diff --git a/.github/ISSUE_TEMPLATE/11-language-change.yml b/.github/ISSUE_TEMPLATE/11-language-change.yml index 37ba2d7e408fbc..ac26ef6cb445bc 100644 --- a/.github/ISSUE_TEMPLATE/11-language-change.yml +++ b/.github/ISSUE_TEMPLATE/11-language-change.yml @@ -1,7 +1,7 @@ name: Language Change Proposals description: Changes to the language -labels: ["Proposal", "v2", "LanguageChange"] -title: "proposal: Go 2: proposal title" +labels: ["Proposal", "LanguageChange", "LanguageChangeReview"] +title: "proposal: spec: proposal title" body: diff --git a/LICENSE b/LICENSE index 6a66aea5eafe0c..2a7cf70da6e498 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. +Copyright 2009 The Go Authors. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -10,7 +10,7 @@ notice, this list of conditions and the following disclaimer. copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Google Inc. nor the names of its + * Neither the name of Google LLC nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. diff --git a/api/go1.24.txt b/api/go1.24.txt new file mode 100644 index 00000000000000..05e2006e0747bb --- /dev/null +++ b/api/go1.24.txt @@ -0,0 +1,223 @@ +pkg bytes, func FieldsFuncSeq([]uint8, func(int32) bool) iter.Seq[[]uint8] #61901 +pkg bytes, func FieldsSeq([]uint8) iter.Seq[[]uint8] #61901 +pkg bytes, func Lines([]uint8) iter.Seq[[]uint8] #61901 +pkg bytes, func SplitAfterSeq([]uint8, []uint8) iter.Seq[[]uint8] #61901 +pkg bytes, func SplitSeq([]uint8, []uint8) iter.Seq[[]uint8] #61901 +pkg crypto/cipher, func NewCFBDecrypter //deprecated #69445 +pkg crypto/cipher, func NewCFBEncrypter //deprecated #69445 +pkg crypto/cipher, func NewGCMWithRandomNonce(Block) (AEAD, error) #69981 +pkg crypto/cipher, func NewOFB //deprecated #69445 +pkg crypto/fips140, func Enabled() bool #70123 +pkg crypto/hkdf, func Expand[$0 hash.Hash](func() $0, []uint8, string, int) ([]uint8, error) #61477 +pkg crypto/hkdf, func Extract[$0 hash.Hash](func() $0, []uint8, []uint8) ([]uint8, error) #61477 +pkg crypto/hkdf, func Key[$0 hash.Hash](func() $0, []uint8, []uint8, string, int) ([]uint8, error) #61477 +pkg crypto/mlkem, const CiphertextSize1024 = 1568 #70122 +pkg crypto/mlkem, const CiphertextSize1024 ideal-int #70122 +pkg crypto/mlkem, const CiphertextSize768 = 1088 #70122 +pkg crypto/mlkem, const CiphertextSize768 ideal-int #70122 +pkg crypto/mlkem, const EncapsulationKeySize1024 = 1568 #70122 +pkg crypto/mlkem, const EncapsulationKeySize1024 ideal-int #70122 +pkg crypto/mlkem, const EncapsulationKeySize768 = 1184 #70122 +pkg crypto/mlkem, const EncapsulationKeySize768 ideal-int #70122 +pkg crypto/mlkem, const SeedSize = 64 #70122 +pkg crypto/mlkem, const SeedSize ideal-int #70122 +pkg crypto/mlkem, const SharedKeySize = 32 #70122 +pkg crypto/mlkem, const SharedKeySize ideal-int #70122 +pkg crypto/mlkem, func GenerateKey1024() (*DecapsulationKey1024, error) #70122 +pkg crypto/mlkem, func GenerateKey768() (*DecapsulationKey768, error) #70122 +pkg crypto/mlkem, func NewDecapsulationKey1024([]uint8) (*DecapsulationKey1024, error) #70122 +pkg crypto/mlkem, func NewDecapsulationKey768([]uint8) (*DecapsulationKey768, error) #70122 +pkg crypto/mlkem, func NewEncapsulationKey1024([]uint8) (*EncapsulationKey1024, error) #70122 +pkg crypto/mlkem, func NewEncapsulationKey768([]uint8) (*EncapsulationKey768, error) #70122 +pkg crypto/mlkem, method (*DecapsulationKey1024) Bytes() []uint8 #70122 +pkg crypto/mlkem, method (*DecapsulationKey1024) Decapsulate([]uint8) ([]uint8, error) #70122 +pkg crypto/mlkem, method (*DecapsulationKey1024) EncapsulationKey() *EncapsulationKey1024 #70122 +pkg crypto/mlkem, method (*DecapsulationKey768) Bytes() []uint8 #70122 +pkg crypto/mlkem, method (*DecapsulationKey768) Decapsulate([]uint8) ([]uint8, error) #70122 +pkg crypto/mlkem, method (*DecapsulationKey768) EncapsulationKey() *EncapsulationKey768 #70122 +pkg crypto/mlkem, method (*EncapsulationKey1024) Bytes() []uint8 #70122 +pkg crypto/mlkem, method (*EncapsulationKey1024) Encapsulate() ([]uint8, []uint8) #70122 +pkg crypto/mlkem, method (*EncapsulationKey768) Bytes() []uint8 #70122 +pkg crypto/mlkem, method (*EncapsulationKey768) Encapsulate() ([]uint8, []uint8) #70122 +pkg crypto/mlkem, type DecapsulationKey1024 struct #70122 +pkg crypto/mlkem, type DecapsulationKey768 struct #70122 +pkg crypto/mlkem, type EncapsulationKey1024 struct #70122 +pkg crypto/mlkem, type EncapsulationKey768 struct #70122 +pkg crypto/pbkdf2, func Key[$0 hash.Hash](func() $0, string, []uint8, int, int) ([]uint8, error) #69488 +pkg crypto/rand, func Text() string #67057 +pkg crypto/sha3, func New224() *SHA3 #69982 +pkg crypto/sha3, func New256() *SHA3 #69982 +pkg crypto/sha3, func New384() *SHA3 #69982 +pkg crypto/sha3, func New512() *SHA3 #69982 +pkg crypto/sha3, func NewCSHAKE128([]uint8, []uint8) *SHAKE #69982 +pkg crypto/sha3, func NewCSHAKE256([]uint8, []uint8) *SHAKE #69982 +pkg crypto/sha3, func NewSHAKE128() *SHAKE #69982 +pkg crypto/sha3, func NewSHAKE256() *SHAKE #69982 +pkg crypto/sha3, func Sum224([]uint8) [28]uint8 #69982 +pkg crypto/sha3, func Sum256([]uint8) [32]uint8 #69982 +pkg crypto/sha3, func Sum384([]uint8) [48]uint8 #69982 +pkg crypto/sha3, func Sum512([]uint8) [64]uint8 #69982 +pkg crypto/sha3, func SumSHAKE128([]uint8, int) []uint8 #69982 +pkg crypto/sha3, func SumSHAKE256([]uint8, int) []uint8 #69982 +pkg crypto/sha3, method (*SHA3) AppendBinary([]uint8) ([]uint8, error) #69982 +pkg crypto/sha3, method (*SHA3) BlockSize() int #69982 +pkg crypto/sha3, method (*SHA3) MarshalBinary() ([]uint8, error) #69982 +pkg crypto/sha3, method (*SHA3) Reset() #69982 +pkg crypto/sha3, method (*SHA3) Size() int #69982 +pkg crypto/sha3, method (*SHA3) Sum([]uint8) []uint8 #69982 +pkg crypto/sha3, method (*SHA3) UnmarshalBinary([]uint8) error #69982 +pkg crypto/sha3, method (*SHA3) Write([]uint8) (int, error) #69982 +pkg crypto/sha3, method (*SHAKE) AppendBinary([]uint8) ([]uint8, error) #69982 +pkg crypto/sha3, method (*SHAKE) BlockSize() int #69982 +pkg crypto/sha3, method (*SHAKE) MarshalBinary() ([]uint8, error) #69982 +pkg crypto/sha3, method (*SHAKE) Read([]uint8) (int, error) #69982 +pkg crypto/sha3, method (*SHAKE) Reset() #69982 +pkg crypto/sha3, method (*SHAKE) UnmarshalBinary([]uint8) error #69982 +pkg crypto/sha3, method (*SHAKE) Write([]uint8) (int, error) #69982 +pkg crypto/sha3, type SHA3 struct #69982 +pkg crypto/sha3, type SHAKE struct #69982 +pkg crypto/subtle, func WithDataIndependentTiming(func()) #66450 +pkg crypto/tls, const X25519MLKEM768 = 4588 #69985 +pkg crypto/tls, const X25519MLKEM768 CurveID #69985 +pkg crypto/tls, type ClientHelloInfo struct, Extensions []uint16 #32936 +pkg crypto/tls, type Config struct, EncryptedClientHelloKeys []EncryptedClientHelloKey #68500 +pkg crypto/tls, type EncryptedClientHelloKey struct #68500 +pkg crypto/tls, type EncryptedClientHelloKey struct, Config []uint8 #68500 +pkg crypto/tls, type EncryptedClientHelloKey struct, PrivateKey []uint8 #68500 +pkg crypto/tls, type EncryptedClientHelloKey struct, SendAsRetry bool #68500 +pkg crypto/x509, const NoValidChains = 10 #68484 +pkg crypto/x509, const NoValidChains InvalidReason #68484 +pkg crypto/x509, method (OID) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg crypto/x509, method (OID) AppendText([]uint8) ([]uint8, error) #62384 +pkg crypto/x509, type Certificate struct, InhibitAnyPolicy int #68484 +pkg crypto/x509, type Certificate struct, InhibitAnyPolicyZero bool #68484 +pkg crypto/x509, type Certificate struct, InhibitPolicyMapping int #68484 +pkg crypto/x509, type Certificate struct, InhibitPolicyMappingZero bool #68484 +pkg crypto/x509, type Certificate struct, PolicyMappings []PolicyMapping #68484 +pkg crypto/x509, type Certificate struct, RequireExplicitPolicy int #68484 +pkg crypto/x509, type Certificate struct, RequireExplicitPolicyZero bool #68484 +pkg crypto/x509, type PolicyMapping struct #68484 +pkg crypto/x509, type PolicyMapping struct, IssuerDomainPolicy OID #68484 +pkg crypto/x509, type PolicyMapping struct, SubjectDomainPolicy OID #68484 +pkg crypto/x509, type VerifyOptions struct, CertificatePolicies []OID #68484 +pkg debug/elf, const VER_FLG_BASE = 1 #63952 +pkg debug/elf, const VER_FLG_BASE DynamicVersionFlag #63952 +pkg debug/elf, const VER_FLG_INFO = 4 #63952 +pkg debug/elf, const VER_FLG_INFO DynamicVersionFlag #63952 +pkg debug/elf, const VER_FLG_WEAK = 2 #63952 +pkg debug/elf, const VER_FLG_WEAK DynamicVersionFlag #63952 +pkg debug/elf, method (*File) DynamicVersionNeeds() ([]DynamicVersionNeed, error) #63952 +pkg debug/elf, method (*File) DynamicVersions() ([]DynamicVersion, error) #63952 +pkg debug/elf, type DynamicVersion struct #63952 +pkg debug/elf, type DynamicVersion struct, Deps []string #63952 +pkg debug/elf, type DynamicVersion struct, Flags DynamicVersionFlag #63952 +pkg debug/elf, type DynamicVersion struct, Name string #63952 +pkg debug/elf, type DynamicVersion struct, Index uint16 #63952 +pkg debug/elf, type DynamicVersionDep struct #63952 +pkg debug/elf, type DynamicVersionDep struct, Dep string #63952 +pkg debug/elf, type DynamicVersionDep struct, Flags DynamicVersionFlag #63952 +pkg debug/elf, type DynamicVersionDep struct, Index uint16 #63952 +pkg debug/elf, type DynamicVersionFlag uint16 #63952 +pkg debug/elf, type DynamicVersionNeed struct #63952 +pkg debug/elf, type DynamicVersionNeed struct, Name string #63952 +pkg debug/elf, type DynamicVersionNeed struct, Needs []DynamicVersionDep #63952 +pkg debug/elf, type Symbol struct, HasVersion bool #63952 +pkg debug/elf, type Symbol struct, VersionIndex VersionIndex #63952 +pkg debug/elf, method (VersionIndex) Index() uint16 #63952 +pkg debug/elf, method (VersionIndex) IsHidden() bool #63952 +pkg debug/elf, type VersionIndex uint16 #63952 +pkg encoding, type BinaryAppender interface { AppendBinary } #62384 +pkg encoding, type BinaryAppender interface, AppendBinary([]uint8) ([]uint8, error) #62384 +pkg encoding, type TextAppender interface { AppendText } #62384 +pkg encoding, type TextAppender interface, AppendText([]uint8) ([]uint8, error) #62384 +pkg go/types, method (*Interface) EmbeddedTypes() iter.Seq[Type] #66626 +pkg go/types, method (*Interface) ExplicitMethods() iter.Seq[*Func] #66626 +pkg go/types, method (*Interface) Methods() iter.Seq[*Func] #66626 +pkg go/types, method (*MethodSet) Methods() iter.Seq[*Selection] #66626 +pkg go/types, method (*Named) Methods() iter.Seq[*Func] #66626 +pkg go/types, method (*Scope) Children() iter.Seq[*Scope] #66626 +pkg go/types, method (*Struct) Fields() iter.Seq[*Var] #66626 +pkg go/types, method (*Tuple) Variables() iter.Seq[*Var] #66626 +pkg go/types, method (*TypeList) Types() iter.Seq[Type] #66626 +pkg go/types, method (*TypeParamList) TypeParams() iter.Seq[*TypeParam] #66626 +pkg go/types, method (*Union) Terms() iter.Seq[*Term] #66626 +pkg hash/maphash, func Comparable[$0 comparable](Seed, $0) uint64 #54670 +pkg hash/maphash, func WriteComparable[$0 comparable](*Hash, $0) #54670 +pkg log/slog, method (*LevelVar) AppendText([]uint8) ([]uint8, error) #62384 +pkg log/slog, method (Level) AppendText([]uint8) ([]uint8, error) #62384 +pkg log/slog, var DiscardHandler Handler #62005 +pkg math/big, method (*Float) AppendText([]uint8) ([]uint8, error) #62384 +pkg math/big, method (*Int) AppendText([]uint8) ([]uint8, error) #62384 +pkg math/big, method (*Rat) AppendText([]uint8) ([]uint8, error) #62384 +pkg math/rand/v2, method (*ChaCha8) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg math/rand/v2, method (*PCG) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg net, method (IP) AppendText([]uint8) ([]uint8, error) #62384 +pkg net/http, method (*Protocols) SetHTTP1(bool) #67814 +pkg net/http, method (*Protocols) SetHTTP2(bool) #67814 +pkg net/http, method (*Protocols) SetUnencryptedHTTP2(bool) #67816 +pkg net/http, method (Protocols) HTTP1() bool #67814 +pkg net/http, method (Protocols) HTTP2() bool #67814 +pkg net/http, method (Protocols) String() string #67814 +pkg net/http, method (Protocols) UnencryptedHTTP2() bool #67816 +pkg net/http, type HTTP2Config struct #67813 +pkg net/http, type HTTP2Config struct, CountError func(string) #67813 +pkg net/http, type HTTP2Config struct, MaxConcurrentStreams int #67813 +pkg net/http, type HTTP2Config struct, MaxDecoderHeaderTableSize int #67813 +pkg net/http, type HTTP2Config struct, MaxEncoderHeaderTableSize int #67813 +pkg net/http, type HTTP2Config struct, MaxReadFrameSize int #67813 +pkg net/http, type HTTP2Config struct, MaxReceiveBufferPerConnection int #67813 +pkg net/http, type HTTP2Config struct, MaxReceiveBufferPerStream int #67813 +pkg net/http, type HTTP2Config struct, PermitProhibitedCipherSuites bool #67813 +pkg net/http, type HTTP2Config struct, PingTimeout time.Duration #67813 +pkg net/http, type HTTP2Config struct, SendPingTimeout time.Duration #67813 +pkg net/http, type HTTP2Config struct, WriteByteTimeout time.Duration #67813 +pkg net/http, type Protocols struct #67814 +pkg net/http, type Server struct, HTTP2 *HTTP2Config #67813 +pkg net/http, type Server struct, Protocols *Protocols #67814 +pkg net/http, type Transport struct, HTTP2 *HTTP2Config #67813 +pkg net/http, type Transport struct, Protocols *Protocols #67814 +pkg net/netip, method (Addr) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg net/netip, method (Addr) AppendText([]uint8) ([]uint8, error) #62384 +pkg net/netip, method (AddrPort) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg net/netip, method (AddrPort) AppendText([]uint8) ([]uint8, error) #62384 +pkg net/netip, method (Prefix) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg net/netip, method (Prefix) AppendText([]uint8) ([]uint8, error) #62384 +pkg net/url, method (*URL) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg os, func OpenInRoot(string, string) (*File, error) #67002 +pkg os, func OpenRoot(string) (*Root, error) #67002 +pkg os, method (*Root) Close() error #67002 +pkg os, method (*Root) Create(string) (*File, error) #67002 +pkg os, method (*Root) FS() fs.FS #67002 +pkg os, method (*Root) Lstat(string) (fs.FileInfo, error) #67002 +pkg os, method (*Root) Mkdir(string, fs.FileMode) error #67002 +pkg os, method (*Root) Name() string #67002 +pkg os, method (*Root) Open(string) (*File, error) #67002 +pkg os, method (*Root) OpenFile(string, int, fs.FileMode) (*File, error) #67002 +pkg os, method (*Root) OpenRoot(string) (*Root, error) #67002 +pkg os, method (*Root) Remove(string) error #67002 +pkg os, method (*Root) Stat(string) (fs.FileInfo, error) #67002 +pkg os, type Root struct #67002 +pkg regexp, method (*Regexp) AppendText([]uint8) ([]uint8, error) #62384 +pkg runtime, func AddCleanup[$0 interface{}, $1 interface{}](*$0, func($1), $1) Cleanup #67535 +pkg runtime, func GOROOT //deprecated #51473 +pkg runtime, method (Cleanup) Stop() #67535 +pkg runtime, type Cleanup struct #67535 +pkg strings, func FieldsFuncSeq(string, func(int32) bool) iter.Seq[string] #61901 +pkg strings, func FieldsSeq(string) iter.Seq[string] #61901 +pkg strings, func Lines(string) iter.Seq[string] #61901 +pkg strings, func SplitAfterSeq(string, string) iter.Seq[string] #61901 +pkg strings, func SplitSeq(string, string) iter.Seq[string] #61901 +pkg testing, method (*B) Chdir(string) #62516 +pkg testing, method (*B) Context() context.Context #36532 +pkg testing, method (*B) Loop() bool #61515 +pkg testing, method (*F) Chdir(string) #62516 +pkg testing, method (*F) Context() context.Context #36532 +pkg testing, method (*T) Chdir(string) #62516 +pkg testing, method (*T) Context() context.Context #36532 +pkg testing, type TB interface, Chdir(string) #62516 +pkg testing, type TB interface, Context() context.Context #36532 +pkg time, method (Time) AppendBinary([]uint8) ([]uint8, error) #62384 +pkg time, method (Time) AppendText([]uint8) ([]uint8, error) #62384 +pkg weak, func Make[$0 interface{}](*$0) Pointer[$0] #67552 +pkg weak, method (Pointer[$0]) Value() *$0 #67552 +pkg weak, type Pointer[$0 interface{}] struct #67552 diff --git a/api/go1.25.txt b/api/go1.25.txt new file mode 100644 index 00000000000000..d50d19545f163a --- /dev/null +++ b/api/go1.25.txt @@ -0,0 +1,111 @@ +pkg crypto, func SignMessage(Signer, io.Reader, []uint8, SignerOpts) ([]uint8, error) #63405 +pkg crypto, type MessageSigner interface { Public, Sign, SignMessage } #63405 +pkg crypto, type MessageSigner interface, Public() PublicKey #63405 +pkg crypto, type MessageSigner interface, Sign(io.Reader, []uint8, SignerOpts) ([]uint8, error) #63405 +pkg crypto, type MessageSigner interface, SignMessage(io.Reader, []uint8, SignerOpts) ([]uint8, error) #63405 +pkg crypto/ecdsa, func ParseRawPrivateKey(elliptic.Curve, []uint8) (*PrivateKey, error) #63963 +pkg crypto/ecdsa, func ParseUncompressedPublicKey(elliptic.Curve, []uint8) (*PublicKey, error) #63963 +pkg crypto/ecdsa, method (*PrivateKey) Bytes() ([]uint8, error) #63963 +pkg crypto/ecdsa, method (*PublicKey) Bytes() ([]uint8, error) #63963 +pkg crypto/sha3, method (*SHA3) Clone() (hash.Cloner, error) #69521 +pkg crypto/tls, type Config struct, GetEncryptedClientHelloKeys func(*ClientHelloInfo) ([]EncryptedClientHelloKey, error) #71920 +pkg crypto/tls, type ConnectionState struct, CurveID CurveID #67516 +pkg debug/elf, const PT_RISCV_ATTRIBUTES = 1879048195 #72843 +pkg debug/elf, const PT_RISCV_ATTRIBUTES ProgType #72843 +pkg debug/elf, const SHT_RISCV_ATTRIBUTES = 1879048195 #72843 +pkg debug/elf, const SHT_RISCV_ATTRIBUTES SectionType #72843 +pkg go/ast, const FilterFuncDuplicates //deprecated #73088 +pkg go/ast, const FilterImportDuplicates //deprecated #73088 +pkg go/ast, const FilterUnassociatedComments //deprecated #73088 +pkg go/ast, func FilterPackage //deprecated #73088 +pkg go/ast, func MergePackageFiles //deprecated #73088 +pkg go/ast, func PackageExports //deprecated #73088 +pkg go/ast, func PreorderStack(Node, []Node, func(Node, []Node) bool) #73319 +pkg go/ast, type MergeMode //deprecated #73088 +pkg go/parser, func ParseDir //deprecated #71122 +pkg go/token, method (*FileSet) AddExistingFiles(...*File) #73205 +pkg go/types, const FieldVar = 6 #70250 +pkg go/types, const FieldVar VarKind #70250 +pkg go/types, const LocalVar = 2 #70250 +pkg go/types, const LocalVar VarKind #70250 +pkg go/types, const PackageVar = 1 #70250 +pkg go/types, const PackageVar VarKind #70250 +pkg go/types, const ParamVar = 4 #70250 +pkg go/types, const ParamVar VarKind #70250 +pkg go/types, const RecvVar = 3 #70250 +pkg go/types, const RecvVar VarKind #70250 +pkg go/types, const ResultVar = 5 #70250 +pkg go/types, const ResultVar VarKind #70250 +pkg go/types, func LookupSelection(Type, bool, *Package, string) (Selection, bool) #70737 +pkg go/types, method (*Var) Kind() VarKind #70250 +pkg go/types, method (*Var) SetKind(VarKind) #70250 +pkg go/types, method (VarKind) String() string #70250 +pkg go/types, type VarKind uint8 #70250 +pkg hash, type Cloner interface { BlockSize, Clone, Reset, Size, Sum, Write } #69521 +pkg hash, type Cloner interface, BlockSize() int #69521 +pkg hash, type Cloner interface, Clone() (Cloner, error) #69521 +pkg hash, type Cloner interface, Reset() #69521 +pkg hash, type Cloner interface, Size() int #69521 +pkg hash, type Cloner interface, Sum([]uint8) []uint8 #69521 +pkg hash, type Cloner interface, Write([]uint8) (int, error) #69521 +pkg hash, type XOF interface { BlockSize, Read, Reset, Write } #69518 +pkg hash, type XOF interface, BlockSize() int #69518 +pkg hash, type XOF interface, Read([]uint8) (int, error) #69518 +pkg hash, type XOF interface, Reset() #69518 +pkg hash, type XOF interface, Write([]uint8) (int, error) #69518 +pkg hash/maphash, method (*Hash) Clone() (hash.Cloner, error) #69521 +pkg io/fs, func Lstat(FS, string) (FileInfo, error) #49580 +pkg io/fs, func ReadLink(FS, string) (string, error) #49580 +pkg io/fs, type ReadLinkFS interface { Lstat, Open, ReadLink } #49580 +pkg io/fs, type ReadLinkFS interface, Lstat(string) (FileInfo, error) #49580 +pkg io/fs, type ReadLinkFS interface, Open(string) (File, error) #49580 +pkg io/fs, type ReadLinkFS interface, ReadLink(string) (string, error) #49580 +pkg log/slog, func GroupAttrs(string, ...Attr) Attr #66365 +pkg log/slog, method (Record) Source() *Source #70280 +pkg mime/multipart, func FileContentDisposition(string, string) string #46771 +pkg net/http, func NewCrossOriginProtection() *CrossOriginProtection #73626 +pkg net/http, method (*CrossOriginProtection) AddInsecureBypassPattern(string) #73626 +pkg net/http, method (*CrossOriginProtection) AddTrustedOrigin(string) error #73626 +pkg net/http, method (*CrossOriginProtection) Check(*Request) error #73626 +pkg net/http, method (*CrossOriginProtection) Handler(Handler) Handler #73626 +pkg net/http, method (*CrossOriginProtection) SetDenyHandler(Handler) #73626 +pkg net/http, type CrossOriginProtection struct #73626 +pkg os, method (*Root) Chmod(string, fs.FileMode) error #67002 +pkg os, method (*Root) Chown(string, int, int) error #67002 +pkg os, method (*Root) Chtimes(string, time.Time, time.Time) error #67002 +pkg os, method (*Root) Lchown(string, int, int) error #67002 +pkg os, method (*Root) Link(string, string) error #67002 +pkg os, method (*Root) MkdirAll(string, fs.FileMode) error #67002 +pkg os, method (*Root) ReadFile(string) ([]uint8, error) #73126 +pkg os, method (*Root) Readlink(string) (string, error) #67002 +pkg os, method (*Root) RemoveAll(string) error #67002 +pkg os, method (*Root) Rename(string, string) error #67002 +pkg os, method (*Root) Symlink(string, string) error #67002 +pkg os, method (*Root) WriteFile(string, []uint8, fs.FileMode) error #73126 +pkg reflect, func TypeAssert[$0 interface{}](Value) ($0, bool) #62121 +pkg runtime, func SetDefaultGOMAXPROCS() #73193 +pkg runtime/trace, func NewFlightRecorder(FlightRecorderConfig) *FlightRecorder #63185 +pkg runtime/trace, method (*FlightRecorder) Enabled() bool #63185 +pkg runtime/trace, method (*FlightRecorder) Start() error #63185 +pkg runtime/trace, method (*FlightRecorder) Stop() #63185 +pkg runtime/trace, method (*FlightRecorder) WriteTo(io.Writer) (int64, error) #63185 +pkg runtime/trace, type FlightRecorder struct #63185 +pkg runtime/trace, type FlightRecorderConfig struct #63185 +pkg runtime/trace, type FlightRecorderConfig struct, MaxBytes uint64 #63185 +pkg runtime/trace, type FlightRecorderConfig struct, MinAge time.Duration #63185 +pkg sync, method (*WaitGroup) Go(func()) #63796 +pkg testing, method (*B) Attr(string, string) #43936 +pkg testing, method (*B) Output() io.Writer #59928 +pkg testing, method (*F) Attr(string, string) #43936 +pkg testing, method (*F) Output() io.Writer #59928 +pkg testing, method (*T) Attr(string, string) #43936 +pkg testing, method (*T) Output() io.Writer #59928 +pkg testing, type TB interface, Attr(string, string) #43936 +pkg testing, type TB interface, Output() io.Writer #59928 +pkg testing/fstest, method (MapFS) Lstat(string) (fs.FileInfo, error) #49580 +pkg testing/fstest, method (MapFS) ReadLink(string) (string, error) #49580 +pkg testing/synctest, func Test(*testing.T, func(*testing.T)) #67434 +pkg testing/synctest, func Wait() #67434 +pkg unicode, var CategoryAliases map[string]string #70780 +pkg unicode, var Cn *RangeTable #70780 +pkg unicode, var LC *RangeTable #70780 diff --git a/api/next/49097.txt b/api/next/49097.txt new file mode 100644 index 00000000000000..f7240954c66228 --- /dev/null +++ b/api/next/49097.txt @@ -0,0 +1,4 @@ +pkg net, method (*Dialer) DialIP(context.Context, string, netip.Addr, netip.Addr) (*IPConn, error) #49097 +pkg net, method (*Dialer) DialTCP(context.Context, string, netip.AddrPort, netip.AddrPort) (*TCPConn, error) #49097 +pkg net, method (*Dialer) DialUDP(context.Context, string, netip.AddrPort, netip.AddrPort) (*UDPConn, error) #49097 +pkg net, method (*Dialer) DialUnix(context.Context, string, *UnixAddr, *UnixAddr) (*UnixConn, error) #49097 diff --git a/api/next/51945.txt b/api/next/51945.txt new file mode 100644 index 00000000000000..7db1f093e5a2a5 --- /dev/null +++ b/api/next/51945.txt @@ -0,0 +1 @@ +pkg errors, func AsType[$0 error](error) ($0, bool) #51945 diff --git a/api/next/61642.txt b/api/next/61642.txt new file mode 100644 index 00000000000000..dd67874ab9fba4 --- /dev/null +++ b/api/next/61642.txt @@ -0,0 +1 @@ +pkg net/netip, method (Prefix) Compare(Prefix) int #61642 diff --git a/api/next/63963.txt b/api/next/63963.txt new file mode 100644 index 00000000000000..c1aaae5044c0a4 --- /dev/null +++ b/api/next/63963.txt @@ -0,0 +1,3 @@ +pkg crypto/ecdsa, type PrivateKey struct, D //deprecated #63963 +pkg crypto/ecdsa, type PublicKey struct, X //deprecated #63963 +pkg crypto/ecdsa, type PublicKey struct, Y //deprecated #63963 diff --git a/api/next/65954.txt b/api/next/65954.txt new file mode 100644 index 00000000000000..88d7c2668fe02d --- /dev/null +++ b/api/next/65954.txt @@ -0,0 +1,6 @@ +pkg log/slog, func NewMultiHandler(...Handler) *MultiHandler #65954 +pkg log/slog, method (*MultiHandler) Enabled(context.Context, Level) bool #65954 +pkg log/slog, method (*MultiHandler) Handle(context.Context, Record) error #65954 +pkg log/slog, method (*MultiHandler) WithAttrs([]Attr) Handler #65954 +pkg log/slog, method (*MultiHandler) WithGroup(string) Handler #65954 +pkg log/slog, type MultiHandler struct #65954 diff --git a/api/next/67546.txt b/api/next/67546.txt new file mode 100644 index 00000000000000..0b5b4b981c19a6 --- /dev/null +++ b/api/next/67546.txt @@ -0,0 +1,5 @@ +pkg database/sql/driver, type RowsColumnScanner interface { Close, Columns, Next, ScanColumn } #67546 +pkg database/sql/driver, type RowsColumnScanner interface, Close() error #67546 +pkg database/sql/driver, type RowsColumnScanner interface, Columns() []string #67546 +pkg database/sql/driver, type RowsColumnScanner interface, Next([]Value) error #67546 +pkg database/sql/driver, type RowsColumnScanner interface, ScanColumn(interface{}, int) error #67546 diff --git a/api/next/67813.txt b/api/next/67813.txt new file mode 100644 index 00000000000000..b420a141e1fbc7 --- /dev/null +++ b/api/next/67813.txt @@ -0,0 +1 @@ +pkg net/http, type HTTP2Config struct, StrictMaxConcurrentRequests bool #67813 diff --git a/api/next/68021.txt b/api/next/68021.txt new file mode 100644 index 00000000000000..46156e06654878 --- /dev/null +++ b/api/next/68021.txt @@ -0,0 +1,13 @@ +pkg go/ast, func ParseDirective(token.Pos, string) (Directive, bool) #68021 +pkg go/ast, method (*Directive) End() token.Pos #68021 +pkg go/ast, method (*Directive) ParseArgs() ([]DirectiveArg, error) #68021 +pkg go/ast, method (*Directive) Pos() token.Pos #68021 +pkg go/ast, type Directive struct #68021 +pkg go/ast, type Directive struct, Args string #68021 +pkg go/ast, type Directive struct, ArgsPos token.Pos #68021 +pkg go/ast, type Directive struct, Name string #68021 +pkg go/ast, type Directive struct, Slash token.Pos #68021 +pkg go/ast, type Directive struct, Tool string #68021 +pkg go/ast, type DirectiveArg struct #68021 +pkg go/ast, type DirectiveArg struct, Arg string #68021 +pkg go/ast, type DirectiveArg struct, Pos token.Pos #68021 diff --git a/api/next/70352.txt b/api/next/70352.txt new file mode 100644 index 00000000000000..cd264046c3641d --- /dev/null +++ b/api/next/70352.txt @@ -0,0 +1,2 @@ +pkg os, method (*Process) WithHandle(func(uintptr)) error #70352 +pkg os, var ErrNoHandle error #70352 diff --git a/api/next/71287.txt b/api/next/71287.txt new file mode 100644 index 00000000000000..c1e09a1f523082 --- /dev/null +++ b/api/next/71287.txt @@ -0,0 +1,4 @@ +pkg testing, method (*B) ArtifactDir() string #71287 +pkg testing, method (*F) ArtifactDir() string #71287 +pkg testing, method (*T) ArtifactDir() string #71287 +pkg testing, type TB interface, ArtifactDir() string #71287 diff --git a/api/next/73161.txt b/api/next/73161.txt new file mode 100644 index 00000000000000..86526b597a9463 --- /dev/null +++ b/api/next/73161.txt @@ -0,0 +1 @@ +pkg net/http/httputil, type ReverseProxy struct, Director //deprecated #73161 diff --git a/doc/go1.17_spec.html b/doc/go1.17_spec.html deleted file mode 100644 index 9f408bcc253c46..00000000000000 --- a/doc/go1.17_spec.html +++ /dev/null @@ -1,6864 +0,0 @@ - - -

Introduction

- -

-This is the reference manual for the Go programming language as it was for -language version 1.17, in October 2021, before the introduction of generics. -It is provided for historical interest. -The current reference manual can be found here. -For more information and other documents, see go.dev. -

- -

-Go is a general-purpose language designed with systems programming -in mind. It is strongly typed and garbage-collected and has explicit -support for concurrent programming. Programs are constructed from -packages, whose properties allow efficient management of -dependencies. -

- -

-The grammar is compact and simple to parse, allowing for easy analysis -by automatic tools such as integrated development environments. -

- -

Notation

-

-The syntax is specified using Extended Backus-Naur Form (EBNF): -

- -
-Production  = production_name "=" [ Expression ] "." .
-Expression  = Alternative { "|" Alternative } .
-Alternative = Term { Term } .
-Term        = production_name | token [ "…" token ] | Group | Option | Repetition .
-Group       = "(" Expression ")" .
-Option      = "[" Expression "]" .
-Repetition  = "{" Expression "}" .
-
- -

-Productions are expressions constructed from terms and the following -operators, in increasing precedence: -

-
-|   alternation
-()  grouping
-[]  option (0 or 1 times)
-{}  repetition (0 to n times)
-
- -

-Lower-case production names are used to identify lexical tokens. -Non-terminals are in CamelCase. Lexical tokens are enclosed in -double quotes "" or back quotes ``. -

- -

-The form a … b represents the set of characters from -a through b as alternatives. The horizontal -ellipsis is also used elsewhere in the spec to informally denote various -enumerations or code snippets that are not further specified. The character -(as opposed to the three characters ...) is not a token of the Go -language. -

- -

Source code representation

- -

-Source code is Unicode text encoded in -UTF-8. The text is not -canonicalized, so a single accented code point is distinct from the -same character constructed from combining an accent and a letter; -those are treated as two code points. For simplicity, this document -will use the unqualified term character to refer to a Unicode code point -in the source text. -

-

-Each code point is distinct; for instance, upper and lower case letters -are different characters. -

-

-Implementation restriction: For compatibility with other tools, a -compiler may disallow the NUL character (U+0000) in the source text. -

-

-Implementation restriction: For compatibility with other tools, a -compiler may ignore a UTF-8-encoded byte order mark -(U+FEFF) if it is the first Unicode code point in the source text. -A byte order mark may be disallowed anywhere else in the source. -

- -

Characters

- -

-The following terms are used to denote specific Unicode character classes: -

-
-newline        = /* the Unicode code point U+000A */ .
-unicode_char   = /* an arbitrary Unicode code point except newline */ .
-unicode_letter = /* a Unicode code point classified as "Letter" */ .
-unicode_digit  = /* a Unicode code point classified as "Number, decimal digit" */ .
-
- -

-In The Unicode Standard 8.0, -Section 4.5 "General Category" defines a set of character categories. -Go treats all characters in any of the Letter categories Lu, Ll, Lt, Lm, or Lo -as Unicode letters, and those in the Number category Nd as Unicode digits. -

- -

Letters and digits

- -

-The underscore character _ (U+005F) is considered a letter. -

-
-letter        = unicode_letter | "_" .
-decimal_digit = "0" … "9" .
-binary_digit  = "0" | "1" .
-octal_digit   = "0" … "7" .
-hex_digit     = "0" … "9" | "A" … "F" | "a" … "f" .
-
- -

Lexical elements

- -

Comments

- -

-Comments serve as program documentation. There are two forms: -

- -
    -
  1. -Line comments start with the character sequence // -and stop at the end of the line. -
  2. -
  3. -General comments start with the character sequence /* -and stop with the first subsequent character sequence */. -
  4. -
- -

-A comment cannot start inside a rune or -string literal, or inside a comment. -A general comment containing no newlines acts like a space. -Any other comment acts like a newline. -

- -

Tokens

- -

-Tokens form the vocabulary of the Go language. -There are four classes: identifiers, keywords, operators -and punctuation, and literals. White space, formed from -spaces (U+0020), horizontal tabs (U+0009), -carriage returns (U+000D), and newlines (U+000A), -is ignored except as it separates tokens -that would otherwise combine into a single token. Also, a newline or end of file -may trigger the insertion of a semicolon. -While breaking the input into tokens, -the next token is the longest sequence of characters that form a -valid token. -

- -

Semicolons

- -

-The formal grammar uses semicolons ";" as terminators in -a number of productions. Go programs may omit most of these semicolons -using the following two rules: -

- -
    -
  1. -When the input is broken into tokens, a semicolon is automatically inserted -into the token stream immediately after a line's final token if that token is - -
  2. - -
  3. -To allow complex statements to occupy a single line, a semicolon -may be omitted before a closing ")" or "}". -
  4. -
- -

-To reflect idiomatic use, code examples in this document elide semicolons -using these rules. -

- - -

Identifiers

- -

-Identifiers name program entities such as variables and types. -An identifier is a sequence of one or more letters and digits. -The first character in an identifier must be a letter. -

-
-identifier = letter { letter | unicode_digit } .
-
-
-a
-_x9
-ThisVariableIsExported
-αβ
-
- -

-Some identifiers are predeclared. -

- - -

Keywords

- -

-The following keywords are reserved and may not be used as identifiers. -

-
-break        default      func         interface    select
-case         defer        go           map          struct
-chan         else         goto         package      switch
-const        fallthrough  if           range        type
-continue     for          import       return       var
-
- -

Operators and punctuation

- -

-The following character sequences represent operators -(including assignment operators) and punctuation: -

-
-+    &     +=    &=     &&    ==    !=    (    )
--    |     -=    |=     ||    <     <=    [    ]
-*    ^     *=    ^=     <-    >     >=    {    }
-/    <<    /=    <<=    ++    =     :=    ,    ;
-%    >>    %=    >>=    --    !     ...   .    :
-     &^          &^=
-
- -

Integer literals

- -

-An integer literal is a sequence of digits representing an -integer constant. -An optional prefix sets a non-decimal base: 0b or 0B -for binary, 0, 0o, or 0O for octal, -and 0x or 0X for hexadecimal. -A single 0 is considered a decimal zero. -In hexadecimal literals, letters a through f -and A through F represent values 10 through 15. -

- -

-For readability, an underscore character _ may appear after -a base prefix or between successive digits; such underscores do not change -the literal's value. -

-
-int_lit        = decimal_lit | binary_lit | octal_lit | hex_lit .
-decimal_lit    = "0" | ( "1" … "9" ) [ [ "_" ] decimal_digits ] .
-binary_lit     = "0" ( "b" | "B" ) [ "_" ] binary_digits .
-octal_lit      = "0" [ "o" | "O" ] [ "_" ] octal_digits .
-hex_lit        = "0" ( "x" | "X" ) [ "_" ] hex_digits .
-
-decimal_digits = decimal_digit { [ "_" ] decimal_digit } .
-binary_digits  = binary_digit { [ "_" ] binary_digit } .
-octal_digits   = octal_digit { [ "_" ] octal_digit } .
-hex_digits     = hex_digit { [ "_" ] hex_digit } .
-
- -
-42
-4_2
-0600
-0_600
-0o600
-0O600       // second character is capital letter 'O'
-0xBadFace
-0xBad_Face
-0x_67_7a_2f_cc_40_c6
-170141183460469231731687303715884105727
-170_141183_460469_231731_687303_715884_105727
-
-_42         // an identifier, not an integer literal
-42_         // invalid: _ must separate successive digits
-4__2        // invalid: only one _ at a time
-0_xBadFace  // invalid: _ must separate successive digits
-
- - -

Floating-point literals

- -

-A floating-point literal is a decimal or hexadecimal representation of a -floating-point constant. -

- -

-A decimal floating-point literal consists of an integer part (decimal digits), -a decimal point, a fractional part (decimal digits), and an exponent part -(e or E followed by an optional sign and decimal digits). -One of the integer part or the fractional part may be elided; one of the decimal point -or the exponent part may be elided. -An exponent value exp scales the mantissa (integer and fractional part) by 10exp. -

- -

-A hexadecimal floating-point literal consists of a 0x or 0X -prefix, an integer part (hexadecimal digits), a radix point, a fractional part (hexadecimal digits), -and an exponent part (p or P followed by an optional sign and decimal digits). -One of the integer part or the fractional part may be elided; the radix point may be elided as well, -but the exponent part is required. (This syntax matches the one given in IEEE 754-2008 §5.12.3.) -An exponent value exp scales the mantissa (integer and fractional part) by 2exp. -

- -

-For readability, an underscore character _ may appear after -a base prefix or between successive digits; such underscores do not change -the literal value. -

- -
-float_lit         = decimal_float_lit | hex_float_lit .
-
-decimal_float_lit = decimal_digits "." [ decimal_digits ] [ decimal_exponent ] |
-                    decimal_digits decimal_exponent |
-                    "." decimal_digits [ decimal_exponent ] .
-decimal_exponent  = ( "e" | "E" ) [ "+" | "-" ] decimal_digits .
-
-hex_float_lit     = "0" ( "x" | "X" ) hex_mantissa hex_exponent .
-hex_mantissa      = [ "_" ] hex_digits "." [ hex_digits ] |
-                    [ "_" ] hex_digits |
-                    "." hex_digits .
-hex_exponent      = ( "p" | "P" ) [ "+" | "-" ] decimal_digits .
-
- -
-0.
-72.40
-072.40       // == 72.40
-2.71828
-1.e+0
-6.67428e-11
-1E6
-.25
-.12345E+5
-1_5.         // == 15.0
-0.15e+0_2    // == 15.0
-
-0x1p-2       // == 0.25
-0x2.p10      // == 2048.0
-0x1.Fp+0     // == 1.9375
-0X.8p-0      // == 0.5
-0X_1FFFP-16  // == 0.1249847412109375
-0x15e-2      // == 0x15e - 2 (integer subtraction)
-
-0x.p1        // invalid: mantissa has no digits
-1p-2         // invalid: p exponent requires hexadecimal mantissa
-0x1.5e-2     // invalid: hexadecimal mantissa requires p exponent
-1_.5         // invalid: _ must separate successive digits
-1._5         // invalid: _ must separate successive digits
-1.5_e1       // invalid: _ must separate successive digits
-1.5e_1       // invalid: _ must separate successive digits
-1.5e1_       // invalid: _ must separate successive digits
-
- - -

Imaginary literals

- -

-An imaginary literal represents the imaginary part of a -complex constant. -It consists of an integer or -floating-point literal -followed by the lower-case letter i. -The value of an imaginary literal is the value of the respective -integer or floating-point literal multiplied by the imaginary unit i. -

- -
-imaginary_lit = (decimal_digits | int_lit | float_lit) "i" .
-
- -

-For backward compatibility, an imaginary literal's integer part consisting -entirely of decimal digits (and possibly underscores) is considered a decimal -integer, even if it starts with a leading 0. -

- -
-0i
-0123i         // == 123i for backward-compatibility
-0o123i        // == 0o123 * 1i == 83i
-0xabci        // == 0xabc * 1i == 2748i
-0.i
-2.71828i
-1.e+0i
-6.67428e-11i
-1E6i
-.25i
-.12345E+5i
-0x1p-2i       // == 0x1p-2 * 1i == 0.25i
-
- - -

Rune literals

- -

-A rune literal represents a rune constant, -an integer value identifying a Unicode code point. -A rune literal is expressed as one or more characters enclosed in single quotes, -as in 'x' or '\n'. -Within the quotes, any character may appear except newline and unescaped single -quote. A single quoted character represents the Unicode value -of the character itself, -while multi-character sequences beginning with a backslash encode -values in various formats. -

- -

-The simplest form represents the single character within the quotes; -since Go source text is Unicode characters encoded in UTF-8, multiple -UTF-8-encoded bytes may represent a single integer value. For -instance, the literal 'a' holds a single byte representing -a literal a, Unicode U+0061, value 0x61, while -'ä' holds two bytes (0xc3 0xa4) representing -a literal a-dieresis, U+00E4, value 0xe4. -

- -

-Several backslash escapes allow arbitrary values to be encoded as -ASCII text. There are four ways to represent the integer value -as a numeric constant: \x followed by exactly two hexadecimal -digits; \u followed by exactly four hexadecimal digits; -\U followed by exactly eight hexadecimal digits, and a -plain backslash \ followed by exactly three octal digits. -In each case the value of the literal is the value represented by -the digits in the corresponding base. -

- -

-Although these representations all result in an integer, they have -different valid ranges. Octal escapes must represent a value between -0 and 255 inclusive. Hexadecimal escapes satisfy this condition -by construction. The escapes \u and \U -represent Unicode code points so within them some values are illegal, -in particular those above 0x10FFFF and surrogate halves. -

- -

-After a backslash, certain single-character escapes represent special values: -

- -
-\a   U+0007 alert or bell
-\b   U+0008 backspace
-\f   U+000C form feed
-\n   U+000A line feed or newline
-\r   U+000D carriage return
-\t   U+0009 horizontal tab
-\v   U+000B vertical tab
-\\   U+005C backslash
-\'   U+0027 single quote  (valid escape only within rune literals)
-\"   U+0022 double quote  (valid escape only within string literals)
-
- -

-All other sequences starting with a backslash are illegal inside rune literals. -

-
-rune_lit         = "'" ( unicode_value | byte_value ) "'" .
-unicode_value    = unicode_char | little_u_value | big_u_value | escaped_char .
-byte_value       = octal_byte_value | hex_byte_value .
-octal_byte_value = `\` octal_digit octal_digit octal_digit .
-hex_byte_value   = `\` "x" hex_digit hex_digit .
-little_u_value   = `\` "u" hex_digit hex_digit hex_digit hex_digit .
-big_u_value      = `\` "U" hex_digit hex_digit hex_digit hex_digit
-                           hex_digit hex_digit hex_digit hex_digit .
-escaped_char     = `\` ( "a" | "b" | "f" | "n" | "r" | "t" | "v" | `\` | "'" | `"` ) .
-
- -
-'a'
-'ä'
-'本'
-'\t'
-'\000'
-'\007'
-'\377'
-'\x07'
-'\xff'
-'\u12e4'
-'\U00101234'
-'\''         // rune literal containing single quote character
-'aa'         // illegal: too many characters
-'\xa'        // illegal: too few hexadecimal digits
-'\0'         // illegal: too few octal digits
-'\uDFFF'     // illegal: surrogate half
-'\U00110000' // illegal: invalid Unicode code point
-
- - -

String literals

- -

-A string literal represents a string constant -obtained from concatenating a sequence of characters. There are two forms: -raw string literals and interpreted string literals. -

- -

-Raw string literals are character sequences between back quotes, as in -`foo`. Within the quotes, any character may appear except -back quote. The value of a raw string literal is the -string composed of the uninterpreted (implicitly UTF-8-encoded) characters -between the quotes; -in particular, backslashes have no special meaning and the string may -contain newlines. -Carriage return characters ('\r') inside raw string literals -are discarded from the raw string value. -

- -

-Interpreted string literals are character sequences between double -quotes, as in "bar". -Within the quotes, any character may appear except newline and unescaped double quote. -The text between the quotes forms the -value of the literal, with backslash escapes interpreted as they -are in rune literals (except that \' is illegal and -\" is legal), with the same restrictions. -The three-digit octal (\nnn) -and two-digit hexadecimal (\xnn) escapes represent individual -bytes of the resulting string; all other escapes represent -the (possibly multi-byte) UTF-8 encoding of individual characters. -Thus inside a string literal \377 and \xFF represent -a single byte of value 0xFF=255, while ÿ, -\u00FF, \U000000FF and \xc3\xbf represent -the two bytes 0xc3 0xbf of the UTF-8 encoding of character -U+00FF. -

- -
-string_lit             = raw_string_lit | interpreted_string_lit .
-raw_string_lit         = "`" { unicode_char | newline } "`" .
-interpreted_string_lit = `"` { unicode_value | byte_value } `"` .
-
- -
-`abc`                // same as "abc"
-`\n
-\n`                  // same as "\\n\n\\n"
-"\n"
-"\""                 // same as `"`
-"Hello, world!\n"
-"日本語"
-"\u65e5本\U00008a9e"
-"\xff\u00FF"
-"\uD800"             // illegal: surrogate half
-"\U00110000"         // illegal: invalid Unicode code point
-
- -

-These examples all represent the same string: -

- -
-"日本語"                                 // UTF-8 input text
-`日本語`                                 // UTF-8 input text as a raw literal
-"\u65e5\u672c\u8a9e"                    // the explicit Unicode code points
-"\U000065e5\U0000672c\U00008a9e"        // the explicit Unicode code points
-"\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e"  // the explicit UTF-8 bytes
-
- -

-If the source code represents a character as two code points, such as -a combining form involving an accent and a letter, the result will be -an error if placed in a rune literal (it is not a single code -point), and will appear as two code points if placed in a string -literal. -

- - -

Constants

- -

There are boolean constants, -rune constants, -integer constants, -floating-point constants, complex constants, -and string constants. Rune, integer, floating-point, -and complex constants are -collectively called numeric constants. -

- -

-A constant value is represented by a -rune, -integer, -floating-point, -imaginary, -or -string literal, -an identifier denoting a constant, -a constant expression, -a conversion with a result that is a constant, or -the result value of some built-in functions such as -unsafe.Sizeof applied to any value, -cap or len applied to -some expressions, -real and imag applied to a complex constant -and complex applied to numeric constants. -The boolean truth values are represented by the predeclared constants -true and false. The predeclared identifier -iota denotes an integer constant. -

- -

-In general, complex constants are a form of -constant expression -and are discussed in that section. -

- -

-Numeric constants represent exact values of arbitrary precision and do not overflow. -Consequently, there are no constants denoting the IEEE 754 negative zero, infinity, -and not-a-number values. -

- -

-Constants may be typed or untyped. -Literal constants, true, false, iota, -and certain constant expressions -containing only untyped constant operands are untyped. -

- -

-A constant may be given a type explicitly by a constant declaration -or conversion, or implicitly when used in a -variable declaration or an -assignment or as an -operand in an expression. -It is an error if the constant value -cannot be represented as a value of the respective type. -

- -

-An untyped constant has a default type which is the type to which the -constant is implicitly converted in contexts where a typed value is required, -for instance, in a short variable declaration -such as i := 0 where there is no explicit type. -The default type of an untyped constant is bool, rune, -int, float64, complex128 or string -respectively, depending on whether it is a boolean, rune, integer, floating-point, -complex, or string constant. -

- -

-Implementation restriction: Although numeric constants have arbitrary -precision in the language, a compiler may implement them using an -internal representation with limited precision. That said, every -implementation must: -

- - - -

-These requirements apply both to literal constants and to the result -of evaluating constant -expressions. -

- - -

Variables

- -

-A variable is a storage location for holding a value. -The set of permissible values is determined by the -variable's type. -

- -

-A variable declaration -or, for function parameters and results, the signature -of a function declaration -or function literal reserves -storage for a named variable. - -Calling the built-in function new -or taking the address of a composite literal -allocates storage for a variable at run time. -Such an anonymous variable is referred to via a (possibly implicit) -pointer indirection. -

- -

-Structured variables of array, slice, -and struct types have elements and fields that may -be addressed individually. Each such element -acts like a variable. -

- -

-The static type (or just type) of a variable is the -type given in its declaration, the type provided in the -new call or composite literal, or the type of -an element of a structured variable. -Variables of interface type also have a distinct dynamic type, -which is the concrete type of the value assigned to the variable at run time -(unless the value is the predeclared identifier nil, -which has no type). -The dynamic type may vary during execution but values stored in interface -variables are always assignable -to the static type of the variable. -

- -
-var x interface{}  // x is nil and has static type interface{}
-var v *T           // v has value nil, static type *T
-x = 42             // x has value 42 and dynamic type int
-x = v              // x has value (*T)(nil) and dynamic type *T
-
- -

-A variable's value is retrieved by referring to the variable in an -expression; it is the most recent value -assigned to the variable. -If a variable has not yet been assigned a value, its value is the -zero value for its type. -

- - -

Types

- -

-A type determines a set of values together with operations and methods specific -to those values. A type may be denoted by a type name, if it has one, -or specified using a type literal, which composes a type from existing types. -

- -
-Type      = TypeName | TypeLit | "(" Type ")" .
-TypeName  = identifier | QualifiedIdent .
-TypeLit   = ArrayType | StructType | PointerType | FunctionType | InterfaceType |
-	    SliceType | MapType | ChannelType .
-
- -

-The language predeclares certain type names. -Others are introduced with type declarations. -Composite types—array, struct, pointer, function, -interface, slice, map, and channel types—may be constructed using -type literals. -

- -

-Each type T has an underlying type: If T -is one of the predeclared boolean, numeric, or string types, or a type literal, -the corresponding underlying -type is T itself. Otherwise, T's underlying type -is the underlying type of the type to which T refers in its -type declaration. -

- -
-type (
-	A1 = string
-	A2 = A1
-)
-
-type (
-	B1 string
-	B2 B1
-	B3 []B1
-	B4 B3
-)
-
- -

-The underlying type of string, A1, A2, B1, -and B2 is string. -The underlying type of []B1, B3, and B4 is []B1. -

- -

Method sets

-

-A type has a (possibly empty) method set associated with it. -The method set of an interface type is its interface. -The method set of any other type T consists of all -methods declared with receiver type T. -The method set of the corresponding pointer type *T -is the set of all methods declared with receiver *T or T -(that is, it also contains the method set of T). -Further rules apply to structs containing embedded fields, as described -in the section on struct types. -Any other type has an empty method set. -In a method set, each method must have a -unique -non-blank method name. -

- -

-The method set of a type determines the interfaces that the -type implements -and the methods that can be called -using a receiver of that type. -

- -

Boolean types

- -

-A boolean type represents the set of Boolean truth values -denoted by the predeclared constants true -and false. The predeclared boolean type is bool; -it is a defined type. -

- -

Numeric types

- -

-A numeric type represents sets of integer or floating-point values. -The predeclared architecture-independent numeric types are: -

- -
-uint8       the set of all unsigned  8-bit integers (0 to 255)
-uint16      the set of all unsigned 16-bit integers (0 to 65535)
-uint32      the set of all unsigned 32-bit integers (0 to 4294967295)
-uint64      the set of all unsigned 64-bit integers (0 to 18446744073709551615)
-
-int8        the set of all signed  8-bit integers (-128 to 127)
-int16       the set of all signed 16-bit integers (-32768 to 32767)
-int32       the set of all signed 32-bit integers (-2147483648 to 2147483647)
-int64       the set of all signed 64-bit integers (-9223372036854775808 to 9223372036854775807)
-
-float32     the set of all IEEE 754 32-bit floating-point numbers
-float64     the set of all IEEE 754 64-bit floating-point numbers
-
-complex64   the set of all complex numbers with float32 real and imaginary parts
-complex128  the set of all complex numbers with float64 real and imaginary parts
-
-byte        alias for uint8
-rune        alias for int32
-
- -

-The value of an n-bit integer is n bits wide and represented using -two's complement arithmetic. -

- -

-There is also a set of predeclared numeric types with implementation-specific sizes: -

- -
-uint     either 32 or 64 bits
-int      same size as uint
-uintptr  an unsigned integer large enough to store the uninterpreted bits of a pointer value
-
- -

-To avoid portability issues all numeric types are defined -types and thus distinct except -byte, which is an alias for uint8, and -rune, which is an alias for int32. -Explicit conversions -are required when different numeric types are mixed in an expression -or assignment. For instance, int32 and int -are not the same type even though they may have the same size on a -particular architecture. -

- -

String types

- -

-A string type represents the set of string values. -A string value is a (possibly empty) sequence of bytes. -The number of bytes is called the length of the string and is never negative. -Strings are immutable: once created, -it is impossible to change the contents of a string. -The predeclared string type is string; -it is a defined type. -

- -

-The length of a string s can be discovered using -the built-in function len. -The length is a compile-time constant if the string is a constant. -A string's bytes can be accessed by integer indices -0 through len(s)-1. -It is illegal to take the address of such an element; if -s[i] is the i'th byte of a -string, &s[i] is invalid. -

- - -

Array types

- -

-An array is a numbered sequence of elements of a single -type, called the element type. -The number of elements is called the length of the array and is never negative. -

- -
-ArrayType   = "[" ArrayLength "]" ElementType .
-ArrayLength = Expression .
-ElementType = Type .
-
- -

-The length is part of the array's type; it must evaluate to a -non-negative constant -representable by a value -of type int. -The length of array a can be discovered -using the built-in function len. -The elements can be addressed by integer indices -0 through len(a)-1. -Array types are always one-dimensional but may be composed to form -multi-dimensional types. -

- -
-[32]byte
-[2*N] struct { x, y int32 }
-[1000]*float64
-[3][5]int
-[2][2][2]float64  // same as [2]([2]([2]float64))
-
- -

Slice types

- -

-A slice is a descriptor for a contiguous segment of an underlying array and -provides access to a numbered sequence of elements from that array. -A slice type denotes the set of all slices of arrays of its element type. -The number of elements is called the length of the slice and is never negative. -The value of an uninitialized slice is nil. -

- -
-SliceType = "[" "]" ElementType .
-
- -

-The length of a slice s can be discovered by the built-in function -len; unlike with arrays it may change during -execution. The elements can be addressed by integer indices -0 through len(s)-1. The slice index of a -given element may be less than the index of the same element in the -underlying array. -

-

-A slice, once initialized, is always associated with an underlying -array that holds its elements. A slice therefore shares storage -with its array and with other slices of the same array; by contrast, -distinct arrays always represent distinct storage. -

-

-The array underlying a slice may extend past the end of the slice. -The capacity is a measure of that extent: it is the sum of -the length of the slice and the length of the array beyond the slice; -a slice of length up to that capacity can be created by -slicing a new one from the original slice. -The capacity of a slice a can be discovered using the -built-in function cap(a). -

- -

-A new, initialized slice value for a given element type T is -made using the built-in function -make, -which takes a slice type -and parameters specifying the length and optionally the capacity. -A slice created with make always allocates a new, hidden array -to which the returned slice value refers. That is, executing -

- -
-make([]T, length, capacity)
-
- -

-produces the same slice as allocating an array and slicing -it, so these two expressions are equivalent: -

- -
-make([]int, 50, 100)
-new([100]int)[0:50]
-
- -

-Like arrays, slices are always one-dimensional but may be composed to construct -higher-dimensional objects. -With arrays of arrays, the inner arrays are, by construction, always the same length; -however with slices of slices (or arrays of slices), the inner lengths may vary dynamically. -Moreover, the inner slices must be initialized individually. -

- -

Struct types

- -

-A struct is a sequence of named elements, called fields, each of which has a -name and a type. Field names may be specified explicitly (IdentifierList) or -implicitly (EmbeddedField). -Within a struct, non-blank field names must -be unique. -

- -
-StructType    = "struct" "{" { FieldDecl ";" } "}" .
-FieldDecl     = (IdentifierList Type | EmbeddedField) [ Tag ] .
-EmbeddedField = [ "*" ] TypeName .
-Tag           = string_lit .
-
- -
-// An empty struct.
-struct {}
-
-// A struct with 6 fields.
-struct {
-	x, y int
-	u float32
-	_ float32  // padding
-	A *[]int
-	F func()
-}
-
- -

-A field declared with a type but no explicit field name is called an embedded field. -An embedded field must be specified as -a type name T or as a pointer to a non-interface type name *T, -and T itself may not be -a pointer type. The unqualified type name acts as the field name. -

- -
-// A struct with four embedded fields of types T1, *T2, P.T3 and *P.T4
-struct {
-	T1        // field name is T1
-	*T2       // field name is T2
-	P.T3      // field name is T3
-	*P.T4     // field name is T4
-	x, y int  // field names are x and y
-}
-
- -

-The following declaration is illegal because field names must be unique -in a struct type: -

- -
-struct {
-	T     // conflicts with embedded field *T and *P.T
-	*T    // conflicts with embedded field T and *P.T
-	*P.T  // conflicts with embedded field T and *T
-}
-
- -

-A field or method f of an -embedded field in a struct x is called promoted if -x.f is a legal selector that denotes -that field or method f. -

- -

-Promoted fields act like ordinary fields -of a struct except that they cannot be used as field names in -composite literals of the struct. -

- -

-Given a struct type S and a defined type -T, promoted methods are included in the method set of the struct as follows: -

- - -

-A field declaration may be followed by an optional string literal tag, -which becomes an attribute for all the fields in the corresponding -field declaration. An empty tag string is equivalent to an absent tag. -The tags are made visible through a reflection interface -and take part in type identity for structs -but are otherwise ignored. -

- -
-struct {
-	x, y float64 ""  // an empty tag string is like an absent tag
-	name string  "any string is permitted as a tag"
-	_    [4]byte "ceci n'est pas un champ de structure"
-}
-
-// A struct corresponding to a TimeStamp protocol buffer.
-// The tag strings define the protocol buffer field numbers;
-// they follow the convention outlined by the reflect package.
-struct {
-	microsec  uint64 `protobuf:"1"`
-	serverIP6 uint64 `protobuf:"2"`
-}
-
- -

Pointer types

- -

-A pointer type denotes the set of all pointers to variables of a given -type, called the base type of the pointer. -The value of an uninitialized pointer is nil. -

- -
-PointerType = "*" BaseType .
-BaseType    = Type .
-
- -
-*Point
-*[4]int
-
- -

Function types

- -

-A function type denotes the set of all functions with the same parameter -and result types. The value of an uninitialized variable of function type -is nil. -

- -
-FunctionType   = "func" Signature .
-Signature      = Parameters [ Result ] .
-Result         = Parameters | Type .
-Parameters     = "(" [ ParameterList [ "," ] ] ")" .
-ParameterList  = ParameterDecl { "," ParameterDecl } .
-ParameterDecl  = [ IdentifierList ] [ "..." ] Type .
-
- -

-Within a list of parameters or results, the names (IdentifierList) -must either all be present or all be absent. If present, each name -stands for one item (parameter or result) of the specified type and -all non-blank names in the signature -must be unique. -If absent, each type stands for one item of that type. -Parameter and result -lists are always parenthesized except that if there is exactly -one unnamed result it may be written as an unparenthesized type. -

- -

-The final incoming parameter in a function signature may have -a type prefixed with .... -A function with such a parameter is called variadic and -may be invoked with zero or more arguments for that parameter. -

- -
-func()
-func(x int) int
-func(a, _ int, z float32) bool
-func(a, b int, z float32) (bool)
-func(prefix string, values ...int)
-func(a, b int, z float64, opt ...interface{}) (success bool)
-func(int, int, float64) (float64, *[]int)
-func(n int) func(p *T)
-
- - -

Interface types

- -

-An interface type specifies a method set called its interface. -A variable of interface type can store a value of any type with a method set -that is any superset of the interface. Such a type is said to -implement the interface. -The value of an uninitialized variable of interface type is nil. -

- -
-InterfaceType      = "interface" "{" { ( MethodSpec | InterfaceTypeName ) ";" } "}" .
-MethodSpec         = MethodName Signature .
-MethodName         = identifier .
-InterfaceTypeName  = TypeName .
-
- -

-An interface type may specify methods explicitly through method specifications, -or it may embed methods of other interfaces through interface type names. -

- -
-// A simple File interface.
-interface {
-	Read([]byte) (int, error)
-	Write([]byte) (int, error)
-	Close() error
-}
-
- -

-The name of each explicitly specified method must be unique -and not blank. -

- -
-interface {
-	String() string
-	String() string  // illegal: String not unique
-	_(x int)         // illegal: method must have non-blank name
-}
-
- -

-More than one type may implement an interface. -For instance, if two types S1 and S2 -have the method set -

- -
-func (p T) Read(p []byte) (n int, err error)
-func (p T) Write(p []byte) (n int, err error)
-func (p T) Close() error
-
- -

-(where T stands for either S1 or S2) -then the File interface is implemented by both S1 and -S2, regardless of what other methods -S1 and S2 may have or share. -

- -

-A type implements any interface comprising any subset of its methods -and may therefore implement several distinct interfaces. For -instance, all types implement the empty interface: -

- -
-interface{}
-
- -

-Similarly, consider this interface specification, -which appears within a type declaration -to define an interface called Locker: -

- -
-type Locker interface {
-	Lock()
-	Unlock()
-}
-
- -

-If S1 and S2 also implement -

- -
-func (p T) Lock() { … }
-func (p T) Unlock() { … }
-
- -

-they implement the Locker interface as well -as the File interface. -

- -

-An interface T may use a (possibly qualified) interface type -name E in place of a method specification. This is called -embedding interface E in T. -The method set of T is the union -of the method sets of T’s explicitly declared methods and of -T’s embedded interfaces. -

- -
-type Reader interface {
-	Read(p []byte) (n int, err error)
-	Close() error
-}
-
-type Writer interface {
-	Write(p []byte) (n int, err error)
-	Close() error
-}
-
-// ReadWriter's methods are Read, Write, and Close.
-type ReadWriter interface {
-	Reader  // includes methods of Reader in ReadWriter's method set
-	Writer  // includes methods of Writer in ReadWriter's method set
-}
-
- -

-A union of method sets contains the (exported and non-exported) -methods of each method set exactly once, and methods with the -same names must -have identical signatures. -

- -
-type ReadCloser interface {
-	Reader   // includes methods of Reader in ReadCloser's method set
-	Close()  // illegal: signatures of Reader.Close and Close are different
-}
-
- -

-An interface type T may not embed itself -or any interface type that embeds T, recursively. -

- -
-// illegal: Bad cannot embed itself
-type Bad interface {
-	Bad
-}
-
-// illegal: Bad1 cannot embed itself using Bad2
-type Bad1 interface {
-	Bad2
-}
-type Bad2 interface {
-	Bad1
-}
-
- -

Map types

- -

-A map is an unordered group of elements of one type, called the -element type, indexed by a set of unique keys of another type, -called the key type. -The value of an uninitialized map is nil. -

- -
-MapType     = "map" "[" KeyType "]" ElementType .
-KeyType     = Type .
-
- -

-The comparison operators -== and != must be fully defined -for operands of the key type; thus the key type must not be a function, map, or -slice. -If the key type is an interface type, these -comparison operators must be defined for the dynamic key values; -failure will cause a run-time panic. - -

- -
-map[string]int
-map[*T]struct{ x, y float64 }
-map[string]interface{}
-
- -

-The number of map elements is called its length. -For a map m, it can be discovered using the -built-in function len -and may change during execution. Elements may be added during execution -using assignments and retrieved with -index expressions; they may be removed with the -delete built-in function. -

-

-A new, empty map value is made using the built-in -function make, -which takes the map type and an optional capacity hint as arguments: -

- -
-make(map[string]int)
-make(map[string]int, 100)
-
- -

-The initial capacity does not bound its size: -maps grow to accommodate the number of items -stored in them, with the exception of nil maps. -A nil map is equivalent to an empty map except that no elements -may be added. -

- -

Channel types

- -

-A channel provides a mechanism for -concurrently executing functions -to communicate by -sending and -receiving -values of a specified element type. -The value of an uninitialized channel is nil. -

- -
-ChannelType = ( "chan" | "chan" "<-" | "<-" "chan" ) ElementType .
-
- -

-The optional <- operator specifies the channel direction, -send or receive. If no direction is given, the channel is -bidirectional. -A channel may be constrained only to send or only to receive by -assignment or -explicit conversion. -

- -
-chan T          // can be used to send and receive values of type T
-chan<- float64  // can only be used to send float64s
-<-chan int      // can only be used to receive ints
-
- -

-The <- operator associates with the leftmost chan -possible: -

- -
-chan<- chan int    // same as chan<- (chan int)
-chan<- <-chan int  // same as chan<- (<-chan int)
-<-chan <-chan int  // same as <-chan (<-chan int)
-chan (<-chan int)
-
- -

-A new, initialized channel -value can be made using the built-in function -make, -which takes the channel type and an optional capacity as arguments: -

- -
-make(chan int, 100)
-
- -

-The capacity, in number of elements, sets the size of the buffer in the channel. -If the capacity is zero or absent, the channel is unbuffered and communication -succeeds only when both a sender and receiver are ready. Otherwise, the channel -is buffered and communication succeeds without blocking if the buffer -is not full (sends) or not empty (receives). -A nil channel is never ready for communication. -

- -

-A channel may be closed with the built-in function -close. -The multi-valued assignment form of the -receive operator -reports whether a received value was sent before -the channel was closed. -

- -

-A single channel may be used in -send statements, -receive operations, -and calls to the built-in functions -cap and -len -by any number of goroutines without further synchronization. -Channels act as first-in-first-out queues. -For example, if one goroutine sends values on a channel -and a second goroutine receives them, the values are -received in the order sent. -

- -

Properties of types and values

- -

Type identity

- -

-Two types are either identical or different. -

- -

-A defined type is always different from any other type. -Otherwise, two types are identical if their underlying type literals are -structurally equivalent; that is, they have the same literal structure and corresponding -components have identical types. In detail: -

- - - -

-Given the declarations -

- -
-type (
-	A0 = []string
-	A1 = A0
-	A2 = struct{ a, b int }
-	A3 = int
-	A4 = func(A3, float64) *A0
-	A5 = func(x int, _ float64) *[]string
-)
-
-type (
-	B0 A0
-	B1 []string
-	B2 struct{ a, b int }
-	B3 struct{ a, c int }
-	B4 func(int, float64) *B0
-	B5 func(x int, y float64) *A1
-)
-
-type	C0 = B0
-
- -

-these types are identical: -

- -
-A0, A1, and []string
-A2 and struct{ a, b int }
-A3 and int
-A4, func(int, float64) *[]string, and A5
-
-B0 and C0
-[]int and []int
-struct{ a, b *T5 } and struct{ a, b *T5 }
-func(x int, y float64) *[]string, func(int, float64) (result *[]string), and A5
-
- -

-B0 and B1 are different because they are new types -created by distinct type definitions; -func(int, float64) *B0 and func(x int, y float64) *[]string -are different because B0 is different from []string. -

- - -

Assignability

- -

-A value x is assignable to a variable of type T -("x is assignable to T") if one of the following conditions applies: -

- - - - -

Representability

- -

-A constant x is representable -by a value of type T if one of the following conditions applies: -

- - - -
-x                   T           x is representable by a value of T because
-
-'a'                 byte        97 is in the set of byte values
-97                  rune        rune is an alias for int32, and 97 is in the set of 32-bit integers
-"foo"               string      "foo" is in the set of string values
-1024                int16       1024 is in the set of 16-bit integers
-42.0                byte        42 is in the set of unsigned 8-bit integers
-1e10                uint64      10000000000 is in the set of unsigned 64-bit integers
-2.718281828459045   float32     2.718281828459045 rounds to 2.7182817 which is in the set of float32 values
--1e-1000            float64     -1e-1000 rounds to IEEE -0.0 which is further simplified to 0.0
-0i                  int         0 is an integer value
-(42 + 0i)           float32     42.0 (with zero imaginary part) is in the set of float32 values
-
- -
-x                   T           x is not representable by a value of T because
-
-0                   bool        0 is not in the set of boolean values
-'a'                 string      'a' is a rune, it is not in the set of string values
-1024                byte        1024 is not in the set of unsigned 8-bit integers
--1                  uint16      -1 is not in the set of unsigned 16-bit integers
-1.1                 int         1.1 is not an integer value
-42i                 float32     (0 + 42i) is not in the set of float32 values
-1e1000              float64     1e1000 overflows to IEEE +Inf after rounding
-
- - -

Blocks

- -

-A block is a possibly empty sequence of declarations and statements -within matching brace brackets. -

- -
-Block = "{" StatementList "}" .
-StatementList = { Statement ";" } .
-
- -

-In addition to explicit blocks in the source code, there are implicit blocks: -

- -
    -
  1. The universe block encompasses all Go source text.
  2. - -
  3. Each package has a package block containing all - Go source text for that package.
  4. - -
  5. Each file has a file block containing all Go source text - in that file.
  6. - -
  7. Each "if", - "for", and - "switch" - statement is considered to be in its own implicit block.
  8. - -
  9. Each clause in a "switch" - or "select" statement - acts as an implicit block.
  10. -
- -

-Blocks nest and influence scoping. -

- - -

Declarations and scope

- -

-A declaration binds a non-blank identifier to a -constant, -type, -variable, -function, -label, or -package. -Every identifier in a program must be declared. -No identifier may be declared twice in the same block, and -no identifier may be declared in both the file and package block. -

- -

-The blank identifier may be used like any other identifier -in a declaration, but it does not introduce a binding and thus is not declared. -In the package block, the identifier init may only be used for -init function declarations, -and like the blank identifier it does not introduce a new binding. -

- -
-Declaration   = ConstDecl | TypeDecl | VarDecl .
-TopLevelDecl  = Declaration | FunctionDecl | MethodDecl .
-
- -

-The scope of a declared identifier is the extent of source text in which -the identifier denotes the specified constant, type, variable, function, label, or package. -

- -

-Go is lexically scoped using blocks: -

- -
    -
  1. The scope of a predeclared identifier is the universe block.
  2. - -
  3. The scope of an identifier denoting a constant, type, variable, - or function (but not method) declared at top level (outside any - function) is the package block.
  4. - -
  5. The scope of the package name of an imported package is the file block - of the file containing the import declaration.
  6. - -
  7. The scope of an identifier denoting a method receiver, function parameter, - or result variable is the function body.
  8. - -
  9. The scope of a constant or variable identifier declared - inside a function begins at the end of the ConstSpec or VarSpec - (ShortVarDecl for short variable declarations) - and ends at the end of the innermost containing block.
  10. - -
  11. The scope of a type identifier declared inside a function - begins at the identifier in the TypeSpec - and ends at the end of the innermost containing block.
  12. -
- -

-An identifier declared in a block may be redeclared in an inner block. -While the identifier of the inner declaration is in scope, it denotes -the entity declared by the inner declaration. -

- -

-The package clause is not a declaration; the package name -does not appear in any scope. Its purpose is to identify the files belonging -to the same package and to specify the default package name for import -declarations. -

- - -

Label scopes

- -

-Labels are declared by labeled statements and are -used in the "break", -"continue", and -"goto" statements. -It is illegal to define a label that is never used. -In contrast to other identifiers, labels are not block scoped and do -not conflict with identifiers that are not labels. The scope of a label -is the body of the function in which it is declared and excludes -the body of any nested function. -

- - -

Blank identifier

- -

-The blank identifier is represented by the underscore character _. -It serves as an anonymous placeholder instead of a regular (non-blank) -identifier and has special meaning in declarations, -as an operand, and in assignments. -

- - -

Predeclared identifiers

- -

-The following identifiers are implicitly declared in the -universe block: -

-
-Types:
-	bool byte complex64 complex128 error float32 float64
-	int int8 int16 int32 int64 rune string
-	uint uint8 uint16 uint32 uint64 uintptr
-
-Constants:
-	true false iota
-
-Zero value:
-	nil
-
-Functions:
-	append cap close complex copy delete imag len
-	make new panic print println real recover
-
- - -

Exported identifiers

- -

-An identifier may be exported to permit access to it from another package. -An identifier is exported if both: -

-
    -
  1. the first character of the identifier's name is a Unicode upper case - letter (Unicode class "Lu"); and
  2. -
  3. the identifier is declared in the package block - or it is a field name or - method name.
  4. -
-

-All other identifiers are not exported. -

- - -

Uniqueness of identifiers

- -

-Given a set of identifiers, an identifier is called unique if it is -different from every other in the set. -Two identifiers are different if they are spelled differently, or if they -appear in different packages and are not -exported. Otherwise, they are the same. -

- -

Constant declarations

- -

-A constant declaration binds a list of identifiers (the names of -the constants) to the values of a list of constant expressions. -The number of identifiers must be equal -to the number of expressions, and the nth identifier on -the left is bound to the value of the nth expression on the -right. -

- -
-ConstDecl      = "const" ( ConstSpec | "(" { ConstSpec ";" } ")" ) .
-ConstSpec      = IdentifierList [ [ Type ] "=" ExpressionList ] .
-
-IdentifierList = identifier { "," identifier } .
-ExpressionList = Expression { "," Expression } .
-
- -

-If the type is present, all constants take the type specified, and -the expressions must be assignable to that type. -If the type is omitted, the constants take the -individual types of the corresponding expressions. -If the expression values are untyped constants, -the declared constants remain untyped and the constant identifiers -denote the constant values. For instance, if the expression is a -floating-point literal, the constant identifier denotes a floating-point -constant, even if the literal's fractional part is zero. -

- -
-const Pi float64 = 3.14159265358979323846
-const zero = 0.0         // untyped floating-point constant
-const (
-	size int64 = 1024
-	eof        = -1  // untyped integer constant
-)
-const a, b, c = 3, 4, "foo"  // a = 3, b = 4, c = "foo", untyped integer and string constants
-const u, v float32 = 0, 3    // u = 0.0, v = 3.0
-
- -

-Within a parenthesized const declaration list the -expression list may be omitted from any but the first ConstSpec. -Such an empty list is equivalent to the textual substitution of the -first preceding non-empty expression list and its type if any. -Omitting the list of expressions is therefore equivalent to -repeating the previous list. The number of identifiers must be equal -to the number of expressions in the previous list. -Together with the iota constant generator -this mechanism permits light-weight declaration of sequential values: -

- -
-const (
-	Sunday = iota
-	Monday
-	Tuesday
-	Wednesday
-	Thursday
-	Friday
-	Partyday
-	numberOfDays  // this constant is not exported
-)
-
- - -

Iota

- -

-Within a constant declaration, the predeclared identifier -iota represents successive untyped integer -constants. Its value is the index of the respective ConstSpec -in that constant declaration, starting at zero. -It can be used to construct a set of related constants: -

- -
-const (
-	c0 = iota  // c0 == 0
-	c1 = iota  // c1 == 1
-	c2 = iota  // c2 == 2
-)
-
-const (
-	a = 1 << iota  // a == 1  (iota == 0)
-	b = 1 << iota  // b == 2  (iota == 1)
-	c = 3          // c == 3  (iota == 2, unused)
-	d = 1 << iota  // d == 8  (iota == 3)
-)
-
-const (
-	u         = iota * 42  // u == 0     (untyped integer constant)
-	v float64 = iota * 42  // v == 42.0  (float64 constant)
-	w         = iota * 42  // w == 84    (untyped integer constant)
-)
-
-const x = iota  // x == 0
-const y = iota  // y == 0
-
- -

-By definition, multiple uses of iota in the same ConstSpec all have the same value: -

- -
-const (
-	bit0, mask0 = 1 << iota, 1<<iota - 1  // bit0 == 1, mask0 == 0  (iota == 0)
-	bit1, mask1                           // bit1 == 2, mask1 == 1  (iota == 1)
-	_, _                                  //                        (iota == 2, unused)
-	bit3, mask3                           // bit3 == 8, mask3 == 7  (iota == 3)
-)
-
- -

-This last example exploits the implicit repetition -of the last non-empty expression list. -

- - -

Type declarations

- -

-A type declaration binds an identifier, the type name, to a type. -Type declarations come in two forms: alias declarations and type definitions. -

- -
-TypeDecl = "type" ( TypeSpec | "(" { TypeSpec ";" } ")" ) .
-TypeSpec = AliasDecl | TypeDef .
-
- -

Alias declarations

- -

-An alias declaration binds an identifier to the given type. -

- -
-AliasDecl = identifier "=" Type .
-
- -

-Within the scope of -the identifier, it serves as an alias for the type. -

- -
-type (
-	nodeList = []*Node  // nodeList and []*Node are identical types
-	Polar    = polar    // Polar and polar denote identical types
-)
-
- - -

Type definitions

- -

-A type definition creates a new, distinct type with the same -underlying type and operations as the given type, -and binds an identifier to it. -

- -
-TypeDef = identifier Type .
-
- -

-The new type is called a defined type. -It is different from any other type, -including the type it is created from. -

- -
-type (
-	Point struct{ x, y float64 }  // Point and struct{ x, y float64 } are different types
-	polar Point                   // polar and Point denote different types
-)
-
-type TreeNode struct {
-	left, right *TreeNode
-	value *Comparable
-}
-
-type Block interface {
-	BlockSize() int
-	Encrypt(src, dst []byte)
-	Decrypt(src, dst []byte)
-}
-
- -

-A defined type may have methods associated with it. -It does not inherit any methods bound to the given type, -but the method set -of an interface type or of elements of a composite type remains unchanged: -

- -
-// A Mutex is a data type with two methods, Lock and Unlock.
-type Mutex struct         { /* Mutex fields */ }
-func (m *Mutex) Lock()    { /* Lock implementation */ }
-func (m *Mutex) Unlock()  { /* Unlock implementation */ }
-
-// NewMutex has the same composition as Mutex but its method set is empty.
-type NewMutex Mutex
-
-// The method set of PtrMutex's underlying type *Mutex remains unchanged,
-// but the method set of PtrMutex is empty.
-type PtrMutex *Mutex
-
-// The method set of *PrintableMutex contains the methods
-// Lock and Unlock bound to its embedded field Mutex.
-type PrintableMutex struct {
-	Mutex
-}
-
-// MyBlock is an interface type that has the same method set as Block.
-type MyBlock Block
-
- -

-Type definitions may be used to define different boolean, numeric, -or string types and associate methods with them: -

- -
-type TimeZone int
-
-const (
-	EST TimeZone = -(5 + iota)
-	CST
-	MST
-	PST
-)
-
-func (tz TimeZone) String() string {
-	return fmt.Sprintf("GMT%+dh", tz)
-}
-
- - -

Variable declarations

- -

-A variable declaration creates one or more variables, -binds corresponding identifiers to them, and gives each a type and an initial value. -

- -
-VarDecl     = "var" ( VarSpec | "(" { VarSpec ";" } ")" ) .
-VarSpec     = IdentifierList ( Type [ "=" ExpressionList ] | "=" ExpressionList ) .
-
- -
-var i int
-var U, V, W float64
-var k = 0
-var x, y float32 = -1, -2
-var (
-	i       int
-	u, v, s = 2.0, 3.0, "bar"
-)
-var re, im = complexSqrt(-1)
-var _, found = entries[name]  // map lookup; only interested in "found"
-
- -

-If a list of expressions is given, the variables are initialized -with the expressions following the rules for assignments. -Otherwise, each variable is initialized to its zero value. -

- -

-If a type is present, each variable is given that type. -Otherwise, each variable is given the type of the corresponding -initialization value in the assignment. -If that value is an untyped constant, it is first implicitly -converted to its default type; -if it is an untyped boolean value, it is first implicitly converted to type bool. -The predeclared value nil cannot be used to initialize a variable -with no explicit type. -

- -
-var d = math.Sin(0.5)  // d is float64
-var i = 42             // i is int
-var t, ok = x.(T)      // t is T, ok is bool
-var n = nil            // illegal
-
- -

-Implementation restriction: A compiler may make it illegal to declare a variable -inside a function body if the variable is -never used. -

- -

Short variable declarations

- -

-A short variable declaration uses the syntax: -

- -
-ShortVarDecl = IdentifierList ":=" ExpressionList .
-
- -

-It is shorthand for a regular variable declaration -with initializer expressions but no types: -

- -
-"var" IdentifierList = ExpressionList .
-
- -
-i, j := 0, 10
-f := func() int { return 7 }
-ch := make(chan int)
-r, w, _ := os.Pipe()  // os.Pipe() returns a connected pair of Files and an error, if any
-_, y, _ := coord(p)   // coord() returns three values; only interested in y coordinate
-
- -

-Unlike regular variable declarations, a short variable declaration may redeclare -variables provided they were originally declared earlier in the same block -(or the parameter lists if the block is the function body) with the same type, -and at least one of the non-blank variables is new. -As a consequence, redeclaration can only appear in a multi-variable short declaration. -Redeclaration does not introduce a new variable; it just assigns a new value to the original. -

- -
-field1, offset := nextField(str, 0)
-field2, offset := nextField(str, offset)  // redeclares offset
-a, a := 1, 2                              // illegal: double declaration of a or no new variable if a was declared elsewhere
-
- -

-Short variable declarations may appear only inside functions. -In some contexts such as the initializers for -"if", -"for", or -"switch" statements, -they can be used to declare local temporary variables. -

- -

Function declarations

- -

-A function declaration binds an identifier, the function name, -to a function. -

- -
-FunctionDecl = "func" FunctionName Signature [ FunctionBody ] .
-FunctionName = identifier .
-FunctionBody = Block .
-
- -

-If the function's signature declares -result parameters, the function body's statement list must end in -a terminating statement. -

- -
-func IndexRune(s string, r rune) int {
-	for i, c := range s {
-		if c == r {
-			return i
-		}
-	}
-	// invalid: missing return statement
-}
-
- -

-A function declaration may omit the body. Such a declaration provides the -signature for a function implemented outside Go, such as an assembly routine. -

- -
-func min(x int, y int) int {
-	if x < y {
-		return x
-	}
-	return y
-}
-
-func flushICache(begin, end uintptr)  // implemented externally
-
- -

Method declarations

- -

-A method is a function with a receiver. -A method declaration binds an identifier, the method name, to a method, -and associates the method with the receiver's base type. -

- -
-MethodDecl = "func" Receiver MethodName Signature [ FunctionBody ] .
-Receiver   = Parameters .
-
- -

-The receiver is specified via an extra parameter section preceding the method -name. That parameter section must declare a single non-variadic parameter, the receiver. -Its type must be a defined type T or a -pointer to a defined type T. T is called the receiver -base type. A receiver base type cannot be a pointer or interface type and -it must be defined in the same package as the method. -The method is said to be bound to its receiver base type and the method name -is visible only within selectors for type T -or *T. -

- -

-A non-blank receiver identifier must be -unique in the method signature. -If the receiver's value is not referenced inside the body of the method, -its identifier may be omitted in the declaration. The same applies in -general to parameters of functions and methods. -

- -

-For a base type, the non-blank names of methods bound to it must be unique. -If the base type is a struct type, -the non-blank method and field names must be distinct. -

- -

-Given defined type Point, the declarations -

- -
-func (p *Point) Length() float64 {
-	return math.Sqrt(p.x * p.x + p.y * p.y)
-}
-
-func (p *Point) Scale(factor float64) {
-	p.x *= factor
-	p.y *= factor
-}
-
- -

-bind the methods Length and Scale, -with receiver type *Point, -to the base type Point. -

- -

-The type of a method is the type of a function with the receiver as first -argument. For instance, the method Scale has type -

- -
-func(p *Point, factor float64)
-
- -

-However, a function declared this way is not a method. -

- - -

Expressions

- -

-An expression specifies the computation of a value by applying -operators and functions to operands. -

- -

Operands

- -

-Operands denote the elementary values in an expression. An operand may be a -literal, a (possibly qualified) -non-blank identifier denoting a -constant, -variable, or -function, -or a parenthesized expression. -

- -

-The blank identifier may appear as an -operand only on the left-hand side of an assignment. -

- -
-Operand     = Literal | OperandName | "(" Expression ")" .
-Literal     = BasicLit | CompositeLit | FunctionLit .
-BasicLit    = int_lit | float_lit | imaginary_lit | rune_lit | string_lit .
-OperandName = identifier | QualifiedIdent .
-
- -

Qualified identifiers

- -

-A qualified identifier is an identifier qualified with a package name prefix. -Both the package name and the identifier must not be -blank. -

- -
-QualifiedIdent = PackageName "." identifier .
-
- -

-A qualified identifier accesses an identifier in a different package, which -must be imported. -The identifier must be exported and -declared in the package block of that package. -

- -
-math.Sin	// denotes the Sin function in package math
-
- -

Composite literals

- -

-Composite literals construct values for structs, arrays, slices, and maps -and create a new value each time they are evaluated. -They consist of the type of the literal followed by a brace-bound list of elements. -Each element may optionally be preceded by a corresponding key. -

- -
-CompositeLit  = LiteralType LiteralValue .
-LiteralType   = StructType | ArrayType | "[" "..." "]" ElementType |
-                SliceType | MapType | TypeName .
-LiteralValue  = "{" [ ElementList [ "," ] ] "}" .
-ElementList   = KeyedElement { "," KeyedElement } .
-KeyedElement  = [ Key ":" ] Element .
-Key           = FieldName | Expression | LiteralValue .
-FieldName     = identifier .
-Element       = Expression | LiteralValue .
-
- -

-The LiteralType's underlying type must be a struct, array, slice, or map type -(the grammar enforces this constraint except when the type is given -as a TypeName). -The types of the elements and keys must be assignable -to the respective field, element, and key types of the literal type; -there is no additional conversion. -The key is interpreted as a field name for struct literals, -an index for array and slice literals, and a key for map literals. -For map literals, all elements must have a key. It is an error -to specify multiple elements with the same field name or -constant key value. For non-constant map keys, see the section on -evaluation order. -

- -

-For struct literals the following rules apply: -

- - -

-Given the declarations -

-
-type Point3D struct { x, y, z float64 }
-type Line struct { p, q Point3D }
-
- -

-one may write -

- -
-origin := Point3D{}                            // zero value for Point3D
-line := Line{origin, Point3D{y: -4, z: 12.3}}  // zero value for line.q.x
-
- -

-For array and slice literals the following rules apply: -

- - -

-Taking the address of a composite literal -generates a pointer to a unique variable initialized -with the literal's value. -

- -
-var pointer *Point3D = &Point3D{y: 1000}
-
- -

-Note that the zero value for a slice or map -type is not the same as an initialized but empty value of the same type. -Consequently, taking the address of an empty slice or map composite literal -does not have the same effect as allocating a new slice or map value with -new. -

- -
-p1 := &[]int{}    // p1 points to an initialized, empty slice with value []int{} and length 0
-p2 := new([]int)  // p2 points to an uninitialized slice with value nil and length 0
-
- -

-The length of an array literal is the length specified in the literal type. -If fewer elements than the length are provided in the literal, the missing -elements are set to the zero value for the array element type. -It is an error to provide elements with index values outside the index range -of the array. The notation ... specifies an array length equal -to the maximum element index plus one. -

- -
-buffer := [10]string{}             // len(buffer) == 10
-intSet := [6]int{1, 2, 3, 5}       // len(intSet) == 6
-days := [...]string{"Sat", "Sun"}  // len(days) == 2
-
- -

-A slice literal describes the entire underlying array literal. -Thus the length and capacity of a slice literal are the maximum -element index plus one. A slice literal has the form -

- -
-[]T{x1, x2, … xn}
-
- -

-and is shorthand for a slice operation applied to an array: -

- -
-tmp := [n]T{x1, x2, … xn}
-tmp[0 : n]
-
- -

-Within a composite literal of array, slice, or map type T, -elements or map keys that are themselves composite literals may elide the respective -literal type if it is identical to the element or key type of T. -Similarly, elements or keys that are addresses of composite literals may elide -the &T when the element or key type is *T. -

- -
-[...]Point{{1.5, -3.5}, {0, 0}}     // same as [...]Point{Point{1.5, -3.5}, Point{0, 0}}
-[][]int{{1, 2, 3}, {4, 5}}          // same as [][]int{[]int{1, 2, 3}, []int{4, 5}}
-[][]Point{{{0, 1}, {1, 2}}}         // same as [][]Point{[]Point{Point{0, 1}, Point{1, 2}}}
-map[string]Point{"orig": {0, 0}}    // same as map[string]Point{"orig": Point{0, 0}}
-map[Point]string{{0, 0}: "orig"}    // same as map[Point]string{Point{0, 0}: "orig"}
-
-type PPoint *Point
-[2]*Point{{1.5, -3.5}, {}}          // same as [2]*Point{&Point{1.5, -3.5}, &Point{}}
-[2]PPoint{{1.5, -3.5}, {}}          // same as [2]PPoint{PPoint(&Point{1.5, -3.5}), PPoint(&Point{})}
-
- -

-A parsing ambiguity arises when a composite literal using the -TypeName form of the LiteralType appears as an operand between the -keyword and the opening brace of the block -of an "if", "for", or "switch" statement, and the composite literal -is not enclosed in parentheses, square brackets, or curly braces. -In this rare case, the opening brace of the literal is erroneously parsed -as the one introducing the block of statements. To resolve the ambiguity, -the composite literal must appear within parentheses. -

- -
-if x == (T{a,b,c}[i]) { … }
-if (x == T{a,b,c}[i]) { … }
-
- -

-Examples of valid array, slice, and map literals: -

- -
-// list of prime numbers
-primes := []int{2, 3, 5, 7, 9, 2147483647}
-
-// vowels[ch] is true if ch is a vowel
-vowels := [128]bool{'a': true, 'e': true, 'i': true, 'o': true, 'u': true, 'y': true}
-
-// the array [10]float32{-1, 0, 0, 0, -0.1, -0.1, 0, 0, 0, -1}
-filter := [10]float32{-1, 4: -0.1, -0.1, 9: -1}
-
-// frequencies in Hz for equal-tempered scale (A4 = 440Hz)
-noteFrequency := map[string]float32{
-	"C0": 16.35, "D0": 18.35, "E0": 20.60, "F0": 21.83,
-	"G0": 24.50, "A0": 27.50, "B0": 30.87,
-}
-
- - -

Function literals

- -

-A function literal represents an anonymous function. -

- -
-FunctionLit = "func" Signature FunctionBody .
-
- -
-func(a, b int, z float64) bool { return a*b < int(z) }
-
- -

-A function literal can be assigned to a variable or invoked directly. -

- -
-f := func(x, y int) int { return x + y }
-func(ch chan int) { ch <- ACK }(replyChan)
-
- -

-Function literals are closures: they may refer to variables -defined in a surrounding function. Those variables are then shared between -the surrounding function and the function literal, and they survive as long -as they are accessible. -

- - -

Primary expressions

- -

-Primary expressions are the operands for unary and binary expressions. -

- -
-PrimaryExpr =
-	Operand |
-	Conversion |
-	MethodExpr |
-	PrimaryExpr Selector |
-	PrimaryExpr Index |
-	PrimaryExpr Slice |
-	PrimaryExpr TypeAssertion |
-	PrimaryExpr Arguments .
-
-Selector       = "." identifier .
-Index          = "[" Expression "]" .
-Slice          = "[" [ Expression ] ":" [ Expression ] "]" |
-                 "[" [ Expression ] ":" Expression ":" Expression "]" .
-TypeAssertion  = "." "(" Type ")" .
-Arguments      = "(" [ ( ExpressionList | Type [ "," ExpressionList ] ) [ "..." ] [ "," ] ] ")" .
-
- - -
-x
-2
-(s + ".txt")
-f(3.1415, true)
-Point{1, 2}
-m["foo"]
-s[i : j + 1]
-obj.color
-f.p[i].x()
-
- - -

Selectors

- -

-For a primary expression x -that is not a package name, the -selector expression -

- -
-x.f
-
- -

-denotes the field or method f of the value x -(or sometimes *x; see below). -The identifier f is called the (field or method) selector; -it must not be the blank identifier. -The type of the selector expression is the type of f. -If x is a package name, see the section on -qualified identifiers. -

- -

-A selector f may denote a field or method f of -a type T, or it may refer -to a field or method f of a nested -embedded field of T. -The number of embedded fields traversed -to reach f is called its depth in T. -The depth of a field or method f -declared in T is zero. -The depth of a field or method f declared in -an embedded field A in T is the -depth of f in A plus one. -

- -

-The following rules apply to selectors: -

- -
    -
  1. -For a value x of type T or *T -where T is not a pointer or interface type, -x.f denotes the field or method at the shallowest depth -in T where there -is such an f. -If there is not exactly one f -with shallowest depth, the selector expression is illegal. -
  2. - -
  3. -For a value x of type I where I -is an interface type, x.f denotes the actual method with name -f of the dynamic value of x. -If there is no method with name f in the -method set of I, the selector -expression is illegal. -
  4. - -
  5. -As an exception, if the type of x is a defined -pointer type and (*x).f is a valid selector expression denoting a field -(but not a method), x.f is shorthand for (*x).f. -
  6. - -
  7. -In all other cases, x.f is illegal. -
  8. - -
  9. -If x is of pointer type and has the value -nil and x.f denotes a struct field, -assigning to or evaluating x.f -causes a run-time panic. -
  10. - -
  11. -If x is of interface type and has the value -nil, calling or -evaluating the method x.f -causes a run-time panic. -
  12. -
- -

-For example, given the declarations: -

- -
-type T0 struct {
-	x int
-}
-
-func (*T0) M0()
-
-type T1 struct {
-	y int
-}
-
-func (T1) M1()
-
-type T2 struct {
-	z int
-	T1
-	*T0
-}
-
-func (*T2) M2()
-
-type Q *T2
-
-var t T2     // with t.T0 != nil
-var p *T2    // with p != nil and (*p).T0 != nil
-var q Q = p
-
- -

-one may write: -

- -
-t.z          // t.z
-t.y          // t.T1.y
-t.x          // (*t.T0).x
-
-p.z          // (*p).z
-p.y          // (*p).T1.y
-p.x          // (*(*p).T0).x
-
-q.x          // (*(*q).T0).x        (*q).x is a valid field selector
-
-p.M0()       // ((*p).T0).M0()      M0 expects *T0 receiver
-p.M1()       // ((*p).T1).M1()      M1 expects T1 receiver
-p.M2()       // p.M2()              M2 expects *T2 receiver
-t.M2()       // (&t).M2()           M2 expects *T2 receiver, see section on Calls
-
- -

-but the following is invalid: -

- -
-q.M0()       // (*q).M0 is valid but not a field selector
-
- - -

Method expressions

- -

-If M is in the method set of type T, -T.M is a function that is callable as a regular function -with the same arguments as M prefixed by an additional -argument that is the receiver of the method. -

- -
-MethodExpr    = ReceiverType "." MethodName .
-ReceiverType  = Type .
-
- -

-Consider a struct type T with two methods, -Mv, whose receiver is of type T, and -Mp, whose receiver is of type *T. -

- -
-type T struct {
-	a int
-}
-func (tv  T) Mv(a int) int         { return 0 }  // value receiver
-func (tp *T) Mp(f float32) float32 { return 1 }  // pointer receiver
-
-var t T
-
- -

-The expression -

- -
-T.Mv
-
- -

-yields a function equivalent to Mv but -with an explicit receiver as its first argument; it has signature -

- -
-func(tv T, a int) int
-
- -

-That function may be called normally with an explicit receiver, so -these five invocations are equivalent: -

- -
-t.Mv(7)
-T.Mv(t, 7)
-(T).Mv(t, 7)
-f1 := T.Mv; f1(t, 7)
-f2 := (T).Mv; f2(t, 7)
-
- -

-Similarly, the expression -

- -
-(*T).Mp
-
- -

-yields a function value representing Mp with signature -

- -
-func(tp *T, f float32) float32
-
- -

-For a method with a value receiver, one can derive a function -with an explicit pointer receiver, so -

- -
-(*T).Mv
-
- -

-yields a function value representing Mv with signature -

- -
-func(tv *T, a int) int
-
- -

-Such a function indirects through the receiver to create a value -to pass as the receiver to the underlying method; -the method does not overwrite the value whose address is passed in -the function call. -

- -

-The final case, a value-receiver function for a pointer-receiver method, -is illegal because pointer-receiver methods are not in the method set -of the value type. -

- -

-Function values derived from methods are called with function call syntax; -the receiver is provided as the first argument to the call. -That is, given f := T.Mv, f is invoked -as f(t, 7) not t.f(7). -To construct a function that binds the receiver, use a -function literal or -method value. -

- -

-It is legal to derive a function value from a method of an interface type. -The resulting function takes an explicit receiver of that interface type. -

- -

Method values

- -

-If the expression x has static type T and -M is in the method set of type T, -x.M is called a method value. -The method value x.M is a function value that is callable -with the same arguments as a method call of x.M. -The expression x is evaluated and saved during the evaluation of the -method value; the saved copy is then used as the receiver in any calls, -which may be executed later. -

- -
-type S struct { *T }
-type T int
-func (t T) M() { print(t) }
-
-t := new(T)
-s := S{T: t}
-f := t.M                    // receiver *t is evaluated and stored in f
-g := s.M                    // receiver *(s.T) is evaluated and stored in g
-*t = 42                     // does not affect stored receivers in f and g
-
- -

-The type T may be an interface or non-interface type. -

- -

-As in the discussion of method expressions above, -consider a struct type T with two methods, -Mv, whose receiver is of type T, and -Mp, whose receiver is of type *T. -

- -
-type T struct {
-	a int
-}
-func (tv  T) Mv(a int) int         { return 0 }  // value receiver
-func (tp *T) Mp(f float32) float32 { return 1 }  // pointer receiver
-
-var t T
-var pt *T
-func makeT() T
-
- -

-The expression -

- -
-t.Mv
-
- -

-yields a function value of type -

- -
-func(int) int
-
- -

-These two invocations are equivalent: -

- -
-t.Mv(7)
-f := t.Mv; f(7)
-
- -

-Similarly, the expression -

- -
-pt.Mp
-
- -

-yields a function value of type -

- -
-func(float32) float32
-
- -

-As with selectors, a reference to a non-interface method with a value receiver -using a pointer will automatically dereference that pointer: pt.Mv is equivalent to (*pt).Mv. -

- -

-As with method calls, a reference to a non-interface method with a pointer receiver -using an addressable value will automatically take the address of that value: t.Mp is equivalent to (&t).Mp. -

- -
-f := t.Mv; f(7)   // like t.Mv(7)
-f := pt.Mp; f(7)  // like pt.Mp(7)
-f := pt.Mv; f(7)  // like (*pt).Mv(7)
-f := t.Mp; f(7)   // like (&t).Mp(7)
-f := makeT().Mp   // invalid: result of makeT() is not addressable
-
- -

-Although the examples above use non-interface types, it is also legal to create a method value -from a value of interface type. -

- -
-var i interface { M(int) } = myVal
-f := i.M; f(7)  // like i.M(7)
-
- - -

Index expressions

- -

-A primary expression of the form -

- -
-a[x]
-
- -

-denotes the element of the array, pointer to array, slice, string or map a indexed by x. -The value x is called the index or map key, respectively. -The following rules apply: -

- -

-If a is not a map: -

- - -

-For a of array type A: -

- - -

-For a of pointer to array type: -

- - -

-For a of slice type S: -

- - -

-For a of string type: -

- - -

-For a of map type M: -

- - -

-Otherwise a[x] is illegal. -

- -

-An index expression on a map a of type map[K]V -used in an assignment or initialization of the special form -

- -
-v, ok = a[x]
-v, ok := a[x]
-var v, ok = a[x]
-
- -

-yields an additional untyped boolean value. The value of ok is -true if the key x is present in the map, and -false otherwise. -

- -

-Assigning to an element of a nil map causes a -run-time panic. -

- - -

Slice expressions

- -

-Slice expressions construct a substring or slice from a string, array, pointer -to array, or slice. There are two variants: a simple form that specifies a low -and high bound, and a full form that also specifies a bound on the capacity. -

- -

Simple slice expressions

- -

-For a string, array, pointer to array, or slice a, the primary expression -

- -
-a[low : high]
-
- -

-constructs a substring or slice. The indices low and -high select which elements of operand a appear -in the result. The result has indices starting at 0 and length equal to -high - low. -After slicing the array a -

- -
-a := [5]int{1, 2, 3, 4, 5}
-s := a[1:4]
-
- -

-the slice s has type []int, length 3, capacity 4, and elements -

- -
-s[0] == 2
-s[1] == 3
-s[2] == 4
-
- -

-For convenience, any of the indices may be omitted. A missing low -index defaults to zero; a missing high index defaults to the length of the -sliced operand: -

- -
-a[2:]  // same as a[2 : len(a)]
-a[:3]  // same as a[0 : 3]
-a[:]   // same as a[0 : len(a)]
-
- -

-If a is a pointer to an array, a[low : high] is shorthand for -(*a)[low : high]. -

- -

-For arrays or strings, the indices are in range if -0 <= low <= high <= len(a), -otherwise they are out of range. -For slices, the upper index bound is the slice capacity cap(a) rather than the length. -A constant index must be non-negative and -representable by a value of type -int; for arrays or constant strings, constant indices must also be in range. -If both indices are constant, they must satisfy low <= high. -If the indices are out of range at run time, a run-time panic occurs. -

- -

-Except for untyped strings, if the sliced operand is a string or slice, -the result of the slice operation is a non-constant value of the same type as the operand. -For untyped string operands the result is a non-constant value of type string. -If the sliced operand is an array, it must be addressable -and the result of the slice operation is a slice with the same element type as the array. -

- -

-If the sliced operand of a valid slice expression is a nil slice, the result -is a nil slice. Otherwise, if the result is a slice, it shares its underlying -array with the operand. -

- -
-var a [10]int
-s1 := a[3:7]   // underlying array of s1 is array a; &s1[2] == &a[5]
-s2 := s1[1:4]  // underlying array of s2 is underlying array of s1 which is array a; &s2[1] == &a[5]
-s2[1] = 42     // s2[1] == s1[2] == a[5] == 42; they all refer to the same underlying array element
-
- - -

Full slice expressions

- -

-For an array, pointer to array, or slice a (but not a string), the primary expression -

- -
-a[low : high : max]
-
- -

-constructs a slice of the same type, and with the same length and elements as the simple slice -expression a[low : high]. Additionally, it controls the resulting slice's capacity -by setting it to max - low. Only the first index may be omitted; it defaults to 0. -After slicing the array a -

- -
-a := [5]int{1, 2, 3, 4, 5}
-t := a[1:3:5]
-
- -

-the slice t has type []int, length 2, capacity 4, and elements -

- -
-t[0] == 2
-t[1] == 3
-
- -

-As for simple slice expressions, if a is a pointer to an array, -a[low : high : max] is shorthand for (*a)[low : high : max]. -If the sliced operand is an array, it must be addressable. -

- -

-The indices are in range if 0 <= low <= high <= max <= cap(a), -otherwise they are out of range. -A constant index must be non-negative and -representable by a value of type -int; for arrays, constant indices must also be in range. -If multiple indices are constant, the constants that are present must be in range relative to each -other. -If the indices are out of range at run time, a run-time panic occurs. -

- -

Type assertions

- -

-For an expression x of interface type -and a type T, the primary expression -

- -
-x.(T)
-
- -

-asserts that x is not nil -and that the value stored in x is of type T. -The notation x.(T) is called a type assertion. -

-

-More precisely, if T is not an interface type, x.(T) asserts -that the dynamic type of x is identical -to the type T. -In this case, T must implement the (interface) type of x; -otherwise the type assertion is invalid since it is not possible for x -to store a value of type T. -If T is an interface type, x.(T) asserts that the dynamic type -of x implements the interface T. -

-

-If the type assertion holds, the value of the expression is the value -stored in x and its type is T. If the type assertion is false, -a run-time panic occurs. -In other words, even though the dynamic type of x -is known only at run time, the type of x.(T) is -known to be T in a correct program. -

- -
-var x interface{} = 7          // x has dynamic type int and value 7
-i := x.(int)                   // i has type int and value 7
-
-type I interface { m() }
-
-func f(y I) {
-	s := y.(string)        // illegal: string does not implement I (missing method m)
-	r := y.(io.Reader)     // r has type io.Reader and the dynamic type of y must implement both I and io.Reader
-	…
-}
-
- -

-A type assertion used in an assignment or initialization of the special form -

- -
-v, ok = x.(T)
-v, ok := x.(T)
-var v, ok = x.(T)
-var v, ok interface{} = x.(T) // dynamic types of v and ok are T and bool
-
- -

-yields an additional untyped boolean value. The value of ok is true -if the assertion holds. Otherwise it is false and the value of v is -the zero value for type T. -No run-time panic occurs in this case. -

- - -

Calls

- -

-Given an expression f of function type -F, -

- -
-f(a1, a2, … an)
-
- -

-calls f with arguments a1, a2, … an. -Except for one special case, arguments must be single-valued expressions -assignable to the parameter types of -F and are evaluated before the function is called. -The type of the expression is the result type -of F. -A method invocation is similar but the method itself -is specified as a selector upon a value of the receiver type for -the method. -

- -
-math.Atan2(x, y)  // function call
-var pt *Point
-pt.Scale(3.5)     // method call with receiver pt
-
- -

-In a function call, the function value and arguments are evaluated in -the usual order. -After they are evaluated, the parameters of the call are passed by value to the function -and the called function begins execution. -The return parameters of the function are passed by value -back to the caller when the function returns. -

- -

-Calling a nil function value -causes a run-time panic. -

- -

-As a special case, if the return values of a function or method -g are equal in number and individually -assignable to the parameters of another function or method -f, then the call f(g(parameters_of_g)) -will invoke f after binding the return values of -g to the parameters of f in order. The call -of f must contain no parameters other than the call of g, -and g must have at least one return value. -If f has a final ... parameter, it is -assigned the return values of g that remain after -assignment of regular parameters. -

- -
-func Split(s string, pos int) (string, string) {
-	return s[0:pos], s[pos:]
-}
-
-func Join(s, t string) string {
-	return s + t
-}
-
-if Join(Split(value, len(value)/2)) != value {
-	log.Panic("test fails")
-}
-
- -

-A method call x.m() is valid if the method set -of (the type of) x contains m and the -argument list can be assigned to the parameter list of m. -If x is addressable and &x's method -set contains m, x.m() is shorthand -for (&x).m(): -

- -
-var p Point
-p.Scale(3.5)
-
- -

-There is no distinct method type and there are no method literals. -

- -

Passing arguments to ... parameters

- -

-If f is variadic with a final -parameter p of type ...T, then within f -the type of p is equivalent to type []T. -If f is invoked with no actual arguments for p, -the value passed to p is nil. -Otherwise, the value passed is a new slice -of type []T with a new underlying array whose successive elements -are the actual arguments, which all must be assignable -to T. The length and capacity of the slice is therefore -the number of arguments bound to p and may differ for each -call site. -

- -

-Given the function and calls -

-
-func Greeting(prefix string, who ...string)
-Greeting("nobody")
-Greeting("hello:", "Joe", "Anna", "Eileen")
-
- -

-within Greeting, who will have the value -nil in the first call, and -[]string{"Joe", "Anna", "Eileen"} in the second. -

- -

-If the final argument is assignable to a slice type []T and -is followed by ..., it is passed unchanged as the value -for a ...T parameter. In this case no new slice is created. -

- -

-Given the slice s and call -

- -
-s := []string{"James", "Jasmine"}
-Greeting("goodbye:", s...)
-
- -

-within Greeting, who will have the same value as s -with the same underlying array. -

- - -

Operators

- -

-Operators combine operands into expressions. -

- -
-Expression = UnaryExpr | Expression binary_op Expression .
-UnaryExpr  = PrimaryExpr | unary_op UnaryExpr .
-
-binary_op  = "||" | "&&" | rel_op | add_op | mul_op .
-rel_op     = "==" | "!=" | "<" | "<=" | ">" | ">=" .
-add_op     = "+" | "-" | "|" | "^" .
-mul_op     = "*" | "/" | "%" | "<<" | ">>" | "&" | "&^" .
-
-unary_op   = "+" | "-" | "!" | "^" | "*" | "&" | "<-" .
-
- -

-Comparisons are discussed elsewhere. -For other binary operators, the operand types must be identical -unless the operation involves shifts or untyped constants. -For operations involving constants only, see the section on -constant expressions. -

- -

-Except for shift operations, if one operand is an untyped constant -and the other operand is not, the constant is implicitly converted -to the type of the other operand. -

- -

-The right operand in a shift expression must have integer type -or be an untyped constant representable by a -value of type uint. -If the left operand of a non-constant shift expression is an untyped constant, -it is first implicitly converted to the type it would assume if the shift expression were -replaced by its left operand alone. -

- -
-var a [1024]byte
-var s uint = 33
-
-// The results of the following examples are given for 64-bit ints.
-var i = 1<<s                   // 1 has type int
-var j int32 = 1<<s             // 1 has type int32; j == 0
-var k = uint64(1<<s)           // 1 has type uint64; k == 1<<33
-var m int = 1.0<<s             // 1.0 has type int; m == 1<<33
-var n = 1.0<<s == j            // 1.0 has type int32; n == true
-var o = 1<<s == 2<<s           // 1 and 2 have type int; o == false
-var p = 1<<s == 1<<33          // 1 has type int; p == true
-var u = 1.0<<s                 // illegal: 1.0 has type float64, cannot shift
-var u1 = 1.0<<s != 0           // illegal: 1.0 has type float64, cannot shift
-var u2 = 1<<s != 1.0           // illegal: 1 has type float64, cannot shift
-var v float32 = 1<<s           // illegal: 1 has type float32, cannot shift
-var w int64 = 1.0<<33          // 1.0<<33 is a constant shift expression; w == 1<<33
-var x = a[1.0<<s]              // panics: 1.0 has type int, but 1<<33 overflows array bounds
-var b = make([]byte, 1.0<<s)   // 1.0 has type int; len(b) == 1<<33
-
-// The results of the following examples are given for 32-bit ints,
-// which means the shifts will overflow.
-var mm int = 1.0<<s            // 1.0 has type int; mm == 0
-var oo = 1<<s == 2<<s          // 1 and 2 have type int; oo == true
-var pp = 1<<s == 1<<33         // illegal: 1 has type int, but 1<<33 overflows int
-var xx = a[1.0<<s]             // 1.0 has type int; xx == a[0]
-var bb = make([]byte, 1.0<<s)  // 1.0 has type int; len(bb) == 0
-
- -

Operator precedence

-

-Unary operators have the highest precedence. -As the ++ and -- operators form -statements, not expressions, they fall -outside the operator hierarchy. -As a consequence, statement *p++ is the same as (*p)++. -

- -

-There are five precedence levels for binary operators. -Multiplication operators bind strongest, followed by addition -operators, comparison operators, && (logical AND), -and finally || (logical OR): -

- -
-Precedence    Operator
-    5             *  /  %  <<  >>  &  &^
-    4             +  -  |  ^
-    3             ==  !=  <  <=  >  >=
-    2             &&
-    1             ||
-
- -

-Binary operators of the same precedence associate from left to right. -For instance, x / y * z is the same as (x / y) * z. -

- -
-+x
-23 + 3*x[i]
-x <= f()
-^a >> b
-f() || g()
-x == y+1 && <-chanInt > 0
-
- - -

Arithmetic operators

-

-Arithmetic operators apply to numeric values and yield a result of the same -type as the first operand. The four standard arithmetic operators (+, --, *, /) apply to integer, -floating-point, and complex types; + also applies to strings. -The bitwise logical and shift operators apply to integers only. -

- -
-+    sum                    integers, floats, complex values, strings
--    difference             integers, floats, complex values
-*    product                integers, floats, complex values
-/    quotient               integers, floats, complex values
-%    remainder              integers
-
-&    bitwise AND            integers
-|    bitwise OR             integers
-^    bitwise XOR            integers
-&^   bit clear (AND NOT)    integers
-
-<<   left shift             integer << integer >= 0
->>   right shift            integer >> integer >= 0
-
- - -

Integer operators

- -

-For two integer values x and y, the integer quotient -q = x / y and remainder r = x % y satisfy the following -relationships: -

- -
-x = q*y + r  and  |r| < |y|
-
- -

-with x / y truncated towards zero -("truncated division"). -

- -
- x     y     x / y     x % y
- 5     3       1         2
--5     3      -1        -2
- 5    -3      -1         2
--5    -3       1        -2
-
- -

-The one exception to this rule is that if the dividend x is -the most negative value for the int type of x, the quotient -q = x / -1 is equal to x (and r = 0) -due to two's-complement integer overflow: -

- -
-			 x, q
-int8                     -128
-int16                  -32768
-int32             -2147483648
-int64    -9223372036854775808
-
- -

-If the divisor is a constant, it must not be zero. -If the divisor is zero at run time, a run-time panic occurs. -If the dividend is non-negative and the divisor is a constant power of 2, -the division may be replaced by a right shift, and computing the remainder may -be replaced by a bitwise AND operation: -

- -
- x     x / 4     x % 4     x >> 2     x & 3
- 11      2         3         2          3
--11     -2        -3        -3          1
-
- -

-The shift operators shift the left operand by the shift count specified by the -right operand, which must be non-negative. If the shift count is negative at run time, -a run-time panic occurs. -The shift operators implement arithmetic shifts if the left operand is a signed -integer and logical shifts if it is an unsigned integer. -There is no upper limit on the shift count. Shifts behave -as if the left operand is shifted n times by 1 for a shift -count of n. -As a result, x << 1 is the same as x*2 -and x >> 1 is the same as -x/2 but truncated towards negative infinity. -

- -

-For integer operands, the unary operators -+, -, and ^ are defined as -follows: -

- -
-+x                          is 0 + x
--x    negation              is 0 - x
-^x    bitwise complement    is m ^ x  with m = "all bits set to 1" for unsigned x
-                                      and  m = -1 for signed x
-
- - -

Integer overflow

- -

-For unsigned integer values, the operations +, --, *, and << are -computed modulo 2n, where n is the bit width of -the unsigned integer's type. -Loosely speaking, these unsigned integer operations -discard high bits upon overflow, and programs may rely on "wrap around". -

-

-For signed integers, the operations +, --, *, /, and << may legally -overflow and the resulting value exists and is deterministically defined -by the signed integer representation, the operation, and its operands. -Overflow does not cause a run-time panic. -A compiler may not optimize code under the assumption that overflow does -not occur. For instance, it may not assume that x < x + 1 is always true. -

- - -

Floating-point operators

- -

-For floating-point and complex numbers, -+x is the same as x, -while -x is the negation of x. -The result of a floating-point or complex division by zero is not specified beyond the -IEEE 754 standard; whether a run-time panic -occurs is implementation-specific. -

- -

-An implementation may combine multiple floating-point operations into a single -fused operation, possibly across statements, and produce a result that differs -from the value obtained by executing and rounding the instructions individually. -An explicit floating-point type conversion rounds to -the precision of the target type, preventing fusion that would discard that rounding. -

- -

-For instance, some architectures provide a "fused multiply and add" (FMA) instruction -that computes x*y + z without rounding the intermediate result x*y. -These examples show when a Go implementation can use that instruction: -

- -
-// FMA allowed for computing r, because x*y is not explicitly rounded:
-r  = x*y + z
-r  = z;   r += x*y
-t  = x*y; r = t + z
-*p = x*y; r = *p + z
-r  = x*y + float64(z)
-
-// FMA disallowed for computing r, because it would omit rounding of x*y:
-r  = float64(x*y) + z
-r  = z; r += float64(x*y)
-t  = float64(x*y); r = t + z
-
- -

String concatenation

- -

-Strings can be concatenated using the + operator -or the += assignment operator: -

- -
-s := "hi" + string(c)
-s += " and good bye"
-
- -

-String addition creates a new string by concatenating the operands. -

- - -

Comparison operators

- -

-Comparison operators compare two operands and yield an untyped boolean value. -

- -
-==    equal
-!=    not equal
-<     less
-<=    less or equal
->     greater
->=    greater or equal
-
- -

-In any comparison, the first operand -must be assignable -to the type of the second operand, or vice versa. -

-

-The equality operators == and != apply -to operands that are comparable. -The ordering operators <, <=, >, and >= -apply to operands that are ordered. -These terms and the result of the comparisons are defined as follows: -

- - - -

-A comparison of two interface values with identical dynamic types -causes a run-time panic if values -of that type are not comparable. This behavior applies not only to direct interface -value comparisons but also when comparing arrays of interface values -or structs with interface-valued fields. -

- -

-Slice, map, and function values are not comparable. -However, as a special case, a slice, map, or function value may -be compared to the predeclared identifier nil. -Comparison of pointer, channel, and interface values to nil -is also allowed and follows from the general rules above. -

- -
-const c = 3 < 4            // c is the untyped boolean constant true
-
-type MyBool bool
-var x, y int
-var (
-	// The result of a comparison is an untyped boolean.
-	// The usual assignment rules apply.
-	b3        = x == y // b3 has type bool
-	b4 bool   = x == y // b4 has type bool
-	b5 MyBool = x == y // b5 has type MyBool
-)
-
- -

Logical operators

- -

-Logical operators apply to boolean values -and yield a result of the same type as the operands. -The right operand is evaluated conditionally. -

- -
-&&    conditional AND    p && q  is  "if p then q else false"
-||    conditional OR     p || q  is  "if p then true else q"
-!     NOT                !p      is  "not p"
-
- - -

Address operators

- -

-For an operand x of type T, the address operation -&x generates a pointer of type *T to x. -The operand must be addressable, -that is, either a variable, pointer indirection, or slice indexing -operation; or a field selector of an addressable struct operand; -or an array indexing operation of an addressable array. -As an exception to the addressability requirement, x may also be a -(possibly parenthesized) -composite literal. -If the evaluation of x would cause a run-time panic, -then the evaluation of &x does too. -

- -

-For an operand x of pointer type *T, the pointer -indirection *x denotes the variable of type T pointed -to by x. -If x is nil, an attempt to evaluate *x -will cause a run-time panic. -

- -
-&x
-&a[f(2)]
-&Point{2, 3}
-*p
-*pf(x)
-
-var x *int = nil
-*x   // causes a run-time panic
-&*x  // causes a run-time panic
-
- - -

Receive operator

- -

-For an operand ch of channel type, -the value of the receive operation <-ch is the value received -from the channel ch. The channel direction must permit receive operations, -and the type of the receive operation is the element type of the channel. -The expression blocks until a value is available. -Receiving from a nil channel blocks forever. -A receive operation on a closed channel can always proceed -immediately, yielding the element type's zero value -after any previously sent values have been received. -

- -
-v1 := <-ch
-v2 = <-ch
-f(<-ch)
-<-strobe  // wait until clock pulse and discard received value
-
- -

-A receive expression used in an assignment or initialization of the special form -

- -
-x, ok = <-ch
-x, ok := <-ch
-var x, ok = <-ch
-var x, ok T = <-ch
-
- -

-yields an additional untyped boolean result reporting whether the -communication succeeded. The value of ok is true -if the value received was delivered by a successful send operation to the -channel, or false if it is a zero value generated because the -channel is closed and empty. -

- - -

Conversions

- -

-A conversion changes the type of an expression -to the type specified by the conversion. -A conversion may appear literally in the source, or it may be implied -by the context in which an expression appears. -

- -

-An explicit conversion is an expression of the form T(x) -where T is a type and x is an expression -that can be converted to type T. -

- -
-Conversion = Type "(" Expression [ "," ] ")" .
-
- -

-If the type starts with the operator * or <-, -or if the type starts with the keyword func -and has no result list, it must be parenthesized when -necessary to avoid ambiguity: -

- -
-*Point(p)        // same as *(Point(p))
-(*Point)(p)      // p is converted to *Point
-<-chan int(c)    // same as <-(chan int(c))
-(<-chan int)(c)  // c is converted to <-chan int
-func()(x)        // function signature func() x
-(func())(x)      // x is converted to func()
-(func() int)(x)  // x is converted to func() int
-func() int(x)    // x is converted to func() int (unambiguous)
-
- -

-A constant value x can be converted to -type T if x is representable -by a value of T. -As a special case, an integer constant x can be explicitly converted to a -string type using the -same rule -as for non-constant x. -

- -

-Converting a constant yields a typed constant as result. -

- -
-uint(iota)               // iota value of type uint
-float32(2.718281828)     // 2.718281828 of type float32
-complex128(1)            // 1.0 + 0.0i of type complex128
-float32(0.49999999)      // 0.5 of type float32
-float64(-1e-1000)        // 0.0 of type float64
-string('x')              // "x" of type string
-string(0x266c)           // "♬" of type string
-MyString("foo" + "bar")  // "foobar" of type MyString
-string([]byte{'a'})      // not a constant: []byte{'a'} is not a constant
-(*int)(nil)              // not a constant: nil is not a constant, *int is not a boolean, numeric, or string type
-int(1.2)                 // illegal: 1.2 cannot be represented as an int
-string(65.0)             // illegal: 65.0 is not an integer constant
-
- -

-A non-constant value x can be converted to type T -in any of these cases: -

- - - -

-Struct tags are ignored when comparing struct types -for identity for the purpose of conversion: -

- -
-type Person struct {
-	Name    string
-	Address *struct {
-		Street string
-		City   string
-	}
-}
-
-var data *struct {
-	Name    string `json:"name"`
-	Address *struct {
-		Street string `json:"street"`
-		City   string `json:"city"`
-	} `json:"address"`
-}
-
-var person = (*Person)(data)  // ignoring tags, the underlying types are identical
-
- -

-Specific rules apply to (non-constant) conversions between numeric types or -to and from a string type. -These conversions may change the representation of x -and incur a run-time cost. -All other conversions only change the type but not the representation -of x. -

- -

-There is no linguistic mechanism to convert between pointers and integers. -The package unsafe -implements this functionality under -restricted circumstances. -

- -

Conversions between numeric types

- -

-For the conversion of non-constant numeric values, the following rules apply: -

- -
    -
  1. -When converting between integer types, if the value is a signed integer, it is -sign extended to implicit infinite precision; otherwise it is zero extended. -It is then truncated to fit in the result type's size. -For example, if v := uint16(0x10F0), then uint32(int8(v)) == 0xFFFFFFF0. -The conversion always yields a valid value; there is no indication of overflow. -
  2. -
  3. -When converting a floating-point number to an integer, the fraction is discarded -(truncation towards zero). -
  4. -
  5. -When converting an integer or floating-point number to a floating-point type, -or a complex number to another complex type, the result value is rounded -to the precision specified by the destination type. -For instance, the value of a variable x of type float32 -may be stored using additional precision beyond that of an IEEE 754 32-bit number, -but float32(x) represents the result of rounding x's value to -32-bit precision. Similarly, x + 0.1 may use more than 32 bits -of precision, but float32(x + 0.1) does not. -
  6. -
- -

-In all non-constant conversions involving floating-point or complex values, -if the result type cannot represent the value the conversion -succeeds but the result value is implementation-dependent. -

- -

Conversions to and from a string type

- -
    -
  1. -Converting a signed or unsigned integer value to a string type yields a -string containing the UTF-8 representation of the integer. Values outside -the range of valid Unicode code points are converted to "\uFFFD". - -
    -string('a')       // "a"
    -string(-1)        // "\ufffd" == "\xef\xbf\xbd"
    -string(0xf8)      // "\u00f8" == "ø" == "\xc3\xb8"
    -type MyString string
    -MyString(0x65e5)  // "\u65e5" == "日" == "\xe6\x97\xa5"
    -
    -
  2. - -
  3. -Converting a slice of bytes to a string type yields -a string whose successive bytes are the elements of the slice. - -
    -string([]byte{'h', 'e', 'l', 'l', '\xc3', '\xb8'})   // "hellø"
    -string([]byte{})                                     // ""
    -string([]byte(nil))                                  // ""
    -
    -type MyBytes []byte
    -string(MyBytes{'h', 'e', 'l', 'l', '\xc3', '\xb8'})  // "hellø"
    -
    -
  4. - -
  5. -Converting a slice of runes to a string type yields -a string that is the concatenation of the individual rune values -converted to strings. - -
    -string([]rune{0x767d, 0x9d6c, 0x7fd4})   // "\u767d\u9d6c\u7fd4" == "白鵬翔"
    -string([]rune{})                         // ""
    -string([]rune(nil))                      // ""
    -
    -type MyRunes []rune
    -string(MyRunes{0x767d, 0x9d6c, 0x7fd4})  // "\u767d\u9d6c\u7fd4" == "白鵬翔"
    -
    -
  6. - -
  7. -Converting a value of a string type to a slice of bytes type -yields a slice whose successive elements are the bytes of the string. - -
    -[]byte("hellø")   // []byte{'h', 'e', 'l', 'l', '\xc3', '\xb8'}
    -[]byte("")        // []byte{}
    -
    -MyBytes("hellø")  // []byte{'h', 'e', 'l', 'l', '\xc3', '\xb8'}
    -
    -
  8. - -
  9. -Converting a value of a string type to a slice of runes type -yields a slice containing the individual Unicode code points of the string. - -
    -[]rune(MyString("白鵬翔"))  // []rune{0x767d, 0x9d6c, 0x7fd4}
    -[]rune("")                 // []rune{}
    -
    -MyRunes("白鵬翔")           // []rune{0x767d, 0x9d6c, 0x7fd4}
    -
    -
  10. -
- -

Conversions from slice to array pointer

- -

-Converting a slice to an array pointer yields a pointer to the underlying array of the slice. -If the length of the slice is less than the length of the array, -a run-time panic occurs. -

- -
-s := make([]byte, 2, 4)
-s0 := (*[0]byte)(s)      // s0 != nil
-s1 := (*[1]byte)(s[1:])  // &s1[0] == &s[1]
-s2 := (*[2]byte)(s)      // &s2[0] == &s[0]
-s4 := (*[4]byte)(s)      // panics: len([4]byte) > len(s)
-
-var t []string
-t0 := (*[0]string)(t)    // t0 == nil
-t1 := (*[1]string)(t)    // panics: len([1]string) > len(t)
-
-u := make([]byte, 0)
-u0 := (*[0]byte)(u)      // u0 != nil
-
- -

Constant expressions

- -

-Constant expressions may contain only constant -operands and are evaluated at compile time. -

- -

-Untyped boolean, numeric, and string constants may be used as operands -wherever it is legal to use an operand of boolean, numeric, or string type, -respectively. -

- -

-A constant comparison always yields -an untyped boolean constant. If the left operand of a constant -shift expression is an untyped constant, the -result is an integer constant; otherwise it is a constant of the same -type as the left operand, which must be of -integer type. -

- -

-Any other operation on untyped constants results in an untyped constant of the -same kind; that is, a boolean, integer, floating-point, complex, or string -constant. -If the untyped operands of a binary operation (other than a shift) are of -different kinds, the result is of the operand's kind that appears later in this -list: integer, rune, floating-point, complex. -For example, an untyped integer constant divided by an -untyped complex constant yields an untyped complex constant. -

- -
-const a = 2 + 3.0          // a == 5.0   (untyped floating-point constant)
-const b = 15 / 4           // b == 3     (untyped integer constant)
-const c = 15 / 4.0         // c == 3.75  (untyped floating-point constant)
-const Θ float64 = 3/2      // Θ == 1.0   (type float64, 3/2 is integer division)
-const Π float64 = 3/2.     // Π == 1.5   (type float64, 3/2. is float division)
-const d = 1 << 3.0         // d == 8     (untyped integer constant)
-const e = 1.0 << 3         // e == 8     (untyped integer constant)
-const f = int32(1) << 33   // illegal    (constant 8589934592 overflows int32)
-const g = float64(2) >> 1  // illegal    (float64(2) is a typed floating-point constant)
-const h = "foo" > "bar"    // h == true  (untyped boolean constant)
-const j = true             // j == true  (untyped boolean constant)
-const k = 'w' + 1          // k == 'x'   (untyped rune constant)
-const l = "hi"             // l == "hi"  (untyped string constant)
-const m = string(k)        // m == "x"   (type string)
-const Σ = 1 - 0.707i       //            (untyped complex constant)
-const Δ = Σ + 2.0e-4       //            (untyped complex constant)
-const Φ = iota*1i - 1/1i   //            (untyped complex constant)
-
- -

-Applying the built-in function complex to untyped -integer, rune, or floating-point constants yields -an untyped complex constant. -

- -
-const ic = complex(0, c)   // ic == 3.75i  (untyped complex constant)
-const iΘ = complex(0, Θ)   // iΘ == 1i     (type complex128)
-
- -

-Constant expressions are always evaluated exactly; intermediate values and the -constants themselves may require precision significantly larger than supported -by any predeclared type in the language. The following are legal declarations: -

- -
-const Huge = 1 << 100         // Huge == 1267650600228229401496703205376  (untyped integer constant)
-const Four int8 = Huge >> 98  // Four == 4                                (type int8)
-
- -

-The divisor of a constant division or remainder operation must not be zero: -

- -
-3.14 / 0.0   // illegal: division by zero
-
- -

-The values of typed constants must always be accurately -representable by values -of the constant type. The following constant expressions are illegal: -

- -
-uint(-1)     // -1 cannot be represented as a uint
-int(3.14)    // 3.14 cannot be represented as an int
-int64(Huge)  // 1267650600228229401496703205376 cannot be represented as an int64
-Four * 300   // operand 300 cannot be represented as an int8 (type of Four)
-Four * 100   // product 400 cannot be represented as an int8 (type of Four)
-
- -

-The mask used by the unary bitwise complement operator ^ matches -the rule for non-constants: the mask is all 1s for unsigned constants -and -1 for signed and untyped constants. -

- -
-^1         // untyped integer constant, equal to -2
-uint8(^1)  // illegal: same as uint8(-2), -2 cannot be represented as a uint8
-^uint8(1)  // typed uint8 constant, same as 0xFF ^ uint8(1) = uint8(0xFE)
-int8(^1)   // same as int8(-2)
-^int8(1)   // same as -1 ^ int8(1) = -2
-
- -

-Implementation restriction: A compiler may use rounding while -computing untyped floating-point or complex constant expressions; see -the implementation restriction in the section -on constants. This rounding may cause a -floating-point constant expression to be invalid in an integer -context, even if it would be integral when calculated using infinite -precision, and vice versa. -

- - -

Order of evaluation

- -

-At package level, initialization dependencies -determine the evaluation order of individual initialization expressions in -variable declarations. -Otherwise, when evaluating the operands of an -expression, assignment, or -return statement, -all function calls, method calls, and -communication operations are evaluated in lexical left-to-right -order. -

- -

-For example, in the (function-local) assignment -

-
-y[f()], ok = g(h(), i()+x[j()], <-c), k()
-
-

-the function calls and communication happen in the order -f(), h(), i(), j(), -<-c, g(), and k(). -However, the order of those events compared to the evaluation -and indexing of x and the evaluation -of y is not specified. -

- -
-a := 1
-f := func() int { a++; return a }
-x := []int{a, f()}            // x may be [1, 2] or [2, 2]: evaluation order between a and f() is not specified
-m := map[int]int{a: 1, a: 2}  // m may be {2: 1} or {2: 2}: evaluation order between the two map assignments is not specified
-n := map[int]int{a: f()}      // n may be {2: 3} or {3: 3}: evaluation order between the key and the value is not specified
-
- -

-At package level, initialization dependencies override the left-to-right rule -for individual initialization expressions, but not for operands within each -expression: -

- -
-var a, b, c = f() + v(), g(), sqr(u()) + v()
-
-func f() int        { return c }
-func g() int        { return a }
-func sqr(x int) int { return x*x }
-
-// functions u and v are independent of all other variables and functions
-
- -

-The function calls happen in the order -u(), sqr(), v(), -f(), v(), and g(). -

- -

-Floating-point operations within a single expression are evaluated according to -the associativity of the operators. Explicit parentheses affect the evaluation -by overriding the default associativity. -In the expression x + (y + z) the addition y + z -is performed before adding x. -

- -

Statements

- -

-Statements control execution. -

- -
-Statement =
-	Declaration | LabeledStmt | SimpleStmt |
-	GoStmt | ReturnStmt | BreakStmt | ContinueStmt | GotoStmt |
-	FallthroughStmt | Block | IfStmt | SwitchStmt | SelectStmt | ForStmt |
-	DeferStmt .
-
-SimpleStmt = EmptyStmt | ExpressionStmt | SendStmt | IncDecStmt | Assignment | ShortVarDecl .
-
- -

Terminating statements

- -

-A terminating statement interrupts the regular flow of control in -a block. The following statements are terminating: -

- -
    -
  1. - A "return" or - "goto" statement. - - -
  2. - -
  3. - A call to the built-in function - panic. - - -
  4. - -
  5. - A block in which the statement list ends in a terminating statement. - - -
  6. - -
  7. - An "if" statement in which: - -
  8. - -
  9. - A "for" statement in which: - -
  10. - -
  11. - A "switch" statement in which: - -
  12. - -
  13. - A "select" statement in which: - -
  14. - -
  15. - A labeled statement labeling - a terminating statement. -
  16. -
- -

-All other statements are not terminating. -

- -

-A statement list ends in a terminating statement if the list -is not empty and its final non-empty statement is terminating. -

- - -

Empty statements

- -

-The empty statement does nothing. -

- -
-EmptyStmt = .
-
- - -

Labeled statements

- -

-A labeled statement may be the target of a goto, -break or continue statement. -

- -
-LabeledStmt = Label ":" Statement .
-Label       = identifier .
-
- -
-Error: log.Panic("error encountered")
-
- - -

Expression statements

- -

-With the exception of specific built-in functions, -function and method calls and -receive operations -can appear in statement context. Such statements may be parenthesized. -

- -
-ExpressionStmt = Expression .
-
- -

-The following built-in functions are not permitted in statement context: -

- -
-append cap complex imag len make new real
-unsafe.Add unsafe.Alignof unsafe.Offsetof unsafe.Sizeof unsafe.Slice
-
- -
-h(x+y)
-f.Close()
-<-ch
-(<-ch)
-len("foo")  // illegal if len is the built-in function
-
- - -

Send statements

- -

-A send statement sends a value on a channel. -The channel expression must be of channel type, -the channel direction must permit send operations, -and the type of the value to be sent must be assignable -to the channel's element type. -

- -
-SendStmt = Channel "<-" Expression .
-Channel  = Expression .
-
- -

-Both the channel and the value expression are evaluated before communication -begins. Communication blocks until the send can proceed. -A send on an unbuffered channel can proceed if a receiver is ready. -A send on a buffered channel can proceed if there is room in the buffer. -A send on a closed channel proceeds by causing a run-time panic. -A send on a nil channel blocks forever. -

- -
-ch <- 3  // send value 3 to channel ch
-
- - -

IncDec statements

- -

-The "++" and "--" statements increment or decrement their operands -by the untyped constant 1. -As with an assignment, the operand must be addressable -or a map index expression. -

- -
-IncDecStmt = Expression ( "++" | "--" ) .
-
- -

-The following assignment statements are semantically -equivalent: -

- -
-IncDec statement    Assignment
-x++                 x += 1
-x--                 x -= 1
-
- - -

Assignments

- -
-Assignment = ExpressionList assign_op ExpressionList .
-
-assign_op = [ add_op | mul_op ] "=" .
-
- -

-Each left-hand side operand must be addressable, -a map index expression, or (for = assignments only) the -blank identifier. -Operands may be parenthesized. -

- -
-x = 1
-*p = f()
-a[i] = 23
-(k) = <-ch  // same as: k = <-ch
-
- -

-An assignment operation x op= -y where op is a binary arithmetic operator -is equivalent to x = x op -(y) but evaluates x -only once. The op= construct is a single token. -In assignment operations, both the left- and right-hand expression lists -must contain exactly one single-valued expression, and the left-hand -expression must not be the blank identifier. -

- -
-a[i] <<= 2
-i &^= 1<<n
-
- -

-A tuple assignment assigns the individual elements of a multi-valued -operation to a list of variables. There are two forms. In the -first, the right hand operand is a single multi-valued expression -such as a function call, a channel or -map operation, or a type assertion. -The number of operands on the left -hand side must match the number of values. For instance, if -f is a function returning two values, -

- -
-x, y = f()
-
- -

-assigns the first value to x and the second to y. -In the second form, the number of operands on the left must equal the number -of expressions on the right, each of which must be single-valued, and the -nth expression on the right is assigned to the nth -operand on the left: -

- -
-one, two, three = '一', '二', '三'
-
- -

-The blank identifier provides a way to -ignore right-hand side values in an assignment: -

- -
-_ = x       // evaluate x but ignore it
-x, _ = f()  // evaluate f() but ignore second result value
-
- -

-The assignment proceeds in two phases. -First, the operands of index expressions -and pointer indirections -(including implicit pointer indirections in selectors) -on the left and the expressions on the right are all -evaluated in the usual order. -Second, the assignments are carried out in left-to-right order. -

- -
-a, b = b, a  // exchange a and b
-
-x := []int{1, 2, 3}
-i := 0
-i, x[i] = 1, 2  // set i = 1, x[0] = 2
-
-i = 0
-x[i], i = 2, 1  // set x[0] = 2, i = 1
-
-x[0], x[0] = 1, 2  // set x[0] = 1, then x[0] = 2 (so x[0] == 2 at end)
-
-x[1], x[3] = 4, 5  // set x[1] = 4, then panic setting x[3] = 5.
-
-type Point struct { x, y int }
-var p *Point
-x[2], p.x = 6, 7  // set x[2] = 6, then panic setting p.x = 7
-
-i = 2
-x = []int{3, 5, 7}
-for i, x[i] = range x {  // set i, x[2] = 0, x[0]
-	break
-}
-// after this loop, i == 0 and x == []int{3, 5, 3}
-
- -

-In assignments, each value must be assignable -to the type of the operand to which it is assigned, with the following special cases: -

- -
    -
  1. - Any typed value may be assigned to the blank identifier. -
  2. - -
  3. - If an untyped constant - is assigned to a variable of interface type or the blank identifier, - the constant is first implicitly converted to its - default type. -
  4. - -
  5. - If an untyped boolean value is assigned to a variable of interface type or - the blank identifier, it is first implicitly converted to type bool. -
  6. -
- -

If statements

- -

-"If" statements specify the conditional execution of two branches -according to the value of a boolean expression. If the expression -evaluates to true, the "if" branch is executed, otherwise, if -present, the "else" branch is executed. -

- -
-IfStmt = "if" [ SimpleStmt ";" ] Expression Block [ "else" ( IfStmt | Block ) ] .
-
- -
-if x > max {
-	x = max
-}
-
- -

-The expression may be preceded by a simple statement, which -executes before the expression is evaluated. -

- -
-if x := f(); x < y {
-	return x
-} else if x > z {
-	return z
-} else {
-	return y
-}
-
- - -

Switch statements

- -

-"Switch" statements provide multi-way execution. -An expression or type is compared to the "cases" -inside the "switch" to determine which branch -to execute. -

- -
-SwitchStmt = ExprSwitchStmt | TypeSwitchStmt .
-
- -

-There are two forms: expression switches and type switches. -In an expression switch, the cases contain expressions that are compared -against the value of the switch expression. -In a type switch, the cases contain types that are compared against the -type of a specially annotated switch expression. -The switch expression is evaluated exactly once in a switch statement. -

- -

Expression switches

- -

-In an expression switch, -the switch expression is evaluated and -the case expressions, which need not be constants, -are evaluated left-to-right and top-to-bottom; the first one that equals the -switch expression -triggers execution of the statements of the associated case; -the other cases are skipped. -If no case matches and there is a "default" case, -its statements are executed. -There can be at most one default case and it may appear anywhere in the -"switch" statement. -A missing switch expression is equivalent to the boolean value -true. -

- -
-ExprSwitchStmt = "switch" [ SimpleStmt ";" ] [ Expression ] "{" { ExprCaseClause } "}" .
-ExprCaseClause = ExprSwitchCase ":" StatementList .
-ExprSwitchCase = "case" ExpressionList | "default" .
-
- -

-If the switch expression evaluates to an untyped constant, it is first implicitly -converted to its default type. -The predeclared untyped value nil cannot be used as a switch expression. -The switch expression type must be comparable. -

- -

-If a case expression is untyped, it is first implicitly converted -to the type of the switch expression. -For each (possibly converted) case expression x and the value t -of the switch expression, x == t must be a valid comparison. -

- -

-In other words, the switch expression is treated as if it were used to declare and -initialize a temporary variable t without explicit type; it is that -value of t against which each case expression x is tested -for equality. -

- -

-In a case or default clause, the last non-empty statement -may be a (possibly labeled) -"fallthrough" statement to -indicate that control should flow from the end of this clause to -the first statement of the next clause. -Otherwise control flows to the end of the "switch" statement. -A "fallthrough" statement may appear as the last statement of all -but the last clause of an expression switch. -

- -

-The switch expression may be preceded by a simple statement, which -executes before the expression is evaluated. -

- -
-switch tag {
-default: s3()
-case 0, 1, 2, 3: s1()
-case 4, 5, 6, 7: s2()
-}
-
-switch x := f(); {  // missing switch expression means "true"
-case x < 0: return -x
-default: return x
-}
-
-switch {
-case x < y: f1()
-case x < z: f2()
-case x == 4: f3()
-}
-
- -

-Implementation restriction: A compiler may disallow multiple case -expressions evaluating to the same constant. -For instance, the current compilers disallow duplicate integer, -floating point, or string constants in case expressions. -

- -

Type switches

- -

-A type switch compares types rather than values. It is otherwise similar -to an expression switch. It is marked by a special switch expression that -has the form of a type assertion -using the keyword type rather than an actual type: -

- -
-switch x.(type) {
-// cases
-}
-
- -

-Cases then match actual types T against the dynamic type of the -expression x. As with type assertions, x must be of -interface type, and each non-interface type -T listed in a case must implement the type of x. -The types listed in the cases of a type switch must all be -different. -

- -
-TypeSwitchStmt  = "switch" [ SimpleStmt ";" ] TypeSwitchGuard "{" { TypeCaseClause } "}" .
-TypeSwitchGuard = [ identifier ":=" ] PrimaryExpr "." "(" "type" ")" .
-TypeCaseClause  = TypeSwitchCase ":" StatementList .
-TypeSwitchCase  = "case" TypeList | "default" .
-TypeList        = Type { "," Type } .
-
- -

-The TypeSwitchGuard may include a -short variable declaration. -When that form is used, the variable is declared at the end of the -TypeSwitchCase in the implicit block of each clause. -In clauses with a case listing exactly one type, the variable -has that type; otherwise, the variable has the type of the expression -in the TypeSwitchGuard. -

- -

-Instead of a type, a case may use the predeclared identifier -nil; -that case is selected when the expression in the TypeSwitchGuard -is a nil interface value. -There may be at most one nil case. -

- -

-Given an expression x of type interface{}, -the following type switch: -

- -
-switch i := x.(type) {
-case nil:
-	printString("x is nil")                // type of i is type of x (interface{})
-case int:
-	printInt(i)                            // type of i is int
-case float64:
-	printFloat64(i)                        // type of i is float64
-case func(int) float64:
-	printFunction(i)                       // type of i is func(int) float64
-case bool, string:
-	printString("type is bool or string")  // type of i is type of x (interface{})
-default:
-	printString("don't know the type")     // type of i is type of x (interface{})
-}
-
- -

-could be rewritten: -

- -
-v := x  // x is evaluated exactly once
-if v == nil {
-	i := v                                 // type of i is type of x (interface{})
-	printString("x is nil")
-} else if i, isInt := v.(int); isInt {
-	printInt(i)                            // type of i is int
-} else if i, isFloat64 := v.(float64); isFloat64 {
-	printFloat64(i)                        // type of i is float64
-} else if i, isFunc := v.(func(int) float64); isFunc {
-	printFunction(i)                       // type of i is func(int) float64
-} else {
-	_, isBool := v.(bool)
-	_, isString := v.(string)
-	if isBool || isString {
-		i := v                         // type of i is type of x (interface{})
-		printString("type is bool or string")
-	} else {
-		i := v                         // type of i is type of x (interface{})
-		printString("don't know the type")
-	}
-}
-
- -

-The type switch guard may be preceded by a simple statement, which -executes before the guard is evaluated. -

- -

-The "fallthrough" statement is not permitted in a type switch. -

- -

For statements

- -

-A "for" statement specifies repeated execution of a block. There are three forms: -The iteration may be controlled by a single condition, a "for" clause, or a "range" clause. -

- -
-ForStmt = "for" [ Condition | ForClause | RangeClause ] Block .
-Condition = Expression .
-
- -

For statements with single condition

- -

-In its simplest form, a "for" statement specifies the repeated execution of -a block as long as a boolean condition evaluates to true. -The condition is evaluated before each iteration. -If the condition is absent, it is equivalent to the boolean value -true. -

- -
-for a < b {
-	a *= 2
-}
-
- -

For statements with for clause

- -

-A "for" statement with a ForClause is also controlled by its condition, but -additionally it may specify an init -and a post statement, such as an assignment, -an increment or decrement statement. The init statement may be a -short variable declaration, but the post statement must not. -Variables declared by the init statement are re-used in each iteration. -

- -
-ForClause = [ InitStmt ] ";" [ Condition ] ";" [ PostStmt ] .
-InitStmt = SimpleStmt .
-PostStmt = SimpleStmt .
-
- -
-for i := 0; i < 10; i++ {
-	f(i)
-}
-
- -

-If non-empty, the init statement is executed once before evaluating the -condition for the first iteration; -the post statement is executed after each execution of the block (and -only if the block was executed). -Any element of the ForClause may be empty but the -semicolons are -required unless there is only a condition. -If the condition is absent, it is equivalent to the boolean value -true. -

- -
-for cond { S() }    is the same as    for ; cond ; { S() }
-for      { S() }    is the same as    for true     { S() }
-
- -

For statements with range clause

- -

-A "for" statement with a "range" clause -iterates through all entries of an array, slice, string or map, -or values received on a channel. For each entry it assigns iteration values -to corresponding iteration variables if present and then executes the block. -

- -
-RangeClause = [ ExpressionList "=" | IdentifierList ":=" ] "range" Expression .
-
- -

-The expression on the right in the "range" clause is called the range expression, -which may be an array, pointer to an array, slice, string, map, or channel permitting -receive operations. -As with an assignment, if present the operands on the left must be -addressable or map index expressions; they -denote the iteration variables. If the range expression is a channel, at most -one iteration variable is permitted, otherwise there may be up to two. -If the last iteration variable is the blank identifier, -the range clause is equivalent to the same clause without that identifier. -

- -

-The range expression x is evaluated once before beginning the loop, -with one exception: if at most one iteration variable is present and -len(x) is constant, -the range expression is not evaluated. -

- -

-Function calls on the left are evaluated once per iteration. -For each iteration, iteration values are produced as follows -if the respective iteration variables are present: -

- -
-Range expression                          1st value          2nd value
-
-array or slice  a  [n]E, *[n]E, or []E    index    i  int    a[i]       E
-string          s  string type            index    i  int    see below  rune
-map             m  map[K]V                key      k  K      m[k]       V
-channel         c  chan E, <-chan E       element  e  E
-
- -
    -
  1. -For an array, pointer to array, or slice value a, the index iteration -values are produced in increasing order, starting at element index 0. -If at most one iteration variable is present, the range loop produces -iteration values from 0 up to len(a)-1 and does not index into the array -or slice itself. For a nil slice, the number of iterations is 0. -
  2. - -
  3. -For a string value, the "range" clause iterates over the Unicode code points -in the string starting at byte index 0. On successive iterations, the index value will be the -index of the first byte of successive UTF-8-encoded code points in the string, -and the second value, of type rune, will be the value of -the corresponding code point. If the iteration encounters an invalid -UTF-8 sequence, the second value will be 0xFFFD, -the Unicode replacement character, and the next iteration will advance -a single byte in the string. -
  4. - -
  5. -The iteration order over maps is not specified -and is not guaranteed to be the same from one iteration to the next. -If a map entry that has not yet been reached is removed during iteration, -the corresponding iteration value will not be produced. If a map entry is -created during iteration, that entry may be produced during the iteration or -may be skipped. The choice may vary for each entry created and from one -iteration to the next. -If the map is nil, the number of iterations is 0. -
  6. - -
  7. -For channels, the iteration values produced are the successive values sent on -the channel until the channel is closed. If the channel -is nil, the range expression blocks forever. -
  8. -
- -

-The iteration values are assigned to the respective -iteration variables as in an assignment statement. -

- -

-The iteration variables may be declared by the "range" clause using a form of -short variable declaration -(:=). -In this case their types are set to the types of the respective iteration values -and their scope is the block of the "for" -statement; they are re-used in each iteration. -If the iteration variables are declared outside the "for" statement, -after execution their values will be those of the last iteration. -

- -
-var testdata *struct {
-	a *[7]int
-}
-for i, _ := range testdata.a {
-	// testdata.a is never evaluated; len(testdata.a) is constant
-	// i ranges from 0 to 6
-	f(i)
-}
-
-var a [10]string
-for i, s := range a {
-	// type of i is int
-	// type of s is string
-	// s == a[i]
-	g(i, s)
-}
-
-var key string
-var val interface{}  // element type of m is assignable to val
-m := map[string]int{"mon":0, "tue":1, "wed":2, "thu":3, "fri":4, "sat":5, "sun":6}
-for key, val = range m {
-	h(key, val)
-}
-// key == last map key encountered in iteration
-// val == map[key]
-
-var ch chan Work = producer()
-for w := range ch {
-	doWork(w)
-}
-
-// empty a channel
-for range ch {}
-
- - -

Go statements

- -

-A "go" statement starts the execution of a function call -as an independent concurrent thread of control, or goroutine, -within the same address space. -

- -
-GoStmt = "go" Expression .
-
- -

-The expression must be a function or method call; it cannot be parenthesized. -Calls of built-in functions are restricted as for -expression statements. -

- -

-The function value and parameters are -evaluated as usual -in the calling goroutine, but -unlike with a regular call, program execution does not wait -for the invoked function to complete. -Instead, the function begins executing independently -in a new goroutine. -When the function terminates, its goroutine also terminates. -If the function has any return values, they are discarded when the -function completes. -

- -
-go Server()
-go func(ch chan<- bool) { for { sleep(10); ch <- true }} (c)
-
- - -

Select statements

- -

-A "select" statement chooses which of a set of possible -send or -receive -operations will proceed. -It looks similar to a -"switch" statement but with the -cases all referring to communication operations. -

- -
-SelectStmt = "select" "{" { CommClause } "}" .
-CommClause = CommCase ":" StatementList .
-CommCase   = "case" ( SendStmt | RecvStmt ) | "default" .
-RecvStmt   = [ ExpressionList "=" | IdentifierList ":=" ] RecvExpr .
-RecvExpr   = Expression .
-
- -

-A case with a RecvStmt may assign the result of a RecvExpr to one or -two variables, which may be declared using a -short variable declaration. -The RecvExpr must be a (possibly parenthesized) receive operation. -There can be at most one default case and it may appear anywhere -in the list of cases. -

- -

-Execution of a "select" statement proceeds in several steps: -

- -
    -
  1. -For all the cases in the statement, the channel operands of receive operations -and the channel and right-hand-side expressions of send statements are -evaluated exactly once, in source order, upon entering the "select" statement. -The result is a set of channels to receive from or send to, -and the corresponding values to send. -Any side effects in that evaluation will occur irrespective of which (if any) -communication operation is selected to proceed. -Expressions on the left-hand side of a RecvStmt with a short variable declaration -or assignment are not yet evaluated. -
  2. - -
  3. -If one or more of the communications can proceed, -a single one that can proceed is chosen via a uniform pseudo-random selection. -Otherwise, if there is a default case, that case is chosen. -If there is no default case, the "select" statement blocks until -at least one of the communications can proceed. -
  4. - -
  5. -Unless the selected case is the default case, the respective communication -operation is executed. -
  6. - -
  7. -If the selected case is a RecvStmt with a short variable declaration or -an assignment, the left-hand side expressions are evaluated and the -received value (or values) are assigned. -
  8. - -
  9. -The statement list of the selected case is executed. -
  10. -
- -

-Since communication on nil channels can never proceed, -a select with only nil channels and no default case blocks forever. -

- -
-var a []int
-var c, c1, c2, c3, c4 chan int
-var i1, i2 int
-select {
-case i1 = <-c1:
-	print("received ", i1, " from c1\n")
-case c2 <- i2:
-	print("sent ", i2, " to c2\n")
-case i3, ok := (<-c3):  // same as: i3, ok := <-c3
-	if ok {
-		print("received ", i3, " from c3\n")
-	} else {
-		print("c3 is closed\n")
-	}
-case a[f()] = <-c4:
-	// same as:
-	// case t := <-c4
-	//	a[f()] = t
-default:
-	print("no communication\n")
-}
-
-for {  // send random sequence of bits to c
-	select {
-	case c <- 0:  // note: no statement, no fallthrough, no folding of cases
-	case c <- 1:
-	}
-}
-
-select {}  // block forever
-
- - -

Return statements

- -

-A "return" statement in a function F terminates the execution -of F, and optionally provides one or more result values. -Any functions deferred by F -are executed before F returns to its caller. -

- -
-ReturnStmt = "return" [ ExpressionList ] .
-
- -

-In a function without a result type, a "return" statement must not -specify any result values. -

-
-func noResult() {
-	return
-}
-
- -

-There are three ways to return values from a function with a result -type: -

- -
    -
  1. The return value or values may be explicitly listed - in the "return" statement. Each expression must be single-valued - and assignable - to the corresponding element of the function's result type. -
    -func simpleF() int {
    -	return 2
    -}
    -
    -func complexF1() (re float64, im float64) {
    -	return -7.0, -4.0
    -}
    -
    -
  2. -
  3. The expression list in the "return" statement may be a single - call to a multi-valued function. The effect is as if each value - returned from that function were assigned to a temporary - variable with the type of the respective value, followed by a - "return" statement listing these variables, at which point the - rules of the previous case apply. -
    -func complexF2() (re float64, im float64) {
    -	return complexF1()
    -}
    -
    -
  4. -
  5. The expression list may be empty if the function's result - type specifies names for its result parameters. - The result parameters act as ordinary local variables - and the function may assign values to them as necessary. - The "return" statement returns the values of these variables. -
    -func complexF3() (re float64, im float64) {
    -	re = 7.0
    -	im = 4.0
    -	return
    -}
    -
    -func (devnull) Write(p []byte) (n int, _ error) {
    -	n = len(p)
    -	return
    -}
    -
    -
  6. -
- -

-Regardless of how they are declared, all the result values are initialized to -the zero values for their type upon entry to the -function. A "return" statement that specifies results sets the result parameters before -any deferred functions are executed. -

- -

-Implementation restriction: A compiler may disallow an empty expression list -in a "return" statement if a different entity (constant, type, or variable) -with the same name as a result parameter is in -scope at the place of the return. -

- -
-func f(n int) (res int, err error) {
-	if _, err := f(n-1); err != nil {
-		return  // invalid return statement: err is shadowed
-	}
-	return
-}
-
- -

Break statements

- -

-A "break" statement terminates execution of the innermost -"for", -"switch", or -"select" statement -within the same function. -

- -
-BreakStmt = "break" [ Label ] .
-
- -

-If there is a label, it must be that of an enclosing -"for", "switch", or "select" statement, -and that is the one whose execution terminates. -

- -
-OuterLoop:
-	for i = 0; i < n; i++ {
-		for j = 0; j < m; j++ {
-			switch a[i][j] {
-			case nil:
-				state = Error
-				break OuterLoop
-			case item:
-				state = Found
-				break OuterLoop
-			}
-		}
-	}
-
- -

Continue statements

- -

-A "continue" statement begins the next iteration of the -innermost "for" loop at its post statement. -The "for" loop must be within the same function. -

- -
-ContinueStmt = "continue" [ Label ] .
-
- -

-If there is a label, it must be that of an enclosing -"for" statement, and that is the one whose execution -advances. -

- -
-RowLoop:
-	for y, row := range rows {
-		for x, data := range row {
-			if data == endOfRow {
-				continue RowLoop
-			}
-			row[x] = data + bias(x, y)
-		}
-	}
-
- -

Goto statements

- -

-A "goto" statement transfers control to the statement with the corresponding label -within the same function. -

- -
-GotoStmt = "goto" Label .
-
- -
-goto Error
-
- -

-Executing the "goto" statement must not cause any variables to come into -scope that were not already in scope at the point of the goto. -For instance, this example: -

- -
-	goto L  // BAD
-	v := 3
-L:
-
- -

-is erroneous because the jump to label L skips -the creation of v. -

- -

-A "goto" statement outside a block cannot jump to a label inside that block. -For instance, this example: -

- -
-if n%2 == 1 {
-	goto L1
-}
-for n > 0 {
-	f()
-	n--
-L1:
-	f()
-	n--
-}
-
- -

-is erroneous because the label L1 is inside -the "for" statement's block but the goto is not. -

- -

Fallthrough statements

- -

-A "fallthrough" statement transfers control to the first statement of the -next case clause in an expression "switch" statement. -It may be used only as the final non-empty statement in such a clause. -

- -
-FallthroughStmt = "fallthrough" .
-
- - -

Defer statements

- -

-A "defer" statement invokes a function whose execution is deferred -to the moment the surrounding function returns, either because the -surrounding function executed a return statement, -reached the end of its function body, -or because the corresponding goroutine is panicking. -

- -
-DeferStmt = "defer" Expression .
-
- -

-The expression must be a function or method call; it cannot be parenthesized. -Calls of built-in functions are restricted as for -expression statements. -

- -

-Each time a "defer" statement -executes, the function value and parameters to the call are -evaluated as usual -and saved anew but the actual function is not invoked. -Instead, deferred functions are invoked immediately before -the surrounding function returns, in the reverse order -they were deferred. That is, if the surrounding function -returns through an explicit return statement, -deferred functions are executed after any result parameters are set -by that return statement but before the function returns to its caller. -If a deferred function value evaluates -to nil, execution panics -when the function is invoked, not when the "defer" statement is executed. -

- -

-For instance, if the deferred function is -a function literal and the surrounding -function has named result parameters that -are in scope within the literal, the deferred function may access and modify -the result parameters before they are returned. -If the deferred function has any return values, they are discarded when -the function completes. -(See also the section on handling panics.) -

- -
-lock(l)
-defer unlock(l)  // unlocking happens before surrounding function returns
-
-// prints 3 2 1 0 before surrounding function returns
-for i := 0; i <= 3; i++ {
-	defer fmt.Print(i)
-}
-
-// f returns 42
-func f() (result int) {
-	defer func() {
-		// result is accessed after it was set to 6 by the return statement
-		result *= 7
-	}()
-	return 6
-}
-
- -

Built-in functions

- -

-Built-in functions are -predeclared. -They are called like any other function but some of them -accept a type instead of an expression as the first argument. -

- -

-The built-in functions do not have standard Go types, -so they can only appear in call expressions; -they cannot be used as function values. -

- -

Close

- -

-For a channel c, the built-in function close(c) -records that no more values will be sent on the channel. -It is an error if c is a receive-only channel. -Sending to or closing a closed channel causes a run-time panic. -Closing the nil channel also causes a run-time panic. -After calling close, and after any previously -sent values have been received, receive operations will return -the zero value for the channel's type without blocking. -The multi-valued receive operation -returns a received value along with an indication of whether the channel is closed. -

- - -

Length and capacity

- -

-The built-in functions len and cap take arguments -of various types and return a result of type int. -The implementation guarantees that the result always fits into an int. -

- -
-Call      Argument type    Result
-
-len(s)    string type      string length in bytes
-          [n]T, *[n]T      array length (== n)
-          []T              slice length
-          map[K]T          map length (number of defined keys)
-          chan T           number of elements queued in channel buffer
-
-cap(s)    [n]T, *[n]T      array length (== n)
-          []T              slice capacity
-          chan T           channel buffer capacity
-
- -

-The capacity of a slice is the number of elements for which there is -space allocated in the underlying array. -At any time the following relationship holds: -

- -
-0 <= len(s) <= cap(s)
-
- -

-The length of a nil slice, map or channel is 0. -The capacity of a nil slice or channel is 0. -

- -

-The expression len(s) is constant if -s is a string constant. The expressions len(s) and -cap(s) are constants if the type of s is an array -or pointer to an array and the expression s does not contain -channel receives or (non-constant) -function calls; in this case s is not evaluated. -Otherwise, invocations of len and cap are not -constant and s is evaluated. -

- -
-const (
-	c1 = imag(2i)                    // imag(2i) = 2.0 is a constant
-	c2 = len([10]float64{2})         // [10]float64{2} contains no function calls
-	c3 = len([10]float64{c1})        // [10]float64{c1} contains no function calls
-	c4 = len([10]float64{imag(2i)})  // imag(2i) is a constant and no function call is issued
-	c5 = len([10]float64{imag(z)})   // invalid: imag(z) is a (non-constant) function call
-)
-var z complex128
-
- -

Allocation

- -

-The built-in function new takes a type T, -allocates storage for a variable of that type -at run time, and returns a value of type *T -pointing to it. -The variable is initialized as described in the section on -initial values. -

- -
-new(T)
-
- -

-For instance -

- -
-type S struct { a int; b float64 }
-new(S)
-
- -

-allocates storage for a variable of type S, -initializes it (a=0, b=0.0), -and returns a value of type *S containing the address -of the location. -

- -

Making slices, maps and channels

- -

-The built-in function make takes a type T, -which must be a slice, map or channel type, -optionally followed by a type-specific list of expressions. -It returns a value of type T (not *T). -The memory is initialized as described in the section on -initial values. -

- -
-Call             Type T     Result
-
-make(T, n)       slice      slice of type T with length n and capacity n
-make(T, n, m)    slice      slice of type T with length n and capacity m
-
-make(T)          map        map of type T
-make(T, n)       map        map of type T with initial space for approximately n elements
-
-make(T)          channel    unbuffered channel of type T
-make(T, n)       channel    buffered channel of type T, buffer size n
-
- - -

-Each of the size arguments n and m must be of integer type -or an untyped constant. -A constant size argument must be non-negative and representable -by a value of type int; if it is an untyped constant it is given type int. -If both n and m are provided and are constant, then -n must be no larger than m. -If n is negative or larger than m at run time, -a run-time panic occurs. -

- -
-s := make([]int, 10, 100)       // slice with len(s) == 10, cap(s) == 100
-s := make([]int, 1e3)           // slice with len(s) == cap(s) == 1000
-s := make([]int, 1<<63)         // illegal: len(s) is not representable by a value of type int
-s := make([]int, 10, 0)         // illegal: len(s) > cap(s)
-c := make(chan int, 10)         // channel with a buffer size of 10
-m := make(map[string]int, 100)  // map with initial space for approximately 100 elements
-
- -

-Calling make with a map type and size hint n will -create a map with initial space to hold n map elements. -The precise behavior is implementation-dependent. -

- - -

Appending to and copying slices

- -

-The built-in functions append and copy assist in -common slice operations. -For both functions, the result is independent of whether the memory referenced -by the arguments overlaps. -

- -

-The variadic function append -appends zero or more values x -to s of type S, which must be a slice type, and -returns the resulting slice, also of type S. -The values x are passed to a parameter of type ...T -where T is the element type of -S and the respective -parameter passing rules apply. -As a special case, append also accepts a first argument -assignable to type []byte with a second argument of -string type followed by .... This form appends the -bytes of the string. -

- -
-append(s S, x ...T) S  // T is the element type of S
-
- -

-If the capacity of s is not large enough to fit the additional -values, append allocates a new, sufficiently large underlying -array that fits both the existing slice elements and the additional values. -Otherwise, append re-uses the underlying array. -

- -
-s0 := []int{0, 0}
-s1 := append(s0, 2)                // append a single element     s1 == []int{0, 0, 2}
-s2 := append(s1, 3, 5, 7)          // append multiple elements    s2 == []int{0, 0, 2, 3, 5, 7}
-s3 := append(s2, s0...)            // append a slice              s3 == []int{0, 0, 2, 3, 5, 7, 0, 0}
-s4 := append(s3[3:6], s3[2:]...)   // append overlapping slice    s4 == []int{3, 5, 7, 2, 3, 5, 7, 0, 0}
-
-var t []interface{}
-t = append(t, 42, 3.1415, "foo")   //                             t == []interface{}{42, 3.1415, "foo"}
-
-var b []byte
-b = append(b, "bar"...)            // append string contents      b == []byte{'b', 'a', 'r' }
-
- -

-The function copy copies slice elements from -a source src to a destination dst and returns the -number of elements copied. -Both arguments must have identical element type T and must be -assignable to a slice of type []T. -The number of elements copied is the minimum of -len(src) and len(dst). -As a special case, copy also accepts a destination argument assignable -to type []byte with a source argument of a string type. -This form copies the bytes from the string into the byte slice. -

- -
-copy(dst, src []T) int
-copy(dst []byte, src string) int
-
- -

-Examples: -

- -
-var a = [...]int{0, 1, 2, 3, 4, 5, 6, 7}
-var s = make([]int, 6)
-var b = make([]byte, 5)
-n1 := copy(s, a[0:])            // n1 == 6, s == []int{0, 1, 2, 3, 4, 5}
-n2 := copy(s, s[2:])            // n2 == 4, s == []int{2, 3, 4, 5, 4, 5}
-n3 := copy(b, "Hello, World!")  // n3 == 5, b == []byte("Hello")
-
- - -

Deletion of map elements

- -

-The built-in function delete removes the element with key -k from a map m. The -type of k must be assignable -to the key type of m. -

- -
-delete(m, k)  // remove element m[k] from map m
-
- -

-If the map m is nil or the element m[k] -does not exist, delete is a no-op. -

- - -

Manipulating complex numbers

- -

-Three functions assemble and disassemble complex numbers. -The built-in function complex constructs a complex -value from a floating-point real and imaginary part, while -real and imag -extract the real and imaginary parts of a complex value. -

- -
-complex(realPart, imaginaryPart floatT) complexT
-real(complexT) floatT
-imag(complexT) floatT
-
- -

-The type of the arguments and return value correspond. -For complex, the two arguments must be of the same -floating-point type and the return type is the complex type -with the corresponding floating-point constituents: -complex64 for float32 arguments, and -complex128 for float64 arguments. -If one of the arguments evaluates to an untyped constant, it is first implicitly -converted to the type of the other argument. -If both arguments evaluate to untyped constants, they must be non-complex -numbers or their imaginary parts must be zero, and the return value of -the function is an untyped complex constant. -

- -

-For real and imag, the argument must be -of complex type, and the return type is the corresponding floating-point -type: float32 for a complex64 argument, and -float64 for a complex128 argument. -If the argument evaluates to an untyped constant, it must be a number, -and the return value of the function is an untyped floating-point constant. -

- -

-The real and imag functions together form the inverse of -complex, so for a value z of a complex type Z, -z == Z(complex(real(z), imag(z))). -

- -

-If the operands of these functions are all constants, the return -value is a constant. -

- -
-var a = complex(2, -2)             // complex128
-const b = complex(1.0, -1.4)       // untyped complex constant 1 - 1.4i
-x := float32(math.Cos(math.Pi/2))  // float32
-var c64 = complex(5, -x)           // complex64
-var s int = complex(1, 0)          // untyped complex constant 1 + 0i can be converted to int
-_ = complex(1, 2<<s)               // illegal: 2 assumes floating-point type, cannot shift
-var rl = real(c64)                 // float32
-var im = imag(a)                   // float64
-const c = imag(b)                  // untyped constant -1.4
-_ = imag(3 << s)                   // illegal: 3 assumes complex type, cannot shift
-
- -

Handling panics

- -

Two built-in functions, panic and recover, -assist in reporting and handling run-time panics -and program-defined error conditions. -

- -
-func panic(interface{})
-func recover() interface{}
-
- -

-While executing a function F, -an explicit call to panic or a run-time panic -terminates the execution of F. -Any functions deferred by F -are then executed as usual. -Next, any deferred functions run by F's caller are run, -and so on up to any deferred by the top-level function in the executing goroutine. -At that point, the program is terminated and the error -condition is reported, including the value of the argument to panic. -This termination sequence is called panicking. -

- -
-panic(42)
-panic("unreachable")
-panic(Error("cannot parse"))
-
- -

-The recover function allows a program to manage behavior -of a panicking goroutine. -Suppose a function G defers a function D that calls -recover and a panic occurs in a function on the same goroutine in which G -is executing. -When the running of deferred functions reaches D, -the return value of D's call to recover will be the value passed to the call of panic. -If D returns normally, without starting a new -panic, the panicking sequence stops. In that case, -the state of functions called between G and the call to panic -is discarded, and normal execution resumes. -Any functions deferred by G before D are then run and G's -execution terminates by returning to its caller. -

- -

-The return value of recover is nil if any of the following conditions holds: -

- - -

-The protect function in the example below invokes -the function argument g and protects callers from -run-time panics raised by g. -

- -
-func protect(g func()) {
-	defer func() {
-		log.Println("done")  // Println executes normally even if there is a panic
-		if x := recover(); x != nil {
-			log.Printf("run time panic: %v", x)
-		}
-	}()
-	log.Println("start")
-	g()
-}
-
- - -

Bootstrapping

- -

-Current implementations provide several built-in functions useful during -bootstrapping. These functions are documented for completeness but are not -guaranteed to stay in the language. They do not return a result. -

- -
-Function   Behavior
-
-print      prints all arguments; formatting of arguments is implementation-specific
-println    like print but prints spaces between arguments and a newline at the end
-
- -

-Implementation restriction: print and println need not -accept arbitrary argument types, but printing of boolean, numeric, and string -types must be supported. -

- -

Packages

- -

-Go programs are constructed by linking together packages. -A package in turn is constructed from one or more source files -that together declare constants, types, variables and functions -belonging to the package and which are accessible in all files -of the same package. Those elements may be -exported and used in another package. -

- -

Source file organization

- -

-Each source file consists of a package clause defining the package -to which it belongs, followed by a possibly empty set of import -declarations that declare packages whose contents it wishes to use, -followed by a possibly empty set of declarations of functions, -types, variables, and constants. -

- -
-SourceFile       = PackageClause ";" { ImportDecl ";" } { TopLevelDecl ";" } .
-
- -

Package clause

- -

-A package clause begins each source file and defines the package -to which the file belongs. -

- -
-PackageClause  = "package" PackageName .
-PackageName    = identifier .
-
- -

-The PackageName must not be the blank identifier. -

- -
-package math
-
- -

-A set of files sharing the same PackageName form the implementation of a package. -An implementation may require that all source files for a package inhabit the same directory. -

- -

Import declarations

- -

-An import declaration states that the source file containing the declaration -depends on functionality of the imported package -(§Program initialization and execution) -and enables access to exported identifiers -of that package. -The import names an identifier (PackageName) to be used for access and an ImportPath -that specifies the package to be imported. -

- -
-ImportDecl       = "import" ( ImportSpec | "(" { ImportSpec ";" } ")" ) .
-ImportSpec       = [ "." | PackageName ] ImportPath .
-ImportPath       = string_lit .
-
- -

-The PackageName is used in qualified identifiers -to access exported identifiers of the package within the importing source file. -It is declared in the file block. -If the PackageName is omitted, it defaults to the identifier specified in the -package clause of the imported package. -If an explicit period (.) appears instead of a name, all the -package's exported identifiers declared in that package's -package block will be declared in the importing source -file's file block and must be accessed without a qualifier. -

- -

-The interpretation of the ImportPath is implementation-dependent but -it is typically a substring of the full file name of the compiled -package and may be relative to a repository of installed packages. -

- -

-Implementation restriction: A compiler may restrict ImportPaths to -non-empty strings using only characters belonging to -Unicode's -L, M, N, P, and S general categories (the Graphic characters without -spaces) and may also exclude the characters -!"#$%&'()*,:;<=>?[\]^`{|} -and the Unicode replacement character U+FFFD. -

- -

-Assume we have compiled a package containing the package clause -package math, which exports function Sin, and -installed the compiled package in the file identified by -"lib/math". -This table illustrates how Sin is accessed in files -that import the package after the -various types of import declaration. -

- -
-Import declaration          Local name of Sin
-
-import   "lib/math"         math.Sin
-import m "lib/math"         m.Sin
-import . "lib/math"         Sin
-
- -

-An import declaration declares a dependency relation between -the importing and imported package. -It is illegal for a package to import itself, directly or indirectly, -or to directly import a package without -referring to any of its exported identifiers. To import a package solely for -its side-effects (initialization), use the blank -identifier as explicit package name: -

- -
-import _ "lib/math"
-
- - -

An example package

- -

-Here is a complete Go package that implements a concurrent prime sieve. -

- -
-package main
-
-import "fmt"
-
-// Send the sequence 2, 3, 4, … to channel 'ch'.
-func generate(ch chan<- int) {
-	for i := 2; ; i++ {
-		ch <- i  // Send 'i' to channel 'ch'.
-	}
-}
-
-// Copy the values from channel 'src' to channel 'dst',
-// removing those divisible by 'prime'.
-func filter(src <-chan int, dst chan<- int, prime int) {
-	for i := range src {  // Loop over values received from 'src'.
-		if i%prime != 0 {
-			dst <- i  // Send 'i' to channel 'dst'.
-		}
-	}
-}
-
-// The prime sieve: Daisy-chain filter processes together.
-func sieve() {
-	ch := make(chan int)  // Create a new channel.
-	go generate(ch)       // Start generate() as a subprocess.
-	for {
-		prime := <-ch
-		fmt.Print(prime, "\n")
-		ch1 := make(chan int)
-		go filter(ch, ch1, prime)
-		ch = ch1
-	}
-}
-
-func main() {
-	sieve()
-}
-
- -

Program initialization and execution

- -

The zero value

-

-When storage is allocated for a variable, -either through a declaration or a call of new, or when -a new value is created, either through a composite literal or a call -of make, -and no explicit initialization is provided, the variable or value is -given a default value. Each element of such a variable or value is -set to the zero value for its type: false for booleans, -0 for numeric types, "" -for strings, and nil for pointers, functions, interfaces, slices, channels, and maps. -This initialization is done recursively, so for instance each element of an -array of structs will have its fields zeroed if no value is specified. -

-

-These two simple declarations are equivalent: -

- -
-var i int
-var i int = 0
-
- -

-After -

- -
-type T struct { i int; f float64; next *T }
-t := new(T)
-
- -

-the following holds: -

- -
-t.i == 0
-t.f == 0.0
-t.next == nil
-
- -

-The same would also be true after -

- -
-var t T
-
- -

Package initialization

- -

-Within a package, package-level variable initialization proceeds stepwise, -with each step selecting the variable earliest in declaration order -which has no dependencies on uninitialized variables. -

- -

-More precisely, a package-level variable is considered ready for -initialization if it is not yet initialized and either has -no initialization expression or -its initialization expression has no dependencies on uninitialized variables. -Initialization proceeds by repeatedly initializing the next package-level -variable that is earliest in declaration order and ready for initialization, -until there are no variables ready for initialization. -

- -

-If any variables are still uninitialized when this -process ends, those variables are part of one or more initialization cycles, -and the program is not valid. -

- -

-Multiple variables on the left-hand side of a variable declaration initialized -by single (multi-valued) expression on the right-hand side are initialized -together: If any of the variables on the left-hand side is initialized, all -those variables are initialized in the same step. -

- -
-var x = a
-var a, b = f() // a and b are initialized together, before x is initialized
-
- -

-For the purpose of package initialization, blank -variables are treated like any other variables in declarations. -

- -

-The declaration order of variables declared in multiple files is determined -by the order in which the files are presented to the compiler: Variables -declared in the first file are declared before any of the variables declared -in the second file, and so on. -

- -

-Dependency analysis does not rely on the actual values of the -variables, only on lexical references to them in the source, -analyzed transitively. For instance, if a variable x's -initialization expression refers to a function whose body refers to -variable y then x depends on y. -Specifically: -

- - - -

-For example, given the declarations -

- -
-var (
-	a = c + b  // == 9
-	b = f()    // == 4
-	c = f()    // == 5
-	d = 3      // == 5 after initialization has finished
-)
-
-func f() int {
-	d++
-	return d
-}
-
- -

-the initialization order is d, b, c, a. -Note that the order of subexpressions in initialization expressions is irrelevant: -a = c + b and a = b + c result in the same initialization -order in this example. -

- -

-Dependency analysis is performed per package; only references referring -to variables, functions, and (non-interface) methods declared in the current -package are considered. If other, hidden, data dependencies exists between -variables, the initialization order between those variables is unspecified. -

- -

-For instance, given the declarations -

- -
-var x = I(T{}).ab()   // x has an undetected, hidden dependency on a and b
-var _ = sideEffect()  // unrelated to x, a, or b
-var a = b
-var b = 42
-
-type I interface      { ab() []int }
-type T struct{}
-func (T) ab() []int   { return []int{a, b} }
-
- -

-the variable a will be initialized after b but -whether x is initialized before b, between -b and a, or after a, and -thus also the moment at which sideEffect() is called (before -or after x is initialized) is not specified. -

- -

-Variables may also be initialized using functions named init -declared in the package block, with no arguments and no result parameters. -

- -
-func init() { … }
-
- -

-Multiple such functions may be defined per package, even within a single -source file. In the package block, the init identifier can -be used only to declare init functions, yet the identifier -itself is not declared. Thus -init functions cannot be referred to from anywhere -in a program. -

- -

-A package with no imports is initialized by assigning initial values -to all its package-level variables followed by calling all init -functions in the order they appear in the source, possibly in multiple files, -as presented to the compiler. -If a package has imports, the imported packages are initialized -before initializing the package itself. If multiple packages import -a package, the imported package will be initialized only once. -The importing of packages, by construction, guarantees that there -can be no cyclic initialization dependencies. -

- -

-Package initialization—variable initialization and the invocation of -init functions—happens in a single goroutine, -sequentially, one package at a time. -An init function may launch other goroutines, which can run -concurrently with the initialization code. However, initialization -always sequences -the init functions: it will not invoke the next one -until the previous one has returned. -

- -

-To ensure reproducible initialization behavior, build systems are encouraged -to present multiple files belonging to the same package in lexical file name -order to a compiler. -

- - -

Program execution

-

-A complete program is created by linking a single, unimported package -called the main package with all the packages it imports, transitively. -The main package must -have package name main and -declare a function main that takes no -arguments and returns no value. -

- -
-func main() { … }
-
- -

-Program execution begins by initializing the main package and then -invoking the function main. -When that function invocation returns, the program exits. -It does not wait for other (non-main) goroutines to complete. -

- -

Errors

- -

-The predeclared type error is defined as -

- -
-type error interface {
-	Error() string
-}
-
- -

-It is the conventional interface for representing an error condition, -with the nil value representing no error. -For instance, a function to read data from a file might be defined: -

- -
-func Read(f *File, b []byte) (n int, err error)
-
- -

Run-time panics

- -

-Execution errors such as attempting to index an array out -of bounds trigger a run-time panic equivalent to a call of -the built-in function panic -with a value of the implementation-defined interface type runtime.Error. -That type satisfies the predeclared interface type -error. -The exact error values that -represent distinct run-time error conditions are unspecified. -

- -
-package runtime
-
-type Error interface {
-	error
-	// and perhaps other methods
-}
-
- -

System considerations

- -

Package unsafe

- -

-The built-in package unsafe, known to the compiler -and accessible through the import path "unsafe", -provides facilities for low-level programming including operations -that violate the type system. A package using unsafe -must be vetted manually for type safety and may not be portable. -The package provides the following interface: -

- -
-package unsafe
-
-type ArbitraryType int  // shorthand for an arbitrary Go type; it is not a real type
-type Pointer *ArbitraryType
-
-func Alignof(variable ArbitraryType) uintptr
-func Offsetof(selector ArbitraryType) uintptr
-func Sizeof(variable ArbitraryType) uintptr
-
-type IntegerType int  // shorthand for an integer type; it is not a real type
-func Add(ptr Pointer, len IntegerType) Pointer
-func Slice(ptr *ArbitraryType, len IntegerType) []ArbitraryType
-
- -

-A Pointer is a pointer type but a Pointer -value may not be dereferenced. -Any pointer or value of underlying type uintptr can be converted to -a type of underlying type Pointer and vice versa. -The effect of converting between Pointer and uintptr is implementation-defined. -

- -
-var f float64
-bits = *(*uint64)(unsafe.Pointer(&f))
-
-type ptr unsafe.Pointer
-bits = *(*uint64)(ptr(&f))
-
-var p ptr = nil
-
- -

-The functions Alignof and Sizeof take an expression x -of any type and return the alignment or size, respectively, of a hypothetical variable v -as if v was declared via var v = x. -

-

-The function Offsetof takes a (possibly parenthesized) selector -s.f, denoting a field f of the struct denoted by s -or *s, and returns the field offset in bytes relative to the struct's address. -If f is an embedded field, it must be reachable -without pointer indirections through fields of the struct. -For a struct s with field f: -

- -
-uintptr(unsafe.Pointer(&s)) + unsafe.Offsetof(s.f) == uintptr(unsafe.Pointer(&s.f))
-
- -

-Computer architectures may require memory addresses to be aligned; -that is, for addresses of a variable to be a multiple of a factor, -the variable's type's alignment. The function Alignof -takes an expression denoting a variable of any type and returns the -alignment of the (type of the) variable in bytes. For a variable -x: -

- -
-uintptr(unsafe.Pointer(&x)) % unsafe.Alignof(x) == 0
-
- -

-Calls to Alignof, Offsetof, and -Sizeof are compile-time constant expressions of type uintptr. -

- -

-The function Add adds len to ptr -and returns the updated pointer unsafe.Pointer(uintptr(ptr) + uintptr(len)). -The len argument must be of integer type or an untyped constant. -A constant len argument must be representable by a value of type int; -if it is an untyped constant it is given type int. -The rules for valid uses of Pointer still apply. -

- -

-The function Slice returns a slice whose underlying array starts at ptr -and whose length and capacity are len. -Slice(ptr, len) is equivalent to -

- -
-(*[len]ArbitraryType)(unsafe.Pointer(ptr))[:]
-
- -

-except that, as a special case, if ptr -is nil and len is zero, -Slice returns nil. -

- -

-The len argument must be of integer type or an untyped constant. -A constant len argument must be non-negative and representable by a value of type int; -if it is an untyped constant it is given type int. -At run time, if len is negative, -or if ptr is nil and len is not zero, -a run-time panic occurs. -

- -

Size and alignment guarantees

- -

-For the numeric types, the following sizes are guaranteed: -

- -
-type                                 size in bytes
-
-byte, uint8, int8                     1
-uint16, int16                         2
-uint32, int32, float32                4
-uint64, int64, float64, complex64     8
-complex128                           16
-
- -

-The following minimal alignment properties are guaranteed: -

-
    -
  1. For a variable x of any type: unsafe.Alignof(x) is at least 1. -
  2. - -
  3. For a variable x of struct type: unsafe.Alignof(x) is the largest of - all the values unsafe.Alignof(x.f) for each field f of x, but at least 1. -
  4. - -
  5. For a variable x of array type: unsafe.Alignof(x) is the same as - the alignment of a variable of the array's element type. -
  6. -
- -

-A struct or array type has size zero if it contains no fields (or elements, respectively) that have a size greater than zero. Two distinct zero-size variables may have the same address in memory. -

diff --git a/doc/go_mem.html b/doc/go_mem.html index c0b81d3fac900b..8db7d65e7342a8 100644 --- a/doc/go_mem.html +++ b/doc/go_mem.html @@ -82,7 +82,7 @@

Memory Model

The following formal definition of Go's memory model closely follows the approach presented by Hans-J. Boehm and Sarita V. Adve in -“Foundations of the C++ Concurrency Memory Model”, +“Foundations of the C++ Concurrency Memory Model”, published in PLDI 2008. The definition of data-race-free programs and the guarantee of sequential consistency for race-free programs are equivalent to the ones in that work. @@ -231,7 +231,7 @@

Implementation Restrictions for Programs Containing Data R

A read of an array, struct, or complex number -may by implemented as a read of each individual sub-value +may be implemented as a read of each individual sub-value (array element, struct field, or real/imaginary component), in any order. Similarly, a write of an array, struct, or complex number @@ -453,7 +453,7 @@

Channel communication

-The kth receive on a channel with capacity C is synchronized before the completion of the k+Cth send from that channel completes. +The kth receive from a channel with capacity C is synchronized before the completion of the k+Cth send on that channel.

diff --git a/doc/go_spec.html b/doc/go_spec.html index b5b7f15bbd369b..d6c7962a961b88 100644 --- a/doc/go_spec.html +++ b/doc/go_spec.html @@ -1,6 +1,6 @@ @@ -8,8 +8,6 @@

Introduction

This is the reference manual for the Go programming language. -The pre-Go1.18 version, without generics, can be found -here. For more information and other documents, see go.dev.

@@ -798,7 +796,6 @@

Variables

zero value for its type.

-

Types

@@ -810,12 +807,12 @@

Types

-Type      = TypeName [ TypeArgs ] | TypeLit | "(" Type ")" .
-TypeName  = identifier | QualifiedIdent .
-TypeArgs  = "[" TypeList [ "," ] "]" .
-TypeList  = Type { "," Type } .
-TypeLit   = ArrayType | StructType | PointerType | FunctionType | InterfaceType |
-            SliceType | MapType | ChannelType .
+Type     = TypeName [ TypeArgs ] | TypeLit | "(" Type ")" .
+TypeName = identifier | QualifiedIdent .
+TypeArgs = "[" TypeList [ "," ] "]" .
+TypeList = Type { "," Type } .
+TypeLit  = ArrayType | StructType | PointerType | FunctionType | InterfaceType |
+           SliceType | MapType | ChannelType .
 

@@ -1086,7 +1083,7 @@

Struct types

An embedded field must be specified as a type name T or as a pointer to a non-interface type name *T, and T itself may not be -a pointer type. The unqualified type name acts as the field name. +a pointer type or type parameter. The unqualified type name acts as the field name.

@@ -1127,7 +1124,7 @@ 

Struct types

-Given a struct type S and a named type +Given a struct type S and a type name T, promoted methods are included in the method set of the struct as follows:

+ +

Go 1.24

+ +

Type unification rules

@@ -8727,9 +8833,9 @@

Type unification rules

  • Exactly one type is an unbound - type parameter with a core type, - and that core type unifies with the other type per the - unification rules for A + type parameter, and all the types in its type set unify with + the other type + per the unification rules for A (loose unification at the top level and exact unification for element types).
  • diff --git a/doc/godebug.md b/doc/godebug.md index b3a43664c42cd4..c12ce5311d90d1 100644 --- a/doc/godebug.md +++ b/doc/godebug.md @@ -34,6 +34,7 @@ For example, if a Go program is running in an environment that contains then that Go program will disable the use of HTTP/2 by default in both the HTTP client and the HTTP server. +Unrecognized settings in the `GODEBUG` environment variable are ignored. It is also possible to set the default `GODEBUG` for a given program (discussed below). @@ -108,7 +109,9 @@ Only the work module's `go.mod` is consulted for `godebug` directives. Any directives in required dependency modules are ignored. It is an error to list a `godebug` with an unrecognized setting. (Toolchains older than Go 1.23 reject all `godebug` lines, since they do not -understand `godebug` at all.) +understand `godebug` at all.) When a workspace is in use, `godebug` +directives in `go.mod` files are ignored, and `go.work` will be consulted +for `godebug` directives instead. The defaults from the `go` and `godebug` lines apply to all main packages that are built. For more fine-grained control, @@ -150,6 +153,132 @@ for example, see the [runtime documentation](/pkg/runtime#hdr-Environment_Variables) and the [go command documentation](/cmd/go#hdr-Build_and_test_caching). +### Go 1.26 + +Go 1.26 added a new `httpcookiemaxnum` setting that controls the maximum number +of cookies that net/http will accept when parsing HTTP headers. If the number of +cookie in a header exceeds the number set in `httpcookiemaxnum`, cookie parsing +will fail early. The default value is `httpcookiemaxnum=3000`. Setting +`httpcookiemaxnum=0` will allow the cookie parsing to accept an indefinite +number of cookies. To avoid denial of service attacks, this setting and default +was backported to Go 1.25.2 and Go 1.24.8. + +### Go 1.25 + +Go 1.25 added a new `decoratemappings` setting that controls whether the Go +runtime annotates OS anonymous memory mappings with context about their +purpose. These annotations appear in /proc/self/maps and /proc/self/smaps as +"[anon: Go: ...]". This setting is only used on Linux. For Go 1.25, it defaults +to `decoratemappings=1`, enabling annotations. Using `decoratemappings=0` +reverts to the pre-Go 1.25 behavior. This setting is fixed at program startup +time, and can't be modified by changing the `GODEBUG` environment variable +after the program starts. + +Go 1.25 added a new `embedfollowsymlinks` setting that controls whether the +Go command will follow symlinks to regular files embedding files. +The default value `embedfollowsymlinks=0` does not allow following +symlinks. `embedfollowsymlinks=1` will allow following symlinks. + +Go 1.25 added a new `containermaxprocs` setting that controls whether the Go +runtime will consider cgroup CPU limits when setting the default GOMAXPROCS. +The default value `containermaxprocs=1` will use cgroup limits in addition to +the total logical CPU count and CPU affinity. `containermaxprocs=0` will +disable consideration of cgroup limits. This setting only affects Linux. + +Go 1.25 added a new `updatemaxprocs` setting that controls whether the Go +runtime will periodically update GOMAXPROCS for new CPU affinity or cgroup +limits. The default value `updatemaxprocs=1` will enable periodic updates. +`updatemaxprocs=0` will disable periodic updates. + +Go 1.25 disabled SHA-1 signature algorithms in TLS 1.2 according to RFC 9155. +The default can be reverted using the `tlssha1=1` setting. + +Go 1.25 switched to SHA-256 to fill in missing SubjectKeyId in +crypto/x509.CreateCertificate. The setting `x509sha256skid=0` reverts to SHA-1. + +Go 1.25 corrected the semantics of contention reports for runtime-internal locks, +and so removed the [`runtimecontentionstacks` setting](/pkg/runtime#hdr-Environment_Variables). + +Go 1.25 (starting with Go 1.25 RC 2) disabled build information stamping when +multiple VCS are detected due to concerns around VCS injection attacks. This +behavior and setting was backported to Go 1.24.5 and Go 1.23.11. This behavior +can be renabled with the setting `allowmultiplevcs=1`. + +### Go 1.24 + +Go 1.24 added a new `fips140` setting that controls whether the Go +Cryptographic Module operates in FIPS 140-3 mode. +The possible values are: +- "off": no special support for FIPS 140-3 mode. This is the default. +- "on": the Go Cryptographic Module operates in FIPS 140-3 mode. +- "only": like "on", but cryptographic algorithms not approved by + FIPS 140-3 return an error or panic. +For more information, see [FIPS 140-3 Compliance](/doc/security/fips140). +This setting is fixed at program startup time, and can't be modified +by changing the `GODEBUG` environment variable after the program starts. + +Go 1.24 changed the global [`math/rand.Seed`](/pkg/math/rand/#Seed) to be a +no-op. This behavior is controlled by the `randseednop` setting. +For Go 1.24 it defaults to `randseednop=1`. +Using `randseednop=0` reverts to the pre-Go 1.24 behavior. + +Go 1.24 added new values for the `multipathtcp` setting. +The possible values for `multipathtcp` are now: +- "0": disable MPTCP on dialers and listeners by default +- "1": enable MPTCP on dialers and listeners by default +- "2": enable MPTCP on listeners only by default +- "3": enable MPTCP on dialers only by default + +For Go 1.24, it now defaults to multipathtcp="2", thus +enabled by default on listeners. Using multipathtcp="0" reverts to the +pre-Go 1.24 behavior. + +Go 1.24 changed the behavior of `go test -json` to emit build errors as JSON +instead of text. +These new JSON events are distinguished by new `Action` values, +but can still cause problems with CI systems that aren't robust to these events. +This behavior can be controlled with the `gotestjsonbuildtext` setting. +Using `gotestjsonbuildtext=1` restores the 1.23 behavior. +This setting will be removed in a future release, Go 1.28 at the earliest. + +Go 1.24 changed [`crypto/rsa`](/pkg/crypto/rsa) to require RSA keys to be at +least 1024 bits. This behavior can be controlled with the `rsa1024min` setting. +Using `rsa1024min=0` restores the Go 1.23 behavior. + +Go 1.24 introduced a mechanism for enabling platform specific Data Independent +Timing (DIT) modes in the [`crypto/subtle`](/pkg/crypto/subtle) package. This +mode can be enabled for an entire program with the `dataindependenttiming` setting. +For Go 1.24 it defaults to `dataindependenttiming=0`. There is no change in default +behavior from Go 1.23 when `dataindependenttiming` is unset. +Using `dataindependenttiming=1` enables the DIT mode for the entire Go program. +When enabled, DIT will be enabled when calling into C from Go. When enabled, +calling into Go code from C will enable DIT, and disable it before returning to +C if it was not enabled when Go code was entered. +This currently only affects arm64 programs. For all other platforms it is a no-op. + +Go 1.24 removed the `x509sha1` setting. `crypto/x509` no longer supports verifying +signatures on certificates that use SHA-1 based signature algorithms. + +Go 1.24 changes the default value of the [`x509usepolicies` +setting.](/pkg/crypto/x509/#CreateCertificate) from `0` to `1`. When marshalling +certificates, policies are now taken from the +[`Certificate.Policies`](/pkg/crypto/x509/#Certificate.Policies) field rather +than the +[`Certificate.PolicyIdentifiers`](/pkg/crypto/x509/#Certificate.PolicyIdentifiers) +field by default. + +Go 1.24 enabled the post-quantum key exchange mechanism +X25519MLKEM768 by default. The default can be reverted using the +[`tlsmlkem` setting](/pkg/crypto/tls/#Config.CurvePreferences). +This can be useful when dealing with buggy TLS servers that do not handle large records correctly, +causing a timeout during the handshake (see [TLS post-quantum TL;DR fail](https://tldr.fail/)). +Go 1.24 also removed X25519Kyber768Draft00 and the Go 1.23 `tlskyber` setting. + +Go 1.24 made [`ParsePKCS1PrivateKey`](/pkg/crypto/x509/#ParsePKCS1PrivateKey) +use and validate the CRT parameters in the encoded private key. This behavior +can be controlled with the `x509rsacrt` setting. Using `x509rsacrt=0` restores +the Go 1.23 behavior. + ### Go 1.23 Go 1.23 changed the channels created by package time to be unbuffered @@ -179,6 +308,8 @@ Previous versions default to `winreadlinkvolume=0`. Go 1.23 enabled the experimental post-quantum key exchange mechanism X25519Kyber768Draft00 by default. The default can be reverted using the [`tlskyber` setting](/pkg/crypto/tls/#Config.CurvePreferences). +This can be useful when dealing with buggy TLS servers that do not handle large records correctly, +causing a timeout during the handshake (see [TLS post-quantum TL;DR fail](https://tldr.fail/)). Go 1.23 changed the behavior of [crypto/x509.ParseCertificate](/pkg/crypto/x509/#ParseCertificate) to reject @@ -253,7 +384,7 @@ In particular, a common default Linux kernel configuration can result in significant memory overheads, and Go 1.22 no longer works around this default. To work around this issue without adjusting kernel settings, transparent huge pages can be disabled for Go memory with the -[`disablethp` setting](/pkg/runtime#hdr-Environment_Variable). +[`disablethp` setting](/pkg/runtime#hdr-Environment_Variables). This behavior was backported to Go 1.21.1, but the setting is only available starting with Go 1.21.6. This setting may be removed in a future release, and users impacted by this issue @@ -265,7 +396,7 @@ Go 1.22 added contention on runtime-internal locks to the [`mutex` profile](/pkg/runtime/pprof#Profile). Contention on these locks is always reported at `runtime._LostContendedRuntimeLock`. Complete stack traces of runtime locks can be enabled with the [`runtimecontentionstacks` -setting](/pkg/runtime#hdr-Environment_Variable). These stack traces have +setting](/pkg/runtime#hdr-Environment_Variables). These stack traces have non-standard semantics, see setting documentation for details. Go 1.22 added a new [`crypto/x509.Certificate`](/pkg/crypto/x509/#Certificate) @@ -274,7 +405,7 @@ certificate policy OIDs with components larger than 31 bits. By default this field is only used during parsing, when it is populated with policy OIDs, but not used during marshaling. It can be used to marshal these larger OIDs, instead of the existing PolicyIdentifiers field, by using the -[`x509usepolicies` setting.](/pkg/crypto/x509/#CreateCertificate). +[`x509usepolicies` setting](/pkg/crypto/x509/#CreateCertificate). ### Go 1.21 @@ -342,7 +473,7 @@ There is no plan to remove this setting. Go 1.18 removed support for SHA1 in most X.509 certificates, controlled by the [`x509sha1` setting](/pkg/crypto/x509#InsecureAlgorithmError). -This setting will be removed in a future release, Go 1.22 at the earliest. +This setting was removed in Go 1.24. ### Go 1.10 diff --git a/doc/initial/1-intro.md b/doc/initial/1-intro.md index 8c9948ddf607ae..84ffee855a4095 100644 --- a/doc/initial/1-intro.md +++ b/doc/initial/1-intro.md @@ -1,9 +1,3 @@ - - diff --git a/doc/initial/6-stdlib/99-minor/0-heading.md b/doc/initial/6-stdlib/99-minor/0-heading.md index a98105e8ccba7f..e80ebc64c380fd 100644 --- a/doc/initial/6-stdlib/99-minor/0-heading.md +++ b/doc/initial/6-stdlib/99-minor/0-heading.md @@ -1,3 +1,10 @@ ### Minor changes to the library {#minor_library_changes} +#### go/types +The `Var.Kind` method returns an enumeration of type `VarKind` that +classifies the variable (package-level, local, receiver, parameter, +result, or struct field). See issue #70250. + +Callers of `NewVar` or `NewParam` are encouraged to call `Var.SetKind` +to ensure that this attribute is set correctly in all cases. diff --git a/doc/next/1-intro.md b/doc/next/1-intro.md index 3f6dc9f5d898b9..3cd0d66b5ad4e3 100644 --- a/doc/next/1-intro.md +++ b/doc/next/1-intro.md @@ -1,14 +1,8 @@ - - -## DRAFT RELEASE NOTES — Introduction to Go 1.24 {#introduction} +## DRAFT RELEASE NOTES — Introduction to Go 1.26 {#introduction} -**Go 1.24 is not yet released. These are work-in-progress release notes. -Go 1.24 is expected to be released in February 2025.** +**Go 1.26 is not yet released. These are work-in-progress release notes. +Go 1.26 is expected to be released in February 2026.** diff --git a/doc/next/2-language.md b/doc/next/2-language.md index 61030bd67606b0..71da62f59e59ab 100644 --- a/doc/next/2-language.md +++ b/doc/next/2-language.md @@ -1,3 +1,32 @@ ## Changes to the language {#language} + +The built-in `new` function, which creates a new variable, now allows +its operand to be an expression, specifying the initial value of the +variable. + +This feature is particularly useful when working with serialization +packages such as `encoding/json` or protocol buffers that use a +pointer to represent an optional value, as it enables an optional +field to be populated in a simple expression, for example: + +```go +import "encoding/json" + +type Person struct { + Name string `json:"name"` + Age *int `json:"age"` // age if known; nil otherwise +} + +func personJSON(name string, born time.Time) ([]byte, error) { + return json.Marshal(Person{ + Name: name, + Age: new(yearsSince(born)), + }) +} + +func yearsSince(t time.Time) int { + return int(time.Since(t).Hours() / (365.25 * 24)) // approximately +} +``` diff --git a/doc/next/3-tools.md b/doc/next/3-tools.md index b1412874689e1a..c0a4601c0b9e74 100644 --- a/doc/next/3-tools.md +++ b/doc/next/3-tools.md @@ -2,11 +2,20 @@ ### Go command {#go-command} + +`cmd/doc`, and `go tool doc` have been deleted. `go doc` can be used as +a replacement for `go tool doc`: it takes the same flags and arguments and +has the same behavior. + + +The `go fix` command, following the pattern of `go vet` in Go 1.10, +now uses the Go analysis framework (`golang.org/x/tools/go/analysis`). +This means the same analyzers that provide diagnostics in `go vet` +can be used to suggest and apply fixes in `go fix`. +The `go fix` command's historical fixers, all of which were obsolete, +have been removed and replaced by a suite of new analyzers that +offer fixes to use newer features of the language and library. + + ### Cgo {#cgo} -Cgo currently refuses to compile calls to a C function which has multiple -incompatible declarations. For instance, if `f` is declared as both `void f(int)` -and `void f(double)`, cgo will report an error instead of possibly generating an -incorrect call sequence for `f(0)`. New in this release is a better detector for -this error condition when the incompatible declarations appear in different -files. See [#67699](/issue/67699). diff --git a/doc/next/5-toolchain.md b/doc/next/5-toolchain.md index 0f4a816479754c..cc32f30a521a6d 100644 --- a/doc/next/5-toolchain.md +++ b/doc/next/5-toolchain.md @@ -4,4 +4,9 @@ ## Linker {#linker} +## Bootstrap {#bootstrap} + +As mentioned in the [Go 1.24 release notes](/doc/go1.24#bootstrap), Go 1.26 now requires +Go 1.24.6 or later for bootstrap. +We expect that Go 1.28 will require a minor release of Go 1.26 or later for bootstrap. diff --git a/doc/next/6-stdlib/99-minor/0-heading.md b/doc/next/6-stdlib/99-minor/0-heading.md index a98105e8ccba7f..e80ebc64c380fd 100644 --- a/doc/next/6-stdlib/99-minor/0-heading.md +++ b/doc/next/6-stdlib/99-minor/0-heading.md @@ -1,3 +1,10 @@ ### Minor changes to the library {#minor_library_changes} +#### go/types +The `Var.Kind` method returns an enumeration of type `VarKind` that +classifies the variable (package-level, local, receiver, parameter, +result, or struct field). See issue #70250. + +Callers of `NewVar` or `NewParam` are encouraged to call `Var.SetKind` +to ensure that this attribute is set correctly in all cases. diff --git a/doc/next/6-stdlib/99-minor/crypto/ecdsa/63963.md b/doc/next/6-stdlib/99-minor/crypto/ecdsa/63963.md new file mode 100644 index 00000000000000..81efc00bb5d657 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/crypto/ecdsa/63963.md @@ -0,0 +1 @@ +The `big.Int` fields of [PublicKey] and [PrivateKey] are now deprecated. diff --git a/doc/next/6-stdlib/99-minor/crypto/rsa/74115.md b/doc/next/6-stdlib/99-minor/crypto/rsa/74115.md new file mode 100644 index 00000000000000..a3647a79f2efd8 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/crypto/rsa/74115.md @@ -0,0 +1,5 @@ +If [PrivateKey] fields are modified after calling [PrivateKey.Precompute], +[PrivateKey.Validate] now fails. + +[PrivateKey.D] is now checked for consistency with precomputed values, even if +it is not used. diff --git a/doc/next/6-stdlib/99-minor/database/sql/driver/67546.md b/doc/next/6-stdlib/99-minor/database/sql/driver/67546.md new file mode 100644 index 00000000000000..8cb9089583a89d --- /dev/null +++ b/doc/next/6-stdlib/99-minor/database/sql/driver/67546.md @@ -0,0 +1 @@ +A database driver may implement [RowsColumnScanner] to entirely override `Scan` behavior. diff --git a/doc/next/6-stdlib/99-minor/errors/51945.md b/doc/next/6-stdlib/99-minor/errors/51945.md new file mode 100644 index 00000000000000..44ac7222e6d990 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/errors/51945.md @@ -0,0 +1,2 @@ +The new [AsType] function is a generic version of [As]. It is type-safe, faster, +and, in most cases, easier to use. diff --git a/doc/next/6-stdlib/99-minor/go/ast/68021.md b/doc/next/6-stdlib/99-minor/go/ast/68021.md new file mode 100644 index 00000000000000..0ff1a0b11e8e0c --- /dev/null +++ b/doc/next/6-stdlib/99-minor/go/ast/68021.md @@ -0,0 +1,4 @@ +The new [ParseDirective] function parses [directive +comments](/doc/comment#Syntax), which are comments such as `//go:generate`. +Source code tools can support their own directive comments and this new API +should help them implement the conventional syntax. diff --git a/doc/next/6-stdlib/99-minor/image/jpeg/75603.md b/doc/next/6-stdlib/99-minor/image/jpeg/75603.md new file mode 100644 index 00000000000000..43421761fdad82 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/image/jpeg/75603.md @@ -0,0 +1,2 @@ +The JPEG encoder and decoder have been replaced with new, faster, more accurate implementations. +Code that expects specific bit-for-bit outputs from the encoder or decoder may need to be updated. diff --git a/doc/next/6-stdlib/99-minor/log/slog/65954.md b/doc/next/6-stdlib/99-minor/log/slog/65954.md new file mode 100644 index 00000000000000..631ed665df24a7 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/log/slog/65954.md @@ -0,0 +1,6 @@ +The [`NewMultiHandler`](/pkg/log/slog#NewMultiHandler) function creates a +[`MultiHandler`](/pkg/log/slog#MultiHandler) that invokes all the given Handlers. +Its `Enable` method reports whether any of the handlers' `Enabled` methods +return true. +Its `Handle`, `WithAttr` and `WithGroup` methods call the corresponding method +on each of the enabled handlers. diff --git a/doc/next/6-stdlib/99-minor/net/49097.md b/doc/next/6-stdlib/99-minor/net/49097.md new file mode 100644 index 00000000000000..bb7947b0a11377 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/net/49097.md @@ -0,0 +1 @@ +Added context aware dial functions for TCP, UDP, IP and Unix networks. diff --git a/doc/next/6-stdlib/99-minor/net/http/67813.md b/doc/next/6-stdlib/99-minor/net/http/67813.md new file mode 100644 index 00000000000000..74b8c7644f831e --- /dev/null +++ b/doc/next/6-stdlib/99-minor/net/http/67813.md @@ -0,0 +1,4 @@ +The new +[HTTP2Config.StrictMaxConcurrentRequests](/pkg/net/http#HTTP2Config.StrictMaxConcurrentRequests) +field controls whether a new connection should be opened +if an existing HTTP/2 connection has exceeded its stream limit. diff --git a/doc/next/6-stdlib/99-minor/net/http/httptest/31054.md b/doc/next/6-stdlib/99-minor/net/http/httptest/31054.md new file mode 100644 index 00000000000000..ef6a4898f26a9d --- /dev/null +++ b/doc/next/6-stdlib/99-minor/net/http/httptest/31054.md @@ -0,0 +1,2 @@ +The HTTP client returned by [Server.Client] will now redirect requests for +`example.com` and any subdomains to the server being tested. diff --git a/doc/next/6-stdlib/99-minor/net/http/httputil/73161.md b/doc/next/6-stdlib/99-minor/net/http/httputil/73161.md new file mode 100644 index 00000000000000..f6318f85534746 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/net/http/httputil/73161.md @@ -0,0 +1,11 @@ +The [ReverseProxy.Director] configuration field is deprecated +in favor of [ReverseProxy.Rewrite]. + +A malicious client can remove headers added by a `Director` function +by designating those headers as hop-by-hop. Since there is no way to address +this problem within the scope of the `Director` API, we added a new +`Rewrite` hook in Go 1.20. `Rewrite` hooks are provided with both the +unmodified inbound request received by the proxy and the outbound request +which will be sent by the proxy. + +Since the `Director` hook is fundamentally unsafe, we are now deprecating it. diff --git a/doc/next/6-stdlib/99-minor/net/netip/61642.md b/doc/next/6-stdlib/99-minor/net/netip/61642.md new file mode 100644 index 00000000000000..3d79f2e76aee04 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/net/netip/61642.md @@ -0,0 +1 @@ +The new [Prefix.Compare] method compares two prefixes. diff --git a/doc/next/6-stdlib/99-minor/os/70352.md b/doc/next/6-stdlib/99-minor/os/70352.md new file mode 100644 index 00000000000000..5651639dadb499 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/os/70352.md @@ -0,0 +1,4 @@ +The new [Process.WithHandle] method provides access to an internal process +handle on supported platforms (Linux 5.4 or later and Windows). On Linux, +the process handle is a pidfd. The method returns [ErrNoHandle] on unsupported +platforms or when no process handle is available. diff --git a/doc/next/6-stdlib/99-minor/os/73676.md b/doc/next/6-stdlib/99-minor/os/73676.md new file mode 100644 index 00000000000000..70d01f262d86fc --- /dev/null +++ b/doc/next/6-stdlib/99-minor/os/73676.md @@ -0,0 +1,4 @@ +On Windows, the [OpenFile] `flag` parameter can now contain any combination of +Windows-specific file flags, such as `FILE_FLAG_OVERLAPPED` and +`FILE_FLAG_SEQUENTIAL_SCAN`, for control of file or device caching behavior, +access modes, and other special-purpose flags. \ No newline at end of file diff --git a/doc/next/6-stdlib/99-minor/testing/71287.md b/doc/next/6-stdlib/99-minor/testing/71287.md new file mode 100644 index 00000000000000..82cac638101099 --- /dev/null +++ b/doc/next/6-stdlib/99-minor/testing/71287.md @@ -0,0 +1,18 @@ +The new methods [T.ArtifactDir], [B.ArtifactDir], and [F.ArtifactDir] +return a directory in which to write test output files (artifacts). + +When the `-artifacts` flag is provided to `go test`, +this directory will be located under the output directory +(specified with `-outputdir`, or the current directory by default). +Otherwise, artifacts are stored in a temporary directory +which is removed after the test completes. + +The first call to `ArtifactDir` when `-artifacts` is provided +writes the location of the directory to the test log. + +For example, in a test named `TestArtifacts`, +`t.ArtifactDir()` emits: + +``` +=== ARTIFACTS Test /path/to/artifact/dir +``` diff --git a/doc/next/7-ports.md b/doc/next/7-ports.md index 8bea3f8fbc33f9..07445454227bcb 100644 --- a/doc/next/7-ports.md +++ b/doc/next/7-ports.md @@ -1,2 +1,6 @@ ## Ports {#ports} +### Windows + + +As [announced](/doc/go1.25#windows) in the Go 1.25 release notes, the [broken](/doc/go1.24#windows) 32-bit windows/arm port (`GOOS=windows` `GOARCH=arm`) is removed. diff --git a/lib/fips140/Makefile b/lib/fips140/Makefile new file mode 100644 index 00000000000000..e8fc82e08aee02 --- /dev/null +++ b/lib/fips140/Makefile @@ -0,0 +1,46 @@ +# Copyright 2024 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Rules for building and testing new FIPS snapshots. +# For example: +# +# make v1.2.3.zip +# make v1.2.3.test +# +# and then if changes are needed, check them into master +# and run 'make v1.2.3.rm' and repeat. +# +# Note that once published a snapshot zip file should never +# be modified. We record the sha256 hashes of the zip files +# in fips140.sum, and the cmd/go/internal/fips140 test checks +# that the zips match. +# +# When the zip file is finalized, run 'make updatesum' to update +# fips140.sum. + +default: + @echo nothing to make + +# make v1.2.3.zip builds a v1.2.3.zip file +# from the current origin/master. +# copy and edit the 'go run' command by hand to use a different branch. +v%.zip: + git fetch origin master + go run ../../src/cmd/go/internal/fips140/mkzip.go v$* + +# normally mkzip refuses to overwrite an existing zip file. +# make v1.2.3.rm removes the zip file and unpacked +# copy from the module cache. +v%.rm: + rm -f v$*.zip + chmod -R u+w $$(go env GOMODCACHE)/golang.org/fips140@v$* 2>/dev/null || true + rm -rf $$(go env GOMODCACHE)/golang.org/fips140@v$* + +# make v1.2.3.test runs the crypto tests using that snapshot. +v%.test: + GOFIPS140=v$* go test -short crypto... + +# make updatesum updates the fips140.sum file. +updatesum: + go test cmd/go/internal/fips140 -update diff --git a/lib/fips140/README.md b/lib/fips140/README.md new file mode 100644 index 00000000000000..38ca130d6fa22e --- /dev/null +++ b/lib/fips140/README.md @@ -0,0 +1,9 @@ +This directory holds snapshots of the crypto/internal/fips140 tree +that are being validated and certified for FIPS-140 use. +The file x.txt (for example, inprocess.txt, certified.txt) +defines the meaning of the FIPS version alias x, listing +the exact version to use. + +The zip files are created by cmd/go/internal/fips140/mkzip.go. +The fips140.sum file lists checksums for the zip files. +See the Makefile for recipes. diff --git a/lib/fips140/fips140.sum b/lib/fips140/fips140.sum new file mode 100644 index 00000000000000..703d1dc60e58ab --- /dev/null +++ b/lib/fips140/fips140.sum @@ -0,0 +1,12 @@ +# SHA256 checksums of snapshot zip files in this directory. +# These checksums are included in the FIPS security policy +# (validation instructions sent to the lab) and MUST NOT CHANGE. +# That is, the zip files themselves must not change. +# +# It is okay to add new zip files to the list, and it is okay to +# remove zip files from the list when they are removed from +# this directory. To update this file: +# +# go test cmd/go/internal/fips140 -update +# +v1.0.0-c2097c7c.zip daf3614e0406f67ae6323c902db3f953a1effb199142362a039e7526dfb9368b diff --git a/lib/fips140/inprocess.txt b/lib/fips140/inprocess.txt new file mode 100644 index 00000000000000..efd3caba85ba62 --- /dev/null +++ b/lib/fips140/inprocess.txt @@ -0,0 +1 @@ +v1.0.0-c2097c7c diff --git a/lib/fips140/v1.0.0-c2097c7c.zip b/lib/fips140/v1.0.0-c2097c7c.zip new file mode 100644 index 00000000000000..aabf762d0fecb2 Binary files /dev/null and b/lib/fips140/v1.0.0-c2097c7c.zip differ diff --git a/lib/fips140/v1.0.0.txt b/lib/fips140/v1.0.0.txt new file mode 100644 index 00000000000000..efd3caba85ba62 --- /dev/null +++ b/lib/fips140/v1.0.0.txt @@ -0,0 +1 @@ +v1.0.0-c2097c7c diff --git a/lib/time/update.bash b/lib/time/update.bash index bed82b4f40dcb7..66494752ea3c4b 100755 --- a/lib/time/update.bash +++ b/lib/time/update.bash @@ -24,8 +24,8 @@ # in the CL match the update.bash in the CL. # Versions to use. -CODE=2024a -DATA=2024a +CODE=2025b +DATA=2025b set -e @@ -40,7 +40,12 @@ curl -sS -L -O https://www.iana.org/time-zones/repository/releases/tzdata$DATA.t tar xzf tzcode$CODE.tar.gz tar xzf tzdata$DATA.tar.gz -if ! make CFLAGS=-DSTD_INSPIRED AWK=awk TZDIR=zoneinfo posix_only >make.out 2>&1; then +# The PACKRATLIST and PACKRATDATA options are copied from Ubuntu: +# https://git.launchpad.net/ubuntu/+source/tzdata/tree/debian/rules?h=debian/sid +# +# You can see the description of these make variables in the tzdata Makefile: +# https://github.com/eggert/tz/blob/main/Makefile +if ! make CFLAGS=-DSTD_INSPIRED AWK=awk TZDIR=zoneinfo PACKRATDATA=backzone PACKRATLIST=zone.tab posix_only >make.out 2>&1; then cat make.out exit 2 fi diff --git a/lib/time/zoneinfo.zip b/lib/time/zoneinfo.zip index bb38801b7a188a..b107695d1ef042 100644 Binary files a/lib/time/zoneinfo.zip and b/lib/time/zoneinfo.zip differ diff --git a/misc/wasm/go_js_wasm_exec b/lib/wasm/go_js_wasm_exec similarity index 100% rename from misc/wasm/go_js_wasm_exec rename to lib/wasm/go_js_wasm_exec diff --git a/misc/wasm/go_wasip1_wasm_exec b/lib/wasm/go_wasip1_wasm_exec similarity index 92% rename from misc/wasm/go_wasip1_wasm_exec rename to lib/wasm/go_wasip1_wasm_exec index 3b2d12ec458c54..2de1758793f212 100755 --- a/misc/wasm/go_wasip1_wasm_exec +++ b/lib/wasm/go_wasip1_wasm_exec @@ -14,7 +14,7 @@ case "$GOWASIRUNTIME" in exec wazero run -mount /:/ -env-inherit -cachedir "${TMPDIR:-/tmp}"/wazero ${GOWASIRUNTIMEARGS:-} "$1" "${@:2}" ;; "wasmtime" | "") - exec wasmtime run --dir=/ --env PWD="$PWD" --env PATH="$PATH" -W max-wasm-stack=1048576 ${GOWASIRUNTIMEARGS:-} "$1" "${@:2}" + exec wasmtime run --dir=/ --env PWD="$PWD" --env PATH="$PATH" -W max-wasm-stack=8388608 ${GOWASIRUNTIMEARGS:-} "$1" "${@:2}" ;; *) echo "Unknown Go WASI runtime specified: $GOWASIRUNTIME" diff --git a/misc/wasm/wasm_exec.js b/lib/wasm/wasm_exec.js similarity index 97% rename from misc/wasm/wasm_exec.js rename to lib/wasm/wasm_exec.js index bc6f210242824c..d71af9e97e8ca9 100644 --- a/misc/wasm/wasm_exec.js +++ b/lib/wasm/wasm_exec.js @@ -14,7 +14,7 @@ if (!globalThis.fs) { let outputBuf = ""; globalThis.fs = { - constants: { O_WRONLY: -1, O_RDWR: -1, O_CREAT: -1, O_TRUNC: -1, O_APPEND: -1, O_EXCL: -1 }, // unused + constants: { O_WRONLY: -1, O_RDWR: -1, O_CREAT: -1, O_TRUNC: -1, O_APPEND: -1, O_EXCL: -1, O_DIRECTORY: -1 }, // unused writeSync(fd, buf) { outputBuf += decoder.decode(buf); const nl = outputBuf.lastIndexOf("\n"); @@ -73,6 +73,14 @@ } } + if (!globalThis.path) { + globalThis.path = { + resolve(...pathSegments) { + return pathSegments.join("/"); + } + } + } + if (!globalThis.crypto) { throw new Error("globalThis.crypto is not available, polyfill required (crypto.getRandomValues only)"); } @@ -208,10 +216,16 @@ return decoder.decode(new DataView(this._inst.exports.mem.buffer, saddr, len)); } + const testCallExport = (a, b) => { + this._inst.exports.testExport0(); + return this._inst.exports.testExport(a, b); + } + const timeOrigin = Date.now() - performance.now(); this.importObject = { _gotest: { add: (a, b) => a + b, + callExport: testCallExport, }, gojs: { // Go's SP does not change as long as no Go code is running. Some operations (e.g. calls, getters and setters) diff --git a/misc/wasm/wasm_exec_node.js b/lib/wasm/wasm_exec_node.js similarity index 97% rename from misc/wasm/wasm_exec_node.js rename to lib/wasm/wasm_exec_node.js index 986069087bc0bc..dd65b19867dcf4 100644 --- a/misc/wasm/wasm_exec_node.js +++ b/lib/wasm/wasm_exec_node.js @@ -11,6 +11,7 @@ if (process.argv.length < 3) { globalThis.require = require; globalThis.fs = require("fs"); +globalThis.path = require("path"); globalThis.TextEncoder = require("util").TextEncoder; globalThis.TextDecoder = require("util").TextDecoder; diff --git a/misc/ios/go_ios_exec.go b/misc/ios/go_ios_exec.go index b21ce1aaf3f0cc..e58457d0519948 100644 --- a/misc/ios/go_ios_exec.go +++ b/misc/ios/go_ios_exec.go @@ -212,7 +212,7 @@ func copyLocalData(dstbase string) (pkgpath string, err error) { // Copy all immediate files and testdata directories between // the package being tested and the source root. pkgpath = "" - for _, element := range strings.Split(finalPkgpath, string(filepath.Separator)) { + for element := range strings.SplitSeq(finalPkgpath, string(filepath.Separator)) { if debug { log.Printf("copying %s", pkgpath) } diff --git a/misc/linkcheck/linkcheck.go b/misc/linkcheck/linkcheck.go deleted file mode 100644 index efe400965b2e06..00000000000000 --- a/misc/linkcheck/linkcheck.go +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// The linkcheck command finds missing links in the godoc website. -// It crawls a URL recursively and notes URLs and URL fragments -// that it's seen and prints a report of missing links at the end. -package main - -import ( - "errors" - "flag" - "fmt" - "io" - "log" - "net/http" - "os" - "regexp" - "strings" - "sync" -) - -var ( - root = flag.String("root", "http://localhost:6060/", "Root to crawl") - verbose = flag.Bool("verbose", false, "verbose") -) - -var wg sync.WaitGroup // outstanding fetches -var urlq = make(chan string) // URLs to crawl - -// urlFrag is a URL and its optional #fragment (without the #) -type urlFrag struct { - url, frag string -} - -var ( - mu sync.Mutex - crawled = make(map[string]bool) // URL without fragment -> true - neededFrags = make(map[urlFrag][]string) // URL#frag -> who needs it -) - -var aRx = regexp.MustCompile(`]+)`) - -// Owned by crawlLoop goroutine: -var ( - linkSources = make(map[string][]string) // url no fragment -> sources - fragExists = make(map[urlFrag]bool) - problems []string -) - -func localLinks(body string) (links []string) { - seen := map[string]bool{} - mv := aRx.FindAllStringSubmatch(body, -1) - for _, m := range mv { - ref := m[1] - if strings.HasPrefix(ref, "/src/") { - continue - } - if !seen[ref] { - seen[ref] = true - links = append(links, m[1]) - } - } - return -} - -var idRx = regexp.MustCompile(`\bid=['"]?([^\s'">]+)`) - -func pageIDs(body string) (ids []string) { - mv := idRx.FindAllStringSubmatch(body, -1) - for _, m := range mv { - ids = append(ids, m[1]) - } - return -} - -// url may contain a #fragment, and the fragment is then noted as needing to exist. -func crawl(url string, sourceURL string) { - if strings.Contains(url, "/devel/release") { - return - } - mu.Lock() - defer mu.Unlock() - if u, frag, ok := strings.Cut(url, "#"); ok { - url = u - if frag != "" { - uf := urlFrag{url, frag} - neededFrags[uf] = append(neededFrags[uf], sourceURL) - } - } - if crawled[url] { - return - } - crawled[url] = true - - wg.Add(1) - go func() { - urlq <- url - }() -} - -func addProblem(url, errmsg string) { - msg := fmt.Sprintf("Error on %s: %s (from %s)", url, errmsg, linkSources[url]) - if *verbose { - log.Print(msg) - } - problems = append(problems, msg) -} - -func crawlLoop() { - for url := range urlq { - if err := doCrawl(url); err != nil { - addProblem(url, err.Error()) - } - } -} - -func doCrawl(url string) error { - defer wg.Done() - - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return err - } - res, err := http.DefaultTransport.RoundTrip(req) - if err != nil { - return err - } - // Handle redirects. - if res.StatusCode/100 == 3 { - newURL, err := res.Location() - if err != nil { - return fmt.Errorf("resolving redirect: %v", err) - } - if !strings.HasPrefix(newURL.String(), *root) { - // Skip off-site redirects. - return nil - } - crawl(newURL.String(), url) - return nil - } - if res.StatusCode != 200 { - return errors.New(res.Status) - } - slurp, err := io.ReadAll(res.Body) - res.Body.Close() - if err != nil { - log.Fatalf("Error reading %s body: %v", url, err) - } - if *verbose { - log.Printf("Len of %s: %d", url, len(slurp)) - } - body := string(slurp) - for _, ref := range localLinks(body) { - if *verbose { - log.Printf(" links to %s", ref) - } - dest := *root + ref - linkSources[dest] = append(linkSources[dest], url) - crawl(dest, url) - } - for _, id := range pageIDs(body) { - if *verbose { - log.Printf(" url %s has #%s", url, id) - } - fragExists[urlFrag{url, id}] = true - } - return nil -} - -func main() { - flag.Parse() - - go crawlLoop() - crawl(*root, "") - - wg.Wait() - close(urlq) - for uf, needers := range neededFrags { - if !fragExists[uf] { - problems = append(problems, fmt.Sprintf("Missing fragment for %+v from %v", uf, needers)) - } - } - - for _, s := range problems { - fmt.Println(s) - } - if len(problems) > 0 { - os.Exit(1) - } -} diff --git a/misc/wasm/wasm_exec.html b/misc/wasm/wasm_exec.html index 72e64473eb588d..694b526df14927 100644 --- a/misc/wasm/wasm_exec.html +++ b/misc/wasm/wasm_exec.html @@ -17,7 +17,7 @@ (see https://caniuse.com/#feat=textencoder) --> - + + + +` diff --git a/src/crypto/tls/bogo_shim_unix_test.go b/src/crypto/tls/bogo_shim_unix_test.go new file mode 100644 index 00000000000000..3b5f5f92c228a7 --- /dev/null +++ b/src/crypto/tls/bogo_shim_unix_test.go @@ -0,0 +1,18 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix && !wasm + +package tls + +import ( + "os" + "syscall" +) + +func pauseProcess() { + pid := os.Getpid() + process, _ := os.FindProcess(pid) + process.Signal(syscall.SIGSTOP) +} diff --git a/src/crypto/tls/boring.go b/src/crypto/tls/boring.go deleted file mode 100644 index c44ae92f2528f3..00000000000000 --- a/src/crypto/tls/boring.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build boringcrypto - -package tls - -import "crypto/internal/boring/fipstls" - -// needFIPS returns fipstls.Required(), which is not available without the -// boringcrypto build tag. -func needFIPS() bool { - return fipstls.Required() -} diff --git a/src/crypto/tls/boring_test.go b/src/crypto/tls/boring_test.go deleted file mode 100644 index be10b71bd2269b..00000000000000 --- a/src/crypto/tls/boring_test.go +++ /dev/null @@ -1,626 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build boringcrypto - -package tls - -import ( - "crypto/ecdsa" - "crypto/elliptic" - "crypto/internal/boring/fipstls" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "fmt" - "internal/obscuretestdata" - "math/big" - "net" - "runtime" - "strings" - "testing" - "time" -) - -func TestBoringServerProtocolVersion(t *testing.T) { - test := func(t *testing.T, name string, v uint16, msg string) { - t.Run(name, func(t *testing.T) { - serverConfig := testConfig.Clone() - serverConfig.MinVersion = VersionSSL30 - clientConfig := testConfig.Clone() - clientConfig.MinVersion = v - clientConfig.MaxVersion = v - _, _, err := testHandshake(t, clientConfig, serverConfig) - if msg == "" { - if err != nil { - t.Fatalf("got error: %v, expected success", err) - } - } else { - if err == nil { - t.Fatalf("got success, expected error") - } - if !strings.Contains(err.Error(), msg) { - t.Fatalf("got error %v, expected %q", err, msg) - } - } - }) - } - - test(t, "VersionTLS10", VersionTLS10, "") - test(t, "VersionTLS11", VersionTLS11, "") - test(t, "VersionTLS12", VersionTLS12, "") - test(t, "VersionTLS13", VersionTLS13, "") - - t.Run("fipstls", func(t *testing.T) { - fipstls.Force() - defer fipstls.Abandon() - test(t, "VersionTLS10", VersionTLS10, "supported versions") - test(t, "VersionTLS11", VersionTLS11, "supported versions") - test(t, "VersionTLS12", VersionTLS12, "") - test(t, "VersionTLS13", VersionTLS13, "supported versions") - }) -} - -func isBoringVersion(v uint16) bool { - return v == VersionTLS12 -} - -func isBoringCipherSuite(id uint16) bool { - switch id { - case TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - TLS_RSA_WITH_AES_128_GCM_SHA256, - TLS_RSA_WITH_AES_256_GCM_SHA384: - return true - } - return false -} - -func isBoringCurve(id CurveID) bool { - switch id { - case CurveP256, CurveP384, CurveP521: - return true - } - return false -} - -func isECDSA(id uint16) bool { - for _, suite := range cipherSuites { - if suite.id == id { - return suite.flags&suiteECSign == suiteECSign - } - } - panic(fmt.Sprintf("unknown cipher suite %#x", id)) -} - -func isBoringSignatureScheme(alg SignatureScheme) bool { - switch alg { - default: - return false - case PKCS1WithSHA256, - ECDSAWithP256AndSHA256, - PKCS1WithSHA384, - ECDSAWithP384AndSHA384, - PKCS1WithSHA512, - ECDSAWithP521AndSHA512, - PSSWithSHA256, - PSSWithSHA384, - PSSWithSHA512: - // ok - } - return true -} - -func TestBoringServerCipherSuites(t *testing.T) { - serverConfig := testConfig.Clone() - serverConfig.CipherSuites = allCipherSuites() - serverConfig.Certificates = make([]Certificate, 1) - - for _, id := range allCipherSuites() { - if isECDSA(id) { - serverConfig.Certificates[0].Certificate = [][]byte{testECDSACertificate} - serverConfig.Certificates[0].PrivateKey = testECDSAPrivateKey - } else { - serverConfig.Certificates[0].Certificate = [][]byte{testRSACertificate} - serverConfig.Certificates[0].PrivateKey = testRSAPrivateKey - } - serverConfig.BuildNameToCertificate() - t.Run(fmt.Sprintf("suite=%#x", id), func(t *testing.T) { - clientHello := &clientHelloMsg{ - vers: VersionTLS12, - random: make([]byte, 32), - cipherSuites: []uint16{id}, - compressionMethods: []uint8{compressionNone}, - supportedCurves: defaultCurvePreferences(), - supportedPoints: []uint8{pointFormatUncompressed}, - } - - testClientHello(t, serverConfig, clientHello) - t.Run("fipstls", func(t *testing.T) { - fipstls.Force() - defer fipstls.Abandon() - msg := "" - if !isBoringCipherSuite(id) { - msg = "no cipher suite supported by both client and server" - } - testClientHelloFailure(t, serverConfig, clientHello, msg) - }) - }) - } -} - -func TestBoringServerCurves(t *testing.T) { - serverConfig := testConfig.Clone() - serverConfig.Certificates = make([]Certificate, 1) - serverConfig.Certificates[0].Certificate = [][]byte{testECDSACertificate} - serverConfig.Certificates[0].PrivateKey = testECDSAPrivateKey - serverConfig.BuildNameToCertificate() - - for _, curveid := range defaultCurvePreferences() { - t.Run(fmt.Sprintf("curve=%d", curveid), func(t *testing.T) { - clientConfig := testConfig.Clone() - clientConfig.CurvePreferences = []CurveID{curveid} - if curveid == x25519Kyber768Draft00 { - // x25519Kyber768Draft00 is not supported standalone. - clientConfig.CurvePreferences = append(clientConfig.CurvePreferences, X25519) - } - if _, _, err := testHandshake(t, clientConfig, serverConfig); err != nil { - t.Fatalf("got error: %v, expected success", err) - } - - // With fipstls forced, bad curves should be rejected. - t.Run("fipstls", func(t *testing.T) { - fipstls.Force() - defer fipstls.Abandon() - _, _, err := testHandshake(t, clientConfig, serverConfig) - if err != nil && isBoringCurve(curveid) { - t.Fatalf("got error: %v, expected success", err) - } else if err == nil && !isBoringCurve(curveid) { - t.Fatalf("got success, expected error") - } - }) - }) - } -} - -func boringHandshake(t *testing.T, clientConfig, serverConfig *Config) (clientErr, serverErr error) { - c, s := localPipe(t) - client := Client(c, clientConfig) - server := Server(s, serverConfig) - done := make(chan error, 1) - go func() { - done <- client.Handshake() - c.Close() - }() - serverErr = server.Handshake() - s.Close() - clientErr = <-done - return -} - -func TestBoringServerSignatureAndHash(t *testing.T) { - defer func() { - testingOnlyForceClientHelloSignatureAlgorithms = nil - }() - - for _, sigHash := range defaultSupportedSignatureAlgorithms { - t.Run(fmt.Sprintf("%v", sigHash), func(t *testing.T) { - serverConfig := testConfig.Clone() - serverConfig.Certificates = make([]Certificate, 1) - - testingOnlyForceClientHelloSignatureAlgorithms = []SignatureScheme{sigHash} - - sigType, _, _ := typeAndHashFromSignatureScheme(sigHash) - switch sigType { - case signaturePKCS1v15, signatureRSAPSS: - serverConfig.CipherSuites = []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256} - serverConfig.Certificates[0].Certificate = [][]byte{testRSA2048Certificate} - serverConfig.Certificates[0].PrivateKey = testRSA2048PrivateKey - case signatureEd25519: - serverConfig.CipherSuites = []uint16{TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256} - serverConfig.Certificates[0].Certificate = [][]byte{testEd25519Certificate} - serverConfig.Certificates[0].PrivateKey = testEd25519PrivateKey - case signatureECDSA: - serverConfig.CipherSuites = []uint16{TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256} - serverConfig.Certificates[0].Certificate = [][]byte{testECDSACertificate} - serverConfig.Certificates[0].PrivateKey = testECDSAPrivateKey - } - serverConfig.BuildNameToCertificate() - // PKCS#1 v1.5 signature algorithms can't be used standalone in TLS - // 1.3, and the ECDSA ones bind to the curve used. - serverConfig.MaxVersion = VersionTLS12 - - clientErr, serverErr := boringHandshake(t, testConfig, serverConfig) - if clientErr != nil { - t.Fatalf("expected handshake with %#x to succeed; client error: %v; server error: %v", sigHash, clientErr, serverErr) - } - - // With fipstls forced, bad curves should be rejected. - t.Run("fipstls", func(t *testing.T) { - fipstls.Force() - defer fipstls.Abandon() - clientErr, _ := boringHandshake(t, testConfig, serverConfig) - if isBoringSignatureScheme(sigHash) { - if clientErr != nil { - t.Fatalf("expected handshake with %#x to succeed; err=%v", sigHash, clientErr) - } - } else { - if clientErr == nil { - t.Fatalf("expected handshake with %#x to fail, but it succeeded", sigHash) - } - } - }) - }) - } -} - -func TestBoringClientHello(t *testing.T) { - // Test that no matter what we put in the client config, - // the client does not offer non-FIPS configurations. - fipstls.Force() - defer fipstls.Abandon() - - c, s := net.Pipe() - defer c.Close() - defer s.Close() - - clientConfig := testConfig.Clone() - // All sorts of traps for the client to avoid. - clientConfig.MinVersion = VersionSSL30 - clientConfig.MaxVersion = VersionTLS13 - clientConfig.CipherSuites = allCipherSuites() - clientConfig.CurvePreferences = defaultCurvePreferences() - - go Client(c, clientConfig).Handshake() - srv := Server(s, testConfig) - msg, err := srv.readHandshake(nil) - if err != nil { - t.Fatal(err) - } - hello, ok := msg.(*clientHelloMsg) - if !ok { - t.Fatalf("unexpected message type %T", msg) - } - - if !isBoringVersion(hello.vers) { - t.Errorf("client vers=%#x, want %#x (TLS 1.2)", hello.vers, VersionTLS12) - } - for _, v := range hello.supportedVersions { - if !isBoringVersion(v) { - t.Errorf("client offered disallowed version %#x", v) - } - } - for _, id := range hello.cipherSuites { - if !isBoringCipherSuite(id) { - t.Errorf("client offered disallowed suite %#x", id) - } - } - for _, id := range hello.supportedCurves { - if !isBoringCurve(id) { - t.Errorf("client offered disallowed curve %d", id) - } - } - for _, sigHash := range hello.supportedSignatureAlgorithms { - if !isBoringSignatureScheme(sigHash) { - t.Errorf("client offered disallowed signature-and-hash %v", sigHash) - } - } -} - -func TestBoringCertAlgs(t *testing.T) { - // NaCl, arm and wasm time out generating keys. Nothing in this test is architecture-specific, so just don't bother on those. - if runtime.GOOS == "nacl" || runtime.GOARCH == "arm" || runtime.GOOS == "js" { - t.Skipf("skipping on %s/%s because key generation takes too long", runtime.GOOS, runtime.GOARCH) - } - - // Set up some roots, intermediate CAs, and leaf certs with various algorithms. - // X_Y is X signed by Y. - R1 := boringCert(t, "R1", boringRSAKey(t, 2048), nil, boringCertCA|boringCertFIPSOK) - R2 := boringCert(t, "R2", boringRSAKey(t, 512), nil, boringCertCA) - - M1_R1 := boringCert(t, "M1_R1", boringECDSAKey(t, elliptic.P256()), R1, boringCertCA|boringCertFIPSOK) - M2_R1 := boringCert(t, "M2_R1", boringECDSAKey(t, elliptic.P224()), R1, boringCertCA) - - I_R1 := boringCert(t, "I_R1", boringRSAKey(t, 3072), R1, boringCertCA|boringCertFIPSOK) - I_R2 := boringCert(t, "I_R2", I_R1.key, R2, boringCertCA|boringCertFIPSOK) - I_M1 := boringCert(t, "I_M1", I_R1.key, M1_R1, boringCertCA|boringCertFIPSOK) - I_M2 := boringCert(t, "I_M2", I_R1.key, M2_R1, boringCertCA|boringCertFIPSOK) - - L1_I := boringCert(t, "L1_I", boringECDSAKey(t, elliptic.P384()), I_R1, boringCertLeaf|boringCertFIPSOK) - L2_I := boringCert(t, "L2_I", boringRSAKey(t, 1024), I_R1, boringCertLeaf) - - // client verifying server cert - testServerCert := func(t *testing.T, desc string, pool *x509.CertPool, key interface{}, list [][]byte, ok bool) { - clientConfig := testConfig.Clone() - clientConfig.RootCAs = pool - clientConfig.InsecureSkipVerify = false - clientConfig.ServerName = "example.com" - - serverConfig := testConfig.Clone() - serverConfig.Certificates = []Certificate{{Certificate: list, PrivateKey: key}} - serverConfig.BuildNameToCertificate() - - clientErr, _ := boringHandshake(t, clientConfig, serverConfig) - - if (clientErr == nil) == ok { - if ok { - t.Logf("%s: accept", desc) - } else { - t.Logf("%s: reject", desc) - } - } else { - if ok { - t.Errorf("%s: BAD reject (%v)", desc, clientErr) - } else { - t.Errorf("%s: BAD accept", desc) - } - } - } - - // server verifying client cert - testClientCert := func(t *testing.T, desc string, pool *x509.CertPool, key interface{}, list [][]byte, ok bool) { - clientConfig := testConfig.Clone() - clientConfig.ServerName = "example.com" - clientConfig.Certificates = []Certificate{{Certificate: list, PrivateKey: key}} - - serverConfig := testConfig.Clone() - serverConfig.ClientCAs = pool - serverConfig.ClientAuth = RequireAndVerifyClientCert - - _, serverErr := boringHandshake(t, clientConfig, serverConfig) - - if (serverErr == nil) == ok { - if ok { - t.Logf("%s: accept", desc) - } else { - t.Logf("%s: reject", desc) - } - } else { - if ok { - t.Errorf("%s: BAD reject (%v)", desc, serverErr) - } else { - t.Errorf("%s: BAD accept", desc) - } - } - } - - // Run simple basic test with known answers before proceeding to - // exhaustive test with computed answers. - r1pool := x509.NewCertPool() - r1pool.AddCert(R1.cert) - testServerCert(t, "basic", r1pool, L2_I.key, [][]byte{L2_I.der, I_R1.der}, true) - testClientCert(t, "basic (client cert)", r1pool, L2_I.key, [][]byte{L2_I.der, I_R1.der}, true) - fipstls.Force() - testServerCert(t, "basic (fips)", r1pool, L2_I.key, [][]byte{L2_I.der, I_R1.der}, false) - testClientCert(t, "basic (fips, client cert)", r1pool, L2_I.key, [][]byte{L2_I.der, I_R1.der}, false) - fipstls.Abandon() - - if t.Failed() { - t.Fatal("basic test failed, skipping exhaustive test") - } - - if testing.Short() { - t.Logf("basic test passed; skipping exhaustive test in -short mode") - return - } - - for l := 1; l <= 2; l++ { - leaf := L1_I - if l == 2 { - leaf = L2_I - } - for i := 0; i < 64; i++ { - reachable := map[string]bool{leaf.parentOrg: true} - reachableFIPS := map[string]bool{leaf.parentOrg: leaf.fipsOK} - list := [][]byte{leaf.der} - listName := leaf.name - addList := func(cond int, c *boringCertificate) { - if cond != 0 { - list = append(list, c.der) - listName += "," + c.name - if reachable[c.org] { - reachable[c.parentOrg] = true - } - if reachableFIPS[c.org] && c.fipsOK { - reachableFIPS[c.parentOrg] = true - } - } - } - addList(i&1, I_R1) - addList(i&2, I_R2) - addList(i&4, I_M1) - addList(i&8, I_M2) - addList(i&16, M1_R1) - addList(i&32, M2_R1) - - for r := 1; r <= 3; r++ { - pool := x509.NewCertPool() - rootName := "," - shouldVerify := false - shouldVerifyFIPS := false - addRoot := func(cond int, c *boringCertificate) { - if cond != 0 { - rootName += "," + c.name - pool.AddCert(c.cert) - if reachable[c.org] { - shouldVerify = true - } - if reachableFIPS[c.org] && c.fipsOK { - shouldVerifyFIPS = true - } - } - } - addRoot(r&1, R1) - addRoot(r&2, R2) - rootName = rootName[1:] // strip leading comma - testServerCert(t, listName+"->"+rootName[1:], pool, leaf.key, list, shouldVerify) - testClientCert(t, listName+"->"+rootName[1:]+"(client cert)", pool, leaf.key, list, shouldVerify) - fipstls.Force() - testServerCert(t, listName+"->"+rootName[1:]+" (fips)", pool, leaf.key, list, shouldVerifyFIPS) - testClientCert(t, listName+"->"+rootName[1:]+" (fips, client cert)", pool, leaf.key, list, shouldVerifyFIPS) - fipstls.Abandon() - } - } - } -} - -const ( - boringCertCA = iota - boringCertLeaf - boringCertFIPSOK = 0x80 -) - -func boringRSAKey(t *testing.T, size int) *rsa.PrivateKey { - k, err := rsa.GenerateKey(rand.Reader, size) - if err != nil { - t.Fatal(err) - } - return k -} - -func boringECDSAKey(t *testing.T, curve elliptic.Curve) *ecdsa.PrivateKey { - k, err := ecdsa.GenerateKey(curve, rand.Reader) - if err != nil { - t.Fatal(err) - } - return k -} - -type boringCertificate struct { - name string - org string - parentOrg string - der []byte - cert *x509.Certificate - key interface{} - fipsOK bool -} - -func boringCert(t *testing.T, name string, key interface{}, parent *boringCertificate, mode int) *boringCertificate { - org := name - parentOrg := "" - if i := strings.Index(org, "_"); i >= 0 { - org = org[:i] - parentOrg = name[i+1:] - } - tmpl := &x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{ - Organization: []string{org}, - }, - NotBefore: time.Unix(0, 0), - NotAfter: time.Unix(0, 0), - - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - BasicConstraintsValid: true, - } - if mode&^boringCertFIPSOK == boringCertLeaf { - tmpl.DNSNames = []string{"example.com"} - } else { - tmpl.IsCA = true - tmpl.KeyUsage |= x509.KeyUsageCertSign - } - - var pcert *x509.Certificate - var pkey interface{} - if parent != nil { - pcert = parent.cert - pkey = parent.key - } else { - pcert = tmpl - pkey = key - } - - var pub interface{} - switch k := key.(type) { - case *rsa.PrivateKey: - pub = &k.PublicKey - case *ecdsa.PrivateKey: - pub = &k.PublicKey - default: - t.Fatalf("invalid key %T", key) - } - - der, err := x509.CreateCertificate(rand.Reader, tmpl, pcert, pub, pkey) - if err != nil { - t.Fatal(err) - } - cert, err := x509.ParseCertificate(der) - if err != nil { - t.Fatal(err) - } - - fipsOK := mode&boringCertFIPSOK != 0 - return &boringCertificate{name, org, parentOrg, der, cert, key, fipsOK} -} - -// A self-signed test certificate with an RSA key of size 2048, for testing -// RSA-PSS with SHA512. SAN of example.golang. -var ( - testRSA2048Certificate []byte - testRSA2048PrivateKey *rsa.PrivateKey -) - -func init() { - block, _ := pem.Decode(obscuretestdata.Rot13([]byte(` ------ORTVA PREGVSVPNGR----- -ZVVP/mPPNrrtNjVONtVENYUUK/xu4+4mZH9QnemORpDjQDLWXbMVuipANDRYODNj -RwRDZN4TN1HRPuZUDJAgMFOQomNrSj0kZGNkZQRkAGN0ZQInSj0lZQRlZwxkAGN0 -ZQInZOVkRQNBOtAIONbGO0SwoJHtD28jttRvZN0TPFdTFVo3QDRONDHNN4VOQjNj -ttRXNbVONDPs8sx0A6vrPOK4VBIVsXvgg4xTpBDYrvzPsfwddUplfZVITRgSFZ6R -4Nl141s/7VdqJ0HgVdAo4CKuEBVQ7lQkE284kY6KoPhi/g5uC3HpruLp3uzYvlIq -ZxMDvMJgsHHWs/1dBgZ+buAt59YEJc4q+6vK0yn1WY3RjPVpxxAwW9uDoS7Co2PF -+RF9Lb55XNnc8XBoycpE8ZOFA38odajwsDqPKiBRBwnz2UHkXmRSK5ZN+sN0zr4P -vbPpPEYJXy+TbA9S8sNOsbM+G+2rny4QYhB95eKE8FeBVIOu3KSBe/EIuwgKpAIS -MXpiQg6q68I6wNXNLXz5ayw9TCcq4i+eNtZONNTwHQOBZN4TN1HqQjRO/jDRNjVS -bQNGOtAIUFHRQQNXOtteOtRSODpQNGNZOtAIUEZONs8RNwNNZOxTN1HqRDDFZOPP -QzI4LJ1joTHhM29fLJ5aZN0TPFdTFVo3QDROPjHNN4VONDPBbLfIpSPOuobdr3JU -qP6I7KKKRPzawu01e8u80li0AE379aFQ3pj2Z+UXinKlfJdey5uwTIXj0igjQ81e -I4WmQh7VsVbt5z8+DAP+7YdQMfm88iQXBefblFIBzHPtzPXSKrj+YN+rB/vDRWGe -7rafqqBrKWRc27Rq5iJ+xzJJ3Dztyp2Tjl8jSeZQVdaeaBmON4bPaQRtgKWg0mbt -aEjosRZNJv1nDEl5qG9XN3FC9zb5FrGSFmTTUvR4f4tUHr7wifNSS2dtgQ6+jU6f -m9o6fukaP7t5VyOXuV7FIO/Hdg2lqW+xU1LowZpVd6ANZ5rAZXtMhWe3+mjfFtju -TAnR ------RAQ PREGVSVPNGR-----`))) - testRSA2048Certificate = block.Bytes - - block, _ = pem.Decode(obscuretestdata.Rot13([]byte(` ------ORTVA EFN CEVINGR XRL----- -ZVVRcNVONNXPNDRNa/U5AQrbattI+PQyFUlbeorWOaQxP3bcta7V6du3ZeQPSEuY -EHwBuBNZgrAK/+lXaIgSYFXwJ+Q14HGvN+8t8HqiBZF+y2jee/7rLG91UUbJUA4M -v4fyKGWTHVzIeK1SPK/9nweGCdVGLBsF0IdrUshby9WJgFF9kZNvUWWQLlsLHTkr -m29txiuRiJXBrFtTdsPwz5nKRsQNHwq/T6c8V30UDy7muQb2cgu1ZFfkOI+GNCaj -AWahNbdNaNxF1vcsudQsEsUjNK6Tsx/gazcrNl7wirn10sRdmvSDLq1kGd/0ILL7 -I3QIEJFaYj7rariSrbjPtTPchM5L/Ew6KrY/djVQNDNONbVONDPAcZMvsq/it42u -UqPiYhMnLF0E7FhaSycbKRfygTqYSfac0VsbWM/htSDOFNVVsYjZhzH6bKN1m7Hi -98nVLI61QrCeGPQIQSOfUoAzC8WNb8JgohfRojq5mlbO7YLT2+pyxWxyJR73XdHd -ezV+HWrlFpy2Tva7MGkOKm1JCOx9IjpajxrnKctNFVOJ23suRPZ9taLRRjnOrm5G -6Zr8q1gUgLDi7ifXr7eb9j9/UXeEKrwdLXX1YkxusSevlI+z8YMWMa2aKBn6T3tS -Ao8Dx1Hx5CHORAOzlZSWuG4Z/hhFd4LgZeeB2tv8D+sCuhTmp5FfuLXEOc0J4C5e -zgIPgRSENbTONZRAOVSYeI2+UfTw0kLSnfXbi/DCr6UFGE1Uu2VMBAc+bX4bfmJR -wOG4IpaVGzcy6gP1Jl4TpekwAtXVSMNw+1k1YHHYqbeKxhT8le0gNuT9mAlsJfFl -CeFbiP0HIome8Wkkyn+xDIkRDDdJDkCyRIhY8xKnVQN6Ylg1Uchn2YiCNbTONADM -p6Yd2G7+OkYkAqv2z8xMmrw5xtmOc/KqIfoSJEyroVK2XeSUfeUmG9CHx3QR1iMX -Z6cmGg94aDuJFxQtPnj1FbuRyW3USVSjphfS1FWNp3cDrcq8ht6VLqycQZYgOw/C -/5C6OIHgtb05R4+V/G3vLngztyDkGgyM0ExFI2yyNbTONYBKxXSK7nuCis0JxfQu -hGshSBGCbbjtDT0RctJ0jEqPkrt/WYvp3yFQ0tfggDI2JfErpelJpknryEt10EzB -38OobtzunS4kitfFihwBsvMGR8bX1G43Z+6AXfVyZY3LVYocH/9nWkCJl0f2QdQe -pDWuMeyx+cmwON7Oas/HEqjkNbTNXE/PAj14Q+zeY3LYoovPKvlqdkIjki5cqMqm -8guv3GApfJP4vTHEqpIdosHvaICqWvKr/Xnp3JTPrEWnSItoXNBkYgv1EO5ZxVut -Q8rlhcOdx4J1Y1txekdfqw4GSykxjZljwy2R2F4LlD8COg6I04QbIEMfVXmdm+CS -HvbaCd0PtLOPLKidvbWuCrjxBd/L5jeQOrMJ1SDX5DQ9J5Z8/5mkq4eqiWgwuoWc -bBegiZqey6hcl9Um4OWQ3SKjISvCSR7wdrAdv0S21ivYkOCZZQ3HBQS6YY5RlYvE -9I4kIZF8XKkit7ekfhdmZCfpIvnJHY6JAIOufQ2+92qUkFKmm5RWXD== ------RAQ EFN CEVINGR XRL-----`))) - var err error - testRSA2048PrivateKey, err = x509.ParsePKCS1PrivateKey(block.Bytes) - if err != nil { - panic(err) - } -} diff --git a/src/crypto/tls/cache.go b/src/crypto/tls/cache.go index a7677611fdc7bf..a2c255af88af2d 100644 --- a/src/crypto/tls/cache.go +++ b/src/crypto/tls/cache.go @@ -8,78 +8,19 @@ import ( "crypto/x509" "runtime" "sync" - "sync/atomic" + "weak" ) -type cacheEntry struct { - refs atomic.Int64 - cert *x509.Certificate -} - -// certCache implements an intern table for reference counted x509.Certificates, -// implemented in a similar fashion to BoringSSL's CRYPTO_BUFFER_POOL. This -// allows for a single x509.Certificate to be kept in memory and referenced from -// multiple Conns. Returned references should not be mutated by callers. Certificates -// are still safe to use after they are removed from the cache. -// -// Certificates are returned wrapped in an activeCert struct that should be held by -// the caller. When references to the activeCert are freed, the number of references -// to the certificate in the cache is decremented. Once the number of references -// reaches zero, the entry is evicted from the cache. -// -// The main difference between this implementation and CRYPTO_BUFFER_POOL is that -// CRYPTO_BUFFER_POOL is a more generic structure which supports blobs of data, -// rather than specific structures. Since we only care about x509.Certificates, -// certCache is implemented as a specific cache, rather than a generic one. -// -// See https://boringssl.googlesource.com/boringssl/+/master/include/openssl/pool.h -// and https://boringssl.googlesource.com/boringssl/+/master/crypto/pool/pool.c -// for the BoringSSL reference. -type certCache struct { - sync.Map -} - -var globalCertCache = new(certCache) - -// activeCert is a handle to a certificate held in the cache. Once there are -// no alive activeCerts for a given certificate, the certificate is removed -// from the cache by a finalizer. -type activeCert struct { - cert *x509.Certificate -} +// weakCertCache provides a cache of *x509.Certificates, allowing multiple +// connections to reuse parsed certificates, instead of re-parsing the +// certificate for every connection, which is an expensive operation. +type weakCertCache struct{ sync.Map } -// active increments the number of references to the entry, wraps the -// certificate in the entry in an activeCert, and sets the finalizer. -// -// Note that there is a race between active and the finalizer set on the -// returned activeCert, triggered if active is called after the ref count is -// decremented such that refs may be > 0 when evict is called. We consider this -// safe, since the caller holding an activeCert for an entry that is no longer -// in the cache is fine, with the only side effect being the memory overhead of -// there being more than one distinct reference to a certificate alive at once. -func (cc *certCache) active(e *cacheEntry) *activeCert { - e.refs.Add(1) - a := &activeCert{e.cert} - runtime.SetFinalizer(a, func(_ *activeCert) { - if e.refs.Add(-1) == 0 { - cc.evict(e) +func (wcc *weakCertCache) newCert(der []byte) (*x509.Certificate, error) { + if entry, ok := wcc.Load(string(der)); ok { + if v := entry.(weak.Pointer[x509.Certificate]).Value(); v != nil { + return v, nil } - }) - return a -} - -// evict removes a cacheEntry from the cache. -func (cc *certCache) evict(e *cacheEntry) { - cc.Delete(string(e.cert.Raw)) -} - -// newCert returns a x509.Certificate parsed from der. If there is already a copy -// of the certificate in the cache, a reference to the existing certificate will -// be returned. Otherwise, a fresh certificate will be added to the cache, and -// the reference returned. The returned reference should not be mutated. -func (cc *certCache) newCert(der []byte) (*activeCert, error) { - if entry, ok := cc.Load(string(der)); ok { - return cc.active(entry.(*cacheEntry)), nil } cert, err := x509.ParseCertificate(der) @@ -87,9 +28,17 @@ func (cc *certCache) newCert(der []byte) (*activeCert, error) { return nil, err } - entry := &cacheEntry{cert: cert} - if entry, loaded := cc.LoadOrStore(string(der), entry); loaded { - return cc.active(entry.(*cacheEntry)), nil + wp := weak.Make(cert) + if entry, loaded := wcc.LoadOrStore(string(der), wp); !loaded { + runtime.AddCleanup(cert, func(_ any) { wcc.CompareAndDelete(string(der), entry) }, any(string(der))) + } else if v := entry.(weak.Pointer[x509.Certificate]).Value(); v != nil { + return v, nil + } else { + if wcc.CompareAndSwap(string(der), entry, wp) { + runtime.AddCleanup(cert, func(_ any) { wcc.CompareAndDelete(string(der), wp) }, any(string(der))) + } } - return cc.active(entry), nil + return cert, nil } + +var globalCertCache = new(weakCertCache) diff --git a/src/crypto/tls/cache_test.go b/src/crypto/tls/cache_test.go index 2846734195d2d2..75a0508ec0defa 100644 --- a/src/crypto/tls/cache_test.go +++ b/src/crypto/tls/cache_test.go @@ -1,63 +1,60 @@ -// Copyright 2022 The Go Authors. All rights reserved. +// Copyright 2025 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. - package tls import ( "encoding/pem" - "fmt" "runtime" "testing" "time" ) -func TestCertCache(t *testing.T) { - cc := certCache{} +func TestWeakCertCache(t *testing.T) { + wcc := weakCertCache{} p, _ := pem.Decode([]byte(rsaCertPEM)) if p == nil { t.Fatal("Failed to decode certificate") } - certA, err := cc.newCert(p.Bytes) + certA, err := wcc.newCert(p.Bytes) if err != nil { t.Fatalf("newCert failed: %s", err) } - certB, err := cc.newCert(p.Bytes) + certB, err := wcc.newCert(p.Bytes) if err != nil { t.Fatalf("newCert failed: %s", err) } - if certA.cert != certB.cert { + if certA != certB { t.Fatal("newCert returned a unique reference for a duplicate certificate") } - if entry, ok := cc.Load(string(p.Bytes)); !ok { + if _, ok := wcc.Load(string(p.Bytes)); !ok { t.Fatal("cache does not contain expected entry") - } else { - if refs := entry.(*cacheEntry).refs.Load(); refs != 2 { - t.Fatalf("unexpected number of references: got %d, want 2", refs) - } } - timeoutRefCheck := func(t *testing.T, key string, count int64) { + timeoutRefCheck := func(t *testing.T, key string, present bool) { t.Helper() - c := time.After(4 * time.Second) + timeout := time.After(4 * time.Second) for { select { - case <-c: + case <-timeout: t.Fatal("timed out waiting for expected ref count") default: - e, ok := cc.Load(key) - if !ok && count != 0 { - t.Fatal("cache does not contain expected key") - } else if count == 0 && !ok { - return - } - - if e.(*cacheEntry).refs.Load() == count { + _, ok := wcc.Load(key) + if ok == present { return } } + // Explicitly yield to the scheduler. + // + // On single-threaded platforms like js/wasm a busy-loop might + // never call into the scheduler for the full timeout, meaning + // that if we arrive here and the cleanup hasn't already run, + // we'll simply loop until the timeout. Busy-loops put us at the + // mercy of the Go scheduler, making this test fragile on some + // platforms. + runtime.Gosched() } } @@ -68,7 +65,7 @@ func TestCertCache(t *testing.T) { certA = nil runtime.GC() - timeoutRefCheck(t, string(p.Bytes), 1) + timeoutRefCheck(t, string(p.Bytes), true) // Keep certB alive until at least now, so that we can // purposefully nil it and force the finalizer to be @@ -77,41 +74,5 @@ func TestCertCache(t *testing.T) { certB = nil runtime.GC() - timeoutRefCheck(t, string(p.Bytes), 0) -} - -func BenchmarkCertCache(b *testing.B) { - p, _ := pem.Decode([]byte(rsaCertPEM)) - if p == nil { - b.Fatal("Failed to decode certificate") - } - - cc := certCache{} - b.ReportAllocs() - b.ResetTimer() - // We expect that calling newCert additional times after - // the initial call should not cause additional allocations. - for extra := 0; extra < 4; extra++ { - b.Run(fmt.Sprint(extra), func(b *testing.B) { - actives := make([]*activeCert, extra+1) - b.ResetTimer() - for i := 0; i < b.N; i++ { - var err error - actives[0], err = cc.newCert(p.Bytes) - if err != nil { - b.Fatal(err) - } - for j := 0; j < extra; j++ { - actives[j+1], err = cc.newCert(p.Bytes) - if err != nil { - b.Fatal(err) - } - } - for j := 0; j < extra+1; j++ { - actives[j] = nil - } - runtime.GC() - } - }) - } + timeoutRefCheck(t, string(p.Bytes), false) } diff --git a/src/crypto/tls/cipher_suites.go b/src/crypto/tls/cipher_suites.go index eebc66880d631f..6ed63ccc2dc1f5 100644 --- a/src/crypto/tls/cipher_suites.go +++ b/src/crypto/tls/cipher_suites.go @@ -11,6 +11,8 @@ import ( "crypto/des" "crypto/hmac" "crypto/internal/boring" + fipsaes "crypto/internal/fips140/aes" + "crypto/internal/fips140/aes/gcm" "crypto/rc4" "crypto/sha1" "crypto/sha256" @@ -76,8 +78,8 @@ func CipherSuites() []*CipherSuite { // Most applications should not use the cipher suites in this list, and should // only use those returned by [CipherSuites]. func InsecureCipherSuites() []*CipherSuite { - // This list includes RC4, CBC_SHA256, and 3DES cipher suites. See - // cipherSuitesPreferenceOrder for details. + // This list includes legacy RSA kex, RC4, CBC_SHA256, and 3DES cipher + // suites. See cipherSuitesPreferenceOrder for details. return []*CipherSuite{ {TLS_RSA_WITH_RC4_128_SHA, "TLS_RSA_WITH_RC4_128_SHA", supportedUpToTLS12, true}, {TLS_RSA_WITH_3DES_EDE_CBC_SHA, "TLS_RSA_WITH_3DES_EDE_CBC_SHA", supportedUpToTLS12, true}, @@ -147,8 +149,8 @@ type cipherSuite struct { } var cipherSuites = []*cipherSuite{ // TODO: replace with a map, since the order doesn't matter. - {TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, 32, 0, 12, ecdheRSAKA, suiteECDHE | suiteTLS12, nil, nil, aeadChaCha20Poly1305}, - {TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, 32, 0, 12, ecdheECDSAKA, suiteECDHE | suiteECSign | suiteTLS12, nil, nil, aeadChaCha20Poly1305}, + {TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, 32, 0, 12, ecdheRSAKA, suiteECDHE | suiteTLS12, nil, nil, aeadChaCha20Poly1305}, + {TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, 32, 0, 12, ecdheECDSAKA, suiteECDHE | suiteECSign | suiteTLS12, nil, nil, aeadChaCha20Poly1305}, {TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, 16, 0, 4, ecdheRSAKA, suiteECDHE | suiteTLS12, nil, nil, aeadAESGCM}, {TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, 16, 0, 4, ecdheECDSAKA, suiteECDHE | suiteECSign | suiteTLS12, nil, nil, aeadAESGCM}, {TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, 32, 0, 4, ecdheRSAKA, suiteECDHE | suiteTLS12 | suiteSHA384, nil, nil, aeadAESGCM}, @@ -233,7 +235,7 @@ var cipherSuitesTLS13 = []*cipherSuiteTLS13{ // TODO: replace with a map. // - Anything else comes before CBC_SHA256 // // SHA-256 variants of the CBC ciphersuites don't implement any Lucky13 -// countermeasures. See http://www.isg.rhul.ac.uk/tls/Lucky13.html and +// countermeasures. See https://www.isg.rhul.ac.uk/tls/Lucky13.html and // https://www.imperialviolet.org/2013/02/04/luckythirteen.html. // // - Anything else comes before 3DES @@ -282,7 +284,7 @@ var cipherSuitesPreferenceOrder = []uint16{ // AEADs w/ ECDHE TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, // CBC w/ ECDHE TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, @@ -311,7 +313,7 @@ var cipherSuitesPreferenceOrder = []uint16{ var cipherSuitesPreferenceOrderNoAES = []uint16{ // ChaCha20Poly1305 - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, // AES-GCM w/ ECDHE TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, @@ -365,15 +367,13 @@ var tdesCiphers = map[uint16]bool{ } var ( - hasGCMAsmAMD64 = cpu.X86.HasAES && cpu.X86.HasPCLMULQDQ + // Keep in sync with crypto/internal/fips140/aes/gcm.supportsAESGCM. + hasGCMAsmAMD64 = cpu.X86.HasAES && cpu.X86.HasPCLMULQDQ && cpu.X86.HasSSE41 && cpu.X86.HasSSSE3 hasGCMAsmARM64 = cpu.ARM64.HasAES && cpu.ARM64.HasPMULL - // Keep in sync with crypto/aes/cipher_s390x.go. - hasGCMAsmS390X = cpu.S390X.HasAES && cpu.S390X.HasAESCBC && cpu.S390X.HasAESCTR && - (cpu.S390X.HasGHASH || cpu.S390X.HasAESGCM) + hasGCMAsmS390X = cpu.S390X.HasAES && cpu.S390X.HasAESCTR && cpu.S390X.HasGHASH + hasGCMAsmPPC64 = runtime.GOARCH == "ppc64" || runtime.GOARCH == "ppc64le" - hasAESGCMHardwareSupport = runtime.GOARCH == "amd64" && hasGCMAsmAMD64 || - runtime.GOARCH == "arm64" && hasGCMAsmARM64 || - runtime.GOARCH == "s390x" && hasGCMAsmS390X + hasAESGCMHardwareSupport = hasGCMAsmAMD64 || hasGCMAsmARM64 || hasGCMAsmS390X || hasGCMAsmPPC64 ) var aesgcmCiphers = map[uint16]bool{ @@ -387,9 +387,13 @@ var aesgcmCiphers = map[uint16]bool{ TLS_AES_256_GCM_SHA384: true, } -// aesgcmPreferred returns whether the first known cipher in the preference list -// is an AES-GCM cipher, implying the peer has hardware support for it. -func aesgcmPreferred(ciphers []uint16) bool { +// isAESGCMPreferred returns whether we have hardware support for AES-GCM, and the +// first known cipher in the peer's preference list is an AES-GCM cipher, +// implying the peer also has hardware support for it. +func isAESGCMPreferred(ciphers []uint16) bool { + if !hasAESGCMHardwareSupport { + return false + } for _, cID := range ciphers { if c := cipherSuiteByID(cID); c != nil { return aesgcmCiphers[cID] @@ -523,7 +527,7 @@ func aeadAESGCM(key, noncePrefix []byte) aead { aead, err = boring.NewGCMTLS(aes) } else { boring.Unreachable() - aead, err = cipher.NewGCM(aes) + aead, err = gcm.NewGCMForTLS12(aes.(*fipsaes.Block)) } if err != nil { panic(err) @@ -552,7 +556,13 @@ func aeadAESGCMTLS13(key, nonceMask []byte) aead { if err != nil { panic(err) } - aead, err := cipher.NewGCM(aes) + var aead cipher.AEAD + if boring.Enabled { + aead, err = boring.NewGCMTLS13(aes) + } else { + boring.Unreachable() + aead, err = gcm.NewGCMForTLS13(aes.(*fipsaes.Block)) + } if err != nil { panic(err) } diff --git a/src/crypto/tls/common.go b/src/crypto/tls/common.go index 5fd92d3c639b53..6fe6f34cd210d8 100644 --- a/src/crypto/tls/common.go +++ b/src/crypto/tls/common.go @@ -15,6 +15,7 @@ import ( "crypto/rand" "crypto/rsa" "crypto/sha512" + "crypto/tls/internal/fips140tls" "crypto/x509" "errors" "fmt" @@ -144,17 +145,21 @@ const ( type CurveID uint16 const ( - CurveP256 CurveID = 23 - CurveP384 CurveID = 24 - CurveP521 CurveID = 25 - X25519 CurveID = 29 - - // Experimental codepoint for X25519Kyber768Draft00, specified in - // draft-tls-westerbaan-xyber768d00-03. Not exported, as support might be - // removed in the future. - x25519Kyber768Draft00 CurveID = 0x6399 // X25519Kyber768Draft00 + CurveP256 CurveID = 23 + CurveP384 CurveID = 24 + CurveP521 CurveID = 25 + X25519 CurveID = 29 + X25519MLKEM768 CurveID = 4588 ) +func isTLS13OnlyKeyExchange(curve CurveID) bool { + return curve == X25519MLKEM768 +} + +func isPQKeyExchange(curve CurveID) bool { + return curve == X25519MLKEM768 +} + // TLS 1.3 Key Share. See RFC 8446, Section 4.2.8. type keyShare struct { group CurveID @@ -242,6 +247,11 @@ type ConnectionState struct { // TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_AES_128_GCM_SHA256). CipherSuite uint16 + // CurveID is the key exchange mechanism used for the connection. The name + // refers to elliptic curves for legacy reasons, see [CurveID]. If a legacy + // RSA key exchange is used, this value is zero. + CurveID CurveID + // NegotiatedProtocol is the application protocol negotiated with ALPN. NegotiatedProtocol string @@ -300,9 +310,9 @@ type ConnectionState struct { // testingOnlyDidHRR is true if a HelloRetryRequest was sent/received. testingOnlyDidHRR bool - // testingOnlyCurveID is the selected CurveID, or zero if an RSA exchanges - // is performed. - testingOnlyCurveID CurveID + // testingOnlyPeerSignatureAlgorithm is the signature algorithm used by the + // peer to sign the handshake. It is not set for resumed connections. + testingOnlyPeerSignatureAlgorithm SignatureScheme } // ExportKeyingMaterial returns length bytes of exported key material in a new @@ -418,9 +428,12 @@ type ClientHelloInfo struct { // client is using SNI (see RFC 4366, Section 3.1). ServerName string - // SupportedCurves lists the elliptic curves supported by the client. - // SupportedCurves is set only if the Supported Elliptic Curves - // Extension is being used (see RFC 4492, Section 5.1.1). + // SupportedCurves lists the key exchange mechanisms supported by the + // client. It was renamed to "supported groups" in TLS 1.3, see RFC 8446, + // Section 4.2.7 and [CurveID]. + // + // SupportedCurves may be nil in TLS 1.2 and lower if the Supported Elliptic + // Curves Extension is not being used (see RFC 4492, Section 5.1.1). SupportedCurves []CurveID // SupportedPoints lists the point formats supported by the client. @@ -447,6 +460,10 @@ type ClientHelloInfo struct { // might be rejected if used. SupportedVersions []uint16 + // Extensions lists the IDs of the extensions presented by the client + // in the ClientHello. + Extensions []uint16 + // Conn is the underlying net.Conn for the connection. Do not read // from, or write to, this connection; that will cause the TLS // connection to fail. @@ -756,14 +773,15 @@ type Config struct { // which is currently TLS 1.3. MaxVersion uint16 - // CurvePreferences contains the elliptic curves that will be used in - // an ECDHE handshake, in preference order. If empty, the default will - // be used. The client will use the first preference as the type for - // its key share in TLS 1.3. This may change in the future. + // CurvePreferences contains a set of supported key exchange mechanisms. + // The name refers to elliptic curves for legacy reasons, see [CurveID]. + // The order of the list is ignored, and key exchange mechanisms are chosen + // from this list using an internal preference order. If empty, the default + // will be used. // - // From Go 1.23, the default includes the X25519Kyber768Draft00 hybrid + // From Go 1.24, the default includes the [X25519MLKEM768] hybrid // post-quantum key exchange. To disable it, set CurvePreferences explicitly - // or use the GODEBUG=tlskyber=0 environment variable. + // or use the GODEBUG=tlsmlkem=0 environment variable. CurvePreferences []CurveID // DynamicRecordSizingDisabled disables adaptive sizing of TLS records. @@ -786,8 +804,10 @@ type Config struct { // EncryptedClientHelloConfigList is a serialized ECHConfigList. If // provided, clients will attempt to connect to servers using Encrypted - // Client Hello (ECH) using one of the provided ECHConfigs. Servers - // currently ignore this field. + // Client Hello (ECH) using one of the provided ECHConfigs. + // + // Servers do not use this field. In order to configure ECH for servers, see + // the EncryptedClientHelloKeys field. // // If the list contains no valid ECH configs, the handshake will fail // and return an error. @@ -796,7 +816,7 @@ type Config struct { // be VersionTLS13. // // When EncryptedClientHelloConfigList is set, the handshake will only - // succeed if ECH is sucessfully negotiated. If the server rejects ECH, + // succeed if ECH is successfully negotiated. If the server rejects ECH, // an ECHRejectionError error will be returned, which may contain a new // ECHConfigList that the server suggests using. // @@ -805,9 +825,11 @@ type Config struct { EncryptedClientHelloConfigList []byte // EncryptedClientHelloRejectionVerify, if not nil, is called when ECH is - // rejected, in order to verify the ECH provider certificate in the outer - // Client Hello. If it returns a non-nil error, the handshake is aborted and - // that error results. + // rejected by the remote server, in order to verify the ECH provider + // certificate in the outer ClientHello. If it returns a non-nil error, the + // handshake is aborted and that error results. + // + // On the server side this field is not used. // // Unlike VerifyPeerCertificate and VerifyConnection, normal certificate // verification will not be performed before calling @@ -819,6 +841,37 @@ type Config struct { // when ECH is rejected, even if set, and InsecureSkipVerify is ignored. EncryptedClientHelloRejectionVerify func(ConnectionState) error + // GetEncryptedClientHelloKeys, if not nil, is called when by a server when + // a client attempts ECH. + // + // If GetEncryptedClientHelloKeys is not nil, [EncryptedClientHelloKeys] is + // ignored. + // + // If GetEncryptedClientHelloKeys returns an error, the handshake will be + // aborted and the error will be returned. Otherwise, + // GetEncryptedClientHelloKeys must return a non-nil slice of + // [EncryptedClientHelloKey] that represents the acceptable ECH keys. + // + // For further details, see [EncryptedClientHelloKeys]. + GetEncryptedClientHelloKeys func(*ClientHelloInfo) ([]EncryptedClientHelloKey, error) + + // EncryptedClientHelloKeys are the ECH keys to use when a client + // attempts ECH. + // + // If EncryptedClientHelloKeys is set, MinVersion, if set, must be + // VersionTLS13. + // + // If a client attempts ECH, but it is rejected by the server, the server + // will send a list of configs to retry based on the set of + // EncryptedClientHelloKeys which have the SendAsRetry field set. + // + // If GetEncryptedClientHelloKeys is non-nil, EncryptedClientHelloKeys is + // ignored. + // + // On the client side, this field is ignored. In order to configure ECH for + // clients, see the EncryptedClientHelloConfigList field. + EncryptedClientHelloKeys []EncryptedClientHelloKey + // mutex protects sessionTicketKeys and autoSessionTicketKeys. mutex sync.RWMutex // sessionTicketKeys contains zero or more ticket keys. If set, it means @@ -832,6 +885,24 @@ type Config struct { autoSessionTicketKeys []ticketKey } +// EncryptedClientHelloKey holds a private key that is associated +// with a specific ECH config known to a client. +type EncryptedClientHelloKey struct { + // Config should be a marshalled ECHConfig associated with PrivateKey. This + // must match the config provided to clients byte-for-byte. The config + // should only specify the DHKEM(X25519, HKDF-SHA256) KEM ID (0x0020), the + // HKDF-SHA256 KDF ID (0x0001), and a subset of the following AEAD IDs: + // AES-128-GCM (0x0001), AES-256-GCM (0x0002), ChaCha20Poly1305 (0x0003). + Config []byte + // PrivateKey should be a marshalled private key. Currently, we expect + // this to be the output of [ecdh.PrivateKey.Bytes]. + PrivateKey []byte + // SendAsRetry indicates if Config should be sent as part of the list of + // retry configs when ECH is requested by the client but rejected by the + // server. + SendAsRetry bool +} + const ( // ticketKeyLifetime is how long a ticket key remains valid and can be used to // resume a client connection. @@ -885,6 +956,7 @@ func (c *Config) Clone() *Config { GetCertificate: c.GetCertificate, GetClientCertificate: c.GetClientCertificate, GetConfigForClient: c.GetConfigForClient, + GetEncryptedClientHelloKeys: c.GetEncryptedClientHelloKeys, VerifyPeerCertificate: c.VerifyPeerCertificate, VerifyConnection: c.VerifyConnection, RootCAs: c.RootCAs, @@ -908,6 +980,7 @@ func (c *Config) Clone() *Config { KeyLogWriter: c.KeyLogWriter, EncryptedClientHelloConfigList: c.EncryptedClientHelloConfigList, EncryptedClientHelloRejectionVerify: c.EncryptedClientHelloRejectionVerify, + EncryptedClientHelloKeys: c.EncryptedClientHelloKeys, sessionTicketKeys: c.sessionTicketKeys, autoSessionTicketKeys: c.autoSessionTicketKeys, } @@ -962,6 +1035,7 @@ func (c *Config) ticketKeys(configForClient *Config) []ticketKey { if configForClient != nil { configForClient.mutex.RLock() if configForClient.SessionTicketsDisabled { + configForClient.mutex.RUnlock() return nil } configForClient.initLegacySessionTicketKeyRLocked() @@ -1055,20 +1129,28 @@ func (c *Config) time() time.Time { return t() } -func (c *Config) cipherSuites() []uint16 { +func (c *Config) cipherSuites(aesGCMPreferred bool) []uint16 { + var cipherSuites []uint16 if c.CipherSuites == nil { - if needFIPS() { - return defaultCipherSuitesFIPS - } - return defaultCipherSuites() + cipherSuites = defaultCipherSuites(aesGCMPreferred) + } else { + cipherSuites = supportedCipherSuites(aesGCMPreferred) + cipherSuites = slices.DeleteFunc(cipherSuites, func(id uint16) bool { + return !slices.Contains(c.CipherSuites, id) + }) } - if needFIPS() { - cipherSuites := slices.Clone(c.CipherSuites) - return slices.DeleteFunc(cipherSuites, func(id uint16) bool { - return !slices.Contains(defaultCipherSuitesFIPS, id) + if fips140tls.Required() { + cipherSuites = slices.DeleteFunc(cipherSuites, func(id uint16) bool { + return !slices.Contains(allowedCipherSuitesFIPS, id) }) } - return c.CipherSuites + return cipherSuites +} + +// supportedCipherSuites returns the supported TLS 1.0–1.2 cipher suites in an +// undefined order. For preference ordering, use [Config.cipherSuites]. +func (c *Config) supportedCipherSuites() []uint16 { + return c.cipherSuites(false) } var supportedVersions = []uint16{ @@ -1085,10 +1167,12 @@ const roleServer = false var tls10server = godebug.New("tls10server") +// supportedVersions returns the list of supported TLS versions, sorted from +// highest to lowest (and hence also in preference order). func (c *Config) supportedVersions(isClient bool) []uint16 { versions := make([]uint16, 0, len(supportedVersions)) for _, v := range supportedVersions { - if needFIPS() && !slices.Contains(defaultSupportedVersionsFIPS, v) { + if fips140tls.Required() && !slices.Contains(allowedSupportedVersionsFIPS, v) { continue } if (c == nil || c.MinVersion == 0) && v < VersionTLS12 { @@ -1133,45 +1217,34 @@ func supportedVersionsFromMax(maxVersion uint16) []uint16 { } func (c *Config) curvePreferences(version uint16) []CurveID { - var curvePreferences []CurveID + curvePreferences := defaultCurvePreferences() + if fips140tls.Required() { + curvePreferences = slices.DeleteFunc(curvePreferences, func(x CurveID) bool { + return !slices.Contains(allowedCurvePreferencesFIPS, x) + }) + } if c != nil && len(c.CurvePreferences) != 0 { - curvePreferences = slices.Clone(c.CurvePreferences) - if needFIPS() { - return slices.DeleteFunc(curvePreferences, func(c CurveID) bool { - return !slices.Contains(defaultCurvePreferencesFIPS, c) - }) - } - } else if needFIPS() { - curvePreferences = slices.Clone(defaultCurvePreferencesFIPS) - } else { - curvePreferences = defaultCurvePreferences() + curvePreferences = slices.DeleteFunc(curvePreferences, func(x CurveID) bool { + return !slices.Contains(c.CurvePreferences, x) + }) } if version < VersionTLS13 { - return slices.DeleteFunc(curvePreferences, func(c CurveID) bool { - return c == x25519Kyber768Draft00 - }) + curvePreferences = slices.DeleteFunc(curvePreferences, isTLS13OnlyKeyExchange) } return curvePreferences } func (c *Config) supportsCurve(version uint16, curve CurveID) bool { - for _, cc := range c.curvePreferences(version) { - if cc == curve { - return true - } - } - return false + return slices.Contains(c.curvePreferences(version), curve) } // mutualVersion returns the protocol version to use given the advertised -// versions of the peer. Priority is given to the peer preference order. +// versions of the peer. The highest supported version is preferred. func (c *Config) mutualVersion(isClient bool, peerVersions []uint16) (uint16, bool) { supportedVersions := c.supportedVersions(isClient) - for _, peerVersion := range peerVersions { - for _, v := range supportedVersions { - if v == peerVersion { - return v, true - } + for _, v := range supportedVersions { + if slices.Contains(peerVersions, v) { + return v, true } } return 0, false @@ -1292,7 +1365,7 @@ func (chi *ClientHelloInfo) SupportsCertificate(c *Certificate) error { } // Finally, there needs to be a mutual cipher suite that uses the static // RSA key exchange instead of ECDHE. - rsaCipherSuite := selectCipherSuite(chi.CipherSuites, config.cipherSuites(), func(c *cipherSuite) bool { + rsaCipherSuite := selectCipherSuite(chi.CipherSuites, config.supportedCipherSuites(), func(c *cipherSuite) bool { if c.flags&suiteECDHE != 0 { return false } @@ -1323,7 +1396,11 @@ func (chi *ClientHelloInfo) SupportsCertificate(c *Certificate) error { } // The only signed key exchange we support is ECDHE. - if !supportsECDHE(config, vers, chi.SupportedCurves, chi.SupportedPoints) { + ecdheSupported, err := supportsECDHE(config, vers, chi.SupportedCurves, chi.SupportedPoints) + if err != nil { + return err + } + if !ecdheSupported { return supportsRSAFallback(errors.New("client doesn't support ECDHE, can only use legacy RSA key exchange")) } @@ -1369,7 +1446,7 @@ func (chi *ClientHelloInfo) SupportsCertificate(c *Certificate) error { // Make sure that there is a mutually supported cipher suite that works with // this certificate. Cipher suite selection will then apply the logic in // reverse to pick it. See also serverHandshakeState.cipherSuiteOk. - cipherSuite := selectCipherSuite(chi.CipherSuites, config.cipherSuites(), func(c *cipherSuite) bool { + cipherSuite := selectCipherSuite(chi.CipherSuites, config.supportedCipherSuites(), func(c *cipherSuite) bool { if c.flags&suiteECDHE == 0 { return false } @@ -1611,23 +1688,68 @@ func unexpectedMessageError(wanted, got any) error { return fmt.Errorf("tls: received unexpected handshake message of type %T when waiting for %T", got, wanted) } -// supportedSignatureAlgorithms returns the supported signature algorithms. -func supportedSignatureAlgorithms() []SignatureScheme { - if !needFIPS() { - return defaultSupportedSignatureAlgorithms +var testingOnlySupportedSignatureAlgorithms []SignatureScheme + +// supportedSignatureAlgorithms returns the supported signature algorithms for +// the given minimum TLS version, to advertise in ClientHello and +// CertificateRequest messages. +func supportedSignatureAlgorithms(minVers uint16) []SignatureScheme { + sigAlgs := defaultSupportedSignatureAlgorithms() + if testingOnlySupportedSignatureAlgorithms != nil { + sigAlgs = slices.Clone(testingOnlySupportedSignatureAlgorithms) } - return defaultSupportedSignatureAlgorithmsFIPS + return slices.DeleteFunc(sigAlgs, func(s SignatureScheme) bool { + return isDisabledSignatureAlgorithm(minVers, s, false) + }) } -func isSupportedSignatureAlgorithm(sigAlg SignatureScheme, supportedSignatureAlgorithms []SignatureScheme) bool { - for _, s := range supportedSignatureAlgorithms { - if s == sigAlg { +var tlssha1 = godebug.New("tlssha1") + +func isDisabledSignatureAlgorithm(version uint16, s SignatureScheme, isCert bool) bool { + if fips140tls.Required() && !slices.Contains(allowedSignatureAlgorithmsFIPS, s) { + return true + } + + // For the _cert extension we include all algorithms, including SHA-1 and + // PKCS#1 v1.5, because it's more likely that something on our side will be + // willing to accept a *-with-SHA1 certificate (e.g. with a custom + // VerifyConnection or by a direct match with the CertPool), than that the + // peer would have a better certificate but is just choosing not to send it. + // crypto/x509 will refuse to verify important SHA-1 signatures anyway. + if isCert { + return false + } + + // TLS 1.3 removed support for PKCS#1 v1.5 and SHA-1 signatures, + // and Go 1.25 removed support for SHA-1 signatures in TLS 1.2. + if version > VersionTLS12 { + sigType, sigHash, _ := typeAndHashFromSignatureScheme(s) + if sigType == signaturePKCS1v15 || sigHash == crypto.SHA1 { + return true + } + } else if tlssha1.Value() != "1" { + _, sigHash, _ := typeAndHashFromSignatureScheme(s) + if sigHash == crypto.SHA1 { return true } } + return false } +// supportedSignatureAlgorithmsCert returns the supported algorithms for +// signatures in certificates. +func supportedSignatureAlgorithmsCert() []SignatureScheme { + sigAlgs := defaultSupportedSignatureAlgorithms() + return slices.DeleteFunc(sigAlgs, func(s SignatureScheme) bool { + return isDisabledSignatureAlgorithm(0, s, true) + }) +} + +func isSupportedSignatureAlgorithm(sigAlg SignatureScheme, supportedSignatureAlgorithms []SignatureScheme) bool { + return slices.Contains(supportedSignatureAlgorithms, sigAlg) +} + // CertificateVerificationError is returned when certificate verification fails during the handshake. type CertificateVerificationError struct { // UnverifiedCertificates and its contents should not be modified. @@ -1642,3 +1764,42 @@ func (e *CertificateVerificationError) Error() string { func (e *CertificateVerificationError) Unwrap() error { return e.Err } + +// fipsAllowedChains returns chains that are allowed to be used in a TLS connection +// based on the current fips140tls enforcement setting. +// +// If fips140tls is not required, the chains are returned as-is with no processing. +// Otherwise, the returned chains are filtered to only those allowed by FIPS 140-3. +// If this results in no chains it returns an error. +func fipsAllowedChains(chains [][]*x509.Certificate) ([][]*x509.Certificate, error) { + if !fips140tls.Required() { + return chains, nil + } + + permittedChains := make([][]*x509.Certificate, 0, len(chains)) + for _, chain := range chains { + if fipsAllowChain(chain) { + permittedChains = append(permittedChains, chain) + } + } + + if len(permittedChains) == 0 { + return nil, errors.New("tls: no FIPS compatible certificate chains found") + } + + return permittedChains, nil +} + +func fipsAllowChain(chain []*x509.Certificate) bool { + if len(chain) == 0 { + return false + } + + for _, cert := range chain { + if !isCertificateAllowedFIPS(cert) { + return false + } + } + + return true +} diff --git a/src/crypto/tls/common_string.go b/src/crypto/tls/common_string.go index 1752f810505bd1..e15dd48838b530 100644 --- a/src/crypto/tls/common_string.go +++ b/src/crypto/tls/common_string.go @@ -71,13 +71,13 @@ func _() { _ = x[CurveP384-24] _ = x[CurveP521-25] _ = x[X25519-29] - _ = x[x25519Kyber768Draft00-25497] + _ = x[X25519MLKEM768-4588] } const ( _CurveID_name_0 = "CurveP256CurveP384CurveP521" _CurveID_name_1 = "X25519" - _CurveID_name_2 = "X25519Kyber768Draft00" + _CurveID_name_2 = "X25519MLKEM768" ) var ( @@ -91,7 +91,7 @@ func (i CurveID) String() string { return _CurveID_name_0[_CurveID_index_0[i]:_CurveID_index_0[i+1]] case i == 29: return _CurveID_name_1 - case i == 25497: + case i == 4588: return _CurveID_name_2 default: return "CurveID(" + strconv.FormatInt(int64(i), 10) + ")" diff --git a/src/crypto/tls/conn.go b/src/crypto/tls/conn.go index bdbc2bde416ac0..2de120a1329f87 100644 --- a/src/crypto/tls/conn.go +++ b/src/crypto/tls/conn.go @@ -51,12 +51,10 @@ type Conn struct { didHRR bool // whether a HelloRetryRequest was sent/received cipherSuite uint16 curveID CurveID + peerSigAlg SignatureScheme ocspResponse []byte // stapled OCSP response scts [][]byte // signed certificate timestamps from server peerCertificates []*x509.Certificate - // activeCertHandles contains the cache handles to certificates in - // peerCertificates that are used to track active references. - activeCertHandles []*activeCert // verifiedChains contains the certificate chains that we built, as // opposed to the ones presented by the server. verifiedChains [][]*x509.Certificate @@ -222,9 +220,7 @@ func (hc *halfConn) changeCipherSpec() error { hc.mac = hc.nextMac hc.nextCipher = nil hc.nextMac = nil - for i := range hc.seq { - hc.seq[i] = 0 - } + clear(hc.seq[:]) return nil } @@ -233,9 +229,7 @@ func (hc *halfConn) setTrafficSecret(suite *cipherSuiteTLS13, level QUICEncrypti hc.level = level key, iv := suite.trafficKey(secret) hc.cipher = suite.aead(key, iv) - for i := range hc.seq { - hc.seq[i] = 0 - } + clear(hc.seq[:]) } // incSeq increments the sequence number. @@ -724,6 +718,15 @@ func (c *Conn) readRecordOrCCS(expectChangeCipherSpec bool) error { return c.in.setErrorLocked(io.EOF) } if c.vers == VersionTLS13 { + // TLS 1.3 removed warning-level alerts except for alertUserCanceled + // (RFC 8446, § 6.1). Since at least one major implementation + // (https://bugs.openjdk.org/browse/JDK-8323517) misuses this alert, + // many TLS stacks now ignore it outright when seen in a TLS 1.3 + // handshake (e.g. BoringSSL, NSS, Rustls). + if alert(data[1]) == alertUserCanceled { + // Like TLS 1.2 alertLevelWarning alerts, we drop the record and retry. + return c.retryReadRecord(expectChangeCipherSpec) + } return c.in.setErrorLocked(&net.OpError{Op: "remote error", Err: alert(data[1])}) } switch data[0] { @@ -1173,7 +1176,7 @@ func (c *Conn) unmarshalHandshakeMessage(data []byte, transcript transcriptHash) data = append([]byte(nil), data...) if !m.unmarshal(data) { - return nil, c.in.setErrorLocked(c.sendAlert(alertUnexpectedMessage)) + return nil, c.in.setErrorLocked(c.sendAlert(alertDecodeError)) } if transcript != nil { @@ -1517,7 +1520,7 @@ func (c *Conn) handshakeContext(ctx context.Context) (ret error) { } handshakeCtx, cancel := context.WithCancel(ctx) - // Note: defer this before starting the "interrupter" goroutine + // Note: defer this before calling context.AfterFunc // so that we can tell the difference between the input being canceled and // this cancellation. In the former case, we need to close the connection. defer cancel() @@ -1526,28 +1529,14 @@ func (c *Conn) handshakeContext(ctx context.Context) (ret error) { c.quic.cancelc = handshakeCtx.Done() c.quic.cancel = cancel } else if ctx.Done() != nil { - // Start the "interrupter" goroutine, if this context might be canceled. - // (The background context cannot). - // - // The interrupter goroutine waits for the input context to be done and - // closes the connection if this happens before the function returns. - done := make(chan struct{}) - interruptRes := make(chan error, 1) + // Close the connection if ctx is canceled before the function returns. + stop := context.AfterFunc(ctx, func() { + _ = c.conn.Close() + }) defer func() { - close(done) - if ctxErr := <-interruptRes; ctxErr != nil { + if !stop() { // Return context error to user. - ret = ctxErr - } - }() - go func() { - select { - case <-handshakeCtx.Done(): - // Close the connection, discarding the error - _ = c.conn.Close() - interruptRes <- handshakeCtx.Err() - case <-done: - interruptRes <- nil + ret = ctx.Err() } }() } @@ -1589,9 +1578,9 @@ func (c *Conn) handshakeContext(ctx context.Context) (ret error) { // the handshake (RFC 9001, Section 5.7). c.quicSetReadSecret(QUICEncryptionLevelApplication, c.cipherSuite, c.in.trafficSecret) } else { - var a alert c.out.Lock() - if !errors.As(c.out.err, &a) { + a, ok := errors.AsType[alert](c.out.err) + if !ok { a = alertInternalError } c.out.Unlock() @@ -1624,8 +1613,8 @@ func (c *Conn) connectionStateLocked() ConnectionState { state.NegotiatedProtocol = c.clientProtocol state.DidResume = c.didResume state.testingOnlyDidHRR = c.didHRR - // c.curveID is not set on TLS 1.0–1.2 resumptions. Fix that before exposing it. - state.testingOnlyCurveID = c.curveID + state.testingOnlyPeerSignatureAlgorithm = c.peerSigAlg + state.CurveID = c.curveID state.NegotiatedProtocolIsMutual = true state.ServerName = c.serverName state.CipherSuite = c.cipherSuite diff --git a/src/crypto/tls/conn_test.go b/src/crypto/tls/conn_test.go index 5e090a017dc8e4..5fd48d6bd5af66 100644 --- a/src/crypto/tls/conn_test.go +++ b/src/crypto/tls/conn_test.go @@ -230,6 +230,8 @@ func runDynamicRecordSizingTest(t *testing.T, config *Config) { } func TestDynamicRecordSizingWithStreamCipher(t *testing.T) { + skipFIPS(t) // No RC4 in FIPS mode. + config := testConfig.Clone() config.MaxVersion = VersionTLS12 config.CipherSuites = []uint16{TLS_RSA_WITH_RC4_128_SHA} @@ -237,6 +239,8 @@ func TestDynamicRecordSizingWithStreamCipher(t *testing.T) { } func TestDynamicRecordSizingWithCBC(t *testing.T) { + skipFIPS(t) // No CBC cipher suites in defaultCipherSuitesFIPS. + config := testConfig.Clone() config.MaxVersion = VersionTLS12 config.CipherSuites = []uint16{TLS_RSA_WITH_AES_256_CBC_SHA} diff --git a/src/crypto/tls/defaults.go b/src/crypto/tls/defaults.go index 9b28acdc2d866a..489a2750dff390 100644 --- a/src/crypto/tls/defaults.go +++ b/src/crypto/tls/defaults.go @@ -13,41 +13,52 @@ import ( // Defaults are collected in this file to allow distributions to more easily patch // them to apply local policies. -var tlskyber = godebug.New("tlskyber") +var tlsmlkem = godebug.New("tlsmlkem") +// defaultCurvePreferences is the default set of supported key exchanges, as +// well as the preference order. func defaultCurvePreferences() []CurveID { - if tlskyber.Value() == "0" { + if tlsmlkem.Value() == "0" { return []CurveID{X25519, CurveP256, CurveP384, CurveP521} } - // For now, x25519Kyber768Draft00 must always be followed by X25519. - return []CurveID{x25519Kyber768Draft00, X25519, CurveP256, CurveP384, CurveP521} + return []CurveID{X25519MLKEM768, X25519, CurveP256, CurveP384, CurveP521} } -// defaultSupportedSignatureAlgorithms contains the signature and hash algorithms that -// the code advertises as supported in a TLS 1.2+ ClientHello and in a TLS 1.2+ +// defaultSupportedSignatureAlgorithms returns the signature and hash algorithms that +// the code advertises and supports in a TLS 1.2+ ClientHello and in a TLS 1.2+ // CertificateRequest. The two fields are merged to match with TLS 1.3. // Note that in TLS 1.2, the ECDSA algorithms are not constrained to P-256, etc. -var defaultSupportedSignatureAlgorithms = []SignatureScheme{ - PSSWithSHA256, - ECDSAWithP256AndSHA256, - Ed25519, - PSSWithSHA384, - PSSWithSHA512, - PKCS1WithSHA256, - PKCS1WithSHA384, - PKCS1WithSHA512, - ECDSAWithP384AndSHA384, - ECDSAWithP521AndSHA512, - PKCS1WithSHA1, - ECDSAWithSHA1, +func defaultSupportedSignatureAlgorithms() []SignatureScheme { + return []SignatureScheme{ + PSSWithSHA256, + ECDSAWithP256AndSHA256, + Ed25519, + PSSWithSHA384, + PSSWithSHA512, + PKCS1WithSHA256, + PKCS1WithSHA384, + PKCS1WithSHA512, + ECDSAWithP384AndSHA384, + ECDSAWithP521AndSHA512, + PKCS1WithSHA1, + ECDSAWithSHA1, + } } var tlsrsakex = godebug.New("tlsrsakex") var tls3des = godebug.New("tls3des") -func defaultCipherSuites() []uint16 { - suites := slices.Clone(cipherSuitesPreferenceOrder) - return slices.DeleteFunc(suites, func(c uint16) bool { +func supportedCipherSuites(aesGCMPreferred bool) []uint16 { + if aesGCMPreferred { + return slices.Clone(cipherSuitesPreferenceOrder) + } else { + return slices.Clone(cipherSuitesPreferenceOrderNoAES) + } +} + +func defaultCipherSuites(aesGCMPreferred bool) []uint16 { + cipherSuites := supportedCipherSuites(aesGCMPreferred) + return slices.DeleteFunc(cipherSuites, func(c uint16) bool { return disabledCipherSuites[c] || tlsrsakex.Value() != "1" && rsaKexCiphers[c] || tls3des.Value() != "1" && tdesCiphers[c] @@ -89,41 +100,3 @@ var defaultCipherSuitesTLS13NoAES = []uint16{ TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384, } - -var defaultSupportedVersionsFIPS = []uint16{ - VersionTLS12, -} - -// defaultCurvePreferencesFIPS are the FIPS-allowed curves, -// in preference order (most preferable first). -var defaultCurvePreferencesFIPS = []CurveID{CurveP256, CurveP384, CurveP521} - -// defaultSupportedSignatureAlgorithmsFIPS currently are a subset of -// defaultSupportedSignatureAlgorithms without Ed25519 and SHA-1. -var defaultSupportedSignatureAlgorithmsFIPS = []SignatureScheme{ - PSSWithSHA256, - PSSWithSHA384, - PSSWithSHA512, - PKCS1WithSHA256, - ECDSAWithP256AndSHA256, - PKCS1WithSHA384, - ECDSAWithP384AndSHA384, - PKCS1WithSHA512, - ECDSAWithP521AndSHA512, -} - -// defaultCipherSuitesFIPS are the FIPS-allowed cipher suites. -var defaultCipherSuitesFIPS = []uint16{ - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - TLS_RSA_WITH_AES_128_GCM_SHA256, - TLS_RSA_WITH_AES_256_GCM_SHA384, -} - -// defaultCipherSuitesTLS13FIPS are the FIPS-allowed cipher suites for TLS 1.3. -var defaultCipherSuitesTLS13FIPS = []uint16{ - TLS_AES_128_GCM_SHA256, - TLS_AES_256_GCM_SHA384, -} diff --git a/src/crypto/tls/defaults_boring.go b/src/crypto/tls/defaults_boring.go new file mode 100644 index 00000000000000..e88f05cc505cfb --- /dev/null +++ b/src/crypto/tls/defaults_boring.go @@ -0,0 +1,69 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build boringcrypto + +package tls + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rsa" + "crypto/x509" +) + +// These Go+BoringCrypto policies mostly match BoringSSL's +// ssl_compliance_policy_fips_202205, which is based on NIST SP 800-52r2. +// https://cs.opensource.google/boringssl/boringssl/+/master:ssl/ssl_lib.cc;l=3289;drc=ea7a88fa +// +// P-521 is allowed per https://go.dev/issue/71757. +// +// They are applied when crypto/tls/fipsonly is imported with GOEXPERIMENT=boringcrypto. + +var ( + allowedSupportedVersionsFIPS = []uint16{ + VersionTLS12, + VersionTLS13, + } + allowedCurvePreferencesFIPS = []CurveID{ + CurveP256, + CurveP384, + CurveP521, + } + allowedSignatureAlgorithmsFIPS = []SignatureScheme{ + PSSWithSHA256, + PSSWithSHA384, + PSSWithSHA512, + PKCS1WithSHA256, + ECDSAWithP256AndSHA256, + PKCS1WithSHA384, + ECDSAWithP384AndSHA384, + PKCS1WithSHA512, + ECDSAWithP521AndSHA512, + } + allowedCipherSuitesFIPS = []uint16{ + TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + } + allowedCipherSuitesTLS13FIPS = []uint16{ + TLS_AES_128_GCM_SHA256, + TLS_AES_256_GCM_SHA384, + } +) + +func isCertificateAllowedFIPS(c *x509.Certificate) bool { + // The key must be RSA 2048, RSA 3072, RSA 4096, + // or ECDSA P-256, P-384, P-521. + switch k := c.PublicKey.(type) { + case *rsa.PublicKey: + size := k.N.BitLen() + return size == 2048 || size == 3072 || size == 4096 + case *ecdsa.PublicKey: + return k.Curve == elliptic.P256() || k.Curve == elliptic.P384() || k.Curve == elliptic.P521() + } + + return false +} diff --git a/src/crypto/tls/defaults_fips140.go b/src/crypto/tls/defaults_fips140.go new file mode 100644 index 00000000000000..00176795eba6b7 --- /dev/null +++ b/src/crypto/tls/defaults_fips140.go @@ -0,0 +1,76 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !boringcrypto + +package tls + +import ( + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rsa" + "crypto/x509" +) + +// These FIPS 140-3 policies allow anything approved by SP 800-140C +// and SP 800-140D, and tested as part of the Go Cryptographic Module. +// +// Notably, not SHA-1, 3DES, RC4, ChaCha20Poly1305, RSA PKCS #1 v1.5 key +// transport, or TLS 1.0—1.1 (because we don't test its KDF). +// +// These are not default lists, but filters to apply to the default or +// configured lists. Missing items are treated as if they were not implemented. +// +// They are applied when the fips140 GODEBUG is "on" or "only". + +var ( + allowedSupportedVersionsFIPS = []uint16{ + VersionTLS12, + VersionTLS13, + } + allowedCurvePreferencesFIPS = []CurveID{ + X25519MLKEM768, + CurveP256, + CurveP384, + CurveP521, + } + allowedSignatureAlgorithmsFIPS = []SignatureScheme{ + PSSWithSHA256, + ECDSAWithP256AndSHA256, + Ed25519, + PSSWithSHA384, + PSSWithSHA512, + PKCS1WithSHA256, + PKCS1WithSHA384, + PKCS1WithSHA512, + ECDSAWithP384AndSHA384, + ECDSAWithP521AndSHA512, + } + allowedCipherSuitesFIPS = []uint16{ + TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, + } + allowedCipherSuitesTLS13FIPS = []uint16{ + TLS_AES_128_GCM_SHA256, + TLS_AES_256_GCM_SHA384, + } +) + +func isCertificateAllowedFIPS(c *x509.Certificate) bool { + switch k := c.PublicKey.(type) { + case *rsa.PublicKey: + return k.N.BitLen() >= 2048 + case *ecdsa.PublicKey: + return k.Curve == elliptic.P256() || k.Curve == elliptic.P384() || k.Curve == elliptic.P521() + case ed25519.PublicKey: + return true + default: + return false + } +} diff --git a/src/crypto/tls/ech.go b/src/crypto/tls/ech.go index 7bf68589f8702c..d3472d8dc4aafb 100644 --- a/src/crypto/tls/ech.go +++ b/src/crypto/tls/ech.go @@ -5,13 +5,28 @@ package tls import ( + "bytes" "crypto/internal/hpke" "errors" + "fmt" + "slices" "strings" "golang.org/x/crypto/cryptobyte" ) +// sortedSupportedAEADs is just a sorted version of hpke.SupportedAEADS. +// We need this so that when we insert them into ECHConfigs the ordering +// is stable. +var sortedSupportedAEADs []uint16 + +func init() { + for aeadID := range hpke.SupportedAEADs { + sortedSupportedAEADs = append(sortedSupportedAEADs, aeadID) + } + slices.Sort(sortedSupportedAEADs) +} + type echCipher struct { KDFID uint16 AEADID uint16 @@ -38,86 +53,111 @@ type echConfig struct { Extensions []echExtension } -var errMalformedECHConfig = errors.New("tls: malformed ECHConfigList") +var errMalformedECHConfigList = errors.New("tls: malformed ECHConfigList") + +type echConfigErr struct { + field string +} + +func (e *echConfigErr) Error() string { + if e.field == "" { + return "tls: malformed ECHConfig" + } + return fmt.Sprintf("tls: malformed ECHConfig, invalid %s field", e.field) +} + +func parseECHConfig(enc []byte) (skip bool, ec echConfig, err error) { + s := cryptobyte.String(enc) + ec.raw = []byte(enc) + if !s.ReadUint16(&ec.Version) { + return false, echConfig{}, &echConfigErr{"version"} + } + if !s.ReadUint16(&ec.Length) { + return false, echConfig{}, &echConfigErr{"length"} + } + if len(ec.raw) < int(ec.Length)+4 { + return false, echConfig{}, &echConfigErr{"length"} + } + ec.raw = ec.raw[:ec.Length+4] + if ec.Version != extensionEncryptedClientHello { + s.Skip(int(ec.Length)) + return true, echConfig{}, nil + } + if !s.ReadUint8(&ec.ConfigID) { + return false, echConfig{}, &echConfigErr{"config_id"} + } + if !s.ReadUint16(&ec.KemID) { + return false, echConfig{}, &echConfigErr{"kem_id"} + } + if !readUint16LengthPrefixed(&s, &ec.PublicKey) { + return false, echConfig{}, &echConfigErr{"public_key"} + } + var cipherSuites cryptobyte.String + if !s.ReadUint16LengthPrefixed(&cipherSuites) { + return false, echConfig{}, &echConfigErr{"cipher_suites"} + } + for !cipherSuites.Empty() { + var c echCipher + if !cipherSuites.ReadUint16(&c.KDFID) { + return false, echConfig{}, &echConfigErr{"cipher_suites kdf_id"} + } + if !cipherSuites.ReadUint16(&c.AEADID) { + return false, echConfig{}, &echConfigErr{"cipher_suites aead_id"} + } + ec.SymmetricCipherSuite = append(ec.SymmetricCipherSuite, c) + } + if !s.ReadUint8(&ec.MaxNameLength) { + return false, echConfig{}, &echConfigErr{"maximum_name_length"} + } + var publicName cryptobyte.String + if !s.ReadUint8LengthPrefixed(&publicName) { + return false, echConfig{}, &echConfigErr{"public_name"} + } + ec.PublicName = publicName + var extensions cryptobyte.String + if !s.ReadUint16LengthPrefixed(&extensions) { + return false, echConfig{}, &echConfigErr{"extensions"} + } + for !extensions.Empty() { + var e echExtension + if !extensions.ReadUint16(&e.Type) { + return false, echConfig{}, &echConfigErr{"extensions type"} + } + if !extensions.ReadUint16LengthPrefixed((*cryptobyte.String)(&e.Data)) { + return false, echConfig{}, &echConfigErr{"extensions data"} + } + ec.Extensions = append(ec.Extensions, e) + } + + return false, ec, nil +} // parseECHConfigList parses a draft-ietf-tls-esni-18 ECHConfigList, returning a // slice of parsed ECHConfigs, in the same order they were parsed, or an error // if the list is malformed. func parseECHConfigList(data []byte) ([]echConfig, error) { s := cryptobyte.String(data) - // Skip the length prefix var length uint16 if !s.ReadUint16(&length) { - return nil, errMalformedECHConfig + return nil, errMalformedECHConfigList } if length != uint16(len(data)-2) { - return nil, errMalformedECHConfig + return nil, errMalformedECHConfigList } var configs []echConfig for len(s) > 0 { - var ec echConfig - ec.raw = []byte(s) - if !s.ReadUint16(&ec.Version) { - return nil, errMalformedECHConfig - } - if !s.ReadUint16(&ec.Length) { - return nil, errMalformedECHConfig + if len(s) < 4 { + return nil, errors.New("tls: malformed ECHConfig") } - if len(ec.raw) < int(ec.Length)+4 { - return nil, errMalformedECHConfig - } - ec.raw = ec.raw[:ec.Length+4] - if ec.Version != extensionEncryptedClientHello { - s.Skip(int(ec.Length)) - continue + configLen := uint16(s[2])<<8 | uint16(s[3]) + skip, ec, err := parseECHConfig(s) + if err != nil { + return nil, err } - if !s.ReadUint8(&ec.ConfigID) { - return nil, errMalformedECHConfig + s = s[configLen+4:] + if !skip { + configs = append(configs, ec) } - if !s.ReadUint16(&ec.KemID) { - return nil, errMalformedECHConfig - } - if !s.ReadUint16LengthPrefixed((*cryptobyte.String)(&ec.PublicKey)) { - return nil, errMalformedECHConfig - } - var cipherSuites cryptobyte.String - if !s.ReadUint16LengthPrefixed(&cipherSuites) { - return nil, errMalformedECHConfig - } - for !cipherSuites.Empty() { - var c echCipher - if !cipherSuites.ReadUint16(&c.KDFID) { - return nil, errMalformedECHConfig - } - if !cipherSuites.ReadUint16(&c.AEADID) { - return nil, errMalformedECHConfig - } - ec.SymmetricCipherSuite = append(ec.SymmetricCipherSuite, c) - } - if !s.ReadUint8(&ec.MaxNameLength) { - return nil, errMalformedECHConfig - } - var publicName cryptobyte.String - if !s.ReadUint8LengthPrefixed(&publicName) { - return nil, errMalformedECHConfig - } - ec.PublicName = publicName - var extensions cryptobyte.String - if !s.ReadUint16LengthPrefixed(&extensions) { - return nil, errMalformedECHConfig - } - for !extensions.Empty() { - var e echExtension - if !extensions.ReadUint16(&e.Type) { - return nil, errMalformedECHConfig - } - if !extensions.ReadUint16LengthPrefixed((*cryptobyte.String)(&e.Data)) { - return nil, errMalformedECHConfig - } - ec.Extensions = append(ec.Extensions, e) - } - - configs = append(configs, ec) } return configs, nil } @@ -195,6 +235,195 @@ func encodeInnerClientHello(inner *clientHelloMsg, maxNameLength int) ([]byte, e return append(h, make([]byte, paddingLen)...), nil } +func skipUint8LengthPrefixed(s *cryptobyte.String) bool { + var skip uint8 + if !s.ReadUint8(&skip) { + return false + } + return s.Skip(int(skip)) +} + +func skipUint16LengthPrefixed(s *cryptobyte.String) bool { + var skip uint16 + if !s.ReadUint16(&skip) { + return false + } + return s.Skip(int(skip)) +} + +type rawExtension struct { + extType uint16 + data []byte +} + +func extractRawExtensions(hello *clientHelloMsg) ([]rawExtension, error) { + s := cryptobyte.String(hello.original) + if !s.Skip(4+2+32) || // header, version, random + !skipUint8LengthPrefixed(&s) || // session ID + !skipUint16LengthPrefixed(&s) || // cipher suites + !skipUint8LengthPrefixed(&s) { // compression methods + return nil, errors.New("tls: malformed outer client hello") + } + var rawExtensions []rawExtension + var extensions cryptobyte.String + if !s.ReadUint16LengthPrefixed(&extensions) { + return nil, errors.New("tls: malformed outer client hello") + } + + for !extensions.Empty() { + var extension uint16 + var extData cryptobyte.String + if !extensions.ReadUint16(&extension) || + !extensions.ReadUint16LengthPrefixed(&extData) { + return nil, errors.New("tls: invalid inner client hello") + } + rawExtensions = append(rawExtensions, rawExtension{extension, extData}) + } + return rawExtensions, nil +} + +func decodeInnerClientHello(outer *clientHelloMsg, encoded []byte) (*clientHelloMsg, error) { + // Reconstructing the inner client hello from its encoded form is somewhat + // complicated. It is missing its header (message type and length), session + // ID, and the extensions may be compressed. Since we need to put the + // extensions back in the same order as they were in the raw outer hello, + // and since we don't store the raw extensions, or the order we parsed them + // in, we need to reparse the raw extensions from the outer hello in order + // to properly insert them into the inner hello. This _should_ result in raw + // bytes which match the hello as it was generated by the client. + innerReader := cryptobyte.String(encoded) + var versionAndRandom, sessionID, cipherSuites, compressionMethods []byte + var extensions cryptobyte.String + if !innerReader.ReadBytes(&versionAndRandom, 2+32) || + !readUint8LengthPrefixed(&innerReader, &sessionID) || + len(sessionID) != 0 || + !readUint16LengthPrefixed(&innerReader, &cipherSuites) || + !readUint8LengthPrefixed(&innerReader, &compressionMethods) || + !innerReader.ReadUint16LengthPrefixed(&extensions) { + return nil, errors.New("tls: invalid inner client hello") + } + + // The specification says we must verify that the trailing padding is all + // zeros. This is kind of weird for TLS messages, where we generally just + // throw away any trailing garbage. + for _, p := range innerReader { + if p != 0 { + return nil, errors.New("tls: invalid inner client hello") + } + } + + rawOuterExts, err := extractRawExtensions(outer) + if err != nil { + return nil, err + } + + recon := cryptobyte.NewBuilder(nil) + recon.AddUint8(typeClientHello) + recon.AddUint24LengthPrefixed(func(recon *cryptobyte.Builder) { + recon.AddBytes(versionAndRandom) + recon.AddUint8LengthPrefixed(func(recon *cryptobyte.Builder) { + recon.AddBytes(outer.sessionId) + }) + recon.AddUint16LengthPrefixed(func(recon *cryptobyte.Builder) { + recon.AddBytes(cipherSuites) + }) + recon.AddUint8LengthPrefixed(func(recon *cryptobyte.Builder) { + recon.AddBytes(compressionMethods) + }) + recon.AddUint16LengthPrefixed(func(recon *cryptobyte.Builder) { + for !extensions.Empty() { + var extension uint16 + var extData cryptobyte.String + if !extensions.ReadUint16(&extension) || + !extensions.ReadUint16LengthPrefixed(&extData) { + recon.SetError(errors.New("tls: invalid inner client hello")) + return + } + if extension == extensionECHOuterExtensions { + if !extData.ReadUint8LengthPrefixed(&extData) { + recon.SetError(errors.New("tls: invalid inner client hello")) + return + } + var i int + for !extData.Empty() { + var extType uint16 + if !extData.ReadUint16(&extType) { + recon.SetError(errors.New("tls: invalid inner client hello")) + return + } + if extType == extensionEncryptedClientHello { + recon.SetError(errors.New("tls: invalid outer extensions")) + return + } + for ; i <= len(rawOuterExts); i++ { + if i == len(rawOuterExts) { + recon.SetError(errors.New("tls: invalid outer extensions")) + return + } + if rawOuterExts[i].extType == extType { + break + } + } + recon.AddUint16(rawOuterExts[i].extType) + recon.AddUint16LengthPrefixed(func(recon *cryptobyte.Builder) { + recon.AddBytes(rawOuterExts[i].data) + }) + } + } else { + recon.AddUint16(extension) + recon.AddUint16LengthPrefixed(func(recon *cryptobyte.Builder) { + recon.AddBytes(extData) + }) + } + } + }) + }) + + reconBytes, err := recon.Bytes() + if err != nil { + return nil, err + } + inner := &clientHelloMsg{} + if !inner.unmarshal(reconBytes) { + return nil, errors.New("tls: invalid reconstructed inner client hello") + } + + if !bytes.Equal(inner.encryptedClientHello, []byte{uint8(innerECHExt)}) { + return nil, errInvalidECHExt + } + + hasTLS13 := false + for _, v := range inner.supportedVersions { + // Skip GREASE values (values of the form 0x?A0A). + // GREASE (Generate Random Extensions And Sustain Extensibility) is a mechanism used by + // browsers like Chrome to ensure TLS implementations correctly ignore unknown values. + // GREASE values follow a specific pattern: 0x?A0A, where ? can be any hex digit. + // These values should be ignored when processing supported TLS versions. + if v&0x0F0F == 0x0A0A && v&0xff == v>>8 { + continue + } + + // Ensure at least TLS 1.3 is offered. + if v == VersionTLS13 { + hasTLS13 = true + } else if v < VersionTLS13 { + // Reject if any non-GREASE value is below TLS 1.3, as ECH requires TLS 1.3+. + return nil, errors.New("tls: client sent encrypted_client_hello extension with unsupported versions") + } + } + + if !hasTLS13 { + return nil, errors.New("tls: client sent encrypted_client_hello extension but did not offer TLS 1.3") + } + + return inner, nil +} + +func decryptECHPayload(context *hpke.Recipient, hello, payload []byte) ([]byte, error) { + outerAAD := bytes.Replace(hello[4:], payload, make([]byte, len(payload)), 1) + return context.Open(outerAAD, payload) +} + func generateOuterECHExt(id uint8, kdfID, aeadID uint16, encodedKey []byte, payload []byte) ([]byte, error) { var b cryptobyte.Builder b.AddUint8(0) // outer @@ -206,7 +435,7 @@ func generateOuterECHExt(id uint8, kdfID, aeadID uint16, encodedKey []byte, payl return b.Bytes() } -func computeAndUpdateOuterECHExtension(outer, inner *clientHelloMsg, ech *echContext, useKey bool) error { +func computeAndUpdateOuterECHExtension(outer, inner *clientHelloMsg, ech *echClientContext, useKey bool) error { var encapKey []byte if useKey { encapKey = ech.encapsulatedKey @@ -281,3 +510,149 @@ type ECHRejectionError struct { func (e *ECHRejectionError) Error() string { return "tls: server rejected ECH" } + +var errMalformedECHExt = errors.New("tls: malformed encrypted_client_hello extension") +var errInvalidECHExt = errors.New("tls: client sent invalid encrypted_client_hello extension") + +type echExtType uint8 + +const ( + innerECHExt echExtType = 1 + outerECHExt echExtType = 0 +) + +func parseECHExt(ext []byte) (echType echExtType, cs echCipher, configID uint8, encap []byte, payload []byte, err error) { + data := make([]byte, len(ext)) + copy(data, ext) + s := cryptobyte.String(data) + var echInt uint8 + if !s.ReadUint8(&echInt) { + err = errMalformedECHExt + return + } + echType = echExtType(echInt) + if echType == innerECHExt { + if !s.Empty() { + err = errMalformedECHExt + return + } + return echType, cs, 0, nil, nil, nil + } + if echType != outerECHExt { + err = errInvalidECHExt + return + } + if !s.ReadUint16(&cs.KDFID) { + err = errMalformedECHExt + return + } + if !s.ReadUint16(&cs.AEADID) { + err = errMalformedECHExt + return + } + if !s.ReadUint8(&configID) { + err = errMalformedECHExt + return + } + if !readUint16LengthPrefixed(&s, &encap) { + err = errMalformedECHExt + return + } + if !readUint16LengthPrefixed(&s, &payload) { + err = errMalformedECHExt + return + } + + // NOTE: clone encap and payload so that mutating them does not mutate the + // raw extension bytes. + return echType, cs, configID, bytes.Clone(encap), bytes.Clone(payload), nil +} + +func (c *Conn) processECHClientHello(outer *clientHelloMsg, echKeys []EncryptedClientHelloKey) (*clientHelloMsg, *echServerContext, error) { + echType, echCiphersuite, configID, encap, payload, err := parseECHExt(outer.encryptedClientHello) + if err != nil { + if errors.Is(err, errInvalidECHExt) { + c.sendAlert(alertIllegalParameter) + } else { + c.sendAlert(alertDecodeError) + } + + return nil, nil, errInvalidECHExt + } + + if echType == innerECHExt { + return outer, &echServerContext{inner: true}, nil + } + + if len(echKeys) == 0 { + return outer, nil, nil + } + + for _, echKey := range echKeys { + skip, config, err := parseECHConfig(echKey.Config) + if err != nil || skip { + c.sendAlert(alertInternalError) + return nil, nil, fmt.Errorf("tls: invalid EncryptedClientHelloKeys Config: %s", err) + } + if skip { + continue + } + echPriv, err := hpke.ParseHPKEPrivateKey(config.KemID, echKey.PrivateKey) + if err != nil { + c.sendAlert(alertInternalError) + return nil, nil, fmt.Errorf("tls: invalid EncryptedClientHelloKeys PrivateKey: %s", err) + } + info := append([]byte("tls ech\x00"), echKey.Config...) + hpkeContext, err := hpke.SetupRecipient(hpke.DHKEM_X25519_HKDF_SHA256, echCiphersuite.KDFID, echCiphersuite.AEADID, echPriv, info, encap) + if err != nil { + // attempt next trial decryption + continue + } + + encodedInner, err := decryptECHPayload(hpkeContext, outer.original, payload) + if err != nil { + // attempt next trial decryption + continue + } + + // NOTE: we do not enforce that the sent server_name matches the ECH + // configs PublicName, since this is not particularly important, and + // the client already had to know what it was in order to properly + // encrypt the payload. This is only a MAY in the spec, so we're not + // doing anything revolutionary. + + echInner, err := decodeInnerClientHello(outer, encodedInner) + if err != nil { + c.sendAlert(alertIllegalParameter) + return nil, nil, errInvalidECHExt + } + + c.echAccepted = true + + return echInner, &echServerContext{ + hpkeContext: hpkeContext, + configID: configID, + ciphersuite: echCiphersuite, + }, nil + } + + return outer, nil, nil +} + +func buildRetryConfigList(keys []EncryptedClientHelloKey) ([]byte, error) { + var atLeastOneRetryConfig bool + var retryBuilder cryptobyte.Builder + retryBuilder.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + for _, c := range keys { + if !c.SendAsRetry { + continue + } + atLeastOneRetryConfig = true + b.AddBytes(c.Config) + } + }) + if !atLeastOneRetryConfig { + return nil, nil + } + return retryBuilder.Bytes() +} diff --git a/src/crypto/tls/fips140_test.go b/src/crypto/tls/fips140_test.go new file mode 100644 index 00000000000000..d3fa61dc97d398 --- /dev/null +++ b/src/crypto/tls/fips140_test.go @@ -0,0 +1,712 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tls + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/internal/boring" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "internal/obscuretestdata" + "internal/testenv" + "math/big" + "net" + "os" + "runtime" + "strings" + "testing" + "time" +) + +func allCipherSuitesIncludingTLS13() []uint16 { + s := allCipherSuites() + for _, suite := range cipherSuitesTLS13 { + s = append(s, suite.id) + } + return s +} + +func isTLS13CipherSuite(id uint16) bool { + for _, suite := range cipherSuitesTLS13 { + if id == suite.id { + return true + } + } + return false +} + +func generateKeyShare(group CurveID) keyShare { + key, err := generateECDHEKey(rand.Reader, group) + if err != nil { + panic(err) + } + return keyShare{group: group, data: key.PublicKey().Bytes()} +} + +func TestFIPSServerProtocolVersion(t *testing.T) { + test := func(t *testing.T, name string, v uint16, msg string) { + t.Run(name, func(t *testing.T) { + serverConfig := testConfig.Clone() + serverConfig.MinVersion = VersionSSL30 + clientConfig := testConfig.Clone() + clientConfig.MinVersion = v + clientConfig.MaxVersion = v + _, _, err := testHandshake(t, clientConfig, serverConfig) + if msg == "" { + if err != nil { + t.Fatalf("got error: %v, expected success", err) + } + } else { + if err == nil { + t.Fatalf("got success, expected error") + } + if !strings.Contains(err.Error(), msg) { + t.Fatalf("got error %v, expected %q", err, msg) + } + } + }) + } + + runWithFIPSDisabled(t, func(t *testing.T) { + test(t, "VersionTLS10", VersionTLS10, "") + test(t, "VersionTLS11", VersionTLS11, "") + test(t, "VersionTLS12", VersionTLS12, "") + test(t, "VersionTLS13", VersionTLS13, "") + }) + + runWithFIPSEnabled(t, func(t *testing.T) { + test(t, "VersionTLS10", VersionTLS10, "supported versions") + test(t, "VersionTLS11", VersionTLS11, "supported versions") + test(t, "VersionTLS12", VersionTLS12, "") + test(t, "VersionTLS13", VersionTLS13, "") + }) +} + +func isFIPSVersion(v uint16) bool { + return v == VersionTLS12 || v == VersionTLS13 +} + +func isFIPSCipherSuite(id uint16) bool { + name := CipherSuiteName(id) + if isTLS13CipherSuite(id) { + switch id { + case TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384: + return true + case TLS_CHACHA20_POLY1305_SHA256: + return false + default: + panic("unknown TLS 1.3 cipher suite: " + name) + } + } + switch id { + case TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384: + return true + case TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256: + // Only for the native module. + return !boring.Enabled + } + switch { + case strings.Contains(name, "CHACHA20"): + return false + case strings.HasSuffix(name, "_SHA"): // SHA-1 + return false + case strings.HasPrefix(name, "TLS_RSA"): // RSA kex + return false + default: + panic("unknown cipher suite: " + name) + } +} + +func isFIPSCurve(id CurveID) bool { + switch id { + case CurveP256, CurveP384, CurveP521: + return true + case X25519MLKEM768: + // Only for the native module. + return !boring.Enabled + case X25519: + return false + default: + panic("unknown curve: " + id.String()) + } +} + +func isECDSA(id uint16) bool { + for _, suite := range cipherSuites { + if suite.id == id { + return suite.flags&suiteECSign == suiteECSign + } + } + return false // TLS 1.3 cipher suites are not tied to the signature algorithm. +} + +func isFIPSSignatureScheme(alg SignatureScheme) bool { + switch alg { + case PKCS1WithSHA256, + ECDSAWithP256AndSHA256, + PKCS1WithSHA384, + ECDSAWithP384AndSHA384, + PKCS1WithSHA512, + ECDSAWithP521AndSHA512, + PSSWithSHA256, + PSSWithSHA384, + PSSWithSHA512: + return true + case Ed25519: + // Only for the native module. + return !boring.Enabled + case PKCS1WithSHA1, ECDSAWithSHA1: + return false + default: + panic("unknown signature scheme: " + alg.String()) + } +} + +func TestFIPSServerCipherSuites(t *testing.T) { + serverConfig := testConfig.Clone() + serverConfig.Certificates = make([]Certificate, 1) + + for _, id := range allCipherSuitesIncludingTLS13() { + if isECDSA(id) { + serverConfig.Certificates[0].Certificate = [][]byte{testECDSACertificate} + serverConfig.Certificates[0].PrivateKey = testECDSAPrivateKey + } else { + serverConfig.Certificates[0].Certificate = [][]byte{testRSACertificate} + serverConfig.Certificates[0].PrivateKey = testRSAPrivateKey + } + serverConfig.BuildNameToCertificate() + t.Run(fmt.Sprintf("suite=%s", CipherSuiteName(id)), func(t *testing.T) { + clientHello := &clientHelloMsg{ + vers: VersionTLS12, + random: make([]byte, 32), + cipherSuites: []uint16{id}, + compressionMethods: []uint8{compressionNone}, + supportedCurves: defaultCurvePreferences(), + keyShares: []keyShare{generateKeyShare(CurveP256)}, + supportedPoints: []uint8{pointFormatUncompressed}, + supportedVersions: []uint16{VersionTLS12}, + supportedSignatureAlgorithms: allowedSignatureAlgorithmsFIPS, + } + if isTLS13CipherSuite(id) { + clientHello.supportedVersions = []uint16{VersionTLS13} + } + + runWithFIPSDisabled(t, func(t *testing.T) { + testClientHello(t, serverConfig, clientHello) + }) + + runWithFIPSEnabled(t, func(t *testing.T) { + msg := "" + if !isFIPSCipherSuite(id) { + msg = "no cipher suite supported by both client and server" + } + testClientHelloFailure(t, serverConfig, clientHello, msg) + }) + }) + } +} + +func TestFIPSServerCurves(t *testing.T) { + serverConfig := testConfig.Clone() + serverConfig.CurvePreferences = nil + serverConfig.BuildNameToCertificate() + + for _, curveid := range defaultCurvePreferences() { + t.Run(fmt.Sprintf("curve=%v", curveid), func(t *testing.T) { + clientConfig := testConfig.Clone() + clientConfig.CurvePreferences = []CurveID{curveid} + + runWithFIPSDisabled(t, func(t *testing.T) { + if _, _, err := testHandshake(t, clientConfig, serverConfig); err != nil { + t.Fatalf("got error: %v, expected success", err) + } + }) + + // With fipstls forced, bad curves should be rejected. + runWithFIPSEnabled(t, func(t *testing.T) { + _, _, err := testHandshake(t, clientConfig, serverConfig) + if err != nil && isFIPSCurve(curveid) { + t.Fatalf("got error: %v, expected success", err) + } else if err == nil && !isFIPSCurve(curveid) { + t.Fatalf("got success, expected error") + } + }) + }) + } +} + +func fipsHandshake(t *testing.T, clientConfig, serverConfig *Config) (clientErr, serverErr error) { + c, s := localPipe(t) + client := Client(c, clientConfig) + server := Server(s, serverConfig) + done := make(chan error, 1) + go func() { + done <- client.Handshake() + c.Close() + }() + serverErr = server.Handshake() + s.Close() + clientErr = <-done + return +} + +func TestFIPSServerSignatureAndHash(t *testing.T) { + defer func() { + testingOnlySupportedSignatureAlgorithms = nil + }() + defer func(godebug string) { + os.Setenv("GODEBUG", godebug) + }(os.Getenv("GODEBUG")) + os.Setenv("GODEBUG", "tlssha1=1") + + for _, sigHash := range defaultSupportedSignatureAlgorithms() { + t.Run(fmt.Sprintf("%v", sigHash), func(t *testing.T) { + serverConfig := testConfig.Clone() + serverConfig.Certificates = make([]Certificate, 1) + + testingOnlySupportedSignatureAlgorithms = []SignatureScheme{sigHash} + + sigType, _, _ := typeAndHashFromSignatureScheme(sigHash) + switch sigType { + case signaturePKCS1v15, signatureRSAPSS: + serverConfig.CipherSuites = []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256} + serverConfig.Certificates[0].Certificate = [][]byte{testRSAPSS2048Certificate} + serverConfig.Certificates[0].PrivateKey = testRSAPSS2048PrivateKey + case signatureEd25519: + serverConfig.CipherSuites = []uint16{TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256} + serverConfig.Certificates[0].Certificate = [][]byte{testEd25519Certificate} + serverConfig.Certificates[0].PrivateKey = testEd25519PrivateKey + case signatureECDSA: + serverConfig.CipherSuites = []uint16{TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256} + serverConfig.Certificates[0].Certificate = [][]byte{testECDSACertificate} + serverConfig.Certificates[0].PrivateKey = testECDSAPrivateKey + } + serverConfig.BuildNameToCertificate() + // PKCS#1 v1.5 signature algorithms can't be used standalone in TLS + // 1.3, and the ECDSA ones bind to the curve used. + serverConfig.MaxVersion = VersionTLS12 + + runWithFIPSDisabled(t, func(t *testing.T) { + clientErr, serverErr := fipsHandshake(t, testConfig, serverConfig) + if clientErr != nil { + t.Fatalf("expected handshake with %v to succeed; client error: %v; server error: %v", sigHash, clientErr, serverErr) + } + }) + + // With fipstls forced, bad curves should be rejected. + runWithFIPSEnabled(t, func(t *testing.T) { + clientErr, _ := fipsHandshake(t, testConfig, serverConfig) + if isFIPSSignatureScheme(sigHash) { + if clientErr != nil { + t.Fatalf("expected handshake with %v to succeed; err=%v", sigHash, clientErr) + } + } else { + if clientErr == nil { + t.Fatalf("expected handshake with %v to fail, but it succeeded", sigHash) + } + } + }) + }) + } +} + +func TestFIPSClientHello(t *testing.T) { + runWithFIPSEnabled(t, testFIPSClientHello) +} + +func testFIPSClientHello(t *testing.T) { + // Test that no matter what we put in the client config, + // the client does not offer non-FIPS configurations. + + c, s := net.Pipe() + defer c.Close() + defer s.Close() + + clientConfig := testConfig.Clone() + // All sorts of traps for the client to avoid. + clientConfig.MinVersion = VersionSSL30 + clientConfig.MaxVersion = VersionTLS13 + clientConfig.CipherSuites = allCipherSuites() + clientConfig.CurvePreferences = defaultCurvePreferences() + + go Client(c, clientConfig).Handshake() + srv := Server(s, testConfig) + msg, err := srv.readHandshake(nil) + if err != nil { + t.Fatal(err) + } + hello, ok := msg.(*clientHelloMsg) + if !ok { + t.Fatalf("unexpected message type %T", msg) + } + + if !isFIPSVersion(hello.vers) { + t.Errorf("client vers=%#x", hello.vers) + } + for _, v := range hello.supportedVersions { + if !isFIPSVersion(v) { + t.Errorf("client offered disallowed version %#x", v) + } + } + for _, id := range hello.cipherSuites { + if !isFIPSCipherSuite(id) { + t.Errorf("client offered disallowed suite %v", CipherSuiteName(id)) + } + } + for _, id := range hello.supportedCurves { + if !isFIPSCurve(id) { + t.Errorf("client offered disallowed curve %v", id) + } + } + for _, sigHash := range hello.supportedSignatureAlgorithms { + if !isFIPSSignatureScheme(sigHash) { + t.Errorf("client offered disallowed signature-and-hash %v", sigHash) + } + } +} + +func TestFIPSCertAlgs(t *testing.T) { + // arm and wasm time out generating keys. Nothing in this test is + // architecture-specific, so just don't bother on those. + if testenv.CPUIsSlow() { + t.Skipf("skipping on %s/%s because key generation takes too long", runtime.GOOS, runtime.GOARCH) + } + + // Set up some roots, intermediate CAs, and leaf certs with various algorithms. + // X_Y is X signed by Y. + R1 := fipsCert(t, "R1", fipsRSAKey(t, 2048), nil, fipsCertCA|fipsCertFIPSOK) + R2 := fipsCert(t, "R2", fipsRSAKey(t, 1024), nil, fipsCertCA) + R3 := fipsCert(t, "R3", fipsRSAKey(t, 4096), nil, fipsCertCA|fipsCertFIPSOK) + + M1_R1 := fipsCert(t, "M1_R1", fipsECDSAKey(t, elliptic.P256()), R1, fipsCertCA|fipsCertFIPSOK) + M2_R1 := fipsCert(t, "M2_R1", fipsECDSAKey(t, elliptic.P224()), R1, fipsCertCA) + + I_R1 := fipsCert(t, "I_R1", fipsRSAKey(t, 3072), R1, fipsCertCA|fipsCertFIPSOK) + I_R2 := fipsCert(t, "I_R2", I_R1.key, R2, fipsCertCA|fipsCertFIPSOK) + I_M1 := fipsCert(t, "I_M1", I_R1.key, M1_R1, fipsCertCA|fipsCertFIPSOK) + I_M2 := fipsCert(t, "I_M2", I_R1.key, M2_R1, fipsCertCA|fipsCertFIPSOK) + + I_R3 := fipsCert(t, "I_R3", fipsRSAKey(t, 3072), R3, fipsCertCA|fipsCertFIPSOK) + fipsCert(t, "I_R3", I_R3.key, R3, fipsCertCA|fipsCertFIPSOK) + + L1_I := fipsCert(t, "L1_I", fipsECDSAKey(t, elliptic.P384()), I_R1, fipsCertLeaf|fipsCertFIPSOK) + L2_I := fipsCert(t, "L2_I", fipsRSAKey(t, 1024), I_R1, fipsCertLeaf) + + // client verifying server cert + testServerCert := func(t *testing.T, desc string, pool *x509.CertPool, key interface{}, list [][]byte, ok bool) { + clientConfig := testConfig.Clone() + clientConfig.RootCAs = pool + clientConfig.InsecureSkipVerify = false + clientConfig.ServerName = "example.com" + + serverConfig := testConfig.Clone() + serverConfig.Certificates = []Certificate{{Certificate: list, PrivateKey: key}} + serverConfig.BuildNameToCertificate() + + clientErr, _ := fipsHandshake(t, clientConfig, serverConfig) + + if (clientErr == nil) == ok { + if ok { + t.Logf("%s: accept", desc) + } else { + t.Logf("%s: reject", desc) + } + } else { + if ok { + t.Errorf("%s: BAD reject (%v)", desc, clientErr) + } else { + t.Errorf("%s: BAD accept", desc) + } + } + } + + // server verifying client cert + testClientCert := func(t *testing.T, desc string, pool *x509.CertPool, key interface{}, list [][]byte, ok bool) { + clientConfig := testConfig.Clone() + clientConfig.ServerName = "example.com" + clientConfig.Certificates = []Certificate{{Certificate: list, PrivateKey: key}} + + serverConfig := testConfig.Clone() + serverConfig.ClientCAs = pool + serverConfig.ClientAuth = RequireAndVerifyClientCert + + _, serverErr := fipsHandshake(t, clientConfig, serverConfig) + + if (serverErr == nil) == ok { + if ok { + t.Logf("%s: accept", desc) + } else { + t.Logf("%s: reject", desc) + } + } else { + if ok { + t.Errorf("%s: BAD reject (%v)", desc, serverErr) + } else { + t.Errorf("%s: BAD accept", desc) + } + } + } + + // Run simple basic test with known answers before proceeding to + // exhaustive test with computed answers. + r1pool := x509.NewCertPool() + r1pool.AddCert(R1.cert) + + runWithFIPSDisabled(t, func(t *testing.T) { + testServerCert(t, "basic", r1pool, L2_I.key, [][]byte{L2_I.der, I_R1.der}, true) + testClientCert(t, "basic (client cert)", r1pool, L2_I.key, [][]byte{L2_I.der, I_R1.der}, true) + }) + + runWithFIPSEnabled(t, func(t *testing.T) { + testServerCert(t, "basic (fips)", r1pool, L2_I.key, [][]byte{L2_I.der, I_R1.der}, false) + testClientCert(t, "basic (fips, client cert)", r1pool, L2_I.key, [][]byte{L2_I.der, I_R1.der}, false) + }) + + if t.Failed() { + t.Fatal("basic test failed, skipping exhaustive test") + } + + if testing.Short() { + t.Logf("basic test passed; skipping exhaustive test in -short mode") + return + } + + for l := 1; l <= 2; l++ { + leaf := L1_I + if l == 2 { + leaf = L2_I + } + for i := 0; i < 64; i++ { + reachable := map[string]bool{leaf.parentOrg: true} + reachableFIPS := map[string]bool{leaf.parentOrg: leaf.fipsOK} + list := [][]byte{leaf.der} + listName := leaf.name + addList := func(cond int, c *fipsCertificate) { + if cond != 0 { + list = append(list, c.der) + listName += "," + c.name + if reachable[c.org] { + reachable[c.parentOrg] = true + } + if reachableFIPS[c.org] && c.fipsOK { + reachableFIPS[c.parentOrg] = true + } + } + } + addList(i&1, I_R1) + addList(i&2, I_R2) + addList(i&4, I_M1) + addList(i&8, I_M2) + addList(i&16, M1_R1) + addList(i&32, M2_R1) + + for r := 1; r <= 3; r++ { + pool := x509.NewCertPool() + rootName := "," + shouldVerify := false + shouldVerifyFIPS := false + addRoot := func(cond int, c *fipsCertificate) { + if cond != 0 { + rootName += "," + c.name + pool.AddCert(c.cert) + if reachable[c.org] { + shouldVerify = true + } + if reachableFIPS[c.org] && c.fipsOK { + shouldVerifyFIPS = true + } + } + } + addRoot(r&1, R1) + addRoot(r&2, R2) + rootName = rootName[1:] // strip leading comma + + runWithFIPSDisabled(t, func(t *testing.T) { + testServerCert(t, listName+"->"+rootName[1:], pool, leaf.key, list, shouldVerify) + testClientCert(t, listName+"->"+rootName[1:]+"(client cert)", pool, leaf.key, list, shouldVerify) + }) + + runWithFIPSEnabled(t, func(t *testing.T) { + testServerCert(t, listName+"->"+rootName[1:]+" (fips)", pool, leaf.key, list, shouldVerifyFIPS) + testClientCert(t, listName+"->"+rootName[1:]+" (fips, client cert)", pool, leaf.key, list, shouldVerifyFIPS) + }) + } + } + } +} + +const ( + fipsCertCA = iota + fipsCertLeaf + fipsCertFIPSOK = 0x80 +) + +func fipsRSAKey(t *testing.T, size int) *rsa.PrivateKey { + k, err := rsa.GenerateKey(rand.Reader, size) + if err != nil { + t.Fatal(err) + } + return k +} + +func fipsECDSAKey(t *testing.T, curve elliptic.Curve) *ecdsa.PrivateKey { + k, err := ecdsa.GenerateKey(curve, rand.Reader) + if err != nil { + t.Fatal(err) + } + return k +} + +type fipsCertificate struct { + name string + org string + parentOrg string + der []byte + cert *x509.Certificate + key interface{} + fipsOK bool +} + +func fipsCert(t *testing.T, name string, key interface{}, parent *fipsCertificate, mode int) *fipsCertificate { + org := name + parentOrg := "" + if i := strings.Index(org, "_"); i >= 0 { + org = org[:i] + parentOrg = name[i+1:] + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + Organization: []string{org}, + }, + NotBefore: time.Unix(0, 0), + NotAfter: time.Unix(0, 0), + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + } + if mode&^fipsCertFIPSOK == fipsCertLeaf { + tmpl.DNSNames = []string{"example.com"} + } else { + tmpl.IsCA = true + tmpl.KeyUsage |= x509.KeyUsageCertSign + } + + var pcert *x509.Certificate + var pkey interface{} + if parent != nil { + pcert = parent.cert + pkey = parent.key + } else { + pcert = tmpl + pkey = key + } + + var pub interface{} + var desc string + switch k := key.(type) { + case *rsa.PrivateKey: + pub = &k.PublicKey + desc = fmt.Sprintf("RSA-%d", k.N.BitLen()) + case *ecdsa.PrivateKey: + pub = &k.PublicKey + desc = "ECDSA-" + k.Curve.Params().Name + default: + t.Fatalf("invalid key %T", key) + } + + der, err := x509.CreateCertificate(rand.Reader, tmpl, pcert, pub, pkey) + if err != nil { + t.Fatal(err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + + fipsOK := mode&fipsCertFIPSOK != 0 + runWithFIPSEnabled(t, func(t *testing.T) { + if isCertificateAllowedFIPS(cert) != fipsOK { + t.Errorf("fipsAllowCert(cert with %s key) = %v, want %v", desc, !fipsOK, fipsOK) + } + }) + + return &fipsCertificate{name, org, parentOrg, der, cert, key, fipsOK} +} + +// A self-signed test certificate with an RSA key of size 2048, for testing +// RSA-PSS with SHA512. SAN of example.golang. +var ( + testRSAPSS2048Certificate []byte + testRSAPSS2048PrivateKey *rsa.PrivateKey +) + +func init() { + block, _ := pem.Decode(obscuretestdata.Rot13([]byte(` +-----ORTVA PREGVSVPNGR----- +ZVVP/mPPNrrtNjVONtVENYUUK/xu4+4mZH9QnemORpDjQDLWXbMVuipANDRYODNj +RwRDZN4TN1HRPuZUDJAgMFOQomNrSj0kZGNkZQRkAGN0ZQInSj0lZQRlZwxkAGN0 +ZQInZOVkRQNBOtAIONbGO0SwoJHtD28jttRvZN0TPFdTFVo3QDRONDHNN4VOQjNj +ttRXNbVONDPs8sx0A6vrPOK4VBIVsXvgg4xTpBDYrvzPsfwddUplfZVITRgSFZ6R +4Nl141s/7VdqJ0HgVdAo4CKuEBVQ7lQkE284kY6KoPhi/g5uC3HpruLp3uzYvlIq +ZxMDvMJgsHHWs/1dBgZ+buAt59YEJc4q+6vK0yn1WY3RjPVpxxAwW9uDoS7Co2PF ++RF9Lb55XNnc8XBoycpE8ZOFA38odajwsDqPKiBRBwnz2UHkXmRSK5ZN+sN0zr4P +vbPpPEYJXy+TbA9S8sNOsbM+G+2rny4QYhB95eKE8FeBVIOu3KSBe/EIuwgKpAIS +MXpiQg6q68I6wNXNLXz5ayw9TCcq4i+eNtZONNTwHQOBZN4TN1HqQjRO/jDRNjVS +bQNGOtAIUFHRQQNXOtteOtRSODpQNGNZOtAIUEZONs8RNwNNZOxTN1HqRDDFZOPP +QzI4LJ1joTHhM29fLJ5aZN0TPFdTFVo3QDROPjHNN4VONDPBbLfIpSPOuobdr3JU +qP6I7KKKRPzawu01e8u80li0AE379aFQ3pj2Z+UXinKlfJdey5uwTIXj0igjQ81e +I4WmQh7VsVbt5z8+DAP+7YdQMfm88iQXBefblFIBzHPtzPXSKrj+YN+rB/vDRWGe +7rafqqBrKWRc27Rq5iJ+xzJJ3Dztyp2Tjl8jSeZQVdaeaBmON4bPaQRtgKWg0mbt +aEjosRZNJv1nDEl5qG9XN3FC9zb5FrGSFmTTUvR4f4tUHr7wifNSS2dtgQ6+jU6f +m9o6fukaP7t5VyOXuV7FIO/Hdg2lqW+xU1LowZpVd6ANZ5rAZXtMhWe3+mjfFtju +TAnR +-----RAQ PREGVSVPNGR-----`))) + testRSAPSS2048Certificate = block.Bytes + + block, _ = pem.Decode(obscuretestdata.Rot13([]byte(` +-----ORTVA EFN CEVINGR XRL----- +ZVVRcNVONNXPNDRNa/U5AQrbattI+PQyFUlbeorWOaQxP3bcta7V6du3ZeQPSEuY +EHwBuBNZgrAK/+lXaIgSYFXwJ+Q14HGvN+8t8HqiBZF+y2jee/7rLG91UUbJUA4M +v4fyKGWTHVzIeK1SPK/9nweGCdVGLBsF0IdrUshby9WJgFF9kZNvUWWQLlsLHTkr +m29txiuRiJXBrFtTdsPwz5nKRsQNHwq/T6c8V30UDy7muQb2cgu1ZFfkOI+GNCaj +AWahNbdNaNxF1vcsudQsEsUjNK6Tsx/gazcrNl7wirn10sRdmvSDLq1kGd/0ILL7 +I3QIEJFaYj7rariSrbjPtTPchM5L/Ew6KrY/djVQNDNONbVONDPAcZMvsq/it42u +UqPiYhMnLF0E7FhaSycbKRfygTqYSfac0VsbWM/htSDOFNVVsYjZhzH6bKN1m7Hi +98nVLI61QrCeGPQIQSOfUoAzC8WNb8JgohfRojq5mlbO7YLT2+pyxWxyJR73XdHd +ezV+HWrlFpy2Tva7MGkOKm1JCOx9IjpajxrnKctNFVOJ23suRPZ9taLRRjnOrm5G +6Zr8q1gUgLDi7ifXr7eb9j9/UXeEKrwdLXX1YkxusSevlI+z8YMWMa2aKBn6T3tS +Ao8Dx1Hx5CHORAOzlZSWuG4Z/hhFd4LgZeeB2tv8D+sCuhTmp5FfuLXEOc0J4C5e +zgIPgRSENbTONZRAOVSYeI2+UfTw0kLSnfXbi/DCr6UFGE1Uu2VMBAc+bX4bfmJR +wOG4IpaVGzcy6gP1Jl4TpekwAtXVSMNw+1k1YHHYqbeKxhT8le0gNuT9mAlsJfFl +CeFbiP0HIome8Wkkyn+xDIkRDDdJDkCyRIhY8xKnVQN6Ylg1Uchn2YiCNbTONADM +p6Yd2G7+OkYkAqv2z8xMmrw5xtmOc/KqIfoSJEyroVK2XeSUfeUmG9CHx3QR1iMX +Z6cmGg94aDuJFxQtPnj1FbuRyW3USVSjphfS1FWNp3cDrcq8ht6VLqycQZYgOw/C +/5C6OIHgtb05R4+V/G3vLngztyDkGgyM0ExFI2yyNbTONYBKxXSK7nuCis0JxfQu +hGshSBGCbbjtDT0RctJ0jEqPkrt/WYvp3yFQ0tfggDI2JfErpelJpknryEt10EzB +38OobtzunS4kitfFihwBsvMGR8bX1G43Z+6AXfVyZY3LVYocH/9nWkCJl0f2QdQe +pDWuMeyx+cmwON7Oas/HEqjkNbTNXE/PAj14Q+zeY3LYoovPKvlqdkIjki5cqMqm +8guv3GApfJP4vTHEqpIdosHvaICqWvKr/Xnp3JTPrEWnSItoXNBkYgv1EO5ZxVut +Q8rlhcOdx4J1Y1txekdfqw4GSykxjZljwy2R2F4LlD8COg6I04QbIEMfVXmdm+CS +HvbaCd0PtLOPLKidvbWuCrjxBd/L5jeQOrMJ1SDX5DQ9J5Z8/5mkq4eqiWgwuoWc +bBegiZqey6hcl9Um4OWQ3SKjISvCSR7wdrAdv0S21ivYkOCZZQ3HBQS6YY5RlYvE +9I4kIZF8XKkit7ekfhdmZCfpIvnJHY6JAIOufQ2+92qUkFKmm5RWXD== +-----RAQ EFN CEVINGR XRL-----`))) + var err error + testRSAPSS2048PrivateKey, err = x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + panic(err) + } +} diff --git a/src/crypto/tls/fipsonly/fipsonly.go b/src/crypto/tls/fipsonly/fipsonly.go index e5e47835e2f48d..e702f44e986746 100644 --- a/src/crypto/tls/fipsonly/fipsonly.go +++ b/src/crypto/tls/fipsonly/fipsonly.go @@ -19,11 +19,11 @@ package fipsonly // new source file and not modifying any existing source files. import ( - "crypto/internal/boring/fipstls" "crypto/internal/boring/sig" + "crypto/tls/internal/fips140tls" ) func init() { - fipstls.Force() + fips140tls.Force() sig.FIPSOnly() } diff --git a/src/crypto/tls/fipsonly/fipsonly_test.go b/src/crypto/tls/fipsonly/fipsonly_test.go index f8485dc3ca1c29..027bc22c33c921 100644 --- a/src/crypto/tls/fipsonly/fipsonly_test.go +++ b/src/crypto/tls/fipsonly/fipsonly_test.go @@ -7,12 +7,12 @@ package fipsonly import ( - "crypto/internal/boring/fipstls" + "crypto/tls/internal/fips140tls" "testing" ) func Test(t *testing.T) { - if !fipstls.Required() { - t.Fatal("fipstls.Required() = false, must be true") + if !fips140tls.Required() { + t.Fatal("fips140tls.Required() = false, must be true") } } diff --git a/src/crypto/tls/handshake_client.go b/src/crypto/tls/handshake_client.go index 5025657590d32c..90c5bdacd82318 100644 --- a/src/crypto/tls/handshake_client.go +++ b/src/crypto/tls/handshake_client.go @@ -10,18 +10,20 @@ import ( "crypto" "crypto/ecdsa" "crypto/ed25519" + "crypto/internal/fips140/mlkem" + "crypto/internal/fips140/tls13" "crypto/internal/hpke" - "crypto/internal/mlkem768" "crypto/rsa" "crypto/subtle" + "crypto/tls/internal/fips140tls" "crypto/x509" "errors" "fmt" "hash" - "internal/byteorder" "internal/godebug" "io" "net" + "slices" "strconv" "strings" "time" @@ -39,9 +41,7 @@ type clientHandshakeState struct { ticket []byte // a fresh ticket received during this handshake } -var testingOnlyForceClientHelloSignatureAlgorithms []SignatureScheme - -func (c *Conn) makeClientHello() (*clientHelloMsg, *keySharePrivateKeys, *echContext, error) { +func (c *Conn) makeClientHello() (*clientHelloMsg, *keySharePrivateKeys, *echClientContext, error) { config := c.config if len(config.ServerName) == 0 && !config.InsecureSkipVerify { return nil, nil, nil, errors.New("tls: either ServerName or InsecureSkipVerify must be specified in the tls.Config") @@ -63,7 +63,10 @@ func (c *Conn) makeClientHello() (*clientHelloMsg, *keySharePrivateKeys, *echCon if len(supportedVersions) == 0 { return nil, nil, nil, errors.New("tls: no supported versions satisfy MinVersion and MaxVersion") } - maxVersion := config.maxSupportedVersion(roleClient) + // Since supportedVersions is sorted in descending order, the first element + // is the maximum version and the last element is the minimum version. + maxVersion := supportedVersions[0] + minVersion := supportedVersions[len(supportedVersions)-1] hello := &clientHelloMsg{ vers: maxVersion, @@ -91,24 +94,12 @@ func (c *Conn) makeClientHello() (*clientHelloMsg, *keySharePrivateKeys, *echCon hello.secureRenegotiation = c.clientFinished[:] } - preferenceOrder := cipherSuitesPreferenceOrder - if !hasAESGCMHardwareSupport { - preferenceOrder = cipherSuitesPreferenceOrderNoAES - } - configCipherSuites := config.cipherSuites() - hello.cipherSuites = make([]uint16, 0, len(configCipherSuites)) - - for _, suiteId := range preferenceOrder { - suite := mutualCipherSuite(configCipherSuites, suiteId) - if suite == nil { - continue - } - // Don't advertise TLS 1.2-only cipher suites unless - // we're attempting TLS 1.2. - if maxVersion < VersionTLS12 && suite.flags&suiteTLS12 != 0 { - continue - } - hello.cipherSuites = append(hello.cipherSuites, suiteId) + hello.cipherSuites = config.cipherSuites(hasAESGCMHardwareSupport) + // Don't advertise TLS 1.2-only cipher suites unless we're attempting TLS 1.2. + if maxVersion < VersionTLS12 { + hello.cipherSuites = slices.DeleteFunc(hello.cipherSuites, func(id uint16) bool { + return cipherSuiteByID(id).flags&suiteTLS12 != 0 + }) } _, err := io.ReadFull(config.rand(), hello.random) @@ -129,47 +120,55 @@ func (c *Conn) makeClientHello() (*clientHelloMsg, *keySharePrivateKeys, *echCon } if maxVersion >= VersionTLS12 { - hello.supportedSignatureAlgorithms = supportedSignatureAlgorithms() - } - if testingOnlyForceClientHelloSignatureAlgorithms != nil { - hello.supportedSignatureAlgorithms = testingOnlyForceClientHelloSignatureAlgorithms + hello.supportedSignatureAlgorithms = supportedSignatureAlgorithms(minVersion) + hello.supportedSignatureAlgorithmsCert = supportedSignatureAlgorithmsCert() } var keyShareKeys *keySharePrivateKeys - if hello.supportedVersions[0] == VersionTLS13 { + if maxVersion >= VersionTLS13 { // Reset the list of ciphers when the client only supports TLS 1.3. - if len(hello.supportedVersions) == 1 { + if minVersion >= VersionTLS13 { hello.cipherSuites = nil } - if hasAESGCMHardwareSupport { + + if fips140tls.Required() { + hello.cipherSuites = append(hello.cipherSuites, allowedCipherSuitesTLS13FIPS...) + } else if hasAESGCMHardwareSupport { hello.cipherSuites = append(hello.cipherSuites, defaultCipherSuitesTLS13...) } else { hello.cipherSuites = append(hello.cipherSuites, defaultCipherSuitesTLS13NoAES...) } - curveID := config.curvePreferences(maxVersion)[0] + if len(hello.supportedCurves) == 0 { + return nil, nil, nil, errors.New("tls: no supported elliptic curves for ECDHE") + } + curveID := hello.supportedCurves[0] keyShareKeys = &keySharePrivateKeys{curveID: curveID} - if curveID == x25519Kyber768Draft00 { + // Note that if X25519MLKEM768 is supported, it will be first because + // the preference order is fixed. + if curveID == X25519MLKEM768 { keyShareKeys.ecdhe, err = generateECDHEKey(config.rand(), X25519) if err != nil { return nil, nil, nil, err } - seed := make([]byte, mlkem768.SeedSize) + seed := make([]byte, mlkem.SeedSize) if _, err := io.ReadFull(config.rand(), seed); err != nil { return nil, nil, nil, err } - keyShareKeys.kyber, err = mlkem768.NewKeyFromSeed(seed) + keyShareKeys.mlkem, err = mlkem.NewDecapsulationKey768(seed) if err != nil { return nil, nil, nil, err } - // For draft-tls-westerbaan-xyber768d00-03, we send both a hybrid - // and a standard X25519 key share, since most servers will only - // support the latter. We reuse the same X25519 ephemeral key for - // both, as allowed by draft-ietf-tls-hybrid-design-09, Section 3.2. + mlkemEncapsulationKey := keyShareKeys.mlkem.EncapsulationKey().Bytes() + x25519EphemeralKey := keyShareKeys.ecdhe.PublicKey().Bytes() hello.keyShares = []keyShare{ - {group: x25519Kyber768Draft00, data: append(keyShareKeys.ecdhe.PublicKey().Bytes(), - keyShareKeys.kyber.EncapsulationKey()...)}, - {group: X25519, data: keyShareKeys.ecdhe.PublicKey().Bytes()}, + {group: X25519MLKEM768, data: append(mlkemEncapsulationKey, x25519EphemeralKey...)}, + } + // If both X25519MLKEM768 and X25519 are supported, we send both key + // shares (as a fallback) and we reuse the same X25519 ephemeral + // key, as allowed by draft-ietf-tls-hybrid-design-09, Section 3.2. + if slices.Contains(hello.supportedCurves, X25519) { + hello.keyShares = append(hello.keyShares, keyShare{group: X25519, data: x25519EphemeralKey}) } } else { if _, ok := curveForCurveID(curveID); !ok { @@ -194,7 +193,7 @@ func (c *Conn) makeClientHello() (*clientHelloMsg, *keySharePrivateKeys, *echCon hello.quicTransportParameters = p } - var ech *echContext + var ech *echClientContext if c.config.EncryptedClientHelloConfigList != nil { if c.config.MinVersion != 0 && c.config.MinVersion < VersionTLS13 { return nil, nil, nil, errors.New("tls: MinVersion must be >= VersionTLS13 if EncryptedClientHelloConfigList is populated") @@ -210,7 +209,7 @@ func (c *Conn) makeClientHello() (*clientHelloMsg, *keySharePrivateKeys, *echCon if echConfig == nil { return nil, nil, nil, errors.New("tls: EncryptedClientHelloConfigList contains no valid configs") } - ech = &echContext{config: echConfig} + ech = &echClientContext{config: echConfig} hello.encryptedClientHello = []byte{1} // indicate inner hello // We need to explicitly set these 1.2 fields to nil, as we do not // marshal them when encoding the inner hello, otherwise transcripts @@ -239,7 +238,7 @@ func (c *Conn) makeClientHello() (*clientHelloMsg, *keySharePrivateKeys, *echCon return hello, keyShareKeys, ech, nil } -type echContext struct { +type echClientContext struct { config *echConfig hpkeContext *hpke.Sender encapsulatedKey []byte @@ -248,6 +247,7 @@ type echContext struct { kdfID uint16 aeadID uint16 echRejected bool + retryConfigs []byte } func (c *Conn) clientHandshake(ctx context.Context) (err error) { @@ -258,6 +258,7 @@ func (c *Conn) clientHandshake(ctx context.Context) (err error) { // This may be a renegotiation handshake, in which case some fields // need to be reset. c.didResume = false + c.curveID = 0 hello, keyShareKeys, ech, err := c.makeClientHello() if err != nil { @@ -319,7 +320,7 @@ func (c *Conn) clientHandshake(ctx context.Context) (err error) { if err := transcriptMsg(hello, transcript); err != nil { return err } - earlyTrafficSecret := suite.deriveSecret(earlySecret, clientEarlyTrafficLabel, transcript) + earlyTrafficSecret := earlySecret.ClientEarlyTrafficSecret(transcript) c.quicSetWriteSecret(QUICEncryptionLevelEarly, suite.id, earlyTrafficSecret) } @@ -377,7 +378,7 @@ func (c *Conn) clientHandshake(ctx context.Context) (err error) { } func (c *Conn) loadSession(hello *clientHelloMsg) ( - session *SessionState, earlySecret, binderKey []byte, err error) { + session *SessionState, earlySecret *tls13.EarlySecret, binderKey []byte, err error) { if c.config.SessionTicketsDisabled || c.config.ClientSessionCache == nil { return nil, nil, nil, nil } @@ -449,6 +450,11 @@ func (c *Conn) loadSession(hello *clientHelloMsg) ( return nil, nil, nil, nil } + // FIPS 140-3 requires the use of Extended Master Secret. + if !session.extMasterSecret && fips140tls.Required() { + return nil, nil, nil, nil + } + hello.sessionTicket = session.ticket return } @@ -504,8 +510,8 @@ func (c *Conn) loadSession(hello *clientHelloMsg) ( hello.pskBinders = [][]byte{make([]byte, cipherSuite.hash.Size())} // Compute the PSK binders. See RFC 8446, Section 4.2.11.2. - earlySecret = cipherSuite.extract(session.secret, nil) - binderKey = cipherSuite.deriveSecret(earlySecret, resumptionBinderLabel, nil) + earlySecret = tls13.NewEarlySecret(cipherSuite.hash.New, session.secret) + binderKey = earlySecret.ResumptionBinderKey() transcript := cipherSuite.hash.New() if err := computeAndUpdatePSK(hello, binderKey, transcript, cipherSuite.finishedHash); err != nil { return nil, nil, nil, err @@ -539,6 +545,19 @@ func (c *Conn) pickTLSVersion(serverHello *serverHelloMsg) error { func (hs *clientHandshakeState) handshake() error { c := hs.c + // If we did not load a session (hs.session == nil), but we did set a + // session ID in the transmitted client hello (hs.hello.sessionId != nil), + // it means we tried to negotiate TLS 1.3 and sent a random session ID as a + // compatibility measure (see RFC 8446, Section 4.1.2). + // + // Since we're now handshaking for TLS 1.2, if the server echoed the + // transmitted ID back to us, we know mischief is afoot: the session ID + // was random and can't possibly be recognized by the server. + if hs.session == nil && hs.hello.sessionId != nil && bytes.Equal(hs.hello.sessionId, hs.serverHello.sessionId) { + c.sendAlert(alertIllegalParameter) + return errors.New("tls: server echoed TLS 1.3 compatibility session ID in TLS 1.2") + } + isResume, err := hs.processServerHello() if err != nil { return err @@ -626,11 +645,11 @@ func (hs *clientHandshakeState) pickCipherSuite() error { return errors.New("tls: server chose an unconfigured cipher suite") } - if hs.c.config.CipherSuites == nil && !needFIPS() && rsaKexCiphers[hs.suite.id] { + if hs.c.config.CipherSuites == nil && !fips140tls.Required() && rsaKexCiphers[hs.suite.id] { tlsrsakex.Value() // ensure godebug is initialized tlsrsakex.IncNonDefault() } - if hs.c.config.CipherSuites == nil && !needFIPS() && tdesCiphers[hs.suite.id] { + if hs.c.config.CipherSuites == nil && !fips140tls.Required() && tdesCiphers[hs.suite.id] { tls3des.Value() // ensure godebug is initialized tls3des.IncNonDefault() } @@ -704,11 +723,12 @@ func (hs *clientHandshakeState) doFullHandshake() error { if ok { err = keyAgreement.processServerKeyExchange(c.config, hs.hello, hs.serverHello, c.peerCertificates[0], skx) if err != nil { - c.sendAlert(alertUnexpectedMessage) + c.sendAlert(alertIllegalParameter) return err } - if len(skx.key) >= 3 && skx.key[0] == 3 /* named curve */ { - c.curveID = CurveID(byteorder.BeUint16(skx.key[1:])) + if keyAgreement, ok := keyAgreement.(*ecdheKeyAgreement); ok { + c.curveID = keyAgreement.curveID + c.peerSigAlg = keyAgreement.signatureAlgorithm } msg, err = c.readHandshake(&hs.finishedHash) @@ -768,6 +788,10 @@ func (hs *clientHandshakeState) doFullHandshake() error { hs.masterSecret = extMasterFromPreMasterSecret(c.vers, hs.suite, preMasterSecret, hs.finishedHash.Sum()) } else { + if fips140tls.Required() { + c.sendAlert(alertHandshakeFailure) + return errors.New("tls: FIPS 140-3 requires the use of Extended Master Secret") + } hs.masterSecret = masterFromPreMasterSecret(c.vers, hs.suite, preMasterSecret, hs.hello.random, hs.serverHello.random) } @@ -790,7 +814,7 @@ func (hs *clientHandshakeState) doFullHandshake() error { if c.vers >= VersionTLS12 { signatureAlgorithm, err := selectSignatureScheme(c.vers, chainToSend, certReq.supportedSignatureAlgorithms) if err != nil { - c.sendAlert(alertIllegalParameter) + c.sendAlert(alertHandshakeFailure) return err } sigType, sigHash, err = typeAndHashFromSignatureScheme(signatureAlgorithm) @@ -799,6 +823,10 @@ func (hs *clientHandshakeState) doFullHandshake() error { } certVerify.hasSignatureAlgorithm = true certVerify.signatureAlgorithm = signatureAlgorithm + if sigHash == crypto.SHA1 { + tlssha1.Value() // ensure godebug is initialized + tlssha1.IncNonDefault() + } } else { sigType, sigHash, err = legacyTypeAndHashFromPublicKey(key.Public()) if err != nil { @@ -865,10 +893,23 @@ func (hs *clientHandshakeState) processServerHello() (bool, error) { } if hs.serverHello.compressionMethod != compressionNone { - c.sendAlert(alertUnexpectedMessage) + c.sendAlert(alertIllegalParameter) return false, errors.New("tls: server selected unsupported compression format") } + supportsPointFormat := false + offeredNonCompressedFormat := false + for _, format := range hs.serverHello.supportedPoints { + if format == pointFormatUncompressed { + supportsPointFormat = true + } else { + offeredNonCompressedFormat = true + } + } + if !supportsPointFormat && offeredNonCompressedFormat { + return false, errors.New("tls: server offered only incompatible point formats") + } + if c.handshakes == 0 && hs.serverHello.secureRenegotiationSupported { c.secureRenegotiation = true if len(hs.serverHello.secureRenegotiation) != 0 { @@ -919,20 +960,20 @@ func (hs *clientHandshakeState) processServerHello() (bool, error) { hs.masterSecret = hs.session.secret c.extMasterSecret = hs.session.extMasterSecret c.peerCertificates = hs.session.peerCertificates - c.activeCertHandles = hs.c.activeCertHandles c.verifiedChains = hs.session.verifiedChains c.ocspResponse = hs.session.ocspResponse // Let the ServerHello SCTs override the session SCTs from the original - // connection, if any are provided + // connection, if any are provided. if len(c.scts) == 0 && len(hs.session.scts) != 0 { c.scts = hs.session.scts } + c.curveID = hs.session.curveID return true, nil } // checkALPN ensure that the server's choice of ALPN protocol is compatible with -// the protocols that we advertised in the Client Hello. +// the protocols that we advertised in the ClientHello. func checkALPN(clientProtos []string, serverProto string, quic bool) error { if serverProto == "" { if quic && len(clientProtos) > 0 { @@ -1069,23 +1110,21 @@ func checkKeySize(n int) (max int, ok bool) { // verifyServerCertificate parses and verifies the provided chain, setting // c.verifiedChains and c.peerCertificates or sending the appropriate alert. func (c *Conn) verifyServerCertificate(certificates [][]byte) error { - activeHandles := make([]*activeCert, len(certificates)) certs := make([]*x509.Certificate, len(certificates)) for i, asn1Data := range certificates { cert, err := globalCertCache.newCert(asn1Data) if err != nil { - c.sendAlert(alertBadCertificate) + c.sendAlert(alertDecodeError) return errors.New("tls: failed to parse certificate from server: " + err.Error()) } - if cert.cert.PublicKeyAlgorithm == x509.RSA { - n := cert.cert.PublicKey.(*rsa.PublicKey).N.BitLen() + if cert.PublicKeyAlgorithm == x509.RSA { + n := cert.PublicKey.(*rsa.PublicKey).N.BitLen() if max, ok := checkKeySize(n); !ok { c.sendAlert(alertBadCertificate) return fmt.Errorf("tls: server sent certificate containing RSA key larger than %d bits", max) } } - activeHandles[i] = cert - certs[i] = cert.cert + certs[i] = cert } echRejected := c.config.EncryptedClientHelloConfigList != nil && !c.echAccepted @@ -1106,8 +1145,13 @@ func (c *Conn) verifyServerCertificate(certificates [][]byte) error { for _, cert := range certs[1:] { opts.Intermediates.AddCert(cert) } - var err error - c.verifiedChains, err = certs[0].Verify(opts) + chains, err := certs[0].Verify(opts) + if err != nil { + c.sendAlert(alertBadCertificate) + return &CertificateVerificationError{UnverifiedCertificates: certs, Err: err} + } + + c.verifiedChains, err = fipsAllowedChains(chains) if err != nil { c.sendAlert(alertBadCertificate) return &CertificateVerificationError{UnverifiedCertificates: certs, Err: err} @@ -1124,8 +1168,13 @@ func (c *Conn) verifyServerCertificate(certificates [][]byte) error { for _, cert := range certs[1:] { opts.Intermediates.AddCert(cert) } - var err error - c.verifiedChains, err = certs[0].Verify(opts) + chains, err := certs[0].Verify(opts) + if err != nil { + c.sendAlert(alertBadCertificate) + return &CertificateVerificationError{UnverifiedCertificates: certs, Err: err} + } + + c.verifiedChains, err = fipsAllowedChains(chains) if err != nil { c.sendAlert(alertBadCertificate) return &CertificateVerificationError{UnverifiedCertificates: certs, Err: err} @@ -1140,7 +1189,6 @@ func (c *Conn) verifyServerCertificate(certificates [][]byte) error { return fmt.Errorf("tls: server's certificate contains an unsupported type of public key: %T", certs[0].PublicKey) } - c.activeCertHandles = activeHandles c.peerCertificates = certs if c.config.VerifyPeerCertificate != nil && !echRejected { diff --git a/src/crypto/tls/handshake_client_test.go b/src/crypto/tls/handshake_client_test.go index 501f9c6755f9e3..6020c0f055c776 100644 --- a/src/crypto/tls/handshake_client_test.go +++ b/src/crypto/tls/handshake_client_test.go @@ -11,6 +11,7 @@ import ( "crypto/elliptic" "crypto/rand" "crypto/rsa" + "crypto/tls/internal/fips140tls" "crypto/x509" "crypto/x509/pkix" "encoding/base64" @@ -207,7 +208,7 @@ func (test *clientTest) connFromCommand() (conn *recordingConn, child *exec.Cmd, var serverInfo bytes.Buffer for _, ext := range test.extensions { pem.Encode(&serverInfo, &pem.Block{ - Type: fmt.Sprintf("SERVERINFO FOR EXTENSION %d", byteorder.BeUint16(ext)), + Type: fmt.Sprintf("SERVERINFO FOR EXTENSION %d", byteorder.BEUint16(ext)), Bytes: ext, }) } @@ -433,7 +434,7 @@ func (test *clientTest) run(t *testing.T, write bool) { } if write { - clientConn.Close() + client.Close() path := test.dataPath() out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { @@ -637,7 +638,7 @@ func TestHandshakeClientHelloRetryRequest(t *testing.T) { func TestHandshakeClientECDHERSAChaCha20(t *testing.T) { config := testConfig.Clone() - config.CipherSuites = []uint16{TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305} + config.CipherSuites = []uint16{TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} test := &clientTest{ name: "ECDHE-RSA-CHACHA20-POLY1305", @@ -650,7 +651,7 @@ func TestHandshakeClientECDHERSAChaCha20(t *testing.T) { func TestHandshakeClientECDHEECDSAChaCha20(t *testing.T) { config := testConfig.Clone() - config.CipherSuites = []uint16{TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305} + config.CipherSuites = []uint16{TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256} test := &clientTest{ name: "ECDHE-ECDSA-CHACHA20-POLY1305", @@ -848,13 +849,17 @@ func testResumption(t *testing.T, version uint16) { if testing.Short() { t.Skip("skipping in -short mode") } + + // Note: using RSA 2048 test certificates because they are compatible with FIPS mode. + testCertificates := []Certificate{{Certificate: [][]byte{testRSA2048Certificate}, PrivateKey: testRSA2048PrivateKey}} serverConfig := &Config{ MaxVersion: version, - CipherSuites: []uint16{TLS_RSA_WITH_RC4_128_SHA, TLS_ECDHE_RSA_WITH_RC4_128_SHA}, - Certificates: testConfig.Certificates, + CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384}, + Certificates: testCertificates, + Time: testTime, } - issuer, err := x509.ParseCertificate(testRSACertificateIssuer) + issuer, err := x509.ParseCertificate(testRSA2048CertificateIssuer) if err != nil { panic(err) } @@ -864,10 +869,11 @@ func testResumption(t *testing.T, version uint16) { clientConfig := &Config{ MaxVersion: version, - CipherSuites: []uint16{TLS_RSA_WITH_RC4_128_SHA}, + CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256}, ClientSessionCache: NewLRUClientSessionCache(32), RootCAs: rootCAs, ServerName: "example.golang", + Time: testTime, } testResumeState := func(test string, didResume bool) { @@ -914,7 +920,7 @@ func testResumption(t *testing.T, version uint16) { // An old session ticket is replaced with a ticket encrypted with a fresh key. ticket = getTicket() - serverConfig.Time = func() time.Time { return time.Now().Add(24*time.Hour + time.Minute) } + serverConfig.Time = func() time.Time { return testTime().Add(24*time.Hour + time.Minute) } testResumeState("ResumeWithOldTicket", true) if bytes.Equal(ticket, getTicket()) { t.Fatal("old first ticket matches the fresh one") @@ -922,13 +928,13 @@ func testResumption(t *testing.T, version uint16) { // Once the session master secret is expired, a full handshake should occur. ticket = getTicket() - serverConfig.Time = func() time.Time { return time.Now().Add(24*8*time.Hour + time.Minute) } + serverConfig.Time = func() time.Time { return testTime().Add(24*8*time.Hour + time.Minute) } testResumeState("ResumeWithExpiredTicket", false) if bytes.Equal(ticket, getTicket()) { t.Fatal("expired first ticket matches the fresh one") } - serverConfig.Time = func() time.Time { return time.Now() } // reset the time back + serverConfig.Time = testTime // reset the time back key1 := randomKey() serverConfig.SetSessionTicketKeys([][32]byte{key1}) @@ -945,11 +951,11 @@ func testResumption(t *testing.T, version uint16) { testResumeState("KeyChangeFinish", true) // Age the session ticket a bit, but not yet expired. - serverConfig.Time = func() time.Time { return time.Now().Add(24*time.Hour + time.Minute) } + serverConfig.Time = func() time.Time { return testTime().Add(24*time.Hour + time.Minute) } testResumeState("OldSessionTicket", true) ticket = getTicket() // Expire the session ticket, which would force a full handshake. - serverConfig.Time = func() time.Time { return time.Now().Add(24*8*time.Hour + time.Minute) } + serverConfig.Time = func() time.Time { return testTime().Add(24*8*time.Hour + 2*time.Minute) } testResumeState("ExpiredSessionTicket", false) if bytes.Equal(ticket, getTicket()) { t.Fatal("new ticket wasn't provided after old ticket expired") @@ -957,7 +963,7 @@ func testResumption(t *testing.T, version uint16) { // Age the session ticket a bit at a time, but don't expire it. d := 0 * time.Hour - serverConfig.Time = func() time.Time { return time.Now().Add(d) } + serverConfig.Time = func() time.Time { return testTime().Add(d) } deleteTicket() testResumeState("GetFreshSessionTicket", false) for i := 0; i < 13; i++ { @@ -968,7 +974,7 @@ func testResumption(t *testing.T, version uint16) { // handshake occurs for TLS 1.2. Resumption should still occur for // TLS 1.3 since the client should be using a fresh ticket sent over // by the server. - d += 12 * time.Hour + d += 12*time.Hour + time.Minute if version == VersionTLS13 { testResumeState("ExpiredSessionTicket", true) } else { @@ -982,8 +988,9 @@ func testResumption(t *testing.T, version uint16) { // before the serverConfig is used works. serverConfig = &Config{ MaxVersion: version, - CipherSuites: []uint16{TLS_RSA_WITH_RC4_128_SHA, TLS_ECDHE_RSA_WITH_RC4_128_SHA}, - Certificates: testConfig.Certificates, + CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384}, + Certificates: testCertificates, + Time: testTime, } serverConfig.SetSessionTicketKeys([][32]byte{key2}) @@ -992,7 +999,7 @@ func testResumption(t *testing.T, version uint16) { // In TLS 1.3, cross-cipher suite resumption is allowed as long as the KDF // hash matches. Also, Config.CipherSuites does not apply to TLS 1.3. if version != VersionTLS13 { - clientConfig.CipherSuites = []uint16{TLS_ECDHE_RSA_WITH_RC4_128_SHA} + clientConfig.CipherSuites = []uint16{TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384} testResumeState("DifferentCipherSuite", false) testResumeState("DifferentCipherSuiteRecovers", true) } @@ -1008,7 +1015,8 @@ func testResumption(t *testing.T, version uint16) { // Use a different curve than the client to force a HelloRetryRequest. CurvePreferences: []CurveID{CurveP521, CurveP384, CurveP256}, MaxVersion: version, - Certificates: testConfig.Certificates, + Certificates: testCertificates, + Time: testTime, } testResumeState("InitialHandshake", false) testResumeState("WithHelloRetryRequest", true) @@ -1016,8 +1024,9 @@ func testResumption(t *testing.T, version uint16) { // Reset serverConfig back. serverConfig = &Config{ MaxVersion: version, - CipherSuites: []uint16{TLS_RSA_WITH_RC4_128_SHA, TLS_ECDHE_RSA_WITH_RC4_128_SHA}, - Certificates: testConfig.Certificates, + CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384}, + Certificates: testCertificates, + Time: testTime, } } @@ -1272,7 +1281,7 @@ func TestServerSelectingUnconfiguredApplicationProtocol(t *testing.T) { go func() { client := Client(c, &Config{ ServerName: "foo", - CipherSuites: []uint16{TLS_RSA_WITH_AES_128_GCM_SHA256}, + CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256}, NextProtos: []string{"http", "something-else"}, }) errChan <- client.Handshake() @@ -1292,7 +1301,7 @@ func TestServerSelectingUnconfiguredApplicationProtocol(t *testing.T) { serverHello := &serverHelloMsg{ vers: VersionTLS12, random: make([]byte, 32), - cipherSuite: TLS_RSA_WITH_AES_128_GCM_SHA256, + cipherSuite: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, alpnProtocol: "how-about-this", } serverHelloBytes := mustMarshal(t, serverHello) @@ -1308,7 +1317,7 @@ func TestServerSelectingUnconfiguredApplicationProtocol(t *testing.T) { s.Close() if err := <-errChan; !strings.Contains(err.Error(), "server selected unadvertised ALPN protocol") { - t.Fatalf("Expected error about unconfigured cipher suite but got %q", err) + t.Fatalf("Expected error about unconfigured ALPN protocol but got %q", err) } } @@ -1724,7 +1733,10 @@ func testVerifyConnection(t *testing.T, version uint16) { }, } for _, test := range tests { - issuer, err := x509.ParseCertificate(testRSACertificateIssuer) + // Note: using RSA 2048 test certificates because they are compatible with FIPS mode. + testCertificates := []Certificate{{Certificate: [][]byte{testRSA2048Certificate}, PrivateKey: testRSA2048PrivateKey}} + + issuer, err := x509.ParseCertificate(testRSA2048CertificateIssuer) if err != nil { panic(err) } @@ -1735,7 +1747,8 @@ func testVerifyConnection(t *testing.T, version uint16) { serverConfig := &Config{ MaxVersion: version, - Certificates: []Certificate{testConfig.Certificates[0]}, + Certificates: testCertificates, + Time: testTime, ClientCAs: rootCAs, NextProtos: []string{"protocol1"}, } @@ -1748,7 +1761,8 @@ func testVerifyConnection(t *testing.T, version uint16) { ClientSessionCache: NewLRUClientSessionCache(32), RootCAs: rootCAs, ServerName: "example.golang", - Certificates: []Certificate{testConfig.Certificates[0]}, + Certificates: testCertificates, + Time: testTime, NextProtos: []string{"protocol1"}, } test.configureClient(clientConfig, &clientCalled) @@ -1783,7 +1797,8 @@ func TestVerifyPeerCertificate(t *testing.T) { } func testVerifyPeerCertificate(t *testing.T, version uint16) { - issuer, err := x509.ParseCertificate(testRSACertificateIssuer) + // Note: using RSA 2048 test certificates because they are compatible with FIPS mode. + issuer, err := x509.ParseCertificate(testRSA2048CertificateIssuer) if err != nil { panic(err) } @@ -1791,8 +1806,6 @@ func testVerifyPeerCertificate(t *testing.T, version uint16) { rootCAs := x509.NewCertPool() rootCAs.AddCert(issuer) - now := func() time.Time { return time.Unix(1476984729, 0) } - sentinelErr := errors.New("TestVerifyPeerCertificate") verifyPeerCertificateCallback := func(called *bool, rawCerts [][]byte, validatedChains [][]*x509.Certificate) error { @@ -2038,11 +2051,11 @@ func testVerifyPeerCertificate(t *testing.T, version uint16) { config.ServerName = "example.golang" config.ClientAuth = RequireAndVerifyClientCert config.ClientCAs = rootCAs - config.Time = now + config.Time = testTime config.MaxVersion = version config.Certificates = make([]Certificate, 1) - config.Certificates[0].Certificate = [][]byte{testRSACertificate} - config.Certificates[0].PrivateKey = testRSAPrivateKey + config.Certificates[0].Certificate = [][]byte{testRSA2048Certificate} + config.Certificates[0].PrivateKey = testRSA2048PrivateKey config.Certificates[0].SignedCertificateTimestamps = [][]byte{[]byte("dummy sct 1"), []byte("dummy sct 2")} config.Certificates[0].OCSPStaple = []byte("dummy ocsp") test.configureServer(config, &serverCalled) @@ -2053,9 +2066,10 @@ func testVerifyPeerCertificate(t *testing.T, version uint16) { }() config := testConfig.Clone() + config.Certificates = []Certificate{{Certificate: [][]byte{testRSA2048Certificate}, PrivateKey: testRSA2048PrivateKey}} config.ServerName = "example.golang" config.RootCAs = rootCAs - config.Time = now + config.Time = testTime config.MaxVersion = version test.configureClient(config, &clientCalled) clientErr := Client(c, config).Handshake() @@ -2336,8 +2350,8 @@ var getClientCertificateTests = []struct { panic("empty AcceptableCAs") } cert := &Certificate{ - Certificate: [][]byte{testRSACertificate}, - PrivateKey: testRSAPrivateKey, + Certificate: [][]byte{testRSA2048Certificate}, + PrivateKey: testRSA2048PrivateKey, } return cert, nil } @@ -2357,25 +2371,34 @@ func TestGetClientCertificate(t *testing.T) { } func testGetClientCertificate(t *testing.T, version uint16) { - issuer, err := x509.ParseCertificate(testRSACertificateIssuer) + // Note: using RSA 2048 test certificates because they are compatible with FIPS mode. + issuer, err := x509.ParseCertificate(testRSA2048CertificateIssuer) if err != nil { panic(err) } for i, test := range getClientCertificateTests { serverConfig := testConfig.Clone() + serverConfig.Certificates = []Certificate{{Certificate: [][]byte{testRSA2048Certificate}, PrivateKey: testRSA2048PrivateKey}} serverConfig.ClientAuth = VerifyClientCertIfGiven serverConfig.RootCAs = x509.NewCertPool() serverConfig.RootCAs.AddCert(issuer) serverConfig.ClientCAs = serverConfig.RootCAs - serverConfig.Time = func() time.Time { return time.Unix(1476984729, 0) } + serverConfig.Time = testTime serverConfig.MaxVersion = version clientConfig := testConfig.Clone() + clientConfig.Certificates = []Certificate{{Certificate: [][]byte{testRSA2048Certificate}, PrivateKey: testRSA2048PrivateKey}} clientConfig.MaxVersion = version test.setup(clientConfig, serverConfig) + // TLS 1.1 isn't available for FIPS required + if fips140tls.Required() && clientConfig.MaxVersion == VersionTLS11 { + t.Logf("skipping test %d for FIPS mode", i) + continue + } + type serverResult struct { cs ConnectionState err error @@ -2514,11 +2537,15 @@ func TestDowngradeCanary(t *testing.T) { if err := testDowngradeCanary(t, VersionTLS12, VersionTLS12); err != nil { t.Errorf("client didn't ignore expected TLS 1.2 canary") } - if err := testDowngradeCanary(t, VersionTLS11, VersionTLS11); err != nil { - t.Errorf("client unexpectedly reacted to a canary in TLS 1.1") - } - if err := testDowngradeCanary(t, VersionTLS10, VersionTLS10); err != nil { - t.Errorf("client unexpectedly reacted to a canary in TLS 1.0") + if !fips140tls.Required() { + if err := testDowngradeCanary(t, VersionTLS11, VersionTLS11); err != nil { + t.Errorf("client unexpectedly reacted to a canary in TLS 1.1") + } + if err := testDowngradeCanary(t, VersionTLS10, VersionTLS10); err != nil { + t.Errorf("client unexpectedly reacted to a canary in TLS 1.0") + } + } else { + t.Logf("skiping TLS 1.1 and TLS 1.0 downgrade canary checks in FIPS mode") } } @@ -2528,7 +2555,8 @@ func TestResumptionKeepsOCSPAndSCT(t *testing.T) { } func testResumptionKeepsOCSPAndSCT(t *testing.T, ver uint16) { - issuer, err := x509.ParseCertificate(testRSACertificateIssuer) + // Note: using RSA 2048 test certificates because they are compatible with FIPS mode. + issuer, err := x509.ParseCertificate(testRSA2048CertificateIssuer) if err != nil { t.Fatalf("failed to parse test issuer") } @@ -2539,8 +2567,10 @@ func testResumptionKeepsOCSPAndSCT(t *testing.T, ver uint16) { ClientSessionCache: NewLRUClientSessionCache(32), ServerName: "example.golang", RootCAs: roots, + Time: testTime, } serverConfig := testConfig.Clone() + serverConfig.Certificates = []Certificate{{Certificate: [][]byte{testRSA2048Certificate}, PrivateKey: testRSA2048PrivateKey}} serverConfig.MaxVersion = ver serverConfig.Certificates[0].OCSPStaple = []byte{1, 2, 3} serverConfig.Certificates[0].SignedCertificateTimestamps = [][]byte{{4, 5, 6}} @@ -2625,8 +2655,8 @@ func TestClientHandshakeContextCancellation(t *testing.T) { if err != context.Canceled { t.Errorf("Unexpected client handshake error: %v", err) } - if runtime.GOARCH == "wasm" { - t.Skip("conn.Close does not error as expected when called multiple times on WASM") + if runtime.GOOS == "js" || runtime.GOOS == "wasip1" { + t.Skip("conn.Close does not error as expected when called multiple times on GOOS=js or GOOS=wasip1") } err = cli.Close() if err == nil { @@ -2663,7 +2693,6 @@ func TestTLS13OnlyClientHelloCipherSuite(t *testing.T) { }, } for _, tt := range tls13Tests { - tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() testTLS13OnlyClientHelloCipherSuite(t, tt.ciphers) @@ -2675,11 +2704,15 @@ func testTLS13OnlyClientHelloCipherSuite(t *testing.T, ciphers []uint16) { serverConfig := &Config{ Certificates: testConfig.Certificates, GetConfigForClient: func(chi *ClientHelloInfo) (*Config, error) { - if len(chi.CipherSuites) != len(defaultCipherSuitesTLS13NoAES) { + expectedCiphersuites := defaultCipherSuitesTLS13NoAES + if fips140tls.Required() { + expectedCiphersuites = allowedCipherSuitesTLS13FIPS + } + if len(chi.CipherSuites) != len(expectedCiphersuites) { t.Errorf("only TLS 1.3 suites should be advertised, got=%x", chi.CipherSuites) } else { - for i := range defaultCipherSuitesTLS13NoAES { - if want, got := defaultCipherSuitesTLS13NoAES[i], chi.CipherSuites[i]; want != got { + for i := range expectedCiphersuites { + if want, got := expectedCiphersuites[i], chi.CipherSuites[i]; want != got { t.Errorf("cipher at index %d does not match, want=%x, got=%x", i, want, got) } } diff --git a/src/crypto/tls/handshake_client_tls13.go b/src/crypto/tls/handshake_client_tls13.go index db5e35d9a46c2e..7018bb2336b8f3 100644 --- a/src/crypto/tls/handshake_client_tls13.go +++ b/src/crypto/tls/handshake_client_tls13.go @@ -8,8 +8,10 @@ import ( "bytes" "context" "crypto" + "crypto/hkdf" "crypto/hmac" - "crypto/internal/mlkem768" + "crypto/internal/fips140/mlkem" + "crypto/internal/fips140/tls13" "crypto/rsa" "crypto/subtle" "errors" @@ -26,7 +28,7 @@ type clientHandshakeStateTLS13 struct { keyShareKeys *keySharePrivateKeys session *SessionState - earlySecret []byte + earlySecret *tls13.EarlySecret binderKey []byte certReq *certificateRequestMsgTLS13 @@ -34,10 +36,10 @@ type clientHandshakeStateTLS13 struct { sentDummyCCS bool suite *cipherSuiteTLS13 transcript hash.Hash - masterSecret []byte + masterSecret *tls13.MasterSecret trafficSecret []byte // client_application_traffic_secret_0 - echContext *echContext + echContext *echClientContext } // handshake requires hs.c, hs.hello, hs.serverHello, hs.keyShareKeys, and, @@ -45,10 +47,6 @@ type clientHandshakeStateTLS13 struct { func (hs *clientHandshakeStateTLS13) handshake() error { c := hs.c - if needFIPS() { - return errors.New("tls: internal error: TLS 1.3 reached in FIPS mode") - } - // The server must not select TLS 1.3 in a renegotiation. See RFC 8446, // sections 4.1.2 and 4.1.3. if c.handshakes > 0 { @@ -87,18 +85,18 @@ func (hs *clientHandshakeStateTLS13) handshake() error { } } - var echRetryConfigList []byte if hs.echContext != nil { confTranscript := cloneHash(hs.echContext.innerTranscript, hs.suite.hash) confTranscript.Write(hs.serverHello.original[:30]) confTranscript.Write(make([]byte, 8)) confTranscript.Write(hs.serverHello.original[38:]) - acceptConfirmation := hs.suite.expandLabel( - hs.suite.extract(hs.echContext.innerHello.random, nil), - "ech accept confirmation", - confTranscript.Sum(nil), - 8, - ) + h := hs.suite.hash.New + prk, err := hkdf.Extract(h, hs.echContext.innerHello.random, nil) + if err != nil { + c.sendAlert(alertInternalError) + return err + } + acceptConfirmation := tls13.ExpandLabel(h, prk, "ech accept confirmation", confTranscript.Sum(nil), 8) if subtle.ConstantTimeCompare(acceptConfirmation, hs.serverHello.random[len(hs.serverHello.random)-8:]) == 1 { hs.hello = hs.echContext.innerHello c.serverName = c.config.ServerName @@ -107,7 +105,7 @@ func (hs *clientHandshakeStateTLS13) handshake() error { if hs.serverHello.encryptedClientHello != nil { c.sendAlert(alertUnsupportedExtension) - return errors.New("tls: unexpected encrypted_client_hello extension in server hello despite ECH being accepted") + return errors.New("tls: unexpected encrypted client hello extension in server hello despite ECH being accepted") } if hs.hello.serverName == "" && hs.serverHello.serverNameAck { @@ -116,9 +114,6 @@ func (hs *clientHandshakeStateTLS13) handshake() error { } } else { hs.echContext.echRejected = true - // If the server sent us retry configs, we'll return these to - // the user so they can update their Config. - echRetryConfigList = hs.serverHello.encryptedClientHello } } @@ -157,7 +152,7 @@ func (hs *clientHandshakeStateTLS13) handshake() error { if hs.echContext != nil && hs.echContext.echRejected { c.sendAlert(alertECHRequired) - return &ECHRejectionError{echRetryConfigList} + return &ECHRejectionError{hs.echContext.retryConfigs} } c.isHandshakeComplete.Store(true) @@ -202,8 +197,8 @@ func (hs *clientHandshakeStateTLS13) checkServerHelloOrHRR() error { } if hs.serverHello.compressionMethod != compressionNone { - c.sendAlert(alertIllegalParameter) - return errors.New("tls: server selected unsupported compression format") + c.sendAlert(alertDecodeError) + return errors.New("tls: server sent non-zero legacy TLS compression method") } selectedSuite := mutualCipherSuiteTLS13(hs.hello.cipherSuites, hs.serverHello.cipherSuite) @@ -270,12 +265,13 @@ func (hs *clientHandshakeStateTLS13) processHelloRetryRequest() error { copy(hrrHello, hs.serverHello.original) hrrHello = bytes.Replace(hrrHello, hs.serverHello.encryptedClientHello, make([]byte, 8), 1) confTranscript.Write(hrrHello) - acceptConfirmation := hs.suite.expandLabel( - hs.suite.extract(hs.echContext.innerHello.random, nil), - "hrr ech accept confirmation", - confTranscript.Sum(nil), - 8, - ) + h := hs.suite.hash.New + prk, err := hkdf.Extract(h, hs.echContext.innerHello.random, nil) + if err != nil { + c.sendAlert(alertInternalError) + return err + } + acceptConfirmation := tls13.ExpandLabel(h, prk, "hrr ech accept confirmation", confTranscript.Sum(nil), 8) if subtle.ConstantTimeCompare(acceptConfirmation, hs.serverHello.encryptedClientHello) == 1 { hello = hs.echContext.innerHello c.serverName = c.config.ServerName @@ -290,7 +286,7 @@ func (hs *clientHandshakeStateTLS13) processHelloRetryRequest() error { } else if hs.serverHello.encryptedClientHello != nil { // Unsolicited ECH extension should be rejected c.sendAlert(alertUnsupportedExtension) - return errors.New("tls: unexpected ECH extension in serverHello") + return errors.New("tls: unexpected encrypted client hello extension in serverHello") } // The only HelloRetryRequest extensions we support are key_share and @@ -324,12 +320,11 @@ func (hs *clientHandshakeStateTLS13) processHelloRetryRequest() error { c.sendAlert(alertIllegalParameter) return errors.New("tls: server sent an unnecessary HelloRetryRequest key_share") } - // Note: we don't support selecting X25519Kyber768Draft00 in a HRR, - // because we currently only support it at all when CurvePreferences is - // empty, which will cause us to also send a key share for it. + // Note: we don't support selecting X25519MLKEM768 in a HRR, because it + // is currently first in preference order, so if it's enabled we'll + // always send a key share for it. // - // This will have to change once we support selecting hybrid KEMs - // without sending key shares for them. + // This will have to change once we support multiple hybrid KEMs. if _, ok := curveForCurveID(curveID); !ok { c.sendAlert(alertInternalError) return errors.New("tls: CurvePreferences includes unsupported curve") @@ -471,7 +466,6 @@ func (hs *clientHandshakeStateTLS13) processServerHello() error { hs.usingPSK = true c.didResume = true c.peerCertificates = hs.session.peerCertificates - c.activeCertHandles = hs.session.activeCertHandles c.verifiedChains = hs.session.verifiedChains c.ocspResponse = hs.session.ocspResponse c.scts = hs.session.scts @@ -482,12 +476,12 @@ func (hs *clientHandshakeStateTLS13) establishHandshakeKeys() error { c := hs.c ecdhePeerData := hs.serverHello.serverShare.data - if hs.serverHello.serverShare.group == x25519Kyber768Draft00 { - if len(ecdhePeerData) != x25519PublicKeySize+mlkem768.CiphertextSize { + if hs.serverHello.serverShare.group == X25519MLKEM768 { + if len(ecdhePeerData) != mlkem.CiphertextSize768+x25519PublicKeySize { c.sendAlert(alertIllegalParameter) - return errors.New("tls: invalid server key share") + return errors.New("tls: invalid server X25519MLKEM768 key share") } - ecdhePeerData = hs.serverHello.serverShare.data[:x25519PublicKeySize] + ecdhePeerData = hs.serverHello.serverShare.data[mlkem.CiphertextSize768:] } peerKey, err := hs.keyShareKeys.ecdhe.Curve().NewPublicKey(ecdhePeerData) if err != nil { @@ -499,33 +493,30 @@ func (hs *clientHandshakeStateTLS13) establishHandshakeKeys() error { c.sendAlert(alertIllegalParameter) return errors.New("tls: invalid server key share") } - if hs.serverHello.serverShare.group == x25519Kyber768Draft00 { - if hs.keyShareKeys.kyber == nil { + if hs.serverHello.serverShare.group == X25519MLKEM768 { + if hs.keyShareKeys.mlkem == nil { return c.sendAlert(alertInternalError) } - ciphertext := hs.serverHello.serverShare.data[x25519PublicKeySize:] - kyberShared, err := kyberDecapsulate(hs.keyShareKeys.kyber, ciphertext) + ciphertext := hs.serverHello.serverShare.data[:mlkem.CiphertextSize768] + mlkemShared, err := hs.keyShareKeys.mlkem.Decapsulate(ciphertext) if err != nil { c.sendAlert(alertIllegalParameter) - return errors.New("tls: invalid Kyber server key share") + return errors.New("tls: invalid X25519MLKEM768 server key share") } - sharedKey = append(sharedKey, kyberShared...) + sharedKey = append(mlkemShared, sharedKey...) } c.curveID = hs.serverHello.serverShare.group earlySecret := hs.earlySecret if !hs.usingPSK { - earlySecret = hs.suite.extract(nil, nil) + earlySecret = tls13.NewEarlySecret(hs.suite.hash.New, nil) } - handshakeSecret := hs.suite.extract(sharedKey, - hs.suite.deriveSecret(earlySecret, "derived", nil)) + handshakeSecret := earlySecret.HandshakeSecret(sharedKey) - clientSecret := hs.suite.deriveSecret(handshakeSecret, - clientHandshakeTrafficLabel, hs.transcript) + clientSecret := handshakeSecret.ClientHandshakeTrafficSecret(hs.transcript) c.out.setTrafficSecret(hs.suite, QUICEncryptionLevelHandshake, clientSecret) - serverSecret := hs.suite.deriveSecret(handshakeSecret, - serverHandshakeTrafficLabel, hs.transcript) + serverSecret := handshakeSecret.ServerHandshakeTrafficSecret(hs.transcript) c.in.setTrafficSecret(hs.suite, QUICEncryptionLevelHandshake, serverSecret) if c.quic != nil { @@ -547,8 +538,7 @@ func (hs *clientHandshakeStateTLS13) establishHandshakeKeys() error { return err } - hs.masterSecret = hs.suite.extract(nil, - hs.suite.deriveSecret(handshakeSecret, "derived", nil)) + hs.masterSecret = handshakeSecret.MasterSecret() return nil } @@ -608,9 +598,13 @@ func (hs *clientHandshakeStateTLS13) readServerParameters() error { return errors.New("tls: server accepted 0-RTT with the wrong ALPN") } } - if hs.echContext != nil && !hs.echContext.echRejected && encryptedExtensions.echRetryConfigs != nil { - c.sendAlert(alertUnsupportedExtension) - return errors.New("tls: server sent ECH retry configs after accepting ECH") + if hs.echContext != nil { + if hs.echContext.echRejected { + hs.echContext.retryConfigs = encryptedExtensions.echRetryConfigs + } else if encryptedExtensions.echRetryConfigs != nil { + c.sendAlert(alertUnsupportedExtension) + return errors.New("tls: server sent encrypted client hello retry configs after accepting encrypted client hello") + } } return nil @@ -681,7 +675,10 @@ func (hs *clientHandshakeStateTLS13) readServerCertificate() error { } // See RFC 8446, Section 4.4.3. - if !isSupportedSignatureAlgorithm(certVerify.signatureAlgorithm, supportedSignatureAlgorithms()) { + // We don't use hs.hello.supportedSignatureAlgorithms because it might + // include PKCS#1 v1.5 and SHA-1 if the ClientHello also supported TLS 1.2. + if !isSupportedSignatureAlgorithm(certVerify.signatureAlgorithm, supportedSignatureAlgorithms(c.vers)) || + !isSupportedSignatureAlgorithm(certVerify.signatureAlgorithm, signatureSchemesForPublicKey(c.vers, c.peerCertificates[0].PublicKey)) { c.sendAlert(alertIllegalParameter) return errors.New("tls: certificate used with invalid signature algorithm") } @@ -690,8 +687,7 @@ func (hs *clientHandshakeStateTLS13) readServerCertificate() error { return c.sendAlert(alertInternalError) } if sigType == signaturePKCS1v15 || sigHash == crypto.SHA1 { - c.sendAlert(alertIllegalParameter) - return errors.New("tls: certificate used with invalid signature algorithm") + return c.sendAlert(alertInternalError) } signed := signedMessage(sigHash, serverSignatureContext, hs.transcript) if err := verifyHandshakeSignature(sigType, c.peerCertificates[0].PublicKey, @@ -699,6 +695,7 @@ func (hs *clientHandshakeStateTLS13) readServerCertificate() error { c.sendAlert(alertDecryptError) return errors.New("tls: invalid signature by the server certificate: " + err.Error()) } + c.peerSigAlg = certVerify.signatureAlgorithm if err := transcriptMsg(certVerify, hs.transcript); err != nil { return err @@ -736,10 +733,8 @@ func (hs *clientHandshakeStateTLS13) readServerFinished() error { // Derive secrets that take context through the server Finished. - hs.trafficSecret = hs.suite.deriveSecret(hs.masterSecret, - clientApplicationTrafficLabel, hs.transcript) - serverSecret := hs.suite.deriveSecret(hs.masterSecret, - serverApplicationTrafficLabel, hs.transcript) + hs.trafficSecret = hs.masterSecret.ClientApplicationTrafficSecret(hs.transcript) + serverSecret := hs.masterSecret.ServerApplicationTrafficSecret(hs.transcript) c.in.setTrafficSecret(hs.suite, QUICEncryptionLevelApplication, serverSecret) err = c.config.writeKeyLog(keyLogLabelClientTraffic, hs.hello.random, hs.trafficSecret) @@ -846,8 +841,7 @@ func (hs *clientHandshakeStateTLS13) sendClientFinished() error { c.out.setTrafficSecret(hs.suite, QUICEncryptionLevelApplication, hs.trafficSecret) if !c.config.SessionTicketsDisabled && c.config.ClientSessionCache != nil { - c.resumptionSecret = hs.suite.deriveSecret(hs.masterSecret, - resumptionLabel, hs.transcript) + c.resumptionSecret = hs.masterSecret.ResumptionMasterSecret(hs.transcript) } if c.quic != nil { @@ -880,6 +874,11 @@ func (c *Conn) handleNewSessionTicket(msg *newSessionTicketMsgTLS13) error { return errors.New("tls: received a session ticket with invalid lifetime") } + if len(msg.label) == 0 { + c.sendAlert(alertDecodeError) + return errors.New("tls: received a session ticket with empty opaque ticket label") + } + // RFC 9001, Section 4.6.1 if c.quic != nil && msg.maxEarlyData != 0 && msg.maxEarlyData != 0xffffffff { c.sendAlert(alertIllegalParameter) @@ -891,7 +890,7 @@ func (c *Conn) handleNewSessionTicket(msg *newSessionTicketMsgTLS13) error { return c.sendAlert(alertInternalError) } - psk := cipherSuite.expandLabel(c.resumptionSecret, "resumption", + psk := tls13.ExpandLabel(cipherSuite.hash.New, c.resumptionSecret, "resumption", msg.nonce, cipherSuite.hash.Size()) session := c.sessionState() diff --git a/src/crypto/tls/handshake_messages.go b/src/crypto/tls/handshake_messages.go index 8620b66a474903..aa0b7db75dd493 100644 --- a/src/crypto/tls/handshake_messages.go +++ b/src/crypto/tls/handshake_messages.go @@ -97,6 +97,8 @@ type clientHelloMsg struct { pskBinders [][]byte quicTransportParameters []byte encryptedClientHello []byte + // extensions are only populated on the server-side of a handshake + extensions []uint16 } func (m *clientHelloMsg) marshalMsg(echInner bool) ([]byte, error) { @@ -467,6 +469,7 @@ func (m *clientHelloMsg) unmarshal(data []byte) bool { return false } seenExts[extension] = true + m.extensions = append(m.extensions, extension) switch extension { case extensionServerName: @@ -659,6 +662,10 @@ func (m *clientHelloMsg) unmarshal(data []byte) bool { } m.pskBinders = append(m.pskBinders, binder) } + case extensionEncryptedClientHello: + if !extData.ReadBytes(&m.encryptedClientHello, len(extData)) { + return false + } default: // Ignore unknown extensions. continue @@ -998,6 +1005,7 @@ type encryptedExtensionsMsg struct { quicTransportParameters []byte earlyData bool echRetryConfigs []byte + serverNameAck bool } func (m *encryptedExtensionsMsg) marshal() ([]byte, error) { @@ -1033,6 +1041,10 @@ func (m *encryptedExtensionsMsg) marshal() ([]byte, error) { b.AddBytes(m.echRetryConfigs) }) } + if m.serverNameAck { + b.AddUint16(extensionServerName) + b.AddUint16(0) // empty extension_data + } }) }) @@ -1049,6 +1061,7 @@ func (m *encryptedExtensionsMsg) unmarshal(data []byte) bool { return false } + seenExts := make(map[uint16]bool) for !extensions.Empty() { var extension uint16 var extData cryptobyte.String @@ -1057,6 +1070,11 @@ func (m *encryptedExtensionsMsg) unmarshal(data []byte) bool { return false } + if seenExts[extension] { + return false + } + seenExts[extension] = true + switch extension { case extensionALPN: var protoList cryptobyte.String @@ -1082,6 +1100,11 @@ func (m *encryptedExtensionsMsg) unmarshal(data []byte) bool { if !extData.CopyBytes(m.echRetryConfigs) { return false } + case extensionServerName: + if len(extData) != 0 { + return false + } + m.serverNameAck = true default: // Ignore unknown extensions. continue @@ -1777,7 +1800,7 @@ func (m *certificateRequestMsg) unmarshal(data []byte) bool { } sigAndHashLen := uint16(data[0])<<8 | uint16(data[1]) data = data[2:] - if sigAndHashLen&1 != 0 { + if sigAndHashLen&1 != 0 || sigAndHashLen == 0 { return false } if len(data) < int(sigAndHashLen) { diff --git a/src/crypto/tls/handshake_messages_test.go b/src/crypto/tls/handshake_messages_test.go index 197a1c55eeb5a8..fa81a72b0de018 100644 --- a/src/crypto/tls/handshake_messages_test.go +++ b/src/crypto/tls/handshake_messages_test.go @@ -72,8 +72,16 @@ func TestMarshalUnmarshal(t *testing.T) { break } - if m, ok := m.(*SessionState); ok { - m.activeCertHandles = nil + if ch, ok := m.(*clientHelloMsg); ok { + // extensions is special cased, as it is only populated by the + // server-side of a handshake and is not expected to roundtrip + // through marshal + unmarshal. m ends up with the list of + // extensions necessary to serialize the other fields of + // clientHelloMsg, so check that it is non-empty, then clear it. + if len(ch.extensions) == 0 { + t.Errorf("expected ch.extensions to be populated on unmarshal") + } + ch.extensions = nil } // clientHelloMsg and serverHelloMsg, when unmarshalled, store @@ -171,10 +179,10 @@ func (*clientHelloMsg) Generate(rand *rand.Rand, size int) reflect.Value { } } if rand.Intn(10) > 5 { - m.supportedSignatureAlgorithms = supportedSignatureAlgorithms() + m.supportedSignatureAlgorithms = supportedSignatureAlgorithms(VersionTLS12) } if rand.Intn(10) > 5 { - m.supportedSignatureAlgorithmsCert = supportedSignatureAlgorithms() + m.supportedSignatureAlgorithmsCert = supportedSignatureAlgorithms(VersionTLS12) } for i := 0; i < rand.Intn(5); i++ { m.alpnProtocols = append(m.alpnProtocols, randomString(rand.Intn(20)+1, rand)) @@ -220,6 +228,9 @@ func (*clientHelloMsg) Generate(rand *rand.Rand, size int) reflect.Value { if rand.Intn(10) > 5 { m.earlyData = true } + if rand.Intn(10) > 5 { + m.encryptedClientHello = randomBytes(rand.Intn(50)+1, rand) + } return reflect.ValueOf(m) } @@ -409,11 +420,13 @@ func (*SessionState) Generate(rand *rand.Rand, size int) reflect.Value { if rand.Intn(10) > 5 && s.EarlyData { s.alpnProtocol = string(randomBytes(rand.Intn(10), rand)) } - if s.isClient { - if isTLS13 { + if isTLS13 { + if s.isClient { s.useBy = uint64(rand.Int63()) s.ageAdd = uint32(rand.Int63() & math.MaxUint32) } + } else { + s.curveID = CurveID(rand.Intn(30000) + 1) } return reflect.ValueOf(s) } @@ -460,10 +473,10 @@ func (*certificateRequestMsgTLS13) Generate(rand *rand.Rand, size int) reflect.V m.scts = true } if rand.Intn(10) > 5 { - m.supportedSignatureAlgorithms = supportedSignatureAlgorithms() + m.supportedSignatureAlgorithms = supportedSignatureAlgorithms(VersionTLS12) } if rand.Intn(10) > 5 { - m.supportedSignatureAlgorithmsCert = supportedSignatureAlgorithms() + m.supportedSignatureAlgorithmsCert = supportedSignatureAlgorithms(VersionTLS12) } if rand.Intn(10) > 5 { m.certificateAuthorities = make([][]byte, 3) diff --git a/src/crypto/tls/handshake_server.go b/src/crypto/tls/handshake_server.go index ac3d915d1746d7..a2cf176a86c0d8 100644 --- a/src/crypto/tls/handshake_server.go +++ b/src/crypto/tls/handshake_server.go @@ -11,11 +11,11 @@ import ( "crypto/ed25519" "crypto/rsa" "crypto/subtle" + "crypto/tls/internal/fips140tls" "crypto/x509" "errors" "fmt" "hash" - "internal/byteorder" "io" "time" ) @@ -40,7 +40,7 @@ type serverHandshakeState struct { // serverHandshake performs a TLS handshake as a server. func (c *Conn) serverHandshake(ctx context.Context) error { - clientHello, err := c.readClientHello(ctx) + clientHello, ech, err := c.readClientHello(ctx) if err != nil { return err } @@ -50,6 +50,7 @@ func (c *Conn) serverHandshake(ctx context.Context) error { c: c, ctx: ctx, clientHello: clientHello, + echContext: ech, } return hs.handshake() } @@ -130,17 +131,35 @@ func (hs *serverHandshakeState) handshake() error { } // readClientHello reads a ClientHello message and selects the protocol version. -func (c *Conn) readClientHello(ctx context.Context) (*clientHelloMsg, error) { +func (c *Conn) readClientHello(ctx context.Context) (*clientHelloMsg, *echServerContext, error) { // clientHelloMsg is included in the transcript, but we haven't initialized // it yet. The respective handshake functions will record it themselves. msg, err := c.readHandshake(nil) if err != nil { - return nil, err + return nil, nil, err } clientHello, ok := msg.(*clientHelloMsg) if !ok { c.sendAlert(alertUnexpectedMessage) - return nil, unexpectedMessageError(clientHello, msg) + return nil, nil, unexpectedMessageError(clientHello, msg) + } + + // ECH processing has to be done before we do any other negotiation based on + // the contents of the client hello, since we may swap it out completely. + var ech *echServerContext + if len(clientHello.encryptedClientHello) != 0 { + echKeys := c.config.EncryptedClientHelloKeys + if c.config.GetEncryptedClientHelloKeys != nil { + echKeys, err = c.config.GetEncryptedClientHelloKeys(clientHelloInfo(ctx, c, clientHello)) + if err != nil { + c.sendAlert(alertInternalError) + return nil, nil, err + } + } + clientHello, ech, err = c.processECHClientHello(clientHello, echKeys) + if err != nil { + return nil, nil, err + } } var configForClient *Config @@ -149,7 +168,7 @@ func (c *Conn) readClientHello(ctx context.Context) (*clientHelloMsg, error) { chi := clientHelloInfo(ctx, c, clientHello) if configForClient, err = c.config.GetConfigForClient(chi); err != nil { c.sendAlert(alertInternalError) - return nil, err + return nil, nil, err } else if configForClient != nil { c.config = configForClient } @@ -157,24 +176,44 @@ func (c *Conn) readClientHello(ctx context.Context) (*clientHelloMsg, error) { c.ticketKeys = originalConfig.ticketKeys(configForClient) clientVersions := clientHello.supportedVersions - if len(clientHello.supportedVersions) == 0 { + if clientHello.vers >= VersionTLS13 && len(clientVersions) == 0 { + // RFC 8446 4.2.1 indicates when the supported_versions extension is not sent, + // compatible servers MUST negotiate TLS 1.2 or earlier if supported, even + // if the client legacy version is TLS 1.3 or later. + // + // Since we reject empty extensionSupportedVersions in the client hello unmarshal + // finding the supportedVersions empty indicates the extension was not present. + clientVersions = supportedVersionsFromMax(VersionTLS12) + } else if len(clientVersions) == 0 { clientVersions = supportedVersionsFromMax(clientHello.vers) } c.vers, ok = c.config.mutualVersion(roleServer, clientVersions) if !ok { c.sendAlert(alertProtocolVersion) - return nil, fmt.Errorf("tls: client offered only unsupported versions: %x", clientVersions) + return nil, nil, fmt.Errorf("tls: client offered only unsupported versions: %x", clientVersions) } c.haveVers = true c.in.version = c.vers c.out.version = c.vers + // This check reflects some odd specification implied behavior. Client-facing servers + // are supposed to reject hellos with outer ECH and inner ECH that offers 1.2, but + // backend servers are allowed to accept hellos with inner ECH that offer 1.2, since + // they cannot expect client-facing servers to behave properly. Since we act as both + // a client-facing and backend server, we only enforce 1.3 being negotiated if we + // saw a hello with outer ECH first. The spec probably should've made this an error, + // but it didn't, and this matches the boringssl behavior. + if c.vers != VersionTLS13 && (ech != nil && !ech.inner) { + c.sendAlert(alertIllegalParameter) + return nil, nil, errors.New("tls: Encrypted Client Hello cannot be used pre-TLS 1.3") + } + if c.config.MinVersion == 0 && c.vers < VersionTLS12 { tls10server.Value() // ensure godebug is initialized tls10server.IncNonDefault() } - return clientHello, nil + return clientHello, ech, nil } func (hs *serverHandshakeState) processClientHello() error { @@ -193,7 +232,7 @@ func (hs *serverHandshakeState) processClientHello() error { } if !foundCompression { - c.sendAlert(alertHandshakeFailure) + c.sendAlert(alertIllegalParameter) return errors.New("tls: client does not support uncompressed connections") } @@ -248,7 +287,11 @@ func (hs *serverHandshakeState) processClientHello() error { hs.hello.scts = hs.cert.SignedCertificateTimestamps } - hs.ecdheOk = supportsECDHE(c.config, c.vers, hs.clientHello.supportedCurves, hs.clientHello.supportedPoints) + hs.ecdheOk, err = supportsECDHE(c.config, c.vers, hs.clientHello.supportedCurves, hs.clientHello.supportedPoints) + if err != nil { + c.sendAlert(alertMissingExtension) + return err + } if hs.ecdheOk && len(hs.clientHello.supportedPoints) > 0 { // Although omitting the ec_point_formats extension is permitted, some @@ -314,12 +357,12 @@ func negotiateALPN(serverProtos, clientProtos []string, quic bool) (string, erro if http11fallback { return "", nil } - return "", fmt.Errorf("tls: client requested unsupported application protocols (%s)", clientProtos) + return "", fmt.Errorf("tls: client requested unsupported application protocols (%q)", clientProtos) } // supportsECDHE returns whether ECDHE key exchanges can be used with this // pre-TLS 1.3 client. -func supportsECDHE(c *Config, version uint16, supportedCurves []CurveID, supportedPoints []uint8) bool { +func supportsECDHE(c *Config, version uint16, supportedCurves []CurveID, supportedPoints []uint8) (bool, error) { supportsCurve := false for _, curve := range supportedCurves { if c.supportsCurve(version, curve) { @@ -329,10 +372,12 @@ func supportsECDHE(c *Config, version uint16, supportedCurves []CurveID, support } supportsPointFormat := false + offeredNonCompressedFormat := false for _, pointFormat := range supportedPoints { if pointFormat == pointFormatUncompressed { supportsPointFormat = true - break + } else { + offeredNonCompressedFormat = true } } // Per RFC 8422, Section 5.1.2, if the Supported Point Formats extension is @@ -341,42 +386,31 @@ func supportsECDHE(c *Config, version uint16, supportedCurves []CurveID, support // the parser. See https://go.dev/issue/49126. if len(supportedPoints) == 0 { supportsPointFormat = true + } else if offeredNonCompressedFormat && !supportsPointFormat { + return false, errors.New("tls: client offered only incompatible point formats") } - return supportsCurve && supportsPointFormat + return supportsCurve && supportsPointFormat, nil } func (hs *serverHandshakeState) pickCipherSuite() error { c := hs.c - preferenceOrder := cipherSuitesPreferenceOrder - if !hasAESGCMHardwareSupport || !aesgcmPreferred(hs.clientHello.cipherSuites) { - preferenceOrder = cipherSuitesPreferenceOrderNoAES - } - - configCipherSuites := c.config.cipherSuites() - preferenceList := make([]uint16, 0, len(configCipherSuites)) - for _, suiteID := range preferenceOrder { - for _, id := range configCipherSuites { - if id == suiteID { - preferenceList = append(preferenceList, id) - break - } - } - } + preferenceList := c.config.cipherSuites(isAESGCMPreferred(hs.clientHello.cipherSuites)) hs.suite = selectCipherSuite(preferenceList, hs.clientHello.cipherSuites, hs.cipherSuiteOk) if hs.suite == nil { c.sendAlert(alertHandshakeFailure) - return errors.New("tls: no cipher suite supported by both client and server") + return fmt.Errorf("tls: no cipher suite supported by both client and server; client offered: %x", + hs.clientHello.cipherSuites) } c.cipherSuite = hs.suite.id - if c.config.CipherSuites == nil && !needFIPS() && rsaKexCiphers[hs.suite.id] { + if c.config.CipherSuites == nil && !fips140tls.Required() && rsaKexCiphers[hs.suite.id] { tlsrsakex.Value() // ensure godebug is initialized tlsrsakex.IncNonDefault() } - if c.config.CipherSuites == nil && !needFIPS() && tdesCiphers[hs.suite.id] { + if c.config.CipherSuites == nil && !fips140tls.Required() && tdesCiphers[hs.suite.id] { tls3des.Value() // ensure godebug is initialized tls3des.IncNonDefault() } @@ -473,7 +507,7 @@ func (hs *serverHandshakeState) checkForResumption() error { // Check that we also support the ciphersuite from the session. suite := selectCipherSuite([]uint16{sessionState.cipherSuite}, - c.config.cipherSuites(), hs.cipherSuiteOk) + c.config.supportedCipherSuites(), hs.cipherSuiteOk) if suite == nil { return nil } @@ -503,6 +537,10 @@ func (hs *serverHandshakeState) checkForResumption() error { // weird downgrade in client capabilities. return errors.New("tls: session supported extended_master_secret but client does not") } + if !sessionState.extMasterSecret && fips140tls.Required() { + // FIPS 140-3 requires the use of Extended Master Secret. + return nil + } c.peerCertificates = sessionState.peerCertificates c.ocspResponse = sessionState.ocspResponse @@ -511,6 +549,7 @@ func (hs *serverHandshakeState) checkForResumption() error { c.extMasterSecret = sessionState.extMasterSecret hs.sessionState = sessionState hs.suite = suite + c.curveID = sessionState.curveID c.didResume = true return nil } @@ -555,6 +594,10 @@ func (hs *serverHandshakeState) doFullHandshake() error { hs.hello.ocspStapling = true } + if hs.clientHello.serverName != "" { + hs.hello.serverNameAck = true + } + hs.hello.ticketSupported = hs.clientHello.ticketSupported && !c.config.SessionTicketsDisabled hs.hello.cipherSuite = hs.suite.id @@ -592,8 +635,9 @@ func (hs *serverHandshakeState) doFullHandshake() error { return err } if skx != nil { - if len(skx.key) >= 3 && skx.key[0] == 3 /* named curve */ { - c.curveID = CurveID(byteorder.BeUint16(skx.key[1:])) + if keyAgreement, ok := keyAgreement.(*ecdheKeyAgreement); ok { + c.curveID = keyAgreement.curveID + c.peerSigAlg = keyAgreement.signatureAlgorithm } if _, err := hs.c.writeHandshakeRecord(skx, &hs.finishedHash); err != nil { return err @@ -610,7 +654,7 @@ func (hs *serverHandshakeState) doFullHandshake() error { } if c.vers >= VersionTLS12 { certReq.hasSignatureAlgorithm = true - certReq.supportedSignatureAlgorithms = supportedSignatureAlgorithms() + certReq.supportedSignatureAlgorithms = supportedSignatureAlgorithms(c.vers) } // An empty list of certificateAuthorities signals to @@ -681,7 +725,7 @@ func (hs *serverHandshakeState) doFullHandshake() error { preMasterSecret, err := keyAgreement.processClientKeyExchange(c.config, hs.cert, ckx, c.vers) if err != nil { - c.sendAlert(alertHandshakeFailure) + c.sendAlert(alertIllegalParameter) return err } if hs.hello.extendedMasterSecret { @@ -689,6 +733,10 @@ func (hs *serverHandshakeState) doFullHandshake() error { hs.masterSecret = extMasterFromPreMasterSecret(c.vers, hs.suite, preMasterSecret, hs.finishedHash.Sum()) } else { + if fips140tls.Required() { + c.sendAlert(alertHandshakeFailure) + return errors.New("tls: FIPS 140-3 requires the use of Extended Master Secret") + } hs.masterSecret = masterFromPreMasterSecret(c.vers, hs.suite, preMasterSecret, hs.clientHello.random, hs.hello.random) } @@ -728,6 +776,10 @@ func (hs *serverHandshakeState) doFullHandshake() error { if err != nil { return c.sendAlert(alertInternalError) } + if sigHash == crypto.SHA1 { + tlssha1.Value() // ensure godebug is initialized + tlssha1.IncNonDefault() + } } else { sigType, sigHash, err = legacyTypeAndHashFromPublicKey(pub) if err != nil { @@ -741,6 +793,7 @@ func (hs *serverHandshakeState) doFullHandshake() error { c.sendAlert(alertDecryptError) return errors.New("tls: invalid signature by the client certificate: " + err.Error()) } + c.peerSigAlg = certVerify.signatureAlgorithm if err := transcriptMsg(certVerify, &hs.finishedHash); err != nil { return err @@ -870,14 +923,14 @@ func (hs *serverHandshakeState) sendFinished(out []byte) error { } // processCertsFromClient takes a chain of client certificates either from a -// Certificates message and verifies them. +// certificateMsg message or a certificateMsgTLS13 message and verifies them. func (c *Conn) processCertsFromClient(certificate Certificate) error { certificates := certificate.Certificate certs := make([]*x509.Certificate, len(certificates)) var err error for i, asn1Data := range certificates { if certs[i], err = x509.ParseCertificate(asn1Data); err != nil { - c.sendAlert(alertBadCertificate) + c.sendAlert(alertDecodeError) return errors.New("tls: failed to parse client certificate: " + err.Error()) } if certs[i].PublicKeyAlgorithm == x509.RSA { @@ -893,7 +946,7 @@ func (c *Conn) processCertsFromClient(certificate Certificate) error { if c.vers == VersionTLS13 { c.sendAlert(alertCertificateRequired) } else { - c.sendAlert(alertBadCertificate) + c.sendAlert(alertHandshakeFailure) } return errors.New("tls: client didn't provide a certificate") } @@ -912,10 +965,9 @@ func (c *Conn) processCertsFromClient(certificate Certificate) error { chains, err := certs[0].Verify(opts) if err != nil { - var errCertificateInvalid x509.CertificateInvalidError - if errors.As(err, &x509.UnknownAuthorityError{}) { + if _, ok := errors.AsType[x509.UnknownAuthorityError](err); ok { c.sendAlert(alertUnknownCA) - } else if errors.As(err, &errCertificateInvalid) && errCertificateInvalid.Reason == x509.Expired { + } else if errCertificateInvalid, ok := errors.AsType[x509.CertificateInvalidError](err); ok && errCertificateInvalid.Reason == x509.Expired { c.sendAlert(alertCertificateExpired) } else { c.sendAlert(alertBadCertificate) @@ -923,7 +975,11 @@ func (c *Conn) processCertsFromClient(certificate Certificate) error { return &CertificateVerificationError{UnverifiedCertificates: certs, Err: err} } - c.verifiedChains = chains + c.verifiedChains, err = fipsAllowedChains(chains) + if err != nil { + c.sendAlert(alertBadCertificate) + return &CertificateVerificationError{UnverifiedCertificates: certs, Err: err} + } } c.peerCertificates = certs @@ -963,6 +1019,7 @@ func clientHelloInfo(ctx context.Context, c *Conn, clientHello *clientHelloMsg) SignatureSchemes: clientHello.supportedSignatureAlgorithms, SupportedProtos: clientHello.alpnProtocols, SupportedVersions: supportedVersions, + Extensions: clientHello.extensions, Conn: c.conn, config: c.config, ctx: ctx, diff --git a/src/crypto/tls/handshake_server_test.go b/src/crypto/tls/handshake_server_test.go index 94d3d0f6dc87bc..43183db2a19770 100644 --- a/src/crypto/tls/handshake_server_test.go +++ b/src/crypto/tls/handshake_server_test.go @@ -11,6 +11,7 @@ import ( "crypto/ecdh" "crypto/elliptic" "crypto/rand" + "crypto/tls/internal/fips140tls" "crypto/x509" "encoding/pem" "errors" @@ -23,11 +24,13 @@ import ( "runtime" "slices" "strings" + "sync/atomic" "testing" "time" ) func testClientHello(t *testing.T, serverConfig *Config, m handshakeMessage) { + t.Helper() testClientHelloFailure(t, serverConfig, m, "") } @@ -52,12 +55,13 @@ func testClientHelloFailure(t *testing.T, serverConfig *Config, m handshakeMessa }() ctx := context.Background() conn := Server(s, serverConfig) - ch, err := conn.readClientHello(ctx) + ch, ech, err := conn.readClientHello(ctx) if conn.vers == VersionTLS13 { hs := serverHandshakeStateTLS13{ c: conn, ctx: ctx, clientHello: ch, + echContext: ech, } if err == nil { err = hs.processClientHello() @@ -116,7 +120,7 @@ func TestRejectBadProtocolVersion(t *testing.T) { func TestNoSuiteOverlap(t *testing.T) { clientHello := &clientHelloMsg{ - vers: VersionTLS10, + vers: VersionTLS12, random: make([]byte, 32), cipherSuites: []uint16{0xff00}, compressionMethods: []uint8{compressionNone}, @@ -126,9 +130,9 @@ func TestNoSuiteOverlap(t *testing.T) { func TestNoCompressionOverlap(t *testing.T) { clientHello := &clientHelloMsg{ - vers: VersionTLS10, + vers: VersionTLS12, random: make([]byte, 32), - cipherSuites: []uint16{TLS_RSA_WITH_RC4_128_SHA}, + cipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256}, compressionMethods: []uint8{0xff}, } testClientHelloFailure(t, testConfig, clientHello, "client does not support uncompressed connections") @@ -136,7 +140,7 @@ func TestNoCompressionOverlap(t *testing.T) { func TestNoRC4ByDefault(t *testing.T) { clientHello := &clientHelloMsg{ - vers: VersionTLS10, + vers: VersionTLS12, random: make([]byte, 32), cipherSuites: []uint16{TLS_RSA_WITH_RC4_128_SHA}, compressionMethods: []uint8{compressionNone}, @@ -153,16 +157,16 @@ func TestRejectSNIWithTrailingDot(t *testing.T) { vers: VersionTLS12, random: make([]byte, 32), serverName: "foo.com.", - }, "unexpected message") + }, "decoding message") } func TestDontSelectECDSAWithRSAKey(t *testing.T) { // Test that, even when both sides support an ECDSA cipher suite, it // won't be selected if the server's private key doesn't support it. clientHello := &clientHelloMsg{ - vers: VersionTLS10, + vers: VersionTLS12, random: make([]byte, 32), - cipherSuites: []uint16{TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA}, + cipherSuites: []uint16{TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384}, compressionMethods: []uint8{compressionNone}, supportedCurves: []CurveID{CurveP256}, supportedPoints: []uint8{pointFormatUncompressed}, @@ -186,9 +190,9 @@ func TestDontSelectRSAWithECDSAKey(t *testing.T) { // Test that, even when both sides support an RSA cipher suite, it // won't be selected if the server's private key doesn't support it. clientHello := &clientHelloMsg{ - vers: VersionTLS10, + vers: VersionTLS12, random: make([]byte, 32), - cipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA}, + cipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256}, compressionMethods: []uint8{compressionNone}, supportedCurves: []CurveID{CurveP256}, supportedPoints: []uint8{pointFormatUncompressed}, @@ -208,6 +212,8 @@ func TestDontSelectRSAWithECDSAKey(t *testing.T) { } func TestRenegotiationExtension(t *testing.T) { + skipFIPS(t) // #70505 + clientHello := &clientHelloMsg{ vers: VersionTLS12, compressionMethods: []uint8{compressionNone}, @@ -230,7 +236,6 @@ func TestRenegotiationExtension(t *testing.T) { n, err := c.Read(buf) if err != nil { t.Errorf("Server read returned error: %s", err) - return } c.Close() bufChan <- buf[:n] @@ -260,6 +265,8 @@ func TestRenegotiationExtension(t *testing.T) { } func TestTLS12OnlyCipherSuites(t *testing.T) { + skipFIPS(t) // No TLS 1.1 in FIPS mode. + // Test that a Server doesn't select a TLS 1.2-only cipher suite when // the client negotiates TLS 1.1. clientHello := &clientHelloMsg{ @@ -321,13 +328,18 @@ func TestTLSPointFormats(t *testing.T) { supportedPoints []uint8 wantSupportedPoints bool }{ - {"ECC", []uint16{TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA}, []CurveID{CurveP256}, []uint8{pointFormatUncompressed}, true}, - {"ECC without ec_point_format", []uint16{TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA}, []CurveID{CurveP256}, nil, false}, - {"ECC with extra values", []uint16{TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA}, []CurveID{CurveP256}, []uint8{13, 37, pointFormatUncompressed, 42}, true}, + {"ECC", []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256}, []CurveID{CurveP256}, []uint8{pointFormatUncompressed}, true}, + {"ECC without ec_point_format", []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256}, []CurveID{CurveP256}, nil, false}, + {"ECC with extra values", []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256}, []CurveID{CurveP256}, []uint8{13, 37, pointFormatUncompressed, 42}, true}, {"RSA", []uint16{TLS_RSA_WITH_AES_256_GCM_SHA384}, nil, nil, false}, {"RSA with ec_point_format", []uint16{TLS_RSA_WITH_AES_256_GCM_SHA384}, nil, []uint8{pointFormatUncompressed}, false}, } for _, tt := range tests { + // The RSA subtests should be enabled for FIPS 140 required mode: #70505 + if strings.HasPrefix(tt.name, "RSA") && fips140tls.Required() { + t.Logf("skipping in FIPS mode.") + continue + } t.Run(tt.name, func(t *testing.T) { clientHello := &clientHelloMsg{ vers: VersionTLS12, @@ -341,7 +353,9 @@ func TestTLSPointFormats(t *testing.T) { c, s := localPipe(t) replyChan := make(chan any) go func() { - cli := Client(c, testConfig) + clientConfig := testConfig.Clone() + clientConfig.Certificates = []Certificate{{Certificate: [][]byte{testRSA2048Certificate}, PrivateKey: testRSA2048PrivateKey}} + cli := Client(c, clientConfig) cli.vers = clientHello.vers if _, err := cli.writeHandshakeRecord(clientHello, nil); err != nil { testFatal(t, err) @@ -354,9 +368,10 @@ func TestTLSPointFormats(t *testing.T) { replyChan <- reply } }() - config := testConfig.Clone() - config.CipherSuites = clientHello.cipherSuites - Server(s, config).Handshake() + serverConfig := testConfig.Clone() + serverConfig.Certificates = []Certificate{{Certificate: [][]byte{testRSA2048Certificate}, PrivateKey: testRSA2048PrivateKey}} + serverConfig.CipherSuites = clientHello.cipherSuites + Server(s, serverConfig).Handshake() s.Close() reply := <-replyChan if err, ok := reply.(error); ok { @@ -388,8 +403,7 @@ func TestAlertForwarding(t *testing.T) { err := Server(s, testConfig).Handshake() s.Close() - var opErr *net.OpError - if !errors.As(err, &opErr) || opErr.Err != error(alertUnknownCA) { + if opErr, ok := errors.AsType[*net.OpError](err); !ok || opErr.Err != error(alertUnknownCA) { t.Errorf("Got error: %s; expected: %s", err, error(alertUnknownCA)) } } @@ -431,6 +445,8 @@ func TestVersion(t *testing.T) { } func TestCipherSuitePreference(t *testing.T) { + skipFIPS(t) // No RC4 or CHACHA20_POLY1305 in FIPS mode. + serverConfig := &Config{ CipherSuites: []uint16{TLS_RSA_WITH_RC4_128_SHA, TLS_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256}, @@ -499,15 +515,17 @@ func TestCrossVersionResume(t *testing.T) { func testCrossVersionResume(t *testing.T, version uint16) { serverConfig := &Config{ - CipherSuites: []uint16{TLS_RSA_WITH_AES_128_CBC_SHA}, + CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256}, Certificates: testConfig.Certificates, + Time: testTime, } clientConfig := &Config{ - CipherSuites: []uint16{TLS_RSA_WITH_AES_128_CBC_SHA}, + CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256}, InsecureSkipVerify: true, ClientSessionCache: NewLRUClientSessionCache(1), ServerName: "servername", MinVersion: VersionTLS12, + Time: testTime, } // Establish a session at TLS 1.3. @@ -716,7 +734,7 @@ func (test *serverTest) run(t *testing.T, write bool) { t.Fatalf("%s: mismatch on peer list length: %d (wanted) != %d (got)", test.name, len(test.expectedPeerCerts), len(peerCerts)) } - if test.validate != nil { + if test.validate != nil && !t.Failed() { if err := test.validate(connState); err != nil { t.Fatalf("validate callback returned error: %s", err) } @@ -921,22 +939,6 @@ func TestHandshakeServerKeySharePreference(t *testing.T) { runServerTestTLS13(t, test) } -// TestHandshakeServerUnsupportedKeyShare tests a client that sends a key share -// that's not in the supported groups list. -func TestHandshakeServerUnsupportedKeyShare(t *testing.T) { - pk, _ := ecdh.X25519().GenerateKey(rand.Reader) - clientHello := &clientHelloMsg{ - vers: VersionTLS12, - random: make([]byte, 32), - supportedVersions: []uint16{VersionTLS13}, - cipherSuites: []uint16{TLS_CHACHA20_POLY1305_SHA256}, - compressionMethods: []uint8{compressionNone}, - keyShares: []keyShare{{group: X25519, data: pk.PublicKey().Bytes()}}, - supportedCurves: []CurveID{CurveP256}, - } - testClientHelloFailure(t, testConfig, clientHello, "client sent key share for group it does not support") -} - func TestHandshakeServerALPN(t *testing.T) { config := testConfig.Clone() config.NextProtos = []string{"proto1", "proto2"} @@ -1066,6 +1068,65 @@ func TestHandshakeServerSNIGetCertificateNotFound(t *testing.T) { runServerTestTLS12(t, test) } +// TestHandshakeServerGetCertificateExtensions tests to make sure that the +// Extensions passed to GetCertificate match what we expect based on the +// clientHelloMsg +func TestHandshakeServerGetCertificateExtensions(t *testing.T) { + const errMsg = "TestHandshakeServerGetCertificateExtensions error" + // ensure the test condition inside our GetCertificate callback + // is actually invoked + var called atomic.Int32 + + testVersions := []uint16{VersionTLS12, VersionTLS13} + for _, vers := range testVersions { + t.Run(fmt.Sprintf("TLS version %04x", vers), func(t *testing.T) { + pk, _ := ecdh.P256().GenerateKey(rand.Reader) + clientHello := &clientHelloMsg{ + vers: vers, + random: make([]byte, 32), + cipherSuites: []uint16{TLS_AES_128_GCM_SHA256}, + compressionMethods: []uint8{compressionNone}, + serverName: "test", + keyShares: []keyShare{{group: CurveP256, data: pk.PublicKey().Bytes()}}, + supportedCurves: []CurveID{CurveP256}, + supportedSignatureAlgorithms: []SignatureScheme{ECDSAWithP256AndSHA256}, + } + + // the clientHelloMsg initialized just above is serialized with + // two extensions: server_name(0) and application_layer_protocol_negotiation(16) + expectedExtensions := []uint16{ + extensionServerName, + extensionSupportedCurves, + extensionSignatureAlgorithms, + extensionKeyShare, + } + + if vers == VersionTLS13 { + clientHello.supportedVersions = []uint16{VersionTLS13} + expectedExtensions = append(expectedExtensions, extensionSupportedVersions) + } + + // Go's TLS client presents extensions in the ClientHello sorted by extension ID + slices.Sort(expectedExtensions) + + serverConfig := testConfig.Clone() + serverConfig.GetCertificate = func(clientHello *ClientHelloInfo) (*Certificate, error) { + if !slices.Equal(expectedExtensions, clientHello.Extensions) { + t.Errorf("expected extensions on ClientHelloInfo (%v) to match clientHelloMsg (%v)", expectedExtensions, clientHello.Extensions) + } + called.Add(1) + + return nil, errors.New(errMsg) + } + testClientHelloFailure(t, serverConfig, clientHello, errMsg) + }) + } + + if int(called.Load()) != len(testVersions) { + t.Error("expected our GetCertificate test to be called twice") + } +} + // TestHandshakeServerSNIGetCertificateError tests to make sure that errors in // GetCertificate result in a tls alert. func TestHandshakeServerSNIGetCertificateError(t *testing.T) { @@ -1077,9 +1138,9 @@ func TestHandshakeServerSNIGetCertificateError(t *testing.T) { } clientHello := &clientHelloMsg{ - vers: VersionTLS10, + vers: VersionTLS12, random: make([]byte, 32), - cipherSuites: []uint16{TLS_RSA_WITH_RC4_128_SHA}, + cipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256}, compressionMethods: []uint8{compressionNone}, serverName: "test", } @@ -1098,9 +1159,9 @@ func TestHandshakeServerEmptyCertificates(t *testing.T) { serverConfig.Certificates = nil clientHello := &clientHelloMsg{ - vers: VersionTLS10, + vers: VersionTLS12, random: make([]byte, 32), - cipherSuites: []uint16{TLS_RSA_WITH_RC4_128_SHA}, + cipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256}, compressionMethods: []uint8{compressionNone}, } testClientHelloFailure(t, serverConfig, clientHello, errMsg) @@ -1110,9 +1171,9 @@ func TestHandshakeServerEmptyCertificates(t *testing.T) { serverConfig.GetCertificate = nil clientHello = &clientHelloMsg{ - vers: VersionTLS10, + vers: VersionTLS12, random: make([]byte, 32), - cipherSuites: []uint16{TLS_RSA_WITH_RC4_128_SHA}, + cipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256}, compressionMethods: []uint8{compressionNone}, } testClientHelloFailure(t, serverConfig, clientHello, "no certificates") @@ -1317,31 +1378,31 @@ func BenchmarkHandshakeServer(b *testing.B) { }) b.Run("ECDHE-P256-RSA", func(b *testing.B) { b.Run("TLSv13", func(b *testing.B) { - benchmarkHandshakeServer(b, VersionTLS13, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + benchmarkHandshakeServer(b, VersionTLS13, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, CurveP256, testRSACertificate, testRSAPrivateKey) }) b.Run("TLSv12", func(b *testing.B) { - benchmarkHandshakeServer(b, VersionTLS12, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + benchmarkHandshakeServer(b, VersionTLS12, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, CurveP256, testRSACertificate, testRSAPrivateKey) }) }) b.Run("ECDHE-P256-ECDSA-P256", func(b *testing.B) { b.Run("TLSv13", func(b *testing.B) { - benchmarkHandshakeServer(b, VersionTLS13, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + benchmarkHandshakeServer(b, VersionTLS13, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, CurveP256, testP256Certificate, testP256PrivateKey) }) b.Run("TLSv12", func(b *testing.B) { - benchmarkHandshakeServer(b, VersionTLS12, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + benchmarkHandshakeServer(b, VersionTLS12, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, CurveP256, testP256Certificate, testP256PrivateKey) }) }) b.Run("ECDHE-X25519-ECDSA-P256", func(b *testing.B) { b.Run("TLSv13", func(b *testing.B) { - benchmarkHandshakeServer(b, VersionTLS13, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + benchmarkHandshakeServer(b, VersionTLS13, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, X25519, testP256Certificate, testP256PrivateKey) }) b.Run("TLSv12", func(b *testing.B) { - benchmarkHandshakeServer(b, VersionTLS12, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + benchmarkHandshakeServer(b, VersionTLS12, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, X25519, testP256Certificate, testP256PrivateKey) }) }) @@ -1350,11 +1411,11 @@ func BenchmarkHandshakeServer(b *testing.B) { b.Fatal("test ECDSA key doesn't use curve P-521") } b.Run("TLSv13", func(b *testing.B) { - benchmarkHandshakeServer(b, VersionTLS13, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + benchmarkHandshakeServer(b, VersionTLS13, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, CurveP521, testECDSACertificate, testECDSAPrivateKey) }) b.Run("TLSv12", func(b *testing.B) { - benchmarkHandshakeServer(b, VersionTLS12, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + benchmarkHandshakeServer(b, VersionTLS12, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, CurveP521, testECDSACertificate, testECDSAPrivateKey) }) }) @@ -1435,9 +1496,9 @@ func TestSNIGivenOnFailure(t *testing.T) { const expectedServerName = "test.testing" clientHello := &clientHelloMsg{ - vers: VersionTLS10, + vers: VersionTLS12, random: make([]byte, 32), - cipherSuites: []uint16{TLS_RSA_WITH_RC4_128_SHA}, + cipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256}, compressionMethods: []uint8{compressionNone}, serverName: expectedServerName, } @@ -1457,7 +1518,7 @@ func TestSNIGivenOnFailure(t *testing.T) { }() conn := Server(s, serverConfig) ctx := context.Background() - ch, err := conn.readClientHello(ctx) + ch, _, err := conn.readClientHello(ctx) hs := serverHandshakeState{ c: conn, ctx: ctx, @@ -1528,9 +1589,7 @@ var getConfigForClientTests = []struct { }, func(clientHello *ClientHelloInfo) (*Config, error) { config := testConfig.Clone() - for i := range config.SessionTicketKey { - config.SessionTicketKey[i] = 0 - } + clear(config.SessionTicketKey[:]) config.sessionTicketKeys = nil return config, nil }, @@ -1693,7 +1752,7 @@ T+E0J8wlH24pgwQHzy7Ko2qLwn1b5PW8ecrlvP1g func TestMultipleCertificates(t *testing.T) { clientConfig := testConfig.Clone() - clientConfig.CipherSuites = []uint16{TLS_RSA_WITH_AES_128_GCM_SHA256} + clientConfig.CipherSuites = []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256} clientConfig.MaxVersion = VersionTLS12 serverConfig := testConfig.Clone() @@ -1715,6 +1774,8 @@ func TestMultipleCertificates(t *testing.T) { } func TestAESCipherReordering(t *testing.T) { + skipFIPS(t) // No CHACHA20_POLY1305 for FIPS. + currentAESSupport := hasAESGCMHardwareSupport defer func() { hasAESGCMHardwareSupport = currentAESSupport }() @@ -1728,28 +1789,28 @@ func TestAESCipherReordering(t *testing.T) { { name: "server has hardware AES, client doesn't (pick ChaCha)", clientCiphers: []uint16{ - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA, }, serverHasAESGCM: true, - expectedCipher: TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + expectedCipher: TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, }, { name: "client prefers AES-GCM, server doesn't have hardware AES (pick ChaCha)", clientCiphers: []uint16{ TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA, }, serverHasAESGCM: false, - expectedCipher: TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + expectedCipher: TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, }, { name: "client prefers AES-GCM, server has hardware AES (pick AES-GCM)", clientCiphers: []uint16{ TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA, }, serverHasAESGCM: true, @@ -1760,7 +1821,7 @@ func TestAESCipherReordering(t *testing.T) { clientCiphers: []uint16{ 0x0A0A, // GREASE value TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA, }, serverHasAESGCM: true, @@ -1781,27 +1842,27 @@ func TestAESCipherReordering(t *testing.T) { clientCiphers: []uint16{ TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA, - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, }, serverHasAESGCM: false, - expectedCipher: TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + expectedCipher: TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, }, { name: "client prefers AES-GCM over ChaCha and sends GREASE, server doesn't have hardware AES (pick ChaCha)", clientCiphers: []uint16{ 0x0A0A, // GREASE value TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA, }, serverHasAESGCM: false, - expectedCipher: TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + expectedCipher: TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, }, { name: "client supports multiple AES-GCM, server doesn't have hardware AES and doesn't support ChaCha (AES-GCM)", clientCiphers: []uint16{ TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, }, serverHasAESGCM: false, @@ -1815,14 +1876,14 @@ func TestAESCipherReordering(t *testing.T) { name: "client prefers AES-GCM, server has hardware but doesn't support AES (pick ChaCha)", clientCiphers: []uint16{ TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA, }, serverHasAESGCM: true, serverCiphers: []uint16{ - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, }, - expectedCipher: TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + expectedCipher: TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, }, } @@ -1858,6 +1919,8 @@ func TestAESCipherReordering(t *testing.T) { } func TestAESCipherReorderingTLS13(t *testing.T) { + skipFIPS(t) // No CHACHA20_POLY1305 for FIPS. + currentAESSupport := hasAESGCMHardwareSupport defer func() { hasAESGCMHardwareSupport = currentAESSupport }() @@ -1978,8 +2041,8 @@ func TestServerHandshakeContextCancellation(t *testing.T) { if err != context.Canceled { t.Errorf("Unexpected server handshake error: %v", err) } - if runtime.GOARCH == "wasm" { - t.Skip("conn.Close does not error as expected when called multiple times on WASM") + if runtime.GOOS == "js" || runtime.GOOS == "wasip1" { + t.Skip("conn.Close does not error as expected when called multiple times on GOOS=js or GOOS=wasip1") } err = conn.Close() if err == nil { diff --git a/src/crypto/tls/handshake_server_tls13.go b/src/crypto/tls/handshake_server_tls13.go index 503a732e05765e..c5b0552cae6134 100644 --- a/src/crypto/tls/handshake_server_tls13.go +++ b/src/crypto/tls/handshake_server_tls13.go @@ -8,14 +8,20 @@ import ( "bytes" "context" "crypto" + "crypto/hkdf" "crypto/hmac" - "crypto/internal/mlkem768" + "crypto/internal/fips140/mlkem" + "crypto/internal/fips140/tls13" + "crypto/internal/hpke" "crypto/rsa" + "crypto/tls/internal/fips140tls" "errors" + "fmt" "hash" "internal/byteorder" "io" "slices" + "sort" "time" ) @@ -24,6 +30,18 @@ import ( // messages cause too much work in session ticket decryption attempts. const maxClientPSKIdentities = 5 +type echServerContext struct { + hpkeContext *hpke.Recipient + configID uint8 + ciphersuite echCipher + transcript hash.Hash + // inner indicates that the initial client_hello we received contained an + // encrypted_client_hello extension that indicated it was an "inner" hello. + // We don't do any additional processing of the hello in this case, so all + // fields above are unset. + inner bool +} + type serverHandshakeStateTLS13 struct { c *Conn ctx context.Context @@ -35,22 +53,19 @@ type serverHandshakeStateTLS13 struct { suite *cipherSuiteTLS13 cert *Certificate sigAlg SignatureScheme - earlySecret []byte + earlySecret *tls13.EarlySecret sharedKey []byte - handshakeSecret []byte - masterSecret []byte + handshakeSecret *tls13.HandshakeSecret + masterSecret *tls13.MasterSecret trafficSecret []byte // client_application_traffic_secret_0 transcript hash.Hash clientFinished []byte + echContext *echServerContext } func (hs *serverHandshakeStateTLS13) handshake() error { c := hs.c - if needFIPS() { - return errors.New("tls: internal error: TLS 1.3 reached in FIPS mode") - } - // For an overview of the TLS 1.3 handshake, see RFC 8446, Section 2. if err := hs.processClientHello(); err != nil { return err @@ -162,9 +177,12 @@ func (hs *serverHandshakeStateTLS13) processClientHello() error { hs.hello.compressionMethod = compressionNone preferenceList := defaultCipherSuitesTLS13 - if !hasAESGCMHardwareSupport || !aesgcmPreferred(hs.clientHello.cipherSuites) { + if !hasAESGCMHardwareSupport || !isAESGCMPreferred(hs.clientHello.cipherSuites) { preferenceList = defaultCipherSuitesTLS13NoAES } + if fips140tls.Required() { + preferenceList = allowedCipherSuitesTLS13FIPS + } for _, suiteID := range preferenceList { hs.suite = mutualCipherSuiteTLS13(hs.clientHello.cipherSuites, suiteID) if hs.suite != nil { @@ -173,42 +191,51 @@ func (hs *serverHandshakeStateTLS13) processClientHello() error { } if hs.suite == nil { c.sendAlert(alertHandshakeFailure) - return errors.New("tls: no cipher suite supported by both client and server") + return fmt.Errorf("tls: no cipher suite supported by both client and server; client offered: %x", + hs.clientHello.cipherSuites) } c.cipherSuite = hs.suite.id hs.hello.cipherSuite = hs.suite.id hs.transcript = hs.suite.hash.New() - // Pick the key exchange method in server preference order, but give - // priority to key shares, to avoid a HelloRetryRequest round-trip. - var selectedGroup CurveID - var clientKeyShare *keyShare + // First, if a post-quantum key exchange is available, use one. See + // draft-ietf-tls-key-share-prediction-01, Section 4 for why this must be + // first. + // + // Second, if the client sent a key share for a group we support, use that, + // to avoid a HelloRetryRequest round-trip. + // + // Finally, pick in our fixed preference order. preferredGroups := c.config.curvePreferences(c.vers) - for _, preferredGroup := range preferredGroups { - ki := slices.IndexFunc(hs.clientHello.keyShares, func(ks keyShare) bool { - return ks.group == preferredGroup - }) - if ki != -1 { - clientKeyShare = &hs.clientHello.keyShares[ki] - selectedGroup = clientKeyShare.group - if !slices.Contains(hs.clientHello.supportedCurves, selectedGroup) { - c.sendAlert(alertIllegalParameter) - return errors.New("tls: client sent key share for group it does not support") - } - break - } + preferredGroups = slices.DeleteFunc(preferredGroups, func(group CurveID) bool { + return !slices.Contains(hs.clientHello.supportedCurves, group) + }) + if len(preferredGroups) == 0 { + c.sendAlert(alertHandshakeFailure) + return errors.New("tls: no key exchanges supported by both client and server") } - if selectedGroup == 0 { - for _, preferredGroup := range preferredGroups { - if slices.Contains(hs.clientHello.supportedCurves, preferredGroup) { - selectedGroup = preferredGroup - break + hasKeyShare := func(group CurveID) bool { + for _, ks := range hs.clientHello.keyShares { + if ks.group == group { + return true } } + return false } - if selectedGroup == 0 { - c.sendAlert(alertHandshakeFailure) - return errors.New("tls: no ECDHE curve supported by both client and server") + sort.SliceStable(preferredGroups, func(i, j int) bool { + return hasKeyShare(preferredGroups[i]) && !hasKeyShare(preferredGroups[j]) + }) + sort.SliceStable(preferredGroups, func(i, j int) bool { + return isPQKeyExchange(preferredGroups[i]) && !isPQKeyExchange(preferredGroups[j]) + }) + selectedGroup := preferredGroups[0] + + var clientKeyShare *keyShare + for _, ks := range hs.clientHello.keyShares { + if ks.group == selectedGroup { + clientKeyShare = &ks + break + } } if clientKeyShare == nil { ks, err := hs.doHelloRetryRequest(selectedGroup) @@ -221,13 +248,13 @@ func (hs *serverHandshakeStateTLS13) processClientHello() error { ecdhGroup := selectedGroup ecdhData := clientKeyShare.data - if selectedGroup == x25519Kyber768Draft00 { + if selectedGroup == X25519MLKEM768 { ecdhGroup = X25519 - if len(ecdhData) != x25519PublicKeySize+mlkem768.EncapsulationKeySize { + if len(ecdhData) != mlkem.EncapsulationKeySize768+x25519PublicKeySize { c.sendAlert(alertIllegalParameter) - return errors.New("tls: invalid Kyber client key share") + return errors.New("tls: invalid X25519MLKEM768 client key share") } - ecdhData = ecdhData[:x25519PublicKeySize] + ecdhData = ecdhData[mlkem.EncapsulationKeySize768:] } if _, ok := curveForCurveID(ecdhGroup); !ok { c.sendAlert(alertInternalError) @@ -249,14 +276,24 @@ func (hs *serverHandshakeStateTLS13) processClientHello() error { c.sendAlert(alertIllegalParameter) return errors.New("tls: invalid client key share") } - if selectedGroup == x25519Kyber768Draft00 { - ciphertext, kyberShared, err := kyberEncapsulate(clientKeyShare.data[x25519PublicKeySize:]) + if selectedGroup == X25519MLKEM768 { + k, err := mlkem.NewEncapsulationKey768(clientKeyShare.data[:mlkem.EncapsulationKeySize768]) if err != nil { c.sendAlert(alertIllegalParameter) - return errors.New("tls: invalid Kyber client key share") + return errors.New("tls: invalid X25519MLKEM768 client key share") } - hs.sharedKey = append(hs.sharedKey, kyberShared...) - hs.hello.serverShare.data = append(hs.hello.serverShare.data, ciphertext...) + mlkemSharedSecret, ciphertext := k.Encapsulate() + // draft-kwiatkowski-tls-ecdhe-mlkem-02, Section 3.1.3: "For + // X25519MLKEM768, the shared secret is the concatenation of the ML-KEM + // shared secret and the X25519 shared secret. The shared secret is 64 + // bytes (32 bytes for each part)." + hs.sharedKey = append(mlkemSharedSecret, hs.sharedKey...) + // draft-kwiatkowski-tls-ecdhe-mlkem-02, Section 3.1.2: "When the + // X25519MLKEM768 group is negotiated, the server's key exchange value + // is the concatenation of an ML-KEM ciphertext returned from + // encapsulation to the client's encapsulation key, and the server's + // ephemeral X25519 share." + hs.hello.serverShare.data = append(ciphertext, hs.hello.serverShare.data...) } selectedProto, err := negotiateALPN(c.config.NextProtos, hs.clientHello.alpnProtocols, c.quic != nil) @@ -383,8 +420,8 @@ func (hs *serverHandshakeStateTLS13) checkForResumption() error { } } - hs.earlySecret = hs.suite.extract(sessionState.secret, nil) - binderKey := hs.suite.deriveSecret(hs.earlySecret, resumptionBinderLabel, nil) + hs.earlySecret = tls13.NewEarlySecret(hs.suite.hash.New, sessionState.secret) + binderKey := hs.earlySecret.ResumptionBinderKey() // Clone the transcript in case a HelloRetryRequest was recorded. transcript := cloneHash(hs.transcript, hs.suite.hash) if transcript == nil { @@ -412,7 +449,7 @@ func (hs *serverHandshakeStateTLS13) checkForResumption() error { if err := transcriptMsg(hs.clientHello, transcript); err != nil { return err } - earlyTrafficSecret := hs.suite.deriveSecret(hs.earlySecret, clientEarlyTrafficLabel, transcript) + earlyTrafficSecret := hs.earlySecret.ClientEarlyTrafficSecret(transcript) c.quicSetReadSecret(QUICEncryptionLevelEarly, hs.suite.id, earlyTrafficSecret) } @@ -431,10 +468,17 @@ func (hs *serverHandshakeStateTLS13) checkForResumption() error { return nil } -// cloneHash uses the encoding.BinaryMarshaler and encoding.BinaryUnmarshaler +// cloneHash uses [hash.Cloner] to clone in. If [hash.Cloner] +// is not implemented or not supported, then it falls back to the +// [encoding.BinaryMarshaler] and [encoding.BinaryUnmarshaler] // interfaces implemented by standard library hashes to clone the state of in // to a new instance of h. It returns nil if the operation fails. func cloneHash(in hash.Hash, h crypto.Hash) hash.Hash { + if cloner, ok := in.(hash.Cloner); ok { + if out, err := cloner.Clone(); err == nil { + return out + } + } // Recreate the interface to avoid importing encoding. type binaryMarshaler interface { MarshalBinary() (data []byte, err error) @@ -530,6 +574,23 @@ func (hs *serverHandshakeStateTLS13) doHelloRetryRequest(selectedGroup CurveID) selectedGroup: selectedGroup, } + if hs.echContext != nil { + // Compute the acceptance message. + helloRetryRequest.encryptedClientHello = make([]byte, 8) + confTranscript := cloneHash(hs.transcript, hs.suite.hash) + if err := transcriptMsg(helloRetryRequest, confTranscript); err != nil { + return nil, err + } + h := hs.suite.hash.New + prf, err := hkdf.Extract(h, hs.clientHello.random, nil) + if err != nil { + c.sendAlert(alertInternalError) + return nil, err + } + acceptConfirmation := tls13.ExpandLabel(h, prf, "hrr ech accept confirmation", confTranscript.Sum(nil), 8) + helloRetryRequest.encryptedClientHello = acceptConfirmation + } + if _, err := hs.c.writeHandshakeRecord(helloRetryRequest, hs.transcript); err != nil { return nil, err } @@ -550,6 +611,45 @@ func (hs *serverHandshakeStateTLS13) doHelloRetryRequest(selectedGroup CurveID) return nil, unexpectedMessageError(clientHello, msg) } + if hs.echContext != nil { + if len(clientHello.encryptedClientHello) == 0 { + c.sendAlert(alertMissingExtension) + return nil, errors.New("tls: second client hello missing encrypted client hello extension") + } + + echType, echCiphersuite, configID, encap, payload, err := parseECHExt(clientHello.encryptedClientHello) + if err != nil { + c.sendAlert(alertDecodeError) + return nil, errors.New("tls: client sent invalid encrypted client hello extension") + } + + if echType == outerECHExt && hs.echContext.inner || echType == innerECHExt && !hs.echContext.inner { + c.sendAlert(alertDecodeError) + return nil, errors.New("tls: unexpected switch in encrypted client hello extension type") + } + + if echType == outerECHExt { + if echCiphersuite != hs.echContext.ciphersuite || configID != hs.echContext.configID || len(encap) != 0 { + c.sendAlert(alertIllegalParameter) + return nil, errors.New("tls: second client hello encrypted client hello extension does not match") + } + + encodedInner, err := decryptECHPayload(hs.echContext.hpkeContext, clientHello.original, payload) + if err != nil { + c.sendAlert(alertDecryptError) + return nil, errors.New("tls: failed to decrypt second client hello encrypted client hello extension payload") + } + + echInner, err := decodeInnerClientHello(clientHello, encodedInner) + if err != nil { + c.sendAlert(alertIllegalParameter) + return nil, errors.New("tls: client sent invalid encrypted client hello extension") + } + + clientHello = echInner + } + } + if len(clientHello.keyShares) != 1 { c.sendAlert(alertIllegalParameter) return nil, errors.New("tls: client didn't send one key share in second ClientHello") @@ -637,9 +737,28 @@ func illegalClientHelloChange(ch, ch1 *clientHelloMsg) bool { func (hs *serverHandshakeStateTLS13) sendServerParameters() error { c := hs.c + if hs.echContext != nil { + copy(hs.hello.random[32-8:], make([]byte, 8)) + echTranscript := cloneHash(hs.transcript, hs.suite.hash) + echTranscript.Write(hs.clientHello.original) + if err := transcriptMsg(hs.hello, echTranscript); err != nil { + return err + } + // compute the acceptance message + h := hs.suite.hash.New + prk, err := hkdf.Extract(h, hs.clientHello.random, nil) + if err != nil { + c.sendAlert(alertInternalError) + return err + } + acceptConfirmation := tls13.ExpandLabel(h, prk, "ech accept confirmation", echTranscript.Sum(nil), 8) + copy(hs.hello.random[32-8:], acceptConfirmation) + } + if err := transcriptMsg(hs.clientHello, hs.transcript); err != nil { return err } + if _, err := hs.c.writeHandshakeRecord(hs.hello, hs.transcript); err != nil { return err } @@ -650,16 +769,13 @@ func (hs *serverHandshakeStateTLS13) sendServerParameters() error { earlySecret := hs.earlySecret if earlySecret == nil { - earlySecret = hs.suite.extract(nil, nil) + earlySecret = tls13.NewEarlySecret(hs.suite.hash.New, nil) } - hs.handshakeSecret = hs.suite.extract(hs.sharedKey, - hs.suite.deriveSecret(earlySecret, "derived", nil)) + hs.handshakeSecret = earlySecret.HandshakeSecret(hs.sharedKey) - clientSecret := hs.suite.deriveSecret(hs.handshakeSecret, - clientHandshakeTrafficLabel, hs.transcript) + clientSecret := hs.handshakeSecret.ClientHandshakeTrafficSecret(hs.transcript) c.in.setTrafficSecret(hs.suite, QUICEncryptionLevelHandshake, clientSecret) - serverSecret := hs.suite.deriveSecret(hs.handshakeSecret, - serverHandshakeTrafficLabel, hs.transcript) + serverSecret := hs.handshakeSecret.ServerHandshakeTrafficSecret(hs.transcript) c.out.setTrafficSecret(hs.suite, QUICEncryptionLevelHandshake, serverSecret) if c.quic != nil { @@ -693,6 +809,28 @@ func (hs *serverHandshakeStateTLS13) sendServerParameters() error { encryptedExtensions.earlyData = hs.earlyData } + if !hs.c.didResume && hs.clientHello.serverName != "" { + encryptedExtensions.serverNameAck = true + } + + // If client sent ECH extension, but we didn't accept it, + // send retry configs, if available. + echKeys := hs.c.config.EncryptedClientHelloKeys + if hs.c.config.GetEncryptedClientHelloKeys != nil { + echKeys, err = hs.c.config.GetEncryptedClientHelloKeys(clientHelloInfo(hs.ctx, c, hs.clientHello)) + if err != nil { + c.sendAlert(alertInternalError) + return err + } + } + if len(echKeys) > 0 && len(hs.clientHello.encryptedClientHello) > 0 && hs.echContext == nil { + encryptedExtensions.echRetryConfigs, err = buildRetryConfigList(echKeys) + if err != nil { + c.sendAlert(alertInternalError) + return err + } + } + if _, err := hs.c.writeHandshakeRecord(encryptedExtensions, hs.transcript); err != nil { return err } @@ -717,7 +855,8 @@ func (hs *serverHandshakeStateTLS13) sendServerCertificate() error { certReq := new(certificateRequestMsgTLS13) certReq.ocspStapling = true certReq.scts = true - certReq.supportedSignatureAlgorithms = supportedSignatureAlgorithms() + certReq.supportedSignatureAlgorithms = supportedSignatureAlgorithms(c.vers) + certReq.supportedSignatureAlgorithmsCert = supportedSignatureAlgorithmsCert() if c.config.ClientCAs != nil { certReq.certificateAuthorities = c.config.ClientCAs.Subjects() } @@ -784,13 +923,10 @@ func (hs *serverHandshakeStateTLS13) sendServerFinished() error { // Derive secrets that take context through the server Finished. - hs.masterSecret = hs.suite.extract(nil, - hs.suite.deriveSecret(hs.handshakeSecret, "derived", nil)) + hs.masterSecret = hs.handshakeSecret.MasterSecret() - hs.trafficSecret = hs.suite.deriveSecret(hs.masterSecret, - clientApplicationTrafficLabel, hs.transcript) - serverSecret := hs.suite.deriveSecret(hs.masterSecret, - serverApplicationTrafficLabel, hs.transcript) + hs.trafficSecret = hs.masterSecret.ClientApplicationTrafficSecret(hs.transcript) + serverSecret := hs.masterSecret.ServerApplicationTrafficSecret(hs.transcript) c.out.setTrafficSecret(hs.suite, QUICEncryptionLevelApplication, serverSecret) if c.quic != nil { @@ -837,12 +973,7 @@ func (hs *serverHandshakeStateTLS13) shouldSendSessionTickets() bool { } // Don't send tickets the client wouldn't use. See RFC 8446, Section 4.2.9. - for _, pskMode := range hs.clientHello.pskModes { - if pskMode == pskModeDHE { - return true - } - } - return false + return slices.Contains(hs.clientHello.pskModes, pskModeDHE) } func (hs *serverHandshakeStateTLS13) sendSessionTickets() error { @@ -856,8 +987,7 @@ func (hs *serverHandshakeStateTLS13) sendSessionTickets() error { return err } - c.resumptionSecret = hs.suite.deriveSecret(hs.masterSecret, - resumptionLabel, hs.transcript) + c.resumptionSecret = hs.masterSecret.ResumptionMasterSecret(hs.transcript) if !hs.shouldSendSessionTickets() { return nil @@ -872,7 +1002,7 @@ func (c *Conn) sendSessionTicket(earlyData bool, extra [][]byte) error { } // ticket_nonce, which must be unique per connection, is always left at // zero because we only ever send one ticket per connection. - psk := suite.expandLabel(c.resumptionSecret, "resumption", + psk := tls13.ExpandLabel(suite.hash.New, c.resumptionSecret, "resumption", nil, suite.hash.Size()) m := new(newSessionTicketMsgTLS13) @@ -907,7 +1037,7 @@ func (c *Conn) sendSessionTicket(earlyData bool, extra [][]byte) error { if _, err := c.config.rand().Read(ageAdd); err != nil { return err } - m.ageAdd = byteorder.LeUint32(ageAdd) + m.ageAdd = byteorder.LEUint32(ageAdd) if earlyData { // RFC 9001, Section 4.6.1 @@ -977,7 +1107,10 @@ func (hs *serverHandshakeStateTLS13) readClientCertificate() error { } // See RFC 8446, Section 4.4.3. - if !isSupportedSignatureAlgorithm(certVerify.signatureAlgorithm, supportedSignatureAlgorithms()) { + // We don't use certReq.supportedSignatureAlgorithms because it would + // require keeping the certificateRequestMsgTLS13 around in the hs. + if !isSupportedSignatureAlgorithm(certVerify.signatureAlgorithm, supportedSignatureAlgorithms(c.vers)) || + !isSupportedSignatureAlgorithm(certVerify.signatureAlgorithm, signatureSchemesForPublicKey(c.vers, c.peerCertificates[0].PublicKey)) { c.sendAlert(alertIllegalParameter) return errors.New("tls: client certificate used with invalid signature algorithm") } @@ -986,8 +1119,7 @@ func (hs *serverHandshakeStateTLS13) readClientCertificate() error { return c.sendAlert(alertInternalError) } if sigType == signaturePKCS1v15 || sigHash == crypto.SHA1 { - c.sendAlert(alertIllegalParameter) - return errors.New("tls: client certificate used with invalid signature algorithm") + return c.sendAlert(alertInternalError) } signed := signedMessage(sigHash, clientSignatureContext, hs.transcript) if err := verifyHandshakeSignature(sigType, c.peerCertificates[0].PublicKey, @@ -995,6 +1127,7 @@ func (hs *serverHandshakeStateTLS13) readClientCertificate() error { c.sendAlert(alertDecryptError) return errors.New("tls: invalid signature by the client certificate: " + err.Error()) } + c.peerSigAlg = certVerify.signatureAlgorithm if err := transcriptMsg(certVerify, hs.transcript); err != nil { return err diff --git a/src/crypto/tls/handshake_test.go b/src/crypto/tls/handshake_test.go index bc3d23d5adc24e..3e2c5663087828 100644 --- a/src/crypto/tls/handshake_test.go +++ b/src/crypto/tls/handshake_test.go @@ -46,10 +46,15 @@ var ( keyFile = flag.String("keylog", "", "destination file for KeyLogWriter") bogoMode = flag.Bool("bogo-mode", false, "Enabled bogo shim mode, ignore everything else") bogoFilter = flag.String("bogo-filter", "", "BoGo test filter") - bogoLocalDir = flag.String("bogo-local-dir", "", "Local BoGo to use, instead of fetching from source") + bogoLocalDir = flag.String("bogo-local-dir", "", + "If not-present, checkout BoGo into this dir, or otherwise use it as a pre-existing checkout") + bogoReport = flag.String("bogo-html-report", "", "File path to render an HTML report with BoGo results") ) func runTestAndUpdateIfNeeded(t *testing.T, name string, run func(t *testing.T, update bool), wait bool) { + // FIPS mode is non-deterministic and so isn't suited for testing against static test transcripts. + skipFIPS(t) + success := t.Run(name, func(t *testing.T) { if !*update && !wait { t.Parallel() @@ -491,9 +496,10 @@ func testHandshake(t *testing.T, clientConfig, serverConfig *Config) (serverStat if got := string(buf); got != sentinel { t.Errorf("read %q from TLS connection, but expected %q", got, sentinel) } - if err := cli.Close(); err != nil { - t.Errorf("failed to call cli.Close: %v", err) - } + // We discard the error because after ReadAll returns the server must + // have already closed the connection. Sending data (the closeNotify + // alert) can cause a reset, that will make Close return an error. + cli.Close() }() server := Server(s, serverConfig) err = server.Handshake() @@ -518,10 +524,21 @@ func fromHex(s string) []byte { return b } +// testTime is 2016-10-20T17:32:09.000Z, which is within the validity period of +// [testRSACertificate], [testRSACertificateIssuer], [testRSA2048Certificate], +// [testRSA2048CertificateIssuer], and [testECDSACertificate]. +var testTime = func() time.Time { return time.Unix(1476984729, 0) } + var testRSACertificate = fromHex("3082024b308201b4a003020102020900e8f09d3fe25beaa6300d06092a864886f70d01010b0500301f310b3009060355040a1302476f3110300e06035504031307476f20526f6f74301e170d3136303130313030303030305a170d3235303130313030303030305a301a310b3009060355040a1302476f310b300906035504031302476f30819f300d06092a864886f70d010101050003818d0030818902818100db467d932e12270648bc062821ab7ec4b6a25dfe1e5245887a3647a5080d92425bc281c0be97799840fb4f6d14fd2b138bc2a52e67d8d4099ed62238b74a0b74732bc234f1d193e596d9747bf3589f6c613cc0b041d4d92b2b2423775b1c3bbd755dce2054cfa163871d1e24c4f31d1a508baab61443ed97a77562f414c852d70203010001a38193308190300e0603551d0f0101ff0404030205a0301d0603551d250416301406082b0601050507030106082b06010505070302300c0603551d130101ff0402300030190603551d0e041204109f91161f43433e49a6de6db680d79f60301b0603551d230414301280104813494d137e1631bba301d5acab6e7b30190603551d1104123010820e6578616d706c652e676f6c616e67300d06092a864886f70d01010b0500038181009d30cc402b5b50a061cbbae55358e1ed8328a9581aa938a495a1ac315a1a84663d43d32dd90bf297dfd320643892243a00bccf9c7db74020015faad3166109a276fd13c3cce10c5ceeb18782f16c04ed73bbb343778d0c1cf10fa1d8408361c94c722b9daedb4606064df4c1b33ec0d1bd42d4dbfe3d1360845c21d33be9fae7") var testRSACertificateIssuer = fromHex("3082021930820182a003020102020900ca5e4e811a965964300d06092a864886f70d01010b0500301f310b3009060355040a1302476f3110300e06035504031307476f20526f6f74301e170d3136303130313030303030305a170d3235303130313030303030305a301f310b3009060355040a1302476f3110300e06035504031307476f20526f6f7430819f300d06092a864886f70d010101050003818d0030818902818100d667b378bb22f34143b6cd2008236abefaf2852adf3ab05e01329e2c14834f5105df3f3073f99dab5442d45ee5f8f57b0111c8cb682fbb719a86944eebfffef3406206d898b8c1b1887797c9c5006547bb8f00e694b7a063f10839f269f2c34fff7a1f4b21fbcd6bfdfb13ac792d1d11f277b5c5b48600992203059f2a8f8cc50203010001a35d305b300e0603551d0f0101ff040403020204301d0603551d250416301406082b0601050507030106082b06010505070302300f0603551d130101ff040530030101ff30190603551d0e041204104813494d137e1631bba301d5acab6e7b300d06092a864886f70d01010b050003818100c1154b4bab5266221f293766ae4138899bd4c5e36b13cee670ceeaa4cbdf4f6679017e2fe649765af545749fe4249418a56bd38a04b81e261f5ce86b8d5c65413156a50d12449554748c59a30c515bc36a59d38bddf51173e899820b282e40aa78c806526fd184fb6b4cf186ec728edffa585440d2b3225325f7ab580e87dd76") +var testRSA2048Certificate = fromHex("30820316308201fea003020102020900e8f09d3fe25beaa6300d06092a864886f70d01010b0500301f310b3009060355040a1302476f3110300e06035504031307476f20526f6f74301e170d3136303130313030303030305a170d3338303130313030303030305a301a310b3009060355040a1302476f310b300906035504031302476f30820122300d06092a864886f70d01010105000382010f003082010a0282010100e0ac47db9ba1b7f98a996c62dc1d248d4ee570544136fe4e911e22fccc0fe2b20982f3c4cdd8f4065c5068c873ca0a768b80dc915edc66541a5f26cdea44e56e411221e2f9927bf4e009fee76dbe0e118dcc13392efd6f42d8eb2fd5bc8f63ac77800c84d3be90c20c321273254b9137ef61f825dad1ec2c5e75aa4be6d3104899bd5ac400da7ab942b4227a3870ae5bb97870aa09a1082fb8e78b944cd7fd1b0c6fb1cce03b5430b12ef9ce2d95e01821766e998df0cc99202a57cf030577bd2dc0ec85a49f203511bb6f0e9f43398ead0958f8d7534c61e81daf4501faaa68d9cbc725b58401900fa48a3e2333b15c88cf0c5cc8f33fb9464f9d5f5768b8f10203010001a35a3058300e0603551d0f0101ff0404030205a0301d0603551d250416301406082b0601050507030106082b06010505070302300c0603551d130101ff0402300030190603551d1104123010820e6578616d706c652e676f6c616e67300d06092a864886f70d01010b050003820101009e83f835e2da08204ee6f8bdca793cf83c7aec175349c1642dfbe9f4d0dcfb1aedb4d0122e16c2ad92e63dd31cce10ca5dd04be48cded0fdc8fea49e891d9d93e778a67d54b619ac167ce7bb0f6000ca00c5677d09df3eb10080134ba32bfe4132d33954dc479cb266288d53d3f43af9c78c0ca59d396498bdc56d4966dc6b7e49081f7f2ae1d704bb9f9effed93c57d3b738da02edff3999e3f1a5dce2b093951947d233d9c6b6a12b4b1611826aa02544980089eebbcf22a1a96bd35a3ddf638578989334a93d5081fab442b4383ba6213b7cdd74110582244a2abd937828b311d8dd69178756db7874293b9810c5c2e833f91d49d283a62caaf359141997f") + +var testRSA2048CertificateIssuer = fromHex("308203223082020aa003020102020900ca5e4e811a965964300d06092a864886f70d01010b0500301f310b3009060355040a1302476f3110300e06035504031307476f20526f6f74301e170d3136303130313030303030305a170d3235303130313030303030305a301f310b3009060355040a1302476f3110300e06035504031307476f20526f6f7430820122300d06092a864886f70d01010105000382010f003082010a0282010100b308c1720c7054abe66e1be6f8a11246808215a810e8936e47601f7ec1afeb02ad69a5000959d4e08ebc4455ef90b39616f380b8ff2e76f29942d7e009cf010824fe56f69140ac39b761595255ec2aa35155ca2eea884f57b25f8a52f41f56f65b0197cb6c637f9adfa97d8ac27565449f64e67f8b918646ffd630601b0badd8d38aea421fe413ee94f10ea5874c2fd6d8c1b9febaa5ca0ce759993a232c9c48e52230bbf58777b0c30e07e9e0914133730d844b9887b950d5a17c779ac69de2d9c65d26f1ea46c7dd7ac636af6d77df7c9218f78c7b5f08b025867f343ac66cd43a657ac44bfd7e9d07e95a22ff9a0babf72dcffc66eba0a1d90731f67e3bbd0203010001a361305f300e0603551d0f0101ff040403020204301d0603551d250416301406082b0601050507030106082b06010505070302300f0603551d130101ff040530030101ff301d0603551d0e0416041460145a6ce2e8a15b1b68db9a4752ce8684d6ba2d300d06092a864886f70d01010b050003820101001d342fe0b50a25d57a8b13bc14d0abb1eea7431ee752aa423e1306654183e44e9d48bbf592cd32ce77310fdc4e8bbcd724fc43d2723f454bfe605ff90d38d8c6fe60b36c6f4d2d7e4e79bceeb2484f0565274b0d0c4a8562370677624a4c133e332a9e63d4b47544c14e4908ee8685dd0760ae6f4ab089ede2b0cdc595ecefbee7d8be80d57b2d4e4510b6ceda54d1a5980540214191d81cc89a983da43d4043f8efe97a2e231c5153bded520acce87ec8c64a3408f0eb4c742c4a877e8b5b7b7f72497734a41a95994a7a103262ea6d598d03fd5cb0579ed4702424da8893334c58215bc655d49656aedcd02d18676f45d6b9469ae04b89abe9b358391cce99") + +var testRSA2048PrivateKey, _ = x509.ParsePKCS1PrivateKey(fromHex("308204a40201000282010100e0ac47db9ba1b7f98a996c62dc1d248d4ee570544136fe4e911e22fccc0fe2b20982f3c4cdd8f4065c5068c873ca0a768b80dc915edc66541a5f26cdea44e56e411221e2f9927bf4e009fee76dbe0e118dcc13392efd6f42d8eb2fd5bc8f63ac77800c84d3be90c20c321273254b9137ef61f825dad1ec2c5e75aa4be6d3104899bd5ac400da7ab942b4227a3870ae5bb97870aa09a1082fb8e78b944cd7fd1b0c6fb1cce03b5430b12ef9ce2d95e01821766e998df0cc99202a57cf030577bd2dc0ec85a49f203511bb6f0e9f43398ead0958f8d7534c61e81daf4501faaa68d9cbc725b58401900fa48a3e2333b15c88cf0c5cc8f33fb9464f9d5f5768b8f10203010001028201007aac96efca229b199e1bf79a63256677e1c455792bc2a348b2e409a68ea57dda486740430d4290bb885c3f5a741eb567d4f41f7b2098a726f4df4f88cf899edc7c9b31f584dffedece15a7212642c7dbbdd8d806392a183e1fc30af36169c9bab9e528f0bdcd27ad4c8b6a97849da6452c6809de61848db80c3ba3289e785042cdfd46fbfee5f78adcba2927fcd8cbe9dcaa97190457eaa45d77adbe0db820aff0c8511d837ab5b307bad5f85afd2cc70d9659ec58045d97ced1eb7950670ac559449c0305fddefda1bac88d36629a177f65abad182c6470830b39e7f6dbdef4df813ccaef01d5a42d37213b2b9647e2ff56a63e6b6a4b6e8a1567bbfd77042102818100eb66f205e8507c78f7167dbef3ddf02fde6a67bd15152609e9296576e28c79678177145ae98e0a2fee58fdb3d626fb6beae3e0ae0b76bc47d16fcdeb16f0caca8a0902779979382609705ae84514de480c2fb2ddda3049347cc1bde9f1a359747079ef3dce020a3c186c90e63bc20b5489a40d768b1c1c35c679edc5662e18c702818100f454ffff95b126b55cb13b68a3841600fc0bc69ff4064f7ceb122495fa972fdb05ca2fa1c6e2e84432f81c96875ab12226e8ce92ba808c4f6325f27ce058791f05db96e623687d3cfc198e748a07521a8c7ee9e7e8faf95b0985be82b867a49f7d5d50fac3881d2c39dedfdbca3ebe847b859c9864cf7a543e4688f5a60118870281806cee737ac65950704daeebbb8c701c709a54d4f28baa00b33f6137a1bf0e5033d4963d2620c3e8f4eb2fe51eee2f95d3079c31e1784e96ac093fdaa33a376d3032961ebd27990fa192669abab715041385082196461c6813d0d37ac5a25afbcf452937cb7ae438c63c6b28d651bae6b1550c446aa1cefd42e9388d0df6cdc80b02818100cac172c33504923bb494fad8e5c0a9c5dd63244bfe63f238969632b82700a95cd71c2694d887d9f92656d0da75ae640a1441e392cda3f94bb3da7cb4f6335527d2639c809467946e34423cfe26c0d6786398ba20922d1b1a59f79bd5bc937d8040b75c890c13fb298548977a3c05ff71cf535c54f66b5a77684a7e4363a3cb2702818100a4d782f35d5a07f9c1f8f9c378564b220387d1e481cc856b631de7637d8bb77c851db070122050ac230dc6e45edf4523471c717c1cb86a36b2fd3358fae349d51be54d71d7dbeaa6af668323e2b51933f0b8488aa12723e0f32207068b4aa64ed54bcef4acbbbe35b92802faba7ed45ae52bef8313d9ef4393ccc5cf868ddbf8")) + // testRSAPSSCertificate has signatureAlgorithm rsassaPss, but subjectPublicKeyInfo // algorithm rsaEncryption, for use with the rsa_pss_rsae_* SignatureSchemes. // See also TestRSAPSSKeyError. testRSAPSSCertificate is self-signed. diff --git a/src/crypto/tls/internal/fips140tls/fipstls.go b/src/crypto/tls/internal/fips140tls/fipstls.go new file mode 100644 index 00000000000000..0b87185683ab8b --- /dev/null +++ b/src/crypto/tls/internal/fips140tls/fipstls.go @@ -0,0 +1,37 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package fips140tls controls whether crypto/tls requires FIPS-approved settings. +package fips140tls + +import ( + "crypto/fips140" + "sync/atomic" +) + +var required atomic.Bool + +func init() { + if fips140.Enabled() { + Force() + } +} + +// Force forces crypto/tls to restrict TLS configurations to FIPS-approved settings. +// By design, this call is impossible to undo (except in tests). +func Force() { + required.Store(true) +} + +// Required reports whether FIPS-approved settings are required. +// +// Required is true if FIPS 140-3 mode is enabled with GODEBUG=fips140=on, or if +// the crypto/tls/fipsonly package is imported by a Go+BoringCrypto build. +func Required() bool { + return required.Load() +} + +func TestingOnlyAbandon() { + required.Store(false) +} diff --git a/src/crypto/tls/key_agreement.go b/src/crypto/tls/key_agreement.go index 3e96242b9798b8..88116f941e0b9a 100644 --- a/src/crypto/tls/key_agreement.go +++ b/src/crypto/tls/key_agreement.go @@ -14,6 +14,7 @@ import ( "errors" "fmt" "io" + "slices" ) // A keyAgreement implements the client and server side of a TLS 1.0–1.2 key @@ -164,25 +165,29 @@ type ecdheKeyAgreement struct { // and returned in generateClientKeyExchange. ckx *clientKeyExchangeMsg preMasterSecret []byte + + // curveID and signatureAlgorithm are set by processServerKeyExchange and + // generateServerKeyExchange. + curveID CurveID + signatureAlgorithm SignatureScheme } func (ka *ecdheKeyAgreement) generateServerKeyExchange(config *Config, cert *Certificate, clientHello *clientHelloMsg, hello *serverHelloMsg) (*serverKeyExchangeMsg, error) { - var curveID CurveID for _, c := range clientHello.supportedCurves { if config.supportsCurve(ka.version, c) { - curveID = c + ka.curveID = c break } } - if curveID == 0 { + if ka.curveID == 0 { return nil, errors.New("tls: no supported elliptic curves offered") } - if _, ok := curveForCurveID(curveID); !ok { + if _, ok := curveForCurveID(ka.curveID); !ok { return nil, errors.New("tls: CurvePreferences includes unsupported curve") } - key, err := generateECDHEKey(config.rand(), curveID) + key, err := generateECDHEKey(config.rand(), ka.curveID) if err != nil { return nil, err } @@ -192,8 +197,8 @@ func (ka *ecdheKeyAgreement) generateServerKeyExchange(config *Config, cert *Cer ecdhePublic := key.PublicKey().Bytes() serverECDHEParams := make([]byte, 1+2+1+len(ecdhePublic)) serverECDHEParams[0] = 3 // named curve - serverECDHEParams[1] = byte(curveID >> 8) - serverECDHEParams[2] = byte(curveID) + serverECDHEParams[1] = byte(ka.curveID >> 8) + serverECDHEParams[2] = byte(ka.curveID) serverECDHEParams[3] = byte(len(ecdhePublic)) copy(serverECDHEParams[4:], ecdhePublic) @@ -202,18 +207,21 @@ func (ka *ecdheKeyAgreement) generateServerKeyExchange(config *Config, cert *Cer return nil, fmt.Errorf("tls: certificate private key of type %T does not implement crypto.Signer", cert.PrivateKey) } - var signatureAlgorithm SignatureScheme var sigType uint8 var sigHash crypto.Hash if ka.version >= VersionTLS12 { - signatureAlgorithm, err = selectSignatureScheme(ka.version, cert, clientHello.supportedSignatureAlgorithms) + ka.signatureAlgorithm, err = selectSignatureScheme(ka.version, cert, clientHello.supportedSignatureAlgorithms) if err != nil { return nil, err } - sigType, sigHash, err = typeAndHashFromSignatureScheme(signatureAlgorithm) + sigType, sigHash, err = typeAndHashFromSignatureScheme(ka.signatureAlgorithm) if err != nil { return nil, err } + if sigHash == crypto.SHA1 { + tlssha1.Value() // ensure godebug is initialized + tlssha1.IncNonDefault() + } } else { sigType, sigHash, err = legacyTypeAndHashFromPublicKey(priv.Public()) if err != nil { @@ -244,8 +252,8 @@ func (ka *ecdheKeyAgreement) generateServerKeyExchange(config *Config, cert *Cer copy(skx.key, serverECDHEParams) k := skx.key[len(serverECDHEParams):] if ka.version >= VersionTLS12 { - k[0] = byte(signatureAlgorithm >> 8) - k[1] = byte(signatureAlgorithm) + k[0] = byte(ka.signatureAlgorithm >> 8) + k[1] = byte(ka.signatureAlgorithm) k = k[2:] } k[0] = byte(len(sig) >> 8) @@ -279,7 +287,7 @@ func (ka *ecdheKeyAgreement) processServerKeyExchange(config *Config, clientHell if skx.key[0] != 3 { // named curve return errors.New("tls: server selected unsupported curve") } - curveID := CurveID(skx.key[1])<<8 | CurveID(skx.key[2]) + ka.curveID = CurveID(skx.key[1])<<8 | CurveID(skx.key[2]) publicLen := int(skx.key[3]) if publicLen+4 > len(skx.key) { @@ -293,11 +301,15 @@ func (ka *ecdheKeyAgreement) processServerKeyExchange(config *Config, clientHell return errServerKeyExchange } - if _, ok := curveForCurveID(curveID); !ok { + if !slices.Contains(clientHello.supportedCurves, ka.curveID) { + return errors.New("tls: server selected unoffered curve") + } + + if _, ok := curveForCurveID(ka.curveID); !ok { return errors.New("tls: server selected unsupported curve") } - key, err := generateECDHEKey(config.rand(), curveID) + key, err := generateECDHEKey(config.rand(), ka.curveID) if err != nil { return err } @@ -321,19 +333,23 @@ func (ka *ecdheKeyAgreement) processServerKeyExchange(config *Config, clientHell var sigType uint8 var sigHash crypto.Hash if ka.version >= VersionTLS12 { - signatureAlgorithm := SignatureScheme(sig[0])<<8 | SignatureScheme(sig[1]) + ka.signatureAlgorithm = SignatureScheme(sig[0])<<8 | SignatureScheme(sig[1]) sig = sig[2:] if len(sig) < 2 { return errServerKeyExchange } - if !isSupportedSignatureAlgorithm(signatureAlgorithm, clientHello.supportedSignatureAlgorithms) { + if !isSupportedSignatureAlgorithm(ka.signatureAlgorithm, clientHello.supportedSignatureAlgorithms) { return errors.New("tls: certificate used with invalid signature algorithm") } - sigType, sigHash, err = typeAndHashFromSignatureScheme(signatureAlgorithm) + sigType, sigHash, err = typeAndHashFromSignatureScheme(ka.signatureAlgorithm) if err != nil { return err } + if sigHash == crypto.SHA1 { + tlssha1.Value() // ensure godebug is initialized + tlssha1.IncNonDefault() + } } else { sigType, sigHash, err = legacyTypeAndHashFromPublicKey(cert.PublicKey) if err != nil { diff --git a/src/crypto/tls/key_schedule.go b/src/crypto/tls/key_schedule.go index 1636baf79e7288..1426a276bf2de0 100644 --- a/src/crypto/tls/key_schedule.go +++ b/src/crypto/tls/key_schedule.go @@ -7,93 +7,26 @@ package tls import ( "crypto/ecdh" "crypto/hmac" - "crypto/internal/mlkem768" + "crypto/internal/fips140/mlkem" + "crypto/internal/fips140/tls13" "errors" - "fmt" "hash" "io" - - "golang.org/x/crypto/cryptobyte" - "golang.org/x/crypto/hkdf" - "golang.org/x/crypto/sha3" ) // This file contains the functions necessary to compute the TLS 1.3 key // schedule. See RFC 8446, Section 7. -const ( - resumptionBinderLabel = "res binder" - clientEarlyTrafficLabel = "c e traffic" - clientHandshakeTrafficLabel = "c hs traffic" - serverHandshakeTrafficLabel = "s hs traffic" - clientApplicationTrafficLabel = "c ap traffic" - serverApplicationTrafficLabel = "s ap traffic" - exporterLabel = "exp master" - resumptionLabel = "res master" - trafficUpdateLabel = "traffic upd" -) - -// expandLabel implements HKDF-Expand-Label from RFC 8446, Section 7.1. -func (c *cipherSuiteTLS13) expandLabel(secret []byte, label string, context []byte, length int) []byte { - var hkdfLabel cryptobyte.Builder - hkdfLabel.AddUint16(uint16(length)) - hkdfLabel.AddUint8LengthPrefixed(func(b *cryptobyte.Builder) { - b.AddBytes([]byte("tls13 ")) - b.AddBytes([]byte(label)) - }) - hkdfLabel.AddUint8LengthPrefixed(func(b *cryptobyte.Builder) { - b.AddBytes(context) - }) - hkdfLabelBytes, err := hkdfLabel.Bytes() - if err != nil { - // Rather than calling BytesOrPanic, we explicitly handle this error, in - // order to provide a reasonable error message. It should be basically - // impossible for this to panic, and routing errors back through the - // tree rooted in this function is quite painful. The labels are fixed - // size, and the context is either a fixed-length computed hash, or - // parsed from a field which has the same length limitation. As such, an - // error here is likely to only be caused during development. - // - // NOTE: another reasonable approach here might be to return a - // randomized slice if we encounter an error, which would break the - // connection, but avoid panicking. This would perhaps be safer but - // significantly more confusing to users. - panic(fmt.Errorf("failed to construct HKDF label: %s", err)) - } - out := make([]byte, length) - n, err := hkdf.Expand(c.hash.New, secret, hkdfLabelBytes).Read(out) - if err != nil || n != length { - panic("tls: HKDF-Expand-Label invocation failed unexpectedly") - } - return out -} - -// deriveSecret implements Derive-Secret from RFC 8446, Section 7.1. -func (c *cipherSuiteTLS13) deriveSecret(secret []byte, label string, transcript hash.Hash) []byte { - if transcript == nil { - transcript = c.hash.New() - } - return c.expandLabel(secret, label, transcript.Sum(nil), c.hash.Size()) -} - -// extract implements HKDF-Extract with the cipher suite hash. -func (c *cipherSuiteTLS13) extract(newSecret, currentSecret []byte) []byte { - if newSecret == nil { - newSecret = make([]byte, c.hash.Size()) - } - return hkdf.Extract(c.hash.New, newSecret, currentSecret) -} - // nextTrafficSecret generates the next traffic secret, given the current one, // according to RFC 8446, Section 7.2. func (c *cipherSuiteTLS13) nextTrafficSecret(trafficSecret []byte) []byte { - return c.expandLabel(trafficSecret, trafficUpdateLabel, nil, c.hash.Size()) + return tls13.ExpandLabel(c.hash.New, trafficSecret, "traffic upd", nil, c.hash.Size()) } // trafficKey generates traffic keys according to RFC 8446, Section 7.3. func (c *cipherSuiteTLS13) trafficKey(trafficSecret []byte) (key, iv []byte) { - key = c.expandLabel(trafficSecret, "key", nil, c.keyLen) - iv = c.expandLabel(trafficSecret, "iv", nil, aeadNonceLength) + key = tls13.ExpandLabel(c.hash.New, trafficSecret, "key", nil, c.keyLen) + iv = tls13.ExpandLabel(c.hash.New, trafficSecret, "iv", nil, aeadNonceLength) return } @@ -101,7 +34,7 @@ func (c *cipherSuiteTLS13) trafficKey(trafficSecret []byte) (key, iv []byte) { // to RFC 8446, Section 4.4.4. See sections 4.4 and 4.2.11.2 for the baseKey // selection. func (c *cipherSuiteTLS13) finishedHash(baseKey []byte, transcript hash.Hash) []byte { - finishedKey := c.expandLabel(baseKey, "finished", nil, c.hash.Size()) + finishedKey := tls13.ExpandLabel(c.hash.New, baseKey, "finished", nil, c.hash.Size()) verifyData := hmac.New(c.hash.New, finishedKey) verifyData.Write(transcript.Sum(nil)) return verifyData.Sum(nil) @@ -109,51 +42,17 @@ func (c *cipherSuiteTLS13) finishedHash(baseKey []byte, transcript hash.Hash) [] // exportKeyingMaterial implements RFC5705 exporters for TLS 1.3 according to // RFC 8446, Section 7.5. -func (c *cipherSuiteTLS13) exportKeyingMaterial(masterSecret []byte, transcript hash.Hash) func(string, []byte, int) ([]byte, error) { - expMasterSecret := c.deriveSecret(masterSecret, exporterLabel, transcript) +func (c *cipherSuiteTLS13) exportKeyingMaterial(s *tls13.MasterSecret, transcript hash.Hash) func(string, []byte, int) ([]byte, error) { + expMasterSecret := s.ExporterMasterSecret(transcript) return func(label string, context []byte, length int) ([]byte, error) { - secret := c.deriveSecret(expMasterSecret, label, nil) - h := c.hash.New() - h.Write(context) - return c.expandLabel(secret, "exporter", h.Sum(nil), length), nil + return expMasterSecret.Exporter(label, context, length), nil } } type keySharePrivateKeys struct { curveID CurveID ecdhe *ecdh.PrivateKey - kyber *mlkem768.DecapsulationKey -} - -// kyberDecapsulate implements decapsulation according to Kyber Round 3. -func kyberDecapsulate(dk *mlkem768.DecapsulationKey, c []byte) ([]byte, error) { - K, err := mlkem768.Decapsulate(dk, c) - if err != nil { - return nil, err - } - return kyberSharedSecret(K, c), nil -} - -// kyberEncapsulate implements encapsulation according to Kyber Round 3. -func kyberEncapsulate(ek []byte) (c, ss []byte, err error) { - c, ss, err = mlkem768.Encapsulate(ek) - if err != nil { - return nil, nil, err - } - return c, kyberSharedSecret(ss, c), nil -} - -func kyberSharedSecret(K, c []byte) []byte { - // Package mlkem768 implements ML-KEM, which compared to Kyber removed a - // final hashing step. Compute SHAKE-256(K || SHA3-256(c), 32) to match Kyber. - // See https://words.filippo.io/mlkem768/#bonus-track-using-a-ml-kem-implementation-as-kyber-v3. - h := sha3.NewShake256() - h.Write(K) - ch := sha3.Sum256(c) - h.Write(ch[:]) - out := make([]byte, 32) - h.Read(out) - return out + mlkem *mlkem.DecapsulationKey768 } const x25519PublicKeySize = 32 @@ -183,18 +82,3 @@ func curveForCurveID(id CurveID) (ecdh.Curve, bool) { return nil, false } } - -func curveIDForCurve(curve ecdh.Curve) (CurveID, bool) { - switch curve { - case ecdh.X25519(): - return X25519, true - case ecdh.P256(): - return CurveP256, true - case ecdh.P384(): - return CurveP384, true - case ecdh.P521(): - return CurveP521, true - default: - return 0, false - } -} diff --git a/src/crypto/tls/key_schedule_test.go b/src/crypto/tls/key_schedule_test.go index 3ffdf6c109ef84..1710994d91c656 100644 --- a/src/crypto/tls/key_schedule_test.go +++ b/src/crypto/tls/key_schedule_test.go @@ -6,14 +6,80 @@ package tls import ( "bytes" - "crypto/internal/mlkem768" + "crypto/internal/fips140/tls13" + "crypto/sha256" "encoding/hex" - "hash" "strings" "testing" "unicode" ) +func TestACVPVectors(t *testing.T) { + // https://github.com/usnistgov/ACVP-Server/blob/3a7333f63/gen-val/json-files/TLS-v1.3-KDF-RFC8446/prompt.json#L428-L436 + psk := fromHex("56288B726C73829F7A3E47B103837C8139ACF552E7530C7A710B35ED41191698") + dhe := fromHex("EFFE9EC26AA29FD750DFA6A10B944D74071595B27EE88887D5E11C84590B5CC3") + helloClientRandom := fromHex("E9137679E582BA7C1DB41CF725F86C6D09C8C05F297BAD9A65B552EAF524FDE4") + helloServerRandom := fromHex("23ECCFD030790748C8F8D8A656FD98D717F1B62AF3712F97211D2070B499F98A") + finishedClientRandom := fromHex("62A62FA75563ED4FDCAA0BC16567B314871C304ACF06B0FFC3F08C1797594D43") + finishedServerRandom := fromHex("C750EDA6696CD101B142BD79E00E6AC8C5F2C0ABC78DD64F4D991326659E9299") + + // https://github.com/usnistgov/ACVP-Server/blob/3a7333f63/gen-val/json-files/TLS-v1.3-KDF-RFC8446/expectedResults.json#L571-L581 + clientEarlyTrafficSecret := fromHex("3272189698C3594D18F58EFA3F12B638A249515099BE7A2FA9836BABE74F0111") + earlyExporterMasterSecret := fromHex("88E078F562CDC930219F6A5E98A1CE8C6E5F3DAC5AC516459A96F2EF8F114C66") + clientHandshakeTrafficSecret := fromHex("B32306C3CE9932C460A1FE6C0F060593974842036B96FA45049B7352E71C2AD2") + serverHandshakeTrafficSecret := fromHex("22787F8CA269D34BC549AC8BA19F2040938A3AA370D7CC9D60F720882B88D01B") + clientApplicationTrafficSecret := fromHex("47D7EA08397B5871154B0FE85584BCC30A87C69E84D69B56007C5B21F76493BA") + serverApplicationTrafficSecret := fromHex("EFBDB0C873C0480DA57307083839A8984BE25B9A8545E4FCA029940FE2800565") + exporterMasterSecret := fromHex("8A43D787EE3804EAD4A2A5B32972F9896B696295645D7222E1FD081DDD939834") + resumptionMasterSecret := fromHex("5F4C961329C91044011ACBECB0B289282E0E3FED045CB3EA924DFFE5FE654B3D") + + // The "Random" values are undocumented, but they are meant to be written to + // the hash in sequence to develop the transcript. + transcript := sha256.New() + + es := tls13.NewEarlySecret(sha256.New, psk) + + transcript.Write(helloClientRandom) + + if got := es.ClientEarlyTrafficSecret(transcript); !bytes.Equal(got, clientEarlyTrafficSecret) { + t.Errorf("clientEarlyTrafficSecret = %x, want %x", got, clientEarlyTrafficSecret) + } + if got := tls13.TestingOnlyExporterSecret(es.EarlyExporterMasterSecret(transcript)); !bytes.Equal(got, earlyExporterMasterSecret) { + t.Errorf("earlyExporterMasterSecret = %x, want %x", got, earlyExporterMasterSecret) + } + + hs := es.HandshakeSecret(dhe) + + transcript.Write(helloServerRandom) + + if got := hs.ClientHandshakeTrafficSecret(transcript); !bytes.Equal(got, clientHandshakeTrafficSecret) { + t.Errorf("clientHandshakeTrafficSecret = %x, want %x", got, clientHandshakeTrafficSecret) + } + if got := hs.ServerHandshakeTrafficSecret(transcript); !bytes.Equal(got, serverHandshakeTrafficSecret) { + t.Errorf("serverHandshakeTrafficSecret = %x, want %x", got, serverHandshakeTrafficSecret) + } + + ms := hs.MasterSecret() + + transcript.Write(finishedServerRandom) + + if got := ms.ClientApplicationTrafficSecret(transcript); !bytes.Equal(got, clientApplicationTrafficSecret) { + t.Errorf("clientApplicationTrafficSecret = %x, want %x", got, clientApplicationTrafficSecret) + } + if got := ms.ServerApplicationTrafficSecret(transcript); !bytes.Equal(got, serverApplicationTrafficSecret) { + t.Errorf("serverApplicationTrafficSecret = %x, want %x", got, serverApplicationTrafficSecret) + } + if got := tls13.TestingOnlyExporterSecret(ms.ExporterMasterSecret(transcript)); !bytes.Equal(got, exporterMasterSecret) { + t.Errorf("exporterMasterSecret = %x, want %x", got, exporterMasterSecret) + } + + transcript.Write(finishedClientRandom) + + if got := ms.ResumptionMasterSecret(transcript); !bytes.Equal(got, resumptionMasterSecret) { + t.Errorf("resumptionMasterSecret = %x, want %x", got, resumptionMasterSecret) + } +} + // This file contains tests derived from draft-ietf-tls-tls13-vectors-07. func parseVector(v string) []byte { @@ -32,78 +98,6 @@ func parseVector(v string) []byte { return res } -func TestDeriveSecret(t *testing.T) { - chTranscript := cipherSuitesTLS13[0].hash.New() - chTranscript.Write(parseVector(` - payload (512 octets): 01 00 01 fc 03 03 1b c3 ce b6 bb e3 9c ff - 93 83 55 b5 a5 0a db 6d b2 1b 7a 6a f6 49 d7 b4 bc 41 9d 78 76 - 48 7d 95 00 00 06 13 01 13 03 13 02 01 00 01 cd 00 00 00 0b 00 - 09 00 00 06 73 65 72 76 65 72 ff 01 00 01 00 00 0a 00 14 00 12 - 00 1d 00 17 00 18 00 19 01 00 01 01 01 02 01 03 01 04 00 33 00 - 26 00 24 00 1d 00 20 e4 ff b6 8a c0 5f 8d 96 c9 9d a2 66 98 34 - 6c 6b e1 64 82 ba dd da fe 05 1a 66 b4 f1 8d 66 8f 0b 00 2a 00 - 00 00 2b 00 03 02 03 04 00 0d 00 20 00 1e 04 03 05 03 06 03 02 - 03 08 04 08 05 08 06 04 01 05 01 06 01 02 01 04 02 05 02 06 02 - 02 02 00 2d 00 02 01 01 00 1c 00 02 40 01 00 15 00 57 00 00 00 - 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 - 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 - 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 - 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 - 00 29 00 dd 00 b8 00 b2 2c 03 5d 82 93 59 ee 5f f7 af 4e c9 00 - 00 00 00 26 2a 64 94 dc 48 6d 2c 8a 34 cb 33 fa 90 bf 1b 00 70 - ad 3c 49 88 83 c9 36 7c 09 a2 be 78 5a bc 55 cd 22 60 97 a3 a9 - 82 11 72 83 f8 2a 03 a1 43 ef d3 ff 5d d3 6d 64 e8 61 be 7f d6 - 1d 28 27 db 27 9c ce 14 50 77 d4 54 a3 66 4d 4e 6d a4 d2 9e e0 - 37 25 a6 a4 da fc d0 fc 67 d2 ae a7 05 29 51 3e 3d a2 67 7f a5 - 90 6c 5b 3f 7d 8f 92 f2 28 bd a4 0d da 72 14 70 f9 fb f2 97 b5 - ae a6 17 64 6f ac 5c 03 27 2e 97 07 27 c6 21 a7 91 41 ef 5f 7d - e6 50 5e 5b fb c3 88 e9 33 43 69 40 93 93 4a e4 d3 57 fa d6 aa - cb 00 21 20 3a dd 4f b2 d8 fd f8 22 a0 ca 3c f7 67 8e f5 e8 8d - ae 99 01 41 c5 92 4d 57 bb 6f a3 1b 9e 5f 9d`)) - - type args struct { - secret []byte - label string - transcript hash.Hash - } - tests := []struct { - name string - args args - want []byte - }{ - { - `derive secret for handshake "tls13 derived"`, - args{ - parseVector(`PRK (32 octets): 33 ad 0a 1c 60 7e c0 3b 09 e6 cd 98 93 68 0c e2 - 10 ad f3 00 aa 1f 26 60 e1 b2 2e 10 f1 70 f9 2a`), - "derived", - nil, - }, - parseVector(`expanded (32 octets): 6f 26 15 a1 08 c7 02 c5 67 8f 54 fc 9d ba - b6 97 16 c0 76 18 9c 48 25 0c eb ea c3 57 6c 36 11 ba`), - }, - { - `derive secret "tls13 c e traffic"`, - args{ - parseVector(`PRK (32 octets): 9b 21 88 e9 b2 fc 6d 64 d7 1d c3 29 90 0e 20 bb - 41 91 50 00 f6 78 aa 83 9c bb 79 7c b7 d8 33 2c`), - "c e traffic", - chTranscript, - }, - parseVector(`expanded (32 octets): 3f bb e6 a6 0d eb 66 c3 0a 32 79 5a ba 0e - ff 7e aa 10 10 55 86 e7 be 5c 09 67 8d 63 b6 ca ab 62`), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := cipherSuitesTLS13[0] - if got := c.deriveSecret(tt.args.secret, tt.args.label, tt.args.transcript); !bytes.Equal(got, tt.want) { - t.Errorf("cipherSuiteTLS13.deriveSecret() = % x, want % x", got, tt.want) - } - }) - } -} - func TestTrafficKey(t *testing.T) { trafficSecret := parseVector( `PRK (32 octets): b6 7b 7d 69 0c c1 6c 4e 75 e5 42 13 cb 2d 37 b4 @@ -123,90 +117,3 @@ func TestTrafficKey(t *testing.T) { t.Errorf("cipherSuiteTLS13.trafficKey() gotIV = % x, want % x", gotIV, wantIV) } } - -func TestExtract(t *testing.T) { - type args struct { - newSecret []byte - currentSecret []byte - } - tests := []struct { - name string - args args - want []byte - }{ - { - `extract secret "early"`, - args{ - nil, - nil, - }, - parseVector(`secret (32 octets): 33 ad 0a 1c 60 7e c0 3b 09 e6 cd 98 93 68 0c - e2 10 ad f3 00 aa 1f 26 60 e1 b2 2e 10 f1 70 f9 2a`), - }, - { - `extract secret "master"`, - args{ - nil, - parseVector(`salt (32 octets): 43 de 77 e0 c7 77 13 85 9a 94 4d b9 db 25 90 b5 - 31 90 a6 5b 3e e2 e4 f1 2d d7 a0 bb 7c e2 54 b4`), - }, - parseVector(`secret (32 octets): 18 df 06 84 3d 13 a0 8b f2 a4 49 84 4c 5f 8a - 47 80 01 bc 4d 4c 62 79 84 d5 a4 1d a8 d0 40 29 19`), - }, - { - `extract secret "handshake"`, - args{ - parseVector(`IKM (32 octets): 8b d4 05 4f b5 5b 9d 63 fd fb ac f9 f0 4b 9f 0d - 35 e6 d6 3f 53 75 63 ef d4 62 72 90 0f 89 49 2d`), - parseVector(`salt (32 octets): 6f 26 15 a1 08 c7 02 c5 67 8f 54 fc 9d ba b6 97 - 16 c0 76 18 9c 48 25 0c eb ea c3 57 6c 36 11 ba`), - }, - parseVector(`secret (32 octets): 1d c8 26 e9 36 06 aa 6f dc 0a ad c1 2f 74 1b - 01 04 6a a6 b9 9f 69 1e d2 21 a9 f0 ca 04 3f be ac`), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := cipherSuitesTLS13[0] - if got := c.extract(tt.args.newSecret, tt.args.currentSecret); !bytes.Equal(got, tt.want) { - t.Errorf("cipherSuiteTLS13.extract() = % x, want % x", got, tt.want) - } - }) - } -} - -func TestKyberDecapsulate(t *testing.T) { - // From https://pq-crystals.org/kyber/data/kyber-submission-nist-round3.zip - dkBytes, _ := hex.DecodeString("07638FB69868F3D320E5862BD96933FEB311B362093C9B5D50170BCED43F1B536D9A204BB1F22695950BA1F2A9E8EB828B284488760B3FC84FABA04275D5628E39C5B2471374283C503299C0AB49B66B8BBB56A4186624F919A2BA59BB08D8551880C2BEFC4F87F25F59AB587A79C327D792D54C974A69262FF8A78938289E9A87B688B083E0595FE218B6BB1505941CE2E81A5A64C5AAC60417256985349EE47A52420A5F97477B7236AC76BC70E8288729287EE3E34A3DBC3683C0B7B10029FC203418537E7466BA6385A8FF301EE12708F82AAA1E380FC7A88F8F205AB7E88D7E95952A55BA20D09B79A47141D62BF6EB7DD307B08ECA13A5BC5F6B68581C6865B27BBCDDAB142F4B2CBFF488C8A22705FAA98A2B9EEA3530C76662335CC7EA3A00777725EBCCCD2A4636B2D9122FF3AB77123CE0883C1911115E50C9E8A94194E48DD0D09CFFB3ADCD2C1E92430903D07ADBF00532031575AA7F9E7B5A1F3362DEC936D4043C05F2476C07578BC9CBAF2AB4E382727AD41686A96B2548820BB03B32F11B2811AD62F489E951632ABA0D1DF89680CC8A8B53B481D92A68D70B4EA1C3A6A561C0692882B5CA8CC942A8D495AFCB06DE89498FB935B775908FE7A03E324D54CC19D4E1AABD3593B38B19EE1388FE492B43127E5A504253786A0D69AD32601C28E2C88504A5BA599706023A61363E17C6B9BB59BDC697452CD059451983D738CA3FD034E3F5988854CA05031DB09611498988197C6B30D258DFE26265541C89A4B31D6864E9389B03CB74F7EC4323FB9421A4B9790A26D17B0398A26767350909F84D57B6694DF830664CA8B3C3C03ED2AE67B89006868A68527CCD666459AB7F056671000C6164D3A7F266A14D97CBD7004D6C92CACA770B844A4FA9B182E7B18CA885082AC5646FCB4A14E1685FEB0C9CE3372AB95365C04FD83084F80A23FF10A05BF15F7FA5ACC6C0CB462C33CA524FA6B8BB359043BA68609EAA2536E81D08463B19653B5435BA946C9ADDEB202B04B031CC960DCC12E4518D428B32B257A4FC7313D3A7980D80082E934F9D95C32B0A0191A23604384DD9E079BBBAA266D14C3F756B9F2133107433A4E83FA7187282A809203A4FAF841851833D121AC383843A5E55BC2381425E16C7DB4CC9AB5C1B0D91A47E2B8DE0E582C86B6B0D907BB360B97F40AB5D038F6B75C814B27D9B968D419832BC8C2BEE605EF6E5059D33100D90485D378450014221736C07407CAC260408AA64926619788B8601C2A752D1A6CBF820D7C7A04716203225B3895B9342D147A8185CFC1BB65BA06B4142339903C0AC4651385B45D98A8B19D28CD6BAB088787F7EE1B12461766B43CBCCB96434427D93C065550688F6948ED1B5475A425F1B85209D061C08B56C1CC069F6C0A7C6F29358CAB911087732A649D27C9B98F9A48879387D9B00C25959A71654D6F6A946164513E47A75D005986C2363C09F6B537ECA78B9303A5FA457608A586A653A347DB04DFCC19175B3A301172536062A658A95277570C8852CA8973F4AE123A334047DD711C8927A634A03388A527B034BF7A8170FA702C1F7C23EC32D18A2374890BE9C787A9409C82D192C4BB705A2F996CE405DA72C2D9C843EE9F8313ECC7F86D6294D59159D9A879A542E260922ADF999051CC45200C9FFDB60449C49465979272367C083A7D6267A3ED7A7FD47957C219327F7CA73A4007E1627F00B11CC80573C15AEE6640FB8562DFA6B240CA0AD351AC4AC155B96C14C8AB13DD262CDFD51C4BB5572FD616553D17BDD430ACBEA3E95F0B698D66990AB51E5D03783A8B3D278A5720454CF9695CFDCA08485BA099C51CD92A7EA7587C1D15C28E609A81852601B0604010679AA482D51261EC36E36B8719676217FD74C54786488F4B4969C05A8BA27CA3A77CCE73B965923CA554E422B9B61F4754641608AC16C9B8587A32C1C5DD788F88B36B717A46965635DEB67F45B129B99070909C93EB80B42C2B3F3F70343A7CF37E8520E7BCFC416ACA4F18C7981262BA2BFC756AE03278F0EC66DC2057696824BA6769865A601D7148EF6F54E5AF5686AA2906F994CE38A5E0B938F239007003022C03392DF3401B1E4A3A7EBC6161449F73374C8B0140369343D9295FDF511845C4A46EBAAB6CA5492F6800B98C0CC803653A4B1D6E6AAED1932BACC5FEFAA818BA502859BA5494C5F5402C8536A9C4C1888150617F80098F6B2A99C39BC5DC7CF3B5900A21329AB59053ABAA64ED163E859A8B3B3CA3359B750CCC3E710C7AC43C8191CB5D68870C06391C0CB8AEC72B897AC6BE7FBAACC676ED66314C83630E89448C88A1DF04ACEB23ABF2E409EF333C622289C18A2134E650C45257E47475FA33AA537A5A8F7680214716C50D470E3284963CA64F54677AEC54B5272162BF52BC8142E1D4183FC017454A6B5A496831759064024745978CBD51A6CEDC8955DE4CC6D363670A47466E82BE5C23603A17BF22ACDB7CC984AF08C87E14E27753CF587A8EC3447E62C649E887A67C36C9CE98721B697213275646B194F36758673A8ED11284455AFC7A8529F69C97A3C2D7B8C636C0BA55614B768E624E712930F776169B01715725351BC74B47395ED52B25A1313C95164814C34C979CBDFAB85954662CAB485E75087A98CC74BB82CA2D1B5BF2803238480638C40E90B43C7460E7AA917F010151FAB1169987B372ABB59271F7006C24E60236B84B9DDD600623704254617FB498D89E58B0368BCB2103E79353EB587860C1422E476162E425BC2381DB82C6592737E1DD602864B0167A71EC1F223305C02FE25052AF2B3B5A55A0D7A2022D9A798DC0C5874A98702AAF4054C5D80338A5248B5B7BD09C53B5E2A084B047D277A861B1A73BB51488DE04EF573C85230A0470B73175C9FA50594F66A5F50B4150054C93B68186F8B5CBC49316C8548A642B2B36A1D454C7489AC33B2D2CE6668096782A2C1E0866D21A65E16B585E7AF8618BDF3184C1986878508917277B93E10706B1614972B2A94C7310FE9C708C231A1A8AC8D9314A529A97F469BF64962D820648443099A076D55D4CEA824A58304844F99497C10A25148618A315D72CA857D1B04D575B94F85C01D19BEF211BF0AA3362E7041FD16596D808E867B44C4C00D1CDA3418967717F147D0EB21B42AAEE74AC35D0B92414B958531AADF463EC6305AE5ECAF79174002F26DDECC813BF32672E8529D95A4E730A7AB4A3E8F8A8AF979A665EAFD465FC64A0C5F8F3F9003489415899D59A543D8208C54A3166529B53922D4EC143B50F01423B177895EDEE22BB739F647ECF85F50BC25EF7B5A725DEE868626ED79D451140800E03B59B956F8210E556067407D13DC90FA9E8B872BFB8F") - dk, err := mlkem768.NewKeyFromExtendedEncoding(dkBytes) - if err != nil { - t.Fatal(err) - } - ct, _ := hex.DecodeString("B52C56B92A4B7CE9E4CB7C5B1B163167A8A1675B2FDEF84A5B67CA15DB694C9F11BD027C30AE22EC921A1D911599AF0585E48D20DA70DF9F39E32EF95D4C8F44BFEFDAA5DA64F1054631D04D6D3CFD0A540DD7BA3886E4B5F13E878788604C95C096EAB3919F427521419A946C26CC041475D7124CDC01D0373E5B09C7A70603CFDB4FB3405023F2264DC3F983C4FC02A2D1B268F2208A1F6E2A6209BFF12F6F465F0B069C3A7F84F606D8A94064003D6EC114C8E808D3053884C1D5A142FBF20112EB360FDA3F0F28B172AE50F5E7D83801FB3F0064B687187074BD7FE30EDDAA334CF8FC04FA8CED899CEADE4B4F28B68372BAF98FF482A415B731155B75CEB976BE0EA0285BA01A27F1857A8FB377A3AE0C23B2AA9A079BFABFF0D5B2F1CD9B718BEA03C42F343A39B4F142D01AD8ACBB50E38853CF9A50C8B44C3CF671A4A9043B26DDBB24959AD6715C08521855C79A23B9C3D6471749C40725BDD5C2776D43AED20204BAA141EFB3304917474B7F9F7A4B08B1A93DAED98C67495359D37D67F7438BEE5E43585634B26C6B3810D7CDCBC0F6EB877A6087E68ACB8480D3A8CF6900447E49B417F15A53B607A0E216B855970D37406870B4568722DA77A4084703816784E2F16BED18996532C5D8B7F5D214464E5F3F6E905867B0CE119E252A66713253544685D208E1723908A0CE97834652E08AE7BDC881A131B73C71E84D20D68FDEFF4F5D70CD1AF57B78E3491A9865942321800A203C05ED1FEEB5A28E584E19F6535E7F84E4A24F84A72DCAF5648B4A4235DD664464482F03176E888C28BFC6C1CB238CFFA35A321E71791D9EA8ED0878C61121BF8D2A4AB2C1A5E120BC40ABB1892D1715090A0EE48252CA297A99AA0E510CF26B1ADD06CA543E1C5D6BDCD3B9C585C8538045DB5C252EC3C8C3C954D9BE5907094A894E60EAB43538CFEE82E8FFC0791B0D0F43AC1627830A61D56DAD96C62958B0DE780B78BD47A604550DAB83FFF227C324049471F35248CFB849B25724FF704D5277AA352D550958BE3B237DFF473EC2ADBAEA48CA2658AEFCC77BBD4264AB374D70EAE5B964416CE8226A7E3255A0F8D7E2ADCA062BCD6D78D60D1B32E11405BE54B66EF0FDDD567702A3BCCFEDE3C584701269ED14809F06F8968356BB9267FE86E514252E88BB5C30A7ECB3D0E621021EE0FBF7871B09342BF84F55C97EAF86C48189C7FF4DF389F077E2806E5FA73B3E9458A16C7E275F4F602275580EB7B7135FB537FA0CD95D6EA58C108CD8943D70C1643111F4F01CA8A8276A902666ED81B78D168B006F16AAA3D8E4CE4F4D0FB0997E41AEFFB5B3DAA838732F357349447F387776C793C0479DE9E99498CC356FDB0075A703F23C55D47B550EC89B02ADE89329086A50843456FEDC3788AC8D97233C54560467EE1D0F024B18428F0D73B30E19F5C63B9ABF11415BEA4D0170130BAABD33C05E6524E5FB5581B22B0433342248266D0F1053B245CC2462DC44D34965102482A8ED9E4E964D5683E5D45D0C8269") - ss, err := kyberDecapsulate(dk, ct) - if err != nil { - t.Fatal(err) - } - exp, _ := hex.DecodeString("914CB67FE5C38E73BF74181C0AC50428DEDF7750A98058F7D536708774535B29") - if !bytes.Equal(ss, exp) { - t.Fatalf("got %x, want %x", ss, exp) - } -} - -func TestKyberEncapsulate(t *testing.T) { - dk, err := mlkem768.GenerateKey() - if err != nil { - t.Fatal(err) - } - ct, ss, err := kyberEncapsulate(dk.EncapsulationKey()) - if err != nil { - t.Fatal(err) - } - dkSS, err := kyberDecapsulate(dk, ct) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(ss, dkSS) { - t.Fatalf("got %x, want %x", ss, dkSS) - } -} diff --git a/src/crypto/tls/notboring.go b/src/crypto/tls/notboring.go deleted file mode 100644 index bdbc32e05b35dd..00000000000000 --- a/src/crypto/tls/notboring.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !boringcrypto - -package tls - -func needFIPS() bool { return false } diff --git a/src/crypto/tls/prf.go b/src/crypto/tls/prf.go index a7fa3370e66c82..e7369542a73270 100644 --- a/src/crypto/tls/prf.go +++ b/src/crypto/tls/prf.go @@ -7,6 +7,7 @@ package tls import ( "crypto" "crypto/hmac" + "crypto/internal/fips140/tls12" "crypto/md5" "crypto/sha1" "crypto/sha256" @@ -16,6 +17,8 @@ import ( "hash" ) +type prfFunc func(secret []byte, label string, seed []byte, keyLen int) []byte + // Split a premaster secret in two as specified in RFC 4346, Section 5. func splitPreMasterSecret(secret []byte) (s1, s2 []byte) { s1 = secret[0 : (len(secret)+1)/2] @@ -45,7 +48,8 @@ func pHash(result, secret, seed []byte, hash func() hash.Hash) { } // prf10 implements the TLS 1.0 pseudo-random function, as defined in RFC 2246, Section 5. -func prf10(result, secret, label, seed []byte) { +func prf10(secret []byte, label string, seed []byte, keyLen int) []byte { + result := make([]byte, keyLen) hashSHA1 := sha1.New hashMD5 := md5.New @@ -61,16 +65,14 @@ func prf10(result, secret, label, seed []byte) { for i, b := range result2 { result[i] ^= b } + + return result } // prf12 implements the TLS 1.2 pseudo-random function, as defined in RFC 5246, Section 5. -func prf12(hashFunc func() hash.Hash) func(result, secret, label, seed []byte) { - return func(result, secret, label, seed []byte) { - labelAndSeed := make([]byte, len(label)+len(seed)) - copy(labelAndSeed, label) - copy(labelAndSeed[len(label):], seed) - - pHash(result, secret, labelAndSeed, hashFunc) +func prf12(hashFunc func() hash.Hash) prfFunc { + return func(secret []byte, label string, seed []byte, keyLen int) []byte { + return tls12.PRF(hashFunc, secret, label, seed, keyLen) } } @@ -79,13 +81,13 @@ const ( finishedVerifyLength = 12 // Length of verify_data in a Finished message. ) -var masterSecretLabel = []byte("master secret") -var extendedMasterSecretLabel = []byte("extended master secret") -var keyExpansionLabel = []byte("key expansion") -var clientFinishedLabel = []byte("client finished") -var serverFinishedLabel = []byte("server finished") +const masterSecretLabel = "master secret" +const extendedMasterSecretLabel = "extended master secret" +const keyExpansionLabel = "key expansion" +const clientFinishedLabel = "client finished" +const serverFinishedLabel = "server finished" -func prfAndHashForVersion(version uint16, suite *cipherSuite) (func(result, secret, label, seed []byte), crypto.Hash) { +func prfAndHashForVersion(version uint16, suite *cipherSuite) (prfFunc, crypto.Hash) { switch version { case VersionTLS10, VersionTLS11: return prf10, crypto.Hash(0) @@ -99,7 +101,7 @@ func prfAndHashForVersion(version uint16, suite *cipherSuite) (func(result, secr } } -func prfForVersion(version uint16, suite *cipherSuite) func(result, secret, label, seed []byte) { +func prfForVersion(version uint16, suite *cipherSuite) prfFunc { prf, _ := prfAndHashForVersion(version, suite) return prf } @@ -111,17 +113,19 @@ func masterFromPreMasterSecret(version uint16, suite *cipherSuite, preMasterSecr seed = append(seed, clientRandom...) seed = append(seed, serverRandom...) - masterSecret := make([]byte, masterSecretLength) - prfForVersion(version, suite)(masterSecret, preMasterSecret, masterSecretLabel, seed) - return masterSecret + return prfForVersion(version, suite)(preMasterSecret, masterSecretLabel, seed, masterSecretLength) } // extMasterFromPreMasterSecret generates the extended master secret from the // pre-master secret. See RFC 7627. func extMasterFromPreMasterSecret(version uint16, suite *cipherSuite, preMasterSecret, transcript []byte) []byte { - masterSecret := make([]byte, masterSecretLength) - prfForVersion(version, suite)(masterSecret, preMasterSecret, extendedMasterSecretLabel, transcript) - return masterSecret + prf, hash := prfAndHashForVersion(version, suite) + if version == VersionTLS12 { + // Use the FIPS 140-3 module only for TLS 1.2 with EMS, which is the + // only TLS 1.0-1.2 approved mode per IG D.Q. + return tls12.MasterSecret(hash.New, preMasterSecret, transcript) + } + return prf(preMasterSecret, extendedMasterSecretLabel, transcript, masterSecretLength) } // keysFromMasterSecret generates the connection keys from the master @@ -133,8 +137,7 @@ func keysFromMasterSecret(version uint16, suite *cipherSuite, masterSecret, clie seed = append(seed, clientRandom...) n := 2*macLen + 2*keyLen + 2*ivLen - keyMaterial := make([]byte, n) - prfForVersion(version, suite)(keyMaterial, masterSecret, keyExpansionLabel, seed) + keyMaterial := prfForVersion(version, suite)(masterSecret, keyExpansionLabel, seed, n) clientMAC = keyMaterial[:macLen] keyMaterial = keyMaterial[macLen:] serverMAC = keyMaterial[:macLen] @@ -177,7 +180,7 @@ type finishedHash struct { buffer []byte version uint16 - prf func(result, secret, label, seed []byte) + prf prfFunc } func (h *finishedHash) Write(msg []byte) (n int, err error) { @@ -209,17 +212,13 @@ func (h finishedHash) Sum() []byte { // clientSum returns the contents of the verify_data member of a client's // Finished message. func (h finishedHash) clientSum(masterSecret []byte) []byte { - out := make([]byte, finishedVerifyLength) - h.prf(out, masterSecret, clientFinishedLabel, h.Sum()) - return out + return h.prf(masterSecret, clientFinishedLabel, h.Sum(), finishedVerifyLength) } // serverSum returns the contents of the verify_data member of a server's // Finished message. func (h finishedHash) serverSum(masterSecret []byte) []byte { - out := make([]byte, finishedVerifyLength) - h.prf(out, masterSecret, serverFinishedLabel, h.Sum()) - return out + return h.prf(masterSecret, serverFinishedLabel, h.Sum(), finishedVerifyLength) } // hashForClientCertificate returns the handshake messages so far, pre-hashed if @@ -292,8 +291,6 @@ func ekmFromMasterSecret(version uint16, suite *cipherSuite, masterSecret, clien seed = append(seed, context...) } - keyMaterial := make([]byte, length) - prfForVersion(version, suite)(keyMaterial, masterSecret, []byte(label), seed) - return keyMaterial, nil + return prfForVersion(version, suite)(masterSecret, label, seed, length), nil } } diff --git a/src/crypto/tls/quic.go b/src/crypto/tls/quic.go index 9dd6168b62356e..2ba2242b2d93d2 100644 --- a/src/crypto/tls/quic.go +++ b/src/crypto/tls/quic.go @@ -206,7 +206,7 @@ func (q *QUICConn) Start(ctx context.Context) error { } q.conn.quic.started = true if q.conn.config.MinVersion < VersionTLS13 { - return quicError(errors.New("tls: Config MinVersion must be at least TLS 1.13")) + return quicError(errors.New("tls: Config MinVersion must be at least TLS 1.3")) } go q.conn.HandshakeContext(ctx) if _, ok := <-q.conn.quic.blockedc; !ok { @@ -221,7 +221,7 @@ func (q *QUICConn) NextEvent() QUICEvent { qs := q.conn.quic if last := qs.nextEvent - 1; last >= 0 && len(qs.events[last].Data) > 0 { // Write over some of the previous event's data, - // to catch callers erroniously retaining it. + // to catch callers erroneously retaining it. qs.events[last].Data[0] = 0 } if qs.nextEvent >= len(qs.events) && qs.waitingForDrain { @@ -302,6 +302,9 @@ type QUICSessionTicketOptions struct { // Currently, it can only be called once. func (q *QUICConn) SendSessionTicket(opts QUICSessionTicketOptions) error { c := q.conn + if c.config.SessionTicketsDisabled { + return nil + } if !c.isHandshakeComplete.Load() { return quicError(errors.New("tls: SendSessionTicket called before handshake completed")) } @@ -359,12 +362,11 @@ func quicError(err error) error { if err == nil { return nil } - var ae AlertError - if errors.As(err, &ae) { + if _, ok := errors.AsType[AlertError](err); ok { return err } - var a alert - if !errors.As(err, &a) { + a, ok := errors.AsType[alert](err) + if !ok { a = alertInternalError } // Return an error wrapping the original error and an AlertError. diff --git a/src/crypto/tls/quic_test.go b/src/crypto/tls/quic_test.go index 1bb2e55bddcbea..5f4b2b7707d01e 100644 --- a/src/crypto/tls/quic_test.go +++ b/src/crypto/tls/quic_test.go @@ -9,6 +9,7 @@ import ( "context" "errors" "reflect" + "strings" "testing" ) @@ -230,6 +231,18 @@ func TestQUICSessionResumption(t *testing.T) { if !cli2.conn.ConnectionState().DidResume { t.Errorf("second connection did not use session resumption") } + + clientConfig.TLSConfig.SessionTicketsDisabled = true + cli3 := newTestQUICClient(t, clientConfig) + cli3.conn.SetTransportParameters(nil) + srv3 := newTestQUICServer(t, serverConfig) + srv3.conn.SetTransportParameters(nil) + if err := runTestQUICConnection(context.Background(), cli3, srv3, nil); err != nil { + t.Fatalf("error during third connection handshake: %v", err) + } + if cli3.conn.ConnectionState().DidResume { + t.Errorf("third connection unexpectedly used session resumption") + } } func TestQUICFragmentaryData(t *testing.T) { @@ -278,7 +291,7 @@ func TestQUICPostHandshakeClientAuthentication(t *testing.T) { certReq := new(certificateRequestMsgTLS13) certReq.ocspStapling = true certReq.scts = true - certReq.supportedSignatureAlgorithms = supportedSignatureAlgorithms() + certReq.supportedSignatureAlgorithms = supportedSignatureAlgorithms(VersionTLS13) certReqBytes, err := certReq.marshal() if err != nil { t.Fatal(err) @@ -308,11 +321,11 @@ func TestQUICPostHandshakeKeyUpdate(t *testing.T) { if err != nil { t.Fatal(err) } - if err := cli.conn.HandleData(QUICEncryptionLevelApplication, append([]byte{ - byte(typeKeyUpdate), - byte(0), byte(0), byte(len(keyUpdateBytes)), - }, keyUpdateBytes...)); !errors.Is(err, alertUnexpectedMessage) { - t.Fatalf("key update request: got error %v, want alertUnexpectedMessage", err) + expectedErr := "unexpected key update message" + if err = cli.conn.HandleData(QUICEncryptionLevelApplication, keyUpdateBytes); err == nil { + t.Fatalf("key update request: expected error from post-handshake key update, got nil") + } else if !strings.Contains(err.Error(), expectedErr) { + t.Fatalf("key update request: got error %v, expected substring %q", err, expectedErr) } } @@ -355,8 +368,7 @@ func TestQUICHandshakeError(t *testing.T) { if !errors.Is(err, AlertError(alertBadCertificate)) { t.Errorf("connection handshake terminated with error %q, want alertBadCertificate", err) } - var e *CertificateVerificationError - if !errors.As(err, &e) { + if _, ok := errors.AsType[*CertificateVerificationError](err); !ok { t.Errorf("connection handshake terminated with error %q, want CertificateVerificationError", err) } } diff --git a/src/crypto/tls/testdata/Client-TLSv10-ClientCert-ECDSA-ECDSA b/src/crypto/tls/testdata/Client-TLSv10-ClientCert-ECDSA-ECDSA index a35aa57ee0986f..3bd86fd90ed92a 100644 --- a/src/crypto/tls/testdata/Client-TLSv10-ClientCert-ECDSA-ECDSA +++ b/src/crypto/tls/testdata/Client-TLSv10-ClientCert-ECDSA-ECDSA @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 01 00 5d 02 00 00 59 03 01 60 8a bf 2b 6f |....]...Y..`..+o| -00000010 c1 f0 f3 7b d4 78 2c d9 15 3f 33 6a 4b 96 aa 83 |...{.x,..?3jK...| -00000020 54 ae 66 8f 0f e7 b1 68 31 00 39 20 d8 4d f2 2c |T.f....h1.9 .M.,| -00000030 8b 85 98 d6 af e0 07 98 d5 cb 89 72 fb 8d c0 73 |...........r...s| -00000040 cc 68 cf 58 21 4a 8a fc c3 8e 90 0a c0 09 00 00 |.h.X!J..........| +00000000 16 03 01 00 5d 02 00 00 59 03 01 1a 76 c4 d5 b5 |....]...Y...v...| +00000010 7e 30 1c 1e c3 62 8b 92 a8 19 9b 62 4e 36 a1 d3 |~0...b.....bN6..| +00000020 c8 e3 55 f6 64 5f 06 aa 19 9a 48 20 24 a4 2d a9 |..U.d_....H $.-.| +00000030 8a bd cf 4e 9b cf 7d c7 8e 8f e8 ed f5 09 37 ad |...N..}.......7.| +00000040 06 c9 6b a4 d5 de 7f 07 07 a6 9d c7 c0 09 00 00 |..k.............| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 01 02 0e 0b 00 02 0a 00 02 07 00 02 |................| 00000070 04 30 82 02 00 30 82 01 62 02 09 00 b8 bf 2d 47 |.0...0..b.....-G| @@ -57,17 +58,17 @@ 00000250 c9 86 2e dd d7 11 69 7f 85 7c 56 de fb 31 78 2b |......i..|V..1x+| 00000260 e4 c7 78 0d ae cb be 9e 4e 36 24 31 7b 6a 0f 39 |..x.....N6$1{j.9| 00000270 95 12 07 8f 2a 16 03 01 00 b5 0c 00 00 b1 03 00 |....*...........| -00000280 1d 20 68 5a 6f c6 20 26 b5 5c 47 c6 5e fb 23 c4 |. hZo. &.\G.^.#.| -00000290 bc 9c 3f 3a 9b ed d6 8e a3 c8 66 7d 9b cb d0 30 |..?:......f}...0| -000002a0 f9 60 00 8b 30 81 88 02 42 01 b2 fb f8 5f d0 14 |.`..0...B...._..| -000002b0 9d 3c 55 0f 16 50 6f d5 0c 4c 3e 73 2e a9 23 5f |.s..#_| -000002c0 e8 9c 02 5d 4c 6d b0 c1 9e 0d ac 59 36 6c d5 c2 |...]Lm.....Y6l..| -000002d0 4c 94 94 94 6f a4 df 26 1a 54 f5 74 b8 49 75 49 |L...o..&.T.t.IuI| -000002e0 9c aa cd 91 24 f3 52 88 6a 1e 80 02 42 01 81 a2 |....$.R.j...B...| -000002f0 76 e2 e8 b0 2a 8e 4e ed 6d be 2f e3 ca 4c ff f2 |v...*.N.m./..L..| -00000300 d3 14 c0 b5 f8 c5 53 04 97 de 14 b2 8e af 77 86 |......S.......w.| -00000310 de bf 4a 59 cf 7b 8a 73 d3 95 c3 28 ca 25 63 8e |..JY.{.s...(.%c.| -00000320 9e 02 7f 8a 04 bb 69 1e 41 31 76 2b 5a 54 ed 16 |......i.A1v+ZT..| +00000280 1d 20 3e 61 33 da 82 02 f5 6d 70 2d e5 07 df 43 |. >a3....mp-...C| +00000290 d7 e6 1a b2 81 3e 58 03 dc 27 37 34 3b 2e 5a d5 |.....>X..'74;.Z.| +000002a0 d4 1a 00 8b 30 81 88 02 42 00 8b 0c f4 0f fe 8c |....0...B.......| +000002b0 01 70 3a c5 e5 29 53 1c ad 6c f4 35 53 e3 e6 ce |.p:..)S..l.5S...| +000002c0 19 21 03 de f5 e3 df 8e e5 6b 43 00 08 b7 24 2c |.!.......kC...$,| +000002d0 27 9e 2b bc 2f 96 07 a0 f9 a9 6b d4 b7 75 a8 5d |'.+./.....k..u.]| +000002e0 99 13 ab 5a dd 35 56 08 6d 9c a1 02 42 01 29 62 |...Z.5V.m...B.)b| +000002f0 b9 64 8b 6d 91 b7 f5 59 73 9d 2d bc 9d 9a 5b 43 |.d.m...Ys.-...[C| +00000300 d8 d5 76 be 2a 8f c5 f0 2e 5d a8 be 7b d0 36 fe |..v.*....]..{.6.| +00000310 62 97 7f 17 96 f6 c3 b4 06 2a e2 cb 77 80 ff 28 |b........*..w..(| +00000320 02 64 cb bb c7 a9 77 4b 96 7d 07 11 7e e8 28 16 |.d....wK.}..~.(.| 00000330 03 01 00 0a 0d 00 00 06 03 01 02 40 00 00 16 03 |...........@....| 00000340 01 00 04 0e 00 00 00 |.......| >>> Flow 3 (client to server) @@ -106,30 +107,30 @@ 00000200 e4 fa cc b1 8a ce e2 23 a0 87 f0 e1 67 51 eb 16 |.......#....gQ..| 00000210 03 01 00 25 10 00 00 21 20 2f e5 7d a3 47 cd 62 |...%...! /.}.G.b| 00000220 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 cf |C.(.._.).0......| -00000230 c2 ed 90 99 5f 58 cb 3b 74 16 03 01 00 91 0f 00 |...._X.;t.......| -00000240 00 8d 00 8b 30 81 88 02 42 01 57 eb e6 90 3c fe |....0...B.W...<.| -00000250 10 8d e9 7d 50 a5 d3 83 43 64 e7 d0 cb 65 ef b9 |...}P...Cd...e..| -00000260 56 59 b9 52 09 e8 52 d7 d7 4d a7 57 09 dd 1f 83 |VY.R..R..M.W....| -00000270 22 0e 4c 4e 8b 50 0d 68 72 26 9e 2a 8b 6d cb 88 |".LN.P.hr&.*.m..| -00000280 c8 f4 0d 7d 85 80 e2 ec 7f f1 be 02 42 01 68 5d |...}........B.h]| -00000290 f8 91 45 82 61 7f 57 e2 2e b2 54 7d f3 11 11 44 |..E.a.W...T}...D| -000002a0 2f ee 48 91 17 5c 04 3d b8 0e eb ed 66 33 b1 62 |/.H..\.=....f3.b| -000002b0 61 a1 03 e8 77 cf 44 4b 93 fc 4f 12 24 2f d1 67 |a...w.DK..O.$/.g| -000002c0 c8 4b 07 e3 cb a8 7d 5d 82 d4 a2 ec d7 0b f8 14 |.K....}]........| -000002d0 03 01 00 01 01 16 03 01 00 30 28 45 dc 18 db 52 |.........0(E...R| -000002e0 a7 b5 1e 68 7a 06 03 8a 23 87 07 ea 79 38 29 ec |...hz...#...y8).| -000002f0 1b b7 b9 cb 1b 04 ac ba 1d b2 d5 8e 71 e0 27 30 |............q.'0| -00000300 02 d1 c9 4d 35 69 38 71 c9 c7 |...M5i8q..| +00000230 c2 ed 90 99 5f 58 cb 3b 74 16 03 01 00 90 0f 00 |...._X.;t.......| +00000240 00 8c 00 8a 30 81 87 02 41 46 5f cc 88 bc 2b e5 |....0...AF_...+.| +00000250 20 33 a9 ff 52 bb 25 dc cf a1 08 45 b5 82 9a 1b | 3..R.%....E....| +00000260 53 04 18 b4 23 20 c4 d2 d8 92 9b 21 a7 ec 36 d6 |S...# .....!..6.| +00000270 63 35 68 4c 21 c4 0e 20 07 0c 8f 38 2f 9b b7 9e |c5hL!.. ...8/...| +00000280 e7 9d 6d ea da 1c a6 f1 c5 d3 02 42 01 ed 54 c8 |..m........B..T.| +00000290 9e b2 45 f1 1f 77 ee e3 a7 3a 40 9b fe 9b 1d 38 |..E..w...:@....8| +000002a0 ff 9b b4 c6 e7 94 07 b8 5f 93 bd 38 8d 31 dd 80 |........_..8.1..| +000002b0 2f 82 6e b5 85 8f 15 8b cd b2 04 81 e2 0b 2a fa |/.n...........*.| +000002c0 d9 4b 47 0b 86 26 8b 4e 83 a6 15 9e 93 15 14 03 |.KG..&.N........| +000002d0 01 00 01 01 16 03 01 00 30 f3 06 0e 6f 16 1b 0b |........0...o...| +000002e0 46 54 37 fd c6 80 0f ec 4b dd d7 d4 17 b2 d7 8d |FT7.....K.......| +000002f0 4e 6a 7f 2a af d2 ea 19 11 8f 23 57 b7 e7 2f b3 |Nj.*......#W../.| +00000300 97 f6 93 3f ac 0d 5c f8 f5 |...?..\..| >>> Flow 4 (server to client) -00000000 14 03 01 00 01 01 16 03 01 00 30 3d a8 df 2e 80 |..........0=....| -00000010 26 22 66 32 fb 6e bc 9e f5 d6 6a 5e 0a 18 34 92 |&"f2.n....j^..4.| -00000020 f9 42 40 e4 9c b1 7a 28 d2 52 e9 b8 13 ce 89 01 |.B@...z(.R......| -00000030 23 44 ab 2e 75 3e c2 96 f5 59 61 |#D..u>...Ya| +00000000 14 03 01 00 01 01 16 03 01 00 30 03 aa a1 4c bf |..........0...L.| +00000010 ab 53 f3 96 e7 db b1 a4 d7 19 f7 2f c0 b7 32 ea |.S........./..2.| +00000020 bd f3 4a 7c 0d 4c df a4 5f a0 3d f3 1b 57 65 a1 |..J|.L.._.=..We.| +00000030 d1 31 ae 52 6e ed 15 d2 d9 23 68 |.1.Rn....#h| >>> Flow 5 (client to server) -00000000 17 03 01 00 20 64 ac d2 1b 98 4d 4e ec 22 25 35 |.... d....MN."%5| -00000010 7c 7c 79 15 a8 54 0d 93 1a b3 3e ff 68 27 d0 d5 |||y..T....>.h'..| -00000020 82 24 7b dc e7 17 03 01 00 20 98 da 6d 26 de b8 |.${...... ..m&..| -00000030 4b d0 ee 2c 18 52 05 86 59 b0 b6 e7 42 74 54 ea |K..,.R..Y...BtT.| -00000040 e6 96 de ec 82 dc 1f 3a 6c 5c 15 03 01 00 20 cc |.......:l\.... .| -00000050 11 da e4 75 e5 0c 06 84 e5 89 3e a4 ef 3c 28 10 |...u......>..<(.| -00000060 68 0a 6f d4 6e ae 14 a6 ca 7c ba 98 c4 28 7e |h.o.n....|...(~| +00000000 17 03 01 00 20 aa 0c f4 30 17 b7 59 47 33 91 07 |.... ...0..YG3..| +00000010 cc 52 7b 52 f8 a0 82 44 25 03 0e b4 45 6e 0c 5a |.R{R...D%...En.Z| +00000020 37 6d d6 6d 80 17 03 01 00 20 bc c7 70 3b 94 cc |7m.m..... ..p;..| +00000030 09 db 15 84 d8 b9 c5 cf d6 01 91 93 5b 20 5a b1 |............[ Z.| +00000040 6a 43 23 30 7e 02 ea 8b e0 28 15 03 01 00 20 c2 |jC#0~....(.... .| +00000050 7f c9 90 b3 ef c6 52 68 44 df 05 30 17 39 d9 42 |......RhD..0.9.B| +00000060 76 7a 1d a1 17 2b fa b4 92 ca 40 a9 58 d4 87 |vz...+....@.X..| diff --git a/src/crypto/tls/testdata/Client-TLSv10-ClientCert-ECDSA-RSA b/src/crypto/tls/testdata/Client-TLSv10-ClientCert-ECDSA-RSA index 687eba43e6cf8c..de3de89cd16616 100644 --- a/src/crypto/tls/testdata/Client-TLSv10-ClientCert-ECDSA-RSA +++ b/src/crypto/tls/testdata/Client-TLSv10-ClientCert-ECDSA-RSA @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 01 00 5d 02 00 00 59 03 01 48 c6 a1 aa 01 |....]...Y..H....| -00000010 b8 4b a1 f5 6d 71 28 f7 fd 76 9d 12 50 d3 56 e2 |.K..mq(..v..P.V.| -00000020 ca 77 d4 82 07 65 e2 c5 12 40 35 20 e3 ee 0a df |.w...e...@5 ....| -00000030 1f 4a d9 b3 a6 6f a0 24 49 8d ab d9 d8 3c 24 36 |.J...o.$I....<$6| -00000040 35 ba 64 8b 7d ec 29 91 d9 26 5e 61 c0 13 00 00 |5.d.}.)..&^a....| +00000000 16 03 01 00 5d 02 00 00 59 03 01 fc 68 73 9f 1a |....]...Y...hs..| +00000010 9d 3b 9b f9 10 cf b5 84 b9 31 f4 a8 e9 47 ab 33 |.;.......1...G.3| +00000020 55 42 0b c0 f2 8e fa e7 a0 39 55 20 56 cd 87 f6 |UB.......9U V...| +00000030 05 e0 b1 e9 aa b8 b2 ca 33 9d 46 02 fd d8 f0 11 |........3.F.....| +00000040 0e fc 96 7e b4 fa fc c2 f3 da c4 2b c0 13 00 00 |...~.......+....| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 01 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -61,17 +62,17 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 01 00 aa 0c 00 00 a6 03 00 1d 20 55 51 65 |............ UQe| -000002d0 bb 06 22 d7 d6 97 39 d1 f4 dc 95 06 b3 a4 a7 00 |.."...9.........| -000002e0 d1 e5 98 bc 97 12 03 25 03 12 ab 20 4f 00 80 71 |.......%... O..q| -000002f0 8d 3c 54 44 ba df 73 92 76 16 d1 ec b1 de a2 27 |.>> Flow 3 (client to server) @@ -110,30 +111,30 @@ 00000200 e4 fa cc b1 8a ce e2 23 a0 87 f0 e1 67 51 eb 16 |.......#....gQ..| 00000210 03 01 00 25 10 00 00 21 20 2f e5 7d a3 47 cd 62 |...%...! /.}.G.b| 00000220 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 cf |C.(.._.).0......| -00000230 c2 ed 90 99 5f 58 cb 3b 74 16 03 01 00 90 0f 00 |...._X.;t.......| -00000240 00 8c 00 8a 30 81 87 02 41 79 c5 57 74 46 9a 18 |....0...Ay.WtF..| -00000250 ad e0 b0 ba 68 f5 0e e2 58 94 dc 73 84 5a f8 86 |....h...X..s.Z..| -00000260 86 8e 2a 37 82 02 a1 4b 19 cd 71 b3 99 04 64 b0 |..*7...K..q...d.| -00000270 db 4a cc 41 a6 17 28 38 f1 67 bd 59 16 97 71 32 |.J.A..(8.g.Y..q2| -00000280 06 71 24 2c f3 df 34 1b a3 b8 02 42 01 2f 2f db |.q$,..4....B.//.| -00000290 45 07 94 53 89 81 59 0b 92 9d 1a 05 42 b3 1c 40 |E..S..Y.....B..@| -000002a0 38 50 a1 8e a6 35 15 76 ca 75 7e fc 8d 7b 36 f3 |8P...5.v.u~..{6.| -000002b0 e3 e7 bf f9 bd 94 35 a0 c5 2b 35 88 be 5d ee f1 |......5..+5..]..| -000002c0 00 f9 9c a8 01 a8 92 5d d6 17 b6 54 98 e4 14 03 |.......]...T....| -000002d0 01 00 01 01 16 03 01 00 30 1f d6 11 ac 58 b3 20 |........0....X. | -000002e0 31 6d 3b f5 83 98 50 75 ff 4f 79 61 2b fc 0f 6c |1m;...Pu.Oya+..l| -000002f0 a6 4d 9e 65 38 e3 ca 12 76 0a 56 1e dd 73 da e1 |.M.e8...v.V..s..| -00000300 66 5a 33 62 8f 7d c3 ed ad |fZ3b.}...| +00000230 c2 ed 90 99 5f 58 cb 3b 74 16 03 01 00 91 0f 00 |...._X.;t.......| +00000240 00 8d 00 8b 30 81 88 02 42 00 88 3e 14 10 fa b5 |....0...B..>....| +00000250 65 d8 03 71 7e b7 44 a5 db 04 85 d2 f4 5d c5 de |e..q~.D......]..| +00000260 71 e4 f9 ad 5e 47 6c 83 eb 6a 2b fc 8d 60 6b 1b |q...^Gl..j+..`k.| +00000270 55 89 7b 03 60 fb 9c b2 b1 42 ef 02 63 29 59 03 |U.{.`....B..c)Y.| +00000280 02 a8 48 4d 9a 3d f3 e9 6b ac 76 02 42 01 90 36 |..HM.=..k.v.B..6| +00000290 5d d0 ec dd 76 75 0c 97 66 7f 10 ec 1d 39 5e bb |]...vu..f....9^.| +000002a0 2c 81 9e 15 fa 59 3f e8 77 3f 33 03 b6 2d 02 5a |,....Y?.w?3..-.Z| +000002b0 28 82 53 7a 18 69 29 5b d9 7d ce 4f 94 d9 69 29 |(.Sz.i)[.}.O..i)| +000002c0 b2 84 87 4a 15 47 c6 da 6f c3 df ca 8a 58 0b 14 |...J.G..o....X..| +000002d0 03 01 00 01 01 16 03 01 00 30 89 15 96 15 9d 93 |.........0......| +000002e0 e3 ae 94 14 f9 ea 39 d7 3b d6 98 e1 ed c8 0a 3f |......9.;......?| +000002f0 6f 2c a2 9b cd c5 ea 1a 1f 27 89 1a 7d ff 60 07 |o,.......'..}.`.| +00000300 22 1f bc b8 56 3a ee 24 5a ff |"...V:.$Z.| >>> Flow 4 (server to client) -00000000 14 03 01 00 01 01 16 03 01 00 30 92 36 25 46 a5 |..........0.6%F.| -00000010 44 e6 31 25 cd 24 15 df 13 f8 5b a3 af 7c 12 43 |D.1%.$....[..|.C| -00000020 b8 8c 39 84 bc 25 06 02 02 86 78 20 1b ec 98 1a |..9..%....x ....| -00000030 31 b3 b8 cf 82 92 77 ff 08 87 fb |1.....w....| +00000000 14 03 01 00 01 01 16 03 01 00 30 bf 0c 82 bd 43 |..........0....C| +00000010 ba 60 ec df 88 4d 48 be d5 c4 0c b5 7d c4 94 c4 |.`...MH.....}...| +00000020 15 6e 50 45 77 56 ce d5 e0 4c 15 fc da 96 0b 41 |.nPEwV...L.....A| +00000030 fd 70 39 e9 33 3f 57 77 f5 a3 67 |.p9.3?Ww..g| >>> Flow 5 (client to server) -00000000 17 03 01 00 20 2f bd ab e0 2f 9a 81 58 99 35 bb |.... /.../..X.5.| -00000010 86 61 6e 15 be 31 d7 ad 44 1d d9 cf 2f fc d9 f6 |.an..1..D.../...| -00000020 da b6 48 32 27 17 03 01 00 20 76 70 b7 7d 1b 05 |..H2'.... vp.}..| -00000030 ee 54 99 bf 89 79 79 b5 68 c1 84 3c 6d 47 5c d1 |.T...yy.h..>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 01 00 5d 02 00 00 59 03 01 6f 48 26 41 da |....]...Y..oH&A.| -00000010 73 82 d1 cf 84 ad 06 e9 76 39 dc e4 c8 34 af 53 |s.......v9...4.S| -00000020 06 ed ed 68 56 5f 76 11 4e f6 ff 20 55 fd 29 c5 |...hV_v.N.. U.).| -00000030 aa 56 af bd 5d 67 74 4f 08 8a 52 a1 7e cd b9 ac |.V..]gtO..R.~...| -00000040 31 a0 2f 45 25 1f b2 24 71 26 80 17 c0 09 00 00 |1./E%..$q&......| +00000000 16 03 01 00 5d 02 00 00 59 03 01 ef c9 5c 4d 29 |....]...Y....\M)| +00000010 07 24 2a 41 08 94 39 cc d3 fb 92 88 1c ff 64 6b |.$*A..9.......dk| +00000020 0a 14 41 89 c6 5d 9b 25 7e a7 04 20 a0 aa ad 46 |..A..].%~.. ...F| +00000030 14 01 d2 dd 37 44 05 4b 1d 9f ea e5 98 29 1e 36 |....7D.K.....).6| +00000040 09 e2 ab 90 93 ee c1 99 7d 17 77 9b c0 09 00 00 |........}.w.....| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 01 02 0e 0b 00 02 0a 00 02 07 00 02 |................| 00000070 04 30 82 02 00 30 82 01 62 02 09 00 b8 bf 2d 47 |.0...0..b.....-G| @@ -56,20 +57,20 @@ 00000240 8c 25 c1 33 13 83 0d 94 06 bb d4 37 7a f6 ec 7a |.%.3.......7z..z| 00000250 c9 86 2e dd d7 11 69 7f 85 7c 56 de fb 31 78 2b |......i..|V..1x+| 00000260 e4 c7 78 0d ae cb be 9e 4e 36 24 31 7b 6a 0f 39 |..x.....N6$1{j.9| -00000270 95 12 07 8f 2a 16 03 01 00 b3 0c 00 00 af 03 00 |....*...........| -00000280 1d 20 51 4c 6f eb 08 70 52 20 15 78 d3 f8 65 5e |. QLo..pR .x..e^| -00000290 0f 59 03 b7 d1 0e e9 7e be 42 d2 cd f3 57 c6 24 |.Y.....~.B...W.$| -000002a0 6b 2c 00 89 30 81 86 02 41 7b 1b ec 11 08 f4 8f |k,..0...A{......| -000002b0 db 2c c5 1b 33 b2 b0 b4 da 76 b9 a7 78 ed ee 0e |.,..3....v..x...| -000002c0 23 5c 4b b0 10 6e 41 12 b9 d4 68 c1 15 ea 70 0a |#\K..nA...h...p.| -000002d0 0c 91 c1 8d 4a 4d 41 c1 c3 25 f1 90 ef 95 f3 e1 |....JMA..%......| -000002e0 a8 30 4d ce 4d d1 eb 8c cf 9a 02 41 09 e0 8c 8e |.0M.M......A....| -000002f0 70 01 f1 5d b2 8a d1 61 bc 86 f4 85 2f 6c ac 2c |p..]...a..../l.,| -00000300 e8 bf 20 c7 81 46 21 6e 9b bd c1 83 71 34 8d 65 |.. ..F!n....q4.e| -00000310 fc 35 07 be c8 ad d4 3a c6 7b 40 1c f7 17 e3 d9 |.5.....:.{@.....| -00000320 96 19 d4 cd 0d d0 8c 4f b2 09 6a 4b 05 16 03 01 |.......O..jK....| -00000330 00 0a 0d 00 00 06 03 01 02 40 00 00 16 03 01 00 |.........@......| -00000340 04 0e 00 00 00 |.....| +00000270 95 12 07 8f 2a 16 03 01 00 b4 0c 00 00 b0 03 00 |....*...........| +00000280 1d 20 a4 4e 43 df 00 5c c7 1c e8 d4 8e 9b cf b0 |. .NC..\........| +00000290 36 85 64 7c d7 69 95 c0 b4 6f d8 0b 45 b8 a4 34 |6.d|.i...o..E..4| +000002a0 1a 18 00 8a 30 81 87 02 42 01 d0 4a 3f 65 9d 46 |....0...B..J?e.F| +000002b0 20 80 34 28 12 93 56 6e dc e4 0e 91 0b 45 4b 83 | .4(..Vn.....EK.| +000002c0 c5 e9 83 2c 41 d6 dc 49 15 15 e6 65 9f 18 ba a6 |...,A..I...e....| +000002d0 20 a6 de c7 20 7e 09 71 e6 59 86 9e aa 32 be 43 | ... ~.q.Y...2.C| +000002e0 b7 c3 27 98 ba 5b 49 9b 1d b9 67 02 41 4e 36 0e |..'..[I...g.AN6.| +000002f0 6d 29 c8 7d 0b d9 6f 06 92 ca 0b b9 33 7e 11 58 |m).}..o.....3~.X| +00000300 2f cc 06 ae ad 57 80 f4 38 a1 8a e3 6a ef 37 86 |/....W..8...j.7.| +00000310 58 1a 59 f9 4a 9a 64 89 5b 7c 8a 7a c5 78 dd b5 |X.Y.J.d.[|.z.x..| +00000320 6c 96 b8 23 ff fc 88 20 59 0b e9 74 99 b9 16 03 |l..#... Y..t....| +00000330 01 00 0a 0d 00 00 06 03 01 02 40 00 00 16 03 01 |..........@.....| +00000340 00 04 0e 00 00 00 |......| >>> Flow 3 (client to server) 00000000 16 03 01 01 fd 0b 00 01 f9 00 01 f6 00 01 f3 30 |...............0| 00000010 82 01 ef 30 82 01 58 a0 03 02 01 02 02 10 5c 19 |...0..X.......\.| @@ -106,29 +107,29 @@ 00000200 e5 35 16 03 01 00 25 10 00 00 21 20 2f e5 7d a3 |.5....%...! /.}.| 00000210 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 |G.bC.(.._.).0...| 00000220 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 16 03 01 00 |......._X.;t....| -00000230 86 0f 00 00 82 00 80 59 18 58 aa 18 70 84 a3 26 |.......Y.X..p..&| -00000240 8c 1d 77 52 a7 2d 00 e5 00 dc 24 1c f6 9b 9f 63 |..wR.-....$....c| -00000250 c8 84 8b dc e7 61 78 9c 28 0d 4c 3a 1e 5f a7 ce |.....ax.(.L:._..| -00000260 35 17 1c b1 de e0 01 32 35 6f 85 59 22 73 50 2a |5......25o.Y"sP*| -00000270 36 df 39 0c c8 c1 45 70 a2 e4 53 bb de e6 0e 6a |6.9...Ep..S....j| -00000280 fd 45 8e 4d eb 1f de 86 d3 fb 94 e4 8d 22 cd 30 |.E.M.........".0| -00000290 e7 29 de 65 f0 2a cd b8 3d b5 be 91 a3 7d 3b 30 |.).e.*..=....};0| -000002a0 1a a5 63 bd 84 1f 95 19 d1 fe 25 40 b8 8f ab 96 |..c.......%@....| -000002b0 59 42 e6 31 93 c0 39 14 03 01 00 01 01 16 03 01 |YB.1..9.........| -000002c0 00 30 a9 f7 c0 97 f4 b8 06 65 a6 7c fb 80 a1 79 |.0.......e.|...y| -000002d0 5f 44 72 55 13 d1 ca e3 9a 91 5d 1e 08 93 6d b5 |_DrU......]...m.| -000002e0 1c 5e 52 24 b1 ec db aa 3b bc fe db 18 0b 18 fe |.^R$....;.......| -000002f0 51 6a |Qj| +00000230 86 0f 00 00 82 00 80 a4 68 2d 1f 8a 97 43 76 aa |........h-...Cv.| +00000240 f9 24 95 20 62 13 c0 a3 45 c6 18 1c a3 34 70 02 |.$. b...E....4p.| +00000250 ff f5 01 4e ba e2 20 1c f9 06 a6 67 92 d9 e6 9d |...N.. ....g....| +00000260 a3 49 e0 75 3e 11 00 74 52 b1 36 58 4b 1e 54 83 |.I.u>..tR.6XK.T.| +00000270 e0 9a 48 4d df 2c ab fd cd 5e 7a cf c9 b8 32 08 |..HM.,...^z...2.| +00000280 74 e6 ae 75 20 f4 41 3a 7c a9 a3 19 38 a0 8d 05 |t..u .A:|...8...| +00000290 0a e9 3e 50 6c f6 f8 a3 89 a9 55 ea dc 3f be b1 |..>Pl.....U..?..| +000002a0 0a 92 83 cc f0 9b c9 e1 49 13 db 64 be 55 46 b5 |........I..d.UF.| +000002b0 12 b1 0b 88 32 e3 f1 14 03 01 00 01 01 16 03 01 |....2...........| +000002c0 00 30 81 77 0f 6c 7a bc a8 d2 41 f9 8b a7 da 96 |.0.w.lz...A.....| +000002d0 29 f1 2f b1 31 f3 57 03 09 21 5c fa dc f7 5c f6 |)./.1.W..!\...\.| +000002e0 7f a8 24 08 30 70 bb 34 16 22 f8 c6 b2 4d a7 16 |..$.0p.4."...M..| +000002f0 68 61 |ha| >>> Flow 4 (server to client) -00000000 14 03 01 00 01 01 16 03 01 00 30 e6 36 50 e7 a3 |..........0.6P..| -00000010 72 0a 6a 9f 18 17 91 51 ad 48 19 54 c8 6b c0 45 |r.j....Q.H.T.k.E| -00000020 d5 9b cb 08 34 d3 3a 73 74 04 3a 3d 24 b0 95 48 |....4.:st.:=$..H| -00000030 97 9e c5 83 4f 20 29 c8 dd 93 6c |....O )...l| +00000000 14 03 01 00 01 01 16 03 01 00 30 71 d2 ee cd f8 |..........0q....| +00000010 c5 fe b4 96 d5 02 ee cb f7 f8 93 34 f2 8a ed 71 |...........4...q| +00000020 9a b7 1f 01 9d fb 6c 3f ee 22 bb 5c b0 8c 08 f5 |......l?.".\....| +00000030 bf 1e d3 1c 12 ec 7b 86 05 bd e5 |......{....| >>> Flow 5 (client to server) -00000000 17 03 01 00 20 58 a1 0a 68 2c 04 68 7d e9 13 ce |.... X..h,.h}...| -00000010 69 4b 53 4a f6 fc e0 c8 0e 05 ad 19 d8 b3 37 67 |iKSJ..........7g| -00000020 f2 1f 3b 33 62 17 03 01 00 20 c2 e7 d3 d2 38 ba |..;3b.... ....8.| -00000030 27 13 ed d1 73 fc 9b f5 4f 61 f0 35 27 8e 54 52 |'...s...Oa.5'.TR| -00000040 b7 ce bd 6a 05 c6 61 95 1e 8a 15 03 01 00 20 6e |...j..a....... n| -00000050 07 21 86 31 f8 74 6b 45 4d 10 2f 5d e7 2b 64 77 |.!.1.tkEM./].+dw| -00000060 a9 7f 1e 49 ed d5 d4 1e 2e f2 b1 c5 4d 02 46 |...I........M.F| +00000000 17 03 01 00 20 8a 57 b3 89 76 41 f0 b3 51 da f4 |.... .W..vA..Q..| +00000010 e7 6a f8 46 75 77 4d 8b 67 41 f9 f9 eb a0 cd 12 |.j.FuwM.gA......| +00000020 78 08 12 d1 7b 17 03 01 00 20 9d 44 6a dd 48 ad |x...{.... .Dj.H.| +00000030 0a d9 3f 80 da b1 3d b3 50 be 40 c1 85 b5 bb 59 |..?...=.P.@....Y| +00000040 e8 b9 2a 9f f5 2e 98 d3 2b c1 15 03 01 00 20 bd |..*.....+..... .| +00000050 69 41 45 bb 53 de f8 b7 bf a5 87 12 02 32 1a 05 |iAE.S........2..| +00000060 09 94 40 a5 64 b3 31 7d 0d dc 01 ff 25 ca 31 |..@.d.1}....%.1| diff --git a/src/crypto/tls/testdata/Client-TLSv10-ClientCert-RSA-RSA b/src/crypto/tls/testdata/Client-TLSv10-ClientCert-RSA-RSA index 48eb1ce6419bfc..37d05fc481914f 100644 --- a/src/crypto/tls/testdata/Client-TLSv10-ClientCert-RSA-RSA +++ b/src/crypto/tls/testdata/Client-TLSv10-ClientCert-RSA-RSA @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 01 00 5d 02 00 00 59 03 01 24 71 03 c6 1d |....]...Y..$q...| -00000010 d4 f5 09 b2 3c d8 88 51 57 4c 90 07 4b 74 fe 1d |....<..QWL..Kt..| -00000020 c9 7f 00 78 9b a3 b2 1b ef 43 66 20 07 97 7c a6 |...x.....Cf ..|.| -00000030 9b c6 a9 47 42 ae 99 bf 59 fe 52 f6 da d9 1c 68 |...GB...Y.R....h| -00000040 c2 7a 53 3f 27 0d 11 87 9a 47 81 5c c0 13 00 00 |.zS?'....G.\....| +00000000 16 03 01 00 5d 02 00 00 59 03 01 71 32 68 84 f2 |....]...Y..q2h..| +00000010 93 91 32 5b 71 e6 d1 fd 26 83 00 87 0f 56 f3 8a |..2[q...&....V..| +00000020 70 17 5e c9 c3 b3 ce 61 9a 1d 4d 20 85 cc 39 26 |p.^....a..M ..9&| +00000030 32 1a 78 34 d5 d3 6c e2 df 89 f5 a0 51 2c f5 2d |2.x4..l.....Q,.-| +00000040 e0 17 51 e7 51 f4 61 8d 35 72 75 92 c0 13 00 00 |..Q.Q.a.5ru.....| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 01 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -61,17 +62,17 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 01 00 aa 0c 00 00 a6 03 00 1d 20 ad a7 d5 |............ ...| -000002d0 fc 63 e1 22 f8 1f 8d ad 50 09 1a 92 a4 57 ea 88 |.c."....P....W..| -000002e0 6d 9c 20 70 f1 8b 85 15 ec 8e b2 b3 07 00 80 b1 |m. p............| -000002f0 03 03 a7 cf 8f 73 e2 a3 29 0d 55 4a 06 72 db e5 |.....s..).UJ.r..| -00000300 65 41 44 49 05 1c 37 17 1c 02 6e b1 f4 e8 a9 bb |eADI..7...n.....| -00000310 00 24 3e 3a 3c d6 1c f6 dd 21 a4 e3 61 b0 1d 2e |.$>:<....!..a...| -00000320 77 11 53 51 4d 49 1e 07 bf 07 61 d1 54 f4 c7 24 |w.SQMI....a.T..$| -00000330 4f 7d 5c 5a fb 84 c0 9d 0e 52 12 b6 40 72 37 fd |O}\Z.....R..@r7.| -00000340 8c bd d0 8a b1 9e 2c 18 bc 04 a3 c2 90 a3 33 4c |......,.......3L| -00000350 06 7c 14 a8 73 7d d2 23 30 f9 31 12 56 31 60 19 |.|..s}.#0.1.V1`.| -00000360 d1 0d 31 c1 3c 25 66 62 5c 9b 13 5e 34 c6 69 16 |..1.<%fb\..^4.i.| +000002c0 16 03 01 00 aa 0c 00 00 a6 03 00 1d 20 3a 64 e7 |............ :d.| +000002d0 9a 59 c3 4e fc 40 5e 2f 5c 89 cd e1 94 85 4e 1f |.Y.N.@^/\.....N.| +000002e0 29 37 0b 53 fe 3a 13 76 56 25 2a 97 65 00 80 08 |)7.S.:.vV%*.e...| +000002f0 1c de 63 0d 31 9b 72 7b 85 0c 03 b0 08 ea 80 a1 |..c.1.r{........| +00000300 ee 00 03 f9 29 a3 ba 8e c8 71 3e b4 4d b3 28 54 |....)....q>.M.(T| +00000310 2c e7 11 3a 15 e0 43 06 f0 36 15 50 54 5e 88 48 |,..:..C..6.PT^.H| +00000320 ac c4 68 db 83 dc 0c 22 e4 99 4a 08 2a 00 7d 19 |..h...."..J.*.}.| +00000330 0d 74 ba 7a 27 9c 39 dc 29 41 52 cf a2 ac 29 94 |.t.z'.9.)AR...).| +00000340 e6 b0 87 60 e5 d3 58 af 3e 8e 41 bd be 48 ba 90 |...`..X.>.A..H..| +00000350 49 b2 b1 d3 8e b0 49 98 4a 12 70 60 c7 57 d9 a7 |I.....I.J.p`.W..| +00000360 db dc 41 b0 dc 81 37 1b 6d ac 9c 69 12 f4 fa 16 |..A...7.m..i....| 00000370 03 01 00 0a 0d 00 00 06 03 01 02 40 00 00 16 03 |...........@....| 00000380 01 00 04 0e 00 00 00 |.......| >>> Flow 3 (client to server) @@ -110,29 +111,29 @@ 00000200 e5 35 16 03 01 00 25 10 00 00 21 20 2f e5 7d a3 |.5....%...! /.}.| 00000210 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 |G.bC.(.._.).0...| 00000220 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 16 03 01 00 |......._X.;t....| -00000230 86 0f 00 00 82 00 80 59 e2 7d 00 66 f0 59 a0 19 |.......Y.}.f.Y..| -00000240 b6 4f 26 80 ad 31 b4 b3 c2 56 af 84 bf 88 15 7f |.O&..1...V......| -00000250 dc 29 fc 51 f2 cf 0d ec 52 46 c5 c5 cd 24 c1 93 |.).Q....RF...$..| -00000260 5b f6 06 09 ad bb 9a 4a ae 5d ec 57 f8 96 a8 b1 |[......J.].W....| -00000270 4b 81 45 7d 34 38 59 ea 76 47 76 1c 7f 7b 68 e9 |K.E}48Y.vGv..{h.| -00000280 89 7b 5d 4c b9 bc 5a 26 e1 f7 6c 42 f7 c6 6d bb |.{]L..Z&..lB..m.| -00000290 c0 c7 bd f6 5e 9e 51 d8 30 ec 9f da 56 0a b7 2a |....^.Q.0...V..*| -000002a0 6c 5e 50 d1 74 48 70 4a 88 62 16 a1 ba f2 01 e5 |l^P.tHpJ.b......| -000002b0 00 b6 eb cb e3 18 cf 14 03 01 00 01 01 16 03 01 |................| -000002c0 00 30 61 51 84 02 46 3c 10 62 b5 2a a2 8b 4e 2d |.0aQ..F<.b.*..N-| -000002d0 57 55 26 c5 46 f2 5d a1 f5 20 46 0d 63 fd 48 5a |WU&.F.].. F.c.HZ| -000002e0 f3 65 90 8c 1a bd 59 6e 5f d1 5c b6 ee 33 49 12 |.e....Yn_.\..3I.| -000002f0 46 ec |F.| +00000230 86 0f 00 00 82 00 80 21 34 0c d8 10 ac 90 53 f9 |.......!4.....S.| +00000240 18 52 42 05 ca e8 c7 4f 33 c4 43 4b 8e 7c e4 23 |.RB....O3.CK.|.#| +00000250 21 6d e4 07 ef 3f 06 d1 ea 1c 9d 3b e0 d2 66 36 |!m...?.....;..f6| +00000260 b5 c9 a1 da fe 54 fd e0 fe 0c b6 12 90 93 41 1f |.....T........A.| +00000270 43 00 00 e4 a4 04 14 af 00 3e 1b db 16 d6 07 4b |C........>.....K| +00000280 55 2f ed 55 e1 e1 a8 8b d8 e1 fe cb 41 1d fe bc |U/.U........A...| +00000290 6d d9 ba 8f 2b 1c 26 19 9d 93 a9 78 fb 8a 54 59 |m...+.&....x..TY| +000002a0 76 3b 0a df e6 71 2c c0 63 dd 22 8d c6 70 ef 0e |v;...q,.c."..p..| +000002b0 4f 1b 4c da 65 11 f6 14 03 01 00 01 01 16 03 01 |O.L.e...........| +000002c0 00 30 b5 c2 b0 f3 b6 6c 4a 99 de f2 98 2f 37 2b |.0.....lJ..../7+| +000002d0 8a d8 ab 96 91 c2 9b cc 56 5c fb e1 4f 5b 89 09 |........V\..O[..| +000002e0 5f 94 05 60 0e 83 b5 49 9a 15 9b 0f 5d 1a 2b a2 |_..`...I....].+.| +000002f0 11 23 |.#| >>> Flow 4 (server to client) -00000000 14 03 01 00 01 01 16 03 01 00 30 b1 3c 7f c3 15 |..........0.<...| -00000010 e4 fe bc 1a ee 0d ec 28 ce b0 f7 fd a4 0a e4 79 |.......(.......y| -00000020 37 09 68 de 58 f8 e8 76 7d 9c 09 35 42 7e 00 8e |7.h.X..v}..5B~..| -00000030 4c 8d d1 60 26 d4 09 c5 8b d0 96 |L..`&......| +00000000 14 03 01 00 01 01 16 03 01 00 30 63 a2 f0 9b 1c |..........0c....| +00000010 48 6d 23 54 62 b1 2a 19 e7 89 51 fd 0d 83 97 87 |Hm#Tb.*...Q.....| +00000020 9b 73 31 11 5e 42 56 62 62 37 4d e5 e6 72 8a 6d |.s1.^BVbb7M..r.m| +00000030 a3 02 2b 2c 9e a5 1a 5c 34 f2 0d |..+,...\4..| >>> Flow 5 (client to server) -00000000 17 03 01 00 20 6c bb 09 ee 1d 96 6c 63 5b b6 69 |.... l.....lc[.i| -00000010 bf ea ad 7a 85 36 a2 4b d2 f0 73 18 91 b1 42 e5 |...z.6.K..s...B.| -00000020 77 d0 6c ff ab 17 03 01 00 20 ed 85 78 3b bd 49 |w.l...... ..x;.I| -00000030 2c f3 02 24 f2 18 72 96 a0 49 6b 06 bd bb 54 25 |,..$..r..Ik...T%| -00000040 0a 3e ff 07 45 6b 6c cc a6 d2 15 03 01 00 20 07 |.>..Ekl....... .| -00000050 9a fb b7 cc 01 cd f7 b6 90 d3 b6 a7 31 15 99 e2 |............1...| -00000060 e7 e8 2b a5 c0 e8 a9 9d fe 4e 11 0d 4a d4 b7 |..+......N..J..| +00000000 17 03 01 00 20 63 1a 2c 25 27 68 ed de ba 94 52 |.... c.,%'h....R| +00000010 73 f2 7a 28 ed 8c e9 3f a2 48 9a 07 62 22 27 6d |s.z(...?.H..b"'m| +00000020 4e be ba e4 67 17 03 01 00 20 1d ee ac 7c fc a7 |N...g.... ...|..| +00000030 df 74 93 16 c6 ec 58 5e 04 5d 2e 98 0a ba 52 4f |.t....X^.]....RO| +00000040 a0 02 c5 79 c3 d6 f0 ab 8d 33 15 03 01 00 20 bb |...y.....3.... .| +00000050 54 a6 65 a7 c9 03 e7 83 ae a1 f3 26 9f 73 76 6b |T.e........&.svk| +00000060 e6 1e f7 e0 76 e5 ca 9f c8 87 14 ac 27 f1 e3 |....v.......'..| diff --git a/src/crypto/tls/testdata/Client-TLSv10-ECDHE-ECDSA-AES b/src/crypto/tls/testdata/Client-TLSv10-ECDHE-ECDSA-AES index ac354c50bc6bea..a8593e249b0467 100644 --- a/src/crypto/tls/testdata/Client-TLSv10-ECDHE-ECDSA-AES +++ b/src/crypto/tls/testdata/Client-TLSv10-ECDHE-ECDSA-AES @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 01 00 5d 02 00 00 59 03 01 70 d2 b8 0b 74 |....]...Y..p...t| -00000010 93 b4 60 d7 ab 1d 0f f3 7c d0 b5 f8 a2 78 45 30 |..`.....|....xE0| -00000020 b1 41 9b b6 73 bd 05 d8 85 16 66 20 e8 26 8d 7a |.A..s.....f .&.z| -00000030 e2 da e0 a0 7a c9 19 d1 dd db 2b bc b4 b2 56 39 |....z.....+...V9| -00000040 9f e6 62 75 29 96 40 b3 0d c0 db 65 c0 09 00 00 |..bu).@....e....| +00000000 16 03 01 00 5d 02 00 00 59 03 01 3e 61 ab b7 94 |....]...Y..>a...| +00000010 d9 7c 76 db 26 b6 fc 86 00 8f c4 af bd c6 1a 22 |.|v.&.........."| +00000020 dd 72 ce 5d 2a 4c d8 61 a1 20 6b 20 c9 82 ca c9 |.r.]*L.a. k ....| +00000030 99 59 80 37 6c 01 d2 b3 b5 0d 68 9f 65 b0 15 7d |.Y.7l.....h.e..}| +00000040 a6 b0 15 b0 49 5a ae 38 d2 77 5e 06 c0 09 00 00 |....IZ.8.w^.....| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 01 02 0e 0b 00 02 0a 00 02 07 00 02 |................| 00000070 04 30 82 02 00 30 82 01 62 02 09 00 b8 bf 2d 47 |.0...0..b.....-G| @@ -56,37 +57,37 @@ 00000240 8c 25 c1 33 13 83 0d 94 06 bb d4 37 7a f6 ec 7a |.%.3.......7z..z| 00000250 c9 86 2e dd d7 11 69 7f 85 7c 56 de fb 31 78 2b |......i..|V..1x+| 00000260 e4 c7 78 0d ae cb be 9e 4e 36 24 31 7b 6a 0f 39 |..x.....N6$1{j.9| -00000270 95 12 07 8f 2a 16 03 01 00 b4 0c 00 00 b0 03 00 |....*...........| -00000280 1d 20 5f 81 8f 18 31 4e 64 fe 87 f2 da c6 45 45 |. _...1Nd.....EE| -00000290 54 e3 54 c6 a0 bb bf bd d1 01 c5 7b 08 d8 32 6b |T.T........{..2k| -000002a0 a8 31 00 8a 30 81 87 02 42 01 21 7e c2 26 cb 5e |.1..0...B.!~.&.^| -000002b0 f1 2a d2 9b 7f 3f b4 d4 28 5e 63 5a 20 aa 28 87 |.*...?..(^cZ .(.| -000002c0 bb dd 5c 6a 91 a2 2d 03 7a 69 67 8b ca 84 a6 1f |..\j..-.zig.....| -000002d0 d3 58 40 eb 5c 54 95 96 05 d0 37 63 e4 a6 1b 51 |.X@.\T....7c...Q| -000002e0 9a d0 93 31 82 e6 8e a0 af 29 64 02 41 4c ff ac |...1.....)d.AL..| -000002f0 44 74 e0 04 2c ab 1c 23 91 dc 06 36 ff 19 a1 60 |Dt..,..#...6...`| -00000300 19 d4 e5 d3 78 f5 ed d8 09 7b de 61 6f 65 2c 69 |....x....{.aoe,i| -00000310 a1 d2 5d 07 bf 99 26 2b 70 fa 34 26 30 41 59 3b |..]...&+p.4&0AY;| -00000320 bc a3 10 9c 92 fa 60 a7 df ea e4 8a 3a 76 16 03 |......`.....:v..| -00000330 01 00 04 0e 00 00 00 |.......| +00000270 95 12 07 8f 2a 16 03 01 00 b5 0c 00 00 b1 03 00 |....*...........| +00000280 1d 20 73 36 95 70 b0 c6 fd c6 cf 68 f3 77 1e 46 |. s6.p.....h.w.F| +00000290 c2 22 3e f8 e6 f4 37 82 73 8e 8d ec 40 64 0d a9 |.">...7.s...@d..| +000002a0 f7 6f 00 8b 30 81 88 02 42 01 2f 8b 24 b7 24 65 |.o..0...B./.$.$e| +000002b0 6e f8 a3 55 53 3f da 67 a2 b6 35 b3 ef 8b 87 39 |n..US?.g..5....9| +000002c0 2f 44 e8 a6 9d 2c 16 c1 82 a9 a9 f6 20 0f 1b 36 |/D...,...... ..6| +000002d0 32 b8 7a 96 2d 5a b0 4d 43 53 ec c9 06 82 83 0e |2.z.-Z.MCS......| +000002e0 fb 0b 8c f8 0b 47 b6 dd 19 4c 96 02 42 01 c6 7e |.....G...L..B..~| +000002f0 20 9e d9 2f 33 1f 5f 25 bc 79 a3 df 96 9d e0 05 | ../3._%.y......| +00000300 d7 72 75 29 7d b3 f2 0a 5e 81 39 71 b7 f9 68 e8 |.ru)}...^.9q..h.| +00000310 82 07 93 80 88 31 77 2c db 8b 58 49 28 c5 7b c1 |.....1w,..XI(.{.| +00000320 e3 84 3c b9 08 2e a0 ab 66 12 b9 3c b9 a2 e8 16 |..<.....f..<....| +00000330 03 01 00 04 0e 00 00 00 |........| >>> Flow 3 (client to server) 00000000 16 03 01 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 01 00 01 01 |....._X.;t......| -00000030 16 03 01 00 30 6d 2a 91 d7 c3 8f 1c d8 d4 cc 67 |....0m*........g| -00000040 ed 07 41 c2 9c 48 0d bd 7f 11 61 00 58 23 55 88 |..A..H....a.X#U.| -00000050 f9 bd da 92 bc e4 8d 47 c0 ac 72 43 7f 2d 0f b5 |.......G..rC.-..| -00000060 de bb 67 6e 9f |..gn.| +00000030 16 03 01 00 30 9a 24 ee 81 ee 6e b3 fe ab 63 04 |....0.$...n...c.| +00000040 ae b1 1b 11 91 b0 cc 45 b8 67 74 9e 92 15 fd b1 |.......E.gt.....| +00000050 b4 49 49 b4 f4 a5 61 01 1b ec 91 23 ec c0 98 8d |.II...a....#....| +00000060 ee 21 fd 29 95 |.!.).| >>> Flow 4 (server to client) -00000000 14 03 01 00 01 01 16 03 01 00 30 be 25 a7 fa 04 |..........0.%...| -00000010 a5 dd 58 c1 5f a6 31 f8 fc fc db 27 a1 c7 0b 7c |..X._.1....'...|| -00000020 a5 9a 0f bd 07 09 d9 9b 95 bf f4 a5 80 c7 7d bd |..............}.| -00000030 a0 37 25 68 46 63 37 37 5f 3d b2 |.7%hFc77_=.| +00000000 14 03 01 00 01 01 16 03 01 00 30 74 d2 f1 68 1e |..........0t..h.| +00000010 1e 11 8a eb 51 ae a6 19 af 60 09 c4 85 65 3e 71 |....Q....`...e>q| +00000020 af d0 94 14 c3 80 18 91 72 23 97 26 f8 91 2f d6 |........r#.&../.| +00000030 65 3c 02 06 10 d0 bb e7 92 57 b0 |e<.......W.| >>> Flow 5 (client to server) -00000000 17 03 01 00 20 98 6e bd 0c 0e 46 88 05 6d 67 b3 |.... .n...F..mg.| -00000010 a7 92 7a 3c f2 98 8b ab 07 9e 80 bc 7a 5e 70 1a |..z<........z^p.| -00000020 2b 3d 2f 28 c9 17 03 01 00 20 f1 a8 ac 45 f6 60 |+=/(..... ...E.`| -00000030 ac 76 4e 66 1b a1 23 65 9c 55 6b 0b d4 cb 0a 43 |.vNf..#e.Uk....C| -00000040 42 d1 1e 1b 61 45 16 8e a8 c2 15 03 01 00 20 3d |B...aE........ =| -00000050 c0 ac 27 49 be 69 0e 23 cd 1b ea 01 0f 97 66 34 |..'I.i.#......f4| -00000060 f8 bd a6 40 35 af 5b 1f cf 42 25 54 aa 2f 2e |...@5.[..B%T./.| +00000000 17 03 01 00 20 bf f3 92 9c 91 50 02 1e d2 29 17 |.... .....P...).| +00000010 2c 5d 5f 43 c9 de 49 db 7b 0b bf eb 77 c5 9a 37 |,]_C..I.{...w..7| +00000020 c7 e1 c3 83 a3 17 03 01 00 20 df 80 0b 7e 80 39 |......... ...~.9| +00000030 b3 46 d8 7c 09 7a a8 c1 3a 04 77 2f 94 30 eb 8e |.F.|.z..:.w/.0..| +00000040 5d 56 08 95 bb 50 80 76 4b e7 15 03 01 00 20 0e |]V...P.vK..... .| +00000050 91 ba 25 bb d6 f6 ee 42 e0 8e 08 a1 7f d5 8f e3 |..%....B........| +00000060 18 05 85 24 b6 0c 81 80 98 89 ae 2c 04 0e 8a |...$.......,...| diff --git a/src/crypto/tls/testdata/Client-TLSv10-ECDHE-RSA-AES b/src/crypto/tls/testdata/Client-TLSv10-ECDHE-RSA-AES index 694cd4ce89273d..b73fd761c46d7e 100644 --- a/src/crypto/tls/testdata/Client-TLSv10-ECDHE-RSA-AES +++ b/src/crypto/tls/testdata/Client-TLSv10-ECDHE-RSA-AES @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 01 00 5d 02 00 00 59 03 01 5a a9 a1 6f fa |....]...Y..Z..o.| -00000010 19 95 38 9e 68 42 d7 0f 73 23 09 3f 01 5f e8 ae |..8.hB..s#.?._..| -00000020 d7 06 a2 9b d3 01 0f 0d 23 53 7d 20 55 2b 67 4e |........#S} U+gN| -00000030 8b 44 94 a5 25 ef b7 fb e1 c7 85 e8 15 60 f9 d7 |.D..%........`..| -00000040 6d a4 3f 3d 42 d7 78 a6 41 49 b7 12 c0 13 00 00 |m.?=B.x.AI......| +00000000 16 03 01 00 5d 02 00 00 59 03 01 79 f2 d6 96 d6 |....]...Y..y....| +00000010 6d c9 a1 7a 04 ba 6d 7d 29 8d 91 3f 8e 2e 17 0f |m..z..m})..?....| +00000020 c8 c4 3a e1 3c 64 00 28 f8 21 9d 20 16 7b 80 30 |..:...| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 01 00 aa 0c 00 00 a6 03 00 1d 20 a7 ec f4 |............ ...| -000002d0 c7 51 5e a0 fb 55 5a 01 34 cc d1 a6 22 df d6 00 |.Q^..UZ.4..."...| -000002e0 f5 73 e0 9e b8 9d 2a a9 49 3a 99 ba 17 00 80 14 |.s....*.I:......| -000002f0 95 47 77 a1 48 b4 e7 ba 6a e8 0e 4e ee 55 25 f5 |.Gw.H...j..N.U%.| -00000300 ea 82 65 b4 3f db a2 94 03 cc d3 6c 4a e6 cf 57 |..e.?......lJ..W| -00000310 25 4c 79 b8 fb ab dd 1f 68 e4 55 87 b1 46 12 30 |%Ly.....h.U..F.0| -00000320 5e 9f 3d 4a 52 24 76 50 c8 54 83 a2 b9 f5 a6 e0 |^.=JR$vP.T......| -00000330 ec e1 fb ae dd ca d2 c9 b2 31 d5 d4 73 f4 c0 0a |.........1..s...| -00000340 29 1c d3 8e 87 4c b4 0a 37 e8 77 e4 e2 a7 e9 9c |)....L..7.w.....| -00000350 04 ba af 31 dc 2e 5a b3 2e f3 58 e6 21 0a e2 1c |...1..Z...X.!...| -00000360 d4 c6 be b1 70 83 22 05 f8 78 12 6e bd 53 22 16 |....p."..x.n.S".| +000002c0 16 03 01 00 aa 0c 00 00 a6 03 00 1d 20 69 9e db |............ i..| +000002d0 5f 4b 5e 7d 1e 00 1e 91 a6 49 81 e3 d9 ee ea 5e |_K^}.....I.....^| +000002e0 5c 40 f0 68 fd dd eb 6e e4 85 58 91 70 00 80 8e |\@.h...n..X.p...| +000002f0 d6 64 01 3a 56 c2 58 5c 60 28 bc f6 bd 1e bf 73 |.d.:V.X\`(.....s| +00000300 21 b8 1a ea fb c2 df d5 f1 b9 4d d7 6f 1c 8b 24 |!.........M.o..$| +00000310 99 35 a5 ef 20 75 00 3e 83 34 da 40 4e ec e3 43 |.5.. u.>.4.@N..C| +00000320 04 a4 f2 1e bb 97 23 3e 21 32 88 43 80 99 ec 43 |......#>!2.C...C| +00000330 66 c3 09 87 1e e2 ad bb c3 1c db f7 9a 59 3a a8 |f............Y:.| +00000340 46 43 b6 3d 9a 6e c3 42 5b 1a 7d 85 dc db 96 cc |FC.=.n.B[.}.....| +00000350 8d 06 cc 08 d2 f0 09 c3 c6 ed 5f 3b f6 b0 a3 69 |.........._;...i| +00000360 6c a8 9a a5 ef ad 59 c1 07 96 e8 34 1d f0 d9 16 |l.....Y....4....| 00000370 03 01 00 04 0e 00 00 00 |........| >>> Flow 3 (client to server) 00000000 16 03 01 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 01 00 01 01 |....._X.;t......| -00000030 16 03 01 00 30 0f 56 a4 2c c6 11 a2 0a a8 f7 ff |....0.V.,.......| -00000040 6e 71 19 e9 e5 c0 e2 6b 86 9b b1 5d 46 8d 62 93 |nq.....k...]F.b.| -00000050 69 7c 28 6f 25 3d dc ce fa 6f 91 9e b8 8c 15 cd |i|(o%=...o......| -00000060 6a b1 72 ce a8 |j.r..| +00000030 16 03 01 00 30 d1 17 2f 22 e5 fa 78 a0 b3 d3 22 |....0../"..x..."| +00000040 ab cb d2 54 52 e6 b2 74 11 bd 3c 69 9a 71 e8 69 |...TR..t..>> Flow 4 (server to client) -00000000 14 03 01 00 01 01 16 03 01 00 30 d0 aa 00 27 9e |..........0...'.| -00000010 fb 56 22 9f 65 ab 3c cf 34 d9 a0 92 08 62 f2 fe |.V".e.<.4....b..| -00000020 29 4b 0c 14 b0 72 f5 c0 f8 2b 7e 1f 38 ad 68 7e |)K...r...+~.8.h~| -00000030 37 c3 fb 26 5a 5b 42 39 83 33 c0 |7..&Z[B9.3.| +00000000 14 03 01 00 01 01 16 03 01 00 30 d1 2e 43 5c f7 |..........0..C\.| +00000010 fc 5f 96 b7 e6 84 60 08 aa b3 22 d1 d9 e2 f3 4d |._....`..."....M| +00000020 c8 44 62 9b fa 0e 18 7c 84 fe 1e 75 09 4a f4 94 |.Db....|...u.J..| +00000030 67 03 91 79 90 26 70 2d 0e 97 6a |g..y.&p-..j| >>> Flow 5 (client to server) -00000000 17 03 01 00 20 0e 54 7f 5d b9 f2 73 9d a5 49 25 |.... .T.]..s..I%| -00000010 c6 b9 03 1e fb 84 76 a9 69 ca 25 79 c3 a4 60 18 |......v.i.%y..`.| -00000020 94 ff 92 47 4f 17 03 01 00 20 cd 19 d2 cd 7d 7d |...GO.... ....}}| -00000030 3b a1 65 ee 4e a2 f2 02 55 02 28 54 28 92 c9 ec |;.e.N...U.(T(...| -00000040 db 59 f6 7d 2f 9e 7b 66 0e be 15 03 01 00 20 4f |.Y.}/.{f...... O| -00000050 ab cd 12 5b c4 f4 03 0e 57 03 3d f5 4e 09 8b 83 |...[....W.=.N...| -00000060 8b c6 9c c7 eb 30 8c f2 43 f6 10 30 14 80 c5 |.....0..C..0...| +00000000 17 03 01 00 20 c9 ce 68 cc e9 a7 f7 1f 3c 19 76 |.... ..h.....<.v| +00000010 4e a6 2d 64 86 4d 35 9b 5e 5b 8b c7 a1 15 d9 d4 |N.-d.M5.^[......| +00000020 5e 59 67 df 4d 17 03 01 00 20 26 09 76 e2 16 41 |^Yg.M.... &.v..A| +00000030 a3 7f 3e 8e cc 75 ab a0 cf e1 42 8d 3c 51 d4 bb |..>..u....B.>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 01 00 5d 02 00 00 59 03 01 6f 36 7b f5 f4 |....]...Y..o6{..| -00000010 3b 49 31 64 aa 42 a7 8c a2 4d 00 a7 9c b9 b6 9c |;I1d.B...M......| -00000020 0d 2f 09 81 e8 ab 74 77 11 fe 0b 20 e0 b9 92 4d |./....tw... ...M| -00000030 e2 70 fd 58 ec 58 02 f2 b3 44 d2 3d 7f c9 ad ca |.p.X.X...D.=....| -00000040 5c 78 fe f9 36 67 a8 b7 a8 0f 7a 1a c0 13 00 00 |\x..6g....z.....| +00000000 16 03 01 00 5d 02 00 00 59 03 01 97 c9 b8 e6 c1 |....]...Y.......| +00000010 0e b3 68 29 30 ed ff df a6 f3 cd b7 e0 80 a9 76 |..h)0..........v| +00000020 84 1b 45 bb 42 29 e0 c8 7f 52 4b 20 95 97 03 66 |..E.B)...RK ...f| +00000030 6b 45 44 7c 5d 4b 9e 47 fc 72 53 5d 3c 8a 28 6e |kED|]K.G.rS]<.(n| +00000040 bc 40 6d c3 96 67 f9 f2 72 ec 6d 0d c0 13 00 00 |.@m..g..r.m.....| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 01 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -61,36 +62,36 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 01 00 aa 0c 00 00 a6 03 00 1d 20 10 69 5f |............ .i_| -000002d0 db e5 a8 1f 14 81 0c c7 35 d4 3c 22 ee 17 a3 21 |........5.<"...!| -000002e0 47 9b e2 3c 5a f6 f5 46 b9 73 2a 7f 62 00 80 49 |G..ukv.........Mf| -00000340 57 70 1a 4c 21 27 7f c8 69 f4 f4 83 5c 44 32 b4 |Wp.L!'..i...\D2.| -00000350 73 18 a3 c4 4a c1 71 92 04 7a 49 49 15 25 05 a0 |s...J.q..zII.%..| -00000360 61 69 f8 38 4b 9f 92 96 f4 6b 6b 4b 96 60 45 16 |ai.8K....kkK.`E.| +000002c0 16 03 01 00 aa 0c 00 00 a6 03 00 1d 20 15 0b 17 |............ ...| +000002d0 fd fb 69 3e 44 71 3c 68 b2 6a 99 9e 8c 57 ba e8 |..i>Dq.$.`....9...Z| +00000300 12 2c c9 3e a7 db 25 ab 32 32 e4 79 9e 0a 9f 98 |.,.>..%.22.y....| +00000310 99 cd be b3 28 34 40 e1 78 3e 3d 20 35 74 79 7f |....(4@.x>= 5ty.| +00000320 f6 c5 5d 4c 54 30 1d 64 31 49 78 bd a2 cb 62 5a |..]LT0.d1Ix...bZ| +00000330 89 1a bf 65 bf 7f 1c ff 7d 61 6c 7d d2 76 8c 9e |...e....}al}.v..| +00000340 e4 80 56 7d 96 79 48 36 ca c0 99 db 9b ea 3a e7 |..V}.yH6......:.| +00000350 a7 fe cb ed d8 3b 34 8c be d5 ee be 59 d5 e2 5f |.....;4.....Y.._| +00000360 59 17 f3 57 29 eb c8 0e ed 1a 06 79 c0 3c 16 16 |Y..W)......y.<..| 00000370 03 01 00 04 0e 00 00 00 |........| >>> Flow 3 (client to server) 00000000 16 03 01 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 01 00 01 01 |....._X.;t......| -00000030 16 03 01 00 30 e2 8b e2 aa 5a 35 c5 e5 68 ce f4 |....0....Z5..h..| -00000040 fa 51 3f 7b 3e 99 2f 21 b2 c6 b2 70 74 af ae ca |.Q?{>./!...pt...| -00000050 36 86 78 08 ea 8f 02 29 e6 a6 71 fa ad 7f c6 97 |6.x....)..q.....| -00000060 54 6f 88 a3 4b |To..K| +00000030 16 03 01 00 30 c6 d7 ad 2c 52 90 50 3c 50 ec 66 |....0...,R.P>> Flow 4 (server to client) -00000000 14 03 01 00 01 01 16 03 01 00 30 af 26 01 47 8a |..........0.&.G.| -00000010 a3 08 b2 79 1c fe fa 91 e8 4b a6 e1 ad 8e 59 b7 |...y.....K....Y.| -00000020 ea 41 ee 3e 3a aa 2f f2 07 9e 0d 90 3d 6f 92 d5 |.A.>:./.....=o..| -00000030 72 92 ad ae ee c8 5d 48 a0 25 46 |r.....]H.%F| +00000000 14 03 01 00 01 01 16 03 01 00 30 8c a1 03 f1 fb |..........0.....| +00000010 fe f8 1b 43 7b fb 10 59 c3 ed b3 34 b9 74 e1 89 |...C{..Y...4.t..| +00000020 7f 5c 9b 81 b1 4f 13 bf 67 c5 87 92 31 96 7f e7 |.\...O..g...1...| +00000030 35 7b b0 da 2c 79 2e be 43 f4 cf |5{..,y..C..| >>> Flow 5 (client to server) -00000000 17 03 01 00 20 29 2b 6a 75 2b 1b 94 f4 cb 80 14 |.... )+ju+......| -00000010 16 91 ec 08 77 5a 08 45 e7 51 2e a2 ec 03 f0 ca |....wZ.E.Q......| -00000020 35 88 20 3a 2e 17 03 01 00 20 59 0c 48 6f 66 8e |5. :..... Y.Hof.| -00000030 88 62 90 cc 33 39 62 ce 96 28 d0 2f 40 6e ed 14 |.b..39b..(./@n..| -00000040 94 72 88 d8 4a cb 73 24 a9 01 15 03 01 00 20 5c |.r..J.s$...... \| -00000050 8a 91 45 ed c9 38 e7 1d 71 02 b8 30 bf 24 0e 30 |..E..8..q..0.$.0| -00000060 6d fa 0a 4a 39 9f 56 b2 04 99 3e 67 bd bc d9 |m..J9.V...>g...| +00000000 17 03 01 00 20 61 e8 5a b2 15 88 f2 f9 e2 09 61 |.... a.Z.......a| +00000010 53 a5 7f 79 4b e8 c7 8d a9 5d 29 bf b9 0a 1d f7 |S..yK....]).....| +00000020 90 9f 3b 89 39 17 03 01 00 20 86 93 3b 08 7f 77 |..;.9.... ..;..w| +00000030 55 e5 51 06 ca 0e 80 96 9d 40 73 55 45 18 28 c7 |U.Q......@sUE.(.| +00000040 54 6a d8 15 d7 67 1e 4b 52 1e 15 03 01 00 20 56 |Tj...g.KR..... V| +00000050 d8 23 be 5a 8a a8 a4 8f 13 2f f0 34 24 90 96 87 |.#.Z...../.4$...| +00000060 59 48 3f 76 77 47 d5 eb 4d cb 80 00 bc 11 cf |YH?vwG..M......| diff --git a/src/crypto/tls/testdata/Client-TLSv10-RSA-RC4 b/src/crypto/tls/testdata/Client-TLSv10-RSA-RC4 index 8fce559c72b8b7..c217b1112ac1ff 100644 --- a/src/crypto/tls/testdata/Client-TLSv10-RSA-RC4 +++ b/src/crypto/tls/testdata/Client-TLSv10-RSA-RC4 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 01 00 55 02 00 00 51 03 01 9b 6a cd e3 08 |....U...Q...j...| -00000010 29 e7 32 d1 53 14 f5 4a cf b7 61 1a 90 d6 9a 73 |).2.S..J..a....s| -00000020 ad 48 2a 08 ce d7 d7 cb f8 25 b1 20 c1 24 61 85 |.H*......%. .$a.| -00000030 31 d2 81 5e 8e c8 31 40 56 36 79 af bb f7 90 d3 |1..^..1@V6y.....| -00000040 06 5e 4b 5a 62 50 c0 d5 89 7f 35 70 00 05 00 00 |.^KZbP....5p....| +00000000 16 03 01 00 55 02 00 00 51 03 01 6a dd d7 0d bb |....U...Q..j....| +00000010 bd b4 9c de 87 94 32 27 fa 4b 66 e0 8b 95 f2 11 |......2'.Kf.....| +00000020 a0 a5 30 15 34 6f 76 6b f7 23 ec 20 ef 7d 52 7d |..0.4ovk.#. .}R}| +00000030 2c 3b 30 1b f2 16 e7 8f b6 62 64 79 51 5b 31 36 |,;0......bdyQ[16| +00000040 b7 59 b1 f9 d5 26 d6 21 94 ff 7f bd 00 05 00 00 |.Y...&.!........| 00000050 09 ff 01 00 01 00 00 17 00 00 16 03 01 02 59 0b |..............Y.| 00000060 00 02 55 00 02 52 00 02 4f 30 82 02 4b 30 82 01 |..U..R..O0..K0..| 00000070 b4 a0 03 02 01 02 02 09 00 e8 f0 9d 3f e2 5b ea |............?.[.| @@ -72,15 +73,15 @@ 00000060 c5 70 0f 08 83 48 e9 48 ef 6e 50 8b 05 7e e5 84 |.p...H.H.nP..~..| 00000070 25 fa 55 c7 ae 31 02 27 00 ef 3f 98 86 20 12 89 |%.U..1.'..?.. ..| 00000080 91 59 28 b4 f7 d7 af d2 69 61 35 14 03 01 00 01 |.Y(.....ia5.....| -00000090 01 16 03 01 00 24 b3 7b e6 d1 8f 79 04 e2 42 0f |.....$.{...y..B.| -000000a0 61 de d2 be 1e 4d 8f d4 6f 7e 2d 59 cf 23 ff 1e |a....M..o~-Y.#..| -000000b0 74 ea 2a 9c 7e e2 5b 23 05 2e |t.*.~.[#..| +00000090 01 16 03 01 00 24 0e 49 42 d7 a8 ca 08 09 a6 63 |.....$.IB......c| +000000a0 0f b1 4b 06 30 37 5e cb 3a c8 d6 ce f9 9c bf 2f |..K.07^.:....../| +000000b0 4a c1 c7 fb 2e 02 a6 b0 de ed |J.........| >>> Flow 4 (server to client) -00000000 14 03 01 00 01 01 16 03 01 00 24 3c 4e 68 82 8f |..........$>> Flow 5 (client to server) -00000000 17 03 01 00 1a e0 a4 39 3c 9c 5c 8e 60 db 8e 5c |.......9<.\.`..\| -00000010 13 aa 60 4a 1c f0 71 1b f2 4d a9 f8 52 07 bd 15 |..`J..q..M..R...| -00000020 03 01 00 16 9e f3 0a 66 42 77 5a 75 74 18 85 f1 |.......fBwZut...| -00000030 3c fb ea d4 b8 76 6d eb 48 3d |<....vm.H=| +00000000 17 03 01 00 1a d3 71 0b 8e 0d d4 e0 06 04 e2 30 |......q........0| +00000010 59 2c fe 84 81 45 1c e4 59 90 b1 b1 11 85 cb 15 |Y,...E..Y.......| +00000020 03 01 00 16 ad 5d 98 96 4e 9d 83 af b0 50 64 77 |.....]..N....Pdw| +00000030 62 a1 2b 1a 63 59 16 9e 60 da |b.+.cY..`.| diff --git a/src/crypto/tls/testdata/Client-TLSv11-ECDHE-ECDSA-AES b/src/crypto/tls/testdata/Client-TLSv11-ECDHE-ECDSA-AES index 0ad2db05758870..dbaefe8eb36ee7 100644 --- a/src/crypto/tls/testdata/Client-TLSv11-ECDHE-ECDSA-AES +++ b/src/crypto/tls/testdata/Client-TLSv11-ECDHE-ECDSA-AES @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 02 00 5d 02 00 00 59 03 02 51 70 ac 24 75 |....]...Y..Qp.$u| -00000010 53 94 02 c6 20 8e 22 0d b2 79 30 3b ee a1 00 44 |S... ."..y0;...D| -00000020 f3 1c b8 a3 4c 51 c4 d9 30 95 d4 20 ca 82 67 4f |....LQ..0.. ..gO| -00000030 8c 13 5f c2 7e fc e0 0c 04 5c e2 9a c2 a3 02 a1 |.._.~....\......| -00000040 bc fa ca 16 38 83 2e 5a c1 75 19 67 c0 09 00 00 |....8..Z.u.g....| +00000000 16 03 02 00 5d 02 00 00 59 03 02 14 c4 6e 18 e9 |....]...Y....n..| +00000010 5b 74 0d 7f 05 01 75 e3 a3 af 90 f0 ea e3 a8 f0 |[t....u.........| +00000020 6e b5 7d 29 97 4d f3 7e e9 06 20 20 ba 37 13 9f |n.}).M.~.. .7..| +00000030 4e dd 10 6d 52 96 14 d0 93 3b 99 5b c2 cd f3 9c |N..mR....;.[....| +00000040 48 c1 12 78 c2 e5 e7 9d a1 d6 b4 da c0 09 00 00 |H..x............| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 02 02 0e 0b 00 02 0a 00 02 07 00 02 |................| 00000070 04 30 82 02 00 30 82 01 62 02 09 00 b8 bf 2d 47 |.0...0..b.....-G| @@ -57,38 +58,38 @@ 00000250 c9 86 2e dd d7 11 69 7f 85 7c 56 de fb 31 78 2b |......i..|V..1x+| 00000260 e4 c7 78 0d ae cb be 9e 4e 36 24 31 7b 6a 0f 39 |..x.....N6$1{j.9| 00000270 95 12 07 8f 2a 16 03 02 00 b5 0c 00 00 b1 03 00 |....*...........| -00000280 1d 20 4a 31 cd 48 ea 05 09 8a 6d 78 c5 46 5d ea |. J1.H....mx.F].| -00000290 fb aa 62 79 17 70 31 5e 5a 95 c1 ce a9 52 3e 8e |..by.p1^Z....R>.| -000002a0 a1 7b 00 8b 30 81 88 02 42 01 e1 c6 99 40 d7 9f |.{..0...B....@..| -000002b0 15 41 f4 4f 54 5f 5d 14 bd a6 cc 18 6e 67 98 ab |.A.OT_].....ng..| -000002c0 6a 33 9d 54 68 94 2a 89 22 66 2d 20 97 e7 53 c9 |j3.Th.*."f- ..S.| -000002d0 f8 ee ad 64 46 6e 53 1d d6 57 55 6a 4c bc 37 1f |...dFnS..WUjL.7.| -000002e0 e7 9c c4 bb 23 cb af 27 2a 19 94 02 42 01 35 b4 |....#..'*...B.5.| -000002f0 a1 f0 c2 6d 34 f0 05 a7 25 9a 22 6c 3e 41 e7 a0 |...m4...%."l>A..| -00000300 1e 8f 5a 28 b1 5a 88 46 48 7e 40 93 f2 38 9d 33 |..Z(.Z.FH~@..8.3| -00000310 6b b5 92 ed 6d 44 07 03 c8 77 ee 3a 80 37 1e f9 |k...mD...w.:.7..| -00000320 4d 5c 2b 6d c5 15 07 b2 7c eb c5 f5 c5 14 dd 16 |M\+m....|.......| +00000280 1d 20 d1 90 13 d3 6d b1 e1 ec a3 e1 8b a1 d6 a6 |. ....m.........| +00000290 40 ac 8e cf 6e 42 06 7d a8 80 9a 9e a8 06 00 84 |@...nB.}........| +000002a0 69 1f 00 8b 30 81 88 02 42 01 b6 ba 66 1c de c4 |i...0...B...f...| +000002b0 8b a1 4a a7 0e f5 cb aa 5c 76 65 59 ba bb e2 7f |..J.....\veY....| +000002c0 a5 ee 91 26 77 40 e9 5d 25 73 5a d6 f9 b4 aa ac |...&w@.]%sZ.....| +000002d0 c8 bd 2b a4 95 b3 ef e0 c7 bb f1 d0 5e da 76 34 |..+.........^.v4| +000002e0 a4 34 5b 6c d6 c8 3b 13 b1 d0 12 02 42 01 e1 18 |.4[l..;.....B...| +000002f0 d5 90 79 c5 06 00 6b 7a 86 19 e0 2f 67 49 db 9e |..y...kz.../gI..| +00000300 4a 74 07 30 51 15 7b a1 01 89 9c 94 d8 17 18 a6 |Jt.0Q.{.........| +00000310 31 aa fb 4b 57 24 52 00 41 1d cc 36 89 6e b4 ed |1..KW$R.A..6.n..| +00000320 fc 23 33 63 dd 94 3b 40 6f f1 af d7 78 27 0c 16 |.#3c..;@o...x'..| 00000330 03 02 00 04 0e 00 00 00 |........| >>> Flow 3 (client to server) 00000000 16 03 02 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 02 00 01 01 |....._X.;t......| 00000030 16 03 02 00 40 00 00 00 00 00 00 00 00 00 00 00 |....@...........| -00000040 00 00 00 00 00 40 aa 15 c7 e0 2e f6 9e e8 aa 92 |.....@..........| -00000050 81 6d 83 8f 13 7e 7c d5 9f b9 bf ca 5d da fd 3c |.m...~|.....]..<| -00000060 ba c7 8e 8e 9a 93 86 21 49 15 04 9d b8 75 c1 fe |.......!I....u..| -00000070 5b a6 41 d5 46 |[.A.F| +00000040 00 00 00 00 00 bd f5 0b 25 31 b7 e6 3f 74 dc 39 |........%1..?t.9| +00000050 c8 a4 f5 3f 78 f5 6f 66 1b 59 53 51 40 d2 e1 14 |...?x.of.YSQ@...| +00000060 56 96 ea 1f 08 30 3f 66 ca e4 e4 a0 b5 9f 60 9b |V....0?f......`.| +00000070 f8 b0 c7 b8 79 |....y| >>> Flow 4 (server to client) -00000000 14 03 02 00 01 01 16 03 02 00 40 ba a7 6d aa 40 |..........@..m.@| -00000010 b5 94 f8 d6 84 38 7c 19 7f b9 e9 8b 0f a8 db 4c |.....8|........L| -00000020 68 cf 1e d4 e0 b2 7c 78 80 76 18 c7 a5 cf 57 2d |h.....|x.v....W-| -00000030 1e 3d 17 b2 ce 44 93 76 38 91 4b d0 7f 09 00 2f |.=...D.v8.K..../| -00000040 c2 da c7 32 70 de fd cb a4 99 90 |...2p......| +00000000 14 03 02 00 01 01 16 03 02 00 40 68 2d 60 78 1b |..........@h-`x.| +00000010 a7 b8 28 3a 2b f5 cd b8 ef 4e 8e ab 3c d5 67 7b |..(:+....N..<.g{| +00000020 1f 29 9e 59 de ca 43 3a 4c 0c f5 70 43 cb 0b 40 |.).Y..C:L..pC..@| +00000030 69 a3 7c f7 1a 3b 48 2f b8 a2 7c b4 4a 03 36 2a |i.|..;H/..|.J.6*| +00000040 24 c9 78 9a 06 6e 18 2e 4a c0 54 |$.x..n..J.T| >>> Flow 5 (client to server) 00000000 17 03 02 00 30 00 00 00 00 00 00 00 00 00 00 00 |....0...........| -00000010 00 00 00 00 00 86 40 c2 e7 d4 56 43 6b e0 e4 96 |......@...VCk...| -00000020 41 74 2e b1 65 cd c6 93 06 71 f0 25 20 21 c5 1b |At..e....q.% !..| -00000030 d2 ea eb e4 06 15 03 02 00 30 00 00 00 00 00 00 |.........0......| -00000040 00 00 00 00 00 00 00 00 00 00 1e a9 c0 dc d8 23 |...............#| -00000050 f2 08 b0 c5 b2 2e 53 9b 23 50 8a 02 c6 41 9a 36 |......S.#P...A.6| -00000060 59 b1 23 d9 5e 91 45 cf 1b a1 |Y.#.^.E...| +00000010 00 00 00 00 00 f6 ff 38 da 12 09 78 fe de fb 0f |.......8...x....| +00000020 ea d3 5a d8 57 65 73 78 41 2d 0b 1b a4 8d f6 e7 |..Z.WesxA-......| +00000030 ed 58 97 c9 ea 15 03 02 00 30 00 00 00 00 00 00 |.X.......0......| +00000040 00 00 00 00 00 00 00 00 00 00 7d 4a 10 43 93 8a |..........}J.C..| +00000050 6d d2 50 34 92 22 24 f6 7a 9a c4 1f 57 e8 6d f8 |m.P4."$.z...W.m.| +00000060 74 c4 cb 8b e0 4d 99 dd ce 0a |t....M....| diff --git a/src/crypto/tls/testdata/Client-TLSv11-ECDHE-RSA-AES b/src/crypto/tls/testdata/Client-TLSv11-ECDHE-RSA-AES index 83c7d77102fd2a..d0093da76c9ec5 100644 --- a/src/crypto/tls/testdata/Client-TLSv11-ECDHE-RSA-AES +++ b/src/crypto/tls/testdata/Client-TLSv11-ECDHE-RSA-AES @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 02 00 5d 02 00 00 59 03 02 79 2f de f0 f3 |....]...Y..y/...| -00000010 97 6a 5e e3 07 66 d9 f2 8b e9 be 1a 97 9f 6d c8 |.j^..f........m.| -00000020 9d b5 88 57 fc 8c 5d 7f d7 3e 34 20 e5 4c 51 33 |...W..]..>4 .LQ3| -00000030 52 23 a0 52 77 ca 41 4a 1b e4 22 fb 1c 82 f8 e4 |R#.Rw.AJ..".....| -00000040 aa 13 c2 1a ce cd d5 42 de b0 e9 10 c0 13 00 00 |.......B........| +00000000 16 03 02 00 5d 02 00 00 59 03 02 3d 12 00 a1 c0 |....]...Y..=....| +00000010 6f b2 13 96 d8 c3 b6 4e 81 60 03 60 fa 9a 4b 54 |o......N.`.`..KT| +00000020 a9 1d e3 e9 10 e6 8d 84 e3 af 76 20 7c 6d 5c 41 |..........v |m\A| +00000030 f6 19 49 92 b0 d2 1d 74 22 5d 6a 3f c6 5e 77 c0 |..I....t"]j?.^w.| +00000040 c4 bb 31 2d 62 8d 7b 5c 66 6d c4 94 c0 13 00 00 |..1-b.{\fm......| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 02 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -61,38 +62,38 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 02 00 aa 0c 00 00 a6 03 00 1d 20 ef 2a b0 |............ .*.| -000002d0 da 99 1f 17 dc 29 38 43 c2 72 18 b6 ea ad 8f e6 |.....)8C.r......| -000002e0 f2 d2 81 e5 f6 2e 6c 5a 95 89 10 14 34 00 80 11 |......lZ....4...| -000002f0 ff 56 fa 9e 9f 0c b9 a9 57 c4 2c cb a1 ce 70 ca |.V......W.,...p.| -00000300 94 bd 5e 54 0e eb da ce 1b 19 ee bc aa bb 9c 10 |..^T............| -00000310 f8 6e 59 48 82 5e fe 3c 2e 28 6d 8a 2f 08 ce df |.nYH.^.<.(m./...| -00000320 52 e2 07 a6 68 68 69 fc 2b d7 fd 28 ef 48 85 c9 |R...hhi.+..(.H..| -00000330 70 75 b4 f4 04 2f 39 69 ee 1a aa 14 c9 7b e0 f7 |pu.../9i.....{..| -00000340 bc 50 76 df 24 57 8b a4 ca 87 8a b5 ff c9 13 9e |.Pv.$W..........| -00000350 d3 9e 2b e1 d4 6f 0c ce 61 83 e9 38 51 04 2e 23 |..+..o..a..8Q..#| -00000360 b4 f8 90 f4 0f 45 93 34 14 f4 ca 4f 13 02 89 16 |.....E.4...O....| +000002c0 16 03 02 00 aa 0c 00 00 a6 03 00 1d 20 73 f2 57 |............ s.W| +000002d0 12 5a 50 bb 9d 2d 14 f0 ca ee c1 41 bf da 9e 8e |.ZP..-.....A....| +000002e0 d5 a9 25 c3 84 07 e7 5c 35 87 8b 70 3d 00 80 d3 |..%....\5..p=...| +000002f0 86 1b 82 48 5c 14 9b cf e4 a0 2b 24 bc 8c ad e9 |...H\.....+$....| +00000300 7c 1e 4f da c2 22 10 91 76 47 bc 9f 64 ca 1c 69 ||.O.."..vG..d..i| +00000310 77 c0 c7 2c 50 ea 1a 07 d0 8c ec da aa ed 82 9d |w..,P...........| +00000320 a5 6c d6 27 05 f8 24 19 f9 d4 b1 c6 e3 0f 49 6f |.l.'..$.......Io| +00000330 e4 47 25 9a 36 f1 d3 ed b1 b5 a0 cc 66 50 75 64 |.G%.6.......fPud| +00000340 97 ee 3c 65 84 1b 62 f8 1a 8f 02 5a d4 2c 49 b3 |..>> Flow 3 (client to server) 00000000 16 03 02 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 02 00 01 01 |....._X.;t......| 00000030 16 03 02 00 40 00 00 00 00 00 00 00 00 00 00 00 |....@...........| -00000040 00 00 00 00 00 34 fa 43 0a 6f 38 d0 80 5d a1 d6 |.....4.C.o8..]..| -00000050 61 c9 62 a9 b2 14 88 b4 7d b9 dc 20 93 11 27 37 |a.b.....}.. ..'7| -00000060 5a 2a 5b d4 10 54 75 23 8d 39 4a 5b 51 80 48 f2 |Z*[..Tu#.9J[Q.H.| -00000070 16 64 e5 90 38 |.d..8| +00000040 00 00 00 00 00 93 60 ad fa 30 93 3e 31 6d 2c 0c |......`..0.>1m,.| +00000050 21 3d 6a 53 b1 51 f9 a9 1f 74 ee 42 7a 90 8a 08 |!=jS.Q...t.Bz...| +00000060 9e a0 7f 42 19 c4 28 06 77 bb 32 c1 a0 0d ec 71 |...B..(.w.2....q| +00000070 4f 20 89 c1 7d |O ..}| >>> Flow 4 (server to client) -00000000 14 03 02 00 01 01 16 03 02 00 40 f8 77 bf c0 dd |..........@.w...| -00000010 11 e1 3f 90 e0 1a c0 dc a6 6a b2 81 9b 1b 63 bc |..?......j....c.| -00000020 42 b6 23 b2 1f 95 36 38 56 f8 a4 e4 c1 c9 6f d2 |B.#...68V.....o.| -00000030 64 16 13 16 b7 25 53 ec a1 c2 94 de 8d ae 38 b9 |d....%S.......8.| -00000040 07 29 42 1d 16 3d fb 29 8b 76 15 |.)B..=.).v.| +00000000 14 03 02 00 01 01 16 03 02 00 40 fb 97 d9 4a c6 |..........@...J.| +00000010 10 21 6f 7b 77 ba e0 41 b2 50 d3 8c df 54 b1 9f |.!o{w..A.P...T..| +00000020 98 55 e7 0e fb bd 25 67 fb fb 5a 5c 86 b8 f0 17 |.U....%g..Z\....| +00000030 2b 56 b3 81 21 45 58 98 38 63 24 0a ec aa 17 55 |+V..!EX.8c$....U| +00000040 8c 46 67 a6 44 57 00 8d 49 83 28 |.Fg.DW..I.(| >>> Flow 5 (client to server) 00000000 17 03 02 00 30 00 00 00 00 00 00 00 00 00 00 00 |....0...........| -00000010 00 00 00 00 00 59 ac 45 2f 3f 3b d3 5d 73 08 85 |.....Y.E/?;.]s..| -00000020 9a 0e 99 a2 ad 8d 4e de 52 e9 5b 8b 76 71 5d 5e |......N.R.[.vq]^| -00000030 6b ec 05 24 a5 15 03 02 00 30 00 00 00 00 00 00 |k..$.....0......| -00000040 00 00 00 00 00 00 00 00 00 00 92 ce 2d b7 15 d2 |............-...| -00000050 3b c5 42 ad aa 73 a8 e0 ac 26 d2 e7 ef c7 a9 72 |;.B..s...&.....r| -00000060 75 96 5c 8c 2d 60 6e 57 bd ef |u.\.-`nW..| +00000010 00 00 00 00 00 e7 87 56 3d 84 0b 17 94 41 94 67 |.......V=....A.g| +00000020 48 ff 0f e4 2d a9 1c 7a 68 fc 42 36 17 27 b0 66 |H...-..zh.B6.'.f| +00000030 af 97 25 90 a2 15 03 02 00 30 00 00 00 00 00 00 |..%......0......| +00000040 00 00 00 00 00 00 00 00 00 00 55 a4 8a bc 52 98 |..........U...R.| +00000050 07 3a dd 7a dc 4f 2c 8d f2 1b f0 76 09 ca 88 36 |.:.z.O,....v...6| +00000060 d4 fa f9 f4 b7 2e ce 6e 26 82 |.......n&.| diff --git a/src/crypto/tls/testdata/Client-TLSv11-RSA-RC4 b/src/crypto/tls/testdata/Client-TLSv11-RSA-RC4 index dd58620c470e7d..64b06a8f17a035 100644 --- a/src/crypto/tls/testdata/Client-TLSv11-RSA-RC4 +++ b/src/crypto/tls/testdata/Client-TLSv11-RSA-RC4 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 02 00 55 02 00 00 51 03 02 42 bb c8 01 43 |....U...Q..B...C| -00000010 e4 f7 10 09 bf 35 a3 c2 d7 ca 60 a9 60 be 1d 00 |.....5....`.`...| -00000020 92 33 b6 3e 56 2b c2 f7 29 e3 f9 20 8c d7 b6 90 |.3.>V+..).. ....| -00000030 03 f5 8b 55 e9 81 47 5b 84 ec 19 bc 32 58 c6 30 |...U..G[....2X.0| -00000040 f1 79 ea 51 a2 18 cc c0 4e 8a 8e cb 00 05 00 00 |.y.Q....N.......| +00000000 16 03 02 00 55 02 00 00 51 03 02 3d a4 ea 71 81 |....U...Q..=..q.| +00000010 c9 47 24 2b 53 22 83 07 df 5a 9e 76 ef ca d8 1b |.G$+S"...Z.v....| +00000020 1f 16 15 cd 7e e4 62 93 1e 5d a7 20 9d ac ea 5a |....~.b..]. ...Z| +00000030 9e e3 7c 14 94 9d 1b 9e 2a 7b 2d 80 55 85 2f 9e |..|.....*{-.U./.| +00000040 ed 17 20 79 66 a2 6c 88 81 cb b0 79 00 05 00 00 |.. yf.l....y....| 00000050 09 ff 01 00 01 00 00 17 00 00 16 03 02 02 59 0b |..............Y.| 00000060 00 02 55 00 02 52 00 02 4f 30 82 02 4b 30 82 01 |..U..R..O0..K0..| 00000070 b4 a0 03 02 01 02 02 09 00 e8 f0 9d 3f e2 5b ea |............?.[.| @@ -72,15 +73,15 @@ 00000060 c5 70 0f 08 83 48 e9 48 ef 6e 50 8b 05 7e e5 84 |.p...H.H.nP..~..| 00000070 25 fa 55 c7 ae 31 02 27 00 ef 3f 98 86 20 12 89 |%.U..1.'..?.. ..| 00000080 91 59 28 b4 f7 d7 af d2 69 61 35 14 03 02 00 01 |.Y(.....ia5.....| -00000090 01 16 03 02 00 24 a8 40 50 90 31 50 fc 5e d8 f5 |.....$.@P.1P.^..| -000000a0 64 7a a5 62 8d ad ce 03 34 c9 ab 16 31 45 bc 8b |dz.b....4...1E..| -000000b0 26 8b fa 38 7a 2e 60 42 86 46 |&..8z.`B.F| +00000090 01 16 03 02 00 24 30 52 7f 8a 5c 3a 31 65 87 8c |.....$0R..\:1e..| +000000a0 9e 31 8f b1 22 15 ed af 99 6c 19 47 46 fd e1 3b |.1.."....l.GF..;| +000000b0 b3 f4 3a 5b d8 e5 a6 1a 7c 5e |..:[....|^| >>> Flow 4 (server to client) -00000000 14 03 02 00 01 01 16 03 02 00 24 83 1d 5a 04 90 |..........$..Z..| -00000010 1f 8d 07 46 39 1e 3c cf cb 06 45 f5 88 9e 68 5c |...F9.<...E...h\| -00000020 96 a0 06 47 43 21 20 ce 90 1f 24 49 52 30 59 |...GC! ...$IR0Y| +00000000 14 03 02 00 01 01 16 03 02 00 24 c1 5d da 6d 6e |..........$.].mn| +00000010 55 3e 70 a4 52 15 d9 ba 88 a1 b7 f0 40 71 09 fa |U>p.R.......@q..| +00000020 3f 00 6f 39 72 88 89 a1 3d cf 7a 7a 97 15 b7 |?.o9r...=.zz...| >>> Flow 5 (client to server) -00000000 17 03 02 00 1a f9 9f c2 32 02 96 87 95 4b ba 17 |........2....K..| -00000010 1f b9 af fe 6a 87 38 d4 d8 b4 f8 1d ad 05 00 15 |....j.8.........| -00000020 03 02 00 16 54 07 15 34 f4 c3 a2 3e 01 9e 00 cb |....T..4...>....| -00000030 83 f7 58 79 e2 dc a4 b8 15 e9 |..Xy......| +00000000 17 03 02 00 1a 56 ea a4 ed 0f 9d 98 3b 48 bc 76 |.....V......;H.v| +00000010 35 3f fb 78 92 d9 ce ef 53 b2 ef a6 13 9a 4c 15 |5?.x....S.....L.| +00000020 03 02 00 16 b5 41 d0 98 50 73 73 90 c0 fe ec 11 |.....A..Pss.....| +00000030 ec 98 d5 fb 02 c0 11 11 29 1c |........).| diff --git a/src/crypto/tls/testdata/Client-TLSv12-AES128-GCM-SHA256 b/src/crypto/tls/testdata/Client-TLSv12-AES128-GCM-SHA256 index d88acc95f9b5b2..33e01c140b0139 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-AES128-GCM-SHA256 +++ b/src/crypto/tls/testdata/Client-TLSv12-AES128-GCM-SHA256 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 55 02 00 00 51 03 03 d1 5c 30 66 38 |....U...Q...\0f8| -00000010 84 89 85 fe b6 87 de 62 8e b5 86 18 16 f7 8f b0 |.......b........| -00000020 f2 00 09 e5 55 d7 10 a0 76 d8 58 20 23 7d ff 5f |....U...v.X #}._| -00000030 b4 b5 5b de b9 7a f9 d1 bc 90 ff 2c 1d d7 fe ed |..[..z.....,....| -00000040 2f 6e 29 d8 08 61 0b 9d cc 7d 64 6a 00 9c 00 00 |/n)..a...}dj....| +00000000 16 03 03 00 55 02 00 00 51 03 03 c5 da f3 88 41 |....U...Q......A| +00000010 ed 43 a7 4e 49 ad db 74 04 c6 fb c2 13 49 86 14 |.C.NI..t.....I..| +00000020 fd d0 e1 7a ab d3 df 65 62 c6 1f 20 29 1a 03 bf |...z...eb.. )...| +00000030 10 35 3c 58 36 fd 4d 7a 7d dc f1 fd be d0 c6 b5 |.5D..l.C.:...S.| -000000b0 a6 d3 5c f0 5b ca e8 6a c7 13 88 65 50 71 |..\.[..j...ePq| +00000090 01 16 03 03 00 28 00 00 00 00 00 00 00 00 2c 46 |.....(........,F| +000000a0 31 d0 81 b9 6e 55 b0 5b 7b 15 a0 27 da e2 d7 71 |1...nU.[{..'...q| +000000b0 f4 d0 83 6c 98 d9 75 e2 f4 4c 61 0b fe 3c |...l..u..La..<| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 28 3f 30 36 77 c6 |..........(?06w.| -00000010 6d 4f 37 2c ac f1 c2 24 9d e1 3f 0c 51 64 12 67 |mO7,...$..?.Qd.g| -00000020 83 47 ec f1 b8 02 6d 11 11 b7 ec ab 09 26 41 ff |.G....m......&A.| -00000030 e8 32 05 |.2.| +00000000 14 03 03 00 01 01 16 03 03 00 28 e5 b5 71 5d e0 |..........(..q].| +00000010 c8 9a 75 4c 6e 72 4e a2 5f f9 b4 9f f4 40 a0 de |..uLnrN._....@..| +00000020 73 48 9c 01 f3 0b 78 91 5f 85 29 9c 51 dc 77 bc |sH....x._.).Q.w.| +00000030 c0 32 42 |.2B| >>> Flow 5 (client to server) -00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 af 3a 1e |..............:.| -00000010 98 3e 74 5f 66 64 eb 6c 88 36 33 f0 98 e8 46 e9 |.>t_fd.l.63...F.| -00000020 f1 2f 32 15 03 03 00 1a 00 00 00 00 00 00 00 02 |./2.............| -00000030 76 2b 83 08 46 7d 34 58 db e8 e8 e1 e2 cb ad 1e |v+..F}4X........| -00000040 e4 a5 |..| +00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 9e 81 42 |...............B| +00000010 90 9e ab a2 29 b3 f7 a6 31 45 5f a4 2e d5 52 f7 |....)...1E_...R.| +00000020 72 8b 3b 15 03 03 00 1a 00 00 00 00 00 00 00 02 |r.;.............| +00000030 96 5a f4 de 37 1b 2c f1 8d 90 91 17 6f 81 90 7f |.Z..7.,.....o...| +00000040 e7 9d |..| diff --git a/src/crypto/tls/testdata/Client-TLSv12-AES128-SHA256 b/src/crypto/tls/testdata/Client-TLSv12-AES128-SHA256 index 030cbc6f2be591..b373f27aaeb622 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-AES128-SHA256 +++ b/src/crypto/tls/testdata/Client-TLSv12-AES128-SHA256 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 55 02 00 00 51 03 03 bd e7 e2 ea 28 |....U...Q......(| -00000010 09 0b 7f 30 b5 91 fd b4 91 d1 c3 b8 8f 25 38 88 |...0.........%8.| -00000020 62 28 5c 1d 14 1a 6a 50 7d 30 e8 20 c3 91 2d ea |b(\...jP}0. ..-.| -00000030 59 8d 1d 56 98 c9 02 c0 f1 98 94 a6 8c 2b f2 63 |Y..V.........+.c| -00000040 ed b9 97 36 c6 c6 32 97 9e 61 6c 3b 00 3c 00 00 |...6..2..al;.<..| +00000000 16 03 03 00 55 02 00 00 51 03 03 eb c3 6e 78 2e |....U...Q....nx.| +00000010 9f 47 ec 9d 0e bc 8d 49 d2 46 11 75 5e 50 a5 07 |.G.....I.F.u^P..| +00000020 0a 99 90 34 34 2d 81 75 1b f4 ca 20 fb 28 c1 8c |...44-.u... .(..| +00000030 55 27 36 be 4e d1 c7 ee e8 b3 2a eb 7a be f1 2a |U'6.N.....*.z..*| +00000040 fc 81 df d8 5b c2 1f 1e 7b 47 0f 06 00 3c 00 00 |....[...{G...<..| 00000050 09 ff 01 00 01 00 00 17 00 00 16 03 03 02 59 0b |..............Y.| 00000060 00 02 55 00 02 52 00 02 4f 30 82 02 4b 30 82 01 |..U..R..O0..K0..| 00000070 b4 a0 03 02 01 02 02 09 00 e8 f0 9d 3f e2 5b ea |............?.[.| @@ -73,25 +74,25 @@ 00000070 25 fa 55 c7 ae 31 02 27 00 ef 3f 98 86 20 12 89 |%.U..1.'..?.. ..| 00000080 91 59 28 b4 f7 d7 af d2 69 61 35 14 03 03 00 01 |.Y(.....ia5.....| 00000090 01 16 03 03 00 50 00 00 00 00 00 00 00 00 00 00 |.....P..........| -000000a0 00 00 00 00 00 00 f3 6e c5 65 29 75 22 fe 23 6f |.......n.e)u".#o| -000000b0 96 fa b3 8d 98 ac df bf 3a 23 ca 4f d9 52 95 3f |........:#.O.R.?| -000000c0 89 8b 61 cf d2 2d f3 14 31 9d 3a 6f a9 b5 7b 69 |..a..-..1.:o..{i| -000000d0 27 a6 2b 2d 43 6e 77 f1 cd 73 fa 80 7b bf 8a 1d |'.+-Cnw..s..{...| -000000e0 83 c5 9b 8c 0a 03 |......| +000000a0 00 00 00 00 00 00 ee 37 21 73 05 93 63 ca 4f 0e |.......7!s..c.O.| +000000b0 e2 29 7f 90 6f 3a aa 52 a0 e0 71 97 5f f9 66 cf |.)..o:.R..q._.f.| +000000c0 06 19 09 51 03 5f 2e 0c 57 26 42 15 ef 8f 4e a9 |...Q._..W&B...N.| +000000d0 c4 97 95 0a d1 ce 30 2c bf 7a 85 4b a9 d0 a9 e0 |......0,.z.K....| +000000e0 64 11 1a dc 48 1d |d...H.| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 50 f6 93 f3 85 25 |..........P....%| -00000010 06 5c 47 ea d5 6e 90 05 21 91 9a 86 06 42 f6 54 |.\G..n..!....B.T| -00000020 25 4a 61 65 7e 19 c8 1a 58 52 06 81 df dd 19 fc |%Jae~...XR......| -00000030 bd 5a 82 ff ae 80 92 b3 3b 7d 89 c0 64 b1 36 e3 |.Z......;}..d.6.| -00000040 5c bb 2a 5b e8 6d 18 02 43 27 b5 57 bc 3f ab b1 |\.*[.m..C'.W.?..| -00000050 27 59 0e 6a d5 07 6a 66 ad 51 82 |'Y.j..jf.Q.| +00000000 14 03 03 00 01 01 16 03 03 00 50 58 d1 09 93 e9 |..........PX....| +00000010 97 c1 cc c3 db 1d 65 d8 97 35 c1 b4 3f 9f a0 00 |......e..5..?...| +00000020 9f 81 2a 81 61 b5 51 9f ec 15 94 d0 86 0f a1 70 |..*.a.Q........p| +00000030 e9 90 59 dd 16 d3 e7 33 21 b7 d8 6e 4b a8 fd fb |..Y....3!..nK...| +00000040 dd 98 78 95 16 44 f8 da bd e3 e2 3f f0 e1 d9 39 |..x..D.....?...9| +00000050 c0 2e 0d c4 fe a5 ac 41 66 2d f3 |.......Af-.| >>> Flow 5 (client to server) 00000000 17 03 03 00 40 00 00 00 00 00 00 00 00 00 00 00 |....@...........| -00000010 00 00 00 00 00 2a 8b 84 55 57 20 b8 4f 11 d2 cc |.....*..UW .O...| -00000020 5e a0 ef 1a f1 62 fc 59 38 14 30 0f e0 76 04 ca |^....b.Y8.0..v..| -00000030 eb c2 45 a4 75 12 e6 1a 16 fc d8 bd d4 f7 fd 6f |..E.u..........o| -00000040 f3 99 ca d6 41 15 03 03 00 40 00 00 00 00 00 00 |....A....@......| -00000050 00 00 00 00 00 00 00 00 00 00 8c 80 a8 e9 95 4c |...............L| -00000060 c9 fa 4d 9d 06 e5 d8 70 5f fa a9 44 24 15 f4 47 |..M....p_..D$..G| -00000070 5a 0a 24 55 a7 6a 04 1c 2a 39 80 fd 08 47 d7 08 |Z.$U.j..*9...G..| -00000080 41 8b 9d 46 74 cb 61 c6 84 84 |A..Ft.a...| +00000010 00 00 00 00 00 2f d9 4b fc 22 0f 20 dd 2c 20 83 |...../.K.". ., .| +00000020 bd f0 49 5b bc 57 55 bc fb 41 13 79 a7 19 4d e3 |..I[.WU..A.y..M.| +00000030 71 66 ce b5 77 35 9c 54 86 0e fc 21 d6 9e c4 66 |qf..w5.T...!...f| +00000040 ee 95 3f 0a d5 15 03 03 00 40 00 00 00 00 00 00 |..?......@......| +00000050 00 00 00 00 00 00 00 00 00 00 54 25 cd 1a 0a d4 |..........T%....| +00000060 7b a4 ce f1 62 c8 8f 62 af 93 8e 0e e8 fd ef 55 |{...b..b.......U| +00000070 1d 47 a4 ac 1c 80 25 6f c4 a2 51 11 84 e9 63 cb |.G....%o..Q...c.| +00000080 db d8 e6 bf 2c 89 4e 1a d4 11 |....,.N...| diff --git a/src/crypto/tls/testdata/Client-TLSv12-AES256-GCM-SHA384 b/src/crypto/tls/testdata/Client-TLSv12-AES256-GCM-SHA384 index 42454d52f3fdaf..9920621be3b770 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-AES256-GCM-SHA384 +++ b/src/crypto/tls/testdata/Client-TLSv12-AES256-GCM-SHA384 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 55 02 00 00 51 03 03 a3 a8 ef 9d b3 |....U...Q.......| -00000010 36 14 00 fe 5b e4 07 77 e7 78 c8 b4 d8 96 8d 72 |6...[..w.x.....r| -00000020 c5 79 99 10 8f 6e ac e3 fd 99 ee 20 53 99 57 36 |.y...n..... S.W6| -00000030 7e 9b 89 a3 79 c7 57 c6 f6 c7 2c 66 0a 6b 48 a2 |~...y.W...,f.kH.| -00000040 06 87 54 2c 20 3d 65 b9 1c 93 fd a3 00 9d 00 00 |..T, =e.........| +00000000 16 03 03 00 55 02 00 00 51 03 03 9f 42 e3 1b 5f |....U...Q...B.._| +00000010 00 d9 00 0a a5 fa 6d 63 95 11 d2 5a ff 51 b0 f8 |......mc...Z.Q..| +00000020 bc a9 7f 18 3c 8d d7 38 d5 e8 2d 20 be 5b 08 a0 |....<..8..- .[..| +00000030 2f 7b 50 d9 06 0c bf 7c 2d cc f0 39 3a 90 e4 e7 |/{P....|-..9:...| +00000040 9c 27 9b 88 ea ff c0 51 f8 79 cd b9 00 9d 00 00 |.'.....Q.y......| 00000050 09 ff 01 00 01 00 00 17 00 00 16 03 03 02 59 0b |..............Y.| 00000060 00 02 55 00 02 52 00 02 4f 30 82 02 4b 30 82 01 |..U..R..O0..K0..| 00000070 b4 a0 03 02 01 02 02 09 00 e8 f0 9d 3f e2 5b ea |............?.[.| @@ -72,17 +73,17 @@ 00000060 c5 70 0f 08 83 48 e9 48 ef 6e 50 8b 05 7e e5 84 |.p...H.H.nP..~..| 00000070 25 fa 55 c7 ae 31 02 27 00 ef 3f 98 86 20 12 89 |%.U..1.'..?.. ..| 00000080 91 59 28 b4 f7 d7 af d2 69 61 35 14 03 03 00 01 |.Y(.....ia5.....| -00000090 01 16 03 03 00 28 00 00 00 00 00 00 00 00 76 c9 |.....(........v.| -000000a0 1e 50 43 51 5e 4a b6 c8 98 33 8f 18 ac f3 fe 20 |.PCQ^J...3..... | -000000b0 09 76 7c 48 f7 e8 00 2c 3b e1 7e 2f 71 d5 |.v|H...,;.~/q.| +00000090 01 16 03 03 00 28 00 00 00 00 00 00 00 00 0e dd |.....(..........| +000000a0 10 e6 7a dd b2 02 5c 0d 0c 73 b4 f1 a1 89 85 95 |..z...\..s......| +000000b0 91 a9 e4 7a 47 49 9a 52 96 77 93 0b cc 74 |...zGI.R.w...t| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 28 55 b2 34 a3 b7 |..........(U.4..| -00000010 18 a8 36 55 c4 ef 81 82 e4 41 9f 1a 42 2c 42 7e |..6U.....A..B,B~| -00000020 9b 59 c8 9d 3a 0b 59 a6 78 d0 ab 2f c4 e1 f7 2b |.Y..:.Y.x../...+| -00000030 9b ee 9f |...| +00000000 14 03 03 00 01 01 16 03 03 00 28 72 c3 d3 ad 10 |..........(r....| +00000010 71 48 c4 32 9b 24 ac c0 a4 88 db 35 e9 be fe 34 |qH.2.$.....5...4| +00000020 ee 9a bb 5a a5 90 b6 ec 31 9a bc 4c 56 e2 e6 d5 |...Z....1..LV...| +00000030 f9 2b 93 |.+.| >>> Flow 5 (client to server) -00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 ec ad 42 |...............B| -00000010 8a cf 39 ed 82 b7 70 9b 1a 29 4c 3e b7 f6 92 35 |..9...p..)L>...5| -00000020 12 e2 df 15 03 03 00 1a 00 00 00 00 00 00 00 02 |................| -00000030 81 d4 de 3c 03 a9 01 0e e7 64 ae 18 84 dd d3 6c |...<.....d.....l| -00000040 57 6a |Wj| +00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 ed af c7 |................| +00000010 74 44 64 5d e3 1b 0c d4 bd a1 7e 7e 87 10 8d bc |tDd]......~~....| +00000020 26 db e8 15 03 03 00 1a 00 00 00 00 00 00 00 02 |&...............| +00000030 42 6b 3e 59 c5 e4 e5 5b 53 0f fe d9 3f 0c 5c f0 |Bk>Y...[S...?.\.| +00000040 67 d5 |g.| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ALPN b/src/crypto/tls/testdata/Client-TLSv12-ALPN index 9f3f4dc930ed36..6686f6e9c0f7c5 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ALPN +++ b/src/crypto/tls/testdata/Client-TLSv12-ALPN @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 01 12 01 00 01 0e 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 2c 01 00 01 28 03 03 00 00 00 00 00 |....,...(.......| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,22 +7,24 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 93 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 ad 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 10 00 10 |................| -000000d0 00 0e 06 70 72 6f 74 6f 32 06 70 72 6f 74 6f 31 |...proto2.proto1| -000000e0 00 2b 00 09 08 03 04 03 03 03 02 03 01 00 33 00 |.+............3.| -000000f0 26 00 24 00 1d 00 20 2f e5 7d a3 47 cd 62 43 15 |&.$... /.}.G.bC.| -00000100 28 da ac 5f bb 29 07 30 ff f6 84 af c4 cf c2 ed |(.._.).0........| -00000110 90 99 5f 58 cb 3b 74 |.._X.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 10 00 10 00 0e 06 70 72 6f |.............pro| +000000f0 74 6f 32 06 70 72 6f 74 6f 31 00 2b 00 09 08 03 |to2.proto1.+....| +00000100 04 03 03 03 02 03 01 00 33 00 26 00 24 00 1d 00 |........3.&.$...| +00000110 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 | /.}.G.bC.(.._.)| +00000120 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b |.0.........._X.;| +00000130 74 |t| >>> Flow 2 (server to client) -00000000 16 03 03 00 6a 02 00 00 66 03 03 71 ba 32 f4 08 |....j...f..q.2..| -00000010 31 23 ea 71 46 b6 f0 24 f9 80 c2 94 b8 13 f5 cc |1#.qF..$........| -00000020 6b 77 e5 46 f1 76 df a5 0c 56 6a 20 98 53 12 a3 |kw.F.v...Vj .S..| -00000030 d2 a6 97 45 36 11 e6 b6 b6 6e 0b b9 30 2d 65 c8 |...E6....n..0-e.| -00000040 47 c7 af 4c c5 a6 51 90 2f 77 db 8c cc a8 00 00 |G..L..Q./w......| +00000000 16 03 03 00 6a 02 00 00 66 03 03 37 57 f9 cb 6a |....j...f..7W..j| +00000010 dc 6c e2 4b 1d 74 93 8e e1 1a 05 e7 fd 8d 29 57 |.l.K.t........)W| +00000020 5e b4 fc 9a ba 7d df de 56 cc e7 20 57 f3 1a b0 |^....}..V.. W...| +00000030 20 cd ac a1 e3 93 b9 79 f5 1c ce d2 d1 24 da fc | ......y.....$..| +00000040 88 97 25 fb 36 72 75 bb cc a3 2a 79 cc a8 00 00 |..%.6ru...*y....| 00000050 1e ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 10 |................| 00000060 00 09 00 07 06 70 72 6f 74 6f 31 00 17 00 00 16 |.....proto1.....| 00000070 03 03 02 59 0b 00 02 55 00 02 52 00 02 4f 30 82 |...Y...U..R..O0.| @@ -63,31 +65,31 @@ 000002a0 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 4c 72 2b |Cw.......@.a.Lr+| 000002b0 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 bd 42 d4 |...F..M...>...B.| 000002c0 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 16 03 03 |..=.`.\!.;......| -000002d0 00 ac 0c 00 00 a8 03 00 1d 20 76 16 ff 2a 83 9a |......... v..*..| -000002e0 07 b0 01 f6 1e 3e 4b c1 69 62 9b 00 38 35 c5 4f |.....>K.ib..85.O| -000002f0 d3 2c db 50 16 9d 26 b1 cc 28 08 04 00 80 56 73 |.,.P..&..(....Vs| -00000300 39 4d ab f4 68 92 14 6d d5 75 db 10 4b 0c a4 19 |9M..h..m.u..K...| -00000310 9e 67 42 d3 f8 3f 91 26 06 4d f9 34 9b ad 7c 85 |.gB..?.&.M.4..|.| -00000320 ec 11 34 43 dd 7d a8 5b 0f 59 54 ff a3 92 dc 7c |..4C.}.[.YT....|| -00000330 26 ee 07 0e cf bc e4 6a 1a da b9 d2 9a 62 bf de |&......j.....b..| -00000340 37 a6 68 e2 cd 21 8e 4e a3 66 ee e1 74 45 7d 64 |7.h..!.N.f..tE}d| -00000350 b2 d8 1d dc 6a cd 2a 86 9e 0d 76 a3 7b f2 49 11 |....j.*...v.{.I.| -00000360 ae ad a1 d4 f2 a9 e5 56 0e 2a 96 a5 f7 8b 74 d4 |.......V.*....t.| -00000370 78 e0 bc cc 95 a1 f1 76 0b 4b 37 ed 77 1b 16 03 |x......v.K7.w...| +000002d0 00 ac 0c 00 00 a8 03 00 1d 20 27 fd a6 7d 32 28 |......... '..}2(| +000002e0 d2 44 43 d5 2a 95 3b e2 7e 77 5a d2 66 87 4a 92 |.DC.*.;.~wZ.f.J.| +000002f0 12 56 ba 40 3e 09 43 b4 90 02 08 04 00 80 8b e9 |.V.@>.C.........| +00000300 93 0c 5e 72 b8 a8 49 1c c3 b7 fc 39 af 58 9c 3a |..^r..I....9.X.:| +00000310 27 b6 20 23 9a 5d 35 0e dd 21 da 1b be 34 32 23 |'. #.]5..!...42#| +00000320 76 2e c8 9e 3d c8 e5 9e cd 04 11 a9 be 97 52 53 |v...=.........RS| +00000330 81 b6 01 fc 60 5d 0d dd 0d ba 05 a6 15 3b 29 4e |....`].......;)N| +00000340 88 a0 69 b9 c2 46 d9 c2 6d 17 b2 d0 c0 ee de f9 |..i..F..m.......| +00000350 94 fe 25 31 66 df 72 7d 02 77 a6 2c a6 ad da eb |..%1f.r}.w.,....| +00000360 dc 4b ea d4 5b 0a d4 e6 c8 89 15 cf 3f 81 58 20 |.K..[.......?.X | +00000370 0f 83 2f e1 db a1 cc a7 e1 61 79 7d 14 4b 16 03 |../......ay}.K..| 00000380 03 00 04 0e 00 00 00 |.......| >>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| -00000030 16 03 03 00 20 aa 8c 9c ab fb ab ed 5b 2e 3e 4b |.... .......[.>K| -00000040 92 cd 18 dd ac ce af 1a b9 f2 df aa 7f a1 d2 ad |................| -00000050 0e 8a 67 03 3f |..g.?| +00000030 16 03 03 00 20 50 e7 36 59 68 f6 0f cd b0 cc ae |.... P.6Yh......| +00000040 bc 18 23 d3 c0 fb c9 41 49 91 ec 8f cd f0 69 84 |..#....AI.....i.| +00000050 49 3c 68 6e 05 |I>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 20 db cb 3a 73 0c |.......... ..:s.| -00000010 ab 16 44 41 5a 25 df f7 45 bf f0 70 c6 a8 4f bd |..DAZ%..E..p..O.| -00000020 50 99 fb 18 26 e7 cf 71 16 1a 96 |P...&..q...| +00000000 14 03 03 00 01 01 16 03 03 00 20 84 30 04 4f 3e |.......... .0.O>| +00000010 72 f6 d8 6a dc ae fb 55 82 69 8e a3 77 c2 1f 27 |r..j...U.i..w..'| +00000020 88 14 5b 8d 10 79 4c 6a 41 3e e5 |..[..yLjA>.| >>> Flow 5 (client to server) -00000000 17 03 03 00 16 34 82 27 01 99 27 08 58 c4 aa 76 |.....4.'..'.X..v| -00000010 fe 34 9c 1f 99 ea 4c f8 a0 ef 96 15 03 03 00 12 |.4....L.........| -00000020 68 b8 da ac 77 d5 f7 95 9f 68 e0 d6 26 88 cd ef |h...w....h..&...| -00000030 31 d4 |1.| +00000000 17 03 03 00 16 03 a8 d7 eb 5d 79 60 d4 65 1d c1 |.........]y`.e..| +00000010 2b 32 b5 c3 fb b2 2f 10 83 4e 39 15 03 03 00 12 |+2..../..N9.....| +00000020 90 eb 89 fc e7 4c a2 94 02 33 b3 0e 72 6e bc 4e |.....L...3..rn.N| +00000030 d4 e7 |..| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-ECDSA-ECDSA b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-ECDSA-ECDSA index b56ac9f0c4b317..784e95f87e5874 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-ECDSA-ECDSA +++ b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-ECDSA-ECDSA @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 d4 19 2c 08 c6 |....]...Y....,..| -00000010 ef 32 88 79 a1 84 fc 79 38 62 b2 dd 4b a7 0b b3 |.2.y...y8b..K...| -00000020 d3 13 3d d4 f7 c7 4d d9 8b c6 8e 20 56 8e 90 3d |..=...M.... V..=| -00000030 2b 6e 2d cf 7e c1 c6 b0 e5 d8 d2 af 3a 06 88 c6 |+n-.~.......:...| -00000040 ed e7 18 76 ab 45 c5 76 47 67 95 ad c0 09 00 00 |...v.E.vGg......| +00000000 16 03 03 00 5d 02 00 00 59 03 03 4a 0b 4d 4a 05 |....]...Y..J.MJ.| +00000010 a0 5b ca 5d 65 4f f7 7e 82 3c 54 1d a3 42 64 7c |.[.]eO.~..;O....P.......| -000002c0 0d 89 bc 6c 01 93 45 ca b8 3c 09 cf b2 01 e9 99 |...l..E..<......| -000002d0 87 fb 1d ac 91 7f 77 a2 21 5e 07 5e 65 3b ec 31 |......w.!^.^e;.1| -000002e0 d7 b5 b9 1d 88 c8 82 f5 03 a9 37 e8 b9 02 42 01 |..........7...B.| -000002f0 78 c4 90 fb e3 7f 5a 7a 66 0a 44 f5 66 0e 1e ac |x.....Zzf.D.f...| -00000300 cb 53 c4 74 55 3a b8 b7 ec b3 f6 4c e1 a8 b6 cb |.S.tU:.....L....| -00000310 a1 e5 37 ac e2 17 e2 6c 8f 71 37 b0 f1 ca b8 1b |..7....l.q7.....| -00000320 df 91 16 a4 7f 31 b1 4f 3e e4 30 f0 5a 5e df 93 |.....1.O>.0.Z^..| -00000330 85 16 03 03 00 3a 0d 00 00 36 03 01 02 40 00 2e |.....:...6...@..| +00000280 1d 20 b8 ef 9c cd 47 a4 e9 1e e6 02 d5 40 12 95 |. ....G......@..| +00000290 0e da 1d 8f 2a 4f a6 9f 95 f9 1a 2a 1c ce 3c 32 |....*O.....*..<2| +000002a0 2b 4e 04 03 00 8b 30 81 88 02 42 00 c2 7d 24 9c |+N....0...B..}$.| +000002b0 b8 9c b1 69 fc 53 e9 ae 2c 07 39 23 d3 56 c8 96 |...i.S..,.9#.V..| +000002c0 e1 5e a9 a6 6b 18 33 f0 f9 50 29 65 2e dd 2b df |.^..k.3..P)e..+.| +000002d0 4e 58 67 ee 85 b3 da 8d 28 a1 7a 9f 59 df ae 10 |NXg.....(.z.Y...| +000002e0 31 d2 1b 6b 59 6d b9 24 16 79 66 49 7c 02 42 00 |1..kYm.$.yfI|.B.| +000002f0 bd a3 f5 56 db 00 b1 27 63 79 cb 1b 40 b2 e9 90 |...V...'cy..@...| +00000300 26 9f 44 53 ff fb 68 59 10 42 56 f2 36 fc 96 6a |&.DS..hY.BV.6..j| +00000310 a0 37 0e a3 77 5e 4c 0d 7b b6 e3 b5 b5 5b 2e d8 |.7..w^L.{....[..| +00000320 6f 99 6b af 84 57 9c 14 35 aa 09 23 11 08 83 3a |o.k..W..5..#...:| +00000330 63 16 03 03 00 3a 0d 00 00 36 03 01 02 40 00 2e |c....:...6...@..| 00000340 04 03 05 03 06 03 08 07 08 08 08 09 08 0a 08 0b |................| 00000350 08 04 08 05 08 06 04 01 05 01 06 01 03 03 02 03 |................| 00000360 03 01 02 01 03 02 02 02 04 02 05 02 06 02 00 00 |................| @@ -110,31 +111,31 @@ 00000210 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd 62 |...%...! /.}.G.b| 00000220 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 cf |C.(.._.).0......| 00000230 c2 ed 90 99 5f 58 cb 3b 74 16 03 03 00 93 0f 00 |...._X.;t.......| -00000240 00 8f 04 03 00 8b 30 81 88 02 42 01 c9 2c e8 1f |......0...B..,..| -00000250 5b a0 97 0f 70 2d 46 e3 6e b4 4c 82 fc be df 05 |[...p-F.n.L.....| -00000260 09 b0 d7 e4 0a 06 71 66 d6 3b 3e b6 56 2c 22 c7 |......qf.;>.V,".| -00000270 7c 3a 63 5c 34 22 5b 3a 49 a2 97 f3 2f 58 e7 2a ||:c\4"[:I.../X.*| -00000280 cd f4 05 84 db 8b 0b 22 e3 50 1b e7 6d 02 42 01 |.......".P..m.B.| -00000290 fd 1e bc 64 8b 01 3f a0 f8 8b 41 be 4d 97 07 6c |...d..?...A.M..l| -000002a0 3b 09 74 d6 00 76 b3 a0 c0 d0 92 d0 78 24 d7 b2 |;.t..v......x$..| -000002b0 1f c0 86 90 59 3a fb a8 e1 f6 80 23 db f7 90 93 |....Y:.....#....| -000002c0 95 fd 37 14 8c 34 f3 07 10 f8 dd e3 66 28 1c b2 |..7..4......f(..| -000002d0 a0 14 03 03 00 01 01 16 03 03 00 40 00 00 00 00 |...........@....| -000002e0 00 00 00 00 00 00 00 00 00 00 00 00 6c b8 ab 0b |............l...| -000002f0 c1 2d 2f 4e d2 25 14 96 24 c6 18 97 82 93 1c 1b |.-/N.%..$.......| -00000300 b4 ca 77 8b 17 7c cf 08 ca 06 e2 ef f3 97 6a 31 |..w..|........j1| -00000310 88 aa 46 a2 d3 7b 65 56 e2 77 72 df |..F..{eV.wr.| +00000240 00 8f 04 03 00 8b 30 81 88 02 42 01 dd bf fd e0 |......0...B.....| +00000250 db 99 16 0b f0 bc a9 4b 93 10 d9 59 06 d2 eb 30 |.......K...Y...0| +00000260 dc b1 44 69 1d 54 2d 9b b4 a4 c9 3a f8 d8 9d 2e |..Di.T-....:....| +00000270 36 71 3e ff aa 05 98 8a 27 00 dd 65 6a 15 b7 c9 |6q>.....'..ej...| +00000280 fe 1d 4d 45 02 58 09 4f 38 ea cd 88 3c 02 42 00 |..ME.X.O8...<.B.| +00000290 a6 27 a5 d7 e7 9e 66 7e f8 38 37 03 7b 6d df c4 |.'....f~.87.{m..| +000002a0 81 6a 59 af 5b c7 06 2e 57 e9 8f 8d 1f 9c 9d 94 |.jY.[...W.......| +000002b0 70 4c 74 ed 03 24 3d df 35 51 a1 76 a7 8f 9b 40 |pLt..$=.5Q.v...@| +000002c0 16 c6 4f 0e 49 95 34 20 22 dc 5c ce 1d c0 ed bf |..O.I.4 ".\.....| +000002d0 f4 14 03 03 00 01 01 16 03 03 00 40 00 00 00 00 |...........@....| +000002e0 00 00 00 00 00 00 00 00 00 00 00 00 b7 d5 df bf |................| +000002f0 2b 32 a9 9e e4 78 40 84 ca c9 7d 54 4e 38 89 3b |+2...x@...}TN8.;| +00000300 0f 83 23 42 75 95 6e 4d 70 d4 b6 5a 1a 26 dd bb |..#Bu.nMp..Z.&..| +00000310 37 4a 85 88 ab 60 ba 3c 19 bd 02 79 |7J...`.<...y| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 40 b8 19 a7 e2 35 |..........@....5| -00000010 8a be 5f 4c 91 d2 db 3f f3 42 90 8e b5 7f ea f7 |.._L...?.B......| -00000020 52 3e 61 2f 4d ef 25 8d ce 82 77 22 05 6f 0d b6 |R>a/M.%...w".o..| -00000030 04 c1 f0 f0 a2 9d bf 80 a9 f5 e1 62 5c e2 30 ef |...........b\.0.| -00000040 83 e7 c5 78 de 8f 17 4e d3 57 dd |...x...N.W.| +00000000 14 03 03 00 01 01 16 03 03 00 40 07 8e 16 c5 c2 |..........@.....| +00000010 e1 e5 32 41 2d 8b 3c fb 2e 3e b2 84 89 56 4d b5 |..2A-.<..>...VM.| +00000020 1c bd 1e a5 11 a1 b0 18 29 ce eb 46 f8 11 ff dc |........)..F....| +00000030 c0 cc ed 5a 38 33 5a 8f 48 df bf e9 8e 09 be b0 |...Z83Z.H.......| +00000040 9d 70 85 f7 71 f5 1e 5c aa 6f 88 |.p..q..\.o.| >>> Flow 5 (client to server) 00000000 17 03 03 00 30 00 00 00 00 00 00 00 00 00 00 00 |....0...........| -00000010 00 00 00 00 00 cb 79 13 ea 9e 7a 63 8f 5a 2b 8c |......y...zc.Z+.| -00000020 3a f3 bb 7e dc ff d1 2c 4a a6 4a aa ad bf 44 b8 |:..~...,J.J...D.| -00000030 cb 67 ce 3d f0 15 03 03 00 30 00 00 00 00 00 00 |.g.=.....0......| -00000040 00 00 00 00 00 00 00 00 00 00 f4 15 a0 f0 64 d1 |..............d.| -00000050 dd 5f 14 66 6c ce 51 95 bc b5 0b f8 4f 42 48 57 |._.fl.Q.....OBHW| -00000060 cf f3 09 62 75 0d 3e 64 64 e0 |...bu.>dd.| +00000010 00 00 00 00 00 37 b7 99 22 eb 67 a0 32 57 76 b5 |.....7..".g.2Wv.| +00000020 53 86 fc 5b 79 c5 4a 4f 3a 6b a9 0f d8 c8 30 de |S..[y.JO:k....0.| +00000030 0a 5b 9b af e2 15 03 03 00 30 00 00 00 00 00 00 |.[.......0......| +00000040 00 00 00 00 00 00 00 00 00 00 57 9c 4b ca 83 a3 |..........W.K...| +00000050 c0 3a 52 71 24 f5 b1 37 1e 4d c1 5a 2d 29 1c 90 |.:Rq$..7.M.Z-)..| +00000060 9b 96 cf 1e 07 39 f1 1b 03 2b |.....9...+| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-ECDSA-RSA b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-ECDSA-RSA index 692092e34f91dc..c20bd95d7f3065 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-ECDSA-RSA +++ b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-ECDSA-RSA @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 ed c3 6f 59 34 |....]...Y....oY4| -00000010 78 33 49 00 68 50 1f a5 aa 93 45 9a 87 34 c4 4e |x3I.hP....E..4.N| -00000020 71 b0 ab 5e 43 f7 a1 5c 89 e8 2f 20 f7 42 d7 2a |q..^C..\../ .B.*| -00000030 a5 fe 16 76 ac 6f cf 20 1d a6 bc d5 9d 27 9d 81 |...v.o. .....'..| -00000040 80 b4 0d 4e 12 89 de 7b 7a 5b a0 2b c0 2f 00 00 |...N...{z[.+./..| +00000000 16 03 03 00 5d 02 00 00 59 03 03 48 41 85 3e fc |....]...Y..HA.>.| +00000010 9f e7 b6 84 da fe 2f a7 1d e9 ba 12 4e a1 cf cb |....../.....N...| +00000020 9e f8 df 76 7c e7 29 c1 3b 9e 23 20 07 c9 55 c7 |...v|.).;.# ..U.| +00000030 0e 45 5c 26 17 94 b4 14 6b 58 39 27 43 4e dc 9b |.E\&....kX9'CN..| +00000040 65 30 0e f2 bd 59 d9 a2 a1 f3 0a 01 c0 2f 00 00 |e0...Y......./..| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -61,18 +62,18 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 d9 02 16 |............ ...| -000002d0 ba b6 db c0 c6 65 6d fb b8 69 dc 04 1a 8d 4a 36 |.....em..i....J6| -000002e0 27 6b f0 53 c3 72 56 1a 99 07 0f b9 79 08 04 00 |'k.S.rV.....y...| -000002f0 80 5d c6 1e a2 20 7d ae 35 e6 20 2f ff 7a 7b 8d |.]... }.5. /.z{.| -00000300 3f fd 49 d4 ce 26 f7 d0 c2 51 2a 42 03 26 b7 30 |?.I..&...Q*B.&.0| -00000310 af 5f 7d 34 1d 7d 37 ef 8a 1c e6 be 19 99 f1 30 |._}4.}7........0| -00000320 2e 26 6d 7b 83 cd 6e 26 c4 03 f5 7a 30 8f bf e3 |.&m{..n&...z0...| -00000330 c1 64 da 43 fc f0 32 c9 a2 68 e4 97 d0 34 8a c4 |.d.C..2..h...4..| -00000340 fc f8 bb 61 ec df 69 f7 d0 d7 1e 19 c0 5b 21 86 |...a..i......[!.| -00000350 eb 79 93 46 3b 7a 50 83 41 8b d3 7c 59 d5 34 8a |.y.F;zP.A..|Y.4.| -00000360 0c b5 70 e4 86 38 3a 6c 11 04 57 d6 94 c0 78 c7 |..p..8:l..W...x.| -00000370 91 16 03 03 00 3a 0d 00 00 36 03 01 02 40 00 2e |.....:...6...@..| +000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 73 d3 a9 |............ s..| +000002d0 7e 93 32 e3 dd ad 1c b3 c1 ff 03 c2 b9 08 da 09 |~.2.............| +000002e0 d3 1b 67 95 9c 8c d1 05 12 2e 8b dc 7a 08 04 00 |..g.........z...| +000002f0 80 85 af 3b 06 67 b0 ab 07 70 21 02 b1 3a 89 40 |...;.g...p!..:.@| +00000300 d6 90 ef a5 5b 89 49 81 18 20 74 9f 7b dd 58 65 |....[.I.. t.{.Xe| +00000310 28 6f 2a f1 aa 3f 35 91 b9 88 79 27 a0 f3 e7 41 |(o*..?5...y'...A| +00000320 9a a5 77 be 55 5e 70 89 37 b6 4a 7b 3b 8c df ad |..w.U^p.7.J{;...| +00000330 47 cc ac 45 47 43 05 05 ad c9 7b d8 1d d6 a8 fa |G..EGC....{.....| +00000340 38 45 c3 54 35 0c 28 a1 29 be 1f 73 98 a6 02 01 |8E.T5.(.)..s....| +00000350 fb 9d 12 64 1a 9c f3 82 e5 3f f6 0c 20 67 59 72 |...d.....?.. gYr| +00000360 3f a7 59 4e ef b4 58 ba 49 4e c9 b6 ea 95 b2 b3 |?.YN..X.IN......| +00000370 78 16 03 03 00 3a 0d 00 00 36 03 01 02 40 00 2e |x....:...6...@..| 00000380 04 03 05 03 06 03 08 07 08 08 08 09 08 0a 08 0b |................| 00000390 08 04 08 05 08 06 04 01 05 01 06 01 03 03 02 03 |................| 000003a0 03 01 02 01 03 02 02 02 04 02 05 02 06 02 00 00 |................| @@ -113,28 +114,28 @@ 00000200 e4 fa cc b1 8a ce e2 23 a0 87 f0 e1 67 51 eb 16 |.......#....gQ..| 00000210 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd 62 |...%...! /.}.G.b| 00000220 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 cf |C.(.._.).0......| -00000230 c2 ed 90 99 5f 58 cb 3b 74 16 03 03 00 93 0f 00 |...._X.;t.......| -00000240 00 8f 04 03 00 8b 30 81 88 02 42 01 a3 80 63 a4 |......0...B...c.| -00000250 49 60 35 5c 06 87 9f 7f ae 40 37 d0 64 58 b2 60 |I`5\.....@7.dX.`| -00000260 61 59 8b 6d a6 d9 55 67 81 f6 7e 9c de 40 69 00 |aY.m..Ug..~..@i.| -00000270 42 e1 2e 67 0d 48 cf 23 a7 28 f2 e0 9e 26 61 20 |B..g.H.#.(...&a | -00000280 4f 50 b6 e3 85 0b f5 f6 96 ec 03 32 35 02 42 01 |OP.........25.B.| -00000290 bd 3b 3e d6 03 98 b5 09 9a aa a9 73 1d 98 6a a5 |.;>........s..j.| -000002a0 29 ff e5 b3 f8 1f 58 2a a1 92 ea 2d 6d e9 71 d3 |).....X*...-m.q.| -000002b0 ce 04 46 ef d6 fb e4 5e e3 70 19 01 1a 22 85 70 |..F....^.p...".p| -000002c0 06 ec 15 bd ff ef af ed 2b 6d 00 97 cb 67 1f cb |........+m...g..| -000002d0 f6 14 03 03 00 01 01 16 03 03 00 28 00 00 00 00 |...........(....| -000002e0 00 00 00 00 80 da 31 59 19 74 39 ff a4 8c 30 1b |......1Y.t9...0.| -000002f0 25 fd 6b 25 57 ef 26 57 bd a1 df 90 4e d6 6a 47 |%.k%W.&W....N.jG| -00000300 96 08 54 92 |..T.| +00000230 c2 ed 90 99 5f 58 cb 3b 74 16 03 03 00 91 0f 00 |...._X.;t.......| +00000240 00 8d 04 03 00 89 30 81 86 02 41 66 64 90 bc df |......0...Afd...| +00000250 a5 d0 19 89 2b ed fc a5 8f 7e 14 d0 9f a2 07 6b |....+....~.....k| +00000260 d3 09 07 46 f8 29 4d b5 6c 01 e5 2e 0d d8 a4 b9 |...F.)M.l.......| +00000270 1a 86 2f b1 10 4c 29 5b de e7 29 e6 b9 32 53 ca |../..L)[..)..2S.| +00000280 d0 fc 7b a1 82 6e 34 2f 11 7a 2b 98 02 41 74 a4 |..{..n4/.z+..At.| +00000290 51 21 0c 57 ac 99 d1 a3 8c 86 f6 f2 b8 66 b8 1f |Q!.W.........f..| +000002a0 2d db 49 1a c1 34 e6 02 fd ce 50 14 7c 9b a4 52 |-.I..4....P.|..R| +000002b0 17 bc 96 ab 11 5f 97 9a 7f be ab 26 f7 1f 2b cf |....._.....&..+.| +000002c0 30 f1 da 80 b5 82 a0 da 44 be c1 00 51 1d b4 14 |0.......D...Q...| +000002d0 03 03 00 01 01 16 03 03 00 28 00 00 00 00 00 00 |.........(......| +000002e0 00 00 39 c2 3d 4e 74 16 e2 8c 4b f9 11 38 94 12 |..9.=Nt...K..8..| +000002f0 8f d3 16 18 9b ad 41 ef c9 ed 56 7f e3 ed d7 e5 |......A...V.....| +00000300 0e 52 |.R| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 28 0e 1f ce 8a 46 |..........(....F| -00000010 77 28 6d e2 fc c4 e4 39 70 6b ab 6e 14 14 2a 34 |w(m....9pk.n..*4| -00000020 13 a0 5b c5 95 f5 fa a9 a2 f6 60 20 e3 1f c5 84 |..[.......` ....| -00000030 9c 00 5e |..^| +00000000 14 03 03 00 01 01 16 03 03 00 28 c0 9a 2a 35 ef |..........(..*5.| +00000010 fa 87 1f 74 0a e9 b7 ea 3c 1c ab 1c ce 6e bb 95 |...t....<....n..| +00000020 ef 92 f3 cb 07 c0 e6 af b1 2a 60 fb 09 2a d7 68 |.........*`..*.h| +00000030 27 b0 f1 |'..| >>> Flow 5 (client to server) -00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 88 d1 a4 |................| -00000010 c9 1b a6 a5 21 4d 93 e8 04 8b 3b 69 a9 2f bd 7d |....!M....;i./.}| -00000020 97 c1 3d 15 03 03 00 1a 00 00 00 00 00 00 00 02 |..=.............| -00000030 ba 48 db 22 9e ae d8 a9 24 b7 a6 52 13 92 68 d6 |.H."....$..R..h.| -00000040 aa b5 |..| +00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 b1 98 56 |...............V| +00000010 38 68 a7 d0 da c6 83 4b 00 31 40 d7 1e 81 35 1a |8h.....K.1@...5.| +00000020 2f e3 42 15 03 03 00 1a 00 00 00 00 00 00 00 02 |/.B.............| +00000030 1d 8f a1 cf 12 2f 53 37 4d 60 46 90 e2 db 97 ce |...../S7M`F.....| +00000040 3e 99 |>.| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-Ed25519 b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-Ed25519 index edb433fefc5fc8..9eb522b86b968e 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-Ed25519 +++ b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-Ed25519 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 68 ad d5 4c 88 |....]...Y..h..L.| -00000010 60 b8 3b b4 ea e4 f2 55 7d 0f 90 d8 bd 55 b5 53 |`.;....U}....U.S| -00000020 49 6e f7 50 e4 05 70 a2 6d 9e a5 20 d1 0a e5 58 |In.P..p.m.. ...X| -00000030 38 f1 82 94 a7 c3 bf 77 60 d8 51 c2 c7 e9 8e d6 |8......w`.Q.....| -00000040 94 ee 4f 23 51 d3 2c 52 06 12 4b 4a cc a8 00 00 |..O#Q.,R..KJ....| +00000000 16 03 03 00 5d 02 00 00 59 03 03 7f e3 92 d1 05 |....]...Y.......| +00000010 fd a2 f7 b4 24 f8 e6 e2 2f 15 51 f2 4b 72 12 59 |....$.../.Q.Kr.Y| +00000020 e6 af fe c9 0b 86 8b 53 a3 ce ff 20 68 97 1d e3 |.......S... h...| +00000030 6d 6c fb 67 78 05 ba fc 10 5b 83 87 8c 04 e9 57 |ml.gx....[.....W| +00000040 19 32 3f ee 88 a0 30 61 de 1c d2 2d cc a8 00 00 |.2?...0a...-....| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -61,18 +62,18 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 4e 55 f3 |............ NU.| -000002d0 5f 83 c8 88 d4 f2 5c 75 6a ab 0e 25 df 12 ef f0 |_.....\uj..%....| -000002e0 ea e9 3e b4 4f 2c cd bb 43 5b 64 21 12 08 04 00 |..>.O,..C[d!....| -000002f0 80 84 5e ca 36 b2 fa d0 29 47 1c c2 de 4e 1d 81 |..^.6...)G...N..| -00000300 14 66 0a 26 6f 21 5f d5 b5 54 2e 37 f1 7e b1 90 |.f.&o!_..T.7.~..| -00000310 a0 57 ab 40 81 cf e5 77 29 23 54 fa 12 76 1a 4a |.W.@...w)#T..v.J| -00000320 66 c0 52 08 07 3c 3c 56 06 08 02 df 9f 75 75 5e |f.R..<| +000002d0 dd fc 30 52 61 80 f3 e8 7e e2 fb f8 03 ab e4 86 |..0Ra...~.......| +000002e0 bc 8b 69 5b f5 8c fd 4b fc 12 2e 99 1a 08 04 00 |..i[...K........| +000002f0 80 43 60 48 4c 88 e9 fc 78 36 5f a9 6a 61 54 7e |.C`HL...x6_.jaT~| +00000300 a8 a3 67 91 df ff 78 66 d6 b6 ce 95 26 af 8e ac |..g...xf....&...| +00000310 42 fa 95 e1 a3 c6 9b 73 bb 7b dc e6 aa ca 6f a0 |B......s.{....o.| +00000320 17 6b ee 50 ff d9 20 ed 11 c4 e5 81 23 1d 6b 02 |.k.P.. .....#.k.| +00000330 a8 e6 f6 ef fc 1f 69 9e c1 28 d9 bd 36 98 ad f6 |......i..(..6...| +00000340 f3 40 5a e8 a4 e9 3c 62 b2 8d 53 a1 88 06 e6 81 |.@Z...>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 20 10 cb 1b e5 55 |.......... ....U| -00000010 64 49 55 0a bf 25 e4 b0 d0 45 e1 47 b0 05 d0 f2 |dIU..%...E.G....| -00000020 41 b7 f4 6a 84 4e 6a 66 8d 0e 43 |A..j.Njf..C| +00000000 14 03 03 00 01 01 16 03 03 00 20 91 2c 24 55 e8 |.......... .,$U.| +00000010 70 9e 88 07 53 e1 60 9e 86 af 53 d4 21 d9 99 ad |p...S.`...S.!...| +00000020 3a 2b 83 b0 bc f7 93 5e 63 04 50 |:+.....^c.P| >>> Flow 5 (client to server) -00000000 17 03 03 00 16 4a f3 ed 07 22 0d 7d 9b a6 a8 3c |.....J...".}...<| -00000010 88 58 f3 48 c2 f1 aa ea c7 cd 14 15 03 03 00 12 |.X.H............| -00000020 0b 1d 53 98 b3 69 99 e1 cd 4f 37 3c dc 63 17 9f |..S..i...O7<.c..| -00000030 00 7f |..| +00000000 17 03 03 00 16 e0 69 be 8e c5 4d 63 0e 77 88 42 |......i...Mc.w.B| +00000010 4e a8 e9 8f f4 c5 05 56 99 92 70 15 03 03 00 12 |N......V..p.....| +00000020 64 5c 05 8f 23 af e8 aa f6 76 1b b6 90 36 d9 d5 |d\..#....v...6..| +00000030 59 e6 |Y.| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-AES256-GCM-SHA384 b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-AES256-GCM-SHA384 index c6673f6dc0081c..e9f6604b2f5cac 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-AES256-GCM-SHA384 +++ b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-AES256-GCM-SHA384 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 60 13 bb 4b ce |....]...Y..`..K.| -00000010 d6 49 3b b2 3a 0c d4 5a e8 25 26 67 41 fd dd 0f |.I;.:..Z.%&gA...| -00000020 62 ec c1 ab bc 58 92 67 9d 71 fd 20 20 4c ab 4d |b....X.g.q. L.M| -00000030 85 ce 87 49 68 8c cf 9c 2f 7b f9 2b 96 62 8a 75 |...Ih.../{.+.b.u| -00000040 c3 21 05 c6 ab b6 98 5e e7 e5 d8 4a c0 30 00 00 |.!.....^...J.0..| +00000000 16 03 03 00 5d 02 00 00 59 03 03 bc b7 27 59 a9 |....]...Y....'Y.| +00000010 a7 ac ce 8b f5 5b 13 56 60 19 70 2a a7 f0 a2 ea |.....[.V`.p*....| +00000020 1a ce 27 2a 67 53 e0 5c 7e 5e 80 20 d5 c6 de 6d |..'*gS.\~^. ...m| +00000030 d1 3c 6d cb c9 49 77 00 cc 31 87 0a 86 45 06 70 |...| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 85 c8 0e |............ ...| -000002d0 fb 1e b3 4a ec c1 b2 3a b8 4e bb a1 ca 80 8e 13 |...J...:.N......| -000002e0 f5 08 b5 9a b8 3e c4 6c 23 cf 17 a0 31 08 04 00 |.....>.l#...1...| -000002f0 80 28 db ec 9d f9 f4 9a 06 12 8a 32 89 11 4a e5 |.(.........2..J.| -00000300 b6 68 98 21 ec f1 75 9c f9 28 bc b1 99 5b ec 77 |.h.!..u..(...[.w| -00000310 37 8b 6b 05 ad b3 10 de 24 05 84 a4 16 fe 1d 84 |7.k.....$.......| -00000320 af 5d 28 7d d7 bc de 2f 81 da a3 79 fb c0 a5 43 |.](}.../...y...C| -00000330 b6 3b 8d 90 30 00 80 98 16 83 ac 50 43 c7 3e 76 |.;..0......PC.>v| -00000340 c5 dc 85 c2 19 75 a6 90 f8 2f 71 40 d9 a0 73 7e |.....u.../q@..s~| -00000350 fe 3b 1c d7 66 73 0f f0 42 c5 00 b1 2b be 41 2b |.;..fs..B...+.A+| -00000360 40 da 8f b0 f1 9e b0 0a 83 23 04 80 fb 4c fe 7e |@........#...L.~| -00000370 b4 16 03 03 00 3a 0d 00 00 36 03 01 02 40 00 2e |.....:...6...@..| +000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 8b ea d3 |............ ...| +000002d0 e4 1f c2 21 cb 32 99 00 d2 a9 a4 f3 e4 9a d7 6e |...!.2.........n| +000002e0 f0 6e 7c 45 ca 41 51 60 a3 31 61 f0 34 08 04 00 |.n|E.AQ`.1a.4...| +000002f0 80 96 62 58 0c b4 72 a7 49 1c b8 73 72 5a c7 ba |..bX..r.I..srZ..| +00000300 e2 fe dc a6 a4 54 3d 21 83 70 ab ef de d8 d8 db |.....T=!.p......| +00000310 82 56 22 4f 9d f6 04 c2 b6 f7 9d a0 41 88 53 c9 |.V"O........A.S.| +00000320 2b 57 80 71 a2 bd d7 84 b6 4b cd 65 3d c3 10 cb |+W.q.....K.e=...| +00000330 5d 89 03 af 46 d5 b0 39 b8 c8 2d 74 1b 46 8d 0d |]...F..9..-t.F..| +00000340 ca 13 30 34 54 10 ea ec 77 ca 84 a6 2c 88 68 d6 |..04T...w...,.h.| +00000350 69 e6 f3 95 aa 65 af 05 7d e0 c2 84 48 f5 fa ae |i....e..}...H...| +00000360 d5 90 2a 79 0c 45 c6 7d d3 b7 de 85 e8 b5 2b 68 |..*y.E.}......+h| +00000370 bf 16 03 03 00 3a 0d 00 00 36 03 01 02 40 00 2e |.....:...6...@..| 00000380 04 03 05 03 06 03 08 07 08 08 08 09 08 0a 08 0b |................| 00000390 08 04 08 05 08 06 04 01 05 01 06 01 03 03 02 03 |................| 000003a0 03 01 02 01 03 02 02 02 04 02 05 02 06 02 00 00 |................| @@ -113,26 +114,26 @@ 00000200 e5 35 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 |.5....%...! /.}.| 00000210 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 |G.bC.(.._.).0...| 00000220 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 16 03 03 00 |......._X.;t....| -00000230 88 0f 00 00 84 08 04 00 80 41 83 55 9b 3c 8d 29 |.........A.U.<.)| -00000240 ed 6c 58 91 85 cb 9b 19 df 61 09 d0 d3 73 fb 2b |.lX......a...s.+| -00000250 3b 4b 4b bd 64 7e e6 53 87 cb 6d 2f 78 1c 67 cf |;KK.d~.S..m/x.g.| -00000260 13 21 e0 ec 1b d7 d8 aa 06 a0 0e d5 86 6f ba 1e |.!...........o..| -00000270 97 e2 8c 9b 86 e2 8a 4d 62 1a bd 35 0e 2d 63 18 |.......Mb..5.-c.| -00000280 a1 1f 7c ed fc 3b 89 c2 00 6d da 01 42 0a 47 95 |..|..;...m..B.G.| -00000290 12 8e 01 82 62 58 f9 96 eb a9 aa c3 f8 96 ec 20 |....bX......... | -000002a0 e1 65 a9 46 a8 af 4b 7c aa 29 ee 9c 48 18 b3 4a |.e.F..K|.)..H..J| -000002b0 00 7f 79 97 87 27 92 ca 45 14 03 03 00 01 01 16 |..y..'..E.......| -000002c0 03 03 00 28 00 00 00 00 00 00 00 00 da f3 01 f2 |...(............| -000002d0 76 e2 df 95 82 0d cc 0c 7e b0 cc 15 69 ba 8c 64 |v.......~...i..d| -000002e0 ac dc f3 45 58 a3 8f ed f0 2a 43 ce |...EX....*C.| +00000230 88 0f 00 00 84 08 04 00 80 48 ae 01 b6 4e e2 f0 |.........H...N..| +00000240 6b d9 58 e9 95 fb 93 56 10 8f 15 dc e8 3e c7 49 |k.X....V.....>.I| +00000250 b3 26 c2 06 91 4a 02 27 36 6b 21 cd aa 4a 62 6a |.&...J.'6k!..Jbj| +00000260 9d 44 6a b7 6c 16 d4 52 23 12 20 9d e5 5c 83 04 |.Dj.l..R#. ..\..| +00000270 d8 38 45 61 8b 30 5b 7b 6b 77 be 47 5a f8 fe 4d |.8Ea.0[{kw.GZ..M| +00000280 c8 e7 83 97 44 b4 fb 71 89 30 f2 cf d2 49 82 e8 |....D..q.0...I..| +00000290 d4 a8 73 86 44 29 30 17 7b b6 0e 4f 5f 6b 0c 33 |..s.D)0.{..O_k.3| +000002a0 43 7f 9b 84 a4 9b c6 30 18 ff 1b 85 a7 a9 17 23 |C......0.......#| +000002b0 2e 08 d3 57 10 af 49 95 2a 14 03 03 00 01 01 16 |...W..I.*.......| +000002c0 03 03 00 28 00 00 00 00 00 00 00 00 43 b8 4c 8a |...(........C.L.| +000002d0 84 1e 6c 41 02 fb b6 74 1e 4d 69 0d c0 f8 fc 8b |..lA...t.Mi.....| +000002e0 ce 64 53 95 40 c8 e8 52 31 5f a3 65 |.dS.@..R1_.e| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 28 e4 27 5e 80 00 |..........(.'^..| -00000010 ee db 03 4c 17 d1 9c bf f5 68 5f eb a6 fb 0c 3f |...L.....h_....?| -00000020 4f c4 38 84 3e c2 e9 f4 9b 06 39 5f 89 06 3a bd |O.8.>.....9_..:.| -00000030 77 db 25 |w.%| +00000000 14 03 03 00 01 01 16 03 03 00 28 04 f3 da a1 b9 |..........(.....| +00000010 b9 1a cf f3 a8 19 aa 23 12 65 8d 68 dc 37 09 9a |.......#.e.h.7..| +00000020 e4 0c cd e5 d4 3d cb 42 89 e4 ad 9e 49 34 de 05 |.....=.B....I4..| +00000030 74 f0 33 |t.3| >>> Flow 5 (client to server) -00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 2b 98 2b |.............+.+| -00000010 d6 dc d2 69 03 c9 0e 44 46 d1 5e d2 5a 14 89 7b |...i...DF.^.Z..{| -00000020 60 ab 62 15 03 03 00 1a 00 00 00 00 00 00 00 02 |`.b.............| -00000030 b9 7a 17 4b 10 5f 25 90 14 75 bb aa 42 de 7b 63 |.z.K._%..u..B.{c| -00000040 69 e8 |i.| +00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 00 33 c0 |..............3.| +00000010 3f 9c 56 3e 9b ed bb 5e 81 0c 4a 01 65 91 f5 da |?.V>...^..J.e...| +00000020 f6 4b 62 15 03 03 00 1a 00 00 00 00 00 00 00 02 |.Kb.............| +00000030 55 6f 8d 18 50 1f aa 00 94 9b 79 26 f0 3e 0a c6 |Uo..P.....y&.>..| +00000040 45 b0 |E.| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-ECDSA b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-ECDSA index 7b2a3631768e72..25120914e850cd 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-ECDSA +++ b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-ECDSA @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 6d b7 f7 cf 1d |....]...Y..m....| -00000010 f9 c0 02 cb ee 90 23 93 41 8e 26 24 3e 74 31 ce |......#.A.&$>t1.| -00000020 4f 53 f8 9d 0f 94 02 b2 66 c9 87 20 d6 5c 35 52 |OS......f.. .\5R| -00000030 4b b1 f2 bb 2e 1d 95 ff 7d 83 f0 58 a8 0a ed b1 |K.......}..X....| -00000040 54 25 03 ca ea 7b 8d 1a 8f 9f 43 51 c0 09 00 00 |T%...{....CQ....| +00000000 16 03 03 00 5d 02 00 00 59 03 03 80 e4 9b 9d a9 |....]...Y.......| +00000010 59 27 7f 44 b9 4b 90 31 c7 66 03 90 d6 9b ef 17 |Y'.D.K.1.f......| +00000020 f8 fb e5 63 04 68 f6 ce a1 a6 f4 20 cb 84 55 fa |...c.h..... ..U.| +00000030 41 77 48 0f 0a 5a 24 92 e3 59 b2 d9 91 0c 18 4a |AwH..Z$..Y.....J| +00000040 bf af ad db 64 db cb 57 d8 0b de 46 c0 09 00 00 |....d..W...F....| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 0e 0b 00 02 0a 00 02 07 00 02 |................| 00000070 04 30 82 02 00 30 82 01 62 02 09 00 b8 bf 2d 47 |.0...0..b.....-G| @@ -57,17 +58,17 @@ 00000250 c9 86 2e dd d7 11 69 7f 85 7c 56 de fb 31 78 2b |......i..|V..1x+| 00000260 e4 c7 78 0d ae cb be 9e 4e 36 24 31 7b 6a 0f 39 |..x.....N6$1{j.9| 00000270 95 12 07 8f 2a 16 03 03 00 b6 0c 00 00 b2 03 00 |....*...........| -00000280 1d 20 04 b4 79 b4 2c 1d 0f b3 4b ff 67 e7 24 88 |. ..y.,...K.g.$.| -00000290 d6 db 4f 1e 66 da 0e f2 89 5a 53 ed 4e ba ad 4c |..O.f....ZS.N..L| -000002a0 81 0a 04 03 00 8a 30 81 87 02 42 01 fb 16 53 43 |......0...B...SC| -000002b0 2b 86 61 0a 58 a0 68 c1 cd 2c ff ec 79 7f 83 fa |+.a.X.h..,..y...| -000002c0 cc 0b 24 9d 98 54 d0 dc 90 55 e1 b3 e6 48 69 1a |..$..T...U...Hi.| -000002d0 55 62 f4 da 8f 60 db f7 76 80 d5 4d 37 f6 43 49 |Ub...`..v..M7.CI| -000002e0 95 3d 96 f6 e2 fd a4 07 ae 24 8c fa bd 02 41 20 |.=.......$....A | -000002f0 a1 50 78 a3 dd 99 c0 cf 74 f1 c0 79 b1 13 9d bc |.Px.....t..y....| -00000300 0b 37 cf 7c 09 11 b8 a4 71 65 e8 be ff 3a b9 85 |.7.|....qe...:..| -00000310 cd b4 30 f8 1f d6 2e 83 96 6c 01 3e d2 00 a7 5b |..0......l.>...[| -00000320 23 c6 d0 69 eb 90 49 e3 46 ed 45 96 3b 07 d4 a8 |#..i..I.F.E.;...| +00000280 1d 20 2f 6b 4c 95 75 59 25 2f f7 fd b1 1f d2 9e |. /kL.uY%/......| +00000290 2f 38 5e 57 1a 7c 36 99 1e 11 4a 3f f7 13 5b 0b |/8^W.|6...J?..[.| +000002a0 90 33 04 03 00 8a 30 81 87 02 42 01 b4 34 3f d0 |.3....0...B..4?.| +000002b0 41 09 00 12 f4 79 20 f4 b7 80 5c d1 35 9d 8b d2 |A....y ...\.5...| +000002c0 fa c9 4a a9 44 6b 05 95 c5 a7 50 08 0d 73 66 3e |..J.Dk....P..sf>| +000002d0 dd 49 e4 a4 c6 c0 12 ca 0b 4a df bc c1 3e ec 88 |.I.......J...>..| +000002e0 ec 9a 0e 71 15 4d 45 98 04 3a 51 7a 67 02 41 15 |...q.ME..:Qzg.A.| +000002f0 17 de b0 5c 03 a5 74 0e 0f 2b 53 6e 55 17 73 b8 |...\..t..+SnU.s.| +00000300 ac 16 70 1a 95 f9 25 b1 fc 4b 9d c7 b1 f4 71 f6 |..p...%..K....q.| +00000310 86 2d 5b 74 9e d3 4e 1b 40 67 f4 a6 62 2e c8 4d |.-[t..N.@g..b..M| +00000320 66 f7 32 e9 05 df d5 b0 e8 1a b7 b1 48 c4 1c 91 |f.2.........H...| 00000330 16 03 03 00 3a 0d 00 00 36 03 01 02 40 00 2e 04 |....:...6...@...| 00000340 03 05 03 06 03 08 07 08 08 08 09 08 0a 08 0b 08 |................| 00000350 04 08 05 08 06 04 01 05 01 06 01 03 03 02 03 03 |................| @@ -109,31 +110,31 @@ 00000200 e5 35 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 |.5....%...! /.}.| 00000210 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 |G.bC.(.._.).0...| 00000220 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 16 03 03 00 |......._X.;t....| -00000230 88 0f 00 00 84 08 04 00 80 3f 4a c2 4f 36 85 f0 |.........?J.O6..| -00000240 d0 c6 b6 8f f1 cc 45 c1 2f f2 c7 24 1e 0f 04 dc |......E./..$....| -00000250 f5 af 6e 38 eb aa a6 6f 36 f4 80 dd 78 78 a7 d4 |..n8...o6...xx..| -00000260 50 3a df e1 23 c4 3f 58 df 1a c0 1d 57 a5 46 3b |P:..#.?X....W.F;| -00000270 5d 09 ac 62 63 28 8a a2 b5 d4 9b 88 7c b9 4d b4 |]..bc(......|.M.| -00000280 66 b2 9d 53 6e 15 9c f2 9b c7 14 ca 19 7f 00 38 |f..Sn..........8| -00000290 81 a3 7b 44 e8 3d 6d 54 0f b3 81 fd 82 07 4d a1 |..{D.=mT......M.| -000002a0 3e 8c 30 34 cd 6e 55 96 58 bf 86 8b 9c f6 be 94 |>.04.nU.X.......| -000002b0 f4 a8 7e 4d 7f 03 07 7e 98 14 03 03 00 01 01 16 |..~M...~........| +00000230 88 0f 00 00 84 08 04 00 80 70 66 f2 ac fb f7 29 |.........pf....)| +00000240 15 31 a2 12 de 37 3f cc 97 74 5c 5b 7e 6d e4 f8 |.1...7?..t\[~m..| +00000250 b0 b3 3d 9c ee 32 bf d7 64 90 d7 af ad 8f 61 77 |..=..2..d.....aw| +00000260 f2 c0 7e 6f 91 1d 4e 95 92 3e ab 23 f0 ac d8 de |..~o..N..>.#....| +00000270 32 69 cd bc 04 4c d1 a3 77 7a af ac f0 64 41 aa |2i...L..wz...dA.| +00000280 a0 53 f0 89 89 a4 6f 1f 67 21 16 55 4e dc cb a8 |.S....o.g!.UN...| +00000290 12 7d cb a0 5c a9 48 48 d9 af 03 f0 75 ed 32 72 |.}..\.HH....u.2r| +000002a0 d5 da 34 a3 ea 82 08 8f 00 fc 7d 1d b8 11 ff f7 |..4.......}.....| +000002b0 09 52 a8 cc a8 66 b0 06 1e 14 03 03 00 01 01 16 |.R...f..........| 000002c0 03 03 00 40 00 00 00 00 00 00 00 00 00 00 00 00 |...@............| -000002d0 00 00 00 00 c7 88 1e 15 dd 36 31 22 0f 30 d1 4d |.........61".0.M| -000002e0 40 2e 3a dd 05 cc fd a8 d2 ea f8 d9 79 1d 07 46 |@.:.........y..F| -000002f0 2c 80 ab ab 54 3c 10 5a a7 79 d2 1c 16 18 94 eb |,...T<.Z.y......| -00000300 46 69 cc 03 |Fi..| +000002d0 00 00 00 00 9c 7d d4 9e 59 53 e7 66 64 63 9c cb |.....}..YS.fdc..| +000002e0 58 03 03 26 fe d9 15 eb 03 1c 8f a7 9c 5a 86 4a |X..&.........Z.J| +000002f0 6c 4e 06 4a 80 91 94 00 6f 7d 38 6a ea a3 68 df |lN.J....o}8j..h.| +00000300 17 08 14 ed |....| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 40 9e 65 27 5b 92 |..........@.e'[.| -00000010 1e 2b 1a bc 81 ab 85 29 51 c1 38 04 b6 97 e5 4b |.+.....)Q.8....K| -00000020 b1 7d a5 e2 6d e7 b1 1a 33 6c f1 3d a4 9c de 2d |.}..m...3l.=...-| -00000030 b3 8a 01 da cc f1 d7 83 b1 1e 84 cb b7 e7 fe e6 |................| -00000040 26 83 b0 2d 6f a9 77 46 55 44 7a |&..-o.wFUDz| +00000000 14 03 03 00 01 01 16 03 03 00 40 7e 6d 15 90 c5 |..........@~m...| +00000010 99 96 ca bb 16 b5 5a 55 3e b0 ef 3f ab e5 b6 8c |......ZU>..?....| +00000020 51 1d 18 c7 c2 25 86 e0 db c1 d0 38 85 51 4e 8d |Q....%.....8.QN.| +00000030 37 51 92 cc d0 64 37 b7 67 7b 2c fc e7 1e 16 f6 |7Q...d7.g{,.....| +00000040 76 3a 94 48 68 eb dc cc cf 2a 4d |v:.Hh....*M| >>> Flow 5 (client to server) 00000000 17 03 03 00 30 00 00 00 00 00 00 00 00 00 00 00 |....0...........| -00000010 00 00 00 00 00 e2 55 06 b8 6f 63 c4 63 78 76 4b |......U..oc.cxvK| -00000020 c8 63 8b 4b c6 11 2c ff dc fc 20 f7 52 fe fa 5f |.c.K..,... .R.._| -00000030 e3 45 3a f2 a1 15 03 03 00 30 00 00 00 00 00 00 |.E:......0......| -00000040 00 00 00 00 00 00 00 00 00 00 0e cb 88 2f 1f be |............./..| -00000050 9c 76 4d db 75 7f eb 01 ae bd 76 28 07 41 49 6c |.vM.u.....v(.AIl| -00000060 4c 82 84 d5 fc d3 75 f4 4b 81 |L.....u.K.| +00000010 00 00 00 00 00 db 14 ae 78 62 50 37 42 b8 fc c6 |........xbP7B...| +00000020 f1 84 40 75 7c e4 3f 8a 57 b8 1c 12 4a 6f 11 f2 |..@u|.?.W...Jo..| +00000030 ba 1a a6 9b 20 15 03 03 00 30 00 00 00 00 00 00 |.... ....0......| +00000040 00 00 00 00 00 00 00 00 00 00 09 93 aa 80 fd b2 |................| +00000050 66 e2 83 0e f8 83 45 3d e1 39 06 5d a3 12 9e 12 |f.....E=.9.]....| +00000060 fd f5 cb 32 c4 3b ce 20 e4 10 |...2.;. ..| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-RSA b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-RSA index 25fc245b650812..3bfb4aaabd829a 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-RSA +++ b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-RSA @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 5f da ab 10 86 |....]...Y.._....| -00000010 4c 91 60 4c ec 28 20 d4 37 84 c4 d0 87 bb 9d a9 |L.`L.( .7.......| -00000020 5a c0 86 81 d9 91 38 5a ef 3f ae 20 ec 14 b6 de |Z.....8Z.?. ....| -00000030 b7 cf 59 09 43 dc 59 3f 74 89 10 f2 82 ea 60 46 |..Y.C.Y?t.....`F| -00000040 8f a3 6a 0f 61 0f 2d 66 24 c3 fb 7f c0 2f 00 00 |..j.a.-f$..../..| +00000000 16 03 03 00 5d 02 00 00 59 03 03 c0 dd 18 ad 41 |....]...Y......A| +00000010 2b ee fd 45 1f c0 c1 10 75 a7 1d 80 74 3e 8e 66 |+..E....u...t>.f| +00000020 18 c8 64 c2 b4 89 4f 19 ff e0 0c 20 be c6 2a 8e |..d...O.... ..*.| +00000030 73 0a 1e 7a 2a d2 81 71 11 ba e9 de 02 0a aa 52 |s..z*..q.......R| +00000040 76 b7 43 a3 49 a0 81 24 9c 57 4c 26 c0 2f 00 00 |v.C.I..$.WL&./..| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -61,18 +62,18 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 2c c4 b9 |............ ,..| -000002d0 13 19 85 99 cb 09 f3 5c 3a a1 b2 16 ee 77 6a cd |.......\:....wj.| -000002e0 23 6c 08 00 7e 6b c0 67 a8 cd 8b 62 64 08 04 00 |#l..~k.g...bd...| -000002f0 80 90 74 bf ce f7 94 f1 94 b1 95 04 4a ab d6 08 |..t.........J...| -00000300 23 11 8d ce 4e 7b 36 ee 67 d2 26 af 11 0e 33 58 |#...N{6.g.&...3X| -00000310 12 cb e5 3b 5b 5c f3 d1 82 19 2e 55 6b 98 05 6e |...;[\.....Uk..n| -00000320 3c 5e 04 f5 3e df 76 c3 e5 46 e7 a6 bb f7 d4 d5 |<^..>.v..F......| -00000330 e8 fd 20 d3 ce 3e 9d a0 90 9a 10 52 a7 04 c9 e9 |.. ..>.....R....| -00000340 28 6f 3f ff e1 0a 10 d0 25 04 ea e4 84 4c b7 80 |(o?.....%....L..| -00000350 e4 fe a7 a6 da db cc b0 e0 62 80 10 fc 03 16 94 |.........b......| -00000360 f2 dd 0d 3f aa 11 45 af e3 e4 7b e6 bc d6 75 91 |...?..E...{...u.| -00000370 99 16 03 03 00 3a 0d 00 00 36 03 01 02 40 00 2e |.....:...6...@..| +000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 5a f3 c3 |............ Z..| +000002d0 1b 34 9a 1a 01 68 65 fd 25 e5 e4 62 f4 17 d6 00 |.4...he.%..b....| +000002e0 88 ae 5e fa 6b ef 60 96 e9 96 1e 33 0f 08 04 00 |..^.k.`....3....| +000002f0 80 74 91 43 7a 3e 02 93 c6 6c 6d 36 86 2f 74 51 |.t.Cz>...lm6./tQ| +00000300 50 84 a2 0b 1c b4 90 71 72 2b f9 7d 35 2a a6 fc |P......qr+.}5*..| +00000310 56 b1 fa 36 e0 cb ea 5a 4e 89 71 3f 28 d3 fc 90 |V..6...ZN.q?(...| +00000320 de 21 0c 03 84 d9 23 78 b6 58 f2 03 02 27 48 f9 |.!....#x.X...'H.| +00000330 6d 6c 2b eb 62 36 47 66 55 fc d9 77 42 1e 9b 93 |ml+.b6GfU..wB...| +00000340 00 0f 5a 71 76 af 2c d9 b7 c3 6e e8 7a 64 34 0f |..Zqv.,...n.zd4.| +00000350 78 36 d7 cf a1 bb 3d 0a 23 64 c4 70 f0 78 8e 42 |x6....=.#d.p.x.B| +00000360 80 42 1f 0f 1d 7f c9 b6 7b 9c 2a 30 6c 7a ef 0c |.B......{.*0lz..| +00000370 2f 16 03 03 00 3a 0d 00 00 36 03 01 02 40 00 2e |/....:...6...@..| 00000380 04 03 05 03 06 03 08 07 08 08 08 09 08 0a 08 0b |................| 00000390 08 04 08 05 08 06 04 01 05 01 06 01 03 03 02 03 |................| 000003a0 03 01 02 01 03 02 02 02 04 02 05 02 06 02 00 00 |................| @@ -113,26 +114,26 @@ 00000200 e5 35 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 |.5....%...! /.}.| 00000210 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 |G.bC.(.._.).0...| 00000220 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 16 03 03 00 |......._X.;t....| -00000230 88 0f 00 00 84 08 04 00 80 4d c1 8a 3d 2b 1f a0 |.........M..=+..| -00000240 ad 6f 1c b6 44 7c cc 0d 21 6f e8 31 51 58 51 fe |.o..D|..!o.1QXQ.| -00000250 84 f3 41 40 12 48 29 0d 20 16 d1 6a 4b 38 0b c8 |..A@.H). ..jK8..| -00000260 b3 70 3f 4a ca 17 da 05 30 ec 6c 4f 55 e2 ee e5 |.p?J....0.lOU...| -00000270 9a 8c a7 9f c1 96 79 cf 53 b6 67 c8 cb 67 d3 23 |......y.S.g..g.#| -00000280 07 8a 3f ef df e5 e6 84 f6 5f 5e 23 57 89 cc 77 |..?......_^#W..w| -00000290 99 ed 3d 58 8a ec eb bb 64 62 d1 44 7f b6 45 85 |..=X....db.D..E.| -000002a0 89 06 90 56 48 98 8a 43 93 64 53 df 3a 8c b3 93 |...VH..C.dS.:...| -000002b0 91 81 7b 60 95 8f 99 99 eb 14 03 03 00 01 01 16 |..{`............| -000002c0 03 03 00 28 00 00 00 00 00 00 00 00 dd 93 96 02 |...(............| -000002d0 47 0e 1d 2f 6a 76 12 e5 56 53 8f 7f c1 cd f3 5e |G../jv..VS.....^| -000002e0 3c 94 35 1b fe d5 33 97 b2 5b a3 29 |<.5...3..[.)| +00000230 88 0f 00 00 84 08 04 00 80 56 a2 bb f3 05 94 90 |.........V......| +00000240 46 bc ef 79 ea 74 f9 a4 5c 27 59 dd 93 53 c3 9f |F..y.t..\'Y..S..| +00000250 07 b0 bf c1 e2 dc 39 b9 0d 63 62 d1 d0 88 65 e4 |......9..cb...e.| +00000260 2b 28 d3 ad b1 78 60 4c 0d a6 1f 2e a1 bc 36 81 |+(...x`L......6.| +00000270 31 f0 aa 0b 52 e6 04 69 ba 8a 9d c1 fd 25 d3 50 |1...R..i.....%.P| +00000280 43 5b ed fc 78 21 2b cc 13 f8 db f1 8b e7 57 d8 |C[..x!+.......W.| +00000290 59 f3 cf 54 ab d0 ea d6 8c 61 32 79 b8 be a6 c7 |Y..T.....a2y....| +000002a0 87 0a 35 88 70 f7 3f 2d bd 1b 7c 57 35 32 1b 81 |..5.p.?-..|W52..| +000002b0 87 08 e3 b0 1f a1 2b 13 fb 14 03 03 00 01 01 16 |......+.........| +000002c0 03 03 00 28 00 00 00 00 00 00 00 00 49 94 ed a1 |...(........I...| +000002d0 a3 13 e0 f4 20 2a 63 4d ef 24 d3 6c 8d 6c de b3 |.... *cM.$.l.l..| +000002e0 92 16 21 0d b0 6c 64 df 1b 32 ca dc |..!..ld..2..| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 28 de 60 8a ea b5 |..........(.`...| -00000010 74 66 35 35 39 bb 62 ab 17 7d 91 c3 1a 5a e5 6c |tf559.b..}...Z.l| -00000020 49 14 c8 bc bd d6 b6 7e 2f 8e d4 40 5a ae ee 89 |I......~/..@Z...| -00000030 89 ae af |...| +00000000 14 03 03 00 01 01 16 03 03 00 28 b9 2c 96 43 03 |..........(.,.C.| +00000010 f9 8c 28 34 4e 8b 58 ba d7 1a c4 87 d8 67 5c 0f |..(4N.X......g\.| +00000020 36 36 67 54 4e f4 6c 9c 0c 0e 1f df ca a3 8d ab |66gTN.l.........| +00000030 31 f2 84 |1..| >>> Flow 5 (client to server) -00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 3c 92 2d |.............<.-| -00000010 44 23 5b eb 34 dc 9a 4d c2 fe a7 88 cb 6f 8d 62 |D#[.4..M.....o.b| -00000020 4b 2a bf 15 03 03 00 1a 00 00 00 00 00 00 00 02 |K*..............| -00000030 90 82 82 54 5d 63 6e 8b 50 71 6e e8 fd 75 96 9e |...T]cn.Pqn..u..| -00000040 d0 0d |..| +00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 de 90 16 |................| +00000010 8e ec 2c 60 b8 a7 4f b6 26 8a c9 78 ff e0 73 19 |..,`..O.&..x..s.| +00000020 b3 01 c1 15 03 03 00 1a 00 00 00 00 00 00 00 02 |................| +00000030 8a 4b 2c 4b 99 51 81 27 34 28 63 00 15 24 e1 47 |.K,K.Q.'4(c..$.G| +00000040 4a 2f |J/| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-RSAPKCS1v15 b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-RSAPKCS1v15 index 6500aec8b01514..b933f592befe72 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-RSAPKCS1v15 +++ b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-RSAPKCS1v15 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 7b 21 bd ee 4b |....]...Y..{!..K| -00000010 af e3 88 cb 18 15 e5 f3 ef a5 a7 b3 a1 66 06 b2 |.............f..| -00000020 f1 cc d7 36 7e 18 d1 f6 2e 3e cd 20 34 09 01 9f |...6~....>. 4...| -00000030 c6 80 10 43 c1 ed e6 c1 29 1c ed ac 61 36 37 4f |...C....)...a67O| -00000040 8e 00 44 9d b9 cb 51 0b a2 9c 64 be c0 2f 00 00 |..D...Q...d../..| +00000000 16 03 03 00 5d 02 00 00 59 03 03 0e 33 06 1f 9d |....]...Y...3...| +00000010 3d 0e 59 21 38 7d 4c ea a2 2a 2a c7 99 7b e9 bf |=.Y!8}L..**..{..| +00000020 61 e2 44 a1 10 5e 5d a1 e4 fd 90 20 c1 f0 1c 8e |a.D..^].... ....| +00000030 00 2f d9 a0 45 23 76 35 e0 01 2e 49 b8 4d b4 76 |./..E#v5...I.M.v| +00000040 08 65 3f cd 8b dc 80 ea fd 31 9f 47 c0 2f 00 00 |.e?......1.G./..| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -61,18 +62,18 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 d3 22 bb |............ .".| -000002d0 c4 42 90 54 0a 43 f6 26 06 c1 ad 71 c8 82 ba 03 |.B.T.C.&...q....| -000002e0 9b cd be a4 a8 04 5a 30 69 ec b5 c9 79 04 01 00 |......Z0i...y...| -000002f0 80 ab c7 ca 24 9b db 7d 8c 81 c4 c4 46 49 2a 45 |....$..}....FI*E| -00000300 69 31 1b dc ef 01 ce 9f e4 da cf 6c 04 4c e3 4e |i1.........l.L.N| -00000310 16 84 05 fe 48 f4 21 60 fc d4 e1 6c 48 8c 87 2d |....H.!`...lH..-| -00000320 1f 56 2c ad 88 2b 5c 8d 4f 36 93 d6 a3 b1 32 4a |.V,..+\.O6....2J| -00000330 ef 0f e6 db 82 1c f2 ea 38 08 2a 62 8b a3 bd 4e |........8.*b...N| -00000340 8b 2a ae eb 0e e5 f1 88 ff 3c de f8 ed d7 c3 07 |.*.......<......| -00000350 05 92 bb e5 6d 15 23 c8 54 19 a6 cf d1 4f e2 b1 |....m.#.T....O..| -00000360 a3 4d ff 6f 22 32 5b 58 f7 58 c1 9f 58 59 b5 e2 |.M.o"2[X.X..XY..| -00000370 7b 16 03 03 00 0c 0d 00 00 08 01 01 00 02 04 01 |{...............| +000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 09 ce 63 |............ ..c| +000002d0 9c 52 37 83 ef 21 78 60 eb 8b dd 22 91 fb 34 4f |.R7..!x`..."..4O| +000002e0 ee 04 ef 48 75 2f 49 7e 3f 0b 00 62 15 04 01 00 |...Hu/I~?..b....| +000002f0 80 a0 11 99 d2 bc a9 fd 59 ad 38 20 bb 44 85 8e |........Y.8 .D..| +00000300 89 35 2e 42 ff a2 87 81 86 f5 e3 6c 9d 84 2a cf |.5.B.......l..*.| +00000310 0d cc 6a c0 ce 31 01 91 48 78 75 23 24 3e 3d 93 |..j..1..Hxu#$>=.| +00000320 bf ad c9 49 9d 63 66 cd 4b cc 92 0f 6d 64 2c 80 |...I.cf.K...md,.| +00000330 71 22 bf 6d 62 8e 8a f7 19 6f 50 2a 86 46 e4 46 |q".mb....oP*.F.F| +00000340 71 73 df 8c 25 84 6f 28 a7 8c bd 78 01 22 a2 91 |qs..%.o(...x."..| +00000350 f4 17 ef 88 55 9d d5 ac 42 5a 8a c8 7a 1b bf d1 |....U...BZ..z...| +00000360 d9 6d 15 42 b9 37 16 7b 2b 3c c7 58 99 da ab 98 |.m.B.7.{+<.X....| +00000370 f3 16 03 03 00 0c 0d 00 00 08 01 01 00 02 04 01 |................| 00000380 00 00 16 03 03 00 04 0e 00 00 00 |...........| >>> Flow 3 (client to server) 00000000 16 03 03 01 fd 0b 00 01 f9 00 01 f6 00 01 f3 30 |...............0| @@ -110,26 +111,26 @@ 00000200 e5 35 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 |.5....%...! /.}.| 00000210 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 |G.bC.(.._.).0...| 00000220 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 16 03 03 00 |......._X.;t....| -00000230 88 0f 00 00 84 04 01 00 80 8f 22 b5 46 ab 82 02 |..........".F...| -00000240 f8 a8 f8 63 d9 0f eb d4 2b 8d 02 de ce 79 d5 a1 |...c....+....y..| -00000250 20 bb 70 7a b2 36 6a b5 52 81 f5 ea f7 23 b5 41 | .pz.6j.R....#.A| -00000260 86 35 90 cc 64 4a 68 7e e3 ae c1 97 32 3b c2 a9 |.5..dJh~....2;..| -00000270 e0 f0 07 14 63 d5 08 15 59 46 f5 ea a6 39 5c 98 |....c...YF...9\.| -00000280 39 7e 17 d8 74 ae 05 a3 f5 3a 71 74 e8 b4 c9 a4 |9~..t....:qt....| -00000290 1c 82 04 ca fe 5f 97 23 8a c1 f9 ce d1 5d 0e 81 |....._.#.....]..| -000002a0 da 5f e1 b6 76 80 3c cf 9f 19 79 cf 33 d0 0a fe |._..v.<...y.3...| -000002b0 19 fc 2d 9a bb 24 cd d4 79 14 03 03 00 01 01 16 |..-..$..y.......| -000002c0 03 03 00 28 00 00 00 00 00 00 00 00 28 78 b8 0b |...(........(x..| -000002d0 59 d8 a8 11 21 ed 45 74 02 00 50 33 42 b9 0a d2 |Y...!.Et..P3B...| -000002e0 8c 9c a3 45 6e b5 9d 3e 5b 30 a9 2e |...En..>[0..| +00000230 88 0f 00 00 84 04 01 00 80 01 83 0b 8b bf a3 50 |...............P| +00000240 03 89 e8 24 9b 9e 2e f5 7b 72 58 b1 b1 ba b2 64 |...$....{rX....d| +00000250 9f a9 e2 b4 00 78 79 50 ec 55 18 43 e5 c4 a4 76 |.....xyP.U.C...v| +00000260 c8 21 bc 7e e0 c4 2d 78 85 d7 33 ca d7 4d e0 f1 |.!.~..-x..3..M..| +00000270 cd 6d bd 17 2f a0 b9 39 93 29 ba d7 09 4f bb 08 |.m../..9.)...O..| +00000280 b0 b0 05 f1 83 de d0 db 39 f9 b4 34 a3 ca d3 76 |........9..4...v| +00000290 97 35 f6 ea cd 53 fe bb 94 99 30 e3 a2 0a 29 1a |.5...S....0...).| +000002a0 5f f4 28 0e 8c 80 fc 75 84 a4 e2 3a 29 a9 50 be |_.(....u...:).P.| +000002b0 11 99 bb a5 12 64 5c af 5f 14 03 03 00 01 01 16 |.....d\._.......| +000002c0 03 03 00 28 00 00 00 00 00 00 00 00 8e 51 58 0e |...(.........QX.| +000002d0 1a fd 8e 01 f1 6b 59 94 54 ab 00 be 45 8b 4c ae |.....kY.T...E.L.| +000002e0 70 c3 db 53 de 82 12 6c cb df 32 72 |p..S...l..2r| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 28 f8 f5 85 19 36 |..........(....6| -00000010 be 6e 58 45 66 04 c4 f9 e7 fd 50 55 3b 07 6c 50 |.nXEf.....PU;.lP| -00000020 16 8c 2e 72 cf be 78 0a 8a 82 91 ed c1 72 10 d3 |...r..x......r..| -00000030 cf 42 b5 |.B.| +00000000 14 03 03 00 01 01 16 03 03 00 28 7b 7f 81 c9 d7 |..........({....| +00000010 1d 3a 32 48 da 92 d6 9f 78 6e 2f d3 e9 7e 9f 12 |.:2H....xn/..~..| +00000020 9b 0b e1 00 e8 2f 4b 63 2c 3d 94 e4 f8 1e 06 7d |...../Kc,=.....}| +00000030 a7 7e 6a |.~j| >>> Flow 5 (client to server) -00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 53 4b a2 |.............SK.| -00000010 e4 5f 51 70 46 3d e3 41 28 8b 82 02 e2 27 8d eb |._QpF=.A(....'..| -00000020 42 65 7f 15 03 03 00 1a 00 00 00 00 00 00 00 02 |Be..............| -00000030 f3 31 ed 2c ca e5 13 e8 93 cd d7 51 4b c5 16 88 |.1.,.......QK...| -00000040 c0 a4 |..| +00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 ee 21 1b |..............!.| +00000010 45 15 6a 0c 79 12 4d 79 d2 26 db fc e4 da 0f e4 |E.j.y.My.&......| +00000020 20 7e c7 15 03 03 00 1a 00 00 00 00 00 00 00 02 | ~..............| +00000030 2a 46 62 6b 5e c3 14 56 e1 fb d0 61 f5 e8 7b 2f |*Fbk^..V...a..{/| +00000040 cc fd |..| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-RSAPSS b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-RSAPSS index b4c5102535dcee..646026793f0a58 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-RSAPSS +++ b/src/crypto/tls/testdata/Client-TLSv12-ClientCert-RSA-RSAPSS @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 0d 73 c5 43 a1 |....]...Y...s.C.| -00000010 71 b6 21 da 7a f6 76 44 60 8a 84 ec 6b 1d 70 01 |q.!.z.vD`...k.p.| -00000020 d8 b2 29 48 db 5d ea da ea 79 bb 20 3f b9 66 fe |..)H.]...y. ?.f.| -00000030 73 db fc 90 18 3f ac e5 1f 5d 86 24 80 75 ec a3 |s....?...].$.u..| -00000040 c6 aa e9 05 c1 47 21 bd 30 f6 41 43 c0 2f 00 00 |.....G!.0.AC./..| +00000000 16 03 03 00 5d 02 00 00 59 03 03 cd 47 5d a7 66 |....]...Y...G].f| +00000010 aa b1 e1 bd ed 83 bd 3d 4a b7 b2 08 b8 79 08 c5 |.......=J....y..| +00000020 22 7b 8d be fe 55 4c 2d b5 4c b9 20 13 26 2a e6 |"{...UL-.L. .&*.| +00000030 ab 60 4c 53 8c b7 27 c4 d5 12 ff 38 64 b1 a2 1d |.`LS..'....8d...| +00000040 4f a4 c7 28 53 95 01 8f 7e 4a 7b 02 c0 2f 00 00 |O..(S...~J{../..| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 66 0b 00 02 62 00 02 5f 00 02 |......f...b.._..| 00000070 5c 30 82 02 58 30 82 01 8d a0 03 02 01 02 02 11 |\0..X0..........| @@ -62,17 +63,17 @@ 000002a0 fa b1 3f 14 11 89 66 79 d1 8e 88 0e 0b a0 9e 30 |..?...fy.......0| 000002b0 2a c0 67 ef ca 46 02 88 e9 53 81 22 69 22 97 ad |*.g..F...S."i"..| 000002c0 80 93 d4 f7 dd 70 14 24 d7 70 0a 46 a1 16 03 03 |.....p.$.p.F....| -000002d0 00 ac 0c 00 00 a8 03 00 1d 20 03 05 56 dd 40 c0 |......... ..V.@.| -000002e0 b6 4d 5a df 26 e7 4a f1 a8 47 ef af 12 6e 5f 01 |.MZ.&.J..G...n_.| -000002f0 2a 7a 19 1a 45 52 8c 47 d2 53 08 04 00 80 8a 6a |*z..ER.G.S.....j| -00000300 9d 8b 38 73 da 92 bc f6 05 79 90 af 7a 43 59 62 |..8s.....y..zCYb| -00000310 bc 97 b6 af ef ce 5f 59 07 81 93 bc c5 3c 5f f9 |......_Y.....<_.| -00000320 4e 04 45 74 5e cc 7a 6f 82 7a cf 86 0d 68 c9 35 |N.Et^.zo.z...h.5| -00000330 1d 62 f0 3c ee 77 b5 4c 3a 40 ec 89 fc 97 ff a6 |.b.<.w.L:@......| -00000340 34 13 8b bc 6b 83 92 e8 52 c5 c6 42 c9 25 ad 37 |4...k...R..B.%.7| -00000350 41 e7 5a 52 d8 0f 7d fd a8 9e 86 c6 1b b2 8b 50 |A.ZR..}........P| -00000360 2a 1c 15 56 00 d8 a8 85 86 05 28 7f a1 3e ba f2 |*..V......(..>..| -00000370 fe 92 07 c8 a4 4a 2a 5f d1 53 82 09 9c 65 16 03 |.....J*_.S...e..| +000002d0 00 ac 0c 00 00 a8 03 00 1d 20 a8 94 95 94 cc 16 |......... ......| +000002e0 f1 dc 11 b6 c5 52 b3 fe 95 4d 28 34 1f 2d 87 ec |.....R...M(4.-..| +000002f0 8d 56 51 0e 5b 92 03 f9 60 0d 08 04 00 80 09 3a |.VQ.[...`......:| +00000300 99 39 d8 3f 63 b4 86 12 ba ce e6 6d d6 22 aa d8 |.9.?c......m."..| +00000310 31 60 86 eb 48 cc c0 87 97 a5 0d c7 42 86 cc f8 |1`..H.......B...| +00000320 f7 f9 d5 1a 34 86 ec 41 02 08 99 c2 93 9b d1 b5 |....4..A........| +00000330 57 a4 84 3c a8 93 0a c9 f7 1e a1 33 07 ca 60 c4 |W..<.......3..`.| +00000340 4b 40 32 20 a6 22 3f 6d 4e 28 22 3b 52 70 9a 4d |K@2 ."?mN(";Rp.M| +00000350 fa 84 67 88 ba b5 0d 92 eb 23 94 90 00 e2 68 74 |..g......#....ht| +00000360 5f 13 a3 d4 2d fb 24 f2 32 09 b1 e3 63 15 39 b1 |_...-.$.2...c.9.| +00000370 9e 1e a6 61 fd 8d c6 cf 1f f7 f7 43 0c 64 16 03 |...a.......C.d..| 00000380 03 00 0c 0d 00 00 08 01 01 00 02 08 04 00 00 16 |................| 00000390 03 03 00 04 0e 00 00 00 |........| >>> Flow 3 (client to server) @@ -118,26 +119,26 @@ 00000270 10 00 00 21 20 2f e5 7d a3 47 cd 62 43 15 28 da |...! /.}.G.bC.(.| 00000280 ac 5f bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 |._.).0..........| 00000290 5f 58 cb 3b 74 16 03 03 00 88 0f 00 00 84 08 04 |_X.;t...........| -000002a0 00 80 2e 4c af b1 f2 9c de 14 92 81 c4 7d d1 2f |...L.........}./| -000002b0 a3 ba ba df 6f 26 da 2d 86 b4 1f 60 cd fe e7 c1 |....o&.-...`....| -000002c0 77 02 a5 58 3f 6a fc 1d ef 93 23 0a 9c c9 70 93 |w..X?j....#...p.| -000002d0 f8 0c 67 fd 58 83 96 85 62 da 31 e7 a6 78 9f 1c |..g.X...b.1..x..| -000002e0 dc 02 b3 b8 bc 05 26 92 e8 6a e7 3c db 5f 80 ee |......&..j.<._..| -000002f0 12 75 ff e6 36 36 e6 75 c2 77 d9 63 ef 60 89 db |.u..66.u.w.c.`..| -00000300 21 aa 27 85 11 9b 8c eb be a3 e3 34 0e 01 93 ca |!.'........4....| -00000310 a6 65 e3 cd 0f b8 b5 c6 f0 d7 15 69 db 4b 57 d2 |.e.........i.KW.| -00000320 05 0d 14 03 03 00 01 01 16 03 03 00 28 00 00 00 |............(...| -00000330 00 00 00 00 00 b5 09 a0 c4 55 89 df 8a 60 83 f3 |.........U...`..| -00000340 82 12 55 1d d5 50 11 81 74 0f cd b5 ed 9a 00 e7 |..U..P..t.......| -00000350 58 f7 65 0e 8d |X.e..| +000002a0 00 80 86 14 3f 28 40 db 2b 02 d6 56 d3 58 d0 77 |....?(@.+..V.X.w| +000002b0 58 f8 a3 25 1f 18 d1 9c 0a d2 56 09 32 59 34 db |X..%......V.2Y4.| +000002c0 d9 2e 0a bf b4 c2 0a 86 ee 6e 29 56 53 aa 82 11 |.........n)VS...| +000002d0 96 6d 86 29 7c 85 a4 71 56 04 71 5f e6 68 3f 3f |.m.)|..qV.q_.h??| +000002e0 71 54 13 a2 22 b4 3e 38 fa c2 96 a4 c4 ab 83 c6 |qT..".>8........| +000002f0 f5 ca 7a ea 2c 8d 62 4d 76 d6 db 9c f0 0e 8c 2a |..z.,.bMv......*| +00000300 38 2f 9d 24 34 f4 33 73 54 a2 44 9a e3 4c 1f a7 |8/.$4.3sT.D..L..| +00000310 ca 49 16 e8 35 29 29 d1 48 95 0a 38 19 64 4c e7 |.I..5)).H..8.dL.| +00000320 0d a5 14 03 03 00 01 01 16 03 03 00 28 00 00 00 |............(...| +00000330 00 00 00 00 00 09 7f fa f5 c5 e5 b1 46 7e 3d e2 |............F~=.| +00000340 d4 51 a7 b8 ae 0a 56 a3 8c 45 40 97 d4 c5 81 4e |.Q....V..E@....N| +00000350 6f 12 50 b3 f6 |o.P..| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 28 3f eb 87 50 b0 |..........(?..P.| -00000010 ae c2 fd 68 30 95 94 61 1c 78 1e 60 3e 38 17 ef |...h0..a.x.`>8..| -00000020 ed 9b f1 10 26 5e 53 a3 8c 46 2e 6c 24 b2 ab a7 |....&^S..F.l$...| -00000030 6c f0 ea |l..| +00000000 14 03 03 00 01 01 16 03 03 00 28 0b 0d 27 bb 5f |..........(..'._| +00000010 c5 fa 62 08 8f 7a 24 65 90 bc de be 28 ae a4 e2 |..b..z$e....(...| +00000020 1f ac 18 7a 83 3e 90 bb 76 2f 73 ec 35 ca d3 e7 |...z.>..v/s.5...| +00000030 d8 25 7f |.%.| >>> Flow 5 (client to server) -00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 fe de 40 |...............@| -00000010 52 50 79 da 5f 31 52 52 d8 96 04 35 86 92 03 0b |RPy._1RR...5....| -00000020 6f 5f 04 15 03 03 00 1a 00 00 00 00 00 00 00 02 |o_..............| -00000030 9c 85 24 b0 a9 17 24 cf 18 f6 ef f5 0d c6 77 6d |..$...$.......wm| -00000040 e0 b2 |..| +00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 a7 07 a5 |................| +00000010 b5 4c 26 d0 d1 6b 52 10 8f 7c 3a 4c 31 ce 74 fa |.L&..kR..|:L1.t.| +00000020 4a c7 9b 15 03 03 00 1a 00 00 00 00 00 00 00 02 |J...............| +00000030 22 22 5a b9 ef 4f 05 9f 2e 1e 9c 11 e1 6c 0b 1e |""Z..O.......l..| +00000040 bd 10 |..| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES index 774cb40e3dc931..6268dbcc7a4b61 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES +++ b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 15 ed ca 52 59 |....]...Y.....RY| -00000010 e2 56 80 e2 f3 20 5b 7d 52 f3 67 1f 01 0d ad 30 |.V... [}R.g....0| -00000020 36 a6 43 c2 ed 5e 34 c5 24 89 e1 20 c7 4c 5d 06 |6.C..^4.$.. .L].| -00000030 01 7a 80 aa 84 0c 46 48 63 36 49 4a 9b 3f 8d ba |.z....FHc6IJ.?..| -00000040 78 1b cd f9 ef 03 df cd ee 2c b1 79 c0 09 00 00 |x........,.y....| +00000000 16 03 03 00 5d 02 00 00 59 03 03 61 e9 cc eb 72 |....]...Y..a...r| +00000010 7c 11 18 0e 9d 6b 72 e9 be 84 f3 92 8d 5f e6 4c ||....kr......_.L| +00000020 ec 9e f0 00 ca 00 6f af 90 5b de 20 29 97 82 6d |......o..[. )..m| +00000030 57 ac 60 59 41 1d bb 29 b1 12 a4 ac 60 18 3b 51 |W.`YA..)....`.;Q| +00000040 af 5c e7 80 1a dc 9d 72 ed 3a 74 96 c0 09 00 00 |.\.....r.:t.....| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 0e 0b 00 02 0a 00 02 07 00 02 |................| 00000070 04 30 82 02 00 30 82 01 62 02 09 00 b8 bf 2d 47 |.0...0..b.....-G| @@ -56,39 +57,39 @@ 00000240 8c 25 c1 33 13 83 0d 94 06 bb d4 37 7a f6 ec 7a |.%.3.......7z..z| 00000250 c9 86 2e dd d7 11 69 7f 85 7c 56 de fb 31 78 2b |......i..|V..1x+| 00000260 e4 c7 78 0d ae cb be 9e 4e 36 24 31 7b 6a 0f 39 |..x.....N6$1{j.9| -00000270 95 12 07 8f 2a 16 03 03 00 b6 0c 00 00 b2 03 00 |....*...........| -00000280 1d 20 b7 f5 92 bc 9b 4a f9 62 3f 46 b8 b0 33 56 |. .....J.b?F..3V| -00000290 1e 8d fc e9 68 af bf 67 9b a5 d0 9e a9 cc d9 65 |....h..g.......e| -000002a0 7e 41 04 03 00 8a 30 81 87 02 41 08 b2 fe 6e 40 |~A....0...A...n@| -000002b0 b7 5f 04 db eb 39 18 d5 e0 46 81 07 33 3a 29 fb |._...9...F..3:).| -000002c0 0b 19 76 ac 95 0d cd 4d a4 f0 99 df 6d d6 73 79 |..v....M....m.sy| -000002d0 ed 48 52 78 ea 7a 1d 4b b1 67 d6 fd 21 3b c9 9d |.HRx.z.K.g..!;..| -000002e0 89 67 b7 b9 8e cf 35 28 62 28 3f 0b 02 42 01 7e |.g....5(b(?..B.~| -000002f0 4c a5 af 5b ba 3b d6 38 3b 34 b7 0c 69 27 92 ce |L..[.;.8;4..i'..| -00000300 34 ba bb 0d f6 32 65 fe 3c c0 9c 9d 92 88 b4 11 |4....2e.<.......| -00000310 04 88 f5 b3 21 79 6c ed 6e f6 51 f5 e7 b1 3d 0d |....!yl.n.Q...=.| -00000320 bd d6 92 67 c8 90 71 7a db cc cc 21 64 44 22 d4 |...g..qz...!dD".| -00000330 16 03 03 00 04 0e 00 00 00 |.........| +00000270 95 12 07 8f 2a 16 03 03 00 b7 0c 00 00 b3 03 00 |....*...........| +00000280 1d 20 88 7d 69 d3 1b 2d b5 dd 4e 0a f4 98 f3 8c |. .}i..-..N.....| +00000290 42 d7 ee e2 7c 28 dc f2 75 ec b0 9e ca ad cb 21 |B...|(..u......!| +000002a0 59 34 04 03 00 8b 30 81 88 02 42 01 32 2c 84 f3 |Y4....0...B.2,..| +000002b0 d8 7c 05 58 ac d0 25 46 42 11 b4 7c 0a cb ee 83 |.|.X..%FB..|....| +000002c0 7f 68 36 b6 be f8 d2 0a 8d d7 bc 8a 95 d8 2d b3 |.h6...........-.| +000002d0 fa 87 56 ae 4c 01 3f bf 29 d1 81 c1 5d e9 f0 93 |..V.L.?.)...]...| +000002e0 f6 32 91 e7 da 18 84 dd 8d a4 94 3f 81 02 42 01 |.2.........?..B.| +000002f0 70 96 e1 3c c9 aa b8 72 8d 8e 13 68 00 50 65 9c |p..<...r...h.Pe.| +00000300 cf c6 1e 5f 5a d0 80 e8 8b ff 24 df a9 55 42 d9 |..._Z.....$..UB.| +00000310 75 83 67 d7 20 69 f1 29 2d 15 f4 83 f9 ff 3d c8 |u.g. i.)-.....=.| +00000320 0e 79 a9 3e 86 38 4a c5 a2 93 f1 47 a2 43 41 a5 |.y.>.8J....G.CA.| +00000330 bb 16 03 03 00 04 0e 00 00 00 |..........| >>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| 00000030 16 03 03 00 40 00 00 00 00 00 00 00 00 00 00 00 |....@...........| -00000040 00 00 00 00 00 07 03 9e fa a4 84 27 17 b8 dd 53 |...........'...S| -00000050 12 4b 89 a1 47 98 22 e3 7b c8 d4 5c 18 bb 89 09 |.K..G.".{..\....| -00000060 29 3a b9 af d3 38 4a 40 76 d6 15 37 80 19 9c ce |):...8J@v..7....| -00000070 32 5b 94 89 2e |2[...| +00000040 00 00 00 00 00 84 08 13 72 05 4e a7 cb ac c1 40 |........r.N....@| +00000050 9e 31 9c a4 8f 7f f1 41 c6 63 19 39 3d d4 27 74 |.1.....A.c.9=.'t| +00000060 f9 5c 59 88 15 c3 e2 a7 6a 6f 6d fb 52 6f 94 36 |.\Y.....jom.Ro.6| +00000070 1e 8d 47 50 db |..GP.| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 40 2d 53 b0 b0 c1 |..........@-S...| -00000010 05 78 e3 3c 5d 57 a5 04 9a 4b b9 f7 56 ce 5a 76 |.x.<]W...K..V.Zv| -00000020 84 4d f2 46 2e fc c2 9c ec 37 2b 4d 99 88 ab 30 |.M.F.....7+M...0| -00000030 1b 83 7d d8 72 bf e7 c7 f7 cf b8 13 54 d5 a2 a2 |..}.r.......T...| -00000040 dc 76 b3 01 a4 d0 a5 cc eb 66 29 |.v.......f)| +00000000 14 03 03 00 01 01 16 03 03 00 40 9f 65 82 54 e1 |..........@.e.T.| +00000010 50 68 92 33 d1 f2 a1 1d 95 81 e1 df c3 b2 4c 9b |Ph.3..........L.| +00000020 f4 8d ee b2 4b 43 2f e9 e3 af 60 ae d4 4c 7a 4b |....KC/...`..LzK| +00000030 f5 03 22 f0 11 58 de 4d ac 46 5a bf 98 da 01 1d |.."..X.M.FZ.....| +00000040 fa 19 ad a6 48 66 fa c1 25 4b a0 |....Hf..%K.| >>> Flow 5 (client to server) 00000000 17 03 03 00 30 00 00 00 00 00 00 00 00 00 00 00 |....0...........| -00000010 00 00 00 00 00 11 31 80 89 b8 32 e7 66 a6 eb c5 |......1...2.f...| -00000020 ca 4b f7 2b 47 73 e9 cd 9f 4e 9a 5e 3f 99 5a 29 |.K.+Gs...N.^?.Z)| -00000030 00 0d 4f 4e 93 15 03 03 00 30 00 00 00 00 00 00 |..ON.....0......| -00000040 00 00 00 00 00 00 00 00 00 00 d9 18 f4 dd 64 64 |..............dd| -00000050 38 27 a7 96 d5 14 d1 b2 26 9c 14 96 54 2a cf d7 |8'......&...T*..| -00000060 4c d0 b6 c3 20 fe 33 73 78 e5 |L... .3sx.| +00000010 00 00 00 00 00 b3 26 24 39 5d 7c fc 25 23 ee fc |......&$9]|.%#..| +00000020 3a 6a 38 08 f5 bf 5e cc dc 34 83 1f 5e af b0 b4 |:j8...^..4..^...| +00000030 ee 34 fc f5 e1 15 03 03 00 30 00 00 00 00 00 00 |.4.......0......| +00000040 00 00 00 00 00 00 00 00 00 00 84 b2 cc 06 95 78 |...............x| +00000050 de 1d a0 cc d3 15 32 b7 89 74 8d 99 70 13 37 c3 |......2..t..p.7.| +00000060 e2 99 6e fe 8c 2d 69 77 cc 93 |..n..-iw..| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES-GCM b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES-GCM index 86095340d40e2d..e235f27b75f992 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES-GCM +++ b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES-GCM @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 e1 ed 2c 5d 15 |....]...Y....,].| -00000010 15 24 d2 e3 a0 58 fb 2f 89 a3 26 b1 c8 06 0f dc |.$...X./..&.....| -00000020 ef e1 a7 a4 81 4a 71 2c 1d a2 35 20 be ff 22 db |.....Jq,..5 ..".| -00000030 2d 33 1b 11 24 55 96 96 54 5f 62 c7 7f b5 ba 32 |-3..$U..T_b....2| -00000040 e7 e6 7c 11 68 f8 97 d1 17 f7 8f 4f c0 2b 00 00 |..|.h......O.+..| +00000000 16 03 03 00 5d 02 00 00 59 03 03 ac 28 cc 34 03 |....]...Y...(.4.| +00000010 c8 91 48 22 79 a2 98 74 e4 55 df 72 6b 41 cb 40 |..H"y..t.U.rkA.@| +00000020 58 ba 04 93 e7 f3 1c 8e 0a 19 d1 20 98 93 cb 7c |X.......... ...|| +00000030 a3 57 a1 c7 ca 19 13 5c e1 ac c4 44 17 30 cb a3 |.W.....\...D.0..| +00000040 b2 83 81 90 82 1d 37 c3 71 ae b8 45 c0 2b 00 00 |......7.q..E.+..| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 0e 0b 00 02 0a 00 02 07 00 02 |................| 00000070 04 30 82 02 00 30 82 01 62 02 09 00 b8 bf 2d 47 |.0...0..b.....-G| @@ -57,33 +58,33 @@ 00000250 c9 86 2e dd d7 11 69 7f 85 7c 56 de fb 31 78 2b |......i..|V..1x+| 00000260 e4 c7 78 0d ae cb be 9e 4e 36 24 31 7b 6a 0f 39 |..x.....N6$1{j.9| 00000270 95 12 07 8f 2a 16 03 03 00 b6 0c 00 00 b2 03 00 |....*...........| -00000280 1d 20 74 55 23 1a ff ed d3 e7 95 30 ee ed 6d 77 |. tU#......0..mw| -00000290 57 83 fb 09 fc 81 70 69 d8 ae 38 1b e7 da 7f c6 |W.....pi..8.....| -000002a0 e4 01 04 03 00 8a 30 81 87 02 41 36 52 ca 91 b7 |......0...A6R...| -000002b0 a2 b2 31 91 98 96 65 92 67 a6 9c f2 75 8a 00 91 |..1...e.g...u...| -000002c0 8e 17 bd 22 d2 6c cb 16 0a fb 6b 27 08 3b f9 e4 |...".l....k'.;..| -000002d0 7c b5 21 45 60 4c c0 71 0b 28 46 d2 a8 a2 ad ad ||.!E`L.q.(F.....| -000002e0 51 fd bc 2e 40 7d 06 89 e7 b8 07 df 02 42 00 86 |Q...@}.......B..| -000002f0 83 4f 1d c1 c0 6d 3c f3 ed 24 47 70 65 e9 7c 5e |.O...m<..$Gpe.|^| -00000300 27 43 f0 05 b8 cd a4 34 ba 69 bc 81 7c fb b1 c3 |'C.....4.i..|...| -00000310 f8 bc 99 d6 90 15 8f db 21 fe 9f 18 10 23 75 6d |........!....#um| -00000320 22 b2 c6 f4 a5 60 80 31 06 a3 ad 33 88 c4 52 5d |"....`.1...3..R]| +00000280 1d 20 40 9c d1 ec 0a 11 08 c3 fa a0 33 80 00 62 |. @.........3..b| +00000290 5b 65 59 c6 4f 4e 32 8a 79 fc df 63 2f ee f6 59 |[eY.ON2.y..c/..Y| +000002a0 40 4a 04 03 00 8a 30 81 87 02 41 39 af 63 7a fa |@J....0...A9.cz.| +000002b0 dc e3 40 85 19 9d 1b 8f ca 55 26 e4 00 a1 f5 01 |..@......U&.....| +000002c0 73 e6 b5 6f ab bb c3 f3 b4 e0 c2 a1 33 37 ae ec |s..o........37..| +000002d0 e7 90 f6 dd b4 a4 c8 f1 3c 51 67 94 c1 ee d7 f3 |........>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| -00000030 16 03 03 00 28 00 00 00 00 00 00 00 00 6b 3d 83 |....(........k=.| -00000040 88 db aa b0 62 30 8d 4a 6c b2 f1 14 d3 99 43 65 |....b0.Jl.....Ce| -00000050 4c ef 26 a4 ba 46 2f 75 9a 05 0c f2 91 |L.&..F/u.....| +00000030 16 03 03 00 28 00 00 00 00 00 00 00 00 50 16 81 |....(........P..| +00000040 fe 51 5b ac de d7 31 1b 9a 09 4f 1e d4 de a4 af |.Q[...1...O.....| +00000050 c9 62 5d 38 85 23 84 8c ae 4f 55 5d 0d |.b]8.#...OU].| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 28 d5 c2 de 26 c1 |..........(...&.| -00000010 ed 28 f1 d8 81 b3 9a 4f 3d 82 ad 22 d0 2c 41 61 |.(.....O=..".,Aa| -00000020 8b 15 a3 72 01 94 44 ff 8f 01 c1 05 e6 84 56 0a |...r..D.......V.| -00000030 8b 1d c4 |...| +00000000 14 03 03 00 01 01 16 03 03 00 28 d3 0d f6 53 0f |..........(...S.| +00000010 c8 3e f0 e6 c0 2c ed ec bc 8f cc 19 5e 33 30 69 |.>...,......^30i| +00000020 3c d9 55 8b e4 14 e2 0e 3a f9 89 39 fc a7 89 70 |<.U.....:..9...p| +00000030 58 16 44 |X.D| >>> Flow 5 (client to server) -00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 c7 7d a7 |..............}.| -00000010 c4 a5 a1 21 94 7a 23 bc 05 44 7d 4a 2f 16 dc 95 |...!.z#..D}J/...| -00000020 25 6e 17 15 03 03 00 1a 00 00 00 00 00 00 00 02 |%n..............| -00000030 77 fa b6 34 7e 60 77 f9 a5 09 d1 39 b0 1e 66 86 |w..4~`w....9..f.| -00000040 50 41 |PA| +00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 0b 26 61 |..............&a| +00000010 e6 d8 6a 6d db c6 2d 8a c3 8e f4 17 1a f7 8e f3 |..jm..-.........| +00000020 ce 04 ce 15 03 03 00 1a 00 00 00 00 00 00 00 02 |................| +00000030 fa f7 0a 81 bb 8d 83 f3 82 05 ee 26 69 22 90 b2 |...........&i"..| +00000040 bb b1 |..| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES128-SHA256 b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES128-SHA256 index ac5d07e69d6e3a..f81845ab3d91d6 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES128-SHA256 +++ b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES128-SHA256 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 36 c3 50 72 b9 |....]...Y..6.Pr.| -00000010 0d e9 d9 76 1d 31 72 e6 61 ec 3e 13 93 d7 f1 52 |...v.1r.a.>....R| -00000020 a7 36 5c 16 9a cc c5 c0 da 7e f3 20 31 79 7a ec |.6\......~. 1yz.| -00000030 1d db f7 cd 0a fe b3 86 6f 43 12 f4 9d 19 16 51 |........oC.....Q| -00000040 e2 42 1b 94 43 13 71 67 2d 52 7a 86 c0 23 00 00 |.B..C.qg-Rz..#..| +00000000 16 03 03 00 5d 02 00 00 59 03 03 be 62 2f ca 91 |....]...Y...b/..| +00000010 8b 79 1d ce e3 3f 2e 37 bc 74 da c8 7d de a5 85 |.y...?.7.t..}...| +00000020 58 c8 04 55 4e c6 51 f2 18 60 33 20 2f 33 8b c1 |X..UN.Q..`3 /3..| +00000030 b6 be 54 3e 3f 78 12 1b d9 d0 db 71 fa 0c ee f9 |..T>?x.....q....| +00000040 60 fb 49 08 a1 83 a4 f6 22 e8 dc 2c c0 23 00 00 |`.I....."..,.#..| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 0e 0b 00 02 0a 00 02 07 00 02 |................| 00000070 04 30 82 02 00 30 82 01 62 02 09 00 b8 bf 2d 47 |.0...0..b.....-G| @@ -56,43 +57,43 @@ 00000240 8c 25 c1 33 13 83 0d 94 06 bb d4 37 7a f6 ec 7a |.%.3.......7z..z| 00000250 c9 86 2e dd d7 11 69 7f 85 7c 56 de fb 31 78 2b |......i..|V..1x+| 00000260 e4 c7 78 0d ae cb be 9e 4e 36 24 31 7b 6a 0f 39 |..x.....N6$1{j.9| -00000270 95 12 07 8f 2a 16 03 03 00 b7 0c 00 00 b3 03 00 |....*...........| -00000280 1d 20 d2 2d 0f 11 29 eb b4 23 67 1c 40 84 7f 7e |. .-..)..#g.@..~| -00000290 cd 8c 3d c2 62 ed 9b 0f 08 01 5b a5 fd 86 31 c5 |..=.b.....[...1.| -000002a0 d2 76 04 03 00 8b 30 81 88 02 42 01 a4 ea c8 67 |.v....0...B....g| -000002b0 c0 0a 93 23 7c d9 41 17 ba 91 02 78 c7 16 28 85 |...#|.A....x..(.| -000002c0 b8 d6 5c ed ed 20 1e 8b 90 e0 19 db 67 72 7e d3 |..\.. ......gr~.| -000002d0 2c 2e 84 6a d7 bc b7 53 74 8b 1b ef a8 bc 39 d8 |,..j...St.....9.| -000002e0 e0 75 29 6b 1d 89 c3 12 c2 3c 51 25 84 02 42 01 |.u)k.....+| +000002b0 e8 b6 0b c9 76 80 60 6f 9a c5 4c 6e 7c d8 53 00 |....v.`o..Ln|.S.| +000002c0 56 c7 f9 37 36 32 2d dc 82 d8 0c 67 7a 96 5c 34 |V..762-....gz.\4| +000002d0 db 10 71 4e f4 29 e0 20 84 01 09 9a dc 74 76 39 |..qN.). .....tv9| +000002e0 63 e3 b2 c5 b4 7b cc 20 92 77 f1 3b a0 02 41 5d |c....{. .w.;..A]| +000002f0 36 ae 86 8c 27 7f 9e 0d 17 8e 16 48 20 2a 43 ac |6...'......H *C.| +00000300 d4 ce c6 9b 6a 95 47 5a 47 5c 13 b6 eb db 0b a6 |....j.GZG\......| +00000310 fa 23 fe 0c 76 e0 a9 b2 97 76 02 ff 68 3d 68 c5 |.#..v....v..h=h.| +00000320 17 08 54 25 1a 22 c0 a3 06 bd a3 f6 7c 68 07 9e |..T%."......|h..| +00000330 16 03 03 00 04 0e 00 00 00 |.........| >>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| 00000030 16 03 03 00 50 00 00 00 00 00 00 00 00 00 00 00 |....P...........| -00000040 00 00 00 00 00 98 8d 15 68 8c 12 3e 4a 88 64 7c |........h..>J.d|| -00000050 69 46 25 16 66 20 d7 f5 7e 08 d8 9d c4 89 c1 cf |iF%.f ..~.......| -00000060 ac 86 b7 ce 3a 9c 95 0c 5d d1 a2 c2 bf fd 7c e1 |....:...].....|.| -00000070 b5 f4 04 7e 71 ae 3c fc d2 c1 2a ec 81 e4 62 a6 |...~q.<...*...b.| -00000080 e6 05 72 d8 9e |..r..| +00000040 00 00 00 00 00 27 1a a1 cf 9d f5 8f 91 e4 7f dc |.....'..........| +00000050 2f 81 3f 1e 3a c3 40 21 c2 6d dd dd 8f f6 9c 4b |/.?.:.@!.m.....K| +00000060 0b da 8d 10 f6 83 da 35 bb b9 a4 bf 2f 29 27 a5 |.......5..../)'.| +00000070 12 52 6a 74 6e 15 8d 82 cf 86 df f0 52 26 32 ca |.Rjtn.......R&2.| +00000080 02 91 8c 03 f7 |.....| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 50 27 94 9b d3 f1 |..........P'....| -00000010 6c 59 bb fa b7 73 8b d6 fa b2 ca c1 0c d3 60 cf |lY...s........`.| -00000020 50 18 a6 6f 3d b3 46 a1 06 a2 b1 62 4c ea 88 2f |P..o=.F....bL../| -00000030 95 bc 35 6f 55 a6 8a 52 69 6c 5b a5 fc 94 2a b6 |..5oU..Ril[...*.| -00000040 27 7c d7 95 27 72 d4 c1 f2 f9 9c a5 b3 2a 85 52 |'|..'r.......*.R| -00000050 ae bf 14 52 54 51 6a fe a1 99 05 |...RTQj....| +00000000 14 03 03 00 01 01 16 03 03 00 50 1f 1c 35 9b 6c |..........P..5.l| +00000010 a8 9f 58 0d bf 0f db b5 91 c6 74 de b1 b7 68 b6 |..X.......t...h.| +00000020 05 0d 39 68 87 7d b3 b0 8b 86 15 67 ba 7e 54 0c |..9h.}.....g.~T.| +00000030 0d 4f 6e f3 ad d8 27 e8 1d fa 43 31 be 30 67 d2 |.On...'...C1.0g.| +00000040 bc 0a 30 37 78 3e 30 62 bf 81 2e 53 da b3 88 73 |..07x>0b...S...s| +00000050 17 42 79 ee ef 0e d3 3c 87 c8 85 |.By....<...| >>> Flow 5 (client to server) 00000000 17 03 03 00 40 00 00 00 00 00 00 00 00 00 00 00 |....@...........| -00000010 00 00 00 00 00 9b b4 f6 7e 49 48 09 cd 4f ed 22 |........~IH..O."| -00000020 68 db 19 94 ad af de d1 9b b6 bc f5 d6 7f c6 b2 |h...............| -00000030 ff b1 0e b4 0e dd 77 19 e8 9e ff 68 1a 04 fc e5 |......w....h....| -00000040 f2 3b 5b 34 aa 15 03 03 00 40 00 00 00 00 00 00 |.;[4.....@......| -00000050 00 00 00 00 00 00 00 00 00 00 f7 95 04 54 94 ff |.............T..| -00000060 f7 86 df 0a 2e 5b e9 54 fe 13 30 2a 97 f9 21 8e |.....[.T..0*..!.| -00000070 83 8b ba 86 77 67 92 33 5e a8 ea b7 64 65 63 59 |....wg.3^...decY| -00000080 44 2e 65 60 9a 0a 7c 78 97 a4 |D.e`..|x..| +00000010 00 00 00 00 00 2c 39 43 06 0a c3 81 10 24 e8 bb |.....,9C.....$..| +00000020 de b8 74 00 a9 a3 ab 9a 2a a7 8a 0e 62 b8 2a 23 |..t.....*...b.*#| +00000030 10 46 1b 76 dc a7 81 2d 37 2e e8 5f b5 75 5e 65 |.F.v...-7.._.u^e| +00000040 73 11 ef 71 16 15 03 03 00 40 00 00 00 00 00 00 |s..q.....@......| +00000050 00 00 00 00 00 00 00 00 00 00 a5 24 00 67 c6 ae |...........$.g..| +00000060 7f 05 91 ab 17 c9 ea eb 5a f8 8f 07 f3 e5 9b 9c |........Z.......| +00000070 20 41 63 21 69 3e f1 bb 66 7e 0b 3d c0 0a 14 b1 | Ac!i>..f~.=....| +00000080 a3 de 2e d9 10 23 37 84 4d 88 |.....#7.M.| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES256-GCM-SHA384 b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES256-GCM-SHA384 index ad5fa501469a5f..bf3f0192605d4a 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES256-GCM-SHA384 +++ b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-AES256-GCM-SHA384 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 54 28 b4 3e d3 |....]...Y..T(.>.| -00000010 f8 21 2a 34 c9 6b 54 76 aa d2 cf 5e 07 7a aa e2 |.!*4.kTv...^.z..| -00000020 10 cb 3e ed 87 f4 3e 70 44 d0 35 20 6e b4 f0 71 |..>...>pD.5 n..q| -00000030 76 96 1e 2a 57 74 4b db 9a 11 0a af 06 ea 3b 20 |v..*WtK.......; | -00000040 a8 81 51 9d 41 ef f1 f9 23 87 18 cb c0 2c 00 00 |..Q.A...#....,..| +00000000 16 03 03 00 5d 02 00 00 59 03 03 45 43 2c 7b 07 |....]...Y..EC,{.| +00000010 6e 76 10 6b f3 9c 7c 2e b8 c9 6b 8d a0 b6 7b be |nv.k..|...k...{.| +00000020 79 7a 52 ab 8d f8 c6 cb 00 f6 48 20 a3 a7 3c ca |yzR.......H ..<.| +00000030 d7 57 be c7 6e 45 d6 75 c1 70 e0 86 1e 58 14 a3 |.W..nE.u.p...X..| +00000040 b4 7d 78 2c ca 82 2f 38 b8 96 ef c4 c0 2c 00 00 |.}x,../8.....,..| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 0e 0b 00 02 0a 00 02 07 00 02 |................| 00000070 04 30 82 02 00 30 82 01 62 02 09 00 b8 bf 2d 47 |.0...0..b.....-G| @@ -56,34 +57,34 @@ 00000240 8c 25 c1 33 13 83 0d 94 06 bb d4 37 7a f6 ec 7a |.%.3.......7z..z| 00000250 c9 86 2e dd d7 11 69 7f 85 7c 56 de fb 31 78 2b |......i..|V..1x+| 00000260 e4 c7 78 0d ae cb be 9e 4e 36 24 31 7b 6a 0f 39 |..x.....N6$1{j.9| -00000270 95 12 07 8f 2a 16 03 03 00 b7 0c 00 00 b3 03 00 |....*...........| -00000280 1d 20 de df 32 23 35 b3 79 a9 49 c2 b3 ad 0c d6 |. ..2#5.y.I.....| -00000290 23 a6 6a 9b 0f e3 52 51 7b bc f2 79 00 c9 a4 a0 |#.j...RQ{..y....| -000002a0 94 02 04 03 00 8b 30 81 88 02 42 01 8c 12 91 28 |......0...B....(| -000002b0 d4 6d cd 39 62 75 9a e0 e5 dd ea 31 45 f5 cd 8b |.m.9bu.....1E...| -000002c0 28 82 14 f5 e9 61 0e 9d af 42 06 17 42 e9 24 1c |(....a...B..B.$.| -000002d0 2b 1a c8 77 e4 7a 82 2b b0 dd 28 3f 76 5d dd fd |+..w.z.+..(?v]..| -000002e0 20 53 87 e6 77 26 be 45 e4 6d 31 e3 bf 02 42 01 | S..w&.E.m1...B.| -000002f0 70 c5 97 3a e4 7b 80 2b 1b eb c9 b2 0a 79 92 2e |p..:.{.+.....y..| -00000300 3d b1 4d 13 69 d2 60 a9 fd e9 66 56 83 e4 f5 6a |=.M.i.`...fV...j| -00000310 ae cd 8b 94 9f 0b 70 81 b1 ed c6 0d 5f 33 42 04 |......p....._3B.| -00000320 3d f7 b6 d3 e9 09 b8 bf 43 8c 19 d5 7d 60 d6 a9 |=.......C...}`..| -00000330 f7 16 03 03 00 04 0e 00 00 00 |..........| +00000270 95 12 07 8f 2a 16 03 03 00 b5 0c 00 00 b1 03 00 |....*...........| +00000280 1d 20 2c 21 c0 1d 0f 62 0d 25 60 02 a2 5f 10 5b |. ,!...b.%`.._.[| +00000290 68 fb b8 74 a5 2f 33 d6 1e 3f 05 43 19 6d 1c 74 |h..t./3..?.C.m.t| +000002a0 ce 5b 04 03 00 89 30 81 86 02 41 4c 6b 04 c2 3f |.[....0...ALk..?| +000002b0 98 87 0d 54 5a d5 1f 4c 77 1b b6 e5 f8 1f 35 29 |...TZ..Lw.....5)| +000002c0 e9 72 13 34 ba f8 f9 b3 0f 49 ed b5 69 bd 03 3e |.r.4.....I..i..>| +000002d0 10 7d a5 0f 7a aa a6 c5 e0 48 24 2a 61 85 17 5d |.}..z....H$*a..]| +000002e0 ac 93 35 47 41 60 f6 3b d6 41 32 20 02 41 02 4b |..5GA`.;.A2 .A.K| +000002f0 b7 28 37 9d 5d f3 fe 87 7b 13 9a 71 b0 59 7b f0 |.(7.]...{..q.Y{.| +00000300 50 e2 87 aa 9a ef 0e e8 47 bd 85 60 d5 41 32 00 |P.......G..`.A2.| +00000310 aa 26 e4 f3 c9 4c 9d df 6c dc 6a de 2b fc 76 b2 |.&...L..l.j.+.v.| +00000320 99 28 41 1c d3 04 a2 e0 71 92 41 97 0a b3 4b 16 |.(A.....q.A...K.| +00000330 03 03 00 04 0e 00 00 00 |........| >>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| -00000030 16 03 03 00 28 00 00 00 00 00 00 00 00 a7 5d c9 |....(.........].| -00000040 e7 6b 78 13 ae 11 64 d0 e7 77 48 03 d4 cf b9 65 |.kx...d..wH....e| -00000050 7f 07 75 78 7f 3d ae a1 9a 0f fb cb 74 |..ux.=......t| +00000030 16 03 03 00 28 00 00 00 00 00 00 00 00 5a b1 f1 |....(........Z..| +00000040 5a c2 32 50 dd 2b 6c 8a 0b de b7 04 cd 69 de 3a |Z.2P.+l......i.:| +00000050 45 3c 72 94 80 e2 f6 6c 66 98 c3 2d 4d |E>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 28 07 4e 69 71 66 |..........(.Niqf| -00000010 75 a0 bb b4 fd 86 8d 74 40 7e 0d e9 a5 95 d0 2d |u......t@~.....-| -00000020 ca eb 12 2f 7b 09 22 7c 8e bf da 97 18 d9 bd 74 |.../{."|.......t| -00000030 cd 08 6b |..k| +00000000 14 03 03 00 01 01 16 03 03 00 28 be d2 0b b0 a2 |..........(.....| +00000010 ac 18 1e dc 59 4f b4 85 be 3a 81 bd 73 51 47 35 |....YO...:..sQG5| +00000020 b3 2b b2 84 45 3a 08 11 a3 40 13 0e 5d ef 4b 26 |.+..E:...@..].K&| +00000030 9d e6 83 |...| >>> Flow 5 (client to server) -00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 44 26 54 |.............D&T| -00000010 a3 37 94 7a 3d 93 6a 06 35 fe 9a d4 7b 54 ab d4 |.7.z=.j.5...{T..| -00000020 33 fd 93 15 03 03 00 1a 00 00 00 00 00 00 00 02 |3...............| -00000030 bf ff c7 9e 4b db ee a4 a2 b9 a5 91 cd 09 97 a2 |....K...........| -00000040 55 a5 |U.| +00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 0f 8d 3a |...............:| +00000010 5b dd b2 49 18 84 73 e7 a5 c0 46 c0 ad fa cc 44 |[..I..s...F....D| +00000020 60 0d 8c 15 03 03 00 1a 00 00 00 00 00 00 00 02 |`...............| +00000030 f6 4c c6 95 ff e5 01 3a 3b a9 46 23 f3 bc 24 29 |.L.....:;.F#..$)| +00000040 6d 52 |mR| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-CHACHA20-POLY1305 b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-CHACHA20-POLY1305 index 9982121ef8c2ea..db53e96a01e48f 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-CHACHA20-POLY1305 +++ b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-ECDSA-CHACHA20-POLY1305 @@ -1,24 +1,26 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 d4 01 00 00 d0 03 03 00 00 00 00 00 |................| +00000000 16 03 01 00 ee 01 00 00 ea 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000040 00 00 00 00 00 00 00 00 00 00 00 00 00 08 cc a9 |................| -00000050 13 03 13 01 13 02 01 00 00 7f 00 0b 00 02 01 00 |................| +00000050 13 03 13 01 13 02 01 00 00 99 00 0b 00 02 01 00 |................| 00000060 ff 01 00 01 00 00 17 00 00 00 12 00 00 00 05 00 |................| 00000070 05 01 00 00 00 00 00 0a 00 0a 00 08 00 1d 00 17 |................| -00000080 00 18 00 19 00 0d 00 1a 00 18 08 04 04 03 08 07 |................| -00000090 08 05 08 06 04 01 05 01 06 01 05 03 06 03 02 01 |................| -000000a0 02 03 00 2b 00 09 08 03 04 03 03 03 02 03 01 00 |...+............| -000000b0 33 00 26 00 24 00 1d 00 20 2f e5 7d a3 47 cd 62 |3.&.$... /.}.G.b| -000000c0 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 cf |C.(.._.).0......| -000000d0 c2 ed 90 99 5f 58 cb 3b 74 |...._X.;t| +00000080 00 18 00 19 00 0d 00 16 00 14 08 04 04 03 08 07 |................| +00000090 08 05 08 06 04 01 05 01 06 01 05 03 06 03 00 32 |...............2| +000000a0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000b0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| +000000c0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| +000000d0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| +000000e0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| +000000f0 cb 3b 74 |.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 4f ed 35 28 41 |....]...Y..O.5(A| -00000010 4a 0f a2 da 57 ef 60 69 32 00 a5 82 22 3d d9 ea |J...W.`i2..."=..| -00000020 36 28 20 31 e3 4a b6 d1 b6 b4 49 20 97 f3 45 59 |6( 1.J....I ..EY| -00000030 e7 9c f5 40 57 51 b8 f8 5a 8d f2 eb ce 83 18 2d |...@WQ..Z......-| -00000040 e3 28 55 6f 17 08 23 88 57 b1 a0 e4 cc a9 00 00 |.(Uo..#.W.......| +00000000 16 03 03 00 5d 02 00 00 59 03 03 74 9e c0 76 74 |....]...Y..t..vt| +00000010 c1 54 da a1 cf ad 75 4c fe 91 3b e2 06 e9 fb 1a |.T....uL..;.....| +00000020 9c 69 ee bf 30 1f 5c 0c 92 ca 58 20 a9 bb 64 3d |.i..0.\...X ..d=| +00000030 1e 8a 19 db 31 25 15 f2 65 5a 78 ad 7e d2 98 90 |....1%..eZx.~...| +00000040 8a 8b 0f 0e 77 53 db a2 93 b4 76 6d cc a9 00 00 |....wS....vm....| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 0e 0b 00 02 0a 00 02 07 00 02 |................| 00000070 04 30 82 02 00 30 82 01 62 02 09 00 b8 bf 2d 47 |.0...0..b.....-G| @@ -53,32 +55,32 @@ 00000240 8c 25 c1 33 13 83 0d 94 06 bb d4 37 7a f6 ec 7a |.%.3.......7z..z| 00000250 c9 86 2e dd d7 11 69 7f 85 7c 56 de fb 31 78 2b |......i..|V..1x+| 00000260 e4 c7 78 0d ae cb be 9e 4e 36 24 31 7b 6a 0f 39 |..x.....N6$1{j.9| -00000270 95 12 07 8f 2a 16 03 03 00 b7 0c 00 00 b3 03 00 |....*...........| -00000280 1d 20 64 49 59 eb a9 80 c3 0a 9e 13 f4 09 04 6b |. dIY..........k| -00000290 1f e5 3b 0b 77 84 fe 2e 1f ea 7e d4 64 2a 7f 3f |..;.w.....~.d*.?| -000002a0 dc 0f 04 03 00 8b 30 81 88 02 42 00 a4 62 9b d1 |......0...B..b..| -000002b0 ed 60 b7 2d 05 48 64 b6 94 3b a1 75 cb 41 78 e5 |.`.-.Hd..;.u.Ax.| -000002c0 e0 12 70 d9 90 4d c0 34 39 f3 52 13 75 4b fa 10 |..p..M.49.R.uK..| -000002d0 19 6f a6 2d b4 c3 8f 45 0c 7f 34 fb 82 01 e6 a1 |.o.-...E..4.....| -000002e0 be 73 72 3b e5 04 80 70 7d bd 4a f9 e5 02 42 01 |.sr;...p}.J...B.| -000002f0 69 3d 53 e9 8d 70 6d 07 84 7e 56 3f ce 53 b9 6c |i=S..pm..~V?.S.l| -00000300 43 0f 70 74 f1 52 ce d4 95 55 13 8c 07 e7 20 d7 |C.pt.R...U.... .| -00000310 61 2e b7 f4 5b 34 6e 5d 9c d8 73 66 89 a1 f6 7e |a...[4n]..sf...~| -00000320 01 bb 9a 5e 52 59 3f fc 3e 24 fc e7 49 10 83 9d |...^RY?.>$..I...| -00000330 fb 16 03 03 00 04 0e 00 00 00 |..........| +00000270 95 12 07 8f 2a 16 03 03 00 b6 0c 00 00 b2 03 00 |....*...........| +00000280 1d 20 6b 7f 5e 5e cd 37 8e 2a cf 07 bf 99 89 93 |. k.^^.7.*......| +00000290 05 be e6 3c 1d a8 67 07 b4 0d 28 22 be 3f 57 88 |...<..g...(".?W.| +000002a0 e4 41 04 03 00 8a 30 81 87 02 42 00 ce 0f 5f f7 |.A....0...B..._.| +000002b0 52 8c 44 e3 58 4c ed 4a 4f bf 2d 14 77 0f 3c a7 |R.D.XL.JO.-.w.<.| +000002c0 8a 7b fc 2d 0f 94 6e 51 01 51 e0 8c 58 b0 72 e4 |.{.-..nQ.Q..X.r.| +000002d0 57 cd ab ad fd a9 8c c2 e7 92 f3 0e e0 a0 22 64 |W............."d| +000002e0 a0 36 f9 e1 a3 f5 19 c8 8a b8 25 f6 0b 02 41 5d |.6........%...A]| +000002f0 d3 7c a0 42 5b f9 5d b8 f9 c2 a3 d4 8f 0e 4c c7 |.|.B[.].......L.| +00000300 d4 61 11 73 f0 35 7e b1 73 29 0f 97 52 75 8a 1f |.a.s.5~.s)..Ru..| +00000310 00 e9 6e 59 25 bb ca 72 bd 1e 4f 34 fc 55 ef 66 |..nY%..r..O4.U.f| +00000320 33 3d 5f fc e2 8d ac 09 7f 20 85 ca b3 a4 b3 ca |3=_...... ......| +00000330 16 03 03 00 04 0e 00 00 00 |.........| >>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| -00000030 16 03 03 00 20 cf f9 90 26 e8 2c 00 99 ae 0e e6 |.... ...&.,.....| -00000040 a8 d9 66 6b 23 e7 c9 73 1c f7 e9 af e1 71 1d f3 |..fk#..s.....q..| -00000050 34 60 ee 16 9d |4`...| +00000030 16 03 03 00 20 1a fc 19 6c ef eb a5 d1 28 d9 aa |.... ...l....(..| +00000040 56 3f b7 12 a0 18 eb 76 16 be 6d 1e d9 3b 6f 14 |V?.....v..m..;o.| +00000050 51 68 9c 63 b3 |Qh.c.| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 20 39 9f 39 3a 22 |.......... 9.9:"| -00000010 0b 27 cb 75 0d 43 98 11 06 dd 7e 73 17 e0 8a 19 |.'.u.C....~s....| -00000020 6d eb a8 33 c6 42 b4 e9 cb dd b1 |m..3.B.....| +00000000 14 03 03 00 01 01 16 03 03 00 20 38 c9 5f c3 6f |.......... 8._.o| +00000010 1b 84 54 77 40 98 70 cb f1 d1 5f 9e 11 16 24 c6 |..Tw@.p..._...$.| +00000020 c4 7e f0 44 32 2c ce 29 dd 8e 20 |.~.D2,.).. | >>> Flow 5 (client to server) -00000000 17 03 03 00 16 24 1f 08 49 ef 32 16 8d 76 c7 34 |.....$..I.2..v.4| -00000010 e0 08 cb d9 e7 72 aa 0f fb c4 94 15 03 03 00 12 |.....r..........| -00000020 38 0b 8c e0 0b 61 20 e8 a6 7c 77 69 fb 00 96 39 |8....a ..|wi...9| -00000030 b9 a7 |..| +00000000 17 03 03 00 16 86 12 a2 e2 ae df 03 ed e4 a4 74 |...............t| +00000010 d0 8b 8c 17 92 20 4d bd c2 29 ac 15 03 03 00 12 |..... M..)......| +00000020 e1 66 4e 21 16 24 34 9f 0f 95 63 80 d6 22 52 c4 |.fN!.$4...c.."R.| +00000030 67 ac |g.| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-RSA-AES b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-RSA-AES index e2e935cc2e7981..70b43803f15ce5 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-RSA-AES +++ b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-RSA-AES @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 8d 69 e3 6e 69 |....]...Y...i.ni| -00000010 45 96 8b 0d c8 7d 9c dc b6 75 24 ee 61 44 95 99 |E....}...u$.aD..| -00000020 cb a8 da df 35 ac c3 36 02 3f b9 20 2d c2 47 c9 |....5..6.?. -.G.| -00000030 f0 47 80 9a c6 bd e4 1b ca d6 62 17 59 ec c9 83 |.G........b.Y...| -00000040 d5 ed 3f 2c 13 b0 30 0a 5e 1f 0e db c0 13 00 00 |..?,..0.^.......| +00000000 16 03 03 00 5d 02 00 00 59 03 03 c7 61 5d 9b a7 |....]...Y...a]..| +00000010 9f 27 4f de 56 2b 83 b8 dc 21 7d 27 e5 23 c0 b4 |.'O.V+...!}'.#..| +00000020 a7 b1 61 3c 7c b7 af 1a b0 c6 ed 20 82 8c 08 27 |..a<|...... ...'| +00000030 79 99 c6 a7 dd 7e 33 bb 6c d0 4b 4a 02 e7 2e 87 |y....~3.l.KJ....| +00000040 69 24 05 ac 32 6f 40 4f dc 2e 11 3f c0 13 00 00 |i$..2o@O...?....| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -61,38 +62,38 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 0f 4e 12 |............ .N.| -000002d0 ab ce 7e 62 68 64 e0 1d 7e 33 a2 ee 5d e7 2d d5 |..~bhd..~3..].-.| -000002e0 d2 dc d0 ce b2 7e 9b be 33 8d 9c 56 7a 08 04 00 |.....~..3..Vz...| -000002f0 80 44 b8 9b 87 5f 84 f6 8a 99 af e9 f7 f8 88 fd |.D..._..........| -00000300 f7 12 02 48 a1 fa 66 9c 89 a2 dc bf fd 41 08 fa |...H..f......A..| -00000310 6f e4 44 09 8e c1 45 bb 2a e9 e3 45 19 a3 81 50 |o.D...E.*..E...P| -00000320 99 46 5f 86 5b 6b c0 29 d1 e5 21 4b 56 af f4 d0 |.F_.[k.)..!KV...| -00000330 63 c4 75 ac f6 75 61 ca fe 1a b9 70 4d 14 50 b1 |c.u..ua....pM.P.| -00000340 12 7f 29 2d c5 d7 78 da 06 3e 4c 47 75 8a a6 01 |..)-..x..>LGu...| -00000350 37 17 1c e8 ef 94 85 d0 e2 ee d5 25 ae 8d f8 e6 |7..........%....| -00000360 90 10 12 37 b4 0f f5 c7 a2 85 1a 83 8a 38 e6 32 |...7.........8.2| -00000370 98 16 03 03 00 04 0e 00 00 00 |..........| +000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 5c ce 07 |............ \..| +000002d0 a8 0b 8f ff 08 19 e7 d7 38 62 98 bf c1 f7 92 27 |........8b.....'| +000002e0 cb 7b 5e 50 36 51 83 61 8d b4 d6 29 1c 08 04 00 |.{^P6Q.a...)....| +000002f0 80 98 88 66 48 81 8b 9d 9b e0 6e 6e c4 c4 99 fb |...fH.....nn....| +00000300 63 46 fd 8c 7e e1 d0 dd 01 e6 bb 23 97 b8 21 2e |cF..~......#..!.| +00000310 53 06 e0 5d 58 c8 79 c3 01 a5 01 e2 81 7b c7 a9 |S..]X.y......{..| +00000320 04 15 98 9b 53 98 42 0b 3d 4e 97 93 b9 88 80 2c |....S.B.=N.....,| +00000330 78 fd 9b 99 12 7f a8 de a0 c0 c4 1d 15 59 d4 d8 |x............Y..| +00000340 ef 0d bd 0b 46 c3 6e e6 4b 04 69 91 c0 db 74 ca |....F.n.K.i...t.| +00000350 99 2e 6f 47 8b 9b 22 3f b3 4a e3 10 47 2d dd 47 |..oG.."?.J..G-.G| +00000360 d5 e1 9b 66 8f d9 b2 fb b7 52 ae 6a 14 74 32 2d |...f.....R.j.t2-| +00000370 6e 16 03 03 00 04 0e 00 00 00 |n.........| >>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| 00000030 16 03 03 00 40 00 00 00 00 00 00 00 00 00 00 00 |....@...........| -00000040 00 00 00 00 00 6c 72 1c 47 f9 89 fa 2b 0e cf 69 |.....lr.G...+..i| -00000050 bf 95 e4 ea b8 74 e7 99 d4 c4 53 f5 97 f5 08 f9 |.....t....S.....| -00000060 ef 03 e7 76 b8 2c 31 f6 9b 2c cd 27 c8 42 6a a8 |...v.,1..,.'.Bj.| -00000070 c0 e0 b9 5e cd |...^.| +00000040 00 00 00 00 00 99 c9 2d 35 8b 96 18 36 f4 db a4 |.......-5...6...| +00000050 9a 56 67 6a 6b 9d ee b1 cb 95 54 43 93 7c 0e ff |.Vgjk.....TC.|..| +00000060 c0 54 84 83 14 bf d5 95 85 a1 b5 c5 2c fe c3 29 |.T..........,..)| +00000070 81 7b 1e 16 2e |.{...| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 40 89 35 48 2e 99 |..........@.5H..| -00000010 7c 13 76 5e 7f 07 a6 44 d9 95 1c 9f 25 59 3d f5 ||.v^...D....%Y=.| -00000020 df d6 5e cb 85 3e 0c f7 48 69 95 77 29 95 88 1c |..^..>..Hi.w)...| -00000030 1e 76 8a c3 ea 69 07 75 46 68 09 50 fa 30 0a 57 |.v...i.uFh.P.0.W| -00000040 84 e3 c7 d9 34 c4 40 32 7b f4 95 |....4.@2{..| +00000000 14 03 03 00 01 01 16 03 03 00 40 8b 6e b0 cb bf |..........@.n...| +00000010 35 8d c3 7e 48 27 92 77 20 8a 38 c6 81 94 17 d4 |5..~H'.w .8.....| +00000020 b8 18 4e 23 e7 8b 0a 89 7b d1 72 58 aa b9 9b bb |..N#....{.rX....| +00000030 c9 40 c7 c4 8e 74 77 f3 3e 90 9c b0 19 04 6c ea |.@...tw.>.....l.| +00000040 87 85 f1 fd d9 fd 2d 57 57 f1 c9 |......-WW..| >>> Flow 5 (client to server) 00000000 17 03 03 00 30 00 00 00 00 00 00 00 00 00 00 00 |....0...........| -00000010 00 00 00 00 00 24 6d d9 23 68 ad 8d 64 2e 3a d5 |.....$m.#h..d.:.| -00000020 ed eb 4c e9 cb 9d 13 49 4c 44 b9 00 af af 2c e7 |..L....ILD....,.| -00000030 22 c0 bb 31 4e 15 03 03 00 30 00 00 00 00 00 00 |"..1N....0......| -00000040 00 00 00 00 00 00 00 00 00 00 c8 60 b8 c1 66 9b |...........`..f.| -00000050 52 44 3c 26 7f ff 96 a6 21 c7 eb 33 71 f8 40 23 |RD<&....!..3q.@#| -00000060 7c 5b ef b2 e1 b4 3f 84 82 48 ||[....?..H| +00000010 00 00 00 00 00 b9 8c 86 6d 84 db d0 60 f7 11 22 |........m...`.."| +00000020 97 52 b3 fe bb 1c 0c bd 26 9c a1 84 51 3d a2 2b |.R......&...Q=.+| +00000030 c0 79 e5 a5 96 15 03 03 00 30 00 00 00 00 00 00 |.y.......0......| +00000040 00 00 00 00 00 00 00 00 00 00 7a 42 27 d5 ae 30 |..........zB'..0| +00000050 81 b5 94 05 55 c9 93 e0 11 98 6d f8 aa 18 92 f4 |....U.....m.....| +00000060 32 8f d1 aa a6 86 cd df 11 c2 |2.........| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-RSA-AES128-SHA256 b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-RSA-AES128-SHA256 index f8f04ac13b58c4..9081031e737b02 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-RSA-AES128-SHA256 +++ b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-RSA-AES128-SHA256 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 8f cc 68 50 60 |....]...Y....hP`| -00000010 11 c8 c3 ce e7 e9 e1 3c fb 29 d7 4f df 44 16 65 |.......<.).O.D.e| -00000020 1d 3b 25 5b 32 59 f1 8e be d8 49 20 c2 89 a7 d5 |.;%[2Y....I ....| -00000030 cb 2c a2 05 19 6b 65 64 9b 71 2b d3 d2 19 cb 66 |.,...ked.q+....f| -00000040 61 c1 6f 19 b4 7a 15 41 76 14 85 fc c0 27 00 00 |a.o..z.Av....'..| +00000000 16 03 03 00 5d 02 00 00 59 03 03 7b 18 cc 97 f2 |....]...Y..{....| +00000010 59 62 62 18 bc ee e1 09 0c 7b 8c 3e f0 36 3d 8b |Ybb......{.>.6=.| +00000020 96 54 26 93 c8 ac bb 7c 19 63 6a 20 96 8e 51 d2 |.T&....|.cj ..Q.| +00000030 18 6d 8a 72 6b 5d d2 5e 2b 53 4e a5 43 cf 75 4f |.m.rk].^+SN.C.uO| +00000040 e6 9c 3a 30 f5 23 3f bf 66 bc fe a2 c0 27 00 00 |..:0.#?.f....'..| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -61,42 +62,42 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 88 db 33 |............ ..3| -000002d0 96 fb ae 63 de f8 2b 4b 94 fa da c2 8e 75 fa 63 |...c..+K.....u.c| -000002e0 0a 54 2b 68 01 76 2a 76 4f e7 91 e9 2a 08 04 00 |.T+h.v*vO...*...| -000002f0 80 d9 47 dd dd b9 94 b8 1c 1b dc 1f 79 bd f1 26 |..G.........y..&| -00000300 66 19 52 0b 0a 2e ea 0d 69 bf 93 a0 29 0f 6a 30 |f.R.....i...).j0| -00000310 2c 1b 44 25 13 e3 b7 03 db b3 fc 37 9c 3f bb e7 |,.D%.......7.?..| -00000320 c7 4d 65 8f af bb 39 fa 6e 77 09 bb 08 09 7e d1 |.Me...9.nw....~.| -00000330 0f ad 57 78 a6 cb a8 3f 2e 70 fc 38 75 2f a0 23 |..Wx...?.p.8u/.#| -00000340 0b 0c 6c 28 0a 71 51 dc 10 8f 85 fd 97 0f c4 84 |..l(.qQ.........| -00000350 80 cf 70 0a a9 56 9a af 2f ab 12 c7 8b d1 82 1c |..p..V../.......| -00000360 f0 6c 08 00 73 18 dd 6e 5a 66 ed a5 f7 7a 3d 73 |.l..s..nZf...z=s| -00000370 88 16 03 03 00 04 0e 00 00 00 |..........| +000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 e4 05 36 |............ ..6| +000002d0 05 47 cf b0 62 f8 f6 d6 e0 ce a5 f9 45 f6 28 f9 |.G..b.......E.(.| +000002e0 d2 58 47 6c 0a 58 9d 85 74 52 d3 0d 17 08 04 00 |.XGl.X..tR......| +000002f0 80 d0 8b 12 95 dd 2a 7f b2 55 33 e0 13 28 41 47 |......*..U3..(AG| +00000300 06 38 fe fb 11 b3 3f 66 c4 62 db 1c f9 29 9e ea |.8....?f.b...)..| +00000310 2f e4 62 22 4f 99 36 05 fb 89 8f d5 29 bf a6 e0 |/.b"O.6.....)...| +00000320 fc 12 88 d8 60 9a 4d 82 eb 64 ba 99 1c 1c 82 ff |....`.M..d......| +00000330 df 57 a9 b9 14 d5 a7 ad 57 45 1d e0 91 db 61 90 |.W......WE....a.| +00000340 ca 15 9b 0c 49 f8 a9 40 5f 34 6b d2 fe 00 48 e5 |....I..@_4k...H.| +00000350 ea 1e 3b ae 1f 81 a9 d7 91 ff 46 e9 af 48 04 b3 |..;.......F..H..| +00000360 83 0c 95 91 ea 5f 1f 5b 48 98 2a 9c f6 80 2a db |....._.[H.*...*.| +00000370 d2 16 03 03 00 04 0e 00 00 00 |..........| >>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| 00000030 16 03 03 00 50 00 00 00 00 00 00 00 00 00 00 00 |....P...........| -00000040 00 00 00 00 00 93 01 19 de 19 e6 58 a9 b2 bb e6 |...........X....| -00000050 83 08 03 78 72 d8 25 fc 31 70 55 fa cd ec e4 0a |...xr.%.1pU.....| -00000060 a2 7b 00 5c fe 7f 08 26 a5 ca 9f d6 28 95 69 f5 |.{.\...&....(.i.| -00000070 b9 17 23 34 83 fe 36 24 60 17 4c 90 18 30 f9 44 |..#4..6$`.L..0.D| -00000080 af 74 ea 65 4b |.t.eK| +00000040 00 00 00 00 00 c0 4e 63 30 1d 23 c8 dc ef 03 e9 |......Nc0.#.....| +00000050 29 7c 61 b8 7a a4 52 c3 09 f4 aa 97 30 3c 5a 7c |)|a.z.R.....0>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 50 b6 cf 83 c7 0d |..........P.....| -00000010 dc 8f 9a 96 0d 0b 4f 58 ed c9 46 fa 95 0b f1 ab |......OX..F.....| -00000020 9a 50 2a cc af 26 73 03 0b 93 e7 4e 96 a0 e2 44 |.P*..&s....N...D| -00000030 01 99 9e 06 a3 81 3f 0b e4 7c 03 53 e4 65 36 69 |......?..|.S.e6i| -00000040 65 96 79 a2 4e 79 e8 fc b1 6a 50 3e dd 22 1f e3 |e.y.Ny...jP>."..| -00000050 fe e3 77 9f 8c 05 f8 cc 08 4b 6b |..w......Kk| +00000000 14 03 03 00 01 01 16 03 03 00 50 bd 21 3f c1 94 |..........P.!?..| +00000010 2f 3e 00 3c 43 00 17 39 90 74 b2 4f 40 5d 0c 2b |/>.>> Flow 5 (client to server) 00000000 17 03 03 00 40 00 00 00 00 00 00 00 00 00 00 00 |....@...........| -00000010 00 00 00 00 00 f8 39 8e 57 60 18 1c ef 19 c8 da |......9.W`......| -00000020 f4 3b dc 42 85 44 e3 59 28 ea 29 11 60 ba 2b 2b |.;.B.D.Y(.).`.++| -00000030 dc 6a 5d 3a ed ce f2 10 34 17 6e 10 67 00 f0 43 |.j]:....4.n.g..C| -00000040 3e 0f e1 a1 c8 15 03 03 00 40 00 00 00 00 00 00 |>........@......| -00000050 00 00 00 00 00 00 00 00 00 00 13 0d fa 71 e4 df |.............q..| -00000060 76 f3 c9 01 d5 60 b4 34 d4 6f 68 f5 cd af d6 0b |v....`.4.oh.....| -00000070 e6 af ab ec 64 75 38 d6 17 d4 27 81 96 e5 71 74 |....du8...'...qt| -00000080 d5 cd 39 30 35 4a ef 92 56 6e |..905J..Vn| +00000010 00 00 00 00 00 ad 41 d8 27 7d f4 06 b7 6c 89 f4 |......A.'}...l..| +00000020 e3 c5 de 6d 06 80 0c 2e f8 42 5a 2c da f1 e2 b6 |...m.....BZ,....| +00000030 e6 e8 6f 5c ee e0 aa 4b d6 8a 7a 04 6a 5e 8c 99 |..o\...K..z.j^..| +00000040 1b 36 be f9 d0 15 03 03 00 40 00 00 00 00 00 00 |.6.......@......| +00000050 00 00 00 00 00 00 00 00 00 00 f1 cf 76 91 15 bd |............v...| +00000060 33 f3 96 25 c7 d9 77 30 a6 84 1a 06 c1 96 96 27 |3..%..w0.......'| +00000070 70 ff d2 ce 82 1e 39 8f 19 75 c7 f2 0f cb de bc |p.....9..u......| +00000080 9a ef 4c 93 e3 3c 0c 2f 97 ad |..L..<./..| diff --git a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-RSA-CHACHA20-POLY1305 b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-RSA-CHACHA20-POLY1305 index 2ea5debf2d67f9..30ea8ec568a905 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-ECDHE-RSA-CHACHA20-POLY1305 +++ b/src/crypto/tls/testdata/Client-TLSv12-ECDHE-RSA-CHACHA20-POLY1305 @@ -1,24 +1,26 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 d4 01 00 00 d0 03 03 00 00 00 00 00 |................| +00000000 16 03 01 00 ee 01 00 00 ea 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000040 00 00 00 00 00 00 00 00 00 00 00 00 00 08 cc a8 |................| -00000050 13 03 13 01 13 02 01 00 00 7f 00 0b 00 02 01 00 |................| +00000050 13 03 13 01 13 02 01 00 00 99 00 0b 00 02 01 00 |................| 00000060 ff 01 00 01 00 00 17 00 00 00 12 00 00 00 05 00 |................| 00000070 05 01 00 00 00 00 00 0a 00 0a 00 08 00 1d 00 17 |................| -00000080 00 18 00 19 00 0d 00 1a 00 18 08 04 04 03 08 07 |................| -00000090 08 05 08 06 04 01 05 01 06 01 05 03 06 03 02 01 |................| -000000a0 02 03 00 2b 00 09 08 03 04 03 03 03 02 03 01 00 |...+............| -000000b0 33 00 26 00 24 00 1d 00 20 2f e5 7d a3 47 cd 62 |3.&.$... /.}.G.b| -000000c0 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 cf |C.(.._.).0......| -000000d0 c2 ed 90 99 5f 58 cb 3b 74 |...._X.;t| +00000080 00 18 00 19 00 0d 00 16 00 14 08 04 04 03 08 07 |................| +00000090 08 05 08 06 04 01 05 01 06 01 05 03 06 03 00 32 |...............2| +000000a0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000b0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| +000000c0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| +000000d0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| +000000e0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| +000000f0 cb 3b 74 |.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 01 ce 8f f1 84 |....]...Y.......| -00000010 65 dc 47 4e 43 f2 cd 85 6c 98 40 a7 5c 7f ed 53 |e.GNC...l.@.\..S| -00000020 78 63 03 dd 50 3e a4 27 20 21 bf 20 d9 93 9d f6 |xc..P>.' !. ....| -00000030 1f e8 0e 72 fc 64 e0 dd 8e ef b3 ea ff 5b ad 60 |...r.d.......[.`| -00000040 a5 f0 68 ff 7e 35 46 a4 28 71 31 fd cc a8 00 00 |..h.~5F.(q1.....| +00000000 16 03 03 00 5d 02 00 00 59 03 03 1b 61 37 54 ad |....]...Y...a7T.| +00000010 7e 92 6a 93 df 5e f8 16 be e7 06 12 3f e7 23 84 |~.j..^......?.#.| +00000020 be 2a 30 78 98 cf 60 8a 8a 2d 26 20 44 fb 40 31 |.*0x..`..-& D.@1| +00000030 88 38 6e fc ce 1d af 07 e3 c0 49 05 ff bc 92 ae |.8n.......I.....| +00000040 58 79 51 ec a7 24 82 75 9a 16 45 fa cc a8 00 00 |XyQ..$.u..E.....| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -58,31 +60,31 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 ea 03 c7 |............ ...| -000002d0 b0 eb 76 ba 86 25 18 e5 0d b3 70 2a 41 72 4d 88 |..v..%....p*ArM.| -000002e0 55 6d 91 2b 65 fe e0 64 12 69 42 eb 27 08 04 00 |Um.+e..d.iB.'...| -000002f0 80 0f 00 0a b0 35 db 39 fa ba a1 27 44 e0 9b d4 |.....5.9...'D...| -00000300 b8 aa b3 4f 93 46 ed 09 f8 3e 1b c8 47 6e 52 8a |...O.F...>..GnR.| -00000310 8f fd b1 67 f6 05 87 68 de 7b 25 1b 40 df 1e f2 |...g...h.{%.@...| -00000320 04 2b 4f 39 3d 6f a8 f7 25 37 26 34 87 84 06 c4 |.+O9=o..%7&4....| -00000330 0e 31 bc 46 8a d2 99 14 6d 5d 14 32 a2 32 fc 10 |.1.F....m].2.2..| -00000340 5e fb d9 f8 06 d4 05 17 5d 59 f4 38 c5 13 1c 91 |^.......]Y.8....| -00000350 5a 5f cb dd 3e 27 78 10 2c 0c 9b 7f 1a ec 4c da |Z_..>'x.,.....L.| -00000360 fb d7 52 eb 76 83 75 87 d2 a3 7f e9 25 a5 8a 22 |..R.v.u.....%.."| -00000370 dd 16 03 03 00 04 0e 00 00 00 |..........| +000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 8b cc 79 |............ ..y| +000002d0 1e 8e 20 61 5d 00 97 27 cb 24 5c 7c 5f 3f df db |.. a]..'.$\|_?..| +000002e0 2d ba 3b f4 a1 1d 3e 3c 08 da 98 22 68 08 04 00 |-.;...><..."h...| +000002f0 80 10 fc 5b 90 af a6 8e d4 1b 5e 63 13 55 88 55 |...[......^c.U.U| +00000300 f3 bb ad 7a 5b 14 95 55 6b 2f 01 eb 22 88 fb 21 |...z[..Uk/.."..!| +00000310 40 88 d1 7f 52 b2 a2 ff 51 f9 05 8c c8 28 70 77 |@...R...Q....(pw| +00000320 3c 7a b0 48 28 f0 34 19 cc 8e c8 c9 43 93 aa fa |..a...4..r}..| +00000360 98 f2 bb c0 ae 5f b8 8e 00 6a 47 9c 8d cf 91 eb |....._...jG.....| +00000370 e7 16 03 03 00 04 0e 00 00 00 |..........| >>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| -00000030 16 03 03 00 20 28 48 18 85 9d 35 c7 5a 46 2c a7 |.... (H...5.ZF,.| -00000040 5b 8b c0 71 73 66 d0 a7 53 a0 e3 ed c8 57 f3 70 |[..qsf..S....W.p| -00000050 9a 01 58 61 84 |..Xa.| +00000030 16 03 03 00 20 af 23 75 51 f1 3c 68 f8 87 2e 2b |.... .#uQ.>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 20 b9 72 f6 07 40 |.......... .r..@| -00000010 0b 2f 67 8a 97 fa 17 24 81 e5 6f 5b 09 90 72 8d |./g....$..o[..r.| -00000020 48 59 14 00 34 e8 3c 82 b8 2f 87 |HY..4.<../.| +00000000 14 03 03 00 01 01 16 03 03 00 20 c0 52 cb d5 de |.......... .R...| +00000010 b1 43 8a 0b 75 a6 a4 d4 8f af 85 63 71 1d 3b 4f |.C..u......cq.;O| +00000020 7a ad 9b 77 85 1a 1d 7e 4b 42 13 |z..w...~KB.| >>> Flow 5 (client to server) -00000000 17 03 03 00 16 f6 cf b1 93 64 54 56 f3 35 3d 8f |.........dTV.5=.| -00000010 31 5e 66 62 56 aa e9 81 47 4c 8b 15 03 03 00 12 |1^fbV...GL......| -00000020 de 0b 32 2b 58 2c b5 4e 80 2a 66 e7 a0 a9 28 d9 |..2+X,.N.*f...(.| -00000030 db 11 |..| +00000000 17 03 03 00 16 30 b8 0e dd 42 64 a4 17 ee f5 70 |.....0...Bd....p| +00000010 1e 00 e8 da a7 f6 8e 73 c7 70 32 15 03 03 00 12 |.......s.p2.....| +00000020 57 be 7b cf bb 7b fe cb 7e b2 bb 38 9f 31 65 07 |W.{..{..~..8.1e.| +00000030 ba 61 |.a| diff --git a/src/crypto/tls/testdata/Client-TLSv12-Ed25519 b/src/crypto/tls/testdata/Client-TLSv12-Ed25519 index d43ee07eeab5e0..40b2f35137dc42 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-Ed25519 +++ b/src/crypto/tls/testdata/Client-TLSv12-Ed25519 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 93 8a a3 77 84 |....]...Y.....w.| -00000010 fd 41 3d a5 ad b5 eb 91 63 a6 b5 3a 5f 21 08 df |.A=.....c..:_!..| -00000020 72 07 be 1f df d7 4e 6f f3 f8 cb 20 ae d3 e5 fe |r.....No... ....| -00000030 53 a3 c7 84 6c 3e c6 1d d5 65 5d a6 a5 7d f7 5c |S...l>...e]..}.\| -00000040 34 65 a7 df 1e 28 3f 7d 20 08 81 2c cc a9 00 00 |4e...(?} ..,....| +00000000 16 03 03 00 5d 02 00 00 59 03 03 cc 3c fc d0 c6 |....]...Y...<...| +00000010 b8 3e 6b cb 59 8d 68 04 0b 94 10 28 ad 71 06 14 |.>k.Y.h....(.q..| +00000020 08 6c 10 12 aa 8d e4 f9 aa 7d 9b 20 a0 b8 96 40 |.l.......}. ...@| +00000030 1a ce 65 ae a6 a2 4d 4d 92 41 8e 2c 8d 94 d1 02 |..e...MM.A.,....| +00000040 c1 cb bd 6d 8b ee 36 ae c5 ec 6a 40 cc a9 00 00 |...m..6...j@....| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 01 3c 0b 00 01 38 00 01 35 00 01 |......<...8..5..| 00000070 32 30 82 01 2e 30 81 e1 a0 03 02 01 02 02 10 0f |20...0..........| @@ -44,26 +45,26 @@ 00000180 fc ab 7d 4d d4 e0 93 13 d0 a9 42 e0 b6 6b fe 5d |..}M......B..k.]| 00000190 67 48 d7 9f 50 bc 6c cd 4b 03 83 7c f2 08 58 cd |gH..P.l.K..|..X.| 000001a0 ac cf 0c 16 03 03 00 6c 0c 00 00 68 03 00 1d 20 |.......l...h... | -000001b0 e9 e7 75 b4 4c 42 14 7a e0 50 0a 95 f1 d7 35 aa |..u.LB.z.P....5.| -000001c0 39 05 9e 59 73 5e cd d2 ec 56 18 f0 ee 1f 81 3c |9..Ys^...V.....<| -000001d0 08 07 00 40 44 e9 19 b6 00 49 f7 c8 06 62 cb c1 |...@D....I...b..| -000001e0 b5 7c b6 cd 32 47 c9 05 82 3b f0 af d1 cd 75 8b |.|..2G...;....u.| -000001f0 82 8b 47 dd 85 13 78 69 5d 40 7e 9b 91 24 2e ce |..G...xi]@~..$..| -00000200 3e be e0 12 1e ec 56 0b ff 25 21 92 6f 24 12 10 |>.....V..%!.o$..| -00000210 30 33 cf 04 16 03 03 00 04 0e 00 00 00 |03...........| +000001b0 fd f1 12 74 21 62 15 da 36 af d5 6c f3 0a cd 1e |...t!b..6..l....| +000001c0 09 37 aa 55 32 1a 27 a7 5b 37 34 32 35 36 f5 14 |.7.U2.'.[74256..| +000001d0 08 07 00 40 3d 7c 51 33 98 02 91 19 da 47 1a 1a |...@=|Q3.....G..| +000001e0 fe 03 4e 1b e8 5c 5e a7 12 9c e6 8a c2 74 18 dd |..N..\^......t..| +000001f0 18 9a 9d d6 60 f3 c8 63 00 33 d8 81 3e 2e 40 0f |....`..c.3..>.@.| +00000200 c5 98 fc a8 1c 6b 0c 19 70 f5 16 a5 25 ce 7b b8 |.....k..p...%.{.| +00000210 f2 f5 87 04 16 03 03 00 04 0e 00 00 00 |.............| >>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| -00000030 16 03 03 00 20 69 ac be cf f1 3d 8a 4e f3 3a 54 |.... i....=.N.:T| -00000040 05 d8 20 3a f0 67 40 3f 5b 94 cf a4 3e ec 8e 4e |.. :.g@?[...>..N| -00000050 75 c9 92 2c af |u..,.| +00000030 16 03 03 00 20 91 a7 c4 90 65 85 ce 63 66 fa 8b |.... ....e..cf..| +00000040 5c 22 e1 1d fb 14 c2 6c 86 7c 52 7c d1 96 3b 41 |\".....l.|R|..;A| +00000050 20 0d bf 09 e8 | ....| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 20 70 c5 f8 c7 7c |.......... p...|| -00000010 4a 34 6f b0 eb 1b 13 c7 e0 21 32 cf ce 89 aa 53 |J4o......!2....S| -00000020 5f 67 b8 a7 3e 10 bb a3 5f fa 2f |_g..>..._./| +00000000 14 03 03 00 01 01 16 03 03 00 20 d0 3e 15 0e 08 |.......... .>...| +00000010 5b fd a8 b7 5f 01 7f ca 6d af 78 6e 33 7a 93 42 |[..._...m.xn3z.B| +00000020 aa 6e e4 f4 a1 bb 9a 2d bb 17 6f |.n.....-..o| >>> Flow 5 (client to server) -00000000 17 03 03 00 16 54 ae 8d 1c 04 40 47 8a 7e ac c3 |.....T....@G.~..| -00000010 21 a2 8a f0 5d 3d 12 9d b0 b5 8d 15 03 03 00 12 |!...]=..........| -00000020 71 3b 30 89 d4 c3 ef 34 66 92 29 1b e1 af d4 d2 |q;0....4f.).....| -00000030 7e cd |~.| +00000000 17 03 03 00 16 7b f9 2f ce ac bc 43 3c 6b 10 3b |.....{./...C>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 18 47 80 ab 8d |....]...Y...G...| -00000010 42 c7 73 06 75 ef 28 ff 87 46 49 af 3a 3b cb bb |B.s.u.(..FI.:;..| -00000020 ff 6e e3 60 11 40 2c b8 40 87 5c 20 86 3d de c2 |.n.`.@,.@.\ .=..| -00000030 18 41 ff 1a dc 00 77 4e 17 0b 36 f3 69 92 d7 04 |.A....wN..6.i...| -00000040 c4 c6 39 ad ed f7 06 23 8c 52 50 da cc a8 00 00 |..9....#.RP.....| +00000000 16 03 03 00 5d 02 00 00 59 03 03 ad 2d c6 4e 13 |....]...Y...-.N.| +00000010 72 ea 19 e0 65 33 ec cb aa 5c 70 bd f5 70 e0 b4 |r...e3...\p..p..| +00000020 ca 7e 64 da 5e 5f cb a6 69 d4 8c 20 79 45 e2 0a |.~d.^_..i.. yE..| +00000030 c7 b2 be 37 28 b5 c6 35 f9 0c c9 34 39 4d 38 3b |...7(..5...49M8;| +00000040 c0 47 31 44 e2 47 f7 6f 70 0f 8b 2d cc a8 00 00 |.G1D.G.op..-....| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -61,31 +62,31 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 c5 28 44 |............ .(D| -000002d0 4a 31 d9 b6 53 90 8e 41 3c 67 2c 6f 9d b0 a6 d3 |J1..S..A>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| -00000030 16 03 03 00 20 9f 16 49 2a d8 cc 0a 45 92 8d c2 |.... ..I*...E...| -00000040 25 3c b1 5c a8 f2 c4 9b c4 2a 66 fe 7f 87 df 35 |%<.\.....*f....5| -00000050 77 a4 b0 05 f5 |w....| +00000030 16 03 03 00 20 e1 88 c9 ef 40 26 52 ca d2 d9 31 |.... ....@&R...1| +00000040 9c 51 a6 fa b3 40 6b 62 7e ad 00 ea da 8f 54 bf |.Q...@kb~.....T.| +00000050 f5 89 4b c9 a9 |..K..| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 20 37 55 d5 05 71 |.......... 7U..q| -00000010 a0 00 44 56 32 c3 c8 e6 68 5e de d4 e7 65 56 d3 |..DV2...h^...eV.| -00000020 2d 93 07 e6 72 17 e9 49 74 4d c0 |-...r..ItM.| +00000000 14 03 03 00 01 01 16 03 03 00 20 6f e3 54 91 aa |.......... o.T..| +00000010 00 b4 8e f3 b2 51 1e 76 48 91 49 b9 ba dc 5f e4 |.....Q.vH.I..._.| +00000020 9f 0e 49 b5 54 f5 e3 9d 25 75 38 |..I.T...%u8| >>> Flow 5 (client to server) -00000000 17 03 03 00 16 27 a6 31 16 5c 7e bd 05 0a 2c 05 |.....'.1.\~...,.| -00000010 ba 37 06 00 d7 8d 66 a7 8e 16 7b 15 03 03 00 12 |.7....f...{.....| -00000020 70 38 bc 7e 6b 83 83 e1 61 3b bb a5 d9 dc f7 e7 |p8.~k...a;......| -00000030 5e 12 |^.| +00000000 17 03 03 00 16 7b 2a af 79 d3 1c 0d cf 0f 83 04 |.....{*.y.......| +00000010 9e 96 63 69 0c ec f7 98 02 33 12 15 03 03 00 12 |..ci.....3......| +00000020 58 d3 1a 60 74 2c 92 91 61 2c 13 23 6d e9 5f de |X..`t,..a,.#m._.| +00000030 65 42 |eB| diff --git a/src/crypto/tls/testdata/Client-TLSv12-P256-ECDHE b/src/crypto/tls/testdata/Client-TLSv12-P256-ECDHE index 7a0a53c03a097c..0cac7e7545cb29 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-P256-ECDHE +++ b/src/crypto/tls/testdata/Client-TLSv12-P256-ECDHE @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 01 19 01 00 01 15 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 33 01 00 01 2f 03 03 00 00 00 00 00 |....3.../.......| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,22 +7,24 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 9a 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 b4 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| -000000a0 00 0a 00 04 00 02 00 17 00 0d 00 1a 00 18 08 04 |................| +000000a0 00 0a 00 04 00 02 00 17 00 0d 00 16 00 14 08 04 |................| 000000b0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| -000000c0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| -000000d0 02 03 01 00 33 00 47 00 45 00 17 00 41 04 1e 18 |....3.G.E...A...| -000000e0 37 ef 0d 19 51 88 35 75 71 b5 e5 54 5b 12 2e 8f |7...Q.5uq..T[...| -000000f0 09 67 fd a7 24 20 3e b2 56 1c ce 97 28 5e f8 2b |.g..$ >.V...(^.+| -00000100 2d 4f 9e f1 07 9f 6c 4b 5b 83 56 e2 32 42 e9 58 |-O....lK[.V.2B.X| -00000110 b6 d7 49 a6 b5 68 1a 41 03 56 6b dc 5a 89 |..I..h.A.Vk.Z.| +000000c0 06 03 00 32 00 1a 00 18 08 04 04 03 08 07 08 05 |...2............| +000000d0 08 06 04 01 05 01 06 01 05 03 06 03 02 01 02 03 |................| +000000e0 00 2b 00 09 08 03 04 03 03 03 02 03 01 00 33 00 |.+............3.| +000000f0 47 00 45 00 17 00 41 04 1e 18 37 ef 0d 19 51 88 |G.E...A...7...Q.| +00000100 35 75 71 b5 e5 54 5b 12 2e 8f 09 67 fd a7 24 20 |5uq..T[....g..$ | +00000110 3e b2 56 1c ce 97 28 5e f8 2b 2d 4f 9e f1 07 9f |>.V...(^.+-O....| +00000120 6c 4b 5b 83 56 e2 32 42 e9 58 b6 d7 49 a6 b5 68 |lK[.V.2B.X..I..h| +00000130 1a 41 03 56 6b dc 5a 89 |.A.Vk.Z.| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 1c ad ae e9 64 |....]...Y......d| -00000010 35 80 c6 04 f5 69 05 bb 5c 38 78 d3 99 2f ad e6 |5....i..\8x../..| -00000020 56 20 d4 79 89 1a 51 54 14 ac 6d 20 06 6d 1a bd |V .y..QT..m .m..| -00000030 04 cf 4f 42 3d aa 31 3b fb dc c1 85 42 c3 6d 12 |..OB=.1;....B.m.| -00000040 27 98 be 4b 88 93 45 0e 78 ba d8 13 c0 2f 00 00 |'..K..E.x..../..| +00000000 16 03 03 00 5d 02 00 00 59 03 03 d9 70 61 62 c5 |....]...Y...pab.| +00000010 f4 b4 22 c6 ea f6 e9 8c fc 29 e8 8c 24 2d 4b c4 |.."......)..$-K.| +00000020 04 57 01 ae e5 41 7e 7f 6f ec c3 20 86 6b 38 a9 |.W...A~.o.. .k8.| +00000030 ea cd 51 3b 0f 05 5c a3 98 0c 4d ba 46 d6 99 f3 |..Q;..\...M.F...| +00000040 67 da f2 62 56 29 f7 e6 8c 14 8a 7a c0 2f 00 00 |g..bV).....z./..| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -62,37 +64,37 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 03 00 cd 0c 00 00 c9 03 00 17 41 04 3a 1c |............A.:.| -000002d0 84 6b e6 cb 7d 3f bd ea d7 7a c3 63 43 77 f7 14 |.k..}?...z.cCw..| -000002e0 a7 98 52 86 49 4d 05 b2 70 e2 d3 44 c9 05 eb 83 |..R.IM..p..D....| -000002f0 b4 11 c5 3e 9d 3b 3b cd de d0 df 78 83 0f 0c d8 |...>.;;....x....| -00000300 76 e7 19 9d 1f 5a 3d 2a 54 19 16 a0 c2 55 08 04 |v....Z=*T....U..| -00000310 00 80 09 ee 97 8a d1 82 fc 4a fc f2 89 d2 a3 c7 |.........J......| -00000320 8c 25 74 98 cf b5 67 c6 fd f6 0d 3b 62 ba 31 ca |.%t...g....;b.1.| -00000330 d1 67 aa c3 32 89 ee 83 22 d2 b7 3c f8 98 af 16 |.g..2..."..<....| -00000340 dd 05 06 2c c4 57 dd fc 22 3a 2e 80 f9 84 6a ee |...,.W..":....j.| -00000350 f2 a1 21 d1 0f db 17 50 a1 35 94 b3 0a b7 62 ca |..!....P.5....b.| -00000360 2c b3 0b 68 0c 58 d7 24 83 82 d8 34 64 3f a4 61 |,..h.X.$...4d?.a| -00000370 0e b1 ca 93 67 b3 51 11 dc 49 f2 df 26 89 8b ea |....g.Q..I..&...| -00000380 22 09 1a be 62 27 61 c8 93 4b dd 95 31 6a ba e2 |"...b'a..K..1j..| -00000390 c3 2b 16 03 03 00 04 0e 00 00 00 |.+.........| +000002c0 16 03 03 00 cd 0c 00 00 c9 03 00 17 41 04 a0 b7 |............A...| +000002d0 0f 94 44 46 c4 72 5a 3c b8 5c cd f1 9e b0 49 17 |..DF.rZ<.\....I.| +000002e0 ea a0 ca 1d 17 d8 85 5f 0a e3 4a 85 5d f4 bb 52 |......._..J.]..R| +000002f0 23 79 24 a9 50 81 9f 8a 5e e4 2f 5d cc 7d e2 10 |#y$.P...^./].}..| +00000300 db 42 f0 26 98 61 b7 f5 39 16 46 d6 04 f7 08 04 |.B.&.a..9.F.....| +00000310 00 80 05 70 60 b3 39 e0 61 6e 72 b3 13 2c 8c 2d |...p`.9.anr..,.-| +00000320 e1 d9 37 8e cd 13 64 2f 4b 65 4f 7c a0 2a 7e 91 |..7...d/KeO|.*~.| +00000330 e1 fd 7d 3b e3 35 dd 5a c8 ab 56 13 c3 c6 07 90 |..};.5.Z..V.....| +00000340 fb d3 cf 22 dd 78 4c 33 c7 d3 6e 64 7e 65 ba 02 |...".xL3..nd~e..| +00000350 ca 04 50 77 08 65 d5 80 c9 4f c3 b3 5b 97 b2 80 |..Pw.e...O..[...| +00000360 3d 48 37 20 ab 28 15 0c f2 86 dd b6 19 66 46 e3 |=H7 .(.......fF.| +00000370 23 69 e2 5d fb 70 65 fb c6 ae fc a4 02 42 9f fe |#i.].pe......B..| +00000380 b4 5c f0 4b 23 92 2a 4f a4 1a f4 86 3b 5c 25 9d |.\.K#.*O....;\%.| +00000390 86 e2 16 03 03 00 04 0e 00 00 00 |...........| >>> Flow 3 (client to server) 00000000 16 03 03 00 46 10 00 00 42 41 04 1e 18 37 ef 0d |....F...BA...7..| 00000010 19 51 88 35 75 71 b5 e5 54 5b 12 2e 8f 09 67 fd |.Q.5uq..T[....g.| 00000020 a7 24 20 3e b2 56 1c ce 97 28 5e f8 2b 2d 4f 9e |.$ >.V...(^.+-O.| 00000030 f1 07 9f 6c 4b 5b 83 56 e2 32 42 e9 58 b6 d7 49 |...lK[.V.2B.X..I| 00000040 a6 b5 68 1a 41 03 56 6b dc 5a 89 14 03 03 00 01 |..h.A.Vk.Z......| -00000050 01 16 03 03 00 28 00 00 00 00 00 00 00 00 fa c4 |.....(..........| -00000060 4e 47 d1 de 6d 60 fc af 59 43 7f 2f 4c b3 6e aa |NG..m`..YC./L.n.| -00000070 6f c3 bc 8a 40 5e e9 9c d9 b0 87 c5 92 41 |o...@^.......A| +00000050 01 16 03 03 00 28 00 00 00 00 00 00 00 00 3f 54 |.....(........?T| +00000060 35 c8 33 70 66 0f 9d 3d 57 d7 a8 7f 57 45 63 a4 |5.3pf..=W...WEc.| +00000070 67 a2 83 c4 17 5b 22 8f 1c 89 78 1c 43 8d |g....["...x.C.| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 28 eb 92 83 02 be |..........(.....| -00000010 89 55 4e 77 b2 45 1d 15 fd e0 6a 36 ab 0c 7b 4a |.UNw.E....j6..{J| -00000020 5d 50 e2 5a f3 c6 10 e2 ac ac a4 d3 4c 2e d8 31 |]P.Z........L..1| -00000030 84 31 17 |.1.| +00000000 14 03 03 00 01 01 16 03 03 00 28 25 5d 63 2d f3 |..........(%]c-.| +00000010 a5 91 45 d0 7d 68 39 f6 0e 04 6f 4b c9 dd 4a 54 |..E.}h9...oK..JT| +00000020 f5 a2 04 54 d2 57 4b 40 75 4d 2f b3 5d 00 5b 11 |...T.WK@uM/.].[.| +00000030 89 0e 92 |...| >>> Flow 5 (client to server) -00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 90 44 33 |..............D3| -00000010 cf be 4e 29 fc 6f 2f 30 c1 ef aa 8e cb 10 0e f9 |..N).o/0........| -00000020 8f 60 fc 15 03 03 00 1a 00 00 00 00 00 00 00 02 |.`..............| -00000030 80 cb 52 6b e7 c9 45 af 69 6b e8 ee 88 23 61 20 |..Rk..E.ik...#a | -00000040 74 33 |t3| +00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 0c 9c ab |................| +00000010 24 e8 b2 cf a2 6f f9 8e 54 ed 35 43 07 4d 8f 89 |$....o..T.5C.M..| +00000020 5c 08 15 15 03 03 00 1a 00 00 00 00 00 00 00 02 |\...............| +00000030 f8 65 48 1e a4 03 c0 3b 2f 23 c7 9e 92 59 f6 be |.eH....;/#...Y..| +00000040 b4 a2 |..| diff --git a/src/crypto/tls/testdata/Client-TLSv12-RSA-RC4 b/src/crypto/tls/testdata/Client-TLSv12-RSA-RC4 index 24a9b8e4155228..faa48c784371b6 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-RSA-RC4 +++ b/src/crypto/tls/testdata/Client-TLSv12-RSA-RC4 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 55 02 00 00 51 03 03 7c f1 44 f1 2a |....U...Q..|.D.*| -00000010 6f 7f 62 66 08 15 dd c8 bd 4a 30 b5 a3 73 53 82 |o.bf.....J0..sS.| -00000020 93 78 6b f2 c6 70 70 1e 4d 8f 48 20 24 b3 17 46 |.xk..pp.M.H $..F| -00000030 37 c0 0f 79 af f4 48 43 17 6b ea df 6b 73 07 cf |7..y..HC.k..ks..| -00000040 5f 95 c5 5c 8d b2 99 5c fd 30 e5 1b 00 05 00 00 |_..\...\.0......| +00000000 16 03 03 00 55 02 00 00 51 03 03 c4 5b d6 0f d4 |....U...Q...[...| +00000010 64 3a 37 34 a3 84 9d 3d 1b 33 c3 d4 4f 42 f7 3a |d:74...=.3..OB.:| +00000020 e8 da 8c 8b 97 1a e5 f4 56 a4 bd 20 03 30 25 fd |........V.. .0%.| +00000030 d9 01 7a 7c b8 9d 63 f0 a3 7a 1c 00 6b 75 16 d8 |..z|..c..z..ku..| +00000040 68 5c 7a 8d 4a ae ed 52 fc 92 e5 03 00 05 00 00 |h\z.J..R........| 00000050 09 ff 01 00 01 00 00 17 00 00 16 03 03 02 59 0b |..............Y.| 00000060 00 02 55 00 02 52 00 02 4f 30 82 02 4b 30 82 01 |..U..R..O0..K0..| 00000070 b4 a0 03 02 01 02 02 09 00 e8 f0 9d 3f e2 5b ea |............?.[.| @@ -72,15 +73,15 @@ 00000060 c5 70 0f 08 83 48 e9 48 ef 6e 50 8b 05 7e e5 84 |.p...H.H.nP..~..| 00000070 25 fa 55 c7 ae 31 02 27 00 ef 3f 98 86 20 12 89 |%.U..1.'..?.. ..| 00000080 91 59 28 b4 f7 d7 af d2 69 61 35 14 03 03 00 01 |.Y(.....ia5.....| -00000090 01 16 03 03 00 24 fd 67 40 11 7e 4d 8f bb 87 e8 |.....$.g@.~M....| -000000a0 c3 1f e8 b0 15 17 3a f4 6f 5f 87 b2 50 c5 92 e4 |......:.o_..P...| -000000b0 ac c0 c0 98 aa 6f be bc fc 77 |.....o...w| +00000090 01 16 03 03 00 24 83 66 73 cf 08 6f c7 29 1b ea |.....$.fs..o.)..| +000000a0 86 aa 02 66 2d d9 d4 7b 19 0d cc e5 9e dc e9 20 |...f-..{....... | +000000b0 d9 62 82 59 bd e9 91 6b 90 62 |.b.Y...k.b| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 24 e8 ea 6f 8a 0c |..........$..o..| -00000010 79 70 73 a9 7b 2c a8 0a e5 f8 26 f3 eb 05 20 65 |yps.{,....&... e| -00000020 58 9b 0c 8e 28 3d c7 ee cf 57 85 06 a5 d7 46 |X...(=...W....F| +00000000 14 03 03 00 01 01 16 03 03 00 24 ef bb 0c 3b fb |..........$...;.| +00000010 4e 36 53 b9 fa d4 c3 d0 37 5e 2b 22 a2 12 b6 9e |N6S.....7^+"....| +00000020 1c a3 f7 93 74 60 38 18 f1 cc 72 e4 ba 1a 3d |....t`8...r...=| >>> Flow 5 (client to server) -00000000 17 03 03 00 1a 2d 7a c5 d0 12 79 e8 a2 05 d3 a4 |.....-z...y.....| -00000010 51 80 8f 28 37 b9 75 64 f3 11 4c 78 de 0c f3 15 |Q..(7.ud..Lx....| -00000020 03 03 00 16 43 89 2c 8b 9e dc c5 5d 4c ca 7b 4e |....C.,....]L.{N| -00000030 15 e4 e5 62 01 02 b4 d4 6f 1c |...b....o.| +00000000 17 03 03 00 1a 62 a3 b3 3a 89 25 05 89 bc 5a 21 |.....b..:.%...Z!| +00000010 30 61 84 b0 97 52 26 a2 18 87 25 43 17 1a 52 15 |0a...R&...%C..R.| +00000020 03 03 00 16 97 d6 18 0c 30 b3 9b 8f 2b f1 f9 3f |........0...+..?| +00000030 c7 8d a2 e1 ff 36 0c d6 c2 f1 |.....6....| diff --git a/src/crypto/tls/testdata/Client-TLSv12-RenegotiateOnce b/src/crypto/tls/testdata/Client-TLSv12-RenegotiateOnce index 0510d5e2d33e46..868b527396bd00 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-RenegotiateOnce +++ b/src/crypto/tls/testdata/Client-TLSv12-RenegotiateOnce @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 55 71 df 37 76 |....]...Y..Uq.7v| -00000010 48 bb 86 62 66 f1 58 0d 92 f4 c8 bd 17 6c 00 43 |H..bf.X......l.C| -00000020 a9 da f1 6f 19 2c 76 81 6d aa eb 20 4f c7 eb 3f |...o.,v.m.. O..?| -00000030 b8 48 89 7f d8 61 bc e8 3c e6 a0 3d 6c 29 fd 60 |.H...a..<..=l).`| -00000040 7c 0a 09 1c 71 41 07 04 24 dc e7 27 cc a8 00 00 ||...qA..$..'....| +00000000 16 03 03 00 5d 02 00 00 59 03 03 9e b0 dd 2b 97 |....]...Y.....+.| +00000010 86 13 76 2e 65 fc 27 bc fd 01 c0 05 86 b9 e3 bd |..v.e.'.........| +00000020 b9 46 3b 09 37 2a fe 3a eb ee e0 20 65 ad 7c 0b |.F;.7*.:... e.|.| +00000030 1a 0b 9b da b0 8d 56 86 b6 e1 04 1b dd 5d 93 8e |......V......]..| +00000040 a9 ea b6 b7 17 c9 75 26 58 5a c5 37 cc a8 00 00 |......u&XZ.7....| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -61,186 +62,188 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 7a 15 23 |............ z.#| -000002d0 f5 78 c2 16 21 72 bb e5 d3 2e c9 67 55 a8 2c 1f |.x..!r.....gU.,.| -000002e0 00 50 a2 04 4f 71 9a 94 4a 90 55 69 33 08 04 00 |.P..Oq..J.Ui3...| -000002f0 80 53 b3 16 f5 08 3a 5a 84 3c 02 5d c3 b6 0e c2 |.S....:Z.<.]....| -00000300 93 f6 27 74 24 b4 e2 3d 22 04 30 a0 d6 5b c8 da |..'t$..=".0..[..| -00000310 6e 56 30 d4 fb 86 fd 76 8f 1b 47 c0 55 95 cd 15 |nV0....v..G.U...| -00000320 bf a7 27 ce 2d c3 43 6b ed 2d 09 bb eb 55 73 c9 |..'.-.Ck.-...Us.| -00000330 2c ee cd 23 af d9 1c 51 1d 0c 00 14 01 61 8a fd |,..#...Q.....a..| -00000340 8f 95 d1 ce 32 57 ae 2b 09 15 b4 86 41 1e 74 f5 |....2W.+....A.t.| -00000350 d5 96 d7 e1 03 03 09 f4 3c 53 fe 6b 9a c1 52 ab |........>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| -00000030 16 03 03 00 20 bf 35 ae 8c b4 63 82 40 e8 5d d4 |.... .5...c.@.].| -00000040 bb 3d 0a 0f 07 28 3a 74 2c 5b a8 d0 a6 ab 88 ba |.=...(:t,[......| -00000050 7d 5c 1c 42 c7 |}\.B.| +00000030 16 03 03 00 20 5b d2 2a ad 40 50 6f a8 b8 70 d0 |.... [.*.@Po..p.| +00000040 b9 aa ab d5 d7 2a f1 2b f7 d7 9b cd 10 a5 80 e9 |.....*.+........| +00000050 d3 15 3f 4f 00 |..?O.| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 20 e7 86 48 6c 00 |.......... ..Hl.| -00000010 0a 1e f5 5f b8 ff ad 1c 9d 74 24 c5 6e d4 c6 6e |..._.....t$.n..n| -00000020 25 27 7d 6e 18 44 96 29 eb db b0 |%'}n.D.)...| +00000000 14 03 03 00 01 01 16 03 03 00 20 dd 90 d8 8a 0a |.......... .....| +00000010 11 a6 30 9b b0 29 50 c8 b1 70 36 bc fb 84 6e 6d |..0..)P..p6...nm| +00000020 a9 5e 67 3c e2 2e 0b 09 b2 44 38 |.^g<.....D8| >>> Flow 5 (client to server) -00000000 17 03 03 00 16 64 9d 31 09 a1 ce 5a 6d ec 6f c4 |.....d.1...Zm.o.| -00000010 e5 09 df 92 5f d2 1c 7d 62 c4 e7 |...._..}b..| +00000000 17 03 03 00 16 9f e0 6a fc e9 a2 bf cf b4 df 07 |.......j........| +00000010 ba b4 8b 8b 08 ca bd a7 d5 02 2a |..........*| >>> Flow 6 (server to client) -00000000 16 03 03 00 14 9b cb 75 61 99 76 30 9c 14 a5 12 |.......ua.v0....| -00000010 92 de b7 95 cc 8f fe ad f1 |.........| +00000000 16 03 03 00 14 ed 9b 22 44 dd b2 e2 c3 7e ab 0c |......."D....~..| +00000010 69 98 d0 7b 40 9e da e2 67 |i..{@...g| >>> Flow 7 (client to server) -00000000 16 03 03 01 1a c7 9a be e1 2a 62 9f 01 59 e9 6a |.........*b..Y.j| -00000010 5f d6 32 88 bd 39 76 1b 34 fd 39 8d c7 31 cb 97 |_.2..9v.4.9..1..| -00000020 c6 09 8b 00 c9 f3 f9 00 c5 b6 48 e1 28 ae b8 8b |..........H.(...| -00000030 b2 f0 eb 8b cb e8 68 30 50 b5 23 ba cf f3 8d f3 |......h0P.#.....| -00000040 47 5f 94 ac fb 5e 56 9e 5a f8 e7 12 60 13 83 9c |G_...^V.Z...`...| -00000050 34 ca 45 a7 63 5f 2c 6e 46 c8 5a a0 d3 5b 9a a6 |4.E.c_,nF.Z..[..| -00000060 70 be da df 9e 79 21 98 bd fb 94 01 6d 14 f4 5c |p....y!.....m..\| -00000070 2a 13 56 26 47 33 7d 4f ab 5a fc 9c 9e 8d f4 75 |*.V&G3}O.Z.....u| -00000080 d0 64 16 ba f2 0d 04 ca 5d 94 6b da a8 09 b1 29 |.d......].k....)| -00000090 70 a9 37 1e ac 94 e3 81 60 c3 19 f3 a9 99 6a 11 |p.7.....`.....j.| -000000a0 b1 e7 23 45 8a f5 42 f5 50 76 9f 1e 9e a8 e7 75 |..#E..B.Pv.....u| -000000b0 4a 18 84 80 da 10 ed 83 9a 14 a9 a1 90 54 8e 8b |J............T..| -000000c0 d1 32 83 6d e7 7e be 59 f4 66 59 53 75 37 c6 82 |.2.m.~.Y.fYSu7..| -000000d0 15 aa 56 0a 01 e1 11 ba 64 0c 8e 44 93 60 29 a4 |..V.....d..D.`).| -000000e0 cc 9f b0 b9 d5 df 9c aa 64 2c ef 0c 9a 18 2a 97 |........d,....*.| -000000f0 e1 20 07 37 35 28 97 9d 00 53 61 11 81 22 45 9e |. .75(...Sa.."E.| -00000100 c3 d7 b2 b3 4c f2 05 5b e6 a4 de 11 7f a7 a2 82 |....L..[........| -00000110 88 1c cb d9 24 80 3d 34 0c fc 22 58 32 28 1b |....$.=4.."X2(.| +00000000 16 03 03 01 34 0d de 90 41 3c 76 1d 54 9d c2 7e |....4...A?J..}......| +00000070 4c 1c fb 2c 88 4c 13 8d c8 d0 e3 7f f6 cb 2e ca |L..,.L..........| +00000080 cb 92 ae 30 fe ea 3c 3e ae 57 51 6a d6 98 5c d6 |...0..<>.WQj..\.| +00000090 19 85 72 bd 62 56 fa c2 ea 08 eb b4 d9 75 ac de |..r.bV.......u..| +000000a0 a5 40 8d 8b 1f c4 8f f6 6d b2 5f d1 6d fb 14 fa |.@......m._.m...| +000000b0 e2 92 9e 04 37 d3 60 83 28 4a f9 92 8c b3 4a d2 |....7.`.(J....J.| +000000c0 ce 6d 95 e4 40 79 7c 3d 98 11 e7 da 5c 56 0a 95 |.m..@y|=....\V..| +000000d0 e0 6f 36 80 07 da 29 ef 88 44 db cd 2b 5a a5 27 |.o6...)..D..+Z.'| +000000e0 56 5a 94 49 aa 51 ae 48 f9 12 8d df e0 2a a7 67 |VZ.I.Q.H.....*.g| +000000f0 7f 55 3e 7a b6 bb 28 e3 a8 d9 44 67 e9 03 8f e5 |.U>z..(...Dg....| +00000100 2b 40 a3 37 fb e4 51 0a 18 8f bb 14 2e 5b 62 93 |+@.7..Q......[b.| +00000110 ab a4 40 d7 5f 15 12 cf 93 74 37 d0 ec 14 d3 a3 |..@._....t7.....| +00000120 eb 6f 61 35 4a c4 b5 d7 d1 bb ae fe 11 6d 04 ac |.oa5J........m..| +00000130 58 72 a6 72 b6 06 9d 93 a5 |Xr.r.....| >>> Flow 8 (server to client) -00000000 16 03 03 00 85 ba 55 5d 15 b8 6d 8a b6 82 a8 e1 |......U]..m.....| -00000010 88 ea fe c0 a6 a8 fe 78 ed 8b ae 44 eb 7b 6c cc |.......x...D.{l.| -00000020 0f ad a1 da da 86 fb 60 07 4a 0e 5c ba 36 09 8b |.......`.J.\.6..| -00000030 95 3f f0 87 26 0f 7a e7 7c 1f af c7 67 42 c9 39 |.?..&.z.|...gB.9| -00000040 39 6a 4d 8d 87 00 3b 14 76 4a 86 87 46 1e d4 04 |9jM...;.vJ..F...| -00000050 2d ea c1 44 1f e8 87 71 da 1e 26 72 a2 e9 40 0c |-..D...q..&r..@.| -00000060 33 6e 6f 06 43 ed 7f fc 8f 4c d4 f4 0f 83 19 cb |3no.C....L......| -00000070 52 a9 94 f3 6a 64 db dd 20 d1 a7 d4 c7 6e 86 ed |R...jd.. ....n..| -00000080 6e ea 5c 66 d1 34 a6 92 7e b1 16 03 03 02 69 48 |n.\f.4..~.....iH| -00000090 13 97 82 60 46 e1 c9 02 d2 e1 f3 62 da d6 3a c1 |...`F......b..:.| -000000a0 2a a7 47 bd c2 ee 09 d5 2d 7e 5e 35 09 79 88 35 |*.G.....-~^5.y.5| -000000b0 96 50 09 68 79 e9 de e1 71 97 11 60 c5 4e 84 54 |.P.hy...q..`.N.T| -000000c0 03 78 69 84 e3 7f 02 1a 58 04 ed 53 47 01 b6 9e |.xi.....X..SG...| -000000d0 bd 5f 77 34 e7 52 3b 73 45 e1 10 3f c8 6e 4c 09 |._w4.R;sE..?.nL.| -000000e0 d0 4b 79 f8 de 0f 15 e7 2d 42 5d 7a 89 80 1f fb |.Ky.....-B]z....| -000000f0 f9 02 6f a1 72 d4 8d 65 8e d0 a8 43 41 4c 48 57 |..o.r..e...CALHW| -00000100 71 21 b3 14 45 e5 b1 f5 55 7a dd 89 c3 0f af 27 |q!..E...Uz.....'| -00000110 03 d6 5c 2d bc 24 5b 51 fa ae 34 7b f9 56 e8 41 |..\-.$[Q..4{.V.A| -00000120 cd 77 1c 3d c8 45 f5 a5 4a 97 92 ff bb cd 5b ff |.w.=.E..J.....[.| -00000130 d5 33 d6 02 9b 2a 5a 3a e4 1b fb 48 34 b3 0b ce |.3...*Z:...H4...| -00000140 d7 34 5c 4a b4 4b bd 87 b6 72 40 ab 29 b0 65 25 |.4\J.K...r@.).e%| -00000150 c7 1d 71 b6 e2 d7 b1 23 b1 96 a6 bd 74 eb ad 69 |..q....#....t..i| -00000160 59 0c 0f af 8a 64 be e4 a7 27 c2 95 11 05 55 a1 |Y....d...'....U.| -00000170 d6 44 df ad 1d 9c 3a 88 24 52 52 9d 42 f2 74 98 |.D....:.$RR.B.t.| -00000180 08 9f 55 1b 2f 79 ca b4 63 38 e4 f2 fa 99 ce 66 |..U./y..c8.....f| -00000190 77 ac 8d 31 91 05 1c bc 51 0a 31 df 5d 3e f8 69 |w..1....Q.1.]>.i| -000001a0 b7 fa f2 35 af 57 6d 7b c3 bf 1d 98 21 40 dd 02 |...5.Wm{....!@..| -000001b0 1c de ac 02 40 c8 d6 04 23 30 71 16 d0 0a 26 29 |....@...#0q...&)| -000001c0 66 e9 f1 a8 76 f8 52 18 3e 3f c5 66 c9 11 04 6c |f...v.R.>?.f...l| -000001d0 32 1b 35 cc 9a 34 70 07 da db 12 51 78 77 dc bc |2.5..4p....Qxw..| -000001e0 7a bb b8 b9 06 79 bb 04 dd d4 46 8e b5 69 d5 39 |z....y....F..i.9| -000001f0 5e 34 8e 37 dd a2 3e 31 be 28 19 45 21 76 5b dc |^4.7..>1.(.E!v[.| -00000200 7d 25 a2 41 98 16 1b 5b 69 f0 90 f9 28 a8 e9 b5 |}%.A...[i...(...| -00000210 6f 00 e3 3e eb f6 f3 e4 c9 0f bb 74 ce 40 23 df |o..>.......t.@#.| -00000220 12 c6 78 b0 2e bb 68 96 04 a3 e3 43 6b 4b c3 37 |..x...h....CkK.7| -00000230 6d 97 c6 79 1a ff 4f 00 c7 76 64 5c b8 b3 17 20 |m..y..O..vd\... | -00000240 3c 3e 6b d5 2b 72 88 11 73 1e 63 a7 6f 1e ae 83 |<>k.+r..s.c.o...| -00000250 10 77 6f d1 96 86 84 63 fd 27 8e b9 54 da 4b b5 |.wo....c.'..T.K.| -00000260 56 f0 50 8a aa c7 e1 b3 cb 9c 36 9e ec 38 31 39 |V.P.......6..819| -00000270 78 ec ea 34 8a 87 cf 6b 34 fd 5e 81 92 81 61 f1 |x..4...k4.^...a.| -00000280 88 e7 50 62 2e 58 0b d9 b9 ca f3 ed 79 a9 9a 01 |..Pb.X......y...| -00000290 80 9d 7f 84 ae de fb 51 ac 0f 6b b9 76 fd 68 d4 |.......Q..k.v.h.| -000002a0 f4 8d c3 92 6b 5e 9e 99 ff 7a f4 b7 0e ec 0d e7 |....k^...z......| -000002b0 6b 24 75 fb 21 49 6f 5b 31 12 f2 88 8e 3b b4 34 |k$u.!Io[1....;.4| -000002c0 f2 f7 c6 e7 48 0e d6 1d 81 11 32 16 01 cb b7 7f |....H.....2.....| -000002d0 a6 bd 9c 87 8c 0f 12 b2 dc 73 5d a0 46 fd f3 e8 |.........s].F...| -000002e0 b8 6f e4 38 ab 94 96 a8 bd ed 90 0e 31 1f fc 2f |.o.8........1../| -000002f0 8f 50 e2 97 6a f5 5c b4 16 03 03 00 bc 65 bb 7c |.P..j.\......e.|| -00000300 42 03 0d 42 46 e3 ac 9f 8a 96 b6 a3 f4 cf a5 30 |B..BF..........0| -00000310 5b 02 17 dc 20 7d 18 19 21 a4 70 f4 4b 99 bf b5 |[... }..!.p.K...| -00000320 cd d7 07 59 a5 82 ed 85 03 06 5c 34 3f c8 c3 4d |...Y......\4?..M| -00000330 c3 fb 96 05 bf b5 bd bf e2 28 07 7e 51 a6 84 90 |.........(.~Q...| -00000340 bf 9e 2e f6 b5 04 8e 06 7a 63 c8 00 84 a1 a3 2c |........zc.....,| -00000350 f3 6f 52 52 c4 ce 4a 59 31 1f d4 ab 2e f4 75 90 |.oRR..JY1.....u.| -00000360 a5 3b ff ab 20 be 51 92 c5 f4 4d 8b f2 2a a7 ff |.;.. .Q...M..*..| -00000370 90 07 40 3e d6 9c cf 23 54 d1 65 d3 74 79 af 51 |..@>...#T.e.ty.Q| -00000380 31 35 40 aa 29 42 32 7e 42 2d af 79 e8 59 6a 10 |15@.)B2~B-.y.Yj.| -00000390 c5 55 6b 70 53 53 ab 02 48 ba 1f df 07 59 f8 34 |.UkpSS..H....Y.4| -000003a0 ee 1d d3 e9 12 86 8b 37 2b cc 27 78 fe e1 65 2b |.......7+.'x..e+| -000003b0 c3 1e f2 25 a4 6d 3c b1 d4 16 03 03 00 4a 50 98 |...%.m<......JP.| -000003c0 cd ee ab 2c 94 95 d6 46 06 ef 63 65 5c aa 6d 6e |...,...F..ce\.mn| -000003d0 fb a0 99 62 23 af 7a 7d 17 dc a3 c6 2b 9a 56 2e |...b#.z}....+.V.| -000003e0 b8 60 87 1e 88 5d 58 27 3f 02 b4 f5 2f 3b 17 8c |.`...]X'?.../;..| -000003f0 f9 93 0c a9 58 af 11 ed 27 9d af a5 0a 2b d6 55 |....X...'....+.U| -00000400 7e c4 54 4a 36 51 b7 7b 16 03 03 00 14 c1 c4 d7 |~.TJ6Q.{........| -00000410 ea 2d f0 8c 54 0e 19 33 e1 7e a8 ae 49 7c 5e 05 |.-..T..3.~..I|^.| -00000420 23 |#| +00000000 16 03 03 00 85 8b 8d 61 42 37 50 ee 69 d8 fb e9 |.......aB7P.i...| +00000010 89 a4 04 63 5d 87 a4 31 cc 22 26 c0 2c 22 1a 73 |...c]..1."&.,".s| +00000020 cc f0 9f be 65 c4 e0 ad 6d b0 b5 d3 86 2c 0b 59 |....e...m....,.Y| +00000030 e6 ae 58 38 b5 5f 4f 6e 7c ec e9 27 23 5f 4a 92 |..X8._On|..'#_J.| +00000040 78 5f cc 9f 7f 2b bb 2c dd 04 17 ea d4 87 b9 6d |x_...+.,.......m| +00000050 d0 1e 35 01 b4 4d 5d 73 6a 9f de 1d 71 47 78 d6 |..5..M]sj...qGx.| +00000060 89 dc c2 d6 5c 69 0c dc bb 7b 59 60 56 47 e9 5c |....\i...{Y`VG.\| +00000070 04 3e 3c 93 18 9b 40 4b 95 1f 20 e3 17 c6 ab 35 |.><...@K.. ....5| +00000080 db 3f 88 d7 4b 89 5a 53 b7 a9 16 03 03 02 69 85 |.?..K.ZS......i.| +00000090 49 cc 00 5a 45 40 9d be 80 06 97 16 a0 c0 dc bb |I..ZE@..........| +000000a0 32 62 18 88 31 3b 14 ec 9d f2 3f ed c0 36 ad bb |2b..1;....?..6..| +000000b0 69 fa 32 f5 13 54 2b 77 f3 db 6e 20 0a 2b 5c a5 |i.2..T+w..n .+\.| +000000c0 d8 aa 5a 63 8d c1 51 50 41 f6 6c f7 ef 2f 15 81 |..Zc..QPA.l../..| +000000d0 85 af 6e e5 01 95 9a 5e 70 26 cb c1 c8 89 3f b8 |..n....^p&....?.| +000000e0 3d 5a 1b 68 a6 25 08 48 ad dd 29 db bf ce 75 44 |=Z.h.%.H..)...uD| +000000f0 ca cd 92 79 47 d8 25 f1 23 9a 63 62 15 05 d8 2c |...yG.%.#.cb...,| +00000100 c0 30 a4 65 a7 f9 89 a4 fe 87 32 21 07 24 d9 16 |.0.e......2!.$..| +00000110 64 26 48 75 ab a7 00 e1 9b 48 5c 77 3d 66 18 69 |d&Hu.....H\w=f.i| +00000120 35 a9 4c 03 f5 9f 42 ff 50 98 73 b1 82 16 fd ff |5.L...B.P.s.....| +00000130 94 fd aa 3c 46 1f b5 99 fd de ab 2e 6a 24 00 5a |.........."| +00000300 3d 8d 48 5d 28 cd ae 94 db 95 2e a6 44 77 da d5 |=.H](.......Dw..| +00000310 a1 09 7c a3 4c 3f a6 18 80 92 cf 5c 77 9d 18 5d |..|.L?.....\w..]| +00000320 43 78 05 bb cb 8e 27 7c cc 2e 9c 54 95 ff 1c 74 |Cx....'|...T...t| +00000330 48 fb df 47 e8 e1 24 94 94 5d cc 2c b2 df 48 6c |H..G..$..].,..Hl| +00000340 22 1e 61 f1 e3 c6 56 fb 79 70 b0 ab c5 70 ca 7f |".a...V.yp...p..| +00000350 c0 32 8d 6d a5 55 6d 94 8c 7b 9d da 0b 35 d1 da |.2.m.Um..{...5..| +00000360 77 85 01 66 15 be 10 c3 3c 9b 80 61 b3 c0 1a 29 |w..f....<..a...)| +00000370 22 d1 d1 ba 4a ea 01 58 72 94 79 f0 6e 5f 36 23 |"...J..Xr.y.n_6#| +00000380 8a cf 16 fa 2c 5f 6d dd 54 61 df bc 6f 91 d4 df |....,_m.Ta..o...| +00000390 e7 6f 50 8f 9c 27 04 04 19 80 10 a2 8d a5 61 81 |.oP..'........a.| +000003a0 06 1f b0 79 87 c0 95 87 f7 d8 87 22 75 a5 87 30 |...y......."u..0| +000003b0 f4 33 6d cb 59 f7 18 75 0f 16 03 03 00 4a d3 e9 |.3m.Y..u.....J..| +000003c0 2e 47 65 f2 29 70 4c 74 3f e9 13 88 e7 66 c6 73 |.Ge.)pLt?....f.s| +000003d0 0f e4 7a 8f 0e 49 49 39 90 e7 ff d4 1f 8a d6 c6 |..z..II9........| +000003e0 91 16 f6 a6 fe 60 d8 ba 26 a6 5e 44 f5 76 88 63 |.....`..&.^D.v.c| +000003f0 10 d0 89 2f 6b ae 55 64 13 0c 63 76 69 df 6a 4a |.../k.Ud..cvi.jJ| +00000400 74 15 51 0b 4d 74 db 49 16 03 03 00 14 f2 0f e7 |t.Q.Mt.I........| +00000410 38 90 33 ff 5e 20 60 42 32 40 72 2c c8 70 d7 ba |8.3.^ `B2@r,.p..| +00000420 a1 |.| >>> Flow 9 (client to server) -00000000 16 03 03 02 69 83 ef 87 6b aa f1 2a cc c1 b8 5d |....i...k..*...]| -00000010 47 e4 2d da c5 91 62 e4 7d 49 da 54 a3 79 fb 1d |G.-...b.}I.T.y..| -00000020 59 49 e8 a6 ab 79 c0 9f 51 f9 d2 63 0d 8c 6b 7f |YI...y..Q..c..k.| -00000030 b6 77 2d f3 b3 3e 78 86 3a 14 1a ab 3b 5e 23 b2 |.w-..>x.:...;^#.| -00000040 d3 5c b1 2b 5c f4 f0 9d e2 87 08 d0 2f 24 30 d3 |.\.+\......./$0.| -00000050 05 eb f8 a6 b2 d1 52 00 c3 9e 0b 82 34 3e fd 5a |......R.....4>.Z| -00000060 46 d5 b6 b2 f0 a0 96 11 e1 35 c2 32 57 d6 dd b2 |F........5.2W...| -00000070 a8 ad b2 71 a2 6f 83 05 d6 f1 83 38 a0 3c 13 ff |...q.o.....8.<..| -00000080 c1 c1 b0 1e b5 40 5b b5 05 31 65 0b 81 a6 b0 37 |.....@[..1e....7| -00000090 7d 16 3b 7b cb e7 58 7f 81 25 e3 39 37 98 87 b0 |}.;{..X..%.97...| -000000a0 e1 8f c0 d5 de 33 fe de 15 37 41 25 d8 97 5d 84 |.....3...7A%..].| -000000b0 28 12 b4 a5 b0 15 f1 f4 cf 41 28 e9 26 5d ba 36 |(........A(.&].6| -000000c0 da b6 a6 cb 21 92 21 30 4b 21 4d 44 28 4d 76 d1 |....!.!0K!MD(Mv.| -000000d0 61 65 fa 02 05 67 50 ec 7d 98 78 21 46 5d fe 75 |ae...gP.}.x!F].u| -000000e0 78 8e 9d 41 6a 0a d7 0e 27 22 d0 a1 21 57 3d 23 |x..Aj...'"..!W=#| -000000f0 d1 7d 08 f4 0b 1b 90 ec 63 74 4c e7 df c8 8f 8b |.}......ctL.....| -00000100 b8 cd 2b 06 a5 35 f5 c1 62 d5 46 f3 d5 19 5b ce |..+..5..b.F...[.| -00000110 c8 d2 f4 c1 51 5f cd b2 51 23 61 bf 93 7f 5d bd |....Q_..Q#a...].| -00000120 61 f2 b2 e0 ad be ab d7 c9 77 26 c2 61 fb 25 8d |a........w&.a.%.| -00000130 46 38 7a 30 48 9f b5 5c 47 6b ce 10 1d 03 d0 68 |F8z0H..\Gk.....h| -00000140 7f 00 c0 94 f4 35 eb 41 e8 91 f6 d9 5c 44 d6 79 |.....5.A....\D.y| -00000150 72 9f 22 a4 08 fd 74 1b 42 dc 49 06 34 8f b6 f5 |r."...t.B.I.4...| -00000160 12 1c 09 f0 d4 eb e4 6e d5 9a 31 f9 1a 88 c1 bf |.......n..1.....| -00000170 37 42 90 5f c8 e1 38 2b 8b 4b c1 cd 66 72 e6 49 |7B._..8+.K..fr.I| -00000180 d3 19 0e 01 19 60 f7 7d d3 66 b2 bf bd 94 30 c9 |.....`.}.f....0.| -00000190 3a 01 aa b6 dc 2a d6 1a 68 cf a6 31 5e 9a 1d 5b |:....*..h..1^..[| -000001a0 90 bb 77 33 31 f2 28 5a 70 a5 c5 ef 91 91 27 22 |..w31.(Zp.....'"| -000001b0 59 33 d5 22 78 8e 8f 07 91 3c 69 ec b9 81 be e8 |Y3."x.........G...NWQ.Y| -00000250 05 84 dc 4d fd fb d1 11 33 b6 e5 5e df 65 d4 ed |...M....3..^.e..| -00000260 04 af 89 b3 f0 90 9b 7c 5e 83 b1 66 71 b1 16 03 |.......|^..fq...| -00000270 03 00 35 5b 37 fa e1 97 11 25 7c fd da 7e e8 2a |..5[7....%|..~.*| -00000280 9b 28 fa 20 a6 9b 9b ca 99 ed a2 eb 5b 84 df a0 |.(. ........[...| -00000290 b9 14 c2 fe 38 a8 54 06 e4 54 38 87 2a 24 8b 1e |....8.T..T8.*$..| -000002a0 3e ba 0a bb c2 1d a4 74 16 03 03 00 98 8b 39 c7 |>......t......9.| -000002b0 ac e5 80 bf 49 95 ad f4 c0 cf c9 7c 86 bf 11 65 |....I......|...e| -000002c0 53 40 f3 64 3e 04 ad c9 8d 33 8a 10 4a f5 2e c5 |S@.d>....3..J...| -000002d0 22 18 59 1c 81 65 8a 51 47 b4 8d b1 94 57 3c f8 |".Y..e.QG....W<.| -000002e0 d0 06 60 ce 04 f9 50 8f 6c 43 d8 6e b0 9e d9 da |..`...P.lC.n....| -000002f0 4b e5 b5 05 b7 1b 4b 46 59 d6 ad 20 bb 4c fc aa |K.....KFY.. .L..| -00000300 9a 9b ef fd 59 4b 2a 63 01 b0 1b 2d ed b1 f4 5b |....YK*c...-...[| -00000310 bb ca cd fa 13 06 06 7e 1d a5 cd c3 ca 4e bf 7b |.......~.....N.{| -00000320 7e 92 92 09 61 35 68 a2 38 f9 13 41 f4 a9 a5 0f |~...a5h.8..A....| -00000330 42 63 e4 15 e8 86 00 48 90 2b 43 30 da 05 b6 fc |Bc.....H.+C0....| -00000340 8e 7f 52 d3 b8 14 03 03 00 11 82 73 be a9 08 9d |..R........s....| -00000350 5b 2f fe bd 1f 2c fb e2 f3 94 f3 16 03 03 00 20 |[/...,......... | -00000360 d0 47 0f 2d 7e ce 8f 01 e2 7d 01 3c 32 79 a0 26 |.G.-~....}.<2y.&| -00000370 fe 00 ce d3 37 46 20 b4 f8 af 21 81 0f 1e ba 2d |....7F ...!....-| +00000000 16 03 03 02 69 78 39 27 1a 64 94 f7 9c 64 ac 37 |....ix9'.d...d.7| +00000010 28 be d0 a4 74 98 34 51 8e 3b c7 14 02 83 ee a5 |(...t.4Q.;......| +00000020 05 55 59 f8 04 43 59 cd 33 e6 c4 44 57 30 68 db |.UY..CY.3..DW0h.| +00000030 b2 cb 50 db c1 e6 c4 04 a3 79 22 40 15 f9 89 57 |..P......y"@...W| +00000040 7b 7a 68 d6 d1 15 c9 7f 6b d8 3a 6c dc 32 4f 72 |{zh.....k.:l.2Or| +00000050 97 57 32 25 f9 78 e6 07 91 f4 45 67 b1 41 dd f9 |.W2%.x....Eg.A..| +00000060 5b fd 76 53 d5 7d 0f 74 6e 2e ea d9 9c 2d 00 39 |[.vS.}.tn....-.9| +00000070 2f 7f 49 97 24 bf e4 f2 c7 7e 21 f9 dd 30 3a 78 |/.I.$....~!..0:x| +00000080 3c 53 41 58 5c 97 3a ff 6f 60 f4 b6 d6 23 a9 ad |.c....| +000000e0 6c b5 0d 0c d3 c6 6d 47 2e a8 3a d8 ec fd d3 42 |l.....mG..:....B| +000000f0 ef 8b e8 75 a6 04 06 3b 5e 57 ae a3 10 2e 6d 01 |...u...;^W....m.| +00000100 0e ea 94 e6 78 1b 91 f0 a0 1e 55 d3 25 13 15 ac |....x.....U.%...| +00000110 eb a4 7c c3 66 83 62 00 a3 d2 f5 7c 64 c6 39 fb |..|.f.b....|d.9.| +00000120 6d ee 01 50 d4 13 5d 10 5f 31 30 09 90 91 64 ad |m..P..]._10...d.| +00000130 f1 c5 4b ed 9d ef 2a 71 e5 7a b2 5d a8 57 bc b8 |..K...*q.z.].W..| +00000140 1f 32 c6 f0 61 f0 08 9e 07 9b d3 99 5b 5e a5 32 |.2..a.......[^.2| +00000150 1f 8a f7 30 1f e9 e6 39 b1 5a c6 a5 22 c8 98 49 |...0...9.Z.."..I| +00000160 04 f2 58 4d e2 15 e6 cd d1 2b 05 54 81 cc b8 33 |..XM.....+.T...3| +00000170 5d 26 52 65 95 32 84 01 f6 05 fa 19 58 c6 57 53 |]&Re.2......X.WS| +00000180 c8 d7 3d ef 56 4e 6e e4 17 45 1c bc 40 ee 06 32 |..=.VNn..E..@..2| +00000190 60 74 9e 62 02 47 52 a8 92 57 26 1f ba 2d 93 67 |`t.b.GR..W&..-.g| +000001a0 94 46 0f c4 0d f5 df 75 b0 d9 27 02 56 f3 82 e1 |.F.....u..'.V...| +000001b0 da 1c 3b 3e 97 93 c8 2d 12 bd b4 5b 36 7a 5c bf |..;>...-...[6z\.| +000001c0 27 9a 75 11 f9 4d 9b c0 ae 6b be 71 5f 37 33 b5 |'.u..M...k.q_73.| +000001d0 4e d4 5a fb 98 71 c5 77 8c fc e2 f3 91 73 35 f4 |N.Z..q.w.....s5.| +000001e0 e7 13 b1 ea bd c2 a9 46 b9 ea 88 06 58 a2 49 a2 |.......F....X.I.| +000001f0 97 2b cf 1b 25 a3 71 a0 22 95 4c 54 81 eb 41 7f |.+..%.q.".LT..A.| +00000200 80 15 82 ec 19 0c f8 a3 21 fe 37 2e e1 ca 34 02 |........!.7...4.| +00000210 5d ec 78 d9 eb 99 93 5d 96 26 cd 4c 56 3a 35 6a |].x....].&.LV:5j| +00000220 7f d0 0c c7 b7 c3 80 c6 fe d0 04 54 00 8c 4c 8b |...........T..L.| +00000230 43 f4 36 b8 ea 51 35 3c 5e b5 b9 45 dc d2 19 bd |C.6..Q5<^..E....| +00000240 d2 2a 95 bc 7f 22 43 4f 84 be bd 7b ab a8 5c 4d |.*..."CO...{..\M| +00000250 28 99 e6 e5 05 33 05 38 be 17 55 80 2d e1 de cf |(....3.8..U.-...| +00000260 39 4e 5b 7d 89 b4 a6 d3 0a e3 fd 47 e1 2c 16 03 |9N[}.......G.,..| +00000270 03 00 35 f3 2d 2c 7b fa 06 39 06 40 27 5b ea 9e |..5.-,{..9.@'[..| +00000280 2f 4a 6c 32 df 62 c3 bc af ed 21 6a 1a 34 82 cf |/Jl2.b....!j.4..| +00000290 73 e5 b9 1d 8d 42 3d 8e b7 e6 1c 0a e0 81 3e cd |s....B=.......>.| +000002a0 88 23 f3 56 55 09 e2 6e 16 03 03 00 98 73 0f 59 |.#.VU..n.....s.Y| +000002b0 0e 1b 1b d1 db db 6d 47 d9 05 f8 e1 29 32 e5 16 |......mG....)2..| +000002c0 f9 f6 24 2b 0f 2d bb 74 da fd 75 9b ec 6a 3f 02 |..$+.-.t..u..j?.| +000002d0 c8 3a ab 23 ec cb 77 c6 d7 7b ba ce 29 85 66 54 |.:.#..w..{..).fT| +000002e0 ab df c9 74 2a 50 b5 cd 45 03 e2 ce 6f d3 d0 f1 |...t*P..E...o...| +000002f0 1a 3e e4 c0 34 72 e1 53 1e 69 9a d7 41 4f d4 3a |.>..4r.S.i..AO.:| +00000300 31 af 1a 0b 83 82 e7 6c 4f f6 9a d5 84 bb 9a 14 |1......lO.......| +00000310 e3 ec 04 12 c8 83 9c c8 d9 81 c1 c9 16 db 35 6e |..............5n| +00000320 0f af a0 b3 e6 a9 2b ad a5 1f cd a0 fb 45 01 84 |......+......E..| +00000330 ae b5 42 49 21 d8 29 06 0a 0e e8 26 8e f4 f1 56 |..BI!.)....&...V| +00000340 c2 7c 8f d6 4a 14 03 03 00 11 c6 06 37 68 e0 2b |.|..J.......7h.+| +00000350 dd 32 2a b5 c7 b6 ad e2 c8 7c d8 16 03 03 00 20 |.2*......|..... | +00000360 1d 80 95 df 2d d3 38 50 a2 2d bc 09 50 e4 e8 47 |....-.8P.-..P..G| +00000370 5f 3e 26 03 5e d3 9b 5a ee ed d4 bb cb da 05 5b |_>&.^..Z.......[| >>> Flow 10 (server to client) -00000000 14 03 03 00 11 53 7f 72 ce 10 f4 0e a3 ed ed 8d |.....S.r........| -00000010 3a ad 2d 57 c8 e1 16 03 03 00 20 11 e9 69 5e ff |:.-W...... ..i^.| -00000020 22 0d f6 a3 e1 e5 3f 14 34 a0 33 d7 d5 0a 7f bc |".....?.4.3.....| -00000030 7c 69 9b 9a d2 aa 41 87 fe da de 17 03 03 00 19 ||i....A.........| -00000040 bd a1 83 5d 27 9f cc 0b 40 02 23 6e 6f 18 4c bc |...]'...@.#no.L.| -00000050 48 c4 02 7d 45 08 71 ac d7 |H..}E.q..| +00000000 14 03 03 00 11 13 75 b3 44 ec 4c 6b 24 d3 52 90 |......u.D.Lk$.R.| +00000010 28 c1 47 92 5c 0d 16 03 03 00 20 c2 61 ed 34 d2 |(.G.\..... .a.4.| +00000020 c5 cf fd b3 31 3d 1d 5f 30 87 18 65 01 47 da 0f |....1=._0..e.G..| +00000030 d9 eb 4f 0f 50 2e 02 86 fe eb 13 17 03 03 00 19 |..O.P...........| +00000040 3f ac 26 1c 16 0e 16 bf b1 d3 80 86 dd c6 e2 b6 |?.&.............| +00000050 3f 5a 0b 0c 3e 7d 3b a8 d5 |?Z..>};..| >>> Flow 11 (client to server) -00000000 15 03 03 00 12 80 ae fb 45 cb a1 2d 1f c8 b6 02 |........E..-....| -00000010 3a 28 62 d6 13 48 2c |:(b..H,| +00000000 15 03 03 00 12 ea 81 9d df ec c1 0c 3d 21 8b fe |............=!..| +00000010 b5 24 d8 92 43 1d 3d |.$..C.=| diff --git a/src/crypto/tls/testdata/Client-TLSv12-RenegotiateTwice b/src/crypto/tls/testdata/Client-TLSv12-RenegotiateTwice index 7ff58763c5b54a..e83ad4b285c310 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-RenegotiateTwice +++ b/src/crypto/tls/testdata/Client-TLSv12-RenegotiateTwice @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 25 9f b0 d0 c5 |....]...Y..%....| -00000010 56 c8 79 6a 1a ff 0e e9 8b 41 b6 be c7 36 0b 58 |V.yj.....A...6.X| -00000020 c0 9e 9d 6c 4c b4 39 1f 95 ff 8f 20 32 27 d2 39 |...lL.9.... 2'.9| -00000030 ac f6 33 f9 48 a4 0b 0f e1 f9 5f 0a 83 7b 75 95 |..3.H....._..{u.| -00000040 23 55 14 ba 55 2f 47 42 e8 df 09 39 cc a8 00 00 |#U..U/GB...9....| +00000000 16 03 03 00 5d 02 00 00 59 03 03 36 29 60 e0 b5 |....]...Y..6)`..| +00000010 ee 59 a6 3b 1a de 96 8a 51 0c c1 15 30 ce 6a 70 |.Y.;....Q...0.jp| +00000020 ca 5b fa a5 86 f4 59 79 44 cd 73 20 ad b7 a9 d5 |.[....YyD.s ....| +00000030 c1 aa a6 49 6d a8 36 a9 bf 40 1e 3a d4 2c 21 4a |...Im.6..@.:.,!J| +00000040 5d aa fa 75 80 d4 cb fc be 2b 41 36 cc a8 00 00 |]..u.....+A6....| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -61,286 +62,290 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 2b 66 1d |............ +f.| -000002d0 80 f2 60 32 0f af 6f d3 03 db 28 c1 33 18 1c 5d |..`2..o...(.3..]| -000002e0 71 fd 2a 24 f9 3c 4d 76 ca e3 93 8d 25 08 04 00 |q.*$.{W...*| +00000350 85 0d b0 5f bf a5 8e 74 55 72 dc 2c 5c 60 52 93 |..._...tUr.,\`R.| +00000360 1e 28 9d 36 9f fd da a5 3b 51 96 4e 12 d6 d4 ca |.(.6....;Q.N....| +00000370 d3 16 03 03 00 04 0e 00 00 00 |..........| >>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| -00000030 16 03 03 00 20 7b de 6f c8 b7 f1 69 89 0b 29 af |.... {.o...i..).| -00000040 62 1b 92 47 e9 44 f7 26 91 93 9c 3b e2 77 0e d0 |b..G.D.&...;.w..| -00000050 c5 af fd e5 59 |....Y| +00000030 16 03 03 00 20 e4 80 79 12 42 80 88 c3 66 6b 9b |.... ..y.B...fk.| +00000040 a2 03 59 f9 a3 54 2f 7f 03 de 90 97 1d cb 48 08 |..Y..T/.......H.| +00000050 19 a4 bb 13 64 |....d| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 20 94 da e2 2e f3 |.......... .....| -00000010 65 59 31 4f 58 42 57 b5 1f e6 c1 5a c6 07 97 12 |eY1OXBW....Z....| -00000020 f6 6c 20 cc 91 50 76 42 70 dd 2c |.l ..PvBp.,| +00000000 14 03 03 00 01 01 16 03 03 00 20 9e 59 22 68 f2 |.......... .Y"h.| +00000010 3c 38 7d cf 43 eb 77 73 94 6c 35 e5 3f 3e 7e 03 |<8}.C.ws.l5.?>~.| +00000020 29 96 e8 c6 f7 5c b4 39 c3 77 61 |)....\.9.wa| >>> Flow 5 (client to server) -00000000 17 03 03 00 16 fe 0d 78 61 f9 fe 65 3c 42 c0 98 |.......xa..e>> Flow 6 (server to client) -00000000 16 03 03 00 14 e0 bb 6e cc 69 fc a7 02 46 c4 5e |.......n.i...F.^| -00000010 a5 b1 1e ce e9 c7 7a d9 76 |......z.v| +00000000 16 03 03 00 14 a8 ab 9e 3c 00 e4 6f b3 26 dd 3a |........<..o.&.:| +00000010 67 ab ee 28 a0 a1 65 83 91 |g..(..e..| >>> Flow 7 (client to server) -00000000 16 03 03 01 1a a7 03 1c ef 27 0a 04 9b 31 b0 c6 |.........'...1..| -00000010 17 00 1f 6c d5 91 ae 7c 77 87 98 05 67 9a ea ea |...l...|w...g...| -00000020 6c 79 28 da 2c 91 6d a0 c6 9a e9 f3 85 a0 7f 7d |ly(.,.m........}| -00000030 ae 89 ac 0d 01 7f d8 34 dd fb ba c3 d8 ba ff 9b |.......4........| -00000040 75 dc d6 5e 8a fa ad 9c ea fa 15 63 55 85 98 26 |u..^.......cU..&| -00000050 65 7a 19 17 c4 99 98 85 83 c4 06 39 30 d5 63 51 |ez.........90.cQ| -00000060 ff 22 e8 cd 55 6e fc 88 5e d5 54 24 79 52 7e b0 |."..Un..^.T$yR~.| -00000070 71 e4 e6 cb a6 06 24 14 1a c2 a8 52 85 e9 fc e4 |q.....$....R....| -00000080 66 47 9c 9c d1 f3 56 11 47 16 4b 7c a5 f1 81 1e |fG....V.G.K|....| -00000090 00 78 98 8f 70 ae 47 6d 45 3b c4 af fc 88 bd 88 |.x..p.GmE;......| -000000a0 7b cc 8a 78 86 ff 86 25 2d 21 28 f2 c5 d7 4b 61 |{..x...%-!(...Ka| -000000b0 c0 ea 67 38 4c d0 d6 6f fc a0 69 62 98 6f ac 29 |..g8L..o..ib.o.)| -000000c0 29 13 34 7a f0 2d 1b 7e 52 42 6f f1 64 ed 7f 47 |).4z.-.~RBo.d..G| -000000d0 69 82 9b 9d 19 7b 3a c5 54 51 c8 8f 4c 9e 42 a7 |i....{:.TQ..L.B.| -000000e0 0f 9e 05 1d 4f 85 11 05 97 42 fa 24 37 1c 4e ec |....O....B.$7.N.| -000000f0 55 dc e1 4e f6 e2 dd c6 63 b0 dd 60 11 93 23 8a |U..N....c..`..#.| -00000100 eb 68 76 21 ec 19 cf 59 c8 78 80 f0 fa 39 6a 09 |.hv!...Y.x...9j.| -00000110 f8 7f 9d c0 e7 51 3d 75 17 e3 cf a5 bf 1c 10 |.....Q=u.......| +00000000 16 03 03 01 34 d8 7e 6c f6 7b d5 eb 3f 49 63 78 |....4.~l.{..?Icx| +00000010 28 92 1c 8b 67 f2 70 a7 ff e2 91 42 43 d8 90 4e |(...g.p....BC..N| +00000020 01 47 0a 12 bb 9d a2 9a e1 a2 d6 ee 5e 91 8c 4d |.G..........^..M| +00000030 86 8e 29 d4 e0 b3 67 0f c4 5e a9 02 c9 e3 8f f0 |..)...g..^......| +00000040 e6 ea 24 50 da 57 70 35 58 98 81 3e ff b8 8d ea |..$P.Wp5X..>....| +00000050 b3 e0 f0 a1 91 39 69 33 40 bc 0f 67 21 10 67 f4 |.....9i3@..g!.g.| +00000060 93 66 0c f2 66 aa fd a5 2c 8d 2b 38 31 30 b3 50 |.f..f...,.+810.P| +00000070 71 09 de ce 22 59 c3 96 5a 07 84 c0 1f 96 5c e3 |q..."Y..Z.....\.| +00000080 7e 6b c8 7e 3f a5 32 77 20 3c 7b 92 df bc 11 57 |~k.~?.2w <{....W| +00000090 86 e5 a1 03 78 9c 08 ea 49 bc a7 1b 6b fb 47 6d |....x...I...k.Gm| +000000a0 9a 86 14 fa cc 98 e6 9a f8 39 18 57 36 fb 1b ff |.........9.W6...| +000000b0 ae ce 1e 96 20 9d 16 be c6 00 45 51 2a 5a 7b 46 |.... .....EQ*Z{F| +000000c0 9c c9 c1 eb fb fc 37 52 8d b2 22 e3 58 6a d1 fe |......7R..".Xj..| +000000d0 71 55 18 4b ff 92 e1 16 62 29 cb fb d0 0d 94 22 |qU.K....b)....."| +000000e0 13 ee 8e bb 75 2e 1c ae 11 61 3d 37 69 0d 51 b0 |....u....a=7i.Q.| +000000f0 61 e3 7f 94 08 54 3a 49 68 94 b5 64 da ac a9 a4 |a....T:Ih..d....| +00000100 e2 3f 65 ad d7 42 8e 0b 70 5f 4f 6d 6e 0c 27 51 |.?e..B..p_Omn.'Q| +00000110 5a 48 57 71 6e 9b 58 e0 6e 05 40 48 a8 20 36 98 |ZHWqn.X.n.@H. 6.| +00000120 f8 71 a3 ef 2d db 8c ae fa a5 14 d8 1f ff ad 7b |.q..-..........{| +00000130 2a 15 0d fb 3d e8 1b fe 48 |*...=...H| >>> Flow 8 (server to client) -00000000 16 03 03 00 85 14 4e 8e de d0 5b 59 2d f6 a3 b0 |......N...[Y-...| -00000010 c0 ff 4a ff 8b cf af 2f 09 f1 ee 03 0f b9 70 5e |..J..../......p^| -00000020 1b e4 ef b4 3e e7 c1 43 fb ea a8 68 4c 66 0a 55 |....>..C...hLf.U| -00000030 e8 51 a6 30 bb e3 f5 0e df 11 f2 e0 51 94 14 f6 |.Q.0........Q...| -00000040 1e c6 e8 9d 10 a2 20 9f f2 f6 1e f0 3c f3 ba ff |...... .....<...| -00000050 65 d1 f4 df 9e e6 18 9f a1 a5 9c 5f 4b 4f 20 98 |e.........._KO .| -00000060 4a f6 0b 95 da 01 1f ea c0 c4 39 39 8b fd 19 40 |J.........99...@| -00000070 8d 14 da 45 fd 51 3d a7 36 23 a0 b2 bb cb 81 6b |...E.Q=.6#.....k| -00000080 79 0a 02 71 50 84 ac d6 7b ad 16 03 03 02 69 b8 |y..qP...{.....i.| -00000090 a9 6c 87 d7 23 6b c6 1a 4c 08 dd fa 58 e8 e8 d5 |.l..#k..L...X...| -000000a0 44 16 3b de 77 60 33 2a 47 a5 3c e0 cb c3 1a 38 |D.;.w`3*G.<....8| -000000b0 1d 1f 69 8e 33 a3 ba ac 55 b5 77 9b e1 fc a4 21 |..i.3...U.w....!| -000000c0 99 90 01 5e 99 36 a3 cf 74 b4 1f 96 b7 96 1e 4b |...^.6..t......K| -000000d0 cf 0b 92 5f 6b 1f 1c b0 70 80 2e 87 57 ef 52 b0 |..._k...p...W.R.| -000000e0 2b 7d 66 06 cb a1 78 a7 5a e2 af 4c fc a4 8b 99 |+}f...x.Z..L....| -000000f0 af 6c 6d 0e 07 7e da 11 5a b7 89 af 3c ce 1e 22 |.lm..~..Z...<.."| -00000100 1f 7d 97 d5 d4 78 09 c4 e7 fb 33 bf 36 ee 6e 5a |.}...x....3.6.nZ| -00000110 af f0 5b 0f 00 eb cc 3d 76 35 41 98 4c 65 81 34 |..[....=v5A.Le.4| -00000120 74 d9 d7 ba 04 fc c8 c6 c9 ca 4b b0 8e b1 7e 87 |t.........K...~.| -00000130 9d 69 2e d5 68 54 96 48 37 42 ae ff 18 d0 08 b2 |.i..hT.H7B......| -00000140 dc 26 1c c1 99 f1 c3 02 3f 48 51 a8 92 10 18 01 |.&......?HQ.....| -00000150 f3 3b 96 4b de 80 29 27 87 da f4 89 23 1b 49 b8 |.;.K..)'....#.I.| -00000160 5b f5 50 ec 99 b0 c3 48 e2 d3 fe fb a0 95 c3 c6 |[.P....H........| -00000170 83 fe d9 87 ef 0c cf 67 8f e7 52 3d 17 7d c0 ab |.......g..R=.}..| -00000180 02 1a 33 78 c8 d6 13 8b 8e 82 be 0d 5e d4 22 e8 |..3x........^.".| -00000190 63 c3 84 a9 05 12 d3 b7 e2 36 96 a2 22 0a 77 df |c........6..".w.| -000001a0 20 06 5c 0d 8b c1 d6 ac 73 b0 b2 a0 be db 59 17 | .\.....s.....Y.| -000001b0 05 60 68 b9 18 85 d5 73 9b c9 41 b9 c1 f2 02 47 |.`h....s..A....G| -000001c0 ca 95 c8 47 e2 de 04 75 08 d4 b7 a2 41 d7 e2 6b |...G...u....A..k| -000001d0 ad 8c ba c5 fc f8 b1 fe d3 23 c0 22 ec d0 2f a7 |.........#."../.| -000001e0 d9 ed 0e da 71 18 c0 d0 1f 40 8d eb 79 ea 5f aa |....q....@..y._.| -000001f0 07 4f 65 81 c1 9e ea 36 7f 0d 8b 07 8b 71 81 5a |.Oe....6.....q.Z| -00000200 0e cb d4 0f 98 c8 5c 69 d0 61 6e b5 ef 2a dd 5f |......\i.an..*._| -00000210 63 92 b4 3e 6e f4 b8 fc 45 bf 58 79 08 a4 95 6f |c..>n...E.Xy...o| -00000220 34 01 9b 8a 19 a5 e4 11 3f d4 97 47 66 e8 42 ef |4.......?..Gf.B.| -00000230 92 72 01 f4 6f 07 a4 35 81 83 ee 26 d0 96 cc de |.r..o..5...&....| -00000240 42 2a 65 64 cc a9 0c 39 31 d5 7c 2f 42 e5 ce 36 |B*ed...91.|/B..6| -00000250 0b bf b5 22 2a 4f 3c 56 27 29 cd fb 8e 9f 8f ab |..."*O...d...V*.h| +000001e0 45 68 13 da c6 eb 5d f2 7b 51 bd b4 8e 25 0d b7 |Eh....].{Q...%..| +000001f0 f0 03 dd 2b cd 93 7f 00 96 ca 75 c1 1b c7 d0 95 |...+......u.....| +00000200 0f 08 52 e0 d6 c9 2c b2 42 b3 0b 0c ed e1 01 e3 |..R...,.B.......| +00000210 7d ef b1 fb 80 76 c8 08 ec 9f c2 13 a5 04 05 fe |}....v..........| +00000220 33 c2 b7 2e 1a 2b 4a 74 f5 f5 92 ad e8 cf f7 76 |3....+Jt.......v| +00000230 3e 7c 69 4b 2d be 9b 08 3b 80 97 66 63 e5 df db |>|iK-...;..fc...| +00000240 1a a4 aa 8a bd 43 2a 65 44 6d f1 e6 60 92 cd 98 |.....C*eDm..`...| +00000250 a9 ac 81 de fb 22 06 f4 ca 90 4a 67 64 3f 25 ab |....."....Jgd?%.| +00000260 3c db 77 cb bc 32 55 bc 2d 48 4e 32 c7 48 d1 2f |<.w..2U.-HN2.H./| +00000270 bf 80 03 7e 27 92 8b 6a 90 35 ab 8e 93 dd 27 c7 |...~'..j.5....'.| +00000280 7d d6 a7 ea d8 1f 6a 43 57 34 d2 a9 7c c3 23 55 |}.....jCW4..|.#U| +00000290 62 5b 98 80 a1 ec 61 53 63 b9 a7 89 85 7e a8 0e |b[....aSc....~..| +000002a0 31 e9 9f 82 e9 1a b3 25 b8 d3 bf 4b f7 da 40 ce |1......%...K..@.| +000002b0 30 95 8f 41 75 9e 99 f5 3f fd 90 57 77 ee ad b7 |0..Au...?..Ww...| +000002c0 56 1e fd a6 d6 98 ae d3 a0 36 8a e0 19 a5 64 63 |V........6....dc| +000002d0 8f 08 d3 e6 2b c3 8a 29 90 8d e0 d7 1d 8b 84 6d |....+..).......m| +000002e0 a8 7a 0d 8b 6d 0e d5 fa bd c1 31 13 c7 39 61 f6 |.z..m.....1..9a.| +000002f0 e8 b3 fd 4d fe 49 a0 2c 16 03 03 00 bc 6d 99 1c |...M.I.,.....m..| +00000300 76 24 e6 f4 79 ac da e5 71 78 32 8d df a7 74 a4 |v$..y...qx2...t.| +00000310 e1 22 43 86 62 7a 99 34 9e c1 c7 82 82 f2 ad c0 |."C.bz.4........| +00000320 cb e5 54 c7 2d 3f bd 00 ba 4f 9f 6b 90 c2 2c bf |..T.-?...O.k..,.| +00000330 d5 d1 20 48 28 f0 bc 21 d4 1f 8d a9 89 82 ce 3d |.. H(..!.......=| +00000340 b2 52 bd 91 57 65 da 07 74 d9 56 b3 54 a7 2c 20 |.R..We..t.V.T., | +00000350 93 5e 79 59 d0 d3 33 95 a9 de f6 a4 48 37 11 10 |.^yY..3.....H7..| +00000360 a7 34 2a 2b 30 a5 24 03 5b d5 63 78 1f ad ca 70 |.4*+0.$.[.cx...p| +00000370 ff f9 22 ca bc 23 ac 4d eb 5a d7 de af 31 72 25 |.."..#.M.Z...1r%| +00000380 84 24 72 b7 b8 c0 a5 7a b6 7e a9 95 42 bb 5e 52 |.$r....z.~..B.^R| +00000390 6c 13 87 c2 b5 17 04 a8 0a 43 d0 b4 96 12 12 4b |l........C.....K| +000003a0 1e d1 0b ab a1 5d 1a bc 48 15 ec 88 e6 7b 57 aa |.....]..H....{W.| +000003b0 da 3b 7f 0b cc 11 93 8f ee 16 03 03 00 4a 35 4e |.;...........J5N| +000003c0 5f 82 88 e9 d5 24 cc 81 3c 0a cc 49 63 13 ea 0c |_....$..<..Ic...| +000003d0 e6 d7 a9 f2 7b 4f 7f d4 c1 d8 71 4e 1d e6 68 e6 |....{O....qN..h.| +000003e0 7e ba 73 f8 23 a5 af 1a 05 35 4c bf 88 8d 8a 90 |~.s.#....5L.....| +000003f0 09 26 34 1d b6 88 92 2d 60 92 52 2c cb 45 02 a4 |.&4....-`.R,.E..| +00000400 78 54 3d a3 44 d3 f9 46 16 03 03 00 14 58 29 f7 |xT=.D..F.....X).| +00000410 44 f4 a0 9f 65 d8 47 1a 01 7e 1b 95 3b 5f 1f 4a |D...e.G..~..;_.J| +00000420 12 |.| >>> Flow 9 (client to server) -00000000 16 03 03 02 69 8a dc ce d1 c0 85 30 95 6c da 57 |....i......0.l.W| -00000010 23 05 39 c1 4c 20 08 25 a2 96 f1 55 a1 77 3b 42 |#.9.L .%...U.w;B| -00000020 9f 3d 69 9b 56 49 b4 94 8a 3c d5 85 43 f1 b4 4b |.=i.VI...<..C..K| -00000030 cd e0 67 95 83 34 90 89 81 f0 18 87 28 c0 42 88 |..g..4......(.B.| -00000040 b0 1c d1 cc 0a f0 bb 67 91 55 0b e4 ed 22 52 09 |.......g.U..."R.| -00000050 92 0b 56 ac d0 3f c9 41 05 8a 19 7d e7 9a d7 ff |..V..?.A...}....| -00000060 fa 85 f5 23 b0 98 a0 79 b1 8e 0f 90 0e c8 32 92 |...#...y......2.| -00000070 4b 6a 37 47 45 31 70 be 8b cd 7a 26 3a 3c 0b 63 |Kj7GE1p...z&:<.c| -00000080 a7 8c 00 2c 1d be eb 5c 1f 16 90 76 2b d7 82 5e |...,...\...v+..^| -00000090 31 93 40 7a e0 9a e0 4c 7d 2e e3 84 0f cb 06 7d |1.@z...L}......}| -000000a0 8d 76 41 ea 27 52 4a 77 e5 91 02 ab 11 cc 8c f0 |.vA.'RJw........| -000000b0 fe 8e ee b2 f1 ba 1e ab 0d 5c e2 2d da d5 8d f9 |.........\.-....| -000000c0 7c 22 32 e9 50 1d 88 db af c0 82 51 e0 31 14 71 ||"2.P......Q.1.q| -000000d0 ad c9 05 5d 8e 2b 75 4b 9e 74 b0 3a a9 e4 7c 82 |...].+uK.t.:..|.| -000000e0 ac 75 39 31 0a dc dc 0a 5c f4 ee 55 f2 09 c2 d2 |.u91....\..U....| -000000f0 81 c8 70 2e 8f 37 6b 90 79 5c 18 6e 38 21 c6 00 |..p..7k.y\.n8!..| -00000100 08 be ad 21 e9 40 b6 54 06 f0 61 f9 2c 71 25 0e |...!.@.T..a.,q%.| -00000110 13 6c 66 ef b6 a9 61 d0 d0 42 56 58 32 3c 9c 5a |.lf...a..BVX2<.Z| -00000120 ee 13 1d 42 8b 94 41 9c d1 2e c9 ac d4 ed 0f 04 |...B..A.........| -00000130 1c 6f 5c 30 80 12 37 c6 43 07 7e 03 50 86 24 84 |.o\0..7.C.~.P.$.| -00000140 34 2a 70 82 65 67 35 cd 5e 7c 0e 49 30 35 d4 ee |4*p.eg5.^|.I05..| -00000150 43 b7 7a c3 88 30 e5 ba 98 7f 8d 9e 08 a2 8c bb |C.z..0..........| -00000160 48 44 c2 1a 8d 7b 41 de b6 68 ae d2 8a 64 9b 6b |HD...{A..h...d.k| -00000170 8a 47 88 04 49 05 8a 94 66 da d3 90 78 6d 81 64 |.G..I...f...xm.d| -00000180 4e 6b a5 6f d2 9d 1d 36 6f a2 a3 83 0c f5 69 10 |Nk.o...6o.....i.| -00000190 37 c8 b8 59 c7 b1 5b 91 fc e3 26 37 da 14 80 8f |7..Y..[...&7....| -000001a0 d4 1e 63 51 1d 77 85 67 af 7a 7d e4 5a 27 f9 3f |..cQ.w.g.z}.Z'.?| -000001b0 75 b0 9e 2a 1d 42 29 ac d4 29 c6 0d 2f ee 35 38 |u..*.B)..)../.58| -000001c0 38 91 a4 f1 60 1b d6 49 b2 eb c4 df b6 01 99 69 |8...`..I.......i| -000001d0 06 7c ca 2d aa 15 e8 c5 4f 48 5f 77 87 4b fd 41 |.|.-....OH_w.K.A| -000001e0 cf aa 7d 96 b9 12 ee b7 b3 a2 86 5e 4f 79 08 a1 |..}........^Oy..| -000001f0 00 43 12 93 92 99 07 44 23 48 78 46 c8 fc 4a 96 |.C.....D#HxF..J.| -00000200 88 72 45 2a f4 ff 92 41 7b e5 a0 74 93 ff b8 f6 |.rE*...A{..t....| -00000210 3e e0 6a 3b 3b 12 68 50 89 d1 d3 22 e0 a1 3f ef |>.j;;.hP..."..?.| -00000220 da 18 15 5b c2 48 0e 78 d0 af ae d7 81 af 23 16 |...[.H.x......#.| -00000230 ab 71 07 9d 26 b2 8b 34 7e b2 1a 1e f4 fb 02 9a |.q..&..4~.......| -00000240 f4 15 78 a9 d2 19 94 bb 9f b0 ba 2a 21 20 dd 87 |..x........*! ..| -00000250 57 73 e7 01 08 bc 38 ce 2d 9e dc 3f f7 c4 75 aa |Ws....8.-..?..u.| -00000260 00 9d 28 43 e5 de aa 82 fb e5 ca 87 4e 6c 16 03 |..(C........Nl..| -00000270 03 00 35 e0 37 f9 95 cd 96 f9 ba 36 12 6b 35 c5 |..5.7......6.k5.| -00000280 02 9d 71 8a e5 07 d6 1c 77 6a 86 17 d0 01 9b 79 |..q.....wj.....y| -00000290 74 6b 14 27 33 2f 6d c2 af cb dd 47 71 b3 73 4d |tk.'3/m....Gq.sM| -000002a0 7f 42 3e 7f 86 6e 63 ed 16 03 03 00 98 7f ba 78 |.B>..nc........x| -000002b0 bf a3 d4 02 cb 19 0d 1f 48 bf 4d 9f d0 37 3a c9 |........H.M..7:.| -000002c0 c1 76 6e fb 9a 69 63 0a a1 76 13 2d ea 52 38 18 |.vn..ic..v.-.R8.| -000002d0 71 89 9a f5 02 2e 44 94 72 6d 08 e1 89 9a e1 4c |q.....D.rm.....L| -000002e0 ee 2f 15 9c 0d b6 67 1e 0c 17 b0 e9 03 7d 66 1c |./....g......}f.| -000002f0 28 6b 57 6d c8 49 5d ef ca 4c 51 59 ae 17 55 97 |(kWm.I]..LQY..U.| -00000300 54 f1 f5 64 19 ec 36 27 96 33 0e fd 65 42 ac 50 |T..d..6'.3..eB.P| -00000310 bb cf f9 da fe 98 3d 83 79 a0 24 0d 9f 91 5c b0 |......=.y.$...\.| -00000320 5c c5 04 b4 90 53 c3 8b b5 29 c5 73 7f 04 c6 55 |\....S...).s...U| -00000330 93 4a 08 67 8f f7 a8 ba a8 e2 62 09 85 d3 47 b9 |.J.g......b...G.| -00000340 27 bd 6c 81 95 14 03 03 00 11 88 9b 21 1e 06 0a |'.l.........!...| -00000350 fa 8a 9e 4e d9 7a b3 a5 45 7a 96 16 03 03 00 20 |...N.z..Ez..... | -00000360 6f a0 f0 cf 0b 2b c2 d1 36 d8 64 2b 72 86 7b ab |o....+..6.d+r.{.| -00000370 c7 86 5d c3 67 ec d6 c0 32 4d b8 1f fa d9 3a af |..].g...2M....:.| +00000000 16 03 03 02 69 ef e7 da 35 77 b6 d6 09 a1 71 43 |....i...5w....qC| +00000010 1c 2b 72 d6 65 ea b4 38 e4 13 c1 85 c3 36 bb f4 |.+r.e..8.....6..| +00000020 ef 1a b8 94 de 11 22 6e b8 28 14 05 88 2a 5d 7a |......"n.(...*]z| +00000030 7c 0a 00 ac 74 ce 4f f8 b3 94 5d 5c a1 aa 3e 20 ||...t.O...]\..> | +00000040 ee 0b e3 50 3d e7 4f 4d ad 5c 9e 6e 8e 75 26 b4 |...P=.OM.\.n.u&.| +00000050 80 e1 85 3f ae 76 6f 95 6e f0 79 98 3d 86 17 9c |...?.vo.n.y.=...| +00000060 e9 d6 35 1d f5 15 ea c5 29 67 15 9d 03 4b 41 9d |..5.....)g...KA.| +00000070 25 b3 ab 3a b0 12 d1 ff fc 9e 25 af ff 8a c8 1c |%..:......%.....| +00000080 4b 4e f0 10 9d d3 98 6e 66 0a 13 82 b9 04 d7 39 |KN.....nf......9| +00000090 b2 2c c5 f9 cf be 4b 8b 8a 28 e9 24 db c8 fe d6 |.,....K..(.$....| +000000a0 a1 b1 3c 9e 19 92 0b 5d 2e c8 5f 56 35 dc 16 46 |..<....].._V5..F| +000000b0 7b a1 d2 43 d9 ee 3f a7 74 c1 4c ed 2a 84 4a ad |{..C..?.t.L.*.J.| +000000c0 76 a2 bc 90 8d 19 06 11 1a 6d b8 e0 3a 8e 0d 33 |v........m..:..3| +000000d0 f2 ad 06 d7 e5 a3 16 8c 14 07 4c 84 8a 47 13 c1 |..........L..G..| +000000e0 b1 cb 81 8b d7 5b fb 04 2b da 3c 7f d1 0f 2b 8b |.....[..+.<...+.| +000000f0 23 ad f5 f1 09 82 24 80 5c bc f7 68 69 c0 90 5f |#.....$.\..hi.._| +00000100 fa 96 b4 9d 51 0e 96 1e 06 2a d1 98 5d 96 95 68 |....Q....*..]..h| +00000110 de df 40 f9 f6 b1 f2 ef a7 c4 0c 05 ee b3 63 8d |..@...........c.| +00000120 3a 1e a2 d8 34 2a b2 99 c3 17 03 60 18 4f 43 21 |:...4*.....`.OC!| +00000130 99 4f 81 c1 11 8e a4 45 79 d8 fa fc b2 9a f8 d0 |.O.....Ey.......| +00000140 95 10 79 38 45 1b 82 42 f0 bb 75 27 6b a2 53 d3 |..y8E..B..u'k.S.| +00000150 e6 dd 2d 43 f5 80 fd 9a 59 ec 07 42 ee b0 9d bd |..-C....Y..B....| +00000160 33 dd 58 c8 57 e8 16 de a4 21 c9 92 51 d2 e5 8b |3.X.W....!..Q...| +00000170 48 f5 cb a8 3a d8 f6 a3 b2 00 90 9e f0 e0 ca c7 |H...:...........| +00000180 fe 79 70 f7 8e 5c 4b 3c 86 c8 cc ca b1 b6 05 e2 |.yp..\K<........| +00000190 90 66 db 85 fc 7c 87 6f e0 44 ed 2a c9 66 b7 de |.f...|.o.D.*.f..| +000001a0 c5 f0 d2 80 dd f2 c2 26 5b 84 cf b0 c4 78 7e 65 |.......&[....x~e| +000001b0 66 d9 d8 c5 fe 97 b9 27 6e 55 11 c0 0b 3e e2 c9 |f......'nU...>..| +000001c0 ce 4d dd 27 28 6f 62 45 70 b3 e4 0c 18 31 f1 b2 |.M.'(obEp....1..| +000001d0 33 8f 7d 34 7c f6 f3 50 d9 a9 6b ec a7 cf c2 7b |3.}4|..P..k....{| +000001e0 36 21 d7 76 68 c1 0e 90 8d af 2e c5 d5 26 c7 c1 |6!.vh........&..| +000001f0 0b 1c 43 85 a6 43 3e 96 67 46 2d 1e 0a c8 90 99 |..C..C>.gF-.....| +00000200 0f 71 cc 60 49 68 2c df 17 68 e4 fa 79 05 50 ac |.q.`Ih,..h..y.P.| +00000210 16 d0 26 1f 7e 58 a2 dc 73 e9 05 8d 92 f6 8c 56 |..&.~X..s......V| +00000220 db 08 0a b7 99 64 f7 5b d0 5f f3 88 f5 02 f5 d0 |.....d.[._......| +00000230 66 55 e3 db f4 d1 c9 d3 c1 f4 ab 8c 5c 39 fd 64 |fU..........\9.d| +00000240 ec cd ab 63 da 7b 3e 64 1a b4 c3 d9 79 bd 92 e3 |...c.{>d....y...| +00000250 97 68 a4 83 b5 f1 fa f5 05 04 84 39 e9 82 a6 84 |.h.........9....| +00000260 e5 a0 6f a2 6d ea 6b ab 4c f8 a5 87 9c 9c 16 03 |..o.m.k.L.......| +00000270 03 00 35 08 0b 50 f9 90 d3 e9 58 e5 5b df a1 99 |..5..P....X.[...| +00000280 89 ea 0b 1a f1 4d e3 fd a6 79 2c 61 ac 6f da 7c |.....M...y,a.o.|| +00000290 2a 78 e1 38 05 35 2e 8b db eb 25 fd eb 2f d7 a3 |*x.8.5....%../..| +000002a0 27 9d 27 f2 b3 ff 20 8a 16 03 03 00 98 95 42 7e |'.'... .......B~| +000002b0 d5 86 40 42 dc 93 fe 80 ef 70 cc fb 2f b1 51 bd |..@B.....p../.Q.| +000002c0 ca 5f 1e 0d a0 3f c6 e0 c9 69 0e d4 dc 18 25 52 |._...?...i....%R| +000002d0 5c c2 8c 37 f6 a8 e6 4d 48 27 81 4d 7a f3 49 5c |\..7...MH'.Mz.I\| +000002e0 92 92 68 a9 e0 dc d6 c5 b5 ea 99 5c 51 df 93 a1 |..h........\Q...| +000002f0 36 2d 65 a8 cb ca 72 76 53 92 7d c7 81 5f 19 2c |6-e...rvS.}.._.,| +00000300 3e 83 29 16 41 65 4f 81 d7 3f 5c 15 f0 36 d9 75 |>.).AeO..?\..6.u| +00000310 a7 c8 52 6f 5b b6 98 46 5e 39 31 83 1c 99 3f f9 |..Ro[..F^91...?.| +00000320 9e 28 d7 94 ae a7 b4 f7 c9 ce 5b 74 2f 88 9e fd |.(........[t/...| +00000330 3c 1c 1e eb f4 b3 39 65 39 07 14 8e 3e 95 87 b2 |<.....9e9...>...| +00000340 b8 4c e8 7d 3b 14 03 03 00 11 cc eb 21 2a 7c 9a |.L.};.......!*|.| +00000350 5c 35 9f 35 e3 09 dd f8 7a 4f 43 16 03 03 00 20 |\5.5....zOC.... | +00000360 0b ff 00 c0 96 43 18 d9 9d ad 51 55 44 40 11 2f |.....C....QUD@./| +00000370 cd 1c bd ae ea 0a c9 eb de 57 95 c2 81 63 79 4c |.........W...cyL| >>> Flow 10 (server to client) -00000000 14 03 03 00 11 b4 79 7e 62 ea ea f7 37 1d a4 65 |......y~b...7..e| -00000010 9d 02 95 b8 4a 6c 16 03 03 00 20 6a 45 d1 d0 ff |....Jl.... jE...| -00000020 fa 9f 6a ea d5 0d 27 b6 e7 15 8b e8 5b f9 3c d0 |..j...'.....[.<.| -00000030 32 ff 81 24 9a de 1c 31 6a 48 f8 17 03 03 00 19 |2..$...1jH......| -00000040 a9 07 ab fb 37 2f 88 0c 93 4a 0c 34 08 26 39 8c |....7/...J.4.&9.| -00000050 22 00 1b 36 3c 6d 0c 49 61 16 03 03 00 14 bb 63 |"..6>> Flow 11 (client to server) -00000000 16 03 03 01 1a a4 4c dc 47 09 04 ab bb ca 49 f2 |......L.G.....I.| -00000010 63 36 35 fb e3 3f ad ae 82 2a 9b 87 cc 0c 2e ea |c65..?...*......| -00000020 18 af ad e9 ea 7a c8 45 de c6 9b 0e e0 32 26 95 |.....z.E.....2&.| -00000030 ba e3 57 9e 27 ce ff 36 43 a3 bd 79 47 6c 02 86 |..W.'..6C..yGl..| -00000040 81 44 ac b3 c6 21 d8 fa 8c 69 43 b7 66 70 ea 24 |.D...!...iC.fp.$| -00000050 12 1b ac 44 d2 49 e3 5a d1 0e 50 76 46 4f 58 51 |...D.I.Z..PvFOXQ| -00000060 21 4c f8 20 63 60 c0 20 3a 12 e5 0b 86 e3 03 2c |!L. c`. :......,| -00000070 e1 fc 63 e4 44 d4 67 95 aa af 22 3d 40 7b 4e 12 |..c.D.g..."=@{N.| -00000080 98 7d 05 00 57 50 d5 3a d7 af dd 96 0f ac 77 63 |.}..WP.:......wc| -00000090 f8 1c b0 9c d0 96 9f 20 a1 48 55 aa dc 69 fb b7 |....... .HU..i..| -000000a0 67 58 ff 27 de 22 ea 66 48 32 56 1f ef 12 3f 44 |gX.'.".fH2V...?D| -000000b0 38 af ae 4b c2 16 bb c8 f1 fe 13 d7 05 66 de 08 |8..K.........f..| -000000c0 f3 4d 5c a6 e1 2d 0d c3 ce 50 7b 01 40 49 0f 83 |.M\..-...P{.@I..| -000000d0 97 c0 37 a2 6c 2b 64 a9 40 0e e1 7b f4 6e e1 cc |..7.l+d.@..{.n..| -000000e0 1a 5f 0b c5 69 dc dc db 0e a2 22 7b cf ca 73 81 |._..i....."{..s.| -000000f0 fc 47 47 c7 76 18 8e ae 55 6c fe 5e 5e 33 84 9c |.GG.v...Ul.^^3..| -00000100 4c 1b f8 21 29 3c 47 1b c8 01 38 7d c2 50 45 1c |L..!).1....#..tE{..| +00000120 d8 c8 c0 8b 00 72 ee d8 a9 64 b8 1b 48 77 98 b9 |.....r...d..Hw..| +00000130 20 61 b8 23 17 b5 65 7c fb | a.#..e|.| >>> Flow 12 (server to client) -00000000 16 03 03 00 85 bb 34 66 c0 a3 8c 2b 2c fe fe 82 |......4f...+,...| -00000010 53 a1 07 1f 06 db 41 94 ac 21 f2 c4 23 0e b6 48 |S.....A..!..#..H| -00000020 4d 16 97 2a 79 dd 95 4c 93 7f e4 8c 94 90 d4 02 |M..*y..L........| -00000030 3c 44 b4 80 80 7b 45 84 53 3f 6b 9f f6 b6 d3 0c |.c....>TP..alX | -00000200 da a8 5a f7 46 9d a3 57 a1 60 72 59 82 dd f7 08 |..Z.F..W.`rY....| -00000210 9e 48 16 8d b1 c0 f4 d2 e0 4e 5d 3c bf 23 88 0f |.H.......N]<.#..| -00000220 99 90 b0 a9 7e df 0b f5 1a 31 9e 59 e1 05 43 4e |....~....1.Y..CN| -00000230 4f d9 a5 b1 00 2e f2 c8 10 2c 73 ec c4 62 b8 c3 |O........,s..b..| -00000240 59 53 ce 38 37 9d c3 f6 77 36 34 f2 07 a6 21 1f |YS.87...w64...!.| -00000250 62 56 08 d3 ed ae 51 05 4d ed ef 7e 77 c5 7d ae |bV....Q.M..~w.}.| -00000260 b6 aa 74 e9 9a df 5d 89 c2 9f 07 75 a0 c2 35 2d |..t...]....u..5-| -00000270 cc 13 c5 a9 17 85 79 b6 45 0c 82 56 0d 19 9e bc |......y.E..V....| -00000280 e7 1a a6 cc 20 6f f9 7c 29 54 2f 5a 92 1f c0 8b |.... o.|)T/Z....| -00000290 9c a7 a3 85 43 f2 4c e7 c5 e3 af 6e 40 f4 14 d4 |....C.L....n@...| -000002a0 b8 26 5f ac d7 44 16 6d 40 d5 80 c5 49 df 8d fe |.&_..D.m@...I...| -000002b0 06 1c 7c 19 49 94 50 6f e8 8c e0 59 80 9a a0 0e |..|.I.Po...Y....| -000002c0 50 3f 03 de c9 30 88 15 91 83 4e a8 df c3 65 75 |P?...0....N...eu| -000002d0 cd 30 3e 7d 32 4a d9 ff 72 1e 1d 90 50 f3 67 8a |.0>}2J..r...P.g.| -000002e0 6a 94 c3 92 9a d4 0b 20 7e 61 98 6e db d0 71 4a |j...... ~a.n..qJ| -000002f0 8b 43 82 7e 76 e6 93 f7 16 03 03 00 bc e6 e6 85 |.C.~v...........| -00000300 a8 2c 74 47 10 a3 27 26 f7 46 fb da fa 6a 93 2c |.,tG..'&.F...j.,| -00000310 b1 14 b2 90 d3 4e 94 42 c4 37 6d 4d 43 73 b4 a7 |.....N.B.7mMCs..| -00000320 17 c1 3c 62 5d e4 c4 04 7c f7 13 39 9f a8 3f 05 |....d....| -00000390 e5 f8 f3 75 54 65 f4 6f e9 c8 eb a2 6c 06 98 fd |...uTe.o....l...| -000003a0 e9 bb a2 93 ab ee 0b fa c7 b4 fa e1 91 34 a9 a7 |.............4..| -000003b0 b9 14 16 25 32 88 09 3e b1 16 03 03 00 14 d0 c0 |...%2..>........| -000003c0 54 b2 37 35 9b cf 51 71 28 b2 d6 86 05 16 ad a0 |T.75..Qq(.......| -000003d0 18 e7 |..| +00000000 16 03 03 00 85 93 93 97 db ec 52 74 ce 78 ca e6 |..........Rt.x..| +00000010 95 19 ac 68 2c 81 ad 6b c3 6d d5 f7 84 a1 00 d0 |...h,..k.m......| +00000020 15 4c ae 2d 10 6a 07 d3 82 af af f8 d9 5d 9d 65 |.L.-.j.......].e| +00000030 8d c9 1e 8c 61 3e b3 67 71 89 41 7e 94 1e 4a 27 |....a>.gq.A~..J'| +00000040 73 53 83 d4 70 44 9a 4f 8d e7 62 af 9b 71 9e 83 |sS..pD.O..b..q..| +00000050 72 9a a1 e2 03 47 5f c3 11 4f 56 b9 6f 02 b5 b8 |r....G_..OV.o...| +00000060 8b 4a cb 99 ed 62 58 45 2d 0f f1 25 ce e9 de 7d |.J...bXE-..%...}| +00000070 3f 17 a8 35 d5 73 06 6e d6 bc 77 2a 82 27 d7 92 |?..5.s.n..w*.'..| +00000080 91 67 68 07 52 4b ed a0 3c 95 16 03 03 02 69 11 |.gh.RK..<.....i.| +00000090 73 93 d5 ad 00 17 11 3b 83 81 68 6d 3a 8a 02 1b |s......;..hm:...| +000000a0 90 df 84 f6 2b 4a b8 98 cf 50 0d dd 29 22 9d 58 |....+J...P..)".X| +000000b0 ea 7b 2c 30 2d e7 e2 d1 ae ad 00 9e 01 f6 ef 96 |.{,0-...........| +000000c0 ae a4 48 58 29 63 1d 2c 19 f3 c2 49 6f cf c9 7b |..HX)c.,...Io..{| +000000d0 e3 ca 5c e7 30 a0 b5 72 a9 3d 61 a2 0f 96 e4 d6 |..\.0..r.=a.....| +000000e0 8f 93 66 f1 de 24 88 14 9f 8b 14 9a b5 94 f7 70 |..f..$.........p| +000000f0 79 e4 94 68 b7 e0 f1 8e d0 1b 56 da d0 ce 90 b2 |y..h......V.....| +00000100 13 b3 4b b5 20 99 77 4a a4 83 47 4e 1b 1f db 35 |..K. .wJ..GN...5| +00000110 96 16 f0 d8 2b 7e 18 bf 0c b7 a3 da 44 fe 4c 96 |....+~......D.L.| +00000120 86 06 52 81 f9 a1 f4 ab 43 6d a3 fd 50 f6 83 08 |..R.....Cm..P...| +00000130 ba b5 d5 15 55 16 aa 84 95 77 1d 9e cd d6 53 d0 |....U....w....S.| +00000140 d3 c7 1c 9a 12 4d 8d 7a b5 0b 02 34 3f cf b5 0f |.....M.z...4?...| +00000150 98 56 cc 6e 15 ef 60 88 e0 71 2e 47 46 ce 19 11 |.V.n..`..q.GF...| +00000160 81 21 fe c6 80 86 d7 be d9 a7 6f a1 8c 8d ff ba |.!........o.....| +00000170 fb 8b 8b 8f 35 95 03 cb 10 e6 f5 71 18 d5 3c e0 |....5......q..<.| +00000180 60 e7 a2 47 36 41 a0 a3 c4 a3 c1 23 6c da 55 6a |`..G6A.....#l.Uj| +00000190 5f ef cd 58 e7 5d d3 cf 50 7b 3d b5 c6 7c 73 d9 |_..X.]..P{=..|s.| +000001a0 ce c3 ae 6a 46 4a 9a e6 10 57 53 b3 6c e5 ec 9d |...jFJ...WS.l...| +000001b0 f0 25 3c a4 a6 f0 42 1a 1e 8c 2b 40 79 e5 51 84 |.%<...B...+@y.Q.| +000001c0 79 6b c5 8a 3d 25 11 38 97 ca 15 11 e7 51 74 29 |yk..=%.8.....Qt)| +000001d0 66 9e 4d 4d 50 6a ec 1d 94 6b 4d c9 e9 a7 f8 4e |f.MMPj...kM....N| +000001e0 5b c3 06 74 ba 1d a1 79 3c 24 94 c9 3b b0 b1 16 |[..t...y<$..;...| +000001f0 fe 42 31 fe d0 4b e4 51 e6 06 63 89 2d 3b 56 6a |.B1..K.Q..c.-;Vj| +00000200 cf 89 4c 45 49 fd 10 58 9b 6b 4b 35 eb d8 c9 9d |..LEI..X.kK5....| +00000210 c0 31 b6 3e 33 da 7e 87 39 e3 c0 6a f8 7f bd 9a |.1.>3.~.9..j....| +00000220 26 4f 42 51 86 40 fb f7 fb cc 50 80 b6 0a 63 f4 |&OBQ.@....P...c.| +00000230 41 62 68 c0 b8 99 7d 8e ab e1 8a 15 92 f2 35 a4 |Abh...}.......5.| +00000240 cc 0e e7 9e 74 2d c1 b4 44 be 25 10 92 02 4f e3 |....t-..D.%...O.| +00000250 e7 ea c4 77 83 18 62 36 ba 9c 77 a6 d4 c1 6f 66 |...w..b6..w...of| +00000260 fc 07 4b 1a 00 98 a4 10 5e bc d5 93 07 e7 0c e3 |..K.....^.......| +00000270 34 05 32 23 c7 60 22 a8 52 fa 6e de 3f f9 c6 cc |4.2#.`".R.n.?...| +00000280 c2 54 08 d4 2a 98 39 20 09 f4 1d 8e d5 18 77 5a |.T..*.9 ......wZ| +00000290 f0 f4 08 a4 66 a7 8e fe f5 25 50 16 ca 4a 8d f0 |....f....%P..J..| +000002a0 b4 9f f1 37 e6 9b db 24 25 d0 a0 57 06 74 e2 14 |...7...$%..W.t..| +000002b0 46 51 e9 9c 03 f0 e0 16 17 d0 c9 54 54 29 e6 04 |FQ.........TT)..| +000002c0 9a c3 47 93 69 78 1a de ca f1 d4 b1 52 ff c0 5c |..G.ix......R..\| +000002d0 9f 5f 16 a9 35 01 f9 18 47 7b ee 06 f4 f3 3a 1d |._..5...G{....:.| +000002e0 94 b9 d5 2c 29 a0 80 85 b0 31 55 37 63 bc a6 e6 |...,)....1U7c...| +000002f0 67 66 21 5f eb 1c 17 15 16 03 03 00 bc 93 4b d6 |gf!_..........K.| +00000300 07 b0 12 ca 98 9f 52 b8 14 c3 6e d4 3b f2 74 e4 |......R...n.;.t.| +00000310 f5 6f 51 40 04 cd 1a 5f 69 fb 3d 68 2b 4e 09 df |.oQ@..._i.=h+N..| +00000320 c8 c9 6f c2 87 ae b6 f9 14 6a 41 fa 59 08 0b b0 |..o......jA.Y...| +00000330 d9 0b d0 61 fd 64 c6 52 3e 40 f2 96 75 b0 82 7a |...a.d.R>@..u..z| +00000340 8e 68 11 d2 bc 22 a0 bc 30 d8 a6 1a 88 26 00 d5 |.h..."..0....&..| +00000350 b5 26 49 c2 5d f3 19 c5 c0 9d 75 c3 f4 3e 95 85 |.&I.].....u..>..| +00000360 d0 8b de 31 79 b5 5c 27 6e 99 ab 50 fe ef 3a 07 |...1y.\'n..P..:.| +00000370 a0 a9 ce b7 4b 29 6b 93 42 6d db 34 6b 14 03 a9 |....K)k.Bm.4k...| +00000380 a8 1f c2 57 65 64 c3 95 ff 2a f8 39 de c3 36 f4 |...Wed...*.9..6.| +00000390 dd 77 b0 cf 86 be c4 9c 1d d5 ea 88 3c 8c ed 81 |.w..........<...| +000003a0 51 fc a9 80 78 bc 59 7d d1 e4 48 d5 d6 07 88 e1 |Q...x.Y}..H.....| +000003b0 d7 85 80 81 3a 35 9c 57 21 16 03 03 00 14 d9 b1 |....:5.W!.......| +000003c0 fb 95 82 67 c4 35 92 cd 47 cb 7b 0f 63 1a e0 32 |...g.5..G.{.c..2| +000003d0 43 75 |Cu| >>> Flow 13 (client to server) -00000000 16 03 03 00 35 e8 18 a5 be 6e 3b 37 5b d3 81 d3 |....5....n;7[...| -00000010 ad 0b 90 2e e0 df 2b 7f c4 ec 54 58 ef 85 e3 13 |......+...TX....| -00000020 41 07 50 d1 06 9b 3e a3 71 30 bf 7d 0c 7a bd 1e |A.P...>.q0.}.z..| -00000030 b6 e9 1e 10 0d 18 e3 b3 1c 38 14 03 03 00 11 bf |.........8......| -00000040 b3 65 ae 3b e6 c2 ad 30 83 75 65 c0 2b 0d 4b 11 |.e.;...0.ue.+.K.| -00000050 16 03 03 00 20 c6 cb f8 69 c3 bc 36 74 ff 8a 1f |.... ...i..6t...| -00000060 02 ef 32 37 fe 4f 14 51 d2 11 6a 98 05 4c 8c f6 |..27.O.Q..j..L..| -00000070 75 05 f9 39 2c |u..9,| +00000000 16 03 03 00 35 36 32 d0 64 c8 ba 33 1d 4f 31 73 |....562.d..3.O1s| +00000010 68 35 f2 76 0c b1 52 12 f5 4b 8a ea 74 0a 7f c2 |h5.v..R..K..t...| +00000020 a4 90 75 1f 24 0a 76 77 1f 01 15 9d 59 fa 16 56 |..u.$.vw....Y..V| +00000030 cc fd f9 08 74 76 31 cc f4 4c 14 03 03 00 11 c0 |....tv1..L......| +00000040 12 9b 35 78 94 a3 4e c3 b2 30 aa c7 fa 44 20 e6 |..5x..N..0...D .| +00000050 16 03 03 00 20 c6 a5 79 48 7b c3 2b 2d 6a 73 0a |.... ..yH{.+-js.| +00000060 83 a9 2b a7 ba 90 ea 6f b5 c2 c1 13 d7 7a 86 37 |..+....o.....z.7| +00000070 5a 24 8e f9 b3 |Z$...| >>> Flow 14 (server to client) -00000000 14 03 03 00 11 a0 fa 14 67 3f 54 1b 61 89 ed db |........g?T.a...| -00000010 9c 66 9f 49 77 2d 16 03 03 00 20 ca 47 f0 57 8c |.f.Iw-.... .G.W.| -00000020 3e 52 e6 e2 6c 01 cb 16 05 68 5e 19 12 ca 80 48 |>R..l....h^....H| -00000030 dc e9 ed 41 33 08 01 ef 6e b0 a3 17 03 03 00 19 |...A3...n.......| -00000040 d9 c8 68 e8 27 3d cf c0 33 71 55 a9 6a b4 6d a5 |..h.'=..3qU.j.m.| -00000050 da 8d bb 34 94 4f b1 c6 22 |...4.O.."| +00000000 14 03 03 00 11 c4 0e 59 d8 5e f6 5b 1d e6 20 7b |.......Y.^.[.. {| +00000010 45 4f 89 cd 2b a9 16 03 03 00 20 58 ff 80 6c f2 |EO..+..... X..l.| +00000020 fd 94 e7 66 b3 6d e3 37 57 8a 8c 35 98 4e bb c3 |...f.m.7W..5.N..| +00000030 42 87 32 b7 3b 6f ee 0f f1 7d 08 17 03 03 00 19 |B.2.;o...}......| +00000040 8a f6 02 17 4e 8a 1e 2b db 44 24 d4 aa c6 d4 af |....N..+.D$.....| +00000050 ef 5a a3 17 ba 77 f4 54 6e |.Z...w.Tn| >>> Flow 15 (client to server) -00000000 15 03 03 00 12 15 66 43 1e 60 21 d1 f6 94 34 3e |......fC.`!...4>| -00000010 ee 90 fc 87 45 2a f5 |....E*.| +00000000 15 03 03 00 12 84 3f 11 0b 00 bd 12 5e be 74 4d |......?.....^.tM| +00000010 04 e4 44 b2 01 73 66 |..D..sf| diff --git a/src/crypto/tls/testdata/Client-TLSv12-RenegotiateTwiceRejected b/src/crypto/tls/testdata/Client-TLSv12-RenegotiateTwiceRejected index 7698b6791dac41..62b6831a47b7a5 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-RenegotiateTwiceRejected +++ b/src/crypto/tls/testdata/Client-TLSv12-RenegotiateTwiceRejected @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 e3 fe 9a 75 3b |....]...Y.....u;| -00000010 82 ac ff 66 ee c2 e9 75 d9 29 cc 89 6d e0 27 19 |...f...u.)..m.'.| -00000020 62 07 0b 0b c0 49 df 17 26 af 63 20 2a 95 69 54 |b....I..&.c *.iT| -00000030 3c 5d d9 67 05 4c 45 77 d0 d9 46 29 ed 25 fd 7d |<].g.LEw..F).%.}| -00000040 b4 ef 39 d2 ee 3b a3 88 85 c5 78 9a cc a8 00 00 |..9..;....x.....| +00000000 16 03 03 00 5d 02 00 00 59 03 03 a1 1a ae 12 c9 |....]...Y.......| +00000010 55 34 c9 23 81 cb 4f b0 d8 35 97 8e d5 b7 ac 67 |U4.#..O..5.....g| +00000020 e1 9c fd ec 35 61 48 7b d5 c8 8b 20 e1 e9 61 af |....5aH{... ..a.| +00000030 0b 5c 69 7b cb f9 7a ee 73 31 20 07 c3 b9 27 47 |.\i{..z.s1 ...'G| +00000040 07 e1 c0 34 7c c5 8f 31 ac 9d cd 8c cc a8 00 00 |...4|..1........| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -61,189 +62,191 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 ca 74 42 |............ .tB| -000002d0 03 67 f2 31 ac 68 dd 2c db 23 7d 2c 84 df e9 50 |.g.1.h.,.#},...P| -000002e0 ff ef d4 a1 fa a4 5b 0b 0b 73 20 b7 0e 08 04 00 |......[..s .....| -000002f0 80 41 60 3d b1 1c 1d 14 8a 0a 3e 5c ea ca ce 07 |.A`=......>\....| -00000300 d4 45 18 6c 16 52 40 6f b0 f4 33 38 d7 cf 11 75 |.E.l.R@o..38...u| -00000310 08 4e 7d 8b cc 64 8c 79 dd 42 fa 43 2c 9e 84 91 |.N}..d.y.B.C,...| -00000320 e9 d5 bc 9f 84 2d 10 4b db 4e a6 be 37 e9 2e 34 |.....-.K.N..7..4| -00000330 d7 3e a7 17 80 b5 cd 37 04 2a 6c 2a cc 5f 0c a1 |.>.....7.*l*._..| -00000340 ff c2 31 86 17 50 3a 47 4d 99 49 94 9c 11 12 e2 |..1..P:GM.I.....| -00000350 70 fa d1 f4 7e 3a f2 3a df 16 34 45 0a b2 7b 73 |p...~:.:..4E..{s| -00000360 fa 59 23 85 88 74 2b e6 1e cb c7 00 f5 96 c8 8a |.Y#..t+.........| -00000370 02 16 03 03 00 04 0e 00 00 00 |..........| +000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 95 6f d7 |............ .o.| +000002d0 37 b9 49 ff 98 39 6f 25 c5 c0 e0 89 61 23 2d 22 |7.I..9o%....a#-"| +000002e0 5b 86 d6 7f a1 19 07 f7 12 4d 01 e6 1b 08 04 00 |[........M......| +000002f0 80 ca fd 6e 23 09 33 a7 0c 9b 4b 2b 38 37 a0 32 |...n#.3...K+87.2| +00000300 21 a4 e4 fe 56 b6 4e c6 d1 e6 86 fa d9 13 b4 f3 |!...V.N.........| +00000310 72 bf 31 ef c3 33 e9 99 a1 bf a4 e5 10 46 44 7d |r.1..3.......FD}| +00000320 cb ea 38 99 d0 bc 1f 16 81 fb ad 09 7e 9e 54 99 |..8.........~.T.| +00000330 40 c2 44 65 94 53 88 c3 28 db 7f 1b fa 52 f1 63 |@.De.S..(....R.c| +00000340 9a e9 f2 43 7f 19 2b 3e 02 3d 53 ed a1 f9 9a ff |...C..+>.=S.....| +00000350 aa af 8d 4c ab bf d2 7a b4 ea 1e f9 22 fe ee 7e |...L...z...."..~| +00000360 a1 3e dc d4 f1 76 d1 4e 7b 1b 84 f6 b3 a7 f7 a9 |.>...v.N{.......| +00000370 37 16 03 03 00 04 0e 00 00 00 |7.........| >>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| -00000030 16 03 03 00 20 0a a7 0c 5f 47 d1 7c 5e ea 63 82 |.... ..._G.|^.c.| -00000040 83 27 6c bc 29 20 25 ec 3d 15 2d 59 94 e7 9d 2c |.'l.) %.=.-Y...,| -00000050 8b 4e 95 85 fc |.N...| +00000030 16 03 03 00 20 d0 64 37 7c 81 1c cd 86 e7 ab 97 |.... .d7|.......| +00000040 25 1d c5 ee bd 5a 49 44 69 c8 3e db 37 6a dc 9a |%....ZIDi.>.7j..| +00000050 fd e2 5e 82 16 |..^..| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 20 03 84 99 6b f7 |.......... ...k.| -00000010 3e 44 b1 96 6a 09 75 bf 26 4b 67 30 c4 e9 86 74 |>D..j.u.&Kg0...t| -00000020 4d e1 e3 9b fa 15 8e 1e 72 f9 5a |M.......r.Z| +00000000 14 03 03 00 01 01 16 03 03 00 20 18 71 b3 67 a3 |.......... .q.g.| +00000010 98 36 20 8f e4 17 30 e4 73 de 91 88 ef f7 f3 8a |.6 ...0.s.......| +00000020 c6 0e 5e b2 1f e2 e6 5a 52 f6 72 |..^....ZR.r| >>> Flow 5 (client to server) -00000000 17 03 03 00 16 68 29 2f 1b 1c 9d 28 6e b6 e5 09 |.....h)/...(n...| -00000010 4a 77 96 23 20 da 67 9e 14 ec 6b |Jw.# .g...k| +00000000 17 03 03 00 16 20 c3 2a be 39 d2 7c 1c 83 30 cc |..... .*.9.|..0.| +00000010 7b cd 44 ae 2b e8 d4 19 50 69 81 |{.D.+...Pi.| >>> Flow 6 (server to client) -00000000 16 03 03 00 14 66 e8 13 d1 b5 13 cf 5c 71 7b d9 |.....f......\q{.| -00000010 63 29 3d be 68 9f f4 ad 7b |c)=.h...{| +00000000 16 03 03 00 14 96 2f 3a b5 b5 56 c9 d5 38 c0 1a |....../:..V..8..| +00000010 eb 35 c5 b9 dd 1c 5b e6 50 |.5....[.P| >>> Flow 7 (client to server) -00000000 16 03 03 01 1a cb 21 91 08 de 08 87 fc 63 10 93 |......!......c..| -00000010 24 a3 83 e8 99 07 d5 03 19 61 4e 8e df dd d0 f6 |$........aN.....| -00000020 50 3c fd 14 23 67 cd 74 ad 87 9b b7 8f ee a2 23 |P<..#g.t.......#| -00000030 6c 2f 90 18 f4 01 6b 31 fe ca f7 65 f9 64 5a bc |l/....k1...e.dZ.| -00000040 49 3b 25 4c 35 59 50 f2 bc 70 cb 24 2f ec 47 12 |I;%L5YP..p.$/.G.| -00000050 b6 16 22 9e ce 3b fb 17 f9 59 4e 2f 5c 03 90 47 |.."..;...YN/\..G| -00000060 d5 e8 e9 eb d9 55 bf 8f 28 e8 72 e5 15 21 42 ba |.....U..(.r..!B.| -00000070 86 65 ac b4 f5 53 7e 98 38 39 b5 7d 2a 14 d6 7b |.e...S~.89.}*..{| -00000080 3e 69 9d dc 43 33 ed 0b 8a 77 14 54 9d b0 da 88 |>i..C3...w.T....| -00000090 3b 75 50 01 07 02 99 36 39 4f 01 ee 02 cf 05 a5 |;uP....69O......| -000000a0 71 75 03 3a a5 5c c4 ce 0e 1d ba dd 9d 79 c1 84 |qu.:.\.......y..| -000000b0 81 69 7c cd 63 a4 20 a3 a2 13 8e 6e d7 01 29 8d |.i|.c. ....n..).| -000000c0 35 e2 fa 3d 74 ea 40 06 7e 0b 2d ab 3c 54 73 ef |5..=t.@.~.-....&.FU........| -00000110 43 bd e5 9f 5c 35 02 25 54 17 89 c0 be ab bc |C...\5.%T......| +00000000 16 03 03 01 34 ae 4f 4c 78 20 df 3b 9e f3 c1 82 |....4.OLx .;....| +00000010 37 6b ca fc 48 2a 2c 9a 71 7e c3 20 a0 e8 f7 d5 |7k..H*,.q~. ....| +00000020 76 af 11 81 dd 69 e9 93 58 df 58 8c fb f1 82 ad |v....i..X.X.....| +00000030 35 0e fb 6c a9 58 7a 5d 79 38 cb d7 77 1f 93 34 |5..l.Xz]y8..w..4| +00000040 7e bb 88 61 9e 19 1a 98 bb 6f ad 78 2c a6 73 ab |~..a.....o.x,.s.| +00000050 a4 7e 4b d7 75 69 7b 92 47 68 3d 7b 40 03 60 59 |.~K.ui{.Gh={@.`Y| +00000060 2e 04 bc cd fc ca 45 2c 2c f7 e5 0a 66 49 0b 86 |......E,,...fI..| +00000070 c5 ec 86 e6 ae c9 39 4b 05 1f 21 db 78 8d 51 e6 |......9K..!.x.Q.| +00000080 5d fe be 6a 1d 85 e1 15 23 56 86 4c ba 09 bf 7b |]..j....#V.L...{| +00000090 9e 36 e9 7a 8f 06 af 10 7b 4a 4b af 58 db f9 4e |.6.z....{JK.X..N| +000000a0 13 36 61 f4 e3 a9 c6 37 11 d4 ed 9e 6a c7 bf 16 |.6a....7....j...| +000000b0 ea a4 ae 01 99 4a b5 3e 9c ca 6c ce e8 a2 3d b8 |.....J.>..l...=.| +000000c0 81 ef 60 7b 51 79 81 22 87 49 ad ab fb c6 f6 48 |..`{Qy.".I.....H| +000000d0 c0 ba e0 2b d7 90 42 f7 a8 0d f0 cf 2c 7a 55 7d |...+..B.....,zU}| +000000e0 be 65 a6 67 bd 96 7f 01 6f ce db f6 50 2f 45 b6 |.e.g....o...P/E.| +000000f0 6f cc 74 ba 6f 57 3b e3 a4 08 34 92 e6 ac e4 99 |o.t.oW;...4.....| +00000100 2e b6 9b 29 4d 26 dd 7f 5c f5 1b 7e 5e 5c fd f7 |...)M&..\..~^\..| +00000110 d4 da 81 47 51 8c 3f 94 e1 95 5b de b9 6e 4a 76 |...GQ.?...[..nJv| +00000120 bc 59 6a bb 1f 1c 79 02 93 2e 21 b3 6d be ae b4 |.Yj...y...!.m...| +00000130 95 6b 8a 48 cb 84 fb ab b3 |.k.H.....| >>> Flow 8 (server to client) -00000000 16 03 03 00 85 10 dc 85 7d 1e 66 b2 a4 86 68 a7 |........}.f...h.| -00000010 31 f3 b1 d8 c7 c9 26 6d 74 0a c0 c0 c0 39 ad b5 |1.....&mt....9..| -00000020 1b b1 dc d3 15 9f a4 96 27 dd 65 85 ad 83 2f 97 |........'.e.../.| -00000030 42 6f 8a 9b 58 fc f8 1b 54 89 cd 9c 11 10 b9 1d |Bo..X...T.......| -00000040 c3 e1 8e 89 20 a5 2d 0b 31 b5 e0 16 54 ce 93 9b |.... .-.1...T...| -00000050 de cc b1 af 48 48 33 96 4d a6 00 78 7b 60 3f 7c |....HH3.M..x{`?|| -00000060 cd 86 64 5e 38 e8 fd 60 d4 b1 89 cd e4 fb 42 b5 |..d^8..`......B.| -00000070 82 8f e5 23 91 87 6d 54 9e 85 37 34 df ae 70 a5 |...#..mT..74..p.| -00000080 cd 4c de 95 f7 9f a9 b4 87 0b 16 03 03 02 69 d1 |.L............i.| -00000090 c2 87 4e c8 26 b3 89 88 da c0 b9 24 84 88 c0 76 |..N.&......$...v| -000000a0 94 08 eb 56 5d fc 96 81 18 f3 5b 90 f1 38 e5 ae |...V].....[..8..| -000000b0 b5 4e 9a c1 19 19 1a b3 40 55 08 cb 0e 3b 93 b3 |.N......@U...;..| -000000c0 d7 74 80 3a 7c 0d eb be 70 0e a9 50 88 d2 52 11 |.t.:|...p..P..R.| -000000d0 2d 55 cf d9 9d aa 92 bd 15 ab 1a 62 b3 26 09 3f |-U.........b.&.?| -000000e0 50 2b b8 f2 00 a2 5a 2b 98 e3 bb e2 c7 0b 07 0e |P+....Z+........| -000000f0 5a e5 2e 08 27 87 f3 0b 1b f7 f7 82 49 cf 08 d6 |Z...'.......I...| -00000100 7c 78 39 0e 92 e7 28 f4 8f 12 ac 6a c2 72 46 e8 ||x9...(....j.rF.| -00000110 92 bf 36 c7 52 46 7a 01 92 6d a9 08 19 2a 07 70 |..6.RFz..m...*.p| -00000120 8c b2 40 7d 7f 00 27 ea fb e6 24 5c d2 d8 ae 99 |..@}..'...$\....| -00000130 0b cc 2a 4b 35 12 de 85 e9 b5 e2 2d 80 39 6d 17 |..*K5......-.9m.| -00000140 7a ca ef ca bb 58 1e 13 86 10 bc 4d 8a df 32 ba |z....X.....M..2.| -00000150 3b 7e f5 40 ec af 2e 63 be f3 38 02 a9 9d 30 1f |;~.@...c..8...0.| -00000160 c4 d2 04 93 df e1 8a bb 5d f0 6a a8 d9 b9 ad bf |........].j.....| -00000170 da d6 5e bd a1 a8 b8 20 91 26 ab 50 fd 2b bf e1 |..^.... .&.P.+..| -00000180 56 33 e3 e4 4f 3f 21 ea 86 10 e9 35 84 03 70 0b |V3..O?!....5..p.| -00000190 bd dc 44 fc 58 1d bc 9e 3a 0d 3b 61 13 d6 33 1b |..D.X...:.;a..3.| -000001a0 97 a8 67 bf e6 d9 1d 34 69 70 93 4a 6c 28 88 01 |..g....4ip.Jl(..| -000001b0 01 9d 78 59 c0 58 66 c3 70 e4 ba 74 e6 0f f7 27 |..xY.Xf.p..t...'| -000001c0 8b dd 05 ac 0d 99 dc e9 c8 2f 16 ee 59 53 91 d7 |........./..YS..| -000001d0 28 27 db ad 85 e6 d0 72 28 81 31 e1 e7 bf dd 95 |('.....r(.1.....| -000001e0 75 ff d6 a9 dc 4f 30 37 52 07 87 cb 8d a7 4a 00 |u....O07R.....J.| -000001f0 22 53 3c 6d 91 94 11 5d 0a a8 21 ea 1f 18 4e 42 |"S...A...| -00000390 37 c3 62 b5 30 89 4d b6 6c 5c f2 8c ff 3c 2e 48 |7.b.0.M.l\...<.H| -000003a0 bf 0f 5f 3e e0 b2 d7 64 2b dc 42 ad 2f 83 b5 6b |.._>...d+.B./..k| -000003b0 5a 04 51 f8 c4 07 c2 61 e4 16 03 03 00 4a a9 18 |Z.Q....a.....J..| -000003c0 58 79 d8 ad 86 d7 b6 77 39 76 99 b4 0a 29 72 b1 |Xy.....w9v...)r.| -000003d0 eb c3 ec 95 4d 30 4c 21 1a 7f 52 da 56 21 78 b7 |....M0L!..R.V!x.| -000003e0 fe dd d3 0e fc a9 89 40 99 97 12 68 7e 28 6e 32 |.......@...h~(n2| -000003f0 5b c0 e8 b4 42 eb ee 83 c2 ae 0d 28 99 48 46 a9 |[...B......(.HF.| -00000400 64 38 60 59 c2 dc 5c 7b 16 03 03 00 14 78 72 cf |d8`Y..\{.....xr.| -00000410 ac 5d 53 7e 73 b1 6e e2 0a 8e 12 33 be 03 86 6a |.]S~s.n....3...j| -00000420 ce |.| +00000000 16 03 03 00 85 60 c6 c2 9b 61 f1 1f 81 6f f1 0b |.....`...a...o..| +00000010 87 2d f2 93 a6 4e 7e 74 c2 61 cf 1d a1 00 32 8f |.-...N~t.a....2.| +00000020 f7 3d 93 da 91 cc 5b d5 21 c6 ba 4c 1b eb 81 a9 |.=....[.!..L....| +00000030 a1 2a fe 88 6d 3e 1d 1f 57 22 9c ff fe 6c 85 ca |.*..m>..W"...l..| +00000040 0b 13 f3 9f 63 55 b9 49 61 2b dd 5e 2b e1 fd ec |....cU.Ia+.^+...| +00000050 1d 0d 94 06 42 dc 2c 6f 88 22 cb 30 4f 66 6f 16 |....B.,o.".0Ofo.| +00000060 3a a0 c0 23 12 82 46 38 70 68 e5 12 b9 16 12 e7 |:..#..F8ph......| +00000070 38 6b 50 64 55 f0 47 a9 e0 cd 19 01 8e d5 12 96 |8kPdU.G.........| +00000080 09 db 5c 52 4b db 9b 26 43 37 16 03 03 02 69 41 |..\RK..&C7....iA| +00000090 52 a8 ca e5 8f e0 33 4b 52 93 74 ef bb 42 20 d1 |R.....3KR.t..B .| +000000a0 cc 24 79 ce df 51 90 e0 19 b1 11 fb e9 ef b5 e4 |.$y..Q..........| +000000b0 ae da 72 e1 35 a8 41 88 a8 4b 68 d2 50 58 ba ef |..r.5.A..Kh.PX..| +000000c0 b0 1e 20 26 a3 c2 86 a7 68 60 84 2d 23 14 38 21 |.. &....h`.-#.8!| +000000d0 12 60 05 2f 79 9e c0 08 4c 87 a7 41 b3 d3 84 9a |.`./y...L..A....| +000000e0 f1 45 bd 2d ff 7c b5 bd c4 3a b1 48 10 9a d8 cf |.E.-.|...:.H....| +000000f0 ce 58 47 75 e1 6d 01 b6 18 bd 78 6b 86 a1 f2 1b |.XGu.m....xk....| +00000100 c8 03 4d 0a ce ce d4 68 8f 9a 54 1e 83 83 89 2c |..M....h..T....,| +00000110 36 a2 3a b5 95 09 de c5 8e 8c d0 a4 95 59 7e d6 |6.:..........Y~.| +00000120 f8 5f 96 e9 c5 cb 9b 6c c0 b7 55 15 b3 b4 d0 ea |._.....l..U.....| +00000130 bf 11 1c 89 1a 5c f2 09 74 9a 43 73 4f 6f 00 33 |.....\..t.CsOo.3| +00000140 b9 e0 6a 99 b9 e0 02 86 dd cd 07 68 72 63 0d 8b |..j........hrc..| +00000150 e0 e8 12 68 4d f6 3b dc 0a 93 82 48 ce f7 96 ba |...hM.;....H....| +00000160 7f c5 90 07 45 66 3d 47 b8 8d 5f 0d 41 8d 88 76 |....Ef=G.._.A..v| +00000170 bf ce 9c 2a 2e 25 7c 47 f6 96 73 0b 43 42 73 2c |...*.%|G..s.CBs,| +00000180 d5 b3 bc 82 c5 19 2f 5d c4 69 21 7d c8 7b 1b b8 |....../].i!}.{..| +00000190 b6 d1 37 89 92 a6 b7 44 cd e6 23 1f a1 03 08 05 |..7....D..#.....| +000001a0 1e cf 54 78 e9 af 04 bc e3 94 aa 12 ce 67 62 ce |..Tx.........gb.| +000001b0 f4 2f ef f7 2b 8d 65 06 08 07 2f c6 fa b2 d2 c8 |./..+.e.../.....| +000001c0 f1 d7 c4 1f cc 8b 0e b8 f2 b2 1d fb 09 19 5a b0 |..............Z.| +000001d0 a3 23 6e 05 20 9a 28 39 16 05 0e c4 7d d0 59 f5 |.#n. .(9....}.Y.| +000001e0 9a de 88 da a2 c0 fb 7b 84 19 05 7d 80 6f 9a 03 |.......{...}.o..| +000001f0 0a f8 5f 97 a2 b4 c0 d9 6d 79 f5 c0 51 d3 63 d2 |.._.....my..Q.c.| +00000200 b8 ee a6 b0 76 59 7f 92 6d ad 3c bb 96 34 ac 0c |....vY..m.<..4..| +00000210 81 61 c5 07 7e 65 f1 c3 dd 3e 69 ad ee 7f 56 8d |.a..~e...>i...V.| +00000220 d7 92 48 5c 23 be 94 44 2e 8a 13 76 ce a3 cb d2 |..H\#..D...v....| +00000230 86 d5 eb 2c 48 55 fd 31 c1 94 08 55 69 8c 20 cd |...,HU.1...Ui. .| +00000240 65 fa 47 5e 8e 7f 04 a4 a0 54 c3 cf 4e b5 a0 61 |e.G^.....T..N..a| +00000250 3a 66 fd 21 5d 42 ab b3 46 d8 2a 64 69 1a d3 da |:f.!]B..F.*di...| +00000260 41 65 2c 36 50 f1 79 c3 83 01 d2 87 41 d9 10 1d |Ae,6P.y.....A...| +00000270 75 a2 74 de 48 e3 9c 9e c8 96 58 45 43 82 dd d6 |u.t.H.....XEC...| +00000280 e1 46 66 2f 13 e4 1d fe 81 8f ea 3d 6d 83 d7 97 |.Ff/.......=m...| +00000290 4d ed c6 67 0e 4b e5 a1 ca 2b 24 c5 b1 24 af df |M..g.K...+$..$..| +000002a0 7a 0e 44 b9 c1 5f 07 43 c1 6f 94 d3 22 fb 14 df |z.D.._.C.o.."...| +000002b0 a3 23 38 c9 91 ff 12 41 fa 47 f7 83 fb b6 ca ea |.#8....A.G......| +000002c0 19 01 22 8a ba d7 86 c5 d2 82 2f ac f5 4c a4 e9 |.."......./..L..| +000002d0 90 f5 52 c7 88 96 ae f3 0a 91 53 1c db bf 4b a9 |..R.......S...K.| +000002e0 42 43 0d 2d cc f2 6a 79 1e 9a 3a bd 55 da 5d 6c |BC.-..jy..:.U.]l| +000002f0 a3 84 79 76 7d 96 62 5e 16 03 03 00 bc 51 62 b9 |..yv}.b^.....Qb.| +00000300 26 c9 f7 dd 55 83 aa cd 90 5d e9 9a 29 92 8c 6f |&...U....]..)..o| +00000310 b1 df 4c b7 be 75 89 2b dc b1 ad ae 01 38 27 0f |..L..u.+.....8'.| +00000320 36 43 0b 04 69 6b a4 3a 52 b5 4f df 50 1d 04 ee |6C..ik.:R.O.P...| +00000330 4a f1 a0 6c 06 6f 24 2a f3 dc 7c a4 96 12 e2 83 |J..l.o$*..|.....| +00000340 9c d8 2d 63 2e 3b 1c 73 e6 32 ce 15 76 01 9a a9 |..-c.;.s.2..v...| +00000350 3b a7 dc aa 40 82 4d 5a 68 24 78 4e fe 64 db a4 |;...@.MZh$xN.d..| +00000360 ff 87 6d 90 bc d9 ec 0d ed 8b 54 12 bd 74 16 1d |..m.......T..t..| +00000370 2c 30 57 8f 67 56 7c 09 e6 b1 12 f8 4a 9f e3 79 |,0W.gV|.....J..y| +00000380 c1 8b fa 91 f7 1a 29 bd a3 1f 59 59 16 26 04 10 |......)...YY.&..| +00000390 e4 19 c9 91 a6 b3 c5 b5 df a3 b0 11 a5 87 29 4e |..............)N| +000003a0 5b 9f 96 cf 88 19 9e ae b4 e6 63 19 6d c5 ee 7a |[.........c.m..z| +000003b0 c9 38 2d 0a fd f6 3b f8 f5 16 03 03 00 4a f5 3f |.8-...;......J.?| +000003c0 ef a3 f8 86 1b f5 ce 8e 48 f2 d0 cb 75 d6 80 b2 |........H...u...| +000003d0 78 ef bc 77 a6 aa 91 cd 88 39 62 f8 42 78 7c f3 |x..w.....9b.Bx|.| +000003e0 8b c7 86 71 9c a5 9f 1c 5f 40 25 e9 c6 69 82 a1 |...q...._@%..i..| +000003f0 45 ee d1 9d f7 9c a0 b9 34 b8 82 72 f9 f8 1d fb |E.......4..r....| +00000400 a5 74 b5 a5 68 04 82 c4 16 03 03 00 14 07 9c 37 |.t..h..........7| +00000410 77 ad 83 27 66 2c 3a ba 26 22 2f 72 b4 d1 c8 c1 |w..'f,:.&"/r....| +00000420 06 |.| >>> Flow 9 (client to server) -00000000 16 03 03 02 69 33 a1 bb 7d ff 23 ba ad dc a0 1e |....i3..}.#.....| -00000010 3c 7c 1f 53 59 81 13 04 49 f7 6e bf 8f ef 11 92 |<|.SY...I.n.....| -00000020 51 bf 8a e7 fe ef 76 e4 f8 88 27 95 f4 12 f2 ed |Q.....v...'.....| -00000030 cc ec a6 2d 10 87 19 5e c9 fe cd 12 ae 1a 38 f9 |...-...^......8.| -00000040 78 f1 42 f8 47 26 1b c2 5f 82 a5 aa 65 17 eb 06 |x.B.G&.._...e...| -00000050 03 c8 22 9c 9f 8b 87 26 4a 64 18 93 9c f4 16 82 |.."....&Jd......| -00000060 28 eb 8d 2d f8 51 60 3a f4 bb a3 77 f5 97 2b 6a |(..-.Q`:...w..+j| -00000070 19 db 2e 46 8b d8 08 d8 28 88 bb bd d8 3b c2 bb |...F....(....;..| -00000080 8c 87 1b 17 a0 34 a8 c6 e8 a5 a0 e5 cb d6 9c 9d |.....4..........| -00000090 a6 a4 6c 06 6a 85 0c 5e ef 0c 97 6f b6 11 e7 4d |..l.j..^...o...M| -000000a0 60 91 95 74 48 af 1b 0b 47 03 ed 47 e9 b9 28 36 |`..tH...G..G..(6| -000000b0 74 0d 8a b8 1e 5d 1c cb ea 7b b7 8f 18 e3 b9 b9 |t....]...{......| -000000c0 43 ed 53 f2 bf 7e 8c 8f e6 c0 2a f4 31 25 f5 e8 |C.S..~....*.1%..| -000000d0 44 48 2d 36 4f a7 11 f6 ea 7c 3f d6 2d 5d 3e 39 |DH-6O....|?.-]>9| -000000e0 fb 50 f1 c9 eb 25 be 7a c5 af 54 f5 12 07 04 22 |.P...%.z..T...."| -000000f0 ec 68 45 f1 fd d1 91 0a cf 64 0f f3 58 4e e8 ce |.hE......d..XN..| -00000100 f6 9e c0 57 ea 3a bd a0 5d 26 d4 ab ed 1a 34 12 |...W.:..]&....4.| -00000110 9f fa ec 73 d6 38 06 78 85 3c f9 cd aa fb fd 32 |...s.8.x.<.....2| -00000120 cb e0 a7 d0 bb 71 b0 a3 d2 b9 89 5e a9 57 18 7e |.....q.....^.W.~| -00000130 c0 c9 2e e3 e5 07 65 6f 4c ad 22 78 50 f2 87 0f |......eoL."xP...| -00000140 cd 93 b3 19 a9 73 e2 1d bf 38 a0 20 8d 3a 3a 4e |.....s...8. .::N| -00000150 b3 02 b7 11 1f df 12 60 dd 10 1c f8 75 4a 52 86 |.......`....uJR.| -00000160 71 61 88 37 b3 bf f7 2d 45 6d 10 4a 9f 80 b3 fc |qa.7...-Em.J....| -00000170 36 d4 d3 9f 08 f2 bb ce 9c 3a 9a 59 f6 0a 0a aa |6........:.Y....| -00000180 7f 61 29 0b df 29 92 09 55 ec 1c 84 d6 1f 7c 68 |.a)..)..U.....|h| -00000190 72 00 88 5e e3 6a ee 35 45 7a 0b 49 35 0e 2d 7c |r..^.j.5Ez.I5.-|| -000001a0 ac 8b 9f 9c 94 48 2d 1e 74 24 68 14 8e 16 b4 06 |.....H-.t$h.....| -000001b0 24 66 a3 18 f4 ff 21 31 4b 9b bb 09 8a 9d 57 e4 |$f....!1K.....W.| -000001c0 a1 5c ae f1 e1 4b 5c 2a be 64 3c 7a 61 10 3d 73 |.\...K\*.d....^O| -000002f0 f7 4b 38 70 d0 82 42 f2 5a c9 34 d2 76 7b 8d e4 |.K8p..B.Z.4.v{..| -00000300 0b c7 3d be 86 7d 21 8c f8 9f 1a 4a dc 48 07 12 |..=..}!....J.H..| -00000310 ab 4c 98 f7 8b 3e 02 49 c2 d1 b5 e9 9a e0 5a 74 |.L...>.I......Zt| -00000320 58 ce a3 33 be 0f ca 12 54 a2 0e a9 e5 22 8f 82 |X..3....T...."..| -00000330 df 1a 3e 21 bf 66 c6 13 39 77 83 95 b1 4b 1d d3 |..>!.f..9w...K..| -00000340 32 69 24 c2 89 14 03 03 00 11 1c eb 63 31 c9 c9 |2i$.........c1..| -00000350 8a e3 8b 25 7e e9 b7 7d 19 03 3c 16 03 03 00 20 |...%~..}..<.... | -00000360 c6 41 57 04 ab e8 6c 55 0b dc cd 06 f5 b3 4b 95 |.AW...lU......K.| -00000370 40 b5 04 47 da 71 d7 70 ce d6 2e 0b c7 66 16 3c |@..G.q.p.....f.<| +00000000 16 03 03 02 69 70 78 66 7a 28 3b 50 a6 21 b2 cc |....ipxfz(;P.!..| +00000010 52 a6 74 f7 29 8a dc fd c0 1c 79 d0 20 23 d0 74 |R.t.).....y. #.t| +00000020 d3 16 a6 4c 6d df 6e 32 2e 69 46 dd 8d 22 9e 8e |...Lm.n2.iF.."..| +00000030 cb 75 44 75 7e 24 69 ed ce cf 5c 4b a9 a2 b5 d0 |.uDu~$i...\K....| +00000040 48 bf 08 d8 1d 16 d9 9f 6d b4 24 ee f1 f3 58 79 |H.......m.$...Xy| +00000050 4c 19 4c 99 7f 01 a8 80 b1 3c 4e 55 d1 64 75 89 |L.L......?.Pjad*Q...| +00000120 01 2e 5a 1c 8b 12 58 c5 32 e1 3f ea 0b ac 4c 3b |..Z...X.2.?...L;| +00000130 1d 97 11 e9 92 6d 52 f0 1e f7 5a f8 71 5f f5 a6 |.....mR...Z.q_..| +00000140 6b aa 30 4f 85 41 c6 49 83 37 3a 72 86 1a be 7a |k.0O.A.I.7:r...z| +00000150 1d bc d6 ad 67 6c 95 42 5d 74 10 8e ac 4b 8d b3 |....gl.B]t...K..| +00000160 6e 3d 9c 8f 08 71 be ce ce aa 64 26 62 2a 58 8c |n=...q....d&b*X.| +00000170 e4 7b 75 e9 61 90 38 b2 c2 a0 8d c7 a9 11 cf 5b |.{u.a.8........[| +00000180 30 a7 33 3b 2b c2 fd 2a 6b db 3d cf 35 6c 23 28 |0.3;+..*k.=.5l#(| +00000190 14 e4 7b 10 50 e8 00 9f af 60 69 cf a9 d9 93 f3 |..{.P....`i.....| +000001a0 8f 7d 39 49 97 5b 92 4a 35 2a 6c 68 3b 48 25 77 |.}9I.[.J5*lh;H%w| +000001b0 6e d0 57 76 c4 94 83 10 e3 3d 00 e8 b9 fe da d9 |n.Wv.....=......| +000001c0 a5 56 22 61 e0 f4 33 dd c8 dd 4d 2f 39 51 35 12 |.V"a..3...M/9Q5.| +000001d0 9a cb dd e1 03 d3 27 9f 41 71 83 5c c7 6c a3 38 |......'.Aq.\.l.8| +000001e0 9f db 39 1c 2a f6 4b e0 48 44 61 1b 34 3c bc ed |..9.*.K.HDa.4<..| +000001f0 7a 31 c9 9f 8e 82 2d fb b6 bd 6e 4c 57 76 23 f5 |z1....-...nLWv#.| +00000200 7d 68 ff 02 75 42 10 e7 2d 24 83 4a 04 7a 78 d7 |}h..uB..-$.J.zx.| +00000210 60 b2 70 07 3f e4 bc 54 42 65 22 c3 2e aa 35 85 |`.p.?..TBe"...5.| +00000220 df fa e5 c9 c9 f5 ee 6a 50 82 7a bf 92 96 78 5e |.......jP.z...x^| +00000230 a8 ac 2a d1 5e e2 f0 20 6b 51 97 3d 74 71 eb d8 |..*.^.. kQ.=tq..| +00000240 40 5f 06 72 b6 8c 1c 2a 71 dd 53 38 fb de b6 31 |@_.r...*q.S8...1| +00000250 53 8d 8f 4e 0f c1 a5 ea bf 27 fa ed ff 0c 56 34 |S..N.....'....V4| +00000260 55 88 e1 d2 65 a7 e5 10 02 4d ba 8e ad fa 16 03 |U...e....M......| +00000270 03 00 35 94 36 93 c6 5e c8 ec 8e 63 86 7b 13 d6 |..5.6..^...c.{..| +00000280 78 b2 5d 2f 54 55 5a 41 1c 65 5e b2 69 ab c2 bd |x.]/TUZA.e^.i...| +00000290 d9 a3 55 42 d1 96 29 51 c4 38 c4 ec 1f d4 19 6c |..UB..)Q.8.....l| +000002a0 2b 82 75 81 37 07 79 35 16 03 03 00 98 d9 a2 ba |+.u.7.y5........| +000002b0 86 ca f4 24 e1 e0 72 0e 21 81 f6 c5 d7 0a 1c 17 |...$..r.!.......| +000002c0 05 1f ce b8 4d d3 d8 e6 13 c8 a6 4d f3 da 0f 4d |....M......M...M| +000002d0 e3 21 29 84 78 90 fa 11 a5 06 19 3a cd ca f2 1f |.!).x......:....| +000002e0 a2 39 4d b3 03 8d 22 27 eb bc f4 8b b8 61 c0 f3 |.9M..."'.....a..| +000002f0 70 34 56 80 28 a2 85 1e 21 7c 11 b8 c6 0c 0d bd |p4V.(...!|......| +00000300 02 56 8a b5 d5 7b 22 97 3e 70 20 31 00 a0 16 2d |.V...{".>p 1...-| +00000310 87 b5 c7 b0 5e 40 2d 08 82 97 38 64 c4 14 e3 16 |....^@-...8d....| +00000320 0d 6f d1 b1 6b 5f 7c 14 18 66 67 aa 8a b6 6b 66 |.o..k_|..fg...kf| +00000330 7d e3 68 ca 97 13 ad ca 0a 46 06 6b 4a a9 c0 ee |}.h......F.kJ...| +00000340 8b 61 6d 2d cc 14 03 03 00 11 93 6d 0b 63 c5 e2 |.am-.......m.c..| +00000350 3f 0f da dd f4 7c ae 95 2a e8 65 16 03 03 00 20 |?....|..*.e.... | +00000360 62 97 ee b8 34 aa be 0a 2c 24 30 6c 89 f2 80 08 |b...4...,$0l....| +00000370 05 69 bb ad a8 5c 90 21 04 77 96 44 ae 5f d4 d5 |.i...\.!.w.D._..| >>> Flow 10 (server to client) -00000000 14 03 03 00 11 c0 4e fd fa 3a 1c 4a 19 ce 08 f6 |......N..:.J....| -00000010 da 70 56 fa 39 42 16 03 03 00 20 0c bc c4 5e 24 |.pV.9B.... ...^$| -00000020 e3 0e c9 23 8c cd f6 a2 45 07 e6 93 40 7c c8 02 |...#....E...@|..| -00000030 6e 24 f7 4e 06 9a ad ba de cd 10 17 03 03 00 19 |n$.N............| -00000040 52 e0 85 b3 46 25 7d fd 8f d0 4c ca df fe 2f a6 |R...F%}...L.../.| -00000050 14 d6 08 82 70 0f 93 78 bf 16 03 03 00 14 ff 68 |....p..x.......h| -00000060 f1 6d 8d b4 5f 74 19 6b e3 1e 6e ee 9d ee c2 34 |.m.._t.k..n....4| -00000070 0f 44 |.D| +00000000 14 03 03 00 11 03 06 35 7b 36 e4 b2 df 80 68 80 |.......5{6....h.| +00000010 df e3 d6 3c bb b7 16 03 03 00 20 08 9b ed a0 ae |...<...... .....| +00000020 e7 f9 42 9a f5 10 21 7a 38 dc a2 6d 27 a8 bc 00 |..B...!z8..m'...| +00000030 58 85 6a 85 04 b9 3b 3f ca 26 68 17 03 03 00 19 |X.j...;?.&h.....| +00000040 d5 24 ff 37 e7 a2 5d eb f2 45 ca 73 60 3b c3 8c |.$.7..]..E.s`;..| +00000050 87 10 d8 31 c3 a2 e5 3d 0f 16 03 03 00 14 31 8a |...1...=......1.| +00000060 c9 a6 3a 3e 36 e0 4f d6 f1 f5 67 70 34 8a ab dc |..:>6.O...gp4...| +00000070 82 e2 |..| >>> Flow 11 (client to server) -00000000 15 03 03 00 12 81 0d 63 a5 11 7a 03 ab 66 f6 c8 |.......c..z..f..| -00000010 15 f3 d9 23 fa 67 6b 15 03 03 00 12 71 0e 3b 52 |...#.gk.....q.;R| -00000020 1a 05 39 4c 8c 76 c3 a9 00 35 bd 66 80 a1 |..9L.v...5.f..| +00000000 15 03 03 00 12 be b2 bb cf 44 91 f3 b9 71 00 af |.........D...q..| +00000010 9e b6 f5 07 64 36 7b 15 03 03 00 12 ae b2 33 16 |....d6{.......3.| +00000020 de f1 45 31 a5 fd 07 97 6e 57 f8 22 cc b2 |..E1....nW."..| diff --git a/src/crypto/tls/testdata/Client-TLSv12-RenegotiationRejected b/src/crypto/tls/testdata/Client-TLSv12-RenegotiationRejected index 0be2e72bf3cfeb..c907c7cb2ea9cb 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-RenegotiationRejected +++ b/src/crypto/tls/testdata/Client-TLSv12-RenegotiationRejected @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 21 15 e1 02 00 |....]...Y..!....| -00000010 c6 0c e5 3f c9 6f a5 59 38 7e 13 81 1b 26 50 46 |...?.o.Y8~...&PF| -00000020 bf 2b 95 0c eb a8 bc 72 97 bc 26 20 ad 6b 84 ac |.+.....r..& .k..| -00000030 8f 62 26 0b 4b d8 bf 1b 7d a5 27 3b 3e 45 a5 8f |.b&.K...}.';>E..| -00000040 37 fb b2 25 2c d3 82 69 ed d7 c9 f0 cc a8 00 00 |7..%,..i........| +00000000 16 03 03 00 5d 02 00 00 59 03 03 ea a3 8e 85 79 |....]...Y......y| +00000010 9f 41 1f 7e 05 76 70 3d bd f4 bb f4 dd 49 a6 fd |.A.~.vp=.....I..| +00000020 11 83 a9 70 89 86 87 36 2f 09 b9 20 1a 00 de fc |...p...6/.. ....| +00000030 6b e5 93 4e da 98 0b e1 a8 93 8d fc 4b c3 48 6b |k..N........K.Hk| +00000040 15 6d de fd e3 0f 36 67 61 1c 91 8e cc a8 00 00 |.m....6ga.......| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -61,36 +62,36 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 b2 3a 28 |............ .:(| -000002d0 2e 2c 66 f1 9b 44 58 9a 92 03 42 81 f6 1c f4 67 |.,f..DX...B....g| -000002e0 50 b5 92 f3 06 1b 1b 23 93 99 12 c5 6b 08 04 00 |P......#....k...| -000002f0 80 44 05 ae 8d ed c6 82 3a 2a 19 05 9d 70 f9 ba |.D......:*...p..| -00000300 45 66 cb 45 75 9c da 92 cc bb 4a a5 ae 41 8c f8 |Ef.Eu.....J..A..| -00000310 9b e7 06 73 88 9f f9 7d 95 ce 74 a6 05 e9 38 a5 |...s...}..t...8.| -00000320 26 4b b2 26 31 5c e9 f9 a8 f8 6e 6b 05 e4 39 b5 |&K.&1\....nk..9.| -00000330 fe b1 b1 cd 40 23 e9 68 f3 9c ed 91 71 d3 0b c5 |....@#.h....q...| -00000340 91 c5 b2 91 69 a9 4b 2c a7 0c 24 d6 a9 e7 74 89 |....i.K,..$...t.| -00000350 9e ce 8b 00 72 9a c9 86 8b ca 8d 39 01 a0 71 3e |....r......9..q>| -00000360 46 ff a6 7d c6 da 0a af f2 84 c4 01 1c 94 47 22 |F..}..........G"| -00000370 71 16 03 03 00 04 0e 00 00 00 |q.........| +000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 6a e1 91 |............ j..| +000002d0 5d 5d fe 77 50 be a5 52 f1 12 43 30 34 aa 46 87 |]].wP..R..C04.F.| +000002e0 f9 6a 80 37 eb 46 fb 92 d6 ce 18 c4 0b 08 04 00 |.j.7.F..........| +000002f0 80 46 10 ab ff 45 fe c5 88 5a da d0 07 b5 a1 30 |.F...E...Z.....0| +00000300 03 d8 b0 5f 88 37 45 75 ef b7 96 5c c9 1f 47 14 |..._.7Eu...\..G.| +00000310 24 28 de cf 1f c9 c6 a4 0c 41 1b 51 70 7a e6 e8 |$(.......A.Qpz..| +00000320 88 1b 35 ce 63 e7 4f 2b 02 c9 35 0a 56 b2 2a 59 |..5.c.O+..5.V.*Y| +00000330 e9 02 53 11 82 f2 f6 18 06 3a 3e 2e 8e 21 78 9d |..S......:>..!x.| +00000340 41 43 e2 ed 49 ce 87 cd 93 b7 13 6c 35 6e 4e 95 |AC..I......l5nN.| +00000350 68 1b 4b 75 de fa ed 48 53 56 54 40 5f 3f 36 cb |h.Ku...HSVT@_?6.| +00000360 6b 57 de 3c fc 51 f4 fb 9d 8d 55 2e e9 ce fc 79 |kW.<.Q....U....y| +00000370 08 16 03 03 00 04 0e 00 00 00 |..........| >>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| -00000030 16 03 03 00 20 80 26 44 1c a7 83 b7 12 29 9a 44 |.... .&D.....).D| -00000040 7b c1 0b 96 53 c8 0e 9b dd 8d 29 4b 4b b8 75 d7 |{...S.....)KK.u.| -00000050 0a c7 86 c7 d9 |.....| +00000030 16 03 03 00 20 be b4 f9 49 91 3c c8 f8 7c da ef |.... ...I.<..|..| +00000040 06 bd d5 9e d3 b4 74 f6 22 2e b8 5c b6 e2 2d 01 |......t."..\..-.| +00000050 cb d5 3f 94 7f |..?..| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 20 2e d7 9e b0 66 |.......... ....f| -00000010 f1 39 84 a9 d9 93 17 bd 94 5a 31 49 bb eb f0 a0 |.9.......Z1I....| -00000020 7b af d0 3b ae 1a 5d f6 46 31 36 |{..;..].F16| +00000000 14 03 03 00 01 01 16 03 03 00 20 21 a2 e6 72 21 |.......... !..r!| +00000010 6d 45 36 87 6d 3f c9 70 89 1f b3 0a cc c2 5b 18 |mE6.m?.p......[.| +00000020 27 37 d8 1f 4e 55 f9 9d 07 96 2f |'7..NU..../| >>> Flow 5 (client to server) -00000000 17 03 03 00 16 cd 39 a7 64 3b 6a de 14 e0 26 ea |......9.d;j...&.| -00000010 66 b2 73 b1 8e b0 e3 a9 94 62 4f |f.s......bO| +00000000 17 03 03 00 16 81 33 71 a4 ec a6 67 b2 17 ba 80 |......3q...g....| +00000010 27 80 a1 62 fb 1e ab 3f cd b9 fe |'..b...?...| >>> Flow 6 (server to client) -00000000 16 03 03 00 14 c5 d7 88 58 81 44 1f 8d e4 c2 19 |........X.D.....| -00000010 15 3b 5c 43 76 82 fe 03 e1 |.;\Cv....| +00000000 16 03 03 00 14 b3 2b 93 6f bc d1 11 45 fa 9a d3 |......+.o...E...| +00000010 f3 82 75 6c ce 40 38 21 5e |..ul.@8!^| >>> Flow 7 (client to server) -00000000 15 03 03 00 12 d4 b1 0d fa 41 25 ab d4 e1 b7 74 |.........A%....t| -00000010 27 5c 8b c6 0d 49 8e 15 03 03 00 12 65 3d 4e 30 |'\...I......e=N0| -00000020 48 43 8f f6 8b 89 45 af f4 7a e4 eb ad a1 |HC....E..z....| +00000000 15 03 03 00 12 48 45 72 03 40 44 98 c6 db 40 d4 |.....HEr.@D...@.| +00000010 a6 9b 67 5d 8b c1 a9 15 03 03 00 12 21 d2 4b 7d |..g]........!.K}| +00000020 51 c7 3c 53 0b d9 67 90 aa e6 42 e2 8c 83 |Q.>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,21 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 01 ca 02 00 01 c6 03 03 da b0 57 a0 ee |.............W..| -00000010 bf 74 92 06 39 a6 9a 9b 80 71 7e 57 63 ab 31 74 |.t..9....q~Wc.1t| -00000020 d3 22 c1 e3 a6 c6 cf 6f 47 57 49 20 86 52 fb 0a |.".....oGWI .R..| -00000030 fd aa 10 47 00 9a 5d ad 4c d8 90 f1 cf 13 10 73 |...G..].L......s| -00000040 31 f1 df 22 f3 42 58 b0 10 25 43 eb cc a8 00 01 |1..".BX..%C.....| +00000000 16 03 03 01 ca 02 00 01 c6 03 03 28 44 55 f3 f5 |...........(DU..| +00000010 c1 63 ac 0b fd 1c 29 d3 90 a5 5c 96 2a d9 13 20 |.c....)...\.*.. | +00000020 fe 96 11 3f 30 0a e1 a9 8e d9 e6 20 ca 40 de 01 |...?0...... .@..| +00000030 4f 00 88 88 41 0f df af 85 0e 0e 22 9d a7 46 e8 |O...A......"..F.| +00000040 df 9c 8e 8c 47 5b 94 91 94 06 f4 14 cc a8 00 01 |....G[..........| 00000050 7e 00 12 01 69 01 67 00 75 00 a4 b9 09 90 b4 18 |~...i.g.u.......| 00000060 58 14 87 bb 13 a2 cc 67 70 0a 3c 35 98 04 f9 1b |X......gp.<5....| 00000070 df b8 e3 77 cd 0e c8 0d dc 10 00 00 01 47 97 99 |...w.........G..| @@ -84,31 +85,31 @@ 00000400 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 4c 72 2b |Cw.......@.a.Lr+| 00000410 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 bd 42 d4 |...F..M...>...B.| 00000420 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 16 03 03 |..=.`.\!.;......| -00000430 00 ac 0c 00 00 a8 03 00 1d 20 0e e6 72 88 cf d1 |......... ..r...| -00000440 05 c7 8d 03 64 b9 db d8 c3 5e 36 80 67 67 36 94 |....d....^6.gg6.| -00000450 5b b7 92 f9 77 1f 6f b2 44 7f 08 04 00 80 1d 3b |[...w.o.D......;| -00000460 70 d5 72 87 2f 58 20 da 29 e0 f8 e8 7c 8d e5 c7 |p.r./X .)...|...| -00000470 bf da 3c 10 f8 b0 3e 38 7b 06 0b 8e 8f d7 40 b6 |..<...>8{.....@.| -00000480 47 9f 1c 64 6e 80 ab 9d 51 92 8c 60 a2 88 c4 43 |G..dn...Q..`...C| -00000490 7d dc 29 64 45 83 a7 9c 2a 39 e9 bb 2a d9 f5 96 |}.)dE...*9..*...| -000004a0 4c fb fd 0d cc 0f 9b 48 f5 ee af 8f 7a 1c 39 3e |L......H....z.9>| -000004b0 cf 23 15 c6 ab c3 f2 29 fd 6c 4f 17 6b d4 be 1a |.#.....).lO.k...| -000004c0 23 6e 74 81 e5 e5 e8 c0 de 9b b0 ed d2 32 bc 2a |#nt..........2.*| -000004d0 b1 30 b2 eb ac 9e 23 f8 be ea 31 9a b4 55 16 03 |.0....#...1..U..| +00000430 00 ac 0c 00 00 a8 03 00 1d 20 26 24 4e d8 a6 cc |......... &$N...| +00000440 2f c9 0e 68 d8 20 e3 97 5b c1 08 72 1e 79 46 d9 |/..h. ..[..r.yF.| +00000450 a6 00 9c e9 f8 21 9b cd 29 17 08 04 00 80 aa 6d |.....!..)......m| +00000460 36 bb 06 e6 11 66 82 44 87 5d 81 53 e2 9a 95 11 |6....f.D.].S....| +00000470 54 a3 cc a7 c9 9c 19 f6 da 98 4f 91 a6 dd 9e 10 |T.........O.....| +00000480 14 83 04 55 2d 6c 9a af 26 7c 5d f0 aa ca 69 83 |...U-l..&|]...i.| +00000490 af fe a5 cb a4 1a d4 bd 86 91 52 11 03 4d 9a ca |..........R..M..| +000004a0 37 fd 01 48 e8 5d a8 ea ad a2 a5 08 cb ce 5f 52 |7..H.]........_R| +000004b0 92 30 83 de 77 2a 06 c2 f2 53 4d 47 40 b9 2f 61 |.0..w*...SMG@./a| +000004c0 9c 41 c4 05 45 42 5e 42 d5 5e 30 95 30 4e a1 77 |.A..EB^B.^0.0N.w| +000004d0 79 b5 50 5c df d6 e7 74 42 d8 2b 66 02 fa 16 03 |y.P\...tB.+f....| 000004e0 03 00 04 0e 00 00 00 |.......| >>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| -00000030 16 03 03 00 20 a1 58 d5 1a a3 9f 85 4e b6 c8 b7 |.... .X.....N...| -00000040 cb c1 b0 cc 0a 80 59 1d 81 41 cc 4d 0b d3 bd 38 |......Y..A.M...8| -00000050 48 69 01 23 96 |Hi.#.| +00000030 16 03 03 00 20 93 c8 47 ed fd 9b 41 69 97 58 97 |.... ..G...Ai.X.| +00000040 cc 31 9c 93 a6 77 41 36 7a 90 f0 73 13 4a 7d 85 |.1...wA6z..s.J}.| +00000050 12 20 7d 91 fa |. }..| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 20 83 ae fc 0d dd |.......... .....| -00000010 a1 3a 55 b0 2e 5e 21 9c 57 f3 1b 94 80 6c 0e bb |.:U..^!.W....l..| -00000020 78 ae f4 6c 20 d5 7e 23 11 1e 7d |x..l .~#..}| +00000000 14 03 03 00 01 01 16 03 03 00 20 16 00 35 d2 e0 |.......... ..5..| +00000010 ad f5 4c 10 98 fb 4f c7 81 1f 05 4b d6 7d 9c ac |..L...O....K.}..| +00000020 50 94 84 c8 35 80 ec 54 fc f3 ee |P...5..T...| >>> Flow 5 (client to server) -00000000 17 03 03 00 16 d3 16 a3 f0 93 49 44 d4 82 60 35 |..........ID..`5| -00000010 e3 63 08 c0 4e 66 99 37 31 90 6c 15 03 03 00 12 |.c..Nf.71.l.....| -00000020 3c ea 2c 19 34 7b cc 6c 7a a9 15 31 6f 9c 3b b0 |<.,.4{.lz..1o.;.| -00000030 b2 1f |..| +00000000 17 03 03 00 16 83 e6 0f 3a ad c2 28 6c 82 ee 4a |........:..(l..J| +00000010 38 47 cc 86 75 70 e2 94 77 e6 6e 15 03 03 00 12 |8G..up..w.n.....| +00000020 13 c0 a0 4f 02 5e 6f 33 f6 ae d3 52 7d 72 72 a0 |...O.^o3...R}rr.| +00000030 1e 8d |..| diff --git a/src/crypto/tls/testdata/Client-TLSv12-X25519-ECDHE b/src/crypto/tls/testdata/Client-TLSv12-X25519-ECDHE index c851eb1a1f2a5e..6f59dc6636b023 100644 --- a/src/crypto/tls/testdata/Client-TLSv12-X25519-ECDHE +++ b/src/crypto/tls/testdata/Client-TLSv12-X25519-ECDHE @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 f8 01 00 00 f4 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 12 01 00 01 0e 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,20 +7,22 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 79 00 0b 00 02 01 00 ff 01 00 01 00 00 |...y............| +00000080 01 00 00 93 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| -000000a0 00 0a 00 04 00 02 00 1d 00 0d 00 1a 00 18 08 04 |................| +000000a0 00 0a 00 04 00 02 00 1d 00 0d 00 16 00 14 08 04 |................| 000000b0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| -000000c0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| -000000d0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| -000000e0 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| -000000f0 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| +000000c0 06 03 00 32 00 1a 00 18 08 04 04 03 08 07 08 05 |...2............| +000000d0 08 06 04 01 05 01 06 01 05 03 06 03 02 01 02 03 |................| +000000e0 00 2b 00 09 08 03 04 03 03 03 02 03 01 00 33 00 |.+............3.| +000000f0 26 00 24 00 1d 00 20 2f e5 7d a3 47 cd 62 43 15 |&.$... /.}.G.bC.| +00000100 28 da ac 5f bb 29 07 30 ff f6 84 af c4 cf c2 ed |(.._.).0........| +00000110 90 99 5f 58 cb 3b 74 |.._X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 5d 02 00 00 59 03 03 e0 da ea 5e 09 |....]...Y.....^.| -00000010 bb 30 4c 10 db 85 36 44 38 9e a4 83 74 5f 2f 94 |.0L...6D8...t_/.| -00000020 a1 b4 45 61 73 2f 88 b3 ac ad e4 20 45 55 20 90 |..Eas/..... EU .| -00000030 b0 b0 d4 70 e6 10 13 5c f4 49 0e 9c 2a 52 5c 84 |...p...\.I..*R\.| -00000040 b2 55 1a ea e3 4b 5a 9f ac e9 7b 37 c0 2f 00 00 |.U...KZ...{7./..| +00000000 16 03 03 00 5d 02 00 00 59 03 03 64 5a cc e0 63 |....]...Y..dZ..c| +00000010 02 82 64 75 5e 2c b3 5f 23 c6 98 c8 95 07 40 00 |..du^,._#.....@.| +00000020 3e ef fd f6 00 af 81 0c 3a 47 a0 20 a1 fc 21 83 |>.......:G. ..!.| +00000030 cf 2b 34 5e b7 7b bd ec bb 8f 26 6a f2 26 7f 8f |.+4^.{....&j.&..| +00000040 51 13 6f ba 0d 1a f5 b7 1c 9d 6d 91 c0 2f 00 00 |Q.o.......m../..| 00000050 11 ff 01 00 01 00 00 0b 00 04 03 00 01 02 00 17 |................| 00000060 00 00 16 03 03 02 59 0b 00 02 55 00 02 52 00 02 |......Y...U..R..| 00000070 4f 30 82 02 4b 30 82 01 b4 a0 03 02 01 02 02 09 |O0..K0..........| @@ -60,33 +62,33 @@ 00000290 73 bb b3 43 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 |s..Cw.......@.a.| 000002a0 4c 72 2b 9d ae db 46 06 06 4d f4 c1 b3 3e c0 d1 |Lr+...F..M...>..| 000002b0 bd 42 d4 db fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 |.B...=.`.\!.;...| -000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 ca 07 6e |............ ..n| -000002d0 67 12 8e f7 b4 55 c0 d4 88 73 35 f0 f1 ae c4 34 |g....U...s5....4| -000002e0 59 dc 05 df 75 96 54 f1 4f 7f 79 93 25 08 04 00 |Y...u.T.O.y.%...| -000002f0 80 d2 1d 65 15 23 74 e4 57 80 58 62 b5 6d ab 06 |...e.#t.W.Xb.m..| -00000300 11 4c e0 a0 7b 7e b5 78 87 47 71 30 c3 d8 86 b2 |.L..{~.x.Gq0....| -00000310 af 34 24 95 9c 2f 74 bb 6e c8 2a 8e 81 f8 fb d7 |.4$../t.n.*.....| -00000320 06 06 dc e8 95 01 65 fe 5d cb 9e 7f 36 9b 29 40 |......e.]...6.)@| -00000330 09 81 1b df 24 66 13 94 51 2a 48 31 e3 cb 3a 06 |....$f..Q*H1..:.| -00000340 c9 9c f4 36 b8 c0 de c1 3e 5e 79 95 f8 11 4b d8 |...6....>^y...K.| -00000350 0a b1 6c ef f4 ce 89 ba df ad 24 3d 7c 48 aa 70 |..l.......$=|H.p| -00000360 d8 0e 52 a8 12 1c 6b eb 5b c9 c3 a3 fa f0 60 d4 |..R...k.[.....`.| -00000370 54 16 03 03 00 04 0e 00 00 00 |T.........| +000002c0 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 15 d1 98 |............ ...| +000002d0 53 f4 98 b6 58 a5 cd 89 bb 32 21 83 4a e5 ad ba |S...X....2!.J...| +000002e0 1f ba db d0 4c c5 a3 89 0e 96 9f 59 60 08 04 00 |....L......Y`...| +000002f0 80 4e 14 a0 a7 33 e8 47 d5 7d 01 26 19 ff 1d 3c |.N...3.G.}.&...<| +00000300 2a f9 77 58 84 c0 07 b7 dc cc 82 85 a6 18 e9 33 |*.wX...........3| +00000310 70 15 04 75 bb 59 59 9c fa c0 cd b4 b7 f8 a3 0e |p..u.YY.........| +00000320 21 cd ac 7d 18 d2 34 e5 a9 c7 a1 5d 86 f3 74 2b |!..}..4....]..t+| +00000330 9b b0 21 5f cb 91 7f a0 d8 78 f0 7c 56 e3 69 ec |..!_.....x.|V.i.| +00000340 43 55 80 3f 02 d7 e0 10 52 15 4d 1b bc 2c 7f b7 |CU.?....R.M..,..| +00000350 86 cd 07 5d b9 f1 21 7e 18 da fc f8 45 45 cf 39 |...]..!~....EE.9| +00000360 62 3e c8 f1 6c 75 f4 fb fe f8 23 89 6a 0d b5 2e |b>..lu....#.j...| +00000370 4d 16 03 03 00 04 0e 00 00 00 |M.........| >>> Flow 3 (client to server) 00000000 16 03 03 00 25 10 00 00 21 20 2f e5 7d a3 47 cd |....%...! /.}.G.| 00000010 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 |bC.(.._.).0.....| 00000020 cf c2 ed 90 99 5f 58 cb 3b 74 14 03 03 00 01 01 |....._X.;t......| -00000030 16 03 03 00 28 00 00 00 00 00 00 00 00 21 27 7f |....(........!'.| -00000040 32 c6 19 c4 a9 13 bf 5e 4b 53 5f c3 47 64 bb f8 |2......^KS_.Gd..| -00000050 21 d5 10 12 12 60 5e d8 e8 cf 1e fe 18 |!....`^......| +00000030 16 03 03 00 28 00 00 00 00 00 00 00 00 e3 a2 01 |....(...........| +00000040 72 d5 55 57 f7 1a da 10 9a 04 7d 33 f4 18 b1 9a |r.UW......}3....| +00000050 09 38 45 dd 74 13 21 70 52 9c cc c2 3f |.8E.t.!pR...?| >>> Flow 4 (server to client) -00000000 14 03 03 00 01 01 16 03 03 00 28 af e6 ee d0 75 |..........(....u| -00000010 df 97 22 b0 ff 46 8e 51 bb 23 70 8d 4a 22 01 b6 |.."..F.Q.#p.J"..| -00000020 13 34 68 fd 69 fe 0a 0f df 0e e6 41 48 49 8c e7 |.4h.i......AHI..| -00000030 d3 21 13 |.!.| +00000000 14 03 03 00 01 01 16 03 03 00 28 86 a1 0b 2b ef |..........(...+.| +00000010 13 8d 20 82 ad 17 92 64 0b 44 21 0f f0 dd 20 43 |.. ....d.D!... C| +00000020 4c db 92 3c 25 3a 12 ca 27 56 86 16 ca 9f ba 46 |L..<%:..'V.....F| +00000030 dc 96 86 |...| >>> Flow 5 (client to server) -00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 10 16 77 |...............w| -00000010 99 da 71 ef 78 16 d7 cd 2c 7b 71 ba 8a 74 c1 65 |..q.x...,{q..t.e| -00000020 f2 f6 be 15 03 03 00 1a 00 00 00 00 00 00 00 02 |................| -00000030 72 d4 4a f6 6d 63 4c ad 02 46 36 18 61 47 61 7d |r.J.mcL..F6.aGa}| -00000040 50 08 |P.| +00000000 17 03 03 00 1e 00 00 00 00 00 00 00 01 20 7c 62 |............. |b| +00000010 05 52 7a 91 a7 13 96 96 29 11 1b 1a fe fa ad 7f |.Rz.....).......| +00000020 d2 3a 26 15 03 03 00 1a 00 00 00 00 00 00 00 02 |.:&.............| +00000030 c7 2b 60 1b 9c 6b cb 22 08 f1 e3 c0 44 ab 70 21 |.+`..k."....D.p!| +00000040 68 48 |hH| diff --git a/src/crypto/tls/testdata/Client-TLSv13-AES128-SHA256 b/src/crypto/tls/testdata/Client-TLSv13-AES128-SHA256 index 6c7c3d47119ff6..3673b8da39367f 100644 --- a/src/crypto/tls/testdata/Client-TLSv13-AES128-SHA256 +++ b/src/crypto/tls/testdata/Client-TLSv13-AES128-SHA256 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,85 +7,86 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 7a 02 00 00 76 03 03 63 fa e1 78 d5 |....z...v..c..x.| -00000010 53 77 64 44 97 45 79 94 e7 79 dd da 70 81 46 66 |SwdD.Ey..y..p.Ff| -00000020 ac 7c 77 a2 f9 29 ca 48 e6 53 56 20 00 00 00 00 |.|w..).H.SV ....| +00000000 16 03 03 00 7a 02 00 00 76 03 03 07 aa 6b aa b6 |....z...v....k..| +00000010 ef a5 5c 62 d7 05 bb f8 9d ea 03 34 13 bf 00 1f |..\b.......4....| +00000020 e6 95 7b ef a5 2b ad e8 20 2f 21 20 00 00 00 00 |..{..+.. /! ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000040 00 00 00 00 00 00 00 00 00 00 00 00 13 01 00 00 |................| -00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 36 |..+.....3.$... 6| -00000060 e2 c3 a0 14 f4 9c 50 9c 08 03 5e c2 b8 4b 66 85 |......P...^..Kf.| -00000070 71 fb bb 20 6e 97 3a f6 74 be 02 31 29 8e 18 14 |q.. n.:.t..1)...| -00000080 03 03 00 01 01 17 03 03 00 17 00 79 2a 2d 44 f4 |...........y*-D.| -00000090 12 2d 7c d6 8c 7a 27 76 a5 3a 6b c1 2d 40 8f 20 |.-|..z'v.:k.-@. | -000000a0 8b 17 03 03 02 6d 6e 16 f8 0f d2 67 3b e8 1a 95 |.....mn....g;...| -000000b0 2a ce 16 7b 95 21 7e 03 65 bf 01 ce e9 43 83 f6 |*..{.!~.e....C..| -000000c0 0e a9 d2 f2 d1 a3 2f d4 55 d2 27 83 44 3d f9 72 |....../.U.'.D=.r| -000000d0 00 8d 41 44 eb 93 ca 68 09 d1 bd f3 1a cc a0 53 |..AD...h.......S| -000000e0 a7 18 6e b7 27 d0 f5 e1 08 d8 9f 7b 37 bc 8f b9 |..n.'......{7...| -000000f0 84 6d e0 20 cf 99 bf c1 67 c8 00 74 4c 76 3e 2d |.m. ....g..tLv>-| -00000100 63 48 11 f1 75 30 ab a5 5b a2 cb 13 e9 e0 65 36 |cH..u0..[.....e6| -00000110 86 ce 8b 64 f6 dc e1 b7 0d 16 96 d1 ec 4b 93 d5 |...d.........K..| -00000120 6c 10 0d b7 d8 a2 45 b2 f0 e0 aa ab ab 11 8d d0 |l.....E.........| -00000130 2c fc 20 35 99 1b d6 3b 72 6d 88 03 80 5c 59 d4 |,. 5...;rm...\Y.| -00000140 04 18 ee 7a ed fe fb 52 c1 3c 93 95 e2 00 23 0d |...z...R.<....#.| -00000150 cc c1 4e 0a fb 55 33 93 9f c8 31 33 29 f5 12 3e |..N..U3...13)..>| -00000160 39 62 62 27 05 cf d2 be bd 7d 42 51 97 4f 71 95 |9bb'.....}BQ.Oq.| -00000170 10 f8 eb 0f d8 d1 69 aa 22 13 a8 fe 24 f6 87 a1 |......i."...$...| -00000180 9d ca cb 8f 99 39 2b fb 99 d9 18 8a fa fa 25 20 |.....9+.......% | -00000190 df 6a b3 d1 ac ce 56 8b 76 5d a9 46 1c 0b c7 57 |.j....V.v].F...W| -000001a0 90 02 10 55 85 49 48 89 d8 d6 4b 68 29 cb 28 66 |...U.IH...Kh).(f| -000001b0 1c d1 97 e6 a0 10 90 63 83 96 33 ee 5a 5a 5b 66 |.......c..3.ZZ[f| -000001c0 ca b3 0d be 6d 70 c5 de fd 78 c8 f8 e1 c5 78 ef |....mp...x....x.| -000001d0 07 2e 60 06 28 86 ed e6 bd 7b 37 33 0e 06 5e 06 |..`.(....{73..^.| -000001e0 fb 44 2b 96 16 fe 09 30 dc 03 53 bc c9 f9 6f 1e |.D+....0..S...o.| -000001f0 4b 79 a9 ac fe 20 3f 12 58 48 0c 40 a5 2f f2 6a |Ky... ?.XH.@./.j| -00000200 ea 44 69 93 54 4f 4d 6c 98 85 b1 c5 6f 50 83 b8 |.Di.TOMl....oP..| -00000210 8a 20 e3 b9 da b7 7d e4 ec 33 28 96 29 d7 dc a3 |. ....}..3(.)...| -00000220 d0 54 76 cd a6 99 17 e8 0f 14 d2 7c 4b 68 0a 80 |.Tv........|Kh..| -00000230 4f 0b 2b b5 16 68 dc 25 42 b5 0b f7 13 6a c5 81 |O.+..h.%B....j..| -00000240 c0 18 bc fb 40 86 cc 99 2e 70 2b 9a 72 eb be 57 |....@....p+.r..W| -00000250 a0 43 92 83 3b 7e c8 02 43 ab 17 7a 73 87 1a a2 |.C..;~..C..zs...| -00000260 1c 01 db b4 31 b1 d9 44 f1 c0 d9 de 27 bd 4c 8b |....1..D....'.L.| -00000270 1c f5 3d db d6 aa 98 7d 43 fe dd 64 85 8b c1 88 |..=....}C..d....| -00000280 a9 1e f1 04 0b 5d 3b c2 fd 3f b2 27 a7 28 d1 82 |.....];..?.'.(..| -00000290 13 fb 85 46 77 78 50 45 1b 4b c0 75 f2 32 8a 6d |...FwxPE.K.u.2.m| -000002a0 52 37 2e b1 80 e9 81 3b 60 c8 88 78 11 3f 0f 96 |R7.....;`..x.?..| -000002b0 d1 4b 19 49 91 b5 70 44 2d 62 a1 04 8f 98 89 ab |.K.I..pD-b......| -000002c0 75 1f 8a 97 d5 28 dc 88 20 92 7f f2 90 16 1b de |u....(.. .......| -000002d0 54 7c 5f 05 8f c9 70 a1 f5 8c 99 09 43 ab 09 73 |T|_...p.....C..s| -000002e0 dd 5e 62 c3 9f c8 e1 fa 9e 98 65 18 b5 5e 4e f9 |.^b.......e..^N.| -000002f0 85 a4 cc dc b8 12 1b eb 0a bc bc bd 1c 2f 51 0e |............./Q.| -00000300 8f 55 f1 10 22 06 ca cb f2 97 54 63 1c 98 36 9f |.U..".....Tc..6.| -00000310 1d 66 b0 17 03 03 00 99 f0 14 4d 69 84 b3 ca d4 |.f........Mi....| -00000320 96 9c e2 3c a4 55 f4 b6 01 1f 1e 1f 77 98 c4 84 |...<.U......w...| -00000330 5e ce 4c 0b bf ba 39 6b 36 a6 15 9c d1 90 87 d1 |^.L...9k6.......| -00000340 13 39 1c dd 86 0f a0 06 f7 fc f7 c9 10 4e 46 0a |.9...........NF.| -00000350 03 12 a0 bf 3e f3 7d e6 1b b7 ce ec 16 84 52 8e |....>.}.......R.| -00000360 bc 9c 9a 4f 2f a7 d9 aa f3 d6 be 0a f8 39 2d 18 |...O/........9-.| -00000370 28 9e 35 6e 25 25 46 9b 49 61 a6 ee 6a 87 d2 5f |(.5n%%F.Ia..j.._| -00000380 54 e6 fb 96 a3 e9 8a 05 c2 fb 71 0b d7 13 2e 19 |T.........q.....| -00000390 e5 5c a2 d0 f4 b3 34 95 e1 1d 24 ef 33 a2 0a d5 |.\....4...$.3...| -000003a0 65 b2 5f 46 73 45 33 69 d6 c3 f6 49 20 e2 00 bd |e._FsE3i...I ...| -000003b0 16 17 03 03 00 35 89 2f 34 08 44 35 6a 52 1a fd |.....5./4.D5jR..| -000003c0 e4 be 2b 0e 0a dd 2d c1 e7 e3 20 7a 0a 3e 87 1d |..+...-... z.>..| -000003d0 2b 64 87 c5 11 54 46 1a 5c 89 e2 97 3f 83 b2 b4 |+d...TF.\...?...| -000003e0 04 ed d2 c4 1d 3d 37 1f e1 62 f2 |.....=7..b.| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 53 |..+.....3.$... S| +00000060 3c 5e 7d 61 39 39 6c f7 ea 14 d3 f0 40 23 30 8b |<^}a99l.....@#0.| +00000070 7e 1f 41 86 f8 2e 12 0a f5 67 13 28 46 6f 14 14 |~.A......g.(Fo..| +00000080 03 03 00 01 01 17 03 03 00 17 a8 64 4a 9d f1 a0 |...........dJ...| +00000090 3c 1b e3 20 a2 66 80 7f 13 b5 a9 f0 8f ee 5b 4b |<.. .f........[K| +000000a0 af 17 03 03 02 6d 26 d2 8f df b4 d5 d3 9a c8 92 |.....m&.........| +000000b0 4c 32 63 80 6b 5f 99 11 0f 4b be 14 77 55 86 d1 |L2c.k_...K..wU..| +000000c0 27 94 18 50 27 1c 84 9c 38 03 fb 11 d3 5a ea 64 |'..P'...8....Z.d| +000000d0 d1 15 47 1d fc cc 6e bf 5e 4d 45 74 f2 ca 44 77 |..G...n.^MEt..Dw| +000000e0 32 6d 02 3a 9f 44 aa 5c 5d 62 64 ee 44 0d 45 05 |2m.:.D.\]bd.D.E.| +000000f0 de 40 d0 83 c9 b3 21 39 a0 8f 5b 5a e1 96 2f 0b |.@....!9..[Z../.| +00000100 4b 12 da a7 dc 7b 2a 73 14 7f cc 93 fd a8 47 1c |K....{*s......G.| +00000110 f7 88 76 5d 30 29 af 65 ac 52 25 04 13 be 96 92 |..v]0).e.R%.....| +00000120 bb e4 86 a0 b7 c8 67 63 ef cb 1f 78 7e bf 3c 5f |......gc...x~.<_| +00000130 7a a6 89 4a 94 f3 2e 01 8b b9 81 21 57 00 5e 26 |z..J.......!W.^&| +00000140 22 9b eb e0 73 87 cd 77 46 d4 c8 1d 0a 4f d8 83 |"...s..wF....O..| +00000150 0b 0a 43 3f 37 53 e2 0a 4b bb ef e8 8d 2a dc 51 |..C?7S..K....*.Q| +00000160 b0 b9 90 7e 1a c5 31 e0 1b 0f 0c 7e 96 7b aa 49 |...~..1....~.{.I| +00000170 ad 84 b0 0a 75 d6 37 58 bd 7f f6 ac 50 c5 d7 93 |....u.7X....P...| +00000180 70 ce b1 cd 00 7c 79 1a e5 ec 04 7b 12 8f fb 80 |p....|y....{....| +00000190 54 d0 2b a0 9e 83 0d 05 a7 f5 4b e8 b4 36 1c d0 |T.+.......K..6..| +000001a0 88 34 a6 71 51 d6 cf 59 c9 ad 03 d8 38 f2 15 f0 |.4.qQ..Y....8...| +000001b0 63 9d 34 a3 f4 bc de 8f ae 26 97 c0 98 8e 59 4e |c.4......&....YN| +000001c0 1f 74 a1 d4 2c 1a 18 20 ef e8 06 58 74 dd ed 40 |.t..,.. ...Xt..@| +000001d0 50 dd 8b a4 77 15 1b 9e 63 7e c7 11 63 1a a9 d5 |P...w...c~..c...| +000001e0 16 c2 8f f5 6c ce cd 03 e6 2d cc da 75 1a ce cb |....l....-..u...| +000001f0 ea 41 de dd 59 e5 68 90 98 69 76 a3 d0 d4 ed d1 |.A..Y.h..iv.....| +00000200 e9 9a cd b8 29 73 eb 9c e6 6f ee 8d 91 84 9e 2e |....)s...o......| +00000210 b9 23 2b 04 a0 f6 5f 0b 16 07 49 ae 6f 33 b0 ee |.#+..._...I.o3..| +00000220 be ff 75 52 da 7b 06 05 6c 8f 87 1f 48 2f fb 59 |..uR.{..l...H/.Y| +00000230 79 a6 99 cb a6 0b 73 fb 4c d0 cc eb ba 51 1a 6d |y.....s.L....Q.m| +00000240 9a 33 27 e3 f4 cf 16 bc c0 82 da 21 04 4a 7b e0 |.3'........!.J{.| +00000250 12 a5 5e de 22 d2 df b4 c4 c9 fa 11 97 7b 07 ea |..^."........{..| +00000260 11 c8 2c 55 b8 6d c2 64 6d fa e0 6f b8 5c 50 ae |..,U.m.dm..o.\P.| +00000270 a6 2e aa bf 2f 2a 74 2d 1c 11 c2 44 d7 28 b8 5e |..../*t-...D.(.^| +00000280 6a 2d bc 0d 2e a7 a2 b6 b6 ae 12 2c a2 c4 ba 5c |j-.........,...\| +00000290 0b 3f 17 2e 98 30 ce 6c 8e 88 3c f9 a8 e1 68 8d |.?...0.l..<...h.| +000002a0 52 c8 a1 b3 3f 12 e1 35 f0 eb ee 9f 0d bb 3c 90 |R...?..5......<.| +000002b0 2c e2 2c 95 2e f3 e2 f6 f2 f1 be b8 03 02 84 69 |,.,............i| +000002c0 56 8f 3b 1d 68 77 f1 52 eb 48 4f e8 c8 5b 0a df |V.;.hw.R.HO..[..| +000002d0 a1 74 1e e1 47 a2 9a 9a 83 6d c2 ea f3 1b fc 8f |.t..G....m......| +000002e0 b6 aa 18 7e 85 46 ff 33 0b 6e 1f 5b f1 70 c0 45 |...~.F.3.n.[.p.E| +000002f0 f6 bb a0 da 74 e0 9a 3c a7 e2 07 ef 0b e5 9a 5b |....t..<.......[| +00000300 6e 9b aa 6c a4 97 84 5d 77 f5 33 b8 e3 45 c0 00 |n..l...]w.3..E..| +00000310 21 c7 91 17 03 03 00 99 ee bf d2 9c 82 9f 68 b4 |!.............h.| +00000320 5e 15 09 1a 7a c1 f9 72 ff ca 0e d9 f1 9f 7f 8c |^...z..r........| +00000330 dc 4a 8d d3 37 7b 81 42 ef 9b 2b 5f 9a 4b 40 8e |.J..7{.B..+_.K@.| +00000340 6b 24 53 79 32 b0 d1 24 2a 6f c9 fb a8 82 c1 01 |k$Sy2..$*o......| +00000350 1e 0b 11 7f 85 c7 ee d4 29 12 08 99 8f 00 ef af |........).......| +00000360 2c 5d c5 b1 25 a3 a9 f0 f1 a3 2b f8 7f e6 31 1b |,]..%.....+...1.| +00000370 51 20 af 3f a7 4e 5c 3d 73 21 04 0f 2c 5c cd f3 |Q .?.N\=s!..,\..| +00000380 eb d1 01 ef b0 e3 64 f7 33 d3 05 82 9f a6 71 3b |......d.3.....q;| +00000390 b2 ab 08 61 57 ad 74 d1 e2 8f 2c c9 73 c3 e0 ac |...aW.t...,.s...| +000003a0 ed 18 4b 70 0d 9a bf ae b4 db 07 7d 25 3e 7f 3d |..Kp.......}%>.=| +000003b0 14 17 03 03 00 35 db be d2 61 e1 3f a3 0b fe 39 |.....5...a.?...9| +000003c0 b2 86 42 72 a6 ee 02 90 c2 d1 d1 5c 3d 61 c8 28 |..Br.......\=a.(| +000003d0 28 7c 01 40 26 f1 e1 f1 81 50 03 4c 6c 32 6d 8b |(|.@&....P.Ll2m.| +000003e0 dd 60 cf fc a9 3c fe e1 4d d0 d2 |.`...<..M..| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 00 35 07 8e f6 0b 41 |..........5....A| -00000010 be 74 45 99 5e 35 59 4f 2e 59 d4 37 77 ca 53 d4 |.tE.^5YO.Y.7w.S.| -00000020 fd 33 bf dc 65 85 ee ad c5 d6 94 7e 6e e8 02 cf |.3..e......~n...| -00000030 15 d2 04 cb e1 9f 5e 94 dd 5f 37 17 95 17 0c b7 |......^.._7.....| -00000040 17 03 03 00 17 1f aa e9 7d 34 dc 12 da 3b cb dd |........}4...;..| -00000050 1c d3 b0 f1 79 ad cc 66 06 64 69 21 17 03 03 00 |....y..f.di!....| -00000060 13 b7 c7 a2 01 48 27 3a 44 e6 c6 47 a9 19 5f d2 |.....H':D..G.._.| -00000070 f8 96 98 70 |...p| +00000000 14 03 03 00 01 01 17 03 03 00 35 bd 1e ca 07 84 |..........5.....| +00000010 c8 ba 87 6d d3 ed 53 20 79 76 34 af 30 e6 bf c5 |...m..S yv4.0...| +00000020 74 a0 d0 b3 13 17 a9 3c c7 b3 47 5b e7 b4 6d 18 |t......<..G[..m.| +00000030 e5 3f 12 5c ca c9 26 87 c4 96 c4 9b 45 23 7f f7 |.?.\..&.....E#..| +00000040 17 03 03 00 17 28 0b 72 a0 6e ac 55 ec c0 b7 21 |.....(.r.n.U...!| +00000050 3f 7a 16 98 40 76 7d 3c 95 24 98 3b 17 03 03 00 |?z..@v}<.$.;....| +00000060 13 8e 23 e0 33 31 dc 39 fb 54 a6 a2 63 36 e4 2a |..#.31.9.T..c6.*| +00000070 5a 1c bc 09 |Z...| diff --git a/src/crypto/tls/testdata/Client-TLSv13-AES256-SHA384 b/src/crypto/tls/testdata/Client-TLSv13-AES256-SHA384 index c5bac350a6e3b4..5fcaba0d4fd54a 100644 --- a/src/crypto/tls/testdata/Client-TLSv13-AES256-SHA384 +++ b/src/crypto/tls/testdata/Client-TLSv13-AES256-SHA384 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,87 +7,88 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 7a 02 00 00 76 03 03 3d 39 60 81 a4 |....z...v..=9`..| -00000010 cc 92 5d fd 8b 21 58 da 2e 4c e2 70 2b 12 d2 8c |..]..!X..L.p+...| -00000020 35 55 bc ba 79 c1 9e 6f 78 8c cf 20 00 00 00 00 |5U..y..ox.. ....| +00000000 16 03 03 00 7a 02 00 00 76 03 03 76 2d 81 0b a9 |....z...v..v-...| +00000010 70 b7 fe 52 ef d3 80 c4 1d bf f7 19 99 3c 4c aa |p..R.........4.3I8..| -000002a0 d1 04 b4 13 a8 5f 0b f2 41 77 d4 c8 b2 d3 0e d6 |....._..Aw......| -000002b0 8c dc 34 ad 2a d9 4b 5f 4a e2 17 0a be 2a 55 4a |..4.*.K_J....*UJ| -000002c0 44 a7 b0 0f d1 39 b0 e5 9d 77 6b d7 44 15 99 bf |D....9...wk.D...| -000002d0 c8 02 57 90 5b 73 29 fe ab 85 93 c3 dc 1c b0 b2 |..W.[s).........| -000002e0 14 78 01 99 f5 d2 db ef ae 0e 2a d9 17 1e 70 fa |.x........*...p.| -000002f0 bb 74 fe 93 a0 aa ff 79 cd 8b 6d 3d 79 66 d2 c8 |.t.....y..m=yf..| -00000300 90 c7 2c e6 54 26 de f7 4d b8 a7 73 cd 92 fc 42 |..,.T&..M..s...B| -00000310 ad 4e ff 17 03 03 00 99 c6 07 36 ce 58 9c fe 38 |.N........6.X..8| -00000320 aa b4 5c c8 8e f5 11 19 82 4c 4b 2b 0a 08 be 3f |..\......LK+...?| -00000330 11 76 06 4b d3 1b b7 fc 66 d9 db de fb 6a ee 29 |.v.K....f....j.)| -00000340 f6 a5 4b 96 f3 3c a0 67 44 73 d4 46 c5 d4 a1 f8 |..K..<.gDs.F....| -00000350 71 36 7c a5 62 79 fc be f9 34 55 85 31 8f fc e3 |q6|.by...4U.1...| -00000360 46 be 64 1e 2d 1b 21 c3 89 8e 1c cc df 30 05 57 |F.d.-.!......0.W| -00000370 2e d5 d5 67 11 10 46 1a af 99 01 0e fd 40 53 1f |...g..F......@S.| -00000380 b0 f0 76 3e 8e c1 7a 6a fe fe ef c1 08 6e ea 9b |..v>..zj.....n..| -00000390 75 6c 62 86 07 50 ee 1f ef 3b 55 fd 68 44 47 27 |ulb..P...;U.hDG'| -000003a0 07 c1 53 b5 13 c5 e0 a4 d6 9e 8a 3c 78 a0 04 b7 |..S............|.I+NU.t..| +00000130 23 32 9d aa b1 17 15 3d 37 85 42 0e 5f 7c fa 5b |#2.....=7.B._|.[| +00000140 b8 ae f3 b9 4d b2 40 b5 e2 de e2 b3 f0 27 a6 f2 |....M.@......'..| +00000150 0c 28 4b 77 ee 38 dc ec 17 28 18 03 13 85 60 2c |.(Kw.8...(....`,| +00000160 ab 16 e1 1c c6 e1 91 e0 5f f3 b6 f7 1c 3b 5f 03 |........_....;_.| +00000170 6a d5 b0 a0 5b ab 6d 9c b2 ee 8e 50 49 94 7d db |j...[.m....PI.}.| +00000180 3b fe 68 e2 79 3e 2d 33 06 a6 b2 9c 8b 7d 93 04 |;.h.y>-3.....}..| +00000190 b4 9e 07 6d 15 80 63 38 8e 4b c5 e6 88 a0 6a f2 |...m..c8.K....j.| +000001a0 11 60 11 7a f7 6e 7e 47 9f fd 77 12 c9 d5 8e 58 |.`.z.n~G..w....X| +000001b0 84 51 2c f6 fd 2b d1 dc a6 51 d8 20 3f 06 76 ad |.Q,..+...Q. ?.v.| +000001c0 dd 37 b6 e9 86 4e cb 31 76 79 60 26 c7 b4 d5 67 |.7...N.1vy`&...g| +000001d0 2c 70 22 f4 4d b5 60 61 41 48 65 a8 d0 b5 d8 47 |,p".M.`aAHe....G| +000001e0 08 bf 5b 67 e8 99 c4 a5 91 1e 36 6b 37 53 7b 65 |..[g......6k7S{e| +000001f0 ac af 2d df 8e 77 e2 46 1a 65 45 0a 00 f2 8f 9d |..-..w.F.eE.....| +00000200 47 1c 93 64 52 c3 d9 91 47 41 88 7f 81 ea 8f 2b |G..dR...GA.....+| +00000210 a9 fa 4b d5 9d 6f 30 36 dc af e0 96 b3 f0 e4 6a |..K..o06.......j| +00000220 a3 10 cf 0e 53 64 e3 18 52 08 a9 16 e3 6d e9 0a |....Sd..R....m..| +00000230 ec 4d a4 f7 f1 ee 38 28 88 31 aa 38 a7 14 09 f4 |.M....8(.1.8....| +00000240 1a 31 ac 40 46 dd f2 4f 7f ff f4 fc 71 42 61 73 |.1.@F..O....qBas| +00000250 03 47 9f ec 69 7f 77 e3 91 29 cd 89 7f 45 6b 9d |.G..i.w..)...Ek.| +00000260 f5 33 66 8b 74 bc 53 ff 54 c3 af 48 8e d8 25 0a |.3f.t.S.T..H..%.| +00000270 99 49 08 29 f3 b2 a5 d0 5b ca 89 e2 d7 f5 5f 40 |.I.)....[....._@| +00000280 72 6f b4 d6 20 39 79 4a 0e 7c 00 60 68 eb 1e ab |ro.. 9yJ.|.`h...| +00000290 ee 38 35 f2 db 83 9c 0b 52 80 15 14 ae 6b dc f5 |.85.....R....k..| +000002a0 d5 f3 83 cb e7 5f 5c 98 e2 f2 d0 52 68 92 ff f6 |....._\....Rh...| +000002b0 7a ce ff ae 96 6e ad 11 d2 5e bc 60 16 55 cd 81 |z....n...^.`.U..| +000002c0 25 67 5b db 76 59 36 a8 48 d6 91 86 4f cb 87 5f |%g[.vY6.H...O.._| +000002d0 ea 96 e4 71 df ac 54 5a a0 e1 0a bd 88 68 ff 0a |...q..TZ.....h..| +000002e0 97 be 9b da cf aa e1 b1 74 c9 8d 5f 19 f5 df 53 |........t.._...S| +000002f0 a0 57 93 7d c3 fe 37 78 2a ba 6b c2 a1 51 41 f4 |.W.}..7x*.k..QA.| +00000300 ee 57 ed d2 7d 48 14 31 4a f7 14 d4 0e 42 9e fb |.W..}H.1J....B..| +00000310 fb f4 cd 17 03 03 00 99 6b 70 ae fc d3 07 5e 7f |........kp....^.| +00000320 39 2a 27 34 54 8a b6 4d 39 36 d4 38 0c ed 8c 24 |9*'4T..M96.8...$| +00000330 fc 29 1d 59 74 99 f1 e1 a1 23 4b 6d 27 31 f4 2d |.).Yt....#Km'1.-| +00000340 8b b3 23 aa 92 5f 01 3d a1 f6 b5 24 8e 56 ea eb |..#.._.=...$.V..| +00000350 4e 0a dc 23 ae 32 1b 0d cb bc 60 87 b7 35 32 5f |N..#.2....`..52_| +00000360 10 32 fb 89 32 3f 99 0e 30 0d 65 28 69 c4 3b f8 |.2..2?..0.e(i.;.| +00000370 6c 04 a5 bf ec e9 aa c4 bc b4 55 e2 d0 09 a1 4b |l.........U....K| +00000380 82 f8 a5 37 ab ca 69 f4 bc 24 69 c4 ce 07 6c ee |...7..i..$i...l.| +00000390 d6 08 92 14 41 ca 8f c8 c9 37 ae b5 f3 29 ff 69 |....A....7...).i| +000003a0 7a 18 4e 93 dd eb 52 58 d2 6b 22 6d b6 29 70 b3 |z.N...RX.k"m.)p.| +000003b0 ea 17 03 03 00 45 10 35 86 2e d4 e3 92 35 a3 0e |.....E.5.....5..| +000003c0 a1 cd 79 11 05 cd 77 c4 93 f4 6a ba f8 fc 4b bd |..y...w...j...K.| +000003d0 63 5d c0 2d a0 c9 c2 2a 10 79 e9 0c bc 0b 18 c1 |c].-...*.y......| +000003e0 15 18 c0 8a d7 05 39 f1 de 6e 8e cb dd 73 5f ee |......9..n...s_.| +000003f0 28 93 f9 92 35 56 1f c1 e4 8d 89 |(...5V.....| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 00 45 ee 00 18 db 1e |..........E.....| -00000010 67 aa 0f 9e 5c 20 7d 25 27 49 e8 d5 60 02 78 7b |g...\ }%'I..`.x{| -00000020 23 58 70 67 67 93 ca 4d 50 69 b1 cf d9 dc 31 d3 |#Xpgg..MPi....1.| -00000030 03 36 b0 07 39 59 52 e1 c7 36 6d 16 0b 56 03 c6 |.6..9YR..6m..V..| -00000040 a0 ec 0e 6b ed de 7b ba 51 96 83 b6 b8 77 8f 6a |...k..{.Q....w.j| -00000050 17 03 03 00 17 77 f6 6f 83 00 ad eb be 16 26 3d |.....w.o......&=| -00000060 27 e8 58 73 a2 bc 11 ff 14 54 4a 15 17 03 03 00 |'.Xs.....TJ.....| -00000070 13 bc 94 3f 6e ee 1b 40 b0 2e 03 13 47 28 38 4d |...?n..@....G(8M| -00000080 2d 26 5d 15 |-&].| +00000000 14 03 03 00 01 01 17 03 03 00 45 2f 08 54 f6 2b |..........E/.T.+| +00000010 fd 13 5d 87 d3 b6 73 5e 20 88 af 53 db 7e 3c 37 |..]...s^ ..S.~<7| +00000020 1d 52 a4 ae 91 3f 24 a7 30 26 02 da 13 f3 c5 60 |.R...?$.0&.....`| +00000030 28 e6 eb 8f e1 1b 58 2d bb d0 06 c4 67 33 1c a1 |(.....X-....g3..| +00000040 03 cb 16 56 1b 91 06 08 b6 34 c5 de 97 04 ee f0 |...V.....4......| +00000050 17 03 03 00 17 6c 7e 3d 5d 98 4e 42 81 42 c5 2c |.....l~=].NB.B.,| +00000060 b2 c2 d3 c9 58 bc 13 bc 61 cd 63 f2 17 03 03 00 |....X...a.c.....| +00000070 13 83 c7 4a b6 10 69 b4 16 be 25 a1 db cb c9 94 |...J..i...%.....| +00000080 8f a2 27 22 |..'"| diff --git a/src/crypto/tls/testdata/Client-TLSv13-ALPN b/src/crypto/tls/testdata/Client-TLSv13-ALPN index f40340ac152182..ff0989c567b149 100644 --- a/src/crypto/tls/testdata/Client-TLSv13-ALPN +++ b/src/crypto/tls/testdata/Client-TLSv13-ALPN @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 01 12 01 00 01 0e 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 2c 01 00 01 28 03 03 00 00 00 00 00 |....,...(.......| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,87 +7,89 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 93 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 ad 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 10 00 10 |................| -000000d0 00 0e 06 70 72 6f 74 6f 32 06 70 72 6f 74 6f 31 |...proto2.proto1| -000000e0 00 2b 00 09 08 03 04 03 03 03 02 03 01 00 33 00 |.+............3.| -000000f0 26 00 24 00 1d 00 20 2f e5 7d a3 47 cd 62 43 15 |&.$... /.}.G.bC.| -00000100 28 da ac 5f bb 29 07 30 ff f6 84 af c4 cf c2 ed |(.._.).0........| -00000110 90 99 5f 58 cb 3b 74 |.._X.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 10 00 10 00 0e 06 70 72 6f |.............pro| +000000f0 74 6f 32 06 70 72 6f 74 6f 31 00 2b 00 09 08 03 |to2.proto1.+....| +00000100 04 03 03 03 02 03 01 00 33 00 26 00 24 00 1d 00 |........3.&.$...| +00000110 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 | /.}.G.bC.(.._.)| +00000120 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b |.0.........._X.;| +00000130 74 |t| >>> Flow 2 (server to client) -00000000 16 03 03 00 7a 02 00 00 76 03 03 c2 2e 95 0c cf |....z...v.......| -00000010 63 40 d8 a0 bd 51 27 e2 62 79 72 26 b1 d4 38 5b |c@...Q'.byr&..8[| -00000020 50 03 66 d2 2e 8e 46 b7 cf 40 0a 20 00 00 00 00 |P.f...F..@. ....| +00000000 16 03 03 00 7a 02 00 00 76 03 03 bb 72 d9 b8 e5 |....z...v...r...| +00000010 0b 80 14 4c 16 9b 86 ec 26 62 f2 f9 00 85 a6 ae |...L....&b......| +00000020 44 f4 27 c4 13 67 4a 2e 96 e6 33 20 00 00 00 00 |D.'..gJ...3 ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000040 00 00 00 00 00 00 00 00 00 00 00 00 13 03 00 00 |................| -00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 21 |..+.....3.$... !| -00000060 18 6f 64 73 17 22 b0 11 60 92 ab 01 b3 fb 8a 2d |.ods."..`......-| -00000070 27 15 5a 46 ec 6c c3 0c a7 73 00 b2 7e 9f 4f 14 |'.ZF.l...s..~.O.| -00000080 03 03 00 01 01 17 03 03 00 24 68 7f 90 b2 66 74 |.........$h...ft| -00000090 36 f5 fc 65 55 19 c1 78 65 d8 dc 46 fb ad b6 2c |6..eU..xe..F...,| -000000a0 89 f3 d1 fd 3e 46 c2 50 01 24 72 2a 64 98 17 03 |....>F.P.$r*d...| -000000b0 03 02 6d 40 bc da 1a 9c 8b d7 15 24 4f d4 c1 9c |..m@.......$O...| -000000c0 8b 3c 8b 51 37 41 46 a8 cb b4 9c 50 c0 be 72 ef |.<.Q7AF....P..r.| -000000d0 81 d3 7f ed ec 96 e2 c2 08 0e 7e 23 95 c3 4f 79 |..........~#..Oy| -000000e0 01 2d 26 81 c1 ba d0 58 61 d8 c5 b7 e7 79 4f 3e |.-&....Xa....yO>| -000000f0 d8 c1 07 ae de 6a 1e 78 eb 9d 8c a1 2a 57 9c 56 |.....j.x....*W.V| -00000100 1f 6c 6e bb 2d 70 6d 55 92 10 3f 39 7e 8a d2 30 |.ln.-pmU..?9~..0| -00000110 e5 1c 4b df c0 96 d0 de 16 09 c6 34 23 16 54 a8 |..K........4#.T.| -00000120 f5 0a 1a af 9a 27 60 89 19 6a 3e 69 12 de 76 4c |.....'`..j>i..vL| -00000130 88 f2 ca 6c 0b 94 b8 d2 bc 09 f0 d3 86 74 3c 3f |...l.........tq..| -00000290 5c 9a b7 6b f2 d0 1e 55 2c 1c ac 5d 21 64 a2 d2 |\..k...U,..]!d..| -000002a0 a9 dd a8 49 7c 22 26 8e ca 75 a3 ef 52 42 d0 38 |...I|"&..u..RB.8| -000002b0 f4 02 60 af ee 33 30 46 90 a5 b3 7a 66 c8 ab 10 |..`..30F...zf...| -000002c0 3f 76 d1 76 aa 5d da c4 29 ca d7 22 c4 44 b4 e8 |?v.v.]..)..".D..| -000002d0 64 b6 6d 0f 16 cd d9 e9 62 17 91 64 03 0f 99 8f |d.m.....b..d....| -000002e0 c6 97 c8 4b c9 45 c4 31 e4 1f a8 b0 aa 67 02 e7 |...K.E.1.....g..| -000002f0 20 68 2a 89 f9 b8 e8 ce 93 d0 a3 1b 0d d1 4c 45 | h*...........LE| -00000300 c2 19 ac aa 87 43 95 ae c4 56 56 fe 4b 0e af db |.....C...VV.K...| -00000310 08 6a 14 8e 8b 7e 64 9d 6d 6e 0f 78 7c e4 f8 51 |.j...~d.mn.x|..Q| -00000320 17 03 03 00 99 2d 7b fa e6 dc da 89 93 da 93 91 |.....-{.........| -00000330 9c 25 08 50 66 eb a7 e4 f1 20 da 33 e5 cd a5 89 |.%.Pf.... .3....| -00000340 00 a7 47 9c 84 e6 cf 6d e1 86 02 af fb 54 ff b3 |..G....m.....T..| -00000350 b5 a4 e1 46 1a 3a 89 ad f0 c4 63 fe d2 eb bb 6c |...F.:....c....l| -00000360 6e cc a9 95 b0 e4 df f8 44 46 16 43 07 d7 8e 6e |n.......DF.C...n| -00000370 9b ee 0b 22 7a 23 f4 b8 3e fd fe 99 13 9d 18 bc |..."z#..>.......| -00000380 77 99 2f 90 21 a1 5f cc 9f 08 e2 75 44 2f bf 58 |w./.!._....uD/.X| -00000390 70 f2 95 b2 13 20 f3 ed 51 de e8 02 c6 ca 44 57 |p.... ..Q.....DW| -000003a0 eb 0b 9f 3e 61 f1 1a 6e 6b 13 f5 14 f3 43 19 f6 |...>a..nk....C..| -000003b0 7d 46 3a 73 49 2f fd 83 c3 68 c7 ba b4 e8 17 03 |}F:sI/...h......| -000003c0 03 00 35 67 2b 2b 9b f0 f3 d9 09 c4 01 20 ea 22 |..5g++....... ."| -000003d0 c9 7c e6 7a ce 3f 7f 84 c2 1f 6e eb d6 ce 85 b8 |.|.z.?....n.....| -000003e0 2c 3c 9f 77 8b 37 11 dd 64 d1 ae ba 85 50 7e ba |,<.w.7..d....P~.| -000003f0 14 a9 05 ff 82 3b e5 7d |.....;.}| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 69 |..+.....3.$... i| +00000060 d7 31 51 9b e6 1d 98 09 9f 5f 6c 05 10 ba 64 b5 |.1Q......_l...d.| +00000070 b9 8f b8 c0 2a c1 7f 89 13 ed 04 38 e9 78 43 14 |....*......8.xC.| +00000080 03 03 00 01 01 17 03 03 00 24 99 f3 d5 db e2 35 |.........$.....5| +00000090 4b fb b2 05 e6 03 c2 f2 c4 ea 64 d2 e4 d3 17 e7 |K.........d.....| +000000a0 72 34 e8 b2 d6 61 91 3b b3 03 ea 1c 33 57 17 03 |r4...a.;....3W..| +000000b0 03 02 6d 22 da 71 90 42 da 0a 03 52 98 42 e3 71 |..m".q.B...R.B.q| +000000c0 db 57 e0 9d 70 89 4b 4c a9 a5 4d 39 02 19 f1 09 |.W..p.KL..M9....| +000000d0 5a 9c 33 c9 f2 8d 7e 78 c3 b3 ae 1b 2c 7b 73 ef |Z.3...~x....,{s.| +000000e0 3a ad d8 51 82 90 bc 6b 0d 6f a6 f0 46 8d 72 29 |:..Q...k.o..F.r)| +000000f0 ca 84 d8 b1 27 e3 8d 8f f5 ce f0 f0 d9 2f 9c 71 |....'......../.q| +00000100 b9 bc 89 af 92 18 d2 c9 07 b2 eb 94 40 42 d2 7a |............@B.z| +00000110 c8 be 49 35 b6 b7 f3 b9 64 c1 56 fe fc d8 11 04 |..I5....d.V.....| +00000120 6c d7 e9 18 98 29 eb d5 6f 59 0d fa 25 85 f7 3c |l....)..oY..%..<| +00000130 df 76 d9 52 10 72 59 d6 36 5b b0 54 01 1d ac 9e |.v.R.rY.6[.T....| +00000140 9b 86 5f 5b ee 18 74 ec 8e 8b da 0f df 71 50 e2 |.._[..t......qP.| +00000150 cd 58 98 23 a0 21 94 ac eb db 3a fe f6 8d aa cc |.X.#.!....:.....| +00000160 21 23 e6 35 f7 0e 42 0e 50 48 14 22 f3 83 91 3d |!#.5..B.PH."...=| +00000170 73 22 5b 74 e7 e8 d1 b5 ee cd 7e c3 ee 0d 13 a9 |s"[t......~.....| +00000180 aa ae 0b 12 b0 3a 9d 6f 01 68 27 e4 62 e0 0e 67 |.....:.o.h'.b..g| +00000190 e4 de 69 74 8d f1 7a 3b 16 bb 1d a0 ad bf c5 4b |..it..z;.......K| +000001a0 50 9e b0 dc 08 73 14 99 c9 ad de 04 df 75 09 de |P....s.......u..| +000001b0 e4 27 ec 2b 75 79 db 82 8c aa 42 af 40 df 32 0e |.'.+uy....B.@.2.| +000001c0 e6 1d 93 d1 00 f7 18 59 43 9c 03 6c f7 bf ac a2 |.......YC..l....| +000001d0 8b 10 be b1 a8 b6 4e 84 2b c0 ff 87 4d 26 c2 8f |......N.+...M&..| +000001e0 63 64 7a 66 a6 12 0b e4 06 9e ff af fe 03 55 c7 |cdzf..........U.| +000001f0 13 92 22 ec ed 89 76 2d 36 8a 4a 92 f1 a5 b6 ac |.."...v-6.J.....| +00000200 d4 19 46 9d f3 e0 52 6d 01 ee bf ab d5 17 c0 33 |..F...Rm.......3| +00000210 b5 81 e1 81 a2 e9 44 b7 20 0e f4 f0 35 bb d4 8d |......D. ...5...| +00000220 bc 83 fc 59 fb 1d 5e e0 c4 4c 9e cc c3 72 b7 3d |...Y..^..L...r.=| +00000230 01 a4 f2 df e2 a7 44 d1 f1 20 90 31 53 09 50 eb |......D.. .1S.P.| +00000240 80 85 23 52 68 7d b2 51 60 0c 00 87 c3 42 88 07 |..#Rh}.Q`....B..| +00000250 de 59 fd 3d 68 a3 22 50 cb 67 43 94 02 77 24 6a |.Y.=h."P.gC..w$j| +00000260 8d a4 4f 85 7f 19 a0 b5 10 76 2a 85 4c 84 12 e3 |..O......v*.L...| +00000270 f7 ce b8 30 9f d0 9f 8a 26 42 28 f4 cb f7 c4 c2 |...0....&B(.....| +00000280 aa 34 c2 72 64 dc 6d 9d d4 26 2f 14 4d 97 2c 00 |.4.rd.m..&/.M.,.| +00000290 2d 46 0f 07 9c 3d 76 d4 55 a7 15 13 b2 41 e3 f4 |-F...=v.U....A..| +000002a0 33 ae 1b 26 68 43 62 de c1 c2 61 26 08 1b 20 13 |3..&hCb...a&.. .| +000002b0 bc 58 72 c2 16 fe 4d 2e 68 8b 31 61 ac c9 55 c5 |.Xr...M.h.1a..U.| +000002c0 3c 5a 77 c2 61 47 4a d6 0d c4 8a bc bb 63 ca ea |0....8bA.L94..| +000003e0 ee 48 70 3c 23 65 90 cb f9 4f 76 72 dd f3 64 83 |.Hp<#e...Ovr..d.| +000003f0 23 38 79 13 d2 93 57 fb |#8y...W.| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 00 35 39 60 84 ac d0 |..........59`...| -00000010 b9 47 4b 47 49 af 52 88 3c 12 50 0d 65 46 f7 a1 |.GKGI.R.<.P.eF..| -00000020 50 39 c5 39 22 b3 45 e6 d3 25 44 a6 bc 3d bc 69 |P9.9".E..%D..=.i| -00000030 35 1d 61 cf a0 3c 53 77 94 67 66 ae 37 7f 3f 84 |5.a..>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,85 +7,86 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 7a 02 00 00 76 03 03 3d 50 78 7e 02 |....z...v..=Px~.| -00000010 dc 79 3a 32 6b 6a 50 24 90 65 72 5e cd fc 96 18 |.y:2kjP$.er^....| -00000020 e1 47 21 43 50 5a 32 0f cf e7 ae 20 00 00 00 00 |.G!CPZ2.... ....| +00000000 16 03 03 00 7a 02 00 00 76 03 03 f9 23 d9 5d 6c |....z...v...#.]l| +00000010 ae 6b 63 2a d8 2c a0 27 8e 7b 39 29 35 04 0e a9 |.kc*.,.'.{9)5...| +00000020 31 f7 a0 c5 77 a7 f3 c5 25 ca 2f 20 00 00 00 00 |1...w...%./ ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000040 00 00 00 00 00 00 00 00 00 00 00 00 13 03 00 00 |................| -00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 cd |..+.....3.$... .| -00000060 e6 a0 65 6b 1c 9e 5c f0 b7 f9 82 8e 37 d1 c4 35 |..ek..\.....7..5| -00000070 31 c1 3f df 4a 70 33 ba 38 d9 3f 74 0c c2 61 14 |1.?.Jp3.8.?t..a.| -00000080 03 03 00 01 01 17 03 03 00 17 63 c3 02 95 17 f4 |..........c.....| -00000090 a8 76 8d 54 ad 3c 10 c7 cc e8 08 13 d6 9b 0b 0f |.v.T.<..........| -000000a0 db 17 03 03 02 6d 0f ec 8d 0d e5 1a 85 02 3f 92 |.....m........?.| -000000b0 44 09 ac e6 da 2d c8 d7 14 68 00 9d 22 bf 05 1d |D....-...h.."...| -000000c0 cd 07 b5 8b b9 84 3f cf cc c5 48 40 00 2e e1 c9 |......?...H@....| -000000d0 0b 41 6d 39 cb 05 00 d6 7a 10 4a ba 96 e1 96 a9 |.Am9....z.J.....| -000000e0 70 d8 20 76 2f d2 70 83 cb a0 d1 72 43 db 09 cd |p. v/.p....rC...| -000000f0 2d e5 89 11 e8 81 15 13 dd 29 2a c8 88 4a d8 73 |-........)*..J.s| -00000100 35 3d 8e 8a 03 ca ed 32 a9 16 bb 71 88 d5 5f a1 |5=.....2...q.._.| -00000110 1b ec ee 9e 3d 8c f4 d0 c2 5b 72 84 3c 75 aa de |....=....[r......).@..| -00000130 3c f4 32 90 c0 25 b7 f0 fc 6b 16 c1 ab 80 69 79 |<.2..%...k....iy| -00000140 5e dd 95 5a ba e4 5a 68 64 a6 44 f5 a2 3a d6 30 |^..Z..Zhd.D..:.0| -00000150 7e b1 27 b1 79 44 ef 79 95 51 75 02 24 2c ea ab |~.'.yD.y.Qu.$,..| -00000160 d5 e1 fd aa 81 55 89 cf bc 9c 11 2b 0b 4a 99 9f |.....U.....+.J..| -00000170 50 93 68 a1 c8 79 3d d8 aa d8 54 e3 d5 1a 97 90 |P.h..y=...T.....| -00000180 33 87 a9 70 1d 9d ba 78 d0 87 8a 98 6f 1b 1e b0 |3..p...x....o...| -00000190 e8 92 78 df 21 61 88 36 fa 14 4b c9 b1 a4 93 88 |..x.!a.6..K.....| -000001a0 b4 26 9a 13 1b 8f 6c 84 bc b1 78 a8 a1 e7 64 4b |.&....l...x...dK| -000001b0 d4 56 37 b6 d1 80 6d 8d 0f c2 2f 0c 0b 9e 42 8e |.V7...m.../...B.| -000001c0 37 e0 9b 73 e4 ab cf 24 43 86 bc d0 40 fd db 4a |7..s...$C...@..J| -000001d0 c1 f6 55 56 e9 54 5c c7 3c 17 36 44 7b 99 7e 03 |..UV.T\.<.6D{.~.| -000001e0 48 f7 a9 cc 4d b9 b4 bf 09 dd 96 37 8c b5 d5 f0 |H...M......7....| -000001f0 bd af 19 0e 82 6c 5d b6 96 ca 98 a2 56 d3 68 f2 |.....l].....V.h.| -00000200 94 cd 7a 94 8d b3 97 16 03 3d 6a 94 5d 66 32 5d |..z......=j.]f2]| -00000210 11 28 d9 7b c1 8e b1 c4 ec b7 54 94 3b 80 f9 b3 |.(.{......T.;...| -00000220 9d 2d 74 3c 02 55 a0 4a 02 b7 61 19 b0 a2 e6 44 |.-t<.U.J..a....D| -00000230 3e c8 81 e3 a4 18 d7 4c 16 d2 fb e3 90 2c 61 71 |>......L.....,aq| -00000240 a0 82 28 48 de 7c 7c bd ba 27 85 e1 fd c9 df ac |..(H.||..'......| -00000250 b2 61 0f 80 f3 86 31 8f c1 a3 dc b6 43 98 c9 0e |.a....1.....C...| -00000260 73 dd 1d cf 61 f6 25 23 44 ca 5a 4c 12 6c bb 35 |s...a.%#D.ZL.l.5| -00000270 32 c4 32 0d cc 8c 11 b9 4a ee 88 92 d6 27 d4 a8 |2.2.....J....'..| -00000280 25 59 1e fe ba 41 63 50 1e b1 e2 c5 75 95 7d 56 |%Y...AcP....u.}V| -00000290 60 ed 81 d0 e7 12 05 e7 0f b4 5d 6f 27 98 76 72 |`.........]o'.vr| -000002a0 e7 8a e1 ff 5d 6b 29 9f 75 24 a5 fd 6d 1f bc 58 |....]k).u$..m..X| -000002b0 f8 b8 67 d1 b0 21 25 f1 09 c7 f9 41 f5 85 1b eb |..g..!%....A....| -000002c0 56 f6 14 83 bb 08 d5 3e ed 22 e1 c5 9c 14 cc de |V......>."......| -000002d0 d3 45 e1 0e e3 cb 61 b5 ab f6 03 41 bc 71 5d 17 |.E....a....A.q].| -000002e0 99 c6 8f 13 41 f8 78 e8 da 3d 40 1c 1a 47 a9 b2 |....A.x..=@..G..| -000002f0 e7 22 90 2f d8 8f 6f 0d 36 02 b1 36 fa 6d 26 6f |."./..o.6..6.m&o| -00000300 82 e2 54 74 a0 26 3c 0c 57 bb d7 01 ff ee 6f 16 |..Tt.&<.W.....o.| -00000310 63 92 00 17 03 03 00 99 ea 68 e4 08 10 2d 6f b6 |c........h...-o.| -00000320 a2 f8 f5 03 35 00 63 c8 da 23 4c 96 b8 2a 5c f5 |....5.c..#L..*\.| -00000330 9d 24 6f 5a 5b c3 aa d2 df d9 16 b6 9a 91 79 39 |.$oZ[.........y9| -00000340 18 35 65 28 80 97 fe 76 26 79 45 80 4c 8a 7c cc |.5e(...v&yE.L.|.| -00000350 a2 36 1a 8a ad aa f3 06 08 1f b7 4a 9b 22 7f 37 |.6.........J.".7| -00000360 62 3a ae 13 c0 f4 22 05 6f fc 92 5a e0 ed f1 5b |b:....".o..Z...[| -00000370 c6 6b 4f 18 83 a6 39 a7 5c 1c 53 32 08 76 25 03 |.kO...9.\.S2.v%.| -00000380 20 d5 47 57 76 06 38 4e bd b8 dd 7f 0e 37 b3 4e | .GWv.8N.....7.N| -00000390 f1 f3 b1 98 d4 f0 d6 00 ea 6a e9 ec a4 b0 fd 20 |.........j..... | -000003a0 2d 67 20 c8 ef 9f a1 97 00 cd 15 c2 8e c2 ba 46 |-g ............F| -000003b0 76 17 03 03 00 35 d0 f3 18 d0 71 4c 66 8c 40 ef |v....5....qLf.@.| -000003c0 57 92 f3 56 5a 42 1e 4c e6 13 5a cc ab 26 90 6a |W..VZB.L..Z..&.j| -000003d0 9b ca 49 b0 81 ec 38 30 44 27 5a fa ac d6 1f 5c |..I...80D'Z....\| -000003e0 bc 64 4b 0b c3 69 b2 5b ef 7f 4a |.dK..i.[..J| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 ba |..+.....3.$... .| +00000060 2b 6b 27 27 14 c0 42 b6 1f bc ad 77 f1 1f 4a b3 |+k''..B....w..J.| +00000070 42 94 a6 3a 2c 7f e6 aa 72 8e d0 70 e8 0e 53 14 |B..:,...r..p..S.| +00000080 03 03 00 01 01 17 03 03 00 17 33 bb 8a 47 b0 a9 |..........3..G..| +00000090 29 d7 bf e4 94 7d 42 8a 19 95 4e d9 7c b7 9a 87 |)....}B...N.|...| +000000a0 83 17 03 03 02 6d 86 a7 6c 3c 4e 45 95 f6 f6 66 |.....m..l.`.`~.....r| +000001d0 a4 bc 66 a7 bd 46 83 88 6e 9f b3 db de e7 b3 04 |..f..F..n.......| +000001e0 62 4c 61 d6 d0 89 84 e4 30 d7 4d a0 61 4f 56 ad |bLa.....0.M.aOV.| +000001f0 51 c6 1f 10 01 9e 8d 75 55 fb 89 b8 d0 57 e3 ce |Q......uU....W..| +00000200 40 72 41 27 96 e0 5b f4 cc ba d0 ec 8c d4 91 ca |@rA'..[.........| +00000210 8f f0 db 11 86 d5 8e 02 6a 1f 29 00 f6 ff 42 a2 |........j.)...B.| +00000220 7c c9 e7 18 38 39 c9 e4 2b 0e e9 36 40 d3 dd bc ||...89..+..6@...| +00000230 f7 e3 7c 2d e9 f0 52 31 16 85 b4 e2 a8 54 cd d0 |..|-..R1.....T..| +00000240 d6 e2 ed f4 fc 91 9f 50 68 a0 74 af 37 c7 4c bd |.......Ph.t.7.L.| +00000250 de 9e c7 78 06 41 6b 62 07 8c 8b f4 f5 ea ab 16 |...x.Akb........| +00000260 b0 16 b9 ea 06 2c ee 66 d5 da b3 2c 0f 03 d2 f6 |.....,.f...,....| +00000270 ae f7 11 85 d8 b6 b8 bf 7e a5 c8 ff 0d 41 29 86 |........~....A).| +00000280 94 e4 32 58 6c 0e 05 47 b8 c5 77 de d1 81 8d 72 |..2Xl..G..w....r| +00000290 d4 e0 64 66 f7 17 8a 96 88 bf 1c 04 22 40 e0 cf |..df........"@..| +000002a0 28 6d f5 4f 89 e2 4a 23 4b 71 e1 15 3b da 45 0b |(m.O..J#Kq..;.E.| +000002b0 13 ee 1f 7f 4e 45 3d 8e d2 f3 bd 23 e7 4c f3 d2 |....NE=....#.L..| +000002c0 0f 4b 6e 6a 3d 6a f1 26 b7 ae f6 1d 39 73 cd fd |.Knj=j.&....9s..| +000002d0 1c c8 c7 45 fa e0 67 d5 2c eb e9 2b 32 8e 65 7b |...E..g.,..+2.e{| +000002e0 c5 32 cc e6 01 91 fc fd 92 6e ab 9a 13 f6 63 81 |.2.......n....c.| +000002f0 e5 b9 25 6d ce 73 39 9e 82 ae 18 89 a5 32 cb d3 |..%m.s9......2..| +00000300 a3 85 55 f5 cd 26 96 18 91 1c 41 36 69 49 93 f2 |..U..&....A6iI..| +00000310 aa 05 ab 17 03 03 00 99 9c 1d 79 c5 7b 73 b9 f9 |..........y.{s..| +00000320 00 2d ea ff c2 8d f6 57 65 7c 92 fa ba 22 18 03 |.-.....We|..."..| +00000330 69 69 e9 05 e0 74 1a e5 9d f9 30 9f 1c 39 86 30 |ii...t....0..9.0| +00000340 aa 1b 86 fa 20 26 20 13 ea 8d eb 8b 4e 8b c1 50 |.... & .....N..P| +00000350 86 ef a3 c5 8e 48 b2 a1 5a ac 05 e7 8f 23 8a 34 |.....H..Z....#.4| +00000360 ab 1f 8e 90 b1 e5 9a d3 d7 28 90 b6 12 35 dc cb |.........(...5..| +00000370 c5 3c 8d 3d fc e2 99 2a 8b f0 6a f4 8b a9 62 3f |.<.=...*..j...b?| +00000380 b6 19 29 fd 79 b9 35 72 b0 89 59 ab 78 c6 c9 f0 |..).y.5r..Y.x...| +00000390 68 bc 0d f5 9a 45 dd 4f d2 40 75 19 47 af e9 6f |h....E.O.@u.G..o| +000003a0 56 ec 73 ce cd 19 31 c0 39 08 b1 63 e0 ac d4 49 |V.s...1.9..c...I| +000003b0 e5 17 03 03 00 35 0d 34 de e9 22 e4 56 18 4a 33 |.....5.4..".V.J3| +000003c0 d1 05 c4 d4 f2 64 24 62 d7 da 6a 8e 34 3b 51 13 |.....d$b..j.4;Q.| +000003d0 27 69 88 37 4b ba 29 9d c5 78 af 62 2b 62 6e 5a |'i.7K.)..x.b+bnZ| +000003e0 28 7a 93 c5 9a f3 84 1d 50 9a 94 |(z......P..| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 00 35 ef 3b a2 4d 0f |..........5.;.M.| -00000010 69 53 f2 a3 80 a0 06 a9 99 a9 15 1e bc 8b 4a 08 |iS............J.| -00000020 de 87 e7 b6 10 d6 b3 fc 89 f8 24 f9 2d 04 a1 fd |..........$.-...| -00000030 c2 ce eb 14 e9 df bf ab 80 8e f1 00 53 60 88 82 |............S`..| -00000040 17 03 03 00 17 22 aa a9 e0 1f f8 55 03 22 9b 0f |.....".....U."..| -00000050 b2 6d 73 be 10 5d 6c c6 d8 0e 5e 0d 17 03 03 00 |.ms..]l...^.....| -00000060 13 45 f2 81 7f ea ae 0d 77 bb d4 05 1e 55 af a8 |.E......w....U..| -00000070 3e 0d 10 0f |>...| +00000000 14 03 03 00 01 01 17 03 03 00 35 4f 9c 2d 01 8b |..........5O.-..| +00000010 4a 31 16 77 b4 f2 2b 40 cb 8e d5 7d 93 bd 66 59 |J1.w..+@...}..fY| +00000020 f8 f1 f4 45 69 fd f7 9b 88 5a e5 0a 40 67 e2 17 |...Ei....Z..@g..| +00000030 72 de b0 1d 02 ba b5 a7 58 35 4a 3e b6 2a 0c e8 |r.......X5J>.*..| +00000040 17 03 03 00 17 4a 50 b7 f1 94 a4 64 9e a2 95 0a |.....JP....d....| +00000050 6c f6 93 7f d6 6d 12 10 7a 69 8a d2 17 03 03 00 |l....m..zi......| +00000060 13 10 60 24 3f 6e c9 ee c6 27 50 72 5e 19 22 e0 |..`$?n...'Pr^.".| +00000070 76 12 45 e5 |v.E.| diff --git a/src/crypto/tls/testdata/Client-TLSv13-ClientCert-ECDSA-RSA b/src/crypto/tls/testdata/Client-TLSv13-ClientCert-ECDSA-RSA index 8984790aab6801..6611cdfed22fae 100644 --- a/src/crypto/tls/testdata/Client-TLSv13-ClientCert-ECDSA-RSA +++ b/src/crypto/tls/testdata/Client-TLSv13-ClientCert-ECDSA-RSA @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,134 +7,135 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 7a 02 00 00 76 03 03 ea 8b 99 cb 5b |....z...v......[| -00000010 d8 fb e9 14 7f 17 20 9c b8 41 01 dd ce 8a 90 4e |...... ..A.....N| -00000020 a9 f0 fb eb 71 37 24 02 d2 ee 96 20 00 00 00 00 |....q7$.... ....| +00000000 16 03 03 00 7a 02 00 00 76 03 03 ea 4e 78 a5 f4 |....z...v...Nx..| +00000010 71 78 37 5b 2c e4 69 b2 00 4d 49 8c 8b 86 4c 80 |qx7[,.i..MI...L.| +00000020 f9 db 03 3f cc 1e 42 f9 87 ff 7b 20 00 00 00 00 |...?..B...{ ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000040 00 00 00 00 00 00 00 00 00 00 00 00 13 03 00 00 |................| -00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 c8 |..+.....3.$... .| -00000060 c6 52 8e 3c 6f 0d ab 2d be 4c e0 01 c8 af 1f 8e |.R........t| -00000100 69 88 5d 8e a4 fa 02 b0 6f f4 0e 38 cc 95 12 e2 |i.].....o..8....| -00000110 f1 e3 47 89 a6 1d 80 26 cb 23 6e f3 83 1c e4 85 |..G....&.#n.....| -00000120 7b 1a 4d 12 c8 bf ff 07 39 a9 4e 4e d7 45 35 23 |{.M.....9.NN.E5#| -00000130 9b f9 59 6d a5 b0 49 1b 5f e7 42 62 17 00 1e 57 |..Ym..I._.Bb...W| -00000140 53 c5 22 fb 05 89 fd fe 5d de 71 e8 26 fd 6d e3 |S.".....].q.&.m.| -00000150 fc b9 cb 1f d5 d4 84 d1 67 fe 8a a0 74 ff ad ff |........g...t...| -00000160 c8 35 fe c2 9a 17 33 18 51 c2 cd 19 7c dc 5d c5 |.5....3.Q...|.].| -00000170 7c e6 d0 38 ab 28 0b 8c 78 51 aa 7f ef f0 9a c3 ||..8.(..xQ......| -00000180 df 26 d2 bc 1b b6 98 b1 16 35 9d f0 73 b7 15 f7 |.&.......5..s...| -00000190 7d 9f 3e fe 4c 75 e7 c7 5d cb c2 e5 29 4a 30 32 |}.>.Lu..]...)J02| -000001a0 e2 da 3c 2c 16 ba 89 41 28 5c 33 75 b3 ed d1 e6 |..<,...A(\3u....| -000001b0 4f f6 bb 33 62 53 9d be fe d3 53 b5 bd 3e e3 b5 |O..3bS....S..>..| -000001c0 0a 37 67 60 33 c3 72 a8 ea 55 73 3c b2 7f ed 8b |.7g`3.r..Us<....| -000001d0 59 5e 44 e4 76 0d 1e 3c 3e 1c 9c 8c 86 3d 0a a3 |Y^D.v..<>....=..| -000001e0 78 bd 13 77 eb dc 22 e5 96 ff ae 44 94 cd ef ae |x..w.."....D....| -000001f0 ca 64 ec 06 a1 38 49 17 ce a5 c4 34 86 fd 55 1f |.d...8I....4..U.| -00000200 32 31 d5 b0 8c d6 b5 bc b8 29 29 97 b5 39 a0 f8 |21.......))..9..| -00000210 b1 b5 72 24 ff ce 6f 7d 6a 0d 18 26 8f 9a e8 d3 |..r$..o}j..&....| -00000220 e6 e0 7c 9c 56 45 ca 48 44 69 05 53 40 eb 96 c5 |..|.VE.HDi.S@...| -00000230 17 aa 28 56 20 ee 82 fd de d1 55 b8 e9 27 ae 3d |..(V .....U..'.=| -00000240 e1 44 d6 0b b9 7a 25 77 b0 f1 be 64 ae cc 0d 44 |.D...z%w...d...D| -00000250 af 57 32 9f cf bc c0 07 00 0b 19 97 08 0a d5 50 |.W2............P| -00000260 20 0e ef d5 1e 2e 68 82 ae 2d 84 47 3d 9b 5b 24 | .....h..-.G=.[$| -00000270 55 95 b2 93 e0 60 a2 cd e5 72 69 b3 e2 de da 70 |U....`...ri....p| -00000280 76 88 be 1f 5a 8e 7a d8 ff 94 db b0 92 b2 a1 a1 |v...Z.z.........| -00000290 26 5b 3b d5 5e 67 16 b7 6a 9f dc ab 21 7e df 6e |&[;.^g..j...!~.n| -000002a0 2a 73 e9 20 40 4b c8 34 fc 4b be f6 d8 ba 78 d7 |*s. @K.4.K....x.| -000002b0 a4 c2 ed a9 52 f9 ea 88 67 5e 10 92 8f ba 83 cd |....R...g^......| -000002c0 19 79 82 c5 76 06 d9 98 9f 68 2e 34 82 2f 9a 41 |.y..v....h.4./.A| -000002d0 fb 63 b6 8c 79 56 62 9d bb bc b5 22 ab 28 35 f7 |.c..yVb....".(5.| -000002e0 bc c6 a8 6b e7 86 01 c3 6c e5 88 28 57 09 65 31 |...k....l..(W.e1| -000002f0 b2 5c 0a 60 46 af 94 2c d4 37 49 9b 4c e4 4b bb |.\.`F..,.7I.L.K.| -00000300 fb 0d 87 94 82 11 09 26 04 6f ec e9 a6 f4 9c ca |.......&.o......| -00000310 79 a0 d0 48 32 5a 02 58 70 81 d0 b6 6d 77 2f 6f |y..H2Z.Xp...mw/o| -00000320 4e 9b db ca 82 38 ed a6 60 8e e2 b6 14 33 f5 02 |N....8..`....3..| -00000330 4b 97 a4 86 2f 43 9d ee 88 2c 1c 58 7e 47 30 cc |K.../C...,.X~G0.| -00000340 ec 0d aa 22 13 60 be e2 c9 c4 59 90 67 96 9b 2a |...".`....Y.g..*| -00000350 41 79 49 00 71 80 e9 0d 12 c3 17 03 03 00 99 ec |AyI.q...........| -00000360 f5 a1 45 64 61 fc 61 35 d5 2f bf 20 02 68 0b 10 |..Eda.a5./. .h..| -00000370 df c4 4b e7 2c 43 bc f5 3d 0b 7e 9f a4 71 09 2c |..K.,C..=.~..q.,| -00000380 a6 cf f4 f4 b4 2c 08 0c 03 50 ac 47 74 ad 24 f1 |.....,...P.Gt.$.| -00000390 04 f3 d4 83 42 3f 35 a5 57 ff ab 59 0c 9a a2 ca |....B?5.W..Y....| -000003a0 6c 30 b7 87 73 af 53 f9 1d 6b e7 44 ec d1 bd 14 |l0..s.S..k.D....| -000003b0 15 09 cf ff 82 5e a2 6d ba 00 53 b8 b3 7c 0e e5 |.....^.m..S..|..| -000003c0 d1 e2 a2 38 25 88 aa ee 93 c8 d9 d1 88 42 42 90 |...8%........BB.| -000003d0 43 8d 22 d8 48 02 57 22 6a f4 e9 23 71 f0 64 30 |C.".H.W"j..#q.d0| -000003e0 6a 68 12 a5 3c 8c 61 5e bc 73 91 6a 01 3a 14 14 |jh..<.a^.s.j.:..| -000003f0 86 7c 4d f5 aa cc 2f f5 17 03 03 00 35 7e f6 f5 |.|M.../.....5~..| -00000400 6b 75 e2 63 64 5a 6b 9b ce 6b 76 d7 47 bc 1b 47 |ku.cdZk..kv.G..G| -00000410 9e 68 25 fe 47 2b 06 a9 72 d0 a7 3f 23 3d 71 4a |.h%.G+..r..?#=qJ| -00000420 da 29 fb e3 dd ee e6 f6 a5 32 cc eb da 61 2b 52 |.).......2...a+R| -00000430 24 26 |$&| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 f1 |..+.....3.$... .| +00000060 bf 65 1e fd e0 5d 1b ca 02 b5 9e 3f a8 82 e0 5a |.e...].....?...Z| +00000070 97 eb af b5 f1 f7 77 3f 46 a8 66 93 de a8 39 14 |......w?F.f...9.| +00000080 03 03 00 01 01 17 03 03 00 17 16 a7 f6 48 b9 11 |.............H..| +00000090 59 75 3d d8 cf eb ba 72 6c f7 0d 26 50 ab 2d 2a |Yu=....rl..&P.-*| +000000a0 61 17 03 03 00 42 00 0f 78 f4 5a 62 c5 56 bd f5 |a....B..x.Zb.V..| +000000b0 4a df d3 78 76 bd 90 71 49 69 8a b3 60 5b 24 8b |J..xv..qIi..`[$.| +000000c0 45 fb c7 e0 ff 04 d8 7b c6 1f 93 c9 f9 b8 0d 38 |E......{.......8| +000000d0 53 d1 2e 95 99 32 52 2e 45 b2 11 08 45 c7 bc 64 |S....2R.E...E..d| +000000e0 6a c2 32 db 70 96 77 6f 17 03 03 02 6d ba 3e 97 |j.2.p.wo....m.>.| +000000f0 ea d7 dd c1 c5 39 9a c4 b1 eb 8b df 35 57 43 35 |.....9......5WC5| +00000100 31 7a 4c 71 ab 57 79 80 f9 48 96 67 5f 3b fe 98 |1zLq.Wy..H.g_;..| +00000110 0b 5d d0 ce 8c 0c 5a 03 28 8f 64 b4 3e 74 75 51 |.]....Z.(.d.>tuQ| +00000120 81 b3 09 a7 8e 33 45 92 49 72 91 b7 2c 45 68 5c |.....3E.Ir..,Eh\| +00000130 ee 27 e8 5b fd b0 80 e3 43 35 2f a2 bd e5 cf d6 |.'.[....C5/.....| +00000140 c3 fc 43 03 1e aa c9 c4 67 db ef 0c c3 aa 46 18 |..C.....g.....F.| +00000150 39 69 31 26 7f 0a 7a b2 e7 c4 cb 09 3a f1 8e a0 |9i1&..z.....:...| +00000160 05 70 87 a9 12 c6 f2 a1 25 87 72 0a 2d fb d5 51 |.p......%.r.-..Q| +00000170 ce 93 8e f8 08 0e 86 90 04 00 9b 76 fd 57 23 cb |...........v.W#.| +00000180 bc 78 23 07 8a 35 92 6e d6 cd f5 b5 2e e1 8f 33 |.x#..5.n.......3| +00000190 7a bb 61 54 65 4f 4d 12 4b ca db 19 4b 30 6e 2c |z.aTeOM.K...K0n,| +000001a0 56 62 b5 32 7a 98 b6 a4 10 98 5a 16 6b 26 8e 7c |Vb.2z.....Z.k&.|| +000001b0 e0 88 77 a6 20 60 c9 fb c5 e2 66 c3 6d 0d df dd |..w. `....f.m...| +000001c0 21 22 10 da 88 24 21 92 74 3b d8 92 c0 ec 2f 57 |!"...$!.t;..../W| +000001d0 79 d7 42 bd d7 b0 e0 23 d2 a0 45 7f 2a 2a ff df |y.B....#..E.**..| +000001e0 a4 61 53 ec 44 e2 3a 7a 06 15 8a ce 02 84 e7 78 |.aS.D.:z.......x| +000001f0 9b ef 15 0e 84 16 fe 0f dd 36 de 0f 69 14 e4 35 |.........6..i..5| +00000200 90 e4 a0 15 a6 97 e5 c9 a1 3c ed 79 c6 03 ae 39 |.........<.y...9| +00000210 f2 36 1e ca 20 4f 20 59 e6 6f c7 15 ba 90 ac 4a |.6.. O Y.o.....J| +00000220 11 ad 3e f5 48 df 94 fe f8 48 2f 12 25 01 6c 9a |..>.H....H/.%.l.| +00000230 e6 ee 41 7b 13 4c 2f cc 3a 2d 8d b8 e7 d5 62 88 |..A{.L/.:-....b.| +00000240 88 5d d4 6e c4 64 c5 32 0a e7 86 08 64 2c 0a 11 |.].n.d.2....d,..| +00000250 8e 51 63 f9 81 30 00 b1 29 fa b8 1c ab 87 88 22 |.Qc..0..)......"| +00000260 f0 ef 79 04 8c 85 78 df 72 6c 99 d6 c7 3a 9f 2a |..y...x.rl...:.*| +00000270 6d c6 24 05 e7 e5 d7 d0 c7 e5 7c 87 50 f9 b7 69 |m.$.......|.P..i| +00000280 6d 1f 39 77 5d 4c f2 98 35 f7 07 b6 30 0d d6 25 |m.9w]L..5...0..%| +00000290 40 cb a4 2d b2 f1 22 ca 26 5d 92 22 97 0a 65 a9 |@..-..".&]."..e.| +000002a0 a0 46 58 be 0d af a1 1c da c8 bf 98 28 72 ea ee |.FX.........(r..| +000002b0 46 f9 c0 0d b3 f1 11 12 5f b7 69 75 ea dd 60 d0 |F......._.iu..`.| +000002c0 53 51 01 ec 7c ff ef 41 68 04 ec a4 6b 51 bb 89 |SQ..|..Ah...kQ..| +000002d0 8b 8d a2 89 05 6f 78 81 16 f8 7a c2 f6 c3 0d ef |.....ox...z.....| +000002e0 b3 12 e2 57 54 95 a6 cd 5d 04 32 88 70 f9 db db |...WT...].2.p...| +000002f0 b3 45 75 32 6f ac 9a fa ad 54 e5 a5 82 ae 3e 73 |.Eu2o....T....>s| +00000300 2d 07 f4 48 83 e2 10 7d cb 2f 4c 18 94 75 37 24 |-..H...}./L..u7$| +00000310 cd 13 dc 3a 56 93 e8 80 a3 ba de b1 0d 14 79 48 |...:V.........yH| +00000320 08 69 a6 f3 2e 9f 7a 78 6e 9c de 16 ba 73 0b a6 |.i....zxn....s..| +00000330 f2 6c 04 e6 e6 89 3d 46 7d db a4 8f 9a e7 5d 0e |.l....=F}.....].| +00000340 b9 6b 05 59 4d 4d e9 48 1a 18 e9 59 7b 56 70 d6 |.k.YMM.H...Y{Vp.| +00000350 0f fc c2 93 31 03 fa ad b4 c9 17 03 03 00 99 62 |....1..........b| +00000360 c5 b8 11 10 72 af 5b 36 7b 4b 86 83 05 1f 5a c5 |....r.[6{K....Z.| +00000370 66 7b b6 ac 30 8d c0 03 64 ec 29 5d 60 e9 e9 9d |f{..0...d.)]`...| +00000380 6d 58 3e 55 7e 19 a3 59 f5 8b ca 1a f0 c8 ca 1b |mX>U~..Y........| +00000390 c6 ed 7f d7 70 fb 4b a5 6a 96 66 ce 4c 96 d1 b4 |....p.K.j.f.L...| +000003a0 de 99 e2 31 4a 88 64 51 7b 73 60 bb 5c 46 9b b7 |...1J.dQ{s`.\F..| +000003b0 9f d8 97 de 03 06 f3 3e 34 8b 75 43 ce cd 9a 12 |.......>4.uC....| +000003c0 1e 27 b5 28 5e 8f b6 04 d2 02 19 02 02 72 85 46 |.'.(^........r.F| +000003d0 1c cb 5e 63 09 e6 ba 81 69 29 97 5f 31 1e 8f f5 |..^c....i)._1...| +000003e0 1a ed 3b 1e 97 83 bf ee 1f 09 f4 f7 be 32 c7 bc |..;..........2..| +000003f0 b6 7e 70 b9 32 4b 07 99 17 03 03 00 35 67 1c 72 |.~p.2K......5g.r| +00000400 de 27 25 26 1b 12 b2 9a a3 38 5c a7 f7 13 a4 1e |.'%&.....8\.....| +00000410 bf f9 7f 1d 57 af ff 2f 89 4b 75 fc 23 c5 47 00 |....W../.Ku.#.G.| +00000420 cd b7 b6 89 af 4e f0 17 e9 e6 31 cd fb b0 31 19 |.....N....1...1.| +00000430 73 61 |sa| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 02 1e 67 f8 08 c4 15 |...........g....| -00000010 47 a9 da 91 df f1 61 0f 50 21 f6 da dd a9 a6 e4 |G.....a.P!......| -00000020 00 05 29 62 d2 cc fe 14 57 a2 aa 46 16 b3 46 6e |..)b....W..F..Fn| -00000030 82 87 dc bb 1a d9 e7 c6 e9 1a 3b 7e a5 94 9b 7e |..........;~...~| -00000040 bb 07 b8 f3 de cc f1 85 d5 ee ff 0b 5a 19 c1 e3 |............Z...| -00000050 5f 47 f4 81 f4 d3 2d 85 f8 38 90 00 10 54 9a 3e |_G....-..8...T.>| -00000060 56 e4 99 a5 31 b1 dc d4 77 fe 28 3a b4 3e 63 42 |V...1...w.(:.>cB| -00000070 bc 05 c7 8a e5 d9 01 5c c9 18 39 1e 62 4f b4 58 |.......\..9.bO.X| -00000080 d3 9a 6a e1 a4 d3 ef 7f b8 0f 35 ac 2a 4a ba 77 |..j.......5.*J.w| -00000090 24 1e 24 6e 0e a2 8e f1 ba 5f fe 24 03 ed 1d e7 |$.$n....._.$....| -000000a0 43 2d fb bb 2e a9 a5 9a 18 11 39 99 a1 5a b7 92 |C-........9..Z..| -000000b0 37 91 f2 80 b2 f6 57 87 f9 a5 d9 da 36 b5 db 1a |7.....W.....6...| -000000c0 28 d8 9e f7 92 6f cf 0c 57 9a 95 42 ef 6a 50 f6 |(....o..W..B.jP.| -000000d0 38 4f 74 52 09 34 ca 8a d8 c7 a2 d2 69 bb db 13 |8OtR.4......i...| -000000e0 b1 ef f2 6d d3 f0 dd 5d b7 93 25 87 84 cd 87 6d |...m...]..%....m| -000000f0 c7 24 99 2b c8 02 25 1c 58 bc 98 03 33 15 ee 3e |.$.+..%.X...3..>| -00000100 96 d0 af 82 c7 74 71 ec ef a6 eb 88 20 55 1b fa |.....tq..... U..| -00000110 ea 01 38 5b 68 77 f1 3b 2a e6 d1 96 bc 28 b3 97 |..8[hw.;*....(..| -00000120 47 94 14 3f 6b 73 82 bf b0 ba fe 45 2a 45 45 1c |G..?ks.....E*EE.| -00000130 30 68 f5 74 07 15 18 00 a7 4f 62 df 7c ff 4c 0a |0h.t.....Ob.|.L.| -00000140 93 d6 60 8b 1f 0e 7f d3 88 43 90 f6 18 05 f1 ae |..`......C......| -00000150 ac 04 63 8b 43 b8 11 74 b1 87 e6 bc 2d 72 4a 2e |..c.C..t....-rJ.| -00000160 75 ab 16 5e 0e bc 76 6d d6 0e bc 39 0d 3c 76 01 |u..^..vm...9.fDcKC.%(D.-.| -000001b0 39 0e cd 15 a3 04 d9 db 4f a4 03 cb d2 84 da 41 |9.......O......A| -000001c0 5c 60 bf 00 77 19 68 b1 50 99 c2 88 d2 56 1e 29 |\`..w.h.P....V.)| -000001d0 c9 8c 83 98 7a 2f 04 e6 d6 09 5c 27 af a7 69 c4 |....z/....\'..i.| -000001e0 f7 4a ed b5 db d3 7a aa 75 f1 b2 8d 87 15 9a d1 |.J....z.u.......| -000001f0 00 fa 7f 58 06 2d 72 2b f7 27 b3 5b 76 57 21 35 |...X.-r+.'.[vW!5| -00000200 80 ce 8e bd e4 bb 45 01 c4 6b 43 f1 44 0a 5f 21 |......E..kC.D._!| -00000210 76 48 ae ce 8e 1d ba f7 7c 4f ae a2 d8 77 ce 9c |vH......|O...w..| -00000220 6d 69 b7 1e 78 ab 02 ed 15 17 03 03 00 a3 84 ed |mi..x...........| -00000230 ba 37 b6 6f 96 68 ef 9e b7 c6 23 5a 9f 1b ed ad |.7.o.h....#Z....| -00000240 3b 5d ab d4 22 d8 3e ab db db c5 b9 57 f8 68 d1 |;]..".>.....W.h.| -00000250 93 d5 36 fe 0c c0 fe 29 88 8b 63 ac 0e 06 4e bd |..6....)..c...N.| -00000260 9e b9 24 65 b5 9c e7 b9 58 4c 8a 07 10 9c 17 f7 |..$e....XL......| -00000270 c0 67 af ff c8 ff fc 87 1b fa c8 21 21 17 2d 43 |.g.........!!.-C| -00000280 f5 fc 4f 0d bf 01 58 b6 f1 58 08 39 f4 0d 94 69 |..O...X..X.9...i| -00000290 8f f0 c1 14 93 41 56 32 41 11 84 58 73 13 69 2a |.....AV2A..Xs.i*| -000002a0 ed 2a 34 61 73 8d 47 41 62 33 39 66 fa 3d 2a e5 |.*4as.GAb39f.=*.| -000002b0 bf 09 d6 c0 1e 3c 98 b3 86 a6 87 b5 a7 d2 cf d9 |.....<..........| -000002c0 dd f8 2e 86 f7 13 84 4a f7 3b ec 8e e5 06 f5 cd |.......J.;......| -000002d0 42 17 03 03 00 35 b2 38 87 30 58 9e 03 6e 44 dd |B....5.8.0X..nD.| -000002e0 fb 87 11 3a a0 e7 c1 2d 74 3b 35 d0 3f bc de cd |...:...-t;5.?...| -000002f0 71 61 8b 7c a5 7e c6 2d 76 67 44 9e 75 e5 9b 3b |qa.|.~.-vgD.u..;| -00000300 c5 2b 42 8a 4a 7f 0e 12 4c 2e 0f 17 03 03 00 17 |.+B.J...L.......| -00000310 d3 61 ed 75 dd 2e ee dd 79 fe d1 7c 4d 23 b1 95 |.a.u....y..|M#..| -00000320 ea 14 d6 27 d0 02 46 17 03 03 00 13 84 c1 07 6f |...'..F........o| -00000330 1c c6 22 a8 ae 6d a8 e8 62 54 ac b2 53 57 bb |.."..m..bT..SW.| +00000000 14 03 03 00 01 01 17 03 03 02 1e e9 c3 0a e9 35 |...............5| +00000010 c0 d7 d8 1e 01 13 31 2c 39 0c 31 57 df f2 8d 7f |......1,9.1W....| +00000020 ca 2e 8c a0 63 60 e2 61 c6 37 8f cf 66 62 e7 d6 |....c`.a.7..fb..| +00000030 3d 89 6b 3b bd 1c 45 ec 62 a5 1c b3 86 16 bc 03 |=.k;..E.b.......| +00000040 57 9e 2b cd 31 d4 4d ad 57 96 12 12 19 ce 61 14 |W.+.1.M.W.....a.| +00000050 d6 1f 7a d1 c8 9b d4 9e 18 cf 1c f7 d0 ca 84 bb |..z.............| +00000060 8e 93 85 97 4b 53 68 3d f6 45 76 e0 a7 42 df 6a |....KSh=.Ev..B.j| +00000070 96 04 b9 9e 03 97 bb 57 bf 7f 9c 27 68 ba 04 49 |.......W...'h..I| +00000080 a7 c9 a3 f3 77 83 1f 15 0e 02 05 b5 8c e4 a6 ed |....w...........| +00000090 f8 65 5b bb a4 65 08 a1 a3 34 2a 71 66 6c 2e c8 |.e[..e...4*qfl..| +000000a0 e2 5b 08 ca 59 91 da 07 14 61 17 b7 19 28 00 33 |.[..Y....a...(.3| +000000b0 d9 0c 45 df d8 74 83 2e 9b 99 c7 35 cd f4 de 53 |..E..t.....5...S| +000000c0 b0 df 5c 8b 15 b0 e0 81 77 a5 5d d5 33 40 51 60 |..\.....w.].3@Q`| +000000d0 09 36 4f b6 6f ce 95 2b d1 38 88 33 e0 13 a0 c1 |.6O.o..+.8.3....| +000000e0 5f 2c 15 77 56 d1 16 09 9a 78 5c 6f 8c 93 c3 44 |_,.wV....x\o...D| +000000f0 54 04 b4 d3 23 31 ff 61 74 e5 09 3a d3 9b f1 2f |T...#1.at..:.../| +00000100 de f4 a9 0b b8 9d ca 17 2c 3f d5 2b dd c0 9e 07 |........,?.+....| +00000110 48 50 6a 51 72 be 92 10 1a 91 b5 2d 39 76 10 a4 |HPjQr......-9v..| +00000120 42 63 31 df ce 31 9a 5c 0d ee 6e 55 c0 ba 44 9f |Bc1..1.\..nU..D.| +00000130 e4 75 77 0f 88 6d 1d 24 9f 1c a0 12 14 4c ce 68 |.uw..m.$.....L.h| +00000140 24 a5 aa f2 f2 c5 f4 4f a9 c7 e0 dd 5c 6b 4c 53 |$......O....\kLS| +00000150 b5 26 8b e2 a4 af e3 13 77 0a fc dd 2b 56 fa a2 |.&......w...+V..| +00000160 46 7f 9a 11 c6 a0 4c b6 36 29 93 9f 83 99 13 ad |F.....L.6)......| +00000170 a0 62 9a ef 42 d0 dd 49 fe ba e4 fd fd 09 4b d4 |.b..B..I......K.| +00000180 31 6e d9 a5 0e 4d ac 8c 90 7c 26 2b 6b 1a cb 55 |1n...M...|&+k..U| +00000190 47 53 47 e2 d4 a1 d8 2e 5d 9b 36 75 f0 a4 8a 39 |GSG.....].6u...9| +000001a0 88 07 c8 ed 75 40 fd 72 0e 57 02 a6 bf 8a 64 0a |....u@.r.W....d.| +000001b0 98 8a 1c f4 d3 3d c4 af 7d 97 6e b7 e0 f6 d8 10 |.....=..}.n.....| +000001c0 17 0b d1 5c f9 41 c9 5e 5c 8a 38 dd 66 b3 e9 74 |...\.A.^\.8.f..t| +000001d0 41 2a bd 2d f8 6a 27 57 da ef d6 ca 70 49 c6 e2 |A*.-.j'W....pI..| +000001e0 94 ef 1e c6 57 a0 c8 fc d1 05 4f 65 c1 71 e4 ab |....W.....Oe.q..| +000001f0 44 49 f3 e3 6d b1 f4 5a 69 e7 70 30 b5 81 19 3a |DI..m..Zi.p0...:| +00000200 06 33 ef a5 bd 81 8c 3c 7b 2f 9f 41 a5 7d a8 5f |.3.....<{/.A.}._| +00000210 3b 3d 95 6e 04 a8 5f 2a c1 de 6e e4 14 17 91 36 |;=.n.._*..n....6| +00000220 9d 57 9c 34 1a 7d c8 c3 42 17 03 03 00 a3 a0 20 |.W.4.}..B...... | +00000230 06 7e 90 c8 0a b2 db 90 28 38 8f 42 12 89 1d 21 |.~......(8.B...!| +00000240 04 b3 a6 7b 8c 63 60 49 33 4a 70 4b 1a 97 90 51 |...{.c`I3JpK...Q| +00000250 d2 13 6e 56 9a 1d 41 9c 65 19 31 43 56 96 e2 50 |..nV..A.e.1CV..P| +00000260 77 17 67 6e df 3b e1 43 22 fe 0c 49 74 ff b0 e9 |w.gn.;.C"..It...| +00000270 5c f2 66 4e a3 e5 ac 02 1c 69 11 ce de a1 d4 6b |\.fN.....i.....k| +00000280 1c 46 97 02 7e 81 d8 29 78 42 82 5e b7 e2 31 02 |.F..~..)xB.^..1.| +00000290 fb 74 e2 a6 5e 29 f1 28 cf 8a 9c bd f8 a0 09 45 |.t..^).(.......E| +000002a0 67 7e d7 7d 91 8d fe 43 03 92 16 b1 67 cc 7a 32 |g~.}...C....g.z2| +000002b0 7e 3a 88 fc 77 36 b5 69 07 09 a0 1c 5c 76 a7 68 |~:..w6.i....\v.h| +000002c0 93 1a 35 20 f6 c1 20 c7 8f de c9 da 81 25 7a 08 |..5 .. ......%z.| +000002d0 e0 17 03 03 00 35 fa 5b 68 55 45 ee dd d6 a2 cb |.....5.[hUE.....| +000002e0 09 a7 21 23 43 9c c4 ff 1c d6 66 4c ce e1 bd a9 |..!#C.....fL....| +000002f0 77 22 0b b0 d4 ad f9 23 a7 1c 0e 63 27 36 16 c6 |w".....#...c'6..| +00000300 a2 f8 00 ef 36 22 89 a7 d6 76 da 17 03 03 00 17 |....6"...v......| +00000310 ca d3 97 3a 7d cc a0 95 02 4a a8 d8 dd 6d 26 64 |...:}....J...m&d| +00000320 92 be 64 86 00 d9 bf 17 03 03 00 13 18 1e 4f 7b |..d...........O{| +00000330 f9 1a 63 8a 50 6a ee 81 07 76 53 a9 95 ea df |..c.Pj...vS....| diff --git a/src/crypto/tls/testdata/Client-TLSv13-ClientCert-Ed25519 b/src/crypto/tls/testdata/Client-TLSv13-ClientCert-Ed25519 index b524ef43d29dd0..38382edbea7e63 100644 --- a/src/crypto/tls/testdata/Client-TLSv13-ClientCert-Ed25519 +++ b/src/crypto/tls/testdata/Client-TLSv13-ClientCert-Ed25519 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,117 +7,118 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 7a 02 00 00 76 03 03 28 10 3b 44 b1 |....z...v..(.;D.| -00000010 0f a9 77 89 f9 dd 3c 3a ad 83 a0 ca 3e 60 d6 0e |..w...<:....>`..| -00000020 ab 94 3a ec 2e 63 ef b1 41 90 48 20 00 00 00 00 |..:..c..A.H ....| +00000000 16 03 03 00 7a 02 00 00 76 03 03 72 63 6c 83 50 |....z...v..rcl.P| +00000010 e9 4c 88 85 70 4f e3 cb c7 0e 0c c5 e5 44 b9 7e |.L..pO.......D.~| +00000020 4a 88 d3 32 d2 37 5c 16 d4 a1 e5 20 00 00 00 00 |J..2.7\.... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000040 00 00 00 00 00 00 00 00 00 00 00 00 13 03 00 00 |................| -00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 d9 |..+.....3.$... .| -00000060 ce 37 c6 7e 9c 0f 75 bb b6 91 c9 af 59 eb 24 07 |.7.~..u.....Y.$.| -00000070 2e 80 3a 7e d2 fc 39 9e 45 19 94 2d ae 33 2c 14 |..:~..9.E..-.3,.| -00000080 03 03 00 01 01 17 03 03 00 17 c6 b7 f1 35 ee 93 |.............5..| -00000090 75 ca 2e 3a a2 3d ed 32 5c e1 2c e0 5b 1c 01 e0 |u..:.=.2\.,.[...| -000000a0 bb 17 03 03 00 42 02 e5 db dc 80 e1 c2 6f 31 1c |.....B.......o1.| -000000b0 33 7b 76 8f 51 9c f8 a8 b0 1b 1a 8a 46 00 c7 ff |3{v.Q.......F...| -000000c0 ff 4c c9 d8 28 14 32 74 ee 59 b3 15 4b 45 90 b6 |.L..(.2t.Y..KE..| -000000d0 da 56 b3 ef d4 6f 8a eb 96 b9 94 ab b9 9e b1 2e |.V...o..........| -000000e0 fe 29 f2 0d 0d 93 66 1a 17 03 03 02 6d 84 4e 43 |.)....f.....m.NC| -000000f0 92 85 d5 e0 7a cd da a6 c0 5a bf 8b eb 83 db a0 |....z....Z......| -00000100 dc c3 e2 a8 d5 ca ad 87 75 19 08 dd 3e 2e 36 a0 |........u...>.6.| -00000110 a1 53 28 09 cd 1c 1d a0 24 77 e3 e6 4d b1 8f 36 |.S(.....$w..M..6| -00000120 43 29 79 b3 e2 fb 9e dc 39 90 b5 b7 db 6d 0a c0 |C)y.....9....m..| -00000130 55 d2 1c fd 8d 57 5b bf e3 70 b7 6a bc af 4f 34 |U....W[..p.j..O4| -00000140 e7 e1 46 d3 9a cc e4 73 33 fb e6 ad f9 b5 35 af |..F....s3.....5.| -00000150 97 77 88 11 19 7c e6 7d 8b 08 bb 78 2c 50 29 0c |.w...|.}...x,P).| -00000160 87 5f ef 04 19 34 9f 59 54 df 14 71 d1 5f e7 dc |._...4.YT..q._..| -00000170 46 2e 73 1a 07 12 be 6e 74 e1 8e 82 ca db a2 6d |F.s....nt......m| -00000180 35 5b fc 4c 71 42 45 86 35 1c c0 dc 0d 4c 40 a8 |5[.LqBE.5....L@.| -00000190 c7 ea 4a b8 33 0b 30 e2 6d a4 82 dc 28 35 1f f2 |..J.3.0.m...(5..| -000001a0 ac 23 41 10 a4 d4 1a 40 28 4d ea db 32 d5 06 de |.#A....@(M..2...| -000001b0 1e 37 91 77 34 ed b4 c8 f2 1c 44 de ee 2f dd af |.7.w4.....D../..| -000001c0 61 cc 1e 59 87 a6 be 36 b0 8e a3 61 01 aa f4 60 |a..Y...6...a...`| -000001d0 1e c8 3a 3c 53 57 31 28 3e 40 32 95 67 02 41 9c |..:@2.g.A.| -000001e0 58 28 50 51 5e 06 1d f1 57 e1 39 ba e0 ab 30 26 |X(PQ^...W.9...0&| -000001f0 25 fe b6 0a a9 d5 b3 72 61 3a 71 ec a2 44 91 5e |%......ra:q..D.^| -00000200 30 1b d4 dd 07 cf 35 6a 85 4a 95 b2 a9 f5 5c 6e |0.....5j.J....\n| -00000210 35 f9 07 7b d9 b0 a5 64 2f c3 34 23 84 3f 1a 97 |5..{...d/.4#.?..| -00000220 dd 25 75 7c 76 0f 9b 73 5b aa c0 a1 00 4b f6 b1 |.%u|v..s[....K..| -00000230 61 34 51 d0 d1 3b 00 4f 87 93 f5 45 2e 23 93 ee |a4Q..;.O...E.#..| -00000240 e6 ce e5 1c 02 22 3c c1 22 ed 33 b2 71 99 52 26 |....."<.".3.q.R&| -00000250 70 86 ad bc 41 d9 13 54 d3 3e 5d 4b 63 4d d2 6f |p...A..T.>]KcM.o| -00000260 bf a5 24 f8 7b 83 b2 0b 13 02 0d 42 74 9b 04 4e |..$.{......Bt..N| -00000270 8a 86 6b 40 92 42 3a d8 a8 21 2f ce e0 90 71 65 |..k@.B:..!/...qe| -00000280 5f ed c7 65 38 e3 85 e2 32 d5 4b c0 84 61 24 a1 |_..e8...2.K..a$.| -00000290 00 be 67 f3 59 b0 b2 f0 65 fe a6 49 19 41 f9 9b |..g.Y...e..I.A..| -000002a0 40 3a 62 ec 4d 1f 83 77 03 9d 4b 4c d2 5d 93 ca |@:b.M..w..KL.]..| -000002b0 01 39 44 dc c6 f7 18 6b 2a c9 1b 3d ef 27 6d a4 |.9D....k*..=.'m.| -000002c0 2e 8f 33 63 2b c5 fb bb 52 23 68 7e 0b 16 92 64 |..3c+...R#h~...d| -000002d0 d9 72 04 2d 0d b2 ba 67 45 ae 14 16 f1 57 3b 59 |.r.-...gE....W;Y| -000002e0 44 ba db b0 c9 d2 9b 8f 6d b2 a4 8c c2 d1 1c e4 |D.......m.......| -000002f0 ce d3 32 bf 04 d9 09 d1 28 3d 25 2d b5 d3 da 12 |..2.....(=%-....| -00000300 77 01 45 09 08 e0 df cc f3 4c c0 e0 b7 d7 d8 16 |w.E......L......| -00000310 82 19 1d 57 dc b4 27 83 48 1b 9f a9 0e cc d6 39 |...W..'.H......9| -00000320 c1 4a 66 b3 42 85 a5 a7 14 97 5c 58 fe 41 e1 2e |.Jf.B.....\X.A..| -00000330 16 e9 70 89 e7 98 a8 f3 4a 81 d4 d8 ff 5f 48 6e |..p.....J...._Hn| -00000340 60 c6 4a 07 c5 57 9c 22 25 3a 9f bf 7e 71 f3 0d |`.J..W."%:..~q..| -00000350 71 e5 a5 d4 2b 6d 70 2f c5 a3 17 03 03 00 99 b9 |q...+mp/........| -00000360 15 c7 70 bc 06 75 26 50 0f 22 90 43 99 25 0d 0f |..p..u&P.".C.%..| -00000370 05 d3 ce e7 3c 53 fb 01 d4 d2 e8 fb b7 49 87 03 |....K......#..._C.| -000003e0 75 65 ac ba 46 43 61 b4 3d be b8 49 44 ad 15 00 |ue..FCa.=..ID...| -000003f0 3b c0 07 ba 18 b2 14 71 17 03 03 00 35 d1 22 81 |;......q....5.".| -00000400 7f 21 e9 e2 7b 11 f5 df dd 3f 15 ce b5 fc 71 f3 |.!..{....?....q.| -00000410 0e f2 d0 1c a4 85 29 88 f2 ca 73 01 ee 1c e4 66 |......)...s....f| -00000420 07 fa 5c a3 32 b0 e4 2a e4 97 e5 70 b1 cd db 79 |..\.2..*...p...y| -00000430 83 1f |..| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 d7 |..+.....3.$... .| +00000060 fa 97 47 72 9a 7e 63 47 d6 62 2d dc 5d 4f 7f 9b |..Gr.~cG.b-.]O..| +00000070 aa 86 c9 ba 65 4c ae 91 82 40 38 ad 4b 46 73 14 |....eL...@8.KFs.| +00000080 03 03 00 01 01 17 03 03 00 17 27 2e 49 01 db 5e |..........'.I..^| +00000090 e1 45 48 97 54 02 31 9f ad 62 74 58 70 77 21 5b |.EH.T.1..btXpw![| +000000a0 81 17 03 03 00 42 2c 0a 8a a0 e0 40 32 b1 2f 9f |.....B,....@2./.| +000000b0 e6 13 9f fd 62 88 a7 34 bd 9a bc 64 5d 9d 17 b9 |....b..4...d]...| +000000c0 b0 f6 fc 84 ca 3f 80 0b 97 4f 7a d9 67 42 c5 0a |.....?...Oz.gB..| +000000d0 e5 18 21 e8 08 42 b4 e6 f5 7e c3 70 2c 0e d3 d8 |..!..B...~.p,...| +000000e0 15 36 37 57 1a d5 58 64 17 03 03 02 6d 07 11 32 |.67W..Xd....m..2| +000000f0 d3 1e 03 02 fb 04 35 f2 64 a8 78 c2 b2 f7 7d 5c |......5.d.x...}\| +00000100 7c 94 0c 7b 16 9a 87 7d 2f 96 de 01 74 d6 6c 6f ||..{...}/...t.lo| +00000110 ce 95 eb f0 df 6d 52 89 3a 19 ff 5b b5 f2 d8 6d |.....mR.:..[...m| +00000120 e6 10 94 f7 d9 c4 58 e8 80 db f5 30 22 b1 82 b1 |......X....0"...| +00000130 66 05 b5 d0 71 40 0f 68 83 ec 43 b5 51 c3 f8 ce |f...q@.h..C.Q...| +00000140 e9 71 4a c7 cf 57 b5 53 3f 60 99 ae 84 df 98 cc |.qJ..W.S?`......| +00000150 9f 90 d0 fc 1d 03 e9 80 72 7c 60 51 a1 89 93 6d |........r|`Q...m| +00000160 0a 57 18 c6 dc 22 82 71 be 66 87 93 dd 16 41 c8 |.W...".q.f....A.| +00000170 84 38 33 63 fc 82 db 38 63 f8 84 f7 12 08 3b 82 |.83c...8c.....;.| +00000180 18 cb c0 50 0d dd 19 25 16 88 23 97 35 56 6d 46 |...P...%..#.5VmF| +00000190 3d 75 e1 83 c3 62 e4 19 70 6c 03 f0 33 5d 94 ad |=u...b..pl..3]..| +000001a0 6d be d2 db c4 b8 ad d8 78 78 53 76 62 91 f7 cf |m.......xxSvb...| +000001b0 83 5b 1e 44 11 2f 27 6a 29 d4 ea 96 fb 40 1c 94 |.[.D./'j)....@..| +000001c0 69 c2 cc c7 90 2c 60 14 c7 d4 f2 9c f9 0e 66 1c |i....,`.......f.| +000001d0 08 76 6e 9f 3b 3a 47 8a 40 0a de 00 e4 6f 45 ca |.vn.;:G.@....oE.| +000001e0 1d 41 cc 34 5a 2c 67 78 58 34 eb 19 0c a5 0e a2 |.A.4Z,gxX4......| +000001f0 fb c1 0a 25 74 f5 ec 55 f8 c3 97 00 d0 a5 90 c5 |...%t..U........| +00000200 a1 9c aa 19 2b e6 ee c1 9d 73 a1 3c 1f fa 6a 91 |....+....s.<..j.| +00000210 2b 2d 27 be 06 f3 85 54 63 a5 d9 ac 55 73 0a e4 |+-'....Tc...Us..| +00000220 4f dc 25 a0 9f 39 c0 0e 1a 9d a7 4c bd c9 3c 64 |O.%..9.....L..>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 01 50 dd d8 5d 40 64 |..........P..]@d| -00000010 5d 35 b0 82 0d 46 97 4c a2 d1 4d dd 3c c9 0a f9 |]5...F.L..M.<...| -00000020 39 6a 1c 17 c3 f6 1a 6b 42 cc f7 37 23 7a e1 bf |9j.....kB..7#z..| -00000030 de 51 7f 87 6b 3d d6 4d ed 24 b3 de dd 62 18 4a |.Q..k=.M.$...b.J| -00000040 b5 b3 1e 0f 2f a3 c5 e8 67 65 04 bc 37 97 15 a8 |..../...ge..7...| -00000050 a8 f3 a8 79 99 82 e1 10 5d d7 f2 ee 21 46 aa b8 |...y....]...!F..| -00000060 40 9f 3b fc 12 19 c8 82 b4 8e bc b8 9b a9 f6 aa |@.;.............| -00000070 18 6b 48 4e 03 92 43 12 b8 54 f2 b7 29 0b 0a a1 |.kHN..C..T..)...| -00000080 6f c4 1c 2a af 22 c1 3d 1b 43 a5 3c 7d 39 83 5d |o..*.".=.C.<}9.]| -00000090 ee 09 fb f1 9a 4c ff b3 2a bb 85 53 04 18 5e 18 |.....L..*..S..^.| -000000a0 62 e8 cf ec 01 be 76 31 82 1a 60 19 33 c7 37 0a |b.....v1..`.3.7.| -000000b0 fa 45 4d b1 9a 1e 07 e9 9c 45 c3 89 60 a2 c9 59 |.EM......E..`..Y| -000000c0 8c 7d 02 f6 d6 5c 12 26 bd 6d 97 15 e2 07 27 be |.}...\.&.m....'.| -000000d0 2f 59 93 1b 11 0e 05 ef 83 c6 a2 c0 1a 9a 4c 60 |/Y............L`| -000000e0 a6 84 c0 72 b6 2b 7c a1 37 4c d7 eb 76 0f 00 97 |...r.+|.7L..v...| -000000f0 43 b6 25 ed 36 e1 eb 3a 3d 71 05 d8 d0 f9 80 18 |C.%.6..:=q......| -00000100 b2 9b c0 4e 2c 8e 05 41 52 a5 40 57 3c 1f 48 0c |...N,..AR.@W<.H.| -00000110 55 ea d3 76 6d 3a 26 5d 5d 71 74 1c d5 8d 96 92 |U..vm:&]]qt.....| -00000120 79 c5 9c 78 68 5f 20 12 01 06 c3 52 49 66 e4 5b |y..xh_ ....RIf.[| -00000130 41 09 fa 72 c5 d7 d9 26 1b fa a5 1d 23 b2 4a 1e |A..r...&....#.J.| -00000140 da 83 5e 89 c8 3a c4 26 f2 42 70 6d 1b 90 67 eb |..^..:.&.Bpm..g.| -00000150 37 15 35 b9 a2 b0 2b ef 93 5f 11 17 03 03 00 59 |7.5...+.._.....Y| -00000160 b4 a0 79 ba e4 3f 78 d2 11 1a 71 8a a1 a4 28 fc |..y..?x...q...(.| -00000170 44 c7 bc 32 19 06 71 1b 6d 50 20 26 d5 91 39 48 |D..2..q.mP &..9H| -00000180 59 8b fd 55 0a 35 14 f5 31 47 62 15 ee c1 69 23 |Y..U.5..1Gb...i#| -00000190 eb db 79 ef 20 47 5c ef 31 e1 ae bd 2f 72 f8 9d |..y. G\.1.../r..| -000001a0 92 cc 04 6f 73 22 5d 93 59 9f 06 cb bd 70 9b d8 |...os"].Y....p..| -000001b0 c9 41 44 8f 5b b2 59 2b a9 17 03 03 00 35 22 65 |.AD.[.Y+.....5"e| -000001c0 ef 1e dc 4c 63 ba 6d 6a 46 1c d2 9c 9a 81 4b f3 |...Lc.mjF.....K.| -000001d0 1f 58 4e 08 86 93 f8 3a 53 13 64 1f ca 20 d2 da |.XN....:S.d.. ..| -000001e0 6b df 54 88 a8 f3 af 0d 4a 1b 29 91 3a bb 90 27 |k.T.....J.).:..'| -000001f0 29 3a 97 17 03 03 00 17 09 78 1d f3 95 2e 92 af |):.......x......| -00000200 d8 af 64 59 fb 77 e3 85 de f0 37 be 11 91 2f 17 |..dY.w....7.../.| -00000210 03 03 00 13 4e a2 06 c3 90 6f d3 b0 04 a8 25 32 |....N....o....%2| -00000220 09 65 1c f2 ef 9c 32 |.e....2| +00000000 14 03 03 00 01 01 17 03 03 01 50 90 77 22 64 76 |..........P.w"dv| +00000010 a1 cd 65 49 53 cb f0 2e 21 6c 12 7c 63 e5 ff 49 |..eIS...!l.|c..I| +00000020 6c be e6 ba 70 2d 22 49 6e 5c 5b 65 46 4b 64 01 |l...p-"In\[eFKd.| +00000030 f7 1d 57 a3 5f d5 a3 39 b6 b9 79 62 b3 e2 52 35 |..W._..9..yb..R5| +00000040 95 26 d9 2b 72 39 60 23 c4 5f 7e 88 66 d4 c8 2b |.&.+r9`#._~.f..+| +00000050 2d 89 38 c8 b8 bd 73 0a 02 27 92 ab 89 8a ee 9b |-.8...s..'......| +00000060 a2 bc 9e 55 fd a0 d4 f3 02 d7 d6 4c 6c 49 ef 7e |...U.......LlI.~| +00000070 35 56 90 41 90 53 24 f5 c1 19 f0 ff a0 c8 7c 82 |5V.A.S$.......|.| +00000080 52 01 98 8c ef 87 69 09 d2 17 35 af 9c 4b 3b b3 |R.....i...5..K;.| +00000090 6c 36 70 63 48 a2 88 df cb 87 f3 40 03 a8 b0 64 |l6pcH......@...d| +000000a0 cc 66 fb cf ba b9 be 8c 67 2d 3c 99 ac 7f 93 c1 |.f......g-<.....| +000000b0 15 2e 05 ae 95 51 3d 71 d9 43 09 d7 44 cb df 67 |.....Q=q.C..D..g| +000000c0 b8 b3 33 62 c5 60 cf 22 20 e5 45 17 7c d6 74 12 |..3b.`." .E.|.t.| +000000d0 0f 6d af d0 db dd 91 ad 20 ea 4d 26 fc c4 2b bd |.m...... .M&..+.| +000000e0 ec 3d 75 c8 87 36 b2 d0 1e cc 1d 92 fd 58 5e d5 |.=u..6.......X^.| +000000f0 f7 ad c2 ed 0c b6 c6 bc 25 ff 39 75 ee cf fc 76 |........%.9u...v| +00000100 77 e0 15 2e 22 82 3b 6f 93 e4 e9 a1 5a bb 9d 78 |w...".;o....Z..x| +00000110 e2 3d 18 c8 e5 ea e3 82 52 de bf ca 32 c9 56 21 |.=......R...2.V!| +00000120 ba c9 2b 12 7d 7b f5 18 73 e5 5e 9c 1e e1 23 92 |..+.}{..s.^...#.| +00000130 ec 58 56 e5 a4 aa dc 2b 59 75 71 19 06 77 83 d7 |.XV....+Yuq..w..| +00000140 0b 28 03 e9 fa 2c 89 1a 8c 64 f0 84 b6 13 f0 02 |.(...,...d......| +00000150 22 02 33 cf c4 22 dd fb b0 76 8a 17 03 03 00 59 |".3.."...v.....Y| +00000160 9b 4c 67 95 f4 37 c9 2c b7 33 c7 78 1c e0 1b 49 |.Lg..7.,.3.x...I| +00000170 41 6f 88 2d 99 a7 e3 d3 d1 d1 f6 36 b1 2a 8e df |Ao.-.......6.*..| +00000180 11 b6 8d 04 63 c7 11 49 e2 8c 79 03 6b b8 a4 8b |....c..I..y.k...| +00000190 2a 2c 04 ab b3 e2 50 03 51 77 65 eb 3a 45 f1 ce |*,....P.Qwe.:E..| +000001a0 65 9f 9c be d6 be 7f 28 14 6a 61 37 55 94 8f a7 |e......(.ja7U...| +000001b0 23 2f bc fc bb aa 89 47 76 17 03 03 00 35 f5 be |#/.....Gv....5..| +000001c0 e6 33 9b 48 97 74 b0 ad a4 fc 78 fe 20 ab cd fa |.3.H.t....x. ...| +000001d0 48 ad 8c 55 c3 46 b8 2e d6 ea f0 79 e3 a5 cf 29 |H..U.F.....y...)| +000001e0 56 19 ab 95 c4 7f fd 89 41 f7 a3 6f e4 19 2f 83 |V.......A..o../.| +000001f0 8c 3e 5d 17 03 03 00 17 74 24 c5 8d 30 40 0a 12 |.>].....t$..0@..| +00000200 57 37 d0 27 30 3e 24 ea 81 ca c7 0a f5 12 cf 17 |W7.'0>$.........| +00000210 03 03 00 13 ff 20 1a 76 60 da a1 15 22 16 10 8a |..... .v`..."...| +00000220 9f e1 ee 5d 62 a2 cc |...]b..| diff --git a/src/crypto/tls/testdata/Client-TLSv13-ClientCert-RSA-ECDSA b/src/crypto/tls/testdata/Client-TLSv13-ClientCert-RSA-ECDSA index 0cd5b76d660b4e..10c83c4828d005 100644 --- a/src/crypto/tls/testdata/Client-TLSv13-ClientCert-RSA-ECDSA +++ b/src/crypto/tls/testdata/Client-TLSv13-ClientCert-RSA-ECDSA @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,129 +7,130 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 7a 02 00 00 76 03 03 40 ad 79 0c ad |....z...v..@.y..| -00000010 83 38 58 82 dc b2 5a d1 e2 de f5 43 99 72 8a 93 |.8X...Z....C.r..| -00000020 ba b1 ea 17 21 52 a7 e9 ed dc 48 20 00 00 00 00 |....!R....H ....| +00000000 16 03 03 00 7a 02 00 00 76 03 03 af 7e fd 3d 23 |....z...v...~.=#| +00000010 96 b6 46 26 9c 09 13 f6 80 4f c9 d2 52 d9 df 52 |..F&.....O..R..R| +00000020 0b 7c da 61 52 ba 99 1b 32 5a 0c 20 00 00 00 00 |.|.aR...2Z. ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000040 00 00 00 00 00 00 00 00 00 00 00 00 13 03 00 00 |................| -00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 bd |..+.....3.$... .| -00000060 d2 3a 7e c2 30 84 4f f2 39 7e 26 11 65 49 b9 b2 |.:~.0.O.9~&.eI..| -00000070 ac 63 c8 ef ea 5e f6 23 33 70 ec 16 86 49 03 14 |.c...^.#3p...I..| -00000080 03 03 00 01 01 17 03 03 00 17 16 47 c1 58 83 6b |...........G.X.k| -00000090 66 06 c4 39 42 8f 68 fb 3a 07 9e e7 74 41 0d 10 |f..9B.h.:...tA..| -000000a0 97 17 03 03 00 42 42 b2 84 94 a6 18 c6 b6 21 77 |.....BB.......!w| -000000b0 1e 02 57 4a 14 2d c4 84 17 f1 fa b4 63 0f aa d7 |..WJ.-......c...| -000000c0 4a d5 ae 75 f1 da 14 ec 51 09 93 6b 62 6f 8f 7e |J..u....Q..kbo.~| -000000d0 fe 0c 7f 63 ef 1b 87 d8 43 42 d0 f4 2d 6d 0f cc |...c....CB..-m..| -000000e0 ef 77 79 17 1e 39 1e 75 17 03 03 02 22 9c 82 27 |.wy..9.u...."..'| -000000f0 99 e0 4b 44 a7 a7 bb a9 25 a2 58 7f 70 3f 82 1a |..KD....%.X.p?..| -00000100 4b bc e1 35 3e 66 fa 4f ac 7f 7e da 73 13 06 0f |K..5>f.O..~.s...| -00000110 fe 97 ef 82 53 2e d0 e2 11 c6 ee 8d 25 d2 f0 5c |....S.......%..\| -00000120 2e b8 61 40 09 12 d4 11 9b b8 3d 38 d7 d5 a7 76 |..a@......=8...v| -00000130 ed a7 bf 6f 39 9c 83 49 57 86 de 53 da c5 ff 8a |...o9..IW..S....| -00000140 ee 47 bc 19 72 d3 cc d0 5b fc 6b 25 2f 64 2a 0c |.G..r...[.k%/d*.| -00000150 7d e9 8a 1f f8 48 d1 74 d5 70 97 e7 b4 e2 c8 ce |}....H.t.p......| -00000160 dc 74 27 e9 af b1 64 56 19 28 8a 34 93 4e 1c 03 |.t'...dV.(.4.N..| -00000170 67 ae 41 fb 5f d1 e3 20 45 6b bd 49 25 d8 b4 8b |g.A._.. Ek.I%...| -00000180 d4 96 d6 96 b2 72 72 f0 d0 68 f1 f7 f3 17 76 85 |.....rr..h....v.| -00000190 78 0e dd 35 83 8a 7a 45 c5 91 86 4b a6 b5 77 b4 |x..5..zE...K..w.| -000001a0 ef 02 43 5c 42 c7 f8 a6 1d ed 73 d2 41 02 be cb |..C\B.....s.A...| -000001b0 24 40 61 5a 5f 7a f6 bf f9 d0 bd 05 7f 90 52 16 |$@aZ_z........R.| -000001c0 65 49 c6 74 61 f0 05 fa dd 83 69 b7 ff 5a 2c 4b |eI.ta.....i..Z,K| -000001d0 63 6d d2 57 01 43 67 54 25 90 72 f8 12 7e c2 39 |cm.W.CgT%.r..~.9| -000001e0 26 a9 65 55 02 ca 87 8a e6 13 cd a4 6b eb 3c 4d |&.eU........k.#| -00000250 59 38 6c 9f be 1b ba 60 93 e8 c9 24 fb 13 11 b7 |Y8l....`...$....| -00000260 78 d8 a7 30 6d 06 0f 4d 59 5d d1 1f 81 13 33 99 |x..0m..MY]....3.| -00000270 a8 4d 63 16 53 73 40 22 4b 55 21 79 c1 ce 1c 92 |.Mc.Ss@"KU!y....| -00000280 8c d6 95 14 fd 93 5d c2 54 20 30 c5 79 b0 cd eb |......].T 0.y...| -00000290 ac 1b 50 6a 25 b7 e8 f1 0f da f3 d8 6b 17 2b 12 |..Pj%.......k.+.| -000002a0 06 a8 9a 88 be 2f f3 52 b3 d7 70 05 df 57 88 f9 |...../.R..p..W..| -000002b0 ec 59 d4 e7 ce cb c3 1f 33 3e c0 d8 04 9f 59 a0 |.Y......3>....Y.| -000002c0 04 c0 13 b3 a6 7c ce 6c c4 fa 82 cd da 11 17 b6 |.....|.l........| -000002d0 c2 40 cd e6 5a 6e d3 b5 17 f0 4d c4 34 8f 2a 28 |.@..Zn....M.4.*(| -000002e0 4d ce 93 d5 eb 7e 0a d2 57 0a 98 cf ac 48 ee a8 |M....~..W....H..| -000002f0 8a bd ec c2 d7 d9 39 af e2 a0 9f b2 fd d1 4e 96 |......9.......N.| -00000300 fa fd 34 f7 a4 3e 3d 31 9a f6 a2 a7 97 a3 43 17 |..4..>=1......C.| -00000310 03 03 00 a4 cf 0d e1 8d b7 e4 1a c0 79 0d 7e d9 |............y.~.| -00000320 70 a3 34 57 0f 72 b4 5c 05 7f 1e 16 52 ba d3 e7 |p.4W.r.\....R...| -00000330 fb e5 b8 b0 8d cf 91 7e 96 c4 70 6e cb e5 7e 51 |.......~..pn..~Q| -00000340 19 17 c1 94 e9 49 69 a0 21 59 74 a9 1d 34 9c 10 |.....Ii.!Yt..4..| -00000350 a9 10 f0 4a 51 2c e3 1d e3 59 28 f2 3b ea 66 c1 |...JQ,...Y(.;.f.| -00000360 ed 6a 8d 39 72 ea 5a 24 7a ee 9f cd 85 9d 54 60 |.j.9r.Z$z.....T`| -00000370 d4 95 fd 8d 63 a4 78 6c 2f 7d 6a a5 53 e4 42 97 |....c.xl/}j.S.B.| -00000380 e8 19 36 bf 14 9c a8 48 9f 57 ad 29 7a 30 e5 a9 |..6....H.W.)z0..| -00000390 32 e7 0f 24 9c ca 9d 6d b7 47 46 1c cb 90 7d 18 |2..$...m.GF...}.| -000003a0 39 70 d2 c1 08 28 dc d7 3a 3e 5e 62 a7 4a 4a af |9p...(..:>^b.JJ.| -000003b0 02 93 ce 4b 5d a2 96 f3 17 03 03 00 35 10 07 d6 |...K].......5...| -000003c0 82 f3 28 ef 2f 08 17 f6 0b c2 da 1f b7 52 e1 eb |..(./........R..| -000003d0 ca f4 0b c6 21 5c 54 4d f9 0b 04 c1 5a bf 63 58 |....!\TM....Z.cX| -000003e0 1c e1 da 7f f7 1f 84 77 09 3a 35 7c e6 1f 89 f2 |.......w.:5|....| -000003f0 fd 01 |..| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 83 |..+.....3.$... .| +00000060 95 cf ff 0f f6 eb 66 dc 4f 61 f3 94 43 18 7b f5 |......f.Oa..C.{.| +00000070 91 6a e5 36 75 7b 6b 2c d4 38 f7 4f 36 f0 3c 14 |.j.6u{k,.8.O6.<.| +00000080 03 03 00 01 01 17 03 03 00 17 b7 83 3c 5c 9e ee |............<\..| +00000090 fc c3 14 33 45 62 69 8b 63 25 03 77 fe 7c 8b c0 |...3Ebi.c%.w.|..| +000000a0 0a 17 03 03 00 42 6c cd df 26 24 42 0d 2e 38 64 |.....Bl..&$B..8d| +000000b0 c5 92 7c 6b 18 47 cc 90 09 57 95 0a f9 cc 81 f1 |..|k.G...W......| +000000c0 db 90 29 ba b0 3f db 99 55 78 93 ab 05 34 91 46 |..)..?..Ux...4.F| +000000d0 ce cb c5 2c b8 fb 43 98 31 cc 18 8f 2e 2a 39 78 |...,..C.1....*9x| +000000e0 68 6d 01 29 05 ff 7e 4f 17 03 03 02 22 c1 3d c5 |hm.)..~O....".=.| +000000f0 cb 42 fe 5d a8 ea 65 fd 1a a8 cd a9 28 ed 8d 69 |.B.]..e.....(..i| +00000100 5b eb 28 11 c5 bb 9f 58 8d f2 2d 44 b4 8f 87 d6 |[.(....X..-D....| +00000110 3f af df 3f 13 c0 7e bf 6f b3 e0 fa 45 5c ee a2 |?..?..~.o...E\..| +00000120 13 70 08 94 2d 87 a7 1c 23 a9 aa a1 64 d2 49 ed |.p..-...#...d.I.| +00000130 33 2c ae 02 9e a7 03 24 3f 4c 43 d4 2e 54 b9 fc |3,.....$?LC..T..| +00000140 39 6c 32 8c b1 0c bb f6 31 60 d9 48 82 5b ed 2b |9l2.....1`.H.[.+| +00000150 ea dd e3 2d 1a 35 a3 22 be e1 f1 13 04 9c aa 1a |...-.5."........| +00000160 24 39 4a 0d 63 fb ce 31 71 af e6 1c f3 a3 dd c1 |$9J.c..1q.......| +00000170 51 40 28 5a 11 a0 9e 19 0a a5 74 e2 40 56 9c 55 |Q@(Z......t.@V.U| +00000180 40 45 e6 20 5d 23 aa 85 ec 42 5f 2c 24 10 fb ff |@E. ]#...B_,$...| +00000190 8a 52 e6 33 8f e1 1e e1 51 8a 9f 5d 6f 63 b8 04 |.R.3....Q..]oc..| +000001a0 13 ab fa 5d 85 ba d3 eb cf b9 0b 89 08 b7 7b a9 |...]..........{.| +000001b0 ea 07 b9 41 07 7a 08 7b 57 01 35 11 a1 65 99 4a |...A.z.{W.5..e.J| +000001c0 4c 4a 1f c2 94 5a 00 09 9a 13 37 23 16 60 45 62 |LJ...Z....7#.`Eb| +000001d0 96 fa 6b 7a 0f d6 68 14 f5 cd 40 d7 0a eb ea 75 |..kz..h...@....u| +000001e0 f0 29 cb ea 7f e7 55 36 d9 02 b2 a0 bc 54 ac 04 |.)....U6.....T..| +000001f0 1b 00 10 c0 db 45 81 e6 97 2a 4a 57 1d e7 de e3 |.....E...*JW....| +00000200 d1 0a 09 c7 73 6a 9e a0 3c 79 7f a8 26 6a 98 05 |....sj..B ]^..| +000003f0 a3 91 |..| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 02 11 f1 43 fc 28 7d |............C.(}| -00000010 43 0b 17 e4 ed cc 95 d5 09 4e 8f 44 3c 3b 46 23 |C........N.D<;F#| -00000020 59 a4 62 a7 94 1e da b6 a7 a5 22 15 6a ed e3 eb |Y.b.......".j...| -00000030 4f 6d 0e 91 c6 20 74 af 97 80 95 db 6a b6 2d ac |Om... t.....j.-.| -00000040 a8 d9 95 84 95 5b 32 27 ec 42 42 a8 fd 05 fc ad |.....[2'.BB.....| -00000050 2e 00 e3 4a 9d 93 d1 5e fd dc 22 30 1f 9c e9 cb |...J...^.."0....| -00000060 3f 2e ac ad 0f ab b7 46 82 69 38 db c6 8a be 77 |?......F.i8....w| -00000070 7e 83 77 4c e1 f4 f2 cf c2 b1 86 ff 17 d3 1d 79 |~.wL...........y| -00000080 3e ac f8 6e 73 15 ab 02 4c bf 2a 63 41 ce 93 3b |>..ns...L.*cA..;| -00000090 af 11 70 8c 91 91 f4 14 5c 1b 63 4b 42 3b d8 ca |..p.....\.cKB;..| -000000a0 00 f6 02 b3 c8 d3 ab 01 1d f2 71 86 7b f9 13 de |..........q.{...| -000000b0 1e 39 1c e7 b5 3b b5 bb 26 33 f7 a9 3f 29 e2 8c |.9...;..&3..?)..| -000000c0 d1 67 64 11 b4 29 be 6e c9 cf c5 6e 41 43 41 e7 |.gd..).n...nACA.| -000000d0 43 9c 5d fb a1 b6 b3 f9 0f 4c 17 11 3c 56 0e 49 |C.]......L...z.U.....4..| -00000130 11 86 8a 14 8e d6 5c c0 c2 55 de da 37 c4 ac 69 |......\..U..7..i| -00000140 95 42 84 5f 1d 5f c7 32 91 a9 45 02 c3 a3 da 4f |.B._._.2..E....O| -00000150 96 da c0 a2 09 7a a6 2b 25 f4 a2 94 89 fa bd 0f |.....z.+%.......| -00000160 87 04 e1 87 00 c4 4f 45 39 7a 5a 7d 75 c5 c9 f6 |......OE9zZ}u...| -00000170 27 34 2d ba 12 c5 2c 76 06 b0 aa d0 34 e1 99 e9 |'4-...,v....4...| -00000180 9b 01 db 2a b4 d5 dd 55 2b 92 6d e3 9e fc f1 09 |...*...U+.m.....| -00000190 2b 37 8b 37 8f 70 31 ee d2 f0 86 5a 1f d3 4a 29 |+7.7.p1....Z..J)| -000001a0 7c c9 0e 97 7c 4f e6 87 b7 c8 94 f3 d2 0b 4a 1d ||...|O........J.| -000001b0 59 60 09 81 55 6a 01 38 9f ca 20 e0 6e 44 f7 b5 |Y`..Uj.8.. .nD..| -000001c0 f6 3e 2e d0 81 f6 e6 1d 15 cf c3 d8 ae 98 84 d1 |.>..............| -000001d0 0b c9 e4 65 96 f9 eb c9 60 6d 50 37 49 9f 6d b1 |...e....`mP7I.m.| -000001e0 b9 f6 25 20 81 2d cf b4 17 2b 4c a9 8a 76 8a db |..% .-...+L..v..| -000001f0 77 8c 29 47 05 0b 27 e1 52 8d 37 27 fb c0 70 26 |w.)G..'.R.7'..p&| -00000200 3b 6d 2e 74 05 75 88 e4 63 3d ac 98 3f 74 67 3b |;m.t.u..c=..?tg;| -00000210 c6 dc 41 39 2b 20 ef f9 8b 2f 4e 20 17 03 03 00 |..A9+ .../N ....| -00000220 99 7d 8a 83 dd 34 5e 37 e2 c3 85 22 f3 6b ee bc |.}...4^7...".k..| -00000230 a5 21 1b c5 2c 38 cf 99 09 5b 31 97 5e a0 8b 69 |.!..,8...[1.^..i| -00000240 75 3a 23 64 18 22 fb 31 ed ac 8b 0f ae 9c 4e ee |u:#d.".1......N.| -00000250 06 0e 3c 3a 18 d3 8e 0c dc d4 3f 7b 25 e3 90 85 |..<:......?{%...| -00000260 ca f4 8a 5c 5e 5e 44 f3 1b 57 cd 14 e0 8d 67 36 |...\^^D..W....g6| -00000270 6e c2 a6 95 60 f3 86 20 29 a6 a8 8f b3 b0 0d 57 |n...`.. )......W| -00000280 d2 1e 4a b9 d2 63 33 09 f9 c5 87 94 89 c5 fd 2e |..J..c3.........| -00000290 de b3 f0 0e 7b 4e 3e d6 21 b4 3b 58 bd 2a 35 49 |....{N>.!.;X.*5I| -000002a0 5e b0 f4 5a ca 58 dc a5 8b 45 a5 d1 7b 13 b2 00 |^..Z.X...E..{...| -000002b0 52 c4 b2 6c d1 9a b8 c5 1b fe 17 03 03 00 35 c9 |R..l..........5.| -000002c0 9b c7 94 b3 eb 5f 4e ab 8c c0 de cf b6 b4 5c ee |....._N.......\.| -000002d0 bb 5f fa 48 a2 06 f0 ed f5 b1 ba 74 0c c5 64 1a |._.H.......t..d.| -000002e0 ce 2e 9e 90 4b 6e 98 13 a0 98 99 a3 b4 6f 73 76 |....Kn.......osv| -000002f0 7b e3 26 15 17 03 03 00 17 3e a1 f0 68 e6 00 86 |{.&......>..h...| -00000300 10 f4 bf 26 b6 f5 e6 77 85 ca b7 c1 75 bb f9 78 |...&...w....u..x| -00000310 17 03 03 00 13 43 7f f3 84 ee d7 a9 59 9d c3 5c |.....C......Y..\| -00000320 c0 b7 c8 20 4e 83 ba 23 |... N..#| +00000000 14 03 03 00 01 01 17 03 03 02 11 e0 62 97 8a 4d |............b..M| +00000010 01 cf 8f e2 d2 a8 0d d4 9a fc f5 b2 d7 82 2b 7a |..............+z| +00000020 8c e7 ce 2a 99 3b 41 a2 a3 ca c4 9d 15 79 7f 2f |...*.;A......y./| +00000030 04 7a 18 e4 5b b6 01 d6 b5 03 d2 08 15 41 c7 18 |.z..[........A..| +00000040 8a 66 78 05 ed 33 19 8e 93 9e c3 ff 4a 92 bb 7b |.fx..3......J..{| +00000050 72 e5 81 a6 2a 7f b3 e1 72 67 01 7a 95 f4 3f bc |r...*...rg.z..?.| +00000060 cd e4 bc 45 74 9b 4c 1c 69 68 d0 75 89 9f d5 d0 |...Et.L.ih.u....| +00000070 de 59 d4 1c 47 cc 5e 2e dd bd f9 1f e4 16 c9 c8 |.Y..G.^.........| +00000080 9b 4e 07 6b 7a 12 e7 13 71 ae c1 26 48 32 d6 4e |.N.kz...q..&H2.N| +00000090 8a 15 c2 a0 91 59 9d 21 0d 28 d2 94 3f e9 fc 74 |.....Y.!.(..?..t| +000000a0 98 5b e5 be 50 ad 10 7d d9 a1 da 41 59 15 b3 85 |.[..P..}...AY...| +000000b0 2d 7b 81 b3 ae fb db 4c 00 24 31 57 f2 54 b7 f7 |-{.....L.$1W.T..| +000000c0 64 cc 73 23 bc 6a 93 e0 91 a7 5e 1b 63 f1 59 89 |d.s#.j....^.c.Y.| +000000d0 a9 d9 7b 88 35 17 7a 4d b9 66 d0 a5 f2 d6 79 ed |..{.5.zM.f....y.| +000000e0 c1 3a e7 98 88 96 e7 2f 31 66 bf 16 34 e2 fd 5f |.:...../1f..4.._| +000000f0 fb 0f fb 9a aa ba 78 d8 6a c9 72 d6 39 32 3a 99 |......x.j.r.92:.| +00000100 a2 11 8a 32 79 cf 18 d9 22 da 40 31 3e d3 c8 17 |...2y...".@1>...| +00000110 c4 1f e2 4f c8 7f b9 2f 83 d7 5e 59 48 d3 9b b9 |...O.../..^YH...| +00000120 68 9c c2 e4 45 c2 a3 26 91 cd 3a 26 47 c4 b8 7f |h...E..&..:&G...| +00000130 8f 91 c0 06 b5 6e 5e b4 65 05 42 ea 48 9e 40 bc |.....n^.e.B.H.@.| +00000140 0c 04 22 86 6a 54 6c 27 c3 77 b4 b9 22 99 6a f0 |..".jTl'.w..".j.| +00000150 91 bc 41 ca 24 41 52 fd e2 18 0f 64 13 7e a3 6b |..A.$AR....d.~.k| +00000160 ab 27 1e 15 87 0d 7f 71 1e 29 16 f9 af 81 ec bc |.'.....q.)......| +00000170 28 0b 45 bd 76 fd ff 0e fb 8d c5 0c aa ef a5 17 |(.E.v...........| +00000180 55 49 a6 3d 74 5c 8d 77 60 99 a1 8f aa a9 eb 0f |UI.=t\.w`.......| +00000190 75 1a 55 21 3a 96 da 08 a2 cd ad 11 78 15 6a ce |u.U!:.......x.j.| +000001a0 ef f6 fb 8b a3 dd fd ad 2d a5 2d 59 25 37 fe 53 |........-.-Y%7.S| +000001b0 48 90 fa 9a d3 3c 09 69 47 d3 d1 e4 48 30 fd df |H....<.iG...H0..| +000001c0 15 d7 64 ff ca 91 46 c2 36 82 30 ae 4e 75 12 be |..d...F.6.0.Nu..| +000001d0 58 5d da 63 da bd dc be 81 be ad 37 87 ea 0a 26 |X].c.......7...&| +000001e0 31 cf 1b 1e 7d de a8 04 e3 b8 e5 65 5a 21 db b6 |1...}......eZ!..| +000001f0 2b 7c e7 23 7c 2b e1 89 3f 28 27 97 dd 1c c6 00 |+|.#|+..?('.....| +00000200 0e e4 05 68 0f 9a 8a 1d e6 bd bd aa 1f 46 6e a2 |...h.........Fn.| +00000210 d9 69 91 9b e7 e3 6c 39 33 77 b8 76 17 03 03 00 |.i....l93w.v....| +00000220 99 c9 3e 7e 78 3b 91 65 35 cb 19 44 92 f7 77 f5 |..>~x;.e5..D..w.| +00000230 60 3c 19 2a 97 c5 a0 92 b2 28 e2 44 94 ec 1b 3d |`<.*.....(.D...=| +00000240 f8 1a c1 65 eb 41 3f 61 f1 db 42 1a 0d b8 32 12 |...e.A?a..B...2.| +00000250 f6 1b 83 be 37 d8 fe 78 bd 5b 66 d1 f2 6c 3e e3 |....7..x.[f..l>.| +00000260 8a 0f 3a 28 57 71 1a 78 ab 2a b5 5f ad a3 6e 2c |..:(Wq.x.*._..n,| +00000270 a3 d9 3a 0b d0 99 95 d6 dc 8e 7a f0 b6 e4 cb 46 |..:.......z....F| +00000280 ab cb eb cf ec 86 b3 fe e6 e6 73 2c a2 64 d2 d5 |..........s,.d..| +00000290 9c 8c 25 39 62 07 51 93 12 92 2b e3 4b 2e 3d f5 |..%9b.Q...+.K.=.| +000002a0 f1 d4 22 69 c3 90 cf 91 35 2f e4 60 35 44 6f bf |.."i....5/.`5Do.| +000002b0 7a 75 3d fb 70 bd 20 05 a8 f8 17 03 03 00 35 6e |zu=.p. .......5n| +000002c0 09 00 fa 13 8f d4 17 40 ee 9e 5f 8a 56 ba 7c 69 |.......@.._.V.|i| +000002d0 05 ee 65 8e fd 9c 62 7f f9 af 04 c7 46 20 07 da |..e...b.....F ..| +000002e0 bc 79 f8 cc 53 c6 fc 47 b5 54 9c fb 4d a3 cf 56 |.y..S..G.T..M..V| +000002f0 a4 57 c5 aa 17 03 03 00 17 d6 ea 39 50 a5 8e 67 |.W.........9P..g| +00000300 b9 79 76 17 77 86 a8 58 fb 86 03 74 1a e4 12 37 |.yv.w..X...t...7| +00000310 17 03 03 00 13 e7 07 59 c3 d0 27 b3 d8 e8 a3 7b |.......Y..'....{| +00000320 df e8 17 08 78 4e 9a 6b |....xN.k| diff --git a/src/crypto/tls/testdata/Client-TLSv13-ClientCert-RSA-RSAPSS b/src/crypto/tls/testdata/Client-TLSv13-ClientCert-RSA-RSAPSS index e557181b29550d..4892e2b5555158 100644 --- a/src/crypto/tls/testdata/Client-TLSv13-ClientCert-RSA-RSAPSS +++ b/src/crypto/tls/testdata/Client-TLSv13-ClientCert-RSA-RSAPSS @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,138 +7,139 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 7a 02 00 00 76 03 03 46 b8 dc eb 72 |....z...v..F...r| -00000010 ac 41 0a 3c 58 14 84 a0 75 91 e1 ca d7 3c c7 2c |.A..| -00000070 42 63 2e 2a a9 13 0c a9 f2 e6 74 14 1e 5c 52 14 |Bc.*......t..\R.| -00000080 03 03 00 01 01 17 03 03 00 17 23 71 3e 39 7a 0c |..........#q>9z.| -00000090 2a fa 89 f1 65 d1 b0 23 96 8e 6a e4 55 2c fd 34 |*...e..#..j.U,.4| -000000a0 ea 17 03 03 00 20 b7 a2 d6 ff b3 b4 db ce f8 90 |..... ..........| -000000b0 f9 a3 9c 0c 8f f0 77 3e 37 f2 d9 e7 84 65 28 f6 |......w>7....e(.| -000000c0 dd fe b9 42 03 3c 17 03 03 02 7a d4 cf 93 f6 2b |...B.<....z....+| -000000d0 64 e3 13 75 41 62 7d bd cf 4f 42 80 23 f4 03 92 |d..uAb}..OB.#...| -000000e0 35 02 5e ed bb c6 2f b1 a8 57 cf d4 69 94 6a db |5.^.../..W..i.j.| -000000f0 39 b8 3a 05 07 dd f1 1d 63 13 22 9b d7 58 68 6f |9.:.....c."..Xho| -00000100 4f ac 79 6a 47 0a 23 35 3c 9b 9f 15 04 e7 6b d8 |O.yjG.#5<.....k.| -00000110 ff df 00 6c 68 5e a0 41 6a a9 34 9f 37 98 c2 3d |...lh^.Aj.4.7..=| -00000120 51 29 73 c4 ad 34 e5 69 78 18 eb 03 55 c0 c6 0d |Q)s..4.ix...U...| -00000130 43 d5 4a 5d cb 53 e6 b9 df e8 8b 0e 98 04 89 bb |C.J].S..........| -00000140 f4 a1 48 01 af d3 42 ef 17 e7 f9 27 b4 b1 63 99 |..H...B....'..c.| -00000150 91 a6 c5 c1 cb 6d 1c 55 b1 69 1c ec b1 b7 c3 cd |.....m.U.i......| -00000160 7a 93 f2 d8 e4 c4 96 43 0a e1 18 b5 9a 3b 57 83 |z......C.....;W.| -00000170 98 ce 74 f2 4c 65 ad 01 aa 76 b5 b0 7c a0 7d 09 |..t.Le...v..|.}.| -00000180 ad d6 20 7d 12 67 f8 4c 2c 41 37 34 66 86 fe 3d |.. }.g.L,A74f..=| -00000190 44 52 f0 08 26 4c b3 e7 71 b3 89 b3 5c 78 7e db |DR..&L..q...\x~.| -000001a0 34 31 dc dc 7c 7d 28 e1 41 96 f4 2f 28 60 9f 5b |41..|}(.A../(`.[| -000001b0 b2 2b 37 6b 3f 3a 37 e2 38 7a 66 40 fe 44 44 72 |.+7k?:7.8zf@.DDr| -000001c0 b4 e3 97 f3 db 73 c8 af 96 c8 a6 24 ef ea 34 8f |.....s.....$..4.| -000001d0 5e 4b 21 22 19 d8 4a db 2e f1 23 90 d0 d2 38 f3 |^K!"..J...#...8.| -000001e0 c6 08 5d 1d c9 a3 f2 6d 01 eb 30 5e cd f4 98 f5 |..]....m..0^....| -000001f0 0d d8 68 4f 00 f4 cd 24 9f c4 3c 6a 31 5d c0 e5 |..hO...$..S8B.| -00000350 6f 99 20 c4 98 59 48 61 e0 13 80 8e 96 25 96 f8 |o. ..YHa.....%..| -00000360 c9 c8 35 9c 35 c0 13 d7 34 24 0b 6e 59 4f 1b 77 |..5.5...4$.nYO.w| -00000370 1f 9a 71 6b 6b dc 71 23 b1 a2 e8 1e c2 a0 b1 8c |..qkk.q#........| -00000380 27 5e 90 c5 f8 65 30 a2 de fc 6b b2 e3 5d 8d 9f |'^...e0...k..]..| -00000390 70 32 3c 92 47 c7 91 10 49 0d f1 3b 41 f1 14 3b |p2<.G...I..;A..;| -000003a0 09 55 f5 78 0e 91 1b 75 a0 58 fc c9 34 82 fe 51 |.U.x...u.X..4..Q| -000003b0 20 73 71 3d 7f 82 a4 29 bc 04 67 d7 30 c7 a5 4b | sq=...)..g.0..K| -000003c0 f3 94 4d 15 35 2a 1e 46 82 3f 36 f4 7b ff 9f 88 |..M.5*.F.?6.{...| -000003d0 3b e6 60 bc 86 51 59 ad 97 58 95 7e 6a 4b c9 6c |;.`..QY..X.~jK.l| -000003e0 34 96 03 17 03 03 00 35 03 94 eb e1 e5 9d eb 8f |4......5........| -000003f0 34 b1 28 3c 9d 26 40 a9 e7 4c 2b bf 37 32 b6 aa |4.(<.&@..L+.72..| -00000400 78 d2 45 0c 6c 5b 0a 0e 8a 3a b6 1a 01 09 3e 0a |x.E.l[...:....>.| -00000410 45 25 c6 83 8a 4a cc 22 d9 29 c8 8a 32 |E%...J.".)..2| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 e0 |..+.....3.$... .| +00000060 70 9a 3d 69 69 1f 8f de d4 c8 7c dc 65 53 9a c6 |p.=ii.....|.eS..| +00000070 77 c2 c4 21 1b 06 2d c4 4f 24 61 52 5b 00 69 14 |w..!..-.O$aR[.i.| +00000080 03 03 00 01 01 17 03 03 00 17 96 99 40 9f d8 9d |............@...| +00000090 70 f0 cf f0 86 3e 7f 4a 3a 8a a4 63 6e e1 7f 24 |p....>.J:..cn..$| +000000a0 f4 17 03 03 00 20 72 bb 5a aa 0e 2e d9 cc 38 0c |..... r.Z.....8.| +000000b0 c8 b9 88 08 6f 19 d8 38 14 14 d7 97 42 6c 45 43 |....o..8....BlEC| +000000c0 51 dc 2b f6 02 e2 17 03 03 02 7a 8d 04 2f 64 c0 |Q.+.......z../d.| +000000d0 e2 7b b5 20 9d 04 69 5e 88 82 aa f3 c6 b2 8d 6c |.{. ..i^.......l| +000000e0 41 08 c3 b1 0c 7d d1 5f d6 e6 b7 1d c0 ad 2b 0d |A....}._......+.| +000000f0 02 c5 f7 6d a0 87 91 07 20 d3 d4 2e 5a 7d 5e 29 |...m.... ...Z}^)| +00000100 37 2b ac 5f d6 cb eb e9 99 a7 05 f7 e7 ea 9c 99 |7+._............| +00000110 cd 33 25 f8 45 95 43 f4 7a 59 2a 91 8c e2 ac 84 |.3%.E.C.zY*.....| +00000120 ba 99 ef d0 d7 ea 9b db 31 b0 27 0c 81 3b b2 83 |........1.'..;..| +00000130 20 c4 8b 7e 05 ac c6 82 e9 56 5a a2 b6 7c c7 33 | ..~.....VZ..|.3| +00000140 a8 31 2f ad 60 be ec c3 2d c0 a8 b4 40 50 98 ea |.1/.`...-...@P..| +00000150 83 d1 cc a8 34 20 11 7c 2e f7 7c ca d8 27 03 a2 |....4 .|..|..'..| +00000160 58 86 e8 e6 41 46 07 18 c7 a7 61 f9 cc 2f 7d 1e |X...AF....a../}.| +00000170 bb 34 7c 6e cb b3 9c 03 2f b0 dd e9 a1 32 13 10 |.4|n..../....2..| +00000180 85 1d fa a6 bf c7 4d ec 94 8f 26 07 94 34 b3 5f |......M...&..4._| +00000190 b8 e4 da 6f 3d 5b 0a 61 8f 44 58 10 1a 4c f3 bc |...o=[.a.DX..L..| +000001a0 ff c7 8f 09 0a e4 1b 35 e8 a9 dc 9c 14 86 e5 67 |.......5.......g| +000001b0 7e 96 e9 79 8d b2 d7 34 a0 7f 43 07 11 f1 03 68 |~..y...4..C....h| +000001c0 95 5e f5 fa e7 8f c8 d6 5b 23 c4 84 e4 c5 40 58 |.^......[#....@X| +000001d0 80 4a ac b7 a7 07 21 93 c1 a9 ad 9a 89 f7 f3 9e |.J....!.........| +000001e0 bb 7d 2a 00 d0 e0 66 9d f7 86 4a ba b1 c9 1e 3b |.}*...f...J....;| +000001f0 f1 c6 6e c1 9c 09 85 38 9d fa 5a b9 ca bb a4 f5 |..n....8..Z.....| +00000200 9e 3f ba d4 31 7d aa d7 f9 bf 83 05 5c 1f 61 18 |.?..1}......\.a.| +00000210 d4 5e 16 98 c6 92 7b bd 12 96 18 c7 33 75 ad 08 |.^....{.....3u..| +00000220 d1 0b 47 4c c8 73 3e 68 fb 53 ff e0 1a 10 6c 5a |..GL.s>h.S....lZ| +00000230 7f 9d 92 32 2f c2 2d 95 7d c6 ef 18 9b 44 a1 bc |...2/.-.}....D..| +00000240 6a f5 88 79 e1 00 a3 14 8f 66 07 03 16 2a 53 80 |j..y.....f...*S.| +00000250 9b 26 80 0d 0b 5e 0c c6 c9 fa a1 3b c9 a6 91 1f |.&...^.....;....| +00000260 bd fb 79 24 ab 93 e4 25 d1 a6 41 8b 9e a9 06 0f |..y$...%..A.....| +00000270 80 2e 4e 8c 20 2c 1e a8 7e 63 7a 4f b8 90 c3 56 |..N. ,..~czO...V| +00000280 a3 9e 63 2a 8b 85 9e ef 66 f5 16 be 79 c3 9b 47 |..c*....f...y..G| +00000290 dc 1c 75 0b 30 3c db 32 e8 ec 33 f9 9f 26 3d 56 |..u.0<.2..3..&=V| +000002a0 36 e9 ea 83 57 c4 59 ac 73 db 04 5a 1d 38 9b e4 |6...W.Y.s..Z.8..| +000002b0 47 50 f8 92 92 7f c9 09 4f f4 9c ab 3e 03 df 80 |GP......O...>...| +000002c0 cc 5b 50 0b 06 ef 8c 59 d2 f6 f3 a4 16 e7 0f 90 |.[P....Y........| +000002d0 c8 79 95 0a 39 0f 33 69 31 29 1c 30 77 72 58 b5 |.y..9.3i1).0wrX.| +000002e0 cf 0e 4c 30 fc 0a 01 93 b1 20 21 34 2a ce 28 8d |..L0..... !4*.(.| +000002f0 57 71 3e c9 b1 51 c9 4f e5 e4 09 0b 1e 32 52 4e |Wq>..Q.O.....2RN| +00000300 d0 be f2 a2 90 75 f4 a8 61 66 43 84 74 4e bb 28 |.....u..afC.tN.(| +00000310 7b ea 68 96 92 6d 4b 8c af 50 13 84 92 b9 6b 48 |{.h..mK..P....kH| +00000320 60 1c 60 62 28 a2 37 d1 1c 86 d4 60 27 a6 5b 6d |`.`b(.7....`'.[m| +00000330 88 d7 56 21 b7 86 f5 b6 34 f9 55 cf 47 f8 4f 90 |..V!....4.U.G.O.| +00000340 78 5c 56 52 97 17 03 03 00 99 35 71 51 6f 73 ba |x\VR......5qQos.| +00000350 76 60 e1 4a 5f ec ce 52 b6 91 58 3d 0c ea e6 58 |v`.J_..R..X=...X| +00000360 7e ee da 1e aa df bb 8e c9 89 ce 43 bd e7 34 9b |~..........C..4.| +00000370 b8 4b dd 50 73 0c f7 e5 15 e0 a1 c3 bd 07 08 62 |.K.Ps..........b| +00000380 a2 d5 2e 9f 46 f7 4d 9c ae ad be 22 89 0f b0 3e |....F.M...."...>| +00000390 82 1c 80 60 38 39 4f 06 f9 fc 4c f8 02 bf b3 5a |...`89O...L....Z| +000003a0 25 22 f4 62 61 ba eb 78 b2 40 68 ea 99 ef 95 90 |%".ba..x.@h.....| +000003b0 b1 b5 25 17 81 43 5c 15 bd a9 1f e8 0f b9 91 be |..%..C\.........| +000003c0 f1 47 91 99 ec 50 3b fa c2 06 fc 89 09 a3 d7 b3 |.G...P;.........| +000003d0 ed e3 3d 41 57 07 4b b0 3d 02 51 7e 13 bb d3 68 |..=AW.K.=.Q~...h| +000003e0 49 4c 6a 17 03 03 00 35 63 f7 b5 68 31 22 c7 f5 |ILj....5c..h1"..| +000003f0 50 ae c0 81 0f 50 fb ba 50 72 8a d7 e0 c0 73 07 |P....P..Pr....s.| +00000400 d0 88 ed 6d 7b e7 66 b3 8d e3 10 ca 2f 68 c3 39 |...m{.f...../h.9| +00000410 d9 b9 09 44 78 4a 11 91 fb 51 ea 9f 9a |...DxJ...Q...| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 02 7a 3e 62 91 75 b4 |..........z>b.u.| -00000010 58 7d 61 ac 42 4b 1b 91 86 cd 3f c0 6d 30 5a 01 |X}a.BK....?.m0Z.| -00000020 11 1e 77 87 92 50 bb a0 d2 c6 84 b6 10 37 02 ab |..w..P.......7..| -00000030 2d c2 c2 12 0a db f5 8f e4 1f dd 16 ec 83 ad db |-...............| -00000040 2d 8b ec fa aa 56 44 4e d4 af ce d2 9f 44 3d de |-....VDN.....D=.| -00000050 12 29 db bc 29 2a df 99 d7 d0 53 79 84 a4 b8 2d |.)..)*....Sy...-| -00000060 8b 5c e6 a4 b7 98 07 5a f4 63 10 21 87 13 09 07 |.\.....Z.c.!....| -00000070 28 98 49 48 07 70 c1 ab 8b 62 6d 40 a8 47 27 67 |(.IH.p...bm@.G'g| -00000080 d6 ad 29 8d 2e 14 34 14 18 6b dd f0 76 3b 91 f9 |..)...4..k..v;..| -00000090 e1 93 a2 1c 3a 5e 18 a3 c9 ca b0 c9 cf d9 3c f5 |....:^........<.| -000000a0 a3 38 c7 83 53 67 14 ba 65 6f 11 a2 ef 78 7e ea |.8..Sg..eo...x~.| -000000b0 4a 90 c9 aa bf e9 8e ff 8c 9d 7c 5f 56 c3 9b 16 |J.........|_V...| -000000c0 c1 8a a9 51 24 20 86 dc 99 ac d2 28 af 7b 2d 6e |...Q$ .....(.{-n| -000000d0 16 9b 71 71 82 00 94 51 a4 4f 22 c1 26 a2 6f 67 |..qq...Q.O".&.og| -000000e0 d9 45 ff 80 13 39 16 0f 9c 43 76 54 e5 12 0f 89 |.E...9...CvT....| -000000f0 95 ed 75 df eb cb 6b d2 77 fd fa 37 d7 a3 31 b5 |..u...k.w..7..1.| -00000100 61 47 69 8f 99 55 70 d0 96 b0 f7 a9 1b 53 69 a3 |aGi..Up......Si.| -00000110 e4 59 cc a1 d9 4b d5 4f e2 47 c1 53 30 60 50 03 |.Y...K.O.G.S0`P.| -00000120 c7 07 67 3f 68 ce 8c 32 7e f1 93 12 ba d8 67 21 |..g?h..2~.....g!| -00000130 71 4f c9 ca ae 7b 56 8d f7 64 14 e3 8d d6 a6 ed |qO...{V..d......| -00000140 d0 63 5e 00 1e f7 8a 0c 1e c3 24 e0 bd e2 e8 99 |.c^.......$.....| -00000150 da ca 01 9e fb 21 9e 1f 57 98 f1 36 7d c8 a8 7b |.....!..W..6}..{| -00000160 4b 27 13 fb 8c ab d6 b3 27 3e f5 b5 08 eb cb a9 |K'......'>......| -00000170 b0 a4 d4 3e c9 14 08 75 51 83 48 03 e7 79 65 30 |...>...uQ.H..ye0| -00000180 a4 1a 4f a8 7a 41 c8 af 9e 74 89 4d ec eb 58 9b |..O.zA...t.M..X.| -00000190 20 db 80 ce e0 c8 a7 1c b2 62 fc 85 7d 87 ca 00 | ........b..}...| -000001a0 64 4c 25 4d 3c b2 bf 9d 46 c5 04 b5 dd 7a 29 87 |dL%M<...F....z).| -000001b0 29 30 2e bc 42 31 9b f2 eb 7b 58 63 91 28 6f 43 |)0..B1...{Xc.(oC| -000001c0 6b c1 4e 00 b2 4b 9d 0c ef 18 5b 61 ec 77 84 e1 |k.N..K....[a.w..| -000001d0 88 49 f9 e2 71 2e 35 23 b8 73 6d c2 52 d1 2f ad |.I..q.5#.sm.R./.| -000001e0 55 40 4e a2 b0 14 1b 50 d7 9d 12 80 89 ab 88 a5 |U@N....P........| -000001f0 01 16 88 d5 4f 95 0c ce 79 c7 49 27 4b 44 4d fe |....O...y.I'KDM.| -00000200 f5 cd 95 08 b9 26 8d 52 ff d7 bc 47 11 16 d5 d0 |.....&.R...G....| -00000210 7d 4c 3c 04 74 92 5b 85 d2 2c 95 b8 4b 02 0b 66 |}L<.t.[..,..K..f| -00000220 60 7d b9 12 6e d3 06 73 cc c2 69 b9 fe ad f9 4c |`}..n..s..i....L| -00000230 f6 24 a0 eb 96 f0 b8 7c 32 dc 4c 67 04 83 15 84 |.$.....|2.Lg....| -00000240 f9 ef 70 e7 55 85 0e 8f b3 78 20 2c ed 69 29 89 |..p.U....x ,.i).| -00000250 d4 d8 5e 97 16 59 90 0f 79 41 6e cf 02 d3 9d 08 |..^..Y..yAn.....| -00000260 e4 0c 27 0c 74 97 c6 d6 89 6b c9 65 37 7b ae 6b |..'.t....k.e7{.k| -00000270 b7 0f e8 70 57 a3 1c 95 8d 66 da 4e 34 51 c8 e0 |...pW....f.N4Q..| -00000280 0e 83 e3 24 64 17 03 03 00 99 e3 31 2c ae ae 1c |...$d......1,...| -00000290 64 6c fc 36 c8 9f c2 88 16 c7 b2 ae 4a ef 3a 90 |dl.6........J.:.| -000002a0 f4 ad f2 91 58 37 7c 94 c2 e3 ed 76 c5 90 fb e6 |....X7|....v....| -000002b0 88 08 a6 a2 53 5a 78 cb bd f2 3c 6e 36 b4 cd 87 |....SZx...XCK.......N..ZR..| +000000c0 bd fa f0 a7 88 d5 9e 76 9b 2b 23 fb 37 c0 f1 0a |.......v.+#.7...| +000000d0 d2 df d4 2c 5d 8c 64 f0 98 3c 85 3f bb ed 30 44 |...,].d..<.?..0D| +000000e0 eb cb fc 47 e5 8a 57 ff 37 23 6d e2 75 a7 15 e4 |...G..W.7#m.u...| +000000f0 dc f9 02 8f c4 af d8 e8 bb 07 43 90 5b c2 86 f3 |..........C.[...| +00000100 40 22 a7 bb b6 f1 f6 4c 2e d6 fd d8 02 f0 3d 27 |@".....L......='| +00000110 89 38 97 7d 6e d3 6c b1 4c 87 ca 30 e9 ec fe b3 |.8.}n.l.L..0....| +00000120 db 62 ea 2f 3a 83 95 18 4e 44 4f 97 c4 ff f6 bb |.b./:...NDO.....| +00000130 0b b0 98 c7 c6 c7 94 0a 8b c9 c0 d9 51 d3 6a e1 |............Q.j.| +00000140 36 55 79 c4 37 75 6f e7 14 53 60 9e 22 00 fe d7 |6Uy.7uo..S`."...| +00000150 ef e4 ca 60 38 12 25 8c c6 c9 7e 5f 1a 7c f1 c1 |...`8.%...~_.|..| +00000160 a7 15 b2 f4 32 31 de c8 a2 d4 59 46 1e 68 a6 94 |....21....YF.h..| +00000170 b8 18 52 df 44 01 0b a0 a0 02 60 83 84 fc ad 99 |..R.D.....`.....| +00000180 fc 60 0b 23 0a 7a f9 73 bf d8 a5 af 9b ea dd a3 |.`.#.z.s........| +00000190 fb 5b 41 17 95 0a 57 4a 94 7e 24 9f 31 74 06 65 |.[A...WJ.~$.1t.e| +000001a0 95 90 40 20 17 0b 95 22 b8 49 4e a6 a4 be e1 74 |..@ ...".IN....t| +000001b0 01 a2 7e cd 60 50 46 5c fd 2a 88 90 ef 7f 6c b8 |..~.`PF\.*....l.| +000001c0 3e a1 40 8e 91 cd 8d db ad 81 d0 08 b0 d0 d7 ae |>.@.............| +000001d0 18 4f 47 9b 1a 66 8b a5 d2 fb 01 d3 67 79 46 71 |.OG..f......gyFq| +000001e0 5a 49 dd df 06 c6 57 c7 e0 b1 60 39 d4 a9 37 88 |ZI....W...`9..7.| +000001f0 31 0d c0 92 4e c7 2a 25 c4 df b0 d8 df bb 31 1b |1...N.*%......1.| +00000200 ff 0a 34 46 46 88 0b 11 7d 20 32 cd 01 d0 3f 11 |..4FF...} 2...?.| +00000210 e1 6e 63 42 d7 4a 83 ab ad f7 51 bd c3 37 2b 76 |.ncB.J....Q..7+v| +00000220 9e bb 45 3b 81 d8 47 71 02 b9 fe 7c 56 46 78 3e |..E;..Gq...|VFx>| +00000230 9e 94 00 1b 98 17 72 37 c0 e5 36 a9 f0 02 b7 cf |......r7..6.....| +00000240 f0 b4 66 10 1b 91 07 5f bb 22 e8 12 d9 e3 33 dd |..f...._."....3.| +00000250 87 66 e4 f6 00 f6 3a dc 19 59 af ce 8e ae c4 d3 |.f....:..Y......| +00000260 03 d8 25 02 40 7e 95 70 31 85 53 bc 9e 96 11 69 |..%.@~.p1.S....i| +00000270 5b 2f b9 17 48 f4 ae c5 a1 c5 d1 7f 4a 10 06 b6 |[/..H.......J...| +00000280 2e 34 6c 09 12 17 03 03 00 99 16 4c 04 64 97 e0 |.4l........L.d..| +00000290 26 e1 66 e9 34 05 d7 d3 75 e3 b9 de 56 15 51 67 |&.f.4...u...V.Qg| +000002a0 fb fe 9a c6 9a 3c 38 08 2c c8 8c dd fe 49 c5 ed |.....<8.,....I..| +000002b0 a8 54 86 90 2c f6 7c d1 12 02 99 94 5e 2e 4b 4e |.T..,.|.....^.KN| +000002c0 84 e9 b4 96 5c dc 56 28 3d ea a9 4f 8f ad 51 ff |....\.V(=..O..Q.| +000002d0 02 b1 b9 7a 29 e5 32 7c 2a 8b 60 5d e2 fc b4 8f |...z).2|*.`]....| +000002e0 06 32 4a ea 37 ed 03 f0 68 72 b7 83 1f 04 10 2b |.2J.7...hr.....+| +000002f0 24 db 5b 10 6c 41 55 40 54 69 07 39 d4 db ac 10 |$.[.lAU@Ti.9....| +00000300 77 9f 04 f4 b9 3f 35 7e 04 af ab 7a a0 47 b9 d9 |w....?5~...z.G..| +00000310 4c 6e 25 00 ce ef 93 3f 28 2c 2d f6 42 e4 5f 3e |Ln%....?(,-.B._>| +00000320 26 92 13 17 03 03 00 35 fb 14 eb 5f 18 61 75 ba |&......5..._.au.| +00000330 e4 dc d2 95 fe 93 bb 54 29 e3 38 e3 59 54 81 9f |.......T).8.YT..| +00000340 4e 29 be c6 e6 cd ad 8c 9d 6a ad 28 ec d3 a6 e4 |N).......j.(....| +00000350 bc 5e 8c df a8 7e 14 d8 69 3c 30 7c 0a 17 03 03 |.^...~..i<0|....| +00000360 00 17 10 73 c0 75 88 af 51 90 ff 3f b2 83 47 27 |...s.u..Q..?..G'| +00000370 19 c0 e6 cd 14 a4 c7 8d a0 17 03 03 00 13 fb 1f |................| +00000380 ed 8e 47 c8 79 f9 53 df 6d 97 a7 1d 53 8d 80 85 |..G.y.S.m...S...| +00000390 dd |.| diff --git a/src/crypto/tls/testdata/Client-TLSv13-ECDSA b/src/crypto/tls/testdata/Client-TLSv13-ECDSA index b6a3b2addbe924..eeab8711723b55 100644 --- a/src/crypto/tls/testdata/Client-TLSv13-ECDSA +++ b/src/crypto/tls/testdata/Client-TLSv13-ECDSA @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,81 +7,82 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 7a 02 00 00 76 03 03 24 80 76 3d db |....z...v..$.v=.| -00000010 cf 32 53 04 de ce 2e 74 95 bb 30 a8 d5 5e ab ed |.2S....t..0..^..| -00000020 3c 1e 1a ae 92 63 8f 40 0b ce a4 20 00 00 00 00 |<....c.@... ....| +00000000 16 03 03 00 7a 02 00 00 76 03 03 a5 7b 07 50 bc |....z...v...{.P.| +00000010 01 11 74 76 99 fb 85 e5 40 6b 02 14 d9 64 b1 8a |..tv....@k...d..| +00000020 8b 78 a3 ee 2e 4d b0 96 14 3a fb 20 00 00 00 00 |.x...M...:. ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000040 00 00 00 00 00 00 00 00 00 00 00 00 13 03 00 00 |................| -00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 d2 |..+.....3.$... .| -00000060 b3 86 f1 35 22 26 89 f4 07 ab b8 f5 27 a6 d5 c9 |...5"&......'...| -00000070 87 00 59 43 b8 df da 92 f6 f5 ed f4 8d 04 26 14 |..YC..........&.| -00000080 03 03 00 01 01 17 03 03 00 17 ed 6b 14 3a 82 c5 |...........k.:..| -00000090 88 9d a4 96 c7 c4 c7 73 63 c6 58 65 d0 d4 a6 cb |.......sc.Xe....| -000000a0 dc 17 03 03 02 22 e8 c5 15 54 3c 26 7b 87 8c d3 |....."...T<&{...| -000000b0 99 b0 af 98 21 1b 8f be f5 d8 40 c9 05 97 39 fd |....!.....@...9.| -000000c0 3d b2 da 2d 80 26 c9 24 cd 14 0b 48 2e d7 c4 dc |=..-.&.$...H....| -000000d0 4c 61 1e 5d 46 ce 66 cd 94 50 a4 e8 c4 d7 f0 48 |La.]F.f..P.....H| -000000e0 20 16 52 fb 12 04 17 48 7d 65 49 50 c5 b8 e6 9c | .R....H}eIP....| -000000f0 be f6 58 09 ea 8b 94 51 93 cf 0d 7a 76 88 78 cb |..X....Q...zv.x.| -00000100 37 83 43 fa 37 d5 5b 3f 10 1d 07 b1 bb 9b ac 3a |7.C.7.[?.......:| -00000110 43 e0 62 b1 fe 64 43 83 53 49 96 9e 81 1c 05 64 |C.b..dC.SI.....d| -00000120 40 2a 1b 98 5f ed 8d 47 52 2c db 84 8b 6a 1e 3b |@*.._..GR,...j.;| -00000130 de bb 7d 20 e6 cf 2a 79 2e c4 54 71 56 2f 79 45 |..} ..*y..TqV/yE| -00000140 07 3f 05 6c 82 e8 ab f7 62 0a 28 e0 db 83 6b bd |.?.l....b.(...k.| -00000150 55 81 8f 3f b6 1f d0 83 cd 97 67 71 56 f5 13 aa |U..?......gqV...| -00000160 45 95 65 ce 1e 42 bf 6a 83 6e bc 2e 41 28 79 b7 |E.e..B.j.n..A(y.| -00000170 b8 3f 6e 64 14 8d ed 3a a4 02 06 46 17 e5 c7 f8 |.?nd...:...F....| -00000180 72 f1 5f 9b 14 96 c9 c6 53 e6 32 b2 0b cc d5 d7 |r._.....S.2.....| -00000190 89 1e d9 52 fb 8f 48 8b 41 6e c6 f3 55 57 41 5d |...R..H.An..UWA]| -000001a0 95 03 9c 34 ea 3c a8 58 8d b7 89 82 e1 40 e0 60 |...4.<.X.....@.`| -000001b0 22 ab 23 30 e8 f0 4e 21 a1 11 0a 7f 18 80 ad de |".#0..N!........| -000001c0 41 ba a1 12 60 31 60 0f 84 f6 91 b1 ae ca f5 81 |A...`1`.........| -000001d0 3e d6 4e 3a 89 b6 b0 fe 3f 27 18 20 65 a9 d4 6c |>.N:....?'. e..l| -000001e0 ac 8a 46 08 e2 7d 06 6d 52 8c 69 9e 24 9b 8d f9 |..F..}.mR.i.$...| -000001f0 b7 a7 74 87 bd f9 84 ca 18 7d 5b 50 1d 8e 64 e1 |..t......}[P..d.| -00000200 39 6f f7 56 5a ef 08 6b e2 a1 87 e4 7a 5e ea b6 |9o.VZ..k....z^..| -00000210 12 75 2d ee e9 af a4 36 99 91 88 11 97 f3 d5 fb |.u-....6........| -00000220 8b 52 59 bc 9c 18 03 5b b1 59 47 ae d4 bb a1 65 |.RY....[.YG....e| -00000230 cf bf 40 54 e5 e7 6e e0 0c 26 0e a1 d2 4d 41 19 |..@T..n..&...MA.| -00000240 36 b5 b6 48 33 96 8a e3 a5 56 9b 34 16 ae 36 48 |6..H3....V.4..6H| -00000250 c5 ff 12 a7 33 f4 76 40 de d1 4b 41 ed 18 3b 04 |....3.v@..KA..;.| -00000260 06 32 6e f3 57 c6 be 72 58 7f 78 b7 91 65 00 a8 |.2n.W..rX.x..e..| -00000270 8d 5c 7f ff 0a 62 d4 99 82 b2 6b c8 80 3e 89 30 |.\...b....k..>.0| -00000280 dd 31 60 7a 00 6e a2 13 c7 58 08 b0 d5 32 03 2e |.1`z.n...X...2..| -00000290 08 86 a2 97 7a f0 a1 95 10 5f a2 d6 fc e7 a3 93 |....z...._......| -000002a0 16 a2 3d 33 04 e7 aa 3b 99 a5 ce c7 2c 28 71 bb |..=3...;....,(q.| -000002b0 ed e6 b9 5a a5 ec d1 44 54 e0 ff 41 cf 1a cb ff |...Z...DT..A....| -000002c0 67 43 6b af 87 19 40 74 17 03 03 00 a4 bc ef 32 |gCk...@t.......2| -000002d0 22 da e9 d5 93 ce 06 bd 3d 6c 2e f0 0c de ec b8 |".......=l......| -000002e0 12 d6 28 17 c6 64 aa 8d 4b 04 76 9e 41 3c 4a 94 |..(..d..K.v.Aru@.0.| -000003a0 fd 46 9d 5e 79 39 e1 a8 f9 27 2b |.F.^y9...'+| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 67 |..+.....3.$... g| +00000060 b0 dc 13 4f 2b 92 43 27 38 b5 2c 7a 3a 87 83 6f |...O+.C'8.,z:..o| +00000070 a1 cc 43 fa 90 9e 2e 4d 88 f6 60 d4 20 32 6a 14 |..C....M..`. 2j.| +00000080 03 03 00 01 01 17 03 03 00 17 9b 51 d0 81 30 4b |...........Q..0K| +00000090 52 65 8b b5 fe 1a cc 44 c9 7a 10 83 41 0b 1a 00 |Re.....D.z..A...| +000000a0 76 17 03 03 02 22 4e 55 d0 64 9c 41 c8 22 2c 9c |v...."NU.d.A.",.| +000000b0 0e 0a 0c 2b 2b 2e 6e 08 d5 54 75 61 99 bf ea f5 |...++.n..Tua....| +000000c0 ca 12 3e 44 7f 81 a8 40 d1 7c 4d a0 a3 d2 f3 07 |..>D...@.|M.....| +000000d0 40 af 68 81 69 dd 7c 77 24 26 87 73 73 5e d2 0c |@.h.i.|w$&.ss^..| +000000e0 2a 32 c5 34 50 ef a8 0f 63 12 3a 47 34 8f f2 c6 |*2.4P...c.:G4...| +000000f0 9f bd 81 bf ea 69 47 04 8b 87 64 4e 67 81 10 a5 |.....iG...dNg...| +00000100 f1 76 92 5e d6 11 a5 48 5b cc ef a8 43 dd cf f5 |.v.^...H[...C...| +00000110 20 9d b8 2f f4 0d 92 10 55 af d1 fa ab 64 5f 99 | ../....U....d_.| +00000120 3e 6b e9 70 70 f1 22 d9 05 04 89 3a fa 65 ae 91 |>k.pp."....:.e..| +00000130 9d 07 ea 54 93 2d 02 0c c0 70 d9 e9 f0 9a 5a 81 |...T.-...p....Z.| +00000140 c6 c6 79 e0 e1 90 ad 34 78 bf d3 c8 9c 68 2e ac |..y....4x....h..| +00000150 e6 2d c0 e5 c8 3c 77 80 d7 dd e5 a4 ac b8 36 4f |.-...@..I<..FW.Ye.| +00000170 4e db 21 ea ab 72 47 79 e6 c1 4f ea 17 e3 b4 73 |N.!..rGy..O....s| +00000180 9d e2 e5 72 be 88 0a 60 1e 35 02 67 33 a0 7a 05 |...r...`.5.g3.z.| +00000190 b8 ae 05 b2 53 6c cd c5 e3 a5 16 56 2b b0 0e 8e |....Sl.....V+...| +000001a0 a0 ca 54 c0 34 6b 00 22 69 de e1 57 81 48 c6 1a |..T.4k."i..W.H..| +000001b0 cc 88 f6 15 1c 7e b6 1f b0 48 82 75 6b ff 42 78 |.....~...H.uk.Bx| +000001c0 f9 85 10 86 5b af 31 62 f6 7b d4 8a a6 86 3f 99 |....[.1b.{....?.| +000001d0 9b ce 91 c0 4d 44 5b 3f 16 81 d0 a9 4a e0 2d 85 |....MD[?....J.-.| +000001e0 a4 80 91 64 13 20 33 dc 84 48 da 79 2e 73 cb 78 |...d. 3..H.y.s.x| +000001f0 9c 05 8a 6c 42 ce b0 b0 45 3c 05 47 47 f7 92 3b |...lB...E<.GG..;| +00000200 d2 fb 65 f0 40 a3 52 b7 aa 04 ae 71 84 35 12 7a |..e.@.R....q.5.z| +00000210 33 f8 8b 0d 21 a3 29 ad 78 de df 7f 53 71 fb 5f |3...!.).x...Sq._| +00000220 a5 ab 07 89 3a 1f e9 ca 6d 8d bb 04 39 8d b3 50 |....:...m...9..P| +00000230 7e 2e bd 0f b8 ff 4f 3f d1 a5 23 b9 80 33 da b0 |~.....O?..#..3..| +00000240 2c dc 0f c2 26 b2 cf 59 d9 10 ef 66 52 40 e4 54 |,...&..Y...fR@.T| +00000250 28 5e 7f c1 94 62 8f 4f 9b 94 fc c9 32 af f5 17 |(^...b.O....2...| +00000260 5d 04 32 08 83 f4 90 68 68 01 d3 00 ed 82 f3 da |].2....hh.......| +00000270 81 2e d5 df 1e 13 c3 c3 76 83 c4 67 23 c4 32 c0 |........v..g#.2.| +00000280 59 5d dd 56 78 d9 74 ef 7b d5 c9 13 4d 62 29 85 |Y].Vx.t.{...Mb).| +00000290 ca 24 53 1a b1 2b 09 a1 a6 26 db 13 cf 2a 2b 92 |.$S..+...&...*+.| +000002a0 85 a3 51 2b 24 e1 90 2a fe 0c 74 ee 86 cc 3b 6a |..Q+$..*..t...;j| +000002b0 07 21 a6 b6 97 a4 e0 9f 3d 75 9a 2b 9e 91 ef d8 |.!......=u.+....| +000002c0 c9 94 28 ba 40 e4 cc 6a 17 03 03 00 a4 ab c3 c1 |..(.@..j........| +000002d0 32 1d 44 0d f7 85 e3 85 df 9e bb 0c 82 18 a2 9f |2.D.............| +000002e0 27 de 78 4f 06 9b 43 59 27 b5 8e 34 4d 96 b3 96 |'.xO..CY'..4M...| +000002f0 af 8b 98 d0 36 91 52 df 9c aa c2 fe e7 0f ed ca |....6.R.........| +00000300 ef 57 73 97 cc c8 dc 7c c8 15 73 ad a6 3e 54 93 |.Ws....|..s..>T.| +00000310 1a ab 72 f6 e0 e3 cb bb fe a5 d1 45 47 f3 1a 89 |..r........EG...| +00000320 40 f7 9d f5 e4 61 16 fa 12 0a 62 24 a7 34 ab 6f |@....a....b$.4.o| +00000330 08 85 b2 fe b2 5c 49 59 7b cb 05 b2 e3 1a 37 79 |.....\IY{.....7y| +00000340 b1 27 28 a5 ab ee ae 72 11 19 61 c0 3b ed 32 ec |.'(....r..a.;.2.| +00000350 57 26 76 8e 42 3c 98 4a ec 10 4e e0 eb e2 19 3d |W&v.B<.J..N....=| +00000360 67 47 99 9c e5 97 a4 07 ff 7a b5 15 65 0e b9 e4 |gG.......z..e...| +00000370 e3 17 03 03 00 35 61 d9 d8 36 a0 1e 83 b0 f8 1f |.....5a..6......| +00000380 74 61 a0 4b 93 7f 98 8b 63 1c 63 82 f4 cd 57 8d |ta.K....c.c...W.| +00000390 1f 75 7a 6d a9 1d e2 35 0b 2b 9a 5f 5f 71 cd 46 |.uzm...5.+.__q.F| +000003a0 48 7f 27 af a5 da 8b 35 b1 18 39 |H.'....5..9| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 00 35 09 35 dd b0 97 |..........5.5...| -00000010 7b a4 5b ef 16 90 e4 15 aa 19 e4 5b e8 f1 11 c0 |{.[........[....| -00000020 e0 fa 4b 0e df a3 4a 7c cd 9a 0a c4 05 65 e9 5b |..K...J|.....e.[| -00000030 8b 0b e0 3c b2 5a cc eb 87 2b 67 0d 9f 19 d0 ec |...<.Z...+g.....| -00000040 17 03 03 00 17 68 1f ad 70 7c 49 62 6b 2f 67 15 |.....h..p|Ibk/g.| -00000050 d1 bf bb 35 b3 b3 2b f3 12 27 b6 0e 17 03 03 00 |...5..+..'......| -00000060 13 23 39 42 9f 8a 9e 68 41 39 fa 3b 50 bc 56 59 |.#9B...hA9.;P.VY| -00000070 d0 3a 5a 9a |.:Z.| +00000000 14 03 03 00 01 01 17 03 03 00 35 3d 13 d4 8c 88 |..........5=....| +00000010 73 62 97 8b 18 04 f7 05 8d c8 56 d2 f4 0a 86 93 |sb........V.....| +00000020 d4 6f 08 1c d8 7f 57 be 50 03 a7 4a c8 37 80 04 |.o....W.P..J.7..| +00000030 43 1b 2a 86 21 25 ae 84 53 80 0c f6 4f 2e 73 67 |C.*.!%..S...O.sg| +00000040 17 03 03 00 17 43 99 86 f2 2f a2 22 05 36 d8 47 |.....C.../.".6.G| +00000050 c5 57 92 93 c7 79 d9 fd 13 70 8f 75 17 03 03 00 |.W...y...p.u....| +00000060 13 15 90 47 ea af 57 89 d4 19 70 e7 e1 ce 68 8a |...G..W...p...h.| +00000070 7b 83 8c a5 |{...| diff --git a/src/crypto/tls/testdata/Client-TLSv13-Ed25519 b/src/crypto/tls/testdata/Client-TLSv13-Ed25519 index 3a2915d613e2e3..274620a2576af4 100644 --- a/src/crypto/tls/testdata/Client-TLSv13-Ed25519 +++ b/src/crypto/tls/testdata/Client-TLSv13-Ed25519 @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,63 +7,64 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 7a 02 00 00 76 03 03 b2 4d aa 76 5b |....z...v...M.v[| -00000010 b7 38 95 18 e6 81 66 f8 27 82 26 2c b3 44 50 b0 |.8....f.'.&,.DP.| -00000020 f4 a8 f5 f2 87 f0 9d 85 4f ac 60 20 00 00 00 00 |........O.` ....| +00000000 16 03 03 00 7a 02 00 00 76 03 03 5a e2 b6 f0 cf |....z...v..Z....| +00000010 f2 85 49 e0 28 8c d7 89 41 39 5f a4 e8 41 9b f6 |..I.(...A9_..A..| +00000020 1d 2b 3a 8a 3e a3 40 29 76 17 22 20 00 00 00 00 |.+:.>.@)v." ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000040 00 00 00 00 00 00 00 00 00 00 00 00 13 03 00 00 |................| -00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 e7 |..+.....3.$... .| -00000060 c2 3e 68 73 d9 fd 6e 69 16 15 85 84 24 45 36 47 |.>hs..ni....$E6G| -00000070 1b ad d9 c4 dc 4a 61 c6 12 5c 22 bf 1e 2f 59 14 |.....Ja..\"../Y.| -00000080 03 03 00 01 01 17 03 03 00 17 0f a4 05 81 10 48 |...............H| -00000090 c7 b7 78 9b 56 14 6c 9f 55 28 e3 4c 42 80 cc 1c |..x.V.l.U(.LB...| -000000a0 31 17 03 03 01 50 0d af 3b 08 02 84 89 42 b5 75 |1....P..;....B.u| -000000b0 c5 8f 9f c2 b1 dc cb 7c 6a d5 31 41 9a 16 8f fa |.......|j.1A....| -000000c0 74 40 21 5d f9 cc b0 1d 30 80 87 9a b2 57 58 53 |t@!]....0....WXS| -000000d0 2c 9e 5c 97 20 9c 81 77 16 25 62 9b fc a4 e1 9a |,.\. ..w.%b.....| -000000e0 80 c8 9a 4c f6 fc ae 79 a5 c2 26 4b 55 41 ff d0 |...L...y..&KUA..| -000000f0 25 4b 70 c9 ad b6 04 80 65 19 e0 fe 05 e7 b4 dc |%Kp.....e.......| -00000100 53 8f f8 3b ed 2c df b9 08 c2 da 60 b9 23 17 50 |S..;.,.....`.#.P| -00000110 da 0f 24 76 15 21 e6 e9 a8 f5 3e 08 cc 1b ee 92 |..$v.!....>.....| -00000120 2b 01 92 8d f9 4f 5a 3a 53 11 fc 32 52 cc af cd |+....OZ:S..2R...| -00000130 7b 94 0e 76 10 c2 16 36 2d a4 64 69 1c 05 70 20 |{..v...6-.di..p | -00000140 0d 23 cd 4a 33 c5 c7 db db 0f f8 b6 42 0c 83 0a |.#.J3.......B...| -00000150 a1 73 68 fb 87 2c 9d d2 d3 cf d7 3a bb 36 7e 83 |.sh..,.....:.6~.| -00000160 c3 3f bc e2 61 d9 c2 8b 15 a2 cc bf 14 a0 69 f4 |.?..a.........i.| -00000170 22 02 a9 ff 5e 55 37 6e 61 86 71 73 94 2f 7e 50 |"...^U7na.qs./~P| -00000180 45 96 cf 23 dd 39 90 5d 57 04 a3 d0 9f 97 f8 56 |E..#.9.]W......V| -00000190 fa cf 7a a9 e6 57 1f cb 33 a0 82 7a 84 94 1f 6d |..z..W..3..z...m| -000001a0 d8 0b c7 fb bc 62 94 36 4a d6 96 ce f6 f2 7f 06 |.....b.6J.......| -000001b0 18 59 66 77 77 12 8c 74 06 61 8f b6 37 35 65 fd |.Yfww..t.a..75e.| -000001c0 f1 0b fe e6 cc a9 64 f5 b5 0d 06 4e 38 4b 4b 78 |......d....N8KKx| -000001d0 be 6a cc 67 9e 39 f8 f2 27 33 a9 2d bc 9c 5e 06 |.j.g.9..'3.-..^.| -000001e0 b9 fb 76 0c be 8d 2a a5 6d cf 83 55 4f c5 6e 6e |..v...*.m..UO.nn| -000001f0 d0 08 8b f6 5a 61 17 03 03 00 59 c1 cc 58 14 33 |....Za....Y..X.3| -00000200 5a 39 b5 40 46 e8 b1 28 06 08 22 d3 27 b4 e6 ef |Z9.@F..(..".'...| -00000210 10 79 08 d4 ce 9e e2 cb 0b 86 a4 b1 80 e4 1b ac |.y..............| -00000220 71 15 fd 16 5b aa 01 c2 7b e3 c2 84 01 c0 f6 04 |q...[...{.......| -00000230 a7 31 2b ec f2 50 bf 5a 07 6f b8 03 9f d2 c6 ad |.1+..P.Z.o......| -00000240 73 18 81 f5 fd 9a 5a b2 bc 9f 85 94 9f f6 da da |s.....Z.........| -00000250 dd 56 bc 66 17 03 03 00 35 9f 7e 2e 02 66 bb ac |.V.f....5.~..f..| -00000260 3d d8 92 70 c9 35 c6 df ff 0f 81 b5 ca d4 56 4a |=..p.5........VJ| -00000270 24 06 06 57 28 e9 9d 82 0e 40 06 a7 f8 cc bc 6c |$..W(....@.....l| -00000280 a2 ea 2b 0e 88 77 05 de 3d 30 a0 6f bc 98 |..+..w..=0.o..| +00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 d1 |..+.....3.$... .| +00000060 41 fa 43 c4 c8 0a 98 e0 da 88 c8 58 36 ab 12 61 |A.C........X6..a| +00000070 d5 2d 9a 5b 10 fd 1f 10 bf d4 df 23 d0 de 2c 14 |.-.[.......#..,.| +00000080 03 03 00 01 01 17 03 03 00 17 bf 64 74 65 06 48 |...........dte.H| +00000090 41 72 b9 9f 41 24 a9 d0 05 00 4e 8c 93 a8 bd 91 |Ar..A$....N.....| +000000a0 ce 17 03 03 01 50 20 58 c1 4f 42 ed e9 e6 50 86 |.....P X.OB...P.| +000000b0 5f 49 88 ac 8b b8 3d 25 1d 87 b5 7f 9f 1a a2 cd |_I....=%........| +000000c0 f2 90 d0 6d 13 ed a6 6c f9 1e af c8 73 dd 68 d4 |...m...l....s.h.| +000000d0 46 1f ce d4 74 87 57 0f c1 b7 dd bd 5a 44 7c 08 |F...t.W.....ZD|.| +000000e0 2d 60 ca 34 1f 94 ee 4a cd e7 4a 69 1c 8d 4c 64 |-`.4...J..Ji..Ld| +000000f0 e8 a1 90 89 75 e2 3b 5b 0e 58 f8 7a 0a fe 2d 6d |....u.;[.X.z..-m| +00000100 ee 82 ab 07 e3 09 ae 37 d5 d6 31 9f 47 a5 eb ca |.......7..1.G...| +00000110 fd d9 5e b3 f1 ab 8a 64 8f 66 7d e7 a5 66 b4 46 |..^....d.f}..f.F| +00000120 3c 3c 2c 2f f7 5a 46 b3 e7 8b a4 3e 5f 21 ab 66 |<<,/.ZF....>_!.f| +00000130 60 4d ca 20 4b 7b 75 5a 88 a5 65 97 1b 70 5b 69 |`M. K{uZ..e..p[i| +00000140 7b 22 87 48 e4 a2 a8 32 06 65 00 91 e6 ac 6c 9b |{".H...2.e....l.| +00000150 42 04 53 f5 e5 46 72 d6 3a 7d d8 c1 e4 f1 79 dc |B.S..Fr.:}....y.| +00000160 de 7b 21 83 d0 98 90 99 2d 9d 78 67 89 31 2a 7d |.{!.....-.xg.1*}| +00000170 6a 97 c7 80 a9 2a 91 4e e2 30 29 96 25 97 e9 94 |j....*.N.0).%...| +00000180 2d 0a e0 30 8c e3 54 00 2a 29 e9 60 ef 4c 60 1c |-..0..T.*).`.L`.| +00000190 76 51 db 5d d7 7c 0c ae f3 a8 9e 3b 49 c6 a7 ce |vQ.].|.....;I...| +000001a0 a3 e8 e6 8c 13 ea fa bd 7f 59 36 06 1e 0c 54 2d |.........Y6...T-| +000001b0 d2 75 8a 75 94 f3 5c 77 0a 8e a0 23 9f 21 d8 ed |.u.u..\w...#.!..| +000001c0 65 87 fa f0 65 e1 81 2b 45 50 91 9e 48 8f 4c 80 |e...e..+EP..H.L.| +000001d0 7a c7 32 07 9a d9 d4 59 7a 7c 11 01 c2 75 fd 25 |z.2....Yz|...u.%| +000001e0 15 7b de b1 72 54 42 9e 02 41 a8 1a 04 6a 64 ba |.{..rTB..A...jd.| +000001f0 97 48 48 48 24 3d 17 03 03 00 59 d7 53 12 64 01 |.HHH$=....Y.S.d.| +00000200 d4 b7 e4 bf e1 33 f6 49 24 cf e8 6f 48 56 d6 1b |.....3.I$..oHV..| +00000210 e3 11 e9 a5 fb 7d f7 f5 b7 06 6c ce 4d e7 3c ca |.....}....l.M.<.| +00000220 bc 30 27 2e 02 4a 50 ec 7e 8c d4 cd 7a 78 43 3f |.0'..JP.~...zxC?| +00000230 5d 17 d7 bd 3e 78 68 21 ad 98 93 ad 90 de f7 c9 |]...>xh!........| +00000240 ab 7d 36 59 7e 7b e8 10 92 de ee 44 a2 e3 df 41 |.}6Y~{.....D...A| +00000250 6c a8 6c 9b 17 03 03 00 35 3a 9b 88 fc 4d 78 aa |l.l.....5:...Mx.| +00000260 a3 96 c7 f9 80 87 75 88 41 e6 58 9c f7 f6 88 0c |......u.A.X.....| +00000270 8f 57 8f 41 b7 49 a9 c2 db d6 44 51 a6 cb eb c3 |.W.A.I....DQ....| +00000280 8d e5 32 19 4b e5 dd 4e 19 6b f4 38 cd 3d |..2.K..N.k.8.=| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 00 35 1c 60 8f 01 6b |..........5.`..k| -00000010 7c e3 d4 28 24 d8 7c 85 7f 63 44 23 97 7c a0 de ||..($.|..cD#.|..| -00000020 70 61 9e d9 ce 35 d9 3b 81 68 09 12 c1 b9 5e f5 |pa...5.;.h....^.| -00000030 32 23 9b 47 fc be 74 3c e6 1b 3f cd e8 c1 f1 4f |2#.G..t<..?....O| -00000040 17 03 03 00 17 1a 16 bc fe 99 30 d8 97 c1 00 d9 |..........0.....| -00000050 fd 1a 44 bd 84 35 dc 66 62 eb 28 3d 17 03 03 00 |..D..5.fb.(=....| -00000060 13 b4 8a 54 df ae 97 ce 2e ae fa 38 ca 02 a1 d6 |...T.......8....| -00000070 f5 40 64 2b |.@d+| +00000000 14 03 03 00 01 01 17 03 03 00 35 64 73 84 51 69 |..........5ds.Qi| +00000010 81 4c a7 ce 8b 39 08 0e 0f 25 d2 24 49 3c 9d 98 |.L...9...%.$I<..| +00000020 c7 46 f5 7e ba bd d1 9d 79 d3 f6 cb 29 00 95 90 |.F.~....y...)...| +00000030 c2 fa c6 65 01 f2 9d 12 a7 db 14 49 e6 0b 40 c9 |...e.......I..@.| +00000040 17 03 03 00 17 df ad 2a 5f f7 bd 94 90 dd 11 24 |.......*_......$| +00000050 b8 97 6e fc d3 93 91 47 36 26 2d e1 17 03 03 00 |..n....G6&-.....| +00000060 13 40 ad 3c c3 49 76 47 c0 02 f0 bc 80 66 3c e4 |.@.<.IvG.....f<.| +00000070 35 28 eb b5 |5(..| diff --git a/src/crypto/tls/testdata/Client-TLSv13-ExportKeyingMaterial b/src/crypto/tls/testdata/Client-TLSv13-ExportKeyingMaterial index 13ae97153af48b..13fb6048748dac 100644 --- a/src/crypto/tls/testdata/Client-TLSv13-ExportKeyingMaterial +++ b/src/crypto/tls/testdata/Client-TLSv13-ExportKeyingMaterial @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,85 +7,86 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 7a 02 00 00 76 03 03 48 e6 a7 a9 4c |....z...v..H...L| -00000010 c3 a7 cc 18 b7 71 7d ed c5 6a cb ca b5 9f 00 fd |.....q}..j......| -00000020 f8 2c ac 9c 1f 24 27 b9 c6 55 8e 20 00 00 00 00 |.,...$'..U. ....| +00000000 16 03 03 00 7a 02 00 00 76 03 03 67 c5 21 e2 8b |....z...v..g.!..| +00000010 b6 6a cc 42 40 24 b2 4f 55 61 d9 20 78 e2 e3 40 |.j.B@$.OUa. x..@| +00000020 83 62 3d 08 e3 0a 31 0b 55 f5 b6 20 00 00 00 00 |.b=...1.U.. ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000040 00 00 00 00 00 00 00 00 00 00 00 00 13 03 00 00 |................| -00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 dd |..+.....3.$... .| -00000060 be 27 eb a0 9c b1 22 6a 8c 29 9a d4 47 c2 ee 14 |.'...."j.)..G...| -00000070 39 0c 60 81 c9 06 3f dc e5 e0 24 9a c4 88 35 14 |9.`...?...$...5.| -00000080 03 03 00 01 01 17 03 03 00 17 25 70 5a e5 6b 9e |..........%pZ.k.| -00000090 56 b9 cf 83 48 b0 bc 99 6a 86 e1 cb 4e ce b5 10 |V...H...j...N...| -000000a0 e0 17 03 03 02 6d c7 a6 79 ef b0 81 d6 e4 0e 02 |.....m..y.......| -000000b0 59 32 88 cc b1 0d 53 f6 33 9b d2 e8 74 a9 0a a7 |Y2....S.3...t...| -000000c0 f9 76 e9 6e 0d 16 75 0b e0 8f 5c b5 31 47 6b 68 |.v.n..u...\.1Gkh| -000000d0 52 c7 c2 84 cb 48 81 a3 da bd a1 50 5c ec 5c a7 |R....H.....P\.\.| -000000e0 10 01 58 cc 03 c3 53 04 03 69 80 f4 ad 4d ce 72 |..X...S..i...M.r| -000000f0 26 4e 6c c7 2c 31 69 2b fd 97 67 5e 7d e0 05 b3 |&Nl.,1i+..g^}...| -00000100 f4 40 64 a1 bd a3 fd a8 f9 7b 18 82 89 8f 25 f9 |.@d......{....%.| -00000110 ca ca c4 8f e4 90 7b 26 7a d5 b2 1e fa 05 db ad |......{&z.......| -00000120 8a 9f 93 e9 13 5b 28 cc cb 8b 30 f2 4c 1d 73 09 |.....[(...0.L.s.| -00000130 7f 6b 63 5c 29 36 2f fc a5 6e eb 24 79 f8 7c 63 |.kc\)6/..n.$y.|c| -00000140 1f ef 41 72 98 69 7c d6 8d f9 76 d4 4d af b0 71 |..Ar.i|...v.M..q| -00000150 2e f7 f8 b5 73 45 05 52 fa 25 46 02 28 0d d9 7a |....sE.R.%F.(..z| -00000160 60 13 b9 6c 6d fb f3 be e3 04 74 76 72 d6 a4 91 |`..lm.....tvr...| -00000170 d1 2c 0d 1e fa 23 ef c7 80 ff 1e aa 1b af 50 58 |.,...#........PX| -00000180 77 ea 49 d9 22 4d ed bc bf a6 0a 41 8e e7 5b 31 |w.I."M.....A..[1| -00000190 de 33 05 10 46 a5 54 aa 5e 90 5c 15 64 2d 1b e9 |.3..F.T.^.\.d-..| -000001a0 5c fc 93 8d 2f b2 af 74 d7 d2 c6 7f 27 68 fd 44 |\.../..t....'h.D| -000001b0 13 60 70 87 e8 08 e1 e2 af 7f 1a 2c 29 5f 45 fe |.`p........,)_E.| -000001c0 49 9a d0 42 c9 51 ef f7 5b ae 02 df 27 1c 29 20 |I..B.Q..[...'.) | -000001d0 35 4f 3d 7d 74 97 0c 20 be f8 a3 c9 b7 ff 65 69 |5O=}t.. ......ei| -000001e0 08 89 92 fe 85 65 9f 8a 00 4b 9f 39 8d 6f 29 7c |.....e...K.9.o)|| -000001f0 7c e9 16 e4 bd 06 a3 b0 5b 7f cf f0 74 14 56 a2 ||.......[...t.V.| -00000200 76 61 b8 79 10 44 55 4f 25 55 a7 be a4 eb 2e 7d |va.y.DUO%U.....}| -00000210 9a b8 7a d8 d7 34 b6 ef 6c f7 fb ef fd 16 c2 61 |..z..4..l......a| -00000220 89 bb 98 22 c6 80 9e 33 7f e9 35 7a 58 b6 33 1c |..."...3..5zX.3.| -00000230 d6 87 68 b7 62 21 3b 26 9b f1 b1 f2 92 d5 4b 19 |..h.b!;&......K.| -00000240 02 58 05 3c 81 cf 00 5a 54 86 a5 61 8f 71 ae 32 |.X.<...ZT..a.q.2| -00000250 f2 0f 08 3b 13 4d f3 e6 03 2e 73 9c 50 4a b7 6c |...;.M....s.PJ.l| -00000260 d8 0a 04 fc b5 44 a5 45 c8 86 c9 9f 29 b4 00 90 |.....D.E....)...| -00000270 d8 8b e0 c8 ba 63 9f 42 65 ef ba 5b dc b2 61 53 |.....c.Be..[..aS| -00000280 e6 4b 29 72 51 c9 21 d4 d7 2d 14 56 82 80 32 36 |.K)rQ.!..-.V..26| -00000290 fd 72 b6 16 6f 06 71 f9 60 4f 32 ce f6 83 94 75 |.r..o.q.`O2....u| -000002a0 d9 23 d3 41 f8 e7 90 60 80 a8 a5 95 c0 a2 dd 2e |.#.A...`........| -000002b0 e7 60 73 5b c0 a5 a0 bd 8b bc cc 32 8a 9e 30 6a |.`s[.......2..0j| -000002c0 72 2f 61 24 56 0b 1e 3e 52 92 d2 e0 11 cd 52 69 |r/a$V..>R.....Ri| -000002d0 c4 73 7f 72 95 fd f5 c4 72 d7 77 73 85 bf be e0 |.s.r....r.ws....| -000002e0 cd 3c 3b 3d 92 63 91 ba c8 a8 d2 32 40 6a 33 91 |.<;=.c.....2@j3.| -000002f0 c0 71 fe ea 76 9f a9 96 dc c0 a9 bd 67 b6 23 42 |.q..v.......g.#B| -00000300 d9 1b 3d 8d d4 5f 81 38 74 1c db 36 9e bd 79 fd |..=.._.8t..6..y.| -00000310 30 c5 db 17 03 03 00 99 61 1e 25 cb 11 26 61 72 |0.......a.%..&ar| -00000320 a0 9a de 5b ad 5b ad 8a 29 43 eb 76 5a 16 a4 d3 |...[.[..)C.vZ...| -00000330 d4 a1 84 e4 7e a1 2b 76 75 b6 a5 a6 36 da d3 bf |....~.+vu...6...| -00000340 0c 3a 39 ca ef 3f 02 22 63 3d 5d c1 30 94 ba 32 |.:9..?."c=].0..2| -00000350 fe c0 20 c1 ba c8 fa 29 e8 1c c6 ab f0 c9 53 a4 |.. ....)......S.| -00000360 73 d0 9a 2d 50 d8 b4 0c db e5 4a 9f 98 38 34 c9 |s..-P.....J..84.| -00000370 9f ff 63 a9 50 7b 26 00 64 26 4d 0d 20 d5 e5 27 |..c.P{&.d&M. ..'| -00000380 e1 41 7b 5a 03 97 0a 14 8c 55 3c 45 4b 72 ce de |.A{Z.....U...,..N.....| +000001e0 6b 24 30 5d f8 a0 52 ca 02 9a 50 58 43 b6 22 04 |k$0]..R...PXC.".| +000001f0 47 c6 ba 1c e1 aa c2 2c ca 50 13 de 2b 2b 34 ec |G......,.P..++4.| +00000200 80 8c 34 77 20 ab f3 f3 42 d2 9a 2b 9d 38 32 f9 |..4w ...B..+.82.| +00000210 de 9c 7b ee 22 4f 41 9e ea 35 88 8c e7 74 f6 ac |..{."OA..5...t..| +00000220 a5 fe af d9 c4 b1 a4 87 aa 9f d9 02 3b 4b 92 2b |............;K.+| +00000230 f4 ea 6f 81 b8 54 ca 77 3d a2 37 f0 60 d8 26 e4 |..o..T.w=.7.`.&.| +00000240 27 04 c8 5b bf bc 5e 9c 8b a9 f6 c1 58 31 de 48 |'..[..^.....X1.H| +00000250 67 5b c5 d7 ae 35 cc d8 8d 01 9b e0 d0 c7 33 f8 |g[...5........3.| +00000260 36 c1 e8 50 32 27 87 6c 64 71 27 3f 66 82 fa 92 |6..P2'.ldq'?f...| +00000270 d3 fe b9 9e d3 e2 63 43 64 13 73 4b e9 34 d9 ae |......cCd.sK.4..| +00000280 49 dd 9c 17 f3 a6 34 27 3e 07 68 ed 6b 98 0a 69 |I.....4'>.h.k..i| +00000290 9c f5 4f 1e fc ac 72 08 7e bb bd 97 fe f9 c1 09 |..O...r.~.......| +000002a0 a2 69 75 c4 e7 a1 83 df 03 0f 74 21 e8 4c 4c d7 |.iu.......t!.LL.| +000002b0 a2 90 5a 4d d9 a2 fa f1 e4 dc ff 75 e8 fe a1 91 |..ZM.......u....| +000002c0 46 a4 c2 0b c3 9e 83 45 40 b4 ba fe ed 9e 3a 81 |F......E@.....:.| +000002d0 a9 27 b9 fa 6f 9e 38 10 2f e6 05 9a 00 43 e9 fb |.'..o.8./....C..| +000002e0 60 b8 26 2e 18 4a 0c 14 6a 9f aa 26 1d 53 9e 4f |`.&..J..j..&.S.O| +000002f0 fc 7e e1 90 60 28 b1 89 db b6 68 58 5d 2a fa 43 |.~..`(....hX]*.C| +00000300 91 99 c1 94 74 77 a9 30 cf 1e 9d 49 fc cd be 6c |....tw.0...I...l| +00000310 5a 72 90 17 03 03 00 99 1b 1f 16 9b 8e 85 5a 8a |Zr............Z.| +00000320 3c c7 d8 e5 86 20 eb 71 83 12 e7 3b 02 ca a1 18 |<.... .q...;....| +00000330 aa b7 ef 67 9d bf 57 46 14 3b ed b2 9b 2e f1 96 |...g..WF.;......| +00000340 67 b0 cc 20 ab 67 63 5c a9 96 d8 22 8e fa f0 94 |g.. .gc\..."....| +00000350 1e 83 6c 53 8a 75 a5 37 1d f1 de 1f a7 22 13 08 |..lS.u.7....."..| +00000360 b7 13 dd 25 26 eb 7a 85 72 33 ed a8 27 be cc ff |...%&.z.r3..'...| +00000370 8d 42 fa 6a f8 0a 59 44 6f 48 4d 58 27 d5 28 fb |.B.j..YDoHMX'.(.| +00000380 37 b0 85 25 49 d0 32 7c dd b8 46 1a aa dc 16 b7 |7..%I.2|..F.....| +00000390 f2 fd 99 17 67 41 c0 2d 75 0d 66 ef 65 e2 5f af |....gA.-u.f.e._.| +000003a0 d8 b4 aa 15 1c 24 40 0d 0d 4f a8 bc c9 92 1c ab |.....$@..O......| +000003b0 91 17 03 03 00 35 f9 69 7e 2a d6 fa 6e 42 25 6c |.....5.i~*..nB%l| +000003c0 da 96 f7 79 5f 02 44 1f d8 1c a2 41 fc eb a5 12 |...y_.D....A....| +000003d0 bf 37 ab b0 40 92 55 50 64 8e 7b e9 e5 f3 04 b5 |.7..@.UPd.{.....| +000003e0 0f 04 93 98 fa ca 24 69 a1 73 26 |......$i.s&| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 00 35 a6 7f ef 71 68 |..........5...qh| -00000010 8b 86 84 20 a6 e7 65 3f 3f f8 c6 8b 62 40 31 e2 |... ..e??...b@1.| -00000020 1a ee 8d 0a 64 88 ea 4b 83 5f c5 ff 1b 9d aa 5f |....d..K._....._| -00000030 ce a4 31 76 06 90 da ad b9 17 49 0b a6 b5 37 80 |..1v......I...7.| -00000040 17 03 03 00 17 2b be e8 98 11 37 0d db 46 69 0d |.....+....7..Fi.| -00000050 51 34 ee 13 00 c3 f1 12 8a 13 21 b7 17 03 03 00 |Q4........!.....| -00000060 13 8d ec 94 6f 55 b1 d5 c0 d6 a3 6e a2 8d 67 76 |....oU.....n..gv| -00000070 7b c7 b8 1d |{...| +00000000 14 03 03 00 01 01 17 03 03 00 35 80 f8 9b f1 1d |..........5.....| +00000010 89 31 a8 ea b8 cc 44 a7 6c c3 f7 ea 0f fd f4 a3 |.1....D.l.......| +00000020 4c c2 15 06 bd 79 16 90 a1 bf 44 df e5 21 03 ff |L....y....D..!..| +00000030 e5 83 08 67 70 09 11 21 8e 96 47 e6 f4 e3 36 6e |...gp..!..G...6n| +00000040 17 03 03 00 17 7b 87 be 4e f8 ce 6f 30 b8 5b 95 |.....{..N..o0.[.| +00000050 20 79 23 f7 50 6b 17 b8 13 a2 38 6a 17 03 03 00 | y#.Pk....8j....| +00000060 13 2a d5 e0 a4 17 7d 4f 3b 5a 6c bc 9d 5a 79 5b |.*....}O;Zl..Zy[| +00000070 a8 f6 b2 4a |...J| diff --git a/src/crypto/tls/testdata/Client-TLSv13-HelloRetryRequest b/src/crypto/tls/testdata/Client-TLSv13-HelloRetryRequest index a669920d938177..4d58f8d440da5b 100644 --- a/src/crypto/tls/testdata/Client-TLSv13-HelloRetryRequest +++ b/src/crypto/tls/testdata/Client-TLSv13-HelloRetryRequest @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fa 01 00 00 f6 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 14 01 00 01 10 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,14 +7,16 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7b 00 0b 00 02 01 00 ff 01 00 01 00 00 |...{............| +00000080 01 00 00 95 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| -000000a0 00 0a 00 06 00 04 00 1d 00 17 00 0d 00 1a 00 18 |................| +000000a0 00 0a 00 06 00 04 00 1d 00 17 00 0d 00 16 00 14 |................| 000000b0 08 04 04 03 08 07 08 05 08 06 04 01 05 01 06 01 |................| -000000c0 05 03 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 |.........+......| -000000d0 03 03 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f |......3.&.$... /| -000000e0 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| -000000f0 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |.........._X.;t| +000000c0 05 03 06 03 00 32 00 1a 00 18 08 04 04 03 08 07 |.....2..........| +000000d0 08 05 08 06 04 01 05 01 06 01 05 03 06 03 02 01 |................| +000000e0 02 03 00 2b 00 09 08 03 04 03 03 03 02 03 01 00 |...+............| +000000f0 33 00 26 00 24 00 1d 00 20 2f e5 7d a3 47 cd 62 |3.&.$... /.}.G.b| +00000100 43 15 28 da ac 5f bb 29 07 30 ff f6 84 af c4 cf |C.(.._.).0......| +00000110 c2 ed 90 99 5f 58 cb 3b 74 |...._X.;t| >>> Flow 2 (server to client) 00000000 16 03 03 00 58 02 00 00 54 03 03 cf 21 ad 74 e5 |....X...T...!.t.| 00000010 9a 61 11 be 1d 8c 02 1e 65 b8 91 c2 a2 11 16 7a |.a......e......z| @@ -24,7 +26,7 @@ 00000050 0c 00 2b 00 02 03 04 00 33 00 02 00 17 14 03 03 |..+.....3.......| 00000060 00 01 01 |...| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 16 03 03 01 1b 01 00 01 17 03 |................| +00000000 14 03 03 00 01 01 16 03 03 01 35 01 00 01 31 03 |..........5...1.| 00000010 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000030 00 20 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |. ..............| @@ -32,88 +34,89 @@ 00000050 00 00 00 32 cc a9 cc a8 c0 2b c0 2f c0 2c c0 30 |...2.....+./.,.0| 00000060 c0 09 c0 13 c0 0a c0 14 00 9c 00 9d 00 2f 00 35 |............./.5| 00000070 c0 12 00 0a c0 23 c0 27 00 3c c0 07 c0 11 00 05 |.....#.'.<......| -00000080 13 03 13 01 13 02 01 00 00 9c 00 0b 00 02 01 00 |................| +00000080 13 03 13 01 13 02 01 00 00 b6 00 0b 00 02 01 00 |................| 00000090 ff 01 00 01 00 00 17 00 00 00 12 00 00 00 05 00 |................| 000000a0 05 01 00 00 00 00 00 0a 00 06 00 04 00 1d 00 17 |................| -000000b0 00 0d 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 |................| -000000c0 04 01 05 01 06 01 05 03 06 03 02 01 02 03 00 2b |...............+| -000000d0 00 09 08 03 04 03 03 03 02 03 01 00 33 00 47 00 |............3.G.| -000000e0 45 00 17 00 41 04 1e 18 37 ef 0d 19 51 88 35 75 |E...A...7...Q.5u| -000000f0 71 b5 e5 54 5b 12 2e 8f 09 67 fd a7 24 20 3e b2 |q..T[....g..$ >.| -00000100 56 1c ce 97 28 5e f8 2b 2d 4f 9e f1 07 9f 6c 4b |V...(^.+-O....lK| -00000110 5b 83 56 e2 32 42 e9 58 b6 d7 49 a6 b5 68 1a 41 |[.V.2B.X..I..h.A| -00000120 03 56 6b dc 5a 89 |.Vk.Z.| +000000b0 00 0d 00 16 00 14 08 04 04 03 08 07 08 05 08 06 |................| +000000c0 04 01 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 |...........2....| +000000d0 08 04 04 03 08 07 08 05 08 06 04 01 05 01 06 01 |................| +000000e0 05 03 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 |.........+......| +000000f0 03 03 02 03 01 00 33 00 47 00 45 00 17 00 41 04 |......3.G.E...A.| +00000100 1e 18 37 ef 0d 19 51 88 35 75 71 b5 e5 54 5b 12 |..7...Q.5uq..T[.| +00000110 2e 8f 09 67 fd a7 24 20 3e b2 56 1c ce 97 28 5e |...g..$ >.V...(^| +00000120 f8 2b 2d 4f 9e f1 07 9f 6c 4b 5b 83 56 e2 32 42 |.+-O....lK[.V.2B| +00000130 e9 58 b6 d7 49 a6 b5 68 1a 41 03 56 6b dc 5a 89 |.X..I..h.A.Vk.Z.| >>> Flow 4 (server to client) -00000000 16 03 03 00 9b 02 00 00 97 03 03 9a 1f 7a fa 06 |.............z..| -00000010 c4 5f 92 61 62 d2 a5 f3 ae b9 a6 d8 6b eb 13 a7 |._.ab.......k...| -00000020 2f 17 62 79 3a ca 42 8b 18 06 15 20 00 00 00 00 |/.by:.B.... ....| +00000000 16 03 03 00 9b 02 00 00 97 03 03 3e b5 60 d1 cd |...........>.`..| +00000010 f5 a6 76 2c bd 49 f6 0f c2 ff ff 6b c9 f6 45 19 |..v,.I.....k..E.| +00000020 f6 e3 e7 fc 31 8b 49 4e 88 29 55 20 00 00 00 00 |....1.IN.)U ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000040 00 00 00 00 00 00 00 00 00 00 00 00 13 03 00 00 |................| 00000050 4f 00 2b 00 02 03 04 00 33 00 45 00 17 00 41 04 |O.+.....3.E...A.| -00000060 d1 56 18 67 59 a5 e1 ed 99 0f ca e5 6b 5f e3 d9 |.V.gY.......k_..| -00000070 e3 fd e6 a2 50 93 bf 87 85 60 e7 7f 0b a0 96 c4 |....P....`......| -00000080 29 55 63 de fe 6b 75 99 b2 a6 75 92 61 d0 96 19 |)Uc..ku...u.a...| -00000090 f5 60 8f ec e9 d3 98 4a b9 6b 98 26 c1 3e 27 7e |.`.....J.k.&.>'~| -000000a0 17 03 03 00 17 5a aa bb 35 67 ec 83 da 8e 85 bd |.....Z..5g......| -000000b0 ae a5 0d c9 8b cd 03 bf 96 2b 35 61 17 03 03 02 |.........+5a....| -000000c0 6d 85 0c 7c d5 ff 0f 77 0a a4 b7 97 aa d3 f7 af |m..|...w........| -000000d0 4b 58 cb 24 4d 9f a6 d8 e2 c3 10 ce 3a 5d e4 cc |KX.$M.......:]..| -000000e0 84 7d 1f 71 bd b6 25 0c 10 75 d3 8d b6 12 04 56 |.}.q..%..u.....V| -000000f0 63 af 53 e5 cc 29 ce f7 18 7a 93 91 73 ae ff c1 |c.S..)...z..s...| -00000100 2e 2b 3d 76 ab 99 c8 e0 fd 69 b3 67 39 e9 58 40 |.+=v.....i.g9.X@| -00000110 4a 63 e8 71 c5 8d de 44 fb ef 4d 0a dc ae 51 12 |Jc.q...D..M...Q.| -00000120 7e ea 6a 25 32 8e 5b 08 6e c7 b2 87 00 4e 35 d8 |~.j%2.[.n....N5.| -00000130 cd 3b bd 8c dc e8 55 a9 2f 65 d1 7f c1 28 33 f9 |.;....U./e...(3.| -00000140 70 e1 af d8 2d 64 4c ac 5a 3a 16 6d 3e 97 a7 0e |p...-dL.Z:.m>...| -00000150 cd 28 05 1c ea 28 8d 63 a3 41 e2 3a 24 27 f4 43 |.(...(.c.A.:$'.C| -00000160 97 c8 89 09 ed 8b 96 74 b3 dc 98 f4 0c c2 6f 25 |.......t......o%| -00000170 fc 30 8a f8 25 49 e1 91 fb c6 39 23 ca d5 16 08 |.0..%I....9#....| -00000180 5f bf 10 88 71 a4 c5 ac e7 ad 9e 5a 4d e8 d3 1f |_...q......ZM...| -00000190 f0 5d 83 9d c9 e2 b3 f8 e4 a3 a5 57 5d ca 30 74 |.].........W].0t| -000001a0 a6 1b c7 51 50 05 5f 7e 93 71 1a 8d fc 04 d6 fe |...QP._~.q......| -000001b0 4e 4b 95 54 18 43 19 4d d2 dc 81 a8 94 76 7e 23 |NK.T.C.M.....v~#| -000001c0 56 8c 86 f9 15 d2 a8 0c 48 4e 0e 81 df e0 31 52 |V.......HN....1R| -000001d0 c9 2e b4 39 f3 cf ae e2 29 72 01 ed 34 4f 09 ce |...9....)r..4O..| -000001e0 64 de a9 9b 84 61 3a bb 0b ac 4e b1 6d 37 41 10 |d....a:...N.m7A.| -000001f0 5b cf 99 b1 3e e6 f5 7b 1c d4 d0 d5 34 fc ab c6 |[...>..{....4...| -00000200 29 8e 0b 53 e4 7b 35 33 e4 e3 f6 77 9b f6 c6 ae |)..S.{53...w....| -00000210 73 3c c5 80 c8 72 f7 56 6c 4c ce e5 a7 05 63 ee |s<...r.VlL....c.| -00000220 07 e0 1d 3e 73 3e 55 73 ba 63 a8 d7 99 6a 48 57 |...>s>Us.c...jHW| -00000230 0c 55 51 42 4b 27 91 4b 75 b3 3b 4c 29 f1 52 e8 |.UQBK'.Ku.;L).R.| -00000240 85 f9 49 53 ed 87 6d 9e 8f 5c 29 7c 5c 9c 4e 9a |..IS..m..\)|\.N.| -00000250 90 a8 99 2a 53 65 f7 d9 61 ef 6a 56 2a da 76 ed |...*Se..a.jV*.v.| -00000260 04 84 11 f5 1c 40 82 52 7d fa 00 79 43 12 42 b4 |.....@.R}..yC.B.| -00000270 ca 7a 86 d5 8d d6 c5 7b 04 13 79 08 72 0c ce d7 |.z.....{..y.r...| -00000280 88 9a f2 2c b5 9a 18 a4 d2 87 48 ce ab 17 33 fd |...,......H...3.| -00000290 cf 22 3c 72 03 86 af 35 2a b3 fe 24 0e 86 d8 82 |."......D..| -000002c0 14 12 cd 16 e5 6a 58 1b 3f f8 70 d6 47 e6 1b 98 |.....jX.?.p.G...| -000002d0 74 9c 21 2b 6d f8 8a 47 05 86 b1 fa 63 9d cc 19 |t.!+m..G....c...| -000002e0 da 9a a0 ea 5a 26 98 e6 b7 d0 da 03 b3 1d 33 e8 |....Z&........3.| -000002f0 55 74 ee 4b 6a 06 0f f1 45 35 72 81 bb c9 61 4e |Ut.Kj...E5r...aN| -00000300 60 d6 83 e6 ab 08 77 a4 f9 73 e5 d8 cb 66 9f 41 |`.....w..s...f.A| -00000310 00 15 2c 09 0e ad 16 02 6c 06 5f bc bf 47 f1 5a |..,.....l._..G.Z| -00000320 85 97 ab 88 e8 45 b0 31 96 ad 56 00 97 59 17 03 |.....E.1..V..Y..| -00000330 03 00 99 5e 77 80 46 1d 2f 47 9f b3 43 4d a6 2b |...^w.F./G..CM.+| -00000340 e9 f7 72 a2 d0 c6 69 7c df 5d ce a0 3b 89 e2 69 |..r...i|.]..;..i| -00000350 e7 35 5e 00 fb 87 10 4a a8 41 02 b7 89 fa 88 fc |.5^....J.A......| -00000360 72 63 ce bf 41 61 16 91 55 06 cd b6 fa 78 09 bc |rc..Aa..U....x..| -00000370 f6 21 a0 c2 1f f3 fa 34 2a 9e b9 8a 0c 72 81 02 |.!.....4*....r..| -00000380 66 a7 da 6b d6 e2 aa 99 6d 4b 42 f5 19 83 52 89 |f..k....mKB...R.| -00000390 9a 26 56 5d d5 a4 a7 bd ad b4 10 db ee 87 1d 15 |.&V]............| -000003a0 15 df 30 95 d8 61 e2 c4 88 47 e1 91 e3 85 33 fe |..0..a...G....3.| -000003b0 7d 02 0c 22 6b 52 22 a7 cf bf bf f0 9d 25 65 e0 |}.."kR"......%e.| -000003c0 6b da a2 8c 9a 00 3d 68 78 55 38 69 17 03 03 00 |k.....=hxU8i....| -000003d0 35 c8 ca 51 96 aa c1 4b a0 09 d3 bc d9 ea 35 db |5..Q...K......5.| -000003e0 f2 8d bb ab b3 23 09 a1 d1 9c 40 3e e4 84 d3 1c |.....#....@>....| -000003f0 60 64 5c ff 01 c7 78 69 c2 4a fb e1 c3 d6 f0 57 |`d\...xi.J.....W| -00000400 fa 3a 5f 7a 8b 70 |.:_z.p| +00000060 f8 a5 4d 81 0a 05 43 42 0b d6 ca 3b 3e 02 44 20 |..M...CB...;>.D | +00000070 56 89 80 d3 03 50 1a b4 70 50 78 aa 81 2b 5b 03 |V....P..pPx..+[.| +00000080 6c 38 22 c1 17 4f e5 b9 57 ab 58 33 76 8e ae c9 |l8"..O..W.X3v...| +00000090 a0 74 47 a0 10 c0 5c a7 8c 77 24 c8 90 bb f4 2a |.tG...\..w$....*| +000000a0 17 03 03 00 17 0c 2a 67 cb 9f a8 e8 5b 97 4a 75 |......*g....[.Ju| +000000b0 24 80 65 28 15 a7 d4 71 eb 24 74 7e 17 03 03 02 |$.e(...q.$t~....| +000000c0 6d 89 a3 fa 0f a6 dd dc b5 5a f9 03 93 5a 2a a6 |m........Z...Z*.| +000000d0 2f 65 81 9e 0d 2c 21 25 0b 7c 04 41 40 3e 95 a4 |/e...,!%.|.A@>..| +000000e0 31 21 2a 08 db 35 bc 5a 93 aa b4 8f 30 28 0c e5 |1!*..5.Z....0(..| +000000f0 11 f6 31 d6 7a b0 df 9c f3 76 30 72 e2 27 1c d3 |..1.z....v0r.'..| +00000100 75 68 3d 63 71 27 da 72 7a 18 19 da 25 94 ee e3 |uh=cq'.rz...%...| +00000110 62 6f 45 c5 38 67 ba 4a 1f 6f f3 ea 46 c6 09 ad |boE.8g.J.o..F...| +00000120 db 3c 42 3c d7 8d 9c 49 8d 2c 22 15 74 98 60 5b |...| +00000190 c6 b2 c3 eb ec 03 7d d2 85 2b c3 7f e7 07 c1 50 |......}..+.....P| +000001a0 c9 b6 6f ab 7a 86 7b b3 fd 8b 9a 6a 3c ea 18 06 |..o.z.{....j<...| +000001b0 13 e4 a4 9b d0 2c ef 8c 3f ea 8e d8 c4 d4 54 33 |.....,..?.....T3| +000001c0 2e e5 b6 40 85 a8 4b 5d 9b 3c f2 9e 70 5a 23 d7 |...@..K].<..pZ#.| +000001d0 c1 e1 b4 7a 42 69 fb b1 49 99 cf 60 5c 8a c0 7f |...zBi..I..`\...| +000001e0 c2 8c 8c 26 4f 0f 6c 10 bd e4 51 ab 19 4d 2a 63 |...&O.l...Q..M*c| +000001f0 b3 39 1e d6 55 00 c9 89 3a c0 5a 2b 5b 0a e9 10 |.9..U...:.Z+[...| +00000200 56 4c 8a 88 2d 37 27 50 07 2a 77 19 92 6a a0 67 |VL..-7'P.*w..j.g| +00000210 c9 34 e7 b2 e4 f8 f8 fc e0 24 c1 8e 6a 04 b0 6a |.4.......$..j..j| +00000220 35 1b 5a 74 78 34 81 09 d5 ed bc 61 55 bd bf 6f |5.Ztx4.....aU..o| +00000230 e4 a1 13 e9 b1 9a de de 54 3b 99 19 f7 90 74 be |........T;....t.| +00000240 48 b0 80 92 e7 19 91 17 12 12 9a 08 77 bf 2c 21 |H...........w.,!| +00000250 33 51 9a 02 fc 8c c5 7b 90 52 89 61 df 64 1f af |3Q.....{.R.a.d..| +00000260 1e 49 2a 7b 33 1b 47 df 34 7f aa ca e3 16 2d 28 |.I*{3.G.4.....-(| +00000270 94 cc 07 d1 0d 1a 07 80 a4 c6 cc 10 a7 72 58 bc |.............rX.| +00000280 65 92 de a1 c7 8b e5 63 9a 83 85 eb ee e8 8d e9 |e......c........| +00000290 63 06 f0 6d 14 81 5a d5 a6 1b 84 d3 a2 5d f1 94 |c..m..Z......]..| +000002a0 a4 af f7 eb 85 d3 00 41 12 b3 bc bf 4a 1a c1 c0 |.......A....J...| +000002b0 b0 46 38 63 f4 05 de e0 c2 b7 ee 5b f8 c8 6c e7 |.F8c.......[..l.| +000002c0 fe a4 3d a2 52 e8 25 4c d0 b4 dd 1a aa e4 9c 5b |..=.R.%L.......[| +000002d0 3c 5c 9b 0f e2 ca 18 a8 59 1d 22 d8 2d cb ca 0d |<\......Y.".-...| +000002e0 61 f0 8e be 55 5e 0b b4 72 2f 32 f1 47 fe b8 94 |a...U^..r/2.G...| +000002f0 ec 14 af c4 59 dd f9 b1 1b 18 84 44 54 c0 16 f0 |....Y......DT...| +00000300 f4 e6 19 82 fb 00 84 6e c0 87 05 82 33 7b 62 d8 |.......n....3{b.| +00000310 e3 83 86 55 d6 06 bd e5 a3 1c c0 f1 ef b3 ae 2c |...U...........,| +00000320 f7 6e fa 9a 3d 6c cf de 5d 77 54 08 03 0c 17 03 |.n..=l..]wT.....| +00000330 03 00 99 03 00 b3 cc ca 76 27 18 16 0a 8e d0 81 |........v'......| +00000340 32 5d 9e 32 75 5e bb b4 c5 5c 6b 61 62 38 3d 2e |2].2u^...\kab8=.| +00000350 21 1a 93 c9 fc b7 90 74 17 78 ae 18 78 86 fc 6d |!......t.x..x..m| +00000360 0b 0c 50 d3 a6 5a 98 a8 76 6f ff 3b 98 f5 2f 11 |..P..Z..vo.;../.| +00000370 3d af 84 92 04 42 93 33 33 5c 9a f2 fc 5f 05 dd |=....B.33\..._..| +00000380 89 a8 c3 c1 e7 92 0f 76 3a ed da 7c 25 e0 96 89 |.......v:..|%...| +00000390 27 97 c1 3e 27 70 3f a4 15 f3 24 6a 47 43 7f d7 |'..>'p?...$jGC..| +000003a0 70 b9 7b 2d 3d 53 c0 f6 53 85 fb f7 62 4a 01 8e |p.{-=S..S...bJ..| +000003b0 3e 30 3b b4 79 17 af 2e 73 56 e5 de ba 48 0e 0c |>0;.y...sV...H..| +000003c0 e0 d3 0a 80 07 b9 53 f1 1d a6 cf 29 17 03 03 00 |......S....)....| +000003d0 35 f8 8f 94 ec 0a b9 d9 b1 2c d3 02 c9 0b ef 19 |5........,......| +000003e0 89 ee a9 c0 b1 f3 9c 87 09 0e fb 9a e2 ad 4a 1e |..............J.| +000003f0 16 6d 84 af b3 fa b9 af 98 79 54 7d 45 29 99 85 |.m.......yT}E)..| +00000400 8c a9 69 b6 ee 4e |..i..N| >>> Flow 5 (client to server) -00000000 17 03 03 00 35 24 4a 53 55 49 92 7a 21 a2 db 69 |....5$JSUI.z!..i| -00000010 e5 2b 00 29 83 cb 91 32 17 48 73 96 89 d1 b2 11 |.+.)...2.Hs.....| -00000020 c4 23 48 41 eb df eb 5a 10 c2 78 4e a2 e4 f0 38 |.#HA...Z..xN...8| -00000030 4a 1c b6 d3 bb 52 bc 7e f6 94 17 03 03 00 17 74 |J....R.~.......t| -00000040 04 34 8d 27 19 da f3 d8 14 30 1b 85 ad 1e 99 cd |.4.'.....0......| -00000050 06 f0 a3 50 94 48 17 03 03 00 13 ac 8f 7e 2e 9c |...P.H.......~..| -00000060 90 96 db 33 e2 e9 94 c1 c9 95 f2 7a d1 8f |...3.......z..| +00000000 17 03 03 00 35 4c c3 af 0b 49 46 e0 51 3a 73 a8 |....5L...IF.Q:s.| +00000010 28 46 95 22 ef d7 05 8a fc 4f 12 98 2f 09 ca a9 |(F.".....O../...| +00000020 4d b8 17 e4 15 15 b5 13 3f 1a cb 8e 3d 19 00 d6 |M.......?...=...| +00000030 19 af 90 66 0e 93 d4 22 4d 54 17 03 03 00 17 56 |...f..."MT.....V| +00000040 31 7c 05 02 1d 84 64 d0 32 21 8d c1 61 7e 7f ce |1|....d.2!..a~..| +00000050 d3 45 b2 df ba df 17 03 03 00 13 3a 57 b6 2e 79 |.E.........:W..y| +00000060 96 09 e9 66 f2 85 7a 21 88 45 cc b4 ff 12 |...f..z!.E....| diff --git a/src/crypto/tls/testdata/Client-TLSv13-KeyUpdate b/src/crypto/tls/testdata/Client-TLSv13-KeyUpdate index 1b8004ad0a08ac..7d04d95a272d9c 100644 --- a/src/crypto/tls/testdata/Client-TLSv13-KeyUpdate +++ b/src/crypto/tls/testdata/Client-TLSv13-KeyUpdate @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 fe 01 00 00 fa 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 18 01 00 01 14 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,97 +7,98 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 7f 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 99 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| 000000a0 00 0a 00 0a 00 08 00 1d 00 17 00 18 00 19 00 0d |................| -000000b0 00 1a 00 18 08 04 04 03 08 07 08 05 08 06 04 01 |................| -000000c0 05 01 06 01 05 03 06 03 02 01 02 03 00 2b 00 09 |.............+..| -000000d0 08 03 04 03 03 03 02 03 01 00 33 00 26 00 24 00 |..........3.&.$.| -000000e0 1d 00 20 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f |.. /.}.G.bC.(.._| -000000f0 bb 29 07 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 |.).0.........._X| -00000100 cb 3b 74 |.;t| +000000b0 00 16 00 14 08 04 04 03 08 07 08 05 08 06 04 01 |................| +000000c0 05 01 06 01 05 03 06 03 00 32 00 1a 00 18 08 04 |.........2......| +000000d0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| +000000e0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| +000000f0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| +00000100 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| +00000110 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 7a 02 00 00 76 03 03 dc 81 c4 82 2e |....z...v.......| -00000010 a2 4f c4 c2 53 c2 bc 6a bd f3 46 84 b5 ba 66 b5 |.O..S..j..F...f.| -00000020 8b 67 7c 90 51 40 12 39 18 e1 bf 20 00 00 00 00 |.g|.Q@.9... ....| +00000000 16 03 03 00 7a 02 00 00 76 03 03 b2 4e f2 da 8b |....z...v...N...| +00000010 79 31 33 74 d5 37 8e 44 9d ad 6f 91 c4 bb 5d d4 |y13t.7.D..o...].| +00000020 fd f1 e2 42 50 43 f5 3c 7f cf 92 20 00 00 00 00 |...BPC.<... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000040 00 00 00 00 00 00 00 00 00 00 00 00 13 03 00 00 |................| -00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 3b |..+.....3.$... ;| -00000060 6b 2a e6 c2 7e b6 59 68 e4 e3 f2 f3 14 e5 72 bc |k*..~.Yh......r.| -00000070 c9 61 b4 b0 0a c6 41 0d a9 8e d9 9b 7d 2a 11 14 |.a....A.....}*..| -00000080 03 03 00 01 01 17 03 03 00 17 43 af 38 b9 56 06 |..........C.8.V.| -00000090 2d 10 e3 e5 1d 1b 1e a9 5f 90 ca 0d a9 52 33 86 |-......._....R3.| -000000a0 85 17 03 03 02 6d ac 6a a1 8f 42 27 74 80 25 f9 |.....m.j..B't.%.| -000000b0 1f 48 49 2d c2 33 38 e7 93 7e b0 b2 50 b8 6a ea |.HI-.38..~..P.j.| -000000c0 a6 81 ef 9b 55 83 4e 93 df 92 97 6f 00 f5 c4 fc |....U.N....o....| -000000d0 ec b1 19 dd 68 b5 bd c4 bb ba 63 9a e4 c9 24 af |....h.....c...$.| -000000e0 88 13 65 11 bf ea d9 07 e9 46 fd 5b 60 ce 57 46 |..e......F.[`.WF| -000000f0 8b a9 bd c6 58 1a 3b bd 5e fb 0f 46 ec fc 8b 2c |....X.;.^..F...,| -00000100 ea a7 19 06 6a e5 6f 10 7a 27 04 6b aa a4 2c f4 |....j.o.z'.k..,.| -00000110 ef 3b e8 8a 51 88 fd e0 ae 33 b1 4c b3 04 5e 91 |.;..Q....3.L..^.| -00000120 b0 98 0b 9f 38 a3 3c fb 9f d5 d2 36 e4 09 19 18 |....8.<....6....| -00000130 a5 b3 12 aa c9 03 ac b5 ab bb f1 7a 02 d2 dd 75 |...........z...u| -00000140 0e cb 60 09 39 23 c3 b2 c1 8e e0 18 57 72 54 61 |..`.9#......WrTa| -00000150 4c 99 35 1d ba 31 01 0c 48 d2 f2 88 22 9c 91 7d |L.5..1..H..."..}| -00000160 e3 74 f9 b3 52 bf 0e 0b e1 31 7c 2c cb fd f2 8c |.t..R....1|,....| -00000170 bf 27 40 6d 26 b7 62 47 56 91 22 00 67 9a df 4f |.'@m&.bGV.".g..O| -00000180 f0 47 57 3c a6 46 4d 16 f6 8d fc 2d 91 c4 1a bf |.GW<.FM....-....| -00000190 38 63 ec 63 fe 97 14 80 aa 5b 60 ff c8 77 57 9c |8c.c.....[`..wW.| -000001a0 d5 86 a6 76 96 b3 e9 db c9 eb dd 94 84 2a 46 f5 |...v.........*F.| -000001b0 6a c1 10 66 59 f1 13 b9 41 f3 89 26 ba 52 69 95 |j..fY...A..&.Ri.| -000001c0 b1 f3 66 30 f8 aa 90 f7 90 49 19 48 4c 25 4a 1f |..f0.....I.HL%J.| -000001d0 12 9d 67 32 79 bb 53 d8 c5 d1 b4 6e 89 75 49 c0 |..g2y.S....n.uI.| -000001e0 65 86 ac 72 23 2f 97 d3 ae e2 64 79 5e e2 10 4e |e..r#/....dy^..N| -000001f0 55 0c c6 70 d3 2e 4a 6c b0 73 0a 11 eb ae f7 a1 |U..p..Jl.s......| -00000200 a1 f0 5f 67 45 46 d3 8c 11 ff 21 62 7d ed f9 0e |.._gEF....!b}...| -00000210 2a ba b3 82 f5 6b c1 4a 4e cc 11 90 48 81 96 7a |*....k.JN...H..z| -00000220 df f9 22 ae 53 31 14 9d c9 5c 85 e7 db a2 dd 02 |..".S1...\......| -00000230 56 eb d0 fe 20 35 21 c9 33 63 b5 b7 a8 93 30 7f |V... 5!.3c....0.| -00000240 86 1f d1 af b1 ff 3e 9a d1 a8 90 d9 9c 86 55 e8 |......>.......U.| -00000250 d0 4c c4 6d a3 ce c6 c0 df f8 a8 b6 43 03 ae fc |.L.m........C...| -00000260 7d 94 7b fe be 85 46 d8 42 9d b6 15 b9 a3 27 3d |}.{...F.B.....'=| -00000270 80 64 54 c8 53 c4 a7 94 52 8f 9d 4c 58 54 a5 c4 |.dT.S...R..LXT..| -00000280 e0 e7 2e cb f3 8c d0 82 3f 95 76 c9 ea ea 80 41 |........?.v....A| -00000290 21 5d 3e a3 1e be 4e 0b ce 10 ab 61 a5 76 ef 62 |!]>...N....a.v.b| -000002a0 50 1b 52 a8 75 23 fd eb ea 76 f9 d8 41 3c a2 e4 |P.R.u#...v..A<..| -000002b0 21 cb 56 f7 40 81 78 56 22 06 2a 38 fc 1f d8 9f |!.V.@.xV".*8....| -000002c0 38 b0 7a 93 f0 8c ad 3e 54 27 a2 d7 8b 2c 79 46 |8.z....>T'...,yF| -000002d0 15 65 f1 55 b2 2a 06 a1 97 9b 47 23 f8 9a 3f 88 |.e.U.*....G#..?.| -000002e0 8e 26 7e 13 cd 6e 8b cb d5 a5 78 48 f7 ba ad d8 |.&~..n....xH....| -000002f0 08 3b 34 5b 52 cd e3 2d 12 ac 81 00 c0 d0 4d df |.;4[R..-......M.| -00000300 56 d6 40 86 91 31 3d ba 6b 41 bc 51 6f ac b2 df |V.@..1=.kA.Qo...| -00000310 90 4b 78 17 03 03 00 99 21 0f 5b 18 54 84 98 0c |.Kx.....!.[.T...| -00000320 3a 7b 0b db 99 0e 09 f6 b7 4f a9 cc da bf 4c ac |:{.......O....L.| -00000330 5f 44 fc ba 9d 5d 52 d5 ec 2e 08 0c cc 3c e1 72 |_D...]R......<.r| -00000340 10 77 5b 7b 55 f7 c1 44 a3 25 e0 48 20 9a 3a de |.w[{U..D.%.H .:.| -00000350 2f ae 30 a9 e9 5b 75 84 e3 59 f0 6b 23 a6 d8 20 |/.0..[u..Y.k#.. | -00000360 16 51 2a 19 61 60 35 28 74 41 32 fa 97 8d 3b ed |.Q*.a`5(tA2...;.| -00000370 ff 64 94 d7 27 4c 9b 1e 5e b8 89 43 e3 ae 2a b6 |.d..'L..^..C..*.| -00000380 60 a8 bb f0 d1 75 85 15 bb 95 3c 82 f1 62 da b1 |`....u....<..b..| -00000390 25 d2 58 7d 67 7f 02 5b b4 91 68 3c 70 10 09 94 |%.X}g..[..h...-2.| +000003c0 cd 16 ab da cd e7 93 0d c2 90 a2 62 7b 3c 54 bb |...........b{>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 00 35 c9 09 ac f5 44 |..........5....D| -00000010 40 83 88 5a b9 46 70 b9 ff 9b 2e bb b4 7e 72 b0 |@..Z.Fp......~r.| -00000020 85 26 d6 37 33 ec d1 ac ce f4 db 72 8c e1 07 b5 |.&.73......r....| -00000030 d0 ce ee 2f 19 77 62 ec 97 ae 1b e1 5f 85 bf c4 |.../.wb....._...| -00000040 17 03 03 00 17 1c ae 0f 1d 50 be 4b d0 64 4f 23 |.........P.K.dO#| -00000050 41 60 d9 c7 f5 60 a6 5c 38 14 fd d5 |A`...`.\8...| +00000000 14 03 03 00 01 01 17 03 03 00 35 a8 d6 93 8f 7e |..........5....~| +00000010 65 3a ee 6f c8 37 2a 27 14 dc b7 19 0e aa 14 b6 |e:.o.7*'........| +00000020 22 21 2d ea f4 00 d1 60 55 c0 f1 56 db 43 7a f0 |"!-....`U..V.Cz.| +00000030 1a 47 ce 93 d9 07 7e 61 43 92 3a 95 a9 54 7b c6 |.G....~aC.:..T{.| +00000040 17 03 03 00 17 e6 e4 52 74 15 e2 18 d7 14 59 29 |.......Rt.....Y)| +00000050 59 d8 ce 49 68 3a a8 79 9e f4 ee f1 |Y..Ih:.y....| >>> Flow 4 (server to client) -00000000 17 03 03 00 16 ad 61 13 66 9c 3c dd 88 42 2c 6c |......a.f.<..B,l| -00000010 d1 22 5d b4 b2 6f f0 68 4f 37 4d |."]..o.hO7M| +00000000 17 03 03 00 16 e6 cd 54 e3 31 d9 70 20 25 f4 aa |.......T.1.p %..| +00000010 cd 8f fe e7 88 6b 7f 62 9b 7b de |.....k.b.{.| >>> Flow 5 (client to server) -00000000 17 03 03 00 16 e5 b5 ff ad d4 f5 e5 55 04 83 a7 |............U...| -00000010 59 43 9a 3e 68 4d 38 6f b1 1c 30 |YC.>hM8o..0| +00000000 17 03 03 00 16 bd e9 c4 24 4a 6d 76 aa 83 bb da |........$Jmv....| +00000010 ec 9e d0 c9 38 cc 6b 3c 65 38 91 |....8.k>> Flow 6 (server to client) -00000000 17 03 03 00 1a 64 0c f4 8e 8f 2b 04 e2 36 28 77 |.....d....+..6(w| -00000010 a2 28 97 4b 15 ba 1b b2 10 31 b3 4d 87 09 af |.(.K.....1.M...| +00000000 17 03 03 00 1a 6a e1 ea df 76 a9 fc 4a 80 7b 49 |.....j...v..J.{I| +00000010 dd fa c3 87 e9 e7 c9 b3 ab bf bc a7 d8 cd 36 |..............6| >>> Flow 7 (client to server) -00000000 17 03 03 00 1d 90 c9 33 bf fc 76 12 3d af 9e c8 |.......3..v.=...| -00000010 8c ca e7 a1 63 6e 80 91 b1 7f 5e e4 dc c2 e6 1c |....cn....^.....| -00000020 b6 2b 17 03 03 00 13 ef a8 30 91 b1 20 fe 82 79 |.+.......0.. ..y| -00000030 44 31 a8 af 99 bb 5e 97 75 a3 |D1....^.u.| +00000000 17 03 03 00 1d 1e c5 6f 93 48 41 07 03 e5 8c 6f |.......o.HA....o| +00000010 09 8e 2f 8e da cb 6e 01 e9 b8 6e f6 3f 59 5c 17 |../...n...n.?Y\.| +00000020 91 46 17 03 03 00 13 35 33 12 5e d3 97 0a 1c 54 |.F.....53.^....T| +00000030 cf 54 bb 7d 78 2e 57 a2 5b 39 |.T.}x.W.[9| diff --git a/src/crypto/tls/testdata/Client-TLSv13-P256-ECDHE b/src/crypto/tls/testdata/Client-TLSv13-P256-ECDHE index fa8d312384fce6..410f0dd189a8c5 100644 --- a/src/crypto/tls/testdata/Client-TLSv13-P256-ECDHE +++ b/src/crypto/tls/testdata/Client-TLSv13-P256-ECDHE @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 01 19 01 00 01 15 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 33 01 00 01 2f 03 03 00 00 00 00 00 |....3.../.......| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,88 +7,90 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 9a 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| +00000080 01 00 00 b4 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| -000000a0 00 0a 00 04 00 02 00 17 00 0d 00 1a 00 18 08 04 |................| +000000a0 00 0a 00 04 00 02 00 17 00 0d 00 16 00 14 08 04 |................| 000000b0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| -000000c0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| -000000d0 02 03 01 00 33 00 47 00 45 00 17 00 41 04 1e 18 |....3.G.E...A...| -000000e0 37 ef 0d 19 51 88 35 75 71 b5 e5 54 5b 12 2e 8f |7...Q.5uq..T[...| -000000f0 09 67 fd a7 24 20 3e b2 56 1c ce 97 28 5e f8 2b |.g..$ >.V...(^.+| -00000100 2d 4f 9e f1 07 9f 6c 4b 5b 83 56 e2 32 42 e9 58 |-O....lK[.V.2B.X| -00000110 b6 d7 49 a6 b5 68 1a 41 03 56 6b dc 5a 89 |..I..h.A.Vk.Z.| +000000c0 06 03 00 32 00 1a 00 18 08 04 04 03 08 07 08 05 |...2............| +000000d0 08 06 04 01 05 01 06 01 05 03 06 03 02 01 02 03 |................| +000000e0 00 2b 00 09 08 03 04 03 03 03 02 03 01 00 33 00 |.+............3.| +000000f0 47 00 45 00 17 00 41 04 1e 18 37 ef 0d 19 51 88 |G.E...A...7...Q.| +00000100 35 75 71 b5 e5 54 5b 12 2e 8f 09 67 fd a7 24 20 |5uq..T[....g..$ | +00000110 3e b2 56 1c ce 97 28 5e f8 2b 2d 4f 9e f1 07 9f |>.V...(^.+-O....| +00000120 6c 4b 5b 83 56 e2 32 42 e9 58 b6 d7 49 a6 b5 68 |lK[.V.2B.X..I..h| +00000130 1a 41 03 56 6b dc 5a 89 |.A.Vk.Z.| >>> Flow 2 (server to client) -00000000 16 03 03 00 9b 02 00 00 97 03 03 1c 1a ee 30 2f |..............0/| -00000010 02 10 0c 84 ee 6a 2c d1 67 76 00 8a 16 e5 15 c5 |.....j,.gv......| -00000020 4c d7 a9 26 33 43 9a 9a d6 f8 e7 20 00 00 00 00 |L..&3C..... ....| +00000000 16 03 03 00 9b 02 00 00 97 03 03 1f 9f 7f 3f 27 |..............?'| +00000010 c5 5c f2 f1 bb cc a2 89 0d 7f 66 6a ff 85 7e 55 |.\........fj..~U| +00000020 ec 47 a6 5e 9c 09 00 ec 96 e2 a7 20 00 00 00 00 |.G.^....... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000040 00 00 00 00 00 00 00 00 00 00 00 00 13 03 00 00 |................| 00000050 4f 00 2b 00 02 03 04 00 33 00 45 00 17 00 41 04 |O.+.....3.E...A.| -00000060 f0 2e aa 75 ea 32 f4 3c ea b7 61 13 0f 22 8d 39 |...u.2.<..a..".9| -00000070 9e 0a 53 c8 d6 92 ea 5e 87 5b 46 d3 71 2d d1 df |..S....^.[F.q-..| -00000080 fa 1e 9a 6a 0e b2 bc d7 91 6e 26 56 17 24 11 d8 |...j.....n&V.$..| -00000090 0f be c0 1d f5 fd c7 9d 2d 3c a0 0f ee 03 70 1f |........-<....p.| -000000a0 14 03 03 00 01 01 17 03 03 00 17 8e 76 66 0d 44 |............vf.D| -000000b0 aa a2 d9 80 84 7c 6d 42 f0 ee 9f a3 6c b8 83 21 |.....|mB....l..!| -000000c0 5d 7b 17 03 03 02 6d 18 d3 53 cc 09 6a 23 fc c5 |]{....m..S..j#..| -000000d0 a9 2e 73 b4 3d ea 54 56 42 f5 1f 71 3e 8f 8e 7b |..s.=.TVB..q>..{| -000000e0 12 18 d7 d6 ab ed 24 5f 16 c8 18 5e e0 28 84 40 |......$_...^.(.@| -000000f0 89 49 a7 91 57 d6 2b a0 9c ab 5d 85 ac 4f 6b 70 |.I..W.+...]..Okp| -00000100 c3 31 e0 57 87 d7 7c 45 27 34 54 eb 85 02 14 2c |.1.W..|E'4T....,| -00000110 a0 53 4f ec 21 9f 04 91 38 b8 df 2c 5d 0b 79 0a |.SO.!...8..,].y.| -00000120 4f 96 79 15 72 de b5 fb 43 1f d8 71 62 d8 e0 69 |O.y.r...C..qb..i| -00000130 21 13 87 ab e5 b8 e5 86 01 74 25 b7 39 fc 86 e2 |!........t%.9...| -00000140 2f de 0b 1f 8c db b2 98 b3 47 c1 4e dd db 36 d3 |/........G.N..6.| -00000150 28 de cf 4c 4d 54 8e a3 e5 d3 38 19 1f 4b 05 7f |(..LMT....8..K..| -00000160 9a e1 59 4c a3 4e 42 b6 71 0d 9c 96 e4 d8 29 73 |..YL.NB.q.....)s| -00000170 4e e6 f5 87 56 04 99 c2 56 15 6c 8b 04 7c e5 83 |N...V...V.l..|..| -00000180 8b f3 42 e8 97 7a b0 cc 7d 15 b8 84 d3 08 dd c1 |..B..z..}.......| -00000190 89 f5 d4 19 d2 f6 fd 32 b4 37 6e 6a cc 0c 7a 69 |.......2.7nj..zi| -000001a0 09 57 78 36 d4 5b 8c 23 0f e6 39 08 4c 58 3e 3f |.Wx6.[.#..9.LX>?| -000001b0 b1 a0 ae 2e 85 b7 64 45 dd e1 ae c4 91 53 8c 6f |......dE.....S.o| -000001c0 02 70 45 f0 97 7b c8 90 88 0e 95 ed 1e 81 fc 86 |.pE..{..........| -000001d0 27 83 3d 32 43 d6 e4 f3 84 c6 c4 c1 b2 1c 0d 4a |'.=2C..........J| -000001e0 22 78 cd 1c f8 0a aa 19 1a 96 cc 46 6d a0 7f c7 |"x.........Fm...| -000001f0 2f d8 68 de 48 7d ae 42 b3 75 2a d5 ab cf e7 a7 |/.h.H}.B.u*.....| -00000200 a1 e6 66 e0 88 d8 14 25 81 21 4a 6f f7 7c 84 57 |..f....%.!Jo.|.W| -00000210 9c cd 45 8a c0 2d 5f 97 b6 66 96 f1 be d8 02 d2 |..E..-_..f......| -00000220 a4 8e fb 0d c4 cc 02 12 1d b7 76 73 8d 73 8e 71 |..........vs.s.q| -00000230 ec 62 4b ba d1 94 98 e5 de 45 f4 92 22 79 9a 2e |.bK......E.."y..| -00000240 b4 12 a6 f8 ab 8a 52 c3 df f8 77 a9 71 8b ff d6 |......R...w.q...| -00000250 2b bc 47 0b 63 5b 0f cf f8 f1 86 7a 72 91 78 7b |+.G.c[.....zr.x{| -00000260 b8 52 74 96 40 4c 08 f3 ca 46 ef d9 b9 6d 7a dc |.Rt.@L...F...mz.| -00000270 2c da f1 c2 c4 1f 19 38 84 cf 7d b7 5d f2 19 a7 |,......8..}.]...| -00000280 bc 81 70 48 1c 5d da e7 e1 8f 94 33 fa b9 44 97 |..pH.].....3..D.| -00000290 52 7a 22 75 f3 3e 08 f5 d3 66 be 63 8e e6 fc 7b |Rz"u.>...f.c...{| -000002a0 54 6d 14 2e 88 09 c7 a1 a6 b7 8e f3 9f 66 dd a4 |Tm...........f..| -000002b0 ec 81 e9 4b 44 69 29 78 95 dd 44 80 f5 f9 41 23 |...KDi)x..D...A#| -000002c0 97 53 71 63 2d cc d0 47 d6 f7 3f 6b 61 5f 44 33 |.Sqc-..G..?ka_D3| -000002d0 61 d1 35 94 42 7b 88 b9 af 6f e2 c1 71 7b d9 12 |a.5.B{...o..q{..| -000002e0 2c b6 57 fa 42 98 27 6d 8a da f8 dd de 6f b2 ea |,.W.B.'m.....o..| -000002f0 c8 de 5c 09 cd 47 36 10 0f 52 72 39 b2 e5 92 19 |..\..G6..Rr9....| -00000300 81 2a e0 ae aa cc b5 d1 68 98 c4 69 02 fc 86 1a |.*......h..i....| -00000310 79 9c 11 54 fc 43 8a ed 63 50 83 49 ac bf 4c ad |y..T.C..cP.I..L.| -00000320 80 04 98 fd 33 19 76 49 8f fe 5a 0f 22 69 3e eb |....3.vI..Z."i>.| -00000330 d8 eb 51 71 17 03 03 00 99 42 4d 4b db 37 6c c3 |..Qq.....BMK.7l.| -00000340 fb 93 1f a6 7a 86 3c 5e 88 e5 75 d4 a1 73 4a b3 |....z.<^..u..sJ.| -00000350 0b 99 f1 a6 98 5f 6d 0f 00 0e 4f fa a3 5e 7d f0 |....._m...O..^}.| -00000360 41 dd dd 5c 71 3b 10 f9 75 07 d5 6c f2 83 83 5c |A..\q;..u..l...\| -00000370 60 32 d9 d2 b3 13 1b 6c 84 2b 02 58 da b6 18 4e |`2.....l.+.X...N| -00000380 ea 7d b3 46 2b a9 ec e7 65 3a 8e 2f 77 48 92 72 |.}.F+...e:./wH.r| -00000390 6a 82 47 bd 82 2b f7 01 6b c4 22 e2 43 78 78 03 |j.G..+..k.".Cxx.| -000003a0 3c 40 d9 6c d1 b3 5e 21 4d 52 74 e4 07 70 56 21 |<@.l..^!MRt..pV!| -000003b0 05 37 37 82 13 cd e6 ce 3f 93 61 74 b4 9a 42 7d |.77.....?.at..B}| -000003c0 26 fa 15 4f ed b7 63 22 3c 92 f3 72 9c d2 80 17 |&..O..c"<..r....| -000003d0 30 f0 17 03 03 00 35 8c f6 08 33 76 5a a4 7b ef |0.....5...3vZ.{.| -000003e0 0d 91 b6 bf fa 9e 39 19 6f 79 77 4c 4d 74 95 71 |......9.oywLMt.q| -000003f0 00 73 37 17 50 ef 55 fb 71 04 36 87 2e 16 5e 25 |.s7.P.U.q.6...^%| -00000400 9b d4 9c 52 f9 ec c5 26 7e 58 ed 9f |...R...&~X..| +00000060 f0 c5 09 c6 22 3f 23 06 65 c7 e2 1c f8 e7 c2 59 |...."?#.e......Y| +00000070 a9 9f 00 3d 1a 48 fb b5 7f e4 ad 3d 4f 2d 91 d8 |...=.H.....=O-..| +00000080 f4 2f d5 31 5c dc 2b 53 63 2b 25 1b 60 47 31 3a |./.1\.+Sc+%.`G1:| +00000090 30 72 d8 5d a4 60 9d 68 51 48 45 69 32 73 58 a5 |0r.].`.hQHEi2sX.| +000000a0 14 03 03 00 01 01 17 03 03 00 17 e9 38 fa fe 25 |............8..%| +000000b0 89 ba 85 3b 0e 66 1e 3f b9 52 55 20 e0 3c 07 6f |...;.f.?.RU .<.o| +000000c0 2c 2f 17 03 03 02 6d cf 73 3f de 91 5b 86 ff 90 |,/....m.s?..[...| +000000d0 ca a3 c9 ba e2 9e 56 53 fc 72 28 e5 98 1f 08 ca |......VS.r(.....| +000000e0 6d 0e 80 96 d2 45 74 33 3b 45 ee 2b ab 12 94 57 |m....Et3;E.+...W| +000000f0 be 00 7e f6 14 93 64 62 7b 0d 81 0e 79 8b 63 49 |..~...db{...y.cI| +00000100 36 8a 67 5c bb 33 72 fa 73 0d e1 ab 66 58 fa a3 |6.g\.3r.s...fX..| +00000110 dd 66 c2 bb 0e 64 82 39 59 e1 a0 e1 5c 84 b1 b6 |.f...d.9Y...\...| +00000120 0d 60 94 f2 ff 94 39 72 62 45 cd f6 d8 f3 33 b5 |.`....9rbE....3.| +00000130 20 99 29 c8 d5 ba 8c 49 cb 29 54 e9 cf 7e 26 1f | .)....I.)T..~&.| +00000140 25 96 41 93 82 4c 38 15 0d 0c 6d 7f 89 51 7f 43 |%.A..L8...m..Q.C| +00000150 9d 04 3a 1a 58 86 97 4e 0c 29 c0 2e f0 e7 78 42 |..:.X..N.)....xB| +00000160 f0 b4 11 cb 50 ec 45 62 c5 0a 42 11 cc aa c4 d4 |....P.Eb..B.....| +00000170 33 15 ee 43 e1 80 6f 46 16 ea 30 27 09 d1 34 e6 |3..C..oF..0'..4.| +00000180 59 f7 b0 e0 6f ea 51 bd 02 79 17 06 3f 84 75 08 |Y...o.Q..y..?.u.| +00000190 63 91 b3 58 d3 1c 61 47 bf d9 2a 9e 11 75 b2 5e |c..X..aG..*..u.^| +000001a0 7e 09 62 f2 3c ba 63 4b 2b 09 c4 90 b5 78 9a c4 |~.b.<.cK+....x..| +000001b0 be 35 9b 7f e3 11 5a 82 d8 50 b3 67 20 7a 53 ec |.5....Z..P.g zS.| +000001c0 cb f9 1a b3 b3 b6 b6 39 1b f1 fe 7a 28 27 9c ed |.......9...z('..| +000001d0 25 35 f4 f1 21 11 eb 33 61 64 1d 54 eb ef c3 d0 |%5..!..3ad.T....| +000001e0 2c 0d 76 0d 7e 90 3f 8d a4 d0 b5 d9 27 49 c7 fc |,.v.~.?.....'I..| +000001f0 cc 59 8c c5 af 59 fc 09 98 63 9c fb 63 d8 09 6f |.Y...Y...c..c..o| +00000200 55 2d 1f d2 c7 83 5f 3d ae 00 ef ca 27 03 9d b9 |U-...._=....'...| +00000210 00 03 c5 c7 e5 2d 91 47 b0 6c 67 2a 09 3e bc 57 |.....-.G.lg*.>.W| +00000220 a3 02 3f cf b2 88 d8 3d 25 c5 e1 0e 2b a2 b7 5f |..?....=%...+.._| +00000230 57 18 5c eb 8e ae 5f d6 5f 11 1b 5c 48 3b f5 84 |W.\..._._..\H;..| +00000240 b0 6e 29 3e 58 71 36 6e 66 6e e0 d0 98 b0 1b 84 |.n)>Xq6nfn......| +00000250 ca 8c 53 83 86 ca 26 7a 15 6b 6a 7c 61 91 55 45 |..S...&z.kj|a.UE| +00000260 58 cb bd b2 df df 78 22 83 36 8e 4f b6 40 70 ea |X.....x".6.O.@p.| +00000270 d1 72 33 7d 85 00 01 48 f1 75 73 db b3 aa 1d da |.r3}...H.us.....| +00000280 78 c9 e3 37 7c 1c 52 74 62 6c f6 eb af af 0b 2d |x..7|.Rtbl.....-| +00000290 ef d4 85 3d 9f 48 de 5c 4a 6e 24 69 b9 7b 87 e7 |...=.H.\Jn$i.{..| +000002a0 62 3c 26 67 8a 52 b2 73 66 90 56 07 6f ec 14 62 |b<&g.R.sf.V.o..b| +000002b0 88 15 20 e6 27 59 d4 cf 0e f0 59 cd cb 8e bd e5 |.. .'Y....Y.....| +000002c0 ca 10 39 40 54 a5 57 4e 99 70 93 b4 86 60 5f f1 |..9@T.WN.p...`_.| +000002d0 70 f5 d7 21 ef f2 11 b2 da 7a 7f 27 a8 6b fe 7f |p..!.....z.'.k..| +000002e0 e4 eb 35 9c d5 24 ef 70 4b 3c ed bd 9b b9 b9 f3 |..5..$.pK<......| +000002f0 50 a4 61 7f 24 d0 8c 7f 4f 48 09 4f 1e 26 fb 46 |P.a.$...OH.O.&.F| +00000300 ec c2 84 12 a5 6a 5f 19 f0 03 cc 50 cc 07 26 23 |.....j_....P..&#| +00000310 58 6e 68 d8 20 ae eb db 2f be a6 7b a8 ed 86 20 |Xnh. .../..{... | +00000320 61 07 03 47 e5 19 9b a7 0d 27 e0 56 9f 2a d5 19 |a..G.....'.V.*..| +00000330 e7 b3 39 07 17 03 03 00 99 3a 9b b2 61 6a 40 3a |..9......:..aj@:| +00000340 46 a8 0b 50 e3 95 d2 80 5f 5a 08 c3 54 2e 47 ac |F..P...._Z..T.G.| +00000350 f1 83 07 13 21 7f 5e 4e 1b 67 66 cc 38 a7 f3 91 |....!.^N.gf.8...| +00000360 f0 5f ac 9a 39 55 de bf 02 30 2a 87 35 85 f8 1d |._..9U...0*.5...| +00000370 32 3a 8c f8 90 6a 22 21 b2 e8 87 5e 54 6a 45 de |2:...j"!...^TjE.| +00000380 74 ff 6d 37 00 92 07 6c 09 13 c1 5e 68 bc 28 80 |t.m7...l...^h.(.| +00000390 d6 24 e5 32 fc 13 ee 68 68 91 9a e6 15 74 21 7d |.$.2...hh....t!}| +000003a0 2d b2 c8 f5 1d 05 69 5c 80 ee 8b 7c 3e 23 be 35 |-.....i\...|>#.5| +000003b0 d4 54 3c 52 66 2a f1 12 98 5d 3a 3f 44 26 84 d8 |.T>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 00 35 f5 7f 27 09 c2 |..........5..'..| -00000010 54 9d e7 a7 3b 14 b3 f2 a6 6d 27 5f 9b 79 04 17 |T...;....m'_.y..| -00000020 28 5c de 4f 67 cf a5 24 e4 d3 c5 e1 38 fa 7e e8 |(\.Og..$....8.~.| -00000030 97 03 7e 66 3b d0 6b e7 f8 7e 97 e0 db 6a da 79 |..~f;.k..~...j.y| -00000040 17 03 03 00 17 30 1a e2 fe 4a 7a 03 03 82 f6 05 |.....0...Jz.....| -00000050 e8 18 67 1d 14 ab 4f 3c 22 c6 45 f5 17 03 03 00 |..g...O<".E.....| -00000060 13 f8 8f 43 e7 74 3c a0 28 b2 71 5a 85 69 e5 86 |...C.t<.(.qZ.i..| -00000070 06 e3 4c 91 |..L.| +00000000 14 03 03 00 01 01 17 03 03 00 35 70 3d 2c 7e 6a |..........5p=,~j| +00000010 d9 c5 58 7b 1d b5 d8 af 9e ca b8 30 2a 9b c3 c8 |..X{.......0*...| +00000020 fa 1c 76 f0 64 f8 4c 01 24 19 18 b2 2e 70 b9 e1 |..v.d.L.$....p..| +00000030 d3 45 96 e7 08 7e 7b c8 7d 1a c8 b7 87 13 9d 1d |.E...~{.}.......| +00000040 17 03 03 00 17 3f 62 68 a3 27 6c c3 12 33 ed e6 |.....?bh.'l..3..| +00000050 0e 87 e6 c4 3e 63 55 2a 15 11 aa 35 17 03 03 00 |....>cU*...5....| +00000060 13 f8 81 0f 72 73 8b d0 14 af fe f2 9d b7 c7 fd |....rs..........| +00000070 16 17 b9 60 |...`| diff --git a/src/crypto/tls/testdata/Client-TLSv13-X25519-ECDHE b/src/crypto/tls/testdata/Client-TLSv13-X25519-ECDHE index 855f63de210766..9e53626d5dc344 100644 --- a/src/crypto/tls/testdata/Client-TLSv13-X25519-ECDHE +++ b/src/crypto/tls/testdata/Client-TLSv13-X25519-ECDHE @@ -1,5 +1,5 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 f8 01 00 00 f4 03 03 00 00 00 00 00 |................| +00000000 16 03 01 01 12 01 00 01 0e 03 03 00 00 00 00 00 |................| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 |........... ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| @@ -7,84 +7,86 @@ 00000050 cc a8 c0 2b c0 2f c0 2c c0 30 c0 09 c0 13 c0 0a |...+./.,.0......| 00000060 c0 14 00 9c 00 9d 00 2f 00 35 c0 12 00 0a c0 23 |......./.5.....#| 00000070 c0 27 00 3c c0 07 c0 11 00 05 13 03 13 01 13 02 |.'.<............| -00000080 01 00 00 79 00 0b 00 02 01 00 ff 01 00 01 00 00 |...y............| +00000080 01 00 00 93 00 0b 00 02 01 00 ff 01 00 01 00 00 |................| 00000090 17 00 00 00 12 00 00 00 05 00 05 01 00 00 00 00 |................| -000000a0 00 0a 00 04 00 02 00 1d 00 0d 00 1a 00 18 08 04 |................| +000000a0 00 0a 00 04 00 02 00 1d 00 0d 00 16 00 14 08 04 |................| 000000b0 04 03 08 07 08 05 08 06 04 01 05 01 06 01 05 03 |................| -000000c0 06 03 02 01 02 03 00 2b 00 09 08 03 04 03 03 03 |.......+........| -000000d0 02 03 01 00 33 00 26 00 24 00 1d 00 20 2f e5 7d |....3.&.$... /.}| -000000e0 a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 |.G.bC.(.._.).0..| -000000f0 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |........_X.;t| +000000c0 06 03 00 32 00 1a 00 18 08 04 04 03 08 07 08 05 |...2............| +000000d0 08 06 04 01 05 01 06 01 05 03 06 03 02 01 02 03 |................| +000000e0 00 2b 00 09 08 03 04 03 03 03 02 03 01 00 33 00 |.+............3.| +000000f0 26 00 24 00 1d 00 20 2f e5 7d a3 47 cd 62 43 15 |&.$... /.}.G.bC.| +00000100 28 da ac 5f bb 29 07 30 ff f6 84 af c4 cf c2 ed |(.._.).0........| +00000110 90 99 5f 58 cb 3b 74 |.._X.;t| >>> Flow 2 (server to client) -00000000 16 03 03 00 7a 02 00 00 76 03 03 6c f6 24 8a 4a |....z...v..l.$.J| -00000010 82 87 8f c9 9a a3 5a d2 4a 24 d3 52 c8 5d 52 95 |......Z.J$.R.]R.| -00000020 0f 12 e8 8e e6 0c a6 5d 60 de 3c 20 00 00 00 00 |.......]`.< ....| +00000000 16 03 03 00 7a 02 00 00 76 03 03 5a 42 7b 30 69 |....z...v..ZB{0i| +00000010 87 6c 87 6a 9c c4 4f 41 b5 18 58 d5 7a 1d 7d da |.l.j..OA..X.z.}.| +00000020 92 29 4c dd 57 6b 4d 0c f9 7c 74 20 00 00 00 00 |.)L.WkM..|t ....| 00000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000040 00 00 00 00 00 00 00 00 00 00 00 00 13 03 00 00 |................| -00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 1b |..+.....3.$... .| -00000060 e6 5f 3d 83 17 1c a6 90 45 f9 00 d9 96 be 4a 58 |._=.....E.....JX| -00000070 ec 82 73 4d e6 8c 74 e9 80 bc 3a 48 1b 93 71 14 |..sM..t...:H..q.| -00000080 03 03 00 01 01 17 03 03 00 17 71 c1 fc 68 5e 33 |..........q..h^3| -00000090 54 8f 43 df 20 55 ac 3b e8 a4 ee 68 ff 88 7c 71 |T.C. U.;...h..|q| -000000a0 92 17 03 03 02 6d 2a 26 5c 45 13 5f 75 46 56 41 |.....m*&\E._uFVA| -000000b0 25 92 a1 6f cd 7b 01 e2 cb 76 81 4d 33 f0 9d d1 |%..o.{...v.M3...| -000000c0 0d db b1 90 1a a7 d9 a8 4a 79 b1 12 59 62 0d c8 |........Jy..Yb..| -000000d0 cc 92 16 26 27 3e 07 b6 8a 15 77 a8 f6 c3 02 e7 |...&'>....w.....| -000000e0 44 0d b5 d3 11 57 6b 81 15 7d 23 b3 91 82 21 3c |D....Wk..}#...!<| -000000f0 0a 0c ef d5 5b 96 5d 85 aa 3a 1e a6 7b fc 3a eb |....[.]..:..{.:.| -00000100 24 af 13 fa 7f 90 0a 83 b5 0a 0b d0 f7 0c 2d 99 |$.............-.| -00000110 75 9b 84 0c d8 dc 60 a3 62 ed 64 55 02 73 a1 74 |u.....`.b.dU.s.t| -00000120 c1 53 c6 97 a5 23 7c 19 c8 f7 1c 97 e9 e9 f8 ee |.S...#|.........| -00000130 4c 47 7e c4 5e 0f 03 fc 60 9e ba 47 ea a9 9b ef |LG~.^...`..G....| -00000140 9a 04 62 e3 db bf 30 d7 5a 1f 20 c9 4c 37 ff 0e |..b...0.Z. .L7..| -00000150 46 d9 ce 85 c5 47 16 43 9a 89 0b 97 44 7a be 09 |F....G.C....Dz..| -00000160 5d 03 a0 b1 1f a6 ca 78 0d fb f4 1a 6d 3a ec 40 |]......x....m:.@| -00000170 1e 58 fc 67 5e bb 69 a1 ae 1d 6d bc b4 80 b9 71 |.X.g^.i...m....q| -00000180 1a 11 12 35 6a 2a 0c dc b3 b5 4b 0a 06 a7 8b be |...5j*....K.....| -00000190 38 4b 70 32 d4 51 e3 99 5b 60 28 e7 9a 60 90 6b |8Kp2.Q..[`(..`.k| -000001a0 1f 3a 9b 4b 66 fd e6 76 b5 8c 29 c3 36 ae a8 81 |.:.Kf..v..).6...| -000001b0 7e 7c bf e7 46 7d 13 27 0d 38 75 f0 15 e1 64 93 |~|..F}.'.8u...d.| -000001c0 1e 26 bc a8 5d 95 60 a2 e5 42 7f 2f 08 a1 e8 d3 |.&..].`..B./....| -000001d0 79 e8 5e 1f 02 45 cc 05 ca 63 7b f2 d9 ad af ab |y.^..E...c{.....| -000001e0 5a 94 f0 16 e7 60 14 c2 3d 05 3f 8d bb 33 c1 ca |Z....`..=.?..3..| -000001f0 79 28 04 c7 20 07 c8 13 89 7a 11 a2 54 6d b8 d5 |y(.. ....z..Tm..| -00000200 e1 b7 b1 43 48 9b a9 b9 e8 e0 40 7e 5a 36 24 70 |...CH.....@~Z6$p| -00000210 42 9a 51 70 6e 22 8a 2b 4b 9d b0 58 ff ae a9 45 |B.Qpn".+K..X...E| -00000220 34 b2 8b c1 35 d8 65 a5 7e 1f 32 fd 51 21 79 00 |4...5.e.~.2.Q!y.| -00000230 79 20 f7 5f 8a e2 e0 b3 4a 78 3b 03 a8 03 b6 ef |y ._....Jx;.....| -00000240 b2 25 37 e9 f3 ce 22 f0 2b fc f8 dd be 50 10 22 |.%7...".+....P."| -00000250 0b fa fb 04 ab a7 33 07 48 2f ca ec 0d fe 19 75 |......3.H/.....u| -00000260 cc c5 3d 17 36 06 5c 07 08 15 df 36 6c 4f f4 73 |..=.6.\....6lO.s| -00000270 50 49 ff 47 ba 47 6d e1 bb 2d 8f 77 d5 44 a9 87 |PI.G.Gm..-.w.D..| -00000280 2a 05 12 52 bf 2b e2 4b 64 94 9c 89 bb 2c 65 cb |*..R.+.Kd....,e.| -00000290 59 2a f0 1a 15 b2 e3 6e 5a cc 48 b4 44 6c 44 07 |Y*.....nZ.H.DlD.| -000002a0 80 01 93 25 86 83 f2 8f 01 e6 ef 5e 9a 36 4e 7f |...%.......^.6N.| -000002b0 bc 27 0e 4d f0 67 3a de 29 b5 e9 6a 7f 4b b4 77 |.'.M.g:.)..j.K.w| -000002c0 9b e2 3b 73 c9 51 e5 a6 df 97 a7 02 fa f2 f7 db |..;s.Q..........| -000002d0 df 71 fb d1 ad 64 37 46 3e de 57 de 1d fc 8f 6e |.q...d7F>.W....n| -000002e0 bb 6f 58 3f 87 00 d1 a3 52 a5 35 12 17 83 19 a4 |.oX?....R.5.....| -000002f0 2b be 31 bd dc a6 62 ca c3 09 39 e1 cd 0b 64 44 |+.1...b...9...dD| -00000300 1a ef 65 26 f3 e5 31 fb 61 56 df d3 11 d9 cc 65 |..e&..1.aV.....e| -00000310 7f 8f ab 17 03 03 00 99 94 ce 37 3c e1 7f b1 f5 |..........7<....| -00000320 7e 2b 04 17 45 3d 38 40 1b 82 0e f7 22 ef 28 ce |~+..E=8@....".(.| -00000330 1a 69 33 a6 ad c6 1f ab 08 12 31 b4 c7 41 ac c5 |.i3.......1..A..| -00000340 b3 e5 4b 84 56 d9 0c 53 58 4c 8f 40 3a 34 66 d0 |..K.V..SXL.@:4f.| -00000350 c7 8f 9b b3 26 25 1f 68 fc 97 ba a7 de ac cf c5 |....&%.h........| -00000360 8b b3 26 99 68 6e ca e9 65 47 07 bd 38 15 f3 0c |..&.hn..eG..8...| -00000370 d0 14 8e 6d 89 a2 ad cd ad ef 00 54 07 5a 4b ef |...m.......T.ZK.| -00000380 02 ce b9 a1 b0 d0 d6 f7 bb e0 91 51 72 87 99 50 |...........Qr..P| -00000390 f4 68 4c 0e d8 fa 00 d4 dd bd 1d 2e 7b f6 e2 61 |.hL.........{..a| -000003a0 d3 a8 46 9c f7 dc 2a 04 be 3c 42 85 4f ad 59 20 |..F...*..'.........| +000000f0 14 3c a1 5b 2e 60 55 34 73 f2 1d c2 24 df 09 55 |.<.[.`U4s...$..U| +00000100 e5 5b b6 43 de d7 48 6b cd 15 da 06 f1 b6 56 0b |.[.C..Hk......V.| +00000110 fb 7e 97 f4 40 2f 6b c9 5b 4d 0c 70 8d d5 1c 21 |.~..@/k.[M.p...!| +00000120 d6 a2 4f ab e9 c7 cb 69 0e d0 86 d0 b8 1e c5 6b |..O....i.......k| +00000130 3d 2f 78 f7 ab fd db 06 5c 59 69 fe 74 69 2f c9 |=/x.....\Yi.ti/.| +00000140 a0 36 e2 08 2e 42 23 05 da 89 33 e7 b5 c3 b2 5a |.6...B#...3....Z| +00000150 d4 7e 4c 99 1c 24 a8 62 9b 92 7d bb c0 e1 4a 90 |.~L..$.b..}...J.| +00000160 72 83 4d b2 74 13 45 3f ef 91 7c b0 c4 6b 4e f5 |r.M.t.E?..|..kN.| +00000170 5c 3d 56 6f fe 96 63 c2 fe 2c 88 23 ab 92 08 bf |\=Vo..c..,.#....| +00000180 c5 08 25 81 a5 49 a6 5c 95 da 09 b2 c9 4b e0 f2 |..%..I.\.....K..| +00000190 59 4e 5a 67 6b 71 c7 ba 69 ff a8 f5 6d 21 52 68 |YNZgkq..i...m!Rh| +000001a0 05 d6 61 0e d6 67 d9 64 3d 26 a3 65 3c 55 11 42 |..a..g.d=&.e.}..E0L-rC.| +00000260 4f a3 08 d2 7f 70 e9 31 7b c8 cd 77 fc 3e 82 70 |O....p.1{..w.>.p| +00000270 48 ed 83 55 79 16 c1 8c 69 50 60 53 48 08 5b 66 |H..Uy...iP`SH.[f| +00000280 fe 78 02 06 16 ba 02 90 85 11 7b 49 36 71 58 35 |.x........{I6qX5| +00000290 46 7f fd 0f de dc 11 ae 3a bd df 79 93 f9 e3 8d |F.......:..y....| +000002a0 4c d8 c5 55 74 34 7f 01 dd 4d d5 ef ef 4e 3d d3 |L..Ut4...M...N=.| +000002b0 31 94 3f 1c 9b 6c 44 d0 3b 73 d1 5f b4 b6 48 5d |1.?..lD.;s._..H]| +000002c0 16 ba 17 0b d0 f5 b4 16 18 1f 85 be dc f1 75 71 |..............uq| +000002d0 17 f8 b0 8a f3 d2 80 24 d5 3b cd 58 f4 9b 03 e5 |.......$.;.X....| +000002e0 64 c5 ae 5e 9a 25 d2 bb a2 28 91 4b 2e 81 4d e4 |d..^.%...(.K..M.| +000002f0 4e 25 57 12 fc 7a 87 36 66 0b 76 21 66 17 a0 e5 |N%W..z.6f.v!f...| +00000300 0a f7 00 31 48 62 a9 70 f1 75 1b e0 78 c9 fa de |...1Hb.p.u..x...| +00000310 11 6f 3d 17 03 03 00 99 72 f9 ad 67 80 2a 9f 04 |.o=.....r..g.*..| +00000320 a9 0a fb 6f 03 90 88 35 7f cb 97 5f 4e a2 1e a8 |...o...5..._N...| +00000330 34 80 96 d1 60 b4 b4 12 ec c3 2b 15 c9 a0 f9 28 |4...`.....+....(| +00000340 50 1c 53 1d fc 32 a2 c7 c0 03 58 1b 83 a6 ee 94 |P.S..2....X.....| +00000350 2d 98 99 83 62 6a 52 a0 c5 42 94 4f 3a 26 2e 44 |-...bjR..B.O:&.D| +00000360 7b 03 f5 2a 0e 9e 48 e4 2a 18 2e 9b 42 7f 34 d2 |{..*..H.*...B.4.| +00000370 3c b2 92 52 cc 51 0e f0 fd c7 4d e6 d7 1b b2 44 |<..R.Q....M....D| +00000380 99 48 e7 da 1e 43 37 d1 ac 80 c5 34 21 1b d8 0c |.H...C7....4!...| +00000390 9c 92 0d b4 27 b0 78 43 06 a2 90 42 c4 99 5e 5f |....'.xC...B..^_| +000003a0 a4 36 69 a5 a1 92 12 5e 51 47 7f ff 50 07 ac 8a |.6i....^QG..P...| +000003b0 35 17 03 03 00 35 86 65 34 2f d0 12 2a 0b 7c 50 |5....5.e4/..*.|P| +000003c0 b4 c5 67 30 cc 4d cf 4f f7 be a5 0d 2c b2 96 ae |..g0.M.O....,...| +000003d0 da 70 a3 74 c2 e7 9a 18 61 17 95 da 00 ac cc 89 |.p.t....a.......| +000003e0 66 a1 e8 aa ca 8f 02 3c b5 16 3d |f......<..=| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 00 35 19 50 85 5d 08 |..........5.P.].| -00000010 c9 5e 25 05 37 ed cc 54 fe ba 86 3e 8a d4 e9 fd |.^%.7..T...>....| -00000020 66 54 6e a3 1f 8b 4f 95 cf 04 51 db 60 96 ae 78 |fTn...O...Q.`..x| -00000030 6b 7d 19 63 03 2a b6 e5 97 37 26 79 98 3f 85 52 |k}.c.*...7&y.?.R| -00000040 17 03 03 00 17 c9 80 91 3e 92 1e a7 9b 00 a3 01 |........>.......| -00000050 19 31 d0 6f 2e 69 0c 15 03 73 20 36 17 03 03 00 |.1.o.i...s 6....| -00000060 13 97 85 32 6f a2 9c 72 be ca 43 db 18 a7 98 b7 |...2o..r..C.....| -00000070 67 44 71 ce |gDq.| +00000000 14 03 03 00 01 01 17 03 03 00 35 a0 a0 1c ae a9 |..........5.....| +00000010 1a 1f 74 01 92 fe bf 91 c4 aa 8b f2 84 7e 39 c1 |..t..........~9.| +00000020 cb f4 ae a8 76 0c de 9e 54 7f ca 4a a2 d3 07 ef |....v...T..J....| +00000030 8e ed 8a be 54 ad 76 fe b6 62 24 4d a7 88 d5 7a |....T.v..b$M...z| +00000040 17 03 03 00 17 a2 d4 e7 78 ef 4a 2e 66 52 6f 2a |........x.J.fRo*| +00000050 e3 07 e0 9c fc 19 f6 f8 96 88 bb be 17 03 03 00 |................| +00000060 13 43 01 14 16 42 27 8a b9 a9 98 5f c7 06 4d 20 |.C...B'...._..M | +00000070 8b c3 cb 33 |...3| diff --git a/src/crypto/tls/testdata/Server-TLSv10-ECDHE-ECDSA-AES b/src/crypto/tls/testdata/Server-TLSv10-ECDHE-ECDSA-AES index c5d947c2ce8c46..3bc3a75b366e7a 100644 --- a/src/crypto/tls/testdata/Server-TLSv10-ECDHE-ECDSA-AES +++ b/src/crypto/tls/testdata/Server-TLSv10-ECDHE-ECDSA-AES @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 51 01 00 00 4d 03 01 5e bf ff e7 c2 |....Q...M..^....| -00000010 c1 98 4a a3 cf 5a e8 8d 8f 19 9e 85 48 5b 92 cc |..J..Z......H[..| -00000020 7d 0c 14 1e 2e 50 5b d7 dd fe ef 00 00 04 c0 0a |}....P[.........| +00000000 16 03 01 00 51 01 00 00 4d 03 01 76 4b 58 ef 57 |....Q...M..vKX.W| +00000010 d5 8d ba b7 0b fe d4 b3 2b a7 76 7c f1 72 59 39 |........+.v|.rY9| +00000020 fa 02 66 88 4a 55 72 15 9e 42 8c 00 00 04 c0 0a |..f.JUr..B......| 00000030 00 ff 01 00 00 20 00 0b 00 04 03 00 01 02 00 0a |..... ..........| 00000040 00 0c 00 0a 00 1d 00 17 00 1e 00 19 00 18 00 16 |................| 00000050 00 00 00 17 00 00 |......| @@ -43,37 +43,37 @@ 00000220 c1 33 13 83 0d 94 06 bb d4 37 7a f6 ec 7a c9 86 |.3.......7z..z..| 00000230 2e dd d7 11 69 7f 85 7c 56 de fb 31 78 2b e4 c7 |....i..|V..1x+..| 00000240 78 0d ae cb be 9e 4e 36 24 31 7b 6a 0f 39 95 12 |x.....N6$1{j.9..| -00000250 07 8f 2a 16 03 01 00 b5 0c 00 00 b1 03 00 1d 20 |..*............ | +00000250 07 8f 2a 16 03 01 00 b3 0c 00 00 af 03 00 1d 20 |..*............ | 00000260 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 |/.}.G.bC.(.._.).| 00000270 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |0.........._X.;t| -00000280 00 8b 30 81 88 02 42 00 c3 eb 60 b8 d3 af cb 2d |..0...B...`....-| -00000290 4f ca 46 6d e4 fe 47 41 82 1e d4 14 0f 08 ab b6 |O.Fm..GA........| -000002a0 b8 41 8b 46 f5 28 bb 87 28 73 a0 5c e9 ce f5 56 |.A.F.(..(s.\...V| -000002b0 11 02 17 2c 39 b6 28 6c ec de 12 bf 22 91 3d 06 |...,9.(l....".=.| -000002c0 ac 8e 0a 92 b1 46 69 86 44 02 42 01 fd 70 6e 63 |.....Fi.D.B..pnc| -000002d0 1b 2a 21 47 9b 42 9c d4 4a 38 20 dd 94 05 c4 0f |.*!G.B..J8 .....| -000002e0 5d b2 48 c8 17 90 01 4d 4f 7e 7a ef bb b2 5b 26 |].H....MO~z...[&| -000002f0 7e e1 24 f5 80 93 69 72 3f cf bb 5d 52 ee ec b4 |~.$...ir?..]R...| -00000300 cc 0c 96 1f 93 4c d6 a8 c7 b2 91 f5 6d 16 03 01 |.....L......m...| -00000310 00 04 0e 00 00 00 |......| +00000280 00 89 30 81 86 02 41 00 b5 7c a4 63 77 fa 75 cd |..0...A..|.cw.u.| +00000290 82 a5 75 15 08 09 e8 6d e9 ba 07 1f f9 9c 24 a5 |..u....m......$.| +000002a0 30 08 d0 51 3b d1 82 14 14 dd 5a 5d c9 2d 91 6f |0..Q;.....Z].-.o| +000002b0 b3 92 30 f1 38 36 e8 34 9e 99 50 a0 c4 29 04 ef |..0.86.4..P..)..| +000002c0 97 f3 cd dc be 22 86 b9 02 41 6a dd 3a 57 5b 61 |....."...Aj.:W[a| +000002d0 ff 68 7d 1e 5e bb 67 5f 76 44 7c f2 f2 03 95 f2 |.h}.^.g_vD|.....| +000002e0 e0 1a 53 70 ce b0 fa cc 7a f3 9a e3 2f 37 a3 cf |..Sp....z.../7..| +000002f0 b5 ca 1d fb fe a3 0d e2 d6 c1 d2 7d 48 80 5b 82 |...........}H.[.| +00000300 56 29 1b b7 43 2e b3 38 19 39 49 16 03 01 00 04 |V)..C..8.9I.....| +00000310 0e 00 00 00 |....| >>> Flow 3 (client to server) -00000000 16 03 01 00 25 10 00 00 21 20 ec f2 2d ca 02 ce |....%...! ..-...| -00000010 11 2d eb 26 d7 d9 fc b2 a7 2d 34 5b a9 3a 0b 2f |.-.&.....-4[.:./| -00000020 5c 49 a9 69 1a 3a 83 90 ec 5f 14 03 01 00 01 01 |\I.i.:..._......| -00000030 16 03 01 00 30 9f 06 c7 a7 a0 c3 a5 3d 60 6e fb |....0.......=`n.| -00000040 c6 18 a4 d2 80 2e ad 8f cf 92 84 94 36 f8 81 28 |............6..(| -00000050 c5 3f 37 e8 d6 e7 6d a3 f5 32 63 a0 ab 7a db 12 |.?7...m..2c..z..| -00000060 17 e1 e4 33 d6 |...3.| +00000000 16 03 01 00 25 10 00 00 21 20 82 03 ad 10 45 0a |....%...! ....E.| +00000010 b9 a4 85 0d 88 bb 9e 16 f1 6c 6a 17 c0 11 09 48 |.........lj....H| +00000020 b4 8b 27 4e 3a 45 a1 b7 a2 03 14 03 01 00 01 01 |..'N:E..........| +00000030 16 03 01 00 30 33 5d a1 85 df 96 6d cf a1 b3 c4 |....03]....m....| +00000040 3f 3c 40 aa 05 25 af 62 ee e9 ce 48 ba e8 08 88 |?<@..%.b...H....| +00000050 95 77 c7 f1 87 c6 ce 46 a2 50 2f 41 3c 8f bf 1a |.w.....F.P/A<...| +00000060 1e 1e 1b 39 9c |...9.| >>> Flow 4 (server to client) -00000000 14 03 01 00 01 01 16 03 01 00 30 18 29 35 d7 c5 |..........0.)5..| -00000010 a2 31 3b 26 85 de 50 26 39 4d 16 22 58 a2 17 bd |.1;&..P&9M."X...| -00000020 4b 73 33 8d dc 3f 92 20 f2 ca 22 00 f5 31 db a7 |Ks3..?. .."..1..| -00000030 18 79 fc 71 87 68 a5 1d a6 db 33 17 03 01 00 20 |.y.q.h....3.... | -00000040 0d be 57 e4 12 6d 2d 3a 33 24 a0 0c c4 9b 27 09 |..W..m-:3$....'.| -00000050 85 e0 0e 42 04 79 21 9a bf 47 fa 0b 38 1a ce 8f |...B.y!..G..8...| -00000060 17 03 01 00 30 6d 27 f1 9b cf 55 4d 65 48 38 1b |....0m'...UMeH8.| -00000070 d9 dd 1d 5b 81 2f 10 a5 65 28 83 93 b3 b1 3a 72 |...[./..e(....:r| -00000080 f0 15 9a e5 9f 21 80 f1 59 a5 0e f1 0c 2b d1 0c |.....!..Y....+..| -00000090 d4 27 73 f3 7e 15 03 01 00 20 6f 08 27 3a d2 60 |.'s.~.... o.':.`| -000000a0 c3 27 bc 73 55 bb 43 53 e2 e0 87 16 ca 8f 49 f0 |.'.sU.CS......I.| -000000b0 88 a8 20 30 9d 42 86 d9 c3 36 |.. 0.B...6| +00000000 14 03 01 00 01 01 16 03 01 00 30 7b 30 af df 92 |..........0{0...| +00000010 2b ee 4d 02 e3 6c 6f 8b 72 32 16 0e 4d ba 71 0d |+.M..lo.r2..M.q.| +00000020 86 0d f5 7d fe dd 07 05 3d fe 70 9b 7f d9 2b c6 |...}....=.p...+.| +00000030 7e 04 82 5f ef 0c 0b c1 e7 a5 18 17 03 01 00 20 |~.._........... | +00000040 ad bd 43 a6 10 e8 e2 41 39 35 69 af a0 00 5f 81 |..C....A95i..._.| +00000050 1e 0a 5e 78 45 2f 66 27 cb 4f db 06 22 c0 ea 0f |..^xE/f'.O.."...| +00000060 17 03 01 00 30 16 69 7f fa 1c 89 36 9f 99 c6 49 |....0.i....6...I| +00000070 e6 0d 9a b7 81 00 75 f5 2d cc 89 e8 be 47 64 76 |......u.-....Gdv| +00000080 ef 34 27 a2 46 bd 8c 14 5b 15 67 ab f1 d9 30 c3 |.4'.F...[.g...0.| +00000090 df 96 a1 59 17 15 03 01 00 20 77 ea 6f e2 ae 81 |...Y..... w.o...| +000000a0 80 4f 37 7b ee 37 3f 40 df a3 e4 be 80 1e 3e 83 |.O7{.7?@......>.| +000000b0 9b 42 f7 95 50 af ab a5 60 54 |.B..P...`T| diff --git a/src/crypto/tls/testdata/Server-TLSv10-ExportKeyingMaterial b/src/crypto/tls/testdata/Server-TLSv10-ExportKeyingMaterial index b3d0f7d05b82fa..114ec3708ee8ee 100644 --- a/src/crypto/tls/testdata/Server-TLSv10-ExportKeyingMaterial +++ b/src/crypto/tls/testdata/Server-TLSv10-ExportKeyingMaterial @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 55 01 00 00 51 03 01 e0 8d 7b f2 8d |....U...Q....{..| -00000010 45 9f c5 40 1b be 81 05 a1 83 82 c1 54 4a c7 1c |E..@........TJ..| -00000020 f1 f8 d5 6c 7a ff 93 81 e2 a2 ba 00 00 04 c0 14 |...lz...........| +00000000 16 03 01 00 55 01 00 00 51 03 01 fb 2f 2f 8e 61 |....U...Q...//.a| +00000010 23 39 d1 13 76 62 4e f8 d5 40 82 a3 89 78 bf fe |#9..vbN..@...x..| +00000020 31 e9 60 d5 e1 e2 1c 54 7a bc 0b 00 00 04 c0 14 |1.`....Tz.......| 00000030 00 ff 01 00 00 24 00 0b 00 04 03 00 01 02 00 0a |.....$..........| 00000040 00 0c 00 0a 00 1d 00 17 00 1e 00 19 00 18 00 23 |...............#| 00000050 00 00 00 16 00 00 00 17 00 00 |..........| @@ -51,42 +51,42 @@ 000002a0 fa e7 16 03 01 00 aa 0c 00 00 a6 03 00 1d 20 2f |.............. /| 000002b0 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| 000002c0 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 00 |.........._X.;t.| -000002d0 80 3d ff b7 ad d5 15 4f 10 6e 8a d2 ad 8a 6b 1b |.=.....O.n....k.| -000002e0 9d 6c f2 92 99 c7 d7 d8 07 d5 c7 77 09 22 41 4f |.l.........w."AO| -000002f0 7f ca 3e 8c 22 ba 2b f2 75 5f 47 c9 7e 0c 03 5d |..>.".+.u_G.~..]| -00000300 1a 66 c3 c8 f3 76 f0 f6 fa 03 40 3a 9b e7 2b 35 |.f...v....@:..+5| -00000310 bc c7 5e 62 a6 97 8a 1a 17 e3 13 4c 1f 88 39 2a |..^b.......L..9*| -00000320 5b cc 9c 65 df 27 1e b3 26 d7 46 3e 76 a9 ae 71 |[..e.'..&.F>v..q| -00000330 11 4d d6 10 b4 2e 30 37 a1 b4 ff 46 91 77 c7 4c |.M....07...F.w.L| -00000340 f9 8e e3 96 88 d2 1e c5 9d fb a1 be c6 ef 5d f0 |..............].| -00000350 52 16 03 01 00 04 0e 00 00 00 |R.........| +000002d0 80 ca 44 f4 0f ef d9 cb a9 88 61 a3 b4 f2 1b 9c |..D.......a.....| +000002e0 e9 a1 c2 c7 84 58 0e 3e ee 95 21 52 61 be 80 64 |.....X.>..!Ra..d| +000002f0 46 17 d5 c7 71 7c 43 41 70 2d 84 9a 49 1c bf 34 |F...q|CAp-..I..4| +00000300 f4 05 1a 0f 9c 00 c5 2d 64 37 84 34 5e d7 5c 06 |.......-d7.4^.\.| +00000310 50 99 f9 d5 a0 19 4b 2d aa 67 e4 17 c7 b4 23 26 |P.....K-.g....#&| +00000320 94 a1 cd e0 cb b1 33 9b e6 c6 a3 a7 25 93 87 7e |......3.....%..~| +00000330 37 ee 9c a0 42 b6 fd 60 59 02 4b 17 4a 4d f3 f2 |7...B..`Y.K.JM..| +00000340 2d 2a e7 8d 96 41 86 43 0a 7b 4e fc c0 7d 38 f6 |-*...A.C.{N..}8.| +00000350 f6 16 03 01 00 04 0e 00 00 00 |..........| >>> Flow 3 (client to server) -00000000 16 03 01 00 25 10 00 00 21 20 01 39 8b 2b 21 99 |....%...! .9.+!.| -00000010 fd fc b8 20 f1 51 97 c7 85 13 05 64 55 41 6b c4 |... .Q.....dUAk.| -00000020 1a 5e d5 b2 7c 8b 31 08 0f 78 14 03 01 00 01 01 |.^..|.1..x......| -00000030 16 03 01 00 30 d8 3b e6 9f f8 a8 b2 6b 8b fb 89 |....0.;.....k...| -00000040 71 3b 55 cd c3 c9 78 3c 45 1b 8d 5f 70 4f bd 64 |q;U...x>> Flow 4 (server to client) -00000000 16 03 01 00 83 04 00 00 7f 00 00 00 00 00 79 00 |..............y.| +00000000 16 03 01 00 85 04 00 00 81 00 00 00 00 00 7b 00 |..............{.| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 94 |................| -00000020 6d 2d 70 97 51 ed 14 ef 68 ca 42 c5 4c 71 8e 74 |m-p.Q...h.B.Lq.t| -00000030 d4 83 d6 4a 5b 69 f8 af 61 3a 98 83 19 d5 7c 60 |...J[i..a:....|`| -00000040 4a 1e f4 b7 26 b8 99 b5 45 6f a3 8d 97 63 5f 1b |J...&...Eo...c_.| -00000050 ab f4 84 59 db ce 99 ce b8 6a 23 d5 15 49 38 16 |...Y.....j#..I8.| -00000060 7e 51 5c e5 15 c0 58 7d c0 ee 59 1b e4 6e 1f c8 |~Q\...X}..Y..n..| -00000070 fc d4 2c 33 ed 0a 2b e0 78 04 64 4b 56 e4 af 61 |..,3..+.x.dKV..a| -00000080 c6 b5 7d f5 a0 86 9f e3 14 03 01 00 01 01 16 03 |..}.............| -00000090 01 00 30 73 2b f0 16 d3 a8 02 b3 73 98 5e 4e a0 |..0s+......s.^N.| -000000a0 ca 5b c4 50 fb 5a 92 11 43 97 e9 e3 16 9f 08 0a |.[.P.Z..C.......| -000000b0 56 73 e6 44 67 70 aa 3d bb c1 36 c8 63 1c 2b 51 |Vs.Dgp.=..6.c.+Q| -000000c0 1f 3b 81 17 03 01 00 20 4c 93 10 5c 01 e2 63 12 |.;..... L..\..c.| -000000d0 97 6b e1 89 fb e7 14 cf ec 70 d1 fe 6f ea 8b 09 |.k.......p..o...| -000000e0 63 5f 8c 8a 9e b5 ac b8 17 03 01 00 30 a1 ad dd |c_..........0...| -000000f0 92 ac a8 6e 77 ed c2 ed 59 b6 a8 41 ad 45 59 8c |...nw...Y..A.EY.| -00000100 4e 1d 16 36 57 e6 2f 47 3d 10 0f 36 04 00 b0 c1 |N..6W./G=..6....| -00000110 a7 94 25 8e 77 1e 69 20 41 6c c0 9d 26 15 03 01 |..%.w.i Al..&...| -00000120 00 20 c5 83 26 5d 20 cb 16 7e 27 63 d7 96 aa 96 |. ..&] ..~'c....| -00000130 37 19 2a 7a 18 d4 85 08 25 32 85 d5 b5 e3 4e 9b |7.*z....%2....N.| -00000140 98 f5 |..| +00000020 6d 2d 70 97 51 ed 14 ef 68 ca 42 c5 4c ed db 91 |m-p.Q...h.B.L...| +00000030 26 40 46 a4 da 9a 13 33 d7 75 7c e0 2f 98 9f 5a |&@F....3.u|./..Z| +00000040 9c a9 12 db 59 ba 75 b2 a1 cb cf f9 75 05 c3 55 |....Y.u.....u..U| +00000050 04 ee 2a 61 94 99 df 73 b3 0b 81 68 f3 49 38 16 |..*a...s...h.I8.| +00000060 7e 51 5c e5 15 c0 58 7d 52 07 a3 db 42 00 b3 4c |~Q\...X}R...B..L| +00000070 77 09 cd 17 5f c1 da 85 f3 09 46 d6 e9 ae 7c e8 |w..._.....F...|.| +00000080 3f 6a 74 38 f9 e7 de 23 0d 90 14 03 01 00 01 01 |?jt8...#........| +00000090 16 03 01 00 30 fa 7e 6e 18 87 06 c8 26 ae a0 34 |....0.~n....&..4| +000000a0 1a 58 05 9e 0c 47 60 93 8c 83 15 98 ad ee de 62 |.X...G`........b| +000000b0 53 6f 1b 44 90 45 d9 22 0b e3 d8 25 32 75 68 ae |So.D.E."...%2uh.| +000000c0 c4 39 b9 05 93 17 03 01 00 20 ac 7a ac 04 59 6a |.9....... .z..Yj| +000000d0 75 a3 26 96 49 c8 f3 ef 39 a6 1f 07 20 d2 e6 bf |u.&.I...9... ...| +000000e0 b8 06 69 55 97 6c c4 68 01 b9 17 03 01 00 30 8b |..iU.l.h......0.| +000000f0 67 6e 9a ea 62 2c dc eb aa 9b 57 e4 5f 82 14 c6 |gn..b,....W._...| +00000100 11 d2 44 e7 5a 9d 13 c0 3e 38 de a7 82 33 44 8e |..D.Z...>8...3D.| +00000110 10 c2 20 c8 6b d2 af 12 b5 44 84 17 a9 2a ec 15 |.. .k....D...*..| +00000120 03 01 00 20 56 39 68 ce 01 c1 52 dd 21 cb 65 a0 |... V9h...R.!.e.| +00000130 5d 28 00 d6 7f f0 c1 38 51 51 98 4f cb 13 5a 41 |](.....8QQ.O..ZA| +00000140 7b 34 be f8 |{4..| diff --git a/src/crypto/tls/testdata/Server-TLSv12-ALPN b/src/crypto/tls/testdata/Server-TLSv12-ALPN index ccd4a086a6ed94..6ee4bc8086d989 100644 --- a/src/crypto/tls/testdata/Server-TLSv12-ALPN +++ b/src/crypto/tls/testdata/Server-TLSv12-ALPN @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 9d 01 00 00 99 03 03 f7 12 13 92 75 |...............u| -00000010 34 ab f3 e8 a2 19 2d 3c 0c 8b 9e c3 e8 22 7e d8 |4.....-<....."~.| -00000020 66 f9 08 88 70 9b cc 37 95 43 a7 00 00 04 cc a8 |f...p..7.C......| +00000000 16 03 01 00 9d 01 00 00 99 03 03 4c 3c dd 9a 33 |...........L<..3| +00000010 a3 3d c3 9d 54 4c a8 e7 d4 2d 20 59 11 bc 48 71 |.=..TL...- Y..Hq| +00000020 bc 5d 6b 24 fd 97 a2 30 4a 2f c8 00 00 04 cc a8 |.]k$...0J/......| 00000030 00 ff 01 00 00 6c 00 0b 00 04 03 00 01 02 00 0a |.....l..........| 00000040 00 0c 00 0a 00 1d 00 17 00 1e 00 19 00 18 00 23 |...............#| 00000050 00 00 00 10 00 10 00 0e 06 70 72 6f 74 6f 32 06 |.........proto2.| @@ -57,35 +57,35 @@ 000002b0 03 03 00 ac 0c 00 00 a8 03 00 1d 20 2f e5 7d a3 |........... /.}.| 000002c0 47 cd 62 43 15 28 da ac 5f bb 29 07 30 ff f6 84 |G.bC.(.._.).0...| 000002d0 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 08 04 00 80 |......._X.;t....| -000002e0 2a 3d 85 27 96 fe 41 e2 5a cc 39 dd 8a 8e 64 73 |*=.'..A.Z.9...ds| -000002f0 ef 98 04 5c ac d2 8f 5e 55 b8 37 da 10 68 33 b8 |...\...^U.7..h3.| -00000300 63 83 e1 c9 9a e6 3a e9 c9 20 cc 57 58 e2 ba bc |c.....:.. .WX...| -00000310 e3 ac ab aa 08 e2 1e 6f 66 90 d7 66 c5 73 60 0d |.......of..f.s`.| -00000320 19 4f eb 99 9d d1 b1 91 36 80 b9 20 aa f5 d9 c8 |.O......6.. ....| -00000330 44 a7 99 c9 a6 4d 2c ff ca 4d 84 f2 a5 bf 02 c5 |D....M,..M......| -00000340 61 77 7e 4a e6 7c dd bf 48 fc a6 53 fb c4 d3 dd |aw~J.|..H..S....| -00000350 e6 20 b9 74 90 82 4a 3a 73 0a 81 74 07 a3 23 fe |. .t..J:s..t..#.| +000002e0 82 9a 38 98 24 59 07 8b a9 7e d3 9f c3 70 8d 87 |..8.$Y...~...p..| +000002f0 2d 1b 53 f3 36 96 4a 07 83 80 1e 62 23 b4 79 c9 |-.S.6.J....b#.y.| +00000300 93 48 0a 54 ad 03 5e 71 c3 69 d9 b7 be 93 c0 e8 |.H.T..^q.i......| +00000310 13 bd 10 67 b1 ea 8f f0 72 ed e1 54 b1 e5 a8 ca |...g....r..T....| +00000320 c7 b2 ac 2e 14 ab 6a 84 2b 97 e6 8f 68 1c e9 83 |......j.+...h...| +00000330 73 70 24 40 99 f7 86 2a c7 08 1f bc bd df a2 24 |sp$@...*.......$| +00000340 75 33 81 29 18 69 d5 5e 93 91 63 62 ee e9 8f b6 |u3.).i.^..cb....| +00000350 fc d1 00 5d b2 b5 cc 5f c9 83 8d fd f8 dd 7a cd |...]..._......z.| 00000360 16 03 03 00 04 0e 00 00 00 |.........| >>> Flow 3 (client to server) -00000000 16 03 03 00 25 10 00 00 21 20 d1 bb f1 17 6c 41 |....%...! ....lA| -00000010 8f 14 84 d2 98 99 30 0c 8a 00 4c 39 37 15 f5 be |......0...L97...| -00000020 81 8d 08 e0 11 c1 f7 65 43 0b 14 03 03 00 01 01 |.......eC.......| -00000030 16 03 03 00 20 ab 15 bb 47 30 42 c9 7d 45 f8 5d |.... ...G0B.}E.]| -00000040 21 79 3b 4d 5e a9 99 f5 7d f3 4e 7e ba b9 9b 30 |!y;M^...}.N~...0| -00000050 b6 14 4d ba f9 |..M..| +00000000 16 03 03 00 25 10 00 00 21 20 61 d8 61 58 30 ba |....%...! a.aX0.| +00000010 44 5b 31 35 e3 4e 92 87 d0 10 0c 83 96 98 b5 73 |D[15.N.........s| +00000020 31 40 7a fd 78 8a cf b4 af 53 14 03 03 00 01 01 |1@z.x....S......| +00000030 16 03 03 00 20 14 80 2b 46 28 9e a8 1b d5 9a bd |.... ..+F(......| +00000040 6c da 22 62 a9 7d d8 c1 e5 d7 63 6f 26 3f ca 1c |l."b.}....co&?..| +00000050 5e 53 8b 3c f3 |^S.<.| >>> Flow 4 (server to client) -00000000 16 03 03 00 83 04 00 00 7f 00 00 00 00 00 79 00 |..............y.| +00000000 16 03 03 00 85 04 00 00 81 00 00 00 00 00 7b 00 |..............{.| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 94 |................| -00000020 6f 2d 7c 2b 51 ed 14 ef 68 ca 42 c5 4c f3 5c b9 |o-|+Q...h.B.L.\.| -00000030 84 7d 30 9e 2f 9d 4d 0e 59 b4 28 fd 17 10 cd 1e |.}0./.M.Y.(.....| -00000040 1c d3 2c 5e d9 dc db 26 d0 b9 00 4b 0a 13 54 90 |..,^...&...K..T.| -00000050 f2 7b 68 75 6b 00 34 66 9e 43 29 06 16 49 38 16 |.{huk.4f.C)..I8.| -00000060 7e 51 5c e5 15 c0 58 7d 52 0b 16 21 d8 2c e8 c8 |~Q\...X}R..!.,..| -00000070 8e 3a f6 aa fa 21 45 4a 17 02 67 7d 93 1c 95 88 |.:...!EJ..g}....| -00000080 36 a5 19 53 74 74 81 e1 14 03 03 00 01 01 16 03 |6..Stt..........| -00000090 03 00 20 3d 66 04 37 0c 40 cc 20 2c 1c 16 ba 05 |.. =f.7.@. ,....| -000000a0 d6 7b 40 04 27 40 6f cc d7 af 68 fb 32 49 6c 4f |.{@.'@o...h.2IlO| -000000b0 f3 01 bf 17 03 03 00 1d 99 10 78 bc fa 7e 8a 86 |..........x..~..| -000000c0 4c b8 e4 7c e2 79 70 eb ad 33 44 e1 ab 7a c9 ae |L..|.yp..3D..z..| -000000d0 47 fe 39 50 d1 15 03 03 00 12 9e 9a be b0 55 c3 |G.9P..........U.| -000000e0 3a 5f 5c e0 4b 8f 4f 81 52 d3 89 09 |:_\.K.O.R...| +00000020 6f 2d 7c 2b 51 ed 14 ef 68 ca 42 c5 4c a3 4e 47 |o-|+Q...h.B.L.NG| +00000030 75 8d 90 24 a8 60 43 a8 3b 00 81 b1 1d 41 ce bf |u..$.`C.;....A..| +00000040 ec 75 e9 32 6b 9b 21 9f 0f 56 27 b2 e5 9e 9a 01 |.u.2k.!..V'.....| +00000050 aa c7 63 81 8b 90 45 fa 64 75 96 e3 c8 49 38 16 |..c...E.du...I8.| +00000060 7e 51 5c e5 15 c0 58 7d 52 07 32 1b 54 23 7f 75 |~Q\...X}R.2.T#.u| +00000070 8a 30 1c 6a 94 57 27 7d 06 25 05 b7 ae ce 5c a4 |.0.j.W'}.%....\.| +00000080 58 21 47 f2 04 bc 3a f1 6d 20 14 03 03 00 01 01 |X!G...:.m ......| +00000090 16 03 03 00 20 94 0c cf 54 2b fb 49 26 19 d4 06 |.... ...T+.I&...| +000000a0 0e b6 71 b5 d9 24 f6 d1 99 36 78 1c 96 b4 12 e0 |..q..$...6x.....| +000000b0 20 5a 2a a7 ad 17 03 03 00 1d 19 36 ef 7d 53 24 | Z*........6.}S$| +000000c0 0c 5e 48 24 c6 ad 91 ca 44 0d 2e fb 10 fd 58 2f |.^H$....D.....X/| +000000d0 86 5c be c6 64 01 c6 15 03 03 00 12 26 94 16 38 |.\..d.......&..8| +000000e0 6c 23 4e 29 03 96 c6 6a a8 af 32 0b 2e 9e |l#N)...j..2...| diff --git a/src/crypto/tls/testdata/Server-TLSv12-ALPN-Fallback b/src/crypto/tls/testdata/Server-TLSv12-ALPN-Fallback index 070201253d2d62..a453fcc5227f92 100644 --- a/src/crypto/tls/testdata/Server-TLSv12-ALPN-Fallback +++ b/src/crypto/tls/testdata/Server-TLSv12-ALPN-Fallback @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 a6 01 00 00 a2 03 03 ea e2 1b 90 0e |................| -00000010 91 d5 9f b2 c6 ee 72 37 19 f5 14 cd ca a9 ca 03 |......r7........| -00000020 98 c4 2e d4 85 05 4a a5 02 e1 4b 00 00 04 cc a8 |......J...K.....| +00000000 16 03 01 00 a6 01 00 00 a2 03 03 74 b9 ce 14 2f |...........t.../| +00000010 85 a9 93 bf 60 4e 5c 5c 6e 47 34 cf b8 27 f8 dc |....`N\\nG4..'..| +00000020 b1 5b a8 eb e0 fa da a0 1b b6 9c 00 00 04 cc a8 |.[..............| 00000030 00 ff 01 00 00 75 00 0b 00 04 03 00 01 02 00 0a |.....u..........| 00000040 00 0c 00 0a 00 1d 00 17 00 1e 00 19 00 18 00 23 |...............#| 00000050 00 00 00 10 00 19 00 17 06 70 72 6f 74 6f 33 08 |.........proto3.| @@ -56,35 +56,35 @@ 000002a0 fa e7 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 2f |.............. /| 000002b0 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| 000002c0 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 08 |.........._X.;t.| -000002d0 04 00 80 85 e9 a6 5c 79 bc db ed 97 fb 30 ca fd |......\y.....0..| -000002e0 32 13 19 3f da 6f fd c1 11 74 fe e9 6f 60 ec 7e |2..?.o...t..o`.~| -000002f0 48 7e 17 33 9b 8d 2a c2 82 e0 18 38 f3 0f 20 27 |H~.3..*....8.. '| -00000300 81 0f c9 47 bf 5f 2b 2f 65 1c 6b e3 b7 72 85 46 |...G._+/e.k..r.F| -00000310 5c 15 dc fd e6 be cf 50 51 62 f5 d9 17 e2 e8 bf |\......PQb......| -00000320 08 7f 37 71 91 88 83 7f e3 90 66 66 c4 d8 60 25 |..7q......ff..`%| -00000330 53 f7 9f 44 20 89 48 ff c2 3b 6d 21 e5 8c dc e5 |S..D .H..;m!....| -00000340 42 ea d8 14 93 96 2f 53 24 66 e7 bb e7 2c 1f 92 |B...../S$f...,..| -00000350 90 80 23 16 03 03 00 04 0e 00 00 00 |..#.........| +000002d0 04 00 80 22 97 55 0d 4c fe 5e 4e 45 b7 9b b2 f8 |...".U.L.^NE....| +000002e0 29 c7 7a 33 b5 e0 06 92 4f b3 6e 67 ad 4e 69 20 |).z3....O.ng.Ni | +000002f0 e5 82 b6 93 84 52 05 fd 99 d1 94 67 e4 7d bc 1d |.....R.....g.}..| +00000300 f7 16 d7 24 95 61 db ed 92 16 11 ee c1 c5 6f 82 |...$.a........o.| +00000310 8e 6b 10 69 31 d2 17 1a 6f 25 a0 d5 4b 7e c9 ba |.k.i1...o%..K~..| +00000320 13 3e c4 94 46 63 e2 6e c6 ca d0 e4 09 5a 2a 39 |.>..Fc.n.....Z*9| +00000330 12 c0 fc 37 14 4e a8 1f 74 4e 44 86 1a 29 d4 a0 |...7.N..tND..)..| +00000340 5f e5 0a 22 6c 09 78 29 be 33 a5 2c d9 b3 5f ec |_.."l.x).3.,.._.| +00000350 f1 5e 87 16 03 03 00 04 0e 00 00 00 |.^..........| >>> Flow 3 (client to server) -00000000 16 03 03 00 25 10 00 00 21 20 e8 d1 46 5e 70 b5 |....%...! ..F^p.| -00000010 34 1c 6f cd be f0 86 24 2a d6 55 ae 97 de 52 0c |4.o....$*.U...R.| -00000020 67 10 a0 02 ed ae f8 47 aa 52 14 03 03 00 01 01 |g......G.R......| -00000030 16 03 03 00 20 52 cf 5d 07 bb bc e8 86 d4 f4 3e |.... R.].......>| -00000040 49 51 a7 1d f5 df 10 c4 5a 77 37 ba 68 3d 4e c5 |IQ......Zw7.h=N.| -00000050 11 ac 67 b7 e2 |..g..| +00000000 16 03 03 00 25 10 00 00 21 20 a4 f2 f3 f8 81 68 |....%...! .....h| +00000010 6f 8d d5 2c 93 fe ee cf f6 28 ae 06 9f 81 fa 0d |o..,.....(......| +00000020 ac 31 2b cf 05 4e cb a0 b3 14 14 03 03 00 01 01 |.1+..N..........| +00000030 16 03 03 00 20 ca 3e f2 cd 68 42 34 26 61 40 29 |.... .>..hB4&a@)| +00000040 dd 71 e1 52 b7 0e ec 1f 77 8e 1d 1b 95 dd 07 4f |.q.R....w......O| +00000050 c4 4d d8 02 83 |.M...| >>> Flow 4 (server to client) -00000000 16 03 03 00 83 04 00 00 7f 00 00 00 00 00 79 00 |..............y.| +00000000 16 03 03 00 85 04 00 00 81 00 00 00 00 00 7b 00 |..............{.| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 94 |................| -00000020 6f 2d 7c 2b 51 ed 14 ef 68 ca 42 c5 4c 26 1d 23 |o-|+Q...h.B.L&.#| -00000030 c4 90 54 85 8b 21 f5 0d e8 48 f2 5f 4e 6b f1 25 |..T..!...H._Nk.%| -00000040 e8 46 8a e5 3c 09 57 df dd 37 a7 57 c5 a5 28 5a |.F..<.W..7.W..(Z| -00000050 21 83 2a 98 4b a5 44 aa 5b cc 30 e9 62 49 38 16 |!.*.K.D.[.0.bI8.| -00000060 7e 51 5c e5 15 c0 58 7d a4 aa c5 93 39 bb e2 b6 |~Q\...X}....9...| -00000070 4f c4 3e 1e 03 dc 46 b1 f3 0d d2 61 6c 1e c5 e1 |O.>...F....al...| -00000080 8f 18 2a 3c 85 83 c4 33 14 03 03 00 01 01 16 03 |..*<...3........| -00000090 03 00 20 63 76 4f b3 77 4d 63 6c eb 73 f3 b2 ec |.. cvO.wMcl.s...| -000000a0 b8 49 3e c5 81 d5 53 0c 96 77 2f 3f 52 d0 e1 5b |.I>...S..w/?R..[| -000000b0 62 fa 0b 17 03 03 00 1d 2f 60 09 31 db e9 c5 23 |b......./`.1...#| -000000c0 98 5c 46 23 a6 58 80 66 7d 50 84 f1 42 b8 65 65 |.\F#.X.f}P..B.ee| -000000d0 77 2d d2 e4 be 15 03 03 00 12 b7 e8 e1 13 04 68 |w-.............h| -000000e0 d5 21 c8 98 db 1b 1c 6e 4f b5 0b 9c |.!.....nO...| +00000020 6f 2d 7c 2b 51 ed 14 ef 68 ca 42 c5 4c 23 bb 4f |o-|+Q...h.B.L#.O| +00000030 57 15 7b bb 5c 23 ff bd b0 3b c9 ce d7 8e b9 d8 |W.{.\#...;......| +00000040 b6 35 dd 0b 5b fd 3f bf c6 c9 74 86 d4 4e 1c 22 |.5..[.?...t..N."| +00000050 fb 4e ea 39 16 d3 9d 08 c8 08 c8 94 c7 49 38 16 |.N.9.........I8.| +00000060 7e 51 5c e5 15 c0 58 7d 52 07 e6 23 d9 47 91 c4 |~Q\...X}R..#.G..| +00000070 55 a3 14 46 22 5a 68 ec 70 f1 cd 8b e0 36 5d 20 |U..F"Zh.p....6] | +00000080 bb 33 6b d2 cc e0 bc 81 f6 ba 14 03 03 00 01 01 |.3k.............| +00000090 16 03 03 00 20 06 6c 46 46 01 02 e9 42 de 4a dc |.... .lFF...B.J.| +000000a0 4b 55 15 6d e4 2c da 02 67 af 08 f1 15 f6 5a 72 |KU.m.,..g.....Zr| +000000b0 0b 70 d3 28 ba 17 03 03 00 1d 5b 4f 16 d3 78 dd |.p.(......[O..x.| +000000c0 fb cb 38 70 cc dc 26 36 99 ad 67 e3 dc 2b c8 62 |..8p..&6..g..+.b| +000000d0 1f a1 ad 3b e2 fd d7 15 03 03 00 12 e4 ca da 87 |...;............| +000000e0 78 97 b2 b3 27 0f 3e 1a 97 8d ab fc 7c b9 |x...'.>.....|.| diff --git a/src/crypto/tls/testdata/Server-TLSv12-ALPN-NotConfigured b/src/crypto/tls/testdata/Server-TLSv12-ALPN-NotConfigured index 79f0748af40083..97d300d76b9283 100644 --- a/src/crypto/tls/testdata/Server-TLSv12-ALPN-NotConfigured +++ b/src/crypto/tls/testdata/Server-TLSv12-ALPN-NotConfigured @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 9d 01 00 00 99 03 03 19 26 ad 3f c0 |............&.?.| -00000010 d6 a0 cc ac 9b 2a 91 d3 1a d5 96 78 5f 7c 3f e0 |.....*.....x_|?.| -00000020 23 08 75 a1 ca cb aa da d7 c8 0b 00 00 04 cc a8 |#.u.............| +00000000 16 03 01 00 9d 01 00 00 99 03 03 23 13 3f 28 85 |...........#.?(.| +00000010 56 f7 0f d1 b4 ee 5f 18 a0 58 8c c5 83 a4 13 24 |V....._..X.....$| +00000020 9b 53 29 6e 28 35 a8 8c 0e 07 2a 00 00 04 cc a8 |.S)n(5....*.....| 00000030 00 ff 01 00 00 6c 00 0b 00 04 03 00 01 02 00 0a |.....l..........| 00000040 00 0c 00 0a 00 1d 00 17 00 1e 00 19 00 18 00 23 |...............#| 00000050 00 00 00 10 00 10 00 0e 06 70 72 6f 74 6f 32 06 |.........proto2.| @@ -56,35 +56,35 @@ 000002a0 fa e7 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 2f |.............. /| 000002b0 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| 000002c0 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 08 |.........._X.;t.| -000002d0 04 00 80 4f 9e 64 41 a6 8a 41 ab 9c c8 09 3e 94 |...O.dA..A....>.| -000002e0 ee d2 9b ad 1b 3e a9 3c 7b 43 96 95 eb 4d b5 04 |.....>.<{C...M..| -000002f0 1a 5f 0c b2 b3 a6 2c a4 e6 78 a8 b8 d5 6c 7f d0 |._....,..x...l..| -00000300 16 e8 56 31 e0 4a 69 d3 6b 27 18 a3 4e f5 d1 6a |..V1.Ji.k'..N..j| -00000310 36 15 b5 fc 4d 15 50 90 a0 30 b9 49 3d ac 8c 84 |6...M.P..0.I=...| -00000320 d2 15 31 70 df e5 a6 97 d0 64 f7 1d 8a a1 87 4d |..1p.....d.....M| -00000330 3c ee da 69 20 e4 31 67 ca f2 c0 09 ee 13 7c 78 |<..i .1g......|x| -00000340 d6 c2 c0 39 e0 b8 00 52 a9 bf d0 99 e0 b0 66 70 |...9...R......fp| -00000350 46 ae 62 16 03 03 00 04 0e 00 00 00 |F.b.........| +000002d0 04 00 80 66 f8 99 59 3a 3a 64 36 75 11 53 eb 34 |...f..Y::d6u.S.4| +000002e0 1d d7 56 1b fb 73 58 63 69 2d 3d b5 d0 05 ce 4d |..V..sXci-=....M| +000002f0 6d 6e 49 46 c2 ad 91 43 de b3 63 12 4b e6 e2 c8 |mnIF...C..c.K...| +00000300 59 09 09 45 f2 b8 1e 95 71 b2 38 60 78 36 c5 46 |Y..E....q.8`x6.F| +00000310 15 85 66 4b 83 e2 6f 07 df 3e 87 60 eb 85 2d 01 |..fK..o..>.`..-.| +00000320 c4 ae 50 b8 0e e5 19 b4 1d a4 90 af 97 b7 87 9e |..P.............| +00000330 cb 3a 13 1f ec 78 6c d9 5d 14 03 b7 4b 8b 8d 92 |.:...xl.]...K...| +00000340 06 cf a3 dc 59 30 36 0f 7f 24 11 ca b8 ff 46 4b |....Y06..$....FK| +00000350 0c 4e c2 16 03 03 00 04 0e 00 00 00 |.N..........| >>> Flow 3 (client to server) -00000000 16 03 03 00 25 10 00 00 21 20 b0 1a 5b c3 55 5f |....%...! ..[.U_| -00000010 0b b8 f3 69 ba 4f 49 93 05 0f b1 f1 d7 6b 6c 0c |...i.OI......kl.| -00000020 98 d0 22 78 0c ad 15 6b 24 5b 14 03 03 00 01 01 |.."x...k$[......| -00000030 16 03 03 00 20 9d aa 3f 17 b3 16 88 d5 44 3d 03 |.... ..?.....D=.| -00000040 3c 3c 8d 92 f1 2f e4 38 cc 42 20 2f ef 6a 29 c6 |<<.../.8.B /.j).| -00000050 5c ca 44 81 f6 |\.D..| +00000000 16 03 03 00 25 10 00 00 21 20 16 8d 5b 4a f7 e1 |....%...! ..[J..| +00000010 c0 dd 5d e2 82 7d 4e c6 ef 66 ef 87 50 85 bf 0d |..]..}N..f..P...| +00000020 b5 3f 03 cc 64 f4 48 93 79 27 14 03 03 00 01 01 |.?..d.H.y'......| +00000030 16 03 03 00 20 2c 69 26 7b 1b 84 2b 1d 33 43 cf |.... ,i&{..+.3C.| +00000040 95 c5 72 d6 7a 88 8f f9 aa 82 72 f4 02 c5 6e aa |..r.z.....r...n.| +00000050 a9 f7 f0 b9 44 |....D| >>> Flow 4 (server to client) -00000000 16 03 03 00 83 04 00 00 7f 00 00 00 00 00 79 00 |..............y.| +00000000 16 03 03 00 85 04 00 00 81 00 00 00 00 00 7b 00 |..............{.| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 94 |................| -00000020 6f 2d 7c 2b 51 ed 14 ef 68 ca 42 c5 4c 9a f3 2b |o-|+Q...h.B.L..+| -00000030 7a 66 76 26 4f 73 12 14 ef a1 f4 8c c2 08 03 42 |zfv&Os.........B| -00000040 4d d5 f9 d7 ab 31 78 51 f3 f4 94 49 5f 9d bf 23 |M....1xQ...I_..#| -00000050 b2 11 7b ac 42 df 71 1a 37 db 64 99 a0 49 38 16 |..{.B.q.7.d..I8.| -00000060 7e 51 5c e5 15 c0 58 7d 2d 89 ac 0d 05 31 27 ae |~Q\...X}-....1'.| -00000070 85 ff 27 56 24 4c 26 b3 bc 6c f6 20 80 dd bd ba |..'V$L&..l. ....| -00000080 a3 34 c2 32 a8 58 1b b9 14 03 03 00 01 01 16 03 |.4.2.X..........| -00000090 03 00 20 74 e1 8a e6 a6 02 0d f7 e1 28 3a f4 c4 |.. t........(:..| -000000a0 a6 8c 32 81 84 85 ec 58 6a 10 8a 6d c4 cc 10 3a |..2....Xj..m...:| -000000b0 32 3e df 17 03 03 00 1d fd a8 94 23 3e 5d 96 b1 |2>.........#>]..| -000000c0 68 a6 24 55 bf 29 08 93 c7 7b 9b 05 fc 0b 97 ff |h.$U.)...{......| -000000d0 7c 93 b0 34 82 15 03 03 00 12 43 9f 44 e4 63 e7 ||..4......C.D.c.| -000000e0 3c 30 a5 da 9f 58 ac 01 e4 e2 a7 30 |<0...X.....0| +00000020 6f 2d 7c 2b 51 ed 14 ef 68 ca 42 c5 4c 2e ba 80 |o-|+Q...h.B.L...| +00000030 92 a2 2f 66 80 b9 56 b8 7b be 8f 7f 3e f1 92 8d |../f..V.{...>...| +00000040 bf a9 6d 23 58 04 c5 70 85 af a7 db e4 0f e5 87 |..m#X..p........| +00000050 6e aa a1 58 2a 7c 3f 9b 15 36 ac a3 00 49 38 16 |n..X*|?..6...I8.| +00000060 7e 51 5c e5 15 c0 58 7d 52 07 ff f3 10 e5 6f 3a |~Q\...X}R.....o:| +00000070 6f e9 dd 79 00 6d 46 a8 9d a3 6c 3b 1b 39 da 98 |o..y.mF...l;.9..| +00000080 5a 36 f1 64 1f a6 4b f8 2b ff 14 03 03 00 01 01 |Z6.d..K.+.......| +00000090 16 03 03 00 20 d1 e3 4a cc 06 f0 a9 b6 f7 66 2d |.... ..J......f-| +000000a0 3d 07 70 e2 93 39 a1 2a c2 72 f3 e7 b3 ca a0 77 |=.p..9.*.r.....w| +000000b0 ff cf 9b 0f 2f 17 03 03 00 1d 4b e5 61 b4 4c 6b |..../.....K.a.Lk| +000000c0 b0 ca 5e e4 96 7b f2 18 9a c2 19 b2 be c2 39 8b |..^..{........9.| +000000d0 ee 88 e7 53 27 3a 8d 15 03 03 00 12 2a 21 52 fc |...S':......*!R.| +000000e0 3b ae 6e fb ac 9c 42 bc 1a 7d 6b 8d 84 5d |;.n...B..}k..]| diff --git a/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndECDSAGiven b/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndECDSAGiven index 41ed6c4e1b0ed9..73d54e6579036f 100644 --- a/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndECDSAGiven +++ b/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndECDSAGiven @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 6d 01 00 00 69 03 03 5e 92 9d 30 27 |....m...i..^..0'| -00000010 23 da fa a0 07 30 03 c8 bd 60 f2 db e9 5e b3 fc |#....0...`...^..| -00000020 65 d3 c5 e1 49 35 63 86 53 ec 87 00 00 04 00 2f |e...I5c.S....../| +00000000 16 03 01 00 6d 01 00 00 69 03 03 5e 1c 8d af f1 |....m...i..^....| +00000010 4a ab 9d e1 c7 9c 85 c3 ab b8 84 84 ee 39 66 0b |J............9f.| +00000020 6d d7 b2 3b 7b bf 4e d1 90 a4 92 00 00 04 00 2f |m..;{.N......../| 00000030 00 ff 01 00 00 3c 00 16 00 00 00 17 00 00 00 0d |.....<..........| 00000040 00 30 00 2e 04 03 05 03 06 03 08 07 08 08 08 09 |.0..............| 00000050 08 0a 08 0b 08 04 08 05 08 06 04 01 05 01 06 01 |................| @@ -49,10 +49,10 @@ 00000260 ee b1 87 82 f1 6c 04 ed 73 bb b3 43 77 8d 0c 1c |.....l..s..Cw...| 00000270 f1 0f a1 d8 40 83 61 c9 4c 72 2b 9d ae db 46 06 |....@.a.Lr+...F.| 00000280 06 4d f4 c1 b3 3e c0 d1 bd 42 d4 db fe 3d 13 60 |.M...>...B...=.`| -00000290 84 5c 21 d3 3b e9 fa e7 16 03 03 00 23 0d 00 00 |.\!.;.......#...| -000002a0 1f 02 01 40 00 18 08 04 04 03 08 07 08 05 08 06 |...@............| -000002b0 04 01 05 01 06 01 05 03 06 03 02 01 02 03 00 00 |................| -000002c0 16 03 03 00 04 0e 00 00 00 |.........| +00000290 84 5c 21 d3 3b e9 fa e7 16 03 03 00 1f 0d 00 00 |.\!.;...........| +000002a0 1b 02 01 40 00 14 08 04 04 03 08 07 08 05 08 06 |...@............| +000002b0 04 01 05 01 06 01 05 03 06 03 00 00 16 03 03 00 |................| +000002c0 04 0e 00 00 00 |.....| >>> Flow 3 (client to server) 00000000 16 03 03 02 0a 0b 00 02 06 00 02 03 00 02 00 30 |...............0| 00000010 82 01 fc 30 82 01 5e 02 09 00 9a 30 84 6c 26 35 |...0..^....0.l&5| @@ -87,40 +87,40 @@ 000001e0 be e8 91 b3 da 1a f5 5d a3 23 f5 26 8b 45 70 8d |.......].#.&.Ep.| 000001f0 65 62 9b 7e 01 99 3d 18 f6 10 9a 38 61 9b 2e 57 |eb.~..=....8a..W| 00000200 e4 fa cc b1 8a ce e2 23 a0 87 f0 e1 67 51 eb 16 |.......#....gQ..| -00000210 03 03 00 86 10 00 00 82 00 80 02 50 e4 cc a3 ad |...........P....| -00000220 fb 33 24 a1 b3 0a 7c 0f 00 e6 1a 06 2b 9f 1e 1f |.3$...|.....+...| -00000230 cc b8 b2 80 90 e7 86 20 32 40 06 ac 1b b0 41 b7 |....... 2@....A.| -00000240 0d 9c 4c 41 90 01 0b 7a 7e b2 b2 46 39 dc 51 25 |..LA...z~..F9.Q%| -00000250 98 e0 b9 ec 36 fc 08 64 f0 51 2a 41 e1 e5 61 3d |....6..d.Q*A..a=| -00000260 fc 07 c1 9b 1f 6f 48 d4 1f 46 bf 14 e6 92 61 1a |.....oH..F....a.| -00000270 bd 5f 25 1f 5e b1 3c ac c7 58 63 02 0d 3a e0 d6 |._%.^.<..Xc..:..| -00000280 e9 39 fc ec 59 66 2e 91 b2 65 37 eb a8 b5 60 d9 |.9..Yf...e7...`.| -00000290 49 05 9f 6f cc 71 79 bb f7 68 16 03 03 00 93 0f |I..o.qy..h......| -000002a0 00 00 8f 04 03 00 8b 30 81 88 02 42 00 bd 6a 29 |.......0...B..j)| -000002b0 21 06 1a e2 67 a1 7f 10 ab ca 3f 74 5a bc 2f 5d |!...g.....?tZ./]| -000002c0 53 d0 59 90 f2 d0 b4 2d 75 47 67 0b 67 55 b6 4f |S.Y....-uGg.gU.O| -000002d0 75 7d 32 d8 a7 25 c8 4c 90 0b 56 65 be 60 5d ee |u}2..%.L..Ve.`].| -000002e0 f7 b3 80 79 26 e5 25 1d 17 cc d8 36 fc 39 02 42 |...y&.%....6.9.B| -000002f0 01 c3 32 d6 f2 59 9e 10 c8 bf 7f 74 27 a1 00 df |..2..Y.....t'...| -00000300 55 05 f0 b3 81 a1 6e 10 a6 fb 0b e4 1c 3f 62 02 |U.....n......?b.| -00000310 c9 cc c2 4b 97 ad 0c 88 98 07 6c 98 6d db 9d 9f |...K......l.m...| -00000320 68 a0 56 ab 5f f9 a2 21 33 86 64 53 de 37 ff 68 |h.V._..!3.dS.7.h| -00000330 04 9d 14 03 03 00 01 01 16 03 03 00 40 85 14 34 |............@..4| -00000340 d6 74 a9 d0 0b e9 1f 34 a9 e9 6c cf 5a ac 88 22 |.t.....4..l.Z.."| -00000350 51 4d ae 16 05 dd 9e c1 36 5e e3 cf b1 5a b5 48 |QM......6^...Z.H| -00000360 6c 24 b1 d6 fb 7f 03 6a 98 41 90 de 6d c7 b2 49 |l$.....j.A..m..I| -00000370 d9 a3 c7 45 ff 18 7c f7 a4 cf 05 59 87 |...E..|....Y.| +00000210 03 03 00 86 10 00 00 82 00 80 b1 e6 1d 71 51 c1 |.............qQ.| +00000220 3a bd 0a 32 95 14 0c 83 7a 2b ec 89 24 f2 29 d8 |:..2....z+..$.).| +00000230 72 84 ae 13 33 90 58 93 b6 46 6c 54 11 54 5b d3 |r...3.X..FlT.T[.| +00000240 59 da 02 4a de 2a 56 67 04 32 3b 44 6b ac 6d 6c |Y..J.*Vg.2;Dk.ml| +00000250 c5 de 9d b2 9e 7b ec 27 05 9d 47 6d a1 0b 50 71 |.....{.'..Gm..Pq| +00000260 ea 19 cc 60 5e db 6c 2f 06 b7 6e ce 51 bf 93 a9 |...`^.l/..n.Q...| +00000270 0e c7 85 c1 83 d2 ac fe 6a d2 a9 bd b3 54 4f 45 |........j....TOE| +00000280 4b e4 40 68 fb 30 21 ec 1c fc 76 a6 db 8b e1 46 |K.@h.0!...v....F| +00000290 8c 0c 56 1f c0 e5 9b 2c 54 eb 16 03 03 00 93 0f |..V....,T.......| +000002a0 00 00 8f 04 03 00 8b 30 81 88 02 42 01 11 9a de |.......0...B....| +000002b0 07 19 df 8b d0 56 e7 b5 b0 d2 d4 c1 32 58 93 88 |.....V......2X..| +000002c0 ea a6 73 86 f9 e6 be b5 c5 1f d6 0d da 28 59 89 |..s..........(Y.| +000002d0 21 73 fe e8 30 b9 f0 d1 01 d3 e0 54 79 a6 67 0d |!s..0......Ty.g.| +000002e0 84 88 94 2c b9 b6 0e 19 06 34 cc f1 5f 01 02 42 |...,.....4.._..B| +000002f0 00 fe 02 83 e8 46 a8 5f ef 5a f0 e1 6f 3f 73 b9 |.....F._.Z..o?s.| +00000300 0f a1 64 db a7 c4 fb 1e 9f ac da 33 ac a2 7f ff |..d........3....| +00000310 64 c1 26 37 17 41 c0 5e f4 37 5c 76 23 7d 92 3a |d.&7.A.^.7\v#}.:| +00000320 ea 35 7f 83 03 7b a4 65 44 5d fb 84 08 39 c7 90 |.5...{.eD]...9..| +00000330 cc 4a 14 03 03 00 01 01 16 03 03 00 40 7b 06 98 |.J..........@{..| +00000340 ff f7 d5 a1 68 60 23 25 bc df 12 27 7f 64 1e c8 |....h`#%...'.d..| +00000350 bc 6d 26 28 29 d0 9f 56 6a f1 5b cd 4e 17 6c 32 |.m&()..Vj.[.N.l2| +00000360 15 b9 7a 55 02 9b 66 1c e3 97 40 26 69 7b e7 02 |..zU..f...@&i{..| +00000370 b0 37 d1 ec ed 96 2e 92 5a 5f 90 c1 be |.7......Z_...| >>> Flow 4 (server to client) 00000000 14 03 03 00 01 01 16 03 03 00 40 00 00 00 00 00 |..........@.....| -00000010 00 00 00 00 00 00 00 00 00 00 00 63 1a 77 66 2a |...........c.wf*| -00000020 49 3a b2 17 83 74 e1 d9 70 96 de 01 84 09 f4 88 |I:...t..p.......| -00000030 c3 e7 3b 65 11 6f 13 32 b8 b4 f4 41 ca 6a d6 d7 |..;e.o.2...A.j..| -00000040 51 a3 a1 f0 2d 5b b4 55 29 f9 d3 17 03 03 00 40 |Q...-[.U)......@| +00000010 00 00 00 00 00 00 00 00 00 00 00 0f e2 df ef c7 |................| +00000020 0e 11 83 70 ba a8 fa 90 e3 d5 df 76 dd 7a f1 63 |...p.......v.z.c| +00000030 ca a3 12 c7 42 45 ae 1a a3 0f 3b 4c 46 52 91 8e |....BE....;LFR..| +00000040 bf df 21 be cb ed 93 12 8a ba 88 17 03 03 00 40 |..!............@| 00000050 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -00000060 d7 30 0f 03 89 22 4c 19 5f 06 a7 4b 95 59 91 52 |.0..."L._..K.Y.R| -00000070 2a 65 ab 99 cb 71 99 8b 13 82 44 92 6b ff 59 07 |*e...q....D.k.Y.| -00000080 28 ca 01 68 ab ad ba ee 6c 6a 19 0b e5 6d 82 24 |(..h....lj...m.$| +00000060 12 fd 68 02 2d 6e aa 2f df e4 0b a1 2c 13 e1 23 |..h.-n./....,..#| +00000070 f9 78 4b 18 a3 1f 28 78 4d f6 25 83 4c 0c 8c df |.xK...(xM.%.L...| +00000080 8b ed a9 b2 87 8c 95 e3 87 8e 71 ad d9 23 05 91 |..........q..#..| 00000090 15 03 03 00 30 00 00 00 00 00 00 00 00 00 00 00 |....0...........| -000000a0 00 00 00 00 00 fc 07 f4 d4 bb 24 a3 f1 cf dc 3c |..........$....<| -000000b0 ac 14 63 50 32 34 fd 73 c0 eb f2 78 7b 3b ea 58 |..cP24.s...x{;.X| -000000c0 cc 3e ff 7f e5 |.>...| +000000a0 00 00 00 00 00 80 9a 8c 0e 27 b9 7a f9 61 a4 a6 |.........'.z.a..| +000000b0 4e c9 24 02 ce 1c 93 5c 5a 7a c9 1e 5f b2 a1 9b |N.$....\Zz.._...| +000000c0 e3 0d 47 85 ab |..G..| diff --git a/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndEd25519Given b/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndEd25519Given index f8cc96058dfc1b..4ca8a034ed21da 100644 --- a/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndEd25519Given +++ b/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndEd25519Given @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 6d 01 00 00 69 03 03 8a fe ad ad 75 |....m...i......u| -00000010 e4 8c bf bf b7 b6 66 14 92 eb 84 85 9c c8 a7 66 |......f........f| -00000020 04 2a d0 63 5e a6 bf 85 e9 4f 49 00 00 04 00 2f |.*.c^....OI..../| +00000000 16 03 01 00 6d 01 00 00 69 03 03 73 b2 f2 a9 ed |....m...i..s....| +00000010 88 e0 79 65 b4 3b 58 0d b2 d8 ab cf d7 12 12 c6 |..ye.;X.........| +00000020 99 f9 36 75 d3 f8 3e 94 cf 39 25 00 00 04 00 2f |..6u..>..9%..../| 00000030 00 ff 01 00 00 3c 00 16 00 00 00 17 00 00 00 0d |.....<..........| 00000040 00 30 00 2e 04 03 05 03 06 03 08 07 08 08 08 09 |.0..............| 00000050 08 0a 08 0b 08 04 08 05 08 06 04 01 05 01 06 01 |................| @@ -49,10 +49,10 @@ 00000260 ee b1 87 82 f1 6c 04 ed 73 bb b3 43 77 8d 0c 1c |.....l..s..Cw...| 00000270 f1 0f a1 d8 40 83 61 c9 4c 72 2b 9d ae db 46 06 |....@.a.Lr+...F.| 00000280 06 4d f4 c1 b3 3e c0 d1 bd 42 d4 db fe 3d 13 60 |.M...>...B...=.`| -00000290 84 5c 21 d3 3b e9 fa e7 16 03 03 00 23 0d 00 00 |.\!.;.......#...| -000002a0 1f 02 01 40 00 18 08 04 04 03 08 07 08 05 08 06 |...@............| -000002b0 04 01 05 01 06 01 05 03 06 03 02 01 02 03 00 00 |................| -000002c0 16 03 03 00 04 0e 00 00 00 |.........| +00000290 84 5c 21 d3 3b e9 fa e7 16 03 03 00 1f 0d 00 00 |.\!.;...........| +000002a0 1b 02 01 40 00 14 08 04 04 03 08 07 08 05 08 06 |...@............| +000002b0 04 01 05 01 06 01 05 03 06 03 00 00 16 03 03 00 |................| +000002c0 04 0e 00 00 00 |.....| >>> Flow 3 (client to server) 00000000 16 03 03 01 3c 0b 00 01 38 00 01 35 00 01 32 30 |....<...8..5..20| 00000010 82 01 2e 30 81 e1 a0 03 02 01 02 02 10 17 d1 81 |...0............| @@ -74,36 +74,36 @@ 00000110 8a 4e 34 40 39 d6 b3 10 dc 19 fe a0 22 71 b3 f5 |.N4@9......."q..| 00000120 8f a1 58 0d cd f4 f1 85 24 bf e6 3d 14 df df ed |..X.....$..=....| 00000130 0e e1 17 d8 11 a2 60 d0 8a 37 23 2a c2 46 aa 3a |......`..7#*.F.:| -00000140 08 16 03 03 00 86 10 00 00 82 00 80 77 8b 9f 34 |............w..4| -00000150 b4 db a7 0d 5b ed 1b 2f 4a 41 64 f5 ce 4a 00 7c |....[../JAd..J.|| -00000160 91 32 b3 cf 61 18 41 04 ae fa 3b 14 de 19 0e 64 |.2..a.A...;....d| -00000170 f9 ec 75 a6 48 7e 28 57 26 f5 1c 75 1d 42 73 fc |..u.H~(W&..u.Bs.| -00000180 11 51 2b ef e5 08 83 ac 17 ec 78 b8 5b 14 84 c9 |.Q+.......x.[...| -00000190 bc 7f 22 fd 54 69 7a 82 36 c7 21 bc d6 04 c4 e7 |..".Tiz.6.!.....| -000001a0 bc 48 c8 72 56 5d 1e 65 41 21 0a 26 85 a0 d8 c3 |.H.rV].eA!.&....| -000001b0 50 f0 b6 07 25 ee 79 b8 f5 e6 17 85 d4 09 e7 d7 |P...%.y.........| -000001c0 ab 8f 17 cb c2 13 a0 5a 50 cb e4 a7 16 03 03 00 |.......ZP.......| -000001d0 48 0f 00 00 44 08 07 00 40 b7 24 50 46 db d4 8c |H...D...@.$PF...| -000001e0 68 17 f5 5e 79 a9 80 8c 40 23 92 33 4e 1e cc ee |h..^y...@#.3N...| -000001f0 d5 35 4d b8 2a 52 f0 7f 50 8e c6 d5 5f bc 08 35 |.5M.*R..P..._..5| -00000200 a2 6d db cb 96 52 ec 92 c7 62 c7 59 ab d8 6f 9d |.m...R...b.Y..o.| -00000210 d7 46 35 71 28 41 89 59 02 14 03 03 00 01 01 16 |.F5q(A.Y........| -00000220 03 03 00 40 3e 12 44 bc c6 3d 88 71 ba d3 0c 26 |...@>.D..=.q...&| -00000230 20 72 b0 7f 25 83 9f fd 77 c1 f5 1e 47 28 2e 60 | r..%...w...G(.`| -00000240 53 e0 ac 52 e8 94 e4 87 90 3f af f3 a4 c0 d3 ba |S..R.....?......| -00000250 fe b7 06 54 f7 13 33 36 47 8f 5e 45 22 84 18 3a |...T..36G.^E"..:| -00000260 1f 14 21 85 |..!.| +00000140 08 16 03 03 00 86 10 00 00 82 00 80 1c aa 0a c6 |................| +00000150 76 22 2b bc 67 c7 db 5a 59 0c 2b 1d 1a 66 9b c5 |v"+.g..ZY.+..f..| +00000160 55 ac 80 bf 23 11 68 96 82 df 44 cf bc 44 4f 54 |U...#.h...D..DOT| +00000170 ce 0c 32 01 59 5e 3e a8 28 e1 33 7d 7d fb 2a 87 |..2.Y^>.(.3}}.*.| +00000180 53 d1 32 25 b8 29 5a 5e 45 24 4d a8 47 58 bc 9c |S.2%.)Z^E$M.GX..| +00000190 6f f3 61 a9 ca e0 ad 32 88 04 1a da 83 ff fd 31 |o.a....2.......1| +000001a0 84 65 9e 33 bb 79 d4 71 55 52 bc 57 fd 2e d5 98 |.e.3.y.qUR.W....| +000001b0 46 b9 dc 74 58 7c c9 25 44 3c 07 97 5d bc 65 b5 |F..tX|.%D<..].e.| +000001c0 b5 46 50 fa 52 f9 45 d7 0f f5 d2 4e 16 03 03 00 |.FP.R.E....N....| +000001d0 48 0f 00 00 44 08 07 00 40 e2 1c ab 11 6c 52 e6 |H...D...@....lR.| +000001e0 e8 7f 67 f0 6e 6a e4 a8 4f 25 89 31 d7 f8 dd 6f |..g.nj..O%.1...o| +000001f0 fd c7 84 e9 59 6b 77 b6 3b fb bc b3 d6 a7 96 4c |....Ykw.;......L| +00000200 2f 54 d2 cf 6b 06 5f a5 69 b6 85 0e a9 a2 90 aa |/T..k._.i.......| +00000210 c4 b2 89 17 b3 c7 b9 73 00 14 03 03 00 01 01 16 |.......s........| +00000220 03 03 00 40 45 8d 48 5b 23 74 21 05 ae 22 ce c0 |...@E.H[#t!.."..| +00000230 8a 05 9a 15 7e fb 61 73 dd 45 fd d0 97 a4 ca f5 |....~.as.E......| +00000240 84 f0 01 c4 e4 44 78 c2 14 4f b6 27 0f e8 5a 9d |.....Dx..O.'..Z.| +00000250 69 7c 9b c4 c5 a3 4d 42 bf 2b 89 c6 a3 c2 ca 7a |i|....MB.+.....z| +00000260 d3 6c 5e 51 |.l^Q| >>> Flow 4 (server to client) 00000000 14 03 03 00 01 01 16 03 03 00 40 00 00 00 00 00 |..........@.....| -00000010 00 00 00 00 00 00 00 00 00 00 00 d4 e6 b8 6f 53 |..............oS| -00000020 6a d9 37 2b a4 95 9f 04 e5 99 2f f9 9a 16 fd a7 |j.7+....../.....| -00000030 2d 39 d9 aa 7c 26 9e 44 4b 7f 8f d5 c6 24 4d ac |-9..|&.DK....$M.| -00000040 13 ca 8a 45 1e 66 dc 9a bf 76 22 17 03 03 00 40 |...E.f...v"....@| +00000010 00 00 00 00 00 00 00 00 00 00 00 d0 d7 ea c0 57 |...............W| +00000020 b8 c4 0e ad 2b ba 7e f7 40 0e 92 42 0b c1 55 38 |....+.~.@..B..U8| +00000030 89 ac d8 9f 46 96 89 c8 a0 06 e7 84 ac 42 6f a8 |....F........Bo.| +00000040 e2 67 49 fe 5b 2f 66 3e 47 c3 14 17 03 03 00 40 |.gI.[/f>G......@| 00000050 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -00000060 3f 5c 60 f8 22 7b aa 82 38 c4 4a 2e 07 50 cb 6c |?\`."{..8.J..P.l| -00000070 3f 6f a9 39 bf 21 ce 7a 30 72 03 90 ec bc 9c 18 |?o.9.!.z0r......| -00000080 1f a9 7f 82 3a d9 46 d9 d8 b8 77 65 e8 b3 e7 f5 |....:.F...we....| +00000060 95 78 c4 75 cf 05 a9 ce aa 85 0b 8e 4e fc 4b dc |.x.u........N.K.| +00000070 59 70 3e 68 85 68 97 9a eb 22 22 3a 8c 61 91 a4 |Yp>h.h..."":.a..| +00000080 89 06 bd 9e fc 8d 1d 4b ed fe 4b d6 e7 0a 6e 2b |.......K..K...n+| 00000090 15 03 03 00 30 00 00 00 00 00 00 00 00 00 00 00 |....0...........| -000000a0 00 00 00 00 00 6d 29 d4 87 0a b4 1d b4 9d f4 12 |.....m).........| -000000b0 bc 3d a3 1b 79 21 85 0d e7 10 64 92 40 39 05 99 |.=..y!....d.@9..| -000000c0 c8 a7 dd ef 0e |.....| +000000a0 00 00 00 00 00 92 d4 46 1f 6b d5 63 a7 95 0d c2 |.......F.k.c....| +000000b0 2f a9 a2 5f 0d 70 8f a5 31 e3 5c 1d fa ac f6 2e |/.._.p..1.\.....| +000000c0 02 6d e8 9f 95 |.m...| diff --git a/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndGiven b/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndGiven index cc6450a4ce8b34..585e6af6579bd1 100644 --- a/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndGiven +++ b/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndGiven @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 6d 01 00 00 69 03 03 e9 31 0f d0 25 |....m...i...1..%| -00000010 ef 25 a7 1a 9b 8c 4b a3 ca 2b a6 54 89 1c e1 68 |.%....K..+.T...h| -00000020 6f b2 b2 60 6f 8a dc 87 24 8c 7b 00 00 04 00 2f |o..`o...$.{..../| +00000000 16 03 01 00 6d 01 00 00 69 03 03 0e c0 95 b1 0b |....m...i.......| +00000010 7b b5 57 cc 04 e7 03 d0 66 8d ee 9d da 65 dc 74 |{.W.....f....e.t| +00000020 0d de 11 47 38 cd 19 12 f4 06 17 00 00 04 00 2f |...G8........../| 00000030 00 ff 01 00 00 3c 00 16 00 00 00 17 00 00 00 0d |.....<..........| 00000040 00 30 00 2e 04 03 05 03 06 03 08 07 08 08 08 09 |.0..............| 00000050 08 0a 08 0b 08 04 08 05 08 06 04 01 05 01 06 01 |................| @@ -49,10 +49,10 @@ 00000260 ee b1 87 82 f1 6c 04 ed 73 bb b3 43 77 8d 0c 1c |.....l..s..Cw...| 00000270 f1 0f a1 d8 40 83 61 c9 4c 72 2b 9d ae db 46 06 |....@.a.Lr+...F.| 00000280 06 4d f4 c1 b3 3e c0 d1 bd 42 d4 db fe 3d 13 60 |.M...>...B...=.`| -00000290 84 5c 21 d3 3b e9 fa e7 16 03 03 00 23 0d 00 00 |.\!.;.......#...| -000002a0 1f 02 01 40 00 18 08 04 04 03 08 07 08 05 08 06 |...@............| -000002b0 04 01 05 01 06 01 05 03 06 03 02 01 02 03 00 00 |................| -000002c0 16 03 03 00 04 0e 00 00 00 |.........| +00000290 84 5c 21 d3 3b e9 fa e7 16 03 03 00 1f 0d 00 00 |.\!.;...........| +000002a0 1b 02 01 40 00 14 08 04 04 03 08 07 08 05 08 06 |...@............| +000002b0 04 01 05 01 06 01 05 03 06 03 00 00 16 03 03 00 |................| +000002c0 04 0e 00 00 00 |.....| >>> Flow 3 (client to server) 00000000 16 03 03 01 fd 0b 00 01 f9 00 01 f6 00 01 f3 30 |...............0| 00000010 82 01 ef 30 82 01 58 a0 03 02 01 02 02 10 5c 19 |...0..X.......\.| @@ -86,40 +86,40 @@ 000001d0 ac 11 b1 28 56 be 1d cd 61 62 84 09 bf d6 80 c6 |...(V...ab......| 000001e0 45 8d 82 2c b4 d8 83 9b db c9 22 b7 2a 12 11 7b |E..,......".*..{| 000001f0 fa 02 3b c1 c9 ff ea c9 9d a8 49 d3 95 d7 d5 0e |..;.......I.....| -00000200 e5 35 16 03 03 00 86 10 00 00 82 00 80 29 51 da |.5...........)Q.| -00000210 8e 5c 3e fb 44 8a 0f 97 42 23 8b e2 73 cc e2 90 |.\>.D...B#..s...| -00000220 11 c4 98 01 e9 60 96 9e a9 96 30 c5 95 f8 56 0e |.....`....0...V.| -00000230 4a 2e 77 e7 7e 23 b7 49 31 c4 87 c5 69 c6 ca 6f |J.w.~#.I1...i..o| -00000240 ea 53 41 b4 2e 1e f6 0b 33 f5 e1 40 69 c0 91 6f |.SA.....3..@i..o| -00000250 88 c1 68 c8 18 99 6e fe b3 5f 9b ee f1 4a 76 41 |..h...n.._...JvA| -00000260 1f d1 05 f5 39 76 61 e6 a6 ea 75 0e 50 32 a1 19 |....9va...u.P2..| -00000270 20 6a 4c 5d 62 6e 2a 6e af f9 9c 38 b6 3a bc 86 | jL]bn*n...8.:..| -00000280 eb ac 6d d3 b5 48 30 11 4d 98 2e 61 34 16 03 03 |..m..H0.M..a4...| -00000290 00 88 0f 00 00 84 08 04 00 80 82 ed 3f da b5 50 |............?..P| -000002a0 d2 50 51 14 cf ee f7 b9 7b a9 0c 77 2f 88 42 0a |.PQ.....{..w/.B.| -000002b0 34 a9 5d e7 32 26 3a 28 87 49 fb c4 83 31 68 c6 |4.].2&:(.I...1h.| -000002c0 0d 32 d4 31 0a d1 d6 1e 6f 7f 89 93 bf b7 7c c7 |.2.1....o.....|.| -000002d0 95 f8 c3 69 d8 58 4e e4 76 07 36 84 b7 c3 e7 22 |...i.XN.v.6...."| -000002e0 01 4c 59 ae 89 95 bb e0 07 e0 31 6a e2 95 4c d4 |.LY.......1j..L.| -000002f0 01 54 9d 27 82 60 31 13 39 07 47 c2 0c 08 5c d4 |.T.'.`1.9.G...\.| -00000300 03 5a 6f d7 89 a0 67 5e 2d a0 11 03 bf 0e 35 d8 |.Zo...g^-.....5.| -00000310 d0 78 2f 1e d8 15 47 ce c9 d3 14 03 03 00 01 01 |.x/...G.........| -00000320 16 03 03 00 40 d0 0a 0e 93 dd 9a 51 4f a9 7f 5f |....@......QO.._| -00000330 93 a6 60 a6 f2 10 f1 bd bd ae 13 5d 11 b7 0d 1a |..`........]....| -00000340 3d 1e f3 0c b7 53 7c 10 ed fa 8c d7 3f 20 ec f2 |=....S|.....? ..| -00000350 7d e9 15 87 3d d3 05 21 3a bc a5 54 fa 40 3b 53 |}...=..!:..T.@;S| -00000360 41 7c ea c6 28 |A|..(| +00000200 e5 35 16 03 03 00 86 10 00 00 82 00 80 cc d2 66 |.5.............f| +00000210 37 df f1 5d cb 6f 1d 6b 64 ea 62 45 97 dd 47 f8 |7..].o.kd.bE..G.| +00000220 e5 a0 f6 84 46 9b 6c 5b c9 79 60 07 b0 d1 5a e6 |....F.l[.y`...Z.| +00000230 5c 1a 43 b1 04 9f f2 3d 7a 09 da e0 45 ea 30 8a |\.C....=z...E.0.| +00000240 5c 08 07 67 17 2e 55 f5 0a 13 96 5c 92 e5 61 66 |\..g..U....\..af| +00000250 92 fe c6 44 9c 4f 62 54 10 12 df f7 e4 11 74 f6 |...D.ObT......t.| +00000260 35 81 bb 55 4f ce 43 dc 7f 4d bc 4f 8b 0c ef 13 |5..UO.C..M.O....| +00000270 43 8a e5 80 dc 38 3e 8a f5 7b 5c 5d 1c 76 10 06 |C....8>..{\].v..| +00000280 3d c5 05 5b cb 9a 17 20 13 29 a5 36 5d 16 03 03 |=..[... .).6]...| +00000290 00 88 0f 00 00 84 08 04 00 80 a7 7f 2c 3b d1 82 |............,;..| +000002a0 8a 17 50 2a f6 c4 ac ce 47 1b 25 23 4b 0c d0 17 |..P*....G.%#K...| +000002b0 89 18 98 a6 e4 b8 51 70 6a 59 72 1a aa 68 e8 25 |......QpjYr..h.%| +000002c0 f5 4d 72 66 be bb 25 61 9b 36 e0 24 a5 34 e4 36 |.Mrf..%a.6.$.4.6| +000002d0 23 0a 36 a8 81 f4 19 62 98 2b af 1b f5 c4 55 d2 |#.6....b.+....U.| +000002e0 d5 65 58 b8 31 21 f4 fe ce 44 cc ea 77 8e 1d f7 |.eX.1!...D..w...| +000002f0 bc a4 4c e5 cc 90 10 f3 a0 8d 10 72 08 d4 50 1c |..L........r..P.| +00000300 88 82 a7 61 da cb 35 ba 26 3c 18 11 6c 14 d6 1a |...a..5.&<..l...| +00000310 7f 65 3d 2c 74 f0 92 a3 aa fd 14 03 03 00 01 01 |.e=,t...........| +00000320 16 03 03 00 40 f3 3a f0 23 48 35 41 7f d0 ed 22 |....@.:.#H5A..."| +00000330 5b 1a 47 71 60 08 b0 6b cd 32 56 c0 d0 05 90 b3 |[.Gq`..k.2V.....| +00000340 0b 35 3c 03 49 ec 06 5d b4 ce 1d 10 4e bc 75 bf |.5<.I..]....N.u.| +00000350 23 3b f1 d6 8f f6 f0 70 b6 94 8f 51 fd 4f 47 0b |#;.....p...Q.OG.| +00000360 e1 c9 ad c8 14 |.....| >>> Flow 4 (server to client) 00000000 14 03 03 00 01 01 16 03 03 00 40 00 00 00 00 00 |..........@.....| -00000010 00 00 00 00 00 00 00 00 00 00 00 a8 8e 30 08 f0 |.............0..| -00000020 87 7b 13 31 99 6d 7e 9a 9b 03 d3 6f 84 d8 d9 31 |.{.1.m~....o...1| -00000030 2b d2 aa d4 0e ae 6e 72 03 ac e7 7e 5c 22 cc ac |+.....nr...~\"..| -00000040 33 b5 df 04 b2 4a 2b 6f bb a1 6f 17 03 03 00 40 |3....J+o..o....@| +00000010 00 00 00 00 00 00 00 00 00 00 00 6e d3 79 f1 02 |...........n.y..| +00000020 4d 8e ad 3b 33 5a 92 10 55 79 94 b7 43 ed 08 c1 |M..;3Z..Uy..C...| +00000030 e1 5f 04 c6 01 82 ce 96 70 c7 97 8d cc 0a ca d3 |._......p.......| +00000040 46 d4 2f 9f b8 78 57 27 ee 14 aa 17 03 03 00 40 |F./..xW'.......@| 00000050 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -00000060 50 9c 81 04 9b 1d 61 8a 30 9c 18 68 c7 e1 c9 f3 |P.....a.0..h....| -00000070 70 f0 1b b6 4a dd fc c7 e3 e3 20 e2 4d 6f 9f bf |p...J..... .Mo..| -00000080 17 b0 5e 5b 45 73 29 1e d4 30 b4 03 ca 8e 69 63 |..^[Es)..0....ic| +00000060 ed 9f 38 f6 99 84 f5 cf 5e ee 27 64 f7 29 3c 18 |..8.....^.'d.)<.| +00000070 a0 55 23 b7 db 4a 6d 2d 80 c9 75 a5 a3 1f 38 24 |.U#..Jm-..u...8$| +00000080 0a 99 18 0d 0b 5d 7d 03 f7 8d d2 55 fd 98 7f 69 |.....]}....U...i| 00000090 15 03 03 00 30 00 00 00 00 00 00 00 00 00 00 00 |....0...........| -000000a0 00 00 00 00 00 28 ca 6a 4c 1b 3c 11 61 ce b2 58 |.....(.jL.<.a..X| -000000b0 94 e7 e4 7d c5 ce 51 03 c4 ae b5 4c 33 0b 3c 95 |...}..Q....L3.<.| -000000c0 ec b1 65 ea da |..e..| +000000a0 00 00 00 00 00 aa 43 ca 95 7d 1a 47 d5 0b 0e c7 |......C..}.G....| +000000b0 cc 61 3d 43 5b 69 05 a1 39 eb 03 52 41 05 54 e9 |.a=C[i..9..RA.T.| +000000c0 5d b6 ca 56 2c |]..V,| diff --git a/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndPKCS1v15Given b/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndPKCS1v15Given index 875fe1becb17bd..10949e5e3badeb 100644 --- a/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndPKCS1v15Given +++ b/src/crypto/tls/testdata/Server-TLSv12-ClientAuthRequestedAndPKCS1v15Given @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 6d 01 00 00 69 03 03 dc f3 c6 52 a4 |....m...i.....R.| -00000010 9a 9c 53 e0 5a 3c cc 4c 4f 09 32 7f f1 7c 86 6b |..S.Z<.LO.2..|.k| -00000020 75 59 68 a5 81 72 45 46 fb 94 a8 00 00 04 00 2f |uYh..rEF......./| +00000000 16 03 01 00 6d 01 00 00 69 03 03 c2 c7 15 c0 0f |....m...i.......| +00000010 0d fc 44 60 25 22 6c 4a ec f1 b0 66 5d c4 f3 bc |..D`%"lJ...f]...| +00000020 fe da b2 9a af 24 04 b7 bb 74 79 00 00 04 00 2f |.....$...ty..../| 00000030 00 ff 01 00 00 3c 00 16 00 00 00 17 00 00 00 0d |.....<..........| 00000040 00 30 00 2e 04 03 05 03 06 03 08 07 08 08 08 09 |.0..............| 00000050 08 0a 08 0b 08 04 08 05 08 06 04 01 05 01 06 01 |................| @@ -49,10 +49,10 @@ 00000260 ee b1 87 82 f1 6c 04 ed 73 bb b3 43 77 8d 0c 1c |.....l..s..Cw...| 00000270 f1 0f a1 d8 40 83 61 c9 4c 72 2b 9d ae db 46 06 |....@.a.Lr+...F.| 00000280 06 4d f4 c1 b3 3e c0 d1 bd 42 d4 db fe 3d 13 60 |.M...>...B...=.`| -00000290 84 5c 21 d3 3b e9 fa e7 16 03 03 00 23 0d 00 00 |.\!.;.......#...| -000002a0 1f 02 01 40 00 18 08 04 04 03 08 07 08 05 08 06 |...@............| -000002b0 04 01 05 01 06 01 05 03 06 03 02 01 02 03 00 00 |................| -000002c0 16 03 03 00 04 0e 00 00 00 |.........| +00000290 84 5c 21 d3 3b e9 fa e7 16 03 03 00 1f 0d 00 00 |.\!.;...........| +000002a0 1b 02 01 40 00 14 08 04 04 03 08 07 08 05 08 06 |...@............| +000002b0 04 01 05 01 06 01 05 03 06 03 00 00 16 03 03 00 |................| +000002c0 04 0e 00 00 00 |.....| >>> Flow 3 (client to server) 00000000 16 03 03 01 fd 0b 00 01 f9 00 01 f6 00 01 f3 30 |...............0| 00000010 82 01 ef 30 82 01 58 a0 03 02 01 02 02 10 5c 19 |...0..X.......\.| @@ -86,40 +86,40 @@ 000001d0 ac 11 b1 28 56 be 1d cd 61 62 84 09 bf d6 80 c6 |...(V...ab......| 000001e0 45 8d 82 2c b4 d8 83 9b db c9 22 b7 2a 12 11 7b |E..,......".*..{| 000001f0 fa 02 3b c1 c9 ff ea c9 9d a8 49 d3 95 d7 d5 0e |..;.......I.....| -00000200 e5 35 16 03 03 00 86 10 00 00 82 00 80 b5 77 6b |.5............wk| -00000210 fa 10 fb df 48 8f e7 51 b4 cb 14 c5 48 bd 63 d6 |....H..Q....H.c.| -00000220 0e 19 d0 81 a8 5a d7 b5 43 84 35 85 37 b7 8d 2e |.....Z..C.5.7...| -00000230 c7 c8 70 4c f4 45 bf be 17 86 e7 40 1d 6f 88 2a |..pL.E.....@.o.*| -00000240 91 b5 aa aa 34 f7 9a f3 96 e4 dd 51 15 88 be f1 |....4......Q....| -00000250 80 a9 6f 94 ed c7 5d 28 66 b4 37 e8 22 4f 42 c3 |..o...](f.7."OB.| -00000260 b5 f0 2f dd 57 dc 8d e5 5a c0 9d fa ce 3c 7a 2d |../.W...Z....>> Flow 4 (server to client) 00000000 14 03 03 00 01 01 16 03 03 00 40 00 00 00 00 00 |..........@.....| -00000010 00 00 00 00 00 00 00 00 00 00 00 a2 b0 ad 7e 71 |..............~q| -00000020 0c 2c db df 4c b1 4f 19 e6 00 4f 11 ff 5e 4a c5 |.,..L.O...O..^J.| -00000030 c2 9d 8c 6c 03 50 12 3d 81 ec 44 5a 75 ba 2d 48 |...l.P.=..DZu.-H| -00000040 7a 74 c3 a3 68 5a 26 ee 7e f5 a2 17 03 03 00 40 |zt..hZ&.~......@| +00000010 00 00 00 00 00 00 00 00 00 00 00 0c 22 6e 84 b2 |............"n..| +00000020 a1 f9 18 62 3a 86 16 7e e6 17 3b 8e e5 88 b8 8e |...b:..~..;.....| +00000030 3c c5 08 11 8e 0a df df d4 69 bc 01 7d c8 63 33 |<........i..}.c3| +00000040 b5 15 bf 03 5e df 50 29 c5 c4 c2 17 03 03 00 40 |....^.P).......@| 00000050 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -00000060 ad 1b 16 8e 39 99 64 7c c5 49 24 83 c4 4e f6 86 |....9.d|.I$..N..| -00000070 6b 5d 68 ae f4 0b 58 23 83 eb ab 01 52 4d 07 a1 |k]h...X#....RM..| -00000080 59 00 e8 dc a5 a1 6f 76 e2 e9 f2 e1 21 58 6b a0 |Y.....ov....!Xk.| +00000060 45 76 91 63 fa 48 9b c9 47 6a f6 7b fa 72 ab 78 |Ev.c.H..Gj.{.r.x| +00000070 4f cb c4 bb 68 78 7c 71 13 f9 47 32 33 59 d4 16 |O...hx|q..G23Y..| +00000080 83 fa 8e db 37 b6 cb d5 a1 1a e5 21 1b 50 a6 d3 |....7......!.P..| 00000090 15 03 03 00 30 00 00 00 00 00 00 00 00 00 00 00 |....0...........| -000000a0 00 00 00 00 00 3c 6e a7 81 36 d0 8c 99 d8 f3 55 |.....>> Flow 1 (client to server) -00000000 16 03 01 00 6d 01 00 00 69 03 03 ac ea d9 49 98 |....m...i.....I.| -00000010 9a 0c 7c 86 64 7c 73 72 6d 79 3f 7b e9 11 8b 1d |..|.d|srmy?{....| -00000020 79 95 f5 f5 23 9f b2 f1 9c f4 b5 00 00 04 00 2f |y...#........../| +00000000 16 03 01 00 6d 01 00 00 69 03 03 f2 1b 6e dc ce |....m...i....n..| +00000010 fe af a8 4b 44 88 5f ba eb c5 d7 92 7e 69 d7 19 |...KD._.....~i..| +00000020 b3 32 d3 99 e6 be 1f 4f 94 04 e6 00 00 04 00 2f |.2.....O......./| 00000030 00 ff 01 00 00 3c 00 16 00 00 00 17 00 00 00 0d |.....<..........| 00000040 00 30 00 2e 04 03 05 03 06 03 08 07 08 08 08 09 |.0..............| 00000050 08 0a 08 0b 08 04 08 05 08 06 04 01 05 01 06 01 |................| @@ -49,37 +49,37 @@ 00000260 ee b1 87 82 f1 6c 04 ed 73 bb b3 43 77 8d 0c 1c |.....l..s..Cw...| 00000270 f1 0f a1 d8 40 83 61 c9 4c 72 2b 9d ae db 46 06 |....@.a.Lr+...F.| 00000280 06 4d f4 c1 b3 3e c0 d1 bd 42 d4 db fe 3d 13 60 |.M...>...B...=.`| -00000290 84 5c 21 d3 3b e9 fa e7 16 03 03 00 23 0d 00 00 |.\!.;.......#...| -000002a0 1f 02 01 40 00 18 08 04 04 03 08 07 08 05 08 06 |...@............| -000002b0 04 01 05 01 06 01 05 03 06 03 02 01 02 03 00 00 |................| -000002c0 16 03 03 00 04 0e 00 00 00 |.........| +00000290 84 5c 21 d3 3b e9 fa e7 16 03 03 00 1f 0d 00 00 |.\!.;...........| +000002a0 1b 02 01 40 00 14 08 04 04 03 08 07 08 05 08 06 |...@............| +000002b0 04 01 05 01 06 01 05 03 06 03 00 00 16 03 03 00 |................| +000002c0 04 0e 00 00 00 |.....| >>> Flow 3 (client to server) 00000000 16 03 03 00 07 0b 00 00 03 00 00 00 16 03 03 00 |................| -00000010 86 10 00 00 82 00 80 2e 37 44 fb d7 1d 2f 3d a5 |........7D.../=.| -00000020 1b 43 cf f4 1c cf 9d 95 fa be 9f 9d 96 8a 27 5d |.C............']| -00000030 7b be 19 10 bd 5e 9a 3e 49 49 d2 af 85 07 70 f8 |{....^.>II....p.| -00000040 c8 4f 69 02 ff 4e 9d ee f4 0d 4d 54 a1 aa 61 a3 |.Oi..N....MT..a.| -00000050 e0 cc db a7 2c 46 80 6e eb 10 fb cd 2e 3b c5 50 |....,F.n.....;.P| -00000060 2b a5 d9 a0 bf 01 d2 f8 d8 51 2b ad 40 6f c6 6f |+........Q+.@o.o| -00000070 0e 30 53 27 73 89 b7 1b c1 28 ff ff 18 4c fa 6f |.0S's....(...L.o| -00000080 fa 5f 16 b3 38 36 9f f4 07 74 ca bb bb c2 3f aa |._..86...t....?.| -00000090 0d e7 42 24 fb f8 4c 14 03 03 00 01 01 16 03 03 |..B$..L.........| -000000a0 00 40 19 02 9e 3a ce b9 38 40 ce d6 3b 87 b2 f6 |.@...:..8@..;...| -000000b0 1b 7d ee 76 62 f8 6e 04 80 8f cb 1b f7 1e 1d a6 |.}.vb.n.........| -000000c0 50 8a 59 b1 ad 7d c5 9d 2f 2d 14 56 2e e5 3b b3 |P.Y..}../-.V..;.| -000000d0 db da 7e 37 10 97 71 91 d3 7b 93 f6 64 a4 d7 8b |..~7..q..{..d...| -000000e0 d2 f0 |..| +00000010 86 10 00 00 82 00 80 36 1c ba 9d 08 27 52 8b f7 |.......6....'R..| +00000020 24 d9 e6 18 d7 21 75 1d 76 e0 13 a0 35 d5 08 7d |$....!u.v...5..}| +00000030 c1 8e 3f b2 aa 10 b4 f9 d4 77 e6 cd b3 92 94 0e |..?......w......| +00000040 7a c9 0b 5f e2 34 88 ad fc 02 1b 84 10 ff e8 2a |z.._.4.........*| +00000050 dd 2d 82 5c bb ca 15 f8 73 74 ad dd 9f 9d e7 38 |.-.\....st.....8| +00000060 7c cd 74 8e 37 0f 87 62 cf 30 68 8a e2 15 9d d2 ||.t.7..b.0h.....| +00000070 43 4b e3 29 69 e4 db 94 9b 5a 7c c6 9b e8 7d 26 |CK.)i....Z|...}&| +00000080 4b a8 4a 28 c1 47 cf 15 7b 22 a2 1d 6b ac 16 e4 |K.J(.G..{"..k...| +00000090 e1 62 6e be 9a 05 67 14 03 03 00 01 01 16 03 03 |.bn...g.........| +000000a0 00 40 98 4e 3d 6c 72 d8 7f 81 b6 b8 ed 32 2e 98 |.@.N=lr......2..| +000000b0 8d fc c1 1d 56 97 82 ef 3f 7c 86 79 e3 27 06 ed |....V...?|.y.'..| +000000c0 87 c4 28 2d 6b f1 b1 88 d0 67 34 64 ba e1 d9 34 |..(-k....g4d...4| +000000d0 a4 2c ff e0 b2 38 21 5d 04 64 99 a4 34 62 aa 81 |.,...8!].d..4b..| +000000e0 cf 21 |.!| >>> Flow 4 (server to client) 00000000 14 03 03 00 01 01 16 03 03 00 40 00 00 00 00 00 |..........@.....| -00000010 00 00 00 00 00 00 00 00 00 00 00 46 c8 31 06 11 |...........F.1..| -00000020 01 8e df b0 e7 cc 16 d3 97 2e a2 68 e7 a4 d1 0f |...........h....| -00000030 91 71 dd ba db 97 20 45 60 c2 47 c7 ee 56 c4 68 |.q.... E`.G..V.h| -00000040 a4 b1 05 09 e2 68 4d 54 fa ff 01 17 03 03 00 40 |.....hMT.......@| +00000010 00 00 00 00 00 00 00 00 00 00 00 66 31 30 4e 0f |...........f10N.| +00000020 de bd de 72 65 0c 74 ab 64 d8 59 47 fc 6b ad d8 |...re.t.d.YG.k..| +00000030 73 24 da 77 62 ba 0a 8c 69 d3 c4 6f 89 ef 5b 92 |s$.wb...i..o..[.| +00000040 d3 ca 3b e8 67 2b 7c bf 39 7c 8b 17 03 03 00 40 |..;.g+|.9|.....@| 00000050 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -00000060 79 bf ad 63 e3 11 2b d0 41 0e 24 85 92 c4 9b b5 |y..c..+.A.$.....| -00000070 b2 d3 2e fc aa 46 84 85 a7 37 90 fc f0 2b 5a 7e |.....F...7...+Z~| -00000080 28 9f 2e 57 1d 8f c3 ca eb 40 32 79 af 4b b8 28 |(..W.....@2y.K.(| +00000060 5c a8 b1 1c 7e e1 76 b8 d0 0c 4d 3d 3f 36 a8 26 |\...~.v...M=?6.&| +00000070 66 00 dd 47 3c ae 1c 8b 6f b0 6b 80 75 c4 0b 7e |f..G<...o.k.u..~| +00000080 ee fb 8c fe 2f 2f 65 1b 9b e1 72 a9 ac 8f cf da |....//e...r.....| 00000090 15 03 03 00 30 00 00 00 00 00 00 00 00 00 00 00 |....0...........| -000000a0 00 00 00 00 00 8b 6c 27 b8 ff f9 ea ca 68 75 54 |......l'.....huT| -000000b0 bf bf a7 f4 b1 58 a5 b3 31 01 4d c7 85 58 31 d4 |.....X..1.M..X1.| -000000c0 e7 da 7e 77 68 |..~wh| +000000a0 00 00 00 00 00 9d 96 95 12 3b 42 41 a8 30 b2 8c |.........;BA.0..| +000000b0 3d 18 f6 27 b7 77 30 d9 29 0c 68 ec 2b 09 26 91 |=..'.w0.).h.+.&.| +000000c0 23 0c e2 10 07 |#....| diff --git a/src/crypto/tls/testdata/Server-TLSv12-ECDHE-ECDSA-AES b/src/crypto/tls/testdata/Server-TLSv12-ECDHE-ECDSA-AES index 697b8102dd38d3..f0d266bdc7af13 100644 --- a/src/crypto/tls/testdata/Server-TLSv12-ECDHE-ECDSA-AES +++ b/src/crypto/tls/testdata/Server-TLSv12-ECDHE-ECDSA-AES @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 85 01 00 00 81 03 03 83 21 a6 e4 ea |............!...| -00000010 e9 7b 3a 7c 72 28 ee 68 c5 c7 fa f1 98 ed 4a be |.{:|r(.h......J.| -00000020 b8 42 13 fb d3 ab 63 16 d2 74 c8 00 00 04 c0 0a |.B....c..t......| +00000000 16 03 01 00 85 01 00 00 81 03 03 19 8a e1 c7 50 |...............P| +00000010 ba 63 15 9b d5 85 f1 8c 55 43 d3 ce 9c d6 35 20 |.c......UC....5 | +00000020 f3 49 3d 55 a5 11 57 6d db 42 1d 00 00 04 c0 0a |.I=U..Wm.B......| 00000030 00 ff 01 00 00 54 00 0b 00 04 03 00 01 02 00 0a |.....T..........| 00000040 00 0c 00 0a 00 1d 00 17 00 1e 00 19 00 18 00 16 |................| 00000050 00 00 00 17 00 00 00 0d 00 30 00 2e 04 03 05 03 |.........0......| @@ -46,39 +46,39 @@ 00000220 c1 33 13 83 0d 94 06 bb d4 37 7a f6 ec 7a c9 86 |.3.......7z..z..| 00000230 2e dd d7 11 69 7f 85 7c 56 de fb 31 78 2b e4 c7 |....i..|V..1x+..| 00000240 78 0d ae cb be 9e 4e 36 24 31 7b 6a 0f 39 95 12 |x.....N6$1{j.9..| -00000250 07 8f 2a 16 03 03 00 b7 0c 00 00 b3 03 00 1d 20 |..*............ | +00000250 07 8f 2a 16 03 03 00 b6 0c 00 00 b2 03 00 1d 20 |..*............ | 00000260 2f e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 |/.}.G.bC.(.._.).| 00000270 30 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 |0.........._X.;t| -00000280 04 03 00 8b 30 81 88 02 42 00 b9 39 44 59 12 77 |....0...B..9DY.w| -00000290 8d e2 79 25 01 d1 6a 05 3d 53 ea f3 91 d6 c5 09 |..y%..j.=S......| -000002a0 24 bd 0c ad 24 cc 1c a7 fb 03 eb 0a 0d f4 30 96 |$...$.........0.| -000002b0 8d 28 a1 b3 64 ba 30 27 95 29 23 22 91 62 c3 1f |.(..d.0'.)#".b..| -000002c0 51 aa c8 be 17 85 31 8e f5 40 3e 02 42 00 ee a1 |Q.....1..@>.B...| -000002d0 64 14 a1 52 b3 e5 54 c9 24 53 94 5a 43 d8 4f 79 |d..R..T.$S.ZC.Oy| -000002e0 69 4b a8 51 ee de b3 b0 f7 1a 57 a3 28 72 d2 13 |iK.Q......W.(r..| -000002f0 a6 d3 17 0b c4 45 34 7f 10 3b 81 cb 0c 8d 51 b6 |.....E4..;....Q.| -00000300 0b 86 21 d0 ee 1d 7e 73 6b ea 77 8c 66 dc 65 16 |..!...~sk.w.f.e.| -00000310 03 03 00 04 0e 00 00 00 |........| +00000280 04 03 00 8a 30 81 87 02 42 01 f2 09 77 4a e7 f5 |....0...B...wJ..| +00000290 a8 35 3b dd 9d 62 5a 07 97 1e 76 93 b6 07 21 3e |.5;..bZ...v...!>| +000002a0 c8 fd 99 35 50 8a 8b ad e5 de 03 07 c8 5e fe 03 |...5P........^..| +000002b0 c1 99 04 ad 53 b6 76 67 eb 04 99 54 11 4d 4d e9 |....S.vg...T.MM.| +000002c0 74 3f 89 6e d9 c8 02 98 c5 3c cf 02 41 4e 64 21 |t?.n.....<..ANd!| +000002d0 1a 01 5f 2e 89 17 cc 65 33 d0 59 ed 17 59 c4 43 |.._....e3.Y..Y.C| +000002e0 0a fc 68 30 9c e2 c3 86 fb 2a c1 4a ae 32 ef 1d |..h0.....*.J.2..| +000002f0 06 27 36 7d d5 cd 68 23 4c e9 7e 64 b8 eb 42 05 |.'6}..h#L.~d..B.| +00000300 ef 83 36 b2 9e a7 ae 1a cd b0 3a 17 3a 46 16 03 |..6.......:.:F..| +00000310 03 00 04 0e 00 00 00 |.......| >>> Flow 3 (client to server) -00000000 16 03 03 00 25 10 00 00 21 20 ed 3e ba a7 43 53 |....%...! .>..CS| -00000010 5e e4 60 aa 31 3f e1 69 60 32 25 3d fd 8b 32 da |^.`.1?.i`2%=..2.| -00000020 f2 c5 db c7 02 e6 4d d0 de 15 14 03 03 00 01 01 |......M.........| -00000030 16 03 03 00 40 ee 28 f2 27 82 24 9d 17 d1 48 7a |....@.(.'.$...Hz| -00000040 74 2d dd 16 18 b7 70 97 2f 2b 91 47 eb c2 1d ae |t-....p./+.G....| -00000050 3f 48 52 cd ff e7 9e 0b 35 ad 1f 60 5e 07 b1 5e |?HR.....5..`^..^| -00000060 1c ba 6a 85 bb 6b 30 94 41 8a 59 81 cf 37 5f 26 |..j..k0.A.Y..7_&| -00000070 b1 52 36 5f df |.R6_.| +00000000 16 03 03 00 25 10 00 00 21 20 73 43 c2 08 92 f5 |....%...! sC....| +00000010 db bf 2f 8a eb 49 55 f7 5d 6b 80 64 f7 d9 75 1f |../..IU.]k.d..u.| +00000020 67 f6 35 21 3c 95 3f 1c 04 1a 14 03 03 00 01 01 |g.5!<.?.........| +00000030 16 03 03 00 40 59 bb 5a 5d 76 73 a5 30 0e 29 d3 |....@Y.Z]vs.0.).| +00000040 17 d8 2f 30 e6 ed 02 c6 83 12 44 42 d8 79 86 e0 |../0......DB.y..| +00000050 78 7b 43 8d 5b 7c 85 42 fb 7c 67 b0 d0 71 03 0e |x{C.[|.B.|g..q..| +00000060 d0 6b b6 06 f1 16 72 c0 16 66 cf 53 df ae 62 3b |.k....r..f.S..b;| +00000070 f3 57 52 4d 08 |.WRM.| >>> Flow 4 (server to client) 00000000 14 03 03 00 01 01 16 03 03 00 40 00 00 00 00 00 |..........@.....| -00000010 00 00 00 00 00 00 00 00 00 00 00 f5 05 5a a6 22 |.............Z."| -00000020 90 4e 8d d9 f1 55 c4 78 f2 ec 9d 97 cd fe af ae |.N...U.x........| -00000030 b7 62 00 67 2e b2 d9 1e 0c a3 c8 6a bf d2 3c 42 |.b.g.......j..u......| +00000080 cd 14 5b 4b 0a 7b a2 e6 54 b3 bd 3c f0 eb ca 78 |..[K.{..T..<...x| 00000090 15 03 03 00 30 00 00 00 00 00 00 00 00 00 00 00 |....0...........| -000000a0 00 00 00 00 00 13 3e 42 a5 61 84 ae 49 8b b9 91 |......>B.a..I...| -000000b0 c2 a3 76 74 1e 4f 53 0a fc 71 de 0d d2 44 c8 ac |..vt.OS..q...D..| -000000c0 2e 09 27 e6 ad |..'..| +000000a0 00 00 00 00 00 e6 4b 35 cc 69 58 89 49 67 99 f4 |......K5.iX.Ig..| +000000b0 c2 14 2a bb e7 21 2b fe fe b5 60 ae b2 2a 96 15 |..*..!+...`..*..| +000000c0 e0 65 d2 54 0b |.e.T.| diff --git a/src/crypto/tls/testdata/Server-TLSv12-ExportKeyingMaterial b/src/crypto/tls/testdata/Server-TLSv12-ExportKeyingMaterial index ff8d63568d7876..a670f05d58c54f 100644 --- a/src/crypto/tls/testdata/Server-TLSv12-ExportKeyingMaterial +++ b/src/crypto/tls/testdata/Server-TLSv12-ExportKeyingMaterial @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 89 01 00 00 85 03 03 ad 13 87 9e b3 |................| -00000010 c7 71 bb bf be e3 b9 80 3f 17 bf 41 37 95 22 e6 |.q......?..A7.".| -00000020 f2 98 a9 15 62 1d 65 06 69 ea 53 00 00 04 c0 14 |....b.e.i.S.....| +00000000 16 03 01 00 89 01 00 00 85 03 03 ba bc af a2 f7 |................| +00000010 a2 a0 19 81 f0 3b c0 76 56 10 e6 95 ce ab 89 82 |.....;.vV.......| +00000020 d5 27 b3 78 69 f2 d3 5b 2d 97 77 00 00 04 c0 14 |.'.xi..[-.w.....| 00000030 00 ff 01 00 00 58 00 0b 00 04 03 00 01 02 00 0a |.....X..........| 00000040 00 0c 00 0a 00 1d 00 17 00 1e 00 19 00 18 00 23 |...............#| 00000050 00 00 00 16 00 00 00 17 00 00 00 0d 00 30 00 2e |.............0..| @@ -54,43 +54,43 @@ 000002a0 fa e7 16 03 03 00 ac 0c 00 00 a8 03 00 1d 20 2f |.............. /| 000002b0 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| 000002c0 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 08 |.........._X.;t.| -000002d0 04 00 80 41 24 c2 f9 e8 40 21 47 3c ab 8e 99 5e |...A$...@!G<...^| -000002e0 0e 08 27 86 6c 29 ae 36 ed 21 18 23 67 cc f7 d5 |..'.l).6.!.#g...| -000002f0 3f e2 2c 48 2f 3d 47 e5 af d5 61 86 0f 91 69 30 |?.,H/=G...a...i0| -00000300 cf 84 56 f2 d3 c1 9a a3 a1 a2 c8 ef 4d 33 de 12 |..V.........M3..| -00000310 d6 46 55 5b c6 6a 65 a5 36 b5 51 5b db 04 25 aa |.FU[.je.6.Q[..%.| -00000320 1c af a0 b0 2d ee db 00 c5 ad 1b 94 d3 90 11 86 |....-...........| -00000330 10 83 35 41 65 9e a4 2c a9 ee 37 ac d4 cc 05 76 |..5Ae..,..7....v| -00000340 92 59 f9 51 68 79 6d 9e 5f eb 80 47 3a 7c e0 74 |.Y.Qhym._..G:|.t| -00000350 ac f5 36 16 03 03 00 04 0e 00 00 00 |..6.........| +000002d0 04 00 80 59 86 4b 13 6a 34 c5 bd 82 a9 ae d8 bf |...Y.K.j4.......| +000002e0 7b 9b f2 c2 0a aa 81 99 25 d8 14 68 32 49 00 ab |{.......%..h2I..| +000002f0 b0 18 4e 05 50 31 0e 25 f3 db 5f 93 45 13 a4 3e |..N.P1.%.._.E..>| +00000300 38 76 a1 0d d8 87 0f 85 81 0c af cb cd e8 43 cd |8v............C.| +00000310 67 01 f2 15 e0 7e 11 44 2a 8d ba 12 33 94 01 c9 |g....~.D*...3...| +00000320 81 bd 99 0d 9f 84 04 a0 7c 0f 24 dd 79 34 53 ba |........|.$.y4S.| +00000330 fa ae 32 16 0c 30 6c f0 76 5d 75 c1 ba d9 35 86 |..2..0l.v]u...5.| +00000340 b0 94 2f 1f 35 7f 1c 1f 92 10 c5 88 55 cc 2c 5b |../.5.......U.,[| +00000350 89 b3 58 16 03 03 00 04 0e 00 00 00 |..X.........| >>> Flow 3 (client to server) -00000000 16 03 03 00 25 10 00 00 21 20 d2 ed 26 ce 1f 5d |....%...! ..&..]| -00000010 50 b8 f4 19 fc 63 e2 b6 3d 7d 39 54 c2 c1 61 a9 |P....c..=}9T..a.| -00000020 2a 82 d8 e3 a9 2f 22 8c b2 18 14 03 03 00 01 01 |*..../".........| -00000030 16 03 03 00 40 82 78 f0 1e e6 03 20 67 66 4e d6 |....@.x.... gfN.| -00000040 93 25 69 9e 38 c6 dd 17 92 02 18 7f 5f 9c 9c f0 |.%i.8......._...| -00000050 a3 f7 45 d3 ba 82 e3 01 38 e5 4f cf 8b 0e 77 6e |..E.....8.O...wn| -00000060 91 99 83 e0 f1 3d e8 a1 39 d4 ea b3 2e 1c 67 59 |.....=..9.....gY| -00000070 c5 5d 83 30 dc |.].0.| +00000000 16 03 03 00 25 10 00 00 21 20 7b e4 41 19 92 fc |....%...! {.A...| +00000010 73 0d 4b 88 2d bd e2 a1 f3 6a ad 5b 8c 10 9b 9e |s.K.-....j.[....| +00000020 46 7a c0 81 96 03 0b 4c 03 7d 14 03 03 00 01 01 |Fz.....L.}......| +00000030 16 03 03 00 40 e5 e4 11 bd 7c 54 b8 be 80 44 82 |....@....|T...D.| +00000040 03 22 51 7f f5 de 92 20 7d 34 b0 9a 7b 17 ce 12 |."Q.... }4..{...| +00000050 b4 75 44 9b 37 b1 cd 1f 0c f8 86 4c 75 d9 1e 75 |.uD.7......Lu..u| +00000060 d3 7e bf 6e 3e 9d be 3b c1 47 11 6d f1 09 10 2e |.~.n>..;.G.m....| +00000070 d0 0d cd 1f 26 |....&| >>> Flow 4 (server to client) -00000000 16 03 03 00 83 04 00 00 7f 00 00 00 00 00 79 00 |..............y.| +00000000 16 03 03 00 85 04 00 00 81 00 00 00 00 00 7b 00 |..............{.| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 94 |................| -00000020 6f 2d 70 97 51 ed 14 ef 68 ca 42 c5 4c ff 6b a2 |o-p.Q...h.B.L.k.| -00000030 3e f9 07 a9 91 ad 0d c7 23 bd 7f 04 cf 4d a0 eb |>.......#....M..| -00000040 58 e0 e1 37 73 d3 cc 4b e2 7f 6d 3a 2e 47 b5 b4 |X..7s..K..m:.G..| -00000050 60 dd e6 9a ea 30 1e 6e 7a e7 8e 84 ca 49 38 16 |`....0.nz....I8.| -00000060 7e 51 5c e5 15 c0 58 7d a2 ba e2 ca 90 24 11 ea |~Q\...X}.....$..| -00000070 53 9c 7d cb 47 13 91 cf f6 05 f0 2f db 57 1a 40 |S.}.G....../.W.@| -00000080 57 b0 d4 97 8e 23 7e f5 14 03 03 00 01 01 16 03 |W....#~.........| -00000090 03 00 40 00 00 00 00 00 00 00 00 00 00 00 00 00 |..@.............| -000000a0 00 00 00 72 43 0e f2 f3 bb e8 6d 3b f2 ff 96 53 |...rC.....m;...S| -000000b0 12 36 07 e0 f0 17 35 e7 52 87 a3 12 7b 53 d4 83 |.6....5.R...{S..| -000000c0 cc d2 d3 06 4b e2 3a fc 38 4f a7 75 d8 3c 6a a4 |....K.:.8O.u..u.L| +00000040 72 93 e7 50 f9 17 7b 90 0d 70 e7 b8 5c 7b 93 81 |r..P..{..p..\{..| +00000050 66 e1 64 b0 a9 25 34 84 bc 49 42 f5 13 49 38 16 |f.d..%4..IB..I8.| +00000060 7e 51 5c e5 15 c0 58 7d 52 07 da 2f 56 e0 05 62 |~Q\...X}R../V..b| +00000070 16 1e 0c 62 76 cc b5 ff 25 a1 c3 2e 1f 28 71 29 |...bv...%....(q)| +00000080 9c d0 c1 0a 05 dd 22 57 7a 19 14 03 03 00 01 01 |......"Wz.......| +00000090 16 03 03 00 40 00 00 00 00 00 00 00 00 00 00 00 |....@...........| +000000a0 00 00 00 00 00 9a 9d 01 bc db e5 21 c2 d2 f8 d8 |...........!....| +000000b0 b8 c1 86 87 fc b6 79 df 69 be d2 97 cd 69 76 9d |......y.i....iv.| +000000c0 04 95 7f d4 e8 c7 78 52 c0 d1 ac bc 55 08 57 a7 |......xR....U.W.| +000000d0 9c 28 d4 7b df 17 03 03 00 40 00 00 00 00 00 00 |.(.{.....@......| +000000e0 00 00 00 00 00 00 00 00 00 00 81 f0 80 62 50 04 |.............bP.| +000000f0 7f 86 ee f1 73 46 b0 c3 c1 0d 92 ab dd 4f b9 2a |....sF.......O.*| +00000100 58 4f 17 9f be 60 ff 8b 1a d6 e3 94 aa dc 8b 60 |XO...`.........`| +00000110 d8 2b 4c c8 5a 69 18 74 65 49 15 03 03 00 30 00 |.+L.Zi.teI....0.| +00000120 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 18 |................| +00000130 36 4f 05 fe ab 47 92 6e 48 42 4f 06 c4 f3 e1 70 |6O...G.nHBO....p| +00000140 c6 66 00 1d aa 84 6b 2b d4 23 37 c9 42 fb 6d |.f....k+.#7.B.m| diff --git a/src/crypto/tls/testdata/Server-TLSv12-IssueTicket b/src/crypto/tls/testdata/Server-TLSv12-IssueTicket index ee293586102610..b05669d073b047 100644 --- a/src/crypto/tls/testdata/Server-TLSv12-IssueTicket +++ b/src/crypto/tls/testdata/Server-TLSv12-IssueTicket @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 71 01 00 00 6d 03 03 bf f8 80 5d 1b |....q...m.....].| -00000010 ea 95 cb 32 3b 8f ff 5e f9 4d 58 7d dc a4 50 cc |...2;..^.MX}..P.| -00000020 68 4d 40 98 11 af f3 e4 d7 31 43 00 00 04 00 2f |hM@......1C..../| +00000000 16 03 01 00 71 01 00 00 6d 03 03 ae 71 d4 07 74 |....q...m...q..t| +00000010 03 93 b0 1f 88 72 ef d2 54 61 44 34 5f 3f ea 16 |.....r..TaD4_?..| +00000020 32 41 11 a9 00 9b 59 ba 50 a8 ab 00 00 04 00 2f |2A....Y.P....../| 00000030 00 ff 01 00 00 40 00 23 00 00 00 16 00 00 00 17 |.....@.#........| 00000040 00 00 00 0d 00 30 00 2e 04 03 05 03 06 03 08 07 |.....0..........| 00000050 08 08 08 09 08 0a 08 0b 08 04 08 05 08 06 04 01 |................| @@ -52,39 +52,39 @@ 00000290 fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 16 03 03 00 |.=.`.\!.;.......| 000002a0 04 0e 00 00 00 |.....| >>> Flow 3 (client to server) -00000000 16 03 03 00 86 10 00 00 82 00 80 35 d7 dd 48 68 |...........5..Hh| -00000010 17 8c 9b 02 75 0c 6b 76 17 0a e1 f8 06 c0 d7 1d |....u.kv........| -00000020 bf a5 97 d2 59 0f c9 90 4a f0 6f 40 dc e7 30 c4 |....Y...J.o@..0.| -00000030 75 ab 74 9c 32 48 a4 84 3c 40 a0 bd 03 aa 09 2b |u.t.2H..<@.....+| -00000040 b2 4c 80 82 17 b8 3d 27 16 9a b7 90 66 f2 10 4e |.L....='....f..N| -00000050 41 7e 78 24 de 27 91 f9 e9 bc bf 15 3a 35 1b ae |A~x$.'......:5..| -00000060 28 9e e1 09 f0 7a 4d 66 7e de d1 43 bf f5 e4 09 |(....zMf~..C....| -00000070 a7 21 cb 0e 1d 59 6d a0 a6 41 44 58 f4 ab ac 6a |.!...Ym..ADX...j| -00000080 98 db 25 e3 57 ee 94 87 85 51 ea 14 03 03 00 01 |..%.W....Q......| -00000090 01 16 03 03 00 40 c9 64 79 e7 15 1d 30 15 95 89 |.....@.dy...0...| -000000a0 b1 9b 12 42 69 4b 22 20 54 5a aa b6 71 02 1c 3f |...BiK" TZ..q..?| -000000b0 7c b5 66 07 b5 1f 55 96 3f ce 47 1f 66 52 d8 6b ||.f...U.?.G.fR.k| -000000c0 65 71 c0 4e 0b 7e 55 e0 f5 af 42 29 af 2b 1d 0e |eq.N.~U...B).+..| -000000d0 e6 96 cd 7b fc d3 |...{..| +00000000 16 03 03 00 86 10 00 00 82 00 80 1a 4a c4 37 68 |............J.7h| +00000010 6a c0 28 f1 ea ea 14 20 fe c3 b6 61 28 67 75 7b |j.(.... ...a(gu{| +00000020 74 e0 6b ab 2c dd c8 13 0a be d6 71 a1 13 96 5a |t.k.,......q...Z| +00000030 bf 89 2e 6e 6a 61 24 ca d4 88 3f f8 20 ed 20 1f |...nja$...?. . .| +00000040 0a 9c 11 9b 96 e1 cd d8 38 42 05 be b8 6e e5 fe |........8B...n..| +00000050 54 c8 93 b8 56 67 01 97 a0 bc 37 33 7c 40 f7 77 |T...Vg....73|@.w| +00000060 5d 8b 63 bc 3f 7b e3 e9 0a b0 13 06 12 6e 8e 1c |].c.?{.......n..| +00000070 7c e7 ed 99 6d c3 5a 93 92 d2 4a fe fa d9 10 1c ||...m.Z...J.....| +00000080 76 e2 9e d7 d4 cd c7 b9 7a 40 54 14 03 03 00 01 |v.......z@T.....| +00000090 01 16 03 03 00 40 ac 11 71 60 db b6 db b9 db fb |.....@..q`......| +000000a0 09 20 8d 00 e9 69 25 15 f9 14 8f 08 7a 6c 8c 29 |. ...i%.....zl.)| +000000b0 0d f4 9b d3 ca c8 c8 f3 11 0a 85 d6 c2 cc 60 a7 |..............`.| +000000c0 8c a3 32 06 08 15 bd 84 a0 4f 17 b9 6b 9b 6a 7c |..2......O..k.j|| +000000d0 23 9d 74 22 7f fd |#.t"..| >>> Flow 4 (server to client) -00000000 16 03 03 00 83 04 00 00 7f 00 00 00 00 00 79 00 |..............y.| +00000000 16 03 03 00 85 04 00 00 81 00 00 00 00 00 7b 00 |..............{.| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 94 |................| -00000020 6f 2d b0 ac 51 ed 14 ef 68 ca 42 c5 4c 85 f6 26 |o-..Q...h.B.L..&| -00000030 0d a4 ad a8 f5 14 64 4f b9 c3 fb 1e 55 c1 1f c7 |......dO....U...| -00000040 31 57 72 68 db 03 37 a8 c9 07 f4 ca 62 6c 5c f3 |1Wrh..7.....bl\.| -00000050 8b 5a 3d 76 dd 63 ea 68 61 6b a1 2d 95 49 38 16 |.Z=v.c.hak.-.I8.| -00000060 7e 51 5c e5 15 c0 58 7d c5 67 4a 6f 64 b6 79 1a |~Q\...X}.gJod.y.| -00000070 41 9b b1 33 15 38 74 92 5c a5 48 c3 f2 94 bb 33 |A..3.8t.\.H....3| -00000080 ec af cf d7 e7 c9 3e 35 14 03 03 00 01 01 16 03 |......>5........| -00000090 03 00 40 00 00 00 00 00 00 00 00 00 00 00 00 00 |..@.............| -000000a0 00 00 00 19 51 7c 1c a3 80 34 e1 81 30 3f f9 a4 |....Q|...4..0?..| -000000b0 a0 97 97 fd 94 fb ab e8 80 48 25 7b 83 ca 38 61 |.........H%{..8a| -000000c0 34 95 d0 52 6f 09 ad 4f 74 35 c5 3d e8 bb aa 5d |4..Ro..Ot5.=...]| -000000d0 d0 fc 85 17 03 03 00 40 00 00 00 00 00 00 00 00 |.......@........| -000000e0 00 00 00 00 00 00 00 00 e7 19 f9 fd 10 7c 17 04 |.............|..| -000000f0 2d ce 5f a6 41 33 3d 05 b0 29 91 ff a0 a5 76 52 |-._.A3=..)....vR| -00000100 e1 b9 ba 6a ca d3 79 60 11 ac 43 b5 30 f7 15 dc |...j..y`..C.0...| -00000110 6f b1 d2 b2 00 85 43 40 15 03 03 00 30 00 00 00 |o.....C@....0...| -00000120 00 00 00 00 00 00 00 00 00 00 00 00 00 19 99 8a |................| -00000130 4c 18 e0 03 cc 27 7a be 2c e5 d2 16 95 f6 a4 6e |L....'z.,......n| -00000140 11 d3 1d f4 01 52 2b fc 98 04 b1 0b 31 |.....R+.....1| +00000020 6f 2d b0 ac 51 ed 14 ef 68 ca 42 c5 4c 52 2e 96 |o-..Q...h.B.LR..| +00000030 6b e5 cc b4 0c ee 82 5b c1 57 52 8e dd 26 c8 58 |k......[.WR..&.X| +00000040 27 01 5f ec 58 a0 5c ad 74 e8 82 b7 ab 86 71 25 |'._.X.\.t.....q%| +00000050 aa ed ec ef 69 5f 7e 1d f2 58 30 13 75 49 38 16 |....i_~..X0.uI8.| +00000060 7e 51 5c e5 15 c0 58 7d 52 1a 43 47 27 99 9d 0f |~Q\...X}R.CG'...| +00000070 e0 4c f4 3b e0 b0 76 ae e6 5d a4 a0 34 38 8b b0 |.L.;..v..]..48..| +00000080 3a ba 26 90 a3 dd c2 dc 26 98 14 03 03 00 01 01 |:.&.....&.......| +00000090 16 03 03 00 40 00 00 00 00 00 00 00 00 00 00 00 |....@...........| +000000a0 00 00 00 00 00 55 e7 9a 32 83 32 d1 01 6e 0e 5a |.....U..2.2..n.Z| +000000b0 dd 72 c5 f9 90 24 6b da 73 d9 ed 39 b8 d5 f6 93 |.r...$k.s..9....| +000000c0 e0 f0 3e 20 db d0 0c 8a b0 10 78 1e 08 fb 47 44 |..> ......x...GD| +000000d0 27 74 30 c4 73 17 03 03 00 40 00 00 00 00 00 00 |'t0.s....@......| +000000e0 00 00 00 00 00 00 00 00 00 00 0a ae 7e 4e c4 37 |............~N.7| +000000f0 65 b9 bf 66 a5 c5 5d a2 bc 76 7a db c9 cd 0b 85 |e..f..]..vz.....| +00000100 de 86 e4 94 2c f2 87 1f 7a 4b 31 b7 4d 77 18 8a |....,...zK1.Mw..| +00000110 6d 5e af d0 eb 87 bf d9 c5 e3 15 03 03 00 30 00 |m^............0.| +00000120 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 9f |................| +00000130 8a 39 93 10 94 78 d1 dc 81 07 b6 1a 1a c3 96 c1 |.9...x..........| +00000140 28 d3 30 eb 3d 1a d3 d6 d8 3a d2 33 ec ed 6c |(.0.=....:.3..l| diff --git a/src/crypto/tls/testdata/Server-TLSv12-IssueTicketPreDisable b/src/crypto/tls/testdata/Server-TLSv12-IssueTicketPreDisable index 20ce3c3cc8c75b..a86d19c35a8178 100644 --- a/src/crypto/tls/testdata/Server-TLSv12-IssueTicketPreDisable +++ b/src/crypto/tls/testdata/Server-TLSv12-IssueTicketPreDisable @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 71 01 00 00 6d 03 03 a1 ba 69 29 39 |....q...m....i)9| -00000010 b5 fc c7 90 90 54 35 be 5a ad 4a e2 b2 3d b9 01 |.....T5.Z.J..=..| -00000020 f0 48 fd 77 b5 9e bc 89 f5 d4 df 00 00 04 00 2f |.H.w.........../| +00000000 16 03 01 00 71 01 00 00 6d 03 03 9d 49 68 d2 e5 |....q...m...Ih..| +00000010 4a 77 05 39 fb b6 c2 e8 79 b8 cd e7 42 dd f5 29 |Jw.9....y...B..)| +00000020 4f 24 92 27 bf 94 95 89 8a 0c 5f 00 00 04 00 2f |O$.'......_..../| 00000030 00 ff 01 00 00 40 00 23 00 00 00 16 00 00 00 17 |.....@.#........| 00000040 00 00 00 0d 00 30 00 2e 04 03 05 03 06 03 08 07 |.....0..........| 00000050 08 08 08 09 08 0a 08 0b 08 04 08 05 08 06 04 01 |................| @@ -52,39 +52,39 @@ 00000290 fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 16 03 03 00 |.=.`.\!.;.......| 000002a0 04 0e 00 00 00 |.....| >>> Flow 3 (client to server) -00000000 16 03 03 00 86 10 00 00 82 00 80 cc 4e 62 e0 bf |............Nb..| -00000010 21 65 dc f2 57 fb fe 6e e4 cc de b4 1f aa af 52 |!e..W..n.......R| -00000020 a4 e0 0e 1b fa 8b 23 4b d9 e2 4b 62 6a 26 80 f7 |......#K..Kbj&..| -00000030 15 82 ba 44 4a 18 b8 97 ca a1 79 4b 11 59 90 7d |...DJ.....yK.Y.}| -00000040 ea 89 7c f9 6b 5f 29 c7 ca 32 bf 3b 53 b2 bb bb |..|.k_)..2.;S...| -00000050 77 0a 5c 1f c2 d8 20 cf 59 19 4e a0 ff ef ca ca |w.\... .Y.N.....| -00000060 25 39 ac c7 64 b9 e8 68 09 f2 49 96 8e 49 c7 4c |%9..d..h..I..I.L| -00000070 cd ff 28 6f d8 0d d3 7a ae 7a 51 9e 04 70 8a 59 |..(o...z.zQ..p.Y| -00000080 8c 05 61 c9 2b bd e1 05 5a 12 63 14 03 03 00 01 |..a.+...Z.c.....| -00000090 01 16 03 03 00 40 ac 30 bb 83 2f e2 a1 98 a3 c5 |.....@.0../.....| -000000a0 9c e3 55 36 70 a0 10 fc 53 7e 2d ae f1 02 d7 04 |..U6p...S~-.....| -000000b0 1f 4e 5d ed 33 29 99 04 54 8e 51 74 d5 2a 73 21 |.N].3)..T.Qt.*s!| -000000c0 4f bf 8b 0c 04 b2 f2 d4 3e a7 f6 ee 8b fb 3a 0b |O.......>.....:.| -000000d0 86 27 7c a7 bb 32 |.'|..2| +00000000 16 03 03 00 86 10 00 00 82 00 80 2b 09 f3 6c 39 |...........+..l9| +00000010 25 51 ab 73 c9 5d e4 de bb b3 7d 8e 40 96 df f9 |%Q.s.]....}.@...| +00000020 28 24 82 47 21 40 2d cb bd 7d a5 2a 89 91 7d e0 |($.G!@-..}.*..}.| +00000030 4c 92 ca 8f 2c 2d be 93 d1 a2 00 ef 3a 4b 6b de |L...,-......:Kk.| +00000040 e9 ab 38 0b 19 21 35 5d fb 06 b1 1f dd 75 db d6 |..8..!5].....u..| +00000050 6b 16 7e 1e 32 ef 58 11 78 ef 6e 7f cc 1f cd 8e |k.~.2.X.x.n.....| +00000060 57 01 96 eb 06 bf 09 10 99 ed 3c 35 94 9f 03 66 |W.........<5...f| +00000070 a4 e1 96 22 eb f5 cd 28 1f 4e 2c b9 2c 48 29 bf |..."...(.N,.,H).| +00000080 b3 43 c1 b9 f0 aa 2b 29 47 a4 38 14 03 03 00 01 |.C....+)G.8.....| +00000090 01 16 03 03 00 40 75 ac e5 40 a2 19 82 90 ef 25 |.....@u..@.....%| +000000a0 9c 3b c2 95 fb 58 b9 c8 72 2c b3 94 d5 23 e9 f6 |.;...X..r,...#..| +000000b0 0d 03 2e 24 54 73 c3 5b 0d 84 2d 5b 12 f6 f9 5a |...$Ts.[..-[...Z| +000000c0 59 6d ea 80 5e b6 ab 34 5f 57 98 fb 2c c8 e2 d0 |Ym..^..4_W..,...| +000000d0 3e fb 32 4c b1 93 |>.2L..| >>> Flow 4 (server to client) -00000000 16 03 03 00 83 04 00 00 7f 00 00 00 00 00 79 00 |..............y.| +00000000 16 03 03 00 85 04 00 00 81 00 00 00 00 00 7b 00 |..............{.| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 94 |................| -00000020 6f 2d b0 ac 51 ed 14 ef 68 ca 42 c5 4c fa 53 68 |o-..Q...h.B.L.Sh| -00000030 d9 20 e9 d9 c1 9c 90 3b f2 e6 57 af 04 e5 db 6b |. .....;..W....k| -00000040 36 0b b5 b8 e2 a5 a4 bf 52 31 80 32 b9 da d9 32 |6.......R1.2...2| -00000050 36 e7 31 d3 22 78 12 ae 7a 80 ac fa 6d 49 38 16 |6.1."x..z...mI8.| -00000060 7e 51 5c e5 15 c0 58 7d d6 77 d5 17 1b d9 a8 74 |~Q\...X}.w.....t| -00000070 be 93 25 54 84 a7 1a 93 1f 20 a4 49 eb 26 e7 8e |..%T..... .I.&..| -00000080 d3 0f cf 9c 75 cc 6f 36 14 03 03 00 01 01 16 03 |....u.o6........| -00000090 03 00 40 00 00 00 00 00 00 00 00 00 00 00 00 00 |..@.............| -000000a0 00 00 00 eb f1 52 79 ff b8 6c 9c 23 f3 22 bc 96 |.....Ry..l.#."..| -000000b0 2a bc e7 73 05 32 4b 2e e4 5e 31 97 62 86 cc 12 |*..s.2K..^1.b...| -000000c0 ae 22 77 92 37 5d 82 41 57 48 aa f4 0a f3 94 30 |."w.7].AWH.....0| -000000d0 5d 06 7b 17 03 03 00 40 00 00 00 00 00 00 00 00 |].{....@........| -000000e0 00 00 00 00 00 00 00 00 8f 8d a7 06 a7 d6 52 5a |..............RZ| -000000f0 b9 66 5e ef e3 8d 1d 91 d0 6d 30 29 92 4e 6a 81 |.f^......m0).Nj.| -00000100 f4 77 97 06 de a8 c8 d1 4c 6b 15 07 1f 9b 59 6d |.w......Lk....Ym| -00000110 cb 4f 23 20 58 aa 22 21 15 03 03 00 30 00 00 00 |.O# X."!....0...| -00000120 00 00 00 00 00 00 00 00 00 00 00 00 00 06 55 3d |..............U=| -00000130 42 f5 12 b2 66 aa af 00 91 5e b8 31 ae 19 0e 35 |B...f....^.1...5| -00000140 a2 d7 a6 e7 0c 3c 2b 95 62 69 d7 a0 81 |.....<+.bi...| +00000020 6f 2d b0 ac 51 ed 14 ef 68 ca 42 c5 4c ba fa 12 |o-..Q...h.B.L...| +00000030 47 15 bc 82 f4 35 e2 0f 0f 2b d3 02 30 a7 c3 bb |G....5...+..0...| +00000040 48 06 b7 80 c5 21 70 95 bf fd e5 fa d8 aa ee 9c |H....!p.........| +00000050 d6 10 79 3b f9 e6 9e 21 21 bd e9 50 ba 49 38 16 |..y;...!!..P.I8.| +00000060 7e 51 5c e5 15 c0 58 7d 52 1a 2e ee ae e1 df 6c |~Q\...X}R......l| +00000070 d2 82 e8 11 4b d9 3a b4 ed 46 56 6a 8e cd 4b 70 |....K.:..FVj..Kp| +00000080 37 5c 82 fc ef d7 ff 38 65 e6 14 03 03 00 01 01 |7\.....8e.......| +00000090 16 03 03 00 40 00 00 00 00 00 00 00 00 00 00 00 |....@...........| +000000a0 00 00 00 00 00 8a 27 b1 df e3 23 8d 5e a8 06 e0 |......'...#.^...| +000000b0 3b 70 3a 8a f2 36 09 e5 0b 91 03 0c ab c7 6b 2c |;p:..6........k,| +000000c0 ee 8d fd b3 d1 3b 0b d6 08 45 af 26 40 ad c7 fd |.....;...E.&@...| +000000d0 8f fa fa f6 c0 17 03 03 00 40 00 00 00 00 00 00 |.........@......| +000000e0 00 00 00 00 00 00 00 00 00 00 c0 62 b3 b3 3d 17 |...........b..=.| +000000f0 5d 7f a6 7d 7b eb ea 35 f5 46 3c 69 94 a0 37 ca |]..}{..5.F>> Flow 1 (client to server) -00000000 16 03 01 01 0a 01 00 01 06 03 03 2d b6 ca ea 39 |...........-...9| -00000010 59 17 86 df 90 2f 73 e0 a0 5c 6e 28 09 78 69 d6 |Y..../s..\n(.xi.| -00000020 30 06 b7 7b 17 a9 79 30 2a d8 57 20 c5 5c ed 86 |0..{..y0*.W .\..| -00000030 15 f4 3b c8 d2 5f 7a 80 2a 6a cd 40 c2 da 6f a8 |..;.._z.*j.@..o.| -00000040 cd d7 e7 bf 48 bd fb a1 e9 4b 9b a9 00 04 00 2f |....H....K...../| -00000050 00 ff 01 00 00 b9 00 23 00 79 00 00 00 00 00 00 |.......#.y......| +00000000 16 03 01 01 0c 01 00 01 08 03 03 a6 8e 75 2d a5 |.............u-.| +00000010 52 ef 15 c3 e3 42 53 88 55 21 76 a9 8c 44 e8 df |R....BS.U!v..D..| +00000020 f4 1c 40 08 1a 35 46 4b 6d 4f ff 20 e3 91 5e 82 |..@..5FKmO. ..^.| +00000030 58 5e 5a 37 a7 16 d8 fe 96 0e 2c 45 13 ac 4f 30 |X^Z7......,E..O0| +00000040 96 a3 17 a2 16 49 00 d6 8e b6 6a 4d 00 04 00 2f |.....I....jM.../| +00000050 00 ff 01 00 00 bb 00 23 00 7b 00 00 00 00 00 00 |.......#.{......| 00000060 00 00 00 00 00 00 00 00 00 00 94 6f 2d b0 ac 51 |...........o-..Q| -00000070 ed 14 ef 68 ca 42 c5 4c 85 f6 26 0d a4 ad a8 f5 |...h.B.L..&.....| -00000080 14 64 4f b9 c3 fb 1e 55 c1 1f c7 31 57 72 68 db |.dO....U...1Wrh.| -00000090 03 37 a8 c9 07 f4 ca 62 6c 5c f3 8b 5a 3d 76 dd |.7.....bl\..Z=v.| -000000a0 63 ea 68 61 6b a1 2d 95 49 38 16 7e 51 5c e5 15 |c.hak.-.I8.~Q\..| -000000b0 c0 58 7d c5 67 4a 6f 64 b6 79 1a 41 9b b1 33 15 |.X}.gJod.y.A..3.| -000000c0 38 74 92 5c a5 48 c3 f2 94 bb 33 ec af cf d7 e7 |8t.\.H....3.....| -000000d0 c9 3e 35 00 16 00 00 00 17 00 00 00 0d 00 30 00 |.>5...........0.| -000000e0 2e 04 03 05 03 06 03 08 07 08 08 08 09 08 0a 08 |................| -000000f0 0b 08 04 08 05 08 06 04 01 05 01 06 01 03 03 02 |................| -00000100 03 03 01 02 01 03 02 02 02 04 02 05 02 06 02 |...............| +00000070 ed 14 ef 68 ca 42 c5 4c 52 2e 96 6b e5 cc b4 0c |...h.B.LR..k....| +00000080 ee 82 5b c1 57 52 8e dd 26 c8 58 27 01 5f ec 58 |..[.WR..&.X'._.X| +00000090 a0 5c ad 74 e8 82 b7 ab 86 71 25 aa ed ec ef 69 |.\.t.....q%....i| +000000a0 5f 7e 1d f2 58 30 13 75 49 38 16 7e 51 5c e5 15 |_~..X0.uI8.~Q\..| +000000b0 c0 58 7d 52 1a 43 47 27 99 9d 0f e0 4c f4 3b e0 |.X}R.CG'....L.;.| +000000c0 b0 76 ae e6 5d a4 a0 34 38 8b b0 3a ba 26 90 a3 |.v..]..48..:.&..| +000000d0 dd c2 dc 26 98 00 16 00 00 00 17 00 00 00 0d 00 |...&............| +000000e0 30 00 2e 04 03 05 03 06 03 08 07 08 08 08 09 08 |0...............| +000000f0 0a 08 0b 08 04 08 05 08 06 04 01 05 01 06 01 03 |................| +00000100 03 02 03 03 01 02 01 03 02 02 02 04 02 05 02 06 |................| +00000110 02 |.| >>> Flow 2 (server to client) 00000000 16 03 03 00 59 02 00 00 55 03 03 00 00 00 00 00 |....Y...U.......| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -00000020 00 00 00 44 4f 57 4e 47 52 44 01 20 c5 5c ed 86 |...DOWNGRD. .\..| -00000030 15 f4 3b c8 d2 5f 7a 80 2a 6a cd 40 c2 da 6f a8 |..;.._z.*j.@..o.| -00000040 cd d7 e7 bf 48 bd fb a1 e9 4b 9b a9 00 2f 00 00 |....H....K.../..| +00000020 00 00 00 44 4f 57 4e 47 52 44 01 20 e3 91 5e 82 |...DOWNGRD. ..^.| +00000030 58 5e 5a 37 a7 16 d8 fe 96 0e 2c 45 13 ac 4f 30 |X^Z7......,E..O0| +00000040 96 a3 17 a2 16 49 00 d6 8e b6 6a 4d 00 2f 00 00 |.....I....jM./..| 00000050 0d 00 23 00 00 ff 01 00 01 00 00 17 00 00 16 03 |..#.............| -00000060 03 00 83 04 00 00 7f 00 00 00 00 00 79 00 00 00 |............y...| +00000060 03 00 85 04 00 00 81 00 00 00 00 00 7b 00 00 00 |............{...| 00000070 00 00 00 00 00 00 00 00 00 00 00 00 00 94 6f 2d |..............o-| -00000080 b0 ac 51 ed 14 ef 68 ca 42 c5 4c 85 f6 26 0d a4 |..Q...h.B.L..&..| -00000090 ad a8 f5 14 64 4f b9 c3 fb 1e 55 c1 1f c7 31 57 |....dO....U...1W| -000000a0 72 68 db 03 37 a8 c9 07 f4 ca 62 6c 5c f3 8b 5a |rh..7.....bl\..Z| -000000b0 3d 76 dd 63 ea 68 61 6b a1 2d 95 49 38 16 7e 51 |=v.c.hak.-.I8.~Q| -000000c0 5c e5 15 c0 58 7d c5 67 4a 6f 64 b6 79 1a 41 9b |\...X}.gJod.y.A.| -000000d0 b1 33 15 38 74 92 5c a5 48 c3 f2 94 bb 33 ec af |.3.8t.\.H....3..| -000000e0 cf d7 e7 c9 3e 35 14 03 03 00 01 01 16 03 03 00 |....>5..........| -000000f0 40 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |@...............| -00000100 00 47 68 3a 66 5b d6 ed b7 60 a9 fb e8 37 d6 9d |.Gh:f[...`...7..| -00000110 a6 b9 4d d5 f3 9f 0f c6 3c 21 6e d5 80 08 a8 34 |..M.......k| -00000130 40 |@| +00000080 b0 ac 51 ed 14 ef 68 ca 42 c5 4c 52 2e 96 6b e5 |..Q...h.B.LR..k.| +00000090 cc b4 0c ee 82 5b c1 57 52 8e dd 26 c8 58 27 01 |.....[.WR..&.X'.| +000000a0 5f ec 58 a0 5c ad 74 e8 82 b7 ab 86 71 25 aa ed |_.X.\.t.....q%..| +000000b0 ec ef 69 5f 7e 1d f2 58 30 13 75 49 38 16 7e 51 |..i_~..X0.uI8.~Q| +000000c0 5c e5 15 c0 58 7d 52 1a 43 47 27 99 9d 0f e0 4c |\...X}R.CG'....L| +000000d0 f4 3b e0 b0 76 ae e6 5d a4 a0 34 38 8b b0 3a ba |.;..v..]..48..:.| +000000e0 26 90 a3 dd c2 dc 26 98 14 03 03 00 01 01 16 03 |&.....&.........| +000000f0 03 00 40 00 00 00 00 00 00 00 00 00 00 00 00 00 |..@.............| +00000100 00 00 00 fb 6c 2b 7d a1 a6 a1 aa 6f 38 3f e2 8a |....l+}....o8?..| +00000110 09 da 48 94 ce 2a 70 70 8c d3 5d bd 8c e5 74 fc |..H..*pp..]...t.| +00000120 91 05 dc 9e f1 2d a6 db 3c d6 06 50 b7 9d 4d 8b |.....-..<..P..M.| +00000130 b7 d4 06 |...| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 16 03 03 00 40 46 34 b3 97 54 |..........@F4..T| -00000010 20 5b 95 f3 22 f8 a1 89 c8 95 93 ba 7b a4 a8 8f | [..".......{...| -00000020 46 a8 d6 c1 b3 ac f0 e0 49 3d 8d e4 1c ac b8 a4 |F.......I=......| -00000030 01 21 5e d8 f0 f5 10 10 f7 de 8b 33 9d 94 cf f6 |.!^........3....| -00000040 f2 9b 39 22 5c e6 c0 5e b4 1d cd |..9"\..^...| +00000000 14 03 03 00 01 01 16 03 03 00 40 b0 47 45 3b 24 |..........@.GE;$| +00000010 ae 4b 98 d6 cc 37 28 ab 3b ea 7e 6b bc 1f ed a5 |.K...7(.;.~k....| +00000020 bb 67 e2 5e 72 bf d5 28 90 dc 1b 98 87 2c 49 c6 |.g.^r..(.....,I.| +00000030 90 73 45 3f 1b 8c a2 c5 50 84 48 09 41 e1 ea 52 |.sE?....P.H.A..R| +00000040 9e 17 ad 8f d6 cd cd 16 7a 90 64 |........z.d| >>> Flow 4 (server to client) 00000000 17 03 03 00 40 00 00 00 00 00 00 00 00 00 00 00 |....@...........| -00000010 00 00 00 00 00 5c bc 45 06 2e d3 7b 30 99 a6 af |.....\.E...{0...| -00000020 64 0e 63 93 73 6f 0a e7 a4 1d ac 94 25 11 a5 63 |d.c.so......%..c| -00000030 8d b2 44 aa 98 44 f8 b5 51 ea 2c fb 26 99 f6 a4 |..D..D..Q.,.&...| -00000040 2c f8 15 c3 90 15 03 03 00 30 00 00 00 00 00 00 |,........0......| -00000050 00 00 00 00 00 00 00 00 00 00 c6 58 8e 7c 97 de |...........X.|..| -00000060 3b b8 39 cd 7b 1d 67 77 27 da 93 39 52 a7 81 9b |;.9.{.gw'..9R...| -00000070 ab 5a bc e9 00 1a 64 3a ca f5 |.Z....d:..| +00000010 00 00 00 00 00 a6 e7 61 5a 2b 01 ce 94 48 d2 09 |.......aZ+...H..| +00000020 13 a5 90 a8 58 47 8d fd ab 69 85 fa 42 00 3d 99 |....XG...i..B.=.| +00000030 d1 fd 16 88 0b 2a 43 92 54 e7 55 2e dd 70 86 e1 |.....*C.T.U..p..| +00000040 f1 c5 5c c8 1e 15 03 03 00 30 00 00 00 00 00 00 |..\......0......| +00000050 00 00 00 00 00 00 00 00 00 00 ba 42 cf 74 9f 00 |...........B.t..| +00000060 4a 76 d5 cf 48 23 ed 4a 42 62 be 1a 35 d2 8c d2 |Jv..H#.JBb..5...| +00000070 ee 86 2f 3a f5 4d 96 64 f7 b8 |../:.M.d..| diff --git a/src/crypto/tls/testdata/Server-TLSv12-SNI b/src/crypto/tls/testdata/Server-TLSv12-SNI index c3dc1b6e52d9d2..b7889def656bd4 100644 --- a/src/crypto/tls/testdata/Server-TLSv12-SNI +++ b/src/crypto/tls/testdata/Server-TLSv12-SNI @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 81 01 00 00 7d 03 03 cc a1 7b c4 56 |........}....{.V| -00000010 9f 65 31 01 b0 23 09 18 10 50 7c 1e 14 7b b5 dd |.e1..#...P|..{..| -00000020 d4 70 07 3e 0b 19 19 31 6b f7 4d 00 00 04 00 2f |.p.>...1k.M..../| +00000000 16 03 01 00 81 01 00 00 7d 03 03 5b 8e 50 b3 0e |........}..[.P..| +00000010 1f d7 4f a4 0f 53 de 37 1a c6 86 2f 01 c0 0b 15 |..O..S.7.../....| +00000020 cf a9 f4 f5 30 c0 aa 7e cc 5b 4e 00 00 04 00 2f |....0..~.[N..../| 00000030 00 ff 01 00 00 50 00 00 00 10 00 0e 00 00 0b 73 |.....P.........s| 00000040 6e 69 74 65 73 74 2e 63 6f 6d 00 16 00 00 00 17 |nitest.com......| 00000050 00 00 00 0d 00 30 00 2e 04 03 05 03 06 03 08 07 |.....0..........| @@ -9,75 +9,75 @@ 00000070 05 01 06 01 03 03 02 03 03 01 02 01 03 02 02 02 |................| 00000080 04 02 05 02 06 02 |......| >>> Flow 2 (server to client) -00000000 16 03 03 00 35 02 00 00 31 03 03 00 00 00 00 00 |....5...1.......| +00000000 16 03 03 00 39 02 00 00 35 03 03 00 00 00 00 00 |....9...5.......| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 44 4f 57 4e 47 52 44 01 00 00 2f 00 00 |...DOWNGRD.../..| -00000030 09 ff 01 00 01 00 00 17 00 00 16 03 03 02 59 0b |..............Y.| -00000040 00 02 55 00 02 52 00 02 4f 30 82 02 4b 30 82 01 |..U..R..O0..K0..| -00000050 b4 a0 03 02 01 02 02 09 00 e8 f0 9d 3f e2 5b ea |............?.[.| -00000060 a6 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05 00 |.0...*.H........| -00000070 30 1f 31 0b 30 09 06 03 55 04 0a 13 02 47 6f 31 |0.1.0...U....Go1| -00000080 10 30 0e 06 03 55 04 03 13 07 47 6f 20 52 6f 6f |.0...U....Go Roo| -00000090 74 30 1e 17 0d 31 36 30 31 30 31 30 30 30 30 30 |t0...16010100000| -000000a0 30 5a 17 0d 32 35 30 31 30 31 30 30 30 30 30 30 |0Z..250101000000| -000000b0 5a 30 1a 31 0b 30 09 06 03 55 04 0a 13 02 47 6f |Z0.1.0...U....Go| -000000c0 31 0b 30 09 06 03 55 04 03 13 02 47 6f 30 81 9f |1.0...U....Go0..| -000000d0 30 0d 06 09 2a 86 48 86 f7 0d 01 01 01 05 00 03 |0...*.H.........| -000000e0 81 8d 00 30 81 89 02 81 81 00 db 46 7d 93 2e 12 |...0.......F}...| -000000f0 27 06 48 bc 06 28 21 ab 7e c4 b6 a2 5d fe 1e 52 |'.H..(!.~...]..R| -00000100 45 88 7a 36 47 a5 08 0d 92 42 5b c2 81 c0 be 97 |E.z6G....B[.....| -00000110 79 98 40 fb 4f 6d 14 fd 2b 13 8b c2 a5 2e 67 d8 |y.@.Om..+.....g.| -00000120 d4 09 9e d6 22 38 b7 4a 0b 74 73 2b c2 34 f1 d1 |...."8.J.ts+.4..| -00000130 93 e5 96 d9 74 7b f3 58 9f 6c 61 3c c0 b0 41 d4 |....t{.X.la<..A.| -00000140 d9 2b 2b 24 23 77 5b 1c 3b bd 75 5d ce 20 54 cf |.++$#w[.;.u]. T.| -00000150 a1 63 87 1d 1e 24 c4 f3 1d 1a 50 8b aa b6 14 43 |.c...$....P....C| -00000160 ed 97 a7 75 62 f4 14 c8 52 d7 02 03 01 00 01 a3 |...ub...R.......| -00000170 81 93 30 81 90 30 0e 06 03 55 1d 0f 01 01 ff 04 |..0..0...U......| -00000180 04 03 02 05 a0 30 1d 06 03 55 1d 25 04 16 30 14 |.....0...U.%..0.| -00000190 06 08 2b 06 01 05 05 07 03 01 06 08 2b 06 01 05 |..+.........+...| -000001a0 05 07 03 02 30 0c 06 03 55 1d 13 01 01 ff 04 02 |....0...U.......| -000001b0 30 00 30 19 06 03 55 1d 0e 04 12 04 10 9f 91 16 |0.0...U.........| -000001c0 1f 43 43 3e 49 a6 de 6d b6 80 d7 9f 60 30 1b 06 |.CC>I..m....`0..| -000001d0 03 55 1d 23 04 14 30 12 80 10 48 13 49 4d 13 7e |.U.#..0...H.IM.~| -000001e0 16 31 bb a3 01 d5 ac ab 6e 7b 30 19 06 03 55 1d |.1......n{0...U.| -000001f0 11 04 12 30 10 82 0e 65 78 61 6d 70 6c 65 2e 67 |...0...example.g| -00000200 6f 6c 61 6e 67 30 0d 06 09 2a 86 48 86 f7 0d 01 |olang0...*.H....| -00000210 01 0b 05 00 03 81 81 00 9d 30 cc 40 2b 5b 50 a0 |.........0.@+[P.| -00000220 61 cb ba e5 53 58 e1 ed 83 28 a9 58 1a a9 38 a4 |a...SX...(.X..8.| -00000230 95 a1 ac 31 5a 1a 84 66 3d 43 d3 2d d9 0b f2 97 |...1Z..f=C.-....| -00000240 df d3 20 64 38 92 24 3a 00 bc cf 9c 7d b7 40 20 |.. d8.$:....}.@ | -00000250 01 5f aa d3 16 61 09 a2 76 fd 13 c3 cc e1 0c 5c |._...a..v......\| -00000260 ee b1 87 82 f1 6c 04 ed 73 bb b3 43 77 8d 0c 1c |.....l..s..Cw...| -00000270 f1 0f a1 d8 40 83 61 c9 4c 72 2b 9d ae db 46 06 |....@.a.Lr+...F.| -00000280 06 4d f4 c1 b3 3e c0 d1 bd 42 d4 db fe 3d 13 60 |.M...>...B...=.`| -00000290 84 5c 21 d3 3b e9 fa e7 16 03 03 00 04 0e 00 00 |.\!.;...........| -000002a0 00 |.| +00000030 0d ff 01 00 01 00 00 17 00 00 00 00 00 00 16 03 |................| +00000040 03 02 59 0b 00 02 55 00 02 52 00 02 4f 30 82 02 |..Y...U..R..O0..| +00000050 4b 30 82 01 b4 a0 03 02 01 02 02 09 00 e8 f0 9d |K0..............| +00000060 3f e2 5b ea a6 30 0d 06 09 2a 86 48 86 f7 0d 01 |?.[..0...*.H....| +00000070 01 0b 05 00 30 1f 31 0b 30 09 06 03 55 04 0a 13 |....0.1.0...U...| +00000080 02 47 6f 31 10 30 0e 06 03 55 04 03 13 07 47 6f |.Go1.0...U....Go| +00000090 20 52 6f 6f 74 30 1e 17 0d 31 36 30 31 30 31 30 | Root0...1601010| +000000a0 30 30 30 30 30 5a 17 0d 32 35 30 31 30 31 30 30 |00000Z..25010100| +000000b0 30 30 30 30 5a 30 1a 31 0b 30 09 06 03 55 04 0a |0000Z0.1.0...U..| +000000c0 13 02 47 6f 31 0b 30 09 06 03 55 04 03 13 02 47 |..Go1.0...U....G| +000000d0 6f 30 81 9f 30 0d 06 09 2a 86 48 86 f7 0d 01 01 |o0..0...*.H.....| +000000e0 01 05 00 03 81 8d 00 30 81 89 02 81 81 00 db 46 |.......0.......F| +000000f0 7d 93 2e 12 27 06 48 bc 06 28 21 ab 7e c4 b6 a2 |}...'.H..(!.~...| +00000100 5d fe 1e 52 45 88 7a 36 47 a5 08 0d 92 42 5b c2 |]..RE.z6G....B[.| +00000110 81 c0 be 97 79 98 40 fb 4f 6d 14 fd 2b 13 8b c2 |....y.@.Om..+...| +00000120 a5 2e 67 d8 d4 09 9e d6 22 38 b7 4a 0b 74 73 2b |..g....."8.J.ts+| +00000130 c2 34 f1 d1 93 e5 96 d9 74 7b f3 58 9f 6c 61 3c |.4......t{.X.la<| +00000140 c0 b0 41 d4 d9 2b 2b 24 23 77 5b 1c 3b bd 75 5d |..A..++$#w[.;.u]| +00000150 ce 20 54 cf a1 63 87 1d 1e 24 c4 f3 1d 1a 50 8b |. T..c...$....P.| +00000160 aa b6 14 43 ed 97 a7 75 62 f4 14 c8 52 d7 02 03 |...C...ub...R...| +00000170 01 00 01 a3 81 93 30 81 90 30 0e 06 03 55 1d 0f |......0..0...U..| +00000180 01 01 ff 04 04 03 02 05 a0 30 1d 06 03 55 1d 25 |.........0...U.%| +00000190 04 16 30 14 06 08 2b 06 01 05 05 07 03 01 06 08 |..0...+.........| +000001a0 2b 06 01 05 05 07 03 02 30 0c 06 03 55 1d 13 01 |+.......0...U...| +000001b0 01 ff 04 02 30 00 30 19 06 03 55 1d 0e 04 12 04 |....0.0...U.....| +000001c0 10 9f 91 16 1f 43 43 3e 49 a6 de 6d b6 80 d7 9f |.....CC>I..m....| +000001d0 60 30 1b 06 03 55 1d 23 04 14 30 12 80 10 48 13 |`0...U.#..0...H.| +000001e0 49 4d 13 7e 16 31 bb a3 01 d5 ac ab 6e 7b 30 19 |IM.~.1......n{0.| +000001f0 06 03 55 1d 11 04 12 30 10 82 0e 65 78 61 6d 70 |..U....0...examp| +00000200 6c 65 2e 67 6f 6c 61 6e 67 30 0d 06 09 2a 86 48 |le.golang0...*.H| +00000210 86 f7 0d 01 01 0b 05 00 03 81 81 00 9d 30 cc 40 |.............0.@| +00000220 2b 5b 50 a0 61 cb ba e5 53 58 e1 ed 83 28 a9 58 |+[P.a...SX...(.X| +00000230 1a a9 38 a4 95 a1 ac 31 5a 1a 84 66 3d 43 d3 2d |..8....1Z..f=C.-| +00000240 d9 0b f2 97 df d3 20 64 38 92 24 3a 00 bc cf 9c |...... d8.$:....| +00000250 7d b7 40 20 01 5f aa d3 16 61 09 a2 76 fd 13 c3 |}.@ ._...a..v...| +00000260 cc e1 0c 5c ee b1 87 82 f1 6c 04 ed 73 bb b3 43 |...\.....l..s..C| +00000270 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 4c 72 2b 9d |w.......@.a.Lr+.| +00000280 ae db 46 06 06 4d f4 c1 b3 3e c0 d1 bd 42 d4 db |..F..M...>...B..| +00000290 fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 16 03 03 00 |.=.`.\!.;.......| +000002a0 04 0e 00 00 00 |.....| >>> Flow 3 (client to server) -00000000 16 03 03 00 86 10 00 00 82 00 80 04 36 4c 25 96 |............6L%.| -00000010 b0 1a 33 80 88 98 4f 2a c8 93 24 81 0a 78 6c 85 |..3...O*..$..xl.| -00000020 06 4d f1 cf 25 18 e0 f0 61 50 c5 45 c1 24 1b b3 |.M..%...aP.E.$..| -00000030 d6 3c d3 49 a6 40 81 2c bb ef 49 76 c0 10 4c ad |.<.I.@.,..Iv..L.| -00000040 2e 7d 4d f4 0b 96 bc 1c eb 3d 1d 7d 18 25 34 14 |.}M......=.}.%4.| -00000050 ed 76 f2 a1 17 aa 87 1b ef ff 11 93 a7 44 0c 33 |.v...........D.3| -00000060 86 27 38 3d 5d 3f bb f1 8d a9 f5 44 28 d1 28 41 |.'8=]?.....D(.(A| -00000070 bb b7 9a fb 83 81 91 92 4e 7d 71 55 43 ed 42 12 |........N}qUC.B.| -00000080 86 5f de 02 13 1f c4 63 08 87 db 14 03 03 00 01 |._.....c........| -00000090 01 16 03 03 00 40 32 01 5f a2 e1 08 cf 6b ce 11 |.....@2._....k..| -000000a0 db 82 94 c5 f1 12 9a ac 68 dc f9 c8 2c 00 a5 dd |........h...,...| -000000b0 6b 49 c8 8b b7 9f e3 90 27 a5 c2 45 fc 75 e5 ac |kI......'..E.u..| -000000c0 77 0c 80 bd 43 41 d4 00 c0 fb 8d 08 a6 f4 f7 63 |w...CA.........c| -000000d0 07 01 09 06 e5 fc |......| +00000000 16 03 03 00 86 10 00 00 82 00 80 25 3d 8b d1 c0 |...........%=...| +00000010 ef 86 34 20 a5 4b 4b 94 d9 c8 04 ff 02 13 24 57 |..4 .KK.......$W| +00000020 21 7f f1 c1 06 0b ea 1f b1 06 e8 fa 9b 5c 34 96 |!............\4.| +00000030 23 2a 4b ef cb d7 47 75 05 74 f3 7f ed fb 28 6a |#*K...Gu.t....(j| +00000040 cd b8 16 12 96 4b b7 cf 0c c3 b0 93 c3 ea b0 78 |.....K.........x| +00000050 65 93 9d 6d a9 d5 b7 ed be 8b 3a f6 12 bb 5d ae |e..m......:...].| +00000060 2b 17 2f 62 ca 21 68 7d 12 52 e3 2c cc 32 4b 94 |+./b.!h}.R.,.2K.| +00000070 4b 1d 73 9a 2e 60 60 da e6 32 dd d3 4d 39 69 c8 |K.s..``..2..M9i.| +00000080 b7 9d 8a 1d d8 57 90 13 4c 2a a9 14 03 03 00 01 |.....W..L*......| +00000090 01 16 03 03 00 40 c4 71 f6 06 63 08 15 02 63 0a |.....@.q..c...c.| +000000a0 59 40 55 52 28 17 3f 16 c8 48 93 af 80 87 e6 a8 |Y@UR(.?..H......| +000000b0 37 a4 4f 20 f0 37 88 5b 55 f3 32 60 c7 c4 1d ce |7.O .7.[U.2`....| +000000c0 b2 b8 d1 2d 8b fb a6 87 39 c8 75 31 22 77 33 82 |...-....9.u1"w3.| +000000d0 64 0f f2 10 9d ee |d.....| >>> Flow 4 (server to client) 00000000 14 03 03 00 01 01 16 03 03 00 40 00 00 00 00 00 |..........@.....| -00000010 00 00 00 00 00 00 00 00 00 00 00 6d 31 fe 01 3d |...........m1..=| -00000020 2a c3 97 67 9f 08 f8 c9 ce 57 5c 4a e6 da 17 f2 |*..g.....W\J....| -00000030 f8 47 2b d9 9d 7e af 59 b8 a9 23 9d 7e d5 ed 77 |.G+..~.Y..#.~..w| -00000040 3b cd d4 b7 76 5b 6f 6d 09 bd 0c 17 03 03 00 40 |;...v[om.......@| +00000010 00 00 00 00 00 00 00 00 00 00 00 f2 12 bf 38 31 |..............81| +00000020 ee c9 9a 6a 8d fb 1c 53 41 f1 06 3a 44 9c 31 31 |...j...SA..:D.11| +00000030 25 7b 28 08 f5 3a 85 84 f1 83 61 9b 8c e3 cf 79 |%{(..:....a....y| +00000040 3a c2 ce e2 9c b8 52 ca 4f 5c b1 17 03 03 00 40 |:.....R.O\.....@| 00000050 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -00000060 d2 8c 47 46 36 47 fa d8 f8 1c b4 fc f6 fd fb 4b |..GF6G.........K| -00000070 79 e1 a3 39 df ac 6c 94 61 dd 20 1a e7 c0 4c 9c |y..9..l.a. ...L.| -00000080 45 69 69 cf 73 cb b1 6c fc 71 49 de 41 ca 4d 4f |Eii.s..l.qI.A.MO| +00000060 56 57 a0 ee c3 3e 4a 13 70 d5 05 2a 7d ed 49 81 |VW...>J.p..*}.I.| +00000070 52 37 e0 dc bd d0 e3 de f8 8e 18 a2 8b f8 62 71 |R7............bq| +00000080 7f a9 35 50 91 81 6f 33 63 e3 c5 ec cf fa 1b 05 |..5P..o3c.......| 00000090 15 03 03 00 30 00 00 00 00 00 00 00 00 00 00 00 |....0...........| -000000a0 00 00 00 00 00 d1 66 64 ce 59 eb 23 13 e9 92 28 |......fd.Y.#...(| -000000b0 a4 2a 7a b0 e1 79 ce 92 34 77 6e b3 8d d3 bb e6 |.*z..y..4wn.....| -000000c0 ad 90 e8 a2 1a |.....| +000000a0 00 00 00 00 00 e5 05 33 38 e8 33 35 a3 f0 aa f8 |.......38.35....| +000000b0 8c b7 c5 2b 8c d0 9e 40 57 c5 c9 52 61 ae 5e c7 |...+...@W..Ra.^.| +000000c0 50 f1 5a 28 50 |P.Z(P| diff --git a/src/crypto/tls/testdata/Server-TLSv12-SNI-GetCertificate b/src/crypto/tls/testdata/Server-TLSv12-SNI-GetCertificate index 474ab1ab10a04a..b7756a3d10af1b 100644 --- a/src/crypto/tls/testdata/Server-TLSv12-SNI-GetCertificate +++ b/src/crypto/tls/testdata/Server-TLSv12-SNI-GetCertificate @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 81 01 00 00 7d 03 03 02 34 82 a7 1a |........}...4...| -00000010 fe 81 b0 1c 2e df cc 04 2d f7 22 39 34 95 c7 c1 |........-."94...| -00000020 b2 92 a2 d2 aa ca 57 0f 9c be b4 00 00 04 00 2f |......W......../| +00000000 16 03 01 00 81 01 00 00 7d 03 03 37 94 a0 f3 65 |........}..7...e| +00000010 7b 07 88 ab 9f 29 dd 9a 56 a8 27 84 75 29 4f 24 |{....)..V.'.u)O$| +00000020 ce a2 ef 9b 34 ff 69 06 4c c8 e5 00 00 04 00 2f |....4.i.L....../| 00000030 00 ff 01 00 00 50 00 00 00 10 00 0e 00 00 0b 73 |.....P.........s| 00000040 6e 69 74 65 73 74 2e 63 6f 6d 00 16 00 00 00 17 |nitest.com......| 00000050 00 00 00 0d 00 30 00 2e 04 03 05 03 06 03 08 07 |.....0..........| @@ -9,75 +9,75 @@ 00000070 05 01 06 01 03 03 02 03 03 01 02 01 03 02 02 02 |................| 00000080 04 02 05 02 06 02 |......| >>> Flow 2 (server to client) -00000000 16 03 03 00 35 02 00 00 31 03 03 00 00 00 00 00 |....5...1.......| +00000000 16 03 03 00 39 02 00 00 35 03 03 00 00 00 00 00 |....9...5.......| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 44 4f 57 4e 47 52 44 01 00 00 2f 00 00 |...DOWNGRD.../..| -00000030 09 ff 01 00 01 00 00 17 00 00 16 03 03 02 59 0b |..............Y.| -00000040 00 02 55 00 02 52 00 02 4f 30 82 02 4b 30 82 01 |..U..R..O0..K0..| -00000050 b4 a0 03 02 01 02 02 09 00 e8 f0 9d 3f e2 5b ea |............?.[.| -00000060 a6 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05 00 |.0...*.H........| -00000070 30 1f 31 0b 30 09 06 03 55 04 0a 13 02 47 6f 31 |0.1.0...U....Go1| -00000080 10 30 0e 06 03 55 04 03 13 07 47 6f 20 52 6f 6f |.0...U....Go Roo| -00000090 74 30 1e 17 0d 31 36 30 31 30 31 30 30 30 30 30 |t0...16010100000| -000000a0 30 5a 17 0d 32 35 30 31 30 31 30 30 30 30 30 30 |0Z..250101000000| -000000b0 5a 30 1a 31 0b 30 09 06 03 55 04 0a 13 02 47 6f |Z0.1.0...U....Go| -000000c0 31 0b 30 09 06 03 55 04 03 13 02 47 6f 30 81 9f |1.0...U....Go0..| -000000d0 30 0d 06 09 2a 86 48 86 f7 0d 01 01 01 05 00 03 |0...*.H.........| -000000e0 81 8d 00 30 81 89 02 81 81 00 db 46 7d 93 2e 12 |...0.......F}...| -000000f0 27 06 48 bc 06 28 21 ab 7e c4 b6 a2 5d fe 1e 52 |'.H..(!.~...]..R| -00000100 45 88 7a 36 47 a5 08 0d 92 42 5b c2 81 c0 be 97 |E.z6G....B[.....| -00000110 79 98 40 fb 4f 6d 14 fd 2b 13 8b c2 a5 2e 67 d8 |y.@.Om..+.....g.| -00000120 d4 09 9e d6 22 38 b7 4a 0b 74 73 2b c2 34 f1 d1 |...."8.J.ts+.4..| -00000130 93 e5 96 d9 74 7b f3 58 9f 6c 61 3c c0 b0 41 d4 |....t{.X.la<..A.| -00000140 d9 2b 2b 24 23 77 5b 1c 3b bd 75 5d ce 20 54 cf |.++$#w[.;.u]. T.| -00000150 a1 63 87 1d 1e 24 c4 f3 1d 1a 50 8b aa b6 14 43 |.c...$....P....C| -00000160 ed 97 a7 75 62 f4 14 c8 52 d7 02 03 01 00 01 a3 |...ub...R.......| -00000170 81 93 30 81 90 30 0e 06 03 55 1d 0f 01 01 ff 04 |..0..0...U......| -00000180 04 03 02 05 a0 30 1d 06 03 55 1d 25 04 16 30 14 |.....0...U.%..0.| -00000190 06 08 2b 06 01 05 05 07 03 01 06 08 2b 06 01 05 |..+.........+...| -000001a0 05 07 03 02 30 0c 06 03 55 1d 13 01 01 ff 04 02 |....0...U.......| -000001b0 30 00 30 19 06 03 55 1d 0e 04 12 04 10 9f 91 16 |0.0...U.........| -000001c0 1f 43 43 3e 49 a6 de 6d b6 80 d7 9f 60 30 1b 06 |.CC>I..m....`0..| -000001d0 03 55 1d 23 04 14 30 12 80 10 48 13 49 4d 13 7e |.U.#..0...H.IM.~| -000001e0 16 31 bb a3 01 d5 ac ab 6e 7b 30 19 06 03 55 1d |.1......n{0...U.| -000001f0 11 04 12 30 10 82 0e 65 78 61 6d 70 6c 65 2e 67 |...0...example.g| -00000200 6f 6c 61 6e 67 30 0d 06 09 2a 86 48 86 f7 0d 01 |olang0...*.H....| -00000210 01 0b 05 00 03 81 81 00 9d 30 cc 40 2b 5b 50 a0 |.........0.@+[P.| -00000220 61 cb ba e5 53 58 e1 ed 83 28 a9 58 1a a9 38 a4 |a...SX...(.X..8.| -00000230 95 a1 ac 31 5a 1a 84 66 3d 43 d3 2d d9 0b f2 97 |...1Z..f=C.-....| -00000240 df d3 20 64 38 92 24 3a 00 bc cf 9c 7d b7 40 20 |.. d8.$:....}.@ | -00000250 01 5f aa d3 16 61 09 a2 76 fd 13 c3 cc e1 0c 5c |._...a..v......\| -00000260 ee b1 87 82 f1 6c 04 ed 73 bb b3 43 77 8d 0c 1c |.....l..s..Cw...| -00000270 f1 0f a1 d8 40 83 61 c9 4c 72 2b 9d ae db 46 06 |....@.a.Lr+...F.| -00000280 06 4d f4 c1 b3 3e c0 d1 bd 42 d4 db fe 3d 13 60 |.M...>...B...=.`| -00000290 84 5c 21 d3 3b e9 fa e7 16 03 03 00 04 0e 00 00 |.\!.;...........| -000002a0 00 |.| +00000030 0d ff 01 00 01 00 00 17 00 00 00 00 00 00 16 03 |................| +00000040 03 02 59 0b 00 02 55 00 02 52 00 02 4f 30 82 02 |..Y...U..R..O0..| +00000050 4b 30 82 01 b4 a0 03 02 01 02 02 09 00 e8 f0 9d |K0..............| +00000060 3f e2 5b ea a6 30 0d 06 09 2a 86 48 86 f7 0d 01 |?.[..0...*.H....| +00000070 01 0b 05 00 30 1f 31 0b 30 09 06 03 55 04 0a 13 |....0.1.0...U...| +00000080 02 47 6f 31 10 30 0e 06 03 55 04 03 13 07 47 6f |.Go1.0...U....Go| +00000090 20 52 6f 6f 74 30 1e 17 0d 31 36 30 31 30 31 30 | Root0...1601010| +000000a0 30 30 30 30 30 5a 17 0d 32 35 30 31 30 31 30 30 |00000Z..25010100| +000000b0 30 30 30 30 5a 30 1a 31 0b 30 09 06 03 55 04 0a |0000Z0.1.0...U..| +000000c0 13 02 47 6f 31 0b 30 09 06 03 55 04 03 13 02 47 |..Go1.0...U....G| +000000d0 6f 30 81 9f 30 0d 06 09 2a 86 48 86 f7 0d 01 01 |o0..0...*.H.....| +000000e0 01 05 00 03 81 8d 00 30 81 89 02 81 81 00 db 46 |.......0.......F| +000000f0 7d 93 2e 12 27 06 48 bc 06 28 21 ab 7e c4 b6 a2 |}...'.H..(!.~...| +00000100 5d fe 1e 52 45 88 7a 36 47 a5 08 0d 92 42 5b c2 |]..RE.z6G....B[.| +00000110 81 c0 be 97 79 98 40 fb 4f 6d 14 fd 2b 13 8b c2 |....y.@.Om..+...| +00000120 a5 2e 67 d8 d4 09 9e d6 22 38 b7 4a 0b 74 73 2b |..g....."8.J.ts+| +00000130 c2 34 f1 d1 93 e5 96 d9 74 7b f3 58 9f 6c 61 3c |.4......t{.X.la<| +00000140 c0 b0 41 d4 d9 2b 2b 24 23 77 5b 1c 3b bd 75 5d |..A..++$#w[.;.u]| +00000150 ce 20 54 cf a1 63 87 1d 1e 24 c4 f3 1d 1a 50 8b |. T..c...$....P.| +00000160 aa b6 14 43 ed 97 a7 75 62 f4 14 c8 52 d7 02 03 |...C...ub...R...| +00000170 01 00 01 a3 81 93 30 81 90 30 0e 06 03 55 1d 0f |......0..0...U..| +00000180 01 01 ff 04 04 03 02 05 a0 30 1d 06 03 55 1d 25 |.........0...U.%| +00000190 04 16 30 14 06 08 2b 06 01 05 05 07 03 01 06 08 |..0...+.........| +000001a0 2b 06 01 05 05 07 03 02 30 0c 06 03 55 1d 13 01 |+.......0...U...| +000001b0 01 ff 04 02 30 00 30 19 06 03 55 1d 0e 04 12 04 |....0.0...U.....| +000001c0 10 9f 91 16 1f 43 43 3e 49 a6 de 6d b6 80 d7 9f |.....CC>I..m....| +000001d0 60 30 1b 06 03 55 1d 23 04 14 30 12 80 10 48 13 |`0...U.#..0...H.| +000001e0 49 4d 13 7e 16 31 bb a3 01 d5 ac ab 6e 7b 30 19 |IM.~.1......n{0.| +000001f0 06 03 55 1d 11 04 12 30 10 82 0e 65 78 61 6d 70 |..U....0...examp| +00000200 6c 65 2e 67 6f 6c 61 6e 67 30 0d 06 09 2a 86 48 |le.golang0...*.H| +00000210 86 f7 0d 01 01 0b 05 00 03 81 81 00 9d 30 cc 40 |.............0.@| +00000220 2b 5b 50 a0 61 cb ba e5 53 58 e1 ed 83 28 a9 58 |+[P.a...SX...(.X| +00000230 1a a9 38 a4 95 a1 ac 31 5a 1a 84 66 3d 43 d3 2d |..8....1Z..f=C.-| +00000240 d9 0b f2 97 df d3 20 64 38 92 24 3a 00 bc cf 9c |...... d8.$:....| +00000250 7d b7 40 20 01 5f aa d3 16 61 09 a2 76 fd 13 c3 |}.@ ._...a..v...| +00000260 cc e1 0c 5c ee b1 87 82 f1 6c 04 ed 73 bb b3 43 |...\.....l..s..C| +00000270 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 4c 72 2b 9d |w.......@.a.Lr+.| +00000280 ae db 46 06 06 4d f4 c1 b3 3e c0 d1 bd 42 d4 db |..F..M...>...B..| +00000290 fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 16 03 03 00 |.=.`.\!.;.......| +000002a0 04 0e 00 00 00 |.....| >>> Flow 3 (client to server) -00000000 16 03 03 00 86 10 00 00 82 00 80 d9 90 3c 11 be |.............<..| -00000010 f3 48 de f0 8f 9e 12 ca e0 ab 86 e0 7e e7 8b ea |.H..........~...| -00000020 1a 76 3e 65 8d 7a d6 1c 72 2a f7 1e aa 0a 12 8f |.v>e.z..r*......| -00000030 54 ac 33 95 9d 00 a9 a6 94 54 7b 6a d9 e3 f4 67 |T.3......T{j...g| -00000040 a6 d3 b1 c1 5d 86 51 aa 63 67 6b 6e cb 3b 5e 59 |....].Q.cgkn.;^Y| -00000050 02 c2 57 fd 37 39 1b 73 9a 61 b0 78 de e8 cc f8 |..W.79.s.a.x....| -00000060 b3 01 11 e5 e9 31 85 4d fe 60 d4 12 70 71 64 45 |.....1.M.`..pqdE| -00000070 e8 7d fb be 5b 82 c0 c4 e1 57 09 2c f2 d7 a3 79 |.}..[....W.,...y| -00000080 1c 40 08 e1 e6 cd e2 3e e7 55 da 14 03 03 00 01 |.@.....>.U......| -00000090 01 16 03 03 00 40 29 9e b7 cf 5e 7c e9 40 91 5f |.....@)...^|.@._| -000000a0 b6 12 d4 42 ec 6a bc 03 d9 fa e4 d8 bf c7 2c c5 |...B.j........,.| -000000b0 52 74 17 77 b1 aa 13 87 f0 81 da 0d ca 7f d9 ca |Rt.w............| -000000c0 18 46 55 62 3f 90 21 60 fa 85 8c 80 6b 23 45 e7 |.FUb?.!`....k#E.| -000000d0 0b 6e 8c e2 c3 f6 |.n....| +00000000 16 03 03 00 86 10 00 00 82 00 80 1f fb a2 ec cf |................| +00000010 39 a3 cd db ee 86 8e 22 91 e1 47 5b ac 3b c0 f6 |9......"..G[.;..| +00000020 37 0f d0 b6 19 c5 a4 4c 1a 8f 8b 67 8a 20 0e 06 |7......L...g. ..| +00000030 6a 25 d9 13 58 37 cb dc 9b 3a 0f 9d 12 02 45 3f |j%..X7...:....E?| +00000040 2d 51 f5 cd 9f 45 be 5e f9 af 13 53 c2 15 a6 ca |-Q...E.^...S....| +00000050 8a cb 27 e0 d1 23 7a 19 06 26 d6 86 de 76 e7 9c |..'..#z..&...v..| +00000060 eb f8 15 1d 85 3f be 38 c4 bc 48 c3 74 d4 10 9b |.....?.8..H.t...| +00000070 9e 97 4c 1c 56 18 9d 65 1c be 33 3c 4c 90 e0 e4 |..L.V..e..3.]| +000000b0 3e c1 c2 0c c5 20 eb 76 e1 14 16 95 9c 56 10 67 |>.... .v.....V.g| +000000c0 02 61 2f a6 af 01 b3 64 73 4a 80 53 4a 94 b3 a0 |.a/....dsJ.SJ...| +000000d0 ee b5 95 b6 6a 20 |....j | >>> Flow 4 (server to client) 00000000 14 03 03 00 01 01 16 03 03 00 40 00 00 00 00 00 |..........@.....| -00000010 00 00 00 00 00 00 00 00 00 00 00 ee 8d 0f cd 15 |................| -00000020 db b4 cd 25 27 b6 7e 9b 82 91 2f 01 e1 4f f9 0c |...%'.~.../..O..| -00000030 68 4c bf 26 2b 4b 49 f5 0a 67 8a 4f 12 35 37 75 |hL.&+KI..g.O.57u| -00000040 16 fe cc 26 35 66 60 8c ed 42 40 17 03 03 00 40 |...&5f`..B@....@| +00000010 00 00 00 00 00 00 00 00 00 00 00 0e 0d f6 1c 84 |................| +00000020 d7 1a 4b 45 a1 9b e1 22 78 31 89 0c 4d f3 5b b8 |..KE..."x1..M.[.| +00000030 41 22 4f b2 aa 99 9e 5c 7c ff 2d ca db 32 01 eb |A"O....\|.-..2..| +00000040 55 2a f4 66 58 4a c2 fd 9f e5 7e 17 03 03 00 40 |U*.fXJ....~....@| 00000050 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -00000060 f5 7a ee 53 aa 85 bb 81 c4 57 68 12 f1 40 4c 20 |.z.S.....Wh..@L | -00000070 2a ff fc 6c dd 73 65 fc 41 e6 5b 96 6b 35 2f 8a |*..l.se.A.[.k5/.| -00000080 62 49 4a da f4 df 93 a0 ab e1 12 4d 8d 34 2c 6a |bIJ........M.4,j| +00000060 6c e0 c3 a0 c9 bd 12 83 58 56 e7 f4 cf 31 8f 1d |l.......XV...1..| +00000070 02 17 ce 2b 24 1c 2f 04 11 cc b2 15 38 62 d2 7d |...+$./.....8b.}| +00000080 1b 75 bc 20 a6 3a 65 48 2e 47 14 17 19 51 aa 71 |.u. .:eH.G...Q.q| 00000090 15 03 03 00 30 00 00 00 00 00 00 00 00 00 00 00 |....0...........| -000000a0 00 00 00 00 00 1c 08 e5 52 29 79 d6 15 07 10 44 |........R)y....D| -000000b0 95 07 07 cb 3b 2b 37 2f e3 dc 17 f9 27 b6 5d 44 |....;+7/....'.]D| -000000c0 d0 30 4b 2e 21 |.0K.!| +000000a0 00 00 00 00 00 34 81 ed 3f e0 b9 5d 01 6e d7 e8 |.....4..?..].n..| +000000b0 45 9f 2c 93 27 28 11 34 b4 a9 32 d5 97 9f ea 05 |E.,.'(.4..2.....| +000000c0 39 90 90 dc e5 |9....| diff --git a/src/crypto/tls/testdata/Server-TLSv12-SNI-GetCertificateNotFound b/src/crypto/tls/testdata/Server-TLSv12-SNI-GetCertificateNotFound index 0c06ce98d753c2..975b9fbe599b79 100644 --- a/src/crypto/tls/testdata/Server-TLSv12-SNI-GetCertificateNotFound +++ b/src/crypto/tls/testdata/Server-TLSv12-SNI-GetCertificateNotFound @@ -1,7 +1,7 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 81 01 00 00 7d 03 03 77 e7 c3 97 fa |........}..w....| -00000010 59 80 de d1 f5 9f ce e5 a5 38 60 2c 30 b2 64 5b |Y........8`,0.d[| -00000020 6c 0a 56 49 1d 6f 19 57 5a ac 05 00 00 04 00 2f |l.VI.o.WZ....../| +00000000 16 03 01 00 81 01 00 00 7d 03 03 7a 13 72 a9 8d |........}..z.r..| +00000010 6d 7e 8e 9c ba c1 9d 5c 09 87 9e 2f 7b e1 ba 39 |m~.....\.../{..9| +00000020 f8 ee fd 1c a7 08 61 73 b9 d7 be 00 00 04 00 2f |......as......./| 00000030 00 ff 01 00 00 50 00 00 00 10 00 0e 00 00 0b 73 |.....P.........s| 00000040 6e 69 74 65 73 74 2e 63 6f 6d 00 16 00 00 00 17 |nitest.com......| 00000050 00 00 00 0d 00 30 00 2e 04 03 05 03 06 03 08 07 |.....0..........| @@ -9,75 +9,75 @@ 00000070 05 01 06 01 03 03 02 03 03 01 02 01 03 02 02 02 |................| 00000080 04 02 05 02 06 02 |......| >>> Flow 2 (server to client) -00000000 16 03 03 00 35 02 00 00 31 03 03 00 00 00 00 00 |....5...1.......| +00000000 16 03 03 00 39 02 00 00 35 03 03 00 00 00 00 00 |....9...5.......| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 00000020 00 00 00 44 4f 57 4e 47 52 44 01 00 00 2f 00 00 |...DOWNGRD.../..| -00000030 09 ff 01 00 01 00 00 17 00 00 16 03 03 02 59 0b |..............Y.| -00000040 00 02 55 00 02 52 00 02 4f 30 82 02 4b 30 82 01 |..U..R..O0..K0..| -00000050 b4 a0 03 02 01 02 02 09 00 e8 f0 9d 3f e2 5b ea |............?.[.| -00000060 a6 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05 00 |.0...*.H........| -00000070 30 1f 31 0b 30 09 06 03 55 04 0a 13 02 47 6f 31 |0.1.0...U....Go1| -00000080 10 30 0e 06 03 55 04 03 13 07 47 6f 20 52 6f 6f |.0...U....Go Roo| -00000090 74 30 1e 17 0d 31 36 30 31 30 31 30 30 30 30 30 |t0...16010100000| -000000a0 30 5a 17 0d 32 35 30 31 30 31 30 30 30 30 30 30 |0Z..250101000000| -000000b0 5a 30 1a 31 0b 30 09 06 03 55 04 0a 13 02 47 6f |Z0.1.0...U....Go| -000000c0 31 0b 30 09 06 03 55 04 03 13 02 47 6f 30 81 9f |1.0...U....Go0..| -000000d0 30 0d 06 09 2a 86 48 86 f7 0d 01 01 01 05 00 03 |0...*.H.........| -000000e0 81 8d 00 30 81 89 02 81 81 00 db 46 7d 93 2e 12 |...0.......F}...| -000000f0 27 06 48 bc 06 28 21 ab 7e c4 b6 a2 5d fe 1e 52 |'.H..(!.~...]..R| -00000100 45 88 7a 36 47 a5 08 0d 92 42 5b c2 81 c0 be 97 |E.z6G....B[.....| -00000110 79 98 40 fb 4f 6d 14 fd 2b 13 8b c2 a5 2e 67 d8 |y.@.Om..+.....g.| -00000120 d4 09 9e d6 22 38 b7 4a 0b 74 73 2b c2 34 f1 d1 |...."8.J.ts+.4..| -00000130 93 e5 96 d9 74 7b f3 58 9f 6c 61 3c c0 b0 41 d4 |....t{.X.la<..A.| -00000140 d9 2b 2b 24 23 77 5b 1c 3b bd 75 5d ce 20 54 cf |.++$#w[.;.u]. T.| -00000150 a1 63 87 1d 1e 24 c4 f3 1d 1a 50 8b aa b6 14 43 |.c...$....P....C| -00000160 ed 97 a7 75 62 f4 14 c8 52 d7 02 03 01 00 01 a3 |...ub...R.......| -00000170 81 93 30 81 90 30 0e 06 03 55 1d 0f 01 01 ff 04 |..0..0...U......| -00000180 04 03 02 05 a0 30 1d 06 03 55 1d 25 04 16 30 14 |.....0...U.%..0.| -00000190 06 08 2b 06 01 05 05 07 03 01 06 08 2b 06 01 05 |..+.........+...| -000001a0 05 07 03 02 30 0c 06 03 55 1d 13 01 01 ff 04 02 |....0...U.......| -000001b0 30 00 30 19 06 03 55 1d 0e 04 12 04 10 9f 91 16 |0.0...U.........| -000001c0 1f 43 43 3e 49 a6 de 6d b6 80 d7 9f 60 30 1b 06 |.CC>I..m....`0..| -000001d0 03 55 1d 23 04 14 30 12 80 10 48 13 49 4d 13 7e |.U.#..0...H.IM.~| -000001e0 16 31 bb a3 01 d5 ac ab 6e 7b 30 19 06 03 55 1d |.1......n{0...U.| -000001f0 11 04 12 30 10 82 0e 65 78 61 6d 70 6c 65 2e 67 |...0...example.g| -00000200 6f 6c 61 6e 67 30 0d 06 09 2a 86 48 86 f7 0d 01 |olang0...*.H....| -00000210 01 0b 05 00 03 81 81 00 9d 30 cc 40 2b 5b 50 a0 |.........0.@+[P.| -00000220 61 cb ba e5 53 58 e1 ed 83 28 a9 58 1a a9 38 a4 |a...SX...(.X..8.| -00000230 95 a1 ac 31 5a 1a 84 66 3d 43 d3 2d d9 0b f2 97 |...1Z..f=C.-....| -00000240 df d3 20 64 38 92 24 3a 00 bc cf 9c 7d b7 40 20 |.. d8.$:....}.@ | -00000250 01 5f aa d3 16 61 09 a2 76 fd 13 c3 cc e1 0c 5c |._...a..v......\| -00000260 ee b1 87 82 f1 6c 04 ed 73 bb b3 43 77 8d 0c 1c |.....l..s..Cw...| -00000270 f1 0f a1 d8 40 83 61 c9 4c 72 2b 9d ae db 46 06 |....@.a.Lr+...F.| -00000280 06 4d f4 c1 b3 3e c0 d1 bd 42 d4 db fe 3d 13 60 |.M...>...B...=.`| -00000290 84 5c 21 d3 3b e9 fa e7 16 03 03 00 04 0e 00 00 |.\!.;...........| -000002a0 00 |.| +00000030 0d ff 01 00 01 00 00 17 00 00 00 00 00 00 16 03 |................| +00000040 03 02 59 0b 00 02 55 00 02 52 00 02 4f 30 82 02 |..Y...U..R..O0..| +00000050 4b 30 82 01 b4 a0 03 02 01 02 02 09 00 e8 f0 9d |K0..............| +00000060 3f e2 5b ea a6 30 0d 06 09 2a 86 48 86 f7 0d 01 |?.[..0...*.H....| +00000070 01 0b 05 00 30 1f 31 0b 30 09 06 03 55 04 0a 13 |....0.1.0...U...| +00000080 02 47 6f 31 10 30 0e 06 03 55 04 03 13 07 47 6f |.Go1.0...U....Go| +00000090 20 52 6f 6f 74 30 1e 17 0d 31 36 30 31 30 31 30 | Root0...1601010| +000000a0 30 30 30 30 30 5a 17 0d 32 35 30 31 30 31 30 30 |00000Z..25010100| +000000b0 30 30 30 30 5a 30 1a 31 0b 30 09 06 03 55 04 0a |0000Z0.1.0...U..| +000000c0 13 02 47 6f 31 0b 30 09 06 03 55 04 03 13 02 47 |..Go1.0...U....G| +000000d0 6f 30 81 9f 30 0d 06 09 2a 86 48 86 f7 0d 01 01 |o0..0...*.H.....| +000000e0 01 05 00 03 81 8d 00 30 81 89 02 81 81 00 db 46 |.......0.......F| +000000f0 7d 93 2e 12 27 06 48 bc 06 28 21 ab 7e c4 b6 a2 |}...'.H..(!.~...| +00000100 5d fe 1e 52 45 88 7a 36 47 a5 08 0d 92 42 5b c2 |]..RE.z6G....B[.| +00000110 81 c0 be 97 79 98 40 fb 4f 6d 14 fd 2b 13 8b c2 |....y.@.Om..+...| +00000120 a5 2e 67 d8 d4 09 9e d6 22 38 b7 4a 0b 74 73 2b |..g....."8.J.ts+| +00000130 c2 34 f1 d1 93 e5 96 d9 74 7b f3 58 9f 6c 61 3c |.4......t{.X.la<| +00000140 c0 b0 41 d4 d9 2b 2b 24 23 77 5b 1c 3b bd 75 5d |..A..++$#w[.;.u]| +00000150 ce 20 54 cf a1 63 87 1d 1e 24 c4 f3 1d 1a 50 8b |. T..c...$....P.| +00000160 aa b6 14 43 ed 97 a7 75 62 f4 14 c8 52 d7 02 03 |...C...ub...R...| +00000170 01 00 01 a3 81 93 30 81 90 30 0e 06 03 55 1d 0f |......0..0...U..| +00000180 01 01 ff 04 04 03 02 05 a0 30 1d 06 03 55 1d 25 |.........0...U.%| +00000190 04 16 30 14 06 08 2b 06 01 05 05 07 03 01 06 08 |..0...+.........| +000001a0 2b 06 01 05 05 07 03 02 30 0c 06 03 55 1d 13 01 |+.......0...U...| +000001b0 01 ff 04 02 30 00 30 19 06 03 55 1d 0e 04 12 04 |....0.0...U.....| +000001c0 10 9f 91 16 1f 43 43 3e 49 a6 de 6d b6 80 d7 9f |.....CC>I..m....| +000001d0 60 30 1b 06 03 55 1d 23 04 14 30 12 80 10 48 13 |`0...U.#..0...H.| +000001e0 49 4d 13 7e 16 31 bb a3 01 d5 ac ab 6e 7b 30 19 |IM.~.1......n{0.| +000001f0 06 03 55 1d 11 04 12 30 10 82 0e 65 78 61 6d 70 |..U....0...examp| +00000200 6c 65 2e 67 6f 6c 61 6e 67 30 0d 06 09 2a 86 48 |le.golang0...*.H| +00000210 86 f7 0d 01 01 0b 05 00 03 81 81 00 9d 30 cc 40 |.............0.@| +00000220 2b 5b 50 a0 61 cb ba e5 53 58 e1 ed 83 28 a9 58 |+[P.a...SX...(.X| +00000230 1a a9 38 a4 95 a1 ac 31 5a 1a 84 66 3d 43 d3 2d |..8....1Z..f=C.-| +00000240 d9 0b f2 97 df d3 20 64 38 92 24 3a 00 bc cf 9c |...... d8.$:....| +00000250 7d b7 40 20 01 5f aa d3 16 61 09 a2 76 fd 13 c3 |}.@ ._...a..v...| +00000260 cc e1 0c 5c ee b1 87 82 f1 6c 04 ed 73 bb b3 43 |...\.....l..s..C| +00000270 77 8d 0c 1c f1 0f a1 d8 40 83 61 c9 4c 72 2b 9d |w.......@.a.Lr+.| +00000280 ae db 46 06 06 4d f4 c1 b3 3e c0 d1 bd 42 d4 db |..F..M...>...B..| +00000290 fe 3d 13 60 84 5c 21 d3 3b e9 fa e7 16 03 03 00 |.=.`.\!.;.......| +000002a0 04 0e 00 00 00 |.....| >>> Flow 3 (client to server) -00000000 16 03 03 00 86 10 00 00 82 00 80 67 0c a0 f8 da |...........g....| -00000010 2e d9 9e 9d ef c8 9b 77 f5 fe 25 32 74 49 1d a7 |.......w..%2tI..| -00000020 7d 69 37 2a 94 07 5a 15 e8 f1 1d 36 25 ae 32 e4 |}i7*..Z....6%.2.| -00000030 9c f5 35 fb 54 81 f2 19 4f 8d 6b 64 1a b2 a2 2e |..5.T...O.kd....| -00000040 c4 cb 5b 73 9d 46 97 01 33 d3 b8 a9 18 39 2c ad |..[s.F..3....9,.| -00000050 f2 eb 6b 02 38 44 f8 cf ae ac a6 e6 54 92 29 ae |..k.8D......T.).| -00000060 a6 8a 4e 82 99 f3 77 8c b6 3a a1 5c 4f 25 3b 7f |..N...w..:.\O%;.| -00000070 39 2f cd 51 dc e3 fc 7c 5a 5a 33 e4 af 43 d0 d3 |9/.Q...|ZZ3..C..| -00000080 eb 3b 86 71 af 92 53 6e 02 b9 59 14 03 03 00 01 |.;.q..Sn..Y.....| -00000090 01 16 03 03 00 40 8b e4 6f d3 88 e7 6a e9 aa f2 |.....@..o...j...| -000000a0 4f 67 69 80 bc f1 78 ca a9 f9 29 ce 44 93 81 46 |Ogi...x...).D..F| -000000b0 0e 18 d1 2a 14 8b 3b b5 15 e4 b5 2a bb 88 d4 80 |...*..;....*....| -000000c0 7b 2f 03 c7 83 7c 61 24 29 fe dd bc 49 8a b1 88 |{/...|a$)...I...| -000000d0 41 ac 8a 12 f8 d6 |A.....| +00000000 16 03 03 00 86 10 00 00 82 00 80 6a be 75 35 e3 |...........j.u5.| +00000010 38 f9 8e b5 c3 64 bb 5f 95 95 5e 2c 6a 61 84 8d |8....d._..^,ja..| +00000020 aa 41 88 de 30 55 ba ae 38 48 e5 d9 19 fa ad 09 |.A..0U..8H......| +00000030 94 16 93 df 08 4a a6 0b 0b 53 2a 2a 37 65 cb ed |.....J...S**7e..| +00000040 2c 07 6b 7d 99 6e 14 1d 9b de 60 e8 25 da 0d c5 |,.k}.n....`.%...| +00000050 73 e5 a9 87 25 ce c7 8f 68 88 c5 68 14 ee ac 91 |s...%...h..h....| +00000060 ab 44 fe 31 e0 b5 e1 cd 9b 56 b7 0a 5a d6 b3 54 |.D.1.....V..Z..T| +00000070 9c aa 30 17 ea e2 8c b5 61 89 a7 b1 96 d6 25 0f |..0.....a.....%.| +00000080 30 91 ba 95 2e a8 c0 53 ad 18 e4 14 03 03 00 01 |0......S........| +00000090 01 16 03 03 00 40 b1 5f 16 fb 6e 9d 3b 05 20 be |.....@._..n.;. .| +000000a0 19 0e 6d 74 c1 4a aa 00 db af 58 92 4a 83 f3 23 |..mt.J....X.J..#| +000000b0 e2 6c 0f f2 00 08 0f fd f4 f8 71 b3 c3 8e 8c 57 |.l........q....W| +000000c0 db 98 ff d6 7c f9 26 0b 13 8c 83 36 32 9a 5a 58 |....|.&....62.ZX| +000000d0 0a 41 7d 3f 30 c0 |.A}?0.| >>> Flow 4 (server to client) 00000000 14 03 03 00 01 01 16 03 03 00 40 00 00 00 00 00 |..........@.....| -00000010 00 00 00 00 00 00 00 00 00 00 00 e7 d3 34 46 88 |.............4F.| -00000020 0e 7f ae 5b d6 e5 70 d2 7d 99 25 1b 27 89 8a a4 |...[..p.}.%.'...| -00000030 02 03 01 a4 e1 d6 72 af c3 5a 55 f7 56 69 60 91 |......r..ZU.Vi`.| -00000040 49 29 68 36 99 e5 09 ac 7a e3 4f 17 03 03 00 40 |I)h6....z.O....@| +00000010 00 00 00 00 00 00 00 00 00 00 00 88 5b 28 f3 14 |............[(..| +00000020 18 d4 cd f1 1b b8 4b 47 50 0c 76 07 8a f9 ad 9d |......KGP.v.....| +00000030 ba 87 e7 69 cf 47 d7 79 2f 45 55 74 e8 2a 8f d3 |...i.G.y/EUt.*..| +00000040 02 10 ee d8 80 a0 36 9a 5b 70 4d 17 03 03 00 40 |......6.[pM....@| 00000050 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -00000060 71 d0 36 7d 2c 3b 40 e3 a9 3f c1 d0 5e 3a ee d2 |q.6},;@..?..^:..| -00000070 4d 0d b7 f4 83 3e 75 e0 ed 8a fc b2 9b ed 98 a8 |M....>u.........| -00000080 ec 49 65 83 53 e0 79 52 03 2b 78 8a 64 3e 4c 5e |.Ie.S.yR.+x.d>L^| +00000060 43 e0 95 80 4c ce 68 fb 8c ad a8 77 99 d7 53 7f |C...L.h....w..S.| +00000070 b7 bf 03 59 23 f1 b4 3c d8 db 38 f4 b5 9b f5 3f |...Y#..<..8....?| +00000080 12 b6 f3 2a f5 f8 66 57 0c 03 7b 00 16 01 8a 70 |...*..fW..{....p| 00000090 15 03 03 00 30 00 00 00 00 00 00 00 00 00 00 00 |....0...........| -000000a0 00 00 00 00 00 76 38 c3 3d 86 d8 58 f2 16 48 94 |.....v8.=..X..H.| -000000b0 46 65 ea 80 46 74 fe 66 7c 72 99 30 b3 05 08 14 |Fe..Ft.f|r.0....| -000000c0 19 e3 ee 6f cf |...o.| +000000a0 00 00 00 00 00 3a fa 64 7c f3 25 08 c6 06 fb 02 |.....:.d|.%.....| +000000b0 ce a0 97 81 c9 ea 53 fc cf 39 b9 8a f7 0c df 2f |......S..9...../| +000000c0 5f 56 62 78 86 |_Vbx.| diff --git a/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedAndECDSAGiven b/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedAndECDSAGiven index a957c26399e8e6..686011b0fa9b82 100644 --- a/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedAndECDSAGiven +++ b/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedAndECDSAGiven @@ -1,180 +1,182 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 ca 01 00 00 c6 03 03 cd 87 c0 5e 7c |..............^|| -00000010 7b d7 c6 77 d9 21 6f 46 00 07 88 61 18 8c b9 d4 |{..w.!oF...a....| -00000020 ad 25 6a 9d 7e 54 cc 70 52 7c 0f 20 67 9b dd 18 |.%j.~T.pR|. g...| -00000030 84 bb 23 7d 53 10 b9 6c 01 ef 30 6f 79 7d 64 5c |..#}S..l..0oy}d\| -00000040 79 3e c1 11 8f 75 cf 83 02 d3 e8 d9 00 04 13 01 |y>...u..........| +00000000 16 03 01 00 ca 01 00 00 c6 03 03 af 29 1d da b2 |............)...| +00000010 1b 14 96 f9 c9 6a 4d 28 cc 1d 9f c4 95 d0 a6 16 |.....jM(........| +00000020 1f 83 3d d6 17 80 5e 4f 9f d9 87 20 cb 34 16 64 |..=...^O... .4.d| +00000030 39 bb 98 16 55 43 38 3a a2 b4 e0 38 02 04 ee 6b |9...UC8:...8...k| +00000040 a6 4e b9 f8 83 09 aa 7b f5 de e0 79 00 04 13 01 |.N.....{...y....| 00000050 00 ff 01 00 00 79 00 0b 00 04 03 00 01 02 00 0a |.....y..........| 00000060 00 0c 00 0a 00 1d 00 17 00 1e 00 19 00 18 00 16 |................| 00000070 00 00 00 17 00 00 00 0d 00 1e 00 1c 04 03 05 03 |................| 00000080 06 03 08 07 08 08 08 09 08 0a 08 0b 08 04 08 05 |................| 00000090 08 06 04 01 05 01 06 01 00 2b 00 03 02 03 04 00 |.........+......| -000000a0 2d 00 02 01 01 00 33 00 26 00 24 00 1d 00 20 d7 |-.....3.&.$... .| -000000b0 18 8a c1 a5 4d cb 7c f2 7d e4 cf 7a c6 92 28 ee |....M.|.}..z..(.| -000000c0 a6 b4 79 65 bf 2b fb 71 2e a5 2a 58 da 6f 5e |..ye.+.q..*X.o^| +000000a0 2d 00 02 01 01 00 33 00 26 00 24 00 1d 00 20 ee |-.....3.&.$... .| +000000b0 90 71 47 4a 18 b1 78 c4 9a 7d 65 5d 4e b3 88 96 |.qGJ..x..}e]N...| +000000c0 6c b1 bc 2d a3 9d 2d 8e a2 7a 7b eb 94 11 4e |l..-..-..z{...N| >>> Flow 2 (server to client) 00000000 16 03 03 00 7a 02 00 00 76 03 03 00 00 00 00 00 |....z...v.......| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -00000020 00 00 00 00 00 00 00 00 00 00 00 20 67 9b dd 18 |........... g...| -00000030 84 bb 23 7d 53 10 b9 6c 01 ef 30 6f 79 7d 64 5c |..#}S..l..0oy}d\| -00000040 79 3e c1 11 8f 75 cf 83 02 d3 e8 d9 13 01 00 00 |y>...u..........| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 cb 34 16 64 |........... .4.d| +00000030 39 bb 98 16 55 43 38 3a a2 b4 e0 38 02 04 ee 6b |9...UC8:...8...k| +00000040 a6 4e b9 f8 83 09 aa 7b f5 de e0 79 13 01 00 00 |.N.....{...y....| 00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 2f |..+.....3.$... /| 00000060 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| 00000070 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 14 |.........._X.;t.| -00000080 03 03 00 01 01 17 03 03 00 17 4d 4b 33 12 cb ed |..........MK3...| -00000090 f7 a2 55 e7 a6 ab 5b b1 55 16 30 c3 ee e1 5c 14 |..U...[.U.0...\.| -000000a0 b8 17 03 03 00 3e 30 04 2a e5 e8 b8 f3 25 9b a9 |.....>0.*....%..| -000000b0 92 e6 eb d9 fb b6 62 64 e2 de 9d c0 48 68 b3 d7 |......bd....Hh..| -000000c0 0f 8b ad 21 69 c7 f0 d7 96 ff 6a 24 2f 01 c9 e7 |...!i.....j$/...| -000000d0 e9 06 4b 93 94 67 97 44 46 c4 69 2f 0e aa e0 9b |..K..g.DF.i/....| -000000e0 8d 19 f2 2c 17 03 03 02 6d a1 fb 2b 5f 10 e9 62 |...,....m..+_..b| -000000f0 8b ad c1 33 58 f6 a7 89 78 a6 dd 64 ff c2 c7 57 |...3X...x..d...W| -00000100 80 9f 59 be 2d bd f5 b9 8a 6b d9 22 1c fe 10 ea |..Y.-....k."....| -00000110 c4 a5 3e 45 d9 d7 12 cf 48 d5 26 18 49 50 80 69 |..>E....H.&.IP.i| -00000120 44 28 03 3b b6 f8 0a 0e b4 cb 5e 5b 57 0f c0 2f |D(.;......^[W../| -00000130 59 4f 13 e3 9c 02 18 b1 ce 94 78 05 18 64 73 c6 |YO........x..ds.| -00000140 05 89 d1 54 37 18 ea 09 61 08 c5 6a 54 f6 48 44 |...T7...a..jT.HD| -00000150 40 63 6a 53 b9 41 5f 4f 8e 05 e7 31 7b 08 d0 67 |@cjS.A_O...1{..g| -00000160 8e bf 56 c4 56 0a 82 b2 74 6a 89 dd b4 f7 3c 0b |..V.V...tj....<.| -00000170 3c fd 21 1f 00 72 1d 4f be b1 50 44 9a 14 67 7d |<.!..r.O..PD..g}| -00000180 a9 93 30 a2 4e ea 61 c9 fd 44 de 5c 88 36 59 a2 |..0.N.a..D.\.6Y.| -00000190 e3 63 b1 9c ea dd 47 0a ca 63 9e 50 9d ca 57 12 |.c....G..c.P..W.| -000001a0 05 9b fc f1 26 a2 5e 18 9b 32 00 38 1f ce a6 58 |....&.^..2.8...X| -000001b0 58 0f 61 e2 44 c2 89 34 cc f4 fd 9a dc 1a 67 a6 |X.a.D..4......g.| -000001c0 e8 b1 fc 9f dc bd 0b 21 01 49 0d 9b e1 40 00 f6 |.......!.I...@..| -000001d0 33 1a 57 c5 84 c1 98 3d 7f 53 d3 4d 2e 04 5e 40 |3.W....=.S.M..^@| -000001e0 7d 38 80 66 bc c5 40 d9 14 f6 83 26 82 9b c8 14 |}8.f..@....&....| -000001f0 61 aa 6c 1c a1 53 81 f8 b0 7f 06 92 5c af be 57 |a.l..S......\..W| -00000200 1a 54 97 02 27 31 1f 58 52 cf 39 2f 82 26 ae 6c |.T..'1.XR.9/.&.l| -00000210 86 d9 46 cd 38 16 e2 67 62 82 2e 53 7a 86 15 30 |..F.8..gb..Sz..0| -00000220 08 0c a7 e2 85 18 55 79 16 44 4d 50 9e b5 e7 e1 |......Uy.DMP....| -00000230 2c 1e 1d eb e1 83 f3 9e d0 7b 45 b8 1e 51 d9 79 |,........{E..Q.y| -00000240 91 7a b8 90 bc 18 94 69 ad 94 08 e5 23 de 2b fa |.z.....i....#.+.| -00000250 8d ef 23 4c 40 df e1 43 0d 71 51 ef 88 a9 bb 89 |..#L@..C.qQ.....| -00000260 59 87 9c db e1 d4 31 94 a7 f5 af 7d 51 be e9 d0 |Y.....1....}Q...| -00000270 f2 49 12 72 47 65 a0 5b 7d 9f 91 85 f7 e8 d6 90 |.I.rGe.[}.......| -00000280 b3 4d f3 5d 6a 51 96 a9 81 84 72 95 47 e8 0a f5 |.M.]jQ....r.G...| -00000290 8d d6 2a 64 c2 34 1b d8 f0 f9 62 0c be 17 12 9b |..*d.4....b.....| -000002a0 40 a2 c3 8a eb 30 20 04 e4 69 a3 27 90 a6 1a 4f |@....0 ..i.'...O| -000002b0 3f 95 65 e6 9e c7 ad 03 e1 d2 34 d2 84 d5 f6 8c |?.e.......4.....| -000002c0 1e 8a aa e4 75 c0 7f 1d 79 4e 70 10 4e 18 9c eb |....u...yNp.N...| -000002d0 17 76 0c 66 6c 82 72 41 83 98 fc 41 41 4a 07 03 |.v.fl.rA...AAJ..| -000002e0 a6 16 51 0d 60 96 43 0a 97 27 72 42 31 70 6b 02 |..Q.`.C..'rB1pk.| -000002f0 e4 58 b0 15 4f 2e a3 5a ed dc 82 99 82 47 d5 6c |.X..O..Z.....G.l| -00000300 4b b4 68 70 f5 a3 31 36 52 8d af ab d1 ac f6 28 |K.hp..16R......(| -00000310 2e 18 bc 4f 1b 7c a8 ad c3 1f 2f 70 a6 c4 39 c6 |...O.|..../p..9.| -00000320 ae 0f 2e b7 58 c9 c0 2a 4b 34 c2 42 12 e3 5d ed |....X..*K4.B..].| -00000330 d1 ac e1 f1 14 66 d4 09 1c a0 99 82 d3 04 13 2a |.....f.........*| -00000340 a4 20 c4 e7 38 1e 0a 02 4e 96 02 71 9d f6 f7 86 |. ..8...N..q....| -00000350 f7 30 1a 5d 65 4f 17 03 03 00 99 3f 3d 4a 91 ae |.0.]eO.....?=J..| -00000360 0f 80 52 0f 1c d1 a2 75 83 e5 08 d8 1f 9d c8 24 |..R....u.......$| -00000370 fc 85 ba 76 1f 9e 1e 35 a3 dd 45 83 8e b9 55 a6 |...v...5..E...U.| -00000380 3b 26 ae 82 4f 1f 2e 8f e5 25 fb d6 22 0f 55 d6 |;&..O....%..".U.| -00000390 ac fa 93 6d d1 d3 7c 41 af c0 15 5c 8b e1 64 c1 |...m..|A...\..d.| -000003a0 3f a2 c8 9e 48 f5 63 61 3a df 13 6e f4 e3 60 9d |?...H.ca:..n..`.| -000003b0 bc 52 bd b7 94 e9 4b 7a 65 97 28 ac 39 6a 77 a7 |.R....Kze.(.9jw.| -000003c0 86 1d b4 6b e4 15 c4 bd 2b 41 b6 06 ac ff b5 9f |...k....+A......| -000003d0 17 47 b4 a7 1d 69 8b 6e 82 eb f7 39 03 95 10 dd |.G...i.n...9....| -000003e0 18 78 50 58 c8 78 80 ae 45 dc 54 0f 33 cb 4a d2 |.xPX.x..E.T.3.J.| -000003f0 90 00 12 d4 17 03 03 00 35 76 fa bf ab 3f c3 3d |........5v...?.=| -00000400 dd 78 65 cc 35 1e 24 35 4f 7d 3c e4 bb 73 d8 19 |.xe.5.$5O}<..s..| -00000410 56 94 f4 ce ad bc 1d 6a fb 1b 75 01 93 36 2b 44 |V......j..u..6+D| -00000420 3a 3c a3 9b 7c 57 6f 98 24 a3 64 b1 13 47 |:<..|Wo.$.d..G| +00000080 03 03 00 01 01 17 03 03 00 17 c3 9d 29 f6 f8 18 |............)...| +00000090 b9 39 9f 93 b2 f5 ab 30 41 d6 95 40 21 ba f8 8e |.9.....0A..@!...| +000000a0 25 17 03 03 00 52 08 a9 80 f3 24 c6 a6 b6 aa bc |%....R....$.....| +000000b0 30 c2 f7 dd e6 47 10 88 db 0c dd 43 ab 78 bd 16 |0....G.....C.x..| +000000c0 82 7b 4f 26 9c 0e 47 6b 75 79 4e da 8e 43 8b e6 |.{O&..GkuyN..C..| +000000d0 6a 25 0d 47 50 bf 26 c8 16 70 41 1c b2 b5 cd d1 |j%.GP.&..pA.....| +000000e0 34 9b 26 13 92 45 b5 f5 03 04 cf 55 ee ff c8 e6 |4.&..E.....U....| +000000f0 92 9e 04 d9 c5 f5 fd 0f 17 03 03 02 6d f6 33 26 |............m.3&| +00000100 26 da 41 f2 4e 6e ed 41 23 44 29 5f 43 38 7d 24 |&.A.Nn.A#D)_C8}$| +00000110 f7 a1 01 d4 23 a5 bf 85 db 39 86 27 ff 9c e9 ac |....#....9.'....| +00000120 6c 0e 0e 51 c7 95 8d aa f6 59 53 0a 89 d4 e7 2d |l..Q.....YS....-| +00000130 6e f8 66 7d e5 b2 ac e4 7b b2 91 2b 31 c9 2b 4e |n.f}....{..+1.+N| +00000140 92 8f a9 c8 79 21 9a 8f a2 79 29 47 c4 ef ab 80 |....y!...y)G....| +00000150 25 02 d6 8c ae 32 3b 1a 55 0e f9 4a ba 92 b9 61 |%....2;.U..J...a| +00000160 86 39 04 0a a7 d6 eb 95 50 4f 86 1b c9 d1 1d 1e |.9......PO......| +00000170 61 06 43 00 c2 2a a9 e2 06 ff ce 4a fd 6f d0 19 |a.C..*.....J.o..| +00000180 2a 0e ec 14 86 19 9c bc 1d 96 e1 b1 00 b2 19 c4 |*...............| +00000190 15 64 4c 33 54 89 fc 15 01 90 13 3b ff 99 e6 23 |.dL3T......;...#| +000001a0 19 38 4f 67 97 ee ab b3 9f ca 1f 18 7d 45 ef 87 |.8Og........}E..| +000001b0 42 7b 57 51 d0 c3 43 50 35 a3 33 ff 19 df 54 8e |B{WQ..CP5.3...T.| +000001c0 e9 98 2f 46 93 87 b4 b1 c2 73 04 4f 06 5b 3c 0a |../F.....s.O.[<.| +000001d0 c3 87 6c 68 68 b4 61 a2 33 bd 11 06 93 19 fb 05 |..lhh.a.3.......| +000001e0 c3 e6 3f 14 e5 a6 6d ec 48 56 a6 37 60 81 a4 cd |..?...m.HV.7`...| +000001f0 89 b5 4e c8 dc 99 79 66 5e 40 44 de 4a 22 9a ce |..N...yf^@D.J"..| +00000200 3d c4 a0 17 03 45 e6 28 70 27 f5 24 e1 81 7a 5f |=....E.(p'.$..z_| +00000210 5d 6e 6c bc 28 c3 fb b1 8b 18 fa 3d 9d f3 c3 a5 |]nl.(......=....| +00000220 f0 41 7a 0d 87 c1 70 77 d3 3a fa ef 2b 11 9b 61 |.Az...pw.:..+..a| +00000230 12 f8 d1 25 73 bc 79 bf ee 9a 5e e0 3b fb 06 da |...%s.y...^.;...| +00000240 7e d6 08 aa 13 73 8a 8c 97 d7 5a e4 dc 88 51 60 |~....s....Z...Q`| +00000250 8c 61 51 65 f6 d0 4e 7d b0 b1 23 2c 69 77 20 fe |.aQe..N}..#,iw .| +00000260 d1 ff 13 57 d8 c5 58 1f 3c e9 86 f4 3a 45 1a 9c |...W..X.<...:E..| +00000270 c2 e0 4d da 67 1b 81 62 bf 07 de a6 ea 74 81 e3 |..M.g..b.....t..| +00000280 ba 1b 1f bf b0 d3 d3 08 c0 23 5f 05 d5 1f 41 51 |.........#_...AQ| +00000290 f2 11 2e 36 ed 00 f8 b5 ea f1 4c 0b b8 d0 a3 cc |...6......L.....| +000002a0 c6 a7 2a ee 1f 6c 6b 33 0d 38 5b 5c 6e f4 53 d2 |..*..lk3.8[\n.S.| +000002b0 6e 90 5b ce 44 e7 f1 50 1e 12 21 76 35 d1 2f 49 |n.[.D..P..!v5./I| +000002c0 a1 27 66 00 93 27 27 34 5d e5 ed 4b 6f 7e 3d e3 |.'f..''4]..Ko~=.| +000002d0 78 de 2f 48 9f 4c 36 a7 5d 62 98 1e 2c ee 82 3c |x./H.L6.]b..,..<| +000002e0 18 17 2d 80 f4 a2 ac e9 a6 c8 2c b9 92 d1 9e 2b |..-.......,....+| +000002f0 5f 3e 33 56 03 b5 76 51 ea c7 af 1c 42 8e 0e 65 |_>3V..vQ....B..e| +00000300 f7 0f ad 36 9f fe b5 a0 06 31 f0 89 02 b4 bf c3 |...6.....1......| +00000310 8e 2f 52 4d 97 99 dd 95 0f c4 32 9e 81 55 d1 2a |./RM......2..U.*| +00000320 05 ff b5 7a 02 0f 6f 1b ef f1 de 48 dc a8 f8 44 |...z..o....H...D| +00000330 35 8c eb df 6b 78 cd 89 75 ca ef 31 ba 8f b8 ff |5...kx..u..1....| +00000340 a3 74 39 ce 42 a0 01 8b db 68 34 21 0a bf 84 fd |.t9.B....h4!....| +00000350 5f 28 87 d4 09 b2 91 c2 f3 1f b8 39 da 2b 55 3c |_(.........9.+U<| +00000360 46 a2 fa 3d 65 eb 41 0c 37 c8 17 03 03 00 99 e8 |F..=e.A.7.......| +00000370 25 eb 99 2f 80 e2 09 ed 2f b9 78 90 7c dc e2 4a |%../..../.x.|..J| +00000380 78 19 49 46 a8 e7 82 a0 85 84 71 da 80 fb d2 5d |x.IF......q....]| +00000390 28 c4 69 cf ac 1c bc 1e ba 90 5c 0a ec dd 31 5c |(.i.......\...1\| +000003a0 87 d0 4f 31 3f 62 cd 91 1e 99 a5 08 c6 de 6c 75 |..O1?b........lu| +000003b0 91 be 31 83 12 5b fd 5a eb 96 ae 70 9c 06 c9 ee |..1..[.Z...p....| +000003c0 63 e9 3c a0 5b 9c 55 cb cd 9d a2 31 33 17 92 fa |c.<.[.U....13...| +000003d0 dc 34 20 43 71 92 8b 16 dd 1c 29 f5 ec a2 e4 ad |.4 Cq.....).....| +000003e0 d6 21 27 37 32 0c 7b f1 60 7f 58 aa fc d6 ae f3 |.!'72.{.`.X.....| +000003f0 c9 7b 5f 59 39 cf 79 ab 5b a3 7e ea 13 72 51 9f |.{_Y9.y.[.~..rQ.| +00000400 a9 c5 fc fb c0 da 39 2a 17 03 03 00 35 b7 6d ed |......9*....5.m.| +00000410 8d d3 c6 09 fd 9b 6b be 6e bc 01 64 3b a1 4e 2d |......k.n..d;.N-| +00000420 3d 85 e3 5b 4b 7b 44 49 81 fd c8 d9 07 fe e5 53 |=..[K{DI.......S| +00000430 ab 5f a3 64 f6 be 22 79 80 52 f0 ee 3c 1e 4e 65 |._.d.."y.R..<.Ne| +00000440 0e e3 |..| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 02 1e e0 29 6d d1 38 |............)m.8| -00000010 a6 03 d7 fc 2b df a4 a4 d0 ae 87 01 b8 82 5d eb |....+.........].| -00000020 3c 25 3e a1 33 89 b5 05 eb bc 02 0f ad 84 f0 2b |<%>.3..........+| -00000030 5f 14 af 0d f8 f1 08 e3 ca c1 8e ad 4b a5 a0 09 |_...........K...| -00000040 b8 a0 4f bf e4 0f 93 a9 4a 35 35 26 22 d7 04 03 |..O.....J55&"...| -00000050 70 95 4f 80 df d4 47 5b 21 14 27 d6 17 ea 32 6e |p.O...G[!.'...2n| -00000060 ea 3a ca 31 7c f2 d1 46 a2 ef 47 b9 16 d6 18 d5 |.:.1|..F..G.....| -00000070 17 35 71 9d 14 37 63 cd e1 c6 ff 3d ec a5 d8 53 |.5q..7c....=...S| -00000080 48 d5 d8 0b 75 39 c1 d2 66 ad 97 4e 5f eb c3 c7 |H...u9..f..N_...| -00000090 e0 77 95 fd 16 84 e2 a3 6a f2 a8 88 5e 4c 86 be |.w......j...^L..| -000000a0 81 a5 f2 6b 12 86 63 f1 4f a6 8d 63 7c 07 0f 8d |...k..c.O..c|...| -000000b0 53 d5 d4 00 c7 b7 2a ef b2 1d 07 4e 43 d6 25 35 |S.....*....NC.%5| -000000c0 c9 b1 fa 91 d1 f7 87 c6 98 c7 e7 c2 c3 7d 42 09 |.............}B.| -000000d0 e5 3c 50 83 1b 20 ef ec e4 ac 2e f2 3e 03 73 52 |..sR| -000000e0 2e 1d 20 cf e2 8d 15 c3 c7 a6 2f 68 b5 8e 5f bc |.. ......./h.._.| -000000f0 c5 73 61 ff 52 3c b0 7b 47 82 47 a0 73 f8 1c ab |.sa.R<.{G.G.s...| -00000100 4d ea 15 fa 94 e0 7f 70 c1 c8 9c 55 f0 96 38 42 |M......p...U..8B| -00000110 76 d4 26 6d a7 73 11 59 43 19 2f 49 70 a9 18 1c |v.&m.s.YC./Ip...| -00000120 12 9e ee d3 eb ca 1c c0 3b e1 99 e3 c2 25 de 39 |........;....%.9| -00000130 1a 15 e0 d7 20 9d 1b 95 74 8c ee 96 7b 5e de 13 |.... ...t...{^..| -00000140 99 56 54 a2 31 7b e9 96 02 9e 86 7b 15 9d c6 3e |.VT.1{.....{...>| -00000150 24 a5 19 e5 8e de 85 97 05 68 4f 39 d1 49 05 c9 |$........hO9.I..| -00000160 7a 54 90 29 e7 93 ec 8e 6f cc 73 73 82 7d 72 8d |zT.)....o.ss.}r.| -00000170 4a 85 29 7f a5 53 13 26 16 b2 fa c3 ce 1f 8b ae |J.)..S.&........| -00000180 e6 60 8c 2d b0 64 66 d4 29 7c b5 2d 1d 11 c5 09 |.`.-.df.)|.-....| -00000190 bb a5 44 c6 c8 af e6 f1 d5 f8 d9 45 97 64 7f 03 |..D........E.d..| -000001a0 02 f7 f5 9f cb 31 1e 89 e4 5d a0 e9 34 db be 28 |.....1...]..4..(| -000001b0 51 15 68 54 01 7c e5 1d b4 05 d2 d6 24 ca 10 69 |Q.hT.|......$..i| -000001c0 31 bf 8b 7a ee d9 bf e1 2a d4 7c c8 e8 79 ca dd |1..z....*.|..y..| -000001d0 0f 73 09 c8 cf 97 28 78 04 5b 04 51 44 c4 5f d1 |.s....(x.[.QD._.| -000001e0 dc da 4f f5 d9 5c 9c b1 ea f8 1b f7 43 90 c0 c0 |..O..\......C...| -000001f0 fd 82 56 e3 71 15 18 5e 7e 5c 61 5e b3 80 c1 1c |..V.q..^~\a^....| -00000200 22 92 32 67 23 f6 3b 74 e4 20 4e 1f fb f8 89 55 |".2g#.;t. N....U| -00000210 e7 3c 18 30 24 77 7c 33 5c 89 91 60 65 14 06 9e |.<.0$w|3\..`e...| -00000220 e2 6d f0 07 84 4b b4 14 e8 17 03 03 00 a3 a7 e1 |.m...K..........| -00000230 f7 26 48 56 da 6d ef a5 f4 5f 19 52 37 9e e7 6d |.&HV.m..._.R7..m| -00000240 28 07 70 ee 1e de 85 2b 7a 2d bf eb 48 06 f5 d4 |(.p....+z-..H...| -00000250 ea 4d 83 86 59 d4 14 4f 46 bc 17 89 f1 f5 37 0e |.M..Y..OF.....7.| -00000260 84 60 6e ba 73 d7 c1 bc 6f d7 aa cf f0 36 96 a2 |.`n.s...o....6..| -00000270 83 60 81 6f 48 6c 9d 87 e5 b6 5e 77 ab c5 e3 cb |.`.oHl....^w....| -00000280 8e 55 94 dc 94 f3 8a ce cb f7 b4 d5 33 55 df 88 |.U..........3U..| -00000290 22 44 04 ee 4e f5 30 e7 30 94 dc 95 2d 97 2c e4 |"D..N.0.0...-.,.| -000002a0 34 58 4d 38 9e 25 61 96 c1 37 66 34 2b be ee e6 |4XM8.%a..7f4+...| -000002b0 ee 39 73 89 a3 aa 1b 0a 5a bf 44 23 4e 19 5c c4 |.9s.....Z.D#N.\.| -000002c0 3f 27 47 5c 40 67 6c 50 b8 3f 7c c9 97 f0 55 02 |?'G\@glP.?|...U.| -000002d0 16 17 03 03 00 35 82 8b 0d 4e 87 3c c6 bc 41 8b |.....5...N.<..A.| -000002e0 ec ab 71 9d 57 7b e6 22 e4 87 82 61 5d f8 69 31 |..q.W{."...a].i1| -000002f0 8a 2c be 2c d8 4d 2f dc 9a 91 31 7a ab d8 a4 0e |.,.,.M/...1z....| -00000300 ba cb fc ef 17 a4 7c 87 ca 13 bb 17 03 03 00 13 |......|.........| -00000310 3b d0 da 9f d4 be fa 59 7a 30 8b 7e 8e a0 99 c0 |;......Yz0.~....| -00000320 c2 36 f3 |.6.| +00000000 14 03 03 00 01 01 17 03 03 02 1e a8 48 8d 47 f6 |............H.G.| +00000010 3d d8 59 5b 26 9e bd fc d1 15 7c e1 37 1b db 3c |=.Y[&.....|.7..<| +00000020 e4 02 98 fd aa cc a6 45 97 6e c5 18 4e 45 8e d6 |.......E.n..NE..| +00000030 84 7a b5 e4 da 8b 17 34 e5 fb 21 3c f6 02 c2 5c |.z.....4..!<...\| +00000040 ea 57 ea 16 67 4a 6d b2 0e 5a 39 f8 6b 11 84 bd |.W..gJm..Z9.k...| +00000050 43 80 03 40 5e d2 d2 54 2e d1 38 bc 9a 16 4f 21 |C..@^..T..8...O!| +00000060 0e 07 38 b5 80 21 16 c6 a1 bb 23 79 a6 df ef 27 |..8..!....#y...'| +00000070 dc 30 3c 4a 53 b1 5c 54 a7 fb ff 73 e2 12 ab 90 |.0..*..`.d~q| +000001b0 45 26 b8 91 26 cc d6 bf d5 4d 26 3f 4f c8 73 99 |E&..&....M&?O.s.| +000001c0 a7 50 9d 14 09 b6 07 1a af aa 9b 02 31 b2 c6 22 |.P..........1.."| +000001d0 ca 3e e0 2f 27 87 27 2e 96 f4 b4 ba c6 01 77 d3 |.>./'.'.......w.| +000001e0 52 09 ea 79 9b ee b4 6a 41 29 b1 de de 74 6b b2 |R..y...jA)...tk.| +000001f0 11 b1 58 90 5c 6a 0d b5 51 4b 2e 1b 3e e6 f4 17 |..X.\j..QK..>...| +00000200 8e 8a 1d 1e c8 bd 55 3a fc f4 90 73 14 5f 63 1b |......U:...s._c.| +00000210 22 f0 81 fd 70 05 ca b2 fd 90 3c 9d 2e 73 d3 8c |"...p.....<..s..| +00000220 f0 0a 59 9e 46 fe 8c e2 22 17 03 03 00 a4 78 2b |..Y.F...".....x+| +00000230 ec c5 5e 71 85 c6 e5 5d 75 f3 b5 3d 55 de 1c f9 |..^q...]u..=U...| +00000240 06 92 be df ef 77 31 4b f3 13 b8 c7 7a 68 ec 1b |.....w1K....zh..| +00000250 e2 7c c0 ff b5 6f c2 62 bd 34 23 fd 6a 39 c7 ef |.|...o.b.4#.j9..| +00000260 91 47 77 7e 32 2e a9 b2 85 ae 01 3e 61 43 8a 93 |.Gw~2......>aC..| +00000270 72 14 e1 b7 a5 14 4b ca 6e cb 4d e7 a3 cd 74 98 |r.....K.n.M...t.| +00000280 4e 08 a4 d4 34 ea 73 17 4b 20 8b 40 03 10 a0 33 |N...4.s.K .@...3| +00000290 c9 2f c0 4c 3b a2 2c 61 3c ab ec e3 c0 e8 e2 d6 |./.L;.,a<.......| +000002a0 a0 85 fa 26 26 a9 65 dc 70 5b 8f b7 3f 9e b3 fb |...&&.e.p[..?...| +000002b0 44 36 62 79 75 af ef 94 05 6b d2 44 07 51 ae 91 |D6byu....k.D.Q..| +000002c0 ea e3 e7 d2 f4 2f 19 17 38 aa 1c ae cb 88 db 0b |...../..8.......| +000002d0 66 5a 17 03 03 00 35 6d d9 23 d1 3c c3 25 7b 5d |fZ....5m.#.<.%{]| +000002e0 8a 1a 41 00 00 f8 0c a7 3e 53 4e e5 f6 cb 11 3f |..A.....>SN....?| +000002f0 9c 66 62 80 98 6c 55 19 b1 6f 00 5d 46 93 d2 0b |.fb..lU..o.]F...| +00000300 79 58 1c d6 d7 f9 fc fb 38 c5 32 63 17 03 03 00 |yX......8.2c....| +00000310 13 72 a9 fd 60 5c cf 68 b3 32 15 04 33 1c e1 5f |.r..`\.h.2..3.._| +00000320 11 11 9e 8c |....| >>> Flow 4 (server to client) -00000000 17 03 03 02 90 a2 a0 e4 83 63 ad 8d d5 45 25 dc |.........c...E%.| -00000010 41 02 31 28 8d b0 87 85 66 b8 9c 36 f3 1f 97 87 |A.1(....f..6....| -00000020 8c c9 e6 b4 67 1e 42 ab 84 c4 eb 0a 41 b1 0f 50 |....g.B.....A..P| -00000030 25 c3 7e 69 20 cf 8a d0 56 79 61 e3 e1 5b cc a4 |%.~i ...Vya..[..| -00000040 24 5a c7 2d 3b 17 33 92 59 6c 7e 29 a9 a2 2c 73 |$Z.-;.3.Yl~)..,s| -00000050 3e b1 65 32 3d 6a 2b 61 5d 76 c4 66 0e 4f f1 da |>.e2=j+a]v.f.O..| -00000060 dd d3 7e 29 3d f5 42 99 9e 04 60 a4 9a a1 c0 f7 |..~)=.B...`.....| -00000070 54 4e d5 58 73 85 02 83 38 ba 4e 93 9a 69 68 07 |TN.Xs...8.N..ih.| -00000080 71 c9 a7 e4 83 51 4c 11 21 26 b2 dd ad 4b 2a ef |q....QL.!&...K*.| -00000090 23 9b b0 f6 d6 96 9f 99 1f 69 fe 35 28 86 49 bc |#........i.5(.I.| -000000a0 ec 97 11 4d 4e b7 c1 c2 da 6c ae c7 40 b9 2a 1f |...MN....l..@.*.| -000000b0 ff 9d 9d ea 90 b9 61 6f 76 ae fe 55 70 f6 ee 54 |......aov..Up..T| -000000c0 54 62 6d 19 9a fc 40 16 e5 c2 e2 3e d1 68 c6 09 |Tbm...@....>.h..| -000000d0 be 54 64 25 8a a0 2b e6 b3 14 7e 74 17 91 f0 de |.Td%..+...~t....| -000000e0 87 e8 3c 3e 58 8d 1b b2 4e 2d 7d c1 f8 aa 16 ca |..<>X...N-}.....| -000000f0 2a 3e 8f aa 52 14 e4 f2 2a b9 6e 62 46 ab ed 3a |*>..R...*.nbF..:| -00000100 ff a1 51 53 92 7a 78 c4 ed 9d fd 1f b4 62 2e e7 |..QS.zx......b..| -00000110 cc 9d 09 fa 06 2e 9e 55 e5 98 9e b8 be e9 12 94 |.......U........| -00000120 33 58 6b be 71 a1 4f f8 a0 83 85 f8 7e b4 28 a7 |3Xk.q.O.....~.(.| -00000130 ee 1b ac 3c 46 b6 ac 60 af 5d cc b0 a8 2c 3f 95 |.....Z.| -00000210 6b 5c d2 4d b2 20 35 63 9e 83 c0 7e 83 60 46 57 |k\.M. 5c...~.`FW| -00000220 c0 80 0d d7 b9 9f 14 c0 58 2d 48 2a cc 8c 1d 32 |........X-H*...2| -00000230 2c 34 ec 10 f2 34 b4 28 e1 0e 83 38 c4 2e 5a 09 |,4...4.(...8..Z.| -00000240 ff e6 3f d3 9a 32 8e 33 9e 31 18 e5 1d 6b 97 12 |..?..2.3.1...k..| -00000250 9b 93 84 86 62 8a 0a 8a ab 8d 37 68 af a1 ec 8e |....b.....7h....| -00000260 38 c8 47 ef 10 f8 64 7c e1 13 0a 33 eb c2 4b bf |8.G...d|...3..K.| -00000270 47 49 6d 93 3c c9 44 aa 74 67 a9 93 dd de 66 47 |GIm.<.D.tg....fG| -00000280 1e b3 55 47 b1 16 88 68 82 24 d6 b8 81 b0 5a b6 |..UG...h.$....Z.| -00000290 27 7a f8 0b 5c 17 03 03 00 1e a5 9b 8c d2 3e 96 |'z..\.........>.| -000002a0 f3 49 7a ed d9 fc 33 62 15 12 43 76 11 f9 dc fc |.Iz...3b..Cv....| -000002b0 ea d2 d7 87 22 6f 9e 3d 17 03 03 00 13 e1 30 4e |...."o.=......0N| -000002c0 4c 1d 02 78 57 a5 ee 8f 1c 5f 19 f0 57 3a 8d 7e |L..xW...._..W:.~| +00000000 17 03 03 02 90 2e 71 49 81 51 71 9c 2f da 1e dc |......qI.Qq./...| +00000010 22 c0 a2 59 13 ad 41 ae 7b 18 56 d0 00 7b b5 cf |"..Y..A.{.V..{..| +00000020 47 e7 bc 8e 92 9e dc 88 78 3e 40 8e 8b 01 b0 30 |G.......x>@....0| +00000030 8f 7e 88 95 77 ed de 31 bf c5 78 ba a1 44 55 ec |.~..w..1..x..DU.| +00000040 b6 df 49 a9 df a0 92 b2 be b7 2c 1c 39 1b a0 63 |..I.......,.9..c| +00000050 18 e5 12 b7 5a a5 85 14 97 7d 9e e1 80 b7 15 c5 |....Z....}......| +00000060 2f ca e3 3b 48 91 42 07 f4 49 25 99 2c ee b8 b5 |/..;H.B..I%.,...| +00000070 05 a2 34 7d bb c0 87 4c f3 a8 07 89 fe b7 08 2c |..4}...L.......,| +00000080 e2 0b 27 6f 49 88 80 2f 3b 9c ae a6 3e 38 74 23 |..'oI../;...>8t#| +00000090 53 70 f5 84 0f 77 d4 94 68 f9 86 18 2f dc e1 94 |Sp...w..h.../...| +000000a0 3c e1 2c f2 84 07 f5 3c 88 99 63 ef a1 96 73 f9 |<.,....<..c...s.| +000000b0 46 43 c2 92 be 04 b8 6b 9e db e1 7f 63 b1 d4 15 |FC.....k....c...| +000000c0 36 2c 37 e3 c2 24 b2 4c c5 60 96 15 3b 20 01 67 |6,7..$.L.`..; .g| +000000d0 e5 52 9a 4f 17 6c 0a f1 a0 a3 31 48 e5 31 8e d3 |.R.O.l....1H.1..| +000000e0 b9 49 38 d2 55 4d 40 8e a2 bf dd 0f 19 25 42 55 |.I8.UM@......%BU| +000000f0 06 57 22 c5 06 c9 59 13 f7 c7 56 b1 68 20 f4 62 |.W"...Y...V.h .b| +00000100 19 0a 5d ab 5f 81 c6 50 b7 a2 f8 ce 55 f2 b6 6c |..]._..P....U..l| +00000110 67 39 75 82 1b a1 6b 85 58 37 c0 a5 12 64 a6 ee |g9u...k.X7...d..| +00000120 2e f8 f0 dc d2 50 08 47 0f 15 c8 37 2d ba 69 64 |.....P.G...7-.id| +00000130 e9 ea 7b fa 16 91 ed b6 24 0d fb e8 aa 74 6b 74 |..{.....$....tkt| +00000140 2d 78 2d 1c 2b 98 8e ab 74 6d 06 a3 00 8a e5 be |-x-.+...tm......| +00000150 c0 68 09 05 e0 95 0d c1 27 8a cf 05 09 0a 7c b9 |.h......'.....|.| +00000160 a1 3a 9b 45 3d d4 5d 16 64 5b d5 a3 6c d2 12 78 |.:.E=.].d[..l..x| +00000170 4f 6b ba 3a fe ad 35 9e 5e 0c 5f d4 f3 32 a2 0b |Ok.:..5.^._..2..| +00000180 f1 af d4 e5 f1 91 c6 6c 47 6b 30 a7 2f bf 60 05 |.......lGk0./.`.| +00000190 e3 c0 2d 9c 34 f2 f0 6b f2 ce 7f 19 d0 86 26 b0 |..-.4..k......&.| +000001a0 e1 7e 7b e0 0e 3d e7 f6 3c 7f 42 bc 9f a6 0a 25 |.~{..=..<.B....%| +000001b0 eb cf 1b 4b 44 10 df 7d f1 1f e3 a9 ea 4f 6d 52 |...KD..}.....OmR| +000001c0 70 b2 f0 0b a9 c5 bc 66 46 21 4d 92 c4 c3 f6 71 |p......fF!M....q| +000001d0 eb 46 c0 f3 e7 6f 4a ee 37 ad 8a ea 3b aa 51 79 |.F...oJ.7...;.Qy| +000001e0 d5 d5 18 7e e8 ae 31 1a db 16 04 d8 3c 94 17 f6 |...~..1.....<...| +000001f0 ad 8f 8b 5d 7d 14 fb b4 e9 b4 7e bb 75 f1 df de |...]}.....~.u...| +00000200 6f 0a 0d a8 f9 b5 d8 53 73 36 88 77 db e6 0e 84 |o......Ss6.w....| +00000210 fc 94 6b c9 cc 74 0f 01 40 39 c4 d0 66 2d 23 76 |..k..t..@9..f-#v| +00000220 9f 0d a1 53 94 a2 dd 56 83 9e 62 39 10 13 39 40 |...S...V..b9..9@| +00000230 55 09 b6 77 d2 22 9c 49 ef f9 af 93 ee d1 cf 3d |U..w.".I.......=| +00000240 1f 29 ef 27 94 3c 6d 23 d6 7c 33 e1 f7 cd 70 05 |.).'.Z..~..i7.| +000002b0 b0 90 43 11 e6 76 5e 2d 17 03 03 00 13 4b 69 75 |..C..v^-.....Kiu| +000002c0 3d 46 35 f6 85 4b 0b 6e f4 6b 30 2a 28 37 3c f1 |=F5..K.n.k0*(7<.| diff --git a/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedAndEd25519Given b/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedAndEd25519Given index 6943bf65690f77..016b7da4dfe90e 100644 --- a/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedAndEd25519Given +++ b/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedAndEd25519Given @@ -1,149 +1,151 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 ca 01 00 00 c6 03 03 e4 83 a9 75 06 |..............u.| -00000010 0f 8b c9 35 1e 62 89 7f a8 df 7c 93 b6 f0 8f 94 |...5.b....|.....| -00000020 ea 31 dc 66 11 66 bd 77 33 54 bf 20 38 77 15 8d |.1.f.f.w3T. 8w..| -00000030 b4 21 50 72 6f 95 61 6c 15 b8 35 c9 92 10 72 99 |.!Pro.al..5...r.| -00000040 bc 41 03 53 7c 5e 7b b3 a4 2e b4 19 00 04 13 01 |.A.S|^{.........| +00000000 16 03 01 00 ca 01 00 00 c6 03 03 c3 82 3b bf 9c |.............;..| +00000010 31 25 31 c7 8d 18 7f c1 28 22 0f 23 0c 13 b8 81 |1%1.....(".#....| +00000020 a8 e8 35 00 92 0a 13 21 8d 76 43 20 9e 3a 45 91 |..5....!.vC .:E.| +00000030 73 a0 99 70 ab 96 99 2b ae 4a b1 2a 89 3c 74 94 |s..p...+.J.*.>> Flow 2 (server to client) 00000000 16 03 03 00 7a 02 00 00 76 03 03 00 00 00 00 00 |....z...v.......| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -00000020 00 00 00 00 00 00 00 00 00 00 00 20 38 77 15 8d |........... 8w..| -00000030 b4 21 50 72 6f 95 61 6c 15 b8 35 c9 92 10 72 99 |.!Pro.al..5...r.| -00000040 bc 41 03 53 7c 5e 7b b3 a4 2e b4 19 13 01 00 00 |.A.S|^{.........| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 9e 3a 45 91 |........... .:E.| +00000030 73 a0 99 70 ab 96 99 2b ae 4a b1 2a 89 3c 74 94 |s..p...+.J.*.8..`...1&.| -000000b0 db ef e2 b1 99 55 c8 f9 f3 a0 81 19 12 a7 63 85 |.....U........c.| -000000c0 5a 26 2d d1 1c ad 5d ae d6 4b 66 93 62 d7 fe 08 |Z&-...]..Kf.b...| -000000d0 40 e9 fa 16 8b 89 f8 04 e8 33 67 20 2b 21 91 a0 |@........3g +!..| -000000e0 c6 0a 87 ff 17 03 03 02 6d 79 97 6c 2f f6 01 7b |........my.l/..{| -000000f0 3a 49 0e 1a 00 96 10 fd 7f 77 db 76 b2 d4 e4 68 |:I.......w.v...h| -00000100 46 4e 4f 3c 64 54 ca 27 9a 5c 78 98 f4 96 a4 fe |FNO.....u.(..| -00000120 ff d8 ec 27 2c f2 4c e5 a0 6e 88 ce 67 6e 35 f4 |...',.L..n..gn5.| -00000130 e5 d5 96 2d 40 af fa 88 12 8a 48 24 2c f9 82 f5 |...-@.....H$,...| -00000140 cb a4 6e 95 a6 53 bc 79 f7 6a ef 66 77 bc 46 f0 |..n..S.y.j.fw.F.| -00000150 1b 0d 6b 5c 76 82 15 c4 d0 1c dd ec cc ce 09 93 |..k\v...........| -00000160 ce 21 55 9b d8 8a 11 1b 0c 24 fa 9e 5f 29 4a f1 |.!U......$.._)J.| -00000170 2a 2e ad c0 6d 6d 46 06 5b c9 75 b3 3e 32 45 67 |*...mmF.[.u.>2Eg| -00000180 05 26 cc d8 a8 4a a9 b1 67 71 a6 82 1c dc f0 15 |.&...J..gq......| -00000190 6d 25 f5 6e be a2 5f 45 39 dc d1 2e df fa e1 e9 |m%.n.._E9.......| -000001a0 48 ca 7a 78 fa 0e 53 d1 5c 8f c2 40 91 d5 fa 40 |H.zx..S.\..@...@| -000001b0 7e a1 52 23 c8 56 1f 31 17 91 5c 38 bb 54 56 f3 |~.R#.V.1..\8.TV.| -000001c0 1e 14 90 43 b7 ef fd 56 b5 ae 13 90 97 dc 60 15 |...C...V......`.| -000001d0 67 72 fc c2 0a 32 90 be ec de 69 16 d3 1b 22 2c |gr...2....i...",| -000001e0 25 9f 91 27 a7 6d 8c a4 de 02 fd 0e da bf ca 71 |%..'.m.........q| -000001f0 77 9b 56 b8 07 e8 80 00 9b d9 36 1c 09 4f 9f 54 |w.V.......6..O.T| -00000200 76 d5 76 f4 9a 03 94 bb 9e 93 f0 b5 3c a1 71 ec |v.v.........<.q.| -00000210 b3 83 3a 06 b4 46 97 bf ef bb f0 26 94 4e b0 08 |..:..F.....&.N..| -00000220 3b ec 81 20 66 92 11 85 a0 c2 90 fd c5 bc ae 39 |;.. f..........9| -00000230 2c 87 ec e4 5d 59 ee b4 e9 0d f7 2a e0 3b 2a 94 |,...]Y.....*.;*.| -00000240 1a 79 2f e8 5c 88 d3 61 2e 47 c0 f3 c4 01 84 a9 |.y/.\..a.G......| -00000250 cf f6 36 13 cb 4b 0b f7 9a 14 f1 d5 0e 10 80 fd |..6..K..........| -00000260 11 79 20 20 ae 56 5e de 58 53 19 38 26 e2 ac bc |.y .V^.XS.8&...| -00000270 0c 40 38 8b f9 67 62 4c 42 7d 18 4f 27 e9 53 96 |.@8..gbLB}.O'.S.| -00000280 78 4b fa 44 fe c2 c3 d9 99 f2 2c 59 2b 2b 2c 88 |xK.D......,Y++,.| -00000290 5b dc a8 98 3d 17 14 09 70 ce e4 02 8b 3c 5d 94 |[...=...p....<].| -000002a0 44 ac ba 57 2d a9 bf b8 70 e9 b8 a8 c3 b8 90 da |D..W-...p.......| -000002b0 ec b1 b4 57 d6 e3 0f 41 82 bb 21 4a 57 dc ac 4b |...W...A..!JW..K| -000002c0 89 34 75 fc c4 56 6b 70 3d 83 2b fa be c8 2b cd |.4u..Vkp=.+...+.| -000002d0 f8 4f 9f 9e 9a 0e d2 d0 46 cd 21 a5 f7 07 a6 2a |.O......F.!....*| -000002e0 85 7b 30 92 78 a2 da a5 1d 1c 1c 54 63 4b 66 b2 |.{0.x......TcKf.| -000002f0 f1 a7 c4 43 57 97 7f 28 37 e7 15 62 9b 1c f5 90 |...CW..(7..b....| -00000300 0c 19 36 1a c1 48 48 e5 7d 56 93 3c 13 e3 cd 6a |..6..HH.}V.<...j| -00000310 aa aa ba d5 24 95 c7 df 9c a9 76 6c 07 bf 09 2d |....$.....vl...-| -00000320 4b 7b 55 94 37 ec d4 69 ce ab 0f 48 37 74 37 99 |K{U.7..i...H7t7.| -00000330 83 0d 60 8a 73 56 fb e2 9e 0c 39 0e 23 bf 68 b2 |..`.sV....9.#.h.| -00000340 92 51 12 bc cf 1b af 9d 7c fe 77 14 c8 66 4a 6f |.Q......|.w..fJo| -00000350 91 06 55 6a 11 61 17 03 03 00 99 c2 bf 26 a6 fa |..Uj.a.......&..| -00000360 67 16 a3 b9 1f 36 f8 4f 5d 59 b1 be 43 3a 70 01 |g....6.O]Y..C:p.| -00000370 c0 3a 4b c5 20 b1 22 49 04 22 bb 7f 5f f4 bb f8 |.:K. ."I.".._...| -00000380 35 03 0e dc ba ce de 2a 25 ea 96 dd 3d 64 34 90 |5......*%...=d4.| -00000390 30 f8 34 22 bb e4 94 00 bb b3 ea 3c d2 87 90 9a |0.4".......<....| -000003a0 86 76 6b b7 e3 78 fc 35 10 50 ce b6 c0 71 52 ae |.vk..x.5.P...qR.| -000003b0 a5 f7 bf 8c 5e 5d c1 96 c7 92 6f f0 04 87 d9 a8 |....^]....o.....| -000003c0 72 f4 9e ed 6d ab 28 42 7c c8 60 39 81 66 74 a1 |r...m.(B|.`9.ft.| -000003d0 79 79 6a 59 02 29 b8 14 12 34 a7 96 8f e0 c1 d6 |yyjY.)...4......| -000003e0 4e da e2 63 22 c1 60 b1 87 64 d3 80 b9 c4 df 9a |N..c".`..d......| -000003f0 5f 2c 22 91 17 03 03 00 35 9a 62 4d a2 ba 27 31 |_,".....5.bM..'1| -00000400 fc 8e 23 cc 5f f0 5c 8c 9b c1 b0 ae 7b b8 fa e2 |..#._.\.....{...| -00000410 f3 af 6c 6c ac 86 1e e1 2b 9f 14 a1 f3 5f b5 f9 |..ll....+...._..| -00000420 76 b6 dd 73 f5 6a 08 29 f1 29 9e 79 87 aa |v..s.j.).).y..| +00000080 03 03 00 01 01 17 03 03 00 17 84 17 5a 31 cf 40 |............Z1.@| +00000090 61 37 2b 5a 28 91 d9 e1 7b f2 50 14 8e 7b 40 30 |a7+Z(...{.P..{@0| +000000a0 4b 17 03 03 00 52 ad d1 15 51 c8 89 48 44 23 64 |K....R...Q..HD#d| +000000b0 8a b6 8d ac 08 47 ed 65 1d a3 ea cd 3c 89 43 4b |.....G.e....<.CK| +000000c0 36 28 4b b7 bc e4 93 93 ce 37 4c f5 63 ad 3b 3f |6(K......7L.c.;?| +000000d0 05 b9 92 56 88 35 89 30 5e 40 2d 46 ac 9d 1f ec |...V.5.0^@-F....| +000000e0 3d 0a 2a 32 d0 98 79 97 e2 93 2b b0 43 35 37 90 |=.*2..y...+.C57.| +000000f0 64 e3 a6 fd 09 01 14 61 17 03 03 02 6d 3c 5b ed |d......a....m<[.| +00000100 d8 cf c6 d9 e4 b1 7b 15 dc a1 c1 f6 88 87 1a 5c |......{........\| +00000110 22 37 53 87 a7 10 13 50 1c 1c 65 70 09 67 f3 63 |"7S....P..ep.g.c| +00000120 e7 63 88 76 50 45 e6 a8 1a 38 93 3b c0 ef fd c0 |.c.vPE...8.;....| +00000130 17 c2 c5 04 d6 15 c6 e9 70 bd d2 be 9a 38 50 8a |........p....8P.| +00000140 2b 4c a2 ea 4e ad 80 09 8f 24 2c 17 4d 17 e4 d8 |+L..N....$,.M...| +00000150 a1 47 13 fb 76 18 2d 04 8c 39 23 03 67 64 d2 d6 |.G..v.-..9#.gd..| +00000160 3d c8 4f 1d 1c 59 44 08 80 52 6e f3 3f 90 35 bc |=.O..YD..Rn.?.5.| +00000170 7e 58 1e 16 2c b0 7d 89 2a 4a a9 53 5a 2a ce 65 |~X..,.}.*J.SZ*.e| +00000180 c9 85 08 0b 6d e9 a4 9c e7 25 30 49 3c 85 bb 83 |....m....%0I<...| +00000190 7c 78 be 9d ee de 8f fb 01 d7 59 d5 4e 42 bc 21 ||x........Y.NB.!| +000001a0 e6 ba ed 0c 84 3c 8a aa 70 43 8a 46 c5 7f 81 65 |.....<..pC.F...e| +000001b0 b3 18 73 d9 73 8f 4e 67 5e 64 e3 92 0c 2f 03 e4 |..s.s.Ng^d.../..| +000001c0 81 ec 8a a2 0c 7d e9 97 5f 4b 5f 7e 55 48 24 8c |.....}.._K_~UH$.| +000001d0 ea d4 34 18 2b 88 6e 07 17 a9 7a be 6a 99 59 76 |..4.+.n...z.j.Yv| +000001e0 f5 1d 9b e9 4c ad f7 eb 94 8b bb 7d 93 46 ba a7 |....L......}.F..| +000001f0 d5 f8 3b 82 91 6d 5c 7e dc 0f da 08 19 06 40 f0 |..;..m\~......@.| +00000200 fd 16 68 24 fc 85 84 12 65 71 cf 91 63 46 ea 2f |..h$....eq..cF./| +00000210 d6 45 be 8a f3 27 0a 66 cb ed 8a d0 38 02 59 34 |.E...'.f....8.Y4| +00000220 4a 3d dc 6d 2d ef a4 f6 b4 c0 9d 29 85 b3 da 20 |J=.m-......)... | +00000230 4d 60 ec 90 eb 0f f1 5f 83 b4 bd 6a d5 0b b1 01 |M`....._...j....| +00000240 dd df a3 4c 1c b1 f6 b3 0c e0 8d 94 59 84 08 c4 |...L........Y...| +00000250 31 58 83 11 0b 04 5c 5e e3 ef 5a f4 6d 95 3a ca |1X....\^..Z.m.:.| +00000260 7a 11 b9 db e4 dd 38 66 65 b1 24 40 17 22 8f 90 |z.....8fe.$@."..| +00000270 f3 88 c6 a0 76 21 67 be 1f 6c aa ca ad c1 58 20 |....v!g..l....X | +00000280 dd c7 c0 20 32 a8 d7 db 9a 81 c9 82 38 a0 76 6b |... 2.......8.vk| +00000290 54 cf 79 cb 01 f1 59 50 3b 99 e2 68 96 2f 49 01 |T.y...YP;..h./I.| +000002a0 f6 7a ec a5 4f a0 91 be fb 85 52 ef 9b 6d 0c c1 |.z..O.....R..m..| +000002b0 36 82 a2 3f 0b 4e 4e 01 31 43 29 2a c0 26 41 83 |6..?.NN.1C)*.&A.| +000002c0 31 7b 9a b0 bf 0d 0f 96 b7 64 60 4d 07 2b 29 7e |1{.......d`M.+)~| +000002d0 d2 f6 c3 92 89 50 a7 19 b7 5b 15 6d 9c 0b 6e e3 |.....P...[.m..n.| +000002e0 01 d4 58 18 c0 22 95 e6 42 a7 1b 65 a8 2f 5f 6e |..X.."..B..e./_n| +000002f0 fc bc 00 ed 01 21 ec a9 6c ac 9e c7 c0 f0 f5 fc |.....!..l.......| +00000300 bd 0c c1 26 98 20 fa 1d a5 2a b6 fc 01 17 1c ba |...&. ...*......| +00000310 62 17 7d 39 25 76 bb 8c 77 5a bb 5e 74 22 f7 82 |b.}9%v..wZ.^t"..| +00000320 f1 65 8a 25 b5 05 53 db 84 de e6 59 1a a6 e7 09 |.e.%..S....Y....| +00000330 45 8e 14 fc e2 bc 9c 20 42 28 8d e2 ae 48 ad f4 |E...... B(...H..| +00000340 a0 ab 6b 3e 23 02 0a 0f ee 92 68 2f 29 1c 7e 4f |..k>#.....h/).~O| +00000350 8b 92 85 32 15 2f c1 dc fa 87 14 1d 21 3e 95 c6 |...2./......!>..| +00000360 e4 96 f1 66 d6 af bb eb d8 d4 17 03 03 00 99 31 |...f...........1| +00000370 1a 82 6b b3 f2 1d 12 83 7c df 15 6a 5b a8 af 96 |..k.....|..j[...| +00000380 a3 81 92 59 7a 19 c5 6b b4 ec e0 90 14 f1 00 ca |...Yz..k........| +00000390 0d d5 15 c5 24 49 a7 fc c2 2f 40 89 61 c1 6d d4 |....$I.../@.a.m.| +000003a0 da 39 f7 c3 42 50 e2 47 62 b2 5f 2b 91 70 e7 61 |.9..BP.Gb._+.p.a| +000003b0 57 e0 a3 20 a2 79 1f 18 ce f9 ad 5d ea 87 fa 16 |W.. .y.....]....| +000003c0 85 4b 50 a0 43 8f 0a 40 d7 93 cf 82 b5 82 6c 11 |.KP.C..@......l.| +000003d0 b7 cb 7c b2 4d 7d 23 ec a7 f8 2f b6 69 a8 83 29 |..|.M}#.../.i..)| +000003e0 4a ff 72 86 8e 05 79 91 a7 ed 4b 3d c8 86 fc 29 |J.r...y...K=...)| +000003f0 74 1e 88 7b a5 b2 2c 6e 62 08 2e a9 02 c9 05 d8 |t..{..,nb.......| +00000400 66 db 9c 28 35 79 ed 6f 17 03 03 00 35 24 4d e5 |f..(5y.o....5$M.| +00000410 81 c9 80 a9 1b c8 a9 20 27 c8 17 b3 0e 42 91 70 |....... '....B.p| +00000420 cb 4d ac 6f b8 b7 9a 96 94 95 fb ce f1 51 65 21 |.M.o.........Qe!| +00000430 ff 2c c5 53 42 1c b3 1c b1 9b 38 1a 1c a0 bf 3c |.,.SB.....8....<| +00000440 92 06 |..| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 01 50 ef 68 27 d6 ec |..........P.h'..| -00000010 76 00 e1 c6 ed 3c f6 a1 83 b4 4b 26 28 ba 0f d6 |v....<....K&(...| -00000020 2a fd f0 4a 10 8f 9c ed 84 3f 0a 0e 5b 77 e2 7d |*..J.....?..[w.}| -00000030 1e 03 2a 76 5b 2b 87 78 ad bd 45 8b 03 b3 8d e7 |..*v[+.x..E.....| -00000040 b7 66 ca 5e 36 f8 53 87 90 3c 1a 33 46 1d 32 4f |.f.^6.S..<.3F.2O| -00000050 f1 90 fb 36 da 96 1c 1a db 9f 9b e6 9f 85 f8 13 |...6............| -00000060 7d e1 ab e1 ca c6 05 df 15 ea af dd 55 58 c7 5f |}...........UX._| -00000070 de 62 1b 93 60 a4 fc 39 0a ef 95 bc 0c ca 8f 84 |.b..`..9........| -00000080 98 0a 6d 5b fd c6 0c ad 02 7f 0c f8 b4 be fe 5a |..m[...........Z| -00000090 fb 22 00 08 09 5d c7 47 76 89 e5 06 d1 90 5b e6 |."...].Gv.....[.| -000000a0 63 64 06 28 37 d9 1b e9 0d 27 45 f7 72 30 d7 f2 |cd.(7....'E.r0..| -000000b0 db 8e bf 95 97 29 43 e7 16 bf a0 59 9c fa d9 59 |.....)C....Y...Y| -000000c0 a0 a6 9b 1f b5 74 80 87 d0 61 2f d5 a5 ac dd b2 |.....t...a/.....| -000000d0 8d 27 fc e6 68 eb 07 b3 3d 97 a9 93 5b 35 99 e9 |.'..h...=...[5..| -000000e0 ba 99 fe 49 d6 39 1a 0a 38 98 cd 47 b9 67 9b 9a |...I.9..8..G.g..| -000000f0 77 65 45 f8 48 fb d3 1c 0f a2 2e af e0 29 68 bc |weE.H........)h.| -00000100 81 24 3b 9b 36 0a ef 51 75 ff 61 6a d4 6c 59 42 |.$;.6..Qu.aj.lYB| -00000110 54 31 47 e9 02 9e 58 33 9e 89 65 b6 65 db b2 81 |T1G...X3..e.e...| -00000120 bd c1 f4 0a 34 eb f3 26 f5 8d 36 6d da 78 e6 88 |....4..&..6m.x..| -00000130 00 8f 92 24 dc 76 e3 95 dc 13 b5 92 91 ee c0 82 |...$.v..........| -00000140 cb 63 85 b6 59 67 dc 14 2e 2d 58 8e 56 7e 7c db |.c..Yg...-X.V~|.| -00000150 2f 54 01 ed 17 8d 9a 97 22 39 7f 17 03 03 00 59 |/T......"9.....Y| -00000160 a1 f2 0d 19 e7 d8 a8 6d cd ea f6 82 ee 5d 0a 55 |.......m.....].U| -00000170 22 61 11 21 f7 b0 1d 86 a8 4d c2 e2 9b ac bb 87 |"a.!.....M......| -00000180 a2 82 67 ee 78 76 9b e0 c0 00 85 bf 1e 2b ab e6 |..g.xv.......+..| -00000190 f1 43 79 69 a0 3d 04 b7 d9 7f 31 c7 7a b7 4f 5c |.Cyi.=....1.z.O\| -000001a0 9f 62 84 dc f4 6d a1 ce 3d ff 24 88 15 10 4a e6 |.b...m..=.$...J.| -000001b0 5c 12 68 08 3c 55 a2 a9 d7 17 03 03 00 35 f4 d9 |\.h..z....5=.| +000001c0 94 02 6e 65 f2 98 8d 45 3e d5 af 22 46 f0 70 3e |..ne...E>.."F.p>| +000001d0 d4 90 9c 02 0a f9 8d e6 c7 8f 52 af ab 74 67 12 |..........R..tg.| +000001e0 6d 8e 25 f8 03 73 02 78 5d 9c 8b 38 68 fe e9 96 |m.%..s.x]..8h...| +000001f0 c6 83 29 |..)| >>> Flow 4 (server to client) -00000000 17 03 03 01 c2 80 69 97 9a 20 30 2a 1c f4 31 f9 |......i.. 0*..1.| -00000010 0f cf f7 79 c0 01 e1 f3 35 f5 16 a0 33 d6 eb 21 |...y....5...3..!| -00000020 44 db bc c6 c4 91 6b a6 75 da ca d3 63 78 47 8b |D.....k.u...cxG.| -00000030 96 e5 6f 63 2c 77 c0 33 29 d8 3e ee bf 8e 6b d4 |..oc,w.3).>...k.| -00000040 de f7 1b 0e e6 ae ce cd 17 0d 24 77 10 3d e4 89 |..........$w.=..| -00000050 06 07 a3 77 68 ac 20 ec 0b ae 47 41 3b 80 4e 95 |...wh. ...GA;.N.| -00000060 02 aa 13 36 19 03 06 1c 47 b3 f7 f0 4b 6d 5a c6 |...6....G...KmZ.| -00000070 42 14 95 03 20 c7 46 96 42 d3 18 5a 40 bd a1 03 |B... .F.B..Z@...| -00000080 b6 d2 8b f9 ff 2d d1 b1 3c 8a 34 af 23 64 31 7d |.....-..<.4.#d1}| -00000090 46 47 21 b4 82 16 df a2 a4 0f 96 03 4b 38 3d 5d |FG!.........K8=]| -000000a0 d0 d1 78 d1 6e 6a a2 95 c0 a7 e6 ee 07 eb 77 68 |..x.nj........wh| -000000b0 35 78 72 3e df d9 4b e9 1b ab 34 99 2d fe 52 99 |5xr>..K...4.-.R.| -000000c0 6e 57 63 13 39 ae 5e ce b6 43 21 07 fd fe b7 6d |nWc.9.^..C!....m| -000000d0 8f 72 a4 f8 7e 0a 56 60 61 d5 5a d4 01 b3 47 8e |.r..~.V`a.Z...G.| -000000e0 f0 48 cd 85 61 a3 d2 d1 eb ba 04 39 6b 5e 5f fc |.H..a......9k^_.| -000000f0 e3 90 c1 cb 3f 40 30 00 5c 94 df bf 5b 89 6d ab |....?@0.\...[.m.| -00000100 15 1e 72 50 ac 56 ee 16 7d 84 4c e6 0c 89 68 fa |..rP.V..}.L...h.| -00000110 d5 8d 5f 09 85 25 5f 8c 70 df 0b b7 94 15 40 20 |.._..%_.p.....@ | -00000120 b1 ff 41 50 5d 9f c6 8a 9b 7f 40 6f dc bd 4f 54 |..AP].....@o..OT| -00000130 d8 1f e6 f1 44 00 11 97 45 ca 80 bc 15 eb 93 01 |....D...E.......| -00000140 dd 54 c6 75 7e 08 b9 38 f5 4d 97 c5 50 56 97 3c |.T.u~..8.M..PV.<| -00000150 3c 72 9a 33 7c 68 b1 73 2b 38 c7 b8 a8 3c 5d af |.....6....[..z| -00000170 f5 74 8d ac a4 0f 38 34 ee cd 08 50 e0 33 b0 e1 |.t....84...P.3..| -00000180 52 e1 5d f2 7d c6 7b 64 c7 46 f6 9e d0 a2 cf 89 |R.].}.{d.F......| -00000190 3c c9 ab 2f fb 8a f3 ba 78 e9 4c c5 2d 62 32 6b |<../....x.L.-b2k| -000001a0 50 4c a0 7e d3 bb 2a 66 57 99 38 69 e8 77 ef 18 |PL.~..*fW.8i.w..| -000001b0 24 af b3 cb e5 c0 37 a2 97 f6 00 d4 68 8a 71 af |$.....7.....h.q.| -000001c0 24 a4 ab 19 07 3b c0 17 03 03 00 1e 60 b4 fc 15 |$....;......`...| -000001d0 f9 9c b1 a1 60 1a ba f5 1b b9 c1 33 f1 8b e5 c0 |....`......3....| -000001e0 48 77 4f 11 42 21 ad 8c d2 d6 17 03 03 00 13 5c |HwO.B!.........\| -000001f0 db 07 5e 65 40 58 74 a4 7f ab 5f cc f0 9a 91 0c |..^e@Xt..._.....| -00000200 17 3d |.=| +00000000 17 03 03 01 c2 48 66 ba b6 48 e4 d9 1a 73 81 09 |.....Hf..H...s..| +00000010 cc 26 f8 2b bd 70 e7 15 c7 29 ac 30 58 21 ee 0c |.&.+.p...).0X!..| +00000020 9b c6 f0 b2 85 23 20 1a 7b ad ce fc 5f b6 ce aa |.....# .{..._...| +00000030 af a3 40 52 bf b3 3f 3d 61 a9 14 d1 eb 3c 21 78 |..@R..?=a......J.........| +00000170 fb ff 37 ce b7 5a 75 34 91 35 de 45 a2 36 71 05 |..7..Zu4.5.E.6q.| +00000180 20 65 d3 e4 fa 48 00 c1 25 3f f2 98 e8 0f 32 2a | e...H..%?....2*| +00000190 ee c5 60 f6 83 25 40 91 f6 13 0b fe 26 a9 f1 4c |..`..%@.....&..L| +000001a0 67 0d e8 c9 25 7e f5 f3 e3 ca 34 c3 e7 0e 4b e1 |g...%~....4...K.| +000001b0 c8 f4 40 94 69 88 a2 81 75 02 7d 6d d4 6a b3 c3 |..@.i...u.}m.j..| +000001c0 61 84 17 29 2a e9 90 17 03 03 00 1e 43 69 c6 7c |a..)*.......Ci.|| +000001d0 1a fa 1a cf bf 78 54 c0 8f b7 8d 83 d8 74 f9 54 |.....xT......t.T| +000001e0 6a 62 2f b1 68 63 69 b7 46 2e 17 03 03 00 13 08 |jb/.hci.F.......| +000001f0 49 af 4e 5f f1 e1 8a 7d 20 4e 55 2b 86 4f 03 a0 |I.N_...} NU+.O..| +00000200 d2 b5 |..| diff --git a/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedAndGiven b/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedAndGiven index 8ba88ebf3fa2ee..2a608bda2618bb 100644 --- a/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedAndGiven +++ b/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedAndGiven @@ -1,177 +1,180 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 ca 01 00 00 c6 03 03 56 2e 3c 64 35 |...........V.>> Flow 2 (server to client) 00000000 16 03 03 00 7a 02 00 00 76 03 03 00 00 00 00 00 |....z...v.......| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -00000020 00 00 00 00 00 00 00 00 00 00 00 20 7c d3 26 20 |........... |.& | -00000030 a0 dc e9 7a 07 71 02 2d 3b 27 85 16 fb 6e 26 e5 |...z.q.-;'...n&.| -00000040 c3 67 e5 0b bd e5 50 8f bd 39 6f 2c 13 01 00 00 |.g....P..9o,....| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 b9 21 2b 29 |........... .!+)| +00000030 79 05 38 92 c6 fe 41 fa be d7 98 08 0a e4 12 b7 |y.8...A.........| +00000040 7a ea 10 5a 0d 1a f2 b4 81 5d 9d 01 13 01 00 00 |z..Z.....]......| 00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 2f |..+.....3.$... /| 00000060 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| 00000070 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 14 |.........._X.;t.| -00000080 03 03 00 01 01 17 03 03 00 17 43 8e 41 a4 04 13 |..........C.A...| -00000090 01 08 9f 6e 1a fd 00 68 38 5c 93 d9 9a f7 1e 36 |...n...h8\.....6| -000000a0 ea 17 03 03 00 3e 9f ae 80 ef f4 20 66 e7 44 fc |.....>..... f.D.| -000000b0 4d a4 3f 0e dc bb 33 47 0f 13 96 fa 03 07 d6 6f |M.?...3G.......o| -000000c0 cc 9f 3c fd 01 f5 87 d9 ec c9 d0 fc dd bf c8 5c |..<............\| -000000d0 0b 3d aa a7 c6 1d 90 18 d9 a1 1a a0 a1 ea 49 32 |.=............I2| -000000e0 9e 45 86 f7 17 03 03 02 6d 61 29 fe 45 7e 5c b3 |.E......ma).E~\.| -000000f0 8a 73 f2 65 c1 90 4e 93 0d 84 b2 bd e4 46 93 c2 |.s.e..N......F..| -00000100 52 3f 07 38 e7 23 db 25 5e 71 98 a9 7d fd e9 ef |R?.8.#.%^q..}...| -00000110 3f 2c d8 9c ae 41 c4 d3 c7 9e d5 6e f0 0c 33 78 |?,...A.....n..3x| -00000120 98 cf bf 22 3d 1f d0 f3 c8 a2 34 e8 ce 5e 0d 37 |..."=.....4..^.7| -00000130 52 e2 b8 e3 50 ea 35 3e e5 59 a0 6d ed 9e 09 36 |R...P.5>.Y.m...6| -00000140 59 20 33 08 a9 41 f4 72 aa 2d 0c b5 d6 96 d9 04 |Y 3..A.r.-......| -00000150 1a f1 d8 45 ed 67 ab f9 15 fa 25 ef 6d 87 72 ad |...E.g....%.m.r.| -00000160 f0 06 59 a5 8e 61 80 8b 28 f9 a0 df 5b b2 a2 3a |..Y..a..(...[..:| -00000170 1c 91 43 18 f4 a2 f3 4e db dc 24 1b c3 0e 77 22 |..C....N..$...w"| -00000180 83 ae 88 9f 8e 8d 48 38 f4 60 51 42 fa f2 a4 de |......H8.`QB....| -00000190 33 78 35 d0 b6 01 3d 7a f5 54 68 51 fd 0e 4c 9b |3x5...=z.ThQ..L.| -000001a0 92 7c a5 01 96 52 7e de 38 b9 b0 ee 60 1e aa eb |.|...R~.8...`...| -000001b0 8c e6 b4 f4 7c 35 d2 0b 9b 83 94 ac ac ce 7e 58 |....|5........~X| -000001c0 51 6e c3 ae 3b cd f5 85 8a 1e 43 78 19 ee dc a1 |Qn..;.....Cx....| -000001d0 a3 d0 93 24 0d 3b 6a 4b cd dc 78 9c 0b 2f bc 41 |...$.;jK..x../.A| -000001e0 46 2f 64 3c 23 95 04 8b 60 75 bf 4d 45 3b e4 1e |F/d<#...`u.ME;..| -000001f0 9c 5b 1a 46 bb f3 4d a9 1b 59 33 b4 40 78 bd ff |.[.F..M..Y3.@x..| -00000200 30 7d d4 cc 5e 83 03 de 8d a3 a6 27 b4 bc 12 6e |0}..^......'...n| -00000210 5e f2 88 e8 b6 60 f3 01 e8 4a 53 c7 a9 fc a1 cc |^....`...JS.....| -00000220 27 45 c1 06 90 38 9a fb 1b 2a 9e ed 9e f6 19 85 |'E...8...*......| -00000230 dd f7 8a 7f 95 08 3a 25 c0 5b 63 96 44 71 c2 16 |......:%.[c.Dq..| -00000240 9c e1 10 69 e5 6a 5c 4a e8 2a ed 6f bd de f5 98 |...i.j\J.*.o....| -00000250 c0 a0 c5 54 7c cc 06 11 b2 54 1a c3 b4 46 c2 b4 |...T|....T...F..| -00000260 97 d8 9c 7d f1 f3 d4 6f 3c a0 ef 18 c5 a6 e9 13 |...}...o<.......| -00000270 e9 f4 9d bf 9b 25 a2 da c6 ba 7a 6d 91 fd 41 a4 |.....%....zm..A.| -00000280 e8 88 e3 79 2c 99 df 4d 21 48 89 57 5a bf 2a 2d |...y,..M!H.WZ.*-| -00000290 72 4e 1e 3a e8 c9 82 7b c0 ff 6b 7c e8 8f 41 bf |rN.:...{..k|..A.| -000002a0 83 19 9e 96 d1 3f 2b 60 8f 7f 0b f8 6c 70 82 dd |.....?+`....lp..| -000002b0 34 da 91 62 17 20 e9 99 2e e2 a9 9a 9d fd 5d f8 |4..b. ........].| -000002c0 a5 c0 ac e8 a3 df 11 b5 df 2c bd e1 e8 0f 7e 0a |.........,....~.| -000002d0 f2 47 4c 92 33 7b 6b 49 e5 30 31 8b 2e 16 81 3e |.GL.3{kI.01....>| -000002e0 79 25 f7 d7 d2 8f 5e e6 e3 2d ed 0d e1 08 97 13 |y%....^..-......| -000002f0 f3 ce 7d b1 36 0d 7c b0 4f 23 6b 12 ef 3a a4 8b |..}.6.|.O#k..:..| -00000300 b5 d0 c9 ee 48 77 70 28 61 ff ad 49 f6 48 4c 37 |....Hwp(a..I.HL7| -00000310 7b 00 c4 01 5d 8b 54 bc 44 5c 5f 98 6f 7d 84 84 |{...].T.D\_.o}..| -00000320 c8 d0 55 88 f9 17 f6 02 f1 84 b6 3c 1a 03 e8 7b |..U........<...{| -00000330 b9 4e 24 c6 d8 0a f6 8c b9 49 c6 10 38 53 e1 10 |.N$......I..8S..| -00000340 8f 91 cd 39 9a 3a e8 c7 10 f3 c3 91 84 3c 8c a6 |...9.:.......<..| -00000350 55 ab f4 f5 ab e7 17 03 03 00 99 e5 40 f6 35 34 |U...........@.54| -00000360 2d 42 3e 7f e1 51 26 56 50 4e 60 b0 2f 65 e3 cd |-B>..Q&VPN`./e..| -00000370 c7 08 0e 96 77 08 c7 f6 4f e6 70 90 bc 80 95 e7 |....w...O.p.....| -00000380 b2 df 98 83 94 4d 9e 5c 8a af d3 45 da e9 d7 fa |.....M.\...E....| -00000390 d8 d2 60 f8 b1 06 d9 27 64 45 4d e8 d3 07 8a bb |..`....'dEM.....| -000003a0 72 7a c6 71 00 7e 8b b0 2b 7d d0 f7 ab 1a bf d2 |rz.q.~..+}......| -000003b0 50 be a9 3a 0c 68 b8 48 9a 91 ee db 26 4d d3 66 |P..:.h.H....&M.f| -000003c0 5b 00 ef c3 cc b8 f2 4e 1e 51 c7 9b 34 3c e3 01 |[......N.Q..4<..| -000003d0 7f 75 4e 41 e4 56 34 ec 14 92 0f 1e 6d dd 51 9a |.uNA.V4.....m.Q.| -000003e0 e0 8b 33 54 df 77 1f ff d3 72 67 4c 62 16 b7 f8 |..3T.w...rgLb...| -000003f0 4f 8f f8 ee 17 03 03 00 35 b7 83 d2 27 a3 15 f2 |O.......5...'...| -00000400 75 55 aa 06 8c 5f c4 fa 0f 43 88 c9 c5 e3 c7 36 |uU..._...C.....6| -00000410 40 6b 35 0b 7e 60 a1 1e 48 ef 46 2c d6 e4 48 80 |@k5.~`..H.F,..H.| -00000420 91 b0 e5 3b c0 58 5d fe 5d bd 0a 6c 19 ea |...;.X].]..l..| +00000080 03 03 00 01 01 17 03 03 00 17 4d 00 8b d1 bc 63 |..........M....c| +00000090 c3 8c af c4 0f 2e 36 78 8d 2b 52 32 f6 2c 29 0e |......6x.+R2.,).| +000000a0 17 17 03 03 00 52 aa 37 4e 5d 56 9a 8d 72 e7 77 |.....R.7N]V..r.w| +000000b0 f1 b7 0c 0b be 7d 63 92 58 0f 09 d7 a0 36 12 75 |.....}c.X....6.u| +000000c0 c9 0d ec d8 44 b4 8f ad dd a1 84 58 3f 0a 7c 47 |....D......X?.|G| +000000d0 0b c1 6f 56 7a 90 bf 82 6d 29 22 8b b3 50 03 ea |..oVz...m)"..P..| +000000e0 18 96 5e f5 1b f0 67 7d 10 c8 3f a4 f5 44 84 b3 |..^...g}..?..D..| +000000f0 f3 af 3e f2 b3 93 a1 57 17 03 03 02 6d 54 0a b5 |..>....W....mT..| +00000100 be 3f e0 77 de 6f e3 d7 83 60 2e 17 94 77 fb e1 |.?.w.o...`...w..| +00000110 11 e8 8d 22 1a 7b ac 36 94 35 d5 29 e5 fa 9e 07 |...".{.6.5.)....| +00000120 62 87 34 f8 b4 c5 ae d5 ee 84 76 c8 fd 30 cf a8 |b.4.......v..0..| +00000130 b1 a4 fe 42 12 e1 36 80 05 76 e2 d2 04 2b 6e c7 |...B..6..v...+n.| +00000140 0d 7d dc 99 de 3d 62 41 2f 3c c2 78 22 ec 6f 37 |.}...=bA/<.x".o7| +00000150 ab 0d 03 22 fe fc f4 82 41 49 44 fd 0a 02 4a 79 |..."....AID...Jy| +00000160 bd 5b 40 63 46 58 cd 84 b6 cf 46 c3 31 68 38 2a |.[@cFX....F.1h8*| +00000170 cf e0 3f ee de 06 ee de f8 e8 3e 37 ab 92 a6 9b |..?.......>7....| +00000180 0a f3 a8 64 6a 81 86 5d ed e3 a2 8c d0 0f 09 1e |...dj..]........| +00000190 ba e9 b0 6f 6b 1e 32 be 03 de 97 81 e0 42 ad 6e |...ok.2......B.n| +000001a0 f6 af bd 12 7a 2f f4 41 63 c4 f4 6a 61 8c e5 d4 |....z/.Ac..ja...| +000001b0 60 99 f2 d5 92 54 2f 28 ee 3f 9b ab fa c1 51 9b |`....T/(.?....Q.| +000001c0 f5 6b d1 d6 6d b1 17 28 95 5f 3c cc e4 41 bf 9b |.k..m..(._<..A..| +000001d0 c5 6e 29 4b 39 f8 ad b0 fe 51 4c 2b 92 76 ce 22 |.n)K9....QL+.v."| +000001e0 ee 5c 9e ad 0e 50 12 3d 2a a0 7a 6d a1 63 18 f7 |.\...P.=*.zm.c..| +000001f0 f2 a0 2a b7 b6 c9 f3 60 54 47 a8 90 dd 90 7d c3 |..*....`TG....}.| +00000200 6e f8 58 1c 28 e7 21 b4 52 33 da 67 67 2b b1 51 |n.X.(.!.R3.gg+.Q| +00000210 b0 41 97 a3 1f 9b cd c8 28 63 c9 f5 e5 d1 a3 3f |.A......(c.....?| +00000220 ac 1c 39 2e 72 1b e5 16 4d 8c b1 6b 4c 35 07 e9 |..9.r...M..kL5..| +00000230 24 a4 98 fb 67 75 4b 76 b7 2d cc 35 d9 a4 db 74 |$...guKv.-.5...t| +00000240 96 56 96 e0 74 8e f9 4a 45 19 30 57 a0 22 27 a9 |.V..t..JE.0W."'.| +00000250 d9 34 d7 73 f0 16 87 e5 be 8e b3 d2 53 f2 9c 19 |.4.s........S...| +00000260 c1 0a df 07 e7 e8 30 fb 44 ac 34 13 2b 22 4c a2 |......0.D.4.+"L.| +00000270 00 da 45 fc 95 1f 83 a0 c9 fa b3 b9 16 95 83 0f |..E.............| +00000280 a9 fa ae 2f b7 bc a1 55 d2 ab 2d 31 af 39 c2 82 |.../...U..-1.9..| +00000290 f7 bf 9a 9a 5c e6 61 d7 f2 9c d0 b5 53 85 58 cf |....\.a.....S.X.| +000002a0 9d 2e 50 eb 46 a1 a4 a0 aa c4 44 66 f7 61 41 0f |..P.F.....Df.aA.| +000002b0 75 68 36 76 11 7d 1e d7 be 0f a4 7a a8 bb 9e 26 |uh6v.}.....z...&| +000002c0 a9 7b ea 30 14 51 37 78 58 2a 92 e2 2c d3 9e 27 |.{.0.Q7xX*..,..'| +000002d0 86 92 83 a0 bc e8 50 ed da 9c 06 f4 d8 3c d5 6c |......P......<.l| +000002e0 a5 3a cc 1a ce 00 55 30 20 74 ad 17 5c bd 98 2d |.:....U0 t..\..-| +000002f0 1f 11 e7 38 99 2d b4 fc 1c 42 c9 ae f1 af 46 3e |...8.-...B....F>| +00000300 67 13 d2 31 16 34 fd e9 26 7c dd bb a0 25 10 7d |g..1.4..&|...%.}| +00000310 ca 94 20 6a b9 10 b7 94 0d 26 31 46 f2 bf 45 a8 |.. j.....&1F..E.| +00000320 ed 8f 08 15 7d db dd 3a ad 45 9f bd 32 8d ff 80 |....}..:.E..2...| +00000330 c6 1f bf 57 5e a0 dc 74 3c 8a b3 8f 03 ef a1 ef |...W^..t<.......| +00000340 12 65 2c c3 16 4f bc 63 3b 62 ab fa ae 9f d1 84 |.e,..O.c;b......| +00000350 79 80 25 04 bc e0 f9 70 98 e1 8c 15 55 ba cf a3 |y.%....p....U...| +00000360 0a 24 64 8c 3a b2 b7 a8 04 c1 17 03 03 00 99 c0 |.$d.:...........| +00000370 87 16 ce 52 d1 e9 83 f3 46 02 91 89 a1 6d 24 17 |...R....F....m$.| +00000380 dc 6a 05 bb c2 fb bd 93 94 05 60 8c 6c 47 f6 4a |.j........`.lG.J| +00000390 d8 ad 42 99 8b 77 92 91 f3 58 1e 10 3b 59 b1 19 |..B..w...X..;Y..| +000003a0 55 2c 3b f2 5b f3 e5 67 b2 e1 9c a8 68 dd 0e b9 |U,;.[..g....h...| +000003b0 67 c4 61 63 b3 bc 15 7b d6 76 7d 44 42 46 57 96 |g.ac...{.v}DBFW.| +000003c0 b3 10 40 39 2b 8c d5 fa cf a0 32 c0 d5 1a aa 96 |..@9+.....2.....| +000003d0 95 3b d5 55 fe 06 d2 34 31 7c 29 95 86 ac 7e f5 |.;.U...41|)...~.| +000003e0 e6 6b 7d a6 b5 52 37 2d 02 56 b8 12 66 33 67 80 |.k}..R7-.V..f3g.| +000003f0 14 5f 78 b3 a3 bf fc 4f cb 16 82 4e f5 c4 1b a1 |._x....O...N....| +00000400 33 3e 6c ed df 7b 4d 0b 17 03 03 00 35 dd de ca |3>l..{M.....5...| +00000410 4d 9e 09 ba 14 09 72 72 23 9f 3c 7e 2c 46 e5 5e |M.....rr#.<~,F.^| +00000420 50 a2 21 02 f7 03 4b 76 ab 4c a8 0b a2 b3 9b 7a |P.!...Kv.L.....z| +00000430 fc b9 5d a2 ac 0b c3 55 15 b8 b8 23 2b 61 46 00 |..]....U...#+aF.| +00000440 1c 5c |.\| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 02 11 9e 9e cf 44 84 |..............D.| -00000010 df e7 23 47 2d 4e 85 fa f2 02 2a 79 4d 3a 3d df |..#G-N....*yM:=.| -00000020 48 95 2e d7 0d 3c d0 05 fc bf bb 23 0a 25 59 76 |H....<.....#.%Yv| -00000030 fd 04 f8 f0 81 88 85 9f 99 f0 55 91 9a 79 8a 39 |..........U..y.9| -00000040 f4 6e 49 92 be ed 1d d1 20 19 77 bc 55 9b b0 da |.nI..... .w.U...| -00000050 d1 c4 9f b5 2f 5c 1a 60 38 e1 92 9b f9 a1 97 6b |..../\.`8......k| -00000060 1b d0 d0 d0 05 7a 41 26 d4 6b 12 a0 b3 c6 33 13 |.....zA&.k....3.| -00000070 4f 74 10 d6 91 fb b0 69 46 46 ca de 7c 05 d6 62 |Ot.....iFF..|..b| -00000080 51 23 12 58 00 ff 25 8e c8 5f 54 85 f4 2f e9 f9 |Q#.X..%.._T../..| -00000090 f1 1e 32 b9 f1 a6 fb 90 9b a9 65 d2 c9 ea a7 1a |..2.......e.....| -000000a0 e8 c9 a8 bb 7e 3d 3f 03 62 fc f4 06 a4 a2 f7 41 |....~=?.b......A| -000000b0 60 f5 f1 df 3e d6 31 51 f4 dc b1 d3 60 4b 84 4f |`...>.1Q....`K.O| -000000c0 99 e2 9f b2 f0 d5 fd b2 f3 5c 24 5f 91 a6 94 cd |.........\$_....| -000000d0 37 61 91 4e cc ae cd c1 b5 cc 58 5f 9a 91 a1 13 |7a.N......X_....| -000000e0 23 42 8a c5 27 4c 66 32 69 9c 75 9f 2a ed 73 1d |#B..'Lf2i.u.*.s.| -000000f0 29 2e 36 50 34 b4 80 d2 08 e6 9f c6 3e da df 9f |).6P4.......>...| -00000100 e0 2a 08 88 47 b6 d3 ff f7 6c 6c 91 1a 8e 53 89 |.*..G....ll...S.| -00000110 53 6c b8 d7 83 37 ab 03 59 e6 4f 41 42 af d4 f9 |Sl...7..Y.OAB...| -00000120 ac 9d ae d1 77 f1 71 79 a0 16 c4 c7 b5 a6 a9 9f |....w.qy........| -00000130 59 da 55 fb c5 88 8c 13 04 c2 06 39 8f ae 7e ca |Y.U........9..~.| -00000140 99 ce cd aa 86 0a 00 bd 56 f1 98 d9 c8 d2 c1 c0 |........V.......| -00000150 df 16 c2 6f 78 da 66 3f cc 43 c7 38 33 1b 3b 5b |...ox.f?.C.83.;[| -00000160 f8 7e d8 b7 ef 4c 6d c7 f6 84 56 8d 76 f7 0d 83 |.~...Lm...V.v...| -00000170 94 e6 ad c5 f3 05 5c f4 17 69 d5 83 98 c3 43 8e |......\..i....C.| -00000180 9f a5 5c 7b 12 ea f4 1f 39 ce 0d 83 15 b1 e6 ce |..\{....9.......| -00000190 c1 35 9b 8c ec c6 d2 d7 f9 02 36 90 24 f4 3a 70 |.5........6.$.:p| -000001a0 ce bb 8d bb 4b b2 aa 64 f0 b5 c0 88 cc 06 e3 9c |....K..d........| -000001b0 f7 70 64 fd 5d 48 f5 c8 07 48 c2 09 c4 07 56 b9 |.pd.]H...H....V.| -000001c0 86 e9 d6 7b dc ac a5 00 2c 1d 80 8f 47 f2 c4 c7 |...{....,...G...| -000001d0 ab dc 7b 41 7b 3d 47 db 09 12 02 9f 1b 63 e7 cc |..{A{=G......c..| -000001e0 38 1d 33 56 e0 ae 63 7a 95 73 5f e6 da 13 53 49 |8.3V..cz.s_...SI| -000001f0 5f 69 e9 ff 86 26 bd 9a dd be 5c 75 e7 69 66 b4 |_i...&....\u.if.| -00000200 27 3f 79 d2 2e 8e 79 9e 89 42 58 20 a2 ca 8c 2a |'?y...y..BX ...*| -00000210 37 b8 99 81 66 3a 31 66 1a 95 4c 47 17 03 03 00 |7...f:1f..LG....| -00000220 99 66 08 07 06 6d 95 fe f1 72 2a 7c de 84 06 b0 |.f...m...r*|....| -00000230 3d d7 d1 6b 47 0e 4d fb 9e ab 55 f7 71 c5 5d 11 |=..kG.M...U.q.].| -00000240 cb c7 fb 45 90 9d 22 eb ec 03 d6 ce 8c 01 ff 81 |...E..".........| -00000250 a2 90 23 1d 7a f3 cb 16 76 a5 05 57 77 f6 af f0 |..#.z...v..Ww...| -00000260 29 6c 6c 39 9e 99 55 c0 38 c3 31 68 49 b3 bf cc |)ll9..U.8.1hI...| -00000270 31 e2 6d a4 4c e7 99 53 2f 31 3f 05 2c 7c 1b 10 |1.m.L..S/1?.,|..| -00000280 60 ce 8b 60 47 16 eb e5 8b be 1b 7a 95 b9 3c 60 |`..`G......z..<`| -00000290 1c d6 b5 13 5f ad b9 bb 13 dd d6 08 8e 70 cb 2a |...._........p.*| -000002a0 26 3e df ce 7c 21 e5 27 7e 27 ec 75 b6 47 a0 89 |&>..|!.'~'.u.G..| -000002b0 2e e1 3c ab 0d 72 90 d1 d5 07 17 03 03 00 35 82 |..<..r........5.| -000002c0 6b 48 4a 9a 63 16 07 8f b2 d3 4a 65 7d e0 c4 e8 |kHJ.c.....Je}...| -000002d0 27 3e ce 4a 0a c8 63 e0 f9 70 98 c0 6a 12 39 ec |'>.J..c..p..j.9.| -000002e0 e1 52 de 73 58 2d f0 7c bc 8d 41 16 be 89 a0 88 |.R.sX-.|..A.....| -000002f0 56 e5 ef f6 |V...| +00000000 14 03 03 00 01 01 17 03 03 02 11 79 f8 bd a7 4e |...........y...N| +00000010 90 84 77 6f d1 a8 95 67 b7 0b e2 7a 4e 16 e9 ca |..wo...g...zN...| +00000020 1c f3 31 df a8 0e 99 02 c0 09 68 de 6a bb c2 bd |..1.......h.j...| +00000030 19 10 30 65 e6 5d d1 43 e2 aa 81 ba ea 3d 30 9a |..0e.].C.....=0.| +00000040 bb a8 66 ef 91 d1 1d 4c cd 93 97 19 f5 b7 52 1f |..f....L......R.| +00000050 c3 4b f8 dd 2f 39 53 54 68 06 dd 90 c1 75 57 9a |.K../9STh....uW.| +00000060 d6 9d c5 79 36 f9 5e bc c6 9b b8 bd 3d 14 40 71 |...y6.^.....=.@q| +00000070 a8 db 5c a8 d0 28 f3 a6 c1 37 29 c1 38 3f a3 51 |..\..(...7).8?.Q| +00000080 29 5c 01 eb 5c 7e c7 09 e6 2b fd 48 6f 53 77 25 |)\..\~...+.HoSw%| +00000090 c8 55 bb fd 68 21 a4 f0 f8 0e 52 41 97 81 20 78 |.U..h!....RA.. x| +000000a0 5e b8 77 3e 4e 6c 0a b1 32 98 2e d4 58 dd 64 fa |^.w>Nl..2...X.d.| +000000b0 8a 3e ba ea 79 67 c1 01 97 d2 5c d6 09 81 57 b0 |.>..yg....\...W.| +000000c0 fa c3 b8 3d 91 ee 7e 06 47 a8 3e 7b 59 74 14 44 |...=..~.G.>{Yt.D| +000000d0 29 28 29 1f 52 00 b8 31 4f ae 60 d5 18 72 fb a7 |)().R..1O.`..r..| +000000e0 21 0f 79 73 d8 e2 5c 6b b7 aa 78 53 f0 83 b5 ec |!.ys..\k..xS....| +000000f0 89 fc d0 fd 84 2e f6 3b 65 1d ac fe e8 75 10 f6 |.......;e....u..| +00000100 52 01 7f 5a 67 6d 1c 1d 82 4d eb 9d 2f fa 54 4e |R..Zgm...M../.TN| +00000110 0f 05 c8 d0 c3 50 6b cb 15 f0 27 5e 1d 15 b4 60 |.....Pk...'^...`| +00000120 85 bc b0 0e 54 3a 10 54 f9 4c 43 db 57 74 96 4b |....T:.T.LC.Wt.K| +00000130 12 1c c8 1b 4c b7 d6 44 e7 df 95 9b 1d 28 cc d0 |....L..D.....(..| +00000140 20 a5 9f 28 5d da 20 1a 46 42 1c b5 88 81 6b 3e | ..(]. .FB....k>| +00000150 0f 7b 37 5f b8 8e e5 b8 57 3c e7 03 6f 4f be 83 |.{7_....W<..oO..| +00000160 5a 3a a9 46 c6 e9 42 68 4f 10 38 bc ba 4d 67 7b |Z:.F..BhO.8..Mg{| +00000170 19 9a b0 bd ce 9a 8f 21 ff eb 82 59 a6 2f 0a 0f |.......!...Y./..| +00000180 b2 eb 76 c0 e2 dc 3f f4 3b c5 26 d3 7e 8c ed c3 |..v...?.;.&.~...| +00000190 5d 0f 80 56 a0 07 76 88 4c ab 04 3c aa 8a 44 65 |]..V..v.L..<..De| +000001a0 a7 5a 7b 31 29 5f 41 b8 12 c0 c0 f1 cd 95 08 0e |.Z{1)_A.........| +000001b0 f7 89 a0 af 91 e6 52 14 5a 2b 39 c5 c6 4e b7 72 |......R.Z+9..N.r| +000001c0 93 e2 a7 6f 00 b4 57 9b 20 e4 90 b9 57 f0 67 40 |...o..W. ...W.g@| +000001d0 e3 87 7d f6 6b f6 87 91 7f f6 3b 99 f2 93 b3 35 |..}.k.....;....5| +000001e0 e7 d4 fe 7c 1c b7 08 4a 00 de 7c 6c 85 df 2d 37 |...|...J..|l..-7| +000001f0 3b 69 62 18 c6 f9 e0 e3 6e 1d 30 cd 35 7d 83 5c |;ib.....n.0.5}.\| +00000200 1d 0c d8 61 df 4b f3 20 77 81 b1 9e 5f 84 79 44 |...a.K. w..._.yD| +00000210 93 b6 30 60 2d 76 4e 6d 4d 18 3a 21 17 03 03 00 |..0`-vNmM.:!....| +00000220 99 30 99 a3 ff 5c 70 58 5a d7 f0 ec 97 a0 d9 b8 |.0...\pXZ.......| +00000230 9e 98 08 f6 ec 87 48 3b c6 46 c2 45 c4 29 ab 08 |......H;.F.E.)..| +00000240 f2 ee 94 4d 89 c9 19 b4 0d b0 f6 df f7 a5 8d 4b |...M...........K| +00000250 2f 53 62 b8 ee 4c 8a cf 81 be fb 7e 16 2f ec f3 |/Sb..L.....~./..| +00000260 03 49 18 93 dc d3 53 67 fc a2 03 f8 f6 ab 26 87 |.I....Sg......&.| +00000270 92 32 7c 06 1a 12 14 29 cf a2 43 e4 36 04 2c ad |.2|....)..C.6.,.| +00000280 68 ea 5f 5a d2 83 81 d7 96 7d 8c 2e 9a 98 14 56 |h._Z.....}.....V| +00000290 a7 a3 23 69 d5 fd 21 9e 2f 81 43 2b 8a ad 2e c9 |..#i..!./.C+....| +000002a0 50 a8 a5 5c 28 b3 a0 62 c0 95 78 92 f7 3b 7b c0 |P..\(..b..x..;{.| +000002b0 1c a9 29 c4 34 28 39 c4 0b ef 17 03 03 00 35 8d |..).4(9.......5.| +000002c0 35 f3 1d 55 34 aa 92 ff 8f e6 2e c2 9b 2f c6 c8 |5..U4......../..| +000002d0 29 69 66 08 db fb 6f b5 22 4e bf 62 35 43 6c 72 |)if...o."N.b5Clr| +000002e0 c6 2a c4 57 8c a9 42 2c b8 5e 82 25 b2 92 aa 82 |.*.W..B,.^.%....| +000002f0 77 5f a4 1f 17 03 03 00 13 cc 12 01 75 9c 9f d3 |w_..........u...| +00000300 3f 17 17 a8 b8 8b b3 ae dc f6 08 34 |?..........4| >>> Flow 4 (server to client) -00000000 17 03 03 02 83 ce c4 10 39 1a fe 62 a2 ff 27 6d |........9..b..'m| -00000010 b7 e3 1d d6 8e b8 a2 7e f5 30 87 35 16 41 fb 04 |.......~.0.5.A..| -00000020 3c 79 9e 02 9b 06 4c a7 ba 01 5b cf 94 bc c8 08 |.| -00000050 58 7d fe c1 e3 78 79 31 48 d2 74 c0 8d 17 97 6f |X}...xy1H.t....o| -00000060 30 bb 8a 2c 8c d4 76 3d 3f f0 20 24 3d 5a 21 0c |0..,..v=?. $=Z!.| -00000070 37 7b 14 45 e6 69 db ed 52 50 a0 77 e9 a2 84 59 |7{.E.i..RP.w...Y| -00000080 0c 96 c1 ad 48 ed 8d 9f 00 4d f2 15 86 71 c0 fa |....H....M...q..| -00000090 14 b9 77 cb 9f 04 d9 1b be da 68 8e 31 8f 25 14 |..w.......h.1.%.| -000000a0 f5 43 bd e5 6e c5 10 ab f7 68 22 7f c2 ba 5c a6 |.C..n....h"...\.| -000000b0 88 31 c0 a5 fb 63 05 95 52 b3 04 94 14 fe eb 0c |.1...c..R.......| -000000c0 53 a0 c2 bf ae 58 e3 f9 84 22 6b ca 95 33 12 80 |S....X..."k..3..| -000000d0 09 e2 97 b0 2b 4b ed fa 34 e1 5a b1 de 52 b1 2c |....+K..4.Z..R.,| -000000e0 a0 aa 11 d6 fa 07 e1 41 ed 36 9f 9a 1a 56 18 b0 |.......A.6...V..| -000000f0 ef e7 85 dc 5b 53 23 56 c2 ac 34 64 c8 9d 4b 49 |....[S#V..4d..KI| -00000100 6d 29 7e 4b 73 4f 0b 8e 30 86 87 ea cf 1c dd 62 |m)~KsO..0......b| -00000110 c0 a4 96 aa fe 41 e7 25 94 8e 08 b5 4d 42 26 d3 |.....A.%....MB&.| -00000120 ba 84 98 bf 27 2b d5 3d 37 b9 b1 b5 24 33 e3 4d |....'+.=7...$3.M| -00000130 3f 05 38 54 fe 2c 15 63 20 2e 70 c0 c6 da 0e 89 |?.8T.,.c .p.....| -00000140 b6 99 07 db e3 7c 3e 58 d0 a3 2b 50 c4 f3 21 92 |.....|>X..+P..!.| -00000150 62 e0 e9 b5 d4 7a 6b 23 a2 05 ef 9c f5 f5 05 57 |b....zk#.......W| -00000160 cd d4 4d 0a 2f 17 ca ac fc 9d 7e 60 ec 3c 80 8c |..M./.....~`.<..| -00000170 1f 99 da b7 f6 14 7a e1 86 76 50 8d f8 6b 92 24 |......z..vP..k.$| -00000180 f2 b7 82 fa 3c 9b 14 af 0f 37 40 ae 7f 10 f5 0a |....<....7@.....| -00000190 f2 0f 0f bd 01 b7 0a f7 b6 d0 7f cf bf e6 67 55 |..............gU| -000001a0 cc 36 af a6 d8 c4 ca 80 c7 af 35 ff 6e 83 56 30 |.6........5.n.V0| -000001b0 62 26 49 bb 1a 04 8b 39 10 7b 0f 09 19 2b 0f 95 |b&I....9.{...+..| -000001c0 08 9d c8 85 3b 5d 8c 97 16 ae cd 92 00 d5 3e 50 |....;]........>P| -000001d0 54 66 85 8f 42 9b 60 3d f8 99 ca c7 07 3b 51 18 |Tf..B.`=.....;Q.| -000001e0 d5 20 37 57 35 0a d8 c6 13 0f 48 94 8f 50 7d 0e |. 7W5.....H..P}.| -000001f0 5a f0 98 b2 5d 5f 46 fb ba 85 c8 4f ba 02 19 86 |Z...]_F....O....| -00000200 0b ef 4a 04 49 1f 06 cd be dc d5 32 74 14 d8 60 |..J.I......2t..`| -00000210 17 a5 b5 a2 70 8e b1 75 29 bc e0 02 e0 a2 1c 7b |....p..u)......{| -00000220 58 cd 96 69 84 0f 95 7b 78 3e 09 72 a6 e0 50 7e |X..i...{x>.r..P~| -00000230 76 b0 7b 44 11 9b b8 7e 7d 09 49 91 75 5b 9b 6f |v.{D...~}.I.u[.o| -00000240 42 a4 e6 54 4d c6 21 65 c4 64 7d c6 29 74 13 8d |B..TM.!e.d}.)t..| -00000250 69 5f 20 8c f1 88 e8 1e 2d c3 13 d5 f5 52 70 24 |i_ .....-....Rp$| -00000260 de ec 64 c7 00 3f 57 e2 3f c4 23 fe e9 65 e6 d9 |..d..?W.?.#..e..| -00000270 92 ae f0 2f 05 4d 01 72 be 0d 7d c0 f4 30 ac 69 |.../.M.r..}..0.i| -00000280 3c 7c c1 72 c1 b7 c4 c9 17 03 03 00 1e 73 8f 48 |<|.r.........s.H| -00000290 cf 34 35 31 9b 63 59 59 1b 23 e1 6b b2 3a 0f df |.451.cYY.#.k.:..| -000002a0 21 69 8e 73 1a 61 09 5b 82 c7 78 17 03 03 00 13 |!i.s.a.[..x.....| -000002b0 4f a0 ab 25 3f a4 85 9e 78 58 69 d2 2c c3 b6 df |O..%?...xXi.,...| -000002c0 32 4a 7f |2J.| +00000000 17 03 03 02 83 63 68 c7 ae a5 c2 03 56 49 d3 13 |.....ch.....VI..| +00000010 45 f0 de 02 20 70 1a cb a5 cc 6f fa 0e 4b 28 65 |E... p....o..K(e| +00000020 4f 6d 76 2e b0 61 cf fe be 43 82 02 44 06 e4 1e |Omv..a...C..D...| +00000030 91 0c 64 1b 0e 58 53 c0 15 84 e8 6a 23 64 a1 cb |..d..XS....j#d..| +00000040 cd af e6 98 c1 c2 10 23 2b dd 54 9c ff d8 11 af |.......#+.T.....| +00000050 41 0c 88 7b 77 5c 12 47 7e f9 61 97 d6 f4 65 a7 |A..{w\.G~.a...e.| +00000060 5e 9c 06 0e 8a 96 5a 4c 60 f9 ce fb 52 a6 61 44 |^.....ZL`...R.aD| +00000070 7a d1 41 54 10 51 b6 26 d4 18 fb 37 75 55 7f 18 |z.AT.Q.&...7uU..| +00000080 10 39 ea b1 f7 d5 8f 3a a4 e8 d8 be f5 2f c2 90 |.9.....:...../..| +00000090 1a dd bc d7 50 b7 0e 7e c8 71 88 8f a0 04 34 2d |....P..~.q....4-| +000000a0 6f 35 f5 3f 43 08 1d 28 c7 30 db 8e c0 6c cd 14 |o5.?C..(.0...l..| +000000b0 93 10 fc b4 58 43 92 6d a3 3c 2e 87 71 12 70 c5 |....XC.m.<..q.p.| +000000c0 ec f8 64 5a 28 ee 3c 67 05 f4 00 f8 16 11 a6 49 |..dZ(.`3e.^_.| +00000110 7b 16 78 d9 8d 2a 92 16 62 02 fc 27 ee 72 f4 2e |{.x..*..b..'.r..| +00000120 86 11 32 55 a2 9b b7 d3 3a 3b 87 be 28 8f 30 e7 |..2U....:;..(.0.| +00000130 c7 ca 34 51 e5 3b 96 55 c1 d2 b2 b1 4e 84 85 0c |..4Q.;.U....N...| +00000140 df d9 25 3e 29 b8 56 4a 32 b7 45 1a b3 05 c1 d6 |..%>).VJ2.E.....| +00000150 ca ac 13 74 59 b6 0d 03 14 f2 fa b7 05 c4 2f da |...tY........./.| +00000160 11 65 a4 89 06 c6 fd 2b 10 3c 13 3d 92 d5 6a 1c |.e.....+.<.=..j.| +00000170 c5 01 10 b5 1a 3d a0 a5 ea ef 11 74 8b 68 24 20 |.....=.....t.h$ | +00000180 04 58 68 e4 e3 69 b7 12 03 f1 4c dd 78 65 04 58 |.Xh..i....L.xe.X| +00000190 d9 85 45 39 d2 74 18 f7 5c b4 59 fc 08 fb e0 49 |..E9.t..\.Y....I| +000001a0 c6 73 e0 0d 18 05 6a 84 95 03 11 ad 73 1d 45 d3 |.s....j.....s.E.| +000001b0 d7 66 c9 14 14 f4 ed 39 a6 b2 bd 81 ac 23 c0 d9 |.f.....9.....#..| +000001c0 ba 6a 82 16 75 2f c1 31 4f 83 25 ce 87 fb 23 97 |.j..u/.1O.%...#.| +000001d0 a0 0d 7c c8 2d 35 ce a6 e8 76 65 cb 72 cc 95 fc |..|.-5...ve.r...| +000001e0 8c 65 eb f7 74 7f a6 fc 1e 03 05 cd dc 56 8d 3c |.e..t........V.<| +000001f0 09 0b 74 48 24 72 19 07 c0 c4 a3 32 28 15 33 45 |..tH$r.....2(.3E| +00000200 ef c6 02 f2 8d bf 39 a9 b9 41 bd 99 77 be 36 67 |......9..A..w.6g| +00000210 0e eb 62 2b fa cd c9 6e b8 59 d8 f9 2b dd 09 1e |..b+...n.Y..+...| +00000220 4c 4a 90 e5 d4 4c 54 10 b9 f1 e3 5a 88 4b 3f 30 |LJ...LT....Z.K?0| +00000230 c9 7c d5 be e5 b0 44 4f bb 41 7a 83 e2 47 0c 73 |.|....DO.Az..G.s| +00000240 28 b9 f2 c0 6c 28 0c 3c 3a cd 34 92 5c 13 f8 15 |(...l(.<:.4.\...| +00000250 88 5b 02 fa 65 fd 1c b9 c0 cc 78 17 37 cc 53 50 |.[..e.....x.7.SP| +00000260 0f 53 5e a0 fa 08 48 93 c2 4e 7d c1 f9 bb f7 6a |.S^...H..N}....j| +00000270 e0 0e 8b 14 1a 64 60 b6 67 3a 07 68 13 20 e1 8a |.....d`.g:.h. ..| +00000280 f2 7b 0d 25 c6 11 2f 9e 17 03 03 00 1e 1e a5 9c |.{.%../.........| +00000290 f5 56 23 e2 91 d8 e1 fd da 88 04 24 b5 66 79 06 |.V#........$.fy.| +000002a0 b2 3d e8 95 60 e1 86 2d 34 f4 c8 17 03 03 00 13 |.=..`..-4.......| +000002b0 2a c7 60 04 0d 2f b5 d7 d2 c2 f9 15 bf 26 d7 84 |*.`../.......&..| +000002c0 05 98 ee |...| diff --git a/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedNotGiven b/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedNotGiven index b839c9587945c5..7a06770a57889b 100644 --- a/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedNotGiven +++ b/src/crypto/tls/testdata/Server-TLSv13-ClientAuthRequestedNotGiven @@ -1,103 +1,105 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 ca 01 00 00 c6 03 03 4a ea 7e 77 82 |...........J.~w.| -00000010 17 37 46 db 49 14 d2 41 e4 14 b0 46 20 9d 50 45 |.7F.I..A...F .PE| -00000020 d1 75 08 82 2b 8d bc 9a 75 e3 71 20 ce 77 9a 79 |.u..+...u.q .w.y| -00000030 98 24 bc 15 be ac 30 fe 66 35 ab 51 be bd b4 fa |.$....0.f5.Q....| -00000040 6f 53 1f e9 5f 54 58 75 ce 94 f9 47 00 04 13 01 |oS.._TXu...G....| +00000000 16 03 01 00 ca 01 00 00 c6 03 03 00 69 af 00 d0 |............i...| +00000010 54 e8 0e 25 4c 62 db 87 e1 af 15 9e cf 14 ac b6 |T..%Lb..........| +00000020 ce 8c 5a 65 99 f3 c0 48 2e b6 00 20 c4 bf 7a 0b |..Ze...H... ..z.| +00000030 d8 da 4f 37 b8 12 7c ff 2e 1e ac 80 d4 32 77 0f |..O7..|......2w.| +00000040 92 8c b4 76 53 57 ba 7d bb 15 78 c8 00 04 13 01 |...vSW.}..x.....| 00000050 00 ff 01 00 00 79 00 0b 00 04 03 00 01 02 00 0a |.....y..........| 00000060 00 0c 00 0a 00 1d 00 17 00 1e 00 19 00 18 00 16 |................| 00000070 00 00 00 17 00 00 00 0d 00 1e 00 1c 04 03 05 03 |................| 00000080 06 03 08 07 08 08 08 09 08 0a 08 0b 08 04 08 05 |................| 00000090 08 06 04 01 05 01 06 01 00 2b 00 03 02 03 04 00 |.........+......| -000000a0 2d 00 02 01 01 00 33 00 26 00 24 00 1d 00 20 e0 |-.....3.&.$... .| -000000b0 64 7e 58 b6 e7 32 fc c9 d6 3e f7 e0 f5 6a 9c 3a |d~X..2...>...j.:| -000000c0 e6 8f 83 d7 1c 27 62 72 71 06 71 de 49 96 05 |.....'brq.q.I..| +000000a0 2d 00 02 01 01 00 33 00 26 00 24 00 1d 00 20 30 |-.....3.&.$... 0| +000000b0 a0 2a f8 e2 67 08 84 a8 c9 fc f8 de 0e fb 3c 0c |.*..g.........<.| +000000c0 59 10 2d 50 e9 cc d4 32 5c 86 e2 6e 0d ba 03 |Y.-P...2\..n...| >>> Flow 2 (server to client) 00000000 16 03 03 00 7a 02 00 00 76 03 03 00 00 00 00 00 |....z...v.......| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -00000020 00 00 00 00 00 00 00 00 00 00 00 20 ce 77 9a 79 |........... .w.y| -00000030 98 24 bc 15 be ac 30 fe 66 35 ab 51 be bd b4 fa |.$....0.f5.Q....| -00000040 6f 53 1f e9 5f 54 58 75 ce 94 f9 47 13 01 00 00 |oS.._TXu...G....| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 c4 bf 7a 0b |........... ..z.| +00000030 d8 da 4f 37 b8 12 7c ff 2e 1e ac 80 d4 32 77 0f |..O7..|......2w.| +00000040 92 8c b4 76 53 57 ba 7d bb 15 78 c8 13 01 00 00 |...vSW.}..x.....| 00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 2f |..+.....3.$... /| 00000060 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| 00000070 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 14 |.........._X.;t.| -00000080 03 03 00 01 01 17 03 03 00 17 d0 8f 0a 7e 6c 33 |.............~l3| -00000090 0b be 2a 24 0e fc af df 6e 7d ad 22 28 82 77 60 |..*$....n}."(.w`| -000000a0 3d 17 03 03 00 3e 8f 87 8f f1 5a f6 6f eb eb d9 |=....>....Z.o...| -000000b0 da aa fc 3c 6d ac 73 94 a3 13 5f fe 01 bb 75 eb |........| -000001f0 50 76 d6 79 39 e8 25 bc 3b d9 5a a4 a8 5e 08 6a |Pv.y9.%.;.Z..^.j| -00000200 1c 48 b1 11 f0 d9 b9 48 39 e1 23 db 41 8c bf bd |.H.....H9.#.A...| -00000210 20 27 7b 0c 89 10 1f b0 ae 70 18 9a 7f f2 b4 1d | '{......p......| -00000220 20 cd 6d 80 38 00 a4 33 de 22 ef f6 42 52 c7 66 | .m.8..3."..BR.f| -00000230 83 4a 67 18 6b a6 38 27 d3 40 cf a2 a9 3e 58 06 |.Jg.k.8'.@...>X.| -00000240 91 a7 36 08 29 10 4d 8f 1b f9 7c 5a 17 05 81 b9 |..6.).M...|Z....| -00000250 4b 60 48 40 49 73 63 8a ef 9f f2 9e 80 85 57 fa |K`H@Isc.......W.| -00000260 0a b8 72 83 f3 26 fa 07 ae d2 47 99 b5 a6 5d c5 |..r..&....G...].| -00000270 1e b5 fc ea 0f 17 f8 ba e2 5c 7d 59 70 53 2e 23 |.........\}YpS.#| -00000280 f7 55 75 cb de 82 dc ca b1 bf 3f 5f 7d 7d 92 3c |.Uu.......?_}}.<| -00000290 29 29 64 30 74 0a 01 0b c0 95 db 45 fe 20 be 38 |))d0t......E. .8| -000002a0 c5 87 b7 e4 a9 93 63 67 6b 9a 2f 24 9e 62 8f 1d |......cgk./$.b..| -000002b0 bd 8c 4a d4 b0 0f 95 2f 56 b2 1c e8 5a 58 81 2e |..J..../V...ZX..| -000002c0 b5 b5 b5 f2 1b 7f 6c 39 58 75 51 dc 83 2a 59 0b |......l9XuQ..*Y.| -000002d0 78 5e 22 7e af ee 59 af b9 8f dc 65 97 af a5 b5 |x^"~..Y....e....| -000002e0 b8 50 af 35 51 30 e7 0a 75 e2 d0 33 e2 fb f4 b1 |.P.5Q0..u..3....| -000002f0 99 cd 5f 72 6b a9 f8 85 a1 a5 19 7e 2b 91 01 19 |.._rk......~+...| -00000300 09 dd 88 6e a7 d6 54 57 cd ef d0 97 6a 68 d9 6e |...n..TW....jh.n| -00000310 52 38 ef a5 fa 84 63 70 f0 6d 64 ec 66 1a c9 b5 |R8....cp.md.f...| -00000320 78 ba 17 74 f4 b4 2b a2 fe 9a 7f 38 b8 5e 8b 56 |x..t..+....8.^.V| -00000330 a6 7b 2c 92 7f db 58 c8 fa f9 2d 6b 00 25 dc 0a |.{,...X...-k.%..| -00000340 aa 13 e8 40 f3 fd 47 23 f6 bf 1c 30 fc 91 18 95 |...@..G#...0....| -00000350 ac a8 82 3d f5 ef 17 03 03 00 99 7e 30 4f f1 00 |...=.......~0O..| -00000360 1e dd eb c6 54 d2 f5 f7 21 aa 6b b0 83 0c fa 8b |....T...!.k.....| -00000370 12 af ac 15 3e 54 b6 1c 85 9b 0c 80 02 d8 e3 5f |....>T........._| -00000380 36 57 64 fe 7a b8 31 d0 aa 59 f1 e6 af e0 27 c5 |6Wd.z.1..Y....'.| -00000390 b8 d8 2f ab e0 cc c3 02 18 73 30 36 b5 2a 0d 12 |../......s06.*..| -000003a0 a4 45 e6 c3 79 9f 54 cb 51 61 1a b8 aa 87 45 43 |.E..y.T.Qa....EC| -000003b0 8e 93 58 66 f2 97 cb 3b 44 df ae 93 17 de 22 99 |..Xf...;D.....".| -000003c0 3c b4 9d 21 a6 db 03 ce 7b fb 67 b9 6e fb ab 50 |<..!....{.g.n..P| -000003d0 f8 33 9f 20 a0 fb e9 54 bb 62 16 19 d6 df 8c fe |.3. ...T.b......| -000003e0 3d 63 42 7c b0 72 2b 0d 87 1e f7 7d bb 59 ba f5 |=cB|.r+....}.Y..| -000003f0 d6 e8 f3 57 17 03 03 00 35 9e 6f 39 92 2e 32 10 |...W....5.o9..2.| -00000400 03 cd f0 28 2c 1a 32 77 19 c8 39 38 60 0c 28 b7 |...(,.2w..98`.(.| -00000410 8c 3a d8 50 a1 44 e4 d6 c5 64 59 88 2d a4 23 c9 |.:.P.D...dY.-.#.| -00000420 26 d1 96 0c c9 5d da 84 3e 8a 7d fe 80 77 |&....]..>.}..w| +00000080 03 03 00 01 01 17 03 03 00 17 47 a8 92 ce cd bb |..........G.....| +00000090 b5 b2 f6 47 50 c3 e7 c4 44 59 20 07 13 9a ca 98 |...GP...DY .....| +000000a0 f5 17 03 03 00 52 b6 9d 37 b2 ed bf 53 10 e2 28 |.....R..7...S..(| +000000b0 09 25 e5 f1 f8 6d 10 99 f8 b0 b8 1d cd 5c 82 ff |.%...m.......\..| +000000c0 de fd 9d 8e 89 94 2e 7a 7f ed 30 80 7e f0 e5 ef |.......z..0.~...| +000000d0 13 06 7b cf 8f 91 c9 38 e0 77 52 80 0d 74 a2 28 |..{....8.wR..t.(| +000000e0 a7 73 cd 06 df c4 7e 04 ad 6e e3 44 04 86 98 95 |.s....~..n.D....| +000000f0 b7 2d 76 99 3e 8b cd 7c 17 03 03 02 6d cb e5 77 |.-v.>..|....m..w| +00000100 58 3e c2 74 44 db 85 f8 54 12 02 89 8d 2b 8d 24 |X>.tD...T....+.$| +00000110 2a c0 db 71 25 b8 23 a1 86 1a 09 66 8e 60 b9 58 |*..q%.#....f.`.X| +00000120 40 0f 72 34 95 e2 6b 7c 76 8a 85 e5 dd cd be 0a |@.r4..k|v.......| +00000130 1f a3 13 42 aa 58 d7 95 0a 69 fc 60 5a 4d 17 db |...B.X...i.`ZM..| +00000140 19 59 d3 86 04 3e c2 1b 93 76 68 27 53 ae 07 af |.Y...>...vh'S...| +00000150 47 4e e6 03 b8 fe 8e c5 06 f1 84 54 0c e5 24 be |GN.........T..$.| +00000160 83 43 1f da 1e 3e 7e a3 2f 2f 4d bc cd e5 a5 b5 |.C...>~.//M.....| +00000170 5d 66 e1 2a 9b 1e 76 6c be e7 9d f7 ed c5 1b e9 |]f.*..vl........| +00000180 91 39 18 30 22 b5 d8 71 66 4d 62 b8 6f 94 7f 5f |.9.0"..qfMb.o.._| +00000190 84 77 a2 b5 79 02 a1 44 6a 5c d2 39 c2 95 d6 8e |.w..y..Dj\.9....| +000001a0 49 36 91 97 2e 02 59 47 e8 fa 78 b9 a7 0e 41 f2 |I6....YG..x...A.| +000001b0 df 92 44 7b 59 8f 9c e9 c8 1b 0a 15 cf 1a fe 5a |..D{Y..........Z| +000001c0 b2 5a 9d 9c b2 0b 1e 48 03 a9 2b 73 df 9f dd b2 |.Z.....H..+s....| +000001d0 30 d1 ba bd 46 e6 50 30 ed e3 c5 f7 d8 96 d5 cc |0...F.P0........| +000001e0 d2 3a 39 3b f0 a1 f2 8e 29 c4 be e9 ee af cb 7b |.:9;....)......{| +000001f0 f6 7d c3 be 4e 20 6e 63 21 50 d5 34 9a b4 9c 7f |.}..N nc!P.4....| +00000200 95 22 15 e3 ad 12 c8 6a 9c 68 a3 c1 35 c1 e1 fa |.".....j.h..5...| +00000210 ce 15 ef 12 b5 50 5b d3 56 8a 24 b0 61 ca 67 1d |.....P[.V.$.a.g.| +00000220 60 12 b3 4b 42 6a 3d ba ef 80 b1 b9 74 cf ba e0 |`..KBj=.....t...| +00000230 95 5f 1a b9 c6 24 d7 c5 ec 96 3f 6c e8 13 39 66 |._...$....?l..9f| +00000240 da e6 49 39 dc ca bc 2a 50 10 cf 51 fe d0 eb 80 |..I9...*P..Q....| +00000250 9c 21 dc 9d ae 10 31 d6 9c 9b ac 3e d6 84 36 e6 |.!....1....>..6.| +00000260 76 a1 d2 86 2a a0 2b d8 91 ba c3 f0 17 f7 7a 69 |v...*.+.......zi| +00000270 0d d3 14 6f ca ba 33 d2 d0 2e 85 a9 d7 dc d2 4a |...o..3........J| +00000280 dc bf 88 68 73 4f b1 35 e0 36 3b ed bd f6 37 0b |...hsO.5.6;...7.| +00000290 07 32 a1 96 81 29 63 e1 38 4d 52 7c 5a d0 4c 6c |.2...)c.8MR|Z.Ll| +000002a0 dd 65 83 75 1d 9e bc e4 c8 e7 75 13 5f 71 11 87 |.e.u......u._q..| +000002b0 ea 4b f3 ee d2 fd 38 2a 88 a6 35 e4 a8 63 77 0e |.K....8*..5..cw.| +000002c0 5f 57 ac a6 a2 e7 43 3c 79 1a 2d c9 1d 5f 65 22 |_W....C.K.5.| +000003c0 29 1d 3e ba a0 91 22 0a 8a aa a3 f8 9b eb 77 fe |).>...".......w.| +000003d0 19 ac 2e a7 29 ab 8f 61 91 af 09 b1 3b b3 4b 69 |....)..a....;.Ki| +000003e0 e6 96 a3 05 46 db 28 9b 1d d0 79 71 3d 93 4e 66 |....F.(...yq=.Nf| +000003f0 5d f4 22 38 91 a1 30 c5 c3 b7 72 fa e6 91 d8 a4 |]."8..0...r.....| +00000400 e9 a0 47 cc 5a 87 5e 60 17 03 03 00 35 ce b7 6e |..G.Z.^`....5..n| +00000410 2c 11 dc d4 b8 e1 a4 4b d8 d6 09 1e fe e3 d3 d3 |,......K........| +00000420 52 be fc cc af 5d 73 45 db 1e 05 b6 b1 38 59 65 |R....]sE.....8Ye| +00000430 df 72 6d 7b af b5 7a 3d e1 8a 27 44 6e 72 25 04 |.rm{..z=..'Dnr%.| +00000440 7c 61 ||a| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 00 19 87 49 99 a6 d9 |............I...| -00000010 ed cd f7 7a 75 14 e1 26 41 d2 6e 5c 79 a6 be 7c |...zu..&A.n\y..|| -00000020 5d 9d 44 36 17 03 03 00 35 1b 51 a9 b1 ce 11 ed |].D6....5.Q.....| -00000030 95 47 34 b9 3d 2f 6e 27 b2 e5 31 54 7f e3 8a 11 |.G4.=/n'..1T....| -00000040 fd 54 75 2c b6 8a 56 25 00 29 a7 5f 7a 1e 16 be |.Tu,..V%.)._z...| -00000050 16 e3 86 3a 72 84 0e bc 40 ef fd ad 18 33 |...:r...@....3| +00000000 14 03 03 00 01 01 17 03 03 00 19 01 ab 91 41 25 |..............A%| +00000010 d3 52 97 b2 f8 4e 57 9d c5 d3 7d 34 c4 c3 44 93 |.R...NW...}4..D.| +00000020 79 a5 0c fe 17 03 03 00 35 ba 56 09 d8 9d 73 5e |y.......5.V...s^| +00000030 f4 f4 03 4d bf f7 e4 ab cc 83 26 28 cc 68 78 91 |...M......&(.hx.| +00000040 9f 6f f6 dc d5 34 e5 a5 e6 86 11 b3 71 89 70 25 |.o...4......q.p%| +00000050 8c 83 79 b7 c8 56 92 1d 98 c7 35 cd bb 56 |..y..V....5..V| >>> Flow 4 (server to client) -00000000 17 03 03 00 8b 69 2e 81 c4 4d 43 a6 1f 96 b7 8e |.....i...MC.....| -00000010 87 4a 9b be 48 3c 31 18 98 f4 8c 04 24 b2 52 96 |.J..H<1.....$.R.| -00000020 04 b5 12 7c 54 37 83 6d 51 42 c6 52 f4 a5 bc d3 |...|T7.mQB.R....| -00000030 d1 c8 29 ab 4f e4 02 da 74 ec 8e 13 ad 03 f3 e0 |..).O...t.......| -00000040 7f 44 58 6b c7 28 a5 6a 75 30 b8 b1 31 38 fe ba |.DXk.(.ju0..18..| -00000050 e7 27 ae b3 e7 cb 5e 78 24 82 03 61 ba ae dd 4c |.'....^x$..a...L| -00000060 c6 7b f3 45 cf 6f a8 dc 7d 5d 73 65 db ae dc 10 |.{.E.o..}]se....| -00000070 ff 32 dc 4c b4 5e dc ce 4c 34 37 83 a0 0c d5 20 |.2.L.^..L47.... | -00000080 f1 f6 81 42 bc 63 65 47 80 d0 d6 f3 08 aa e2 58 |...B.ceG.......X| -00000090 17 03 03 00 1e 85 84 f3 e4 0f a8 24 c0 fe fa 2c |...........$...,| -000000a0 8b 60 52 32 73 2b 95 e9 37 3d 1c bd 2f ee ff e2 |.`R2s+..7=../...| -000000b0 70 13 df 17 03 03 00 13 65 2b 11 5f 50 7c 11 eb |p.......e+._P|..| -000000c0 3b 06 75 23 28 13 ca 4a b5 fb dc |;.u#(..J...| +00000000 17 03 03 00 8b eb fd 9e 19 36 7f 54 7f 9f 5a 6b |.........6.T..Zk| +00000010 31 0e e8 ca f3 40 7d 74 0d 82 6f 97 41 fb 15 52 |1....@}t..o.A..R| +00000020 36 00 50 c3 c5 36 ea e3 8f ca 02 b1 de ed 61 8e |6.P..6........a.| +00000030 60 0a d8 cf 0a 7c 47 60 87 3f fa 00 6f 74 a8 42 |`....|G`.?..ot.B| +00000040 c1 8d df f7 4d 7c ac c0 52 7c c9 21 5c 45 32 3f |....M|..R|.!\E2?| +00000050 42 0d 75 64 76 93 18 29 a1 50 06 36 94 bb 1e 52 |B.udv..).P.6...R| +00000060 31 5b b7 32 9a b1 9f 59 30 8a da ff 93 4f c1 b3 |1[.2...Y0....O..| +00000070 e9 8d f2 ec 4a 7b 01 51 5e a2 9b ea 8d a8 53 bd |....J{.Q^.....S.| +00000080 db 94 88 73 3b 9f da 77 a8 de 11 55 7b 06 0f c6 |...s;..w...U{...| +00000090 17 03 03 00 1e 1b 76 21 ea fd a4 47 6e ec 6e 26 |......v!...Gn.n&| +000000a0 00 3e fc 2f 87 8f 4e ee 08 58 e7 cc 46 aa b7 8c |.>./..N..X..F...| +000000b0 c1 5d dd 17 03 03 00 13 dd ed 88 b0 f5 e8 17 f1 |.]..............| +000000c0 66 85 c4 f5 6b 47 7d 5f 19 21 7f |f...kG}_.!.| diff --git a/src/crypto/tls/testdata/Server-TLSv13-ECDHE-ECDSA-AES b/src/crypto/tls/testdata/Server-TLSv13-ECDHE-ECDSA-AES index d2ed2ee64e9bb5..d80662d169a446 100644 --- a/src/crypto/tls/testdata/Server-TLSv13-ECDHE-ECDSA-AES +++ b/src/crypto/tls/testdata/Server-TLSv13-ECDHE-ECDSA-AES @@ -1,93 +1,93 @@ >>> Flow 1 (client to server) -00000000 16 03 01 00 ca 01 00 00 c6 03 03 be 5b 8c 08 2b |............[..+| -00000010 26 a0 41 0f e3 4e b6 5c 9f 5d 53 04 67 4a 1d a2 |&.A..N.\.]S.gJ..| -00000020 26 3b 83 ab b4 7b c6 ec f8 a6 41 20 a6 de ad e2 |&;...{....A ....| -00000030 0c fd 02 99 11 51 c6 be e8 52 df 0b e2 b3 6f fe |.....Q...R....o.| -00000040 33 3e 2f 90 ac d2 e8 a2 53 8b d9 05 00 04 13 01 |3>/.....S.......| +00000000 16 03 01 00 ca 01 00 00 c6 03 03 1d 95 21 d3 93 |.............!..| +00000010 6b 69 ad 44 69 28 2d 2e 74 c3 77 24 86 82 52 91 |ki.Di(-.t.w$..R.| +00000020 a8 15 64 82 15 2e 02 f8 41 3d c5 20 87 ff 55 4c |..d.....A=. ..UL| +00000030 00 16 80 c2 f7 44 15 18 bc 00 81 d8 7b d8 2c 88 |.....D......{.,.| +00000040 cb 19 31 89 23 d0 82 be d8 7f a4 26 00 04 13 01 |..1.#......&....| 00000050 00 ff 01 00 00 79 00 0b 00 04 03 00 01 02 00 0a |.....y..........| 00000060 00 0c 00 0a 00 1d 00 17 00 1e 00 19 00 18 00 16 |................| 00000070 00 00 00 17 00 00 00 0d 00 1e 00 1c 04 03 05 03 |................| 00000080 06 03 08 07 08 08 08 09 08 0a 08 0b 08 04 08 05 |................| 00000090 08 06 04 01 05 01 06 01 00 2b 00 03 02 03 04 00 |.........+......| -000000a0 2d 00 02 01 01 00 33 00 26 00 24 00 1d 00 20 31 |-.....3.&.$... 1| -000000b0 8e dd f4 7c cf 22 04 c1 c3 04 5c 24 49 db ae ab |...|."....\$I...| -000000c0 0a d0 42 e8 70 51 c7 4f 88 e2 4e 2e 0b 80 65 |..B.pQ.O..N...e| +000000a0 2d 00 02 01 01 00 33 00 26 00 24 00 1d 00 20 8d |-.....3.&.$... .| +000000b0 18 6e 7e 5a 97 58 25 0d 07 9e af 9c 9b bd 6f 92 |.n~Z.X%.......o.| +000000c0 e9 08 8f 92 55 28 d2 90 3f fe bc dd db b7 00 |....U(..?......| >>> Flow 2 (server to client) 00000000 16 03 03 00 7a 02 00 00 76 03 03 00 00 00 00 00 |....z...v.......| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| -00000020 00 00 00 00 00 00 00 00 00 00 00 20 a6 de ad e2 |........... ....| -00000030 0c fd 02 99 11 51 c6 be e8 52 df 0b e2 b3 6f fe |.....Q...R....o.| -00000040 33 3e 2f 90 ac d2 e8 a2 53 8b d9 05 13 01 00 00 |3>/.....S.......| +00000020 00 00 00 00 00 00 00 00 00 00 00 20 87 ff 55 4c |........... ..UL| +00000030 00 16 80 c2 f7 44 15 18 bc 00 81 d8 7b d8 2c 88 |.....D......{.,.| +00000040 cb 19 31 89 23 d0 82 be d8 7f a4 26 13 01 00 00 |..1.#......&....| 00000050 2e 00 2b 00 02 03 04 00 33 00 24 00 1d 00 20 2f |..+.....3.$... /| 00000060 e5 7d a3 47 cd 62 43 15 28 da ac 5f bb 29 07 30 |.}.G.bC.(.._.).0| 00000070 ff f6 84 af c4 cf c2 ed 90 99 5f 58 cb 3b 74 14 |.........._X.;t.| -00000080 03 03 00 01 01 17 03 03 00 17 cc d0 60 a1 dc d6 |............`...| -00000090 46 57 69 3d df 0e 0f 7f a8 34 2d 87 71 84 16 54 |FWi=.....4-.q..T| -000000a0 9d 17 03 03 02 22 cb 9c f9 3e a0 fd bf 07 03 7c |....."...>.....|| -000000b0 53 0c 15 22 0b 78 e5 02 36 e6 e7 6c 5b f9 aa 8d |S..".x..6..l[...| -000000c0 54 8e b1 15 d4 23 05 12 5e 6e 0f 0f 65 77 bf b5 |T....#..^n..ew..| -000000d0 32 28 0e 32 ca 9f 61 c3 37 23 87 e8 ec 19 1d ba |2(.2..a.7#......| -000000e0 ef f5 18 eb ba 49 2d 86 a6 d1 f7 c1 9e 67 10 9f |.....I-......g..| -000000f0 a1 d2 62 bd 4c 6c 5e a4 41 f6 1e fa fd e7 55 bc |..b.Ll^.A.....U.| -00000100 16 ad 91 91 de 03 86 d7 e1 88 87 ab 0e f4 f5 bb |................| -00000110 16 da 37 bb a4 ce 4e 6c 5f 88 41 f9 a2 90 9a 2d |..7...Nl_.A....-| -00000120 5c 14 d5 01 28 06 a9 20 a4 ae 92 17 c5 95 b1 dc |\...(.. ........| -00000130 02 a8 3f 3b a7 97 91 5a 4f 56 bb db b6 30 0d 80 |..?;...ZOV...0..| -00000140 35 ac 91 6f 4f ba 1e 10 c6 fc d2 ca 96 e4 9d 1f |5..oO...........| -00000150 2f 29 00 3a 11 b7 77 dd d5 ed 76 9f 67 a1 b1 0c |/).:..w...v.g...| -00000160 5d 34 34 eb 42 49 23 15 49 12 24 24 73 be a9 65 |]44.BI#.I.$$s..e| -00000170 99 b6 b4 3f 18 0c d1 32 26 eb 86 93 5c 0e e8 06 |...?...2&...\...| -00000180 fb d7 9f 0e d9 26 14 47 b8 e5 67 8c c8 cb 0c 55 |.....&.G..g....U| -00000190 61 70 a9 ce 0d 4e bf ca 40 9c e7 0d 2b 5d 54 b7 |ap...N..@...+]T.| -000001a0 5a 64 50 e6 a1 c9 fc 47 7d e7 0a 13 36 8d 70 eb |ZdP....G}...6.p.| -000001b0 68 65 e4 9b 9d 12 d1 d9 0d 32 72 59 0f 46 b2 e2 |he.......2rY.F..| -000001c0 21 ab 13 d4 ab f3 6e b6 44 16 b8 85 bb dc f4 f7 |!.....n.D.......| -000001d0 d6 ce 5c 9f c0 4c 28 8f 36 39 ec 29 c7 33 bd ea |..\..L(.69.).3..| -000001e0 2d 10 16 84 50 c5 18 5b 2c a3 99 bb 3b 0b 70 66 |-...P..[,...;.pf| -000001f0 72 9a 83 01 06 2a bf 4a 60 c5 5d 41 a1 f0 92 bb |r....*.J`.]A....| -00000200 3b 2a 1a 41 3a 57 c3 22 13 2c b4 7b 3e 47 52 ea |;*.A:W.".,.{>GR.| -00000210 79 8a bf ef 2e 2c f7 89 c7 36 5a df 38 c2 04 b6 |y....,...6Z.8...| -00000220 6f 96 cd 7c 01 b3 e3 cd 4a 83 56 40 06 58 8a 7c |o..|....J.V@.X.|| -00000230 8c 75 df b6 b8 76 63 71 89 72 0a 64 de 23 7d 50 |.u...vcq.r.d.#}P| -00000240 77 a8 f6 a0 81 9d e9 ed 81 5e 20 c8 9f 65 3c 95 |w........^ ..e<.| -00000250 cf ed 99 80 71 06 5e 00 46 0d 0c 22 b3 88 f0 c5 |....q.^.F.."....| -00000260 33 3e 13 6b f2 07 9c db 20 31 9c 8d ea d7 73 e8 |3>.k.... 1....s.| -00000270 00 e1 2b f4 c8 d7 34 37 4a 98 b9 4d 28 db 15 8a |..+...47J..M(...| -00000280 af 53 14 3b 02 54 a3 0b 5f 10 ff 5d 20 1c 19 ae |.S.;.T.._..] ...| -00000290 6b 8a 99 a5 8f e0 ac dd c1 ba 1f 85 56 a3 94 bc |k...........V...| -000002a0 79 03 5f d5 dd e1 8e 62 b7 82 fa 92 c3 d5 8a fc |y._....b........| -000002b0 6b 17 24 d9 af db 3d 9c 0f 51 82 3d a2 ec 5f 9c |k.$...=..Q.=.._.| -000002c0 dc 69 a5 ce db d8 8b 87 17 03 03 00 a3 69 cd 7b |.i...........i.{| -000002d0 9f ac ad 72 11 b2 5d ee 19 63 d0 35 12 6d 5e 3f |...r..]..c.5.m^?| -000002e0 81 a8 18 4a d4 09 f3 80 38 4a 31 08 3e a0 4c 78 |...J....8J1.>.Lx| -000002f0 48 08 e9 90 ba e7 2a b4 73 2e 2b 2b 15 60 ce 09 |H.....*.s.++.`..| -00000300 7d df 49 31 e1 9d ff 92 1d b4 af 2e 8c f8 a6 2e |}.I1............| -00000310 93 d7 b9 10 69 10 7f 04 0d 8d e2 37 09 a7 d0 2a |....i......7...*| -00000320 ac ea 51 49 50 1d 1c 54 7f b9 15 ad 8c 77 ef 1d |..QIP..T.....w..| -00000330 a6 59 a3 bf b2 53 f7 6c 21 92 e0 36 c5 0d 61 94 |.Y...S.l!..6..a.| -00000340 be 61 5e 77 25 35 df e4 5f 67 c1 c6 af 51 e4 ce |.a^w%5.._g...Q..| -00000350 c4 28 c5 4e bc f6 c6 ba 32 dc 8e c7 45 f3 4d a1 |.(.N....2...E.M.| -00000360 70 53 98 46 8f 39 c2 cc b7 fc f7 24 11 97 72 b3 |pS.F.9.....$..r.| -00000370 17 03 03 00 35 76 be b6 7a 3f e3 08 7a a2 65 25 |....5v..z?..z.e%| -00000380 fd 0b c3 87 be ba eb ca cb 3d c1 25 10 e0 7b 00 |.........=.%..{.| -00000390 37 7a 52 9e d6 b2 e7 ba 8e 51 de 15 c4 e8 16 eb |7zR......Q......| -000003a0 c6 21 92 42 b1 62 f4 ce 27 ba 17 03 03 00 8b 54 |.!.B.b..'......T| -000003b0 03 de d7 a7 85 2f 4b 23 2d d5 3a b4 3d 3d f6 00 |...../K#-.:.==..| -000003c0 ac ab bd 6f dd bf 9f 24 fb 1b d4 01 39 3e c0 87 |...o...$....9>..| -000003d0 bb 32 ca f6 61 b2 ef 5d 9c 2c 1b a5 10 66 7b fd |.2..a..].,...f{.| -000003e0 4b d0 03 dc 53 a9 0d 5a d5 c4 4c 25 9c 55 e6 f8 |K...S..Z..L%.U..| -000003f0 d1 d8 49 dc 36 a1 92 ae f1 3e 2f 11 66 87 93 69 |..I.6....>/.f..i| -00000400 24 2e 5d 6c f6 79 15 68 a8 99 2e 1a 9c e2 85 4e |$.]l.y.h.......N| -00000410 5f d6 a8 3c 70 e1 67 cb df b2 1b ab 2b ed dc b6 |_....b...O| +000000a0 e8 17 03 03 02 22 3f 87 53 63 dc 59 f7 32 60 4b |....."?.Sc.Y.2`K| +000000b0 bd 9f e1 fc 4c 9a 98 18 94 e1 c1 07 ab 11 33 dc |....L.........3.| +000000c0 f1 48 67 e6 66 83 3c 88 53 c7 dc af e2 87 bc 0b |.Hg.f.<.S.......| +000000d0 d7 60 99 83 29 a1 1c 30 09 ba 4a e1 a9 c2 0e 34 |.`..)..0..J....4| +000000e0 cb a6 f2 8b 1b a0 b0 e6 21 27 3d b8 b4 90 0c 61 |........!'=....a| +000000f0 af 38 db fe fe 9c 34 09 1e 1a c8 f2 e9 05 68 ee |.8....4.......h.| +00000100 9c ec 74 b8 10 25 29 3d 52 71 87 c6 88 22 5a e9 |..t..%)=Rq..."Z.| +00000110 33 d0 d3 75 a8 94 b2 6d 48 4f 63 d1 32 f2 a3 70 |3..u...mHOc.2..p| +00000120 f1 a5 0b 4c 5d 7c 91 9b 04 d4 c3 9e 37 dd 67 a1 |...L]|......7.g.| +00000130 aa 23 6f 2b d0 42 b9 30 5c ed ae 12 36 f1 7c b3 |.#o+.B.0\...6.|.| +00000140 92 de 02 3a 99 c3 98 91 a3 09 43 ef 24 8d 67 e7 |...:......C.$.g.| +00000150 0d 68 22 e1 cc 99 8e 8e 64 09 be 50 f7 4a 37 0a |.h".....d..P.J7.| +00000160 02 af 88 db 8b a0 68 0d 7e 97 d9 9c 48 c3 bd aa |......h.~...H...| +00000170 db 01 69 2b 2b e6 f5 4b 66 c0 7a 8c fe 4d 8f 7b |..i++..Kf.z..M.{| +00000180 94 be 58 b5 44 67 df 26 3f 79 ee 55 bf bf aa 52 |..X.Dg.&?y.U...R| +00000190 95 ec 6b 7f 2b 68 f0 5a 81 4e 13 25 91 bd 9a df |..k.+h.Z.N.%....| +000001a0 dd 2c ae 6d c3 47 27 c2 3f 51 98 a3 b7 06 ec 2f |.,.m.G'.?Q...../| +000001b0 d6 c0 7f 1f e5 5e 3c 50 d3 6e 82 33 be 07 48 0b |.....^..}.....HUB..| +00000260 d0 b2 8e 4d c6 26 bb 77 9e 3f e0 0a 90 a4 3b eb |...M.&.w.?....;.| +00000270 37 94 c4 e8 39 12 82 24 b3 8b 6d 0d ed 9c 31 f0 |7...9..$..m...1.| +00000280 d0 5a cb b0 79 9b d2 ed ab 08 8b 9d ad 25 7a ce |.Z..y........%z.| +00000290 d7 6d c8 11 0a 60 f4 81 e9 e3 e3 42 7b 3d 95 67 |.m...`.....B{=.g| +000002a0 c2 4e 3e 80 11 2e 09 53 94 03 c0 88 cb 23 7e d2 |.N>....S.....#~.| +000002b0 ad e2 dc e7 e2 0b ba 74 9c 04 ad 75 e6 7f 5a fb |.......t...u..Z.| +000002c0 53 5a 98 14 18 4f 1d 2b 17 03 03 00 a4 7a ce c7 |SZ...O.+.....z..| +000002d0 9c de bc 27 04 f7 8b 4b a1 73 7d 0d fa b5 a1 e2 |...'...K.s}.....| +000002e0 fe 8b 33 8d 48 64 65 13 68 e2 5d e2 d7 3e 67 f2 |..3.Hde.h.]..>g.| +000002f0 db bd ff f9 e5 3e 4c b1 56 e3 22 95 88 23 48 fe |.....>L.V."..#H.| +00000300 0f 80 4c 5c 1c 74 0e 26 d4 7c 17 83 65 d6 a3 51 |..L\.t.&.|..e..Q| +00000310 5a 01 a5 12 9c db 0b c9 0b 8b 53 c7 03 75 b9 04 |Z.........S..u..| +00000320 a0 62 df 11 75 ae ff 33 7b 98 6b 7b 35 3e 41 4c |.b..u..3{.k{5>AL| +00000330 9b 16 12 b4 39 ce 9a d5 e9 83 78 b3 4b 3e d6 82 |....9.....x.K>..| +00000340 75 66 bf 73 e4 26 e6 22 8e 2f fe 4d 49 e4 b5 03 |uf.s.&."./.MI...| +00000350 04 a6 65 59 c2 aa e2 e6 ec f0 e2 99 b5 c4 55 75 |..eY..........Uu| +00000360 e1 90 a4 73 cb 21 78 df 4e 96 e2 99 75 15 77 59 |...s.!x.N...u.wY| +00000370 db 17 03 03 00 35 bc 1c 15 d7 b0 62 21 d4 dd 09 |.....5.....b!...| +00000380 1d aa 05 3c e3 ea 0a 9d 89 1f aa 2f f7 75 93 86 |...<......./.u..| +00000390 35 ee 5f 06 20 99 17 ca 4c 05 65 07 f7 56 9f 62 |5._. ...L.e..V.b| +000003a0 2a ea e2 05 f0 be fe bf d6 09 46 17 03 03 00 8b |*.........F.....| +000003b0 37 1a 6c e5 ea 27 e7 b2 d7 87 9a 1a a1 41 b5 64 |7.l..'.......A.d| +000003c0 61 8b bb 1c 64 a2 37 de 39 b3 5b f4 5b 9f bf d8 |a...d.7.9.[.[...| +000003d0 e7 3d be ad 96 6c 69 19 ce 8e a8 12 14 5d 1e 79 |.=...li......].y| +000003e0 c5 12 53 c3 13 81 5a 22 44 e5 6e c4 97 cc 18 19 |..S...Z"D.n.....| +000003f0 c4 04 08 cf 16 dd df 3d 4f 13 40 5f 33 38 f5 0f |.......=O.@_38..| +00000400 4f bb 41 e2 85 85 43 de d0 b5 7a 61 d8 3a 53 41 |O.A...C...za.:SA| +00000410 d2 ad 7b e4 bf 02 d2 14 7d f7 0c 05 b8 bb 21 90 |..{.....}.....!.| +00000420 a5 61 76 7e 07 5d bf e2 a1 f8 1a a6 77 42 2a 7c |.av~.]......wB*|| +00000430 7a 41 a7 5e 04 c2 49 02 45 a8 f5 |zA.^..I.E..| >>> Flow 3 (client to server) -00000000 14 03 03 00 01 01 17 03 03 00 35 46 8b fa 42 0d |..........5F..B.| -00000010 fa 3e 9e 80 76 12 ce 73 ae 85 67 ee af 1e 25 6e |.>..v..s..g...%n| -00000020 0b 46 4c bd 5a 46 8e 5c 27 7a 0a 8d d3 9c 3c 29 |.FL.ZF.\'z....<)| -00000030 4c c8 08 78 ac 9f f4 7a 38 8d 49 6a 01 b6 f5 83 |L..x...z8.Ij....| +00000000 14 03 03 00 01 01 17 03 03 00 35 3d f2 27 fe 81 |..........5=.'..| +00000010 4c 6e 61 1f 34 f8 3d 25 1f 33 d6 22 aa 7f ab 08 |Lna.4.=%.3."....| +00000020 2c 48 44 39 74 2c e9 be 78 7f c7 db 27 b0 b0 6d |,HD9t,..x...'..m| +00000030 b2 8b 87 6c e5 5c 38 79 9f ed 3d 4f 92 81 dc ea |...l.\8y..=O....| >>> Flow 4 (server to client) -00000000 17 03 03 00 1e c2 1a dc 0a cb 9a 11 f7 a1 c2 1f |................| -00000010 54 7d 32 6f 0e 13 b6 6b 9f e1 c6 14 63 fc 18 b9 |T}2o...k....c...| -00000020 81 53 44 17 03 03 00 13 c9 72 ae 5e 2b c1 6f 64 |.SD......r.^+.od| -00000030 e0 70 47 15 b1 ec c3 25 00 7f 4e |.pG....%..N| +00000000 17 03 03 00 1e f7 99 f9 d8 a2 00 d9 e3 48 d9 b2 |.............H..| +00000010 35 37 93 6f b0 1f d5 81 b1 16 b1 e4 d8 b4 40 ce |57.o..........@.| +00000020 97 9f 16 17 03 03 00 13 a4 cb 62 61 70 e9 67 c3 |..........bap.g.| +00000030 21 02 19 bc 01 01 5d 9b 15 d4 84 |!.....]....| diff --git a/src/crypto/tls/ticket.go b/src/crypto/tls/ticket.go index 06aec5aa63f901..c56898c6f79813 100644 --- a/src/crypto/tls/ticket.go +++ b/src/crypto/tls/ticket.go @@ -44,20 +44,21 @@ type SessionState struct { // case 0: Empty; // case 1: opaque alpn<1..2^8-1>; // }; - // select (SessionState.type) { - // case server: Empty; - // case client: struct { - // select (SessionState.version) { - // case VersionTLS10..VersionTLS12: Empty; - // case VersionTLS13: struct { - // uint64 use_by; - // uint32 age_add; - // }; + // select (SessionState.version) { + // case VersionTLS10..VersionTLS12: uint16 curve_id; + // case VersionTLS13: select (SessionState.type) { + // case server: Empty; + // case client: struct { + // uint64 use_by; + // uint32 age_add; // }; // }; // }; // } SessionState; // + // The format can be extended backwards-compatibly by adding new fields at + // the end. Otherwise, a new SessionStateType must be used, as different Go + // versions may share the same session ticket encryption key. // Extra is ignored by crypto/tls, but is encoded by [SessionState.Bytes] // and parsed by [ParseSessionState]. @@ -83,20 +84,22 @@ type SessionState struct { // createdAt is the generation time of the secret on the sever (which for // TLS 1.0–1.2 might be earlier than the current session) and the time at // which the ticket was received on the client. - createdAt uint64 // seconds since UNIX epoch - secret []byte // master secret for TLS 1.2, or the PSK for TLS 1.3 - extMasterSecret bool - peerCertificates []*x509.Certificate - activeCertHandles []*activeCert - ocspResponse []byte - scts [][]byte - verifiedChains [][]*x509.Certificate - alpnProtocol string // only set if EarlyData is true + createdAt uint64 // seconds since UNIX epoch + secret []byte // master secret for TLS 1.2, or the PSK for TLS 1.3 + extMasterSecret bool + peerCertificates []*x509.Certificate + ocspResponse []byte + scts [][]byte + verifiedChains [][]*x509.Certificate + alpnProtocol string // only set if EarlyData is true // Client-side TLS 1.3-only fields. useBy uint64 // seconds since UNIX epoch ageAdd uint32 ticket []byte + + // TLS 1.0–1.2 only fields. + curveID CurveID } // Bytes encodes the session, including any private fields, so that it can be @@ -161,11 +164,13 @@ func (s *SessionState) Bytes() ([]byte, error) { b.AddBytes([]byte(s.alpnProtocol)) }) } - if s.isClient { - if s.version >= VersionTLS13 { + if s.version >= VersionTLS13 { + if s.isClient { addUint64(&b, s.useBy) b.AddUint32(s.ageAdd) } + } else { + b.AddUint16(uint16(s.curveID)) } return b.Bytes() } @@ -187,7 +192,6 @@ func ParseSessionState(data []byte) (*SessionState, error) { var extra cryptobyte.String if !s.ReadUint16(&ss.version) || !s.ReadUint8(&typ) || - (typ != 1 && typ != 2) || !s.ReadUint16(&ss.cipherSuite) || !readUint64(&s, &ss.createdAt) || !readUint8LengthPrefixed(&s, &ss.secret) || @@ -205,6 +209,14 @@ func ParseSessionState(data []byte) (*SessionState, error) { } ss.Extra = append(ss.Extra, e) } + switch typ { + case 1: + ss.isClient = false + case 2: + ss.isClient = true + default: + return nil, errors.New("tls: unknown session encoding") + } switch extMasterSecret { case 0: ss.extMasterSecret = false @@ -226,8 +238,10 @@ func ParseSessionState(data []byte) (*SessionState, error) { if err != nil { return nil, err } - ss.activeCertHandles = append(ss.activeCertHandles, c) - ss.peerCertificates = append(ss.peerCertificates, c.cert) + ss.peerCertificates = append(ss.peerCertificates, c) + } + if ss.isClient && len(ss.peerCertificates) == 0 { + return nil, errors.New("tls: no server certificates in client session") } ss.ocspResponse = cert.OCSPStaple ss.scts = cert.SignedCertificateTimestamps @@ -254,8 +268,7 @@ func ParseSessionState(data []byte) (*SessionState, error) { if err != nil { return nil, err } - ss.activeCertHandles = append(ss.activeCertHandles, c) - chain = append(chain, c.cert) + chain = append(chain, c) } ss.verifiedChains = append(ss.verifiedChains, chain) } @@ -266,24 +279,16 @@ func ParseSessionState(data []byte) (*SessionState, error) { } ss.alpnProtocol = string(alpn) } - if isClient := typ == 2; !isClient { - if !s.Empty() { - return nil, errors.New("tls: invalid session encoding") + if ss.version >= VersionTLS13 { + if ss.isClient { + if !s.ReadUint64(&ss.useBy) || !s.ReadUint32(&ss.ageAdd) { + return nil, errors.New("tls: invalid session encoding") + } } - return ss, nil - } - ss.isClient = true - if len(ss.peerCertificates) == 0 { - return nil, errors.New("tls: no server certificates in client session") - } - if ss.version < VersionTLS13 { - if !s.Empty() { + } else { + if !s.ReadUint16((*uint16)(&ss.curveID)) { return nil, errors.New("tls: invalid session encoding") } - return ss, nil - } - if !s.ReadUint64(&ss.useBy) || !s.ReadUint32(&ss.ageAdd) || !s.Empty() { - return nil, errors.New("tls: invalid session encoding") } return ss, nil } @@ -292,17 +297,17 @@ func ParseSessionState(data []byte) (*SessionState, error) { // from the current connection. func (c *Conn) sessionState() *SessionState { return &SessionState{ - version: c.vers, - cipherSuite: c.cipherSuite, - createdAt: uint64(c.config.time().Unix()), - alpnProtocol: c.clientProtocol, - peerCertificates: c.peerCertificates, - activeCertHandles: c.activeCertHandles, - ocspResponse: c.ocspResponse, - scts: c.scts, - isClient: c.isClient, - extMasterSecret: c.extMasterSecret, - verifiedChains: c.verifiedChains, + version: c.vers, + cipherSuite: c.cipherSuite, + createdAt: uint64(c.config.time().Unix()), + alpnProtocol: c.clientProtocol, + peerCertificates: c.peerCertificates, + ocspResponse: c.ocspResponse, + scts: c.scts, + isClient: c.isClient, + extMasterSecret: c.extMasterSecret, + verifiedChains: c.verifiedChains, + curveID: c.curveID, } } diff --git a/src/crypto/tls/tls.go b/src/crypto/tls/tls.go index f3089f0ed68dda..a0fd8e835da7ae 100644 --- a/src/crypto/tls/tls.go +++ b/src/crypto/tls/tls.go @@ -4,6 +4,18 @@ // Package tls partially implements TLS 1.2, as specified in RFC 5246, // and TLS 1.3, as specified in RFC 8446. +// +// # FIPS 140-3 mode +// +// When the program is in [FIPS 140-3 mode], this package behaves as if only +// SP 800-140C and SP 800-140D approved protocol versions, cipher suites, +// signature algorithms, certificate public key types and sizes, and key +// exchange and derivation algorithms were implemented. Others are silently +// ignored and not negotiated, or rejected. This set may depend on the +// algorithms supported by the FIPS 140-3 Go Cryptographic Module selected with +// GOFIPS140, and may change across Go versions. +// +// [FIPS 140-3 mode]: https://go.dev/doc/security/fips140 package tls // BUG(agl): The crypto/tls package only implements some countermeasures diff --git a/src/crypto/tls/tls_test.go b/src/crypto/tls/tls_test.go index fc5040635fbbf7..6539009df6242a 100644 --- a/src/crypto/tls/tls_test.go +++ b/src/crypto/tls/tls_test.go @@ -8,9 +8,12 @@ import ( "bytes" "context" "crypto" + "crypto/ecdh" "crypto/ecdsa" "crypto/elliptic" + "crypto/internal/hpke" "crypto/rand" + "crypto/tls/internal/fips140tls" "crypto/x509" "crypto/x509/pkix" "encoding/asn1" @@ -29,6 +32,8 @@ import ( "strings" "testing" "time" + + "golang.org/x/crypto/cryptobyte" ) var rsaCertPEM = `-----BEGIN CERTIFICATE----- @@ -163,6 +168,7 @@ func TestX509MixedKeyPair(t *testing.T) { } func newLocalListener(t testing.TB) net.Listener { + t.Helper() ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { ln, err = net.Listen("tcp6", "[::1]:0") @@ -173,6 +179,40 @@ func newLocalListener(t testing.TB) net.Listener { return ln } +func runWithFIPSEnabled(t *testing.T, testFunc func(t *testing.T)) { + originalFIPS := fips140tls.Required() + defer func() { + if originalFIPS { + fips140tls.Force() + } else { + fips140tls.TestingOnlyAbandon() + } + }() + + fips140tls.Force() + t.Run("fips140tls", testFunc) +} + +func runWithFIPSDisabled(t *testing.T, testFunc func(t *testing.T)) { + originalFIPS := fips140tls.Required() + defer func() { + if originalFIPS { + fips140tls.Force() + } else { + fips140tls.TestingOnlyAbandon() + } + }() + + fips140tls.TestingOnlyAbandon() + t.Run("no-fips140tls", testFunc) +} + +func skipFIPS(t *testing.T) { + if fips140tls.Required() { + t.Skip("skipping test in FIPS mode") + } +} + func TestDialTimeout(t *testing.T) { if testing.Short() { t.Skip("skipping in short mode") @@ -771,7 +811,7 @@ func TestWarningAlertFlood(t *testing.T) { } func TestCloneFuncFields(t *testing.T) { - const expectedCount = 9 + const expectedCount = 10 called := 0 c1 := Config{ @@ -811,6 +851,10 @@ func TestCloneFuncFields(t *testing.T) { called |= 1 << 8 return nil }, + GetEncryptedClientHelloKeys: func(*ClientHelloInfo) ([]EncryptedClientHelloKey, error) { + called |= 1 << 9 + return nil, nil + }, } c2 := c1.Clone() @@ -824,6 +868,7 @@ func TestCloneFuncFields(t *testing.T) { c2.UnwrapSession(nil, ConnectionState{}) c2.WrapSession(ConnectionState{}, nil) c2.EncryptedClientHelloRejectionVerify(ConnectionState{}) + c2.GetEncryptedClientHelloKeys(nil) if called != (1<= 0 { - org = org[:i] - parentOrg = name[i+1:] - } - tmpl := &Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{ - Organization: []string{org}, - }, - NotBefore: time.Unix(0, 0), - NotAfter: time.Unix(0, 0), - - KeyUsage: KeyUsageKeyEncipherment | KeyUsageDigitalSignature, - ExtKeyUsage: []ExtKeyUsage{ExtKeyUsageServerAuth, ExtKeyUsageClientAuth}, - BasicConstraintsValid: true, - } - if mode&^boringCertFIPSOK == boringCertLeaf { - tmpl.DNSNames = []string{"example.com"} - } else { - tmpl.IsCA = true - tmpl.KeyUsage |= KeyUsageCertSign - } - - var pcert *Certificate - var pkey interface{} - if parent != nil { - pcert = parent.cert - pkey = parent.key - } else { - pcert = tmpl - pkey = key - } - - var pub interface{} - var desc string - switch k := key.(type) { - case *rsa.PrivateKey: - pub = &k.PublicKey - desc = fmt.Sprintf("RSA-%d", k.N.BitLen()) - case *ecdsa.PrivateKey: - pub = &k.PublicKey - desc = "ECDSA-" + k.Curve.Params().Name - default: - t.Fatalf("invalid key %T", key) - } - - der, err := CreateCertificate(rand.Reader, tmpl, pcert, pub, pkey) - if err != nil { - t.Fatal(err) - } - cert, err := ParseCertificate(der) - if err != nil { - t.Fatal(err) - } - - // Tell isBoringCertificate to enforce FIPS restrictions for this check. - fipstls.Force() - defer fipstls.Abandon() - - fipsOK := mode&boringCertFIPSOK != 0 - if boringAllowCert(cert) != fipsOK { - t.Errorf("boringAllowCert(cert with %s key) = %v, want %v", desc, !fipsOK, fipsOK) - } - return &boringCertificate{name, org, parentOrg, der, cert, key, fipsOK} -} diff --git a/src/crypto/x509/internal/macos/corefoundation.go b/src/crypto/x509/internal/macos/corefoundation.go index b4032a5d91c2a0..23fbe35c248dd9 100644 --- a/src/crypto/x509/internal/macos/corefoundation.go +++ b/src/crypto/x509/internal/macos/corefoundation.go @@ -4,10 +4,10 @@ //go:build darwin -// Package macOS provides cgo-less wrappers for Core Foundation and +// Package macos provides cgo-less wrappers for Core Foundation and // Security.framework, similarly to how package syscall provides access to // libSystem.dylib. -package macOS +package macos import ( "bytes" diff --git a/src/crypto/x509/internal/macos/security.go b/src/crypto/x509/internal/macos/security.go index a6972c0c09dfe2..6af19bafe59ce9 100644 --- a/src/crypto/x509/internal/macos/security.go +++ b/src/crypto/x509/internal/macos/security.go @@ -4,7 +4,7 @@ //go:build darwin -package macOS +package macos import ( "errors" @@ -20,37 +20,6 @@ import ( // Based on https://opensource.apple.com/source/Security/Security-59306.41.2/base/Security.h -type SecTrustSettingsResult int32 - -const ( - SecTrustSettingsResultInvalid SecTrustSettingsResult = iota - SecTrustSettingsResultTrustRoot - SecTrustSettingsResultTrustAsRoot - SecTrustSettingsResultDeny - SecTrustSettingsResultUnspecified -) - -type SecTrustResultType int32 - -const ( - SecTrustResultInvalid SecTrustResultType = iota - SecTrustResultProceed - SecTrustResultConfirm // deprecated - SecTrustResultDeny - SecTrustResultUnspecified - SecTrustResultRecoverableTrustFailure - SecTrustResultFatalTrustFailure - SecTrustResultOtherError -) - -type SecTrustSettingsDomain int32 - -const ( - SecTrustSettingsDomainUser SecTrustSettingsDomain = iota - SecTrustSettingsDomainAdmin - SecTrustSettingsDomainSystem -) - const ( // various macOS error codes that can be returned from // SecTrustEvaluateWithError that we can map to Go cert @@ -69,54 +38,6 @@ func (s OSStatus) Error() string { return s.call + " error: " + strconv.Itoa(int(s.status)) } -// Dictionary keys are defined as build-time strings with CFSTR, but the Go -// linker's internal linking mode can't handle CFSTR relocations. Create our -// own dynamic strings instead and just never release them. -// -// Note that this might be the only thing that can break over time if -// these values change, as the ABI arguably requires using the strings -// pointed to by the symbols, not values that happen to be equal to them. - -var SecTrustSettingsResultKey = StringToCFString("kSecTrustSettingsResult") -var SecTrustSettingsPolicy = StringToCFString("kSecTrustSettingsPolicy") -var SecTrustSettingsPolicyString = StringToCFString("kSecTrustSettingsPolicyString") -var SecPolicyOid = StringToCFString("SecPolicyOid") -var SecPolicyAppleSSL = StringToCFString("1.2.840.113635.100.1.3") // defined by POLICYMACRO - -var ErrNoTrustSettings = errors.New("no trust settings found") - -const errSecNoTrustSettings = -25263 - -//go:cgo_import_dynamic x509_SecTrustSettingsCopyCertificates SecTrustSettingsCopyCertificates "/System/Library/Frameworks/Security.framework/Versions/A/Security" - -func SecTrustSettingsCopyCertificates(domain SecTrustSettingsDomain) (certArray CFRef, err error) { - ret := syscall(abi.FuncPCABI0(x509_SecTrustSettingsCopyCertificates_trampoline), uintptr(domain), - uintptr(unsafe.Pointer(&certArray)), 0, 0, 0, 0) - if int32(ret) == errSecNoTrustSettings { - return 0, ErrNoTrustSettings - } else if ret != 0 { - return 0, OSStatus{"SecTrustSettingsCopyCertificates", int32(ret)} - } - return certArray, nil -} -func x509_SecTrustSettingsCopyCertificates_trampoline() - -const errSecItemNotFound = -25300 - -//go:cgo_import_dynamic x509_SecTrustSettingsCopyTrustSettings SecTrustSettingsCopyTrustSettings "/System/Library/Frameworks/Security.framework/Versions/A/Security" - -func SecTrustSettingsCopyTrustSettings(cert CFRef, domain SecTrustSettingsDomain) (trustSettings CFRef, err error) { - ret := syscall(abi.FuncPCABI0(x509_SecTrustSettingsCopyTrustSettings_trampoline), uintptr(cert), uintptr(domain), - uintptr(unsafe.Pointer(&trustSettings)), 0, 0, 0) - if int32(ret) == errSecItemNotFound { - return 0, ErrNoTrustSettings - } else if ret != 0 { - return 0, OSStatus{"SecTrustSettingsCopyTrustSettings", int32(ret)} - } - return trustSettings, nil -} -func x509_SecTrustSettingsCopyTrustSettings_trampoline() - //go:cgo_import_dynamic x509_SecTrustCreateWithCertificates SecTrustCreateWithCertificates "/System/Library/Frameworks/Security.framework/Versions/A/Security" func SecTrustCreateWithCertificates(certs CFRef, policies CFRef) (CFRef, error) { @@ -184,19 +105,6 @@ func SecTrustEvaluate(trustObj CFRef) (CFRef, error) { } func x509_SecTrustEvaluate_trampoline() -//go:cgo_import_dynamic x509_SecTrustGetResult SecTrustGetResult "/System/Library/Frameworks/Security.framework/Versions/A/Security" - -func SecTrustGetResult(trustObj CFRef, result CFRef) (CFRef, CFRef, error) { - var chain, info CFRef - ret := syscall(abi.FuncPCABI0(x509_SecTrustGetResult_trampoline), uintptr(trustObj), uintptr(unsafe.Pointer(&result)), - uintptr(unsafe.Pointer(&chain)), uintptr(unsafe.Pointer(&info)), 0, 0) - if int32(ret) != 0 { - return 0, 0, OSStatus{"SecTrustGetResult", int32(ret)} - } - return chain, info, nil -} -func x509_SecTrustGetResult_trampoline() - //go:cgo_import_dynamic x509_SecTrustEvaluateWithError SecTrustEvaluateWithError "/System/Library/Frameworks/Security.framework/Versions/A/Security" func SecTrustEvaluateWithError(trustObj CFRef) (int, error) { @@ -214,25 +122,6 @@ func SecTrustEvaluateWithError(trustObj CFRef) (int, error) { } func x509_SecTrustEvaluateWithError_trampoline() -//go:cgo_import_dynamic x509_SecTrustGetCertificateCount SecTrustGetCertificateCount "/System/Library/Frameworks/Security.framework/Versions/A/Security" - -func SecTrustGetCertificateCount(trustObj CFRef) int { - ret := syscall(abi.FuncPCABI0(x509_SecTrustGetCertificateCount_trampoline), uintptr(trustObj), 0, 0, 0, 0, 0) - return int(ret) -} -func x509_SecTrustGetCertificateCount_trampoline() - -//go:cgo_import_dynamic x509_SecTrustGetCertificateAtIndex SecTrustGetCertificateAtIndex "/System/Library/Frameworks/Security.framework/Versions/A/Security" - -func SecTrustGetCertificateAtIndex(trustObj CFRef, i int) (CFRef, error) { - ret := syscall(abi.FuncPCABI0(x509_SecTrustGetCertificateAtIndex_trampoline), uintptr(trustObj), uintptr(i), 0, 0, 0, 0) - if ret == 0 { - return 0, OSStatus{"SecTrustGetCertificateAtIndex", int32(ret)} - } - return CFRef(ret), nil -} -func x509_SecTrustGetCertificateAtIndex_trampoline() - //go:cgo_import_dynamic x509_SecCertificateCopyData SecCertificateCopyData "/System/Library/Frameworks/Security.framework/Versions/A/Security" func SecCertificateCopyData(cert CFRef) ([]byte, error) { @@ -245,3 +134,14 @@ func SecCertificateCopyData(cert CFRef) ([]byte, error) { return b, nil } func x509_SecCertificateCopyData_trampoline() + +//go:cgo_import_dynamic x509_SecTrustCopyCertificateChain SecTrustCopyCertificateChain "/System/Library/Frameworks/Security.framework/Versions/A/Security" + +func SecTrustCopyCertificateChain(trustObj CFRef) (CFRef, error) { + ret := syscall(abi.FuncPCABI0(x509_SecTrustCopyCertificateChain_trampoline), uintptr(trustObj), 0, 0, 0, 0, 0) + if ret == 0 { + return 0, OSStatus{"SecTrustCopyCertificateChain", int32(ret)} + } + return CFRef(ret), nil +} +func x509_SecTrustCopyCertificateChain_trampoline() diff --git a/src/crypto/x509/internal/macos/security.s b/src/crypto/x509/internal/macos/security.s index ed726f1127c8ce..ca5337c788d673 100644 --- a/src/crypto/x509/internal/macos/security.s +++ b/src/crypto/x509/internal/macos/security.s @@ -9,10 +9,6 @@ // The trampolines are ABIInternal as they are address-taken in // Go code. -TEXT ·x509_SecTrustSettingsCopyCertificates_trampoline(SB),NOSPLIT,$0-0 - JMP x509_SecTrustSettingsCopyCertificates(SB) -TEXT ·x509_SecTrustSettingsCopyTrustSettings_trampoline(SB),NOSPLIT,$0-0 - JMP x509_SecTrustSettingsCopyTrustSettings(SB) TEXT ·x509_SecTrustCreateWithCertificates_trampoline(SB),NOSPLIT,$0-0 JMP x509_SecTrustCreateWithCertificates(SB) TEXT ·x509_SecCertificateCreateWithData_trampoline(SB),NOSPLIT,$0-0 @@ -23,13 +19,9 @@ TEXT ·x509_SecTrustSetVerifyDate_trampoline(SB),NOSPLIT,$0-0 JMP x509_SecTrustSetVerifyDate(SB) TEXT ·x509_SecTrustEvaluate_trampoline(SB),NOSPLIT,$0-0 JMP x509_SecTrustEvaluate(SB) -TEXT ·x509_SecTrustGetResult_trampoline(SB),NOSPLIT,$0-0 - JMP x509_SecTrustGetResult(SB) TEXT ·x509_SecTrustEvaluateWithError_trampoline(SB),NOSPLIT,$0-0 JMP x509_SecTrustEvaluateWithError(SB) -TEXT ·x509_SecTrustGetCertificateCount_trampoline(SB),NOSPLIT,$0-0 - JMP x509_SecTrustGetCertificateCount(SB) -TEXT ·x509_SecTrustGetCertificateAtIndex_trampoline(SB),NOSPLIT,$0-0 - JMP x509_SecTrustGetCertificateAtIndex(SB) TEXT ·x509_SecCertificateCopyData_trampoline(SB),NOSPLIT,$0-0 JMP x509_SecCertificateCopyData(SB) +TEXT ·x509_SecTrustCopyCertificateChain_trampoline(SB),NOSPLIT,$0-0 + JMP x509_SecTrustCopyCertificateChain(SB) diff --git a/src/crypto/x509/name_constraints_test.go b/src/crypto/x509/name_constraints_test.go index 008c7028f4e4c4..a5851845164d10 100644 --- a/src/crypto/x509/name_constraints_test.go +++ b/src/crypto/x509/name_constraints_test.go @@ -1607,6 +1607,23 @@ var nameConstraintsTests = []nameConstraintsTest{ leaf: leafSpec{sans: []string{"dns:.example.com"}}, expectedError: "cannot parse dnsName \".example.com\"", }, + // #86: URIs with IPv6 addresses with zones and ports are rejected + { + roots: []constraintsSpec{ + { + ok: []string{"uri:example.com"}, + }, + }, + intermediates: [][]constraintsSpec{ + { + {}, + }, + }, + leaf: leafSpec{ + sans: []string{"uri:http://[2006:abcd::1%25.example.com]:16/"}, + }, + expectedError: "URI with IP", + }, } func makeConstraintsCACert(constraints constraintsSpec, name string, key *ecdsa.PrivateKey, parent *Certificate, parentKey *ecdsa.PrivateKey) (*Certificate, error) { diff --git a/src/crypto/x509/notboring.go b/src/crypto/x509/notboring.go deleted file mode 100644 index c83a7272c9f01f..00000000000000 --- a/src/crypto/x509/notboring.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !boringcrypto - -package x509 - -func boringAllowCert(c *Certificate) bool { return true } diff --git a/src/crypto/x509/oid.go b/src/crypto/x509/oid.go index fd438eacf95420..c60daa7540c830 100644 --- a/src/crypto/x509/oid.go +++ b/src/crypto/x509/oid.go @@ -112,9 +112,14 @@ func appendBase128BigInt(dst []byte, n *big.Int) []byte { return dst } +// AppendText implements [encoding.TextAppender] +func (o OID) AppendText(b []byte) ([]byte, error) { + return append(b, o.String()...), nil +} + // MarshalText implements [encoding.TextMarshaler] func (o OID) MarshalText() ([]byte, error) { - return []byte(o.String()), nil + return o.AppendText(nil) } // UnmarshalText implements [encoding.TextUnmarshaler] @@ -180,9 +185,14 @@ func (o *OID) unmarshalOIDText(oid string) error { return nil } +// AppendBinary implements [encoding.BinaryAppender] +func (o OID) AppendBinary(b []byte) ([]byte, error) { + return append(b, o.der...), nil +} + // MarshalBinary implements [encoding.BinaryMarshaler] func (o OID) MarshalBinary() ([]byte, error) { - return bytes.Clone(o.der), nil + return o.AppendBinary(nil) } // UnmarshalBinary implements [encoding.BinaryUnmarshaler] @@ -276,7 +286,7 @@ func (oid OID) EqualASN1OID(other asn1.ObjectIdentifier) bool { return i == len(other) } -// Strings returns the string representation of the Object Identifier. +// String returns the string representation of the Object Identifier. func (oid OID) String() string { var b strings.Builder b.Grow(32) diff --git a/src/crypto/x509/oid_test.go b/src/crypto/x509/oid_test.go index 270dca0bb58663..ce3a0672a6aa7c 100644 --- a/src/crypto/x509/oid_test.go +++ b/src/crypto/x509/oid_test.go @@ -133,12 +133,12 @@ func TestOIDEqual(t *testing.T) { oid2 OID eq bool }{ - {oid: mustNewOIDFromInts(t, []uint64{1, 2, 3}), oid2: mustNewOIDFromInts(t, []uint64{1, 2, 3}), eq: true}, - {oid: mustNewOIDFromInts(t, []uint64{1, 2, 3}), oid2: mustNewOIDFromInts(t, []uint64{1, 2, 4}), eq: false}, - {oid: mustNewOIDFromInts(t, []uint64{1, 2, 3}), oid2: mustNewOIDFromInts(t, []uint64{1, 2, 3, 4}), eq: false}, - {oid: mustNewOIDFromInts(t, []uint64{2, 33, 22}), oid2: mustNewOIDFromInts(t, []uint64{2, 33, 23}), eq: false}, + {oid: mustNewOIDFromInts([]uint64{1, 2, 3}), oid2: mustNewOIDFromInts([]uint64{1, 2, 3}), eq: true}, + {oid: mustNewOIDFromInts([]uint64{1, 2, 3}), oid2: mustNewOIDFromInts([]uint64{1, 2, 4}), eq: false}, + {oid: mustNewOIDFromInts([]uint64{1, 2, 3}), oid2: mustNewOIDFromInts([]uint64{1, 2, 3, 4}), eq: false}, + {oid: mustNewOIDFromInts([]uint64{2, 33, 22}), oid2: mustNewOIDFromInts([]uint64{2, 33, 23}), eq: false}, {oid: OID{}, oid2: OID{}, eq: true}, - {oid: OID{}, oid2: mustNewOIDFromInts(t, []uint64{2, 33, 23}), eq: false}, + {oid: OID{}, oid2: mustNewOIDFromInts([]uint64{2, 33, 23}), eq: false}, } for _, tt := range cases { @@ -228,6 +228,14 @@ func TestOIDMarshal(t *testing.T) { continue } + textAppend := make([]byte, 4) + textAppend, err = o.AppendText(textAppend) + textAppend = textAppend[4:] + if string(textAppend) != tt.in || err != nil { + t.Errorf("(%#v).AppendText() = (%v, %v); want = (%v, nil)", o, string(textAppend), err, tt.in) + continue + } + binary, err := o.MarshalBinary() if err != nil { t.Errorf("(%#v).MarshalBinary() = %v; want = nil", o, err) @@ -242,6 +250,23 @@ func TestOIDMarshal(t *testing.T) { t.Errorf("(*OID).UnmarshalBinary(%v) = %v; want = %v", binary, o3, tt.out) continue } + + binaryAppend := make([]byte, 4) + binaryAppend, err = o.AppendBinary(binaryAppend) + binaryAppend = binaryAppend[4:] + if err != nil { + t.Errorf("(%#v).AppendBinary() = %v; want = nil", o, err) + } + + var o4 OID + if err := o4.UnmarshalBinary(binaryAppend); err != nil { + t.Errorf("(*OID).UnmarshalBinary(%v) = %v; want = nil", binaryAppend, err) + } + + if !o4.Equal(tt.out) { + t.Errorf("(*OID).UnmarshalBinary(%v) = %v; want = %v", binaryAppend, o4, tt.out) + continue + } } } @@ -252,32 +277,32 @@ func TestOIDEqualASN1OID(t *testing.T) { oid2 asn1.ObjectIdentifier eq bool }{ - {oid: mustNewOIDFromInts(t, []uint64{1, 2, 3}), oid2: asn1.ObjectIdentifier{1, 2, 3}, eq: true}, - {oid: mustNewOIDFromInts(t, []uint64{1, 2, 3}), oid2: asn1.ObjectIdentifier{1, 2, 4}, eq: false}, - {oid: mustNewOIDFromInts(t, []uint64{1, 2, 3}), oid2: asn1.ObjectIdentifier{1, 2, 3, 4}, eq: false}, - {oid: mustNewOIDFromInts(t, []uint64{1, 33, 22}), oid2: asn1.ObjectIdentifier{1, 33, 23}, eq: false}, - {oid: mustNewOIDFromInts(t, []uint64{1, 33, 23}), oid2: asn1.ObjectIdentifier{1, 33, 22}, eq: false}, - {oid: mustNewOIDFromInts(t, []uint64{1, 33, 127}), oid2: asn1.ObjectIdentifier{1, 33, 127}, eq: true}, - {oid: mustNewOIDFromInts(t, []uint64{1, 33, 128}), oid2: asn1.ObjectIdentifier{1, 33, 127}, eq: false}, - {oid: mustNewOIDFromInts(t, []uint64{1, 33, 128}), oid2: asn1.ObjectIdentifier{1, 33, 128}, eq: true}, - {oid: mustNewOIDFromInts(t, []uint64{1, 33, 129}), oid2: asn1.ObjectIdentifier{1, 33, 129}, eq: true}, - {oid: mustNewOIDFromInts(t, []uint64{1, 33, 128}), oid2: asn1.ObjectIdentifier{1, 33, 129}, eq: false}, - {oid: mustNewOIDFromInts(t, []uint64{1, 33, 129}), oid2: asn1.ObjectIdentifier{1, 33, 128}, eq: false}, - {oid: mustNewOIDFromInts(t, []uint64{1, 33, 255}), oid2: asn1.ObjectIdentifier{1, 33, 255}, eq: true}, - {oid: mustNewOIDFromInts(t, []uint64{1, 33, 256}), oid2: asn1.ObjectIdentifier{1, 33, 256}, eq: true}, - {oid: mustNewOIDFromInts(t, []uint64{2, 33, 257}), oid2: asn1.ObjectIdentifier{2, 33, 256}, eq: false}, - {oid: mustNewOIDFromInts(t, []uint64{2, 33, 256}), oid2: asn1.ObjectIdentifier{2, 33, 257}, eq: false}, - - {oid: mustNewOIDFromInts(t, []uint64{1, 33}), oid2: asn1.ObjectIdentifier{1, 33, math.MaxInt32}, eq: false}, - {oid: mustNewOIDFromInts(t, []uint64{1, 33, math.MaxInt32}), oid2: asn1.ObjectIdentifier{1, 33}, eq: false}, - {oid: mustNewOIDFromInts(t, []uint64{1, 33, math.MaxInt32}), oid2: asn1.ObjectIdentifier{1, 33, math.MaxInt32}, eq: true}, + {oid: mustNewOIDFromInts([]uint64{1, 2, 3}), oid2: asn1.ObjectIdentifier{1, 2, 3}, eq: true}, + {oid: mustNewOIDFromInts([]uint64{1, 2, 3}), oid2: asn1.ObjectIdentifier{1, 2, 4}, eq: false}, + {oid: mustNewOIDFromInts([]uint64{1, 2, 3}), oid2: asn1.ObjectIdentifier{1, 2, 3, 4}, eq: false}, + {oid: mustNewOIDFromInts([]uint64{1, 33, 22}), oid2: asn1.ObjectIdentifier{1, 33, 23}, eq: false}, + {oid: mustNewOIDFromInts([]uint64{1, 33, 23}), oid2: asn1.ObjectIdentifier{1, 33, 22}, eq: false}, + {oid: mustNewOIDFromInts([]uint64{1, 33, 127}), oid2: asn1.ObjectIdentifier{1, 33, 127}, eq: true}, + {oid: mustNewOIDFromInts([]uint64{1, 33, 128}), oid2: asn1.ObjectIdentifier{1, 33, 127}, eq: false}, + {oid: mustNewOIDFromInts([]uint64{1, 33, 128}), oid2: asn1.ObjectIdentifier{1, 33, 128}, eq: true}, + {oid: mustNewOIDFromInts([]uint64{1, 33, 129}), oid2: asn1.ObjectIdentifier{1, 33, 129}, eq: true}, + {oid: mustNewOIDFromInts([]uint64{1, 33, 128}), oid2: asn1.ObjectIdentifier{1, 33, 129}, eq: false}, + {oid: mustNewOIDFromInts([]uint64{1, 33, 129}), oid2: asn1.ObjectIdentifier{1, 33, 128}, eq: false}, + {oid: mustNewOIDFromInts([]uint64{1, 33, 255}), oid2: asn1.ObjectIdentifier{1, 33, 255}, eq: true}, + {oid: mustNewOIDFromInts([]uint64{1, 33, 256}), oid2: asn1.ObjectIdentifier{1, 33, 256}, eq: true}, + {oid: mustNewOIDFromInts([]uint64{2, 33, 257}), oid2: asn1.ObjectIdentifier{2, 33, 256}, eq: false}, + {oid: mustNewOIDFromInts([]uint64{2, 33, 256}), oid2: asn1.ObjectIdentifier{2, 33, 257}, eq: false}, + + {oid: mustNewOIDFromInts([]uint64{1, 33}), oid2: asn1.ObjectIdentifier{1, 33, math.MaxInt32}, eq: false}, + {oid: mustNewOIDFromInts([]uint64{1, 33, math.MaxInt32}), oid2: asn1.ObjectIdentifier{1, 33}, eq: false}, + {oid: mustNewOIDFromInts([]uint64{1, 33, math.MaxInt32}), oid2: asn1.ObjectIdentifier{1, 33, math.MaxInt32}, eq: true}, { - oid: mustNewOIDFromInts(t, []uint64{1, 33, math.MaxInt32 + 1}), + oid: mustNewOIDFromInts([]uint64{1, 33, math.MaxInt32 + 1}), oid2: asn1.ObjectIdentifier{1, 33 /*convert to int, so that it compiles on 32bit*/, int(maxInt32PlusOne)}, eq: false, }, - {oid: mustNewOIDFromInts(t, []uint64{1, 33, 256}), oid2: asn1.ObjectIdentifier{}, eq: false}, + {oid: mustNewOIDFromInts([]uint64{1, 33, 256}), oid2: asn1.ObjectIdentifier{}, eq: false}, {oid: OID{}, oid2: asn1.ObjectIdentifier{1, 33, 256}, eq: false}, {oid: OID{}, oid2: asn1.ObjectIdentifier{}, eq: false}, } @@ -306,7 +331,7 @@ func TestOIDUnmarshalBinary(t *testing.T) { } func BenchmarkOIDMarshalUnmarshalText(b *testing.B) { - oid := mustNewOIDFromInts(b, []uint64{1, 2, 3, 9999, 1024}) + oid := mustNewOIDFromInts([]uint64{1, 2, 3, 9999, 1024}) for range b.N { text, err := oid.MarshalText() if err != nil { @@ -318,11 +343,3 @@ func BenchmarkOIDMarshalUnmarshalText(b *testing.B) { } } } - -func mustNewOIDFromInts(t testing.TB, ints []uint64) OID { - oid, err := OIDFromInts(ints) - if err != nil { - t.Fatalf("OIDFromInts(%v) unexpected error: %v", ints, err) - } - return oid -} diff --git a/src/crypto/x509/parser.go b/src/crypto/x509/parser.go index 3ba5f6a4e1dc61..680dcee203a828 100644 --- a/src/crypto/x509/parser.go +++ b/src/crypto/x509/parser.go @@ -17,6 +17,7 @@ import ( "errors" "fmt" "internal/godebug" + "math" "math/big" "net" "net/url" @@ -60,7 +61,21 @@ func isPrintable(b byte) bool { func parseASN1String(tag cryptobyte_asn1.Tag, value []byte) (string, error) { switch tag { case cryptobyte_asn1.T61String: - return string(value), nil + // T.61 is a defunct ITU 8-bit character encoding which preceded Unicode. + // T.61 uses a code page layout that _almost_ exactly maps to the code + // page layout of the ISO 8859-1 (Latin-1) character encoding, with the + // exception that a number of characters in Latin-1 are not present + // in T.61. + // + // Instead of mapping which characters are present in Latin-1 but not T.61, + // we just treat these strings as being encoded using Latin-1. This matches + // what most of the world does, including BoringSSL. + buf := make([]byte, 0, len(value)) + for _, v := range value { + // All the 1-byte UTF-8 runes map 1-1 with Latin-1. + buf = utf8.AppendRune(buf, rune(v)) + } + return string(buf), nil case cryptobyte_asn1.PrintableString: for _, b := range value { if !isPrintable(b) { @@ -74,6 +89,14 @@ func parseASN1String(tag cryptobyte_asn1.Tag, value []byte) (string, error) { } return string(value), nil case cryptobyte_asn1.Tag(asn1.TagBMPString): + // BMPString uses the defunct UCS-2 16-bit character encoding, which + // covers the Basic Multilingual Plane (BMP). UTF-16 was an extension of + // UCS-2, containing all of the same code points, but also including + // multi-code point characters (by using surrogate code points). We can + // treat a UCS-2 encoded string as a UTF-16 encoded string, as long as + // we reject out the UTF-16 specific code points. This matches the + // BoringSSL behavior. + if len(value)%2 != 0 { return "", errors.New("invalid BMPString") } @@ -85,7 +108,16 @@ func parseASN1String(tag cryptobyte_asn1.Tag, value []byte) (string, error) { s := make([]uint16, 0, len(value)/2) for len(value) > 0 { - s = append(s, uint16(value[0])<<8+uint16(value[1])) + point := uint16(value[0])<<8 + uint16(value[1]) + // Reject UTF-16 code points that are permanently reserved + // noncharacters (0xfffe, 0xffff, and 0xfdd0-0xfdef) and surrogates + // (0xd800-0xdfff). + if point == 0xfffe || point == 0xffff || + (point >= 0xfdd0 && point <= 0xfdef) || + (point >= 0xd800 && point <= 0xdfff) { + return "", errors.New("invalid BMPString") + } + s = append(s, point) value = value[2:] } @@ -342,14 +374,16 @@ func parseBasicConstraintsExtension(der cryptobyte.String) (bool, int, error) { return false, 0, errors.New("x509: invalid basic constraints") } } + maxPathLen := -1 if der.PeekASN1Tag(cryptobyte_asn1.INTEGER) { - if !der.ReadASN1Integer(&maxPathLen) { + var mpl uint + if !der.ReadASN1Integer(&mpl) || mpl > math.MaxInt { return false, 0, errors.New("x509: invalid basic constraints") } + maxPathLen = int(mpl) } - // TODO: map out.MaxPathLen to 0 if it has the -1 default value? (Issue 19285) return isCA, maxPathLen, nil } @@ -395,10 +429,8 @@ func parseSANExtension(der cryptobyte.String) (dnsNames, emailAddresses []string if err != nil { return fmt.Errorf("x509: cannot parse URI %q: %s", uriStr, err) } - if len(uri.Host) > 0 { - if _, ok := domainToReverseLabels(uri.Host); !ok { - return fmt.Errorf("x509: cannot parse URI %q: invalid domain", uriStr) - } + if len(uri.Host) > 0 && !domainNameValid(uri.Host, false) { + return fmt.Errorf("x509: cannot parse URI %q: invalid domain", uriStr) } uris = append(uris, uri) case nameTypeIP: @@ -458,6 +490,7 @@ func parseExtKeyUsageExtension(der cryptobyte.String) ([]ExtKeyUsage, []asn1.Obj func parseCertificatePoliciesExtension(der cryptobyte.String) ([]OID, error) { var oids []OID + seenOIDs := map[string]bool{} if !der.ReadASN1(&der, cryptobyte_asn1.SEQUENCE) { return nil, errors.New("x509: invalid certificate policies") } @@ -467,6 +500,10 @@ func parseCertificatePoliciesExtension(der cryptobyte.String) ([]OID, error) { if !der.ReadASN1(&cp, cryptobyte_asn1.SEQUENCE) || !cp.ReadASN1(&OIDBytes, cryptobyte_asn1.OBJECT_IDENTIFIER) { return nil, errors.New("x509: invalid certificate policies") } + if seenOIDs[string(OIDBytes)] { + return nil, errors.New("x509: invalid certificate policies") + } + seenOIDs[string(OIDBytes)] = true oid, ok := newOIDFromDER(OIDBytes) if !ok { return nil, errors.New("x509: invalid certificate policies") @@ -559,15 +596,7 @@ func parseNameConstraintsExtension(out *Certificate, e pkix.Extension) (unhandle return nil, nil, nil, nil, errors.New("x509: invalid constraint value: " + err.Error()) } - trimmedDomain := domain - if len(trimmedDomain) > 0 && trimmedDomain[0] == '.' { - // constraints can have a leading - // period to exclude the domain - // itself, but that's not valid in a - // normal domain name. - trimmedDomain = trimmedDomain[1:] - } - if _, ok := domainToReverseLabels(trimmedDomain); !ok { + if !domainNameValid(domain, true) { return nil, nil, nil, nil, fmt.Errorf("x509: failed to parse dnsName constraint %q", domain) } dnsNames = append(dnsNames, domain) @@ -608,12 +637,7 @@ func parseNameConstraintsExtension(out *Certificate, e pkix.Extension) (unhandle return nil, nil, nil, nil, fmt.Errorf("x509: failed to parse rfc822Name constraint %q", constraint) } } else { - // Otherwise it's a domain name. - domain := constraint - if len(domain) > 0 && domain[0] == '.' { - domain = domain[1:] - } - if _, ok := domainToReverseLabels(domain); !ok { + if !domainNameValid(constraint, true) { return nil, nil, nil, nil, fmt.Errorf("x509: failed to parse rfc822Name constraint %q", constraint) } } @@ -629,15 +653,7 @@ func parseNameConstraintsExtension(out *Certificate, e pkix.Extension) (unhandle return nil, nil, nil, nil, fmt.Errorf("x509: failed to parse URI constraint %q: cannot be IP address", domain) } - trimmedDomain := domain - if len(trimmedDomain) > 0 && trimmedDomain[0] == '.' { - // constraints can have a leading - // period to exclude the domain itself, - // but that's not valid in a normal - // domain name. - trimmedDomain = trimmedDomain[1:] - } - if _, ok := domainToReverseLabels(trimmedDomain); !ok { + if !domainNameValid(domain, true) { return nil, nil, nil, nil, fmt.Errorf("x509: failed to parse URI constraint %q", domain) } uriDomains = append(uriDomains, domain) @@ -747,13 +763,41 @@ func processExtensions(out *Certificate) error { if err != nil { return err } + case 36: + val := cryptobyte.String(e.Value) + if !val.ReadASN1(&val, cryptobyte_asn1.SEQUENCE) { + return errors.New("x509: invalid policy constraints extension") + } + if val.PeekASN1Tag(cryptobyte_asn1.Tag(0).ContextSpecific()) { + var v int64 + if !val.ReadASN1Int64WithTag(&v, cryptobyte_asn1.Tag(0).ContextSpecific()) { + return errors.New("x509: invalid policy constraints extension") + } + out.RequireExplicitPolicy = int(v) + // Check for overflow. + if int64(out.RequireExplicitPolicy) != v { + return errors.New("x509: policy constraints requireExplicitPolicy field overflows int") + } + out.RequireExplicitPolicyZero = out.RequireExplicitPolicy == 0 + } + if val.PeekASN1Tag(cryptobyte_asn1.Tag(1).ContextSpecific()) { + var v int64 + if !val.ReadASN1Int64WithTag(&v, cryptobyte_asn1.Tag(1).ContextSpecific()) { + return errors.New("x509: invalid policy constraints extension") + } + out.InhibitPolicyMapping = int(v) + // Check for overflow. + if int64(out.InhibitPolicyMapping) != v { + return errors.New("x509: policy constraints inhibitPolicyMapping field overflows int") + } + out.InhibitPolicyMappingZero = out.InhibitPolicyMapping == 0 + } case 37: out.ExtKeyUsage, out.UnknownExtKeyUsage, err = parseExtKeyUsageExtension(e.Value) if err != nil { return err } - case 14: - // RFC 5280, 4.2.1.2 + case 14: // RFC 5280, 4.2.1.2 if e.Critical { // Conforming CAs MUST mark this extension as non-critical return errors.New("x509: subject key identifier incorrectly marked critical") @@ -775,6 +819,27 @@ func processExtensions(out *Certificate) error { out.PolicyIdentifiers = append(out.PolicyIdentifiers, oid) } } + case 33: + val := cryptobyte.String(e.Value) + if !val.ReadASN1(&val, cryptobyte_asn1.SEQUENCE) { + return errors.New("x509: invalid policy mappings extension") + } + for !val.Empty() { + var s cryptobyte.String + var issuer, subject cryptobyte.String + if !val.ReadASN1(&s, cryptobyte_asn1.SEQUENCE) || + !s.ReadASN1(&issuer, cryptobyte_asn1.OBJECT_IDENTIFIER) || + !s.ReadASN1(&subject, cryptobyte_asn1.OBJECT_IDENTIFIER) { + return errors.New("x509: invalid policy mappings extension") + } + out.PolicyMappings = append(out.PolicyMappings, PolicyMapping{OID{issuer}, OID{subject}}) + } + case 54: + val := cryptobyte.String(e.Value) + if !val.ReadASN1Integer(&out.InhibitAnyPolicy) { + return errors.New("x509: invalid inhibit any policy extension") + } + out.InhibitAnyPolicyZero = out.InhibitAnyPolicy == 0 default: // Unknown extensions are recorded if critical. unhandled = true @@ -1024,7 +1089,7 @@ func ParseCertificate(der []byte) (*Certificate, error) { if len(der) != len(cert.Raw) { return nil, errors.New("x509: trailing data") } - return cert, err + return cert, nil } // ParseCertificates parses one or more certificates from the given ASN.1 DER @@ -1229,3 +1294,62 @@ func ParseRevocationList(der []byte) (*RevocationList, error) { return rl, nil } + +// domainNameValid is an alloc-less version of the checks that +// domainToReverseLabels does. +func domainNameValid(s string, constraint bool) bool { + // TODO(#75835): This function omits a number of checks which we + // really should be doing to enforce that domain names are valid names per + // RFC 1034. We previously enabled these checks, but this broke a + // significant number of certificates we previously considered valid, and we + // happily create via CreateCertificate (et al). We should enable these + // checks, but will need to gate them behind a GODEBUG. + // + // I have left the checks we previously enabled, noted with "TODO(#75835)" so + // that we can easily re-enable them once we unbreak everyone. + + // TODO(#75835): this should only be true for constraints. + if len(s) == 0 { + return true + } + + // Do not allow trailing period (FQDN format is not allowed in SANs or + // constraints). + if s[len(s)-1] == '.' { + return false + } + + // TODO(#75835): domains must have at least one label, cannot have + // a leading empty label, and cannot be longer than 253 characters. + // if len(s) == 0 || (!constraint && s[0] == '.') || len(s) > 253 { + // return false + // } + + lastDot := -1 + if constraint && s[0] == '.' { + s = s[1:] + } + + for i := 0; i <= len(s); i++ { + if i < len(s) && (s[i] < 33 || s[i] > 126) { + // Invalid character. + return false + } + if i == len(s) || s[i] == '.' { + labelLen := i + if lastDot >= 0 { + labelLen -= lastDot + 1 + } + if labelLen == 0 { + return false + } + // TODO(#75835): labels cannot be longer than 63 characters. + // if labelLen > 63 { + // return false + // } + lastDot = i + } + } + + return true +} diff --git a/src/crypto/x509/parser_test.go b/src/crypto/x509/parser_test.go index b31f9cdb248e08..d53b805b786990 100644 --- a/src/crypto/x509/parser_test.go +++ b/src/crypto/x509/parser_test.go @@ -5,7 +5,13 @@ package x509 import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" "encoding/asn1" + "encoding/pem" + "os" + "strings" "testing" cryptobyte_asn1 "golang.org/x/crypto/cryptobyte/asn1" @@ -22,8 +28,8 @@ func TestParseASN1String(t *testing.T) { { name: "T61String", tag: cryptobyte_asn1.T61String, - value: []byte{80, 81, 82}, - expected: string("PQR"), + value: []byte{0xbf, 0x61, 0x3f}, + expected: string("¿a?"), }, { name: "PrintableString", @@ -61,6 +67,30 @@ func TestParseASN1String(t *testing.T) { value: []byte{255}, expectedErr: "invalid BMPString", }, + { + name: "BMPString (invalid surrogate)", + tag: cryptobyte_asn1.Tag(asn1.TagBMPString), + value: []byte{80, 81, 216, 1}, + expectedErr: "invalid BMPString", + }, + { + name: "BMPString (invalid noncharacter 0xfdd1)", + tag: cryptobyte_asn1.Tag(asn1.TagBMPString), + value: []byte{80, 81, 253, 209}, + expectedErr: "invalid BMPString", + }, + { + name: "BMPString (invalid noncharacter 0xffff)", + tag: cryptobyte_asn1.Tag(asn1.TagBMPString), + value: []byte{80, 81, 255, 255}, + expectedErr: "invalid BMPString", + }, + { + name: "BMPString (invalid noncharacter 0xfffe)", + tag: cryptobyte_asn1.Tag(asn1.TagBMPString), + value: []byte{80, 81, 255, 254}, + expectedErr: "invalid BMPString", + }, { name: "IA5String", tag: cryptobyte_asn1.IA5String, @@ -101,3 +131,230 @@ func TestParseASN1String(t *testing.T) { }) } } + +const policyPEM = `-----BEGIN CERTIFICATE----- +MIIGeDCCBWCgAwIBAgIUED9KQBi0ScBDoufB2mgAJ63G5uIwDQYJKoZIhvcNAQEL +BQAwVTELMAkGA1UEBhMCVVMxGDAWBgNVBAoTD1UuUy4gR292ZXJubWVudDENMAsG +A1UECxMERlBLSTEdMBsGA1UEAxMURmVkZXJhbCBCcmlkZ2UgQ0EgRzQwHhcNMjAx +MDIyMTcwNDE5WhcNMjMxMDIyMTcwNDE5WjCBgTELMAkGA1UEBhMCVVMxHTAbBgNV +BAoTFFN5bWFudGVjIENvcnBvcmF0aW9uMR8wHQYDVQQLExZTeW1hbnRlYyBUcnVz +dCBOZXR3b3JrMTIwMAYDVQQDEylTeW1hbnRlYyBDbGFzcyAzIFNTUCBJbnRlcm1l +ZGlhdGUgQ0EgLSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL2p +75cMpx86sS2aH4r+0o8r+m/KTrPrknWP0RA9Kp6sewAzkNa7BVwg0jOhyamiv1iP +Cns10usoH93nxYbXLWF54vOLRdYU/53KEPNmgkj2ipMaTLuaReBghNibikWSnAmy +S8RItaDMs8tdF2goKPI4xWiamNwqe92VC+pic2tq0Nva3Y4kvMDJjtyje3uduTtL +oyoaaHkrX7i7gE67psnMKj1THUtre1JV1ohl9+oOuyot4p3eSxVlrMWiiwb11bnk +CakecOz/mP2DHMGg6pZ/BeJ+ThaLUylAXECARIqHc9UwRPKC9BfLaCX4edIoeYiB +loRs4KdqLdg/I9eTwKkCAwEAAaOCAxEwggMNMB0GA1UdDgQWBBQ1Jn1QleGhwb0F +1cOdd0LHDBOWjDAfBgNVHSMEGDAWgBR58ABJ6393wl1BAmU0ipAjmx4HbzAOBgNV +HQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zCBiAYDVR0gBIGAMH4wDAYKYIZI +AWUDAgEDAzAMBgpghkgBZQMCAQMMMAwGCmCGSAFlAwIBAw4wDAYKYIZIAWUDAgED +DzAMBgpghkgBZQMCAQMSMAwGCmCGSAFlAwIBAxMwDAYKYIZIAWUDAgEDFDAMBgpg +hkgBZQMCAQMlMAwGCmCGSAFlAwIBAyYwggESBgNVHSEEggEJMIIBBTAbBgpghkgB +ZQMCAQMDBg1ghkgBhvhFAQcXAwEGMBsGCmCGSAFlAwIBAwwGDWCGSAGG+EUBBxcD +AQcwGwYKYIZIAWUDAgEDDgYNYIZIAYb4RQEHFwMBDjAbBgpghkgBZQMCAQMPBg1g +hkgBhvhFAQcXAwEPMBsGCmCGSAFlAwIBAxIGDWCGSAGG+EUBBxcDARIwGwYKYIZI +AWUDAgEDEwYNYIZIAYb4RQEHFwMBETAbBgpghkgBZQMCAQMUBg1ghkgBhvhFAQcX +AwEUMBsGCmCGSAFlAwIBAyUGDWCGSAGG+EUBBxcDAQgwGwYKYIZIAWUDAgEDJgYN +YIZIAYb4RQEHFwMBJDBgBggrBgEFBQcBCwRUMFIwUAYIKwYBBQUHMAWGRGh0dHA6 +Ly9zc3Atc2lhLnN5bWF1dGguY29tL1NUTlNTUC9DZXJ0c19Jc3N1ZWRfYnlfQ2xh +c3MzU1NQQ0EtRzMucDdjMA8GA1UdJAQIMAaAAQCBAQAwCgYDVR02BAMCAQAwUQYI +KwYBBQUHAQEERTBDMEEGCCsGAQUFBzAChjVodHRwOi8vcmVwby5mcGtpLmdvdi9i +cmlkZ2UvY2FDZXJ0c0lzc3VlZFRvZmJjYWc0LnA3YzA3BgNVHR8EMDAuMCygKqAo +hiZodHRwOi8vcmVwby5mcGtpLmdvdi9icmlkZ2UvZmJjYWc0LmNybDANBgkqhkiG +9w0BAQsFAAOCAQEAA751TycC1f/WTkHmedF9ZWxP58Jstmwvkyo8bKueJ0eF7LTG +BgQlzE2B9vke4sFhd4V+BdgOPGE1dsGzllYKCWg0BhkCBs5kIJ7F6Ay6G1TBuGU1 +Ie8247GL+P9pcC5TVvXHC/62R2w3DuD/vAPLbYEbSQjobXlsqt8Kmtd6yK/jVuDV +BTZMdZmvoNtjemqmgcBXHsf0ctVm0m6tH5uYqyVxu8tfyUis6Cf303PHj+spWP1k +gc5PYnVF0ot7qAmNFENIpbKg3BdusBkF9rGxLaDSUBvSc7+s9iQz9d/iRuAebrYu ++eqUlJ2lsjS1U8qyPmlH+spfPNbAEQEsuP32Aw== +-----END CERTIFICATE----- +` + +func TestPolicyParse(t *testing.T) { + b, _ := pem.Decode([]byte(policyPEM)) + c, err := ParseCertificate(b.Bytes) + if err != nil { + t.Fatal(err) + } + if len(c.Policies) != 9 { + t.Errorf("unexpected number of policies: got %d, want %d", len(c.Policies), 9) + } + if len(c.PolicyMappings) != 9 { + t.Errorf("unexpected number of policy mappings: got %d, want %d", len(c.PolicyMappings), 9) + } + if !c.RequireExplicitPolicyZero { + t.Error("expected RequireExplicitPolicyZero to be set") + } + if !c.InhibitPolicyMappingZero { + t.Error("expected InhibitPolicyMappingZero to be set") + } + if !c.InhibitAnyPolicyZero { + t.Error("expected InhibitAnyPolicyZero to be set") + } +} + +func TestParsePolicies(t *testing.T) { + for _, tc := range []string{ + "testdata/policy_leaf_duplicate.pem", + "testdata/policy_leaf_invalid.pem", + } { + t.Run(tc, func(t *testing.T) { + b, err := os.ReadFile(tc) + if err != nil { + t.Fatal(err) + } + p, _ := pem.Decode(b) + _, err = ParseCertificate(p.Bytes) + if err == nil { + t.Error("parsing should've failed") + } + }) + } +} + +func TestParseCertificateNegativeMaxPathLength(t *testing.T) { + certs := []string{ + // Certificate with MaxPathLen set to -1. + ` +-----BEGIN CERTIFICATE----- +MIIByTCCATKgAwIBAgIBATANBgkqhkiG9w0BAQsFADAPMQ0wCwYDVQQDEwRURVNU +MB4XDTcwMDEwMTAwMTY0MFoXDTcwMDEwMjAzNDY0MFowDzENMAsGA1UEAxMEVEVT +VDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsaHglFuSicTT8TKfipgsSi3N +Wb/TcvuAhanFF1VGB+vS95kO7yFqyfRgX3GgOwT0KlJVsVjPjghEGR9RGTSLqkTD +UFbiBgm8+VEPMOrUtIHIHXhl+ye44AkOEStxfz7gjN/EAS2h8ffPKhvDTHOlShKw +Y3LQlxR0LdeJXq3eSqUCAwEAAaM1MDMwEgYDVR0TAQH/BAgwBgEB/wIB/zAdBgNV +HQ4EFgQUrbrk0tqQAEsce8uYifP0BIVhuFAwDQYJKoZIhvcNAQELBQADgYEAIkhV +ZBj1ThT+eyh50XsoU570NUysTg3Nj/3lbkEolzdcE+wu0CPXvgxLRM6Y62u1ey82 +8d5VQHstzF4dXgc3W+O9UySa+CKdcHx/q7o7seOGXdysT0IJtAY3w66mFkuF7PIn +y9b7M5t6pmWjb7N0QqGuWeNqi4ZvS8gLKmVEgGY= +-----END CERTIFICATE----- +`, + // Certificate with MaxPathLen set to -2. + ` +-----BEGIN CERTIFICATE----- +MIIByTCCATKgAwIBAgIBATANBgkqhkiG9w0BAQsFADAPMQ0wCwYDVQQDEwRURVNU +MB4XDTcwMDEwMTAwMTY0MFoXDTcwMDEwMjAzNDY0MFowDzENMAsGA1UEAxMEVEVT +VDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAsaHglFuSicTT8TKfipgsSi3N +Wb/TcvuAhanFF1VGB+vS95kO7yFqyfRgX3GgOwT0KlJVsVjPjghEGR9RGTSLqkTD +UFbiBgm8+VEPMOrUtIHIHXhl+ye44AkOEStxfz7gjN/EAS2h8ffPKhvDTHOlShKw +Y3LQlxR0LdeJXq3eSqUCAwEAAaM1MDMwEgYDVR0TAQH/BAgwBgEB/wIB/jAdBgNV +HQ4EFgQUrbrk0tqQAEsce8uYifP0BIVhuFAwDQYJKoZIhvcNAQELBQADgYEAGjIr +YGQc7Ods+BuKck7p+vpAMONM8SLEuUtKorCP3ecsO51MoA4/niLbgMHaOGNHwzMp +ajg0zLbY0Dj6Ml0VZ+lS3rjgTEhYXc626eZkoQqgUzL1jhe3S0ZbSxxmHMBKjJFl +d5l1tRhScKu2NBgm74nYmJxJYgvuTA38wGhRrGU= +-----END CERTIFICATE----- +`, + } + + for _, cert := range certs { + b, _ := pem.Decode([]byte(cert)) + _, err := ParseCertificate(b.Bytes) + if err == nil || err.Error() != "x509: invalid basic constraints" { + t.Errorf(`ParseCertificate() = %v; want = "x509: invalid basic constraints"`, err) + } + } +} + +func TestDomainNameValid(t *testing.T) { + for _, tc := range []struct { + name string + dnsName string + constraint bool + valid bool + }{ + // TODO(#75835): these tests are for stricter name validation, which we + // had to disable. Once we reenable these strict checks, behind a + // GODEBUG, we should add them back in. + // {"empty name, name", "", false, false}, + // {"254 char label, name", strings.Repeat("a.a", 84) + "aaa", false, false}, + // {"254 char label, constraint", strings.Repeat("a.a", 84) + "aaa", true, false}, + // {"253 char label, name", strings.Repeat("a.a", 84) + "aa", false, false}, + // {"253 char label, constraint", strings.Repeat("a.a", 84) + "aa", true, false}, + // {"64 char single label, name", strings.Repeat("a", 64), false, false}, + // {"64 char single label, constraint", strings.Repeat("a", 64), true, false}, + // {"64 char label, name", "a." + strings.Repeat("a", 64), false, false}, + // {"64 char label, constraint", "a." + strings.Repeat("a", 64), true, false}, + + // TODO(#75835): these are the inverse of the tests above, they should be removed + // once the strict checking is enabled. + {"254 char label, name", strings.Repeat("a.a", 84) + "aaa", false, true}, + {"254 char label, constraint", strings.Repeat("a.a", 84) + "aaa", true, true}, + {"253 char label, name", strings.Repeat("a.a", 84) + "aa", false, true}, + {"253 char label, constraint", strings.Repeat("a.a", 84) + "aa", true, true}, + {"64 char single label, name", strings.Repeat("a", 64), false, true}, + {"64 char single label, constraint", strings.Repeat("a", 64), true, true}, + {"64 char label, name", "a." + strings.Repeat("a", 64), false, true}, + {"64 char label, constraint", "a." + strings.Repeat("a", 64), true, true}, + + // Check we properly enforce properties of domain names. + {"empty name, constraint", "", true, true}, + {"empty label, name", "a..a", false, false}, + {"empty label, constraint", "a..a", true, false}, + {"period, name", ".", false, false}, + {"period, constraint", ".", true, false}, // TODO(roland): not entirely clear if this is a valid constraint (require at least one label?) + {"valid, name", "a.b.c", false, true}, + {"valid, constraint", "a.b.c", true, true}, + {"leading period, name", ".a.b.c", false, false}, + {"leading period, constraint", ".a.b.c", true, true}, + {"trailing period, name", "a.", false, false}, + {"trailing period, constraint", "a.", true, false}, + {"bare label, name", "a", false, true}, + {"bare label, constraint", "a", true, true}, + {"63 char single label, name", strings.Repeat("a", 63), false, true}, + {"63 char single label, constraint", strings.Repeat("a", 63), true, true}, + {"63 char label, name", "a." + strings.Repeat("a", 63), false, true}, + {"63 char label, constraint", "a." + strings.Repeat("a", 63), true, true}, + } { + t.Run(tc.name, func(t *testing.T) { + valid := domainNameValid(tc.dnsName, tc.constraint) + if tc.valid != valid { + t.Errorf("domainNameValid(%q, %t) = %v; want %v", tc.dnsName, tc.constraint, !tc.valid, tc.valid) + } + // Also check that we enforce the same properties as domainToReverseLabels + trimmedName := tc.dnsName + if tc.constraint && len(trimmedName) > 1 && trimmedName[0] == '.' { + trimmedName = trimmedName[1:] + } + _, revValid := domainToReverseLabels(trimmedName) + if valid != revValid { + t.Errorf("domainNameValid(%q, %t) = %t != domainToReverseLabels(%q) = %t", tc.dnsName, tc.constraint, valid, trimmedName, revValid) + } + }) + } +} + +func TestRoundtripWeirdSANs(t *testing.T) { + // TODO(#75835): check that certificates we create with CreateCertificate that have malformed SAN values + // can be parsed by ParseCertificate. We should eventually restrict this, but for now we have to maintain + // this property as people have been relying on it. + k, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + badNames := []string{ + "baredomain", + "baredomain.", + strings.Repeat("a", 255), + strings.Repeat("a", 65) + ".com", + } + tmpl := &Certificate{ + EmailAddresses: badNames, + DNSNames: badNames, + } + b, err := CreateCertificate(rand.Reader, tmpl, tmpl, &k.PublicKey, k) + if err != nil { + t.Fatal(err) + } + _, err = ParseCertificate(b) + if err != nil { + t.Fatalf("Couldn't roundtrip certificate: %v", err) + } +} + +func FuzzDomainNameValid(f *testing.F) { + f.Fuzz(func(t *testing.T, data string) { + domainNameValid(data, false) + domainNameValid(data, true) + }) +} diff --git a/src/crypto/x509/pkcs1.go b/src/crypto/x509/pkcs1.go index 94c7bbb2304fd1..68aa8dd980c10e 100644 --- a/src/crypto/x509/pkcs1.go +++ b/src/crypto/x509/pkcs1.go @@ -8,6 +8,7 @@ import ( "crypto/rsa" "encoding/asn1" "errors" + "internal/godebug" "math/big" ) @@ -19,10 +20,9 @@ type pkcs1PrivateKey struct { D *big.Int P *big.Int Q *big.Int - // We ignore these values, if present, because rsa will calculate them. - Dp *big.Int `asn1:"optional"` - Dq *big.Int `asn1:"optional"` - Qinv *big.Int `asn1:"optional"` + Dp *big.Int `asn1:"optional"` + Dq *big.Int `asn1:"optional"` + Qinv *big.Int `asn1:"optional"` AdditionalPrimes []pkcs1AdditionalRSAPrime `asn1:"optional,omitempty"` } @@ -41,9 +41,16 @@ type pkcs1PublicKey struct { E int } +// x509rsacrt, if zero, makes ParsePKCS1PrivateKey ignore and recompute invalid +// CRT values in the RSA private key. +var x509rsacrt = godebug.New("x509rsacrt") + // ParsePKCS1PrivateKey parses an [RSA] private key in PKCS #1, ASN.1 DER form. // // This kind of key is commonly encoded in PEM blocks of type "RSA PRIVATE KEY". +// +// Before Go 1.24, the CRT parameters were ignored and recomputed. To restore +// the old behavior, use the GODEBUG=x509rsacrt=0 environment variable. func ParsePKCS1PrivateKey(der []byte) (*rsa.PrivateKey, error) { var priv pkcs1PrivateKey rest, err := asn1.Unmarshal(der, &priv) @@ -64,7 +71,10 @@ func ParsePKCS1PrivateKey(der []byte) (*rsa.PrivateKey, error) { return nil, errors.New("x509: unsupported private key version") } - if priv.N.Sign() <= 0 || priv.D.Sign() <= 0 || priv.P.Sign() <= 0 || priv.Q.Sign() <= 0 { + if priv.N.Sign() <= 0 || priv.D.Sign() <= 0 || priv.P.Sign() <= 0 || priv.Q.Sign() <= 0 || + priv.Dp != nil && priv.Dp.Sign() <= 0 || + priv.Dq != nil && priv.Dq.Sign() <= 0 || + priv.Qinv != nil && priv.Qinv.Sign() <= 0 { return nil, errors.New("x509: private key contains zero or negative value") } @@ -78,6 +88,9 @@ func ParsePKCS1PrivateKey(der []byte) (*rsa.PrivateKey, error) { key.Primes = make([]*big.Int, 2+len(priv.AdditionalPrimes)) key.Primes[0] = priv.P key.Primes[1] = priv.Q + key.Precomputed.Dp = priv.Dp + key.Precomputed.Dq = priv.Dq + key.Precomputed.Qinv = priv.Qinv for i, a := range priv.AdditionalPrimes { if a.Prime.Sign() <= 0 { return nil, errors.New("x509: private key contains zero or negative prime") @@ -87,11 +100,23 @@ func ParsePKCS1PrivateKey(der []byte) (*rsa.PrivateKey, error) { // them as needed. } - err = key.Validate() - if err != nil { + key.Precompute() + if err := key.Validate(); err != nil { + // If x509rsacrt=0 is set, try dropping the CRT values and + // rerunning precomputation and key validation. + if x509rsacrt.Value() == "0" { + key.Precomputed.Dp = nil + key.Precomputed.Dq = nil + key.Precomputed.Qinv = nil + key.Precompute() + if err := key.Validate(); err == nil { + x509rsacrt.IncNonDefault() + return key, nil + } + } + return nil, err } - key.Precompute() return key, nil } @@ -101,6 +126,10 @@ func ParsePKCS1PrivateKey(der []byte) (*rsa.PrivateKey, error) { // This kind of key is commonly encoded in PEM blocks of type "RSA PRIVATE KEY". // For a more flexible key format which is not [RSA] specific, use // [MarshalPKCS8PrivateKey]. +// +// The key must have passed validation by calling [rsa.PrivateKey.Validate] +// first. MarshalPKCS1PrivateKey calls [rsa.PrivateKey.Precompute], which may +// modify the key if not already precomputed. func MarshalPKCS1PrivateKey(key *rsa.PrivateKey) []byte { key.Precompute() diff --git a/src/crypto/x509/pkcs8.go b/src/crypto/x509/pkcs8.go index 08e9da404c33e9..d0ab573ff33236 100644 --- a/src/crypto/x509/pkcs8.go +++ b/src/crypto/x509/pkcs8.go @@ -32,6 +32,9 @@ type pkcs8 struct { // in the future. // // This kind of key is commonly encoded in PEM blocks of type "PRIVATE KEY". +// +// Before Go 1.24, the CRT parameters of RSA keys were ignored and recomputed. +// To restore the old behavior, use the GODEBUG=x509rsacrt=0 environment variable. func ParsePKCS8PrivateKey(der []byte) (key any, err error) { var privKey pkcs8 if _, err := asn1.Unmarshal(der, &privKey); err != nil { @@ -98,6 +101,8 @@ func ParsePKCS8PrivateKey(der []byte) (key any, err error) { // Unsupported key types result in an error. // // This kind of key is commonly encoded in PEM blocks of type "PRIVATE KEY". +// +// MarshalPKCS8PrivateKey runs [rsa.PrivateKey.Precompute] on RSA keys. func MarshalPKCS8PrivateKey(key any) ([]byte, error) { var privKey pkcs8 @@ -107,6 +112,10 @@ func MarshalPKCS8PrivateKey(key any) ([]byte, error) { Algorithm: oidPublicKeyRSA, Parameters: asn1.NullRawValue, } + k.Precompute() + if err := k.Validate(); err != nil { + return nil, err + } privKey.PrivateKey = MarshalPKCS1PrivateKey(k) case *ecdsa.PrivateKey: diff --git a/src/crypto/x509/pkits_test.go b/src/crypto/x509/pkits_test.go new file mode 100644 index 00000000000000..b1139bbf9c638c --- /dev/null +++ b/src/crypto/x509/pkits_test.go @@ -0,0 +1,186 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package x509 + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" +) + +var nistTestPolicies = map[string]OID{ + "anyPolicy": anyPolicyOID, + "NIST-test-policy-1": mustNewOIDFromInts([]uint64{2, 16, 840, 1, 101, 3, 2, 1, 48, 1}), + "NIST-test-policy-2": mustNewOIDFromInts([]uint64{2, 16, 840, 1, 101, 3, 2, 1, 48, 2}), + "NIST-test-policy-3": mustNewOIDFromInts([]uint64{2, 16, 840, 1, 101, 3, 2, 1, 48, 3}), + "NIST-test-policy-6": mustNewOIDFromInts([]uint64{2, 16, 840, 1, 101, 3, 2, 1, 48, 6}), +} + +func TestNISTPKITSPolicy(t *testing.T) { + // This test runs a subset of the NIST PKI path validation test suite that + // focuses of policy validation, rather than the entire suite. Since the + // suite assumes you are only validating the path, rather than building + // _and_ validating the path, we take the path as given and run + // policiesValid on it. + + certDir := "testdata/nist-pkits/certs" + + var testcases []struct { + Name string + CertPath []string + InitialPolicySet []string + InitialPolicyMappingInhibit bool + InitialExplicitPolicy bool + InitialAnyPolicyInhibit bool + ShouldValidate bool + Skipped bool + } + b, err := os.ReadFile("testdata/nist-pkits/vectors.json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(b, &testcases); err != nil { + t.Fatal(err) + } + + policyTests := map[string]bool{ + "4.8.1 All Certificates Same Policy Test1 (Subpart 1)": true, + "4.8.1 All Certificates Same Policy Test1 (Subpart 2)": true, + "4.8.1 All Certificates Same Policy Test1 (Subpart 3)": true, + "4.8.1 All Certificates Same Policy Test1 (Subpart 4)": true, + "4.8.2 All Certificates No Policies Test2 (Subpart 1)": true, + "4.8.2 All Certificates No Policies Test2 (Subpart 2)": true, + "4.8.3 Different Policies Test3 (Subpart 1)": true, + "4.8.3 Different Policies Test3 (Subpart 2)": true, + "4.8.3 Different Policies Test3 (Subpart 3)": true, + "4.8.4 Different Policies Test4": true, + "4.8.5 Different Policies Test5": true, + "4.8.6 Overlapping Policies Test6 (Subpart 1)": true, + "4.8.6 Overlapping Policies Test6 (Subpart 2)": true, + "4.8.6 Overlapping Policies Test6 (Subpart 3)": true, + "4.8.7 Different Policies Test7": true, + "4.8.8 Different Policies Test8": true, + "4.8.9 Different Policies Test9": true, + "4.8.10 All Certificates Same Policies Test10 (Subpart 1)": true, + "4.8.10 All Certificates Same Policies Test10 (Subpart 2)": true, + "4.8.10 All Certificates Same Policies Test10 (Subpart 3)": true, + "4.8.11 All Certificates AnyPolicy Test11 (Subpart 1)": true, + "4.8.11 All Certificates AnyPolicy Test11 (Subpart 2)": true, + "4.8.12 Different Policies Test12": true, + "4.8.13 All Certificates Same Policies Test13 (Subpart 1)": true, + "4.8.13 All Certificates Same Policies Test13 (Subpart 2)": true, + "4.8.13 All Certificates Same Policies Test13 (Subpart 3)": true, + "4.8.14 AnyPolicy Test14 (Subpart 1)": true, + "4.8.14 AnyPolicy Test14 (Subpart 2)": true, + "4.8.15 User Notice Qualifier Test15": true, + "4.8.16 User Notice Qualifier Test16": true, + "4.8.17 User Notice Qualifier Test17": true, + "4.8.18 User Notice Qualifier Test18 (Subpart 1)": true, + "4.8.18 User Notice Qualifier Test18 (Subpart 2)": true, + "4.8.19 User Notice Qualifier Test19": true, + "4.8.20 CPS Pointer Qualifier Test20": true, + "4.9.1 Valid RequireExplicitPolicy Test1": true, + "4.9.2 Valid RequireExplicitPolicy Test2": true, + "4.9.3 Invalid RequireExplicitPolicy Test3": true, + "4.9.4 Valid RequireExplicitPolicy Test4": true, + "4.9.5 Invalid RequireExplicitPolicy Test5": true, + "4.9.6 Valid Self-Issued requireExplicitPolicy Test6": true, + "4.9.7 Invalid Self-Issued requireExplicitPolicy Test7": true, + "4.9.8 Invalid Self-Issued requireExplicitPolicy Test8": true, + "4.10.1.1 Valid Policy Mapping Test1 (Subpart 1)": true, + "4.10.1.2 Valid Policy Mapping Test1 (Subpart 2)": true, + "4.10.1.3 Valid Policy Mapping Test1 (Subpart 3)": true, + "4.10.2 Invalid Policy Mapping Test2 (Subpart 1)": true, + "4.10.2 Invalid Policy Mapping Test2 (Subpart 2)": true, + "4.10.3 Valid Policy Mapping Test3 (Subpart 1)": true, + "4.10.3 Valid Policy Mapping Test3 (Subpart 2)": true, + "4.10.4 Invalid Policy Mapping Test4": true, + "4.10.5 Valid Policy Mapping Test5 (Subpart 1)": true, + "4.10.5 Valid Policy Mapping Test5 (Subpart 2)": true, + "4.10.6 Valid Policy Mapping Test6 (Subpart 1)": true, + "4.10.6 Valid Policy Mapping Test6 (Subpart 2)": true, + "4.10.7 Invalid Mapping From anyPolicy Test7": true, + "4.10.8 Invalid Mapping To anyPolicy Test8": true, + "4.10.9 Valid Policy Mapping Test9": true, + "4.10.10 Invalid Policy Mapping Test10": true, + "4.10.11 Valid Policy Mapping Test11": true, + "4.10.12 Valid Policy Mapping Test12 (Subpart 1)": true, + "4.10.12 Valid Policy Mapping Test12 (Subpart 2)": true, + "4.10.13 Valid Policy Mapping Test13 (Subpart 1)": true, + "4.10.13 Valid Policy Mapping Test13 (Subpart 2)": true, + "4.10.13 Valid Policy Mapping Test13 (Subpart 3)": true, + "4.10.14 Valid Policy Mapping Test14": true, + "4.11.1 Invalid inhibitPolicyMapping Test1": true, + "4.11.2 Valid inhibitPolicyMapping Test2": true, + "4.11.3 Invalid inhibitPolicyMapping Test3": true, + "4.11.4 Valid inhibitPolicyMapping Test4": true, + "4.11.5 Invalid inhibitPolicyMapping Test5": true, + "4.11.6 Invalid inhibitPolicyMapping Test6": true, + "4.11.7 Valid Self-Issued inhibitPolicyMapping Test7": true, + "4.11.8 Invalid Self-Issued inhibitPolicyMapping Test8": true, + "4.11.9 Invalid Self-Issued inhibitPolicyMapping Test9": true, + "4.11.10 Invalid Self-Issued inhibitPolicyMapping Test10": true, + "4.11.11 Invalid Self-Issued inhibitPolicyMapping Test11": true, + "4.12.1 Invalid inhibitAnyPolicy Test1": true, + "4.12.2 Valid inhibitAnyPolicy Test2": true, + "4.12.3 inhibitAnyPolicy Test3 (Subpart 1)": true, + "4.12.3 inhibitAnyPolicy Test3 (Subpart 2)": true, + "4.12.4 Invalid inhibitAnyPolicy Test4": true, + "4.12.5 Invalid inhibitAnyPolicy Test5": true, + "4.12.6 Invalid inhibitAnyPolicy Test6": true, + "4.12.7 Valid Self-Issued inhibitAnyPolicy Test7": true, + "4.12.8 Invalid Self-Issued inhibitAnyPolicy Test8": true, + "4.12.9 Valid Self-Issued inhibitAnyPolicy Test9": true, + "4.12.10 Invalid Self-Issued inhibitAnyPolicy Test10": true, + } + + for _, tc := range testcases { + if !policyTests[tc.Name] { + continue + } + t.Run(tc.Name, func(t *testing.T) { + var chain []*Certificate + for _, c := range tc.CertPath { + certDER, err := os.ReadFile(filepath.Join(certDir, c)) + if err != nil { + t.Fatal(err) + } + cert, err := ParseCertificate(certDER) + if err != nil { + t.Fatal(err) + } + chain = append(chain, cert) + } + slices.Reverse(chain) + + var initialPolicies []OID + for _, pstr := range tc.InitialPolicySet { + policy, ok := nistTestPolicies[pstr] + if !ok { + t.Fatalf("unknown test policy: %s", pstr) + } + initialPolicies = append(initialPolicies, policy) + } + + valid := policiesValid(chain, VerifyOptions{ + CertificatePolicies: initialPolicies, + inhibitPolicyMapping: tc.InitialPolicyMappingInhibit, + requireExplicitPolicy: tc.InitialExplicitPolicy, + inhibitAnyPolicy: tc.InitialAnyPolicyInhibit, + }) + if !valid { + if !tc.ShouldValidate { + return + } + t.Fatalf("Failed to validate: %s", err) + } + if !tc.ShouldValidate { + t.Fatal("Expected path validation to fail") + } + }) + } +} diff --git a/src/crypto/x509/platform_test.go b/src/crypto/x509/platform_test.go index b425e02f3bc144..44ceff43f454df 100644 --- a/src/crypto/x509/platform_test.go +++ b/src/crypto/x509/platform_test.go @@ -202,7 +202,6 @@ func TestPlatformVerifier(t *testing.T) { } for _, tc := range tests { - tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() parent := testRoot diff --git a/src/crypto/x509/root.go b/src/crypto/x509/root.go index fbd43430afa8f9..600f75979d6e9e 100644 --- a/src/crypto/x509/root.go +++ b/src/crypto/x509/root.go @@ -20,11 +20,12 @@ import ( // //go:linkname systemRoots var ( - once sync.Once - systemRootsMu sync.RWMutex - systemRoots *CertPool - systemRootsErr error - fallbacksSet bool + once sync.Once + systemRootsMu sync.RWMutex + systemRoots *CertPool + systemRootsErr error + fallbacksSet bool + useFallbackRoots bool ) func systemRootsPool() *CertPool { @@ -37,10 +38,28 @@ func systemRootsPool() *CertPool { func initSystemRoots() { systemRootsMu.Lock() defer systemRootsMu.Unlock() + + fallbackRoots := systemRoots systemRoots, systemRootsErr = loadSystemRoots() if systemRootsErr != nil { systemRoots = nil } + + if fallbackRoots == nil { + return // no fallbacks to try + } + + systemCertsAvail := systemRoots != nil && (systemRoots.len() > 0 || systemRoots.systemPool) + + if !useFallbackRoots && systemCertsAvail { + return + } + + if useFallbackRoots && systemCertsAvail { + x509usefallbackroots.IncNonDefault() // overriding system certs with fallback certs. + } + + systemRoots, systemRootsErr = fallbackRoots, nil } var x509usefallbackroots = godebug.New("x509usefallbackroots") @@ -63,10 +82,6 @@ func SetFallbackRoots(roots *CertPool) { panic("roots must be non-nil") } - // trigger initSystemRoots if it hasn't already been called before we - // take the lock - _ = systemRootsPool() - systemRootsMu.Lock() defer systemRootsMu.Unlock() @@ -75,11 +90,28 @@ func SetFallbackRoots(roots *CertPool) { } fallbacksSet = true - if systemRoots != nil && (systemRoots.len() > 0 || systemRoots.systemPool) { - if x509usefallbackroots.Value() != "1" { - return - } - x509usefallbackroots.IncNonDefault() + // Handle case when initSystemRoots was not yet executed. + // We handle that specially instead of calling loadSystemRoots, to avoid + // spending excessive amount of cpu here, since the SetFallbackRoots in most cases + // is going to be called at program startup. + if systemRoots == nil && systemRootsErr == nil { + systemRoots = roots + useFallbackRoots = x509usefallbackroots.Value() == "1" + return + } + + once.Do(func() { panic("unreachable") }) // asserts that system roots were indeed loaded before. + + forceFallbackRoots := x509usefallbackroots.Value() == "1" + systemCertsAvail := systemRoots != nil && (systemRoots.len() > 0 || systemRoots.systemPool) + + if !forceFallbackRoots && systemCertsAvail { + return + } + + if forceFallbackRoots && systemCertsAvail { + x509usefallbackroots.IncNonDefault() // overriding system certs with fallback certs. } + systemRoots, systemRootsErr = roots, nil } diff --git a/src/crypto/x509/root_darwin.go b/src/crypto/x509/root_darwin.go index 469e907a8e154e..3e9aa1ba097b36 100644 --- a/src/crypto/x509/root_darwin.go +++ b/src/crypto/x509/root_darwin.go @@ -5,51 +5,51 @@ package x509 import ( - macOS "crypto/x509/internal/macos" + "crypto/x509/internal/macos" "errors" "fmt" ) func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate, err error) { - certs := macOS.CFArrayCreateMutable() - defer macOS.ReleaseCFArray(certs) - leaf, err := macOS.SecCertificateCreateWithData(c.Raw) + certs := macos.CFArrayCreateMutable() + defer macos.ReleaseCFArray(certs) + leaf, err := macos.SecCertificateCreateWithData(c.Raw) if err != nil { return nil, errors.New("invalid leaf certificate") } - macOS.CFArrayAppendValue(certs, leaf) + macos.CFArrayAppendValue(certs, leaf) if opts.Intermediates != nil { for _, lc := range opts.Intermediates.lazyCerts { c, err := lc.getCert() if err != nil { return nil, err } - sc, err := macOS.SecCertificateCreateWithData(c.Raw) + sc, err := macos.SecCertificateCreateWithData(c.Raw) if err != nil { return nil, err } - macOS.CFArrayAppendValue(certs, sc) + macos.CFArrayAppendValue(certs, sc) } } - policies := macOS.CFArrayCreateMutable() - defer macOS.ReleaseCFArray(policies) - sslPolicy, err := macOS.SecPolicyCreateSSL(opts.DNSName) + policies := macos.CFArrayCreateMutable() + defer macos.ReleaseCFArray(policies) + sslPolicy, err := macos.SecPolicyCreateSSL(opts.DNSName) if err != nil { return nil, err } - macOS.CFArrayAppendValue(policies, sslPolicy) + macos.CFArrayAppendValue(policies, sslPolicy) - trustObj, err := macOS.SecTrustCreateWithCertificates(certs, policies) + trustObj, err := macos.SecTrustCreateWithCertificates(certs, policies) if err != nil { return nil, err } - defer macOS.CFRelease(trustObj) + defer macos.CFRelease(trustObj) if !opts.CurrentTime.IsZero() { - dateRef := macOS.TimeToCFDateRef(opts.CurrentTime) - defer macOS.CFRelease(dateRef) - if err := macOS.SecTrustSetVerifyDate(trustObj, dateRef); err != nil { + dateRef := macos.TimeToCFDateRef(opts.CurrentTime) + defer macos.CFRelease(dateRef) + if err := macos.SecTrustSetVerifyDate(trustObj, dateRef); err != nil { return nil, err } } @@ -59,13 +59,13 @@ func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate // always enforce its SCT requirements, and there are still _some_ people // using TLS or OCSP for that. - if ret, err := macOS.SecTrustEvaluateWithError(trustObj); err != nil { + if ret, err := macos.SecTrustEvaluateWithError(trustObj); err != nil { switch ret { - case macOS.ErrSecCertificateExpired: + case macos.ErrSecCertificateExpired: return nil, CertificateInvalidError{c, Expired, err.Error()} - case macOS.ErrSecHostNameMismatch: + case macos.ErrSecHostNameMismatch: return nil, HostnameError{c, opts.DNSName} - case macOS.ErrSecNotTrusted: + case macos.ErrSecNotTrusted: return nil, UnknownAuthorityError{Cert: c} default: return nil, fmt.Errorf("x509: %s", err) @@ -73,12 +73,13 @@ func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate } chain := [][]*Certificate{{}} - numCerts := macOS.SecTrustGetCertificateCount(trustObj) - for i := 0; i < numCerts; i++ { - certRef, err := macOS.SecTrustGetCertificateAtIndex(trustObj, i) - if err != nil { - return nil, err - } + chainRef, err := macos.SecTrustCopyCertificateChain(trustObj) + if err != nil { + return nil, err + } + defer macos.CFRelease(chainRef) + for i := 0; i < macos.CFArrayGetCount(chainRef); i++ { + certRef := macos.CFArrayGetValueAtIndex(chainRef, i) cert, err := exportCertificate(certRef) if err != nil { return nil, err @@ -87,7 +88,7 @@ func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate } if len(chain[0]) == 0 { // This should _never_ happen, but to be safe - return nil, errors.New("x509: macOS certificate verification internal error") + return nil, errors.New("x509: macos certificate verification internal error") } if opts.DNSName != "" { @@ -117,8 +118,8 @@ func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate } // exportCertificate returns a *Certificate for a SecCertificateRef. -func exportCertificate(cert macOS.CFRef) (*Certificate, error) { - data, err := macOS.SecCertificateCopyData(cert) +func exportCertificate(cert macos.CFRef) (*Certificate, error) { + data, err := macos.SecCertificateCopyData(cert) if err != nil { return nil, err } diff --git a/src/crypto/x509/root_linux_test.go b/src/crypto/x509/root_linux_test.go new file mode 100644 index 00000000000000..d4f92e0972b123 --- /dev/null +++ b/src/crypto/x509/root_linux_test.go @@ -0,0 +1,289 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux + +package x509 + +import ( + "encoding/pem" + "fmt" + "internal/testenv" + "os" + "os/exec" + "syscall" + "testing" +) + +func TestSetFallbackRoots(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } + testenv.MustHaveExec(t) + + test := func(t *testing.T, name string, f func(t *testing.T)) { + t.Run(name, func(t *testing.T) { + if os.Getenv("CRYPTO_X509_SETFALLBACKROOTS_TEST") != "1" { + // Execute test in a separate process with CRYPTO_X509_SETFALBACKROOTS_TEST env. + cmd := exec.Command(os.Args[0], fmt.Sprintf("-test.run=^%v$", t.Name())) + cmd.Env = append(os.Environ(), "CRYPTO_X509_SETFALLBACKROOTS_TEST=1") + cmd.SysProcAttr = &syscall.SysProcAttr{ + Cloneflags: syscall.CLONE_NEWNS | syscall.CLONE_NEWUSER, + UidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getuid(), Size: 1}}, + GidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getgid(), Size: 1}}, + } + out, err := cmd.CombinedOutput() + if err != nil { + if testenv.SyscallIsNotSupported(err) { + t.Skipf("skipping: could not start process with CLONE_NEWNS and CLONE_NEWUSER: %v", err) + } + t.Errorf("%v\n%s", err, out) + } + return + } + + // This test is executed in a separate user and mount namespace, thus + // we can mount a separate "/etc" empty bind mount, without the need for root access. + // On linux all certs reside in /etc, so as we bind an empty dir, we + // get a full control over the system CAs, required for this test. + if err := syscall.Mount(t.TempDir(), "/etc", "", syscall.MS_BIND, ""); err != nil { + if testenv.SyscallIsNotSupported(err) { + t.Skipf("Failed to mount /etc: %v", err) + } + t.Fatalf("Failed to mount /etc: %v", err) + } + + t.Cleanup(func() { + if err := syscall.Unmount("/etc", 0); err != nil { + t.Errorf("failed to unmount /etc: %v", err) + } + }) + + f(t) + }) + } + + newFallbackCertPool := func(t *testing.T) *CertPool { + t.Helper() + + const fallbackCert = `-----BEGIN CERTIFICATE----- +MIICGzCCAaGgAwIBAgIQQdKd0XLq7qeAwSxs6S+HUjAKBggqhkjOPQQDAzBPMQsw +CQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2gg +R3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBYMjAeFw0yMDA5MDQwMDAwMDBaFw00 +MDA5MTcxNjAwMDBaME8xCzAJBgNVBAYTAlVTMSkwJwYDVQQKEyBJbnRlcm5ldCBT +ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEVMBMGA1UEAxMMSVNSRyBSb290IFgyMHYw +EAYHKoZIzj0CAQYFK4EEACIDYgAEzZvVn4CDCuwJSvMWSj5cz3es3mcFDR0HttwW ++1qLFNvicWDEukWVEYmO6gbf9yoWHKS5xcUy4APgHoIYOIvXRdgKam7mAHf7AlF9 +ItgKbppbd9/w+kHsOdx1ymgHDB/qo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T +AQH/BAUwAwEB/zAdBgNVHQ4EFgQUfEKWrt5LSDv6kviejM9ti6lyN5UwCgYIKoZI +zj0EAwMDaAAwZQIwe3lORlCEwkSHRhtFcP9Ymd70/aTSVaYgLXTWNLxBo1BfASdW +tL4ndQavEi51mI38AjEAi/V3bNTIZargCyzuFJ0nN6T5U6VR5CmD1/iQMVtCnwr1 +/q4AaOeMSQ+2b1tbFfLn +-----END CERTIFICATE----- +` + b, _ := pem.Decode([]byte(fallbackCert)) + cert, err := ParseCertificate(b.Bytes) + if err != nil { + t.Fatal(err) + } + p := NewCertPool() + p.AddCert(cert) + return p + } + + installSystemRootCAs := func(t *testing.T) { + t.Helper() + + const systemCAs = `-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw +TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh +cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4 +WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu +ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY +MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc +h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+ +0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U +A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW +T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH +B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC +B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv +KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn +OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn +jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw +qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI +rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV +HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq +hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL +ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ +3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK +NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5 +ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur +TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC +jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc +oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq +4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA +mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d +emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= +-----END CERTIFICATE----- +` + if err := os.MkdirAll("/etc/ssl/certs", 06660); err != nil { + t.Fatal(err) + } + + if err := os.WriteFile("/etc/ssl/certs/ca-certificates.crt", []byte(systemCAs), 0666); err != nil { + t.Fatal(err) + } + } + + test(t, "after_first_load_no_system_CAs", func(t *testing.T) { + SystemCertPool() // load system certs, before setting fallbacks + fallback := newFallbackCertPool(t) + SetFallbackRoots(fallback) + got, err := SystemCertPool() + if err != nil { + t.Fatal(err) + } + if !got.Equal(fallback) { + t.Fatal("SystemCertPool returned a non-fallback CertPool") + } + }) + + test(t, "after_first_load_system_CA_read_error", func(t *testing.T) { + // This will fail to load in SystemCertPool since this is a directory, + // rather than a file with certificates. + if err := os.MkdirAll("/etc/ssl/certs/ca-certificates.crt", 0666); err != nil { + t.Fatal(err) + } + + _, err := SystemCertPool() // load system certs, before setting fallbacks + if err == nil { + t.Fatal("unexpected success") + } + + fallback := newFallbackCertPool(t) + SetFallbackRoots(fallback) + got, err := SystemCertPool() + if err != nil { + t.Fatal(err) + } + if !got.Equal(fallback) { + t.Fatal("SystemCertPool returned a non-fallback CertPool") + } + }) + + test(t, "after_first_load_with_system_CAs", func(t *testing.T) { + installSystemRootCAs(t) + + SystemCertPool() // load system certs, before setting fallbacks + + fallback := newFallbackCertPool(t) + SetFallbackRoots(fallback) + got, err := SystemCertPool() + if err != nil { + t.Fatal(err) + } + if got.Equal(fallback) { + t.Fatal("SystemCertPool returned the fallback CertPool") + } + }) + + test(t, "before_first_load_no_system_CAs", func(t *testing.T) { + fallback := newFallbackCertPool(t) + SetFallbackRoots(fallback) + got, err := SystemCertPool() + if err != nil { + t.Fatal(err) + } + if !got.Equal(fallback) { + t.Fatal("SystemCertPool returned a non-fallback CertPool") + } + }) + + test(t, "before_first_load_system_CA_read_error", func(t *testing.T) { + // This will fail to load in SystemCertPool since this is a directory, + // rather than a file with certificates. + if err := os.MkdirAll("/etc/ssl/certs/ca-certificates.crt", 0666); err != nil { + t.Fatal(err) + } + + fallback := newFallbackCertPool(t) + SetFallbackRoots(fallback) + got, err := SystemCertPool() + if err != nil { + t.Fatal(err) + } + if !got.Equal(fallback) { + t.Fatal("SystemCertPool returned a non-fallback CertPool") + } + }) + + test(t, "before_first_load_with_system_CAs", func(t *testing.T) { + installSystemRootCAs(t) + + fallback := newFallbackCertPool(t) + SetFallbackRoots(fallback) + got, err := SystemCertPool() + if err != nil { + t.Fatal(err) + } + if got.Equal(fallback) { + t.Fatal("SystemCertPool returned the fallback CertPool") + } + }) + + test(t, "before_first_load_force_godebug", func(t *testing.T) { + if err := os.Setenv("GODEBUG", "x509usefallbackroots=1"); err != nil { + t.Fatal(err) + } + + installSystemRootCAs(t) + + fallback := newFallbackCertPool(t) + SetFallbackRoots(fallback) + got, err := SystemCertPool() + if err != nil { + t.Fatal(err) + } + if !got.Equal(fallback) { + t.Fatal("SystemCertPool returned a non-fallback CertPool") + } + }) + + test(t, "after_first_load_force_godebug", func(t *testing.T) { + if err := os.Setenv("GODEBUG", "x509usefallbackroots=1"); err != nil { + t.Fatal(err) + } + + installSystemRootCAs(t) + SystemCertPool() // load system certs, before setting fallbacks + + fallback := newFallbackCertPool(t) + SetFallbackRoots(fallback) + got, err := SystemCertPool() + if err != nil { + t.Fatal(err) + } + if !got.Equal(fallback) { + t.Fatal("SystemCertPool returned a non-fallback CertPool") + } + }) + + test(t, "after_first_load_force_godebug_no_system_certs", func(t *testing.T) { + if err := os.Setenv("GODEBUG", "x509usefallbackroots=1"); err != nil { + t.Fatal(err) + } + + SystemCertPool() // load system certs, before setting fallbacks + + fallback := newFallbackCertPool(t) + SetFallbackRoots(fallback) + got, err := SystemCertPool() + if err != nil { + t.Fatal(err) + } + if !got.Equal(fallback) { + t.Fatal("SystemCertPool returned a non-fallback CertPool") + } + }) +} diff --git a/src/crypto/x509/root_test.go b/src/crypto/x509/root_test.go index 94ee6a632d9f02..218d2b6f98c7a6 100644 --- a/src/crypto/x509/root_test.go +++ b/src/crypto/x509/root_test.go @@ -79,8 +79,10 @@ func TestFallback(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + useFallbackRoots = false fallbacksSet = false systemRoots = tc.systemRoots + if systemRoots != nil { systemRoots.systemPool = tc.systemPool } diff --git a/src/crypto/x509/root_unix_test.go b/src/crypto/x509/root_unix_test.go index 2ea69e252a2403..b03a03d1169c06 100644 --- a/src/crypto/x509/root_unix_test.go +++ b/src/crypto/x509/root_unix_test.go @@ -17,7 +17,6 @@ import ( ) const ( - testDir = "testdata" testDirCN = "test-dir" testFile = "test-file.crt" testFileCN = "test-file" @@ -25,6 +24,17 @@ const ( ) func TestEnvVars(t *testing.T) { + tmpDir := t.TempDir() + testCert, err := os.ReadFile("testdata/test-dir.crt") + if err != nil { + t.Fatalf("failed to read test cert: %s", err) + } + if err := os.WriteFile(filepath.Join(tmpDir, testFile), testCert, 0644); err != nil { + if err != nil { + t.Fatalf("failed to write test cert: %s", err) + } + } + testCases := []struct { name string fileEnv string @@ -39,7 +49,7 @@ func TestEnvVars(t *testing.T) { fileEnv: testMissing, dirEnv: testMissing, files: []string{testFile}, - dirs: []string{testDir}, + dirs: []string{tmpDir}, cns: nil, }, { @@ -55,7 +65,7 @@ func TestEnvVars(t *testing.T) { // Directory environment overrides default directory locations. name: "dir", fileEnv: "", - dirEnv: testDir, + dirEnv: tmpDir, files: nil, dirs: nil, cns: []string{testDirCN}, @@ -64,7 +74,7 @@ func TestEnvVars(t *testing.T) { // File & directory environment overrides both default locations. name: "file+dir", fileEnv: testFile, - dirEnv: testDir, + dirEnv: tmpDir, files: nil, dirs: nil, cns: []string{testFileCN, testDirCN}, @@ -75,7 +85,7 @@ func TestEnvVars(t *testing.T) { fileEnv: "", dirEnv: "", files: []string{testFile}, - dirs: []string{testDir}, + dirs: []string{tmpDir}, cns: []string{testFileCN, testDirCN}, }, } @@ -151,7 +161,6 @@ func TestLoadSystemCertsLoadColonSeparatedDirs(t *testing.T) { rootPEMs := []string{ gtsRoot, googleLeaf, - startComRoot, } var certDirs []string diff --git a/src/crypto/x509/testdata/nist-pkits/README.md b/src/crypto/x509/testdata/nist-pkits/README.md new file mode 100644 index 00000000000000..7b761260d6c046 --- /dev/null +++ b/src/crypto/x509/testdata/nist-pkits/README.md @@ -0,0 +1,6 @@ +Test vectors and certificates for the "Path Validation Testing Program" + portion of the NIST Public Key Infrastructure Testing suite: https://csrc.nist.gov/projects/pki-testing. + +Vectors are extracted from the provided PDF: https://csrc.nist.gov/CSRC/media/Projects/PKI-Testing/documents/PKITS.pdf. + +Vectors and test material are public domain (United States Government Work under 17 U.S.C. 105). \ No newline at end of file diff --git a/src/crypto/x509/testdata/nist-pkits/certs/AllCertificatesNoPoliciesTest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/AllCertificatesNoPoliciesTest2EE.crt new file mode 100644 index 00000000000000..ae6be6c4c8f63f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/AllCertificatesNoPoliciesTest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/AllCertificatesSamePoliciesTest10EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/AllCertificatesSamePoliciesTest10EE.crt new file mode 100644 index 00000000000000..e36fdb8fc37216 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/AllCertificatesSamePoliciesTest10EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/AllCertificatesSamePoliciesTest13EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/AllCertificatesSamePoliciesTest13EE.crt new file mode 100644 index 00000000000000..c296e5a4308848 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/AllCertificatesSamePoliciesTest13EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/AllCertificatesanyPolicyTest11EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/AllCertificatesanyPolicyTest11EE.crt new file mode 100644 index 00000000000000..7439f85152a785 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/AllCertificatesanyPolicyTest11EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/AnyPolicyTest14EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/AnyPolicyTest14EE.crt new file mode 100644 index 00000000000000..a6cf3528fa3c26 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/AnyPolicyTest14EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/BadCRLIssuerNameCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/BadCRLIssuerNameCACert.crt new file mode 100644 index 00000000000000..05e4b3ddbe858b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/BadCRLIssuerNameCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/BadCRLSignatureCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/BadCRLSignatureCACert.crt new file mode 100644 index 00000000000000..6dfa00d6b8ebf7 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/BadCRLSignatureCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/BadSignedCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/BadSignedCACert.crt new file mode 100644 index 00000000000000..0a598fcb8eb03f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/BadSignedCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/BadnotAfterDateCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/BadnotAfterDateCACert.crt new file mode 100644 index 00000000000000..7a7dcec665123a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/BadnotAfterDateCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/BadnotBeforeDateCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/BadnotBeforeDateCACert.crt new file mode 100644 index 00000000000000..33cfbd7ce8c7bc Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/BadnotBeforeDateCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedCRLSigningKeyCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedCRLSigningKeyCACert.crt new file mode 100644 index 00000000000000..4e1245299d42e8 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedCRLSigningKeyCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedCRLSigningKeyCRLCert.crt b/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedCRLSigningKeyCRLCert.crt new file mode 100644 index 00000000000000..7f86064c26b95d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedCRLSigningKeyCRLCert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedNewKeyCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedNewKeyCACert.crt new file mode 100644 index 00000000000000..1f83cb863f1d83 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedNewKeyCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedNewKeyOldWithNewCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedNewKeyOldWithNewCACert.crt new file mode 100644 index 00000000000000..8773e48464cb13 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedNewKeyOldWithNewCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedOldKeyCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedOldKeyCACert.crt new file mode 100644 index 00000000000000..b00748cc262b20 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedOldKeyCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedOldKeyNewWithOldCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedOldKeyNewWithOldCACert.crt new file mode 100644 index 00000000000000..963f57a485d4a9 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/BasicSelfIssuedOldKeyNewWithOldCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/CPSPointerQualifierTest20EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/CPSPointerQualifierTest20EE.crt new file mode 100644 index 00000000000000..706d98d63b8d5d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/CPSPointerQualifierTest20EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/DSACACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/DSACACert.crt new file mode 100644 index 00000000000000..14787b05801bd1 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/DSACACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/DSAParametersInheritedCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/DSAParametersInheritedCACert.crt new file mode 100644 index 00000000000000..5e2fa5bc921a70 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/DSAParametersInheritedCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest12EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest12EE.crt new file mode 100644 index 00000000000000..7873bd8d36716b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest12EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest3EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest3EE.crt new file mode 100644 index 00000000000000..57f1df4334e967 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest3EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest4EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest4EE.crt new file mode 100644 index 00000000000000..4967f41d30245f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest4EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest5EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest5EE.crt new file mode 100644 index 00000000000000..b6d31236e2c5eb Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest5EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest7EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest7EE.crt new file mode 100644 index 00000000000000..4c9c82bbcdae62 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest7EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest8EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest8EE.crt new file mode 100644 index 00000000000000..6c01f377f46c86 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest8EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest9EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest9EE.crt new file mode 100644 index 00000000000000..b2e30bd69217cc Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/DifferentPoliciesTest9EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/GeneralizedTimeCRLnextUpdateCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/GeneralizedTimeCRLnextUpdateCACert.crt new file mode 100644 index 00000000000000..f4acda66ec5cd9 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/GeneralizedTimeCRLnextUpdateCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/GoodCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/GoodCACert.crt new file mode 100644 index 00000000000000..edbfa648f24723 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/GoodCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/GoodsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/GoodsubCACert.crt new file mode 100644 index 00000000000000..7a770c31ae7342 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/GoodsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/GoodsubCAPanyPolicyMapping1to2CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/GoodsubCAPanyPolicyMapping1to2CACert.crt new file mode 100644 index 00000000000000..9f4d95f39533c5 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/GoodsubCAPanyPolicyMapping1to2CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidBadCRLIssuerNameTest5EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidBadCRLIssuerNameTest5EE.crt new file mode 100644 index 00000000000000..e24d88d444d0db Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidBadCRLIssuerNameTest5EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidBadCRLSignatureTest4EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidBadCRLSignatureTest4EE.crt new file mode 100644 index 00000000000000..4b35bd248ed3f5 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidBadCRLSignatureTest4EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidBasicSelfIssuedCRLSigningKeyTest7EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidBasicSelfIssuedCRLSigningKeyTest7EE.crt new file mode 100644 index 00000000000000..348df8fe0db496 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidBasicSelfIssuedCRLSigningKeyTest7EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidBasicSelfIssuedCRLSigningKeyTest8EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidBasicSelfIssuedCRLSigningKeyTest8EE.crt new file mode 100644 index 00000000000000..3ca799546005f8 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidBasicSelfIssuedCRLSigningKeyTest8EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidBasicSelfIssuedNewWithOldTest5EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidBasicSelfIssuedNewWithOldTest5EE.crt new file mode 100644 index 00000000000000..6cc192b8ba2195 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidBasicSelfIssuedNewWithOldTest5EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidBasicSelfIssuedOldWithNewTest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidBasicSelfIssuedOldWithNewTest2EE.crt new file mode 100644 index 00000000000000..18033bc34b08de Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidBasicSelfIssuedOldWithNewTest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidCASignatureTest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidCASignatureTest2EE.crt new file mode 100644 index 00000000000000..1f4ad3e1a15d5a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidCASignatureTest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidCAnotAfterDateTest5EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidCAnotAfterDateTest5EE.crt new file mode 100644 index 00000000000000..a9938aa80e9552 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidCAnotAfterDateTest5EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidCAnotBeforeDateTest1EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidCAnotBeforeDateTest1EE.crt new file mode 100644 index 00000000000000..f15d6a9ed22aff Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidCAnotBeforeDateTest1EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNSnameConstraintsTest31EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNSnameConstraintsTest31EE.crt new file mode 100644 index 00000000000000..5f7ad1535aa50a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNSnameConstraintsTest31EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNSnameConstraintsTest33EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNSnameConstraintsTest33EE.crt new file mode 100644 index 00000000000000..fa59d6fbd0d9f4 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNSnameConstraintsTest33EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNSnameConstraintsTest38EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNSnameConstraintsTest38EE.crt new file mode 100644 index 00000000000000..334fed1f111783 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNSnameConstraintsTest38EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNandRFC822nameConstraintsTest28EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNandRFC822nameConstraintsTest28EE.crt new file mode 100644 index 00000000000000..f724473de877be Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNandRFC822nameConstraintsTest28EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNandRFC822nameConstraintsTest29EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNandRFC822nameConstraintsTest29EE.crt new file mode 100644 index 00000000000000..468cb7bede2ab3 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNandRFC822nameConstraintsTest29EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest10EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest10EE.crt new file mode 100644 index 00000000000000..806ebf3ce7fcbd Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest10EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest12EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest12EE.crt new file mode 100644 index 00000000000000..5f3a49f93e402b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest12EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest13EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest13EE.crt new file mode 100644 index 00000000000000..d64ddf53c7a101 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest13EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest15EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest15EE.crt new file mode 100644 index 00000000000000..fd864ced34c176 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest15EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest16EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest16EE.crt new file mode 100644 index 00000000000000..455658dbc97ae5 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest16EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest17EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest17EE.crt new file mode 100644 index 00000000000000..63f262b99fb256 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest17EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest20EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest20EE.crt new file mode 100644 index 00000000000000..a7ef3220436c1d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest20EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest2EE.crt new file mode 100644 index 00000000000000..3fd895c9242de0 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest3EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest3EE.crt new file mode 100644 index 00000000000000..decbf34aaceec2 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest3EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest7EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest7EE.crt new file mode 100644 index 00000000000000..6ac76654e5e7e3 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest7EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest8EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest8EE.crt new file mode 100644 index 00000000000000..48adc0a6d55c90 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest8EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest9EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest9EE.crt new file mode 100644 index 00000000000000..ed753d42e66649 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDNnameConstraintsTest9EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidDSASignatureTest6EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDSASignatureTest6EE.crt new file mode 100644 index 00000000000000..a1725b19dacab8 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidDSASignatureTest6EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidEESignatureTest3EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidEESignatureTest3EE.crt new file mode 100644 index 00000000000000..9238109b646e7d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidEESignatureTest3EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidEEnotAfterDateTest6EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidEEnotAfterDateTest6EE.crt new file mode 100644 index 00000000000000..af6fdf8c5a91ce Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidEEnotAfterDateTest6EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidEEnotBeforeDateTest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidEEnotBeforeDateTest2EE.crt new file mode 100644 index 00000000000000..3ddef09cabffac Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidEEnotBeforeDateTest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidIDPwithindirectCRLTest23EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidIDPwithindirectCRLTest23EE.crt new file mode 100644 index 00000000000000..5cf92f7ce45695 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidIDPwithindirectCRLTest23EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidIDPwithindirectCRLTest26EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidIDPwithindirectCRLTest26EE.crt new file mode 100644 index 00000000000000..c4b45f87831eb7 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidIDPwithindirectCRLTest26EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidLongSerialNumberTest18EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidLongSerialNumberTest18EE.crt new file mode 100644 index 00000000000000..56b1ab45831fb5 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidLongSerialNumberTest18EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidMappingFromanyPolicyTest7EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidMappingFromanyPolicyTest7EE.crt new file mode 100644 index 00000000000000..eec4c3c3a6de84 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidMappingFromanyPolicyTest7EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidMappingToanyPolicyTest8EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidMappingToanyPolicyTest8EE.crt new file mode 100644 index 00000000000000..ee6914c15aba20 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidMappingToanyPolicyTest8EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidMissingCRLTest1EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidMissingCRLTest1EE.crt new file mode 100644 index 00000000000000..30b027590338a4 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidMissingCRLTest1EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidMissingbasicConstraintsTest1EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidMissingbasicConstraintsTest1EE.crt new file mode 100644 index 00000000000000..80ba7a03dd1ad0 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidMissingbasicConstraintsTest1EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidNameChainingOrderTest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidNameChainingOrderTest2EE.crt new file mode 100644 index 00000000000000..6b7d7de29c65e3 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidNameChainingOrderTest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidNameChainingTest1EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidNameChainingTest1EE.crt new file mode 100644 index 00000000000000..ee18fa08fbb48d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidNameChainingTest1EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidNegativeSerialNumberTest15EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidNegativeSerialNumberTest15EE.crt new file mode 100644 index 00000000000000..2c479ca231f10f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidNegativeSerialNumberTest15EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidOldCRLnextUpdateTest11EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidOldCRLnextUpdateTest11EE.crt new file mode 100644 index 00000000000000..1ec410d7554529 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidOldCRLnextUpdateTest11EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidPolicyMappingTest10EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidPolicyMappingTest10EE.crt new file mode 100644 index 00000000000000..053a608d7e63f7 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidPolicyMappingTest10EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidPolicyMappingTest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidPolicyMappingTest2EE.crt new file mode 100644 index 00000000000000..1ed661582c49fe Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidPolicyMappingTest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidPolicyMappingTest4EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidPolicyMappingTest4EE.crt new file mode 100644 index 00000000000000..a194a040a70a07 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidPolicyMappingTest4EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidRFC822nameConstraintsTest22EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidRFC822nameConstraintsTest22EE.crt new file mode 100644 index 00000000000000..c9ad311ac0a01b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidRFC822nameConstraintsTest22EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidRFC822nameConstraintsTest24EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidRFC822nameConstraintsTest24EE.crt new file mode 100644 index 00000000000000..28ef8f7491e37d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidRFC822nameConstraintsTest24EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidRFC822nameConstraintsTest26EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidRFC822nameConstraintsTest26EE.crt new file mode 100644 index 00000000000000..0e7f71937a5054 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidRFC822nameConstraintsTest26EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidRevokedCATest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidRevokedCATest2EE.crt new file mode 100644 index 00000000000000..80545971141725 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidRevokedCATest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidRevokedEETest3EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidRevokedEETest3EE.crt new file mode 100644 index 00000000000000..455cb0240c4aea Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidRevokedEETest3EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitAnyPolicyTest10EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitAnyPolicyTest10EE.crt new file mode 100644 index 00000000000000..2e85ce5c21c860 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitAnyPolicyTest10EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitAnyPolicyTest8EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitAnyPolicyTest8EE.crt new file mode 100644 index 00000000000000..ee48b7fc85ec2d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitAnyPolicyTest8EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitPolicyMappingTest10EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitPolicyMappingTest10EE.crt new file mode 100644 index 00000000000000..e729fe77cda740 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitPolicyMappingTest10EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitPolicyMappingTest11EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitPolicyMappingTest11EE.crt new file mode 100644 index 00000000000000..103e0940fe5c26 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitPolicyMappingTest11EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitPolicyMappingTest8EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitPolicyMappingTest8EE.crt new file mode 100644 index 00000000000000..3eaa74deb815d6 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitPolicyMappingTest8EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitPolicyMappingTest9EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitPolicyMappingTest9EE.crt new file mode 100644 index 00000000000000..1a1da9fe7a0007 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedinhibitPolicyMappingTest9EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedpathLenConstraintTest16EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedpathLenConstraintTest16EE.crt new file mode 100644 index 00000000000000..2ff84b8b7d3467 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedpathLenConstraintTest16EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedrequireExplicitPolicyTest7EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedrequireExplicitPolicyTest7EE.crt new file mode 100644 index 00000000000000..d4050e6f4f0474 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedrequireExplicitPolicyTest7EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedrequireExplicitPolicyTest8EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedrequireExplicitPolicyTest8EE.crt new file mode 100644 index 00000000000000..77b6a3c1472506 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSelfIssuedrequireExplicitPolicyTest8EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidSeparateCertificateandCRLKeysTest20EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSeparateCertificateandCRLKeysTest20EE.crt new file mode 100644 index 00000000000000..2cbab480b15658 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSeparateCertificateandCRLKeysTest20EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidSeparateCertificateandCRLKeysTest21EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSeparateCertificateandCRLKeysTest21EE.crt new file mode 100644 index 00000000000000..e703d679051ee4 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidSeparateCertificateandCRLKeysTest21EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidURInameConstraintsTest35EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidURInameConstraintsTest35EE.crt new file mode 100644 index 00000000000000..65096685fe2b37 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidURInameConstraintsTest35EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidURInameConstraintsTest37EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidURInameConstraintsTest37EE.crt new file mode 100644 index 00000000000000..e64db473af5f95 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidURInameConstraintsTest37EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidUnknownCRLEntryExtensionTest8EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidUnknownCRLEntryExtensionTest8EE.crt new file mode 100644 index 00000000000000..8630e99cb240db Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidUnknownCRLEntryExtensionTest8EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidUnknownCRLExtensionTest10EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidUnknownCRLExtensionTest10EE.crt new file mode 100644 index 00000000000000..42fda8fc122d29 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidUnknownCRLExtensionTest10EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidUnknownCRLExtensionTest9EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidUnknownCRLExtensionTest9EE.crt new file mode 100644 index 00000000000000..c3f93b5bd7c654 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidUnknownCRLExtensionTest9EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidUnknownCriticalCertificateExtensionTest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidUnknownCriticalCertificateExtensionTest2EE.crt new file mode 100644 index 00000000000000..9200cccb39a7cb Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidUnknownCriticalCertificateExtensionTest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidWrongCRLTest6EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidWrongCRLTest6EE.crt new file mode 100644 index 00000000000000..148f9fb23aba36 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidWrongCRLTest6EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidcAFalseTest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidcAFalseTest2EE.crt new file mode 100644 index 00000000000000..3d5b82946bc875 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidcAFalseTest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidcAFalseTest3EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidcAFalseTest3EE.crt new file mode 100644 index 00000000000000..f791140cedab3a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidcAFalseTest3EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidcRLIssuerTest27EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidcRLIssuerTest27EE.crt new file mode 100644 index 00000000000000..2433e3b95e129d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidcRLIssuerTest27EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidcRLIssuerTest31EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidcRLIssuerTest31EE.crt new file mode 100644 index 00000000000000..210bb41fef9900 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidcRLIssuerTest31EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidcRLIssuerTest32EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidcRLIssuerTest32EE.crt new file mode 100644 index 00000000000000..5509dda847de04 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidcRLIssuerTest32EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidcRLIssuerTest34EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidcRLIssuerTest34EE.crt new file mode 100644 index 00000000000000..8b9041f5ba49f5 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidcRLIssuerTest34EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidcRLIssuerTest35EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidcRLIssuerTest35EE.crt new file mode 100644 index 00000000000000..32e72a225ec311 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidcRLIssuerTest35EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLIndicatorNoBaseTest1EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLIndicatorNoBaseTest1EE.crt new file mode 100644 index 00000000000000..10da321247087f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLIndicatorNoBaseTest1EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLTest10EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLTest10EE.crt new file mode 100644 index 00000000000000..d60812c6a41620 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLTest10EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLTest3EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLTest3EE.crt new file mode 100644 index 00000000000000..6b3c374331d8de Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLTest3EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLTest4EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLTest4EE.crt new file mode 100644 index 00000000000000..b959414934419d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLTest4EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLTest6EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLTest6EE.crt new file mode 100644 index 00000000000000..ea141b173af1ac Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLTest6EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLTest9EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLTest9EE.crt new file mode 100644 index 00000000000000..de4da9d69b05c1 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddeltaCRLTest9EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvaliddistributionPointTest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddistributionPointTest2EE.crt new file mode 100644 index 00000000000000..a60b030e9e0087 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddistributionPointTest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvaliddistributionPointTest3EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddistributionPointTest3EE.crt new file mode 100644 index 00000000000000..bbb8271d6bb33a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddistributionPointTest3EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvaliddistributionPointTest6EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddistributionPointTest6EE.crt new file mode 100644 index 00000000000000..a47f7b208569dc Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddistributionPointTest6EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvaliddistributionPointTest8EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddistributionPointTest8EE.crt new file mode 100644 index 00000000000000..af3a366dd74f38 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddistributionPointTest8EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvaliddistributionPointTest9EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddistributionPointTest9EE.crt new file mode 100644 index 00000000000000..3456831e0b9420 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvaliddistributionPointTest9EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitAnyPolicyTest1EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitAnyPolicyTest1EE.crt new file mode 100644 index 00000000000000..828203b11c7db7 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitAnyPolicyTest1EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitAnyPolicyTest4EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitAnyPolicyTest4EE.crt new file mode 100644 index 00000000000000..2ffd9dd8ce5218 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitAnyPolicyTest4EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitAnyPolicyTest5EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitAnyPolicyTest5EE.crt new file mode 100644 index 00000000000000..2fc212d33ed986 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitAnyPolicyTest5EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitAnyPolicyTest6EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitAnyPolicyTest6EE.crt new file mode 100644 index 00000000000000..9aafebfc25f67c Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitAnyPolicyTest6EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitPolicyMappingTest1EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitPolicyMappingTest1EE.crt new file mode 100644 index 00000000000000..65ca6340ea6e3a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitPolicyMappingTest1EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitPolicyMappingTest3EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitPolicyMappingTest3EE.crt new file mode 100644 index 00000000000000..c8b06f07e40055 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitPolicyMappingTest3EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitPolicyMappingTest5EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitPolicyMappingTest5EE.crt new file mode 100644 index 00000000000000..f3526efb69fb23 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitPolicyMappingTest5EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitPolicyMappingTest6EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitPolicyMappingTest6EE.crt new file mode 100644 index 00000000000000..733c152685dd34 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidinhibitPolicyMappingTest6EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidkeyUsageCriticalcRLSignFalseTest4EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidkeyUsageCriticalcRLSignFalseTest4EE.crt new file mode 100644 index 00000000000000..cfddd3a4359dc1 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidkeyUsageCriticalcRLSignFalseTest4EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidkeyUsageCriticalkeyCertSignFalseTest1EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidkeyUsageCriticalkeyCertSignFalseTest1EE.crt new file mode 100644 index 00000000000000..16c103f7446fea Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidkeyUsageCriticalkeyCertSignFalseTest1EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidkeyUsageNotCriticalcRLSignFalseTest5EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidkeyUsageNotCriticalcRLSignFalseTest5EE.crt new file mode 100644 index 00000000000000..5583f19690c7c8 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidkeyUsageNotCriticalcRLSignFalseTest5EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidkeyUsageNotCriticalkeyCertSignFalseTest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidkeyUsageNotCriticalkeyCertSignFalseTest2EE.crt new file mode 100644 index 00000000000000..f3062e9e4803a5 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidkeyUsageNotCriticalkeyCertSignFalseTest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlyContainsAttributeCertsTest14EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlyContainsAttributeCertsTest14EE.crt new file mode 100644 index 00000000000000..279306ed188999 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlyContainsAttributeCertsTest14EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlyContainsCACertsTest12EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlyContainsCACertsTest12EE.crt new file mode 100644 index 00000000000000..f206348963c0c5 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlyContainsCACertsTest12EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlyContainsUserCertsTest11EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlyContainsUserCertsTest11EE.crt new file mode 100644 index 00000000000000..ecf51285fa49fb Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlyContainsUserCertsTest11EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlySomeReasonsTest15EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlySomeReasonsTest15EE.crt new file mode 100644 index 00000000000000..f536fc6d2bdab1 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlySomeReasonsTest15EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlySomeReasonsTest16EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlySomeReasonsTest16EE.crt new file mode 100644 index 00000000000000..af5aa4b0d4123d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlySomeReasonsTest16EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlySomeReasonsTest17EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlySomeReasonsTest17EE.crt new file mode 100644 index 00000000000000..59722f96227983 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlySomeReasonsTest17EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlySomeReasonsTest20EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlySomeReasonsTest20EE.crt new file mode 100644 index 00000000000000..4a0f1916509cf7 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlySomeReasonsTest20EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlySomeReasonsTest21EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlySomeReasonsTest21EE.crt new file mode 100644 index 00000000000000..59a02de9d7807f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidonlySomeReasonsTest21EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest10EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest10EE.crt new file mode 100644 index 00000000000000..447115e63684e6 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest10EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest11EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest11EE.crt new file mode 100644 index 00000000000000..c28c455abbec72 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest11EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest12EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest12EE.crt new file mode 100644 index 00000000000000..dc6d0dda96abdd Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest12EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest5EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest5EE.crt new file mode 100644 index 00000000000000..b8830a2405533a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest5EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest6EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest6EE.crt new file mode 100644 index 00000000000000..b96d3c626fdb7b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest6EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest9EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest9EE.crt new file mode 100644 index 00000000000000..c339f6fae75b31 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidpathLenConstraintTest9EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/Invalidpre2000CRLnextUpdateTest12EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/Invalidpre2000CRLnextUpdateTest12EE.crt new file mode 100644 index 00000000000000..3e1ba073e1338a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/Invalidpre2000CRLnextUpdateTest12EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/Invalidpre2000UTCEEnotAfterDateTest7EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/Invalidpre2000UTCEEnotAfterDateTest7EE.crt new file mode 100644 index 00000000000000..4a7e31caf02fa8 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/Invalidpre2000UTCEEnotAfterDateTest7EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidrequireExplicitPolicyTest3EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidrequireExplicitPolicyTest3EE.crt new file mode 100644 index 00000000000000..e9b7cf2510f60e Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidrequireExplicitPolicyTest3EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/InvalidrequireExplicitPolicyTest5EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/InvalidrequireExplicitPolicyTest5EE.crt new file mode 100644 index 00000000000000..971d0a5de61b21 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/InvalidrequireExplicitPolicyTest5EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/LongSerialNumberCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/LongSerialNumberCACert.crt new file mode 100644 index 00000000000000..12830d90697364 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/LongSerialNumberCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/Mapping1to2CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/Mapping1to2CACert.crt new file mode 100644 index 00000000000000..4b70c9a4fc00fd Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/Mapping1to2CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/MappingFromanyPolicyCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/MappingFromanyPolicyCACert.crt new file mode 100644 index 00000000000000..0a8f1e9811441b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/MappingFromanyPolicyCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/MappingToanyPolicyCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/MappingToanyPolicyCACert.crt new file mode 100644 index 00000000000000..d93d8c79c98875 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/MappingToanyPolicyCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/MissingbasicConstraintsCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/MissingbasicConstraintsCACert.crt new file mode 100644 index 00000000000000..e6f41a446b322a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/MissingbasicConstraintsCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/NameOrderingCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/NameOrderingCACert.crt new file mode 100644 index 00000000000000..f1c4a55fbf53a9 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/NameOrderingCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/NegativeSerialNumberCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/NegativeSerialNumberCACert.crt new file mode 100644 index 00000000000000..1a4d9ba374cebe Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/NegativeSerialNumberCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/NoCRLCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/NoCRLCACert.crt new file mode 100644 index 00000000000000..71c607dac44de7 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/NoCRLCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/NoPoliciesCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/NoPoliciesCACert.crt new file mode 100644 index 00000000000000..3a94cb157d27ab Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/NoPoliciesCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/NoissuingDistributionPointCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/NoissuingDistributionPointCACert.crt new file mode 100644 index 00000000000000..c4f182ad7f9771 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/NoissuingDistributionPointCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/OldCRLnextUpdateCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/OldCRLnextUpdateCACert.crt new file mode 100644 index 00000000000000..2666670afb5e8e Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/OldCRLnextUpdateCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/OverlappingPoliciesTest6EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/OverlappingPoliciesTest6EE.crt new file mode 100644 index 00000000000000..82b5b5e0ee1c7e Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/OverlappingPoliciesTest6EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/P12Mapping1to3CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/P12Mapping1to3CACert.crt new file mode 100644 index 00000000000000..9139bd730d3373 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/P12Mapping1to3CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/P12Mapping1to3subCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/P12Mapping1to3subCACert.crt new file mode 100644 index 00000000000000..3b9c2a751c4a84 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/P12Mapping1to3subCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/P12Mapping1to3subsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/P12Mapping1to3subsubCACert.crt new file mode 100644 index 00000000000000..91fc36a7273717 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/P12Mapping1to3subsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/P1Mapping1to234CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/P1Mapping1to234CACert.crt new file mode 100644 index 00000000000000..3500737ab8e8ed Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/P1Mapping1to234CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/P1Mapping1to234subCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/P1Mapping1to234subCACert.crt new file mode 100644 index 00000000000000..eb900ebc1cbeb6 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/P1Mapping1to234subCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/P1anyPolicyMapping1to2CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/P1anyPolicyMapping1to2CACert.crt new file mode 100644 index 00000000000000..3818b6a7f5193e Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/P1anyPolicyMapping1to2CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/PanyPolicyMapping1to2CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/PanyPolicyMapping1to2CACert.crt new file mode 100644 index 00000000000000..db220487cc05a1 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/PanyPolicyMapping1to2CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP1234CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP1234CACert.crt new file mode 100644 index 00000000000000..36cf4ce24ea9f0 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP1234CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP1234subCAP123Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP1234subCAP123Cert.crt new file mode 100644 index 00000000000000..1ab7ab104f6564 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP1234subCAP123Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP1234subsubCAP123P12Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP1234subsubCAP123P12Cert.crt new file mode 100644 index 00000000000000..df834464bbad67 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP1234subsubCAP123P12Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP123CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP123CACert.crt new file mode 100644 index 00000000000000..26262a3d72f10d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP123CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP123subCAP12Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP123subCAP12Cert.crt new file mode 100644 index 00000000000000..cef6abeb29c680 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP123subCAP12Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP123subsubCAP12P1Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP123subsubCAP12P1Cert.crt new file mode 100644 index 00000000000000..49e66b5be012d0 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP123subsubCAP12P1Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP123subsubCAP12P2Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP123subsubCAP12P2Cert.crt new file mode 100644 index 00000000000000..d7b5a42353b557 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP123subsubCAP12P2Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP123subsubsubCAP12P2P1Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP123subsubsubCAP12P2P1Cert.crt new file mode 100644 index 00000000000000..3a79422477f813 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP123subsubsubCAP12P2P1Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP12CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP12CACert.crt new file mode 100644 index 00000000000000..dc1b60de0e8592 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP12CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP12subCAP1Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP12subCAP1Cert.crt new file mode 100644 index 00000000000000..081f951b80d946 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP12subCAP1Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP12subsubCAP1P2Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP12subsubCAP1P2Cert.crt new file mode 100644 index 00000000000000..e8d0bb8ba8df7a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP12subsubCAP1P2Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP2subCA2Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP2subCA2Cert.crt new file mode 100644 index 00000000000000..c734009d050a16 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP2subCA2Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP2subCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP2subCACert.crt new file mode 100644 index 00000000000000..0f3fbbb01a1922 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP2subCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP3CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP3CACert.crt new file mode 100644 index 00000000000000..9740b309d40f40 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/PoliciesP3CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/RFC3280MandatoryAttributeTypesCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/RFC3280MandatoryAttributeTypesCACert.crt new file mode 100644 index 00000000000000..9c648a30be041d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/RFC3280MandatoryAttributeTypesCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/RFC3280OptionalAttributeTypesCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/RFC3280OptionalAttributeTypesCACert.crt new file mode 100644 index 00000000000000..306303a8465ec7 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/RFC3280OptionalAttributeTypesCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/RevokedsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/RevokedsubCACert.crt new file mode 100644 index 00000000000000..25705b2f670f8e Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/RevokedsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/RolloverfromPrintableStringtoUTF8StringCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/RolloverfromPrintableStringtoUTF8StringCACert.crt new file mode 100644 index 00000000000000..32ddfe3e315253 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/RolloverfromPrintableStringtoUTF8StringCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/SeparateCertificateandCRLKeysCA2CRLSigningCert.crt b/src/crypto/x509/testdata/nist-pkits/certs/SeparateCertificateandCRLKeysCA2CRLSigningCert.crt new file mode 100644 index 00000000000000..17b3cbba30f022 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/SeparateCertificateandCRLKeysCA2CRLSigningCert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/SeparateCertificateandCRLKeysCA2CertificateSigningCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/SeparateCertificateandCRLKeysCA2CertificateSigningCACert.crt new file mode 100644 index 00000000000000..d747ea1fe5127c Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/SeparateCertificateandCRLKeysCA2CertificateSigningCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/SeparateCertificateandCRLKeysCRLSigningCert.crt b/src/crypto/x509/testdata/nist-pkits/certs/SeparateCertificateandCRLKeysCRLSigningCert.crt new file mode 100644 index 00000000000000..3c1730f41ab288 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/SeparateCertificateandCRLKeysCRLSigningCert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/SeparateCertificateandCRLKeysCertificateSigningCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/SeparateCertificateandCRLKeysCertificateSigningCACert.crt new file mode 100644 index 00000000000000..e75eb4cd707fb2 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/SeparateCertificateandCRLKeysCertificateSigningCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/TrustAnchorRootCertificate.crt b/src/crypto/x509/testdata/nist-pkits/certs/TrustAnchorRootCertificate.crt new file mode 100644 index 00000000000000..04efaa0659ba5a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/TrustAnchorRootCertificate.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/TwoCRLsCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/TwoCRLsCACert.crt new file mode 100644 index 00000000000000..28eb60a0713091 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/TwoCRLsCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/UIDCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/UIDCACert.crt new file mode 100644 index 00000000000000..ec04d744557706 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/UIDCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/UTF8StringCaseInsensitiveMatchCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/UTF8StringCaseInsensitiveMatchCACert.crt new file mode 100644 index 00000000000000..2d653ef65b1614 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/UTF8StringCaseInsensitiveMatchCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/UTF8StringEncodedNamesCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/UTF8StringEncodedNamesCACert.crt new file mode 100644 index 00000000000000..ae2ce8a7b49e84 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/UTF8StringEncodedNamesCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/UnknownCRLEntryExtensionCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/UnknownCRLEntryExtensionCACert.crt new file mode 100644 index 00000000000000..69128811ba3751 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/UnknownCRLEntryExtensionCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/UnknownCRLExtensionCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/UnknownCRLExtensionCACert.crt new file mode 100644 index 00000000000000..2e2c3ef3d694eb Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/UnknownCRLExtensionCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/UserNoticeQualifierTest15EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/UserNoticeQualifierTest15EE.crt new file mode 100644 index 00000000000000..afb3455e36b7c3 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/UserNoticeQualifierTest15EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/UserNoticeQualifierTest16EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/UserNoticeQualifierTest16EE.crt new file mode 100644 index 00000000000000..7d3bcc5d0bc7ae Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/UserNoticeQualifierTest16EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/UserNoticeQualifierTest17EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/UserNoticeQualifierTest17EE.crt new file mode 100644 index 00000000000000..5fefe19944457f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/UserNoticeQualifierTest17EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/UserNoticeQualifierTest18EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/UserNoticeQualifierTest18EE.crt new file mode 100644 index 00000000000000..1168b580e8c60c Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/UserNoticeQualifierTest18EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/UserNoticeQualifierTest19EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/UserNoticeQualifierTest19EE.crt new file mode 100644 index 00000000000000..3cb86cd1ce612a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/UserNoticeQualifierTest19EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidBasicSelfIssuedCRLSigningKeyTest6EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidBasicSelfIssuedCRLSigningKeyTest6EE.crt new file mode 100644 index 00000000000000..c91b9f3665cddf Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidBasicSelfIssuedCRLSigningKeyTest6EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidBasicSelfIssuedNewWithOldTest3EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidBasicSelfIssuedNewWithOldTest3EE.crt new file mode 100644 index 00000000000000..34197f036016d1 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidBasicSelfIssuedNewWithOldTest3EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidBasicSelfIssuedNewWithOldTest4EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidBasicSelfIssuedNewWithOldTest4EE.crt new file mode 100644 index 00000000000000..9a7919b00acb49 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidBasicSelfIssuedNewWithOldTest4EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidBasicSelfIssuedOldWithNewTest1EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidBasicSelfIssuedOldWithNewTest1EE.crt new file mode 100644 index 00000000000000..038e4d7a80a743 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidBasicSelfIssuedOldWithNewTest1EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidCertificatePathTest1EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidCertificatePathTest1EE.crt new file mode 100644 index 00000000000000..69ba3019d4b6bf Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidCertificatePathTest1EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidDNSnameConstraintsTest30EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNSnameConstraintsTest30EE.crt new file mode 100644 index 00000000000000..e5235c7ff251f2 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNSnameConstraintsTest30EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidDNSnameConstraintsTest32EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNSnameConstraintsTest32EE.crt new file mode 100644 index 00000000000000..8bc3e87b9fc4a1 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNSnameConstraintsTest32EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidDNandRFC822nameConstraintsTest27EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNandRFC822nameConstraintsTest27EE.crt new file mode 100644 index 00000000000000..2332d4c189a118 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNandRFC822nameConstraintsTest27EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest11EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest11EE.crt new file mode 100644 index 00000000000000..f8fe1223249f33 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest11EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest14EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest14EE.crt new file mode 100644 index 00000000000000..4364e1bcbf58b2 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest14EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest18EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest18EE.crt new file mode 100644 index 00000000000000..3b5ac8be530528 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest18EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest19EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest19EE.crt new file mode 100644 index 00000000000000..20fa140e19dd01 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest19EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest1EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest1EE.crt new file mode 100644 index 00000000000000..c59e921bac7e06 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest1EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest4EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest4EE.crt new file mode 100644 index 00000000000000..c6cfcbb77862b1 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest4EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest5EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest5EE.crt new file mode 100644 index 00000000000000..f2c4dfc553fdd1 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest5EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest6EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest6EE.crt new file mode 100644 index 00000000000000..675711970cea2d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidDNnameConstraintsTest6EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidDSAParameterInheritanceTest5EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidDSAParameterInheritanceTest5EE.crt new file mode 100644 index 00000000000000..d8b6ce36d0de3b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidDSAParameterInheritanceTest5EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidDSASignaturesTest4EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidDSASignaturesTest4EE.crt new file mode 100644 index 00000000000000..2fc40a6c2f3978 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidDSASignaturesTest4EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidGeneralizedTimeCRLnextUpdateTest13EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidGeneralizedTimeCRLnextUpdateTest13EE.crt new file mode 100644 index 00000000000000..7f77ee81965fe4 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidGeneralizedTimeCRLnextUpdateTest13EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidGeneralizedTimenotAfterDateTest8EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidGeneralizedTimenotAfterDateTest8EE.crt new file mode 100644 index 00000000000000..f97ed0a3e958fb Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidGeneralizedTimenotAfterDateTest8EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidGeneralizedTimenotBeforeDateTest4EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidGeneralizedTimenotBeforeDateTest4EE.crt new file mode 100644 index 00000000000000..2ef73e1f69732d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidGeneralizedTimenotBeforeDateTest4EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidIDPwithindirectCRLTest22EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidIDPwithindirectCRLTest22EE.crt new file mode 100644 index 00000000000000..66296ac7e75436 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidIDPwithindirectCRLTest22EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidIDPwithindirectCRLTest24EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidIDPwithindirectCRLTest24EE.crt new file mode 100644 index 00000000000000..0a1b85dc68062b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidIDPwithindirectCRLTest24EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidIDPwithindirectCRLTest25EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidIDPwithindirectCRLTest25EE.crt new file mode 100644 index 00000000000000..6f69c0c8bca864 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidIDPwithindirectCRLTest25EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidLongSerialNumberTest16EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidLongSerialNumberTest16EE.crt new file mode 100644 index 00000000000000..44e890546d5472 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidLongSerialNumberTest16EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidLongSerialNumberTest17EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidLongSerialNumberTest17EE.crt new file mode 100644 index 00000000000000..96186587225d25 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidLongSerialNumberTest17EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidNameChainingCapitalizationTest5EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidNameChainingCapitalizationTest5EE.crt new file mode 100644 index 00000000000000..c0a6b3d03ecf19 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidNameChainingCapitalizationTest5EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidNameChainingWhitespaceTest3EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidNameChainingWhitespaceTest3EE.crt new file mode 100644 index 00000000000000..fc0f65d0796a9f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidNameChainingWhitespaceTest3EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidNameChainingWhitespaceTest4EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidNameChainingWhitespaceTest4EE.crt new file mode 100644 index 00000000000000..a8ffc872cae3fc Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidNameChainingWhitespaceTest4EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidNameUIDsTest6EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidNameUIDsTest6EE.crt new file mode 100644 index 00000000000000..7d0b7061137453 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidNameUIDsTest6EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidNegativeSerialNumberTest14EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidNegativeSerialNumberTest14EE.crt new file mode 100644 index 00000000000000..ab39228409f9d5 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidNegativeSerialNumberTest14EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidNoissuingDistributionPointTest10EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidNoissuingDistributionPointTest10EE.crt new file mode 100644 index 00000000000000..89eac753f30a25 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidNoissuingDistributionPointTest10EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest11EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest11EE.crt new file mode 100644 index 00000000000000..865c97542eecba Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest11EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest12EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest12EE.crt new file mode 100644 index 00000000000000..eb4306ab5a6e26 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest12EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest13EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest13EE.crt new file mode 100644 index 00000000000000..2d1b18c33f4194 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest13EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest14EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest14EE.crt new file mode 100644 index 00000000000000..2487d626f7f9d9 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest14EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest1EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest1EE.crt new file mode 100644 index 00000000000000..f2bd7d381de216 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest1EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest3EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest3EE.crt new file mode 100644 index 00000000000000..e941bbbad003d6 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest3EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest5EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest5EE.crt new file mode 100644 index 00000000000000..d084fc7215863e Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest5EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest6EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest6EE.crt new file mode 100644 index 00000000000000..97dd2e72c11b23 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest6EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest9EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest9EE.crt new file mode 100644 index 00000000000000..ef1ac897e08a3b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidPolicyMappingTest9EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidRFC3280MandatoryAttributeTypesTest7EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidRFC3280MandatoryAttributeTypesTest7EE.crt new file mode 100644 index 00000000000000..15825d7eb3b99b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidRFC3280MandatoryAttributeTypesTest7EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidRFC3280OptionalAttributeTypesTest8EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidRFC3280OptionalAttributeTypesTest8EE.crt new file mode 100644 index 00000000000000..60a20316813bbc Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidRFC3280OptionalAttributeTypesTest8EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidRFC822nameConstraintsTest21EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidRFC822nameConstraintsTest21EE.crt new file mode 100644 index 00000000000000..576a1b81716124 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidRFC822nameConstraintsTest21EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidRFC822nameConstraintsTest23EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidRFC822nameConstraintsTest23EE.crt new file mode 100644 index 00000000000000..c0ff7596a0929f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidRFC822nameConstraintsTest23EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidRFC822nameConstraintsTest25EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidRFC822nameConstraintsTest25EE.crt new file mode 100644 index 00000000000000..75f67b73c86c16 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidRFC822nameConstraintsTest25EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidRolloverfromPrintableStringtoUTF8StringTest10EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidRolloverfromPrintableStringtoUTF8StringTest10EE.crt new file mode 100644 index 00000000000000..0a4e150700fa7c Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidRolloverfromPrintableStringtoUTF8StringTest10EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedinhibitAnyPolicyTest7EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedinhibitAnyPolicyTest7EE.crt new file mode 100644 index 00000000000000..16968ab59b17dc Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedinhibitAnyPolicyTest7EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedinhibitAnyPolicyTest9EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedinhibitAnyPolicyTest9EE.crt new file mode 100644 index 00000000000000..1516f1ee703deb Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedinhibitAnyPolicyTest9EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedinhibitPolicyMappingTest7EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedinhibitPolicyMappingTest7EE.crt new file mode 100644 index 00000000000000..a4385c1d959fb0 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedinhibitPolicyMappingTest7EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedpathLenConstraintTest15EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedpathLenConstraintTest15EE.crt new file mode 100644 index 00000000000000..1cb0924ec360a2 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedpathLenConstraintTest15EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedpathLenConstraintTest17EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedpathLenConstraintTest17EE.crt new file mode 100644 index 00000000000000..ed346760878c83 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedpathLenConstraintTest17EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedrequireExplicitPolicyTest6EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedrequireExplicitPolicyTest6EE.crt new file mode 100644 index 00000000000000..44e5c1e25329a0 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidSelfIssuedrequireExplicitPolicyTest6EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidSeparateCertificateandCRLKeysTest19EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidSeparateCertificateandCRLKeysTest19EE.crt new file mode 100644 index 00000000000000..08260919768f0c Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidSeparateCertificateandCRLKeysTest19EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidTwoCRLsTest7EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidTwoCRLsTest7EE.crt new file mode 100644 index 00000000000000..c42779d70ce106 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidTwoCRLsTest7EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidURInameConstraintsTest34EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidURInameConstraintsTest34EE.crt new file mode 100644 index 00000000000000..be8ef42f19a6de Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidURInameConstraintsTest34EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidURInameConstraintsTest36EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidURInameConstraintsTest36EE.crt new file mode 100644 index 00000000000000..6a24838f5dcf25 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidURInameConstraintsTest36EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidUTF8StringCaseInsensitiveMatchTest11EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidUTF8StringCaseInsensitiveMatchTest11EE.crt new file mode 100644 index 00000000000000..d1f80a74a4bd83 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidUTF8StringCaseInsensitiveMatchTest11EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidUTF8StringEncodedNamesTest9EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidUTF8StringEncodedNamesTest9EE.crt new file mode 100644 index 00000000000000..b14d789b5dc166 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidUTF8StringEncodedNamesTest9EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidUnknownNotCriticalCertificateExtensionTest1EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidUnknownNotCriticalCertificateExtensionTest1EE.crt new file mode 100644 index 00000000000000..d55dcb1a6f493b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidUnknownNotCriticalCertificateExtensionTest1EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidbasicConstraintsNotCriticalTest4EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidbasicConstraintsNotCriticalTest4EE.crt new file mode 100644 index 00000000000000..4059c017a74405 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidbasicConstraintsNotCriticalTest4EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidcRLIssuerTest28EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidcRLIssuerTest28EE.crt new file mode 100644 index 00000000000000..9145515308cc35 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidcRLIssuerTest28EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidcRLIssuerTest29EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidcRLIssuerTest29EE.crt new file mode 100644 index 00000000000000..b10632b2093202 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidcRLIssuerTest29EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidcRLIssuerTest30EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidcRLIssuerTest30EE.crt new file mode 100644 index 00000000000000..593ef98e3508a0 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidcRLIssuerTest30EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidcRLIssuerTest33EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidcRLIssuerTest33EE.crt new file mode 100644 index 00000000000000..2ae810abf9d992 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidcRLIssuerTest33EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValiddeltaCRLTest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValiddeltaCRLTest2EE.crt new file mode 100644 index 00000000000000..a2eb9a7dc42e66 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValiddeltaCRLTest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValiddeltaCRLTest5EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValiddeltaCRLTest5EE.crt new file mode 100644 index 00000000000000..1a3f7f51427177 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValiddeltaCRLTest5EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValiddeltaCRLTest7EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValiddeltaCRLTest7EE.crt new file mode 100644 index 00000000000000..43b44bc5d892f1 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValiddeltaCRLTest7EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValiddeltaCRLTest8EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValiddeltaCRLTest8EE.crt new file mode 100644 index 00000000000000..8be24581eb99dc Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValiddeltaCRLTest8EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValiddistributionPointTest1EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValiddistributionPointTest1EE.crt new file mode 100644 index 00000000000000..b2c832fa41d7ce Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValiddistributionPointTest1EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValiddistributionPointTest4EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValiddistributionPointTest4EE.crt new file mode 100644 index 00000000000000..47feb00fd03667 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValiddistributionPointTest4EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValiddistributionPointTest5EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValiddistributionPointTest5EE.crt new file mode 100644 index 00000000000000..a93d6663841408 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValiddistributionPointTest5EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValiddistributionPointTest7EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValiddistributionPointTest7EE.crt new file mode 100644 index 00000000000000..107f102c98d459 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValiddistributionPointTest7EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidinhibitAnyPolicyTest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidinhibitAnyPolicyTest2EE.crt new file mode 100644 index 00000000000000..df4ba444504e65 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidinhibitAnyPolicyTest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidinhibitPolicyMappingTest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidinhibitPolicyMappingTest2EE.crt new file mode 100644 index 00000000000000..f13524a0dce9bc Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidinhibitPolicyMappingTest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidinhibitPolicyMappingTest4EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidinhibitPolicyMappingTest4EE.crt new file mode 100644 index 00000000000000..75daa870284699 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidinhibitPolicyMappingTest4EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidkeyUsageNotCriticalTest3EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidkeyUsageNotCriticalTest3EE.crt new file mode 100644 index 00000000000000..6da79065ea1d44 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidkeyUsageNotCriticalTest3EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidonlyContainsCACertsTest13EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidonlyContainsCACertsTest13EE.crt new file mode 100644 index 00000000000000..3eec5cc6fe8aef Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidonlyContainsCACertsTest13EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidonlySomeReasonsTest18EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidonlySomeReasonsTest18EE.crt new file mode 100644 index 00000000000000..f255d3ad71181d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidonlySomeReasonsTest18EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidonlySomeReasonsTest19EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidonlySomeReasonsTest19EE.crt new file mode 100644 index 00000000000000..912968e9502997 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidonlySomeReasonsTest19EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidpathLenConstraintTest13EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidpathLenConstraintTest13EE.crt new file mode 100644 index 00000000000000..1ad52efdb6ce88 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidpathLenConstraintTest13EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidpathLenConstraintTest14EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidpathLenConstraintTest14EE.crt new file mode 100644 index 00000000000000..76800f5159cec3 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidpathLenConstraintTest14EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidpathLenConstraintTest7EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidpathLenConstraintTest7EE.crt new file mode 100644 index 00000000000000..f3368edd5d1d17 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidpathLenConstraintTest7EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidpathLenConstraintTest8EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidpathLenConstraintTest8EE.crt new file mode 100644 index 00000000000000..8ff0a131e7c063 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidpathLenConstraintTest8EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/Validpre2000UTCnotBeforeDateTest3EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/Validpre2000UTCnotBeforeDateTest3EE.crt new file mode 100644 index 00000000000000..15b2928401a98e Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/Validpre2000UTCnotBeforeDateTest3EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidrequireExplicitPolicyTest1EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidrequireExplicitPolicyTest1EE.crt new file mode 100644 index 00000000000000..7cf888e16ad198 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidrequireExplicitPolicyTest1EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidrequireExplicitPolicyTest2EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidrequireExplicitPolicyTest2EE.crt new file mode 100644 index 00000000000000..23889360cc222f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidrequireExplicitPolicyTest2EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/ValidrequireExplicitPolicyTest4EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/ValidrequireExplicitPolicyTest4EE.crt new file mode 100644 index 00000000000000..e93a0e1fe91fed Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/ValidrequireExplicitPolicyTest4EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/WrongCRLCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/WrongCRLCACert.crt new file mode 100644 index 00000000000000..3a96d87cfc52f4 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/WrongCRLCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/anyPolicyCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/anyPolicyCACert.crt new file mode 100644 index 00000000000000..df54668adbd277 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/anyPolicyCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/basicConstraintsCriticalcAFalseCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/basicConstraintsCriticalcAFalseCACert.crt new file mode 100644 index 00000000000000..4b678fee0c5687 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/basicConstraintsCriticalcAFalseCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/basicConstraintsNotCriticalCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/basicConstraintsNotCriticalCACert.crt new file mode 100644 index 00000000000000..d6c7fb805fa6a7 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/basicConstraintsNotCriticalCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/basicConstraintsNotCriticalcAFalseCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/basicConstraintsNotCriticalcAFalseCACert.crt new file mode 100644 index 00000000000000..27e670ec162ba9 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/basicConstraintsNotCriticalcAFalseCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/deltaCRLCA1Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/deltaCRLCA1Cert.crt new file mode 100644 index 00000000000000..6815e4f888f146 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/deltaCRLCA1Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/deltaCRLCA2Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/deltaCRLCA2Cert.crt new file mode 100644 index 00000000000000..2f64a74e135563 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/deltaCRLCA2Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/deltaCRLCA3Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/deltaCRLCA3Cert.crt new file mode 100644 index 00000000000000..31e6b33a463e9a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/deltaCRLCA3Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/deltaCRLIndicatorNoBaseCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/deltaCRLIndicatorNoBaseCACert.crt new file mode 100644 index 00000000000000..7cd82a4363c5af Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/deltaCRLIndicatorNoBaseCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/distributionPoint1CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/distributionPoint1CACert.crt new file mode 100644 index 00000000000000..23250812d93bea Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/distributionPoint1CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/distributionPoint2CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/distributionPoint2CACert.crt new file mode 100644 index 00000000000000..205b62ad165f2e Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/distributionPoint2CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA1Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA1Cert.crt new file mode 100644 index 00000000000000..046deefaec87e8 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA1Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA2Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA2Cert.crt new file mode 100644 index 00000000000000..de9a0be510efb6 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA2Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA3Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA3Cert.crt new file mode 100644 index 00000000000000..03bb3eb2da9102 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA3Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA3cRLIssuerCert.crt b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA3cRLIssuerCert.crt new file mode 100644 index 00000000000000..20e8267eeef085 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA3cRLIssuerCert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA4Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA4Cert.crt new file mode 100644 index 00000000000000..f1cb26b375bc0b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA4Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA4cRLIssuerCert.crt b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA4cRLIssuerCert.crt new file mode 100644 index 00000000000000..ff1203df3ad288 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA4cRLIssuerCert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA5Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA5Cert.crt new file mode 100644 index 00000000000000..c4f9f17874606d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA5Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA6Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA6Cert.crt new file mode 100644 index 00000000000000..46443aab941682 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/indirectCRLCA6Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy0CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy0CACert.crt new file mode 100644 index 00000000000000..cf3611025edc96 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy0CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1CACert.crt new file mode 100644 index 00000000000000..0494c8fe5bbbb3 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1SelfIssuedCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1SelfIssuedCACert.crt new file mode 100644 index 00000000000000..6512e9d2e9c981 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1SelfIssuedCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1SelfIssuedsubCA2Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1SelfIssuedsubCA2Cert.crt new file mode 100644 index 00000000000000..42e00344afc255 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1SelfIssuedsubCA2Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1subCA1Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1subCA1Cert.crt new file mode 100644 index 00000000000000..633536c33ae9c0 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1subCA1Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1subCA2Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1subCA2Cert.crt new file mode 100644 index 00000000000000..319e8098786ab4 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1subCA2Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1subCAIAP5Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1subCAIAP5Cert.crt new file mode 100644 index 00000000000000..a3c4f2134e940b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1subCAIAP5Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1subsubCA2Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1subsubCA2Cert.crt new file mode 100644 index 00000000000000..3c4512ac28a231 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy1subsubCA2Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy5CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy5CACert.crt new file mode 100644 index 00000000000000..fc9b42329920c2 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy5CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy5subCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy5subCACert.crt new file mode 100644 index 00000000000000..11ceeb78cc9c15 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy5subCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy5subsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy5subsubCACert.crt new file mode 100644 index 00000000000000..32bbffeb446c7c Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicy5subsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicyTest3EE.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicyTest3EE.crt new file mode 100644 index 00000000000000..2c8fd4f6d12529 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitAnyPolicyTest3EE.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping0CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping0CACert.crt new file mode 100644 index 00000000000000..16808f7c50adce Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping0CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping0subCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping0subCACert.crt new file mode 100644 index 00000000000000..846abc924da146 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping0subCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P12CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P12CACert.crt new file mode 100644 index 00000000000000..5baaf35e0f1f8c Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P12CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P12subCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P12subCACert.crt new file mode 100644 index 00000000000000..b2f0979ccee951 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P12subCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P12subCAIPM5Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P12subCAIPM5Cert.crt new file mode 100644 index 00000000000000..4ad9f1e174dd63 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P12subCAIPM5Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P12subsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P12subsubCACert.crt new file mode 100644 index 00000000000000..f514e5d88b7f69 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P12subsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P12subsubCAIPM5Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P12subsubCAIPM5Cert.crt new file mode 100644 index 00000000000000..b1e9ff8d06883a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P12subsubCAIPM5Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P1CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P1CACert.crt new file mode 100644 index 00000000000000..ec47ee63731771 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P1CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P1SelfIssuedCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P1SelfIssuedCACert.crt new file mode 100644 index 00000000000000..65155c7b5a0376 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P1SelfIssuedCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P1SelfIssuedsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P1SelfIssuedsubCACert.crt new file mode 100644 index 00000000000000..ae1891624b9422 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P1SelfIssuedsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P1subCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P1subCACert.crt new file mode 100644 index 00000000000000..80135df869524a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P1subCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P1subsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P1subsubCACert.crt new file mode 100644 index 00000000000000..3a72ec12fb8beb Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping1P1subsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping5CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping5CACert.crt new file mode 100644 index 00000000000000..fd092230fbdd0c Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping5CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping5subCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping5subCACert.crt new file mode 100644 index 00000000000000..93857ab656577d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping5subCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping5subsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping5subsubCACert.crt new file mode 100644 index 00000000000000..134b7f8cb186c3 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping5subsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping5subsubsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping5subsubsubCACert.crt new file mode 100644 index 00000000000000..dfb268d1d33d16 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/inhibitPolicyMapping5subsubsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/keyUsageCriticalcRLSignFalseCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/keyUsageCriticalcRLSignFalseCACert.crt new file mode 100644 index 00000000000000..2467c945add087 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/keyUsageCriticalcRLSignFalseCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/keyUsageCriticalkeyCertSignFalseCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/keyUsageCriticalkeyCertSignFalseCACert.crt new file mode 100644 index 00000000000000..aa19cec73d2a0d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/keyUsageCriticalkeyCertSignFalseCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/keyUsageNotCriticalCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/keyUsageNotCriticalCACert.crt new file mode 100644 index 00000000000000..bab8307e33090b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/keyUsageNotCriticalCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/keyUsageNotCriticalcRLSignFalseCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/keyUsageNotCriticalcRLSignFalseCACert.crt new file mode 100644 index 00000000000000..a6d878c8dff500 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/keyUsageNotCriticalcRLSignFalseCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/keyUsageNotCriticalkeyCertSignFalseCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/keyUsageNotCriticalkeyCertSignFalseCACert.crt new file mode 100644 index 00000000000000..ef1056f1c39c7f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/keyUsageNotCriticalkeyCertSignFalseCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN1CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN1CACert.crt new file mode 100644 index 00000000000000..206359f9131673 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN1CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN1SelfIssuedCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN1SelfIssuedCACert.crt new file mode 100644 index 00000000000000..452ea547524e5f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN1SelfIssuedCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN1subCA1Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN1subCA1Cert.crt new file mode 100644 index 00000000000000..645f0ae7c4b41d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN1subCA1Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN1subCA2Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN1subCA2Cert.crt new file mode 100644 index 00000000000000..6cfc5926a50581 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN1subCA2Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN1subCA3Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN1subCA3Cert.crt new file mode 100644 index 00000000000000..840d073f6b53fa Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN1subCA3Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN2CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN2CACert.crt new file mode 100644 index 00000000000000..c68d496e6521b3 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN2CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN3CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN3CACert.crt new file mode 100644 index 00000000000000..87ba14d13a64bb Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN3CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN3subCA1Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN3subCA1Cert.crt new file mode 100644 index 00000000000000..7eed575fb4806c Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN3subCA1Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN3subCA2Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN3subCA2Cert.crt new file mode 100644 index 00000000000000..08f2245ef6cde7 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN3subCA2Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN4CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN4CACert.crt new file mode 100644 index 00000000000000..3b114631869447 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN4CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN5CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN5CACert.crt new file mode 100644 index 00000000000000..c190f7a7f24c17 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDN5CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDNS1CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDNS1CACert.crt new file mode 100644 index 00000000000000..a7ec3bd1ebb5da Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDNS1CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDNS2CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDNS2CACert.crt new file mode 100644 index 00000000000000..c70846206c561c Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsDNS2CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsRFC822CA1Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsRFC822CA1Cert.crt new file mode 100644 index 00000000000000..1be8e993355f7f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsRFC822CA1Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsRFC822CA2Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsRFC822CA2Cert.crt new file mode 100644 index 00000000000000..58308f8939d67e Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsRFC822CA2Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsRFC822CA3Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsRFC822CA3Cert.crt new file mode 100644 index 00000000000000..ff6ba166ba488a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsRFC822CA3Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsURI1CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsURI1CACert.crt new file mode 100644 index 00000000000000..5f638c093c03d1 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsURI1CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsURI2CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsURI2CACert.crt new file mode 100644 index 00000000000000..e06b6377a9b356 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/nameConstraintsURI2CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/onlyContainsAttributeCertsCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/onlyContainsAttributeCertsCACert.crt new file mode 100644 index 00000000000000..e8d2b7224a82bb Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/onlyContainsAttributeCertsCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/onlyContainsCACertsCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/onlyContainsCACertsCACert.crt new file mode 100644 index 00000000000000..d75988ad008877 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/onlyContainsCACertsCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/onlyContainsUserCertsCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/onlyContainsUserCertsCACert.crt new file mode 100644 index 00000000000000..0d0b95030b17b3 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/onlyContainsUserCertsCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/onlySomeReasonsCA1Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/onlySomeReasonsCA1Cert.crt new file mode 100644 index 00000000000000..ca247b06b402a6 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/onlySomeReasonsCA1Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/onlySomeReasonsCA2Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/onlySomeReasonsCA2Cert.crt new file mode 100644 index 00000000000000..c1cce6e0cecefd Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/onlySomeReasonsCA2Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/onlySomeReasonsCA3Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/onlySomeReasonsCA3Cert.crt new file mode 100644 index 00000000000000..cd65a820e42b66 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/onlySomeReasonsCA3Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/onlySomeReasonsCA4Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/onlySomeReasonsCA4Cert.crt new file mode 100644 index 00000000000000..f205db0a3bef4d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/onlySomeReasonsCA4Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint0CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint0CACert.crt new file mode 100644 index 00000000000000..ce9b90d28445d0 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint0CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint0SelfIssuedCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint0SelfIssuedCACert.crt new file mode 100644 index 00000000000000..6e8f97c2035ca2 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint0SelfIssuedCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint0subCA2Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint0subCA2Cert.crt new file mode 100644 index 00000000000000..2fc8fb590f88b1 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint0subCA2Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint0subCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint0subCACert.crt new file mode 100644 index 00000000000000..b156179e3a730a Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint0subCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint1CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint1CACert.crt new file mode 100644 index 00000000000000..a424261672868d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint1CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint1SelfIssuedCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint1SelfIssuedCACert.crt new file mode 100644 index 00000000000000..87590c3d262021 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint1SelfIssuedCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint1SelfIssuedsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint1SelfIssuedsubCACert.crt new file mode 100644 index 00000000000000..f2c43ea8936352 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint1SelfIssuedsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint1subCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint1subCACert.crt new file mode 100644 index 00000000000000..05a2bac1da8277 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint1subCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6CACert.crt new file mode 100644 index 00000000000000..c254a2376d3db5 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subCA0Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subCA0Cert.crt new file mode 100644 index 00000000000000..0a8c99dd3e41c4 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subCA0Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subCA1Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subCA1Cert.crt new file mode 100644 index 00000000000000..bd686290efec31 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subCA1Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subCA4Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subCA4Cert.crt new file mode 100644 index 00000000000000..822a383d053b26 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subCA4Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subsubCA00Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subsubCA00Cert.crt new file mode 100644 index 00000000000000..e2fd7ae3cd6556 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subsubCA00Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subsubCA11Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subsubCA11Cert.crt new file mode 100644 index 00000000000000..44c0162e945b26 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subsubCA11Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subsubCA41Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subsubCA41Cert.crt new file mode 100644 index 00000000000000..284f4a9e483050 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subsubCA41Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subsubsubCA11XCert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subsubsubCA11XCert.crt new file mode 100644 index 00000000000000..9766cf01596982 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subsubsubCA11XCert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subsubsubCA41XCert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subsubsubCA41XCert.crt new file mode 100644 index 00000000000000..e14753174b4128 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pathLenConstraint6subsubsubCA41XCert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/pre2000CRLnextUpdateCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/pre2000CRLnextUpdateCACert.crt new file mode 100644 index 00000000000000..30aff16129db1b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/pre2000CRLnextUpdateCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy0CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy0CACert.crt new file mode 100644 index 00000000000000..16594b9e970fdd Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy0CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy0subCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy0subCACert.crt new file mode 100644 index 00000000000000..b7a1518eb86a3f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy0subCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy0subsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy0subsubCACert.crt new file mode 100644 index 00000000000000..db57e9b337f313 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy0subsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy0subsubsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy0subsubsubCACert.crt new file mode 100644 index 00000000000000..4952094eef3e7f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy0subsubsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy10CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy10CACert.crt new file mode 100644 index 00000000000000..3a54e7f2b82d31 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy10CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy10subCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy10subCACert.crt new file mode 100644 index 00000000000000..650a53f4c2e4d8 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy10subCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy10subsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy10subsubCACert.crt new file mode 100644 index 00000000000000..139be532a59c87 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy10subsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy10subsubsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy10subsubsubCACert.crt new file mode 100644 index 00000000000000..a7c216c1640d08 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy10subsubsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy2CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy2CACert.crt new file mode 100644 index 00000000000000..f7ca7ae7e2a271 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy2CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy2SelfIssuedCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy2SelfIssuedCACert.crt new file mode 100644 index 00000000000000..9d1626909055a6 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy2SelfIssuedCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy2SelfIssuedsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy2SelfIssuedsubCACert.crt new file mode 100644 index 00000000000000..b53bec1560d18f Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy2SelfIssuedsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy2subCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy2subCACert.crt new file mode 100644 index 00000000000000..36fc0d8df48dfe Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy2subCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy4CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy4CACert.crt new file mode 100644 index 00000000000000..723ae42a47c9ec Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy4CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy4subCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy4subCACert.crt new file mode 100644 index 00000000000000..1bd237f766cb0b Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy4subCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy4subsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy4subsubCACert.crt new file mode 100644 index 00000000000000..1a37158581b428 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy4subsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy4subsubsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy4subsubsubCACert.crt new file mode 100644 index 00000000000000..3047d74341e9a0 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy4subsubsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy5CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy5CACert.crt new file mode 100644 index 00000000000000..c6b69ad95d551d Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy5CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy5subCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy5subCACert.crt new file mode 100644 index 00000000000000..16958532f00d75 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy5subCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy5subsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy5subsubCACert.crt new file mode 100644 index 00000000000000..093963aeca5a65 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy5subsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy5subsubsubCACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy5subsubsubCACert.crt new file mode 100644 index 00000000000000..58da176c465517 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy5subsubsubCACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy7CACert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy7CACert.crt new file mode 100644 index 00000000000000..aba4a7fde45e75 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy7CACert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy7subCARE2Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy7subCARE2Cert.crt new file mode 100644 index 00000000000000..c57e9e4a5bdd40 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy7subCARE2Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy7subsubCARE2RE4Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy7subsubCARE2RE4Cert.crt new file mode 100644 index 00000000000000..343efa5ec2d0aa Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy7subsubCARE2RE4Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy7subsubsubCARE2RE4Cert.crt b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy7subsubsubCARE2RE4Cert.crt new file mode 100644 index 00000000000000..9a8e72a1ce3b91 Binary files /dev/null and b/src/crypto/x509/testdata/nist-pkits/certs/requireExplicitPolicy7subsubsubCARE2RE4Cert.crt differ diff --git a/src/crypto/x509/testdata/nist-pkits/vectors.json b/src/crypto/x509/testdata/nist-pkits/vectors.json new file mode 100644 index 00000000000000..5842b4326df33d --- /dev/null +++ b/src/crypto/x509/testdata/nist-pkits/vectors.json @@ -0,0 +1,5010 @@ +[ + { + "Name": "4.1.1 Valid Signatures Test1", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "ValidCertificatePathTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.1.2 Invalid CA Signature Test2", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "BadSignedCACert.crt", + "InvalidCASignatureTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "BadSignedCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.1.3 Invalid EE Signature Test3", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "InvalidEESignatureTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.1.4 Valid DSA Signatures Test4", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "DSACACert.crt", + "ValidDSASignaturesTest4EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "DSACACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.1.5 Valid DSA Parameter Inheritance Test5", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "DSACACert.crt", + "DSAParametersInheritedCACert.crt", + "ValidDSAParameterInheritanceTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "DSACACRL.crl", + "DSAParametersInheritedCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.1.6 Invalid DSA Signature Test6", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "DSACACert.crt", + "InvalidDSASignatureTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "DSACACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.2.1 Invalid CA notBefore Date Test1", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "BadnotBeforeDateCACert.crt", + "InvalidCAnotBeforeDateTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "BadnotBeforeDateCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.2.2 Invalid EE notBefore Date Test2", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "InvalidEEnotBeforeDateTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.2.3 Valid pre2000 UTC notBefore Date Test3", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "Validpre2000UTCnotBeforeDateTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.2.4 Valid GeneralizedTime notBefore Date Test4", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "ValidGeneralizedTimenotBeforeDateTest4EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.2.5 Invalid CA notAfter Date Test5", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "BadnotAfterDateCACert.crt", + "InvalidCAnotAfterDateTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "BadnotAfterDateCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.2.6 Invalid EE notAfter Date Test6", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "InvalidEEnotAfterDateTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.2.7 Invalid pre2000 UTC EE notAfter Date Test7", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "Invalidpre2000UTCEEnotAfterDateTest7EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.2.8 Valid GeneralizedTime notAfter Date Test8", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "ValidGeneralizedTimenotAfterDateTest8EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.3.1 Invalid Name Chaining EE Test1", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "InvalidNameChainingTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.3.2 Invalid Name Chaining Order Test2", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "NameOrderingCACert.crt", + "InvalidNameChainingOrderTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "NameOrderCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.3.3 Valid Name Chaining Whitespace Test3", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "ValidNameChainingWhitespaceTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.3.4 Valid Name Chaining Whitespace Test4", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "ValidNameChainingWhitespaceTest4EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.3.5 Valid Name Chaining Capitalization Test5", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "ValidNameChainingCapitalizationTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.3.6 Valid Name Chaining UIDs Test6", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "UIDCACert.crt", + "ValidNameUIDsTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "UIDCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.3.7 Valid RFC3280 Mandatory Attribute Types Test7", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "RFC3280MandatoryAttributeTypesCACert.crt", + "ValidRFC3280MandatoryAttributeTypesTest7EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "RFC3280MandatoryAttributeTypesCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.3.8 Valid RFC3280 Optional Attribute Types Test8", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "RFC3280OptionalAttributeTypesCACert.crt", + "ValidRFC3280OptionalAttributeTypesTest8EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "RFC3280OptionalAttributeTypesCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.3.9 Valid UTF8String Encoded Names Test9", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "UTF8StringEncodedNamesCACert.crt", + "ValidUTF8StringEncodedNamesTest9EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "UTF8StringEncodedNamesCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.3.10 Valid Rollover from PrintableString to UTF8String Test10", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "RolloverfromPrintableStringtoUTF8StringCACert.crt", + "ValidRolloverfromPrintableStringtoUTF8StringTest10EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "RolloverfromPrintableStringtoUTF8StringCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.3.11 Valid UTF8String Case Insensitive Match Test11", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "UTF8StringCaseInsensitiveMatchCACert.crt", + "ValidUTF8StringCaseInsensitiveMatchTest11EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "UTF8StringCaseInsensitiveMatchCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.1 Missing CRL Test1", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "NoCRLCACert.crt", + "InvalidMissingCRLTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.2 Invalid Revoked CA Test2", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "RevokedsubCACert.crt", + "InvalidRevokedCATest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl", + "RevokedsubCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.3 Invalid Revoked EE Test3", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "InvalidRevokedEETest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.4 Invalid Bad CRL Signature Test4", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "BadCRLSignatureCACert.crt", + "InvalidBadCRLSignatureTest4EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "BadCRLSignatureCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.5 Invalid Bad CRL Issuer Name Test5", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "BadCRLIssuerNameCACert.crt", + "InvalidBadCRLIssuerNameTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "BadCRLIssuerNameCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.6 Invalid Wrong CRL Test6", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "WrongCRLCACert.crt", + "InvalidWrongCRLTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "WrongCRLCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.7 Valid Two CRLs Test7", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "TwoCRLsCACert.crt", + "ValidTwoCRLsTest7EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "TwoCRLsCAGoodCRL.crl", + "TwoCRLsCABadCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.8 Invalid Unknown CRL Entry Extension Test8", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "UnknownCRLEntryExtensionCACert.crt", + "InvalidUnknownCRLEntryExtensionTest8EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "UnknownCRLEntryExtensionCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.9 Invalid Unknown CRL Extension Test9", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "UnknownCRLExtensionCACert.crt", + "InvalidUnknownCRLExtensionTest9EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "UnknownCRLExtensionCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.10 Invalid Unknown CRL Extension Test10", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "UnknownCRLExtensionCACert.crt", + "InvalidUnknownCRLExtensionTest10EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "UnknownCRLExtensionCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.11 Invalid Old CRL nextUpdate Test11", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "OldCRLnextUpdateCACert.crt", + "InvalidOldCRLnextUpdateTest11EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "OldCRLnextUpdateCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.12 Invalid pre2000 CRL nextUpdate Test12", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "pre2000CRLnextUpdateCACert.crt", + "Invalidpre2000CRLnextUpdateTest12EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "pre2000CRLnextUpdateCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.13 Valid GeneralizedTime CRL nextUpdate Test13", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GeneralizedTimeCRLnextUpdateCACert.crt", + "ValidGeneralizedTimeCRLnextUpdateTest13EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GeneralizedTimeCRLnextUpdateCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.14 Valid Negative Serial Number Test14", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "NegativeSerialNumberCACert.crt", + "ValidNegativeSerialNumberTest14EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "NegativeSerialNumberCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.15 Invalid Negative Serial Number Test15", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "NegativeSerialNumberCACert.crt", + "InvalidNegativeSerialNumberTest15EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "NegativeSerialNumberCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.16 Valid Long Serial Number Test16", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "LongSerialNumberCACert.crt", + "ValidLongSerialNumberTest16EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "LongSerialNumberCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.17 Valid Long Serial Number Test17", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "LongSerialNumberCACert.crt", + "ValidLongSerialNumberTest17EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "LongSerialNumberCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.18 Invalid Long Serial Number Test18", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "LongSerialNumberCACert.crt", + "InvalidLongSerialNumberTest18EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "LongSerialNumberCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.19 Valid Separate Certificate and CRL Keys Test19", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "SeparateCertificateandCRLKeysCertificateSigningCACert.crt", + "SeparateCertificateandCRLKeysCRLSigningCert.crt", + "ValidSeparateCertificateandCRLKeysTest19EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "SeparateCertificateandCRLKeysCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.20 Invalid Separate Certificate and CRL Keys Test20", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "SeparateCertificateandCRLKeysCertificateSigningCACert.crt", + "SeparateCertificateandCRLKeysCRLSigningCert.crt", + "InvalidSeparateCertificateandCRLKeysTest20EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "SeparateCertificateandCRLKeysCRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.4.21 Invalid Separate Certificate and CRL Keys Test21", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "SeparateCertificateandCRLKeysCA2CertificateSigningCACert.crt", + "SeparateCertificateandCRLKeysCA2CRLSigningCert.crt", + "InvalidSeparateCertificateandCRLKeysTest21EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "SeparateCertificateandCRLKeysCA2CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.5.1 Valid Basic Self-Issued Old With New Test1", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "BasicSelfIssuedNewKeyCACert.crt", + "BasicSelfIssuedNewKeyOldWithNewCACert.crt", + "ValidBasicSelfIssuedOldWithNewTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "BasicSelfIssuedNewKeyCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.5.2 Invalid Basic Self-Issued Old With New Test2", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "BasicSelfIssuedNewKeyCACert.crt", + "BasicSelfIssuedNewKeyOldWithNewCACert.crt", + "InvalidBasicSelfIssuedOldWithNewTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "BasicSelfIssuedNewKeyCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.5.3 Valid Basic Self-Issued New With Old Test3", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "BasicSelfIssuedOldKeyCACert.crt", + "BasicSelfIssuedOldKeyNewWithOldCACert.crt", + "ValidBasicSelfIssuedNewWithOldTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "BasicSelfIssuedOldKeySelfIssuedCertCRL.crl", + "BasicSelfIssuedOldKeyCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.5.4 Valid Basic Self-Issued New With Old Test4", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "BasicSelfIssuedOldKeyCACert.crt", + "BasicSelfIssuedOldKeyNewWithOldCACert.crt", + "ValidBasicSelfIssuedNewWithOldTest4EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "BasicSelfIssuedOldKeySelfIssuedCertCRL.crl", + "BasicSelfIssuedOldKeyCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.5.5 Invalid Basic Self-Issued New With Old Test5", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "BasicSelfIssuedOldKeyCACert.crt", + "BasicSelfIssuedOldKeyNewWithOldCACert.crt", + "InvalidBasicSelfIssuedNewWithOldTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "BasicSelfIssuedOldKeySelfIssuedCertCRL.crl", + "BasicSelfIssuedOldKeyCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.5.6 Valid Basic Self-Issued CRL Signing Key Test6", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "BasicSelfIssuedCRLSigningKeyCACert.crt", + "BasicSelfIssuedCRLSigningKeyCRLCert.crt", + "ValidBasicSelfIssuedCRLSigningKeyTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "BasicSelfIssuedCRLSigningKeyCRLCertCRL.crl", + "BasicSelfIssuedCRLSigningKeyCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.5.7 Invalid Basic Self-Issued CRL Signing Key Test7", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "BasicSelfIssuedCRLSigningKeyCACert.crt", + "BasicSelfIssuedCRLSigningKeyCRLCert.crt", + "InvalidBasicSelfIssuedCRLSigningKeyTest7EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "BasicSelfIssuedCRLSigningKeyCRLCertCRL.crl", + "BasicSelfIssuedCRLSigningKeyCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.5.8 Invalid Basic Self-Issued CRL Signing Key Test8", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "BasicSelfIssuedCRLSigningKeyCACert.crt", + "BasicSelfIssuedCRLSigningKeyCRLCert.crt", + "InvalidBasicSelfIssuedCRLSigningKeyTest8EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "BasicSelfIssuedCRLSigningKeyCRLCertCRL.crl", + "BasicSelfIssuedCRLSigningKeyCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.1 Invalid Missing basicConstraints Test1", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "MissingbasicConstraintsCACert.crt", + "InvalidMissingbasicConstraintsTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "MissingbasicConstraintsCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.2 Invalid cA False Test2", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "basicConstraintsCriticalcAFalseCACert.crt", + "InvalidcAFalseTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "basicConstraintsCriticalcAFalseCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.3 Invalid cA False Test3", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "basicConstraintsNotCriticalcAFalseCACert.crt", + "InvalidcAFalseTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "basicConstraintsNotCriticalcAFalseCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.4 Valid basicConstraints Not Critical Test4", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "basicConstraintsNotCriticalCACert.crt", + "ValidbasicConstraintsNotCriticalTest4EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "basicConstraintsNotCriticalCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.5 Invalid pathLenConstraint Test5", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "pathLenConstraint0CACert.crt", + "pathLenConstraint0subCACert.crt", + "InvalidpathLenConstraintTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "pathLenConstraint0CACRL.crl", + "pathLenConstraint0subCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.6 Invalid pathLenConstraint Test6", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "pathLenConstraint0CACert.crt", + "pathLenConstraint0subCACert.crt", + "InvalidpathLenConstraintTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "pathLenConstraint0CACRL.crl", + "pathLenConstraint0subCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.7 Valid pathLenConstraint Test7", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "pathLenConstraint0CACert.crt", + "ValidpathLenConstraintTest7EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "pathLenConstraint0CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.8 Valid pathLenConstraint Test8", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "pathLenConstraint0CACert.crt", + "ValidpathLenConstraintTest8EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "pathLenConstraint0CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.9 Invalid pathLenConstraint Test9", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "pathLenConstraint6CACert.crt", + "pathLenConstraint6subCA0Cert.crt", + "pathLenConstraint6subsubCA00Cert.crt", + "InvalidpathLenConstraintTest9EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "pathLenConstraint6CACRL.crl", + "pathLenConstraint6subCA0CRL.crl", + "pathLenConstraint6subsubCA00CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.10 Invalid pathLenConstraint Test10", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "pathLenConstraint6CACert.crt", + "pathLenConstraint6subCA0Cert.crt", + "pathLenConstraint6subsubCA00Cert.crt", + "InvalidpathLenConstraintTest10EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "pathLenConstraint6CACRL.crl", + "pathLenConstraint6subCA0CRL.crl", + "pathLenConstraint6subsubCA00CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.11 Invalid pathLenConstraint Test11", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "pathLenConstraint6CACert.crt", + "pathLenConstraint6subCA1Cert.crt", + "pathLenConstraint6subsubCA11Cert.crt", + "pathLenConstraint6subsubsubCA11XCert.crt", + "InvalidpathLenConstraintTest11EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "pathLenConstraint6CACRL.crl", + "pathLenConstraint6subCA1CRL.crl", + "pathLenConstraint6subsubCA11CRL.crl", + "pathLenConstraint6subsubsubCA11XCRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.12 Invalid pathLenConstraint Test12", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "pathLenConstraint6CACert.crt", + "pathLenConstraint6subCA1Cert.crt", + "pathLenConstraint6subsubCA11Cert.crt", + "pathLenConstraint6subsubsubCA11XCert.crt", + "InvalidpathLenConstraintTest12EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "pathLenConstraint6CACRL.crl", + "pathLenConstraint6subCA1CRL.crl", + "pathLenConstraint6subsubCA11CRL.crl", + "pathLenConstraint6subsubsubCA11XCRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.13 Valid pathLenConstraint Test13", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "pathLenConstraint6CACert.crt", + "pathLenConstraint6subCA4Cert.crt", + "pathLenConstraint6subsubCA41Cert.crt", + "pathLenConstraint6subsubsubCA41XCert.crt", + "ValidpathLenConstraintTest13EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "pathLenConstraint6CACRL.crl", + "pathLenConstraint6subCA4CRL.crl", + "pathLenConstraint6subsubCA41CRL.crl", + "pathLenConstraint6subsubsubCA41XCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.14 Valid pathLenConstraint Test14", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "pathLenConstraint6CACert.crt", + "pathLenConstraint6subCA4Cert.crt", + "pathLenConstraint6subsubCA41Cert.crt", + "pathLenConstraint6subsubsubCA41XCert.crt", + "ValidpathLenConstraintTest14EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "pathLenConstraint6CACRL.crl", + "pathLenConstraint6subCA4CRL.crl", + "pathLenConstraint6subsubCA41CRL.crl", + "pathLenConstraint6subsubsubCA41XCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.15 Valid Self-Issued pathLenConstraint Test15", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "pathLenConstraint0CACert.crt", + "pathLenConstraint0SelfIssuedCACert.crt", + "ValidSelfIssuedpathLenConstraintTest15EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "pathLenConstraint0CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.16 Invalid Self-Issued pathLenConstraint Test16", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "pathLenConstraint0CACert.crt", + "pathLenConstraint0SelfIssuedCACert.crt", + "pathLenConstraint0subCA2Cert.crt", + "InvalidSelfIssuedpathLenConstraintTest16EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "pathLenConstraint0CACRL.crl", + "pathLenConstraint0subCA2CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.6.17 Valid Self-Issued pathLenConstraint Test17", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "pathLenConstraint1CACert.crt", + "pathLenConstraint1SelfIssuedCACert.crt", + "pathLenConstraint1subCACert.crt", + "pathLenConstraint1SelfIssuedsubCACert.crt", + "ValidSelfIssuedpathLenConstraintTest17EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "pathLenConstraint1CACRL.crl", + "pathLenConstraint1subCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.7.1 Invalid keyUsage Critical keyCertSign False Test1", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "keyUsageCriticalkeyCertSignFalseCACert.crt", + "InvalidkeyUsageCriticalkeyCertSignFalseTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "keyUsageCriticalkeyCertSignFalseCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.7.2 Invalid keyUsage Not Critical keyCertSign False Test2", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "keyUsageNotCriticalkeyCertSignFalseCACert.crt", + "InvalidkeyUsageNotCriticalkeyCertSignFalseTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "keyUsageNotCriticalkeyCertSignFalseCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.7.3 Valid keyUsage Not Critical Test3", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "keyUsageNotCriticalCACert.crt", + "ValidkeyUsageNotCriticalTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "keyUsageNotCriticalCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.7.4 Invalid keyUsage Critical cRLSign False Test4", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "keyUsageCriticalcRLSignFalseCACert.crt", + "InvalidkeyUsageCriticalcRLSignFalseTest4EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "keyUsageCriticalcRLSignFalseCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.7.5 Invalid keyUsage Not Critical cRLSign False Test5", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "keyUsageNotCriticalcRLSignFalseCACert.crt", + "InvalidkeyUsageNotCriticalcRLSignFalseTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "keyUsageNotCriticalcRLSignFalseCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.1 All Certificates Same Policy Test1 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "ValidCertificatePathTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": true, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.1 All Certificates Same Policy Test1 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "ValidCertificatePathTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-1" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": true, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.1 All Certificates Same Policy Test1 (Subpart 3)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "ValidCertificatePathTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "NIST-test-policy-2" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": true, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.1 All Certificates Same Policy Test1 (Subpart 4)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "ValidCertificatePathTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-1", + "NIST-test-policy-2" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": true, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.2 All Certificates No Policies Test2 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "NoPoliciesCACert.crt", + "AllCertificatesNoPoliciesTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "NoPoliciesCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.2 All Certificates No Policies Test2 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "NoPoliciesCACert.crt", + "AllCertificatesNoPoliciesTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "NoPoliciesCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": true, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.3 Different Policies Test3 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "PoliciesP2subCACert.crt", + "DifferentPoliciesTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl", + "PoliciesP2subCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.3 Different Policies Test3 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "PoliciesP2subCACert.crt", + "DifferentPoliciesTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl", + "PoliciesP2subCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": true, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.3 Different Policies Test3 (Subpart 3)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "PoliciesP2subCACert.crt", + "DifferentPoliciesTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl", + "PoliciesP2subCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "NIST-test-policy-1", + "NIST-test-policy-2" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": true, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.4 Different Policies Test4", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "GoodsubCACert.crt", + "DifferentPoliciesTest4EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl", + "GoodsubCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.5 Different Policies Test5", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "PoliciesP2subCA2Cert.crt", + "DifferentPoliciesTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl", + "PoliciesP2subCA2CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.6 Overlapping Policies Test6 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PoliciesP1234CACert.crt", + "PoliciesP1234subCAP123Cert.crt", + "PoliciesP1234subsubCAP123P12Cert.crt", + "OverlappingPoliciesTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PoliciesP1234CACRL.crl", + "PoliciesP1234subCAP123CRL.crl", + "PoliciesP1234subsubCAP123P12CRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.6 Overlapping Policies Test6 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PoliciesP1234CACert.crt", + "PoliciesP1234subCAP123Cert.crt", + "PoliciesP1234subsubCAP123P12Cert.crt", + "OverlappingPoliciesTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PoliciesP1234CACRL.crl", + "PoliciesP1234subCAP123CRL.crl", + "PoliciesP1234subsubCAP123P12CRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-1" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.6 Overlapping Policies Test6 (Subpart 3)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PoliciesP1234CACert.crt", + "PoliciesP1234subCAP123Cert.crt", + "PoliciesP1234subsubCAP123P12Cert.crt", + "OverlappingPoliciesTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PoliciesP1234CACRL.crl", + "PoliciesP1234subCAP123CRL.crl", + "PoliciesP1234subsubCAP123P12CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "NIST-test-policy-2" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.7 Different Policies Test7", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PoliciesP123CACert.crt", + "PoliciesP123subCAP12Cert.crt", + "PoliciesP123subsubCAP12P1Cert.crt", + "DifferentPoliciesTest7EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PoliciesP123CACRL.crl", + "PoliciesP123subCAP12CRL.crl", + "PoliciesP123subsubCAP12P1CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.8 Different Policies Test8", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PoliciesP12CACert.crt", + "PoliciesP12subCAP1Cert.crt", + "PoliciesP12subsubCAP1P2Cert.crt", + "DifferentPoliciesTest8EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PoliciesP12CACRL.crl", + "PoliciesP12subCAP1CRL.crl", + "PoliciesP12subsubCAP1P2CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.9 Different Policies Test9", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PoliciesP123CACert.crt", + "PoliciesP123subCAP12Cert.crt", + "PoliciesP123subsubCAP12P2Cert.crt", + "PoliciesP123subsubsubCAP12P2P1Cert.crt", + "DifferentPoliciesTest9EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PoliciesP123CACRL.crl", + "PoliciesP123subCAP12CRL.crl", + "PoliciesP123subsubCAP2P2CRL.crl", + "PoliciesP123subsubsubCAP12P2P1CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.10 All Certificates Same Policies Test10 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PoliciesP12CACert.crt", + "AllCertificatesSamePoliciesTest10EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PoliciesP12CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.10 All Certificates Same Policies Test10 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PoliciesP12CACert.crt", + "AllCertificatesSamePoliciesTest10EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PoliciesP12CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-1" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.10 All Certificates Same Policies Test10 (Subpart 3)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PoliciesP12CACert.crt", + "AllCertificatesSamePoliciesTest10EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PoliciesP12CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-2" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.11 All Certificates AnyPolicy Test11 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "anyPolicyCACert.crt", + "AllCertificatesanyPolicyTest11EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "anyPolicyCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.11 All Certificates AnyPolicy Test11 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "anyPolicyCACert.crt", + "AllCertificatesanyPolicyTest11EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "anyPolicyCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-1" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.12 Different Policies Test12", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PoliciesP3CACert.crt", + "DifferentPoliciesTest12EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PoliciesP3CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.13 All Certificates Same Policies Test13 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PoliciesP123CACert.crt", + "AllCertificatesSamePoliciesTest13EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PoliciesP123CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-1" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.13 All Certificates Same Policies Test13 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PoliciesP123CACert.crt", + "AllCertificatesSamePoliciesTest13EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PoliciesP123CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-2" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.13 All Certificates Same Policies Test13 (Subpart 3)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PoliciesP123CACert.crt", + "AllCertificatesSamePoliciesTest13EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PoliciesP123CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-3" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.14 AnyPolicy Test14 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "anyPolicyCACert.crt", + "AnyPolicyTest14EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "anyPolicyCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-1" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.14 AnyPolicy Test14 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "anyPolicyCACert.crt", + "AnyPolicyTest14EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "anyPolicyCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "NIST-test-policy-2" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.15 User Notice Qualifier Test15", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "UserNoticeQualifierTest15EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.16 User Notice Qualifier Test16", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "UserNoticeQualifierTest16EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.17 User Notice Qualifier Test17", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "UserNoticeQualifierTest17EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.18 User Notice Qualifier Test18 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PoliciesP12CACert.crt", + "UserNoticeQualifierTest18EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PoliciesP12CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-1" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.18 User Notice Qualifier Test18 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PoliciesP12CACert.crt", + "UserNoticeQualifierTest18EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PoliciesP12CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-2" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.19 User Notice Qualifier Test19", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "UserNoticeQualifierTest19EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.8.20 CPS Pointer Qualifier Test20", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "CPSPointerQualifierTest20EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-1" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": true, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.9.1 Valid RequireExplicitPolicy Test1", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "requireExplicitPolicy10CACert.crt", + "requireExplicitPolicy10subCACert.crt", + "requireExplicitPolicy10subsubCACert.crt", + "requireExplicitPolicy10subsubsubCACert.crt", + "ValidrequireExplicitPolicyTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "requireExplicitPolicy10CACRL.crl", + "requireExplicitPolicy10subCACRL.crl", + "requireExplicitPolicy10subsubCACRL.crl", + "requireExplicitPolicy10subsubsubCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.9.2 Valid RequireExplicitPolicy Test2", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "requireExplicitPolicy5CACert.crt", + "requireExplicitPolicy5subCACert.crt", + "requireExplicitPolicy5subsubCACert.crt", + "requireExplicitPolicy5subsubsubCACert.crt", + "ValidrequireExplicitPolicyTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "requireExplicitPolicy5CACRL.crl", + "requireExplicitPolicy5subCACRL.crl", + "requireExplicitPolicy5subsubCACRL.crl", + "requireExplicitPolicy5subsubsubCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.9.3 Invalid RequireExplicitPolicy Test3", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "requireExplicitPolicy4CACert.crt", + "requireExplicitPolicy4subCACert.crt", + "requireExplicitPolicy4subsubCACert.crt", + "requireExplicitPolicy4subsubsubCACert.crt", + "InvalidrequireExplicitPolicyTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "requireExplicitPolicy4CACRL.crl", + "requireExplicitPolicy4subCACRL.crl", + "requireExplicitPolicy4subsubCACRL.crl", + "requireExplicitPolicy4subsubsubCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.9.4 Valid RequireExplicitPolicy Test4", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "requireExplicitPolicy0CACert.crt", + "requireExplicitPolicy0subCACert.crt", + "requireExplicitPolicy0subsubCACert.crt", + "requireExplicitPolicy0subsubsubCACert.crt", + "ValidrequireExplicitPolicyTest4EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "requireExplicitPolicy0CACRL.crl", + "requireExplicitPolicy0subCACRL.crl", + "requireExplicitPolicy0subsubCACRL.crl", + "requireExplicitPolicy0subsubsubCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.9.5 Invalid RequireExplicitPolicy Test5", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "requireExplicitPolicy7CACert.crt", + "requireExplicitPolicy7subCARE2Cert.crt", + "requireExplicitPolicy7subsubCARE2RE4Cert.crt", + "requireExplicitPolicy7subsubsubCARE2RE4Cert.crt", + "InvalidrequireExplicitPolicyTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "requireExplicitPolicy7CACRL.crl", + "requireExplicitPolicy7subCARE2CRL.crl", + "requireExplicitPolicy7subsubCARE2RE4CRL.crl", + "requireExplicitPolicy7subsubsubCARE2RE4CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.9.6 Valid Self-Issued requireExplicitPolicy Test6", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "requireExplicitPolicy2CACert.crt", + "requireExplicitPolicy2SelfIssuedCACert.crt", + "ValidSelfIssuedrequireExplicitPolicyTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "requireExplicitPolicy2CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.9.7 Invalid Self-Issued requireExplicitPolicy Test7", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "requireExplicitPolicy2CACert.crt", + "requireExplicitPolicy2SelfIssuedCACert.crt", + "requireExplicitPolicy2subCACert.crt", + "InvalidSelfIssuedrequireExplicitPolicyTest7EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "requireExplicitPolicy2CACRL.crl", + "requireExplicitPolicy2subCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.9.8 Invalid Self-Issued requireExplicitPolicy Test8", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "requireExplicitPolicy2CACert.crt", + "requireExplicitPolicy2SelfIssuedCACert.crt", + "requireExplicitPolicy2subCACert.crt", + "requireExplicitPolicy2SelfIssuedsubCACert.crt", + "InvalidSelfIssuedrequireExplicitPolicyTest8EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "requireExplicitPolicy2CACRL.crl", + "requireExplicitPolicy2subCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.1.1 Valid Policy Mapping Test1 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "Mapping1to2CACert.crt", + "ValidPolicyMappingTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "Mapping1to2CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-1" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.1.2 Valid Policy Mapping Test1 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "Mapping1to2CACert.crt", + "ValidPolicyMappingTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "Mapping1to2CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "NIST-test-policy-2" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.1.3 Valid Policy Mapping Test1 (Subpart 3)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "Mapping1to2CACert.crt", + "ValidPolicyMappingTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "Mapping1to2CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": true, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.2 Invalid Policy Mapping Test2 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "Mapping1to2CACert.crt", + "InvalidPolicyMappingTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "Mapping1to2CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.2 Invalid Policy Mapping Test2 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "Mapping1to2CACert.crt", + "InvalidPolicyMappingTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "Mapping1to2CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": true, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.3 Valid Policy Mapping Test3 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "P12Mapping1to3CACert.crt", + "P12Mapping1to3subCACert.crt", + "P12Mapping1to3subsubCACert.crt", + "ValidPolicyMappingTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "P12Mapping1to3CACRL.crl", + "P12Mapping1to3subCACRL.crl", + "P12Mapping1to3subsubCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "NIST-test-policy-1" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.3 Valid Policy Mapping Test3 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "P12Mapping1to3CACert.crt", + "P12Mapping1to3subCACert.crt", + "P12Mapping1to3subsubCACert.crt", + "ValidPolicyMappingTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "P12Mapping1to3CACRL.crl", + "P12Mapping1to3subCACRL.crl", + "P12Mapping1to3subsubCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-2" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.4 Invalid Policy Mapping Test4", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "P12Mapping1to3CACert.crt", + "P12Mapping1to3subCACert.crt", + "P12Mapping1to3subsubCACert.crt", + "InvalidPolicyMappingTest4EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "P12Mapping1to3CACRL.crl", + "P12Mapping1to3subCACRL.crl", + "P12Mapping1to3subsubCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.5 Valid Policy Mapping Test5 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "P1Mapping1to234CACert.crt", + "P1Mapping1to234subCACert.crt", + "ValidPolicyMappingTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "P1Mapping1to234CACRL.crl", + "P1Mapping1to234subCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-1" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.5 Valid Policy Mapping Test5 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "P1Mapping1to234CACert.crt", + "P1Mapping1to234subCACert.crt", + "ValidPolicyMappingTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "P1Mapping1to234CACRL.crl", + "P1Mapping1to234subCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "NIST-test-policy-6" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.6 Valid Policy Mapping Test6 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "P1Mapping1to234CACert.crt", + "P1Mapping1to234subCACert.crt", + "ValidPolicyMappingTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "P1Mapping1to234CACRL.crl", + "P1Mapping1to234subCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-1" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.6 Valid Policy Mapping Test6 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "P1Mapping1to234CACert.crt", + "P1Mapping1to234subCACert.crt", + "ValidPolicyMappingTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "P1Mapping1to234CACRL.crl", + "P1Mapping1to234subCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "NIST-test-policy-6" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.7 Invalid Mapping From anyPolicy Test7", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "MappingFromanyPolicyCACert.crt", + "InvalidMappingFromanyPolicyTest7EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "MappingFromanyPolicyCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.8 Invalid Mapping To anyPolicy Test8", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "MappingToanyPolicyCACert.crt", + "InvalidMappingToanyPolicyTest8EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "MappingToanyPolicyCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.9 Valid Policy Mapping Test9", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "PanyPolicyMapping1to2CACert.crt", + "ValidPolicyMappingTest9EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "PanyPolicyMapping1to2CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.10 Invalid Policy Mapping Test10", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "GoodsubCAPanyPolicyMapping1to2CACert.crt", + "InvalidPolicyMappingTest10EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl", + "GoodsubCAPanyPolicyMapping1to2CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.11 Valid Policy Mapping Test11", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "GoodCACert.crt", + "GoodsubCAPanyPolicyMapping1to2CACert.crt", + "ValidPolicyMappingTest11EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl", + "GoodsubCAPanyPolicyMapping1to2CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.12 Valid Policy Mapping Test12 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "P12Mapping1to3CACert.crt", + "ValidPolicyMappingTest12EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "P12Mapping1to3CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-1" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.12 Valid Policy Mapping Test12 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "P12Mapping1to3CACert.crt", + "ValidPolicyMappingTest12EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "P12Mapping1to3CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-2" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.13 Valid Policy Mapping Test13 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "P1anyPolicyMapping1to2CACert.crt", + "ValidPolicyMappingTest13EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "P1anyPolicyMapping1to2CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.13 Valid Policy Mapping Test13 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "P1anyPolicyMapping1to2CACert.crt", + "ValidPolicyMappingTest13EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "P1anyPolicyMapping1to2CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "NIST-test-policy-1", + "NIST-test-policy-2" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.13 Valid Policy Mapping Test13 (Subpart 3)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "P1anyPolicyMapping1to2CACert.crt", + "ValidPolicyMappingTest13EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "P1anyPolicyMapping1to2CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "NIST-test-policy-2" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.10.14 Valid Policy Mapping Test14", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "P1anyPolicyMapping1to2CACert.crt", + "ValidPolicyMappingTest14EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "P1anyPolicyMapping1to2CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.11.1 Invalid inhibitPolicyMapping Test1", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitPolicyMapping0CACert.crt", + "inhibitPolicyMapping0subCACert.crt", + "InvalidinhibitPolicyMappingTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitPolicyMapping0CACRL.crl", + "inhibitPolicyMapping0subCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.11.2 Valid inhibitPolicyMapping Test2", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitPolicyMapping1P12CACert.crt", + "inhibitPolicyMapping1P12subCACert.crt", + "ValidinhibitPolicyMappingTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitPolicyMapping1P12CACRL.crl", + "inhibitPolicyMapping1P12subCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.11.3 Invalid inhibitPolicyMapping Test3", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitPolicyMapping1P12CACert.crt", + "inhibitPolicyMapping1P12subCACert.crt", + "inhibitPolicyMapping1P12subsubCACert.crt", + "InvalidinhibitPolicyMappingTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitPolicyMapping1P12CACRL.crl", + "inhibitPolicyMapping1P12subCACRL.crl", + "inhibitPolicyMapping1P12subsubCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.11.4 Valid inhibitPolicyMapping Test4", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitPolicyMapping1P12CACert.crt", + "inhibitPolicyMapping1P12subCACert.crt", + "inhibitPolicyMapping1P12subsubCACert.crt", + "ValidinhibitPolicyMappingTest4EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitPolicyMapping1P12CACRL.crl", + "inhibitPolicyMapping1P12subCACRL.crl", + "inhibitPolicyMapping1P12subsubCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.11.5 Invalid inhibitPolicyMapping Test5", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitPolicyMapping5CACert.crt", + "inhibitPolicyMapping5subCACert.crt", + "inhibitPolicyMapping5subsubCACert.crt", + "inhibitPolicyMapping5subsubsubCACert.crt", + "InvalidinhibitPolicyMappingTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitPolicyMapping5CACRL.crl", + "inhibitPolicyMapping5subCACRL.crl", + "inhibitPolicyMapping5subsubCACRL.crl", + "inhibitPolicyMapping5subsubsubCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.11.6 Invalid inhibitPolicyMapping Test6", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitPolicyMapping1P12CACert.crt", + "inhibitPolicyMapping1P12subCAIPM5Cert.crt", + "inhibitPolicyMapping1P12subsubCAIPM5Cert.crt", + "InvalidinhibitPolicyMappingTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitPolicyMapping1P12CACRL.crl", + "inhibitPolicyMapping1P12subCAIPM5CRL.crl", + "inhibitPolicyMapping1P12subsubCAIPM5CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.11.7 Valid Self-Issued inhibitPolicyMapping Test7", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitPolicyMapping1P1CACert.crt", + "inhibitPolicyMapping1P1SelfIssuedCACert.crt", + "inhibitPolicyMapping1P1subCACert.crt", + "ValidSelfIssuedinhibitPolicyMappingTest7EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitPolicyMapping1P1CACRL.crl", + "inhibitPolicyMapping1P1subCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.11.8 Invalid Self-Issued inhibitPolicyMapping Test8", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitPolicyMapping1P1CACert.crt", + "inhibitPolicyMapping1P1SelfIssuedCACert.crt", + "inhibitPolicyMapping1P1subCACert.crt", + "inhibitPolicyMapping1P1subsubCACert.crt", + "InvalidSelfIssuedinhibitPolicyMappingTest8EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitPolicyMapping1P1CACRL.crl", + "inhibitPolicyMapping1P1subCACRL.crl", + "inhibitPolicyMapping1P1subsubCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.11.9 Invalid Self-Issued inhibitPolicyMapping Test9", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitPolicyMapping1P1CACert.crt", + "inhibitPolicyMapping1P1SelfIssuedCACert.crt", + "inhibitPolicyMapping1P1subCACert.crt", + "inhibitPolicyMapping1P1subsubCACert.crt", + "InvalidSelfIssuedinhibitPolicyMappingTest9EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitPolicyMapping1P1CACRL.crl", + "inhibitPolicyMapping1P1subCACRL.crl", + "inhibitPolicyMapping1P1subsubCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.11.10 Invalid Self-Issued inhibitPolicyMapping Test10", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitPolicyMapping1P1CACert.crt", + "inhibitPolicyMapping1P1SelfIssuedCACert.crt", + "inhibitPolicyMapping1P1subCACert.crt", + "inhibitPolicyMapping1P1SelfIssuedsubCACert.crt", + "InvalidSelfIssuedinhibitPolicyMappingTest10EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitPolicyMapping1P1CACRL.crl", + "inhibitPolicyMapping1P1subCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.11.11 Invalid Self-Issued inhibitPolicyMapping Test11", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitPolicyMapping1P1CACert.crt", + "inhibitPolicyMapping1P1SelfIssuedCACert.crt", + "inhibitPolicyMapping1P1subCACert.crt", + "inhibitPolicyMapping1P1SelfIssuedsubCACert.crt", + "InvalidSelfIssuedinhibitPolicyMappingTest11EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitPolicyMapping1P1CACRL.crl", + "inhibitPolicyMapping1P1subCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.12.1 Invalid inhibitAnyPolicy Test1", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitAnyPolicy0CACert.crt", + "InvalidinhibitAnyPolicyTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitAnyPolicy0CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.12.2 Valid inhibitAnyPolicy Test2", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitAnyPolicy0CACert.crt", + "ValidinhibitAnyPolicyTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitAnyPolicy0CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.12.3 inhibitAnyPolicy Test3 (Subpart 1)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitAnyPolicy1CACert.crt", + "inhibitAnyPolicy1subCA1Cert.crt", + "inhibitAnyPolicyTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitAnyPolicy1CACRL.crl", + "inhibitAnyPolicy1subCA1CRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.12.3 inhibitAnyPolicy Test3 (Subpart 2)", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitAnyPolicy1CACert.crt", + "inhibitAnyPolicy1subCA1Cert.crt", + "inhibitAnyPolicyTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitAnyPolicy1CACRL.crl", + "inhibitAnyPolicy1subCA1CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": true + }, + { + "Name": "4.12.4 Invalid inhibitAnyPolicy Test4", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitAnyPolicy1CACert.crt", + "inhibitAnyPolicy1subCA1Cert.crt", + "InvalidinhibitAnyPolicyTest4EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitAnyPolicy1CACRL.crl", + "inhibitAnyPolicy1subCA1CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.12.5 Invalid inhibitAnyPolicy Test5", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitAnyPolicy5CACert.crt", + "inhibitAnyPolicy5subCACert.crt", + "inhibitAnyPolicy5subsubCACert.crt", + "InvalidinhibitAnyPolicyTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitAnyPolicy5CACRL.crl", + "inhibitAnyPolicy5subCACRL.crl", + "inhibitAnyPolicy5subsubCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.12.6 Invalid inhibitAnyPolicy Test6", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitAnyPolicy1CACert.crt", + "inhibitAnyPolicy1subCAIAP5Cert.crt", + "InvalidinhibitAnyPolicyTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitAnyPolicy1CACRL.crl", + "inhibitAnyPolicy1subCAIAP5CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.12.7 Valid Self-Issued inhibitAnyPolicy Test7", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitAnyPolicy1CACert.crt", + "inhibitAnyPolicy1SelfIssuedCACert.crt", + "inhibitAnyPolicy1subCA2Cert.crt", + "ValidSelfIssuedinhibitAnyPolicyTest7EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitAnyPolicy1CACRL.crl", + "inhibitAnyPolicy1subCA2CRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.12.8 Invalid Self-Issued inhibitAnyPolicy Test8", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitAnyPolicy1CACert.crt", + "inhibitAnyPolicy1SelfIssuedCACert.crt", + "inhibitAnyPolicy1subCA2Cert.crt", + "inhibitAnyPolicy1subsubCA2Cert.crt", + "InvalidSelfIssuedinhibitAnyPolicyTest8EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitAnyPolicy1CACRL.crl", + "inhibitAnyPolicy1subCA2CRL.crl", + "inhibitAnyPolicy1subsubCA2CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.12.9 Valid Self-Issued inhibitAnyPolicy Test9", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitAnyPolicy1CACert.crt", + "inhibitAnyPolicy1SelfIssuedCACert.crt", + "inhibitAnyPolicy1subCA2Cert.crt", + "inhibitAnyPolicy1SelfIssuedsubCA2Cert.crt", + "ValidSelfIssuedinhibitAnyPolicyTest9EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitAnyPolicy1CACRL.crl", + "inhibitAnyPolicy1subCA2CRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.12.10 Invalid Self-Issued inhibitAnyPolicy Test10", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "inhibitAnyPolicy1CACert.crt", + "inhibitAnyPolicy1SelfIssuedCACert.crt", + "inhibitAnyPolicy1subCA2Cert.crt", + "InvalidSelfIssuedinhibitAnyPolicyTest10EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "inhibitAnyPolicy1CACRL.crl", + "inhibitAnyPolicy1subCA2CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.1 Valid DN nameConstraints Test1", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN1CACert.crt", + "ValidDNnameConstraintsTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN1CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.2 Invalid DN nameConstraints Test2", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN1CACert.crt", + "InvalidDNnameConstraintsTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN1CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.3 Invalid DN nameConstraints Test3", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN1CACert.crt", + "InvalidDNnameConstraintsTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN1CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.4 Valid DN nameConstraints Test4", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN1CACert.crt", + "ValidDNnameConstraintsTest4EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN1CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.5 Valid DN nameConstraints Test5", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN2CACert.crt", + "ValidDNnameConstraintsTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN2CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.6 Valid DN nameConstraints Test6", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN3CACert.crt", + "ValidDNnameConstraintsTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN3CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.7 Invalid DN nameConstraints Test7", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN3CACert.crt", + "InvalidDNnameConstraintsTest7EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN3CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.8 Invalid DN nameConstraints Test8", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN4CACert.crt", + "InvalidDNnameConstraintsTest8EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN4CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.9 Invalid DN nameConstraints Test9", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN4CACert.crt", + "InvalidDNnameConstraintsTest9EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN4CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.10 Invalid DN nameConstraints Test10", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN5CACert.crt", + "InvalidDNnameConstraintsTest10EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN5CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.11 Valid DN nameConstraints Test11", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN5CACert.crt", + "ValidDNnameConstraintsTest11EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN5CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.12 Invalid DN nameConstraints Test12", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN1CACert.crt", + "nameConstraintsDN1subCA1Cert.crt", + "InvalidDNnameConstraintsTest12EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN1CACRL.crl", + "nameConstraintsDN1subCA1CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.13 Invalid DN nameConstraints Test13", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN1CACert.crt", + "nameConstraintsDN1subCA2Cert.crt", + "InvalidDNnameConstraintsTest13EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN1CACRL.crl", + "nameConstraintsDN1subCA2CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.14 Valid DN nameConstraints Test14", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN1CACert.crt", + "nameConstraintsDN1subCA2Cert.crt", + "ValidDNnameConstraintsTest14EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN1CACRL.crl", + "nameConstraintsDN1subCA2CRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.15 Invalid DN nameConstraints Test15", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN3CACert.crt", + "nameConstraintsDN3subCA1Cert.crt", + "InvalidDNnameConstraintsTest15EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN3CACRL.crl", + "nameConstraintsDN3subCA1CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.16 Invalid DN nameConstraints Test16", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN3CACert.crt", + "nameConstraintsDN3subCA1Cert.crt", + "InvalidDNnameConstraintsTest16EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN3CACRL.crl", + "nameConstraintsDN3subCA1CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.17 Invalid DN nameConstraints Test17", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN3CACert.crt", + "nameConstraintsDN3subCA2Cert.crt", + "InvalidDNnameConstraintsTest17EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN3CACRL.crl", + "nameConstraintsDN3subCA2CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.18 Valid DN nameConstraints Test18", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN3CACert.crt", + "nameConstraintsDN3subCA2Cert.crt", + "ValidDNnameConstraintsTest18EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN3CACRL.crl", + "nameConstraintsDN3subCA2CRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.19 Valid Self-Issued DN nameConstraints Test19", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN1CACert.crt", + "nameConstraintsDN1SelfIssuedCACert.crt", + "ValidDNnameConstraintsTest19EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN1CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.20 Invalid Self-Issued DN nameConstraints Test20", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN1CACert.crt", + "InvalidDNnameConstraintsTest20EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN1CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.21 Valid RFC822 nameConstraints Test21", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsRFC822CA1Cert.crt", + "ValidRFC822nameConstraintsTest21EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsRFC822CA1CRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.22 Invalid RFC822 nameConstraints Test22", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsRFC822CA1Cert.crt", + "InvalidRFC822nameConstraintsTest22EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsRFC822CA1CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.23 Valid RFC822 nameConstraints Test23", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsRFC822CA2Cert.crt", + "ValidRFC822nameConstraintsTest23EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsRFC822CA2CRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.24 Invalid RFC822 nameConstraints Test24", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsRFC822CA2Cert.crt", + "InvalidRFC822nameConstraintsTest24EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsRFC822CA2CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.25 Valid RFC822 nameConstraints Test25", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsRFC822CA3Cert.crt", + "ValidRFC822nameConstraintsTest25EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsRFC822CA3CRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.26 Invalid RFC822 nameConstraints Test26", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsRFC822CA3Cert.crt", + "InvalidRFC822nameConstraintsTest26EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsRFC822CA3CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.27 Valid DN and RFC822 nameConstraints Test27", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN1CACert.crt", + "nameConstraintsDN1subCA3Cert.crt", + "ValidDNandRFC822nameConstraintsTest27EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN1CACRL.crl", + "nameConstraintsDN1subCA3CRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.28 Invalid DN and RFC822 nameConstraints Test28", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN1CACert.crt", + "nameConstraintsDN1subCA3Cert.crt", + "InvalidDNandRFC822nameConstraintsTest28EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN1CACRL.crl", + "nameConstraintsDN1subCA3CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.29 Invalid DN and RFC822 nameConstraints Test29", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDN1CACert.crt", + "nameConstraintsDN1subCA3Cert.crt", + "InvalidDNandRFC822nameConstraintsTest29EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDN1CACRL.crl", + "nameConstraintsDN1subCA3CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.30 Valid DNS nameConstraints Test30", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDNS1CACert.crt", + "ValidDNSnameConstraintsTest30EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDNS1CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.31 Invalid DNS nameConstraints Test31", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDNS1CACert.crt", + "InvalidDNSnameConstraintsTest31EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDNS1CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.32 Valid DNS nameConstraints Test32", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDNS2CACert.crt", + "ValidDNSnameConstraintsTest32EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDNS2CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.33 Invalid DNS nameConstraints Test33", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDNS2CACert.crt", + "InvalidDNSnameConstraintsTest33EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDNS2CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.34 Valid URI nameConstraints Test34", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsURI1CACert.crt", + "ValidURInameConstraintsTest34EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsURI1CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.35 Invalid URI nameConstraints Test35", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsURI1CACert.crt", + "InvalidURInameConstraintsTest35EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsURI1CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.36 Valid URI nameConstraints Test36", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsURI2CACert.crt", + "ValidURInameConstraintsTest36EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsURI2CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.37 Invalid URI nameConstraints Test37", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsURI2CACert.crt", + "InvalidURInameConstraintsTest37EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsURI2CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.13.38 Invalid DNS nameConstraints Test38", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "nameConstraintsDNS1CACert.crt", + "InvalidDNSnameConstraintsTest38EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "nameConstraintsDNS1CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.1 Valid distributionPoint Test1", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "distributionPoint1CACert.crt", + "ValiddistributionPointTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "distributionPoint1CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.2 Invalid distributionPoint Test2", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "distributionPoint1CACert.crt", + "InvaliddistributionPointTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "distributionPoint1CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.3 Invalid distributionPoint Test3", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "distributionPoint1CACert.crt", + "InvaliddistributionPointTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "distributionPoint1CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.4 Valid distributionPoint Test4", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "distributionPoint1CACert.crt", + "ValiddistributionPointTest4EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "distributionPoint1CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.5 Valid distributionPoint Test5", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "distributionPoint2CACert.crt", + "ValiddistributionPointTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "distributionPoint2CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.6 Invalid distributionPoint Test6", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "distributionPoint2CACert.crt", + "InvaliddistributionPointTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "distributionPoint2CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.7 Valid distributionPoint Test7", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "distributionPoint2CACert.crt", + "ValiddistributionPointTest7EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "distributionPoint2CACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.8 Invalid distributionPoint Test8", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "distributionPoint2CACert.crt", + "InvaliddistributionPointTest8EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "distributionPoint2CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.9 Invalid distributionPoint Test9", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "distributionPoint2CACert.crt", + "InvaliddistributionPointTest9EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "distributionPoint2CACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.10 Valid No issuingDistributionPoint Test10", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "NoissuingDistributionPointCACert.crt", + "ValidNoissuingDistributionPointTest10EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "NoissuingDistributionPointCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.11 Invalid onlyContainsUserCerts CRL Test11", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "onlyContainsUserCertsCACert.crt", + "InvalidonlyContainsUserCertsTest11EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "onlyContainsUserCertsCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.12 Invalid onlyContainsCACerts CRL Test12", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "onlyContainsCACertsCACert.crt", + "InvalidonlyContainsCACertsTest12EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "onlyContainsCACertsCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.13 Valid onlyContainsCACerts CRL Test13", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "onlyContainsCACertsCACert.crt", + "ValidonlyContainsCACertsTest13EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "onlyContainsCACertsCACRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.14 Invalid onlyContainsAttributeCerts Test14", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "onlyContainsAttributeCertsCACert.crt", + "InvalidonlyContainsAttributeCertsTest14EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "onlyContainsAttributeCertsCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.15 Invalid onlySomeReasons Test15", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "onlySomeReasonsCA1Cert.crt", + "InvalidonlySomeReasonsTest15EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "onlySomeReasonsCA1compromiseCRL.crl", + "onlySomeReasonsCA1otherreasonsCRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.16 Invalid onlySomeReasons Test16", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "onlySomeReasonsCA1Cert.crt", + "InvalidonlySomeReasonsTest16EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "onlySomeReasonsCA1compromiseCRL.crl", + "onlySomeReasonsCA1otherreasonsCRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.17 Invalid onlySomeReasons Test17", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "onlySomeReasonsCA2Cert.crt", + "InvalidonlySomeReasonsTest17EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "onlySomeReasonsCA2CRL1.crl", + "onlySomeReasonsCA2CRL2.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.18 Valid onlySomeReasons Test18", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "onlySomeReasonsCA3Cert.crt", + "ValidonlySomeReasonsTest18EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "onlySomeReasonsCA3compromiseCRL.crl", + "onlySomeReasonsCA3otherreasonsCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.19 Valid onlySomeReasons Test19", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "onlySomeReasonsCA4Cert.crt", + "ValidonlySomeReasonsTest19EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "onlySomeReasonsCA4compromiseCRL.crl", + "onlySomeReasonsCA4otherreasonsCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.20 Invalid onlySomeReasons Test20", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "onlySomeReasonsCA4Cert.crt", + "InvalidonlySomeReasonsTest20EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "onlySomeReasonsCA4compromiseCRL.crl", + "onlySomeReasonsCA4otherreasonsCRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.21 Invalid onlySomeReasons Test21", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "onlySomeReasonsCA4Cert.crt", + "InvalidonlySomeReasonsTest21EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "onlySomeReasonsCA4compromiseCRL.crl", + "onlySomeReasonsCA4otherreasonsCRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.22 Valid IDP with indirectCRL Test22", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "indirectCRLCA1Cert.crt", + "ValidIDPwithindirectCRLTest22EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "indirectCRLCA1CRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.23 Invalid IDP with indirectCRL Test23", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "indirectCRLCA1Cert.crt", + "InvalidIDPwithindirectCRLTest23EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "indirectCRLCA1CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.24 Valid IDP with indirectCRL Test24", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "indirectCRLCA2Cert.crt", + "indirectCRLCA1Cert.crt", + "ValidIDPwithindirectCRLTest24EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "indirectCRLCA1CRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.25 Valid IDP with indirectCRL Test25", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "indirectCRLCA2Cert.crt", + "indirectCRLCA1Cert.crt", + "ValidIDPwithindirectCRLTest25EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "indirectCRLCA1CRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.26 Invalid IDP with indirectCRL Test26", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "indirectCRLCA2Cert.crt", + "indirectCRLCA1Cert.crt", + "InvalidIDPwithindirectCRLTest26EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "indirectCRLCA1CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.27 Invalid cRLIssuer Test27", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "indirectCRLCA2Cert.crt", + "GoodCACert.crt", + "InvalidcRLIssuerTest27EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "GoodCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.28 Valid cRLIssuer Test28", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "indirectCRLCA3Cert.crt", + "indirectCRLCA3cRLIssuerCert.crt", + "ValidcRLIssuerTest28EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "indirectCRLCA3CRL.crl", + "indirectCRLCA3cRLIssuerCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.29 Valid cRLIssuer Test29", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "indirectCRLCA3Cert.crt", + "indirectCRLCA3cRLIssuerCert.crt", + "ValidcRLIssuerTest29EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "indirectCRLCA3CRL.crl", + "indirectCRLCA3cRLIssuerCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.30 Valid cRLIssuer Test30", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "indirectCRLCA4Cert.crt", + "indirectCRLCA4cRLIssuerCert.crt", + "ValidcRLIssuerTest30EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "indirectCRLCA4cRLIssuerCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.31 Invalid cRLIssuer Test31", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "indirectCRLCA5Cert.crt", + "indirectCRLCA6Cert.crt", + "InvalidcRLIssuerTest31EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "indirectCRLCA5CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.32 Invalid cRLIssuer Test32", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "indirectCRLCA5Cert.crt", + "indirectCRLCA6Cert.crt", + "InvalidcRLIssuerTest32EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "indirectCRLCA5CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.33 Valid cRLIssuer Test33", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "indirectCRLCA5Cert.crt", + "indirectCRLCA6Cert.crt", + "ValidcRLIssuerTest33EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "indirectCRLCA5CRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.34 Invalid cRLIssuer Test34", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "indirectCRLCA5Cert.crt", + "InvalidcRLIssuerTest34EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "indirectCRLCA5CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.14.35 Invalid cRLIssuer Test35", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "indirectCRLCA5Cert.crt", + "InvalidcRLIssuerTest35EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "indirectCRLCA5CRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.15.1 Invalid deltaCRLIndicator No Base Test1", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "deltaCRLIndicatorNoBaseCACert.crt", + "InvaliddeltaCRLIndicatorNoBaseTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "deltaCRLIndicatorNoBaseCACRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.15.2 Valid delta-CRL Test2", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "deltaCRLCA1Cert.crt", + "ValiddeltaCRLTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "deltaCRLCA1CRL.crl", + "deltaCRLCA1deltaCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.15.3 Invalid delta-CRL Test3", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "deltaCRLCA1Cert.crt", + "InvaliddeltaCRLTest3EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "deltaCRLCA1CRL.crl", + "deltaCRLCA1deltaCRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.15.4 Invalid delta-CRL Test4", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "deltaCRLCA1Cert.crt", + "InvaliddeltaCRLTest4EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "deltaCRLCA1CRL.crl", + "deltaCRLCA1deltaCRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.15.5 Valid delta-CRL Test5", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "deltaCRLCA1Cert.crt", + "ValiddeltaCRLTest5EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "deltaCRLCA1CRL.crl", + "deltaCRLCA1deltaCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.15.6 Invalid delta-CRL Test6", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "deltaCRLCA1Cert.crt", + "InvaliddeltaCRLTest6EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "deltaCRLCA1CRL.crl", + "deltaCRLCA1deltaCRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.15.7 Valid delta-CRL Test7", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "deltaCRLCA1Cert.crt", + "ValiddeltaCRLTest7EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "deltaCRLCA1CRL.crl", + "deltaCRLCA1deltaCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.15.8 Valid delta-CRL Test8", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "deltaCRLCA2Cert.crt", + "ValiddeltaCRLTest8EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "deltaCRLCA2CRL.crl", + "deltaCRLCA2deltaCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.15.9 Invalid delta-CRL Test9", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "deltaCRLCA2Cert.crt", + "InvaliddeltaCRLTest9EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "deltaCRLCA2CRL.crl", + "deltaCRLCA2deltaCRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.15.10 Invalid delta-CRL Test10", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "deltaCRLCA3Cert.crt", + "InvaliddeltaCRLTest10EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl", + "deltaCRLCA3CRL.crl", + "deltaCRLCA3deltaCRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.16.1 Valid Unknown Not Critical Certificate Extension Test1", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "ValidUnknownNotCriticalCertificateExtensionTest1EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl" + ], + "ShouldValidate": true, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + }, + { + "Name": "4.16.2 Invalid Unknown Critical Certificate Extension Test2", + "CertPath": [ + "TrustAnchorRootCertificate.crt", + "InvalidUnknownCriticalCertificateExtensionTest2EE.crt" + ], + "CRLPath": [ + "TrustAnchorRootCRL.crl" + ], + "ShouldValidate": false, + "InitialPolicySet": [ + "anyPolicy" + ], + "InitialPolicyMappingInhibit": false, + "InitialExplicitPolicy": false, + "InitialAnyPolicyInhibit": false + } +] \ No newline at end of file diff --git a/src/crypto/x509/testdata/policy_intermediate.pem b/src/crypto/x509/testdata/policy_intermediate.pem new file mode 100644 index 00000000000000..759deb4c43a646 --- /dev/null +++ b/src/crypto/x509/testdata/policy_intermediate.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBqjCCAVGgAwIBAgIBAjAKBggqhkjOPQQDAjAWMRQwEgYDVQQDEwtQb2xpY3kg +Um9vdDAgFw0wMDAxMDEwMDAwMDBaGA8yMTAwMDEwMTAwMDAwMFowHjEcMBoGA1UE +AxMTUG9saWN5IEludGVybWVkaWF0ZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BOI6fKiM3jFLkLyAn88cvlw4SwxuygRjopP3FFBKHyUQvh3VVvfqSpSCSmp50Qia +jQ6Dg7CTpVZVVH+bguT7JTCjgYUwgYIwDgYDVR0PAQH/BAQDAgIEMBMGA1UdJQQM +MAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJDS9/4O7qhr +CIRhwsXrPVBagG2uMCsGA1UdIAQkMCIwDwYNKoZIhvcSBAGEtwkCATAPBg0qhkiG +9xIEAYS3CQICMAoGCCqGSM49BAMCA0cAMEQCIFN2ZtknXQ9vz23qD1ecprC9iIo7 +j/SI42Ub64qZQaraAiA+CRCWJz/l+NQ1+TPWYDDWY6Wh2L9Wbddh1Nj5KJEkhQ== +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_intermediate_any.pem b/src/crypto/x509/testdata/policy_intermediate_any.pem new file mode 100644 index 00000000000000..0931964f520b89 --- /dev/null +++ b/src/crypto/x509/testdata/policy_intermediate_any.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBkDCCATWgAwIBAgIBAjAKBggqhkjOPQQDAjAWMRQwEgYDVQQDEwtQb2xpY3kg +Um9vdDAgFw0wMDAxMDEwMDAwMDBaGA8yMTAwMDEwMTAwMDAwMFowHjEcMBoGA1UE +AxMTUG9saWN5IEludGVybWVkaWF0ZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BOI6fKiM3jFLkLyAn88cvlw4SwxuygRjopP3FFBKHyUQvh3VVvfqSpSCSmp50Qia +jQ6Dg7CTpVZVVH+bguT7JTCjajBoMA4GA1UdDwEB/wQEAwICBDATBgNVHSUEDDAK +BggrBgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSQ0vf+Du6oawiE +YcLF6z1QWoBtrjARBgNVHSAECjAIMAYGBFUdIAAwCgYIKoZIzj0EAwIDSQAwRgIh +AJbyXshUwjsFCiqrJkg91GzJdhZZ+3WXOekCJgi8uEESAiEAhv4sEE0wRRqgHDjl +vIt26IELfFE2Z/FBF3ihGmi6NoI= +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_intermediate_duplicate.pem b/src/crypto/x509/testdata/policy_intermediate_duplicate.pem new file mode 100644 index 00000000000000..0eafe8d86a8804 --- /dev/null +++ b/src/crypto/x509/testdata/policy_intermediate_duplicate.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBvDCCAWKgAwIBAgIBAjAKBggqhkjOPQQDAjAWMRQwEgYDVQQDEwtQb2xpY3kg +Um9vdDAgFw0wMDAxMDEwMDAwMDBaGA8yMTAwMDEwMTAwMDAwMFowHjEcMBoGA1UE +AxMTUG9saWN5IEludGVybWVkaWF0ZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BOI6fKiM3jFLkLyAn88cvlw4SwxuygRjopP3FFBKHyUQvh3VVvfqSpSCSmp50Qia +jQ6Dg7CTpVZVVH+bguT7JTCjgZYwgZMwDgYDVR0PAQH/BAQDAgIEMBMGA1UdJQQM +MAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJDS9/4O7qhr +CIRhwsXrPVBagG2uMDwGA1UdIAQ1MDMwDwYNKoZIhvcSBAGEtwkCATAPBg0qhkiG +9xIEAYS3CQICMA8GDSqGSIb3EgQBhLcJAgIwCgYIKoZIzj0EAwIDSAAwRQIgUpG6 +FUeWrC62BtTPHiSlWBdnLWUYH0llS6uYUkpJFJECIQCWfhoZYXvHdMhgBDSI/vzY +Sw4uNdcMxrC2kP6lIioUSw== +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_intermediate_invalid.pem b/src/crypto/x509/testdata/policy_intermediate_invalid.pem new file mode 100644 index 00000000000000..11c95afcea48df --- /dev/null +++ b/src/crypto/x509/testdata/policy_intermediate_invalid.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBjDCCATKgAwIBAgIBAjAKBggqhkjOPQQDAjAWMRQwEgYDVQQDEwtQb2xpY3kg +Um9vdDAgFw0wMDAxMDEwMDAwMDBaGA8yMTAwMDEwMTAwMDAwMFowHjEcMBoGA1UE +AxMTUG9saWN5IEludGVybWVkaWF0ZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BOI6fKiM3jFLkLyAn88cvlw4SwxuygRjopP3FFBKHyUQvh3VVvfqSpSCSmp50Qia +jQ6Dg7CTpVZVVH+bguT7JTCjZzBlMA4GA1UdDwEB/wQEAwICBDATBgNVHSUEDDAK +BggrBgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSQ0vf+Du6oawiE +YcLF6z1QWoBtrjAOBgNVHSAEB0lOVkFMSUQwCgYIKoZIzj0EAwIDSAAwRQIgS2uK +cYlZ1bxeqgMy3X0Sfi0arAnqpePsAqAeEf+HJHQCIQDwfCnXrWyHET9lM/gJSkfN +j/JRJvJELDrAMVewCxZWKA== +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_intermediate_mapped.pem b/src/crypto/x509/testdata/policy_intermediate_mapped.pem new file mode 100644 index 00000000000000..fa45e604b43af1 --- /dev/null +++ b/src/crypto/x509/testdata/policy_intermediate_mapped.pem @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICrjCCAlSgAwIBAgIBAjAKBggqhkjOPQQDAjAWMRQwEgYDVQQDEwtQb2xpY3kg +Um9vdDAgFw0wMDAxMDEwMDAwMDBaGA8yMTAwMDEwMTAwMDAwMFowHjEcMBoGA1UE +AxMTUG9saWN5IEludGVybWVkaWF0ZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BOI6fKiM3jFLkLyAn88cvlw4SwxuygRjopP3FFBKHyUQvh3VVvfqSpSCSmp50Qia +jQ6Dg7CTpVZVVH+bguT7JTCjggGHMIIBgzAOBgNVHQ8BAf8EBAMCAgQwEwYDVR0l +BAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUkNL3/g7u +qGsIhGHCxes9UFqAba4wXgYDVR0gBFcwVTAPBg0qhkiG9xIEAYS3CQIBMA8GDSqG +SIb3EgQBhLcJAgIwDwYNKoZIhvcSBAGEtwkCAzAPBg0qhkiG9xIEAYS3CQIEMA8G +DSqGSIb3EgQBhLcJAgUwgcsGA1UdIQSBwzCBwDAeBg0qhkiG9xIEAYS3CQIDBg0q +hkiG9xIEAYS3CQIBMB4GDSqGSIb3EgQBhLcJAgMGDSqGSIb3EgQBhLcJAgIwHgYN +KoZIhvcSBAGEtwkCBAYNKoZIhvcSBAGEtwkCBDAeBg0qhkiG9xIEAYS3CQIEBg0q +hkiG9xIEAYS3CQIFMB4GDSqGSIb3EgQBhLcJAgUGDSqGSIb3EgQBhLcJAgQwHgYN +KoZIhvcSBAGEtwkCBQYNKoZIhvcSBAGEtwkCBTAKBggqhkjOPQQDAgNIADBFAiAe +Ah2vJMZsW/RV35mM7b7/NjsjScjPEIxfDJu49inNXQIhANmGBqyWUogh/gXyVB0/ +IfDro27pANW3R02A+zH34q5k +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_intermediate_mapped_any.pem b/src/crypto/x509/testdata/policy_intermediate_mapped_any.pem new file mode 100644 index 00000000000000..ae47bf45ceae6a --- /dev/null +++ b/src/crypto/x509/testdata/policy_intermediate_mapped_any.pem @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICYjCCAgegAwIBAgIBAjAKBggqhkjOPQQDAjAWMRQwEgYDVQQDEwtQb2xpY3kg +Um9vdDAgFw0wMDAxMDEwMDAwMDBaGA8yMTAwMDEwMTAwMDAwMFowHjEcMBoGA1UE +AxMTUG9saWN5IEludGVybWVkaWF0ZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BOI6fKiM3jFLkLyAn88cvlw4SwxuygRjopP3FFBKHyUQvh3VVvfqSpSCSmp50Qia +jQ6Dg7CTpVZVVH+bguT7JTCjggE6MIIBNjAOBgNVHQ8BAf8EBAMCAgQwEwYDVR0l +BAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUkNL3/g7u +qGsIhGHCxes9UFqAba4wEQYDVR0gBAowCDAGBgRVHSAAMIHLBgNVHSEEgcMwgcAw +HgYNKoZIhvcSBAGEtwkCAwYNKoZIhvcSBAGEtwkCATAeBg0qhkiG9xIEAYS3CQID +Bg0qhkiG9xIEAYS3CQICMB4GDSqGSIb3EgQBhLcJAgQGDSqGSIb3EgQBhLcJAgQw +HgYNKoZIhvcSBAGEtwkCBAYNKoZIhvcSBAGEtwkCBTAeBg0qhkiG9xIEAYS3CQIF +Bg0qhkiG9xIEAYS3CQIEMB4GDSqGSIb3EgQBhLcJAgUGDSqGSIb3EgQBhLcJAgUw +CgYIKoZIzj0EAwIDSQAwRgIhAIOx3GL5xlldQGdTLIvTTAvczm8wiYHzZDAif2yj +wAjEAiEAg4K02kTYX9x7PC/u1PYdwvo+LVbnGbO6AN6U3K2d7gs= +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_intermediate_mapped_oid3.pem b/src/crypto/x509/testdata/policy_intermediate_mapped_oid3.pem new file mode 100644 index 00000000000000..c04a38a48f1298 --- /dev/null +++ b/src/crypto/x509/testdata/policy_intermediate_mapped_oid3.pem @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICajCCAhCgAwIBAgIBAjAKBggqhkjOPQQDAjAWMRQwEgYDVQQDEwtQb2xpY3kg +Um9vdDAgFw0wMDAxMDEwMDAwMDBaGA8yMTAwMDEwMTAwMDAwMFowHjEcMBoGA1UE +AxMTUG9saWN5IEludGVybWVkaWF0ZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BOI6fKiM3jFLkLyAn88cvlw4SwxuygRjopP3FFBKHyUQvh3VVvfqSpSCSmp50Qia +jQ6Dg7CTpVZVVH+bguT7JTCjggFDMIIBPzAOBgNVHQ8BAf8EBAMCAgQwEwYDVR0l +BAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUkNL3/g7u +qGsIhGHCxes9UFqAba4wGgYDVR0gBBMwETAPBg0qhkiG9xIEAYS3CQIDMIHLBgNV +HSEEgcMwgcAwHgYNKoZIhvcSBAGEtwkCAwYNKoZIhvcSBAGEtwkCATAeBg0qhkiG +9xIEAYS3CQIDBg0qhkiG9xIEAYS3CQICMB4GDSqGSIb3EgQBhLcJAgQGDSqGSIb3 +EgQBhLcJAgQwHgYNKoZIhvcSBAGEtwkCBAYNKoZIhvcSBAGEtwkCBTAeBg0qhkiG +9xIEAYS3CQIFBg0qhkiG9xIEAYS3CQIEMB4GDSqGSIb3EgQBhLcJAgUGDSqGSIb3 +EgQBhLcJAgUwCgYIKoZIzj0EAwIDSAAwRQIhAK0bRaGgd5qQlX+zTw3IUynFHxfk +zRbZagnTzjYtkNNmAiBJ2kOnvRdW930eHAwZPGpc1Hn5hMSOQdUhNZ3XZDASkQ== +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_intermediate_require.pem b/src/crypto/x509/testdata/policy_intermediate_require.pem new file mode 100644 index 00000000000000..5cf5d5bfe62331 --- /dev/null +++ b/src/crypto/x509/testdata/policy_intermediate_require.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBuDCCAV+gAwIBAgIBAjAKBggqhkjOPQQDAjAWMRQwEgYDVQQDEwtQb2xpY3kg +Um9vdDAgFw0wMDAxMDEwMDAwMDBaGA8yMTAwMDEwMTAwMDAwMFowHjEcMBoGA1UE +AxMTUG9saWN5IEludGVybWVkaWF0ZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BOI6fKiM3jFLkLyAn88cvlw4SwxuygRjopP3FFBKHyUQvh3VVvfqSpSCSmp50Qia +jQ6Dg7CTpVZVVH+bguT7JTCjgZMwgZAwDgYDVR0PAQH/BAQDAgIEMBMGA1UdJQQM +MAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJDS9/4O7qhr +CIRhwsXrPVBagG2uMCsGA1UdIAQkMCIwDwYNKoZIhvcSBAGEtwkCATAPBg0qhkiG +9xIEAYS3CQICMAwGA1UdJAQFMAOAAQAwCgYIKoZIzj0EAwIDRwAwRAIgbPUZ9ezH +SgTqom7VLPOvrQQXwy3b/ijSobs7+SOouKMCIDaqcb9143BG005etqeTvlgUyOGF +GQDWhiW8bizH+KEl +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_intermediate_require1.pem b/src/crypto/x509/testdata/policy_intermediate_require1.pem new file mode 100644 index 00000000000000..7087404b3f1165 --- /dev/null +++ b/src/crypto/x509/testdata/policy_intermediate_require1.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBujCCAV+gAwIBAgIBAjAKBggqhkjOPQQDAjAWMRQwEgYDVQQDEwtQb2xpY3kg +Um9vdDAgFw0wMDAxMDEwMDAwMDBaGA8yMTAwMDEwMTAwMDAwMFowHjEcMBoGA1UE +AxMTUG9saWN5IEludGVybWVkaWF0ZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BOI6fKiM3jFLkLyAn88cvlw4SwxuygRjopP3FFBKHyUQvh3VVvfqSpSCSmp50Qia +jQ6Dg7CTpVZVVH+bguT7JTCjgZMwgZAwDgYDVR0PAQH/BAQDAgIEMBMGA1UdJQQM +MAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJDS9/4O7qhr +CIRhwsXrPVBagG2uMCsGA1UdIAQkMCIwDwYNKoZIhvcSBAGEtwkCATAPBg0qhkiG +9xIEAYS3CQICMAwGA1UdJAQFMAOAAQEwCgYIKoZIzj0EAwIDSQAwRgIhAIAwvhHB +GQDN5YXlidd+n3OT/SqoeXfp7RiEonBnCkW4AiEA+iFc47EOBchHb+Gy0gg8F9Po +RnlpoulWDfbDwx9r4lc= +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_intermediate_require2.pem b/src/crypto/x509/testdata/policy_intermediate_require2.pem new file mode 100644 index 00000000000000..350f41919879a3 --- /dev/null +++ b/src/crypto/x509/testdata/policy_intermediate_require2.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBuTCCAV+gAwIBAgIBAjAKBggqhkjOPQQDAjAWMRQwEgYDVQQDEwtQb2xpY3kg +Um9vdDAgFw0wMDAxMDEwMDAwMDBaGA8yMTAwMDEwMTAwMDAwMFowHjEcMBoGA1UE +AxMTUG9saWN5IEludGVybWVkaWF0ZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BOI6fKiM3jFLkLyAn88cvlw4SwxuygRjopP3FFBKHyUQvh3VVvfqSpSCSmp50Qia +jQ6Dg7CTpVZVVH+bguT7JTCjgZMwgZAwDgYDVR0PAQH/BAQDAgIEMBMGA1UdJQQM +MAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJDS9/4O7qhr +CIRhwsXrPVBagG2uMCsGA1UdIAQkMCIwDwYNKoZIhvcSBAGEtwkCATAPBg0qhkiG +9xIEAYS3CQICMAwGA1UdJAQFMAOAAQIwCgYIKoZIzj0EAwIDSAAwRQIgOpliSKKA ++wy/auQnKKl+wwtn/hGw6eZXgIOtFgDmyMYCIQC84zoJL87AE64gsrdX4XSHq6lb +WhZQp9ZnDaNu88SQLQ== +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_intermediate_require_duplicate.pem b/src/crypto/x509/testdata/policy_intermediate_require_duplicate.pem new file mode 100644 index 00000000000000..733087af91c17c --- /dev/null +++ b/src/crypto/x509/testdata/policy_intermediate_require_duplicate.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIByjCCAXCgAwIBAgIBAjAKBggqhkjOPQQDAjAWMRQwEgYDVQQDEwtQb2xpY3kg +Um9vdDAgFw0wMDAxMDEwMDAwMDBaGA8yMTAwMDEwMTAwMDAwMFowHjEcMBoGA1UE +AxMTUG9saWN5IEludGVybWVkaWF0ZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BOI6fKiM3jFLkLyAn88cvlw4SwxuygRjopP3FFBKHyUQvh3VVvfqSpSCSmp50Qia +jQ6Dg7CTpVZVVH+bguT7JTCjgaQwgaEwDgYDVR0PAQH/BAQDAgIEMBMGA1UdJQQM +MAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJDS9/4O7qhr +CIRhwsXrPVBagG2uMDwGA1UdIAQ1MDMwDwYNKoZIhvcSBAGEtwkCATAPBg0qhkiG +9xIEAYS3CQICMA8GDSqGSIb3EgQBhLcJAgIwDAYDVR0kBAUwA4ABADAKBggqhkjO +PQQDAgNIADBFAiA2GxzMRYYo7NNq8u/ZvffXkCj/phqXQ8I64tEDd0X8pgIhAOJJ +e+dzzf4vbWfMlYkOQ4kf6ei5Zf+J2PL6VrqVrHQa +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_intermediate_require_no_policies.pem b/src/crypto/x509/testdata/policy_intermediate_require_no_policies.pem new file mode 100644 index 00000000000000..1e81e0c1165d8f --- /dev/null +++ b/src/crypto/x509/testdata/policy_intermediate_require_no_policies.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBizCCATCgAwIBAgIBAjAKBggqhkjOPQQDAjAWMRQwEgYDVQQDEwtQb2xpY3kg +Um9vdDAgFw0wMDAxMDEwMDAwMDBaGA8yMTAwMDEwMTAwMDAwMFowHjEcMBoGA1UE +AxMTUG9saWN5IEludGVybWVkaWF0ZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BOI6fKiM3jFLkLyAn88cvlw4SwxuygRjopP3FFBKHyUQvh3VVvfqSpSCSmp50Qia +jQ6Dg7CTpVZVVH+bguT7JTCjZTBjMA4GA1UdDwEB/wQEAwICBDATBgNVHSUEDDAK +BggrBgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSQ0vf+Du6oawiE +YcLF6z1QWoBtrjAMBgNVHSQEBTADgAEAMAoGCCqGSM49BAMCA0kAMEYCIQDJYPgf +50fFDVho5TFeqkNVONx0ArVNgULPB27yPDHLrwIhAN+eua6oM4Q/O0jUESQ4VAKt +ts7ZCquTZbvgRgyqtjuT +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_leaf.pem b/src/crypto/x509/testdata/policy_leaf.pem new file mode 100644 index 00000000000000..fb70306c8a610f --- /dev/null +++ b/src/crypto/x509/testdata/policy_leaf.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBpzCCAU2gAwIBAgIBAzAKBggqhkjOPQQDAjAeMRwwGgYDVQQDExNQb2xpY3kg +SW50ZXJtZWRpYXRlMCAXDTAwMDEwMTAwMDAwMFoYDzIxMDAwMTAxMDAwMDAwWjAa +MRgwFgYDVQQDEw93d3cuZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMB +BwNCAASRKti8VW2Rkma+Kt9jQkMNitlCs0l5w8u3SSwm7HZREvmcBCJBjVIREacR +qI0umhzR2V5NLzBBP9yPD/A+Ch5Xo34wfDAOBgNVHQ8BAf8EBAMCAgQwEwYDVR0l +BAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAaBgNVHREEEzARgg93d3cuZXhh +bXBsZS5jb20wKwYDVR0gBCQwIjAPBg0qhkiG9xIEAYS3CQIBMA8GDSqGSIb3EgQB +hLcJAgIwCgYIKoZIzj0EAwIDSAAwRQIgBEOriD1N3/cqoAofxEtf73M7Wi4UfjFK +jiU9nQhwnnoCIQD1v/XDp2BkWNHxNq7TaPnil3xXTvMX97yUbkUg8IRo0w== +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_leaf_any.pem b/src/crypto/x509/testdata/policy_leaf_any.pem new file mode 100644 index 00000000000000..d2c1b9e9555d16 --- /dev/null +++ b/src/crypto/x509/testdata/policy_leaf_any.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBjTCCATOgAwIBAgIBAzAKBggqhkjOPQQDAjAeMRwwGgYDVQQDExNQb2xpY3kg +SW50ZXJtZWRpYXRlMCAXDTAwMDEwMTAwMDAwMFoYDzIxMDAwMTAxMDAwMDAwWjAa +MRgwFgYDVQQDEw93d3cuZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMB +BwNCAASRKti8VW2Rkma+Kt9jQkMNitlCs0l5w8u3SSwm7HZREvmcBCJBjVIREacR +qI0umhzR2V5NLzBBP9yPD/A+Ch5Xo2QwYjAOBgNVHQ8BAf8EBAMCAgQwEwYDVR0l +BAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAaBgNVHREEEzARgg93d3cuZXhh +bXBsZS5jb20wEQYDVR0gBAowCDAGBgRVHSAAMAoGCCqGSM49BAMCA0gAMEUCIQC4 +UwAf1R4HefSzyO8lyQ3fmMjkptVEhFBee0a7N12IvwIgJMYZgQ52VTbqXyXqraJ8 +V+y+o7eHds7NewqnyuLbc78= +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_leaf_duplicate.pem b/src/crypto/x509/testdata/policy_leaf_duplicate.pem new file mode 100644 index 00000000000000..bdeb13cbd68ec0 --- /dev/null +++ b/src/crypto/x509/testdata/policy_leaf_duplicate.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBsTCCAVigAwIBAgIBAzAKBggqhkjOPQQDAjAWMRQwEgYDVQQDEwtQb2xpY3kg +Um9vdDAgFw0wMDAxMDEwMDAwMDBaGA8yMTAwMDEwMTAwMDAwMFowGjEYMBYGA1UE +AxMPd3d3LmV4YW1wbGUuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEkSrY +vFVtkZJmvirfY0JDDYrZQrNJecPLt0ksJux2URL5nAQiQY1SERGnEaiNLpoc0dle +TS8wQT/cjw/wPgoeV6OBkDCBjTAOBgNVHQ8BAf8EBAMCAgQwEwYDVR0lBAwwCgYI +KwYBBQUHAwEwDAYDVR0TAQH/BAIwADAaBgNVHREEEzARgg93d3cuZXhhbXBsZS5j +b20wPAYDVR0gBDUwMzAPBg0qhkiG9xIEAYS3CQIBMA8GDSqGSIb3EgQBhLcJAgIw +DwYNKoZIhvcSBAGEtwkCAjAKBggqhkjOPQQDAgNHADBEAiBjYDwsWcs35hU/wPqa +5gf0QUMvV/8z5LPX14fB2y4RGQIgMw0ekrt9K5UcgkvFupV/XXIjLRFQvc8URA3C +/+w+2/4= +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_leaf_invalid.pem b/src/crypto/x509/testdata/policy_leaf_invalid.pem new file mode 100644 index 00000000000000..de7a5e9b20f7be --- /dev/null +++ b/src/crypto/x509/testdata/policy_leaf_invalid.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBgjCCASigAwIBAgIBAzAKBggqhkjOPQQDAjAWMRQwEgYDVQQDEwtQb2xpY3kg +Um9vdDAgFw0wMDAxMDEwMDAwMDBaGA8yMTAwMDEwMTAwMDAwMFowGjEYMBYGA1UE +AxMPd3d3LmV4YW1wbGUuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEkSrY +vFVtkZJmvirfY0JDDYrZQrNJecPLt0ksJux2URL5nAQiQY1SERGnEaiNLpoc0dle +TS8wQT/cjw/wPgoeV6NhMF8wDgYDVR0PAQH/BAQDAgIEMBMGA1UdJQQMMAoGCCsG +AQUFBwMBMAwGA1UdEwEB/wQCMAAwGgYDVR0RBBMwEYIPd3d3LmV4YW1wbGUuY29t +MA4GA1UdIAQHSU5WQUxJRDAKBggqhkjOPQQDAgNIADBFAiAgfcDIeqmV+u5YtUe4 +aBnj13tZAJAQh6ttum1xZ+xHEgIhAJqvGX5c0/d1qYelBlm/jE3UuivijdEjVsLX +GVH+X1VA +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_leaf_none.pem b/src/crypto/x509/testdata/policy_leaf_none.pem new file mode 100644 index 00000000000000..13ad7cec0175d9 --- /dev/null +++ b/src/crypto/x509/testdata/policy_leaf_none.pem @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE----- +MIIBezCCASCgAwIBAgIBAzAKBggqhkjOPQQDAjAeMRwwGgYDVQQDExNQb2xpY3kg +SW50ZXJtZWRpYXRlMCAXDTAwMDEwMTAwMDAwMFoYDzIxMDAwMTAxMDAwMDAwWjAa +MRgwFgYDVQQDEw93d3cuZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMB +BwNCAASRKti8VW2Rkma+Kt9jQkMNitlCs0l5w8u3SSwm7HZREvmcBCJBjVIREacR +qI0umhzR2V5NLzBBP9yPD/A+Ch5Xo1EwTzAOBgNVHQ8BAf8EBAMCAgQwEwYDVR0l +BAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAaBgNVHREEEzARgg93d3cuZXhh +bXBsZS5jb20wCgYIKoZIzj0EAwIDSQAwRgIhAIDFeeYJ8nmYo09OnJFpNS3A6fYO +ZliHkAqOsg193DTnAiEA3OSHLCczcvRjMG+qd/FI61u2sKU1hhHh7uHtD/YO/dA= +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_leaf_oid1.pem b/src/crypto/x509/testdata/policy_leaf_oid1.pem new file mode 100644 index 00000000000000..94cd1a77b45ff8 --- /dev/null +++ b/src/crypto/x509/testdata/policy_leaf_oid1.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBlTCCATygAwIBAgIBAzAKBggqhkjOPQQDAjAeMRwwGgYDVQQDExNQb2xpY3kg +SW50ZXJtZWRpYXRlMCAXDTAwMDEwMTAwMDAwMFoYDzIxMDAwMTAxMDAwMDAwWjAa +MRgwFgYDVQQDEw93d3cuZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMB +BwNCAASRKti8VW2Rkma+Kt9jQkMNitlCs0l5w8u3SSwm7HZREvmcBCJBjVIREacR +qI0umhzR2V5NLzBBP9yPD/A+Ch5Xo20wazAOBgNVHQ8BAf8EBAMCAgQwEwYDVR0l +BAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAaBgNVHREEEzARgg93d3cuZXhh +bXBsZS5jb20wGgYDVR0gBBMwETAPBg0qhkiG9xIEAYS3CQIBMAoGCCqGSM49BAMC +A0cAMEQCIHh4Bo8l/HVJhLMWcYusPOE0arqoDrJ5E0M6nEi3nRhgAiAArK8bBohG +fZ3DmVMq/2BJtQZwRRj+50VKWuf9mBSflQ== +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_leaf_oid2.pem b/src/crypto/x509/testdata/policy_leaf_oid2.pem new file mode 100644 index 00000000000000..10adf86c5213fa --- /dev/null +++ b/src/crypto/x509/testdata/policy_leaf_oid2.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBlzCCATygAwIBAgIBAzAKBggqhkjOPQQDAjAeMRwwGgYDVQQDExNQb2xpY3kg +SW50ZXJtZWRpYXRlMCAXDTAwMDEwMTAwMDAwMFoYDzIxMDAwMTAxMDAwMDAwWjAa +MRgwFgYDVQQDEw93d3cuZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMB +BwNCAASRKti8VW2Rkma+Kt9jQkMNitlCs0l5w8u3SSwm7HZREvmcBCJBjVIREacR +qI0umhzR2V5NLzBBP9yPD/A+Ch5Xo20wazAOBgNVHQ8BAf8EBAMCAgQwEwYDVR0l +BAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAaBgNVHREEEzARgg93d3cuZXhh +bXBsZS5jb20wGgYDVR0gBBMwETAPBg0qhkiG9xIEAYS3CQICMAoGCCqGSM49BAMC +A0kAMEYCIQDvW7rdL6MSW/0BPNET4hEeECO6LWmZZHKCHIu6o33dsAIhAPwgm6lD +KV2hMOxkE6rBDQzlCr+zAkQrxSzQZqJp5p+W +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_leaf_oid3.pem b/src/crypto/x509/testdata/policy_leaf_oid3.pem new file mode 100644 index 00000000000000..e5c103151bd836 --- /dev/null +++ b/src/crypto/x509/testdata/policy_leaf_oid3.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBlzCCATygAwIBAgIBAzAKBggqhkjOPQQDAjAeMRwwGgYDVQQDExNQb2xpY3kg +SW50ZXJtZWRpYXRlMCAXDTAwMDEwMTAwMDAwMFoYDzIxMDAwMTAxMDAwMDAwWjAa +MRgwFgYDVQQDEw93d3cuZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMB +BwNCAASRKti8VW2Rkma+Kt9jQkMNitlCs0l5w8u3SSwm7HZREvmcBCJBjVIREacR +qI0umhzR2V5NLzBBP9yPD/A+Ch5Xo20wazAOBgNVHQ8BAf8EBAMCAgQwEwYDVR0l +BAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAaBgNVHREEEzARgg93d3cuZXhh +bXBsZS5jb20wGgYDVR0gBBMwETAPBg0qhkiG9xIEAYS3CQIDMAoGCCqGSM49BAMC +A0kAMEYCIQDBPnPpRsOH20ncg8TKUdlONfbO62WafQj9SKgyi/nGBQIhAMhT8J7f +fTEou6jlAilaIQwlAgZzVKRqgghIHezFY86T +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_leaf_oid4.pem b/src/crypto/x509/testdata/policy_leaf_oid4.pem new file mode 100644 index 00000000000000..7dd7a547af20d1 --- /dev/null +++ b/src/crypto/x509/testdata/policy_leaf_oid4.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBlzCCATygAwIBAgIBAzAKBggqhkjOPQQDAjAeMRwwGgYDVQQDExNQb2xpY3kg +SW50ZXJtZWRpYXRlMCAXDTAwMDEwMTAwMDAwMFoYDzIxMDAwMTAxMDAwMDAwWjAa +MRgwFgYDVQQDEw93d3cuZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMB +BwNCAASRKti8VW2Rkma+Kt9jQkMNitlCs0l5w8u3SSwm7HZREvmcBCJBjVIREacR +qI0umhzR2V5NLzBBP9yPD/A+Ch5Xo20wazAOBgNVHQ8BAf8EBAMCAgQwEwYDVR0l +BAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAaBgNVHREEEzARgg93d3cuZXhh +bXBsZS5jb20wGgYDVR0gBBMwETAPBg0qhkiG9xIEAYS3CQIEMAoGCCqGSM49BAMC +A0kAMEYCIQD2gnpCTMxUalCtEV52eXzqeJgsKMYvEpJTuU/VqH5KwQIhAPEavAkt +cSJsgMgJcJnbBzAdSrbOgHXF2etDHmFbg0hz +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_leaf_oid5.pem b/src/crypto/x509/testdata/policy_leaf_oid5.pem new file mode 100644 index 00000000000000..2a9aee73b59ff5 --- /dev/null +++ b/src/crypto/x509/testdata/policy_leaf_oid5.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBlzCCATygAwIBAgIBAzAKBggqhkjOPQQDAjAeMRwwGgYDVQQDExNQb2xpY3kg +SW50ZXJtZWRpYXRlMCAXDTAwMDEwMTAwMDAwMFoYDzIxMDAwMTAxMDAwMDAwWjAa +MRgwFgYDVQQDEw93d3cuZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMB +BwNCAASRKti8VW2Rkma+Kt9jQkMNitlCs0l5w8u3SSwm7HZREvmcBCJBjVIREacR +qI0umhzR2V5NLzBBP9yPD/A+Ch5Xo20wazAOBgNVHQ8BAf8EBAMCAgQwEwYDVR0l +BAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAaBgNVHREEEzARgg93d3cuZXhh +bXBsZS5jb20wGgYDVR0gBBMwETAPBg0qhkiG9xIEAYS3CQIFMAoGCCqGSM49BAMC +A0kAMEYCIQDDFVjhlQ1Wu0KITcRX8kELpVDeYSKSlvEbZc3rn1QjkQIhAMPthqBi +I0acz8DPQcdFmHXV0xR2xyC1yuen0gES5WLR +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_leaf_require.pem b/src/crypto/x509/testdata/policy_leaf_require.pem new file mode 100644 index 00000000000000..169b8444199ee8 --- /dev/null +++ b/src/crypto/x509/testdata/policy_leaf_require.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBuDCCAV2gAwIBAgIBAzAKBggqhkjOPQQDAjAeMRwwGgYDVQQDExNQb2xpY3kg +SW50ZXJtZWRpYXRlMCAXDTAwMDEwMTAwMDAwMFoYDzIxMDAwMTAxMDAwMDAwWjAa +MRgwFgYDVQQDEw93d3cuZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMB +BwNCAASRKti8VW2Rkma+Kt9jQkMNitlCs0l5w8u3SSwm7HZREvmcBCJBjVIREacR +qI0umhzR2V5NLzBBP9yPD/A+Ch5Xo4GNMIGKMA4GA1UdDwEB/wQEAwICBDATBgNV +HSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMBoGA1UdEQQTMBGCD3d3dy5l +eGFtcGxlLmNvbTArBgNVHSAEJDAiMA8GDSqGSIb3EgQBhLcJAgEwDwYNKoZIhvcS +BAGEtwkCAjAMBgNVHSQEBTADgAEAMAoGCCqGSM49BAMCA0kAMEYCIQDrNQPi/mdK +l7Nd/YmMXWYTHJBWWin1zA64Ohkd7z4jGgIhAJpw/umk5MxS1MwSi+YTkkcSQKpl +YROQH6+T53DauoW6 +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_leaf_require1.pem b/src/crypto/x509/testdata/policy_leaf_require1.pem new file mode 100644 index 00000000000000..261ef954f12ae7 --- /dev/null +++ b/src/crypto/x509/testdata/policy_leaf_require1.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBuDCCAV2gAwIBAgIBAzAKBggqhkjOPQQDAjAeMRwwGgYDVQQDExNQb2xpY3kg +SW50ZXJtZWRpYXRlMCAXDTAwMDEwMTAwMDAwMFoYDzIxMDAwMTAxMDAwMDAwWjAa +MRgwFgYDVQQDEw93d3cuZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMB +BwNCAASRKti8VW2Rkma+Kt9jQkMNitlCs0l5w8u3SSwm7HZREvmcBCJBjVIREacR +qI0umhzR2V5NLzBBP9yPD/A+Ch5Xo4GNMIGKMA4GA1UdDwEB/wQEAwICBDATBgNV +HSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMBoGA1UdEQQTMBGCD3d3dy5l +eGFtcGxlLmNvbTArBgNVHSAEJDAiMA8GDSqGSIb3EgQBhLcJAgEwDwYNKoZIhvcS +BAGEtwkCAjAMBgNVHSQEBTADgAEBMAoGCCqGSM49BAMCA0kAMEYCIQCtXENGJrKv +IOeLHO/3Nu/SMRXc69Vb3q+4b/uHBFbuqwIhAK22Wfh/ZIHKu3FwbjL+sN0Z39pf +Dsak6fp1y4tqNuvK +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_root.pem b/src/crypto/x509/testdata/policy_root.pem new file mode 100644 index 00000000000000..595f8a132a50cf --- /dev/null +++ b/src/crypto/x509/testdata/policy_root.pem @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE----- +MIIBdTCCARqgAwIBAgIBATAKBggqhkjOPQQDAjAWMRQwEgYDVQQDEwtQb2xpY3kg +Um9vdDAgFw0wMDAxMDEwMDAwMDBaGA8yMTAwMDEwMTAwMDAwMFowFjEUMBIGA1UE +AxMLUG9saWN5IFJvb3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQmdqXYl1Gv +Y7y3jcTTK6MVXIQr44TqChRYI6IeV9tIB6jIsOY+Qol1bk8x/7A5FGOnUWFVLEAP +EPSJwPndjolto1cwVTAOBgNVHQ8BAf8EBAMCAgQwEwYDVR0lBAwwCgYIKwYBBQUH +AwEwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU0GnnoB+yeN63WMthnh6Uh1HH +dRIwCgYIKoZIzj0EAwIDSQAwRgIhAKVxVAaJnmvt+q4SqegGS23QSzKPM9Yakw9e +bOUU9+52AiEAjXPRBdd90YDey4VFu4f/78yVe0cxMK30lll7lLl7TTA= +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_root2.pem b/src/crypto/x509/testdata/policy_root2.pem new file mode 100644 index 00000000000000..1350035fd46282 --- /dev/null +++ b/src/crypto/x509/testdata/policy_root2.pem @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE----- +MIIBeDCCAR6gAwIBAgIBATAKBggqhkjOPQQDAjAYMRYwFAYDVQQDEw1Qb2xpY3kg +Um9vdCAyMCAXDTAwMDEwMTAwMDAwMFoYDzIxMDAwMTAxMDAwMDAwWjAYMRYwFAYD +VQQDEw1Qb2xpY3kgUm9vdCAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJnal +2JdRr2O8t43E0yujFVyEK+OE6goUWCOiHlfbSAeoyLDmPkKJdW5PMf+wORRjp1Fh +VSxADxD0icD53Y6JbaNXMFUwDgYDVR0PAQH/BAQDAgIEMBMGA1UdJQQMMAoGCCsG +AQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNBp56Afsnjet1jLYZ4e +lIdRx3USMAoGCCqGSM49BAMCA0gAMEUCIQDm9rw9ODVtJUPBn2lWoK8s7ElbyY4/ +Gc2thHR50UUzbgIgKRenEDhKiBR6cGC77RaIiaaafW8b7HMd7obuZdDU/58= +-----END CERTIFICATE----- diff --git a/src/crypto/x509/testdata/policy_root_cross_inhibit_mapping.pem b/src/crypto/x509/testdata/policy_root_cross_inhibit_mapping.pem new file mode 100644 index 00000000000000..9273a53086f015 --- /dev/null +++ b/src/crypto/x509/testdata/policy_root_cross_inhibit_mapping.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBljCCAT2gAwIBAgIBATAKBggqhkjOPQQDAjAYMRYwFAYDVQQDEw1Qb2xpY3kg +Um9vdCAyMCAXDTAwMDEwMTAwMDAwMFoYDzIxMDAwMTAxMDAwMDAwWjAWMRQwEgYD +VQQDEwtQb2xpY3kgUm9vdDBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCZ2pdiX +Ua9jvLeNxNMroxVchCvjhOoKFFgjoh5X20gHqMiw5j5CiXVuTzH/sDkUY6dRYVUs +QA8Q9InA+d2OiW2jeDB2MA4GA1UdDwEB/wQEAwICBDATBgNVHSUEDDAKBggrBgEF +BQcDATAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTQaeegH7J43rdYy2GeHpSH +Ucd1EjARBgNVHSAECjAIMAYGBFUdIAAwDAYDVR0kBAUwA4EBADAKBggqhkjOPQQD +AgNHADBEAiBzR3JGEf9PITYuiXTx+vx9gXji5idGsVog9wRUbY98wwIgVVeYNQQb +x+RN2wYp3kmm8iswUOrqiI6J4PSzT8CYP8Q= +-----END CERTIFICATE----- diff --git a/src/crypto/x509/verify.go b/src/crypto/x509/verify.go index 7170087287644e..12e59335b2d88f 100644 --- a/src/crypto/x509/verify.go +++ b/src/crypto/x509/verify.go @@ -10,10 +10,14 @@ import ( "crypto/x509/pkix" "errors" "fmt" + "iter" + "maps" "net" + "net/netip" "net/url" "reflect" "runtime" + "slices" "strings" "time" "unicode/utf8" @@ -56,6 +60,8 @@ const ( // CANotAuthorizedForExtKeyUsage results when an intermediate or root // certificate does not permit a requested extended key usage. CANotAuthorizedForExtKeyUsage + // NoValidChains results when there are no valid chains to return. + NoValidChains ) // CertificateInvalidError results when an odd error occurs. Users of this @@ -86,6 +92,12 @@ func (e CertificateInvalidError) Error() string { return "x509: issuer has name constraints but leaf doesn't have a SAN extension" case UnconstrainedName: return "x509: issuer has name constraints but leaf contains unknown or unconstrained name: " + e.Detail + case NoValidChains: + s := "x509: no valid chains built" + if e.Detail != "" { + s = fmt.Sprintf("%s: %s", s, e.Detail) + } + return s } return "x509: unknown error" } @@ -201,6 +213,27 @@ type VerifyOptions struct { // certificates from consuming excessive amounts of CPU time when // validating. It does not apply to the platform verifier. MaxConstraintComparisions int + + // CertificatePolicies specifies which certificate policy OIDs are + // acceptable during policy validation. An empty CertificatePolices + // field implies any valid policy is acceptable. + CertificatePolicies []OID + + // The following policy fields are unexported, because we do not expect + // users to actually need to use them, but are useful for testing the + // policy validation code. + + // inhibitPolicyMapping indicates if policy mapping should be allowed + // during path validation. + inhibitPolicyMapping bool + + // requireExplicitPolicy indidicates if explicit policies must be present + // for each certificate being validated. + requireExplicitPolicy bool + + // inhibitAnyPolicy indicates if the anyPolicy policy should be + // processed if present in a certificate being validated. + inhibitAnyPolicy bool } const ( @@ -359,6 +392,7 @@ func parseRFC2821Mailbox(in string) (mailbox rfc2821Mailbox, ok bool) { // domainToReverseLabels converts a textual domain name like foo.example.com to // the list of labels in reverse order, e.g. ["com", "example", "foo"]. func domainToReverseLabels(domain string) (reverseLabels []string, ok bool) { + reverseLabels = make([]string, 0, strings.Count(domain, ".")+1) for len(domain) > 0 { if i := strings.LastIndexByte(domain, '.'); i == -1 { reverseLabels = append(reverseLabels, domain) @@ -396,7 +430,7 @@ func domainToReverseLabels(domain string) (reverseLabels []string, ok bool) { return reverseLabels, true } -func matchEmailConstraint(mailbox rfc2821Mailbox, constraint string) (bool, error) { +func matchEmailConstraint(mailbox rfc2821Mailbox, constraint string, reversedDomainsCache map[string][]string, reversedConstraintsCache map[string][]string) (bool, error) { // If the constraint contains an @, then it specifies an exact mailbox // name. if strings.Contains(constraint, "@") { @@ -409,10 +443,10 @@ func matchEmailConstraint(mailbox rfc2821Mailbox, constraint string) (bool, erro // Otherwise the constraint is like a DNS constraint of the domain part // of the mailbox. - return matchDomainConstraint(mailbox.domain, constraint) + return matchDomainConstraint(mailbox.domain, constraint, reversedDomainsCache, reversedConstraintsCache) } -func matchURIConstraint(uri *url.URL, constraint string) (bool, error) { +func matchURIConstraint(uri *url.URL, constraint string, reversedDomainsCache map[string][]string, reversedConstraintsCache map[string][]string) (bool, error) { // From RFC 5280, Section 4.2.1.10: // “a uniformResourceIdentifier that does not include an authority // component with a host name specified as a fully qualified domain @@ -434,12 +468,14 @@ func matchURIConstraint(uri *url.URL, constraint string) (bool, error) { } } - if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") || - net.ParseIP(host) != nil { + // netip.ParseAddr will reject the URI IPv6 literal form "[...]", so we + // check if _either_ the string parses as an IP, or if it is enclosed in + // square brackets. + if _, err := netip.ParseAddr(host); err == nil || (strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]")) { return false, fmt.Errorf("URI with IP (%q) cannot be matched against constraints", uri.String()) } - return matchDomainConstraint(host, constraint) + return matchDomainConstraint(host, constraint, reversedDomainsCache, reversedConstraintsCache) } func matchIPConstraint(ip net.IP, constraint *net.IPNet) (bool, error) { @@ -456,16 +492,21 @@ func matchIPConstraint(ip net.IP, constraint *net.IPNet) (bool, error) { return true, nil } -func matchDomainConstraint(domain, constraint string) (bool, error) { +func matchDomainConstraint(domain, constraint string, reversedDomainsCache map[string][]string, reversedConstraintsCache map[string][]string) (bool, error) { // The meaning of zero length constraints is not specified, but this // code follows NSS and accepts them as matching everything. if len(constraint) == 0 { return true, nil } - domainLabels, ok := domainToReverseLabels(domain) - if !ok { - return false, fmt.Errorf("x509: internal error: cannot parse domain %q", domain) + domainLabels, found := reversedDomainsCache[domain] + if !found { + var ok bool + domainLabels, ok = domainToReverseLabels(domain) + if !ok { + return false, fmt.Errorf("x509: internal error: cannot parse domain %q", domain) + } + reversedDomainsCache[domain] = domainLabels } // RFC 5280 says that a leading period in a domain name means that at @@ -479,9 +520,14 @@ func matchDomainConstraint(domain, constraint string) (bool, error) { constraint = constraint[1:] } - constraintLabels, ok := domainToReverseLabels(constraint) - if !ok { - return false, fmt.Errorf("x509: internal error: cannot parse domain %q", constraint) + constraintLabels, found := reversedConstraintsCache[constraint] + if !found { + var ok bool + constraintLabels, ok = domainToReverseLabels(constraint) + if !ok { + return false, fmt.Errorf("x509: internal error: cannot parse domain %q", constraint) + } + reversedConstraintsCache[constraint] = constraintLabels } if len(domainLabels) < len(constraintLabels) || @@ -590,94 +636,8 @@ func (c *Certificate) isValid(certType int, currentChain []*Certificate, opts *V } } - maxConstraintComparisons := opts.MaxConstraintComparisions - if maxConstraintComparisons == 0 { - maxConstraintComparisons = 250000 - } - comparisonCount := 0 - - if certType == intermediateCertificate || certType == rootCertificate { - if len(currentChain) == 0 { - return errors.New("x509: internal error: empty chain when appending CA cert") - } - } - - if (certType == intermediateCertificate || certType == rootCertificate) && - c.hasNameConstraints() { - toCheck := []*Certificate{} - for _, c := range currentChain { - if c.hasSANExtension() { - toCheck = append(toCheck, c) - } - } - for _, sanCert := range toCheck { - err := forEachSAN(sanCert.getSANExtension(), func(tag int, data []byte) error { - switch tag { - case nameTypeEmail: - name := string(data) - mailbox, ok := parseRFC2821Mailbox(name) - if !ok { - return fmt.Errorf("x509: cannot parse rfc822Name %q", mailbox) - } - - if err := c.checkNameConstraints(&comparisonCount, maxConstraintComparisons, "email address", name, mailbox, - func(parsedName, constraint any) (bool, error) { - return matchEmailConstraint(parsedName.(rfc2821Mailbox), constraint.(string)) - }, c.PermittedEmailAddresses, c.ExcludedEmailAddresses); err != nil { - return err - } - - case nameTypeDNS: - name := string(data) - if _, ok := domainToReverseLabels(name); !ok { - return fmt.Errorf("x509: cannot parse dnsName %q", name) - } - - if err := c.checkNameConstraints(&comparisonCount, maxConstraintComparisons, "DNS name", name, name, - func(parsedName, constraint any) (bool, error) { - return matchDomainConstraint(parsedName.(string), constraint.(string)) - }, c.PermittedDNSDomains, c.ExcludedDNSDomains); err != nil { - return err - } - - case nameTypeURI: - name := string(data) - uri, err := url.Parse(name) - if err != nil { - return fmt.Errorf("x509: internal error: URI SAN %q failed to parse", name) - } - - if err := c.checkNameConstraints(&comparisonCount, maxConstraintComparisons, "URI", name, uri, - func(parsedName, constraint any) (bool, error) { - return matchURIConstraint(parsedName.(*url.URL), constraint.(string)) - }, c.PermittedURIDomains, c.ExcludedURIDomains); err != nil { - return err - } - - case nameTypeIP: - ip := net.IP(data) - if l := len(ip); l != net.IPv4len && l != net.IPv6len { - return fmt.Errorf("x509: internal error: IP SAN %x failed to parse", data) - } - - if err := c.checkNameConstraints(&comparisonCount, maxConstraintComparisons, "IP address", ip.String(), ip, - func(parsedName, constraint any) (bool, error) { - return matchIPConstraint(parsedName.(net.IP), constraint.(*net.IPNet)) - }, c.PermittedIPRanges, c.ExcludedIPRanges); err != nil { - return err - } - - default: - // Unknown SAN types are ignored. - } - - return nil - }) - - if err != nil { - return err - } - } + if (certType == intermediateCertificate || certType == rootCertificate) && len(currentChain) == 0 { + return errors.New("x509: internal error: empty chain when appending CA cert") } // KeyUsage status flags are ignored. From Engineering Security, Peter @@ -708,13 +668,6 @@ func (c *Certificate) isValid(certType int, currentChain []*Certificate, opts *V } } - if !boringAllowCert(c) { - // IncompatibleUsage is not quite right here, - // but it's also the "no chains found" error - // and is close enough. - return CertificateInvalidError{c, IncompatibleUsage, ""} - } - return nil } @@ -750,7 +703,7 @@ func (c *Certificate) isValid(certType int, currentChain []*Certificate, opts *V // Certificates other than c in the returned chains should not be modified. // // WARNING: this function doesn't do any revocation checking. -func (c *Certificate) Verify(opts VerifyOptions) (chains [][]*Certificate, err error) { +func (c *Certificate) Verify(opts VerifyOptions) ([][]*Certificate, error) { // Platform-specific verification needs the ASN.1 contents so // this makes the behavior consistent across platforms. if len(c.Raw) == 0 { @@ -792,15 +745,15 @@ func (c *Certificate) Verify(opts VerifyOptions) (chains [][]*Certificate, err e } } - err = c.isValid(leafCertificate, nil, &opts) + err := c.isValid(leafCertificate, nil, &opts) if err != nil { - return + return nil, err } if len(opts.DNSName) > 0 { err = c.VerifyHostname(opts.DNSName) if err != nil { - return + return nil, err } } @@ -814,30 +767,61 @@ func (c *Certificate) Verify(opts VerifyOptions) (chains [][]*Certificate, err e } } - if len(opts.KeyUsages) == 0 { - opts.KeyUsages = []ExtKeyUsage{ExtKeyUsageServerAuth} - } - + anyKeyUsage := false for _, eku := range opts.KeyUsages { if eku == ExtKeyUsageAny { - // If any key usage is acceptable, no need to check the chain for - // key usages. - return candidateChains, nil + // The presence of anyExtendedKeyUsage overrides any other key usage. + anyKeyUsage = true + break } } - chains = make([][]*Certificate, 0, len(candidateChains)) - for _, candidate := range candidateChains { - if checkChainForKeyUsage(candidate, opts.KeyUsages) { - chains = append(chains, candidate) - } + if len(opts.KeyUsages) == 0 { + opts.KeyUsages = []ExtKeyUsage{ExtKeyUsageServerAuth} } - if len(chains) == 0 { - return nil, CertificateInvalidError{c, IncompatibleUsage, ""} + var invalidPoliciesChains int + var incompatibleKeyUsageChains int + var constraintsHintErr error + candidateChains = slices.DeleteFunc(candidateChains, func(chain []*Certificate) bool { + if !policiesValid(chain, opts) { + invalidPoliciesChains++ + return true + } + // If any key usage is acceptable, no need to check the chain for + // key usages. + if !anyKeyUsage && !checkChainForKeyUsage(chain, opts.KeyUsages) { + incompatibleKeyUsageChains++ + return true + } + if err := checkChainConstraints(chain, opts); err != nil { + if constraintsHintErr == nil { + constraintsHintErr = err + } + return true + } + return false + }) + + if len(candidateChains) == 0 { + if constraintsHintErr != nil { + return nil, constraintsHintErr // Preserve previous constraint behavior + } + var details []string + if incompatibleKeyUsageChains > 0 { + if invalidPoliciesChains == 0 { + return nil, CertificateInvalidError{c, IncompatibleUsage, ""} + } + details = append(details, fmt.Sprintf("%d candidate chains with incompatible key usage", incompatibleKeyUsageChains)) + } + if invalidPoliciesChains > 0 { + details = append(details, fmt.Sprintf("%d candidate chains with invalid policies", invalidPoliciesChains)) + } + err = CertificateInvalidError{c, NoValidChains, strings.Join(details, ", ")} + return nil, err } - return chains, nil + return candidateChains, nil } func appendToFreshChain(chain []*Certificate, cert *Certificate) []*Certificate { @@ -869,7 +853,10 @@ func alreadyInChain(candidate *Certificate, chain []*Certificate) bool { if !bytes.Equal(candidate.RawSubject, cert.RawSubject) { continue } - if !candidate.PublicKey.(pubKeyEqual).Equal(cert.PublicKey) { + // We enforce the canonical encoding of SPKI (by only allowing the + // correct AI paremeter encodings in parseCertificate), so it's safe to + // directly compare the raw bytes. + if !bytes.Equal(candidate.RawSubjectPublicKeyInfo, cert.RawSubjectPublicKeyInfo) { continue } var certSAN *pkix.Extension @@ -1194,3 +1181,469 @@ NextCert: return true } + +func checkChainConstraints(chain []*Certificate, opts VerifyOptions) error { + maxConstraintComparisons := opts.MaxConstraintComparisions + if maxConstraintComparisons == 0 { + maxConstraintComparisons = 250000 + } + comparisonCount := 0 + + // Each time we do constraint checking, we need to check the constraints in + // the current certificate against all of the names that preceded it. We + // reverse these names using domainToReverseLabels, which is a relatively + // expensive operation. Since we check each name against each constraint, + // this requires us to do N*C calls to domainToReverseLabels (where N is the + // total number of names that preceed the certificate, and C is the total + // number of constraints in the certificate). By caching the results of + // calling domainToReverseLabels, we can reduce that to N+C calls at the + // cost of keeping all of the parsed names and constraints in memory until + // we return from isValid. + reversedDomainsCache := map[string][]string{} + reversedConstraintsCache := map[string][]string{} + + for i, c := range chain { + if !c.hasNameConstraints() { + continue + } + for _, sanCert := range chain[:i] { + if !sanCert.hasSANExtension() { + continue + } + err := forEachSAN(sanCert.getSANExtension(), func(tag int, data []byte) error { + switch tag { + case nameTypeEmail: + name := string(data) + mailbox, ok := parseRFC2821Mailbox(name) + if !ok { + return fmt.Errorf("x509: cannot parse rfc822Name %q", mailbox) + } + + if err := c.checkNameConstraints(&comparisonCount, maxConstraintComparisons, "email address", name, mailbox, + func(parsedName, constraint any) (bool, error) { + return matchEmailConstraint(parsedName.(rfc2821Mailbox), constraint.(string), reversedDomainsCache, reversedConstraintsCache) + }, c.PermittedEmailAddresses, c.ExcludedEmailAddresses); err != nil { + return err + } + + case nameTypeDNS: + name := string(data) + if !domainNameValid(name, false) { + return fmt.Errorf("x509: cannot parse dnsName %q", name) + } + + if err := c.checkNameConstraints(&comparisonCount, maxConstraintComparisons, "DNS name", name, name, + func(parsedName, constraint any) (bool, error) { + return matchDomainConstraint(parsedName.(string), constraint.(string), reversedDomainsCache, reversedConstraintsCache) + }, c.PermittedDNSDomains, c.ExcludedDNSDomains); err != nil { + return err + } + + case nameTypeURI: + name := string(data) + uri, err := url.Parse(name) + if err != nil { + return fmt.Errorf("x509: internal error: URI SAN %q failed to parse", name) + } + + if err := c.checkNameConstraints(&comparisonCount, maxConstraintComparisons, "URI", name, uri, + func(parsedName, constraint any) (bool, error) { + return matchURIConstraint(parsedName.(*url.URL), constraint.(string), reversedDomainsCache, reversedConstraintsCache) + }, c.PermittedURIDomains, c.ExcludedURIDomains); err != nil { + return err + } + + case nameTypeIP: + ip := net.IP(data) + if l := len(ip); l != net.IPv4len && l != net.IPv6len { + return fmt.Errorf("x509: internal error: IP SAN %x failed to parse", data) + } + + if err := c.checkNameConstraints(&comparisonCount, maxConstraintComparisons, "IP address", ip.String(), ip, + func(parsedName, constraint any) (bool, error) { + return matchIPConstraint(parsedName.(net.IP), constraint.(*net.IPNet)) + }, c.PermittedIPRanges, c.ExcludedIPRanges); err != nil { + return err + } + + default: + // Unknown SAN types are ignored. + } + + return nil + }) + + if err != nil { + return err + } + } + } + + return nil +} + +func mustNewOIDFromInts(ints []uint64) OID { + oid, err := OIDFromInts(ints) + if err != nil { + panic(fmt.Sprintf("OIDFromInts(%v) unexpected error: %v", ints, err)) + } + return oid +} + +type policyGraphNode struct { + validPolicy OID + expectedPolicySet []OID + // we do not implement qualifiers, so we don't track qualifier_set + + parents map[*policyGraphNode]bool + children map[*policyGraphNode]bool +} + +func newPolicyGraphNode(valid OID, parents []*policyGraphNode) *policyGraphNode { + n := &policyGraphNode{ + validPolicy: valid, + expectedPolicySet: []OID{valid}, + children: map[*policyGraphNode]bool{}, + parents: map[*policyGraphNode]bool{}, + } + for _, p := range parents { + p.children[n] = true + n.parents[p] = true + } + return n +} + +type policyGraph struct { + strata []map[string]*policyGraphNode + // map of OID -> nodes at strata[depth-1] with OID in their expectedPolicySet + parentIndex map[string][]*policyGraphNode + depth int +} + +var anyPolicyOID = mustNewOIDFromInts([]uint64{2, 5, 29, 32, 0}) + +func newPolicyGraph() *policyGraph { + root := policyGraphNode{ + validPolicy: anyPolicyOID, + expectedPolicySet: []OID{anyPolicyOID}, + children: map[*policyGraphNode]bool{}, + parents: map[*policyGraphNode]bool{}, + } + return &policyGraph{ + depth: 0, + strata: []map[string]*policyGraphNode{{string(anyPolicyOID.der): &root}}, + } +} + +func (pg *policyGraph) insert(n *policyGraphNode) { + pg.strata[pg.depth][string(n.validPolicy.der)] = n +} + +func (pg *policyGraph) parentsWithExpected(expected OID) []*policyGraphNode { + if pg.depth == 0 { + return nil + } + return pg.parentIndex[string(expected.der)] +} + +func (pg *policyGraph) parentWithAnyPolicy() *policyGraphNode { + if pg.depth == 0 { + return nil + } + return pg.strata[pg.depth-1][string(anyPolicyOID.der)] +} + +func (pg *policyGraph) parents() iter.Seq[*policyGraphNode] { + if pg.depth == 0 { + return nil + } + return maps.Values(pg.strata[pg.depth-1]) +} + +func (pg *policyGraph) leaves() map[string]*policyGraphNode { + return pg.strata[pg.depth] +} + +func (pg *policyGraph) leafWithPolicy(policy OID) *policyGraphNode { + return pg.strata[pg.depth][string(policy.der)] +} + +func (pg *policyGraph) deleteLeaf(policy OID) { + n := pg.strata[pg.depth][string(policy.der)] + if n == nil { + return + } + for p := range n.parents { + delete(p.children, n) + } + for c := range n.children { + delete(c.parents, n) + } + delete(pg.strata[pg.depth], string(policy.der)) +} + +func (pg *policyGraph) validPolicyNodes() []*policyGraphNode { + var validNodes []*policyGraphNode + for i := pg.depth; i >= 0; i-- { + for _, n := range pg.strata[i] { + if n.validPolicy.Equal(anyPolicyOID) { + continue + } + + if len(n.parents) == 1 { + for p := range n.parents { + if p.validPolicy.Equal(anyPolicyOID) { + validNodes = append(validNodes, n) + } + } + } + } + } + return validNodes +} + +func (pg *policyGraph) prune() { + for i := pg.depth - 1; i > 0; i-- { + for _, n := range pg.strata[i] { + if len(n.children) == 0 { + for p := range n.parents { + delete(p.children, n) + } + delete(pg.strata[i], string(n.validPolicy.der)) + } + } + } +} + +func (pg *policyGraph) incrDepth() { + pg.parentIndex = map[string][]*policyGraphNode{} + for _, n := range pg.strata[pg.depth] { + for _, e := range n.expectedPolicySet { + pg.parentIndex[string(e.der)] = append(pg.parentIndex[string(e.der)], n) + } + } + + pg.depth++ + pg.strata = append(pg.strata, map[string]*policyGraphNode{}) +} + +func policiesValid(chain []*Certificate, opts VerifyOptions) bool { + // The following code implements the policy verification algorithm as + // specified in RFC 5280 and updated by RFC 9618. In particular the + // following sections are replaced by RFC 9618: + // * 6.1.2 (a) + // * 6.1.3 (d) + // * 6.1.3 (e) + // * 6.1.3 (f) + // * 6.1.4 (b) + // * 6.1.5 (g) + + if len(chain) == 1 { + return true + } + + // n is the length of the chain minus the trust anchor + n := len(chain) - 1 + + pg := newPolicyGraph() + var inhibitAnyPolicy, explicitPolicy, policyMapping int + if !opts.inhibitAnyPolicy { + inhibitAnyPolicy = n + 1 + } + if !opts.requireExplicitPolicy { + explicitPolicy = n + 1 + } + if !opts.inhibitPolicyMapping { + policyMapping = n + 1 + } + + initialUserPolicySet := map[string]bool{} + for _, p := range opts.CertificatePolicies { + initialUserPolicySet[string(p.der)] = true + } + // If the user does not pass any policies, we consider + // that equivalent to passing anyPolicyOID. + if len(initialUserPolicySet) == 0 { + initialUserPolicySet[string(anyPolicyOID.der)] = true + } + + for i := n - 1; i >= 0; i-- { + cert := chain[i] + + isSelfSigned := bytes.Equal(cert.RawIssuer, cert.RawSubject) + + // 6.1.3 (e) -- as updated by RFC 9618 + if len(cert.Policies) == 0 { + pg = nil + } + + // 6.1.3 (f) -- as updated by RFC 9618 + if explicitPolicy == 0 && pg == nil { + return false + } + + if pg != nil { + pg.incrDepth() + + policies := map[string]bool{} + + // 6.1.3 (d) (1) -- as updated by RFC 9618 + for _, policy := range cert.Policies { + policies[string(policy.der)] = true + + if policy.Equal(anyPolicyOID) { + continue + } + + // 6.1.3 (d) (1) (i) -- as updated by RFC 9618 + parents := pg.parentsWithExpected(policy) + if len(parents) == 0 { + // 6.1.3 (d) (1) (ii) -- as updated by RFC 9618 + if anyParent := pg.parentWithAnyPolicy(); anyParent != nil { + parents = []*policyGraphNode{anyParent} + } + } + if len(parents) > 0 { + pg.insert(newPolicyGraphNode(policy, parents)) + } + } + + // 6.1.3 (d) (2) -- as updated by RFC 9618 + // NOTE: in the check "n-i < n" our i is different from the i in the specification. + // In the specification chains go from the trust anchor to the leaf, whereas our + // chains go from the leaf to the trust anchor, so our i's our inverted. Our + // check here matches the check "i < n" in the specification. + if policies[string(anyPolicyOID.der)] && (inhibitAnyPolicy > 0 || (n-i < n && isSelfSigned)) { + missing := map[string][]*policyGraphNode{} + leaves := pg.leaves() + for p := range pg.parents() { + for _, expected := range p.expectedPolicySet { + if leaves[string(expected.der)] == nil { + missing[string(expected.der)] = append(missing[string(expected.der)], p) + } + } + } + + for oidStr, parents := range missing { + pg.insert(newPolicyGraphNode(OID{der: []byte(oidStr)}, parents)) + } + } + + // 6.1.3 (d) (3) -- as updated by RFC 9618 + pg.prune() + + if i != 0 { + // 6.1.4 (b) -- as updated by RFC 9618 + if len(cert.PolicyMappings) > 0 { + // collect map of issuer -> []subject + mappings := map[string][]OID{} + + for _, mapping := range cert.PolicyMappings { + if policyMapping > 0 { + if mapping.IssuerDomainPolicy.Equal(anyPolicyOID) || mapping.SubjectDomainPolicy.Equal(anyPolicyOID) { + // Invalid mapping + return false + } + mappings[string(mapping.IssuerDomainPolicy.der)] = append(mappings[string(mapping.IssuerDomainPolicy.der)], mapping.SubjectDomainPolicy) + } else { + // 6.1.4 (b) (3) (i) -- as updated by RFC 9618 + pg.deleteLeaf(mapping.IssuerDomainPolicy) + + // 6.1.4 (b) (3) (ii) -- as updated by RFC 9618 + pg.prune() + } + } + + for issuerStr, subjectPolicies := range mappings { + // 6.1.4 (b) (1) -- as updated by RFC 9618 + if matching := pg.leafWithPolicy(OID{der: []byte(issuerStr)}); matching != nil { + matching.expectedPolicySet = subjectPolicies + } else if matching := pg.leafWithPolicy(anyPolicyOID); matching != nil { + // 6.1.4 (b) (2) -- as updated by RFC 9618 + n := newPolicyGraphNode(OID{der: []byte(issuerStr)}, []*policyGraphNode{matching}) + n.expectedPolicySet = subjectPolicies + pg.insert(n) + } + } + } + } + } + + if i != 0 { + // 6.1.4 (h) + if !isSelfSigned { + if explicitPolicy > 0 { + explicitPolicy-- + } + if policyMapping > 0 { + policyMapping-- + } + if inhibitAnyPolicy > 0 { + inhibitAnyPolicy-- + } + } + + // 6.1.4 (i) + if (cert.RequireExplicitPolicy > 0 || cert.RequireExplicitPolicyZero) && cert.RequireExplicitPolicy < explicitPolicy { + explicitPolicy = cert.RequireExplicitPolicy + } + if (cert.InhibitPolicyMapping > 0 || cert.InhibitPolicyMappingZero) && cert.InhibitPolicyMapping < policyMapping { + policyMapping = cert.InhibitPolicyMapping + } + // 6.1.4 (j) + if (cert.InhibitAnyPolicy > 0 || cert.InhibitAnyPolicyZero) && cert.InhibitAnyPolicy < inhibitAnyPolicy { + inhibitAnyPolicy = cert.InhibitAnyPolicy + } + } + } + + // 6.1.5 (a) + if explicitPolicy > 0 { + explicitPolicy-- + } + + // 6.1.5 (b) + if chain[0].RequireExplicitPolicyZero { + explicitPolicy = 0 + } + + // 6.1.5 (g) (1) -- as updated by RFC 9618 + var validPolicyNodeSet []*policyGraphNode + // 6.1.5 (g) (2) -- as updated by RFC 9618 + if pg != nil { + validPolicyNodeSet = pg.validPolicyNodes() + // 6.1.5 (g) (3) -- as updated by RFC 9618 + if currentAny := pg.leafWithPolicy(anyPolicyOID); currentAny != nil { + validPolicyNodeSet = append(validPolicyNodeSet, currentAny) + } + } + + // 6.1.5 (g) (4) -- as updated by RFC 9618 + authorityConstrainedPolicySet := map[string]bool{} + for _, n := range validPolicyNodeSet { + authorityConstrainedPolicySet[string(n.validPolicy.der)] = true + } + // 6.1.5 (g) (5) -- as updated by RFC 9618 + userConstrainedPolicySet := maps.Clone(authorityConstrainedPolicySet) + // 6.1.5 (g) (6) -- as updated by RFC 9618 + if len(initialUserPolicySet) != 1 || !initialUserPolicySet[string(anyPolicyOID.der)] { + // 6.1.5 (g) (6) (i) -- as updated by RFC 9618 + for p := range userConstrainedPolicySet { + if !initialUserPolicySet[p] { + delete(userConstrainedPolicySet, p) + } + } + // 6.1.5 (g) (6) (ii) -- as updated by RFC 9618 + if authorityConstrainedPolicySet[string(anyPolicyOID.der)] { + for policy := range initialUserPolicySet { + userConstrainedPolicySet[policy] = true + } + } + } + + if explicitPolicy == 0 && len(userConstrainedPolicySet) == 0 { + return false + } + + return true +} diff --git a/src/crypto/x509/verify_test.go b/src/crypto/x509/verify_test.go index 7f6b74b7a0f5bb..17e7aaf897ab56 100644 --- a/src/crypto/x509/verify_test.go +++ b/src/crypto/x509/verify_test.go @@ -6,6 +6,7 @@ package x509 import ( "crypto" + "crypto/dsa" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" @@ -16,6 +17,7 @@ import ( "fmt" "internal/testenv" "math/big" + "os" "os/exec" "runtime" "slices" @@ -134,45 +136,6 @@ var verifyTests = []verifyTest{ // entry so the results don't match. systemLax: true, }, - { - name: "dnssec-exp", - leaf: dnssecExpLeaf, - intermediates: []string{startComIntermediate}, - roots: []string{startComRoot}, - currentTime: 1302726541, - - // The StartCom root is not trusted by Windows when the default - // ServerAuth EKU is requested. - systemSkip: true, - - expectedChains: [][]string{ - {"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, - }, - }, - { - name: "dnssec-exp/AnyEKU", - leaf: dnssecExpLeaf, - intermediates: []string{startComIntermediate}, - roots: []string{startComRoot}, - currentTime: 1302726541, - keyUsages: []ExtKeyUsage{ExtKeyUsageAny}, - - expectedChains: [][]string{ - {"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, - }, - }, - { - name: "dnssec-exp/RootInIntermediates", - leaf: dnssecExpLeaf, - intermediates: []string{startComIntermediate, startComRoot}, - roots: []string{startComRoot}, - currentTime: 1302726541, - systemSkip: true, // see dnssec-exp test - - expectedChains: [][]string{ - {"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, - }, - }, { name: "InvalidHash", leaf: googleLeafWithInvalidHash, @@ -227,15 +190,15 @@ var verifyTests = []verifyTest{ leaf: nameConstraintsLeaf, intermediates: []string{nameConstraintsIntermediate1, nameConstraintsIntermediate2}, roots: []string{globalSignRoot}, - currentTime: 1382387896, - dnsName: "secure.iddl.vt.edu", + currentTime: 1524771953, + dnsName: "udctest.ads.vt.edu", expectedChains: [][]string{ { - "Technology-enhanced Learning and Online Strategies", + "udctest.ads.vt.edu", "Virginia Tech Global Qualified Server CA", - "Trusted Root CA G2", - "GlobalSign Root CA", + "Trusted Root CA SHA256 G2", + "GlobalSign", }, }, }, @@ -552,10 +515,6 @@ func testVerify(t *testing.T, test verifyTest, useSystemRoots bool) { } func TestGoVerify(t *testing.T) { - // Temporarily enable SHA-1 verification since a number of test chains - // require it. TODO(filippo): regenerate test chains. - t.Setenv("GODEBUG", "x509sha1=1") - for _, test := range verifyTests { t.Run(test.name, func(t *testing.T) { testVerify(t, test, false) @@ -724,128 +683,6 @@ c78zEqNswn6xGKXpWF5xVwdFcsx9HKhJ6UAi2bQ/KQ1yb7LPUOR6wXXWrG1cLnNP i8eNLnKL9PXQ+5SwJFCzfEhcIZuhzg== -----END CERTIFICATE-----` -const dnssecExpLeaf = `-----BEGIN CERTIFICATE----- -MIIGzTCCBbWgAwIBAgIDAdD6MA0GCSqGSIb3DQEBBQUAMIGMMQswCQYDVQQGEwJJ -TDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0 -YWwgQ2VydGlmaWNhdGUgU2lnbmluZzE4MDYGA1UEAxMvU3RhcnRDb20gQ2xhc3Mg -MSBQcmltYXJ5IEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwHhcNMTAwNzA0MTQ1MjQ1 -WhcNMTEwNzA1MTA1NzA0WjCBwTEgMB4GA1UEDRMXMjIxMTM3LWxpOWE5dHhJRzZM -NnNyVFMxCzAJBgNVBAYTAlVTMR4wHAYDVQQKExVQZXJzb25hIE5vdCBWYWxpZGF0 -ZWQxKTAnBgNVBAsTIFN0YXJ0Q29tIEZyZWUgQ2VydGlmaWNhdGUgTWVtYmVyMRsw -GQYDVQQDExJ3d3cuZG5zc2VjLWV4cC5vcmcxKDAmBgkqhkiG9w0BCQEWGWhvc3Rt -YXN0ZXJAZG5zc2VjLWV4cC5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK -AoIBAQDEdF/22vaxrPbqpgVYMWi+alfpzBctpbfLBdPGuqOazJdCT0NbWcK8/+B4 -X6OlSOURNIlwLzhkmwVsWdVv6dVSaN7d4yI/fJkvgfDB9+au+iBJb6Pcz8ULBfe6 -D8HVvqKdORp6INzHz71z0sghxrQ0EAEkoWAZLh+kcn2ZHdcmZaBNUfjmGbyU6PRt -RjdqoP+owIaC1aktBN7zl4uO7cRjlYFdusINrh2kPP02KAx2W84xjxX1uyj6oS6e -7eBfvcwe8czW/N1rbE0CoR7h9+HnIrjnVG9RhBiZEiw3mUmF++Up26+4KTdRKbu3 -+BL4yMpfd66z0+zzqu+HkvyLpFn5AgMBAAGjggL/MIIC+zAJBgNVHRMEAjAAMAsG -A1UdDwQEAwIDqDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4EFgQUy04I5guM -drzfh2JQaXhgV86+4jUwHwYDVR0jBBgwFoAU60I00Jiwq5/0G2sI98xkLu8OLEUw -LQYDVR0RBCYwJIISd3d3LmRuc3NlYy1leHAub3Jngg5kbnNzZWMtZXhwLm9yZzCC -AUIGA1UdIASCATkwggE1MIIBMQYLKwYBBAGBtTcBAgIwggEgMC4GCCsGAQUFBwIB -FiJodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS9wb2xpY3kucGRmMDQGCCsGAQUFBwIB -FihodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS9pbnRlcm1lZGlhdGUucGRmMIG3Bggr -BgEFBQcCAjCBqjAUFg1TdGFydENvbSBMdGQuMAMCAQEagZFMaW1pdGVkIExpYWJp -bGl0eSwgc2VlIHNlY3Rpb24gKkxlZ2FsIExpbWl0YXRpb25zKiBvZiB0aGUgU3Rh -cnRDb20gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgUG9saWN5IGF2YWlsYWJsZSBh -dCBodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS9wb2xpY3kucGRmMGEGA1UdHwRaMFgw -KqAooCaGJGh0dHA6Ly93d3cuc3RhcnRzc2wuY29tL2NydDEtY3JsLmNybDAqoCig -JoYkaHR0cDovL2NybC5zdGFydHNzbC5jb20vY3J0MS1jcmwuY3JsMIGOBggrBgEF -BQcBAQSBgTB/MDkGCCsGAQUFBzABhi1odHRwOi8vb2NzcC5zdGFydHNzbC5jb20v -c3ViL2NsYXNzMS9zZXJ2ZXIvY2EwQgYIKwYBBQUHMAKGNmh0dHA6Ly93d3cuc3Rh -cnRzc2wuY29tL2NlcnRzL3N1Yi5jbGFzczEuc2VydmVyLmNhLmNydDAjBgNVHRIE -HDAahhhodHRwOi8vd3d3LnN0YXJ0c3NsLmNvbS8wDQYJKoZIhvcNAQEFBQADggEB -ACXj6SB59KRJPenn6gUdGEqcta97U769SATyiQ87i9er64qLwvIGLMa3o2Rcgl2Y -kghUeyLdN/EXyFBYA8L8uvZREPoc7EZukpT/ZDLXy9i2S0jkOxvF2fD/XLbcjGjM -iEYG1/6ASw0ri9C0k4oDDoJLCoeH9++yqF7SFCCMcDkJqiAGXNb4euDpa8vCCtEQ -CSS+ObZbfkreRt3cNCf5LfCXe9OsTnCfc8Cuq81c0oLaG+SmaLUQNBuToq8e9/Zm -+b+/a3RVjxmkV5OCcGVBxsXNDn54Q6wsdw0TBMcjwoEndzpLS7yWgFbbkq5ZiGpw -Qibb2+CfKuQ+WFV1GkVQmVA= ------END CERTIFICATE-----` - -const startComIntermediate = `-----BEGIN CERTIFICATE----- -MIIGNDCCBBygAwIBAgIBGDANBgkqhkiG9w0BAQUFADB9MQswCQYDVQQGEwJJTDEW -MBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0YWwg -Q2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3RhcnRDb20gQ2VydGlmaWNh -dGlvbiBBdXRob3JpdHkwHhcNMDcxMDI0MjA1NDE3WhcNMTcxMDI0MjA1NDE3WjCB -jDELMAkGA1UEBhMCSUwxFjAUBgNVBAoTDVN0YXJ0Q29tIEx0ZC4xKzApBgNVBAsT -IlNlY3VyZSBEaWdpdGFsIENlcnRpZmljYXRlIFNpZ25pbmcxODA2BgNVBAMTL1N0 -YXJ0Q29tIENsYXNzIDEgUHJpbWFyeSBJbnRlcm1lZGlhdGUgU2VydmVyIENBMIIB -IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtonGrO8JUngHrJJj0PREGBiE -gFYfka7hh/oyULTTRwbw5gdfcA4Q9x3AzhA2NIVaD5Ksg8asWFI/ujjo/OenJOJA -pgh2wJJuniptTT9uYSAK21ne0n1jsz5G/vohURjXzTCm7QduO3CHtPn66+6CPAVv -kvek3AowHpNz/gfK11+AnSJYUq4G2ouHI2mw5CrY6oPSvfNx23BaKA+vWjhwRRI/ -ME3NO68X5Q/LoKldSKqxYVDLNM08XMML6BDAjJvwAwNi/rJsPnIO7hxDKslIDlc5 -xDEhyBDBLIf+VJVSH1I8MRKbf+fAoKVZ1eKPPvDVqOHXcDGpxLPPr21TLwb0pwID -AQABo4IBrTCCAakwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYD -VR0OBBYEFOtCNNCYsKuf9BtrCPfMZC7vDixFMB8GA1UdIwQYMBaAFE4L7xqkQFul -F2mHMMo0aEPQQa7yMGYGCCsGAQUFBwEBBFowWDAnBggrBgEFBQcwAYYbaHR0cDov -L29jc3Auc3RhcnRzc2wuY29tL2NhMC0GCCsGAQUFBzAChiFodHRwOi8vd3d3LnN0 -YXJ0c3NsLmNvbS9zZnNjYS5jcnQwWwYDVR0fBFQwUjAnoCWgI4YhaHR0cDovL3d3 -dy5zdGFydHNzbC5jb20vc2ZzY2EuY3JsMCegJaAjhiFodHRwOi8vY3JsLnN0YXJ0 -c3NsLmNvbS9zZnNjYS5jcmwwgYAGA1UdIAR5MHcwdQYLKwYBBAGBtTcBAgEwZjAu -BggrBgEFBQcCARYiaHR0cDovL3d3dy5zdGFydHNzbC5jb20vcG9saWN5LnBkZjA0 -BggrBgEFBQcCARYoaHR0cDovL3d3dy5zdGFydHNzbC5jb20vaW50ZXJtZWRpYXRl -LnBkZjANBgkqhkiG9w0BAQUFAAOCAgEAIQlJPqWIbuALi0jaMU2P91ZXouHTYlfp -tVbzhUV1O+VQHwSL5qBaPucAroXQ+/8gA2TLrQLhxpFy+KNN1t7ozD+hiqLjfDen -xk+PNdb01m4Ge90h2c9W/8swIkn+iQTzheWq8ecf6HWQTd35RvdCNPdFWAwRDYSw -xtpdPvkBnufh2lWVvnQce/xNFE+sflVHfXv0pQ1JHpXo9xLBzP92piVH0PN1Nb6X -t1gW66pceG/sUzCv6gRNzKkC4/C2BBL2MLERPZBOVmTX3DxDX3M570uvh+v2/miI -RHLq0gfGabDBoYvvF0nXYbFFSF87ICHpW7LM9NfpMfULFWE7epTj69m8f5SuauNi -YpaoZHy4h/OZMn6SolK+u/hlz8nyMPyLwcKmltdfieFcNID1j0cHL7SRv7Gifl9L -WtBbnySGBVFaaQNlQ0lxxeBvlDRr9hvYqbBMflPrj0jfyjO1SPo2ShpTpjMM0InN -SRXNiTE8kMBy12VLUjWKRhFEuT2OKGWmPnmeXAhEKa2wNREuIU640ucQPl2Eg7PD -wuTSxv0JS3QJ3fGz0xk+gA2iCxnwOOfFwq/iI9th4p1cbiCJSS4jarJiwUW0n6+L -p/EiO/h94pDQehn7Skzj0n1fSoMD7SfWI55rjbRZotnvbIIp3XUZPD9MEI3vu3Un -0q6Dp6jOW6c= ------END CERTIFICATE-----` - -const startComRoot = `-----BEGIN CERTIFICATE----- -MIIHyTCCBbGgAwIBAgIBATANBgkqhkiG9w0BAQUFADB9MQswCQYDVQQGEwJJTDEW -MBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0YWwg -Q2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3RhcnRDb20gQ2VydGlmaWNh -dGlvbiBBdXRob3JpdHkwHhcNMDYwOTE3MTk0NjM2WhcNMzYwOTE3MTk0NjM2WjB9 -MQswCQYDVQQGEwJJTDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMi -U2VjdXJlIERpZ2l0YWwgQ2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3Rh -cnRDb20gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUA -A4ICDwAwggIKAoICAQDBiNsJvGxGfHiflXu1M5DycmLWwTYgIiRezul38kMKogZk -pMyONvg45iPwbm2xPN1yo4UcodM9tDMr0y+v/uqwQVlntsQGfQqedIXWeUyAN3rf -OQVSWff0G0ZDpNKFhdLDcfN1YjS6LIp/Ho/u7TTQEceWzVI9ujPW3U3eCztKS5/C -Ji/6tRYccjV3yjxd5srhJosaNnZcAdt0FCX+7bWgiA/deMotHweXMAEtcnn6RtYT -Kqi5pquDSR3l8u/d5AGOGAqPY1MWhWKpDhk6zLVmpsJrdAfkK+F2PrRt2PZE4XNi -HzvEvqBTViVsUQn3qqvKv3b9bZvzndu/PWa8DFaqr5hIlTpL36dYUNk4dalb6kMM -Av+Z6+hsTXBbKWWc3apdzK8BMewM69KN6Oqce+Zu9ydmDBpI125C4z/eIT574Q1w -+2OqqGwaVLRcJXrJosmLFqa7LH4XXgVNWG4SHQHuEhANxjJ/GP/89PrNbpHoNkm+ -Gkhpi8KWTRoSsmkXwQqQ1vp5Iki/untp+HDH+no32NgN0nZPV/+Qt+OR0t3vwmC3 -Zzrd/qqc8NSLf3Iizsafl7b4r4qgEKjZ+xjGtrVcUjyJthkqcwEKDwOzEmDyei+B -26Nu/yYwl/WL3YlXtq09s68rxbd2AvCl1iuahhQqcvbjM4xdCUsT37uMdBNSSwID -AQABo4ICUjCCAk4wDAYDVR0TBAUwAwEB/zALBgNVHQ8EBAMCAa4wHQYDVR0OBBYE -FE4L7xqkQFulF2mHMMo0aEPQQa7yMGQGA1UdHwRdMFswLKAqoCiGJmh0dHA6Ly9j -ZXJ0LnN0YXJ0Y29tLm9yZy9zZnNjYS1jcmwuY3JsMCugKaAnhiVodHRwOi8vY3Js -LnN0YXJ0Y29tLm9yZy9zZnNjYS1jcmwuY3JsMIIBXQYDVR0gBIIBVDCCAVAwggFM -BgsrBgEEAYG1NwEBATCCATswLwYIKwYBBQUHAgEWI2h0dHA6Ly9jZXJ0LnN0YXJ0 -Y29tLm9yZy9wb2xpY3kucGRmMDUGCCsGAQUFBwIBFilodHRwOi8vY2VydC5zdGFy -dGNvbS5vcmcvaW50ZXJtZWRpYXRlLnBkZjCB0AYIKwYBBQUHAgIwgcMwJxYgU3Rh -cnQgQ29tbWVyY2lhbCAoU3RhcnRDb20pIEx0ZC4wAwIBARqBl0xpbWl0ZWQgTGlh -YmlsaXR5LCByZWFkIHRoZSBzZWN0aW9uICpMZWdhbCBMaW1pdGF0aW9ucyogb2Yg -dGhlIFN0YXJ0Q29tIENlcnRpZmljYXRpb24gQXV0aG9yaXR5IFBvbGljeSBhdmFp -bGFibGUgYXQgaHR0cDovL2NlcnQuc3RhcnRjb20ub3JnL3BvbGljeS5wZGYwEQYJ -YIZIAYb4QgEBBAQDAgAHMDgGCWCGSAGG+EIBDQQrFilTdGFydENvbSBGcmVlIFNT -TCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTANBgkqhkiG9w0BAQUFAAOCAgEAFmyZ -9GYMNPXQhV59CuzaEE44HF7fpiUFS5Eyweg78T3dRAlbB0mKKctmArexmvclmAk8 -jhvh3TaHK0u7aNM5Zj2gJsfyOZEdUauCe37Vzlrk4gNXcGmXCPleWKYK34wGmkUW -FjgKXlf2Ysd6AgXmvB618p70qSmD+LIU424oh0TDkBreOKk8rENNZEXO3SipXPJz -ewT4F+irsfMuXGRuczE6Eri8sxHkfY+BUZo7jYn0TZNmezwD7dOaHZrzZVD1oNB1 -ny+v8OqCQ5j4aZyJecRDjkZy42Q2Eq/3JR44iZB3fsNrarnDy0RLrHiQi+fHLB5L -EUTINFInzQpdn4XBidUaePKVEFMy3YCEZnXZtWgo+2EuvoSoOMCZEoalHmdkrQYu -L6lwhceWD3yJZfWOQ1QOq92lgDmUYMA0yZZwLKMS9R9Ie70cfmu3nZD0Ijuu+Pwq -yvqCUqDvr0tVk+vBtfAii6w0TiYiBKGHLHVKt+V9E9e4DGTANtLJL4YSjCMJwRuC -O3NJo2pXh5Tl1njFmUNj403gdy3hZZlyaQQaRwnmDwFWJPsfvw55qVguucQJAX6V -um0ABj6y6koQOdjQK/W/7HW/lwLFCRsI3FU34oH7N4RDYiDK51ZLZer+bMEkkySh -NOsF/5oirpt9P/FlUQqmMGqz9IgcgA38corog14= ------END CERTIFICATE-----` - const smimeLeaf = `-----BEGIN CERTIFICATE----- MIIIPDCCBiSgAwIBAgIQaMDxFS0pOMxZZeOBxoTJtjANBgkqhkiG9w0BAQsFADCB nTELMAkGA1UEBhMCRVMxFDASBgNVBAoMC0laRU5QRSBTLkEuMTowOAYDVQQLDDFB @@ -971,168 +808,136 @@ QyYBNWNgVYkDOnXYukrZVP/u3oDYLdE41V4tC5h9Pmzb/CaIxw== -----END CERTIFICATE-----` var nameConstraintsLeaf = `-----BEGIN CERTIFICATE----- -MIIHMTCCBRmgAwIBAgIIIZaV/3ezOJkwDQYJKoZIhvcNAQEFBQAwgcsxCzAJBgNV -BAYTAlVTMREwDwYDVQQIEwhWaXJnaW5pYTETMBEGA1UEBxMKQmxhY2tzYnVyZzEj -MCEGA1UECxMaR2xvYmFsIFF1YWxpZmllZCBTZXJ2ZXIgQ0ExPDA6BgNVBAoTM1Zp -cmdpbmlhIFBvbHl0ZWNobmljIEluc3RpdHV0ZSBhbmQgU3RhdGUgVW5pdmVyc2l0 -eTExMC8GA1UEAxMoVmlyZ2luaWEgVGVjaCBHbG9iYWwgUXVhbGlmaWVkIFNlcnZl -ciBDQTAeFw0xMzA5MTkxNDM2NTVaFw0xNTA5MTkxNDM2NTVaMIHNMQswCQYDVQQG -EwJVUzERMA8GA1UECAwIVmlyZ2luaWExEzARBgNVBAcMCkJsYWNrc2J1cmcxPDA6 -BgNVBAoMM1ZpcmdpbmlhIFBvbHl0ZWNobmljIEluc3RpdHV0ZSBhbmQgU3RhdGUg -VW5pdmVyc2l0eTE7MDkGA1UECwwyVGVjaG5vbG9neS1lbmhhbmNlZCBMZWFybmlu -ZyBhbmQgT25saW5lIFN0cmF0ZWdpZXMxGzAZBgNVBAMMEnNlY3VyZS5pZGRsLnZ0 -LmVkdTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKkOyPpsOK/6IuPG -WnIBlVwlHzeYf+cUlggqkLq0b0+vZbiTXgio9/VCuNQ8opSoss7J7o3ygV9to+9Y -YwJKVC5WDT/y5JWpQey0CWILymViJnpNSwnxBc8A+Q8w5NUGDd/UhtPx/U8/hqbd -WPDYj2hbOqyq8UlRhfS5pwtnv6BbCTaY11I6FhCLK7zttISyTuWCf9p9o/ggiipP -ii/5oh4dkl+r5SfuSp5GPNHlYO8lWqys5NAPoDD4fc/kuflcK7Exx7XJ+Oqu0W0/ -psjEY/tES1ZgDWU/ParcxxFpFmKHbD5DXsfPOObzkVWXIY6tGMutSlE1Froy/Nn0 -OZsAOrcCAwEAAaOCAhMwggIPMIG4BggrBgEFBQcBAQSBqzCBqDBYBggrBgEFBQcw -AoZMaHR0cDovL3d3dy5wa2kudnQuZWR1L2dsb2JhbHF1YWxpZmllZHNlcnZlci9j -YWNlcnQvZ2xvYmFscXVhbGlmaWVkc2VydmVyLmNydDBMBggrBgEFBQcwAYZAaHR0 -cDovL3Z0Y2EtcC5lcHJvdi5zZXRpLnZ0LmVkdTo4MDgwL2VqYmNhL3B1YmxpY3dl -Yi9zdGF0dXMvb2NzcDAdBgNVHQ4EFgQUp7xbO6iHkvtZbPE4jmndmnAbSEcwDAYD -VR0TAQH/BAIwADAfBgNVHSMEGDAWgBS8YmAn1eM1SBfpS6tFatDIqHdxjDBqBgNV -HSAEYzBhMA4GDCsGAQQBtGgFAgICATAOBgwrBgEEAbRoBQICAQEwPwYMKwYBBAG0 -aAUCAgMBMC8wLQYIKwYBBQUHAgEWIWh0dHA6Ly93d3cucGtpLnZ0LmVkdS9nbG9i -YWwvY3BzLzBKBgNVHR8EQzBBMD+gPaA7hjlodHRwOi8vd3d3LnBraS52dC5lZHUv -Z2xvYmFscXVhbGlmaWVkc2VydmVyL2NybC9jYWNybC5jcmwwDgYDVR0PAQH/BAQD -AgTwMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAdBgNVHREEFjAUghJz -ZWN1cmUuaWRkbC52dC5lZHUwDQYJKoZIhvcNAQEFBQADggIBAEgoYo4aUtatY3gI -OyyKp7QlIOaLbTJZywESHqy+L5EGDdJW2DJV+mcE0LDGvqa2/1Lo+AR1ntsZwfOi -Y718JwgVVaX/RCd5+QKP25c5/x72xI8hb/L1bgS0ED9b0YAhd7Qm1K1ot82+6mqX -DW6WiGeDr8Z07MQ3143qQe2rBlq+QI69DYzm2GOqAIAnUIWv7tCyLUm31b4DwmrJ -TeudVreTKUbBNB1TWRFHEPkWhjjXKZnNGRO11wHXcyBu6YekIvVZ+vmx8ePee4jJ -3GFOi7lMuWOeq57jTVL7KOKaKLVXBb6gqo5aq+Wwt8RUD5MakrCAEeQZj7DKaFmZ -oQCO0Pxrsl3InCGvxnGzT+bFVO9nJ/BAMj7hknFdm9Jr6Bg5q33Z+gnf909AD9QF -ESqUSykaHu2LVdJx2MaCH1CyKnRgMw5tEwE15EXpUjCm24m8FMOYC+rNtf18pgrz -5D8Jhh+oxK9PjcBYqXNtnioIxiMCYcV0q5d4w4BYFEh71tk7/bYB0R55CsBUVPmp -timWNOdRd57Tfpk3USaVsumWZAf9MP3wPiC7gb4d5tYEEAG5BuDT8ruFw838wU8G -1VvAVutSiYBg7k3NYO7AUqZ+Ax4klQX3aM9lgonmJ78Qt94UPtbptrfZ4/lSqEf8 -GBUwDrQNTb+gsXsDkjd5lcYxNx6l +MIIG+jCCBOKgAwIBAgIQWj9gbtPPkZs65N6TKyutRjANBgkqhkiG9w0BAQsFADCB +yzELMAkGA1UEBhMCVVMxETAPBgNVBAgTCFZpcmdpbmlhMRMwEQYDVQQHEwpCbGFj +a3NidXJnMSMwIQYDVQQLExpHbG9iYWwgUXVhbGlmaWVkIFNlcnZlciBDQTE8MDoG +A1UEChMzVmlyZ2luaWEgUG9seXRlY2huaWMgSW5zdGl0dXRlIGFuZCBTdGF0ZSBV +bml2ZXJzaXR5MTEwLwYDVQQDEyhWaXJnaW5pYSBUZWNoIEdsb2JhbCBRdWFsaWZp +ZWQgU2VydmVyIENBMB4XDTE4MDQyNjE5NDU1M1oXDTE5MTIxMDAwMDAwMFowgZAx +CzAJBgNVBAYTAlVTMREwDwYDVQQIEwhWaXJnaW5pYTETMBEGA1UEBxMKQmxhY2tz +YnVyZzE8MDoGA1UEChMzVmlyZ2luaWEgUG9seXRlY2huaWMgSW5zdGl0dXRlIGFu +ZCBTdGF0ZSBVbml2ZXJzaXR5MRswGQYDVQQDExJ1ZGN0ZXN0LmFkcy52dC5lZHUw +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCcoVBeV3AzdSGMzRWH0tuM +VluEj+sq4r9PuLDBAdgjjHi4ED8npT2/fgOalswInXspRvFS+pkEwTrmeZ7HPzRJ +HUE5YlX5Nc6WI8ZXPVg5E6GyoMy6gNlALwqsIvDCvqxBMc39oG6yOuGmQXdF6s0N +BJMrXc4aPz60s4QMWNO2OHL0pmnZqE1TxYRBHUY/dk3cfsIepIDDuSxRsNE/P/MI +pxm/uVOyiLEnPmOMsL430SZ7nC8PxUMqya9ok6Zaf7k54g7JJXDjE96VMCjMszIv +Ud9qe1PbokTOxlG/4QW7Qm0dPbiJhTUuoBzVAxzlOOkSFdXqSYKjC9tFcbr8y+pT +AgMBAAGjggIRMIICDTCBtgYIKwYBBQUHAQEEgakwgaYwXwYIKwYBBQUHMAKGU2h0 +dHA6Ly93d3cucGtpLnZ0LmVkdS9nbG9iYWxxdWFsaWZpZWRzZXJ2ZXIvY2FjZXJ0 +L2dsb2JhbHF1YWxpZmllZHNlcnZlcl9zaGEyNTYuY3J0MEMGCCsGAQUFBzABhjdo +dHRwOi8vdnRjYS5wa2kudnQuZWR1OjgwODAvZWpiY2EvcHVibGljd2ViL3N0YXR1 +cy9vY3NwMB0GA1UdDgQWBBSzDLXee0wbgXpVQxvBQCophQDZbTAMBgNVHRMBAf8E +AjAAMB8GA1UdIwQYMBaAFLxiYCfV4zVIF+lLq0Vq0Miod3GMMGoGA1UdIARjMGEw +DgYMKwYBBAG0aAUCAgIBMA4GDCsGAQQBtGgFAgIBATA/BgwrBgEEAbRoBQICAwEw +LzAtBggrBgEFBQcCARYhaHR0cDovL3d3dy5wa2kudnQuZWR1L2dsb2JhbC9jcHMv +MEoGA1UdHwRDMEEwP6A9oDuGOWh0dHA6Ly93d3cucGtpLnZ0LmVkdS9nbG9iYWxx +dWFsaWZpZWRzZXJ2ZXIvY3JsL2NhY3JsLmNybDAOBgNVHQ8BAf8EBAMCBeAwHQYD +VR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMB0GA1UdEQQWMBSCEnVkY3Rlc3Qu +YWRzLnZ0LmVkdTANBgkqhkiG9w0BAQsFAAOCAgEAD79kuyZbwQJCSBOVq9lA0lj4 +juHM7RMBfp2GuWvhk5F90OMKQCNdITva3oq4uQzt013TtwposYXq/d0Jobk6RHxj +OJzRZVvEPsXLvKm8oLhz7/qgI8gcVeJFR9WgdNhjN1upn++EnABHUdDR77fgixuH +FFwNC0WSZ6G0+WgYV7MKD4jYWh1DXEaJtQCN763IaWGxgvQaLUwS423xgwsx+8rw +hCRYns5u8myTbUlEu2b+GYimiogtDFMT01A7y88vKl9g+3bx42dJHQNNmSzmYPfs +IljtQbVwJIyNL/rwjoz7BTk8e9WY0qUK7ZYh+oGK8kla8yfPKtkvOJV29KdFKzTm +42kNm6cH+U5gGwEEg+Xj66Q2yFH5J9kAoBazTepgQ/13wwTY0mU9PtKVBtMH5Y/u +MoNVZz6p7vWWRrY5qSXIaW9qyF3bZnmPEHHYTplWsyAyh8blGlqPnpayDflPiQF/ +9y37kax5yhT0zPZW1ZwIZ5hDTO7pu5i83bYh3pzhvJNHtv74Nn/SX1dTZrWBi/HG +OSWK3CLz8qAEBe72XGoBjBzuk9VQxg6k52qjxCyYf7CBSQpTZhsNMf0gzu+JNATc +b+XaOqJT6uI/RfqAJVe16ZeXZIFZgQlzIwRS9vobq9fqTIpH/QxqgXROGqAlbBVp +/ByH6FEe6+oH1UCklhg= -----END CERTIFICATE-----` var nameConstraintsIntermediate1 = `-----BEGIN CERTIFICATE----- -MIINLjCCDBagAwIBAgIRIqpyf/YoGgvHc8HiDAxAI8owDQYJKoZIhvcNAQEFBQAw -XDELMAkGA1UEBhMCQkUxFTATBgNVBAsTDFRydXN0ZWQgUm9vdDEZMBcGA1UEChMQ -R2xvYmFsU2lnbiBudi1zYTEbMBkGA1UEAxMSVHJ1c3RlZCBSb290IENBIEcyMB4X -DTEyMTIxMzAwMDAwMFoXDTE3MTIxMzAwMDAwMFowgcsxCzAJBgNVBAYTAlVTMREw -DwYDVQQIEwhWaXJnaW5pYTETMBEGA1UEBxMKQmxhY2tzYnVyZzEjMCEGA1UECxMa -R2xvYmFsIFF1YWxpZmllZCBTZXJ2ZXIgQ0ExPDA6BgNVBAoTM1ZpcmdpbmlhIFBv -bHl0ZWNobmljIEluc3RpdHV0ZSBhbmQgU3RhdGUgVW5pdmVyc2l0eTExMC8GA1UE -AxMoVmlyZ2luaWEgVGVjaCBHbG9iYWwgUXVhbGlmaWVkIFNlcnZlciBDQTCCAiIw -DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALgIZhEaptBWADBqdJ45ueFGzMXa -GHnzNxoxR1fQIaaRQNdCg4cw3A4dWKMeEgYLtsp65ai3Xfw62Qaus0+KJ3RhgV+r -ihqK81NUzkls78fJlADVDI4fCTlothsrE1CTOMiy97jKHai5mVTiWxmcxpmjv7fm -5Nhc+uHgh2hIz6npryq495mD51ZrUTIaqAQN6Pw/VHfAmR524vgriTOjtp1t4lA9 -pXGWjF/vkhAKFFheOQSQ00rngo2wHgCqMla64UTN0oz70AsCYNZ3jDLx0kOP0YmM -R3Ih91VA63kLqPXA0R6yxmmhhxLZ5bcyAy1SLjr1N302MIxLM/pSy6aquEnbELhz -qyp9yGgRyGJay96QH7c4RJY6gtcoPDbldDcHI9nXngdAL4DrZkJ9OkDkJLyqG66W -ZTF5q4EIs6yMdrywz0x7QP+OXPJrjYpbeFs6tGZCFnWPFfmHCRJF8/unofYrheq+ -9J7Jx3U55S/k57NXbAM1RAJOuMTlfn9Etf9Dpoac9poI4Liav6rBoUQk3N3JWqnV -HNx/NdCyJ1/6UbKMJUZsStAVglsi6lVPo289HHOE4f7iwl3SyekizVOp01wUin3y -cnbZB/rXmZbwapSxTTSBf0EIOr9i4EGfnnhCAVA9U5uLrI5OEB69IY8PNX0071s3 -Z2a2fio5c8m3JkdrAgMBAAGjggh5MIIIdTAOBgNVHQ8BAf8EBAMCAQYwTAYDVR0g -BEUwQzBBBgkrBgEEAaAyATwwNDAyBggrBgEFBQcCARYmaHR0cHM6Ly93d3cuZ2xv -YmFsc2lnbi5jb20vcmVwb3NpdG9yeS8wEgYDVR0TAQH/BAgwBgEB/wIBADCCBtAG -A1UdHgSCBscwggbDoIIGvzASghAzZGJsYWNrc2J1cmcub3JnMBiCFmFjY2VsZXJh -dGV2aXJnaW5pYS5jb20wGIIWYWNjZWxlcmF0ZXZpcmdpbmlhLm9yZzALgglhY3Zj -cC5vcmcwCYIHYmV2Lm5ldDAJggdiZXYub3JnMAuCCWNsaWdzLm9yZzAMggpjbWl3 -ZWIub3JnMBeCFWVhc3Rlcm5icm9va3Ryb3V0Lm5ldDAXghVlYXN0ZXJuYnJvb2t0 -cm91dC5vcmcwEYIPZWNvcnJpZG9ycy5pbmZvMBOCEWVkZ2FycmVzZWFyY2gub3Jn -MBKCEGdldC1lZHVjYXRlZC5jb20wE4IRZ2V0LWVkdWNhdGVkLmluZm8wEYIPZ2V0 -ZWR1Y2F0ZWQubmV0MBKCEGdldC1lZHVjYXRlZC5uZXQwEYIPZ2V0ZWR1Y2F0ZWQu -b3JnMBKCEGdldC1lZHVjYXRlZC5vcmcwD4INaG9raWVjbHViLmNvbTAQgg5ob2tp -ZXBob3RvLmNvbTAPgg1ob2tpZXNob3AuY29tMBGCD2hva2llc3BvcnRzLmNvbTAS -ghBob2tpZXRpY2tldHMuY29tMBKCEGhvdGVscm9hbm9rZS5jb20wE4IRaHVtYW53 -aWxkbGlmZS5vcmcwF4IVaW5uYXR2aXJnaW5pYXRlY2guY29tMA+CDWlzY2hwMjAx -MS5vcmcwD4INbGFuZHJlaGFiLm9yZzAggh5uYXRpb25hbHRpcmVyZXNlYXJjaGNl -bnRlci5jb20wFYITbmV0d29ya3ZpcmdpbmlhLm5ldDAMggpwZHJjdnQuY29tMBiC -FnBldGVkeWVyaXZlcmNvdXJzZS5jb20wDYILcmFkaW9pcS5vcmcwFYITcml2ZXJj -b3Vyc2Vnb2xmLmNvbTALgglzZGltaS5vcmcwEIIOc292YW1vdGlvbi5jb20wHoIc -c3VzdGFpbmFibGUtYmlvbWF0ZXJpYWxzLmNvbTAeghxzdXN0YWluYWJsZS1iaW9t -YXRlcmlhbHMub3JnMBWCE3RoaXNpc3RoZWZ1dHVyZS5jb20wGIIWdGhpcy1pcy10 -aGUtZnV0dXJlLmNvbTAVghN0aGlzaXN0aGVmdXR1cmUubmV0MBiCFnRoaXMtaXMt -dGhlLWZ1dHVyZS5uZXQwCoIIdmFkcy5vcmcwDIIKdmFsZWFmLm9yZzANggt2YXRl -Y2guaW5mbzANggt2YXRlY2gubW9iaTAcghp2YXRlY2hsaWZlbG9uZ2xlYXJuaW5n -LmNvbTAcghp2YXRlY2hsaWZlbG9uZ2xlYXJuaW5nLm5ldDAcghp2YXRlY2hsaWZl -bG9uZ2xlYXJuaW5nLm9yZzAKggh2Y29tLmVkdTASghB2aXJnaW5pYXZpZXcubmV0 -MDSCMnZpcmdpbmlhcG9seXRlY2huaWNpbnN0aXR1dGVhbmRzdGF0ZXVuaXZlcnNp -dHkuY29tMDWCM3ZpcmdpbmlhcG9seXRlY2huaWNpbnN0aXR1dGVhbmRzdGF0ZXVu -aXZlcnNpdHkuaW5mbzA0gjJ2aXJnaW5pYXBvbHl0ZWNobmljaW5zdGl0dXRlYW5k -c3RhdGV1bml2ZXJzaXR5Lm5ldDA0gjJ2aXJnaW5pYXBvbHl0ZWNobmljaW5zdGl0 -dXRlYW5kc3RhdGV1bml2ZXJzaXR5Lm9yZzAZghd2aXJnaW5pYXB1YmxpY3JhZGlv -Lm9yZzASghB2aXJnaW5pYXRlY2guZWR1MBOCEXZpcmdpbmlhdGVjaC5tb2JpMByC -GnZpcmdpbmlhdGVjaGZvdW5kYXRpb24ub3JnMAiCBnZ0LmVkdTALggl2dGFyYy5v -cmcwDIIKdnQtYXJjLm9yZzALggl2dGNyYy5jb20wCoIIdnRpcC5vcmcwDIIKdnRs -ZWFuLm9yZzAWghR2dGtub3dsZWRnZXdvcmtzLmNvbTAYghZ2dGxpZmVsb25nbGVh -cm5pbmcuY29tMBiCFnZ0bGlmZWxvbmdsZWFybmluZy5uZXQwGIIWdnRsaWZlbG9u -Z2xlYXJuaW5nLm9yZzATghF2dHNwb3J0c21lZGlhLmNvbTALggl2dHdlaS5jb20w -D4INd2l3YXR3ZXJjLmNvbTAKggh3dnRmLm9yZzAIgQZ2dC5lZHUwd6R1MHMxCzAJ -BgNVBAYTAlVTMREwDwYDVQQIEwhWaXJnaW5pYTETMBEGA1UEBxMKQmxhY2tzYnVy -ZzE8MDoGA1UEChMzVmlyZ2luaWEgUG9seXRlY2huaWMgSW5zdGl0dXRlIGFuZCBT -dGF0ZSBVbml2ZXJzaXR5MCcGA1UdJQQgMB4GCCsGAQUFBwMCBggrBgEFBQcDAQYI -KwYBBQUHAwkwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDovL2NybC5nbG9iYWxzaWdu -LmNvbS9ncy90cnVzdHJvb3RnMi5jcmwwgYQGCCsGAQUFBwEBBHgwdjAzBggrBgEF -BQcwAYYnaHR0cDovL29jc3AyLmdsb2JhbHNpZ24uY29tL3RydXN0cm9vdGcyMD8G -CCsGAQUFBzAChjNodHRwOi8vc2VjdXJlLmdsb2JhbHNpZ24uY29tL2NhY2VydC90 -cnVzdHJvb3RnMi5jcnQwHQYDVR0OBBYEFLxiYCfV4zVIF+lLq0Vq0Miod3GMMB8G -A1UdIwQYMBaAFBT25YsxtkWASkxt/MKHico2w5BiMA0GCSqGSIb3DQEBBQUAA4IB -AQAyJm/lOB2Er4tHXhc/+fSufSzgjohJgYfMkvG4LknkvnZ1BjliefR8tTXX49d2 -SCDFWfGjqyJZwavavkl/4p3oXPG/nAMDMvxh4YAT+CfEK9HH+6ICV087kD4BLegi -+aFJMj8MMdReWCzn5sLnSR1rdse2mo2arX3Uod14SW+PGrbUmTuWNyvRbz3fVmxp -UdbGmj3laknO9YPsBGgHfv73pVVsTJkW4ZfY/7KdD/yaVv6ophpOB3coXfjl2+kd -Z4ypn2zK+cx9IL/LSewqd/7W9cD55PCUy4X9OTbEmAccwiz3LB66mQoUGfdHdkoB -jUY+v9vLQXmaVwI0AYL7g9LN +MIIHVTCCBj2gAwIBAgINAecHzcaPEeFvu7X4TTANBgkqhkiG9w0BAQsFADBjMQsw +CQYDVQQGEwJCRTEVMBMGA1UECxMMVHJ1c3RlZCBSb290MRkwFwYDVQQKExBHbG9i +YWxTaWduIG52LXNhMSIwIAYDVQQDExlUcnVzdGVkIFJvb3QgQ0EgU0hBMjU2IEcy +MB4XDTE3MTIwNjAwMDAwMFoXDTIyMTIwNjAwMDAwMFowgcsxCzAJBgNVBAYTAlVT +MREwDwYDVQQIEwhWaXJnaW5pYTETMBEGA1UEBxMKQmxhY2tzYnVyZzEjMCEGA1UE +CxMaR2xvYmFsIFF1YWxpZmllZCBTZXJ2ZXIgQ0ExPDA6BgNVBAoTM1Zpcmdpbmlh +IFBvbHl0ZWNobmljIEluc3RpdHV0ZSBhbmQgU3RhdGUgVW5pdmVyc2l0eTExMC8G +A1UEAxMoVmlyZ2luaWEgVGVjaCBHbG9iYWwgUXVhbGlmaWVkIFNlcnZlciBDQTCC +AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALgIZhEaptBWADBqdJ45ueFG +zMXaGHnzNxoxR1fQIaaRQNdCg4cw3A4dWKMeEgYLtsp65ai3Xfw62Qaus0+KJ3Rh +gV+rihqK81NUzkls78fJlADVDI4fCTlothsrE1CTOMiy97jKHai5mVTiWxmcxpmj +v7fm5Nhc+uHgh2hIz6npryq495mD51ZrUTIaqAQN6Pw/VHfAmR524vgriTOjtp1t +4lA9pXGWjF/vkhAKFFheOQSQ00rngo2wHgCqMla64UTN0oz70AsCYNZ3jDLx0kOP +0YmMR3Ih91VA63kLqPXA0R6yxmmhhxLZ5bcyAy1SLjr1N302MIxLM/pSy6aquEnb +ELhzqyp9yGgRyGJay96QH7c4RJY6gtcoPDbldDcHI9nXngdAL4DrZkJ9OkDkJLyq +G66WZTF5q4EIs6yMdrywz0x7QP+OXPJrjYpbeFs6tGZCFnWPFfmHCRJF8/unofYr +heq+9J7Jx3U55S/k57NXbAM1RAJOuMTlfn9Etf9Dpoac9poI4Liav6rBoUQk3N3J +WqnVHNx/NdCyJ1/6UbKMJUZsStAVglsi6lVPo289HHOE4f7iwl3SyekizVOp01wU +in3ycnbZB/rXmZbwapSxTTSBf0EIOr9i4EGfnnhCAVA9U5uLrI5OEB69IY8PNX00 +71s3Z2a2fio5c8m3JkdrAgMBAAGjggKdMIICmTAOBgNVHQ8BAf8EBAMCAQYwHQYD +VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMBIGA1UdEwEB/wQIMAYBAf8CAQAw +HQYDVR0OBBYEFLxiYCfV4zVIF+lLq0Vq0Miod3GMMB8GA1UdIwQYMBaAFMhjmwhp +VMKYyNnN4zO3UF74yQGbMIGNBggrBgEFBQcBAQSBgDB+MDcGCCsGAQUFBzABhito +dHRwOi8vb2NzcDIuZ2xvYmFsc2lnbi5jb20vdHJ1c3Ryb290c2hhMmcyMEMGCCsG +AQUFBzAChjdodHRwOi8vc2VjdXJlLmdsb2JhbHNpZ24uY29tL2NhY2VydC90cnVz +dHJvb3RzaGEyZzIuY3J0MIHyBgNVHR4EgeowgeeggbIwCIEGdnQuZWR1MAmCB2Jl +di5uZXQwCoIIdmNvbS5lZHUwCIIGdnQuZWR1MAyCCnZ0Y2dpdC5jb20wd6R1MHMx +CzAJBgNVBAYTAlVTMREwDwYDVQQIEwhWaXJnaW5pYTETMBEGA1UEBxMKQmxhY2tz +YnVyZzE8MDoGA1UEChMzVmlyZ2luaWEgUG9seXRlY2huaWMgSW5zdGl0dXRlIGFu +ZCBTdGF0ZSBVbml2ZXJzaXR5oTAwCocIAAAAAAAAAAAwIocgAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAwQQYDVR0fBDowODA2oDSgMoYwaHR0cDovL2Ny +bC5nbG9iYWxzaWduLmNvbS9ncy90cnVzdHJvb3RzaGEyZzIuY3JsMEwGA1UdIARF +MEMwQQYJKwYBBAGgMgE8MDQwMgYIKwYBBQUHAgEWJmh0dHBzOi8vd3d3Lmdsb2Jh +bHNpZ24uY29tL3JlcG9zaXRvcnkvMA0GCSqGSIb3DQEBCwUAA4IBAQArHocpEKTv +DW1Hw0USj60KN96aLJXTLm05s0LbjloeTePtDFtuisrbE85A0IhCwxdIl/VsQMZB +7mQZBEmLzR+NK1/Luvs7C6WTmkqrE8H7D73dSOab5fMZIXS91V/aEtEQGpJMhwi1 +svd9TiiQrVkagrraeRWmTTz9BtUA3CeujuW2tShxF1ew4Q4prYw97EsE4HnKDJtu +RtyTqKsuh/rRvKMmgUdEPZbVI23yzUKhi/mTbyml/35x/f6f5p7OYIKcQ/34sts8 +xoW9dfkWBQKAXCstXat3WJVilGXBFub6GoVZdnxTDipyMZhUT/vzXq2bPphjcdR5 +YGbmwyYmChfa -----END CERTIFICATE-----` var nameConstraintsIntermediate2 = `-----BEGIN CERTIFICATE----- -MIIEXTCCA0WgAwIBAgILBAAAAAABNuk6OrMwDQYJKoZIhvcNAQEFBQAwVzELMAkG -A1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExEDAOBgNVBAsTB1Jv -b3QgQ0ExGzAZBgNVBAMTEkdsb2JhbFNpZ24gUm9vdCBDQTAeFw0xMjA0MjUxMTAw -MDBaFw0yNzA0MjUxMTAwMDBaMFwxCzAJBgNVBAYTAkJFMRUwEwYDVQQLEwxUcnVz -dGVkIFJvb3QxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExGzAZBgNVBAMTElRy -dXN0ZWQgUm9vdCBDQSBHMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB -AKyuvqrtcMr7g7EuNbu4sKwxM127UsCmx1RxbxxgcArGS7rjiefpBH/w4LYrymjf -vcw1ueyMNoqLo9nJMz/ORXupb35NNfE667prQYHa+tTjl1IiKpB7QUwt3wXPuTMF -Ja1tXtjKzkqJyuJlNuPKT76HcjgNqgV1s9qG44MD5I2JvI12du8zI1bgdQ+l/KsX -kTfbGjUvhOLOlVNWVQDpL+YMIrGqgBYxy5TUNgrAcRtwpNdS2KkF5otSmMweVb5k -hoUVv3u8UxQH/WWbNhHq1RrIlg/0rBUfi/ziShYFSB7U+aLx5DxPphTFBiDquQGp -tB+FC4JvnukDStFihZCZ1R8CAwEAAaOCASMwggEfMA4GA1UdDwEB/wQEAwIBBjAP -BgNVHRMBAf8EBTADAQH/MEcGA1UdIARAMD4wPAYEVR0gADA0MDIGCCsGAQUFBwIB -FiZodHRwczovL3d3dy5nbG9iYWxzaWduLmNvbS9yZXBvc2l0b3J5LzAdBgNVHQ4E -FgQUFPblizG2RYBKTG38woeJyjbDkGIwMwYDVR0fBCwwKjAooCagJIYiaHR0cDov -L2NybC5nbG9iYWxzaWduLm5ldC9yb290LmNybDA+BggrBgEFBQcBAQQyMDAwLgYI -KwYBBQUHMAGGImh0dHA6Ly9vY3NwMi5nbG9iYWxzaWduLmNvbS9yb290cjEwHwYD -VR0jBBgwFoAUYHtmGkUNl8qJUC99BM00qP/8/UswDQYJKoZIhvcNAQEFBQADggEB -AL7IG0l+k4LkcpI+a/kvZsSRwSM4uA6zGX34e78A2oytr8RG8bJwVb8+AHMUD+Xe -2kYdh/Uj/waQXfqR0OgxQXL9Ct4ZM+JlR1avsNKXWL5AwYXAXCOB3J5PW2XOck7H -Zw0vRbGQhjWjQx+B4KOUFg1b3ov/z6Xkr3yaCfRQhXh7KC0Bc0RXPPG5Nv5lCW+z -tbbg0zMm3kyfQITRusMSg6IBsDJqOnjaiaKQRcXiD0Sk43ZXb2bUKMxC7+Td3QL4 -RyHcWJbQ7YylLTS/x+jxWIcOQ0oO5/54t5PTQ14neYhOz9x4gUk2AYAW6d1vePwb -hcC8roQwkHT7HvfYBoc74FM= +MIIEXDCCA0SgAwIBAgILBAAAAAABNumCOV0wDQYJKoZIhvcNAQELBQAwTDEgMB4G +A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjMxEzARBgNVBAoTCkdsb2JhbFNp +Z24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTIwNDI1MTEwMDAwWhcNMjcwNDI1 +MTEwMDAwWjBjMQswCQYDVQQGEwJCRTEVMBMGA1UECxMMVHJ1c3RlZCBSb290MRkw +FwYDVQQKExBHbG9iYWxTaWduIG52LXNhMSIwIAYDVQQDExlUcnVzdGVkIFJvb3Qg +Q0EgU0hBMjU2IEcyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz80+ +/Q2PAhLuYwe04YTLBLGKr1/JScHtDvAY5E94GjGxCbSR1/1VhL880UPJyN85tddO +oxZPgtIyZixDvvK+CgpT5webyBBbqK/ap7aoByghAJ7X520XZMRwKA6cEWa6tjCL +WH1zscxQxGzgtV50rn2ux2SapoCPxMpM4+tpEVwWJf3KP3NT+jd9GRaXWgNei5JK +Quo9l+cZkSeuoWijvaer5hcLCufPywMMQd0r6XXIM/l7g9DjMaE24d+fa2bWxQXC +8WT/PZ+D1KUEkdtn/ixADqsoiIibGn7M84EE9/NLjbzPrwROlBUJFz6cuw+II0rZ +8OFFeZ/OkHHYZq2h9wIDAQABo4IBJjCCASIwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud +EwEB/wQFMAMBAf8wRwYDVR0gBEAwPjA8BgRVHSAAMDQwMgYIKwYBBQUHAgEWJmh0 +dHBzOi8vd3d3Lmdsb2JhbHNpZ24uY29tL3JlcG9zaXRvcnkvMB0GA1UdDgQWBBTI +Y5sIaVTCmMjZzeMzt1Be+MkBmzA2BgNVHR8ELzAtMCugKaAnhiVodHRwOi8vY3Js +Lmdsb2JhbHNpZ24ubmV0L3Jvb3QtcjMuY3JsMD4GCCsGAQUFBwEBBDIwMDAuBggr +BgEFBQcwAYYiaHR0cDovL29jc3AyLmdsb2JhbHNpZ24uY29tL3Jvb3RyMzAfBgNV +HSMEGDAWgBSP8Et/qC5FJK5NUPpjmove4t0bvDANBgkqhkiG9w0BAQsFAAOCAQEA +XzbLwBjJiY6j3WEcxD3eVnsIY4pY3bl6660tgpxCuLVx4o1xyiVkS/BcQFD7GIoX +FBRrf5HibO1uSEOw0QZoRwlsio1VPg1PRaccG5C1sB51l/TL1XH5zldZBCnRYrrF +qCPorxi0xoRogj8kqkS2xyzYLElhx9X7jIzfZ8dC4mgOeoCtVvwM9xvmef3n6Vyb +7/hl3w/zWwKxWyKJNaF7tScD5nvtLUzyBpr++aztiyJ1WliWcS6W+V2gKg9rxEC/ +rc2yJS70DvfkPiEnBJ2x2AHZV3yKTALUqurkV705JledqUT9I5frAwYNXZ8pNzde +n+DIcSIo7yKy6MX9czbFWQ== -----END CERTIFICATE-----` var globalSignRoot = `-----BEGIN CERTIFICATE----- -MIIDdTCCAl2gAwIBAgILBAAAAAABFUtaw5QwDQYJKoZIhvcNAQEFBQAwVzELMAkG -A1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExEDAOBgNVBAsTB1Jv -b3QgQ0ExGzAZBgNVBAMTEkdsb2JhbFNpZ24gUm9vdCBDQTAeFw05ODA5MDExMjAw -MDBaFw0yODAxMjgxMjAwMDBaMFcxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9i -YWxTaWduIG52LXNhMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJHbG9iYWxT -aWduIFJvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDaDuaZ -jc6j40+Kfvvxi4Mla+pIH/EqsLmVEQS98GPR4mdmzxzdzxtIK+6NiY6arymAZavp -xy0Sy6scTHAHoT0KMM0VjU/43dSMUBUc71DuxC73/OlS8pF94G3VNTCOXkNz8kHp -1Wrjsok6Vjk4bwY8iGlbKk3Fp1S4bInMm/k8yuX9ifUSPJJ4ltbcdG6TRGHRjcdG -snUOhugZitVtbNV4FpWi6cgKOOvyJBNPc1STE4U6G7weNLWLBYy5d4ux2x8gkasJ -U26Qzns3dLlwR5EiUWMWea6xrkEmCMgZK9FGqkjWZCrXgzT/LCrBbBlDSgeF59N8 -9iFo7+ryUp9/k5DPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E -BTADAQH/MB0GA1UdDgQWBBRge2YaRQ2XyolQL30EzTSo//z9SzANBgkqhkiG9w0B -AQUFAAOCAQEA1nPnfE920I2/7LqivjTFKDK1fPxsnCwrvQmeU79rXqoRSLblCKOz -yj1hTdNGCbM+w6DjY1Ub8rrvrTnhQ7k4o+YviiY776BQVvnGCv04zcQLcFGUl5gE -38NflNUVyRRBnMRddWQVDf9VMOyGj/8N7yy5Y0b2qvzfvGn9LhJIZJrglfCm7ymP -AbEVtQwdpf5pLGkkeB6zpxxxYu7KyJesF12KwvhHhm4qxFYxldBniYUr+WymXUad -DKqC5JlR3XC321Y9YeRq4VzW9v493kHMB65jUr9TU/Qr6cf9tveCX4XSQRjbgbME -HMUfpIBvFSDJ3gyICh3WZlXi/EjJKSZp4A== +MIIDXzCCAkegAwIBAgILBAAAAAABIVhTCKIwDQYJKoZIhvcNAQELBQAwTDEgMB4G +A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjMxEzARBgNVBAoTCkdsb2JhbFNp +Z24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDkwMzE4MTAwMDAwWhcNMjkwMzE4 +MTAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMzETMBEG +A1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAMwldpB5BngiFvXAg7aEyiie/QV2EcWtiHL8 +RgJDx7KKnQRfJMsuS+FggkbhUqsMgUdwbN1k0ev1LKMPgj0MK66X17YUhhB5uzsT +gHeMCOFJ0mpiLx9e+pZo34knlTifBtc+ycsmWQ1z3rDI6SYOgxXG71uL0gRgykmm +KPZpO/bLyCiR5Z2KYVc3rHQU3HTgOu5yLy6c+9C7v/U9AOEGM+iCK65TpjoWc4zd +QQ4gOsC0p6Hpsk+QLjJg6VfLuQSSaGjlOCZgdbKfd/+RFO+uIEn8rUAVSNECMWEZ +XriX7613t2Saer9fwRPvm2L7DWzgVGkWqQPabumDk3F2xmmFghcCAwEAAaNCMEAw +DgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI/wS3+o +LkUkrk1Q+mOai97i3Ru8MA0GCSqGSIb3DQEBCwUAA4IBAQBLQNvAUKr+yAzv95ZU +RUm7lgAJQayzE4aGKAczymvmdLm6AC2upArT9fHxD4q/c2dKg8dEe3jgr25sbwMp +jjM5RcOO5LlXbKr8EpbsU8Yt5CRsuZRj+9xTaGdWPoO4zzUhw8lo/s7awlOqzJCK +6fBdRoyV3XpYKBovHd7NADdBj+1EbddTKJd+82cEHhXXipa0095MJ6RMG3NzdvQX +mcIfeg7jLQitChws/zyrVQ4PkX4268NXSb7hLi18YIvDQVETI53O9zJrlAGomecs +Mx86OyXShkDOOyyGeMlhLxS67ttVb9+E7gUJTb0o2HLO02JQZR7rkpeDMdmztcpH +WD9f -----END CERTIFICATE-----` const digicertRoot = `-----BEGIN CERTIFICATE----- @@ -1547,7 +1352,7 @@ var nameConstraintTests = []struct { func TestNameConstraints(t *testing.T) { for i, test := range nameConstraintTests { - result, err := matchDomainConstraint(test.domain, test.constraint) + result, err := matchDomainConstraint(test.domain, test.constraint, map[string][]string{}, map[string][]string{}) if err != nil && !test.expectError { t.Errorf("unexpected error for test #%d: domain=%s, constraint=%s, err=%s", i, test.domain, test.constraint, err) @@ -2715,7 +2520,7 @@ func TestEKUEnforcement(t *testing.T) { if err == nil && tc.err != "" { t.Errorf("expected error") } else if err != nil && err.Error() != tc.err { - t.Errorf("unexpected error: want %q, got %q", err.Error(), tc.err) + t.Errorf("unexpected error: got %q, want %q", err.Error(), tc.err) } }) } @@ -2836,3 +2641,537 @@ func TestVerifyBareWildcard(t *testing.T) { t.Fatalf("VerifyHostname unexpected success with bare wildcard SAN") } } + +func TestPoliciesValid(t *testing.T) { + // These test cases, the comments, and the certificates they rely on, are + // stolen from BoringSSL [0]. We skip the tests which involve certificate + // parsing as part of the verification process. Those tests are in + // TestParsePolicies. + // + // [0] https://boringssl.googlesource.com/boringssl/+/264f4f7a958af6c4ccb04662e302a99dfa7c5b85/crypto/x509/x509_test.cc#5913 + + testOID1 := mustNewOIDFromInts([]uint64{1, 2, 840, 113554, 4, 1, 72585, 2, 1}) + testOID2 := mustNewOIDFromInts([]uint64{1, 2, 840, 113554, 4, 1, 72585, 2, 2}) + testOID3 := mustNewOIDFromInts([]uint64{1, 2, 840, 113554, 4, 1, 72585, 2, 3}) + testOID4 := mustNewOIDFromInts([]uint64{1, 2, 840, 113554, 4, 1, 72585, 2, 4}) + testOID5 := mustNewOIDFromInts([]uint64{1, 2, 840, 113554, 4, 1, 72585, 2, 5}) + + loadTestCert := func(t *testing.T, path string) *Certificate { + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + p, _ := pem.Decode(b) + c, err := ParseCertificate(p.Bytes) + if err != nil { + t.Fatal(err) + } + return c + } + + root := loadTestCert(t, "testdata/policy_root.pem") + root_cross_inhibit_mapping := loadTestCert(t, "testdata/policy_root_cross_inhibit_mapping.pem") + root2 := loadTestCert(t, "testdata/policy_root2.pem") + intermediate := loadTestCert(t, "testdata/policy_intermediate.pem") + intermediate_any := loadTestCert(t, "testdata/policy_intermediate_any.pem") + intermediate_mapped := loadTestCert(t, "testdata/policy_intermediate_mapped.pem") + intermediate_mapped_any := loadTestCert(t, "testdata/policy_intermediate_mapped_any.pem") + intermediate_mapped_oid3 := loadTestCert(t, "testdata/policy_intermediate_mapped_oid3.pem") + intermediate_require := loadTestCert(t, "testdata/policy_intermediate_require.pem") + intermediate_require1 := loadTestCert(t, "testdata/policy_intermediate_require1.pem") + intermediate_require2 := loadTestCert(t, "testdata/policy_intermediate_require2.pem") + intermediate_require_no_policies := loadTestCert(t, "testdata/policy_intermediate_require_no_policies.pem") + leaf := loadTestCert(t, "testdata/policy_leaf.pem") + leaf_any := loadTestCert(t, "testdata/policy_leaf_any.pem") + leaf_none := loadTestCert(t, "testdata/policy_leaf_none.pem") + leaf_oid1 := loadTestCert(t, "testdata/policy_leaf_oid1.pem") + leaf_oid2 := loadTestCert(t, "testdata/policy_leaf_oid2.pem") + leaf_oid3 := loadTestCert(t, "testdata/policy_leaf_oid3.pem") + leaf_oid4 := loadTestCert(t, "testdata/policy_leaf_oid4.pem") + leaf_oid5 := loadTestCert(t, "testdata/policy_leaf_oid5.pem") + leaf_require := loadTestCert(t, "testdata/policy_leaf_require.pem") + leaf_require1 := loadTestCert(t, "testdata/policy_leaf_require1.pem") + + type testCase struct { + chain []*Certificate + policies []OID + requireExplicitPolicy bool + inhibitPolicyMapping bool + inhibitAnyPolicy bool + valid bool + } + + tests := []testCase{ + // The chain is good for |oid1| and |oid2|, but not |oid3|. + { + chain: []*Certificate{leaf, intermediate, root}, + requireExplicitPolicy: true, + valid: true, + }, + { + chain: []*Certificate{leaf, intermediate, root}, + policies: []OID{testOID1}, + requireExplicitPolicy: true, + valid: true, + }, + { + chain: []*Certificate{leaf, intermediate, root}, + policies: []OID{testOID2}, + requireExplicitPolicy: true, + valid: true, + }, + { + chain: []*Certificate{leaf, intermediate, root}, + policies: []OID{testOID3}, + requireExplicitPolicy: true, + valid: false, + }, + { + chain: []*Certificate{leaf, intermediate, root}, + policies: []OID{testOID1, testOID2}, + requireExplicitPolicy: true, + valid: true, + }, + { + chain: []*Certificate{leaf, intermediate, root}, + policies: []OID{testOID1, testOID3}, + requireExplicitPolicy: true, + valid: true, + }, + // Without |X509_V_FLAG_EXPLICIT_POLICY|, the policy tree is built and + // intersected with user-specified policies, but it is not required to result + // in any valid policies. + { + chain: []*Certificate{leaf, intermediate, root}, + policies: []OID{testOID1}, + valid: true, + }, + { + chain: []*Certificate{leaf, intermediate, root}, + policies: []OID{testOID3}, + valid: true, + }, + // However, a CA with policy constraints can require an explicit policy. + { + chain: []*Certificate{leaf, intermediate_require, root}, + policies: []OID{testOID1}, + valid: true, + }, + { + chain: []*Certificate{leaf, intermediate_require, root}, + policies: []OID{testOID3}, + valid: false, + }, + // requireExplicitPolicy applies even if the application does not configure a + // user-initial-policy-set. If the validation results in no policies, the + // chain is invalid. + { + chain: []*Certificate{leaf_none, intermediate_require, root}, + requireExplicitPolicy: true, + valid: false, + }, + // A leaf can also set requireExplicitPolicy. + { + chain: []*Certificate{leaf_require, intermediate, root}, + valid: true, + }, + { + chain: []*Certificate{leaf_require, intermediate, root}, + policies: []OID{testOID1}, + valid: true, + }, + { + chain: []*Certificate{leaf_require, intermediate, root}, + policies: []OID{testOID3}, + valid: false, + }, + // requireExplicitPolicy is a count of certificates to skip. If the value is + // not zero by the end of the chain, it doesn't count. + { + chain: []*Certificate{leaf, intermediate_require1, root}, + policies: []OID{testOID3}, + valid: false, + }, + { + chain: []*Certificate{leaf, intermediate_require2, root}, + policies: []OID{testOID3}, + valid: true, + }, + { + chain: []*Certificate{leaf_require1, intermediate, root}, + policies: []OID{testOID3}, + valid: true, + }, + // If multiple certificates specify the constraint, the more constrained value + // wins. + { + chain: []*Certificate{leaf_require1, intermediate_require1, root}, + policies: []OID{testOID3}, + valid: false, + }, + { + chain: []*Certificate{leaf_require, intermediate_require2, root}, + policies: []OID{testOID3}, + valid: false, + }, + // An intermediate that requires an explicit policy, but then specifies no + // policies should fail verification as a result. + { + chain: []*Certificate{leaf, intermediate_require_no_policies, root}, + policies: []OID{testOID1}, + valid: false, + }, + // A constrained intermediate's policy extension has a duplicate policy, which + // is invalid. + // { + // chain: []*Certificate{leaf, intermediate_require_duplicate, root}, + // policies: []OID{testOID1}, + // valid: false, + // }, + // The leaf asserts anyPolicy, but the intermediate does not. The resulting + // valid policies are the intersection. + { + chain: []*Certificate{leaf_any, intermediate, root}, + policies: []OID{testOID1}, + requireExplicitPolicy: true, + valid: true, + }, + { + chain: []*Certificate{leaf_any, intermediate, root}, + policies: []OID{testOID3}, + requireExplicitPolicy: true, + valid: false, + }, + // The intermediate asserts anyPolicy, but the leaf does not. The resulting + // valid policies are the intersection. + { + chain: []*Certificate{leaf, intermediate_any, root}, + policies: []OID{testOID1}, + requireExplicitPolicy: true, + valid: true, + }, + { + chain: []*Certificate{leaf, intermediate_any, root}, + policies: []OID{testOID3}, + requireExplicitPolicy: true, + valid: false, + }, + // Both assert anyPolicy. All policies are valid. + { + chain: []*Certificate{leaf_any, intermediate_any, root}, + policies: []OID{testOID1}, + requireExplicitPolicy: true, + valid: true, + }, + { + chain: []*Certificate{leaf_any, intermediate_any, root}, + policies: []OID{testOID3}, + requireExplicitPolicy: true, + valid: true, + }, + // With just a trust anchor, policy checking silently succeeds. + { + chain: []*Certificate{root}, + policies: []OID{testOID1}, + requireExplicitPolicy: true, + valid: true, + }, + // Although |intermediate_mapped_oid3| contains many mappings, it only accepts + // OID3. Nodes should not be created for the other mappings. + { + chain: []*Certificate{leaf_oid1, intermediate_mapped_oid3, root}, + policies: []OID{testOID3}, + requireExplicitPolicy: true, + valid: true, + }, + { + chain: []*Certificate{leaf_oid4, intermediate_mapped_oid3, root}, + policies: []OID{testOID4}, + requireExplicitPolicy: true, + valid: false, + }, + // Policy mapping can be inhibited, either by the caller or a certificate in + // the chain, in which case mapped policies are unassertable (apart from some + // anyPolicy edge cases). + { + chain: []*Certificate{leaf_oid1, intermediate_mapped_oid3, root}, + policies: []OID{testOID3}, + requireExplicitPolicy: true, + inhibitPolicyMapping: true, + valid: false, + }, + { + chain: []*Certificate{leaf_oid1, intermediate_mapped_oid3, root_cross_inhibit_mapping, root2}, + policies: []OID{testOID3}, + requireExplicitPolicy: true, + valid: false, + }, + } + + for _, useAny := range []bool{false, true} { + var intermediate *Certificate + if useAny { + intermediate = intermediate_mapped_any + } else { + intermediate = intermediate_mapped + } + extraTests := []testCase{ + // OID3 is mapped to {OID1, OID2}, which means OID1 and OID2 (or both) are + // acceptable for OID3. + { + chain: []*Certificate{leaf, intermediate, root}, + policies: []OID{testOID3}, + requireExplicitPolicy: true, + valid: true, + }, + { + chain: []*Certificate{leaf_oid1, intermediate, root}, + policies: []OID{testOID3}, + requireExplicitPolicy: true, + valid: true, + }, + { + chain: []*Certificate{leaf_oid2, intermediate, root}, + policies: []OID{testOID3}, + requireExplicitPolicy: true, + valid: true, + }, + // If the intermediate's policies were anyPolicy, OID3 at the leaf, despite + // being mapped, is still acceptable as OID3 at the root. Despite the OID3 + // having expected_policy_set = {OID1, OID2}, it can match the anyPolicy + // node instead. + // + // If the intermediate's policies listed OIDs explicitly, OID3 at the leaf + // is not acceptable as OID3 at the root. OID3 has expected_polciy_set = + // {OID1, OID2} and no other node allows OID3. + { + chain: []*Certificate{leaf_oid3, intermediate, root}, + policies: []OID{testOID3}, + requireExplicitPolicy: true, + valid: useAny, + }, + // If the intermediate's policies were anyPolicy, OID1 at the leaf is no + // longer acceptable as OID1 at the root because policies only match + // anyPolicy when they match no other policy. + // + // If the intermediate's policies listed OIDs explicitly, OID1 at the leaf + // is acceptable as OID1 at the root because it will match both OID1 and + // OID3 (mapped) policies. + { + chain: []*Certificate{leaf_oid1, intermediate, root}, + policies: []OID{testOID1}, + requireExplicitPolicy: true, + valid: !useAny, + }, + // All pairs of OID4 and OID5 are mapped together, so either can stand for + // the other. + { + chain: []*Certificate{leaf_oid4, intermediate, root}, + policies: []OID{testOID4}, + requireExplicitPolicy: true, + valid: true, + }, + { + chain: []*Certificate{leaf_oid4, intermediate, root}, + policies: []OID{testOID5}, + requireExplicitPolicy: true, + valid: true, + }, + { + chain: []*Certificate{leaf_oid5, intermediate, root}, + policies: []OID{testOID4}, + requireExplicitPolicy: true, + valid: true, + }, + { + chain: []*Certificate{leaf_oid5, intermediate, root}, + policies: []OID{testOID5}, + requireExplicitPolicy: true, + valid: true, + }, + { + chain: []*Certificate{leaf_oid4, intermediate, root}, + policies: []OID{testOID4, testOID5}, + requireExplicitPolicy: true, + valid: true, + }, + } + tests = append(tests, extraTests...) + } + + for i, tc := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + valid := policiesValid(tc.chain, VerifyOptions{ + CertificatePolicies: tc.policies, + requireExplicitPolicy: tc.requireExplicitPolicy, + inhibitPolicyMapping: tc.inhibitPolicyMapping, + inhibitAnyPolicy: tc.inhibitAnyPolicy, + }) + if valid != tc.valid { + t.Errorf("policiesValid: got %t, want %t", valid, tc.valid) + } + }) + } +} + +func TestInvalidPolicyWithAnyKeyUsage(t *testing.T) { + loadTestCert := func(t *testing.T, path string) *Certificate { + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + p, _ := pem.Decode(b) + c, err := ParseCertificate(p.Bytes) + if err != nil { + t.Fatal(err) + } + return c + } + + testOID3 := mustNewOIDFromInts([]uint64{1, 2, 840, 113554, 4, 1, 72585, 2, 3}) + root, intermediate, leaf := loadTestCert(t, "testdata/policy_root.pem"), loadTestCert(t, "testdata/policy_intermediate_require.pem"), loadTestCert(t, "testdata/policy_leaf.pem") + + expectedErr := "x509: no valid chains built: 1 candidate chains with invalid policies" + + roots, intermediates := NewCertPool(), NewCertPool() + roots.AddCert(root) + intermediates.AddCert(intermediate) + + _, err := leaf.Verify(VerifyOptions{ + Roots: roots, + Intermediates: intermediates, + KeyUsages: []ExtKeyUsage{ExtKeyUsageAny}, + CertificatePolicies: []OID{testOID3}, + }) + if err == nil { + t.Fatal("unexpected success, invalid policy shouldn't be bypassed by passing VerifyOptions.KeyUsages with ExtKeyUsageAny") + } else if err.Error() != expectedErr { + t.Fatalf("unexpected error, got %q, want %q", err, expectedErr) + } +} + +func TestCertificateChainSignedByECDSA(t *testing.T) { + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + root := &Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "X"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + IsCA: true, + KeyUsage: KeyUsageCertSign | KeyUsageCRLSign, + BasicConstraintsValid: true, + } + caDER, err := CreateCertificate(rand.Reader, root, root, &caKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + root, err = ParseCertificate(caDER) + if err != nil { + t.Fatal(err) + } + + leafKey, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + leaf := &Certificate{ + SerialNumber: big.NewInt(42), + Subject: pkix.Name{CommonName: "leaf"}, + NotBefore: time.Now().Add(-10 * time.Minute), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: KeyUsageDigitalSignature, + ExtKeyUsage: []ExtKeyUsage{ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + leafDER, err := CreateCertificate(rand.Reader, leaf, root, &leafKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + leaf, err = ParseCertificate(leafDER) + if err != nil { + t.Fatal(err) + } + + inter, err := ParseCertificate(dsaSelfSignedCNX(t)) + if err != nil { + t.Fatal(err) + } + + inters := NewCertPool() + inters.AddCert(root) + inters.AddCert(inter) + + wantErr := "certificate signed by unknown authority" + _, err = leaf.Verify(VerifyOptions{Intermediates: inters, Roots: NewCertPool()}) + if !strings.Contains(err.Error(), wantErr) { + t.Errorf("got %v, want %q", err, wantErr) + } +} + +// dsaSelfSignedCNX produces DER-encoded +// certificate with the properties: +// +// Subject=Issuer=CN=X +// DSA SPKI +// Matching inner/outer signature OIDs +// Dummy ECDSA signature +func dsaSelfSignedCNX(t *testing.T) []byte { + t.Helper() + var params dsa.Parameters + if err := dsa.GenerateParameters(¶ms, rand.Reader, dsa.L1024N160); err != nil { + t.Fatal(err) + } + + var dsaPriv dsa.PrivateKey + dsaPriv.Parameters = params + if err := dsa.GenerateKey(&dsaPriv, rand.Reader); err != nil { + t.Fatal(err) + } + dsaPub := &dsaPriv.PublicKey + + type dsaParams struct{ P, Q, G *big.Int } + paramDER, err := asn1.Marshal(dsaParams{dsaPub.P, dsaPub.Q, dsaPub.G}) + if err != nil { + t.Fatal(err) + } + yDER, err := asn1.Marshal(dsaPub.Y) + if err != nil { + t.Fatal(err) + } + + spki := publicKeyInfo{ + Algorithm: pkix.AlgorithmIdentifier{ + Algorithm: oidPublicKeyDSA, + Parameters: asn1.RawValue{FullBytes: paramDER}, + }, + PublicKey: asn1.BitString{Bytes: yDER, BitLength: 8 * len(yDER)}, + } + + rdn := pkix.Name{CommonName: "X"}.ToRDNSequence() + b, err := asn1.Marshal(rdn) + if err != nil { + t.Fatal(err) + } + rawName := asn1.RawValue{FullBytes: b} + + algoIdent := pkix.AlgorithmIdentifier{Algorithm: oidSignatureDSAWithSHA256} + tbs := tbsCertificate{ + Version: 0, + SerialNumber: big.NewInt(1002), + SignatureAlgorithm: algoIdent, + Issuer: rawName, + Validity: validity{NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * time.Hour)}, + Subject: rawName, + PublicKey: spki, + } + c := certificate{ + TBSCertificate: tbs, + SignatureAlgorithm: algoIdent, + SignatureValue: asn1.BitString{Bytes: []byte{0}, BitLength: 8}, + } + dsaDER, err := asn1.Marshal(c) + if err != nil { + t.Fatal(err) + } + return dsaDER +} diff --git a/src/crypto/x509/x509.go b/src/crypto/x509/x509.go index 50433058f74c3d..1f06b4fbc578fe 100644 --- a/src/crypto/x509/x509.go +++ b/src/crypto/x509/x509.go @@ -29,6 +29,7 @@ import ( "crypto/elliptic" "crypto/rsa" "crypto/sha1" + "crypto/sha256" "crypto/x509/pkix" "encoding/asn1" "encoding/pem" @@ -786,11 +787,89 @@ type Certificate struct { // cannot be represented by asn1.ObjectIdentifier, it will not be included in // PolicyIdentifiers, but will be present in Policies, which contains all parsed // policy OIDs. + // See CreateCertificate for context about how this field and the Policies field + // interact. PolicyIdentifiers []asn1.ObjectIdentifier // Policies contains all policy identifiers included in the certificate. + // See CreateCertificate for context about how this field and the PolicyIdentifiers field + // interact. // In Go 1.22, encoding/gob cannot handle and ignores this field. Policies []OID + + // InhibitAnyPolicy and InhibitAnyPolicyZero indicate the presence and value + // of the inhibitAnyPolicy extension. + // + // The value of InhibitAnyPolicy indicates the number of additional + // certificates in the path after this certificate that may use the + // anyPolicy policy OID to indicate a match with any other policy. + // + // When parsing a certificate, a positive non-zero InhibitAnyPolicy means + // that the field was specified, -1 means it was unset, and + // InhibitAnyPolicyZero being true mean that the field was explicitly set to + // zero. The case of InhibitAnyPolicy==0 with InhibitAnyPolicyZero==false + // should be treated equivalent to -1 (unset). + InhibitAnyPolicy int + // InhibitAnyPolicyZero indicates that InhibitAnyPolicy==0 should be + // interpreted as an actual maximum path length of zero. Otherwise, that + // combination is interpreted as InhibitAnyPolicy not being set. + InhibitAnyPolicyZero bool + + // InhibitPolicyMapping and InhibitPolicyMappingZero indicate the presence + // and value of the inhibitPolicyMapping field of the policyConstraints + // extension. + // + // The value of InhibitPolicyMapping indicates the number of additional + // certificates in the path after this certificate that may use policy + // mapping. + // + // When parsing a certificate, a positive non-zero InhibitPolicyMapping + // means that the field was specified, -1 means it was unset, and + // InhibitPolicyMappingZero being true mean that the field was explicitly + // set to zero. The case of InhibitPolicyMapping==0 with + // InhibitPolicyMappingZero==false should be treated equivalent to -1 + // (unset). + InhibitPolicyMapping int + // InhibitPolicyMappingZero indicates that InhibitPolicyMapping==0 should be + // interpreted as an actual maximum path length of zero. Otherwise, that + // combination is interpreted as InhibitAnyPolicy not being set. + InhibitPolicyMappingZero bool + + // RequireExplicitPolicy and RequireExplicitPolicyZero indicate the presence + // and value of the requireExplicitPolicy field of the policyConstraints + // extension. + // + // The value of RequireExplicitPolicy indicates the number of additional + // certificates in the path after this certificate before an explicit policy + // is required for the rest of the path. When an explicit policy is required, + // each subsequent certificate in the path must contain a required policy OID, + // or a policy OID which has been declared as equivalent through the policy + // mapping extension. + // + // When parsing a certificate, a positive non-zero RequireExplicitPolicy + // means that the field was specified, -1 means it was unset, and + // RequireExplicitPolicyZero being true mean that the field was explicitly + // set to zero. The case of RequireExplicitPolicy==0 with + // RequireExplicitPolicyZero==false should be treated equivalent to -1 + // (unset). + RequireExplicitPolicy int + // RequireExplicitPolicyZero indicates that RequireExplicitPolicy==0 should be + // interpreted as an actual maximum path length of zero. Otherwise, that + // combination is interpreted as InhibitAnyPolicy not being set. + RequireExplicitPolicyZero bool + + // PolicyMappings contains a list of policy mappings included in the certificate. + PolicyMappings []PolicyMapping +} + +// PolicyMapping represents a policy mapping entry in the policyMappings extension. +type PolicyMapping struct { + // IssuerDomainPolicy contains a policy OID the issuing certificate considers + // equivalent to SubjectDomainPolicy in the subject certificate. + IssuerDomainPolicy OID + // SubjectDomainPolicy contains a OID the issuing certificate considers + // equivalent to IssuerDomainPolicy in the subject certificate. + SubjectDomainPolicy OID } // ErrUnsupportedAlgorithm results from attempting to perform an operation that @@ -799,18 +878,10 @@ var ErrUnsupportedAlgorithm = errors.New("x509: cannot verify signature: algorit // An InsecureAlgorithmError indicates that the [SignatureAlgorithm] used to // generate the signature is not secure, and the signature has been rejected. -// -// To temporarily restore support for SHA-1 signatures, include the value -// "x509sha1=1" in the GODEBUG environment variable. Note that this option will -// be removed in a future release. type InsecureAlgorithmError SignatureAlgorithm func (e InsecureAlgorithmError) Error() string { - var override string - if SignatureAlgorithm(e) == SHA1WithRSA || SignatureAlgorithm(e) == ECDSAWithSHA1 { - override = " (temporarily override with GODEBUG=x509sha1=1)" - } - return fmt.Sprintf("x509: cannot verify signature: insecure algorithm %v", SignatureAlgorithm(e)) + override + return fmt.Sprintf("x509: cannot verify signature: insecure algorithm %v", SignatureAlgorithm(e)) } // ConstraintViolationError results when a requested usage is not permitted by @@ -887,8 +958,6 @@ func signaturePublicKeyAlgoMismatchError(expectedPubKeyAlgo PublicKeyAlgorithm, return fmt.Errorf("x509: signature algorithm specifies an %s public key, but have public key of type %T", expectedPubKeyAlgo.String(), pubKey) } -var x509sha1 = godebug.New("x509sha1") - // checkSignature verifies that signature is a valid signature over signed from // a crypto.PublicKey. func checkSignature(algo SignatureAlgorithm, signed, signature []byte, publicKey crypto.PublicKey, allowSHA1 bool) (err error) { @@ -911,12 +980,9 @@ func checkSignature(algo SignatureAlgorithm, signed, signature []byte, publicKey case crypto.MD5: return InsecureAlgorithmError(algo) case crypto.SHA1: - // SHA-1 signatures are mostly disabled. See go.dev/issue/41682. + // SHA-1 signatures are only allowed for CRLs and CSRs. if !allowSHA1 { - if x509sha1.Value() != "1" { - return InsecureAlgorithmError(algo) - } - x509sha1.IncNonDefault() + return InsecureAlgorithmError(algo) } fallthrough default: @@ -1198,7 +1264,7 @@ func buildCertExtensions(template *Certificate, subjectIsEmpty bool, authorityKe n++ } - usePolicies := x509usepolicies.Value() == "1" + usePolicies := x509usepolicies.Value() != "0" if ((!usePolicies && len(template.PolicyIdentifiers) > 0) || (usePolicies && len(template.Policies) > 0)) && !oidInExtensions(oidExtensionCertificatePolicies, template.ExtraExtensions) { ret[n], err = marshalCertificatePolicies(template.Policies, template.PolicyIdentifiers) @@ -1391,7 +1457,7 @@ func marshalCertificatePolicies(policies []OID, policyIdentifiers []asn1.ObjectI b := cryptobyte.NewBuilder(make([]byte, 0, 128)) b.AddASN1(cryptobyte_asn1.SEQUENCE, func(child *cryptobyte.Builder) { - if x509usepolicies.Value() == "1" { + if x509usepolicies.Value() != "0" { x509usepolicies.IncNonDefault() for _, v := range policies { child.AddASN1(cryptobyte_asn1.SEQUENCE, func(child *cryptobyte.Builder) { @@ -1503,13 +1569,7 @@ func signingParamsForKey(key crypto.Signer, sigAlgo SignatureAlgorithm) (Signatu } func signTBS(tbs []byte, key crypto.Signer, sigAlg SignatureAlgorithm, rand io.Reader) ([]byte, error) { - signed := tbs hashFunc := sigAlg.hashFunc() - if hashFunc != 0 { - h := hashFunc.New() - h.Write(signed) - signed = h.Sum(nil) - } var signerOpts crypto.SignerOpts = hashFunc if sigAlg.isRSAPSS() { @@ -1519,7 +1579,7 @@ func signTBS(tbs []byte, key crypto.Signer, sigAlg SignatureAlgorithm, rand io.R } } - signature, err := key.Sign(rand, signed, signerOpts) + signature, err := crypto.SignMessage(key, rand, tbs, signerOpts) if err != nil { return nil, err } @@ -1581,7 +1641,7 @@ var emptyASN1Subject = []byte{0x30, 0} // // The currently supported key types are *rsa.PublicKey, *ecdsa.PublicKey and // ed25519.PublicKey. pub must be a supported key type, and priv must be a -// crypto.Signer with a supported public key. +// crypto.Signer or crypto.MessageSigner with a supported public key. // // The AuthorityKeyId will be taken from the SubjectKeyId of parent, if any, // unless the resulting certificate is self-signed. Otherwise the value from @@ -1590,10 +1650,13 @@ var emptyASN1Subject = []byte{0x30, 0} // If SubjectKeyId from template is empty and the template is a CA, SubjectKeyId // will be generated from the hash of the public key. // -// The PolicyIdentifier and Policies fields are both used to marshal certificate -// policy OIDs. By default, only the PolicyIdentifier is marshaled, but if the -// GODEBUG setting "x509usepolicies" has the value "1", the Policies field will -// be marshaled instead of the PolicyIdentifier field. The Policies field can +// If template.SerialNumber is nil, a serial number will be generated which +// conforms to RFC 5280, Section 4.1.2.2 using entropy from rand. +// +// The PolicyIdentifier and Policies fields can both be used to marshal certificate +// policy OIDs. By default, only the Policies is marshaled, but if the +// GODEBUG setting "x509usepolicies" has the value "0", the PolicyIdentifiers field will +// be marshaled instead of the Policies field. This changed in Go 1.24. The Policies field can // be used to marshal policy OIDs which have components that are larger than 31 // bits. func CreateCertificate(rand io.Reader, template, parent *Certificate, pub, priv any) ([]byte, error) { @@ -1602,19 +1665,36 @@ func CreateCertificate(rand io.Reader, template, parent *Certificate, pub, priv return nil, errors.New("x509: certificate private key does not implement crypto.Signer") } - if template.SerialNumber == nil { - return nil, errors.New("x509: no SerialNumber given") + serialNumber := template.SerialNumber + if serialNumber == nil { + // Generate a serial number following RFC 5280, Section 4.1.2.2 if one + // is not provided. The serial number must be positive and at most 20 + // octets *when encoded*. + serialBytes := make([]byte, 20) + if _, err := io.ReadFull(rand, serialBytes); err != nil { + return nil, err + } + // If the top bit is set, the serial will be padded with a leading zero + // byte during encoding, so that it's not interpreted as a negative + // integer. This padding would make the serial 21 octets so we clear the + // top bit to ensure the correct length in all cases. + serialBytes[0] &= 0b0111_1111 + serialNumber = new(big.Int).SetBytes(serialBytes) } - // RFC 5280 Section 4.1.2.2: serial number must positive + // RFC 5280 Section 4.1.2.2: serial number must be positive // // We _should_ also restrict serials to <= 20 octets, but it turns out a lot of people // get this wrong, in part because the encoding can itself alter the length of the // serial. For now we accept these non-conformant serials. - if template.SerialNumber.Sign() == -1 { + if serialNumber.Sign() == -1 { return nil, errors.New("x509: serial number must be positive") } + if template.BasicConstraintsValid && template.MaxPathLen < -1 { + return nil, errors.New("x509: invalid MaxPathLen, must be greater or equal to -1") + } + if template.BasicConstraintsValid && !template.IsCA && template.MaxPathLen != -1 && (template.MaxPathLen != 0 || template.MaxPathLenZero) { return nil, errors.New("x509: only CAs are allowed to specify MaxPathLen") } @@ -1649,12 +1729,22 @@ func CreateCertificate(rand io.Reader, template, parent *Certificate, pub, priv subjectKeyId := template.SubjectKeyId if len(subjectKeyId) == 0 && template.IsCA { - // SubjectKeyId generated using method 1 in RFC 5280, Section 4.2.1.2: - // (1) The keyIdentifier is composed of the 160-bit SHA-1 hash of the - // value of the BIT STRING subjectPublicKey (excluding the tag, - // length, and number of unused bits). - h := sha1.Sum(publicKeyBytes) - subjectKeyId = h[:] + if x509sha256skid.Value() == "0" { + x509sha256skid.IncNonDefault() + // SubjectKeyId generated using method 1 in RFC 5280, Section 4.2.1.2: + // (1) The keyIdentifier is composed of the 160-bit SHA-1 hash of the + // value of the BIT STRING subjectPublicKey (excluding the tag, + // length, and number of unused bits). + h := sha1.Sum(publicKeyBytes) + subjectKeyId = h[:] + } else { + // SubjectKeyId generated using method 1 in RFC 7093, Section 2: + // 1) The keyIdentifier is composed of the leftmost 160-bits of the + // SHA-256 hash of the value of the BIT STRING subjectPublicKey + // (excluding the tag, length, and number of unused bits). + h := sha256.Sum256(publicKeyBytes) + subjectKeyId = h[:20] + } } // Check that the signer's public key matches the private key, if available. @@ -1675,7 +1765,7 @@ func CreateCertificate(rand io.Reader, template, parent *Certificate, pub, priv encodedPublicKey := asn1.BitString{BitLength: len(publicKeyBytes) * 8, Bytes: publicKeyBytes} c := tbsCertificate{ Version: 2, - SerialNumber: template.SerialNumber, + SerialNumber: serialNumber, SignatureAlgorithm: algorithmIdentifier, Issuer: asn1.RawValue{FullBytes: asn1Issuer}, Validity: validity{template.NotBefore.UTC(), template.NotAfter.UTC()}, @@ -1702,6 +1792,8 @@ func CreateCertificate(rand io.Reader, template, parent *Certificate, pub, priv }) } +var x509sha256skid = godebug.New("x509sha256skid") + // pemCRLPrefix is the magic string that indicates that we have a PEM encoded // CRL. var pemCRLPrefix = []byte("-----BEGIN X509 CRL") @@ -1950,10 +2042,10 @@ func parseCSRExtensions(rawAttributes []asn1.RawValue) ([]pkix.Extension, error) // - Attributes (deprecated) // // priv is the private key to sign the CSR with, and the corresponding public -// key will be included in the CSR. It must implement crypto.Signer and its -// Public() method must return a *rsa.PublicKey or a *ecdsa.PublicKey or a -// ed25519.PublicKey. (A *rsa.PrivateKey, *ecdsa.PrivateKey or -// ed25519.PrivateKey satisfies this.) +// key will be included in the CSR. It must implement crypto.Signer or +// crypto.MessageSigner and its Public() method must return a *rsa.PublicKey or +// a *ecdsa.PublicKey or a ed25519.PublicKey. (A *rsa.PrivateKey, +// *ecdsa.PrivateKey or ed25519.PrivateKey satisfies this.) // // The returned slice is the certificate request in DER encoding. func CreateCertificateRequest(rand io.Reader, template *CertificateRequest, priv any) (csr []byte, err error) { @@ -2295,8 +2387,9 @@ type tbsCertificateList struct { // CreateRevocationList creates a new X.509 v2 [Certificate] Revocation List, // according to RFC 5280, based on template. // -// The CRL is signed by priv which should be the private key associated with -// the public key in the issuer certificate. +// The CRL is signed by priv which should be a crypto.Signer or +// crypto.MessageSigner associated with the public key in the issuer +// certificate. // // The issuer may not be nil, and the crlSign bit must be set in [KeyUsage] in // order to use it as a CRL issuer. diff --git a/src/crypto/x509/x509_test.go b/src/crypto/x509/x509_test.go index 351fe6ad186b14..98f3f7941c8d63 100644 --- a/src/crypto/x509/x509_test.go +++ b/src/crypto/x509/x509_test.go @@ -22,6 +22,7 @@ import ( "encoding/gob" "encoding/hex" "encoding/pem" + "errors" "fmt" "internal/testenv" "io" @@ -59,6 +60,32 @@ func TestParsePKCS1PrivateKey(t *testing.T) { if _, err := ParsePKCS1PrivateKey(data); err == nil { t.Errorf("parsing invalid private key did not result in an error") } + + // A partial key without CRT values should still parse. + b, _ := asn1.Marshal(struct { + Version int + N *big.Int + E int + D *big.Int + P *big.Int + Q *big.Int + }{ + N: priv.N, + E: priv.PublicKey.E, + D: priv.D, + P: priv.Primes[0], + Q: priv.Primes[1], + }) + p2, err := ParsePKCS1PrivateKey(b) + if err != nil { + t.Fatalf("parsing partial private key resulted in an error: %v", err) + } + if !p2.Equal(priv) { + t.Errorf("partial private key did not match original key") + } + if p2.Precomputed.Dp == nil || p2.Precomputed.Dq == nil || p2.Precomputed.Qinv == nil { + t.Errorf("precomputed values not recomputed") + } } func TestPKCS1MismatchPublicKeyFormat(t *testing.T) { @@ -251,7 +278,62 @@ func TestMarshalRSAPrivateKey(t *testing.T) { priv.Primes[0].Cmp(priv2.Primes[0]) != 0 || priv.Primes[1].Cmp(priv2.Primes[1]) != 0 || priv.Primes[2].Cmp(priv2.Primes[2]) != 0 { - t.Errorf("got:%+v want:%+v", priv, priv2) + t.Errorf("wrong priv:\ngot %+v\nwant %+v", priv2, priv) + } + + if priv.Precomputed.Dp == nil { + t.Fatalf("Precomputed.Dp is nil") + } +} + +func TestMarshalRSAPrivateKeyInvalid(t *testing.T) { + block, _ := pem.Decode([]byte(strings.ReplaceAll( + `-----BEGIN RSA TESTING KEY----- +MIIEowIBAAKCAQEAsPnoGUOnrpiSqt4XynxA+HRP7S+BSObI6qJ7fQAVSPtRkqso +tWxQYLEYzNEx5ZSHTGypibVsJylvCfuToDTfMul8b/CZjP2Ob0LdpYrNH6l5hvFE +89FU1nZQF15oVLOpUgA7wGiHuEVawrGfey92UE68mOyUVXGweJIVDdxqdMoPvNNU +l86BU02vlBiESxOuox+dWmuVV7vfYZ79Toh/LUK43YvJh+rhv4nKuF7iHjVjBd9s +B6iDjj70HFldzOQ9r8SRI+9NirupPTkF5AKNe6kUhKJ1luB7S27ZkvB3tSTT3P59 +3VVJvnzOjaA1z6Cz+4+eRvcysqhrRgFlwI9TEwIDAQABAoIBAEEYiyDP29vCzx/+ +dS3LqnI5BjUuJhXUnc6AWX/PCgVAO+8A+gZRgvct7PtZb0sM6P9ZcLrweomlGezI +FrL0/6xQaa8bBr/ve/a8155OgcjFo6fZEw3Dz7ra5fbSiPmu4/b/kvrg+Br1l77J +aun6uUAs1f5B9wW+vbR7tzbT/mxaUeDiBzKpe15GwcvbJtdIVMa2YErtRjc1/5B2 +BGVXyvlJv0SIlcIEMsHgnAFOp1ZgQ08aDzvilLq8XVMOahAhP1O2A3X8hKdXPyrx +IVWE9bS9ptTo+eF6eNl+d7htpKGEZHUxinoQpWEBTv+iOoHsVunkEJ3vjLP3lyI/ +fY0NQ1ECgYEA3RBXAjgvIys2gfU3keImF8e/TprLge1I2vbWmV2j6rZCg5r/AS0u +pii5CvJ5/T5vfJPNgPBy8B/yRDs+6PJO1GmnlhOkG9JAIPkv0RBZvR0PMBtbp6nT +Y3yo1lwamBVBfY6rc0sLTzosZh2aGoLzrHNMQFMGaauORzBFpY5lU50CgYEAzPHl +u5DI6Xgep1vr8QvCUuEesCOgJg8Yh1UqVoY/SmQh6MYAv1I9bLGwrb3WW/7kqIoD +fj0aQV5buVZI2loMomtU9KY5SFIsPV+JuUpy7/+VE01ZQM5FdY8wiYCQiVZYju9X +Wz5LxMNoz+gT7pwlLCsC4N+R8aoBk404aF1gum8CgYAJ7VTq7Zj4TFV7Soa/T1eE +k9y8a+kdoYk3BASpCHJ29M5R2KEA7YV9wrBklHTz8VzSTFTbKHEQ5W5csAhoL5Fo +qoHzFFi3Qx7MHESQb9qHyolHEMNx6QdsHUn7rlEnaTTyrXh3ifQtD6C0yTmFXUIS +CW9wKApOrnyKJ9nI0HcuZQKBgQCMtoV6e9VGX4AEfpuHvAAnMYQFgeBiYTkBKltQ +XwozhH63uMMomUmtSG87Sz1TmrXadjAhy8gsG6I0pWaN7QgBuFnzQ/HOkwTm+qKw +AsrZt4zeXNwsH7QXHEJCFnCmqw9QzEoZTrNtHJHpNboBuVnYcoueZEJrP8OnUG3r +UjmopwKBgAqB2KYYMUqAOvYcBnEfLDmyZv9BTVNHbR2lKkMYqv5LlvDaBxVfilE0 +2riO4p6BaAdvzXjKeRrGNEKoHNBpOSfYCOM16NjL8hIZB1CaV3WbT5oY+jp7Mzd5 +7d56RZOE+ERK2uz/7JX9VSsM/LbH9pJibd4e8mikDS9ntciqOH/3 +-----END RSA TESTING KEY-----`, "TESTING KEY", "PRIVATE KEY"))) + testRSA2048, _ := ParsePKCS1PrivateKey(block.Bytes) + + broken := *testRSA2048 + broken.Precomputed.Dp = new(big.Int).SetUint64(42) + + parsed, err := ParsePKCS1PrivateKey(MarshalPKCS1PrivateKey(&broken)) + if err == nil { + t.Errorf("expected error, got success") + } + + t.Setenv("GODEBUG", "x509rsacrt=0") + + parsed, err = ParsePKCS1PrivateKey(MarshalPKCS1PrivateKey(&broken)) + if err != nil { + t.Fatalf("expected success, got error: %v", err) + } + // Dp should have been recomputed. + if parsed.Precomputed.Dp.Cmp(testRSA2048.Precomputed.Dp) != 0 { + t.Errorf("Dp recomputation failed: got %v, want %v", parsed.Precomputed.Dp, testRSA2048.Precomputed.Dp) } } @@ -673,8 +755,7 @@ func TestCreateSelfSignedCertificate(t *testing.T) { IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1).To4(), net.ParseIP("2001:4860:0:2001::68")}, URIs: []*url.URL{parseURI("https://foo.com/wibble#foo")}, - PolicyIdentifiers: []asn1.ObjectIdentifier{[]int{1, 2, 3}}, - Policies: []OID{mustNewOIDFromInts(t, []uint64{1, 2, 3, math.MaxUint32, math.MaxUint64})}, + Policies: []OID{mustNewOIDFromInts([]uint64{1, 2, 3, math.MaxUint32, math.MaxUint64})}, PermittedDNSDomains: []string{".example.com", "example.com"}, ExcludedDNSDomains: []string{"bar.example.com"}, PermittedIPRanges: []*net.IPNet{parseCIDR("192.168.1.1/16"), parseCIDR("1.2.3.4/8")}, @@ -712,8 +793,8 @@ func TestCreateSelfSignedCertificate(t *testing.T) { continue } - if len(cert.PolicyIdentifiers) != 1 || !cert.PolicyIdentifiers[0].Equal(template.PolicyIdentifiers[0]) { - t.Errorf("%s: failed to parse policy identifiers: got:%#v want:%#v", test.name, cert.PolicyIdentifiers, template.PolicyIdentifiers) + if len(cert.Policies) != 1 || !cert.Policies[0].Equal(template.Policies[0]) { + t.Errorf("%s: failed to parse policy identifiers: got:%#v want:%#v", test.name, cert.PolicyIdentifiers, template.Policies) } if len(cert.PermittedDNSDomains) != 2 || cert.PermittedDNSDomains[0] != ".example.com" || cert.PermittedDNSDomains[1] != "example.com" { @@ -1472,6 +1553,7 @@ func TestCreateCertificateRequest(t *testing.T) { } func marshalAndParseCSR(t *testing.T, template *CertificateRequest) *CertificateRequest { + t.Helper() derBytes, err := CreateCertificateRequest(rand.Reader, template, testPrivateKey) if err != nil { t.Fatal(err) @@ -1636,6 +1718,7 @@ func TestCriticalFlagInCSRRequestedExtensions(t *testing.T) { // serialiseAndParse generates a self-signed certificate from template and // returns a parsed version of it. func serialiseAndParse(t *testing.T, template *Certificate) *Certificate { + t.Helper() derBytes, err := CreateCertificate(rand.Reader, template, template, &testPrivateKey.PublicKey, testPrivateKey) if err != nil { t.Fatalf("failed to create certificate: %s", err) @@ -1815,8 +1898,8 @@ func TestInsecureAlgorithmErrorString(t *testing.T) { want string }{ {MD5WithRSA, "x509: cannot verify signature: insecure algorithm MD5-RSA"}, - {SHA1WithRSA, "x509: cannot verify signature: insecure algorithm SHA1-RSA (temporarily override with GODEBUG=x509sha1=1)"}, - {ECDSAWithSHA1, "x509: cannot verify signature: insecure algorithm ECDSA-SHA1 (temporarily override with GODEBUG=x509sha1=1)"}, + {SHA1WithRSA, "x509: cannot verify signature: insecure algorithm SHA1-RSA"}, + {ECDSAWithSHA1, "x509: cannot verify signature: insecure algorithm ECDSA-SHA1"}, {MD2WithRSA, "x509: cannot verify signature: insecure algorithm 1"}, {-1, "x509: cannot verify signature: insecure algorithm -1"}, {0, "x509: cannot verify signature: insecure algorithm 0"}, @@ -1897,11 +1980,6 @@ func TestSHA1(t *testing.T) { if _, ok := err.(InsecureAlgorithmError); !ok { t.Fatalf("certificate verification returned %v (%T), wanted InsecureAlgorithmError", err, err) } - - t.Setenv("GODEBUG", "x509sha1=1") - if err = cert.CheckSignatureFrom(cert); err != nil { - t.Fatalf("SHA-1 certificate did not verify with GODEBUG=x509sha1=1: %v", err) - } } // certMissingRSANULL contains an RSA public key where the AlgorithmIdentifier @@ -2327,6 +2405,37 @@ func TestAdditionFieldsInGeneralSubtree(t *testing.T) { } } +func TestEmptySerialNumber(t *testing.T) { + template := Certificate{ + DNSNames: []string{"example.com"}, + } + + for range 100 { + derBytes, err := CreateCertificate(rand.Reader, &template, &template, &testPrivateKey.PublicKey, testPrivateKey) + if err != nil { + t.Fatalf("failed to create certificate: %s", err) + } + + cert, err := ParseCertificate(derBytes) + if err != nil { + t.Fatalf("failed to parse certificate: %s", err) + } + + if sign := cert.SerialNumber.Sign(); sign != 1 { + t.Fatalf("generated a non positive serial, sign: %d", sign) + } + + b, err := asn1.Marshal(cert.SerialNumber) + if err != nil { + t.Fatalf("failed to marshal generated serial number: %s", err) + } + // subtract 2 for tag and length + if l := len(b) - 2; l > 20 { + t.Fatalf("generated serial number larger than 20 octets when encoded: %d", l) + } + } +} + func TestEmptySubject(t *testing.T) { template := Certificate{ SerialNumber: big.NewInt(1), @@ -2979,12 +3088,8 @@ func TestUnknownExtKey(t *testing.T) { DNSNames: []string{"foo"}, ExtKeyUsage: []ExtKeyUsage{ExtKeyUsage(-1)}, } - signer, err := rsa.GenerateKey(rand.Reader, 1024) - if err != nil { - t.Errorf("failed to generate key for TestUnknownExtKey") - } - _, err = CreateCertificate(rand.Reader, template, template, signer.Public(), signer) + _, err := CreateCertificate(rand.Reader, template, template, testPrivateKey.Public(), testPrivateKey) if !strings.Contains(err.Error(), errorContains) { t.Errorf("expected error containing %q, got %s", errorContains, err) } @@ -3923,7 +4028,9 @@ func TestDuplicateAttributesCSR(t *testing.T) { } } -func TestCertificateOIDPolicies(t *testing.T) { +func TestCertificateOIDPoliciesGODEBUG(t *testing.T) { + t.Setenv("GODEBUG", "x509usepolicies=0") + template := Certificate{ SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "Cert"}, @@ -3937,7 +4044,7 @@ func TestCertificateOIDPolicies(t *testing.T) { } var expectPolicies = []OID{ - mustNewOIDFromInts(t, []uint64{1, 2, 3}), + mustNewOIDFromInts([]uint64{1, 2, 3}), } certDER, err := CreateCertificate(rand.Reader, &template, &template, rsaPrivateKey.Public(), rsaPrivateKey) @@ -3959,17 +4066,21 @@ func TestCertificateOIDPolicies(t *testing.T) { } } -func TestCertificatePoliciesGODEBUG(t *testing.T) { +func TestCertificatePolicies(t *testing.T) { + if x509usepolicies.Value() == "0" { + t.Skip("test relies on default x509usepolicies GODEBUG") + } + template := Certificate{ SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "Cert"}, NotBefore: time.Unix(1000, 0), NotAfter: time.Unix(100000, 0), PolicyIdentifiers: []asn1.ObjectIdentifier{[]int{1, 2, 3}}, - Policies: []OID{mustNewOIDFromInts(t, []uint64{1, 2, math.MaxUint32 + 1})}, + Policies: []OID{mustNewOIDFromInts([]uint64{1, 2, math.MaxUint32 + 1})}, } - expectPolicies := []OID{mustNewOIDFromInts(t, []uint64{1, 2, 3})} + expectPolicies := []OID{mustNewOIDFromInts([]uint64{1, 2, math.MaxUint32 + 1})} certDER, err := CreateCertificate(rand.Reader, &template, &template, rsaPrivateKey.Public(), rsaPrivateKey) if err != nil { t.Fatalf("CreateCertificate() unexpected error: %v", err) @@ -3985,7 +4096,7 @@ func TestCertificatePoliciesGODEBUG(t *testing.T) { } t.Setenv("GODEBUG", "x509usepolicies=1") - expectPolicies = []OID{mustNewOIDFromInts(t, []uint64{1, 2, math.MaxUint32 + 1})} + expectPolicies = []OID{mustNewOIDFromInts([]uint64{1, 2, math.MaxUint32 + 1})} certDER, err = CreateCertificate(rand.Reader, &template, &template, rsaPrivateKey.Public(), rsaPrivateKey) if err != nil { @@ -4086,3 +4197,69 @@ func TestRejectCriticalSKI(t *testing.T) { t.Fatalf("ParseCertificate() unexpected error: %v, want: %s", err, expectedErr) } } + +type messageSigner struct{} + +func (ms *messageSigner) Public() crypto.PublicKey { return rsaPrivateKey.Public() } + +func (ms *messageSigner) Sign(rand io.Reader, digest []byte, opts crypto.SignerOpts) (signature []byte, err error) { + return nil, errors.New("unimplemented") +} + +func (ms *messageSigner) SignMessage(rand io.Reader, msg []byte, opts crypto.SignerOpts) (signature []byte, err error) { + if _, ok := opts.(*rsa.PSSOptions); ok { + return nil, errors.New("PSSOptions passed instead of hash") + } + h := opts.HashFunc().New() + h.Write(msg) + tbs := h.Sum(nil) + return rsa.SignPKCS1v15(rand, rsaPrivateKey, opts.HashFunc(), tbs) +} + +func TestMessageSigner(t *testing.T) { + template := Certificate{ + SignatureAlgorithm: SHA256WithRSA, + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "Cert"}, + NotBefore: time.Unix(1000, 0), + NotAfter: time.Unix(100000, 0), + BasicConstraintsValid: true, + IsCA: true, + } + certDER, err := CreateCertificate(rand.Reader, &template, &template, rsaPrivateKey.Public(), &messageSigner{}) + if err != nil { + t.Fatalf("CreateCertificate failed: %s", err) + } + cert, err := ParseCertificate(certDER) + if err != nil { + t.Fatalf("ParseCertificate failed: %s", err) + } + if err := cert.CheckSignatureFrom(cert); err != nil { + t.Fatalf("CheckSignatureFrom failed: %s", err) + } +} + +func TestCreateCertificateNegativeMaxPathLength(t *testing.T) { + template := Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "TEST"}, + NotBefore: time.Unix(1000, 0), + NotAfter: time.Unix(100000, 0), + BasicConstraintsValid: true, + IsCA: true, + + // CreateCertificate treats -1 in the same way as: MaxPathLen == 0 && MaxPathLenZero == false. + MaxPathLen: -1, + } + + _, err := CreateCertificate(rand.Reader, &template, &template, rsaPrivateKey.Public(), rsaPrivateKey) + if err != nil { + t.Fatalf("CreateCertificate() unexpected error: %v", err) + } + + template.MaxPathLen = -2 + _, err = CreateCertificate(rand.Reader, &template, &template, rsaPrivateKey.Public(), rsaPrivateKey) + if err == nil || err.Error() != "x509: invalid MaxPathLen, must be greater or equal to -1" { + t.Fatalf(`CreateCertificate() = %v; want = "x509: invalid MaxPathLen, must be greater or equal to -1"`, err) + } +} diff --git a/src/crypto/x509/x509_test_import.go b/src/crypto/x509/x509_test_import.go index 2474e3d810edfd..e68f89c91d21be 100644 --- a/src/crypto/x509/x509_test_import.go +++ b/src/crypto/x509/x509_test_import.go @@ -43,13 +43,19 @@ func main() { } var pemPrivateKey = testingKey(`-----BEGIN RSA TESTING KEY----- -MIIBOgIBAAJBALKZD0nEffqM1ACuak0bijtqE2QrI/KLADv7l3kK3ppMyCuLKoF0 -fd7Ai2KW5ToIwzFofvJcS/STa6HA5gQenRUCAwEAAQJBAIq9amn00aS0h/CrjXqu -/ThglAXJmZhOMPVn4eiu7/ROixi9sex436MaVeMqSNf7Ex9a8fRNfWss7Sqd9eWu -RTUCIQDasvGASLqmjeffBNLTXV2A5g4t+kLVCpsEIZAycV5GswIhANEPLmax0ME/ -EO+ZJ79TJKN5yiGBRsv5yvx5UiHxajEXAiAhAol5N4EUyq6I9w1rYdhPMGpLfk7A -IU2snfRJ6Nq2CQIgFrPsWRCkV+gOYcajD17rEqmuLrdIRexpg8N1DOSXoJ8CIGlS -tAboUGBxTDq3ZroNism3DaMIbKPyYrAqhKov1h5V +MIICXQIBAAKBgQCw0YNSqI9T1VFvRsIOejZ9feiKz1SgGfbe9Xq5tEzt2yJCsbyg ++xtcuCswNhdqY5A1ZN7G60HbL4/Hh/TlLhFJ4zNHVylz9mDDx3yp4IIcK2lb566d +fTD0B5EQ9Iqub4twLUdLKQCBfyhmJJvsEqKxm4J4QWgI+Brh/Pm3d4piPwIDAQAB +AoGASC6fj6TkLfMNdYHLQqG9kOlPfys4fstarpZD7X+fUBJ/H/7y5DzeZLGCYAIU ++QeAHWv6TfZIQjReW7Qy00RFJdgwFlTFRCsKXhG5x+IB+jL0Grr08KbgPPDgy4Jm +xirRHZVtU8lGbkiZX+omDIU28EHLNWL6rFEcTWao/tERspECQQDp2G5Nw0qYWn7H +Wm9Up1zkUTnkUkCzhqtxHbeRvNmHGKE7ryGMJEk2RmgHVstQpsvuFY4lIUSZEjAc +DUFJERhFAkEAwZH6O1ULORp8sHKDdidyleYcZU8L7y9Y3OXJYqELfddfBgFUZeVQ +duRmJj7ryu0g0uurOTE+i8VnMg/ostxiswJBAOc64Dd8uLJWKa6uug+XPr91oi0n +OFtM+xHrNK2jc+WmcSg3UJDnAI3uqMc5B+pERLq0Dc6hStehqHjUko3RnZECQEGZ +eRYWciE+Cre5dzfZkomeXE0xBrhecV0bOq6EKWLSVE+yr6mAl05ThRK9DCfPSOpy +F6rgN3QiyCA9J/1FluUCQQC5nX+PTU1FXx+6Ri2ZCi6EjEKMHr7gHcABhMinZYOt +N59pra9UdVQw9jxCU9G7eMyb0jJkNACAuEwakX3gi27b -----END RSA TESTING KEY----- `) diff --git a/src/database/sql/convert.go b/src/database/sql/convert.go index c261046b187e52..26b139ababd178 100644 --- a/src/database/sql/convert.go +++ b/src/database/sql/convert.go @@ -290,7 +290,7 @@ func convertAssignRows(dest, src any, rows *Rows) error { if d == nil { return errNilPtr } - *d = []byte(s.Format(time.RFC3339Nano)) + *d = s.AppendFormat(make([]byte, 0, len(time.RFC3339Nano)), time.RFC3339Nano) return nil case *RawBytes: if d == nil { @@ -335,7 +335,6 @@ func convertAssignRows(dest, src any, rows *Rows) error { if rows == nil { return errors.New("invalid context to convert cursor rows, missing parent *Rows") } - rows.closemu.Lock() *d = Rows{ dc: rows.dc, releaseConn: func(error) {}, @@ -351,7 +350,6 @@ func convertAssignRows(dest, src any, rows *Rows) error { parentCancel() } } - rows.closemu.Unlock() return nil } } diff --git a/src/database/sql/convert_test.go b/src/database/sql/convert_test.go index f94db8e5f85edb..1b2e61c143ae13 100644 --- a/src/database/sql/convert_test.go +++ b/src/database/sql/convert_test.go @@ -7,6 +7,7 @@ package sql import ( "database/sql/driver" "fmt" + "internal/asan" "reflect" "runtime" "strings" @@ -353,6 +354,9 @@ func TestRawBytesAllocs(t *testing.T) { {"bool", false, "false"}, {"time", time.Unix(2, 5).UTC(), "1970-01-01T00:00:02.000000005Z"}, } + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } var buf RawBytes rows := &Rows{} diff --git a/src/database/sql/driver/driver.go b/src/database/sql/driver/driver.go index d0892e80fc28d5..487870be63209e 100644 --- a/src/database/sql/driver/driver.go +++ b/src/database/sql/driver/driver.go @@ -515,6 +515,18 @@ type RowsColumnTypePrecisionScale interface { ColumnTypePrecisionScale(index int) (precision, scale int64, ok bool) } +// RowsColumnScanner may be implemented by [Rows]. It allows the driver to completely +// take responsibility for how values are scanned and replace the normal [database/sql]. +// scanning path. This allows drivers to directly support types that do not implement +// [database/sql.Scanner]. +type RowsColumnScanner interface { + Rows + + // ScanColumn copies the column in the current row into the value pointed at by + // dest. It returns [ErrSkip] to fall back to the normal [database/sql] scanning path. + ScanColumn(dest any, index int) error +} + // Tx is a transaction. type Tx interface { Commit() error diff --git a/src/database/sql/fakedb_test.go b/src/database/sql/fakedb_test.go index 3dfcd447b52bca..003e6c62986f31 100644 --- a/src/database/sql/fakedb_test.go +++ b/src/database/sql/fakedb_test.go @@ -5,6 +5,7 @@ package sql import ( + "bytes" "context" "database/sql/driver" "errors" @@ -15,7 +16,6 @@ import ( "strconv" "strings" "sync" - "sync/atomic" "testing" "time" ) @@ -91,8 +91,6 @@ func (cc *fakeDriverCtx) OpenConnector(name string) (driver.Connector, error) { type fakeDB struct { name string - useRawBytes atomic.Bool - mu sync.Mutex tables map[string]*table badConn bool @@ -684,8 +682,6 @@ func (c *fakeConn) PrepareContext(ctx context.Context, query string) (driver.Stm switch cmd { case "WIPE": // Nothing - case "USE_RAWBYTES": - c.db.useRawBytes.Store(true) case "SELECT": stmt, err = c.prepareSelect(stmt, parts) case "CREATE": @@ -789,9 +785,6 @@ func (s *fakeStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (d case "WIPE": db.wipe() return driver.ResultNoRows, nil - case "USE_RAWBYTES": - s.c.db.useRawBytes.Store(true) - return driver.ResultNoRows, nil case "CREATE": if err := db.createTable(s.table, s.colName, s.colType); err != nil { return nil, err @@ -1076,10 +1069,9 @@ type rowsCursor struct { errPos int err error - // a clone of slices to give out to clients, indexed by the - // original slice's first byte address. we clone them - // just so we're able to corrupt them on close. - bytesClone map[*byte][]byte + // Data returned to clients. + // We clone and stash it here so it can be invalidated by Close and Next. + driverOwnedMemory [][]byte // Every operation writes to line to enable the race detector // check for data races. @@ -1096,9 +1088,19 @@ func (rc *rowsCursor) touchMem() { rc.line++ } +func (rc *rowsCursor) invalidateDriverOwnedMemory() { + for _, buf := range rc.driverOwnedMemory { + for i := range buf { + buf[i] = 'x' + } + } + rc.driverOwnedMemory = nil +} + func (rc *rowsCursor) Close() error { rc.touchMem() rc.parentMem.touchMem() + rc.invalidateDriverOwnedMemory() rc.closed = true return rc.closeErr } @@ -1129,6 +1131,8 @@ func (rc *rowsCursor) Next(dest []driver.Value) error { if rc.posRow >= len(rc.rows[rc.posSet]) { return io.EOF // per interface spec } + // Corrupt any previously returned bytes. + rc.invalidateDriverOwnedMemory() for i, v := range rc.rows[rc.posSet][rc.posRow].cols { // TODO(bradfitz): convert to subset types? naah, I // think the subset types should only be input to @@ -1136,20 +1140,13 @@ func (rc *rowsCursor) Next(dest []driver.Value) error { // a wider range of types coming out of drivers. all // for ease of drivers, and to prevent drivers from // messing up conversions or doing them differently. - dest[i] = v - - if bs, ok := v.([]byte); ok && !rc.db.useRawBytes.Load() { - if rc.bytesClone == nil { - rc.bytesClone = make(map[*byte][]byte) - } - clone, ok := rc.bytesClone[&bs[0]] - if !ok { - clone = make([]byte, len(bs)) - copy(clone, bs) - rc.bytesClone[&bs[0]] = clone - } - dest[i] = clone + if bs, ok := v.([]byte); ok { + // Clone []bytes and stash for later invalidation. + bs = bytes.Clone(bs) + rc.driverOwnedMemory = append(rc.driverOwnedMemory, bs) + v = bs } + dest[i] = v } return nil } diff --git a/src/database/sql/sql.go b/src/database/sql/sql.go index de774a051093df..85b9ffc37d9445 100644 --- a/src/database/sql/sql.go +++ b/src/database/sql/sql.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "maps" "math/rand/v2" "reflect" "runtime" @@ -75,12 +76,7 @@ func unregisterAllDrivers() { func Drivers() []string { driversMu.RLock() defer driversMu.RUnlock() - list := make([]string, 0, len(drivers)) - for name := range drivers { - list = append(list, name) - } - slices.Sort(list) - return list + return slices.Sorted(maps.Keys(drivers)) } // A NamedArg is a named argument. NamedArg values may be used as @@ -414,6 +410,8 @@ func (n NullTime) Value() (driver.Value, error) { // } else { // // NULL value // } +// +// T should be one of the types accepted by [driver.Value]. type Null[T any] struct { V T Valid bool @@ -432,7 +430,17 @@ func (n Null[T]) Value() (driver.Value, error) { if !n.Valid { return nil, nil } - return n.V, nil + v := any(n.V) + // See issue 69728. + if valuer, ok := v.(driver.Valuer); ok { + val, err := callValuerValue(valuer) + if err != nil { + return val, err + } + v = val + } + // See issue 69837. + return driver.DefaultParameterConverter.ConvertValue(v) } // Scanner is an interface used by [Rows.Scan]. @@ -1042,7 +1050,7 @@ func (db *DB) SetConnMaxLifetime(d time.Duration) { } db.mu.Lock() // Wake cleaner up when lifetime is shortened. - if d > 0 && d < db.maxLifetime && db.cleanerCh != nil { + if d > 0 && d < db.shortestIdleTimeLocked() && db.cleanerCh != nil { select { case db.cleanerCh <- struct{}{}: default: @@ -1066,7 +1074,7 @@ func (db *DB) SetConnMaxIdleTime(d time.Duration) { defer db.mu.Unlock() // Wake cleaner up when idle time is shortened. - if d > 0 && d < db.maxIdleTime && db.cleanerCh != nil { + if d > 0 && d < db.shortestIdleTimeLocked() && db.cleanerCh != nil { select { case db.cleanerCh <- struct{}{}: default: @@ -1368,8 +1376,8 @@ func (db *DB) conn(ctx context.Context, strategy connReuseStrategy) (*driverConn db.waitDuration.Add(int64(time.Since(waitStart))) - // If we failed to delete it, that means something else - // grabbed it and is about to send on it. + // If we failed to delete it, that means either the DB was closed or + // something else grabbed it and is about to send on it. if !deleted { // TODO(bradfitz): rather than this best effort select, we // should probably start a goroutine to read from req. This best @@ -3360,38 +3368,45 @@ func (rs *Rows) Scan(dest ...any) error { // without calling Next. return fmt.Errorf("sql: Scan called without calling Next (closemuScanHold)") } + rs.closemu.RLock() + rs.raw = rs.raw[:0] + err := rs.scanLocked(dest...) + if err == nil && scanArgsContainRawBytes(dest) { + rs.closemuScanHold = true + } else { + rs.closemu.RUnlock() + } + return err +} +func (rs *Rows) scanLocked(dest ...any) error { if rs.lasterr != nil && rs.lasterr != io.EOF { - rs.closemu.RUnlock() return rs.lasterr } if rs.closed { - err := rs.lasterrOrErrLocked(errRowsClosed) - rs.closemu.RUnlock() - return err - } - - if scanArgsContainRawBytes(dest) { - rs.closemuScanHold = true - rs.raw = rs.raw[:0] - } else { - rs.closemu.RUnlock() + return rs.lasterrOrErrLocked(errRowsClosed) } if rs.lastcols == nil { - rs.closemuRUnlockIfHeldByScan() return errors.New("sql: Scan called without calling Next") } if len(dest) != len(rs.lastcols) { - rs.closemuRUnlockIfHeldByScan() return fmt.Errorf("sql: expected %d destination arguments in Scan, not %d", len(rs.lastcols), len(dest)) } for i, sv := range rs.lastcols { - err := convertAssignRows(dest[i], sv, rs) + err := driver.ErrSkip + + if rcs, ok := rs.rowsi.(driver.RowsColumnScanner); ok { + err = rcs.ScanColumn(dest[i], i) + } + + if err == driver.ErrSkip { + err = convertAssignRows(dest[i], sv, rs) + } + if err != nil { - rs.closemuRUnlockIfHeldByScan() return fmt.Errorf(`sql: Scan error on column index %d, name %q: %w`, i, rs.rowsi.Columns()[i], err) } } @@ -3594,6 +3609,7 @@ type connRequestAndIndex struct { // and clears the set. func (s *connRequestSet) CloseAndRemoveAll() { for _, v := range s.s { + *v.curIdx = -1 close(v.req) } s.s = nil diff --git a/src/database/sql/sql_test.go b/src/database/sql/sql_test.go index ff65e877a5af6b..c3f228ef0ba104 100644 --- a/src/database/sql/sql_test.go +++ b/src/database/sql/sql_test.go @@ -5,6 +5,7 @@ package sql import ( + "bytes" "context" "database/sql/driver" "errors" @@ -1374,8 +1375,7 @@ func TestConnQuery(t *testing.T) { db := newTestDB(t, "people") defer closeDB(t, db) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() conn, err := db.Conn(ctx) if err != nil { t.Fatal(err) @@ -1402,8 +1402,7 @@ func TestConnRaw(t *testing.T) { db := newTestDB(t, "people") defer closeDB(t, db) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() conn, err := db.Conn(ctx) if err != nil { t.Fatal(err) @@ -1518,8 +1517,7 @@ func TestInvalidNilValues(t *testing.T) { db := newTestDB(t, "people") defer closeDB(t, db) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() conn, err := db.Conn(ctx) if err != nil { t.Fatal(err) @@ -1547,8 +1545,7 @@ func TestConnTx(t *testing.T) { db := newTestDB(t, "people") defer closeDB(t, db) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() conn, err := db.Conn(ctx) if err != nil { t.Fatal(err) @@ -2793,8 +2790,7 @@ func TestManyErrBadConn(t *testing.T) { // Conn db = manyErrBadConnSetup() defer closeDB(t, db) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() conn, err := db.Conn(ctx) if err != nil { t.Fatal(err) @@ -2935,8 +2931,7 @@ func TestConnExpiresFreshOutOfPool(t *testing.T) { } defer func() { nowFunc = time.Now }() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() db := newTestDB(t, "magicquery") defer closeDB(t, db) @@ -2944,7 +2939,6 @@ func TestConnExpiresFreshOutOfPool(t *testing.T) { db.SetMaxOpenConns(1) for _, ec := range execCases { - ec := ec name := fmt.Sprintf("expired=%t,badReset=%t", ec.expired, ec.badReset) t.Run(name, func(t *testing.T) { db.clearAllConns(t) @@ -3786,8 +3780,7 @@ func TestIssue20647(t *testing.T) { db := newTestDB(t, "people") defer closeDB(t, db) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() conn, err := db.Conn(ctx) if err != nil { @@ -4142,9 +4135,7 @@ func TestNamedValueChecker(t *testing.T) { } defer db.Close() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - + ctx := t.Context() _, err = db.ExecContext(ctx, "WIPE") if err != nil { t.Fatal("exec wipe", err) @@ -4192,9 +4183,7 @@ func TestNamedValueCheckerSkip(t *testing.T) { } defer db.Close() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - + ctx := t.Context() _, err = db.ExecContext(ctx, "WIPE") if err != nil { t.Fatal("exec wipe", err) @@ -4211,6 +4200,102 @@ func TestNamedValueCheckerSkip(t *testing.T) { } } +type rcsDriver struct { + fakeDriver +} + +func (d *rcsDriver) Open(dsn string) (driver.Conn, error) { + c, err := d.fakeDriver.Open(dsn) + fc := c.(*fakeConn) + fc.db.allowAny = true + return &rcsConn{fc}, err +} + +type rcsConn struct { + *fakeConn +} + +func (c *rcsConn) PrepareContext(ctx context.Context, q string) (driver.Stmt, error) { + stmt, err := c.fakeConn.PrepareContext(ctx, q) + if err != nil { + return stmt, err + } + return &rcsStmt{stmt.(*fakeStmt)}, nil +} + +type rcsStmt struct { + *fakeStmt +} + +func (s *rcsStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + rows, err := s.fakeStmt.QueryContext(ctx, args) + if err != nil { + return rows, err + } + return &rcsRows{rows.(*rowsCursor)}, nil +} + +type rcsRows struct { + *rowsCursor +} + +func (r *rcsRows) ScanColumn(dest any, index int) error { + switch d := dest.(type) { + case *int64: + *d = 42 + return nil + } + + return driver.ErrSkip +} + +func TestRowsColumnScanner(t *testing.T) { + Register("RowsColumnScanner", &rcsDriver{}) + db, err := Open("RowsColumnScanner", "") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, err = db.ExecContext(ctx, "CREATE|t|str=string,n=int64") + if err != nil { + t.Fatal("exec create", err) + } + + _, err = db.ExecContext(ctx, "INSERT|t|str=?,n=?", "foo", int64(1)) + if err != nil { + t.Fatal("exec insert", err) + } + var ( + str string + i64 int64 + i int + f64 float64 + ui uint + ) + err = db.QueryRowContext(ctx, "SELECT|t|str,n,n,n,n|").Scan(&str, &i64, &i, &f64, &ui) + if err != nil { + t.Fatal("select", err) + } + + list := []struct{ got, want any }{ + {str, "foo"}, + {i64, int64(42)}, + {i, int(1)}, + {f64, float64(1)}, + {ui, uint(1)}, + } + + for index, item := range list { + if !reflect.DeepEqual(item.got, item.want) { + t.Errorf("got %#v wanted %#v for index %d", item.got, item.want, index) + } + } +} + func TestOpenConnector(t *testing.T) { Register("testctx", &fakeDriverCtx{}) db, err := Open("testctx", "people") @@ -4305,8 +4390,7 @@ func TestQueryExecContextOnly(t *testing.T) { } defer db.Close() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() conn, err := db.Conn(ctx) if err != nil { @@ -4446,10 +4530,6 @@ func testContextCancelDuringRawBytesScan(t *testing.T, mode string) { db := newTestDB(t, "people") defer closeDB(t, db) - if _, err := db.Exec("USE_RAWBYTES"); err != nil { - t.Fatal(err) - } - // cancel used to call close asynchronously. // This test checks that it waits so as not to interfere with RawBytes. ctx, cancel := context.WithCancel(context.Background()) @@ -4541,6 +4621,61 @@ func TestContextCancelBetweenNextAndErr(t *testing.T) { } } +type testScanner struct { + scanf func(src any) error +} + +func (ts testScanner) Scan(src any) error { return ts.scanf(src) } + +func TestContextCancelDuringScan(t *testing.T) { + db := newTestDB(t, "people") + defer closeDB(t, db) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + scanStart := make(chan any) + scanEnd := make(chan error) + scanner := &testScanner{ + scanf: func(src any) error { + scanStart <- src + return <-scanEnd + }, + } + + // Start a query, and pause it mid-scan. + want := []byte("Alice") + r, err := db.QueryContext(ctx, "SELECT|people|name|name=?", string(want)) + if err != nil { + t.Fatal(err) + } + if !r.Next() { + t.Fatalf("r.Next() = false, want true") + } + go func() { + r.Scan(scanner) + }() + got := <-scanStart + defer close(scanEnd) + gotBytes, ok := got.([]byte) + if !ok { + t.Fatalf("r.Scan returned %T, want []byte", got) + } + if !bytes.Equal(gotBytes, want) { + t.Fatalf("before cancel: r.Scan returned %q, want %q", gotBytes, want) + } + + // Cancel the query. + // Sleep to give it a chance to finish canceling. + cancel() + time.Sleep(10 * time.Millisecond) + + // Cancelling the query should not have changed the result. + if !bytes.Equal(gotBytes, want) { + t.Fatalf("after cancel: r.Scan result is now %q, want %q", gotBytes, want) + } +} + func TestNilErrorAfterClose(t *testing.T) { db := newTestDB(t, "people") defer closeDB(t, db) @@ -4574,10 +4709,6 @@ func TestRawBytesReuse(t *testing.T) { db := newTestDB(t, "people") defer closeDB(t, db) - if _, err := db.Exec("USE_RAWBYTES"); err != nil { - t.Fatal(err) - } - var raw RawBytes // The RawBytes in this query aliases driver-owned memory. @@ -4920,6 +5051,17 @@ func TestConnRequestSet(t *testing.T) { t.Error("wasn't random") } }) + t.Run("close-delete", func(t *testing.T) { + reset() + ch := make(chan connRequest) + dh := s.Add(ch) + wantLen(1) + s.CloseAndRemoveAll() + wantLen(0) + if s.Delete(dh) { + t.Error("unexpected delete after CloseAndRemoveAll") + } + }) } func BenchmarkConnRequestSet(b *testing.B) { @@ -4946,3 +5088,50 @@ func BenchmarkConnRequestSet(b *testing.B) { } } } + +func TestIssue69837(t *testing.T) { + u := Null[uint]{V: 1, Valid: true} + val, err := driver.DefaultParameterConverter.ConvertValue(u) + if err != nil { + t.Errorf("ConvertValue() error = %v, want nil", err) + } + + if v, ok := val.(int64); !ok { + t.Errorf("val.(type): got %T, expected int64", val) + } else if v != 1 { + t.Errorf("val: got %d, expected 1", v) + } +} + +type issue69728Type struct { + ID int + Name string +} + +func (t issue69728Type) Value() (driver.Value, error) { + return []byte(fmt.Sprintf("%d, %s", t.ID, t.Name)), nil +} + +func TestIssue69728(t *testing.T) { + forValue := Null[issue69728Type]{ + Valid: true, + V: issue69728Type{ + ID: 42, + Name: "foobar", + }, + } + + v1, err := forValue.Value() + if err != nil { + t.Errorf("forValue.Value() error = %v, want nil", err) + } + + v2, err := forValue.V.Value() + if err != nil { + t.Errorf("forValue.V.Value() error = %v, want nil", err) + } + + if !reflect.DeepEqual(v1, v2) { + t.Errorf("not equal; v1 = %v, v2 = %v", v1, v2) + } +} diff --git a/src/debug/buildinfo/buildinfo.go b/src/debug/buildinfo/buildinfo.go index 8338f03fa580de..d202d5050a2786 100644 --- a/src/debug/buildinfo/buildinfo.go +++ b/src/debug/buildinfo/buildinfo.go @@ -57,12 +57,17 @@ var errNotGoExe = errors.New("not a Go executable") // fields. var buildInfoMagic = []byte("\xff Go buildinf:") +const ( + buildInfoAlign = 16 + buildInfoHeaderSize = 32 +) + // ReadFile returns build information embedded in a Go binary // file at the given path. Most information is only available for binaries built // with module support. func ReadFile(name string) (info *BuildInfo, err error) { defer func() { - if pathErr := (*fs.PathError)(nil); errors.As(err, &pathErr) { + if _, ok := errors.AsType[*fs.PathError](err); ok { err = fmt.Errorf("could not read Go build info: %w", err) } else if err != nil { err = fmt.Errorf("could not read Go build info from %s: %w", name, err) @@ -94,13 +99,14 @@ func Read(r io.ReaderAt) (*BuildInfo, error) { } type exe interface { - // ReadData reads and returns up to size bytes starting at virtual address addr. - ReadData(addr, size uint64) ([]byte, error) - // DataStart returns the virtual address and size of the segment or section that // should contain build information. This is either a specially named section // or the first writable non-zero data segment. DataStart() (uint64, uint64) + + // DataReader returns an io.ReaderAt that reads from addr until the end + // of segment or section that contains addr. + DataReader(addr uint64) (io.ReaderAt, error) } // readRawBuildInfo extracts the Go toolchain version and module information @@ -165,14 +171,24 @@ func readRawBuildInfo(r io.ReaderAt) (vers, mod string, err error) { if dataSize == 0 { return "", "", errNotGoExe } - data, err := x.ReadData(dataAddr, dataSize) + + addr, err := searchMagic(x, dataAddr, dataSize) if err != nil { return "", "", err } - const ( - buildInfoAlign = 16 - buildInfoHeaderSize = 32 + // Read in the full header first. + header, err := readData(x, addr, buildInfoHeaderSize) + if err == io.EOF { + return "", "", errNotGoExe + } else if err != nil { + return "", "", err + } + if len(header) < buildInfoHeaderSize { + return "", "", errNotGoExe + } + + const ( ptrSizeOffset = 14 flagsOffset = 15 versPtrOffset = 16 @@ -185,17 +201,6 @@ func readRawBuildInfo(r io.ReaderAt) (vers, mod string, err error) { flagsVersionPtr = 0x0 flagsVersionInl = 0x2 ) - for { - i := bytes.Index(data, buildInfoMagic) - if i < 0 || len(data)-i < buildInfoHeaderSize { - return "", "", errNotGoExe - } - if i%buildInfoAlign == 0 && len(data)-i >= buildInfoHeaderSize { - data = data[i:] - break - } - data = data[(i+buildInfoAlign-1)&^(buildInfoAlign-1):] - } // Decode the blob. The blob is a 32-byte header, optionally followed // by 2 varint-prefixed string contents. @@ -220,13 +225,19 @@ func readRawBuildInfo(r io.ReaderAt) (vers, mod string, err error) { // the header is followed by the string contents inline as // length-prefixed (as varint) string contents. First is the version // string, followed immediately by the modinfo string. - flags := data[flagsOffset] + flags := header[flagsOffset] if flags&flagsVersionMask == flagsVersionInl { - vers, data = decodeString(data[buildInfoHeaderSize:]) - mod, data = decodeString(data) + vers, addr, err = decodeString(x, addr+buildInfoHeaderSize) + if err != nil { + return "", "", err + } + mod, _, err = decodeString(x, addr) + if err != nil { + return "", "", err + } } else { // flagsVersionPtr (<1.18) - ptrSize := int(data[ptrSizeOffset]) + ptrSize := int(header[ptrSizeOffset]) bigEndian := flags&flagsEndianMask == flagsEndianBig var bo binary.ByteOrder if bigEndian { @@ -242,8 +253,8 @@ func readRawBuildInfo(r io.ReaderAt) (vers, mod string, err error) { } else { return "", "", errNotGoExe } - vers = readString(x, ptrSize, readPtr, readPtr(data[versPtrOffset:])) - mod = readString(x, ptrSize, readPtr, readPtr(data[versPtrOffset+ptrSize:])) + vers = readString(x, ptrSize, readPtr, readPtr(header[versPtrOffset:])) + mod = readString(x, ptrSize, readPtr, readPtr(header[versPtrOffset+ptrSize:])) } if vers == "" { return "", "", errNotGoExe @@ -270,42 +281,172 @@ func hasPlan9Magic(magic []byte) bool { return false } -func decodeString(data []byte) (s string, rest []byte) { - u, n := binary.Uvarint(data) - if n <= 0 || u > uint64(len(data)-n) { - return "", nil +func decodeString(x exe, addr uint64) (string, uint64, error) { + // varint length followed by length bytes of data. + + // N.B. ReadData reads _up to_ size bytes from the section containing + // addr. So we don't need to check that size doesn't overflow the + // section. + b, err := readData(x, addr, binary.MaxVarintLen64) + if err == io.EOF { + return "", 0, errNotGoExe + } else if err != nil { + return "", 0, err } - return string(data[n : uint64(n)+u]), data[uint64(n)+u:] + + length, n := binary.Uvarint(b) + if n <= 0 { + return "", 0, errNotGoExe + } + addr += uint64(n) + + b, err = readData(x, addr, length) + if err == io.EOF { + return "", 0, errNotGoExe + } else if err == io.ErrUnexpectedEOF { + // Length too large to allocate. Clearly bogus value. + return "", 0, errNotGoExe + } else if err != nil { + return "", 0, err + } + if uint64(len(b)) < length { + // Section ended before we could read the full string. + return "", 0, errNotGoExe + } + + return string(b), addr + length, nil } // readString returns the string at address addr in the executable x. func readString(x exe, ptrSize int, readPtr func([]byte) uint64, addr uint64) string { - hdr, err := x.ReadData(addr, uint64(2*ptrSize)) + hdr, err := readData(x, addr, uint64(2*ptrSize)) if err != nil || len(hdr) < 2*ptrSize { return "" } dataAddr := readPtr(hdr) dataLen := readPtr(hdr[ptrSize:]) - data, err := x.ReadData(dataAddr, dataLen) + data, err := readData(x, dataAddr, dataLen) if err != nil || uint64(len(data)) < dataLen { return "" } return string(data) } +const searchChunkSize = 1 << 20 // 1 MB + +// searchMagic returns the aligned first instance of buildInfoMagic in the data +// range [addr, addr+size). Returns false if not found. +func searchMagic(x exe, start, size uint64) (uint64, error) { + end := start + size + if end < start { + // Overflow. + return 0, errUnrecognizedFormat + } + + // Round up start; magic can't occur in the initial unaligned portion. + start = (start + buildInfoAlign - 1) &^ (buildInfoAlign - 1) + if start >= end { + return 0, errNotGoExe + } + + var buf []byte + for start < end { + // Read in chunks to avoid consuming too much memory if data is large. + // + // Normally it would be somewhat painful to handle the magic crossing a + // chunk boundary, but since it must be 16-byte aligned we know it will + // fall within a single chunk. + remaining := end - start + chunkSize := uint64(searchChunkSize) + if chunkSize > remaining { + chunkSize = remaining + } + + if buf == nil { + buf = make([]byte, chunkSize) + } else { + // N.B. chunkSize can only decrease, and only on the + // last chunk. + buf = buf[:chunkSize] + clear(buf) + } + + n, err := readDataInto(x, start, buf) + if err == io.EOF { + // EOF before finding the magic; must not be a Go executable. + return 0, errNotGoExe + } else if err != nil { + return 0, err + } + + data := buf[:n] + for len(data) > 0 { + i := bytes.Index(data, buildInfoMagic) + if i < 0 { + break + } + if remaining-uint64(i) < buildInfoHeaderSize { + // Found magic, but not enough space left for the full header. + return 0, errNotGoExe + } + if i%buildInfoAlign != 0 { + // Found magic, but misaligned. Keep searching. + next := (i + buildInfoAlign - 1) &^ (buildInfoAlign - 1) + if next > len(data) { + // Corrupt object file: the remaining + // count says there is more data, + // but we didn't read it. + return 0, errNotGoExe + } + data = data[next:] + continue + } + // Good match! + return start + uint64(i), nil + } + + start += chunkSize + } + + return 0, errNotGoExe +} + +func readData(x exe, addr, size uint64) ([]byte, error) { + r, err := x.DataReader(addr) + if err != nil { + return nil, err + } + + b, err := saferio.ReadDataAt(r, size, 0) + if len(b) > 0 && err == io.EOF { + err = nil + } + return b, err +} + +func readDataInto(x exe, addr uint64, b []byte) (int, error) { + r, err := x.DataReader(addr) + if err != nil { + return 0, err + } + + n, err := r.ReadAt(b, 0) + if n > 0 && err == io.EOF { + err = nil + } + return n, err +} + // elfExe is the ELF implementation of the exe interface. type elfExe struct { f *elf.File } -func (x *elfExe) ReadData(addr, size uint64) ([]byte, error) { +func (x *elfExe) DataReader(addr uint64) (io.ReaderAt, error) { for _, prog := range x.f.Progs { if prog.Vaddr <= addr && addr <= prog.Vaddr+prog.Filesz-1 { - n := prog.Vaddr + prog.Filesz - addr - if n > size { - n = size - } - return saferio.ReadDataAt(prog, n, int64(addr-prog.Vaddr)) + remaining := prog.Vaddr + prog.Filesz - addr + return io.NewSectionReader(prog, int64(addr-prog.Vaddr), int64(remaining)), nil } } return nil, errUnrecognizedFormat @@ -340,15 +481,12 @@ func (x *peExe) imageBase() uint64 { return 0 } -func (x *peExe) ReadData(addr, size uint64) ([]byte, error) { +func (x *peExe) DataReader(addr uint64) (io.ReaderAt, error) { addr -= x.imageBase() for _, sect := range x.f.Sections { if uint64(sect.VirtualAddress) <= addr && addr <= uint64(sect.VirtualAddress+sect.Size-1) { - n := uint64(sect.VirtualAddress+sect.Size) - addr - if n > size { - n = size - } - return saferio.ReadDataAt(sect, n, int64(addr-uint64(sect.VirtualAddress))) + remaining := uint64(sect.VirtualAddress+sect.Size) - addr + return io.NewSectionReader(sect, int64(addr-uint64(sect.VirtualAddress)), int64(remaining)), nil } } return nil, errUnrecognizedFormat @@ -381,7 +519,7 @@ type machoExe struct { f *macho.File } -func (x *machoExe) ReadData(addr, size uint64) ([]byte, error) { +func (x *machoExe) DataReader(addr uint64) (io.ReaderAt, error) { for _, load := range x.f.Loads { seg, ok := load.(*macho.Segment) if !ok { @@ -391,11 +529,8 @@ func (x *machoExe) ReadData(addr, size uint64) ([]byte, error) { if seg.Name == "__PAGEZERO" { continue } - n := seg.Addr + seg.Filesz - addr - if n > size { - n = size - } - return saferio.ReadDataAt(seg, n, int64(addr-seg.Addr)) + remaining := seg.Addr + seg.Filesz - addr + return io.NewSectionReader(seg, int64(addr-seg.Addr), int64(remaining)), nil } } return nil, errUnrecognizedFormat @@ -424,14 +559,11 @@ type xcoffExe struct { f *xcoff.File } -func (x *xcoffExe) ReadData(addr, size uint64) ([]byte, error) { +func (x *xcoffExe) DataReader(addr uint64) (io.ReaderAt, error) { for _, sect := range x.f.Sections { if sect.VirtualAddress <= addr && addr <= sect.VirtualAddress+sect.Size-1 { - n := sect.VirtualAddress + sect.Size - addr - if n > size { - n = size - } - return saferio.ReadDataAt(sect, n, int64(addr-sect.VirtualAddress)) + remaining := sect.VirtualAddress + sect.Size - addr + return io.NewSectionReader(sect, int64(addr-sect.VirtualAddress), int64(remaining)), nil } } return nil, errors.New("address not mapped") @@ -456,14 +588,11 @@ func (x *plan9objExe) DataStart() (uint64, uint64) { return 0, 0 } -func (x *plan9objExe) ReadData(addr, size uint64) ([]byte, error) { +func (x *plan9objExe) DataReader(addr uint64) (io.ReaderAt, error) { for _, sect := range x.f.Sections { if uint64(sect.Offset) <= addr && addr <= uint64(sect.Offset+sect.Size-1) { - n := uint64(sect.Offset+sect.Size) - addr - if n > size { - n = size - } - return saferio.ReadDataAt(sect, n, int64(addr-uint64(sect.Offset))) + remaining := uint64(sect.Offset+sect.Size) - addr + return io.NewSectionReader(sect, int64(addr-uint64(sect.Offset)), int64(remaining)), nil } } return nil, errors.New("address not mapped") diff --git a/src/debug/buildinfo/buildinfo_test.go b/src/debug/buildinfo/buildinfo_test.go index e78099da21676c..ceab14e8bff622 100644 --- a/src/debug/buildinfo/buildinfo_test.go +++ b/src/debug/buildinfo/buildinfo_test.go @@ -11,6 +11,7 @@ import ( "encoding/binary" "flag" "fmt" + "internal/obscuretestdata" "internal/testenv" "os" "os/exec" @@ -237,16 +238,13 @@ func TestReadFile(t *testing.T) { } for _, p := range platforms { - p := p t.Run(p.goos+"_"+p.goarch, func(t *testing.T) { if p != runtimePlatform && !*flagAll { t.Skipf("skipping platforms other than %s_%s because -all was not set", runtimePlatform.goos, runtimePlatform.goarch) } for _, mode := range buildModes { - mode := mode t.Run(mode, func(t *testing.T) { for _, tc := range cases { - tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() name := tc.build(t, p.goos, p.goarch, mode) @@ -275,24 +273,16 @@ func TestReadFile(t *testing.T) { // Test117 verifies that parsing of the old, pre-1.18 format works. func Test117(t *testing.T) { - // go117 was generated for linux-amd64 with: - // - // main.go: - // - // package main - // func main() {} - // - // GOTOOLCHAIN=go1.17 go mod init example.com/go117 - // GOTOOLCHAIN=go1.17 go build - // - // TODO(prattmic): Ideally this would be built on the fly to better - // cover all executable formats, but then we need a network connection - // to download an old Go toolchain. - info, err := buildinfo.ReadFile("testdata/go117") + b, err := obscuretestdata.ReadFile("testdata/go117/go117.base64") if err != nil { t.Fatalf("ReadFile got err %v, want nil", err) } + info, err := buildinfo.Read(bytes.NewReader(b)) + if err != nil { + t.Fatalf("Read got err %v, want nil", err) + } + if info.GoVersion != "go1.17" { t.Errorf("GoVersion got %s want go1.17", info.GoVersion) } @@ -306,20 +296,14 @@ func Test117(t *testing.T) { // TestNotGo verifies that parsing of a non-Go binary returns the proper error. func TestNotGo(t *testing.T) { - // notgo was generated for linux-amd64 with: - // - // main.c: - // - // int main(void) { return 0; } - // - // cc -o notgo main.c - // - // TODO(prattmic): Ideally this would be built on the fly to better - // cover all executable formats, but then we need to encode the - // intricacies of calling each platform's C compiler. - _, err := buildinfo.ReadFile("testdata/notgo") + b, err := obscuretestdata.ReadFile("testdata/notgo/notgo.base64") + if err != nil { + t.Fatalf("ReadFile got err %v, want nil", err) + } + + _, err = buildinfo.Read(bytes.NewReader(b)) if err == nil { - t.Fatalf("ReadFile got nil err, want non-nil") + t.Fatalf("Read got nil err, want non-nil") } // The precise error text here isn't critical, but we want something @@ -408,3 +392,21 @@ func TestIssue54968(t *testing.T) { }) } } + +func FuzzRead(f *testing.F) { + go117, err := obscuretestdata.ReadFile("testdata/go117/go117.base64") + if err != nil { + f.Errorf("Error reading go117: %v", err) + } + f.Add(go117) + + notgo, err := obscuretestdata.ReadFile("testdata/notgo/notgo.base64") + if err != nil { + f.Errorf("Error reading notgo: %v", err) + } + f.Add(notgo) + + f.Fuzz(func(t *testing.T, in []byte) { + buildinfo.Read(bytes.NewReader(in)) + }) +} diff --git a/src/debug/buildinfo/search_test.go b/src/debug/buildinfo/search_test.go new file mode 100644 index 00000000000000..c598dcdf8d5dd5 --- /dev/null +++ b/src/debug/buildinfo/search_test.go @@ -0,0 +1,146 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package buildinfo + +import ( + "bytes" + "fmt" + "io" + "testing" +) + +type byteExe struct { + b []byte +} + +func (x *byteExe) DataReader(addr uint64) (io.ReaderAt, error) { + if addr >= uint64(len(x.b)) { + return nil, fmt.Errorf("ReadData(%d) out of bounds of %d-byte slice", addr, len(x.b)) + } + return bytes.NewReader(x.b[addr:]), nil +} + +func (x *byteExe) DataStart() (uint64, uint64) { + return 0, uint64(len(x.b)) +} + +func TestSearchMagic(t *testing.T) { + tests := []struct { + name string + data []byte + want uint64 + wantErr error + }{ + { + name: "beginning", + data: func() []byte { + b := make([]byte, buildInfoHeaderSize) + copy(b, buildInfoMagic) + return b + }(), + want: 0, + }, + { + name: "offset", + data: func() []byte { + b := make([]byte, 512) + copy(b[4*buildInfoAlign:], buildInfoMagic) + return b + }(), + want: 4 * buildInfoAlign, + }, + { + name: "second_chunk", + data: func() []byte { + b := make([]byte, 4*searchChunkSize) + copy(b[searchChunkSize+4*buildInfoAlign:], buildInfoMagic) + return b + }(), + want: searchChunkSize + 4*buildInfoAlign, + }, + { + name: "second_chunk_short", + data: func() []byte { + // Magic is 64-bytes into the second chunk, + // which is short; only exactly long enough to + // hold the header. + b := make([]byte, searchChunkSize+4*buildInfoAlign+buildInfoHeaderSize) + copy(b[searchChunkSize+4*buildInfoAlign:], buildInfoMagic) + return b + }(), + want: searchChunkSize + 4*buildInfoAlign, + }, + { + name: "missing", + data: func() []byte { + b := make([]byte, buildInfoHeaderSize) + return b + }(), + wantErr: errNotGoExe, + }, + { + name: "too_short", + data: func() []byte { + // There needs to be space for the entire + // header, not just the magic. + b := make([]byte, len(buildInfoMagic)) + copy(b, buildInfoMagic) + return b + }(), + wantErr: errNotGoExe, + }, + { + name: "misaligned", + data: func() []byte { + b := make([]byte, 512) + copy(b[7:], buildInfoMagic) + return b + }(), + wantErr: errNotGoExe, + }, + { + name: "misaligned_across_chunk", + data: func() []byte { + // Magic crosses chunk boundary. By definition, + // it has to be misaligned. + b := make([]byte, 2*searchChunkSize) + copy(b[searchChunkSize-8:], buildInfoMagic) + return b + }(), + wantErr: errNotGoExe, + }, + { + name: "header_across_chunk", + data: func() []byte { + // The magic is aligned within the first chunk, + // but the rest of the 32-byte header crosses + // the chunk boundary. + b := make([]byte, 2*searchChunkSize) + copy(b[searchChunkSize-buildInfoAlign:], buildInfoMagic) + return b + }(), + want: searchChunkSize - buildInfoAlign, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + x := &byteExe{tc.data} + dataAddr, dataSize := x.DataStart() + addr, err := searchMagic(x, dataAddr, dataSize) + if tc.wantErr == nil { + if err != nil { + t.Errorf("searchMagic got err %v want nil", err) + } + if addr != tc.want { + t.Errorf("searchMagic got addr %d want %d", addr, tc.want) + } + } else { + if err != tc.wantErr { + t.Errorf("searchMagic got err %v want %v", err, tc.wantErr) + } + } + }) + } +} diff --git a/src/debug/buildinfo/testdata/fuzz/FuzzRead/36aeb674e3454016 b/src/debug/buildinfo/testdata/fuzz/FuzzRead/36aeb674e3454016 new file mode 100644 index 00000000000000..85d3bebd1f459c --- /dev/null +++ b/src/debug/buildinfo/testdata/fuzz/FuzzRead/36aeb674e3454016 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00>\x00\x01\x00\x00\x00@\x10\x00\x00\x00\x00\x00\x00@\x00\x00\x00\x00\x00\x00\x00X6\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00@\x008\x00\r\x00@\x00\x1e\x00\x1d\x00\x06\x00\x00\x00\x04\x00\x00\x00@\x00\x00\x00\x00\x00\x00\x00@\x00\x00\x00\x00\x00\x00\x00@\x00\x00\x00\x00\x00\x00\x00\xd8\x02\x00\x00\x00\x00\x00\x00\xd8\x02\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00\x18\x03\x00\x00\x00\x00\x00\x00\x18\x03\x00\x00\x00\x00\x00\x00\x18\x03\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe0\x05\x00\x00\x00\x00\x00\x00\xe0\x05\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x05\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00=\x01\x00\x00\x00\x00\x00\x00=\x01\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x04\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\xdc\x00\x00\x00\x00\x00\x00\x00\xdc\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x06\x00\x00\x00\x00.\x00\x00\x00\x00\x00\x00\x00>\x00\x00\x00\x00\x00\x00\x00>\x00\x00\x00\x00\x00\x00\x10\x02\x00\x00\x00\x00\x00\x00~~~~~~~\x00\x00\x10\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x06\x00\x00\x00\x10.\x00\x00\x00\x00\x00\x00\x10>\x00\x00\x00\x00\x00\x00\x10>\x00\x00\x00\x00\x00\x00\xb0\x01\x00\x00\x00\x00\x00\x00\xb0\x01\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x04\x00\x00\x008\x03\x00\x00\x00\x00\x00\x008\x03\x00\x00\x00\x00\x00\x008\x03\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x04\x00\x00\x00X\x03\x00\x00\x00\x00\x00\x00X\x03\x00\x00\x00\x00\x00\x00X\x03\x00\x00\x00\x00\x00\x00D\x00\x00\x00\x00\x00\x00\x00D\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00S\xe5td\x04\x00\x00\x008\x03\x00\x00\x00\x00\x00\x008\x03\x00\x00\x00\x00\x00\x008\x03\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00P\xe5td\x04\x00\x00\x00\x04 \x00\x00\x00\x00\x00\x00\x04 \x00\x00\x00\x00\x00\x00\x04 \x00\x00\x00\x00\x00\x00,\x00\x00\x00\x00\x00\x00\x00,\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00Q\xe5td\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00R\xe5td\x04\x00\x00\x00\x00.\x00\x00\x00\x00\x00\x00\x00>\x00\x00\x00\x00\x00\x00\x00>\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00/lib64/ld-linux-x86-64.so.2\x00\x00\x00\x00\x00\x04\x00\x00\x00\x10\x00\x00\x00\x05\x00\x00\x00GNU\x00\x02\x80\x00\xc0\x04\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x14\x00\x00\x00\x03\x00\x00\x00GNU\x00ɳ>\vWT\xe4N\x97\x1e\x8f\xe4\x13\xe2\xa8\xd5\xe8Fm\xcd\x04\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00GNU\x00\x00\x00\x00\x00\x03\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x05\x00\x00\x00\x01\x00\x00\x00\x06\x00\x00\x00\x00\x00\x81\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\xd1e\xcem\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00C\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00_\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00n\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00__cxa_finalize\x00__libc_start_main\x00libc.so.6\x00GLIBC_2.2.5\x00GLIBC_2.34\x00_ITM_deregisterTMCloneTable\x00__gmon_start__\x00_ITM_registerTMCloneTable\x00\x00\x00\x02\x00\x01\x00\x01\x00\x01\x00\x03\x00\x00\x00\x00\x00\x01\x00\x02\x00\"\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00u\x1ai\t\x00\x00\x03\x00,\x00\x00\x00\x10\x00\x00\x00\xb4\x91\x96\x06\x00\x00\x02\x008\x00\x00\x00\x00\x00\x00\x00\x00>\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00 \x11\x00\x00\x00\x00\x00\x00\b>\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\xe0\x10\x00\x00\x00\x00\x00\x00\b@\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\b@\x00\x00\x00\x00\x00\x00\xc0?\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc8?\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd0?\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd8?\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe0?\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x7f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00$\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00H\x83\xec\bH\x8b\x05\xc5/\x00\x00H\x85\xc0t\x02\xff\xd0H\x83\xc4\b\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff5\xca/\x00\x00\xff%\xcc/\x00\x00\x0f\x1f@\x00\xff%\xaa/\x00\x00f\x90\x00\x00\x00\x00\x00\x00\x00\x001\xedI\x89\xd1^H\x89\xe2H\x83\xe4\xf0PTE1\xc01\xc9H\x8d=\xce\x00\x00\x00\xff\x15_/\x00\x00\xf4f.\x0f\x1f\x84\x00\x00\x00\x00\x00\x0f\x1f@\x00H\x8d=\x99/\x00\x00H\x8d\x05\x92/\x00\x00H9\xf8t\x15H\x8b\x05>/\x00\x00H\x85\xc0t\t\xff\xe0\x0f\x1f\x80\x00\x00\x00\x00\xc3\x0f\x1f\x80\x00\x00\x00\x00H\x8d=i/\x00\x00H\x8d5b/\x00\x00H)\xfeH\x89\xf0H\xc1\xee?H\xc1\xf8\x03H\x01\xc6H\xd1\xfet\x14H\x8b\x05\r/\x00\x00H\x85\xc0t\b\xff\xe0f\x0f\x1fD\x00\x00\xc3\x0f\x1f\x80\x00\x00\x00\x00\xf3\x0f\x1e\xfa\x80=%/\x00\x00\x00u+UH\x83=\xea.\x00\x00\x00H\x89\xe5t\fH\x8b=\x06/\x00\x00\xe8)\xff\xff\xff\xe8d\xff\xff\xff\xc6\x05\xfd.\x00\x00\x01]\xc3\x0f\x1f\x00\xc3\x0f\x1f\x80\x00\x00\x00\x00\xf3\x0f\x1e\xfa\xe9w\xff\xff\xffUH\x89\xe5\xb8\x00\x00\x00\x00]\xc3H\x83\xec\bH\x83\xc4\b\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x02\x00\x01\x1b\x03;(\x00\x00\x00\x04\x00\x00\x00\x1c\xf0\xff\xfft\x00\x00\x00,\xf0\xff\xff\x9c\x00\x00\x00<\xf0\xff\xffD\x00\x00\x00%\xf1\xff\xff\xb4\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x01zR\x00\x01x\x10\x01\x1b\f\a\b\x90\x01\a\x10\x14\x00\x00\x00\x1c\x00\x00\x00\xf0\xef\xff\xff\"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x01zR\x00\x01x\x10\x01\x1b\f\a\b\x90\x01\x00\x00$\x00\x00\x00\x1c\x00\x00\x00\xa0\xef\xff\xff\x10\x00\x00\x00\x00\x0e\x10F\x0e\x18J\x0f\vw\b\x80\x00?\x1a;*3$\"\x00\x00\x00\x00\x14\x00\x00\x00D\x00\x00\x00\x88\xef\xff\xff\b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00\\\x00\x00\x00i\xf0\xff\xff\v\x00\x00\x00\x00A\x0e\x10\x86\x02C\r\x06F\f\a\b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \x11\x00\x00\x00\x00\x00\x00\xe0\x10\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\"\x00\x00\x00\x00\x00\x00\x00\f\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\r\x00\x00\x00\x00\x00\x00\x004\x11\x00\x00\x00\x00\x00\x00\x19\x00\x00\x00\x00\x00\x00\x00\x00>\x00\x00\x00\x00\x00\x00\x1b\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x1a\x00\x00\x00\x00\x00\x00\x00\b>\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\xf5\xfe\xffo\x00\x00\x00\x00\xa0\x03\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00X\x04\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\xc8\x03\x00\x00\x00\x00\x00\x00\n\x00\x00\x00\x00\x00\x00\x00\x88\x00\x00\x00\x00\x00\x00\x00\v\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x15\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\xe8?\x00\x00\x00\x00\x00\x00\a\x00\x00\x00\x00\x00\x00\x00 \x05\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\xc0\x00\x00\x00\x00\x00\x00\x00\t\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\xfb\xff\xffo\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\xfe\xff\xffo\x00\x00\x00\x00\xf0\x04\x00\x00\x00\x00\x00\x00\xff\xff\xffo\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\xf0\xff\xffo\x00\x00\x00\x00\xe0\x04\x00\x00\x00\x00\x00\x00\xf9\xff\xffo\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\b@\x00\x00\x00\x00\x00\x00GCC: (Debian 13.2.0-13) 13.2.0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x04\x00\xf1\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\t\x00\x00\x00\x01\x00\x04\x00|\x03\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00\x13\x00\x00\x00\x04\x00\xf1\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1e\x00\x00\x00\x02\x00\x0e\x00p\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x02\x00\x0e\x00\xa0\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x003\x00\x00\x00\x02\x00\x0e\x00\xe0\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00I\x00\x00\x00\x01\x00\x19\x00\x10@\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00U\x00\x00\x00\x01\x00\x14\x00\b>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00|\x00\x00\x00\x02\x00\x0e\x00 \x11\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x88\x00\x00\x00\x01\x00\x13\x00\x00>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa7\x00\x00\x00\x04\x00\xf1\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x13\x00\x00\x00\x04\x00\xf1\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xae\x00\x00\x00\x01\x00\x12\x00\xd8 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\xf1\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbc\x00\x00\x00\x01\x00\x15\x00\x10>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc5\x00\x00\x00\x00\x00\x11\x00\x04 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd8\x00\x00\x00\x01\x00\x17\x00\xe8?\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xee\x00\x00\x00\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\v\x01\x00\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x006\x01\x00\x00 \x00\x18\x00\x00@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'\x01\x00\x00\x10\x00\x18\x00\x10@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00.\x01\x00\x00\x12\x02\x0f\x004\x11\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x004\x01\x00\x00\x10\x00\x18\x00\x00@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00A\x01\x00\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00P\x01\x00\x00\x11\x02\x18\x00\b@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00]\x01\x00\x00\x11\x00\x10\x00\x00 \x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00l\x01\x00\x00\x10\x00\x19\x00\x18@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00:\x01\x00\x00\x12\x00\x0e\x00@\x10\x00\x00\x00\x00\x00\x00\"\x00\x00\x00\x00\x00\x00\x00q\x01\x00\x00\x10\x00\x19\x00\x10@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00}\x01\x00\x00\x12\x00\x0e\x00)\x11\x00\x00\x00\x00\x00\x00\v\x00\x00\x00\x00\x00\x00\x00\x82\x01\x00\x00\x11\x02\x18\x00\x10@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8e\x01\x00\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa8\x01\x00\x00\"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc3\x01\x00\x00\x12\x02\v\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Scrt1.o\x00__abi_tag\x00crtstuff.c\x00deregister_tm_clones\x00__do_global_dtors_aux\x00completed.0\x00__do_global_dtors_aux_fini_array_entry\x00frame_dummy\x00__frame_dummy_init_array_entry\x00main.c\x00__FRAME_END__\x00_DYNAMIC\x00__GNU_EH_FRAME_HDR\x00_GLOBAL_OFFSET_TABLE_\x00__libc_start_main@GLIBC_2.34\x00_ITM_deregisterTMCloneTable\x00_edata\x00_fini\x00__data_start\x00__gmon_start__\x00__dso_handle\x00_IO_stdin_used\x00_end\x00__bss_start\x00main\x00__TMC_END__\x00_ITM_registerTMCloneTable\x00__cxa_finalize@GLIBC_2.2.5\x00_init\x00\x00.symtab\x00.strtab\x00.shstrtab\x00.interp\x00.note.gnu.property\x00.note.gnu.build-id\x00.note.ABI-tag\x00.gnu.hash\x00.dynsym\x00.dynstr\x00.gnu.version\x00.gnu.version_r\x00.rela.dyn\x00.init\x00.plt.got\x00.text\x00.fini\x00.rodata\x00.eh_frame_hdr\x00.eh_frame\x00.init_array\x00.fini_array\x00.dynamic\x00.got.plt\x00.data\x00.bss\x00.comment\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1b\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x18\x03\x00\x00\x00\x00\x00\x00\x18\x03\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00#\x00\x00\x00\a\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x008\x03\x00\x00\x00\x00\x00\x008\x03\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x006\x00\x00\x00\a\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00X\x03\x00\x00\x00\x00\x00\x00X\x03\x00\x00\x00\x00\x00\x00$\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00I\x00\x00\x00\a\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00|\x03\x00\x00\x00\x00\x00\x00|\x03\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00W\x00\x00\x00\xf6\xff\xffo\x02\x00\x00\x00\x00\x00\x00\x00\xa0\x03\x00\x00\x00\x00\x00\x00\xa0\x03\x00\x00\x00\x00\x00\x00$\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00a\x00\x00\x00\v\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\xc8\x03\x00\x00\x00\x00\x00\x00\xc8\x03\x00\x00\x00\x00\x00\x00\x90\x00\x00\x00\x00\x00\x00\x00\a\x00\x00\x00\x01\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00i\x00\x00\x00\x03\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00X\x04\x00\x00\x00\x00\x00\x00X\x04\x00\x00\x00\x00\x00\x00\x88\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00q\x00\x00\x00\xff\xff\xffo\x02\x00\x00\x00\x00\x00\x00\x00\xe0\x04\x00\x00\x00\x00\x00\x00\xe0\x04\x00\x00\x00\x00\x00\x00\f\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00~\x00\x00\x00\xfe\xff\xffo\x02\x00\x00\x00\x00\x00\x00\x00\xf0\x04\x00\x00\x00\x00\x00\x00\xf0\x04\x00\x00\x00\x00\x00\x000\x00\x00\x00\x00\x00\x00\x00\a\x00\x00\x00\x01\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8d\x00\x00\x00\x04\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00 \x05\x00\x00\x00\x00\x00\x00 \x05\x00\x00\x00\x00\x00\x00\xc0\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x97\x00\x00\x00\x01\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x17\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf7\x00\x00\x00\x01\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00 \x10\x00\x00\x00\x00\x00\x00 \x10\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x9d\x00\x00\x00\x01\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x000\x10\x00\x00\x00\x00\x00\x000\x10\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\xa6\x00\x00\x00\x01\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00@\x10\x00\x00\x00\x00\x00\x00@\x10\x00\x00\x00\x00\x00\x00\xf4\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xac\x00\x00\x00\x01\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x004\x11\x00\x00\x00\x00\x00\x004\x11\x00\x00\x00\x00\x00\x00\t\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xb2\x00\x00\x00\x01\x00\x00\x00\x12\x00\x00\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\xba\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x04 \x00\x00\x00\x00\x00\x00\x04 \x00\x00\x00\x00\x00\x00,\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xc8\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x000 \x00\x00\x00\x00\x00\x000 \x00\x00\x00\x00\x00\x00\xac\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd2\x00\x00\x00\x0e\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00>\x00\x00\x00\x00\x00\x00\x00.\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\xde\x00\x00\x00\x0f\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\b>\x00\x00\x00\x00\x00\x00\b.\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\xea\x00\x00\x00\x06\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x10>\x00\x00\x00\x00\x00\x00\x10.\x00\x00\x00\x00\x00\x00\xb0\x01\x00\x00\x00\x00\x00\x00\a\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\xa1\x00\x00\x00\x01\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\xc0?\x00\x00\x00\x00\x00\x00\xc0/\x00\x00\x00\x00\x00\x00(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\xf3\x00\x00\x00\x01\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\xe8?\x00\x00\x00\x00\x00\x00\xe8/\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\xfc\x00\x00\x00\x01\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00@\x00\x00\x00\x00\x00\x00\x000\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x01\x00\x00\b\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x10@\x00\x00\x00\x00\x00\x00\x100\x00\x00\x00\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\a\x01\x00\x00\x01\x00\x00\x000\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x100\x00\x00\x00\x00\x00\x00\x1f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0000\x00\x00\x00\x00\x00\x00H\x03\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00\x12\x00\x00\x00\b\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\t\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00x3\x00\x00\x00\x00\x00\x00\xc9\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x11\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00A5\x00\x00\x00\x00\x00\x00\x10\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00") diff --git a/src/debug/buildinfo/testdata/go117 b/src/debug/buildinfo/testdata/go117 deleted file mode 100755 index d7acbeef28cdb3..00000000000000 Binary files a/src/debug/buildinfo/testdata/go117 and /dev/null differ diff --git a/src/debug/buildinfo/testdata/go117/README.md b/src/debug/buildinfo/testdata/go117/README.md new file mode 100644 index 00000000000000..a36c1f4fcf42c0 --- /dev/null +++ b/src/debug/buildinfo/testdata/go117/README.md @@ -0,0 +1,15 @@ +go117.base64 is a base64-encoded Go 1.17 hello world binary used to test +debug/buildinfo of pre-1.18 buildinfo encoding. + +The binary is base64 encoded to hide it from security scanners that believe a +Go 1.17 is inherently insecure. + +Generate go117.base64 with: + +$ GOTOOLCHAIN=go1.17 GOOS=linux GOARCH=amd64 go build -trimpath +$ base64 go117 > go117.base64 +$ rm go117 + +TODO(prattmic): Ideally this would be built on the fly to better cover all +executable formats, but then we need a network connection to download an old Go +toolchain. diff --git a/src/debug/buildinfo/testdata/go117/go.mod b/src/debug/buildinfo/testdata/go117/go.mod new file mode 100644 index 00000000000000..4497b1c655d036 --- /dev/null +++ b/src/debug/buildinfo/testdata/go117/go.mod @@ -0,0 +1,3 @@ +module example.com/go117 + +go 1.17 diff --git a/src/debug/buildinfo/testdata/go117/go117.base64 b/src/debug/buildinfo/testdata/go117/go117.base64 new file mode 100644 index 00000000000000..f714ba6dd089d4 --- /dev/null +++ b/src/debug/buildinfo/testdata/go117/go117.base64 @@ -0,0 +1,20314 @@ +f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAAADtFAAAAAABAAAAAAAAAAMgBAAAAAAAAAAAAAEAAOAAH +AEAAFwADAAYAAAAEAAAAQAAAAAAAAABAAEAAAAAAAEAAQAAAAAAAiAEAAAAAAACIAQAAAAAAAAAQ +AAAAAAAABAAAAAQAAACcDwAAAAAAAJwPQAAAAAAAnA9AAAAAAABkAAAAAAAAAGQAAAAAAAAABAAA +AAAAAAABAAAABQAAAAAAAAAAAAAAAABAAAAAAAAAAEAAAAAAAJBTBQAAAAAAkFMFAAAAAAAAEAAA +AAAAAAEAAAAEAAAAAGAFAAAAAAAAYEUAAAAAAABgRQAAAAAACBYGAAAAAAAIFgYAAAAAAAAQAAAA +AAAAAQAAAAYAAAAAgAsAAAAAAACASwAAAAAAAIBLAAAAAACgMwAAAAAAAAByAwAAAAAAABAAAAAA +AABR5XRkBgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAA +AIAVBGUAKgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAEAAAABAAAABgAAAAAAAAAAEEAAAAAAAAAQAAAAAAAAkEMFAAAAAAAAAAAAAAAAACAA +AAAAAAAAAAAAAAAAAABqAAAAAQAAAAIAAAAAAAAAAGBFAAAAAAAAYAUAAAAAALgkAgAAAAAAAAAA +AAAAAAAgAAAAAAAAAAAAAAAAAAAAcAEAAAMAAAAAAAAAAAAAAAAAAAAAAAAAwIQHAAAAAAB6AQAA +AAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAHIAAAABAAAAAgAAAAAAAABAhkcAAAAAAECGBwAA +AAAAwAIAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAB8AAAAAQAAAAIAAAAAAAAAAIlHAAAA +AAAAiQcAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAhgAAAAEAAAACAAAAAAAA +AAiJRwAAAAAACIkHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAJAAAAABAAAA +AgAAAAAAAAAgiUcAAAAAACCJBwAAAAAA6OwDAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAABC +AAAAAQAAAAMAAAAAAAAAAIBLAAAAAAAAgAsAAAAAACAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAA +AAAAAAAABwAAAAEAAAADAAAAAAAAACCASwAAAAAAIIALAAAAAACAEQAAAAAAAAAAAAAAAAAAIAAA +AAAAAAAAAAAAAAAAABIAAAABAAAAAwAAAAAAAACgkUsAAAAAAKCRCwAAAAAA8CEAAAAAAAAAAAAA +AAAAACAAAAAAAAAAAAAAAAAAAAAYAAAACAAAAAMAAAAAAAAAoLNLAAAAAACgswsAAAAAACjrAgAA +AAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAHQAAAAgAAAADAAAAAAAAAOCeTgAAAAAA4J4OAAAA +AAAgUwAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAALkAAAABAAAAAAAAAAAAAAAAAE8AAAAA +AADACwAAAAAAGQEAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAfAQAAAQAAAAAAAAAAAAAA +GQFPAAAAAAAZwQsAAAAAAAczAQAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAA1QAAAAEAAAAA +AAAAAAAAACA0UAAAAAAAIPQMAAAAAADDOwAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAACwB +AAABAAAAAAAAAAAAAADjb1AAAAAAAOMvDQAAAAAAGAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAA +AAAAAADvAAAAAQAAAAAAAAAAAAAA+29QAAAAAAD7Lw0AAAAAAPwUAgAAAAAAAAAAAAAAAAABAAAA +AAAAAAAAAAAAAAAABwEAAAEAAAAAAAAAAAAAAPeEUgAAAAAA90QPAAAAAAB3FQEAAAAAAAAAAAAA +AAAAAQAAAAAAAAAAAAAAAAAAAGEBAAABAAAAAAAAAAAAAABumlMAAAAAAG5aEAAAAAAAB2gAAAAA +AAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAABQAAAABwAAAAIAAAAAAAAAnA9AAAAAAACcDwAAAAAA +AGQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAmwAAAAIAAAAAAAAAAAAAAAAAAAAAAAAA +eMIQAAAAAABQeQAAAAAAABYAAABZAAAACAAAAAAAAAAYAAAAAAAAAKMAAAADAAAAAAAAAAAAAAAA +AAAAAAAAAMg7EQAAAAAAJm8AAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAABAAAAFMAAAAEAAAAR28AADFDYzRpMEdraHBBdl9kSEpwWkVQL2dDZEVtUDh5OUxLYkkz +UTRjNzlxL1c0S0g1LTV2WlRyOVlINy13M3MtL1hOcTF0WlBNcGxpdUFwcVRDMnlGAEk7ZhB2OEiD +7BhIiWwkEEiNbCQQSIlEJCBIiVwkKGaQ6HsGAABIi0QkIEiLXCQo6CwAAABIi2wkEEiDxBjDSIlE +JAhIiVwkEOizEQUASItEJAhIi1wkEOunzMzMzMzMzEyNZCTYTTtmEA+G+AUAAEiB7KgAAABIiawk +oAAAAEiNrCSgAAAASImEJLAAAADrBkiJ8EyJw0iF2w+EHwIAADHJ6YsDAABIx8H/////SIXJfQ5F +McAx9kiJ2esxDx9AAEg52Q+HlQUAAEiNcQFIOfMPgn0FAABIKctIjXv/SYn4SPffSMH/P0gh/kgB +xkiD+QR8nYE4Y3B1LnWVMdIPHwDpSQMAAEjHwv////9IiXQkcEyJRCRQSIXSD4xLAQAAkEg5yg+H +HAUAAEiD+gQPggIFAABIjXr8SYn5SPffSMH/P0iD5wRIjRw4TI1SAUw50Q+C1wQAAEyJTCRISIlc +JGBIKdFMjVn/TIlcJBhNidxJ99tJwfs/TSHaTo0cEEyJXCRYSIP5A3UPRg+3FBBmQYH6b250Ietm +SIP5BHVgRg+3LBBmQYH9b2Z1U0YPtlQQAkGA+mZ1R0iD+gd1KUQPtxQ4ZkGB+mFsdRwPtnw4AkCA +/2x1EUiLPd2iCwAxwA8fAOnAAwAASIlMJEBIixXHogsASIlUJDgxwOlyAgAADx8A6Du+AgBIjQVS +FAYAuxAAAADoSscCAEiLRCRYSItcJBjoO8cCAEiNBbU2BgC7IAAAAOgqxwIASItEJGBIi1wkSOgb +xwIASI0FvwIGALsCAAAA6ArHAgDoZb4CAEiLdCRwTItEJFDpJv7//0iJTCQgSIlEJGjox70CAEiN +BWE4BgC7IQAAAOjWxgIASItEJGhIi1wkIOjHxgIASI0FawIGALsCAAAA6LbGAgDoEb4CAEiLdCRw +TItEJFDp0v3//0iLBfOhCwBIiw3koQsASIXAfglIiUQkUDHS6xRIi6wkoAAAAEiBxKgAAADDSIPB +IA8QAQ8RhCSAAAAADxBBEA8RhCSQAAAAgLwkmAAAAAAPhAkBAABIiVQkSEiJTCR4D7acJJkAAACE +23R5SIu0JJAAAACAPgB0BoTbdXLrZkiLhCSAAAAASIlEJHBIi4wkiAAAAEiJTCRA6OW8AgBIjQWh +JQYAuxkAAADo9MUCAEiLRCRwSItcJEDo5cUCAEiNBTUhBgC7FwAAAOjUxQIA6C+9AgBIi0QkUEiL +TCR4SItUJEjreoC8JJoAAAAAdQxIi7QkkAAAAIge62RIi4QkgAAAAEiJRCRwSIuMJIgAAABIiUwk +QOhpvAIASI0FGScGALsaAAAA6HjFAgBIi0QkcEiLXCRA6GnFAgBIjQVUIwYAuxgAAADoWMUCAOiz +vAIASItEJFBIi0wkeEiLVCRISP/CkEg50A+Pwf7//+ms/v//SP/BSDnZD41s/P//D7Y0CGaQQID+ +LHXo6WL8//9I/8JIOcoPja78//8PtjwQDx9EAABAgP89deXpofz//0mNQgFIOdAPjcgAAABIiz0x +oAsATIsVMqALAEw50A+DBQEAAEmJwkjB4AVMi1wHCEiLPAeQTTnLdcZIiUQkMEyJVCQoSIn4TInZ +6CYMAACEwHUlSItMJEBIi1QkOEiLXCRgSIt0JHBMi0QkUEyLTCRITItUJCjriEiLDc6fCwBIizW/ +nwsASItEJChIOcgPg5AAAABIi3wkMMZEPhgBSIsNqJ8LAEiLNZmfCwBIOchzbkiNPD5IjX8ZTItM +JEBJg/kDD5QHSIt0JHBMi0QkUOlR+///kOj7ugIASI0FKC8GALseAAAA6ArEAgBIi0QkYEiLXCRI +6PvDAgBIjQWf/wUAuwIAAADo6sMCAOhFuwIASIt0JHBMi0QkUOkG+///6NETBQDozBMFAEyJ0ejE +EwUARYhUBBlJjUEBSDn4D43i+v//TIsNA58LAEyLFfSeCwAPH0AATDnIc2BJicFIweAFQcZEAhgB +SIP5A3UKQboBAAAAZpDrGUyLFc+eCwBMix3AngsATTnRcyRFD7ZUAxpMix22ngsATIslp54LAE05 +2XKOTInITInZ6EcTBQBMichMidGQ6DsTBQBMicnoMxMFAEyJ0OjrEwUAuAQAAABIidEPHwDo2xMF +AEiJyEiJ0UiJwuhNEwUASInwSInZ6MITBQBIidroOhMFAJBIiUQkCEiJXCQQ6IoLBQBIi0QkCEiL +XCQQ6dv5///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4YwBAAASIPsMEiJbCQoSI1s +JChIjQVhjgUAkOg7lgAASInHSI01cWUHAGYPH4QAAAAAAA8fhAAAAAAASIlsJPBIjWwk8OgNFwUA +SIttAIM9BooOAABmDx9EAAAPhcsAAABIjQ3Ujg4ASIlIEEiNDciODgBIiUgwSI0Nv44OAEiJSFBI +jQ21jg4ASIlIcEiNDauODgBIiYiQAAAASI0Nno4OAEiJiLAAAABIjQ2Rjg4ASImI0AAAAEiNDYSO +DgBIiYjwAAAASI0NeI4OAEiJiBABAABIjQ1rjg4ASImIMAEAAEiNDV+ODgBIiYhQAQAASI0NU44O +AEiJiHABAABIjQ1Gjg4ASImIkAEAAEiNDTaODgBIiYiwAQAASI0NJo4OAEiJiNABAADpGgEAAEiN +eBBIjQ0Fjg4ADx9AAOjbDQUASI14MEiNDfCNDgDoyw0FAEiNeFBIjQ3ijQ4A6LsNBQBIjXhwSI0N +040OAOirDQUASI24kAAAAEiNDcGNDgDomA0FAEiNuLAAAABIjQ2vjQ4A6IUNBQBIjbjQAAAASI0N +nY0OAOhyDQUASI248AAAAEiNDYuNDgAPH0AA6FsNBQBIjbgQAQAASI0Ndo0OAOhIDQUASI24MAEA +AEiNDWSNDgDoNQ0FAEiNuFABAABIjQ1TjQ4A6CINBQBIjbhwAQAASI0NQo0OAOgPDQUASI24kAEA +AEiNDTCNDgCQ6PsMBQBIjbiwAQAASI0NGo0OAOjoDAUASI240AEAAEiNDQWNDgDo1QwFAEjHBcKb +CwAPAAAASMcFv5sLAA8AAACDPfiHDgAAdQlIiQWfmwsA6wxIjT2WmwsA6KELBQBIxwQkAAAAAOjU +AQAARQ9X/2RMizQl+P///4tEJAgPHwCD+AEPgocBAACJRCQguAAAAIBIiQQk6KUBAABFD1f/ZEyL +NCX4////i0QkCIkFioUOAEjHBCQBAAAA6IEBAABFD1f/ZEyLNCX4////i0QkEItMJBQPuuEaD5IF +TIwOAA+64AAPkgVCjA4AD7rgAQ+SBTSMDgAPuuAJD5IFLYwOAA+64BMPkgUjjA4AD7rgFA+SBRmM +DgAPuuAXD5IFCYwOAA+64BkPkgX0iw4AD7rgGw+SwYgN74sOAA+64AwPksIhyogV34sOAA+64Bty +BDHJ6zCJRCQk6AUBAABFD1f/ZEyLNCX4////iwQkD7rgAXMJD7rgAg+SwOsCMcCJwYtEJCQPuuAc +D5LAIciIBZGLDgCLRCQgg/gHcwpIi2wkKEiDxDDDiEwkH0jHBCQHAAAA6IsAAABFD1f/ZEyLNCX4 +////i0QkDA+64AMPkgVTiw4AD7rgBQ+SwQ+2VCQfIcqIFT6LDgAPuuAID5IFNYsOAA+64AkPkgUr +iw4AD7rgEw+SBRuLDgBIi2wkKEiDxDDDSItsJChIg8Qww+ghBwUAkOm7+///zMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMi0QkCItMJAwPoolEJBCJXCQUiUwkGIlUJBzDzMzMzMy5AAAAAA8B0IlE +JAiJVCQMw8zMzMzMzMzMzMzMzMzMzEk7ZhB2Y0iD7CBIiWwkGEiNbCQYSItICEiLE0iLMGaQSDlL +CHUZSIlEJChIiVwkMEiJ8EiJ0+hlBQAAhMB1BDHA6xxIi1QkKEiNQhBIi1QkMEiNWhC5CwAAAOhB +BQAASItsJBhIg8Qgw0iJRCQISIlcJBDoSAYFAEiLRCQISItcJBDpef///8zMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMxJO2YQdm9Ig+wgSIlsJBhIjWwkGEiJRCQoSIlcJDAxyesTSItUJBBIjUoBSItE +JChIi1wkMEiD+Q99K0iJTCQQSMHhBUiNNAFIAdlIifBIicvoCv///4TAdcgxwEiLbCQYSIPEIMO4 +AQAAAEiLbCQYSIPEIMNIiUQkCEiJXCQQkOibBQUASItEJAhIi1wkEOls////zMzMzMzMzMzMzMzM +SInBSNHoSLpVVVVVVVVVVUgh0EghykiNDAJIicpIwekCSLszMzMzMzMzM0gh2Ugh00gB2UiJykjB +6QRIAdFIug8PDw8PDw8PSCHKSInRSMHqCEgB0UiJykjB6RBIAdFIicpIwekgSI0EEUiD4H/DzMzM +zMzMzMzMzMzMzMzMzMyAPdyIDgAAdA1IxwVEgg4APwAAAOsLSMcFN4IOAB8AAADDzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMSDn+D4QiAQAASDnTSYnQTA9Mw0mD+AgPgrYAAABJg/g/dhKA +PXmIDgABD4ShAQAA6QsBAABJg/gQdlvzD28G8w9vD2YPdMhmD9fBSDX//wAAdSpIg8YQSIPHEEmD +6BDr1EiDxjBIg8cw6xJIg8YgSIPHIOsISIPGEEiDxxBID7zYSDHAigweOgwfD5fASI0ERf/////D +SYP4CHYLSIsGSIsPSDnIdQ9Ki0QG+EqLTAf4SDnIdHVID8hID8lIMcFID73JSNPoSIPgAUiNBEX/ +////w0qNDMUAAAAASPfZdEtAgP74dwVIizbrCEqLdAb4SNPuSNPmQID/+HcFSIs/6whKi3wH+EjT +70jT50gPzkgPz0gx93QUSA+9z0jT7kiD5gFIjQR1/////8NIMcBIMclIOdMPn8APlMFIjURB/8Pz +D28G8w9vD2YPdMhmD9fBSDX//wAAD4Uh////8w9vRhDzD29PEGYPdMhmD9fBSDX//wAAD4X7/v// +8w9vRiDzD29PIGYPdMhmD9fBSDX//wAAD4XT/v//8w9vRjDzD29PMGYPdMhmD9fBSDX//wAAD4Wr +/v//SIPGQEiDx0BJg+hASYP4QA+Gaf7//+lv////xf5vFsX+bx/F/m9mIMX+b28gxeV0wsX918A1 +/////3UjxdV09MX918Y1/////3UcSIPGQEiDx0BJg+hASYP4QHIS67zF+HfpYP7//8X4d+lG/v// +xfh36Qj+///MzMzMzMzMzMzMzMzMzMzMzMzMSInGSIn6SInP6bL9///MzMzMzMzMzMzMzMzMzMzM +zMxIg/sID4LzAAAASIP7QA+CtwAAAIA9KIYOAAF0aEiD+0APgqQAAADzD28G8w9vD/MPb1YQ8w9v +XxDzD29mIPMPb28g8w9vdjDzD29/MGYPdMFmD3TTZg905WYPdPdmD9vCZg/b5mYP28RmD9fQSIPG +QEiDx0BIg+tAgfr//wAAdJxIMcDDSIP7QHI9xf5vBsX+bw/F/m9WIMX+b18gxf104cXldOrF1dv0 +xf3X1kiDxkBIg8dASIPrQIH6/////3TExfh3SDHAw8X4d0iD+wh2G0iLDkiLF0iDxghIg8cISIPr +CEg50XTjSDHAw0iLTB74SItUH/hIOdEPlMDDSIP7AHQ3SI0M3QAAAABI99lAgP74dwVIizbrCEiL +dB74SNPuQID/+HcFSIs/6whIi3wf+EjT70gp90jT5w+UwMPMzEg52HUISMfAAQAAAMNIicZIid9I +icvppf7//8zMzMzMSDnYdQhIx8ABAAAAw0iJxkiJ30iLWgjphP7//8zMzMxmSA9uwGYPYMBmD2DA +Zg9wwABIg/sQfFRIifdIg/sgD4eNAAAASI1EHvDrFfMPbw9mD3TIZg/X0Q+80nUlSIPHEEg5x3Lm +SInH8w9vCGYPdMhmD9fRD7zSdQhJxwD/////w0gp90gB10mJOMNIhdt06UiNRhBmqfAPdBnzD28O +Zg90yGYP19EPvNJ0zjnac8pJiRDD8w9vTB7wZg90yGYP19GJ2dPiweoQD7zSdKxJiRDDgD0KhA4A +AQ+FZv///2ZID27ATI1cHuDE4n14yMX+bxfF7XTZxOJ9F9t1JkiDxyBMOd986EyJ38X+bxfF7XTZ +xOJ9F9t1C8X4d0nHAP/////Dxf3X0w+80kgp90gB+kmJEMX4d8PMzMzMzMzMzMxIi3QkCEiLXCQQ +ikQkGEyNRCQg6cj+///MzMzMzMzMzEk7ZhB2IkiD7CBIiWwkGEiNbCQYuQoBAADoYv7//0iLbCQY +SIPEIMNIiUQkCEiJXCQQ6Gn/BABIi0QkCEiLXCQQ673MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzEk7ZhB2IkiD7CBIiWwkGEiNbCQYuRAAAADo4gAFAEiLbCQYSIPEIMNIiUQkCEiJXCQQ6An/ +BABIi0QkCEiLXCQQ673MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzLgBAAAAw8zMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMD7YIOAsPlMDDzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMwPtwhmOQsP +lMDDzMzMzMzMzMzMzMzMzMzMzMzMzMzMzIsIOQsPlMDDzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +SIsISDkLD5TAw8zMzMzMzMzMzMzMzMzMzMzMzMzMzMxIiwhIOQt1DUiLSAhIOUsID5TB6wIxyYnI +w8zMzMzMzPMPEADzDxALDy7BD5TBD5vAIcGJyMPMzMzMzMzMzMzM8g8QAPIPEAtmDy7BD5TBD5vA +IcGJyMPMzMzMzMzMzMxIiUQkCPMPEADzDxALDy7BD5TBD5vAIcFIi0QkCPMPEEAE8w8QSwQPLsEP +lMIPm8AhwiHRicjDzMzMzMzMzMzMSIlEJAjyDxAA8g8QC2YPLsEPlMEPm8AhwUiLRCQI8g8QQAjy +DxBLCGYPLsEPlMIPm8AhwiHRicjDzMzMzMzMzEk7ZhB2NkiD7CBIiWwkGEiNbCQYSItICEiLAEiL +E2aQSDlLCHQEMcDrCEiJ0+gO/P//SItsJBhIg8Qgw0iJRCQISIlcJBDoFf0EAEiLRCQISItcJBDr +qczMzMzMzMzMzEk7ZhB2OEiD7CBIiWwkGEiNbCQYSIsQSItwCEiLSwiQSDkTdAQxwOsLSInQSInz +6GwBAABIi2wkGEiDxCDDSIlEJAhIiVwkEOiz/AQASItEJAhIi1wkEOunzMzMzMzMzEk7ZhB2OEiD +7CBIiWwkGEiNbCQYSIsQSItwCEiLSwiQSDkTdAQxwOsLSInQSInz6CwAAABIi2wkGEiDxCDDSIlE +JAhIiVwkEOhT/AQASItEJAhIi1wkEOunzMzMzMzMzEk7ZhAPhpAAAABIg+wwSIlsJChIjWwkKEiF +wHQ5SItQGEiF0nQ/D7ZwF0D2xiB0EUg5yw+UwEiLbCQoSIPEMJDDSIsySInYSInL/9ZIi2wkKEiD +xDDDuAEAAABIi2wkKEiDxDDD6NatBAC5HAAAAEiJx0iJ3jHASI0dvhkGAGaQ6JsTBADolmIAAEiJ +w0iNBQyTBQDo54kCAJBIiUQkCEiJXCQQSIlMJBjokvsEAEiLRCQISItcJBBIi0wkGA8fAOk7//// +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GkQAAAEiD7DBIiWwkKEiNbCQoSIXAdDxI +i0AISItQGEiF0nQ+D7ZwF0D2xiB0EEg5yw+UwEiLbCQoSIPEMMNIizJIidhIicv/1kiLbCQoSIPE +MMO4AQAAAEiLbCQoSIPEMMPo86wEALkcAAAASInHSIneMcBIjR3bGAYA6LoSBADotWEAAEiJw0iN +BSuSBQDoBokCAJBIiUQkCEiJXCQQSIlMJBjosfoEAEiLRCQISItcJBBIi0wkGGaQ6Tv////MzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4aOAAAASIPsIEiJbCQYSI1sJBiAPUF+DgAAkHQ4 +gD1Efg4AAHQvgD08fg4AAHQmkMYFNncOAAFIjQV9ew4Au4AAAABIidnocF4CAEiLbCQYSIPEIMNI +jQVfeQ4AuyAAAABIidnoUl4CAEiDDUp5DgABSIMNSnkOAAFIgw1KeQ4AAUiDDUp5DgABSItsJBhI +g8Qgw+jj+QQADx8A6Vv////MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxIg+wYSIlsJBBIjWwk +EEmLTjBIi4nQAAAAhAFIi5HAFgAASIswSInfSIkySIlaCEiLkcAWAABIg8IQSImRwBYAAEg5kcgW +AAB1CEiJ++gORwIASItsJBBIg8QYw8zMzMxIg+wYSIlsJBBIjWwkEIA9S3gOAAB0GUiJXCQoSIlE +JCDoev///0iLRCQgSItcJChIhxhIi2wkEEiDxBjDzMzMSIPscEiJbCRoSI1sJGhIgz26igsAAA+E +kwAAAEjHRCQwAAAAAEiNVCQ4RA8ROkyNTCRIRQ8ROUyNTCRYRQ8ROUyNDbkAAABMiUwkOEiJRCRA +SIlcJEiJTCRQiXwkVIl0JFhEiUQkXEiNRCQwSIlEJGBIiRQk6Gb3BABFD1f/ZEyLNCX4////SItE +JDBIPQAQAABzD0iJwzHASItsJGhIg8RwwzHbSItsJGhIg8Rww0iJBCRIiVwkCIlMJBCJfCQUiXQk +GESJRCQc6NIWBQBFD1f/ZEyLNCX4////SItEJCBIi1wkKEiLbCRoSIPEcMPMzMzMzMzMzMzMzMzM +zMzMzEk7ZhB2bUiD7DhIiWwkMEiNbCQwSItCEItKGItaHItyIIt6JEyLQihMiUQkKEiLUghIiRQk +SIlEJAiJTCQQiVwkFIl0JBiJfCQc6LEWBQBFD1f/ZEyLNCX4////SItEJCBIi0wkKEiJAUiLbCQw +SIPEOMPoCPcEAOuGzMzMzMzMSTtmEA+GhwAAAEiD7DBIiWwkKEiNbCQoSIM9OIkLAAB0SkQPEXwk +EEjHRCQgAAAAAEiNDYgAAABIiUwkEEiJRCQYSIlcJCBIjUQkEEiJBCToC/YEAEUPV/9kTIs0Jfj/ +//9Ii2wkKEiDxDDDSIkEJEiJXCQI6GYWBQBFD1f/ZEyLNCX4////SItsJChIg8Qww0iJRCQISIlc +JBAPH0QAAOj79gQASItEJAhIi1wkEOlM////zMzMzMzMzMzMzMzMSTtmEHY7SIPsGEiJbCQQSI1s +JBBIi0IQSItKCEiJDCRIiUQkCOg2FgUARQ9X/2RMizQl+P///0iLbCQQSIPEGMPo+vUEAOu4zMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMSIPsYEiJbCRYSI1sJFiJRCRoSIM9LogLAABmDx9EAAAPhCoB +AACAPTxzDgAAD4UdAQAAx0QkJAAAAACAPS1zDgAAdAxMiXQkKEiLRCQo6wIxwEiJTCR4SIlcJHBI +jVQkaEiFwA+EngAAAEg5EHcGSDlQCHcxi0QkaEiJBCRIiVwkCEiJTCQQ6JQSBQBFD1f/ZEyLNCX4 +////i0QkGIlEJCTpjgAAAEjHRCQwAAAAAEiNRCQ4RA8ROEiNRCRIRA8ROEiNBZgAAABIiUQkMEiN +RCRoSIlEJDhIiVwkQEiJTCRISI1EJCRIiUQkUEiNRCQwSIkEJOhH9AQARQ9X/2RMizQl+P///+ss +i0QkaEiJBCRIiVwkCEiJTCQQ6AESBQBFD1f/ZEyLNCX4////i0QkGIlEJCSDfCQkFnUai0QkaEiL +XCRwSItMJHjoEl0CAOsF6AtdAgBIi2wkWEiDxGDDzEk7ZhB2W0iD7DBIiWwkKEiNbCQoSItCEEiL +ShhIi1ogSIlcJCBIi1IIixJIiRQkSIlEJAhIiUwkEOiCEQUARQ9X/2RMizQl+P///4tEJBhIi0wk +IIkBSItsJChIg8QwkMPoGvQEAOuYzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSIPsMEiJbCQoSI1s +JCiAPXdxDgAAD4S1AAAADx9EAABIhcAPhJYAAABIiVwkQEiJRCQ4SYtGMEiJRCQgSP+AMAEAAP+A +OAEAAEiLiEABAABIxwEAAAAA6KTgBABIi0QkIMaAGAEAAAFIi0wkOEiJDCRIi1QkQEiJVCQIDx9E +AADom/QEAEUPV/9kTIs0Jfj///+LRCQQiUQkHEiLTCQgxoEYAQAAAP+JOAEAAOiP4AQAi0QkHEiL +bCQoSIPEMMNIjQXK8AUAuwsAAADosIkCAEiNBQb+BQC7EwAAAA8fQADom4kCAJDMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzEiD7BBIiWwkCEiNbCQISIXAdEJIiUQkGOijswEADx8AhMB1IkiLDb2F +CwBIhcl0CUiLEUiLSQjrBDHJMdJIi1wkGDHA6yG4AQAAAEiLbCQISIPEEMMxwEiLbCQISIPEEMNI +/8BMicNIOcF+SEiLNMJIi77YAAAASYnYZg8fRAAASDme0AAAAHcFSDn7chdIi77oAAAASDme4AAA +AHfDZpBIOftzvLgBAAAASItsJAhIg8QQwzHASItsJAhIg8QQkMPMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMSIPsKEiJbCQgSI1sJCBIiVwkOEiJRCQwSInYDx9EAADo+/7//4TAdFpIi0Qk +MOjt/v//hMB1QkmLRjBMifFmkEg5CHQqSDlIUHQkg7jwAAAAAHURSItEJDBIicFIixXNcA4A6ytI +i2wkIEiDxCjDSItsJCBIg8Qow0iLbCQgSIPEKMNIi2wkIEiDxCjDSIsSSIXSdBtIOdBy80iNmgAA +BABIOdhz50iLbCQgSIPEKMNEDxF8JAhIx0QkGAAAAABIjQU6AAAASIlEJAhIi0QkOEiJRCQQSIlM +JBhIjUQkCEiJBCTomPAEAEUPV/9kTIs0Jfj///9Ii2wkIEiDxCjDzEk7ZhAPhoEAAABIg+woSIls +JCBIjWwkIEiLQhBIi0oISIlMJBBIiUQkGOjRnwIASI0Frf8FALsUAAAADx9EAADo26gCAEiLRCQQ +6DGnAgBIjQXt+QUAuxIAAAAPH0QAAOi7qAIASItEJBjoEacCAOgMogIA6AegAgBIjQXuHwYAuyQA +AADoFocCAJDokPAEAOlr////zMzMzMzMzMzMzMxIg+xISIlsJEBIjWwkQEiDeAgAdGxIiXwkKEiJ +dCQgSIlEJFBIiVwkOEiJTCQwSInI6Er9//+EwHQ8SItEJDiQ6Dv9//+EwHQKSItsJEBIg8RIw0iL +RCRQSItcJDBIi0wkKEiLfCQg6PQAAABIi2wkQEiDxEjDSItsJEBIg8RIkMNIi2wkQEiDxEjDzMzM +zMzMzMzMzMzMzMzMzMzMzMzMSIPsQEiJbCQ4SI1sJDhIg3gIAHRMSIl8JGBIiVwkMEiJTCRYSIlE +JEhIicjor/z//4TAdCJIi0QkMOih/P//kITAdQkxwEiLTCRY61RIi2wkOEiDxEDDSItsJDhIg8RA +w0iLbCQ4SIPEQMNIiUQkIEiJTCQoSItEJEhIizhIicsxyeg5AAAASItUJCBIjUIBSItUJEhIizJM +i0QkKEmNDDBIi1QkYEg5wn/ASItsJDhIg8RAw8zMzMzMzMzMzMzMSIPEgEiJbCR4SI1sJHhIi1AI +SDnRc0pIKcpIOddID0f6D7ZQF/bCQHQeSIsV14ELAEiF0nQJSIsySItSCOsEMdIx9kUxwOspSItQ +IEiJ2EiJ0+gJAwAASItsJHhIg+yAw0iLbCR4SIPsgMNJ/8BMieNMOcIPjosAAABOiwzGTYuR0AAA +AE2LmdgAAABJidxMOdNyBUw523IaTYuR4AAAAE2LmegAAABMOdNyv0w523O66ylIidpMKdNJKdJJ +i5HoAQAASAHZS40EIkiJ0+iOAgAASItsJHhIg+yAw0iJ2kwp00kp0kmLkfgBAABIAdlLjQQiSInT +6GUCAABIi2wkeEiD7IDDSIsVDAMOAIQCSInekEm4AAAAAACAAABNjQwYScHpGkmB+QAAQAAPg+AB +AABKixTKSMHrDUiB4/8fAABMi4zaAAAgAIQCQYpRY4D6AnRrSIsVvgIOAIQCSYnxkEmNBDBIwega +SD0AAEAAD4OTAQAASIsUwkiF0nQiSYnwSMHuBUiB5v//HwBIAdZJwegDSYPgA0iBwv//HwDrCTHA +RTHAMdIx9kiJjCSYAAAATIlMJEhIiXwkMDHb62RIx0QkUAAAAABIjVQkWEQPETpIjVQkaEQPETpI +jRVAAQAASIlUJFBIiUQkWEiJdCRgSIlMJGhIiXwkcEiNRCRQSIkEJOh57AQARQ9X/2RMizQl+P// +/0iLbCR4SIPsgMNIg8MITI0UD0w50w+DvQAAAEiJXCQoRA+2Fkg52XdVRQ+jwnNPSIl0JEBIiVQk +OIlEJCREiUQkIEmLBBnowfn//5CEwA+FjgAAAItEJCRIi4wkmAAAAEiLVCQ4SItcJChIi3QkQEiL +fCQwRItEJCBMi0wkSEGD+ANzCEH/wOuBDx8ASDnydAtI/8ZFMcDpbv///0SJw4nBSInXSInwDx9E +AADom6kAAEyLTCRIQYnYSInGSIn6SItcJChIi3wkMInISIuMJJgAAADpMv///0iLbCR4SIPsgMNI +jQVxGwYAuyQAAADomYICALkAAEAA6C/0BABMici5AABAAOgi9AQAkMxJO2YQdi1Ig+woSIlsJCBI +jWwkIEiLWhBIi0oYSIt6IEiLQgjo9wAAAEiLbCQgSIPEKMPoyOsEAOvGzMzMzMzMSIPsSEiJbCRA +SI1sJEBIicpIwekGSInOSMHhBpCQSCnKSAHXSIl8JChIAfNIAcFIiUwkMDHAMfbrBkiDwAhmkEg5 ++HNlSKk/AAAAdQgPtjNI/8PrAtHuSIXSdgZIg8L469gPuuYAc9JIiUQkGIl0JBRIiVwkOEiJVCQg +SIsUAUiJ0Og5+P//hMB1KUiLRCQYSItMJDBIi1QkIEiLXCQ4i3QkFEiLfCQo65BIi2wkQEiDxEjD +SI0FUxoGALskAAAA6HuBAgCQzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YYD4blAQAASIPs +eEiJbCRwSI1sJHBIi1AIDx9AAEiF0nR0SDnRc2VIKcpIOddID0f6D7ZQF/bCQHQ2g+IfgPoRdQ9I +iYQkgAAAADHS6R0BAACA+hkPhfIAAABIi1BASItwOEiF0n45SIlUJEgxwOtHSItQIEiJ2EiJ0w8f +AOib/v//SItsJHBIg8R4w0iLbCRwSIPEeMNIi2wkcEiDxHjDSItsJHBIg8R4w0iDxhhMidhMiclM +idNMi0YIZpBJOQh2SUiJRCRASIl0JGhIiUwkKEiJfCQ4SIlcJFhMiUQkYEyJwOgV////SItEJEBI +i0wkKEiLVCRISItcJFhIi3QkaEiLfCQ4TItEJGBNiwBMOcFJiclJD0fIkE2JwkkpyEkpyUkB2kw5 +x3YYTI1YAUwpx0w52g+PcP///w8fAOle////SItsJHBIg8R4w0iNBfboBQC7DAAAAA8fRAAA6Pt/ +AgBI/8JIKfdMicFMictIOVBAD4Yo////SItwMJBIOQ52PEiJVCQwSIl8JDhIiVwkUEiJTCQgSInw +Dx9AAOhb/v//SIuEJIAAAABIi0wkIEiLVCQwSItcJFBIi3wkOEiLcDBIizZIOfFJichID0fOSYnx +SCnOSSnISQHZZpBIOfd3gEiLbCRwSIPEeMNIiUQkCEiJXCQQSIlMJBhIiXwkIOi4EAUASItEJAhI +i1wkEEiLTCQYSIt8JCAPH0AA6dv9///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4ae +AQAASIPsOEiJbCQwSI1sJDBIi1AwSIsykEiB/gAAAQAPg2kBAACAehUID4dOAQAASIlUJCBIidhI +9+YPgCkBAABIuqD//////wAASDnQD4cWAQAADx9EAABIhdsPjAgBAABIiVwkSEiFwA+ElQAAAEiL +VCQgSIN6CAB1NkiDwGAx27kBAAAA6M1pAACEAEiNUGCDPaBnDgAAdQlIiVAQ6Y8AAABIjXgQ6Gzs +BADpgQAAAEiJRCQYSI0Fu7wFAOh2cwAASIlEJChIi1wkILkBAAAASItEJBhmkOh7aQAAgz1UZw4A +AHULSItMJChIiUEQ6w5Ii0wkKEiNeRDo+eoEAEiJyOsuuGAAAAAx27kBAAAA6ENpAACEAEiNeBCD +PRZnDgAAdQZIiXgQ6whIifnoxusEAEiLTCQgSIsRZolQGIM982YOAAB1BkiJSCDrCUiNeCDoousE +AEiLTCRISIlICEiLbCQwSIPEOMNIjQWogAUASI0dkS8HAJDo+3UCAEiNBVH7BQC7FwAAAOiKfQIA +SI0FLhkGALsmAAAA6Hl9AgCQSIlEJAhIiVwkEOiJ5wQASItEJAhIi1wkEOk6/v//zMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMxIg+woSIlsJCBIjWwkILkBAAAASIt8JCjoIwAAAEiLbCQgSIPEKMPM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+G9QUAAEiDxIBIiWwkeEiNbCR4Dx+EAAAAAABI +hcAPhKUAAACEyXUvg3gcAHUpSItQCEiF0nUKSIN4OAAPlMLrBkg5EA+UwoTSdAwxwEiLbCR4SIPs +gMOIjCSYAAAASImcJJAAAABIiYQkiAAAAEiDPUVkDgAAdwUx0pDrIeg56AQARQ9X/2RMizQl+P// +/0iLBCRIicJIi4QkiAAAAEiJVCQwSI1IWEiJTCRAkEiJyOgmUgAASIuEJIgAAACDeBwAD4UTBQAA +6zyEyXUMMcBIi2wkeEiD7IDDMcAx27kEAAAAvxAAAAC+AgAAAOhKpgIASI0FuOMFALsLAAAA6Pl7 +AgBIichIi1A4SIXSD4S7AAAASI14OEyLQgiQTYXAdE6DPfRkDgAAdRZJx0AQAAAAAEyJQDhIx0II +AAAAAOtQTY1IEEiJ+UyJz0iJ0zHS6KjpBABIic8PH0QAAOj76QQASI17COiS6QQASIna6yGDPaZk +DgAAdQdEDxF4OOsRRTHA6NXpBABIjXhA6MzpBACAejQAdCFMiwJIicExwEG5AQAAAPBFD7GIeAEA +AEEPlMBBg/AB6wxIicFBuQEAAABFMcBFhMAPhTf////rDkiJwTHSZg8fhAAAAAAASIXSD4VwAwAA +SIsRSDlRCA+HAwMAAA+2lCSYAAAAZpCE0nUYkJBIi0QkQOiwUgAAMcBIi2wkeEiD7IDDTIl0JEjo +GqcCAEjHQCgAAAAASItMJDBIhcl0CEjHQCj/////gz3ZYw4AAHUeSIuUJJAAAABIiVAYSMdAQAAA +AABMi0QkSEyJAOsuSI14GEiLlCSQAAAA6IjoBABIjXhARTHAkOjb6AQASInHTItMJEjo7ugEAE2J +yMZANABIjXhQgz18Yw4AAHUOTIuMJIgAAABMiUhQ6xFMi4wkiAAAAA8fQADou+gEAEGEAE2NkEgB +AABNjZiIAAAAgz1DYw4AAHUUSYmASAEAAEnHgIgAAAAAAAAA6x5IiftMidfo4uYEAEyJ30iJ1jHS +6PXnBABIifJIid+Qgz0HYw4AAHUKSMdACAAAAADrGkyNYAhIiftMiedIidYx0ujH5wQASInySInf +SIlEJDhIiXwkYEyJVCRYTIlcJFBNjWFQTYtpUA8fRAAATYXtdEiDPbRiDgAAdQ5MiWgQSYlFCEmJ +QVDrd0yNeBBIiftMif9Mic5Nieno7+cEAEmNeQjoRuYEAEyJ5w8fAOg75gQASInfSYnx60aDPWxi +DgAAdRJIx0AQAAAAAEmJQUhJiUFQ6ytMjWgQSIn7TInvSInWMdLoJOcEAEmNeUjo++UEAEyJ5+jz +5QQASInySInfSY2QuQAAAEG5AQAAAESGCkiNBfYpBgBIi1wkQLkPAAAAvxYAAAC+AgAAAGaQ6Buj +AgBIi0QkOEiLVCRISDmCSAEAAA+FkgEAAIM93WEOAAB1DUjHgkgBAAAAAAAA6wxIi3wkWDHJ6ILm +BADGgrgAAAAAD7ZINYM9sGEOAAB1DUjHgogAAAAAAAAA6wxIi3wkUDHS6HXmBACITCQvSItQKEiF +0n4fSItMJDBIKcpIidC7AgAAAOizEQIASItEJDgPtkwkL4M9YmEOAABmkHUKSMdAUAAAAADrDEiL +fCRgMdLoKOYEAOhjpwIAD7ZMJC+EyQ+EsgAAALgBAAAASItsJHhIg+yAw0iLWShIi0EgD7dRGEgP +r9pIA1kQSIuMJJAAAADohpcAAEiLlCSIAAAASItyKEj/xkiJcihIOXIIdQhIx0IoAAAAAEj/ApCQ +SItEJEDodk8AALgBAAAASItsJHhIg+yAw0QPEXwkaEyNBdoAAABMiUQkaEiJTCRwSInISInTSIuM +JJAAAABIjXwkaL4DAAAA6PMAAAC4AQAAAEiLbCR4SIPsgMNIi4wkiAAAAIN5HAB1EUiNBQj5BQC7 +GQAAAOhFdwIASI0FPnoFAEiNHTcpBwDokm8CAEiNBeL7BQC7GwAAAOghdwIAkJBIi0QkQOjVTgAA +SI0FDnoFAEiNHQcpBwDoYm8CAJBIiUQkCEiJXCQQiEwkGEiJfCQg6AnhBABIi0QkCEiLXCQQD7ZM +JBhIi3wkIOnQ+f//zMzMzMzMzMzMzMzMzMzMzEk7ZhB2KUiD7BBIiWwkCEiNbCQISItKCIQBSI1B +WJCQ6FtOAABIi2wkCEiDxBDD6AzgBADryszMzMzMzMzMzMxJO2YQD4bpAAAASIPsKEiJbCQgSI1s +JCBIiXQkUEiJXCQ4SIN7GAB0REiJfCRISItAIOgJAQAAgz1iXw4AAGaQdQ9Ii0QkOEjHQBgAAAAA +6xRIi0QkOEiNeBgxyQ8fQADo++MEAEiJw0iLfCRISIsDSIlEJBhIiw9Iifr/0UiLRCQYhACDPRVf +DgAAdQ5Ii0wkOEiJiIgAAADrEUiNuIgAAABIi0wkOOi04wQAxkE1AUiDeSgAdCTohOEEAEUPV/9k +TIs0Jfj///9IiwQkSItMJDhIiUEoSItEJBhIi0wkUEiNWQHoF6ECAEiLbCQgSIPEKMNIiUQkCEiJ +XCQQSIlMJBhIiXwkIEiJdCQo6I/fBABIi0QkCEiLXCQQSItMJBhIi3wkIEiLdCQo6dH+///MzMzM +zMzMzMzMzMzMzMzMzEk7ZhB2SkiD7DBIiWwkKEiNbCQoSIlMJEhIiUQkOEiLWxhIiVwkIEiLOOiR +pQAASItUJDhIiwpIi0QkIEiLXCRI6JrvBABIi2wkKEiDxDDDSIlEJAhIiVwkEEiJTCQYkOj73gQA +SItEJAhIi1wkEEiLTCQY64rMzMzMzMzMzMzMSTtmEHZQSIPsMEiJbCQoSI1sJChIiUwkSEiJRCQ4 +SItTGEiJVCQgSInLSIs4SInR6AulAABIi1QkOEiLCkiLRCRISItcJCDoFO8EAEiLbCQoSIPEMMNI +iUQkCEiJXCQQSIlMJBjodt4EAEiLRCQISItcJBBIi0wkGOuFzMzMzMxJO2YQD4bmAwAASIPsOEiJ +bCQwSI1sJDAPH4QAAAAAAEiFwA+EswMAAEiJRCRASI1IWEiJTCQokEiJyA8fRAAA6NtJAABIi0wk +QIN5HAAPhWkDAADHQRwBAAAASMdEJBAAAAAA6xfGQjUAkEiLVCQQSImWoAAAAJBIiXQkEA8fAOnF +AAAAQbgBAAAAMdJIhdIPhI8BAABIiVQkGEiLWhiQSIXbdEFIi0Eg6LKUAACDPatcDgAAdQ9Ii0wk +GEjHQRgAAAAA6xBIi0wkGEiNeRgx0uhq4QQASItMJEBIi1QkGEG4AQAAAEiDeigAdC3oDt8EAEUP +V/9kTIs0Jfj///9IiwQkSItMJBhIiUEoSInKQbgBAAAASItMJEBIizKEBoM9OlwOAAB1DEiJlogA +AADpMv///0iNvogAAAAPH0QAAOj74AQA6Rz///9Ii1E4SIXSD4Qu////SI15OEiLcgiQSIX2dESD +PfRbDgAAdRZIx0YQAAAAAEiJcThIx0IIAAAAAOtKTI1GEEiJ+EyJx0Uxyegq4QQASInH6OLgBABI +jXoI6BnhBADrJYM9sFsOAAB1B0QPEXk46xUx9g8fRAAA6LvgBABIjXlA6LLgBACAejQAdB1IizIx +wEG4AQAAAPBED7GGeAEAAEAPlMaD9gHrD0G4AQAAADH2Dx+AAAAAAECE9g+FQf///+l//v//xkI1 +AJBIi1QkEEiJk6AAAACQSIlcJBDpjgAAADHSSIXSD4Q4AQAAgz0kWw4AAHUKSMdCGAAAAADrC0iN +ehgx2+gN4AQASIN6KAB0M0iJVCQgkOib3QQARQ9X/2RMizQl+P///0iLBCRIi0wkIEiJQShIicpB +uAEAAABIi0wkQEiLGoQDgz3HWg4AAHUMSImTiAAAAOln////SI27iAAAAOiN3wQA6Vb///9Ii1FI +Dx9AAEiF0g+EYf///0iNeUhIi1oISIXbdESDPYNaDgAAdRZIx0MQAAAAAEiJWUhIx0IIAAAAAOtF +SI1zEEiJ+EiJ90Uxyei53wQASInH6FHfBABIjXoI6KjfBADrIIM9P1oOAAB1B0QPEXlI6xAx2+gv +3wQASI15UOgm3wQAgHo0AGaQdBZIixoxwPBED7GDeAEAAA+Uw4PzAesIMdtmDx9EAACE2w+FUP// +/+m//v//kJBIi0QkKOiHSAAA6xVIx4CgAAAAAAAAALsDAAAA6DCcAgBIi0QkEEiFwHQZkJAPH0AA +SIXAdNZIi4igAAAASIlMJBDryEiLbCQwSIPEOMOQkEiLRCQo6DdIAABIjQVwcwUASI0diSIHAOjE +aAIASI0FXXMFAEiNHWYiBwDosWgCAJBIiUQkCOhm2gQASItEJAiQ6fv7///MzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMxIg+wgSIlsJBhIjWwkGLkBAAAA6CgAAABIi2wkGEiDxCDDzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMTI1kJPhNO2YQD4aNBgAASIHsiAAAAEiJrCSAAAAASI2sJIAA +AABIhcAPhEABAACEyXQEMdLrHEiDeAgAdQxIi1BISIXSD5TC6wlIizBIhfYPlMKE0nRii1AchdJ0 +R0iDeAgAdQxIi1BISIXSD5TC6wlIizBIhfYPlMKE0nQ7ZpBIhdt0CUiLQCDocpAAALgBAAAAMdtI +i6wkgAAAAEiBxIgAAADDMcCJw0iLrCSAAAAASIHEiAAAAMOIjCSgAAAASImcJJgAAABIiYQkkAAA +AEiDPchWDgAAdwQx0usjZpDou9oEAEUPV/9kTIs0Jfj///9IiwQkSInCSIuEJJAAAABIiVQkMEiN +SFhIiUwkSJBIicjoqEQAAEiLhCSQAAAAg3gcAHRHSIM4AHVBkJBIi0QkSOhoRgAASIucJJgAAABI +hdt0EUiLjCSQAAAASItBIOiqjwAAuAEAAAAx20iLrCSAAAAASIHEiAAAAMPrRITJdRQxwInDSIus +JIAAAABIgcSIAAAAwzHAMdu5AwAAAL8QAAAAvgIAAADogZgCAEiNBe/VBQC7CwAAAOgwbgIASInI +SItQSGYPH4QAAAAAAEiF0g+EwgAAAEiNeEhMi0IITYXAdEmDPSNXDgAAdRZJx0AQAAAAAEyJQEhI +x0IIAAAAAOtQTY1IEEiJ+UyJz0iJ0zHS6NfbBABIic/oL9wEAEiNewjoxtsEAEiJ2usmgz3aVg4A +AHUHRA8ReEjrFkUxwOgJ3AQASI14UA8fRAAA6PvbBACAejQAdCFMiwJIicExwEG5AQAAAPBFD7GI +eAEAAEEPlMBBg/AB6xRIicFBuQEAAABFMcAPH4QAAAAAAEWEwA+FJ////+sFSInBMdJIhdIPhaYD +AABIgzkADx8AD4aWAAAASItZMA+3URhID6/aSANZEEiLlCSYAAAAZpBIhdJ0JEiJXCQ4SItBIEiJ +3kiJ00iJ8eikjAAASIuMJJAAAABIi1wkOEiLQSDoDo4AAEiLjCSQAAAASItRMEj/wkiJUTBIOVEI +dQhIx0EwAAAAAEj/CZCQSItEJEgPHwDoe0QAALgBAAAAicNIi6wkgAAAAEiBxIgAAADDD7aUJKAA +AACE0nUgkJBIi0QkSOhMRAAAMcCJw0iLrCSAAAAASIHEiAAAAMNMiXQkUOiumAIASMdAKAAAAABI +i0wkMJBIhcl0CEjHQCj/////gz1sVQ4AAHUWSIuUJJgAAABIiVAYSMdAQAAAAADrHEiNeBhIi5Qk +mAAAAOgj2gQASI14QDHS6BjaBABIi1QkUIQCSI26SAEAAIM9I1UOAAB1DEiJgkgBAABIiRDrE+jQ +2AQASIn7SInH6OXZBABIid/GQDQATI1AUEyNiogAAACDPexUDgAAdRlMi5QkkAAAAEyJUFBIx4KI +AAAAAAAAAOswSIn7TInHSInWSIuUJJAAAADom9kEAEyJz02JwkUxwOjt2QQASInfTYnQSYnSSIny +kIM9mVQOAAB1CkjHQAgAAAAA6xtMjVgISIn7TInfTInGRTHA6LjZBABIid9JifBIiUQkQEiJfCRo +TIlEJGBMiUwkWE2NWkBNi2JATYXkdEaDPUpUDgAAdQ9MiWAQSYlEJAhJiUJA63VMjVAQSIn7TInX +TInGTYng6GTZBABJjXgI6NvXBABMid/o09cEAEiJ30mJ8OtHgz0EVA4AAHUSSMdAEAAAAABJiUI4 +SYlCQOssTI1gEEiJ+0yJ50yJxkUxwOgb2QQASY16OOiS1wQATInf6IrXBABIid9JifBMjYK5AAAA +QbkBAAAARYYISI0FjRsGAEiLXCRIuQ4AAAC/FwAAAL4CAAAA6LSUAgBIi0QkQEiLVCRQZg8fhAAA +AAAAkEg5gkgBAAAPhf0AAACDPWxTDgAAdQ1Ix4JIAQAAAAAAAOsMSIt8JGgxyegR2AQAxoK4AAAA +AEiLSChIhcl+H0iLVCQwSCnRSInIuwIAAADobAMCAEiLRCRASItUJFAPtkg1iEwkL4M9E1MOAAB1 +FUjHgogAAAAAAAAASMdAUAAAAADrGEiLfCRYMdLo0NcEAEiLfCRgMdLoxNcEAA8fQADo+5gCALgB +AAAAD7ZcJC9Ii6wkgAAAAEiBxIgAAADDRA8RfCRwTI0FlAAAAEyJRCRwSIlMJHhIichIidNIi4wk +mAAAAEiNfCRwvgMAAADorQAAALgBAAAAicNIi6wkgAAAAEiBxIgAAADDSI0FBu4FALsbAAAA6EVp +AgCQSIlEJAhIiVwkEIhMJBjoUdMEAEiLRCQISItcJBAPtkwkGGaQ6Tv5///MzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMxJO2YQdilIg+wQSIlsJAhIjWwkCEiLSgiEAUiNQViQkOibQAAASItsJAhI +g8QQw+hM0gQA68rMzMzMzMzMzMzMSTtmEA+GiwEAAEiD7DBIiWwkKEiNbCQoSIl8JFBIiXQkWEiJ +XCRASIN4CAB1K0iFyQ+ErgAAAEiLQCAPH0QAAOi78///SItcJEBIi3QkWEiLfCRQ6YwAAABIiUQk +OEiLUDBED7dAGEkPr9BIA1AQSIXJdCZIiVQkGEiLcCBIictIidFIifDo1YcAAEiLRCQ4SItUJBhI +i1wkQEiLcCBIi0sYSInwSInT6LOHAABIi1QkOEiLcjBI/8ZIiXIwDx8ASDlyCHUISMdCMAAAAABI +i0IwSIlCKEiLXCRASIt0JFhIi3wkUIQDgz3yUA4AAHUKSMdDGAAAAADrFUiNQxhIiflIicdFMcDo +FNYEAEiJz0iLA0iJRCQgSIsPSIn6/9FIi0QkIIQAgz2zUA4AAHUOSItMJEBIiYiIAAAA6xFIjbiI +AAAASItMJEDoUtUEAMZBNQFIg3koAHQk6CLTBABFD1f/ZEyLNCX4////SIsEJEiLTCRASIlBKEiL +RCQgSItMJFhIjVkB6LWSAgBIi2wkKEiDxDDDSIlEJAhIiVwkEEiJTCQYSIl8JCBIiXQkKOgt0QQA +SItEJAhIi1wkEEiLTCQYSIt8JCBIi3QkKOkv/v//zMzMzMzMzMzMzMzMzMzMSTtmEHY5SIPsEEiJ +bCQISI1sJAjGgLgAAAABSI2IuQAAADHShhGQkEiJ2OhwPgAAuAEAAABIi2wkCEiDxBDDSIlEJAhI +iVwkEOiy0AQASItEJAhIi1wkEOumzMzMzMzMiwXeTQ4AJfA//w+APXhNDgAAdAw9oAYCAHQTDx9E +AAA90AYCAHQHPaAGAwB1B7gBAAAA6wg94AYDAA+UwIA9PlQOAAB0BYPwAesCMcCIBT5NDgDDzMzM +zMzMzMzMzMzMSTtmEA+GMAEAAEiD7FhIiWwkUEiNbCRQSIlcJGhIiXwkeEiJTCRwSIlEJGBIibQk +gAAAAOsu6GXxBABFD1f/ZEyLNCX4////SItEJGBIi0wkcEiLXCRoSIu0JIAAAABIi3wkeEiJwjHA +TI0FaE4OAEG5AQAAAPBFD7EIQQ+UwpBFhNJ0sYM9UE4OAAAPhJMAAACEAkiDulgfAAAAfxRIg7pg +HwAAAHcKSIO6aB8AAAB2CEiJ0OjmAQAASMdEJEgAAAAASMdEJEgBAAAA6O/qBABFD1f/ZEyLNCX4 +////SItEJGhIjYhoAQAASIXAuwAAAABID0XZSIsF84ILAEiLDCRIjXwkSL4BAAAASYnwTItMJHBM +i1QkeEyLnCSAAAAA6BteAwAxwEiNDapNDgCHAUiLbCRQSIPEWMNIiUQkCEiJXCQQSIlMJBhIiXwk +IEiJdCQo6OjOBABIi0QkCEiLXCQQSItMJBhIi3wkIEiLdCQo6Yr+///MzMzMzMzMzMzMSIPsIEiJ +bCQYSI1sJBhIiUwkOEiJXCQw6xzoAfAEAEUPV/9kTIs0Jfj///9Ii0wkOEiLXCQwMcBIjRUZTQ4A +vgEAAADwD7EyQA+Ux0CE/3TJSIsFYKELAEiNNAFIjXYBSIH+6AMAAH15SD3oAwAAD4OCAAAASI1x +AUiNPfiBCwBIiTTHSI2wGfz//0mJ8Ej33kg58UgPTPFMjUgBScHhA0nB+D9NIcFKjQQPSDnDdBhI +weYDSInx6ITeBABIi0wkOEiNFZBMDgBIiwXpoAsASI0ECEiNQAFIiQXaoAsAZpDrB0j/BdegCwAx +wIcCSItsJBhIg8Qgw7noAwAADx9AAOgb1QQAkMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMTI1k +JPBNO2YQD4ZLAgAASIHskAAAAEiJrCSIAAAASI2sJIgAAABIiYQkmAAAAEjHRCRgAAAAAEjHRCRg +AQAAADHJ62BIiUwkSEyLaBBMjZkZ/P//SMHiA02J30nB+z9MIdpMjQwQTY1JGE2NVCT/SfffMdtI +jXwkYL4BAAAASYnwTYn7TInoMcnoElwDAEiLVCRISIuEJJgAAABMi2TQGEmNDBSEAEg5iFgfAAB+ +NkiB+egDAAAPg5ABAABMi2TIGE2NLAxJgf3oAwAAD4dtAQAASI1RAUw56g+GZf///5DpTwEAAEjH +gFgfAAAAAAAASIO4YB8AAAAPho8AAABIx0QkWAAAAABIi5BgHwAASIlUJFhIjRXmEgYAhAJIjRXN +EgYAhAJIixXUEgYATIslvRIGAEyNTCR4RQ8ROUj/wkiJVCR4SY1UJAFIiZQkgAAAAEiLUBAx2zHJ +SI18JFi+AQAAAEmJ8EG6AgAAAE2J00iJ0OgpWwMASIuUJJgAAABIx4JgHwAAAAAAAEiJ0EiDuGgf +AAAAD4aJAAAASMdEJFAAAAAASIuQaB8AAEiJVCRQSI0VURIGAIQCSI0VUBIGAIQCSIsVPxIGAEyL +JUASBgBMjUwkaEUPETlI/8JIiVQkaEmNVCQBSIlUJHBIi1AQMdsxyUiNfCRQvgEAAABJifBBugIA +AABNidNIidDoj1oDAEiLlCSYAAAASMeCaB8AAAAAAABIi6wkiAAAAEiBxJAAAADDSInQTInp6IHT +BABMiem66AMAAOj00gQASInIuegDAADop9IEAJBIiUQkCJDoO8sEAEiLRCQI6ZH9///MzMzMzMzM +zMzMzMzMzMzMzEiD7DhIiWwkMEiNbCQwSIlEJEBJi04wTInyZg8fRAAASDmRwAAAAA+FjwAAAEiN +TCRASDkKc29IOUoIcmlEDxF8JAhEDxF8JBhIx0QkKAAAAABIjQWHAAAASIlEJBhIjUQkQEiJRCQg +SI1EJAhIiUQkKEiNRCQYSIkEJA8fRAAA6FvJBABFD1f/ZEyLNCX4////SItEJAhIi1wkEEiLbCQw +SIPEOMNIjQXa6gUAux0AAABIi2wkMEiDxDjDSI0FxOoFALsdAAAASItsJDBIg8Q4w8zMzMzMzMzM +zMzMzMzMSTtmEA+GOgMAAEiD7GBIiWwkWEiNbCRYSItKEEiJTCRISItSCEiJVCRQSIsC6C7xAwBI +hcB1OEiLfCRISMdHCBoAAACDPfVIDgAAdQxIjQUX4wUASIkH6wxIjQUL4wUA6JvMBABIi2wkWEiD +xGDDSIlcJEBIiUQkOOjC9wMAZpBIg/sMD4+7AAAASIP7C3VVSIsQSbhkZWJ1Z0NhbA8fAEw5wnUe +ZoF4CGwzdROAeAoyD4TAAQAATDnC6wcPH0AATDnCD4W9AQAAZoF4CGw2D4WxAQAAgHgKNA+EmAEA +AJDpoQEAAEiD+wwPhZcBAABIixBJuGRlYnVnQ2FsDx9AAEw5wnUQgXgIbDEyOA+EZQEAAEw5wnUS +gXgIbDI1NmaQD4RRAQAATDnCD4VXAQAAgXgIbDUxMg+EOwEAAA8fQADpQQEAAEiD+w0PhIgAAABI +g/sOD4UtAQAASIsQSbhkZWJ1Z0NhbEw5wnUjgXgIbDE2M3UXZoF4DDg0Zg8fRAAAD4TxAAAATDnC +6wNMOcJ1HoF4CGwzMjd1EmaBeAw2OJAPhNEAAABMOcLrA0w5wg+F0gAAAIF4CGw2NTUPH0QAAA+F +wAAAAGaBeAwzNg+EpQAAAOmvAAAASIlcJChIiUQkMEiNDSTJBQC/DQAAAOjOxf//SIXAf0FIi0Qk +MEiLEEm4ZGVidWdDYWxMOcJ1F4F4CGwxMDJ1C4B4DDR0Wkw5wusDTDnCdVqBeAhsMjA0dVGAeAw4 +dEHrSUiLRCQwSIsQSbhkZWJ1Z0NhbEw5wnUXgXgIbDQwOXULgHgMNnQZTDnC6wNMOcJ1GYF4CGw4 +MTl1EIB4DDJ1CkiLbCRYSIPEYMNIi1wkKEiD+wh+UEiNHSa/BQC5CAAAAOiDxv//Dx8AhMB0OEiL +fCRISMdHCB8AAACDPYhGDgAAdQxIjQVt7AUASIkH6wxIjQVh7AUA6C7KBABIi2wkWEiDxGDDSItU +JFBMiwJIi0QkOEw5AHQGSf/ITIkCSIs6SItcJEAxyTH26Dv8AwCD+P90MEiLfCRISMdHCBYAAACD +PSJGDgAAZpB1DEiNBWHXBQBIiQfrDEiNBVXXBQDoxskEAEiLbCRYSIPEYMPoV8YEAOmy/P//zMzM +zMzMzMzMzMzMzMzMzMzMSIPsUEiJbCRISI1sJEjGRCQLAMdEJAwAAAAATIl0JBBIjUwkGEQPETlI +jVQkKEQPETpIjVQkOEQPETpIjRWeAAAASIlUJBhMifJIiVQkIEiLXCRQSIlcJChIiUQkMEiNRCQL +SIlEJDhIjUQkDEiJRCRASIkMJOglxQQARQ9X/2RMizQl+P///0iNBUkNBgDojMQEAIB8JAsAdC1I +i0QkEEiLSDCLVCQMiZFwAgAA/4l0AgAAkEiJwkiJgWgBAABIiYroAAAA6wVIi1QkEMaCtAAAAABI +i2wkSEiDxFDDzMxJO2YQD4Y8AQAASIPsWEiJbCRQSI1sJFBIi3IQTItCGEyJRCQoTItKIEyJTCRA +TItSKEyJVCRISIt6CEiJfCQ4SI0FwQwGADHbMcnoMAYDAEiJRCQwSI0FhHAFAA8fQADom1AAAEiL +VCQoSIkQgz2MRA4AAHUMSItMJDhIiUgIkOsOSI14CEiLTCQ46DDJBABIi1QkMIQCgz1iRA4AAGaQ +dQlIiYKIAAAA6wxIjbqIAAAA6AnIBABIi7HoAAAAZpBIhfZ0TkiLeTBIOf51YUiLRCRAxgABi4dw +AgAASItcJEiJA/+HdAIAAMeHcAIAAAAAAACQSInQSImXaAEAAJBIibjoAAAASMeB6AAAAAAAAADr +A0iJ0MaBtAAAAAGQSImBoAAAAEiLbCRQSIPEWMNIjQXp0QUAuxQAAAAPH0QAAOibWgIAkOgVxAQA +6bD+///MzMzMzMzMzMzMzMzMzMzMSTtmEHZwSIPsGEiJbCQQSI1sJBBJi46IAAAASIsBSItJCEiJ +TCQIgz1yQw4AAHUNSceGiAAAAAAAAADrEUyJ8kiNuogAAAAx0ugyyAQA6C0AAACQSItMJAhJiY6g +AAAASI0FOQsGAOh0wgQASItsJBBIg8QYw+glxAQA64PMzMxJO2YQD4anAAAASIPsOEiJbCQwSI1s +JDBJx8UAAAAATIlsJCjGRCQHAEjHRCQQAAAAAEjHRCQIAAAAAEiJRCQISI1EJAhIiUQkEMZEJAYA +RA8RfCQYSI0FhAAAAEiJRCQYSI1EJAZIiUQkIEiNRCQYSIlEJCjGRCQHAUiLVCQQSIsC/9DGRCQG +AcZEJAcASItUJChIiwL/0EiLbCQwSIPEOMPoWUACAEiLbCQwSIPEOMNIiUQkCOhlwwQASItEJAjp +O////8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2RkiD7BhIiWwkEEiNbCQQSItKCIA5 +AHUlSI1EJCDouVgCAEiJBCRIiVwkCOgrygQARQ9X/2RMizQl+P///0iLbCQQSIPEGMPoT8IEAOut +zMzMzMzMzMzMzMzMzEk7ZhAPhhwBAABIg+xISIlsJEBIjWwkQEiJRCRQSIsVLFYLAEiLNS1WCwBI +hdIPhN4AAABIhfZ+E0iJXCRYSIlEJFBIiXQkGDHJ6xUxwDHbSItsJEBIg8RIw0iDwhBIiflIi3oI +TIsCSDn7fVpFD7YMGEGA+T10BUUxyetNSIlMJChIiVQkOEiJfCQgTIlEJDBIidlIicNMicDoIcH/ +/0iLTCQoSItUJDhIi1wkWEiLdCQYSIt8JCBMi0QkMEGJwUiLRCRQ6wNFMclFhMl1DkiNeQFIOf5/ +gulv////SI1DAUg5x3IkSCnfSI1f/0iJ2Uj320jB+z9IIdhMAcBIictIi2wkQEiDxEjDSIn56O7J +BABIjQVT0gUAuxYAAABmkOibVwIAkEiJRCQISIlcJBDoq8EEAEiLRCQISItcJBCQ6bv+///MzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxMjWQkgE07ZhAPhoYDAABIgewAAQAASImsJPgAAABIjawk ++AAAAEiJhCQIAQAASIsISIXJdQ67CQAAAEiNDSK6BQDrE0iJyOg0cwQASInBSIuEJAgBAABIiVwk +UEiJTCR4SItQEEiJ0OgTcwQASIuMJAgBAABIi1EIDx+AAAAAAEiF0g+EzAIAAEiJXCRgSImEJIgA +AABIidDo4nIEAEiLjCQIAQAASIN5IAAPhcQBAABIiVwkWEiJhCSAAAAASI28JJgAAABIjX/gSIls +JPBIjWwk8OiSygQASIttAEiNFWrRBQBIiZQkmAAAAEjHhCSgAAAAFgAAAEiLVCR4SImUJKgAAABI +i1QkUEiJlCSwAAAASI0V17MFAEiJlCS4AAAASMeEJMAAAAAEAAAASImEJMgAAABIiZwk0AAAAEiN +FXi1BQBIiZQk2AAAAEjHhCTgAAAABgAAAEiLlCSIAAAASImUJOgAAABIi3QkYEiJtCTwAAAAuQYA +AABIic8xwEiNnCSYAAAA6DTVAwBIi0wkWEiLVCRgSDnRD4XGAAAASIlEJHBIiVwkSEiLhCSAAAAA +SIucJIgAAADoor7//2aQhMB1D0iLXCRISItEJHDpkgAAAEiLjCQIAQAASItBCJDou3IEAEiJhCSQ +AAAASIlcJGhIi4wkCAEAAEiLSRBIicjomnIEAEiLTCRoSDnLdRlIicNIi4QkkAAAAA8fRAAA6Du+ +//+EwHUfMcBIi1wkcEiLTCRISI09xeUFAL4gAAAA6BrXAwDrHTHASItcJHBIi0wkSEiNPT/hBQC+ +HgAAAOj71gMASIusJPgAAABIgcQAAQAAw0iNvCSYAAAASI1/4EiJbCTwSI1sJPDo28gEAEiLbQBI +jRWzzwUASImUJJgAAABIx4QkoAAAABYAAABIiYQkqAAAAEiJnCSwAAAASI0Vm7UFAEiJlCS4AAAA +SMeEJMAAAAAIAAAASIuUJIgAAABIiZQkyAAAAEiLVCRgSImUJNAAAABIjRXjxAUASImUJNgAAABI +x4Qk4AAAABEAAABIi1EgSItxGEiJtCToAAAASImUJPAAAAAxwEiNnCSYAAAAuQYAAABIic8PH0AA +6HvTAwBIi6wk+AAAAEiBxAABAADDuRYAAABIi3wkeEiLdCRQTI0FK74FAEG5DQAAAEmJwkmJ2zHA +SI0dzs4FAOh71gMASIusJPgAAABIgcQAAQAAw0iJRCQI6OG9BABIi0QkCOlX/P//zMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMxJO2YQdjZIg+wwSIlsJChIjWwkKEiJRCQ4uQ8AAABIicdIid4xwEiNHQnB +BQDobtUDAEiLbCQoSIPEMMNIiUQkCEiJXCQQ6HW9BABIi0QkCEiLXCQQ66nMzMzMzMzMzMxJO2YQ +djZIg+wwSIlsJChIjWwkKEiJRCQ4uQ8AAABIicdIid4xwEiNHanABQDoDtUDAEiLbCQoSIPEMMNI +iUQkCEiJXCQQSIlMJBjoEL0EAEiLRCQISItcJBBIi0wkGJDrnszMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzEiJRCQIw8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMTI2kJDj///9NO2YQD4bk +BQAASIHsSAEAAEiJrCRAAQAASI2sJEABAABAD7bXSIP6CQ+DsQUAAEmJ0EjB4gRMjQ37LwsATosU +Ck6LTAoIhMl0KEiFwH0jZg8fhAAAAAAASYP4CA+DcQUAAEyNBU8vCwBOixQCTotMAghMiZQkMAEA +AIhMJEdIiZwkiAAAAEiJhCSAAAAATIlMJFBIx4QkvAAAAAAAAABIjbwkwAAAAEiNf+BmDx+EAAAA +AAAPH0AASIlsJPBIjWwk8OjcxQQASIttAEi6cnVudGltZSBIiZQkvAAAAEi6IGVycm9yOiBIiZQk +wwAAADHSvg8AAAC/ZAAAAEyNhCS8AAAA6wZI/8JMieZJOdEPjukBAAAPH0AAD4aqBAAASIl0JFhF +D7YcEkGA+yV0b0yNZgFMOedzW0iJVCRIRIhcJEZIjQXwOwUATInDSInxTInm6OKZAwBMjWMBSItU +JEhIi5wkiAAAAEiLdCRYTItMJFBMi5QkMAEAAEQPtlwkRkmJwEiJz0iLhCSAAAAAD7ZMJEdFiBww +ZpDpaP///0yNWgFNOdkPhhMEAABMiVwkeEIPtlQSAQ8fAID6eA+FlwAAAITJdGZIhcB9YUiNVgFI +OddzS0iNBV47BQBMicNIifFIidboUJkDAEiNUwFIi5wkiAAAAEiLdCRYTItMJFBMi5QkMAEAAEyL +XCR4SYnASInPSIuEJIAAAAAPtkwkR0HGBDAtSYnESPfY6wZJicRIifJIx4QklAAAAAAAAABMjawk +mAAAAEUPEX0AvhMAAADpOwIAAA8fQACA+nkPhYwAAABIhdt9YUiNVgFIOddzS0iNBcI6BQBMicNI +ifFIidbotJgDAEiNUwFIi5wkiAAAAEiLdCRYTItMJFBMi5QkMAEAAEyLXCR4SYnASInPSIuEJIAA +AAAPtkwkR0HGBDAtSYncSPfb6wZJidxIifJIx4QkqAAAAAAAAABMjawkrAAAAEUPEX0AvhMAAADr +UEyJ2kmJ9A8fRAAA6Qj+//8xwEyJw0iJ8ehO0wMASIusJEABAABIgcRIAQAAw02J7EyNazBEiKw0 +qAAAAEj/zkyJ4EyLpCSIAAAASInTTIn6SIP7CnI1SYnFSLjNzMzMzMzMzEmJ10j340jB6gNMjSSS +SdHkTCnjSIP+FHKw6fsAAABmDx+EAAAAAABIg/4UD4PbAAAATI1rMESIrDSoAAAASYnVSCnySIPC +FEiJVCRYTI1+7EyJ+0nB/z9MIf5Mjbw0qAAAAA8fRAAASDnXcz9IiVwkcEyJbCRoTIm8JCABAABI +jQViOQUATInDTInpSInW6FSXAwBIi1wkcEyLbCRoTIu8JCABAABJicBIic9MiYQkOAEAAEiJfCRg +S40EKEj320iJ2UyJ+2aQ6BvJBABIi4QkgAAAAA+2TCRHSIucJIgAAABMi0wkUEyLlCQwAQAATItc +JHhIi3wkYEiLdCRYTIuEJDgBAABmkOmQ/v//SInwuRQAAADorr8EAEiJ8LkUAAAA6KG/BABJg8Uw +RIisNJQAAABI/85JicRIidBMifpIg/gKci9JicVIuM3MzMzMzMzMSYnXSfflSMHqA0yJ4EyNJJJJ +0eRNKeVIg/4Ucrjp7AAAAEiD/hQPg9UAAABMjWgwRIisNJQAAABJidVIKfJIg8IUSIlUJFhMjX7s +TIn4ScH/P0wh/kyNvDSUAAAASDnXcz9IiUQkcEyJvCQoAQAATIlsJGhIjQUbOAUATInDTInpSInW +6A2WAwBMi2wkaEyLvCQoAQAASYnASInPSItEJHBMiYQkOAEAAEiJfCRgS40UKEj32EyJ+0iJwUiJ +0OjTxwQASIuEJIAAAAAPtkwkR0iLnCSIAAAATItMJFBMi5QkMAEAAEyLXCR4SIt8JGBIi3QkWEyL +hCQ4AQAA6Ur9//9IifC5FAAAAOhovgQASInwuRQAAADoW74EAEyJ2EyJyehQvgQASInQTInJ6EW+ +BABMicC5CAAAAOg4vgQASInQuQkAAADoK74EAJBIiUQkCEiJXCQQiEwkGECIfCQZ6LK2BABIi0Qk +CEiLXCQQD7ZMJBgPtnwkGenZ+f//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEyNZCT4TTtmEA+G +LwUAAEiB7IgAAABIiawkgAAAAEiNrCSAAAAASImEJJAAAABIiZwkmAAAAEiFwA+E1AQAAItIEIH5 +bVQasw+HVwIAAIH5jAIleQ+HMAEAAGYPH0QAAIH5+3+iLg+HgwAAAIH5xQb/E3U2SI0NRS8FAA8f +RAAASDnID4WDBAAAD7YDiEQkE+grZAIAD7ZEJBPo4WYCAJDom2QCAOmEBAAAgfn7f6IuD4VWBAAA +SI0NgzAFAA8fAEg5yA+FQwQAAPIPEAPyDxFEJDjo6GMCAPIPEEQkOGaQ6PtmAgDoVmQCAOk/BAAA +gflfQj5mdThIjQ0CNgUAZpBIOcgPhQMEAAAPtgNIiUQkcOiqYwIASItEJHAPH0QAAOi7aQIA6BZk +AgDp/wMAAIH5jAIleQ+F0QMAAEiNDX4vBQBIOcgPhcEDAADzDxBDBPMPEUQkGPMPEAvzDxFMJBTo +W2MCAPMPEEQkFPMPWsDzDxBMJBjzD1rJ6MJoAgBmkOi7YwIA6aQDAACB+f+bP5Z3fYH5Lo0xhnU1 +SI0NHzUFAEg5yA+FYgMAAEiLA0iJRCRQ6AljAgBIi0QkUA8fQADoG2kCAOh2YwIA6V8DAACB+f+b +P5YPhTEDAABIjQ2eMAUASDnID4UhAwAASIsDSIlEJEjoyGICAEiLRCRIDx8A6NtpAgDoNmMCAOkf +AwAAgfnTPsKwdTpIjQ3iLgUAZpBIOcgPheMCAADzDxAD8w8RRCQc6IhiAgDzDxBEJBzzD1rA6Jll +AgDo9GICAOndAgAAgfltVBqzD4WvAgAASI0NHC4FAEg5yA+FnwIAAPIPEEMI8g8RRCQw8g8QC/IP +EUwkIOg5YgIA8g8QRCQg8g8QTCQw6KhnAgDoo2ICAA8fAOmJAgAAgfk96ErQD4cPAQAAgfmS10q9 +D4eDAAAADx8AgfkCQa27dThIjQ1xLwUASDnID4U0AgAASGMDSIlEJGjo22ECAEiLRCRo6PFoAgDo +TGICAOk1AgAADx+AAAAAAIH5ktdKvQ+FAAIAAEiNDS00BQBIOcgPhfABAABIiwNIiUQkQOiXYQIA +SItEJEDorWcCAOgIYgIA6fEBAAAPHwCB+SfABsx1OEiNDXEvBQBIOcgPhbQBAABID74DSIlEJGjo +WmECAEiLRCRo6HBoAgDoy2ECAOm0AQAAZg8fRAAAgfk96ErQD4WAAQAASI0N7TIFAEg5yA+FcAEA +AIsDSIlEJHDoGGECAEiLRCRw6C5nAgDoiWECAOlyAQAADx9AAIH5tFz/4A+HlAAAAIH5Ene41XU7 +SI0NJTIFAA8fRAAASDnID4UjAQAASIsDSIlEJFjoymACAEiLRCRYDx9EAADo22YCAOg2YQIA6R8B +AACB+bRc/+APhfEAAABIjQ2eMQUASDnID4XhAAAASItDCEiJRCQoSIsLSIlMJHgPH0AA6HtgAgBI +i0QkeEiLXCQo6IxpAgDo52ACAOnQAAAAZpCB+c6A1ex1OEiNDZEtBQBIOcgPhZQAAABID78DSIlE +JGjoOmACAEiLRCRo6FBnAgDoq2ACAOmUAAAAZg8fRAAAgfmgDvLvdS5IjQ2RMQUASDnIdVgPtwNI +iUQkcA8fQADo+18CAEiLRCRw6BFmAgDobGACAOtYgfn6cVP3dS5IjQ3bLAUASDnIdSJIiwNIiUQk +YOjJXwIASItEJGAPH0AA6NtmAgDoNmACAOsi6E8AAADrG+ioXwIASI0FvKQFALsDAAAA6LdoAgDo +EmACAEiLrCSAAAAASIHEiAAAAMNIiUQkCEiJXCQQ6DOxBABIi0QkCEiLXCQQ6aT6///MzMzMSTtm +EA+GcgcAAEiDxIBIiWwkeEiNbCR4SImEJIgAAABIiZwkkAAAAEiLhCSIAAAA6OtiBABIiUQkYEiJ +XCQgSIuMJIgAAAAPtkkXgPkID4cxAwAAgPkED4eSAQAADx8AgPkCD4fGAAAAgPkBdGOA+QIPhZcF +AABIi4QkkAAAAEiLAEiJRCRY6NReAgBIi0QkYEiLXCQg6OVnAgBIjQVfowUAuwEAAADo1GcCAEiL +RCRY6MplAgBIjQVFowUAuwEAAADouWcCAOgUXwIA6aEGAABIi4QkkAAAAA+2AIhEJBfoe14CAEiL +RCRgSItcJCDojGcCAEiNBQajBQC7AQAAAOh7ZwIAD7ZEJBfoEWECAEiNBeyiBQC7AQAAAA8fRAAA +6FtnAgDotl4CAOlDBgAAgPkDdVtIi4QkkAAAAEgPvgBIiUQkUOgWXgIASItEJGBIi1wkIOgnZwIA +SI0FoaIFALsBAAAA6BZnAgBIi0QkUOgMZQIASI0Fh6IFALsBAAAA6PtmAgDoVl4CAOnjBQAASIuE +JJAAAABID78ASIlEJFDou10CAEiLRCRgSItcJCDozGYCAEiNBUaiBQC7AQAAAOi7ZgIASItEJFDo +sWQCAEiNBSyiBQC7AQAAAA8fRAAA6JtmAgDo9l0CAOmDBQAAgPkGD4fNAAAAgPkFdWhIi4QkkAAA +AEhjAEiJRCRQ6E5dAgBIi0QkYEiLXCQgDx9AAOhbZgIASI0F1aEFALsBAAAA6EpmAgBIi0QkUA8f +RAAA6DtkAgBIjQW2oQUAuwEAAADoKmYCAOiFXQIADx9EAADpDQUAAEiLhCSQAAAASIsASIlEJFDo +5lwCAEiLRCRgSItcJCDo92UCAEiNBXGhBQC7AQAAAOjmZQIASItEJFCQ6NtjAgBIjQVWoQUAuwEA +AADoymUCAOglXQIADx9EAADprQQAAID5B3VbSIuEJJAAAABIiwBIiUQkSOiBXAIASItEJGBIi1wk +IOiSZQIASI0FDKEFALsBAAAA6IFlAgBIi0QkSOh3YgIASI0F8qAFALsBAAAA6GZlAgDowVwCAJDp +TQQAAEiLhCSQAAAAD7YASIlEJEDoJlwCAEiLRCRgSItcJCDoN2UCAEiNBbGgBQC7AQAAAOgmZQIA +SItEJECQ6BtiAgBIjQWWoAUAuwEAAADoCmUCAOhlXAIADx9EAADp7QMAAID5DA+HlwEAAID5Cg+H +zgAAAID5CXVpSIuEJJAAAAAPtwBIiUQkQOivWwIASItEJGBIi1wkIA8fRAAA6LtkAgBIjQU1oAUA +uwEAAADoqmQCAEiLRCRADx9EAADom2ECAEiNBRagBQC7AQAAAOiKZAIA6OVbAgAPH0QAAOltAwAA +SIuEJJAAAACLAEiJRCRA6EdbAgBIi0QkYEiLXCQg6FhkAgBIjQXSnwUAuwEAAADoR2QCAEiLRCRA +ZpDoO2ECAEiNBbafBQC7AQAAAOgqZAIA6IVbAgAPH0QAAOkNAwAAgPkLdVtIi4QkkAAAAEiLAEiJ +RCRA6OFaAgBIi0QkYEiLXCQg6PJjAgBIjQVsnwUAuwEAAADo4WMCAEiLRCRA6NdgAgBIjQVSnwUA +uwEAAADoxmMCAOghWwIAkOmtAgAASIuEJJAAAABIiwBIiUQkOOiGWgIASItEJGBIi1wkIOiXYwIA +SI0FEZ8FALsBAAAA6IZjAgBIi0QkOJDoe2ACAEiNBfaeBQC7AQAAAOhqYwIA6MVaAgAPH0QAAOlN +AgAAgPkOD4fDAAAAgPkNdWFIi4QkkAAAAPMPEADzDxFEJBzoFloCAEiLRCRgSItcJCDoJ2MCAEiN +BaGeBQC7AQAAAOgWYwIA8w8QRCQc8w9awOgHXQIASI0Fgp4FALsBAAAA6PZiAgDoUVoCAOneAQAA +SIuEJJAAAADyDxAA8g8RRCQw6LVZAgBIi0QkYEiLXCQg6MZiAgBIjQVAngUAuwEAAADotWICAPIP +EEQkMOiqXAIASI0FJZ4FALsBAAAA6JliAgDo9FkCAOmBAQAAgPkPD4QoAQAAZg8fRAAAgPkQD4TO +AAAAgPkYdGFIi4QkkAAAAEiJRCRwDx9EAADoO1kCAEiNBdWdBQC7AQAAAOhKYgIASItEJGBIi1wk +IOg7YgIASI0F0Z0FALsCAAAA6CpiAgBIi0QkcA8fRAAA6JthAgDodlkCAOkDAQAASIuEJJAAAABI +iwhIiUwkaEiLQAhIiUQkWOjTWAIASItEJGBIi1wkIOjkYQIASI0FeJ0FALsCAAAA6NNhAgBIi0Qk +aEiLXCRY6MRhAgBIjQVmnQUAuwIAAADos2ECAOgOWQIA6ZsAAABIi4QkkAAAAPIPEADyDxFEJDDy +DxBICPIPEUwkKOhnWAIASItEJGBIi1wkIOh4YQIA8g8QRCQw8g8QTCQo6MddAgDowlgCAGaQ61BI +i4QkkAAAAPMPEADzDxFEJBzzDxBIBPMPEUwkGJDoG1gCAEiLRCRgSItcJCDoLGECAPMPEEQkHPMP +WsDzDxBMJBjzD1rJ6HNdAgDoblgCAEiLbCR4SIPsgMNIiUQkCEiJXCQQ6JWpBABIi0QkCEiLXCQQ +6Wb4///MzMzMzMxMjaQkWP///007ZhAPhkwDAABIgewoAQAASImsJCABAABIjawkIAEAAEiLhCQo +AQAA6GrQAwDoRdcDAEiJhCSAAAAASIlcJDBIiQQkSIlcJAjGRCQQKOhlqf//RQ9X/2RMizQl+P// +/0iLTCQYSIXJD4y/AgAASI1B/0iLVCQwkEg5wg+CpAIAAEyNQQJMOcJ/IzHASI0dV+YFALkxAAAA +SIu8JIAAAABIidbosMADAOirPgIAD4JqAgAADx9EAABJOcAPglACAABIi7wkgAAAAEQPtkw5/0QP +thQ5QcHiCEUJ0WZBgfkuKHWnRA+2TDkBQYD5KnWbSIlEJChIKcpIiVQkOEiNQv5IiUQkMEiJwUj3 +2EjB+D9MIcBIAfhIiYQkiAAAAEiJBCRIiUwkCMZEJBAp6Iio//9FD1f/ZEyLNCX4////SItEJBhI +hcAPjJ4BAABIjUgCSItUJDAPH0AASDnRfCUxwEiNHTjiBQC5LgAAAEiLvCSIAAAASInWZpDo278D +AOjWPQIAD4dcAQAASDnID4dOAQAASIu8JIgAAABED7cEB2ZBgfgpLnW3SIn6SI28JJAAAABIjX/Q +Dx8ASIlsJPBIjWwk8OiKsQQASIttAEiNNfSoBQBIibQkkAAAAEjHhCSYAAAADQAAAEiLtCSAAAAA +SIm0JKAAAABIi3QkKEiJtCSoAAAASI01VJoFAEiJtCSwAAAASMeEJLgAAAABAAAASImUJMAAAABI +iYQkyAAAAEiJtCTQAAAASMeEJNgAAAABAAAASIt0JDhIKcZIg8b8SYnwSPfeSMH+P0ghzkgB1kiJ +tCTgAAAATImEJOgAAABIjTWCsAUASIm0JPAAAABIx4Qk+AAAABMAAABIiZQkAAEAAEiJhCQIAQAA +SI0V8Z0FAEiJlCQQAQAASMeEJBgBAAAIAAAAMcBIjZwkkAAAALkJAAAASInP6O27AwDoiA0AAEiJ +w0iNBX4/BQDo2TQCAOi0rgQA6C+uBABIjUQkQEiNHbexBQC5EwAAAEiLvCSIAAAASIt0JDDoTL4D +AOhHPAIATInBDx9AAOh7rgQATInB6POtBABIicHo660EAEiNRCRgSI0dYLEFALkTAAAASIu8JIAA +AABIi3QkMOgIvgMA6AM8AgCQZpDoG6YEAOmW/P//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzEgzHTkl +DgBIui9kvXhkHXagSDHTSIXJD4SnAAAADx8ASIP5BHJudGRIg/kIckR0OEiD+RB2HEiD+TB2DkiJ +ykiJ3kiJ9+mgAQAASInK6d0AAABIicJIjTQBSI12+JCQSIsSkEiLNutgkJBIizBIifLrVkiJwkiN +PAFIjX/8kJCLMpCLF4nQifKJxus8kJCLEInW6zRIicZIjTwBSI1//w+2FkmJyEjR6Q+2DA5IweEI +SAnKD7YPSMHhEEgJykyJwTH26wRIidjDSL/bKLSg0X4D50gx10gx3kiJ8Ej350i7TxJ9xCdOjh1I +MctIMdBI9+NIMdDDSIswSL/bKLSg0X4D50gx/kyLQAhJMdhIicNMicBJidBI9+ZIg8HwkJCQSDHQ +SI1zEEyJwkiJw0iJ8EiD+RB3wEiNPAFIjX/wTI0ECE2NQPiQSIs/kEmLMEiJ0UiJ+ulw////TIsA +SbnbKLSg0X4D500xyEyLUAhJMdpIicNMidBJidJJ9+BMi0MQSbvjxoic8Gq8jk0x2EyLYxhJMfRI +icZMieBJidRJ9+BMi0MgSb3DTDd1zGWZWE0x6EyLeyhJMf9IicdMifhJiddJ9+BIg8HQkJCQTDHm +kJBMMf+QkEgx0EyNQzBMidJIifNIif5IicdMicBIg/kwD4dk////SDH+SDHz6TD////MiwBIudso +tKDRfgPnSDHBSDHYSDMFJyMOAEi6L2S9eGQddqBIMcJIichI9+JIMdBIicFIuEsSfcQnTo4dSPfh +kJBIMdDDzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSIsASLnbKLSg0X4D50gxwUgx2EgzBcYi +DgBIui9kvXhkHXagSDHCSInISPfiSDHQSInBSLhHEn3EJ06OHUj34ZCQSDHQw8zMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMSTtmEA+GvQIAAEiD7EBIiWwkOEiNbCQ4SItQOEiDeEAAD4SOAgAASIlc +JFBIiUQkSA+2cxRA9sYBdRkPH0QAAITJD4TNAQAAMcBIi2wkOEiDxEDDiEwkWEiLFZESCwBIidlI +icNIidDokwIAAEiFwA+F4gAAAJCQSI0F2SAOAJDoew4AAEiLBWQSCwBIi1wkSEiLTCRQ6GUCAAAP +H0QAAEiFwHQgSIlEJCCQkEiNBaUgDgDoKBAAAEiLRCQgDx8A6ZMAAABIi1QkSEiLckBI/85IweYD +SI1GIDHbSI0NPjcOAOi5MAAAhACDPVAhDgAAdRNIi0wkSEiJCEiLVCRQSIlQCOsgSInHSItMJEjo +7qUEAEiNUAhIiddIi1wkUGaQ6BumBABIiUQkMMdAEAAAAADoygMAAEiLRCQwDx9EAADoGwIAAJCQ +SI0FCiAOAOiNDwAASItEJDBIg3gYAHUVD7ZMJFiEyXQWMcBIi2wkOEiDxEDDSItsJDhIg8RAw2aQ +6HsDAABIiUQkKEiJXCQYSI0FCmQFAOilLAAAgz2eIA4AAHVRSItMJFBIiUgISItMJEhIiUgQSItM +JBhIiUgggz16IA4AAHUdSItMJChIiUgYSInDSI0FhTsFAA8fRAAA6JsvAgBIjXgYSItMJCjoDaUE +AOvcSI14CEiLTCRQZpDo+6QEAEiNeBBIi0wkSOjtpAQA66GLGugEVQQADx9AAOhbXQQASIlEJChI +iVwkGEiNBWpjBQDoBSwAAIM9/h8OAAB1UUiLTCRQSIlICEiLTCRISIlIEEiLTCQYSIlIIIM92h8O +AAB1HUiLTCQoSIlIGEiJw0iNBeU6BQAPH0QAAOj7LgIASI14GEiLTCQo6G2kBADr3EiNeAhIi0wk +UGaQ6FukBABIjXgQSItMJEjoTaQEAOuhSI0FDcYFALsfAAAA6Fo2AgCQSIlEJAhIiVwkEIhMJBjo +ZqAEAEiLRCQISItcJBAPtkwkGOkS/f//zMzMzMzMzMzMzMzMzMzMzMzMSIsQSP/Ki3MQM3EQSCHW +vwEAAADrDUyNRwFIAf5IIdZMiceQTI0E8E2NQBBNiwBNhcB0D0k5GHXdSTlICHXXTInAwzHAw8zM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GHAEAAEiD7EhIiWwkQEiNbCRASYtWMIO68AAA +AAAPhecAAABIixVgDwsASItyCEiLOkmJ+EjB7wJIjTx/SDn+D4KfAAAASIlUJBhIiUQkKEuNBABI +jUACSMHgAzHbuQEAAADokyAAAEiJRCQgSItUJBhIizJI0eZIiTBEDxF8JDBIjTXTwgQASIl0JDBI +iUQkOEiNRCQwDx9AAOjbBwAASItcJCBIi1MISIt0JBhIOVYIdUaQgD0fHg4AAHQRSI0Fxg4LAOhR +pf//SItcJCBIjQ21DgsASIcZSIsVqw4LAEiLRCQoSInDSInQ6EsAAABIi2wkQEiDxEjDSI0FJNEF +ALsnAAAA6LA0AgBIjQXloQUAuw8AAAAPH0AA6Js0AgCQSIlEJAjosJ4EAEiLRCQI6cb+///MzMzM +zMxIiwtIi1MISIswSP/Oi0kQM0oQSCHxugEAAADrDUiNegFIAdFIIfFIifqQSIt8yBBMjQTITY1A +EEg5+3QQDx8ASIX/ddhJhxhI/0AIw8PMzMzMzMzMzMzMzMzMzMzMzMxMjWQk2E07ZhAPhoUDAABI +geyoAAAASImsJKAAAABIjawkoAAAAEiJhCSwAAAASIsISIlMJHhIi1AISIlUJGhIidDomVAEAEiL +TCR4SItRQA+3WASLcAhIAcZmDx9EAABIgfsAAAEAD4cZAwAASIH6AAABAA+HAgMAAEiJRCRgSIl0 +JFBIiVQkGEiJXCRIMf9FMcBFMcnrKEiNeQFMi1QkGEiLTCR4TItEJChMidJIi1wkSEiLdCRQSYnB +SItEJGBIOdcPjR0BAABMi1E4TItZQEw53w+DlwIAAEiJfCQgTImUJJgAAABMiUQkKEyJjCSIAAAA +QYtc+gRIicgPH0AA6LtTBABIiUQkWEiLTCQgSIuUJJgAAACLHMpIi0QkeJDo+1AEAEiJhCSQAAAA +6E5ZBABIiYQkgAAAAEiJXCQ4SIuEJJAAAADo9FoEAEiF23UOSItMJHhIi0Ew6CFZBABIiUQkcEiJ +XCQwSItUJGhIi3QkUEiLTCRITItEJCgPHwDppgAAAEiLTCRASItUJFCLXAoISItEJGjo41UEAEiL +TCQgSIXJdQ1Ii5QksAAAAOnk/v//SIuUJLAAAABIjTzKSI1/GIM9dRsOAAB1B0iJRMoY6wXoJ58E +AEiLhCSIAAAA6bL+//9Ii4wksAAAAEyJSRgxwDHbSIusJKAAAABIgcSoAAAAw0iLfCQoTI1HAUiL +VCRoSIt0JFBIi3wkSEiLRCRwSIn5SItcJDBJOcgPjQgBAAAPgy8BAABMiUQkKEnB4ARMiUQkQEKL +HAZIidDozE8EAEiJhCSQAAAASItMJEBIi1QkUItcCgRIi0QkaOhMUgQASItMJFhIOch0BDHA6zVI +i4QkkAAAAOjxVwQASItMJDhIOdl0BDHA6xVIidlIi5wkgAAAAOhzmv//SItMJDhIi0wkWITAD4RH +////SIuEJJAAAADodFkEAEiF23UZSItMJGCLGUiLRCRoDx8A6DtPBADollcEAEiLlCSQAAAAD7YS +9sIBdAxIi1QkMLgBAAAA6yRIi1QkMEg52nQEMcDrFkiJ2UiLXCRwDx9AAOj7mf//SItUJDCEwA+F +U/7//0iLTCRY6cr+//9Ii4wksAAAAEjHQRgAAAAASIuEJIAAAABIi1wkOEiLrCSgAAAASIHEqAAA +AMNMicDoL6IEAEiJ+EyJ2egkogQAuwAAAQDoGqMEAEiJ2rsAAAEA6A2jBACQSIlEJAjoopoEAEiL +RCQI6Vj8///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4a+AAAASIPsQEiJbCQ4SI1sJDiQ +SI0FeBgOAOgbBgAASIsNnCwLAEiFyXQJSIsRSItJCOsEMckx0kiJVCQoSIlMJAgxwOsDSP/ASDnB +fiVIiUQkIEiLHMJIi7NYAQAASIl0JDBIi5tgAQAASIlcJBgx/+tIkJBIjQUWGA4A6JkHAABIi2wk +OEiDxEDDSIl8JBBIiwT+6AH6//9Ii0wkEEiNeQFIi0QkIEiLTCQISItUJChIi1wkGEiLdCQwSDnf +fMvrhOizmQQA6S7////MzMzMzMzMzMzMzMzMzEk7ZhB2WUiD7DBIiWwkKEiNbCQoSIlcJCBIiUQk +OEiLELkBAAAASInGSInQSInz6IwaAABIiUQkGEiJw0iLTCQgSItEJDjo1U4AAEiLRCQ4SItcJBhI +i2wkKEiDxDDDSIlEJAhIiVwkEOgymQQASItEJAhIi1wkEOuGzMzMzMzMSTtmEHZ0SIPsIEiJbCQY +SI1sJBhIiUQkKEiF23UJSI0FOykOAOtJSIlcJDBIiUQkKEiLHVArCwC4EAAAALkBAAAA6PkZAABI +i1QkMEiJUAiDPckXDgAAdQpIi0wkKEiJCOsNSInHSItMJCjocJwEAEiLbCQYSIPEIMNIiUQkCEiJ +XCQQ6JeYBABIi0QkCEiLXCQQ6Wj////MzMzMzMzMzEk7ZhB2VkiD7DBIiWwkKEiNbCQoSIlcJCBI +iUQkOEiLEDHJSInGSInQSInz6G8ZAABIiUQkGEiLVCQ4SIsKSItcJCDouKgEAEiLRCQ4SItcJBhI +i2wkKEiDxDDDSIlEJAhIiVwkEOgVmAQASItEJAhIi1wkEOuJzMzMzMzMzMzMSTtmEHZeSIPsKEiJ +bCQgSI1sJCBIiVwkOEiJTCRAZpBIhdt0MUiJTCQYuQEAAADojPT//0iFwHQPSItcJBhIi2wkIEiD +xCjDMcAx20iLbCQgSIPEKMMxwDHbSItsJCBIg8Qow0iJRCQISIlcJBBIiUwkGOiIlwQASItEJAhI +i1wkEEiLTCQY6XT////MzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2YEiD7CBIiWwkGEiNbCQYSIlE +JChIiw3wBgsASIlMJBAx0usDSP/CSDkRditIi1zREEiF23TuSIlUJAhIiwhIicJIidj/0UiLRCQo +SItMJBBIi1QkCOvNSItsJBhIg8Qgw0iJRCQI6PCWBABIi0QkCOuJzMzMzMzMzMzMSTtmEHYgSIPs +GEiJbCQQSI1sJBBIjQXCyAUAuyYAAADomywCAJDotZYEAOvTzMzMzMzMzMzMzMzMzMzMzMzMzEk7 +ZhAPhhwBAABIg+w4SIlsJDBIjWwkMEiLSwhI/8FIiUsIkEiJ2kjB4xBIic5IgeH//wcASAnLSInZ +SMH7E0jB4wNIOdp0BeslSInwSIsYSIkaSInGSInY8EgPsQ4PlMOE23TlSItsJDBIg8Q4w0iJdCQY +SIlUJChIiUwkEEiJXCQg6FhEAgBIjQX7zgUAuywAAADoZ00CAEiLRCQoZpDo20wCAEiNBR+KBQC7 +BQAAAOhKTQIASItEJBgPH0QAAOibSwIASI0F5YwFALsIAAAA6CpNAgBIi0QkEA8fRAAA6HtLAgBI +jQVmjQUAuwkAAADoCk0CAEiLRCQgDx9EAADoe0wCAOhWRgIA6FFEAgBIjQW2lAUAuwwAAAAPH0QA +AOhbKwIAkEiJRCQISIlcJBDoa5UEAEiLRCQISItcJBCQ6bv+///MzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMxJO2YQD4aCAAAASIPsKEiJbCQgSI1sJCBIicFIweAQSA3//wcASMH4E0jB4ANIOcF1 +CkiLbCQgSIPEKMNIiUwkGOg6QwIASItEJBhIiUQkEOgrQwIASI0FaLQFALscAAAA6DpMAgBIi0Qk +EOiQSgIA6ItFAgDohkMCAEiNBWadBQC7EgAAAOiVKgIAkEiJRCQI6KqUBABIi0QkCA8fRAAA6Vv/ +///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdh9Ig+wYSIlsJBBIjWwkEDHb6OUHAABI +i2wkEEiDxBjDSIlEJAjoUZQEAEiLRCQI68rMzMzMzMzMzMzMSTtmEA+GgAEAAEiD7EBIiWwkOEiN +bCQ4SYtWMIuyCAEAAIX2D4xOAQAA/8aJsggBAAC6AQAAAIcQhdJ0IUiJRCRIgz0jEQ4AAb4AAAAA +vwQAAABID0/3SIl0JCDrKkiLbCQ4SIPEQMO7AgAAAEjHwf/////ox+0BAEiLRCRISIt0JCC6AgAA +AIlUJBwxyetKMcnrfLsCAAAAhxiF23XJkOt5SIlMJCjHBCQeAAAA6K2TBABFD1f/ZEyLNCX4//// +SItEJChIjUgBSItEJEiLVCQcSItcJCBIid5IOfF8c5DrrkiJTCQw6LS0BABFD1f/ZEyLNCX4//// +SItEJDBIjUgBSItEJEiLVCQcSIt0JCBIg/kBfBLpef///0iLbCQ4SIPEQMNIidhIgzgAdbRIicMx +wPAPsRNAD5THDx9EAABAhP904EiLbCQ4SIPEQMNIidhIgzgAD4VG////SInDMcDwD7ETQA+Ux0CE +/3ThSItsJDhIg8RAw0iNBTurBQC7GQAAAOiXKAIAkEiJRCQI6KySBABIi0QkCOli/v//zMxJO2YQ +dh1Ig+wQSIlsJAhIjWwkCOhHBgAASItsJAhIg8QQw0iJRCQI6HOSBABIi0QkCOvMzMzMzMzMzMzM +zMzMSTtmEA+GggAAAEiD7BhIiWwkEEiNbCQQMcmHCA8fQACFyXRWg/kCdQq7AQAAAOgN7QEASYtO +MP+JCAEAAEmLTjCLiQgBAACFyXwedRJBgL6xAAAAAHQIScdGEN76//9Ii2wkEEiDxBjDSI0FiK4F +ALsbAAAA6MYnAgBIjQUDpwUAuxcAAADotScCAJBIiUQkCOjKkQQASItEJAgPH0QAAOlb////zMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEHZ6SIPsIEiJbCQYSI1sJBi7AQAAAIcYhdt1FLsB +AAAA6FfsAQBIi2wkGEiDxCDDiVwkFOikPwIASI0FqbAFALscAAAA6LNIAgCLRCQUicDoqEUCAEiN +BUCEBQC7AgAAAOiXSAIA6PI/AgBIjQUMqwUAuxoAAADoAScCAJBIiUQkCOgWkQQASItEJAjpbP// +/8zMzMzMzMzMzMzMzEk7ZhAPhskAAABIg+wwSIlsJChIjWwkKEmLVjBMifaQSDkyD4WYAAAASIlE +JDhIiXQkIEiLFW4ACwBIgzoASMfB/////7qAlpgASA9FykiJTCQY6xpIi3QkIEiLVjDGghUBAAAA +SItEJDhIi0wkGIsQhdJ1QkiLVjDGghUBAAABMdvoeeoBAEiLFRoACwBIixJIhdJ0v0iJFCRIx0Qk +CAAAAADo2JAEAEUPV/9kTIs0Jfj////rnkiLbCQoSIPEMMNIjQX6mgUAuxMAAADoDiYCAJBIiUQk +COgjkAQASItEJAjpGf///8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxIg+w4SIlsJDBIjWwkMEiJ +RCRATIl0JChIhdt9HkiLFYT/CgBIgzoAuoCWmABID0XaSIlcJCDpKAEAAIsIDx8Ahcl1KkiJXCRI +6DKrBABFD1f/ZEyLNCX4////SIsEJEiLXCRISAHYSIlEJBjrGLgBAAAASItsJDBIg8Q4w0iJy0gp +w0iJyEiLFRv/CgBIgzoAdA5IgfuAlpgAfgW7gJaYAEiLVCQoSItyMMaGFQEAAAFIi0QkQEiJ2THb +Dx9EAADoO+kBAEiLFdz+CgBIixJIhdJ0H0iJFCRIx0QkCAAAAADomo8EAEUPV/9kTIs0Jfj///9I +i0wkKEiLUTDGghUBAAAASItUJECLGoXbdSqQ6GyqBABFD1f/ZEyLNCX4////SIsEJEiLTCQYSDnI +D4xK////SItUJECLCoXJD5XASItsJDBIg8Q4w0iLVCQoSItyMMaGFQEAAABIi0QkQEiLXCQgixCF +0nVMSItUJChIi3IwxoYVAQAAAUiJ2THbZpDoe+gBAEiLFRz+CgBIixJIhdJ0tUiJFCRIx0QkCAAA +AADo2o4EAEUPV/9kTIs0Jfj////rlLgBAAAASItsJDBIg8Q4w8zMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMxJO2YQdkpIg+wYSIlsJBBIjWwkEEmLTjBMifIPH0QAAEg5EXQKSIO5AAEAAAB1D+js +/f//SItsJBBIg8QYw0iNBSKbBQC7FAAAAOixIwIAkEiJRCQISIlcJBDowY0EAEiLRCQISItcJBDr +lczMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2aUiD7DBIiWwkKEiNbCQoSYtOMEyJ8g8fRAAASDkR +dDhIiVwkGEiJRCQg6GzCAgBIi0QkIEiLXCQYZpDoW/3//4hEJBfo8nkEAA+2RCQXSItsJChIg8Qw +w0iNBYuUBQC7EQAAAOgSIwIAkEiJRCQISIlcJBDoIo0EAEiLRCQISItcJBDpc////8zMzMzMzMzM +zMzMzMzMzMzMzMxIg+wYSIlsJBBIjWwkEEiFwHRLSD3oAwAAdC1IixVO/goASIsNT/4KAEg5yHNG +SMHgBEiLDAJIi1wCCEiJyEiLbCQQSIPEGMNIjQVRgAUAuwQAAABIi2wkEEiDxBjDSI0FdoIFALsH +AAAASItsJBBIg8QYw+jnkwQAkMzMzMzMzEk7ZhB2HUiD7BBIiWwkCEiNbCQI6Cf4//9Ii2wkCEiD +xBDDSIlEJAhIiVwkEOhOjAQASItEJAhIi1wkEOvCzMxJO2YQdh1Ig+wQSIlsJAhIjWwkCOjH+f// +SItsJAhIg8QQw0iJRCQI6BOMBABIi0QkCOvMzMzMzMzMzMzMzMzMSTtmEA+GBQMAAEiD7CBIiWwk +GEiNbCQYZoM9hO0KABAPhdcCAADoFf0BADHA6x1IjRRASI01Zu0KAA+3PEZMjQXLMA4AQYk80Ej/ +wEiD+ER83UiLFS8KDgAPH4AAAAAASIXSD4SDAgAASIH6AAAIAA+HCgIAAGYPH4QAAAAAAJBIgfoA +EAAAD4J/AQAASI1a/0iF2g+FJgEAAEiLFd8JDgBIjVr/SIXaD4XBAAAASIH6AABAAHYLSMcFvgkO +AAAAAABIgz22CQ4AAHV8SI0F7Z0MAOhITAEA6ONsAABIiQVMCQ4AuH8AAADrQ0iJRCQQSI0FgQcO +AOjkiwAASItMJBBIicpIweEoSLsAAAAAwAAAAEgJ2UiJCEiLDWGfDQBIiUgQSIkFVp8NAEiNQv9I +hcB9uEiLbCQYSIPEIMNIjVEBSIkVMAkOAEiLDSkJDgBIg/lASBnSuwEAAABI0+NIIdNIOR0YCQ4A +ddPpXf///+jMOAIASI0Fe58FALsXAAAA6NtBAgBIiwX0CA4A6M8+AgBIjQXDnAUAuxcAAAAPHwDo +u0ECAOgWOQIASI0Fz6EFALsZAAAA6CUgAgAPH0QAAOh7OAIASI0Fj5MFALsSAAAA6IpBAgBIiwWr +CA4ADx8A6Hs+AgBIjQVvnAUAuxcAAADoakECAOjFOAIASI0FVZYFALsUAAAA6NQfAgDoLzgCAEiN +BUOTBQC7EgAAAA8fAOg7QQIASIsFXAgOAOgvPgIASI0FurkFALslAAAADx8A6BtBAgC4ABAAAOgR +PwIASI0FqXwFALsCAAAADx9EAADo+0ACAOhWOAIASI0F5pUFALsUAAAA6GUfAgAPH0QAAOi7NwIA +SI0Fz5IFALsSAAAA6MpAAgBIiwXrBw4ADx8A6Ls9AgBIjQW6twUAuyQAAADoqkACALgAAAgADx9E +AADomz4CAEiNBTN8BQC7AgAAAOiKQAIA6OU3AgBIjQV1lQUAuxQAAADo9B4CAEiNBXGsBQC7HgAA +AOjjHgIASI0Fso8FALsRAAAA6NIeAgCQ6OyIBADp5/z//8zMzMzMzMxMjWQk8E07ZhAPhmEGAABI +geyQAAAASImsJIgAAABIjawkiAAAAEiJhCSYAAAAhACQSI2QyAEBAEiBw////wNIgeMAAAD8SIlc +JCC5AAAABEiNPfkcDgBIidDowRoAAJBIhcB1EkiLlCSYAAAASItcJCDpxQIAAEiLTCQgSImEJIAA +AABIiUwkYEiJwki+AAAAAACAAABIAcZIwe4aTIuEJJgAAADrJ0yLTCRYTokMwkyLVCR4SYcCSf/B +SIuUJIAAAABIi0wkYEmJ8EyJzkiJ0EgBykm5//////9/AABMAcpIweoaSDnWD4eoAQAASIl0JFhJ +i5CYAQEASIXSdVC4AAAAArsIAAAAMcnoMBYAAEiFwA+E3QEAAEiLtCSYAAAASInCSIeGmAEBAEiL +hCSAAAAASItMJGBIi3QkWEyLhCSYAAAASbn//////38AAIQCSIH+AABAAA+DiwEAAEyLFPJIjRTy +TYXSD4VpAQAASIlUJHhJjYCgAQEAuxAMIQC5CAAAAEiNPSwcDgDojxkAAEiFwHUfuBAMIQC7CAAA +AEiNDREcDgDolBUAAEiFwA+EEgEAAEiLtCSYAAAATIuG+AEBAEw5hvABAQAPhZUAAABIiUQkMEnB +4ARNhcB1B0yLBYIFDgBMiUQkKEyJwLsIAAAASI0NvhsOAOhBFQAAkEiFwA+ErQAAAEiLtCSYAAAA +SIue6AEBAEyLhvABAQBIiYboAQEATItMJChJwekDTImO+AEBAEiLjvABAQBMOcFJD0/ISIuG6AEB +AEg52HQRSMHhA+gHlwQASIu0JJgAAABIi0QkMEyLhvABAQBIi5b4AQEASY1IAUg5ynI0SImO8AEB +AEiLlugBAQBJOcgPghP+///rE0iJy0iLrCSIAAAASIHEkAAAAMNMicDoj40EAOgKjgQASI0Fg7IF +ALsiAAAA6PkbAgBIjQXQvgUAuywAAADo6BsCAEiNBU+dBQC7GQAAAOjXGwIASInwuQAAQADoao0E +AEiNBXu4BQC7JwAAAOi5GwIATYtREEyJksABAQCQTIuS6GkBAEwrkrhpAQBMiZLoaQEATIuS0GkB +AE2JEUyJitBpAQBMiehMi4rAAQEATYXJD4QBAQAATIlMJFBJiwFFD7ZRCEmJw0gp2E2F0kwPRdhO +jRQbTTnTdg5JvP//////fwAAMcDrcEm8//////9/AABPjSwUScHtGkmB/QAAQAByBDHA61FMiVwk +OEyJVCRgTInYMcm/IgAAAL7/////RTHA6AOM//8PHwBIhdt0AjHASIuUJJgAAABIi1wkIEyLTCRQ +TItUJGBMi1wkOEm8//////9/AABJicVMOdh0RE2F7Q+ECf///0yJrCSAAAAATInoMcnosIEAAEiL +lCSYAAAASItcJCBMi0wkUEm8//////9/AABMi6wkgAAAAOnO/v//QQ+2cQhIhfZND0XTTYkRSInY +6xNJvP//////fwAASYnFSInYMduQSIXbD4WwAAAASInDuQAAAAQxwOgoAgAADx+EAAAAAABIhcAP +hCsBAABIiYQkgAAAAEiJXCRgSIuMJJgAAABIjYG4aQEASIlEJHDoEYUAAEiLjCSAAAAASIkIxkAI +AUiLDZuYDQBIiUgQSIkFkJgNAEiLRCRw6OaEAABMi6wkgAAAAEyJ6UiLXCRgSY1UHQBIiRBIixVn +mA0ASIlQEEiJBVyYDQBIi5QkmAAAAEm8//////9/AABJic1Mie5OjQQrTTnFdhlJuQAAAAAAgAAA +QbocAAAATI0dDKMFAOtaSbkAAAAAAIAAAE+NFClJweoaSYH6AABAAHIPQbohAAAATI0d36wFAOsw +T40UBEnB6hpJgfoAAEAAQboAAAAAQbsgAAAATQ9D00G7AAAAAEyNPU2qBQBND0PfTYXSdTwPHwBJ +98X///8DdR9IifBIidnpyPr//zHAMdtIi6wkiAAAAEiBxJAAAADDSI0FPK0FALshAAAA6OcYAgBM +iWwkQEyJRCRgTIlcJGhMiVQkSOguMQIASI0Ft60FALshAAAAZpDoOzoCAEiLRCRA6JE4AgBIjQXL +dQUAuwIAAAAPH0QAAOgbOgIASItEJGDocTgCAEiNBRynBQC7HwAAAA8fRAAA6Ps5AgBIi0QkaEiL +XCRI6Ow5AgDoRzMCAOhCMQIASI0Fc7wFALsuAAAA6FEYAgCQSIlEJAhIiVwkEOhhggQASItEJAhI +i1wkEOly+f//zMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+G8AAAAEiD7EBIiWwkOEiNbCQ4SIlMJFhI +iVwkUEiNFAtIiVQkML8iAAAAvv////9FMcBIidMxyQ8fAOjbiP//SIXbugAAAABID0XCSYnBSIXA +D4SOAAAASItcJFhIjVP/SIXQdG5IiUQkKJBIjRQYSI1S/0j320gh00iJXCQgSCnDMclMicjoj34A +AEiLRCQgSItcJFBIjRQDSIt0JChIAd5Ii3wkWEgB/kgp1kiF9nYYSInQSInzMcmQ6Ft+AABIi0Qk +IEiLXCRQSItsJDhIg8RAw0yJyEiLXCQwSItsJDhIg8RAw0iJ0DHbSItsJDhIg8RAw0iJRCQISIlc +JBBIiUwkGOgygQQASItEJAhIi1wkEEiLTCQYDx8A6dv+///MzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMxJO2YQD4bpAQAASIPsQEiJbCQ4SI1sJDiEAA+20w8fAEiB+ogAAAAPg7gBAABIiVQkMIhc +JFBIiUQkSEiLTNAoSIlMJChIicjoczgAAEiLTCQoSItROEg50HVUSIlUJCAPt0lgSDnKD4UJAQAA +SIlEJBhIi0QkSA+2XCRQDx9AAOibZAAASItMJDBIi1QkSEiLRMooSIlEJCjoIjgAAEiLVCQgSItc +JBhIi0wkKOsDSInDSDlBOA+GqQAAAEiLcWhID6/GSANBGA+3cWD/xmaJcWAPt/5Mi0E4STn4chVI +OdMPlMJIicuJ0UiLbCQ4SIPEQMNmiXQkFkyJRCQgDx8A6DsuAgBIjQVyggUAuw4AAADoSjcCAA+3 +RCQWD7fAZpDoOzQCAEiNBUt8BQC7CwAAAOgqNwIASItEJCAPH0QAAOgbNAIA6HYwAgDocS4CAEiN +BS2UBQC7FwAAAA8fRAAA6HsVAgBIjQUKkAUAuxYAAADoahUCAGaJTCQWDx9EAADouy0CAEiNBeCT +BQC7FwAAAOjKNgIAD7dEJBYPH0QAAOi7MwIASI0Fy3sFALsLAAAA6Ko2AgBIi0QkIA8fRAAA6Jsz +AgDo9i8CAOjxLQIASI0FzbwFALsxAAAADx9EAADo+xQCAEiJ0LmIAAAA6G6GBACQSIlEJAiIXCQQ +Dx9AAOj7fgQASItEJAgPtlwkEOns/f//zMzMzMzMzMzMzMzMSTtmEA+GzwgAAEiD7HhIiWwkcEiN +bCRwgz3l+w0AApAPhKEIAABIhcAPhB0IAACAPcb/DQAAD4SHAAAAgz3F/w0AAHReSIXbdDtIqQcA +AAB1DroIAAAA6zEPH4AAAAAASKkDAAAAdQe6BAAAAOsbSKkBAAAAugEAAAC+AgAAAEgPRNbrBboQ +AAAASInTSI0NLBMOAOinDAAASItsJHBIg8R4w4A9lv0NAAB0F0iLFZX9DQBJOZaYAAAAdQdI/wWN +/Q0AiIwkkAAAAEiJnCSIAAAASImEJIAAAACDPRv7DQAAdGFJi1YwSIuSwAAAAEiF0nUKTIl0JGBI +i1QkYEiLsoABAABIKcZIibKAAQAASIX2fTNIiVQkaEiJ0A8fRAAA6Du5AABIi4QkgAAAAA+2jCSQ +AAAASItUJGhIi5wkiAAAAOsCMdJJi3YwkP+GCAEAAEmLdjBNifBBhACDvvAAAAAAD4VOBwAATYnw +TDlGUA+EMAcAAMeG8AAAAAEAAABNi0YwTYuA0AAAAE2FwHQGTYtAQOsLTIsFdPsNAA8fQABNhcAP +hOkGAABIhdt0C0iDewgAQQ+UwesGQbkBAAAASIl0JFBMiUQkQESITCQmSIlUJGBIPQCAAAAPh8MD +AACQRYTJD4SpAQAASIP4EA+DnwEAAE2LUBhIqQcAAAB1CkmDwgdJg+L46yFIqQMAAAB1CkmDwgNJ +g+L86w8PuuAAkHIIkEn/wkmD4v5OjRwQSYP7EHdQTYtgEE2F5HRHS40EFE2JWBhJ/0Agx4bwAAAA +AAAAAJCLjggBAACNUf+JlggBAACD+QF1EkGAvrEAAAAAdAhJx0YQ3vr//0iLbCRwSIPEeMNNi1BQ +TYtaQE0PvONBvUAAAABND0TlZg8fRAAASYP8QH1vTYtqME+NfCUASInXSYtSOEw5+nZdT41sJQBN +jW0BSffFPwAAAHUJTDnqdAQx0utESY1UJAFIg/pATRnkSInRSdPrTSHjTYlaQE2JajBBD7dSYP/C +ZkGJUmBJi1JoSQ+v10kDUhgPtowkkAAAAOsFSInXMdJIhdJ0BUUx2+tDTInAuwUAAADor/r//0iL +dCRQSIt8JGBMi0QkQEQPtkwkJkiJwkmJ2kGJy0iLhCSAAAAAD7aMJJAAAABIi5wkiAAAAEQPETpJ +OUAYdwdJg3gQAHUNSYnUSYlQEEmJQBjrA0mJ1LoQAAAA6QcCAABIPfgDAAB3I0yNUAdJweoDSYH6 +gQAAAA+D1gQAAEyNHSrcCgBHD7YUGuskTI2Qf/z//0nB6gdJgfr5AAAAD4OiBAAATI0dRN0KAEcP +thQaSYP6RA+DfwQAAEyNHY7cCgBHD7ccU5BB0eJFD7bhRQniRQ+24kmB/IgAAAAPg0oEAABPi2Tg +KE2LbCRATQ+8/UiJ17pAAAAATA9E+kmD/0APjZUAAABJi1QkMEqNNDpNi0QkOEk58HZ4So0UOkiN +UgFI98I/AAAAdRgPH0QAAEk50HQOSIt0JFBMi0QkQDHS61tNjUcBSYP4QE0Z/0yJwUnT7U0h/U2J +bCRASYlUJDBBD7dUJGD/wmZBiVQkYEmLVCRoSA+v1kkDVCQYD7aMJJAAAABIi3QkUEyLRCRAkOsM +SIt0JFBMi0QkQDHSTIlcJEhIhdJ0BUUx0utGTInARInT6Pj4//9Ii3QkUEiLfCRgTItEJEBED7ZM +JCZMi1wkSEiJwkmJ3EGJykiLhCSAAAAAD7aMJJAAAABIi5wkiAAAAITJdGRBgHwkZAB0XEiJVCRY +RIhUJCVMiWQkKEiJ0EyJ2+hXhwQASIuEJIAAAAAPtowkkAAAAEiLVCRYSIucJIgAAABIi3QkUEiL +fCRgTItEJEBED7ZMJCZED7ZUJCVMi1wkSEyLZCQoRYnTTYniSYnUSItUJEhBvQEAAACQ63OEyXQJ +RInPQYPxAesGRInPRTHJSInDRInJTInAZpDo218AAEjHQDABAAAAZsdAYAEATItgGJBIi1BoD7aM +JJAAAABIi3QkUEiLfCRgTItEJEBED7ZMJCZJicJBid1BuwEAAABIi4QkgAAAAEiLnCSIAAAATIlk +JFhIiVQkSESIXCQlRIhsJCdMiVQkKEWEyXQFRTHJ621IOR3JCgsAvkgAAABID0TGSIlEJDhIicFI +id9MieBIidPogkIAAEiLlCSIAAAASIsyTItEJDhMOcZzHEiLQghMicFJKfBJjRwASIXAuAAAAABI +D0XD6wdIi0IITInBSItcJEBIAUMISYnBSInISIlEJDhMiUwkMOhseAQARQ9X/2RMizQl+P///4M9 +PPUNAAB0G0iLXCRYSItEJChIi0wkSEiLfCQwZpDoO9sAAEiLFWT1DQBIhdJ+O0iD+gF0GkiLVCRA +SIsySItMJEhIOfFzDUgpzkiJMusgSItMJEhIi0QkUEiLXCRY6NsDAABIi0wkSOsFSItMJEhIi1Qk +UMeC8AAAAAAAAACQi7IIAQAAjX7/iboIAQAAg/4BdRJBgL6xAAAAAHQIScdGEN76//8PtlQkJ4TS +dR4PtpQkkAAAAITSdBJIichIi1wkWOiXAQAASItMJEiAPWf4DQAAdEWDPWL4DQAAZpB0GkiLRCRY +SInLSIuMJIgAAADoaakBAEiLTCRIgD199g0AAHQXSIsVfPYNAEk5lpgAAAB1B0gBDXz2DQBIi1Qk +YEiF0nQPSIt0JDhIKfFIKYqAAQAAD7ZUJCWE0nQ/gD3jGw4AAHQlgz0S9A0AAHUcgz3t8w0AAHUT +SIsVMPsNAEg5FRH7DQAPlsLrAjHShNJ0CzHAMdsxyejDegAASItEJFhIi2wkcEiDxHjDSI0FffUN +AEiLbCRwSIPEeMNMieC5iAAAAOjWfQQARInQuUQAAADoyX0EAEyJ0Ln5AAAAkOjbfQQATInQuYEA +AADozn0EAEiNBeO1BQC7NAAAAGaQ6BsMAgBIjQVngwUAuxQAAADoCgwCAEiNBT95BQC7DwAAAOj5 +CwIASI0FSbQFALsyAAAA6OgLAgCQSIlEJAhIiVwkEIhMJBjo9HUEAEiLRCQISItcJBAPtkwkGA8f +RAAA6fv2///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdn5Ig+woSIlsJCBIjWwkIEiN +DANIiUwkEOsvSCnZSIH5AAAEAL4AAAQASA9HzkiJ2EiJy+hDgwQASItMJBhIjZkAAAQASItMJBBI +OctzJ0iJXCQYQYC+sQAAAAB0vUiNBSe9BQDoknMEAEiLTCQQSItcJBjrpUiLbCQgSIPEKMNIiUQk +CEiJXCQQ6C11BABIi0QkCEiLXCQQDx8A6Vv////MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJ +O2YQditIg+wgSIlsJBhIjWwkGEiLEEiJw7kBAAAASInQ6Pn1//9Ii2wkGEiDxCDDSIlEJAjoxXQE +AEiLRCQI677MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4aDAAAASIPsIEiJbCQY +SI1sJBhIg/sBdD5IixBIicFIidhI9+JwTUi6AAAAAAAAAQBIOdB3Pg8fRAAASIXbfDRIicu5AQAA +AOhu9f//SItsJBhIg8Qgw0iLEEiJw7kBAAAASInQ6FH1//9Ii2wkGEiDxCDDSI0FAA0FAEiNHSm8 +BgDoVAICAJBIiUQkCEiJXCQQ6AR0BABIi0QkCEiLXCQQ6VX////MzMzMzMzMzMzMzMzMzMzMzMzM +zMxJO2YQD4acAAAASIPsMEiJbCQoSI1sJChJi1YwSIuS0AAAAEiF0nQGSItSQOsHSIsV2/ENAEiF +0nRVSIsFD/ENAEiD+AF1BDHA6ylIiVQkEEiJTCQYSIlcJCDogQAAAEhjyEiLVCQQSItcJCBIichI +i0wkGEiJAkiJ2EiJyw8fAOjbnwEASItsJChIg8Qww0iNBQG1BQC7OAAAAA8fRAAA6BsJAgCQSIlE +JAhIiVwkEEiJTCQY6CZzBABIi0QkCEiLXCQQSItMJBjpMv///8zMzMzMzMzMzMzMzMzMzMzMzEiD +7BhIiWwkEEiNbCQQSD0AAAAHfgq4AAAAB+sMDx8ASIXAD4QFAQAASYtWMIuaIAEAAIuyJAEAAImy +IAEAAInfweMRMfuJ9zHewesHMfOJ/sHvEDHfibokAQAAjRQ3geL///8D/8IPV8DySA8qwpBmSA9+ +wkiJ00jB6i9Ig+IfSI01udUKAPIPEATWSI16AUiD/yEPg6AAAABIidlIwes0SIHj/wcAAEiBwwH8 +//9IwekbSIHh//8PAPIPEEzWCA9X0vJIDyrT8g9Y0PIPXMgPV8DySA8qwfIPWcHyDxANZLgGAPIP +WcjyD1jR8g8QBcS4BgDyD1zQD1fAZg8u0JB2Aw9X0g9XwPJIDyrA8g8QDeu4BgDyD1nB8g9Z0PIP +LMqNQQFIi2wkEEiDxBjDMcBIi2wkEEiDxBjDSIn4uSEAAADoOXkEAJDMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMxJO2YQD4aBAAAASIPsQEiJbCQ4SI1sJDhIx0QkCAAAAABIx0QkEAAAAABIjVQkGEQP +ETpIjVQkKEQPETpIjRV9AAAASIlUJBBIiUQkGEiJXCQgSIlMJChIjUQkCEiJRCQwSI1EJBBIiQQk +6PFvBABFD1f/ZEyLNCX4////SItEJAhIi2wkOEiDxEDDSIlEJAhIiVwkEEiJTCQY6AFxBABIi0Qk +CEiLXCQQSItMJBjpTf///8zMzMzMzMzMzMzMzMxJO2YQdjtIg+woSIlsJCBIjWwkIEiLWhBIi0oY +SItyIEiJdCQYSItCCOgyAAAASItUJBhIiQJIi2wkIEiDxCiQw+j6bwQA67jMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMxJO2YYD4aNAgAASIPsQEiJbCQ4SI1sJDgPH4QAAAAAAEiFwA+EXAIAAEiF23Qg +SI1T/0iF0w+FNQIAAA8fRAAASIH7ACAAAHYK6RECAAC7CAAAAEg9AAABAA+DxgAAAEiJTCRYSIlE +JEhIiVwkEEmLVjCQ/4IIAQAASYtWMEiJVCQgSIXSdBdIi7LQAAAASIX2dAuEBkiBxlAWAADrKZCQ +SI0F6e4NAOiU2///SItEJEhIi0wkWEiLVCQgSItcJBBIjTXR7g0ASIl0JChIi34ISI08O0iNf/9I +99tIiVwkGEgh30iJfghIAcdIgf8AAAQAdwZIgz4AdT64AAAEAEiNHVsEDgDodmgAAEiLTCQoSIkB +SIM5AA+FxwAAAA8fQADpDQEAAEiJy+hTaAAASItsJDhIg8RAw0iLPkyLRghOjQwATIlOCJBEi4oI +AQAARY1R/0SJkggBAABKjRQHDx8AQYP5AXUSQYC+sQAAAAB0CEnHRhDe+v//SIlUJDBIjT0U7g0A +SDn+dR2QkEiNBf7tDQDoidz//0iLRCRISItMJFhIi1QkMEiNNbMDDgBIOfF0JEiJw0iJyOhjuQEA +SItcJEhI99tIjQWUAw4A6E+5AQBIi1QkMEiJ0EiLbCQ4SIPEQMNIixFIiwUD7Q0ASIkCSIsRSI01 +9uwNAPBID7EWD5TChNJ03UiLfCQQSIPHB0yLRCQYTCHHSIl5CEiLRCRISItUJCBIic5Ii0wkWOkF +////SI0VX+0NAEg50XQRSI0FQJQFALsfAAAA6BEEAgCQkEiNBTjtDQDow9v//+vfSI0FF5wFALsj +AAAA6PADAgBIjQVupAUAuyoAAAAPH0AA6NsDAgBIjQUJiAUAuxoAAADoygMCAJBIiUQkCEiJXCQQ +SIlMJBjoFZUEAEiLRCQISItcJBBIi0wkGOlB/f//zEk7ZhAPhrIAAABIg+wwSIlsJChIjWwkKEiL +EEiNFBFIjVL/SPfZSCHKSI00Gkg5cBByfUiJMEiLHQvsDQBMi0AISI00HkiNdv5I99tIIfNJOdhz +T4B4GAB0RUiJRCQ4SIlUJCBIiVwkGEyJwEwpw0iJ+ehpagAASItUJDhIi0IISInGSItcJBhIKfPo +cGkAAEiLRCQ4SItUJCBIi1wkGEiJWAhIidBIi2wkKEiDxDDDMcBIi2wkKEiDxDDDSIlEJAhIiVwk +EEiJTCQYSIl8JCDo62wEAEiLRCQISItcJBBIi0wkGEiLfCQg6RL////MzMzMzMzMzMzMzMzMzMzM +zMwPtlAJgPoQcl9Ji14wi7MgAQAAi7skAQAAibsgAQAAQYnwweYRQTHwif5EMcdBwegHQTH4iffB +7hBEMcaJsyQBAACNSvGA+SAZ0rsBAAAA0+Mh041T/40cPoXadQoPt0gK/8FmiUgKww+3SAr/wWaJ +SArDzMzMzMzMzMzMzMzMzEk7ZhAPhiUCAABIg+xYSIlsJFBIjWwkUEiJXCRoSIlMJHBIiUQkYEiL +UChIhdJ0f0iLchBIhfZ0dkQPt0NSSYnxSY08MEiNf/iEA0iDPwB0QJCDPefqDQAAdQlIxwcAAAAA +6wcx0ui1bwQASItQKIQCgz3I6g0AAHUKSMdCEAAAAADrSUiNehAx0uiRbwQA6zxLjTQBgz2k6g0A +AHUGSIlyEOspSI16EOizbwQA6x5Ii1NASIsCuQEAAABIidNmkOib7P//SYnBSItEJGBMiUwkQOip +/v//SItMJGhIi1FASIN6CAAPhQkBAACQSItUJGBIg3ooAHU0SI0FISMFAJDoO/b//4M9NOoNAAB1 +C0iLVCRgSIlCKOsOSItUJGBIjXoo6NltBABIi0wkaEyLQihJgzgAdTdIjQUD3QQADx8A6Pv1//9I +i0wkYEiLeSiEB4M96ekNAAB1B0iJB+sHZpDom20EAEiJykiLTCRoSItSKEyLAoQCSYtQCEmLGEmL +eBBIjXIBSDn3cz5MiUQkSEiNBUoMBQBIidHoIkkDAEiLfCRISIlPEIM9kukNAAB1BUiJB+sF6EZt +BABIi0wkaEmJ+EiJ2kiJw0iNcgFJiXAISI0804M9ZekNAAB1C0iLRCRASIkE0+sRSItEJEDoDm0E +AOsFSItEJEAPt0lSSItUJHBIjTwKSI1/+IQHkIM9LOkNAAB1CkiJB+sKDx9EAADo22wEAEiLbCRQ +SIPEWMNIiUQkCEiJXCQQSIlMJBhmkOj7aQQASItEJAhIi1wkEEiLTCQY6af9///MzMzMzMzMSTtm +EA+GeQEAAEiD7EBIiWwkOEiNbCQ4SItQQEiLEkiJxkiJ0Ej343APSLoAAAAAAAABAEg50HYHMdsP +H0QAAEiFyXUjSIl0JDBIiVwkGEiNBQo7BQDohfT//0iLXCQYSIt0JDBIicFJi1Ywi7ogAQAARIuC +JAEAAESJgiABAABBifnB5xFEMc9FicFBMfjB7wdEMcdFichBwekQQTH5RImKJAEAAEONFAiJUQwx +wOsF/8BIidFIg/sIfh9IicqJwb8BAAAASNPnSNHvTI0Ef0qNPIdIOft32OsFSInKicGIQgmEwA+E +iwAAAEiJVCQoicMxyUiJ8OjEAAAAgz3d5w0AAHULSItMJChIiUEQ6xBIi0wkKEiNeRDogmsEAGaQ +SIXbdE5IiVwkIEiNBY8gBQDoqvP//4M9o+cNAAB1FEiLTCQoSIlBKEiLVCQgSIlQEOsgSItMJChI +jXkoDx9AAOg7awQASI14EEiLVCQg6E1sBABIicpIidBIi2wkOEiDxEDDSIlEJAhIiVwkEEiJTCQY +6EloBABIi0QkCEiLXCQQSItMJBjpVf7//8zMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhhwCAABI +g+w4SIlsJDBIjWwkMEiJyonZvgEAAABI0+aA+QQPgvsAAABIi3hAg8H8QbgBAAAASdPgSQHwSIs/ +TYnBTA+vx0mB+ACAAAAPg4YAAAAPH4QAAAAAAEmB+PgDAAB3OU2NUAdJweoDSYH6gQAAAA+DlQEA +AEyNHZvICgBHD7YUGkmD+kQPg3IBAABIjQ0lyQoAQg+3DFHrXk2NkH/8//9JweoHSYH6+QAAAA+D +PwEAAEyNHZ/JCgBHD7YUGkmD+kQPgxwBAABMjR3pyAoAQw+3DFPrIkmNiAAgAABJOch2BUyJwesR +kEmNiP8fAABIgeEA4P//ZpBMOcF0GUiF/w+E2QAAAEiJw0iJyEiJ0THSSPf36xRIidFIicNMicjr +CUiJ0UiJw0iJ8EiJXCQoSIl0JBhIiUQkEEiFyXQ2SIlMJCBIi1NASIs6SA+v+EiDeggAdA1IichI +ifvoGB4AAOsLSInISIn76Gt0BABIi0QkIOsSSItLQEiJwkiJyEiJ0+jy8f//SItMJBBIi1QkGEg5 +0XQ7SIt0JCgPt3ZSSP/JSA+vzkgBwUiNPDFIjX/4hAdID6/ykEiNDDCDPVblDQAAdQVIiQfrCegK +aQQA6wIxyUiJy0iLbCQwSIPEOMPoFNcBAEyJ0LlEAAAA6IdtBABMidC5+QAAAOiabQQATInQuUQA +AADobW0EAEyJ0LmBAAAADx9EAADoe20EAJBIiUQkCIhcJBBIiUwkGOjnZQQASItEJAgPtlwkEEiL +TCQY6bP9///MzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+G/AEAAEiD7DhIiWwkMEiNbCQwDx+EAAAA +AABIhdsPhLkAAABIgzsAD4SvAAAAD7ZzCGYPH4QAAAAAAED2xgQPhaUBAABIiUQkQEiJXCRISIlM +JChIi1BISIsyi3sMSInISIn7/9ZIi0wkSA+2cQlIicqJ8b8BAAAASNPnSI1P/0iJzkghwUiLfCRA +RA+3R1JJD6/ITItKGEgDShBNhcl0K0QPtlIIQfbCCHUDSNHuSCHGTA+vxkuNNAFHD7YEAUGDwP5B +gPgDcgNIifFIweg4PAVzA4PABYhEJBfrPotwVA+65gRzDkiLUEhIizJIicgx2//WSI0FAPUNADHb +SItsJDBIg8Q4ww+3d1JIjTQxSI12+EiLDg8fRAAASIXJdAlIiUwkIDHS6xZIjQXL9A0AMdtIi2wk +MEiDxDjDSP/CSIP6CHPCD7Y0CkA48HQHQIT2denr1EiJVCQYD7Z3UEgPr/JIjQwOSI1JCIt3VA+6 +5gBzA0iLCUiLdzBIi1YYSIsySItEJChIicv/1oTAdRYPtkQkF0iLTCQgSItUJBhIi3wkQOuXSItM +JEAPtlFQD7ZxUUiLfCQYSA+v90iNFNZIi3QkIEiNFBZIjVIIi0lUD7rhAXMDSIsSSInQuwEAAABI +i2wkMEiDxDjDSI0FUI0FALshAAAADx9EAADom/kBAJBIiUQkCEiJXCQQSIlMJBjopmMEAEiLRCQI +SItcJBBIi0wkGOnS/f//zMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+G4AQAAEiD7HhIiWwkcEiNbCRw +Dx+EAAAAAABIhdsPhK0EAAAPtnMIQPbGBA+FjgQAAEiJnCSIAAAASImEJIAAAABIiUwkaEiLUEhI +izKLewxIichIifv/1kiJRCQoSIucJIgAAAAPtksIg/EEiEsISIN7EAB1RkiLlCSAAAAASItaQEiL +A7kBAAAA6Avk//+DPeThDQAAdQ5Ii5wkiAAAAEiJQxDrEUiLnCSIAAAASI17EOiDZQQASItEJChI +i7QkgAAAAOsoSInw6EwEAABIi5wkiAAAAEiLhCSAAAAASItMJChIichIi7QkgAAAAA+2Swm/AQAA +AEjT50iNT/9IIcFIg3sYAHQnSIlMJDBIifDo5QUAAEiLRCQoSItMJDBIi5wkiAAAAEiLtCSAAAAA +D7d+UkgPr89IA0sQSInHSMHoODwFcwODwAWIRCQfMdJFMcBFMckPH0QAAOl7AAAATInZSIN7GAAP +heEBAABMixNJ/8JED7ZbCWaQSYP6CH4nSYnMRInZQb0BAAAASdPlSdHtT418bQBPjWy9AE051Q+C +If///+sGSYnMRInZRA+3UwqA+Q92BbkPAAAAg+EPQbsBAAAAQdPjZkU52g+D9P7//+l6AQAATInR +SIlMJFhFMdLrI0QPt1ZSSYnLTo0UEU2NUvhNixIPHwBNhdJ12Olb////Sf/CSYP6CHPXhAFGD7Yc +EU6NJBFmkEQ42HRPQYD7AXc/SIXSdTpED7ZuUE2J100Pr9VIifoPtn5RSQ+v/0qNPO9Jic1NjQQK +TY1ACEyNDA9NjUkITInpSInXTYn6TIniRYTbdZvp9P7//0yJVCQgTIlMJFBMiUQkQEiJVCRIRA+2 +XlBND6/aSY0MC0iNSQhEi15UQQ+64wBzA0iLCUiJTCQ4SIt+MEiLVxhIizpIi0QkaEiJy5D/14TA +dTgPtkQkH0iLTCRYSIucJIgAAABIi7QkgAAAAEiLfCQoTItUJCBIi1QkSEyLRCRATItMJFDpDP// +/0iLlCSAAAAAi3JUD7rmA3MeSItCMEiLXCQ4SItMJGgPHwDo2xUAAEiLlCSAAAAAD7ZKUA+2clFI +i3wkIEgPr/dIjQzOSIt0JFhIjQwOSI1JCOktAQAASYnMSIXSdTxIidhIifNMieHo1vP//4QAkEiL +lCSAAAAAD7ZyUEyNQAhMjQzwTY1JCEiLnCSIAAAASInWSInCD7ZEJB9MiUwkUEiJVCRIi35UD7rn +AHNWTIlEJGBIi14wSIsDuQEAAAAPH0AA6Nvg//9Ii3wkYIQHgz2t3g0AAHUFSIkH6wXoYWIEAEiL +VCRISIucJIgAAABIi7QkgAAAAEyLTCRQSYnAD7ZEJB+LflQPuucBc0JMiUQkQEiLXjhIiwO5AQAA +AA8fRAAA6Hvg//9Ii3wkUIQHgz1N3g0AAHUFSIkH6wXoAWIEAEiLtCSAAAAATItEJEBIi0YwTInD +SItMJGjooxQAAA+2VCQfSIt0JEiIFkiLlCSIAAAASP8CSIuUJIAAAABIi0wkUEiLtCSIAAAAD7Z+ +CED2xwR0IIPn+0CIfgiLUlQPuuIBcwNIiwlIichIi2wkcEiDxHjDSI0FZm0FALsVAAAAkOib9AEA +SI0FVG0FALsVAAAA6Ir0AQBIjQWD9wQASI0dvKYGAOjX7AEAkEiJRCQISIlcJBBIiUwkGOiCXgQA +SItEJAhIi1wkEEiLTCQY6e76///MzMzMzMzMzMzMzMzMzEk7ZhAPhq0BAABIg+w4SIlsJDBIjWwk +MEiLE0j/wg+2SwlIg/oIfiS+AQAAAEjT5kjR7kiNPHZIjTS+Dx9EAABIOdZzB7kBAAAA6wwPtlMI +g8oIiFMIMclIiVwkSIhMJB9Ii1MQSIlUJCAPtnMJAc6J8zHJ6Mf1//9Ii1QkSA+2cgiJ94Pm/ED2 +xwF0A4POAg+2SglED7ZEJB9EAcGISglAiHIIgz213A0AAHUPSItMJCBIiUoYSIlCEOsXSI16GEiL +TCQg6FZhBABIjXoQ6E1gBABIx0IgAAAAAGbHQgoAAEiLSihIhcl0SkiLMUiF9nRCSIN5CAAPhagA +AACDPVzcDQAAdQZIiXEI6wlIjXkI6GthBABIi3oohAeDPT7cDQAAdQlIxwcAAAAA6wcxyejsYAQA +SIXbdGJIg3ooAGaQdTtIiVwkKEiNBfIUBQDoDej//4M9BtwNAAB1C0iLTCRISIlBKOsOSItMJEhI +jXko6KtfBABIicpIi1wkKEiLQiiEAIM91tsNAAB1BkiJWBDrCUiNeBDoxWAEAEiLbCQwSIPEOMNI +jQXEbQUAuxYAAADoivIBAJBIiUQkCEiJXCQQ6JpcBABIi0QkCEiLXCQQ6Sv+///MzMzMzMzMzMzM +zEk7ZhB2akiD7ChIiWwkIEiNbCQgD7ZTCQ+2cwgPH0AAQPbGCHUC/8pIiVwkOEiJRCQYSInOidG/ +AQAAAEjT50iNT/9IIfGQ6FQAAABIi1wkOEiDexgAdA5Ii0sgSItEJBjoOgAAAEiLbCQgSIPEKMNI +iUQkCEiJXCQQSIlMJBiQ6PtbBABIi0QkCEiLXCQQSItMJBjpZ////8zMzMzMzMxMjWQk0E07ZhAP +hlsFAABIgeywAAAASImsJKgAAABIjawkqAAAAA+3cFJID6/xSANzGEQPtkMJRA+2SwgPH0AAQfbB +CHUDQf/ISInKRInBQbkBAAAASdPhRA+2FkGDwv5BgPoDD4K/AAAATI1UJGhFDxE6TI1UJHhFDxE6 +TI2UJIgAAABFDxE6TI2UJJgAAABFDxE6RA+3UFJMD6/STANTEEyJVCRoTY1aCEyJXCR4RA+2WFBP +jRTaTY1SCEyJlCSAAAAARA+2UwhB9sIIdTpNjRQRRA+3WFJND6/TTANTEEyJlCSIAAAATY1aCEyJ +nCSYAAAARA+2WFBPjRTaTY1SCEyJlCSgAAAASIlUJECITCQfSImEJLgAAABIiZwkwAAAAEyJTCQ4 +6zxIOVMgdRZMiclIicJIidhIidMPH0QAAOhbBAAASIusJKgAAABIgcSwAAAAw0QPt1BSTo0UFk2N +UvhJizJIhfZ0G0iJdCRgkEQPtlBQTI1eCE6NFNZNjVIIMf/rag+2cwhA9sYCdZpIi3BASIN+CAB0 +jw+3SFJIic5ID6/KkEiDxvhIA0sYSIPBCEiJyEiJ8+iDEQAASIuEJLgAAABIi1QkQEiLnCTAAAAA +TItMJDjpTf///0j/x0QPtmBQRA+2aFFNAeNNAepIg/8ID41c////RA+2JD5mkEGA/AF3BsYEPgTr +0EGA/AUPgkIDAABEi2hUQQ+65QCQcwVNiyvrA02J3UiJfCQgTIlUJFhMiWwkSEyJXCRQRA+2ewhB +9scIdAhFMf/p6wAAAESIZCQdSItQSEiLCotzDEyJ6EiJ8//RSIuMJMAAAAAPtnEIQPbGAXRLSIuc +JLgAAACLc1QPuuYCcwQx0utASIlEJChIi0swSItRGEiLCkiLRCRISInD/9GD8AFIi4wkwAAAAEiL +nCS4AAAAicJIi0QkKOsKSIucJLgAAAAx0oTSdBVIweg4PAVzA4PABQ+2dCQdg+YB6xNED7ZEJB9M +D6PAQA+Sxg+2RCQdD7ZMJB9Ii1QkQEiLnCTAAAAASIt8JCBBichMi0wkOEyLVCRYTItcJFBMi2wk +SEGJ90GJxEiLhCS4AAAASIt0JGBFjU8CRIgMPkUPts9Jg/kCD4P5AQAAScHhBUyJTCQwTot8DHBJ +g/8ID4WPAAAARIhkJB5Ki0wMaEiJwkiJ2EiJ0w8fAOgb7P//SItUJDBIiUQUaEjHRBRwAAAAAEiN +cAhIiXQUeEiLnCS4AAAAD7ZzUEiNNPBIjXYISIm0FIAAAABIidgPtkwkH0iLVCRASIucJMAAAABI +i3QkYEiLfCQgQYnITItMJDBMi1QkWEyLXCRQRA+2ZCQeTItsJEhOi3wMaEGEB06LRAxwSYPgB0eI +JAdEi0BUQQ+64ABzNE6LRAx4QYQAgz241g0AAHUFTYko62lJifxMicdJifdMie4PH0QAAOi7WwQA +TIn+TInn60tIi1AwSotcDHhMidlIidAPH0AA6PsMAABIi4QkuAAAAA+2TCQfSItUJEBIi5wkwAAA +AEiLdCRgSIt8JCBMi0wkMEyLVCRYTItcJFBEi0BUQQ+64AFzNU6LhAyAAAAAQYQATYsigz0o1g0A +AHUFTYkg62NJif1MicdJifdMieboMFsEAEyJ/kyJ7+tKSItQOEqLnAyAAAAATInRSInQ6HEMAABI +i4QkuAAAAA+2TCQfSItUJEBIi5wkwAAAAEiLdCRgSIt8JCBMi0wkMEyLVCRYTItcJFBK/0QMcEQP +tkBQTgFEDHhED7ZAUU4BhAyAAAAAQYnITItMJDjpkfz//0yJyLkCAAAA6OhdBABIjQVAVwUAuw0A +AADoV+wBAJBIiUQkCEiJXCQQSIlMJBjoYlYEAEiLRCQISItcJBBIi0wkGOlu+v//zMzMzMzMzMzM +zMzMzMxJO2YQD4a5AAAASIPsCEiJLCRIjSwkSItQIEiNcgFIiXAgSIHCAQQAAEg50UgPQtHrB0j/ +xkiJcCBIi3AgDx8ASDnWdCFED7dDUkwPr8aQTANAGEUPtgBBg8D+Dx9EAABBgPgDcsxIOfF1T4M9 +ztQNAAB1CkjHQBgAAAAA6wtIjXgYMcnod1kEAEiLSChIhcl0HoM9p9QNAAB1CkjHQQgAAAAA6wtI +jXkIMcnoUFkEAA+2SAiD4feISAhIiywkSIPECMNIiUQkCEiJXCQQSIlMJBjoaVUEAEiLRCQISItc +JBBIi0wkGOkV////zMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GYAEAAEiD7BhIiWwkEEiNbCQQ +iUwkMA8fQABIhdsPhLQAAABIgzsAD4SqAAAAD7ZLCGYPH4QAAAAAAPbBBA+FDwEAAIB7CQB1CUiL +SxDpqAAAAEiJRCQgSIlcJChIi1BISIsKi3MMSI1EJDBIifP/0UiLTCQoD7ZxCUiJyonxvwEAAABI +0+dIjU//SInOSCHBSIt8JCBED7dHUkkPr8hMi0oYSANKEE2FyXQnD7ZSCPbCCHUDSNHuSCHGTA+v +xkuNFAFDD7YcAYPD/oD7A3IDSInRSIn46yNIjQWc5A0ASItsJBBIg8QYww+3UFJIjRQRSI1S+IQA +SIsKkEiFyXQJkEiNUQgx2+sYSI0Fa+QNAEiLbCQQSIPEGMNI/8NIg8IESIP7CHPCizI5dCQwdesP +tjQLQID+AXbhD7ZQUUgPr9pIjQQLSI1AKEiLbCQQSIPEGMNIjQVnfQUAuyEAAADot+kBAJBIiUQk +CEiJXCQQiUwkGOjDUwQASItEJAhIi1wkEItMJBjpcP7//8zMzMzMzMzMzMzMzMzMzMxJO2YQD4Zt +AQAASIPsGEiJbCQQSI1sJBCJTCQwDx9AAEiF2w+EtQAAAEiDOwAPhKsAAAAPtksIZg8fhAAAAAAA +9sEED4UcAQAAgHsJAHUJSItLEOmqAAAASIlEJCBIiVwkKEiLUEhIiwqLcwxIjUQkMEiJ8//RSItM +JCgPtnEJSInKifG/AQAAAEjT50iNT/9Iic5IIcFIi3wkIEQPt0dSSQ+vyEyLShhIA0oQTYXJdCgP +tlII9sIIdQNI0e5IIcZMD6/GS40UAUMPtjQBg8b+QID+A3IDSInRSIn46yRIjQX74g0AMdtIi2wk +EEiDxBjDD7dQUkiNFBFIjVL4hABIiwpIhcl0CZBIjVEIMdvrGkiNBcniDQAx20iLbCQQSIPEGMNI +/8NIg8IESIP7CHPBizI5dCQwdesPtjQLDx9AAECA/gF23Q+2UFFID6/aSI0EC0iNQCi7AQAAAEiL +bCQQSIPEGMNIjQW6ewUAuyEAAADoCugBAJBIiUQkCEiJXCQQiUwkGOgWUgQASItEJAhIi1wkEItM +JBjpY/7//8zMzEk7ZhAPhs8CAABIg+wwSIlsJChIjWwkKIlMJEgPH0AASIXbD4ScAgAAD7ZLCPbB +BA+FfgIAAEiJXCRASIlEJDhIi1BISIsKi3MMSI1EJEhIifP/0UiJRCQYSItcJEAPtksIg/EEiEsI +SIN7EAB1PUiLVCQ4SItaQEiLA7kBAAAA6JvS//+DPXTQDQAAdQtIi1wkQEiJQxDrDkiLXCRASI17 +EOgZVAQASItEJBhIi1QkOOsfSInQ6OXy//9Ii0QkOEiLXCRASItMJBhIichIi1QkOA+2Swm+AQAA +AEjT5kiNTv9IIcFIg3sYAHQhSIlMJCBIidDoBwIAAEiLRCQYSItMJCBIi1QkOEiLXCRAD7dyUkgP +r85IA0sQMf9FMcBmkOt3TYnLSIn5TYnBSIN7GAAPhekAAABIizNI/8YPtnsJSIP+CH4lSYnIiflB +ugEAAABJ0+JJ0epPjSRST40UopBJOfIPgkr////rBUmJyIn5D7dzCoD5D3YFuQ8AAACD4Q+/AQAA +ANPnZjn+D4Mi////6YwAAABMidFFMcnrKEmJyUyNFDFNjVL4TYsSTYXSdeXpbP///0yNVwFIic9N +ichMidlNidFJg/kIc9KEAUYPthQJQYD6AXcbSIX/SYnLSA9Fz0yJz00PRchFhNJ1x+k3////SYnK +To0ciU2NWwhFixtEOVwkSHQOTYnTSIn5TInPTYnB659MidFMic7rW0mJyE2FwHUjSInYSInTTInZ +6Ivj//9Ii1QkOEiLXCRASYnARTHJSItEJBhIweg4PAVzA4PABUGEAEyJzkmD4QdDiAQITInBSY08 +sEiNfwiQRItEJEhEiQdI/wMPtlJRSA+v8g+2UwhIjQQOSI1AKPbCBHQQg+L7iFMISItsJChIg8Qw +w0iNBfZdBQC7FQAAAOgs5QEASI0F5V0FALsVAAAA6BvlAQBIjQUU6AQASI0dTZcGAOho3QEAkEiJ +RCQISIlcJBCJTCQY6BRPBABIi0QkCEiLXCQQi0wkGOkB/f//zEk7ZhB2akiD7ChIiWwkIEiNbCQg +D7ZTCQ+2cwgPH0AAQPbGCHUC/8pIiVwkOEiJRCQYSInOidG/AQAAAEjT50iNT/9IIfGQ6FQAAABI +i1wkOEiDexgAdA5Ii0sgSItEJBjoOgAAAEiLbCQgSIPEKMNIiUQkCEiJXCQQSIlMJBiQ6HtOBABI +i0QkCEiLXCQQSItMJBjpZ////8zMzMzMzMxMjWQk4E07ZhAPhosDAABIgeygAAAASImsJJgAAABI +jawkmAAAAA+3cFJID6/xSANzGA+2ewlED7ZDCA8fRAAAQfbACHUC/89IicqJ+UG4AQAAAEnT4EQP +tg5Bg8H+ZpBBgPkDD4KqAAAATI1MJFhFDxE5TI1MJGhFDxE5TI1MJHhFDxE5TI2MJIgAAABFDxE5 +RA+3SFJMD6/KTANLEEyJTCRYTY1RCEyJVCRoSYPBKEyJTCRwRA+2SwhmDx9EAABB9sEIdS5NjQwQ +RA+3UFJND6/KTANLEEyJTCR4TY1RCEyJlCSIAAAASYPBKEyJjCSQAAAASIlUJCiITCQfSImcJLAA +AABIiYQkqAAAAEyJRCQ46zdIOVMgdRFMicFIicJIidhIidPo9fb//0iLrCSYAAAASIHEoAAAAMNE +D7dIUk6NDA5NjUn4SYsxSIX2dBRIiXQkUJCQTI1OCEyNVihFMdvrZg+2cwhA9sYCdaZIi3BASIN+ +CAB0mw+3SFJIic5ID6/KkEiDxvhIA0sYSIPBCEiJyEiJ8+gkBAAASIuEJKgAAABIi1QkKEiLnCSw +AAAATItEJDjpWf///0n/w0QPtmBRSYPBBE0B4kmD+wgPjWf///9GD7YkHkGA/AF3B0LGBB4E69VB +gPwFD4KYAQAATIlcJCBEiGQkHkyJVCRITIlMJEBED7ZrCEH2xQh0BUUx7etYSItQSEiLCotzDEyJ +yEiJ8//RD7ZMJB9ID6PIQA+SxkiLhCSoAAAASItUJChIi5wksAAAAInPTItEJDhMi0wkQEyLVCRI +TItcJCBED7ZkJB5BifVIi3QkUEWNfQJGiDweRQ+27Q8fhAAAAAAASYP9Ag+D8QAAAEnB5QVMiWwk +ME6LfCxgSYP/CHVTSotMLFhIicJIidhIidPob9///0iLVCQwSIlEFFhIx0QUYAAAAABIjXAISIl0 +FGhIjXAoSIl0FHBIi4QkqAAAAEyLTCRATItUJEhED7ZkJB5JidVKi1QsWIQCSot0LGBIg+YHRIgk +MkqLVCxoQYsxiTJIi1A4SotcLHBMidFIidBmkOi7AAAASItUJDBI/0QUYEiDRBRoBEiLnCSoAAAA +D7ZzUUgBdBRwSInYD7ZMJB9Ii1QkKEiLnCSwAAAASIt0JFCJz0yLRCQ4TItMJEBMi1QkSEyLXCQg +6UD+//9Miei5AgAAAOg4UgQASI0FkEsFALsNAAAA6KfgAQCQSIlEJAhIiVwkEEiJTCQY6LJKBABI +i0QkCEiLXCQQSItMJBgPHwDpO/z//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEiD7DBIiWwk +KEiNbCQoSDnLdH5IiUQkOEiJTCRISIlcJECAPVvJDQAAdClIi1AISIXSdCBIidhIictIidEPHwDo +2wgAAEiLRCQ4SItMJEhIi1wkQEiLEEiJ2EiJy0iJ0eibWgQAgD0ZyQ0AAHQZSItEJDhIizBIi1wk +QEiLTCRIMf/o2Vj//0iLbCQoSIPEMMNIi2wkKEiDxDDDzMzMzMxIg+w4SIlsJDBIjWwkMEg5zkgP +TM5IhckPhKQAAABIiUwkKEiJXCRISIl8JFiAPbHIDQAAdCpIiUQkQEiJykiJ+UiJ1+gXWf//SItE +JEBIi0wkKEiLXCRISIt8JFgPHwBIOd90UEiLEEgPr8qAPXHIDQAAdCZIiUwkIEgp0UgDSAhIidhI +ifvo9AcAAEiLTCQgSItcJEhIi3wkWEiJ2EiJ++i6WQQASItEJChIi2wkMEiDxDjDSInISItsJDBI +g8Q4wzHASItsJDBIg8Q4w8zMzMzMzMzMzMzMzMzMzMzMzEiD7CBIiWwkGEiNbCQYgD3vxw0AAHQs +SItICA8fRAAASIXJdB5IiUQkKEiJXCQwSInYMdvoZwcAAEiLRCQoSItcJDBIiwhIidhIicvob1YE +AEiLbCQYSIPEIMPMzMzMzEiD7ChIiWwkIEiNbCQgSIlEJDBIiVwkGEiJ2THbDx8A6BsHAABIi0Qk +MEiLXCQY6CxWBABIi2wkIEiDxCjDzMxIA1hISIsLSPfRkEiJSEDDzMzMzMzMzMzMzMzMzMzMzEk7 +ZhAPhj4BAABIg+wwSIlsJChIjWwkKEiLUDBIi3A4SDnWdCQPggsBAABIiXQkEEiJRCQ4SIt4QEgP +vP9BuEAAAABJD0T460ZIidBIi2wkKEiDxDDDSIlcJBhIwesDkOh7////SItEJDhIi0hASA+8yUG4 +QAAAAEkPRMhIi1QkEEiLXCQYSInWSInPSInaSIP/QHUhSI1aQEiD48APHwBIOd53sUiJcDBIifBI +i2wkKEiDxDDDTI0EOkw5xnZmSI1PAUiD+UBNGclMi1BASdPqTSHKTIlQQEiNHDpIjVsBSPfDPwAA +AHUrSDnedCZMiUQkIEiJXCQYSMHrAw8fQADo2/7//0iLRCQ4SItcJBhMi0QkIEiJWDBMicBIi2wk +KEiDxDDDSIlwMEiJ8EiLbCQoSIPEMMNIjQWBWAUAuxYAAADo2dwBAJBIiUQkCOjuRgQASItEJAjp +pP7//8zMzMxJO2YQD4blAQAASIPsSEiJbCRASI1sJEBIiUQkUEiJTCRgSIlcJDhIiXwkMOjv9AEA +6Or0AQBIjQU2TgUAuxEAAADo+f0BAEiLRCQ46E/8AQDoSvUBAEiLRCRQDx9EAABIhcAPhK4AAACQ +ikhjiEwkJ4D5AQ+EUAEAAOih9AEASI0FTVIFALsUAAAA6LD9AQDoC/UBAEiLRCRQSItIcEiJTCQ4 +SItAGEiJRCQo6G/0AQBIjQVlRgUAuw0AAAAPHwDoe/0BAEiLRCQo6NH7AQBIjQVSRAUAuwwAAAAP +H0QAAOhb/QEASItEJDjosfsBAEiNBT5EBQC7DAAAAA8fRAAA6Dv9AQAPtkQkJw+2wOgu+gEA6In0 +AQDoBPQBAA8fQADoe/YBAOh29AEASItEJGBIhcB1HEmLTjDGgSkBAAACSI0FmokFALs+AAAA6HDb +AQDoy/MBAEiNBbppBQC7HgAAAOja/AEASItEJGDoMPsBAEiNBUw4BQC7AQAAAA8fQADou/wBAEiL +RCQw6BH7AQBIjQVJOAUAuwIAAAAPH0QAAOib/AEA6PbzAQBIjQV9OgUAuwYAAABIi0wkYEiLfCQw +6DukAADpav///+hR8wEASI0F9FsFALsZAAAADx9EAADoW/wBAOi28wEA6ab+//9IiUQkCEiJXCQQ +SIlMJBhIiXwkIOjYRAQASItEJAhIi1wkEEiLTCQYSIt8JCAPH0AA6dv9///MzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMxIg+wwSIlsJChIjWwkKJBIugAAAAAAgAAASAHCSMHqGkiB+gAAQAByBDHS +6y5IizXEWA0AhAZIixTWSIXSdBpIicZIwegNSCX/HwAASIuUwgAAIABIifDrAjHSSIlUJCBIhdIP +hI4AAABAinJjQID+AXUTTItCGEw5wHIKDx9AAEg5QnB3SUCA/gJ0MYM9DcUNAAB0FkiJz0iJ2UiJ +w0iJ0Oga/f//SItUJCAxwEiJ00iJwUiLbCQoSIPEMMMxwEiJ00iJwUiLbCQoSIPEMMOQSItyaEwp +wItKXEgPr8hIwekgSA+v8UqNBAZIidNIi2wkKEiDxDDDSL6t3q3erd6t3kg58HUkgz2TxA0AAHQb +SInPSInZSInDSInQDx9EAADom/z//0iLVCQgMcBIidNIicFIi2wkKEiDxDDDzMzMzEiD7BhIiWwk +EEiNbCQQSIlEJCBIiXwkMEiLFZlXDQCQSIXSdD+NQQFIPQAAQABzR0iLFMJIhdJ0GEiNuv//HwAx +24nBSInQSItsJBBIg8QYwzHAMduJ2UiJx0iLbCQQSIPEGMMxwDHbidlIicdIi2wkEEiDxBjDuQAA +QAAPH0AA6FtKBACQzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxIg+wYSIlsJBBIjWwkEEiJRCQg +SIl8JDCJ2kiNHDJIicJIid5IwesCTI0EA0iD5gNJiflMOcdzZUkp+EmNUP9JidBIweoVjQwRjUkB +SIsVxVYNAEiF0nQtDx+EAAAAAABIgfkAAEAAc0hIixTKSIXSdBNJgeD//x8ASQHQSIHC//8fAOsF +MdJFMcBMicBIifNIiddIi2wkEEiDxBjDSI0EGkiJ80yJz0iLbCQQSIPEGMOJyLkAAEAA6INJBACQ +zMxJO2YQdk9Ig+wwSIlsJChIjWwkKEiJRCQ4SIl8JEhIifpJicBIKcdMjU8BScHhAkw5zkkPR/FI +iXQkIEyJwEiJ1+j6/v//SIt0JCBIi2wkKEiDxDDDSIlEJAiJXCQQiUwkFEiJfCQYSIl0JCDoj0EE +AEiLRCQIi1wkEItMJBRIi3wkGEiLdCQg6XP////MzMzMzMzMzMzMzMzMzMzMzMzMSIPsaEiJbCRg +SI1sJGBIicdICdhICchmDx+EAAAAAABIqQcAAAAPhesDAACAPTHADQAAD4QrAQAAkEi6AAAAAACA +AABIjQQXSMHoGkg9AABAAHIEMdLrL0iLFVlVDQCEAkiLFMJIhdJ0G0mJ+EjB7w1Igef/HwAASIuU ++gAAIABMicfrAjHSSIXSD4S2AAAAQIpyY0CA/gEPhZ4AAABmkEg5ehgPh5IAAABIOXpwD4aIAAAA +SYtWMEiLktAAAACEAkiLNfBUDQCEBkg9AABAAA+DNwMAAEiLNMZIhfZ0JkmJ+EjB7wVIgef//x8A +TI0MPk2JwknB6ANJg+ADSIHG//8fAOsNSYn6McBFMcAx9kUxyUyJVCRASImMJIAAAABIiVQkWEiF +23UHMdvp+AEAAEiJXCR4Mf/p5AAAAEiLbCRgSIPEaMNIixVb0goASIXSdAlMiwJIi1II6wUx0kUx +wDHA6w1Ii2wkYEiDxGjDSP/ASDnCfjlNiwzATYuR0AAAAEw513LoSTm52AAAAHbfSIn4TCnXSYux +6AEAAA8fQADoewQAAEiLbCRgSIPEaMNIixXy0QoASIXSdAlMiwJIi1II6wUx0kUxwDHA6wNI/8BI +OcJ+Ok2LDMBNi5HgAAAATDnXcuhJObnoAAAAdt9IifhMKddJi7H4AQAADx9EAADoGwQAAEiLbCRg +SIPEaMNIi2wkYEiDxGjDSIPHCA8fAEg5+Q+G8QAAAEiJfCQ4RQ+2GUUPo8MPg4QAAABOjRwXTI0k +H0yLqsAWAABNixtNiyQkTYldAE2JZQhMi5rAFgAASYPDEEyJmsAWAABMOZrIFgAAdUtEiUQkKEyJ +TCRQSIl0JEiJRCQsMcAx2+hvjAEAi0QkLEiLjCSAAAAASItUJFhIi1wkeEiLdCRISIt8JDhEi0Qk +KEyLTCRQTItUJEBBg/gDcwhB/8DpS////0w5znQLSf/BRTHA6Tv///9EicOJwUiJ90yJyOgS+/// +SItUJFhMi1QkQEGJ2EiJ/kmJwUiLXCR4SIt8JDiJyEiLjCSAAAAA6f/+//9Ii2wkYEiDxGjDSIPD +CEg52XbtSIlcJDBFD7YZRQ+jw3N8To0cE0yLosAWAABNixtNiRwkScdEJAgAAAAATIuawBYAAEmD +wxBMiZrAFgAATDmayBYAAHVGTIlMJFBIiXQkSIlEJCxEiUQkKDHAMdvocosBAItEJCxIi4wkgAAA +AEiLVCRYSItcJDBIi3QkSESLRCQoTItMJFBMi1QkQEGD+ANzCEH/wOle////TDnOdAtJ/8FFMcDp +Tv///0SJw4nBSIn3TInI6Br6//9Ii1QkWEyLVCRAQYnYSIn+SYnBSItcJDCJyEiLjCSAAAAA6Rf/ +//+5AABAAOjJRAQASI0FnXAFALsoAAAA6BjTAQCQzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxIg+xQ +SIlsJEhIjWwkSEiJ2kgJw0gJy2YPH4QAAAAAAEj3wwcAAAAPhZ4BAACAPfC7DQAAD4SAAAAASYt2 +MEiLttAAAACEBkyLBSpRDQBBhACQSbkAAAAAAIAAAEkBwUnB6RpJgfkAAEAAD4NOAQAAT4sEyE2F +wHQiSYnCSMHoBUgl//8fAE2NHABJweoDSYPiA0mBwP//HwDrDEUxyUUx0kUxwEUx20iJVCQoSIlM +JGhIiXQkQDHA6w5Ii2wkSEiDxFDDSIPACEg5wQ+G4wAAAEiJRCQgRQ+2I0UPo9RmkHN7TI0kEEyL +rsAWAABNiyQkScdFAAAAAABNiWUITIumwBYAAEmDxBBMiabAFgAATDmmyBYAAHVFRIlUJBhMiVwk +OEyJRCQwRIlMJBwxwDHb6IuJAQBIi0QkIEiLTCRoSItUJChIi3QkQEyLRCQwRItMJBxEi1QkGEyL +XCQ4Dx8AQYP6A3MIQf/C6Vb///9NOdh0C0n/w0Ux0ulG////TInYRInTRInJTInH6DH4//9Ii1Qk +KEiLdCRAQYnJQYnaSYn4SYnDSItEJCBIi0wkaOkQ////SItsJEhIg8RQw0yJyLkAAEAA6NVCBABI +jQWpbgUAuygAAADoJNEBAJDMzMxIg+w4SIlsJDBIjWwkMEiJRCRASIlcJEhIiUwkUEmLVjBIi5LQ +AAAASIlUJCiEAkmJ+EjB7wNJwegGSIPnB0mJyUiJ+UG6AQAAAEHT4kwBxjHJ6wxIg8EISIn+Dx9E +AABJOckPhhwBAABFhNJ1GEiNfgGAfgEAdQZIg8E469ZBugEAAADrA0iJ9w+2N0SE1g+E5wAAAEiJ +TCQYSIl8JCBEiFQkF0iNNAFIhdt1b0yLgsAWAABIizZJiTBJx0AIAAAAAEiLssAWAABIg8YQSImy +wBYAAGYPH4QAAAAAAA8fAEg5ssgWAAAPhY8AAAAxwDHb6OqHAQBIi0QkQEiLTCQYSItUJChIi1wk +SEiLfCQgTItMJFBED7ZUJBfrYEyNBBlMi5rAFgAASIs2TYsASYkzTYlDCEiLssAWAABIg8YQSImy +wBYAAEg5ssgWAAB1LTHAMdvoiIcBAEiLRCRASItMJBhIi1QkKEiLXCRISIt8JCBMi0wkUEQPtlQk +F0HR4pDpz/7//0iLbCQwSIPEOMPMzMzMzMzMzMzMzMzMzMzMzEiD7FBIiWwkSEiNbCRISIXAD4QO +AgAASIlEJFgPH0AASDk4D4VSAQAAD7ZQF/bCQA+F5wAAAIA9Z7gNAAB0J0iJXCRgSIlMJGhJi1Yw +SIuS0AAAAEiJVCRAhAJIi3AgMf9FMcDrDkiLbCRISIPEUMNIg8cISDl4CA+GlQAAAGaQSPfHPwAA +AHUJRA+2Bkj/xusDQdHoQQ+64ABz1EyNDB9MjRQPTIuawBYAAE2LCU2LEk2JC02JUwhMi4rAFgAA +SYPBEEyJisAWAABMOYrIFgAAdZ1IiXwkGESJRCQUSIl0JDAxwDHb6FWGAQBIi0QkWEiLTCRoSItU +JEBIi1wkYEiLdCQwSIt8JBhEi0QkFOld////SItsJEhIg8RQww8fAOh76gMASIlEJDhIiVwkKOis +5gEASI0FiGwFALsoAAAA6LvvAQBIi0QkOEiLXCQo6KzvAQBIjQUlOwUAuw8AAADom+8BAOj25gEA +SI0FJWcFALskAAAA6AXOAQBIiXwkcOgb6gMASIlEJDhIiVwkKEiLTCRYSIsJSIlMJCAPH0AA6Dvm +AQBIjQUXbAUAuygAAADoSu8BAEiLRCQ4SItcJCjoO+8BAEiNBdIzBQC7CwAAAOgq7wEASItEJCAP +H0QAAOgb7AEASI0FE0AFALsSAAAA6ArvAQBIi0QkcA8fRAAA6PvrAQDoVugBAOhR5gEASI0FgGYF +ALskAAAADx9EAADoW80BAEiNBbVsBQC7KQAAAOhKzQEAkMzMzMzMzMzMzEk7ZhAPhg0BAABIg+xg +SIlsJFhIjWwkWEiJRCRoSIl8JHhIi1YgSMHiDUjB6gNI98IDAAAAD4XKAAAADx9EAACF2w+FqAAA +AEyLRmhMiUQkOOskSItUJChMi0wkMEwpykyLRCQ4SItEJEiLTCQgi1wkJEiLfCRASIXSdlVIiVQk +KEiJRCRQSInW6NL0//9IiUQkSIlcJCSJTCQgSIl8JEBIiXQkMEiJ8kjB7gJMi0QkOEmD+Ah1C0yL +TCRQRTHSkOskSItEJFBIifPoUUQEAOuCSItsJFhIg8Rgw0HGAf9J/8KQSf/BSTnycvDpY////0iN +BWNMBQC7GAAAAA8fQADoO8wBAEiNBRtQBQC7GgAAAOgqzAEAkEiJRCQIiVwkEIlMJBRIiXwkGEiJ +dCQg6C02BABIi0QkCItcJBCLTCQUSIt8JBhIi3QkIOmx/v//zMzMzMzMzMzMzMzMzMzMzMxMjaQk +eP///007ZhAPhqELAABIgewIAQAASImsJAABAABIjawkAAEAAEiD+wgPhKEDAABIixUeSg0AhAKQ +SbwAAAAAAIAAAEkBxEnB7BpJgfwAAEAAD4NKCwAASosU4pBIhdJ0KEmJxUjB6AVIJf//HwBMjTwC +TInoScHtA0mD5QNIgcL//x8ATInm6w5MieZFMf9FMe0x0kUx5EWJ6EyLbyBIg/sQD4SaAgAASIP7 +GA+EWgEAAE2J6UyNLANFieJFieRJidNIuv//////fwAATAHqSMHqGkk51HQGSYnHRTHbSImEJMgA +AABIiZQkwAAAAEyJpCS4AAAASImcJBgBAABIibQksAAAAEQPtm8XQfbFQA+FtwAAAEyLFw8fQABM +OdEPhn4AAABMiZQkqAAAAEyLXwhJwesDSYP7OXcpSImMJCABAABNic1FMclMiUwkcE2J6UUx7UyJ +rCSIAAAATYnN6UgJAABJg8MHScHrA0mJ1UmNU/+QScHqA0jB4gNJKdJMicpPjQwLTY1J/0mJ0zHS +SIlUJGgx0kiJVCRgTIna6YMIAABNictFMe1FMclMiYwk4AAAAEUxyUyJTCRgRTHJTIlMJHhNidnp +hAQAAEiLdwhIixdNjWEETIn4TInfSYnJTYnjRInRSYnaRInDSYnQ6AgKAABIi5QkuAAAAEyLpCTA +AAAATDni6dQBAABBD7Z1AEiDPwh1Bb4HAAAAg+YHQYnxweYERAnOQYnxg84Qg+ZAQNDuRAnOQYnx +g84QQYP4AXcxRYXAdRlBD7YHg+CIRAnIg8gQQYgHDx9AAOnRAAAAQQ+2B4PgEdHmCcZBiDfpvgAA +AECIdCRFDx8AQYP4AnVaRQ+2D0GD4TNBifKD5jPB5gJECc5BiDdBjVgBg/sDciFJOdd0BUn/x+sX +TIn4RInhSInX6OLv//9ED7ZUJEVJicdBD7YHg+DuQcDqAkGD4hFBCcJFiBfrWGaQQYP4A3VQRQ+2 +D0GD4XdBifKD5hHB5gNECc5BiDcPHwBJOdd0B5BJjUcB6xdMifhEicNEieFIidfog+///0QPtlQk +RQ+2CIPhzEHQ6kGD4jNBCcpEiBBIi6wkAAEAAEiBxAgBAADDSIM/CHUrQYP4IBnAQQ+2F0SJwbsz +AAAA0+MhwwnaQYgXSIusJAABAABIgcQIAQAAw0iLTwhIwekDSIP5IBnAQYP4IBnSGdtBD7Z1AIPm +A78QAAAA0+ch+P/IJfAAAAAJxkEPtgdEicG/MwAAANPnIdf31yHH0+Yh8wn7QYgfSIusJAABAABI +gcQIAQAAw0iLrCQAAQAASIHECAEAAMMPhPEAAABIixVnRg0AhAJMi4QksAAAAEqLFMKQSIXSdC1I +i4QkyAAAAEmJwUjB6AVIJf//HwBMjRQCTInIScHpA0mD4QNIgcL//x8A6xNIi4QkyAAAAEUx0kUx +yTHSRTHATIucJBgBAABJwesDSYnEQYP5Ag+FQQEAAEUPtipBg+UzRA+2OEUJ70WIOkGNWQGD+wNz +B0GDwQLrUJBMOdJ0CEn/wkUxyetCSImEJPgAAABMiZwkoAAAAEyJ0ESJwUiJ1+j17f//TIucJKAA +AABMi6QkyAAAAEGJyEGJ2UiJ+kmJwkiLhCT4AAAASYPD/kj/wOnEAAAASIusJAABAABIgcQIAQAA +w0yJlCTYAAAATImcJIAAAABMid5Iwe4CSMHmAkyJ0ESJy0SJwUiJ1+jp7v//SImEJPAAAACJXCRM +iUwkSEiJvCToAAAASIl0JFBIwe4CSIm0JMAAAABIi4Qk2AAAAEiLnCTQAAAASInx6ChBBABMi5wk +gAAAAEiLVCRQSSnTkEiLlCTAAAAATIuEJNAAAABJjQQQTIukJMgAAABMi5Qk8AAAAESLTCRMSIuU +JOgAAABEi0QkSEiJhCTQAAAASYP7BA+DOv///0mD+wIPhYMAAABBD7Yyg+bMRA+2GEEJ80WIGpBB +g/kDcwZBjXEB6zhMOdJ0B0n/wjH26yxMidBEictEicFIidfoquz//0yLpCTIAAAAid5IifpJicJB +ichIi4Qk0AAAAIP+A3IlSTnSdSBMidCJ80SJwUiJ1+h17P//SIuEJNAAAABMi6QkyAAAAEj/wEyJ +4UgpyEiJw0yJ4OhxPQQA6Yf+//9Nhcl0KkmJ1UEPthGQSf/BTDnRSImEJJAAAAC4CAAAAEiJRCRo +SIuEJJAAAADrFUw50UiLVCRgTIlsJGhMi6wkwAAAAHUKTItXCEnB6gPrL02F0g+EeAMAAEiJyEiJ +0THSSffySI1Q/0wPr9JMA1cIScHqA0iLhCTIAAAASInKTYXSD4TdAgAADx9AAEWFwHU9SYnQSIPi +D0iByvAAAABJg/oEdwy5BAAAAGaQ6QwCAABBiBeQScHoBEiLVCRoSIPC/En/x7kEAAAAZpDreUGD ++AIPhXwCAABJidBIg+IDSMHiAkiJ0UiDykBIgcnAAAAASYP6AUgPR9FMieFFD7YnQYPkM0QJ4kGI +F5BJjVcBSYP6AncSSYnMuQYAAABJidcx0umWAQAAScHoAkyLZCRoSYPE/rkCAAAASYnXTIniTIuk +JLgAAABIg8L8SIt8JGDrLUWIRwGQTIuEJIgAAABJwegESYPHAkiLTCR4TIukJLgAAABMielMi6wk +wAAAAEyJhCSIAAAASYPgD0mByPAAAABMjWEETTniD4YQAQAASIlMJFhFiAeQTIuEJIgAAABJwegE +TIukJOAAAABNOeF0MUiD+ghzIkUPtilIidFJ0+VNCehJ/8FIi0wkWEyLrCTAAAAA6ZgAAABIg8L4 +6Y8AAABNhcl0XEiD+kBNGe1FD7YJSInRSdPhTSHNTQnoTItMJHhOjSwJSYP9CHMTQQ+2E0yJ6UjT +4kkJ0EmNUwHrDk6NLAlNjW34TInaTInpTIusJMAAAABJidFIicpIi0wkWOsuSIP6CHMkSInRSNPn +SQn4TItsJHhKjRQpSItMJFhIi3wkYEyLrCTAAAAASIPC+EyJhCSIAAAASYPgD0mByPAAAABMjWkI +TTnqD4ex/v//Sf/HTYnsTIusJMAAAABMieFMicJMi6QkuAAAAEw50XYzSYnITCnRSIPB/Ej32UiD ++UBNGclBugEAAABJ0+JNIcpNjUr/TYnKScHhBE0J0UwhyusDSYnISYnZSMHrA0k52HcOQYgXkEmN +UARNjUcB6zNMjVMCTTnQdRZFD7YHQYPgzEQJwkGIF0057Olf+v//TTns6Vf6//9BxgAASIPCBJBJ +/8BIOdp2702Jx0mJ0DHS675IjQU6VgUAuyEAAADoJ8IBAEiJ+A8fQADoO94DAEiJhCTwAAAASImc +JJgAAADoZtoBAEiNBV1ABQC7FwAAAOh14wEASIuEJPAAAABIi5wkmAAAAA8fRAAA6FvjAQDottwB +AOix2gEASI0FLWUFALstAAAADx9EAADou8EBAOi2nAEATItsJGhMiYwk4AAAAEyJVCR4TIuUJKgA +AABJidFIi5QkwAAAAOkA/P//SIP6QEgZwEiNcghFD7YhSInRSdPkTCHgSIuMJIgAAABICcGQSf/B +SIuEJMgAAABIi5QkwAAAAEyLpCS4AAAASIl0JHBIiYwkiAAAAEiLjCQgAQAASIu0JLAAAABIi1Qk +cEw52nKaTInSScHqA0+NDBJJg/k5dw1Mi4wkiAAAAE2J0+tUTIuMJIgAAABNietMi6wkwAAAADHS +TIlUJGhMiUwkYEUxyekr////TIlMJGBIGcBMidFJ0+FJIcFIi0QkYEkJwUyNFAlIi4QkyAAAAEiL +jCQgAQAASYP6QHbNRYTbdGlNidpFD7bbZkSJXCRGQbs5AAAARQ+220SJ2EQPt1wkRjHSZkH380QP +tthND6/TSYP6QE0Z20iJyEyJ0UyJ2kG7AQAAAEnT40kh00n/y00h2UiJwUiLlCSoAAAASIuEJMgA +AADpQf///5DoO5sBAEyJ4LkAAEAA6M4xBACQSIlEJAhIiVwkEEiJTCQYSIl8JCDoNCoEAEiLRCQI +SItcJBBIi0wkGEiLfCQg6Rv0///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4YfAwAA +SIPsaEiJbCRgSI1sJGBIiUQkcEiJvCSAAAAASffCHwAAAA+F5QIAAEyJlCSgAAAASIm0JIgAAABI +iUQkWE05wXR5TYXAD4S9AgAASInBTInIMdJJ9/BIx0QkMAAAAABIjVQkOEQPETpIjVQkSEQPETpM +icJJwegDSYnxSMHuA02JxEkp8E2FwHYoxkQkMAFJg/gBdhPGRCQygUmNcP+7AwAAAOkeAgAAuwIA +AADpyQAAADHbZpDpwAAAADHbSInBvwIAAABMidjoyQMAAEiJwkjB4ANIi7QkiAAAAEg5xnU5SI1K +A0jB6QJIi5QkoAAAAEjB6gVIi3QkWEiNHBZIjQQOSInBSCnDSInI6KY2BABIi2wkYEiDxGjDSIlU +JCjoEtcBAEiNBYZgBQC7KwAAAOgh4AEASItEJCjoF90BAEiNBTIqBQC7DgAAAOgG4AEASIuEJIgA +AADo+dwBAOhU2QEA6E/XAQBIjQWXXwUAuysAAAAPHwDoW74BAEiD+ygPgx0BAADGRBwwgEiNcwHr +F02J4EkPuuwHRIhkNDBJwegHSP/GTYnESYH8gAAAAHIRZg8fRAAASIP+KHLU6dQAAABIg/4oD4O9 +AAAARIhkNDBI/8ZMjUD/TInA6xpNicRJD7roB0SIRDQwScHsB0j/xk2J4A8fAEmB+IAAAAByCEiD +/ihy1+t0SIP+KHNfRIhENDBMjUYBSYP4KHNDSIlUJChIiUQkIMZENDEATInYSI1cJDC/AgAAAOhZ +AgAASItUJChIi3QkIEgPr9ZIi7QkiAAAAEgB8kjB6gPpgv7//0yJwLkoAAAA6OouBABIifC5KAAA +AGaQ6NsuBABIifC5KAAAAOjOLgQASInwuSgAAADowS4EAEiJ8LkoAAAA6LQuBABIidi5KAAAAOin +LgQASYnwSA+67gdAiHQcMEnB6AdI/8NMicZIgf6AAAAAcglIg/soctqQ6yNIg/socw1AiHQcMEj/ +w+mQ/v//SInYuSgAAAAPHwDoWy4EAEiJ2LkoAAAA6E4uBADoyZcBAEiNBQVZBQC7JwAAAOi4vAEA +kEiJRCQIiVwkEIlMJBRIiXwkGEiJdCQgTIlEJChMiUwkMEyJVCQ4TIlcJEDopyYEAEiLRCQIi1wk +EItMJBRIi3wkGEiLdCQgTItEJChMi0wkMEyLVCQ4TItcJEDpd/z//8zMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMSTtmEA+GvAAAAEiD7EhIiWwkQEiNbCRASIlEJDhIwesDSI1TB0jB6gNIiVQkKEiNcgFI +iXQkILsBAAAASI0N3boNAEiJ8Oh1tP//hABIi0wkIEiB+QAAAEB3YEiLVCQoSDnKc05IiUQkMMYE +AqEx20iJwb8BAAAASItEJDgPHwDoewAAAEiLXCQwSItUJCgPthQagPqhdQpIi2wkQEiDxEjDSI0F +kkEFALsbAAAA6I27AQBIidDoBS0EALoAAABA6FstBACQSIlEJAhIiVwkEOiLJQQASItEJAhIi1wk +EJDpG////8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEiD7BhIiWwkEEiNbCQQSIlMJDBIicox +9kUxwA8fRAAA6a8BAABJicFIidhMicvpoQEAAEQPtgiQTYnKSYPhf0yNWAFBD7riB3IdTYXJdQ1I +hdsPhM4BAAAxwOvJScHpAzHA6SAFAABNhcl1BzHA6dMEAABIiUwkCDHARTHSDx8A6ekBAABJD6/B +SYP5OXcwSIP/AXUVkEyLVCQITY1i/0iJ8U2JxemIBAAAkEyLVCQITY1i/0iJ8U2JxenBAgAASSnx +SIP/AXVeTY1RB0nB6gNJg+EHTItkJAhNieVNKdRNieJNhcl0M0mNSfhI99lIg/lATRn/QQ+2FCRI +0+pMIfpIifFI0+JJCdCQSY00CUwpyE2NYgFIi1QkMEiJwUjB6APp+gEAAE2NUQNJweoCSYPhA0yL +ZCQITYnlTSnUTYniTYXJdDdJjUn8SPfZSIP5QE0Z/0EPthQkSIPiD0jT6kwh+kiJ8UjT4kkJ0JBJ +jTQJTCnITY1iAUiLVCQwSInBSMHoAulaAQAASIPhA0iFyXYuSIP+QE0ZyUUPthQkQbwBAAAASdPk +Sf/MTSHUSInISInxSdPkTSHMTQngSI00CEiJ2EyJ6UyJ2+lY/v//SIPG+EyJyUiD/ggPglX+//9m +kEiD/wF1DkSIAZBJwegITI1JAevZTYnCSYPgD0mByPAAAABEiAFNidBJweoESYPiD0mByvAAAABE +iFEBkEnB6AhMjUkC66dIg/8BdRxIictIKdFIjRTOSIn3SPfeSIPmB0gB/ukZBAAASInLSCnRSI0U +jkiJ90j33kiD5gNIAf6Q6dsDAABMjVEHSItMJAhJg/pATRnkRQ+2K5BNie9Jg+V/TInRSdPlTSHs +TAngSf/DQQ+65wdyzunp/f//SIP+QE0ZyUUPthQkSYPiD0mJz0iJ8UnT4k0hyk0J0JBNicFJg+AP +SYHI8AAAAEWIRQBI/8iQScHpBEn/xEn/xUyJ+U2JyEiFwHe16Zz+//9Ig/5ATRnJRQ+2FCRJic9I +ifFJ0+JNIcpNCdCQRYhFAEj/yJBJwegISf/ESf/FTIn5Dx8ASIXAd8dIg+EHSIXJD4aN/v//SIP+ +QE0ZyUUPthQkQbwBAAAASdPkSf/MTSHUSInISInxSdPkTSHMTQngSI00COla/v//ScHgBEUPtjwk +SYPnD00J+JBIg8YESf/MDx8ATDnOcuBMOc52GEwpzkiD/kBNGeRJic9IifFJ0+hNIeDrBkmJz0mJ +8UmD+QF1HEmD+AF1Ebk5AAAASbj/////////AespSInB6yRLjTQJSIP+OXcITInJ6fgAAABMicmQ +6wxMKchMiclNieBJifdIOcF3LUmD/0BIGfZJiclMiflNicRJ0+BJIfBNCcVJjTQJSIP/AQ+EjgAA +AOtyDx9AAEiFwHY1SIP4QEgZ9kmD/0BNGclIicFBvAEAAABJ0+RJIfRJjXQk/0kh8EyJ+UnT4E0h +yE0JxUyNPAFMif5IidhMidFNiehMidvpt/v//02J6EmD5Q9Jgc3wAAAARYgqkEnB6ARIg8b8Sf/C +TYnFSIP+BHPa6Un///9FiCqQScHtCEiDxvhJ/8JIg/4Ic+vpL////0yJBCRIGfZJicxMiclJ0+BJ +IfBIizQkSQnwTI0MCUyJ4UmD+UB22UiFyXQ6SInGuDkAAABJidEx0kj38UgPr8hIg/lATRnkSInw +vgEAAABI0+ZMIeZI/85JIfBMicpJicnpw/7//+gikQEAScHgCEUPtjwkTQn4kEiDxghJ/8xMOc5y +5+lJ/v//SIPAB0yJ+U2J00iD+EBNGdJFD7YjkE2J5UmD5H9Jic9IicFJ0+RNIdRNCeFNjVMBQQ+6 +5QdyyUyJ+U2J0+n1+v//SP/ASf/DTInhTDnIc1xFD7YjSYnNSInxSdPkTQngkA8fAEiD/wF1D0WI +RQCQScHoCE2NZQHryU2Jx0mD4A9JgcjwAAAARYhFAE2J+EnB7wRJg+cPSYHP8AAAAEWIfQGQScHo +CE2NZQLrlkmD4gdNhdJ2GkUPtgtIichIifFJ0+FNCciQSY00Ckn/w+sDSInISInBSInYTInb6Qn6 +//9MicFJg+APSYHI8AAAAESIA0iDxvyQSMHpBEj/w0mJyA8fQABIhfZ310iJ0EiLbCQQSIPEGMNE +iANIg8b4kEnB6AhI/8NIhfZ37OvdzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEHZvSIPsMEiJ +bCQoSI1sJChIiVwkQJBIjVA/SMHqBkiBwv8fAABIweoNSI0FbDEMALkCAAAASInTDx9AAOi77AAA +SIlEJCBIi0gYSItUJEBIg8IEMdu/AQAAAEiJ0Oga+f//SItEJCBIi2wkKEiDxDDDSIlEJAhIiVwk +EJDoWx4EAEiLRCQISItcJBDpbP///8zMzMzMzMzMzMzMzEk7ZhAPhqAAAABIg+woSIlsJCBIjWwk +IEjHRCQIAAAAAEQPEXwkEEiNBZIAAABIiUQkEEiNRCQISIlEJBhIjUQkEEiJBCTotRwEAEUPV/9k +TIs0Jfj///8xwOsWSItMJAiEAUiNFTagDQBIiVTBKEj/wEg9iAAAAHziSIsFL5sNAEiD+AF1BDHJ +6wjosKr//0hjyEiLVCQISIkKSItEJAhIi2wkIEiDxCjD6JEdBADpTP///8zMzMzMzMzMzMzMzEk7 +ZhB2XUiD7BhIiWwkEEiNbCQQSItKCEiJTCQIkJBIjQUaMAwA6BWJ//9IjQWemA0A6CkeAABIi0wk +CEiJAYsNIzENAImIqAQAAJCQSI0F7C8MAOjHiv//SItsJBBIg8QYw+h4HAQA65bMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMSTtmEHZKSIPsIEiJbCQYSI1sJBhEDxF8JAhIjQ1fAAAASIlMJAhIiUQkEEiN +RCQISIkEJOiHGwQARQ9X/2RMizQl+P///0iLbCQYSIPEIMNIiUQkCOimHAQASItEJAiQ657MzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdntIg+wYSIlsJBBIjWwkEEiLQghIiUQkCA8f +AOibBQAASItEJAjosQcDAJCQSI0FCC8MAOgDiP//kEiLDbuXDQBIKw2Elw0ASIkNrZcNAEiLDY6X +DQBIi1QkCEiJCkiJFX+XDQCQkEiNBc4uDADoqYn//0iLbCQQSIPEGMPoWhsEAOl1////zMzMzMzM +zMzMzMzMzMzMzMzMzMzMSTtmEA+GYwIAAEiD7EhIiWwkQEiNbCRAhAAPttMPHwBIgfqIAAAAD4My +AgAASItM0CgPt3FgZg8fhAAAAAAAkEg5cTgPhQQCAABIiUQkUIhcJFhIiVQkOEiNNeGdDQCQSDnx +dDKLNV0vDQCDxgM5cVgPhcYBAABIjTRSSMHmBkiNPWIwDQBIjQQ+SInL6E4OAABIi1QkOEiNDFJI +weEGSI0dQjANAEiNBBnoMQcAAEiFwA+EdAEAAA+3SGAPH0AASDlIOA+EUQEAAEiJRCQoiw3zLg0A +g8EDiUhYSI0FhsYNAOhZZgEAhAAPtkwkWInK0OlID77JZg8fhAAAAAAAkEiD+UQPgwQBAABIjQzI +SI1JSEiLXCQoSItzOA+3e2BIKf7wSA/BMYD6BXUXSItMJFBIi1Eg8EgPwVAwSMdBIAAAAABIjQUe +xg0A6PFmAQBIi0wkKA+3UWBIi1loSA+v00iLWSBIweMNSCnTSI0V3p4NAPBID8EaSItUJFBIi1oI +SI010Z4NAPBID8EeSMdCCAAAAACAPTUsCwAAdD6QSMdEJDAAAAAASIsVop4NAEiJVCQwuCEAAABI +x8P/////SI1MJDC/AQAAAEiJ/uhPdAMASItMJChIi1QkUIM9GpcNAAB0FkiNBTWeDQDoEIIAAEiL +TCQoSItUJFBIi0QkOEiJTMIoSItsJEBIg8RIw0iJyLlEAAAA6CUhBABIjQV+KwUAuxYAAADolK8B +AEiNBboaBQC7DQAAAOiDrwEASI0F0CkFALsWAAAA6HKvAQBIjQU2TQUAuygAAADoYa8BAEiJ0LmI +AAAA6NQgBACQSIlEJAiIXCQQ6GUZBABIi0QkCA+2XCQQ6Xb9///MzMzMzMzMzMzMzMzMzMzMzMzM +zMzMSTtmEA+GNgIAAEiD7FhIiWwkUEiNbCRQSI2TACAAAJBIOdMPhwUCAABIiVwkaIhMJC9AiHwk +cUiJ2EjB6A1IjUgBSPfD/x8AAEgPRcFIiUQkOEiJwUjB4A1IiUQkQEiJy+jaxwAAkA+2TCRxD7bJ +iEwkLkiNBYYrDABIi1wkOA+2fCQv6BflAABIhcAPhIsBAABIiUQkMIhcJC1IjQUmxA0A6PljAQBI +i0wkQPBID8FIOLkBAAAA8EgPwUhASI0FBMQNAOjXZAEASItMJEBIjRXbnA0A8EgPwQqAPU8qCwAA +dDSQSMdEJEgAAAAASIsVvJwNAEiJVCRIuCEAAABIx8P/////SI1MJEi/AQAAAEiJ/uhpcgMAgz0+ +lQ0AAGaQdAxIjQVXnA0A6DKAAAAPtkQkLkg9iAAAAA+D1AAAAEiNDEBIweEGixXbKw0ASI019CwN +AEiNDA5IjUlY0eqD4gFIjRSSSI0E0UiLXCQw6M1cAQBIi3QkMEiLThhIi1QkaEgB0UiJTnBIiw0J +LA0AhAFIi1YYkEi/AAAAAACAAABIjQQXSMHoGkg9AABAAHNZTIsEwU2FwHQiSYnRSMHqBUiB4v// +HwBMAcJJwekDSYPhA0mBwP//HwDrCjHARTHJRTHAMdJEicuJwUyJx0iJ0Ojr3///SItEJDAPtlwk +LUiLbCRQSIPEWMO5AABAAOitHgQAuYgAAADogx4EAEiNBSkYBQC7DQAAAOjyrAEASI0FGBgFALsN +AAAA6OGsAQCQSIlEJAhIiVwkEIhMJBhAiHwkGejoFgQASItEJAhIi1wkEA+2TCQYD7Z8JBnpj/3/ +/8zMzMzMzMzMzMzMzMzMzEk7ZhAPhqsBAABIg+xASIlsJDhIjWwkOEiJRCRISItICEiNFRCbDQDw +SA/BCkjHQAgAAAAAiw1tKg0AiUwkFDHS6wNI/8JIgfqIAAAAD43+AAAASItc0ChIjTXBmA0AkEg5 +83TeSIlUJCBIiVwkGEiLSzhIiUwkMA+3U2BIiVQkKEiNBcDBDQDok2EBAIQASItMJCDQ6UgPvslm +Dx9EAABIg/lED4P9AAAASI0MyEiNSUhIi1QkKEiLXCQwSCna8EgPwRFIjQV9wQ0A6FBiAQCLTCQU +jVEBSItcJBgPH0AAOVNYdCFIi1QkKEiLdCQwSCnySItzaEgPr9ZIjTUvmg0A8EgPwRZIi0wkIEiN +FElIweIGSI01rioNAEiNBBZmkOibCAAASItMJCBIjRXvlw0ASIt0JEhIiVTOKEiJ8EiJykiNNdiX +DQCLTCQU6fL+//9EDxF4EEiNBevADQAPHwDou2ABAEiLTCRISItRIPBID8FQMEjHQSAAAAAASI0F +xcANAOiYYQEAgz1Nkg0AAHQMSI0FaJkNAOhDfQAASItsJDhIg8RAw0iJyLlEAAAA6GwcBACQSIlE +JAjoARUEAEiLRCQI6Tf+///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhsAAAABIg+wgSIls +JBhIjWwkGIsNqigNAIuQqAQAADnKdDmNWf452nU8SIlEJCjo5/3//0iLRCQoZpDo+/8CAIsNfSgN +AEiLVCQoh4qoBAAASItsJBhIg8QgkMNIi2wkGEiDxCDDiUwkEIlUJBToqMIBAEiNBRMVBQC7DQAA +AOi3ywEAi0QkFOiuyAEASI0FfzYFALseAAAAZpDom8sBAItEJBDoksgBAOjtxAEA6OjCAQBIjQXJ +EgUAuwwAAADo96kBAJBIiUQkCOgMFAQASItEJAjpIv///8zMSTtmEA+G3AYAAEiD7HBIiWwkaEiN +bCRoD7YQ0OpID77SSIP6RA+DqAYAAEiJRCR4SI0NCXQKAA+2BBFIweANMdvousIAAIA9oyULAAB0 +BegsfQMAiw2OJw0AiUwkIMZEJCQAi0wkINHpg+EBiUwkFEiNFIlIiVQkYEiLXCR4SI0E00iNQAjo +FVoBAEiFwHUVi1QkFEiLRCR4uWQAAABmkOmIAAAAMclIiUQkOIA9PSULAAB0DoTJdQrown0DAEiL +RCQ4SItIOA+3UGBIOdF0OkiLWDBIOct0MUiD48BIwesD6LrK//9Ii0QkOEiLSEBIi1AwSInLSInR +SNPrSIlYQEiLbCRoSIPEcMNIjQWpKQUAuxgAAADoxagBAEiLXCQYSI1L/4tUJBRIi0QkeEiFyQ+M +CgIAAI1a//fbDx9EAABIg/sCD4N9BQAASIlMJBhIjQybSI0MyEiNSQhIichmkOg7WQEASIXAD4So +AQAAi1BYi3QkIIPG/jnydAYxyTHS60uQgHwkJAB1FboBAAAASI01WiYNAPAPwRbGRCQkAYtUJCCN +cv7/ykiJwYnw8A+xUVgPlMKE0nQNSInIuQEAAABIicLrB0iJyDHJMdJIiVQkMITJD4RB////SIlE +JDhIjUQkMLsBAAAA6DKyAACQgHwkJAAPhBABAADGRCQkALn/////SI0V5SUNAPAPwQr/yYXJdQ2L +DdElDQCFyQ+VwesCMcmEyQ+E3gAAAJCDPbySDQAAD47QAAAASIsFL5YNAEiJRCRYSIsN4yUNAEiJ +TCRQSIsVxyUNAEiJVCRI8g8QBdIlDQDyDxFEJEDoz78BAEiNBU83BQC7HwAAAA8fAOjbyAEASItE +JFhIwegU6M3FAQBIjQVKEwUAuw4AAACQ6LvIAQBIi0QkWEiLTCRQSCnISMHoFOilxQEASI0FsCMF +ALsXAAAA6JTIAQBIi0QkSOiKxQEASI0FzQsFALsKAAAA6HnIAQDyDxBEJEDobsIBAEiNBR8PBQC7 +DAAAAGaQ6FvIAQDotr8BADHJSItEJDjpkf3//0iLRCR4SItMJBiLVCQU6xpIi1wkGEiNS/9Ii0Qk +eItUJBQPH4QAAAAAAEiFyQ+MEgIAAI1a//fbSIP7Ag+DYwMAAEiJTCQYSI0Mm0iNDMhIjUlYSInI +6C9XAQBIhcAPhNwBAACLUFiLdCQgg8b+OfJ0BjHJMdLrS5CAfCQkAHUVugEAAABIjTVOJA0A8A/B +FsZEJCQBi1QkII1y/v/KSInBifDwD7FRWA+UwoTSdA1Iici5AQAAAEiJwusHSInIMckx0kiJVCQo +ZpCEyQ+EPv///0iJRCQ4SI1EJCi7AQAAAOgksAAASItEJCjousf//0iLXCQoSDlDOHUcSItMJGBI +i1QkeEiNBMpIjUBY6NhUAQDp+f7//0iJQzCQgHwkJAAPhA0BAADGRCQkALn/////SI0VoiMNAPAP +wQr/yYXJdQ2LDY4jDQCFyQ+VwesCMcmEyQ+E2wAAAJCDPXmQDQAAD47NAAAASIsF7JMNAEiJRCRY +SIsNoCMNAEiJTCRQSIsVhCMNAEiJVCRI8g8QBY8jDQDyDxFEJEDojL0BAEiNBQw1BQC7HwAAAOib +xgEASItEJFhIwegU6I3DAQBIjQUKEQUAuw4AAACQ6HvGAQBIi0QkWEiLTCRQSCnISMHoFOhlwwEA +SI0FcCEFALsXAAAA6FTGAQBIi0QkSOhKwwEASI0FjQkFALsKAAAA6DnGAQDyDxBEJEDoLsABAEiN +Bd8MBQC7DAAAAGaQ6BvGAQDodr0BADHJSItEJDjpUfv//0iLRCR4kIB8JCQAD4QZAQAAxkQkJAC5 +/////0iNFXgiDQDwD8EK/8lmkIXJdQ2LDWIiDQCFyQ+VwesCMcmEyQ+E5QAAAJCDPU2PDQAAD47X +AAAASIsFwJINAEiJRCRYSIsNdCINAEiJTCRQSIsVWCINAEiJVCRI8g8QBWMiDQDyDxFEJEAPH0QA +AOhbvAEASI0F2zMFALsfAAAA6GrFAQBIi0QkWEjB6BSQ6FvCAQBIjQXYDwUAuw4AAADoSsUBAEiL +RCRYSItMJFBIKchIwegU6DTCAQBIjQU/IAUAuxcAAADoI8UBAEiLRCRI6BnCAQBIjQVcCAUAuwoA +AADoCMUBAPIPEEQkQGaQ6Pu+AQBIjQWsCwUAuwwAAADo6sQBAOhFvAEASItEJHgPtg1pHwsAiEwk +E4TJdAro7HcDAEiLRCR46GIBAABmkEiFwHQKD7ZMJBPp+Pn//zHASItsJGhIg8Rww4nYuQIAAADo +mRQEAInYuQIAAADojRQEAEiJ0LlEAAAADx9EAADoexQEAJBIiUQkCOgQDQQASItEJAjpBvn//8zM +zMzMzEk7ZhAPhrwAAABIg+wgSIlsJBhIjWwkGGaDe2AADx8AD4SNAAAAiw28IA0AjVEBi3NYOdZ1 +DI15/4d7WDnWZpDrB4nKh0tYidF0REiLUzgPt3NgSCnySIXSfhqEANHpg+EBSI0MiUiNBMhIjUAI +6I1RAQDrNIQA0emD4QFIjQyJSI0EyEiNQFjoc1EBAOsaSMdEJBAAAAAASIlcJBBIjUQkEDHb6Hes +AABIi2wkGEiDxCDDSI0FFDwFALskAAAAkOgbogEAkEiJRCQISIlcJBDoKwwEAEiLRCQISItcJBCQ +6Rv////MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4Y0AQAASIPsQEiJbCQ4SI1sJDgP +tgiJytDpSA++wUiD+EQPgwYBAABIjTUMbAoAD7YcBkiJXCQwSI01XG0KAA+3NEZIiXQkKEiNBWwe +DACJ0b8BAAAADx9EAADo+9cAAEiFwA+ErAAAAEiLVCQwSMHiDUSLQFxJD6/QSMHqIEyLRCQoTA+v +wkwDQBhMiUBwSIsVvR8NAIQCTItAGJBJuQAAAAAAgAAATQHBScHpGkmB+QAAQABzakqLFMpIhdJ0 +Ik2JwknB6AVJgeD//x8ASQHQScHqA0mD4gNIgcL//x8A6wtFMclFMdIx0kUxwEiJRCQgRInTRInJ +SInXSInGTInA6JXT//9Ii0QkIEiLbCQ4SIPEQMMxwEiLbCQ4SIPEQMNMici5AABAAOhNEgQAuUQA +AADoIxIEAJBIiUQkCOi4CgQASItEJAjprv7//8zMzMzMzMzMzMzMzMzMSTtmEA+G/AAAAEiD7EBI +iWwkOEiNbCQ4SIsVKR8NAEiJVCQwSIs1JR8NAEiJdCQoMcDrE0iLfCQgSI1HAUiLdCQoSItUJDBI +OfB9b0iLPaUeDQCEB0yLBMIPH4AAAAAASYH4AABAAA+DhgAAAEiJRCQgSos8x4QHTIuHAAwhAJBN +hcB0BDHJ61xIiXwkGLgAABAAMdtIjQ3cng0ADx9AAOhbmP//SIXAdCJIi1QkGEiJggAMIQDpef// +/8YF0YYNAAFIi2wkOEiDxEDDSI0FCEAFALsqAAAA6KOfAQBBxgQIAEj/wUiB+QAAEAB87+lB//// +TInAuQAAQAAPH0QAAOgbEQQAkOiVCQQA6fD+///MzMzMzMzMzMzMzMzMzMzMSTtmEHZkSIPsGEiJ +bCQQSI1sJBBIiw2lkQ0ADx9EAABIhcl1GYsNHZINADkNG5INAHYHuAEAAADrCTHA6wW4AQAAAITA +dRHGBSWGDQAASItsJBBIg8QYw0iNBdkSBQC7EwAAAOj3ngEAkOgRCQQA64/MzMzMzMzMzMzMzMzM +zMxJO2YQD4acAQAASIPsOEiJbCQwSI1sJDBIiXwkWA+2F0CE8g+EiAAAAEiLFSgdDQCEApBIuwAA +AAAAgAAASAHDSMHrGkiB+wAAQABzV0iLFNqEAkiLkgAMIQCEAkiJwUjB6B1IJf//DwBIAcKKGkjB +6RpIg+EHvgEAAADT5g8fAECE3nQPuAEAAABIi2wkMEiDxDjD8EAIMjHASItsJDBIg8Q4w0iJ2LkA +AEAA6M8PBABIiVwkKEiJTCQgSIlEJEDoe7YBAOh2tgEASI0FkEoFALs5AAAA6IW/AQBIi0QkQOjb +vQEA6Na4AQDo0bYBAOhMtgEASI0Fjx4FALsYAAAA6Fu/AQBIi0QkKOixvQEASI0FzfoEALsBAAAA +Dx9EAADoO78BAEiLRCQg6JG9AQBIjQXJ+gQAuwIAAAAPH0QAAOgbvwEA6Ha2AQBIjQVr+wQAuwQA +AABIi0wkKEiLfCQg6LtmAABIjQXy+gQAuwMAAABIi0wkQEjHx/////8PHwDom2YAAEmLRjDGgCkB +AAACSI0FlSwFALsfAAAADx9AAOg7nQEAkEiJRCQISIlcJBBIiUwkGEiJfCQgQIh0JChMiUQkMOg3 +BwQASItEJAhIi1wkEEiLTCQYSIt8JCAPtnQkKEyLRCQw6RT+///MzMzMzMzMzMzMzMzMzMzMzMzM +zEiD7EhIiWwkQEiNbCRASIlcJDhIiUQkKLkDAAAAvyIAAAC+/////0UxwEiJwjHASInT6KkN//9m +Dx+EAAAAAABIhdsPhIoAAABIg/sNdT5IiVwkIOjntAEASI0F0icFALsdAAAA6Pa9AQDoUbUBAMcE +JAIAAADopR8EAEUPV/9kTIs0Jfj///9Ii1wkIEiD+wt1NOiotAEASI0FNEkFALs7AAAA6Le9AQDo +ErUBAMcEJAIAAADoZh8EAEUPV/9kTIs0Jfj///8xwEiLbCRASIPESMNIiUQkMEiLRCQ4SItcJCjo +uVABAEiLRCQwSItsJEBIg8RIw8zMzMzMzMzMzMxJO2YQD4YEAgAASIPsQEiJbCQ4SI1sJDhIiUQk +SEiJXCRQSIsNV4QNAEiFyQ+EEAEAAEiJwkiNcf9IhfB0C0iJz0j32UghwesFSInPMclMjQQDTIXG +dBNIjTQDSI12/0mJ+Ej330gh/usFSYn4MfZIhckPhIgAAABKjTwBDx9AAEg5/nRISIXJdHZIiXQk +KEiJTCQwSIkMJEyJRCQIx0QkEA8AAADoFiUEAEUPV/9kTIs0Jfj///9Ii0wkMEiLVCRISItcJFBI +i3QkKOszSIkMJEnR4EyJRCQIx0QkEA8AAADo2iQEAEUPV/9kTIs0Jfj///9Ii1QkSEiLXCRQ60aQ +SIX2dEBIOc50O0iLBW+DDQBIiTQkSIlEJAjHRCQQDwAAAOiZJAQARQ9X/2RMizQl+P///0iLVCRI +SItcJFBmkOsDSInCSInRSIs1OYMNAEj/zkiF8g+FoQAAAA8fRAAASIXeD4WTAAAAgz2UhQ0AAHQH +uAQAAADrBosFpWIKAIlEJCRIiQwkSIlcJAiJRCQQ6CskBABFD1f/ZEyLNCX4////i0QkGItMJCSD ++Qh1Q4XAdD+4BAAAAEiNDWViCgCHAUiLRCRISIkEJEiLRCRQSIlEJAjHRCQQBAAAAA8fQADo2yME +AEUPV/9kTIs0Jfj///9Ii2wkOEiDxEDDSI0Fow8FALsTAAAA6NOZAQCQSIlEJAhIiVwkEOjjAwQA +SItEJAhIi1wkEOnU/f//zMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdnZIg+woSIlsJCBIjWwkIEiL +DSWCDQAPH0QAAEiFyXRNkEiNFANIjQQISI1A/0j32UghyEghyg8fRAAASDnCdi1IicFIiQQkSCnK +SIlUJAjHRCQQDgAAAA8fQADoGyMEAEUPV/9kTIs0Jfj///9Ii2wkIEiDxCjDSIlEJAhIiVwkEOg1 +AwQASItEJAhIi1wkEOlm////zMzMzMzMSIPsIEiJbCQYSI1sJBhIiVwkMEiJRCQQSPfbSInIZpDo +m00BAEiLRCQQSItcJDDoTAv//0iLbCQYSIPEIMPMzEk7ZhAPho0AAABIg+wwSIlsJChIjWwkKEiJ +RCQ4SIlcJCBIicjoVk0BAEiLRCQ4SItcJCC5AwAAAL8yAAAAvv////9FMcDodQn//0iD+wx0NEiL +TCQ4SDnIdRQPH0QAAEiF23UKSItsJChIg8Qww0iNBek+BQC7MAAAAA8fRAAA6DuYAQBIjQWQEwUA +uxYAAADoKpgBAJBIiUQkCEiJXCQQSIlMJBjoNQIEAEiLRCQISItcJBBIi0wkGOlB////zEk7ZhAP +hhwCAABIg+wgSIlsJBhIjWwkGIM9JX8NAACQD4XqAQAASIl8JEBIiUwkOEiJRCQoSIl0JEhIiVwk +MJCQSI0FyH8NAOiTbf//SIsNxH8NAEiFyXQGg3kQZXVwSIM9oX8NAAB1QrgAEAAAMdtIjQ2Blg0A +6ASQ//9IiQWFfw0ASIsVDn8NAEiJEEiLFXR/DQBIiRX9fg0AgD2+gQ0AAHUHMcDpKQEAAEiLFVZ/ +DQBIi1oISIkdS38NAEiLHVR/DQBIiVoISIkVSX8NAEiLFUJ/DQCLQhAPH4AAAAAASIP4ZQ+DxwAA +ALkBAAAA8A/BShBIjQyASI08ykiNfxhIjRzKSI1bMEiNNMpIjXY4TI0Eyk2NQCCDPQGADQAAkHUM +TItMJDBMiUzKGOsKTItMJDDoSAUEAEyLTCQ4TIlMyiiDPdd/DQAAdSBIi1wkQEiJXMowSItcJEhI +iVzKOEiLXCQoSIlcyiDrJ0iJ30iLTCRA6GgEBABIifdIi0wkSOhbBAQATInHSItMJCjoTgQEAMYF +b30NAAGQkEiNBW5+DQDoGW7//0iLbCQYSIPEIMO5ZQAAAOjFBwQASI0Vll4KAA+2HAJIjTWTgA0A +iBwOSI1BAUiD+EAPjc3+//9IicFIuM3MzMzMzMzMSPfpSAHKSMH6AkiNFJJIichIKdCQSIP4BXK1 +uQUAAADocAcEAEiNBTMWBQC7GAAAAA8fQADo25UBAJBIiUQkCEiJXCQQSIlMJBhIiXwkIEiJdCQo +kOjb/wMASItEJAhIi1wkEEiLTCQYSIt8JCBIi3QkKGaQ6Zv9///MzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMxJO2YQdm5Ig+wYSIlsJBBIjWwkEJBIjQV0fQ0ADx9AAOg7a///gD1bfA0AAHQggD1T +fA0AAHQXxgVJfA0AAMYFQ3wNAABIiw2EkQoA6wIxyUiJTCQIkJBIjQUyfQ0AZpDo22z//0iLRCQI +SItsJBBIg8QYw+gn/wMA64XMzMzMzEk7ZhAPhgQBAABIg+wwSIlsJChIjWwkKEiLMA8fRAAASIX2 +D4S4AAAASIt4GEiF/3Q2SIsPSIlIGEiLCEgBSDCAeEAAdBVIiXwkGEiLGEiJ+OiKDAQASIt8JBhI +ifhIi2wkKEiDxDDDSIlEJDiLeChIOf52I0iLSDgx27gAQAAA6PuM//9Ii1QkOEiJQiDHQigAQAAA +SInQSItYIEiLUAhIhdJ0G0iJXCQgSIsKSItwEEiJ8P/RSItEJDhIi1wkIEiLCEgBSCBIiwgpSChI +iwhIAUgwSInYSItsJChIg8Qww+h6rAEASI0FAz4FALs0AAAA6Im1AQDo5KwBAEiNBUQSBQC7FwAA +AOjzkwEAkEiJRCQI6Aj+AwBIi0QkCA8fAOnb/v//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +STtmEHZISIPsGEiJbCQQSI1sJBDHBa4RDQABAAAAZpDoO3MAAInDSI0F8oENAOjNYgAASLgBAAAA +AQAAAEiJBbiGDQBIi2wkEEiDxBjD6I39AwDrq8zMzMzMzMzMzMzMSTtmEA+GsAAAAEiD7BhIiWwk +EEiNbCQQSI0FIXcEALsCAAAA6BcU//+DPVB8DQAAdQlIiQV/jwoA6wxIjT12jwoA6Pn/AwAxwEiN +HehDBQDoizwCADHASI0d0kMFAGaQ6Hs8AgBIiwVMjwoAMdvozSL//0iLBT6PCgAx2w8fQADouyL/ +/4M99HsNAAB1DUjHBR+PCgAAAAAA6w5IjT0WjwoAMcDol/8DAMYF2KENAAFIi2wkEEiDxBjD6MH8 +AwCQ6Tv////MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4adAAAASIPsEEiJbCQISI1s +JAjoAxgEAEUPV/9kTIs0Jfj///9IiwUngQ0ASIsMJEiJykgpwQ8fRAAASIXJfw+4AQAAAEiLbCQI +SIPEEMNJi14wSIub0AAAAIQDSIuzkBYAAEgp8kgDk4AWAAAPV8DySA8qwg9XyfJIDyrJ8g9ewfIP +EA2yQgYA8g9ZDeKADQBmDy7BD5fASItsJAhIg8QQw+j0+wMA6U/////MzMzMzMzMzMzMzMzMzMxJ +O2YQD4ZgBgAASIPseEiJbCRwSI1sJHBJi1YwkP+CCAEAAEmLVjBMifaEBkyJ9kg5MnRTRIuCCAEA +AA8fRAAAQYP4AX9BSIO6AAEAAAB1N5BFjUj/RImKCAEAAA8fQABBg/gBdRGAvrEAAAAAdAhIx0YQ +3vr//4lMJCxIiUQkSEiJXCRA6zmQi4IIAQAAjUj/iYoIAQAAg/gBdRGAvrEAAAAAdAhIx0YQ3vr/ +/0iLbCRwSIPEeMP/BbePCgBIifOAPSGgDQAAD4SbAAAAgz1MeA0AAA+FjgAAAIM9I3gNAAAPhYEA +AABIhcB1FkiLFV1/DQBIORU+fw0AD5bCSIne62tIg/gBdTiDPRF/DQAAfQdIid4x0utVSIsVOaYN +AJBIhdJ0EkiJ3kgp00g5Hf5YCgAPnMLrNkiJ3jHS6y9mkEiD+AJ1GIs1FIQNAEGJyCnxhckPn8JE +icFIid7rD0iJ3roBAAAA6wVIid4x0oTSdQQx0usc6KaTAABIg/j/D5XAi0wkLEiLdCRAicJIi0Qk +SITSD4Ue////kEiNBVuDDQAx2zHJMf/oVJ8CAIA9NZ8NAAAPhJMAAACDPWB3DQAAD4WGAAAAgz03 +dw0AAHV9SItUJEhIhdJ1FEiLNXB+DQBIOTVRfg0AQA+WxutmSIP6AXU0gz0mfg0AAH0GMfZmkOtR +SIs1T6UNAEiF9nQVTItEJEBJKfBMOQUTWAoAQA+cxuswMfbrLEiD+gJ1GESLBSyDDQBEi0wkLEUp +wUWFyUAPn8brDr4BAAAA6wdIi1QkSDH2QIT2D4SxAAAASIP6Ag+UBbmCDQCLFWt6DQCD+gF1B7oB +AAAA6xGD+gK6AAAAAL4CAAAASA9E1kiJVCQ4SI0FdlcKADHbMckx/w8fQADoW54CAJBIjQVnVwoA +MdsxyTH/6EieAgCAPRELCwAAkHQ5kEjHRCRQAAAAAEiLFS0LCwBIiVQkULgHAAAAuwMAAABIjUwk +UL8BAAAASIn+6CxTAwBI/wUFCwsASIsFDowKAEiLDQ+MCgAx0usekEiNBeeBDQAx2zHJ6CKhAgBI +i2wkcEiDxHjDSP/CSDnKfR1IixzQSItzQIu2qAQAAIs9hAwNADn+dODplAIAAOguFQAASI0FL0AF +AEiJBCQPHwDoO/cDAEUPV/9kTIs0Jfj///+LBZB1DQCJBeaBDQCJBeSBDQCLDYJ1DQA5wX0GiQ3Q +gQ0ASIsFtXwNAEiJBf6BDQBIxwXjgQ0AAAAAAEiLRCQ4SIkFX4ENAOiiEwQARQ9X/2RMizQl+P// +/0iLBCRIiUQkMEiJBZWBDQBIiQW2gQ0AgD3nCQsAAHQxkEjHRCRQAAAAAEjHRCRQAQAAALgJAAAA +SMfD/////0iNTCRQvwEAAABIif7oBFIDAEiNBYVABQBIiQQk6HT2AwBFD1f/ZEyLNCX4////SI0F +UD8FAEiJBCToV/YDAEUPV/9kTIs0Jfj////oRSIAAP8F/4ANAEiNBbh7DQDo81wAAEiLBcx7DQBI +iQU9gQ0ASItEJDhIhcB0DDHA6FRwAgBIi0QkOJC5AQAAAEiNFWZ0DQCHCosNXnQNAIP5AXUHuQEA +AADrBoP5Ag+UwYgNJnYNAGaQhMl0B7kBAAAA6wcPtg0Tdg0AiA0Idg0AkMcFzX8NAP/////HBdN/ +DQD/////6A4lAADoKVsAALgBAAAASI0N+XMNAIcBSItEJDBIiQWHew0ASYtGMJD/gAgBAABJi0Yw +SIlEJFhEDxF8JGBIjQ1uAQAASIlMJGBIjUwkMEiJTCRoSI1MJGBIiQwk6FH1AwBFD1f/ZEyLNCX4 +////kEiNBbBUCgAx2zHJ6NOeAgCQSItEJFiLiAgBAACNUf+JkAgBAACD+QF1JkGAvrEAAAAAdBJJ +x0YQ3vr//0iLVCQ4SIXS6xJIi1QkOEiF0usISItUJDhIhdJ0DpBIjQXpPQUAkOhb9AMAkEiNBS9/ +DQAx2zHJ6GqeAgBIi2wkcEiDxHiQw4l0JCiJfCQsSGMDSIlEJEDoKqQBAEiNBVfzBAC7CwAAAOg5 +rQEASItEJEDoL6sBAEiNBV7wBAC7CgAAAA8fAOgbrQEAi0QkKInA6BCqAQBIjQVx9QQAuw0AAAAP +H0AA6PusAQCLRCQs6PKpAQDoTaYBAOhIpAEASI0F3AIFALsUAAAA6FeLAQCQSIlEJAhIiVwkEIlM +JBjoY/UDAEiLRCQISItcJBCLTCQY6XD5///MzMzMzMzMzMzMzMzMzMzMSTtmEA+GHAEAAEiD7CBI +iWwkGEiNbCQYSItKCEiJTCQQD7YFCAcLAOgT2AEASItMJBBIiQFIixW8fg0ASCnQSAEFqn4NAEiL +EUiJFYh+DQBIiwlIKw2efg0ASIXJfRa4AQAAAEiNDc3EDQDwSA/BAemXAAAASIP5EHxnSA+90UjH +w/////9ID0TTSI1a/UiJ3kjB4wRIgfvQAgAAcgy+LAAAALoPAAAA6z1Ig8L8SIP6QEgZ/0j310gJ ++kiJyEiJ0UjT+EiJwkjB+D9Iweg8SI08EEjB/wRIwecESCn6ZpDrBTH2SInKSMHmBEiNBBZIPdAC +AABzH0iNDbqtDQBIjQTBuQEAAADwSA/BCEiLbCQYSIPEIMO50AIAAA8fRAAA6Jv7AwCQ6HXzAwDp +0P7//8zMzMzMzMzMzMzMzMzMzMxJO2YQD4aYAgAASIPsUEiJbCRISI1sJEhIjQUBfQ0AMdsxyTH/ +6PaYAgDrLkiNBUU7BQBIiQQk6ITyAwBFD1f/ZEyLNCX4////kEiNBeNRCgAx2zHJ6AacAgCDPcNw +DQABD4UgAgAAixVzfA0AORVdfA0AD4UOAgAASIsVwHsNAEiF0nUZixU9fA0AORU7fA0Adge4AQAA +AOsJMcDrBbgBAAAAhMAPhdwBAACQSI0FgVEKADHbMckx/+himAIAxwVYcA0AAAAAAEiNFaE6BQBI +iRQk6OjxAwBFD1f/ZEyLNCX4////gz00cA0AAHQXkEiNBT5RCgAx2zHJ6GGbAgCQ6VX///+Q6HUO +BABFD1f/ZEyLNCX4////SIsEJEiJBX18DQBIiQWOfA0ASYtGMEjHgAABAAAFAAAAgz3AcQ0AAHUQ +SI0VPOcEAEiJkPgAAADrE0iNuPgAAABIjRUl5wQA6Hv2AwCAPYQECwAAdDGQSMdEJDAAAAAASMdE +JDAAAAAAuAkAAABIx8P/////SI1MJDC/AQAAAEiJ/uihTAMASI0FIjsFAEiJBCToEfEDAEUPV/9k +TIs0Jfj////GRCQvAEQPEXwkOEiNBfIAAABIiUQkOEiNTCQvSIlMJEBIjUwkOEiJDCTo1fADAEUP +V/9kTIs0Jfj///+AfCQvAHQ7SYtGMEjHgAABAAAAAAAAgz3rcA0AAHUQSMeA+AAAAAAAAADpBf7/ +/0iNuPgAAAAxwOiL9AMA6fL9//8xwEiNDdluDQCHAeg2MwAAkEiNBc56DQAx2zHJ6AWaAgC4AQAA +AOibagIAD7Yd1HoNAEiNBc11DQDoqFoAAOgDAQAASItsJEhIg8RQw5BIjQWReg0AMdsxyejImQIA +SItsJEhIg8RQw+hZ8QMA6VT9///MzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhqIAAABIg+w4SIls +JDBIjWwkMEiLSghIiUwkKEiLFUiECgBIiVQkIEiLHUSECgBIiVwkEDHA6xhIi3QkCEiNRgFIi0wk +KEiLVCQgSItcJBBIOdh9SEiJRCQISIsMwkiJTCQYSInI6DA/AQBIi0wkGIQBSIuRmBYAAGaQSIXS +dLhIg3oQAHUOSIuJoBYAAEiDeRAAdKNIi0QkKMYAAUiLbCQwSIPEOMPo7+8DAOlK////zMzMzMzM +zMzMzEyNpCTg/v//TTtmEA+GtAwAAEiB7KABAABIiawkmAEAAEiNrCSYAQAAuAIAAABIjQ1vbQ0A +hwGLBWdtDQAPHwCD+AF1B7gBAAAA6waD+AIPlMDyDxGEJKAAAACIBSNvDQCEwHQHuAEAAADrBw+2 +BRJvDQCIBQdvDQBIiwVwdA0ASIkFwXkNAJDocwsEAEUPV/9kTIs0Jfj///9Ji0YwkEiLDCRIiUwk +MP+ACAEAAEmLRjBIiYQkcAEAAEyJ8oQCSMeAAAEAAAUAAABIjbj4AAAASIm8JJABAACDPaBuDQAA +dRBIjRUc5AQASImQ+AAAAOsMSI0VDOQEAOhi8wMATIm0JIABAABJi1YwxoIpAQAAAkmLVjBIi4LA +AAAASImEJHgBAAC7AgAAALkEAAAA6K3KAQBIi5QkeAEAAMaCsAAAAAZEDxG8JKgAAABIjTWOCwAA +SIm0JKgAAABIi3QkMEiJtCSwAAAASI20JKgAAABIiTQk6MjtAwBFD1f/ZEyLNCX4////SI0VdDYF +AEiJFCToq+0DAEUPV/9kTIs0Jfj///9Ii5QkgAEAAEiLUjDGgikBAAAASIuEJHgBAAC7BAAAALkC +AAAA6BTKAQCAPZ0ACwAAdBmQuAgAAABIx8P/////Mckx/0iJ/ujSSAMASIuMJHABAABIx4EAAQAA +AAAAAIM9eG0NAAB1DUjHgfgAAAAAAAAA6w9Ii7wkkAEAADHS6DryAwCDPXdrDQAAD4V8CgAASIsN +pnINAEiJDadyDQBIiw2wgg0ASIkNsZkNAEiNBWpyDQDyDxCEJKAAAACQ6HtfAACQ6JUJBABFD1f/ +ZEyLNCX4////SIsMJEiJTCQo6JoQBABFD1f/ZEyLNCX4////SIsMJEhjVCQISIsdlXcNAEiLdCQo +SIn3SCneSAE1e3cNAEiJPWx3DQBIackAypo7SAHRSIsVa3cNAEiJ+0gp10iF/30WugEAAABIjTWU +vQ0A8EgPwRbpnQAAAEiD/xB8aEgPvddIx8b/////SA9E1kiNcv1JifBIweYESIH+0AIAAHINQbgs +AAAAug8AAADrPkiDwvxIg/pATRnJSffRTAnKSInISInRSNP/SIn6SMH/P0jB7zxIAddIwf8ESMHn +BEgp+kiJwesGRTHASIn6ScHgBEmNBBBIPdACAAAPgy8JAABIjRV7pg0ASI0Uwr4BAAAA8EgPwTJI +jRXWgQ0ASInOSIcKSI0NWZgNAEiJ2kiHGQ+2DcyRDQBIix19dg0ASI09voENAEiJHM8Ptg2zkQ0A +SI0drIkNAEiJNMtIiw1Zdg0ASAENkoENAEiLDTN2DQBIKw0kdg0ASGMdEXYNAEgPr8tIiUwkOEiL +NR1xDQBIAzUOcQ0ASAM1F3ENAEiLPRB2DQBIKz0Bdg0ASA+v+0iJfCRYSI0cDkgB+0gDHZJ1DQBI +iR2LdQ0ASIs1JIkKAEgp8khjNVppDQBID6/WSAMVF4kKAA9XwPJIDyrDD1fJ8kgPKsryD17B8g8R +BQuRDQBIxwWUgAoAAAAAAIA9OXUNAAB0Bv8F7ZANAJCQSI0FWHUNAOijV////wXVkA0ASI0FTnUN +AOjRBgIAkJBIjQU4dQ0A6GNZ//8PHwDoexYBAIsNnf8MAIlMJEDGRCREAJC5AQAAAEiNFY//DADw +D8EKxkQkRAFIjQ0XMwUASIkMJOhG6gMARQ9X/2RMizQl+P///+i0FgEA6C+nAABIjQ24MgUASIkM +JA8fQADoG+oDAEUPV/9kTIs0Jfj///9IjQ3fMgUASIkMJA8fAOj76QMARQ9X/2RMizQl+P///5CA +fCREAA+ELAEAAMZEJEQAuv////9IjTUB/wwA8A/BFv/KhdJ1DYsV7f4MAIXSD5XC6wIx0oTSD4T6 +AAAAkIM92GsNAAAPjuwAAABIiwVLbw0ASImEJJgAAABIiw38/gwASImMJJAAAABIixXd/gwASImU +JIgAAADyDxAF5f4MAPIPEYQkoAAAAA8fQADo25gBAEiNBVsQBQC7HwAAAOjqoQEASIuEJJgAAABI +wegU6NmeAQBIjQVW7AQAuw4AAADoyKEBAEiLhCSYAAAASIuMJJAAAABIKchIwegU6KyeAQBIjQW3 +/AQAuxcAAADom6EBAEiLhCSIAAAA6I6eAQBIjQXR5AQAuwoAAABmkOh7oQEA8g8QhCSgAAAA6G2b +AQBIjQUe6AQAuwwAAACQ6FuhAQDotpgBAIM962oNAAB+X/IPEAWlMAYA8g9ZBd2ODQDyDxGEJKAA +AABIx4QkuAAAAAAAAABIjYQkwAAAAEQPETjo95cBAEiLBUBzDQBIKwVRaA0ASInBSLi2aWyvBb03 +hkj34UjB6hO4FwAAAOtmSI0Fu0cKADHbMckPHwDo25ECAJBIjQWfRwoAMdsxyejKkQIAkEiLlCRw +AQAAi7IIAQAAjX7/iboIAQAAg/4BdRJBgL6xAAAAAHQIScdGEN76//9Ii6wkmAEAAEiBxKABAADD +SInwSIP6CnMIZpBIg/gUfE9IicFIuM3MzMzMzMzMSInTSPfiSMHqA0iNNJJI0eZIKfNIg/kYD4MH +BQAASI1zMECItAy4AAAASI1x/0iD+RV1rsaEDLcAAAAuSI1x/uugSIP4GA+DzwQAAEiDwjCIlAS4 +AAAAixWojQ0ASImUJJgAAABIjUjoSInOSMH5P0ghyEiNnAS4AAAASPfeSI2EJCgBAABIifEPH0AA +6BsCAwBIiYQkiAEAAEiJnCSAAAAA6KaWAQBIjQW02wQAuwMAAADotZ8BAEiLhCSYAAAA6KicAQBI +jQU22wQAuwIAAADol58BAEiLhCSIAQAASIucJIAAAADogp8BAEiNBTDbBAC7AgAAAOhxnwEA8g8Q +hCSgAAAA8kgPLMAPHwDoW50BAEiNBSXbBAC7AwAAAOhKnwEA6KWWAQBIixVucQ0ASMeEJNAAAAAA +AAAASI20JNgAAABEDxE+SIs1V3ENAEiJtCTQAAAASIs1UHENAEiJtCTYAAAASIs1SXENAEiJtCTg +AAAAMcDrbUgp10iNhCS4AAAAuxgAAABIidlmkOh7EwAASInZSInDSI2EJAgBAADoCAEDAEiJhCSI +AQAASImcJIAAAADok5UBAEiLhCSIAQAASIucJIAAAAAPHwDom54BAOj2lQEASItUJGBIjUIBSItU +JEhIg/gDfUxIiUQkYEiLvMTQAAAASIl8JEhIhcAPhHL///9IiVQkeGaQ6DuVAQBIjQXX2QQAuwEA +AADoSp4BAOillQEASItUJHhIi3wkSOlB////6BGVAQBIjQUg4wQAuwsAAAAPH0QAAOgbngEA6HaV +AQBIx4QkSAEAAAAAAABIjYQkUAEAAEQPEThIjYQkYAEAAEQPEThIi0QkOEiJhCRIAQAASIsFFmsN +AEiJhCRQAQAASIsFF2sNAEgDBQhrDQBIiYQkWAEAAEiLBQlrDQBIiYQkYAEAAEiLRCRYSImEJGgB +AAAxwGaQ62BIjYQkuAAAALsYAAAASInZ6CkSAABIidlIicNIjYQk6AAAAOi2/wIASImEJIgBAABI +iZwkgAAAAOhBlAEASIuEJIgBAABIi5wkgAAAAOhMnQEA6KeUAQBIi1QkaEiNQgFIg/gFD41+AAAA +SIlEJGhIi7zESAEAAEiJfCRQZpBIg/gCdAZIg/gDdSjo75MBAEiNBY/YBAC7AQAAAA8fAOj7nAEA +6FaUAQBIi3wkUOlO////SIXAD4RF////Dx8A6LuTAQBIjQVX2AQAuwEAAADoypwBAOgllAEASItE +JGhIi3wkUOkY////SIsFD28NAEiJhCSYAAAASIsNCG8NAEiJjCSQAAAASIsVAW8NAEiJlCSIAAAA +SIsd+m4NAEiJXCRwSGM1nm4NAEiJdCR46EyTAQBIjQX/3AQAuwkAAADoW5wBAEiLhCSYAAAASMHo +FOhKmQEASI0F5tcEALsCAAAA6DmcAQBIi4QkkAAAAEjB6BToKJkBAEiNBcTXBAC7AgAAAOgXnAEA +SIuEJIgAAABIwegU6AaZAQBIjQW72AQAuwUAAADo9ZsBAEiLRCRwSMHoFOjnmAEASI0FAt8EALsK +AAAA6NabAQBIi0QkeOjMmQEASI0FXNcEALsCAAAA6LubAQDoFpMBAIA9j20NAAB0G+iIkgEASI0F +6tsEALsJAAAA6JebAQDo8pIBAOhtkgEA6OiUAQDo45IBAEmLRjCQD7aIFwEAAI1R/4iQFwEAAID5 +AQ+Fd/r//5CQSI0FxGENAOiXUf//6WT6//+5GAAAAOhI6wMASInIuRgAAADoO+sDALnQAgAA6FHr +AwBIjQVOBAUAux0AAAAPH0QAAOibeQEAkPIPEUQkCOiv4wMA8g8QRCQI6STz///MzMzMSTtmEHYh +SIPsEEiJbCQISI1sJAhIi0II6KMGAABIi2wkCEiDxBDD6NTiAwDr0szMzMzMzMzMzMzMzMzMzMzM +zEk7ZhB2XEiD7BhIiWwkEEiNbCQQ6zQxwEiNHaEqBQCQ6JsiAgBIjQVcbA0ASMfD/////+iIVf// +kEjHBURsDQAAAAAA/wUuYA0Aiw0oYA0AOQ0yYA0Af75Ii2wkEEiDxBjD6PniAwDrl8zMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMSTtmEA+GPAQAAEiD7HhIiWwkcEiNbCRwTIl0JEhJi04wSMeBAAEAAA4A +AACDPa1hDQAAdRBIjRV65AQASImR+AAAAOsTSI25+AAAAEiNDWPkBADoSOYDAEiNBeGYBACQ6Htt +//9Ii0wkSEiLUTBIx4IAAQAAAAAAAIM9YGENAAB1DUjHgvgAAAAAAAAA6w5Ijbr4AAAAMdLoI+YD +AEiJRCRAkEiJSBBJi04wkP+BCAEAAJBJi04wSIlIGEiNBUZrDQDoeVD//0iNBWopBQBIi1wkQLkY +AAAAvxQAAAAx9pDoG6IBAEmLVjCQ/4IIAQAAkEmLVjBMi0QkQEmJUBhIi1QkSEyLSjBNi4nQAAAA +gz3nXg0AAA+E2wIAAEGEAUmDuYgWAAAAD4S5AgAATIlMJFCQ6Cn9AwBFD1f/ZEyLNCX4////SIsE +JEiLTCRQSImBkBYAALr/////SI0dYGoNAPAPwRP/yos1RGoNADnWD4QRAgAASIlEJDBEDxF8JFhI +x0QkaAAAAABIjQXRAgAASIlEJFhIi1QkSEiJVCRgSIlMJGhIjVwkWEiJHCTo798DAEUPV/9kTIs0 +Jfj///9mkOib/AMARQ9X/2RMizQl+P///0iLBCRIi0wkMEgpyEiLTCRQSIuRiBYAAEiD+gF1H0iN +FYllDQDwSA/BAroBAAAATI0FmGUNAPBJD8EQ6zJIg/oCdRpIjRVsZQ0ASYnA8EgPwQLwTA/BgYAW +AADrEkiD+gN1DEiNFVRlDQDwSA/BAroBAAAATI0Fc2kNAPBBD8EQ/8JEiwVVaQ0ADx9EAABBOdAP +go0AAABIx4GIFgAAAAAAADkVNmkNAHUwSIsVnWgNAEiF0nUaixUaaQ0AORUYaQ0Adge4AQAAAOsK +McCQ6wW4AQAAAIPwAesCMcCEwA+EEf7//0iLRCRASItIGIuRCAEAAI1a/4mZCAEAAIP6AXUSQYC+ +sQAAAAB0CEnHRhDe+v//kDHJSIlIGOjv6///6dH9//+JVCQkRIlEJCxIi4GIFgAASIlEJDjoEI4B +AEiNBRgBBQC7HQAAAA8fQADoG5cBAEiLRCQ46BGVAQBIjQUO4AQAuw0AAAAPH0QAAOj7lgEAi0Qk +JOjykwEASI0F4t8EALsNAAAA6OGWAQCLRCQs6NiTAQDoM5ABAOgujgEASI0FovQEALsXAAAAZpDo +O3UBAIlUJCiJdCQs6I6NAQBIjQW57gQAuxUAAABmkOiblgEAi0QkKOiSkwEASI0Fgt8EALsNAAAA +6IGWAQCLRCQs6HiTAQDo048BAOjOjQEASI0FXPwEALsbAAAAZpDo23QBAEiNBWD9BAC7HAAAAOjK +dAEAQYQBSYuBiBYAAEiJRCQ46BaNAQBIjQWL3gQAuwwAAADoJZYBAEiLRCQ46BuUAQDodo8BAOhx +jQEASI0F6Q8FALsmAAAADx9EAADoe3QBAJDold4DAOmw+///zMzMzMzMzMzMzMzMzMzMzEk7ZhAP +hnwBAABIg+xISIlsJEBIjWwkQEiLchBIiXQkKEiLQghIiUQkMLsCAAAAuQQAAADop7kBAEiLVCQo +hAJIi7KIFgAASIP+AXQ9SIP+AnQgSIP+Aw+FFQEAAEiNgpgWAAC7BwAAAOhyLAAA6eEAAABIjYKY +FgAAuwsAAACQ6FssAADpygAAAEiNgpgWAABIiUQkOLsDAAAADx9EAADoOywAAEiLTCQwgLmxAAAA +AA+EjgAAAEiLRCQoDx9AAOj7YQIASIlEJBhIiVwkIIXJdnKJTCQUkJBIjQVYeQoA6FtJ//+QSItM +JCBIi1QkGEiFyXQxSInLSMeBoAAAAAAAAABIiw1+eQoASIXJdAlIiZGgAAAA6wdIiRVheQoASIkd +YnkKAItMJBQBDWB5CgBEDxF8JBiQkEiNBfl4CgCQ6NtK//9Ii0QkOLsCAAAA6IwrAABIi0QkMLsE +AAAAuQIAAADoeLgBAEiLbCRASIPESMNIjQXAEwUAuysAAABmkOjbcgEAkOhV3AMA6XD+///MzMzM +zMzMzMzMzMzMzMzMTI1kJPBNO2YQD4bIBAAASIHskAAAAEiJrCSIAAAASI2sJIgAAACDPdNdDQAA +fhVIiYQkmAAAAOgEEwEASIuEJJgAAACDPblZDQACD4V0BAAASIkFYGUNAEiLFcFkDQBIiVQkaEiF +0g+FCQMAAESLBTRlDQBmDx+EAAAAAAAPHwBEOQUlZQ0AD4fpAgAAgz04XQ0AAH4F6EUMAABIgz19 +ZA0AAA+FvAIAAEiLFWBvCgBIiZQkgAAAAEyLBVlvCgBMiUQkYDHA6yJIjYGYFgAA6MyQAABIi0wk +WEiNQQFIi5QkgAAAAEyLRCRgTDnAfWRIiUQkWEiLDMJIiUwkeIM9yFwNAAB+CkiJyOgyKgEA6w6E +AUiNgcAWAADo4igBAEiLTCR4hAFIi5GYFgAASIXSdJdIg3oQAA+F0QAAAEiLkaAWAABIg3oQAA+E +ev///+m6AAAASIsVRmQNAEiJFf9fDQBIixWobgoATIsFqW4KADHA6wNI/8BMOcB9KUyLDMJNi0lA +TYXJdOtMixXJXw0ATQNRCEyJFb5fDQBJx0EIAAAAAOvPSIsV9WMNAEiJFZ5fDQBIixWvXw0ASIkV +mF8NAIA9Ce0KAAB0NJBIx0QkcAAAAABIixV2Xw0ASIlUJHC4IQAAAEjHw/////9IjUwkcL8BAAAA +SIn+6CM1AwBIi6wkiAAAAEiBxJAAAADD6A6JAQBIi0QkeEhjCEiJTCRQD7aQuBYAAIhUJC/o8YgB +AEiNBRPYBAC7CwAAAA8fRAAA6PuRAQBIi0QkUOjxjwEASI0FhtoEALsNAAAADx9EAADo25EBAA+2 +RCQv6HGLAQDoLIkBAEiLRCR4SIuImBYAAEiFyQ+EtAAAAEiLQRBIiUQkYOiJiAEASI0FhNIEALsJ +AAAA6JiRAQBIi0QkYOiOjwEA6OmIAQBIi0QkeEiLgKAWAABIhcB0WEiLQBBIiUQkYOhKiAEASI0F +TtIEALsJAAAA6FmRAQBIi0QkYOhPjwEA6KqIAQDoJYgBAA8fRAAA6JuKAQDologBAEiNBcEVBQC7 +LwAAAOilbwEADx9EAADo+4cBAEiNBSzYBAC7DAAAAOgKkQEA6GWIAQDruQ8fAOjbhwEASI0FANgE +ALsMAAAA6OqQAQDoRYgBAA8fRAAA6VL///9IjQVG3AQAuw4AAADoSm8BAIsFLGINAEiJRCRIiw0l +Yg0ASIlMJEBIixU1Yg0ASIlUJGBIix0xYg0ASIlcJFhIizUtYg0ASIl0JDhIiz0pYg0ASIl8JDAP +H0AA6FuHAQBIjQWE2wQAuw4AAADoapABAEiLRCRoDx9EAADou44BAEiNBQbOBAC7BgAAAOhKkAEA +SItEJEgPH0QAAOg7jQEASI0F1M0EALsGAAAA6CqQAQBIi0QkQA8fRAAA6BuNAQBIjQW01gQAuwwA +AADoCpABAEiLRCRgDx9EAADo+40BAEiNBfXUBAC7CwAAAOjqjwEASItEJFgPH0QAAOjbjQEASI0F +gNYEALsMAAAA6MqPAQBIi0QkOA8fRAAA6LuNAQBIjQV32AQAuw0AAADoqo8BAEiLRCQwDx9EAADo +m40BAOj2iAEA6PGGAQBIjQVqVwQASI0dUyAGAA8fAOhbZgEASI0FkxkFALs4AAAA6OptAQCQSIlE +JAgPH0AA6PvXAwBIi0QkCOkR+///zMzMzMzMzMzMzMzMzMzMzMxJO2YYD4Z8AQAASIPsIEiJbCQY +SI1sJBiDPeVUDQAAkA+FTgEAAEiJRCQokJBIjQVs6gsA6GdD//+DBYjrDAACxwWC6wwAAAAAAEjH +BavrDAAAAAAASIsNLOwMAEiLFS3sDABIix0u7AwASIkVN+wMAEiJHTjsDACDPWFWDQAAkHUJSIkN +F+wMAOsMSI09DuwMAOgJ2wMARA8RPYnrDACQkEiNBfjpCwDo00T//5AxyUiNFcVrCgCHCkiLTCQo +ZpBIg/kCdSmQkEiNBdHpCwDozEL//w9XwPIPEQU56wwAkJBIjQW46QsA6JNE///rVJCQSI0FaGsK +AOijQv//gD1sawoAAHQfxgVjawoAAEiLBVRrCgAx27kBAAAADx9EAADoe6oBAJCQSI0FMmsKAOhN +RP//SItsJBhIg8Qgw/8FNWsKAOjYbwAASIP4/3Xv6E2SAAAxwOhGkwAAhMB19WaQ6DsBAQDotgEB +AEiLbCQYSIPEIMNIjQUCCwUAuykAAADoO2wBAJBIiUQkCOiQ/QMASItEJAjpZv7//8zMzMzMzEk7 +ZhgPhtwAAABIg+wwSIlsJChIjWwkKEiNBcEdBQCQ6DuhAQCQkEiNBdLoCwDozUH//0iLDbbqDABI +iUwkEEiLFaLqDABIiVQkIJCQSI0FrOgLAOiHQ///SItMJBBIi1QkIDHA6zFIiUQkGEiLDP6EAUiN +gQAEIQC7AAQAAGaQ6HvjAwBIi0wkGEiNQQFIi0wkEEiLVCQgSDnIfRhIizX06QwAhAZIizzCSIH/ +AABAAHK06yNIxwVAXg0AAAAAAEiLBelZDQBIiQXCXg0ASItsJChIg8Qww0iJ+LkAAEAA6NvcAwCQ +6JX8AwDpEP///8zMzMzMzMzMzMzMzMzMzMxJO2YQD4YmAQAASIPsEEiJbCQISI1sJAhIixV5ZwoA +kEiF0nQFSIsC/9CQkEiNBWVxCgDoyED//0iLDWFxCgCQ6wNIidFIhcl0JUiLUQiDPetTDQAAdQtI +x0EIAAAAAJDr4EiNeQgxyeiT2AMA69ODPcpTDQAAdQ1IxwUdcQoAAAAAAOsOSI09FHEKADHJ6G3Y +AwCQkEiNBfxwCgAPH0AA6DtC//+QSI0F+3AKAOhOQP//McDrA0j/wEiD+AV9LUiNDepwCgBIixTB +6zuDPW1TDQAAdQpIxwTBAAAAAOvXSI08wTHJ6BbYAwDrypCQSI0Fs3AKAOjmQf//SItsJAhIg8QQ +w0iJ2kiF0nTASItaKIM9KVMNAAB1CkjHQigAAAAA6+FIjXooMdLo8tcDAOvU6AvUAwDpxv7//8zM +zMzMzEiD7BhIiWwkEEiNbCQQSIlEJCBIgf+AlpgAciNIicJIuLZpbK8FvTeGSInWSPfnSMHqE0iN +e/9IifjpOAEAAEjR70iJwki4PN9PjZduEoNIidZI9+dIweoISIXSdAe4AwAAAOtPSIXbdijGBjBI +g/kBchJIifC7AQAAAEiLbCQQSIPEGMNIicq5AQAAAOhO2wMAMcBIidnoxNoDAEiJx0i4zczMzMzM +zMxI9+JIweoDSI1H/0iD+mRz4kiNe/9JidhIKcNMjUv/6wNIicdIg/oKcwVMOc98SEi4zczMzMzM +zMxIidNI9+JIweoDTI0UkknR4kwp00k5+HZrTI1TMESIFD5IjUf/TDnIdb5JOcB2TcZEN/8uSI1H +/uuuDx9AAEk5+HYuSIPCMIgUPkgp+UiJykj32UjB+T9IIflIjQQxSSn4TInDSInRSItsJBBIg8QY +w0iJ+EyJwegC2gMATInB6PrZAwBIifhMicHo79kDAEiJx0yJwEiD+gpzCA8fAEg5x3xLSYnASLjN +zMzMzMzMzEmJ0Uj34kjB6gNMjRSSSdHiTSnRSDn7dmlJg8EwRIgMPkiNR/9MOcB1tQ8fQABIOcN2 +R8ZEN/8uSI1H/uuhSDn7dixIg8IwiBQ+SCn5SInKSPfZSMH5P0gh+UiNBA5IKftIidFIi2wkEEiD +xBiQw0iJ+EiJ2ehU2QMASInZ6EzZAwBIifhIidnoQdkDAJBJO2YQD4atAQAASIPsCEiJLCRIjSwk +RA8RPapaDQBIiwUDZAoASIXAdAlIiwhIi0AI6wQxwDHJMdLrBUj/wmaQSDnQfjBIixzRSIuz2AAA +AEgrs9AAAACQSI2e//8DAEjB6xJIOR1eWg0Afc9IiR1VWg0A68ZIiwWsYwoADx9AAEiFwHQJSIsI +SItACOsEMcAxyTHS6wNI/8JIOdB+NEiLHNFIi7PoAAAASCuz4AAAAJBIjZ7//wMASMHrEg8fQABI +OR0JWg0Afc1IiR0AWg0A68RIiwW35QwASIsNqOUMAEiJBdnlDABIiQXa5QwAgz3rTw0AAHULSIkN +uuUMAGaQ6wxIjT2v5QwA6JLUAwBIweAESIkFv1kNAEiLBUBODQBIiQW5WQ0AxwV3WQ0AAAAAAEiL +BZBZDQBIAwWRWQ0ASAMFklkNAEgDBZNZDQBIg8ACiQVVWQ0AxwWHWQ0AAgAAAEiLBWBZDQCNSAKJ +DXtZDQBIiw1YWQ0AjRQBjVICiRVsWQ0AAchIiw1LWQ0AjRQBjVICiRVbWQ0AAchIiw0+WQ0AjQQB +jUACiQVKWQ0ASIssJEiDxAjD6CTQAwAPH0AA6Tv+///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMxJO2YQD4ayAAAASIPsOEiJbCQwSI1sJDCLDapYDQCLFahYDQA5ync5SMdEJBgAAAAARA8RfCQg +SI0NogAAAEiJTCQgSI1MJBhIiUwkKEiNRCQg6ImbAQBIi2wkMEiDxDjDiUwkFIlUJBDo0n0BAItE +JBTo6YMBAEiNBfrCBAC7BAAAAOjYhgEAi0QkEOjPgwEASI0FR9sEALsUAAAADx8A6LuGAQDoFn4B +AEiNBdXiBAC7FwAAAOglZQEAkA8fQADoO88DAOk2////zMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7 +ZhAPhgEBAABIg+woSIlsJCBIjWwkIEiLSghIixGQSDkV+VcNAH4agLi2AAAAAHQbSI1CAUiJAUiL +bCQgSIPEKMNIi2wkIEiDxCjDSIlEJDCQi4iQAAAAiUwkFEiLkJgAAABIiVQkGA+2mLYAAACIXCQT +6Od8AQBIjQX4wQQAuwMAAADo9oUBAEiLRCQw6GyFAQBIjQV/wwQAuwYAAADo24UBAEiLRCQY6NGD +AQBIjQWTxQQAuwgAAAAPH0QAAOi7hQEAi0QkFInA6LCCAQBIjQUZzAQAuwwAAAAPH0AA6JuFAQAP +tkQkE+gxfwEA6Ox+AQDo53wBAEiNBeHRBAC7DwAAAOj2YwEAkEiJRCQI6GvNAwBIi0QkCOnh/v// +zEk7ZhAPhtECAABIg+x4SIlsJHBIjWwkcImcJIgAAABIiYQkgAAAADkd21YNAHc2OR3XVg0Adi5I +ixUKYAoAZpBIhdJ0CUyLAkiLUgjrBTHSRTHASIlUJChMiUQkSDHJkOlsAgAAOR2hVg0Adzg5HZ1W +DQB2MEiLFcxfCgAPH0AASIXSdAlMiwJIi1II6wUx0kUxwEiJVCQwTIlEJFAxyZDpwgEAAIXbdQxI +ixXISg0A6UUBAACD+wEPhMEAAACLFUxWDQA503IUOR1GVg0Adgwp0+hJBQAA6cAAAACLFTJWDQA5 +0w+CwwAAADkdKFYNAA+GtwAAAEiLDRtgCgBIizUMYAoAKdNIOdkPhpcAAABIiwzekIuRkAAAAIP6 +BHQGkIP6A3UYSIO5qAAAAAB1DkiLFaJVDQBIiZGoAAAARA8RfCRYSMdEJGgAAAAASI0VrQEAAEiJ +VCRYSIlMJGBIiUQkaEiNRCRYSIkEJOhQywMARQ9X/2RMizQl+P///+sdSI0FchQFAEiJBCToMcsD +AEUPV/9kTIs0Jfj///9Ii2wkcEiDxHjDidjos9MDAEiNBdXWBAC7EwAAAOgiYgEASIlUJDhEi0IQ +RYnATI1KGEuNHIBIweMDSI0NY0wNAEiJxzH2TInI6HYgAABIi1QkOEiLEkiLhCSAAAAASIXSdb+Q +65pIiUwkQEmLFMhMi4rgAAAATIuS6AAAAE0pykiLkvgBAABEix3iVA0Aid5EKd5IicdMichMidNI +idEPH0QAAOj7AQAASItUJEBIjUoBSIuEJIAAAABIi1QkMIucJIgAAABMi0QkUEg5yn+W6Sv///9I +iUwkQEmLFMhMi4rQAAAATIuS2AAAAE0pykiLkugBAABEix1vVA0Aid5EKd5IicdMichMidNIidHo +kQEAAEiLVCRASI1KAUiLhCSAAAAASItUJCiLnCSIAAAATItEJEhIOcp/m+nB/v//SIlEJAiJXCQQ +6BfLAwBIi0QkCItcJBDpCf3//8zMzMzMzMzMzEk7ZhAPhhYBAABIg+xASIlsJDhIjWwkOEmLdjBI +i3oQSItCCEiLlsAAAABIOcJ1D4uykAAAAIP+AkAPlMbrAjH2SIlEJCBIiXwkKEiJVCQYQIh0JBVA +hPZ0JkiJ0LsCAAAAuQQAAADo9KUBAEiLVCQYxoKwAAAAB0iLRCQgDx8A6NtqAQCE23VvSItUJCCA +urYAAAAAdXdIiUQkMIhMJBeIXCQWSInQSItcJCjoLw8AAEiLTCQgxoG2AAAAAUiLRCQwD7ZcJBYP +tkwkF+gPcAEAD7ZMJBWEyXQUSItEJBi7BAAAALkCAAAA6HKlAQBIi2wkOEiDxEDDSItEJCDGgLYA +AAABSItsJDhIg8RAw0iNBeXQBAC7EQAAAOjBXwEAkOg7yQMA6db+///MzMzMzMzMzMzMzMzMzMzM +zMzMzMzMSTtmEHZhSIPsMEiJbCQoSI1sJChIifJIweYSDx9EAABIOfN3CkiLbCQoSIPEMMNMjYYA +AAQASYnZSCnzTTnBQbgAAAQATA9Cw0gB8EjB4gxIAdFMicMx9ujDHQAASItsJChIg8Qww0iJRCQI +SIlcJBBIiUwkGEiJfCQgSIl0JCjoO8kDAEiLRCQISItcJBBIi0wkGEiLfCQgSIt0JChmkOlb//// +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+G+wAAAEiD7EhIiWwkQEiNbCRAkEiNBRhl +CgDomzT//0iLDRRlCgBIiUwkEEjHBQRlCgAAAAAAkJBIjQXzZAoA6FY2//9Ig3wkEAB0IUQPEXwk +KEiLTCQQSIlMJChIi0wkEEiJTCQwSItMJBDrNEiLbCRASIPESMNIiUwkOEiLWQhIiwHosrcCAEiL +TCQ4RA8ROZBIicpIiUwkMEiLiqAAAABIhcl10ZCQSI0Fg2QKAOgGNP//SItMJDBIi1QkKEiJVCQY +SIlMJCBIhdJ0GkiLFW5kCgBIiZGgAAAASItMJBhIiQ1bZAoAkJBIjQVCZAoA6KU1//9Ii2wkQEiD +xEjD6PbHAwDp8f7//8zMzMzMzMzMzMzMzMzMzMzMTI1kJPhNO2YQD4YfAwAASIHsiAAAAEiJrCSA +AAAASI2sJIAAAACLNZzbDABIiw2N3AwASIs9ftwMAEmJ2EjB+z9Iwes8TAHDSMH7BEg52Q+G0AIA +AEyLDdzbDABBhAFIixzfSIH7AABAAA+DpAIAAEmLHNmEA0nB4AlJgeD/HwAATInHScHoA0mB+AAE +AAAPh3ICAABJjZAA/P//SYnRSMH6P0kh0E6NBANNjYAACCEASffZSYP5QA+CPAIAAIl0JDBIiVwk +UEiJfCRgTIlEJGhIiUQkeDHJ6wZI/8EPHwBIg/lAfRhJjRQIihKE0nTqSIlMJEiIVCQuRTHJ6xdI +i6wkgAAAAEiBxIgAAADDTI1JAUyJ0UmD+Qhzv0mJykyJyUG7AQAAAEHT40SE2nTfTo0c10kBy2YP +H4QAAAAAAJBJgfsAIAAAD4OgAQAATouc2wAAIABFimNjDx+AAAAAAEGA/AEPhT8BAABIiUwkWIA9 +XUMNAAB1GkWLY1gPH0AAQTn0dA1EjW4DRTnsD4XIAAAATIlcJEBJjUN4SIlEJHCQ6Pox//9Ii0wk +QEiLkYAAAADrPZBIi0QkcOjBM///SItEJHhIi0wkWA+2VCQuSItcJFCLdCQwSIt8JGBMi0QkaEmJ +yUyLVCRI6SD///9IixJIhdJ0voB6CgF18g+3QghIi3FoSIt5GEiF9nRISIlUJDgx0kj39kgPr8ZI +AfhIi1wkeOi4GwAASItMJDhIjUEQuwgAAABIi3wkeDH2SI0NvCMKAOj3GQAASItMJEBIi1QkOOuZ +6GY2AQBEiWQkNJDou3MBAEiNBdS6BAC7BgAAAOjKfAEAi0QkNOjBeQEAkOjbdQEAi0QkMOiyeQEA +6A12AQDoCHQBAEiNBR/KBAC7EAAAAOgXWwEARIhkJC/obXMBAEiNBdTABAC7CgAAAJDoe3wBAA+2 +RCQvD7bA6G55AQDoyXUBAOjEcwEASI0FN/wEALsrAAAA6NNaAQBMidi5ACAAAOhmzAMAuUAAAABM +icroucwDAEyJwLkABAAA6AzNAwBIidi5AABAAA8fQADoO8wDAEiJ2OgTzAMAkEiJRCQISIlcJBDo +o8QDAEiLRCQISItcJBDptPz//8zMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GcwIAAEiD7FhIiWwk +UEiNbCRQSItQME2J8JBMOQJ0KkmLUDCDuggBAAAAfxNIg7oAAQAAAHUJSIlEJGAxyeseSItsJFBI +g8RYw0iLbCRQSIPEWMNIi0QkYA+2TCQvSIsV3kgNAEyLBd9IDQBMi4iAAQAATIsVkUgNAEn32Q9X +wPJJDyrBZkgPbsryD1nB8kgPLNBmSQ9uwPIPEA2rCgYA8g9ZyPJMDyzBSIH6AAABAE0PTMhBuAAA +AQBJD0zQTYXSflFJOdJ9JA9XyfJJDyrK8g9ZyPJMDyzJTAOIgAEAAEn/wUyJiIABAADrCkwBiIAB +AABJidJNidFJ99pMjR0ISA0A8E0PwRNMKcpIhdIPhBEBAACAPVDVCgAAdDKEyXUuSIlUJDCQuCsA +AAC7AQAAADHJMf9Iif4PHwDoex0DAEiLRCRgSItUJDC5AQAAAIhMJC9EDxF8JDhIx0QkSAAAAABI +jR0yAQAASIlcJDhIiUQkQEiJVCRISI1UJDhIiRQk6LXBAwBFD1f/ZEyLNCX4////SItEJGBIi4iI +AAAASIXJgz3SQQ0AAHUSSMeAiAAAAAAAAABIhclmkOsRSI24iAAAADHS6JDGAwBIhcl0Cuimzv// +SItEJGBIg7iAAQAAAH1ogLixAAAAAHQhkEiNBU4KBQDowcADAEiLTCRgD7ZUJC9IiciJ0elQ/v// +6GgEAAAPH4QAAAAAAITAD4Qx/v//6yeEyXQZkLgsAAAASMfD/////zHJMf9Iif7oeRwDAEiLbCRQ +SIPEWMMPtlQkL4TSdBmQuCwAAABIx8P/////Mckx/0iJ/uhNHAMASItsJFBIg8RYw0iJRCQI6PnB +AwBIi0QkCOlv/f//zMzMzMzMzMzMzMzMzMzMSTtmEHYpSIPsGEiJbCQQSI1sJBBIi1oQSItCCA8f +QADoGwAAAEiLbCQQSIPEGMPoDMEDAOvKzMzMzMzMzMzMzEk7ZhgPhtwCAABIg+xISIlsJEBIjWwk +QIQASI24iAAAAIM9eEANAAB1DUjHgIgAAAAAAAAA6wcxyegixQMAiw14Pg0Ahcl1FUjHgIABAAAA +AAAASItsJEBIg8RIw0iJfCQ4SIlcJDBIiUQkUJDortwDAEUPV/9kTIs0Jfj///9IiwQkuf////9I +jRXxSQ0A8A/BCv/Jix3VSQ0ADx9EAAA5yw+E3QEAAEiJRCQgSItEJFC7AgAAALkEAAAADx9AAOg7 +nAEASItUJFDGgrAAAAABSYt2MEiLttAAAACEBkiNhpgWAABIi1wkMOgREwAASIlEJCi7BAAAALkC +AAAASItEJFDo+JsBAEiLFWFFDQBmSA9uwkiLVCQoD1fJ8kgPKsryD1nI8kgPLNFIi3QkUEgDloAB +AABI/8JIiZaAAQAAugEAAABIjT00SQ0A8A/BF//Ciz0YSQ0ADx+EAAAAAAA51w+CuQAAAHVOSIsF +b0gNAEiFwHUdiwXsSA0ADx9AADkF5kgNAHYHuAEAAADrCTHA6wW4AQAAAITAdRyDPf0+DQAAdQlI +ibaIAAAA6wpIi3wkOOgIxAMA6GPbAwBFD1f/ZEyLNCX4////SItEJFBIi0AwSIuA0AAAAIQASIsM +JEiLVCQgSCnRSAOIeBYAAEiJiHgWAABIgfmIEwAAfhdIjRUyRA0A8EgPwQpIx4B4FgAAAAAAAEiL +bCRASIPESJDDiVQkFIl8JBzosm0BAEiNBd3OBAC7FQAAAOjBdgEAi0QkFOi4cwEASI0FqL8EALsN +AAAA6Kd2AQCLRCQcDx8A6JtzAQDo9m8BAOjxbQEASI0FZdQEALsXAAAADx9EAADo+1QBAIlMJBiJ +XCQc6E5tAQBIjQVZ0AQAuxYAAABmkOhbdgEAi0QkGOhScwEASI0FQr8EALsNAAAA6EF2AQCLRCQc +6DhzAQDok28BAOiObQEASI0FnMkEALsTAAAAZpDom1QBAJBIiUQkCEiJXCQQ6OvlAwBIi0QkCEiL +XCQQkOn7/P//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEHZXSIPsGEiJbCQQSI1sJBCQ +SI0FvEcNAA8fQADoGyr//0iLDbRHDQBEDxE9rEcNAEiJTCQISI1EJAhmkOg72QEAkJBIjQWKRw0A +6M0r//9Ii2wkEEiDxBjDDx8A6Bu+AwDrmczMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4YN +AQAASIPsMEiJbCQoSI1sJCiQSI0FOEcNAOibKf//iw3xOg0AhckPhMcAAABMiXQkIEiLFSlHDQBM +iwUaRw0AkEnHhqAAAAAAAAAATIsND0cNAE2FyXQMTYnyTYmRoAAAAOsLkE2J8UyJDetGDQCQTItM +JCBMiQ3mRg0ATIsNB0INAE2FyX46TIkFy0YNAEiJFcxGDQAPH0AASIXSdAkxyUiJiqAAAACQkEiN +BaFGDQDo5Cr//zHASItsJChIg8Qww5BIjQWQBQUASI0dgUYNALkLAAAAvyoAAAC+AgAAAOg1fQEA +uAEAAABIi2wkKEiDxDDDkJBIjQVVRg0A6Jgq//+4AQAAAEiLbCQoSIPEMMPo5LwDAA8fQADp2/7/ +/8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhmUBAABIg+wwSIlsJChIjWwkKEiDPQBG +DQAAdEVIiUQkOEiLDWJBDQBIiUwkIJCQSI0F3EUNAA8fQADoOyj//0iLTCQgZkgPbsFIi0wkOA9X +yfJIDyrJ8g9ZwfJIDyzI6zdIjQ3iQA0A8EgPwQFIi2wkKEiDxDDDSIlUJBhIx4CAAQAAAAAAADHb +Mcno6o8BAEiLVCQYSInRSIsFe0UNAEiFwHR/SIXJfnqQkEiFwHQeSIuQoAAAAEiJFVxFDQBIhdJ1 +C0jHBVRFDQAAAAAASIuQgAEAAEgBymaQSIXSfZhIiZCAAQAAkEjHgKAAAAAAAAAASIsNKUUNAJBI +hcl0DEiJwkiJgaAAAADrDpBIicFIiQUERQ0ASInKkEiJFQFFDQAxyUiFyX4pSIsVU0ANAGZID27C +D1fJ8kgPKsnyD1nI8kgPLMlIjRX+Pw0A8EgPwQqQkEiNBbhEDQDo+yj//0iLbCQoSIPEMMNIiUQk +COhHuwMASItEJAhmkOl7/v//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMTI2kJIj+//9NO2YY +D4Z0BgAASIHs+AEAAEiJrCTwAQAASI2sJPABAABIiYQkAAIAAIuQkAAAAA+64gwPH0QAAA+DpQUA +AIuQkAAAAA+68gyD+gJ3C4P6AXQP6e0EAACQg/oED4deAQAATInySDnCD4TGBAAASIN4cAB1GoC4 +tAAAAAB0BDHS6w+KkLkAAACE0g+UwusCMdJIiZwkCAIAAITSdQnGgLMAAAAB6xXoLsQCAEiLhCQA +AgAASIucJAgCAABIjXwkeEiNf9BIiWwk8EiNbCTw6NrDAwBIi20ASItQCEyLIEyJpCR4AQAASImU +JIABAABIg3hQAHQwSI1QUEiNDTkYCgBIid9IjXQkeLsIAAAASInQ6GQOAABIi4QkAAIAAEiLnCQI +AgAARA8RvCTYAQAASMeEJOgBAAAAAAAASI0VeAUAAEiJlCTYAQAASI1UJHhIiZQk4AEAAEiJnCTo +AQAASMcEJAAAAAAxyUiJxzH2RTHAQbn///9/TI2UJNgBAABFMdtIx8D/////SInD6GktAwBIi4Qk +AAIAAEiNnCTYAQAAMcnoMisDAEiLlCQAAgAATItiKEyJpCTQAQAA6bgAAACD+gZ1EEiLrCTwAQAA +SIHE+AEAAMOLiJAAAACJTCRUSIuQmAAAAEiJVCRw6GlnAQBIjQWWuAQAuwwAAADoeHABAEiLhCQA +AgAA6OtvAQBIjQW8rgQAuwcAAADoWnABAEiLRCRw6FBuAQBIjQWOwgQAuxMAAAAPH0AA6DtwAQCL +RCRUicDoMG0BAOiLaQEA6IZnAQBIjQXsvwQAuxEAAADolU4BAEyLhCTQAQAATYtAKEyJhCTQAQAA +TIuEJNABAABNhcAPhLUAAABJg3gYAHQqSY1AGLsIAAAASI0NmRYKAEiLvCQIAgAASI10JHjoxwwA +AEiLlCQAAgAATIuEJNABAABJg3goAHQqSY1AKLsIAAAASI0NYBYKAEiLvCQIAgAASI10JHjojgwA +AEiLlCQAAgAATIuEJNABAABBgHgFAA+EXv///0iNhCTQAQAAuwgAAABIjQ0fFgoASIu8JAgCAABI +jXQkeOhNDAAASIuUJAACAAAPH0QAAOkm////SItaIEiF23QMSI1EJHgxyejGRwAAkEiLhCSoAQAA +SIuMJLgBAAAx2+jOSwAASImEJMABAABIjUQkeJDo20gAAEiFwA+EMAEAAEiLlCR4AQAASCnQSIuU +JMABAADpaAEAADHSSIXSdM1Mi0IITYXAdMSQRTHJTIlKCE2LSBBFi1AIRYXSfAQxwOs1iFwkU0iJ +VCRgTImEJMgBAABB99pJY8JMicvoUpj//0yLSBhIi1QkYA+2XCRTTIuEJMgBAABIiUQkaIsSSAOU +JHgBAACE23Q2RYtACA8fRAAARYXAfQhB99hNY8DrA01jwEiJ0EyJw0yJyUiLvCQIAgAASI10JHjo +NRAAAOsvRYtACEWFwH0IQffYTWPA6wNNY8BIidBMicNMiclIi7wkCAIAAEiNdCR46AQLAABIi1wk +aEiF2w+E8P7//5BIjQVOyQsAuQIAAADo5JEAAA8fQADp1f7//0iLSBhIiYwkqAEAAEjHQBAAAAAA +6AJwAABIi4QkqAEAAEiFwHXaSIO8JJABAAAAdSdIg7wkoAEAAACQdRtIg7wkmAEAAAB1EEiLrCTw +AQAASIHE+AEAAMNIjQUyzgQAuxkAAADo8ksBAEiF0g+Ej/7//0SLAkE5wHYGSItSEOvpRItKBEUB +yEQ5wA+Cc/7//0iLUhjr00iNBZDLBAC7GAAAAOi0SwEAg/oCD4WH/P//i4iQAAAAiUwkWEiLkJgA +AABIiVQkcOjwYwEASI0FHbUEALsMAAAADx9AAOj7bAEASIuEJAACAADobmwBAEiNBT+rBAC7BwAA +AGaQ6NtsAQBIi0QkcOjRagEASI0FD78EALsTAAAADx9EAADou2wBAItEJFiJwOiwaQEA6AtmAQDo +BmQBAEiNBYDdBAC7IAAAAOgVSwEAi4iQAAAAiUwkXEiLkJgAAABIiVQkcOhaYwEASI0Fe8YEALsW +AAAA6GlsAQBIi4QkAAIAAJDo22sBAEiNBayqBAC7BwAAAOhKbAEASItEJHAPH0QAAOg7agEASI0F +eb4EALsTAAAA6CpsAQCLRCRcicAPH0AA6HtqAQDodmUBAOhxYwEASI0FPsYEALsWAAAADx9EAADo +e0oBAJBIiUQkCEiJXCQQ6MvbAwBIi0QkCEiLXCQQkOlb+f//zMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMSTtmEHYuSIPsIEiJbCQYSI1sJBhIi0oQSItaCA8fQADoOwAAALgBAAAASItsJBhIg8Qg +w0iJRCQISIlcJBBmkOh7swMASItEJAhIi1wkEOuvzMzMzMzMzMzMzMzMzMzMSTtmEA+GKAIAAEiD +7GhIiWwkYEiNbCRgSIsQDx9EAABIhdJ0DYB6KANBD5TASIXS6wZBuAAAAAB0CYB6KAUPlMLrAjHS +SIlMJFhIiUQkcEiJXCR4gLsQAQAAAA8fRAAAD4W/AAAARYTAD4W2AAAAhNIPha4AAAAxyeiCwAIA +SIl0JEBMiUQkMIXAfkRIiXwkUIlMJCxIY9BIweIDTItEJHBJi0A4SCnQSInZSIt8JFhIi3QkeEiJ +0+ijBwAAi0wkLEiLdCRASIt8JFBMi0QkMIXJfixIi1QkcEiLQkBIY9lIweMDSIn5SIt8JFhIi3Qk +eOhqBwAASIt0JEBMi0QkMEiLVCRwSIN6OAAPhLMAAABNhcAPjqoAAABIifAxyWaQ6a8AAABEiEQk +K4hUJCpMi0g4TYXJdDtMi1AoTSnRZpBNhcl2LUyJ0EiJz0iJ3kyJyzHJ6AgMAABIi0QkcEiLTCRY +D7ZUJCpIi1wkeEQPtkQkK0yLSEhNhcl0JUiLQEBIic9Iid5MicsxyejRCwAAD7ZUJCpIi1wkeEQP +tkQkK5BFhMB1BITSdAnGgxABAAAB6wfGgxABAAAASItsJGBIg8Row0iLbCRgSIPEaMNIg8YYSIn5 +iz5Mi0o4hf98BEyLSkBIY/9JjRw5SDlaKHc4SIlMJDhIiXQkSEiNFElIjRTQSItEJHhIidHockQA +AEiLRCRASItMJDhIi1QkcEiLdCRITItEJDBIjXkBSTn4f53rkUiJRCQISIlcJBBIiUwkGOi6sQMA +SItEJAhIi1wkEEiLTCQY6ab9///MzMzMzMxJO2YQD4a8AwAASIPsUEiJbCRISI1sJEiAPYUwDQAA +kA+EiQMAAEmLVjBIi5LAAAAASItwGEj3wwwAAAB0Lw+64wJzCUiNPcX5BADrF0j3wwgAAAC/AAAA +AEyNBaj5BABJD0X4TI2GoIYBAOsMSbj/////////fzH/SIlcJGBIiVQkOEiJRCRYSIl8JEBEiw3U +OQ0ARDkN0TkNAHYnSIl0JBBMiUQkGOlwAgAATItMJDhMicpIifNIi3wkQEiJxkiLRCRYgLqxAAAA +AHQeD7rjAHMIQbkBAAAA6xNEiw1eTQoARYXJQQ+VwesDRTHJRYTJD4WrAQAASIl0JDBMiUQkGEiD +Pdc4DQAAdSPoUGYAAEiLRCRYSItUJDhIi1wkYEiLdCQwSIt8JEBMi0QkGEyLCE2FyXQpTYtREE2F +0nUEMcnrHk2NWv9NiVkQSYH7/QAAAA+DlgEAAEuLTNEQ6wIxyUiFyXVA6BVkAABIhcB1FTHAMdvo +x/0AAEiLRCRYZpDo+2MAAEiLVCQ4SItcJGBIi3QkMEiLfCRATItEJBhIicFIi0QkWEiFyQ+E5AAA +AEiJw0iJyOjmBQAASItcJFhIi0sYSIH50AcAAA+MqgAAAEiNNVE0DQDwSA/BDkiLdCRgD7rmAXIH +SItEJDDrHUiLQxhIi0wkMEgpyOjh8v//SItcJFhIi3QkYDHASIt7GEjHQxgAAAAATItEJBhJKfiQ +TYXAf0tIi1QkQEiF0nUEMcnrLUyJRCQoSIlEJCBIiwL/0EiLVCRASItcJFhIi3QkYEyLRCQoicFI +i0QkIA8fAITJdUlJgcCghgEA6Tz+//9Ii1QkQOky/v//SItUJEBIi3QkYEyLRCQYSItEJDDpGf7/ +/0iJ3kiLRCQwSItcJFjrC0iJ8EiJ3kiLXCRYSItLGEiFyX4uSI0VazMNAPBID8EKD7rmAXMUSItL +GEgpwUiJyOgJ8v//SItcJFhIx0MYAAAAAEiLbCRISIPEUMNMidi5/QAAAOgFtgMASItEJFhIi1wk +YEyLTCQ4TInKSIt0JBBIi3wkQEyLRCQYgLqxAAAAAHQeD7rjAHMIQbkBAAAA6xNEiw0GSwoARYXJ +QQ+VwesDRTHJRYTJD4V1/f//QbkBAAAATI0VDDcNAPBFD8EKRIsdBDcNAEU52Q+DU/3//0SJy+gn +4P//SItUJEBmkEiF0nUEMcDrCkiLAv/QSItUJECEwA+EYP///0iLXCRYSIt0JGBIi0QkEOn+/v// +SI0FFMEEALsXAAAADx9EAADou0MBAJBIiUQkCEiJXCQQ6MutAwBIi0QkCEiLXCQQkOkb/P//zMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmGA+GzgEAAEiD7ChIiWwkIEiNbCQggD2FLA0AAJAP +hKABAABIiVwkOEiJRCQwSYtWMEiLcBhI995Ii5LAAAAASIlUJBjrEEiLVCQYSIt8JDhIidhIifuA +urEAAAAAD4VBAQAASIt4GEgB90g5+w+OMQEAAEiJdCQQSIM9XTUNAAB1GejWYgAASItEJDBIi1Qk +GEiLXCQ4SIt0JBBIizhIhf90LkyLRxBNhcB1BDHJ6yNNjUj/TIlPEA8fRAAASYH5/QAAAA+D7AAA +AEqLTMcQ6wIxyUiFyXU5Dx9EAADom2AAAEiFwHUTMcAx2+hN+gAASItEJDDog2AAAEiLVCQYSItc +JDhIi3QkEEiJwUiLRCQwSIXJdUGLDUk1DQCQOQ1GNQ0Adn+5AQAAAEiNPTQ1DQDwD8EPiz0uNQ0A +ZpA5+XNjicvoVd7//0iLXCQwSIt0JBDp7/7//0iJw0iJyOg7AgAASItcJDBIi0sYSIH50AcAAHwm +SI0VqjANAPBID8EKSItTGEjHQxgAAAAASIt8JBBIjTQX6az+//9Ii3QkEOmi/v//SItIGEiNBA5I +i2wkIEiDxCjDTInIuf0AAADoOrMDAEiNBbXBBAC7GAAAAOipQQEAkEiJRCQISIlcJBDo+dIDAEiL +RCQISItcJBDpCv7//8zMzMzMzMzMzMxJO2YQD4ZBAQAASIPsYEiJbCRYSI1sJFhIiVwkcEiJTCR4 +SIm0JIgAAABIiXwkUEiJRCRoMdLrCEyJwg8fRAAASDnTdh1JidBIweoGD7YUEYXSdQlJjVBASYnQ +69tFMcnrE0iLbCRYSIPEYMNJ/8HR6kmDwAhJg/kIfb0PH0QAAEw5w3azD7riAHPhTY0UAE2LEk2F +0nTVTIlMJEBMiVQkOEyJRCRIiVQkNEiJw0yJwUyJ0OhIZv//SIXAdT1Ii4QkiAAAAEiFwHRSSItc +JDhIOZgAAQAAd0QPH4QAAAAAAEg5mAgBAAB2MzHJ6LA6AABIi4QkiAAAAOsiSInfSIt0JFBJichI +i1wkaEiLTCRI6KwGAABIi4QkiAAAAEiLRCRoSItMJHiLVCQ0SItcJHBIi7QkiAAAAEiLfCRQTItE +JEhMi0wkQOkh////SIlEJAhIiVwkEEiJTCQYSIl8JCBIiXQkKOg3qgMASItEJAhIi1wkEEiLTCQY +SIt8JCBIi3QkKOl5/v//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEyNZCTwTTtmEA+GFwMAAEiB +7JAAAABIiawkiAAAAEiNrCSIAAAASIsVK74MAIQCkEi+AAAAAACAAABIAcZIwe4aSIH+AABAAA+D +yAIAAEiLFPJIhdJ0JUmJwEjB6AVIJf//HwBMjQwCTYnCScHoA0mD4ANMjZr//x8A6w5JicIx9kUx +wEUx20UxyU2J1EnB6g1JgeL/HwAATouU0gAAIACEApBJi1JoSIXSD4RVAgAASImcJKAAAABMiWQk +YA8fAEiB+gAAAgB2UU05YhgPhXkBAABFD7ZqYkH2xQF1KEyJVCRATIlcJGiJdCQ4RIlEJDBMiYwk +gAAAAEmNlCQAAAIA6W4BAABIAVMQSIusJIgAAABIgcSQAAAAw0iJVCRQMcDrBEiDwAhIOcIPhgQB +AABFD7YRRInBRYnVQdPqQQ+64gQPg+wAAABIiUQkWEEPo80Pg4cAAABOjRQgTYsSZpBNhdJ0eU2J +1U0p4kw50nduiUwkPEyJTCR4TIlcJHCJdCQ0TInjSInBTIno6NBj//9IhcB0HUiJ30iLtCSgAAAA +SYnISItcJGBIi0wkWOhuBAAASItEJFiLTCQ8SItUJFBIi5wkoAAAAIt0JDRBichMi0wkeEyLXCRw +TItkJGBmkIP5A3MJRI1BAek1////TTnLdAtJ/8FFMcDpJf///0yJyInLifFMid/ok2T//0iLVCRQ +TItkJGCJzkGJ2EmJ+0mJwUiLRCRYSIucJKAAAADp7/7//0gBUxBIAUMYSIusJIgAAABIgcSQAAAA +w0mLUmhJA1IYTCniSIH6AAACAEG6AAACAEkPR9Lprf7//0iBwgAAAgBNi2poTQNqGA8fQABMOepz +x0yLK02F7XQiSYtFEEg9/QAAAHUEMcDrFHNYSYlUxRhJ/0UQuAEAAADrAjHAhMB1uEiJVCRISInY +SInT6PtXAABIi1QkSEiLnCSgAAAAi3QkOESLRCQwTIuMJIAAAABMi1QkQEyLXCRoTItkJGDpdv// +/7n9AAAADx9AAOhbrgMASI0Fy64EALsRAAAA6Mo8AQBIifC5AABAAGaQ6FuuAwCQSIlEJAhIiVwk +EOjLpgMASItEJAhIi1wkEJDpu/z//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhu4B +AABIg+xQSIlsJEhIjWwkSEiJXCRgSIlMJGhIiXwkQEiJRCRYSIl0JHgx0usLSIPCCA8fgAAAAABI +OdMPhpMBAABIhcl0RUmJ0EjB6gYPthQRhNJ1Fg8fAEn3wD8AAAAPhXkBAABJjVA468JNicFJwegD +SYPgB0mJykyJwdLq9sIBdQ5MidFMicrrokmJ0UmJykyJTCQ4So0UCEiLEkiF9nRFSDmWAAEAAHc8 +SDmWCAEAAHYzSInwSInTuQEAAADo1DUAAEiLRCRYSItMJGhIi1wkYEiLdCR4SIt8JEBIi1QkOOlG +////SIlUJDBIidDoRGYAAA8fQABIhcAPhLAAAACQSItQGEiLTCQwSCnRRItAXEkPr8hJichJwegg +TDlAMHdGkEjB6SNIA0hIRA+2CUyJwUiD4QdBugEAAABB0+KQRYTRdSNIi0QkWEiLTCRoSItcJGBI +i3QkeEiLfCRASItUJDjpxP7//0yLSGhND6/ITAHKSItcJFhIi0wkOEiJx0iLdCRASInQ6CoBAABI +i0QkWEiLTCRoSItcJGBIi3QkeEiLfCRASItUJDjpfP7//0iLRCRYSItMJGhIi1wkYEiLdCR4SIt8 +JEBIi1QkOOlZ/v//SItsJEhIg8RQw0iNBd6nBAC7DwAAAOiJOgEAkEiJRCQISIlcJBBIiUwkGEiJ +fCQgSIl0JCjoiqQDAEiLRCQISItcJBBIi0wkGEiLfCQgSIt0JCjpzP3//8zMzMzMzMzMzMzMzEk7 +ZhB2TUiD7DhIiWwkMEiNbCQwMdtIidnool///2aQSIXAdCRJi1YwSIuS0AAAAIQCSI2ymBYAAEiJ +30mJyDHbSInZ6DcAAABIi2wkMEiDxDjDSIlEJAjoA6QDAEiLRCQI65zMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMSTtmEA+GrQIAAEiD7EhIiWwkQEiNbCRADx+EAAAAAABIqQcAAAAPhXkCAABI +iUQkUJBIi1dQTYnBScHoA02JykmD4QdJictMiclBvAEAAABB0+SQTo0MAoA9dCANAACQD4XVAAAA +gz1fJA0AAH4aTDlXMHcUTItXSEcPthQCZpBFhOIPhEABAABBD7YUEEGE1A+FmgAAAJDwRQghSIsV +lbcMAIQCkJBIi08YkEm4AAAAAACAAABJAchJwegaDx8ASYH4AABAAA+D7wAAAEqLFMKEAkmJyEjB +6RBIgeH/AwAARA+2jAoABCEASI0UCkiNkgAEIQBJwegNSYPgB0yJwUG4AQAAAEHT4EWEwXUE8EQI +Ag+2V2L2wgF0REiLRhBIA0doSIlGEEiLbCRASIPESMNIi2wkQEiDxEjDSIl0JHBMidlMic9EieZN +idDoj5n//4TAdVBIi0QkUEiLdCRwSIsWSIXSdCNIi3oQSIH//QAAAHUEMcnrFHM1SIlE+hhI/0IQ +uQEAAADrAjHJhMl1C0iJw0iJ8OgFUwAASItsJEBIg8RIw0iLbCRASIPESMNIifi5/QAAAOiEqQMA +TInAuQAAQADol6kDAEyJXCQ4SIlcJDDoSFABAEiNBRbDBAC7HQAAAOhXWQEASItEJFDorVcBAEiN +BaqfBAC7DAAAAJDoO1kBAEiLRCQw6JFXAQBIjQWtlAQAuwEAAAAPH0QAAOgbWQEASItEJDjocVcB +AEiNBamUBAC7AgAAAA8fRAAA6PtYAQDoVlABAEiNBUuVBAC7BAAAAEiLTCQwSIt8JDjomwAAAEiN +BdKUBAC7AwAAAEiLTCRQSMfH/////w8fAOh7AAAASYtGMMaAKQEAAAJIjQW/qwQAuxMAAAAPH0AA +6Bs3AQBIjQXrzgQAuyMAAADoCjcBAJBIiUQkCEiJXCQQSIlMJBhIiXwkIEiJdCQoTIlEJDDoBqED +AEiLRCQISItcJBBIi0wkGEiLfCQgSIt0JChMi0QkMOkD/f//zMzMSTtmEA+GzAMAAEiD7GhIiWwk +YEiNbCRgSIm8JIgAAABIiUQkcEiJjCSAAAAASIlcJHiQSLoAAAAAAIAAAEgBykjB6hpIgfoAAEAA +cgQx0usuSIs14LQMAIQGSIsU1maQSIXSdBhIwekNSIHh/x8AAEiLjMoAACAASInK6wIx0kiJVCQg +6JdOAQBIi0QkcEiLXCR46KhXAQBIjQUrkwQAuwEAAADol1cBAEiLhCSAAAAA6OpVAQDo5U4BAEiL +RCQgSIXAD4SMAQAASItIcEiJTCRQD7ZQYkiJVCRISItYaEiJXCRASItwGEiJdCQY6C5OAQBIjQW5 +mgQAuwoAAABmkOg7VwEASItEJBjokVUBAEiNBdaXBAC7CQAAAA8fRAAA6BtXAQBIi0QkUOhxVQEA +SI0F2p8EALsNAAAADx9EAADo+1YBAEiLRCRI6PFTAQBIjQW6nQQAuwwAAAAPH0QAAOjbVgEASItE +JEDo0VMBAEiNBX+XBAC7CQAAAA8fRAAA6LtWAQDoFk4BAEiLRCQgikhjD7bRSIsd1BAKAEiLNdUQ +CgBIOdZ+OkjB4gRIiwQTSIlEJFhIi0wTCEiJTCQ4kOhbTQEASItEJFhIi1wkOOhsVgEA6MdPAQDo +wk0BAGaQ6z2ITCQX6DVNAQBIjQWPlgQAuwgAAADoRFYBAA+2RCQXD7bA6DdTAQBIjQXPkQQAuwIA +AADoJlYBAOiBTQEASItEJCBIi0hoikBjPAJ1E0iFyXUOSIuEJIgAAABIjUgI6whIi4QkiAAAAEiJ +TCQoMdIx2+sr6MZMAQBIjQUplAQAuwcAAADo1VUBAOgwTQEASItsJGBIg8Row0iDwghmkEg50Q+G +JAEAAEiB+gAEAAByHEiNcIBIOfJ2DEiNsIAAAABIOfJyB7sBAAAA68xIiVQkMITbdCDoZEwBAEiN +BSWSBAC7BQAAAOhzVQEA6M5MAQBIi1QkMEiLhCSAAAAASI0MAkiLCUiJTCRQ6DBMAQBIjQULkQQA +uwMAAAAPH0AA6DtVAQBIi0QkcEiLXCR46CxVAQBIjQWokAQAuwEAAADoG1UBAEiLRCQw6BFSAQBI +jQUykQQAuwQAAAAPH0QAAOj7VAEASItEJFDoUVMBAOhMTAEASIuEJIgAAABIi0wkMEg5yHUf6LVL +AQBIjQXSkAQAuwQAAADoxFQBAA8fQADoG0wBAOiWSwEA6BFOAQDoDEwBAEiLhCSIAAAASItMJChI +i1QkMDHb6c3+//+E23Qb6GpLAQBIjQUrkQQAuwUAAADoeVQBAOjUSwEASItsJGBIg8Row0iJRCQI +SIlcJBBIiUwkGEiJfCQg6PGcAwBIi0QkCEiLXCQQSItMJBhIi3wkIOn4+///zMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMSIPsGEiJbCQQSI1sJBCAPZ4ZDQAAD4XRAAAAkJCQkEgrWBiQi1BcSA+v2kiJ +2kjB6yCQSMHqI0iD4wdIic5IidlBuAEAAABB0+CQSANQUPBECAJIixW+sAwAhAKQkEiLSBiQSLsA +AAAAAIAAAEiNBAtIwegaSD0AAEAAc2VIiwTChABIicpIwekQSIHh/wMAAA+2nAgABCEASI0ECEiN +gAAEIQBIweoNSIPiB0iJ0boBAAAA0+KE2nUD8AgQSYtGMEiLgNAAAACEAEgBsKgWAABIAbiwFgAA +SItsJBBIg8QYw7kAAEAA6FSjAwBIjQVf1AQAuywAAADoozEBAJDMzEk7ZhAPhqQAAABIg+xgSIls +JFhIjWwkWEiLFdEuCgBIiVQkUEiLNc0uCgBIiXQkQDHA6wNI/8BIOfB9aEiLPMJMi0dATYXAdOtN +i0gQTYXJdOJIiUQkOEiJfCRITIlEJDBMicgx20iJ2eiyVv//SItUJEhIjbKYFgAASItUJDBIi0IQ +SInfSYnIMdtIidnoTff//0iLRCQ4SItUJFBIi3QkQOuQSItsJFhIg8Rgw+gNmwMA6Uj////MzMzM +zMzMzMPMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEHY7SIPsGEiJbCQQSI1sJBBI +x0AIAABAAPIPEAU84QUA8g8RQBBIx0BAIiIiAOiqDwAASItsJBBIg8QYkMNIiUQkCIlcJBDokZoD +AEiLRCQIi1wkEOumzMzMzMzMSTtmEA+GbwIAAEiD7EhIiWwkQEiNbCRARA8ReEhEDxF4WEQPEXho +SItIMEiBwQAAEABIOUggcwRIiUggiw1mFw0AD1fA8g8qwfIPEA2P4AUA8g9ZwfIPEA2T4AUA8g9Y +yPJIDyzJSImIgAAAAA9XyfJIDyrJDxDR8g9eyPIPEB2M4AUA8g9cy/IPEB3o4AUAZg8u2XcO8g8Q +HUrgBQBmDy7Ldj5mDy7QdgpI/8lIiYiAAAAASIuIgAAAAA9XyfJIDyrJ8g9cwYsN2xYNAA9XyfIP +KsnyD17B8g8RgJgAAADrCw9XwPIPEYCYAAAAgz2EGg0AAH4ZSGMNqxYNAEiJiIAAAAAPV8DyDxGA +mAAAAEiLDZosCgBIixWbLAoAMduQ6xFIizTZhAZEDxG+eBYAAEj/w0g503zqSIlEJFAPHwDoWwEA +AIM9JBoNAAAPjh0BAABIi0QkUEiLiIgAAABIiUwkOEiLFY4dDQBIiVQkMEiLHVoiDQBIiVwkKEiL +cCBIiXQkIEiLuIAAAABIiXwkGPIPEICYAAAA8g8RRCQQZpDoG0cBAEiNBUOmBAC7FAAAAOgqUAEA +SItEJDhmSA9uwOgbSgEASI0FJo4EALsHAAAA6ApQAQBIi0QkMEjB6BSQ6PtMAQBIjQUUjgQAuwcA +AADo6k8BAEiLRCQoSMHoFJDo20wBAEiNBXeLBAC7AgAAAOjKTwEASItEJCBIwegUkOi7TAEASI0F +KZgEALsNAAAA6KpPAQBIi0QkGA8fRAAA6JtNAQBIjQUXiwQAuwEAAADoik8BAPIPEEQkEA8fQADo +e0kBAOjWSAEA6NFGAQBIi2wkQEiDxEjDSIlEJAhmkOj7lwMASItEJAjpcf3//8zMzMzMzMzMzMzM +zMzMzMzMiwhIi1AwSItYOEiLcEhIi3gghclBuKCGAQBBD0zIZpBIhdt8Cg9XwPJIDyrD6x9JidhI +g+MBTYnBSdHoSQnYD1fA8kkPKsDyD1jATInL8g8QDUreBQDyD1nIg8FkD1fA8g8qwfIPXshIOdd8 +CvJIDyzJSDnOfhsPV8DySA8qx/IPEA3x3QUA8g9ZyPJIDyz56wNIictIKfNIgfvoAwAAuegDAABI +D0zZSCnXSIX/uQEAAABID075D1fA8kgPKscPV8nySA8qyw8Q0fIPXshmSA9+yUiHiIgAAADyD17C +ZkgPfsFIh4iQAAAAw8zMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhi8EAABIg8SASIlsJHhIjWwk +eA8fhAAAAAAAhNsPhQIEAABIi0ggSItQQEgp0UiFyXwKD1fA8kgPKsHrHkiJy0iD4QFI0etICcsP +V8DySA8qw/IPWMAPH0QAAEiF0nwKD1fJ8kgPKsrrH0iJ0UiD4gFIictI0elICdEPV8nySA8qyfIP +WMlIidryD17BD1fJZg8uyHYDD1fASItIMEiFyXwND1fJ8kgPKslIhdLrHEiJy0iD4QFI0etICcsP +V8nySA8qy/IPWMlIhdJIiYQkiAAAAPIPEUQkGPIPEUwkcHwKD1fS8kgPKtLrGUiJ0UiD4gFI0elI +CdEPV9LySA8q0fIPWNLyDxFUJGjoTLEDAEUPV/9kTIs0Jfj////yDxBEJHDyDxBMJGjyD17BSIsE +JEiLjCSIAAAASCtBeEiFwH4xSItRWA9XyfJIDyrKSGMVrxINAEgPr8IPV9LySA8q0PIPXsryDxAV +z9sFAPIPWNHrCPIPEBXB2wUA8g8QDenbBQDyD1zB8g8QWRDyDxBkJBjyD1zj8g8QNabbBQAPEPry +D17WRA8QwPIPXMNEDxDI8g9ZwkQPENTyD1zg8g8QBYfbBQDyD1nE8g9Yw4M9+BUNAAAPjkICAADy +RA8RRCRw8g8RXCRo8kQPEVQkYPIPEXwkWPIPEVQkUPJEDxFMJEjyDxFEJBBIi0FASIlEJChIi1EY +SIlUJDhIi1kwSIlcJDBIhcB8Cg9X5PJIDyrg6xlIicZIg+ABSNHuSAnGD1fk8kgPKubyD1jk8g8R +ZCRASItBSEiJRCQgDx8A6LtCAQBIjQXymQQAuxAAAADoyksBAEiLRCQoDx9EAADou0gBAEiNBY6I +BAC7BQAAAOiqSwEA8g8QRCRoDx9AAOibRQEASI0FQYgEALsFAAAA6IpLAQBIi0QkOA8fRAAA6HtI +AQBIjQVEiAQAuwUAAADoaksBAPIPEEQkcA8fQADoW0UBAEiNBQaIBAC7BQAAAOhKSwEASItEJDAP +H0QAAOg7SAEASI0FCYgEALsFAAAA6CpLAQDyDxBEJBgPH0AA6BtFAQBIjQXLhwQAuwUAAADoCksB +APIPEAUy2gUA8g8QTCQY8g9YwfIPEEwkQPIPWcHySA8swOjkSAEASI0F2ocEALsFAAAA6NNKAQDy +DxBEJFjoyEQBAEiNBcOHBAC7BQAAAOi3SgEA8g8QBbfZBQDoqkQBAEiNBWSHBAC7BQAAAOiZSgEA +SItEJCDoj0gBAEiNBQGKBAC7CAAAAA8fAOh7SgEA8g8QRCRg6HBEAQBIjQWVjQQAuwoAAAAPH0AA +6FtKAQDyDxBEJEjoUEQBAEiNBRmLBAC7CQAAAA8fQADoO0oBAPIPEEQkUOgwRAEA6ItDAQDohkEB +APIPEEQkEEiLbCR4SIPsgMPyDxBAEEiLbCR4SIPsgMNIiUQkCIhcJBDomZIDAEiLRCQID7ZcJBDp +qvv//8zMzMzMzMzMzMxJO2YQD4YrAQAASIPsKEiJbCQgSI1sJCBIg7iAAAAAAH5Egz1/Dw0AAX4x +TInySIXSdB9Ii1IwSIXSdBZIi5LQAAAASIXSdAqLEolUJBQxwOshSItsJCBIg8Qow0iLbCQgSIPE +KMNIi2wkIEiDxCjDSP/ASIP4BQ+NpQAAAEmLXjCLNSEPDQCLuyABAABEi4MkAQAARImDIAEAAEGJ ++cHnEUEx+USJx0UxyEHB6QdFMcFBifjB7xBEMc+JuyQBAACNXv9BjTQ4SA+v80jB7iA58n8C/8ZI +Y95Iiw3ZJAoASIs1yiQKAEg52XZBSIsc3oN7BAEPhXf///9IiUQkGEiJ2Oi6AQIAhMB1DkiLRCQY +i1QkFOlY////SItsJCBIg8Qow0iLbCQgSIPEKMNIidjorJgDAJBIiUQkCOhBkQMASItEJAjpt/7/ +/8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GRwIAAEiD7DBIiWwkKEiNbCQogz0dDg0AAJAP +hBkCAABIhdt0KEiLk5gWAABIhdJ0HEiDehAAdQ5Ii5OgFgAASIN6EAB0B7kBAAAA6y1IixUGGQ0A +SIXSdRmLFYMZDQA5FYEZDQB2B7kBAAAA6wwxyesIuQEAAAAPHwCEyXQC6w8xwEiLbCQoSIPEMMNI +ichIixWEDg0ADx9AAEiF0nQsSInWSMH6E0jB4gOQSIs6SInBSInwTI0FXw4NAPBJD7E4QA+UxkCE +9nTD6w9IicFMjQVFDg0AMdIPHwBIhdJ0BIQB6wwxwEiLbCQoSIPEMMNIi7GAAAAADx9AAEiF9n4e +SI1+/0iJ8PBID7G5gAAAAEAPlMdAhP9012aQSIX2D4/NAAAA8g8QgZgAAAAPV8lmDy7BdQxmDx9E +AAAPi5kAAABIiUwkIEiJXCRASIlUJBDoRqsDAEUPV/9kTIs0Jfj///9IiwQkSItMJCBIK0F4SIXA +fk1Ii1QkQIQCSIuygBYAAA9XwPJIDyrGD1fJ8kgPKsjyD17B8g8QiZgAAABmDy7BdiJIjQVwDQ0A +SItcJBDoxvj+/zHASItsJChIg8Qww0iLVCRAhAJIx4KIFgAAAgAAAEiLVCQQ6yRMicBIidPolvj+ +/zHASItsJChIg8Qww4QDSMeDiBYAAAEAAABIi0IQSIlEJBi7BAAAALkBAAAA6GVqAQCAPe6gCgAA +dAxIi0QkGDHb6FD9AgBIi0QkGEiLbCQoSIPEMMNIjQVIzwQAuzYAAADosCQBAJBIiUQkCEiJXCQQ +Dx9EAADou44DAEiLRCQISItcJBDpjP3//8zMzMzMzMzMzMzMzEk7ZhAPhlwDAABIg+xQSIlsJEhI +jWwkSIsIhcl8LEiLUEBIY9lID6/aSNHrSInGSLgL16NwPQrXo0iJ10j340jB6gVIAfqFyesKSInG +SMfC/////3w/D1fJ8g8qyfIPEBX71AUA8g9eyvIPEBW31AUA8g9Z0WYPLsJ3Aw8Q0PIPEAWS1AUA +8g9ZwWYPLsJ3EQ8QwusMD1fJZg8uyHYDD1fA8g8RRhCDPgAPjK8AAABIi05ASIXJfAoPV8nySA8q +yesZSInLSNHpSIPjAUgJyw9XyfJIDyrL8g9YyfIPEBVN1AUA8g9Y0PIPWdHyDxANjdQFAGYPLsp2 +CfJIDyzKZpDrDvIPXNHySA8sykgPuuk/SIteCIs9cqEMAIX/dA6LPWyhDACF/0APlMfrAjH/QIT/ +dRJIi34wSIHHAAAQAEg5+0gPQt9IOctID0fLSIXJD4wBAQAASDnKSA9C0esHSMfB/////0iJdCRA +SIlMJBBIiU4YSIdWIIA9Ep8KAABmkHQP6Dn+AgBIi0wkEEiLdCRAgz0sCg0AAHQVSInwDx8A6Bv1 +//9Ii0wkEEiLdCRAiwXXoAwAhcB0DYsF0aAMAIXAD5TA6wIxwITAdA0PV8DyDxEFAaEMAOtuSItG +MEiLFdygDABIix3NoAwASCnBSIHBAADw/0iB+QAgAAC+ACAAAEgPTM5IKdNIhdt/DQ9XwPIPEQXA +oAwA6y0PV8DySA8qww9XyfJIDyrJ8g9ewfIPEQWioAwASIkFk6AMAEiNBYSgDABIhxDoVAIAAEiL +bCRISIPEUMPyDxFEJDhIiVwkEEiLRiBIiUQkMEiLTkBIiUwkKEiLVjBIiVQkIEiLHW0VDQBIiVwk +GOhTOgEASI0FMZUEALsSAAAA6GJDAQBIi0QkMOhYQAEASI0FzYkEALsMAAAA6EdDAQBIi0QkKGaQ +6DtAAQBIjQUYngQAuxcAAADoKkMBAEiLRCQgDx9EAADoG0ABAEiNBeKRBAC7EQAAAOgKQwEASItE +JBgPH0QAAOj7PwEASI0F1YwEALsNAAAA6OpCAQDyDxBEJDgPH0AA6Ns8AQBIjQVoiQQAuwwAAADo +ykIBAEiLRCQQDx9EAADouz8BAOgWPAEA6BE6AQBIjQV2kwQAuxEAAAAPH0QAAOgbIQEAkEiJRCQI +8g8RRCQQ6CqLAwBIi0QkCPIPEEQkEOl6/P//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQ +dl1Ig+wgSIlsJBhIjWwkGIXbuf////8PTNmLCIlMJBSJGEhj00jB4hZI0epIicNIuAvXo3A9Ctej +SPfiSMHqBUiJUwjyDxBDEEiJ2OgL/P//i0QkFEiLbCQYSIPEIMNIiUQkCIlcJBDoj4oDAEiLRCQI +i1wkEOuEzMzMzEk7ZhB2a0iD7BhIiWwkEEiNbCQQSI0FAX4EALsEAAAA6HvH/v9Ig/sDdRxmgThv +ZnUVgHgCZnUPuP////9Ii2wkEEiDxBjD6BSmAgBIY8hIOch1DoTbdApIi2wkEEiDxBjDuGQAAABI +i2wkEEiDxBjD6AqKAwDriMzMzMzMzMzMSIM9YA4NAAAPhBsBAABIiw1LDg0ASIXJfAoPV8DySA8q +wesZSInKSIPhAUjR6kgJyg9XwPJIDyrC8g9YwEiLDSQODQBIhcl8Cg9XyfJIDyrJ6xlIicpIg+EB +SNHqSAnKD1fJ8kgPKsryD1jJ8g9ewUiLDQk1DQBIhcl8Cg9XyfJIDyrJ6xlIicpIg+EBSNHqSAnK +D1fJ8kgPKsryD1jJ8g9ZyPIPEAU10AUAZg8uwXYH8kgPLMnrDvIPXMjySA8syUgPuuk/SIsVuwcN +AJBIix2bHQ0ASIs1pB0NAEi4zczMzMzMzMxIiddI9+FIweoDSAHRSI0MOUiNSf9I999IIc9IKfNI +Od9zDEgp+0g5HXUHDQB2DEjHBSCdDAD/////w0iJPRidDADDSMcFDJ0MAP/////DzMzMzMzMzMzM +zMxJO2YQD4aFAAAASIPsGEiJbCQQSI1sJBCQSI0FQB0KAOhb9P7/gD1EHQoAAHRJMclIjRVJHQoA +hwpIiwU4HQoA6PvBAgDGBSQdCgAASMdEJAgAAAAASIsNDB0KAEjHgaAAAAAAAAAAkEiJTCQISI1E +JAjoSaMBAJCQSI0F4BwKAOjb9f7/SItsJBBIg8QYw+gsiAMA6Wf////MzMzMzMzMSTtmEA+GvgAA +AEiD7EhIiWwkQEiNbCRASIlEJFCQSI0FmxwKAOi28/7/kOhwowMARQ9X/2RMizQl+P///0iLBZQc +CgBIiwwkSIlMJDhIi1QkUEiNHBGQSItQEEiLeBhIi3AgTItAKEyLSDBIidHobccCAMYFVhwKAAGQ +SI0F/s8EAEiNHTccCgC5EwAAAL8TAAAAvgIAAADoo0cBAA8fAOj7ogMARQ9X/2RMizQl+P///0iL +BCRIi0wkOEgpyEiLbCRASIPESMNIiUQkCOhOhwMASItEJAjpJP///8zMzMxJO2YQD4ayAgAASIPs +WEiJbCRQSI1sJFCDPSEGDQAAkHUMTInxSIkNvBsKAOsPSI09sxsKAEyJ8ejDigMAkJBIjQWaGwoA +6LXy/v/GBZ4bCgABSI0FB1YEAOjiEf//gz3bBQ0AAHUUSIkFihsKAEiNDXPNBABIiUgY6xxIjT12 +GwoA6HmJAwBIjXgYSI0NVs0EAOhpigMASIsF2hgKAEiNHZPMBADoNp/+/5BIjQXuzgQASI0dJxsK +ALkNAAAAvxQAAAC+AQAAAOiTRgEA8g8QBcPMBQDyDxFEJCBIx0QkKAAAAAAPV8nyDxFMJDBEDxF8 +JDhIx0QkSAAAAABIjQXdAQAASIlEJDhIjUwkKEiJTCRASI1MJDBIiUwkSEiNTCQ4SIkMJOjWhAMA +RQ9X/2RMizQl+P///0iLRCQoDx9AAEiFwA+EBwEAAEiLDXgEDQBIOcEPh1EBAADyDxBEJDAPV8lm +Dy7IckpIhckPhDQBAAAx0kj38UiFwHwKD1fA8kgPKsDrGUiJwUjR6EiD4QFICcgPV8DySA8qwPIP +WMDyDxAVYswFAPIPWcLyDxFEJDDrCPIPEBVOzAUA8g8QRCQw8g8RRCQw8g8QHUrMBQBmDy7Ddgby +DxFcJDDyDxBEJCDyDxANsMsFAPIPXsHyD1lEJDDyD17B8kgPLMAPH0QAAOgb/f//8g8QRCQwD1fJ +8kgPKsjyD1jI8g9ewfIPEA1tywUAZg8uyHYI8g8QBV/LBQDyDxAVd8sFAPIPWcLyDxBcJCDyD1na +8g9Yw+mF/v//kJBIjQWHGQoA6KLw/v/GBYsZCgABkEiNBTPNBABIjR1sGQoAuQ0AAAC/FAAAAL4B +AAAA6NhEAQDyDxANAMsFAPIPEBUYywUA8g8QRCQgZpDpMP7//+h29QAASI0F474EALsuAAAA6GUa +AQCQDx9AAOh7hAMA6Tb9///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+G8QAAAEiD7DhIiWwk +MEiNbCQwSItKEEiJTCQoSItSCEiJVCQgkJBIjQXtlgsA6Ojv/v9Iiw2RGA0ASIsVmhgNAEiLHVOY +DABIKdFIOctyGpCQSI0FwpYLAGaQ6Jvx/v9Ii2wkMEiDxDjDkOhrnwMARQ9X/2RMizQl+P///0iL +HV8CDQBIiwQkSIlEJBi5AQAAAEiNBYqWCwBmkOh7AAAASItMJCBIiQFIAQVklwwA6CefAwBFD1f/ +ZEyLNCX4////SIsEJEiLTCQYSCnID1fA8kgPKsBIi0QkKPIPEQCQkEiNBTSWCwDoD/H+/0iLbCQw +SIPEOMMPH0QAAOi7ggMA6fb+///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmGA+GGwEAAEiD7EBI +iWwkOEiNbCQ4SIlcJFBIiUQkSIhMJCuEADHSMfYx/0UxwOs9RIlEJCxIKdOJzkyJ0UmJ2kiJ+0yJ +1+hyBQAASItUJDBIAcJIi0QkSEiJzkiJ30SLRCQsD7ZMJCtIi1wkUEiJVCQwSDnTD4aKAAAASbkA +AAAAAIAAAE6NFA9OjRwOTTnTdgmQSYnySCn+6wVJifIx9kiF9nWL6FMDAABIugAAAAAAgAAASI00 +AkiNPBqQSDn3dgmQSYnYSCnD6wVJidgx20iF23QiSItUJDBIi1wkUEiJx02JwkGJyEiLRCRID7ZM +JCvpOf///0yJxkiJx0iLRCRIQYnISIn7SInxRInH6MoDAABIi0QkMEiLbCQ4SIPEQMNIiUQkCEiJ +XCQQiEwkGOhIqQMASItEJAhIi1wkEA+2TCQY6bT+///MzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAP +hk0BAABIg+woSIlsJCBIjWwkIIlEJDBIiVwkOIhMJEDo9i8BAEiLBS8WDQBIiw0gFg0ASIsVERYN +AEiLHRoWDQBIKdpIhdIPhP8AAABIiUQkGEhrwWRIidEx0kj38UiJRCQQ6LMvAQBIjQUPdgQAuwUA +AADowjgBAItMJDCJyOi3NQEA6NIxAQBIi0QkOEjB6AropDUBAEiNBVF9BAC7CwAAAOiTOAEASItE +JBhIwegK6IU1AQBIjQW+fgQAuwwAAADodDgBAEiLRCQQ6Go1AQBIjQUtdgQAuwYAAADoWTgBAOi0 +LwEAD7ZMJECEyXQb6CYvAQBIjQWIeAQAuwkAAADoNTgBAOiQLwEA6AsvAQDohjEBAOiBLwEASYtG +MJAPtogXAQAAjVH/iJAXAQAAgPkBdQ6QkEiNBWb+DADoOe7+/0iLbCQgSIPEKMPoavEAAJCJRCQI +SIlcJBCITCQY6HeAAwCLRCQISItcJBAPtkwkGOmE/v//zMzMzEk7ZhgPhuUAAABIg+woSIlsJCBI +jWwkIEiJRCQwhACDPUIBDQAAfhuLkOAAAQBIi5jwAAEAMcmJ0OhC/v//SItEJDBIjYiQAAEASI2Y +uAABAEiJXCQYSInI6CLEAABIi0wkMEiLmfgAAQBIi5EAAQEASL4AAAAAAIAAAEiNPDNIAdZIOf5I +D0faSItEJBjorcEAAEiLTCQwSIuRqAABAEiBwv//PwBIgeIAAMD/SMHqBkiJkegAAQD/geAAAQBI +x4HwAAEAAAAAAEiLFRDeCQBIiZEAAQEASIsV4t0JAEiJkfgAAQBIi2wkIEiDxCjDSIlEJAjop6YD +AEiLRCQIZpDp+/7//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhgPhrAAAABIg+wwSIls +JChIjWwkKEiJRCQ4hABIjYi4AAEASIlMJCBIi5joAAEASInI6GbAAABIuQAAAAAAgAAASI0UCEgB +2Ug50XYJkEiJ2kgpw+sHSInaMdtmkEiF23UYSIt0JDiLjuAAAQBIidNIi2wkKEiDxDDDSIlUJBiQ +SCUAAMD/SIlEJBBIicNIi0QkIOiFwAAASItMJDiLieAAAQBIi0QkEEiLXCQYSItsJChIg8Qww0iJ +RCQIkOi7pQMASItEJAjpMf///8zMzMzMzMzMzMzMzMzMzMzMSTtmGA+GiQAAAEiD7CBIiWwkGEiN +bCQYhABIugAAAAAAgAAASI00GkgBykg58nYJkEiJykgp2esHSInKMclmkEiFyXQIObjgAAEAdBNI +i2wkGEiDxCDDZg8fhAAAAAAASPfD//8/AHUYSAW4AAEASInR6Om5AABIi2wkGEiDxCDDSI0FzJ0E +ALscAAAA6K4TAQCQSIlEJAhIiVwkEEiJTCQYiXwkIOj1pAMASItEJAhIi1wkEEiLTCQYi3wkIGaQ +6Tv////MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxMjWQk+E07ZhgPhuwDAABIgeyIAAAASIms +JIAAAABIjawkgAAAAIQASbgAAAAAAIAAAE2NDBhNjRQITTnKdgmQSYnKSCnZ6wVJicoxyUiFyQ+E +wAEAAGYPH4QAAAAAAJBI98P//z8AD4V8AwAASIsNlPsMAEjB6Q1Ig/kBQbsBAAAASQ9Cy0mJ+0jB +7w1MjWcBSffD/x8AAEkPRfyQTItYaEyLYGBJvf//////fwAATQHVTYnvScHtFg8fgAAAAABNOesP +hhUDAABPixzsSQ+64z9zCEG7AAAgAOsLScHrFUmB4///HwBMiUwkYEiJhCSQAAAAQIi0JLAAAABI +iVwkWEiJfCRASIlMJDhJOcsPgscAAABJwe8jDx+EAAAAAABJgf8AIAAAD4OkAgAATIlsJFBKi1T4 +eIQCSY1a/0mB5f8fAABJweUHTAHqSIHj//8/AEjB6w1IidDo4ggAAGaQSIXbdTVIi4QkkAAAAEiL +TCQ4SItcJFgPtrQksAAAAEiLfCRASbgAAAAAAIAAAEyLTCRgTItsJFDrQUiJXCQwSInBSInfSIuE +JJAAAABIi1wkUOgJBAAASItUJDBIweINSItcJFhIicFIidBIi6wkgAAAAEiBxIgAAADDkEQPEXwk +aEjHRCR4AAAAAEyNFU4CAABMiVQkaEiJRCRwSIlMJHhJweUWSboAAAAAAID//0+NXBUA61sxwEyJ +0UiLrCSAAAAASIHEiAAAAMOQTItkJEhJweQWSboAAAAAAID//0+NHBRIi1wkWA+2tCSwAAAATItM +JGBIi4QkkAAAAEiLTCQ4SIt8JEBJuAAAAAAAgAAAT40kA005zHYJkE2J3Ekp2+sGTYncRTHbTYXb +D4QgAQAAkECE9nQfTIlkJFBIi4gIAQEAkEiJyOhz6P7/SItcJFhMi2QkUEiLTCRoSInYTInjSI1U +JGj/0UiJRCRIkA+2jCSwAAAAhMl0K4hcJCdIi4wkkAAAAEiLgQgBAQCQ6Ezm/v9Ii0QkSA+2jCSw +AAAAD7ZcJCeE2w+EnAAAAJBIicJIwegNZg8fhAAAAAAAkEg9ACAAAA+DmwAAAEiLtCSQAAAATItE +xnhBhABIgeL/HwAASMHiB0mNBBC7/wEAAEiLTCQ4SIt8JEDo4QYAAJBIhdsPhsL+//9IiVwkKEiJ +wUiJ30iLhCSQAAAASItcJEjoOgIAAEiLVCQoSMHiDUiLXCRYSInBSInQSIusJIAAAABIgcSIAAAA +w0iLXCRYSYncMcBMieFIi6wkgAAAAEiBxIgAAADDuQAgAADoL4EDAEyJ+LkAIAAA6CKBAwBMiehM +idnoF4EDAEiNBS2zBAC7LQAAAOhmDwEAkEiJRCQISIlcJBBIiUwkGEiJfCQgQIh0JCjop6ADAEiL +RCQISItcJBBIi0wkGEiLfCQgD7Z0JCjpyfv//8zMzMzMzMzMzEk7ZhAPhjwBAABIg+w4SIlsJDBI +jWwkMEiJRCQgSItyEEiJdCQQSItSCEiJVCQoSL///////38AAEgB30jB7xbrBEmNef9JuAAAAAAA +gAAATo0MAEnB6RYPH4QAAAAAAEw5zw+CuAAAAEiLSmhMi0pgSDnPD4PCAAAATYsM+UkPuuE/cwhB +uQAAIADrC0nB6RVJgeH//x8ASTnxcwVJifnroEmJ+UjB7w1Igf8AIAAAc31IjTz6SI1/eEiLPw8f +AEiF/w+Eef///0yJTCQYSYHh/x8AAEnB4QdKjQQPSInz6NsDAACEwHUjSItEJCBIi1QkKEiLdCQQ +SbgAAAAAAIAAAEyLTCQY6Tb///9Ii0QkGLsBAAAASItsJDBIg8Q4kMMxwDHbSItsJDBIg8Q4w0iJ ++LkAIAAA6IR/AwBIifiQ6Ht/AwCQSIlEJAhIiVwkEOhLdwMASItEJAhIi1wkEJDpm/7//8zMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhjABAABIg+w4SIlsJDBIjWwkMIQASInaSMHrDUiB ++wAgAAAPg/4AAABIiUQkQEiJVCQoSIl8JFhIiUwkUEiLdNh4hAZIgeL/HwAASMHiB0iNFBZIjVJA +SInLSIn5SInQ6HaSAABIi1QkUEjB4g1Ii3QkKEjB5hZIAfJIvgAAAAAAgP//SI0EFki+AAAAAACA +AABIi3wkQEgDt/gAAQBIOdZ2B0iJh/gAAQCAvxgBAQAAdApIi2wkMEiDxDjDSIlEJBhIi1wkWEjB +4w3o63D//0iLTCRYSMHhDUiJTCQgSI0VVgsNAPBID8EKSI0FUiINAOglwgAASItMJCBIicpI99nw +SA/BCPBID8FQCEiNBTAiDQDoA8MAAEiLRCQYSItsJDBIg8Q4w0iJ2LkAIAAA6Ad+AwCQSIlEJAhI +iVwkEEiJTCQYSIl8JCDobXYDAEiLRCQISItcJBBIi0wkGEiLfCQg6ZT+///MzMzMzMzMzMzMzMzM +zMzMzMzMzEk7ZhAPhnwBAABIg+wYSIlsJBBIjWwkEEiD+wR3cmaQSIP7AXRgSIP7AnUoSLpVVVVV +VVVVVUghwki+VVVVVVVVVVVIAfJICdBICcZI99bp8gAAAEiD+wQPhRIBAABIvnd3d3d3d3d3SCHG +SLp3d3d3d3d3d0gB1kgJxkgJ1kj31unAAAAASItsJBBIg8QYw0iD+xB3XkiD+wh1KUi+f39/f39/ +f39IIcZIun9/f39/f39/SAHWSAnGSAnWSPfWkOmBAAAASIP7EA+FoQAAAEi+/3//f/9//39IIcZI +uv9//3//f/9/SAHWSAnGSAnWSPfW61JIg/sgdSZIvv///3////9/SCHGSLr///9/////f0gB1kgJ +xkgJ1kj31usnkEiD+0B1SpBIicZID7rwP0i6/////////39IjTwQSAn+SAnWSPfWSI1L/0iD+UBI +GdJIifBI0+5IIfJIicNIKdBICdhI99BIi2wkEEiDxBjDSI0Fo3EEALsLAAAADx9AAOibCgEAkEiJ +RCQISIlcJBDoq3QDAEiLRCQISItcJBCQ6Vv+///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJ +O2YQD4YNAQAASIPsIEiJbCQYSI1sJBhIiVwkMEiNS/9IhdkPhawAAABIhdsPhKMAAABIg/tAd11I +iUQkKLkHAAAA6xNIi1QkEEiNSv9Ii0QkKEiLXCQwSIXJfC1IiUwkEIQASItUyEBICxTISInQ6Mv9 +//9Ig/j/dMq4AQAAAEiLbCQYSIPEIMMxwEiLbCQYSIPEIMPoJSIBAEiNBYF3BAC7DwAAAOg0KwEA +SItEJDDoKigBAOiFJAEADx9EAADoeyIBAEiNBad0BAC7DQAAAOiKCQEA6OUhAQBIjQVBdwQAuw8A +AADo9CoBAEiLRCQw6OonAQDoRSQBAA8fRAAA6DsiAQBIjQV+nQQAuyEAAADoSgkBAJBIiUQkCEiJ +XCQQ6FpzAwBIi0QkCEiLXCQQ6cv+///MzMzMzMzMzMzMzEk7ZhAPhgIDAABIg+xISIlsJEBIjWwk +QEiJTCRgSI1R/0iFyg+FpgIAAEiFyQ+EnQIAAEiD+UAPh1gCAAAPHwBIhf91BUiJyusVkEiNFDlI +jVL/SInOSPfZSCHRSInySIlEJFBIiUwkOEjB6wbrGEiLdCQgSI1e/0iLRCRQSItUJGBIi0wkOEiJ +XCQgSIXbfEKEAEiD+wgPg+sBAABIiwzYSAtM2EBIichIidPoT/z//0iD+P90uEiLRCQgSIXASItE +JFBIi0wkOEiLVCRgSItcJCAPjKQAAACEAA8fRAAASIP7CA+DkQEAAEiLDNhIC0zYQEiJyEiJ0+gC +/P//SInBSPfQSA+90EjHxv////9ID0TWSI16wUj330iD/0BIGf9Mi0QkIE2JwUnB4AZOjRQCTY1S +AU6NBAJNjUABTI1aAUn320iJyEyJ2UjT4Egh+EiFwHQUSA+9yEgPRM5Ig8HBSPfZ6ZMAAABMiVQk +KEyJRCQwSI1KAUmNUf/rKzHASInDSItsJEBIg8RIw0iLfCQYSP/PSMfG/////0yLRCQwTItUJChI +ifpIhdJ8T0iJVCQYSIlMJBBIi0wkUEiLBNFIC0TRQEiLXCRg6DH7//9ID73ISMfC/////0gPRMpI +/8FIi3QkEEgpzkiNTkBIhcB0mkyLRCQwTItUJChIi1QkOEg5ykiJzkgPQspMicJJKchIiz2o7wwA +Dx+EAAAAAABIgf8AIAAAdj9IOT2Y7wwAczZIwe8NkE2NDDhNjUn/SPffSSH5TTnKchmQSCnyTCHH +SDnXcghJKfhMAcHrDUyJx+sITInH6wNMicdIifhIictIi2wkQEiDxEjDSInYuQgAAADoGHgDAEiJ +2LkIAAAA6At4AwDo5h4BAEiNBUJ0BAC7DwAAAOj1JwEASItEJGDo6yQBAOhGIQEA6EEfAQBIjQVt +cQQAuw0AAADoUAYBAOirHgEASI0FB3QEALsPAAAA6LonAQBIi0QkYOiwJAEA6AshAQDoBh8BAEiN +BUmaBAC7IQAAAOgVBgEAkEiJRCQISIlcJBBIiUwkGEiJfCQg6BtwAwBIi0QkCEiLXCQQSItMJBhI +i3wkIOnC/P//zMxJO2YQD4bzAAAASIPsIEiJbCQYSI1sJBgPH4QAAAAAAEg5mAABAAAPh74AAABI +OZgIAQAAD4axAAAASI2QGAEAAITJdAdIjZAoAQAASIlUJBBIiVwkMEiLMkiF9nRKSIF+EPwAAAB1 +WkiLsCABAABIhfZ0DkjHgCABAAAAAAAAkOsS6LkmAABIi1QkEEiLXCQwSInGSMdGEAAAAABIizpI +iX4YSIky6xrokyYAAEQPEXgQSItMJBBIiQFIi1wkMEiJxkiLRhBIPfwAAABzE0iJXMYgSP9GEEiL +bCQYSIPEIMO5/AAAAOh1dgMASI0F9okEALsbAAAA6OQEAQCQSIlEJAhIiVwkEIhMJBjo8G4DAEiL +RCQISItcJBAPtkwkGJDp2/7//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhhwBAABI +g+xISIlsJEBIjWwkQEiJRCRQSI1UJDBEDxE6hABIjZAYAQAASIlUJDBIjZAoAQAASIlUJChIiVQk +ODHJ6wNI/8FIg/kCD42OAAAASIt0zDBIiz5mkEiF/3TkSIN/EAB1T0yLgCABAABNhcB0MEiJTCQY +SIl0JCBIiXwkEEyJwOjRJwAASItEJFBIi0wkGEiLVCQoSIt0JCBIi3wkEEiJuCABAABIi38YSIk+ +SIX/dI5Mi0cQSY1A/0iJRxBIPfwAAABzR0g51g+Uw0qLRMcYSItsJEBIg8RIw0iLiCABAABIhcl0 +GEiJyOhoJwAASItMJFBIx4EgAQAAAAAAADHAMdtIi2wkQEiDxEjDufwAAAAPH0QAAOj7dAMAkEiJ +RCQI6JBtAwBIi0QkCOnG/v//zMzMzMzMSTtmEA+GUAEAAEiD7CBIiWwkGEiNbCQYSIlEJChIiVwk +MEiJTCQ4SIuQOAEAAEiF0nUy6IgkAABIx0AYAAAAAEiLTCQoSImBMAEAAEiJgTgBAABIi0wkOEiL +XCQwSInCSItEJChIi3IQSIX2fnpIi7gAAQAASYnYSCn7SI1+/5BIg/8/D4PCAAAASMHnBUSLTDog +RANMOiQPH4QAAAAAAEE52Q+HkgAAAEiD/j91PEiJVCQQ6AckAABIx0AYAAAAAEiLTCQQSIlBGEiL +TCQoSImBOAEAAEyLRCQwSInCSInISItMJDjrA0mJ2EiLWhBIg/s/czlIjXMBSIlyEEjB4wVIi7AA +AQAASSnwRIlEGiCLcQSJdBokkEiJTBooSP+AQAEAAEiLbCQYSIPEIMNIidi5PwAAAOilcwMASI0F +VqEEALspAAAA6BQCAQBIifi5PwAAAOiHcwMAkEiJRCQISIlcJBBIiUwkGOgSbAMASItEJAhIi1wk +EEiLTCQYDx8A6Xv+///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4bjAAAASIPsSEiJ +bCRASI1sJEAPH4QAAAAAAEiFyQ+EpAAAAEiJTCRgSInKSMHpP0gB0UjR+UiJTCQ46Lv///+EA0iD ++T8Pg44AAABIiUQkGEiJXCQwSInKSMHhBUiJTCQoSI00C0iNdiBIiXQkIEiNegFIg/o+dQhIi1MY +Mf/rA0iJ2kiLdCRgTItEJDhMKcZIjU7/SInQSIn76Fv///9Ii1QkKEiLdCQYSIt8JDBIiXQXMEiJ +RBc4SItEJCBIi2wkQEiDxEjDSInZSInDMcBIi2wkQEiDxEjDSInIuT8AAADoVHIDAJBIiUQkCEiJ +XCQQSIlMJBgPH0AA6NtqAwBIi0QkCEiLXCQQSItMJBjp5/7//8zMzMzMzMxJO2YQD4aKAQAASIPs +IEiJbCQYSI1sJBhIiUQkKIQAi5AoAQEAiVQkEIsdLf8JAOsQi3QkFI1eAUiLRCQoi1QkEIH7EAEA +AA+DqgAAAJCJ3tHrD7bbZg8fhAAAAAAAZpBIgfuIAAAAD4MZAQAAiXQkFEiNHFtIweMGD7rmAHIt +SI0cGEiNm6ACAQDR6oPiAf/K99pIg/oCD4PdAAAASI0MkkiNBMvotrAAAOs0SI0cGEiNm1ACAQDR +6oPiAf/K99pIg/oCD4OkAAAASI0MkkiNBMvoibAAAGYPH4QAAAAAAEiFwA+ESP///5CQiw1r/gkA +i1QkFOtKkJCLDV3+CQCQ6wmQix1T/gkAidmD+f9zGYnISI0NQ/4JALr/////8A+xEQ+Uw4TbdNkx +wEiLbCQYSIPEIMOQiz0g/gkASInYifk50XMZSInDichIjTUL/gkA8A+xFg+UwYTJdNnrA0iJw0iJ +2EiLbCQYSIPEIMOJ0LkCAAAA6KZwAwCJ0LkCAAAA6JpwAwBIidi5iAAAAOiNcAMAkEiJRCQI6CJp +AwBIi0QkCOlY/v//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+G3AAAAEiD7DhIiWwkMEiN +bCQw6wb/BXj9CQDoGwIAAEiD+P9174sVt3wMAIlUJBQxwOtnSIlEJBhIweMGSIlcJChIjQ26fQwA +SI00GUiNFJJIiVQkIEiNBNZIjUAI6HiwAABIi0wkKEiNFZR9DABIjQwKSI1JWEiLVCQgSI0E0ehW +sAAASItMJBhIjUEBi0wkFInKDx+AAAAAAEg9iAAAAH0aSI0cQNHqg+IB/8r32kiD+gIPgnr///+Q +6xToed///+i0UQAASItsJDBIg8Q4w4nQuQIAAAAPHwDoe28DAJDoFWgDAOkQ////zMzMzMzMzMzM +zMzMzMzMzEk7ZhAPhiQBAABIg+woSIlsJCBIjWwkIIM94eYMAACQdQxMifFIiQ1c/AkA6w9IjT1T +/AkATInx6INrAwCQkEiNBTr8CQDoddP+/8YFPvwJAAFIiwXf+QkASI0dmK0EAOg7gP7/kEiNBfOv +BABIjR0M/AkAuQwAAAC/FAAAAL4BAAAA6JgnAQDrdpCQSI0F7fsJAOgo0/7/iw1OewwAZpCFyXQ2 +ixVGewwAhdJ1LMYF2/sJAAGQSI0Fo68EAEiNHbz7CQC5DAAAAL8UAAAAvgEAAADoSCcBAOsmkJBI +jQWd+wkA6LjU/v/rFv8FpPsJAJBIjQXQrgQA6ENlAwAPHwDoOwAAAEiD+P913+sNkEiNBbOuBADo +JmUDALgBAAAAkOibIwAAhMB15Olc////6M1mAwDpyP7//8zMzMzMzMzMSTtmEA+GPAMAAEiD7GhI +iWwkYEiNbCRgSYtOMP+BCAEAAIsNhHoMAIXJdRZMiXQkWIsNcXoMAIlMJCDGRCQkAOsbSYtOMP+J +CAEAAEjHwP////9Ii2wkYEiDxGjDSI0FHHkLAOiX+///SIXAD4S3AAAAikhjgPkBdBeLUFiLXCQg +OdN02I1zAznydNHpNAIAAItIWItUJCCDwv450XQGMdsxwOtCkIB8JCQAdRW5AQAAAEiNFfd5DADw +D8EKxkQkJAGLTCQgjVH+/8lIicOJ0PAPsUtYD5TBhMl0B7gBAAAA6wQx2zHASIlcJCiEwA+Eaf// +/0iLSyBIiUwkMEiNRCQoMdvo1wUAAITAdBZIi0wkMEiNFfd5DABIicvwSA/BCusCMdsxyescMcBI +jQ16eQwAugEAAADwD7ERD5TBSMfD/////0iJXCQwkIB8JCQAD4QeAQAAxkQkJAC6/////0iNNUl5 +DADwD8EW/8qF0nUNixU1eQwAhdIPlcLrAjHShNIPhOwAAACQgz0g5gwAAA+O3gAAAIhMJBZIiwWP +6QwASIlEJFBIiw1DeQwASIlMJEhIixUneQwASIlUJEDyDxAFMnkMAPIPEUQkOOgvEwEASI0Fr4oE +ALsfAAAADx8A6DscAQBIi0QkUEjB6BToLRkBAEiNBapmBAC7DgAAAJDoGxwBAEiLRCRQSItMJEhI +KchIwegU6AUZAQBIjQUQdwQAuxcAAADo9BsBAEiLRCRA6OoYAQBIjQUtXwQAuwoAAADo2RsBAPIP +EEQkOOjOFQEASI0Ff2IEALsMAAAAZpDouxsBAOgWEwEAD7ZMJBZIi1wkMITJdDFIjQUZrQQASIkE +JOj4YgMARQ9X/2RMizQl+P///5C4AQAAAEiNDd74CQCHAUiLXCQwSItMJFhIi0kw/4kIAQAASInY +SItsJGBIg8Row4hMJBeJVCQciVwkGOgqEgEASI0FEn4EALsaAAAA6DkbAQAPtkQkFw+2wOgsGAEA +SI0FAWIEALsMAAAA6BsbAQCLRCQc6BIYAQBIjQWlXgQAuwoAAADoARsBAItEJBjo+BcBAOhTFAEA +6E4SAQBIjQUviQQAux8AAABmkOhb+QAAkOh1YwMA6bD8///MzMzMzMzMzMzMzMzMzMzMSTtmEA+G +TwMAAEiD7EhIiWwkQEiNbCRASYtOMIO5CAEAAAB1FYO58AAAAAB1DEyJ8kg5EQ+FDQMAAIsNCHcM +AIlMJBDGRCQUAItIWItUJBCDwv450XQGMckx2+tIkIB8JBQAdRW5AQAAAEiNFd92DADwD8EKxkQk +FAGLTCQQjVH+/8lIicOJ0PAPsUtYD5TBhMl0CkiJ2LkBAAAA6wdIidgxyTHbSIlcJBiEyQ+FMwEA +AEiJRCRQkIB8JBQADx8AD4RlAgAAxkQkFAC5/////0iNFXl2DADwD8EK/8kPHwCFyXUNiw1idgwA +hckPlcHrAjHJhMkPhDACAACQgz1N4wwAAA+OIgIAAEiLBcDmDABIiUQkOEiLDXR2DABIiUwkMEiL +FVh2DABIiVQkKPIPEAVjdgwA8g8RRCQgDx9EAADoWxABAEiNBduHBAC7HwAAAOhqGQEASItEJDhI +wegUkOhbFgEASI0F2GMEALsOAAAA6EoZAQBIi0QkOEiLTCQwSCnISMHoFOg0FgEASI0FP3QEALsX +AAAA6CMZAQBIi0QkKOgZFgEASI0FXFwEALsKAAAA6AgZAQDyDxBEJCBmkOj7EgEASI0FrF8EALsM +AAAA6OoYAQDoRRABAEiLRCRQ6UYBAABIjUQkGDHb6I8BAACQgHwkFAAPhA0BAADGRCQUALj///// +SI0NQnUMAPAPwQH/yIXAdQ2LBS51DACFwA+VwOsCMcCEwA+E2wAAAJCDPRniDAAAD47NAAAASIsF +jOUMAEiJRCQ4SIsNQHUMAEiJTCQwSIsVJHUMAEiJVCQo8g8QBS91DADyDxFEJCDoLA8BAEiNBayG +BAC7HwAAAOg7GAEASItEJDhIwegU6C0VAQBIjQWqYgQAuw4AAACQ6BsYAQBIi0QkOEiLTCQwSCnI +SMHoFOgFFQEASI0FEHMEALsXAAAA6PQXAQBIi0QkKOjqFAEASI0FLVsEALsKAAAA6NkXAQDyDxBE +JCDozhEBAEiNBX9eBAC7DAAAAGaQ6LsXAQDoFg8BAEiLbCRASIPESMPop4EDAEUPV/9kTIs0Jfj/ +//9Ii0QkUItIWItUJBA50XQHg8IDOdF110iLbCRASIPESMNIjQVQjAQAuyIAAADo6PUAAJBIiUQk +CGaQ6PtfAwBIi0QkCOmR/P//zMzMzMzMzMzMzMzMzMzMzMxMjWQksE07ZhAPhp4LAABIgezQAAAA +SImsJMgAAABIjawkyAAAAEmLVjCDuggBAAAAdRaDuvAAAAAAdQ1MifaQSDkyD4VSCwAASIsQhNt1 +B0jHAAAAAACLNWtzDACJdCQoQIp6Y0CIfCQjQID/AQ+FoQoAAESNRv8PH4QAAAAAAEQ5QlgPhYsK +AACInCTgAAAASIlUJEhEiUQkLIA9LnEKAAB0I0iLQiBIweAN6A/JAgBIi1QkSA+2nCTgAAAAi3Qk +KESLRCQsSIt6IEyNDS5zDADwSQ/BOQ+2emJAiHwkIEyLSmhMiUwkQEyLkoAAAABMiVQkMEyNmoAA +AABMiZwkiAAAAEyJlCSQAAAA6wxMifqLdCQoD7Z8JCBMi5wkkAAAAE2F2w+E5wAAAEEPt0MITYXJ +D4TXCQAASInRMdJJ9/FIicJJD6/BTIthGEkBxJCQSYnVSMHqA0mD5QdJic9Miem/AQAAANPnkEkD +V1APtjJAhP50cEGAewoDdBmQTImcJIgAAABJixNIiZQkkAAAAOl4////kEmLE0iJlCSQAAAASIu0 +JIgAAABIiRZBxkMRAUyJ2EyJ40yJyQ8fRAAA6PtCAAAPtpwk4AAAAESLRCQsTItMJEBMi1QkMEyL +fCRI6Sj///9KjTwI6WYIAACQD6vOQIgyuAEAAADrBzHAkOsCMcBIiXwkaIhEJCHpZggAAE2F0g+E +ewAAAEiDuoAAAAAAdXFMixUMcgwAQYQCkJBIi0IYSbsAAAAAAIAAAEkBw0nB6xpJgfsAAEAAD4Py +BwAAT4sU2kGEAkjB6A1IicFIJf8fAABIwegDSD0ABAAAD4PEBwAATY0UAk2NkgAIIQBIg+EHQbsB +AAAAQdPjQffT8EUgGoM9Y94MAAB1CYM9Ht4MAAB0R0yLUlBMiZQkmAAAAMaEJKAAAAABSMeEJKgA +AAAAAAAATItSSEyJlCSwAAAAxoQkuAAAAAFIx4QkwAAAAAAAAAAxwOkRBgAASItKMEg5Sjh2ckyL +UlBJictIwekDRg+2FBFMi2JIRg+2JCFB99RFIdRJg+MHSInITInZQdLsRYTkdDhIiYQkgAAAAEiJ +0OimCAAASIuEJIAAAABIi1QkSA+2nCTgAAAAi3QkKA+2fCQgRItEJCxMi0wkQEiNSAHpHQUAAJBI +i0o4SIPBB0jB6QNIiYwkgAAAADHARTHS6wdIg8AITQHaTIlUJHBIOchzbEyLWlBNixwDTA+2JS3Z +DABNheR0B/NND7jb69NIiUQkWEyJ2A8fQADom1b+/0iLjCSAAAAASItUJEgPtpwk4AAAAIt0JCgP +tnwkIESLRCQsTItMJEBMi1QkcEwPtiXb2AwASYnDSItEJFjrgw+3SmBBictEKdFmiUwkJGZFOdoP +h64DAABmRIlSYEjHQjAAAAAAgD2fbQoAAHQhTYteME2Lm9AAAABBhANED7fhTItqaE0Pr+xNAatI +FgAASItKUEiJSkhIi0I46JtBAABIi0wkSEiJQVBIicgx2+gIE///SItcJEiKS2OITCQjgPkBD4XD +AgAAi1NYRItMJCxEOcoPhbICAABEi0wkKEWNUQFEOdIPhI8CAABFjVEDRDnSD4SCAgAARInKRIdL +WA+2RCQgQYnB0OiEwA+ERgEAAA+3dCQkDx9EAABmhfZ3DQ+2tCTgAAAAQIT262aIRCQixkNkAUiN +BWcGDQDoOqYAAIQAD7ZMJCJID77JSIP5RA+DFwIAAEiNDMhIjYl4AgAAD7dUJCQPt9LwSA/BEUiN +BS4GDQDoAacAAA+2jCTgAAAAhMmLVCQoSItcJEhED7ZMJCAPhZkBAABIi3QkcGaF9g+EjwAAAA+3 +9kg5czh1S2YPH4QAAAAAAGaQSYH5iAAAAA+DmAEAAEuNDElIweEGSI01TG8MAEiNDDFIjUlY0eqD +4gFIjRSSSI0E0egqnwAA6TwBAAAPH0QAAEmB+YgAAAAPg0kBAABLjQxJSMHhBkiNNQxvDABIjQwO +SI1JCNHqg+IBSI0UkkiNBNHo6p4AAOn8AAAASI0FnmwLAOhZNAAAuAEAAABIi6wkyAAAAEiBxNAA +AADDRA+2lCTgAAAARYTSD4XJAAAARA+3VCQkZkWF0g+EhwAAAIM9g9oMAAB+KUjHQ3AAAAAAkEiL +QxiQSItcJEAxyb8yAAAAvv////9FMcDoUmD+/+sMSI0FKWwLAOjkMwAASI0F5QQNAOi4pAAAuQEA +AADwSA/BiHACAABIi0wkQPBID8GIaAIAAEiNBb0EDQDokKUAALgBAAAASIusJMgAAABIgcTQAAAA +w0mB+YgAAABzO0uNDElIweEGSI01C24MAEiNDDFIjUlY0eqD4gFIjRSSSI0E0ejpnQAAMcBIi6wk +yAAAAEiBxNAAAADDRInIuYgAAADoKmADAESJyLmIAAAAZpDoG2ADAESJyLmIAAAA6A5gAwBIici5 +RAAAAOgBYAMASI0FpGAEALsRAAAA6HDuAACLQ1hIiUQkeOjDBgEASI0FK2MEALsTAAAA6NIPAQAP +tkQkIw+2wOjFDAEASI0FWFMEALsKAAAA6LQPAQBIi0QkeOiqDAEASI0FQVwEALsQAAAA6JkPAQCL +RCQo6JAMAQDo6wgBAOjmBgEASI0FkIoEALsnAAAA6PXtAABmRIlcJCZIi0I4SImEJIAAAAAPHwDo +OwYBAEiNBeJdBAC7EAAAAOhKDwEASIuEJIAAAABmkOg7DAEASI0FzU4EALsIAAAA6CoPAQBIi0Qk +cA+3wGaQ6BsMAQBIjQXrZQQAuxUAAADoCg8BAA+3RCQmDx9EAADo+wsBAEiNBZ1OBAC7CAAAAOjq +DgEAD7dEJCQPt8BmkOjbCwEA6DYIAQDoMQYBAEiNBSuABAC7IAAAAA8fRAAA6DvtAABI/8FMi1I4 +SYPCB0nB6gNMOdEPg876//9Mi1JQRg+2FBFMi1pIRg+2HBlB99NFhNp0zkiJTCRgSInQkOgbAwAA +SItMJGBIi1QkSA+2nCTgAAAAi3QkKA+2fCQgRItEJCxMi0wkQOuZSP+EJMAAAABI/8BmDx+EAAAA +AABIOUI4D4bl+f//TIuUJJgAAABED7acJKAAAABFD7YSkEWE2g+FiQAAAEyLlCTAAAAATDlSMHca +TIuUJLAAAABED7acJLgAAABFD7YSRYTadGFMi1JoTA+v0EwDUhiDPZzXDAAAdD9IiUQkUEyJVCQ4 +TInQTInL6CWLAABIi0QkUEiLVCRID7acJOAAAACLdCQoD7Z8JCBEi0QkLEyLTCRATItUJDiDPRjX +DAAAdAQxyet/kEQPtpQkoAAAAA8fQABBgPqAdRJI/4QkmAAAAMaEJKAAAAAB6wtB0eJEiJQkoAAA +AEj/hCSoAAAAkEQPtpQkuAAAAEGA+oB1FUj/hCSwAAAAxoQkuAAAAAHp3P7//0HR4kSIlCS4AAAA +Dx9EAADpx/7//0HHBArvvq3eSIPBBEw5yXLv6Xf///+5AAQAAOgbXQMATInYuQAAQADoDl0DAE2L +G02F2w+EpPf//0UPt2MISTn8D4OR9///QYB7CgF13+l39///SIuUJJAAAABIhdIPhIX2//8Pt3II +SDn+D4N49v//TYtfGEwB3oB6CgF0GoTAdBaQSImUJIgAAABIixJIiZQkkAAAAOu7kEiLOkiJvCSQ +AAAATIuEJIgAAABJiThIidBIifNMickPHwDo2zkAAA+2RCQhD7acJOAAAABIi3wkaESLRCQsTItM +JEBMi1QkMEyLfCRI6Wj////oq8UAAItCWEiJRCR4Dx8A6PsCAQBIjQVjXwQAuxMAAADoCgwBAA+2 +RCQjD7bAZpDo+wgBAEiNBY5PBAC7CgAAAOjqCwEASItEJHgPH0QAAOjbCAEASI0FclgEALsQAAAA +6MoLAQCLRCQo6MEIAQCQ6BsFAQDoFgMBAEiNBQ9wBAC7GwAAAOgl6gAASI0FNnMEALscAAAA6BTq +AACQSIlEJAiIXCQQ6CVUAwBIi0QkCA+2XCQQ6Tb0///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtm +EA+G3AIAAEiD7HhIiWwkcEiNbCRwSImEJIAAAADoGwIBAEiLhCSAAAAASItIaEiJTCQ4SItQMEiJ +VCQwkOj7AQEASI0FzoIEALskAAAA6AoLAQBIi4QkgAAAAGaQ6LsKAQBIjQU3UAQAuwsAAADo6goB +AEiLRCQ4Dx9EAADo2wcBAEiNBZNPBAC7CwAAAOjKCgEASItEJDAPH0QAAOi7BwEASI0FHo0EALsu +AAAA6KoKAQDoBQIBAEiLhCSAAAAASItIUEiJTCRAxkQkSAFIx0QkUAAAAABIi0hISIlMJFjGRCRg +AUjHRCRoAAAAADHJ6xZI/0QkaEiLVCQgSI1KAUiLhCSAAAAASDlIOA+GuwEAAEiJTCQgSItQaEgP +r9FIA1AYSIlUJCjoEAEBAEiLRCQo6IYIAQDogQEBAEiLRCQgSIuMJIAAAABIOUEwdge6AQAAAOsS +SItUJFgPtlwkYA+2EoTaD5XCiFQkH4TSdB3oyAABAEiNBVdHBAC7BgAAAOjXCQEA6DIBAQDrG+ir +AAEASI0FWEcEALsGAAAA6LoJAQDoFQEBAEiLRCRAD7ZMJEgPtgCEyHQkDx9AAOh7AAEASI0FJUoE +ALsJAAAA6IoJAQDo5QABAOseDx8A6FsAAQBIjQVNSgQAuwkAAADoagkBAOjFAAEASItEJEAPtkwk +SA+2AITIdAoPtkQkH4PwAesCMcCIRCQeDx9AAITAdBzoFwABAEiNBYFHBAC7BwAAAOgmCQEA6IEA +AQCQ6Pv/AADodgIBAOhxAAEAD7ZEJB6EwHQtSIuUJIAAAABIi3JoSIH+AAQAAL8ABAAASA9H90iL +RCQoSI0cBjHJkOg7CgEAkA+2TCRIgPmAdQxI/0QkQMZEJEgB6wbR4YhMJEhI/0QkUJAPtkwkYID5 +gHUTSP9EJFjGRCRgAQ8fQADpMP7//9HhiEwkYOkl/v//SI0FeW8EALscAAAADx9AAOj75gAAkEiJ +RCQI6BBRAwBIi0QkCOkG/f//zMzMzMzMSTtmEA+GJwEAAEiD7BhIiWwkEEiNbCQQ8g8QBRhlDAAP +V8lmDy7BdQJ7KUiJRCQgSIlcJCiAPbRiCgAAdC1mkOg7ugIASItEJCBIi1wkKA9XyesXSItsJBBI +g8QYw0iLRCQgSItcJCgPV8lIiw22ZAwASIsV99QMAEiLNbBkDABIKfJIAcLyDxAFqmQMAEiF0nwK +D1fS8kgPKtLrGUiJ1kiD4gFI0e5ICdYPV9LySA8q1vIPWNJIiQwk8g9ZwvJIDyzQSCnaSIlUJAjr +EkiLRCQgSItUJAhIi1wkKA9XyUiLNTpkDABIKc5IOdZ9K+hN6f//SIP4/3QVSIsFKGQMAEiLDCRI +Och0xOlQ////D1fA8g8RBR9kDACAPdBhCgAAdAXoWboCAEiLbCQQSIPEGMNIiUQkCEiJXCQQDx9E +AADou08DAEiLRCQISItcJBDprP7//8zMzMzMzMzMzMzMzEk7ZhB2P0iD7AhIiSwkSI0sJEiJRCQQ +6MQGAABIi0wkEEiJAejXCQAASIXAdQXorQYAAEiLTCQQSIlBCEiLLCRIg8QIw0iJRCQI6FFPAwBI +i0QkCOuqzMzMzMzMzMzMzEk7ZhAPht4AAABIg+wgSIlsJBhIjWwkGEiJXCQwSIlEJChIixBIhdJ0 +ZkiBehD9AAAAdVVIi3AISIkwSIlQCEiLVhBIgfr9AAAAdTdIiVQkEEiJ8OjnCAAASItMJCjGQSAB +6BkGAABIi0wkKEiJAUiLTCQQSIH5/QAAAEiLXCQwSInGQA+Ux+sbSInWMf/rFOgL////SItMJChI +izFIi1wkMDH/SItGEEg9/QAAAHMtSIlcxhhI/0YQQIT/dBWDPZ/LDAABdQxIjQWy0gwA6O27//9I +i2wkGEiDxCDDuf0AAADouVUDAJBIiUQkCEiJXCQQ6ElOAwBIi0QkCEiLXCQQ6fr+///MzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhqoBAABIg+xQSIlsJEhIjWwkSEiJXCRgDx8ASIXJdD9I +iUQkWEiLEEiF0nUuSIl8JHBIiUwkaEiJXCRg6Dr+//9Ii0wkWEiLEUiJyEiLTCRoSItcJGBIi3wk +cDH26ydIi2wkSEiDxFDDTCnPTCnJSYn4SPffScHhA0jB/z9MIc9IAftMicdIhckPjpcAAABIiXwk +OEiJXCRASIlMJDDp5AAAAA+H+AAAAE2NiAP///9NicpJ99lMOclMD0zJScHgA0nB+j9NIdBOjQQC +TY1AGEw5w3RBSIlUJCBMiUwkKECIdCQfTInJSMHhA0yJwOioXQMASItEJFhIi0wkMEiLVCQgSItc +JEAPtnQkH0iLfCQ4TItMJChMAUoQkEw5yQ+DRf///+tzQIT2dBWDPQ3KDAABdQxIjQUg0QwA6Fu6 +//9Ii2wkSEiDxFDDSInQ6MkGAABIi0wkWMZBIAHo+wMAAEiLTCRYSItRCEiJEUiJQQhIichIi0wk +MEiLXCRASIt8JDi+AQAAAEyLQhBJgfj9AAAAdLbpCv///0yJyOiaVAMATInAuf0AAADojVQDAJBI +iUQkCEiJXCQQSIlMJBhIiXwkIOhTTAMASItEJAhIi1wkEEiLTCQYSIt8JCDpGv7//8zMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GvAAAAEiD7ChIiWwkIEiNbCQgSIlEJDBIixBIhdJ1EOhW +/P//SItMJDBIixFIichIg3oQAHVTSItQCEiLGEiJEEiJWAhIg3oQAHU+SIlUJBjoJQYAAA8fRAAA +SIXAdB5IiUQkEEiLRCQY6EwFAABIi1QkEEiLTCQwSIkR6wwxwEiLbCQgSIPEKMNIi1oQSI1D/0iJ +QhAPH0QAAEg9/QAAAHMPSItE2hBIi2wkIEiDxCjDuf0AAAAPH0AA6LtSAwCQSIlEJAjoUEsDAEiL +RCQI6Sb////MzMzMzMxJO2YQD4bGAAAASIPsEEiJbCQISI1sJAhIiwgPH0QAAEiFyXRkSIlEJBhI +g3kQAHURSInI6KcEAABIi0wkGGaQ6xFIicjo9gQAAEiLTCQYxkEgAUjHAQAAAABIi0EISIN4EAB1 +DOh2BAAASItMJBjrDujKBAAASItMJBjGQSABSMdBCAAAAADrA0iJwUiLQRBIhcB0FEiNFWTTDADw +SA/BAkjHQRAAAAAASItBGEiFwHQUSI0VD88MAPBID8ECSMdBGAAAAABIi2wkCEiDxBDDSIlEJAjo +ZkoDAEiLRCQIkOkb////zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GmwAAAEiD7BBI +iWwkCEiNbCQISIsIDx9EAABIhcl0dkiJRCQYSItQCEiDehAAdShIg3kQBH4XSInIkOj7BAAASItM +JBhIiQHGQSAB6ylIi2wkCEiDxBDDSInQ6NsDAABIi0wkGMZBIAHoDQEAAEiLTCQYSIlBCIM94cYM +AAF1DEiNBfTNDADoL7f//0iLbCQISIPEEMNIi2wkCEiDxBDDSIlEJAjokUkDAEiLRCQI6Uf////M +zMzMzMzMSTtmEHYxSIPsGEiJbCQQSI1sJBBIg3gQAHQKSItsJBBIg8QYw0iNBXJPBAC7EAAAAOgq +3wAAkEiJRCQIDx9AAOg7SQMASItEJAjrtMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEHYxSIPsGEiJ +bCQQSI1sJBBIg3gQAHUKSItsJBBIg8QYw0iNBQNXBAC7FAAAAOjK3gAAkEiJRCQIDx9AAOjbSAMA +SItEJAjrtMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+G6AEAAEiD7FhIiWwkUEiNbCRQSIM96NAM +AAB0Bel3AQAAMcBIhcAPheoAAABIiUQkIEjHRCQQAAAAAEiDPRDRDAAAdE+QkEiNBf3QDADoKLT+ +/0iLHfnQDABIiVwkEEiF23QdSI0F6NAMAOirJwAASItcJBBIjQXn0AwA6BopAACQkEiNBcHQDADo +zLX+/0iLRCQgSIN8JBAAkHV2RA8RfCRASI0FUQEAAEiJRCRASI1EJBBIiUQkSEiNRCRASIkEJOi0 +RgMARQ9X/2RMizQl+P///0iDfCQQAJAPhKcAAACQkEiNBWHQDADojLP+/0iLXCQQSI0FaNAMAOib +KAAAkJBIjQVC0AwA6E21/v9Ii0QkIDHJ6w9Ii2wkUEiDxFjDSItMJDBIjZEACAAASIH6AIAAAHfh +SIlMJBhIiVQkMEiJRCQoSItUJBBIi1oYSI0EC0iJRCQ4hAJIx0AQAAAAAOgVsv7/SItMJBhIhcl1 +B0iLRCQ466pIi0QkOOjaAAAASItEJCjrmUiNBTlIBAC7DQAAAOgC3QAASIsNY88MAEiFyXQkSInI +SMH5E0jB4QOQSIsRSI0dSM8MAPBID7ETD5TChNJ00usCMclIhcl0FUiJTCQ4SInIDx8A6Lv9//9I +i0wkOEiJyOk3/v//6MlGAwDpBP7//8zMzMxJO2YQdj9Ig+woSIlsJCBIjWwkIEiLUghIiVQkGEiN +BVxZCwC7BAAAALkDAAAA6K0UAABIi1QkGEiJAkiLbCQgSIPEKMPo1kUDAOu0zMzMzMzMzMzMzMzM +zMzMzMzMzMxJO2YQdjVIg+wYSIlsJBBIjWwkEEiJRCQg6CL9//9Ii1wkIIQDSI0FfM4MAOiPr/7/ +SItsJBBIg8QYw0iJRCQI6BtGAwBIi0QkCOu0zMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdjVIg+wY +SIlsJBBIjWwkEEiJRCQg6GL8//9Ii1wkIIQDSI0FFM4MAOgvr/7/SItsJBBIg8QYw0iJRCQI6LtF +AwBIi0QkCOu0zMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdnNIg+wYSIlsJBBIjWwkEEiLDcXNDAAP +H0QAAEiFyXQkSInISMH5E0jB4QOQSIsRSI0dpc0MAPBID7ETD5TChNJ0zesCMclIhcl0HEiJTCQI +SInI6MP7//9Ii0QkCEiLbCQQSIPEGMNIichIi2wkEEiDxBjD6CJFAwBmkOl7////zMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GkAAAAEiD7ChIiWwkIEiNbCQgSIlEJDAPHwDoG/z//0iL +TCQwSItREEiJ00jB6j9IAdpI0fpIKdNIiVkQSIlQEEiLWRBIgfv9AAAAczlIiUQkGEiNcBhIjRzZ +SI1bGEjB4gNIifBIidHoDVUDAEiLRCQw6IP+//9Ii0QkGEiLbCQgSIPEKMNIidi5/QAAAOjHSwMA +kEiJRCQIkOhbRAMASItEJAjpUf///8zMzMzMzMzMzMzMzMzMzMzMSTtmEA+G4gAAAEiD7BhIiWwk +EEiNbCQQkEiNBbDMDADo26/+/0iDPVPMDAAAD4WnAAAASMcFSswMAAAAAABIiw2jzAwASIXJdSaQ +kEiNBX3MDADoiLH+/0iLbCQQSIPEGMNIjRVvzAwASIlREEiLCUiFyXXtSIsNXMwMAA8fQABIhcl0 +LEiLFWTMDABIiQpIiw1CzAwASIsVU8wMAEiJUQhIiw1AzAwASIkNKcwMAOscSIsNOMwMAEiLFSnM +DABIiRUSzAwASIkNE8wMAEQPET0TzAwA6XD///9IjQXidgQAuygAAADoNdkAAJDoT0MDAOkK//// +zMzMzMzMzMzMzEk7ZhAPhrkAAABIg+woSIlsJCBIjWwkIIhEJDCQSI0FrMsMAOjXrv7/gz00wAwA +AHUKSIM9nssMAAB1GpCQSI0Fi8sMAOiWsP7/McBIi2wkIEiDxCjDSMdEJBAAAAAAxkQkGABIjQV1 +AAAASIlEJBAPtkQkMIhEJBhIjUQkEEiJBCToeUEDAEUPV/9kTIs0Jfj///+QkEiLBTvLDABIiUQk +CEiNBSfLDADoMrD+/0iLRCQISIXAD5XASItsJCBIg8Qow4hEJAjodEIDAA+2RCQI6Sr////MzMzM +zMzMzMzMSTtmEA+GnwAAAEiD7EBIiWwkOEiNbCQ4SYtOMEiLicAAAABIiUwkMA+2UgiIVCQfMcDr +TEiJRCQoSIlcJCBIjQWzygwA6HYhAABIjQXPVAsASItcJCC5AwAAAA8fRAAA6FsdAABIi0wkKEiN +QQEPtkwkH0iLVCQwSInRD7ZUJB9Ig/hAfRmE0nQJgLmxAAAAAHUMSIsdXsoMAEiF23WVSItsJDhI +g8RAw+gSQQMA6U3////MzMzMzMzMzMzMzMzMSTtmEA+GfwEAAEiD7EBIiWwkOEiNbCQ4hABIi7BI +AQEASDmwQAEBAA+MBAEAAEiJRCRISIlcJFBIx0QkIAAAAABEDxF8JChIjQx2SInKSMHpP0gB0UjR ++UiB+QAgAAC4ACAAAEgPT8FIiUQkGEjB4ANIjR0I1gwA6CM6//9IiUQkIEiFwA+E7AAAAEiLdCRI +SIuOQAEBAEiJTCQoSIt8JBhIiXwkMEiLvkABAQBIi544AQEASIX/fh9IOc9ID0zPSItEJCBIOdh0 +DkjB4QPoTVEDAEiLdCRISIuGOAEBAEiLnkgBAQBIi3wkKEyLRCQwTItMJCBIg75AAQEAAEyJjjgB +AQBIib5AAQEATImGSAEBAHQWSMHjA0iNDWHVDACQ6Fs9//9Ii3QkSEiJ8EiLXCRQSIuwQAEBAEiL +kEgBAQBIjU4BSDnKcilIiYhAAQEASIuQOAEBAEg5znMOSIkc8kiLbCQ4SIPEQMNIifDojkcDAOgJ +SAMASI0FJ2YEALsfAAAA6PjVAACQSIlEJAhIiVwkEOgIQAMASItEJAhIi1wkEOlZ/v//zMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzJBIuQAAAAAAgAAASAHBSMHpGkiB+QAAQAByBDHJ6y5IixUSVAwA +hAJIiwzKSIXJdBpIicJIwegNSCX/HwAASIuMwQAAIABIidDrAjHJSIXJdB5IOUEYdxiQilFj/8pm +kID6AXcISDlBcA+XwMMxwMMxwMPMzMzMzMzMzMzMzMzMkEi5AAAAAACAAABIAcFIwekaSIH5AABA +AHIEMcnrLkiLFZJTDACEAkiLDMpIhcl0GkiJwkjB6A1IJf8fAABIi4zBAAAgAEiJ0OsCMclIhcl0 +C4pRY4D6AQ+VwusFugEAAACE0nUMSDlBGHcGSDlBcHcDMcDDSInIw8zMzMzMzMxJO2YQD4aIAwAA +SIPsIEiJbCQYSI1sJBiEAJBIx4BIaAEAiAAAAIM9s70MAAB1F0iNFTKHBABIiZBQaAEASImAWGgB +AOsfSI24UGgBAEiNFRSHBADoZ0IDAEiNuFhoAQDoO0EDAEQPEbhgaAEAx4BwaAEAAAAAAEjHgHho +AQAAAAAAgz1XvQwAAHUQSI0V9tIMAEiJkIBoAQDrE0iNuIBoAQBIjRXf0gwA6BJCAwDGgIhoAQAB +kEjHgJBoAQCwBAAAgz0YvQwAAHUKRA8RuJhoAQDrH0iNuJhoAQAx0g8fAOjbQQMASI24oGgBADH2 +6A1CAwBEDxG4qGgBAMeAuGgBAAAAAABIx4DAaAEAAAAAAIM9ybwMAAB1EEiNFXjSDABIiZDIaAEA +6xNIjbjIaAEASI0VYdIMAOiEQQMAxoDQaAEAAZBIx4DYaAEAMAAAAIM9irwMAAB1CkQPEbjgaAEA +6xxIjbjgaAEAMdLoUEEDAEiNuOhoAQAx9uiCQQMARA8RuPBoAQDHgABpAQAAAAAASMeACGkBAAAA +AACDPT68DAAAdRBIjRUV0gwASImQEGkBAOsTSI24EGkBAEiNFf7RDADo+UADAMaAGGkBAAGQSMeA +IGkBABgAAACDPf+7DAAAdQpEDxG4KGkBAOsdSI24KGkBADH26AVBAwBIjbgwaQEARTHA6BZBAwBE +DxG4OGkBAMeASGkBAAAAAABIx4BQaQEAAAAAAIM9srsMAAB1CUiJkFhpAQDrDEiNuFhpAQDoe0AD +AMaAYGkBAAGQSMeAaGkBABgAAACDPYG7DAAAkHUKRA8RuHBpAQDrHUiNuHBpAQAx9uiGQAMASI24 +eGkBAEUxwOiXQAMARA8RuIBpAQDHgJBpAQAAAAAASMeAmGkBAAAAAACDPTO7DAAAdQlIiZCgaQEA +6w1IjbigaQEAkOj7PwMAxoCoaQEAAZBIx4C4aQEAGAAAAIM9AbsMAACQdQpEDxG4wGkBAOscSI24 +wGkBADH26AZAAwBIjbjIaQEAMfbo+D8DAEQPEbjQaQEAx4DgaQEAAAAAAEjHgOhpAQAAAAAAgz20 +ugwAAHUJSImQ8GkBAOsOSI248GkBAGaQ6Hs/AwDGgPhpAQABxoCIaAEAADHJ6xJIjRRJSMHiBoiM +EEgCAQBI/8FIgfmIAAAAfOVIjVAISInDSI0NONAMAEiJ0OhYJgAASItsJBhIg8Qgw0iJRCQI6EQ7 +AwBIi0QkCOla/P//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4YEAgAASIPsaEiJbCRg +SI1sJGCEAEiLkIgBAQBJuQAAAAAAAACATDnKc21IiUQkcEmLVjCQ/4IIAQAASYtWMEiJVCRQgD3A +TAoAAHQjSIlcJHjoRKQCAEiLRCRwSItUJFBIi1wkeEm5AAAAAAAAAIBIi4gIAgEASIlMJDhMi5AA +AgEATIlUJFhIi7gQAgEASIl8JEAx9usQSItsJGBIg8Row0yJwEyJ60iF2w+GAQEAAEyLmJABAQAP +HwBNhdt2Nkk5202J3EwPR9tNieVNKdxJicBMiejwTQ+xoJABAQBBD5TERQ+25EmJ3Uwp202F5EwP +RevrrEG7AAIAAPBMD8GYiAEBAE2J3EnB6w1MOdkPhpYAAABIiVwkMJBAhPZ1JUyJZCRIkJDor6X+ +/0iLRCRwSItMJDhIi3wkQEyLVCRYTItkJEhMidNMieZBuAACAADo5QAAAEiLVCQwSDnQdwpIKcJM +i0wkcOsTSCnQTItMJHDwSQ/BgZABAQAx0kiLTCQ4SIt8JEBNichJuQAAAAAAAACATItUJFhJidW+ +AQAAAEiLVCRQ6ff+//9Mh4iIAQEAQIT2dA6QkGaQ6Pum/v9Ii1QkUIA9P0sKAAB0CujIowIASItU +JFCQi4IIAQAAjUj/iYoIAQAAg/gBdRJBgL6xAAAAAHQIScdGEN76//9Ii2wkYEiDxGjDSIlEJAhI +iVwkEOgDOQMASItEJAhIi1wkEOnU/f//zMzMzMzMzMzMzMzMzMzMzMzMzMxMjWQkyE07ZhAPhuQE +AABIgey4AAAASImsJLAAAABIjawksAAAAEiJhCTAAAAATImEJOgAAABIiZwkyAAAAEiJjCTQAAAA +hACLFXpMDACJVCQYxkQkHABMicIx/+sOS400wknB4ANNKcdNifhNhcAPhtAAAABJifFIwe4NDx9A +AEg58Q+GYQQAAEyLkJgBAQBBhAJIizTzZg8fhAAAAAAASIH+AABAAA+DMAQAAEmLNPKEBk2JykmB +4f8fAABNictJwekDSYH5AAQAAA+H/wMAAE2NoQD8//9NieVJwfw/TSHhTo0kDk2NpCQAACEASffd +To0MDk2NiQAEIQBNicdJwegDTTnFdwNNiehMibwkkAAAAEiJdCRITImcJIgAAABMiaQkoAAAAEyJ +jCSYAAAATIlEJDhMiZQkgAAAAEUx7emdAQAASIl8JDCQgHwkHAAPhCkBAADGRCQcALn/////SI0d +aUsMAPAPwQv/yYXJdQ2LDVVLDACFyQ+VwesCMcmEyQ+E9wAAAJCDPUC4DAAAD47pAAAASIsFs7sM +AEiJRCR4SIsNZ0sMAEiJTCRwSIsVS0sMAEiJVCRo8g8QBVZLDADyDxFEJGDoU+UAAEiNBdNcBAC7 +HwAAAOhi7gAASItEJHhIwegU6FTrAABIjQXROAQAuw4AAADoQ+4AAEiLRCR4SItMJHBIKchIwegU +6C3rAABIjQU4SQQAuxcAAACQ6BvuAABIi0QkaOgR6wAASI0FVDEEALsKAAAADx9EAADo++0AAPIP +EEQkYOjw5wAASI0FoTQEALsMAAAADx9AAOjb7QAA6DblAABIi4QkwAAAAEiLlCToAAAASIt8JDCA +PUpICgAAdDSQkOjxo/7/SIuEJOgAAABIi0wkMEgpyEjB4A3oGKACAJCQSIuEJMAAAADo6aH+/0iL +fCQwSIn4SIusJLAAAABIgcS4AAAAw0n/xUiLlCToAAAASIucJMgAAABNOegPjnP9//9LjRQsSImU +JKgAAACKEg8fgAAAAABNOegPhrwBAABDD7ZcDQD30yHahNJ0uEyJbCRAMdtmkOsUSI1ZAUiLjCTQ +AAAATIu8JJAAAABIg/sIc5NIidlBvwEAAABB0+dBhNd01U+NPOtJAc9mDx+EAAAAAAAPHwBJgf8A +IAAAD4NHAQAATou8/gAAIABFi1dYRItMJBhBg8H+RTnKdAhFMf9FMcnrW5CAfCQcAHUXQbkBAAAA +TI0VLEkMAPBFD8EKxkQkHAFEi0wkGEWNUf5B/8lEidDwRQ+xT1hBD5TBRYTJdBBIi4QkwAAAAEG5 +AQAAAOsOSIuEJMAAAABFMf9FMclMiXwkIEWEyQ+EpwAAAEiJTCRYSIl8JFBJi08gSIlMJCiQkOhu +ov7/SI1EJCAx2+ji1P//iEQkF5CQSIuEJMAAAADob6D+/0iLjCSoAAAAihFIi1wkKEiLfCRQSAH7 +D7Z0JBdAD7b2SIX2SA9F+0iLnCSYAAAASIt0JEBED7YEHkH30EQhwkiLhCTAAAAASItMJFhIictI +i3QkSEyLRCQ4TIucJIgAAABMi6QkoAAAAEyLbCRATIuMJJgAAABMi5QkgAAAAOlu/v//TIn4uQAg +AAAPH0AA6Js7AwBMiehMicHocDsDAEyJyLkABAAA6EM8AwBIifC5AABAAOh2OwMASInw6G47AwCQ +SIlEJAhIiVwkEEiJTCQYSIl8JCBIiXQkKEyJRCQw6MozAwBIi0QkCEiLXCQQSItMJBhIi3wkIEiL +dCQoTItEJDDpx/r//8zMzMzMzMxJO2YQD4bbAAAASIPsSEiJbCRASI1sJEBAiHwkYUjHRCQQAAAA +AEjHRCQYAAAAAEiNVCQgRA8ROkiNVCQwRA8ROkiNFdgAAABIiVQkGEiJRCQgSIlcJCiITCQwSI1E +JBBIiUQkOEiNRCQYSIkEJOjtMQMARQ9X/2RMizQl+P///0iLRCQQSIXAdE0PtkhkhMkPtlQkYYTS +dB+EyXQdSItIGEiLWCBIweMNSInI6K9AAwC5AQAAAOsFhMkPlMFIi1QkEMZCZABIi0QkEInLSIts +JEBIg8RIwzHAMdtIi2wkQEiDxEjDSIlEJAhIiVwkEIhMJBhAiHwkGeijMgMASItEJAhIi1wkEA+2 +TCQYD7Z8JBnp6v7//8zMzMzMzMzMzMxJO2YQdnVIg+xASIlsJDhIjWwkOEiLWhAPtnoYQIh8JB9I +i3IgSIl0JDBIi0IIixU4RgwAhdJ0DIsVMkYMAGaQhdJ0HkiJRCQoSIlcJCDoDff//0iLRCQoSItc +JCAPtnwkHzHJ6LcDAABIi1QkMEiJAkiLbCQ4SIPEQMMPH0QAAOhbMQMA6Xb////MzMzMzMzMzMzM +zMzMzMzMzMzMzMzMSTtmGHY1SIPsIEiJbCQYSI1sJBiEyXQRMf/oYQMAAEiLbCQYSIPEIMNIjQX6 +dAQAuzwAAADohscAAJBIiUQkCEiJXCQQiEwkGOjSWAMASItEJAhIi1wkEA+2TCQY66HMSIPsGEiJ +bCQQSI1sJBCEAEiLkJgBAQCEApBIvgAAAAAAgAAATI0EHknB6BpJgfgAAEAAc3lJidlIwesNSosU +wkUxwOsOhAJKibzSAAAgAE2NQwFMOcF2Pk6NFANJgeL/HwAATYXSdSlIi5CYAQEAhAJNicNJweAN +TQHISQHwScHoGkmB+AAAQABzFkqLFMLrtE2Jw+uvSItsJBBIg8QYkMNMicC5AABAAOhSOAMATInA +uQAAQADoRTgDAJDMzMzMSTtmEA+GEAEAAEiD7BhIiWwkEEiNbCQQMdLrF0kp3EuNHAxJwewNTSnj +SInITInZRInSSIXJD4Z9AAAAhABIi7CYAQEAhAaQSL8AAAAAAIAAAEyNBB9JwegaZg8fRAAASYH4 +AABAAA+DnwAAAEqLNMaEBkyLhggMIQBJidlIgeP///8DSTnYQQ+XwkmJy0jB4Q1MjSQLQQnSDx+A +AAAAAEmB/AAAAAR2IkG8AAAABOsaSInB6WP///+J0EiLbCQQSIPEGMNIichJidBNOcR24UiJwUyJ +wPBMD7GmCAwhAA+UwoTSD4Uy////SIuWCAwhAEw54nfPSDnadspIjQW/bgQAuzMAAADolMUAAEyJ +wLkAAEAA6Cc3AwCQSIlEJAhIiVwkEEiJTCQY6JIvAwBIi0QkCEiLXCQQSItMJBgPHwDpu/7//8zM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhgPhskAAABIg+woSIlsJCBIjWwkIIQASYtWMEiL +ktAAAABIhdJ0GEiDuigSAAAAdSNIiUQkMEiJVCQYMcnrfkgFSGgBAOjzL///SItsJCBIg8Qow0iL +migSAABIjUP/SD2AAAAAcxxIi4zaKBIAAEiJgigSAABIichIi2wkIEiDxCjDuYAAAADoMDYDAEiJ +TCQQSI2ISGgBAEiJyJDomy///0iLTCQQSItUJBhIiYTKMBIAAEj/wUiLRCQwkEiD+UB8ykjHgigS +AABAAAAA64RIiUQkCOjDVQMASItEJAjpGf///8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YY +D4aJBQAASIPsaEiJbCRgSI1sJGCIjCSAAAAAQIi8JIEAAABIiUQkcEiJXCR4SYtWMEiLktAAAAAP +H0AASIXSD4S7AAAASIP7EA+DsQAAAEiDelAAdUJIiVQkWIQAkOi5mf7/SItMJHBIjUEI6CtGAABI +i1QkWEiJQkhIiVpQSIlKWJCQSItEJHDobpv+/0iLVCRYSItcJHhIjUJI6NtBAABIhcB0S0mLVjBI +i5LQAAAASIXSdC5Ii7IoEgAASIX2dCJIjX7/SIH/gAAAAA+DrgQAAEiLtPIoEgAASIm6KBIAAOsC +MfZIhfYPhecAAADrAjH2SInCSYnYSItEJHDrBzHSMfZFMcBMiUQkKEiJdCRASIlUJDCEAJAPHwDo ++5j+/0iLTCQwSIXJdApIichIi1wkKOtgSItMJHBIjUEISIlEJFBIi1wkeOivMwAASIXAdUNIi0Qk +cEiLXCR46HsEAACEwHUYkJBIi0QkcOiLmv7/McBIi2wkYEiDxGjDSItEJFBIi1wkeOhwMwAASIXA +D4TZAwAASIlcJChIiUQkMEiLTCRASIXJdQ1Ii0QkcOhJ/f//SInBSIlMJECQkEiLRCRw6DWa/v9I +i3QkQEiLRCQwSItcJChIiXQkIEiJRCQwSIlcJChEDxE+SMdGEAAAAABIiUYYSItMJHhIiU4gZsdG +YAAAxkZiAEjHRmgAAAAARA8RfnjGRmQASMdGMAAAAABEDxF+SJBIjVZjSIlUJEgx/0CGOkiJx0iL +RCRwSIn76Ff7//+EwHQLSItUJCDGQmQB6wVIi1QkIEiLRCR4SInDSMHjDUiJXCQ4D7a0JIAAAABA +hPZ0MUjHQigAAAAASMdCOAAAAABIi0ogSMHhDUgDShhIiUpwuQIAAABIi3wkSIYP6eAAAAAPtrwk +gQAAAECIemJA0O8PH0QAAECE/3UYSIlaaEjHQjgBAAAAx0JcAAAAAEmJ0etJSA++/0iD/0QPg3oC +AABIjQ0OjQkAD7cMeUiJSmhmDx9EAABIhckPhFcCAABIidhJidEx0kj38UmJQThIjQ2hjwkAiwy5 +QYlJXEnHQTAAAAAAScdBQP////9Ji0E46EERAABIi0wkIEiJQVBIi0E46E8UAABIi0wkIEiJQUhI +i1QkcIQCi5ooAQEAh1lYkLsBAAAASIt0JEiGHkiLRCR4SInKSItcJDgPtrQkgAAAAEiLTCQoSIXJ +dQVAhPbrRUiLRCQw6Psm//9Ii0wkKEiJykj32UiNNSm/DADwSA/BDg+2jCSAAAAAhMlIi0QkeEiJ +0UiLVCQgSItcJDgPtrQkgAAAAHUUSI09774MAEmJ2PBID8EfQIT26wNJidh0EkyJw0j320iNBci+ +DADo83QAAEiNBdTVDADop3UAAEiLTCQoSInK8EgPwQhI99rwSA/BUAgPtowkgAAAAID5AXckDx9A +AITJdQ1Ii1QkOPBID8FQEOsxSItUJDjwSA/BUBjrJGaQgPkCdQ1Ii1QkOPBID8FQKOsQgPkDdQtI +i1QkOPBID8FQIEiNBV/VDADoMnYAAEiLfCQgSItfGEiLRCRwSItMJHjoOvj//w+2jCSAAAAAhMl1 +f0iLFf89DACEApCQSItcJCBIi0sYkEi+AAAAAACAAABIjQQOSMHoGg8fgAAAAABIPQAAQABzakiL +BMKEAEiJykjB6RBIgeH/AwAASI0ECEiNgAAAIQBIweoNSIPiB0iJ0boBAAAA0+LwCBBIi0QkcIQA +SItUJHjwSA/BkFgBAQDoaikDAEUPV/9kTIs0Jfj///9Ii0QkIEiLbCRgSIPEaMO5AABAAOiEMAMA +Dx9AAOjbmQAASIn4uUQAAADoTjADAEiNBctfBAC7KwAAAGaQ6Lu+AABIifi5gAAAAOguMAMAkEiJ +RCQISIlcJBCITCQYQIh8JBno9U8DAEiLRCQISItcJBAPtkwkGA+2fCQZkOk7+v//zMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GygIAAEiD7GBIiWwkWEiNbCRYSIlEJGiEAEiBw/8BAABI +geMA/v//SMHjDUiLkDACAQBIjTQTSIs9xaYMAEyNBDdNjUD/SPffSSH4TDmAOAIBAHIPZg8fRAAA +SDnyD4Z+AQAASIlcJEDoLZ/+/0iFwA+E+gAAAEiJwkiLdCRoSIu+OAIBAEg5+HUVSI0UGEiJljgC +AQAxwA8fAOmrAAAASIuGMAIBAEgpx0iF/3UHMcDpgAAAAEiJVCRQSIlcJDBIiXwkKEiJ+0iNDRy8 +DADoxyT//0iLVCQoSI01G7wMAPBID8EWSI0FF9MMAOjqcgAASItUJCjwSA/BUAhIjQUA0wwA6NNz +AABIi1QkaEiNQghIi5owAgEASItMJCjo2RMAAEiLVCRQSItcJDBIi3QkaEiLRCQoSInXSImWMAIB +AEiNFB9IiZY4AgEATIsFsaUMAEiLljACAQBIi3wkQEgB+kmNFBBIjVL/SffYSSHQ63ZIiwVzuwwA +SIlEJDjoOdUAAEiNBe1aBAC7KAAAAOhI3gAASItEJEAPHwDoO9sAAEiNBVInBAC7DQAAAOgq3gAA +SItEJDgPH0QAAOgb2wAASI0FgR4EALsJAAAA6AreAADoZdUAADHASItsJFhIg8Rgw0iJxjHASIlE +JCBIi5YwAgEASIlUJBhMiYYwAgEASInXSSn4TIlEJEBMicNIjQ3XugwASInQDx9AAOh7I///SItU +JEBIjTXPugwA8EgPwRZIjQXL0QwADx8A6JtxAABIi1QkQPBID8FQCEiNBbHRDADohHIAAEiLVCRo +SI1CCEiJRCRISItcJBhIi0wkQOiHEgAASIsVcLoMAEiLNXm6DABIi3wkIEyLRCRASY0cOEgp8kgB +2kiLdCRoSIu2gAEBAEg51nMZSCnySDnaSA9C2kiLRCRIMckPHwDoe6L//7gBAAAASItsJFhIg8Rg +w0iJRCQISIlcJBBmkOibJQMASItEJAhIi1wkEOkM/f//zMzMzMzMzMzMzMzMSTtmEHZYSIPsKEiJ +bCQgSI1sJCBEDxF8JAhIx0QkGAAAAABIjQ1WAAAASIlMJAhIiUQkEEiJXCQYSI1EJAhIiQQk6Pkj +AwBFD1f/ZEyLNCX4////SItsJCBIg8Qow0iJRCQISIlcJBDoEyUDAEiLRCQISItcJBDrh8zMzMzM +zMxJO2YQdk9Ig+wwSIlsJChIjWwkKEiLQghIiUQkIIQASItKEEiJTCQYkOiSkP7/SItEJCBIi1wk +GDHJ6KEAAACQkEiLRCQg6FWS/v9Ii2wkKEiDxDDD6AYkAwDrpMzMzMxJO2YYdlVIg+woSIlsJCBI +jWwkIIhMJB9IiVwkOEiJRCQwxkNkAYQAkOgykP7/SItEJDBIi1wkOA+2TCQfDx8A6DsAAACQkEiL +RCQw6O+R/v9Ii2wkIEiDxCjDSIlEJAhIiVwkEIhMJBjocksDAEiLRCQISItcJBAPtkwkGOuBzEk7 +ZhAPhjwDAABIg+xISIlsJEBIjWwkQEiJXCRYhACKU2OA+gEPhaAAAAAPt1NgZolUJB5mhdIPhS4C +AACLc1g5sCgBAQAPhR8CAABIi1MgSPfa8EgPwZBYAQEASIsVFTgMAIQCkJBIi3MYkEi/AAAAAACA +AABIAfdIwe8aDx8ASIH/AABAAA+D0QEAAEiLFPqEAkiJ90jB7hBIgeb/AwAASI0UMkiNkgAAIQBI +we8NSIPnB4nOSIn5QbgBAAAAQdPgQffQ8EQgAusWgPoCD4V5AQAAZoN7YAAPhV0BAACJzkiLSyBI +weENQIT2dRdIicpI99lIjT2dtwwA8EgPwQ9AhPbrA0iJykiJRCRQQIh0JB1IiVQkOHQPSI0FcLcM +AEiJ0+iYbQAASI0Fec4MAOhMbgAAD7ZMJB2A+QF3JmaQhMl1EEiLTCQ4SPfZ8EgPwUgQ6zhIi0wk +OEj32fBID8FIGOsogPkCdRBIi0wkOEj32fBID8FIKOsTgPkDdQ5Ii0wkOEj32fBID8FIIEiNBRXO +DADo6G4AAEiLTCRQSI1BCEiLVCRYSItaIEiLchhIid9IifNIifnoxCsAAJBIi0wkWEiNUWMx24Ya +SYtWMEiLktAAAABIhdJ0IkiLgigSAABIPYAAAAB9E3NHSImMwjASAABI/4IoEgAA6yyQSItEJFBI +i5B4aAEASCuQSGgBAEiJkHhoAQBIi5BgaAEASIkRSImIYGgBAEiLbCRASIPESMO5gAAAAOhEKQMA +SI0FzFYEALspAAAA6LO3AABIjQWSVgQAuykAAADoorcAAEiJ+LkAAEAA6DUpAwCLS1hIiUwkMIuA +KAEBAEiJRCQoSItTGEiJVCQg6NTPAABIjQVNQAQAuxwAAADo49gAAEiLRCRY6JnYAABIjQW7FQQA +uwUAAADoyNgAAEiLRCQgDx8A6BvXAABIjQUMHwQAuwwAAADoqtgAAA+3RCQeDx9EAADom9UAAEiN +BSQcBAC7CgAAAOiK2AAASItEJDAPH0QAAOh71QAASI0F+xMEALsBAAAA6GrYAABIi0QkKA8fRAAA +6FvVAADottEAAOixzwAASI0FxE4EALsjAAAADx9EAADou7YAAJBIiUQkCEiJXCQQiEwkGOjHIAMA +SItEJAhIi1wkEA+2TCQY6ZP8///MzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GQQEAAEiD7DBIiWwk +KEiNbCQoSItLEA8fQABIOch1TEg5GHUISIsLSIkI6w5Ii0sISIsTSIkRDx9AAEg5WAh1CkiLSwhI +iUgI6wtIiwNIi0sISIlICEQPETtIx0MQAAAAAEiLbCQoSIPEMMNIiUQkOEiJXCRASIlMJCBIi0Mg +SIlEJBhIi0sISIlMJBDoSc4AAEiNBXJZBAC7LQAAAOhY1wAASItEJBjoTtQAAEiNBQsVBAC7BgAA +AGaQ6DvXAABIi0QkQOjx1gAASI0F6BQEALsGAAAADx9EAADoG9cAAEiLRCQQ6NHWAABIjQU3HAQA +uwsAAAAPH0QAAOj71gAASItEJCDosdYAAEiNBZAUBAC7BgAAAA8fRAAA6NvWAABIi0QkOOiR1gAA +6CzQAADoJ84AAEiNBZ4kBAC7EAAAAOg2tQAAkEiJRCQISIlcJBDoRh8DAEiLRCQISItcJBDpl/7/ +/8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+G3AAAAEiD7DBIiWwkKEiNbCQoSIsLSIlMJCBI +hcl1PEiDewgAdTVIg3sQAHUuSIsISIkLSIsIDx9AAEiFyXQGSIlZCOsESIlYCEiJGEiJQxBIi2wk +KEiDxDCQw0iJXCRASItDCEiJRCQYSItLEEiJTCQQ6OPMAABIjQUqSQQAuyEAAADo8tUAAEiLRCRA +6KjVAADoA88AAEiLRCQg6JnVAADo9M4AAEiLRCQY6IrVAADo5c4AAEiLRCQQ6HvVAADoFs8AAOgR +zQAASI0FeCMEALsQAAAADx9EAADoG7QAAJBIiUQkCEiJXCQQ6CseAwBIi0QkCEiLXCQQkOn7/v// +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+G7QEAAEiD7EBIiWwkOEiNbCQ4SIlEJEhI +iVwkUOh53v//SIXAD4S1AQAASIlEJBhJi04wkP+BCAEAAEmLTjBIiUwkKOhSuv//SItMJEhIi1Qk +GEgrShhIiUwkIEiLXCRQD7ZzCkCIdCQXSI1CeEiJRCQwkOhCif7/SItMJBhIjZGAAAAAD7ZcJBdI +i3QkIOsDSIn6SIs6Dx9AAEiF/3RmRA+3RwhJOfB1CDhfCnQTSTnwd1J12w8fRAAAOF8KdtHrRJCQ +SItEJDDozYr+/5BIi0wkKIuRCAEAAI1a/4mZCAEAAIP6AXUSQYC+sQAAAAB0CEnHRhDe+v//McBI +i2wkOEiDxEDDSItcJFBmiXMISIsySIkzSIkaSIsVNDEMAIQCkJBIi0EYSLsAAAAAAIAAAEgBw0jB +6xoPHwBIgfsAAEAAD4OJAAAASIsU2oQCSMHoDUiJwUgl/x8AAEjB6ANIPQAEAABzYEiNFAJIjZIA +CCEASIPhB7sBAAAA0+PwCBqQkEiLRCQw6A+K/v+QSItUJCiLmggBAACNc/+JsggBAACD+wF1EkGA +vrEAAAAAdAhJx0YQ3vr//7gBAAAASItsJDhIg8RAw7kABAAA6KojAwBIidi5AABAAGaQ6JsjAwBI +jQVePAQAux0AAADo6rEAAJBIiUQkCEiJXCQQ6PobAwBIi0QkCEiLXCQQ6ev9///MzMzMzMzMzMzM +zEk7ZhAPhoAAAABIg+woSIlsJCBIjWwkIEiJXCQ4SIlEJBiQSI0FJpgMAOhxh/7/SI0FipcMAOiF +HP//SIlEJBCQkEiNBQeYDADoMon+/0iLXCQQxkMKAkiLTCQ4SIlLEEiLRCQY6Hb9//+EwHQKSIts +JCBIg8Qow0iNBVZMBAC7JQAAAOg3sQAAkEiJRCQISIlcJBDoRxsDAEiLRCQISItcJBDpWP///8zM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhgoBAABIg+wwSIlsJChIjWwkKEiJRCQ4D7ZQCoD6 +AXRvgPoCdBKA+gMPhc4AAADGQBAB6bsAAABIi1AQSInLSInQ6FRJAACQkEiNBTuXDADohob+/5BI +iw3OlgwASCsNl5YMAEiJDcCWDABIiw2hlgwASItUJDhIiQpIiRWSlgwAkEiNBQKXDADoLYj+/+tj +SItQEEiLSBhIi3ggSItwKEiJ2EiJ0+hQGP//kJBIjQXXlgwA6CKG/v+QSIsVIpYMAEgrFeuVDABI +iRUUlgwASIsV9ZUMAEyLRCQ4SYkQTIkF5pUMAJCQSI0FnZYMAOjIh/7/SItsJChIg8Qww0iNBbUe +BAC7EAAAAOjtrwAAkEiJRCQISIlcJBBIiUwkGOj4GQMASItEJAhIi1wkEEiLTCQY6cT+///MzMzM +STtmEA+G/gIAAEiD7EBIiWwkOEiNbCQ4SIsVUZkMAEiNWD9IwesGSIXSdBZIizJIjTTeSIH+8P8A +AEAPl8ZmkOsFvgEAAABIid9IweMDQIT2dAdIid4xyes6SIne8EgPwRpIjQT7SD3w/wAAdgQxyesi +SCnwZg8fhAAAAAAAZpBIPfD/AAAPg3ECAABIjQwCSI1JEEiFyQ+FHgIAAEiJdCQwSIl8JCiQkEiN +BbCYDADo64T+/0iLDbSYDAAPH0AASIXJdBhIixFIi1wkKEiNFNpIgfrw/wAAD5fC6wpIi1wkKLoB +AAAAhNJ0CUiLdCQwMcnrQEiLVCQwSInW8EgPwRFIjQTaSD3w/wAAdgQxyesjSCnwZg8fhAAAAAAA +Dx8ASD3w/wAAD4PHAQAASI0MCEiNSRBIhckPhVwBAAAPHwDo2wIAAEiLDSSYDABIhcl0GEiLEUiL +dCQoSI0U8kiB+vD/AAAPl8LrCkiLdCQougEAAACE0nQLSIt8JDAx0maQ6z5Ii1QkMEiJ1/BID8ER +SI0U8kiB+vD/AAB2BjHSZpDrHkgp+kiB+vD/AAAPgzQBAABIjRQKSI1SEGYPH0QAAEiF0g+FnAAA +AEiFwHQUSIsQSI0U8kiB+vD/AAAPl8KQ6wW6AQAAAITSdAQxyes2SIn68EgPwThIjTT3Dx9EAABI +gf7w/wAAdgQxyesYSCnWSIH+8P8AAA+DvAAAAEiNDDBIjUkQSIXJD4SaAAAASIlMJBhIiw02lwwA +SIlICEiNDSuXDABIhwGQkEiNBQ+XDADoKoX+/0iLRCQYSItsJDhIg8RAw0iJVCQQSIsN95YMAEiJ +SAhIiQXslgwAkJBIjQXblgwA6PaE/v9Ii0QkEEiLbCQ4SIPEQMNIiUwkIJCQSI0FuZYMAOjUhP7/ +SItEJCBIi2wkOEiDxEDDSInISItsJDhIg8RAw0iNBU8eBAC7EQAAAOjnrAAASInwufD/AADoeh4D +AEiJ0Lnw/wAA6G0eAwC58P8AAOhjHgMAufD/AADoWR4DAJBIiUQkCOjOFgMASItEJAjp5Pz//8zM +zMxJO2YQdh1Ig+wQSIlsJAhIjWwkCOjH/P//SItsJAhIg8QQw0iJRCQI6JMWAwBIi0QkCOvMzMzM +zMzMzMzMzMzMSTtmEA+GmgAAAEiD7BBIiWwkCEiNbCQIkEiNBeCVDADoG4L+/0iLDfSVDABIhcl0 +E0iLFdCVDABIhdJ1SkiJDcSVDABIiw3NlQwASIkNzpUMAEiLDbeVDABIiQ24lQwAMclIjRWnlQwA +SIcKkJBIjQWLlQwA6KaD/v9Ii2wkCEiDxBDDSInZSItZCEiF23X0SIlRCEiLDYWVDABIiQ1mlQwA +66Do1xUDAOlS////zMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GowAAAEiD7ChIiWwkIEiNbCQgSIsF +KZUMAJBIhcB0IUiJRCQQSItICEiJDROVDAC7AAABAOhBIwMASItEJBDrPJCQSI0F8ZQMAOgMg/7/ +uAAAAQBIjR04qgwA6FsO//9IhcB0MUiJRCQYkJBIjQXIlAwA6AOB/v9Ii0QkGEjHQAgAAAAASMcA +AAAAAEiLbCQgSIPEKMNIjQUjOwQAux8AAADo9KoAAJDoDhUDAOlJ////zMzMzMzMzMzMSTtmEA+G +PAEAAEiD7CBIiWwkGEiNbCQYSIsVIXQJAJBIg/oVD4eZAAAASIlcJDCEAIM9yJMMAAB1CUiJiBAB +AQDrDEiNuBABAQDocRgDAEiJRCQoSI2QkAABAEiJy0iJ0Oi6TQAASItEJCjoMCIAAEiLDfFyCQBI +i1QkKEiJingAAQCDPXaTDAAAdQ5Ii0QkMEiJgggBAQDrEUiNuggBAQBIi0QkMOgVFwMASIsFtnIJ +AEiJgvgAAQBIi2wkGEiDxCDDSIlUJBDoc8IAAEiNBc07BAC7IAAAAOiCywAASItMJBBIg/lASBnA +ugEAAABI0+JIIdDoZskAAOjBxAAAkOi7wgAA6DbCAABIjQWVNQQAux0AAADoRcsAALgAACAA6DvJ +AADolsQAAOiRwgAASI0FL0sEALsrAAAADx9EAADom6kAAJBIiUQkCEiJXCQQSIlMJBjophMDAEiL +RCQISItcJBBIi0wkGOmS/v//zMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GHQIAAEiD7GhIiWwkYEiN +bCRgSImMJIAAAABIiUQkcIQAkJBIjRQZSI2S//8/AEiB4gAAwP9IiVQkKEiB4wAAwP9IiVwkIEiJ +0ehMIgAASItUJHBIi7KAAAEAkJBIi0QkIEi/AAAAAACAAABMjQQHTYnBScHoFg8fQABIhfZ0BUw5 +xnYHTImCgAABAEyJTCRQTIlEJEhIi1wkKEiNDB9IwekWSIlMJEBIOYqIAAEAcwpIiYqIAAEADx8A +6NtJAABIi0wkcEiNkZAAAQBIicZIidBIid9IifNIifnom04AAEiLRCRwSIuIeAABAEi6AAAAAACA +AABIAdFIi1QkUEg50XYOSItcJCBIiZh4AAEA6wVIi1wkIEiLVCRATItEJEjrREqLVMB4hAJIgeb/ +HwAASMHmB0iNFDJIjVJAMdu5AAIAAEiJ0OhQLQAASItUJDhMjUIBSItUJEBIi3QkcEiJ8EiLXCQg +STnQc3ZMicZJwegNZg8fhAAAAAAASYH4ACAAAA+DjAAAAEiJdCQ4Sot8wHhOjQzATY1JeJBIhf91 +gkyJRCQwTIlMJFhIi5gQAQEAuAAAEAAPH0QAAOi7Cv//SIXAdD5Ii1QkWEiHAkiLRCRwSIt0JDhM +i0QkMOlB////SIuMJIAAAABIwekNvwEAAAAx9uhiAAAASItsJGBIg8Row0iNBYMnBAC7GAAAAOhH +pwAATInAuQAgAADo2hgDAJBIiUQkCEiJXCQQSIlMJBjoRREDAEiLRCQISItcJBBIi0wkGOmx/f// +zMzMzMzMzMzMzMzMzMzMzMxMjWQkyE07ZhAPhoQFAABIgey4AAAASImsJLAAAABIjawksAAAAEiJ +hCTAAAAAhABIweENTI0EGZCQSbkAAAAAAIAAAEkB2UyJjCSYAAAATYnKScHpFkyJjCSQAAAASbv/ +/////38AAE0B2EyJhCSIAAAATYnDScHoFmYPH0QAAE05yA+ENgEAAEyJhCSAAAAAQIT/dRpIi3Bo +SIl0JDBMi2BgTImkJKgAAADpdQQAAEiLUGhIi1hgScHqIw8fQABJgfoAIAAAD4MYBAAAQIi0JNkA +AABIiVQkOEiJnCSgAAAASotM0HiEAUmB4f8fAABJweEHTAHJSInI6AEwAABIi0wkOEiLlCSQAAAA +SDnRD4bDAwAASIu0JKAAAABIiQTWSIu8JMAAAABMi0dwTItPYEyLlCSAAAAADx+AAAAAAE050A+C +hAMAAEiNQgFJOcIPgm8DAABJKdBJ/8hJ99hIweADScH4P0whwEwByE2J0Ekp0kmNWv8PtpQk2QAA +AJCE0nUHMdLpKwMAAEiF2w+EmQIAAEjB4wPoQx0DAEiLTCQ4SIu0JKAAAABIi7wkwAAAAEyLhCSA +AAAA6W4CAABIi0hoSItQYEw5yQ+GVQIAAEqLFMpJweojDx+EAAAAAABJgfoAIAAAD4MrAgAASIlU +JChKi0zQeIQBSYHh/x8AAEnB4QdMAclIicjo8S4AAEiLTCQoSDnIdClIi7QkwAAAAEiLTmhMi0Zg +TIuMJJAAAABMOckPhtYBAABLiQTIZpDrEEiLrCSwAAAASIHEuAAAAMNMi4QkmAAAAEyLjCSIAAAA +uAMAAAC5AQAAAOsJSP/ISYnQSYnZSIXAfGAPH0AAhMl0WEiJRCRQTI0VEG4JAEmLDMJIg/lATRnb +TI0lfm0JAE2LbMQITIlsJEhMjT2tbQkASYt8xwhIiXwkQJBMicJJ0+hNIdhMictJ0+lNIdlJ/8FM +iUwkYDHJ61NIi6wksAAAAEiBxLgAAADDTItcJFBMi0wkYEyLbCRISIt8JEBMi0QkeEiLlCSYAAAA +SIucJIgAAABMjRWEbQkATI0l/WwJAEyNPTZtCQCJwUyJ2E05yA+NNv///0yJRCRYiEwkJ0mD/UBN +GdtMjTxATotk/ihOi1T+GEyJ6UnT4E0h2EyLTCRYSf/BTIlMJHhJ0+FNIdlNOcwPgooAAAAPH0QA +AE05wXJyTIl8JHBNKcRNKcFMieFJ99xJweADScH8P00h4EuNBAJMicvosxkAAEiLVCRwSIu0JMAA +AABIi0zWCEiLFNZMi0QkWEk5yHMiTosMwkk5wXQPSokEwrgBAAAAkOkH////D7ZEJCfp/f7//0yJ +wOiJFAMATInATInJDx8A6DsVAwBMiclMieLo8BQDAEyJyOiIFAMATInQuQAgAADoexQDAEyJyOhz +FAMASIuEJIgAAABIwegjDx+AAAAAAEg9ACAAAHNQSItMx3iEAUmB4P8fAABJweAHSo0EAWaQ6Jss +AABIi0wkOEiLlCSAAAAASDnRdhlIi7QkoAAAAEiJBNZIi7QkwAAAAOnC/f//SInQ6AgUAwC5ACAA +AA8fAOj7EwMASbkAAgBAAAAIAEyJDNBI/8JIOdp86g8fRAAA6Wj///9MidHokxQDAEyJ0UyJwuhI +FAMASInQDx9EAADouxMDAEyJ0LkAIAAA6K4TAwBMi6QkqAAAAEmJBMxMjUkBTIuEJIAAAABIi4Qk +wAAAAEiJ1kyLlCSYAAAATIucJIgAAABNOcEPh1X///9MicpJwekNSYH5ACAAAHNHSIlUJGhKi0zI +eIQBSIHi/x8AAEjB4gdIAdFIicgPH0QAAOibKwAASItMJGhIi1QkMEg50Q+Cev///0iJyEiJ0WaQ +6BsTAwBMici5ACAAAOgOEwMAkEiJRCQISIlcJBBIiUwkGECIfCQgQIh0JCHobwsDAEiLRCQISItc +JBBIi0wkGA+2fCQgD7Z0JCHpMfr//8zMzMzMzMzMzMzMzMzMzMzMSTtmEA+GNQMAAEiDxIBIiWwk +eEiNbCR4SImMJJgAAABIiZwkkAAAAEiJhCSIAAAAhABIweENSI00C0iNPAtIjX//kJCQkEm4AAAA +AACAAABJAdhNicFJwegWSbr//////38AAEwB1kmJ8kjB7hZIgeP//z8ASMHrDUiJXCRoSIHn//8/ +AEjB7w1MOcYPhLMAAACQScHpI2YPH4QAAAAAAEmB+QAgAAAPg4QCAABMiUQkYEyJVCRYSIl8JFBI +iXQkSEqLVMh4SIlUJHCEAkmB4P8fAABJweAHTIlEJEBKjTwCSI1/QEiNiwD+//9I99lIiUwkOEiJ ++OjkJwAASIlEJDBIi1QkQEiLdCRwSAHySItcJGhIi0wkOEiJ0A8fRAAA6NsuAABIi1QkYEj/wkiL +tCSIAAAASIt8JEhMi0QkMJDpMgEAAJBJwekjSYH5ACAAAA+DoQAAAEqLVMh4SIlUJHCEAkmB4P8f +AABJweAHTIlEJEBKjTQCSI12QEgp30iNTwFIiUwkOEiJ8OhRJwAASIlEJDBIi1QkQEiLdCRwSAHy +SItcJGhIi0wkOEiJ0OhNLgAASItEJDBIiUQkIEiLnCSQAAAASIuMJJgAAAC/AQAAAIn+SIuEJIgA +AAAPH0AA6Dv4//9Ii0QkIEjB4A1Ii2wkeEiD7IDDTInIuQAgAADouxADAEiJVCQoSIt8xnhIiXwk +cIQHSIHi/x8AAEjB4gdIiVQkQEiNBBdIjUBAMdu5AAIAAOimJgAASIlEJDhIi1QkQEiLdCRwSAHy +SInQ6CwuAABIi1QkKEj/wkiLdCQ4SIt8JCBMjQQ3SIu0JIgAAABIi3wkSEyJRCQgDx9AAEg5+nMZ +kEiJ0EjB6A1IPQAgAAAPgmz////piQAAAJBIi0QkWEjB6CNIPQAgAABzbUiLVMZ4SIlUJHCEAkiB +5/8fAABIwecHSIl8JEBIjQQ6SI1AQEyLRCRQSY1IAUiJTCQ4Mdvo9yUAAEiJRCQwSItUJEBIi3Qk +cEgB8jHbSItMJDhIidDo9iwAAEiLVCQwSIt0JCBIjQQW6Zv+//+5ACAAAOiZDwMAuQAgAADojw8D +AEyJyLkAIAAA6IIPAwCQSIlEJAhIiVwkEEiJTCQY6O0HAwBIi0QkCEiLXCQQSItMJBjpmfz//8zM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4aPAAAASIPsGEiJbCQQSI1sJBCEAJCAuBgBAQAA +dT1IixXtGwwASIXSdDFIvgAAAAAAgAAASAHeSMHuGkiB/gAAQABzQUiLDPJIhcl0DkiJ2EiLbCQQ +SIPEGJDDSAWQAAEA6DRCAACE23QKSItsJBBIg8QYw0iLBZ9lCQBIi2wkEEiDxBjDSInwuQAAQADo +qA4DAJBIiUQkCEiJXCQQ6BgHAwBIi0QkCEiLXCQQ6Un////MzMzMzMzMzMxMjaQkaP///007ZhAP +hrQLAABIgewYAQAASImsJBABAABIjawkEAEAAEiJhCQgAQAASImcJCgBAACEAEQPEbwk6AAAAEiL +NTVlCQBIibQk6AAAAEiLNQZlCQBIibQk8AAAAEQPEbwkAAEAAEiNNY4LAABIibQkAAEAAEiNtCTo +AAAASIm0JAgBAAAxyTHSSMfG/////zH/6ydMi5Qk4AAAAEmNSgFIi4QkIAEAAEiLtCSgAAAASIny +Dx+EAAAAAABIg/kFD40OAQAATI0FL2UJAE2LDMhJg/lATRnSSYnLTInJQbwBAAAASdPkTSHUTI0t +S2UJAE+LfN0ASNPiTCHST40MW06LVMgQTosMyEqNDCJJOcoPgp4KAABmDx9EAABIOcoPh4cKAABI +iZQkqAAAAEiJdCRwSIl8JHhIjTU+ZQkAToss3kmD/UBNGcBMKdJMi5QkqAAAAEnB4gNIwfo/SSHS +TQHRTIuQeAABAEyJ4kn33Ei/AAAAAACAAABJAfpMielJ0+pNIcJNIdRMi4QkqAAAAE05xHUJTI1i +/00h1OsDRTHkTImcJOAAAABIiZQk2AAAAEyJfCRoTImMJPgAAABMiaQkgAAAAEyJ4UUx0kUx7emo +AgAASInWSMHqDZBIgfoAIAAAD4NtAgAASIm0JOAAAABIi1TQeIQCSIHm/x8AAEjB5gdIAfIxyUiJ +0OisJgAASIP4/w+EowAAAEiJhCSYAAAASMHjDUiLjCTgAAAASInOSMHhFkiJjCTQAAAASI08C0m4 +AAAAAACA//9KjQQHTIuMJAABAABIjV4BSMHjFkgp+0iNlCQAAQAAQf/RSIucJOgAAABIi4QkIAEA +AOi6/P//SIuMJJgAAABIweENSIu0JNAAAABIAfFIvgAAAAAAgP//SAHxSInDSInISIusJBABAABI +gcQYAQAAkMNIi5QkIAEAAEiLSmhIi1JgSIuEJOAAAAAPH4AAAAAASDnID4NsAQAASIsMwkiJykiB +4f//HwBIuwAAAAAAAACASIXTuwAAIABID0XLSA+64j8PgyUBAAC7AAAgAEiJXCQYSIlMJEgPgwAB +AAC6AAAgAEiJVCRA6ASyAABIjQVhCwQAuxEAAADoE7sAALgEAAAA6Am5AABIjQWx9gMAuwIAAADo ++LoAAEiLhCTgAAAA6Ou4AABIjQXw9wMAuwUAAADo2roAAEiLRCRI6NC3AABIjQVq9gMAuwIAAAAP +H0AA6Lu6AABIi0QkGOixtwAASI0FS/YDALsCAAAADx9EAADom7oAAEiLRCRA6JG3AABIjQUp9gMA +uwIAAAAPH0QAAOh7ugAA6NaxAADoUbEAAEiNBUEMBAC7EgAAAA8fRAAA6Fu6AABIi4QkKAEAAOhO +twAA6KmzAADopLEAAEiNBYsHBAC7EAAAAOizmAAASMHqKkiB4v//HwDp9f7//0iJ00jB6xVIgeP/ +/x8ASA+64j/pyP7//+gGCgMASInQuQAgAADoGQoDAEn/xEmJ+ki/AAAAAACAAABmDx+EAAAAAABJ +OdQPjWcCAAAPgxYHAABLizzhSIX/dQQx/+vLTImkJIgAAABIibwkyAAAAEyJrCSwAAAATIlUJGBJ +g/9ASBnbSos83kiD/0BNGclMi5QkAAEAAE+NLARMiawkoAAAAEyJ+UG4AQAAAEnT4Ewhw0jB4w1I +iflJ0+VNIc1IvwAAAAAAgP//To0ML0iNlCQAAQAATInIQf/SSIu0JMgAAABIifdIgeb//x8ASbgA +AAAAAAAAgEyFx0G5AAAgAEkPRfFMi1QkYE2NHDJIi5wkKAEAAEw52w+GMwEAAEgPuuc/kHMHuAAA +IADrF0mJ+0jB7xVIgef//x8ASIn4TInfDx8ASDnDD4YQ+///TYXSdQxID7rnP0iLTCRo6yZIi0wk +aEiD+UBNGdtBvAEAAABJ0+RNIdxMOeYPg30AAABID7rnP3MLSIP5QL8AACAA6w9Iwe8qSIHn//8f +AEiD+UBNGe1Ii7QkiAAAAEyNVgFJ0+JNIdVJKf1Ii4QkIAEAAEiLjCSAAAAASIuUJNgAAABMi4Qk +qAAAAEyLjCT4AAAATIucJOAAAABJifRMi3wkaEiNNU1gCQDpL/7//0uNPCJIi4QkIAEAAEiLjCSA +AAAASIuUJNgAAABIjTUlYAkATIuEJKgAAABMi4wk+AAAAEyLnCTgAAAATIukJIgAAABMi3wkaEyL +rCSwAAAA6dr9//9Ii0wkaEiD+UBIGdJIi7QkiAAAAEjT5kgh1k2F0kyLrCSwAAAATA9E7kiLhCQg +AQAASIuMJIAAAABIi5Qk2AAAAEiNNatfCQBMi4QkqAAAAEyLjCT4AAAATYnaTIucJOAAAABMOdMP +hrYCAAAPH4AAAAAATYXbD4SNAgAASIt8JHhJifpIgef//x8ASbwAAAAAAAAAgE2F1EG9AAAgAEkP +Rf1JD7riP3MIQb8AACAA6yNNiddJweoVSYHi//8fAEkPuuc/TInWTYn6SYn3SI01H18JAEyJfCQw +SIl8JChzCEG6AAAgAOsLScHqKkmB4v//HwBMiVQkOOi2rQAASI0FEwcEALsRAAAA6MW2AABIi4Qk +4AAAAEiNSP9IicjosbQAAEiNBVnyAwC7AgAAAA8fRAAA6Ju2AABIi0QkcOiRtAAASI0F0vIDALsE +AAAADx9EAADoe7YAAEiLRCQo6HGzAABIjQUL8gMAuwIAAAAPH0QAAOhbtgAASItEJDDoUbMAAEiN +BevxAwC7AgAAAA8fRAAA6Du2AABIi0QkOOgxswAA6IyvAADoh60AAOgCrQAASI0FLAYEALsRAAAA +6BG2AABIi4Qk4AAAAOgEtAAASI0FS/sDALsLAAAA6PO1AABIi4QkKAEAAOjmsgAASI0FPvQDALsH +AAAA6NW1AABIi4QkgAAAAOjIswAA6COvAAAPHwDoG60AAEiLhCQgAQAASIuAeAABAEiJhCTQAAAA +Dx9AAOh7rAAASI0F1hQEALsYAAAA6Iq1AABIi4Qk0AAAAGaQ6NuzAABIjQVK8wMAuwYAAADoarUA +AEiLhCSoAAAAZpDoW7MAAOi2rgAA6LGsAABIi4Qk4AAAAEiNDWJdCQBIiwzBSImMJMAAAABIjRXP +XAkASIsUwkiJlCS4AAAADx8A6PurAABIjQWsHgQAux0AAADoCrUAAEiLhCTAAAAAZpDo+7EAAEiN +BeALBAC7FQAAAOjqtAAASIuEJLgAAABmkOjbsQAA6DauAADoMawAADHA6V8BAABIix3LWwkAMcBI +i6wkEAEAAEiBxBgBAADDTImsJLAAAABIi5wk6AAAAEqLDN5IiYwkwAAAAOhw9f//SIuMJMAAAABI +g/lASBnSSIu0JLAAAABIweYNSIu8JKgAAABI0+dIIddIjQw3SLoAAAAAAID//0gB0UiJw0iJyEiL +rCQQAQAASIHEGAEAAJDDSIl0JFjoFasAAEiNBXIEBAC7EQAAAOgktAAASIuEJOAAAADoF7IAAEiN +Bb/vAwC7AgAAAOgGtAAASIuEJKgAAABIi4wkkAAAAEiNFAFIidDo6rEAAEiNBe/wAwC7BQAAAOjZ +swAASItEJCDoz7AAAEiNBWnvAwC7AgAAAA8fAOi7swAASItEJFDosbAAAEiNBUvvAwC7AgAAAA8f +RAAA6JuzAABIi0QkWOiRsAAASI0FKe8DALsCAAAADx9EAADoe7MAAOjWqgAASIuEJJAAAABI/8BI +i4wk2AAAAA8fAEg5yA+NhgAAAEiLlCT4AAAASIscwkiJ3kiB4///HwBIvwAAAAAAAACASIX3QbgA +ACAASQ9F2EgPuuY/cwhBuQAAIADrHEmJ8UjB7hVIgeb//x8ASQ+64T9JifJMic5NidFIiYQkkAAA +AEyJTCRQSIlcJCBzCr4AACAA6aL+//9Iwe4qSIHm//8fAOmS/v//SI0FGAAEALsQAAAADx9EAADo +O5EAAEyJ4EiJ0eiwAgMASInQ6GgDAwBMidIPH0QAAOgbAwMAkEiJRCQISIlcJBDoK/sCAEiLRCQI +SItcJBCQ6Rv0///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4ZcAQAASIPsMEiJbCQo +SI1sJChIi0oISIsRSL4AAAAAAIAAAEgB8kiNPAZIOddyLkyLQQhMjQwDSbr//////38AAE0B0UkB +8E05yHIbSIkBSI0EA0iNQP9IiUEI6yJJuv//////fwAATI0EA00B0Ek50HIMSItRCEgB8kg5+nMK +SItsJChIg8Qww0iJTCQgSIlEJBhIiVwkQGaQ6JuoAABIjQUCAAQAuxAAAADoqrEAAEiLRCQYDx9E +AADo+68AAEiNBazyAwC7CQAAAOiKsQAASItEJEAPH0QAAOh7rgAA6NaqAADo0agAAEiLRCQgSIsI +SIlMJBhIi0AISIlEJBDoNqgAAEiNBa3/AwC7EAAAAOhFsQAASItEJBjom68AAEiNBfb0AwC7CgAA +AOgqsQAASItEJBAPH0QAAOh7rwAA6HaqAADocagAAEiNBewPBAC7GAAAAA8fRAAA6HuPAACQSIlE +JAhIiVwkEOjr+AIASItEJAhIi1wkEJDpe/7//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7 +ZhgPhpQCAABIg+w4SIlsJDBIjWwkMIQASIuIeAABAEi6AAAAAACAAABIjTQRSIn3SMHuFmYPH4QA +AAAAAJBIObCIAAEAD4ZmAQAASIlcJEhIiUQkQEiB4f//PwBIwekNTI2BAP7//0n32Ew5ww+HpAAA +AEyLQGhMi0hgDx8ASTnwD4YJAgAATYsE8UkPuuA/cwhBuAAAIADrC0nB6BVJgeD//x8ATDnDd21I +we8jSIH/ACAAAA+DyAEAAEiJdCQoTIlEJBhIi1T4eIQCSIHm/x8AAEjB5gdIAfJIidCQ6DsaAABI +g/j/D4TTAAAASMHgDUiLVCQoSMHiFkiNNBBIvwAAAAAAgP//SAH+SMHjDUgB2kgB+us/6ELx//9m +kEiFwHUtSItMJEhIg/kBdRNIiw2RVgkASItUJEBIiYp4AAEAMcBIicNIi2wkMEiDxDjDSInGSIna +SIl0JChIiVQkIEiLRCRASInzSItMJEjorOz//0iLVCRASIuyeAABAEi/AAAAAACAAABIAf5Mi0Qk +IEwBx0g593YHTImCeAABAEiJw0iLRCQoSItsJDBIg8Q4wzHASInDSItsJDBIg8Q4w+jZpQAASI0F +JvsDALsPAAAA6OiuAABIi0QkGA8fAOjbqwAASI0FIvQDALsLAAAA6MquAABIi0QkSA8fRAAA6Lur +AADoFqgAAOgRpgAASItEJEBIi4B4AAEASIlEJCjoe6UAAEiNBZEGBAC7FQAAAOiKrgAASItEJChI +Jf//PwBIwegN6HarAABIjQVf/QMAuxEAAADoZa4AAEiLRCQo6LusAADotqcAAOixpQAASI0FmPsD +ALsQAAAADx9EAADou4wAAEiJ+LkAIAAA6E7+AgBIifBMicHoQ/4CAJBIiUQkCEiJXCQQ6PMdAwBI +i0QkCEiLXCQQ6UT9///MzMzMSTtmGA+GqwIAAEiD7FhIiWwkUEiNbCRQhABIi5B4AAEASbgAAAAA +AIAAAE2NDBhMAcJMOcp2B0iJmHgAAQBIicpIweENTI0UGUyNHBlNjVv/TIugAAEBAE0B4Em8//// +//9/AABNAeJNOcJ2B0yJmAABAQBIg/oBD4TbAAAASIlUJEhIiVwkaEiJRCRgkJCQkEyJzknB6RZM +iddJweoWSIHj//8/AEjB6w1JgeP//z8AScHrDU05ynRlSMHuI2aQSIH+ACAAAA+D2gEAAEiJfCRA +TIlUJDhMiUwkMEyJXCQoSItU8HiEAkmB4f8fAABJweEHTAHKSI2LAP7//0j32UiJ0Oj4EQAASItU +JDBI/8JIi3QkYEiLfCQ46SEBAABIwe4jSIH+ACAAAA+DygAAAEiLVPB4hAJJgeH/HwAAScHhB0wB +ykkp20mNSwFIidDorBEAAOmQAAAAkE2JyEnB6SNJgfkAIAAAc3JOi0zIeEGEAUnB6BZJgeD/HwAA +ScHgB00ByEiJ2UiB4f//PwCQSYnJSMHpE0iD+QhzM0nB6Q1Iic5MiclBugEAAABJ0+JJ99JNIRTw +SInRvwEAAAAx9ui74///SItsJFBIg8RYw0iJyLkIAAAA6ET8AgBMici5ACAAAOg3/AIASItEJGBI +i1QkSEiLXCRo67xIifC5ACAAAOgZ/AIASIlUJCBIi0zGeIQBSIHi/x8AAEjB4gdIjQQR6PkRAABI +i0wkIEiNUQFIi0wkYEiLXCQ4SInOSInfSDn6cxFIidBIwegNSD0AIAAAcrPrR0iLRCRASMHoI5BI +PQAgAABzK0iLVMZ4hAJIgef/HwAASMHnB0iNBDpIi1QkKEiNSgEx2+hyEAAA6Vb///+5ACAAAOiD ++wIAuQAgAADoefsCAEiJ8LkAIAAA6Gz7AgCQSIlEJAhIiVwkEEiJTCQY6BcbAwBIi0QkCEiLXCQQ +SItMJBjpI/3//8zMzEiD7CBIiWwkGEiNbCQYSIlEJChIhdsPhjgBAABIixBID7riP3MSugAAIAC+ +AAAgAEG4AAAgAOsjSInWSIHi//8fAEmJ8EjB7hVIgeb//x8AScHoKkmB4P//HwC5AQAAAOteSIlM +JBBIg/9ATRnkSYnNSIn5SdPlTSHlTo08Ckw56kkPRNdBvQEAAABJ0+VNIeVPjWQFAE05600PRNxN +AcFMOc5JD0LxSTnySQ9H8kyLTCQQSf/BTInJTYnYDx9AAEg5y35KTIsMyEkPuuE/cxdBuQAAIABB +ugAAIABBuwAAIADpe////02JykmB4f//HwBNidNJweoVSYHi//8fAEnB6ypJgeP//x8A6VP///9I +gf4AACAAdQxJuAAAAAAAAACA6yNIgeL//x8ASIHm//8fAEjB5hVICdZJgeD//x8AScHgKkkJ8EyJ +wEiLbCQYSIPEIMMxwEiJ2ejC+QIAkMxJO2YQD4ZKAQAASIPseEiJbCRwSI1sJHBIiYQkgAAAAEiL +FblRCQBIiVQkSA8QBbVRCQAPEUQkUA8QBblRCQAPEUQkYDHJ6wdJjUsBTInQSIP5BQ+N3wAAAEiJ +TCQgSItUzEhIg8LQSPfaSIP6QE0ZyUiJ0UG6AQAAAEnT4k0hykyJVCQoSIsdXHAMAEqNFNNIjVL/ +SPfbSCHTMcm/IgAAAL7/////RTHAMcDokPj9/0iF27oAAAAASA9Fwg8fQABIhcB0fUQPEXwkMEjH +RCRAAAAAAEiJRCQwSMdEJDgAAAAATItMJChMiUwkQEyLlCSAAAAAQYQCTItcJCBPjSRbTItsJDBL +x0TiCAAAAABPiUziEEuNPOKDPUJwDAAAZpB1CU+JLOLpHf///02J6eiN9QIA6RD///9Ii2wkcEiD +xHjDSI0FeCEEALslAAAA6O2GAACQSIlEJAjoAvECAEiLRCQI6Zj+///MzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMxMjWQk4E07ZhAPhsADAABIgeygAAAASImsJJgAAABIjawkmAAAAEiJjCS4AAAASImc +JLAAAABmDx+EAAAAAACQSPfD//8/AA+FJQMAAEj3wf//PwAPhRgDAABIiYQkqAAAAEQPEXwkcEiN +DREEAABIiUwkcEiJRCR4RA8RvCSAAAAASMeEJJAAAAAAAAAASI0NawMAAEiJjCSAAAAASI0NhDYE +AEiJjCSIAAAASI1UJHBIiZQkkAAAAIQASI2QkAABAEiJ0Oj2KQAASIlEJEgxyesNSP/BZg8fhAAA +AAAAkEiD+QUPjVACAABIiUwkQEiLhCSwAAAASIucJLgAAACQ6PsmAABIiw0cNgQASI0VFTYEAEiJ +xkiLRCRASInfSInzSYnISIn5Qf/QSItMJEBIjRRJSIu0JKgAAABIi3zWEEyLRNYISTnYfQ5IOfsP +hxQCAABIiVzWCJBIweADSIs9BG4MAEyLRNYITIsM1kmJ+kj330gh+E2NFNpNjVL/SSH6TYXAD4bT +AQAASo08CE+NBApIi0QkSEiFwH53TIuOkAABAEyLlpgAAQBMjVj/TTnaD4abAQAASIl8JChMiUQk +IEiLtCSAAAAAScHjBEuLHAtLi3wLCEiJyEiJ+UiNlCSAAAAA/9ZIicFIid9Ii0QkKEiLXCQg6IUm +AABIi0wkQEiLtCSoAAAASYnYSInHSItEJEhMi46QAAEATIuWmAABAEw50H1iD4MeAQAATIlEJDBI +iXwkOEiLtCSAAAAASMHgBEqLHAhOi0QICEiJyEyJwUiNlCSAAAAA/9ZIicFIid9Ii0QkOEiLXCQw +6BAmAABIi0wkQEiLtCSoAAAASInHSYnYSItEJEhIugAAAAAAgAAATI0MF02NFBBNOcp2CZBNicNJ +KfjrBk2Jw0UxwE2FwA+EHP7//0yJTCRoTIlUJGBMiVwkWE05ynYGkEkp++sDRTHbSIl8JFBIifhI +i44QAQEATInb6BXr/v9Ii1QkYEiLdCRoSDnydhCQSItMJFhIi0QkUEgpwesHSItEJFAxyUiJy+gH +6v7/SItEJEhIi0wkQEi6AAAAAACAAABIi7QkqAAAAOmZ/f//SIusJJgAAABIgcSgAAAAw0yJ0eju +9AIATInYTInR6OP0AgAxwEyJwejZ9AIASInZSIn66E71AgDoqZsAAEiNBSDzAwC7EAAAAOi4pAAA +SIuEJLAAAADoC6MAAEiNBXDoAwC7CgAAAOiapAAASIuEJLgAAADo7aIAAOjonQAA6OObAABIjQXg +KAQAuy4AAADo8oIAAJBIiUQkCEiJXCQQSIlMJBhmkOj77AIASItEJAhIi1wkEEiLTCQY6Qf8///M +zMzMzMzMSTtmEHZGSIPsMEiJbCQoSI1sJChIiUQkGEiLchBIiXQkIEiLUghIizr/10iLVCQgSIsy +SInZSInDSItEJBhmkP/WSItsJChIg8Qww0iJRCQISIlcJBBIiUwkGA8fRAAA6NvrAgBIi0QkCEiL +XCQQSItMJBjriszMzMzMzMzMzMxIg+wYSIlsJBBIjWwkEEiLUgiEAkiLNc1qDABIweMDkEiJ90j3 +3kgh80iNPM9IjX//SCH3SIP4BXMuSI00QEiLTPIISIsU8kiFyXYVSI0EGpBIjRwXkEiLbCQQSIPE +GJDDMcDoWPMCALkFAAAA6E7zAgCQzMzMzMzMzMzMzMzMzEk7ZhAPho0AAABIg+wYSIlsJBBIjWwk +EEiLUAgPH0AASIXSdGNIg/sBdU5ID7zySIP+QEgZ20iLeBBIifFBuAEAAABJ0+BJIdhJ99BMIcJI +iVAITCFAEEjT70gh+0iD4wFIweENSAMISMHjDUiJyEiLbCQQSIPEGMPoQgAAAEiLbCQQSIPEGMMx +wEiJw0iLbCQQSIPEGMNIiUQkCEiJXCQQ6DrrAgBIi0QkCEiLXCQQ6Uv////MzMzMzMzMzMzMzEk7 +ZhAPhgYBAABIg+wgSIlsJBhIjWwkGEiLUAhIjXP/uQEAAADrCUgpzkjR4UiJ+kiF9nYmSDnxcxVI +iddI0+pIIddIhf9137pAAAAA6xlIifFIidZI0+pIIfJID7zSvkAAAABID0TWSIP6QA+DjwAAAEiD ++0BIGfZIidm/AQAAAEjT50gh/kj/zkiJ0UjT5kiLeBBIIfdMD7YFdmcMAJBNhcB0B/NID7j/6y5I +iUQkKEiJTCQQSIl0JAhIifhmkOjb5P3/SItMJBBIicpIi3QkCEiJx0iLRCQoSPfWSCFwCEghcBBI +weENSAMISMHnDUiJyEiJ+0iLbCQYSIPEIJDDMcBIicNIi2wkGEiDxCDDSIlEJAhIiVwkEOgB6gIA +SItEJAhIi1wkEOnS/v//zMzMzMzMzMzMzMzMzMzMzMzMSTtmGA+G6QEAAEiD7EhIiWwkQEiNbCRA +hANIg3gIAJB0QUiJRCRQSIlcJFhIixCQSbgAAAAAAIAAAE2NDBBMiUwkOE2JyknB6RZMiUwkMEiB +4v//PwBIweoNSIlUJCgxyesdSItsJEBIg8RIw0iNTgFNieFNidpmDx+EAAAAAABIg/lAD4PuAAAA +TItYCEkPo8tzX02J00nB6iMPH0QAAEmB+gAgAAAPgzgBAABOi1TTeEGEAk2JzEmB4f8fAABJweEH +TQHRTI0UCk2J1UnB6gZJg/oID4P5AAAASInOTInpQb8BAAAASdPnSffXTyE80esJSInOTYnTTYnM +TItIEEkPo/EPg2P///9JwesjSYH7ACAAAA+DrAAAAEiJdCQgSot823iEB0mB5P8fAABJweQHSo0E +J0iNQEBIjTwyuQEAAABIifvotQMAAEiLRCRQSItUJChIi1wkWEiLdCQgSbgAAAAAAIAAAEyLXCQ4 +TItkJDDp9f7//0iLEEyLi3gAAQBNjRQQTQHITTnQdgdIiZN4AAEASIsQuUAAAAAx/4n+SInYSInT +6BTX//9Ii1QkUEQPETpIx0IQAAAAAEiLbCRASIPESMNMidi5ACAAAOiM7wIATInQuQgAAAAPH0AA +6HvvAgBMidC5ACAAAOhu7wIAkEiJRCQISIlcJBAPHwDoGw8DAEiLRCQISItcJBDp7P3//8zMzMzM +zMzMzMzMzEk7ZhgPhqsCAABIg+xgSIlsJFhIjWwkWIQASIuIeAABAEi6AAAAAACAAABIjTQRSIn3 +SMHuFmYPH4QAAAAAAJBIObCIAAEAD4YMAgAATItAaEyLSGBJOfAPhksCAABIiUQkaE2LBPFNhcAP +hb0AAAC7AQAAAOhG4P//Zg8fRAAASIXAD4SAAAAAkJBIugAAAAAAgAAASAHCSInWSMHqI5BIgfoA +IAAAD4PvAQAASIt8JGhIi1TXeIQCkEjB7hZIgeb/HwAASMHmB0yNBDJJicFIJf//PwCQSMHoEw8f +RAAASIP4CA+DqAEAAE2LBMBJ99BIjRQySI1SQEmB4QAA+P9IixTC6eoAAABIixUYRQkASIt0JGhI +iZZ4AAEAMcAx20iJ2UiLbCRYSIPEYMOQSMHvI0iB/wAgAAAPg0YBAABIiXQkQEiLVPh4SIlUJFCE +AkiB5v8fAABIweYHSIl0JDhMjQQyTIlEJEhIgeH//z8ASMHpDbsBAAAATInADx9AAOj7BwAASIP4 +/w+E5QAAAJBIicJIwegGZg8fhAAAAAAASIP4CA+DwAAAAEiD4sBIweINSIt0JEBIweYWSAHWSL8A +AAAAAID//0yNDDdIi3QkSEyLBMZJ99BIi3QkOEiLfCRQSI00N0iNdkBIixTGSIt8JGhIiVQkIEyJ +RCQoTIlMJDBIifhMicu5QAAAAOhu2v//SItEJGhIi1wkMLlAAAAAMf++AQAAAOhz1P//SItEJDBI +jZAA4AcASIt0JGhIiZZ4AAEASItcJChIi0wkIEiLbCRYSIPEYMMxwDHbSInZSItsJFhIg8Rgw7kI +AAAA6MzsAgBIjQXz6QMAuxAAAADoG3sAAEiJ+LkAIAAA6K7sAgC5CAAAAOik7AIASInQuQAgAADo +l+wCAEiJ8EyJweiM7AIAkEiJRCQI6EEMAwBIi0QkCOk3/f//zMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMxIg+wYSIlsJBBIjWwkEIQASInaSMHrBmYPH4QAAAAAAEiD+wgPg8kAAABIg/kBdGBIjTQRSI12 +/0iJ90jB7gaQSDnedCFIg/4ID4OXAAAASInRSMfC/////0jT4kgJFNhIjVMB609Ig/lASBn2vwEA +AABI0+dIIfdIjXf/SInRSNPmSAk02EiLbCQQSIPEGMOQSInRugEAAABI0+JICRTYSItsJBBIg8QY +w0jHBND/////SP/CSDnycvBIg+c/SI1PAUiD+UBIGdK7AQAAAEjT40gh00iNU/9ICRTwSItsJBBI +g8QYw0iJ8LkIAAAA6G3rAgBIidi5CAAAAA8fRAAA6FvrAgCQzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMwxyesNhABIxwTI/////0j/wUiD+Qh87cPMzMzMzMzMzEiD7BhIiWwkEEiNbCQQhABIidpI +wesGZg8fhAAAAAAASIP7CA+D0wAAAEiD+QF0ZkiNNBFIjXb/SIn3SMHuBpBIOd50JEiD/ggPg6EA +AABIidFIx8L/////SNPiSPfSSCEU2EiNUwHrV0iD+UBIGfa/AQAAAEjT50gh90iNd/9IidFI0+ZI +99ZIITTYSItsJBBIg8QYw5BIidG6AQAAAEjT4kj30kghFNhIi2wkEEiDxBjDSMcE0AAAAABI/8Jm +kEg58nLuSIPnP0iNTwFIg/lASBnSuwEAAABI0+NIIdNI99tIIRzwSItsJBBIg8QYw0iJ8LkIAAAA +6CPqAgBIidi5CAAAAOgW6gIAkMzMzMzMzMzMzMzMzMzMzMzMzMzMzDHJ6w2EAEjHBMgAAAAASP/B +SIP5CHztw8zMzMzMzMzMSTtmEA+G/gEAAEiD7DhIiWwkMEiNbCQwDx+EAAAAAABIg/kBD4TvAAAA +hABIidpIwesGSIP7CA+DvQEAAEiNNBFIjXb/SIn3SMHuBkg53nR0SIP+CA+DkgEAAEiJRCRASIl8 +JChIiXQkIEyLBNhIidFJ0+hID7YV2V4MAA8fQABIhdJ0B/NND7jA6yxIiVwkGEyJwOhH3P3/SA+2 +FbReDABIi1wkGEiLdCQgSIt8JChJicBIi0QkQEj/ww8fRAAA6YsAAABIg/lASBn2SIsE2EiJy0iJ +0UjT6EiJ2boBAAAASNPiSCHySP/KSCHQSA+2FWBeDABIhdJ0CkgxyfNID7jI6wjo19v9/0iJwUiJ +yEiLbCQwSIPEOMOEAEiJ2UjB6wZIg/sIcxVIiwTYSNPoSIPgAUiLbCQwSIPEOMNIidi5CAAAAOiW +6AIASP/DTQHITIlEJBhIOfNzRUyLDNhmkEiF0nQH800PuMnr3kiJXCQQTInI6Gfb/f9ID7YV1F0M +AEiLXCQQSIt0JCBIi3wkKEyLRCQYSYnBSItEJEDrq0iD5z9IjU8BSIP5QEgZ278BAAAASNPnSCH7 +SP/LSCMc8EiF0nQKSDHJ80gPuMvrEEiJ2OgJ2/3/TItEJBhIicFKjQQBSItsJDBIg8Q4w0iJ8LkI +AAAA6ObnAgBIidi5CAAAAOjZ5wIAkEiJRCQISIlcJBBIiUwkGOhE4AIASItEJAhIi1wkEEiLTCQY +6dD9///MzMzMzMzMzMzMzMzMzMzMMclIx8L/////Mdsx9usDSP/BSIP5CH1IhABIizzIZpBIhf91 +BkiDw0Dr5EwPvMdMD73PSQHYSIP6/0kPRNBMOcZJD0LwSA+9/0jHx/////9MD0TPkEmNWcFI99vr +sWaQSIP6/3RWSDnzSA9H80iD/j5zBDHJ61RmDx+EAAAAAABIgf4AACAAdQxIvgAAAAAAAACA6yNI +geL//x8ASIHm//8fAEjB5hVICdZIgeP//x8ASMHjKkgJ3kiJ8MNIuAACAEAAAAgAw0mNSgFIg/kI +fUOEAEiLPMhMD7zHQblAAAAATQ9EwUmJykyJwUjT70yNRwFJhfh00LkBAAAASYnwkOmzAAAATInG +671MicbruEyJxuuzSIH+AAAgAHUMSL4AAAAAAAAAgOsjSIHi//8fAEiB5v//HwBIweYVSAnWSIHj +//8fAEjB4ypICd5IifDDTInZSYn0SIn+TInn61pJictIifFJifxI0+9JCfxJjXQkAUmF9HUI65FJ +ictJifxMieZJ99RJD7z8SQ9E+UiJ+UjT7kgPvP5JD0T5SIn5SNPuSQHITI1mAUyF5nWj6V3///9I +Kc5I0eFIhfZ2vA8fAEg58XOZSYn7SNPvTAnfTI1fAUyF33Xb6Sn////MzMzMSTtmEHZ5SIPsIEiJ +bCQYSI1sJBhIg/sBdQiEAEjB6QbrJ0iD+0B3D+iTAAAASItsJBhIg8Qgw+iEAQAASItsJBhIg8Qg +w0j/wUiD+QhzGUiLFMhI99JIhdJ060gPvNJIweEGSAHR6wdIx8H/////SInISInDSItsJBhIg8Qg +w0iJRCQISIlcJBBIiUwkGOit3QIASItEJAhIi1wkEEiLTCQY6Vn////MzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMSMHpBkjHwv////8x9usESY1KAUiD+QgPg50AAACEAEiLPMhJifhI99dIhf91B0mJ +yjH269pIg/r/dRFMD7zPSYnKSMHhBkqNFAnrA0mJyk0PvMhBu0AAAABND0TLSQHxTDnLdkdIjXP/ +uQEAAADrXr5AAAAA6xRIifFIif5I0+9IIfdID7z3SQ9E80iD/kByV0kPvfBIx8f/////SA9E90iD +xsFI997pY////0nB4gZJKfJMidBIidPDSMfA/////0iJ08NIKc5I0eFMic9IhfZ2sEg58XOfSYn5 +SNPvSSH5TYXJdd/riEnB4gZKjQQWSInTw8zMzMzMzMzMzMzMzEjB6QYx0kjHxv////9Ix8f///// +6wpI/8EPH4AAAAAASIP5CA+D4wAAAIQATIsEyEmD+P91CjHS69xmDx9EAABIg/7/dSFNicFJ99BN +D7zAQbpAAAAATQ9EwkmJykjB4QZKjTQB6wZNicFJicpIhdJ1NEkPvclIx8L/////SA9EypBMidJJ +weIGSY08CkiNfwFIg8HBSPfZSYnISInRTInC6XD///9JD7zJQbhAAAAASQ9EyEyNBBFMOcN2SQ8f +AEiD+UBzMUkPvdFJx8D/////SQ9E0JBNidBJweIGSY08EkiNfwFIjUrBSPfZSInKTInB6R////9I +jUpASInKTInR6RD///9IifhIifPDSDnTdgtIx8D/////SInzw0iJ+EiJ88PMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMSTtmEHZJSIPsMEiJbCQoSI1sJChIiUQkOEiJTCQgSIlcJBiEAOg29v// +SItUJDhIjUJASItcJBhIi0wkIA8fAOhb9///SItsJChIg8Qww0iJRCQISIlcJBBIiUwkGGaQ6Nva +AgBIi0QkCEiLXCQQSItMJBjriszMzMzMzMzMzMxJO2YQdjdIg+wQSIlsJAhIjWwkCEiJRCQYhAAP +H0QAAOjb9v//SItMJBhIjUFA6A34//9Ii2wkCEiDxBDDSIlEJAjoedoCAEiLRCQI67LMzMzMzMzM +zMzMzMzMzMzMzMxJO2YQD4aJAAAASIPsKEiJbCQgSI1sJCBIiVwkOEiJRCQwSMHjA0iD+AF1CUiN +s7AAAADrDkiNcP5Ig/4Bd0JIjXNASIl0JBhIifAx20iNDcxuDADoZ2j+/0iLVCQYSAEVo1cMAEiL +VCQwSIlQEEiLVCQ4SIlQKEiLbCQgSIPEKMNIjQUs9QMAuxsAAADorm8AAJBIiUQkCEiJXCQQDx8A +6LvZAgBIi0QkCEiLXCQQ6Uz////MzMzMzMzMzMzMzMxJO2YQdkBIg+wYSIlsJBBIjWwkEEiDeBAB +dRZIi0goSI0EyEiNQDBIi2wkEEiDxBjDSI0F5+UDALsUAAAADx8A6DtvAACQSIlEJAjoUNkCAEiL +RCQI66nMzMzMzMzMzMxJO2YQdkhIg+wYSIlsJBBIjWwkEEiLSBBIg/kCdAhmkEiD+QN1FkiLSChI +jQTISI1AMEiLbCQQSIPEGMNIjQVo5QMAuxQAAADo024AAJBIiUQkCOjo2AIASItEJAjrocxJO2YQ +D4b8AgAASIPsYEiJbCRYSI1sJFhEiIQkkAAAAEiJXCRwSIlMJHhIiUQkaEiJtCSIAAAASIl8JFBI +gz2sagkAAHVluPj4FQBIjR1ObQwA6InR/v+DPYJXDAAAZpB1CUiJBYdqCQDrDEiNPX5qCQDoKdsC +AEiDPXFqCQAAkA+EawIAAEiLRCRoSItMJHhIi1wkcEiLtCSIAAAASIt8JFBED7aEJJAAAAAx0kUx +yesfTIsU0Uj/wk0Byk2J00nB4gpPjQwTTYnKScHpBk0x0Ug513/cTIsVE2oJAEGEAkkB2U2Jy0nB +4QpNAdlNictJwekGTTHZT40MyU2Jy0nB6QtNMdlIicJIuJIaGVO8H2q6SYnTSffhSMHqEUxp4h+/ +AgBMichMKeBIPR+/AgAPg7IBAABMiUwkMEiJRCRITYsUwusDTYsSTYXSD4SPAAAATTlaEHXuTTlK +GHXoDx8ASTlaIHXfkEmLUihIg/ogD4doAQAATIlUJEBJjUIwSInTSYnwSIn+SInPSInZ6M4BAACE +wHU5SItEJEhIi0wkeEiLXCRwSIu0JIgAAABIi3wkUEQPtoQkkAAAAEyLTCQwTItUJEBMi1wkaOl2 +////SItEJEBIi2wkWEiDxGDDZpBFhMAPhN4AAABMidhIifvojPz//5BIi1AoDx+AAAAAAEiD+iAP +h8kAAABIi3QkUEg51kgPTNZIjXAwSItcJHiQSDnzdBlIiUQkOEjB4gNIifBIidHoJ+cCAEiLRCQ4 +SItMJDBIiUgYSItMJHBIiUggSIsNmWgJAIQBSItUJEhIiwzRSIkISIsNhGgJAIQBSIkE0UiLTCRo +SIP5AXUUSIsNhFQMAEiJSAhIiQV5VAwA6yxIg/kDdRRIiw0qVQwASIlICEiJBR9VDADrEkiLDcZT +DABIiUgISIkFu1MMAEiLbCRYSIPEYMMxwEiLbCRYSIPEYMO7IAAAAOiD3gIAuyAAAADoed4CALkf +vwIA6E/dAgBIjQXt+wMAux8AAAAPHwDou2sAAJBIiUQkCEiJXCQQSIlMJBhIiXwkIEiJdCQoRIhE +JDDot9UCAEiLRCQISItcJBBIi0wkGEiLfCQgSIt0JChED7ZEJDDps/z//8zMzMzMzMzMzMzMzMzM +zMzMzMxIiUQkCEiJfCQgSDnedQQxyesGMcDDSP/BSDnLfhBIixTISIs0z0g51nTrMcDDuAEAAADD +zMzMzMzMzMzMzMzMSTtmEHZpSIPsEEiJbCQISI1sJAiQSI0FxFMMAA8fQADo20D+/4sNNVMMAI1R +AburqqqqSA+v2kjB6zqNFFvB4hkp0f/BiQ0VUwwAxgUSUwwAAJCQSI0FhVMMAA8fRAAA6HtC/v9I +i2wkCEiDxBDD6MzUAgDriszMzMzMzMzMzMxJO2YQdkxIg+wQSIlsJAhIjWwkCJBIjQVEUwwADx9A +AOhbQP7/gD24UgwAAHUM6C0AAADGBapSDAABkJBIjQUdUwwA6BhC/v9Ii2wkCEiDxBDD6GnUAgDr +p8zMzMzMzMxJO2YQD4a3AAAASIPsIEiJbCQYSI1sJBiLDWJSDACJTCQMSIsVX1IMAOmAAAAASIlU +JBBIidDohfr//7mrqqqqi1QkDEgPr8pIwekhjQxJidMpykjB4gWQSItMECBIAQhIi0wQKEgBSAhI +i0gQSANMEDBIiUgQSItMEDhIAUgYSMdEECAAAAAASMdEECgAAAAASMdEEDAAAAAASMdEEDgAAAAA +SItMJBBIi1EIidlIhdIPhXf///9Ii2wkGEiDxCDD6JrTAgDpNf///8zMzMzMzMzMzMzMzMzMzMzM +zMzMzEyNpCQY////TTtmEA+GdAEAAEiB7GgBAABIiawkYAEAAEiNrCRgAQAASImEJHABAABIiZwk +eAEAAEiNfCRIZpBIiWwk8EiNbCTw6OXcAgBIi20AuSAAAABIic+4BAAAAEiNXCRI6FZ0AgBIiUQk +OJCQSI0FqFEMAOjDPv7/SItMJDhIg/kgD4fuAAAAuAEAAABIi5wkeAEAAEiJz74gAAAAQbgBAAAA +SI1MJEjo7/n//0iJRCRAixXkUAwAiVQkNOgb+f//hACLVCQ0RI1KAkG6q6qqqk0Pr9FJweohR40M +UkQpyoPCAkjB4gVI/0QQIEyLjCR4AQAATAFMEDCQkEiNBRpRDADoFUD+/0QPEbwkSAEAAEjHhCRY +AQAAAAAAAEiNFZkAAABIiZQkSAEAAEiLlCRwAQAASImUJFABAABIi1QkQEiJlCRYAQAASI2UJEgB +AABIiRQk6OPQAgBFD1f/ZEyLNCX4////SIusJGABAABIgcRoAQAAw7ogAAAAkOib2QIAkEiJRCQI +SIlcJBDo69ECAEiLRCQISItcJBCQ6Vv+///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQ +dilIg+wYSIlsJBBIjWwkEEiLWhBIi0IIDx9AAOi7tf//SItsJBBIg8QYw+js0AIA68rMzMzMzMzM +zMzMSTtmEA+GhQAAAEiD7CBIiWwkGEiNbCQYSIlcJDBIiUQkEJBIjQX2TwwA6BE9/v+LDWtPDACJ +TCQMSItEJBBmkOib9///hACLTCQMjVEBu6uqqqpID6/aSMHrIY0UWynR/8FIweEFSP9ECChIi1Qk +MEgBVAg4kJBIjQWiTwwAZpDomz7+/0iLbCQYSIPEIMNIiUQkCEiJXCQQ6OLQAgBIi0QkCEiLXCQQ +6VP////MzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GxQAAAEiD7ChIiWwkIEiNbCQgSIsVQU4MAEiF +wL4BAAAASA9OxkiF0n8QSInBSYnTugEAAADrbQ8fAEg5wn5dSYt2MESLhiABAABEi44kAQAARYnC +QcHgEUUx0EWJykUxwUHB6AdFMchFidFBweoQRTHCR40EEUiJwUSJwEmJ00iZSff7RImOIAEAAESJ +liQBAABIOcoPn8JmkOsISInBSYnTMdKE0nUXSI1TAUiJyEyJ20iJ0b8CAAAA6DsAAABIi2wkIEiD +xCjDSIlEJAhIiVwkEOjizwIASItEJAhIi1wkEOkT////zMzMzMzMzMzMzMzMzMzMzMzMzEyNpCQY +////TTtmEA+GtQEAAEiB7GgBAABIiawkYAEAAEiNrCRgAQAASImEJHABAABIiZwkeAEAAEiJvCSI +AQAASI18JGBIiWwk8EiNbCTw6B/ZAgBIi20ATYtmMEmLvCTAAAAASIX/dDhNifRJOfx0MEjHBCQA +AAAASMfA/////0iJw0iJzkyNRCRgQbkgAAAARTHSRTHbMcnoAUMCAJDrFUiJyEiNXCRguSAAAABI +ic/oSXACAEiJRCRQkJBIjQWbTQwA6LY6/v9Ii0wkUEiD+SAPh+MAAABIi4QkiAEAADHbSInPviAA +AABBuAEAAABIjUwkYOjl9f//SIlEJFhIi5QkiAEAAEiD+gJ1ZUiLjCR4AQAASIuUJHABAABmkEg5 +0X5O6Fb1//9Ii4wkeAEAAA9XwPJIDyrBSIuUJHABAAAPV8nySA8qyvIPXsHyD1gA8g8RAEiLRCRY +Dx9EAADoG/X//0iLjCR4AQAASAFICOsr6Aj1///yDxAF0BQFAPIPWADyDxEASItEJFjo7vT//0iL +jCRwAQAASAFICJCQSI0FuUwMAOi0O/7/SIusJGABAABIgcRoAQAAw7ogAAAA6JrVAgCQSIlEJAhI +iVwkEEiJTCQYSIl8JCAPH0QAAOjbzQIASItEJAhIi1wkEEiLTCQYSIt8JCDpAv7//8zMSTtmEA+G +DQIAAEiD7HhIiWwkcEiNbCRwSImMJJAAAABIiVwkMEiJRCRIkEiNBV5MDADoSTn+/0yJdCQ4SYtO +MMaBKQEAAAJIi4QkkAAAAEiFwA+EhQAAAOhjfwIASIlEJEBIiVwkKOiUewAASI0F18oDALsLAAAA +6KOEAABIi0QkSOgZhAAASI0FM8ADALsCAAAA6IiEAABIi0QkMA8fAOjbggAASI0FFcADALsCAAAA +6GqEAABIi0QkQEiLXCQo6FuEAABIjQXzvwMAuwIAAADoSoQAAOilewAA614PHwDoG3sAAEiNBV7K +AwC7CwAAAOgqhAAASItEJEgPH0QAAOibgwAASI0Ftb8DALsCAAAA6AqEAABIi0QkMA8fRAAA6FuC +AABIjQWTvwMAuwIAAADo6oMAAOhFewAASItEJDhIi0gwSIuJwAAAAEiFyXQ3SDnIdDJIicjoY3IC +AEiLTCQ4SItRMEiLusAAAABIx8D/////SInDMfYxyQ8fRAAA6LtlAgDrWeg0cgIARA8RfCRQRA8R +fCRgSI0NoQAAAEiJTCRQSItMJHhIiUwkWEiNjCSAAAAASIlMJGBIi0wkOEiJTCRoSI1UJFBIiRQk +6K3KAgBFD1f/ZEyLNCX4////6Bt6AADolnwAAOiRegAASItEJDhIi0AwxoApAQAAAJCQSI0FiEoM +AOhTOf7/SItsJHBIg8R4w0iJRCQISIlcJBBIiUwkGOiVywIASItEJAhIi1wkEEiLTCQY6cH9///M +STtmEHYuSIPsMEiJbCQoSI1sJChIi1oQSIt6GEiLQgiQMckx9ujWZAIASItsJChIg8Qww+inygIA +68XMzMzMzEk7ZhAPhigBAABIg+xQSIlsJEhIjWwkSEiJXCQQSIlEJCCQSI0F5kkMAOjRNv7/TIl0 +JBhJi04wxoEpAQAAApDoO3kAAEiNBdTGAwC7CgAAAOhKggAASItEJCAPH0QAAOi7gQAASI0F1b0D +ALsCAAAA6CqCAABIi0QkEA8fRAAA6HuAAABIjQWzvQMAuwIAAADoCoIAAOhleQAASItEJBjom3AC +AEQPEXwkKEQPEXwkOEiNDagAAABIiUwkKEiLTCRQSIlMJDBIjUwkWEiJTCQ4SItMJBhIiUwkQEiN +VCQoSIkUJOgXyQIARQ9X/2RMizQl+P///+iFeAAADx9EAADo+3oAAOj2eAAASItMJBhIi0kwxoEp +AQAAAJCQSI0F7UgMAOi4N/7/SItsJEhIg8RQw0iJRCQISIlcJBAPH0AA6PvJAgBIi0QkCEiLXCQQ +6az+///MzMzMzMzMzMzMzMxJO2YQdi5Ig+wwSIlsJChIjWwkKEiLWhBIi3oYSItCCJAxyTH26DZj +AgBIi2wkKEiDxDDD6AfJAgDrxczMzMzMSTtmEA+GpQAAAEiD7CBIiWwkGEiNbCQYkEiNBVBIDADo +OzX+/0yJdCQQSYtOMMaBKQEAAALopncAAEiNBUnFAwC7CgAAAOi1gAAA6BB4AABIi0QkEOiGcQIA +6IF3AABIjQU2yAMAuwwAAADokIAAAOjrdwAA6GZ3AADo4XkAAJDo23cAAEiLTCQQSItJMMaBKQEA +AACQkEiNBdJHDABmkOibNv7/SItsJBhIg8Qgw+jsyAIA6Uf////MzMzMzMzMSTtmEHZLSIPsGEiJ +bCQQSI1sJBBIuQAAAAAAgAAASI0UAUg50A+WwkgB2Ug5yw+WwTjRdQpIi2wkEEiDxBjDSI0FqAsE +ALs8AAAA6HBeAACQSIlEJAhIiVwkEA8fRAAA6HvIAgBIi0QkCEiLXCQQ64/MzMzMzMzMzMzMzMzM +zMxJO2YQD4ahAAAASIPsGEiJbCQQSI1sJBBIugAAAAAAgAAASI00CkyNBAJJOfByIkyNDBpMjRQX +TTnKcwZJOfCQ6w8xwEiJw0iLbCQQSIPEGMNzEkyNDBdMjRQaDx9EAABNOcp3NEyNDBdIAdpMOcp2 +EU05wXYFSIn46xRJOfBmkOsDSTnwcwhIOfJ2A0iJy0iLbCQQSIPEGMNIjQVlwAMAuwkAAADoll0A +AJBIiUQkCEiJXCQQSIlMJBhIiXwkIJDom8cCAEiLRCQISItcJBBIi0wkGEiLfCQg6SL////MzEk7 +ZhB2WEiD7BhIiWwkEEiNbCQQSLoAAAAAAIAAAEiNNApIjTwCSDn3cyRIAdpIOfJ3CkiLbCQQSIPE +GMNIicvoW/7//0iLbCQQSIPEGMMxwEiJw0iLbCQQSIPEGMNIiUQkCEiJXCQQSIlMJBjoDscCAEiL +RCQISItcJBBIi0wkGOl6////zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdnxIg+wgSIls +JBhIjWwkGEiJRCQoSIlcJDBIx0AIAAAAAEjHQBAQAAAASInZuwgAAAC4AAEAAA8fRAAA6PtU/v9I +i1QkKEiJAoM9jEUMAAB1DEiLRCQwSIlCIJDrDkiNeiBIi0QkMOgwyQIASMdCGAAAAABIi2wkGEiD +xCDDSIlEJAhIiVwkEOhPxgIASItEJAhIi1wkEA8fRAAA6Vv////MzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMxIg+wYSIlsJBBIjWwkEEiLSAhIixBIicgx9usDTInRSInPSCnxSIP5CH51SYnISMHp +P0kByEnR+E2NDDAPH0AATDnID4anAAAATYnKScHhBE2LHBFOi0wKCEm8AAAAAACAAABNAeNNjSwc +TTndcgtNAeFNOel3Ek053XKgSY00MEiNdgFJifrrk0mNBDBIjUABSItsJBBIg8QYw0mNcAGQSDn+ +fTNIOcZzO0mJ8EjB5gRIizQWSbkAAAAAAIAAAE2NFBlMAc5MOdZ20EyJwEiLbCQQSIPEGMNIifhI +i2wkEEiDxBjDSInBSInw6JDMAgBIicFMicjohcwCAJDMzMzMSTtmEA+G7gAAAEiD7BhIiWwkEEiN +bCQQSIlEJCBIiVwkKOjZ/v//SIXAdS9Ii1QkIEiLSghIixIPH4QAAAAAAEiFyQ+GpwAAAEiLArsB +AAAASItsJBBIg8QYw0iLVCQgSItKCEiLEkiNcP9IOfF2eEjB5gRIizwWSIt0MghJuAAAAAAAgAAA +TAHHTItMJChPjRQISTn6ch9MAcYPH0QAAEw51nYSTInIuwEAAABIi2wkEEiDxBjDSDnIfRlzJUjB +4ARIiwQQuwEAAABIi2wkEEiDxBjDMcAx20iLbCQQSIPEGMPomMsCAEiJ8OiQywIAMcDoicsCAJBI +iUQkCEiJXCQQ6BnEAgBIi0QkCEiLXCQQ6er+///MzMzMzMzMzMzMSTtmEA+GEgUAAEiD7GBIiWwk +WEiNbCRYSLoAAAAAAIAAAEiNNBNIAcpIOfJ2CZBIic9IKdnrBUiJzzHJSIl8JEhIiVwkQEiFyQ+E +XgQAAEiJdCQ4SIlUJDBIiUQkaGaQ6Hv9//9IhcB+OEiLdCRoSItOCEiLPkyNQP9mDx9EAABMOcEP +hhwEAABJweAESot8BwhMi0QkQEw5x0APlMdmkOsMSIt0JGhMi0QkQDH/SItOCEyLDkiLVhBIOch9 +IWaQD4PaAwAASYnCSMHgBE2LHAFMi2QkSE0540EPlMPrC0mJwkyLZCRIRTHbRYTbD4TxAAAAQIT/ +D4TjAAAADx9AAEk5yg+DjgMAAEmNQv9MiddJweIET4tcEQgPH4AAAAAASDnBD4ZpAwAASMHgBE2J +XAEISItOCEyLDkyLXhAPHwBIOc8Ph0EDAABJic1IKflJKftNid9J99tJwfs/TSHTS40EC0j/x0k5 +/Q+CEAMAAEyNUf9MOdFJD0/KTY1X/0n32kjB5wRJwfo/SSH6S40cEUg5w3QYSMHhBOji0gIASIt0 +JGhMi0QkQEyLZCRISItWEEiLRghIjUj/Dx+AAAAAAEg5yg+CsQIAAEiJTghIi0QkMEiLTCQ4SDnI +6TICAABFhNvrMECE/3QoSY1C/0g5wQ+GfwIAAEjB4ARNiWQBCEiLRCQwSItMJDhIOcjpAAIAAEWE +23QjSTnKD4NOAgAAScHiBE+JBBFIi0QkMEiLTCQ4SDnI6dgBAABMiVQkKEiNeQFIOfoPjJMAAAAP +ghQCAABIiX4ISIsWTItOEEmNQgFIOfgPh+8BAABMKdFNKdFNjVn/SffbSMHgBEnB+z9JIcNKjQQa +STn6D4e/AQAASI15AUg5+UgPT89J99lMiddJweIEScH5P00h0UmNHBFIOcMPhDQBAABIweEEDx9E +AADou9ECAEiLdCRoSIt8JChMi0QkQEyLZCRI6Q0BAABIiVQkIEiJTCQYTIlMJFBIiX4ISInQSNHi +SIlWEEjB4AVIi04guwgAAADoU0/+/0iLVCRoSIkCSItyEEiLTCQoZpBIOfEPhxwBAABIi3QkIEg5 +8Q+HAwEAAEiLXCRQDx9AAEg52HQdSMHhBOgy0QIASItMJChIi1QkaEiLXCRQSIt0JCBIi3oITIsC +TItKEEiNQQFIOccPgrgAAABIKc9I/89Mi1QkGEkpykk5+kkPTPpJKclJ/8lJ99lIweAEScH5P0kh +wUuNBAhJichIKfFMicZJweAESMH5P0whwUgBy2aQSDnDdBZIwecESIn56K/QAgBIi1QkaEiLdCQo +SIn3TItEJEBMi2QkSEiJ1kiLFkiLTghIOc9zM0jB5wRMiQQ6TIlkOghIi0QkMEiLTCQ4SDnIdgaQ +TSnE6wNFMeRMAWYYSItsJFhIg8Rgw0iJ+OguxwIASIn56ObHAgBIifIPHwDom8cCAEiJ8uiTxwIA +TInQSIn56MjHAgBIifkPH0QAAOi7xwIASIn56HPHAgBMidDo68YCAOjmxgIA6GHHAgBIifhMieno +lscCAEiJ+OiOxwIA6MnGAgBMidDowcYCAJDou8YCAEyJwOizxgIA6I5tAABIjQWQyAMAuxIAAABm +kOibdgAASItEJEDo8XQAAEiNBSuyAwC7AgAAAA8fRAAA6Ht2AABIi0QkSOjRdAAASI0FIbIDALsC +AAAADx9EAADoW3YAAOi2bQAASI0FY/MDALspAAAA6MVUAACQSIlEJAhIiVwkEEiJTCQY6NC+AgBI +i0QkCEiLXCQQSItMJBiQ6bv6///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxIiwhIi1AISIXS +dGJI/8pIidZIweIESIt8EQhMiwQRSbkAAAAAAIAAAE+NFAFJAflNOdF2CZBJiflMKcfrBUmJ+TH/ +SDn7cxeQTInOSSnZTIlMEQhIKVgYTInISInzw0iJcAhIKXgYTInATInLwzHASInDw8zMzMzMzMzM +zMzMzEk7ZhAPhgECAABIg+xQSIlsJEhIjWwkSEiJRCRYSIlcJGDo2ff//0iFwHRJSIt0JFhIiz5I +i04ITItGEA8fQABIOcgPh7wBAABIKcFJicFMKcBNichJweEESMH4P0khwUkB+UiFyX4JMcAx0ulR +AQAAMcDrH0iLRCRYSMdAGAAAAABIx0AIAAAAAEiLbCRISIPEUMNNjUj/TYnKScHhBE6LHA9Ki1wP +CEi/AAAAAACAAABOjSQfSItMJGBMjSw5kE055Q+CxAAAAEyNJB9NOewPhrcAAABMiUQkQEyJVCQ4 +SIlcJDBMiVwkKEyJTCQgSIlEJBiQTInY6KP1//9Ii1QkMEiLdCQoSCnySIt0JBhIAfJIvgAAAAAA +gAAASI08BkgB3kg5/nYJkEmJ2Egpw+sFSYnYMdtIhdt1DEiLdCRYSItEJDjrQEg5/nYJkEyJw0kp +wOsGTInDRTHATCnCSIt0JFhIiz5Ii04ITItEJDhMOcF2PEyLRCQgSokEB0qJXAcISItEJEBJicBI +idBIi1YQSTnQdxJMiUYISClGGEiLbCRISIPEUMNMicHoUMQCAEyJwOjIwwIASYPBEEyJ2EyJ0k2L +UQhNixlJvAAAAAAAgAAAT40sHE+NPBQPH0QAAE0573YGkE0p2usDRTHSTI1YAUkB0kw52X++TInQ +Dx8A6Y/+///oNsQCAJBIiUQkCEiJXCQQ6Aa8AgBIi0QkCEiLXCQQ6df9///MzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzEk7ZhAPhqsAAABIg+wgSIlsJBhIjWwkGEiLcAgPH0AASDlzEH0/SIlEJChIiVwk +MEjHQwgAAAAASItQEEiJUxBIweIESItLIEiJ0LsIAAAA6OtJ/v9Ii1QkMEiJAkiLRCQoSInTSItT +EEiLSAhIOcpyPUiJSwhIi1AYSIlTGEiLUAhIOcpID0zKSIsTSIsYSDnadBFIweEESInQDx9EAADo +u8sCAEiLbCQYSIPEIMPoDMMCAJBIiUQkCEiJXCQQkOgbuwIASItEJAhIi1wkEOks////zMzMzMzM +zMzMzMzMSTtmEA+GawEAAEiD7FBIiWwkSEiNbCRISIlEJFhIiVwkYIQASI1IIEiJyOiQBAAASItM +JFhIi1EQjVj/iVwkHIneSMHrCUiJXCQw6xqQkOhMKP7/SItMJFhIi1wkMIt0JBxIi1QkKEg52g+H +1gAAAJCQSInI6EYm/v9Ii0QkWEiLSBBIiUwkKEiLVCQwSDnRd7tIi3AYSDnxdWVI0eZIhfa6AAEA +AEgPRPJIiXQkIEiLHYoYCQBIifBIweADSI0NFE8MAOiXSP7/SItUJFhIi0oYSIXJdBxIiUQkOEiL +WghIweED6JfKAgBIi0QkOEiLVCRYSIdCCEiLTCQgSIlKGEiNBak4DADoFAMAAEiJRCRASItMJDBI +weEDSItUJFhIA0oISIcBSItMJChI/8FIh0oQkEiJ0OhlJ/7/i3QkHEiLTCRA6wxIi0kIkEiNDNlI +iwmEAUiB5v8BAABIjQTxSI1AGEiLTCRgSIcISItsJEhIg8RQw0iJRCQISIlcJBCQ6Hu5AgBIi0Qk +CEiLXCQQ6Wz+///MzMzMzMzMzMzMzMxJO2YQD4YPAQAASIPsIEiJbCQYSI1sJBhIi0ggkJBIicpI +wekgOcp2JUiLcBBIiddIweopSDnWdglIicpIifuQ6y8xwEiLbCQYSIPEIMMxwEiLbCQYSIPEIMOQ +SItzIJCQSInxSMHpIEiJ2EiJ90iJ+znRdaONcgFIweYgQYnYTAnGSInDSIn48EgPsXMgQA+UxkCE +9nTDSItTCInOwekJkEiNDMpIixGEAoHm/wEAAEiNNPJIjXYYSIs+6wZMiwZMicdIhf909UUxwEyH +Br4BAAAA8A/BchD/xmYPH0QAAIH+AAIAAHUkSIl8JBAx9kiHMZAxyYdKEEiNBRE3DABIidPouSH+ +/0iLfCQQSIn4SItsJBhIg8Qgw0iJRCQIZpDoO7gCAEiLRCQI6dH+///MzMzMzMzMzMzMzMzMzMzM +zEk7ZhAPhgcBAABIg+woSIlsJCBIjWwkIEiLSCCQSInKSMHpIDnKD4eCAAAAwekJSDlIEHZASMHh +A0gDSAhIixlIhdt0MItTEIXSdFGB+gACAAB0OEiJRCQwMdJIhxGQMcmHSxBIjQVlNgwA6BAh/v9I +i0QkMJAxyUiHSCAxyUiHSBBIi2wkIEiDxCjDSI0FnfQDALsxAAAA6GNNAABIjQWA9wMAuzQAAADo +Uk0AAEiJVCQYSIlMJBDoo2UAAEiNBXatAwC7BwAAAOiybgAASItEJBDoqGsAAEiNBcKvAwC7CQAA +AOiXbgAASItEJBiJwOiLawAA6OZnAADo4WUAAEiNBWjkAwC7IwAAAOjwTAAAkEiJRCQI6AW3AgBI +i0QkCOnb/v//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEHZ7SIPsIEiJbCQYSI1sJBiE +AOsDSInISIsQZpBIhdJ0JUiJ1kjB+hNIweIDkEiLOkiJwUiJ8PBID7E5QA+UxkCE9nTQ6wIx0kiF +0nQNSInQSItsJBhIg8Qgw0iLHbsUCQC4GBAAAEiNDUdLDADoykT+/0iLbCQYSIPEIJDDSIlEJAjo +VbYCAEiLRCQI6Wv////MzMzMzMzMzMzMzEk7ZhAPho0AAABIg+wgSIlsJBhIjWwkGLkBAAAA8EgP +wQhIjUEBhcB0CkiLbCQYSIPEIMNIiUQkEOhCZAAASI0F2bsDALsQAAAA6FFtAABIi0QkEEjB6CDo +Q2oAAEiNBV2uAwC7CQAAAOgybQAASItEJBCJwOgmagAA6IFmAACQ6HtkAABIjQVWxgMAuxYAAADo +iksAAJBIiUQkCA8fQADom7UCAEiLRCQI6VH////MzMzMzMzMzMzMzMzMzMzMzMPMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMSIPsIEiJbCQYSI1sJBhIhcB0PEiJXCQwSInZ8EgPwRhIjQQZ +SIlEJBBIhcl+CEg5wX8mSIXJfQ1IjRRLDx9AAEg50X8USItsJBhIg8Qgw0iLbCQYSIPEIMPoQmMA +AEiNBSK2AwC7DQAAAOhRbAAASItEJBDoR2kAAEiNBQ6oAwC7AwAAAOg2bAAASItEJDDoLGoAAOiH +ZQAA6IJjAABIjQVOwAMAuxMAAADokUoAAJDMzMzMzMzMzMzMzMzMzMzMSTtmEA+G3AAAAEiD7CBI +iWwkGEiNbCQYSYtWMEiLktAAAABIhdJ0GrsBAAAA8A/BmtQmAACNUwEPuuIAciBmkOtcSIlEJCiE +AEiNiOgNAACQSInI6Acg/v9Ii0QkKIuQ4A0AAInTvquqqqpID6/zSMHuIY0cdinaSIP6A3MUSGnK +oAQAAEgByEiLbCQYSIPEIMOJ0LkDAAAA6GK7AgCJVCQU6DliAABIjQUMtQMAuw0AAADoSGsAAItE +JBQPH0AA6DtoAADolmQAAOiRYgAASI0F9L0DALsTAAAADx9EAADom0kAAJBIiUQkCOiwswIASItE +JAjpBv///8zMzMzMzEk7ZhAPhpIAAABIg+wgSIlsJBhIjWwkGEmLTjBIi4nQAAAASIXJdBe4AQAA +APAPwYHUJgAA/8APuuAAcxDrGIQASAXoDQAAkOjzIP7/SItsJBhIg8Qgw4lEJBQPH0QAAOh7YQAA +SI0FTrQDALsNAAAA6IpqAACLRCQU6IFnAACQ6NtjAADo1mEAAEiNBTm9AwC7EwAAAOjlSAAAkEiJ +RCQI6PqyAgBIi0QkCOlQ////zMzMzMzMzMzMzMzMzMzMzEk7ZhB2YEiD7BhIiWwkEEiNbCQQSI1I +EEiJCIA9wzEMAAB0CkiNSCBIiUgI6wtIgcEAEAAASIlICEiLSAhIKwhI98EPAAAAdQpIi2wkEEiD +xBjDSI0FVdcDALsfAAAAkOhbSAAAkEiJRCQI6HCyAgBIi0QkCOuJzMzMzMzMzMzMSIPsGEiJbCQQ +SI1sJBBJi04wg7kMAQAAAH9ogD1DMQwAAHQ4SIXAdDPocr/9/4A9LzEMAAB1JUmLRjBIi4DQAAAA +hABIjYjQFgAASImIwBYAAEiLbCQQSIPEGMNIjQUN+wMASIkEJOi0sAIARQ9X/2RMizQl+P///0iL +bCQQSIPEGMNIi4HQAAAAhABIjYjQFgAASImIwBYAAEiLbCQQSIPEGMPMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMSTtmGA+GkwIAAEiD7EBIiWwkOEiNbCQ4hABIjbDQFgAASIuIwBYAAEgp8UjB +6QNIgfkAAgAAD4dWAgAASIlEJEhIiUwkMEjHgMAWAAAAAAAAgD07LgwAAHUIMdIx22aQ60kx0uso +SIlUJChIi4zQ0BYAAEiJyOjFDP//SItMJChIjVEBSItEJEhIi0wkMEg5ynzTSAXAFgAA6CL+//9I +i2wkOEiDxEDDSP/CSDnKD41xAQAASIu00NAWAAAPH0AASIH+ABAAAHLfSIlUJChIiVwkIEiJ8DHb +SInZDx9EAADoG2z+/0iFwHUYSItEJEhIi0wkMEiLVCQoSItcJCBmkOumkEiJzkjB6QNIg+YHSInK +SInxQbgBAAAAQdPgkEgDU1APtjJBhPB0G0iLRCRISItMJDBIi1QkKEiLXCQgZpDpY////5DwRAgC +SIs1p8QLAIQGkJBIi0sYkEm4AAAAAACAAABNjQwIScHpGkmB+QAAQAAPgwwBAABKizTOhAZJiclI +wekQSIHh/wMAAEQPtpQOAAQhAEiNNA5IjbYABCEAScHpDUmD4QdMiclBuQEAAABB0+FFhNF1BPBE +CA4PtnNiQPbGAXQuSIt0JEhMi46oFgAATANLaEyJjqgWAABIifBIi0wkMEiLVCQoSItcJCDpsv7/ +/0iLdCQgSItMJDBIOc5zd0yLTCRISYmE8dAWAABIjV4BTInISItUJChmkOmD/v//SIH7AAIAAHdB +SI2Q0BYAAEiNsJgWAABIidm/AAIAAEiJ00iJ8OgxYf//SItUJEhIjYLAFgAADx9EAADoW/z//0iL +bCQ4SIPEQMNIidm6AAIAAOgEtwIASInwkOh7tgIATInIuQAAQADojrYCALoAAgAA6MS2AgCQSIlE +JAjoOdYCAEiLRCQI6U/9///MzMzMzMzMzMzMzMzMzMxJO2YQD4b8AAAASIPsMEiJbCQoSI1sJCjH +BCQACAgAkOjbyAIARQ9X/2RMizQl+P///4tEJBCLTCQIi1QkDGaQg/jaD4WaAAAA6JLIAgBFD1f/ +ZEyLNCX4////i0wkCItEJASLFCSFyQ+FhwAAAIlUJCSJRCQgiUwkHOih0AIARQ9X/2RMizQl+P// +/4tEJCSJBCToqNACAEUPV/9kTIs0Jfj///+LRCQgiQQk6G/QAgBFD1f/ZEyLNCX4////i0QkIIkE +JOh20AIARQ9X/2RMizQl+P///4tMJCSLVCQgi0QkHInTicKJyInRSItsJChIg8Qww7j/////icNI +i2wkKEiDxDDD6NWtAgDp8P7//8zMzMzMzMzMzMzMzMzMzMxJO2YQdltIg+wQSIlsJAhIjWwkCIsN +1ioMAIXJdTiQkEiNBfkrDADoVBn+/4M9vSoMAAB1E+hGAQAAuAEAAABIjQ2qKgwAhwGQkEiNBc8r +DADoChv+/0iLbCQISIPEEJDD6FqtAgDrmMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzIP5cnQIgfnp +AAAAdQeEA+mNAAAAMdKD+Xd0DQ8fRAAAgfnpAAAAdQSEA+suMclIhdJ0D5BIixhIiZqgAAAAkEiJ +EEiFyXQPkEiLEEiJkaAAAACQSIkIw0iJ8EiLi4gAAABIg/kBdC1IicZIici/AQAAAPBID7G7iAAA +AEEPlMCQRYTAdNJIg/kCuAAAAABID0TI6wVIicYxyUiJ8OuQSInwSItTKEiD+gF0KkiJxkiJ0L8B +AAAA8EgPsXsoQQ+UwEWEwHTZSIP6AkG4AAAAAEkPRNDrDUiJxr8BAAAARTHAMdJIifDpLP///8zM +zMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4blAQAASIPsSEiJbCRASI1sJEDHBCQAAAgAkOjbzQIA +RQ9X/2RMizQl+P///4tEJAiJBSwKCQAPH0AAhcB9QMcEJAAEAADokM0CAEUPV/9kTIs0Jfj///+L +RCQIiQUBCgkAhcAPjDkBAACJBCToCc4CAEUPV/9kTIs0Jfj////o1/z//4XJD4XUAAAAiUQkKIlc +JCRIx0QkNAAAAABIx0QkOAAAAADHRCQ0AQAAAEiNDdYpDABIiUwkOIsNowkJAIkMJMdEJAQBAAAA +iUQkCEiNTCQ0SIlMJBBmkOg7zQIARQ9X/2RMizQl+P///4tEJBiFwHUni0QkKEhjwEiJBYgpDACL +RCQkSGPASIkFgikMAEiLbCRASIPESJDDiUQkLOhWWQAASI0FJ88DALseAAAA6GViAACLRCQs99hI +Y8DoV2AAAOiyWwAA6K1ZAABIjQVYwQMAuxgAAACQ6LtAAACJTCQs6BJZAABIjQUUxQMAuxoAAADo +IWIAAItEJCz32EhjwOgTYAAA6G5bAADoaVkAAEiNBU24AwC7FAAAAOh4QAAAiUQkMOjPWAAASI0F +9dQDALshAAAADx8A6NthAACLRCQw99hIY8DozV8AAOgoWwAA6CNZAABIjQW+xgMAuxsAAADoMkAA +AJDoTKoCAOkH/v//zMzMzMzMzEk7ZhAPhtIAAABIg+wwSIlsJChIjWwkKDHASI0NVycMALoBAAAA +8A+xEQ+UwYTJdALrCkiLbCQoSIPEMMPGRCQjAEiLBU8oDABIiQQkSI1EJCNIiUQkCMdEJBABAAAA +kOibwwIARQ9X/2RMizQl+P///4tEJBiD+AF0uA8fRAAAg/j8dLiD+PV1CkiLbCQoSIPEMMOJRCQk +6ONXAABIjQVv3QMAuygAAADo8mAAAItEJCT32EhjwOjkXgAADx9AAOg7WgAA6DZYAABIjQUT1gMA +uyIAAADoRT8AAJAPH0AA6FupAgDpFv///8zMzMzMzMzMzMzMzMzMzMzMzMzMzMxMjaQkKPr//007 +ZhAPhgUDAABIgexYBgAASImsJFAGAABIjawkUAYAAIM9OAcJAP90ckiFwH0LSInBuv////+Q60h1 +B0iJwTHS6z9IPUBCDwB9CkiJwboBAAAA6y1IugCAxqR+jQMASDnQfRZIicFIuNs0tteC3htDSPfp +SMH6EusISInBugDKmjtIiUwkOIlUJCBIjXwkUDHAucAAAADzSKvrFDHASIusJFAGAABIgcRYBgAA +w4nKiwWqBgkAiQQkSI1EJFBIiUQkCMdEJBCAAAAAiVQkFOhkygIARQ9X/2RMizQl+P///4tEJBiJ +RCQkhcB9I4P4/A+FuAEAAItMJCCFyX6wMcBIi6wkUAYAAEiBxFgGAADDSMdEJDAAAAAASItUJDgx +yesC/8E5wQ+NIgEAAEhj8WYPH4QAAAAAAEiB/oAAAAAPg1wBAABIjTR2i3y0UIX/dQlMjQ0wJgwA +68iJTCQoTI1EtFRMjQ0eJgwATTkIdXJmDx+EAAAAAACD/wEPheIAAABIhdJ0nEiNRCRARA8ROEiL +BfIlDACJBCRIjUQkQEiJRCQIx0QkEBAAAADoaMECAEUPV/9kTIs0Jfj///8xwEiNDaokDACHAYtE +JCSLTCQoSItUJDhMjQ2sJQwA6UH////3xxkgAABBugAAAABBu3IAAABFD0XTRY1id/fHHAAAAEUP +RdRFhdIPhBT///9JixjGQxkAi3S0UIP+CHUExkMZAUiNRCQwRInR6Mn5//+LRCQki0wkKEiLVCQ4 +TI0NRSUMAEG7cgAAAOnU/v//SItEJDBIi6wkUAYAAEiBxFgGAADDiXwkLOgMVQAASI0FcNcDALsl +AAAA6BteAACLRCQs6BJbAADobVcAAOhoVQAASI0FO+kDALs5AAAA6Hc8AABIifC5gAAAAOjqrQIA +6MVUAABIjQWtvgMAuxkAAADo1F0AAEhjBZUECQDoyFsAAEiNBVCmAwC7DQAAAOi3XQAAi0QkJPfY +SGPA6KlbAADoBFcAAA8fQADo+1QAAEiNBYm6AwC7FwAAAOgKPAAAkEiJRCQIDx9AAOgbpgIASItE +JAjp0fz//8zMzMzMzMzMzMzMzMzMzMzMSIPsSEiJbCRASI1sJEBIhcl9OkiJBCTHRCQIgAAAAIlc +JAxEDxF8JBDHRCQgAAAAAOiqxQIARQ9X/2RMizQl+P///0iLbCRASIPESMNEDxF8JDCQSInCSLjM +UlqboC+4REiJ1kj36UjB+hxIiVQkMEhp0gDKmjtIKdFIiUwkOEiJNCTHRCQIgAAAAIlcJAxIjUwk +MEiJTCQQSMdEJBgAAAAAx0QkIAAAAADoMcUCAEUPV/9kTIs0Jfj///9Ii2wkQEiDxEjDzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMxIg+xQSIlsJEhIjWwkSEiJRCRYSIkEJMdEJAiBAAAAiVwkDEQP +EXwkEMdEJCAAAAAA6MrEAgBFD1f/ZEyLNCX4////i0QkKIXAfApIi2wkSEiDxFDDRA8RfCQwx0Qk +QAAAAABIjQ1WAAAASIlMJDBIi0wkWEiJTCQ4iUQkQEiNRCQwSIkEJOhVowIARQ9X/2RMizQl+P// +/7gGEAAAxwAGEAAASItsJEhIg8RQw8zMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdnNIg+woSIlsJCBI +jWwkIEhjQhBIiUQkEEiLSghIiUwkGOh1UgAASI0FF6sDALsRAAAA6IRbAABIi0QkGOj6WgAASI0F +x54DALsKAAAA6GlbAABIi0QkEA8fQADoW1kAAOi2VAAA6LFSAABIi2wkIEiDxCjD6EKjAgBmkOl7 +////zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSYnkSYHsqB8AAA+C3QAAAE07ZhAPhtMAAABI +gewoIAAASImsJCAgAABIjawkICAAAEiNfCQguQAEAAAxwPNIq0jHBCQAAAAASMdEJAgAIAAASI1U +JCBIiVQkEA8fRAAA6NvEAgBFD1f/ZEyLNCX4////i1QkGIXSfBVIY8oPHwBIgfkAIAAAd1kxwDHS +6xi4AQAAAEiLrCQgIAAASIHEKCAAAMNI/8BIOch9Bw+2XAQg6yiF0rkBAAAAD0TRidBIi6wkICAA +AEiBxCggAADDid6D4wEB2kDQ7onzhNt18JDrwroAIAAA6HSqAgCQ6M6iAgDpCf///8zMzMzMzMzM +zEk7ZhAPhp0BAABIg+xYSIlsJFBIjWwkUEiJRCRgSIsISItJCEiJTCRISMdEJDgAAAAAkMcEJAIA +AABIjRUPAQkASIlUJAhIjVQkOEiJVCQQx0QkGAgAAADoo74CAEUPV/9kTIs0Jfj///9IjQWX6gMA +hABIi0QkYEiLCEiLFYbqAwDHBCQADwUASItcJEhIiVwkCEiJRCQQSIlMJBhIiVQkIOg5wgIARQ9X +/2RMizQl+P///4tEJCiJRCQ0kMcEJAIAAABIjUwkOEiJTCQISMdEJBAAAAAAx0QkGAgAAACQ6Bu+ +AgBFD1f/ZEyLNCX4////i0QkNIXAfApIi2wkUEiDxFjDSIsFhT0JAEgrBY49CQBIiUQkQOjkTwAA +SI0FH9wDALsuAAAA6PNYAABIi0QkQEhjwOjmVgAASI0FbaUDALsQAAAA6NVYAACLRCQ099hIY8Do +x1YAAEiNBV+UAwC7AgAAAOi2WAAA6BFQAACLRCQ0g/j1dBFIjQVAmgMAuwkAAADoFzcAAOhyTwAA +SI0F7uQDALs9AAAA6IFYAACQ6NtPAADr0UiJRCQI6A+hAgBIi0QkCOlF/v//zMzMzMxMjaQkQPz/ +/007ZhAPhisCAABIgexABAAASImsJDgEAABIjawkOAQAAI1QAesC/8JIY/JIizTzSIX2dfL/wkhj +0kiNBNO7AAAAEEiJ2egOAgAASIXAD4UuAQAASIsNdhIJAEiLFWcSCQBIhckPhsMBAABIiRQkSMdE +JAgAAAAAkOjbuQIARQ9X/2RMizQl+P///4tEJBCFwH1FMcC7AAAEALkDAAAAvyIAAAC+/////0Ux +wOgop/3/SIXbdRJIiYQkMAQAALkAEAAA6doAAABIi6wkOAQAAEiBxEAEAADDiUQkIEiNfCQwSIls +JPBIjWwk8OipqAIASIttAIkEJEiNTCQwSIlMJAjHRCQQAAQAAOjLuQIARQ9X/2RMizQl+P///4tE +JBiJRCQki0wkIIkMJOhquQIARQ9X/2RMizQl+P///4tEJCSFwH0QSIusJDgEAABIgcRABAAAw0jH +hCQgBAAAAAAAAEiNRCQwu4AAAABIidno5wAAAEiLrCQ4BAAASIHEQAQAAMNIi6wkOAQAAEiBxEAE +AADDSItMJChI0eFIi4QkMAQAAEiB+QAABABzT0iJTCQoSI0UCEiJFCRIx0QkCAEAAABIjRUSHAwA +SIlUJBDoh7oCAEUPV/9kTIs0Jfj///+DfCQYAHWsSItMJChIiQ2PHQwASIuEJDAEAABIgz1/HQwA +AHULSMcFch0MAAAABAC7AAAEAA8fRAAA6Dun/f9Ii6wkOAQAAEiBxEAEAADDMcDoJKYCAJCJRCQI +SIlcJBDotZ4CAItEJAhIi1wkEOmn/f//zMzMzMzMzEk7ZhAPhuwAAABIg+wgSIlsJBhIjWwkGEiJ +RCQoSIlcJDAxyeskSInQSInz6K97AgBIi0wkEEiDwQJIi1QkKEiLdCQwSInQSInzSDnLD4aXAAAA +SIsUyEiF0nRrSI1xAUg583Z6SIlMJBBIi3TICEiD+gZ1C0iJNawcDADrqGaQSIP6GXWgSYnwhAZI +xwUCMgkAEAAAAEjHBf8xCQAQAAAAgz34HAwAAHUJSIk13zEJAOsMSI091jEJAOgBogIATInG6V// +//9IicpIwek/SI0EEUjR+EiLbCQYSIPEIMNIifBIidnoFqUCAEiJyEiJ2egLpQIAkEiJRCQISIlc +JBBIiUwkGOiWnQIASItEJAhIi1wkEEiLTCQY6eL+///MzEk7ZhAPhhwBAABIg+xISIlsJEBIjWwk +QEjHRCQsAAAAAEiNVCQwRA8ROkiLDUcPCQBIixU4DwkADx+EAAAAAABIhckPhtUAAABIiRQkSMdE +JAgAAAAA6IW2AgBFD1f/ZEyLNCX4////i0QkEIXAD4yeAAAAiUQkKIkEJEiNTCQsSIlMJAjHRCQQ +FAAAAOjOtgIARQ9X/2RMizQl+P///4tEJBiJRCQki0wkKIkMJOhttgIARQ9X/2RMizQl+P///4tE +JCSFwH4+jUj/SGPZSI1EJCzoiLgBAITbdAkPH0AASIXAfQIxwEiNSP9Ihch0DDHASItsJEBIg8RI +w0iLbCRASIPESMMxwEiLbCRASIPESMMxwEiLbCRASIPESMMxwOi7owIAkOhVnAIA6dD+///MzMzM +zMzMzMzMzMzMzMzMSTtmEHZlSIPsCEiJLCRIjSwk6Gn4//+JBU8ZDAAPHwDom/7//0iJBZQaDACA +PfkYDAAAdC2QiwWM+ggAD7rwH4kFgvoIAJCLBX/6CACJwYPg/okFdPoIAJCD4fyJDWr6CABIiywk +SIPECMPo0JsCAOuOzMzMzMzMzMzMzMzMzMxJO2YQD4Y9AQAASIPsSEiJbCRASI1sJEBIiUQkUEiJ +XCQ4SIlMJGBIixWCLwkASIs1gy8JAEiF0nRCSDnzSInfSA9P/kg50HQkSIl8JChIidNIifno56sC +AEiLRCRQSItMJGBIi1wkOEiLfCQo6G48AQBIi2wkQEiDxEjDSIsVVQ0JAEiLNUYNCQBIhdIPhqkA +AABIiTQkSMdEJAgAAAAA6Hu0AgBFD1f/ZEyLNCX4////i0QkEEiLTCQ4Dx9EAABIhcl2cIlEJCSJ +BCRIi1QkUEiJVCQIiUwkEOjBtAIARQ9X/2RMizQl+P///0hjRCQYSIlEJDCLTCQkiQwkDx8A6Fu0 +AgBFD1f/ZEyLNCX4////SItEJFBIi1wkOEiLTCRgSIt8JDDotTsBAEiLbCRASIPESMMxwOjkoQIA +McBIidHo2qECAJBIiUQkCEiJXCQQSIlMJBjoZZoCAEiLRCQISItcJBBIi0wkGOmR/v//zMzMzMzM +zMzMzMzMzMzMzMxJO2YQdmNIg+wQSIlsJAhIjWwkCEiJRCQYuACAAABmkOgb2AAASItMJBiEAYM9 +DRkMAAB1BkiJQVDrCkiNeVCQ6LucAgCEAIM98hgMAAB1BkiJSDDrCUiNeDDooZ0CAEiLbCQISIPE +EMNIiUQkCOjNmQIASItEJAjrhszMzMzMzEk7ZhB2O0iD7BBIiWwkCEiNbCQI6IdwAQDoQrQCAEUP +V/9kTIs0Jfj///9Ji0YwiwwkSIlISEiLbCQISIPEEJDD6HqZAgDruMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzEiD7EBIiWwkOEiNbCQ4RA8RfCQYRA8RfCQoSMdEJCAEAAAckEjHRCQw/////0iNFeTh +AwCEAkiLFdvhAwBIiVQkKEiNFb/hAwCEAkg5HbbhAwB1LoA98RUMAAB0EkiNFczfAwCEAkiLFcPf +AwDrFkiNNarhAwCEBkiLFaHhAwCQ6wNIidpIiVQkGEiNXCQYMcnoSqL9/0iLbCQ4SIPEQJDDzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEiD7EhIiWwkQEiNbCRAiUQkHEQPEXwkIEQPEXwk +MDHbSI1MJCDo9qH9/0iLVCQoD7riG3MKSItsJEBIg8RIw0gPuuobSIlUJCiLRCQcSI1cJCAxyejH +of3/SItsJEBIg8RIw8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSIPsMEiJbCQoSI1sJCiJ +RCQ4icJIiRQkSIlcJAhIiUwkEEjHRCQYCAAAAOhwtAIARQ9X/2RMizQl+P///4N8JCAAdDCLRCQ4 +g/ggdCeD+CF0IoP4QHQdSI0FouADAEiJBCToeZYCAEUPV/9kTIs0Jfj///9Ii2wkKEiDxDDDzMxJ +O2YQdmBIg+woSIlsJCBIjWwkIEiJRCQwSIlcJBhmkOibsgIARQ9X/2RMizQl+P///0iLRCQwSItA +SEiJRCQISItEJBhIiUQkEOiRsgIARQ9X/2RMizQl+P///0iLbCQgSIPEKMNIiUQkCEiJXCQQ6CuX +AgBIi0QkCEiLXCQQkOl7////zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GggAAAEiD +7CBIiWwkGEiNbCQYSIlMJDhIiVwkMOj5vQEA6NTEAQBIg/sIfBdIjR1AjgMAuQgAAABmkOiblf3/ +hMB1M0yJ8UiFyXQSSItJMEiFyXQJg7nwAAAAAHUKSItsJBhIg8Qgw0iLRCQwSItcJDjoZCwAAEiL +RCQwSItcJDjoVSwAAJBIiUQkCEiJXCQQSIlMJBgPH0QAAOhblgIASItEJAhIi1wkEEiLTCQY6Uf/ +///MzMzMzMzMSTtmEHZBSIPsGEiJbCQQSI1sJBBIiUQkIEyJ8Q8fQABIhcl0EkiLSTBIhcl0CYO5 +8AAAAAB1CkiLbCQQSIPEGMPo2isAAJBIiUQkCEiJXCQQ6OqVAgBIi0QkCEiLXCQQ657MzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdnBIg+w4SIlsJDBIjWwkMEiJRCRASIlcJEi5EgAA +AEiLRCQ4SI0dkp4DAOiM/v//RA8RfCQYZsdEJCgAAEiLVCRASIlUJBjGRCQoAUiLVCRISIlUJCDG +RCQpAEiNBTpeAwBIjVwkGOjQ/P3/6IsjAACQSIlEJAhIiVwkEOg7lQIASItEJAhIi1wkEOls//// +zMzMzMzMzMzMzMzMSTtmEHZwSIPsOEiJbCQwSI1sJDBIiUQkQEiJXCRIuRIAAABIi0QkOEiNHfKd +AwDo7P3//0QPEXwkGGbHRCQoAABIi1QkQEiJVCQYxkQkKABIi1QkSEiJVCQgxkQkKQBIjQWaXQMA +SI1cJBjoMPz9/+jrIgAAkEiJRCQISIlcJBDom5QCAEiLRCQISItcJBDpbP///8zMzMzMzMzMzMzM +zEk7ZhB2cEiD7DhIiWwkMEiNbCQwSIlEJEBIiVwkSLkZAAAASItEJDhIjR3krAMA6Ez9//9EDxF8 +JBhmx0QkKAAASItUJEBIiVQkGMZEJCgBSItUJEhIiVQkIMZEJCkBSI0F+lwDAEiNXCQY6JD7/f/o +SyIAAJBIiUQkCEiJXCQQ6PuTAgBIi0QkCEiLXCQQ6Wz////MzMzMzMzMzMzMzMxJO2YQdnBIg+w4 +SIlsJDBIjWwkMEiJRCRASIlcJEi5GQAAAEiLRCQ4SI0dRKwDAOis/P//RA8RfCQYZsdEJCgAAEiL +VCRASIlUJBjGRCQoAEiLVCRISIlUJCDGRCQpAUiNBVpcAwBIjVwkGOjw+v3/6KshAACQSIlEJAhI +iVwkEOhbkwIASItEJAhIi1wkEOls////zMzMzMzMzMzMzMzMSTtmEHZwSIPsOEiJbCQwSI1sJDBI +iUQkQEiJXCRIuRkAAABIi0QkOEiNHaSrAwDoDPz//0QPEXwkGGbHRCQoAABIi1QkQEiJVCQYxkQk +KAFIi1QkSEiJVCQgxkQkKQJIjQW6WwMASI1cJBjoUPr9/+gLIQAAkEiJRCQISIlcJBDou5ICAEiL +RCQISItcJBDpbP///8zMzMzMzMzMzMzMzEk7ZhB2cEiD7DhIiWwkMEiNbCQwSIlEJEBIiVwkSLkZ +AAAASItEJDhIjR0EqwMA6Gz7//9EDxF8JBhmx0QkKAAASItUJEBIiVQkGMZEJCgASItUJEhIiVQk +IMZEJCkCSI0FGlsDAEiNXCQY6LD5/f/oayAAAJBIiUQkCEiJXCQQ6BuSAgBIi0QkCEiLXCQQ6Wz/ +///MzMzMzMzMzMzMzMxJO2YQdnBIg+w4SIlsJDBIjWwkMEiJRCRASIlcJEi5GQAAAEiLRCQ4SI0d +ZKoDAOjM+v//RA8RfCQYZsdEJCgAAEiLVCRASIlUJBjGRCQoAUiLVCRISIlUJCDGRCQpA0iNBXpa +AwBIjVwkGOgQ+f3/6MsfAACQSIlEJAhIiVwkEOh7kQIASItEJAhIi1wkEOls////zMzMzMzMzMzM +zMzMSTtmEHZwSIPsOEiJbCQwSI1sJDBIiUQkQEiJXCRIuRkAAABIi0QkOEiNHcSpAwDoLPr//0QP +EXwkGGbHRCQoAABIi1QkQEiJVCQYxkQkKABIi1QkSEiJVCQgxkQkKQNIjQXaWQMASI1cJBjocPj9 +/+grHwAAkEiJRCQISIlcJBDo25ACAEiLRCQISItcJBDpbP///8zMzMzMzMzMzMzMzEk7ZhB2cEiD +7DhIiWwkMEiNbCQwSIlEJEBIiVwkSLkZAAAASItEJDhIjR0kqQMA6Iz5//9EDxF8JBhmx0QkKAAA +SItUJEBIiVQkGMZEJCgBSItUJEhIiVQkIMZEJCkESI0FOlkDAEiNXCQY6ND3/f/oix4AAJBIiUQk +CEiJXCQQ6DuQAgBIi0QkCEiLXCQQ6Wz////MzMzMzMzMzMzMzMxJO2YQdnBIg+w4SIlsJDBIjWwk +MEiJRCRASIlcJEi5GQAAAEiLRCQ4SI0dhKgDAOjs+P//RA8RfCQYZsdEJCgAAEiLVCRASIlUJBjG +RCQoAEiLVCRISIlUJCDGRCQpBEiNBZpYAwBIjVwkGOgw9/3/6OsdAACQSIlEJAhIiVwkEOibjwIA +SItEJAhIi1wkEOls////zMzMzMzMzMzMzMzMSTtmEHZESIPsIEiJbCQYSI1sJBhIi0QkIEiNHSqe +AwC5FQAAAOhW+P//SIsVHwAJAEiLHSAACQBIhdJ0BEiLUghIidDodx0AAJDoMY8CAOuvzMzMzMzM +zMzMzMzMzMzMSTtmEHZASIPsGEiJbCQQSI1sJBBIjQW9nwMAuxYAAADou/j//0iLDWT/CABIix1l +/wgASIXJdARIi0kISInIkOgbHQAAkOjVjgIA67PMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GNgIA +AEiD7EhIiWwkQEiNbCRASMdEJCwAAAAASI1UJDBEDxE6McDrC8dEhCz/////SP/ASIP4BXzvMcDr +A0j/wEiD+Ah3BDHS6xBIjVAHSMHqBA8fhAAAAAAASIP6BQ+DDwEAAEiD+Ah3B7tQAAAA6wRIjVhI +Dx9EAABIgfsAgAAAD4ORAAAASIH7+AMAAHdESIPDB0jB6wNmkEiB+4EAAAAPg4UBAABIjTUM7wgA +D7YcMw8fhAAAAAAASIP7RA+DWwEAAEiNPY/vCAAPtxxf6X0AAABIgcN//P//SMHrB0iB+/kAAAAP +gyUBAABIjTUH8AgAD7YcHg8fAEiD+0QPgwABAABIjT1P7wgAD7ccX0iNNaTuCADrOUiNswAgAABI +OfN2EEiNNY/uCABIjT0o7wgA6x2QSIHD/x8AAEiB4wDg//9IjTVw7ggASI09Ce8IAESLRJQsDx9A +AEWFwH0JiVyULOnV/v//RDnDD4TM/v//6wpIi2wkQEiDxEjDSIlUJBhIiVwkEEiJRCQg6Gk7AABI +jQWkogMAuxgAAADoeEQAAEiLRCQg6G5BAABIjQVVgQMAuwUAAABmkOhbRAAASItEJBDoUUEAAEiN +Ba6EAwC7CQAAAA8fRAAA6DtEAABIi0QkGOgxQQAA6Iw9AADohzsAAEiNBQOZAwC7FAAAAOiWIgAA +SInYuUQAAADoCZQCAEiJ2Ln5AAAAkOgblAIASInYuUQAAADo7pMCAEiJ2LmBAAAA6AGUAgCQ6HuM +AgDptv3//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdldIg+wYSIlsJBBIjWwkEEQPETwkSI0F +QO8CAEiJBCRIx0QkCAAAAABIiwQkSItAMIM9JAsMAAB1CUiJBUMeCQDrDEiNPToeCQDozY4CAEiL +bCQQSIPEGMMPHwDo+4sCAOuZzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEiD7GBIiWwkWEiNbCRY +SMdEJBgAAAAASGPYZg8fRAAASIP7CHcEMdvrCEiDwwdIwesEiUQkaGYPH4QAAAAAAJBIg/sFD4Pq +AAAASYt2MEiLttAAAACEBkyNBFtOi0zGcE2FyXVoTI0N5CcJAE2LDNlNhcl0WEiJdCQgTIlEJBBE +DxF8JEBIx0QkUAAAAABIjQWLAgAASIlEJEBIiXQkSEiJXCRQSI1EJEBIiQQk6O6JAgBFD1f/ZEyL +NCX4////i0QkaEiLdCQgTItEJBBKi1zGcE6LTMZoSIXbflhNi0zZ+EyJTCQYSI1L/06LTMZwTotU +xmhJOckPhrYAAABJjTzaSI1/+IM91AkMAAB1C0nHRNr4AAAAAOsMMdsPH0QAAOi7jgIASotUxnhI +Ocpyf0qJTMZwSIN8JBgAdVNIx0QkKAAAAADHRCQwAAAAAEjHRCQ4AAAAAEiNDWMAAABIiUwkKIlE +JDBIjUwkGEiJTCQ4SI1MJChIiQwk6CKJAgBFD1f/ZEyLNCX4////i0QkaEiLTCQYiQFIi0wkGMZB +BQFIi0QkGEiLbCRYSIPEYMPoDZICAEiJyEyJyeiCkQIAkMxJO2YQD4YwAQAASIPsKEiJbCQgSI1s +JCBIY3IISItSEEiD/gh3B75QAAAA6wRIg8ZISIlUJBhIgf4AgAAAc3WQSIH++AMAAHc2SI1GB0jB +6ANIPYEAAAAPg9IAAABIjTW86ggAD7YEMEiD+EQPg7MAAABIjT1H6wgAD7c0R+tVSI2Gf/z//0jB +6AdIPfkAAAAPg4MAAABIjTXD6wgAD7YEMEiD+ERzaEiNPRLrCAAPtzRH6yBIjb4AIAAADx9EAABI +Of53D5BIgcb/HwAASIHmAOD//0iLHW0bCQBIifC5AQAAAOhYCv7/gz0xCAwAAHUKSItMJBhIiQHr +CkiLfCQY6NuLAgBIi2wkIEiDxCjDuUQAAADoZ5ACALn5AAAAZpDoe5ACALlEAAAA6FGQAgC5gQAA +AOhnkAIAkOhBiAIAkOm7/v//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GfAEAAEiD +7HBIiWwkaEiNbCRoSItKEEiJTCRASItSCEiJVCRQkEiNBfYkCQDoSfT9/0iLTCRQSItUJECEAUiD ++gUPgygBAABMjQRSTotMwXhOjRTBTY1SaE6LXMFwSdHpTTnLD43uAAAATI0NuiQJAE2LHNFmDx9E +AABNhdsPhNQAAABNi2Mogz0sBwwAAHUOTYkk0UnHQygAAAAA6xhJjTzRTYnh6HCMAgBJjXsoRTHJ +6GSMAgBOi0zBcEqLXMFoSot8wXhJjXEBSDn3c11MiVQkYEyJXCRYTIlEJEhIjQXW6gIATInJ6E5m +AQBIi1QkSEyLRCRQSYlM0HiDPbgGDAAAdQdJiUTQaOsKSIt8JGDoZYoCAEyJwUmJ0EyLXCRYSYnZ +SInDSItUJEBNjVEBTolUwXBKjTzLgz17BgwAAHUJTokcy+nx/v//TYnY6KiLAgDp5P7//5CQSI0F +wiMJAOj19P3/SItsJGhIg8Rww0iJ0LkFAAAADx8A6LuOAgCQ6JWGAgDpcP7//8zMzMzMzMzMzMzM +zMzMzMxIg+x4SIlsJHBIjWwkcEiJhCSAAAAASIN4IAB0EA8fAOhbAwAASIuEJIAAAABIg3gYAHQN +6IcDAABIi4QkgAAAAIB4BQAPhJgBAABIYxBIg/oIdwQx0usISIPCB0jB6gRIg/oFD4NvAQAATYtG +ME2LgNAAAABMiUQkUEGEAEyNDFJMiUwkQE+LVMhwT40cyE2NW2hMiVwkSE+LZMh4Dx8ATTnUdVdE +DxF8JFhIx0QkaAAAAABIjQVFAQAASIlEJFhMiUQkYEiJVCRoSI1EJFhIiQQk6AiFAgBFD1f/ZEyL +NCX4////SIuEJIAAAABMi0QkUEyLTCRATItcJEjHAAAAAADGQAQAxkAGAEQPEXgISMdAQAAAAABI +x0A4AAAAAIM9+gQMAAB1EkjHQDAAAAAASMdAKAAAAADrFkiNeDAx0ui7iQIASI14KDHS6LCJAgBL +i0zIcEuLXMhoS4t8yHhIjXEBSDn3c0ZIjQWx6AIA6CxkAQBIi1QkQEyLRCRQSYlM0HiDPZYEDAAA +dQdJiUTQaOsKSIt8JEjoQ4gCAEmJ0UiJ2UiJw0iLhCSAAAAASI1RAUuJVMhwSI08y4M9XgQMAAB1 +BkiJBMvrBegRiAIASItsJHBIg8R4w0iLbCRwSIPEeMNIi2wkcEiDxHjDzMzMzMzMzMzMzMzMzMzM +zMzMzEk7ZhAPhmYBAABIg+wwSIlsJChIjWwkKEiLQhBIi1oIMckx0usDTInZhANIg/gFD4MwAQAA +SI00QEyLRPNwTItM82hMi1TzeEnR6k050H59TY1Q/0050A+G/QAAAE+LXMH4S408wUiNf/iDPa8D +DAAAdQtLx0TB+AAAAADrCEUxwOjaiAIATItE83hNOdAPgrsAAABMiVTzcEiF0nQnhAGDPXkDDAAA +dQlMiVko6XT///9IjXkoTIne6IKIAgBmkOlh////TIna6Vn///9IiUQkEEiJVCQgSIlMJBiQkEiN +BaMgCQDo9u/9/0iLTCQYhAFIi1QkEEiNHZMgCQBIizTTgz0YAwwAAHUPSIlxKEiLTCQgSIkM0+sa +SI15KA8fAOgbiAIASI0800iLTCQg6K2HAgCQkEiNBUwgCQAPH0AA6Hvx/f9Ii2wkKEiDxDDDTInR +TInC6KaLAgBMidBMicHoG4sCALkFAAAA6DGLAgCQ6AuDAgDphv7//8zMzMzMzEk7ZhB2IEiD7BhI +iWwkEEiNbCQQSI0FB6cDALseAAAA6FsZAACQ6HWDAgDr08zMzMzMzMzMzMzMzMzMzMzMzMxJO2YQ +diBIg+wYSIlsJBBIjWwkEEiNBfKcAwC7GgAAAOgbGQAAkOg1gwIA69PMzMzMzMzMzMzMzMzMzMzM +zMzMSIPsQEiJbCQ4SI1sJDhJi0YoSIXAD4RDAQAASI1UJEhIOVAID4UqAQAATIl0JBhIiUQkKEyJ +9kiNfihIiXwkMIB4BgAPhb4AAABEiwBFhcB0NUGD+Ah1EEiNSEiEAUiLSEiQSIkK6x9JY8iQSI1Y +SEiJ0OgokwIASItEJChIi3QkGEiLfCQwSItIGIM9jgEMAAB1CkjHQBgAAAAA6xVIjVAYSIn7SInX +RTHA6LCGAgBIid9IiUwkIIQGSItQKIM9WwEMAAB1BkiJVijrBeguhgIA6Cn7//9Ii0wkIIQBSIkM +JEiNTCRISIlMJAjoj4ICAEUPV/9kTIs0Jfj///9Ii2wkOEiDxEDDSInDSInw6A0HAACEwHRRSItM +JBiEAUiLRCQoSItQKIM98gAMAAB1BkiJUSjrD0iLfCQwDx9EAADou4UCAOi2+v//SItsJDhIg8RA +w0iLbCQ4SIPEQMNIi2wkOEiDxEDDSI0F7bkDALsrAAAA6IcXAACQzMzMzMzMSTtmEA+GNAEAAEiD +7FhIiWwkUEiNbCRQScfFAAAAAEyJbCRIxkQkLwBIjRXwyQMASIlUJEjGRCQvAesESItAGEiFwA+E +0AAAAEiLWAhIi0gQSIXbdOZIiUQkQEiJXCQwSIlMJDhIjQXUIQMA6C/p/f9IhcB0PEiLSBhIidj/ +0ZDo++f9/0iNDVQAAwBIi1QkQEiJSgiDPQQADAAAdQZIiUIQ6wlIjXoQ6LODAgBIidDrikiNBQci +AwBIi1wkMEiLTCQ46Njo/f9IhcB1CkiLRCRA6WX///9Ii0gYSInYZpD/0eiZ5/3/SI0N8v8CAEiL +VCRASIlKCIM9ov8LAABmkHUGSIlCEOsJSI16EOhPgwIASInQ6SP////GRCQvAGaQ6HtnAgBIi2wk +UEiDxFjD6Ez9//9Ii2wkUEiDxFjDSIlEJAjoWIACAEiLRCQI6a7+///MzMzMzMzMzMzMzMzMzEk7 +ZhAPht4AAABIg+wYSIlsJBBIjWwkEEiJRCQgSItIGEiFyXQ9SInI6NL///9Ii0wkIEiLURiAejIA +dSMPHwDoOy4AAEiNBeJyAwC7AQAAAOhKNwAA6KUuAABIi0wkIEiJyIB4MgB1degSLgAASI0F7HUD +ALsHAAAA6CE3AACQ6HsuAABIi0QkIEiLSAhIi1gQSInI6CbJ/f9Ii0QkIIB4MAB0IOjWLQAASI0F +O30DALsMAAAA6OU2AAAPH0QAAOg7LgAA6LYtAADoMTAAAOgsLgAASItsJBBIg8QYw0iLbCQQSIPE +GMNIiUQkCOhOfwIASItEJAjpBP///8zMzMxJO2YQD4aEAAAASIPsOEiJbCQwSI1sJDBIhcl0BTHS +kOsMSItQKEiLWkBIi0oISMdEJAgAAAAASI10JBBEDxE+SI10JCBEDxE+SI01cAAAAEiJdCQISIlc +JBBIiUwkGEiJRCQgSIlUJChIjUQkCEiJBCToiX0CAEUPV/9kTIs0Jfj///9Ii2wkMEiDxDjDSIlE +JAhIiVwkEEiJTCQYDx8A6Jt+AgBIi0QkCEiLXCQQSItMJBjpR////8zMzMzMzMxJO2YQdm9Ig+xw +SIlsJGhIjWwkaEiLehhIi1oQTItiIEiLQghEDxF8JFBIx0QkYAAAAABIjRVGAAAASIlUJFBMiWQk +WEiJfCRgSMcEJAAAAAAxyTH2RTHAQbn///9/TI1UJFBFMdvo9fEBAEiLbCRoSIPEcMPoZn0CAOuE +zMzMzEk7ZhAPhs0CAABIg+xISIlsJEBIjWwkQEiLchBIi1IISIXSdBlIi1IISDlQKHUPuAEAAABI +i2wkQEiDxEjDSIlEJFBIiXQkIEiLEIB6KwR3BDHS63JIjXorRItCIEqNPIdIjX8BSYn4QQ+64AJz +VEmJ0A+64gJzRkiJfCQQTIlEJDjouSsAAEiNBbqVAwC7GQAAAOjINAAASItEJDgPHwDoOzQAAOgW +LgAA6BEsAABIi0QkUEiLdCQgSIt8JBCQSIPHBEiLVyBIhdJ0CEiLfigxyesZuAEAAABIi2wkQEiD +xEjDTItHKEiJ+UyJx0iF/3QpTItHCEyLSChNOcFyHJB14YB/BgAPhLkBAAC4AQAAAEiLbCRASIPE +SMNMiwBBg3gQAA+EiwEAAJBIidNFMcBFMcnrBkj/wkyJ4UQPthKQQYD6gHI6TYXJD4xgAQAASYP5 +IEUZ20EPuvIHSYnMTInJQdPiRSHaRQHQTI1JB0mD+Rx+wekjAQAADx+AAAAAAE2FyQ+MDgEAAEiJ +XCQwSIl8JChIiUwkGEmD+SAZ0kyJyUHT4kEh0kONBBDocPD//8ZABgGDPUX7CwAAdQpIx0AgAAAA +AOsLSI14IDHJ6O5/AgBIi0wkUEiLEYtaEEgDGkiJWBBIi1E4SIlQOIM9DfsLAAB1DUiLVCQwSIlQ +MGaQ6w5IjXgwSItUJDDo0H8CAEiLURBIiVBASItJKEiJSAiDPdn6CwAAdQtIi0wkKEiJSCjrEUiN +eChIi0wkKA8fAOh7fwIASItMJBhIhcl0GoM9qvoLAAB1BkiJQSjrLUiNeSjoWX4CAOsigz2Q+gsA +AHULSItMJCBIiUEo6w5Ii0wkIEiNeSjoNX4CADHASItsJEBIg8RIw+jk6///SI0FnfoCAEiNHZbD +BADokQkAAOjM6///SI0F4YUDALsTAAAA6BsRAABIjQWUiwMAuxYAAADoChEAAJBIiUQkCEiJXCQQ +6Hp6AgBIi0QkCEiLXCQQ6Qv9///MzMzMzMzMzMzMzEk7ZhAPhjQIAABIg+x4SIlsJHBIjWwkcEiL +UzAxwGaQ6wZIidBIifKQSI1yAYA6gHIZSIXAD4z9BwAASI1QB5BIg/ocftzp2gcAAEiFwA+MzAcA +ADHAMcnrBYnQTInBD7YWkEj/xoD6gHIqSIXJD4ykBwAASIP5IEUZwA+68gfT4kQhwgHCTI1BB0mD ++Bx+yulwBwAASIXJD4xiBwAASIP5IEUZwJDT4kQhwgHCMcAxyesGRInATInJRA+2BpBI/8ZBgPiA +ci1IhckPjCgHAABIg/kgRRnJQQ+68AdB0+BFIchBAcBMjUkHSYP5HH7E6fEGAABIhckPjOMGAABI +iZwkiAAAAIlUJCBMi0s4SSnRSIP5IEUZ0kUPtglB0+BFIdBBAcBJ/8jrF0yLVCRATY1C/0iJy0iJ +xkGJ0YtUJCCQTYXAD4ysAQAAMcAxyem4AQAASIXJD4xlBgAASIP5IEUZ25BB0+JFIdpBAcIxwDHJ +6d4BAABIhckPjCIGAABIg/kgRRnkkEHT40Uh40EBwzHAMcnpBQIAAA8fhAAAAAAASIXJD4zXBQAA +SIP5IEUZ7UHT5EUh7EEBxA8fgAAAAABFhdIPhaEFAABMiUQkQEmD+CBFGe1MicFBvwEAAABB0+dF +If1FhM11BzHA6XIEAABMi3s4TSnfTYsfSI17GIM9+/cLAAB1BkyJWxjrC02J2OgrfQIASYnIgzsA +dQVFMdvrBZBMjVtIRIlUJDREiWQkLESIbCQfSIl8JGhMiVwkSESITCQeMcDpCAIAAEiJdCRYSItL +OEgp0UH31UUh6USITCQfRIgJSItDIEiJRCRQSItLGEiJy+jlBQAASItMJFBIhcl0CoB5MQAPhUsC +AACDPWr3CwAAdRJIi4wkiAAAAEjHQRgAAAAA6wxIi3wkaDHJ6Ap8AgBIi0QkSItcJDSQ6PuFAgBI +i4wkiAAAAEiLUSBIhdJ0EIB6MABmDx9EAAAPhf8BAABIi0QkWA+2VCQf6TT+//+4AQAAAEiLbCRw +SIPEeMNEidBMidlED7YWkEj/xkGA+oAPgjb+//8PH0AASIXJD4yvBAAASIP5IEUZ20EPuvIHQdPi +RSHaQQHCTI1ZB0mD+xx+vOl4BAAARInYTInhRA+2HpBI/8YPH0QAAEGA+4APggv+//9IhckPjEUE +AABIg/kgRRnkQQ+68wdB0+NFIeNBAcNMjWEHSYP8HH676Q4EAABEieBMielED7YmkEj/xkGA/IAP +gvH9//9IhckPjOADAABIg/kgRRntQQ+69AdB0+RFIexBAcRMjWkHSYP9HH7A6akDAACJRCQwSIl0 +JGBJg/ogGdKJyEyJ0UHT50Eh10KNFDhJjQQTSItTOEgp+kSJwUiJ0+hwhwIAi1QkMI1CAYtUJCxI +i3QkYEiLTCRAi1QkIEiLnCSIAAAASIt8JGhJichED7ZMJB5Ei1QkNEyLXCRIRItkJCxED7ZsJB9E +OeAPg+/9//9FMf8xyUiJTCQ4TInB6Z8AAABIi3wkOEiF/w+MwwEAAEiD/yBFGdKQSIn5QdPgRSHQ +Q408BzHJRTHA6cYAAAAPH0AATYXAD4x3AQAASYP4IEUZ/5BMicFB0+JFIfpEi0QkJEUB0DHJRTHS +6e8AAABNhdIPjQf////pJQEAALgBAAAAkOkV/v//D7ZMJB+EyQ+UwOkG/v//SItMJEBEi1QkNEWJ +x0iJfCQ4SIt8JGhJichED7YGkEj/xmYPH4QAAAAAAJBBgPiAD4JF////SIt8JDhIhf8PjCABAABI +g/8gRRnSQQ+68AdIiflB0+BFIdBFAfhIjXkHSIP/HH6a6eYAAABEidGJTCQkRA+2FpBI/8ZBgPqA +D4Io////Dx+EAAAAAABNhcAPjK8AAABJg/ggRRn/QQ+68gdMicFB0+JFIfpEi3wkJEUB+kyNQQdJ +g/gcfq/rc0SJyUQPtkwkHkQPtj6QSP/GQYD/gA+C//7//02F0nxGiUwkKEmD+iBFGclBD7r3B0yJ +0UHT50Uhz0SLTCQoRQH5TI1RB0mD+hx+tesGkOh75f//SI0FNPQCAEiNHS29BADoKAMAAOhj5f// +Dx8A6Fvl//9IjQUU9AIASI0dDb0EAOgIAwAA6EPl//8PHwDoO+X//0iNBfTzAgBIjR3tvAQA6OgC +AADoI+X////ATInWRDngcy0x/+tASIX/D4zXAAAAMfbrWA8fhAAAAAAASIX2D4ylAAAAMfbrZ0iF +9n3L631IidlIifBEicoPHwDphPr//0iJ90yJ1pBMjVYBgD6AcrZIhf8PjKcAAABIjXcHSIP+HH7d +6YMAAABNidqQTY1aAUGAOoBypQ8fRAAASIX2fGFIg8YHSIP+HH7e60JNidOQTY1TAUGAO4ByjpBI +hfZ8JEiDxgdIg/4cfuLrBehq5P//SI0FI/MCAEiNHRy8BADoFwIAAOhS5P//6E3k//9IjQUG8wIA +SI0d/7sEAOj6AQAA6DXk///oMOT//0iNBenyAgBIjR3iuwQAZpDo2wEAAOgW5P//SI0F7owDALsa +AAAA6GUJAAAPH0QAAOj74///SI0FtPICAEiNHa27BADoqAEAAOjj4///Dx8A6Nvj//9IjQWU8gIA +SI0djbsEAOiIAQAA6MPj//8PHwDou+P//0iNBXTyAgBIjR1tuwQA6GgBAADoo+P//w8fAOib4/// +SI0FVPICAEiNHU27BADoSAEAAOiD4///Dx8A6Hvj//9IjQU08gIASI0dLbsEAOgoAQAA6GPj//8P +HwDoW+P//0iNBRTyAgBIjR0NuwQA6AgBAADoQ+P//5BIiUQkCEiJXCQQ6LNyAgBIi0QkCEiLXCQQ +6aT3///MzMzMSTtmEA+GsQAAAEiD7AhIiSwkSI0sJEiJRCQQDx9EAABIhcB0VUiJXCQY6PEHAACD +PWrxCwAAdQpIi0wkEEiJAesNSIt8JBDoFHUCAEiJ+UiLRCQISIlBIEiNRCQQgz088QsAAHUGSIlB +KOsJSI15KOjrdAIASItcJBhIiwNIidr/0EiLRCQQSIXAdCZIx0AgAAAAAIM9BfELAAB1CkjHQCgA +AAAA6wtIjXgoMcDornQCAEiLLCRIg8QIw0iJRCQISIlcJBDo1nECAEiLRCQISItcJBDpJ////8zM +zMzMzMxMjWQkyE07ZhAPhvcGAABIgey4AAAASImsJLAAAABIjawksAAAAEiJhCTAAAAASImcJMgA +AABJi1YwTIn2Dx8ASDmywAAAAA+FYwYAAIO68AAAAAAPhQEGAABIiXQkcEiDugABAAAAD4UuBQAA +g7oIAQAAAA+FywQAAEjHRCR4AAAAAEiNlCSAAAAARA8ROkiNlCSQAAAARA8ROkiNlCSgAAAARA8R +OkiJhCSAAAAASImcJIgAAABIi1YgSImUJJAAAABIjX4gSIl8JGiDPe3vCwAAdQ1IjVQkeEiJViBm +kOsKSI1UJHjotHQCALoBAAAASI09EO4LAPAPwRdIjYwkwAAAAEiJ8EiLnCS4AAAA6Gzx//9Ii0Qk +cEiLWCgPHwBIhdsPhGUCAABIjXgogHsEAA+EkAAAAEiLSyAPH0QAAEiFyXQExkExAYM9cO8LAAB1 +CkjHQyAAAAAA6xRIjUsgSIn6SInPMfboc3QCAEiJ14B7BgB1UYM9Q+8LAAB1CkjHQxgAAAAA6xRI +jUsYSIn6SInPMfboRnQCAEiJ10iLSyiDPRjvCwAAdQZIiUgo6wXoy3MCAEiJ2Ojj6P//SItMJHDp +Tf///8ZDBAFIjUsggz3q7gsAAHULSI1UJHhIiVMg6xhIifpIic9IjXQkeOjtcwIASInXSI1UJHhI +iXwkYEiJXCRISIlMJFiAewYAdD3oq/T//4TAdC1Ii1QkSEiLciCAfjAAdUiIRCQeSItEJHAx2zHJ +6Ebw//8PtkQkHkiLVCRI6ypIi1QkSOsj6O4EAABIiUQkeJBIi0QkSEiLUBhIiwr/0UiLVCRIuAEA +AABIx0QkeAAAAABIi0wkcGaQSDlRKA+FpQIAAIM9L+4LAAB1CkjHQiAAAAAA6wxIi3wkWDH26Ddz +AgBIi3IQSIl0JCBMi0IITIlEJDCEwHRbgz367QsAAHUKSMdCGAAAAADrC0iNehgx2+jjcgIAiEQk +H0iLWiiDPdTtCwAAdQZIiVko6wpIi3wkYOjCcgIASInQ6Jrn//8PtkQkH0iLTCRwSIt0JCBMi0Qk +MIC8JKgAAAAAD4Ts/f//SIuUJJAAAACDPYntCwAAdQtIiVEgSIt8JGjrCkiLfCRo6FJyAgBIhdJ0 +E4B6MgB0DYB6MQAPHwAPhZYBAAC6/////0iNHZbrCwDwD8ETSItRKITAdAQxwOtFSItaKEiJ0EiJ +2us5SItIIEiJyOiG7P//SItMJHBIi0Eg6FgHAAAxyUjHAQAAAABIi6wksAAAAEiBxLgAAADDSInQ +SInaSIXSD4SpAAAAgHoEAA+FnwAAAIB6BgBmkHUGSItaKOvZSIlEJDhIhcB0J0iLWiiDPcPsCwAA +dQZIiVgo60BMjUgoSYn6TInP6KxxAgBMidfrLEiLWiiDPZzsCwAAdRBIiVkoSIt8JGBMi0wkaOsP +SYn5SIt8JGBmkOh7cQIASItKKEiJTCRASInQ6Erm//9Ii0wkcEiLdCQgSIt8JGhMi0QkMEiLXCRA +SItUJDjpSP///0iLlCSQAAAAgz047AsAAHUGSIlRIOsF6AtxAgBIi1EgSIXSdFKAejEAdBxIi1IY +gz0R7AsAAHUGSIlRIOve6ORwAgDr12aQSIXSdCtMiYEQAQAASImxGAEAAEiNBXa1AwDoIWsCAEiN +BUFwAwC7DwAAAOiwAgAAx4HwAAAAAAAAAOvJSItaKEiJmRABAABIi1IgSImRGAEAAEiNBTe1AwDo +4moCAEiNBTWCAwC7GAAAAOhxAgAASI0F9IEDALsYAAAADx9EAADoWwIAAOi2GgAASI0FkGIDALsH +AAAA6MUjAAAPH0QAAOgbGwAASIuEJMAAAABIi5wkyAAAAOjGtf3/6IEaAACQ6PscAADo9hoAAEiN +BT13AwC7EwAAAOgFAgAADx9EAADoWxoAAEiNBTViAwC7BwAAAOhqIwAA6MUaAABIi4QkwAAAAEiL +nCTIAAAA6HC1/f/oKxoAAOimHAAA6KEaAACQ6BsaAABIjQVXeQMAuxQAAADoKiMAAOiFGgAASItE +JHBIi0AwSIuI+AAAAEiJTCRQSIuAAAEAAEiJRCQoDx9AAOjbGQAASItEJFBIi1wkKOjsIgAA6Eca +AADowhkAAGaQ6DscAADoNhoAAEiNBVF/AwC7FwAAAOhFAQAADx9EAADomxkAAEiNBXVhAwC7BwAA +AOiqIgAA6AUaAABIi4QkwAAAAEiLnCTIAAAA6LC0/f/oaxkAAOjmGwAA6OEZAABIjQUCdgMAuxMA +AADo8AAAAOhLGQAASI0FJWEDALsHAAAA6FoiAADotRkAAEiLhCTAAAAASIucJMgAAAAPH0QAAOhb +tP3/6BYZAADokRsAAOiMGQAASI0FpHkDALsVAAAA6JsAAACQSIlEJAhIiVwkEOiragIASItEJAhI +i1wkEJDp2/j//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEiNRCQIw8zMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMSYtOIEiFyXQhgHkyAHUbgHkwAHUVSIsRSDnQdQ3GQTABSItBCEiLWRDDMcAx +28PMzMzMzMzMzMzMzMzMzMzMzEiD7ChIiWwkIEiNbCQgSIlEJDBIx0QkCAAAAABEDxF8JBBIjQ13 +AAAASIlMJAhIiUQkEEiJXCQYSI1EJAhIiQQk6JpoAgBFD1f/ZEyLNCX4////SYtGMIO49AAAAABm +kHUKx4D0AAAAAQAAAOjvAQAAMcBIxwAAAAAASItsJCBIg8Qow8zMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMxJO2YQdllIg+woSIlsJCBIjWwkIEiLQghIiUQkGEiLShBIiUwkEOiVFwAASI0FJ2oD +ALsNAAAA6KQgAABIi0QkGEiLXCQQ6JUgAADo8BkAAOjrFwAASItsJCBIg8Qow5Doe2gCAOuZzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhhIBAABIg+wwSIlsJChIjWwkKEiLiBABAABIi5AY +AQAASIXJdBtIiUwkEEiLGEiJXCQYSDnLd0UPHwBIOUgIcjxIiUg4SIlQQEjHQGAAAAAASMdAWAEA +AABIg8A4SIkEJOi1ZgIARQ9X/2RMizQl+P///0iLbCQoSIPEMMNIi0AISIlEJCDosBYAAEiNBYxh +AwC7CQAAAA8fQADoux8AAEiLRCQQ6BEeAABIjQVNYAMAuwkAAAAPH0QAAOibHwAASItEJBjo8R0A +AEiNBStbAwC7AgAAAA8fRAAA6HsfAABIi0QkIOjRHQAASI0FG1sDALsCAAAADx9EAADoWx8AAOi2 +FgAASI0Fr2YDALsMAAAA6MX9//+QSIlEJAjo2mcCAEiLRCQI6dD+///MzMzMzMzMzMzMzMzMzMzM +SIPsMEiJbCQoSI1sJChEDxF8JAhEDxF8JBhIjQVfAAAASIlEJAhMifBIiUQkEEiLRCQwSIlEJBhI +jUQkOEiJRCQgSI1EJAhIiQQk6DBmAgBFD1f/ZEyLNCX4////McBIxwAAAAAASItsJChIg8Qww8zM +zMzMzMzMzMzMzMzMzMxJO2YQdm5Ig+w4SIlsJDBIjWwkMEiLQhBIiUQkIEiLShhIiUwkGEiLUghI +iVQkKOjMAQAASItEJChIi1wkIEiLTCQY6FgDAACEwHQF6C84AQDHBCQCAAAA6AOAAgBFD1f/ZEyL +NCX4////SItsJDBIg8Q4w+gnZgIA64XMzMzMzEiD7EhIiWwkQEiNbCRAxkQkDwBIjUwkEEQPETlI +jVQkIEQPETpIjVQkMEQPETpIjRWLAAAASIlUJBBIiUQkGEyJ8EiJRCQgSItEJEhIiUQkKEiNRCRQ +SIlEJDBIjUQkD0iJRCQ4SIkMJOgSZQIARQ9X/2RMizQl+P///4B8JA8AdAuQuAYAAADoMzUBAEiN +BWytAwBIiQQk6ONkAgBFD1f/ZEyLNCX4////McBIxwAAAAAASItsJEBIg8RIw8zMzEk7ZhAPhowA +AABIg+xISIlsJEBIjWwkQEiLQhBIiUQkMEiLShhIiUwkIEiLWiBIiVwkGEiLcihIiXQkOEiLUghI +iVQkKOh2AAAAhMB0H0iLRCQoSIXAdBW5/////0iNFeTiCwDwD8EK6FPl//9Ii0QkMEiLXCQgSItM +JBgPH0AA6NsBAABIi1QkOIgCSItsJEBIg8RIw+jFZAIADx9EAADpW////8zMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzEk7ZhAPhm8BAABIg+wgSIlsJBhIjWwkGEyJdCQQSIM9a+ALAAB1HuhUEwAA +SI0F658DALsuAAAA6GMcAAAPHwDouxMAAEiLTCQQSItRMP+C8AAAAEiLUTCDuggBAAAAkH0Kx4II +AQAAAQAAAEiLSTCLkQwBAACF0g+EoAAAAGaQg/oBdGiD+gJ1PseBDAEAAAMAAADo5xIAAEiNBXJ7 +AwC7GAAAAOj2GwAA6FETAADHBCQEAAAA6KV9AgBFD1f/ZEyLNCX4////xwQkBQAAAOiMfQIARQ9X +/2RMizQl+P///zHASItsJBhIg8Qgw8eBDAEAAAIAAADohBIAAEiNBThvAwC7EwAAAOiTGwAA6O4S +AAAxwEiLbCQYSIPEIMPHgQwBAAABAAAAuQEAAABIjRVM4QsA8A/BCkiNBWniCwCQ6HvP/f+DPQTl +CwAAfwmDPffkCwAAfg64AQAAAA8fQADo29QAAOi2OQAAuAEAAABIi2wkGEiDxCDD6MJjAgBmkOl7 +/v//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GzAIAAEiD7HBIiWwkaEiNbCRoSIlM +JFhIiVwkUEiJRCR4i5DwAAAAhdIPhDUBAACD+kFyCEUxwDH2kOsVSI00UkiNPbPeCABMi0T3EEiL +dPcITYXAdDZMiUQkMEiJdCRg6HURAABIjQVnWgMAuwgAAADohBoAAEiLRCRgSItcJDDodRoAAOjQ +EQAA6yiJVCQs6EURAABIjQU3WgMAuwgAAADoVBoAAItEJCzoqxgAAOimEQAASItEJHhIi4gQAQAA +SIlMJEhIi5AYAQAASIlUJEBIi5ggAQAASIlcJDjo+BAAAEiNBZNXAwC7BgAAAOgHGgAASItEJEhm +kOhbGAAASI0FZFcDALsGAAAA6OoZAABIi0QkQA8fRAAA6DsYAABIjQXwVQMAuwQAAADoyhkAAEiL +RCQ4Dx9EAADoGxgAAEiNBWVVAwC7AgAAAOiqGQAA6AURAABIi0QkeEiLTCRYSItcJFCLFWDACABJ +i3Ywg770AAAAAH4HvwEAAADrCA+64gFAD5LHiVQkKA+2tikBAABBidDB6gJAhPYPRdaF0g+O1gAA +AEiLcDBIOYbAAAAAQQ+VwUEJ+USITCQnSDkGdVyD+gJ9EUmLVjCDuvQAAAAAD46FAAAADx8A6PsP +AABIjQUyZgMAuxAAAADoChkAAOhlEAAASItEJFBIi1wkWDHJSIt8JHjoD/oBAEiLRCR4RItEJChE +D7ZMJCfrP+i4DwAA6DMSAADoLhAAAEiLRCR46GQHAgBIi0QkUEiLXCRYMclIi3wkeOjO+QEASItE +JHhEi0QkKEQPtkwkJ4A9HN4LAAB1FkWEyXQRxgUO3gsAAehkCQIARItEJCiQkEiNBX7fCwDosc79 +/7n/////SI0VRd4LAPAPwQr/yYXJdCCQkEiNBbLeCwDorcz9/5CQSI0FpN4LAA8fQADom8z9/4tE +JCiD4AFIi2wkaEiDxHDDSIlEJAhIiVwkEEiJTCQY6LZgAgBIi0QkCEiLXCQQSItMJBjpAv3//8zM +STtmEHZ3SIPsGEiJbCQQSI1sJBBIhdt0WEiLUDCAuhgBAAAAdRJIiUwkCEiJ2OiNhwEASIXAdA+4 +AQAAAEiLbCQQSIPEGMNIi0QkCOhvhwEASIXAdAwxwEiLbCQQSIPEGMO4AQAAAEiLbCQQSIPEGMMx +wEiLbCQQSIPEGMNIiUQkCEiJXCQQSIlMJBjoD2ACAEiLRCQISItcJBBIi0wkGOlb////zMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMSIPsEEiJbCQISI1sJAjo7YYBAEiFwHQRgHgoAQ+UwEiLbCQI +SIPEEMMxwEiLbCQISIPEEMPMzMzMzMzMzMzMzEk7ZhgPhlMFAABIg+xoSIlsJGBIjWwkYEmLVjBI +i5LAAAAASIXSdA+LkpAAAACD+gIPhBQFAABIiUQkcDHJMdIx2zH2Mf9FMcDrIUyLTCQ4SY15AUiL +RCRwSIt0JDCLXCQgSItUJEgPtkwkFkyJRCQoSIl0JDCJXCQgSIlUJEiITCQWSIl8JDhEi4iQAAAA +QYP5BA+H5wEAAA8fgAAAAABBg/kBD4RIAgAAQYP5Ag+FwQEAAIC4sgAAAAB0OIC4sQAAAAB0KkiB +eBDe+v//dRtIOVAwdRBEi4pQAwAAQTnZQQ+UwesaRTHJ6xVFMcnrEEUxyesLRTHJDx+EAAAAAABF +hMkPhScCAAC7AgAAALkCEAAA6Ag5AAAPH4QAAAAAAITAD4QmAQAASItEJHBmx4CxAAAAAQFIx0AQ +3vr//0iLUDBIiVQkWIuyUAMAAIl0JCRIi3wkSEg513QHvwEAAADrCot8JCA5/kAPlcdAiHwkF7sC +EAAAuQIAAAAPH0QAAOjbNAAAgz0s3wsAAA+FjQAAAA+2RCQXhMAPhIAAAACQ6Ht5AgBFD1f/ZEyL +NCX4////SIsEJEiLTCQwDx9EAABIOcF/S5BIicExwEiLVCRYvgEAAADwD7GyVAMAAEAPlMdAhP90 +IUiJTCRASInQuxcAAADoCMb//0iLTCRASItUJFi+AQAAAEiNgYgTAADrHkiLVCRYvgEAAABIicjr +D0iLVCRYvgEAAABIi0QkMEiLfCQ4TItEJCgPtkwkFotcJCRIicZIi0QkcOniAAAASItEJHBIi3wk +OEyLRCQoD7ZMJBZIi1QkSItcJCBIi3QkMOm7AAAAQYP5A3If63UPHwBBg/kGD4SIAQAAQYP5CA+E +nAAAAEGD+Ql0EUEPuuEMkA+CigAAAOl6AQAAuwkAAAC5BAAAAOjmPAAAhMB1JEiLRCRwSIt8JDhM +i0QkKA+2TCQWSItUJEiLXCQgSIt0JDDrTkiLRCRwQbkEAAAAuQEAAACITCQWRInLQQ+66QxEicno +GjcAAITAD4XXAAAASItEJHBIi3wkOEyLRCQoD7ZMJBZIi1QkSItcJCBIi3QkMEiJdCQwiVwkIEiJ +VCRIiEwkFkiF/3Ud6NR3AgBFD1f/ZEyLNCX4////SIsEJEyNgBAnAABMiUQkKOiydwIARQ9X/2RM +izQl+P///0iLRCQoSDkEJH0jxwQkCgAAAOguXAIARQ9X/2RMizQl+P///0yLRCQo6aL8///oUn0C +AEUPV/9kTIs0Jfj///8PH0QAAOhbdwIARQ9X/2RMizQl+P///0iLBCRMjYCIEwAADx8A6Wb8//9I +i0QkcGbHgLEAAAAAAEiLEEiBwqADAABIiVAQMdsPtkwkFkiLbCRgSIPEaMMxwLsBAAAAMclIi2wk +YEiDxGjDkEyJdCRQi4iQAAAAiUwkGEiLkJgAAABIiVQkQOiaCQAASI0FIWEDALsQAAAA6KkSAABI +i0QkcA8fQADoGxIAAEiNBexQAwC7BwAAAOiKEgAASItEJEAPH0QAAOh7EAAASI0FuWQDALsTAAAA +6GoSAACLRCQYicAPH0AA6FsPAADotgsAAOixCQAASItEJFCLiJAAAACJTCQcSIuQmAAAAEiJVCRA +6BEJAABIjQVoYAMAuxAAAAAPH0QAAOgbEgAASItEJFDokREAAEiNBWJQAwC7BwAAAA8fRAAA6PsR +AABIi0QkQOjxDwAASI0FHGQDALsTAAAADx9EAADo2xEAAItEJByJwOjQDgAA6CsLAADoJgkAAEiN +BW1fAwC7EAAAAOg18P//SI0FW40DALsnAAAA6CTw//+QSIlEJAjoeYECAEiLRCQI6Y/6///MzMzM +zMzMzMzMzMzMzMxJO2YQD4aiAQAASIPsSEiJbCRASI1sJEBIiUQkUA8fAITbdVSITCQfSIlEJDiQ +i5CQAAAAgfoBEAAAdAuNsv3v//+D/gF3O4nRD7rxDInT6IwwAAAPtlQkH4TSdBFIi0QkODHbuQEA +AADoci0AAEiLbCRASIPESMNIi2wkQEiDxEjDkEyJdCQwi4iQAAAAiUwkIEiLkJgAAABIiVQkKGaQ +6LsHAABIjQVCXwMAuxAAAADoyhAAAEiLRCQ4Dx9EAADoOxAAAEiNBQxPAwC7BwAAAOiqEAAASItE +JCgPH0QAAOibDgAASI0F2WIDALsTAAAA6IoQAACLRCQgicAPH0AA6HsNAADo1gkAAOjRBwAASItE +JDCLiJAAAACJTCQkSIuQmAAAAEiJVCQo6DEHAABIjQWIXgMAuxAAAAAPH0QAAOg7EAAASItEJDDo +sQ8AAEiNBYJOAwC7BwAAAA8fRAAA6BsQAABIi0QkKOgRDgAASI0FPGIDALsTAAAADx9EAADo+w8A +AItEJCSJwOjwDAAA6EsJAADoRgcAAEiNBThkAwC7EwAAAOhV7v//kEiJRCQIiFwkEIhMJBHoYlgC +AEiLRCQID7ZcJBAPtkwkEeku/v//zMzMzMzMzMzMzMzMzMxIg+wYSIlsJBBIjWwkEEyJdCQIQcaG +tAAAAAFBgL6yAAAAAHQOSI0FjKADAOhPVgIA6wxIjQXGnwMA6EFWAgBIi0QkCMaAtAAAAABIi2wk +EEiDxBjDzMzMzMzMzMzMzMxJO2YQD4ayAAAASIPsIEiJbCQYSI1sJBhIjQ1ZngMAhAFIiwVQngMA +6NN+AQDorosBAIlEJBRIjQ0zngMAhAFIiw0qngMASInI6LJ+AQDojYsBAItMJBQBwUhjyUiDwUBI +iQ3BtQgASIH5IAMAAHcKSItsJBhIg8Qgw+ihBQAASI0Fa3MDALsbAAAA6LAOAABIiwWRtQgA6KQL +AAAPH0AA6PsHAADo9gUAAEiNBaVlAwC7FQAAAOgF7f//kA8fQADoG1cCAOk2////zMzMzMzMzMzM +zMzMzMzMzMzMzMzMzEk7ZhAPhnYDAABIg+xwSIlsJGhIjWwkaEiLUDAPH0AASDmCwAAAAA+FOgMA +AEiLstAAAABIhfYPhBwDAACDuggBAAAAD4UPAwAAg7rwAAAAAA+FAgMAAEiDugABAAAAkA+F8wIA +AIN+BAEPhekCAABIixBIOdFyEUgp0Q8fRAAASDkNwbQIAHYOMcAx20iLbCRoSIPEcMNIiZwkgAAA +AEiJ2Oh5fQEASIXAD4SbAgAASIlcJGBIiUQkWDHJSIu8JIAAAADo14sBAIP4/w+FawIAAEiLRCRY +SI1IK0iJTCRQgHgrAXcKSInCSInOMcnrcItQIEiJzkiNDJFIjUkBSInKD7riAnNNSInCD7rgAnM+ +SIlMJDjoJgQAAEiNBSduAwC7GQAAAOg1DQAASItEJFjoqwwAAOiGBgAA6IEEAABIi0wkOEiLVCRY +SIt0JFBIg8EE6wNIicJIi0kIDx9EAABIhckPhMYBAABIjT2gnAQASDn5D4S2AQAASInQSItcJGDo +eoMBAEiJRCRISIlcJChIi0wkWIB5KwN3B0mJyDHS63iLUSBMi0QkUEmNFJBIjVIBSYnQQQ+64AJz +V0mJyA+64QJzSEiJVCQw6HEDAABIjQVybQMAuxkAAAAPH0QAAOh7DAAASItEJFjo8QsAAOjMBQAA +6McDAABIi0QkSEiLVCQwSItcJChMi0QkWEiDwgTrA0mJyEiLUhhIhdJ0YUiJVCRATInASItcJGC5 +AgAAAEiLvCSAAAAAMfYPH0QAAOi7iQEAhcB8LUhjwEg9AAAQAA+DHQEAAEiNFIBIi3QkQItMlgxI +i0QkWEiLXCRg6OyDAQDrCkiLXCQoSItEJEhIiVwkKEiJRCRISIP7CH0EMcnrHUiNHd5LAwC5CAAA +AOg7U/3/SItcJCiJwUiLRCRIhMl0B7kBAAAA6ysPH0AASIP7EX0EMcnrHUiNHY9bAwC5EQAAAOgF +U/3/SItcJCiJwUiLRCRIhMl1H0iD+wh9BDHA6xFIjR1tSwMAuQgAAADo2lL9/4TAdA4xwDHbSIts +JGhIg8Rww7gBAAAASIucJIAAAABIi2wkaEiDxHDDMcAx20iLbCRoSIPEcMMxwDHbSItsJGhIg8Rw +wzHAMdtIi2wkaEiDxHDDMcAx20iLbCRoSIPEcMMxwDHbSItsJGhIg8Rww7kAABAA6OFaAgCQSIlE +JAhIiVwkEEiJTCQYSIl8JCDoZ1MCAEiLRCQISItcJBBIi0wkGEiLfCQg6U78///MzMzMzMzMzMzM +zMzMzEk7ZhAPhjwBAABIg+wwSIlsJChIjWwkKEiJRCQ4SIlcJEBIiUwkSOhUAQAAiwVO0AsAhcB1 +E0iLTCRASItUJEhIi3QkODHA60DosgEAAEiLbCQoSIPEMMNMiwVx0QsATQHITYnEScH4P0nB6DdN +AeBJwfgJScHgCU0pxEyJJU7RCwBLjQQZSIn5SDnBfrtIiz070QsAZg8fhAAAAAAAZpBIgf8AAgAA +D4eLAAAATI2HAP7//02JwUnB+D9JIfhmkEg5wXJuSffZSInPSCnBSTnJTA9PyUyNFUTcCwBNAdBJ +icNIKdBIwfg/TCHYSI0cBkw5ww+EX////0yJXCQgTIlMJBhMicBMicnosmICAEiLVCRISIt0JDhI +i3wkQEyLTCQYTI0V99sLAEyLXCQg6SX////oKFoCAEiJ+LkAAgAA6BtaAgCQSIlEJAhIiVwkEEiJ +TCQY6OZRAgBIi0QkCEiLXCQQSItMJBjpkv7//8zMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2VEiD7BhI +iWwkEEiNbCQQSYtOMP+BCAEAAA+2kRcBAACNWgGImRcBAACE0nUYSIlMJAiQkEiNBUjPCwDoO739 +/0iLTCQI/4kIAQAASItsJBBIg8QYw+hhUQIAkOuezMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMSTtmEHY/SIPsEEiJbCQISI1sJAhJi04wD7aRFwEAAI1a/4iZFwEAAID6AXUOkJBIjQXSzgsA +6KW+/f9Ii2wkCEiDxBDD6PZQAgDrtMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GIAEAAEiD7DhI +iWwkMEiNbCQwSIlEJEAPHwBIhdsPhO0AAABIiVwkSEiJRCRA6Gj9//9MifIPH0QAAEiF0g+ElAAA +AEiLsvgAAABIi7oAAQAASIuKCAEAAGaQSIX2dHhMi0IwQYO4DAEAAAB/akgp+UyLRCRISTnISYnJ +SQ9MyEn32UnB+T9MIc9IjQQ+SItcJEBIOcN0GUiJVCQoSIlMJCDotGACAEiLTCQgSItUJChIi4II +AQAASIuaAAEAAEgB2Ug5yHJVSImKAAEAAEiLbCQwSIPEOMOQkEjHBCQCAAAASItEJEBIiUQkCEiL +RCRIiUQkEOihaQIARQ9X/2RMizQl+P///0iLbCQwSIPEOMNIi2wkMEiDxDiQw0iJwuiXVwIAkEiJ +RCQISIlcJBBIiUwkGOiiTwIASItEJAhIi1wkEEiLTCQY6a7+///MzMzMzMzMzMzMzMzMzEk7ZhB2 +KUiD7BhIiWwkEEiNbCQQSI0FRUIDALsBAAAA6LsGAABIi2wkEEiDxBjD6ExPAgDryszMzMzMzMzM +zMxJO2YQdilIg+wYSIlsJBBIjWwkEEiNBTWVAwC7AQAAAOh7BgAASItsJBBIg8QYw+gMTwIA68rM +zMzMzMzMzMzMSTtmEHZASIPsGEiJbCQQSI1sJBCEwHQTSI0F5UIDALsEAAAA6DcGAADrEUiNBURD +AwC7BQAAAOgkBgAASItsJBBIg8QYw4hEJAjosU4CAA+2RCQI66rMzMzMzMzMzMzMSTtmEA+GMwIA +AEiD7DBIiWwkKEiNbCQoZg8uwA8fQAAPhcUAAAAPir8AAAAPEMjyD1jAZg8uyHVWelQPV9JmDy7K +dytmDy7IdUh6RmYPLtF2QEiNBdtBAwC7BAAAAGaQ6JsFAABIi2wkKEiDxDDDSI0FukEDALsEAAAA +Dx9EAADoewUAAEiLbCQoSIPEMMMPV9JIx0QkGgAAAABIx0QkIAAAAADGRCQaK2YPLtF1IHoe8g8Q +BXeUBADyD17BZg8u0HYFxkQkGi0xwOmQAAAAdhPGRCQaLfIPEAWylAQAZg/vwesDDxDBMcDrIkiN +BehAAwC7AwAAAOgEBQAASItsJChIg8Qww0j/wPIPXsHyDxANO5QEAGYPLsFz6+sHSP/I8g9ZwfIP +EBUElAQAZg8u0HfrMcnyDxAVDJQEAOsHSP/B8g9e0UiD+Qd88/IPWNBmDy7RcgdI/8DyD17RDxDK +McnrKfJIDyzRSI1yMECIdAwcSP/BD1fA8kgPKsLyD1zI8g8QBciTBADyD1nISIP5B3zRD7ZUJByI +VCQbxkQkHC5mx0QkI2UrSIXAfQjGRCQkLUj32EiJwUi4C9ejcD0K16NI9+lIAcpIwfoGSInOSMH5 +P0gpyoPCMIhUJCVIuM3MzMzMzMzMSPfuSAHySMH6A0gpyg+2+mn/mgEAAMHvDI08v9HnSYnQKfqD +wjCIVCQmS40UgEjR4kgp1o1WMIhUJCdIjUQkGrsOAAAASInZ6I37//9Ii2wkKEiDxDDD8g8RRCQI +6FhMAgDyDxBEJAjprf3//8zMzMzMzMzMzMzMzMxJO2YQdm5Ig+wYSIlsJBBIjWwkEPIPEUQkIPIP +EUwkKOhb+v//SI0F9T4DALsBAAAA6GoDAADyDxBEJCAPH0AA6Fv9///yDxBEJCjoUP3//0iNBfw+ +AwC7AgAAAA8fQADoOwMAAOiW+v//SItsJBBIg8QYw/IPEUQkCPIPEUwkEOi7SwIA8g8QRCQI8g8Q +TCQQ6Wr////MzMzMzMzMzMzMTI1kJPhNO2YQD4bOAAAASIHsiAAAAEiJrCSAAAAASI2sJIAAAABI +x0QkHAAAAABIjXwkIEiNf+APH4QAAAAAAEiJbCTwSI1sJPDoPFUCAEiLbQC5YwAAAOsGSP/JSInQ +SIXJfjNIicJIuM3MzMzMzMzMSInTSPfiSMHqA0iNNJJI0eZIid9IKfNIjXMwQIh0DBxIg/8Kc8JI +g/lkdzFIjVmcSInaSMH7P0gh2UiNRAwcSPfaSInTSInZ6AH6//9Ii6wkgAAAAEiBxIgAAADDSInI +uWQAAADo5FICAJBIiUQkCOi5SgIASItEJAjpD////8zMzMzMzMzMzMzMzMzMzEk7ZhB2QEiD7BhI +iWwkEEiNbCQQSIXAfR5IiUQkIEiNBWA9AwC7AQAAAOjRAQAASItEJCBI99joxP7//0iLbCQQSIPE +GMNIiUQkCOhQSgIASItEJAjrqczMzMzMzMzMzEyNZCT4TTtmEA+G7gAAAEiB7IgAAABIiawkgAAA +AEiNrCSAAAAASMdEJBwAAAAASI18JCBIjX/gDx+EAAAAAABIiWwk8EiNbCTw6NxTAgBIi20AuWMA +AADrCkj/yUjB6gRIidBIhcl+LUiJwkiD4A9IjTX3TQMAD7Y0BkCIdAwcSIP6EHPUSI1xnEj33kg5 +NeTHCwB/xEiNQf9mDx9EAABIg/hkc0zGRAwbeEiNQf5Ig/hkczPGRAwaMEiDwZpIictIwfk/SCHI +SI1EBBxI99tIidnoiPj//0iLrCSAAAAASIHEiAAAAMO5ZAAAAOiuUAIAuWQAAADopFACAJBIiUQk +COg5SQIASItEJAjp7/7//8zMzMzMzMzMzMzMzMzMzEk7ZhB2HUiD7BBIiWwkCEiNbCQI6Mf+//9I +i2wkCEiDxBDDSIlEJAjo80gCAEiLRCQI68zMzMzMzMzMzMzMzMxJO2YQdh1Ig+wQSIlsJAhIjWwk +COiH/v//SItsJAhIg8QQw0iJRCQI6LNIAgBIi0QkCOvMzMzMzMzMzMzMzMzMSTtmEHZeSIPsSEiJ +bCRASI1sJEBIiUQkUEiJRCQYSIlcJCBIx0QkKAAAAABEDxF8JDBIi1QkGEiJVCQoSItcJCBIiVwk +MEiLTCQgSIlMJDhIi0QkKOhm9///SItsJEBIg8RIw0iJRCQISIlcJBDoLUgCAEiLRCQISItcJBDr +gcxJO2YQD4aUAAAASIPsKEiJbCQgSI1sJCBIiVwkGEiJTCQQSIlEJDBIiVwkOEiJTCRA6Cr2//9I +jQXQOgMAuwEAAADoOf///0iLRCQY6C/9//9IjQWvOgMAuwEAAAAPHwDoG////0iLRCQQ6BH9//9I +jQWZOgMAuwEAAAAPH0QAAOj7/v//6Fb2//9Ii0QkMOhM/f//SItsJCBIg8Qow0iJRCQISIlcJBBI +iUwkGOhuRwIASItEJAhIi1wkEEiLTCQY6Tr////MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7 +ZhAPhvQBAABIg+xYSIlsJFBIjWwkUEiJRCRgSIlcJGhIiUwkcOhU9f//xkQkHwDGRCQfIEjHBTfF +CwAQAAAAMcDrCUiLdCQoSI1GCEiLdCRgSI08MEyLRCRoDx9AAEk5+A+GaQEAAEiJRCQoSIl8JDhI +qQ8AAAB1VA8fRAAASIXAdA/o9vT//+hx9///6Gz1///o5/T//0iLRCQ4ZpDoW/z//0iNBZk5AwC7 +AgAAAOjq/f//6EX1//9Ii0QkKEiLdCRgSIt8JDhMi0QkaEiLVCRwSIXSdBlIiwpIifiQ/9GIRCQf +D7bIhMl1BcZEJB8gSI1EJB+7AQAAAEiJ2ZDoW/X//0iLVCQ4SIsSSIlUJCDoafT//0iLRCQgDx9A +AOjb+///6Nb0///oUfT//+iM9v//6Mf0//9Ii0QkIGaQ6BttAQBIhcAPhPf+//9IiUQkSOjocwEA +SIlEJEBIiVwkMEiLTCRISIsJSIlMJDjoDPT//0iNBa44AwC7AQAAAOgb/f//SItEJEBIi1wkMOgM +/f//SI0FiDgDALsBAAAA6Pv8//9Ii0QkIEiLTCQ4SCnI6En7//9IjQWJOAMAuwIAAADo2Pz//+gz +9P//6XP+//9IxwWbwwsAAAAAAA8fAOib8///6Bb2///oEfT//+gM9P//SItsJFBIg8RYw0iJRCQI +SIlcJBBIiUwkGOguRQIASItEJAhIi1wkEEiLTCQY6dr9///MzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzEk7ZhAPhjIDAABIg+xYSIlsJFBIjWwkUEnHxQAAAABMiWwkSMZEJCcATIl0JDBJi0YwSIsA +SMeAQAEAAAAAAABIxwUtowgAAMqaO0jHBRqjCAAAlDV3xgWSwQsAAbgBAAAASI0NV+EIAIcBSI0F +dowDAEiJBCToRUMCAEUPV/9kTIs0Jfj///9Ji0YwkP+AdAIAAEmLRjCQTInxSImIaAEAAEmLRjBJ +iYboAAAASItEJDBIi0AwSI0N49sIAA8fAEg5yA+FUgIAAMYFGN0IAAHoq18CAEUPV/9kTIs0Jfj/ +//9IiwQkSIkFu8ILAEiFwA+EFAIAAIM9J8ULAAB0FUmLjpgAAABIiQ1bwwsAxgVMwwsAAUiNBYWk +CAAPH0QAAOgbzAAAxkQkJgFEDxF8JDhIjQ0pAgAASIlMJDhIjUwkJkiJTCRASI1MJDhIiUwkSMZE +JCcB6CZG/v9IjQUfvQIAMdvoWFr9/4M9kcILAAB1CUiJBcjVCADrDEiNPb/VCADoOkYCAIA9X8AL +AAB0S0iDPTnVCAAAD4RcAQAASIM9O9UIAAAPhD0BAABIgz011QgAAA+EHgEAAEiDPf/UCAAAD4T/ +AAAA6JQ7AABIiwXt1AgAMdvohk79/0iNBd+hCADoWssAAMYFc8ILAABIiwVM1QgA6Kdk/f/GRCQm +AGaQ6FuPAACAPd+/CwAAdSeAPdi/CwAAdR5IiwXSigMASI0Vy4oDAP/QiwUbwAsAhcB0HTHA63nG +RCQnAEiLVCRISIsC/9BIi2wkUEiDxFjDixXsvwsAhdJ0GDHAMdu5CAAAAL8QAAAAvgEAAADosAIA +AMcEJAAAAADopFsCAEUPV/9kTIs0Jfj///8xwMcAAAAAAOv2SIlEJCiQSI0FKIoDAOibQAIASItM +JChIjUEBSD3oAwAAfZiLFYy/CwAPH0AAhdJ1z+uISI0FSnIDALslAAAA6AnY//9IjQWDUAMAuxUA +AADo+Nf//0iNBQJMAwC7EwAAAOjn1///SI0FHFkDALsZAAAA6NbX//9IjQW6VQMAuxcAAADoxdf/ +/0iNBQRTAwC7FgAAAOi01///kOiuvv//SItsJFBIg8RYww8fQADou0ECAOm2/P//zMzMzMzMzMzM +zMzMzMzMzMzMzMzMzEk7ZhB2KEiD7AhIiSwkSI0sJEiLQgiAOAB0Cg8fRAAA6NuNAABIiywkSIPE +CMPozUACAOvLzMzMzMzMzMzMzMxJO2YQdilIg+wYSIlsJBBIjWwkEDHASI0di4gDAA8fAOibgAAA +SItsJBBIg8QYw+gsQQIA68rMzMzMzMzMzMzMSTtmEA+G3gAAAEiD7ChIiWwkIEiNbCQggz0BwAsA +AJB1DEyJ8UiJDYzUCADrN0iNPYPUCABMifHoo0QCAOsmkOhbXAIARQ9X/2RMizQl+P///0iLHCS4 +AQAAADHJDx8A6NtE/v+QSI0FQ9QIAOhurP3/gz1H1AgAAHVbugEAAABMjQU51AgAQYcQkEiNBe6I +AwBIjR0X1AgAuREAAAC/FAAAAL4BAAAA6JMAAACDPWjBCwAAfonope7//0iNBZ47AwC7CgAAAOi0 +9///6A/v///paf///0iNBSZNAwC7FAAAAOgZ1v//kOgzQAIA6Q7////MzMzMzMzMzMzMzMzMzEiD +7BBIiWwkCEiNbCQISI0F04cDAOhGPgIASItsJAhIg8QQw8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMxJO2YQD4bpAAAASIPsGEiJbCQQSI1sJBBJi1YwkP+CCAEAAEmLVjBNifBBhABMi4LAAAAA +kEWLiJAAAAAPH0AAQYP5AnQNQYH5AhAAAA+FjgAAAIM9hr4LAAB1EEiJmogCAABIiYKAAgAA6yFM +jYqIAgAAQYn6TInP6GJDAgBIjbqAAgAA6BZCAgBEiddBiIiwAAAAQIi6kAIAAEiJspgCAACQi4oI +AQAAjVn/iZoIAQAAg/kBdRNBgL6xAAAAAJB0CEnHRhDe+v//SI0FZ4cDAOhKPQIASItsJBBIg8QY +kMNIjQUDTAMAuxQAAADoztT//5BIiUQkCEiJXCQQiEwkGECIfCQZSIl0JCDo0D4CAEiLRCQISItc +JBAPtkwkGA+2fCQZSIt0JCDp0v7//8zMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2WEiD7ChIiWwkIEiN +bCQgRA8RfCQISMdEJBgAAAAASI0NVgAAAEiJTCQISIlEJBBIiVwkGEiNRCQISIkEJOgZPQIARQ9X +/2RMizQl+P///0iLbCQgSIPEKMNIiUQkCEiJXCQQ6DM+AgBIi0QkCEiLXCQQ64fMzMzMzMzMSTtm +EHYqSIPsIEiJbCQYSI1sJBhIi1oQSItCCLkBAAAA6LoRAABIi2wkGEiDxCDD6Es9AgDryczMzMzM +zMzMzEiD7GhIiWwkYEiNbCRgSYt2MJD/hggBAABJi3YwTIuG0AAAAE2J8UGEAUmDuBgOAAAAdTFI +iXQkQEyJRCRYkEiNBfLZCADoVan9/0iLTCRYSI2REA4AAEiJVCRQDx9AAOnAAAAASYuIGA4AAE2L +iBAOAABIjUH/Zg8fhAAAAAAASDnBD4aXAAAATYtUyfhJjTzJSI1/+IM9Q7wLAAB1C0nHRMn4AAAA +AOsIRTHJ6I5BAgBJi5AgDgAASDnCcldJiYAYDgAASYN6GAB1OJCLjggBAACNUf+JlggBAAAPH0AA +g/kBdRJBgL6xAAAAAHQIScdGEN76//9MidBIi2wkYEiDxGjDSI0FknIDALsqAAAA6KvS//9IicHo +o0QCAA8fAOgbRAIATIuBIA4AAEnR6Ew5gRgOAAAPjekAAABMiwX92AgATYXAD4TZAAAATYtICIM9 +ibsLAAB1EUyJDeDYCABJx0AIAAAAAOsbSI09z9gIAOjKQAIASY14CEUxyQ8fAOi7QAIATIuJGA4A +AEmNcQFIi5kQDgAASIu5IA4AAGaQSDn3c1JMiUQkSEiNBS+pAgBMicnopxoBAEiLVCRYSImKIA4A +AIM9FLsLAAB1DkiJghAOAABIi3wkUOsKSIt8JFDouj4CAEiJ0UiJ+kyLRCRISYnZSInDTY1RAUyJ +kRgOAABKjTzLgz3TugsAAHUJTokEy+kN////6ANAAgAPHwDpAP///5CQSI0FCtgIAOhNqf3/SItM +JFhIg7kYDgAAAA+FnwAAAEiNBVMUAwDojsb9/0iLTCRYSIuRGA4AAEiLmRAOAABIi7kgDgAASI1y +AUg593NKSIlEJEhIjQVfqAIASInR6NcZAQBIi1QkWEiJiiAOAACDPUS6CwAAdQlIiYIQDgAA6wpI +i3wkUOjvPQIASInRSInaSInDSItEJEhIjXIBSImxGA4AAEiNPNODPQu6CwAAdQlIiQTT6wgPHwDo +uz0CAEiLdCRASYnI6XP9///MzMzMzMzMzMzMzMzMzEiD7GhIiWwkYEiNbCRgSIN4GAAPhQ4DAACA +eDQADx8AD4XwAgAASIN4CAAPhdQCAABIg3gQAA+FswIAAEiDeEAAD4WXAgAASIN4UAAPhXsCAABJ +g76IAAAAAGYPH0QAAA+FVgIAAEiJRCRwTYtGMJBB/4AIAQAATYtGMEyJRCRATYuI0AAAAEyJTCRY +TYnyQYQCTYuRIA4AAA8fgAAAAABNOZEYDgAAdQkxyTHS6cAAAABJi4kYDgAASI1xAUmLmRAOAABJ +i7kgDgAASDn3c0lIjQUJpwIA6IQYAQBIi1QkWEiJiiAOAACDPfG4CwAAdQlIiYIQDgAA6wxIjboQ +DgAA6Jo8AgBMi0QkQEmJ0UiJ2UiJw0iLRCRwSI1RAUmJkRgOAABIjTzLgz2xuAsAAHUGSIkEy+sF +6GQ8AgCQQYuACAEAAI1I/0GJiAgBAACD+AF1EkGAvrEAAAAAdAhJx0YQ3vr//0iLbCRgSIPEaMNM +idlJi5kQDgAASYuxIA4AAE2LkRgOAABI0e5JOfIPjpIAAABJjXL/Zg8fhAAAAAAASTnyD4YIAQAA +Totc0/hKjTzTSI1/+IM9I7gLAAB1C0rHRNP4AAAAAOsHMdvoDz0CAEmLmSAOAAAPH4QAAAAAAEg5 +8w+CvQAAAEmJsRgOAABIhdJ0J4QBgz3itwsAAGaQdQlMiVkI6WT///9IjXkITInb6Mk8AgDpU/// +/0yJ2pDpSv///0iJVCRQSIlMJEiQkEiNBQDVCADoY6T9/0iLTCRIhAFIixX11AgAgz2OtwsAAHUS +SIlRCEiLTCRQSIkN3NQIAOsfSI15COhRPAIASI09ytQIAEiLTCRQDx9EAADoGzwCAJBIjQWr1AgA +6O6l/f9Ii0QkcEyLRCRATItMJFjpDP7//0iJ8UiJ2ugPQAIASInwTInR6IQ/AgBIjQUDcAMAuysA +AADo883//0iNBZVZAwC7HQAAAOjizf//SI0FXWcDALskAAAA6NHN//9IjQULYAMAuyAAAAAPH0QA +AOi7zf//SI0F1V8DALsgAAAA6KrN//9IjQWaaQMAuyYAAADomc3//0iNBZNfAwC7IAAAAOiIzf// +kMzMzMzMzMxJO2YQdiBIg+wYSIlsJBBIjWwkEEiNBcNmAwC7JAAAAOhbzf//kOh1NwIA69PMzMzM +zMzMzMzMzMzMzMzMzMzMSTtmEHYgSIPsGEiJbCQQSI1sJBBIjQWmXgMAuyAAAADoG83//5DoNTcC +AOvTzMzMzMzMzMzMzMzMzMzMzMzMzEiD7ChIiWwkIEiNbCQgSIsFO6cIAEiLDTynCABIxwQkAgAA +AEiJRCQIiUwkEOiuUAIARQ9X/2RMizQl+P///0iLbCQgSIPEKMPMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzEiD7ChIiWwkIEiNbCQgSIsF66YIAEiLDeymCABIxwQkAgAAAEiJRCQIiUwkEOhOUAIARQ9X +/2RMizQl+P///0iLbCQgSIPEKMPMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEiD7BhIiWwkEEiNbCQQ +SI0FCy8DALsJAAAA6CHM//+QSTtmEA+GPgEAAEiD7FBIiWwkSEiNbCRIi4iQAAAAZpCFyQ+EDgEA +AEiJRCRYkJBIjQWSswsA6MWh/f9Iiw0WyQgASI1xAUiLHQPJCABIiz0MyQgASDn3czdIjQVgmwIA +6FsUAQBIiQ30yAgAgz3NtAsAAHUJSIkF1MgIAOsMSI09y8gIAOh2OAIASInZSInDSI1RAUiJFb3I +CABIjTzLgz2atAsAAHULSItUJFhIiRTL6wpIi1QkWOhjOQIASIsdjMgIAEiLDY3ICABIhcl2X0g5 +HWHHCAB0KpCAPV+0CwAAdBZIiVwkQEiNBUnHCADojDv9/0iLXCRASI0NOMcIAEiHGUiLDU7ICABI +jRWvsgsASIcKkJBIjQWrsgsADx8A6Lui/f9Ii2wkSEiDxFDDMcDoajwCAEiNBThMAwC7GQAAAOjZ +yv//kEiJRCQI6O40AgBIi0QkCOmk/v//zMzMzEk7ZhAPhoUAAABIg+woSIlsJCBIjWwkIEiJRCQw +kEiNBUOyCwDodqD9/0iLDb/HCABIiUwkGEiLFbvHCABIiVQkEDHA6ylIiUQkCEiLHMFIi1QkMEiL +MkiJ2P/WSItMJAhIjUEBSItMJBhIi1QkEEg50HzSkJBIjQXrsQsADx8A6Puh/f9Ii2wkIEiDxCjD +SIlEJAjoRzQCAEiLRCQIZpDpW////8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2bkiD +7ChIiWwkIEiNbCQgSIlEJDBIiw2AsQsASIlMJBBIix3sxQgASIlcJBgx0uswSIlUJAiQSIsISIs0 +00iJx0iJ8EiJ+v/RSItMJAhIjVEBSItEJDBIi0wkEEiLXCQYSDnRd8tIi2wkIEiDxCjDSIlEJAjo +ojMCAEiLRCQI6Xj////MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4ZgAQAASIPsQEiJbCQ4 +SI1sJDjGBUGwCwABMcDrAv/Aiw1ZsAsAjQwIjUkBSGPJSMHhA0gDDUnFCABIgzkAdd6JRCQYMcnr +E4tUJByNSgGLRCQYDx+EAAAAAAA5wQ+NmgAAAIlMJByLFRKwCwCNFAqNUgFIY9JIweIDSAMVAsUI +AEiLAkiJRCQg6K1PAQBEDxF8JChIi0wkIEiJTCQoSIlEJDBIi1QkKEiD+Ah9BDHA6xlIidBIjR3w +KQMAuQgAAADolTH9/0iLTCQghMAPhHX///9IicgPH0QAAOibIgIASIP7CHJxSI1L+EiJykj32UjB ++T9Ig+EISAHB6wQx0jHJSInISInT6G8g/f8Ptg1StgsAiA1XrwsAD7YNSbYLAIgNS68LAA+2DTW2 +CwCIDTyvCwAPtg1BtAsAiA0erwsAD7YNebULAIgNEK8LAEiLbCQ4SIPEQMO4CAAAAEiJ2eg3OgIA +kOgRMgIA6Yz+///MzMzMzMzMzMzMzMxJO2YQD4aSAgAASIPsWEiJbCRQSI1sJFBMiXQkQMcFsc0I +ABAnAABIjQVSlwgA6xZIiUQkOOgmUgEASItMJDhIi4EQAgAASIXAdeXoEBcBAOirpf3/kEiNBYOv +CwC7CAAAAEiJ2ejWlf//SItEJEBIi0gwSMfD/////0iJyA8fAOg7AwAA6Pb9///o8Tb9/+iMTwEA +6MfwAQDo4pb9/0iLRCRASItIMIQBkMcEJAIAAABIx0QkCAAAAABIg+mASIlMJBDHRCQYCAAAAOhw +TQIARQ9X/2RMizQl+P///0iLRCRASItAMEiLgIAAAABIiQUcrwsA6KfGAACQ6MHHAACQ6PvOAADo +FjP+/5CQSI0FpcwIAOionP3/6GNMAgBFD1f/ZEyLNCX4////SIsEJEiJBXPMCACLBfGtCwCJRCQk +uwoAAABIjQX/KwMA6NBt/f/oi0wBAEhjyEg5yLkAAAAAD0TID5TAIdiEwHQEhcl/BItMJCSJyOgF +igAADx9EAABIhcAPhQEBAACQkEiNBSbMCADoCZ79/4M9QrELAAFmkH4txgVcrwsAAcYFUK8LAAFI +iw15wwgASIlMJEhIixV1wwgASIlUJDAxwOmxAAAASIM9kaAIAAB1N0jHBYSgCAAHAAAAgz0VrwsA +AHUQSI0FxyYDAEiJBWWgCADrE0iNPVygCABIjQWwJgMA6LAyAgBIgz2QoAgAAXUvSMcFg6AIAAAA +AACDPdSuCwAAdQ1IxwVnoAgAAAAAAOsOSI09XqAIADHA6HcyAgBIi2wkUEiDxFjDSIlEJChIixTB +hAJIgcLAFgAASInQ6LN8//9Ii0wkKEiNQQFIi0wkSEiLVCQwSDnQfM7pRf///0iNBdZnAwC7KwAA +AOhFxf//kA8fQADoWy8CAOlW/f//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhoMAAABIg+wg +SIlsJBhIjWwkGIsF+soIAEiLDevKCABIKw30yggAOch8CkiLbCQYSIPEIMOJRCQUZpDoO93//0iN +BVVHAwC7GQAAAOhK5v//i0QkFEhjwA8fAOg75P//SI0FjjADALsOAAAA6Crm///ohd3//0iNBdk2 +AwC7EQAAAOiUxP//kOiuLgIA6Wn////MzMzMzMzMzMxJO2YQdlFIg+wgSIlsJBhIjWwkGEiLDVXK +CABIjVEBkEg50X8gSIlMJBBIiRU/yggA6Cr///9Ii0QkEEiLbCQYSIPEIMNIjQXRSgMAuxsAAADo +KsT//5DoRC4CAOuizMxJO2YQD4a5AQAASIPsOEiJbCQwSI1sJDBIiUQkQEiJXCRISYtWMEyJ9kg5 +MnQbhABIjZhwAQAAuSAAAABIic+4AQAAAOg3zwEAkJBIjQWmyQgA6KmZ/f9Ii0wkSA8fQABIhcl8 +DkiLVCRASImK6AAAAOsU6Cj///9Ii0wkQEiJgegAAABIicpIix2SqwsASIuK6AAAAEiJTCQgSI1E +JCCQ6LsvAgBIi0wkQImBIAEAAOgrLwIARQ9X/2RMizQl+P///0iLDCRIix1TqwsASIlMJChI99NI +jUQkKOiBLwIASItMJECJgSQBAACLkSABAAAJwoXSdQrHgSQBAAABAAAASInI6PiS//9Ii1wkQEiL +S1BIhcl0DkiLEUiBwqADAABIiVEYSIsNFb8IAIM9DqwLAAB1CUiJi1gBAADrDEiNu1gBAADotzAC +AJCAPe+rCwAAdBFIjQXmvggA6CEz/f9Ii1wkQEiNDdW+CABIhxmQkEiNBYHICADoZJr9/4A9qakL +AAB0NEiNBfTCAgDor7f9/4M9qKsLAAB1DkiLTCRASImBQAEAAOsRSItMJEBIjblAAQAA6EcvAgBI +i2wkMEiDxDjDSIlEJAhIiVwkEOhuLAIASItEJAhIi1wkEA8fQADpG/7//8zMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzEk7ZhAPhuIBAABIg+xQSIlsJEhIjWwkSIhMJB9IiUQkWIA9CD4JAAB0D+hx +mgEASItEJFgPtkwkH5CLkJAAAABJi3YwkP+GCAEAAA+68gxJi3Ywg/oEdXNMiXQkQEiJdCQwuwQA +AAC5AQAAAOgvBwAASItUJEBIi1IwSIuC0AAAAEiLXCRYD7ZMJB/oMKsAAOhLLgAAkEiLVCQwi7II +AQAAjX7/iboIAQAAg/4BdRJBgL6xAAAAAHQIScdGEN76//9Ii2wkSEiDxFDDkEyJdCQ4i4iQAAAA +iUwkIEiLkJgAAABIiVQkKOiU2f//SI0FGzEDALsQAAAA6KPi//9Ii0QkWOgZ4v//SI0F6iADALsH +AAAA6Iji//9Ii0QkKA8fAOh74P//SI0FuTQDALsTAAAA6Gri//+LRCQgicAPH0AA6Fvf///ottv/ +/+ix2f//SItEJDiLiJAAAACJTCQkSIuQmAAAAEiJVCQo6BHZ//9IjQVoMAMAuxAAAAAPH0QAAOgb +4v//SItEJDjokeH//0iNBWIgAwC7BwAAAA8fRAAA6Pvh//9Ii0QkKOjx3///SI0FHDQDALsTAAAA +Dx9EAADo2+H//4tEJCSJwOjQ3v//6Cvb///oJtn//0iNBX06AwC7FgAAAOg1wP//kEiJRCQISIlc +JBCITCQY6EEqAgBIi0QkCEiLXCQQD7ZMJBjp7f3//8zMzMzMzMzMzMzMzMxJO2YQD4ayAAAASIPs +GEiJbCQQSI1sJBC4AQAAAEiNDRCnCwCHATHA6yHHBCToAwAA6CpEAgBFD1f/ZEyLNCX4////SItE +JAhI/8BIg/gFfSZIiUQkCMcFZMYIAP///3+4AQAAAEiNDVTGCACHAehNmQAAhMB1s8cEJOgDAABm +kOjbQwIARQ9X/2RMizQl+P///+gpmQAAxwQk6AMAAGaQ6LtDAgBFD1f/ZEyLNCX4////SItsJBBI +g8QYww8fQADoWykCAOk2////zMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhoIDAABIg+xASIls +JDhIjWwkOIlcJFCJTCRUSIlEJEiNk//v//+D+gN2DIH7CRAAAA+FrwEAAInaD7rzDDnZdSBIicOJ +0PAPsYuQAAAAQA+UxkCE9nQKSItsJDhIg8RAw+gV1///SI0FjVsDALsnAAAA6CTg//9Ii0QkSOia +3///SI0FGSEDALsJAAAA6Ang//+LRCRQicAPHwDoW97//0iNBfEgAwC7CQAAAOjq3///i0QkVInA +Dx9AAOg73v//6DbZ///oMdf//5BMiXQkKEiLRCRIi4iQAAAAiUwkHEiLkJgAAABIiVQkIOiL1v// +SI0FEi4DALsQAAAA6Jrf//9Ii0QkSOgQ3///SI0F4R0DALsHAAAADx9AAOh73///SItEJCDocd3/ +/0iNBa8xAwC7EwAAAA8fRAAA6Fvf//+LRCQcicDoUNz//+ir2P//6KbW//9Ii0QkKIuIkAAAAIlM +JBRIi5CYAAAASIlUJCDoBtb//0iNBV0tAwC7EAAAAOgV3///SItEJCjoi97//0iNBVwdAwC7BwAA +AOj63v//SItEJCDo8Nz//0iNBRsxAwC7EwAAAA8fQADo297//4tEJBSJwOjQ2///6CvY///oJtb/ +/0iNBZNmAwC7NAAAAOg1vf//6JDV//9IjQXZXgMAuysAAAAPH0AA6Jve//9Ii0QkSOgR3v//SI0F +kB8DALsJAAAADx9EAADoe97//4tEJFCJwOjQ3P//SI0FZh8DALsJAAAADx9AAOhb3v//i0QkVInA +6LDc///oq9f//+im1f//kEyJdCQwSItEJEiLiJAAAACJTCQYSIuQmAAAAEiJVCQgDx9EAADo+9T/ +/0iNBYIsAwC7EAAAAOgK3v//SItEJEgPH0QAAOh73f//SI0FTBwDALsHAAAA6Ord//9Ii0QkIA8f +RAAA6Nvb//9IjQUZMAMAuxMAAADoyt3//4tEJBiJwA8fQADou9r//+gW1///6BHV//9Ii0QkMIuI +kAAAAIlMJBBIi5CYAAAASIlUJCDocdT//0iNBcgrAwC7EAAAAA8fRAAA6Hvd//9Ii0QkMOjx3P// +SI0FwhsDALsHAAAADx9EAADoW93//0iLRCQg6FHb//9IjQV8LwMAuxMAAAAPH0QAAOg73f//i0Qk +EInA6DDa///oi9b//+iG1P//SI0FmWYDALs3AAAA6JW7//+QSIlEJAiJXCQQiUwkFOiiJQIASItE +JAiLXCQQi0wkFOlQ/P//zMzMzMzMzMzMzMzMzMzMzEk7ZhAPhqIAAABIg+wYSIlsJBBIjWwkEIlc +JCiJTCQsjVP/g/oDdyaJ2g+66ww52XUcSInDidDwD7GLkAAAAA+UwYnISItsJBBIg8QYw+ht0/// +SI0Fck8DALshAAAAkOh73P//i0QkKInA6NDa//9IjQUKHAMAuwgAAAAPH0AA6Fvc//+LRCQsicDo +sNr//+ir1f//6KbT//9IjQWtKQMAuxAAAADotbr//5BIiUQkCIlcJBCJTCQU6MIkAgBIi0QkCItc +JBCLTCQU6TD////MzMzMzMzMzMzMzMzMzMzMSIPsQEiJbCQ4SI1sJDhIiUQkSIlMJFSJXCRQD7rj +DJByCg+64QxyBDnLdUJEDxF8JChIjQUnAwAASIlEJCiJXCQwiUwkNEiNRCQoSIkEJOgMIwIARQ9X +/2RMizQl+P///0iLRCRIi0wkVItcJFAx0jH26xZIi3wkIEiNVwFIi0QkSItcJFCLTCRUSInHidjw +D7GPkAAAAEEPlMAPH0QAAEWEwA+FoQAAAIP7BHUNg7+QAAAAAQ+EdQIAAEiJVCQgSIXSdR3oVj8C +AEUPV/9kTIs0Jfj///9IiwQkSI2wiBMAAEiJdCQY6DQ/AgBFD1f/ZEyLNCX4////SItEJBhmkEg5 +BCR9EItMJFBIi1QkSDHb6fwBAADo5UQCAEUPV/9kTIs0Jfj////o8z4CAEUPV/9kTIs0Jfj///9I +iwQkSI2wxAkAAOkq////g/sCdSEPtoe9AAAAqAd1B8aHvAAAAAEPtoe9AAAA/8CIh70AAACAv7wA +AAAAD4Q+AQAAkGaQ6Js+AgBFD1f/ZEyLNCX4////SIsEJItMJFCD+QF1JkiLVCRISIuawAAAAEiJ +xkgp2EgBgsgAAABIx4LAAAAAAAAAAOsISItUJEhIicaLXCRUg/sBdQ9IibLAAAAADx8A6dYAAACD ++wIPhc0AAADGgrwAAAAASIuayAAAAA8fQABIhdt9G7gBAAAASI0NF9YIAPBID8EB6ZUAAAAPH0QA +AEiD+xB8X0gPvfNIx8f/////SA9E90iNfv1JifhIwecESIH/0AIAAHINQbgsAAAAvg8AAADrNUiN +TvxIg/lASBn2SPfWSAnxSNP7SIneSMH7P0jB6zxIAfNIwfsESMHjBEgp3usGRTHASIneScHgBEmN +BDBIPdACAABzKkiNDQa/CABIjQTBuQEAAADwSA/BCEjHgsgAAAAAAAAASItsJDhIg8RAw7nQAgAA +6FEpAgBIiVwkEMcEJAEAAAAPH0QAAOjbIQIARQ9X/2RMizQl+P///0iLRCQQSI1YAYtEJFBIi0wk +SEiLRCQYi0wkUEiLVCRISIP7Cn0QDx+EAAAAAAA5ipAAAAB1p0iJxulE/f//SI0FSF4DALsxAAAA +Dx9AAOg7t///kMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GfAAAAEiD7ChIiWwkIEiN +bCQgi0IMSIlEJBiLSghIiUwkEOhTz///SI0FrEADALscAAAA6GLY//9Ii0QkEOi41v//SI0F8hcD +ALsIAAAA6EfY//9Ii0QkGGaQ6JvW///oltH//+iRz///SI0F10UDALsfAAAADx9EAADom7b//5Do +FSACAOlw////zMzMzMzMzMzMzMzMzMzMzEk7ZhB2YEiD7BhIiWwkEEiNbCQQg/sCdToPH4AAAAAA +gfkJEAAAdAXrKUiJyEiJwbgCAAAAugkQAADwD7GRkAAAAA+Uw4TbdOFIi2wkEEiDxBjDSI0F1CQD +ALsQAAAAkOgbtv//kEiJRCQIiVwkEIlMJBToKCACAEiLRCQIi1wkEItMJBTpdv///8zMzMzMzMzM +zMzMzMzMzMzMzMzMzMxJO2YQdk5Ig+wYSIlsJBBIjWwkEIP7CXUpg/kEdSRIicG4CQAAALoEAAAA +8A+xkZAAAAAPlMGJyEiLbCQQSIPEGMNIjQVFJAMAuxAAAADojbX//5BIiUQkCIlcJBCJTCQU6Jof +AgBIi0QkCItcJBCLTCQU64vMzMzMzMzMzMzMzEk7ZhAPhlECAABIg+xYSIlsJFBIjWwkUEmLTjCD +uQgBAAAAD48gAgAATIl0JECQkEiNBQG7CADoBIv9/4sNZpwLAIkN3LsIALkBAAAASI0VzLsIAIcK +6MWOAABIi0wkQEiLSTBIi4nQAAAAx0EEAwAAAP8NrLsIAEiLDTGyCABIiUwkSEiLFS2yCABIiVQk +KDHA6wxIjUcBDx+EAAAAAABIOdAPjZEAAABIixzBi3MEg/4CdRdIiceJ8EG4AwAAAPBED7FDBEAP +lMbrC0iJx0G4AwAAADH2QIT2dLiAPY0wCQAAdDpIiXwkIEiJXCQwSInY6OmOAQBIi0QkMA8fQADo +O4cBAEiLTCRISItUJChIi1wkMEiLfCQgQbgDAAAA/0MU/w38uggA6Wf////HQAQDAAAA/w3quggA +6KGcAACQSIXAdeiQkIsN17oIAIlMJBRIjQXguQgA6MOL/f+LTCQUhcl+K+sF6LSNAABIjQW1uggA +u6CGAQDo44/9/w8fAITAdOOQSMcFmLoIAAAAAACDPY26CAAAdRlIiw0QsQgASIsVEbEIADHAMdsx +9umHAAAAuCkAAABIjQ0sUwMAixXQmgsAhdJ0OEiJRCQYSIlMJDiQkEiNBWWbCwAPH0QAAOhbif3/ +kJBIjQVSmwsA6E2J/f9Ii0QkGEiLTCQ4Dx8ASIXAdQpIi2wkUEiDxFjDSInDSInI6Eaz//9IizzB +g38EA78uAAAASA9F30yNBeRYAwBJD0XwSP/ASDnQfNxIidhIifHpdf///0iNBU86AwC7GwAAAOgG +s///kA8fRAAA6BsdAgDplv3//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4a1AQAASIPsWEiJ +bCRQSI1sJFCIRCRgSYtOMJD/gQgBAABJi04wSIlMJDiLFfqZCwCF0nQaMcAPH0AA6Htz//9IiUQk +MEiNRCQw6Kw3AACQkEiNBVu4CAAPHwDoW4j9/4sNvZkLAIsVy5kLAIXSdA3HBb2ZCwAAAAAAkOsC +icqJ0Oj1dQAASIlEJEDHBQa5CAAAAAAAgz0PuQgAAHQWxwUDuQgAAAAAAEiNBQS5CADop4r9/5CQ +SI0F9rcIAOjZif3/SItMJEDrBUiLTCRISIXJdFdIi1EISItxOEiJVCRISIX2dDKQSMdBOAAAAABI +g77YAAAAAA8fRAAAD4WnAAAAkEiJjtgAAABIjYZQAQAA6EaK/f/rsjHASInLSMfB/////+jTEQAA +65+Q6Es3AgBFD1f/ZEyLNCX4////SIsEJEiJRCQoD7ZMJGCEyXQZkLgKAAAASMfD/////zHJMf9I +if7o03UBAOguHgAAkEiLRCQ4i4gIAQAAjVH/iZAIAQAAg/kBdRJBgL6xAAAAAHQIScdGEN76//9I +i0QkKEiLbCRQSIPEWMNIjQVmTAMAuyUAAADoIrH//5CIRCQI6DgbAgAPtkQkCOku/v//zMzMzMzM +zMzMzMzMzMxIg+woSIlsJCBIjWwkIEyJdCQYSYsGSIlEJBAPH0QAAEiFwHU5SYtOCEiJTCQISIXJ +dQlIx0QkCAAgAABIjUwkCEiLVCQYSIlKCEiLXCQISCnZSIHBAAQAAEiJCusFSItUJBhIiwJIBaAD +AABIiUIQSIlCGOgnAAAASItEJBBIhcAPlMDo1wEAAEiLbCQgSIPEKMPMzMzMzMzMzMzMzMzMSTtm +EA+G3wAAAEiD7CBIiWwkGEiNbCQYSYtOMEyJ8pBIOREPha4AAABIiVQkEEmJVkhIi0QkIEmJRkBI +jUQkKEmJRjjoFxgCAEUPV/9kTIs0Jfj////oZYD//0iNBb6xCABIi0wkEEg5QTB1EeiOAAAASI0F +p7EIAEiLTCQQSItZMEiLk7gAAABIhdJ0EUiLAv/QSI0FhrEIAEiLTCQQSItRMEg5wnQgSIuC2AAA +AOiMewAASItMJBBIi0kwSMeB2AAAAAAAAADo0zcAAEiLbCQYSIPEIMNIjQW5IwMAuxMAAADoeK// +/5DokhkCAOkN////zMzMzMzMzMzMzMzMzEk7ZhB2PUiD7BBIiWwkCEiNbCQIgD1RlgsAAHQVgD1A +lgsAAHUMxgU3lgsAAehuCwAAMcDoJ9QAAEiLbCQISIPEEMPoOBkCAOu2zMzMzMzMzMzMzMzMzMzM +zMzMzMzMzEiD7BhIiWwkEEiNbCQQTIl0JAhIi0wkCEiLUTCEAkiNglABAADo9of9/0iLTCQISItR +MEjHglABAAAAAAAAZpDoGxIAAITAdcpIi2wkEEiDxBjDzMzMzMzMzMzMzMzMzEk7ZhAPhhICAABI +g+wgSIlsJBhIjWwkGEmLTjBIjRU9sAgASDnRD4SiAQAAiEQkKEiJTCQQuAEAAADoIe4AAJDoG/IA +AEiLTCQQSItRUEiF0nQ0SItaCEiLAuihBwEAgz1alwsAAHUPSItMJBBIx0FQAAAAAOsQSItMJBBI +jXlQMdLoGRwCAJBIjQXpswgA6OyD/f9Ii0wkEEiNBSCqCADrB0iNglgBAABIixBIhdIPhAgBAABI +Ocp16EiLkVgBAACDPfiWCwAAdQVIiRDrCEiJx+jJGwIAD7ZUJCgPH0AAhNJ1Q7sBAAAAh5kcAQAA +SIsdYrQIAIM9w5YLAAB1EEiJmagCAABIiQ1LtAgA6xhIjbmoAgAA6KUbAgBIjT02tAgA6FkbAgCQ +kEiNBUizCADoK4X9/0iLTCQQSIuRMAEAAEiNHcCVCwDwSA/BE+iuegAA6CkXAACQkEiNBRizCADo +G4P9/0j/BTSzCADoT3wAAJCQSI0F/rIIAOjhhP3/D7ZMJCiEyXQKSItsJBhIg8Qgw0iLRCQQSAUc +AQAASIkEJOhaMAIARQ9X/2RMizQl+P///0iLbCQYSIPEIMNIjQVfIQMAuxMAAADo0qz//+gtegAA +6KgWAACQkEiNBZeyCADomoL9/0j/BbOyCADoznsAAJCQSI0FfbIIAA8fRAAA6FuE/f/olv3//0iN +BcsdAwC7EQAAAOiFrP//kIhEJAjomxYCAA+2RCQI6dH9///MzMzMzMzMzMzMzMzMzMzMzEk7ZhgP +hhwDAABIg+xgSIlsJFhIjWwkWEiJRCRoTIl0JDhJi04wkP+BCAEAAEmLTjBIi4nQAAAAkEiLVCQ4 +SItSMEiJVCQoSIlMJFCQSI0F5LEIAOjngf3/gz3wsggAAA+FqQIAAIsFPJMLAP/IiQXcsggAgz0F +lQsAAHUOSItEJGhIiQW/sggA6xFIjT22sggASItEJGjopBgCAEiLDQ2pCABIixUOqQgASItcJFAx +9usDSP/GSDnWfRtIizzxkEg5+3TuhAdBuAEAAABEh4fQJgAA693oRYUAAEiLBY6xCADrBEiLQQhI +hcB0PEiJwbgBAAAAMdvwD7GZ0CYAAEAPlMZAhPZ03EiJTCRISItUJGhIixpIicj/0/8NMLIIAEiL +TCRIMdvru5CQiw0fsggAiUwkFEiNBfywCAAPH0AA6NuC/f9Ii1QkaEiLCkiLRCRQ/9FIiw1VqAgA +SIlMJEBIixVRqAgASIlUJCAxwOsESI1DAUg50A+NjwAAAEiLNMGLfgSD/wJ1J4O+0CYAAAF1FEiJ +w4n4RTHA8EQPsUYEQA+Ux+sSSInDRTHAMf/rCEiJw0UxwDH/QIT/dLNIiVwkGIA9pyYJAAB0H0iJ +dCQwSInw6AiFAQBIi0QkMA8fAOhbfQEASIt0JDD/RhRIifDoSxQAAEiLTCRASItUJCBIi1wkGEUx +wOlk////i0wkFIXJfijrBegFhAAASI0FNrEIALughgEA6DSG/f+EwHTmkEjHBRyxCAAAAAAAgz0N +sQgAAA+FtQAAAEiLDWCnCABIixVhpwgAMcDrBUj/wGaQSDnQfRFIizTBhAaDvtAmAAAAdOfrd5CQ +SI0Fua8IAJDou3/9/4M99JILAAB1DUjHBa+wCAAAAAAA6w5IjT2msAgAMcnolxcCAJCQSI0Fhq8I +AOhpgf3/kEiLTCQoi5EIAQAAjVr/iZkIAQAAg/oBdRJBgL6xAAAAAHQIScdGEN76//9Ii2wkWEiD +xGDDSI0FHy0DALsaAAAA6GKp//9IjQVYHAMAuxIAAADoUan//0iNBYY/AwC7IgAAAA8fRAAA6Dup +//+QSIlEJAjokDoCAEiLRCQI6cb8///MzMzMzMxJO2YQD4aQAAAASIPsEEiJbCQISI1sJAhJi04w +SIuJ0AAAAIQBuAEAAAAx2/APsZnQJgAAD5TDhNt0VUiLFcavCABIixpIicj/05CQSI0Fpa4IAOio +fv3/iw2yrwgAjVn/iR2prwgAg/kBdQxIjQWlrwgA6CiB/f+QkEiNBXeuCADoWoD9/0iLbCQISIPE +EMNIi2wkCEiDxBDD6KESAgCQ6Vv////MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4Z0 +AgAASIPsUEiJbCRISI1sJEhIiUQkWEiJTCQQSIlcJGBMiXQkMEmLVjCQ/4IIAQAASYtWMEiDutAA +AAAAdRTo9nMAAEiLRCRYSItMJBBIi1wkYEiDPbeuCAAAdCKQkEiNBcytCADoz339/0iLDaCuCABI +iUwkKDHAkOkhAQAASI0FdP4CAOjvnP3/gz3okAsAAHUOSItMJGBIiYi4AAAA6xFIjbi4AAAASItM +JGDohxUCAEiJRCQgSItcJBDoeOP//4A9nY4LAAB0LLj/////6KVPAACDPZ6QCwAAdQpIi0wkIEiJ +Aes6SIt8JCDoSBQCAEiJ+esruAAgAADoeU8AAIM9cpALAAB1CkiLTCQgSIkB6w5Ii3wkIJDoGxQC +AEiJ+UiLEYQCgz1MkAsAAHUGSIlKMOsJSI16MOj7FAIASItUJDBIi1owSIub0AAAAEiLdCRYSDne +dRCQ6Ft0AABIi0wkIEiLVCQwSItSMIuaCAEAAI1z/4myCAEAAIP7AXUSQYC+sQAAAAB0CEnHRhDe ++v//SInISItsJEhIg8RQw0iLTCQoSIXJD4SZAAAAg7kcAQAAAHQ0SIuRqAIAAIM9tY8LAAB1CUiJ +gagCAADrD0iNuagCAAAPHwDoWxMCAEiLRCQoSIlUJCjrtUiJRCQYRA8RfCQ4SI0FvQAAAEiJRCQ4 +SI1MJChIiUwkQEiNTCQ4SIkMJA8fRAAA6BsPAgBFD1f/ZEyLNCX4////SItEJChIi4CoAgAASIlE +JChIi0QkGOlZ////gz0sjwsAAHUJSIkFu6wIAOsMSI09sqwIAOjVEgIAkJBIjQXEqwgA6Kd9/f/p +B/7//0iJRCQISIlcJBBIiUwkGOjuDwIASItEJAhIi1wkEEiLTCQY6Vr9///MzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzEk7ZhB2LkiD7BhIiWwkEEiNbCQQSItKCEiLCUiLCUiLAUiLWQjo1v4AAEiL +bCQQSIPEGMPo5w4CAOvFzMzMzMxIg+w4SIlsJDBIjWwkMIA9V4wLAAB0XoA9RowLAABmkHVTSIsN +r4AIAEiLFaCACABIhckPhkwBAABIxwQkAgAAAEiJVCQIiUwkEOjxKAIARQ9X/2RMizQl+P///8cE +JAEAAADoOCgCAEUPV/9kTIs0Jfj///9Ix0QkIAAAAACQkMcEJAIAAABIx0QkCAAAAABIjUQkIEiJ +RCQQx0QkGAgAAADoGSsCAEUPV/9kTIs0Jfj///8xwOhl5AAAMcAPHwDo2wMAAEiJRCQoSIO4YAEA +AAAPlIAoAQAA/w2tiwsASIuIYAEAAEiNFWOMCwBIhwpIi0wkIEiJiIAAAABIiwhIiQwk6LgPAgBF +D1f/ZEyLNCX4////SI1EJEBIjYgABAAASYlOCEiNiACA//9JiQ5IBaCD//9JiUYQ6CIMAgBFD1f/ +ZEyLNCX4////6HB0//9Ii0QkKEiLgMAAAAC7BgAAALkDAAAA6HXp//+4/////0iNDfGpCADwD8EB +SItsJDBIg8Q4wzHA6FQVAgCQzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2b0iD7BhIiWwkEEiNbCQQ +MclIjRXTigsAhwqQhcl2CIlMJAwxwOtDuAEAAADoygIAAJBIicFIjRVviwsASIcCSIXJdQXoMgAA +AEiLbCQQSIPEGMOJRCQIDx9AAOgbAAAAi0QkCP/Ai0wkDDnBd+Xr2ehmDQIA64TMzMzMSTtmEA+G +SwEAAEiD7DBIiWwkKEiNbCQoMcAx20jHwf/////ouPr//0iJRCQYuAAQAADoKUsAAEiJRCQgSI0V +nQ8CAEj/wkiJUEBIi1AISIPC4EiJUDhIx0BgAAAAAEiJxkiJRkhIi35ASIl+eEiJVnBIi1Y4SImW +gAAAAEiJ8DHbuQYAAADoNej//4M9zosLAAB1EEiLTCQYSItEJCBIiUgw6xNIi0QkIEiNeDBIi0wk +GOhpEAIAhAGDPaCLCwAAdQlIiYHAAAAA6wxIjbnAAAAA6EkPAgD/gXQCAACQSInCSImBaAEAAJBI +iYroAAAAuQEAAABIjTUEqAgA8EgPwQ5I/8FIiYqYAAAASInQ6A3W//+5AQAAAEiNFSmoCADwD8EK +uAEAAADoUwEAAJBIi0wkGEiJgWABAAD/BSyJCwCQSI0V6IkLAEiHCkiLbCQoSIPEMMPoBgwCAOmh +/v//zEk7ZhAPhvkAAABIg+xASIlsJDhIjWwkOEmLVjBIiVQkMEiLgsAAAAC7AwAAALkGAAAA6Cnn +//9Ii1QkMEiLssAAAADGhrIAAAAAvgEAAABIjT2SpwgA8A/BN0iLsoAAAABIiXQkKDHA6DPhAACQ +6C3lAAC4AQAAAOijAAAA/wWJiAsAkEiLVCQwSImCYAEAAEjHBCQAAAAA6KMMAgBFD1f/ZEyLNCX4 +////kEiLVCQwSI01GYkLAEiHFkiLVCQoSIlUJCCQxwQkAgAAAEiNVCQgSIlUJAhIx0QkEAAAAADH +RCQYCAAAAOhUJwIARQ9X/2RMizQl+P///0iLbCQ4SIPEQMPo+AoCAOnz/v//zMzMzMzMzMzMzMzM +zMzMzMzMzEiD7BhIiWwkEEiNbCQQiEQkIDHJ6wkPtlQkIInBidCITCQPSIsVhogLAEiD+gF0eEiF +0nU5hMB1NYTJdRC5AQAAAEiNFaeHCwDwD8EKkMcEJAEAAADoxiQCAEUPV/9kTIs0Jfj///+4AQAA +AOuoSInQSI01OIgLAL8BAAAA8EgPsT5BD5TARYTAdTqQ6K8rAgBFD1f/ZEyLNCX4////D7ZEJA/p +bv///5DokisCAEUPV/9kTIs0Jfj///8PtkQkD+lR////SInQSItsJBBIg8QYw8zMzMzMzMzMzMzM +zMzMSTtmEA+G/AAAAEiD7ChIiWwkIEiNbCQgSIlcJDhIicJIidhIidPoVff//0iLVCQ4SIXSD5WA +SAEAAJBIiZDYAAAASIsVx4cLAEiJkIAAAABMifJIhdIPhIYAAABIi0owSIXJdH1IiUQkGIO5cAIA +AAB1CYC5GAEAAAB0ZpCQSI0FO4kLAOg2df3/gz1PiQsAAHReSIsNLokLAEiLVCQYSImKYAEAAJBI +iRUaiQsAgD0biQsAAHQTxgUSiQsAAEiNBROJCwDolnf9/5CQSI0F7YgLAOjIdv3/SItsJCBIg8Qo +w+hZAAAASItsJCBIg8Qow0iNBRk/AwC7KgAAAA8fAOjbnv//kEiJRCQISIlcJBBIiUwkGOjmCAIA +SItEJAhIi1wkEEiLTCQY6dL+///MzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4bxAAAASIPsOEiJbCQw +SI1sJDCAPY2FCwAAkA+EkgAAAEjHRCQYAAAAAEQPEXwkIEiDPVOaCAAADx8AD4SjAAAASIsISIlM +JBhIjYiIAAAASIlMJCBIjQ2nUAMAhAFIiw2eUAMASIlMJChIjQVKiAsA6MWqAABIiw0OmggASIkM +JEiNTCQYSIlMJAjoqwgCAEUPV/9kTIs0Jfj///9IjQUXiAsA6NKrAABIi2wkMEiDxDjDSIlEJEBI +jQX8hwsA6HeqAABIi0QkQOgtZf//SI0F5ocLAOihqwAASItsJDBIg8Q4w0iNBeweAwC7GQAAAOim +nf//kEiJRCQI6LsHAgBIi0QkCOnx/v//zMzMzMzMzMzMzMzMzMzMzMxJO2YQD4a8AAAASIPsKEiJ +bCQgSI1sJCBJi1YwkP+CCAEAAEmLVjAxwEiNNVCHCwC/AQAAAPAPsT5AD5TGDx8AQIT2dTGQi4II +AQAAjUj/iYoIAQAAg/gBdRJBgL6xAAAAAHQIScdGEN76//9Ii2wkIEiDxCjDSIlUJBhIjQUOUAMA +MdtIx8H/////6BD9//+QSItUJBiLsggBAACNfv+JuggBAACD/gF1EkGAvrEAAAAAdAhJx0YQ3vr/ +/0iLbCQgSIPEKMPo1QYCAOkw////zMzMzMzMzMzMzMzMzMzMzEiD7EhIiWwkQEiNbCRASYtOMIuJ +uAIAAA8fhAAAAAAAhckPhB8BAABMiXQkOEjHRCQgAAAAAJCQxwQkAgAAAEjHRCQIAAAAAEiNRCQg +SIlEJBDHRCQYCAAAAOihIgIARQ9X/2RMizQl+P///zHA6O3bAABIi0QkOEiLSDCEAUiBwbACAACQ +SInI6PJx/f9Ii0QkOEiLSDBIi5HAAgAASIlUJDBIhdJ0N4M9NYMLAAAPhZ4AAABIiwoxwP/RSItM +JDhIi1kwSMeDwAIAAAAAAABIi1kwMfaHs7gCAABIichIi0gwhAFIjYGwAgAAkOhuc/3/SItMJCBI +iUwkKJDHBCQCAAAASI1MJChIiUwkCEjHRCQQAAAAAMdEJBgIAAAAkOjbIQIARQ9X/2RMizQl+P// +/0iLTCQwSIXJD5XASItsJEBIg8RIwzHASItsJEBIg8RIw0iNBQFEAwC7MwAAAJDoO5v//5DMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzOh7/v//6JYmAgBFD1f/ZEyLNCX4////w8zMzMzMzMzMSTtm +EA+G6gAAAEiD7BhIiWwkEEiNbCQQkEiNBbigCADou3D9//8F0aAIAOjwaQAAkJBIjQWfoAgA6IJy +/f9mkOs0xgWnhAsAAZBIxwWjhAsAAAAAAJCQSI0FgoQLAGaQ6Fty/f9IjQWMhAsA6K9z/f/o6v3/ +/5BIjQVihAsAZpDoW3D9/+sOkJBIjQVQhAsA6Etw/f9Iiw1MhAsADx9AAEiFyXSdkEiJTCQISMcF +MoQLAAAAAACQSI0FIoQLAGaQ6Ptx/f9Ii0QkCOshSIuIYAEAAEiJTCQISMeAYAEAAAAAAADoePv/ +/0iLRCQISIXAddrrk+gnBAIA6QL////MzEk7ZhAPhtUAAABIg+wgSIlsJBhIjWwkGEmLTjCDuQgB +AAAAD4WkAAAASIO50AAAAAAPhYUAAACAuRQBAAAAZpB1aUyJdCQQkJBIjQWInwgA6Itv/f9Ii0wk +EEiLQTBmkOhbfgAAkJBIjQVqnwgA6E1x/f/oiOr//0iLTCQQSItRMEiLgtgAAADoU2UAAEiLTCQQ +SItJMEjHgdgAAAAAAAAASItsJBhIg8Qgw0iNBRYGAwC7DgAAAOhEmf//SI0FPAcDALsPAAAA6DOZ +//9IjQXMDgMAuxMAAADoIpn//5CQ6DsDAgDpFv///8zMzMzMzMzMzMzMzMzMzMzMzMzMzMxJi0Yw +xoAUAQAAAcPMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhoACAABIg+xASIlsJDhIjWwkOEiJRCRI +iFwkUEmLTjCQ/4EIAQAAkEmLTjBIiUwkIJBIjQV6nggAZpDoe279/0iLTCRISIXJdRHoDIEAAEiF +wA+E9QAAAEiJwZBIixVYnggASIXSdBdIi5pgAQAASIkdRZ4IAP8NR54IAEiF0kiJTCQodXro0NP/ +/0iJRCQYkJBIjQUanggAZpDo+2/9/w+2RCRQD7bASIXAuAAAAABIjQ2ESgMASA9FwUiLXCQoSItM +JBjoMfj//5BIi0QkIIuICAEAAI1R/4mQCAEAAIP5AXUSQYC+sQAAAAB0CEnHRhDe+v//SItsJDhI +g8RAw0iJVCQwkJBIjQWlnQgA6Ihv/f9Ii0wkMIC5FAEAAAAPhU4BAABIg7nYAAAAAA+FLwEAAA+2 +VCRQDx8AhNJ0CkiLdCQo6dMAAAAxwOt5kJBIjQVdnQgADx9EAADoO2/9/w+2TCRQhMl0Frn///// +SI0Vgp0IAPAPwQr/yYXJfDaQSItEJCCLiAgBAACNUf+JkAgBAACD+QF1EkGAvrEAAAAAdAhJx0YQ +3vr//0iLbCQ4SIPEQMNIjQVDHgMAuxsAAADoFZf//4iBFAEAAEiLVCQoSImR2AAAAEiNgVABAADo +d2/9/5BIi0wkIIuRCAEAAI1a/4mZCAEAAGaQg/oBdRJBgL6xAAAAAHQIScdGEN76//9Ii2wkOEiD +xEDDi77wBQAARIuG9AUAAEyLjvgNAABEi5b0BQAADx9AAEU5wnXcQTn4dQxNhcl1B4nQ6XX///9I +jQVPGQMAuxkAAADoeZb//0iNBWIEAwC7DwAAAOholv//SI0FHRADALsVAAAA6FeW//+QSIlEJAiI +XCQQ6GgAAgBIi0QkCA+2XCQQ6Vn9///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GrAIA +AEiD7CBIiWwkGEiNbCQYi4jwBQAAi5D0BQAASIuw+A0AAIu49AUAADnXdeM5yg+FaAIAAA8fAEiF +9g+FXAIAAIM9AJwIAAAPhU8CAACDPf98CwAAdHBIi4iYFgAASIXJdB5Ig3kQAHUQSIuIoBYAAEiD +eRAAZpB0B7kBAAAA6y9Iiw3whwsASIXJdR6LDW2ICwAPH0QAADkNZogLAHYHuQEAAADrCTHJ6wW5 +AQAAAITJdBMx22aQ6Hv8//9Ii2wkGEiDxCDDSIlEJCiLDWKbCACLFVibCAAB0YXJdTVIicExwEiN +FUqbCAC+AQAAAPAPsTIPlMJmkITSdBdIici7AQAAAOgv/P//SItsJBhIg8Qgw5CQSI0F1JoIAOjX +av3/gz2wmwgAAHRESItMJCjHQQQDAAAAiw2gmwgAjVH/iRWXmwgAg/kBdQxIjQWPmwgA6EJt/f+Q +kEiNBZGaCADodGz9/0iLbCQYSIPEIMNIi0QkKIO40CYAAAB0F0iJwbgBAAAAMdvwD7GZ0CYAAA+U +w+sFSInBMduE23Q0SIsVXZsIAEiLGkiJyP/Tiw1XmwgAjVn/iR1OmwgAg/kBdQxIjQVKmwgA6M1s +/f9Ii0wkKIM9cZoIAACQD4WbAAAAixV8ewsA/8o5FUSaCAB1M0iLFeuZCAAPHwBIhdJ0JJCQSI0F +6pkIAOjNa/3/SItEJCgx2+gh+///SItsJBhIg8Qgw0iLkWgWAABIi5lwFgAASIXSdBEPH0AASIXb +dAVIOdN8A0iJ00iJXCQQSInI6CZ7AACQkEiNBZWZCADoeGv9/0iLRCQQSIXAdAXo6RcAAEiLbCQY +SIPEIMOQkEiNBW6ZCADoUWv9/0iLRCQoMdvopfr//0iLbCQYSIPEIMMx2+iU+v//SItsJBhIg8Qg +w0iJRCQIDx9EAADoe/0BAEiLRCQI6TH9///MzMzMzMzMzMzMzMzMzMzMzEk7ZhB2aEiD7BhIiWwk +EEiNbCQQiw0+mQgAhcl0RosNOJkIAIXJdRwxwEiNDSuZCAC6AQAAAPAPsREPlMEPHwCEyXUKSIts +JBBIg8QYwzHAuwEAAADoBvr//0iLbCQQSIPEGMNIi2wkEEiDxBjD6O38AQDri8zMzMzMzMzMzMzM +STtmEA+GMwIAAEiD7EhIiWwkQEiNbCRASYtOMEiLkWgBAABIhdIPhP8BAABIi5LoAAAASDnKD4Xv +AQAATIl0JChIg7nQAAAAAHQK6NBfAADoS/z//7gBAAAA6CFhAACQ6Fvj//9Ii0wkKEiLUTBIi5Jo +AQAAi5KQAAAAidMPuvIMg/oBdS5Ii1EwSIuC2AAAAOgKXgAASItMJChIi0kwSMeB2AAAAAAAAABI +i2wkQEiDxEjDiVwkHOhjqv//SI0FLTQDALsrAAAA6HKz//+LRCQcicDoZ7D//0iNBYoqAwC7JAAA +AOhWs///6LGq//9Ii0QkKEiLQDBIi4BoAQAASIlEJDhMiXQkMIuIkAAAAIlMJBRIi5CYAAAASIlU +JCCQ6Pup//9IjQWCAQMAuxAAAADoCrP//0iLRCQ4Dx9EAADoe7L//0iNBUzxAgC7BwAAAOjqsv// +SItEJCAPH0QAAOjbsP//SI0FGQUDALsTAAAA6Mqy//+LRCQUicAPH0AA6Luv///oFqz//+gRqv// +SItEJDCLiJAAAACJTCQYSIuQmAAAAEiJVCQg6HGp//9IjQXIAAMAuxAAAAAPH0QAAOh7sv//SItE +JDDo8bH//0iNBcLwAgC7BwAAAA8fRAAA6Fuy//9Ii0QkIOhRsP//SI0FfAQDALsTAAAADx9EAADo +O7L//4tEJBiJwOgwr///6Iur///ohqn//0iNBYQTAwC7GQAAAOiVkP//SI0FRCYDALshAAAA6ISQ +//+QDx8A6Jv6AQDptv3//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4aNAAAASIPsIEiJbCQY +SI1sJBhIi4joAAAAkEk5TjB0X0iDudgAAAAAdT9IiUwkELj/////6OFeAACQ6HtdAACQSItMJBBI +iYHYAAAASI2BUAEAAOiCaP3/ZpDo+/X//0iLbCQYSIPEIMNIjQWRCQMAuxUAAAAPH0QAAOjbj/// +SI0FcRQDALsaAAAA6MqP//+QSIlEJAgPH0AA6Nv5AQBIi0QkCOlR////zMzMzMzMzMzMzMzMzMzM +zMxJO2YQD4bNAAAASIPsIEiJbCQYSI1sJBiDPUGWCAAAkA+EnwAAAEmLTjCAuRQBAAAAdB3GgRQB +AAAAuf////9IjRV2lQgA8A/BCv/Jhcl8ZOirXAAASIlEJBCQkEiNBRWVCADoGGX9/0iLRCQQx0AE +AwAAAIsF6pUIAI1I/4kN4ZUIAIP4AXUMSI0F2ZUIAOiMZ/3/kJBIjQXblAgADx8A6Ltm/f/o9vT/ +/0iLbCQYSIPEIMNIjQWNFwMAuxwAAADo247//0iNBfcTAwC7GwAAAOjKjv//kOjk+AEADx9AAOkb +////zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GPAEAAEiD7CBIiWwkGEiNbCQYTIl0 +JBBJi1YwhAKDPZZ3CwAAdQlIiYLAAAAA6xBIjbrAAAAADx9AAOg7+wEAiFwkMIQASItUJBBIi3Iw +gz1ldwsAAHUGSIlwMOsJSI14MOh0/AEASIlEJCi7AQAAALkCAAAADx9EAADom9P//0iLVCQoSMeC +qAAAAAAAAADGgrEAAAAASIsySIHGoAMAAEiJchAPtnQkMECE9nQHSItMJBDrE0iLTCQQSItZMEiL +m9AAAAD/QxBIi0kwix3IlAgADx+EAAAAAAA5mRABAAB0C0mLTjCQiZkQAQAAgD22CQkAAHQmSIN6 +cAB0FYC6uwAAAAB0DEiLgtAAAADoKGcBAOjDYwEASItUJChIjUI4SIkEJOix9QEARQ9X/2RMizQl ++P///0iLbCQYSIPEIMNIiUQkCIhcJBDobPcBAEiLRCQID7ZcJBBmkOmb/v//zMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMTI1kJJhNO2YQD4Z0CQAASIHs6AAAAEiJrCTgAAAASI2sJOAAAABMibQk +yAAAAEiLjCTIAAAASItRMEiLgtAAAACDPZiTCAAAdAfoMf3//+vdSImEJNgAAACEAIO40CYAAAB0 +DeiX4///SIuEJNgAAAAx2+joGAAASIlEJFhIiVwkUIA9nnMLAAB0KYA9lnMLAAB0IOgH9/3/SIXA +dAwx27kBAAAA6FbK//9Ii0QkWEiLXCRQSIsNLWYIAEiLCWaQSIXJdB9IiQwkSMdEJAgAAAAA6On2 +AQBFD1f/ZEyLNCX4////SIuEJNgAAADoD3oAAEiFwA+FCAYAAIM9T5IIAAB0SZCQSI0F7JEIAOjv +Yf3/SIuEJNgAAAAx2w8fRAAA6BtxAABIiYQkqAAAAJCQSI0FwpEIAOilY/3/SIuEJKgAAABIhcAP +haQFAACLFR5zCwCF0nQhixUYcwsADx9AAIXSdg9IixV9kQgASIXSD5XC6wYx0usCMdKE0nQWMcCQ +6HtM//9IiUQkaEiFwA+F7wQAAEiLlCTIAAAASItyMIs9w3ILAIC+FAEAAAB0B74BAAAA6xiLNYGR +CABEiwV2kQgA0eZEKcc5/kAPksZAhPZ1DEiLTCRYSIt8JFDrYEiLSjCAuRQBAAAAdRfGgRQBAAAB +uQEAAABIjR0+kQgA8A/BC0iLRCRY6HQIAABIhcAPhV0EAABAhPYPhRD+//9mkEiF/3QRSItUJFBI +hdJ0D0g513wK6wVIi1QkUEiJ14M9F3ILAAB0XkiLlCTYAAAASIuymBYAAEiF9nQcSIN+EAB1DkiL +sqAWAABIg34QAHQHuAEAAADrOEiLNQJ9CwBmkEiF9nUZizV9fQsAOTV7fQsAdge4AQAAAOsVMcDr +EbgBAAAA6wpIi5Qk2AAAADHASIl8JFBIiUwkWITAdAiQZpDpIgYAAEiLDaSHCABIiYwkuAAAAEiL +FZ2HCABIiZQkgAAAAEiLHZaHCABIiZwkiAAAAEiLNfeHCABIibQkoAAAAEiLPfCHCABIiXwkcEyL +BeyHCABMiUQkeEyLDTCICABMiYwkmAAAAEyLFSmICABMiVQkQEyLHSWICABMiVwkSJCQSI0Fr48I +AOiyX/3/gz2LkAgAAHURSIuEJNgAAACDuNAmAAAAdBOQkEiNBYePCADoamH9/+mz/P//gz3OjwgA +AA+F4gQAAOjzVgAASIuMJNgAAABIOcgPhTgFAABIicjo2nAAAJCQSI0FSY8IAOgsYf3/SIuMJMgA +AABIi1EwD7aaFAEAAIhcJD+E2w+E2QAAAMaCFAEAAAC6/////0yNDVaPCADwQQ/BEf/KhdIPjNAE +AABIi4QkuAAAAEiLnCSAAAAASIuMJIgAAABIi7wkoAAAAEiLdCRwTItEJHiQ6NsJAABIhcB0LejR +VAAASIuMJMgAAABIi1EwxoIUAQAAAboBAAAASI0d7o4IAPAPwRPp1/v//+iECwAADx9AAEiFwA+F +qQIAAEiLhCS4AAAASIucJIAAAABIi4wkiAAAAEiLvCSYAAAASIt0JEBMi0QkSEyLTCRQ6KMKAABI +i4wkyAAAAA+2XCQ/6wVIi0QkUIsVuW8LAIXSdCeLFbNvCwCF0ncFSIXAdBQx0kiNNRWOCABIhxZI +hdIPlcLrBjHS6wIx0oTSD4Q7AwAASI0V/Y0IAEiJxkiHAkiLeTBIg7/QAAAAAA+FpwMAAIC/FAEA +AAAPhYkDAABIhfZ0WUiLfCRYSIX/dT5IibQkkAAAAJDoiw0CAEUPV/9kTIs0Jfj///9IiwwkSI0V +o40IAA+2XCQ/SIu0JJAAAABIic9Ii4wkyAAAAEgp/kiF9r8AAAAASA9M9+sJMf9Ix8b/////SIM9 +kW8LAABID0X3SInw6G1I//9IiUQkYDHJSI0VT40IAEiHCugXDQIARQ9X/2RMizQl+P///0iLDCRI +jRUnjQgASIcKSIM9TG8LAAB0F0iDfCRgAHUP6EXt//8PH0QAAOlJ+v//kJBIjQUKjQgA6A1d/f/o +qG8AAEiJhCTAAAAAkJBIjQXvjAgA6NJe/f9Ii4QkwAAAAEiFwA+E+AEAAJDo21IAAEiLTCRgSIXJ +D4VzAQAAD7ZMJD+EyXQpSIuMJMgAAABIi1EwxoIUAQAAAboBAAAASI014YwIAPAPwRaQ6cn5//9I +i4wkyAAAAOm8+f//SIusJOAAAABIgcToAAAAw5CQSImEJNgAAABIhcB0DEiLiKAAAABIiUwkaEiN +RCRo6JkLAABIi4Qk2AAAALsEAAAAuQEAAADo4sv//4A9awIJAAB0D0iLhCTYAAAAMdvoyl4BAEiL +hCTYAAAAMdtIi6wk4AAAAEiBxOgAAADDMdtIi6wk4AAAAEiBxOgAAADDSIusJOAAAABIgcToAAAA +w0iJhCTAAAAASImcJNAAAADo2VEAAEiLjCTIAAAASItJMMaBFAEAAAG5AQAAAEiNFfaLCADwD8EK +SIuMJMAAAABIx4GIFgAAAwAAAEiLhCTQAAAAuwQAAAC5AQAAAOgsy///gD21AQkAAHQPSIuEJNAA +AAAx2+gUXgEASIuEJNAAAAAx20iLrCTgAAAASIHE6AAAAMOQkEiJjCTYAAAASIXJdAxIi5GgAAAA +SIlUJGBIjUQkYOh1CgAASIuEJNgAAAC7BAAAALkBAAAADx8A6LvK//+APUQBCQAAdA9Ii4Qk2AAA +ADHb6KNdAQBIi4Qk2AAAADHbSIusJOAAAABIgcToAAAAw0iNRCRgDx9AAOgbCgAA6zlIhcB0DYsV +PmwLAIXSD5XC6wIx0oTSdCFIixWqiggASIXSdAVIOcJ+EOirRP//SIuMJMgAAAAPHwDou+r//+nE +9///Mdvoz2kAAEiJhCSwAAAAkJBIjQV2iggA6Flc/f9Ii4QksAAAADHbSIusJOAAAABIgcToAAAA +w0iNBSwcAwC7IwAAAOhuhP//SI0FuwwDALscAAAAZpDoW4T//0iNBRsYAwC7IQAAAOhKhP//SI0F +GP0CALsVAAAA6DmE//9IizVSbAsASIX2dCZIifBIwf4TSMHmA5BMiwZMjQ03bAsA8E0PsQFBD5TA +RYTAdNDrCUyNDSBsCwAx9kiF9g+Emvn//0jHgogWAAADAAAASItGEEiJhCTYAAAAuwQAAAC5AQAA +AOhPyf//gD3Y/wgAAHQPSIuEJNgAAAAx2+g3XAEASIuEJNgAAAAx20iLrCTgAAAASIHE6AAAAMPo +uO0BAOlz9v//zMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhskAAABIg+wYSIlsJBBIjWwkEIM9kYkI +AACQdQ1Ji04wSIuJ0AAAAOsPuAEAAABIi2wkEEiDxBjDi5HwBQAAi5n0BQAASIux+A0AAIu59AUA +ADnfdeM503VlkEiF9nVfiw1lagsAhcl0DYsNX2oLAIXJD5fB6wQxyWaQhMl0NEiDPbyICAAAdCox +wOjLQ///SIlEJAhIhcB0GUiNRCQI6PcHAAC4AQAAAEiLbCQQSIPEGMMxwEiLbCQQSIPEGMO4AQAA +AEiLbCQQSIPEGMPoyOwBAOkj////zMzMSTtmEA+GRgMAAEiD7HhIiWwkcEiNbCRwSYtWMEiLktAA +AABIiVQkaDHJMdsx9usQSP/BTIniSIn4Dx+AAAAAAEiD+QQPjcEAAABNi0YwRYuIIAEAAEWLkCQB +AABFiZAgAQAARYnLQcHhEUUx2UWJ00UxykHB6QdFMdFFidpBwesQRTHLRYmYJAEAAEeNBBNEiw2K +gAgATIsVk4AIAEyLHYSACABFhckPhJ0CAABIicdEicBJidQx0kH38Q8fRAAARYXSD4R8AgAARInA +QYnQMdJB9/KJ0Ek5wg+GXgIAAEiJTCQoQYsUg8dEJFAAAAAARIlMJFREiUQkWIlUJFxIg/kDD5TC +iFQkHesjSInBSInfMcAx20iLbCRwSIPEeMNBidEx0kH38IlUJFhEicpEi0QkUEQ5RCRUD4Ty/v// +gz0biAgAAA+FugEAAEyLBaZ+CABMiw2XfggAi0QkWA8fAEk5wA+GzAEAAE2LBMFNOcQPhD8BAABI +g/kDdS5Miw1VfwgATIsVRn8IAEGJw8HoBUk5wQ+GlAEAAE2NDIJFiwlFD6PZQQ+SwesDRTHJRYTJ +D4SJAAAATIlEJGBIiVwkSECIdCQeTInASIn76BANAABIhdt0EkiLVCRISIXSdBCQSDnTfArrBUiL +VCRISInThMl1Bw+2TCQe6yxIiUQkIEiJXCRASItEJGjocm4AAEiFwA+F1QAAAEiLRCQgSItcJEC5 +AQAAAA+2VCQdTItEJGBMi2QkaEiJx4nOSItMJChMiw0yfggATIsVI34IAItEJFhBicPB6AVmDx+E +AAAAAABJOcEPhrwAAABNjQyCRYsJRQ+j2XI/SIlcJDhAiHQkH0iJfCQwTIngTInDidHobnEAAEiF +wHU6SItMJCgPtlQkHUiLXCQ4D7Z0JB9Ii3wkMEyLZCRo/0QkUItEJFgDRCRcRItEJFRFhcAPhVD+ +///rTTHbSItMJDBIi3wkOA+2dCQfSItsJHBIg8R4w0iLTCQgSIt8JEAPtnQkHkiLbCRwSIPEeMMx +wEiJ+UiJ374BAAAAMdtIi2wkcEiDxHjDZpDoe1r//0yJyejz8AEATInJ6OvwAQBMicHo4/ABAEyJ +0ejb8AEA6FZa///oUVr//5BIiUQkCOhm6QEASItEJAiQ6Zv8///MzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMxJO2YQD4biAAAASIPsIEiJbCQYSI1sJBhIiUQkKEiJfCRAMcnrBEmNSAFIOct+SUiL +FMhJicjB6QUPH4AAAAAASDnOD4aYAAAATI0Mj0WLCUUPo8FzBDHS6xjrKA8fQABFOcp1CE2F2w+U +wusCMdKD8gGE0nSw6y8xwEiLbCQYSIPEIMNEi4rwBQAARIuS9AUAAEyLmvgNAABEi6L0BQAARTnU +dd/ruZCQSI0FSIQIAOhLVP3/6OZmAABIiUQkEJCQSI0FMIQIAOgTVv3/SItEJBBIhcB0oUiLbCQY +SIPEIMOJyEiJ8ei17wEAkEiJRCQISIlcJBBIiUwkGEiJfCQgSIl0JChMiUQkMOgx6AEASItEJAhI +i1wkEEiLTCQYSIt8JCBIi3QkKEyLRCQw6c7+///MzMzMzMzMzMzMzMzMzEiD7BhIiWwkEEiNbCQQ +SIlEJCBIiXwkODHJ6wdJjUgBSYnRSDnLflpIixTISYnIwekFSDnOdlhMjRSPRYsSRQ+jwnM4hAJM +i5JoFgAASIuScBYAAE2F0nQPSIXSdAdmkEw50nwDTInSSIXSdApNhcl0qkk50X+lTInK66BMicqQ +65pMichIi2wkEEiDxBjDichIifHox+4BAJDMzMzMzMxJO2YQD4ZpAQAASIPsIEiJbCQYSI1sJBiL +DV5kCwBmkIXJD4SjAAAASIsNcW8LAEiFyXUfiw3ubwsAZg8fRAAAOQ3mbwsAdge4AQAAAOsJMcDr +BbgBAAAAhMB0YZCQSI0Fs4IIAOi2Uv3/6FFlAABIhcB0LYM9AWQLAAB0AutdkOgbZAAAkJBIjQWK +gggA6G1U/f8xwDHbSItsJBhIg8Qgw5CQSI0FboIIAOhRVP3/McAx20iLbCQYSIPEIMMxwDHbSIts +JBhIg8QgwzHAMdtIi2wkGEiDxCDDSInwSIsNfWQLAEiFyXQqSInKSMH5E0jB4QOQSIsZSInGSInQ +SI09XGQLAPBID7EfD5TChNJ0yesFSInGMclIhcl0MEiJdCQQSIlMJAiQkEiNBemBCADozFP9/0iL +TCQISItZEEiLRCQQSItsJBhIg8Qgw0iJ8OhMYwAAkJBIjQW7gQgADx8A6JtT/f8xwDHbSItsJBhI +g8Qgw+jo5QEA6YP+///MzMxJO2YQdj5Ig+wISIksJEiNLCRIiw1vgQgASIXJdRhIiw1rgQgASIXJ +dAVIOcF+DOhsO///6wXoRej//0iLLCRIg8QIw0iJRCQI6JLlAQBIi0QkCOurzMzMzMzMzMzMzMxJ +O2YQdnFIg+wYSIlsJBBIjWwkEEmLTjCAuRQBAAAAkHRDxoEUAQAAALn/////SI0VR4EIAPAPwQr/ +yYXJfBCQ6Nvn//9Ii2wkEEiDxBjDSI0F0Q4DALshAAAADx9EAADo+3r//0iNBfoKAwC7HwAAAOjq +ev//kOgE5QEA64LMzEk7ZhAPhuwCAABIg+x4SIlsJHBIjWwkcEiLEA8fRAAASIXSdBZIiYQkgAAA +AIA9vPYIAAB0D+mxAgAASItsJHBIg8R4w0iLEEiJVCRoSInRMdsx9us4SIlUJFhIidC7BAAAALkB +AAAA6PO///9Ii1wkWEiLk6AAAABIi3QkGEj/xkiLhCSAAAAASItMJGhIiXQkGEiF0nW+RA8RfCQ4 +kEiJTCQ4kEiJXCRASMcAAAAAAEmLVjBIi4LQAAAADx9AAEiFwHQVixUtgAgARA8RfCRIidIxyemt +AAAAkJBIjQXVfwgA6NhP/f+QSItMJEBIi1QkOEiFyXQxSInLSMeBoAAAAAAAAABIiw37fwgASIXJ +dAlIiZGgAAAA6wdIiRXefwgASIkd338IAEiLTCQYAQ3cfwgARA8RfCQ4kJBIjQV1fwgA6FhR/f+Q +SItMJBjrFkiJTCQwMcAx2+ii4P//SItMJDBI/8lIhcl0CYM9hn8IAAB13EiLbCRwSIPEeMOQTIlM +JFBI/8FIOdF9WkiLfCQ4SIX/dFCQkEiF/3QaTIuHoAAAAEyJRCQ4TYXAdQlIx0QkQAAAAACQSMeH +oAAAAAAAAABMi0QkUE2FwHQMSYn5SYm4oAAAAOulSYn4SIl8JEhNicHrmEiFyQ+OgwAAAEiJRCRo +SIlMJCCQkEiNBbZ+CADouU79/5BIi0wkUEiLVCRISIXJdDFIictIx4GgAAAAAAAAAEiLDdx+CABI +hcl0CUiJkaAAAADrB0iJFb9+CABIiR3AfggASItMJCABDb1+CABEDxF8JEiQkEiNBVZ+CADoOVD9 +/5BIi0wkIEiJyOs6SIN8JDgAdA1IjVwkOEiJ8ej5ZAAASItsJHBIg8R4w0iJTCQoMcAx2+hh3/// +SItMJChI/8lIi0QkIEiFyXQJgz1AfggAAHXXSIt0JBhIKcZIi0QkaOupSIlUJGBIidAx2+iIUAEA +SItMJGBIi5GgAAAASIuEJIAAAABIhdJ12OlP/f//SIlEJAgPH0QAAOj74QEASItEJAjp8fz//8zM +zMzMzMzMzMzMzMzMzMzMSTtmEA+G0QMAAEiD7EBIiWwkOEiNbCQ4SYtOMIO5CAEAAAAPhaADAABM +iXQkIEiDuWgBAAAAdB/ow+T//0iLRCQgSItIMEiLiWgBAAAx20iJyOjJ6P//SItMJCBIi1EwgLoY +AQAAAHQP6UoDAADo7eb//0iLTCQgSItRMEiLgtAAAACEAMaACCcAAACDPe19CAAAD4X8AgAAg7jQ +JgAAAHQUSIlEJDDo9M3//0iLRCQwSItMJCBIi1EwgLoUAQAAAHQpSIO4+A0AAAAPhdYCAACLkPAF +AABmDx+EAAAAAAA5kPQFAAAPhbsCAAAx2+gNAwAAgD3W8ggAAHUJgD3O8ggAAHQ96JY5AQBIhcB0 +LUiJRCQYuwQAAAC5AQAAAGaQ6Bu8//9Ii0QkGDHb6A9PAQBIi1QkGEiF0kiJ0A+VwZDrBDHAMclI +hcB1NoM9ql0LAAB0LYhMJBVIi0wkIEiLUTBIi5rQAAAASI0FsWQLAOhMT/7/SIXAD5XBD7ZUJBUJ +0YhMJBdIhcB0B0iLVCQg63ZIi1QkIEiLcjBIi7bQAAAAi3YQafYVlwzBgf5TXDIEd1WDPTd8CAAA +fkyQkEiNBdR7CADo10v9/0iLTCQgSItRMEiLgtAAAAC7AQAAAGaQ6PtaAABIiUQkKJCQSI0FpXsI +AOiITf3/D7ZMJBdIi1QkIEiLRCQoSIXAdAQx2+sfSItKMEiLgdAAAAAPH0QAAOh7YwAAD7ZMJBdI +i1QkIEiFwHUP6Gfo//8PtkwkF0iLVCQgiFwkFkiJRCQoSItyMIC+FAEAAAB0GegC+v//SItEJCgP +tkwkF0iLVCQgD7ZcJBaAPX97CAAAdQQx9usjuwEAAADo14sBAIPwAQ+2TCQXSItUJCAPtlwkFonG +SItEJChAhPYPhLAAAACQkEiNBeR6CADo50r9/4A9OHsIAAB1B7gBAAAA6w9Ii0QkKLsBAAAA6IiL +AQCEwHVdkEiLTCQoSMeBoAAAAAAAAABIixUUewgASIXSdAxIictIiYqgAAAA6w6QSInKSIkN8HoI +AEiJ05BIiR3teggA/wXveggAkJBIjQVueggA6FFM/f9Ii0wkIOk//f//kJBIjQVWeggA6DlM/f9I +i0QkKA+2TCQXSItUJCAPtlwkFoTJdBWQ6Bvh//9Ii0QkKEiLVCQgD7ZcJBZIg7joAAAAAA8fQAAP +hej8///oleX//0iLbCQ4SIPEQMPohuT//0iLRCQgSInB6dH8//9IjQUaCwMAuyIAAADoCHT//0iN +BR/kAgC7EAAAAOj3c///SI0FqvICALsXAAAA6OZz//+QDx9EAADo+90BAOkW/P//zMzMzMzMzMzM +zMzMzMzMzMzMzMzMzEk7ZhAPhqUBAABIg+xASIlsJDhIjWwkOIQASIuQaBYAAEiLsHAWAABIhdJ0 +E0iF9nQFSDnWfAlIidZmDx9EAABIhfYPhK0AAABIiUQkSEiF23UmSIl0JBiQ6AL5AQBFD1f/ZEyL +NCX4////SIscJEiLRCRISIt0JBhIiVwkKGaQSDnzfThJi1YwSIuS0AAAAEg50HUWi5D8JgAAi7j4 +JgAAidLB7wJIOfp/EkiJ2EiJ8zHJSItsJDhIg8RAw0iNiNgmAABIiUwkMJBIicjozkj9/0iLRCRI +SIO46CYAAACQfwYx0jHJ62pIi1wkKOgOJAEAMcDrFkiJ2DHbMclIi2wkOEiDxEDDuAEAAABIi0wk +SEiDuegmAAAAfi6IRCQXSInISItcJCjo1CcBAEiFwHTWuQAAAABID0/ID7ZEJBdIicpIi0wkSOsC +MdKJw0iJyInZiEwkF0iJVCQgSYteMEiLm9AAAABIOdh1H4uY/CYAAInbSIuw6CYAAEjB7gJmkEg5 +834F6NYrAQCQkEiLRCQw6OpJ/f9Ii0QkKEiLXCQgD7ZMJBdIi2wkOEiDxEDDSIlEJAhIiVwkEOgi +3AEASItEJAhIi1wkEOkz/v//zMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2JkiD7BBIiWwkCEiNbCQI +kEiJ2OiDSf3/uAEAAABIi2wkCEiDxBDDSIlEJAhIiVwkEOjF2wEASItEJAhIi1wkEOu5zMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhk0BAABIg+woSIlsJCBIjWwkIEiJRCQYTIl0JBCAPWft +CAAAdB9Ji04wD7aBkAIAAEiLmZgCAAAPHwDoO0kBAEiLRCQYuwIAAAC5BAAAAOintv//SYtWMEiL +ksAAAACQkDH2SIlyMEmLVjCQMfZIibLAAAAASItUJBBIi3IwSIu+gAIAAEiF/w+EtQAAAEiLD0iL +nogCAABIi0QkGEiJ+v/RSItMJBBIi3EwhAaDPedZCwAAdQ1Ix4aAAgAAAAAAAOsOSI2+gAIAADHS +6KreAQBIi1EwhAKDPb1ZCwAAdQ1Ix4KIAgAAAAAAAOsTSI26iAIAADHSDx9EAADoe94BAITAdT+A +PYDsCAAAdBNIi0QkGLsCAAAADx9AAOjbSAEASItEJBi7BAAAALkBAAAA6Me1//9Ii0QkGLsBAAAA +6Jjh///oc/j//0iLbCQgSIPEKMNIiUQkCA8fQADoO9oBAEiLRCQI6ZH+///MzMzMzMzMzMzMzMzM +zMzMzEk7ZhAPhuIBAABIg+wwSIlsJChIjWwkKEiJRCQ4i5CQAAAAD7ryDIP6Ag+FnwAAALsCAAAA +uQEAAADoQbX//0mLVjBIi5LAAAAAkJAx9kiJcjBJi1YwkDH2SImywAAAAJCQSI0Fb3UIAOhyRf3/ +kJBIi1QkOEjHgqAAAAAAAAAASIs1oXUIAEiF9nQMSInRSImWoAAAAOsLkEiJ0UiJFX11CACQSIkN +fXUIAP8Ff3UIAJCQSI0FHnUIAOgBR/3/kOh79///SItsJChIg8Qww5BMiXQkIIuIkAAAAIlMJBRI +i5CYAAAASIlUJBjocIf//0iNBffeAgC7EAAAAA8fQADoe5D//0iLRCQ46PGP//9IjQXCzgIAuwcA +AAAPH0QAAOhbkP//SItEJBjoUY7//0iNBY/iAgC7EwAAAA8fRAAA6DuQ//+LRCQUicDoMI3//+iL +if//6IaH//9Ii0QkIIuIkAAAAIlMJBBIi5CYAAAASIlUJBjo5ob//0iNBT3eAgC7EAAAAOj1j/// +SItEJCDoa4///0iNBTzOAgC7BwAAAOjaj///SItEJBjo0I3//0iNBfvhAgC7EwAAAA8fQADou4// +/4tEJBCJwOiwjP//6AuJ///oBof//0iNBfPWAgC7DAAAAOgVbv//kEiJRCQI6CrYAQBIi0QkCA8f +RAAA6fv9///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdjVIg+wYSIlsJBBIjWwkEIA9 +1ekIAAB0D0iJRCQI6FlFAQBIi0QkCOiv/f//SItsJBBIg8QYw0iJRCQI6LvXAQBIi0QkCOu0zMzM +zMzMzMzMzMzMzMzMzMzMzMxJO2YQD4alAAAASIPsMEiJbCQoSI1sJChIiUQkOEiLUDCDuggBAAAA +dSCDuvAAAAAAdRdIg7oAAQAAAHUNSIuS0AAAAIN6BAF0H0iNSDhIiQwk6GnVAQBFD1f/ZEyLNCX4 +////SItEJDiAPSDpCAAAdC5Ji1YwkEiLktAAAABJiZbgAAAAuBEAAAC7AQAAADHJMf9Iif7oRTEB +AEiLRCQ46Nv8//9Ii2wkKEiDxDDDSIlEJAjo59YBAEiLRCQIZpDpO////8zMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzEk7ZhB2WUiD7DhIiWwkMEiNbCQwgD2V6AgAAHQzSIlEJChJi1YwkEiLktAA +AABJiZbgAAAAuBIAAAC7AQAAADHJMf9Iif7otTABAEiLRCQo6Ev8//9Ii2wkMEiDxDjDSIlEJAjo +V9YBAEiLRCQI65DMzMzMzMzMzMzMzMzMzMzMSTtmGA+GYgIAAEiD7EBIiWwkOEiNbCQ4SIlEJEiA +PQzoCAAAdBG4FAAAADHb6O5DAQBIi0QkSJCLkJAAAAAPuvIMg/oCD4X6AAAAxoCwAAAAGYC4tAAA +AAB0I0iLSEBIicjo+fwAAEiFwA+EvwAAAA+2UCn2wgJ1W0iLRCRIuwIAAAC5CRAAAOgTtf//SYtW +MEiLksAAAACQkDH2SIlyMEmLVjCQMfZIibLAAAAASItEJEi7CRAAALkJAAAADx8A6Dus///olvP/ +/0iLbCQ4SIPEQMPoZwMBAEiJRCQwSIlcJCDomIP//0iNBSEGAwC7JQAAAOinjP//SItEJDBIi1wk +IOiYjP//SI0Fot0CALsSAAAA6IeM///o4oP//0iNBXPYAgC7DwAAAOjxav//SI0F/uMCALsVAAAA +Dx9EAADo22r//5BMiXQkKIuIkAAAAIlMJBRIi5CYAAAASIlUJBjoGoP//0iNBaHaAgC7EAAAAOgp +jP//SItEJEgPH0AA6JuL//9IjQVsygIAuwcAAADoCoz//0iLRCQYDx9EAADo+4n//0iNBTneAgC7 +EwAAAOjqi///i0QkFInADx9AAOjbiP//6DaF///oMYP//0iLRCQoi4iQAAAAiUwkEEiLkJgAAABI +iVQkGOiRgv//SI0F6NkCALsQAAAADx9EAADom4v//0iLRCQo6BGL//9IjQXiyQIAuwcAAAAPH0QA +AOh7i///SItEJBjocYn//0iNBZzdAgC7EwAAAA8fRAAA6FuL//+LRCQQicDoUIj//+irhP//6KaC +//9IjQWT0gIAuwwAAADotWn//5BIiUQkCOgK+wEASItEJAgPH0QAAOl7/f//zMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMSTtmEA+GqgAAAEiD7DhIiWwkMEiNbCQwSIlEJECAPWzlCAAAdC5Ji1Yw +kEiLktAAAABJiZbgAAAAuBIAAAC7AQAAADHJMf9Iif7okS0BAEiLRCRASItQMEiLktAAAABIiVQk +KLsCAAAAuQEAAADoja7//0mLVjBIi5LAAAAAkJAx9kiJcjBJi1YwkDH2SImywAAAAEiLRCQoSItc +JEAxyeh7UgAA6Bbx//9Ii2wkMEiDxDjDSIlEJAjo4tIBAEiLRCQI6Tj////MzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMxJO2YQdklIg+wwSIlsJChIjWwkKIA9leQIAAB0GZC4DwAAAEjHw/////8xyTH/ +SIn+6MosAQBIjQU7GgMADx8A6LvQAQBIi2wkKEiDxDDD6GzSAQDrqszMzMzMzMzMzMxJO2YQD4Zg +AgAASIPsMEiJbCQoSI1sJChIiUQkOEyJdCQguwIAAAC5BgAAAOiPrf//SItEJDgx2+ijfgEADx8A +hMB0ELn/////SI0V+G0IAPAPwQpIi1wkOIQDgz3+UAsAAHUKSMdDMAAAAADrC0iNezAxyein1QEA +SIuL6AAAAEjHg+gAAAAAAAAASItUJCBIi3IwSMeGaAEAAAAAAADGg7IAAAAAxoO1AAAAAIM9rFAL +AAB1EkjHQygAAAAASMdDIAAAAADrFkiNeygx9uit1QEASI17IDH26KLVAQBEDxG7AAEAAIM9c1AL +AAB1DUjHg/gAAAAAAAAA6w5Ijbv4AAAAMfbodtUBAMaDsAAAAACDPUhQCwAAdRVIx4OIAAAAAAAA +AEQPEbtoAQAA6yhIjbuIAAAAMfboQ9UBAEiNu2gBAADoN9UBAEiNu3ABAAAx9ugp1QEAgz0eTgsA +AGaQdEVIg7uAAQAAAH47SIs1tVULAGZID27GSIuzgAEAAA9XyfJIDyrO8g9ZyPJIDyzxSI09WVUL +APBID8E3SMeDgAEAAAAAAABJi3YwSIu2wAAAAJCQMf9IiX4wSYt2MJAx/0iJvsAAAABIi3Iwi750 +AgAAhf91UkiJTCQYSIuG0AAAAOiZFwAASItMJBhIhcl0KEiLRCQgSItAMEiLAIQASIPAOEiJBCTo +dM4BAEUPV/9kTIs0Jfj////oYu7//0iLbCQoSIPEMMOJfCQU6G9+//9IjQWX4wIAuxcAAAAPHwDo +e4f//4tEJBTocoT//+jNgP//6Mh+//9IjQU66wIAuxsAAADo12X//5BIiUQkCOjszwEASItEJAjp +gv3//8zMSIPsGEiJbCQQSI1sJBBJi04wTInySDkRdDpmDx9EAABIOVFQdC5IiUJASIlaOEjHQmAA +AAAASMdCWAAAAABIg3pQAHQF6DaZ//9Ii2wkEEiDxBjDSI0FXe8CALscAAAA6Ftl//+QzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMxIg+wwSIlsJChIjWwkKEiJRCQ4SIlcJEBMiXQkEEmLTjD/gQgB +AABJx0YQ3vr//0HGhrcAAAAB6ET///9Ii0wkQEiLRCQQSIlIcEiLVCQ4SIlQeLsCAAAAuQMAAAAP +HwDoW6r//0iLTCQQSItRcEg5EXcGSDlRCHM3RA8RfCQYSI0FmgEAAEiJRCQYSIlMJCBIjUQkGEiJ +BCTogs0BAEUPV/9kTIs0Jfj///9Ii0wkEIA9meAIAAB0MUiNBaAXAwBIiQQk6FfNAQBFD1f/ZEyL +NCX4////SItEJDhIi1wkQOib/v//SItMJBCLBSBrCACFwHQ2SI0FfRUDAEiJBCSQ6BvNAQBFD1f/ +ZEyLNCX4////SItEJDhIi1wkQA8fQADoW/7//0iLTCQQSItBMEiLgNAAAACEAIO40CYAAABmkHQx +SI0FrxYDAEiJBCTozswBAEUPV/9kTIs0Jfj///9Ii0QkOEiLXCRA6BL+//9Ii0wkEEiLQTBIi5DQ +AAAAi1IUiZCkAgAAxoG7AAAAAUiLQTBIi4DQAAAASMdAOAAAAABIi1EwSInDSImC4AAAAEiLQTBI +x4DQAAAAAAAAALgCAAAAh0MEgz0zaggAAHQxSI0FmhQDAEiJBCToQcwBAEUPV/9kTIs0Jfj///9I +i0QkOEiLXCRA6IX9//9Ii0wkEEiLQTD/iAgBAABIi2wkKEiDxDDDzMzMzMzMzMzMzMzMSTtmEA+G +sgAAAEiD7DBIiWwkKEiNbCQoSItCCEiLSHBIiUwkIEiLEEiJVCQYSItACEiJRCQQ6EV7//9IjQV3 +5gIAuxoAAADoVIT//0iLRCQg6KqC//9IjQXcvwIAuwIAAADoOYT//0iLRCQY6I+C//9IjQWsvwIA +uwEAAAAPHwDoG4T//0iLRCQQ6HGC//9IjQW7vwIAuwIAAAAPH0QAAOj7g///6FZ7//9IjQWXywIA +uwwAAADoZWL//5APH0AA6NvLAQDpNv///8zMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdlhIg+wQ +SIlsJAhIjWwkCJBIjQX8ZwgADx9AAOj7N/3/iw3laAgAhcl0FzHJSI0V2GgIAIcKSI0F12gIAOh6 +Ov3/kJBIjQXJZwgA6Kw5/f9Ii2wkCEiDxBDDZpDo+8sBAOuZzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzEk7ZhAPhrUAAABIg+wYSIlsJBBIjWwkEEmLTjBIi4ngAAAAkEiJTCQIkEiNBWdnCADoajf9 +/4M9R2gIAAB+GbgCAAAASItMJAi6AwAAAPAPsVEED5TC6whIi0wkCDHSkITSdEOAPWXdCAAAdBdI +icjoyzsBAEiLRCQI6CE0AQBIi0wkCP9BFIsN92cIAI1R/4kV7mcIAIP5AXUMSI0F5mcIAOiZOf3/ +kJBIjQXoZggA6Ms4/f9Ii2wkEEiDxBjDkOgbywEA6Tb////MzMzMzMzMzMzMzMzMzMzMzMzMzMzM +SIPsYEiJbCRYSI1sJFhMiXQkEEmLTjD/gQgBAABBxoa3AAAAAUnHRhDe+v//SYtOMEiLkdAAAACL +UhSJkaQCAABBxoa7AAAAAUmLTjBIi4nQAAAA/0EUSItEJGBIjVwkaOi6+v//SItEJBBIi0g4SIlI +cEiLUEBIiVB4Dx9EAABIOQh3Bkg5SAhzZEiLUDhIx0QkMAAAAABIjVwkOEQPETtIjVwkSEQPETtI +jR3vAQAASIlcJDBIjVwkaEiJXCQ4SIlUJEBIiUwkSEiJRCRQSI1MJDBIiQwk6OPIAQBFD1f/ZEyL +NCX4////SItEJBC7AgAAALkDAAAA6GKl//9Ii1QkEEiLcnBIOTJ3Bkg5cghzRUQPEXwkGEjHRCQo +AAAAAEiNBXgAAABIiUQkGEiNRCRoSIlEJCBIiVQkKEiNTCQYSIkMJOh2yAEARQ9X/2RMizQl+P// +/0iNBcIQAwBIiQQk6FnIAQBFD1f/ZEyLNCX4////SItEJGBIjVwkaGaQ6Jv5//9Ii0QkEEiLQDD/ +iAgBAABIi2wkWEiDxGDDzMxJO2YQD4bjAAAASIPsQEiJbCQ4SI1sJDhIi0IQSItKCEiJTCQQSItQ +OEiJVCQoSItYcEiJXCQgSIswSIl0JBhIi0AISIlEJDDoU3f//0iNBVfuAgC7HwAAAOhigP//SItE +JBDouH7//+hzef//SItEJCjoqX7//+hkef//SItEJCDomn7//0iNBcy7AgC7AgAAAOgpgP//SItE +JBgPH0AA6Ht+//9IjQWYuwIAuwEAAADoCoD//0iLRCQwDx9EAADoW37//0iNBaW7AgC7AgAAAOjq +f///6EV3//9IjQVWzwIAuxEAAADoVF7//5DozscBAOkJ////zMzMzMzMzMzMSTtmEA+G4wAAAEiD +7EBIiWwkOEiNbCQ4SItCIEiLShBIiUwkGEiLWhhIiVwkEEiLUghIiVQkIEiLMEiJdCQoSItACEiJ +RCQw6FN2//9IjQVX7QIAux8AAADoYn///0iLRCQg6Lh9///oc3j//0iLRCQY6Kl9///oZHj//0iL +RCQQ6Jp9//9IjQXMugIAuwIAAADoKX///0iLRCQoDx9AAOh7ff//SI0FmLoCALsBAAAA6Ap///9I +i0QkMA8fRAAA6Ft9//9IjQWlugIAuwIAAADo6n7//+hFdv//SI0FVs4CALsRAAAA6FRd//+Q6M7G +AQDpCf///8zMzMzMzMzMzEk7ZhB2U0iD7DBIiWwkKEiNbCQogD012QgAAHQokLgcAAAAuwEAAAAx +yTH/SIn+6GwhAQBJi1YwSIuC0AAAAJDoezcBAOhWKgAA6NHG//9Ii2wkKEiDxDDD6ALHAQBmkOue +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSIPsQEiJbCQ4SI1sJDiBPWxjCAD///9/D4TD +AAAATIl0JBBIhcB0OYN4BAJ1M0iJwbgCAAAAMdLwD7FRBA+UwoTSdCBIicjotCgAAOhvAQAAuAEA +AABIi2wkOEiDxECQw0iJwUiDPWRiCAAAdFjGRCQPAEQPEXwkGEQPEXwkKEiNBXoAAABIiUQkGEiN +RCQPSIlEJCBIiUwkKEiLRCQQSIlEJDBIjUQkGEiJBCTo7sQBAEUPV/9kTIs0Jfj///+AfCQPAHUM +McBIi2wkOEiDxEDDuAEAAABIi2wkOEiDxEDDMcBIi2wkOEiDxEDDzMzMzMzMzMzMzMzMzMzMzMzM +zEk7ZhAPho8AAABIg+woSIlsJCBIjWwkIEiLQhBIiUQkCEiLShhIiUwkEEiLUghIiVQkGOiIAQAA +SItMJBiIAZCEwHQVgD2F1wgAAHQMSItMJAhIhcl1LOsKSItsJCBIg8QowzHA6PY0AQDr7ejP5gEA +RQ9X/2RMizQl+P///0iLTCQISItUJBBIi1owi3EUObOkAgAAdNXryuiixAEAZpDpW////8zMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzEiD7ChIiWwkIEiNbCQgSYtGMEiLiNAAAACLgKQCAACQOUEU +dFZMiXQkCIA939YIAAB0NUQPEXwkEEiNBWAAAABIiUQkEEyJ8EiJRCQYSI1MJBBIiQwk6IXDAQBF +D1f/ZEyLNCX4////SItEJAhIi0AwSIuA0AAAAP9AFEiLbCQgSIPEKMPMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMxJO2YQdjNIg+wQSIlsJAhIjWwkCEiLSghIi0kwSIuB0AAAAOi4NAEAMcDo0TMB +AEiLbCQISIPEEMPoosMBAGaQ677MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4aF +AAAASIPsGEiJbCQQSI1sJBCQSI0FuF8IAOi7L/3/6FZCAABIiUQkCEiFwHQhiw2WYAgAhcl0FzHJ +SI0ViWAIAIcKSI0FiGAIAOgrMv3/kJBIjQV6XwgAZpDoWzH9/0iLRCQISIXAdBTobCUAALgBAAAA +SItsJBBIg8QYwzHASItsJBBIg8QYw+iMwwEA6Wf////MzMzMzMzMSTtmEA+GWwEAAEiD7ChIiWwk +IEiNbCQgSIlEJDC7AwAAALkBAAAA6LSe//9Ji1YwSIuSwAAAAJCQMfZIiXIwSYtWMJAx9kiJssAA +AACQkEiNBeJeCADo5S79/4A9Nl8IAAB0E0iLRCQwuwEAAADojW8BAITAdAfoZEEAAOsCMcBIiUQk +GEiFwHQqiw2gXwgAhcl0HDHJSI0Vk18IAIcKSI0Fkl8IAOg1Mf3/SItEJBgx0utUkJBIi0wkMEjH +gaAAAAAAAAAASIsVu14IAEiF0nQMSInLSImKoAAAAOsOkEiJykiJDZdeCABIidOQSInZSIkdkV4I +AP8Fk14IAEiDuegAAAAAD5XCiFQkF5BIjQUkXggA6Acw/f9Ii0QkGGaQSIXAdBHoFiQAAEiLRCQw +Mdvoisn//w+2RCQXhMB0EpDoW8X//0iLRCQwMdvob8n//+gKvv//6EXg//9Ii2wkIEiDxCjDSIlE +JAjoEcIBAEiLRCQI6Yf+///MzMzMzMzMSTtmEA+GywAAAEiD7DhIiWwkMEiNbCQwiUQkQEiNBR2p +AgDo2Ez9/4tMJECFyXwEMdLrEEiLbCQwSIPEOMNIjVEBifFIg/ogGduJzkiJ0b8BAAAA0+ch3zn+ +f+JIiUQkEEjHRCQYAAAAAMdEJCAAAAAASMdEJCgAAAAASI0NfgAAAEiJTCQYiXwkIEiJRCQoSI1M +JBhIiQwk6CLAAQBFD1f/ZEyLNCX4////SItEJBBIiwhIgcGgAwAASIlIEEjHQBj/////SIsISMcB +AAAAAOlf////iUQkCOgiwQEAi0QkCOkZ////zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2 +NUiD7BhIiWwkEEiNbCQQSItKEEiJTCQIi0II6DutAABIi0wkCEiJAUiJWQhIi2wkEEiDxBjDDx9E +AADoG8ABAOu5zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEiD7EBIiWwkOEiNbCQ4SIlcJFCQSI1M +JAhEDxE5SI1UJBhEDxE6SI1UJChEDxE6SI0VagAAAEiJVCQISI1UJFBIiVQkEEiNVCRYSIlUJBiJ +RCQgTInwSIlEJChIi0QkQEiJRCQwSIkMJOjyvgEARQ9X/2RMizQl+P///0iLbCQ4SIPEQMPMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdlxIg+wwSIlsJChIjWwkKEiLWhCLShhIi3ogSIty +KEiLUghIiwLoUQAAAEmLVjBIi5LQAAAASInDuQEAAABIidDoNj8AAIA9njwLAAB0BehIwv//SIts +JChIg8Qww+j5vgEA65fMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhgPhoEDAABIg+xISIlsJEBI +jWwkQA8fhAAAAAAAhckPhVEDAABIhcAPhCkDAABJi1YwkP+CCAEAAEyJ8oQCg8EHg+H4gfnYBwAA +D432AgAASIlUJDiJTCQUSIl8JDBIiUQkUEiJdCRwSItKMEiLgdAAAABIiUQkKA8fRAAA6BsIAABI +hcB1KrgACAAA6Oz8//9IiUQkIDHbuQYAAADoO5r//0iLRCQg6JGI//9Ii0QkIEiDeAgAD4R5AgAA +i4iQAAAAg/kGD4VZAgAASIlEJCBIi0gISIlMJBhIjVA4uzgAAABIidDoU8wBAItMJBRIY8lIjVEg +SPfaSIPiB0gB0UiLVCQYSCnKSI1K4EiLVCQgSIlKOEiJioAAAABIjQ3bwAEASP/BSIlKQEiJ0UiJ +UUhIi1E4SIPC+EiJ1pBIi3wkUEyLB5BMi0lATIkKSIlxOEyJQUCDPSI9CwAAZpB1BkiJeVDrD0iN +UVBIifhIidfoycABAEiLVCRwSImRKAEAAEiLRCQw6FMCAACDPew8CwAAdQ5Ii0wkIEiJgTABAADr +EUiLTCQgSI25MAEAAOiLwAEASItUJFBIixJIiZE4AQAASItUJDhIi3IwSIu2wAAAAEiF9nQlSIu2 +aAEAAIM9mTwLAAB1CUiJsWgBAADrDEiNuWgBAADoosEBAEiJyDHb6PhpAQCEwHQQugEAAABIjTVQ +WQgA8A/BFkmLVjCLsiABAACLuiQBAACJuiABAABBifDB5hFBMfCJ/kQxx0HB6AdBMfiJ98HuEEQx +xomyJAEAAI0UN0iLRCQgiJC9AAAA9sIHdQfGgLwAAAABuwYAAAC5AQAAAOhimP//SItUJChIi7Lg +BQAASDmy6AUAAHUnuRAAAABIjTWBWAgA8EgPwQ5IjXEBSImy4AUAAEiDwRFIiYroBQAASIuK4AUA +AEiLRCQgSImImAAAAEj/guAFAACAPZXOCAAAdBFIi5g4AQAA6NcnAQBIi0QkIEiLTCQ4SItJMIuR +CAEAAI1a/4mZCAEAAIP6AXUSQYC+sQAAAAB0CEnHRhDe+v//SItsJEBIg8RIw0iNBWbbAgC7HAAA +AOgoUv//SI0FcdsCALscAAAA6BdS//9IjQV4/QIAuzcAAADoBlL//0mLTjDHgfQAAAD/////SI0F +CMkCALsUAAAA6OdR//9IjQVBzwIAuxcAAADo1lH//5BIiUQkCEiJXCQQiUwkGEiJfCQgSIl0JCjo +GOMBAEiLRCQISItcJBCLTCQYSIt8JCBIi3QkKOk7/P//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMTI2kJLD8//9NO2YQD4aYAgAASIHs0AMAAEiJrCTIAwAASI2sJMgDAACLFYU8CwCF0n4KSIO4 +mAAAAAB1EjHASIusJMgDAABIgcTQAwAAw0iJhCTYAwAASIuwMAEAAGaQSIX2dAlIiz5Ii3YI6wQx +9jH/SIl0JFhIibwkiAMAAESNRgFEOcJED0zCSWPISIlMJGBIjQWncAIASInLDx9AAOibmAAASItM +JGBIg/kBD4LrAQAASImEJJgDAABIjVH/SYnQSPfaSMH6P0iD4ihIjRwQSIu8JIgDAABIi3QkWEiN +BVpwAgBMicHo0nD9/0iNfCRoSI1/4GYPH4QAAAAAAEiJbCTwSI1sJPDof8MBAEiLbQCQSMcEJAAA +AABIx8D/////SInDMclIi7wk2AMAADH2TI1EJGhBuWQAAABFMdJFMdvoOC4BAEiJRCRQSInDSInZ +SI0FBjsCAOjhlwAASItMJFBIg/lkumQAAABJichID0/KSI1cJGhIOdh0HkiJhCSQAwAASMHhA+iR +ygEASIuEJJADAABMi0QkUEjHhCSgAwAAAAAAAEiNlCSoAwAARA8ROkiNlCS4AwAARA8ROkiJhCSg +AwAATImEJKgDAABMiYQksAMAAEiLlCTYAwAASIuymAAAAEiJtCS4AwAASIuSKAEAAEiJlCTAAwAA +gz2bOAsAAHUtSIuMJKADAABIi5QkmAMAAEiJCg8QhCSoAwAADxFCCA8QhCS4AwAADxFCGOscSI0F +BW8CAEiLnCSYAwAASI2MJKADAADo0G79/0iNBUkuAgDoRET9/0iLTCRgSIlICEiJSBCDPTA4CwAA +dQ1Ii4wkmAMAAEiJCOsQSInHSIuMJJgDAADo0bwBAEiLrCTIAwAASIHE0AMAAMO4AQAAAOgXwQEA +kEiJRCQI6Oy4AQBIi0QkCOlC/f//zMxJO2YQD4bSAQAASIPsYEiJbCRYSI1sJFiLi5AAAABmkIP5 +Bg+FoQEAAEiLSwhIixNIic5IKdFIgfkACAAAdC5IiUQkaEiJXCRwSInQSInz6MynAABIi0wkcEQP +ETlIx0EQAAAAAEiLRCRoSInLSIuIAA4AAEiJi6AAAACQSImYAA4AAIuICA4AAP/BiYgIDgAAg/lA +fBBEDxF8JBhEDxF8JEgxyesMSItsJFhIg8Rgw//Bg7gIDgAAIHxzkEiLkAAOAABIhdJ0DkiLmqAA +AABIiZgADgAA/4gIDgAASIM6AHUnkEiLXCRISImaoAAAAJBIidNIiVQkSEiDfCRQAGaQda5IiVwk +UOunkEiLXCQYSImaoAAAAJBIidNIiVQkGEiDfCQgAHWJSIlcJCDrgolMJBSQkEiNBd1TCAAPH0QA +AOhbI/3/SItMJFBIi1QkSEiJVCQ4SIlMJEBIhdJ0GkiLFcNTCABIiZGgAAAASItMJDhIiQ2wUwgA +SItMJCBIi1QkGEiJVCQoSIlMJDBIhdJ0GkiLFYhTCABIiZGgAAAASItMJChIiQ11UwgAi0wkFAEN +e1MIAJCQSI0FWlMIAGaQ6Lsk/f/p3P7//0iNBbDXAgC7HQAAAOjlTP//kEiJRCQISIlcJBDo9bYB +AEiLRCQISItcJBDpBv7//8zMzMzMzEk7ZhAPhl4BAABIg+woSIlsJCBIjWwkIEiJRCQw6xaQkEiN +BfBSCADoUyT9/0iLTCQwSInISIuIAA4AAA8fQABIhcl1K0iDPdNSCAAAdQpIgz3RUggAAHQXkEiN +BbdSCADoOiL9/0iLTCQw6awAAACQkEiFyXQOSIuRoAAAAEiJkAAOAAB0Zv+ICA4AAEiDOQB1TUiJ +TCQIRA8RfCQQSI0F2wAAAEiJRCQQSIlMJBhIjUQkEEiJBCTo47QBAEUPV/9kTIs0Jfj///9Ii0Qk +CEiLCEiBwaADAABIiUgQSInBSInISItsJCBIg8QowzHASItsJCBIg8Qow/8NMVIIAJBIi5kADgAA +SImaoAAAAJBIiZEADgAA/4EIDgAAg7kIDgAAIA+N9v7//5BIixXvUQgASIXSdA5Ii5qgAAAASIkd +3FEIAHWzkEiLFdpRCABIhdJ0DkiLmqAAAABIiR3HUQgAdZbpt/7//0iJRCQI6G61AQBIi0QkCOmE +/v//zMzMzEk7ZhB2N0iD7BhIiWwkEEiNbCQQSItKCEiJTCQIuAAIAADomaEAAEiLTCQISIkBSIlZ +CEiLbCQQSIPEGMMPHwDoe7QBAOu5zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhjcBAABI +g+xYSIlsJFBIjWwkUEQPEXwkEEQPEXwkQDHJ6wL/wUiLkAAOAABIhdJ0bJCQSIXSdA5Ii5qgAAAA +SImYAA4AAP+ICA4AAEiDOgB1JZBIi1wkQEiJmqAAAACQSInTSIlUJEBIg3wkSAB1s0iJXCRI66yQ +SItcJBBIiZqgAAAAkEiJ00iJVCQQSIN8JBgAdY5IiVwkGJDrholMJAyQkEiNBYlQCADoDCD9/0iL +TCRISItUJEBIiVQkIEiJTCQoSIXSdBpIixV0UAgASImRoAAAAEiLTCQgSIkNYVAIAEiLTCQYSItU +JBBIiVQkMEiJTCQ4Dx9EAABIhdJ0GkiLFTRQCABIiZGgAAAASItMJDBIiQ0hUAgAi0wkDAENJ1AI +AJCQSI0FBlAIAOhpIf3/SItsJFBIg8RYw0iJRCQI6LWzAQBIi0QkCOmr/v//zMzMzMzMzMzMzMxI +g+wYSIlsJBBIjWwkEEyJdCQISYtGMIO4dAIAAABmkHUdSI0FF/oCAEiJBCToLrIBAEUPV/9kTIs0 +Jfj///9Ii0QkCEiLQDD/iHQCAABJi0YwkIO4dAIAAAB1H4O4cAIAAAB1FkjHgGgBAAAAAAAASceG +6AAAAAAAAABIi2wkEEiDxBjDzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEHYgSIPsGEiJ +bCQQSI1sJBBIjQUj9wIAuz4AAADou0j//5Do1bIBAOvTzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2 +G0iD7AhIiSwkSI0sJOjp////SIssJEiDxAiQw+iasgEA69jMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMxJO2YQdhtIg+wISIksJEiNLCTo6f///0iLLCRIg8QIkMPoWrIBAOvYzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMSTtmEHYbSIPsCEiJLCRIjSwk6On///9IiywkSIPECJDD6BqyAQDr2MzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzEk7ZhB2G0iD7AhIiSwkSI0sJOjp////SIssJEiDxAiQw+jasQEA69jM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdhtIg+wISIksJEiNLCTo6f///0iLLCRIg8QIkMPo +mrEBAOvYzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEHYbSIPsCEiJLCRIjSwk6On///9Iiywk +SIPECJDD6FqxAQDr2MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEyNpCQY/v//TTtmEA+GVQMAAEiB +7GgCAABIiawkYAIAAEiNrCRgAgAAgz2sLwsAAA+E1AAAAEiF9nQZg74QAQAAAHUQSIusJGACAABI +gcRoAgAAw0iJtCSQAgAASIm8JFgCAABIiYQkcAIAAE2LZjBB/4Qk8AAAAEiJ+kiNfCRYSIlsJPBI +jWwk8OgJugEASIttAIO+OAEAAAB+SUyLpsAAAABNheR0PUmDfCR4AHQ1SYN8JHAAdC1Ei6Y8AQAA +kEWF5HUZSIueQAEAAEiF23QNSIM7AHQHMcnpFwIAADHJ6akBAABIxwQkBgAAAEiJ1zH2TI1EJFhB +uUAAAABFMdJFMdvoGCQBAOsWSIusJGACAABIgcRoAgAAw2YPH0QAAEiFwA+PDAEAAEyLpCSQAgAA +TYXkdExJi5wkQAMAAGaQSIXbdD1Ji4QkSAMAAEjHBCQGAAAAMclIi7wkWAIAADH2TI1EJFhBuUAA +AABFMdJFMdvoqSMBAEyLpCSQAgAAkOsCMcBIhcAPhagAAABIi4QkcAIAAOimjQEAhMB0GUyNBRP2 +AgBBhABMiwUJ9gIASf/ATInA6y5Mi4QkcAIAAJBMOQWxFQgAcxlMjQXA9QIAQYQATIsFtvUCAEmN +QAFmkOsDTInASIlEJFhMi4QkkAIAAEmDuAABAAAAdBtMjQWV9QIAQYQATIsFi/UCAEn/wEyJRCRg +6xlMjQWS9QIAQYQATIsFiPUCAEn/wEyJRCRguAIAAACDPaAtCwAAdClmkEiD+EB3O0iLnCRYAgAA +SI1MJFhIice+QAAAAEiNBX5iCADomd78/0mLRjD/iPAAAABIi6wkYAIAAEiBxGgCAADDSInBukAA +AADoUrYBAEiLvsAAAABMi2d4SItfcA8fAEiD+UBzP0iJTCRQSMcEJAAAAABMjUTMWEyNScBJ99lM +ieAx9kUx0kUx2zHJ6E8iAQBIi1QkUEgBwkiFwEgPT8LpO/7//0iJyLlAAAAA6K61AQBI/8FIg/kg +fQ5MiyTLkE2F5HXtSIP5IHdRTI1kJFhMOeN0M0iJTCRQSMHhA0yJ4OiavgEASIuEJHACAABIi0wk +UEiLlCRYAgAASIu0JJACAABMjWQkWEyLrkABAABJx0UAAAAAAOky////uiAAAADoerUBAJBIiUQk +CEiJXCQQSIlMJBhIiXwkIEiJdCQo6LutAQBIi0QkCEiLXCQQSItMJBhIi3wkIEiLdCQoZpDpW/z/ +/8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEiD7ChIiWwkIEiNbCQggz0HLAsAAHQEMcDrGDHA +SI0NqCoLAIcBSItsJCBIg8Qow0j/wEiD+CB9FkiNHWAyCwBIizTDSIX2dedIg/gg6wdIjR1KMgsA +dxZIicG/IAAAAEiNBblgCADoVN78/+utSInBuiAAAADopbQBAJDMzMzMSIPsOEiJbCQwSI1sJDCD +PYcrCwAAdDpIjRUy8wIAhAJIixUp8wIASI1cJCBEDxE7SIlEJCBI/8JIiVQkKEiNBVxgCAC5AgAA +AEiJz+jv3fz/SItsJDBIg8Q4w8zMzMzMSTtmEA+GnAEAAEiD7BhIiWwkEEiNbCQQiRjHQAQDAAAA +SMeAGA4AAAAAAABIx4AgDgAAgAAAAEiNiCgOAACDPVsrCwAAdQlIiYgQDgAA6wxIjbgQDgAA6ASw +AQAxyWaQ6wRIjU4BSIP5BX1HSI0USUjHRNBwAAAAAEjHRNB4IAAAAEiNPNBIjX9oSInOSMHhCEiN +DAhIjYngAAAAgz39KgsAAHUHSIlM0GjrtuivrwEA669IiUQkIIlcJChIjYjAFgAASInI6PX4/v9I +i0wkIEiDeUAAdAaLVCQo6zWLVCQohdJ1G0iLNeQpCwBIhfYPhJ4AAABIiXFA6xcPH0QAAOhbjf3/ +SItMJCBIiUFAi1QkKEiLHZ8/CABIizWQPwgAidHB+h/B6hsBysH6BYnXweIFKdGFyXxWg/kgGdJB +uAEAAABB0+BBIdBIY8dIOcN2NUiNFIbwRAkCSIsN+D4IAEiLFek+CABIOcF2FUiNBIJB99DwRCEA +SItsJBBIg8QYw+h6sgEASInZ6HKyAQDojRv//0iNBUCuAgC7DwAAAJDo20D//5BIiUQkCIlcJBDo +7KoBAEiLRCQIi1wkEA8fAOk7/v//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GJgMA +AEiD7GBIiWwkWEiNbCRY6wb/BbBGCACLiPQFAAA5iPAFAAB0Qv/JiYj0BQAAD7bJSIuMyPgFAACQ +kEiLFXZGCABIiZGgAAAAkEiJykiJDWRGCABIgz1kRggAAHW0kEiJFVpGCADrqkiLiPgNAABIhcl0 +P5CQkEiLFTpGCABIiZGgAAAAkEiJykiJDShGCABIgz0oRggAAHUIkEiJFR5GCAD/BSBGCABIx4D4 +DQAAAAAAAEiJRCRoSIO46CYAAAAPjsgAAABJi04wSIuJ0AAAAEiJTCRAhAFIjYHYJgAASIlEJDCQ +6I4V/f9Ii0wkaEiNgdgmAABIiUQkOJDodxX9/0iLTCRoSIuZ4CYAAEiLkegmAABIi7nwJgAASItE +JEBIidHoUO4AAEiLTCRoRA8RuegmAACDPXwoCwAAdQ1Ix4HgJgAAAAAAAOsSSI254CYAADHSDx9A +AOg7rQEASMeB+CYAAAAAAAAx0kiHkWgWAACQkEiLRCQ46NsW/f+QkEiLRCQw6M8W/f9Ii0QkaIM9 +RyYLAAB0HJDoe/f+/0iLTCRoSI2BmBYAAOjKXf7/SItEJGhIjYgoDgAAuwAEAABIicjoUWD9/0iL +TCRoSMeBGA4AAAAAAABIx4EgDgAAgAAAAEiNkSgOAACDPcgnCwAAdQlIiZEQDgAA6wxIjbkQDgAA +6JGsAQAxwOsNSI1BAUiJ8WYPH0QAAEiD+AUPjYEAAABIiUQkIEjB4AhIiUQkKEiNNAFIjbbgAAAA +uwABAABIifDo0F/9/0iLTCQgSI0USUiLdCRoSMdE1nAAAAAASMdE1nggAAAASIt8JChIjTw+SI2/ +4AAAAEyNBNZNjUBogz0xJwsAAHUKSIl81mjpeP///0iJ+EyJx+jaqgEA6Wj///9EDxF8JEhIjQWo +AAAASIlEJEhIiUwkUEiNRCRISIkEJOiwpgEARQ9X/2RMizQl+P///0iLRCRoSItIQEiJyOjSiv3/ +SItEJGhIx0BAAAAAAA8fRAAA6Lvy//9Ii0QkaOgxAQEASItEJGhIx4B4FgAAAAAAAMdABAQAAABI +i2wkWEiDxGDDSIlEJAjohqcBAEiLRCQIkOm7/P//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +STtmEA+GtgAAAEiD7CBIiWwkGEiNbCQYSItSCDHAZpDrMUiLnMIwEgAASIs1ZyILAEgrNTAiCwBI +iTVZIgsASIs1OiILAEiJM0iJHTAiCwBI/8CEAkg5gigSAAB+DGaQSD2AAAAAcrrrS0iJVCQQSMeC +KBIAAAAAAACQkEiNBZ25CQDomBL9/0iLTCQQSI1BSEiNHZC5CQDo47z+/5CQSI0FerkJAOhVFP3/ +SItsJBhIg8Qgw7mAAAAA6AGuAQCQ6PulAQDpNv///8zMzMzMzMzMzMzMzMzMzMzMzMzMzMxMjWQk +2E07ZhAPhhgIAABIgeyoAAAASImsJKAAAABIjawkoAAAAIsVfCMLAIXSD4zhBwAAhcAPjtkHAACJ +VCRAiYQksAAAAIA9IrgIAAB0LJBIx0QkcAAAAABIY9BIiVQkcLsBAAAASI1MJHBIid9Iid64BAAA +AOhEAAEAkA8fAOh7wQEARQ9X/2RMizQl+P///0iLBc9CCABIiwwkDx8ASIXAdBqLVCRATGPCSYnJ +SCnBTA+vwUwBBbNCCADrB4tUJEBJiclMiQ2bQggARIuEJLAAAABFjUgfQcH5H0HB6RtHjQwBRY1J +H0HB+QVEiUwkREyLFb84CAAPH4AAAAAARTnQD47GAQAAkJBIjQX+IgsA6CkR/f9IixWiOAgAi4wk +sAAAADnRfxxIY/FIOfIPgtgGAABIiTV8OAgADx9AAOmJAAAASGPZSIlcJGBIjQXMQgIASInZ6MSC +AABIiYQkmAAAAEiLPUU4CABIizVOOAgASInDSItMJGBIjQWfQgIA6Bpb/f9Ii1QkYEiJFSY4CABI +iRUnOAgAgz3gIwsAAHURSIu0JJgAAABIiTX/NwgA6xRIjT32NwgASIu0JJgAAADo2agBAIuMJLAA +AABIixVrOAgASIs9VDgIAEiLNVU4CABEi0QkREQ5wnwvSWPYSDnaD4IMBgAASIkdODgIAEiLFZk4 +CABIOdoPgukFAABIiR2BOAgA6ZQAAABJY9hIiVwkWEiNBVUkAgBIifHojYAAAEiLXCRYSIkd+TcI +AEiJHfo3CACDPTMjCwAAdQlIiQXaNwgA6w1IjT3RNwgAkOjbpgEASIs9JDgIAEiLDSU4CABIjQUG +JAIA6EGAAABIi1QkWEiJFQ04CABIiRUOOAgAgz3nIgsAAHUJSIkF7jcIAOsMSI095TcIAOiQpgEA +kJBIjQVXIQsA6GIR/f+LVCRARIuEJLAAAABEi0wkRInQ6y1Mi5QkgAAAAEyHEESLVCRQQY1SAUSL +hCSwAAAAi0QkQESLTCREDx+EAAAAAABBOdAPjrMAAABIY/JIiw2lNggASIs9ljYIAGYPH0QAAEg5 +8Q+GyAQAAIlUJFBIiXQkaEiLDPdIhcl1E0iNBf6FAgDoOS79/4tUJFBIicFIiYwkgAAAAEiJyInT +Dx9EAADoe/b//0iLDUw2CABIixU9NggASItEJGgPH4QAAAAAAEg5wQ+GYwQAAEiNBMKAPewhCwAA +D4Q5////SImEJJAAAABIi5wkgAAAAOgRqfz/SIuEJJAAAADpF////0yJtCSIAAAASYtWMEiLktAA +AABIhdIPhKAAAABJidIPH0QAAEQ5Anx2TYXSD4SKAAAAgD17tAgAAHRHkE2JluAAAAC4EQAAALsB +AAAAMckx/0iJ/uir/AAASIuUJIgAAABMi0IwSYuA0AAAAOgTCwEAi0QkQESLhCSwAAAARItMJERI +i5QkiAAAAEiLWjBIi5vQAAAASMdDOAAAAADrJcdCBAEAAABJi04wSIuJ0AAAAEiLQUDoKo39/+tb +SIuUJIgAAABIi1owSMeD0AAAAAAAAABIiw0iNQgASIsdEzUIAA8fAEiFyQ+GPAMAAEiLA0jHQDgA +AAAAx0AEAAAAAA8fRAAA6HsDAACAPaSzCAAAdAXozQ0BAEjHBdIfCwAAAAAAi1QkQIuEJLAAAACJ +wesjiUQkTEiLBN/oxvb//4tMJEyNQQGLTCRAi4wksAAAAItUJEA50H0bSGPYSIs1mjQIAEiLPYs0 +CABIOd53w+mpAgAASIsVgjQIADnRD4SHAAAAkJBIjQXJHgsA6PQM/f+LjCSwAAAASGPRSIsdYzQI +AA8fAEg50w+CZAIAAEiJFUg0CACLXCRESGPbSIsVwjQIAGaQSDnaD4I8AgAASIkdqDQIAEiLFQk1 +CABmDx+EAAAAAABIOdoPghECAABIiR3oNAgAkJBIjQVXHgsA6GIO/f+LjCSwAAAAjVH/TIuEJIgA +AAAxwOsF/8pMichIiUQkeGaQhdIPjK0AAABMY8pMixXGMwgATIsdtzMIAA8fgAAAAABNOcoPhqYB +AABNi1AwT4sMy02LktAAAABNOcp1BUmJweuyQcdBBAAAAADpWgEAAEU503UxTYXkdSyJVCRITInI +6HsdAACLjCSwAAAAi1QkSEyLhCSIAAAATItMJHgPHwDpb////5BMixXTOwgATYXSdBRNi5pgAQAA +TIkdwDsIAP8NwjsIAJBNiVE4kEmJQQjpP////5CJDUQ0CABIxwVJNAgAAAAAALoBAAAA6wWNUwGJ +8TnRD4KUAAAAidOJzpDpuwAAAIP6AXXkSIsNHzQIAEiNUQFMiwUMNAgASIs9FTQIAEg513NTiVwk +VEiNBY0fAgBMicNIidboAn4AAEiJDfMzCACDPXQeCwAAdQlIiQXTMwgA6w5IjT3KMwgAZpDoG6IB +AIu0JLAAAABIidlJicBIi0QkeItcJFRIjVEBSIkVqjMIAEGJHIjpX////0iNFVIcCwCHCkiLrCSg +AAAASIHEqAAAAMNIiceJ0DHS9/FIifhBidCJykSJwQ8fQACFyXXk6Tz///9Fi5HwBQAARYuZ9AUA +AE2LofgNAABFi6n0BQAARTnddd/pgP7//0yJyEyJ0egmpgEASInZDx8A6JumAQBIidnok6YBAEiJ +0UiJ2uiIpgEASInYSInxZpDo+6UBADHA6PSlAQDo76UBAEiJ8OjnpQEASInZDx9AAOhbpgEASInZ +6FOmAQBIifHoS6YBAEiNBV2yAgC7FwAAAOg6NP//kIlEJAjoUJ4BAItEJAjpx/f//8zMzMzMzMxJ +O2YQdj5Ig+wQSIlsJAhIjWwkCEiJRCQY6EIAAABIi0wkGEiLQUDoNIn9/4A9/a8IAAB0BehmBgEA +SItsJAhIg8QQw0iJRCQI6PKdAQBIi0QkCOurzMzMzMzMzMzMzMxIg+wwSIlsJChIjWwkKEmLTjBI +g7nQAAAAAGYPH0QAAA+F5gAAAEiLUDhIiVQkIEiF0nU1g3gEAHUmkEiJwkiJgdAAAABJi0YwSIlC +OMdCBAEAAABIi2wkKEiDxDDDZpBIhdIPhJwAAABIi5LoAAAASIlUJBCLQARIiUQkGA8fAOibS/// +SI0FBJ0CALsMAAAA6KpU//9Ii0QkIA8fRAAA6JtR//9IjQUVkAIAuwEAAADoilT//0iLRCQQDx9E +AADoe1L//0iNBZibAgC7DAAAAOhqVP//SItEJBgPH0QAAOhbUf//6LZN///osUv//0iNBRivAgC7 +FgAAAA8fRAAA6Lsy//8x0ulk////SI0FyKoCALsUAAAA6KMy//+QzMxJO2YQD4YyAQAASIPsQEiJ +bCQ4SI1sJDhJi04wSIuB0AAAAEiFwA+E/gAAAEiJTCQwSIlEJCiQSItQOEiJVCQQSDnKdUuDeAQB +dUVMiXQkIIA9Wa4IAAB0CugiBQEASItEJChIi0wkIEiLSTBIx4HQAAAAAAAAAEjHQDgAAAAAx0AE +AAAAAEiLbCQ4SIPEQMOLSARIiUwkGOhjSv//SI0FhJsCALsMAAAA6HJT//9Ii0QkMOjoUv//SI0F +DZECALsGAAAA6FdT//9Ii0QkKOjNUv//SI0F/pACALsGAAAAkOg7U///SItEJBDokVH//0iNBTaY +AgC7CwAAAA8fRAAA6BtT//9Ii0QkGOgRUP//6GxM///oZ0r//0iNBZ2zAgC7GQAAAOh2Mf//SI0F +mKoCALsVAAAA6GUx//+QDx9AAOh7mwEA6bb+///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEHZK +SIPsEEiJbCQISI1sJAiJRCQYkEiNBfg2CADo+wb9/4tMJBgBDf02CACFyX4F6CgAAACQkEiNBdc2 +CADougj9/0iLbCQISIPEEMOJRCQI6AebAQCLRCQI66HMSTtmEA+GwwIAAEiD7EhIiWwkQEiNbCRA +gD3OFwsAAJB1CYA9whcLAAB0CkiLbCRASIPESMODPQQYCwAADx9AAA+H6gAAAIA9nxcLAAB1CYA9 +jhcLAAB1BDHJ6yK4AQAAAOi6j///kIM9nhcLAABIjQ1bGAsASIcBD5fBD7bJixVENggASIsdRTYI +AEgrHU42CABIid4p04s9LzYIACn7RIsFNjYIAEQpwznZfQpIi2wkQEiDxEjDhdsPjGYBAABIx0Qk +IAAAAABEDxF8JDBIjQ0LAgAASIlMJDBIjUwkIEiJTCQ4SI1EJDDoMmX//0iDfCQgAA+ECwEAAEiD +PdYXCwAAdBroF+4AAEiF23QQSIkFwxcLAEiNBdQ1CADrdEiLDQstCABIixUMLQgAMcDrEEiLbCRA +SIPESMNI/8APHwBIOdB9GkiLHMGEA0iDu+gmAAAAfuVIi2wkQEiDxEjDSYtOMMeB9AAAAP////+Q +kEiNBUI1CADoJQf9/0iNBbrJAgC7JQAAAOhUL///hAFIjUEISIsISIXJdAxIOct17UiLSwhIiQiQ +SIsNEjUIAEiFyXQXSIuRYAEAAEiJFf80CAD/DQE1CABIhcl0I5BIiZnYAAAASI2BUAEAAA8fRAAA +6HsH/f9Ii2wkQEiDxEjDSI0FvLACALsZAAAADx9EAADo2y7//5CQSI0FqjQIAOiNBv3/SI0FitkC +ALs2AAAAkOi7Lv//SIl0JCiJVCQciXwkGESJRCQU6ARH//9IjQUEtQIAuxsAAADoE1D//4tEJBxI +Y8DoB07//0iNBTCaAgC7DgAAAOj2T///i0QkGEhjwOjqTf//SI0FbI8CALsIAAAA6NlP//9Ii0Qk +KEhjwOjMTf//SI0FCI4CALsHAAAA6LtP//+LRCQUSGPA6K9N///oCkn//+gFR///SI0FhLsCALse +AAAA6BQu//+Q6C6YAQDpKf3//8zMzMzMzMzMzEk7ZhAPhtwAAABIg+wwSIlsJChIjWwkKEiJRCQ4 +SItKCEiJTCQgMdvoc0QBAITAdTaQSItEJDiLiJAAAACJyg+68QyNWf+D+wJ2JoP5BHQFg/kJdQhI +i0QkIEj/AEiLbCQoSIPEMMNIi2wkKEiDxDDDiVQkFEiLgJgAAABIiUQkGOjaRf//SI0Fv7MCALsb +AAAA6OlO//9Ii0QkGA8fQADo20z//0iNBbSTAgC7CwAAAOjKTv//i0QkFInADx9AAOi7S///6BZI +///oEUb//0iNBdWlAgC7FQAAAA8fRAAA6Bst//+QSIlEJAjokJYBAEiLRCQI6Qb////MzMzMzMxJ +O2YQD4ZlBAAASIPsYEiJbCRYSI1sJFiQSI0FuDIIAOi7Av3//wXRMggA6PD7//+QkEiNBZ8yCADo +ggT9/zHJSI0VmTMIAIcKMcAxyTHS6yJIiXQkQJCQSI0FuTMIAJDoWwT9/0iLRCRIi0wkHEiLVCRA +SIlUJEBIiUQkSA8fAEiFwHUHuxQAAADrC4nL0eFIg/gyD0/ZgfsQJwAAuBAnAAAPR9iJXCQciRwk +6K+wAQBFD1f/ZEyLNCX4////ZpDom4///5Do1bEBAEUPV/9kTIs0Jfj///9IiwQkgz1NFwsAAH8j +gz3UMggAAHQHuQEAAADrFYsNHTIIAIsVRxMLADnRD5TB6wIxyYTJdQ6LRCQcSItMJEjpKAEAAEiJ +RCQgkJBIjQWuMQgA6LEB/f+LDYsyCACFyXQJuQEAAABmkOsRiw3QMQgAixX6EgsAOdEPlMGEyXUO +i0QkHEiLTCRI6cAAAADo1ukAAEiLTCQgSDnIfxGLRCQcSItMJEgPHwDpoAAAAEiJRCQouQEAAABI +jRU6MggAhwqQkEiNBTcxCADoGgP9/0iLDcPzBwBIicpIwek/SI0cEUjR+0iLTCQoSItUJCBIKdFI +OctID0/ZSI0FAjIIAOglB/3/iEQkG5Doe47//5CQSI0F6jAIAOjtAP3/MclIjRXUMQgAhwqQSMcF +zjEIAAAAAAAPtkwkG4TJdAm4FAAAADHJ6wmLRCQcSItMJEhIiUwkSIlEJByQSI0FojAIAOiFAv3/ +i0QkHEiLTCRISIlMJEiJRCQckEiNBcMxCADohgD9/5APH0QAAOg7sAEARQ9X/2RMizQl+P///0iL +DU8ECABIiwlIixQkSIlUJFBIhcl0JEiJDCRIx0QkCAAAAADoBJUBAEUPV/9kTIs0Jfj///9Ii1Qk +UEiLDRMwCACLHZURCwCF23RTkEiFyXRNSI2ZgJaYAEg52n5BSInISI0N7S8IAPBID7ERD5TBMcDo +9ur+/0iJRCQ4SIXAdB64/////+jC+P//SI1EJDjoGK///7gBAAAA6K74///oSY3//4sFwygIAA8f +AIXAdAXoNwv+/0iLRCRQ6E0BAABIi0wkSEj/wYXAugAAAABID0XKgD3bOAsAAHRWgz0KEQsAAHVN +gz3lEAsAAJB1Q4M99xcLAAB9CUiLfCRQMdvrOEiLHR0/CwAPH0QAAEiF23QXSIt0JFBIifdIKd5I +OTXZ8QcAD5zD6xBIi3wkUDHb6wdIi3wkUDHbSIlMJEiE23Rqix0FJwgAhdt0YJCQSI0F6CYIAOgT +//z/xwXpJggAAAAAAEjHRCQwAAAAAEiLDdEmCABIx4GgAAAAAAAAAJBIiUwkMEiNRCQwDx8A6Buu +//+QkEiNBaImCADorQD9/0iLTCRIMdJIi3wkUIsdCxQLAIXbfjxIY9tIadtAQg8ASIt0JEBIAfMP +H0QAAEg53w+MCPz//4M93BMLAAAPn8DoyAMAAEiLdCRQDx8A6ez7//9Ii3QkQOni+///6KySAQDp +h/v//8zMzMzMzMxJO2YQD4YdAgAASIPsOEiJbCQwSI1sJDBIiUQkQJBIjQULEAsA6Db+/P9Ii0wk +QDHAMdLrA0j/wEiJVCQQSIsdjCUIAEg5BY0lCAAPjrgAAABIixzDSIXbdNtIiUQkGEiJXCQoi3ME +iXQkDIP+AXQFg/4CdWOLexBEi0MYDx9AAEk5+HQLiXsYSIlLIDH/61BIi3sgSIHHgJaYAA8fRAAA +SDn5fC1IiXwkIEiJ2OguAgAASItMJEBIi1QkIEg50UiLRCQYSItUJBBIi1wkKIt0JAxAD53H6wgx +/2YPH0QAAIP+Ag+FTP///0SLQxRAhP91FYt7KEw5x3QNRIlDKEiJSzDpLv///+sdkJBIjQUeDwsA +6Cn//P9Ii0QkEEiLbCQwSIPEOMOLu/AFAABEi4P0BQAATIuL+A0AAESLk/QFAABFOcJ14EE5+HUh +TYXJdRiLPUYtCABEiwU7LQgARAHHhf9AD5fH6wYx/+sCMf9AhP90FkiLezBIgceAlpgAZpBIOfkP +jKz+//+QkEiNBZ4OCwDoqf78/7j/////Dx9AAOib9f//i0QkDEiLTCQoMdLwD7FRBA+Uw4TbdDaA +Pc2iCAAAdBdIicjoMwEBAEiLRCQo6In5AABIi0wkKP9BFEiJyOh5kP//SItMJBBIjVEB6wVIi1Qk +EEiJVCQQuAEAAADoOvX//5BIjQUiDgsA6E38/P9Ii0QkGEiLTCRASItUJBDpDv7//0iJRCQI6G+Q +AQBIi0QkCOnF/f//zMzMzMxJO2YQdnNIg+wwSIlsJChIjWwkKEiLDXUjCABIiUwkIEiLFXEjCABI +iVQkGDHAMdvrA0j/wEg50H0zSIs0wYN+BAF17kiJRCQQiFwkD0iJ8OhLAAAAD7ZcJA8Jw0iLRCQQ +SItMJCBIi1QkGOvFidhIi2wkKEiDxDDD6OKPAQBmkOl7////zMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMSTtmEA+GnAAAAEiD7BhIiWwkEEiNbCQQSItIOA8fQABIhcl0dUk5TjB0b0iLkcAAAABI +hdJ0V0g5EXRSxoKxAAAAAUjHQhDe+v//gz2GEAsAAHUrxoAIJwAAAZAxwLoBAAAA8A+xkVQDAAAP +lMKE0nQNSInIuxcAAADooff+/7gBAAAASItsJBBIg8QYwzHASItsJBBIg8QYwzHASItsJBBIg8QY +w0iJRCQI6BCPAQBIi0QkCOlG////zMzMzMzMTI1kJMhNO2YQD4a+BwAASIHsuAAAAEiJrCSwAAAA +SI2sJLAAAACIhCTAAAAA6E6qAQBFD1f/ZEyLNCX4////SIsEJEiJRCQgSIM9aA0LAAB1B0iJBV8N +CwCQkEiNBVYqCADoWfr8/0iLDUoNCwBIiYwkkAAAAIsVfCoIAEiJlCSIAAAAix1yKggASImcJIAA +AABIYzUvKggASIl0JHhIYz1rKggASIl8JHBMiwUfKggATCsFKCoIAEyJRCRoDx8A6Hs8//9IjQV2 +gwIAuwYAAADoikX//0iLTCQgSIuUJJAAAABIKdFIuNs0tteC3htDSPfpSMH6EkjB+T9IKcpIidDo +WkP//0iNBTyRAgC7DwAAAOhJRf//SGMFCgsLAGaQ6DtD//9IjQUJigIAuwsAAADoKkX//0iLhCSI +AAAAZpDoG0L//0iNBduFAgC7CQAAAOgKRf//SItMJGhIY8FmkOj7Qv//SI0F05MCALsRAAAA6OpE +//9Ii4QkgAAAAGaQ6NtB//9IjQV9jQIAuw0AAADoykT//0iLRCR4Dx9EAADou0L//0iNBRyIAgC7 +CgAAAOiqRP//SItEJHAPH0QAAOibQv//6PY7//8PtowkwAAAAITJD4TKAAAAiwXAKQgASImEJIgA +AABIYw3dKAgASImMJJAAAABIYxWmKQgASIlUJHiLHacpCABIiZwkgAAAAOgqO///SI0FDYkCALsL +AAAA6DlE//9Ii4QkiAAAAOgsQf//SI0FVY4CALsOAAAA6BtE//9Ii4QkkAAAAOgOQv//SI0FjYcC +ALsKAAAAZpDo+0P//0iLRCR46PFB//9IjQXqigIAuwwAAAAPH0QAAOjbQ///SIuEJIAAAADozkD/ +/+gpPf//6CQ7//8PtowkwAAAAEiLFYUfCABIiZQkqAAAAEiLHX4fCABIiVwkYDHA6x1IjUEBD7aM +JMAAAABIi5QkqAAAAEiLXCRgDx9AAEg52A+NJgIAAEiJRCRASIs0wkiLfjhEi4bwBQAARIlEJBxE +i470BQAARIlMJBSEyQ+EYwEAAEiF/3QJSIu/6AAAAOsHSMfH/////0iJfCQ4i0YESImEJIgAAACL +ThBIiYwkgAAAAItWFEiJVCRYSGOeCA4AAEiJnCSQAAAASIu26CYAAEiJdCRQ6NI5//9IjQWqfgIA +uwMAAADo4UL//0iLRCRA6NdA//9IjQX6gwIAuwkAAADoxkL//0iLhCSIAAAA6Lk///9IjQXUhwIA +uwsAAADoqEL//0iLhCSAAAAA6Js///9IjQV+iwIAuw0AAADoikL//0iLRCRYDx9EAADoez///0iN +BT9+AgC7AwAAAOhqQv//SItEJDgPH0QAAOhbQP//SI0FsoUCALsKAAAA6EpC//+LRCQUi0wkHCnI +6Ds///9IjQV0hQIAuwoAAADoKkL//0iLhCSQAAAAZpDoG0D//0iNBUyHAgC7CwAAAOgKQv//SItE +JFAPH0QAAOj7P///6FY7///oUTn//0iLTCRA6Ur+///owjj//2aQ6Ps6///oNjn//0iLRCRASIXA +dRvopzj//0iNBU19AgC7AQAAAOi2Qf//6BE5///ojDj//4tEJBSLTCQcKchmkOibPv//6PY4//9I +iwVnHQgASP/ISItMJEAPH4AAAAAASDnBD4Xa/f//6FI4//9IjQUcfQIAuwIAAADoYUH//5Douzj/ +/0iLTCRA6bT9//+EyXQMSIsN5hsIAOkhAgAASI0FkiUIAOg19/z/SIusJLAAAABIgcS4AAAAw0iJ +jCSYAAAASIlcJChIi4HoAAAASImEJJAAAABIY5HwAAAASIlUJHhIY7H0AAAASIl0JHBIi7n4AAAA +SIm8JKAAAABMi4EAAQAATIlEJGBMY4kIAQAATIlMJGhMY5EMAQAATIlUJEhED7aZFAEAAESIXCQT +RA+2oRUBAABEiGQkEg8fQADoezf//0iNBVB8AgC7AwAAAOiKQP//SIuEJJAAAABmkOh7Pv//SI0F +rHwCALsEAAAA6GpA//+LRCQYSGPADx8A6Fs+//9IjQXifQIAuwYAAADoSkD//0iLRCQwDx9EAADo +Oz7//0iNBR+FAgC7CwAAAOgqQP//SItEJHgPH0QAAOgbPv//SI0FwoMCALsKAAAA6ApA//9Ii0Qk +cA8fRAAA6Ps9//9IjQW4hgIAuwwAAADo6j///0iLhCSgAAAASItcJGDo2D///0iNBQZ+AgC7BwAA +AOjHP///SItEJGhmkOi7Pf//SI0F4n0CALsHAAAA6Ko///9Ii0QkSA8fRAAA6Js9//9IjQUQgwIA +uwoAAADoij///w+2RCQTDx9EAADoGzn//0iNBc9/AgC7CQAAAOhqP///D7ZEJBIPH0QAAOj7OP// +SI0Fyn8CALsJAAAA6Eo///9Ii0QkKA8fRAAA6Ds9///oljj//+iRNv//SIuEJJgAAABIi4hYAQAA +ZpBIhcl0YEiLgdAAAABIi5HAAAAASIuZaAEAAEiFwHQEiwDrBbj/////SIXSdAlIi5KYAAAA6wdI +x8L/////SIlUJDCJRCQYSIXbdAxIi5uYAAAA6aT9//9Ix8P/////ZpDplv3//0iNBfTPAgDob1L/ +/0iNBQAjCADoo/T8/0iLrCSwAAAASIHEuAAAAMOIRCQI6CqHAQAPtkQkCA8fRAAA6Rv4///MzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4YRAQAASIPsIEiJbCQYSI1sJBiIRCQokEiNBZQi +CADol/L8/w+2TCQoicqD8QE4Dd8iCAAPhJQAAACIDdMiCACE0nR6iw3hIggAxwXXIggAAAAAAJBI +ixXHIggASIsduCIIAEiF0nQxSInWSMeCoAAAAAAAAABIixWGIggASIXSdAlIiZqgAAAA6wdIiR1p +IggASIk1aiIIAIlMJBQBDWgiCABEDxE9cCIIAJBIjQUAIggA6OPz/P+LTCQU60mQkEiNBewhCADo +z/P8/+sckJBIjQXcIQgADx9AAOi78/z/SItsJBhIg8Qgw0iLbCQYSIPEIMOJTCQUMcAx2+j6gv// +i0wkFP/Jhcl034M94SEIAAB14OvUiEQkCJDo24UBAA+2RCQI6dH+///MzMzMzMzMzMzMzMzMzMzM +zEk7ZhB2O0iD7BhIiWwkEEiNbCQQgD29IQgAAHQUuwEAAADoGTIBAEiLbCQQSIPEGMO4AQAAAEiL +bCQQSIPEGJDDSIlEJAjodYUBAEiLRCQI667MzMzMzMzMzMzMzMzMzEk7ZhB2NkiD7AhIiSwkSI0s +JEiLDQchCABIiYhgAQAAkEiJBfggCAD/BfogCADoLer//0iLLCRIg8QIw0iJRCQI6BqFAQBIi0Qk +COuzzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhhIBAABIg+w4SIlsJDBIjWwkMIsV8iAIAGaQhdIP +hIkAAACLNfoBCwCF9g+E4AAAAEiJwYnQidcPHwCD/v90BZn3/usE99gx0o1QATnXD0zXhdt+BjnT +fQKJ2oH6gAAAAL6AAAAAD0/WKdeJPZogCACQSIsFgiAIAGaQSIXAdB5Ii7CgAAAASIk1bSAIAEiF +9nULSMcFZSAIAAAAAABIiUQkKEiJTCQg/8rrJjHASItsJDBIg8Q4w0iJyDHJ6LkDAACLVCQc/8pI +i0QkKEiLTCQghdJ+MYlUJByQSIsdGSAIAEiF23TRSIuzoAAAAEiJNQYgCABIhfZ1vkjHBf4fCAAA +AAAA67FIi2wkMEiDxDjD6MX0/v+QSIlEJAiJXCQQ6NaDAQBIi0QkCItcJBDpyP7//8zMzMzMzMzM +STtmEA+GwwAAAEiD7CBIiWwkGEiNbCQYhACLiPgmAACFyQ+HjQAAAEiJRCQoSI2I2CYAAEiJTCQQ +kEiJyA8fAOg77/z/SItMJCiLkfgmAACF0nVLSIsVfRcIAEiLHW4XCACLCYnOwfkfwekbjTwOwf8F +QYn4wecFKf6F9nxJg/4gGf+J8b4BAAAA0+Yh/kljwEg5wnYpSI0Mg/fW8CExkJBIi0QkEOi18Pz/ +SItsJBhIg8Qgw0iLbCQYSIPEIMNIidHoWYoBAOh08/7/kEiJRCQI6OmCAQBIi0QkCA8fQADpG/// +/8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhvIAAABIg+wYSIlsJBBIjWwkEIuI8AUA +AIuQ9AUAAEiLsPgNAACLuPQFAAA513XjOcoPha0AAAAPHwBIhfYPhaEAAABIiUQkIOit/v//SIsN +HhYIAEiLFQ8WCABIi1wkIIsziffB/h/B7hsB/sH+BUGJ8MHmBSn3Dx8Ahf98YYP/IBn2SInIiflB +uQEAAABB0+FBIfFJY/APHwBIOfB2NUiNBLLwRAkISIsF/B0IAEiJQwiQSIkd8B0IALgBAAAASI0N +7B0IAPAPwQFIi2wkEEiDxBjDSInBSInw6DuJAQDoVvL+/0iNBe+vAgC7IwAAAOilF///kEiJRCQI +6LqBAQBIi0QkCOnw/v//zMzMzMzMzMzMzMzMzMzMzEiD7BhIiWwkEEiNbCQQSIsFex0IAEiFwA+E +uAAAAEiLFZMVCABIix2EFQgAiwiJzsH5H8HpG408DsH/BUGJ+MHnBSn+hfYPjLQAAACD/iAZ/4nx +vgEAAADT5iH+SWP4kEg5+g+GiAAAAEiNFLvwCTJIixXhFAgASIsd0hQIAIsIic7B+R/B6RuNPA7B +/wVBifjB5wUp/oX2fFKD/iAZ/4nxvgEAAADT5iH+SWP4SDn6di9IjQy799bwITFIi0gISIkNyhwI +ALn/////SI0VxhwIAPAPwQpIi2wkEEiDxBiQw0iJ+EiJ0egUiAEA6C/x/v9IifhIidHoBIgBAA8f +QADoG/H+/5DMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhrkAAABIg+woSIlsJCBIjWwk +IITJdWtIiUQkMEiJXCQY6wpIi0QkMEiLXCQYi5DwBQAAi7j0BQAAif4p14H/AAEAAHIXidGJ9+it +AAAAhMB00UiLbCQgSIPEKMNAD7bOSImcyPgFAACNTgGHiPQFAABIi2wkIEiDxCiQw0iJyEiJ80iL +kPgNAABIid5IicFIidDwSA+xmfgNAABBD5TARYTAdNhIhdJ0C0iJyEiJ0+lj////SItsJCBIg8Qo +w0iJRCQISIlcJBCITCQY6Kp/AQBIi0QkCEiLXCQQD7ZMJBjpFv///8zMzMzMzMzMzMzMzMzMzMzM +zMzMzMxMjaQkSPz//007ZhAPhqUBAABIgew4BAAASImsJDAEAABIjawkMAQAAEjHRCQoAAAAAIn6 +SI18JDAPH4AAAAAASIlsJPBIjWwk8OjRhwEASIttACnK0epmDx+EAAAAAACB+oAAAAAPhTkBAAAx +9usYhACNPA5AD7b/SIu8+PgFAABIiXz0KP/GOdZy5I00CkiJx4nI8A+xt/AFAABAD5TGQIT2dAlI +iVzUKDHA6x8xwEiLrCQwBAAASIHEOAQAAJDDSIt0xChIibOgAAAAOdBzHEiLXMQohAP/wGYPH0QA +AEg9gQAAAHLZ6a4AAACJVCQURA8RfCQYSItMJChIiUwkGEiLTNQoSIlMJCCQkEiNBSQaCADoJ+r8 +/5BIi0wkIEiLVCQYSIXJdDNIictIx4GgAAAAAAAAAEiLDUoaCABmkEiFyXQJSImRoAAAAOsHSIkV +KxoIAEiJHSwaCACLDS4aCACLVCQUjQwRjUkBiQ0eGggARA8RfCQYkJBIjQW3GQgA6Jrr/P+4AQAA +AEiLrCQwBAAASIHEOAQAAMO5gQAAAOg7hQEASI0FsKECALseAAAA6KoT//+QSIlEJAhIiVwkEIlM +JBiJfCQc6LJ9AQBIi0QkCEiLXCQQi0wkGIt8JBzpG/7//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzEk7ZhAPhv0AAABIg+wYSIlsJBBIjWwkEIuQ8AUAAIuw9AUAADH/6xJFD7bRTomE0PgFAABB +jXEB/8dMiwMPHwBNhcB0LUGJ8SnWgf4AAQAAcyOQkE2FwHTPSYuwoAAAAEiJM0iF9nXASMdDCAAA +AADrtkGJ8USHiPQFAABIKflIgzsAdHhIiVwkKEiJTCQIkJBIjQWgGAgA6KPo/P+QSItMJChIi1EI +SIsZSIXSdDFIidZIx4KgAAAAAAAAAEiLFcQYCABIhdJ0CUiJmqAAAADrB0iJHacYCABIiTWoGAgA +SItUJAgBFaUYCABEDxE5kJBIjQVAGAgA6CPq/P9Ii2wkEEiDxBjDSIlEJAhIiVwkEEiJTCQY6GV8 +AQBIi0QkCEiLXCQQSItMJBjp0f7//8zMzMzMzMzMzMzMzMzMzMzM6wNIidBIi4j4DQAASIXJdCZI +icJIicgx9vBID7Gy+A0AAEAPlMdAhP9010iJyLsBAAAAw0iJ2IuI8AUAAIuQ9AUAADnRdCgPttFI +i5TQ+AUAAI1xAUiJw4nI8A+xs/AFAAAPlMGEyXTLSInQMdvDMcAx28PMzMzMzMzMzMzMzMxIg+wY +SIlsJBBIjWwkEEQPETwkSIuQ+A0AAEiF0nQXSInBSInQMfbwSA+xsfgNAABAD5TG6wpIicEx9g8f +RAAAQIT2dD2QkEjHgqAAAAAAAAAASIt0JAhIhfZ0DEiJ10iJlqAAAADrC5BIidZIiRQkSIn3kEiJ +fCQIuAEAAABmkOsGMcDrAonYi5HwBQAAi7H0BQAAKdaF9nQvZg8fRAAAgf4AAQAAd+CNPBaJw4nQ +8A+xufAFAABAD5THDx9EAABAhP90wTHA6yJIi1wkCEiLFCSJwUiJ0EiLbCQQSIPEGMOQTIlMJAj/ +wP/DOfBzP408EEAPtv9Ii7z5+AUAAJBIx4egAAAAAAAAAEyLRCQITYXAdA5JiflJibigAAAAZpDr +v0mJ+EiJPCRNicHrs0iLVCQISIsEJInZSInTSItsJBBIg8QYw8zMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMSTtmEA+GVQEAAEiD7BhIiWwkEEiNbCQQSIlEJCBIiVwkKIlMJDBAiHwkNOsDSInwi5Dw +BQAAi7D0BQAAKdZBifDR7kEp8EWFwA+FfAAAAECE/w+ExwAAAEiLkPgNAACQSIXSD4S2AAAAg3gE +AXU7SIlUJAjHBCQDAAAADx9EAADoG5QBAEUPV/9kTIs0Jfj///9Ii0QkIItMJDBIi1QkCEiLXCQo +D7Z8JDRIicZIidBFMcDwTA+xhvgNAABBD5TBRYTJD4Rk////6zxBgfiAAAAAdgtIicZFMcDpTv// +/zH262hBjTQQSYnBidDwQQ+xsfAFAAAPlMKE0nVWTInORTHA6Sb///+EAw+2yUiJFMu4AQAAAEiL +bCQQSIPEGMMxwEiLbCQQSIPEGMOEA0SNDBZFD7bJTouMyPgFAABEjRQORQ+20k6JDNP/xkQ5xnLb +65FEicBIi2wkEEiDxBjDSIlEJAhIiVwkEIlMJBhAiHwkHOjpeAEASItEJAhIi1wkEItMJBgPtnwk +HOlx/v//zMzMzMzMzMzMzMzMzMzMzMxJO2YQD4axAAAASIPsKEiJbCQgSI1sJCBIiUQkMIuQ9AUA +AIlUJBxIjbD4BQAAic+J0UiJ2EiJ8+gj/v//Dx8AhcB0WYtMJByNFAiNUv8PtvJIi3wkMEiLtPf4 +BQAAZpCD+AF0K0SLh/AFAABEKcGNDAGNSf+B+QABAABzLIeX9AUAAEiJ8EiLbCQgSIPEKMNIifBI +i2wkIEiDxCjDMcBIi2wkIEiDxCjDSI0Fao4CALsYAAAA6OYN//+QSIlEJAhIiVwkEIhMJBjo8ncB +AEiLRCQISItcJBAPtkwkGA8fAOkb////zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMTI2kJFD/ +//9NO2YQD4YUBQAASIHsMAEAAEiJrCQoAQAASI2sJCgBAABIiwhIg/kBD4TcBAAASIP5AnQTSImE +JDgBAABIxwABAAAAMcnrMkiLrCQoAQAASIHEMAEAAMNIiUwkQEiLVMgYSInQ6I////9Ii0wkQEj/ +wUiLhCQ4AQAASDlICHfYSIN4EAB0boA9i/YKAAB1CDHSMckx2+swkOibkgEARQ9X/2RMizQl+P// +/0iLBXf2CgBIiw149goASIsUJEiJw0iLhCQ4AQAASIlMJEhIiVwkUEiJVCQoSItwCEiJdCRwSI08 +8EiNfxhIibwkEAEAAEUxwOtUSMcAAgAAAEiLrCQoAQAASIHEMAEAAMNMiUQkOJBKiwTHSo0Ux//Q +SItEJDhMjUABSIuEJDgBAABIi0wkSEiLVCQoSItcJFBIi3QkcEiLvCQQAQAATDlAEHe9gD3D9QoA +AA8fAA+EuAEAAJDo1JEBAEUPV/9kTIs0Jfj///9Ii4QkEAEAAIQASIsFpvUKAEiJRCRoSIsNovUK +AEiJTCRgSIsUJEiJVCQgSItcJHBIi7QkOAEAAEiLXN4YSInY6CKdAABmkOi7pAAASImEJBgBAABI +iVwkMEjHRCR4AAAAAEiNjCSAAAAARA8ROegUJP//SI0FYWoCALsFAAAA6CMt//9Ii4QkGAEAAEiL +XCQw6BEt//9IjQWfaAIAuwIAAAAPH0QAAOj7LP//6FYk//9IiwU39AoASIt8JChIKcdIjUQkeLsY +AAAASInZ6HWh/f9IidlIicNIjYQk8AAAAOgCjwAASImEJCABAABIiVwkWOiQI///SIuEJCABAABI +i1wkWA8fAOibLP//SI0FeGkCALsFAAAA6Ios///o5SP//0iLfCQgSItEJChIKcdIjUQkeLsYAAAA +SInZ6Aah/f9IidlIicNIjYQk0AAAAOiTjgAASImEJCABAABIiVwkWOghI///SIuEJCABAABIi1wk +WOgvLP//SI0FHnECALsLAAAADx8A6Bss///odiP//0iLRCRgSItMJEhIKci5FwAAAOsmSMcAAgAA +AEiLrCQoAQAASIHEMAEAAMNIjXMwQIh0DHhI/8lIidBIg/gKcjZIicJIuM3MzMzMzMzMSInTSPfi +SMHqA0iNNJJI0eZIKfNIg/kYcsTpjwEAAGYPH4QAAAAAAJBIg/kYD4NpAQAASI1QMIhUDHhIjVHo +SInWSMH6P0gh0UiNXAx4SPfeSI2EJLAAAABIifHoqI0AAEiJhCQgAQAASIlcJFjoNiL//0iLhCQg +AQAASItcJFjoRCv//0iNBaZqAgC7CAAAAOgzK///6I4i//9Ii1QkaEiLdCRQSCnyuBcAAADrDUiN +czBAiHQMeEiNQf9Ig/oKcjJIicFIuM3MzMzMzMzMSInTSPfiSMHqA0iNNJJI0eZIKfNmDx9EAABI +g/kYcsDpmwAAAEiD+BgPg4cAAABIg8IwiFQEeEiNSOhIicpIwfk/SCHISI1cBHhI99pIjYQkkAAA +AEiJ0WaQ6NuMAABIiYQkIAEAAEiJXCRY6Gkh//9Ii4QkIAEAAEiLXCRY6Hcq//9IjQWXaAIAuwcA +AADoZir//+jBIf//kOg7If//6LYj///osSH//0iLhCQ4AQAA6UL+//+5GAAAAOg6egEASInIuRgA +AADoLXoBAEiJyLkYAAAADx9EAADoG3oBAEiJyLkYAAAA6A56AQBIjQUQsQIAuzIAAABmkOh7CP// +kEiJRCQI6JByAQBIi0QkCOnG+v//zMzMzMzMSIPsCEiJLCRIjSwkSIsIkEiLUAhIi3BQSInPSMHp +IkmJ0EjB6iIp0cHhAsH5AkhjyUgB8UiD+QF8TUiLSDhEKcfB5wLB/wJIY9dIAcpIA1gohcl0PUyJ +wEiJ1jHS9/GJ0kiNPBpIjX8CSDn5fQZIKdFIKc5IjUsCSDnOD53ASIssJEiDxAjDMcBIiywkSIPE +CMPoyeL+/5DMzMzMzMzMzEiD7AhIiSwkSI0sJEiLEJBIi3AISIt4UEmJ0EjB6iJJifFIwe4iKfLB +4gLB+gJIY9JIAfpIg/oCD4yEAAAASItQOEUpyEHB4AJBwfgCSWPwSAHWSIt4KEgB+4XSdG9MichJ +idAx0kH38InSTI0ME02NSQJNOch9DU2JwUkp0EwpxjHS6wNNicFIAdpIKd5IjRwPSI0MD0iNSQJI +jRwTSI1bBEk52X0ISSnRTCnO6wRIg8b+SDnOD53ASIssJEiDxAjDMcBIiywkSIPECMPo7uH+/5DM +zMzMzMzMzMzMzMzMSTtmEA+GEwUAAEiDxIBIiWwkeEiNbCR4SIm8JKAAAABMiYwkuAAAAEiFwA+E +FAEAAEg5cCgPjNAEAABIiZwkkAAAAEiJtCSoAAAASIm8JKAAAABMiYwkuAAAAEyJlCTAAAAASImM +JJgAAABIiYQkiAAAAEyLYBBFheR3BjHSZpDrUUyJZCRouwEAAABMidHojP7//0iLjCSYAAAASIuc +JJAAAABIi7QkqAAAAEiLvCSgAAAATIuMJLgAAABMi5QkwAAAAEyLZCRoicJIi4QkiAAAAITSdA5M +i2AQTItoGJDpcwMAAEWF5HYHugEAAADrRUyJ0+iH/f//g/ABSIuMJJgAAABIi5wkkAAAAEiLtCSo +AAAASIu8JKAAAABMi4wkuAAAAEyLlCTAAAAAicJIi4QkiAAAAITSdBWQDx9EAADpqAIAAEiLbCR4 +SIPsgMOQTIsAkEyLQAhMi1hQTItgSEWF2w+EfAIAAE2JxUnB6CJIicJMicBJidAx0kH382aQSIXb +dBKJ0Ek5ww+GTQIAAEyLG02JHMRNi1g4TYtgMJBFhdsPhC8CAABMiegx0kH384nQTYtoKEkBxU+N +bBUATY1tAk05630cSTnDD4b/AQAASccExAAAAABNi1g4SSnDMcDrA0Ux202LYDBNi2g4SYtQQEw5 +6A+HzAEAAEkpxUgpwkmJ10j32kjB4ANIwfo/SCHCTInYTo0cIkyJy02LSChPjQwRTY1JAk2F7Q+G +jAEAAE2JDBRmDx+EAAAAAACQSYP9AQ+GZwEAAEmJTBQISYtIKEyNSQJmDx+EAAAAAABNOc8Pgj0B +AABJg/kCD4ImAQAASDnOSA9MzkmNV/5I99pIwfo/SIPiEEwB2kg513RdTIlsJFBMiVwkcEiJRCRg +SIlMJFhIweEDSInQSIn76Mx+AQBIi0QkYEiLTCRYSIucJLgAAABMi4QkiAAAAEyLlCTAAAAATItc +JHBMi2wkUOsMScdEyxAAAAAASP/BSTlIKHYRSI1RAkk51XflDx8A6YkAAAAxyesHSYkU80j/wUk5 +yn4aSIsUy0mLcChIjTQOSI12Akw57nLf61dIidCQSYtICEmNFAJJA1AoSIPCAkiJy0jB6SJI/8FI +weEiAdpICdFIicJIidjwSQ+xSAgPlMGEyXTGSA+64yBzDEmNgIAAAADo59v8/0iLbCR4SIPsgMNI +ifBMieno8nQBAEiJ0EyJ6ejndAEAuAIAAABMicnoenUBAEyJyUyJ+uhPdQEAuAEAAABMienoonQB +ADHATInp6Jh0AQBMienoUHUBAEyJ2eiIdAEA6APe/v9Midnoe3QBAOj23f7/SInwSItQEIXSdCOD ++v90GUiNWgFIicZIidDwSA+xXhAPlMKE0nTZ6x9IicbrGkiHSBhIweogSI1KAUjB4SBI/8FIh0gQ +SInGSInw6GYBAABIi2wkeEiD7IDDTIt6EEyJ+EyLehhNif1JicRIidBFheR0LU2J50nB7CBJ/8RJ +weQgSInCTIn48EwPsWIQQQ+UxA8fRAAARYTkdL9Fhf/rDEiJwk2J50G9AAAAAHZrSMdEJEgAAAAA +RYn8TIlkJEhIidAx20yJ6TH/MfZJifBMjUwkSEG6AQAAAE2J0+g1+///SIuMJJgAAABIi5QkiAAA +AEiLnCSQAAAASIu0JKgAAABIi7wkoAAAAEyLjCS4AAAATIuUJMAAAABIidDpRPz//0iNBaJ/AgC7 +FwAAAOjEAf//kEiJRCQISIlcJBBIiUwkGEiJfCQgSIl0JChMiUQkMEyJTCQ4TIlUJEBMiVwkSOix +awEASItEJAhIi1wkEEiLTCQYSIt8JCBIi3QkKEyLRCQwTItMJDhMi1QkQEyLXCRIDx9AAOl7+v// +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEHZPSIPsEEiJbCQISI1sJAjrA0iJ2EiLSAhI +icpID7rpIUiJw0iJ0PBID7FLCA+UwYTJdN5ID7riIHMMSI2DgAAAAOh12fz/SItsJAhIg8QQw0iJ +RCQI6AFrAQBIi0QkCOuazMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdkFIg+wYSIlsJBBI +jWwkEIkFyucKAIM9v+kKAAB1CUiJHb78BwDrDEiNPbX8BwDoqG4BAOijyf7/SItsJBBIg8QYw4lE +JAhIiVwkEOiLagEAi0QkCEiLXCQQZpDrnszMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7 +ZhAPhgkBAABIg+xASIlsJDhIjWwkOEhjDUXnCgBIjQV66QEASInL6NJHAABIYxUv5woASIkVfP0H +AEiJFX39BwCDPRbpCgAAdQlIiQVd/QcA6xBIjT1U/QcADx9AAOi7bAEAMcDrA41BATkF8uYKAA+O +hQAAAIlEJBxIY8hIweEDSAMN3vsHAEiLCUiJTCQgSInI6IaGAABEDxF8JChIi0wkIEiJTCQoSIlE +JDCLTCQcSGPRSIsd8/wHAEiLNfT8BwBIi3wkKEg51nY5SMHiBEiJRBMISI00E4M9fugKAAB1CUiJ +PBPpfP///0iJ+EiJ9+gobAEA6Wz///9Ii2wkOEiDxEDDSInQSInx6K5wAQCQ6EhpAQDp4/7//8zM +zEk7ZhAPhhMBAABIg+wwSIlsJChIjWwkKDHA6wL/wIsVIOYKAI0UEI1SAUhj0kjB4gNIAxUQ+wcA +SIM6AHXeiUQkGEhjyEiJTCQgSInLSI0FLOgBAOiHRgAASItUJCBIiRVT/AcASIkVVPwHAIM9zecK +AAB1CUiJBTT8BwDrDEiNPSv8BwDodmsBADHA6wONRgGLTCQYOch9bolEJByLDaHlCgCNDAGNSQFI +Y8lIweEDSAMNkfoHAEiLCUiJyA8fAOh7WAEASIsN5PsHAEiLFeX7BwCLdCQcSGP+SDn6djJIwecE +SIlcOQhIjRQ5gz1N5woAAHUGSIkEOeuTSInXZpDo+2oBAOuHSItsJChIg8Qww0iJ+EiJ0eiEbwEA +kA8fAOgbaAEA6db+///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+G4AEAAEiD7BhIiWwkEEiN +bCQQSMcFpeYKACoAAABIxwWS5goAAAAAADHASI0NkeYKALoBAAAA8EgPsRFAD5TGQIT2D4WKAQAA +SIM9auYKAAAPhWsBAABIxwVZ5goAKgAAALgqAAAA8EgPsREPlMKE0g+EOgEAAEiDPTzmCgAqD4Ub +AQAASIM9NuYKAAEPhQ0BAABIixUp5goAkEiD+gEPheUAAABIugEAAAAAAQAASIcRSIsVCuYKAEi+ +AQAAAAABAABIOfIPha0AAABIugEAAAAAAQAA8EgPwRFIvgEAAAAAAQAASAHySL4CAAAAAAIAAEg5 +1nVxSIsVxeUKAEg58nVUSLoDAAAAAAMAAEiHEUg58nUxSIsNp+UKAEi6AwAAAAADAABIOdF1CkiL +bCQQSIPEGMNIjQUnaAIAuw0AAADolfz+/0iNBRZoAgC7DQAAAOiE/P7/SI0F+GcCALsNAAAA6HP8 +/v9IjQXnZwIAuw0AAADoYvz+/0iNBTFpAgC7DgAAAOhR/P7/SI0FUGcCALsNAAAADx9EAADoO/z+ +/0iNBSxlAgC7DAAAAOgq/P7/SI0FG2UCALsMAAAA6Bn8/v9IjQUKZQIAuwwAAADoCPz+/0iNBflk +AgC7DAAAAOj3+/7/kOgRZgEA6Qz+///MzMzMzMzMzMzMzMxJO2YQD4bRAwAASIPsQEiJbCQ4SI1s +JDjHRCQkAAAAAA9XwPMPEUQkIPMPEUQkHA9XwPIPEUQkMPIPEUQkKMdEJBgAAAAAMcC5HgAAAEi6 +Mc5XSzoLAADrB0j/yQ8fQABIhcl8G74Aypo7SNPmSDnyfOdIKfIPq8jr32YPH0QAAEiB+gDKmjt8 +D8dEJCQAAAAAuP///3/rCIlUJCQPH0AAPTkwAAAPhR4DAACBfCQkMdQAAA+FEAMAAMdEJBQAAAAA +x0QkFAEAAAC4AQAAAEiNTCQUugIAAADwD7ERD5TCkITSD4TQAgAAg3wkFAIPhbQCAADHRCQUBAAA +ALgFAAAAugYAAADwD7ERD5TChNIPhYICAACDfCQUBA+FZgIAAMdEJBT/////uP////+6/v////AP +sREPlMEPH4QAAAAAAITJD4QoAgAAg3wkFP4PhQwCAADHRCQYAAAAAMdEJBgBAQEBufD///9IjVQk +GfAICoB8JBgBD4XTAQAAgHwkGfEPhcgBAACAfCQaAQ+FvQEAAIB8JBsBD4WyAQAAx0QkGAAAAADH +RCQY/////7kBAAAA8CAKgHwkGP8PhX4BAACAfCQZAQ+FcwEAAIB8JBr/D4VoAQAAgHwkG/9mDx9E +AAAPhVcBAABIx0QkMP/////yDxBEJDBmDy7AdQsPH0QAAA+LJgEAAHUGD4sNAQAASMdEJCj+//// +8g8QRCQo8g8QTCQwZg8uyHUGD4vbAAAAdQYPi8IAAADHRCQg//////MPEEQkIA8uwHUGD4uYAAAA +x0QkHP7////zDxBEJBzzDxBMJCAPLsh1Ants6JD7//8xwOsDSP/ASIP4IBnSSInBvgEAAADT5iHW +gf4ACAAAfON1M+jIZQEARQ9X/2RMizQl+P///4A8JAB0CkiLbCQ4SIPEQMNIjQVMcwIAuxYAAADo +Gvn+/0iNBb+AAgC7HAAAAOgJ+f7/SI0FM2ACALsLAAAA6Pj4/v9IjQV5XgIAuwoAAADo5/j+/0iN +BTJgAgC7CwAAAOjW+P7/SI0FFmACALsLAAAA6MX4/v9IjQX6XwIAuwsAAADotPj+/0iNBT9eAgC7 +CgAAAOij+P7/SI0FEF4CALsKAAAA6JL4/v9IjQVHWwIAuwkAAADogfj+/0iNBXZWAgC7BAAAAOhw ++P7/SI0FYVYCALsEAAAADx9AAOhb+P7/SI0FSFYCALsEAAAA6Er4/v9IjQUzVgIAuwQAAADoOfj+ +/0iNBR5WAgC7BAAAAOgo+P7/SI0FCVYCALsEAAAA6Bf4/v9IjQUVXwIAuwsAAADoBvj+/5APH0QA +AOgbYgEA6Rb8///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+G7gIAAEiDxIBIiWwkeEiNbCR4 +xwW+4goAAQAAAEi5AQAAAAEAAABIiQ3N4goASI0FjFcCALsHAAAAkOjbnvz/6wpIi1wkIEiLRCRw +SIXbD4RqAQAASIlcJCBIiUQkWEiJBCRIiVwkCMZEJBAs6Mlh/P9FD1f/ZEyLNCX4////SItMJBhI +hcl9EEiLVCRYMfYxwEiLTCQg6zZIi1QkIEg5yg+CRgIAAEiNQQFIOcIPgjECAABIKcpI/8pIidZI +99pIwfo/SCHQSItUJFhIAdBIiXQkIEiJRCRwSIlMJDBIiRQkSIlMJAjGRCQQPehKYfz/RQ9X/2RM +izQl+P///0iLTCQYSIXJD4w2////SItUJDBIOcoPgsYBAABIjUEBSDnCD4KsAQAASCnKSI1a/0iJ +2kj320jB+z9IIdhIi1wkWEgB2EiD+Q51PEi+bWVtcHJvZmlIOTN1N4F7CGxlcmF1LmaBewx0ZXUm +SInT6Gp8AACE2w+Eyf7//0iJBevdCgDpvf7//0i+bWVtcHJvZmlIiz2d0QcATIsFjtEHAEiF/w+O +nP7//0iJTCQoSIlUJEhIiUQkUEiJfCRARTHJ60uLDVbhCgALDVThCgALDVLhCgCFyQ+VBT3hCgBI +jQXeXAIAuwsAAADoMJ38/+jLTgEAiw01vgcAiQ1P3QoASItsJHhIg+yAw0mDwBhNi1AITYtYEE2L +IGYPH0QAAEk5yg+FngAAAEyJTCQ4TIlEJGhMiVwkYEyJ4EyJ0WaQ6Jte/P+EwHUvSItEJFBIi0wk +KEiLVCRISItcJFhIvm1lbXByb2ZpSIt8JEBMi0QkaEyLTCQ4609Ii0QkUEiLXCRI6Fl7AABIY8hI +Och1C4TbdAdIi1QkYIkCSItEJFBIi0wkKEiLVCRISItcJFhIvm1lbXByb2ZpSIt8JEBMi0QkaEyL +TCQ4Sf/BTDnPD484////6W/9//9IidEPH0QAAOg7ZwEA6LZmAQBIidHoLmcBAOipZgEAkOgDXwEA +Dx8A6fv8///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4bcAAAASIPsKEiJbCQgSI1s +JCBIiVwkOEiJRCQwSIX/uQAAAABID0z56wZIifBIidNIOft+YUiJfCQY6Bt6AQBFD1f/ZEyLNCX4 +////SItMJBhIg/kQuBAAAABIicpID0/ISCnKSIs0JEiLfCQ4SDnXdl5Ii3wkMEiNBBdIifPoF2AB +AEiLVCQ4SIt0JDBIi3wkGDHJ6xdIi2wkIEiDxCjDiAQ+SP/BSP/HSMHoCEiD+QgPjXP///8PHwBI +OfoPjmf///9320iJ+EiJ0ehqZQEASInQSIn5Dx9AAOhbZQEAkEiJRCQISIlcJBBIiUwkGEiJfCQg +6OFdAQBIi0QkCEiLXCQQSItMJBhIi3wkIOno/v//zMzMzMzMzMw8G3INSI0FgWkCALsTAAAAww+2 +yEjB4QRIjRUh1AcASIsECkiLXAoIw8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEHZqSIPsIEiJ +bCQYSI1sJBhJi04wkP+BCAEAAEyJ8YQBuQEAAADwD8FIKP/Bhcl9MkQPEXwkCEiNDV8AAABIiUwk +CEiJRCQQSI1EJAhIiQQk6OdbAQBFD1f/ZEyLNCX4////SItsJBhIg8Qgw0iJRCQI6AZdAQBIi0Qk +CJDpe////8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhokAAABIg+wgSIlsJBhIjWwk +GEiLQghIiUQkCIQA6HjI/P9Ii0QkCItIEIXJdhP/yYlIEJCQDx9EAADoO8r8/+tCSYtOMEiJTCQQ +SItQCEiJkWABAACQSIlICJCQ6BnK/P9Ii0wkEEiNgVABAADoaMv8/5BIi0wkEEjHgVABAAAAAAAA +SItsJBhIg8Qgw+ioWwEA6WP////MzMxJO2YQD4ayAAAASIPsGEiJbCQQSI1sJBC5//////APwUgo +/8mFyX1Og/n/dH2B+f///790dbn/////8A/BSCz/yYXJdTFIiUQkIJDosMf8/0iLRCQgSItIIEiF +yXQRSI2BUAEAAOg2yvz/SItEJCCQkOhqyfz/SYtGMIuICAEAAI1R/4mQCAEAAIP5AXUSQYC+sQAA +AAB0CEnHRhDe+v//SItsJBBIg8QYw0iNBYV3AgC7GwAAAOhl8f7/kEiJRCQI6HpbAQBIi0QkCOkw +////zMzMzMzMzMzMzMzMzMzMzEk7ZhB2WEiD7CBIiWwkGEiNbCQYSIN4KAB0MUiJRCQoSIlcJBDo +1lwBAEUPV/9kTIs0Jfj///9IiwQkSItMJChIiUEoSInISItcJBBIiwDobBz//0iLbCQYSIPEIMNI +iUQkCEiJXCQQ6PNaAQBIi0QkCEiLXCQQ64fMzMzMzMzMSTtmEA+G8gIAAEiD7GBIiWwkWEiNbCRY +SYtWMEyJ9pBIObLAAAAAD4W9AgAAkOsDTInAixCF0nQejXL/SYnAidDwQQ+xMA+UwoTSdONIi2wk +WEiDxGDDSIlMJHhIibwkgAAAAIhcJCdIiUQkaOiOHP//SIlEJEiQSItMJGhIicpIx0AoAAAAAEjH +QCAAAAAAx0AwAAAAAEjB6QNIicNIuEdBQHN9fxkFSInWSPfhSAHKSNHaSMHqB0hpwvsAAABIKcFI +iUwkOEiLRCR4D7rgAHNKSIM9ptcKAAB2QA8fQADom1sBAEUPV/9kTIs0Jfj///9IiwQkSItMJEhI +x0Eo/////0iLTCQ4SItcJEhIi3QkaEiJwkiLRCR46wIx0g+64AFzOEiDPfTXCgAAdi5IhdJ1JehI +WwEARQ9X/2RMizQl+P///0iLFCRIi0wkOEiLXCRISIt0JGhIiVMgSIlUJChIweEGSIlMJDBIjT2P +LAgATI0ED0yJRCRA6wxIi0wkMEiNPXgsCABIjQQPSIlEJFDoCsX8/0iLTCQwSI0VXiwIAEiNHApI +jVsQvgEAAADwD8EzSIt0JGjpjQAAAA8fAEWFwA+FpwAAAEiLRCRASInzSItMJEgPtnwkJw8fRAAA +6HsDAABIi5QkgAAAAEiNcgRIjQVIoQIASItcJFC5EgAAAL8ZAAAA6PQY//9Ii0QkSIN4MAB0D0iL +TCRoSInDugEAAADrGkiLTCRo6ZYAAACF0usDSInDD5XCZg8fRAAAhNIPhDT////rO0SLBkWFwA+E +bf///0WNSP9EicDwRA+xDkEPlMFFhMl03+lQ////uf/////wD8ELkJBIi0QkUOj7xfz/SItEJEhI +i0goSIXJfiFIi1QkKEgp0UiLlCSAAAAASI1aA0iJyOhxh/7/SItEJEjoRx3//0iLbCRYSIPEYMNI +idiLEYXSD4Rk////jXL/SInDidDwD7ExQA+UxkCE9nTe6Ub///9IjQXPeQIAux0AAADoxe3+/5BI +iUQkCIhcJBBIiUwkGEiJfCQg6MxXAQBIi0QkCA+2XCQQSItMJBhIi3wkIOnT/P//zMzMzMzMzMzM +zMzMzMzMzMzMzEk7ZhAPhtwBAABIg+xISIlsJEBIjWwkQEiJwr4BAAAA8A/BMkjB6ANIicZIuEdB +QHN9fxkFSInXSPfmSAHySNHaSMHqB0hp0vsAAABIKdZIweYGSI0VZioIAEyNBDJNjUAQRYsIRYXJ +D4TiAAAAiFwkWEiJTCRgTIlEJDhIiXQkGEiJfCQwSI0EMkiJRCQo6MzC/P9Ii0wkOIsRDx9EAACF +0g+EkgAAAEiLTCQYSI0VDCoIAEiNBApIi1wkMA8fAOj7BAAASIlEJCBIiVwkEEiFwHQOuf////9I +i1QkOPAPwQqQkEiLRCQo6FLE/P9Ii0QkIEiFwHRiSItIIA8fQABIhcl0G0iLRCQQSCnISItMJGBI +jVkD6AVCAQBIi0QkIIN4MAAPhakAAAAPtkwkWITJdDFIi0wkMOtikJBIi0QkKOj6w/z/SItsJEBI +g8RIw0iLbCRASIPESMNIi2wkQEiDxEjDSItMJGBIjVkF6M76//9Ii0wkIIN5MAF13UmLTjCDuQgB +AAAAddCQSI0F5p0CAOhJVAEA68FIidiLEWaQhdJ0GY1y/0iJw4nQ8A+xMUAPlMZAhPZ04IXS6wNI +icN1BUiJ2Oudx0MwAQAAAEiJ2OuRSI0FC28CALsaAAAAkOib6/7/kEiJRCQIiFwkEEiJTCQY6KdV +AQBIi0QkCA+2XCQQSItMJBjp8/3//8zMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4ZTAwAASIPsGEiJ +bCQQSI1sJBCEAUiNUQhIjXEQgz1X1AoAAHURTYnwTIkBSIlZGEQPEXkI6zRBifhIic9NifHollkB +AEyNSRhMic/oKlkBAEiJ10UxyQ8fQADoe1kBAEiJ9+hzWQEARInHTI1ACEyLSAhFMdLrD02LGkyJ +w02J0E2Jyk2J2U2FyQ+E+gEAAE2LURhMOdN0FEmJ2Ek52nYGTY1REOvRTY1RCOvLQIT/D4RgAQAA +gz2+0woAAHUFSYkI6whMicfob1gBAEGLQTCJQTBJi0EgSIlBIEmLQTiDPZXTCgAAdQZIiUE46wlI +jXk46ERXAQBJi0EQgz150woAAHUGSIlBEOsISIn36ClXAQBJi1kIgz1e0woAAHUGSIlZCOsISInX +6E5YAQBIhcB0GoM9QtMKAABmkHUGSIlIOOsJSI14OOjvVwEASItBCEiFwHQYgz0f0woAAHUGSIlI +OOsJSI14OOjOVwEAgz0H0woAAHUHTIlJQJDrCUiNeUDoVVgBAEiNeUhJi0FIgz3m0goAAHUGSIlB +SOsF6JlWAQBIhcB1GYM9zdIKAAB1C0yJSUjrCg8fRAAA6BtYAQCDPbTSCgAAdSVJx0E4AAAAAEnH +QRAAAAAAScdBCAAAAABJx0FIAAAAAOmSAAAASY15ODHA6EJWAQBJjXkQ6DlWAQBJjXkI6DBWAQBJ +jXlI6CdWAQDrakmLQUiQSIXAdBqDPVTSCgAAdQZIiUhA6yNIjXhA6ANXAQDrGIM9OtIKAAB1BkmJ +SUDrCUmNeUDo6VYBAIM9ItIKAABmkHUOSYlJSEjHQUAAAAAA6xVJjXlI6MdWAQBIjXlAMcCQ6LtV +AQBIi2wkEEiDxBjDSYtWMIuyIAEAAESLiiQBAABEiYogAQAAQYnzweYRQTHzRInORTHZQcHrB0Ux +y0GJ8cHuEEQx3omyJAEAAEGNFDGDygGJUTCDPaPRCgAAdQlMiVE4SYkI6xRIjXk4TInS6GxWAQBM +icfoRFYBAEiJTCQwSIlEJCDrCkiLRCQgSItMJDBIi1k4SIXbdCSLUzBmkDlRMHMaSDlLEHQNSDlL +CHUY6KoDAADr0OjjBAAA68lIi2wkEEiDxBjDSI0FcNEBAEiNHbmZAwDoZOD+/5BIiUQkCEiJXCQQ +SIlMJBhAiHwkIOgKUgEASItEJAhIi1wkEEiLTCQYD7Z8JCDpcfz//8zMzMzMzMzMzMzMzMzMzMzM +STtmEA+GEAMAAEiD7DhIiWwkMEiNbCQwSI14CEiLSAhIifrrDEiLMUyJw0iJz0iJ8UiFyQ+EmAEA +AEiLcRhmkEg583QUSYnYSDnedgZIg8EQ69JIg8EI68xIiVQkKEiJTCQYSIlEJEBIg3kgAHUEMdvr +MkiJfCQg6ANTAQBFD1f/ZEyLNCX4////SIsEJEiLTCQYSItUJChIi3wkIEiJw0iLRCRASItxQEiF +9nUKSIlcJBDpngEAAIM9IdAKAACQdQVIiTfrBeg0VQEAi1EwiVYwSItROIM9A9AKAAB1BkiJVjjr +CUiNfjjo0lQBAEiLURCDPefPCgAAdQdIiVYQkOsJSI1+EOi1VAEASIXSdBiDPcnPCgAAdQZIiXI4 +6wlIjXo46NhUAQBIi1EIgz2tzwoAAHUGSIlWCOsKSI1+CJDoe1QBAEiF0nQbgz2PzwoAAHUGSIly +OOsMSI16OA8fAOibVAEASIN+QAB0HkiLUUiDPWnPCgAAdQZIiVZI6ylIjX5I6DhUAQDrHoM9T88K +AAB1CkjHRkgAAAAA6wtIjX5IMdLoGFQBAEiJXiCDPS3PCgAAdQdEDxF5QOslSI15QDHS6PlTAQBI +jXlI6PBTAQDrDzHAMdtIi2wkMEiDxDiQw4M9+M4KAAB1GEjHQTgAAAAASMdBGAAAAABEDxF5CJDr +LEiNeTgx0uizUwEASI15GDH26OhTAQBIjXkIDx9AAOibUwEASI15EOiSUwEAx0EwAAAAAEiJyEiL +bCQwSIPEOMNIi0QkQEiLTCQYSItUJChIi1wkEEiLcQhIhfZ1DEiDeRAAdCtIhfZ0HEiLeRBIhf90 +CIt/MDl+MHcLSInLkOi7AAAA67tIicvo8QEAAOuxSItxOEiF9nRbZg8fRAAASDlOEHUmgz0zzgoA +AHUNSMdGEAAAAADpJf///0iNfhAx0uj5UgEA6RX///+DPQ3OCgAAdRBIx0YIAAAAAA8fAOn8/v// +SI1+CDHS6NBSAQDp7P7//4M95M0KAAB1DUjHQAgAAAAA6db+//9Iidcx0uirUgEA6cf+//9IiUQk +CEiJXCQQ6LdOAQBIi0QkCEiLXCQQ6cj8///MzMzMzMzMzEk7ZhAPhv8AAABIg+wYSIlsJBBIjWwk +EEiLSwhIi1M4SItxEIM9dc0KAAB1DkiJWRBIiUs4SIlzCOscSI15EJDoW1IBAEiNezjoElIBAEiN +ewjoaVIBAEiF9nQYgz09zQoAAHUGSIleOOsJSI1+OOgsUgEAgz0lzQoAAHUGSIlROOsJSI15OOj0 +UQEASIXSdEJIOVoQdRyDPQLNCgAAZpB1BkiJShDrRUiNehDor1EBAOs6SDlaCHU+gz3gzAoAAHUG +SIlKCOslSI16COiPUQEA6xqEAIM9xMwKAAB1BkiJSAjrCUiNeAjoc1EBAEiLbCQQSIPEGMNIjQUP +WQIAuxMAAADoeOP+/5BIiUQkCEiJXCQQ6IhNAQBIi0QkCEiLXCQQ6dn+///MzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMSTtmEA+G/wAAAEiD7BhIiWwkEEiNbCQQSItLEEiLUzhIi3EIgz01zAoAAHUO +SIlZCEiJSzhIiXMQ6xxIjXkIkOgbUQEASI17OOjSUAEASI17EOgpUQEASIX2dBiDPf3LCgAAdQZI +iV446wlIjX446OxQAQCDPeXLCgAAdQZIiVE46wlIjXk46LRQAQBIhdJ0Qkg5WhB1HIM9wssKAABm +kHUGSIlKEOtFSI16EOhvUAEA6zpIOVoIdT6DPaDLCgAAdQZIiUoI6yVIjXoI6E9QAQDrGoQAgz2E +ywoAAHUGSIlICOsJSI14COgzUAEASItsJBBIg8QYw0iNBTVaAgC7FAAAAOg44v7/kEiJRCQISIlc +JBDoSEwBAEiLRCQISItcJBDp2f7//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxMjWQk0E07ZhAP +hssFAABIgeywAAAASImsJKgAAABIjawkqAAAAEiJhCS4AAAASItICJBIi4mQAAAASImMJIgAAADo +Gfr+/0iNBTJCAgC7BwAAAOgoA///SIuEJIgAAADoewH//+h2/P7/6HH6/v9Ii4QkuAAAAEiLSAiQ +SIuJgAAAAEiJTCR46NP5/v9IjQX6QQIAuwcAAADo4gL//0iLRCR46DgB///oM/z+/+gu+v7/SIuE +JLgAAABIi0gIkEiLiZgAAABIiUwkWOiQ+f7/SI0FvkECALsHAAAADx9AAOibAv//SItEJFjo8QD/ +/+js+/7/6Of5/v9Ii4QkuAAAAEiLSAiQSIuJiAAAAEiJTCRw6En5/v9IjQWFQQIAuwcAAADoWAL/ +/0iLRCRw6K4A///oqfv+/+ik+f7/SIuEJLgAAABIi0gIkEiLSWhIiUwkGOgJ+f7/SI0FPkECALsH +AAAA6BgC//9Ii0QkGOhuAP//6Gn7/v/oZPn+/0iLhCS4AAAASItICJBIi0lwSImMJJgAAADoxvj+ +/0iNBRdBAgC7BwAAAOjVAf//SIuEJJgAAADoKAD//+gj+/7/Dx8A6Bv5/v9Ii4QkuAAAAEiLSAiQ +SItJeEiJjCSAAAAAZpDoe/j+/0iNBZtAAgC7BwAAAOiKAf//SIuEJIAAAABmkOjb//7/6Nb6/v/o +0fj+/0iLhCS4AAAASItICJBIi4mgAAAASIlMJBDoM/j+/0iNBYtAAgC7BwAAAOhCAf//SItEJBDo +mP/+/+iT+v7/6I74/v9Ii4QkuAAAAEiLSAiQSItJKEiJTCQo6PP3/v9IjQX+PwIAuwcAAADoAgH/ +/0iLRCQo6Fj//v/oU/r+/+hO+P7/SIuEJLgAAABIi0gIkEiLSTBIiUwkOOiz9/7/SI0FxT8CALsH +AAAA6MIA//9Ii0QkOOgY//7/6BP6/v/oDvj+/0iLhCS4AAAASItICJBIi0k4SIlMJFDoc/f+/0iN +BVQ/AgC7BwAAAOiCAP//SItEJFDo2P7+/+jT+f7/6M73/v9Ii4QkuAAAAEiLSAiQSItJQEiJTCRg +6DP3/v9IjQUbPwIAuwcAAADoQgD//0iLRCRg6Jj+/v/ok/n+/+iO9/7/SIuEJLgAAABIi0gIkEiL +SUhIiUwkaOjz9v7/SI0F4j4CALsHAAAA6AIA//9Ii0QkaOhY/v7/6FP5/v/oTvf+/0iLhCS4AAAA +SItICJBIi0lQSImMJJAAAADosPb+/0iNBaY+AgC7BwAAAA8fQADou//+/0iLhCSQAAAA6A7+/v/o +Cfn+/+gE9/7/SIuEJLgAAABIi0gIkEiLSVhIiUwkQOhp9v7/SI0FZj4CALsHAAAA6Hj//v9Ii0Qk +QOjO/f7/6Mn4/v/oxPb+/0iLhCS4AAAASItICJBIi0lgSIlMJCDoKfb+/0iNBS0+AgC7BwAAAOg4 +//7/SItEJCDojv3+/+iJ+P7/6IT2/v9Ii4QkuAAAAEiLSAiQSIuJqAAAAEiJTCRI6Ob1/v9IjQUw +PgIAuwcAAADo9f7+/0iLRCRI6Ev9/v/oRvj+/+hB9v7/SIuEJLgAAABIi0gIkEiLibAAAABIiUwk +MOij9f7/SI0F5j0CALsHAAAA6LL+/v9Ii0QkMOgI/f7/6AP4/v8PHwDo+/X+/0iLhCS4AAAASItI +CJAPt4m4AAAASImMJKAAAADoWvX+/0iNBQo9AgC7BwAAAOhp/v7/SIuEJKAAAACQ6Lv8/v/otvf+ +/+ix9f7/SIuEJLgAAABIi0gIkA+3ibwAAABIiYwkoAAAAOgQ9f7/SI0FzjwCALsHAAAADx9AAOgb +/v7/SIuEJKAAAADobvz+/+hp9/7/6GT1/v9Ii4QkuAAAAEiLQAiQD7eAugAAAEiJhCSgAAAA6MP0 +/v9IjQWPPAIAuwcAAADo0v3+/0iLhCSgAAAA6CX8/v8PH0QAAOgb9/7/6Bb1/v9Ii6wkqAAAAEiB +xLAAAADDSIlEJAiQ6DtGAQBIi0QkCOkR+v//zMzMzMzMzMzMzMzMzMzMzMxJO2YQD4a+AAAASIPs +KEiJbCQgSI1sJCBIiUQkMEiLUAhIi7KgAAAAhACQkEiLNpBIi5qoAAAASIlcJBhIichIifHoOeX+ +/4TAdFBIjQV+jgIAhABIi0QkMEiLSAhIi5GgAAAASIPC+EiJ04QBkJBIiw1ajgIAkEiLdCQYSIky +SItQCJCQSImaoAAAAEiLQAiQkEiJiKgAAADrI0iNBS6OAgCEAEiLRCQwSItACJCQSIsNGo4CAJBI +iYioAAAASItsJCBIg8Qow0iJRCQIiVwkEEiJTCQY6EVFAQBIi0QkCItcJBBIi0wkGOkS////zMzM +zMzMzMzMzMzMzMzMzMzMSIPsUEiJbCRISI1sJEiEwHUHxgX3wQoAAYA968EKAAB0BITA6wuAPeDB +CgAAdA6EwHUKSItsJEhIg8RQwzHAkOsC/8CD+EEPg1gBAABIjRRASI01KMAHAIs81oX/dOMPuucE +ct2JRCQcSIlUJCBEDxF8JChEDxF8JDgx20iNTCQo6BpO/P9Ii1QkKItEJBxIjTVq0AoASIkUxo1Q +/w8fAIP6AXcYSI0UxkiLEkiD+gF1C0iNPce/BwAxyetDSItUJCBIjT23vwcAixTXD7riBnMEMcnr +KoA9LsEKAAB1CYA9J8EKAAB0Dw+64gNyCYP4DXQEMcnrCbkBAAAADx9AAITJdW5IixTGSIXSdFVI +g/oBdTSQicHB6AUPH4AAAAAASIP4Aw+DhQAAAEiNFY/ECgBMjQSCQbkBAAAAQdPhRAsMgkWHCOsd +6DSr/v+LTCQcSI01qc8KAEiNPSK/BwBmkOsCicGJyEiNNRO/BwDp0P7//0iNDcfKCgDHBIEBAAAA +SI0VQYwCAIQCSIsdOIwCAOgrqv7/i0QkHEiNNeC+BwDpnf7//0iLbCRISIPEUMO5AwAAAOjHSgEA +kMzMzMzMzEk7ZhAPhvQAAABIg+woSIlsJCBIjWwkIIC4sQAAAACQdRtIi1AwSIuS0AAAAEiF0nQh +hAKAuggnAAAAdBKLkJAAAAAPuvIMg/oCD5TC6wYx0usCMdKE0nR+SIlcJDhIiUQkMEiLUwiQkEiL +sqgAAACQSIuKoAAAADH/SInzDx8A6Nvr/v+EwHRJSI0FaIkCAIQASItEJDhIi0gISIuRoAAAAEiD +wvhIidaEAZCQSIsNRIkCAJBIiRpIi1AIkJBIibKgAAAASItACJCQSImIqAAAAEiLRCQwSItIMLoB +AAAA8A/BkVADAABIi0AwMcmHiFQDAABIi2wkIEiDxCjDSIlEJAhIiVwkEOhTQgEASItEJAhIi1wk +EOnk/v//zMzMzEyJ8MPMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSIPscEiJbCRoSI1sJGiJ +RCR4SImcJIAAAABIiYwkiAAAAOi5FQAAhMAPhbwBAABEDxF8JFhIi4wkgAAAAEiJTCRYSIuUJIgA +AABIiVQkYEiNRCRY6If///9IiUQkUEiJBCTo+UIBAEUPV/9kTIs0Jfj///9Ii0wkUEiFyXVUi0wk +eIP5G3Qug/kXdRODPaXCCgAAdQpIi2wkaEiDxHDDichIjVwkWOh1FAAASItsJGhIg8Rww0iLTCRg +kJBIi4GoAAAA6FiU//9Ii2wkaEiDxHDDSItBMEiLQFBIiQQkZpDoe0IBAEUPV/9kTIs0Jfj///9I +x0QkKAAAAABIjUQkMEQPEThIjUQkQEQPEThIi0QkUEiLWDBIjUwkKItEJHiQ6NsAAACEwHQbSIt8 +JFBIi1cwSItSUEiNdCR4SImygAAAAOsFSIt8JFCIRCQnSIF/EC77//91E4tEJHgPH0QAAOg7EwAA +SIt8JFCLRCR4SIucJIAAAABIi4wkiAAAAGaQ6DsDAABIi1QkUEiJFCTozUEBAEUPV/9kTIs0Jfj/ +//8PtlQkJ4TSdDhJi0YwSItAUJBIi0wkMEiLVCQoSIkQSIlICEiLTCQ4SIlIEEiLTCRASIlIGEiL +TCRISImIgAAAAEiLbCRoSIPEcMNIi2wkaEiDxHDDzMzMzMzMzMzMzMxIg+xgSIlsJFhIjWwkWEiJ +XCRwSIlMJHiJRCRoSItTUEiNdCRoSIl0JBBIOTJ3Ekg5cgh2DDHASItsJFhIg8Rgw0jHRCQoAAAA +AEiNRCQwRA8ROEjHBCQAAAAASI1EJChIiUQkCOh2YAEARQ9X/2RMizQl+P///0iLRCQoi0wkMA+6 +4QEPgs8AAABIi0wkEEg5yA+HxgAAAEiLVCQ4SAHCSDnRD4O1AAAAkEyJdCQYSItMJHhIhcl0SUmL +VjBIi1JQSIsaSItSCEiJGUiJUQhJi1YwSItSUEiLUhBIiVEQSYtWMEiLUlBIi1IYSIlRGEmLVjBI +i1JQSIuSgAAAAEiJUSBIi0wkGEiLUTBIi1JQSItcJChIiRpIi1EwSItSUEiLdCQ4SAHeSIlyCEiL +UTBIi1JQSIHDoAMAAEiJWhBIi0kwSItJUEiJWRi4AQAAAEiLbCRYSIPEYMNIi0wkEEiLVCRwSIsa +SDkLD4fmAAAASDlLCA+G3AAAAEjHRCRAAAAAAEiNTCRIRA8ROUiLCkiLWQhIKxlIiVwkUEiLCkiL +CUiJTCRAkEyJdCQgSItMJHhIhcl0SUmLVjBIi1JQSIsaSItSCEiJGUiJUQhJi1YwSItSUEiLUhBI +iVEQSYtWMEiLUlBIi1IYSIlRGEmLVjBIi1JQSIuSgAAAAEiJUSBIi0wkIEiLUTBIi1JQSItcJEBI +iRpIi1EwSItSUEiLdCRQSAHeSIlyCEiLUTBIi1JQSIHDoAMAAEiJWhBIi0kwSItJUEiJWRi4AQAA +AEiLbCRYSIPEYMNIxwQkAAAAAA8fAOj7PgEARQ9X/2RMizQl+P///+gpLv//i0QkMA+64AGQcwuL +RCRo6BUPAADrCYtEJGjoig8AAOiFMf//McBIi2wkWEiDxGDDzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzEyNZCTYTTtmEA+G6gcAAEiB7KgAAABIiawkoAAAAEiNrCSgAAAARA8RvCSQAAAASImcJJAA +AABIiYwkmAAAAJCD+BsPhCEFAABMiXQkcEiJvCTIAAAAiYQksAAAAA8fAIP4BXVESIsVZM8HAEiF +0nQ4SIsySInYSI2cJJAAAABIifn/1oTAdRGLhCSwAAAASIu8JMgAAADrEEiLrCSgAAAASIHEqAAA +AMOD+Ap1OUiLFSPPBwBIhdJ0LUiLCkiJ+P/RhMB1EYuEJLAAAABIi7wkyAAAAOsQSIusJKAAAABI +gcSoAAAAw4P4F3Uogz2FvQoAAHUfSIn4SI2cJJAAAADo+/j//4uEJLAAAABIi7wkyAAAAIP4QXMS +icFIjQxJSI0VmrcHAIsMyusMSI0VjrcHALkEAAAASIucJJAAAABIY1sISIXbdBQPuuEDcw6Av7cA +AAAAdAW5BAAAAIlMJChIi5QkmAAAAJCQSIuCqAAAAOgK3P7/SIuMJJAAAAAPttBIhdKLVCQouwQA +AAAPRdOJVCQoSGNJCA8fhAAAAAAASIXJD4SQAAAAD7riA3IZSIXJD4SBAAAAD7riAA8fQAAPg4sA +AADrcYucJLAAAABIi4wkyAAAAImZ8AAAAEiLlCSQAAAASGNSCEiJkRABAABIi5QkkAAAAJBIi1IQ +SImRGAEAAEiLlCSYAAAAkJBIi5KoAAAASImRIAEAAEiNhCSQAAAA6Bf1//9Ii6wkoAAAAEiBxKgA +AADDi4QksAAAAOjbFQAAhMAPhQ0DAACLVCQoSIucJJAAAABIY1sIDx8ASIXbdTyLhCSwAAAAicPB +6AVIg/gDD4NzBQAASI0NXrsKAEiNDIFIjUkgiwkPo9lzF0iLrCSgAAAASIHEqAAAAMOLnCSwAAAA +D7riAXMSidjoqwkAAItUJCiLnCSwAAAA98IMAAAAD4R+AgAASItEJHBIi0gwx4H0AAAAAQAAAEiL +SDBIi5QkyAAAAEiJ1kiJkcgAAACDPWy3CgAAdAWD+0HrI+gY1f7/i4QksAAAAIP4QUiLRCRwi5wk +sAAAAEiLtCTIAAAAc0mJ2EiNBEBIjQ2LtQcASItUwQhIiZQkiAAAAEiLRMEQSIlEJGjoT+j+/0iL +hCSIAAAASItcJGhmkOhb8f7/6Lbq/v/osej+/+s06Cro/v9IjQW3LwIAuwcAAADoOfH+/4uEJLAA +AACJwUiJyOgo7v7/6IPq/v8PHwDoe+j+/0iLRCRwSItIMEiLlCSYAAAASIucJJAAAABIi4noAAAA +SIlMJGCQkEiLkqgAAABIiVQkMEhjWwhIiVwkWOi65/7/SI0FsCwCALsDAAAA6Mnw/v9Ii0QkMA8f +QADoG+/+/0iNBX8sAgC7AwAAAOiq8P7/SItEJGAPH0QAAOib7v7/SI0FUjECALsJAAAA6Irw/v9I +i0QkWA8fRAAA6Hvt/v/o1un+/+jR5/7/SItEJHBIi0gwSIO5aAEAAAB0UIO5OAEAAAB+PUiLlCTI +AAAASDkRdULoI+f+/0iNBYZoAgC7JAAAAOgy8P7/6I3n/v9Ii0QkcEiLSDBIi7loAQAASIn66xJI +i5QkyAAAAOsISIuUJMgAAABIiZQkgAAAAIuMJLAAAACD+QR0CYP5CA+FvwAAAEiLjCSYAAAAkEiL +HQG3CgCQkEiLiagAAABIhdsPhPACAABIiUwkOEiJyDHSSPfzSCnTSIP7EHIFuxAAAABIiVwkQOiC +5v7/SI0FTkECALsSAAAA6JHv/v/o7Ob+/0iLRCQ4SIlEJHhIi0wkQDHS6WoCAABIi6wkoAAAAEiB +xKgAAADDSIusJKAAAABIgcSoAAAAw0mLdjCQkEiLgagAAACQSIuZoAAAADHJ6JqG//9Ii6wkoAAA +AEiBxKgAAADD6AXm/v8PH0QAAOh76P7/6Hbm/v+LBeCVBwBJi04wD7aJKQEAAInCwegChMkPRcGF +wA+O9AAAAIlUJCxIi4QkgAAAAOiC3QAASIuMJJgAAACQkEiLgagAAACQSIuZoAAAADHJSIu8JIAA +AADoOtAAAIM9W7QKAAB+PEiLTCRwSItRMEiLksAAAABIi4QkgAAAAEg50HQbSIXSdBKLkpAAAAAP +uvIMg/oCD5TC6xcx0usTMdLrD0iLhCSAAAAASItMJHAx0oTSdR+DPQO0CgAAdUnoNN8AAOgv5f7/ +6Krn/v/opeX+/+szSItRMEiLgsAAAADo09wAAEiLTCRwSItJMEiLucAAAABIx8D/////SInDMckx +9ugw0AAASI2EJJAAAADog+r//4tUJCwPuuIAD4N+AAAAiw2XswoA/8GJDY+zCgBIixVQ0gcASCsV +WdIHACsVf7MKADnKfk3oquT+/0iNBaQtAgC7CAAAAOi57f7/6BTl/v/HBCQDAAAA6EhRAQBFD1f/ +ZEyLNCX4////xwQkQEtMAOhvUAEARQ9X/2RMizQl+P///5C4BgAAAOgXBQAAxwQkAgAAAOgrTwEA +RQ9X/2RMizQl+P///0iLrCSgAAAASIHEqAAAAMNIiVQkSA+2DBBIiUwkUOgb5P7/6Fbm/v9Ii0Qk +UOiM6/7/6Ifk/v9Ii0QkSEiNUAFIi0QkQEiLTCR4SInISItMJEBIOdF2D4QADx9EAABIg/oQcqzr +FOjT4/7/6E7m/v/oSeT+/+m6/f//SInQuRAAAADo9zwBAOhSpv7/uQMAAADoyDwBAJCJRCQISIlc +JBBIiUwkGEiJfCQg6E81AQCLRCQISItcJBBIi0wkGEiLfCQg6df3///MzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzEk7ZhAPhuwDAABIg+xISIlsJEBIjWwkQEmLTjBMifKQSIXSdA1IOZHAAAAAD4RkAwAA +McCEwA+ESQMAAIuK8AAAAIP5Bw+FCwEAAEiDuhABAAACdRFIgboYAQAAABAAAA+CvwAAAIC6tQAA +AAB0YUiLihgBAABIiUwkGEiNBd9xAgC7MQAAAOhUnv7/RA8RfCQoSMdEJDgAAAAASI0Nv3ECAEiJ +TCQoSMdEJDAxAAAASItMJBhIiUwkOEiNBX/3AQBIjVwkKOi1mvz/6JDC/v9Ii4IYAQAASIlEJCAP +H0AA6Hvi/v9IjQUSTQIAuxkAAADoiuv+/0iLRCQgDx9EAADo2+n+/+jW5P7/6NHi/v9IjQWFKAIA +uwUAAAAPH0QAAOjbyf7/kEiNBTRxAgC7MQAAAOipnf7/SIsNcqQHAEiLHXOkBwBIhcl0BEiLSQhI +icjoCsL+/4P5CA+FqAAAAEiLihABAABIg/kBdGpIg/kCdTOQSI0FpDgCALsQAAAAkOhbnf7/SIsN +RKQHAEiLHUWkBwBIhcl0BEiLSQhIiciQ6LvB/v+QSI0FQkACALsUAAAA6Cmd/v9Iiw3iowcASIsd +46MHAEiFyXQESItJCEiJyOiKwf7/kEiNBfpDAgC7FgAAAOj4nP7/SIsNoaMHAEiLHaKjBwBIhcl0 +BEiLSQhIicjoWcH+/4P5Cw+FNQEAAEiLihABAABmDx+EAAAAAABIhckPhf4AAABIgboYAQAAABAA +AA+CvAAAAIC6tQAAAAB0Z0iLihgBAABIiUwkEEiNBQtwAgC7MQAAAA8fRAAA6Huc/v9EDxF8JChI +x0QkOAAAAABIjQ3mbwIASIlMJChIx0QkMDEAAABIi0wkEEiJTCQ4SI0FpvUBAEiNXCQokOjbmPz/ +6LbA/v9Ii4IYAQAASIlEJCDopeD+/0iNBTxLAgC7GQAAAOi06f7/SItEJCDoCuj+/+gF4/7/Dx9E +AADo++D+/0iNBa8mAgC7BQAAAOgKyP7/kEiNBWNvAgC7MQAAAOjYm/7/SIsNoaIHAEiLHaKiBwBI +hcl0BEiLSQhIicjoOcD+/0iD+QEPhPj+//9Ig/kCD4Tu/v//Dx9EAADp9f7//4P5QXIRSI0F3EYC +ALsXAAAA6KXH/v9IjQxJSI0VGq0HAEiLXMoQSItEygjoi5j8/0iJw0iNBQHJAQCQ6Nu//v9IjQUj +aAIAuyoAAADoasf+/4O5CAEAAAB0BzHA6Y78//+DufAAAAAAdfCDufQAAAAAdedIg7kAAQAAAHXd +g7kMAQAAAHXUkIuKkAAAAA+68QyD+QJ0CjHADx8A6U/8//9Ig3pwAHXvuAEAAADpPvz//+glMQEA +Dx9EAADp+/v//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEiD7BhIiWwkEEiNbCQQiUQkIOgp +BwAAi0wkIInIDx8ASIP4QQ+D3QAAAEiNFe+3CgBIjQSCMdKHEIkMJA8fQADoe0sBAEUPV/9kTIs0 +Jfj////oCVIBAEUPV/9kTIs0Jfj////o91EBAEUPV/9kTIs0Jfj////o5VEBAEUPV/9kTIs0Jfj/ +//+LRCQgMdvoDZf+/4tEJCCJBCToIUsBAEUPV/9kTIs0Jfj////or1EBAEUPV/9kTIs0Jfj///9m +kOibUQEARQ9X/2RMizQl+P///+iJUQEARQ9X/2RMizQl+P///8cEJAIAAADoMEkBAEUPV/9kTIs0 +Jfj///9Ii2wkEEiDxBjDuUEAAADoTzcBAJDMzMzMzMzMzMzMzMzMzEk7ZhAPhvcAAABIg+wgSIls +JBhIjWwkGA8fhAAAAAAAg/gbD4TNAAAASIlcJDCD+EFyBDHJ6xCJwUiNFYC7CgBIjQzKSIsJSIlM +JBCJRCQo6MsFAACLRCQoSItcJBBmkOgblv7/gD1frAoAAHUJgD1YrAoAAHQqSItEJBAPH0AASIXA +dRxIi0QkMEiLAEhjQAhIhcB0C0iLbCQYSIPEIJDDi0QkKIkEJOjzSQEARQ9X/2RMizQl+P///8cE +JOgDAADoWkkBAEUPV/9kTIs0Jfj///9IjQWudwIAhABIix2ldwIAi0QkKOiUlf7/SItsJBhIg8Qg +w0iLbCQYSIPEIJDDiUQkCEiJXCQQ6NEuAQCLRCQISItcJBDp4/7//8zMzEiD7BBIiWwkCEiNbCQI +uAYAAADoqP3//0iLbCQISIPEEMPMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdmBI +g+wYSIlsJBBIjWwkEIlEJCDoo9z+/0iNBQklAgC7BwAAAOiy5f7/i0QkIInA6Kfi/v9IjQWLYgIA +uykAAADoluX+/+jx3P7/SI0FS1UCALsgAAAADx9EAADo+8P+/5CJRCQI6BEuAQCLRCQI64vMzMzM +zMzMzMzMzEk7ZhB2YEiD7BhIiWwkEEiNbCQQiUQkIOgj3P7/SI0FiSQCALsHAAAA6DLl/v+LRCQg +icDoJ+L+/0iNBVNjAgC7KgAAAOgW5f7/6HHc/v9IjQXSbwIAuzkAAAAPH0QAAOh7w/7/kIlEJAjo +kS0BAItEJAjri8zMzMzMzMzMzMzMSTtmEHZgSIPsGEiJbCQQSI1sJBCJRCQg6KPb/v9IjQUJJAIA +uwcAAADosuT+/4tEJCCJwOin4f7/SI0FBj0CALsWAAAA6Jbk/v/o8dv+/0iNBd1JAgC7GwAAAA8f +RAAA6PvC/v+QiUQkCOgRLQEAi0QkCOuLzMzMzMzMzMzMzMxIg+wwSIlsJChIjWwkKEiJRCQ4SIlc +JCCAPc2pCgAAkHVfgD27qQoAAHVWSIsF7pwHAEiLDe+cBwBIxwQkAgAAAEiJRCQIiUwkEOhxRgEA +RQ9X/2RMizQl+P///8cEJAIAAADouEUBAEUPV/9kTIs0Jfj///+4ewAAAEjHAAIAAADo+hz//0iL +RCQ46DAHAACEwHURSItEJDhIi1wkIGaQ6Hv8///oViD//0iLbCQoSIPEMMPMzMzMzMzMzMzMzMxI +g+xASIlsJDhIjWwkOIP4QQ+DngEAAIlEJEiJwkiNNfy3CgBIjTTWSIs2SI08UkyNBWqnBwBBizz4 +TI0FH7MKAEmNFJCLEoXSD4TxAAAAgD3aqAoAAA+E5AAAAA8fQABIhfYPhMcAAABIiXQkIEiJTCRY +SIlcJFBEDxF8JChIiVwkKEiJTCQwSGNTCEiF0nQGD7rnA3IRg/gNdAwxwEiLbCQ4SIPEQMNIjUQk +KOhO6f//SIXAdC1Ii0gwDx9EAABIhcl0H0iDucAAAAAAdBWAuRgBAAAAdQwxwEiLbCQ4SIPEQMNI +i0wkIEiD+QF0MkiJDCSLRCRIiUQkCEiLRCRQSIlEJBBIi0QkWEiJRCQY6AxIAQBFD1f/ZEyLNCX4 +////uAEAAABIi2wkOEiDxEDDMcBIi2wkOEiDxEDDDx9AAEiD/gF0C0iF9nU3D7rnCHMQuAEAAABI +i2wkOEiDxECQw0iF9nUcMdvoc5H+/4tEJEjoyvn//zHASItsJDhIg8RAw0iJNCSJRCQISIlcJBBI +iUwkGOiHRwEARQ9X/2RMizQl+P///7gBAAAASItsJDhIg8RAwzHASItsJDhIg8RAw8zMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxIg+woSIlsJCBIjWwkIITAdEGQxwQkAgAAAEiNBceIBwBI +iUQkCEjHRCQQAAAAAMdEJBgIAAAA6GRGAQBFD1f/ZEyLNCX4////SItsJCBIg8Qow5DHBCQCAAAA +SI0FjogHAEiJRCQISMdEJBAAAAAAx0QkGAgAAADoI0YBAEUPV/9kTIs0Jfj///9Ii2wkIEiDxCjD +zMzMzMzMzMzMzMzMSIPsMEiJbCQoSI1sJChIx0QkIAAAAACJwkiNWv9Iwfs/SMHrO0iNFBNIjVL/ +SMH6BUiD+gJzTY1I/7gBAAAA0+AJRJQgkMcEJAEAAABIjUQkIEiJRCQISMdEJBAAAAAAx0QkGAgA +AADok0UBAEUPV/9kTIs0Jfj///9Ii2wkKEiDxDDDSInQuQIAAADojzABAJDMzMzMzMzMzMzMzMzM +zEk7ZhB2H0iD7AhIiSwkSI0sJOgpAAAA6IQBAABIiywkSIPECMPo9igBAOvUzMzMzMzMzMzMzMzM +zMzMzMzMzMxJO2YQD4Y6AQAASIPsOEiJbCQwSI1sJDBMiXQkEEjHRCQYAAAAAEiNRCQgRA8ROEjH +BCQAAAAASI1EJBhIiUQkCOhaSQEARQ9X/2RMizQl+P///4tEJCAPuuABDx9EAAAPgrAAAACAPV+l +CgAAD4SjAAAASItEJBBIi0gwSYtWMEiLUlBIixpIi1IISIlZWEiJUWBJi1YwSItSUEiLUhBIiVFo +SYtWMEiLUlBIi1IYSIlRcEmLVjBIi1JQSIuSgAAAAEiJUXhJi04wSItJUEiLVCQYSIkRSYtOMEiL +SVBIi1wkKEgB00iJWQhJi04wSItJUEiBwqADAABIiVEQSYtOMEiLSVBIiVEYSItAMMaAFgEAAADr +JEiLTCQQSItRMEiLQlCEAOj2AQAASItMJBBIi0kwxoEWAQAAAUiLbCQwSIPEOMPolycBAOmy/v// +zMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+G/AAAAEiD7DBIiWwkKEiNbCQoSYtWMEiLkoAAAABIiVQk +IDHA6wNI/8BIg/hBfXOJwmYPH4QAAAAAAEiD+kEPg6oAAABIjRRSSI0di6IHAIsU0w+64gdyGoA9 +BqQKAAB1xYA9/6MKAAB1vPfCBgAAAHS0SI1Q/0jB+j9Iweo7SI0UAkiNUv9IwfoFSIP6AnNRjUj/ +vgEAAADT5vfWIXSUIOuEkMcEJAIAAABIjUQkIEiJRCQISMdEJBAAAAAAx0QkGAgAAADo8EIBAEUP +V/9kTIs0Jfj///9Ii2wkKEiDxDDDSInQuQIAAADo7C0BAEiJ0LlBAAAADx9AAOjbLQEAkOh1JgEA +6fD+///MzMzMzMzMzMzMzMzMzMzMSIPsMEiJbCQoSI1sJChJi0YwgLgWAQAAAHUxSItIUJBIi1Bg +SItYWEiJGUiJUQhIi1BoSIlREEiLUHBIiVEYSItAeEiJgYAAAADrPkjHRCQQAAAAAEiNRCQYRA8R +OMdEJBgCAAAASI1EJBBIiQQkSMdEJAgAAAAA6KNGAQBFD1f/ZEyLNCX4////SItsJChIg8Qww8zM +zMzMzMzMzMzMzEiD7DBIiWwkKEiNbCQoSMdEJBAAAAAASI1MJBhEDxE5SItICEgrCEiJTCQgSIsA +SIlEJBBIjUQkEEiJBCRIx0QkCAAAAADoNUYBAEUPV/9kTIs0Jfj///9Ii2wkKEiDxDDDzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GYAEAAEiD7BhIiWwkEEiNbCQQg/hgc0a6AQAA +AEiNNdOlCgDwD8EWicLB6AVIg/gDD4MjAQAASI0NkaUKAEiNDIGLCQ+j0XIjuf/////wD8EOMcBI +i2wkEEiDxBjDMcBIi2wkEEiDxBjDidhIjQ1QpQoAizyBD6PXZpByHEiNDIFBifgPq9eJw0SJwPAP +sTkPlMGEyXTT6xi5//////APwQ64AQAAAEiLbCQQSIPEGMOLDTylCgAPH0AAg/kBd0GFyXUdMcBI +jQ0mpQoAugIAAADwD7ERD5TBZpCEyXTS61C4AQAAAEiNDQalCgAx0vAPsREPlMGEyXUkugIAAADr +sIP5AnQrg/kDdT7o6x7//7oCAAAASI0126QKAOuTSI0FlqQKAOhxkvz/SI01xqQKALn/////8A/B +DrgBAAAASItsJBBIg8QYw0iNBblAAgC7GwAAAOjBuf7/uQMAAADoNysBAJCJRCQI6M0jAQCLRCQI +6YT+///MzMzMSTtmEA+GHwEAAEiD7EBIiWwkOEiNbCQ4SDnZc0BIixBIiVQkKEiJxkiJ2Ej34g+A +4AAAAEi6AAAAAAAAAQBmkEg50A+HywAAAEiF2w+MwgAAAEiLVCQoSA+vyusNSIsISA+vy0iJxkiJ +yEiJfCRgSIlMJCBIg34IAHU9SIlEJBgx2zHJ6Fek/P9Ii0wkIEiLXCQYSDnZc1lIiUQkMEgpy0iN +FAhIidDo1DABAEiLRCQwSItMJCDrOUiJ87kBAAAA6Buk/P9Ii0wkIEiFyXYigD3qoQoAAHQZSIlE +JDBIi1wkYOi55fz/SItEJDBIi0wkIEiJRCQwSItcJGAPH0QAAOg7MwEASItEJDBIi2wkOEiDxEDD +kEiNBQS6AQBIjR09agMA6Niw/v+QSIlEJAhIiVwkEEiJTCQYSIl8JCAPHwDoeyIBAEiLRCQISItc +JBBIi0wkGEiLfCQg6aL+///MzEk7ZhAPhqcAAABIg+wgSIlsJBhIjWwkGEiLEEiJxkiJ0EiJx0j3 +4XB8SLoAAAAAAAABAEg50HckSIXbfB8PHwBIOdl8F0iJ87kBAAAA6C6j/P9Ii2wkGEiDxCDDSIn4 +SPfjcRSQSI0FVLkBAEiNHY1pAwDoKLD+/0i5AAAAAAAAAQBIOch33UiF23zYkEiNBSy5AQBIjR11 +aQMADx9EAADo+6/+/0i6AAAAAAAAAQDrq0iJRCQISIlcJBBIiUwkGOibIQEASItEJAhIi1wkEEiL +TCQY6Sf////MzMzMzMzMSTtmEA+GfAYAAEiD7GBIiWwkWEiNbCRYSIlcJHAPHwBIOf4PjEQGAABI +ixBIhdJ0JkyNBD9MOcZ/GGYPH0QAAEiB/wAEAAAPjfoFAADp4AUAAEiJ9+sXSI0FEqAKAEiJy0iJ +8UiLbCRYSIPEYMNIg/oBD4XdAAAADx+EAAAAAABIgf4AgAAAD4OLAAAASIH++AMAAHdCSI1WB0jB +6gNmkEiB+oEAAAAPg3gFAABMjQXMgQcAQg+2FAIPH4AAAAAASIP6RA+DSQUAAEyNBU+CBwBBD7cU +UOthSI2Wf/z//0jB6gdIgfr5AAAAD4MWBQAATI0FyYIHAEIPthQCDx9AAEiD+kQPg+8EAABMjQUP +ggcAQQ+3FFDrIUiNlgAgAACQSDnWdgVIifLrD5BIjZb/HwAASIHiAOD//0m4AAAAAAAAAQBMOcZB +D5fBSInOSYnS6RQDAABIg/oID4XwAAAASInySMHmA0iB/gCAAAAPg4UAAABIgf74AwAAdzxIg8YH +SMHuA0iB/oEAAAAPg1sEAABMjQXogAcAQg+2NAYPHwBIg/5ED4M1BAAATI0Fb4EHAEEPtzRw61xI +gcZ//P//SMHuB0iB/vkAAAAPgwIEAABMjQXpgQcAQg+2NAYPH0AASIP+RA+D2wMAAEyNBS+BBwBB +D7c0cOscTI2GACAAAJBMOcZ3D5BIgcb/HwAASIHmAOD//0m4AAAAAAAgAABMOcJBD5fBSYnISMHh +A0jB5wNIifJIweoDSYnSSInyTInGSbgAAAAAAAABAOkaAgAATI1C/0yFwg+FEwEAAEgPvNJJichI +idFNicFJ0+BI0+dJifJI0+ZmDx+EAAAAAABmkEiB/gCAAAAPg4sAAABIgf74AwAAd0JIg8YHSMHu +A2aQSIH+gQAAAA+DCQMAAEyNHcx/BwBCD7Y0Hg8fgAAAAABIg/5ED4PfAgAATI0dT4AHAEEPtzRz +61xIgcZ//P//SMHuB0iB/vkAAAAPg6wCAABMjR3JgAcAQg+2NB4PH0AASIP+RA+DhQIAAEyNHQ+A +BwBBD7c0c+scTI2eACAAAJBMOd53D5BIgcb/HwAASIHmAOD//0m7AAAAAAAAAQBJ0+tNOdpBD5fC +SYnzSNPuTInaTInBSbgAAAAAAAABAEmJ9EyJzkWJ0U2J4un6AAAASIlUJEBJicBIifBI9+JIi1Qk +QEmJyUgPr8pID6/6SD0AgAAAD4ODAAAADx+AAAAAAEg9+AMAAHc6SIPAB0jB6ANIPYEAAAAPg8kB +AABMjRW9fgcAQg+2BBBIg/hED4OnAQAATI0VR38HAEEPtwRCZpDrU0gFf/z//0jB6AdIPfkAAAAP +g3cBAABMjRXBfwcAQg+2BBBIg/hED4NXAQAATI0VC38HAEEPtwRC6xlMjZAAIAAATDnQdw2QSAX/ +HwAASCUA4P//SYnSSYnDMdJJ9/JIiUQkQEiJ8En34g+QwkyJwEyJzkm4AAAAAAAAAQBBidFMidpM +i1QkQEWEyQ+F2wAAAEw5wg+H0gAAAEiJXCRQSIlMJChIiXQkOEyJVCQgSIN4CAB1P0iJVCQwSIl8 +JBhIidAx2zHJ6PSd/P9IiUQkSEiLXCQwSItUJBhIKdNIAcJIidDodyoBAEiLTCQoSItEJEjrUEiJ +RCRoSInDuQEAAABIidDotp38/0iLTCQoSIXJdjGAPYWbCgAAdChIiUQkSEiLXCRQSIt8JGhMiwdM +KcFIA08I6EXf/P9Ii0QkSEiLTCQoSIlEJEhIi1wkUOjMLAEASItEJEhIi1wkOEiLTCQgSItsJFhI +g8Rgw0iNBYyzAQBIjR3lYwMADx9EAADoW6r+/7lEAAAA6HEjAQC5+QAAAOiHIwEAuUQAAABmkOhb +IwEAuYEAAADocSMBAEiJ8LlEAAAA6EQjAQBIifC5+QAAAOhXIwEASInwuUQAAADoKiMBAEiJ8LmB +AAAAZpDoOyMBAEiJ8LlEAAAA6A4jAQBIifC5+QAAAOghIwEASInwuUQAAADo9CIBAEiJ8LmBAAAA +6AcjAQBIidC5RAAAAOjaIgEASInQufkAAADo7SIBAEiJ0LlEAAAADx9EAADouyIBAEiJ0LmBAAAA +6M4iAQBIifdMicbpMfr//0mJ+EjB/wJMAcdIhf9+Fkg5/n/sSIX/fgxJifhIifdMicaQ6wNIifdJ +ifBIif7rxUiNBWyyAQBIjR3FYgMADx9EAADoO6n+/5BIiUQkCEiJXCQQSIlMJBhIiXwkIEiJdCQo +kOjbGgEASItEJAhIi1wkEEiLTCQYSIt8JCBIi3QkKGaQ6Tv5///MzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMwxwOskSInBSMHgBkiNFa6gCgBIx0QCCAAAAABIx0QCEAAAAABIjUEBSIP4BHzWMcDr +I0iJwUjB4ARIjRVoqAoASMcEAgAAAABIx0QCCAAAAABIjUEBSIP4I3zXw8zMzMxJO2YQD4Z8AQAA +SIPsOEiJbCQwSI1sJDAPttAPH0QAAEiD+gQPg0wBAABIweIGSI01K6AKAEiNPBZIjX8ISItUFghI +hdJ1V4hEJEBIiXwkIEiNBaksCQC7BAAAALkBAAAA6Prn/f9IhcAPhPYAAABmg3hgAA+F1QAAAEiL +UChmkEiF0g+FtQAAAA+2TCRAugAIAABI0+JIiVBoMcnrdkiLQihIicFIhcB0PUiLMEiJcigPt3Jg +/8ZmiXJgSItyKJBIhfZ1FUiJTCQoSIn4SInT6Mv4/f9Ii0wkKEiJyEiLbCQwSIPEOMNIjQUaLgIA +uxcAAADoKK/+/0iLUBhIAcpIidZIi3goSIk6SIlwKEiLUGhIAdFIgfkAgAAActtIiUQkGEiJw0iL +RCQg6PH5/f9Ii3wkIEiLVCQY6WD///9IjQW0IQIAuxIAAADo0a7+/0iNBfgaAgC7DgAAAA8fRAAA +6Luu/v9IjQXhGQIAuw0AAADoqq7+/0iJ0LkEAAAAZpDoGyABAJCIRCQI6LEYAQAPtkQkCOln/v// +zMzMzMzMzEk7ZhAPhmkBAABIg+wwSIlsJChIjWwkKEiLFdksCgCEApCQSL4AAAAAAIAAAEgBxkjB +7hpmDx+EAAAAAAAPHwBIgf4AAEAAD4MYAQAASIsU8kiJxkjB6A1IJf8fAABIi7zCAAAgAIQCildj +gPoCD4XbAAAASIl8JBhIi1coDx8ASIXSdUAPtsNIg/gED4OzAAAAiFwkQEiJdCQgSMHgBkiNDRqe +CgBIjQQBSI1ACEiJ++jK+P3/D7ZcJEBIi3QkIEiLfCQYSInyTItHKEyJBkiJVygPt1dgjXL/Zol3 +YIM935QKAAB1SGaD+gF1Qg+2w0iD+ARzQ0jB4AZIjQ2/nQoASI0EAUiNQAhIifvo7/b9/0iLXCQY +SMdDKAAAAABIjQU7KgkAuQEAAADo0fL9/0iLbCQoSIPEMMO5BAAAAGaQ6LseAQC5BAAAAOixHgEA +SI0FEkECALshAAAADx9EAADoG63+/0iJ8LkAAEAA6K4eAQCQSIlEJAiIXCQQDx9AAOgbFwEASItE +JAgPtlwkEOls/v//zMzMzMzMzMzMzMzMSTtmGA+G7AAAAEiD7DhIiWwkMEiNbCQwD7bTDx9EAABI +g/oED4O+AAAASIlEJEBIiVQkIIhcJEhIweIGSI0d3ZwKAEiNBBNIiUQkKOhvgvz/McAxyes3D7ZE +JEgPH0AA6Fv8//9IicFIi1QkGEiJEEiJyA+2TCRIugAIAABI0+JIi1wkEEgB2kiJwUiJ0EiJRCQQ +SIlMJBgPH4QAAAAAAEg9AEAAAHKvkJBIi0QkKOjsg/z/SItMJECEAUiLVCQgSMHiBEiLXCQYSImc +EWgEAABIi1wkEEiJnBFwBAAASItsJDBIg8Q4w0iJ0LkEAAAA6GsdAQCQSIlEJAiIXCQQkOg7PQEA +SItEJAgPtlwkEOns/v//zMzMzMzMzMzMzMzMSTtmGA+GAQEAAEiD7DhIiWwkMEiNbCQwhAAPttMP +HwBIg/oED4PTAAAASIlEJECIXCRISInRSMHiBEiJVCQgSIucEGgEAABIiVwkEEiLtBBwBAAASIl0 +JBhIweEGSI09nJsKAEgB+UiJTCQoSInI6CyB/P9Ii0QkGEiLTCQQZpDrL0iJyEiLCUiJTCQQD7Zc +JEjoqfz//w+2TCRIugAIAABI0+JIi0QkGEgp0EiLTCQQSIlEJBhIPQBAAAB3xEiJTCQQkJBIi0Qk +KOixgvz/SItMJCBIi1QkEEiLXCRASImUC2gEAABIi1QkGEiJlAtwBAAASItsJDBIg8Q4w0iJ0LkE +AAAA6DYcAQCQSIlEJAiIXCQQ6Ac8AQBIi0QkCA+2XCQQ6dj+///MzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMxJO2YYD4bKAAAASIPsOEiJbCQwSI1sJDBIiUQkQDHJkOswSMeEEWgEAAAAAAAASMeEEXAE +AAAAAAAAkJBIi0QkKOj6gfz/D7ZMJBf/wUiLRCRAgPkEc0WITCQXD7bRSIlUJCBIweIGSI01UpoK +AEiNBBZIiUQkKJDo43/8/0iLTCRAhAFIi1QkIEjB4gRIiVQkIEiLnBFoBAAA6ytIi2wkMEiDxDjD +SIsLSIlMJBgPtlwkF+hI+///SItMJEBIi1QkIEiLXCQYSInYSIXbddfpTv///0iJRCQI6AI7AQBI +i0QkCOkY////zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmGA+GtQIAAEiD7HBIiWwkaEiNbCRo +SYtWMEyJ9pBIOTIPhYQCAACNcP+Fxg+FaAIAAIM9TZQKAAB1JYlEJHg9AIAAAHMJicEx2+lPAQAA +icFIicpIwekNSInLMfaQ615Iiw2/kQoAicJIjRQRSI1S/0j32UghyonQSIlEJFhIjR2xpwoA6BQM +/f9IhcB0E0iLTCRYSI0cCEiLbCRoSIPEcMNIjQXwLAIAuxoAAADoy6j+/0jR6Uj/xg8fRAAASIP5 +AXfvSIlcJFhIiVQkUEiJdCQwkJBIjQXCoAoAZpDoe378/0iLRCQwSIP4Iw+DmgAAAEjB4ARIjQ2p +oAoASIscAUiF23UEMcDrEkiJXCRISAHI6Ofx/f9Ii0QkSEiJRCQokJBIjQV0oAoA6A+A/P9Ii0wk +KEiFyXUnSI0FHiUJAEiLXCRYuQEAAADob+D9/0iFwHQnSItMJFBIiUhoSInBSItJGItUJHiJ0kiN +HBFIichIi2wkaEiDxHDDSI0FKRMCALsNAAAA6PKn/v+5IwAAAOhoGQEA/8PR6A8fQAA9AAgAAHfx +SIuy0AAAAEiF9nQKSIO6AAEAAAB0Uw+2w0iD+AQPg8gAAACIXCQfSMHgBkiNDeeXCgBIAchIiUQk +YJDoeX38/w+2RCQf6G/3//9IiUQkIJBIi0QkYA8fQADoO3/8/4tMJHhIi3QkIOtsSItGQIQAD7bT +SIP6BHNmSMHiBEiLtBBoBAAASIn3SIX2dTFIiUQkOEiJVCRADx8A6Fv6//9Ii0wkQEiLVCQ4SIu0 +CmgEAABIidBIicpIifeLTCR4SIn+SIs/SIm8EGgEAACJz0gpvBBwBAAASInxkOnh/v//SInQuQQA +AADobhgBALkEAAAA6GQYAQBIjQXmLQIAuxsAAADo06b+/0iNBWE8AgC7IQAAAOjCpv7/kIlEJAjo +GDgBAItEJAjpL/3//8zMzMzMzMzMzMzMzMzMzEk7ZhgPht4CAABIg8SASIlsJHhIjWwkeEiJwkmJ +2Ugp00yNU/9MhdMPhagCAABMjRQTTTnRD4eKAgAAgz1mkQoAAA+FjwAAAEiB+wCAAABzFEyJdCRw +SInYMckPH0QAAOlcAQAASIs1rCQKAIQGkJBIvwAAAAAAgAAASAHXSMHvGkiB/wAAQAAPgyEBAABI +izT+SMHqDUiB4v8fAABIi5zWAAAgAIQGilNjgPoCD4WyAAAAgz0qjQoAAHQISItLIDHA60NIjQW1 +IgkAuQEAAADoS+v9/+sfkDHJvzIAAAC+/////0UxwOi0Fvz/SItsJHhIg+yAw0iLbCR4SIPsgJDD +SNHpSP/ASIP5AXf0SIlcJChIiUQkOJCQSI0FoJ0KAOhbe/z/SItEJDhIg/gjcyhIweAESI0NjZ0K +AEgByEiLXCQo6Fjw/f+QkEiNBW+dCgDoCn38/+ueuSMAAAAPHwDouxYBAEiJRCRgSItDGEiJRCQw +6Ii9/v9Ii0QkMA8fAOj7xP7/6La//v9Ii0QkYOgMxv7/6Oe//v/o4r3+/0iNBTcRAgC7DgAAAOjx +pP7/SIn4uQAAQADohBYBAP/BSNHrSIH7AAgAAHfySIlUJFhIi3QkcEiLdjBIi77QAAAAkEiF/3QK +SIO+AAEAAAB0Sg+2wUiD+AQPg74AAACITCQnSMHgBkiNDdWUCgBIAchIiUQkaJDoZ3r8/0iLRCRY +D7ZcJCfo+PX//5CQSItEJGjoLHz8/+m9/v//SIt3QIQGD7b5SIP/BHNlSMHnBEyLhD5wBAAASYH4 +AIAAAHItSIlEJFBIiXQkQEiJfCRISInwicvoavj//0iLRCRQSItUJFhIi3QkQEiLfCRISInRSIuc +PmgEAABIiRpIiYw+aAQAAEgBhD5wBAAA6Un+//9Iifi5BAAAAOhmFQEAuQQAAACQ6FsVAQBIjQUe +EAIAuw4AAADoyqP+/0iNBbkfAgC7FgAAAOi5o/7/kEiJRCQISIlcJBDoCTUBAEiLRCQISItcJBDp ++vz//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GxwEAAEiD7GBIiWwkWEiNbCRYSIm8 +JIAAAABIibQkiAAAAEiJdCRQSIsRTItBCEyLSRBMYxNMiVQkOEmJw0iJRCQwSIuJGAEAAEUx5OsE +TY1lCE051HMXTIlkJCBNieVJwewDTANjCEUPtiQk6xRIi2wkWEiDxGDDTItsJCBMi1QkOEWE5HTH +RQ+8/EWNVCT/RSHURIhkJBdPjRQvT4080+skSItEJDBIi3QkUOvKTQHNT4ks0+vBSItEJDBIi3Qk +UEyLbCQgT4ss00yJbCQYSIX/dCZNjWX/ZpBJgfz/DwAAcxGDPbCNCgAAdUNED7ZkJBfrBkQPtmQk +F0k51Q+Cdf///005xQ+DbP///0g5wXadSIt0JBhMAc5MiejwSQ+xN0APlMZAhPZ0jelz////TIl8 +JEBJi04wxoEpAQAAAkiJ+EiJ8+hQOgAASIlEJEhIiVwkKOiBuv7/SI0FNDACALseAAAA6JDD/v9I +i0QkSEiLXCQo6IHD/v9IjQWC/wEAuwQAAADocMP+/0iLRCRA6ObC/v9IjQUE/wEAuwIAAADoVcP+ +/0iLRCQY6KvB/v/oprz+/+ihuv7/SI0FmC8CALseAAAA6LCh/v+QSIlEJAhIiVwkEEiJTCQYSIl8 +JCBIiXQkKOixCwEASItEJAhIi1wkEEiLTCQYSIt8JCBIi3QkKOnz/f//zMzMzMzMzMzMzMzMzMzM +zMzMzEyNZCTYTTtmEA+GAQMAAEiB7KgAAABIiawkoAAAAEiNrCSgAAAASIN4GAAPhEYBAABIixBI +i3AIgHooFQ8fQAAPhBwBAABIiZwkuAAAAEiJhCSwAAAASIlUJGBIiXQkWIQDSI1TGLkBAAAASInT +6A0YAABIiXQkSEyJRCQ4iUQkaEiJXCRwiUwkeEiJvCSAAAAAi1QkaIXSfjxIY9JIweIDTIuEJLAA +AABJi0A4SCnQSI1cJGhIi4wkuAAAAEiLfCRgSIt0JFjoFP3//0iLdCRITItEJDhIi5QksAAAAEyL +SkBMi1I4TSnRSYP5EHUokE2LCkiLjCS4AAAADx9EAABMOQl3Gkw5SQh2FEyLWRBNAdlNiQrrCEiL +jCS4AAAAg3wkeAB+LEiLQkBIjVwkeDH/MfboqPz//0iLjCS4AAAASIuUJLAAAABIi3QkSEyLRCQ4 +SIN6OAB0M02FwH4uMcDrRrgBAAAASIusJKAAAABIgcSoAAAAw7gBAAAASIusJKAAAABIgcSoAAAA +w7gBAAAASIusJKAAAABIgcSoAAAAw0iDxhhIifhIi14Qi34ERItOCESLFkSJlCSIAAAAibwkjAAA +AESJjCSQAAAASImcJJgAAACLvCSIAAAATItKOIX/fARMi0pASGP/TAHPSDl6KA+HrgAAAESLjCSQ +AAAARYXJfQ5FicpB99lNY8lFhdLrA01jyUiJRCRASIl0JFB8BUUx0utCSIl8JDBMiUwkKEyJyOhn +6vz/SItYGEiLjCS4AAAASIuUJLAAAABIi3QkUEiLfCQwTItEJDhMi0wkKEmJwkiLRCRARTHb61JN +hdJ0M0iNBcYbCQBMidO5AgAAAOhZ5P3/SItEJEBIi4wkuAAAAEiLlCSwAAAASIt0JFBMi0QkOEiN +eAFJOfgPj+7+///p1P7//02NXQhMifmQTTnLc6lNidxJwesGRg+2HBtNieVJwewDSYPkB0mJz0yJ +4UHS60H2wwF0zU2NXD0ATYsjTTknd8BNOWcIdrpIidFJi1cQTAHiSYkTSInK66hIiUQkCEiJXCQQ +6GEIAQBIi0QkCEiLXCQQ6dL8///MzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4YUAQAASIPsIEiJbCQY +SI1sJBhIi1AoDx9AAEg5E3cRSDlTCHYLSItzEEgB8kiJUChIi1Ao6wRIi1IoSIXSD4S8AAAAkEiL +chhIOTN3EUg5cwh2C0iLexBIAf5IiXIYkEiLcghIOTN3EUg5cwh2C0iLexBIAf5IiXIIkEiLciBI +OTN3EUg5cwh2C0iLexBIAf5IiXIgkEiLciiQSDkzdxFIOXMIdgtIi3sQSAH+SIlyKJBIi3I4Dx9E +AABIOTN3EUg5cwh2C0iLexBIAf5IiXI4kEiLcjAPH0QAAEg5Mw+HU////0g5cwgPhkn///9Ii3sQ +SAH+SIlyMGaQ6Tf///9IidlIjR2xTQIA6Ox4AABIi2wkGEiDxCDDSIlEJAhIiVwkEOgTBwEASItE +JAhIi1wkEOnE/v//zMzMzEk7ZhAPhj0BAABIg+w4SIlsJDBIjWwkMEiLkEgBAACQSIXSdBNIiUQk +QEiJXCRISIlMJFAx9usXMcBIi2wkMEiDxDjDSIt6QEiLclBIifpIhdJ0L0iLelBIOf5050iJVCQg +hAdIjUdY6FNy/P9Ii0QkQEiLTCRQSItUJCBIi1wkSOvBkEiLkEgBAADrBEiLUkBIhdJ0IpBIi3IY +Dx9EAABIOTF36Eg5cQh24kiLeRBIAf5IiXIY69VIi5EYAQAAkEiF0nUEMcnrLEiLcQhIKd5Ii3kQ +SI0EPkgp8kiJVCQYSInzSInR6JUWAQBIi0QkQEiLTCQYSIlMJBhIi5BIAQAAMcDrC0iLWkBIi0JQ +SInaSIXSdCVIi1pQSDnYdOdIiVQkKIQDSI1DWOhyc/z/SItMJBhIi1QkKOvLSInISItsJDBIg8Q4 +w0iJRCQISIlcJBBIiUwkGOilBQEASItEJAhIi1wkEEiLTCQY6ZH+///MzMzMzMzMzMzMzMzMzMzM +zEyNpCTY/v//TTtmEA+GNwMAAEiB7KgBAABIiawkoAEAAEiNrCSgAQAASIN4cAAPhQMDAABIi0gI +SIsQDx9EAABIhdIPhN0CAABIiZwkuAEAAEiJhCSwAQAASIlMJHhIiVQkcEiLSDhIiUwkaEiJ2Ohs +8f//SIlEJGBIiVwkWEiNvCSAAAAASI1/4EiJbCTwSI1sJPDojQ4BAEiLbQBIi0wkcEiJjCSAAAAA +SItUJHhIiZQkiAAAAEiJ3kgp00iJnCSQAAAASItcJGhIiddIKdpIiVQkUEiLnCSwAQAAgLu4AAAA +AHQPTIuDSAEAAEUxyenVAQAASYn4SCnPTIuMJLgBAABJOflzEECKu7kAAABAhP8PhZ0BAACQSIu7 +SAEAAOsESIt/QEiF/3QwkEyLTxhMOYwkgAAAAHfoZg8fRAAATDmMJIgAAAB22EyLlCSQAAAATQHR +TIlPGOvHSInwSCnQTInDSCnTSInR6JEUAQCQkEiLhCSwAQAASItQUA8fAEg5lCSAAAAAdxlIOZQk +iAAAAHYPSIuMJJAAAABIAdFIiUhQkEiLSGhIOYwkgAAAAHcZSDmMJIgAAAB2D0iLlCSQAAAASAHR +SIlIaEiNnCSAAAAA6Gj7//+QkEiLvCSwAQAASItPIEg5jCSAAAAAdxlIOYwkiAAAAHYPSIuUJJAA +AABIAcpIiVcgSIuUJJgBAABIhdJ0E0yLpCSQAAAATAHiSImUJJgBAABIi1QkYEiJF0yLZCRYTIln +CEiBwqADAABIiVcQSItUJFBJKdRMiWc4SIuUJJAAAABIAZeAAAAASMcEJAAAAABIx8D/////SInD +Mckx9kUxwEG5////f0yNFXpJAgBMjZwkgAAAAOjNdgAASItEJHBIi1wkeA8fAOgb8v//SIusJKAB +AABIgcSoAQAAw0iNBeU+AgC7LwAAAOiamP7/TYtAQE2FwHQlTYtQUE2LWBhFD7dSGE0B2pBMOdF3 +4Uw513bcTTnRc9dNidHr0kyJjCSYAQAASInYSInTSI2MJIAAAADocfv//0iLVCRQSCnCSIt0JFhM +i0QkeOk1/v//SI0FUwMCALsNAAAA6CmY/v9IjQUoNQIAuycAAADoGJj+/5BIiUQkCEiJXCQQ6CgC +AQBIi0QkCEiLXCQQ6Zn8///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMTI2kJNj+//9NO2YQD4aW +CwAASIHsqAEAAEiJrCSgAQAASI2sJKABAABJi04wSItRGEiJ1kiBehAu+///Dx9AAA+EUAsAAEyJ +tCS4AAAASIuBwAAAAEiJhCTIAAAADx8ASDnCD4XsCQAAgLi3AAAAAA+FzwYAAEiLUQhIiZQk+AAA +AA8QQRAPEYQkAAEAAA8QQSAPEYQkEAEAAA8QQTAPEYQkIAEAAEmLTjBIx0EQAAAAAEmLTjBIx0Ew +AAAAAEmLTjBIx0EIAAAAAEmLTjBIx0EYAAAAAEiLSBBIiUwkcJBIgfne+v//dWJJi1Ywg7oIAQAA +AHUgg7rwAAAAAHUXSIO6AAEAAAB1DUiLktAAAACDegQBdDVIiwhIgcGgAwAASIlIEEiNSDhIiQwk +6O/+AABFD1f/ZEyLNCX4////SIuEJMgAAABIi0wkcEiLEEiF0g+E2QUAAEiLcDhIjX74SIl8JDBI +OfoPhtMBAABIiVQkaEiJdCRgSItICEiJTCRYSIuUJAABAABIiZQksAAAAEiLnCT4AAAASImcJKgA +AABIi7QkIAEAAEiJtCSgAAAASIt4QEiJvCSYAAAATItAYEyJhCSQAAAATItIUEyJjCTwAAAA6Geu +/v9IjQVoDwIAuxUAAADodrf+/0iLRCQw6My1/v9IjQUm9wEAuwgAAADoW7f+/0iLRCRo6LG1/v9I +jQXr8gEAuwIAAAAPH0QAAOg7t/7/SItEJFjokbX+/0iNBbkCAgC7DwAAAA8fRAAA6Bu3/v9Ii4Qk +sAAAAOhutf7/SI0FJ/MBALsEAAAAZpDo+7b+/0iLhCSoAAAA6E61/v9IjQX78gEAuwQAAABmkOjb +tv7/SIuEJKAAAADoLrX+/0iNBdwAAgC7DQAAAGaQ6Lu2/v9Ii4QkmAAAAOgOtf7/SI0Fx/IBALsE +AAAAZpDom7b+/0iLRCRg6PG0/v9IjQWe8gEAuwQAAAAPH0QAAOh7tv7/SIuEJJAAAADozrT+/0iN +Be/zAQC7BgAAAGaQ6Fu2/v9Ii4Qk8AAAAOjOtf7/SI0F/vEBALsCAAAAZpDoO7b+/+iWrf7/SIuE +JMgAAABIi0wkcEiLfCQwDx9AAEg5OA+H7QIAAEiB+d76//91e0iLjCS4AAAASItJMGaQSDkBD4S8 +AgAASIO50AAAAAB1E4O5CAEAAABmDx9EAAAPhIwCAACAuLMAAAAAdBnGgLMAAAAA6CUIAABIi4Qk +yAAAAEiLfCQwgLiyAAAAAHQP6Aoo//9Ii4QkyAAAAGaQ6Hsn//9Ii4QkyAAAAEiLSAhIKwhIiUwk +SEiLUEBIidDoOyUAAEiLTCRISInKSNHhSIXAdDdIiYwkiAAAAA8fAOj7MQAASGPISIHBoAMAAEiL +hCTIAAAASItQCEgrUDhIi7QkiAAAAOnJAQAASIuEJMgAAABIgXgQ7f7//0gPRMpIixUxXAcASDnK +cnhIOQ0dXAcAcm9IiUwkULsCAAAAuQgAAADo99j+/0iLhCTIAAAASItcJFDoBfj//0iLhCTIAAAA +uwgAAAC5AgAAAOjO2P7/SIuUJMgAAABIg8I4SIkUJOh5+wAARQ9X/2RMizQl+P///0iLrCSgAQAA +SIHEqAEAAMNIORWlWwcAD4bbAAAA6Gqr/v9IjQXSJwIAuyEAAADoebT+/0iLBYpbBwDobbH+/0iN +BZb7AQC7DAAAAJDoW7T+/+i2q/7/SIuEJMgAAABIiwhIiUwkaEiLQAhIiUQkYOgYq/7/SI0FUfwB +ALsMAAAA6Ce0/v9Ii0QkMGaQ6Huy/v9IjQXV8wEAuwgAAADoCrT+/0iLRCRoDx9EAADoW7L+/0iN +BZXvAQC7AgAAAOjqs/7/SItEJGAPH0QAAOg7sv7/SI0Fhe8BALsCAAAA6Mqz/v/oJav+/0iNBfj+ +AQC7DgAAAOg0kv7/6I+q/v9IjQX3JgIAuyEAAAAPHwDom7P+/0iLBaRaBwDoj7D+/0iNBbj6AQC7 +DAAAAA8fAOh7s/7/6Naq/v/pG////0jR50iJ/kiJ90gp1g8fRAAASDnOcupIi1QkSEiJ+eka/v// +SI0FaSgCALsiAAAAZpDou5H+/0iNBS4HAgC7EwAAAOiqkf7/i4iQAAAAiUwkLEiLkJgAAABIiZQk +gAAAAOjsqf7/SI0FGfsBALsMAAAA6Puy/v9Ii4QkyAAAAOhusv7/SI0FP/EBALsHAAAAZpDo27L+ +/0iLhCSAAAAA6M6w/v9IjQXY+wEAuw0AAABmkOi7sv7/i0QkLInA6BCx/v9IjQVW7gEAuwIAAAAP +H0AA6Juy/v/o9qn+/0iLhCTIAAAASIsASIlEJGjoYan+/0iNBT4hAgC7HwAAAOhwsv7/SItEJDDo +xrD+/0iNBSfuAQC7AwAAAOhVsv7/SItEJGjoq7D+/+imq/7/6KGp/v9IjQVGHAIAux0AAADosJD+ +/0iNBZQSAgC7GQAAAA8fQADom5D+/0iLUQhIiZQkMAEAAA8QQRAPEYQkOAEAAA8QQSAPEYQkSAEA +AA8QQTAPEYQkWAEAAEiLjCQwAQAASIlIcEiLjCQ4AQAASIlIeEiLSEBIicjohyEAAA8fgAAAAABI +hcAPhX0CAAC4CQAAAEiNDZ/yAQAx0kiJRCRASImMJMAAAABIiVQkOEiLnCTIAAAASItzOEiJdCRo +SIs7SIl8JGBMi0MITIlEJFhMi4wkOAEAAEyJjCSwAAAATIuUJDABAABMiZQkqAAAAEyLnCRYAQAA +TImcJKAAAABMi2NATImkJJgAAABMi2tgTImsJJAAAABMi3tQTIm8JPAAAACQ6Pun/v9IjQXnCAIA +uxUAAADoCrH+/0iLhCTAAAAASItcJEDo+LD+/0iNBXTsAQC7AQAAAOjnsP7/SItEJDhmkOg7r/7/ +SI0F+OwBALsEAAAA6Mqw/v9Ii0QkaA8fRAAA6Buv/v9IjQV18AEAuwgAAADoqrD+/0iLRCRgDx9E +AADo+67+/0iNBTXsAQC7AgAAAOiKsP7/SItEJFgPH0QAAOjbrv7/SI0FA/wBALsPAAAA6Gqw/v9I +i4QksAAAAGaQ6Luu/v9IjQV07AEAuwQAAADoSrD+/0iLhCSoAAAAZpDom67+/0iNBUjsAQC7BAAA +AOgqsP7/SIuEJKAAAABmkOh7rv7/SI0FKfoBALsNAAAA6Aqw/v9Ii4QkmAAAAGaQ6Fuu/v9IjQUU +7AEAuwQAAADo6q/+/0iLRCRoDx9EAADoO67+/0iNBejrAQC7BAAAAOjKr/7/SIuEJJAAAABmkOgb +rv7/SI0FPO0BALsGAAAA6Kqv/v9Ii4Qk8AAAAGaQ6Buv/v9IjQVL6wEAuwIAAADoiq/+/+jlpv7/ +SIuEJLgAAABIi0AwxoApAQAAAkiLhCQ4AQAASIucJDABAABIi4wkWAEAAEiLvCTIAAAAMfboa5EA +AEiNBaUfAgC7IAAAAOi6jf7/SImEJOgAAADozSUAAEiLjCTIAAAASItRQEiLtCToAAAASCsWSInB +SInY6WL9//9IiYwk4AAAAEiJdCR4SIsBSImEJNgAAABIi1FQSImUJNAAAADowqX+/0iNBfweAgC7 +IAAAAOjRrv7/SItEJHjoJ63+/0iNBQTrAQC7BAAAAOi2rv7/SIuEJOAAAADoKa7+/0iNBSrvAQC7 +CQAAAOiYrv7/SIuEJMgAAADoC67+/0iNBcDsAQC7BwAAAOh6rv7/SIuEJNgAAADo7a3+/0iNBe70 +AQC7DAAAAJDoW67+/0iLhCTQAAAA6M6t/v/oqaf+/+ikpf7/SIuEJLgAAABIi0AwhABIi0gISImM +JGgBAAAPEEAQDxGEJHABAAAPEEAgDxGEJIABAAAPEEAwDxGEJJABAABIi4QkcAEAAEiLnCRoAQAA +SIuMJJABAABIi7wkeAEAADH2Dx9AAOj7jwAASI0F+iUCALskAAAA6EqM/v9IjQVCCwIAuxcAAADo +OYz+/5DoU/YAAOlO9P//zMzMzMzMzMzMzMzMzMxJO2YQD4Z/AQAASIPsGEiJbCQQSI1sJBBIgzgA +Dx9AAA+EUQEAAIuIkAAAAA+64QxyLEmLVjBIi5LAAAAADx8ASDnQD4UdAQAATIn2SDnWD4QRAQAA +g/kCD4UIAQAASIN4cAB1GoC4tAAAAAB0BDHJ6w+KiLkAAACEyQ+UwesCMcmEyQ+EzAAAAEmLTjBI +OYHAAAAAdRJIi0gwSIO5AAMAAAAPhZwAAACDPYV2CgAAD4+FAAAASIlEJCBIi4g4AQAASInI6Jcc +AABIhcB0EIB4KAZ1CkiLbCQQSIPEGMNIi0QkIEiLWAhIiwhIidpIKctIidlI0etIgfsACAAAcjFI +i3A4SCnySIHCIAMAAEjB6QJIOdF3C0iLbCQQSIPEGJDD6Jrv//9Ii2wkEEiDxBjDSItsJBBIg8QY +w0iLbCQQSIPEGMNIjQVYDwIAuxoAAADoy4r+/0iNBZUJAgC7FwAAAOi6iv7/SI0FOgwCALsZAAAA +6KmK/v9IjQWeEwIAuxwAAADomIr+/5BIiUQkCOit9AAASItEJAjpY/7//8zMzEk7ZhAPhnsBAABI +g+xoSIlsJGBIjWwkYDHA6xWQkEiLRCRY6Bhi/P9Ii0wkKEiNQQFIg/gEfU5IiUQkKEjB4AZIiUQk +UEiNFXR6CgBIjRwCSIlcJFiQSInY6AJg/P9Ii0wkUEiNFVZ6CgBIjQQKSI1ACEiJRCQwSItMCggP +H0AA6agAAACQSI0FE4IKAOjOX/z/McDrCkiNQQFmDx9EAABIg/gjfSJIiUQkOEiJwUjB4ARIjRXv +gQoASI00AkiJdCRISIsUAutYkJBIjQXPgQoA6Gph/P9Ii2wkYEiDxGiQw0iJVCQYSIsKSIlMJEBI +ifBIidPoB9P9/0iNBWAGCQBIi1wkGLkBAAAA6PHO/f9Ii0wkOEiLdCRISItUJEBmkEiF0nW86Wz/ +//9IifFIhckPhOb+//9IizFmg3lgAGaQdehIiUwkIEiJdCRASInL6KzS/f9Ii1wkIEjHQygAAAAA +SI0F+AUJALkBAAAA6I7O/f9Ii0QkMEiNFUJ5CgBIi3QkQOul6BbzAADpcf7//8zMzMzMzMzMzMzM +zMzMzMzMTI1kJLBNO2YQD4acBwAASIHs0AAAAEiJrCTIAAAASI2sJMgAAABIx4QksAAAAAAAAABE +DxG8JLgAAABIi1AYkEiF0g+EKgQAAEiJhCTYAAAATIsQTImUJKgAAABMi1gITImcJKAAAABJORJ1 +B7n/////6ztIjXr/SIl8JDhMidC5AQAAAEiJ3kyJ2+hwJwAATIuUJKgAAABMi5wkoAAAAInBSItU +JDhIi4Qk2AAAAEiJVCQ4g/n/QbwAAAAAQQ9EzIlMJCxMi2A4TCtgKE2F5A+GCAEAAEyJZCRAQYB6 +KwF3C02J10Ux0umYAAAATY1qK0WLeiBPjWy9AE2NbQFNie9BD7rnAnN2TYnXQQ+64gJzZkyJrCSA +AAAA6CGg/v9IjQUiCgIAuxkAAADoMKn+/0iLhCSoAAAA6KOo/v8PHwDoe6L+/+h2oP7/SIuEJNgA +AACLTCQsSItUJDhMi5wkoAAAAEyLZCRATIusJIAAAABMi7wkqAAAAEmDxQTrA02J102LVQhNhdIP +hG0FAABFiypFhe0PjmEFAABFi2IEZpBFheR+MUyJVCR4hckPjHMEAABBOc0PjmoEAACQRY1sJAdB +wf0DRA+v6U1j7U+NFCpNjVII6xFFMeRFMdLrCU2J10Ux5EUx0kyJlCSYAAAARIlkJDRMi2hIDx+E +AAAAAABNhe0PhjEBAABIidNIi1BQSIXSdBpEixpJwe0DRTnrRQ9P3UiLUghNif3pEwEAAEGAfysA +dw9Nif0x0g8fRAAA6ZsAAABJjVcrRYtvIEqNFKpIjVIBSYnVQQ+65QJzdk2J/UEPuucCc2ZIiVQk +aOjNnv7/SI0FzggCALsZAAAAkOjbp/7/SIuEJKgAAADoTqf+/+gpof7/6CSf/v9Ii4Qk2AAAAItM +JCxIi1QkaEiLXCQ4TIuUJJgAAABMi5wkoAAAAESLZCQ0TIusJKgAAABIg8IE6wNNif1IixIPH0QA +AEiF0g+EgwIAAESLOkWF/w+OdwIAAEiJlCSIAAAADx8AhckPjJIBAABBOc8PjokBAABEi1oERYXb +fhlFjXsHQcH/A0EPr89MY/lKjRQ6SI1SCOsPRTHbMdLrCE2J/UUx2zHSSIN4UAB0NUyLLWBhBwBM +iz1hYQcASInQSIsVX2EHAEyJrCSwAAAATIm8JLgAAABIiZQkwAAAAOnBAAAAQYB9KwJ3CkiJ0DHS +6YsAAABNjX0rSInQQYtVIEmNFJdIjVIBSYnXQQ+65wJzakEPuuUCc15IiVQkcEiJhCSQAAAARIlc +JDDodp3+/0iNBXcHAgC7GQAAAOiFpv7/SIuEJKgAAADo+KX+/+jTn/7/6M6d/v9Ii4QkkAAAAEiL +VCRwTIuUJJgAAABEi1wkMESLZCQ0kEiDwgRIi1IQSIXSdCBMiyqQSIPCCEiJlCSwAAAATImsJLgA +AABMiawkwAAAAEyLhCS4AAAATIuMJMAAAABIi7QksAAAAEyJ00SJ2UiJx0SJ4EiLrCTIAAAASIHE +0AAAAMNIi7QksAAAADHAMduJwUiJ30UxwE2JwUiLrCTIAAAASIHE0AAAAMNMiehMidvoWxwAAEiJ +hCSYAAAASIlcJGBIi4wkiAAAAEhjCUiJTCRY6Hmc/v9IjQV5+QEAuxMAAADoiKX+/4tMJCxIY8GQ +6Huj/v9IjQU64gEAuwUAAADoaqX+/0iLRCRYDx9EAADoW6P+/0iNBWQLAgC7HAAAAOhKpf7/SIuE +JJgAAABIi1wkYOg4pf7/SI0F2ukBALsLAAAA6Cel/v9Ii0QkOGaQ6Huj/v9IjQWz4AEAuwIAAADo +CqX+/+hlnP7/SI0FXPIBALsQAAAA6HSD/v9MiehMidvoiRsAAEiJhCSYAAAASIlcJGBIi4wk2AAA +AEiLUUBIiVQkUEiLSUhIiUwkSGaQ6Jub/v9IjQXZ8AEAuw8AAADoqqT+/0iLhCSYAAAASItcJGDo +mKT+/0iNBd3uAQC7DgAAAOiHpP7/SItEJFBmkOjbov7/SI0F998BALsBAAAA6Gqk/v9Ii0QkSA8f +RAAA6Lui/v/otp3+/+ixm/7/SI0FOPIBALsQAAAADx9EAADou4L+/0yJ+EyJ2+jQGgAASImEJJgA +AABIiVwkYEiLTCR4SGMJSIlMJFjo8Zr+/0iNBfH3AQC7EwAAAA8fRAAA6Puj/v+LTCQsSGPB6O+h +/v9IjQWu4AEAuwUAAAAPHwDo26P+/0iLRCRY6NGh/v9IjQXADgIAux4AAAAPH0QAAOi7o/7/SIuE +JJgAAABIi1wkYOipo/7/SI0FS+gBALsLAAAA6Jij/v9Ii0QkOOjuof7/SI0FJt8BALsCAAAAZpDo +e6P+/+jWmv7/SI0FzfABALsQAAAA6OWB/v9MifhMidvo+hkAAEiJhCSYAAAASIlcJGBIi4wk2AAA +AEiLSThIiUwkUOgXmv7/SI0FVe8BALsPAAAA6Caj/v9Ii4QkmAAAAEiLXCRg6BSj/v9IjQXL7wEA +uxAAAADoA6P+/0iLRCRQSItMJEBIKcjoUaH+/0iNBW3eAQC7AQAAAA8fRAAA6Nui/v9Ii0QkQOgx +of7/6Cyc/v/oJ5r+/0iNBa7wAQC7EAAAAOg2gf7/kEiJRCQISIlcJBCITCQY6ELrAABIi0QkCEiL +XCQQD7ZMJBjpLvj//8zMzMzMzMzMzMzMzMzMSTtmEHZASIPsGEiJbCQQSI1sJBBIiw31fAcAD7ZJ +F5D2wUB1CkiLbCQQSIPEGMNIjQVwLgIAuz0AAAAPH0QAAOi7gP7/kOjV6gAA67PMzMzMzMzMzMzM +zMzMzMzMzMzMSTtmEA+GTQIAAEiD7HBIiWwkaEiNbCRoSImcJIAAAABIhcl+EkiJ2jH2Mf9FMcBF +Mcnp0QEAADHSMfYx/w8fAEiF0g+EvAAAAEiD+gF1U0iFwHQMSInyvgEAAADrSWaQSDnxD4aHAQAA +SInySMHmBEyLBDNIi3QzCEyJRCRYSIl0JGBIi3QkWE2LRghJOTZ3CUw5xkAPksbrAjH2g/YBkOsF +SInyMfZAhPZ1PUiJjCSIAAAASImcJIAAAABIifuQ6JsEAABIi5QkiAAAAEiF0n5JSIlEJEBIiVwk +IEyLhCSAAAAARTHJ60xIOdF2JUjB4gRIiwQTSItcEwhIi2wkaEiDxHDDMcAx20iLbCRoSIPEcMNI +idDoBfEAAEiLbCRoSIPEcMNJg8AQSInxTInOSYn5TInfTYtQCEw510mJ+0kPT/pNiyBMOeF1BU05 +0+tlSIlMJEhMiUwkOEyJRCRQTIlcJDBIiXQkKEyJVCQYSInITInjSIn56MT5AABIi0wkMEiLRCQY +SDnBSIuUJIgAAABIi1wkIEiLdCQoTItEJFBMi0wkOEmJwkmJy0iLRCRASItMJEhyLkmNeQFMKdZN +KdNJifFI995Iwf4/SSHySo00EUg5+g+PSv///w8fRAAA6Tb///9MidBMidno8PAAAEiJ8Ogo8AAA +SIPDEEmJ8kyJzk2JwU2J0EyLUwhNhdJ1Dk2JykmJ8UyJxk2J0OsPT40EEU05wX8dSP/HSYnxSf/B +TDnJf8JIidNIifpMiceQ6fv9//9IjQVxCgIAux0AAADoSn7+/5BIiUQkCEiJXCQQSIlMJBhIiXwk +IOhQ6AAASItEJAhIi1wkEEiLTCQYSIt8JCDpd/3//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtm +EHZYSIPsSEiJbCRASI1sJEBIiVwkWEiJfCRoSI1UJCBEDxE6TI1EJDBFDxE4SIlcJCBIiUwkKEiJ +fCQwSIl0JDhIidO5AgAAAEiJz+gM/f//SItsJEBIg8RIw0iJRCQISIlcJBBIiUwkGEiJfCQgSIl0 +JCjopOcAAEiLRCQISItcJBBIi0wkGEiLfCQgSIt0JCjpZv///8zMzMzMzEk7ZhAPhpQAAABIg+xo +SIlsJGBIjWwkYEiJXCR4SIm8JIgAAABMiYQkmAAAAEyJlCSoAAAASI1UJCBEDxE6TI1kJDBFDxE8 +JEyNZCRARQ8RPCRMjWQkUEUPETwkSIlcJCBIiUwkKEiJfCQwSIl0JDhMiUQkQEyJTCRITIlUJFBM +iVwkWEiJ07kEAAAASInP6Cz8//9Ii2wkYEiDxGjDSIlEJAhIiVwkEEiJTCQYSIl8JCBIiXQkKEyJ +RCQwTIlMJDhMiVQkQEyJXCRI6LDmAABIi0QkCEiLXCQQSItMJBhIi3wkIEiLdCQoTItEJDBMi0wk +OEyLVCRATItcJEgPHwDp+/7//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhrcAAABI +g+wwSIlsJChIjWwkKEQPEXwkGGaQSIXJD4SEAAAASIP5AXROSIlcJEBIhcB0BkiD+SB+G0iJTCRI +SInIMdsxyegwZ/z/SItMJEhIi1wkQEiJRCQYSIlMJCDod/YAAEiLXCQgSItEJBhIi2wkKEiDxDDD +D7YLSI0VeU0HAEiNDMpIiUwkGEjHRCQgAQAAAEiLRCQYuwEAAABIi2wkKEiDxDDDRA8RfCQYMcAx +20iLbCQoSIPEMMNIiUQkCEiJXCQQSIlMJBjoi+UAAEiLRCQISItcJBBIi0wkGOkX////zMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMxJO2YQdm1Ig+wYSIlsJBBIjWwkEEiFwHQbSIP7IH8VkHdDSIXbdjG+ +IAAAAEiJ30iJwesRSInY6GQAAABIicJIichIidFIicJIichIidFIi2wkEEiDxBjDMcBIidkPHwDo +W+wAAEiJ2bogAAAA6I7sAACQSIlEJAhIiVwkEA8fAOjb5AAASItEJAhIi1wkEOls////zMzMzMzM +zMzMzMzMSTtmEHZpSIPsSEiJbCRASI1sJEBIiUQkUEQPEXwkGEjHRCQoAAAAAEQPEXwkMDHbMcno +qWX8/0iJRCQYSIt0JFBIiXQkIEiJRCQoSIl0JDBIiXQkOEiLTCQoSItEJBhIifNIid9Ii2wkQEiD +xEjDSIlEJAjoR+QAAEiLRCQIZpDpe////8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEiJRCQI +SIXbdCcPtgiA+S11Fkj/y0iJ2kj320jB+z9Ig+MBSAHY6wNIidox2zH26w8xwDHbw0j/w0iJ/g8f +QABIOdp+Rg+2PBhEjUfQQYD4CXczSbiZmZmZmZmZGQ8fAEw5xnccSI00tkmJ8UjR5kqNPE9IjX/Q +SDn3c7oxwDHbwzHAMdvDMcAx28OA+S10FUi6/////////39IOdZ3LZCA+S11FEi6AAAAAAAAAIBI +OdZ2BTHAMdvDSInwSPfegPktSA9ExrsBAAAAwzHAMdvDzMzMzMzMzMzMzMzMzMzMSIPsUEiJbCRI +SI1sJEhIhcB0F0iJwUgl/w8AAEiNkADw//9I99oxwOsoMcBIi2wkSEiDxFDDSItcJCBIi3QkMEiN +DB5Ii3QkKEiNBB66ABAAAEiJRCQoSIlUJCBIiUwkMEQPEXwkOEiJTCQ4SIlUJEBIi1wkOEiJHCRI +iVQkCMZEJBAA6Pfi+/9FD1f/ZEyLNCX4////SItEJBgPH0QAAEiD+P90kEiLTCQoSAHISItsJEhI +g8RQw8zMzMzMzMzMSIPsKEiJbCQgSI1sJCBIiwW7UgcASIsNvFIHAEjHBCQCAAAASIlEJAiJTCQQ +6A78AABFD1f/ZEyLNCX4////SItsJCBIg8Qow8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSYtOMIuR +IAEAAIuZJAEAAImZIAEAAInWweIRMfKJ3jHTweoHMdqJ88HuEDHWibEkAQAAjQQzw8zMzMzMzMzM +zEk7ZhAPhhwCAABIg+xYSIlsJFBIjWwkUEiNBUFUAQCQ6Lts/P9IiUQkQEiNDS9HBwDrDUiLiRAC +AABmDx9EAABIhckPhDIBAACAuQgCAAAAdeFIiUwkSEiLUAhIjXIBSIsYSIt4EEg593M5SI0FrkoB +AEiJ0ejmv///SIt8JEBIiU8Qgz1WYAoAAHUFSIkH6wXoCuQAAEiLTCRISInaSInDSIn4TI1CAUyJ +QAhIjTzTgz0pYAoAAHUHSIkM0+sGkOjb5AAAg7ngAQAAAA+FYf///0iDuegBAAAAZg8fRAAAD4VN +////SIuBCAEAAEiLmdgAAABIK5nQAAAADx9EAADoe7r8/0iLTCRIiYHgAQAAgz3JXwoAAHUJSImZ +6AEAAOsMSI256AEAAOiy5AAASIuBEAEAAEiLmegAAABIK5ngAAAA6Di6/P9Ii0wkSImB8AEAAIM9 +hl8KAAB1CUiJmfgBAADrDEiNufgBAADob+QAAEiLRCRA6bj+//9IizhIi1AIMcnrA0j/wUg50X1X +SIs0z4C+2AEAAAB064M9Pl8KAAB1BUiJN+sF6FLkAABIixBIi3AISDnxc11IjTzKgz0bXwoAAHUN +SI01kkUHAEiJNMrrEUiNDYVFBwAPH0QAAOi74wAAkIA9814KAAB0FEiJw0iNBS9yBwDoIub7/0iL +RCRASI0NHnIHAEiHAUiLbCRQSIPEWMNIichIifEPHwDoG+cAAJDotd8AAOnQ/f//zMzMzMzMzMzM +zMzMzMzMzEyNZCSwTTtmEA+GgAYAAEiB7NAAAABIiawkyAAAAEiNrCTIAAAASImEJNgAAABIixCL +Mol0JBxmDx+EAAAAAACD/voPhVQFAACAegQAD4VKBQAAgHoFAA+FQAUAAIB6BgEPhTYFAACAegcI +D4UsBQAASIuQgAAAAEiLiIgAAABIjXH/MdvrA0yJ00g58w+N/gEAAEiJ30jB4wRMiwQTTI1PAUw5 +yQ+G6gQAAE2JyknB4QROixwKTTnDc8xIi0hwTItAaEyLXBoISTnLD4O8BAAATQHDSotUCggPH4AA +AAAASDnKD4OcBAAASIl8JHBMiVQkaEiJXCRgTIlMJFhKjQwCSTnyfA65AwAAAEiNFdrRAQDrKUyJ +nCSYAAAASInDSInI6HkMAABMi5wkmAAAAEiJ2UiJwkiLhCTYAAAASIlMJCBIiVQkeEiJw0yJ2OhO +DAAASIuMJNgAAABIi5GIAAAASIuxgAAAAEiLfCRwSDnXD4MBBAAATItEJGBNiwQwTItMJGhMOcoP +ht8DAABMiUQkUEiJXCRoSImEJJAAAABIi0QkWEiLBDBIiUQkSOgujP7/SI0FTx0CALs0AAAAZpDo +O5X+/0iLRCRQ6JGT/v9Ii4QkkAAAAEiLXCRoDx9AAOgblf7/SI0Fn9ABALsBAAAA6AqV/v9Ii0Qk +SA8fRAAA6FuT/v9Ii0QkeEiLXCQg6OyU/v/oR4z+/0iLhCTYAAAASIuIkAEAAEiLkIgBAABIhcl0 +OkiJTCRoSImUJJAAAADomov+/0iNBcLVAQC7CQAAAOiplP7/SIuEJJAAAABIi1wkaOiXlP7/6PKL +/v/obYv+/+jojf7/6OOL/v9Ii0QkcEiLjCTYAAAAMdLpNQIAAEiLuKAAAAAPH4QAAAAAAEiFyQ+G +owEAAEg5Og+FiQEAAEiLuKgAAABIweYESIsUFkg5+g+FcQEAAEiLkMgBAABIi7DAAQAASIXSfglI +iVQkcDHJ6y9Ii6wkyAAAAEiBxNAAAADDSIu8JIgAAABIg8coSIuEJNgAAABIif5IidFIi1QkcEiL +PkiJvCSgAAAADxBGCA8RhCSoAAAADxBGGA8RhCS4AAAASIu8JMAAAABMi4QkuAAAAEyLjCSwAAAA +SIsfTDlHCHU3SIlMJGhIibQkiAAAAEyJyEyJwegP2/v/hMB0G0iLVCRoSP/CSIt0JHBIOdYPj2v/ +///pVv///0iLhCTYAAAASIuIsAEAAEiJjCSQAAAASIuAuAEAAEiJRCRwSIuUJKAAAABIiZQkgAAA +AEiLnCSoAAAASIlcJGjoC4r+/0iNBc7+AQC7HgAAAOgak/7/SIuEJJAAAABIi1wkcOgIk/7/SI0F +x88BALsFAAAA6PeS/v9Ii4QkgAAAAEiLXCRo6OWS/v8PH0QAAOg7jP7/6DaK/v9IjQUL2gEAuwwA +AADoRXH+/0iNBULsAQC7FgAAAOg0cf7/McDoreIAAEiJXCRYSImEJJAAAABIi0QkaEiLBDBIiUQk +UOhtif7/SI0FFM4BALsBAAAAkOh7kv7/SItEJFDo0ZD+/+iMi/7/SIuEJJAAAABIi1wkWOhakv7/ +6LWL/v/osIn+/0iLRCRgSI1QAUiLRCRwSIuMJNgAAABIOcIPj4wAAABIi3FwSIt5aEyLgYgAAABM +i4mAAAAATDnCc2ZJidBIweIETotMCghMOc52SkiJVCRoTIlEJGBKjQQPSInL6I8IAABIi4wk2AAA +AEiLkYgAAABIi7GAAAAASIt8JGBIOdcPgh7///9IifhIidEPH0QAAOi74QAATInISInx6NDhAABI +idBMicHopeEAAEiNBeL4AQC7HAAAAOgUcP7/TInISInR6InhAABIifhIidEPHwDoe+EAAEiJ0OiT +4QAATInY6IvhAABMicjoY+EAAA+2QgRIiUQkQA+2SgVIiUwkOA+2WgZIiVwkMA+2UgdIiVQkKOga +iP7/SI0FhAsCALsmAAAA6CmR/v+LRCQcDx9EAADoe4/+/0iLRCRA6HGP/v9Ii0QkOOhnj/7/SItE +JDBmkOhbj/7/SItEJCjoUY/+/+hMiP7/SIuEJNgAAABIi4iQAQAASIuAiAEAAEiFyXUg6KyH/v/o +J4r+/+giiP7/SI0F+/wBALseAAAA6DFv/v9IiUwkcEiJhCSQAAAADx9AAOh7h/7/SI0Fo9EBALsJ +AAAA6IqQ/v9Ii4QkkAAAAEiLXCRw6HiQ/v/o04f+/+ugSIlEJAjoB9kAAEiLRCQIZpDpW/n//8zM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEiD7BhIiWwkEEiNbCQQSI0NSz4HAOsJSIuJEAIAAGaQ +SIXJdBRIOYGgAAAAd+lIOYGoAAAAduDrBzHJDx9EAABIhckPhIIAAABIi5GgAAAASInGSCnQSInC +SMHoDEiNPIBIwecCSAO5mAAAAEiB4v8PAABIweoIRIsHZg8fRAAASIP6EA+DCwEAAA+2VBcERAHC +SIu5iAAAAEyLgYAAAAA513cESI1X/4nQSDnHD4baAAAASMHgBE2LDAAPH0QAAEk58Q+HhwAAAOsR +McAx20iLbCQQSIPEGMNEicqNQgGQSDnHdmFBicFIweAETosUAEk58nbkidBIOcd2QkjB4ARJi0QA +CEiD+P90HUiLUXBIi3FoSDnCdh5IAfBIictIi2wkEEiDxBjDMcAx20iLbCQQSIPEGMNIidHoIt8A +AEiJ+ej63gAASIn56PLeAAD/yonQSDnHdi5IweAETYsMAJBJOfF2BoXSd+XrBIXSdYZIjQXbBAIA +uyMAAAAPH0QAAOg7bf7/SIn56LPeAABIifnoq94AAEiJ0LkQAAAADx8A6LveAACQzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMxMjWQk6E07ZhAPhsgEAABIgeyYAAAASImsJJAAAABIjawkkAAAAEiJ +hCSgAAAASImcJKgAAACFyXQZSIX2dCuQSIn6SMHvA0iD5wFFMcnpOgQAALj/////MdtIi6wkkAAA +AEiBxJgAAADDSIXAdHZIi1NYTItLUEyLU2BBictmkEw52g+C7QMAAEiJtCTAAAAARIiEJMgAAABI +iZwkiAAAAEiJhCSAAAAATIlcJGhIibwkuAAAAImMJLAAAABMiyBMiWQkQMdEJDT/////TSnaTCna +TYnVSffaScH6P00h2k0B0et8RYTAdAmDPW9TCgAAdBe4/////zHbSIusJJAAAABIgcSYAAAAw0iL +AEiJRCRg6EmE/v9IjQW+9QEAuxwAAADoWI3+/0iLRCRg6K6L/v/oqYb+/+ikhP7/SI0FMdgBALsO +AAAA6LNr/v9Jic1JicFJidRIi4QkgAAAAEiJ2kyJZCQ4TItUJEBMORBBD5TASInTTInpSI18JEBI +jXQkNEyJyOjWCwAAQIT/D4THAAAASItUJEBMi4wkuAAAAEk50XOoSIuMJMAAAABIhckPhIsAAABJ +i1YwkIuyIAEAAIu6JAEAAIm6IAEAAEGJ8MHmEUEx8In+RDHHQcHoB0Ex+In3we4QRDHGibIkAQAA +TInKScHpA0mD4QFJweEHTo0ECQH+g+YHSMHmBEKLfAkIRotUCQxOixwJTYkcMEGJfDAIRYlUMAyL +dCQ0SokUCYuUJLAAAABCiVQJCEKJdAkMi0QkNEiLXCQ4SIusJJAAAABIgcSYAAAAw4M9/1EKAAAP +hTYBAAAPtpQkyAAAAITSD4QmAQAASIlMJFBIiVwkSEiJRCRwSIuEJIAAAABIi5wkiAAAAOiFAgAA +SIlEJHhIiVwkWEiLTCRASIlMJGDorIL+/0iNBTcDAgC7JAAAAOi7i/7/SItEJHhIi1wkWOisi/7/ +SI0FwccBALsEAAAA6JuL/v9Ii0QkYOjxif7/SI0FLs8BALsKAAAADx9EAADoe4v+/0iLhCS4AAAA +6M6J/v9IjQVayAEAuwUAAABmkOhbi/7/SItEJHBIi1wkSEiLTCRQ6MeL/v/oooT+/2aQ6JuC/v9I +i4wkiAAAAEiLUVhIi3FQSItJYEiLRCRoZpBIOcIPggUBAABMi4wkgAAAAE2LEUyJVCRAx0QkNP// +//9IKcFIKcJJicpI99lIwfk/SCHBTI0cDumTAAAAuP////8x20iLrCSQAAAASIHEmAAAAMNIiUwk +UEiJXCRISIlEJHBIi0QkQEiJRCRg6I6B/v9IjQUwyQEAuwcAAABmkOibiv7/SGNEJDTokYj+/0iN +BULOAQC7CgAAAA8fRAAA6HuK/v9Ii0QkYOjRiP7/6MyD/v/ox4H+/0yLjCSAAAAATItUJFBIi1Qk +SEyLXCRwTItkJEBNOSFBD5TATInYSInTTInRSI18JEBIjXQkNOgMCQAAQIT/D4VX////SI0FYPEB +ALscAAAA6JJo/v9IidHoytoAAEyJ2EiJ0Q8fQADou9oAAE2NTCQBTInXSYP5CH08SYn6SMHnB0yN +HD5NicxJweEER4tsCwhBOc111k+LLAtJOdV1zUOLRAsMMdtIi6wkkAAAAEiBxJgAAADDSInX6ZP7 +//9IiUQkCEiJXCQQiUwkGEiJfCQgSIl0JChEiEQkMOgn0gAASItEJAhIi1wkEItMJBhIi3wkIEiL +dCQoRA+2RCQw6eT6///MzMzMSTtmEA+GfAAAAEiD7DBIiWwkKEiNbCQoSIlEJDhIiVwkQEiFwHQd +i1AIhdJ0FkiLSxBIi1sISGPCSDnBdj9IjQwD6wIxyUiJTCQQSInI6G3u//9EDxF8JBhIi0wkEEiJ +TCQYSIlEJCBIi0wkGEiJw0iJyEiLbCQoSIPEMMMPHwDo29gAAJBIiUQkCEiJXCQQ6GvRAABIi0Qk +CEiLXCQQkOlb////zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEHZzSIPsGEiJbCQQSI1s +JBBIiUQkIEiJXCQoZpDoG////0iNS//rA0j/yUiFyX4PD7Y0AUCA/i917usDSP/BSDnLfg92JA+2 +NAFAgP4udexIOctyDUiJy0iLbCQQSIPEGMNIidroc9gAAEiJyEiJ2ego2AAAkEiJRCQISIlcJBDo +uNAAAEiLRCQISItcJBDpaf///8zMzMzMzMzMzEk7ZhB2d0iD7DBIiWwkKEiNbCQoSIlEJDhIiVwk +QGaQSIXAdBZIi1MQSItbCEhjwUg5wnY/SI0MA+sCMclIiUwkEEiJyOgW7f//RA8RfCQYSItMJBBI +iUwkGEiJRCQgSItMJBhIicNIichIi2wkKEiDxDDDSInR6ITXAACQSIlEJAhIiVwkEIlMJBjoENAA +AEiLRCQISItcJBCLTCQYZpDpW////8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhrUA +AABIg+wwSIlsJChIjWwkKEiJRCQ4SIlcJEBIhcB0ckiLUyhIi3MgA0gkSDnKdn2LBI6D+P90REiL +S0BIi1M4SDnBdmNIAdBIiUQkEOhH7P//RA8RfCQYSItMJBBIiUwkGEiJRCQgSItMJBhIicNIichI +i2wkKEiDxDDDSI0FQsIBALsBAAAASItsJChIg8Qww0iNBSzCAQC7AQAAAEiLbCQoSIPEMMPojNYA +AInISInR6ILWAACQSIlEJAhIiVwkEIlMJBjoDs8AAEiLRCQISItcJBCLTCQY6Rv////MzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4baAAAASIPsYEiJbCRYSI1sJFhIiUQkaEiJXCRwSIXA +D4ShAAAASIlMJEBAiHwkN0iJRCRQSIlcJEiLUBgx9kGJ+EiJz4nR6G/3//+JRCQ4SItUJFCLShxI +i1wkSEiLfCRAMfZED7ZEJDdIidDoSff//4tMJDgPH0QAAIP5/3QTg/j/dA5IY9FIi1wkSEg5U0B/ +GEiNBS3BAQC7AQAAADHJSItsJFhIg8Rgw4lEJDxIi0QkUOhC/v//i0wkPEiLbCRYSIPEYMNIjQX5 +wAEAuwEAAAAxyUiLbCRYSIPEYMNIiUQkCEiJXCQQSIlMJBhAiHwkIOjjzQAASItEJAhIi1wkEEiL +TCQYD7Z8JCDp6v7//8zMzMzMzMzMzMxJO2YQdixIg+woSIlsJCBIjWwkIEiJRCQwSIlcJDi/AQAA +AOi4/v//SItsJCBIg8Qow0iJRCQISIlcJBBIiUwkGOh6zQAASItEJAhIi1wkEEiLTCQY66nMzMzM +zMzMzMxJO2YQD4YJAQAASIPseEiJbCRwSI1sJHBIiYQkgAAAAEiJnCSIAAAASIlMJFBIiVwkaEiJ +RCRgi1AUSIn+QbgBAAAASInPidHo8/X//6kHAAAAD4SxAAAAiUQkNEiLRCRgSItcJGjo9fr//0iJ +RCRYSIlcJEhIi0wkYEiLEUiJVCRAi0kUSIlMJDjoEXv+/0iNBejRAQC7EAAAAA8fRAAA6BuE/v9I +i0QkWEiLXCRI6AyE/v/oJ33+/0iLRCRAZpDoW4L+/+gWff7/SItEJFDoTIL+/+gHff7/SItEJDhm +kOg7gv7/6PZ8/v+LTCQ0SGPB6MqB/v/oJX3+/w8fRAAA6Bt7/v+LRCQ0SItsJHBIg8R4w0iJRCQI +SIlcJBBIiUwkGEiJfCQg6DTMAABIi0QkCEiLXCQQSItMJBhIi3wkIOm7/v//zMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMSTtmEA+GxgAAAEiD7FBIiWwkSEiNbCRISIlEJFhIiVwkYEiLS1hIi1NQ +TItLYESLUBRMOdEPgowAAABIiUQkQEyLGEyJXCQ4x0QkMP////9NKdFMKdFNictJ99lJwfk/TSHK +TAHSMdvrIkSLTCQwRItUJDRFOcpFD0zRSYnLSInZSInCRInTSItEJECJXCQ0TItMJDhMOQhBD5TA +SI18JDhIjXQkMEiJ0EiJy0yJ2eiMAQAAQIT/dbGLRCQ0SItsJEhIg8RQw0yJ0OhR0wAAkEiJRCQI +SIlcJBDoIcsAAEiLRCQISItcJBDpEv///8zMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2VEiD7DhIiWwk +MEiNbCQwSIlEJEBIiVwkSGaQOUggdw+4/////0iLbCQwSIPEOMNIjVArQYnJSo0UikiNUgGLCkG4 +AQAAAOiQ8///SItsJDBIg8Q4w0iJRCQISIlcJBCJTCQYSIl8JCBIiXQkKOiJygAASItEJAhIi1wk +EItMJBhIi3wkIEiLdCQo6Wz////MzMzMzMzMzMzMzMxJO2YQdlhIg+w4SIlsJDBIjWwkMEiJRCRA +SIlcJEhmkDlIIHcRuP////8x20iLbCQwSIPEOMNIjVArQYnJSo0UikiNUgGLCjH2QbgBAAAA6Ozy +//9Ii2wkMEiDxDjDSIlEJAhIiVwkEIlMJBhIiXwkIOjqyQAASItEJAhIi1wkEItMJBhIi3wkIOly +////zMzMzMzMzMzMzMzMzMzMzMzMSIPsGEiJbCQQSI1sJBBIiUQkIEiF2w+GXAEAAA+2EJCF0nUY +RYTAdRMxwDHbSInZMf9Ii2wkEEiDxBjDD7riB3MNMdJFMcBFMcnp6QAAAEG4AQAAAEGJ0YPiAffa +QdHpRDHKARZMOcMPgrgAAABMKcNMKcFIicpI99lIwfk/SSHISo00AEiF2w+GjgAAAEYPtgQAQQ+6 +4AdzCTHAMclFMcDrQ7gBAAAASDnYdyhMAQdIKcJIKcNIidFI99pIwfo/SCHQSAHwvwEAAABIi2wk +EEiDxBjDSInZ6APRAABEjUEHicFEidBIOdhzKUQPtgwwRI1QAUWJy0GD4X+JyESJwUHT4UQJyEH2 +w4B10UGJwESJ0OuUSInZ6ATQAAAxwEiJ2ej6zwAARInASInZ6K/QAABEjUkHTInhSDnTdjFED7YU +Av/CRYnTQYPif0mJzESJyUHT4kUJ0EH2w4B11EyJ4UGJ0USJwkWJyOnn/v//idBIidnoqM8AADHA +SInZDx8A6JvPAACQzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4ZgAQAASIPsUEiJbCRI +SI1sJEhIiUQkWEiJXCRggz0nRQoAAHUP6DAa/v9Ii0QkWEiLXCRgSIM7AGaQD4USAQAAkEiJwkiJ +A4QCSIu66CYAAEyLguAmAABMi4rwJgAASI13AUk58XIFSIn461VIiXwkQEiNBaI1AQBMicNIiflM +ic/oFKb//0iLVCRYSImK8CYAAIM9gUYKAACQdQlIiYLgJgAA6wxIjbrgJgAA6CnKAABIid9JicBI +i0QkQEiLXCRgSI13AUiJsugmAABJjTT4gz1DRgoAAHUGSYkc+OsISIn36DPLAABIi7LgJgAASIua +6CYAAEiLivAmAABIicdIifDoExwAAEiLVCRYSIuK6CYAAEiLsuAmAABIhcl2LEiLRCRgSDkGdQtI +i0AISIeCaBYAALgBAAAA8A/BgvgmAABIi2wkSEiDxFDDMcDoKM4AAEiNBbvyAQC7IgAAAOiXXP7/ +kEiJRCQISIlcJBDop8YAAEiLRCQISItcJBDpeP7//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7 +ZhAPhnMCAABIg+wgSIlsJBhIjWwkGEiJRCQo6wNIidiLSECD+QV3JIP5AXcRhckPhacAAADpQwEA +AA8fQACD+QIPhAEBAADpPQEAAIP5Bw+HggAAAIP5BnRcSYtWMJD/gggBAABJi1YwSInDici+BgAA +APAPsXNAD5TBDx8AhMkPhQ8BAACQi4oIAQAAjXn/iboIAQAAg/kBdYJBgL6xAAAAAA+EdP///0nH +RhDe+v//6Wf////oI+cAAEUPV/9kTIs0Jfj///9Ii1wkKL4GAAAA6Ub///+D+Qh1Z0mLVjCQ/4II +AQAASYtWMEiJw4nIvgYAAADwD7FzQA+UwZCEyQ+FAQEAAJCLiggBAACNef+JuggBAAAPH4QAAAAA +AIP5AQ+F9v7//0GAvrEAAAAAD4To/v//ScdGEN76//+Q6dr+//+D+Ql1IeiR5gAARQ9X/2RMizQl ++P///0iLXCQovgYAAADptP7//+iQHQAASItcJCi+BgAAAOmg/v//McBIi2wkGEiDxCDDMcBIi2wk +GEiDxCDDSIsLifC+AwAAAPAPsXNAD5TDhNt1F0iJFCRIiUwkEOhFHQAASItMJBBIixQkkIuaCAEA +AI1z/4myCAEAAIP7AXUSQYC+sQAAAAB0CEnHRhDe+v//hAG6AQAAAPAPwZH8JgAAuAEAAABIi2wk +GEiDxCDDSIsLifC+AwAAAPAPsXNAD5TDhNt1GUiJVCQISIlMJBDo0hwAAEiLTCQQSItUJAiQi5oI +AQAAjXP/ibIIAQAAg/sBdRJBgL6xAAAAAHQIScdGEN76//+EAboBAAAA8A/BkfwmAAC4AQAAAEiL +bCQYSIPEIMNIiUQkCOj5wwAASItEJAjpb/3//8zMzMzMzMzMzMzMzMzMzEk7ZhAPhsUBAABIg+wo +SIlsJCBIjWwkIIQASIuI6CYAAEiLsOAmAABIOcsPg5UBAABIizTeTIsGDx+EAAAAAABMOcAPhWwB +AABIxwYAAAAASIuI6CYAAEiLsOAmAABMjUH/TDnDdC9MOcEPhj0BAABMi0zO+Eg5yw+DJwEAAEiN +PN6DPVdCCgAAdQZMiQze6wXoqscAAEiLsOAmAABMi4joJgAATTnBD4btAAAASI08zkiNf/iDPSRC +CgAAdQtIx0TO+AAAAADrBzH26DDHAABIi5DwJgAAZg8fhAAAAAAATDnCD4KpAAAATImA6CYAAEw5 +w3RUSIlEJDBIiVwkOEiLkOAmAABIi4jwJgAASInfTInDSInQ6MUXAABIi1QkMEiLguAmAABIi5ro +JgAASIuK8CYAAEiLfCQ46AEZAABIi0QkMEiLXCQ4SIXbdS2QSIuI4CYAAEiDuOgmAAAAZpB1CzHJ +SIeIaBYAAOsOSIsJSItJCEiHiGgWAAC5//////APwYj4JgAASItsJCBIg8Qow0yJwegmygAATInA +TInJ6JvJAABIidjok8kAAEyJwOiLyQAASI0FdMwBALsTAAAA6PpX/v9IidjocskAAJBIiUQkCEiJ +XCQQ6ALCAABIi0QkCEiLXCQQ6RP+///MzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GYwEAAEiD7ChI +iWwkIEiNbCQghABIi4joJgAASIuw4CYAAEiFyQ+GNAEAAEiLNkyLBmYPH4QAAAAAAEw5wA+FBwEA +AEjHBgAAAABIi7DoJgAASIu44CYAAEiNTv9Ihcl+GEyLRPf4gz1tQAoAAHUFTIkH6wXoocUAAEyL +gOAmAABMi4joJgAASTnJD4avAAAASY088EiNf/iDPTtACgAAdQtJx0Tw+AAAAADrBzH26EfFAABI +i5DwJgAASDnKcnZIiYjoJgAASIXJfihIiUQkMEiLkOAmAABIi7DwJgAASInLSInxMf9IidDoTBcA +AEiLRCQwkEiLiOAmAABIg7joJgAAAHULMclIh4hoFgAA6w5IiwlIi0kISIeIaBYAALn/////8A/B +iPgmAABIi2wkIEiDxCjDDx9EAADoe8gAAEiJyEyJyejwxwAASI0FRM0BALsUAAAADx9AAOhbVv7/ +McDo1McAAJBIiUQkCOhpwAAASItEJAgPH0AA6Xv+///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMxJO2YQD4aeBAAASIPsQEiJbCQ4SI1sJDhIiXQkaEyJRCRwSIXbD45rBAAASIXJD4xRBAAASIl8 +JGBIiUwkWEiJXCRQSIlEJEhMiUwkeEyJRCRwSIl0JDDrA0yJ2ItQQJCD+gN3dIP6AXcJhdJ0e+nn +AAAAg/oCD4ROAQAATYtWMJBB/4IIAQAATYtWMEmJw4nQQbwGAAAA8EUPsWNAD5TChNIPhd0BAACQ +QYuSCAEAAESNav9FiaoIAQAADx9EAACD+gF1lEGAvrEAAAAAdIpJx0YQ3vr//+uAg/oFd25mkIP6 +BA+E4QAAAE2LVjCQQf+CCAEAAE2LVjBJicOJ0EG8BgAAAPBFD7FjQA+UwoTSD4WKAQAAkEGLkggB +AABEjWr/RYmqCAEAAIP6AQ+FKP///0GAvrEAAAAAD4Qa////ScdGEN76///pDf///4P6Bg+E6wAA +AIP6CHdrTYtWMJBB/4IIAQAATYtWMEmJw4nQQbwGAAAA8EUPsWNAD5TCZpCE0g+FIAEAAJBBi5II +AQAARI1q/0WJqggBAAAPH0QAAIP6AQ+FsP7//0GAvrEAAAAAD4Si/v//ScdGEN76//+Q6ZT+//+D ++gl1QOjR3wAARQ9X/2RMizQl+P///0iLTCRYSItcJFBIi3QkMEiLfCRgTItEJHBMi0wkeEyLXCRI +QbwGAAAA6U/+///osRYAAEiLTCRYSItcJFBIi3QkMEiLfCRgTItEJHBMi0wkeEyLXCRIQbwGAAAA +6Rz+//8PHwDoW98AAEUPV/9kTIs0Jfj///9Ii0wkWEiLXCRQSIt0JDBIi3wkYEyLRCRwTItMJHhM +i1wkSEG8BgAAAA8fRAAA6dT9//9JixOEAkG9//////BED8Gq/CYAADHAMdLrELgBAAAAMdLrBzHA +ugEAAABJiUsQgz2GPAoAAHUGSYl7GOsYSY1LGEmJ/UiJz0GJ10yJ6uhJwQAARIn6SYlzIIM9WzwK +AAB1Bk2JQyjrCUmNeyjoisEAAEyJVCQYiFQkE02JSzCEwHUoSYlbOEk5Wwi5CAAAAL4HAAAAD0/O +SYszkIP5Bw+FrgAAAJDpIAEAAEmJWwhJi04wSIuJ0AAAAEiJTCQohAFIjYHYJgAASIlEJCCQ6Kko +/P9Ii0QkKEiLXCRI6Lr0//+QkEiLRCQg6G4q/P+4BgAAAEiLTCRIugEAAADwD7FRQA+UwYTJdQXo +LhUAAJBIi0wkGIuRCAEAAI1a/4mZCAEAAIP6AXUSQYC+sQAAAAB0CEnHRhDe+v//SItEJFDomNb+ +/w+2RCQTSItsJDhIg8RAw0SJ4PBBD7FLQEAPlMZAhPZ1HIlMJBTozhQAAItMJBQPtlQkE0iLXCRQ +TItUJBhBi7IIAQAAjX7/QYm6CAEAAIP+AXUcQYC+sQAAAAB0DUnHRhDe+v//g/kH6wiD+QfrA4P5 +B3WMSInYkOgb1v7/D7ZUJBPpef///4QGSIu+cBYAAEiF/3QMDx8ASDnfD4xu////SIn48EgPsZ5w +FgAAQA+Ux0CE/3TRZpDpUv///0iNBWvnAQC7IQAAAOiKUf7/SI0F3dgBALsbAAAA6HlR/v+QSIlE +JAhIiVwkEEiJTCQYSIl8JCBIiXQkKEyJRCQwTIlMJDjocLsAAEiLRCQISItcJBBIi0wkGEiLfCQg +SIt0JChMi0QkMEyLTCQ46Qj7///MzMzMzMzMzEk7ZhAPhgYCAABIg+wwSIlsJChIjWwkKEiJTCRI +SIlcJEBIiUQkIDHS6wZI/8JMicBIOdEPjtMAAABIiVQkEEiLNNNIiXQkGOnNAAAASMcGAAAAAEyJ +wEiJ8+jB8v//uAkAAABIi0wkGLoBAAAA8A+xUUAPlMGEyXUK6EETAAC6AQAAAEiLTCRISItUJBBI +i1wkQEyLRCQgQbkFAAAAZpDriUjHBgAAAADrgEiLTjhIiU4ISMcGAAAAAEyJwEiJ8+hb8v//uAkA +AABIi0wkGLoBAAAA8A+xUUAPlMEPH0AAhMl1CujXEgAAugEAAABIi0wkSEiLVCQQSItcJEBMi0Qk +IEG5BQAAAOke////SItsJChIg8Qww0yJwIt+QA8fAIP/A3ddg/8BdyeF/w+EtQAAAEmJwIn4QbkJ +AAAA8EQPsU5AQA+Ux0CE/3TL6fz+//+D/wIPhIYAAABJicCJ+EG5BQAAAPBED7FOQEAPlMdAhP8P +hSr///9BuQkAAABmkOuVg/8Fdk+D/wZ0NoP/CHcjSYnAifhBuQkAAADwRA+xTkBAD5THQIT/D4Ro +////6ff+//+D/wl0J+gCEgAAZpDrKujZ2gAARQ9X/2RMizQl+P///+sWg/8EdQwPH0QAAOjbEQAA +6wXo1BEAAEiLTCRISItUJBBIi1wkQEiLdCQYTItEJCBBuQkAAADpB////0iJRCQISIlcJBBIiUwk +GEiJfCQg6Be5AABIi0QkCEiLXCQQSItMJBhIi3wkIA8fAOm7/f//zMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMSTtmEA+G6QIAAEiD7HBIiWwkaEiNbCRohABIi5BwFgAASIXSdB1IOdp/GEiJRCR4 +MdJIh5BwFgAAMckx0jHbMfbrDUiLbCRoSIPEcMNI/8FMi4DoJgAATIuI4CYAAGaQTDnBD41QAgAA +D4N4AgAATYsEyU2LCEw5yA+FVwIAAEiJTCRQTIlEJFhIiVwkSEiJVCRASIl0JGBFi0hADx9AAEGD ++QMPh6YAAABBg/kBdxBFhcl1mOl5AQAAZg8fRAAAQYP5Ag+EaQEAAEiJx0SJyEG6BAAAAPBFD7FQ +QEEPlMFFhMl0Y0iJ+EiJy+gQ9P//uAQAAABIi0wkWLoFAAAA8A+xUUAPlMGEyXUK6FAQAAC6BQAA +AL7/////SItEJHjwD8Gw/CYAAEiLdCRQSI1O/0iLVCRASItcJEhIi3QkYEiJx0G6BAAAAEiJ+On9 +/v//QYP5Bg+G0wAAAGYPH0QAAEGD+QgPh7sAAABIicdEichBugkAAADwRQ+xUEBBD5TBRYTJD4SU +AAAASYtQOEmJUAhIifhIicvoZPP//0iLTCRASI1xAUiLfCRISDn3cgdIi0QkYOsdSI0FIyUBAEiL +XCRg6JmV//9IjXMBSInPSItMJEBIjRTIgz0CNgoAAGaQdQtIi1QkWEiJFMjrE0iJ+UiJ10yLRCRY +6CO7AABIic9Ii1QkUEiNSv9BugkAAABIiftIifJIicZIi3wkeEiJ+Oko/v//QYP5CXQI6zNBg/kF +dyPoKA8AAEiLRCR4SItMJFBIi1wkSEiLVCRASIt0JGDp9/3//0GD+QZ0KQ8fQADo+w4AAEiLRCR4 +SItMJFBIi1wkSEiLVCRASIt0JGBmkOnI/f//6LbXAABFD1f/ZEyLNCX4////SItEJFBIjUj/SItE +JHhIi1wkSEiLVCRASIt0JGDplP3//0iF0n4OSInRSInfSInz6FQAAABIi2wkaEiDxHDDSI0FB8AB +ALsTAAAA6NlL/v9IichMicHoTr0AAJBIiUQkCEiJXCQQDx8A6Nu1AABIi0QkCEiLXCQQ6ez8///M +zMzMzMzMzMzMzMxJO2YQD4aQAAAASIPsMEiJbCQoSI1sJChIiUwkSEiJXCRASIlEJCAx0usaSIt0 +JBBI/8ZIi0QkIEiLTCRISItcJEBIifJIOdF+RkiJVCQQSIsM00iJTCQYSInLDx9EAADoO+3//7gJ +AAAASItMJBi6AQAAAPAPsVFAD5TBDx9AAITJdafotw0AALoBAAAA65tIi2wkKEiDxDDDSIlEJAhI +iVwkEEiJTCQYSIl8JCDoDbUAAEiLRCQISItcJBBIi0wkGEiLfCQg6TT////MzMzMzMzMzMzMzMzM +zMzMzMzMzEk7ZhgPhoMCAABIg+woSIlsJCBIjWwkIEiJRCQwSIlcJDjrA0iJyIQASIuI6CYAAEiL +kOAmAABmDx+EAAAAAABIhckPhjwCAABIixJIizJIOfAPhRcCAABIiVQkGItyQIP+Aw+HygAAAIP+ +AXdAhfYPhKABAABIi3oIZg8fRAAASDn7D4zIAQAASInBifC/AgAAAPAPsXpAQA+Uxg8fQABAhPYP +hHv////piQEAAIP+Ag+ETgEAAEiJwYnwvwQAAADwD7F6QEAPlMZAhPZ1Cr8CAAAA6Uv///9IicgP +H0AA6Bvy//+4BAAAAEiLTCQYugUAAADwD7FRQA+UwQ8fQACEyXUK6FcMAAC6BQAAAL7/////SIt8 +JDDwD8G3/CYAAEiDv+gmAAAAD4X3AAAA6SsBAACD/gUPhsQAAAAPH0AAg/4GD4SXAAAAg/4Id3dI +icGJ8L8JAAAA8A+xekBAD5TGQIT2dQq/AgAAAOm0/v//SItaOEiJWghIicgPH0QAAOh78f//SItE +JDBIi1wkGOgs6///uAkAAABIi0wkGLoBAAAA8A+xUUAPlMGEyXUK6KwLAAC6AQAAALoFAAAASIt8 +JDDrYIP+CXQ76JELAAC6BQAAAEiLfCQw60oPH0QAAOhb1AAARQ9X/2RMizQl+P///7oFAAAASIt8 +JDDrJ2aQg/4EdRHoVgsAALoFAAAASIt8JDDrD+hFCwAAugUAAABIi3wkMEiJ+UiLXCQ4vwIAAADp +7f3//0iJyEiJ2UiJ0+h7AAAAMcBIi2wkIEiDxCjDSIn4SItsJCBIg8Qow0jHwP////9Ii2wkIEiD +xCjDSI0FHLYBALsPAAAADx9EAADoO0j+/zHA6LS5AACQSIlEJAhIiVwkEOiE2QAASItEJAhIi1wk +EOlV/f//zMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmGA+GZgEAAEiD7FBIiWwkSEiNbCRISIlcJGBI +iUQkWEiLUxhIiVQkOEiLcyBIiXQkKEyLQyhMiUQkQEyLSzBMiUwkIEyLUxBNhdIPjqoAAABMi1sI +TCnZSInHSInISJlJ9/pIjVABTA+v0kuNFBpIiVMISIXSfQ5Iuv////////9/SIlTCIQHSIuH4CYA +AEiLn+gmAABIi4/wJgAAMf/o2gcAALgCAAAASItUJGC+AQAAAPAPsXJAD5TCDx8AhNJ1BejXCQAA +kEiLTCRYSIuR4CYAAEiDuegmAAAAZpB1CzHSSIeRaBYAAOs9SIsSSItSCEiHkWgWAADrLQ8fAOg7 +7///uAIAAABIi0wkYDHS8A+xUUAPlMGEyXUIDx8A6HsJAABIi0wkWIQBSI2B2CYAAEiJRCQwkOiC +Hvz/SItUJDhIiwpIi0QkKEiLXCRASInOSItMJCD/1pCQSItEJDDoehz8/0iLbCRISIPEUMNIiUQk +CEiJXCQQSIlMJBiQ6NvXAABIi0QkCEiLXCQQSItMJBjpZ/7//8zMzMzMzMxJO2YQD4ZkBAAASIPs +YEiJbCRYSI1sJFhIiUQkaIQAMfZIh7BwFgAASIuw4CYAAEiJdCRISIuI6CYAAEiJTCQwSIuQ8CYA +AEiJVCQ4Mdsx/0UxwEUxyesISP/DDx9EAABIOcsPjYUBAABIiVwkQESJTCQkTIlEJChAiHwkI0yL +FN5MiVQkUOkpAgAAQIT/dGBJOcgPg8wDAABOjRzGgz3cLgoAAHUGTokUxusLTInfTYnR6Cm0AABI +ifBIictIidFMicfouAQAAEiLRCRoSItMJDBIi1QkOEiLXCRASIt0JEgPtnwkI0yLRCQoRItMJCRJ +/8DpYP///0nHAgAAAABEiehBuwUAAADwRQ+xWkBBD5TCRYTSdTTo5AcAAEiLTCQwSItUJDhIi1wk +QEiLdCRITItEJChEi0wkJEG7BQAAAEyLZCRoQb0EAAAAQf/BTIngvwEAAADp/f7//02LWjhNiVoI +STnID4PxAgAASo08xoM9CS4KAAB1Bk6JFMbrCE2J0ehZswAASInwSInLSInRTInH6OgDAAC4CQAA +AEiLVCRQvgEAAADwD7FyQA+UwoTSdQroSAcAAL4BAAAATItUJChNjUIBSItEJGhIi0wkMEiLVCQ4 +SItcJEBIi3QkSL8BAAAARItMJCTpav7//0yJw+sDSf/ASTnIfTEPH0QAAA+DowAAAEqNPMaDPW8t +CgAAdQpKxwTGAAAAAOvWSYnaMdvoWbIAAEyJ0+vHQffZRYnI8EQPwYj8JgAA8EQPwYD4JgAASDnT +d1hIiZjoJgAASImQ8CYAAIM9Ii0KAABmkHUJSImw4CYAAOsMSI244CYAAOgpsgAAkEiF23ULMclI +h4hoFgAA6w5Iiw5Ii0kISIeIaBYAAEiLbCRYSIPEYJDDSInZ6Le1AABMicDoL7UAAEyJ4EWLWkBB +g/sDd0JmkEGD+wF3DkWF2w+EUQEAAOm3/f//QYP7Ag+EAgEAAEmJxESJ2EG9BAAAAPBFD7FqQEEP +lMNFhNt0uOn6/f//ZpBBg/sFD4bMAAAAQYP7BnR8QYP7CHcvSYnERInYQb0JAAAA8EUPsWpAQQ+U +w0WE2w+FJv7//0G9BAAAAA8fRAAA6Wz///9Bg/sJD4SRAAAA6KwFAABIi0wkMEiLVCQ4SItcJEBI +i3QkSA+2fCQjTItEJChEi0wkJEyLVCRQTItkJGhBvQQAAADpJf///+hPzgAARQ9X/2RMizQl+P// +/0iLTCQwSItUJDhIi1wkQEiLdCRID7Z8JCNMi0QkKESLTCQkTItUJFBMi2QkaEG9BAAAAOnb/v// +QYP7BHVEDx9AAOgbBQAASItMJDBIi1QkOEiLXCRASIt0JEgPtnwkI0yLRCQoRItMJCRMi1QkUEyL +ZCRoQb0EAAAA6ZT+//8PHwDo2wQAAEiLTCQwSItUJDhIi1wkQEiLdCRID7Z8JCNMi0QkKESLTCQk +TItUJFBMi2QkaEG9BAAAAOlU/v//TInA6HuzAABMicDoc7MAAJBIiUQkCOgIrAAASItEJAgPHwDp +e/v//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhq8AAABIg+wgSIlsJBhIjWwkGJBI +jQVQKQoA6HsX/P9Iiw3kPgcASIsV5T4HADHASLv/////////fzH26wNI/8BIOdB9QkiLPMFIhf90 +70yLh2gWAAAPH0QAAE2FwHQLSTnYfQZMicNIif5Mi4dwFgAATYXAdMcPH0AASTnYfb5MicNIif7r +tkiJdCQQSIlcJAiQkEiNBdAoCgDo2xj8/0iLRCQISItcJBBIi2wkGEiDxCDD6CKrAABmkOk7//// +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GHgEAAEiD7CBIiWwkGEiNbCQYSIl8JEBI +iVwkMEiJRCQoSDn7fx/oTwMAAEiLTCQwSItEJEBIOcFIi0QkKEiJy0iLfCRAD4bLAAAASIsU+EiL +UghIhdJ/HkiJVCQQ6BgDAABIi0QkKEiLVCQQSItcJDBIi3wkQEiLNPjrA0yJx0iF/34+TI1H/0nB ++AJMOcN2ek6LDMAPH0QAAEk5UQh+Ikg5+3ZbTI0U+IM9SikKAAB1BkyJDPjrxEyJ1+iargAA67pI +Oft2LkiLDPhIjRT4SDnxdBeDPR8pCgAAdQZIiTT46whIidfoL64AAEiLbCQYSIPEIMNIifhIidno +WrEAAEiJ+EiJ2ehPsQAATInASInZ6ESxAABIifhIidnoObEAAJBIiUQkCEiJXCQQSIlMJBhIiXwk +IA8fQADou6kAAEiLRCQISItcJBBIi0wkGEiLfCQg6aL+///MzEk7ZhAPhtQBAABIg+wgSIlsJBhI +jWwkGEiJfCRASIlcJDBIiUQkKEg5+38f6O8BAABIi0wkMEiLRCRASDnBSItEJChIictIi3wkQA+G +gQEAAEiLFPhIi1IISIXSfx5IiVQkEOi4AQAASItEJChIi1QkEEiLXCQwSIt8JEBIizT46wNMid9J +ifhIwecCTI1PAUw5yw+OvAAAAA+GKAEAAEyLVPgITYtSCEyNXwJMOdt+Gw+GBQEAAEyLZPgQTYtk +JAhNOdR9Bk2J4k2J2UyNXwNMOdt+Qw+G1gAAAEyLZPgYTYtkJAhMjW8ETDnrfh0PhrIAAABMi3z4 +IE2LfwgPHwBNOed9Bk2J/E2J60054n8QTYnUTYnL6whNidRNictmkEk51H0wTDnbdnFOiwzYTDnD +dl1KjTzAgz1iJwoAAGaQdQlOiQzA6Tf////osKwAAOkt////TDnDditKiwzASo08wEg58XQUgz0y +JwoAAHUGSok0wOsF6EWsAABIi2wkGEiDxCDDTInASInZ6HCvAABMicBIidnoZa8AAEyJ2EiJ2eha +rwAATInoSInZ6E+vAABMidhIidnoRK8AAEyJ2EiJ2eg5rwAATInISInZ6C6vAABIifhIidnoI68A +AJBIiUQkCEiJXCQQSIlMJBhIiXwkIOippwAASItEJAhIi1wkEEiLTCQYSIt8JCDp8P3//8zMzMzM +zMzMzMzMzMzMzMxJO2YQdiBIg+wYSIlsJBBIjWwkEEiNBRa3AQC7FQAAAOg7Pf7/kOhVpwAA69PM +zMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GqwAAAEiD7BhIiWwkEEiNbCQQSIM9aLkHAAB0E0iDPU65 +BwAAdRWAPf64BwAAdQwxwEiLbCQQSIPEGMOQkEiNBda4BwDosRL8/0iLDTK5BwBIhcl0E0iDPRW5 +BwAAdSOAPcW4BwAAdRqQkEiNBam4BwDoZBT8/zHASItsJBBIg8QYw5CQSIlMJAgx0kiJFfC4BwCQ +kEiNBX+4BwDoOhT8/0iLRCQISItsJBBIg8QYw+iGpgAA6UH////MSTtmEA+GkQAAAEiD7BhIiWwk +EEiNbCQQhABIi4gwFgAASMeAMBYAAAAAAABIhcl0YEiJTCQIkJBIjQUhuAcAkOj7Efz/SItMJAhI +icqQSMcBAAAAAEiDPVu4BwAAdQlIiRVSuAcA6wpIiw1RuAcASIkRSIkVR7gHAJCQSI0F3rcHAOiZ +E/z/SItsJBBIg8QYw0iLbCQQSIPEGMNIiUQkCOjbpQAASItEJAjpUf///8zMzMzMzMzMzMzMzMzM +zMzMSTtmEA+GrgAAAEiD7FhIiWwkUEiNbCRQSIlcJGiIRCRLSIlMJHBIibQkgAAAAEiJfCR46GgG +AACAPXG3BwAAkHUJgLigAgAAAHRcSItUJGhIhdJ+EkyNYgFNifVMOajAAAAASQ9E1IlcJExIic8P +tnQkS0mJ0EyLTCRwTItUJHhMi5wkgAAAAInZSInDMcDobAAAAItEJEzogwYAAEiLbCRQSIPEWMOJ +2OhyBgAASItsJFBIg8RYw4hEJAhIiVwkEEiJTCQYSIl8JCBIiXQkKOjrpAAAD7ZEJAhIi1wkEEiL +TCQYSIt8JCBIi3QkKOkN////zMzMzMzMzMzMzMzMzEk7ZhAPhvwDAABIg+xYSIlsJFBIjWwkUEiJ +fCR4QIi0JIAAAABMiZQkmAAAAEyJhCSIAAAASIlcJEhMiYwkkAAAAEiLF0yNWDRMiVwkMEiJ0EiF +0nQTTItiEEmBxBgE//9J99xNOdx9E4nL6DAGAACQSInCSItMJHhIiQFIiVQkOOjapQAARQ9X/2RM +izQl+P///0iLBCRIwegGSItMJDhIi1EISIlBCEgp0EiLtCSIAAAASIX2fA5Ii5QkmAAAAESNUgHr +C0iLlCSYAAAASYnSQYD6A3YGQboDAAAATItBEEWJ0UHB4gZED7acJIAAAABFCdpmDx+EAAAAAAAP +HwBJgfjo+wAAD4PqAgAARoiUARgEAABMi1EQSf/CTIlREEGA+QN1BzHb6XMCAAAx25BMi0kQ6xhJ +icKDyIBCiIQJGAQAAEnB6gdJ/8FMidBIPYAAAAByD0mB+ej7AABy15DpDQIAAEmB+ej7AAAPg/MB +AABCiIQJGAQAAEn/wUyJSRBMi4wkkAAAADHA6xNGiJQZGAQAAE2NUwFMiVEQSP/ASDnCfiBNixTB +kEyLWRDpfAEAAEmB++j7AABy0Q8fQADphgEAAEiF9nULSItREDHA6fQAAAB+K0yJRCQoSIlcJEBI +jVkYSItEJEi/gAAAAEiJ+eiJAgAASItUJDhMi0IQ609Ii0kQTCnBSItUJDBIOdF/FUiF23QGSI1B +/ogDSItsJFBIg8RYw0iNBU/DAQC7HQAAAOhnOP7/SInGg8iAQoiEAhgEAABIwe4HSf/ASInwSD2A +AAAAchIPH4AAAAAASYH46PsAAHLQ6zpJgfjo+wAAcyJCiIQCGAQAAEmNcAFIiXIQSInRSItcJEBM +i0QkKOls////TInAuej7AABmkOh7qQAATInAuej7AADobqkAAEiJxoPIgIiEERgEAABIwe4HSP/C +SInwSD2AAAAAcg9Igfro+wAActjrLQ8fQABIgfro+wAAcxOIhBEYBAAASP/CSIlREOkG////SInQ +uej7AADoF6kAAEiJ0Lno+wAA6AqpAABNidRBg8qARoiUGRgEAABJwewHSf/DTYniSYH6gAAAAA+C +d/7//w8fQABJgfvo+wAAcs3rDUyJ2Lno+wAA6MioAABMidi56PsAAOi7qAAATInIuej7AADorqgA +AEyJyLno+wAA6KGoAABJidmDy4BCiJwRGAQAAEnB6QdJ/8JMictmDx+EAAAAAABIgfuAAAAAcgtJ +gfro+wAAcs3rNkmB+uj7AABzIEKInBEYBAAATY1KAUyJSRBKjRwRSI2bGAQAAOlS/f//TInQuej7 +AADoNqgAAEyJ0Lno+wAA6CmoAABMicC56PsAAJDoG6gAAJBIiUQkCEiJXCQQiUwkGEiJfCQgQIh0 +JChMiUQkMEyJTCQ4TIlUJEBMiVwkSOiJoAAASItEJAhIi1wkEItMJBhIi3wkIA+2dCQoTItEJDBM +i0wkOEyLVCRATItcJEjpmPv//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhsEAAABIg+w4 +SIlsJDBIjWwkMEiJfCRYSIlcJEhIi4DAAAAASIlEJChNifBJOcB0N0iFwHUEMcnrSUiJ2kiJ80mJ +yEiJ0UmJ+UyJx0yJzuinQgAASItcJEhIi3wkWEiJwUiLRCQo6xtIjUYB6ApBAABIi1wkSEiLfCRY +SInBSItEJChIjXH/SIXJSA9PzkiFyX4NSIO4mAAAAAF1A0j/yUg5z3IYSI0F37EHAOjqAwAAicBI +i2wkMEiDxDjDSIn66FanAACQSIlEJAhIiVwkEEiJTCQYSIl8JCBIiXQkKOhXnwAASItEJAhIi1wk +EEiLTCQYSIt8JCBIi3QkKOn5/v//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2b0iD7BhI +iWwkEEiNbCQQSYtWMJD/gggBAABJi0YwTInyhAJIi5DQAAAASIXSdBOLGkiNijAWAABIi2wkEEiD +xBjDSIlEJAiQkEiNBXOxCADohgr8/0iLRCQIu/////9IjQ1lsQgASItsJBBIg8QYw+imngAA64TM +zMzMSTtmEHZVSIPsEEiJbCQISI1sJAiD+P91DpCQSI0FJrEIAOgZDPz/SYtGMIuICAEAAI1R/4mQ +CAEAAIP5AXUSQYC+sQAAAAB0CEnHRhDe+v//SItsJAhIg8QQw4lEJAiQ6DueAACLRCQI65XMzMzM +zMzMzMzMzMzMzMzMzMzMzMxJO2YQD4ZBAgAASIPsMEiJbCQoSI1sJChIiw3prwcAkEiFyXQQSYtW +MEg5isAAAAAPlcHrBbkBAAAAiVwkQIhMJBeEyXQhSIlEJDiQkEiNBayvBwDohwn8/0iLRCQ4D7ZM +JBeLXCRASIXAdDBIicKQSMcAAAAAAEiDPdmvBwAAkHUJSIkVz68HAOsKSIs1zq8HAEiJFkiJFcSv +BwBIixWtrwcASIXSdA9IidBIizJIiTWbrwcA6xq4AAABAEiNHT0yCgDoWJb8/0iFwA+EaAEAAEiJ +RCQYSIlEJCAx0kiJEEjHQBAAAAAA6NOeAABFD1f/ZEyLNCX4////SIsEJEjB6AZIi0wkIEiJQQiQ +SItREEiB+uj7AAAPgw8BAADGhBEYBAAAQUiLURBI/8JIiVEQi1wkQEhj2+sXSIneg8uAiJwRGAQA +AEjB7gdI/8JIifNIgfuAAAAAchUPH4AAAAAASIH66PsAAHLQ6bEAAABIgfro+wAAD4OXAAAAiJwR +GAQAAEj/wkiJURCQ6xdIicODyICIhBEYBAAASMHrB0j/wkiJ2Eg9gAAAAHILSIH66PsAAHLY609I +gfro+wAAczWIhBEYBAAASP/CSIlREA+2TCQXhMl0D5CQSI0FIa4HAJDo2wn8/0iLRCQYSItsJChI +g8Qww0iJ0Lno+wAADx9AAOh7owAASInQuej7AADobqMAAEiJ0Lno+wAA6GGjAABIidC56PsAAOhU +owAASInQuej7AADoR6MAAEiNBcepAQC7FAAAAOi2Mf7/kEiJRCQIiVwkEOjHmwAASItEJAiLXCQQ +6Zn9///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GhwEAAEiD7EhIiWwkQEiNbCRASIlc +JFgPHwBIhckPhPUAAABIiUwkYEiJfCRoSIlcJFhIiUQkUEjB4QNIidgx2+gVnQAASIlEJDBIi1wk +WEiLTCRgSIt8JGhIicZIi0QkUOh0AQAAhcAPhaAAAABIi0QkUIQAkA8fQADo2wb8/0iLRCRQSItc +JFhIi0wkYEiLfCRoSIt0JDBmkOg7AQAAhcB1TUiLRCRQ/0AISItcJGDoxQEAAEiLTCQwSIlICEiL +VCRQi3IIiXAQSIt0JGBIiXAYkA8fgAAAAABIgf6AAAAAD4eWAAAASIt8JFgx2+tAiUQkLJCQSItE +JFDoOgj8/4tEJCxIi2wkQEiDxEjDSItsJEBIg8RIwzHASItsJEBIg8RIw0yLBN9MiUTYIEj/w0g5 +3n/vSIlEJDhIgeH/HwAASIt0yiBIiTBIjQzKSI1JIEiJw0iJyA8fAOjboPv/kJBIi0QkUOjPB/z/ +SItMJDiLQRBIi2wkQEiDxEjDSInxuoAAAADosKEAAJBIiUQkCEiJXCQQSIlMJBhIiXwkIOj2mQAA +SItEJAhIi1wkEEiLTCQYSIt8JCBmkOk7/v//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSIPs +GEiJbCQQSI1sJBBIiVwkKIQASIn3SIHm/x8AAEiLdPAg6wNIizZIhfZ0H0g5fgh18kyLRhhMOcF1 +6Q8fAEmB+IAAAAB3OTHA6xMxwEiLbCQQSIPEGMNI/8APH0AATDnAfRBMi0zGIEyLFMNNOcp05uux +i0YQSItsJBBIg8QYw0yJwbqAAAAA6NGgAACQzMzMzMzMzMzMzMzMzMzMzEk7ZhB2K0iD7BhIiWwk +EEiNbCQQhABIg8AQSMHjA0iDwyjoOQAAAEiLbCQQSIPEGMNIiUQkCEiJXCQQDx9EAADo25gAAEiL +RCQISItcJBDrr8zMzMzMzMzMzMzMzMzMzEk7ZhAPht4AAABIg+wgSIlsJBhIjWwkGEiNUwdIg+L4 +SIM4AHQaSItwCEgB1kiB/vj/AAB2VmYPH4QAAAAAAJBIgfr4/wAAD4eJAAAASIlEJChIiVQkELgA +AAEASI0dPS0KAOhYkfz/SIXAdFhIi1QkKEiLGkiJGJBIiQJIx0IIAAAAAEiJ0EiLVCQQSIsYhANI +i3AISIH++P8AAHMaSI0MFkiJSAhIjQQzSI1ACEiLbCQYSIPEIMNIifC5+P8AAOh7nwAASI0F26UB +ALsUAAAA6Mot/v9IjQX7qQEAuxYAAADouS3+/5BIiUQkCEiJXCQQ6MmXAABIi0QkCEiLXCQQ6fr+ +///MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2T0iD7DhIiWwkMEiNbCQwSMdEJCgAAAAA +SYtWMEiLkugAAABIiVQkKLgFAAAASMfD/////0iNTCQovwEAAABIif7olfH//0iLbCQwSIPEOMPo +RpcAAOukzMzMzEk7ZhAPhpkAAABIg+xASIlsJDhIjWwkOEmLVjCQ/4IIAQAASYtWMEiJVCQwTYnw +QYQATIuC0AAAAEyJRCQokEiJgtAAAAC4BgAAAEjHw/////8xyTH/SIn+6CLx//9Ii1QkKEyLRCQw +SYmQ0AAAAJBBi5AIAQAARI1K/0WJiAgBAACD+gF1EkGAvrEAAAAAdAhJx0YQ3vr//0iLbCQ4SIPE +QMNIiUQkCOiTlgAASItEJAjpSf///8zMzMzMzMzMzEk7ZhB2T0iD7BhIiWwkEEiNbCQQSYtOMEiL +idAAAACEAYC5OBYAAAB1GcaBOBYAAAFEDxG5QBYAAEiLbCQQSIPEGMNIjQUArAEAuxgAAADoDCz+ +/5DoJpYAAOukzMzMzEk7ZhB2aUiD7DhIiWwkMEiNbCQwSYtWMEiLktAAAACEAoC6OBYAAAB0O0iD +ukAWAAAAdSpIiUQkQEiJVCQouAsAAAC7AQAAADHJMf9Iif7oDPD//0iLRCRASItUJChIAYJAFgAA +SItsJDBIg8Q4w0iJRCQI6KeVAABIi0QkCGaQ6Xv////MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMxJO2YQD4aVAAAASIPsSEiJbCRASI1sJEBJi1YwSIuS0AAAAIQCgLo4FgAAAHRfSIO6QBYAAAB0 +REiJVCQ4SI1MJChEDxE5TIuCQBYAAEyJRCQoTIuCSBYAAEyJRCQwuAwAAABIx8P/////vwIAAABI +if7oSe///0iLVCQ4xoI4FgAAAEiLbCRASIPESMNIjQXQrAEAuxkAAADowir+/5CQ6NuUAADpVv// +/8zMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4alAAAASIPsQEiJbCQ4SI1sJDhIiUQkSEiJXCRQ +SMeA2AAAAAAAAABJi04wSIuJ0AAAAEiJiOAAAABIjQVaGAEA6HUf/P9Ii0wkUEj/wUiJCEiJw7kB +AAAASInPSI0FqKYHAOiz+P//SI1MJChEDxE5SItUJEhIi5KYAAAASIlUJCiJwkiJVCQwuA0AAAC7 +AgAAAEiJ30iJ3maQ6Fvu//9Ii2wkOEiDxEDDSIlEJAhIiVwkEOgClAAASItEJAhIi1wkEOkz//// +zMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhkoBAABIg+xgSIlsJFhIjWwkWEmLVjBIi5LAAAAATItC +ME2LgNAAAABI/4LYAAAATYnBQYQASYO4iBYAAAB0e0jHRCRAAAAAAEyNTCRIRQ8ROUyLipgAAABM +iUwkQEiLktgAAABIiVQkSEmLgIgWAABmDx+EAAAAAACQSIP4BA+DvwAAAEiNFdelCABIixTCSIlU +JFC4KQAAAEjHw/////9IjUwkQL8DAAAASIn+6Gjt///pggAAAA8fAEw5iuAAAAB1NUjHRCQoAAAA +AEiLkpgAAABIiVQkKLgmAAAASMfD/////0iNTCQovwEAAABIif7oJO3//+tBTImK4AAAAEiNTCQw +RA8ROUyLgpgAAABMiUQkMEiLktgAAABIiVQkOLgOAAAASMfD/////78CAAAASIn+6OHs//9Ii2wk +WEiDxGDDuQQAAADo7ZkAAJDoh5IAAOmi/v//zMxJO2YQdkBIg+wwSIlsJChIjWwkKEmLVjBIi5LQ +AAAASYmW4AAAALgRAAAAuwEAAAAxyTH/SIn+6ITs//9Ii2wkKEiDxDDD6DWSAADrs8zMzMzMzMzM +zMzMzMzMzMzMzMxJO2YQdlZIg+w4SIlsJDBIjWwkMKiAdCtIiVwkKIhEJEC4JAAAAEjHw/////8x +yTH/SIn+6Cfs//8PtkQkQEiLXCQog+B/Mckx/0iJ/ugO7P//SItsJDBIg8Q4w4hEJAhIiVwkEOi2 +kQAAD7ZEJAhIi1wkEOuKzMzMzMzMzMzMzEk7ZhAPhpwAAABIg+xISIlsJEBIjWwkQEmLVjBIi5LQ +AAAASP+A2AAAAEg5kOAAAAB1L0jHRCQoAAAAAEiLkJgAAABIiVQkKLgnAAAASI1MJCi/AQAAAEiJ +/uiB6///kOs6SImQ4AAAAEiNTCQwRA8ROUiLkJgAAABIiVQkMEiLkNgAAABIiVQkOLgVAAAAvwIA +AABIif7oROv//0iLbCRASIPESMNIiUQkCEiJXCQQ6OuQAABIi0QkCEiLXCQQkOk7////zMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEHYuSIPsMEiJbCQoSI1sJCi4HAAAALsBAAAAMckx/0iJ +/ujW6v//SItsJChIg8Qww+iHkAAA68XMzMzMzEk7ZhAPhqUAAABIg+xISIlsJEBIjWwkQEiFwHQO +Dx8ASDkFWaIHAH4CMcBJi1YwSIuSwAAAAEj/gtgAAABMi0IwTYuA0AAAAEyJguAAAABIx0QkKAAA +AABMjUQkMEUPEThMi4KYAAAATIlEJChIi5LYAAAASIlUJDBIwegGSIlEJDi4HQAAAEjHw/////9I +jUwkKL8DAAAASIn+Dx9EAADoG+r//0iLbCRASIPESMNIiUQkCOjHjwAASItEJAhmkOk7////zMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GmQAAAEiD7EBIiWwkOEiNbCQ4SYtWMJD/gggB +AABJi1YwSIlUJDBNifBBhABMi4LQAAAATIlEJCiQSImC0AAAALgeAAAASMfD/////zHJMf9Iif7o +gun//0iLVCQoTItEJDBJiZDQAAAAkEGLkAgBAABEjUr/RYmICAEAAIP6AXUSQYC+sQAAAAB0CEnH +RhDe+v//SItsJDhIg8RAw0iJRCQI6POOAABIi0QkCOlJ////zMzMzMzMzMzMSTtmEA+GhQAAAEiD +7DhIiWwkMEiNbCQwSIsVIRMKAJBIg/r/dTJIx0QkKAAAAABIx0QkKAAAAAC4IgAAAEjHw/////9I +jUwkKL8BAAAASIn+6Mro///rLUjHRCQoAAAAAEiJVCQouCIAAABIx8P/////SI1MJCi/AQAAAEiJ +/pDom+j//0iLbCQwSIPEOMPoTI4AAOln////zMzMzMzMzEyNZCTYTTtmEA+G1wEAAEiB7KgAAABI +iawkoAAAAEiNrCSgAAAASImcJLgAAABIiUwkQEjHRCRIAAAAAEiNfCRQSI1/0EiJbCTwSI1sJPDo +3JcAAEiLbQBIi3Ao6xZIi3wkOEiLdyhIi0wkQEiLnCS4AAAASIX2D4QLAQAASIl0JDhIi34YSIX/ +dSlIx0QkWAAAAABEDxF8JEhEDxG8JIgAAABIx4QkmAAAAAAAAADpogAAAEiJfCQwSIsHSIlEJFjo +mrT//0iFwA+EyQAAAEiJRCRISIlcJFBMi0QkOEGDOAB1BUUxyesFkE2NSEhMiYwkiAAAAESLSAxN +Y9FMiZQkkAAAAEjHhCSYAAAAAAAAAEGB+QAAAIB1L0iJ2b8BAAAASIt0JDBIicNIjUQkSOjsHgAA +SImEJJAAAABIiZwkmAAAAEyLRCQ4SItMJEBIi5wkuAAAAEiLdCRYSIl0JGBIizNIjUQkSEiJ2kiJ +y//WhMAPheb+//9Ii6wkoAAAAEiBxKgAAADDSIusJKAAAABIgcSoAAAAw0iLRCRYSIlEJCjo2Tr+ +/0iNBVWuAQC7HQAAAOjoQ/7/SItEJCgPHwDoO0L+/+g2Pf7/6DE7/v9IjQVeiAEAuwoAAAAPH0QA +AOg7Iv7/kEiJRCQISIlcJBBIiUwkGOhGjAAASItEJAhIi1wkEEiLTCQY6fL9///MzMzMzMzMzMzM +zMzMzMzMzMxMjaQkGP3//007ZhAPhtYZAABIgexoAwAASImsJGADAABIjawkYAMAAEiF9n4JTYXS +D4WfGQAATInxSDn5dRFIi1EwSDmKwAAAAA+EdRkAAIsV+ukGAEiLSTAPtokpAQAAweoChMkPRdFI +g/j/dSZIg/v/dSBIi09wSIXJdAlIi194RTHk6xdIi19ASItPOEyLZ1DrCUiJ2UUx5EiJw0jHhCQI +AwAAAAAAAEiJ+EiNvCQQAwAASI1/0A8fgAAAAABIiWwk8EiNbCTw6EGVAABIi20ASImcJBgDAABI +iYwkMAMAAEyLqFABAABMi7hYAQAATYXAdAQx/+sHTYXSQA+Ux0iJtCSYAwAATImUJLADAABIiZwk +EAEAAEiJjCQIAQAAQIh8JC9MiZwkAAMAAEyJjCSoAwAATImEJKADAABIiYQk+AIAAEyJvCSYAAAA +TImsJLgCAABMiaQksAIAAIlUJDxIhdt1H0iLCUiJjCQYAwAASIuMJAgBAABIjVEISImUJDADAABI +i4QkGAMAAOitsf//SIXAD4SJAAAASImEJAgDAABIiZwkEAMAAEiNvCRIAQAAZg8fhAAAAAAADx8A +SIlsJPBIjWwk8OgFlAAASIttAEiLtCSoAwAASIuMJPgCAABMi4wkEAEAAEyLlCQIAQAATIucJJgD +AABMi6wkuAIAAEyLvCSYAAAATIukJLACAAAxwDHSMdsx/w8fRAAA6SoBAABIi4wksAMAAEiFyXUJ +D7ZUJC+E0nRsSIuEJBgDAABIiYQkEAEAAOgQOP7/SI0FiJcBALsUAAAADx9AAOgbQf7/SIuEJBAB +AADobj/+/+hpOv7/6GQ4/v9Ii4Qk+AIAAEiLWAhIiwBIjYwkCAMAADH/6OYzAABIi4QksAMAAEiF +wOsDSIXJdRIxwEiLrCRgAwAASIHEaAMAAMNIjQVPhQEAuwoAAADoMR/+/0yJhCQIAwAATIuEJKAC +AABMiYQkEAMAAEyLhCQoAwAATImEJBgDAABIx4QkKAMAAAAAAABMi4QkOAMAAEyJhCQwAwAASMeE +JDgDAAAAAAAASMeEJFgDAAAAAAAAQYD8E0EPlMBMi6QkEAEAAE2J4UyJ0EUx5ESJwkyLlCQIAQAA +SDnGD47YCwAATImcJAABAABNieBMi6QkCAMAAEyLnCQQAwAAQYN8JBQAD4SxCwAATImcJPACAABM +iaQk6AIAAEUPtlwkKUUPtmQkKA8fRAAAQYD8BHUEQYPj/Uw5jCQYAwAAdRxMOZQkMAMAAHUSZpBM +OUl4dQpMOVFwdQRBg+P9SIlEJHBIiXwkWIhcJDBMiYQk4AIAAIhUJC5Ig7wkOAMAAAB0JUyLpCTo +AgAATIukJPACAABMiaQkmAIAAEyLpCToAgAA6fwBAABMi5QkcAMAAEEPuuICD4MgAQAATItJMEk5 +CQ+FCwEAAE2LicAAAABNhckPhPEAAABBgPwNdFlBgPwUdT5Ni0k4TImMJDADAABMi0kwTYuJwAAA +AE2LqVABAABNi7lYAQAAQYPj/UyLjCToAgAATIukJPACAADpyQAAAEyLjCToAgAATIukJPACAADp +tAAAAEmLQUBIiYQkGAMAAOiGrv//SImEJAgDAABIiZwkEAMAAEiLjCT4AgAASItRMEiLksAAAABE +D7ZYKUiLUjhIiZQkMAMAAEiLUTBIi5LAAAAATIuqUAEAAEyLulgBAAAPtlQkLkiLtCSoAwAASIt8 +JFhMi4Qk4AIAAEyLlCRwAwAASYnBSYncSItEJHAPtlwkMOsiTIuMJBABAADrCEyLjCQQAQAATIuM +JOgCAABMi6Qk8AIAAEyJvCSYAAAATImsJLgCAABMiaQkmAIAAEyJjCSoAgAARIhcJDFIi4wkGAMA +AEyJyEyJ40iNvCRIAQAA6CO5//9IY9BIA5QkMAMAAEiJlCQ4AwAASIPCCEiJlCQ4AwAASItEJHBI +i4wk+AIAAA+2VCQuD7ZcJDBIi7QkqAMAAEiLfCRYTIuEJOACAABMi4wkEAEAAEyLlCQIAQAARA+2 +XCQxTIukJKgCAABMi7wkmAAAAEyLrCS4AgAATIm8JJgAAABMiawkuAIAAEyJpCSoAgAAQfbDAXQi +SMeEJCgDAAAAAAAARTHbRTHbTImcJKACAABFMdvpvAIAAEH2wwJ0UEyLnCSwAwAATYXbdA5IhcB+ +Rk2F2w+F3xIAAEjHhCQoAwAAAAAAAEiJhCSwAAAAMcBIiYQkgAIAADHASImEJHgCAABIi4QksAAA +AOk+AgAATIucJLADAABIg7wkKAMAAAB1HkyLlCQ4AwAASYPC+EyJVCR4TYsSTImUJCgDAADrD0iJ +hCS4AAAAMcBIiUQkeEiLhCQoAwAA6Ees//9IhcB0J0iLjCT4AgAAD7ZUJC9Mi5QkqAIAAEyLnCSw +AwAADx9EAADpaQEAAEQPtkwkL0WEyXQ3SIuMJPgCAABMi1EwQYC6GAEAAAB0F0yLlCSoAgAAQYB6 +KBN1IkSJykUxyesdTIuUJKgCAADrEEiLjCT4AgAATIuUJKgCAABEicpIiYQk2AIAAEiJnCTQAgAA +TIucJLADAABNhdt1DkWEyXUJTYXbkOnjAAAATInQSIucJJgCAADoa7L//0iJhCTIAgAASImcJPgA +AABIi4wkKAMAAEiJjCTwAAAA6IYy/v9IjQUFsQEAuyIAAADolTv+/0iLhCTIAgAASIucJPgAAAAP +H0QAAOh7O/7/SI0F9oMBALsNAAAA6Go7/v9Ii4Qk8AAAAGaQ6Ls5/v/otjT+/+ixMv7/SIuMJPgC +AABIi1kISIsBSIt8JHhIjYwkCAMAAOgwLgAASIuUJLADAABIhdJIi4Qk2AIAAEiLjCT4AgAAD7ZU +JC9Ii5wk0AIAAEyLlCSoAgAATIucJLADAAAPhbcQAAAPtlQkLkiLtCSoAwAASIt8JFhMi4Qk4AIA +AEyLjCQQAQAATIuUJAgBAABMi6QkqAIAAEyLrCS4AgAATIu8JJgAAABIiYQkgAIAAEiJnCR4AgAA +SItEJHAPtlwkMEyLnCSAAgAASImEJKgAAABIi4QkeAIAAEiJhCSgAgAASIuEJKgAAABMi5QkOAMA +AEyJlCRAAwAATY1K+EyJjCRAAwAATDmMJDADAABzDE2NSvBMiYwkQAMAAEyJnCSAAgAATIuMJLAD +AABNhcl0CUQPtlQkL5DrEkQPtlQkL0WE0g+EyQAAAE2FyUyLnCQ4AwAATImcJEgDAAB0D0GBfCQM +AAAAgEEPlMPrA0Ux201jbCQMTImsJFADAABIx4QkWAMAAAAAAABFhNt0dE2FyUAPlcdIjYQkCAMA +AEyJ40iLjCSYAgAATInG6CgUAABIiYQkUAMAAEiJnCRYAwAASItEJHBIi4wk+AIAAA+2VCQuD7Zc +JDBIi7QkqAMAAEiLfCRYTIuMJLADAABED7ZUJC9Mi6QkqAIAAEyLvCSYAAAATIucJIACAABMi6wk +uAIAAE2J2EyLnCQYAwAATImcJCADAACE0nRFTIucJAgDAABFi2sQDx8ARYXtdBhNixtPjRwrTY1b +AUyJnCQgAwAATYXJ6w9Ix4QkIAMAAAAAAABNhclMi6wkuAIAAOsDTYXJdRBMi5wkoAMAAE2F2+mF +AAAASYsJSI2EJAgDAABIi5wkAAMAAEyJyv/RhMAPhFAEAABIi4QkoAMAAEiFwEiLRCRwSIuMJPgC +AAAPtlQkLg+2XCQwSIu0JKgDAABIi3wkWEyLhCSAAgAATIuMJLADAABED7ZUJC9Mi5wkoAMAAEyL +pCSoAgAATIusJLgCAABMi7wkmAAAAA+EgAEAAEyLjCQYAwAAZpBIhcB1E0yLrCRwAwAAQQ+65QFy +FoTS6wqE0kyLrCRwAwAAdQZNOQwkdQZNjXkB6xJNjWn/TYnPTYnpTIusJHADAABBgHwkKwN3H0iJ +hCSgAAAAMcBIiYQkUAIAAEiLhCSgAAAA6ecAAABMiUwkQE2NbCQrRYtMJCBPjUyNAE2NSQFNic1B +D7rlAg+DogAAAE2J5UEPuuQCD4OOAAAATImMJEgCAABMiXwkUOhxLv7/SI0FcpgBALsZAAAADx9E +AADoezf+/0iLhCSoAgAA6O42/v/oyTD+/+jELv7/SItEJHBIi4wk+AIAAA+2VCQuD7ZcJDBIi7Qk +qAMAAEiLfCRYTIuEJIACAABMi4wkSAIAAEQPtlQkL0yLnCSgAwAATIusJKgCAABMi3wkUEmDwQTr +A02J5U2LSRhNiexMi6wkcAMAAEyJjCRQAgAATItMJEBMi6wkUAIAAA8fhAAAAAAATYXtD4WXCgAA +TIuMJAABAADpBAwAAEWE0g+EewEAAEyLlCQYAwAASIXAfxdMi5wkcAMAAEEPuuMBchdMi5wkoAMA +AE05FCRzEYTSdQ1J/8rrCEyLnCSgAwAASImEJPgAAABBgHwkKwN3BzHS6e8AAABMidpNjVwkK0WL +TCQgT40Mi02NSQFNictBD7rjAg+DrgAAAE2J40EPuuQCDx9AAA+DlgAAAEyJjCRYAgAATIlUJEiI +XCQz6AQt/v9IjQUFlwEAuxkAAADoEzb+/0iLhCSoAgAA6IY1/v/oYS/+/5DoWy3+/0iLhCT4AAAA +SIuMJPgCAABIi5QkoAMAAA+2XCQzSIu0JKgDAABIi3wkWEyLhCSAAgAATIuMJFgCAABMi1QkSEyL +nCSoAgAATIusJLgCAABMi7wkmAAAAEmDwQTrA02J402LSRhJidNMi6QkqAIAAEyJykyLjCSwAwAA +kEiF0g+EKgUAAEiJlCRgAgAATI2cJBgBAABFDxE7TI2EJCgBAABFDxE4TI2EJDgBAABFDxE46c4C +AABEidJMjVABQYB8JCgED4WvAAAATYX/D46mAAAATInYT4tc/fhMiZwkkAAAAEyLnCQAAQAATYXb +D4WAAAAATYXJdXtIibwk4AAAAIhcJDKJ00iLjCSQAAAATInX6CYRAABIi4wk+AIAAA+2VCQvD7Zc +JDJIi7QkqAMAAEiLvCTgAAAATIuEJIACAABMi4wksAMAAEyLnCQAAQAATIukJKgCAABMi6wkuAIA +AEyLvCSYAAAASYnCSIuEJKADAABJ/8/rC0yJ2EyLnCQAAQAARQ+2ZCQoTYXAD4W58///6aoBAABI +i0QkcEiLrCRgAwAASIHEaAMAAMMPtlQkLw+20kiF0kgPRcdIi5QksAMAAEiF0nQZSDnGfhRIi5Qk +MAMAAEiLmYAAAABIOdN1EEiLrCRgAwAASIHEaAMAAMNIiZQkEAEAAEiJnCQIAQAASIlEJHBIi4GY +AAAASImEJOgAAADowir+/0iNBR94AQC7CgAAAOjRM/7/SIuEJOgAAADoxDH+/0iNBRZ5AQC7CwAA +AOizM/7/SIuEJBABAADoBjL+/0iNBZdwAQC7BQAAAOiVM/7/SIuEJAgBAADo6DH+/+jjLP7/Dx8A +6Nsq/v9Ii4Qk+AIAAEiLCEiJjCQQAQAASItACEiJhCQIAQAA6Dcq/v9IjQU5cwEAuwgAAADoRjP+ +/0iLhCQQAQAA6Jkx/v9IjQW3bgEAuwEAAADoKDP+/0iLhCQIAQAA6Hsx/v9IjQVgbwEAuwQAAADo +CjP+/0iLRCRwDx9EAADo+zD+/0iNBdNvAQC7BQAAAOjqMv7/SIuEJKgDAABmkOjbMP7/6DYs/v/o +MSr+/0iNBa2pAQC7IwAAAA8fRAAA6DsR/v9MidDpY/7//0iLlCQIAwAATIuEJMACAABBD7ZYAkyL +hCTQAAAATIuMJGACAABPjQSBTY1AEE1jEEwDEkyLpCSoAgAASIm8JNgAAACIXCQwTImUJPAAAABM +ieC5AgAAADH2SIucJJgCAABMidfo6q///4XAD4yYAQAASGPASD0AABAAD4O5BQAASI0UgEiJlCTQ +AAAATIuMJGACAABFi1SRDESJlCQgAQAATY0UkU2NUgJBD7YyQIi0JEABAABNjRSRTImUJMACAABM +i5wkcAMAAEEPuuMAckhIi5Qk2AAAAEiF0kAPlMdIjYQkGAEAAEiLnCSYAgAASIuMJPgCAABED7ZE +JDDosR0AAITAdRJIi7wk2AAAAA8fRAAA6ej+//9IjYQkGAEAAEiLnCSYAgAA6Cao//9IiYQkaAIA +AEiJXCRoSIuMJPAAAABIi4QkqAIAAEiLnCSYAgAAkOg7rP//SImEJJACAABIiZwkgAAAAIlMJDjo +Iij+/0iLhCRoAgAASItcJGjoMDH+/0iNBfluAQC7BgAAAA8fQADoGzH+/+h2KP7/6PEn/v9IjQWY +bAEAuwEAAAAPH0QAAOj7MP7/SIuEJJACAABIi5wkgAAAAOjmMP7/SI0FZ2wBALsBAAAA6NUw/v+L +TCQ4SGPB6Mku/v/oJCr+/w8fQADoGyj+/0iLjCTYAAAASI15Aen3/f//SIuEJPgAAABIi4wk+AIA +AEiLtCSoAwAATIuEJIACAABMi4wksAMAAEyLnCSgAwAATIukJKgCAABMi6wkuAIAAEyLvCSYAAAA +D7ZcJDBMi5Qk8AAAAEiLvCTYAAAASIm8JNgAAABMiZQk8AAAAEyJ0kyLlCRwAwAAQQ+64gBzCrsB +AAAA6YgAAABIhf8PlMJBD7Z0JChMieBBidhIi5wkmAIAAInXDx9EAADo+xsAAEiLjCT4AgAASIuU +JPAAAABIi7QkqAMAAEiLvCTYAAAATIuEJIACAABMi4wksAMAAEyLlCRwAwAATIucJKADAABMi6Qk +qAIAAEyLrCS4AgAATIu8JJgAAACJw0iLhCT4AAAAhNsPhAYDAABMieBIi5wkmAIAAOghpv//SImE +JHACAABIiVwkYEiLjCTwAAAASIuEJKgCAABIi5wkmAIAAOg3qv//SImEJIgCAABIiZwkiAAAAIlM +JDRIi1QkYEiD+g91PEi+cnVudGltZS5Ii7wkcAIAAEg5N3UtgX8IZ29wYXUkZoF/DG5pdRyAfw5j +dRa6BQAAAEiNPTxsAQDrCEiLvCRwAgAASIlUJGBIibwkcAIAAOjCJf7/SIuEJHACAABIi1wkYOjQ +Lv7/SI0FSmoBALsBAAAADx9AAOi7Lv7/6BYm/v9Ii4wkSAMAAEiLhCSoAgAASIucJJgCAADoeQUA +AOh0Jf7/SI0FLGoBALsCAAAA6IMu/v8PHwDo2yX+/+hWJf7/SI0F/WkBALsBAAAA6GUu/v9Ii4Qk +iAIAAEiLnCSIAAAA6FAu/v9IjQXRaQEAuwEAAAAPH0AA6Dsu/v+LRCQ0SGPA6C8s/v/oiiX+/0iL +hCQYAwAASIuMJKgCAABIixFIOcJzTUiJhCTwAAAASImUJMgAAABmkOjbJP7/SI0Fh2kBALsCAAAA +6Oot/v9Ii4Qk8AAAAEiLjCTIAAAASCnI6DIs/v/oLSX+/0iLjCSoAgAASIuEJPgCAABIi1AwSIXS +dBKDuvQAAAAAfglIOYLAAAAAdA2LVCQ8g/oCD4yfAAAASIuEJDgDAABIiYQk8AAAAEiLjCQwAwAA +SImMJMgAAABIi5QkGAMAAEiJlCTAAAAADx9EAADoOyT+/0iNBWBpAQC7BAAAAOhKLf7/SIuEJPAA +AABmkOibK/7/SI0FWGkBALsEAAAA6Cot/v9Ii4QkyAAAAGaQ6Hsr/v9IjQUwaQEAuwQAAADoCi3+ +/0iLhCTAAAAAZpDoWyv+/+hWJP7/6NEj/v/oTCb+/+hHJP7/SIuEJNgAAABIjXgBSIuEJPgAAABI +i4wk+AIAAEiLtCSoAwAATIuEJIACAABMi4wksAMAAEyLlCRwAwAATIucJKADAABMi6QkqAIAAEyL +rCS4AgAATIu8JJgAAABBD7ZcJChED7ZUJC/pNPf//7kAABAA6HB8AABMi6wkCAMAAEEPtlkCSY0U +kEiNUhBMYwpNA00ATY15AUyLpCSoAgAATImUJAABAABMibwk8AAAAEiJhCT4AAAAiFwkMLkCAAAA +TInPSI20JEgBAABMieBIi5wkmAIAAOisqf//hcAPjOkAAABIY8CQSD0AABAAD4OtAQAASI0UgEyL +hCRQAgAATY0MkE2NFJBNjVICQYA6FnVBRA+2VCQwQYD6CXQ1QYD6E3QvDx9AAEGA+g90JUyLnCSo +AwAATIusJKADAABIi4Qk+AAAAEyLlCQAAQAA6SX///9Mi5QkAAEAAE2F0n4gSf/KTIucJKgDAABM +i6wkoAMAAEiLhCT4AAAA6fj+//9Mi5wkqAMAAEiLhCT4AAAASTnDfilIPQAAEAAPg/YAAABMi6Qk +8AAAAEyLrCSgAwAATYlkxQBI/8Dpuv7//0yLrCSgAwAAZpDpq/7//0iLjCT4AgAAD7ZUJC5Ii7Qk +qAMAAEiLfCRYTIuEJIACAABED7ZUJC9Mi5wkoAMAAEyLpCSoAgAAD7ZcJDBIi4Qk+AAAAEyLvCTw +AAAATIuMJAABAABBgHwkKBZ1D4D7CXQKgPsTdAWA+w91I02FyX4KSf/J6xkPH0QAAEg5xn4PSD0A +ABAAczhNiTzDSP/AQQ+2XCQoSP/ITIusJLgCAABMi7wkmAAAAEyJjCQAAQAATIuMJLADAAAPHwDp +kfP//7kAABAA6FF6AAC5AAAQAOhHegAAuQAAEABmkOg7egAASI0FEXsBALsRAAAA6KoI/v9MieBI +i5wkmAIAAOi6oP//SImEJMgCAABIiZwkAAEAAOjlIP7/SI0F0qUBALsnAAAA6PQp/v9Ii4QkyAIA +AEiLnCQAAQAADx9AAOjbKf7/6DYj/v/oMSH+/0iNBdFsAQC7CQAAAA8fRAAA6DsI/v9IjQVDtAEA +uzkAAADoKgj+/0iNBVSzAQC7NwAAAOgZCP7/kEiJRCQQSIlcJBhIiUwkIEiJfCQoSIl0JDBMiUQk +OEyJTCRATIlUJEhMiVwkUOgGcgAASItEJBBIi1wkGEiLTCQgSIt8JChIi3QkMEyLRCQ4TItMJEBM +i1QkSEyLXCRQ6bTl///MzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPht0CAABIg+xYSIlsJFBIjWwk +UEiJRCRgSIlcJGiAeCsFdwQx9ut2SI1wK4t4IEiNNL5IjXYBSIn3D7rnAnNaSInHD7rgAnNMSIl8 +JDhIiXQkKEiJTCRwDx9AAOibH/7/SI0FnIkBALsZAAAA6Koo/v9Ii0QkOA8fRAAA6Bso/v/o9iH+ +/+jxH/7/SItMJHBIi3QkKJBIg8YESIt2KEiF9nQlSIl0JDBEDxF8JEBIjT1nAgAASIl8JEBIiUwk +SMZEJBYBMcDrFEiLbCRQSIPEWMNIi3QkMEyJwGaQSD2rAAAAD4PwAQAAD7Y8BkyNQAFMiUQkGA8f +gAAAAABAgP/8D4eqAAAAQID/+3VQkIB8JBYAdRvo4x7+/0iNBZ1jAQC7AgAAAOjyJ/7/6E0f/v/o +yB7+/0iNBXFjAQC7AQAAAOjXJ/7/6DIf/v9Mi0QkGOleAQAADx+EAAAAAABAgP/8D4XAAAAAkIB8 +JBYAdRvoiR7+/0iNBUNjAQC7AgAAAOiYJ/7/6PMe/v/obh7+/0iNBVtjAQC7AwAAAGaQ6Hsn/v/o +1h7+/0yLRCQY6QIBAABAgP/9D4TYAAAAZpBAgP/+dVqQgHwkFgB1HOgtHv7/SI0F52IBALsCAAAA +kOg7J/7/6JYe/v/oER7+/0iNBbtiAQC7AQAAAA8fRAAA6Bsn/v/odh7+/8ZEJBYBTItEJBjpvf7/ +/w8fgAAAAABAgP//D4SbAAAASIlEJCCQgHwkFgB1OECIfCQXDx9AAOi7Hf7/SI0FdWIBALsCAAAA +6Mom/v/oJR7+/0iLRCQgSIt0JDAPtnwkF0yLRCQYSYH4qwAAAHNXD7ZcMAFIi0wkQEiNVCRAifj/ +0UiLTCQgTI1BAusg6GUd/v9IjQUQYgEAuwEAAADodCb+/+jPHf7/TItEJBjGRCQWAA8fRAAA6RH+ +//9Ii2wkUEiDxFjDTInAuasAAADoRHYAALmrAAAA6Dp2AACQSIlEJAhIiVwkEEiJTCQY6MVuAABI +i0QkCEiLXCQQSItMJBjp8fz//8zMzMzMzMzMzMzMzMzMzMzMSTtmEHZgSIPsGEiJbCQQSI1sJBAP +tsBIA0IISIsAZpCA+whzHsHjA41TwPfagPpASBnS99uJ2UjT4Egh0EjT6Egh0EiJRCQI6JMc/v9I +i0QkCOgJJP7/6AQd/v9Ii2wkEEiDxBjDiEQkCIhcJAnojW0AAA+2RCQID7ZcJAnrgcxJO2YQD4Zc +AQAASIPsSEiJbCRASI1sJEBIiVwkWEiJTCRgi1MMTGPCQIT/D4TGAAAAgfoAAACAD4W6AAAASIlE +JFBIiXQkcEyJRCQgSIlMJDhIiVwkMEiJ2EiJyw8fQADou5v//0iD+xR0IEiD+xd1fUiJ2UiNHbWB +AQAPH0QAAOh7bPv/hMB1FetjSInZSI0dInsBAOhmbPv/hMB0UEiLTCRwSIXJdAQxwOsbSItMJFBI +i0koSInKSIPCIEiLMQ+2ConISInxSIsRSIt0JDBIORZ1OEiLUQiLMsHmA0hj9oTAdRhIi3EQSIPm ++OsOSIt0JCAx0usFTInGMdJIifBIidNIi2wkQEiDxEjDSInwSItcJDjoB5v//0iJRCQoSIlcJBjo +OBv+/0iNBQ98AQC7FQAAAOhHJP7/SItEJChIi1wkGOg4JP7/6JMd/v/ojhv+/0iNBVVyAQC7EAAA +AGaQ6JsC/v+QSIlEJAhIiVwkEEiJTCQYQIh8JCBIiXQkKJDom2wAAEiLRCQISItcJBBIi0wkGA+2 +fCQgSIt0JChmkOlb/v//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMTI2kJPj9//9NO2YQD4Y5 +AgAASIHsiAIAAEiJrCSAAgAASI2sJIACAABIiYQkkAIAAIicJJgCAABIibwkqAIAAEiJtCSwAgAA +SI18JEhmDx+EAAAAAABmDx+EAAAAAACQSIlsJPBIjWwk8OildQAASIttAL8gAAAASInISI1cJEhI +ifno+BwAAEjHhCRIAgAAAAAAAEiNlCRQAgAARA8ROkiNlCRgAgAARA8ROkiNlCRwAgAARA8ROkiN +vCRIAQAASI10JEhIiWwk8EiNbCTw6Ex4AABIi20AD7aUJJgCAABIi7QkkAIAAEyLhCSwAgAATIuM +JKgCAAAxwDHJ6wZI/8BJ/8FMiUwkQEiD+CAPjfUAAABIi7zESAEAAA8fRAAASIX/D4TfAAAATTnI +D47WAAAASIX2dBpmDx+EAAAAAABJgfkAABAAD4PwAAAASok8zoTSdKlIiUQkOIhMJCdMixUT/QYA +TIlUJDBNhdJ0LUyJw0wpy0iJ+EiNjCRIAgAA6HsZAABIi1QkQEyNDBBNjUn/SItUJDBIhdLrQEiJ +fCQo6PoY/v9IjQWtewEAuxYAAADoCSL+/0iLRCQoDx9AAOhbIP7/6FYb/v/oURn+/0iLRCQwSIXA +TItMJEAPlcEPtlQkJwnRSItEJDgPtpQkmAIAAEiLtCSQAgAATIuEJLACAADp9v7//4TJdB5Ix4Qk +SAIAAAAAAABIjYQkSAIAAOjbGgAATItMJEBMichIi6wkgAIAAEiBxIgCAADDTInIuQAAEADodnEA +AJBIiUQkCIhcJBBIiUwkGEiJfCQgSIl0JCjo+GkAAEiLRCQID7ZcJBBIi0wkGEiLfCQgSIt0JCjp +ev3//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GfgAAAEiD7EBIiWwkOEiNbCQ4SIlE +JEhIi4goAQAASIlMJCBIicjor5D//0iFwHUEMcnrJ0iJXCQwSIlEJChIi0wkSDH/MfZBifDoywwA +AEiLXCQwicFIi0QkKITJdBlIi1QkSEiDupgAAAABdApIi0wkIOgiAAAASItsJDhIg8RAw0iJRCQI +6C5pAABIi0QkCOlk////zMzMzEk7ZhAPhk8BAABIg+xoSIlsJGBIjWwkYEiJRCRwSIlcJHhIiYwk +gAAAAEiJXCRYSIlEJFDo55b//0iJRCRISIlcJDjoGBf+/0iNBddlAQC7CwAAAOgnIP7/SItEJEhI +i1wkOOgYIP7/6HMZ/v/obhf+/0iLjCSAAAAASI1R/0iLRCRQSDkISA9CykiLXCRYvwEAAADop5n/ +/0iJRCRASIlcJCiJTCQk6LQW/v9IjQVbWwEAuwEAAADowx/+/0iLRCRASItcJCjotB/+/0iNBTVb +AQC7AQAAAOijH/7/i0wkJEhjweiXHf7/6PIW/v9Ii0wkUEiLCUiLlCSAAAAAZpBIOcp2O0iJTCQw +6FEW/v9IjQX9WgEAuwIAAAAPH0QAAOhbH/7/SIuEJIAAAABIi0wkMEgpyOimHf7/6KEW/v+Q6BsW +/v/olhj+/+iRFv7/SItsJGBIg8Row0iJRCQISIlcJBBIiUwkGOizZwAASItEJAhIi1wkEEiLTCQY +Dx9AAOl7/v//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEHYfSIPsMEiJbCQoSI1sJCgx +9ujlAAAASItsJChIg8Qww0iJRCQISIlcJBBIiUwkGEiJfCQg6EJnAABIi0QkCEiLXCQQSItMJBhI +i3wkIOuszMzMzMzMzMzMzMzMSTtmEHZXSIPsMEiJbCQoSI1sJChIi1cwTIuCAAMAAJBNhcB0JEiL +gvgCAABIi7oIAwAATInDMckx9uhhAAAASItsJChIg8Qww74CAAAA6E0AAABIi2wkKEiDxDDDSIlE +JAhIiVwkEEiJTCQYSIl8JCDoqmYAAEiLRCQISItcJBBIi0wkGEiLfCQg6XH////MzMzMzMzMzMzM +zMzMzMzMzEyNpCTI/v//TTtmEA+GwAIAAEiB7LgBAABIiawksAEAAEiNrCSwAQAASIm8JNgBAABI +iUwkeIA9L+MJAAAPHwAPhP0AAABIi1cwSIXSD4TwAAAAg7o4AQAAAGYPH0QAAA+O3QAAAEiDf3AA +D4TSAAAATIuiQAEAAA8fhAAAAAAATYXkD4S6AAAASYM8JAAPhK8AAABIiYQkwAEAAEiJnCTIAQAA +SIm0JOABAAC5AQAAAIeKPAEAAEiLTzBIi7FAAQAAhAFIifhIjbwkgAAAAGYPH4QAAAAAAA8fhAAA +AAAASIlsJPBIjWwk8OhRcgAASIttAEiLSDBIi4lAAQAASMcBAAAAAEiLSDAx0oeRPAEAAEiNhCSA +AAAA6MISAABIi4QkwAEAAEiLTCR4SIucJMgBAABIi7Qk4AEAAEiLvCTYAQAAi5eQAAAAD7ryDIP6 +A3UMSItHeEiLX3BIg+b9SIlEJHBIiVwkaEiJdCRgSIk0JEUxwEG5ZAAAAEUx0kUx2zH26NnY//9I +hcB1QEiLVCRgD7riAHI1SIPKAUiJFCRIi0QkcEiLXCRoSItMJHhIi7wk2AEAADH2RTHAQblkAAAA +RTHSRTHb6JTY//9Ig/hkdRvo6RL+/0iNBTOJAQC7HwAAAOj4G/7/6FMT/v9Ii4Qk2AEAAOjG+v// +SIuMJNgBAABIi4kwAQAASIXJdBVIi1EITIsBSIXSfhlIiVQkWDHA6zJIi6wksAEAAEiBxLgBAADD +SIusJLABAABIgcS4AQAAw0yLjCSAAQAASYPBKE2JyEiJ0EiJRCRQTImEJIABAABJixBIiZQkiAEA +AEEPEEAIDxGEJJABAABBDxBAGA8RhCSgAQAASIuUJIgBAABIi5wkkAEAAEiLjCSYAQAASIu8JKAB +AABIi7QkqAEAAEiJ0OhpAAAASItUJFBI/8JMi0QkWEk50A+Pdv///+lh////SIlEJAhIiVwkEEiJ +TCQYSIl8JCBIiXQkKOiQYwAASItEJAhIi1wkEEiLTCQYSIt8JCBIi3QkKOny/P//zMzMzMzMzMzM +zMzMzMzMzMzMSTtmEA+GZQEAAEiD7FhIiWwkUEiNbCRQSIl8JDBIiUQkYEiJXCRoSIlMJHBIiXwk +eEiJtCSAAAAA6GIR/v9IjQX7gAEAuxwAAADocRr+/0iLRCQw6GcY/v9IjQVAVgEAuwMAAADoVhr+ +/+ixEf7/SItEJGBIiUQkSEiLTCRoSIlMJCgx0usTSItcJCBIjVMBSItEJEhIi0wkKEg5yn1QSIlU +JCBIiwzQSIlMJBhIicjoy4n//0iJRCRASIlcJDhIi0wkIEiFyQ+UwjH/if6J0eirBgAAhMB0rkiL +RCRASItcJDhIi0wkGOjTAAAA65hIg3wkaGR1G+ikEP7/SI0F7oYBALsfAAAA6LMZ/v/oDhH+/0iL +hCSAAAAA6GGJ//+QSIXAdQQxyeshSIlcJDhIiUQkQDHJMf+J/uhCBgAASItcJDiJwUiLRCRAhMl0 +F0iDfCR4AXQPSIuMJIAAAABmkOjb+P//SItsJFBIg8RYw0iJRCQISIlcJBBIiUwkGEiJfCQgSIl0 +JCjo02EAAEiLRCQISItcJBBIi0wkGEiLfCQgSIt0JCjpVf7//8zMzMzMzMzMzMzMzMzMzMzMzMzM +zEk7ZhAPhp8CAABIg8SASIlsJHhIjWwkeEiJhCSIAAAASImcJJAAAABIiUQkcEiJXCRoSImMJJgA +AADoYY///0iJRCRYSIlcJDBIi0wkcIB5KwN3B0mJyDHS63VIjVErRItBIEqNFIJIjVIBSYnQQQ+6 +4AJzVEmJyA+64QJzRUiJVCRQ6FgP/v9IjQVZeQEAuxkAAADoZxj+/0iLRCRwZpDo2xf+/+i2Ef7/ +6LEP/v9Ii0QkWEiLVCRQSItcJDBMi0QkcEiDwgTrA0mJyEiLUhhIhdJ0ZEiJVCRITInASItcJGi5 +AgAAAEiLvCSYAAAAMfboqpX//4XAfDBIY8APHwBIPQAAEAAPg5IBAABIjRSASIt0JEiLTJYMSItE +JHBIi1wkaOjYj///6wpIi1wkMEiLRCRYTItEJHBIiVwkMEiJRCRYkEiLjCSYAAAAvwEAAABMicBI +i1wkaOhikf//SIlEJGBIiVwkOIlMJCxIi1QkMEiD+g91OUi+cnVudGltZS5Ii3wkWEg5N3UqgX8I +Z29wYXUhZoF/DG5pdRmAfw5jdRO6BQAAAEiNPZBUAQDrBUiLfCRYSIlUJDBIiXwkWJDoGw7+/0iL +RCRYSItcJDDoLBf+/0iNBfVUAQC7BgAAAOgbF/7/6HYO/v/o8Q3+/0iNBZhSAQC7AQAAAA8fRAAA +6PsW/v9Ii0QkYEiLXCQ46OwW/v9IjQVtUgEAuwEAAADo2xb+/4tEJCxIY8DozxT+/+gqDv7/SItE +JHBIiwBIi4wkmAAAAEg5wXY6SIlEJEDoiw3+/0iNBTdSAQC7AgAAAOiaFv7/SIuEJJgAAABIi0wk +QEgpyOjlFP7/Dx9EAADo2w3+/+hWDf7/6NEP/v/ozA3+/0iLbCR4SIPsgMO5AAAQAOhYZgAAkEiJ +RCQISIlcJBBIiUwkGOjjXgAASItEJAhIi1wkEEiLTCQY6S/9///MzMzMzMzMzMzMzMzMzMxJO2YQ +D4a5AAAASIPsYEiJbCRYSI1sJFhIiVwkcEjHRCQIAAAAAEjHRCQQAAAAAEiNVCQYRA8ROkiNVCQo +RA8ROkiNVCQ4RA8ROkiNVCRIRA8ROkiNFaYAAABIiVQkEEiLVCRgSIlUJBhIjVQkaEiJVCQgTIny +SIlUJChIiUQkMEiJXCQ4SIlMJEBIiXwkSEiNRCQISIlEJFBIjUQkEEiJBCTo2VwAAEUPV/9kTIs0 +Jfj///9Ii0QkCEiLbCRYSIPEYMNIiUQkCEiJXCQQSIlMJBhIiXwkIOjkXQAASItEJAhIi1wkEEiL +TCQYSIt8JCDpC////8zMzMzMzMzMzMzMSTtmEHZmSIPsYEiJbCRYSI1sJFhIi0owSItaEEiLehhI +i3IgTItCKEyLYkBIi0IISIXJdi9MiWQkUEjHBCQAAAAASYnJRTHSRTHbMcnoTtH//0iLVCRQSIkC +SItsJFhIg8RgwzHA6LVkAACQ6K9cAADrjczMzMzMzMzMzMzMzMxJO2YQdlhIg+xYSIlsJFBIjWwk +UEiJTCRwSIX/djVIxwQkAAAAAEiJ3kmJyEmJ+UUx0kUx20jHw/////8xyUiJx0iJ2OjX0P//SIts +JFBIg8RYwzHASIn56ENkAACQSIlEJAhIiVwkEEiJTCQYSIl8JCBIiXQkKOjEXAAASItEJAhIi1wk +EEiLTCQYSIt8JCBIi3QkKOlm////zMzMzMzMSTtmEHZpSIPsIEiJbCQYSI1sJBhIiUQkKEiJXCQw +SYtWMIO69AAAAAB+KUiFyXQkSDmKwAAAAHQMSIuSyAAAAEg50XUPuAEAAABIi2wkGEiDxCDDifmJ +90SJxg8fRAAA6FsAAABIi2wkGEiDxCDDSIlEJAhIiVwkEEiJTCQYQIh8JCBAiHQkIUSIRCQi6A5c +AABIi0QkCEiLXCQQSItMJBgPtnwkIA+2dCQhRA+2RCQi6Ur////MzMzMzMzMzMzMSTtmEA+GgwEA +AEiD7DhIiWwkMEiNbCQwSIlEJEBIiVwkSIsV6LkGAE2LRjBFD7aAKQEAAMHqAkWEwEEPRdBmkIP6 +AQ+PNQEAAEiFwA+EIAEAAIhMJFBAgP8WdSIPH0AAQID+CXQYQID+E3QSQID+D3QMMcBIi2wkMEiD +xDjDZpDoW4n//0iD+w91Pki5cnVudGltZS5IOQh1L4F4CGdvcGF1JmaBeAxuaXUegHgOY3UYD7ZU +JFCE0nUPuAEAAABIi2wkMEiDxDjDSIlcJCBIiUQkKEiJBCRIiVwkCMZEJBAu6Dpb+/9FD1f/ZEyL +NCX4////SIN8JBgAfGlIi1QkIEiD+gh9BDHA6yBIi0QkKEiNHUNSAQC5CAAAAA8fRAAA6JtZ+/9I +i1QkIITAdC9Ig/oIfiVIuXJ1bnRpbWUuSItUJChIOQp1EQ+2SgiA+UFyCID5Wg+WwesNMcnrCbkB +AAAA6wIxyYnISItsJDBIg8Q4wzHASItsJDBIg8Q4w7gBAAAASItsJDBIg8Q4w0iJRCQISIlcJBCI +TCQYQIh8JBlAiHQkGug2WgAASItEJAhIi1wkEA+2TCQYD7Z8JBkPtnQkGuk4/v//zMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMSTtmEA+GCgIAAEiD7EBIiWwkOEiNbCQ4i4iQAAAAicoPuvEMg/kKcxmJ +y0jB4wRIjTXqzQYASIs8M0iLXDMIkOsMuwMAAABIjT3pTAEAg/kEdUAPtrCwAAAAQIT2dDGQQID+ +G3IRg/kEuxMAAABIjT1XZQEA6xxIweYESI0d/s8GAEiLPB5Ii1weCIP5BOsDg/kESIlEJEiJVCQU +SIlcJCBIiXwkMHQFg/kDdVJIg7ioAAAAAHRI6MR0AABFD1f/ZEyLNCX4////SIsEJEiLTCRISCuB +qAAAAEiJwki4QEdPP5r/TElIidNI9+pIwfoiSMH7P0gp2kiJyEiJ0esCMclIiUwkGEiLkJgAAABI +iVQkKOgpB/7/SI0FXlQBALsKAAAA6DgQ/v9Ii0QkKOguDv7/SI0FwEsBALsCAAAAZpDoGxD+/0iL +RCQwSItcJCDoDBD+/+hnB/7/i0QkFA+64AxzG+jYBv7/SI0FCk4BALsHAAAA6OcP/v/oQgf+/0iL +RCQYSIP4AXw26LIG/v9IjQVsSwEAuwIAAADowQ/+/0iLRCQY6LcN/v9IjQVBTwEAuwgAAADopg/+ +/+gBB/7/SItEJEhIg7joAAAAAHQc6G0G/v9IjQW7YAEAuxIAAACQ6HsP/v/o1gb+/+hRBv7/SI0F +SksBALsDAAAADx9EAADoWw/+/+i2Bv7/SItsJDhIg8RAw0iJRCQI6OJXAABIi0QkCOnY/f//zMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+G2wAAAEiD7GhIiWwkYEiNbCRgTIl0JDiLDc21BgBJ +i1YwSItcJDhIi1swD7abKQEAAMHpAoTbD0XLSIuSwAAAAEiF0nRVSDnQdFBIiUQkcEiJVCQwiUwk +LGaQ6JsF/v/oFgj+/+gRBv7/SItEJDDoR/3//5BIx8D/////SInDMclIi3wkMDH26K7w//9Ii0Qk +cItMJCxIi1QkMEQPEXwkQEjHRCRQAAAAAMdEJFgAAAAASI0dQgAAAEiJXCRASIlEJEhIiVQkUIlM +JFhIjUQkQOjFIv7/SItsJGBIg8Row0iJRCQI6NFWAABIi0QkCOkH////zMzMzMzMzEk7ZhAPhuUA +AABIg+w4SIlsJDBIjWwkMEiLShCLchiQSDlCCHUHuQEAAADrGEg5yHUHuQEAAADrDIuIkAAAAIP5 +Bg+UwYTJdR1IiUQkQIl0JCwx2+jnAgAAhMB0E4tEJCyD+AJ9CkiLbCQwSIPEOMPoiwT+/+gGB/7/ +6AEF/v9Ii0QkQOg3/P//SIt8JEBIi0cwSTlGMHQ5i5eQAAAAD7ryDIP6AnUq6FQE/v9IjQVFlgEA +uzYAAADoYw3+/w8fAOi7BP7/SItEJEDoMez//+sUkEjHwP////9IicMxyTH26Fvv//9Ii2wkMEiD +xDjDSIlEJAjoJ1UAAEiLRCQIZpDp+/7//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAP +hpkBAABIg+xoSIlsJGBIjWwkYEiLUShIi3EwSIX2dBNIOdZzC0iF9kmJ0EiJ8usGSIX2SYnQTIlE +JEBIiXQkOEiJRCQwSIlcJChIibwkiAAAAEiJjCSAAAAASIlUJBh0CEw5xnYDSYnwTIlEJCDoaAP+ +/0iNBRpdAQC7EQAAAOh3DP7/SItEJEDozQr+/0iNBW1JAQC7BQAAAJDoWwz+/0iLRCQ46LEK/v9I +jQVjTwEAuwkAAAAPH0QAAOg7DP7/SItEJDDokQr+/0iNBa5HAQC7AQAAAA8fRAAA6BsM/v9Ii0Qk +KOhxCv7/SI0FqUcBALsCAAAADx9EAADo+wv+/+hWA/7/RA8RfCRISMdEJFgAAAAASI0FwAAAAEiJ +RCRISIuEJIAAAABIiUQkUEiLhCSIAAAASIlEJFhIi0QkGEgFAP///0iLTCRASI2RAPj//0g5wkgP +R8JIi1QkIEiNmgABAABIgcEACAAASDnZSA9C2UiLTCQoSDnZSA9C2UiLTCQwSDnBSA9HwUiNTCRI +6McM/v9Ii2wkYEiDxGjDSIlEJAhIiVwkEEiJTCQYSIl8JCDo5FMAAEiLRCQISItcJBBIi0wkGEiL +fCQg6Sv+///MzMzMzMzMzMzMzEiLSghIi1IQSDlBMHQaSDlBKHQOSDnQdQa4IQAAAMMxwMO4PAAA +AMO4PgAAAMPMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4anAAAASIPsIEiJbCQYSI1sJBiIXCQwSIuA +OAEAAOh4ev//SIXAdHgPtkgogPkSdAWA+Qp1DDHASItsJBhIg8Qgw4D5EXUpD7ZMJDCEyXQMMcBI +i2wkGEiDxCDDD7YF/s8JAIPwAUiLbCQYSIPEIMPoBoH//2YPH0QAAEiD+wh9BDHA6xFIjR1oSgEA +uQgAAADoxVH7/0iLbCQYSIPEIMMxwEiLbCQYSIPEIMNIiUQkCIhcJBDowVIAAEiLRCQID7ZcJBDp +Mv///8zMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhgcBAABIg+xoSIlsJGBIjWwkYEiJRCRwSIM9k+QG +AAB0K0jHRCQoAAAAAEiNVCQwRA8ROkiNVCRARA8ROkiNVCRQRA8ROjHJ6ZEAAAAxyetKSIlMJCBI +iVQkGOh7AP7/SI0FLmMBALsWAAAA6IoJ/v9Ii0QkGA8fRAAA6NsH/v/o1gL+/+jRAP7/SItEJCBI +jUgBSItEJHAPHwBIg/kgfQuEAEiLFMhIhdJ1pUiLbCRgSIPEaMNIiUwkIEiJ0Lv///9/SI1MJCjo +bgAAAEiLVCQgSI1KAUiLVCRwSInQSIP5IH0LhABIixTISIXSdcdIx0QkKAAAAABIjUQkKOg5AgAA +SItsJGBIg8Row0iJRCQI6IVRAABIi0QkCOnb/v//zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +STtmEA+GywEAAEiD7GhIiWwkYEiNbCRgSIlcJHhIiYwkgAAAAEiJRCRwSIkBMdLrD0iLXCR4SInC +SItEJHBmkEg50w+MfwEAAEiJVCQQSInI6KoBAABIi4wkgAAAAEiLQRhIhcB0TkiJRCQ46K9t//9I +iUQkIEQPEXwkQEiLTCQ4SIlMJEBIiUQkSEiLTCRASIlMJCjoBv/9/0iLRCQoSItcJCDoFwj+/+hy +Af7/6G3//f/rG+jm/v3/SI0FDVYBALsQAAAA6PUH/v/oUP/9/+jL/v3/SI0FckMBALsBAAAA6NoH +/v/oNf/9/0iLhCSAAAAASItICEiFyXR9SIlMJDhIicjoF23//0iJRCQgRA8RfCRQSItMJDhIiUwk +UEiJRCRYSIuMJIAAAABIi1EQSIlUJBhIi1wkUEiJXCQwZpDoW/79/0iLRCQwSItcJCDobAf+/0iN +Be1CAQC7AQAAAOhbB/7/SItEJBjoUQT+/+hsAP7/6Kf+/f/oIv79/0iNBTxDAQC7AwAAAOgxB/7/ +SItEJHDohwX+/+iCAP7/ZpDoe/79/0iLRCQQSP/ASIuMJIAAAABIg3koAGYPH0QAAA+Fa/7//+sD +SInQSItsJGBIg8Row0iJRCQISIlcJBBIiUwkGOh3TwAASItEJAhIi1wkEEiLTCQY6QP+///MzMxJ +O2YQdlZIg+wYSIlsJBBIjWwkEIM9hcwJAAB3EEmLTjBMifZIObHAAAAAdAlIjQ20lQEA6wdIjQ37 +lQEASIsxSIs9MeEGAEiJw0iJykiJ+P/WSItsJBBIg8QYw0iJRCQI6PpOAABIi0QkCOuTzMzMzMzM +zMzMzMzMzMzMzMzMzEk7ZhAPhpwAAABIg+w4SIlsJDBIjWwkMEiJXCRISIM92+AGAAB0bYM98ssJ +AAB3GUmLdjBMifdmDx+EAAAAAABIOb7AAAAAdAlIjRUYlQEA6wdIjRVflQEASIXJdkBEDxF8JBBE +DxF8JCBIiUQkEEiJXCQgSIlMJChIiwpIiwV94AYASI1cJBD/0UiLbCQwSIPEOMNIi2wkMEiDxDjD +McDom1UAAJBIiUQkCEiJXCQQSIlMJBhIiXwkIOghTgAASItEJAhIi1wkEEiLTCQYSIt8JCDpKP// +/8zMzMzMzMzMSTtmEHZxSIPsGEiJbCQQSI1sJBBIiUQkIItYKA8fQADouwEAAOgWCgAASItMJCAP +tkkU9sECdCdIg/sBcitI/8tIidlI99tIwfs/SIPjAUgB2EiJy0iLbCQQSIPEGMNIi2wkEEiDxBjD +uAEAAABIidnoqlUAAJBIiUQkCA8fQADoe00AAEiLRCQI6XH////MzMzMzMzMzMzMzMzMzMzMzA+2 +SBT2wQF0ZA+2SBeD4R+A+RR3K4D5EncVgPkRdAuQgPkSdTlIg8BAw0iDwEjDgPkTdQVIg8A4w0iD +wFDDZpCA+RZ3D4D5FXUFSIPAWMNIg8A4w4D5F3QPgPkZdAVIg8Aww0iDwFDDSIPAOMMxwMPMzMzM +zMzMzMzMzMzMzMzMSTtmEA+GigAAAEiD7BhIiWwkEEiNbCQQSIlEJCAPHwDoW////0iFwHVPSItM +JCAPtlEXg+IfgPoUdCsPH0QAAID6GXUTSItBMOjSCAAASItsJBBIg8QYwzHAMdtIi2wkEEiDxBjD +SItBMOixCAAASItsJBBIg8QYw4sYSItEJCDoOwAAAOiWCAAASItsJBBIg8QYw0iJRCQI6EJMAABI +i0QkCOlY////zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GXAIAAEiD7FBIiWwkSEiNbCRI +hdt0DYlcJGBIjQ15sQYA6xcxwEiLbCRISIPEUMNIi4kQAgAADx9AAEiFyXQySIuRGAEAAEg50HLk +SIuxIAEAAEg58HPYSGPLSI0ECkg5xg+CaQEAAEiLbCRISIPEUMNIiUQkIJCQkEiNBdrfBgDoVbf7 +/0iLHd7fBgBIjQVH4AAAi0wkYA8fAOjb9/v/iFwkH0iLCEiJTCQ4kJCQSI0Fpd8GAA8fRAAA6Pu4 ++/8PtkwkH4TJdWjojfn9/0iNBchSAQC7EQAAAJDomwL+/4tEJGBIY8Do7wD+/0iNBQRAAQC7BgAA +AA8fAOh7Av7/SItEJCDo0QD+/0iNBRhPAQC7EAAAAA8fRAAA6FsC/v/otvn9/0iNBW+wBgDpigAA +AEiLRCQ4SItsJEhIg8RQw0iJRCRASIuIGAEAAEiJTCQwSIuQIAEAAEiJVCQo6Pn4/f9IjQWUQAEA +uwcAAADoCAL+/0iLRCQwDx8A6FsA/v9IjQVlQQEAuwgAAADo6gH+/0iLRCQoDx9EAADoOwD+/+g2 ++/3/6DH5/f9Ii0QkQEiLgBACAAAPH0QAAEiFwHWASI0FBIUBALsuAAAA6Crg/f9IiVQkMEiJdCQo +6Hv4/f9IjQW2UQEAuxEAAADoigH+/4tEJGBIY8APHwDo2//9/0iNBbJLAQC7DgAAAOhqAf7/SItE +JDAPH0QAAOi7//3/SI0FGT0BALsDAAAA6EoB/v9Ii0QkKA8fRAAA6Jv//f/olvr9/+iR+P3/SI0F +u3QBALshAAAADx9EAADom9/9/5BIiUQkCIlcJBDorEkAAEiLRCQIi1wkEA8fAOl7/f//zMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEA+GgQIAAEiD7FBIiWwkSEiNbCRIhdt0Eg8fQACD+/90 +CUiNDdSuBgDrEzHASItsJEhIg8RQw0iLiRACAABIhcl0FEg5gRgBAAB360g5gSABAAB24usCMcmJ +XCRgSIXJdGNIiUwkQEiLkQACAABIjQWI3gAAid5IidOJ8ZDou/P7/0iLAEiFwHUxSItMJEBIi5EY +AQAAi1wkYEhj80iNBDJIi4kgAQAASDnBD4JRAQAASItsJEhIg8RQw0iLbCRISIPEUMNIiUQkGJCQ +kEiNBencBgDoZLT7/0iLHe3cBgBIjQVW3QAAi0wkYOhN8/v/SIsISIlMJDCQkJBIjQW73AYA6Ba2 ++/9Ii0QkMEiFwHVf6Kf2/f9IjQUmUAEAuxEAAADotv/9/4tEJGBIY8DoCv79/0iNBR89AQC7BgAA +AOiZ//3/SItEJBjo7/39/0iNBTZMAQC7EAAAAA8fAOh7//3/6Nb2/f9IjQWPrQYA63hIi2wkSEiD +xFDDSIlEJDhIi4gYAQAASIlMJChIi5AgAQAASIlUJCDoIfb9/0iNBbw9AQC7BwAAAOgw//3/SItE +JCjohv39/0iNBZA+AQC7CAAAAOgV//3/SItEJCDoa/39/+hm+P3/6GH2/f9Ii0QkOEiLgBACAABI +hcB1jUiNBcOCAQC7LgAAAA8fQADoW939/0iJVCQoSIlMJCDorPX9/0iNBStPAQC7EQAAAOi7/v3/ +i0QkYEhjwOgP/f3/SI0F5kgBALsOAAAADx8A6Jv+/f9Ii0QkKOjx/P3/SI0FTzoBALsDAAAADx9E +AADoe/79/0iLRCQg6NH8/f/ozPf9/+jH9f3/SI0FM3IBALshAAAA6Nbc/f+QSIlEJAiJXCQQ6OdG +AABIi0QkCItcJBDpWf3//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4a7AgAASIPsSEiJ +bCRASI1sJECD+/90CUiNDRysBgDrIkiNDZuPAQCEAUiLBZKPAQBIi2wkQEiDxEiQw0iLiRACAABI +hcl0FUg5gRgBAAB360g5gSABAAB24pDrAjHJiVwkWEiFyXQvSIuRKAEAAEiLsTABAAAPH0QAAEiD +/gF+BzHA6fsBAABIY9NIA5GwAAAA6T8BAABIiUQkGJCQkEiNBVXaBgDo0LH7/0iLHVnaBgBIjQXC +2gAAi0wkWOi58Pv/SIsISIlMJDCQkJBIjQUn2gYA6IKz+/9Ii0QkMEiFwHVf6BP0/f9IjQWBTQEA +uxEAAADoIv39/4tEJFhIY8Dodvv9/0iNBYs6AQC7BgAAAOgF/f3/SItEJBjoW/v9/0iNBaJJAQC7 +EAAAAOjq/P3/6EX0/f9IjQX+qgYA6X4AAABIi2wkQEiDxEjDSIlEJDhIi4gYAQAASIlMJChIi5Ag +AQAASIlUJCDojfP9/0iNBSg7AQC7BwAAAJDom/z9/0iLRCQo6PH6/f9IjQX7OwEAuwgAAAAPH0QA +AOh7/P3/SItEJCDo0fr9/+jM9f3/6Mfz/f9Ii0QkOEiLgBACAABIhcB1h0iNBft/AQC7LgAAAOjF +2v3/SIuxuAAAAEg58ncNSInQSItsJEBIg8RIw0iJdCQoSIuBsAAAAEiJRCQg6Pby/f9IjQVkTAEA +uxEAAADoBfz9/4tEJFhIY8DoWfr9/0iNBTBGAQC7DgAAAOjo+/3/SItEJCAPHwDoO/r9/0iNBZk3 +AQC7AwAAAOjK+/3/SItEJCgPH0QAAOgb+v3/6Bb1/f/oEfP9/0iNBVxvAQC7IQAAAA8fRAAA6Bva +/f9I/8BIOfB9MEiNPEBMiwT6TItM+ghMY9MPHwBNOdB34E0BwU05ynPYSItU+hBMAdJMKcLpHv// +/zHSkOkW////SIlEJAiJXCQQ6O1DAABIi0QkCItcJBAPH0AA6Rv9///MzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMxIg+wYSIlsJBBIjWwkEEQPETwkSIXAdAYxyTHS6xpEDxE8JDHAMdtIi2wkEEiD +xBjDSI1LAUiJ8EiJxkiNPAFIjX8BD7Y/TI0ESU6NBEGQTYXAfF1Jg/hATRnJQYn6g+d/SInLTInB +SNPnTCHPSAH6QfbCgHW7SIXSdRNEDxE8JDHAMdtIi2wkEEiDxBjDkEiNDANIjUkCSIkMJEiJVCQI +SIsEJEiJ00iLbCQQSIPEGMPoirP9/5DMzMzMzMzMzMxIg+wYSIlsJBBIjWwkEEQPETwkD7YQ9sIC +dAcxyTHSkOsaRA8RPCQxwDHbSItsJBBIg8QYw0iNSwFIifBIicZIjTwBSI1/AQ+2P0yNBElOjQRB +kE2FwA+MkwAAAEmD+EBNGclBifqD539IictMicFI0+dMIc9IAfpB9sKAdbdIAdoxyTHb6wRJjUsB +SI00CkiNPDBIjX8CD7Y/TI0ESU6NBEGQTYXAfEJJg/hATRnJQYn6g+d/SYnLTInBSNPnTCHPSAH7 +QfbCgHW9kEiNDDBIjUkDSIkMJEiJXCQISIsEJEiLbCQQSIPEGMPokbL9/+iMsv3/kMzMzMzMzMzM +zMzMSTtmEA+GMAEAAEiD7CBIiWwkGEiNbCQYSIXAdA4PthD2wgR0BjHJMdvrGDHAMdtIi2wkGEiD +xCDDSY1LAUiJ8EiJ+0iJxkiNPAFIjX8BD7Y/TI0ESU6NBEGQZg8fRAAATYXAD4zLAAAASYP4QE0Z +yUGJ+oPnf0mJy0yJwUjT50whz0gB30H2woB1rk6NBB9KjTwfSI1/AvbCAnQGMckx0us9x0QkFAAA +AACQSI0MPkiNVCQUSDnKdAeLDD6JTCQUi1wkFEiJ8OgU9f//6G/9//9Ii2wkGEiDxCDDSI1LAUmN +PAhIjTwHSI1/Ag+2P0yNDElOjQxJkE2FyXwzSYP5QE0Z0kGJ+4Pnf0iJy0yJyUjT50wh10gB+kH2 +w4B1vUiNDBpKjTwBSI1/A+lz////6Eyx/f/oR7H9/5BIiUQkCJDou0AAAEiLRCQI6bH+///MzMzM +zMzMzMzMzMzMzMzMzEyNpCTI/f//TTtmEA+GuAUAAEiB7LgCAABIiawksAIAAEiNrCSwAgAASIM9 +/6cGAAAPhNEAAABIix0qpwYASI2MJAABAABEDxE5SI2UJBABAABEDxE6SI2UJCABAABEDxE6SIP7 +CH87SI28JMABAABIjX/wZg8fhAAAAAAADx8ASIlsJPBIjWwk8OjJSQAASIttAEiNlCTAAQAASImU +JBABAABIjQXW1QAA6BHW+/9IixUy0gYASIXSdA1IizJIi3oISItSEOsGMf8x9jHSSIX/D4boBAAA +SImEJIAAAABMiwZI/8pI99pIwfo/SIPiCEgB8kiJlCTIAAAASI13/0iJdCR4McnrGEiLrCSwAgAA +SIHEuAIAAMNI/8FJifhmkEg58Q+NUgEAAEiJTCRwTImEJKAAAABIizzKSIm8JMAAAABNi4hAAQAA +TImMJLgAAABNi5BIAQAATIlUJGgx25DpZgEAAEiDvwACAAAAdalIi59IAQAASI0Fo9QAADHJkOg7 +1fv/SIsN7NIGAEiNcQFIix3Z0gYASIs94tIGAGaQSDn3c0dIiYQkiAAAAEiNBWzUAADoZx3//0iJ +DcDSBgCDPdm9CQAAdQlIiQWg0gYA6wxIjT2X0gYA6IJBAABIidlIicNIi4QkiAAAAEiNUQFIiRWB +0gYASI08y4M9nr0JAAB1FUiJBMtIi5QkwAAAAEiJggACAADrGehCQQAASIuUJMAAAABIjboAAgAA +6C5BAABIi7JAAQAASIm0JLgAAABIi7pIAQAASIl8JGgxwOnUAQAASIuEJIAAAABIi0wkcEiLlCTI +AAAASIt0JHhIi7wkwAAAAOmd/v//SIusJLACAABIgcS4AgAAw0yLXCRgSY1bAUiLhCSAAAAASItM +JHBIi5QkyAAAAEiLdCR4SIu8JMAAAABMi4QkoAAAAEyLjCS4AAAATItUJGhMOdMPjZH+//9IiVwk +YEWLHJlNi6AAAgAATYXkdQxNY9tNA5gYAQAA6x1IjQUk0wAATInjRInZ6Fno+/9MixhIi4QkgAAA +AEyJnCSYAAAAQYtLEEiJw0iNBVjTAADoM+j7/0iLSAhIixhIi3gQSIuUJJgAAAAxwOlhAgAASI1x +AUg593MsSIlMJEhIjQWk/AAADx9AAOi7G///SI1zAUiLlCSYAAAASInDSInPSItMJEhIiZwkqAAA +AEiJdCRYSIl8JFBMjQTLgz0HvAkAAHUHSIkUy5DrCEyJx+jWQAAAi0oQSI0FzNIAAEiLnCSAAAAA +Dx9AAOjb6vv/SItUJFhIiVAISItUJFBIiVAQgz3CuwkAAGaQdRBIi5QkqAAAAEiJEOmZ/v//SInH +SIuUJKgAAAAPHwDoe0AAAOmB/v//TItEJGBJjUABSIuUJMAAAABIi7QkuAAAAEiLfCRoSDn4D40j +/v//SIlEJGCLPIaJfCRETGPHTAOCGAEAAEyJhCSwAAAAQYtIEEiLnCSAAAAASI0FHtIAAOj55vv/ +SIsQSImUJKgAAABIi3AISIl0JFgxwA8fAOl/AAAASIuEJJAAAADrCEiLhCSwAAAASImEJJAAAABI +i5QkwAAAAEiLmgACAACLTCRESI0FZ9EAAOji6fv/hACDPdm6CQAAdRBIi5QkkAAAAEiJEOkx//// +SInHSIuUJJAAAADolz8AAOkc////TItEJFBJjUABSIuUJKgAAABIi3QkWEg58H2GSIlEJFBIiwzC +SImMJJAAAABIjZwk0AAAAEQPETtIjbQk4AAAAEQPET5IjbQk8AAAAEQPET5IjbwkMAEAAEiNf9BI +iWwk8EiNbCTw6CBFAABIi20ASI20JDABAABIibQk4AAAAOgOWf//iYQk3AAAAEiLhCSwAAAASIuc +JJAAAABIjYwk0AAAAOhKAAAAhMAPhFD///9mkOng/v//SP/ASDnID42W/f//TIsEw0k50HXr6cz8 +//8xwEiJ+eg3QgAAkOjROgAA6Sz6///MzMzMzMzMzMzMzMxMjaQkYP///007ZhAPhmwMAABIgewg +AQAASImsJBgBAABIjawkGAEAAEiJnCQwAQAASImEJCgBAABIiYwkAAEAAEiJhCQIAQAASImcJBAB +AABIjQWIzwAASInLSI2MJAgBAADomNT7/4TbD4XZBQAASIuUJCgBAABIiZQkCAEAAEiLtCQwAQAA +SIm0JBABAABIjQVJzwAASIucJAABAABIjYwkCAEAAOiU1vv/hABIi4QkKAEAAEiLlCQwAQAAZpBI +OdAPhGsFAAAPtkgXicuD4R8PtnIXg+YfDx+AAAAAAEA4zg+FOQUAAIhcJBuITCQa6Mrr//9IiYQk ++AAAAEiJXCRoSIuEJDABAADosOv//0iLTCRoSDnLdRRIicNIi4Qk+AAAAOh2OPv/hMB1EzHASIus +JBgBAABIgcQgAQAAkMNIi4QkKAEAAOgS7P//SImEJKAAAABIi4QkMAEAAGaQ6Pvr//9Ii4wkoAAA +AEiFyXUTSIXAD4SSAAAADx9EAABIhcl0BUiFwHUSMcBIi6wkGAEAAEiBxCABAADDSImEJJgAAACL +GUiLhCQoAQAA6O3s///oSPX//0iJhCS4AAAASIlcJDBIi4wkmAAAAIsJSIuEJDABAACJy+jC7P// +ZpDoG/X//0iLTCQwSDnLD4UdBAAASInDSIuEJLgAAABmkOibN/v/hMAPhAMEAAAPtnQkGo1+/0CA +/w8PhtwDAACQQID+FQ+HfwIAAECA/hIPhsQBAABAgP4TD4WtAAAASIu0JCgBAAAPt34yTIuEJDAB +AABmQTl4Mg+FfAAAAA+3TjBBD7d4MGY5z3VukEQPtk4UQfbBAXQHuEgAAADrBbg4AAAATI0MBkiB ++QAAEAAPh98JAACQQQ+2UBT2wgF0B7hIAAAA6wW4OAAAAGaJTCQeZol8JBxMiYwkiAAAAJBJjRQA +SImUJIAAAABMi5QkAAEAADHA6f8HAAAxwEiLrCQYAQAASIHEIAEAAMNAgP4UD4WhAAAASIuMJCgB +AABIi0Ew6Prz//9IiYQk+AAAAEiJXCRoSIuMJDABAABIi1EwSInQ6Nnz//9Ii0wkaEg5y3VTSInD +SIuEJPgAAAAPH0AA6Fs2+/+EwHQ7SIuUJCgBAABIi3JASIu8JDABAAAPHwBIOXdAdQxIiXQkaDHA +6TwFAAAxwEiLrCQYAQAASIHEIAEAAMMxwEiLrCQYAQAASIHEIAEAAMNIi5QkKAEAAEiLQjBIi7Qk +MAEAAEiLXjBIi4wkAAEAAOhF/P//hMB1BDHA6yVIi5QkKAEAAEiLQjhIi5QkMAEAAEiLWjhIi4wk +AAEAAOgY/P//SIusJBgBAABIgcQgAQAAw0CA/hF0UWaQQID+Eg+FVAIAAEiLlCQoAQAASItyOEiL +vCQwAQAAZpBIOXc4dAQxwOsVSItCMEiLXzBIi4wkAAEAAOjB+///SIusJBgBAABIgcQgAQAAw0iL +lCQoAQAASItCMEiLtCQwAQAASIteMEiLjCQAAQAA6Iz7//+EwHQdSIuMJCgBAABIi0lASIuUJDAB +AABIOUpAD5TB6wIxyYnISIusJBgBAABIgcQgAQAAw0CA/hd3cUCA/hZ1NUiLlCQoAQAASItCMEiL +lCQwAQAASItaMEiLjCQAAQAA6Cb7//9Ii6wkGAEAAEiBxCABAADDSIuUJCgBAABIi0IwSIuUJDAB +AABIi1owSIuMJAABAADo8fr//0iLrCQYAQAASIHEIAEAAMOQQID+GA+EvAAAAECA/hkPhawAAABI +i4wkMAEAAEiLUUBIi5wkKAEAAEg5U0B1fUiLQzDoqfH//0iJhCT4AAAASIlcJGhIi4wkMAEAAEiL +UTBIidDoiPH//0iLTCRoDx8ASDnLdTRIicNIi4Qk+AAAAOgLNPv/hMB0IEiLlCQoAQAASItaQEiJ +XCRoSIu0JDABAAAxwOkKAQAAMcBIi6wkGAEAAEiBxCABAADDMcBIi6wkGAEAAEiBxCABAADDDx8A +QID+GnV4uAEAAABIi6wkGAEAAEiBxCABAADDuAEAAABIi6wkGAEAAEiBxCABAADDMcBIi6wkGAEA +AEiBxCABAADDMcBIi6wkGAEAAEiBxCABAADDuAEAAABIi6wkGAEAAEiBxCABAADDuAEAAABIi6wk +GAEAAEiBxCABAADDZpDom+L9/0iNBahYAQC7HgAAAOiq6/3/D7ZEJBuD4B9mkOib6P3/6Pbk/f/o +8eL9/0iNBQVVAQC7HQAAAA8fRAAA6PvJ/f9Ii3wkOEiNRwFIi1wkaEiLtCQwAQAASIuUJCgBAABI +OdgPjaUBAABIi3o4SItKQEg5yA+DrgEAAEyNBEBMi044SItOQEg5yA+DlAEAAEyJRCRgSIm8JPAA +AABMiYwk6AAAAEiJRCQ4SosEx+jr7///SImEJPgAAABIiVwkWEiLTCRgSIuUJOgAAABIizTKSInw +6MXv//9Ii0wkWEg5yw+FFgEAAEiJw0iLhCT4AAAA6Ecy+/8PH4AAAAAAhMAPhPcAAABIi1QkYEiL +tCTwAAAASItE1ghIi7wk6AAAAEiLXNcISIuMJAABAADobPj//4TAD4SxAAAASItMJGBIi5Qk8AAA +AEiLBMroDvD//0iJhCT4AAAASIlcJFhIi0wkYEiLlCToAAAASIs0ykiJ8Ojo7///SItMJFgPHwBI +Oct1VkiJw0iLhCT4AAAA6Ksx+/+EwHRCSItUJGBIi5wk8AAAAEiLXNMQSIu0JOgAAABIi1TWEA8f +hAAAAAAASDnaD4R8/v//McBIi6wkGAEAAEiBxCABAADDMcBIi6wkGAEAAEiBxCABAADDMcBIi6wk +GAEAAEiBxCABAADDMcBIi6wkGAEAAEiBxCABAADDuAEAAABIi6wkGAEAAEiBxCABAADD6JU5AADo +kDkAAEyLRCRISY1AAUiLdCRoSIu8JDABAABIi5QkKAEAAEg58A+NxwEAAEyLQjhIi0pAkEg5yA+D +zwEAAE2NDMBMi1c4SItPQEg5yA+DtQEAAEyJlCTgAAAATImEJNgAAABIiUQkSEyJjCSoAAAAQYsc +wEyJyOiZ5f//SImEJNAAAABIi0wkSEiLlCTgAAAASI00ykiJtCSQAAAAixzKSInw6G3l//9IiYQk +yAAAAEiLhCTQAAAA6Ljt//9IiYQkwAAAAEiJXCRgSIuEJMgAAAAPHwDom+3//0iLTCRgSDnLD4X5 +AAAASInDSIuEJMAAAABmkOgbMPv/hMAPhN8AAABIi4Qk0AAAAOgm7///SImEJPgAAABIiVwkYEiL +hCTIAAAA6Azv//9Ii0wkYA8fgAAAAABIOcsPhZEAAABIicNIi4Qk+AAAAOjHL/v/hMB0fUiLTCRI +SIuUJNgAAACLXMoESIuEJKgAAADoRef//0iJhCSwAAAASItMJEhIi5Qk4AAAAItcygRIi4QkkAAA +AA8fQADoG+f//0iJw0iLjCQAAQAASIuEJLAAAADow/X//w8fAITAD4VI/v//McBIi6wkGAEAAEiB +xCABAADDMcBIi6wkGAEAAEiBxCABAADDMcBIi6wkGAEAAEiBxCABAADDuAEAAABIi6wkGAEAAEiB +xCABAADD6Ig3AADogzcAAEyLXCRQSY1DAQ+3TCQeD7d8JBxMi5QkAAEAAEiLlCSAAAAATIuMJIgA +AABIi7QkKAEAAEyLhCQwAQAASDnIfTlNixzBSDn4D4OCAQAASIlEJFBIixzCTInRTInY6AL1//9m +kITAdZkxwEiLrCQYAQAASIHEIAEAAMOQD7ZWFA8fRAAA9sIBdAe4SAAAAOsFuDgAAAAPt1YygeL/ +fwAAD7d+MAH6SAHGD7fKSDnPD4cVAQAAkEgp+UiNlwAA8P9IwecDSMH6P0gh10iNFD5BD7ZwFED2 +xgF0B7hIAAAA6wW4OAAAAEEPt3Aygeb/fwAAQQ+3eDAB/kkBwA+39g8fgAAAAABIOfcPh68AAABI +iUwkIEiJVCR4SCn+SIl0JChMjY8AAPD/SMHnA0nB+T9MIc9MAcdIiXwkcDHA6ydMi0QkQEmNQAFI +i0wkIEiLdCQoTIuUJAABAABIi3wkcEiLVCR4ZpBIOch9NkyLBMJIOfBzQkiJRCRASIscx0yJ0UyJ +wA8fAOjb8///hMB1sDHASIusJBgBAABIgcQgAQAAw7gBAAAASIusJBgBAABIgcQgAQAAw0iJ8ejI +NQAAifhIifEPHwDoezYAAIn46HQ2AACJ+eitNQAAugAAEADo4zUAAJBIiUQkCEiJXCQQSIlMJBjo +Li4AAEiLRCQISItcJBBIi0wkGOla8///zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4Zg +BAAASIPsGEiJbCQQSI1sJBDGAABIid5IiVgITItGIEwBwzHJMdIx/+sC/8FmOU44dkBED7fBTWvA +OEaLDANBg/kBdR2E0nUSTYtMGAhOK0wDEEwDSAhMiUgQugEAAADryUGD+QJ1w0mLfBgISAN4COu4 +hNJ0eQ8fAEiF/3RxSI1YIEiNcBhMjUBgTI1IaIM9ZKwJAAB1F0jHQCAAAAAASMdAGAAAAABEDxF4 +YOs4SIn5SInfTInKRTHJ6JoxAABIifdFMcnojzEAAEyJx0UxyeiEMQAASInXRTHJ6HkxAABIic9J +idExyTHSRTHS6xBIi2wkEEiDxBiQw0mNTCQBSbv//////z8AAEw52Q+DRQMAAEmJzEjB4QRMiywP +TYXtD4QjAQAATIt8DwhMA3gQZg8fhAAAAAAAkEmD/QYPj3sAAABJg/0EdQVNifrrrEmD/QV1K4M9 +nqsJAAB1BkyJeCDrl0iJ+UiJ30mJ9UyJ/uilMAAATInuSInP6Xv///9Jg/0GD4Vx////gz1pqwkA +AHUMTIl4GA8fAOlc////SIn5SIn3TYnNTYn56KowAABIic9NiekPH0AA6Tz///9Jgf31/v9vdQhM +ifrpK////0mB/fD//291MYM9GqsJAAB1CUyJeGDpEP///0iJ+UyJx0mJ9UyJ/g8fAOgbMAAATInu +SInP6fH+//9Jgf38//9vD4Xk/v//gz3cqgkAAHUJTIl4aOnS/v//SIn5TInPSYn1TIn+Dx9EAADo +2y8AAEyJ7kiJz+mx/v//SIN4IAAPhL8BAABIg3gYAA+EtAEAAE2F0nUJSIXSD4SmAQAASIN4aAB1 +JYM9fqoJAAB1CkjHQGAAAAAA6xJMiccx2+hoLwAADx+EAAAAAABIhdIPhL0AAACLCotaBIlYWIta +CI00G412BEm4////////AABMOcYPh4cBAABJKfBNicFJ99hIweYCScH4P0whxkgB1kw5yQ+HXwEA +AEiJSEhMiUhQjRxZjVsEgz3+qQkAAHUGSIlwQOsJSI14QOgNLwAASLn///////8AAA8fAEg5yw+H +HAEAAEgp2UiJSDBIiUg4SPfZSMHjAkjB+T9IIdlIAdGDPbSpCQAAdQZIiUgo6wlIjXgo6GMuAADG +QFwB6acAAABBixpBi3IEjUsCSLr///////8AAGYPH0QAAEg5yg+CtwAAAEiD+QIPgqEAAABMjUH+ +TIlASEm4/f//////AABMiUBQTY1CCI0cM41bAoM9RqkJAAB1BkyJQEDrCUiNeEDodS4AAEg503dd +SDnLck9IKctIiVgwSCnKSIlQOEj32kjB4QJIwfo/SCHKSY0MEoM9BKkJAAB1BkiJSCjrCUiNeCjo +sy0AAMYAAUiLbCQQSIPEGMNIi2wkEEiDxBjDiciJ2ejzMQAAidnobDEAALgCAAAA6OIxAABmkOhb +MQAAidjo1DEAAEyJyuiMMQAAifBMicHowjEAAEiJyEyJ2ej3MAAAkEiJRCQISIlcJBDohykAAEiL +RCQISItcJBDpePv//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhv8AAABIg+xASIlsJDhI +jWwkOIA4AHQQSIlEJEhIiVwkUEiLSGjrEDHASItsJDhIg8RAw5BIAdEPt1ECZvfCAQAPhYcAAACL +UQyLcQg5cxB1fEiJTCQgSItYIIQDixQRSAHaSIlUJBhIidDoq0X//0QPEXwkKEiLTCQYSIlMJChI +iUQkMEiLTCRQSItRCEiLGUiLdCQoDx9EAABIOdB0BDHA6xdIidhIifNIidHoiSf7/0iLTCRQDx9A +AITAdR5Ii0QkSEiLTCQgSItcJFCLURCF0g+FW////2aQ6xhIi0wkIA+3QQQl/38AAEiLbCQ4SIPE +QMO4/////0iLbCQ4SIPEQMNIiUQkCEiJXCQQ6EgoAABIi0QkCEiLXCQQ6dn+///MzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMTI1kJOhNO2YQD4ZXAwAASIHsmAAAAEiJrCSQAAAASI2sJJAAAACAOAB0 +eUiJhCSgAAAARA8RfCR4x4QkiAAAAAAAAABMjQ1VAwAATIlMJHhIiYQkgAAAAImcJIgAAACAeFwA +dCNMiw0MmgYATIsV/ZkGAE2FyQ+OXQEAAEyJTCRQMcnpbgEAAEyLDemZBgBMixXamQYATYXJfhlM +iUwkUDHJ6ypIi6wkkAAAAEiBxJgAAADDSIusJJAAAABIgcSYAAAAw0mDwiBMifhMidlBi3IUTYtC +GEGLehBJixpNi1oITItgSEyLaEBFheQPhOIAAABIicKJ+EmJ1zHSQff0idBJOcQPhsIAAABIiUwk +SEyJVCRwTIlcJEBIiVwkaIl0JDyJfCQ4TIlEJGBFi2SFAOtbSItMJEhMi0wkUEyLVCRwTIu8JKAA +AABMjVkBTTnZD49u////6Vn///9FiySCSItMJEhIi1wkaIt0JDyLfCQ4TItEJGBMi0wkUEyLVCRw +TItcJEBMi7wkoAAAAEWF5HS3RIlkJDBMi0wkeESJ4EyJ2UiNVCR4Qf/RhMB1hEyLjCSgAAAASYtJ +ME2LUSiLRCQwSDnIcpTotC0AAEyJ4eisLQAA6CeX/f9Ii6wkkAAAAEiBxJgAAADDTI1QIEyJ+EyJ +yUmJ0UGLchRNi0IYSYsaTYtaCEGLehBMi2BITItoQEWF5A+ERgEAAEiJwonwSYnXMdJB9/SJ0Ek5 +xA+GJQEAAEWLZIUARTlnWHYITInQTInK6zpIiUwkSEyJVCRYiXwkPEyJXCRASIlcJGhMiUQkYIl0 +JDjrPUiLRCRYSItMJEhIi1QkUEyLvCSgAAAATI1JAUw5yg+PXf///+lI////RItsJCxB/8VEieZJ +idFJicJFiexEiWQkLE2LbzBMidBNi1coTInKRYtPWEUpzJBNOeUPhoMAAABHiwyiRYnKQYPJAUGJ +9IPOAUQ5znVdRIlUJDRMi0wkeItEJCxMidlEieZIjVQkeEH/0Q8fRAAAhMAPhWD///9Ii0QkWEiL +TCRISItUJFBIi1wkaIt8JDxMi0QkYESLVCQ0TItcJEBEi2QkOEyLvCSgAAAAQQ+64gAPg0r////p +M////0SJ4EyJ6egpLAAATInh6CEsAACQ6JuV/f+QSIlEJAiJXCQQ6KwkAABIi0QkCItcJBAPHwDp +e/z//8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhAPhlwBAABIg+x4SIlsJHBIjWwkcEiJ +nCSIAAAATImEJKAAAABIi3IISIt+GIQHi1IQQYnBT40USU6NHNdNjVsERQ+2G5BFidxBg+MPQYD7 +AnQFRYTbdSBBwOwEkEGA/AF0BkGA/AJ1D06NHNdNjVsGZkGDOwB1DDHASItsJHBIg8R4w0iJdCRA +iVQkHEiJfCRYTIlMJDBIiUwkKEiJXCRQTIlEJEhMiVQkIEiLTiCEAUKLFNdIjQQRSIlEJDjoekD/ +/0QPEXwkYEiLTCQ4SIlMJGBIiUQkaEiLXCRgSItMJChIOch1cEiLRCRQ6Gwi+/+EwHRiSItMJEBI +i1FgSIXSdCqLXCQchdt0IkiLdCQwD7cUcoHi/38AAA8fADnTdAwxwEiLbCRwSIPEeMNIi0kQSItU +JCBIi1wkWEgDTNMISItUJEhIiQq4AQAAAEiLbCRwSIPEeMMxwEiLbCRwSIPEeMOJRCQISIlcJBBI +iUwkGIl8JCCJdCQkTIlEJCjoWiIAAItEJAhIi1wkEEiLTCQYi3wkIIt0JCRMi0QkKOla/v//zMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxMjWQk+E07ZhAPho0AAABIgeyIAAAASImsJIAAAABIjawk +gAAAAEiD+CF1YEiF23RLSI18JBBIjX/wZg8fRAAASIlsJPBIjWwk8OhXLAAASIttAEiNRCQQ6GP0 +//9IjUQkEEiNHYeUBgDo8vj//4nDSI1EJBDoJvr//+sQSIusJIAAAABIgcSIAAAAw0iLrCSAAAAA +SIHEiAAAAMNIiUQkCEiJXCQQ6BUiAABIi0QkCEiLXCQQ6Ub////MzMzMzMxIiw1RlAYASIsVQpQG +AEiFyX4EMdvrBzHAw0iDwiBIi3IYSIs2SIX2dQpI/8NIOdl/6OvjSIsNTaAJAEiJykj32Ughzkg5 +8HIMSI0MFkg5yA+SwesCMcmJyMPMzMzMzMxJO2YQD4aXAAAASIPsIEiJbCQYSI1sJBhIi5CgAAAA +SIlUJBBIx4CgAAAAAAAAAMaAsAAAABqAPVMzBwAAdBlIiUQkKLgUAAAAuwEAAADoLY///0iLRCQo +uwIAAAC5BAAAAOiZ/P3/SYtWMEiLksAAAACQkDH2SIlyMEmLVjCQMfZIibLAAAAASItEJBC7AQAA +AOhJKP7/SItsJBhIg8Qgw0iJRCQI6PUgAABIi0QkCOlL////zMzMzMzMzMzMzMxJO2YQD4ZXAQAA +SIPsOEiJbCQwSI1sJDBIi5CgAAAASMeAoAAAAAAAAABIg7joAAAAAHQaSMeA6AAAAAAAAABMi0Aw +SceAaAEAAAAAAABIiVQkKEiJRCRAgD1xMgcAAJB0LkmLVjCQSIuS0AAAAEmJluAAAAC4EQAAALsB +AAAAMckx/0iJ/uiVev//SItEJEC7AgAAALkBAAAA6KH7/f9Ji1YwSIuSwAAAAJCQMfZIiXIwSYtW +MJAx9kiJssAAAACQkEiNBc+7BgDo0ov7/5CQSItUJEBIx4KgAAAAAAAAAEiLNQG8BgBIhfZ0DEiJ +0UiJlqAAAADrC5BIidFIiRXduwYAkEiJDd27BgD/Bd+7BgCQkEiNBX67BgDoYY37/4A9qjEHAAB0 +DEiLRCQoMdvoDI7//0iLRCQouwQAAAC5AQAAAOj4+v3/SItEJCi7AQAAAOjJJv7/SItsJDBIg8Q4 +w0iJRCQI6HUfAABIi0QkCOmL/v//zMzMzMzMzMzMzMxJO2YQdhtIg+wISIksJEiNLCToSbb8/0iL +LCRIg8QIkMPoOh8AAOvYzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSTtmEHZbSIPsEEiJbCQISI1s +JAhIiUQkGOhibf3/SItMJBiEAUiNgZgWAADor9P8/0iLTCQYgLm4FgAAAHQXuAEAAABIjRX1mwkA +8A/BAsaBuBYAAABIi2wkCEiDxBCQw0iJRCQI6LUeAABIi0QkCOuOzMzMzMzMzMzMzMzMzMxJO2YQ +dldIg+wgSIlsJBhIjWwkGEmLVjBIi4LAAAAASIlEJBC7AgAAALkEAAAA6M35/f9IjQXOZQEA6OEH +/v9Ii0QkELsEAAAAuQIAAADorfn9/0iLbCQYSIPEIMMPHwDoOx4AAOuZzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzEk7ZhAPhvwAAABIg+wYSIlsJBBIjWwkELgBAAAADx8A6PsA/v9Iiw2spwkASInC +SCnISAEFl6cJAEiLDZinCQBIKcpIhdJ9GLgBAAAASI0NxO0JAPBID8EB6ZAAAABmkEiD+hB8XkgP +vdpIx8b/////SA9E3kiNc/1IifdIweYESIH+0AIAAHIMvywAAAC7DwAAAOs0SI1L/EiD+UBIGdtI +99NICdlI0/pIidNIwfo/SMHqPEgB2kjB+gRIweIESCnT6wUx/0iJ00jB5wRIjQQfSD3QAgAAcx9I +jQ241gkASI0EwbkBAAAA8EgPwQhIi2wkEEiDxBjDudACAAAPHwDomyQAAJDoFR0AAOnw/v//zMzM +zMzMzMzMzMzMzMzMzEk7ZhAPhsUAAABIg+wgSIlsJBhIjWwkGEiLDaGlCQBIiQ2qpgkAgz2/nQkA +AH5I6CwS/P/oh0b8/0mLRjBIi4DQAAAAhABIBZgWAABIiUQkEDHb6Ahr/P9Ji0YwSIuA0AAAAOj4 +av3/SItEJBDoTtH8/+gJE/z/kDHJSI0Vo5kJAIcKiw2bmQkAg/kBdQe5AQAAAOsGg/kCD5TBiA1j +mwkAhMl0B7kBAAAA6wcPtg1SmwkAiA1HmwkASIsFeKUJAOhbRPz/SItsJBhIg8Qgw+gsHAAA6Sf/ +///MzMzMzMzMSTtmEHYiSIPsEEiJbCQISI1sJAi4AQAAAOgC//3/SItsJAhIg8QQw+jzGwAA69HM +zMzMzMzMzMzMzMzMzMzMzEk7ZhB2IUiD7BBIiWwkCEiNbCQISItAQOjjBvz/SItsJAhIg8QQw0iJ +RCQI6K8bAABIi0QkCOvIzMzMzMzMzMxJO2YQdilIg+wQSIlsJAhIjWwkCEiNBRVjAQAPH0QAAOj7 +BP7/SItsJAhIg8QQw+hsGwAA68rMzMzMzMzMzMzMSTtmEHZZSIPsGEiJbCQQSI1sJBBIi0sYSIXJ +dCeQi5EIAQAAjXL/ibEIAQAAg/oBdRJBgL6xAAAAAHQIScdGEN76//9IjQUVmQkA6HCE+/+4AQAA +AEiLbCQQSIPEGMNIiUQkCEiJXCQQ6PIaAABIi0QkCEiLXCQQ64bMzMzMzMzGgLYAAAAASMeAgAEA +AAAAAADDzMzMzMzMzMzMzMzMzEk7ZhB2G0iD7AhIiSwkSI0sJOjpkfz/SIssJEiDxAiQw+iaGgAA +69jMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdkNIg+wQSIlsJAhIjWwkCJBIjQUkLQgADx9A +AOgbhvv/SI0FHC0IAOjvmfz/kJBIjQUGLQgA6OGH+/9Ii2wkCEiDxBDD6DIaAADrsMzMzMzMzMzM +zMzMzMzMzMxIg+wYSIlsJBBIjWwkEJCQkEiD+AVzfUiNFYJ5BgBIixTCSIP6QEgZ9kiNPfB4BgBI +izzHSIP/QEgZwEm4AAAAAACAAABJAdhIictIidFJ0+hJIfBJuf//////fwAASQHZSdPpSSHxSIn5 +uwEAAABI0+NIIdiQkEiJw0j320kh2EmNFAFIIdNMicBIi2wkEEiDxBjDuQUAAADo4iAAAJDMSTtm +EHYpSIPsEEiJbCQISI1sJAhJi04wSIuB0AAAAJDou2f9/0iLbCQISIPEEMPoTBkAAOvKzMzMzMzM +zMzMzEk7ZhB2IEiD7BhIiWwkEEiNbCQQSI0FMx8BALsQAAAA6Puu/f+Q6BUZAADr08zMzMzMzMzM +zMzMzMzMzMzMzMxJO2YQdkBIg+wYSIlsJBBIjWwkEEiNRCQg6IKu/f9mkEiFwHUKSItsJBBIg8QY +w0iNBQtAAQC7IAAAAA8fRAAA6Juu/f+Q6LUYAADrs8zMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdjZI +g+wQSIlsJAhIjWwkCMcEJAIAAAAPH0QAAOibMQAARQ9X/2RMizQl+P///0iLbCQISIPEEMMPH0AA +6FsYAADruczMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQdi1Ig+wgSIlsJBhIjWwkGEiNBQ1h +AQAx20jHwf/////oFw7+/0iLbCQYSIPEIMPoCBgAAOvGzMzMzMzMSTtmEA+GTwEAAEiD7EhIiWwk +QEiNbCRASItIMEiLkOgAAABIhcl0CUiLiegAAADrB0jHwf////9IhdJ0CUiLkugAAADrB0jHwv// +//9IiVQkIEiJTCQoi5iQAAAAiVwkFA+2sLAAAABAgP4bcg6+EwAAAEyNBVIjAQDrFEjB5gRIjT35 +jQYATIsEN0iLdDcISIl0JBhMiUQkOEiLgJgAAABIiUQkMOiVxf3/SI0FZwoBALsDAAAA6KTO/f9I +i0QkMOiazP3/SI0FvQ8BALsJAAAA6InO/f+LRCQUicAPHwDoe8v9/0iNBfUJAQC7AQAAAOhqzv3/ +SItEJDhIi1wkGOhbzv3/SI0FgAoBALsEAAAA6ErO/f9Ii0QkKA8fRAAA6DvM/f9IjQWzDgEAuwkA +AADoKs79/0iLRCQgDx9EAADoG8z9/+h2x/3/6HHF/f9Ii2wkQEiDxEjDSIlEJAhmkOibFgAASItE +JAjpkf7//8zMzMzMzMzMzMzMzMzMzMzMSTtmEA+G2AEAAEiD7AhIiSwkSI0sJPIPEAVKXQIA8g8R +BWKUCQBIiwVThwYAgz1MlQkAAHUJSIkFw6gGAOsMSI09uqgGAOj1GAAASIsFPocGAIM9J5UJAAB1 +CUiJBaaoBgDrDEiNPZ2oBgDo0BgAAEiLBSmHBgCDPQKVCQAAZpB1CUiJBYeoBgDrDEiNPX6oBgDo +qRgAAEiLBdKGBgCDPduUCQAAdQlIiQU6qAYA6wxIjT0xqAYA6IQYAABIiwWdhgYAgz22lAkAAHUJ +SIkFDagGAOsQSI09BKgGAA8fQADoWxgAAA+2BTSFBgBIhcC4AAAIALkAAAAASA9FwUiJBeySCQBI +iwU1hgYAgz1ulAkAAHUJSIkFtacGAOsMSI09rKcGAOgXGAAASIsFYIUGAIM9SZQJAAB1CUiJBTCn +BgDrDEiNPSenBgDo8hcAAEiLAEiDwAdIg+D499iJBQ+oBgBIiwUIpwYASIsAiQUDqAYASIsF+KYG +AEiLQAiJBfanBgBIiwXnpgYASItAIIM97JMJAAB1CUiJBeOnBgDrDEiNPdqnBgDolRcAAEiNBb5b +AQCEAEiLBbVbAQBIiQV2kgkASI0Fn1sBAIQASIsFllsBAEiJBVeSCQBIiywkSIPECMPomRQAAOkU +/v//zMzMzMzMzMzMzMzMzMzMzMzMzMxJO2YQD4afAAAASIPsKEiJbCQgSI1sJCBIixWhkgkASIXA +vgAAAABID0zGSIXSfm9Ji3YwRIuGIAEAAESLjiQBAABFicJBweARRTHQRYnKRTHBQcHoB0UxyEWJ +0UHB6hBFMcJHjQQRSInBRInASYnTSJlJ9/tEiY4gAQAARImWJAEAAEiF0nUXSI1TAUiJyEyJ20iJ +0b8DAAAA6CFE/f9Ii2wkIEiDxCjDSIlEJAhIiVwkEOjIEwAASItEJAhIi1wkEOk5////zMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzEiD7BhIiWwkEEiNbCQQSItEJBhIjVwkIOgjRP7/SItsJBBIg8QY +w8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxIg+woSIlsJCBIjWwkIEmLTjD/gQgBAABIjUwkMA8f +AEk5TnAPgtUBAABMiXQkEEnHhqgAAAAAAAAASYtOMEiLgeAAAABIiUQkGEjHgeAAAAAAAAAA6CZM +/v+EwHUnSItMJBBIx4HQAAAAAAAAAIA92yQHAAAPhOAAAABIi0QkGOk/AQAAgD3EJAcAAHRLSItE +JBBIi1AwSIuy0AAAAEiLfCQYSDn+dQuLkqQCAAA5VhR0K0iNBZ5bAQBIiQQkZpDoWxEAAEUPV/9k +TIs0Jfj///9Ii0QkEOsFSItEJBBIi1AwSIuS0AAAAP9CFLsDAAAAuQIAAADoxe39/0iLRCQQSMdA +cAAAAABIi1Aw/4oIAQAAgLixAAAAAHQKSMdAEN76///rDkiLCEiBwaADAABIiUgQxoC3AAAAAIA9 +N64GAAB0DuhYjP7/hMB1Bejv0f3/SItsJCBIg8Qow0iLUTD/iggBAABIjQUcWQEA6C8QAABIi0wk +EEjHQXAAAAAASItRMEiLktAAAAD/QhTGgbcAAAAASItsJCBIg8Qoww8fAOgbMwAARQ9X/2RMizQl ++P///0iLRCQQSItMJBhIichIi0wkEEiFwHQPSItRMItYFDmapAIAAHTF6CMTAABFD1f/ZEyLNCX4 +////SIsEJEiLTCQQSImB0AAAAOlc////SI0FcEoBALstAAAA6DCn/f+QzMzMzMzMzMzMzMzMzMzM +STtmEA+G9wAAAEiD7BhIiWwkEEiNbCQQSIlEJCAPHwBIg/sDfyZIhdt0a0iD+wMPhX8AAABmgThh +bHV4gHgCbHVyuAYAAADphgAAAEiD+wR1DoE4bm9uZXVaMcDrdGaQSIP7BXUagThjcmFzdUaAeARo +dUC4CwAAAOtXDx9EAABIg/sGdS6BOHNpbmd1D2aBeARsZXUHuAQAAADrNYE4c3lzdHUPZoF4BGVt +dQe4CgAAAOse6Gcs//+E23QQicGQSDnIdQjB4AKDyALrBbgCAAAAgD1UjQkAAHUJgD1JjQkAAHQD +g8gBCwWjjQkASI0NfG4GAIcBSItsJBBIg8QYkMNIiUQkCEiJXCQQ6DAQAABIi0QkCEiLXCQQ6eH+ +///MSIPsGEiJbCQQSI1sJBBIjQUQTgEAuzIAAADo4aX9/5BJO2YQdndIg+xASIlsJDhIjWwkOEiJ +RCQw6KIs//9mkEiFwHQ/SIlEJCDoESv//0iF/3Y+SIlcJBhIiUQkKEiJyEiLXCQwSItMJCDoMCAA +AEiLRCQoSItcJBhIi2wkOEiDxEDDMcAx20iLbCQ4SIPEQMMxwEiJ+ejkFgAAkEiJRCQI6HkPAABI +i0QkCOlv////zMzMzMzMzMzMzMzMzMzMZEiJFCX4////SYnWSIsjSItDIEiLUxhIi2swSMcDAAAA +AEjHQyAAAAAASMdDGAAAAABIx0MwAAAAAEiLWwj/40yNDbkNAABNiU5ATI1MJAhNiU44ScdGWAAA +AABJiW5oTYtOUE2FyXQF6FIQAADDzMzMzMzMzMzMzMzMzMzMzMxkSIk8Jfj///9Jif7DzMzMzMzM +zMzMzMzMzMzMzMzMzGZID27DZg/EwQTzD3DAAGYPb8hmD+8F5Y8JAGYPONzASIP5EHIsD4SFAAAA +SIP5IA+GgQAAAEiD+UAPhr4AAABIgfmAAAAAD4ZTAQAA6doCAABIhcl0T0iDwBBmqfAPdC3zD29I +8EgByUiNBZBmAgBmD9sMyGYP78hmDzjcyWYPONzJZg843MlmSA9+yMPzD29MCOBIAclIjQViZwIA +Zg84AAzI689mDzjcwGZID37Aw/MPbwjrvmYP7w1RjwkAZg843MnzD28Q8w9vXAjwZg/v0GYP79lm +Dzjc0mYPONzbZg843NJmDzjc22YPONzSZg843NtmD+/TZkgPftDDZg9v0WYPb9lmD+8NAo8JAGYP +7xUKjwkAZg/vHRKPCQBmDzjcyWYPONzSZg843NvzD28g8w9vaBDzD290CODzD298CPBmD+/gZg/v +6WYP7/JmD+/7Zg843ORmDzjc7WYPONz2Zg843P9mDzjc5GYPONztZg843PZmDzjc/2YPONzkZg84 +3O1mDzjc9mYPONz/Zg/v5mYP7+9mD+/lZkgPfuDDZg9v0WYPb9lmD2/hZg9v6WYPb/FmD2/5Zg/v +DVCOCQBmD+8VWI4JAGYP7x1gjgkAZg/vJWiOCQBmD+8tcI4JAGYP7zV4jgkAZg/vPYCOCQBmDzjc +yWYPONzSZg843NtmDzjc5GYPONztZg843PZmDzjc//NED28A80QPb0gQ80QPb1Ag80QPb1gw80QP +b2QIwPNED29sCNDzRA9vdAjg80QPb3wI8GZED+/AZkQP78lmRA/v0mZED+/bZkQP7+RmRA/v7WZE +D+/2ZkQP7/9mRQ843MBmRQ843MlmRQ843NJmRQ843NtmRQ843ORmRQ843O1mRQ843PZmRQ843P9m +RQ843MBmRQ843MlmRQ843NJmRQ843NtmRQ843ORmRQ843O1mRQ843PZmRQ843P9mRQ843MBmRQ84 +3MlmRQ843NJmRQ843NtmRQ843ORmRQ843O1mRQ843PZmRQ843P9mRQ/vxGZFD+/NZkUP79ZmRQ/v +32ZFD+/CZkUP78tmRQ/vwWZFD+//ZkwPfsDDZg9v0WYPb9lmD2/hZg9v6WYPb/FmD2/5Zg/vDcSM +CQBmD+8VzIwJAGYP7x3UjAkAZg/vJdyMCQBmD+8t5IwJAGYP7zXsjAkAZg/vPfSMCQBmDzjcyWYP +ONzSZg843NtmDzjc5GYPONztZg843PZmDzjc//NED29ECIDzRA9vTAiQ80QPb1QIoPNED29cCLDz +RA9vZAjA80QPb2wI0PNED290CODzRA9vfAjwZkQP78BmRA/vyWZED+/SZkQP79tmRA/v5GZED+/t +ZkQP7/ZmRA/v/0j/yUjB6QdmRQ843MBmRQ843MlmRQ843NJmRQ843NtmRQ843ORmRQ843O1mRQ84 +3PZmRQ843P/zD28A8w9vSBDzD29QIPMPb1gwZkQPONzAZkQPONzJZkQPONzSZkQPONzb8w9vYEDz +D29oUPMPb3Bg8w9veHBmRA843ORmRA843O1mRA843PZmRA843P9IBYAAAABI/8kPhWr///9mRQ84 +3MBmRQ843MlmRQ843NJmRQ843NtmRQ843ORmRQ843O1mRQ843PZmRQ843P9mRQ843MBmRQ843Mlm +RQ843NJmRQ843NtmRQ843ORmRQ843O1mRQ843PZmRQ843P9mRQ843MBmRQ843MlmRQ843NJmRQ84 +3NtmRQ843ORmRQ843O1mRQ843PZmRQ843P9mRQ/vxGZFD+/NZkUP79ZmRQ/v32ZFD+/CZkUP78tm +RQ/vwWZFD+//ZkwPfsDDzMzMzMzMZEyLNCX4////SIPsKEiJbCQgSI1sJCBNi2YgTYXkdRpJx8QA +AAAAzEnHxAEAAADMSItsJCBIg8Qow0yNbCQwTTksJHXbSYkkJOvVzMzMzMzMzMzMzMzMzMzMzMzM +zMzMZEyLNCX4////SIPsSEiJbCRASI1sJEBNi2YgTYXkdRpJx8QAAAAAzEnHxAEAAADMSItsJEBI +g8RIw0yNbCRQTTksJHXbSYkkJOvVzMzMzMzMzMzMzMzMzMzMzMzMzMzMZEyLNCX4////TI1kJPhN +O2YQdkBIgeyIAAAASImsJIAAAABIjawkgAAAAE2LZiBNheR1J0nHxAAAAADMScfEAQAAAMxIi6wk +gAAAAEiBxIgAAADD6EcIAADrpUyNrCSQAAAATTksJHXLSYkkJOvFzMzMzMzMzMzMzMzMzMzMzMxk +TIs0Jfj///9MjaQkeP///007ZhB2QEiB7AgBAABIiawkAAEAAEiNrCQAAQAATYtmIE2F5HUnScfE +AAAAAMxJx8QBAAAAzEiLrCQAAQAASIHECAEAAMPoxAcAAOuiTI2sJBABAABNOSwkdctJiSQk68XM +zMzMzMzMzMzMzMzMzGRMizQl+P///0yNpCR4/v//TTtmEHZASIHsCAIAAEiJrCQAAgAASI2sJAAC +AABNi2YgTYXkdSdJx8QAAAAAzEnHxAEAAADMSIusJAACAABIgcQIAgAAw+hEBwAA66JMjawkEAIA +AE05LCR1y0mJJCTrxczMzMzMzMzMzMzMzMzMZEyLNCX4////TI2kJHj8//9NO2YQdkBIgewIBAAA +SImsJAAEAABIjawkAAQAAE2LZiBNheR1J0nHxAAAAADMScfEAQAAAMxIi6wkAAQAAEiBxAgEAADD +6MQGAADrokyNrCQQBAAATTksJHXLSYkkJOvFzMzMzMzMzMzMzMzMzMxkTIs0Jfj///9MjaQkePj/ +/007ZhB2QEiB7AgIAABIiawkAAgAAEiNrCQACAAATYtmIE2F5HUnScfEAAAAAMxJx8QBAAAAzEiL +rCQACAAASIHECAgAAMPoRAYAAOuiTI2sJBAIAABNOSwkdctJiSQk68XMzMzMzMzMzMzMzMzMzGRM +izQl+P///0mJ5EmB7IgPAAByRk07ZhB2QEiB7AgQAABIiawkABAAAEiNrCQAEAAATYtmIE2F5HUn +ScfEAAAAAMxJx8QBAAAAzEiLrCQAEAAASIHECBAAAMPowAUAAOueTI2sJBAQAABNOSwkdctJiSQk +68XMzMzMzMzMzMzMZEyLNCX4////SYnkSYHsiB8AAHJGTTtmEHZASIHsCCAAAEiJrCQAIAAASI2s +JAAgAABNi2YgTYXkdSdJx8QAAAAAzEnHxAEAAADMSIusJAAgAABIgcQIIAAAw+hABQAA655Mjawk +ECAAAE05LCR1y0mJJCTrxczMzMzMzMzMzMxkTIs0Jfj///9JieRJgeyIPwAAckZNO2YQdkBIgewI +QAAASImsJABAAABIjawkAEAAAE2LZiBNheR1J0nHxAAAAADMScfEAQAAAMxIi6wkAEAAAEiBxAhA +AADD6MAEAADrnkyNrCQQQAAATTksJHXLSYkkJOvFzMzMzMzMzMzMzGRMizQl+P///0mJ5EmB7Ih/ +AAByRk07ZhB2QEiB7AiAAABIiawkAIAAAEiNrCQAgAAATYtmIE2F5HUnScfEAAAAAMxJx8QBAAAA +zEiLrCQAgAAASIHECIAAAMPoQAQAAOueTI2sJBCAAABNOSwkdctJiSQk68XMzMzMzMzMzMzMZEyL +NCX4////SYnkSYHsiP8AAHJGTTtmEHZASIHsCAABAEiJrCQAAAEASI2sJAAAAQBNi2YgTYXkdSdJ +x8QAAAAAzEnHxAEAAADMSIusJAAAAQBIgcQIAAEAw+jAAwAA655MjawkEAABAE05LCR1y0mJJCTr +xczMzMzMzMzMzMxIizwkSI10JAjpEgAAAMzMzMzMzMzMzMzMzMzMzMzMzEiJ+EiJ80iD7CdIg+Tw +SIlEJBBIiVwkGEiNPWGZBgBIjZwkaAD//0iJXxBIiV8YSIkfSIlnCLgAAAAAD6KJxoP4AHQzgftH +ZW51dR6B+mluZUl1FoH5bnRlbHUOxgUHgAkAAcYFBIAJAAG4AQAAAA+iiQVNgAkASIsFqpQGAEiF +wHQxSI01FvT//0jHwgAAAABIx8EAAAAA/9BIjQ3fmAYASIsBSAWgAwAASIlBEEiJQRjrLUiNPe2a +BgDowCMAAGRIxwQl+P///yMBAABIiwXUmgYASD0jAQAAdAXo/wMAAEiNDZiYBgBkSIkMJfj///9I +jQUomgYASIkISIlBMPzoOykAAItEJBCJBCRIi0QkGEiJRCQI6OUoAADoACcAAOj7JwAASI0FXEkC +AFBqAOhsKAAAWFjoRQAAAOigAwAAw0iNBbgGAADDzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzDzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzOjbJwAAw8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMSItcJAhIi1MQSIsK6W/y///MzMzMzMzMzMzMzMzMzMxIicJIixwkSYleQEiNXCQISYleOEmJ +bmhJi14wSIszTDn2dQXpdyYAAEyJ8EmJ9mRMiTQl+P///0mLZjhQTIsiQf/UWOmXJgAAw8zMzMzM +zMzMzMzMzMzMzMzMzMzMzMzDzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEiLfCQIZEiL +BCX4////SItYMEg7Q1B0W0iLE0g50HRTSDuDwAAAAHVS6PLx//9kSIkUJfj///9JidZIi1o4SInc +SIn6SIs//9dkSIsEJfj///9Ii1gwSIuDwAAAAGRIiQQl+P///0iLYDhIx0A4AAAAAMNIifpIiz// +50iNBR4oAAD/0M0DzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxkSIscJfj///9Ii1swSIszZEg5 +NCX4////dQro4CUAAOjbAQAASItzUGRIOTQl+P///3UK6OclAADowgEAAEiLRCQISIlDEEiNRCQQ +SIlDCGRIizQl+P///0iJcxhIiwQkSIlGQEiNRCQISIlGOEiJbmhIiVZQSIsbZEiJHCX4////SItj +OOg1JwAA6HABAADDzMzMzMzMzMzMzMzMzMzMugAAAADpVv///8zMzMzMzMzMzMzMzMzMzMzMzMzM +zMyLRCQI85CD6AF1+cPMzMzMzMzMzMzMzMzMzMzMzMzMzMPMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMSItUJAhIi1wkEEiNY/hIi2wk+EiDLCQFSIsa/+PMzMxIi0QkCEiLXCQQSIniZEyL +BCX4////SYP4AHRzTYtAMEmLMGRIizwl+P///0g5/nReSYtwUEg5/nRVSYsw6B7w//9kSIk0Jfj/ +//9Ii2Y4SIPsQEiD5PBIiXwkMEiLfwhIKddIiXwkKEiJ30iJ2f/QSIt8JDBIi3cISCt0JChkSIk8 +Jfj///9IifSJRCQYw0iD7EBIg+TwSMdEJDAAAAAASIlUJChIid9Iidn/0EiLdCQoSIn0iUQkGMPM +zMzMzMxIi1wkCGRIiRwl+P///8PMzMzMzMzMzMzMzMzMzMzMzM0D6/7MzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMZEiLBCX4////SDlgCHcF6Mz///9IOyB3BejC////w8yAPUd7CQABdQUPrujr +Aw+u8A8xSMHiIEgB0EiJRCQIw4A9K3sJAAB0Bely7///6S1Y+//MzMzMzMzMzMzMzMzMgD0LewkA +AHQsZkgPbsNmDzoiAAJmDzjcBUN/CQBmDzjcBUp/CQBmDzjcBVF/CQBmSA9+wMPp5ln7/8zMzMzM +zIA9y3oJAAB0LWZID27DZkgPOiIAAWYPONwFAn8JAGYPONwFCX8JAGYPONwFEH8JAGZID37Aw+kF +Wvv/zMzMzMxIjQXZVQIASI0d0lYCAEgJ2EipDwAAAA+URCQIw8zMzJDoeiMAAJDMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMZEyLNCX4////RQ9X/+kuyP7/zMzMzMzMzMzMzMzMzMxIg+x4SIlsJHBI +jWwkcEyJZCRgTIlsJGhNi24wTYut0AAAAE2LpcAWAABNjWQkEE2JpcAWAABNO6XIFgAASYlEJPBM +iy9NiWwk+HQXTItkJGBMi2wkaEiJB0iLbCRwSIPEeMNIiTwkSIlEJAhIiVwkEEiJTCQYSIlUJCBI +iXQkKEiJbCQwTIlEJDhMiUwkQEyJVCRITIlcJFBMiXwkWOj9IAAASIs8JEiLRCQISItcJBBIi0wk +GEiLVCQgSIt0JChIi2wkMEyLRCQ4TItMJEBMi1QkSEyLXCRQTIt8JFjpaf///8zMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMSJHo+f7//0iRw8zMzMzMzMzMzMzMzMzMzMzMzMzMzMxIkujZ/v// +SJLDzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEiT6Ln+//9Ik8PMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +SJbomf7//0iWw8zMzMzMzMzMzMzMzMzMzMzMzMzMzMxJkOh5/v//SZDDzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzEmR6Fn+//9JkcPMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSIHsoAAAAEiJrCSYAAAASI2s +JJgAAABMiXwkIEyJdCQoTIlsJDBMiWQkOEyJXCRATIlUJEhMiUwkUEyJRCRYSIl8JGBIiXQkaEiJ +bCRwSIlcJHhIiZQkgAAAAEiLlCSQAAAASImMJIgAAABIiYQkkAAAAEiJVCQYSIuEJKAAAABIiQQk +6NceAABIi0QkCEiFwHQbSIkEJEiLRCQQSIlEJAhJx8QIAAAAzOlvAQAASItEJBhIg/ggdxVIjQVg +8f//SIkEJOjXHgAA6U8BAABIg/hAdxVIjQWl8f//SIkEJOi8HgAA6TQBAABIPYAAAAB3FUiNBejx +//9IiQQk6J8eAADpFwEAAEg9AAEAAHcVSI0FS/L//0iJBCTogh4AAOn6AAAASD0AAgAAdxVIjQWu +8v//SIkEJOhlHgAA6d0AAABIPQAEAAB3FUiNBRHz//9IiQQk6EgeAADpwAAAAEg9AAgAAHcVSI0F +dPP//0iJBCToKx4AAOmjAAAASD0AEAAAdxVIjQXX8///SIkEJOgOHgAA6YYAAABIPQAgAAB3EkiN +BTr0//9IiQQk6PEdAADrbEg9AEAAAHcSSI0FoPT//0iJBCTo1x0AAOtSSD0AgAAAdxJIjQUG9f// +SIkEJOi9HQAA6zhIPQAAAQB3EkiNBWz1//9IiQQk6KMdAADrHkiNBcpBAgBIiQQkSMdEJAgUAAAA +ScfECAAAAMzrAEnHxBAAAADMSIuEJJAAAABIi4wkiAAAAEiLlCSAAAAASItcJHhIi2wkcEiLdCRo +SIt8JGBMi0QkWEyLTCRQTItUJEhMi1wkQEyLZCQ4TItsJDBMi3QkKEyLfCQgSIusJJgAAABIgcSg +AAAAw8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSIPsGEiJbCQQSI1sJBBIi0QkIEiJBCRIi0QkKEiJ +RCQIScfEAgAAAMxIi2wkEEiDxBjDzMzMzMzMzMzMzMzMzEiJy+nYYv3/zMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMSInL6Vhj/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxIichIidPp1WP9/8zMzMzM +zMzMzMzMzMzMzMzMzMzMzEiJyEiJ0+lVZP3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMSInISInT6dVk +/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMxIichIidPpVWX9/8zMzMzMzMzMzMzMzMzMzMzMzMzMzEiJ +y+nYZf3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSInL6Vhm/f/MzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMxIidDp2Gb9/8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEiJ0OlYZ/3/zMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMRA8RP0QPEX8QRA8RfyBEDxF/MEiNf0BEDxE/RA8RfxBEDxF/IEQPEX8wSI1/ +QEQPET9EDxF/EEQPEX8gRA8RfzBIjX9ARA8RP0QPEX8QRA8RfyBEDxF/MEiNf0BEDxE/RA8RfxBE +DxF/IEQPEX8wSI1/QEQPET9EDxF/EEQPEX8gRA8RfzBIjX9ARA8RP0QPEX8QRA8RfyBEDxF/MEiN +f0BEDxE/RA8RfxBEDxF/IEQPEX8wSI1/QEQPET9EDxF/EEQPEX8gRA8RfzBIjX9ARA8RP0QPEX8Q +RA8RfyBEDxF/MEiNf0BEDxE/RA8RfxBEDxF/IEQPEX8wSI1/QEQPET9EDxF/EEQPEX8gRA8RfzBI +jX9ARA8RP0QPEX8QRA8RfyBEDxF/MEiNf0BEDxE/RA8RfxBEDxF/IEQPEX8wSI1/QEQPET9EDxF/ +EEQPEX8gRA8RfzBIjX9ARA8RP0QPEX8QRA8RfyBEDxF/MEiNf0DDzMzMzMzMzMzMzMzMzMzMDxAG +SIPGEA8RB0iDxxAPEAZIg8YQDxEHSIPHEA8QBkiDxhAPEQdIg8cQDxAGSIPGEA8RB0iDxxAPEAZI +g8YQDxEHSIPHEA8QBkiDxhAPEQdIg8cQDxAGSIPGEA8RB0iDxxAPEAZIg8YQDxEHSIPHEA8QBkiD +xhAPEQdIg8cQDxAGSIPGEA8RB0iDxxAPEAZIg8YQDxEHSIPHEA8QBkiDxhAPEQdIg8cQDxAGSIPG +EA8RB0iDxxAPEAZIg8YQDxEHSIPHEA8QBkiDxhAPEQdIg8cQDxAGSIPGEA8RB0iDxxAPEAZIg8YQ +DxEHSIPHEA8QBkiDxhAPEQdIg8cQDxAGSIPGEA8RB0iDxxAPEAZIg8YQDxEHSIPHEA8QBkiDxhAP +EQdIg8cQDxAGSIPGEA8RB0iDxxAPEAZIg8YQDxEHSIPHEA8QBkiDxhAPEQdIg8cQDxAGSIPGEA8R +B0iDxxAPEAZIg8YQDxEHSIPHEA8QBkiDxhAPEQdIg8cQDxAGSIPGEA8RB0iDxxAPEAZIg8YQDxEH +SIPHEA8QBkiDxhAPEQdIg8cQDxAGSIPGEA8RB0iDxxAPEAZIg8YQDxEHSIPHEA8QBkiDxhAPEQdI +g8cQDxAGSIPGEA8RB0iDxxAPEAZIg8YQDxEHSIPHEA8QBkiDxhAPEQdIg8cQDxAGSIPGEA8RB0iD +xxAPEAZIg8YQDxEHSIPHEA8QBkiDxhAPEQdIg8cQDxAGSIPGEA8RB0iDxxAPEAZIg8YQDxEHSIPH +EA8QBkiDxhAPEQdIg8cQDxAGSIPGEA8RB0iDxxAPEAZIg8YQDxEHSIPHEA8QBkiDxhAPEQdIg8cQ +DxAGSIPGEA8RB0iDxxAPEAZIg8YQDxEHSIPHEA8QBkiDxhAPEQdIg8cQDxAGSIPGEA8RB0iDxxAP +EAZIg8YQDxEHSIPHEA8QBkiDxhAPEQdIg8cQDxAGSIPGEA8RB0iDxxAPEAZIg8YQDxEHSIPHEA8Q +BkiDxhAPEQdIg8cQDxAGSIPGEA8RB0iDxxAPEAZIg8YQDxEHSIPHEA8QBkiDxhAPEQdIg8cQDxAG +SIPGEA8RB0iDxxAPEAZIg8YQDxEHSIPHEA8QBkiDxhAPEQdIg8cQDxAGSIPGEA8RB0iDxxAPEAZI +g8YQDxEHSIPHEA8QBkiDxhAPEQdIg8cQDxAGSIPGEA8RB0iDxxDDzMzMzMzMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzEiJx0gxwEiF2w+EuAEAAEiD+wIPhqcBAABIg/sED4alAQAASIP7CA+CpAEA +AA+EpQEAAEiD+xAPhp8BAABIg/sgD4aeAQAASIP7QA+GoQEAAEiB+4AAAAAPhq4BAABIgfsAAQAA +D4bVAQAAgD2xdQkAAQ+ElwAAAPNED38/80QPf38Q80QPf38g80QPf38w80QPf39A80QPf39Q80QP +f39g80QPf39w80QPf7+AAAAA80QPf7+QAAAA80QPf7+gAAAA80QPf7+wAAAA80QPf7/AAAAA80QP +f7/QAAAA80QPf7/gAAAA80QPf7/wAAAASIHrAAEAAEiBxwABAABIgfsAAQAAD4Nu////6ff+///F +/e/ASIH7AAAAAnNGxf5/B8X+f0cgxf5/R0DF/n9HYEiB64AAAABIgceAAAAASIH7gAAAAHPWxf5/ +RB/gxf5/RB/Axf5/RB+gxf5/RB+Axfh3w8X+fwdIif5Ig8cgSIPn4Egp/kgB88X95wfF/edHIMX9 +50dAxf3nR2BIgeuAAAAASIHHgAAAAEiB+4AAAABz1g+u+MX+f0Qf4MX+f0QfwMX+f0QfoMX+f0Qf +gMX4d8OIB4hEH//Dw2aJB2aJRB/+w4kHiUQf/MNIiQfDSIkHSIlEH/jD80QPfz/zRA9/fB/ww/NE +D38/80QPf38Q80QPf3wf4PNED398H/DD80QPfz/zRA9/fxDzRA9/fyDzRA9/fzDzRA9/fB/A80QP +f3wf0PNED398H+DzRA9/fB/ww/NED38/80QPf38Q80QPf38g80QPf38w80QPf39A80QPf39Q80QP +f39g80QPf39w80QPf3wfgPNED398H5DzRA9/fB+g80QPf3wfsPNED398H8DzRA9/fB/Q80QPf3wf +4PNED398H/DDzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSInHSIneSInLSIXbD4T3AAAASIP7Ag+G +4AAAAEiD+wQPgukAAAAPht4AAABIg/sID4LmAAAAD4TtAAAASIP7EA+G6gAAAEiD+yAPhvEAAABI +g/tAD4b8AAAASIH7gAAAAA+GGgEAAEiB+wABAAAPhmQBAAD2BfZrCQABD4UlAwAASDn+djpIgfsA +CAAAD4YJAgAAgD3McgkAAXURifAJ+KkHAAAAdAZIidnzpMNIidlIwekDSIPjB/NIpelJ////SInx +SAHZSDn5drtIAd9IAd79SInZSMHpA0iD4wdIg+8ISIPuCPNIpfxIg8cISIPGCEgp30gp3ukN//// +igaKTB7/iAeITB//w8OLBokHw2aLBopOAmaJB4hPAsOLBotMHvyJB4lMH/zDSIsGSIkHw0iLBkiL +TB74SIkHSIlMH/jD8w9vBvMPb0we8PMPfwfzD39MH/DD8w9vBvMPb04Q8w9vVB7g8w9vXB7w8w9/ +B/MPf08Q8w9/VB/g8w9/XB/ww/MPbwbzD29OEPMPb1Yg8w9vXjDzD29kHsDzD29sHtDzD290HuDz +D298HvDzD38H8w9/TxDzD39XIPMPf18w8w9/ZB/A8w9/bB/Q8w9/dB/g8w9/fB/ww/MPbwbzD29O +EPMPb1Yg8w9vXjDzD29mQPMPb25Q8w9vdmDzD29+cPNED29EHoDzRA9vTB6Q80QPb1QeoPNED29c +HrDzRA9vZB7A80QPb2we0PNED290HuDzRA9vfB7w8w9/B/MPf08Q8w9/VyDzD39fMPMPf2dA8w9/ +b1DzD393YPMPf39w80QPf0QfgPNED39MH5DzRA9/VB+g80QPf1wfsPNED39kH8DzRA9/bB/Q80QP +f3Qf4PNED398H/BmRQ/v/8NIgesAAQAA8w9vBvMPb04Q8w9vViDzD29eMPMPb2ZA8w9vblDzD292 +YPMPb35w80QPb4aAAAAA80QPb46QAAAA80QPb5agAAAA80QPb56wAAAA80QPb6bAAAAA80QPb67Q +AAAA80QPb7bgAAAA80QPb77wAAAA8w9/B/MPf08Q8w9/VyDzD39fMPMPf2dA8w9/b1DzD393YPMP +f39w80QPf4eAAAAA80QPf4+QAAAA80QPf5egAAAA80QPf5+wAAAA80QPf6fAAAAA80QPf6/QAAAA +80QPf7fgAAAA80QPf7/wAAAASIH7AAEAAEiNtgABAABIjb8AAQAAD40A////ZkUP7//pY/z//0iJ ++Ugp8Ug52Q+CrAEAAEiB+wAAEAAPg8MAAABIjQweSYn68w9vaYDzD29xkEjHwIAAAABIg+fgSIPH +IPMPb3mg80QPb0GwSYn7TSnT80QPb0nA80QPb1HQTCnb80QPb1ng80QPb2Hwxf5vJkwB3kgpw8X+ +bwbF/m9OIMX+b1ZAxf5vXmBIAcbF/X8Hxf1/TyDF/X9XQMX9f19gSAHHSCnDd89IAcNIAfvEwX5/ +IsX4d/MPf2uA8w9/c5DzD397oPNED39DsPNED39LwPNED39T0PNED39b4PNED39j8MNIjQwe8w9v +bB6A8w9vcZDzD295oPNED29BsPNED29JwPNED29R0PNED29Z4PNED29h8MX+byZJifhIg+fgSIPH +IEmJ+k0pwkwp00wB1kiNDB9IgeuAAAAADxiGwAEAAA8YhoACAADF/m8Gxf5vTiDF/m9WQMX+b15g +SIHGgAAAAMX95wfF/edPIMX951dAxf3nX2BIgceAAAAASIHrgAAAAHe1D674xMF+fyDF+HfzD39p +gPMPf3GQ8w9/eaDzRA9/QbDzRA9/ScDzRA9/UdDzRA9/WeDzRA9/YfDDSIn48w9vLvMPb3YQSAHf +8w9vfiDzRA9vRjBMjVfgSYn780QPb05A80QPb1ZQSYPjH/NED29eYPNED29mcEwx30gB3sX+b2bg +TCneTCnbSIH7AAAQAHd7SIHrgAAAAMX+b0bgxf5vTsDF/m9WoMX+b16ASIHugAAAAMX9f0fgxf1/ +T8DF/X9XoMX9f1+ASIHvgAAAAEiB64AAAAB3wcTBfn8ixfh38w9/KPMPf3AQ8w9/eCDzRA9/QDDz +RA9/SEDzRA9/UFDzRA9/WGDzRA9/YHDDSIHrgAAAAA8YhkD+//8PGIaA/f//xf5vRuDF/m9OwMX+ +b1agxf5vXoBIge6AAAAAxf3nR+DF/edPwMX951egxf3nX4BIge+AAAAASIHrgAAAAHezD674xMF+ +fyLF+HfzD38o8w9/cBDzD394IPNED39AMPNED39IQPNED39QUPNED39YYPNED39gcMPMVUiJ5ZxI +gexwAQAASIkEJEiJTCQISIlUJBBIiVwkGEiJdCQgSIl8JChMiUQkMEyJTCQ4TIlUJEBMiVwkSEyJ +ZCRQTIlsJFhMiXQkYEyJfCRoDxFEJHAPEYwkgAAAAA8RlCSQAAAADxGcJKAAAAAPEaQksAAAAA8R +rCTAAAAADxG0JNAAAAAPEbwk4AAAAEQPEYQk8AAAAEQPEYwkAAEAAEQPEZQkEAEAAEQPEZwkIAEA +AEQPEaQkMAEAAEQPEawkQAEAAEQPEbQkUAEAAEQPEbwkYAEAAOilDAAARA8QvCRgAQAARA8QtCRQ +AQAARA8QrCRAAQAARA8QpCQwAQAARA8QnCQgAQAARA8QlCQQAQAARA8QjCQAAQAARA8QhCTwAAAA +DxC8JOAAAAAPELQk0AAAAA8QrCTAAAAADxCkJLAAAAAPEJwkoAAAAA8QlCSQAAAADxCMJIAAAAAP +EEQkcEyLfCRoTIt0JGBMi2wkWEyLZCRQTItcJEhMi1QkQEyLTCQ4TItEJDBIi3wkKEiLdCQgSItc +JBhIi1QkEEiLTCQISIsEJEiBxHABAACdXcPMzMzMzMzMzMzMzMzMzMzMzOlb4///zMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMi3wkCLjnAAAADwXDzMzMzMzMzMzMzMzMzMzMzMzMzMxIi0QkCMcA +AAAAAL8AAAAAuDwAAAAPBc0D6/7MzMzMzL+c////SIt0JAiLVCQQRItUJBS4AQEAAA8FSD0B8P// +dgW4/////4lEJBjDzMzMzMzMzMzMzMzMzMzMzMzMzMyLfCQIuAMAAAAPBUg9AfD//3YFuP////+J +RCQQw8zMzEiLfCQISIt0JBCLVCQYuAEAAAAPBYlEJCDDzMzMzMzMi3wkCEiLdCQQi1QkGLgAAAAA +DwWJRCQgw8zMzMzMzMxIjXwkCLgWAAAADwWJRCQQw8zMzMzMzMzMzMzMzMzMzEiNfCQQi3QkCLgl +AQAADwWJRCQYw8zMzMzMzMzMzMzMSIPsGEiJbCQQSI1sJBC6AAAAAItEJCC5QEIPAPfxSIkEJLjo +AwAA9+JIiUQkCEiJ574AAAAAuCMAAAAPBUiLbCQQSIPEGMPMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +zMzMuLoAAAAPBYlEJAjDzMzMzMzMzMzMzMzMzMzMzMzMzMy4JwAAAA8FQYnEuLoAAAAPBYnGRInn +i1QkCLjqAAAADwXDzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMuCcAAAAPBYnHi3QkCLg+ +AAAADwXDzMzMzMzMzMzMzMy4JwAAAA8FSIlEJAjDzMzMzMzMzMzMzMzMzMzMzMzMzEiLfCQISIt0 +JBBIi1QkGLjqAAAADwXDzMzMzMzMzMzMSIt8JAhIi3QkEEiLVCQYuBsAAAAPBYlEJCDDzMzMzMxI +g+wYSIlsJBBIjWwkEEmJ5EmLXjBIi4tIAwAASIuTQAMAAEiJDCRIiVQkCEiNVCQgSItK+EiJi0gD +AABIiZNAAwAATDuzwAAAAHUHSIsTSItiOEiD7BBIg+TwvwEAAABIjTQkSIsF7WIJAEiD+AB0Pv/Q +SIsEJEiLVCQITInkSItMJAhIiYtAAwAASIsMJEiJi0gDAABIacAAypo7SAHQSIlEJCBIi2wkEEiD +xBjDSMfA5AAAAA8F67nMzMzMzMyLfCQISIt0JBBIi1QkGESLVCQguA4AAAAPBUg9AfD//3YLxwQl +8QAAAPEAAADDzMzMzMzMzMzMzMzMzMzMzMzMSIt8JAhIi3QkEEiLVCQYTItUJCC4DQAAAA8FiUQk +KMNIg+wYSIlsJBBIjWwkEEiLfCQgSIt0JChIi1QkMEiLBQR1BgBIieNIg+Tw/9BIidyJRCQ4SIts +JBBIg8QYw8zMSItEJAiLfCQQSIt0JBhIi1QkIFVIieVIg+Tw/9BIiexdw8zMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzEiD7DBIiWwkKEiNbCQoSIkcJEyJZCQITIlsJBBMiXQkGEyJfCQgSIPs +GEiJPCRIiXQkCEiJVCQQ6IMJAABIg8QYSIscJEyLZCQITItsJBBMi3QkGEyLfCQgSItsJChIg8Qw +w8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSIsFeXQGAEiFwHRvSIsF7XMGAEiFwHRjZEiL +BCX4////SIXAdFpIi0AwSIXAdEyLiDgBAACFyXRCSIuIwAAAAEiFyXQ2SItJcEiFyXQtTIuAQAEA +AE2FwHQhi4g8AQAAhcl1F0iLDRV0BgBMjQ0O////SIsFh3MGAP/g6QD///+D/xt19rgAAAAAuQEA +AABMjR0WXwkA8EEPsQt13kiLDdxzBgBMjQXVZgkATI0NDggAAEiLBUdzBgD/4MzMzMzMSMfADwAA +AA8FzQPMzMzMzMzMzMzMzMzMzMzMzMzMzMxIi3wkCEiLdCQQi1QkGESLVCQcRItEJCBEi0wkJLgJ +AAAADwVIPQHw//92FUj30Ej/wEjHRCQoAAAAAEiJRCQww0iJRCQoSMdEJDAAAAAAw8zMzMzMzMzM +zMzMzMzMzMxIg+wYSIlsJBBIjWwkEEiLfCQgSIt0JCiLVCQwi0wkNESLRCQ4RItMJDxIiwWfcgYA +SInjSIPk8EiJHCT/0EiLJCRIiUQkQEiLbCQQSIPEGMPMzMzMzMzMzMzMzMzMzMxIi3wkCEiLdCQQ +SMfACwAAAA8FSD0B8P//dgvHBCXxAAAA8QAAAMPMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSIPs +GEiJbCQQSI1sJBBIi3wkIEiLdCQoSIsFGXIGAEiJ40iD5PBIiRwk/9BIiyQkSItsJBBIg8QYw8zM +zMzMzEiLfCQISIt0JBCLVCQYSMfAHAAAAA8FiUQkIMPMzMzMSIt8JAiLdCQQi1QkFEyLVCQYTItE +JCBEi0wkKLjKAAAADwWJRCQww8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzIt8JAhIi3QkEEjHwgAA +AABJx8IAAAAAScfAAAAAAEyLbCQYTItMJCBMi2QkKEmD/QB0GEmD+QB0Ek2NhYgAAABJg8AISIHP +AAAIALg4AAAADwVIg/gAdAWJRCQww0iJ9EmD/QB0JkmD+QB0ILi6AAAADwVJiUVITYlpMGRMiQwl ++P///02JzujU4P//Qf/Uv28AAAC4PAAAAA8F6/LMzMxIi3wkCEiLdCQQSMfAgwAAAA8FSD0B8P// +dgvHBCXxAAAA8QAAAMPMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMSIPsKEiJbCQgSI1sJCBIg8cI +SIn+SMfHAhAAAEjHwJ4AAAAPBUg9AfD//3YLxwQl8QAAAPEAAABIi2wkIEiDxCjDzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzMzMuBgAAAAPBcPMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMxIi3wk +CEiLdCQQSItUJBi4zAAAAA8FiUQkIMPMzMzMzIt8JAi41QAAAA8FiUQkEMPMzMzMzMzMzMzMzMzM +zMzMi3wkCLgjAQAADwWJRCQQw8zMzMzMzMzMzMzMzMzMzMyLfCQIi3QkDItUJBBMi1QkGLjpAAAA +DwWJRCQgw8zMzIt8JAhIi3QkEItUJBhEi1QkHEnHwAAAAAC4GQEAAA8FiUQkIMPMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMyLfCQISMfGAgAAAEjHwgEAAAC4SAAAAA8Fw8zMzMzMzIt8JAhIx8YD +AAAASMfCAAAAALhIAAAADwWLfCQISMfGBAAAAEjHwgAIAAAJwrhIAAAADwXDzMzMzMzMzMzMzMxI +g+wYSIlsJBBIjWwkEEmJ5EmLXjBIjXQkIEiLi0gDAABIi5NAAwAASIkMJEiJVCQISItO+EiJi0gD +AABIibNAAwAATDuzwAAAAHUHSIsTSItiOEiD7BBIg+TwvwAAAABIiwXRWwkASIP4AHRQ/9C/AQAA +AEiNNCRIiwW5WwkA/9BIiwQkSItUJAhMieRIi0wkCEiJi0ADAABIiwwkSImLSAMAAEhpwADKmjtI +AdBIiUQkMEiLbCQQSIPEGMNIx8DkAAAADwW/AQAAAEiNNCRIx8DkAAAADwXrp8zMzMzMzEk7ZhB2 +MEiD7BhIiWwkEEiNbCQQTYtmIE2F5HUvSItKCEiJw0iJyOi0Pfv/SItsJBBIg8QYw0iJRCQIDx9E +AADom9v//0iLRCQI67RMjWwkIE05LCR1xkmJJCTrwMzMzEiD7BBIiWwkCEiNbCQISItEJBhFD1f/ +ZEyLNCX4////6NsQ+/9IiUQkIEiJXCQoSItsJAhIg8QQw8zMzMzMzMxIg+wQSIlsJAhIjWwkCEiL +RCQYRQ9X/2RMizQl+P///+jbFPv/SItsJAhIg8QQw8zMzMzMzMzMzMzMzMzMzMzMSIPsGEiJbCQQ +SI1sJBBIi0QkIEiLXCQoRQ9X/2RMizQl+P///+gWKf3/SItsJBBIg8QYw8zMzMzMzMzMzMzMzEUP +V/9kTIs0Jfj////pDj/9/8zMzMzMzMzMzMzMzMzMRQ9X/2RMizQl+P///+nugv3/zMzMzMzMzMzM +zMzMzMxIg+wQSIlsJAhIjWwkCEiLRCQYRQ9X/2RMizQl+P///+hbo/3/SItsJAhIg8QQw8zMzMzM +zMzMzMzMzMzMzMzMSIPsEEiJbCQISI1sJAhIi0QkGEUPV/9kTIs0Jfj////oW6P9/0iLbCQISIPE +EMPMzMzMzMzMzMzMzMzMzMzMzEUPV/9kTIs0Jfj////pbqP9/8zMzMzMzMzMzMzMzMzMRQ9X/2RM +izQl+P///+muo/3/zMzMzMzMzMzMzMzMzMxFD1f/ZEyLNCX4////6U6o/f/MzMzMzMzMzMzMzMzM +zOg72P//RQ9X/2RMizQl+P///8PMzMzMzMzMzMzMzMzMRQ9X/2RMizQl+P///+nuvv3/zMzMzMzM +zMzMzMzMzMxFD1f/ZEyLNCX4////6S4H/v/MzMzMzMzMzMzMzMzMzEiD7BhIiWwkEEiNbCQQi0Qk +IEiLXCQoRQ9X/2RMizQl+P///+gXGf7/SItsJBBIg8QYw8zMzMzMzMzMzMzMzMxFD1f/ZEyLNCX4 +////6Q4s/v/MzMzMzMzMzMzMzMzMzEiD7BhIiWwkEEiNbCQQi0QkIEiLXCQoRQ9X/2RMizQl+P// +/+h3bv7/SItsJBBIg8QYw8zMzMzMzMzMzMzMzMxFD1f/ZEyLNCX4////6S5z/v/MzMzMzMzMzMzM +zMzMzEiD7CBIiWwkGEiNbCQYi0QkKEiLXCQwSItMJDhFD1f/ZEyLNCX4////6NKW/v9Ii2wkGEiD +xCDDzMzMzMzMzMxFD1f/ZEyLNCX4////6c7W/v/MzMzMzMzMzMzMzMzMzEUPV/9kTIs0Jfj////p +jsj//8zMzMzMzMzMzMzMzMzMRQ9X/2RMizQl+P///+kO9v7/zMzMzMzMzMzMzMzMzMxIg+wgSIls +JBhIjWwkGEiJBCRIiVwkCOjk2P//RQ9X/2RMizQl+P///4tEJBBIi2wkGEiDxCDDzMzMzMzMzMzM +STtmEHZPSIPsMEiJbCQoSI1sJChNi2YgTYXkdUlIiUQkOEiFwHQoSIs4SItwCDHASI0df9sAALkP +AAAADx9AAOjb7/7/SItsJChIg8Qww+hsLvv/kEiJRCQI6OHX//9Ii0QkCOuaTI1sJDhNOSwkdaxJ +iSQk66bMzMzMzMzMzMxJO2YQdjBIg+wgSIlsJBhIjWwkGEiLUxhIOVAYdAQxwOsKuRQAAADodNb6 +/0iLbCQYSIPEIMNIiUQkCEiJXCQQ6HvX//9Ii0QkCEiLXCQQ66/MzMzMzMzMzMzMzMzMzMxJO2YQ +dnZIg+wgSIlsJBhIjWwkGEiLSAhIixNIizBmkEg5Swh1SkiLeBhIOXsYdUBIi3ggSDl7IHU2SIlE +JChIiVwkMEiJ8EiJ0+jx1fr/hMB0HUiLVCQwSItaEEiLVCQoSItCEEiLShjo0tX6/+sCMcBIi2wk +GEiDxCDDSIlEJAhIiVwkEOjV1v//SItEJAhIi1wkEOlm////zMzMzMzMiwg5C3UMSItICEg5SwgP +lMDDMcDDzMzMzMzMzMzMzMxJO2YQdllIg+wgSIlsJBhIjWwkGEiLUCBIOVMgdRhIiUQkKEiJXCQw +uRgAAADoTtX6/4TAdQQxwOsbSItUJDBIi1oYSItUJChIi0IYSItKIOgr1fr/SItsJBhIg8Qgw0iJ +RCQISIlcJBDoMtb//0iLRCQISItcJBDrhszMzMzMzEk7ZhB2bUiD7CBIiWwkGEiNbCQYSIsQSDkT +dUtIi1AISItLEEiLcBBIOVMIdTlIiUQkKEiJXCQwSInQSInzZpDom9n6/4TAdB5Ii1QkKEiNQhhI +i1QkMEiNWhi5GwAAAOib1Pr/6wIxwEiLbCQYSIPEIMNIiUQkCEiJXCQQDx8A6JvV//9Ii0QkCEiL +XCQQ6Wz////MzMzMzMzMzMzMzMxJO2YQdlBIg+wgSIlsJBhIjWwkGEiJRCQoSIlcJDC5BwAAAOg4 +1Pr/hMB1BDHA6xxIi1QkKEiNQghIi1QkMEiNWgi5QAAAAOgU1Pr/SItsJBhIg8Qgw0iJRCQISIlc +JBDoG9X//0iLRCQISItcJBDrj8zMzMzMzMzMzMzMzMzMzEk7ZhB2IkiD7CBIiWwkGEiNbCQYuRIA +AADowtP6/0iLbCQYSIPEIMNIiUQkCEiJXCQQ6MnU//9Ii0QkCEiLXCQQ673MzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzEk7ZhB2SEiD7CBIiWwkGEiNbCQYTYtmIE2F5HVCSIlEJChIhcB0IUiL +EEiLWAgPtkgQD7Z4EUiJ0OiiF/v/SItsJBhIg8Qgw+jTKvv/kEiJRCQI6EjU//9Ii0QkCOuhTI1s +JChNOSwkdbNJiSQk663MzMzMzMzMzMzMzMzMzMzMiwg5C3UeSItICEg5Swh1FItIEDlLEHUMSItI +GEg5SxgPlMDDMcDDzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2IkiD7CBIiWwkGEiNbCQY +uQsAAADootL6/0iLbCQYSIPEIMNIiUQkCEiJXCQQ6KnT//9Ii0QkCEiLXCQQ673MzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2UEiD7CBIiWwkGEiNbCQYSIlEJChIiVwkMLllAAAA6DjS ++v+EwHUEMcDrHEiLVCQoSI1CaEiLVCQwSI1aaLkgAAAA6BTS+v9Ii2wkGEiDxCDDSIlEJAhIiVwk +EOgb0///SItEJAhIi1wkEOuPzMzMzMzMzMzMzMzMzMzMSTtmEHYiSIPsIEiJbCQYSI1sJBi5rAQA +AOjC0fr/SItsJBhIg8Qgw0iJRCQISIlcJBDoydL//0iLRCQISItcJBDrvczMzMzMzMzMzMzMzMzM +zMzMzMzMzMzMzMzMzMzMSIsISDkLdQqLSAg5SwgPlMDDMcDDzMzMzMzMzMzMzMxJO2YQdlBIg+wg +SIlsJBhIjWwkGEiJRCQoSIlcJDC5GgAAAOg40fr/hMB1BDHA6xxIi1QkKEiNQhxIi1QkMEiNWhy5 +RAAAAOgU0fr/SItsJBhIg8Qgw0iJRCQISIlcJBDoG9L//0iLRCQISItcJBDrj8zMzMzMzMzMzMzM +zMzMzEk7ZhB2UEiD7CBIiWwkGEiNbCQYSIlEJChIiVwkMLk2AAAA6LjQ+v+EwHUEMcDrHEiLVCQo +SI1COEiLVCQwSI1aOLkgAAAA6JTQ+v9Ii2wkGEiDxCDDSIlEJAhIiVwkEOib0f//SItEJAhIi1wk +EOuPzMzMzMzMzMzMzMzMzMzMSTtmEHYiSIPsIEiJbCQYSI1sJBi5IQAAAOhC0Pr/SItsJBhIg8Qg +w0iJRCQISIlcJBDoSdH//0iLRCQISItcJBDrvczMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM +SIPsGEiJbCQQSI1sJBBNi2YgTYXkdXpIiUQkIA8fQABIhcB0ZkiLAEiFwHUTugcAAABIjQ3CxgAA +6zsPH0QAAEg96AMAAHUOugQAAABIjQ1sxAAA6yBIixUzQgYASIsNNEIGAEg5wXYdSMHgBEiLDAJI +i1QCCEiJyEiJ00iLbCQQSIPEGMPo9df//+gQJ/v/kEyNbCQgZg8fhAAAAAAAkE05LCQPhW3///9J +iSQk6WT////MzMzMzMzMzMzMzMzMSIPsCEiJLCRIjSwkTYtmIE2F5HVJSIlEJBBIhcB0OQ+2CID5 +G3IOuRMAAABIjTX52wAA6xRIweEESI0VoEYGAEiLNApIi0wKCEiJ8EiJy0iLLCRIg8QIw+iDJvv/ +kEyNbCQQTTksJHWsSYkkJOumzMzMzMzMzMzMzMzMzMzMzMxJO2YQdkNIg+wgSIlsJBhIjWwkGEiL +SAhIixNIizBmkEg5Swh1F0iLeBBIOXsQdQ1IifBIidPohc76/+sCMcBIi2wkGEiDxCDDSIlEJAhI +iVwkEOiIz///SItEJAhIi1wkEOuczMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzEk7ZhB2T0iD +7DBIiWwkKEiNbCQoTYtmIE2F5HVJSIlEJDhIhcB0KEiLOEiLcAgxwEiNHZ/SAAC5DwAAAA8fQADo +++b+/0iLbCQoSIPEMMPojCX7/5BIiUQkCOgBz///SItEJAjrmkyNbCQ4TTksJHWsSYkkJOumzMzM +zMzMzMzMSIPsCEiJLCRIjSwkTYtmIE2F5HUjSIlEJBBIhcB0E0iLCEiLWAhIichIiywkSIPECMPo +KSX7/5BMjWwkEA8fAE05LCR1z0mJJCTryczMzMzMzMzMzMzMzMzMzMzMzMzMw8zMzMzMzMzMzMzM +zMzMzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEB +QgABXwABYwABZgABZwABbQABbgABcAABcwABeAABeQAAAmJwAAJmZAACZm4AAmcwAAJncAACaGkA +AmlkAAJsbwACbHIAAnBjAAJwcAACcjEAAnIyAAJyZAACcmcAAnJ0AAJzcAACdDEAAnQyAAJ3ZAAC +d2cAAnd0AAgICP8BA0dldAEDU2V0AANhZGQAA2FyZwADYmFkAANic3MAA2J1ZgADY2FzAANlbmQA +A2VycgADZnVuAANnY3cAA2dldAADa2V5AANsZW4AA21PUwADbXNnAANvYmoAA29mZgADcGNzAANw +b3AAA3B0cgADcHV0AANyZXQAA3NlcQADc2V0AANzaWcAA3NpegADc3RyAAN0YWcAA3RscwADdHlw +AAAEKmludAEEQWRkcgEESW50cwEETmFtZQEEUHRycwAEYWRkcgAEYXJncAAEYXJncwAEYmFzZQAE +Yml0cAAEY29kZQAEY3R4dAAEY3VyZwAEZGF0YQAEZWJzcwAEZWxlbQAEZnRhYgAEZ29pZAAEZ29w +YwAEaGFzaAAEaGVhZAAEaGVhcAAEaW5pdAAEaXR5cAAEa2V5cwAEa2luZAAEbGFzdAAEbGluawAE +bGlzdAAEbG9jawAEbWhkcgAEbmFtZQAEbmNnbwAEbmV4dAAEbm9iagAEbm9kZQAEb2xkcAAEcGFk +MQAEcGFkMgAEcGFyawAEcHJldgAEcHRhYgAEcHVzaAAEcmFuawAEcnNlcQAEcnVucQAEc2NhdgAE +c2VsZgAEc2l6ZQAEdGV4dAAEdGlueQAEdXNlZAAEdXNlcgAEdmFycAAEd2hlbgAEd3NlcQAFKmJv +b2wABSppbnQ4AAUqdWludAEFRXJyb3IABV90eXBlAAVhbGlnbgAFYWxsb2MABWJ5dGVwAAVieXRl +cwAFY2FjaGUABWNvdW50AAVjdXRhYgAFZHlpbmcABWVkYXRhAAVlbGVtcwAFZW1wdHkABWVudHJ5 +AAVlcXVhbAAFZXRleHQABWV2ZXJyAAVleHRyYQAFZmlyc3QABWZsYWdzAAVmbHVzaAAFZ0ZyZWUA +BWdMaXN0AAVnY2JzcwAFaGFzaDAABWluY2dvAAVpbnRlcgAFbGltaXQABWxvY2tzAAVtYWdpYwAF +bWF4cGMABW1pbkxDAAVtaW5wYwAFbmV4dHAABW5mdW5jAAVwYXJhbQAFcGN0YWIABXJlY3ZxAAVy +ZWN2eAAFcmVzZXQABXNjaGVkAAVzZW5kcQAFc2VuZHgABXNpZ3BjAAVzdGFjawAFc3RhdGUABXRm +bGFnAAV0aW1lcgAFdHlwZXMABXZhZGRyAAV3YkJ1ZgAFd2J1ZjEABXdidWYyAAAAAAAA4BtAAAAA +AABgIkAAAAAAAEAbQAAAAAAAYEtFAAAAAACATEUAAAAAAOBLRQAAAAAAQEtFAAAAAAAATUUAAAAA +ACBSRQAAAAAAgFBFAAAAAACAT0UAAAAAAEBKRQAAAAAAAE9FAAAAAACgSkUAAAAAAIBORQAAAAAA +IE5FAAAAAAAAUEUAAAAAAOBNRQAAAAAAYE9FAAAAAAAABnVuc2FmZQAGKmVycm9yAAYqaW50MTYA +BippbnQzMgAGKmludDY0AAYqdWludDgBBkVuYWJsZQEGRmxvYXRzAQZTdHJpbmcABl9kZWZlcgAG +X3BhbmljAAZhbGxvY04ABmNsb3NlZAAGZGl2TXVsAAZkaXZtb2QABmV0eXBlcwAGZ2NkYXRhAAZn +b2V4aXQABmluTGlzdAAGaW5zZXJ0AAZpc0ZyZWUABmxhYmVscwAGbGF5b3V0AAZsZW5ndGgABm1G +aXh1cAAGbWNhY2hlAAZuZWxlbXMABm5maWxlcwAGbm9zY2FuAAZucGFnZXMABm9mZnNldAAGcGFs +bG9jAAZwYXJlbnQABnBjYWNoZQAGcGVyaW9kAAZwcm9jaWQABnB0cmJpdAAGcWNvdW50AAZyZWZp +bGwABnJlbW92ZQAGc2lnbmVkAAZzdGF0dXMABnN0cmluZwAGdGlja2V0AAZ0aW1lcnMABnRyeUdl +dAAGdmRzb1BDAAZ2ZHNvU1AAB3J1bnRpbWUABypmdW5jKCkABypzdHJpbmcAByp1aW50MTYAByp1 +aW50MzIAByp1aW50NjQBB0ZlYXR1cmUAB2Fib3J0ZWQAB2FsbGxpbmsAB2JhbGFuY2UAB2Jsb2Nr +ZWQAB2J1Y2tldHMAB2Nnb0N0eHQAB2Nsb3NpbmcAB2RlcXVldWUAB2Rlc3Ryb3kAB2Rpc2NhcmQA +B2Rpc3Bvc2UAB2VucXVldWUAB2ZpbGV0YWIAB2ZyYW1lcGMAB2Z1bmNvZmYAB2dyb3dpbmcAB2dz +aWduYWwAB2hhc21haW4AB2lzQmxhbmsAB2lzRW1wdHkAB2xpYmNhbGwAB2xvY2tlZGcAB2xvY2tl +ZG0AB21ha2VBcmcAB21vcmVidWYAB25hbWVPZmYAB3BrZ1BhdGgAB3BrZ3BhdGgAB3ByZWVtcHQA +B3B0clNpemUAB3B0cmRhdGEAB3B1c2hBbGwAB3B1c2hjbnQAB3B1dEZhc3QAB3JhY2VjdHgAB3J1 +bm5leHQEA3BvcBgGAAAAB3NpZ21hc2sAB3NvcnRrZXkAB3N0YXJ0ZWQAB3N0YXJ0cGMAB3N1Y2Nl +c3MAB3N5c2NhbGwAB3Rha2VBbGwAB3RleHRPZmYAB3RvcGJpdHMAB3RvcGhhc2gAB3R5cGVPZmYA +B3R5cGVtYXAAB3dhaXRpbmcAAAgqW11pbnQzMgAIKltddWludDgACCpmbG9hdDMyAAgqZmxvYXQ2 +NAAIKnVpbnRwdHIBCFJlcXVpcmVkAAhhc3NlcnRlZAAIYmFzZWFkZHIACGJ5dGVkYXRhAAhjYWxs +aW5nRwAIY29uY3JldGUACGN1T2Zmc2V0AAhkYXRhcXNpegAIZGlzcGF0Y2gACGRsb2dQZXJNAAhk +b2VzUGFyawAIZWxlbXNpemUACGVsZW10eXBlAAhmYXN0cmFuZAAIZmx1c2hHZW4ACGZyZWVXYWl0 +AAhmcmVlbGluawAIaXNTZWxlY3QACGxpYmNhbGxnAAhsb2NrQWRkcgAIbXN0YXJ0Zm4ACG5jZ29j +YWxsAAhuZWVkemVybwAIbmV4dEZyZWUACG5leHR3aGVuAAhub3B0cmJzcwAIb2JqSW5kZXgACG92 +ZXJmbG93AAhwY0hlYWRlcgAIcHV0QmF0Y2gACHJhY2VhZGRyAAhydW5xaGVhZAAIcnVucXRhaWwE +BHB1c2gYBgAAAAhzY2FuV29yawAIc2lnY29kZTAACHNpZ2NvZGUxAAhzcGVjaWFscwAIc3Bpbm5p +bmcACHN0YXRzU2VxAAhzdGt0b3BzcAAIc3Vkb2didWYACHN3ZWVwZ2VuAAh0aHJvd2luZwAIdHJh +Y2VidWYACHRyYWNlc2VxAAh0cmFja2luZwAIdW5jb21tb24ACHdhaXRsaW5rAAh3YWl0bG9jawAI +d2FpdHRhaWwACHdyaXRlYnVmAAkqWzRddWludDgACSpbOF1pbnQzMgAJKls4XXVpbnQ4AAkqW111 +aW50MzIACSpbXXVpbnQ2NAAJKmNoYW4gaW50AQlTcGVjaWZpZWQACWFsbG9jQml0cwAJYW5jZXN0 +b3JzAAljYXVnaHRzaWcACWRlZmVycG9vbAAJZW5vcHRyYnNzAAlmcmVlaW5kZXgACWdjYnNzbWFz +awAJZ29pZGNhY2hlAAlpdGFibGlua3MACWxpYmNhbGxwYwAJbGliY2FsbHNwAAlsb2NrZWRFeHQA +CWxvY2tlZEludAAJbG9ja3NIZWxkAAltYWxsb2NpbmcACW5ldmFjdWF0ZQAJbmV4dHdhaXRtAAlu +b3B0cmRhdGEACW5vdmVyZmxvdwAJbnVtVGltZXJzAAlvcGVuRGVmZXIACXBjbG50YWJsZQAJcGtn +aGFzaGVzAAlwcmludGxvY2sACXByb2ZpbGVoegAJcHRyVG9UaGlzAAlyZWNvdmVyZWQEBWVtcHR5 +GAYAAAAJc2NhbkFsbG9jAAlzY2hlZGxpbmsACXNjaGVkdGljawAJc2NoZWR3aGVuAAlzaXplY2xh +c3MACXNwYW5jbGFzcwAJc3RhY2tMb2NrAAlzdGFydEFkZHIACXN5c2NhbGxwYwAJc3lzY2FsbHNw +AAl0cmFjZWJhY2sACXR5cGVsaW5rcwAJd2FpdHNpbmNlAAoqWzJddWludDMyAAoqWzhddWludDMy +AAoqW111aW50cHRyAAoqY2hhbiBib29sAAoqY29tcGxleDY0AAoqcnVudGltZS5nAAoqcnVudGlt +ZS5tAAoqcnVudGltZS5wAAoqc3RydWN0IHt9AApfaW50ZXJmYWNlAAphbGxvY0NhY2hlAAphbGxv +Y0NvdW50AAphbGxvY0xhcmdlAApjZ29DYWxsZXJzAApjaGVja2VtcHR5AApjb3VudEFsbG9jAApl +bm9wdHJkYXRhAApmaWVsZEFsaWduAApnY2RhdGFtYXNrAApnY21hcmtCaXRzAApnY3NjYW5kb25l +AApnb1NpZ1N0YWNrAAppbnNlcnRCYWNrAAppc0V4cG9ydGVkAAptb2R1bGVuYW1lAAptc3BhbmNh +Y2hlAApuZWVkZXh0cmFtAApuZXh0U2FtcGxlAApvbGRidWNrZXRzAApwY2xuT2Zmc2V0AApwbHVn +aW5wYXRoAApwcmVlbXB0R2VuAApwcmVlbXB0b2ZmAApyYWNlaWdub3JlAApyZWFkdmFyaW50AApy +ZWxlYXNlQWxsAApzZWxlY3REb25lAApzdGFja2NhY2hlAApzdWRvZ2NhY2hlAApzeXNtb250aWNr +AAp0aHJvd3NwbGl0AAp0aW1lcjBXaGVuAAp0aW1lcnNMb2NrAAp0aW55QWxsb2NzAAp0aW55b2Zm +c2V0AAp0cmFjZVN3ZWVwAAp0cmFjZVN3ZXB0AAp0cmFjZWxhc3RwAAp0cnlHZXRGYXN0AAp3YWl0 +cmVhc29uAAp3b3JrYnVmaGRyAAsqWzE1XXVpbnQ2NAALKlsxXXVpbnRwdHIACypbNl11aW50cHRy +AAsqWzlddWludHB0cgALKmNvbXBsZXgxMjgACypjcHUub3B0aW9uAQtSZXR1cm5Jc1B0cgALYWNx +dWlyZXRpbWUAC2J5dGVzTWFya2VkAAtjcmVhdGVzdGFjawALZW5zdXJlU3dlcHQAC2ZpbmRmdW5j +dGFiAAtmbHVzaGVkV29yawALZnVuY25hbWV0YWIAC25ld1NpZ3N0YWNrAAtuZXdvdmVyZmxvdwAL +bm9sZGJ1Y2tldHMAC29sZG92ZXJmbG93AAtwY3RhYk9mZnNldAALcHJlZW1wdFN0b3AAC3JhY2Vw +cm9jY3R4AAtyZWxlYXNldGltZQQHcHVzaEFsbBgGAAAAC3J1bnRpbWVoYXNoAAtzZXRvdmVyZmxv +dwALc3BlY2lhbGxvY2sAC3N0YWNrZ3VhcmQwAAtzdGFja2d1YXJkMQALc3lzY2FsbHRpY2sAC3N5 +c2NhbGx3aGVuAAt0ZXh0c2VjdG1hcAALdHJhY2tpbmdTZXEAC3dhaXR0cmFjZWV2AAt3YWl0dW5s +b2NrZgAMaW50ZXJuYWwvYWJpAAxpbnRlcm5hbC9jcHUADCpbMzJddWludHB0cgAMKltdc3RydWN0 +IHt9AQwqYWJpLlJlZ0FyZ3MADCpydW50aW1lLm1PUwEMUnVudGltZUVycm9yAAxhdG9taWNzdGF0 +dXMADGRlZmVycG9vbGJ1ZgAMZGVxdWV1ZVN1ZG9HAAxnY0Fzc2lzdFRpbWUADGdvaWRjYWNoZWVu +ZAAMbGlua3RpbWVoYXNoAAxsb2Nrc0hlbGRMZW4ADG1vZHVsZWhhc2hlcwAMbmV4dE92ZXJmbG93 +AAxwYW5pY29uZmF1bHQADHJ1bm5hYmxlVGltZQAMc2FtZVNpemVHcm93AAxzeXNleGl0dGlja3MA +DHRpbWVyUmFjZUN0eAANKlsyNTNddWludHB0cgANKls1MTJddWludHB0cgANKls4XXN0cnVjdCB7 +fQANKltdY3B1Lm9wdGlvbgANKmludGVyZmFjZSB7fQANKnJ1bnRpbWUuYm1hcAANKnJ1bnRpbWUu +aG1hcAANKnJ1bnRpbWUuaXRhYgANKnJ1bnRpbWUubmFtZQANKnJ1bnRpbWUubm90ZQANY2dvQ2Fs +bGVyc1VzZQANY2hlY2tub25lbXB0eQANZGVsZXRlZFRpbWVycwANZmlsZXRhYk9mZnNldAANZ2NB +c3Npc3RCeXRlcwANaW5jcm5vdmVyZmxvdwANbWlzc2luZ01ldGhvZAANbmV4dEZyZWVJbmRleAAN +b2xkYnVja2V0bWFzawANcGFya2luZ09uQ2hhbgANcHJlZW1wdFNocmluawANcmVwb3J0Wm9tYmll +cwANcnVubmFibGVTdGFtcAANc2lnbmFsUGVuZGluZwANc3RhcnRpbmd0cmFjZQANd2FpdHRyYWNl +c2tpcAAAACFAAAAAAAAABAAAAAAAAAAhQAAAAAAACAQAAAAAAAAAIUAAAAAAAEAEAAAAAAAAACFA +AAAAAAB4AAAAAAAAAAAhQAAAAAAAAAUAAAAAAAAAIUAAAAAAAKAAAAAAAAAAACFAAAAAAADoBwAA +AAAAAAAhQAAAAAAAAAgAAAAAAAAAIUAAAAAAABgAAAAAAAAAACFAAAAAAAAAAQAAAAAAAAAhQAAA +AAAAIAAAAAAAAAAAIUAAAAAAACgAAAAAAAAAACFAAAAAAAAAEAAAAAAAAAAhQAAAAAAAEBAAAAAA +AAAAIUAAAAAAADAAAAAAAAAAACFAAAAAAAA4AAAAAAAAAAAhQAAAAAAAQAAAAAAAAAAAIUAAAAAA +AEgAAAAAAAAAAA4qZnVuYygpIHN0cmluZwAOKnJ1bnRpbWUuX3R5cGUADipydW50aW1lLmdMaXN0 +AA4qcnVudGltZS5nb2J1ZgAOKnJ1bnRpbWUuaGNoYW4ADipydW50aW1lLm1zcGFuAA4qcnVudGlt +ZS5tdXRleAAOKnJ1bnRpbWUuc3RhY2sADipydW50aW1lLnN1ZG9nAA4qcnVudGltZS50ZmxhZwAO +KnJ1bnRpbWUudGltZXIADipydW50aW1lLndhaXRxAA4qcnVudGltZS53YkJ1ZgAOYXN5bmNTYWZl +UG9pbnQADmNyZWF0ZU92ZXJmbG93AA5mdW5jbmFtZU9mZnNldAAObG9ja1JhbmtTdHJ1Y3QADm1h +bnVhbEZyZWVMaXN0AA5ydW5TYWZlUG9pbnRGbgAOc3lzYmxvY2t0cmFjZWQADnRyYWNlUmVjbGFp +bWVkAA8qWzE1XWNwdS5vcHRpb24ADypydW50aW1lLl9kZWZlcgAPKnJ1bnRpbWUuX3BhbmljAA8q +cnVudGltZS5nY0JpdHMADypydW50aW1lLmdjV29yawAPKnJ1bnRpbWUubGZub2RlAA8qcnVudGlt +ZS5tY2FjaGUADypydW50aW1lLnNpZ3NldAEPKnVuc2FmZS5Qb2ludGVyAA9tYXJrQml0c0ZvckJh +c2UAD3ByZXBhcmVGb3JTd2VlcAAAECpbXSpydW50aW1lLmJtYXAAECpbXSpydW50aW1lLml0YWIA +ECpmdW5jKGJvb2wpIGJvb2wAECpydW50aW1lLmZ1bmN0YWIAECpydW50aW1lLmZ1bmN2YWwAECpy +dW50aW1lLmltZXRob2QAECpydW50aW1lLmxpYmNhbGwAECpydW50aW1lLm5hbWVPZmYAECpydW50 +aW1lLnNwZWNpYWwAECpydW50aW1lLnR5cGVPZmYAECpydW50aW1lLndvcmtidWYAEGFjdGl2ZVN0 +YWNrQ2hhbnMAEGRpdmlkZUJ5RWxlbVNpemUAEGdjTWFya1dvcmtlck1vZGUAEG1hcmtCaXRzRm9y +SW5kZXgAEHJlZmlsbEFsbG9jQ2FjaGUAESpbXSpydW50aW1lLl90eXBlABEqW10qcnVudGltZS5t +c3BhbgARKltdKnJ1bnRpbWUuc3Vkb2cAESpbXSpydW50aW1lLnRpbWVyABEqW111bnNhZmUuUG9p +bnRlcgARKnJ1bnRpbWUuZGxvZ1Blck0AESpydW50aW1lLmd1aW50cHRyABEqcnVudGltZS5sb2Nr +UmFuawARKnJ1bnRpbWUubWFwZXh0cmEAESpydW50aW1lLm11aW50cHRyABEqcnVudGltZS5wY0hl +YWRlcgARKnJ1bnRpbWUucG9sbERlc2MAESpydW50aW1lLnB1aW50cHRyABEqcnVudGltZS5zdHJp +bmdlcgARKnJ1bnRpbWUudGV4dHNlY3QAEWFsbG9jQml0c0ZvckluZGV4ABIqWzhdKnJ1bnRpbWUu +X3R5cGUAEipbOF11bnNhZmUuUG9pbnRlcgASKls5XXVuc2FmZS5Qb2ludGVyABIqW10qcnVudGlt +ZS5fZGVmZXIAEipbXXJ1bnRpbWUuZnVuY3RhYgASKltdcnVudGltZS5pbWV0aG9kABIqW11ydW50 +aW1lLnR5cGVPZmYAEipydW50aW1lLl90eXBlUGFpcgASKnJ1bnRpbWUuYml0dmVjdG9yABIqcnVu +dGltZS5nY2xpbmtwdHIAEipydW50aW1lLm1TcGFuTGlzdAASKnJ1bnRpbWUubm90SW5IZWFwABIq +cnVudGltZS5wYWdlQ2FjaGUAEipydW50aW1lLnB0YWJFbnRyeQASKnJ1bnRpbWUuc3BhbkNsYXNz +ABMqWzhdcnVudGltZS50eXBlT2ZmABMqW11bXSpydW50aW1lLl90eXBlABMqW11ydW50aW1lLmd1 +aW50cHRyABMqW11ydW50aW1lLnRleHRzZWN0ABMqcnVudGltZS5jZ29DYWxsZXJzABMqcnVudGlt +ZS5tU3BhblN0YXRlABMqcnVudGltZS5tb2R1bGVkYXRhABMqcnVudGltZS5tb2R1bGVoYXNoABMq +cnVudGltZS5wbGFpbkVycm9yABMqcnVudGltZS5zeXNtb250aWNrABMqcnVudGltZS53YWl0UmVh +c29uABMqcnVudGltZS53b3JrYnVmaGRyABQqWzEyOF0qcnVudGltZS5tc3BhbgAUKlsxMjhdKnJ1 +bnRpbWUuc3Vkb2cAFCpbMTM2XSpydW50aW1lLm1zcGFuABQqWzMyXSpydW50aW1lLl9kZWZlcgAU +Kls4XVtdKnJ1bnRpbWUuX3R5cGUAFCpbXVtdKnJ1bnRpbWUuX2RlZmVyABQqW11ydW50aW1lLl90 +eXBlUGFpcgAUKltdcnVudGltZS5wdGFiRW50cnkBFCphYmkuSW50QXJnUmVnQml0bWFwABQqcnVu +dGltZS5ib3VuZHNFcnJvcgAUKnJ1bnRpbWUuZXJyb3JTdHJpbmcAFCpydW50aW1lLnRyYWNlQnVm +UHRyABRnY0ZyYWN0aW9uYWxNYXJrVGltZQAVKls1XVtdKnJ1bnRpbWUuX2RlZmVyABUqWzhdcnVu +dGltZS5fdHlwZVBhaXIAFSpbXXJ1bnRpbWUubW9kdWxlaGFzaAAVKnJ1bnRpbWUuYW5jZXN0b3JJ +bmZvABUqcnVudGltZS5nc2lnbmFsU3RhY2sAFSpydW50aW1lLmhlbGRMb2NrSW5mbwAVZ2NNYXJr +V29ya2VyU3RhcnRUaW1lABV0aW1lck1vZGlmaWVkRWFybGllc3QAAAAAAAAAFipbMjU2XXJ1bnRp +bWUuZ3VpbnRwdHIAFipbXSpydW50aW1lLm1vZHVsZWRhdGEAFipbXVszMl0qcnVudGltZS5fZGVm +ZXIAFipydW50aW1lLmludGVyZmFjZXR5cGUAFipydW50aW1lLm1TcGFuU3RhdGVCb3gAFipydW50 +aW1lLnN0YWNrZnJlZWxpc3QAFypbNV1bMzJdKnJ1bnRpbWUuX2RlZmVyABcqW11ydW50aW1lLmFu +Y2VzdG9ySW5mbwAXKltdcnVudGltZS5oZWxkTG9ja0luZm8AFypydW50aW1lLmxvY2tSYW5rU3Ry +dWN0ABgqW11ydW50aW1lLnN0YWNrZnJlZWxpc3QAGCpydW50aW1lLmJvdW5kc0Vycm9yQ29kZQAY +KnJ1bnRpbWUucGVyc2lzdGVudEFsbG9jABkqWzEwXXJ1bnRpbWUuaGVsZExvY2tJbmZvABkqWzRd +cnVudGltZS5zdGFja2ZyZWVsaXN0ABkqbWFwW2ludDMyXXVuc2FmZS5Qb2ludGVyABkqcnVudGlt +ZS5nY01hcmtXb3JrZXJNb2RlAAAAGipydW50aW1lLmRlYnVnQ2FsbFdyYXBBcmdzABoqcnVudGlt +ZS5zbGljZUludGVyZmFjZVB0cgEbKnJ1bnRpbWUuVHlwZUFzc2VydGlvbkVycm9yABsqcnVudGlt +ZS5lcnJvckFkZHJlc3NTdHJpbmcAGypydW50aW1lLmdjQmdNYXJrV29ya2VyTm9kZQAbKnJ1bnRp +bWUuc3RyaW5nSW50ZXJmYWNlUHRyABsqcnVudGltZS51aW50MTZJbnRlcmZhY2VQdHIAGypydW50 +aW1lLnVpbnQzMkludGVyZmFjZVB0cgAbKnJ1bnRpbWUudWludDY0SW50ZXJmYWNlUHRyAAAcKmZ1 +bmMoaW50ZXJmYWNlIHt9LCB1aW50cHRyKQAcKm1hcFt1aW50MzJdW10qcnVudGltZS5fdHlwZQAg +Km1hcC5idWNrZXRbaW50MzJddW5zYWZlLlBvaW50ZXIAICptYXBbcnVudGltZS5fdHlwZVBhaXJd +c3RydWN0IHt9ACIqc3RydWN0IHsgcnVudGltZS5nTGlzdDsgbiBpbnQzMiB9ACMqbWFwLmJ1Y2tl +dFt1aW50MzJdW10qcnVudGltZS5fdHlwZQAjKm1hcFtydW50aW1lLnR5cGVPZmZdKnJ1bnRpbWUu +X3R5cGUAAAAAAAAAJipmdW5jKCpydW50aW1lLmcsIHVuc2FmZS5Qb2ludGVyKSBib29sACcqbWFw +LmJ1Y2tldFtydW50aW1lLl90eXBlUGFpcl1zdHJ1Y3Qge30AAAAAKipmdW5jKHVuc2FmZS5Qb2lu +dGVyLCB1bnNhZmUuUG9pbnRlcikgYm9vbAAqKm1hcC5idWNrZXRbcnVudGltZS50eXBlT2ZmXSpy +dW50aW1lLl90eXBlACwqc3RydWN0IHsgbGVuIGludDsgYnVmIFsxMjhdKnJ1bnRpbWUubXNwYW4g +fQAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAABWcpJIICAg2IGpGAAAAAAD4Z0YAAAAAAFwW +AAAAAAAAAJRFAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAADu+6dwCAgINiBqRgAAAAAA+GdG +AAAAAABZHQAAAAAAAACXRQAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAA0fANbggICDYgakYA +AAAAAPhnRgAAAAAAcgIAAAAAAABAm0UAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAANcrpCMI +CAg2IGpGAAAAAAD4Z0YAAAAAAPwOAAAAAAAAAJxFAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAA +AAARasF6CAgINiBqRgAAAAAA+GdGAAAAAACUDAAAAAAAAECcRQAAAAAAAAAAAAAAAAAIAAAAAAAA +AAgAAAAAAAAA3K6qWAgICDYgakYAAAAAAPhnRgAAAAAAoAQAAAAAAACgwkUAAAAAAAAAAAAAAAAA +CAAAAAAAAAAIAAAAAAAAAFHm/h8ICAg2IGpGAAAAAAD4Z0YAAAAAAC4IAAAAAAAAgJxFAAAAAAAA +AAAAAAAAAAgAAAAAAAAACAAAAAAAAAATrVa6CAgINiBqRgAAAAAA+GdGAAAAAAA4CAAAAAAAAMCc +RQAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAA5+siJQgICDYgakYAAAAAAPhnRgAAAAAAIgEA +AAAAAABAnUUAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAABXGZqAICAg2IGpGAAAAAAD4Z0YA +AAAAAKgEAAAAAAAAgJ1FAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAmF1hrCAgINiBqRgAA +AAAA+GdGAAAAAACwBAAAAAAAAMCdRQAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAFwe4JggI +CDYgakYAAAAAAPhnRgAAAAAAuAQAAAAAAAAAnkUAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAA +AO/fyl8ICAg2IGpGAAAAAAD4Z0YAAAAAAHkCAAAAAAAAQJ5FAAAAAAAAAAAAAAAAAAgAAAAAAAAA +CAAAAAAAAADlK1GcCAgINiBqRgAAAAAA+GdGAAAAAAC6EAAAAAAAAKDjRQAAAAAAAAAAAAAAAAAI +AAAAAAAAAAgAAAAAAAAAcBm4KggICDYgakYAAAAAAPhnRgAAAAAACQ8AAAAAAABA6EUAAAAAAAAA +AAAAAAAACAAAAAAAAAAIAAAAAAAAAICUw6QICAg2IGpGAAAAAAD4Z0YAAAAAALEVAAAAAAAAwPdF +AAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAACj+OaLCAgINiBqRgAAAAAA+GdGAAAAAADCFQAA +AAAAAODuRQAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAy6DDFAgICDYgakYAAAAAAPhnRgAA +AAAAOBkAAAAAAACgy0UAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAJ2skW0ICAg2IGpGAAAA +AAD4Z0YAAAAAADccAAAAAAAAoNdFAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAABuU+XCCAgI +NiBqRgAAAAAA+GdGAAAAAAC+HQAAAAAAAICeRQAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAA +SQW5pQgICDYgakYAAAAAAPhnRgAAAAAALBoAAAAAAABAuEUAAAAAAAAAAAAAAAAACAAAAAAAAAAI +AAAAAAAAABE96s4ICAg2IGpGAAAAAAD4Z0YAAAAAAGAeAAAAAAAA4MxFAAAAAAAAAAAAAAAAAAgA +AAAAAAAACAAAAAAAAABvTa2YCAgINiBqRgAAAAAA+GdGAAAAAADbFwAAAAAAAAC5RQAAAAAAAAAA +AAAAAAAIAAAAAAAAAAgAAAAAAAAA72tS9QgICDYgakYAAAAAAPhnRgAAAAAAkhYAAAAAAACAzUUA +AAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAADDRKDYICAg2IGpGAAAAAAD4Z0YAAAAAAKQWAAAA +AAAAIMZFAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAB3wwfjCAgINiBqRgAAAAAA+GdGAAAA +AACgDAAAAAAAAEAJRgAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAiYRdYggICDYgakYAAAAA +APhnRgAAAAAA0h4AAAAAAABg2EUAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAJfBzQICAg2 +IGpGAAAAAAD4Z0YAAAAAAEMeAAAAAAAAAJ9FAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAB+ +bXDhCAgINiBqRgAAAAAA+GdGAAAAAACAFAAAAAAAAMDtRQAAAAAAAAAAAAAAAAAIAAAAAAAAAAgA +AAAAAAAAEe8OZwgICDYgakYAAAAAAPhnRgAAAAAAThwAAAAAAADg5UUAAAAAAAAAAAAAAAAACAAA +AAAAAAAIAAAAAAAAACBX8bEICAg2IGpGAAAAAAD4Z0YAAAAAAGUcAAAAAAAAIM5FAAAAAAAAAAAA +AAAAAAgAAAAAAAAACAAAAAAAAACpx29jCAgINiBqRgAAAAAA+GdGAAAAAAC2FgAAAAAAAMDORQAA +AAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAgJV4WQgICDYgakYAAAAAAPhnRgAAAAAA+BwAAAAA +AAAg2UUAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAKsIte8ICAg2IGpGAAAAAAD4Z0YAAAAA +APUVAAAAAAAAYM9FAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAA2U4z5CAgINiBqRgAAAAAA ++GdGAAAAAADIFgAAAAAAAMDsRQAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAA9JPc8wgICDYg +akYAAAAAAPhnRgAAAAAAix0AAAAAAADAuUUAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAA9N +gTcICAg2IGpGAAAAAAD4Z0YAAAAAAKwMAAAAAAAAgA5GAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAA +AAAAAABgi9UHCAgINiBqRgAAAAAA+GdGAAAAAADIEAAAAAAAACC6RQAAAAAAAAAAAAAAAAAIAAAA +AAAAAAgAAAAAAAAAcZ/EVggICDYgakYAAAAAAPhnRgAAAAAAQRoAAAAAAABAn0UAAAAAAAAAAAAA +AAAACAAAAAAAAAAIAAAAAAAAAB1rwvQICAg2IGpGAAAAAAD4Z0YAAAAAABQYAAAAAAAA4NlFAAAA +AAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAABM5VxuCAgINiBqRgAAAAAA+GdGAAAAAABWGgAAAAAA +AKAARgAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAATTDeCQgICDYgakYAAAAAAPhnRgAAAAAA +axoAAAAAAACg2kUAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAO3YonEICAg2IGpGAAAAAAD4 +Z0YAAAAAALAUAAAAAAAAoNBFAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAA6wgsuCAgINiBq +RgAAAAAA+GdGAAAAAADaFgAAAAAAAICfRQAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAycQg +JQgICDYgakYAAAAAAPhnRgAAAAAALxIAAAAAAACgx0UAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAA +AAAAAGmo2m0ICAg2IGpGAAAAAAD4Z0YAAAAAADoYAAAAAAAAQPlFAAAAAAAAAAAAAAAAAAgAAAAA +AAAACAAAAAAAAABLvURdCAgINiBqRgAAAAAA+GdGAAAAAADYHQAAAAAAAEDRRQAAAAAAAAAAAAAA +AAAIAAAAAAAAAAgAAAAAAAAA5CRdWQgICDYgakYAAAAAAPhnRgAAAAAAsBkAAAAAAADg0UUAAAAA +AAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAALIBTV8ICAg2IGpGAAAAAAD4Z0YAAAAAABcWAAAAAAAA +oLhFAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAABHlo7CCAgINiBqRgAAAAAA+GdGAAAAAAB8 +HgAAAAAAAECvRQAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAM6VxNggICBYgakYAAAAAACxf +RwAAAAAA7BYAAAAAAAAg3EUAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAGm6UvUICAg2IGpG +AAAAAAD4Z0YAAAAAAMAUAAAAAAAAgNJFAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAADY+9kX +CAgINiBqRgAAAAAA+GdGAAAAAAAoHQAAAAAAACDTRQAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAA +AAAAtcTHUggICDYgakYAAAAAAPhnRgAAAAAA7x4AAAAAAADAn0UAAAAAAAAAAAAAAAAACAAAAAAA +AAAIAAAAAAAAALPnLQ8ICAg2IGpGAAAAAAD4Z0YAAAAAAHMYAAAAAAAAIMNFAAAAAAAAAAAAAAAA +AAgAAAAAAAAACAAAAAAAAAAU/KY/CAgINiBqRgAAAAAA+GdGAAAAAADQFAAAAAAAAMD6RQAAAAAA +AAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAA2MSONggICDYgakYAAAAAAPhnRgAAAAAAlRoAAAAAAACg +5kUAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAFMJwlwICAg2IGpGAAAAAAD4Z0YAAAAAAIYY +AAAAAAAA4NxFAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAB66ZgzCAgINiBqRgAAAAAA+GdG +AAAAAADgFAAAAAAAAACgRQAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAe2//jQgICDYgakYA +AAAAAPhnRgAAAAAA8BQAAAAAAAAg8UUAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAB/HCsQI +CAg2IGpGAAAAAAD4Z0YAAAAAAP4WAAAAAAAAQKBFAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAA +AAC52NJoCAgINiBqRgAAAAAA+GdGAAAAAAAMHwAAAAAAAICgRQAAAAAAAAAAAAAAAAAIAAAAAAAA +AAgAAAAAAAAA3elarwgICDYgakYAAAAAAPhnRgAAAAAAKR8AAAAAAADAoEUAAAAAAAAAAAAAAAAA +CAAAAAAAAAAIAAAAAAAAADWm4YAICAg2IGpGAAAAAAD4Z0YAAAAAAEYfAAAAAAAAAKFFAAAAAAAA +AAAAAAAAAAgAAAAAAAAACAAAAAAAAAC6ABKOCAgINiBqRgAAAAAA+GdGAAAAAAC/GgAAAAAAAADV +RQAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAA7VoXWggICDYgakYAAAAAAPhnRgAAAAAAKgYA +AAAAAABAoUUAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAHj52xAICAg2IGpGAAAAAAD4Z0YA +AAAAAIACAAAAAAAAgKFFAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAu9JLuCAgINiBqRgAA +AAAA+GdGAAAAAAAzBgAAAAAAAMChRQAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAA02zGAAgI +CDYgakYAAAAAAPhnRgAAAAAAPAYAAAAAAAAAokUAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAA +APh0Lx4ICAg2IGpGAAAAAAD4Z0YAAAAAAEUGAAAAAAAAQKJFAAAAAAAAAAAAAAAAAAgAAAAAAAAA +CAAAAAAAAACoJtmaCAgINiBqRgAAAAAA+GdGAAAAAADABAAAAAAAAICiRQAAAAAAAAAAAAAAAAAI +AAAAAAAAAAgAAAAAAAAAXsWLlggICDYgakYAAAAAAPhnRgAAAAAAQggAAAAAAADAokUAAAAAAAAA +AAAAAAAACAAAAAAAAAAIAAAAAAAAAOPiQTIICAg2IGpGAAAAAAD4Z0YAAAAAACgWAAAAAAAAAKNF +AAAAAAAAAAAAAAAAABgAAAAAAAAACAAAAAAAAAAvMq18AggIFwAAAAAAAAAA+GdGAAAAAADoGAAA +AAAAAACFRQAAAAAAAAAAAAAAAAAYAAAAAAAAAAgAAAAAAAAAgD/uTwIICBcAAAAAAAAAAPhnRgAA +AAAAfBcAAAAAAABg4UUAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAAAAAP6SIqMCCAgXAAAAAAAA +AAD4Z0YAAAAAAFwWAABAIQAAoMNFAAAAAAAAAAAAAAAAABgAAAAAAAAACAAAAAAAAAArR3C7AggI +FwAAAAAAAAAA+GdGAAAAAABuFgAAAAAAAIC0RQAAAAAAAAAAAAAAAAAYAAAAAAAAAAgAAAAAAAAA +Y9zYyAIICBcAAAAAAAAAAPhnRgAAAAAAyBwAAAAAAABAi0UAAAAAAAAAAAAAAAAAGAAAAAAAAAAI +AAAAAAAAACRMWhECCAgXAAAAAAAAAAD4Z0YAAAAAAI8XAAAAAAAAQPJFAAAAAAAAAAAAAAAAABgA +AAAAAAAACAAAAAAAAACNbljBAggIFwAAAAAAAAAA+GdGAAAAAACiFwAAAAAAAACPRQAAAAAAAAAA +AAAAAAAYAAAAAAAAAAgAAAAAAAAAp92c8QIICBcAAAAAAAAAAPhnRgAAAAAAtRcAAAAAAAAAkEUA +AAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAAAAAKriVJoCCAgXAAAAAAAAAAD4Z0YAAAAAAOAcAAAA +AAAAYKdFAAAAAAAAAAAAAAAAABgAAAAAAAAACAAAAAAAAABbaiBAAggIFwAAAAAAAAAA+GdGAAAA +AABCGwAAAAAAAICTRQAAAAAAAAAAAAAAAAAYAAAAAAAAAAgAAAAAAAAAFYno8wIICBcAAAAAAAAA +APhnRgAAAAAA7RkAAAAAAADAk0UAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAAAAABYqTXMCCAgX +AAAAAAAAAAD4Z0YAAAAAABoIAAAAAAAAwJ1FAAAAAAAAAAAAAAAAABgAAAAAAAAACAAAAAAAAABy +NZMZAggIFwAAAAAAAAAA+GdGAAAAAADVEQAAAAAAAEDoRQAAAAAAAAAAAAAAAAAYAAAAAAAAAAgA +AAAAAAAAVSvnaAIICBcAAAAAAAAAAPhnRgAAAAAAWBsAAAAAAACgy0UAAAAAAAAAAAAAAAAAGAAA +AAAAAAAIAAAAAAAAAHxwkUcCCAgXAAAAAAAAAAD4Z0YAAAAAAFkdAACAIQAAoNdFAAAAAAAAAAAA +AAAAABgAAAAAAAAACAAAAAAAAABewJ2kAggIFwAAAAAAAAAA+GdGAAAAAAD8GAAAAAAAAIDNRQAA +AAAAAAAAAAAAAAAYAAAAAAAAAAgAAAAAAAAAVdmPkQIICBcAAAAAAAAAAPhnRgAAAAAAAhoAAAAA +AAAgsUUAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAAAAAG3oJO8CCAgXAAAAAAAAAAD4Z0YAAAAA +AHIdAAAAAAAAIM5FAAAAAAAAAAAAAAAAABgAAAAAAAAACAAAAAAAAABYXaYpAggIFwAAAAAAAAAA ++GdGAAAAAAAQGQAAAAAAAMDORQAAAAAAAAAAAAAAAAAYAAAAAAAAAAgAAAAAAAAAT39iSgIICBcA +AAAAAAAAAPhnRgAAAAAAIBwAAAAAAACg2kUAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAAAAADQl +Ti8CCAgXAAAAAAAAAAD4Z0YAAAAAAG4bAAAAAAAA4NFFAAAAAAAAAAAAAAAAABgAAAAAAAAACAAA +AAAAAAAPlXtAAggIFwAAAAAAAAAA+GdGAAAAAACkHQAAAAAAACDTRQAAAAAAAAAAAAAAAAAYAAAA +AAAAAAgAAAAAAAAAkfjSDwIICBcAAAAAAAAAAPhnRgAAAAAAFxoAAAAAAADg3EUAAAAAAAAAAAAA +AAAAGAAAAAAAAAAIAAAAAAAAANljNPgCCAgXAAAAAAAAAAD4Z0YAAAAAACQZAAAAAAAAQKBFAAAA +AAAAAAAAAAAAABgAAAAAAAAACAAAAAAAAAC6zKWFAggIFwAAAAAAAAAA+GdGAAAAAACsEAAAAAAA +AGCzRQAAAAAAAAAAAAAAAAAYAAAAAAAAAAgAAAAAAAAA1JEdHwIICBcAAAAAAAAAAPhnRgAAAAAA +dQoAAAAAAAAAokUAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAAAAAD+1aSACCAgXAAAAAAAAAAD4 +Z0YAAAAAAIAKAAAAAAAAQKJFAAAAAAAAAAAAAAAAABgAAAAAAAAACAAAAAAAAADffi44AggIFwAA +AAAAAAAA+GdGAAAAAAAkCAAAAAAAAICiRQAAAAAAAAAAAAAAAAAYAAAAAAAAAAgAAAAAAAAAuzPA +XQIICBcAAAAAAAAAAPhnRgAAAAAAfAwAAAAAAADAokUAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAA +AAAAAEY85xsCCAgXAAAAAAAAAAD4Z0YAAAAAAMgXAAAAAAAAAKNFAAAAAAAAAAAAAAAAAAgAAAAA +AAAACAAAAAAAAAD2vIL2AggIMwAAAAAAAAAA+GdGAAAAAAAhBgAAAAAAAAAAAAAAAAAAAAAAAAAA +AAABAAAAAAAAAAAAAAAAAAAAxQb/Ew8BAQEoakYAAAAAACxfRwAAAAAAcgIAAMAhAAAAAAAAAAAA +ABAAAAAAAAAACAAAAAAAAAAIAAAAAAAAALhI390KCAgyIGpGAAAAAAD4Z0YAAAAAAIgMAAAAAAAA +QJtFAAAAAAADAAAAAAAAAAgAAAAAAAAACAAAAAAAAACRVctxCggIMiBqRgAAAAAA+GdGAAAAAACL +CgAAAAAAAECdRQAAAAAAAwAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAbVQaswcICBDAaEYAAAAAACxf +RwAAAAAA/A4AAAAiAAAAAAAAAAAAABAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAIwCJXkHBAQPyGhG +AAAAAAAsX0cAAAAAAJQMAABAIgAAAAAAAAAAAAAQAAAAAAAAAAQAAAAAAAAAAAAAAAAAAADTPsKw +BwQEDTBpRgAAAAAALF9HAAAAAAAuCAAAwCIAAAAAAAAAAAAAEAAAAAAAAAAIAAAAAAAAAAAAAAAA +AAAA+3+iLgcICA44aUYAAAAAACxfRwAAAAAAOAgAAAAjAAAAAAAAAAAAABAAAAAAAAAACAAAAAAA +AAAIAAAAAAAAAKJtyx4CCAgzAAAAAAAAAAD4Z0YAAAAAAFAUAAAAAAAAAAABAAAAAABAoUUAAAAA +AAgAAAAAAAAAAAAAAAAAAAD6cVP3DwgIAiBqRgAAAAAALF9HAAAAAAAiAQAAQCMAAAAAAAAAAAAA +EAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAzoDV7A8CAgQQakYAAAAAACxfRwAAAAAAqAQAAIAjAAAA +AAAAAAAAABAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAJBrbsPBAQFGGpGAAAAAAAsX0cAAAAAALAE +AADAIwAAAAAAAAAAAAAQAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAD/mz+WDwgIBiBqRgAAAAAALF9H +AAAAAAC4BAAAACQAAAAAAAAAAAAAEAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAJ8AGzA8BAQMoakYA +AAAAACxfRwAAAAAAeQIAAEAkAAAAAAAAAAAAABAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAANyRvfAP +AQEIKGpGAAAAAAAsX0cAAAAAAL4dAAAAJgAAGAYAAAAAAAAQAAAAAAAAAAEAAAAAAAAAAAAAAAAA +AABaFIdzDwEBCChqRgAAAAAALF9HAAAAAADTFQAAoF0AABgGAAAAAAAAEAAAAAAAAAAIAAAAAAAA +AAAAAAAAAAAA/tCmuQ8ICAIgakYAAAAAACxfRwAAAAAAQx4AAAAoAAAYBgAAAAAAABAAAAAAAAAA +AQAAAAAAAAAAAAAAAAAAABucyRgPAQEIKGpGAAAAAAAsX0cAAAAAAEEaAADAKgAAGAYAAAAAAAAQ +AAAAAAAAAAQAAAAAAAAAAAAAAAAAAADze22+DwQEBRhqRgAAAAAALF9HAAAAAADaFgAAACwAABgG +AAAAAAAAEAAAAAAAAAAQAAAAAAAAAAgAAAAAAAAAbsPjqwcICBjQakYAAAAAAPhnRgAAAAAA7x4A +AIAuAAAYBgAAAAAAABAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAG3ZAsgPAQEIKGpGAAAAAAAsX0cA +AAAAAOAUAADALwAAGAYAAAAAAAAQAAAAAAAAAAQAAAAAAAAAAAAAAAAAAABF4F42DwQEBRhqRgAA +AAAALF9HAAAAAAD+FgAAQDAAABgGAAAAAAAAEAAAAAAAAAACAAAAAAAAAAAAAAAAAAAA+a6FgQ8C +AgkQakYAAAAAACxfRwAAAAAADB8AAIAwAAAYBgAAAAAAABAAAAAAAAAABAAAAAAAAAAAAAAAAAAA +AFdKaxMPBAQKGGpGAAAAAAAsX0cAAAAAACkfAADAMAAAGAYAAAAAAAAQAAAAAAAAAAgAAAAAAAAA +AAAAAAAAAAARjti7DwgICyBqRgAAAAAALF9HAAAAAABGHwAAADEAABgGAAAAAAAAEAAAAAAAAAAQ +AAAAAAAAAAgAAAAAAAAAtFz/4AcICBjQakYAAAAAAPhnRgAAAAAAKgYAAIAxAAAAAAAAAAAAABAA +AAAAAAAACAAAAAAAAAAAAAAAAAAAABJ3uNUPCAgHIGpGAAAAAAAsX0cAAAAAAIACAADAMQAAAAAA +AAAAAAAQAAAAAAAAAAIAAAAAAAAAAAAAAAAAAACgDvLvDwICCRBqRgAAAAAALF9HAAAAAAAzBgAA +ADIAAAAAAAAAAAAAEAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAPehK0A8EBAoYakYAAAAAACxfRwAA +AAAAPAYAAEAyAAAAAAAAAAAAABAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAC6NMYYPCAgLIGpGAAAA +AAAsX0cAAAAAAEUGAACAMgAAAAAAAAAAAAAQAAAAAAAAAAEAAAAAAAAAAAAAAAAAAABfQj5mDwEB +CChqRgAAAAAALF9HAAAAAADABAAAwDIAAAAAAAAAAAAAEAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAA +ktdKvQ8ICAwgakYAAAAAACxfRwAAAAAAQggAAAAzAAAAAAAAAAAAABAAAAAAAAAACAAAAAAAAAAI +AAAAAAAAAM3EXBsPCAg6IGpGAAAAAAD4Z0YAAAAAACgWAABAMwAAmAQAAAAAAAAQAAAAAAAAAAA/ +KnN0cnVjdCB7IGxvY2sgcnVudGltZS5tdXRleDsgdXNlZCB1aW50MzI7IGZuIGZ1bmMoYm9vbCkg +Ym9vbCB9AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAAAAAAAAAAAAAAAAAAAdSSL+ +CggIEYBzRQAAAAAALF9HAAAAAADyHQAAAAAAACDORQAAAAAAwJdFAAAAAAAKAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAACsjicsCggIETBzRQAAAAAALF9HAAAA +AADUGgAAAAAAAEDyRQAAAAAAwJRFAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAEAAAAAAAAAAQAAAAAAACgKjG+CggIETBzRQAAAAAAmF9HAAAAAADqGgAAAAAAAACPRQAAAAAA +AJVFAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAEAAAAAAAAAAAAAAAAAAB2 +zqUsCggIEVBzRQAAAAAALF9HAAAAAAAAGwAAAAAAAEDyRQAAAAAAwJRFAAAAAACIAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAOABAAAAAAAA2AEAAAAAAABPqR1WAggIEQBkRQAAAAAAeF9H +AAAAAACgFQAAAAAAAEDoRQAAAAAAgJZFAAAAAAAPAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAHgAAAAAAAAAAAAAAAAAAAAQIip/CggIEWBzRQAAAAAALF9HAAAAAADIDgAAAAAAAECiRQAA +AAAAAJpFAAAAAAAPAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAA +AACaYY7CCggIESBqRgAAAAAALF9HAAAAAADVDgAAAAAAAMCiRQAAAAAAgJpFAAAAAAABAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOgHAAAAAAAAAAAAAAAAAABoOoU5CggIEZBzRQAAAAAA +LF9HAAAAAACoEQAAAAAAAMCiRQAAAAAAgJpFAAAAAAD9AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAwImHaCggIEaBzRQAAAAAALF9HAAAAAACwHAAAAAAAACCx +RQAAAAAAgJdFAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAA +AAAAAABjfRH+CgQEESBqRgAAAAAALF9HAAAAAABkDAAAAAAAAACiRQAAAAAAwJlFAAAAAAACAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAEAAAAAAADu8U1nCggIEcBzRQAA +AAAAZF9HAAAAAAAWGwAAAAAAAACFRQAAAAAAgJNFAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAA3YEvoCggIEcBzRQAAAAAALF9HAAAAAACeEAAAAAAA +AMCiRQAAAAAAgJpFAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAA +AAAAAAAAAACIQFIICggIETB0RQAAAAAALF9HAAAAAAANHgAAAAAAACDTRQAAAAAAwJhFAAAAAAAE +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAACEQhgcCgEBERhq +RgAAAAAALF9HAAAAAABUCgAAAAAAAICiRQAAAAAAQJpFAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAABoiq4oCggIEfBzRQAAAAAALF9HAAAAAAC3EQAA +AAAAAMCiRQAAAAAAgJpFAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFAAAA +AAAAAAUAAAAAAABP24XYCggIEXBzRQAAAAAAqF9HAAAAAABAHQAAAAAAAGCnRQAAAAAAgJVFAAAA +AAAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHgAAAAAAAAAaAAAAAAAAAAZjMioAggI +EQAAAAAAAAAAQl9HAAAAAADyGwAAAAAAAICTRQAAAAAAwJVFAAAAAAAFAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAADpwl6jCggIERB0RQAAAAAALF9HAAAAAADi +DgAAAAAAAMCiRQAAAAAAgJpFAAAAAAAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgA +AAAAAAAAAAAAAAAAAAA++TC0CgEBESBqRgAAAAAALF9HAAAAAABqCgAAAAAAAICiRQAAAAAAQJpF +AAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEgAAAAAAAAAAAAAAAAAAAAtOdYo +CggIEUB0RQAAAAAALF9HAAAAAADvDgAAAAAAAMCiRQAAAAAAgJpFAAAAAAAJAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAEgAAAAAAAAASAAAAAAAAAAFM862CggIEUB0RQAAAAAATF9HAAAA +AADUGAAAAAAAAACjRQAAAAAAwJpFAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAgAAAAAAAAACAAAAAAAAAAKs27iAggIMwAAAAAAAAAA+GdGAAAAAACAFgAAAAAAAAEAAQAAAAAA +QJtFAAAAAABAm0UAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAACt +kqG4AggIMwAAAAAAAAAA+GdGAAAAAABkHwAAAAAAAAIAAAAAAAAAYLBFAAAAAADAokUAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAAHATS7AggIEQAAAAAAAAAAO19H +AAAAAACsGAAAAAAAAGDhRQAAAAAAwJNFAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAMAAAAAAAAAAsAAAAAAAAADxvnI/AggIEQAAAAAAAAAAVF9HAAAAAAAsGwAAAAAAAMCTRQAA +AAAAAJZFAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAA +AAA1arvbAgQEEQAAAAAAAAAALF9HAAAAAABfCgAAAAAAAMCdRQAAAAAAQJZFAAAAAAAIAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAB2UHoQAggIEQAAAAAAAAAA +Tl9HAAAAAAAJHAAAAAAAAKDLRQAAAAAAwJZFAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAACAAAAAAAAAAAAAAAAAAAABEjF6/AgQEEQAAAAAAAAAALF9HAAAAAADYGQAAAAAAAECg +RQAAAAAAQJlFAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAA+g3kgAgEBEQAAAAAAAAAALF9HAAAAAADGEQAAAAAAAGCzRQAAAAAAgJlFAAAAAAAIAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAACliC7AAgQEEQAAAAAA +AAAALF9HAAAAAABwDAAAAAAAAACiRQAAAAAAwJlFAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAAlQOcgAggIEQAAAAAAAAAAO19HAAAAAADAGAAAAAAA +AACjRQAAAAAAwJpFAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABgAAAAAAAAA +CAAAAAAAAADpLZjBBwgIFwAAAAAAAAAA+GdGAAAAAAB8HgAAgC0AAICiRQAAAAAAGAYAAAAAAAAQ +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAC0EEGSAggIMwAA +AAAAAAAA+GdGAAAAAABYIAAAAAAAAAIAAQAAAAAAgIdFAAAAAAAAo0UAAAAAAECbRQAAAAAAAAAA +AAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAABycxZ1AggIMwAAAAAAAAAA+GdGAAAAAACsIAAA +AAAAAAIAAQAAAAAAAKNFAAAAAAAAo0UAAAAAAECbRQAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAA +AAAAEAAAAAAAAADnV6AYAggIFFBqRgAAAAAACGhGAAAAAADkEQAAAAAAAAAAAAAAAAAAsLBFAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAACoZcUaDwgI +DCBqRgAAAAAALF9HAAAAAABgGQAAIFQAABgGAAABAAAAEAAAAAAAAADqAAAA//////////////// +AAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAACM9DF4DwgIDCBqRgAAAAAALF9HAAAAAADu +FwAAoGQAABgGAAABAAAAEAAAAAAAAADqAAAA////////////////AAAAAAAAAAAAAAAAAAAAAAgA +AAAAAAAAAAAAAAAAAAB9mbImDwgIAiBqRgAAAAAALF9HAAAAAAABGAAA4FQAABgGAAABAAEAEAAA +AAAAAADYBAAAAD0AAOBABQAAhQAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAB86fQt +DwgIDCBqRgAAAAAALF9HAAAAAAAnGAAAIF8AABgGAAABAAAAEAAAAAAAAADqAAAA//////////// +////AAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAB0lLlbDwgIDCBqRgAAAAAALF9HAAAA +AABgGAAAoGAAABgGAAABAAAAEAAAAAAAAADqAAAA////////////////AAAAAAAAAAAAAAAAAAAA +AAgAAAAAAAAAAAAAAAAAAADwg/n6DwgIDCBqRgAAAAAALF9HAAAAAADGGwAAoGEAABgGAAABAAAA +EAAAAAAAAADqAAAA////////////////AAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAABA +KR9vDwEBCChqRgAAAAAALF9HAAAAAACqGgAAAFYAABgGAAABAAEAEAAAAAAAAADYBAAAAD0AAKBB +BQBAtAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAbrPYnCgEBGQBqRgAAAAAALF9H +AAAAAADEDAAAAAAAAAAAAAAAAAAAsLNFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAgAAAAAAAAACAAAAAAAAADb3/McCQgINiBqRgAAAAAA+GdGAAAAAABMGQAAAAAAAEDMRQAA +AAAAGAYAAAEAAAAQAAAAAAAAALgFAAD///////////////8AAAAAAAAAAAgAAAAAAAAACAAAAAAA +AABIlWAFCQgINiBqRgAAAAAA+GdGAAAAAABgGQAAAAAAAMCwRQAAAAAAGAYAAAEAAAAQAAAAAAAA +AOoAAAD///////////////8AAAAAAAAAAAgAAAAAAAAACAAAAAAAAAB0IHmFCQgINiBqRgAAAAAA ++GdGAAAAAAAREgAAAAAAAADqRQAAAAAAGAYAAAEAAAAQAAAAAAAAAKYBAAAAPQAAwHMAAMBzAAAA +AAAAAAAAAAgAAAAAAAAACAAAAAAAAAAvukdcCQgINiBqRgAAAAAA+GdGAAAAAAABGAAAAAAAAICx +RQAAAAAAGAYAAAEAAQAQAAAAAAAAANgEAAAAPQAA4EAFAOBABQAAAAAAAAAAAAgAAAAAAAAAAAAA +AAAAAABuln/+CQgIFiBqRgAAAAAALF9HAAAAAACIGQAAAAAAAIC6RQAAAAAAGAYAAAEAAAAQAAAA +AAAAAIsAAAD///////////////8AAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAqQUwqCQgIFiBqRgAA +AAAALF9HAAAAAABNGAAAAAAAAGD8RQAAAAAAGAYAAAEAAAAQAAAAAAAAACYHAAD///////////// +//8AAAAAAAAAAAgAAAAAAAAACAAAAAAAAABDNPNdCQgINiBqRgAAAAAA+GdGAAAAAACqGgAAAAAA +AACzRQAAAAAAGAYAAAEAAQAQAAAAAAAAANgEAAAAPQAAoEEFAKBBBQAAAAAAAAAAAAIAAAAAAAAA +AAAAAAAAAAANvaRaDwEBERBqRgAAAAAALF9HAAAAAACEGwAAoFsAAICiRQAAAAAAQJpFAAAAAAAC +AAAAAAAAAIIQAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAuwhtSAggINQAA +AAAAAAAA+GdGAAAAAAAoHgAAAAAAAMCdRQAAAAAAAKNFAAAAAABg3kUAAAAAADhqRgAAAAAABAhw +AAQAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAACndrzrAggINQAAAAAAAAAA+GdGAAAAAADCHwAA +AAAAAKDLRQAAAAAAYLNFAAAAAAAg30UAAAAAADBqRgAAAAAAEACQAAQAAAAAAAAAAAAAAAgAAAAA +AAAACAAAAAAAAAD3OOqKAggINQAAAAAAAAAA+GdGAAAAAAAtIAAAAAAAAECgRQAAAAAAYOFFAAAA +AADg30UAAAAAADhqRgAAAAAABAhwAAQAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAD4vSH3AggI +NQAAAAAAAAAA+GdGAAAAAACCHwAAAAAAAACiRQAAAAAAwJNFAAAAAACg4EUAAAAAADhqRgAAAAAA +BBjwAAQAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABVRAIdDwgIEcBzRQAAAAAALF9HAAAAAAAs +GgAAQCYAAMCiRQAAAAAAgJpFAAAAAAAgAAAAAAAAABgGAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAgA +AAAAAAAAAAAAAAAAAACErYHEDwQEESBqRgAAAAAALF9HAAAAAAAXFgAAQC0AAACiRQAAAAAAwJlF +AAAAAAACAAAAAAAAABgGAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADFciHL +DwEBGQBqRgAAAAAALF9HAAAAAADbFwAAwCYAAAAAAAAAAAAAYLlFAAAAAAAAAAAAAAAAAAAAAAAA +AAAAGAYAAAAAAAAQAAAAAAAAABAAAAAAAAAACAAAAAAAAACZXwxMBwgIGNBqRgAAAAAA+GdGAAAA +AACwGwAAIF0AABgGAAACAAIAEAAAAAAAAACHAgAAAD0AAMA5BQBAVAAA1hAAAAA7AAD///////// +/wAAAAAAAAAAAAAAAAAAAACjW3qZDwEBGQBqRgAAAAAALF9HAAAAAACLHQAAACoAAAAAAAAAAAAA +ILpFAAAAAAAAAAAAAAAAAAAAAAAAAAAAGAYAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACq +kUoUDwEBGQBqRgAAAAAALF9HAAAAAADIEAAAgCoAAAAAAAAAAAAAgLpFAAAAAAAAAAAAAAAAAAAA +AAAAAAAAGAYAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABfyuBODwEBGQBqRgAAAAAALF9H +AAAAAACIGQAAQFUAAAAAAAAAAAAA4LpFAAAAAAAAAAAAAAAAAAAAAAAAAAAAGAYAAAAAAAAQAAAA +AAAAABAAAAAAAAAACAAAAAAAAAARn0T1BwgIGNBqRgAAAAAA+GdGAAAAAACAGgAAIGAAABgGAAAC +AAIAEAAAAAAAAACHAgAAAD0AACBDBQAgVQAA1hAAAAA7AAD//////////wEAAAAAAAAAAAAAAAAA +AABNnA0/DwEBCChqRgAAAAAALF9HAAAAAADEGQAAIGEAABgGAAACAAAAEAAAAAAAAAB4BQAA//// +////////////AQwAAP///////////////wgAAAAAAAAACAAAAAAAAAAL6ApDCQgINiBqRgAAAAAA ++GdGAAAAAACEGwAAAAAAAGC2RQAAAAAAghAAAAIAAgAQAAAAAAAAAIEAAAD///////////////+G +AAAA////////////////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAMPq +5NIJCAg2IGpGAAAAAAD4Z0YAAAAAAJgeAAAAAAAAYORFAAAAAAAYBgAAAgACABAAAAAAAAAAhwIA +AAA9AACAUAAAgFAAANYQAAAAOwAA//////////8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAA +AAAAAAgAAAAAAAAA905zDQkICDYgakYAAAAAAPhnRgAAAAAAmhsAAAAAAADg6kUAAAAAABgGAAAC +AAIAEAAAAAAAAACHAgAAAD0AAGA9BQBgPQUA1hAAAAA7AAD//////////wAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAABEWsxwCQgINiBqRgAAAAAA+GdGAAAAAACwGwAAAAAA +AGC5RQAAAAAAGAYAAAIAAgAQAAAAAAAAAIcCAAAAPQAAwDkFAMA5BQDWEAAAADsAAP////////// +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAB14FgYJCAgWIGpGAAAAAAAs +X0cAAAAAANMVAAAAAAAAwJ5FAAAAAAAYBgAAAgAAABAAAAAAAAAAWAEAAP///////////////6MC +AAD///////////////8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAZ6c+ +mAkICDYgakYAAAAAAPhnRgAAAAAAkBQAAAAAAABA9kUAAAAAABgGAAACAAAAEAAAAAAAAAB4CQAA +////////////////rQcAAP///////////////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAA +AAAACAAAAAAAAADKUlePCQgINiBqRgAAAAAA+GdGAAAAAAAQHQAAAAAAACDHRQAAAAAAGAYAAAIA +AAAQAAAAAAAAAL0AAAD////////////////+AAAA////////////////AAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAId/OTYJCAg2IGpGAAAAAAD4Z0YAAAAAACcYAAAAAAAA +4LFFAAAAAAAYBgAAAgAAABAAAAAAAAAA6gAAAP////////////////4AAAD///////////////8A +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAwkzTRQkICDYgakYAAAAAAPhn +RgAAAAAAuAwAAAAAAADABEYAAAAAABgGAAACAAAAEAAAAAAAAACfBgAAADsAAEBnAwBAZwMApgEA +AP////9gZQMAYGUDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAD6Ri8 +CQgINiBqRgAAAAAA+GdGAAAAAACAGgAAAAAAAOC6RQAAAAAAGAYAAAIAAgAQAAAAAAAAAIcCAAAA +PQAAIEMFACBDBQDWEAAAADsAAP//////////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAA +AAAIAAAAAAAAALkifswJCAg2IGpGAAAAAAD4Z0YAAAAAAGAYAAAAAAAAQLJFAAAAAAAYBgAAAgAA +ABAAAAAAAAAA6gAAAP////////////////4AAAD///////////////8AAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAA8+aSmgkICDYgakYAAAAAAPhnRgAAAAAAxBkAAAAAAABA +u0UAAAAAABgGAAACAAAAEAAAAAAAAAB4BQAA////////////////AQwAAP///////////////wAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAC24U5RCQgINiBqRgAAAAAA+GdG +AAAAAADGGwAAAAAAAKCyRQAAAAAAGAYAAAIAAAAQAAAAAAAAAOoAAAD////////////////+AAAA +////////////////AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAADvaKNEJ +CAgWIGpGAAAAAAAsX0cAAAAAABAXAAAAAAAAYNRFAAAAAAAYBgAAAgAAABAAAAAAAAAADA0AAAA7 +AAD/////4MgBAE0SAAAAOwAA/////4DIAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAA +ABAAAAAAAAAAy14QDgcICBTgaUYAAAAAAAhoRgAAAAAAoAQAAIAiAAAAAAAAAAAAAADDRQAAAAAA +AQAAAAAAAAABAAAAAAAAAAAAAAAAAAAAGAAAAAAAAACHAgAAAD0AAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAABAAAAAAAAAAEAAAAAAAAACn07N5BwgIFOBpRgAAAAAACGhGAAAAAABzGAAAwC4AABhm +RQAAAAAAgMNFAAAAAAABAAAAAAAAAAEAAAAAAAAAGAYAAAAAAAAYAAAAAAAAANgEAAAAPQAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAMIZzoQJCAg2IGpGAAAAAAD4Z0YA +AAAAAPMRAAAAAAAAoMVFAAAAAAAYBgAAAwAAABAAAAAAAAAAsgEAAP///////////////1oJAAD/ +//////////////8AEAAA////////////////AAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAaGvD6QkI +CDYgakYAAAAAAPhnRgAAAAAAtR4AAAAAAAAg5UUAAAAAABgGAAADAAMAEAAAAAAAAAAoAQAA//// +////////////hwIAAAA9AACgQgUAoEIFANYQAAAAOwAA//////////8AAAAAAAAAAAgAAAAAAAAA +CAAAAAAAAADSoiO/CQgINiBqRgAAAAAA+GdGAAAAAADuFwAAAAAAACCxRQAAAAAAGAYAAAMAAAAQ +AAAAAAAAAKQAAAD////////////////qAAAA/////////////////gAAAP///////////////wAA +AAAAAAAACAAAAAAAAAAIAAAAAAAAAHkRDF4JCAg2IGpGAAAAAAD4Z0YAAAAAAAAVAAAAAAAAwNNF +AAAAAAAYBgAAAwAAABAAAAAAAAAAlgYAAP///////////////wARAAD///////////////+6BgAA +////////////////AAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAxd59Sg8BARkgakYAAAAAACxfRwAA +AAAA8xEAAKBjAAAYZkUAAAAAAADGRQAAAAAAAQAAAAAAAAABAAAAAAAAABgGAAAAAAAAKAAAAAAA +AAD1Z0UAAAAAAGCqRQAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAADFfZuFDwgI +GSBqRgAAAAAALF9HAAAAAACkFgAAQCcAABhmRQAAAAAAgMZFAAAAAAABAAAAAAAAAAEAAAAAAAAA +GAYAAAAAAAAoAAAAAAAAACxgRQAAAAAAwKJFAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAA +AAAAAAAAAGuHKzUPCAgZIGpGAAAAAAAsX0cAAAAAAHAUAAAgaQAAGGZFAAAAAAAAx0UAAAAAAAEA +AAAAAAAAAQAAAAAAAAAYBgAAAAAAACgAAAAAAAAAmmFFAAAAAAAgsUUAAAAAAAAAAAAAAAAAAAAA +AAAAAAABAAAAAAAAAAAAAAAAAAAAUwz4vQ8BARkoakYAAAAAACxfRwAAAAAAEB0AAKBeAAAYZkUA +AAAAAIDHRQAAAAAAAQAAAAAAAAABAAAAAAAAABgGAAAAAAAAKAAAAAAAAAAaYEUAAAAAAECfRQAA +AAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAADn7u4hDwgIGSBqRgAAAAAALF9HAAAA +AAAvEgAAQCwAABhmRQAAAAAAAMhFAAAAAAABAAAAAAAAAAEAAAAAAAAAGAYAAAAAAAAoAAAAAAAA +AMJgRQAAAAAAwKJFAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAANSakbkCCAgZ +kGRFAAAAAAAsX0cAAAAAAOQfAAAAawAAGGZFAAAAAABwyEUAAAAAAAIAAAAAAAAAAgAAAAAAAAAh +Y0UAAAAAAKDGRQAAAAAAAQAAAAAAAAAUYEUAAAAAAMCdRQAAAAAAEAAAAAAAAAAIBAAAAAAAAAAA +AAAAAAAAoZ/9MAoICBlAc0UAAAAAACxfRwAAAAAABCEAAAAAAAAYZkUAAAAAAPDIRQAAAAAAAgAA +AAAAAAACAAAAAAAAAMdgRQAAAAAAQJ1FAAAAAAAAAAAAAAAAAJ9gRQAAAAAAAKRFAAAAAAAQAAAA +AAAAAAgAAAAAAAAACAAAAAAAAAA7l7quCQgINiBqRgAAAAAA+GdGAAAAAABwFAAAAAAAAKDGRQAA +AAAAGAYAAAQAAAAQAAAAAAAAANsCAAD////////////////lAAAA////////////////HgIAAP// +/////////////24HAAD///////////////8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAA +AAgAAAAAAAAAfLFRsgkICDYgakYAAAAAAPhnRgAAAAAAnBkAAAAAAABg20UAAAAAABgGAAAEAAAA +EAAAAAAAAACcAgAA/////yAmAgAgJgIA8AQAAP/////gJgIA4CYCANsCAAD///////////////8T +AwAA/////yAoAgAgKAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAACAD +vLkJCAg2IGpGAAAAAAD4Z0YAAAAAABAVAAAAAAAAoN1FAAAAAAAYBgAABAAAABAAAAAAAAAAqAYA +AAA7AAD//////////9sCAAD///////////////+ABwAA////////////////mAMAAAA7AAAgXwIA +IF8CAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAB3vU9zCQgINiBqRgAA +AAAA+GdGAAAAAADkHwAAAAAAACDIRQAAAAAAAAAAAAQAAAAQAAAAAAAAAMoLAAD///////////// +//+bBwAA////////////////lgkAAP///////////////+YPAAD///////////////8AAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAgNWcRA8ICBkIakYAAAAAABhoRgAAAAAA +OBkAAIAlAAAYZkUAAAAAAADMRQAAAAAAAgAAAAAAAAACAAAAAAAAABgGAAAAAAAAQAAAAAAAAABo +YEUAAAAAAGDhRQAAAAAAAAAAAAAAAABsYEUAAAAAAGDhRQAAAAAAEAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAEAAAAAAAAAAQAAAAAAAAABWXefsHCAgZMGRFAAAAAAAIaEYAAAAAAEwZAADAUwAAGGZF +AAAAAACgzEUAAAAAAAIAAAAAAAAAAgAAAAAAAAAYBgAAAAAAAEAAAAAAAAAAFGBFAAAAAADAnUUA +AAAAAAAAAAAAAAAAamhFAAAAAADAkkUAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAA +AAAAEAAAAAAAAAAXOU8ADwgIGQhqRgAAAAAACGhGAAAAAABgHgAAgCYAABhmRQAAAAAAQM1FAAAA +AAACAAAAAAAAAAIAAAAAAAAAGAYAAAAAAABAAAAAAAAAAJxoRQAAAAAAwKJFAAAAAAAAAAAAAAAA +AHRoRQAAAAAAgIdFAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAA +D1P/vQ8ICBkIakYAAAAAACxfRwAAAAAAkhYAAAAnAAAYZkUAAAAAAODNRQAAAAAAAgAAAAAAAAAC +AAAAAAAAABgGAAAAAAAAQAAAAAAAAADiYkUAAAAAAMCiRQAAAAAAAAAAAAAAAADVZkUAAAAAAMCi +RQAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAO0Pyd4PCAgZCGpG +AAAAAAAsX0cAAAAAAGUcAADAKAAAGGZFAAAAAACAzkUAAAAAAAIAAAAAAAAAAgAAAAAAAAAYBgAA +AAAAAEAAAAAAAAAACmlFAAAAAADAokUAAAAAAAAAAAAAAAAAJGJFAAAAAACAsUUAAAAAABAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAACTEynGDwQEGSBqRgAAAAAALF9HAAAA +AAC2FgAAACkAABhmRQAAAAAAIM9FAAAAAAACAAAAAAAAAAIAAAAAAAAAGAYAAAAAAABAAAAAAAAA +ANxhRQAAAAAAgJ9FAAAAAAAAAAAAAAAAAKxhRQAAAAAAQKBFAAAAAAAIAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAA+kk6kg8ICBkIakYAAAAAACxfRwAAAAAA9RUAAIApAAAY +ZkUAAAAAAMDPRQAAAAAAAgAAAAAAAAACAAAAAAAAABgGAAAAAAAAQAAAAAAAAADoYUUAAAAAAECi +RQAAAAAAAAAAAAAAAAB3Z0UAAAAAAMCiRQAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAA +AAAAAAAAAAAAAAAAAHc6To8PCAgZCGpGAAAAAAAsX0cAAAAAAHQZAADgdgAAGGZFAAAAAABg0EUA +AAAAAAIAAAAAAAAAAgAAAAAAAAAYBgAAAAAAAEAAAAAAAAAABWNFAAAAAABA8kUAAAAAAAAAAAAA +AAAAvmFFAAAAAABA8kUAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAA +AAB0WmzeDwgIGSBqRgAAAAAALF9HAAAAAACwFAAAwCsAABhmRQAAAAAAANFFAAAAAAACAAAAAAAA +AAIAAAAAAAAAGAYAAAAAAABAAAAAAAAAAFB1RQAAAAAAwLlFAAAAAAABAAAAAAAAAMJgRQAAAAAA +wKJFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAsrNBuA8ICBkI +akYAAAAAACxfRwAAAAAA2B0AAMAsAAAYZkUAAAAAAKDRRQAAAAAAAgAAAAAAAAACAAAAAAAAABgG +AAAAAAAAQAAAAAAAAABSYUUAAAAAAEC1RQAAAAAAAAAAAAAAAADbYEUAAAAAAMCiRQAAAAAAEAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAO7WzOAPBAQZIGpGAAAAAAAsX0cA +AAAAALAZAAAALQAAGGZFAAAAAABA0kUAAAAAAAIAAAAAAAAAAgAAAAAAAAAYBgAAAAAAAEAAAAAA +AAAA3GFFAAAAAACAn0UAAAAAAAAAAAAAAAAAHGFFAAAAAABAoEUAAAAAAAgAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAABC9vjqDwgIGQhqRgAAAAAALF9HAAAAAADAFAAAAC4A +ABhmRQAAAAAA4NJFAAAAAAACAAAAAAAAAAIAAAAAAAAAGAYAAAAAAABAAAAAAAAAAEBgRQAAAAAA +wKJFAAAAAAAAAAAAAAAAADhgRQAAAAAAwKJFAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ +AAAAAAAAAAAAAAAAAAAA9SZ8cg8ICBkIakYAAAAAACxfRwAAAAAAKB0AAEAuAAAYZkUAAAAAAIDT +RQAAAAAAAgAAAAAAAAACAAAAAAAAABgGAAAAAAAAQAAAAAAAAADKYUUAAAAAAMCwRQAAAAAAAAAA +AAAAAABCYkUAAAAAAMCiRQAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAQAAAA +AAAAALPuGJ0PCAgZCGpGAAAAAAAYaEYAAAAAAAAVAAAgZQAAGGZFAAAAAAAg1EUAAAAAAAIAAAAA +AAAAAgAAAAAAAAAYBgAAAAAAAEAAAAAAAAAABWNFAAAAAAAAj0UAAAAAAAAAAAAAAAAAvmFFAAAA +AAAAj0UAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAACRLg2DwgI +GaBzRQAAAAAALF9HAAAAAAAQFwAAIGIAABhmRQAAAAAAwNRFAAAAAAACAAAAAAAAAAIAAAAAAAAA +GAYAAAAAAABAAAAAAAAAALxuRQAAAAAAANVFAAAAAAABAAAAAAAAANZgRQAAAAAAQKZFAAAAAAAw +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAcBCgAA8ICBmwc0UAAAAAACxf +RwAAAAAAvxoAAEAxAAAYZkUAAAAAAGDVRQAAAAAAAgAAAAAAAAACAAAAAAAAABgGAAAAAAAAQAAA +AAAAAAD0YUUAAAAAAGDPRQAAAAAAAAAAAAAAAADuYUUAAAAAAECdRQAAAAAAIAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAIbxkhsJCAgWIGpGAAAAAAAsX0cAAAAAAAYWAAAA +AAAAAPBFAAAAAAAYBgAABQAAABAAAAAAAAAA9AwAAP//////////wPgAADIJAAD//////////wCR +AABKFgAAADsAAP////8g/QAAyAUAAP//////////IPYAAAgOAAAAOwAA/////0D7AAAAAAAAAAAA +ABgAAAAAAAAAGAAAAAAAAABmTEkdAggIGQAAAAAAAAAAQGhGAAAAAABAQwAAAAAAABhmRQAAAAAA +kNZFAAAAAAADAAAAAAAAAAMAAAAAAAAA0GFFAAAAAACg0EUAAAAAAAAAAAAAAAAAVGJFAAAAAAAA +okUAAAAAABAAAAAAAAAALGBFAAAAAACAq0UAAAAAACAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAA +AAAAAAAAbE5LsgkICBYgakYAAAAAACxfRwAAAAAAdBkAAAAAAAAA0EUAAAAAABgGAAAGAAAAEAAA +AAAAAACmAQAAADsAAP//////////MAUAAP//////////4PIBAGwNAAD///////////////8CBwAA +////////////////0AUAAP//////////YPEBANoHAAD///////////////8AAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAoAAAAAAAAAAgAAAAAAAAANlLJmAcICBkAAAAAAAAAAPhnRgAAAAAANxwAAMAl +AAAYZkUAAAAAAADYRQAAAAAAAwAAAAAAAAADAAAAAAAAABgGAAAAAAAAWAAAAAAAAADgYEUAAAAA +AICaRQAAAAAAAAAAAAAAAACIYUUAAAAAAACeRQAAAAAAMAAAAAAAAACOYUUAAAAAAMCiRQAAAAAA +QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAd+dJ8A8ICBnQ +c0UAAAAAACxfRwAAAAAA0h4AAMAnAAAYZkUAAAAAAMDYRQAAAAAAAwAAAAAAAAADAAAAAAAAABgG +AAAAAAAAWAAAAAAAAAD0YUUAAAAAAGDPRQAAAAAAAAAAAAAAAAA0YEUAAAAAACCxRQAAAAAAIAAA +AAAAAAARYEUAAAAAAOCxRQAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABQAAAA +AAAAAEAAAAAAAAAApkDEhgcICBkAAAAAAAAAADlfRwAAAAAA+BwAAEApAAAYZkUAAAAAAIDZRQAA +AAAAAwAAAAAAAAADAAAAAAAAABgGAAAAAAAAWAAAAAAAAAAcYUUAAAAAAMD0RQAAAAAAAAAAAAAA +AABKZ0UAAAAAAMDrRQAAAAAAYAAAAAAAAADWYUUAAAAAAACYRQAAAAAAcAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAYAAAAAAAAABgAAAAAAAAAGfOYFA8ICBmwc0UAAAAAADhoRgAAAAAA +FBgAAAArAAAYZkUAAAAAAEDaRQAAAAAAAwAAAAAAAAADAAAAAAAAABgGAAAAAAAAWAAAAAAAAABa +aUUAAAAAAECBRQAAAAAAAAAAAAAAAAClb0UAAAAAAECBRQAAAAAAEAAAAAAAAABUcUUAAAAAAKDD +RQAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAAAAAAAACgAAAAAAAAAiU3P +9QcICBloZEUAAAAAAC9fRwAAAAAAaxoAAIArAAAYZkUAAAAAAADbRQAAAAAAAwAAAAAAAAADAAAA +AAAAABgGAAAAAAAAWAAAAAAAAACEbUUAAAAAAEChRQAAAAAAAAAAAAAAAAAqcUUAAAAAAEChRQAA +AAAAIAAAAAAAAADzb0UAAAAAAICRRQAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAYAAAAAAAAAAAAAAAAAAAAwkHjqg8ICBmwc0UAAAAAACxfRwAAAAAAnBkAAMBpAAAYZkUAAAAA +AMDbRQAAAAAAAwAAAAAAAAADAAAAAAAAABgGAAAAAAAAWAAAAAAAAABSYUUAAAAAAMCiRQAAAAAA +AAAAAAAAAACxYkUAAAAAAECiRQAAAAAAEAAAAAAAAAA2YkUAAAAAAECiRQAAAAAAIAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAWYM96wcICBl4ZEUAAAAAACxf +RwAAAAAA7BYAAMAtAAAYZkUAAAAAAIDcRQAAAAAAAwAAAAAAAAADAAAAAAAAABgGAAAAAAAAWAAA +AAAAAADoYUUAAAAAAMCNRQAAAAAAAAAAAAAAAACIZUUAAAAAAMChRQAAAAAAEAAAAAAAAAC4YUUA +AAAAAICiRQAAAAAAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAA +AAAAdH2SRg8ICBmwc0UAAAAAACxfRwAAAAAAhhgAAIAvAAAYZkUAAAAAAEDdRQAAAAAAAwAAAAAA +AAADAAAAAAAAABgGAAAAAAAAWAAAAAAAAADeY0UAAAAAAMCiRQAAAAAAAAAAAAAAAABQZUUAAAAA +AMCiRQAAAAAAEAAAAAAAAABgaEUAAAAAAMCiRQAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAQEAAAAAAAAAAAAAAAAAAAzJa33A8ICBkAdEUAAAAAACxfRwAAAAAAEBUAAGBqAAAY +ZkUAAAAAAADeRQAAAAAAAwAAAAAAAAADAAAAAAAAABgGAAAAAAAAWAAAAAAAAADoYUUAAAAAAMCi +RQAAAAAAAAAAAAAAAACpYEUAAAAAAMCiRQAAAAAAEAAAAAAAAACfYEUAAAAAAOCoRQAAAAAAIAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABwAAAAAAAAAHAAAAAAAAAAUiZN6AIICBkAAAAA +AAAAAEpfRwAAAAAAoB8AAAAAAAAAYEUAAAAAALDeRQAAAAAABAAAAAAAAAAEAAAAAAAAAOxnRQAA +AAAAYKpFAAAAAAAAAAAAAAAAALJhRQAAAAAAAK1FAAAAAAAQAAAAAAAAANRiRQAAAAAA4K5FAAAA +AABQAAAAAAAAAFppRQAAAAAAAKNFAAAAAADQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACQAAAAAAAA +AJAAAAAAAAAAINo3WQIICBkAAAAAAAAAAFxfRwAAAAAAgCAAAAAAAAAAYEUAAAAAAHDfRQAAAAAA +BAAAAAAAAAAEAAAAAAAAAOxnRQAAAAAAYKpFAAAAAAAAAAAAAAAAALJhRQAAAAAAYK1FAAAAAAAQ +AAAAAAAAANRiRQAAAAAAIK5FAAAAAAAQAQAAAAAAAFppRQAAAAAAAKNFAAAAAAAQAQAAAAAAAAAA +AAAAAAAAAAAAAAAAAABwAAAAAAAAAHAAAAAAAAAAb7gvUQIICBkAAAAAAAAAAEpfRwAAAAAA2CAA +AAAAAAAAYEUAAAAAADDgRQAAAAAABAAAAAAAAAAEAAAAAAAAAOxnRQAAAAAAYKpFAAAAAAAAAAAA +AAAAALJhRQAAAAAAwK1FAAAAAAAQAAAAAAAAANRiRQAAAAAAQKxFAAAAAABQAAAAAAAAAFppRQAA +AAAAAKNFAAAAAADQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADwAAAAAAAAAPAAAAAAAAAAgOI9/AII +CBkAAAAAAAAAAGBfRwAAAAAACCAAAAAAAAAAYEUAAAAAAPDgRQAAAAAABAAAAAAAAAAEAAAAAAAA +AOxnRQAAAAAAYKpFAAAAAAAAAAAAAAAAALJhRQAAAAAAgK5FAAAAAAAQAAAAAAAAANRiRQAAAAAA +oKxFAAAAAABQAAAAAAAAAFppRQAAAAAAAKNFAAAAAADQAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAI +AAAAAAAAAAgAAAAAAAAABShogwkICDYgakYAAAAAAPhnRgAAAAAAYBQAAAAAAADA9EUAAAAAABgG +AAAHAAAAEAAAAAAAAADcAQAAAD0AAP//////////OAcAAP///////////////0oHAAAAPQAAIMUE +ACDFBADoBQAAAD0AAADEBAAAxAQA4wcAAP////9AywQAQMsEAP4HAAD///////////////8iCgAA +/////6DEBACgxAQAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAxmmpwwkICDYgakYAAAAAAPhnRgAA +AAAAAhIAAAAAAACA80UAAAAAABgGAAAHAAAAEAAAAAAAAAAwFQAAADsAAP//////////3gYAAP// +/////////////4kSAAAAOwAAQKUAAEClAACLDwAA/////8ClAADApQAAmA8AAP////////////// +/7YSAAD///////////////9+EQAA////////////////AAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAA +xWetNQkICDYgakYAAAAAAPhnRgAAAAAAIBIAAAAAAADA60UAAAAAABgGAAAHAAAAEAAAAAAAAABw +AQAA////////////////+QYAAP///////////////3gNAAD////////////////cAQAAAD0AAP// +////////QQcAAAA9AAD///////////wNAAD///////////////8SAQAAAD0AAP//////////AAAA +AAAAAAAQAQAAAAAAAAgBAAAAAAAAoB+BkQcICBkIZEUAAAAAAGhfRwAAAAAAuhAAAIAkAAAAAAAA +AAAAAADkRQAAAAAABAAAAAAAAAAEAAAAAAAAAIIQAAAAAAAAcAAAAAAAAAAuYUUAAAAAAMCqRQAA +AAAAAAAAAAAAAADQZEUAAAAAAIClRQAAAAAAkAAAAAAAAAA6YUUAAAAAACCrRQAAAAAAgAEAAAAA +AAAWb0UAAAAAAGC2RQAAAAAAEAIAAAAAAAAoAAAAAAAAACAAAAAAAAAAlRTzBAcICBkYZEUAAAAA +AC1fRwAAAAAAmB4AACBcAAAYZkUAAAAAAMDkRQAAAAAABAAAAAAAAAAEAAAAAAAAABgGAAAAAAAA +cAAAAAAAAADQbEUAAAAAAGDhRQAAAAAAAAAAAAAAAAB+aEUAAAAAAGDhRQAAAAAAEAAAAAAAAABW +aEUAAAAAAGDhRQAAAAAAIAAAAAAAAACYckUAAAAAAEChRQAAAAAAMAAAAAAAAAAYAAAAAAAAAAgA +AAAAAAAAOMknogcICBlAZEUAAAAAAPhnRgAAAAAAtR4AACBkAAAYZkUAAAAAAIDlRQAAAAAAAgAA +AAAAAAACAAAAAAAAABgGAAADAAMAQAAAAAAAAADRYEUAAAAAAEChRQAAAAAAAAAAAAAAAABAYUUA +AAAAAMCiRQAAAAAAIAAAAAAAAAAoAQAA////////////////hwIAAAA9AACgQgUAoFQAANYQAAAA +OwAA//////////8oAAAAAAAAAAAAAAAAAAAAPIegfg8ICBngc0UAAAAAACxfRwAAAAAAThwAAIAo +AAAYZkUAAAAAAEDmRQAAAAAABAAAAAAAAAAEAAAAAAAAABgGAAAAAAAAcAAAAAAAAAC7Y0UAAAAA +AIDSRQAAAAAAAAAAAAAAAAAacEUAAAAAAMCiRQAAAAAAIAAAAAAAAAAncEUAAAAAAMCiRQAAAAAA +MAAAAAAAAADcaUUAAAAAAMCiRQAAAAAAQAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIgsogAcICBmI +ZEUAAAAAACxfRwAAAAAAlRoAAEAvAAAYZkUAAAAAAADnRQAAAAAABAAAAAAAAAAEAAAAAAAAABgG +AAAAAAAAcAAAAAAAAADra0UAAAAAAACiRQAAAAAAAAAAAAAAAAD2a0UAAAAAAACeRQAAAAAAEAAA +AAAAAAA0cEUAAAAAAACiRQAAAAAAIAAAAAAAAABBcEUAAAAAAACeRQAAAAAAMAAAAAAAAAAIAAAA +AAAAAAgAAAAAAAAAgg1N7wkICDYgakYAAAAAAPhnRgAAAAAA5BUAAAAAAAAg6UUAAAAAABgGAAAJ +AAAAEAAAAAAAAABpBgAAADsAAMDHAQDAxwEAsQYAAAA7AADAxgEAwMYBANsCAAD///////////// +//+mAQAAADsAAGDCAQBgwgEA7wAAAP/////AwgEAwMIBAG4JAAD/////4MMBAODDAQCABwAA//// +////////////AAYAAP/////gxQEA4MUBAKQOAAD///////////////8AAAAAAAAAACAAAAAAAAAA +GAAAAAAAAAD6l3abBwgIGRBkRQAAAAAAAGhGAAAAAAAJDwAAwCQAAAAAAAAAAAAAoOhFAAAAAAAF +AAAAAAAAAAUAAAAAAAAAkBAAAAAAAACIAAAAAAAAADRhRQAAAAAAQKFFAAAAAAAAAAAAAAAAAE5m +RQAAAAAAwIFFAAAAAAAgAAAAAAAAAJZqRQAAAAAAQJtFAAAAAAAwAAAAAAAAAMhkRQAAAAAAQJtF +AAAAAAAyAAAAAAAAAExoRQAAAAAAQJtFAAAAAAA0AAAAAAAAAAAAAAAAAAAAKAAAAAAAAAAAAAAA +AAAAAPg5EgQHCAgZSGRFAAAAAAAsX0cAAAAAAOQVAABghwAAGGZFAAAAAACA6UUAAAAAAAUAAAAA +AAAABQAAAAAAAAAYBgAAAAAAAIgAAAAAAAAA7GNFAAAAAAAgwkUAAAAAAAAAAAAAAAAA82NFAAAA +AAAgwkUAAAAAABAAAAAAAAAAMG9FAAAAAABAokUAAAAAACAAAAAAAAAAoGlFAAAAAAAAnkUAAAAA +ADAAAAAAAAAAZG9FAAAAAABAm0UAAAAAAEAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAABAAAAAAAAAA +4Mt8tQcICBlYZEUAAAAAABhoRgAAAAAAERIAAIBUAAAYZkUAAAAAAGDqRQAAAAAABQAAAAAAAAAF +AAAAAAAAABgGAAAAAAAAiAAAAAAAAAA9Y0UAAAAAAECJRQAAAAAAAAAAAAAAAACOYkUAAAAAAGDh +RQAAAAAAEAAAAAAAAACUYUUAAAAAAACiRQAAAAAAIAAAAAAAAAAFYEUAAAAAAICoRQAAAAAAKAAA +AAAAAACzYEUAAAAAAOClRQAAAAAAMAAAAAAAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAAAD5GAXV +BwgIGThkRQAAAAAALF9HAAAAAACaGwAAoFwAABhmRQAAAAAAQOtFAAAAAAAEAAAAAAAAAAQAAAAA +AAAAGAYAAAIAAgBwAAAAAAAAAB1gRQAAAAAAAJ5FAAAAAAAAAAAAAAAAACBgRQAAAAAAQJ1FAAAA +AAAQAAAAAAAAANhlRQAAAAAAQJtFAAAAAAAgAAAAAAAAAF5hRQAAAAAAgJ5FAAAAAAAiAAAAAAAA +AIcCAAAAPQAAYD0FAEBVAADWEAAAADsAAP//////////CAAAAAAAAAAIAAAAAAAAAGITyw8PCAg5 +IGpGAAAAAAD4Z0YAAAAAACASAADgggAAGGZFAAAAAAAg7EUAAAAAAAEAAAAAAAAAAQAAAAAAAAAY +BgAABwAAACgAAAAAAAAAqmJFAAAAAADAkkUAAAAAAAAAAAAAAAAAcAEAAP////////////////kG +AAD///////////////94DQAA////////////////3AEAAAA9AABAzgQAQM4EAEEHAAAAPQAAANAE +AADQBAD8DQAA////////////////EgEAAAA9AAAAzwQAAM8EAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAADAAAAAAAAAAAAAAAAAAAAA3Q9FMDwgIGRB0RQAAAAAALF9HAAAAAADIFgAAwCkAABhmRQAA +AAAAIO1FAAAAAAAGAAAAAAAAAAYAAAAAAAAAGAYAAAAAAACgAAAAAAAAACxgRQAAAAAAwKJFAAAA +AAAAAAAAAAAAABRgRQAAAAAAwKJFAAAAAAAQAAAAAAAAAExhRQAAAAAAwKJFAAAAAAAgAAAAAAAA +AFBgRQAAAAAAwKJFAAAAAAAwAAAAAAAAAFRgRQAAAAAAwKJFAAAAAABAAAAAAAAAAK5gRQAAAAAA +wKJFAAAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA4AAAAAAAAACAAAAAAAAAA3u7Ikg8ICBkg +dEUAAAAAACxfRwAAAAAAgBQAAEAoAAAYZkUAAAAAACDuRQAAAAAABwAAAAAAAAAHAAAAAAAAABgG +AAAAAAAAuAAAAAAAAABkYEUAAAAAAMCiRQAAAAAAAAAAAAAAAABIYEUAAAAAAMCiRQAAAAAAEAAA +AAAAAAAOYEUAAAAAACCxRQAAAAAAIAAAAAAAAABkYUUAAAAAAACjRQAAAAAAMAAAAAAAAAD0YEUA +AAAAAMCiRQAAAAAAQAAAAAAAAABEYEUAAAAAAMCiRQAAAAAAUAAAAAAAAAAkYEUAAAAAAMCiRQAA +AAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA4AAAAAAAAADAAAAAAAAAAhjhqKgcI +CBkoZEUAAAAAAOUURgAAAAAAwhUAAEAlAAAYZkUAAAAAAEDvRQAAAAAACAAAAAAAAAAIAAAAAAAA +ABgGAAAAAAAA0AAAAAAAAABGYUUAAAAAAACjRQAAAAAAAAAAAAAAAACQYEUAAAAAAGCwRQAAAAAA +EAAAAAAAAADEYUUAAAAAAECFRQAAAAAAMAAAAAAAAABIYEUAAAAAAMCiRQAAAAAAQAAAAAAAAABk +YEUAAAAAAACjRQAAAAAAUAAAAAAAAAC/a0UAAAAAAECbRQAAAAAAYAAAAAAAAABXZkUAAAAAAECb +RQAAAAAAYgAAAAAAAAAgZUUAAAAAAECbRQAAAAAAZAAAAAAAAACwBAAAAAAAAAAAAAAAAAAAvbMZ +jQcICBlgZEUAAAAAACxfRwAAAAAABhYAAKB1AAAYZkUAAAAAAGDwRQAAAAAACAAAAAAAAAAIAAAA +AAAAABgGAAAAAAAA0AAAAAAAAACobUUAAAAAAMCiRQAAAAAAAAAAAAAAAADVa0UAAAAAAMCiRQAA +AAAAEAAAAAAAAABOYkUAAAAAAMCiRQAAAAAAIAAAAAAAAAB0bkUAAAAAAMCiRQAAAAAAMAAAAAAA +AABobkUAAAAAAMCiRQAAAAAAQAAAAAAAAACcYkUAAAAAAMCkRQAAAAAAUAAAAAAAAAAgbkUAAAAA +ACCoRQAAAAAA0AgAAAAAAADYaEUAAAAAAACiRQAAAAAAUAkAAAAAAABIAAAAAAAAADAAAAAAAAAA +DJp5swcICBkAAAAAAAAAAOEURgAAAAAA8BQAAAAwAAAYZkUAAAAAAIDxRQAAAAAACAAAAAAAAAAI +AAAAAAAAABgGAAAAAAAA0AAAAAAAAABMYEUAAAAAAECyRQAAAAAAAAAAAAAAAABmYkUAAAAAAACe +RQAAAAAAEAAAAAAAAACoZUUAAAAAAACeRQAAAAAAIAAAAAAAAAALYEUAAAAAAOCrRQAAAAAAMAAA +AAAAAACQYEUAAAAAAGCwRQAAAAAAQAAAAAAAAAD5YEUAAAAAAMCiRQAAAAAAYAAAAAAAAAA8aUUA +AAAAAACeRQAAAAAAcAAAAAAAAADgZUUAAAAAAACiRQAAAAAAgAAAAAAAAAAIAAAAAAAAAAAAAAAA +AAAAyXRxwgkICBYgakYAAAAAACxfRwAAAAAAoBQAAAAAAABA/kUAAAAAABgGAAAPAAAAEAAAAAAA +AACZGAAA////////////////UgEAAP///////////////xgNAAD///////////////80FwAA//// +////////////Sg8AAAA7AAD/////oK4BACgFAAD///////////////+mAQAA//////////////// +OAUAAP///////////////0gFAAD///////////////85FgAA////////////////WBcAAP////// +/////////6cSAAD//////////8DJAABQCQAA////////////////ahcAAP//////////oMkAAOMS +AAAAOwAA/////wC+AQAAAAAAAAAAADAAAAAAAAAAMAAAAAAAAAB37NeaDwgIGRB0RQAAAAAA5BRG +AAAAAAACEgAAIIIAABhmRQAAAAAA4PNFAAAAAAAJAAAAAAAAAAkAAAAAAAAAGAYAAAAAAADoAAAA +AAAAALhiRQAAAAAAQJ1FAAAAAAAAAAAAAAAAAAxjRQAAAAAAgKJFAAAAAAAQAAAAAAAAAAJgRQAA +AAAAgKJFAAAAAAASAAAAAAAAAGdrRQAAAAAAwKFFAAAAAAAUAAAAAAAAAC9jRQAAAAAAAKJFAAAA +AAAYAAAAAAAAAHtmRQAAAAAAAKNFAAAAAAAgAAAAAAAAALRtRQAAAAAAAKNFAAAAAAAwAAAAAAAA +AEZrRQAAAAAAwKJFAAAAAABAAAAAAAAAAP5iRQAAAAAAAItFAAAAAABQAAAAAAAAAAAAAAAAAAAA +MAAAAAAAAAAoAAAAAAAAAELBTqEHCAgZAAAAAAAAAAAxX0cAAAAAAGAUAABggQAAGGZFAAAAAAAg +9UUAAAAAAAsAAAAAAAAACwAAAAAAAAAYBgAAAAAAABgBAAAAAAAAQmJFAAAAAADAokUAAAAAAAAA +AAAAAAAAZWdFAAAAAADAokUAAAAAABAAAAAAAAAAlGFFAAAAAAAAokUAAAAAACAAAAAAAAAAyWNF +AAAAAAAAoEUAAAAAACgAAAAAAAAAlWJFAAAAAACAokUAAAAAACoAAAAAAAAAMG1FAAAAAACAokUA +AAAAACwAAAAAAAAAuGFFAAAAAACAokUAAAAAAC4AAAAAAAAA6WJFAAAAAAAAsEUAAAAAADAAAAAA +AAAAGGVFAAAAAADAkkUAAAAAAEAAAAAAAAAADWFFAAAAAACAn0UAAAAAAFAAAAAAAAAAtGtFAAAA +AABAoEUAAAAAAFgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYAAAAAAAAABYAAAAAAAA +AIBnQVYHCAgZUGRFAAAAAABIX0cAAAAAAJAUAAAgXgAAGGZFAAAAAACg9kUAAAAAAAsAAAAAAAAA +CwAAAAAAAAAYBgAAAAAAABgBAAAAAAAAwGVFAAAAAACAoUUAAAAAAAAAAAAAAAAAkmhFAAAAAACA +oUUAAAAAABAAAAAAAAAAn2BFAAAAAAAAo0UAAAAAACAAAAAAAAAAumhFAAAAAADAoUUAAAAAADAA +AAAAAAAA+GRFAAAAAAAAokUAAAAAADgAAAAAAAAAxGhFAAAAAABg4UUAAAAAAEAAAAAAAAAArWNF +AAAAAACAoUUAAAAAAFAAAAAAAAAAkWNFAAAAAACAoUUAAAAAAGAAAAAAAAAAimNFAAAAAADA00UA +AAAAAHAAAAAAAAAApmNFAAAAAADA00UAAAAAAJAAAAAAAAAA0GFFAAAAAACg0EUAAAAAALAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAASAAAAAAAAAA4AAAAAAAAAFIs3i8HCAgZIGRFAAAA +AAA4X0cAAAAAALEVAAAAJQAAGGZFAAAAAAAg+EUAAAAAAAwAAAAAAAAADAAAAAAAAAAYBgAAAAAA +ADABAAAAAAAACGFFAAAAAADAnUUAAAAAAAAAAAAAAAAAtmdFAAAAAABAm0UAAAAAAAgAAAAAAAAA +oGFFAAAAAABAm0UAAAAAAAoAAAAAAAAAfWtFAAAAAABAm0UAAAAAAAwAAAAAAAAAZGBFAAAAAADA +okUAAAAAABAAAAAAAAAASGBFAAAAAADAokUAAAAAACAAAAAAAAAALGBFAAAAAABAh0UAAAAAADAA +AAAAAAAA6GRFAAAAAABAhUUAAAAAAEAAAAAAAAAAxGFFAAAAAAAAhUUAAAAAAFAAAAAAAAAAKGBF +AAAAAAAAo0UAAAAAAGAAAAAAAAAAYGJFAAAAAADAokUAAAAAAHAAAAAAAAAAzGZFAAAAAADAokUA +AAAAAIAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAHHHfn4PCAgZMHRFAAAAAAAsX0cAAAAAADoYAACA +LAAAGGZFAAAAAACg+UUAAAAAAAwAAAAAAAAADAAAAAAAAAAYBgAAAAAAADABAAAAAAAAUmNFAAAA +AAAAokUAAAAAAAAAAAAAAAAAAGJFAAAAAACAokUAAAAAAAgAAAAAAAAABmJFAAAAAACAokUAAAAA +AAoAAAAAAAAAYGNFAAAAAACAokUAAAAAAAwAAAAAAAAAXGdFAAAAAACAokUAAAAAAA4AAAAAAAAA +dWNFAAAAAABAnUUAAAAAABAAAAAAAAAAcGVFAAAAAACAoUUAAAAAACAAAAAAAAAAQHVFAAAAAADA +okUAAAAAADAAAAAAAAAAiGhFAAAAAADAokUAAAAAAEAAAAAAAAAAa3JFAAAAAADAokUAAAAAAFAA +AAAAAAAAsm9FAAAAAADAokUAAAAAAGAAAAAAAAAAwG1FAAAAAADAokUAAAAAAHAAAAAAAAAAWAAA +AAAAAABYAAAAAAAAACKJCxAHCAgZgGRFAAAAAABGX0cAAAAAANAUAAAALwAAGGZFAAAAAAAg+0UA +AAAAAA0AAAAAAAAADQAAAAAAAAAYBgAAAAAAAEgBAAAAAAAADmBFAAAAAACAh0UAAAAAAAAAAAAA +AAAA6GFFAAAAAAAAj0UAAAAAABAAAAAAAAAAEmJFAAAAAAAAj0UAAAAAACAAAAAAAAAAfGFFAAAA +AAAAo0UAAAAAADAAAAAAAAAAI29FAAAAAAAAnkUAAAAAAEAAAAAAAAAA2W9FAAAAAAAAnkUAAAAA +AFAAAAAAAAAA8GVFAAAAAAAAokUAAAAAAGAAAAAAAAAA9mhFAAAAAABAm0UAAAAAAGgAAAAAAAAA +yGdFAAAAAABAm0UAAAAAAGoAAAAAAAAAmGVFAAAAAAAAj0UAAAAAAHAAAAAAAAAALGpFAAAAAAAA +j0UAAAAAAIAAAAAAAAAAQGpFAAAAAAAAj0UAAAAAAJAAAAAAAAAACGBFAAAAAAAgvkUAAAAAAKAA +AAAAAAAAAAAAAAAAAADoAAAAAAAAAMAAAAAAAAAA9fqcLwcICBkAAAAAAAAAAFBfRwAAAAAATRgA +AKBVAAAYZkUAAAAAAMD8RQAAAAAADwAAAAAAAAAPAAAAAAAAABgGAAAAAAAAeAEAAAAAAADEYUUA +AAAAAKC1RQAAAAAAAAAAAAAAAADQYUUAAAAAAKDQRQAAAAAAEAAAAAAAAAAoYEUAAAAAAMCiRQAA +AAAAIAAAAAAAAACNZkUAAAAAAECbRQAAAAAAMAAAAAAAAAD3YkUAAAAAAECbRQAAAAAAMgAAAAAA +AABaYkUAAAAAAACiRQAAAAAAOAAAAAAAAAAqYkUAAAAAAMCiRQAAAAAAQAAAAAAAAABcYEUAAAAA +AMCiRQAAAAAAUAAAAAAAAABgYEUAAAAAACDxRQAAAAAAYAAAAAAAAABYYEUAAAAAAACeRQAAAAAA +8AAAAAAAAABsYkUAAAAAAMCiRQAAAAAAAAEAAAAAAAB0YEUAAAAAAMCiRQAAAAAAEAEAAAAAAAB4 +YEUAAAAAACDxRQAAAAAAIAEAAAAAAABwYEUAAAAAAACeRQAAAAAAsAEAAAAAAAA8YkUAAAAAAKC1 +RQAAAAAAwAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACIAAAAAAAAAAAAAAAAAAAAXmg/ +fQcICBlwZEUAAAAAACxfRwAAAAAAoBQAAECSAAAYZkUAAAAAAKD+RQAAAAAAFQAAAAAAAAAVAAAA +AAAAABgGAAAAAAAACAIAAAAAAADoYUUAAAAAAEDyRQAAAAAAAAAAAAAAAAASYkUAAAAAAEDyRQAA +AAAAEAAAAAAAAADKYUUAAAAAAODWRQAAAAAAIAAAAAAAAAAibEUAAAAAAMCiRQAAAAAAMAAAAAAA +AACAZUUAAAAAAMCiRQAAAAAAQAAAAAAAAABgdUUAAAAAAMCwRQAAAAAAUAAAAAAAAADYakUAAAAA +AMCiRQAAAAAAYAAAAAAAAABoZUUAAAAAAMCiRQAAAAAAcAAAAAAAAADcbEUAAAAAAECiRQAAAAAA +gAAAAAAAAAChakUAAAAAAKC9RQAAAAAAkAAAAAAAAABIbUUAAAAAAKC9RQAAAAAAoAAAAAAAAADw +aUUAAAAAAACiRQAAAAAAsAAAAAAAAAAAZUUAAAAAAACiRQAAAAAAuAAAAAAAAADobEUAAAAAAMCh +RQAAAAAAwAAAAAAAAAAMbEUAAAAAAEC7RQAAAAAAxAAAAAAAAADCY0UAAAAAACDHRQAAAAAAxgAA +AAAAAAAoaUUAAAAAAICiRQAAAAAAyAAAAAAAAAC6aEUAAAAAAMCiRQAAAAAA0AAAAAAAAABEY0UA +AAAAAMCiRQAAAAAA4AAAAAAAAAANcEUAAAAAAKDQRQAAAAAA8AAAAAAAAAC+aUUAAAAAAMCNRQAA +AAAAAAEAAAAAAAAAAAAAAAAAABgCAAAAAAAAGAIAAAAAAAAXv5bpBwgIGQAAAAAAAAAAgF9HAAAA +AABWGgAAQCsAABhmRQAAAAAAAAFGAAAAAAAnAAAAAAAAACcAAAAAAAAAGAYAAAAAAAC4AwAAAAAA +AGRpRQAAAAAAgIxFAAAAAAAAAAAAAAAAAHFvRQAAAAAAQJpFAAAAAAAQAAAAAAAAAL9iRQAAAAAA +wJlFAAAAAABAAAAAAAAAAMNmRQAAAAAAQJpFAAAAAABwAAAAAAAAAINjRQAAAAAAQJpFAAAAAACg +AAAAAAAAAIhrRQAAAAAAQJpFAAAAAADQAAAAAAAAAIJhRQAAAAAAQJdFAAAAAAAAAQAAAAAAAFdv +RQAAAAAAwKJFAAAAAAAwAQAAAAAAAGdjRQAAAAAAwKJFAAAAAABAAQAAAAAAAFljRQAAAAAAwKJF +AAAAAABQAQAAAAAAAEhiRQAAAAAAwKJFAAAAAABgAQAAAAAAAPBiRQAAAAAAwKJFAAAAAABwAQAA +AAAAAFxrRQAAAAAAwKJFAAAAAACAAQAAAAAAACRtRQAAAAAAwKJFAAAAAACQAQAAAAAAAHBhRQAA +AAAAwKJFAAAAAACgAQAAAAAAAM1iRQAAAAAAwKJFAAAAAACwAQAAAAAAAJpgRQAAAAAAwKJFAAAA +AADAAQAAAAAAAHZhRQAAAAAAwKJFAAAAAADQAQAAAAAAAEZpRQAAAAAAwKJFAAAAAADgAQAAAAAA +AM1qRQAAAAAAwKJFAAAAAADwAQAAAAAAAKlgRQAAAAAAwKJFAAAAAAAAAgAAAAAAABhlRQAAAAAA +wKJFAAAAAAAQAgAAAAAAAChjRQAAAAAAwKJFAAAAAAAgAgAAAAAAANdjRQAAAAAAwKJFAAAAAAAw +AgAAAAAAABBlRQAAAAAAwKJFAAAAAABAAgAAAAAAAE5wRQAAAAAAAJlFAAAAAABQAgAAAAAAAE5s +RQAAAAAAQJZFAAAAAACAAgAAAAAAAPlqRQAAAAAAQJRFAAAAAACwAgAAAAAAABhiRQAAAAAAgJhF +AAAAAADgAgAAAAAAAMxtRQAAAAAAQKFFAAAAAAAQAwAAAAAAAJNrRQAAAAAAQJhFAAAAAAAwAwAA +AAAAAIRtRQAAAAAAQKFFAAAAAABgAwAAAAAAAEZxRQAAAAAAQJhFAAAAAACAAwAAAAAAAPBmRQAA +AAAAgKJFAAAAAACwAwAAAAAAADxtRQAAAAAAQMxFAAAAAADAAwAAAAAAAONqRQAAAAAAQMxFAAAA +AADgAwAAAAAAAAdoRQAAAAAAgLdFAAAAAAAABAAAAAAAAJVgRQAAAAAAQJtFAAAAAAAQBAAAAAAA +AOhhRQAAAAAAQItFAAAAAAAgBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAnAAAAAAAA +6CYAAAAAAAAlsbg4BwgIGQAAAAAAAAAAvF9HAAAAAAC4DAAAoF8AABhmRQAAAAAAIAVGAAAAAAAs +AAAAAAAAACwAAAAAAAAAGAYAAAAAAAAwBAAAAAAAADxgRQAAAAAAwJ1FAAAAAAAAAAAAAAAAAOBl +RQAAAAAAAKJFAAAAAAAIAAAAAAAAAMRhRQAAAAAAQLJFAAAAAAAQAAAAAAAAAOtrRQAAAAAAAKJF +AAAAAAAgAAAAAAAAADRwRQAAAAAAAKJFAAAAAAAoAAAAAAAAADhuRQAAAAAAoOZFAAAAAAAwAAAA +AAAAABFgRQAAAAAA4LFFAAAAAABwAAAAAAAAAGBlRQAAAAAAoNVFAAAAAACAAAAAAAAAAKBlRQAA +AAAAYNtFAAAAAACQAAAAAAAAAMxvRQAAAAAAwKJFAAAAAADAAAAAAAAAAMJqRQAAAAAAoKlFAAAA +AADQAAAAAAAAAPJwRQAAAAAAQKlFAAAAAADAAQAAAAAAAO5qRQAAAAAAQKJFAAAAAADACwAAAAAA +ABxxRQAAAAAAQKJFAAAAAADQCwAAAAAAAIJpRQAAAAAAAKJFAAAAAADgCwAAAAAAAIxpRQAAAAAA +AKJFAAAAAADoCwAAAAAAADBiRQAAAAAAoKZFAAAAAADwCwAAAAAAAJJnRQAAAAAAILFFAAAAAADw +GwAAAAAAABpjRQAAAAAAIMhFAAAAAAAAHAAAAAAAACxuRQAAAAAAAJVFAAAAAAAgHAAAAAAAAOZp +RQAAAAAAYKRFAAAAAABQHAAAAAAAAJBtRQAAAAAAoMhFAAAAAABQJAAAAAAAAARqRQAAAAAAoLJF +AAAAAABgLAAAAAAAAIBuRQAAAAAAQJtFAAAAAABwLAAAAAAAAIxuRQAAAAAAwKJFAAAAAACALAAA +AAAAAJB1RQAAAAAAwKJFAAAAAACQLAAAAAAAAJBlRQAAAAAAQNFFAAAAAACgLAAAAAAAAAVgRQAA +AAAAAKJFAAAAAADALAAAAAAAAFBuRQAAAAAAQKJFAAAAAADQLAAAAAAAAJN8RQAAAAAAQKJFAAAA +AADgLAAAAAAAAA5xRQAAAAAAAJ5FAAAAAADwLAAAAAAAANx7RQAAAAAAAJ5FAAAAAAAALQAAAAAA +AEZ3RQAAAAAAAJ9FAAAAAAAQLQAAAAAAAHx8RQAAAAAAAJ5FAAAAAAAgLQAAAAAAALhgRQAAAAAA +IOlFAAAAAAAwLQAAAAAAAOVjRQAAAAAAoN1FAAAAAACALQAAAAAAAHB1RQAAAAAAAKJFAAAAAACg +TQAAAAAAANJpRQAAAAAAAKJFAAAAAACoTQAAAAAAAFxuRQAAAAAAoNBFAAAAAACwTQAAAAAAAPhl +RQAAAAAAQJVFAAAAAADATQAAAAAAAHJrRQAAAAAAAKJFAAAAAADwTQAAAAAAAFxyRQAAAAAAAKJF +AAAAAAD4TQAAAAAAAJpxRQAAAAAAwKJFAAAAAAAATgAAAAAAAFNnRQAAAAAAQJtFAAAAAAAQTgAA +AAAAAIgBAAAAAAAAeAEAAAAAAAC02YfoBwgIGQAAAAAAAAAAbl9HAAAAAACgDAAAgCcAABhmRQAA +AAAAoAlGAAAAAAAzAAAAAAAAADMAAAAAAAAAGAYAAAAAAADYBAAAAAAAALtjRQAAAAAAgNJFAAAA +AAAAAAAAAAAAABpwRQAAAAAAwKJFAAAAAAAgAAAAAAAAACdwRQAAAAAAwKJFAAAAAAAwAAAAAAAA +AOhkRQAAAAAAQIVFAAAAAABAAAAAAAAAAOBkRQAAAAAAAIVFAAAAAABQAAAAAAAAABFgRQAAAAAA +QIpFAAAAAABgAAAAAAAAAJ9jRQAAAAAAwO1FAAAAAABwAAAAAAAAADhsRQAAAAAAwKJFAAAAAADg +AAAAAAAAAC1sRQAAAAAAwKJFAAAAAADwAAAAAAAAANxpRQAAAAAAwKJFAAAAAAAAAQAAAAAAAHxj +RQAAAAAAAKNFAAAAAAAQAQAAAAAAAORwRQAAAAAAAKJFAAAAAAAgAQAAAAAAABdsRQAAAAAAAKJF +AAAAAAAoAQAAAAAAAIhhRQAAAAAAAJ5FAAAAAAAwAQAAAAAAAOBrRQAAAAAAILFFAAAAAABAAQAA +AAAAAFlsRQAAAAAAAJ5FAAAAAABQAQAAAAAAALBuRQAAAAAAALNFAAAAAABgAQAAAAAAAFNnRQAA +AAAAQJtFAAAAAABiAQAAAAAAAL9vRQAAAAAAQJtFAAAAAABkAQAAAAAAANRyRQAAAAAAQJtFAAAA +AABmAQAAAAAAACB1RQAAAAAAQJtFAAAAAABoAQAAAAAAAGJxRQAAAAAAQJtFAAAAAABqAQAAAAAA +AFRtRQAAAAAAQJtFAAAAAABsAQAAAAAAAERuRQAAAAAAQJtFAAAAAABuAQAAAAAAACJ3RQAAAAAA +QJtFAAAAAABwAQAAAAAAAMVyRQAAAAAAgKJFAAAAAAByAQAAAAAAAPBtRQAAAAAAQJ5FAAAAAAB0 +AQAAAAAAAIB1RQAAAAAAQJtFAAAAAAB2AQAAAAAAABhqRQAAAAAAQJtFAAAAAAB4AQAAAAAAAFtw +RQAAAAAAgKJFAAAAAAB6AQAAAAAAAPJyRQAAAAAAAJ5FAAAAAACAAQAAAAAAAHBxRQAAAAAAAJ5F +AAAAAACQAQAAAAAAAIxxRQAAAAAAAJ5FAAAAAACgAQAAAAAAAA5qRQAAAAAAQKJFAAAAAACwAQAA +AAAAAJhuRQAAAAAAQLJFAAAAAADAAQAAAAAAAB1nRQAAAAAA4LFFAAAAAADQAQAAAAAAAANhRQAA +AAAAAKJFAAAAAADgAQAAAAAAAEpqRQAAAAAAQJpFAAAAAADwAQAAAAAAAKppRQAAAAAAwKJFAAAA +AAAgAgAAAAAAALRpRQAAAAAAwKJFAAAAAAAwAgAAAAAAALRjRQAAAAAAwKJFAAAAAABAAgAAAAAA +AI5hRQAAAAAAwKJFAAAAAABQAgAAAAAAAKxqRQAAAAAAgIFFAAAAAABgAgAAAAAAAL9nRQAAAAAA +wKJFAAAAAABwAgAAAAAAAIlnRQAAAAAAwKJFAAAAAACAAgAAAAAAABBoRQAAAAAAAI9FAAAAAACQ +AgAAAAAAAIRmRQAAAAAAgJpFAAAAAACgAgAAAAAAAEBlRQAAAAAAAKNFAAAAAADQAgAAAAAAANBj +RQAAAAAAAJBFAAAAAADgAgAAAAAAABRuRQAAAAAAAKJFAAAAAADwAgAAAAAAAHpyRQAAAAAAAJ5F +AAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAyAIAAAAAAAAYmHj/ +BwgIGQAAAAAAAAAAjF9HAAAAAACsDAAAQCoAABhmRQAAAAAA4A5GAAAAAABAAAAAAAAAAEAAAAAA +AAAAGAYAAAAAAAAQBgAAAAAAADBgRQAAAAAAgIdFAAAAAAAAAAAAAAAAAC9nRQAAAAAAwO1FAAAA +AAAQAAAAAAAAAAhlRQAAAAAAAKJFAAAAAACAAAAAAAAAALBlRQAAAAAAQKJFAAAAAACQAAAAAAAA +AOdmRQAAAAAAgIdFAAAAAACgAAAAAAAAAGBtRQAAAAAA4OVFAAAAAACwAAAAAAAAAKRnRQAAAAAA +oLhFAAAAAAAAAQAAAAAAABdhRQAAAAAAAKpFAAAAAAAQAQAAAAAAABRpRQAAAAAAAJtFAAAAAABw +AQAAAAAAAGphRQAAAAAAgIdFAAAAAACAAQAAAAAAALdqRQAAAAAAILFFAAAAAACQAQAAAAAAABdg +RQAAAAAAQLJFAAAAAACgAQAAAAAAAG5jRQAAAAAAQLJFAAAAAACwAQAAAAAAAPphRQAAAAAAQLJF +AAAAAADAAQAAAAAAADxgRQAAAAAAAJ5FAAAAAADQAQAAAAAAADtrRQAAAAAAwJ1FAAAAAADgAQAA +AAAAAPppRQAAAAAAwJ1FAAAAAADoAQAAAAAAAORtRQAAAAAAQKFFAAAAAADwAQAAAAAAAEtjRQAA +AAAAwJ1FAAAAAAAQAgAAAAAAAMZiRQAAAAAAwJ1FAAAAAAAYAgAAAAAAAKlrRQAAAAAAwJ1FAAAA +AAAgAgAAAAAAAMhpRQAAAAAAQJtFAAAAAAAoAgAAAAAAAHJmRQAAAAAAQJtFAAAAAAAqAgAAAAAA +AH5vRQAAAAAAQJtFAAAAAAAsAgAAAAAAAJ5rRQAAAAAAQJ5FAAAAAAAuAgAAAAAAADZjRQAAAAAA +QJtFAAAAAAAwAgAAAAAAAOJoRQAAAAAAAKJFAAAAAAA4AgAAAAAAAM5oRQAAAAAAAKdFAAAAAABA +AgAAAAAAAJxtRQAAAAAAQJtFAAAAAABQAgAAAAAAAENsRQAAAAAAgKJFAAAAAABSAgAAAAAAAB5p +RQAAAAAAQKJFAAAAAABgAgAAAAAAAOJhRQAAAAAAwJ1FAAAAAABwAgAAAAAAAD5yRQAAAAAAAKJF +AAAAAAB4AgAAAAAAAABtRQAAAAAAQIZFAAAAAACAAgAAAAAAALBoRQAAAAAAQJtFAAAAAACQAgAA +AAAAAAxiRQAAAAAAoMdFAAAAAACgAgAAAAAAAGBmRQAAAAAAQIpFAAAAAACwAgAAAAAAAOBrRQAA +AAAA4LFFAAAAAADAAgAAAAAAABRnRQAAAAAAILFFAAAAAADQAgAAAAAAAD1vRQAAAAAAwKdFAAAA +AADgAgAAAAAAABprRQAAAAAAAKJFAAAAAADgBAAAAAAAACVrRQAAAAAAAKJFAAAAAADoBAAAAAAA +AFFrRQAAAAAA4LFFAAAAAADwBAAAAAAAAHVwRQAAAAAAoK9FAAAAAAAABQAAAAAAADZqRQAAAAAA +AKNFAAAAAAAQBQAAAAAAAGhwRQAAAAAAgKJFAAAAAAAgBQAAAAAAAB9zRQAAAAAAQJ1FAAAAAAAw +BQAAAAAAABBzRQAAAAAAQJtFAAAAAABABQAAAAAAADRwRQAAAAAAAKJFAAAAAABIBQAAAAAAAOxo +RQAAAAAAQIpFAAAAAABQBQAAAAAAAFhlRQAAAAAAQNZFAAAAAABgBQAAAAAAAAtnRQAAAAAAwOxF +AAAAAACQBQAAAAAAAARrRQAAAAAAwKJFAAAAAADwBQAAAAAAAA9rRQAAAAAAwKJFAAAAAAAABgAA +AAAAAABpRQAAAAAAILFFAAAAAAAQBgAAAAAAANFnRQAAAAAAwOxFAAAAAAAgBgAAAAAAABBmRQAA +AAAAwKJFAAAAAACABgAAAAAAAAhmRQAAAAAAwKJFAAAAAACQBgAAAAAAANhtRQAAAAAAAKJFAAAA +AACgBgAAAAAAAAFzRQAAAAAAAKJFAAAAAACoBgAAAAAAAKZoRQAAAAAAALlFAAAAAACxBgAAAAAA +AMxgRQAAAAAAILpFAAAAAACxBgAAAAAAADhxRQAAAAAAQJ1FAAAAAACwBgAAAAAAADBrRQAAAAAA +oKNFAAAAAADABgAAAAAAACAoKSssLS4vOjw9Pj9bCV1fe30gKyBAIFAgWygiKSApCiwgLT46ID4g +IikiCgogXVtdCmkpcyB9CiAgRyAgTSAgUCAqKCAtICA8ICBtPSBuPSU6IC4uLj8/P05hTlBDPV06 +CmFkeGFlc2F2eGVuZGZpbmZtYWdjIGdwIG5pbG9ianBjPSA8PT0gYXQgIGZwPSBpcyAgbHI6IG9m +ICBwYz0gc3A6IHNwPSkgPSApIG09K0luZi1JbmY6IHA9R09HQ0xFQUYKCW09XSA9IF0gbj1hbGxn +YWxscGF2eDJiYXNlYm1pMWJtaTJjYXMxY2FzMmNhczNjYXM0Y2FzNWNhczZkZWFkZXJtc2lkbGVp +dGFicHJvZnJvb3RzYnJrc3NlMnNzZTN0cnVlIC4uLgogSF9UPSBIX2E9IEhfZz0gTUIsICBXX2E9 +IGFuZCAgY250PSBoX2E9IGhfZz0gaF90PSBtYXg9IG1zLCAgcHRyICBzaXo9IHRhYj0gdG9wPSB1 +X2E9IHVfZz0sIGZwOl0gPSAoZGVmZXJmYWxzZWZhdWx0Z0ZyZWVnY2luZ2dzY2FuaGNoYW5pbml0 +IG1oZWFwcGFuaWNzY2F2IHNjaGVkc2xlZXBzc2U0MXNzZTQyc3NzZTNzdWRvZ3N3ZWVwdHJhY2Ug +YWRkcj0gYWxsb2MgYmFzZSAgY29kZT0gY3R4dDogY3VyZz0gZnJlZSAgZ29pZCAgam9icz0gbGlz +dD0gbS0+cD0gbmV4dD0gcC0+bT0gcHJldj0gc3Bhbj0lIHV0aWwoLi4uKQosIGkgPSAsIG5vdCBT +Q0hFRCBlZmVuY2VvYmplY3Rwb3BjbnRzZWxlY3Rzd2VlcCBzeXNtb250aW1lcnMgKHNjYW4gIChz +Y2FuKSBNQiBpbiAgYWxsb2NzIGR5aW5nPSBsb2Nrcz0gbS0+ZzA9IG5tc3lzPSBzPW5pbAogem9t +YmllLCBnb2lkPSwgajAgPSBHT0RFQlVHSU8gd2FpdFNpZ25hbCBVTktOT1dOCXR5cGVzIAl2YWx1 +ZT1jcHVwcm9mY3MgICAgIGZvcmNlZ2NmcyAgICAgZ2N0cmFjZWdzICAgICBoZWFkID0gcGFuaWM6 +IHIxMCAgICByMTEgICAgcjEyICAgIHIxMyAgICByMTQgICAgcjE1ICAgIHI4ICAgICByOSAgICAg +cmF4ICAgIHJicCAgICByYnggICAgcmN4ICAgIHJkaSAgICByZHggICAgcmZsYWdzIHJpcCAgICBy +c2kgICAgcnNwICAgIHJ1bm5pbmdzaWduYWwgc3lzY2FsbHVua25vd253YWl0aW5nIGJ5dGVzLCAg +ZXR5cGVzICBnb2FszpQ9IGlzIG5vdCAgbWNvdW50PSBtaW51dGVzIG5hbGxvYz0gbmV3dmFsPSBu +ZnJlZWQ9IHBhY2tlZD0gcG9pbnRlciBzdGFjaz1bIHN0YXR1cyBHT0RFQlVHPVtzaWduYWwgCi0t +LS0tCgoJc3RhY2s9W2Nnb2NoZWNrZGVhZGxvY2twb2xsRGVzY3JlZmxlY3QucnVubmFibGVydW50 +aW1lLnJ3bXV0ZXhScndtdXRleFdzY2F2ZW5nZXRyYWNlQnVmdW5rbm93biggKGZvcmNlZCkgLT4g +bm9kZT0gYmxvY2tlZD0gZGVmZXJzYz0gaW4gdXNlKQogbG9ja2VkZz0gbG9ja2VkbT0gbS0+Y3Vy +Zz0gbWFya2VkICAgbXMgY3B1LCAgbm90IGluIFsgcy5saW1pdD0gcy5zdGF0ZT0gc2lnY29kZT0g +dGhyZWFkcz0gdV9hL3VfZz0gdW5tYXJrZWQgd2J1ZjEubj0gd2J1ZjIubj0odW5rbm93biksIG5l +d3ZhbD0sIG9sZHZhbD0sIHBsdWdpbjosIHNpemUgPSAsIHRhaWwgPSA6IHN0YXR1cz1MSU5VWF8y +LjZhdG9taWNvcjhiYWQgcHJ1bmVjaGFuIHNlbmRjb3B5c3RhY2tjdHh0ICE9IDBkZWJ1Z0xvY2to +Y2hhbkxlYWZpbml0dHJhY2VpbnRlcmZhY2VtU3BhbkRlYWRtU3BhbkZyZWVuZXdvc3Byb2NwYW5p +Y3dhaXRwY2xtdWxxZHFwcmVlbXB0ZWRyZWNvdmVyOiBzY2F2dHJhY2VzaWduYWwgMzJzaWduYWwg +MzNzaWduYWwgMzRzaWduYWwgMzVzaWduYWwgMzZzaWduYWwgMzdzaWduYWwgMzhzaWduYWwgMzlz +aWduYWwgNDBzaWduYWwgNDFzaWduYWwgNDJzaWduYWwgNDNzaWduYWwgNDRzaWduYWwgNDVzaWdu +YWwgNDZzaWduYWwgNDdzaWduYWwgNDhzaWduYWwgNDlzaWduYWwgNTBzaWduYWwgNTFzaWduYWwg +NTJzaWduYWwgNTNzaWduYWwgNTRzaWduYWwgNTVzaWduYWwgNTZzaWduYWwgNTdzaWduYWwgNThz +aWduYWwgNTlzaWduYWwgNjBzaWduYWwgNjFzaWduYWwgNjJzaWduYWwgNjNzaWduYWwgNjRzdGFj +a3Bvb2x0cmFjZWJhY2t3YnVmU3BhbnN9IHN0YWNrPVsgTUIgZ29hbCwgIGFjdHVhbM6UPSBmbHVz +aEdlbiAgZ2ZyZWVjbnQ9IHBhZ2VzIGF0ICByZXR1cm5lZCAgcnVucXNpemU9IHJ1bnF1ZXVlPSBz +LmJhc2UoKT0gc3Bpbm5pbmc9IHN0b3B3YWl0PSBzd2VlcGdlbiAgc3dlZXBnZW49IHRhcmdldHBj +PSB0aHJvd2luZz0gdW50aWwgcGM9LCBib3VuZCA9ICwgbGltaXQgPSBCYWQgdmFyaW50R0MgZm9y +Y2VkCkdPTUFYUFJPQ1NhdG9taWNhbmQ4ZGVidWcgY2FsbGZsb2F0MzJuYW5mbG9hdDY0bmFuZ29y +b3V0aW5lIGludmFsaWRwdHJtU3BhbkluVXNlbm90aWZ5TGlzdHJ1bnRpbWU6IGdzLnN0YXRlID0g +c2NoZWR0cmFjZXNlbWFjcXVpcmVzdGFja0xhcmdldGlja3MubG9ja3RyYWNlZnJlZSh0cmFjZWdj +KCkKdW5rbm93biBwYyAgb2Ygc2l6ZSAgICh0YXJnZXRwYz0gS2lCIHdvcmssICBmcmVlaW5kZXg9 +IGdjd2FpdGluZz0gaWRsZXByb2NzPSBpbiBzdGF0dXMgIG1hbGxvY2luZz0gbXMgY2xvY2ssICBu +QlNTUm9vdHM9IHAtPnN0YXR1cz0gcy5uZWxlbXM9ICBzY2hlZHRpY2s9IHNwYW4ubGlzdD0gdGlt +ZXJzbGVuPSwgZWxlbXNpemU9LCBucGFnZXMgPSA6IGZyYW1lLnNwPUdPVFJBQ0VCQUNLYXNzaXN0 +UXVldWViYWQgbSB2YWx1ZWJhZCB0aW1lZGl2Y2dvY2FsbCBuaWxjbG9iYmVyZnJlZWNyZWF0ZWQg +YnkgZmxvYXQzMm5hbjJmbG9hdDY0bmFuMWZsb2F0NjRuYW4yZmxvYXQ2NG5hbjNnY2NoZWNrbWFy +a21TcGFuTWFudWFsbmV0cG9sbEluaXRyZWZsZWN0T2Zmc3J1bnRpbWU6IFAgcnVudGltZTogcCBz +Y2hlZGRldGFpbHRyYWNlYWxsb2ModW5yZWFjaGFibGUgS2lCIHRvdGFsLCAgW3JlY292ZXJlZF0g +YWxsb2NDb3VudCAgZm91bmQgYXQgKiggZ2NzY2FuZG9uZSAgaGVhcE1hcmtlZD0gbS0+Z3NpZ25h +bD0gbWluVHJpZ2dlcj0gbkRhdGFSb290cz0gblNwYW5Sb290cz0gcGFnZXMvYnl0ZQogcHJlZW1w +dG9mZj0gcy5lbGVtc2l6ZT0gcy5zd2VlcGdlbj0gc3Bhbi5saW1pdD0gc3Bhbi5zdGF0ZT0gc3lz +bW9ud2FpdD0gd2J1ZjE9PG5pbD4gd2J1ZjI9PG5pbD4pIHAtPnN0YXR1cz0tYnl0ZSBsaW1pdAph +YmkgbWlzbWF0Y2hiYWQgZmx1c2hHZW5iYWQgZyBzdGF0dXNiYWQgcmVjb3ZlcnljYW4ndCBoYXBw +ZW5jYXM2NCBmYWlsZWRjaGFuIHJlY2VpdmVkdW1waW5nIGhlYXBlbmQgdHJhY2VnYwplbnRlcnN5 +c2NhbGxnY0JpdHNBcmVuYXNnY3BhY2VydHJhY2VsZnN0YWNrLnB1c2htYWR2ZG9udG5lZWRtaGVh +cFNwZWNpYWxtc3BhblNwZWNpYWxyYWNlRmluaUxvY2tyZWxlYXNlcDogbT1ydW50aW1lOiBncD1y +dW50aW1lOiBzcD1zcGFuU2V0U3BpbmVzd2VlcFdhaXRlcnN0cmFjZVN0cmluZ3N3aXJlcDogcC0+ +bT13b3JrZXIgbW9kZSAgIT0gc3dlZXBnZW4gIE1CKSB3b3JrZXJzPSBjYWxsZWQgZnJvbSAgZmFp +bGVkIHdpdGggIGZsdXNoZWRXb3JrICBpZGxldGhyZWFkcz0gaXMgbmlsLCBub3QgIG5TdGFja1Jv +b3RzPSBzLnNwYW5jbGFzcz0gc3Bhbi5iYXNlKCk9IHN5c2NhbGx0aWNrPSB3b3JrLm5wcm9jPSAg +d29yay5ud2FpdD0gLCBncC0+c3RhdHVzPS1ieXRlIGJsb2NrIChHQyBzd2VlcCB3YWl0U0lHS0lM +TDoga2lsbFNJR1FVSVQ6IHF1aXRiYWQgZmx1c2hHZW4gYmFkIG1hcCBzdGF0ZWRlYnVnQ2FsbDIw +NDhmYXRhbCBlcnJvcjogbG9hZDY0IGZhaWxlZG1pbiB0b28gbGFyZ2VuaWwgc3RhY2tiYXNlb3V0 +IG9mIG1lbW9yeXJ1bnRpbWU6IHNlcT1ydW50aW1lOiB2YWw9dHJhY2VTdGFja1RhYnRyaWdnZXJS +YXRpbz12YWx1ZSBtZXRob2QgeGFkZDY0IGZhaWxlZHhjaGc2NCBmYWlsZWR9CglzY2hlZD17cGM6 +IGJ1dCBwcm9nU2l6ZSAgbm1pZGxlbG9ja2VkPSBvdXQgb2YgcmFuZ2UgIHVudHlwZWQgYXJncyAt +dGhyZWFkIGxpbWl0CkdDIGFzc2lzdCB3YWl0R0Mgd29ya2VyIGluaXRNQjsgYWxsb2NhdGVkIFNJ +R0FCUlQ6IGFib3J0YWxsb2NmcmVldHJhY2ViYWQgYWxsb2NDb3VudGJhZCBzcGFuIHN0YXRlYmFk +IHN0YWNrIHNpemVmaW5hbGl6ZXIgd2FpdGdjc3RvcHRoZXdvcmxkbm8gbW9kdWxlIGRhdGFwb2xs +Q2FjaGUubG9ja3J1bnRpbWU6IGZ1bGw9cy5hbGxvY0NvdW50PSBzZW1hUm9vdCBxdWV1ZXN0YWNr +IG92ZXJmbG93c3RvcG0gc3Bpbm5pbmdzdG9yZTY0IGZhaWxlZHN5bmMuQ29uZC5XYWl0d29yay5m +dWxsICE9IDAgIHdpdGggR0MgcHJvZwpdCgltb3JlYnVmPXtwYzphc3luY3ByZWVtcHRvZmZmb3Jj +ZSBnYyAoaWRsZSltYWxsb2MgZGVhZGxvY2ttaXNhbGlnbmVkIG1hc2ttaXNzaW5nIG1jYWNoZT9t +czogZ29tYXhwcm9jcz1wcmVlbXB0IFNQV1JJVEVyZWNvdmVyeSBmYWlsZWRydW50aW1lIGVycm9y +OiBydW50aW1lOiBmcmFtZSBydW50aW1lOiBtYXggPSBydW50aW1lOiBtaW4gPSBydW50aW1lcjog +YmFkIHBzY2FuIG1pc3NlZCBhIGdzdGFydG06IG0gaGFzIHBzdG9wbSBob2xkaW5nIHAgYWxyZWFk +eTsgZXJybm89IG1oZWFwLnN3ZWVwZ2VuPSBub3QgaW4gcmFuZ2VzOgogdW50eXBlZCBsb2NhbHMg +MDEyMzQ1Njc4OWFiY2RlZkdDIHNjYXZlbmdlIHdhaXRHQyB3b3JrZXIgKGlkbGUpR09ERUJVRzog +dmFsdWUgIlNJR05PTkU6IG5vIHRyYXAKcnVudGltZSBzdGFjazoKYmFkIGcgdHJhbnNpdGlvbmJh +ZCBzcGVjaWFsIGtpbmRiYWQgc3VtbWFyeSBkYXRhYmFkIHN5bWJvbCB0YWJsZWNhc3RvZ3NjYW5z +dGF0dXNnYzogdW5zd2VwdCBzcGFuZ2NzaHJpbmtzdGFja29mZmludGVnZXIgb3ZlcmZsb3dpbnZh +bGlkIGcgc3RhdHVzaW52YWxpZCBzcGRlbHRhIG1TcGFuTGlzdC5pbnNlcnRtU3Bhbkxpc3QucmVt +b3ZlbWlzc2luZyBzdGFja21hcG5ld21IYW5kb2ZmLmxvY2tub24tR28gZnVuY3Rpb24KcGFjZXI6 +IEhfbV9wcmV2PXJlZmxlY3QgbWlzbWF0Y2hydW50aW1lOiAgZzogIGc9cnVudGltZTogYWRkciA9 +IHJ1bnRpbWU6IGJhc2UgPSBydW50aW1lOiBncDogZ3A9cnVudGltZTogaGVhZCA9IHJ1bnRpbWU6 +IG5lbGVtcz1zY2hlZHVsZTogaW4gY2dvc2lnYWN0aW9uIGZhaWxlZHdvcmtidWYgaXMgZW1wdHkg +aW5pdGlhbEhlYXBMaXZlPSBzcGlubmluZ3RocmVhZHM9LCBwLnNlYXJjaEFkZHIgPSA6IG1pc3Np +bmcgbWV0aG9kIEdDIGFzc2lzdCBtYXJraW5nU0lHQlVTOiBidXMgZXJyb3JTSUdDT05UOiBjb250 +aW51ZVNJR0lOVDogaW50ZXJydXB0YmFkIFRpbnlTaXplQ2xhc3NkZWJ1Z1B0cm1hc2subG9ja2Vu +dGVyc3lzY2FsbGJsb2NrZnV0ZXh3YWtldXAgYWRkcj1nIGFscmVhZHkgc2Nhbm5lZGdsb2JhbEFs +bG9jLm11dGV4bG9ja2VkIG0wIHdva2UgdXBtYXJrIC0gYmFkIHN0YXR1c21hcmtCaXRzIG92ZXJm +bG93bm90ZXRzbGVlcGcgb24gZzBydW50aW1lL2ludGVybmFsL3J1bnRpbWU6IGxldmVsID0gcnVu +dGltZTogbmFtZU9mZiBydW50aW1lOiBwb2ludGVyIHJ1bnRpbWU6IHN1bW1hcnlbcnVudGltZTog +dGV4dE9mZiBydW50aW1lOiB0eXBlT2ZmIHNjYW5vYmplY3QgbiA9PSAwc2VsZWN0IChubyBjYXNl +cylzdGFjazogZnJhbWU9e3NwOnN3ZXB0IGNhY2hlZCBzcGFudGhyZWFkIGV4aGF1c3Rpb250cmln +Z2VyIHVuZGVyZmxvd3Vua25vd24gY2FsbGVyIHBjd2FpdCBmb3IgR0MgY3ljbGUgIGJ1dCBtZW1v +cnkgc2l6ZSAgaW4gYXN5bmMgcHJlZW1wdAogdG8gbm9uLUdvIG1lbW9yeSAsIGxvY2tlZCB0byB0 +aHJlYWRiYWQgbGZub2RlIGFkZHJlc3NiYWQgbWFudWFsRnJlZUxpc3RmYWtldGltZVN0YXRlLmxv +Y2tmb3JFYWNoUDogbm90IGRvbmVnYXJiYWdlIGNvbGxlY3Rpb25pbmRleCBvdXQgb2YgcmFuZ2Vp +bnN0cnVjdGlvbiBieXRlczpydW50aW1lOiBoZWFwR29hbD1ydW50aW1lOiBucGFnZXMgPSBydW50 +aW1lOiByYW5nZSA9IHtzeXN0ZW0gcGFnZSBzaXplICh0cmFjZWJhY2thbmNlc3RvcnMgY2FsbGVk +IHVzaW5nIG5pbCAqLCAgZy0+YXRvbWljc3RhdHVzPSwgZ3AtPmF0b21pY3N0YXR1cz1HQyB3b3Jr +IG5vdCBmbHVzaGVkU0lHVFJBUDogdHJhY2UgdHJhcF9fdmRzb19nZXR0aW1lb2ZkYXlfY2dvX3Nl +dGVudiBtaXNzaW5nYWRqdXN0dGltZXJzOiBiYWQgcGJhZCBydW50aW1lwrdtc3RhcnRiYWQgc2Vx +dWVuY2UgbnVtYmVyY2dvY2FsbCB1bmF2YWlsYWJsZWRvZGVsdGltZXI6IHdyb25nIFBtIG5vdCBm +b3VuZCBpbiBhbGxtbWFya2luZyBmcmVlIG9iamVjdG1hcmtyb290OiBiYWQgaW5kZXhtaXNzaW5n +IGRlZmVycmV0dXJubXNwYW4uc3dlZXA6IHN0YXRlPW5vdGVzbGVlcCBub3Qgb24gZzBud2FpdCA+ +IHdvcmsubnByb2NzcGFuaWMgZHVyaW5nIG1hbGxvY3BhbmljIGR1cmluZyBwYW5pYwpwYW5pYyBo +b2xkaW5nIGxvY2tzcGFuaWN3cmFwOiBubyAoIGluIHBhbmljd3JhcDogbm8gKSBpbiBydW50aW1l +OiBwY2RhdGEgaXMgcnVudGltZTogcHJlZW1wdCBnMHNlbWFSb290IHJvdGF0ZUxlZnRzdG9wbSBo +b2xkaW5nIGxvY2tzc3lzTWVtU3RhdCBvdmVyZmxvd3VuYWxpZ25lZCBzeXNVbnVzZWR1bmV4cGVj +dGVkIGcgc3RhdHVzdW5rbm93biB3YWl0IHJlYXNvbiBtYXJrcm9vdCBqb2JzIGRvbmUKIHRvIHVu +YWxsb2NhdGVkIHNwYW5TSUdBTFJNOiBhbGFybSBjbG9ja1NJR1RFUk06IHRlcm1pbmF0aW9uX192 +ZHNvX2Nsb2NrX2dldHRpbWViYWQgZGVmZXIgc2l6ZSBjbGFzc2JhZCBzeXN0ZW0gcGFnZSBzaXpl +YmFkIHVzZSBvZiBidWNrZXQuYnBiYWQgdXNlIG9mIGJ1Y2tldC5tcGNoYW4gc2VuZCAobmlsIGNo +YW4pY2xvc2Ugb2YgbmlsIGNoYW5uZWxkb2RlbHRpbWVyMDogd3JvbmcgUGZsb2F0aW5nIHBvaW50 +IGVycm9yZm9yY2VnYzogcGhhc2UgZXJyb3JnbyBvZiBuaWwgZnVuYyB2YWx1ZWdvcGFyazogYmFk +IGcgc3RhdHVzaW5jb25zaXN0ZW50IGxvY2tlZG1tYWxsb2MgZHVyaW5nIHNpZ25hbG5vdGV0c2xl +ZXAgbm90IG9uIGcwcCBtY2FjaGUgbm90IGZsdXNoZWRwYWNlcjogYXNzaXN0IHJhdGlvPXByZWVt +cHQgb2ZmIHJlYXNvbjogcmVmbGVjdC5tYWtlRnVuY1N0dWJydW50aW1lOiBwaXBlIGZhaWxlZHJ1 +bnRpbWU6IHVua25vd24gcGMgc2VtYVJvb3Qgcm90YXRlUmlnaHR0cmFjZTogb3V0IG9mIG1lbW9y +eXdpcmVwOiBhbHJlYWR5IGluIGdvd29ya2J1ZiBpcyBub3QgZW1wdHl3cml0ZSBvZiBHbyBwb2lu +dGVyICBwcmV2aW91cyBhbGxvY0NvdW50PSwgbGV2ZWxCaXRzW2xldmVsXSA9IF9jZ29fdW5zZXRl +bnYgbWlzc2luZ2FzeW5jIHN0YWNrIHRvbyBsYXJnZWNoZWNrZGVhZDogcnVubmFibGUgZ2NvbmN1 +cnJlbnQgbWFwIHdyaXRlc2ZpbmRydW5uYWJsZTogd3JvbmcgcG5lZ2F0aXZlIHNoaWZ0IGFtb3Vu +dHBhbmljIG9uIHN5c3RlbSBzdGFja3ByZWVtcHQgYXQgdW5rbm93biBwY3JlbGVhc2VwOiBpbnZh +bGlkIGFyZ3J1bnRpbWU6IGNvbmZ1c2VkIGJ5IHJ1bnRpbWU6IG5ld3N0YWNrIGF0IHJ1bnRpbWU6 +IG5ld3N0YWNrIHNwPXJ1bnRpbWU6IHNlYXJjaElkeCA9IHJ1bnRpbWU6IHdvcmsubndhaXQ9IHN0 +YXJ0bG9ja2VkbTogbSBoYXMgcHN0YXJ0bTogbSBpcyBzcGlubmluZ3RpbWVyIGRhdGEgY29ycnVw +dGlvbiByZWNlaXZlZCBkdXJpbmcgZm9yawpTSUdTVEtGTFQ6IHN0YWNrIGZhdWx0U0lHVFNUUDog +a2V5Ym9hcmQgc3RvcGFzc2VtYmx5IGNoZWNrcyBmYWlsZWRiYWQgZy0+c3RhdHVzIGluIHJlYWR5 +YmFkIHN3ZWVwZ2VuIGluIHJlZmlsbGNhbGwgbm90IGF0IHNhZmUgcG9pbnRkdXBsaWNhdGVkIGRl +ZmVyIGVudHJ5ZnJlZUluZGV4IGlzIG5vdCB2YWxpZGdldGVudiBiZWZvcmUgZW52IGluaXRoZWFk +VGFpbEluZGV4IG92ZXJmbG93aW50ZWdlciBkaXZpZGUgYnkgemVyb2ludGVyZmFjZSBjb252ZXJz +aW9uOiBtaW5wYyBvciBtYXhwYyBpbnZhbGlkbm9uLUdvIGZ1bmN0aW9uIGF0IHBjPW9sZG92ZXJm +bG93IGlzIG5vdCBuaWxydW50aW1lLm1haW4gbm90IG9uIG0wcnVudGltZTogb3V0IG9mIG1lbW9y +eXJ1bnRpbWU6IHdvcmsubndhaXQgPSBydW50aW1lOnNjYW5zdGFjazogZ3A9cy5mcmVlaW5kZXgg +PiBzLm5lbGVtc3NjYW5zdGFjayAtIGJhZCBzdGF0dXNzZW5kIG9uIGNsb3NlZCBjaGFubmVsc3Bh +biBoYXMgbm8gZnJlZSBzcGFjZXN0YWNrIG5vdCBhIHBvd2VyIG9mIDJ0aW1lciBnb3JvdXRpbmUg +KGlkbGUpdHJhY2UgcmVhZGVyIChibG9ja2VkKXRyYWNlOiBhbGxvYyB0b28gbGFyZ2V3aXJlcDog +aW52YWxpZCBwIHN0YXRlIGdjQ29udHJvbGxlci5oZWFwTGl2ZT0pIG11c3QgYmUgYSBwb3dlciBv +ZiAyCk1CIGR1cmluZyBzd2VlcDsgc3dlcHQgU0lHSU86IGkvbyBub3cgcG9zc2libGVTSUdTWVM6 +IGJhZCBzeXN0ZW0gY2FsbCIsIG1pc3NpbmcgQ1BVIHN1cHBvcnQKY2hhbiByZWNlaXZlIChuaWwg +Y2hhbiljbG9zZSBvZiBjbG9zZWQgY2hhbm5lbGZhdGFsOiBtb3Jlc3RhY2sgb24gZzAKZ2FyYmFn +ZSBjb2xsZWN0aW9uIHNjYW5nY0RyYWluIHBoYXNlIGluY29ycmVjdGdvIHdpdGggbm9uLWVtcHR5 +IGZyYW1laW5kZXggb3V0IG9mIHJhbmdlIFsleF1pbnZhbGlkIG0tPmxvY2tlZEludCA9IGxlZnQg +b3ZlciBtYXJrcm9vdCBqb2JzbWFrZWNoYW46IGJhZCBhbGlnbm1lbnRtaXN1c2Ugb2YgcHJvZkJ1 +Zi53cml0ZW5hbm90aW1lIHJldHVybmluZyB6ZXJvcGFuaWMgZHVyaW5nIHByZWVtcHRvZmZwcm9j +cmVzaXplOiBpbnZhbGlkIGFyZ3JlZmxlY3QubWV0aG9kVmFsdWVDYWxscnVudGltZTogaW50ZXJu +YWwgZXJyb3JydW50aW1lOiBpbnZhbGlkIHR5cGUgIHJ1bnRpbWU6IG5ldHBvbGwgZmFpbGVkcnVu +dGltZTogcy5hbGxvY0NvdW50PSBzLmFsbG9jQ291bnQgPiBzLm5lbGVtc3NjaGVkdWxlOiBob2xk +aW5nIGxvY2tzc2hyaW5rc3RhY2sgYXQgYmFkIHRpbWVzcGFuIGhhcyBubyBmcmVlIHN0YWNrc3N0 +YWNrIGdyb3d0aCBhZnRlciBmb3Jrc3lzdGVtIGh1Z2UgcGFnZSBzaXplICh1bmV4cGVjdGVkIHNp +Z25hbCB2YWx1ZXVubG9jayBvZiB1bmxvY2tlZCBsb2Nrd29yay5ud2FpdCA+IHdvcmsubnByb2Mi +LCByZXF1aXJlZCBDUFUgZmVhdHVyZQpiYWQgZGVmZXIgZW50cnkgaW4gcGFuaWNiYWQgZGVmZXIg +c2l6ZSBjbGFzczogaT1ieXBhc3NlZCByZWNvdmVyeSBmYWlsZWRjYW4ndCBzY2FuIG91ciBvd24g +c3RhY2tkb3VibGUgdHJhY2VHQ1N3ZWVwU3RhcnRnY0RyYWluTiBwaGFzZSBpbmNvcnJlY3Rpbml0 +U3BhbjogdW5hbGlnbmVkIGJhc2VwYWdlQWxsb2M6IG91dCBvZiBtZW1vcnlxdWV1ZWZpbmFsaXpl +ciBkdXJpbmcgR0NyYW5nZSBwYXJ0aWFsbHkgb3ZlcmxhcHNydW5xc3RlYWw6IHJ1bnEgb3ZlcmZs +b3dydW50aW1lOiBlcG9sbGN0bCBmYWlsZWRydW50aW1lOiBmb3VuZCBvYmogYXQgKihydW50aW1l +OiBwLnNlYXJjaEFkZHIgPSBzcGFuIGhhcyBubyBmcmVlIG9iamVjdHNzdGFjayB0cmFjZSB1bmF2 +YWlsYWJsZQogdG8gdW51c2VkIHJlZ2lvbiBvZiBzcGFuR09ERUJVRzogY2FuIG5vdCBlbmFibGUg +Il9jZ29fdGhyZWFkX3N0YXJ0IG1pc3NpbmdhbGxnYWRkOiBiYWQgc3RhdHVzIEdpZGxlYXJlbmEg +YWxyZWFkeSBpbml0aWFsaXplZGJhZCBzdGF0dXMgaW4gc2hyaW5rc3RhY2tiYWQgc3lzdGVtIGh1 +Z2UgcGFnZSBzaXplY2hhbnNlbmQ6IHNwdXJpb3VzIHdha2V1cGNoZWNrZGVhZDogbm8gbSBmb3Ig +dGltZXJtaXNzaW5nIHN0YWNrIGluIG5ld3N0YWNrbWlzc2luZyB0cmFjZUdDU3dlZXBTdGFydHJl +bGVhc2VwOiBpbnZhbGlkIHAgc3RhdGVyZW1haW5pbmcgcG9pbnRlciBidWZmZXJzcnVudGltZTog +ZXBvbGx3YWl0IG9uIGZkIHJ1bnRpbWU6IG1pc2FsaWduZWQgZnVuYyBydW50aW1lOiBwcm9ncmFt +IGV4Y2VlZHMgcnVudGltZcK3bG9jazogbG9jayBjb3VudHNsaWNlIGJvdW5kcyBvdXQgb2YgcmFu +Z2VzdGFydG06IHAgaGFzIHJ1bm5hYmxlIGdzc3RvcGxvY2tlZG06IG5vdCBydW5uYWJsZXVuZXhw +ZWN0ZWQgZmF1bHQgYWRkcmVzcyBHT0RFQlVHOiBjYW4gbm90IGRpc2FibGUgIlNJR1NUT1A6IHN0 +b3AsIHVuYmxvY2thYmxlY2FsbCBmcm9tIHVua25vd24gZnVuY3Rpb25jb3JydXB0ZWQgc2VtYXBo +b3JlIHRpY2tldGRlZmVyIHdpdGggbm9uLWVtcHR5IGZyYW1lZW50ZXJzeXNjYWxsIGluY29uc2lz +dGVudCBmb3JFYWNoUDogUCBkaWQgbm90IHJ1biBmbmZyZWVkZWZlciB3aXRoIGQuZm4gIT0gbmls +aW5pdFNwYW46IHVuYWxpZ25lZCBsZW5ndGhub3Rld2FrZXVwIC0gZG91YmxlIHdha2V1cG91dCBv +ZiBtZW1vcnkgKHN0YWNrYWxsb2MpcGVyc2lzdGVudGFsbG9jOiBzaXplID09IDBydW50aW1lOiBi +YWQgc3BhbiBzLnN0YXRlPXJ1bnRpbWU6IHBpcGUgZmFpbGVkIHdpdGggc2hyaW5raW5nIHN0YWNr +IGluIGxpYmNhbGxzdGFydGxvY2tlZG06IGxvY2tlZCB0byBtZUcgd2FpdGluZyBsaXN0IGlzIGNv +cnJ1cHRlZFNJR0lMTDogaWxsZWdhbCBpbnN0cnVjdGlvblNJR1hDUFU6IGNwdSBsaW1pdCBleGNl +ZWRlZGFkZHJlc3Mgbm90IGEgc3RhY2sgYWRkcmVzc2djc3RvcG06IG5vdCB3YWl0aW5nIGZvciBn +Y2dyb3dzbGljZTogY2FwIG91dCBvZiByYW5nZWludGVybmFsIGxvY2tPU1RocmVhZCBlcnJvcmlu +dmFsaWQgcHJvZmlsZSBidWNrZXQgdHlwZW1ha2VjaGFuOiBzaXplIG91dCBvZiByYW5nZW1ha2Vz +bGljZTogY2FwIG91dCBvZiByYW5nZW1ha2VzbGljZTogbGVuIG91dCBvZiByYW5nZW1zcGFuLnN3 +ZWVwOiBiYWQgc3BhbiBzdGF0ZXByb2dUb1BvaW50ZXJNYXNrOiBvdmVyZmxvd3J1bmxvY2sgb2Yg +dW5sb2NrZWQgcndtdXRleHJ1bnRpbWU6IGFzeW5jUHJlZW1wdFN0YWNrPXJ1bnRpbWU6IGNoZWNr +ZGVhZDogZmluZCBnIHJ1bnRpbWU6IGNoZWNrZGVhZDogbm1pZGxlPXJ1bnRpbWU6IG5ldHBvbGxp +bml0IGZhaWxlZHJ1bnRpbWU6IHRocmVhZCBJRCBvdmVyZmxvd3J1bnRpbWXCt3VubG9jazogbG9j +ayBjb3VudHNpZ25hbCByZWNlaXZlZCBkdXJpbmcgZm9ya3NpZ3NlbmQ6IGluY29uc2lzdGVudCBz +dGF0ZXN0YWNrIHNpemUgbm90IGEgcG93ZXIgb2YgMnN0YXJ0bTogbmVnYXRpdmUgbm1zcGlubmlu +Z3N0b3BUaGVXb3JsZDogaG9sZGluZyBsb2Nrc3RpbWVyIHdoZW4gbXVzdCBiZSBwb3NpdGl2ZXdv +cmsubndhaXQgd2FzID4gd29yay5ucHJvYyBhcmdzIHN0YWNrIG1hcCBlbnRyaWVzIGZvciBGaXhl +ZFN0YWNrIGlzIG5vdCBwb3dlci1vZi0yU0lHSFVQOiB0ZXJtaW5hbCBsaW5lIGhhbmd1cFNJR1dJ +TkNIOiB3aW5kb3cgc2l6ZSBjaGFuZ2Vbb3JpZ2luYXRpbmcgZnJvbSBnb3JvdXRpbmUgY29tcGFy +aW5nIHVuY29tcGFyYWJsZSB0eXBlIGZhdGFsOiBtb3Jlc3RhY2sgb24gZ3NpZ25hbApmaW5kcnVu +bmFibGU6IG5ldHBvbGwgd2l0aCBwZm91bmQgcG9pbnRlciB0byBmcmVlIG9iamVjdGdjQmdNYXJr +V29ya2VyOiBtb2RlIG5vdCBzZXRnY3N0b3BtOiBuZWdhdGl2ZSBubXNwaW5uaW5naW52YWxpZCBy +dW50aW1lIHN5bWJvbCB0YWJsZW1oZWFwLmZyZWVTcGFuTG9ja2VkIC0gc3BhbiBtaXNzaW5nIHN0 +YWNrIGluIHNocmlua3N0YWNrbXNwYW4uc3dlZXA6IG0gaXMgbm90IGxvY2tlZG5ld3Byb2MxOiBu +ZXcgZyBpcyBub3QgR2RlYWRuZXdwcm9jMTogbmV3ZyBtaXNzaW5nIHN0YWNrbm90ZXdha2V1cCAt +IGRvdWJsZSB3YWtldXAgKHJlZ2lvbiBleGNlZWRzIHVpbnRwdHIgcmFuZ2VydW50aW1lOiBiYWQg +bGZub2RlIGFkZHJlc3MgcnVudGltZTogY2FzZ3N0YXR1czogb2xkdmFsPXJ1bnRpbWU6IG5vIG1v +ZHVsZSBkYXRhIGZvciBzYXZlIG9uIHN5c3RlbSBnIG5vdCBhbGxvd2VkdW5yZXNlcnZpbmcgdW5h +bGlnbmVkIHJlZ2lvblNJR1BJUEU6IHdyaXRlIHRvIGJyb2tlbiBwaXBlU0lHUFdSOiBwb3dlciBm +YWlsdXJlIHJlc3RhcnRhZGRzcGVjaWFsIG9uIGludmFsaWQgcG9pbnRlcmV4ZWN1dGluZyBvbiBH +byBydW50aW1lIHN0YWNrZ2MgZG9uZSBidXQgZ2NwaGFzZSAhPSBfR0NvZmZnZnB1dDogYmFkIHN0 +YXR1cyAobm90IEdkZWFkKWludmFsaWQgbGVuZ3RoIG9mIHRyYWNlIGV2ZW50cnVudGltZTogaW1w +b3NzaWJsZSB0eXBlIGtpbmRydW50aW1lOiBsZXZlbFNoaWZ0W2xldmVsXSA9IHJ1bnRpbWU6IG1h +cmtpbmcgZnJlZSBvYmplY3QgcnVudGltZTogbW1hcDogYWNjZXNzIGRlbmllZApydW50aW1lOiBw +LmdjTWFya1dvcmtlck1vZGU9IHJ1bnRpbWU6IHNwbGl0IHN0YWNrIG92ZXJmbG93cnVudGltZTog +c3Vkb2cgd2l0aCBub24tbmlsIGNydW50aW1lOiBzdW1tYXJ5IG1heCBwYWdlcyA9IHJ1bnRpbWU6 +IHVua25vd24gcGMgaW4gZGVmZXIgc2VtYWNxdWlyZSBub3Qgb24gdGhlIEcgc3RhY2tzdHJpbmcg +Y29uY2F0ZW5hdGlvbiB0b28gbG9uZyAodHlwZXMgZnJvbSBkaWZmZXJlbnQgc2NvcGVzKSBpbiBw +cmVwYXJlRm9yU3dlZXA7IHN3ZWVwZ2VuICBsb2NhbHMgc3RhY2sgbWFwIGVudHJpZXMgZm9yIEdP +REVCVUc6IHVua25vd24gY3B1IGZlYXR1cmUgIlNJR1BST0Y6IHByb2ZpbGluZyBhbGFybSBjbG9j +a1NJR1VTUjE6IHVzZXItZGVmaW5lZCBzaWduYWwgMVNJR1VTUjI6IHVzZXItZGVmaW5lZCBzaWdu +YWwgMlNJR1ZUQUxSTTogdmlydHVhbCBhbGFybSBjbG9ja2FiaSBtaXNtYXRjaCBkZXRlY3RlZCBi +ZXR3ZWVuIGFzc2lnbm1lbnQgdG8gZW50cnkgaW4gbmlsIG1hcGNoZWNrZGVhZDogaW5jb25zaXN0 +ZW50IGNvdW50c2ZhaWxlZCB0byBnZXQgc3lzdGVtIHBhZ2Ugc2l6ZWZyZWVkZWZlciB3aXRoIGQu +X3BhbmljICE9IG5pbGludmFsaWQgZnVuY3Rpb24gc3ltYm9sIHRhYmxlCmludmFsaWQgcG9pbnRl +ciBmb3VuZCBvbiBzdGFja3J1bnFwdXRzbG93OiBxdWV1ZSBpcyBub3QgZnVsbHJ1bnRpbWU6IGJh +ZCBwb2ludGVyIGluIGZyYW1lIHJ1bnRpbWU6IGVwb2xsY3RsIGZhaWxlZCB3aXRoIHJ1bnRpbWU6 +IGZvdW5kIGluIG9iamVjdCBhdCAqKHJ1bnRpbWU6IGltcG9zc2libGUgdHlwZSBraW5kICkgbm90 +IGluIHVzYWJsZSBhZGRyZXNzIHNwYWNlOiAuLi5hZGRpdGlvbmFsIGZyYW1lcyBlbGlkZWQuLi4K +U0lHU0VHVjogc2VnbWVudGF0aW9uIHZpb2xhdGlvbmJhZCB3cml0ZSBiYXJyaWVyIGJ1ZmZlciBi +b3VuZHNjYWxsIGZyb20gd2l0aGluIHRoZSBHbyBydW50aW1lY2FzZ3N0YXR1czogYmFkIGluY29t +aW5nIHZhbHVlc2NoZWNrbWFyayBmb3VuZCB1bm1hcmtlZCBvYmplY3RlbnRlcnN5c2NhbGxibG9j +ayBpbmNvbnNpc3RlbnQgZmF0YWw6IGJhZCBnIGluIHNpZ25hbCBoYW5kbGVyCmludGVybmFsIGVy +cm9yIC0gbWlzdXNlIG9mIGl0YWJub24gaW4tdXNlIHNwYW4gaW4gdW5zd2VwdCBsaXN0cGFjZXI6 +IHN3ZWVwIGRvbmUgYXQgaGVhcCBzaXplIHJlc2V0c3Bpbm5pbmc6IG5vdCBhIHNwaW5uaW5nIG1y +dW50aW1lOiBjYW5ub3QgYWxsb2NhdGUgbWVtb3J5cnVudGltZTogc3BsaXQgc3RhY2sgb3ZlcmZs +b3c6IHNsaWNlIGJvdW5kcyBvdXQgb2YgcmFuZ2UgWyV4Ol1zbGljZSBib3VuZHMgb3V0IG9mIHJh +bmdlIFs6JXhdICh0eXBlcyBmcm9tIGRpZmZlcmVudCBwYWNrYWdlcylTSUdGUEU6IGZsb2F0aW5n +LXBvaW50IGV4Y2VwdGlvblNJR1RUT1U6IGJhY2tncm91bmQgd3JpdGUgdG8gdHR5IiBub3Qgc3Vw +cG9ydGVkIGZvciBjcHUgb3B0aW9uICJlbmQgb3V0c2lkZSB1c2FibGUgYWRkcmVzcyBzcGFjZW5v +bi1HbyBjb2RlIGRpc2FibGVkIHNpZ2FsdHN0YWNrcGFuaWMgd2hpbGUgcHJpbnRpbmcgcGFuaWMg +dmFsdWVydW50aW1lOiBtY2FsbCBmdW5jdGlvbiByZXR1cm5lZHJ1bnRpbWU6IG5ld3N0YWNrIGNh +bGxlZCBmcm9tIGc9cnVudGltZTogcm9vdCBsZXZlbCBtYXggcGFnZXMgPSBydW50aW1lOiBzdGFj +ayBzcGxpdCBhdCBiYWQgdGltZXJ1bnRpbWU6IHN1ZG9nIHdpdGggbm9uLW5pbCBlbGVtcnVudGlt +ZTogc3Vkb2cgd2l0aCBub24tbmlsIG5leHRydW50aW1lOiBzdWRvZyB3aXRoIG5vbi1uaWwgcHJl +dnNjYW5zdGFjazogZ29yb3V0aW5lIG5vdCBzdG9wcGVkc2xpY2UgYm91bmRzIG91dCBvZiByYW5n +ZSBbJXg6Ol1zbGljZSBib3VuZHMgb3V0IG9mIHJhbmdlIFs6JXg6XXNsaWNlIGJvdW5kcyBvdXQg +b2YgcmFuZ2UgWzo6JXhdc3dlZXAgaW5jcmVhc2VkIGFsbG9jYXRpb24gY291bnRHT0RFQlVHOiBu +byB2YWx1ZSBzcGVjaWZpZWQgZm9yICJTSUdDSExEOiBjaGlsZCBzdGF0dXMgaGFzIGNoYW5nZWRT +SUdUVElOOiBiYWNrZ3JvdW5kIHJlYWQgZnJvbSB0dHlTSUdYRlNaOiBmaWxlIHNpemUgbGltaXQg +ZXhjZWVkZWRiYXNlIG91dHNpZGUgdXNhYmxlIGFkZHJlc3Mgc3BhY2Vjb25jdXJyZW50IG1hcCBy +ZWFkIGFuZCBtYXAgd3JpdGVmaW5kcnVubmFibGU6IG5lZ2F0aXZlIG5tc3Bpbm5pbmdmcmVlaW5n +IHN0YWNrIG5vdCBpbiBhIHN0YWNrIHNwYW5oZWFwQml0c1NldFR5cGU6IHVuZXhwZWN0ZWQgc2hp +ZnRtaW4gbXVzdCBiZSBhIG5vbi16ZXJvIHBvd2VyIG9mIDJtaXNyb3VuZGVkIGFsbG9jYXRpb24g +aW4gc3lzQWxsb2NydW50aW1lOiBjYXN0b2dzY2Fuc3RhdHVzIG9sZHZhbD1ydW50aW1lOiBlcG9s +bGNyZWF0ZSBmYWlsZWQgd2l0aCBydW50aW1lOiBmYWlsZWQgbVNwYW5MaXN0Lmluc2VydCBydW50 +aW1lOiBnb3JvdXRpbmUgc3RhY2sgZXhjZWVkcyBydW50aW1lOiBtZW1vcnkgYWxsb2NhdGVkIGJ5 +IE9TIFtydW50aW1lOiBuYW1lIG9mZnNldCBvdXQgb2YgcmFuZ2VydW50aW1lOiB0ZXh0IG9mZnNl +dCBvdXQgb2YgcmFuZ2VydW50aW1lOiB0eXBlIG9mZnNldCBvdXQgb2YgcmFuZ2VzbGljZSBib3Vu +ZHMgb3V0IG9mIHJhbmdlIFsleDoleV1zdGFja2FsbG9jIG5vdCBvbiBzY2hlZHVsZXIgc3RhY2tz +dG9wbG9ja2VkbTogaW5jb25zaXN0ZW50IGxvY2tpbmd0aW1lciBwZXJpb2QgbXVzdCBiZSBub24t +bmVnYXRpdmVTSUdVUkc6IHVyZ2VudCBjb25kaXRpb24gb24gc29ja2V0ZG9hZGR0aW1lcjogUCBh +bHJlYWR5IHNldCBpbiB0aW1lcmZvckVhY2hQOiBzY2hlZC5zYWZlUG9pbnRXYWl0ICE9IDBtc3Bh +bi5lbnN1cmVTd2VwdDogbSBpcyBub3QgbG9ja2Vkb3V0IG9mIG1lbW9yeSBhbGxvY2F0aW5nIGFs +bEFyZW5hc3J1bnRpbWU6IGcgaXMgcnVubmluZyBidXQgcCBpcyBub3RydW50aW1lOiBuZXRwb2xs +QnJlYWsgd3JpdGUgZmFpbGVkcnVudGltZTogdW5leHBlY3RlZCByZXR1cm4gcGMgZm9yIHNjaGVk +dWxlOiBzcGlubmluZyB3aXRoIGxvY2FsIHdvcmtzbGljZSBib3VuZHMgb3V0IG9mIHJhbmdlIFsl +eDoleTpdc2xpY2UgYm91bmRzIG91dCBvZiByYW5nZSBbOiV4OiV5XWF0dGVtcHQgdG8gY2xlYXIg +bm9uLWVtcHR5IHNwYW4gc2V0ZmluZGZ1bmM6IGJhZCBmaW5kZnVuY3RhYiBlbnRyeSBpZHhmaW5k +cnVubmFibGU6IG5ldHBvbGwgd2l0aCBzcGlubmluZ2dyZXlvYmplY3Q6IG9iaiBub3QgcG9pbnRl +ci1hbGlnbmVkbWhlYXAuZnJlZVNwYW5Mb2NrZWQgLSBpbnZhbGlkIGZyZWVwZXJzaXN0ZW50YWxs +b2M6IGFsaWduIGlzIHRvbyBsYXJnZXBpZGxlcHV0OiBQIGhhcyBub24tZW1wdHkgcnVuIHF1ZXVl +dHJhY2ViYWNrIGRpZCBub3QgdW53aW5kIGNvbXBsZXRlbHkpIGlzIGxhcmdlciB0aGFuIG1heGlt +dW0gcGFnZSBzaXplICgpIGlzIG5vdCBHcnVubmFibGUgb3IgR3NjYW5ydW5uYWJsZQpHbyBwb2lu +dGVyIHN0b3JlZCBpbnRvIG5vbi1HbyBtZW1vcnlydW50aW1lOiBpbnZhbGlkIHBjLWVuY29kZWQg +dGFibGUgZj1ydW50aW1lOiBpbnZhbGlkIHR5cGVCaXRzQnVsa0JhcnJpZXJydW50aW1lOiBtYXJr +ZWQgZnJlZSBvYmplY3QgaW4gc3BhbiBydW50aW1lOiBtY2FsbCBjYWxsZWQgb24gbS0+ZzAgc3Rh +Y2tydW50aW1lOiBzdWRvZyB3aXRoIG5vbi1uaWwgd2FpdGxpbmtydW50aW1lOiB3cm9uZyBnb3Jv +dXRpbmUgaW4gbmV3c3RhY2tzaWduYWwgYXJyaXZlZCBkdXJpbmcgY2dvIGV4ZWN1dGlvbgp1bmNh +Y2hpbmcgc3BhbiBidXQgcy5hbGxvY0NvdW50ID09IDApIGlzIHNtYWxsZXIgdGhhbiBtaW5pbXVt +IHBhZ2Ugc2l6ZSAoX2Nnb19ub3RpZnlfcnVudGltZV9pbml0X2RvbmUgbWlzc2luZ2FsbCBnb3Jv +dXRpbmVzIGFyZSBhc2xlZXAgLSBkZWFkbG9jayFmYWlsZWQgdG8gcmVzZXJ2ZSBwYWdlIHN1bW1h +cnkgbWVtb3J5cnVudGltZTogYWxsb2NhdGlvbiBzaXplIG91dCBvZiByYW5nZXJ1bnRpbWU6IG5l +dHBvbGw6IGJyZWFrIGZkIHJlYWR5IGZvciBydW50aW1lOiB1bmV4cGVjdGVkIFNQV1JJVEUgZnVu +Y3Rpb24gc2V0cHJvZmlsZWJ1Y2tldDogcHJvZmlsZSBhbHJlYWR5IHNldHN0YXJ0VGhlV29ybGQ6 +IGluY29uc2lzdGVudCBtcC0+bmV4dHBnY0JnTWFya1dvcmtlcjogYmxhY2tlbmluZyBub3QgZW5h +YmxlZGluZGV4IG91dCBvZiByYW5nZSBbJXhdIHdpdGggbGVuZ3RoICV5bWFrZWNoYW46IGludmFs +aWQgY2hhbm5lbCBlbGVtZW50IHR5cGVydW50aW1lOiBmdW5jdGlvbiBzeW1ib2wgdGFibGUgaGVh +ZGVyOnJ1bnRpbWU6IHN1ZG9nIHdpdGggbm9uLWZhbHNlIGlzU2VsZWN0dW5yZWFjaGFibGUgbWV0 +aG9kIGNhbGxlZC4gbGlua2VyIGJ1Zz9oZWFwQml0c1NldFR5cGVHQ1Byb2c6IHNtYWxsIGFsbG9j +YXRpb25taXNtYXRjaGVkIGNvdW50IGR1cmluZyBpdGFiIHRhYmxlIGNvcHltc3Bhbi5zd2VlcDog +YmFkIHNwYW4gc3RhdGUgYWZ0ZXIgc3dlZXBvdXQgb2YgbWVtb3J5IGFsbG9jYXRpbmcgaGVhcCBh +cmVuYSBtYXBydW50aW1lOiBjYXNmcm9tX0dzY2Fuc3RhdHVzIGZhaWxlZCBncD1zdGFjayBncm93 +dGggbm90IGFsbG93ZWQgaW4gc3lzdGVtIGNhbGxzdXNwZW5kRyBmcm9tIG5vbi1wcmVlbXB0aWJs +ZSBnb3JvdXRpbmV0cmFjZWJhY2s6IHVuZXhwZWN0ZWQgU1BXUklURSBmdW5jdGlvbiBidWxrQmFy +cmllclByZVdyaXRlOiB1bmFsaWduZWQgYXJndW1lbnRzY2Fubm90IGZyZWUgd29ya2J1ZnMgd2hl +biB3b3JrLmZ1bGwgIT0gMHJlZmlsbCBvZiBzcGFuIHdpdGggZnJlZSBzcGFjZSByZW1haW5pbmdy +dW50aW1lOiBuZXRwb2xsQnJlYWsgd3JpdGUgZmFpbGVkIHdpdGggcnVudGltZTogb3V0IG9mIG1l +bW9yeTogY2Fubm90IGFsbG9jYXRlIHJ1bnRpbWU6IHR5cGVCaXRzQnVsa0JhcnJpZXIgd2l0aCB0 +eXBlICAgcmVjZWl2ZWQgb24gdGhyZWFkIHdpdGggbm8gc2lnbmFsIHN0YWNrCmF0dGVtcHRlZCB0 +byBhZGQgemVyby1zaXplZCBhZGRyZXNzIHJhbmdlZ2NTd2VlcCBiZWluZyBkb25lIGJ1dCBwaGFz +ZSBpcyBub3QgR0NvZmZtaGVhcC5mcmVlU3BhbkxvY2tlZCAtIGludmFsaWQgc3BhbiBzdGF0ZW1o +ZWFwLmZyZWVTcGFuTG9ja2VkIC0gaW52YWxpZCBzdGFjayBmcmVlb2JqZWN0cyBhZGRlZCBvdXQg +b2Ygb3JkZXIgb3Igb3ZlcmxhcHBpbmdydW50aW1lOiB0eXBlQml0c0J1bGtCYXJyaWVyIHdpdGhv +dXQgdHlwZXN0b3BUaGVXb3JsZDogbm90IHN0b3BwZWQgKHN0b3B3YWl0ICE9IDApIHJlY2VpdmVk +IGJ1dCBoYW5kbGVyIG5vdCBvbiBzaWduYWwgc3RhY2sKYWNxdWlyZVN1ZG9nOiBmb3VuZCBzLmVs +ZW0gIT0gbmlsIGluIGNhY2hlbm9uLWVtcHR5IG1hcmsgcXVldWUgYWZ0ZXIgY29uY3VycmVudCBt +YXJrb24gYSBsb2NrZWQgdGhyZWFkIHdpdGggbm8gdGVtcGxhdGUgdGhyZWFkb3V0IG9mIG1lbW9y +eSBhbGxvY2F0aW5nIGNoZWNrbWFya3MgYml0bWFwcGVyc2lzdGVudGFsbG9jOiBhbGlnbiBpcyBu +b3QgYSBwb3dlciBvZiAydW5leHBlY3RlZCBzaWduYWwgZHVyaW5nIHJ1bnRpbWUgZXhlY3V0aW9u +Z2NCZ01hcmtXb3JrZXI6IHVuZXhwZWN0ZWQgZ2NNYXJrV29ya2VyTW9kZWdyZXcgaGVhcCwgYnV0 +IG5vIGFkZXF1YXRlIGZyZWUgc3BhY2UgZm91bmRoZWFwQml0c1NldFR5cGVHQ1Byb2c6IHVuZXhw +ZWN0ZWQgYml0IGNvdW50bm9uIGluLXVzZSBzcGFuIGZvdW5kIHdpdGggc3BlY2lhbHMgYml0IHNl +dHJvb3QgbGV2ZWwgbWF4IHBhZ2VzIGRvZXNuJ3QgZml0IGluIHN1bW1hcnlydW50aW1lOiBjYXNm +cm9tX0dzY2Fuc3RhdHVzIGJhZCBvbGR2YWwgZ3A9cnVudGltZTogaGVhcEJpdHNTZXRUeXBlR0NQ +cm9nOiB0b3RhbCBiaXRzIHJ1bnRpbWU6IHJlbGVhc2VTdWRvZyB3aXRoIG5vbi1uaWwgZ3AucGFy +YW1ydW50aW1lOnN0b3Bsb2NrZWRtOiBsb2NrZWRnIChhdG9taWNzdGF0dXM9dW5maW5pc2hlZCBv +cGVuLWNvZGVkIGRlZmVycyBpbiBkZWZlcnJldHVybnVua25vd24gcnVubmFibGUgZ29yb3V0aW5l +IGR1cmluZyBib290c3RyYXBnY21hcmtuZXdvYmplY3QgY2FsbGVkIHdoaWxlIGRvaW5nIGNoZWNr +bWFya291dCBvZiBtZW1vcnkgYWxsb2NhdGluZyBoZWFwIGFyZW5hIG1ldGFkYXRhcnVudGltZTog +bGZzdGFjay5wdXNoIGludmFsaWQgcGFja2luZzogbm9kZT1leGl0c3lzY2FsbDogc3lzY2FsbCBm +cmFtZSBpcyBubyBsb25nZXIgdmFsaWRoZWFwQml0c1NldFR5cGU6IGNhbGxlZCB3aXRoIG5vbi1w +b2ludGVyIHR5cGVydW50aW1lOiBmYWlsZWQgbVNwYW5MaXN0LnJlbW92ZSBzcGFuLm5wYWdlcz1z +Y2F2ZW5nZU9uZSBjYWxsZWQgd2l0aCB1bmFsaWduZWQgd29yayByZWdpb24gKGJhZCB1c2Ugb2Yg +dW5zYWZlLlBvaW50ZXI/IHRyeSAtZD1jaGVja3B0cikKbWVtb3J5IHJlc2VydmF0aW9uIGV4Y2Vl +ZHMgYWRkcmVzcyBzcGFjZSBsaW1pdHBhbmljd3JhcDogdW5leHBlY3RlZCBzdHJpbmcgYWZ0ZXIg +dHlwZSBuYW1lOiByZWxlYXNlZCBsZXNzIHRoYW4gb25lIHBoeXNpY2FsIHBhZ2Ugb2YgbWVtb3J5 +cnVudGltZTogZmFpbGVkIHRvIGNyZWF0ZSBuZXcgT1MgdGhyZWFkIChoYXZlIHJ1bnRpbWU6IG5h +bWUgb2Zmc2V0IGJhc2UgcG9pbnRlciBvdXQgb2YgcmFuZ2VydW50aW1lOiBwYW5pYyBiZWZvcmUg +bWFsbG9jIGhlYXAgaW5pdGlhbGl6ZWQKcnVudGltZTogdGV4dCBvZmZzZXQgYmFzZSBwb2ludGVy +IG91dCBvZiByYW5nZXJ1bnRpbWU6IHR5cGUgb2Zmc2V0IGJhc2UgcG9pbnRlciBvdXQgb2YgcmFu +Z2VzbGljZSBib3VuZHMgb3V0IG9mIHJhbmdlIFs6JXhdIHdpdGggbGVuZ3RoICV5c3RvcFRoZVdv +cmxkOiBub3Qgc3RvcHBlZCAoc3RhdHVzICE9IF9QZ2NzdG9wKXN5c0dyb3cgYm91bmRzIG5vdCBh +bGlnbmVkIHRvIHBhbGxvY0NodW5rQnl0ZXNQIGhhcyBjYWNoZWQgR0Mgd29yayBhdCBlbmQgb2Yg +bWFyayB0ZXJtaW5hdGlvbnJhY3kgc3Vkb2cgYWRqdXN0bWVudCBkdWUgdG8gcGFya2luZyBvbiBj +aGFubmVsc2xpY2UgYm91bmRzIG91dCBvZiByYW5nZSBbOjoleF0gd2l0aCBsZW5ndGggJXlydW50 +aW1lOiBjYW5ub3QgbWFwIHBhZ2VzIGluIGFyZW5hIGFkZHJlc3Mgc3BhY2VzbGljZSBib3VuZHMg +b3V0IG9mIHJhbmdlIFs6JXhdIHdpdGggY2FwYWNpdHkgJXljYXNnc3RhdHVzOiB3YWl0aW5nIGZv +ciBHd2FpdGluZyBidXQgaXMgR3J1bm5hYmxlZnVsbHkgZW1wdHkgdW5mcmVlZCBzcGFuIHNldCBi +bG9jayBmb3VuZCBpbiByZXNldGludmFsaWQgbWVtb3J5IGFkZHJlc3Mgb3IgbmlsIHBvaW50ZXIg +ZGVyZWZlcmVuY2VwYW5pY3dyYXA6IHVuZXhwZWN0ZWQgc3RyaW5nIGFmdGVyIHBhY2thZ2UgbmFt +ZTogcy5hbGxvY0NvdW50ICE9IHMubmVsZW1zICYmIGZyZWVJbmRleCA9PSBzLm5lbGVtc3NsaWNl +IGJvdW5kcyBvdXQgb2YgcmFuZ2UgWzo6JXhdIHdpdGggY2FwYWNpdHkgJXlhdHRlbXB0IHRvIGV4 +ZWN1dGUgc3lzdGVtIHN0YWNrIGNvZGUgb24gdXNlciBzdGFja21hbGxvY2djIGNhbGxlZCB3aXRo +IGdjcGhhc2UgPT0gX0dDbWFya3Rlcm1pbmF0aW9ucmVjdXJzaXZlIGNhbGwgZHVyaW5nIGluaXRp +YWxpemF0aW9uIC0gbGlua2VyIHNrZXdHQyBtdXN0IGJlIGRpc2FibGVkIHRvIHByb3RlY3QgdmFs +aWRpdHkgb2YgZm4gdmFsdWVmYXRhbDogc3lzdGVtc3RhY2sgY2FsbGVkIGZyb20gdW5leHBlY3Rl +ZCBnb3JvdXRpbmVwb3RlbnRpYWxseSBvdmVybGFwcGluZyBpbi11c2UgYWxsb2NhdGlvbnMgZGV0 +ZWN0ZWRjYXNmcm9tX0dzY2Fuc3RhdHVzOiBncC0+c3RhdHVzIGlzIG5vdCBpbiBzY2FuIHN0YXRl +ZnVuY3Rpb24gc3ltYm9sIHRhYmxlIG5vdCBzb3J0ZWQgYnkgcHJvZ3JhbSBjb3VudGVyOm1hbGxv +Y2djIGNhbGxlZCB3aXRob3V0IGEgUCBvciBvdXRzaWRlIGJvb3RzdHJhcHBpbmdydW50aW1lOiB1 +c2Ugb2YgRml4QWxsb2NfQWxsb2MgYmVmb3JlIEZpeEFsbG9jX0luaXQKc3BhbiBzZXQgYmxvY2sg +d2l0aCB1bnBvcHBlZCBlbGVtZW50cyBmb3VuZCBpbiByZXNldAlnb3JvdXRpbmUgcnVubmluZyBv +biBvdGhlciB0aHJlYWQ7IHN0YWNrIHVuYXZhaWxhYmxlCmdjQ29udHJvbGxlclN0YXRlLmZpbmRS +dW5uYWJsZTogYmxhY2tlbmluZyBub3QgZW5hYmxlZG5vIGdvcm91dGluZXMgKG1haW4gY2FsbGVk +IHJ1bnRpbWUuR29leGl0KSAtIGRlYWRsb2NrIWNhc2Zyb21fR3NjYW5zdGF0dXM6dG9wIGdwLT5z +dGF0dXMgaXMgbm90IGluIHNjYW4gc3RhdGVnZW50cmFjZWJhY2sgY2FsbGJhY2sgY2Fubm90IGJl +IHVzZWQgd2l0aCBub24temVybyBza2lwbmV3cHJvYzogZnVuY3Rpb24gYXJndW1lbnRzIHRvbyBs +YXJnZSBmb3IgbmV3IGdvcm91dGluZWluIGdjTWFyayBleHBlY3RpbmcgdG8gc2VlIGdjcGhhc2Ug +YXMgX0dDbWFya3Rlcm1pbmF0aW9ucHJvZmlsZWFsbG9jIGNhbGxlZCB3aXRob3V0IGEgUCBvciBv +dXRzaWRlIGJvb3RzdHJhcHBpbmdnZW50cmFjZWJhY2sgY2Fubm90IHRyYWNlIHVzZXIgZ29yb3V0 +aW5lIG9uIGl0cyBvd24gc3RhY2tub24tR28gY29kZSBzZXQgdXAgc2lnbmFsIGhhbmRsZXIgd2l0 +aG91dCBTQV9PTlNUQUNLIGZsYWdydW50aW1lOiBjaGVja21hcmtzIGZvdW5kIHVuZXhwZWN0ZWQg +dW5tYXJrZWQgb2JqZWN0IG9iaj1ydW50aW1lOiBuZXRwb2xsOiBicmVhayBmZCByZWFkeSBmb3Ig +c29tZXRoaW5nIHVuZXhwZWN0ZWRydW50aW1lOiBtbWFwOiB0b28gbXVjaCBsb2NrZWQgbWVtb3J5 +IChjaGVjayAndWxpbWl0IC1sJykuCmFkZHIgcmFuZ2UgYmFzZSBhbmQgbGltaXQgYXJlIG5vdCBp +biB0aGUgc2FtZSBtZW1vcnkgc2VnbWVudG1hbnVhbCBzcGFuIGFsbG9jYXRpb24gY2FsbGVkIHdp +dGggbm9uLW1hbnVhbGx5LW1hbmFnZWQgdHlwZWFiaVJlZ0FyZ3NUeXBlIG5lZWRzIEdDIFByb2cs +IHVwZGF0ZSBtZXRob2RWYWx1ZUNhbGxGcmFtZU9ianNydW50aW1lOiBtYXkgbmVlZCB0byBpbmNy +ZWFzZSBtYXggdXNlciBwcm9jZXNzZXMgKHVsaW1pdCAtdSkKZm91bmQgYmFkIHBvaW50ZXIgaW4g +R28gaGVhcCAoaW5jb3JyZWN0IHVzZSBvZiB1bnNhZmUgb3IgY2dvPylydW50aW1lOiBpbnRlcm5h +bCBlcnJvcjogbWlzdXNlIG9mIGxvY2tPU1RocmVhZC91bmxvY2tPU1RocmVhZGNhbm5vdCBjb252 +ZXJ0IHNsaWNlIHdpdGggbGVuZ3RoICV5IHRvIHBvaW50ZXIgdG8gYXJyYXkgd2l0aCBsZW5ndGgg +JXgwd68MknQIAkHhwQfm1hjmcGF0aAlleGFtcGxlLmNvbS9nbzExNwptb2QJZXhhbXBsZS5jb20v +Z28xMTcJKGRldmVsKQkK+TJDMYYYIHIAgkIQQRbY8v8AAAD/AAAA/wAAAP8AAAAABP8AAAH/AAAI +/wAABAQE/wAAAAAICAH/AAAAAAgIBP8AAAAACAgI/wAAAAAECAj/AAAAAAEBAf8AAAAACAgI/wAA +AP4ACP3/AAAAAAEICP8AAAAAKQEACAAAAAApAQAIAAAAACEBAAgAAAAACAgBEAj/AP4ACP0ICP8A +AAgIBAwE/wD+AAgICP3/AAAICAgQAf8AAAgIBBAI/wAACAgIEAT/AAAI/ggI/f8AAAQICBAB/wAA +BAgIEAj/AAAICAgQCP8AAAgICBAI/wAAAAAAAQAAAAAAAAAFAAAAAAAAAAIAAAAAAAAACgAAAAAA +AAADAAAAAAAAAAwAAAAAAAAADgAAAAAAAAAGAAAAAAAAAAcAAAAAAAAABAAAAAAAAADgB0UAAAAA +AIBvQwAAAAAAAHBDAAAAAADAb0MAAAAAAEBwQwAAAAAAQG9DAAAAAACAcEMAAAAAAIAWRAAAAAAA +gElFAAAAAADAyUIAAAAAAEA5RQAAAAAAAG9DAAAAAABAB0UAAAAAAMCaQQAAAAAAALpBAAAAAABg +JEAAAAAAACAkQAAAAAAAoD9FAAAAAABgLUAAAAAAAABRQAAAAAAAAEhAAAAAAADgOkAAAAAAAGAA +RQAAAAAAIAFFAAAAAABgXUAAAAAAACBWQwAAAAAAoFVDAAAAAACgWkMAAAAAAIBeQwAAAAAA4CNA +AAAAAAAAJEAAAAAAAGAJRQAAAAAA4OBCAAAAAABgLUQAAAAAAKAGRQAAAAAAID9BAAAAAADgAkUA +AAAAAGADRQAAAAAA4ANFAAAAAAAABUUAAAAAAOAFRQAAAAAAIAZFAAAAAABgBkUAAAAAACAHRQAA +AAAAwEtBAAAAAACgAkUAAAAAAKBPQwAAAAAAQEtDAAAAAAAASkMAAAAAAGBKQwAAAAAAYE5DAAAA +AAAAJUAAAAAAAMAJRQAAAAAAgFNFAAAAAAAAWUEAAAAAACAjQAAAAAAAwCNAAAAAAABgI0AAAAAA +AIAjQAAAAAAAoCNAAAAAAABAI0AAAAAAAMAiQAAAAAAA4CNFAAAAAADgHkMAAAAAAMBHRQAAAAAA +YCVAAAAAAABgRkMAAAAAAABGQwAAAAAAYCVBAAAAAABgNEMAAAAAAMBLQwAAAAAAAAlFAAAAAABg +4EEAAAAAAAC5QgAAAAAAwA5DAAAAAAAACkUAAAAAAKDkQwAAAAAAoCRFAAAAAABgQEUAAAAAACA/ +RQAAAAAAgAJDAAAAAACgJEAAAAAAAIAHRQAAAAAAwAhFAAAAAADgikMAAAAAAOAcQwAAAAAAII5E +AAAAAABAkUQAAAAAACCLQAAAAAAAgAhFAAAAAAABAAAABQAAAAAAAAABAAAAAQAAAAAAAAABAAAA +AwAAAAAAAAABAAAAAgAAAAAAAAABAAAABAAAAAAAAAD+AAgICP0QAf8AAAAACAgIEAgYAf8AAAAA +CAgIEAERAf8AAAD+AAgICP0QBP8AAAAACAgIEAgYCP8AAAAACP4ICBAI/f8AAAAACAgIEAEYCP8A +AAD+AAgICBAI/f8AAAD+AAgICBAE/f8AAAD+AAgIAQkB/f8AAAAACAgIEAQUAf8AAAAACAgIEAQU +BP8AAAAACAgBEAgYCP8AAAAABAgIEAgYCP8AAAD+AAgICP0QCP8AAAACAAAAAgAAAAIAAAACAAAA +BgAAAAAAAAACAAAACAAAAKsAAAACAAAAAwAAAAAAAAACAAAAAQAAAAEAAAACAAAABQAAABAAAAAC +AAAABQAAAAAAAAACAAAABwAAAAABAAACAAAAAgAAAAAAAAACAAAAAQAAAAAAAAACAAAAAwAAAAUA +AAACAAAABQAAABMAAAACAAAABAAAAAkAAAACAAAAAgAAAAEAAAACAAAABQAAABkAAAACAAAACAAA +AAAAAAACAAAAAwAAAAABAAACAAAABQAAAAAFAAACAAAABQAAABsAAAACAAAABAAAAAAAAAACAAAA +AwAAAAcAAAACAAAAAQAAAAABAAACAAAAAwAAAAQAAAACAAAABAAAAAgAAAACAAAABAAAAAwAAAAC +AAAABgAAAAAJAAACAAAAAwAAAAAHAAACAAAAAwAAAAYAAAACAAAABQAAAAABAAACAAAABAAAAAAI +AAACAAAABAAAAAABAAACAAAAAgAAAAMAAAACAAAAAgAAAAADAAACAAAABAAAAAsAAAADAAAABQAA +ABwUAAADAAAAAwAAAAABAgADAAAAAgAAAAMAAAADAAAAAwAAAAUBAAADAAAAAgAAAAABAgADAAAA +AQAAAAEAAQADAAAABAAAAAAJCAADAAAAAwAAAAcFAAADAAAAAQAAAAEAAAADAAAAAgAAAAMBAAAD +AAAABQAAAAADAAADAAAABgAAAAABAAADAAAABwAAAAAKAAADAAAABAAAAAkBAAADAAAABQAAABgQ +AAADAAAABAAAAAANAgADAAAAAwAAAAABAAADAAAACAAAAAADAQADAAAABAAAAAsKAAADAAAAAgAA +AAMAAwADAAAAAwAAAAcGAAADAAAAAgAAAAMDAAADAAAAAgAAAAADAgADAAAAAQAAAAABAAADAAAA +AwAAAAUAAAADAAAABAAAAAAPAAADAAAAAgAAAAADAQADAAAAAwAAAAAABwADAAAABAAAAAkAAAAD +AAAABAAAAAgAAAADAAAABAAAAAADAQADAAAABAAAAAAODwADAAAAAwAAAAABBwADAAAAAQAAAAAA +AQADAAAAAgAAAAACAQADAAAABwAAAEtBAAADAAAAAgAAAAIAAgADAAAACAAAAIUAAAADAAAAAwAA +AAAHAwADAAAAAgAAAAIAAAADAAAAAgAAAAABAwADAAAAAQAAAAEBAAADAAAAAgAAAAADAAADAAAA +AwAAAAcAAAADAAAABQAAABAAAAADAAAAAwAAAAAAAgADAAAAAwAAAAAGAAADAAAAAwAAAAcDAAAD +AAAABQAAAAAYAAADAAAAAgAAAAMCAAADAAAAAwAAAAAHBAD+/gAI/f4ICP39/wAACAgIEAgYARkB +/wD+AAgICBABEQH9/wD+AAgICBAI/RgI/wAACAgIEAgYCCAI/wD+/gAICAj9EAj9/wD+AAgICP0Q +CBgB/wD+AAgICP0QCBgI/wAACAgIEAERARgI/wD+AAgIBAwEEAj9/wAACAgIEAQYCCAI/wD+AAgI +CP0QBBgI/wAACAgBEAgYCCAI/wAACP4ICBAIGAj9/wAEAAAAAQAAAAEBAAEEAAAAAwAAAAQEBAAE +AAAAAQAAAAEAAQAEAAAAAQAAAAEBAQAEAAAABAAAAAAECQIEAAAAAwAAAAYGAAAEAAAAAgAAAAAC +AQACAAAADgAAAAAAODEEAAAAAwAAAAAEAgEEAAAAAwAAAAcCAgAEAAAAAgAAAAABAAIEAAAAAgAA +AAACAwACAAAACQAAAAAAAQAEAAAAAwAAAAABAwUEAAAAAgAAAAACAAMEAAAABQAAAAAAAAcEAAAA +AQAAAAEAAAAEAAAABAAAAAAHCwEEAAAABAAAAAAJDwAEAAAACAAAAAAABwQEAAAAAgAAAAMAAAAE +AAAAAgAAAAABAgAEAAAAAgAAAAMAAAEEAAAABAAAAAAJDw0EAAAAAwAAAAAEAwEEAAAAAgAAAAAD +AgAEAAAACAAAAAAAASQEAAAAAQAAAAEAAQEEAAAABAAAAAAADgEEAAAAAgAAAAICAgAEAAAABAAA +AAADAgAEAAAABwAAAAAfFgAEAAAABgAAAAACAwcEAAAAAwAAAAADBgIEAAAAAwAAAAAGAAEEAAAA +AwAAAAAEAwAEAAAABAAAAA8KAgAEAAAAAQAAAAAAAQAEAAAAAgAAAAACAwEEAAAABwAAAAAuACcE +AAAAAwAAAAAHBgQEAAAAAwAAAAADAgQEAAAABQAAAAAAAgEEAAAACAAAAAAEBAAEAAAAAgAAAAMB +AQAEAAAAAgAAAAABAwAEAAAAAQAAAAAAAQEEAAAABQAAAB8AAAAEAAAABQAAABcRAAAEAAAAAgAA +AAADAQAEAAAAAwAAAAABBgIEAAAAAgAAAAMAAwMEAAAAAQAAAAEBAAACAAAACQAAAAAAAAAEAAAA +AwAAAAcHBgAEAAAAAgAAAAMCAgAEAAAAAgAAAAMCAAAEAAAAAgAAAAMDAwAEAAAAAQAAAAABAQAE +AAAAAgAAAAABAQAEAAAAAwAAAAcDAAEEAAAAAgAAAAIAAAAEAAAAAgAAAAMAAQAEAAAAAwAAAAAF +AAIEAAAABQAAABIAAAAEAAAAAwAAAAcBAQAEAAAAAwAAAAABAwAEAAAAAgAAAAMCAAIEAAAAAwAA +AAUAAAAEAAAAAwAAAAAFBgAEAAAAAgAAAAAAAQIEAAAAAwAAAAAHBQAEAAAABAAAAAAGCQAEAAAA +AgAAAAACAAEFAAAABAAAAAACCgMGAAAABQAAAAMAAAAAAQQCAAAAAAUAAAACAAAAAAIAAwEAAAAF +AAAAAgAAAAMDAwMAAAAABQAAAAQAAAAACAwKAQAAAAUAAAADAAAAAAECBAAAAAAFAAAAAQAAAAEB +AAAAAAAABQAAAAMAAAAHBQUBAAAAAAUAAAADAAAABAQEBAAAAAAFAAAAAQAAAAEBAQEAAAAABQAA +AAIAAAAAAwIBAAAAAAUAAAABAAAAAQAAAAAAAAAFAAAABQAAAAAYHh8IAAAABQAAAAIAAAAAAgIB +AAAAAAUAAAAEAAAAAAQFDAYAAAAFAAAABgAAAAACAwEAAAAABQAAAAUAAAAAFxEBCQAAAAUAAAAC +AAAAAgICAgAAAAAFAAAAAwAAAAADAgcGAAAABQAAAAMAAAAABAABAgAAAAUAAAADAAAAAAQFBgAA +AAAFAAAABAAAAAsAAAAAAAAABQAAAAMAAAAABQAEAgAAAAUAAAAFAAAAAAUHBgAAAAAFAAAABQAA +AAAFARIAAAAABQAAAAcAAABKSEAAAAAAAAUAAAAEAAAACQgIAAAAAAAFAAAAAwAAAAUFAQAAAAAA +BQAAAAUAAAAAFwQZAAAAAAUAAAACAAAAAQEBAQAAAAAFAAAAAgAAAAACAAEAAAAABQAAAAMAAAAF +AAAAAAAAAAUAAAAGAAAAACUFHgAAAAAFAAAAAwAAAAAEAgMAAAAABQAAAAEAAAABAAABAAAAAAUA +AAADAAAAAAQHAwAAAAAFAAAAAQAAAAAAAQEAAAAABQAAAAIAAAAAAQACAAAAAAUAAAADAAAAAAYC +AQAAAAAFAAAABwAAAAAEAQIAAAAABQAAAAMAAAAHBAQAAAAAAAUAAAACAAAAAAACAQEAAAAFAAAA +AgAAAAABAwEAAAAABQAAAAQAAAAADgUAAQAAAAUAAAAFAAAAAAISEB8AAAAFAAAAAQAAAAEBAAAB +AAAAAAgICP4QCBgIIAj9/wAAAAAI/v4ICP3+EAj9/f8AAAD+/gAI/f4ICP39EAj/AAAAAAgICBAI +/hgIIAj9/wAAAAAICAgQCBgIIAgoCP8AAAAACP4ICBAI/RgBIAj/AAAAAAj+CAgQCBgI/SAI/wAA +AP4ACAgEDAQQCP0YCP8AAAAACAgIEAQUBBgEHAT/AAAA/gAICAj9EAQYCCAI/wAAAP4ACAgI/RAB +EQESAf8AAAAAAQgI/hAIGAggCP3/AAAABgAAAAMAAAAAAQQCAgAAAAYAAAABAAAAAQEAAAAAAAAG +AAAAAgAAAAAAAQACAAAABgAAAAQAAAAPCgsLAgAAAAYAAAACAAAAAwAAAQAAAAAGAAAAAwAAAAAB +AwAEBAAABgAAAAMAAAAGBgYABgAAAAYAAAABAAAAAQEAAQEAAAAGAAAAAgAAAAMDAQAAAAAABgAA +AAIAAAAAAQEAAgIAAAYAAAABAAAAAQEAAAEAAAAGAAAACAAAAAABBAYEAAAABgAAAAMAAAAABgcA +AQEAAAYAAAADAAAAAAIHAwIAAAADAAAACwAAAAAAAADvBgAABgAAAAEAAAABAQEBAQAAAAYAAAAC +AAAAAwMBAAMAAAAGAAAABAAAAAAMDgQFAAAABgAAAAMAAAAEBAQABAAAAAYAAAACAAAAAwAAAAAA +AAAGAAAAAQAAAAEBAAEAAAAABgAAAAYAAAAACAoKDQAAAAYAAAACAAAAAAIDAgEAAAAGAAAAAwAA +AAAGBgcBAAAABgAAAAMAAAAAAQEDBwAAAAYAAAADAAAAAAUBAgIAAAAGAAAAAwAAAAAEAAUBAQAA +BgAAAAEAAAABAQEBAAAAAAYAAAACAAAAAwMDAwMAAAAGAAAABQAAAAAFBwMBAAAABwAAAAMAAAAA +AQAFBAMCAAcAAAAEAAAAAA4PCwsJAAAHAAAABQAAAAAYAAACAAEABwAAAAMAAAAHAwMDAgMAAAcA +AAADAAAABQQAAAAAAAAHAAAABgAAAAAAIAMZARAABwAAAAYAAAAAITEBCwkAAAcAAAAHAAAAAAQF +BwMBAAAHAAAABQAAAAAQAgoEBQEABwAAAAIAAAADAwEAAwICAAcAAAAEAAAACAAAAAAAAAAHAAAA +BAAAAAAIDQEAAwAABwAAAAQAAAAACAcDAQEAAAcAAAAGAAAAKSkBAQEAAAAHAAAABAAAAAkBAQEA +AAAABwAAAAEAAAABAQEAAAAAAAcAAAACAAAAAgAAAAAAAAAACP4ICBAIGAj9IAgoCP8AAAj+/ggI +/f4QCP39GAT/AAAI/ggIEAj9/hgIIAj9/wD+AAgICP0QBBgIIAgoAf8A/v4ACAgIEAj9GAggCP3/ +AAAICAgQCP4YCCABKAj9/wD+AAgICP0QCBgBGQEaAf8A/gAICAgQCP0YCCAIKAH/AAAICAj+EAgY +CCAI/SgB/wAAAAAACAAAAAcAAAAAQF8QbE4MAAgAAAACAAAAAwADAQEDAAAEAAAADgAAAAAAHyAe +IAAACAAAAAQAAAAAAAwNDwwFBAgAAAAEAAAAAAAIBQcGBgIIAAAABQAAABMAEAAAAAAACAAAAAMA +AAAHAwMDAwMDAAgAAAAHAAAAAABgcnBpbAgIAAAAAgAAAAMDAAMDAQEBCAAAAAIAAAACAAICAgIA +AAQAAAAMAAAAAAAAAN8N3g0IAAAABAAAAA4IAAgAAAAACAAAAAEAAAABAAAAAAAAAAgAAAABAAAA +AQEBAQEAAAAIAAAACAAAAAACAAY9IQACCAAAAAYAAAAAMQA5MTUwNwMAAAAUAAAAAAAAAQAAAwAA +AAAACQAAAAYAAAAAMDY1JCwoIAAAAAAJAAAAAgAAAAMAAAAAAAAAAAAAAAkAAAACAAAAAgAAAAAA +AAIAAAAAAAj+/ggI/f4QCP39GAggAf8AAAD+AAgICBAI/f4YCCAIKAj9/wAAAAAE/v4ICBAI/RgE +HAQgCP3/AAAAAAgICBAIGAj+IAgoCP0wCP8AAAAKAAAAAQAAAAEBAQEBAQEBAQAAAAoAAAABAAAA +AQEBAQEBAAAAAAAACwAAAAIAAAADAQEBAAAAAAABAAD+AAgICBAI/f4YCCAIKAj9MAj/AP//AAAB +AAAA+wAAAL0CAAAuAAAA//8AAAEAAAAMAQAAvQIAADIAAAD//wAAAQAAAD0BAADzAgAANAAAAP// +AAACAAAAFwAAABcDAAAOAAAA//8AAAgAAAAxAAAAIAQAAFQAAAD//wAACAAAAGMAAACPBAAAkAAA +AP//AAAPAAAAcwAAAL8HAAAdAAAA//8AABIAAAAtAAAAUQgAADUAAAD//wAAEgAAAM8AAADmBgAA +UwAAAP//AAAWAAAAEgAAAA4JAAA5AAAA//8AAB8AAACUAAAAMAMAALkAAAD//wAAJQAAAL0EAACJ +DgAAeAAAAP//AAAmAAAA8AAAAP8OAABZAAAA//8AACsAAACVAAAA0BMAAAAAAAD//wAAKwAAAAED +AAAlBQAA4gAAAP//AAAsAAAAXwAAANgOAACJAAAA//8AACwAAAAKAQAAvhYAAI0AAAD//wAAMwAA +ABIBAAC/BwAAHQAAAP//AAAzAAAA1QIAAPkaAABQAAAA//8AADMAAABeAwAARRsAAHAAAAD//wAA +MwAAAGcEAADsGwAAOAAAAP//AAA0AAAAiQAAAGcdAABSAAAA//8AADQAAADTAAAAZx0AACsAAAD/ +/wAANAAAAAUBAACPBAAATAAAAP//AAA0AAAAZAMAAFEIAAAYAAAA//8AADQAAAAIBAAAnyAAAEEB +AAD//wAANAAAAFsEAACfIAAAsQAAAP//AAA1AAAAUAEAADIeAABSAQAA//8AADUAAABMAwAAKiQA +AFYAAAD//wAANwAAACoBAADWHwAAFwEAAP//AAA6AAAAmQMAADgOAAChAAAA//8AADoAAACOAwAA +2yMAAEIAAAD//wAAOgAAANQDAADOBAAARwAAAP//AAA6AAAALQQAAAgVAAAaAAAA//8AADsAAAAm +AgAAzgQAABoAAAD//wAAPQAAADsAAADLMwAAGAAAAP//AAA+AAAAIgAAAHY0AACQAAAA//8AAD4A +AABGAAAAljIAAJYAAAD//wAAPgAAAOUAAABANQAAZgAAAP//AAA+AAAAogEAAL81AAAjAAAA//8A +AD4AAACqAQAA/zUAABkAAAD//wAAPwAAAL8AAACPBAAAGwAAAP//AAA/AAAAyAAAAI8EAAAmAAAA +//8AAD8AAAA5AQAA2DYAAFcAAAD//wAAPwAAAHgDAACcNwAAIAAAAP//AAA/AAAAjAMAAJw3AAAg +AAAA//8AAEMAAABZAQAA0DkAADkAAAD//wAASAAAAFYAAAA0PAAAQgAAAP//AABJAAAAMQAAAGU8 +AABTAAAA//8AAE4AAAAiAAAACT4AACIAAAD//wAATgAAABYEAAAnQQAAVwIAAP//AABOAAAAEgUA +APNBAACCAAAA//8AAE4AAADWBQAAUQgAABYAAAD//wAAUgAAAPcAAAA/RAAAGQAAAP//AABSAAAA +/QAAAAFEAACKAAAA//8AAFIAAAAnAQAAUQgAAEgBAAD//wAAUwAAAPQBAAA0PAAADgAAAP//AABT +AAAA/QEAADQ8AAAOAAAA//8AAFMAAAD2AgAA6jwAABgAAAD//wAAUwAAANwHAAC5SAAANgAAAP// +AABTAAAAgAoAALBKAADsAAAA//8AAFMAAADlDQAAMk0AACYAAAD//wAAUwAAACsOAAB/TQAAHQAA +AP//AABTAAAA5A4AANNHAAChAQAA//8AAFMAAABBDwAAl04AAB0AAAD//wAAUwAAAH4QAABwTwAA +MAAAAP//AABTAAAAOREAAHg3AAATAQAA//8AAFMAAAAdEgAAd1AAAFIAAAD//wAAUwAAAM4SAAAO +CAAANQAAAP//AABTAAAAPBQAAAgVAAA2AAAA//8AAFMAAACEFAAAZx0AADEAAAD//wAAUwAAANwV +AACjQgAAWwAAAP//AABTAAAAYBYAAH4IAAAgAAAA//8AAFMAAAC4FwAAglMAACkAAAD//wAAUwAA +ABMYAACCUwAAwwAAAP//AABUAAAAnAEAAExVAAA2AAAA//8AAFoAAACFAQAAKiQAAJECAAD//wAA +WwAAAJ4DAAC/BwAAHQAAAP//AABcAAAAJgAAALUNAAAYAAAA//8AAF4AAAAsAQAA/w4AANACAAD/ +/wAAYQAAALADAAA0PAAAKwAAAP//AABhAAAA8AMAAE5bAACMAAAA//8AAGEAAABWBAAAClsAAHMA +AAD//wAAYQAAAHsEAADUWgAAHwAAAP//AABhAAAAwQQAACJbAAAsAAAA//8AAGkAAAAvAAAAVmAA +AEkAAAD//wAAagAAAEIAAAA0PAAADgAAAP//AABvAAAAEAIAADADAADlAQAA//8AAG8AAACjAwAA +oVUAAD8AAAD//wAAcgAAACQBAADTRwAARgAAAP//AAByAAAAhgEAAPRiAABOAQAA//8AAHIAAACe +AQAA9GIAAPkAAAD//wAAcgAAALYDAAD0YgAAdwIAAP//AAB2AAAAMwMAAHtlAABLAAAA//8AAHYA +AAAEBAAACBUAABQAAAD//wAAdgAAAA4EAAAIFQAAGAAAAP//AAB2AAAALAQAAAgVAAA4AAAA//8A +AHcAAAD1AgAAUQgAABgAAAD//wAAdwAAAAADAAAkYgAAhgAAAP//AAB3AAAALwMAAGcdAABJAQAA +//8AAHcAAABXAwAAUQgAACMBAAD//wAAeAAAADMAAADCCgAAGQAAAP//AAB4AAAAigAAAMIKAAB5 +AAAA//8AAHgAAAATAgAA2gMAAMYAAAD//wAAMwAAAGkDAAD5GgAAOQAAAP//AAAzAAAAuwMAAKga +AAB3AAAA//8AADMAAADDBAAA2A0AAB0AAAD//wAAPwAAANIBAAD/DgAAKwAAAP//AABTAAAAXA8A +AAgVAAA+AAAA//8AAAAAAAABAAAARgkAAB0AAAD//wAAAAAAAAEAAADuDAAAFwAAAP//AAAAAAAA +AQAAACtWAAAVAAAA//8AAAAAAAABAAAAYAkAAB0AAAAGAAAACwAAAAAADACBAAEAAgAAAP7+AAj9 +/ggI/f3+/hAI/f4YCP39/wAAAAgIEAgYCCAIKAgwCDgIQAhICAAI/wAAAP4ACAgEDAQQCP0YCCAI +KAgwCDgI/wAAAAAICAgQBBgIIAEoCP4wCDgIQAj9/wAAAAAICAgQCP4YCCAIKAj9/jAIOAhACP3/ +AAgAAAAJAAAAAAALAAoACAAMABgAEAAAAAgAAAAJAAAAAAAAAAQABQAEAAwAegA6AAkAAAAQAAAA +AACAW+hbwVsAAQAAx1sAARAAAAAACP4ICBAI/f4YCCAI/f4oCDAI/f44CEAI/f8ACgAAAAoAAAAA +ACgArQKvAK8B7wCtAP8A/wEAAAoAAAAKAAAAAAAQAAEACQAIAKQCCgACAAAACAAVAAAAAwAAAAcD +AwMDAwADAwMDAwMDAwMDAwMDAAAAAAsAAAAMAAAAAACAA4gDwQNiA2ICEABAAAAAAAAEAAAADAAA +AAkAAAAAAEAAQAFEAVMBUwBRAGAAAAHgAIAACAAIAAAAEQAAAAAAAAIAAAoAAAwAAAEAABEAAAEA +AAAAAP//AAABAAAAmAAAADQAAAA/AAAA//8AAAEAAAChAAAANAAAAJsAAAD//wAACAAAAPoAAACP +BAAAGwEAAP//AAAIAAAA6AAAAI8EAAClAQAA//8AAAkAAABmAAAAjwQAAJMAAAD//wAACQAAAGEA +AACgBQAAHQEAAP//AAAJAAAA0gAAACUGAAAUAAAAAAAAACIAAABwAAAANAYAAB4AAAD//wAACQAA +AA4CAAAlBgAAFAAAAAAAAAAiAAAAcAAAADQGAAAeAAAA//8AAAkAAABtAgAAZAYAAF4AAAAAAAAA +CQAAAHoAAACPBAAAYgAAAP//AAAJAAAAmQIAACUGAAAmAAAAAAAAACIAAABwAAAANAYAACcAAAD/ +/wAAEgAAALcAAADmBgAAtAAAAP//AAASAAAAuAAAAH4IAADGAAAA//8AAB8AAABlAAAA+QoAAAwA +AAD//wAAHwAAAGcAAACPBAAAIwAAAP//AAAfAAAAogAAAPkKAAAAAAAA//8AAB8AAACkAAAAjwQA +ACoAAAD//wAAIAAAABsAAAAnDAAAIwAAAP//AAAgAAAAHAAAADsMAABDAAAA//8AACAAAAA+AAAA +JwwAABgAAAD//wAAIAAAAD4AAAA7DAAAGAAAAP//AAAiAAAAxAAAAL8HAABfAAAA//8AACIAAADS +AAAAvwcAAA4BAAD//wAAJQAAAB8DAABcDQAAIgAAAP//AAAlAAAAOwMAADMNAABwAAAA//8AACUA +AADrBAAAxg0AAEsAAAD//wAAJQAAAO8EAADYDgAASQAAAP//AAAmAAAABwQAAGgQAAAYAAAAAAAA +ACYAAAAvBAAAfxAAACIAAAD//wAAKwAAAGEBAAAaFAAAaQAAAP//AAArAAAAZwEAADgOAACVAAAA +//8AACsAAAAPBgAAaAUAAPMAAAD//wAAKwAAAA4GAABoBQAA6wAAAP//AAAsAAAAsgAAAL4WAADn +AAAA//8AACwAAADGAAAA2hYAAJ0BAAD//wAALQAAAOMAAACgFwAAbgAAAP//AAAtAAAA5wAAAA4X +AACIAAAA//8AAC4AAABAAAAA9hgAABQAAAAAAAAAMwAAAD4FAAASGQAAIwAAAP//AAAuAAAASgAA +AEAZAAAdAAAA//8AAC4AAABZAAAAzgQAADIAAAD//wAALwAAAEkAAAB9GQAARwAAAP//AAAvAAAA +TQAAAH0ZAABXAAAA//8AAC8AAACGAAAAMw0AACUAAAD//wAALwAAAIgAAAB9GQAAJgAAAP//AAAz +AAAAeAUAAEUbAABIAQAA//8AADMAAAClBQAA2hYAAMkBAAD//wAAMwAAAFIGAACLGwAAnAAAAP// +AAAzAAAARQYAAIsbAAAfAAAA//8AADUAAAC6AQAAIQ8AANMAAAD//wAANQAAALsBAAAhDwAA4wAA +AP//AAA1AAAA2gEAAMgiAADYAAAA//8AADUAAADcAQAAvwcAADUBAAD//wAANQAAACQCAAAkIwAA +gQAAAAAAAABqAAAAjgAAAP8OAADHAAAA//8AADUAAADDAgAA2yMAAC4BAAD//wAANQAAAOoCAADb +IwAAyAEAAP//AAA2AAAAlAAAAFEkAADNAAAAAAAAADYAAABnAAAAZiQAANUAAAD//wAANgAAAMwB +AABAJQAAXgAAAP//AAA2AAAA4gEAADMNAACTAAAA//8AADgAAACQAAAAwRcAAGMAAAD//wAAOAAA +AJEAAABvGAAAhQAAAP//AAA5AAAApQEAAGgjAAAUAAAAAAAAACAAAAAvAAAAOwwAADAAAAD//wAA +OgAAALwDAADOBAAAGQAAAP//AAA6AAAAwQMAAM4EAABxAAAA//8AADwAAABOAAAAMw0AAIUAAAD/ +/wAAPAAAAE8AAABcDQAAswAAAP//AAA+AAAAGwEAAMszAAB1AAAA//8AAD4AAAAfAQAA5ycAAKcA +AAD//wAAPgAAADwBAADnJwAAgQAAAP//AAA+AAAARgEAAOcnAADVAAAA//8AAD8AAACTAQAATDcA +ACIAAAAAAAAAPwAAAJsBAAD/DgAAKwAAAP//AABAAAAASwAAABAyAAAUAAAA//8AAEAAAABOAAAA +EDIAADEAAAD//wAAQwAAABsBAABoIwAAFAAAAAAAAAAgAAAALwAAADsMAAAwAAAA//8AAEQAAAAj +AwAA6gUAAEcAAAAAAAAAIgAAADAAAAD3BQAAUAAAAP//AABEAAAAOgMAACUGAAA/AAAAAAAAACIA +AABwAAAANAYAAEcAAAD//wAARQAAAMEAAAB/OwAANwAAAP//AABFAAAAtwAAAH87AACDAAAA//8A +AEkAAADIAAAACT0AACwAAAD//wAASQAAANAAAACPBAAAPgAAAP//AABOAAAAqAEAAGA/AAAXAAAA +//8AAE4AAACrAQAACBUAAEoAAAD//wAATgAAAN0BAABgPwAASwAAAP//AABOAAAA4QEAAAgVAABq +AAAA//8AAE4AAAC+BQAAUQgAABkAAAD//wAATgAAAMMFAABRCAAAJgAAAP//AABQAAAAPgEAAA4I +AAAYAAAA//8AAFAAAABAAQAADggAAEYAAAD//wAAUgAAAEcAAADqBQAANwAAAAAAAAAiAAAAMAAA +APcFAAA4AAAA//8AAFIAAABQAAAAJQYAAC0AAAAAAAAAIgAAAHAAAAA0BgAALgAAAP//AABSAAAA +YgAAAGJDAADdAAAAAAAAAH4AAAANAAAANDwAAN4AAAD//wAAUwAAAE4CAADhRQAAGQAAAP//AABT +AAAAUAIAAPRFAAA6AAAA//8AAFMAAAC/CAAA5gYAAEYAAAD//wAAUwAAAMEIAAAOCAAAagAAAP// +AABTAAAAOwoAAMtJAAAfAAAA//8AAFMAAABFCgAA00cAAEUAAAD//wAAUwAAACgMAAB3SwAAQgAA +AP//AABTAAAAKQwAAD9KAABlAAAA//8AAFMAAAChDQAAJQYAABUAAAAAAAAAIgAAAHAAAAA0BgAA +FAAAAP//AABTAAAA2Q0AABZDAAAdAAAA//8AAFMAAADeDQAACU0AAHYAAAD//wAAUwAAABwTAAB1 +UQAAEgEAAP//AABTAAAAHxMAAIdRAABZAQAA//8AAFMAAAAwFAAA00cAADoAAAD//wAAUwAAADEU +AAB+CAAARQAAAP//AABTAAAAGBcAAHVRAAAeAAAA//8AAFMAAAAZFwAAh1EAAHAAAAD//wAAUwAA +AFcXAADmBgAAIQAAAP//AABTAAAASRcAAIJTAACEAAAA//8AAFoAAADRAAAA71UAAEYAAAD//wAA +WgAAABwBAABwTwAAcAIAAP//AABhAAAAPQMAAE5bAABlAAAA//8AAGEAAABOAwAADggAAOMAAAD/ +/wAAYQAAACsEAADWPAAAEgAAAP//AABhAAAALgQAANY8AABTAAAA//8AAGEAAAA5BAAA4FwAABcA +AAD//wAAYQAAADoEAADWPAAARQAAAP//AABmAAAAOQAAAI8EAACdAAAA//8AAGYAAAAqAAAAnl0A +ABQBAAD//wAAZgAAAF0AAACeXQAAZAAAAP//AABmAAAAXwAAAMpdAACMAAAA//8AAGgAAACtAAAA +HV4AAAQAAAD//wAAaAAAALEAAAAdXgAAMgAAAP//AABvAAAA1QIAAFFhAAAOAAAA//8AAG8AAADf +AgAAjwQAAF0AAAD//wAAbwAAAH4DAACiYQAAIgAAAP//AABvAAAAfgMAAKFVAAAiAAAA//8AAG8A +AACZAwAA5GEAACAAAAD//wAAbwAAAJkDAAChVQAAIAAAAP//AABvAAAA3wMAAHRiAABFAAAAAAAA +AG8AAADYAwAAjwQAADQAAAD//wAAbwAAAPEDAAB0YgAARwAAAAAAAABvAAAA2AMAAI8EAAA2AAAA +//8AAG8AAAAMBAAAqmIAAEIAAAD//wAAbwAAABQEAACqYgAAmgAAAP//AAB3AAAAWgIAAI8EAAAU +AAAA//8AAHcAAABaAgAANwoAABQAAAD//wAAdwAAAPADAABRQgAAHQAAAP//AAB3AAAA9wMAAJw3 +AAB5AAAA//8AAHcAAABOBAAAUQgAACsAAAD//wAAdwAAAF4EAAAJPgAAdQAAAP//AAB3AAAAVwUA +AKFVAACVAAAA//8AAHcAAABdBQAAoVUAACMBAAD//wAAfAAAAMsAAACPBAAAOQAAAP//AAB8AAAA +wwAAAKFVAABPAAAA//8AAFMAAAAtFgAAZx0AAE0AAAD//wAAUwAAAC0WAAArVgAATQAAAAIAAACB +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGQAAAAoAAABQA0ABAAEAAQAA +QAFAAUABQAFAAUABQAFAAQAAQAFAAUABQAFAAUABQAFAAUABQAEAAAAA//8AAAYAAABcAgAA2gMA +ADcAAAD//wAABgAAAF0CAADwAwAAcAAAAP//AAAGAAAAXQIAAPADAABwAAAA//8AAAgAAACuAAAA +aAUAABwAAAD//wAACAAAAK8AAACPBAAAHQAAAP//AAAIAAAAtgAAAGgFAABTAAAA//8AABIAAACL +AAAA5gYAAPMAAAD//wAAEgAAAIwAAAB+CAAA/gAAAP//AAASAAAAmAAAAOYGAAAdAQAA//8AACUA +AAAhBQAA/w4AAF8AAAD//wAAJQAAACIFAAAQDwAAZwAAAAEAAAAZAAAAEgAAACEPAABvAAAA//8A +ACUAAADIBQAAMw0AABgAAAD//wAAJQAAAM0FAAAzDQAANgAAAP//AAAlAAAA0QUAALoPAAB3AAAA +//8AACYAAAA+AQAAaBAAAEoAAAD//wAAJgAAADkBAAD/DgAAZQAAAAAAAAAmAAAALwQAAH8QAABb +AAAA//8AACYAAADrBAAAqhIAACwAAAAAAAAAJgAAAGUEAACPBAAARQAAAAAAAAAmAAAAZgQAAAIR +AABOAAAA//8AACsAAADRAgAACBUAAEkAAAD//wAAKwAAANYCAABoBQAAkAAAAP//AAArAAAA3QIA +ABcDAACrAAAA//8AACsAAABTBwAA/g0AABoAAAD//wAAKwAAAFUHAAD+DQAAGQAAAP//AAArAAAA +VwcAAGgFAABKAAAA//8AADQAAACgAAAA2gMAAEwAAAD//wAANAAAAKUAAADaAwAAjAAAAP//AAA0 +AAAAxgAAAGcdAAATAQAA//8AADYAAAAYAgAAKSUAADcAAAAAAAAAQAAAACoAAABAJQAALgAAAAAA +AABAAAAALwAAAFklAAAwAAAA//8AADsAAADPAwAA5TIAABMAAAD//wAAOwAAANIDAADlMgAAxQAA +AP//AAA7AAAA8QMAAP4yAAAiAQAA//8AAD4AAACHAAAA5ycAAFQAAAD//wAAPgAAAJ4AAAD+MgAA +kwAAAP//AAA+AAAA1wAAAP4yAAAnAQAA//8AAEAAAADjAAAAkTgAAFsAAAAAAAAAQAAAADQAAAAQ +MgAAlgAAAAAAAABAAAAANAAAAEAlAACWAAAA//8AAEkAAAA+AQAAUz0AADUAAAD//wAASQAAAD8B +AABTPQAARgAAAP//AABJAAAAQAEAAFM9AABYAAAA//8AAFMAAABhAQAAtQ0AABwAAAD//wAAUwAA +AGMBAABnHQAANAAAAP//AABTAAAAbAEAANgNAACiAAAA//8AAFMAAABpCAAAKEkAAE0AAAD//wAA +UwAAAG8IAAA8SQAAiwAAAP//AABTAAAAXggAADxJAACoAAAA//8AAFMAAADaCAAAtQ0AABwAAAD/ +/wAAUwAAANwIAADYDQAARQAAAP//AABTAAAA4AgAANgNAACQAAAA//8AAFMAAAC1CwAAHEoAAGMA +AAD//wAAUwAAALgLAAC9RwAAbQAAAP//AABTAAAAuQsAAP4GAACOAAAA//8AAFMAAACtFgAASh8A +AHYAAAAAAAAAUwAAAHEYAABpHAAAdwAAAP//AABTAAAAsBYAAEofAADfAAAA//8AAFMAAAACFwAA +HEoAABgAAAD//wAAUwAAAAYXAAB1UQAAUwAAAP//AABTAAAACBcAANNHAAC4AAAA//8AAFoAAABJ +AAAACT0AAK8AAAAAAAAAWgAAADoAAACPBAAAfAAAAP//AABaAAAASQAAAKFVAACvAAAA//8AAGEA +AADjAQAAClsAAKkAAAD//wAAYQAAAPMBAAAiWwAAmgEAAP//AABhAAAA9AEAAApbAAClAQAA//8A +AGEAAABmBAAAOF0AAHYAAAD//wAAYQAAAGcEAABTPQAApgAAAP//AABhAAAAagQAANY8AACoAAAA +//8AAGgAAADcAAAATl4AAKUAAAD//wAAaAAAANUAAAA4DgAA/AAAAP//AABoAAAA1gAAAE5eAAAC +AQAA//8AAGgAAADqAAAAtgQAACEAAAAAAAAAOgAAAJoCAADOBAAAIgAAAP//AABoAAAA8gAAAE5e +AADAAAAA//8AAGgAAABdAgAAaAUAAGIAAAD//wAAaAAAAGECAACPBAAAngAAAP//AABoAAAAZAIA +AFEIAADYAAAA//8AAGgAAAB9BAAAZx0AACwAAAD//wAAaAAAAIYEAAByHwAAgQAAAP//AABoAAAA +lAQAAFEIAADMAAAA//8AAHYAAABmAgAAtQ0AABgAAAD//wAAdgAAAGoCAADqBQAATAAAAAEAAAAi +AAAAMAAAAPcFAABNAAAA//8AAHYAAABxAgAAJQYAABkAAAAAAAAAIgAAAHAAAAA0BgAAGgAAAP// +AAB2AAAAcwIAANgNAAAnAAAA//8AAHYAAACvAwAAMw0AABgAAAD//wAAdgAAALgDAAD3ZQAAbQAA +AP//AAB2AAAAuQMAAPdlAAB4AAAA//8AAHYAAADXAwAAtQ0AABwAAAD//wAAdgAAANkDAADTRwAA +PgAAAP//AAB2AAAA3AMAANgNAABvAAAA//8AAHYAAAByBAAAtQ0AABwAAAD//wAAdgAAAHQEAADT +RwAAPgAAAP//AAB2AAAAdwQAANgNAABvAAAA//8AAHcAAADaAwAAK1YAAHUAAAD//wAAdwAAAMsD +AABnHQAAGAAAAP//AAB3AAAA4AMAAL8HAADNAAAA//8AAHcAAAACBAAAZx0AACQAAAD//wAAdwAA +AAsEAABnHQAAiQAAAP//AAB3AAAADwQAAJw3AADRAAAA//8AAHwAAADYAAAA92sAADwAAAD//wAA +fAAAANkAAAAMbAAASAAAAP//AAB8AAAA3gAAAKFVAACaAAAADgAAAB0AAAAAAAAABQgAAIUAAAAF +AAAABAAAAAUABACHAAQAhQAEAEUABAA9AAAAHQAAABkAAAAJAAAAAQAAABUAAAAVAAAAAAAAAAAE +AAAGQAAEIAAEAAIEAAAAAAAAAIAFAIAHgGAEkHAEkHgEkHwEkGgEkGAGkGAEEEAEAAEEDAAEAwAE +AP//AAAPAAAAqQAAAA4IAAA8AQAA//8AAA8AAACqAAAADggAAEQBAAD//wAADwAAALMAAAAOCAAA +2QEAAP//AAAPAAAAtAAAAA4IAADhAQAA//8AABcAAADIAAAAtAkAAIYAAAD//wAAFwAAAMYAAAC0 +CQAApgUAAAEAAAAXAAAAsAAAAMkJAAA2AgAAAAAAABcAAACwAAAAyQkAANICAAD//wAAHAAAAEkA +AABPCgAAPwAAAAAAAAAcAAAAWAAAAFoKAABAAAAA//8AABwAAABKAAAAcgoAAAIAAAD//wAAHAAA +AEoAAAByCgAAAgAAAP//AAAcAAAATgAAACwKAABAAAAAAAAAABwAAABcAAAANwoAAEEAAAD//wAA +HAAAAE8AAAByCgAAAwAAAP//AAAcAAAATwAAAHIKAAADAAAA//8AACwAAABXAAAA6gUAAB0AAAAA +AAAAIgAAADAAAAD3BQAAHgAAAP//AAAsAAAAWgAAACUGAABLAAAAAgAAACIAAABwAAAANAYAAEwA +AAD//wAAMQAAAFkAAADqBQAAPwAAAAAAAAAiAAAAMAAAAPcFAABAAAAA//8AADEAAAB8AAAAJQYA +AJkBAAACAAAAIgAAAHAAAAA0BgAAmgEAAP//AAAxAAAAiwAAAOoFAAAVAAAAAAAAACIAAAAwAAAA +9wUAABQAAAD//wAAMQAAAJEAAAAlBgAAVQAAAAIAAAAiAAAAcAAAADQGAABWAAAA//8AADMAAAD0 +BQAA6gUAACUAAAAAAAAAIgAAADAAAAD3BQAAJgAAAP//AAAzAAAA9gUAACUGAABLAAAAAgAAACIA +AABwAAAANAYAAEwAAAD//wAANAAAAAoCAAC/BwAAbAAAAP//AAA0AAAALwIAAPYYAABoAQAAAQAA +ADMAAAA+BQAAEhkAAHQBAAD//wAANAAAADYCAAC/BwAA4AEAAP//AAA2AAAA/QEAACklAABXAAAA +AAAAAEAAAAAqAAAAQCUAAE4AAAAAAAAAQAAAAC8AAABZJQAAUAAAAP//AAA2AAAABQIAAH0ZAACC +AAAA//8AADoAAABfAgAAShQAAAEAAAAAAAAAOgAAAHwCAADOBAAAAAAAAP//AAA6AAAAYAIAADgO +AABNAAAA//8AADoAAABjAgAAGhQAAFgAAAD//wAAOgAAAKUCAABKFAAAAQAAAAAAAAA6AAAAfAIA +AM4EAAAAAAAA//8AADoAAACrAgAAOA4AAE0AAAD//wAAOgAAAKsCAAAaFAAATQAAAP//AAA6AAAA +lAUAAOoFAAAdAAAAAAAAACIAAAAwAAAA9wUAACgAAAD//wAAOgAAAJwFAAAlBgAAPwAAAAIAAAAi +AAAAcAAAADQGAABAAAAA//8AADoAAACtBQAA6gUAACYAAAAAAAAAIgAAADAAAAD3BQAAKAAAAP// +AAA6AAAArwUAACUGAABFAAAAAgAAACIAAABwAAAANAYAAEYAAAD//wAAOgAAAEMHAADqBQAAIwAA +AAAAAAAiAAAAMAAAAPcFAAAiAAAA//8AADoAAABFBwAAJQYAAEAAAAACAAAAIgAAAHAAAAA0BgAA +QQAAAP//AAA6AAAAEAgAAOoFAAAZAAAAAAAAACIAAAAwAAAA9wUAABgAAAD//wAAOgAAACAIAAAl +BgAAbAAAAAIAAAAiAAAAcAAAADQGAABtAAAA//8AADoAAAAoCAAAJQYAAEYAAAAAAAAAIgAAAHAA +AAA0BgAARwAAAP//AAA6AAAALQgAAOoFAABvAAAAAgAAACIAAAAwAAAA9wUAAHAAAAD//wAAPAAA +AHYAAAB9GQAAFAAAAP//AAA8AAAAdwAAADMNAAAfAAAA//8AADwAAAB6AAAAjwQAAFAAAAD//wAA +PAAAAHsAAACPBAAAVQAAAP//AAA/AAAA5gAAAGw2AAAlAAAAAAAAAD8AAAC2AAAAjwQAAGYBAAD/ +/wAAPwAAAPEAAABsNgAACgIAAAIAAAA/AAAAtgAAAI8EAAD0AQAA//8AAD8AAAAYAQAA6gUAABUA +AAAAAAAAIgAAADAAAAD3BQAAFAAAAP//AAA/AAAAHgEAACUGAABSAAAAAgAAACIAAABwAAAANAYA +AFMAAAD//wAAPwAAACkBAADqBQAAFQAAAAAAAAAiAAAAMAAAAPcFAAAUAAAA//8AAD8AAAAuAQAA +JQYAADoAAAACAAAAIgAAAHAAAAA0BgAAOwAAAP//AAA/AAAAVwEAAOoFAABvAAAAAAAAACIAAAAw +AAAA9wUAAHAAAAD//wAAPwAAAF4BAAAlBgAA/QAAAAIAAAAiAAAAcAAAADQGAAD+AAAA//8AAD8A +AABrAQAA6gUAACMAAAAAAAAAIgAAADAAAAD3BQAAIgAAAP//AAA/AAAAcQEAACUGAAB1AAAAAgAA +ACIAAABwAAAANAYAAHYAAAD//wAAPwAAAIQDAADqBQAAIwAAAAAAAAAiAAAAMAAAAPcFAAAiAAAA +//8AAD8AAACQAwAAJQYAABoBAAACAAAAIgAAAHAAAAA0BgAAGwEAAP//AAA/AAAAlAMAAOoFAAAZ +AAAAAAAAACIAAAAwAAAA9wUAABgAAAD//wAAPwAAAJ0DAAAlBgAAlQAAAAIAAAAiAAAAcAAAADQG +AACWAAAA//8AAEAAAAC7AAAAkTgAAEAAAAAAAAAAQAAAADQAAAAQMgAAbQAAAAAAAABAAAAANAAA +AEAlAABtAAAA//8AAEAAAADSAAAAQCUAAKUAAAD//wAAQAAAAD8BAAApJQAAPAAAAAAAAABAAAAA +KgAAAEAlAAAzAAAAAAAAAEAAAAAvAAAAWSUAADUAAAD//wAAQAAAAEEBAAAoOQAASAAAAP//AABH +AAAAfQAAAOoFAAAeAAAAAAAAACIAAAAwAAAA9wUAAB8AAAD//wAARwAAAIIAAAAlBgAASAAAAAIA +AAAiAAAAcAAAADQGAABJAAAA//8AAEkAAACXAAAA1jwAADIAAAD//wAASQAAAJgAAAAOCAAAGAAA +AP//AABJAAAAmQAAANY8AAC8AAAA//8AAEkAAACcAAAA6jwAABcBAAD//wAATgAAALABAADqBQAA +KgAAAAAAAAAiAAAAMAAAAPcFAAA3AAAA//8AAE4AAAC3AQAAJQYAAF0BAAACAAAAIgAAAHAAAAA0 +BgAAXgEAAP//AABOAAAA9QEAAOoFAADcAAAAAAAAACIAAAAwAAAA9wUAAN0AAAD//wAATgAAAPgB +AAAlBgAAMwEAAAIAAAAiAAAAcAAAADQGAAA0AQAA//8AAE4AAABHAgAAQUAAAFwAAAAAAAAATgAA +AIYBAACPBAAAYgAAAP//AABOAAAASQIAAEFAAABoAAAAAgAAAE4AAACGAQAAjwQAAGsAAAD//wAA +UAAAAAkBAABnHQAALQAAAP//AABQAAAACwEAALRCAACCAAAAAQAAAFMAAADuAgAAZx0AAJIAAAAB +AAAAUwAAAO8CAABnHQAAHgEAAP//AABTAAAAOAEAAL8HAABWAAAA//8AAFMAAAAtAQAA6gUAAD0A +AAABAAAAIgAAADAAAAD3BQAAZQAAAP//AABTAAAAMgEAAB0fAACKAAAA//8AAFMAAABCAgAA6gUA +AB4AAAAAAAAAIgAAADAAAAD3BQAAHQAAAP//AABTAAAARgIAACUGAAB0AAAAAgAAACIAAABwAAAA +NAYAAHUAAAD//wAAUwAAABcHAADqBQAASgAAAAAAAAAiAAAAMAAAAPcFAABLAAAA//8AAFMAAAAc +BwAAJQYAAHgAAAACAAAAIgAAAHAAAAA0BgAAeQAAAP//AABTAAAAkgcAADQ8AAAiAAAA//8AAFMA +AACfBwAAXEYAAH4AAAABAAAAYQAAAAwEAADWPAAAfwAAAP//AABTAAAAsQcAALlIAADfAAAA//8A +AFMAAAD8BwAA5gYAAN0AAAD//wAAUwAAAP0HAAB+CAAA6AAAAP//AABTAAAADQgAAH4IAAAtAQAA +//8AAFMAAAAPCAAAuUgAAEABAAD//wAAUwAAAGUJAADqBQAARwAAAAAAAAAiAAAAMAAAAPcFAABI +AAAA//8AAFMAAABnCQAAJQYAAGUAAAACAAAAIgAAAHAAAAA0BgAAZgAAAP//AABTAAAAWwoAAOoF +AABaAAAAAAAAACIAAAAwAAAA9wUAAFsAAAD//wAAUwAAAGEKAAAlBgAAlAAAAAIAAAAiAAAAcAAA +ADQGAACVAAAA//8AAFMAAAD6DgAA6gUAABUAAAAAAAAAIgAAADAAAAD3BQAAFAAAAP//AABTAAAA +/w4AACUGAABGAAAAAgAAACIAAABwAAAANAYAAEcAAAD//wAAUwAAANwPAADqBQAAGQAAAAAAAAAi +AAAAMAAAAPcFAAAYAAAA//8AAFMAAADiDwAAJQYAAFUAAAACAAAAIgAAAHAAAAA0BgAAVgAAAP// +AABTAAAAmRIAAA4IAACvAQAA//8AAFMAAACcEgAADggAANoBAAD//wAAUwAAAKASAAAOCAAADQIA +AP//AABTAAAAohIAAA4IAAAoAgAA//8AAFMAAABLFAAA6gUAABkAAAAAAAAAIgAAADAAAAD3BQAA +GAAAAP//AABTAAAAUBQAACUGAAA4AAAAAgAAACIAAABwAAAANAYAADkAAAD//wAAUwAAAOUVAAC/ +BwAAMgAAAP//AABTAAAA6hUAAOoFAABZAAAAAQAAACIAAAAwAAAA9wUAAFoAAAD//wAAUwAAAOsV +AADqPAAAZwAAAP//AABaAAAAUgAAAAk9AAAaAAAAAAAAAFoAAAA6AAAAjwQAACoAAAD//wAAWgAA +AFgAAAAJPQAAnQAAAAIAAABaAAAAOgAAAI8EAACpAAAA//8AAFwAAABGAAAA9wUAAEoAAAD//wAA +XAAAAEsAAAAlBgAAbwAAAAEAAAAiAAAAcAAAADQGAABwAAAA//8AAFwAAABOAAAA2A0AAHYAAAD/ +/wAAYQAAAIQAAAANWgAAiwAAAP//AABhAAAAhgAAABxaAAC7AAAA//8AAGEAAACMAAAAOFoAABMB +AAD//wAAYQAAAJIAAAAOCAAAkAEAAP//AABhAAAAqQEAAH1aAADDAAAAAAAAAF8AAAAqAAAAGFkA +ALoAAAABAAAAYAAAACoAAACpVwAAuwAAAP//AABhAAAAzwEAANRaAACxAQAA//8AAGgAAACmAgAA +KF8AAO8AAAD//wAAaAAAAMsCAAAoXwAAUgIAAP//AABoAAAAwQIAAB4gAAARAgAA//8AAGgAAADP +AgAAQyAAAGkCAAD//wAAbwAAACYDAAB5YQAAPwAAAP//AABvAAAANAMAAFEIAABtAAAA//8AAG8A +AABQAwAA/w4AAAUCAAD//wAAbwAAAE4DAAB5YQAAugEAAP//AAByAAAA9QMAAOoFAAAZAAAAAAAA +ACIAAAAwAAAA9wUAABgAAAD//wAAcgAAAAkEAAAlBgAAlwAAAAIAAAAiAAAAcAAAADQGAACYAAAA +//8AAHcAAAAoAAAAUQgAAMkAAAD//wAAdwAAAC0AAABBQAAA5wAAAAEAAABOAAAAhgEAAI8EAADp +AAAA//8AAHcAAAAvAAAARGcAAP0AAAD//wAAdwAAAGIDAADjQAAAWQAAAP//AAB3AAAAaQMAACRi +AABDAQAAAAAAAG8AAAD4AwAAjwQAAF8AAAAAAAAAbwAAAP0DAACPBAAAxwAAAP//AAA4AAAARAEA +AOoFAAAVAAAAAAAAACIAAAAwAAAA9wUAABQAAAD//wAAOAAAAEYBAAAlBgAAMQAAAAIAAAAiAAAA +cAAAADQGAAAyAAAA//8AAD8AAAD3AQAAtG4AAOwAAAD//wAAaAAAAEcFAAAzDQAATgEAAP//AABd +AAAAGQAAAA4IAAC7AQAA//8AAF0AAAAaAAAADggAANIBAAAKAAAARgAAAAAAAAAAAAAAAAEDAAAA +AAAAAAMDAAAAAAAAAJEDAAAAAAAAAJkDAAAAAAAAALEDAAAAAAAAAMEDAAAAAAAAAIUDAAAAAAAA +AOWzAAAAAAAAAOUDAAAAAAAAAAAA//8AAB8AAAD3AAAA6gUAABkAAAAAAAAAIgAAADAAAAD3BQAA +GAAAAP//AAAfAAAA+AAAANoDAAA4AAAA//8AAB8AAAD9AAAAJQYAAHkAAAADAAAAIgAAAHAAAAA0 +BgAAegAAAP//AAAmAAAAXAQAACESAAAUAAAAAAAAACYAAABWBAAAPxIAAD8AAAABAAAAJgAAAE4E +AADlEAAAJAAAAAEAAAAmAAAAUQQAAH8QAABGAAAA//8AACYAAABfBAAAlxEAAFYAAAD//wAAJwAA +AG0BAAAhEgAAFAAAAAAAAAAmAAAAVgQAAD8SAAA/AAAAAQAAACYAAABOBAAA5RAAACQAAAABAAAA +JgAAAFEEAAB/EAAARgAAAP//AAAnAAAAcAEAAJcRAABWAAAA//8AACsAAACUAgAACBUAAB0AAAD/ +/wAAKwAAAJECAABoBQAANgAAAP//AAArAAAAlwIAAGgFAABYAAAA//8AACsAAACiAgAAFwMAALQA +AAD//wAAKwAAAKcCAAAXAwAAJwEAAP//AAAsAAAAcwAAAOoFAAAvAAAAAAAAACIAAAAwAAAA9wUA +ADAAAAD//wAALAAAAHQAAABDDQAAPQAAAP//AAAsAAAAdQAAACUGAABpAAAAAwAAACIAAABwAAAA +NAYAAGoAAAD//wAALQAAAO4AAAC+FgAAGAAAAP//AAAtAAAA+AAAAHMUAABuAAAA//8AAC0AAAD6 +AAAA4QQAAOUAAAD//wAALQAAAPoAAAA4DgAA5QAAAAIAAAArAAAAPwEAAM4EAAChAAAA//8AADQA +AABDAgAA6gUAABUAAAAAAAAAIgAAADAAAAD3BQAAFAAAAP//AAA0AAAARAIAANMeAAA0AAAA//8A +ADQAAABGAgAAJQYAAEUAAAADAAAAIgAAAHAAAAA0BgAARgAAAP//AAA0AAAAsQUAAEoUAAAzAAAA +AAAAADoAAAB8AgAAzgQAADIAAAD//wAANAAAALcFAAA4DgAAyQAAAP//AAA0AAAAuAUAABoUAACS +AQAA//8AADQAAADABQAAGhQAACsCAAD//wAANgAAALEBAABRJAAAMQAAAAAAAAA2AAAAZwAAAGYk +AAA/AAAA//8AADYAAAC3AQAAmxsAACMBAAACAAAAUgAAAFAAAAAlBgAAOQEAAAMAAAAiAAAAcAAA +ADQGAAA6AQAA//8AADYAAADJAgAAQSYAADgAAAAAAAAAOwAAAFgBAABeJgAAIQAAAAAAAAA7AAAA +WAEAAHImAAAhAAAA//8AADYAAADMAgAAnSYAAGoAAAD//wAANgAAAM8CAABAJQAApgAAAP//AAA6 +AAAAOAUAADMNAAAtAAAA//8AADoAAAA+BQAAMw0AADwAAAD//wAAOgAAAGkFAAAzDQAAUAEAAP// +AAA6AAAAhwUAAFEkAACYAgAAAwAAADYAAABnAAAAZiQAAIACAAD//wAAPwAAAKgBAAB4NwAAoAAA +AP//AAA/AAAAqgEAAOoFAAC8AAAAAQAAACIAAAAwAAAA9wUAAL0AAAD//wAAPwAAALUBAAAlBgAA +ngEAAAMAAAAiAAAAcAAAADQGAACfAQAA//8AAD8AAABrAwAA6gUAACsAAAAAAAAAIgAAADAAAAD3 +BQAAKgAAAP//AAA/AAAAfAMAAJw3AABdAQAA//8AAD8AAACAAwAAJQYAAP8BAAADAAAAIgAAAHAA +AAA0BgAAAAIAAP//AABJAAAAqwEAAKk9AAAjAAAA//8AAEkAAACwAQAADggAAD0AAAD//wAASQAA +ALIBAAAOCAAAUgAAAP//AABJAAAAtAEAAA4IAABtAAAA//8AAEkAAAC2AQAADggAAIAAAAD//wAA +TgAAAMMBAABzPwAAKwAAAP//AABOAAAAwwEAAKsQAAArAAAAAQAAAEIAAAAQAAAA/g0AAFEAAAAB +AAAAQgAAABIAAAD+DQAAigAAAAEAAABCAAAAGAAAADMNAADFAAAA//8AAE4AAAD4AgAA40AAAFMA +AAAAAAAAbwAAAPgDAACPBAAAWQAAAAAAAABvAAAA/QMAAI8EAAC+AAAA//8AAE4AAAASAwAA9EAA +ACkBAAADAAAATgAAADwDAACPBAAAPwEAAP//AABTAAAAKQIAAOoFAAAtAAAAAAAAACIAAAAwAAAA +9wUAAC4AAAD//wAAUwAAACwCAAAwAwAA2QAAAP//AABTAAAALwIAACUGAAAUAQAAAwAAACIAAABw +AAAANAYAABUBAAD//wAAUwAAAGoCAAAJPQAAIQAAAAAAAABaAAAAOgAAAI8EAAAxAAAA//8AAFMA +AABvAgAACT0AAGwAAAACAAAAWgAAADoAAACPBAAAeAAAAP//AABTAAAAcgIAAAk+AADQAAAA//8A +AFMAAADLAwAAvwcAANsAAAD//wAAUwAAAM0DAAC/BwAA4gAAAP//AABTAAAA0wMAAL8HAAA+AQAA +//8AAFMAAADgAwAAvwcAAH0BAAD//wAAUwAAAPEDAAD5GgAA9QEAAP//AABTAAAAOggAAE9IAABt +AAAA//8AAFMAAAA+CAAAfggAAIMAAAD//wAAUwAAAEMIAAC5SAAAqgAAAP//AABTAAAARQgAAAJJ +AAC6AAAAAwAAAGEAAAAXBAAA1jwAAMQAAAD//wAAUwAAACsKAABnHQAAZQAAAP//AABTAAAALgoA +AGkcAADvAAAA//8AAFMAAAAuCgAAtEIAAO8AAAACAAAAUwAAAO4CAABnHQAAEwEAAAIAAABTAAAA +7wIAAGcdAACeAQAA//8AAFMAAAB3DQAAvwcAAFgAAAD//wAAUwAAAIMNAADqBQAAvQAAAAEAAAAi +AAAAMAAAAPcFAADJAAAA//8AAFMAAACbDQAAJQYAAIoBAAADAAAAIgAAAHAAAAA0BgAAiwEAAP// +AABTAAAArg0AAIZMAABkAAAAAAAAAFMAAABbDQAAlEwAAFkAAAABAAAAWwAAADUBAAB+CAAAZQAA +AAAAAABTAAAAXA0AAKVMAABsAAAAAwAAAFsAAAAUAQAA5gYAAHAAAAD//wAAUwAAAFAOAACGTAAA +sgEAAAAAAABTAAAAWw0AAJRMAACnAQAAAQAAAFsAAAA1AQAAfggAALMBAAAAAAAAUwAAAFwNAACl +TAAAugEAAAMAAABbAAAAFAEAAOYGAAC+AQAA//8AAFMAAAAEDwAACBUAABwAAAD//wAAUwAAAAYP +AADqBQAAIwAAAAEAAAAiAAAAMAAAAPcFAAApAAAA//8AAFMAAAARDwAAJQYAAKcAAAADAAAAIgAA +AHAAAAA0BgAAqAAAAP//AABTAAAAWBMAAEMNAAAiAAAA//8AAFMAAABbEwAA6gUAAHoAAAABAAAA +IgAAADAAAAD3BQAAewAAAP//AABTAAAAXRMAACUGAACdAAAAAwAAACIAAABwAAAANAYAAJ4AAAD/ +/wAAUwAAAO8WAADqBQAALQAAAAAAAAAiAAAAMAAAAPcFAAA5AAAA//8AAFMAAADxFgAAh1EAAFQA +AAD//wAAUwAAAPMWAAAlBgAAnwAAAAMAAAAiAAAAcAAAADQGAACgAAAA//8AAFQAAADwAAAAOVQA +AA8AAAD//wAAVAAAAPMAAABUVAAAFAAAAP//AABUAAAA8wAAAFRUAAAUAAAA//8AAFQAAADzAAAA +b1QAABQAAAD//wAAVAAAAPgAAABvVAAAOgAAAP//AABUAAAACgEAADlUAAAPAAAA//8AAFQAAAAN +AQAAVFQAABQAAAD//wAAVAAAAA0BAABUVAAAFAAAAP//AABUAAAADQEAAG9UAAAUAAAA//8AAFQA +AAASAQAAb1QAAD4AAAD//wAAaAAAABkBAADqBQAAGAAAAAAAAAAiAAAAMAAAAPcFAABVAAAA//8A +AGgAAAAcAQAATl4AAGgAAAD//wAAaAAAACABAAAlBgAAqAAAAAMAAAAiAAAAcAAAADQGAACpAAAA +//8AAGgAAAAsAQAA6gUAAFkAAAAAAAAAIgAAADAAAAD3BQAAgAAAAP//AABoAAAALgEAAE5eAACF +AAAA//8AAGgAAAAzAQAAJQYAAMMAAAADAAAAIgAAAHAAAAA0BgAAxAAAAP//AABoAAAARwEAACUG +AAA6AAAAAAAAACIAAABwAAAANAYAADsAAAD//wAAaAAAAD4BAADqBQAAWwAAAAIAAAAiAAAAMAAA +APcFAAB3AAAA//8AAGgAAABAAQAATl4AAJoAAAD//wAAaAAAAC8DAAD3BQAAYgAAAP//AABoAAAA +NQMAAGxfAACDAAAAAQAAAGgAAAAFAwAAKF8AAJYAAAD//wAAaAAAAEYDAAAlBgAAIwEAAAMAAAAi +AAAAcAAAADQGAAA4AQAA//8AAHIAAABKAwAA9GIAAMkAAAD//wAAcgAAAFwDAAAlBgAAJQEAAAEA +AAAiAAAAcAAAADQGAAA4AQAA//8AAHIAAABgAwAA6gUAAFoBAAADAAAAIgAAADAAAAD3BQAAWwEA +AP//AAB3AAAAlAMAAFFCAAAiAAAA//8AAHcAAACaAwAAUQgAAEwAAAD//wAAdwAAAJ4DAABwZwAA +VgAAAP//AAB3AAAArQMAAAk+AADJAAAA//8AAHcAAACtAwAA6GgAAMkAAAD//wAAeAAAAN0BAADf +agAAGAAAAAAAAAB4AAAA0QEAAPdqAABDAAAAAQAAAHgAAADHAQAAjwQAAE4AAAD//wAAeAAAAOIB +AAD3agAAlwAAAAMAAAB4AAAAxwEAAI8EAACOAAAA//8AAAMAAABXAAAAXwAAAFMDAAD//wAAAwAA +AF0AAABfAAAAYAMAAP//AAADAAAAZAAAAF8AAACPAwAA//8AAAMAAABkAAAAXwAAAI8DAAD//wAA +AwAAAGcAAABfAAAAqQMAAP//AAADAAAAbwAAAF8AAACcAwAA//8AACsAAACBAQAAShQAAA8AAAAA +AAAAOgAAAHwCAADOBAAADgAAAP//AAArAAAAkQEAABoUAABtAAAA//8AACsAAACRAQAAOA4AAG0A +AAD//wAAKwAAAJ4BAABZFAAAzwAAAAQAAAArAAAA9wAAAHMUAADUAAAA//8AADQAAABEAAAA2gMA +ADEAAAD//wAANAAAAEUAAAARHQAASQAAAAEAAAA0AAAAPQAAAP4NAABXAAAA//8AADQAAABLAAAA +2gMAAIwAAAD//wAANAAAAEwAAAARHQAAogAAAAQAAAA0AAAAPQAAAP4NAACwAAAA//8AADQAAABm +AQAA6gUAAJcBAAAAAAAAIgAAADAAAAD3BQAAoAEAAP//AAA0AAAAeQEAACUGAAC0AQAAAgAAACIA +AABwAAAANAYAAOkBAAD//wAANAAAAG8BAAA4DgAA/AEAAP//AAA0AAAAWQEAABoUAABVAQAA//8A +ADQAAACSAQAAMh4AAEMAAAD//wAANAAAAJMBAAAyHgAAagAAAP//AAA0AAAAuwEAAEoeAAArAQAA +//8AADQAAADUAQAA0xoAAPIBAAD//wAANAAAALMBAABpHgAALgIAAP//AAA0AAAA6QEAAGkeAABa +AgAA//8AADYAAAAAAQAA6gUAAD0AAAAAAAAAIgAAADAAAAD3BQAAPgAAAP//AAA2AAAACQEAAB0f +AACqAAAA//8AADYAAAA4AQAA6gUAAFACAAADAAAAIgAAADAAAAD3BQAAUQIAAP//AAA2AAAAOgEA +AB0fAABlAgAA//8AADYAAACYAQAAKSUAAJwAAAAAAAAAQAAAACoAAABAJQAAkwAAAAAAAABAAAAA +LwAAAFklAACVAAAA//8AADYAAACZAQAAKSUAAKgAAAADAAAAQAAAACoAAABAJQAAwwAAAAMAAABA +AAAALwAAAFklAADFAAAA//8AADYAAAB+AgAAJCcAAEAAAAD//wAANgAAAH4CAAAYJgAAQAAAAP// +AAA2AAAAfgIAABgmAABAAAAA//8AADYAAACGAgAAKyYAAGkAAAD//wAANgAAAJACAABeJgAAqQAA +AP//AAA2AAAAkQIAAHImAADAAAAA//8AADYAAAAJAwAAdicAAE8AAAD//wAANgAAAAsDAAB2JwAA +gQAAAP//AAA2AAAADQMAAHYnAADAAAAA//8AADYAAAAPAwAAdicAAPIAAAD//wAANgAAABEDAAB2 +JwAAHQEAAP//AAA2AAAAEwMAAHYnAAAmAQAA//8AADYAAABnAwAAMw0AAEoAAAD//wAANgAAAHwD +AADnJwAA/gAAAP//AAA2AAAAgQMAAOcnAABkAQAA//8AADYAAACJAwAA5ycAALIBAAD//wAANgAA +AKYDAAAzDQAANgIAAP//AAA2AAAArAMAAH0ZAABKAgAA//8AADoAAADtAgAAtQ0AADkAAAD//wAA +OgAAACIDAADYDQAA3QEAAP//AAA6AAAADQMAAOoFAAAqAQAAAgAAACIAAAAwAAAA9wUAACsBAAD/ +/wAAOgAAABwDAAAlBgAAvAEAAAQAAAAiAAAAcAAAADQGAAC9AQAA//8AAEAAAAA8AAAAEDIAABgA +AAD//wAAQAAAADwAAAAQMgAAGAAAAP//AABAAAAAPgAAAEAlAABRAAAA//8AAEAAAABAAAAAQCUA +AG8AAAD//wAAQAAAAEAAAABAJQAAbwAAAP//AABAAAAAQgAAAEAlAACFAAAA//8AAEMAAADoAAAA +tDkAABgAAAAAAAAAQwAAAD0BAADQOQAAHAAAAP//AABDAAAA9AAAAI8EAAA1AAAA//8AAEMAAAAJ +AQAAQDoAAF4AAAD//wAAQwAAAAwBAAB7OgAAdQAAAP//AABDAAAA6AAAAOs5AAAYAAAA//8AAEcA +AAB1AQAA9DsAANMAAAD//wAARwAAAHgBAAD0OwAAkgAAAP//AABHAAAAewEAANAGAAAzAAAAAgAA +AFMAAACQGAAA5gYAAD4AAAD//wAARwAAAH4BAADQBgAARwAAAAQAAABTAAAAkBgAAOYGAABSAAAA +//8AAE4AAABuAQAAYD8AAE4AAAD//wAATgAAAHIBAABzPwAAdQAAAP//AABOAAAAcgEAAKsQAAB1 +AAAAAgAAAEIAAAAQAAAA/g0AAKAAAAACAAAAQgAAABIAAAD+DQAA5QAAAAIAAABCAAAAGAAAADMN +AAA6AQAA//8AAFMAAAC5AAAAgkQAAIwAAAAAAAAAUwAAAOwRAACXRAAAlwAAAAEAAABTAAAAxREA +AOYGAACTAAAAAQAAAFMAAADGEQAAfggAAKIAAAD//wAAUwAAAMIAAAC/BwAA5gAAAP//AABTAAAA +DgEAANMaAACYAgAA//8AAFMAAABEAwAAZx0AADkAAAD//wAAUwAAAEgDAAC1DQAARAAAAP//AABT +AAAAUgMAANgNAACVAAAA//8AAFMAAABKAwAAtEIAAMsAAAADAAAAUwAAAO4CAABnHQAA2wAAAAMA +AABTAAAA7wIAAGcdAABeAQAA//8AAFMAAACUAwAAtEIAAM8AAAAAAAAAUwAAAO4CAABnHQAA5AAA +AAAAAABTAAAA7wIAAGcdAABpAQAA//8AAFMAAACHAwAAtEIAAFoCAAADAAAAUwAAAO4CAABnHQAA +bwIAAAMAAABTAAAA7wIAAGcdAAD+AgAA//8AAFMAAAA1BwAAtQ0AADAAAAD//wAAUwAAAD0HAADq +BQAAYwAAAAEAAAAiAAAAMAAAAPcFAABkAAAA//8AAFMAAABjBwAA2A0AAGoBAAD//wAAUwAAAFAH +AAAlBgAAawIAAAQAAAAiAAAAcAAAADQGAABsAgAA//8AAFMAAACbCAAA00cAADoAAAD//wAAUwAA +AKkIAADqBQAAfAAAAAEAAAAiAAAAMAAAAPcFAAB9AAAA//8AAFMAAACuCAAAfggAAKYAAAD//wAA +UwAAALMIAAAlBgAAygAAAAQAAAAiAAAAcAAAADQGAADLAAAA//8AAFMAAAATDAAAd0sAAFQAAAD/ +/wAAUwAAABMMAAAcSgAAVAAAAP//AABTAAAAFAwAAOoFAACnAAAAAgAAACIAAAAwAAAA9wUAAKgA +AAD//wAAUwAAABYMAAAlBgAAvwAAAAQAAAAiAAAAcAAAADQGAADAAAAA//8AAHYAAADiAQAA6gUA +ADYAAAAAAAAAIgAAADAAAAD3BQAANwAAAP//AAB2AAAA4wEAAFRkAABNAAAAAgAAAHYAAADpAQAA +a2QAAE4AAAD//wAAdgAAAOQBAAAlBgAAeQAAAAQAAAAiAAAAcAAAADQGAAB6AAAA//8AAHgAAADK +AAAAa2oAAHwAAAAAAAAAeAAAAKwAAADqBQAAfQAAAAEAAAAiAAAAMAAAAPcFAAB+AAAA//8AAHgA +AADMAAAAg2oAALEAAAADAAAAeAAAALYAAAAlBgAAsgAAAAQAAAAiAAAAcAAAADQGAACzAAAA//8A +AHgAAADqAAAAa2oAAM0AAAAAAAAAeAAAAKwAAADqBQAAzgAAAAEAAAAiAAAAMAAAAPcFAADPAAAA +//8AAHgAAADsAAAAg2oAAPsAAAADAAAAeAAAALYAAAAlBgAA/AAAAAQAAAAiAAAAcAAAADQGAAD9 +AAAA//8AABIAAACeAAAAaRwAABgAAAD//wAAEgAAAKcAAACGTAAAcgAAAAEAAABTAAAAWw0AAJRM +AABnAAAAAgAAAFsAAAA1AQAAfggAAHMAAAABAAAAUwAAAFwNAAClTAAAegAAAAQAAABbAAAAFAEA +AOYGAAB+AAAA//8AADwAAABuAAAAJCcAAA8AAAD//wAAPAAAAG4AAAD8MAAADgAAAP//AAA8AAAA +bwAAAPJtAACHAAAA//8AADwAAABuAAAAJCcAABAAAAACAAAAOwAAAKwAAAB9GQAAdQAAAAIAAAA7 +AAAArAAAADMNAAB2AAAAGQAAACMAAAAAAAAAAABgwAMEAABAAwQAAAAAAAAAAAAAAADIAwQAVMgD +BABUzgMEAFTPAwSAXMADBIFcwAMEhFzAAwSAWMADBAAAQAAAiFzAAwSI3MADBJjcwAMEmN7AAwSI +3sADBKBcwAMEoF3AAwSAXcADBIBZwAMEglzAAwQAAAEAAAAAAP//AAAfAAAAOwAAAOoFAAB2AAAA +AAAAACIAAAAwAAAA9wUAAHcAAAD//wAAHwAAAD0AAAAlBgAAqgAAAAIAAAAiAAAAcAAAADQGAACr +AAAA//8AAB8AAABNAAAAJQYAAEUBAAAEAAAAIgAAAHAAAAA0BgAARgEAAP//AAAfAAAAKwAAAMIK +AAAVAgAA//8AAB8AAADKAAAAjwQAAFQAAAD//wAAHwAAANEAAABQCwAA9AAAAP//AAAfAAAA0gAA +AMIKAAAKAQAA//8AAB8AAADaAAAAwgoAALMAAAD//wAAHwAAANsAAABQCwAAXAIAAP//AAAfAAAA +3gAAAMIKAAC7AAAA//8AAB8AAADgAAAAaQsAAF8BAAD//wAAMwAAAP0EAADqBQAA1wAAAAAAAAAi +AAAAMAAAAPcFAADYAAAA//8AADMAAAD+BAAAMhwAAOUAAAACAAAAUwAAAJIWAABLHAAA5gAAAAMA +AABTAAAAZRgAAGkcAAD4AAAA//8AADMAAAD/BAAAJQYAADYBAAAFAAAAIgAAAHAAAAA0BgAANwEA +AP//AAA0AAAAXQUAAFkUAAAJAQAAAAAAACsAAAD3AAAAOA4AAA4BAAAAAAAAKwAAAPcAAABzFAAA +DgEAAP//AAA0AAAAXgUAAD4hAABMAQAAAwAAACsAAADhAAAAViEAACsBAAAEAAAAOgAAAJgHAADQ +EwAALAEAAAUAAAA6AAAAkgcAAGgFAABIAQAA//8AADYAAAC6AAAA6gUAABkAAAAAAAAAIgAAADAA +AAD3BQAAGAAAAP//AAA2AAAAxQAAAJckAAA5AAAA//8AADYAAADWAAAA0AYAAFUAAAADAAAAUwAA +AJAYAADmBgAAZwAAAP//AAA2AAAA2QAAACUGAAB3AAAABQAAACIAAABwAAAANAYAAHgAAAD//wAA +NgAAAOUAAADqBQAAHgAAAAAAAAAiAAAAMAAAAPcFAAAdAAAA//8AADYAAADsAAAAvwcAACoAAAD/ +/wAANgAAAO0AAAC8JAAAPQAAAAMAAAByAAAA5QAAAMwkAABWAAAA//8AADYAAADxAAAAHR8AAHoA +AAD//wAANgAAAPQAAAC/BwAAtgAAAP//AAA4AAAACgEAAIkXAAA3AAAA//8AADgAAAAVAQAAGhQA +AHUAAAD//wAAOAAAAB8BAADkFwAAowAAAAIAAAA4AAAA2QAAAAYYAAClAAAA//8AADgAAAAxAQAA +LRgAAEoBAAAEAAAAOAAAAPIAAABMGAAAiAEAAP//AAA4AAAASwEAADQpAACVAgAA//8AADgAAABq +AQAAiRcAAEAAAAD//wAAOAAAAGwBAADkFwAAWwAAAAEAAAA4AAAA2QAAAAYYAABdAAAA//8AADgA +AABxAQAALRgAALcAAAADAAAAOAAAAPIAAABMGAAA+wAAAP//AAA4AAAAbgEAAC0YAADxAQAABQAA +ADgAAADyAAAATBgAAC8CAAD//wAAOAAAAJQCAAA/KgAAxwAAAP//AAA4AAAAlQIAAGAqAADeAAAA +//8AADgAAACxAgAAsCoAAKcCAAD//wAAOAAAAJkCAABAGQAAPwEAAP//AAA4AAAAnwIAAEAZAACr +AQAA//8AADgAAACkAgAAQBkAAPsBAAD//wAAOAAAALACAACwKgAAhQIAAP//AAA5AAAAvwEAAOoF +AAAZAAAAAAAAACIAAAAwAAAA9wUAABgAAAD//wAAOQAAAMcBAAB1LAAA1QAAAAIAAAA6AAAAZAYA +AJIsAABIAAAA//8AADkAAADIAQAAJQYAAEoAAAAEAAAAIgAAAHAAAAA0BgAASwAAAAIAAAA6AAAA +bgYAAJIsAACDAAAA//8AADoAAADAAgAAMy0AABoAAAD//wAAOgAAAMECAAAzLQAAtQAAAP//AAA6 +AAAAwgIAADMtAABDAQAA//8AADoAAADDAgAAMy0AAM4BAAD//wAAOgAAAMQCAAAzLQAATAIAAP// +AAA6AAAAxQIAADMtAADMAgAA//8AADoAAADTAgAATC0AAFcDAAD//wAAOwAAAIECAAAQMgAAHAAA +AP//AAA7AAAAgQIAANwxAAAcAAAA//8AADsAAACBAgAAEDIAABwAAAD//wAAOwAAAIUCAADcMQAA +XQAAAP//AAA7AAAAhgIAANwxAAB3AAAA//8AADsAAACGAgAAQCUAAHcAAAD//wAAOwAAAIYCAABA +JQAAdwAAAP//AABAAAAABAEAACklAAAYAAAAAAAAAEAAAAAqAAAAQCUAACwAAAAAAAAAQAAAAC8A +AABZJQAALgAAAP//AABAAAAACwEAAPE4AABlAAAA//8AAEAAAAAMAQAA8TgAADcAAAD//wAAQAAA +ADQBAAApJQAAFwQAAAUAAABAAAAALwAAAFklAAATBAAA//8AAFMAAACHAQAAtQ0AABIAAAD//wAA +UwAAAIgBAAAIFQAAHQAAAP//AABTAAAAigEAAOoFAAA+AAAAAgAAACIAAAAwAAAA9wUAAGAAAAD/ +/wAAUwAAAJ8BAADYDQAAzAAAAP//AABTAAAAkgEAACUGAAAlAgAABQAAACIAAABwAAAANAYAACYC +AAD//wAAUwAAALsBAAC1DQAAbwAAAP//AABTAAAAvAEAAAgVAACAAAAA//8AAFMAAADSAQAA2A0A +ADwBAAD//wAAUwAAAMwBAADqBQAALwIAAAMAAAAiAAAAMAAAAPcFAAAwAgAA//8AAFMAAADPAQAA +JQYAAIUCAAAFAAAAIgAAAHAAAAA0BgAAoQIAAP//AABTAAAAFQMAAOoFAABJAAAAAAAAACIAAAAw +AAAA9wUAAEoAAAD//wAAUwAAAB0DAAC2RgAAhAAAAP//AABTAAAAHgMAALZGAACwAAAA//8AAFMA +AAAuAwAAMAMAAEkBAAD//wAAUwAAAC8DAAAlBgAAbgEAAAUAAAAiAAAAcAAAADQGAABvAQAA//8A +AFMAAAAcDgAAMk0AACoAAAD//wAAUwAAAB4OAAAIFQAATwAAAP//AABTAAAAIA4AAIZMAAB+AAAA +AgAAAFMAAABbDQAAlEwAAHMAAAADAAAAWwAAADUBAAB+CAAAfwAAAAIAAABTAAAAXA0AAKVMAACG +AAAABQAAAFsAAAAUAQAA5gYAAIoAAAD//wAAUwAAAMIQAAC1DQAANQAAAP//AABTAAAA/xAAAMlP +AAA+AQAAAQAAAGgAAABfBAAA308AAEcBAAD//wAAUwAAAM4QAAAIFQAAawAAAP//AABTAAAA2RAA +AGcdAADFAAAA//8AAFMAAAAKEQAA/w4AAFgCAAD//wAAUwAAACARAADYDQAA7gIAAP//AABTAAAA +WRkAAL8HAACfAAAA//8AAFMAAABeGQAAjwQAAMcAAAD//wAAUwAAAGAZAACPBAAAGAEAAP//AABT +AAAAahkAAA4IAAC2AQAA//8AAFMAAABmGQAAvwcAAGYBAAD//wAAUwAAAHAZAADJCQAAxAAAAP// +AABTAAAAcRkAAMkJAAA+AQAA//8AAFwAAAAqAAAA9wUAACEAAAD//wAAXAAAAC4AAAAlBgAAOQAA +AAEAAAAiAAAAcAAAADQGAAA6AAAA//8AAFwAAAA0AAAAfggAAFsAAAD//wAAXAAAADUAAAAlBgAA +YAAAAAQAAAAiAAAAcAAAADQGAABhAAAA//8AAFwAAAA3AAAA7BsAAHgAAAD//wAAaAAAAOsCAAAo +XwAAGAAAAP//AABoAAAA7QIAAChfAABJAAAA//8AAGgAAADuAgAAKF8AAGQAAAD//wAAaAAAAO8C +AAAoXwAAfwAAAP//AABoAAAA8AIAAChfAACaAAAA//8AAGgAAADxAgAAKF8AALYAAAD//wAAaAAA +APICAAAoXwAA1gAAAP//AABoAAAAtwMAAGkcAAAtAAAA//8AAGgAAADwAwAAFkMAAOkAAAD//wAA +aAAAACkEAABRCAAA2QMAAP//AABoAAAABgQAAGcdAABABgAA//8AAGgAAADKAwAAUQgAAKMHAAD/ +/wAAaAAAANQDAACcNwAA7gkAAP//AABoAAAAvQMAAJw3AABaCwAA//8AAHYAAAAXAwAA6gUAAHQA +AAAAAAAAIgAAADAAAAD3BQAAewAAAP//AAB2AAAAIgMAAHtlAADYAAAA//8AAHYAAAAZAwAAJQYA +APoAAAADAAAAIgAAAHAAAAA0BgAA+wAAAP//AAB2AAAAKQMAACUGAABlAQAABQAAACIAAABwAAAA +NAYAAGYBAAD//wAAdwAAAFQCAADjQAAAIgAAAAAAAABvAAAA+AMAAI8EAAAwAAAAAAAAAG8AAAD9 +AwAAjwQAAJkAAAD//wAAdwAAAIACAACeZwAAEAEAAP//AAB3AAAAfQIAAJ5nAABqAQAA//8AAHcA +AAB2AgAAnmcAAMYBAAD//wAAdwAAAIMCAACeZwAALwIAAP//AAB4AAAACQEAAA4IAAA2AAAA//8A +AHgAAAAUAQAAa2oAAKEAAAABAAAAeAAAAKwAAADqBQAAogAAAAIAAAAiAAAAMAAAAPcFAACjAAAA +//8AAHgAAAAWAQAAg2oAAM8AAAAEAAAAeAAAALYAAAAlBgAA0AAAAAUAAAAiAAAAcAAAADQGAADR +AAAA//8AACUAAAB5AgAAMw0AADAAAAD//wAAJQAAANsCAADOBAAAdwAAAP//AAAlAAAA2wIAAM4E +AAB3AAAA//8AACUAAACMAgAAzgQAAI8DAAD//wAAJQAAAKUCAABDDQAAEgMAAP//AAAlAAAAkAIA +AFwNAACmAwAA//8AACUAAADIAgAAzgQAAFAFAAD//wAAJQAAAMoCAADOBAAAcAUAAP//AAAmAAAA ++wAAAAQQAABPAAAAAAAAACYAAADQAAAAjwQAADkAAAD//wAAJgAAAAIBAAAdEAAAUQAAAP//AAAm +AAAA/QAAAI8EAACVAAAA//8AACYAAAAGAQAAoA4AAK8AAAD//wAAJgAAAAoBAAA5EAAA6wAAAP// +AAAmAAAADQEAAB0QAAAMAgAABgAAACYAAADUAAAAjwQAACcBAAD//wAAKwAAAHsCAADhBAAAmQAA +AAAAAAArAAAAPwEAAM4EAABRAAAA//8AACsAAAB6AgAACBUAADoAAAD//wAAKwAAAH0CAAAdFQAA +4AAAAAMAAAArAAAAEQIAAPkEAADaAAAA//8AACsAAAB/AgAAFwMAAO0AAAD//wAAKwAAAIMCAAAP +BQAATgEAAAYAAAArAAAArQEAACUFAAB5AQAA//8AACwAAADiAAAAEQ4AAGYAAAAAAAAAOgAAABsC +AAAnDgAAMgAAAP//AAAsAAAA8AAAANoWAADjAAAA//8AACwAAAD4AAAADhcAAC4BAAD//wAALAAA +APkAAAA4DgAAfAEAAP//AAAsAAAA+gAAAOEEAADZAQAA//8AACwAAAD6AAAAOA4AANkBAAAFAAAA +KwAAAD8BAADOBAAAlQEAAP//AAAzAAAAFQYAAOoFAAAqAAAAAAAAACIAAAAwAAAA9wUAACsAAAD/ +/wAAMwAAABwGAAAlBgAAkwAAAAIAAAAiAAAAcAAAADQGAACUAAAA//8AADMAAAAgBgAA6gUAAKUA +AAAEAAAAIgAAADAAAAD3BQAAtAAAAP//AAAzAAAAKwYAACUGAADsAAAABgAAACIAAABwAAAANAYA +AO0AAAD//wAANQAAADkCAAD2GAAAUQAAAAAAAAAzAAAAOwUAAEUbAAAmAAAAAAAAADMAAAA+BQAA +EhkAAF0AAAD//wAANQAAAEICAABoIwAAUAEAAAMAAAAgAAAALwAAADsMAACwAAAA//8AADUAAABf +AgAAfyMAAJIAAAD//wAANQAAAGwCAAC/BwAAawEAAP//AAA1AAAAdwIAAGkcAADYAQAA//8AADkA +AADPAQAA6gUAAB0AAAAAAAAAIgAAADAAAAD3BQAAHAAAAP//AAA5AAAA0AEAAJIsAAApAAAA//8A +ADkAAADRAQAAJQYAADwAAAADAAAAIgAAAHAAAAA0BgAAPQAAAP//AAA5AAAA4AEAACUGAACUAAAA +//8AADkAAADfAQAAkiwAALYAAAAFAAAAIgAAAHAAAAA0BgAAlQAAAP//AAA7AAAAZAEAADMNAAAn +AAAA//8AADsAAABlAQAAfRkAACgAAAD//wAAOwAAAG8BAAAYJgAAYAAAAP//AAA7AAAAbwEAABgm +AABhAAAA//8AADsAAAB+AQAAQCUAAOUAAAD//wAAOwAAAJMBAABBJgAAOQEAAAUAAAA7AAAAWAEA +AHImAAAnAQAA//8AADsAAACIAQAAXiYAAGMBAAD//wAAQwAAAF8AAAAlBgAATQAAAAAAAAAiAAAA +cAAAADQGAABOAAAA//8AAEMAAABaAAAA6gUAAHAAAAACAAAAIgAAADAAAAD3BQAAcQAAAP//AABD +AAAAfwAAAI8EAAARAQAA//8AAEMAAACDAAAAJQYAADIBAAAFAAAAIgAAAHAAAAA0BgAARAEAAP// +AABDAAAAVQAAAI8EAABKAQAA//8AAE4AAABdBQAAQUIAAEAAAAD//wAATgAAAGYFAABRQgAAoAEA +AP//AABOAAAAeQUAACUGAACBAgAAAgAAACIAAABwAAAANAYAAIICAAD//wAATgAAAIAFAADqBQAA +pQIAAAQAAAAiAAAAMAAAAPcFAACmAgAA//8AAE4AAACBBQAA6gUAALMCAAAGAAAAIgAAADAAAAD3 +BQAAtAIAAP//AABTAAAA/AgAAFxGAAA2AAAAAAAAAGEAAAAMBAAA1jwAADcAAAD//wAAUwAAAP4I +AADqBQAAcwAAAAIAAAAiAAAAMAAAAPcFAACFAAAA//8AAFMAAAAfCQAAJQYAANwAAAAEAAAAIgAA +AHAAAAA0BgAA7AAAAP//AABTAAAAIAkAAAJJAADyAAAABgAAAGEAAAAXBAAA1jwAAPwAAAD//wAA +UwAAAHAUAAC5SAAAZgAAAP//AABTAAAAdhQAAOo8AABaAAAA//8AAFMAAAC0FAAAJQYAAG0BAAAC +AAAAIgAAAHAAAAA0BgAAbgEAAP//AABTAAAAoBQAAA9KAACmAQAA//8AAFMAAACmFAAA00cAAMwB +AAD//wAAUwAAAJEUAAAlBgAABQIAAAYAAAAiAAAAcAAAADQGAAAGAgAA//8AAF4AAABpAAAAwlYA +AC0AAAD//wAAXgAAAHQAAADYVgAAdwAAAP//AABeAAAAiAAAAMJWAADDAQAA//8AAF4AAACEAAAA +9wUAAHwBAAD//wAAXgAAAJAAAAAdHwAA5QEAAP//AABeAAAAkQAAAMJWAAAMAgAA//8AAF4AAACK +AAAAJQYAAHkCAAAGAAAAIgAAAHAAAAA0BgAAegIAAP//AABeAAAAoAAAANhWAAAYAAAA//8AAF4A +AACrAAAA9wUAAIYAAAD//wAAXgAAALYAAAAlBgAA4gAAAAIAAAAiAAAAcAAAADQGAADjAAAA//8A +AF4AAAC/AAAAwlYAADEBAAD//wAAXgAAAK8AAAAlBgAAOgEAAAUAAAAiAAAAcAAAADQGAAA7AQAA +//8AAF4AAADUAAAA/FYAAIoBAAD//wAAaAAAAHsDAACTXwAA9AIAAP//AABoAAAAcgMAAGxfAAAY +AQAAAQAAAGgAAAAFAwAAKF8AACsBAAD//wAAaAAAAIgDAACkXwAAbwEAAAMAAABoAAAA2AIAAChf +AABwAQAAAwAAAGgAAADkAgAAKF8AAKMBAAD//wAAaAAAAIoDAAC3XwAA2AEAAAYAAABoAAAA/gIA +AChfAADZAQAA//8AAGgAAADDBAAAJQYAABwAAAAAAAAAIgAAAHAAAAA0BgAAHQAAAP//AABoAAAA +twQAAOoFAAA8AAAAAgAAACIAAAAwAAAA9wUAAFUAAAD//wAAaAAAAMcEAADqBQAAhQAAAAQAAAAi +AAAAMAAAAPcFAACUAAAA//8AAGgAAADRBAAAJQYAAMgAAAAGAAAAIgAAAHAAAAA0BgAAyQAAAP// +AAByAAAASwEAALUNAABgAAAA//8AAHIAAABYAQAA2A0AAIgAAAD//wAAcgAAADgBAAC1DQAA4gAA +AP//AAByAAAARgEAANgNAAAIAQAA//8AAHIAAABPAQAACBUAAJcBAAD//wAAcgAAAFMBAADYDQAA +xAEAAP//AAByAAAAPQEAAAgVAAAJAgAA//8AAHIAAABBAQAA2A0AADgCAAD//wAAdgAAANABAADq +BQAAQQAAAAAAAAAiAAAAMAAAAPcFAABCAAAA//8AAHYAAADSAQAAJQYAAG4AAAACAAAAIgAAAHAA +AAA0BgAAbwAAAP//AAB2AAAA1QEAAGkcAACIAAAA//8AAHYAAADWAQAA5gYAAIkAAAD//wAAdgAA +ANcBAAAlBgAAmAAAAAYAAAAiAAAAcAAAADQGAACZAAAA//8AAHgAAAD3AQAA32oAACUAAAAAAAAA +eAAAANEBAAD3agAATgAAAAEAAAB4AAAAxwEAAI8EAABZAAAA//8AAHgAAAD6AQAA32oAAJwAAAD/ +/wAAeAAAAP4BAAD3agAArwAAAAQAAAB4AAAAxwEAAI8EAACqAAAAAwAAAHgAAADRAQAA92oAAN8A +AAAGAAAAeAAAAMcBAACPBAAA9gAAAP//AAA0AAAAugQAAOEEAAB6AAAAAAAAACsAAAA/AQAAzgQA +AC8AAAD//wAANAAAALsEAAC2BAAAogAAAP//AAA0AAAAxAQAADgOAADNAAAA//8AADQAAADKBAAA +8iAAANMAAAD//wAANAAAAOgEAAD5BAAAcAEAAP//AAA0AAAA5gQAAA8FAAAaAQAABgAAACsAAACt +AQAAJQUAAPkBAAD//wAANAAAANYEAAALIQAAmwIAAP//AAA6AAAA5gUAAHovAAD6AQAA//8AADoA +AADCBQAAyyEAAGUAAAD//wAAOgAAAMIFAAA4DgAAZgAAAAEAAAA6AAAAtAIAAM4EAABrAAAA//8A +ADoAAADiBQAAOA4AAJgBAAD//wAAOgAAALUFAAAaFAAAHwAAAP//AAA6AAAA5QUAALQuAAC8AQAA +AAAAADoAAABWBAAAQw0AAPwBAAD//wAAOgAAALwFAAA4DgAAawIAAP//AAA7AAAAKwMAABgmAAAa +AAAA//8AADsAAAAyAwAAhiYAAGIAAAD//wAAOwAAADUDAAArJgAAdQAAAP//AAA7AAAANgMAAEEm +AADZAAAAAwAAADsAAABYAQAAXiYAALAAAAADAAAAOwAAAFgBAAByJgAAsAAAAP//AAA7AAAAPAMA +AJ0mAADvAAAA//8AADsAAABWAwAAQCUAAFUBAAD//wAAOwAAADkDAACGJgAADwIAAP//AABDAAAA +kgAAALQ5AAAcAAAAAAAAAEMAAAA9AQAA0DkAAB0AAAD//wAAQwAAAKsAAADrOQAAWgAAAP//AABD +AAAArAAAALQ5AABfAAAAAwAAAEMAAAA9AQAA0DkAAGAAAAD//wAAQwAAAKgAAAAJOgAAPAAAAP// +AABDAAAAqAAAACY6AAA8AAAA//8AAEMAAAC4AAAAjwQAAKAAAAD//wAAQwAAANsAAABAOgAA8gAA +AP//AABFAAAACgEAAI4hAAACAQAAAAAAACsAAAABAQAAViEAAB4BAAABAAAAOgAAAJgHAADQEwAA +AwEAAP//AABFAAAACwEAAEAZAAAoAQAA//8AAEUAAAAOAQAAsCEAAEUBAAD//wAARQAAABEBAADL +IQAAUwEAAP//AABFAAAAEQEAADgOAABUAQAABQAAADoAAAC0AgAAzgQAAFkBAAD//wAARQAAABYB +AADyIAAAvgEAAP//AABTAAAAlAQAAOoFAAAuAAAAAAAAACIAAAAwAAAA9wUAAC8AAAD//wAAUwAA +ALEEAAAlBgAARQEAAAIAAAAiAAAAcAAAADQGAABGAQAA//8AAFMAAAC4BAAA7BsAAIQBAAD//wAA +UwAAAM8EAADqBQAA0gEAAAUAAAAiAAAAMAAAAPcFAADTAQAA//8AAFMAAADQBAAA6gUAAOUBAAAH +AAAAIgAAADAAAAD3BQAA5gEAAP//AABTAAAAyAsAAAgVAAAYAAAA//8AAFMAAADQCwAA/w4AAOgA +AAD//wAAUwAAANALAAAFSwAA6AAAAP//AABTAAAA0AsAACJLAADoAAAA//8AAFMAAADQCwAAPUsA +AOgAAAD//wAAUwAAANULAABYSwAASwEAAP//AABTAAAA5wsAAHdLAAB6AQAA//8AAFMAAAD+CwAA +WEsAAD8CAAD//wAAUwAAAP4LAAB3SwAAPwIAAP//AABTAAAAahUAAOoFAAAeAAAAAAAAACIAAAAw +AAAA9wUAAB0AAAD//wAAUwAAAJAVAAAcSgAATgEAAP//AABTAAAApxUAACUGAAAJAQAAAwAAACIA +AABwAAAANAYAAAoBAAD//wAAUwAAAJQVAAAlBgAAiQEAAAUAAAAiAAAAcAAAADQGAACKAQAA//8A +AFMAAACkFQAA6gUAAAYCAAAHAAAAIgAAADAAAAD3BQAAIgIAAP//AABTAAAAzhcAAIJTAAAaAAAA +//8AAFMAAADPFwAAAh8AAEUAAAABAAAAUwAAAFgYAADmBgAAXwAAAAEAAABTAAAAWhgAAOYGAABo +AAAAAQAAAFMAAABcGAAA5gYAAHMAAAD//wAAUwAAAOsXAAACHwAA/gAAAAUAAABTAAAAXBgAAOYG +AADhAAAABQAAAFMAAABYGAAA5gYAABcBAAAFAAAAUwAAAFoYAADmBgAALAEAAP//AABUAAAAOAEA +AL9UAABrAAAA//8AAFQAAAA8AQAA3lQAAE8EAAD//wAAVAAAAEUBAAD+VAAAOgEAAP//AABUAAAA +SwEAADlUAABPAQAA//8AAFQAAABMAQAAOVQAAFMBAAD//wAAVAAAAF4BAABUVAAAWAEAAP//AABU +AAAAfgEAADlUAAAvAwAA//8AAFQAAAB/AQAAI1UAABMCAAD//wAAVAAAAIABAABMVQAAZAMAAP// +AABhAAAAzgIAAMdbAAA0AAAA//8AAGEAAADZAgAA2FsAAG8AAAD//wAAYQAAANUCAADtWwAAJQEA +AP//AABhAAAA7AIAAP5bAAByAQAA//8AAGEAAADuAgAAFFwAAKUBAAD//wAAYQAAAOoCAAA1PwAA +1gEAAP//AABhAAAA4wIAANhbAABDAgAA//8AAGEAAADfAgAA7VsAAPYCAAAAAAAATgAAAJ0FAABn +HQAAywMAAP//AAB2AAAAIQIAAGtkAABCAAAA//8AAHYAAAAlAgAAa2QAAGkAAAD//wAAdgAAACYC +AACvZAAAcAAAAP//AAB2AAAANgIAAMpkAADhAAAA//8AAHYAAAA6AgAA42QAACYBAAD//wAAdgAA +AD0CAADjZAAALwEAAP//AAB2AAAAPwIAAONkAACpAQAA//8AAHYAAABCAgAA42QAAPcCAAD//wAA +dgAAAEQCAADjZAAALQAAAP//AAB4AAAA6AEAAPdqAAATAAAA//8AAHgAAADrAQAA32oAABsAAAAB +AAAAeAAAANEBAAD3agAARwAAAAIAAAB4AAAAxwEAAI8EAABSAAAA//8AAHgAAADsAQAA32oAAH4A +AAAEAAAAeAAAANEBAAD3agAAiwAAAAUAAAB4AAAAxwEAAI8EAACiAAAA//8AAHgAAADuAQAA92oA +ANMAAAAHAAAAeAAAAMcBAACPBAAAygAAAP//AAAmAAAAWQEAAH8QAAAYAAAA//8AACYAAABhAQAA +fxAAADIAAAD//wAAJgAAAGMBAACrEAAAoAAAAAIAAABCAAAAEAAAAP4NAABxAAAAAgAAAEIAAAAS +AAAA/g0AAK0AAAACAAAAQgAAABgAAAAzDQAA7wAAAP//AAAmAAAAfwEAAI8EAACmAQAA//8AACYA +AACAAQAAHRAAAL4BAAAHAAAAJgAAANQAAACPBAAAuAEAAP//AAAmAAAAfgEAAI8EAAB1AQAA//8A +ADMAAAAIAwAAVRoAABgAAAD//wAAMwAAAGsDAAB9GgAASQAAAP//AAAzAAAAEQMAAPYYAAAqAAAA +AgAAADMAAAA+BQAAEhkAAIMAAAD//wAAMwAAABgDAABVGgAAqwAAAP//AAAzAAAAPQMAAH0aAADu +AAAA//8AADMAAABFAwAAvwcAAAUBAAD//wAAMwAAAEoDAACQGgAAbgEAAP//AAAzAAAAegMAAH0a +AABKAgAA//8AADMAAAASAwAAfRoAAIcCAAD//wAANAAAAIUCAADqBQAAMwAAAAAAAAAiAAAAMAAA +APcFAAA0AAAA//8AADQAAACGAgAA7h0AAGUAAAD//wAANAAAAIcCAABKHwAArwAAAP//AAA0AAAA +nQIAAAIfAADsAAAABAAAAFMAAABYGAAA5gYAAAgBAAAEAAAAUwAAAFoYAADmBgAAEQEAAAQAAABT +AAAAXBgAAOYGAAAfAQAA//8AADQAAACoAgAAJQYAAFcBAAAIAAAAIgAAAHAAAAA0BgAAWAEAAP// +AAA2AAAAJQEAAOoFAAAqAAAAAAAAACIAAAAwAAAA9wUAACsAAAD//wAANgAAACgBAABRJAAARgAA +AAIAAAA2AAAAZwAAAGYkAAA/AAAA//8AADYAAAAqAQAAJQYAAFUAAAAEAAAAIgAAAHAAAAA0BgAA +VgAAAP//AAA2AAAALwEAAL8HAABvAAAA//8AADYAAAAyAQAAvwcAAMoAAAD//wAANgAAADQBAAAl +BgAA4wAAAAgAAAAiAAAAcAAAADQGAADkAAAA//8AADgAAABiAAAArCgAAC8AAAD//wAAOAAAAGMA +AADHKAAATQAAAP//AAA4AAAAZwAAAG8YAAClAAAA//8AADgAAABpAAAAwRcAANIAAAD//wAAOAAA +AG4AAADgKAAA6QAAAAQAAAA4AAAARgAAAKwoAADqAAAA//8AADgAAABzAAAA4CgAAPcAAAAGAAAA +OAAAAEYAAACsKAAA+AAAAAYAAAA4AAAASAAAAKwoAAACAQAABAAAADgAAABIAAAArCgAADUBAAD/ +/wAAOgAAAHkHAABDDQAAvgAAAP//AAA6AAAAfgcAAOoFAABMAAAAAQAAACIAAAAwAAAA9wUAAE0A +AAD//wAAOgAAAH8HAABDDQAAWgAAAP//AAA6AAAAgAcAACUGAACTAAAABAAAACIAAABwAAAANAYA +AIYAAAD//wAAOgAAAHgHAADqBQAAsAAAAAYAAAAiAAAAMAAAAPcFAACxAAAA//8AADoAAAB6BwAA +JQYAAOoAAAAIAAAAIgAAAHAAAAA0BgAA6wAAAP//AABQAAAAagAAAGcdAAAYAAAA//8AAFAAAAB7 +AAAAZx0AAI8AAAD//wAAUAAAAOUAAAC/BwAAnwEAAP//AABQAAAA6AAAAKNCAADFAQAA//8AAFAA +AAD1AAAAvwcAAF0DAAD//wAAUAAAAPcAAAC/BwAAbAAAAP//AABQAAAA+wAAAL8HAADWAwAA//8A +AFAAAACGAAAAtEIAACUEAAAHAAAAUwAAAO4CAABnHQAANQQAAAcAAABTAAAA7wIAAGcdAAC+BAAA +//8AAFMAAADzDQAAtEIAAEUBAAD//wAAUwAAAPENAABnHQAANwAAAP//AABTAAAA/Q0AAFEIAABq +AAAA//8AAFMAAAANDgAAhkwAAJgAAAADAAAAUwAAAFsNAACUTAAAjQAAAAQAAABbAAAANQEAAH4I +AACZAAAAAwAAAFMAAABcDQAApUwAAKAAAAAGAAAAWwAAABQBAADmBgAApAAAAAAAAABTAAAA7gIA +AGcdAAAYAAAAAAAAAFMAAADvAgAAZx0AAN4BAAD//wAAUwAAAH4XAADmBgAAwQAAAP//AABTAAAA +gRcAAOYGAAD3AAAA//8AAFMAAACCFwAA5gYAAAEBAAD//wAAUwAAAIUXAADqBQAACwEAAAMAAAAi +AAAAMAAAAPcFAAAMAQAA//8AAFMAAACGFwAAMhwAABkBAAAFAAAAUwAAAJIWAABLHAAAGgEAAAYA +AABTAAAAZRgAAGkcAAAsAQAA//8AAFMAAACHFwAAJQYAAHgBAAAIAAAAIgAAAHAAAAA0BgAAeQEA +AP//AAB4AAAAbQIAAMIKAACkAQAA//8AAHgAAABuAgAAwgoAAMUBAAD//wAAeAAAAIcCAABYawAA +awIAAAIAAAB4AAAARQEAAI8EAACHAgAA//8AAHgAAACHAgAAWGsAAJQCAAAEAAAAeAAAAEUBAACP +BAAAvQIAAP//AAB4AAAAjQIAAG9rAAAWCwAA//8AAHgAAACNAgAAb2sAAFALAAAGAAAAeAAAAE8B +AACPBAAAOwsAAAcAAAB4AAAATwEAAI8EAACMCwAA//8AACUAAAB0BQAAtQ0AAHIAAAD//wAAJQAA +AHkFAADqBQAAngAAAAEAAAAiAAAAMAAAAPcFAACfAAAA//8AACUAAACSBQAA2A0AAEYBAAD//wAA +JQAAAHwFAAAzDQAAaQAAAP//AAAlAAAAkAUAAIQPAAA3AQAA//8AACUAAACUBQAAJQYAAIkBAAAG +AAAAIgAAAHAAAAA0BgAAigEAAP//AAAlAAAAjgUAADMNAAAXAgAA//8AACUAAACBBQAAJQYAAE8C +AAAJAAAAIgAAAHAAAAA0BgAAUAIAAP//AAA0AAAA6gUAAFkUAAAbAAAAAAAAACsAAAD3AAAAOA4A +ACMAAAAAAAAAKwAAAPcAAABzFAAAHgAAAP//AAA0AAAA6wUAAI4hAAAcAAAAAwAAACsAAAABAQAA +ViEAADIAAAAEAAAAOgAAAJgHAADQEwAAMwAAAAUAAAA6AAAAkgcAAGgFAABKAAAA//8AADQAAADr +BQAAsCEAAB0AAAD//wAANAAAAO4FAADLIQAAXAAAAP//AAA0AAAA7gUAADgOAABdAAAACAAAADoA +AAC0AgAAzgQAAGIAAAD//wAAOAAAAKAAAADqBQAAPQAAAAAAAAAiAAAAMAAAAPcFAAA+AAAA//8A +ADgAAACjAAAAHR8AAGUAAAD//wAAOAAAAK0AAADqBQAAigAAAAMAAAAiAAAAMAAAAPcFAACLAAAA +//8AADgAAACuAAAA2yMAAKwAAAD//wAAOAAAALYAAAAdHwAAtQAAAP//AAA4AAAAsgAAACUGAADa +AAAABwAAACIAAABwAAAANAYAANsAAAD//wAAOAAAAKgAAADTGgAA8AAAAP//AAA4AAAAqwAAANMa +AAANAQAA//8AADkAAABfAQAAaCMAANUBAAD//wAAOQAAAGwBAADqBQAASgAAAAEAAAAiAAAAMAAA +APcFAABLAAAA//8AADkAAAByAQAAJQYAAIYAAAADAAAAIgAAAHAAAAA0BgAAhwAAAP//AAA5AAAA +fAEAAOoFAADmAAAABQAAACIAAAAwAAAA9wUAAOcAAAD//wAAOQAAAH4BAAAlBgAABQEAAAcAAAAi +AAAAcAAAADQGAAAGAQAA//8AADkAAACDAQAAOA4AABgBAAAAAAAAIAAAAC8AAAA7DAAAtQEAAP// +AABQAAAAdgEAABZDAAAtAAAA//8AAFAAAACBAQAAUQgAAKoAAAD//wAAUAAAAJcBAADjQAAA7gAA +AP//AABQAAAApQEAAONAAACgAQAAAwAAAG8AAAD4AwAAjwQAAKIBAAACAAAAbwAAAPgDAACPBAAA +8AAAAAIAAABvAAAA/QMAAI8EAABSAQAAAwAAAG8AAAD9AwAAjwQAABECAAD//wAAUAAAAKwBAAAJ +PgAAhgEAAP//AABQAAAArQEAAAk+AADgAgAA//8AAFAAAACuAQAACT4AAAYDAAD//wAAUwAAAK8C +AAAuRgAAJwAAAP//AABTAAAAsgIAAEdGAABVAAAA//8AAFMAAAC6AgAAXEYAAJ4AAAACAAAAYQAA +AAwEAADWPAAAqQAAAP//AABTAAAAwwIAAGxGAAD5AAAA//8AAFMAAADHAgAA6gUAAAoBAAAFAAAA +IgAAADAAAAD3BQAACwEAAP//AABTAAAAyAIAAL8HAAAuAQAA//8AAFMAAADKAgAAKiQAAD8BAAD/ +/wAAUwAAANACAAAlBgAAiQEAAAkAAAAiAAAAcAAAADQGAACKAQAA//8AAFMAAADcBAAAtQ0AACAA +AAD//wAAUwAAAN0EAAC9RwAAOAAAAP//AABTAAAA4QQAAOoFAABUAAAAAgAAACIAAAAwAAAA9wUA +AFUAAAD//wAAUwAAAO4EAAAlBgAAuQAAAAQAAAAiAAAAcAAAADQGAAC6AAAA//8AAFMAAAD0BAAA +CBUAANgAAAD//wAAUwAAAPsEAADTRwAABgEAAP//AABTAAAABAUAAL8HAAAvAQAA//8AAFMAAAAG +BQAA60cAAFQBAAD//wAAUwAAAA4FAADYDQAAcgEAAP//AABTAAAAOBYAAOoFAAAdAAAAAAAAACIA +AAAwAAAA9wUAABwAAAD//wAAUwAAAEEWAAAyHAAAWQAAAAIAAABTAAAAkhYAAEscAABaAAAAAwAA +AFMAAABlGAAAaRwAAHAAAAD//wAAUwAAAEIWAAAlBgAAsAAAAAUAAAAiAAAAcAAAADQGAAC9AAAA +//8AAFMAAABHFgAAJQYAAMMAAAAHAAAAIgAAAHAAAAA0BgAAxAAAAP//AABTAAAAOhYAACUGAADT +AAAACQAAACIAAABwAAAANAYAANQAAAD//wAAUwAAAJUXAADmBgAAKAAAAP//AABTAAAAkxcAAO4d +AAAmAAAA//8AAFMAAACUFwAASh8AAFIAAAD//wAAUwAAAKYXAADuHQAAgwAAAP//AABTAAAApxcA +AOoFAACPAAAABAAAACIAAAAwAAAA9wUAAJAAAAD//wAAUwAAAKgXAAAyHAAAnQAAAAYAAABTAAAA +khYAAEscAACeAAAABwAAAFMAAABlGAAAaRwAALIAAAD//wAAUwAAAKkXAAAlBgAA7wAAAAkAAAAi +AAAAcAAAADQGAADwAAAA//8AAAgAAAB7AAAA2gMAAD0AAAD//wAACAAAAHwAAADwAwAAfgAAAP// +AAAIAAAAgQAAAPADAACZAAAA//8AAAgAAAB+AAAAjwQAALkAAAD//wAACAAAAIMAAACPBAAA4gAA +AP//AAAIAAAAiAAAALYEAAAOAQAABQAAADoAAACaAgAAzgQAABEBAAD//wAACAAAAJkAAADhBAAA +XAEAAAcAAAArAAAAPwEAAM4EAABfAQAA//8AAAgAAACbAAAA+QQAAEMCAAD//wAACAAAAKIAAAAP +BQAAaAIAAAoAAAArAAAArQEAACUFAACrAgAA//8AACcAAAAdAAAA0hAAAHYAAAAAAAAAJgAAAL4A +AAB/EAAAjAAAAP//AAAnAAAAHgAAAI8EAACQAAAA//8AACcAAAAgAAAA5RAAALgAAAD//wAAJwAA +ACQAAACPBAAAvQAAAP//AAAnAAAAJQAAAAIRAADTAAAA//8AACcAAAAqAAAABBAAANUAAAAGAAAA +JgAAANAAAACPBAAA2AAAAP//AAAnAAAAKwAAANwSAAAMAQAACAAAACYAAADYAAAAjwQAAAUBAAD/ +/wAAJwAAACsAAACPBAAADAEAAP//AAAnAAAALQAAAI8EAAA+AQAA//8AACcAAABFAAAA0hAAAHYA +AAAAAAAAJgAAAL4AAAB/EAAAjAAAAP//AAAnAAAARgAAAI8EAACQAAAA//8AACcAAABIAAAA5RAA +ALgAAAD//wAAJwAAAEwAAACPBAAAvQAAAP//AAAnAAAATQAAAAIRAADUAAAA//8AACcAAABSAAAA +BBAAANYAAAAGAAAAJgAAANAAAACPBAAA2QAAAP//AAAnAAAAUwAAANwSAAAOAQAACAAAACYAAADY +AAAAjwQAAAcBAAD//wAAJwAAAFMAAACPBAAADgEAAP//AAAnAAAAVQAAAI8EAABGAQAA//8AADMA +AACYBAAA5gYAAKIAAAD//wAAMwAAAJoEAAC1DQAAqwAAAP//AAAzAAAAmgQAAH4IAACyAAAA//8A +ADMAAADTBAAAtQ0AAOkAAAD//wAAMwAAANMEAAB+CAAA8AAAAP//AAAzAAAA1AQAAAgVAAD+AAAA +//8AADMAAADfBAAAvwcAADEBAAD//wAAMwAAAA4FAAC/BwAA1gEAAP//AAAzAAAAKwUAAPYYAAB0 +AgAACAAAADMAAAA+BQAAEhkAAIYCAAD//wAAMwAAAC8FAADYDQAAtgIAAP//AAAzAAAAMAUAAH4I +AADlAgAA//8AADQAAAAVAQAA6gUAABkAAAAAAAAAIgAAADAAAAD3BQAAGAAAAP//AAA0AAAAGAEA +ACUGAAA8AAAAAgAAACIAAABwAAAANAYAAD0AAAD//wAANAAAABkBAAD+BgAAUAAAAP//AAA0AAAA +JQEAAOYGAACXAAAA//8AADQAAAApAQAA6gUAAKwAAAAGAAAAIgAAADAAAAD3BQAArQAAAP//AAA0 +AAAAKgEAANUdAAC6AAAACAAAAFMAAACVGAAA7h0AANEAAAD//wAANAAAACsBAAAlBgAA7QAAAAoA +AAAiAAAAcAAAADQGAADuAAAA//8AADQAAABQAgAA6gUAABkAAAAAAAAAIgAAADAAAAD3BQAAGAAA +AP//AAA0AAAAWwIAAAIfAABGAAAAAgAAAFMAAABYGAAA5gYAAGEAAAACAAAAUwAAAFoYAADmBgAA +agAAAAIAAABTAAAAXBgAAOYGAAB1AAAA//8AADQAAABkAgAA5gYAAKcAAAD//wAANAAAAGYCAAAl +BgAArgAAAAcAAAAiAAAAcAAAADQGAACvAAAA//8AADQAAABqAgAAHR8AAMgAAAD//wAANAAAAFUC +AAAlBgAA+gAAAAoAAAAiAAAAcAAAADQGAAD7AAAA//8AADQAAAB/BQAAjiEAADEAAAAAAAAAKwAA +AAEBAABWIQAAMgAAAAEAAAA6AAAAmAcAANATAAA2AAAAAgAAADoAAACSBwAAaAUAAFMAAAD//wAA +NAAAAIcFAAA+IQAAZgAAAP//AAA0AAAAkAUAAEAZAACRAAAA//8AADQAAACTBQAAsCEAAJcAAAD/ +/wAANAAAAJYFAADLIQAApQAAAP//AAA0AAAAlgUAADgOAACmAAAABwAAADoAAAC0AgAAzgQAAKsA +AAD//wAANAAAAJ0FAADyIAAAEgEAAP//AAA0AAAAqQUAAAshAAB2AQAA//8AADoAAACsBgAAtQ0A +ADkAAAD//wAAOgAAALIGAADqBQAAbwAAAAEAAAAiAAAAMAAAAPcFAAB4AAAA//8AADoAAAC8BgAA +JQYAAMcAAAADAAAAIgAAAHAAAAA0BgAAyAAAAP//AAA6AAAAvQYAANgNAADTAAAA//8AADoAAADK +BgAA5i8AACYBAAAGAAAAOgAAAI8GAAA4DgAAUwEAAAYAAAA6AAAAkAYAAM4EAAAnAQAA//8AADoA +AADLBgAAJQYAAIUBAAAJAAAAIgAAAHAAAAA0BgAAhgEAAP//AAA6AAAAzAYAANgNAACRAQAA//8A +ADoAAADLBwAAPzAAAFgAAAD//wAAOgAAANUHAAA/MAAAtQAAAP//AAA6AAAA0QcAAOoFAACnAAAA +AgAAACIAAAAwAAAA9wUAAKgAAAD//wAAOgAAAN8HAAA/MAAARQEAAP//AAA6AAAA6gcAAD8wAADt +AQAA//8AADoAAADzBwAAJQYAAEgCAAAGAAAAIgAAAHAAAAA0BgAASQIAAP//AAA6AAAA5AcAACUG +AAB8AgAACAAAACIAAABwAAAANAYAAH0CAAD//wAAOgAAANYHAAAlBgAAngIAAAoAAAAiAAAAcAAA +ADQGAACfAgAA//8AAD0AAABSAAAAZC4AACAAAAD//wAAPQAAAFUAAAAYJgAALAAAAP//AAA9AAAA +VgAAAIYmAAAvAAAA//8AAD0AAABcAAAAQSYAAMMAAAADAAAAOwAAAFgBAAByJgAAoAAAAAMAAAA7 +AAAAWAEAAF4mAACgAAAA//8AAD0AAABcAAAAejIAAMMAAAAGAAAAPgAAAF4BAACWMgAA8QAAAP// +AAA9AAAAXwAAAEEmAAAyAQAACAAAADsAAABYAQAAXiYAAA4BAAAIAAAAOwAAAFgBAAByJgAADgEA +AP//AAA9AAAAZAAAAEAlAAB4AQAA//8AAFMAAAAoBgAAtQ0AACYAAAD//wAAUwAAACkGAAAIFQAA +LQAAAP//AABTAAAAKwYAAOoFAAA4AAAAAgAAACIAAAAwAAAA9wUAAEwAAAD//wAAUwAAAEgGAAAl +BgAAKQEAAAQAAAAiAAAAcAAAADQGAAAqAQAA//8AAFMAAABjBgAA7BsAADACAAD//wAAUwAAAHIG +AADqBQAAdgIAAAcAAAAiAAAAMAAAAPcFAAB3AgAA//8AAFMAAAB0BgAAJQYAAKkCAAAJAAAAIgAA +AHAAAAA0BgAAqgIAAP//AABTAAAAdQYAANgNAAC3AgAA//8AAHYAAAB+AgAAVGQAAG8AAAD//wAA +dgAAAHsCAADqBQAASwAAAAEAAAAiAAAAMAAAAPcFAABMAAAAAAAAAHYAAADpAQAAa2QAAHAAAAD/ +/wAAdgAAAIICAABrZAAAqwAAAP//AAB2AAAAiQIAAGtkAADRAAAA//8AAHYAAACKAgAAr2QAAN0A +AAD//wAAdgAAAJACAADKZAAACwEAAP//AAB2AAAAkQIAAONkAABjAAAA//8AAHYAAACSAgAA42QA +AIkBAAD//wAAdgAAAJUCAAAlBgAA1gEAAAoAAAAiAAAAcAAAADQGAADXAQAA//8AAAkAAAC/AAAA +3QUAACsAAAD//wAACQAAAMgAAADqBQAAmwAAAAEAAAAiAAAAMAAAAPcFAACxAAAA//8AAAkAAADP +AAAADAYAAOABAAD//wAACQAAAOcAAAAlBgAABAIAAAQAAAAiAAAAcAAAADQGAAAFAgAA//8AAAkA +AAD7AAAASwYAABEDAAD//wAACQAAANgAAABkBgAA+QQAAAcAAAAJAAAAegAAAI8EAAABBQAA//8A +AAkAAADiAAAAJQYAAD4FAAAJAAAAIgAAAHAAAAA0BgAAPwUAAP//AAAJAAAAywAAACUGAADfBQAA +CwAAACIAAABwAAAANAYAAOAFAAD//wAAMwAAALYFAADqBQAAKwAAAAAAAAAiAAAAMAAAAPcFAAAs +AAAA//8AADMAAAC9BQAAJQYAAJ8AAAACAAAAIgAAAHAAAAA0BgAAoAAAAP//AAAzAAAAvwUAAJ0c +AACtAAAA//8AADMAAADEBQAA6gUAAMYAAAAFAAAAIgAAADAAAAD3BQAAxwAAAP//AAAzAAAAxgUA +ACUGAADfAAAABwAAACIAAABwAAAANAYAAOAAAAD//wAAMwAAANgFAADqBQAA7wAAAAkAAAAiAAAA +MAAAAPcFAADwAAAA//8AADMAAADdBQAAJQYAACUBAAALAAAAIgAAAHAAAAA0BgAAJgEAAP//AAA0 +AAAAvAIAAGcdAABMAAAA//8AADQAAADNAgAAch8AAKAAAAD//wAANAAAAL4CAABnHQAA5gEAAP// +AAA0AAAAEQMAAIwfAABaAwAA//8AADQAAAAXAwAAsR8AANMDAAD//wAANAAAACADAADWHwAAtgMA +AP//AAA0AAAALAMAAPcfAADJAwAA//8AADQAAAA2AwAAHiAAAOYDAAD//wAANAAAADwDAAAeIAAA +KwQAAP//AAA0AAAAPgMAAB4gAABcBAAA//8AADQAAABCAwAAQyAAAIoEAAD//wAANAAAAMMCAABn +HQAAXwUAAP//AAA0AAAAuAIAAGcdAAApAAAA//8AADoAAAA1AwAAiRcAAE4AAAD//wAAOgAAAGMD +AAAtGAAASgEAAAEAAAA4AAAA8gAAAEwYAACIAQAA//8AADoAAABlAwAAJQYAAIgCAAADAAAAIgAA +AHAAAAA0BgAAiQIAAP//AAA6AAAAaAMAAOoFAACoAgAABQAAACIAAAAwAAAA9wUAAKkCAAD//wAA +OgAAAE4DAADkFwAAjQMAAAcAAAA4AAAA2QAAAAYYAACPAwAA//8AADoAAABQAwAAJQYAAAsEAAAJ +AAAAIgAAAHAAAAA0BgAADAQAAP//AAA6AAAAVAMAAOoFAAAiBAAACwAAACIAAAAwAAAA9wUAACME +AAD//wAAPAAAAKAAAABOMwAAWAEAAAAAAAA8AAAAdgAAAH0ZAABZAQAAAAAAADwAAAB3AAAAMw0A +AF0BAAAAAAAAPAAAAHoAAACPBAAAtwEAAAAAAAA8AAAAewAAAI8EAAC8AQAA//8AADwAAACtAAAA +KSUAAH0CAAAFAAAAQAAAACoAAABAJQAAnQIAAAUAAABAAAAALwAAAFklAACfAgAA//8AADwAAACy +AAAAKSUAAHUCAAAIAAAAQAAAAC8AAABZJQAAywIAAP//AAA8AAAAswAAACklAAAIAwAACgAAAEAA +AAAvAAAAWSUAAPoCAAD//wAAPAAAALMAAAC6DwAACAMAAP//AABAAAAAWQEAAJE4AACUAAAAAAAA +AEAAAAA0AAAAEDIAAMMAAAAAAAAAQAAAADQAAABAJQAAwwAAAP//AABAAAAAWgEAACklAAAKAQAA +AwAAAEAAAAAvAAAAWSUAAPQAAAD//wAAQAAAAFwBAAApJQAALwEAAAUAAABAAAAAKgAAAEAlAAAm +AQAABQAAAEAAAAAvAAAAWSUAACgBAAD//wAAQAAAAF8BAAApJQAAUwEAAAgAAABAAAAALwAAAFkl +AABMAQAA//8AAEAAAABXAQAAKSUAAL8BAAAKAAAAQAAAACoAAABAJQAA4wEAAAoAAABAAAAALwAA +AFklAADlAQAA//8AAFMAAADQBQAAT0gAAD8AAAD//wAAUwAAAN0FAADqBQAAhwAAAAEAAAAiAAAA +MAAAAPcFAACUAAAA//8AAFMAAADzBQAAJQYAACcBAAADAAAAIgAAAHAAAAA0BgAAKAEAAP//AABT +AAAA/gUAAOoFAABXAQAABQAAACIAAAAwAAAA9wUAAFgBAAD//wAAUwAAAAEGAAAlBgAAcQEAAAcA +AAAiAAAAcAAAADQGAAByAQAA//8AAFMAAADHBQAA6gUAANgBAAAJAAAAIgAAADAAAAD3BQAA2QEA +AP//AABTAAAAygUAACUGAADyAQAACwAAACIAAABwAAAANAYAAPMBAAD//wAAUwAAAIYRAAAlBgAA +HwAAAAAAAAAiAAAAcAAAADQGAAAgAAAA//8AAFMAAAB2EQAA/gYAADIAAAD//wAAUwAAAHYRAAD+ +BgAAMgAAAP//AABTAAAAdhEAAP4GAAAyAAAA//8AAFMAAAB3EQAA6gUAAFkAAAAFAAAAIgAAADAA +AAD3BQAAZgAAAP//AABTAAAAiREAABUHAABwAAAABwAAAFMAAACdGAAAaRwAAHEAAAD//wAAUwAA +AIMRAADQBgAA/wAAAAkAAABTAAAAkBgAAOYGAAAOAQAA//8AAFMAAAB7EQAAFQcAACkBAAD//wAA +UwAAAH0RAAAVBwAARgEAAP//AAB3AAAAWQAAAFFCAABmAAAA//8AAHcAAACGAAAAUQgAAJYBAAD/ +/wAAdwAAAAABAABRCAAA/AYAAP//AAB3AAAAOwEAAERnAACdCQAA//8AAHcAAADwAQAAUQgAAEMG +AAD//wAAdwAAAHkBAADjQAAAkgsAAAUAAABvAAAA+AMAAI8EAACzCwAABQAAAG8AAAD9AwAAjwQA +AG8MAAD//wAAdwAAAKkBAADjQAAA9QMAAAgAAABvAAAA+AMAAI8EAAAZDQAACAAAAG8AAAD9AwAA +jwQAAOANAAD//wAAdwAAAIABAABwZwAAfBcAAP//AAB3AAAAjwEAAHBnAAAuDAAA//8AAAkAAABo +AQAA6gUAAC4AAAAAAAAAIgAAADAAAAD3BQAANwAAAP//AAAJAAAAiwEAANAGAABqAAAAAgAAAFMA +AACQGAAA5gYAAHcAAAD//wAACQAAAHoBAAAMBgAAlgAAAP//AAAJAAAAngEAANAGAAASAgAABQAA +AFMAAACQGAAA5gYAAB8CAAD//wAACQAAAJABAAAMBgAAWgIAAP//AAAJAAAAoAEAACUGAABtAwAA +CAAAACIAAABwAAAANAYAAG4DAAD//wAACQAAAKMBAAD+BgAAeQMAAP//AAAJAAAApAEAABUHAACa +AwAA//8AAAkAAABqAQAAJQYAAL0DAAAMAAAAIgAAAHAAAAA0BgAAvgMAAP//AAAJAAAA1wEAAE0H +AAAxAAAA//8AAAkAAADrAQAATQcAAG8AAAD//wAACQAAAPwBAADqBQAA+QAAAAIAAAAiAAAAMAAA +APcFAAAPAQAA//8AAAkAAAACAgAAJQYAACwBAAAEAAAAIgAAAHAAAAA0BgAALQEAAP//AAAJAAAA +CQIAAAwGAACQAgAA//8AAAkAAAAUAgAAZAYAAKYCAAAHAAAACQAAAHoAAACPBAAAqgIAAP//AAAJ +AAAAIQIAACUGAAAWAwAACQAAACIAAABwAAAANAYAABcDAAD//wAACQAAACYCAAAlBgAASAMAAAsA +AAAiAAAAcAAAADQGAABJAwAA//8AAAkAAAA6AgAASwYAAF8EAAD//wAAJgAAANgBAADSEAAAawAA +AAAAAAAmAAAAvgAAAH8QAACBAAAA//8AACYAAADZAQAAjwQAAIUAAAD//wAAJgAAANsBAADlEAAA +rwAAAP//AAAmAAAA3wEAAI8EAAC0AAAA//8AACYAAADgAQAAAhEAAMwAAAD//wAAJgAAAOQBAAAU +EQAA3AAAAP//AAAmAAAAzwEAACQRAADpAAAA//8AACYAAADmAQAABBAAAM4AAAAIAAAAJgAAANAA +AACPBAAADAEAAP//AAAmAAAA7gEAAI8EAABfAQAA//8AACYAAADvAQAARhEAAHYBAAD//wAAJgAA +APMBAACPBAAAqgEAAP//AAAmAAAA9AEAAGURAADYAQAA//8AAFMAAAA9DAAA9hgAAE0AAAAAAAAA +MwAAAD4FAAASGQAAMgAAAP//AABTAAAAUgwAAOoFAABcAAAAAgAAACIAAAAwAAAA9wUAAF0AAAD/ +/wAAUwAAAGEMAABoIwAAIQEAAP//AABTAAAAXQwAACUGAACFAAAABQAAACIAAABwAAAANAYAAIYA +AAD//wAAUwAAAFUMAAAlBgAAoQAAAAcAAAAiAAAAcAAAADQGAACiAAAABAAAACAAAAAvAAAAOwwA +APMAAAD//wAAUwAAAGgMAAAlBgAAJgEAAAoAAAAiAAAAcAAAADQGAAAnAQAA//8AAFMAAABkDAAA +JQYAAFQBAAAMAAAAIgAAAHAAAAA0BgAAVQEAAP//AABfAAAAQwAAAEtYAAArAAAA//8AAF8AAABC +AAAAGFkAACoAAAABAAAAYAAAACoAAACpVwAALwAAAP//AABfAAAASQAAAA4IAABuAAAA//8AAF8A +AABJAAAAq1kAAG0AAAAEAAAAXwAAAFIAAABLWAAAdgAAAAUAAABgAAAAHgAAAKlXAABdAAAABAAA +AF8AAABVAAAAx1kAAIMAAAAHAAAAYAAAADQAAACpVwAAhAAAAAQAAABfAAAAVgAAAOJZAACQAAAA +CQAAAGAAAAAzAAAAqVcAAJEAAAD//wAAXwAAAEwAAAAOCAAArgAAAP//AABfAAAATAAAAOJZAACt +AAAADAAAAGAAAAAzAAAAqVcAALYAAAD//wAAaAAAAIsBAADZXgAA0AAAAP//AABoAAAAYQEAADMN +AABiAAAA//8AAGgAAACOAQAA6gUAANUAAAACAAAAIgAAADAAAAD3BQAA1gAAAP//AABoAAAAjwEA +AJIsAADlAAAA//8AAGgAAACTAQAAJQYAACMBAAAFAAAAIgAAAHAAAAA0BgAAJAEAAP//AABoAAAA +oAEAADgOAABfAQAA//8AAGgAAAB6AQAA6gUAAL0BAAAIAAAAIgAAADAAAAD3BQAA4QEAAP//AABo +AAAAfAEAACUGAAD2AQAACgAAACIAAABwAAAANAYAAA4CAAD//wAAaAAAAIABAABOXgAALgIAAP// +AABoAAAAhAEAAE5eAABXAgAA//8AAGgAAADkAQAAtgQAAG4AAAAAAAAAOgAAAJoCAADOBAAAbwAA +AP//AABoAAAA9AEAANleAAC8AAAA//8AAGgAAADFAQAAgyoAANcAAAD//wAAaAAAAPUBAADqBQAA +FwEAAAQAAAAiAAAAMAAAAPcFAAAYAQAA//8AAGgAAAD3AQAAJQYAAEgBAAAGAAAAIgAAAHAAAAA0 +BgAASQEAAP//AABoAAAA5gEAADgOAABzAQAA//8AAGgAAADXAQAA6gUAAO8BAAAJAAAAIgAAADAA +AAD3BQAAEwIAAP//AABoAAAA2QEAACUGAAAoAgAACwAAACIAAABwAAAANAYAACkCAAD//wAAaAAA +AN8BAABOXgAAgAIAAP//AABoAAAA+QQAAONAAADoAAAA//8AAGgAAAAaBQAA40AAAEACAAAAAAAA +bwAAAPgDAACPBAAA8QAAAAAAAABvAAAA/QMAAI8EAAB8AQAA//8AAGgAAAAFBQAAHGAAALsBAAAE +AAAAbwAAADgEAABoBQAAvAEAAP//AABoAAAAMgUAAONAAACKAwAAAQAAAG8AAAD4AwAAjwQAAEkC +AAABAAAAbwAAAP0DAACPBAAA0wIAAP//AABoAAAAJQUAABxgAAAxAwAACQAAAG8AAAA4BAAAaAUA +ABoDAAAGAAAAbwAAAPgDAACPBAAAlgMAAAYAAABvAAAA/QMAAI8EAAARBAAA//8AAGgAAAA1BQAA +jwQAACIEAAD//wAAcgAAAM0BAAC1DQAAgAAAAP//AAByAAAA0wEAANgNAACoAAAA//8AAHIAAADA +AQAAtQ0AAO0AAAD//wAAcgAAAMkBAADYDQAAFQEAAP//AAByAAAAtwEAALUNAABeAQAA//8AAHIA +AAC8AQAA2A0AAIgBAAD//wAAcgAAAAECAAAyYwAAKQMAAP//AAByAAAA6AEAAAgVAAAzAwAA//8A +AHIAAADpAQAA6gUAAEMDAAAIAAAAIgAAADAAAAD3BQAAUQMAAP//AAByAAAA6wEAACUGAABmAwAA +CgAAACIAAABwAAAANAYAAGcDAAD//wAAcgAAAO8BAADYDQAAkgMAAP//AAByAAAACAIAANgNAAD2 +AgAA//8AACUAAADEAwAAtQ0AAEoBAAD//wAAJQAAAM8DAADGDQAAxAEAAP//AAAlAAAA+wMAADMN +AAAHAgAA//8AACUAAAAFBAAAMw0AABkCAAD//wAAJQAAAAcEAAAzDQAAIgIAAP//AAAlAAAADwQA +ANgNAABTAgAA//8AACUAAAAUBAAA6Q0AANQCAAD//wAAJQAAACYEAAD+DQAAogMAAP//AAAlAAAA +KAQAAP4NAADIAwAA//8AACUAAAArBAAAEQ4AAPcDAAAJAAAAOgAAABsCAAAnDgAA+AMAAP//AAAl +AAAALQQAAOkNAABxBAAA//8AACUAAAA9BAAAOA4AANcFAAD//wAAJQAAAHsEAADYDQAAQAcAAP// +AAAlAAAAlQQAAE4OAAAqCAAA//8AADsAAACpAQAAGCYAADgAAAD//wAAOwAAAKkBAAAYJgAAOQAA +AP//AAA7AAAAuwEAAEEmAAD0AAAAAgAAADsAAABYAQAAXiYAAMAAAAACAAAAOwAAAFgBAAByJgAA +wAAAAP//AAA7AAAAzAEAAEEmAACdAQAABQAAADsAAABYAQAAciYAAKoBAAD//wAAOwAAALABAABB +JgAABAIAAAcAAAA7AAAAWAEAAF4mAADgAQAABwAAADsAAABYAQAAciYAAOABAAD//wAAOwAAAOUB +AAD8MAAAvQIAAAUAAAA7AAAAWAEAAF4mAACqAQAA//8AADsAAADVAQAAQSYAAPIEAAAMAAAAOwAA +AFgBAABeJgAANQUAAAwAAAA7AAAAWAEAAHImAAA1BQAA//8AAFMAAAA7CQAA6gUAABkAAAAAAAAA +IgAAADAAAAD3BQAAGAAAAP//AABTAAAAPgkAACUGAAAwAAAAAgAAACIAAABwAAAANAYAADEAAAD/ +/wAAUwAAAE8JAADsGwAASQAAAP//AABTAAAAUAkAACUGAABVAAAABQAAACIAAABwAAAANAYAAFYA +AAD//wAAUwAAAEEJAADqBQAAQAAAAAcAAAAiAAAAMAAAAPcFAAB2AAAA//8AAFMAAABMCQAA6gUA +AIcAAAAJAAAAIgAAADAAAAD3BQAAiAAAAP//AABTAAAAQwkAAMtJAAClAAAA//8AAFMAAABFCQAA +JQYAALYAAAAMAAAAIgAAAHAAAAA0BgAAygAAAP//AABTAAAARwkAAMtJAADMAAAA//8AAFMAAADW +CQAAHEoAABgAAAD//wAAUwAAANsJAAD2GAAAVgAAAAEAAAAzAAAAOwUAAEUbAACAAAAAAQAAADMA +AAA+BQAAEhkAAJMAAAD//wAAUwAAAOUJAADqBQAAGwEAAAQAAAAiAAAAMAAAAPcFAAAcAQAA//8A +AFMAAADsCQAAJQYAAF4BAAAGAAAAIgAAAHAAAAA0BgAAXwEAAP//AABTAAAABQoAAD9KAABKAgAA +//8AAFMAAAD+CQAAJQYAAAUCAAAJAAAAIgAAAHAAAAA0BgAABgIAAP//AABTAAAABwoAACUGAABa +AgAACwAAACIAAABwAAAANAYAAFsCAAD//wAAUwAAAPcJAAAlBgAAgQIAAA0AAAAiAAAAcAAAADQG +AACCAgAA//8AAFMAAADyDwAAhkwAADcAAAAAAAAAUwAAAFsNAACUTAAALAAAAAEAAABbAAAANQEA +AH4IAAA4AAAAAAAAAFMAAABcDQAApUwAAD8AAAADAAAAWwAAABQBAADmBgAAQwAAAP//AABTAAAA +8w8AAOoFAABNAAAABQAAACIAAAAwAAAA9wUAAE4AAAD//wAAUwAAAPUPAAA5TAAAdQAAAP//AABT +AAAA+g8AAMpMAAC0AAAACAAAAFMAAAB7FgAAAh8AALUAAAAJAAAAUwAAAFgYAADmBgAA1QAAAAkA +AABTAAAAWhgAAOYGAADeAAAACQAAAFMAAABcGAAA5gYAAOwAAAD//wAAUwAAAAYQAAAlBgAAsAAA +AA0AAAAiAAAAcAAAADQGAAAMAQAA//8AAGEAAABJAQAAZFoAABgAAAAAAAAAUAAAAFgBAABnHQAA +IAAAAP//AABhAAAASgEAAH1aAABeAAAAAgAAAF8AAAAqAAAAGFkAAGcAAAADAAAAYAAAACoAAACp +VwAAaAAAAP//AABhAAAASgEAAJZaAABeAAAABQAAAF8AAAAsAAAAS1gAAHAAAAD//wAAYQAAAEwB +AAAOCAAArAAAAP//AABhAAAATAEAAKtZAACrAAAACAAAAF8AAABSAAAAS1gAALQAAAAJAAAAYAAA +AB4AAACpVwAAmwAAAAgAAABfAAAAVQAAAMdZAAC8AAAACwAAAGAAAAA0AAAAqVcAAL0AAAAIAAAA +XwAAAFYAAADiWQAAyQAAAA0AAABgAAAAMwAAAKlXAADKAAAA//8AACcAAABtAAAAoA4AAG8AAAD/ +/wAAJwAAAHEAAADSEAAApwAAAAEAAAAmAAAAvgAAAH8QAADeAAAA//8AACcAAAByAAAAlxEAAOoA +AAD//wAAJwAAAKUAAAAUEQAAKAEAAP//AAAnAAAAdQAAAI8EAADNAAAA//8AACcAAACbAAAAlxEA +ADABAAD//wAAJwAAAJsAAABoEAAAMAEAAAcAAAAmAAAALwQAAH8QAABAAQAA//8AACcAAACbAAAA +rxEAADABAAD//wAAJwAAAJAAAAAEEAAAlgEAAAoAAAAmAAAA0AAAAI8EAACpAQAA//8AACcAAAB+ +AAAAzhEAAMABAAD//wAAJwAAAIgAAACPBAAA/wEAAP//AAAnAAAApwAAAI8EAABuAgAA//8AACcA +AACuAAAAjwQAABcCAAD//wAALQAAAFEAAAC+FgAAGAAAAP//AAAtAAAAaQAAAIkXAABaAAAA//8A +AC0AAABtAAAAoBcAAIYAAAD//wAALQAAAHMAAADBFwAAWwEAAP//AAAtAAAAdwAAAOQXAACAAQAA +BAAAADgAAADZAAAABhgAAIIBAAD//wAALQAAAHoAAAAtGAAA7gEAAAYAAAA4AAAA8gAAAEwYAAAs +AgAA//8AAC0AAACHAAAAbxgAAGkDAAD//wAALQAAAIsAAADkFwAAjAMAAAkAAAA4AAAA2QAAAAYY +AACOAwAA//8AAC0AAACWAAAADhcAACMEAAD//wAALQAAAJIAAAAtGAAAMQQAAAwAAAA4AAAA8gAA +AEwYAABvBAAA//8AAC0AAACaAAAALRgAAFsFAAAOAAAAOAAAAPIAAABMGAAAmwUAAP//AAA7AAAA +AAIAABgmAABCAAAA//8AADsAAAAAAgAAGCYAAEMAAAD//wAAOwAAAAECAACGJgAARAAAAP//AAA7 +AAAAAQIAAIYmAABFAAAA//8AADsAAAALAgAAQSYAAJIAAAAEAAAAOwAAAFgBAABeJgAAoAAAAAQA +AAA7AAAAWAEAAHImAACgAAAA//8AADsAAAAGAgAAQSYAAEUBAAAHAAAAOwAAAFgBAABeJgAASgEA +AAcAAAA7AAAAWAEAAHImAABKAQAA//8AADsAAAAPAgAAQSYAAIUCAAAKAAAAOwAAAFgBAAByJgAA +CgIAAAoAAAA7AAAAWAEAAF4mAAAKAgAA//8AADsAAAATAgAAQSYAAJ4CAAANAAAAOwAAAFgBAABe +JgAAqAIAAA0AAAA7AAAAWAEAAHImAACoAgAA//8AAFMAAACOCQAAtQ0AACUAAAD//wAAUwAAAI8J +AADqBQAALAAAAAEAAAAiAAAAMAAAAPcFAAA2AAAA//8AAFMAAACfCQAAD0oAAGAAAAD//wAAUwAA +AK4JAAAlBgAAlQAAAAQAAAAiAAAAcAAAADQGAACWAAAA//8AAFMAAAC4CQAA2A0AAM8AAAADAAAA +UwAAAGwWAADLSQAAYQAAAP//AABTAAAAuwkAACUGAAAKAQAACAAAACIAAABwAAAANAYAAAsBAAD/ +/wAAUwAAAMIJAAAcSgAAQgEAAP//AABTAAAAkwkAACUGAABSAQAACwAAACIAAABwAAAANAYAAFMB +AAD//wAAUwAAAJsJAADYDQAAhAEAAP//AABTAAAAxwkAANNHAABdAAAA//8AAFMAAADLCQAA2A0A +AOkBAAD//wAAUwAAAKYRAAD+BgAAJgAAAP//AABTAAAApxEAABUHAAA2AAAA//8AAFMAAACqEQAA +F1AAAFcAAAACAAAAUwAAAE4YAADmBgAAZAAAAAIAAABTAAAAUBgAAOYGAAB6AAAA//8AAFMAAACs +EQAAF1AAAHwAAAAFAAAAUwAAAE4YAADmBgAAiQAAAAUAAABTAAAAUBgAAOYGAACgAAAA//8AAFMA +AACwEQAA6gUAAKYAAAAIAAAAIgAAADAAAAD3BQAApwAAAP//AABTAAAAsREAANUdAAC0AAAACgAA +AFMAAACVGAAA7h0AAMsAAAD//wAAUwAAALIRAADVHQAA5wAAAAwAAABTAAAAlRgAAO4dAAADAQAA +//8AAFMAAAC0EQAAJQYAACkBAAAOAAAAIgAAAHAAAAA0BgAAKgEAAP//AAASAAAA0QAAAGkcAAAY +AAAA//8AABIAAADfAAAACU0AAGYAAAD//wAAEgAAAOIAAACGTAAAqgAAAAIAAABTAAAAWw0AAJRM +AACfAAAAAwAAAFsAAAA1AQAAfggAAKsAAAACAAAAUwAAAFwNAAClTAAAsgAAAAUAAABbAAAAFAEA +AOYGAAC2AAAA//8AABIAAADjAAAA6gUAAMAAAAAHAAAAIgAAADAAAAD3BQAAwQAAAP//AAASAAAA +5AAAAMpMAADOAAAACQAAAFMAAAB7FgAAAh8AAM8AAAAKAAAAUwAAAFgYAADmBgAA7wAAAAoAAABT +AAAAWhgAAOYGAAD4AAAACgAAAFMAAABcGAAA5gYAAAMBAAD//wAAEgAAAOUAAAAlBgAAEQEAAA4A +AAAiAAAAcAAAADQGAAASAQAA//8AAFMAAADuDAAACBUAAHgAAAD//wAAUwAAABwNAADqBQAAuwEA +AAEAAAAiAAAAMAAAAPcFAAC8AQAA//8AAFMAAAAeDQAAJQYAAOoBAAADAAAAIgAAAHAAAAA0BgAA +6wEAAP//AABTAAAAMQ0AADlMAAByAgAA//8AAFMAAAA1DQAA6gUAAKsCAAAGAAAAIgAAADAAAAD3 +BQAArAIAAP//AABTAAAANg0AADlMAADHAgAA//8AAFMAAAA7DQAAAh8AANwCAAAJAAAAUwAAAFgY +AADmBgAA/AIAAAkAAABTAAAAWhgAAOYGAAAFAwAACQAAAFMAAABcGAAA5gYAABMDAAD//wAAUwAA +AD0NAAAlBgAAIQMAAA0AAAAiAAAAcAAAADQGAAAiAwAA//8AAFMAAAA5DQAAJQYAADkDAAAPAAAA +IgAAAHAAAAA0BgAAOgMAAP//AABTAAAAwg0AALRCAADPAAAA//8AAFMAAADGDQAAhkwAAEoAAAAB +AAAAUwAAAFsNAACUTAAAPwAAAAIAAABbAAAANQEAAH4IAABLAAAAAQAAAFMAAABcDQAApUwAAFIA +AAAEAAAAWwAAABQBAADmBgAAVgAAAP//AABTAAAAxw0AAOoFAABgAAAABgAAACIAAAAwAAAA9wUA +AGEAAAD//wAAUwAAAMgNAADKTAAAbgAAAAgAAABTAAAAexYAAAIfAABvAAAACQAAAFMAAABYGAAA +5gYAAI8AAAAJAAAAUwAAAFoYAADmBgAAmAAAAAkAAABTAAAAXBgAAOYGAACjAAAA//8AAFMAAADJ +DQAAJQYAALEAAAANAAAAIgAAAHAAAAA0BgAAsgAAAAAAAABTAAAA7gIAAGcdAAAYAAAAAAAAAFMA +AADvAgAAZx0AAGkBAAD//wAAUwAAAFgRAADQBgAAZQAAAAAAAABTAAAAkBgAAOYGAAB7AAAA//8A +AFMAAABhEQAAFQcAALsAAAD//wAAUwAAAGQRAAAXUAAA4gAAAAMAAABTAAAAThgAAOYGAADvAAAA +AwAAAFMAAABQGAAA5gYAAAcBAAD//wAAUwAAAGYRAAAXUAAACQEAAAYAAABTAAAAThgAAOYGAAAW +AQAABgAAAFMAAABQGAAA5gYAACwBAAD//wAAUwAAAGoRAADqBQAAMgEAAAkAAAAiAAAAMAAAAPcF +AAAzAQAA//8AAFMAAABrEQAA1R0AAEUBAAALAAAAUwAAAJUYAADuHQAAXAEAAP//AABTAAAAbBEA +ANUdAAB4AQAADQAAAFMAAACVGAAA7h0AAI8BAAD//wAAUwAAAG4RAAAlBgAAtQEAAA8AAAAiAAAA +cAAAADQGAAC2AQAA//8AAFMAAAAwEwAAsFEAAEEAAAAAAAAAUwAAAIYWAAAXUAAAQgAAAAEAAABT +AAAAThgAAOYGAABRAAAAAQAAAFMAAABQGAAA5gYAAGYAAAD//wAAUwAAADMTAACwUQAAfAAAAAQA +AABTAAAAhhYAABdQAAB+AAAABQAAAFMAAABOGAAA5gYAAI0AAAAFAAAAUwAAAFAYAADmBgAAogAA +AP//AABTAAAANxMAAAgVAADOAAAA//8AAFMAAAA8EwAA6gUAAN4AAAAJAAAAIgAAADAAAAD3BQAA +7AAAAP//AABTAAAAPRMAAOoFAADyAAAACwAAACIAAAAwAAAA9wUAAAMBAAD//wAAUwAAAEMTAAAl +BgAAeQEAAA0AAAAiAAAAcAAAADQGAAB6AQAA//8AAFMAAABEEwAAJQYAAIUBAAAPAAAAIgAAAHAA +AAA0BgAAhgEAAP//AAArAAAANgIAAEoUAABWAAAAAAAAADoAAAB8AgAAzgQAADkAAAD//wAAKwAA +AEYCAAA4DgAAlAAAAP//AAArAAAARgIAABoUAACUAAAA//8AACsAAABRAgAA4QQAAAMBAAD//wAA +KwAAAFACAAAIFQAAtAAAAP//AAArAAAAOQIAANoDAABZAQAA//8AACsAAAA/AgAA2gMAAMIBAAD/ +/wAAKwAAAF4CAAAdFQAANgIAAAgAAAArAAAAEQIAAPkEAAAyAgAA//8AACsAAABhAgAAFwMAAEsC +AAD//wAAKwAAAGUCAAAPBQAAkQIAAAsAAAArAAAArQEAACUFAADZAgAA//8AACsAAABUAgAAHRUA +ADoDAAANAAAAKwAAABECAAD5BAAANgMAAP//AAArAAAAVgIAABcDAABHAwAA//8AACsAAABaAgAA +DwUAAI4DAAAQAAAAKwAAAK0BAAAlBQAA0QMAAP//AABmAAAA1AAAAKsQAADWAAAAAAAAAEIAAAAQ +AAAA/g0AAKAAAAAAAAAAQgAAABIAAAD+DQAA4wAAAAAAAABCAAAAGAAAADMNAAAqAQAA//8AAGYA +AADaAAAAqxAAAF8BAAAEAAAAQgAAABAAAAD+DQAAhAEAAAQAAABCAAAAEgAAAP4NAADDAQAABAAA +AEIAAAAYAAAAMw0AAAUCAAD//wAAZgAAAN0AAAD2XQAAVgIAAP//AABmAAAA5wAAAKsQAABvAgAA +CQAAAEIAAAAQAAAA/g0AAKACAAAJAAAAQgAAABIAAAD+DQAA4wIAAAkAAABCAAAAGAAAADMNAAAl +AwAA//8AAGYAAADuAAAAqxAAAOADAAANAAAAQgAAABAAAAD+DQAAsAMAAA0AAABCAAAAEgAAAP4N +AADsAwAADQAAAEIAAAAYAAAAMw0AACgEAAD//wAAZgAAAAgBAACPBAAAmwQAAP//AAAzAAAAQAIA +ALUNAAAcAAAA//8AADMAAABFAgAA2A0AAFAAAAD//wAAMwAAAFICAABODgAAAAEAAP//AAAzAAAA +QgIAANgNAACHAAAA//8AADMAAABYAgAAVRoAAJkBAAD//wAAMwAAAFoCAABODgAA6wEAAP//AAAz +AAAAbgIAAFUaAAB3AgAA//8AADMAAABvAgAAVRoAAKUCAAD//wAAMwAAAHICAABoGgAAwgIAAP// +AAAzAAAAWwIAAH0aAAANAwAA//8AADMAAACLAgAAvwcAAM8DAAD//wAAMwAAAI8CAACQGgAA6wMA +AP//AAAzAAAAtQIAAKgaAACRBAAA//8AADMAAAC3AgAAuxoAANgEAAD//wAAMwAAAM4CAAC1DQAA +FQUAAP//AAAzAAAA3AIAAH0aAABcBQAA//8AADMAAADdAgAA2A0AAG0FAAD//wAAMwAAAOICAADT +GgAAtwUAAP//AAAzAAAA5QIAAH0aAADFBQAA//8AAD0AAAB5AAAAGCYAABoAAAD//wAAPQAAAJQA +AAAYJgAAiQAAAP//AAA9AAAAlQAAAEEmAACKAAAAAgAAADsAAABYAQAAXiYAAKAAAAACAAAAOwAA +AFgBAAByJgAAoAAAAP//AAA9AAAAmAAAAIYmAADJAAAA//8AAD0AAACYAAAAHzQAAMkAAAAGAAAA +PgAAAG8BAAA9NAAA1gAAAP//AAA9AAAAlwAAAH0ZAAC5AAAA//8AAD0AAACZAAAAPTQAAPEAAAD/ +/wAAPQAAAIAAAABBJgAALQEAAAoAAAA7AAAAWAEAAF4mAAAyAQAACgAAADsAAABYAQAAciYAADIB +AAD//wAAPQAAAIEAAACGJgAAYAEAAP//AAA9AAAAhwAAAB80AADVAQAADgAAAD4AAABvAQAAPTQA +AI8BAAD//wAAPQAAAIYAAAB9GQAArgEAAP//AAA9AAAAhgAAAJ0mAACuAQAA//8AAD0AAACIAAAA +PTQAANgBAAD//wAAUwAAAAEUAAD2UQAAlQYAAP//AABTAAAAixMAABNSAABQAAAA//8AAFMAAACP +EwAAvwcAAHwAAAD//wAAUwAAAJsTAADqBQAACQEAAAMAAAAiAAAAMAAAAPcFAAAKAQAA//8AAFMA +AACzEwAAJQYAALACAAAFAAAAIgAAAHAAAAA0BgAAsQIAAP//AABTAAAAvRMAADADAABlAwAA//8A +AFMAAADBEwAACBUAAMQDAAD//wAAUwAAANATAAAJTQAA9wMAAP//AABTAAAA6xMAAOoFAAA+BQAA +CgAAACIAAAAwAAAA9wUAAD8FAAD//wAAUwAAAO8TAAAlBgAAsAUAAAwAAAAiAAAAcAAAADQGAACx +BQAA//8AAFMAAAD5EwAAHEoAADcGAAD//wAAUwAAAPwTAAAPSgAAZQYAAP//AABTAAAA/BMAAH4I +AACGBgAA//8AAFMAAAD9EwAA00cAAIsGAAAAAAAAUwAAAAcZAAArUgAAxQYAAP//AAAzAAAAjQMA +AKgaAAApAAAA//8AADMAAACQAwAAvwcAAIcAAAD//wAAMwAAAJIDAAC1DQAAngAAAP//AAAzAAAA +wwMAAHcbAADVAQAA//8AADMAAADVAwAAvwcAAGUCAAD//wAAMwAAANoDAAD5GgAAxgIAAP//AAAz +AAAA9wMAAOoFAACPBAAABgAAACIAAAAwAAAA9wUAAJAEAAD//wAAMwAAAPoDAAAlBgAArwQAAAgA +AAAiAAAAcAAAADQGAACwBAAA//8AADMAAAAFBAAAiRcAAMsEAAD//wAAMwAAAAYEAAAGGAAA1AQA +AP//AAAzAAAAIQQAAC0YAABSBQAADAAAADgAAADyAAAATBgAAJAFAAD//wAAMwAAACwEAACLGwAA +yQYAAP//AAAzAAAASwQAAH0aAACJDAAA//8AADMAAABMBAAAfRoAAAUHAAD//wAAMwAAAE8EAADY +DQAAFgcAAP//AAAzAAAASAQAAJsbAABhDAAAEgAAAFIAAABQAAAAJQYAAHsMAAATAAAAIgAAAHAA +AAA0BgAAfAwAAP//AAA7AAAAqAIAAHYxAABmAQAA//8AADsAAAAGAwAAQSYAAAcCAAABAAAAOwAA +AFgBAABeJgAA4AEAAAEAAAA7AAAAWAEAAHImAADgAQAA//8AADsAAAARAwAAnSYAAIYCAAD//wAA +OwAAABYDAACdJgAAUwIAAP//AAA7AAAACwMAAJIxAAAEAwAA//8AADsAAAALAwAAqjEAAAQDAAD/ +/wAAOwAAAAsDAAArJgAABAMAAP//AAA7AAAAwgIAAMAxAAC5BAAA//8AADsAAADEAgAAkjEAADYF +AAD//wAAOwAAANACAAArJgAAZwUAAP//AAA7AAAA3gIAAKoxAADLBQAA//8AADsAAADoAgAAwDEA +AKwEAAD//wAAOwAAAPMCAACSMQAAMQEAAP//AAA7AAAA8wIAACsmAAAxAQAA//8AADsAAADzAgAA +qjEAADEBAAD//wAAOwAAAOgCAADcMQAArAQAAP//AAA7AAAA+QIAAJIxAAAhCgAA//8AADsAAAD5 +AgAAKyYAACEKAAD//wAAOwAAAPkCAACqMQAAIQoAAP//AAAnAAAAdwEAAD8SAAAmAAAAAAAAACYA +AABOBAAA5RAAAEQAAAAAAAAAJgAAAFEEAAB/EAAA+QAAAP//AAAnAAAAeAEAAAIRAABkAAAA//8A +ACcAAAB/AQAAjwQAAJEAAAD//wAAJwAAAIABAACPBAAApwAAAP//AAAnAAAAgQEAAI8EAACwAAAA +//8AACcAAACDAQAA5RAAAMQAAAD//wAAJwAAAIcBAACPBAAAxgAAAP//AAAnAAAAiAEAAI8EAADg +AAAA//8AACcAAACJAQAAjwQAAOwAAAD//wAAJwAAAIwBAAAEEAAASwEAAAsAAAAmAAAA0AAAAI8E +AAA7AQAA//8AACcAAACNAQAAjwQAAFUBAAD//wAAJwAAAI4BAACPBAAAVgEAAP//AAAnAAAAwQEA +AI8EAAB5AQAA//8AACcAAADEAQAAjwQAAIQBAAD//wAAJwAAAI8BAACPBAAAYgEAAP//AAAnAAAA +jwEAAI8EAABiAQAA//8AACcAAACRAQAAzhEAAN0BAAD//wAAJwAAAJkBAADlEAAADQIAAP//AAAn +AAAAqAEAAI8EAADIAgAA//8AACcAAACpAQAAjwQAANECAAD//wAANgAAADcCAAApJQAARgAAAAAA +AABAAAAAKgAAAEAlAAA9AAAAAAAAAEAAAAAvAAAAWSUAAD8AAAD//wAANgAAAGYCAAAYJgAAnAAA +AP//AAA2AAAAZwIAACsmAAB8AAAA//8AADYAAABqAgAAQSYAAFYBAAAFAAAAOwAAAFgBAABeJgAA +IAEAAAUAAAA7AAAAWAEAAHImAAAgAQAA//8AADYAAABqAgAAhiYAAFYBAAD//wAANgAAAJsCAAAp +JQAAFAIAAAkAAABAAAAALwAAAFklAAB6AgAACQAAAEAAAAAqAAAAQCUAAEMCAAD//wAANgAAAHUC +AACdJgAA2wEAAP//AAA2AAAAtwIAAJ0mAAArAgAA//8AADYAAACcAgAAryYAAJICAAAOAAAANgAA +AFwCAAAlBgAAVQIAAA8AAAAiAAAAcAAAADQGAACkAgAA//8AADYAAACkAgAA1iYAAM4CAAARAAAA +NgAAAFcCAADqBQAA3wIAABIAAAAiAAAAMAAAAPcFAADuAgAA//8AADYAAACtAgAAQSYAAA4DAAAU +AAAAOwAAAFgBAABeJgAAIAMAABQAAAA7AAAAWAEAAHImAAAgAwAA//8AADoAAAB8BAAAZC4AAFgA +AAD//wAAOgAAAHcEAAAIFQAANQAAAP//AAA6AAAAfQQAAOoFAABfAAAAAgAAACIAAAAwAAAA9wUA +AGEAAAD//wAAOgAAAH8EAAAlBgAAhgAAAAQAAAAiAAAAcAAAADQGAACHAAAA//8AADoAAACFBAAA +fy4AAOYAAAD//wAAOgAAAJAEAADqBQAA/QAAAAcAAAAiAAAAMAAAAPcFAAAcAQAA//8AADoAAACc +BAAAJQYAAGkBAAAJAAAAIgAAAHAAAAA0BgAAagEAAP//AAA6AAAAuwQAACUGAAC/AQAACwAAACIA +AABwAAAANAYAAMABAAD//wAAOgAAAMAEAACeLgAAtwEAAA0AAAA6AAAAFwYAALQuAAAqAgAA//8A +ADoAAADFBAAA0S4AAHkCAAD//wAAOgAAAMkEAAC0LgAApwIAAP//AAA6AAAAzgQAAL4WAADAAgAA +//8AADoAAADtBAAAtC4AAGoDAAD//wAAOgAAABYFAAA4DgAAdwMAAP//AAA6AAAA9AQAALoPAACb +AwAA//8AADoAAAAeBQAAyyEAALsEAAD//wAAOgAAAB4FAAA4DgAAvAQAABUAAAA6AAAAtAIAAM4E +AADGBAAA//8AACsAAABWBgAAJQUAADcAAAD//wAAKwAAAKYGAAANFgAAlQAAAP//AAArAAAArgYA +AA0WAACqAAAA//8AACsAAAAGBwAAHxYAAMgAAAD//wAAKwAAAAkHAAAlBQAACQEAAP//AAArAAAA +HQcAAB8WAAAmAQAA//8AACsAAAAgBwAAJQUAAGsBAAD//wAAKwAAAEYGAAAlBQAA6QEAAP//AAAr +AAAATwYAACUFAAAbAgAA//8AACsAAACOBgAAJQUAAHkCAAD//wAAKwAAACgHAAAlBQAAuwIAAP// +AAArAAAAKgcAACUFAADRAgAA//8AACsAAAARBwAAJQUAAAcDAAD//wAAKwAAABMHAAAlBQAADwMA +AP//AAArAAAAsgYAAA0WAAB1AwAA//8AACsAAADwBgAAJQUAAH8EAAD//wAAKwAAAOkGAAAlBQAA +nAQAAP//AAArAAAAqgYAAA0WAAAqBQAA//8AACsAAACCBgAAJQUAAFEFAAD//wAAKwAAAGYGAAAl +BQAAnAUAAP//AAArAAAAaQYAACUFAACqBQAA//8AACsAAAByBgAAJQUAAN0FAAD//wAAKwAAAHgG +AAAlBQAAAQYAAP//AAArAAAARQcAACUFAAAxBgAA//8AACsAAAA8BwAAJQUAAFkGAAD//wAAJgAA +AFICAACgDgAAfAAAAP//AAAmAAAAVgIAANIQAAC9AAAAAQAAACYAAAC+AAAAfxAAAAABAAD//wAA +JgAAAFcCAACXEQAADAEAAP//AAAmAAAAWwIAABQRAABPAQAA//8AACYAAABaAgAAjwQAAOwAAAD/ +/wAAJgAAAIcCAACXEQAAbQEAAP//AAAmAAAAhwIAAGgQAABtAQAABwAAACYAAAAvBAAAfxAAAIAB +AAD//wAAJgAAAIcCAACvEQAAbQEAAP//AAAmAAAAfAIAAAQQAACGAQAACgAAACYAAADQAAAAjwQA +AOoBAAD//wAAJgAAAGQCAADOEQAAKQIAAP//AAAmAAAAZgIAAI8EAAAwAgAA//8AACYAAABnAgAA +jwQAAD8CAAD//wAAJgAAAG4CAACPBAAAiAIAAP//AAAmAAAAbwIAAEYRAACiAgAA//8AACYAAAB2 +AgAA3hEAAA0DAAD//wAAJgAAAHkCAACPBAAAJQMAAP//AAAmAAAAkAIAAI8EAABsAwAA//8AACYA +AACRAgAAjwQAAG0DAAD//wAAJgAAAJUCAABGEQAAqQMAAP//AAAmAAAAlgIAAKAOAACwAwAA//8A +ACYAAACaAgAAZREAAAgEAAD//wAAJgAAAJsCAACgDgAADwQAAP//AAAmAAAApwIAAGURAAChBAAA +//8AAE4AAABRAwAA9EAAABwAAAAAAAAATgAAADwDAACPBAAAKAAAAP//AABOAAAAUgMAAPRAAABU +AAAAAgAAAE4AAAA8AwAAjwQAAGIAAAD//wAATgAAAFMDAAD0QAAApQAAAAQAAABOAAAAPAMAAI8E +AAC9AAAA//8AAE4AAABZAwAA9EAAADUBAAD//wAATgAAAFoDAAD0QAAAYgEAAP//AABOAAAAWwMA +APRAAACFAQAA//8AAE4AAABpAwAAQUAAACACAAAJAAAATgAAAIYBAACPBAAAIgIAAP//AABOAAAA +eQMAACdBAABzAgAABgAAAE4AAAA8AwAAjwQAAA4DAAAHAAAATgAAADwDAACPBAAAVwMAAAgAAABO +AAAAPAMAAI8EAAChAwAA//8AAE4AAABwAwAA9EAAALAEAAAPAAAATgAAADwDAACPBAAAvwUAAP// +AABOAAAAbgMAAPRAAABMAgAA//8AAE4AAABvAwAA9EAAAIQEAAARAAAATgAAADwDAACPBAAAEgUA +ABIAAABOAAAAPAMAAI8EAABqBQAA//8AAE4AAABhAwAA9EAAAF8GAAD//wAATgAAAGIDAAD0QAAA +dAYAAP//AABOAAAAYwMAAPRAAACJBgAAFQAAAE4AAAA8AwAAjwQAAKsGAAAWAAAATgAAADwDAACP +BAAA0AYAABcAAABOAAAAPAMAAI8EAAD0BgAA//8AACYAAABzBAAAPxIAACYAAAAAAAAAJgAAAE4E +AADlEAAARAAAAAAAAAAmAAAAUQQAAH8QAAAOAQAA//8AACYAAAB0BAAAAhEAAGQAAAD//wAAJgAA +AHsEAACPBAAAlAAAAP//AAAmAAAAfAQAAI8EAACqAAAA//8AACYAAAB9BAAAjwQAAK8AAAD//wAA +JgAAAH8EAADlEAAAzQAAAP//AAAmAAAAgwQAAI8EAADPAAAA//8AACYAAACEBAAAjwQAAOwAAAD/ +/wAAJgAAAIUEAACPBAAA9AAAAP//AAAmAAAAiAQAAAQQAABlAQAACwAAACYAAADQAAAAjwQAAFUB +AAD//wAAJgAAAIkEAACPBAAAbwEAAP//AAAmAAAAigQAAI8EAABwAQAA//8AACYAAADVBAAAjwQA +AJoBAAD//wAAJgAAANgEAACPBAAApQEAAP//AAAmAAAAiwQAAI8EAACDAQAA//8AACYAAACLBAAA +jwQAAIMBAAD//wAAJgAAAI0EAADOEQAABAIAAP//AAAmAAAAlQQAAEYRAAAgAgAA//8AACYAAACZ +BAAA5RAAAEcCAAD//wAAJgAAAJ0EAABsEgAAaAIAAP//AAAmAAAAwAQAAEYRAAAXBAAA//8AACYA +AACqBAAAFBEAAOYCAAD//wAAJgAAALwEAACPBAAAnAMAAP//AAAmAAAAvQQAAI8EAAChAwAA//8A +ACYAAADFBAAAZREAAKEEAAD//wAAUwAAAJwMAAD+BgAAGAAAAP//AABTAAAApwwAAGkcAABFAAAA +//8AAFMAAACyDAAA5gYAAJ4AAAD//wAAUwAAALMMAADmBgAApAAAAP//AABTAAAAvgwAAOoFAADa +AAAABAAAACIAAAAwAAAA9wUAANsAAAD//wAAUwAAAL8MAAAyHAAA6AAAAAYAAABTAAAAkhYAAEsc +AADpAAAABwAAAFMAAABlGAAAaRwAAPsAAAD//wAAUwAAAMAMAAAlBgAAOgEAAAkAAAAiAAAAcAAA +ADQGAAA7AQAA//8AAFMAAADBDAAADkwAAEgBAAD//wAAUwAAAMoMAAACHwAAtwEAAAwAAABTAAAA +XBgAAOYGAAB+AQAA//8AAFMAAADIDAAA7h0AANUAAAD//wAAUwAAAMkMAABKHwAAlgEAAAwAAABT +AAAAWBgAAOYGAADQAQAADAAAAFMAAABaGAAA5gYAAOQBAAD//wAAUwAAALwMAAAIFQAAsQAAAP// +AABTAAAAzQwAAOoFAAD5AQAAEwAAACIAAAAwAAAA9wUAAPoBAAD//wAAUwAAAM4MAAAyHAAABwIA +ABUAAABTAAAAkhYAAEscAAAIAgAAFgAAAFMAAABlGAAAaRwAABoCAAD//wAAUwAAAM8MAAAlBgAA +WQIAABgAAAAiAAAAcAAAADQGAABaAgAA//8AAFMAAADQDAAADkwAAGcCAAD//wAAUwAAANQMAADu +HQAAeAIAAP//AABTAAAAwxQAAOoFAAAZAAAAAAAAACIAAAAwAAAA9wUAABgAAAD//wAAUwAAAMYU +AAAlBgAAMAAAAAIAAAAiAAAAcAAAADQGAAAxAAAA//8AAFMAAABXFQAAJQYAAFYAAAAEAAAAIgAA +AHAAAAA0BgAAVwAAAP//AABTAAAA6xQAAL8HAADFAAAA//8AAFMAAADtFAAA6gUAACEBAAAHAAAA +IgAAADAAAAD3BQAAIgEAAP//AABTAAAA8xQAACUGAACYAQAACQAAACIAAABwAAAANAYAAJkBAAD/ +/wAAUwAAAAMVAADqBQAA5QEAAAsAAAAiAAAAMAAAAPcFAADmAQAA//8AAFMAAAAFFQAA7BsAAP4B +AAD//wAAUwAAAAwVAAAlBgAAYAEAAA4AAAAiAAAAcAAAADQGAAAuAgAA//8AAFMAAAAPFQAA6gUA +ABcBAAAQAAAAIgAAADAAAAD3BQAATQIAAP//AABTAAAAEhUAAL8HAABaAgAA//8AAFMAAAAaFQAA +vUcAAKkCAAD//wAAUwAAAB0VAAD+BgAA8gIAAP//AABTAAAASxUAAE4OAAByAwAA//8AAFMAAABM +FQAA6gUAAL8DAAAWAAAAIgAAADAAAAD3BQAAwAMAAP//AABTAAAATxUAANAGAADgAwAAGAAAAFMA +AACQGAAA5gYAAPIDAAD//wAAUwAAAFEVAAAlBgAABQQAABoAAAAiAAAAcAAAADQGAAAGBAAA//8A +ADsAAABnAwAAQCUAABoAAAD//wAAOwAAAGwDAABAJQAAUQAAAP//AAA7AAAAdgMAABgmAACNAAAA +//8AADsAAAB2AwAAGCYAAI4AAAD//wAAOwAAAHcDAACGJgAAjwAAAP//AAA7AAAAdwMAAIYmAACQ +AAAA//8AADsAAAB+AwAAQSYAAPMAAAAGAAAAOwAAAFgBAABeJgAAwAAAAAYAAAA7AAAAWAEAAHIm +AADAAAAA//8AADsAAAB+AwAAXzIAAPMAAAD//wAAOwAAAHsDAABBJgAAQgEAAAoAAAA7AAAAWAEA +AF4mAAAjAQAACgAAADsAAABYAQAAciYAACMBAAD//wAAOwAAAHsDAABfMgAAQgEAAP//AAA7AAAA +cwMAAEEmAACBAQAADgAAADsAAABYAQAAXiYAAGEBAAD//wAAOwAAAHIDAAAYJgAAWQEAAA4AAAA7 +AAAAWAEAAHImAABhAQAA//8AADsAAABzAwAAhiYAAIEBAAD//wAAOwAAAHMDAAB6MgAAgQEAABMA +AAA+AAAAXgEAAJYyAACOAQAA//8AADsAAACAAwAAQSYAAB4CAAAVAAAAOwAAAFgBAAByJgAADAIA +AP//AAA7AAAAgAMAALAyAAAeAgAAFQAAADsAAABYAQAAXiYAAAwCAAD//wAAOwAAAIIDAABBJgAA +egIAABkAAAA7AAAAWAEAAF4mAABgAgAAGQAAADsAAABYAQAAciYAAGACAAD//wAAOwAAAIIDAABf +MgAAegIAAP//AAArAAAAWAMAAOEEAACLAAAAAAAAACsAAAA/AQAAzgQAADwAAAD//wAAKwAAAC0F +AADhBAAAMAQAAP//AAArAAAA0AMAAM4EAAC5AAAA//8AACsAAABaBAAAaAUAAHcBAAD//wAAKwAA +APMDAABoBQAA+gAAAP//AAArAAAAuQMAAA8FAADEAgAA//8AACsAAAC5AwAADwUAAMQCAAAHAAAA +KwAAAK0BAAAlBQAArgIAAP//AAArAAAAwQMAAA8FAAAKAwAACQAAACsAAACtAQAAJQUAAAUDAAD/ +/wAAKwAAAEEFAAAPBQAAuwQAAP//AAArAAAAQQUAAA8FAAC7BAAADAAAACsAAACtAQAAJQUAAIsE +AAD//wAAKwAAAEMFAABoBQAA1gQAAP//AAArAAAAUgUAAGgFAABoBQAA//8AACsAAABcBQAADwUA +AP4FAAAQAAAAKwAAAK0BAAAlBQAA4wUAAP//AAArAAAAWwUAAGgFAADMBQAA//8AACsAAABcBQAA +DwUAAP4FAAD//wAAKwAAAGAEAAAlBQAAYAYAAP//AAArAAAAkwQAACUFAAAIBwAA//8AACsAAACx +BAAAJQUAAF4HAAD//wAAKwAAAP8EAAAlBQAAqgcAAP//AAArAAAA2gQAACUFAAA4CAAA//8AACsA +AADxBAAAJQUAAIIIAAD//wAAKwAAAM8EAAAlBQAA+wcAAP//AAArAAAAFAUAACUFAABbCQAA//8A +ACsAAAAYBQAAJQUAAJUJAAD//wAAKwAAADkEAAAlBQAAdwoAAP//AABTAAAAmQoAAAgVAAAuAAAA +//8AAFMAAAC0CgAA6gUAAAMBAAABAAAAIgAAADAAAAD3BQAABAEAAP//AABTAAAAtgoAACUGAAAt +AQAAAwAAACIAAABwAAAANAYAAC4BAAD//wAAUwAAAMMKAAC9RwAAVAEAAP//AABTAAAAxAoAAP4G +AAB9AQAA//8AAFMAAADxCgAA9hgAANwBAAAHAAAAMwAAADsFAABFGwAAbgIAAAcAAAAzAAAAPgUA +ABIZAACDAgAA//8AAFMAAADyCgAAaCMAAL0CAAD//wAAUwAAABkLAADqBQAAQAMAAAsAAAAiAAAA +MAAAAPcFAABBAwAA//8AAFMAAAAbCwAAJQYAAGgDAAANAAAAIgAAAHAAAAA0BgAAaQMAAP//AABT +AAAAJwsAACUGAACmAwAADwAAACIAAABwAAAANAYAAKcDAAD//wAAUwAAAG0LAAC9RwAAuQQAAP// +AABTAAAAeAsAAL8HAAAvBQAA//8AAFMAAACFCwAAvwcAALoFAAD//wAAUwAAAIYLAAD+BgAAxAUA +AP//AABTAAAAjAsAAOoFAADlBQAAFQAAACIAAAAwAAAA9wUAAOYFAAD//wAAUwAAAI4LAAAlBgAA +AAYAABcAAAAiAAAAcAAAADQGAAABBgAA//8AAFMAAACTCwAA/gYAAC0GAAD//wAAUwAAAMUKAAAV +BwAAggYAABoAAABTAAAAnRgAAGkcAACDBgAA//8AAFMAAACUCwAAFQcAAKYHAAAcAAAAUwAAAJ0Y +AABpHAAApwcAAP//AABTAAAAogsAAL1HAACsBAAA//8AAFMAAAAgCwAAJQYAAHkIAAAfAAAAIgAA +AHAAAAA0BgAAeggAAAoAAAAgAAAALwAAADsMAAD+CAAA//8AAFMAAAD1CgAAaRwAADYJAAD//wAA +OAAAAJIBAAAaFAAAYQAAAP//AAA4AAAAsQEAAIgpAAD8AAAA//8AADgAAAC1AQAAOA4AAEEBAAD/ +/wAAOAAAAD8CAAC+FgAAzgQAAP//AAA4AAAAsgEAAKApAAAMAQAA//8AADgAAAC2AQAAjiEAAE8B +AAAFAAAAKwAAAAEBAABWIQAAUAEAAAYAAAA6AAAAmAcAANATAABRAQAABwAAADoAAACSBwAAaAUA +AGkBAAD//wAAOAAAALcBAABAGQAAdAEAAP//AAA4AAAA3AEAAL4pAAB9AQAA//8AADgAAADXAQAA +2ykAAJYBAAD//wAAOAAAAL8BAAABKgAA7wEAAP//AAA4AAAA4QEAACUqAAAvAgAADQAAADoAAACX +BgAAOA4AAFoCAAANAAAAOgAAAJgGAADOBAAAMAIAAP//AAA4AAAA5wEAAD8qAACsAgAA//8AADgA +AADoAQAAYCoAAMwCAAD//wAAOAAAAHICAACDKgAATwYAAP//AAA4AAAAEgIAAJQqAABrAwAAEwAA +ACsAAAAPAwAA/g0AAGwDAAD//wAAOAAAACsCAAAaFAAAwgAAAP//AAA4AAAAWAIAAA4XAABSBQAA +//8AADgAAABaAgAAoBcAAMAFAAD//wAAOAAAAHICAAA4DgAAVAYAAP//AAA4AAAAfgIAAA4XAADF +BgAA//8AADgAAAAKAgAA/g0AADoDAAD//wAAOAAAAPoBAACwKgAA6wkAAP//AAA4AAAA6gEAAEAZ +AAAKCQAA//8AADgAAADqAQAAQBkAAAoJAAD//wAAOAAAAPABAADMKgAAoAkAAP//AAA4AAAA+QEA +ALAqAACyCQAA//8AADgAAADFAQAAoCkAAA0CAAD//wAAOAAAAMkBAAA4DgAAmgoAAP//AAA4AAAA +0QEAAL4pAACnCgAA//8AADgAAADLAQAA2ykAAL0KAAD//wAAYQAAADYCAABOWwAAzQAAAP//AABh +AAAAOwIAAH1aAABxAQAAAQAAAF8AAAAqAAAAGFkAAGgBAAACAAAAYAAAACoAAACpVwAAaQEAAP// +AABhAAAAQAIAAE5bAACgAQAA//8AAGEAAABKAgAATlsAAOkBAAD//wAAYQAAAEsCAABpWwAA/QEA +AAYAAABfAAAALgAAAIJbAAD4AQAA//8AAGEAAABMAgAAfVoAABUCAAAIAAAAXwAAACoAAAAYWQAA +DAIAAAkAAABgAAAAKgAAAKlXAAANAgAA//8AAGEAAABYAgAATlsAAGACAAD//wAAYQAAAFgCAACd +WwAAYAIAAP//AABhAAAAaQIAAOYGAADfAgAA//8AAGEAAAB1AgAAfVoAABIDAAAOAAAAXwAAACoA +AAAYWQAAygMAAA8AAABgAAAAKgAAAKlXAADLAwAA//8AAGEAAAB1AgAATlsAABIDAAD//wAAYQAA +AIcCAAB9WgAA3wQAABIAAABfAAAAKgAAABhZAADnBAAAEwAAAGAAAAAqAAAAqVcAAOgEAAD//wAA +YQAAAB4CAAB9WgAATgAAABUAAABfAAAAKgAAABhZAABuBQAAFgAAAGAAAAAqAAAAqVcAAG8FAAD/ +/wAAYQAAAB4CAACWWgAATgAAABgAAABfAAAALAAAAEtYAAB3BQAA//8AAGEAAACUAgAAUUIAAMIF +AAD//wAAYQAAAJcCAAB9WgAA9wUAABsAAABfAAAAKgAAABhZAADmBQAAHAAAAGAAAAAqAAAAqVcA +AOcFAAD//wAAYQAAAJcCAACWWgAA9wUAAB4AAABfAAAALAAAAEtYAADvBQAA//8AAGEAAACYAgAA +Zx0AAAYGAAD//wAAYQAAAJsCAACcNwAAjQYAAP//AABhAAAApQIAAOo8AADnBgAA//8AAGEAAAC0 +AgAA80EAAD4HAAD//wAAXwAAABEAAACSVwAAQgAAAAAAAABgAAAAFwAAAKlXAAAyAAAA//8AAF8A +AAASAAAAwVcAAIgAAAACAAAAYAAAABgAAACpVwAAewAAAP//AABfAAAAEwAAANhXAADLAAAABAAA +AGAAAAAZAAAAqVcAAL4AAAD//wAAXwAAABQAAADvVwAAEgEAAAYAAABgAAAAGgAAAKlXAAAFAQAA +//8AAF8AAAAVAAAABlgAAFIBAAAIAAAAYAAAABsAAACpVwAASAEAAP//AABfAAAAFgAAAB1YAACV +AQAACgAAAGAAAAAcAAAAqVcAAIgBAAD//wAAXwAAABcAAAA0WAAA4AEAAAwAAABgAAAAHQAAAKlX +AADRAQAA//8AAF8AAAAYAAAAS1gAACgCAAAOAAAAYAAAAB4AAACpVwAAGwIAAP//AABfAAAAGQAA +AGJYAABoAgAAEAAAAGAAAAAfAAAAqVcAAF4CAAD//wAAXwAAABoAAAB4WAAAqAIAABIAAABgAAAA +IAAAAKlXAACeAgAA//8AAF8AAAAbAAAAjlgAAOgCAAAUAAAAYAAAACEAAACpVwAA3gIAAP//AABf +AAAAHAAAAKVYAAAoAwAAFgAAAGAAAAAiAAAAqVcAAB4DAAD//wAAXwAAAB0AAAC8WAAAaAMAABgA +AABgAAAAIwAAAKlXAABeAwAA//8AAF8AAAAeAAAA01gAAKsDAAAaAAAAYAAAACQAAACpVwAAngMA +AP//AABfAAAAHwAAAOpYAADyAwAAHAAAAGAAAAAlAAAAqVcAAOgDAAD//wAAXwAAACAAAAABWQAA +MgQAAB4AAABgAAAAJgAAAKlXAAAoBAAA//8AAF8AAAAhAAAAGFkAAHUEAAAgAAAAYAAAACoAAACp +VwAAaAQAAP//AABfAAAAIgAAAC9ZAAC4BAAAIgAAAGAAAAAsAAAAqVcAAKsEAAD//wAAXwAAACMA +AABJWQAAAQUAACQAAABgAAAALQAAAKlXAADxBAAA//8AAF8AAAAkAAAAX1kAAEsFAAAmAAAAYAAA +AC4AAACpVwAAOwUAAP//AABfAAAAJQAAAHVZAACYBQAAKAAAAGAAAAAvAAAAqVcAAIgFAAD//wAA +HAAAAC8AAACPBAAAZgAAAP//AAAcAAAALgAAACwKAABdAAAAAQAAABwAAABcAAAANwoAAF4AAAD/ +/wAAHAAAAC8AAAAsCgAAZgAAAAMAAAAcAAAAXAAAADcKAABiAAAA//8AABwAAAArAAAALAoAAGgA +AAAFAAAAHAAAAFwAAAA3CgAAaQAAAP//AAAcAAAAKQAAAI8EAACKAAAA//8AABwAAAAoAAAATwoA +AH0AAAAIAAAAHAAAAFgAAABaCgAAfgAAAP//AAAcAAAAKQAAAE8KAACKAAAACgAAABwAAABYAAAA +WgoAAIEAAAD//wAAHAAAACUAAABPCgAAjAAAAAwAAAAcAAAAWAAAAFoKAACNAAAA//8AABwAAAAj +AAAAjwQAALMAAAD//wAAHAAAAEUAAAByCgAAbQAAAP//AAAcAAAARQAAAHIKAABtAAAA//8AABwA +AAA+AAAALAoAAPgAAAARAAAAHAAAAFwAAAA3CgAAHQEAAP//AAAcAAAAPgAAACwKAAD4AAAAEwAA +ABwAAABcAAAANwoAAB4BAAD//wAAHAAAAD4AAAByCgAA+AAAAP//AAAcAAAAPwAAAI8EAAAcAQAA +//8AABwAAABBAAAAjwQAAAwBAAD//wAAHAAAAEIAAACPBAAAUwEAAP//AAAcAAAAQQAAACwKAAAM +AQAAGQAAABwAAABcAAAANwoAAEUBAAD//wAAHAAAAEIAAAAsCgAAUwEAABsAAAAcAAAAXAAAADcK +AABJAQAA//8AABwAAAA2AAAALAoAAFsBAAAdAAAAHAAAAFwAAAA3CgAAyAEAAP//AAAcAAAANgAA +ACwKAABbAQAAHwAAABwAAABcAAAANwoAAMkBAAD//wAAHAAAADYAAAByCgAAWwEAAP//AAAcAAAA +NwAAACwKAAA/AAAAIgAAABwAAABcAAAANwoAAM0BAAD//wAAHAAAADcAAAAsCgAAPwAAACQAAAAc +AAAAXAAAADcKAADOAQAA//8AABwAAAA3AAAAcgoAAD8AAAD//wAAHAAAADgAAAAsCgAAQgAAACcA +AAAcAAAAXAAAADcKAADSAQAA//8AABwAAAA4AAAALAoAAEIAAAApAAAAHAAAAFwAAAA3CgAA0wEA +AP//AAAcAAAAOAAAAHIKAABCAAAA//8AABwAAAA5AAAAjwQAAMcBAAAIDxAVFhgcHiImMlV42N3/ +AwQoATgxSRJVBY8HlAfgP/8B//8ACqAASZIkAFVVAQD+/wMAIEmSJP////8AAAD/AQBwBAKAQGYA +AAAAVVVVVVVVVQWTJAEAIElKoQUAAAARBICBAAkAAAAAIwH///////////////////////////// +//////////////////8AIEny/////////////////////////w8AAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAOT///////////////////8fAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABB/ +////f0VQRIRERMAiCgAAAG7+A45AABDIDCIIAAMBEoEjCiMBgT4P/SCB2wUCAYHtB38BAQEBAQEB +AQEBAQEBAQEBfwICAgICAgICAgICAgICAgJ/BAQEBAQEBAQEBAQEBAQEBH8ICAgICAgICAgICAgI +CAgIfxAQEBAQEBAQEBAQEBAQEBB/ICAgICAgICAgICAgICAgIH9AQEBAQEBAQEBAQEBAQEBAf4CA +gICAgICAgICAgICAgAB/AQEBAQEBAQEBAQEBAQEBAX8CAgICAgICAgICAgICAgICfwQEBAQEBAQE +BAQEBAQEBAR/CAgICAgICAgICAgICAgICH8QEBAQEBAQEBAQEBAQEBAQfyAgICAgICAgICAgICAg +ICB/QEBAQEBAQEBAQEBAQEBAQGmAgICAgICAgICAgICAAIGQQH8BJEn+//////////////9/f/// +/////////////////39/////////////////////f3////////////////////9/f/////////// +/////////39/////////////////////f3////////////////////9/f/////////////////// +/39/////////////////////f3////////////////////9/f////////////////////39///// +////////////////f3////////////////////9/f////////////////////39///////////// +////////f3////////////////////9/f////////////////////39///////////////////// +f3////////////////////9/f////////////////////39/////////////////////f3////// +//////////////9/f////////////////////39/////////////////////f3////////////// +//////9/f////////////////////39/////////////////////f3////////////////////9/ +f////////////////////39/////////////////////f3////////////////////9/f/////// +/////////////39/////////////////////f3////////////////////9/f/////////////// +/////39/////////////////////f3////////////////////9/f////////////////////39/ +////////////////////f3////////////////////9/f////////////////////39///////// +////////////f3////////////////////9/f////////////////////39///////////////// +////f3////////////////////9/f////////////////////39/////////////////////f3// +//////////////////9/f////////////////////39/////////////////////f3////////// +//////////9/f////////////////////39/////////////////////f3////////////////// +//9/f////////////////////39/////////////////////f3////////////////////9/f/// +/////////////////39/////////////////////f3////////////////////9/f/////////// +/////////39/////////////////////f3////////////////////9/f///////////////ESMY +ARB/gCQghBAghBAghBAghBAgBH8hQAghQAghQAghQAghQAghf4AQQoAQQoAQQoAQQoAQQgB/IYQA +IYQAIYQAIYQAIYQAIX8IAUIIAUIIAUIIAUIIAUIIfwKEEAKEEAKEEAKEEAKEEAJ/CCEECCEECCEE +CCEECCEECH9CCBBCCBBCCBBCCBBCCBBCfxAghBAghBAghBAghBAghBB/QAghQAghQAghQAghQAgh +QH8QQoAQQoAQQoAQQoAQQoAQf4QAIYQAIYQAIYQAIYQAIQR/AUIIAUIIAUIIAUIIAUIIAX+EEAKE +EAKEEAKEEAKEEAIEfyEECCEECCEECCEECCEECCF/CBBCCBBCCBBCCBBCCBBCCH8ghBAghBAghBAg +hBAghBAgfwghQAghQAghQAghQAghQAh/QoAQQoAQQoAQQoAQQoAQQn8AIYQAIYQAIYQAIYQAIYQA +f0IIAUIIAUIIAUIIAUIIAUJ/EAKEEAKEEAKEEAKEEAKEEH8ECCEECCEECCEECCEECCEEfxBCCBBC +CBBCCBBCCBBCCBB/hBAghBAghBAghBAghBAgBH8hQAghQAghQAghQAghYMgQJUOGDDEEAH9WlVWp +qapGRERERFCVWVVVf6oqqqqqbdu2bdu2paqqqip/VVVFVVVVVVVVVVVVVVVVSX8kSZIkSZIkSZIk +SZIkSZIkf5IkSZIkSZIk/P///////39/////////////////////f3////////////////////9/ +f////////////////////39G//////////8/AGdvMS4xNwAAAAAAAAAAsD78qfHSTWJQP3sUrkfh +eoQ/AAAAAAAA0D8zMzMzMzPTPwAAAAAAAOA/MzMzMzMz4z8AAAAAAADsP2ZmZmZmZu4/AAAAAAAA +8D+amZmZmZnxPzMzMzMzM/M/AAAAAAAAFEAAAAAAAAAkQAAAAAAAADpAAAAAAAAAWUAAAAAAAIjD +QAAAAAAAAPBAAAAAANASY0EAAAAAAADgQwAAAAAAAPB/AAAAAAAAAIAzMzMzMzPTv+85+v5CLua/ +AN1CAAAAAAACAAAAAAAAAAEAAAAAAAAAAQAAAAYAAAAAAAAAAAAAAAIAAAAIAAAAAQEAAAAAAAAC +AAAABgAAAAAAAAAAAAAAAgAAAAIAAAAAAAAAAAAAAAIAAAAGAAAAAwMAAAAAAAACAAAAAgAAAAAA +AAAAAAAAAgAAAAIAAAAAAAAAAAAAAIwkRgAAAAAADgAAAAAAAAClPUYAAAAAABsAAAAAAAAAij1G +AAAAAAAbAAAAAAAAAB49RgAAAAAAGwAAAAAAAABvPUYAAAAAABsAAAAAAAAAqjNGAAAAAAAWAAAA +AAAAAKcuRgAAAAAAFAAAAAAAAADlNEYAAAAAABcAAAAAAAAAhlJGAAAAAAAlAAAAAAAAAChFRgAA +AAAAHgAAAAAAAADnV0YAAAAAACoAAAAAAAAANh1GAAAAAAAKAAAAAAAAAGNhbGwgZnJhbWUgdG9v +IGxhcmdlAAAAAAAAAAAAAAAAAAAAAAAAAACQQwUAAAAAAAAQQAAAAAAAoMJFAAAAAABguUUAAAAA +AJlfDEwAAAAAwElFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAKD///9gAAAA +WAAAAAAAAABEX0cAAAAAAAEAAAAAAAAA2P///ygAAAAoAAAAAAAAAC5fRwAAAAAAAQAAAAAAAADw +////EAAAABAAAAAAAAAACGhGAAAAAAABAAAAAAAAAPD///8QAAAAEAAAAAAAAAAIaEYAAAAAAAEA +AAAAAAAA6P///xgAAAAQAAAAAAAAAAhoRgAAAAAAAQAAAAAAAADw////EAAAABAAAAAAAAAACGhG +AAAAAAABAAAAAAAAAPD///8QAAAAEAAAAAAAAAAYaEYAAAAAAAEAAAAAAAAA6P///xgAAAAYAAAA +AAAAADBoRgAAAAAAAQAAAAAAAADY////KAAAACgAAAAAAAAAMl9HAAAAAAABAAAAAAAAAPD///8Q +AAAAEAAAAAAAAAAIaEYAAAAAAAEAAAAAAAAA6P///xgAAAAYAAAAAAAAADBoRgAAAAAAAQAAAAAA +AAC4////SAAAAEgAAAAAAAAAPl9HAAAAAAABAAAAAAAAANj///8oAAAAGAAAAAAAAAAwaEYAAAAA +AAEAAAAAAAAA6P///xgAAAAYAAAAAAAAAEBoRgAAAAAAAQAAAAAAAADg////IAAAABgAAAAAAAAA +QGhGAAAAAAABAAAAAAAAAPD///8QAAAAEAAAAAAAAAAIaEYAAAAAAAEAAAAAAAAA8P///xAAAAAQ +AAAAAAAAAAhoRgAAAAAAAQAAAAAAAADw////EAAAABAAAAAAAAAACGhGAAAAAAABAAAAAAAAAOD/ +//8gAAAAGAAAAAAAAAAAaEYAAAAAAAEAAAAAAAAAwP///0AAAAA4AAAAAAAAADdfRwAAAAAAAQAA +AAAAAADw////EAAAAAgAAAAAAAAA+GdGAAAAAAABAAAAAAAAAPD///8QAAAACAAAAAAAAAD4Z0YA +AAAAAAEAAAAAAAAA0P///zAAAAAwAAAAAAAAADZfRwAAAAAAAQAAAAAAAADA////QAAAAEAAAAAA +AAAAOl9HAAAAAAABAAAAAAAAAMD///9AAAAAQAAAAAAAAAA6X0cAAAAAAAEAAAAAAAAA4P///yAA +AAAgAAAAAAAAAChoRgAAAAAAAQAAAAAAAADw////EAAAABAAAAAAAAAACGhGAAAAAAABAAAAAAAA +AND///8wAAAAMAAAAAAAAAA1X0cAAAAAAAEAAAAAAAAA4P///yAAAAAQAAAAAAAAAAhoRgAAAAAA +AQAAAAAAAADw////EAAAAAgAAAAAAAAA+GdGAAAAAAABAAAAAAAAAOj///8YAAAAEAAAAAAAAAAI +aEYAAAAAAAEAAAAAAAAA8P///xAAAAAIAAAAAAAAAPhnRgAAAAAAAQAAAAAAAADw////EAAAAAgA +AAAAAAAA+GdGAAAAAAABAAAAAAAAAPD///8QAAAACAAAAAAAAAD4Z0YAAAAAAAEAAAAAAAAA6P// +/xgAAAAQAAAAAAAAAAhoRgAAAAAAAQAAAAAAAADo////GAAAABAAAAAAAAAACGhGAAAAAAABAAAA +AAAAAOj///8YAAAAGAAAAAAAAAAwaEYAAAAAAAEAAAAAAAAA8P///xAAAAAQAAAAAAAAAAhoRgAA +AAAAAQAAAAAAAADw////EAAAABAAAAAAAAAACGhGAAAAAAABAAAAAAAAAPD///8QAAAAEAAAAAAA +AAAIaEYAAAAAAAEAAAAAAAAAqP///1gAAABYAAAAAAAAADxfRwAAAAAAAQAAAAAAAADo////GAAA +AAgAAAAAAAAA+GdGAAAAAAABAAAAAAAAAPD///8QAAAAEAAAAAAAAAAIaEYAAAAAAAEAAAAAAAAA +8P///xAAAAAQAAAAAAAAAAhoRgAAAAAAAQAAAAAAAADw////EAAAAAgAAAAAAAAA+GdGAAAAAAAB +AAAAAAAAAMj///84AAAAMAAAAAAAAADlFEYAAAAAAAEAAAAAAAAA6P///xgAAAAQAAAAAAAAAAho +RgAAAAAAAQAAAAAAAADw////EAAAABAAAAAAAAAACGhGAAAAAAABAAAAAAAAAPD///8QAAAAEAAA +AAAAAAAIaEYAAAAAAAEAAAAAAAAA6P///xgAAAAQAAAAAAAAAAhoRgAAAAAAAQAAAAAAAADo//// +EAAAABAAAAAAAAAACGhGAAAAAAABAAAAAAAAAOj///8YAAAAGAAAAAAAAABAaEYAAAAAAAEAAAAA +AAAA6P///xgAAAAYAAAAAAAAADBoRgAAAAAAAQAAAAAAAADo////GAAAAAgAAAAAAAAA+GdGAAAA +AAABAAAAAAAAAND///8wAAAAMAAAAAAAAAA0X0cAAAAAAAEAAAAAAAAA6P///xgAAAAQAAAAAAAA +AAhoRgAAAAAAAQAAAAAAAADw////EAAAAAgAAAAAAAAA+GdGAAAAAAABAAAAAAAAAPD///8QAAAA +CAAAAAAAAAD4Z0YAAAAAAAEAAAAAAAAA6P///xgAAAAYAAAAAAAAADBoRgAAAAAAAQAAAAAAAABw +////kAAAAIgAAAAAAAAAWF9HAAAAAAABAAAAAAAAANj///8oAAAAKAAAAAAAAAAxX0cAAAAAAAEA +AAAAAAAA8P///xAAAAAQAAAAAAAAAAhoRgAAAAAAAQAAAAAAAADI////OAAAACAAAAAAAAAAEGhG +AAAAAAABAAAAAAAAAAAAAAAQAAAAEAAAAAAAAAAIaEYAAAAAAAEAAAAAAAAAAAAAABgAAAAIAAAA +AAAAAPhnRgAAAAAAAQAAAAAAAADo////GAAAAAgAAAAAAAAA+GdGAAAAAAABAAAAAAAAAPD///8Q +AAAAEAAAAAAAAAAIaEYAAAAAAAEAAAAAAAAA2P///ygAAAAoAAAAAAAAADNfRwAAAAAAAQAAAAAA +AADw////EAAAABAAAAAAAAAAGGhGAAAAAAABAAAAAAAAAPD///8QAAAAEAAAAAAAAAAYaEYAAAAA +AAEAAAAAAAAA6P///xgAAAAIAAAAAAAAAPhnRgAAAAAAAQAAAAAAAADo////GAAAAAgAAAAAAAAA ++GdGAAAAAAABAAAAAAAAAPD///8QAAAAEAAAAAAAAAAYaEYAAAAAAAEAAAAAAAAA8P///xAAAAAI +AAAAAAAAAPhnRgAAAAAAAQAAAAAAAADo////GAAAABAAAAAAAAAACGhGAAAAAAABAAAAAAAAAOD/ +//8gAAAAIAAAAAAAAAAsX0cAAAAAAAEAAAAAAAAAyP///zgAAAAgAAAAAAAAABBoRgAAAAAAAQAA +AAAAAADo////GAAAABAAAAAAAAAACGhGAAAAAAABAAAAAAAAAKj///9YAAAAWAAAAAAAAAA8X0cA +AAAAAAEAAAAAAAAA4P///yAAAAAYAAAAAAAAADBoRgAAAAAAAQAAAAAAAADg////IAAAACAAAAAA +AAAALF9HAAAAAAABAAAAAAAAAPD///8QAAAAEAAAAAAAAAAYaEYAAAAAAAEAAAAAAAAA6P///xgA +AAAIAAAAAAAAAPhnRgAAAAAAAQAAAAAAAADw////EAAAAAgAAAAAAAAA+GdGAAAAAAABAAAAAAAA +APD///8QAAAACAAAAAAAAAD4Z0YAAAAAAAEAAAAAAAAA6P///xgAAAAQAAAAAAAAAAhoRgAAAAAA +AQAAAAAAAACQ////cAAAAHAAAAAAAAAAQF9HAAAAAAACAAAAAAAAAND///8YAAAACAAAAAAAAAD4 +Z0YAAAAAAOj///8YAAAACAAAAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAgAAAAAAAADY////EAAAABAA +AAAAAAAACGhGAAAAAADo////GAAAABgAAAAAAAAAMGhGAAAAAAAAAAAAAAAAAAIAAAAAAAAA0P// +/xgAAAAIAAAAAAAAAPhnRgAAAAAA6P///xgAAAAIAAAAAAAAAPhnRgAAAAAAAAAAAAAAAAACAAAA +AAAAAOD///8IAAAACAAAAAAAAAD4Z0YAAAAAAPD///8QAAAAEAAAAAAAAAAIaEYAAAAAAAAAAAAA +AAAAAgAAAAAAAADY////EAAAAAgAAAAAAAAA+GdGAAAAAADo////GAAAABgAAAAAAAAAMGhGAAAA +AAAAAAAAAAAAAAIAAAAAAAAA4P///wgAAAAIAAAAAAAAAPhnRgAAAAAA6P///xAAAAAQAAAAAAAA +AAhoRgAAAAAAAAAAAAAAAAACAAAAAAAAAMD///8YAAAAGAAAAAAAAABAaEYAAAAAANj///8oAAAA +KAAAAAAAAAAuX0cAAAAAAAAAAAAAAAAAAgAAAAAAAADQ////MAAAACgAAAAAAAAAMF9HAAAAAAAI +AAAACAAAAAgAAAAAAAAA+GdGAAAAAAAAAAAAAAAAAAIAAAAAAAAA4P///xAAAAAIAAAAAAAAAPhn +RgAAAAAA8P///xAAAAAIAAAAAAAAAPhnRgAAAAAAAAAAAAAAAAACAAAAAAAAANj///8QAAAACAAA +AAAAAAD4Z0YAAAAAAOj///8YAAAACAAAAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAgAAAAAAAADY//// +EAAAAAgAAAAAAAAA+GdGAAAAAADo////GAAAAAgAAAAAAAAA+GdGAAAAAAAAAAAAAAAAAAIAAAAA +AAAA4P///wgAAAAIAAAAAAAAAPhnRgAAAAAA6P///xgAAAAYAAAAAAAAADBoRgAAAAAAAAAAAAAA +AAADAAAAAAAAAMD///8QAAAACAAAAAAAAAD4Z0YAAAAAAND///8YAAAACAAAAAAAAAD4Z0YAAAAA +AOj///8YAAAACAAAAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAMj///8QAAAA +EAAAAAAAAAAIaEYAAAAAANj///8QAAAAEAAAAAAAAAAIaEYAAAAAAOj///8YAAAAGAAAAAAAAABA +aEYAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAMD///8IAAAACAAAAAAAAAD4Z0YAAAAAAND/ +//8YAAAAGAAAAAAAAABAaEYAAAAAAOj///8YAAAAEAAAAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAA +AAAAAAAEAAAAAAAAACD+//8wAAAAMAAAAAAAAADkFEYAAAAAAFD+//8wAAAAMAAAAAAAAADkFEYA +AAAAAID+//+QAAAAkAAAAAAAAABcX0cAAAAAABD////wAAAA8AAAAAAAAABgX0cAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD/AAAAAAAAAAAAAAAAAAAA//8AAAAA +AAAAAAAAAAAAAP///wAAAAAAAAAAAAAAAAD/////AAAAAAAAAAAAAAAA//////8AAAAAAAAAAAAA +AP///////wAAAAAAAAAAAAD/////////AAAAAAAAAAAA//////////8AAAAAAAAAAP////////// +/wAAAAAAAAD/////////////AAAAAAAA//////////////8AAAAAAP///////////////wAAAAD/ +////////////////AAAA//////////////////8AAP///////////////////wAAAAAAAAAAAAAA +AAAAAAAAD////////////////////w4P//////////////////8NDg//////////////////DA0O +D////////////////wsMDQ4P//////////////8KCwwNDg//////////////CQoLDA0OD/////// +/////wgJCgsMDQ4P//////////8HCAkKCwwNDg//////////BgcICQoLDA0OD////////wUGBwgJ +CgsMDQ4P//////8EBQYHCAkKCwwNDg//////AwQFBgcICQoLDA0OD////wIDBAUGBwgJCgsMDQ4P +//8BAgMEBQYHCAkKCwwNDg//QxVGAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABGFUYAAAAA +AAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEkVRgAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +uBVGAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAFUYAAAAAAAQAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAMQVRgAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA5BVGAAAAAAAEAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAABSFUYAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABEbRgAAAAAACQAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAVBdGAAAAAAAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFkYA +AAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALIWRgAAAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAtxZGAAAAAAAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC8FkYAAAAAAAUAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAPwVRgAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAABAQEBAQECAgIC +AwYICQkLAAAAAAIDBg4SFBUWGRobHR4fICwAAAAAAgMEBAQGBwcICAoKCgoKNgAAAAAAAgMFBQUF +BwcHBwcHBwg/AAAAAAEDAwQFBQYHBwcHCQkLDEwAAAAAAQEBAQMFBQUFBQUGBgYGUgAAAAABAQEB +AQEBAQICAgMDBAZYAAAAAAABAgQEBAQFBwkKDA0PD2kAAAAAAgMDBAYJCQkJCgoKCgoKdAAAAAAB +AQECAgICAgICAgIEBQd7AAAAAAEDAwMEBgYGBwgICAkJCoUAAAAAAAAAAQECAwMDAwMEBQUGjAAA +AAAAAAECAgICAwUICAkJCguZAAAAAAEBAQECAgMDBAQFBgYGBp8AAAAAAAAAAAAAAAEBAQECAwMD +ogAAAAAAAAACBAUGBgcHBwgICgqsAAAAAAAAAAABAgIDBQUGBwcHCbcAAAAAAAACAgQFBgYGBgYG +BwgIvwAAAAABAgICAgICAgICAgICAgTEAAAAAAAAAAEBAgICAgIDBAUGBsoAAAAAAQIDBAQEBQUH +BwgICAkJ0wAAAAACAgIEBAUGBgYGBgYGCAjbAAAAAAEBAQECAgMEBAQEBQUGB+IAAAAAAAEBAQEC +BAYGBgcICAgI6wAAAAABAQECAgIDBQUHCAgICQr1AAAAAAECAwQFBQUFBgYHCAgJCv8AAAAAAAEB +AgMDBAUGBwcICAgJCAEAAAAAAAEBAQEBAQEBAQEBAgIKAQAAAAEBAwQEBQYHCQoKDQ8QERwBAAAA +AQEDBAQEBQUFBgYGBgYHJAEAAAACAwQFBQUFBQUGBgYICgouAQAAAAABAgMDAwUFBgYGCAoKCzkB +AAAAAQEBAQEBAgICAgQEBAQEPQEAAAAAAAAAAAABAQICAgMDAwRCAQAAAAABAQEBAwUFBgYHBwcI +Ck4BAAAAAAEBAgMEBQcJCgoKDA4PXQEAAAACAwQEBQUHBwkLDA4PEBBtAQAAAAAAAAICAgQEBQYG +CAoLDHsBAAAAAQEBAgQFBQYHBwcICQoLhwEAAAABAQECAwQFBggKDA0PEBKbAQAAAAEDBQYGBwgJ +CQoLCwwMD6oBAAAAAQIDBQUFBgYGBgYGBgYGsQEAAAABAQEBAQEBBAYGCAoLCwy9AQAAAAABAwMD +AwMEBAUHBwcHCMUBAAAAAgMGBwcICQsMDxAQERER1gEAAAADBAUHCAgICQkJDA8PERLpAQAAAAAA +AQMDBAQFBgYGBgcICPEBAAAAAQMEBAUFBgcJCgoLCwsM/gEAAAAAAAICBAUFBwcICQoMDQ8NAgAA +AAABAQECAwMEBQYHBwcHBxQCAAAAAAAAAAECAgIDBAUFCAgIHAIAAAABAQEBAgMEBQUGBwkJCQoo +AgAAAAAAAgIDBAYGBwgJCgwODzgCAAAAAQMEBAQEBQUFBgYHCQkLRwIAAAADAwMDBQYGBwcHCAkJ +CQlQAgAAAAAAAAECAwUFBQYHBwcHB1gCAAAAAAEDAwMDAwMDAwQEBwcIYQIAAAABAgMDBAUGBwcI +CQkJCQlrAgAAAAICAgICAgMFBgcHCAgICHQCAAAAAAAAAQMFBgcHBwgICQkJfQIAAAABAQECAwQE +BAQEBAUGBgeGAgAAAAABAQECAgICAgICAgMDA4kCAAAAAQECBQcICAkLDA0OEBARmwIAAAABAQEB +AQEBAwMEBQUGBwijAgAAAAABAQECAgMDAwQFBQYGBqoCAAAAAAAAAAAAAAAAAAABAQIDrQIAAAAA +AAAAAAABAgIDBAUGCAm4AgAAAAEBAgICAgICAgMEBAQEBL0CAAAAAQIDBAYHCAkKCwsMDAwNygIA +AAABAgICAgIDAwMEBAQFBgbRAgAAAAABAQEBAQMDBAQGBwgJCdoCAAAAAAECBAQEBQUHCAoMDQ4P +6wIAAAABAwQGBgcHBwcHBwcHBwfyAgAAAAAAAAAAAAAAAAAAAAAAAPICAAAAAQEBAwMEBAQGBggJ +CQkK/AIAAAABAQECBAUGBwcHCAkKCgwJAwAAAAEBAgQFBwcHCAgICQkJCxUDAAAAAAEBAQEBAQIC +AgICAgICFwMAAAAAAAABAQEBAQIDAwMDBAQdAwAAAAECBAYHCAsPEhUVFhYXGTYDAAAAAQMFBwcH +BwcJCw0PERMVTQMAAAAEBgoQFRgcHBwjKCgpKSl3AwAAAAAAAQEBAQEBAQIDCg8TF5ADAAAAAwcK +EBMVGiEmKy0wMjQ3ygMAAAACAwUAAAAAAAAAAAAudGV4dAAubm9wdHJkYXRhAC5kYXRhAC5ic3MA +Lm5vcHRyYnNzAF9fbGliZnV6emVyX2V4dHJhX2NvdW50ZXJzAC5nby5idWlsZGluZm8ALm5vdGUu +Z28uYnVpbGRpZAAuZWxmZGF0YQAucm9kYXRhAC50eXBlbGluawAuaXRhYmxpbmsALmdvc3ltdGFi +AC5nb3BjbG50YWIALnN5bXRhYgAuc3RydGFiAC5kZWJ1Z19hYmJyZXYALnpkZWJ1Z19hYmJyZXYA +LmRlYnVnX2ZyYW1lAC56ZGVidWdfZnJhbWUALmRlYnVnX2luZm8ALnpkZWJ1Z19pbmZvAC5kZWJ1 +Z19sb2MALnpkZWJ1Z19sb2MALmRlYnVnX2xpbmUALnpkZWJ1Z19saW5lAC5kZWJ1Z19nZGJfc2Ny +aXB0cwAuemRlYnVnX2dkYl9zY3JpcHRzAC5kZWJ1Z19yYW5nZXMALnpkZWJ1Z19yYW5nZXMALnNo +c3RydGFiAAAAAAAAAEAhAACAIQAAoFsAAIAkAADAIQAAACIAAEAiAADAJAAAgCIAAMAiAAAAIwAA +QCMAAIAjAADAIwAAACQAAEAkAAAgXAAAACUAAEAlAABggQAAgCUAAMAlAADAUwAAoGMAAKBcAAAA +JgAAQCYAAIAmAADAJgAAIGQAACBdAAAAJwAAQCcAAIAnAAAgaQAAwCcAAKBdAAAAKAAAYIcAACBU +AABAKAAAgCgAAKBkAAAgXgAAwCgAACCCAAAAKQAAQCkAAIBUAACAKQAAwCkAAOBUAAAAKgAAQCoA +AIAqAADgdgAAwCoAAKBeAAAAKwAAoHUAAEArAACAKwAAQJIAACBfAADAKwAA4IIAAAAsAABAVQAA +QCwAAKBfAADAaQAAgCwAAMAsAAAgYAAAoFUAAAAtAACgYAAAQC0AAIAtAAAgYQAAwC0AAAAuAABA +LgAAgC4AAMAuAAAALwAAQC8AAIAvAADALwAAADAAAKBhAABAMAAAgDAAAMAwAAAAMQAAAFYAACBl +AABgagAAIGIAAEAxAACAMQAAAGsAAMAxAAAAMgAAQDIAAIAyAADAMgAAADMAAEAzAACgQwAAAEQA +AGBEAADARAAAIEUAAIBFAADgRQAAQEYAAKBGAAAARwAAYEcAAMBHAAAgSAAAgEgAAOBIAABASQAA +oEkAAABKAABgSgAAwEoAACBLAACAMwAAwDMAAAA0AABANAAAgDQAAMA0AAAANQAAQDUAAIA1AADA +NQAAADYAAIA2AABANgAAwDYAAAA3AABANwAAgDcAAMA3AAAAOAAAQDgAAIA4AADAOAAAADkAAEA5 +AACAOQAAwDkAAAA6AABAOgAAgDoAAMA6AACAOwAAwDsAAAA7AAAAPQAAoE8AAIBLAADgSwAAAFAA +AMBWAAAgVwAAgFcAAOBXAACgaAAAQHYAACBoAABgUwAAiGpHAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAD6////AAABCNIDAAAAAAAAaAAAAAAAAABAAAAAAAAAAEB7AAAAAAAAAH4AAAAAAACg +hgAAAAAAAMBQAgAAAAAAaW50ZXJuYWwvY3B1LkluaXRpYWxpemUAaW50ZXJuYWwvY3B1LnByb2Nl +c3NPcHRpb25zAGludGVybmFsL2NwdS5pbmRleEJ5dGUAaW50ZXJuYWwvY3B1LmRvaW5pdABpbnRl +cm5hbC9jcHUuaXNTZXQAaW50ZXJuYWwvY3B1LmNwdWlkAGludGVybmFsL2NwdS54Z2V0YnYAdHlw +ZS4uZXEuaW50ZXJuYWwvY3B1Lm9wdGlvbgB0eXBlLi5lcS5bMTVdaW50ZXJuYWwvY3B1Lm9wdGlv +bgBydW50aW1lL2ludGVybmFsL3N5cy5PbmVzQ291bnQ2NABpbnRlcm5hbC9ieXRlYWxnLmluaXQu +MABjbXBib2R5AHJ1bnRpbWUuY21wc3RyaW5nAG1lbWVxYm9keQBydW50aW1lLm1lbWVxdWFsAHJ1 +bnRpbWUubWVtZXF1YWxfdmFybGVuAGluZGV4Ynl0ZWJvZHkAaW50ZXJuYWwvYnl0ZWFsZy5JbmRl +eEJ5dGVTdHJpbmcAdHlwZS4uZXEuaW50ZXJuYWwvYWJpLlJlZ0FyZ3MAcnVudGltZS5tZW1oYXNo +MTI4AHJ1bnRpbWUubWVtZXF1YWwwAHJ1bnRpbWUubWVtZXF1YWw4AHJ1bnRpbWUubWVtZXF1YWwx +NgBydW50aW1lLm1lbWVxdWFsMzIAcnVudGltZS5tZW1lcXVhbDY0AHJ1bnRpbWUubWVtZXF1YWwx +MjgAcnVudGltZS5mMzJlcXVhbABydW50aW1lLmY2NGVxdWFsAHJ1bnRpbWUuYzY0ZXF1YWwAcnVu +dGltZS5jMTI4ZXF1YWwAcnVudGltZS5zdHJlcXVhbABydW50aW1lLmludGVyZXF1YWwAcnVudGlt +ZS5uaWxpbnRlcmVxdWFsAHJ1bnRpbWUuZWZhY2VlcQBydW50aW1lLmlzRGlyZWN0SWZhY2UAcnVu +dGltZS5pZmFjZWVxAHJ1bnRpbWUuYWxnaW5pdABydW50aW1lLmluaXRBbGdBRVMAcnVudGltZS5h +dG9taWN3YgBydW50aW1lLigqd2JCdWYpLnB1dEZhc3QAcnVudGltZS5hdG9taWNzdG9yZXAAcnVu +dGltZS5tbWFwAHJ1bnRpbWUubW1hcC5mdW5jMQBydW50aW1lLm11bm1hcABydW50aW1lLm11bm1h +cC5mdW5jMQBydW50aW1lLnNpZ2FjdGlvbgBydW50aW1lLnNpZ2FjdGlvbi5mdW5jMQBydW50aW1l +LmNnb2NhbGwAcnVudGltZS5jZ29Jc0dvUG9pbnRlcgBydW50aW1lLmFjdGl2ZU1vZHVsZXMAcnVu +dGltZS5jZ29JblJhbmdlAHJ1bnRpbWUuY2dvQ2hlY2tXcml0ZUJhcnJpZXIAcnVudGltZS5pblBl +cnNpc3RlbnRBbGxvYwBydW50aW1lLmNnb0NoZWNrV3JpdGVCYXJyaWVyLmZ1bmMxAHJ1bnRpbWUu +Y2dvQ2hlY2tNZW1tb3ZlAHJ1bnRpbWUuY2dvQ2hlY2tTbGljZUNvcHkAcnVudGltZS5hZGQAcnVu +dGltZS5jZ29DaGVja1R5cGVkQmxvY2sAcnVudGltZS5zcGFuT2ZVbmNoZWNrZWQAcnVudGltZS5h +cmVuYUluZGV4AHJ1bnRpbWUuaGVhcEJpdHNGb3JBZGRyAHJ1bnRpbWUuaGVhcEJpdHMuYml0cwBy +dW50aW1lLmhlYXBCaXRzLm5leHQAcnVudGltZS5hZGQxAHJ1bnRpbWUuY2dvQ2hlY2tUeXBlZEJs +b2NrLmZ1bmMxAHJ1bnRpbWUuY2dvQ2hlY2tCaXRzAHJ1bnRpbWUuYWRkYgBydW50aW1lLmNnb0No +ZWNrVXNpbmdUeXBlAHJ1bnRpbWUubWFrZWNoYW4AcnVudGltZS4oKmhjaGFuKS5yYWNlYWRkcgBy +dW50aW1lLmNoYW5zZW5kMQBydW50aW1lLmNoYW5zZW5kAHJ1bnRpbWUuZnVsbABydW50aW1lLmxv +Y2sAcnVudGltZS5sb2NrV2l0aFJhbmsAcnVudGltZS4oKndhaXRxKS5kZXF1ZXVlAHJ1bnRpbWUu +dW5sb2NrAHJ1bnRpbWUudW5sb2NrV2l0aFJhbmsAcnVudGltZS4oKndhaXRxKS5lbnF1ZXVlAHJ1 +bnRpbWUuY2hhbmJ1ZgBydW50aW1lLmNoYW5zZW5kLmZ1bmMxAHJ1bnRpbWUuc2VuZABydW50aW1l +LnNlbmREaXJlY3QAcnVudGltZS5yZWN2RGlyZWN0AHJ1bnRpbWUuY2xvc2VjaGFuAHJ1bnRpbWUu +KCpnTGlzdCkucHVzaABydW50aW1lLigqZ3VpbnRwdHIpLnNldABydW50aW1lLigqZ0xpc3QpLmVt +cHR5AHJ1bnRpbWUuKCpnTGlzdCkucG9wAHJ1bnRpbWUuY2hhbnJlY3YxAHJ1bnRpbWUuY2hhbnJl +Y3YAcnVudGltZS5lbXB0eQBydW50aW1lLmNoYW5yZWN2LmZ1bmMxAHJ1bnRpbWUucmVjdgBydW50 +aW1lLmNoYW5wYXJrY29tbWl0AHJ1bnRpbWUuaW5pdC4wAHJ1bnRpbWUuKCpjcHVQcm9maWxlKS5h +ZGQAcnVudGltZS5uYW5vdGltZQBydW50aW1lLigqY3B1UHJvZmlsZSkuYWRkTm9uR28AcnVudGlt +ZS4oKmNwdVByb2ZpbGUpLmFkZEV4dHJhAHJ1bnRpbWUuZnVuY1BDAHJ1bnRpbWUuZGVidWdDYWxs +Q2hlY2sAcnVudGltZS5kZWJ1Z0NhbGxDaGVjay5mdW5jMQBydW50aW1lLmZ1bmNJbmZvLnZhbGlk +AHJ1bnRpbWUuZGVidWdDYWxsV3JhcABydW50aW1lLigqbXVpbnRwdHIpLnNldABydW50aW1lLmRl +YnVnQ2FsbFdyYXAuZnVuYzEAcnVudGltZS5kZWJ1Z0NhbGxXcmFwMQBydW50aW1lLmRlYnVnQ2Fs +bFdyYXAyAHJ1bnRpbWUuZGVidWdDYWxsV3JhcDIuZnVuYzEAcnVudGltZS5nb2dldGVudgBydW50 +aW1lLmVudktleUVxdWFsAHJ1bnRpbWUuKCpUeXBlQXNzZXJ0aW9uRXJyb3IpLkVycm9yAHJ1bnRp +bWUuZXJyb3JTdHJpbmcuRXJyb3IAcnVudGltZS5lcnJvckFkZHJlc3NTdHJpbmcuRXJyb3IAcnVu +dGltZS5wbGFpbkVycm9yLkVycm9yAHJ1bnRpbWUuYm91bmRzRXJyb3IuRXJyb3IAcnVudGltZS5h +cHBlbmRJbnRTdHIAcnVudGltZS5pdG9hAHJ1bnRpbWUucHJpbnRhbnkAcnVudGltZS5wcmludGFu +eWN1c3RvbXR5cGUAcnVudGltZS5wYW5pY3dyYXAAcnVudGltZS5tZW1oYXNoRmFsbGJhY2sAcnVu +dGltZS5yOABydW50aW1lLnJlYWRVbmFsaWduZWQ2NABydW50aW1lLnI0AHJ1bnRpbWUucmVhZFVu +YWxpZ25lZDMyAHJ1bnRpbWUubWl4AHJ1bnRpbWUubWVtaGFzaDMyRmFsbGJhY2sAcnVudGltZS5t +ZW1oYXNoNjRGYWxsYmFjawBydW50aW1lLmdldGl0YWIAcnVudGltZS4oKl90eXBlKS5uYW1lT2Zm +AHJ1bnRpbWUuKCppdGFiVGFibGVUeXBlKS5maW5kAHJ1bnRpbWUuaXRhYkhhc2hGdW5jAHJ1bnRp +bWUuaXRhYkFkZABydW50aW1lLigqaXRhYlRhYmxlVHlwZSkuYWRkAHJ1bnRpbWUuKCppdGFiKS5p +bml0AHJ1bnRpbWUuKCpfdHlwZSkudHlwZU9mZgBydW50aW1lLm5hbWUuaXNFeHBvcnRlZABydW50 +aW1lLml0YWJzaW5pdABydW50aW1lLmNvbnZUMkUAcnVudGltZS5jb252VHN0cmluZwBydW50aW1l +LmNvbnZUMkVub3B0cgBydW50aW1lLmFzc2VydEUySTIAcnVudGltZS5pdGVyYXRlX2l0YWJzAHJ1 +bnRpbWUudW5yZWFjaGFibGVNZXRob2QAcnVudGltZS4oKmxmc3RhY2spLnB1c2gAcnVudGltZS5s +ZnN0YWNrUGFjawBydW50aW1lLmxmc3RhY2tVbnBhY2sAcnVudGltZS5sZm5vZGVWYWxpZGF0ZQBy +dW50aW1lLmxvY2syAHJ1bnRpbWUudW5sb2NrMgBydW50aW1lLm5vdGV3YWtldXAAcnVudGltZS5u +b3Rlc2xlZXAAcnVudGltZS5ub3RldHNsZWVwX2ludGVybmFsAHJ1bnRpbWUubm90ZXRzbGVlcABy +dW50aW1lLm5vdGV0c2xlZXBnAHJ1bnRpbWUubG9ja1JhbmsuU3RyaW5nAHJ1bnRpbWUubWFsbG9j +aW5pdABydW50aW1lLigqbWhlYXApLnN5c0FsbG9jAHJ1bnRpbWUuYWxpZ25VcABydW50aW1lLigq +Zml4YWxsb2MpLmZyZWUAcnVudGltZS5zeXNSZXNlcnZlAHJ1bnRpbWUuc3lzUmVzZXJ2ZUFsaWdu +ZWQAcnVudGltZS4oKm1jYWNoZSkubmV4dEZyZWUAcnVudGltZS5tYWxsb2NnYwBydW50aW1lLmFj +cXVpcmVtAHJ1bnRpbWUuZ2V0TUNhY2hlAHJ1bnRpbWUucmVsZWFzZW0AcnVudGltZS5uZXh0RnJl +ZUZhc3QAcnVudGltZS5kaXZSb3VuZFVwAHJ1bnRpbWUubWFrZVNwYW5DbGFzcwBydW50aW1lLmJv +b2wyaW50AHJ1bnRpbWUuKCptc3BhbikuYmFzZQBydW50aW1lLmdjVHJpZ2dlci50ZXN0AHJ1bnRp +bWUubWVtY2xyTm9IZWFwUG9pbnRlcnNDaHVua2VkAHJ1bnRpbWUuZ29zY2hlZGd1YXJkZWQAcnVu +dGltZS5uZXdvYmplY3QAcnVudGltZS5uZXdhcnJheQBydW50aW1lLnByb2ZpbGVhbGxvYwBydW50 +aW1lLm5leHRTYW1wbGUAcnVudGltZS5mYXN0ZXhwcmFuZABydW50aW1lLmZhc3RyYW5kAHJ1bnRp +bWUuZmFzdGxvZzIAcnVudGltZS5mbG9hdDY0Yml0cwBydW50aW1lLnBlcnNpc3RlbnRhbGxvYwBy +dW50aW1lLnBlcnNpc3RlbnRhbGxvYy5mdW5jMQBydW50aW1lLnBlcnNpc3RlbnRhbGxvYzEAcnVu +dGltZS4oKm5vdEluSGVhcCkuYWRkAHJ1bnRpbWUuKCpsaW5lYXJBbGxvYykuYWxsb2MAcnVudGlt +ZS5zeXNVc2VkAHJ1bnRpbWUuKCpobWFwKS5pbmNybm92ZXJmbG93AHJ1bnRpbWUuKCpobWFwKS5u +ZXdvdmVyZmxvdwBydW50aW1lLigqYm1hcCkub3ZlcmZsb3cAcnVudGltZS4oKmJtYXApLnNldG92 +ZXJmbG93AHJ1bnRpbWUuKCpobWFwKS5jcmVhdGVPdmVyZmxvdwBydW50aW1lLm1ha2VtYXAAcnVu +dGltZS5vdmVyTG9hZEZhY3RvcgBydW50aW1lLmJ1Y2tldFNoaWZ0AHJ1bnRpbWUubWFrZUJ1Y2tl +dEFycmF5AHJ1bnRpbWUucm91bmR1cHNpemUAcnVudGltZS5tYXBhY2Nlc3MyAHJ1bnRpbWUuYnVj +a2V0TWFzawBydW50aW1lLigqaG1hcCkuc2FtZVNpemVHcm93AHJ1bnRpbWUuZXZhY3VhdGVkAHJ1 +bnRpbWUudG9waGFzaABydW50aW1lLigqbWFwdHlwZSkuaGFzaE1pZ2h0UGFuaWMAcnVudGltZS4o +Km1hcHR5cGUpLmluZGlyZWN0a2V5AHJ1bnRpbWUuKCptYXB0eXBlKS5pbmRpcmVjdGVsZW0AcnVu +dGltZS5tYXBhc3NpZ24AcnVudGltZS4oKmhtYXApLmdyb3dpbmcAcnVudGltZS50b29NYW55T3Zl +cmZsb3dCdWNrZXRzAHJ1bnRpbWUuaXNFbXB0eQBydW50aW1lLigqbWFwdHlwZSkubmVlZGtleXVw +ZGF0ZQBydW50aW1lLmhhc2hHcm93AHJ1bnRpbWUuZ3Jvd1dvcmsAcnVudGltZS4oKmhtYXApLm9s +ZGJ1Y2tldG1hc2sAcnVudGltZS4oKmhtYXApLm5vbGRidWNrZXRzAHJ1bnRpbWUuZXZhY3VhdGUA +cnVudGltZS4oKm1hcHR5cGUpLnJlZmxleGl2ZWtleQBydW50aW1lLmFkdmFuY2VFdmFjdWF0aW9u +TWFyawBydW50aW1lLmJ1Y2tldEV2YWN1YXRlZABydW50aW1lLm1hcGFjY2VzczFfZmFzdDMyAHJ1 +bnRpbWUuKCpibWFwKS5rZXlzAHJ1bnRpbWUubWFwYWNjZXNzMl9mYXN0MzIAcnVudGltZS5tYXBh +c3NpZ25fZmFzdDMyAHJ1bnRpbWUuZ3Jvd1dvcmtfZmFzdDMyAHJ1bnRpbWUuZXZhY3VhdGVfZmFz +dDMyAHJ1bnRpbWUudHlwZWRtZW1tb3ZlAHJ1bnRpbWUudHlwZWRzbGljZWNvcHkAcnVudGltZS50 +eXBlZG1lbWNscgBydW50aW1lLm1lbWNsckhhc1BvaW50ZXJzAHJ1bnRpbWUuKCptc3BhbikucmVm +aWxsQWxsb2NDYWNoZQBydW50aW1lLigqZ2NCaXRzKS5ieXRlcABydW50aW1lLigqbXNwYW4pLm5l +eHRGcmVlSW5kZXgAcnVudGltZS5iYWRQb2ludGVyAHJ1bnRpbWUuKCptU3BhblN0YXRlQm94KS5n +ZXQAcnVudGltZS5maW5kT2JqZWN0AHJ1bnRpbWUuc3Bhbk9mAHJ1bnRpbWUuKCptc3Bhbikub2Jq +SW5kZXgAcnVudGltZS4oKm1zcGFuKS5kaXZpZGVCeUVsZW1TaXplAHJ1bnRpbWUuaGVhcEJpdHMu +bmV4dEFyZW5hAHJ1bnRpbWUuaGVhcEJpdHMuZm9yd2FyZABydW50aW1lLmhlYXBCaXRzLmZvcndh +cmRPckJvdW5kYXJ5AHJ1bnRpbWUuYnVsa0JhcnJpZXJQcmVXcml0ZQBydW50aW1lLnB1aW50cHRy +LnB0cgBydW50aW1lLmhlYXBCaXRzLmlzUG9pbnRlcgBydW50aW1lLmJ1bGtCYXJyaWVyUHJlV3Jp +dGVTcmNPbmx5AHJ1bnRpbWUuYnVsa0JhcnJpZXJCaXRtYXAAcnVudGltZS50eXBlQml0c0J1bGtC +YXJyaWVyAHJ1bnRpbWUuaGVhcEJpdHMuaW5pdFNwYW4AcnVudGltZS5oZWFwQml0c1NldFR5cGUA +cnVudGltZS5oZWFwQml0c1NldFR5cGVHQ1Byb2cAcnVudGltZS5wcm9nVG9Qb2ludGVyTWFzawBy +dW50aW1lLnJ1bkdDUHJvZwBydW50aW1lLnN1YnRyYWN0MQBydW50aW1lLnN1YnRyYWN0YgBydW50 +aW1lLm1hdGVyaWFsaXplR0NQcm9nAHJ1bnRpbWUuYWxsb2NtY2FjaGUAcnVudGltZS5hbGxvY21j +YWNoZS5mdW5jMQBydW50aW1lLmZyZWVtY2FjaGUAcnVudGltZS5mcmVlbWNhY2hlLmZ1bmMxAHJ1 +bnRpbWUuKCptY2FjaGUpLnJlZmlsbABydW50aW1lLnNwYW5DbGFzcy5zaXplY2xhc3MAcnVudGlt +ZS50cmFjZUhlYXBBbGxvYwBydW50aW1lLigqbWNhY2hlKS5hbGxvY0xhcmdlAHJ1bnRpbWUuKCpt +Y2VudHJhbCkuZnVsbFN3ZXB0AHJ1bnRpbWUuKCptY2FjaGUpLnJlbGVhc2VBbGwAcnVudGltZS4o +Km1jYWNoZSkucHJlcGFyZUZvclN3ZWVwAHJ1bnRpbWUuKCptY2VudHJhbCkuY2FjaGVTcGFuAHJ1 +bnRpbWUubmV3U3dlZXBMb2NrZXIAcnVudGltZS4oKm1jZW50cmFsKS5wYXJ0aWFsU3dlcHQAcnVu +dGltZS4oKm1jZW50cmFsKS5wYXJ0aWFsVW5zd2VwdABydW50aW1lLigqc3dlZXBMb2NrZXIpLnRy +eUFjcXVpcmUAcnVudGltZS4oKnN3ZWVwTG9ja2VyKS5ibG9ja0NvbXBsZXRpb24AcnVudGltZS4o +KnN3ZWVwTG9ja2VyKS5kaXNwb3NlAHJ1bnRpbWUuKCpzd2VlcExvY2tlcikuc3dlZXBJc0RvbmUA +cnVudGltZS4oKm1jZW50cmFsKS5mdWxsVW5zd2VwdABydW50aW1lLigqbWNlbnRyYWwpLnVuY2Fj +aGVTcGFuAHJ1bnRpbWUuKCptY2VudHJhbCkuZ3JvdwBydW50aW1lLnN0YXJ0Q2hlY2ttYXJrcwBy +dW50aW1lLmVuZENoZWNrbWFya3MAcnVudGltZS5nY01hcmtXb3JrQXZhaWxhYmxlAHJ1bnRpbWUu +KCpsZnN0YWNrKS5lbXB0eQBydW50aW1lLnNldENoZWNrbWFyawBydW50aW1lLm1hcmtCaXRzLmlz +TWFya2VkAHJ1bnRpbWUuc3lzQWxsb2MAcnVudGltZS5zeXNVbnVzZWQAcnVudGltZS5hbGlnbkRv +d24AcnVudGltZS5zeXNIdWdlUGFnZQBydW50aW1lLnN5c0ZyZWUAcnVudGltZS5zeXNNYXAAcnVu +dGltZS5xdWV1ZWZpbmFsaXplcgBydW50aW1lLndha2VmaW5nAHJ1bnRpbWUuKCpmaXhhbGxvYyku +YWxsb2MAcnVudGltZS5nY2luaXQAcnVudGltZS5nY2VuYWJsZQBydW50aW1lLnBvbGxGcmFjdGlv +bmFsV29ya2VyRXhpdABydW50aW1lLmdjU3RhcnQAcnVudGltZS5zZW1hY3F1aXJlAHJ1bnRpbWUu +dHJhY2VHQ1N0YXJ0AHJ1bnRpbWUuc2VtcmVsZWFzZQBydW50aW1lLnRyYWNlR0NTVFdTdGFydABy +dW50aW1lLnNldEdDUGhhc2UAcnVudGltZS5nY0JnTWFya1ByZXBhcmUAcnVudGltZS5Hb3NjaGVk +AHJ1bnRpbWUuZ2NTdGFydC5mdW5jMgBydW50aW1lLigqdGltZUhpc3RvZ3JhbSkucmVjb3JkAHJ1 +bnRpbWUuZ2NNYXJrRG9uZQBydW50aW1lLmdjTWFya0RvbmUuZnVuYzIAcnVudGltZS4oKmdjV29y +aykuZW1wdHkAcnVudGltZS5nY01hcmtUZXJtaW5hdGlvbgBydW50aW1lLnRyYWNlR0NEb25lAHJ1 +bnRpbWUuaXRvYURpdgBydW50aW1lLnByaW50dW5sb2NrAHJ1bnRpbWUuZ2NNYXJrVGVybWluYXRp +b24uZnVuYzEAcnVudGltZS5nY0JnTWFya1N0YXJ0V29ya2VycwBydW50aW1lLm5vdGVjbGVhcgBy +dW50aW1lLmdjQmdNYXJrV29ya2VyAHJ1bnRpbWUuZ2NCZ01hcmtXb3JrZXIuZnVuYzIAcnVudGlt +ZS5nbG9icnVucXB1dGJhdGNoAHJ1bnRpbWUuKCpnUXVldWUpLnB1c2hCYWNrQWxsAHJ1bnRpbWUu +Z3VpbnRwdHIucHRyAHJ1bnRpbWUuZ2NNYXJrAHJ1bnRpbWUuZ2NTd2VlcABydW50aW1lLigqc3dl +ZXBDbGFzcykuY2xlYXIAcnVudGltZS5nY1Jlc2V0TWFya1N0YXRlAHJ1bnRpbWUuY2xlYXJwb29s +cwBydW50aW1lLmZtdE5TQXNNUwBydW50aW1lLmdjTWFya1Jvb3RQcmVwYXJlAHJ1bnRpbWUuZ2NN +YXJrUm9vdFByZXBhcmUuZnVuYzEAcnVudGltZS5nY01hcmtSb290Q2hlY2sAcnVudGltZS5nY01h +cmtSb290Q2hlY2suZnVuYzEAcnVudGltZS5yZWFkZ3N0YXR1cwBydW50aW1lLm1hcmtyb290AHJ1 +bnRpbWUubWFya3Jvb3QuZnVuYzEAcnVudGltZS5tYXJrcm9vdEJsb2NrAHJ1bnRpbWUubWFya3Jv +b3RGcmVlR1N0YWNrcwBydW50aW1lLigqZ0xpc3QpLnB1c2hBbGwAcnVudGltZS4oKmdRdWV1ZSku +ZW1wdHkAcnVudGltZS5tYXJrcm9vdFNwYW5zAHJ1bnRpbWUuZ2NBc3Npc3RBbGxvYwBydW50aW1l +LmZsb2F0NjRmcm9tYml0cwBydW50aW1lLnRyYWNlR0NNYXJrQXNzaXN0U3RhcnQAcnVudGltZS50 +cmFjZUdDTWFya0Fzc2lzdERvbmUAcnVudGltZS5nY0Fzc2lzdEFsbG9jLmZ1bmMxAHJ1bnRpbWUu +Z2NBc3Npc3RBbGxvYzEAcnVudGltZS5nY1dha2VBbGxBc3Npc3RzAHJ1bnRpbWUuKCpnUXVldWUp +LnBvcExpc3QAcnVudGltZS5nY1BhcmtBc3Npc3QAcnVudGltZS4oKmdRdWV1ZSkucHVzaEJhY2sA +cnVudGltZS5nb3Bhcmt1bmxvY2sAcnVudGltZS5nY0ZsdXNoQmdDcmVkaXQAcnVudGltZS4oKmdR +dWV1ZSkucG9wAHJ1bnRpbWUuc2NhbnN0YWNrAHJ1bnRpbWUuaXNTaHJpbmtTdGFja1NhZmUAcnVu +dGltZS4oKnN0YWNrU2NhblN0YXRlKS5idWlsZEluZGV4AHJ1bnRpbWUuKCpzdGFja1NjYW5TdGF0 +ZSkuZmluZE9iamVjdABydW50aW1lLigqc3RhY2tPYmplY3QpLnNldFJlY29yZABydW50aW1lLigq +c3RhY2tPYmplY3RSZWNvcmQpLnVzZUdDUHJvZwBydW50aW1lLigqc3RhY2tPYmplY3RSZWNvcmQp +LnB0cmRhdGEAcnVudGltZS5kZW1hdGVyaWFsaXplR0NQcm9nAHJ1bnRpbWUuc2NhbnN0YWNrLmZ1 +bmMxAHJ1bnRpbWUuc2NhbmZyYW1ld29ya2VyAHJ1bnRpbWUuZ2NEcmFpbgBydW50aW1lLigqZ2NX +b3JrKS50cnlHZXRGYXN0AHJ1bnRpbWUuZ2NEcmFpbk4AcnVudGltZS5zY2FuYmxvY2sAcnVudGlt +ZS5zY2Fub2JqZWN0AHJ1bnRpbWUuc3BhbkNsYXNzLm5vc2NhbgBydW50aW1lLigqZ2NXb3JrKS5w +dXRGYXN0AHJ1bnRpbWUuc2NhbkNvbnNlcnZhdGl2ZQBydW50aW1lLigqbXNwYW4pLmlzRnJlZQBy +dW50aW1lLigqZ2NCaXRzKS5iaXRwAHJ1bnRpbWUuc2hhZGUAcnVudGltZS5ncmV5b2JqZWN0AHJ1 +bnRpbWUuKCptc3BhbikubWFya0JpdHNGb3JJbmRleABydW50aW1lLm1hcmtCaXRzLnNldE1hcmtl +ZABydW50aW1lLnBhZ2VJbmRleE9mAHJ1bnRpbWUuZ2NEdW1wT2JqZWN0AHJ1bnRpbWUuZ2NtYXJr +bmV3b2JqZWN0AHJ1bnRpbWUuZ2NNYXJrVGlueUFsbG9jcwBydW50aW1lLmluaXQuMQBydW50aW1l +LigqZ2NDb250cm9sbGVyU3RhdGUpLmluaXQAcnVudGltZS4oKmdjQ29udHJvbGxlclN0YXRlKS5z +dGFydEN5Y2xlAHJ1bnRpbWUuKCpnY0NvbnRyb2xsZXJTdGF0ZSkucmV2aXNlAHJ1bnRpbWUuKCpn +Y0NvbnRyb2xsZXJTdGF0ZSkuZW5kQ3ljbGUAcnVudGltZS4oKmdjQ29udHJvbGxlclN0YXRlKS5l +ZmZlY3RpdmVHcm93dGhSYXRpbwBydW50aW1lLigqZ2NDb250cm9sbGVyU3RhdGUpLmVubGlzdFdv +cmtlcgBydW50aW1lLmZhc3RyYW5kbgBydW50aW1lLigqZ2NDb250cm9sbGVyU3RhdGUpLmZpbmRS +dW5uYWJsZUdDV29ya2VyAHJ1bnRpbWUuKCpsZnN0YWNrKS5wb3AAcnVudGltZS4oKmdjQ29udHJv +bGxlclN0YXRlKS5maW5kUnVubmFibGVHQ1dvcmtlci5mdW5jMQBydW50aW1lLigqZ2NDb250cm9s +bGVyU3RhdGUpLmNvbW1pdABydW50aW1lLmlzU3dlZXBEb25lAHJ1bnRpbWUuKCpnY0NvbnRyb2xs +ZXJTdGF0ZSkuc2V0R0NQZXJjZW50AHJ1bnRpbWUucmVhZEdPR0MAcnVudGltZS5hdG9pMzIAcnVu +dGltZS5nY1BhY2VTY2F2ZW5nZXIAcnVudGltZS5oZWFwUmV0YWluZWQAcnVudGltZS4oKnN5c01l +bVN0YXQpLmxvYWQAcnVudGltZS53YWtlU2NhdmVuZ2VyAHRpbWUuc3RvcFRpbWVyAHJ1bnRpbWUu +c2NhdmVuZ2VTbGVlcAB0aW1lLnJlc2V0VGltZXIAcnVudGltZS5yZXNldHRpbWVyAHJ1bnRpbWUu +YmdzY2F2ZW5nZQBydW50aW1lLmJnc2NhdmVuZ2UuZnVuYzIAcnVudGltZS4oKnBhZ2VBbGxvYyku +c2NhdmVuZ2UAcnVudGltZS5hZGRyUmFuZ2Uuc2l6ZQBydW50aW1lLm9mZkFkZHIubGVzc1RoYW4A +cnVudGltZS5vZmZBZGRyLmRpZmYAcnVudGltZS5wcmludFNjYXZUcmFjZQBydW50aW1lLigqcGFn +ZUFsbG9jKS5zY2F2ZW5nZVN0YXJ0R2VuAHJ1bnRpbWUuKCpwYWdlQWxsb2MpLnNjYXZlbmdlUmVz +ZXJ2ZQBydW50aW1lLigqcGFnZUFsbG9jKS5zY2F2ZW5nZVVucmVzZXJ2ZQBydW50aW1lLigqcGFn +ZUFsbG9jKS5zY2F2ZW5nZU9uZQBydW50aW1lLmNodW5rSW5kZXgAcnVudGltZS5wYWxsb2NTdW0u +bWF4AHJ1bnRpbWUuKCpwYWdlQWxsb2MpLmNodW5rT2YAcnVudGltZS5jaHVua0lkeC5sMQBydW50 +aW1lLmNodW5rSWR4LmwyAHJ1bnRpbWUuY2h1bmtQYWdlSW5kZXgAcnVudGltZS5jaHVua0Jhc2UA +cnVudGltZS4oKnBhZ2VBbGxvYykuc2NhdmVuZ2VPbmUuZnVuYzIAcnVudGltZS4oKnBhZ2VBbGxv +Yykuc2NhdmVuZ2VPbmUuZnVuYzEAcnVudGltZS4oKnBhZ2VBbGxvYykuc2NhdmVuZ2VPbmUuZnVu +YzMAcnVudGltZS5vZmZBZGRyLmFkZHIAcnVudGltZS4oKnBhZ2VBbGxvYykuc2NhdmVuZ2VSYW5n +ZUxvY2tlZABydW50aW1lLmZpbGxBbGlnbmVkAHJ1bnRpbWUuZmlsbEFsaWduZWQuZnVuYzEAcnVu +dGltZS4oKnBhbGxvY0RhdGEpLmhhc1NjYXZlbmdlQ2FuZGlkYXRlAHJ1bnRpbWUuKCpwYWxsb2NE +YXRhKS5maW5kU2NhdmVuZ2VDYW5kaWRhdGUAcnVudGltZS9pbnRlcm5hbC9zeXMuTGVhZGluZ1pl +cm9zNjQAcnVudGltZS4oKnN0YWNrU2NhblN0YXRlKS5wdXRQdHIAcnVudGltZS4oKnN0YWNrU2Nh +blN0YXRlKS5nZXRQdHIAcnVudGltZS4oKnN0YWNrU2NhblN0YXRlKS5hZGRPYmplY3QAcnVudGlt +ZS5iaW5hcnlTZWFyY2hUcmVlAHJ1bnRpbWUuKCptaGVhcCkubmV4dFNwYW5Gb3JTd2VlcABydW50 +aW1lLigqc3dlZXBDbGFzcykubG9hZABydW50aW1lLnN3ZWVwQ2xhc3Muc3BsaXQAcnVudGltZS4o +KnN3ZWVwQ2xhc3MpLnVwZGF0ZQBydW50aW1lLmZpbmlzaHN3ZWVwX20AcnVudGltZS5iZ3N3ZWVw +AHJ1bnRpbWUuc3dlZXBvbmUAcnVudGltZS5yZWFkeUZvclNjYXZlbmdlcgBydW50aW1lLigqbXNw +YW4pLmVuc3VyZVN3ZXB0AHJ1bnRpbWUuKCpzd2VlcExvY2tlZCkuc3dlZXAAcnVudGltZS5uZXdT +cGVjaWFsc0l0ZXIAcnVudGltZS4oKnNwZWNpYWxzSXRlcikudmFsaWQAcnVudGltZS4oKnNwZWNp +YWxzSXRlcikubmV4dABydW50aW1lLigqc3BlY2lhbHNJdGVyKS51bmxpbmtBbmROZXh0AHJ1bnRp +bWUubWFya0JpdHMuc2V0TWFya2VkTm9uQXRvbWljAHJ1bnRpbWUuc3Bhbkhhc05vU3BlY2lhbHMA +cnVudGltZS4oKm1zcGFuKS5tYXJrQml0c0ZvckJhc2UAcnVudGltZS4oKm1zcGFuKS5hbGxvY0Jp +dHNGb3JJbmRleABydW50aW1lLnN5c0ZhdWx0AHJ1bnRpbWUuKCptc3BhbikuY291bnRBbGxvYwBy +dW50aW1lLigqbWFya0JpdHMpLmFkdmFuY2UAcnVudGltZS5jbG9iYmVyZnJlZQBydW50aW1lLigq +bXNwYW4pLnJlcG9ydFpvbWJpZXMAcnVudGltZS5kZWR1Y3RTd2VlcENyZWRpdABydW50aW1lLigq +Z2NXb3JrKS5pbml0AHJ1bnRpbWUuKCpnY1dvcmspLnB1dABydW50aW1lLigqZ2NXb3JrKS5wdXRC +YXRjaABydW50aW1lLigqZ2NXb3JrKS50cnlHZXQAcnVudGltZS4oKmdjV29yaykuZGlzcG9zZQBy +dW50aW1lLigqZ2NXb3JrKS5iYWxhbmNlAHJ1bnRpbWUuKCp3b3JrYnVmKS5jaGVja25vbmVtcHR5 +AHJ1bnRpbWUuKCp3b3JrYnVmKS5jaGVja2VtcHR5AHJ1bnRpbWUuZ2V0ZW1wdHkAcnVudGltZS5n +ZXRlbXB0eS5mdW5jMQBydW50aW1lLnB1dGVtcHR5AHJ1bnRpbWUucHV0ZnVsbABydW50aW1lLnRy +eWdldGZ1bGwAcnVudGltZS5oYW5kb2ZmAHJ1bnRpbWUucHJlcGFyZUZyZWVXb3JrYnVmcwBydW50 +aW1lLigqbVNwYW5MaXN0KS50YWtlQWxsAHJ1bnRpbWUuKCptU3Bhbkxpc3QpLmlzRW1wdHkAcnVu +dGltZS5mcmVlU29tZVdidWZzAHJ1bnRpbWUuZnJlZVNvbWVXYnVmcy5mdW5jMQBydW50aW1lLnJl +Y29yZHNwYW4AcnVudGltZS5pbkhlYXBPclN0YWNrAHJ1bnRpbWUuc3Bhbk9mSGVhcABydW50aW1l +LigqbWhlYXApLmluaXQAcnVudGltZS4oKmZpeGFsbG9jKS5pbml0AHJ1bnRpbWUuKCptY2VudHJh +bCkuaW5pdABydW50aW1lLigqbWhlYXApLnJlY2xhaW0AcnVudGltZS4oKm1oZWFwKS5yZWNsYWlt +Q2h1bmsAcnVudGltZS4oKm1oZWFwKS5hbGxvYwBydW50aW1lLigqbWhlYXApLmFsbG9jLmZ1bmMx +AHJ1bnRpbWUuKCptaGVhcCkuYWxsb2NNYW51YWwAcnVudGltZS4oKm1oZWFwKS5zZXRTcGFucwBy +dW50aW1lLigqbWhlYXApLmFsbG9jTmVlZHNaZXJvAHJ1bnRpbWUuKCptaGVhcCkuYWxsb2NNU3Bh +bkxvY2tlZABydW50aW1lLigqbWhlYXApLmFsbG9jU3BhbgBydW50aW1lLigqcGFnZUNhY2hlKS5l +bXB0eQBydW50aW1lLigqbWhlYXApLnRyeUFsbG9jTVNwYW4AcnVudGltZS4oKm1zcGFuKS5pbml0 +AHJ1bnRpbWUuKCptU3BhblN0YXRlQm94KS5zZXQAcnVudGltZS5zcGFuQWxsb2NUeXBlLm1hbnVh +bABydW50aW1lLigqbWhlYXApLmdyb3cAcnVudGltZS4oKm1oZWFwKS5mcmVlU3BhbgBydW50aW1l +LigqbWhlYXApLmZyZWVTcGFuLmZ1bmMxAHJ1bnRpbWUuKCptaGVhcCkuZnJlZU1hbnVhbABydW50 +aW1lLigqbWhlYXApLmZyZWVTcGFuTG9ja2VkAHJ1bnRpbWUuKCptaGVhcCkuZnJlZU1TcGFuTG9j +a2VkAHJ1bnRpbWUuKCptU3Bhbkxpc3QpLnJlbW92ZQBydW50aW1lLigqbVNwYW5MaXN0KS5pbnNl +cnQAcnVudGltZS5hZGRzcGVjaWFsAHJ1bnRpbWUuc3Bhbkhhc1NwZWNpYWxzAHJ1bnRpbWUuc2V0 +cHJvZmlsZWJ1Y2tldABydW50aW1lLmZyZWVTcGVjaWFsAHJ1bnRpbWUubmV3TWFya0JpdHMAcnVu +dGltZS4oKmdjQml0c0FyZW5hKS50cnlBbGxvYwBydW50aW1lLm5ld0FsbG9jQml0cwBydW50aW1l +Lm5leHRNYXJrQml0QXJlbmFFcG9jaABydW50aW1lLm5ld0FyZW5hTWF5VW5sb2NrAHJ1bnRpbWUu +KCpwYWdlQWxsb2MpLmluaXQAcnVudGltZS4oKnBhZ2VBbGxvYykuZ3JvdwBydW50aW1lLigqcGFn +ZUFsbG9jKS51cGRhdGUAcnVudGltZS5hZGRyc1RvU3VtbWFyeVJhbmdlAHJ1bnRpbWUuKCpwYWdl +QWxsb2MpLmFsbG9jUmFuZ2UAcnVudGltZS4oKnBhZ2VBbGxvYykuZmluZE1hcHBlZEFkZHIAcnVu +dGltZS4oKnBhZ2VBbGxvYykuZmluZABydW50aW1lLm9mZkFkZHJUb0xldmVsSW5kZXgAcnVudGlt +ZS5wYWxsb2NTdW0uc3RhcnQAcnVudGltZS5wYWxsb2NTdW0uZW5kAHJ1bnRpbWUubGV2ZWxJbmRl +eFRvT2ZmQWRkcgBydW50aW1lLm9mZkFkZHIuYWRkAHJ1bnRpbWUuKCpwYWdlQWxsb2MpLmZpbmQu +ZnVuYzEAcnVudGltZS5vZmZBZGRyLmxlc3NFcXVhbABydW50aW1lLigqcGFnZUFsbG9jKS5hbGxv +YwBydW50aW1lLigqcGFnZUFsbG9jKS5mcmVlAHJ1bnRpbWUuKCpwYWxsb2NCaXRzKS5mcmVlAHJ1 +bnRpbWUuKCpwYWxsb2NCaXRzKS5mcmVlMQBydW50aW1lLigqcGFnZUJpdHMpLmNsZWFyAHJ1bnRp +bWUuKCpwYWxsb2NCaXRzKS5mcmVlQWxsAHJ1bnRpbWUubWVyZ2VTdW1tYXJpZXMAcnVudGltZS5w +YWxsb2NTdW0udW5wYWNrAHJ1bnRpbWUucGFja1BhbGxvY1N1bQBydW50aW1lLigqcGFnZUFsbG9j +KS5zeXNJbml0AHJ1bnRpbWUuKCpwYWdlQWxsb2MpLnN5c0dyb3cAcnVudGltZS4oKnBhZ2VBbGxv +Yykuc3lzR3Jvdy5mdW5jMgBydW50aW1lLigqcGFnZUFsbG9jKS5zeXNHcm93LmZ1bmMzAHJ1bnRp +bWUuKCpwYWdlQ2FjaGUpLmFsbG9jAHJ1bnRpbWUuKCpwYWdlQ2FjaGUpLmFsbG9jTgBydW50aW1l +LmZpbmRCaXRSYW5nZTY0AHJ1bnRpbWUuKCpwYWdlQ2FjaGUpLmZsdXNoAHJ1bnRpbWUuKCpwYWdl +QWxsb2MpLmFsbG9jVG9DYWNoZQBydW50aW1lLigqcGFsbG9jQml0cykucGFnZXM2NABydW50aW1l +LigqcGFnZUJpdHMpLmJsb2NrNjQAcnVudGltZS4oKnBhZ2VCaXRzKS5zZXRSYW5nZQBydW50aW1l +LigqcGFnZUJpdHMpLnNldABydW50aW1lLigqcGFnZUJpdHMpLnNldEFsbABydW50aW1lLigqcGFn +ZUJpdHMpLmNsZWFyUmFuZ2UAcnVudGltZS4oKnBhZ2VCaXRzKS5jbGVhckFsbABydW50aW1lLigq +cGFnZUJpdHMpLnBvcGNudFJhbmdlAHJ1bnRpbWUuKCpwYWxsb2NCaXRzKS5zdW1tYXJpemUAcnVu +dGltZS4oKnBhbGxvY0JpdHMpLmZpbmQAcnVudGltZS4oKnBhbGxvY0JpdHMpLmZpbmQxAHJ1bnRp +bWUuKCpwYWxsb2NCaXRzKS5maW5kU21hbGxOAHJ1bnRpbWUuKCpwYWxsb2NCaXRzKS5maW5kTGFy +Z2VOAHJ1bnRpbWUuKCpwYWxsb2NEYXRhKS5hbGxvY1JhbmdlAHJ1bnRpbWUuKCpwYWxsb2NCaXRz +KS5hbGxvY1JhbmdlAHJ1bnRpbWUuKCpwYWxsb2NEYXRhKS5hbGxvY0FsbABydW50aW1lLigqcGFs +bG9jQml0cykuYWxsb2NBbGwAcnVudGltZS5uZXdCdWNrZXQAcnVudGltZS4oKmJ1Y2tldCkubXAA +cnVudGltZS4oKmJ1Y2tldCkuYnAAcnVudGltZS5zdGtidWNrZXQAcnVudGltZS4oKmJ1Y2tldCku +c3RrAHJ1bnRpbWUuZXFzbGljZQBydW50aW1lLm1Qcm9mX05leHRDeWNsZQBydW50aW1lLm1Qcm9m +X0ZsdXNoAHJ1bnRpbWUubVByb2ZfRmx1c2hMb2NrZWQAcnVudGltZS4oKm1lbVJlY29yZEN5Y2xl +KS5hZGQAcnVudGltZS5tUHJvZl9NYWxsb2MAcnVudGltZS5tUHJvZl9NYWxsb2MuZnVuYzEAcnVu +dGltZS5tUHJvZl9GcmVlAHJ1bnRpbWUuYmxvY2tldmVudABydW50aW1lLmJsb2Nrc2FtcGxlZABy +dW50aW1lLnNhdmVibG9ja2V2ZW50AHJ1bnRpbWUuZ2NhbGxlcnMAcnVudGltZS50cmFjZWFsbG9j +AHJ1bnRpbWUudHJhY2ViYWNrAHJ1bnRpbWUudHJhY2VhbGxvYy5mdW5jMQBydW50aW1lLnRyYWNl +ZnJlZQBydW50aW1lLnRyYWNlZnJlZS5mdW5jMQBydW50aW1lLnRyYWNlZ2MAcnVudGltZS5tYWtl +QWRkclJhbmdlAHJ1bnRpbWUuYWRkclJhbmdlLnN1YnRyYWN0AHJ1bnRpbWUuYWRkclJhbmdlLnJl +bW92ZUdyZWF0ZXJFcXVhbABydW50aW1lLigqYWRkclJhbmdlcykuaW5pdABydW50aW1lLigqYWRk +clJhbmdlcykuZmluZFN1Y2MAcnVudGltZS5hZGRyUmFuZ2UuY29udGFpbnMAcnVudGltZS4oKmFk +ZHJSYW5nZXMpLmZpbmRBZGRyR3JlYXRlckVxdWFsAHJ1bnRpbWUuKCphZGRyUmFuZ2VzKS5hZGQA +cnVudGltZS5vZmZBZGRyLmVxdWFsAHJ1bnRpbWUuKCphZGRyUmFuZ2VzKS5yZW1vdmVMYXN0AHJ1 +bnRpbWUub2ZmQWRkci5zdWIAcnVudGltZS4oKmFkZHJSYW5nZXMpLnJlbW92ZUdyZWF0ZXJFcXVh +bABydW50aW1lLigqYWRkclJhbmdlcykuY2xvbmVJbnRvAHJ1bnRpbWUuKCpzcGFuU2V0KS5wdXNo +AHJ1bnRpbWUuKCpzcGFuU2V0KS5wb3AAcnVudGltZS5oZWFkVGFpbEluZGV4LnNwbGl0AHJ1bnRp +bWUuaGVhZFRhaWxJbmRleC5oZWFkAHJ1bnRpbWUuKCpoZWFkVGFpbEluZGV4KS5sb2FkAHJ1bnRp +bWUuKCpoZWFkVGFpbEluZGV4KS5jYXMAcnVudGltZS5tYWtlSGVhZFRhaWxJbmRleABydW50aW1l +Ligqc3BhblNldEJsb2NrQWxsb2MpLmZyZWUAcnVudGltZS4oKnNwYW5TZXQpLnJlc2V0AHJ1bnRp +bWUuKCpoZWFkVGFpbEluZGV4KS5yZXNldABydW50aW1lLigqc3BhblNldEJsb2NrQWxsb2MpLmFs +bG9jAHJ1bnRpbWUuKCpoZWFkVGFpbEluZGV4KS5pbmNUYWlsAHJ1bnRpbWUuaW5pdC40AHJ1bnRp +bWUuKCpzeXNNZW1TdGF0KS5hZGQAcnVudGltZS4oKmNvbnNpc3RlbnRIZWFwU3RhdHMpLmFjcXVp +cmUAcnVudGltZS4oKmNvbnNpc3RlbnRIZWFwU3RhdHMpLnJlbGVhc2UAcnVudGltZS4oKndiQnVm +KS5yZXNldABydW50aW1lLndiQnVmRmx1c2gAcnVudGltZS4oKndiQnVmKS5kaXNjYXJkAHJ1bnRp +bWUud2JCdWZGbHVzaDEAcnVudGltZS5ub25ibG9ja2luZ1BpcGUAcnVudGltZS5uZXRwb2xsR2Vu +ZXJpY0luaXQAcnVudGltZS5uZXRwb2xscmVhZHkAcnVudGltZS5uZXRwb2xsdW5ibG9jawBydW50 +aW1lLm5ldHBvbGxpbml0AHJ1bnRpbWUubmV0cG9sbEJyZWFrAHJ1bnRpbWUud3JpdGUAcnVudGlt +ZS5uZXRwb2xsAHJ1bnRpbWUuZnV0ZXhzbGVlcABydW50aW1lLigqdGltZXNwZWMpLnNldE5zZWMA +cnVudGltZS5mdXRleHdha2V1cABydW50aW1lLmZ1dGV4d2FrZXVwLmZ1bmMxAHJ1bnRpbWUuZ2V0 +cHJvY2NvdW50AHJ1bnRpbWUubmV3b3Nwcm9jAHJ1bnRpbWUuc2lncHJvY21hc2sAcnVudGltZS5t +Y291bnQAcnVudGltZS5zeXNhcmdzAHJ1bnRpbWUuYXJndl9pbmRleABydW50aW1lLnN5c2F1eHYA +cnVudGltZS5nZXRIdWdlUGFnZVNpemUAcnVudGltZS5vc2luaXQAcnVudGltZS5zaWdkZWxzZXQA +cnVudGltZS5nZXRSYW5kb21EYXRhAHJ1bnRpbWUubXByZWluaXQAcnVudGltZS5taW5pdABydW50 +aW1lLnNldHNpZwBydW50aW1lLnNpZ2ZpbGxzZXQAcnVudGltZS5zZXRzaWdzdGFjawBydW50aW1l +LnN5c1NpZ2FjdGlvbgBydW50aW1lLnNpZ25hbE0AcnVudGltZS5wYW5pY0NoZWNrMQBydW50aW1l +Lmhhc1ByZWZpeABydW50aW1lLnBhbmljQ2hlY2syAHJ1bnRpbWUuZ29QYW5pY0luZGV4AHJ1bnRp +bWUuZ29QYW5pY0luZGV4VQBydW50aW1lLmdvUGFuaWNTbGljZUFsZW4AcnVudGltZS5nb1Bhbmlj +U2xpY2VBbGVuVQBydW50aW1lLmdvUGFuaWNTbGljZUFjYXAAcnVudGltZS5nb1BhbmljU2xpY2VB +Y2FwVQBydW50aW1lLmdvUGFuaWNTbGljZUIAcnVudGltZS5nb1BhbmljU2xpY2VCVQBydW50aW1l +LmdvUGFuaWNTbGljZTNBbGVuAHJ1bnRpbWUuZ29QYW5pY1NsaWNlM0FsZW5VAHJ1bnRpbWUucGFu +aWNzaGlmdABydW50aW1lLnBhbmljZGl2aWRlAHJ1bnRpbWUudGVzdGRlZmVyc2l6ZXMAcnVudGlt +ZS5kZWZlcmNsYXNzAHJ1bnRpbWUudG90YWxkZWZlcnNpemUAcnVudGltZS5pbml0LjUAcnVudGlt +ZS5uZXdkZWZlcgBydW50aW1lLm5ld2RlZmVyLmZ1bmMyAHJ1bnRpbWUubmV3ZGVmZXIuZnVuYzEA +cnVudGltZS5mcmVlZGVmZXIAcnVudGltZS5mcmVlZGVmZXIuZnVuYzEAcnVudGltZS5mcmVlZGVm +ZXJwYW5pYwBydW50aW1lLmZyZWVkZWZlcmZuAHJ1bnRpbWUuZGVmZXJyZXR1cm4AcnVudGltZS5k +ZWZlckFyZ3MAcnVudGltZS5wcmVwcmludHBhbmljcwBydW50aW1lLnByaW50cGFuaWNzAHJ1bnRp +bWUuYWRkT25lT3BlbkRlZmVyRnJhbWUAcnVudGltZS5hZGRPbmVPcGVuRGVmZXJGcmFtZS5mdW5j +MQBydW50aW1lLmFkZE9uZU9wZW5EZWZlckZyYW1lLmZ1bmMxLjEAcnVudGltZS5mdW5jZGF0YQBy +dW50aW1lLnJlYWR2YXJpbnRVbnNhZmUAcnVudGltZS5ydW5PcGVuRGVmZXJGcmFtZQBydW50aW1l +LmRlZmVyRnVuYwBydW50aW1lLmRlZmVyQ2FsbFNhdmUAcnVudGltZS5nb3BhbmljAHJ1bnRpbWUu +Z2V0YXJncABydW50aW1lLmdvcmVjb3ZlcgBydW50aW1lLnRocm93AHJ1bnRpbWUudGhyb3cuZnVu +YzEAcnVudGltZS5yZWNvdmVyeQBydW50aW1lLmZhdGFsdGhyb3cAcnVudGltZS5mYXRhbHRocm93 +LmZ1bmMxAHJ1bnRpbWUuZmF0YWxwYW5pYwBydW50aW1lLmNyYXNoAHJ1bnRpbWUuZmF0YWxwYW5p +Yy5mdW5jMQBydW50aW1lLnN0YXJ0cGFuaWNfbQBydW50aW1lLmRvcGFuaWNfbQBydW50aW1lLnNp +Z25hbWUAcnVudGltZS5nb3RyYWNlYmFjawBydW50aW1lLnNob3VsZFB1c2hTaWdwYW5pYwBydW50 +aW1lLmlzQWJvcnRQQwBydW50aW1lLnN1c3BlbmRHAHJ1bnRpbWUucHJlZW1wdE0AcnVudGltZS5k +dW1wZ3N0YXR1cwBydW50aW1lLnJlc3VtZUcAcnVudGltZS5hc3luY1ByZWVtcHQyAHJ1bnRpbWUu +aW5pdC42AHJ1bnRpbWUuaXNBc3luY1NhZmVQb2ludABydW50aW1lLmNhblByZWVtcHRNAHJ1bnRp +bWUucmVjb3JkRm9yUGFuaWMAcnVudGltZS5wcmludGxvY2sAcnVudGltZS5nd3JpdGUAcnVudGlt +ZS53cml0ZUVycgBydW50aW1lLnByaW50c3AAcnVudGltZS5wcmludG5sAHJ1bnRpbWUucHJpbnRi +b29sAHJ1bnRpbWUucHJpbnRmbG9hdABydW50aW1lLnByaW50Y29tcGxleABydW50aW1lLnByaW50 +dWludABydW50aW1lLnByaW50aW50AHJ1bnRpbWUucHJpbnRoZXgAcnVudGltZS5wcmludHBvaW50 +ZXIAcnVudGltZS5wcmludHVpbnRwdHIAcnVudGltZS5wcmludHN0cmluZwBydW50aW1lLmJ5dGVz +AHJ1bnRpbWUucHJpbnRzbGljZQBydW50aW1lLmhleGR1bXBXb3JkcwBydW50aW1lLm1haW4AcnVu +dGltZS5sb2NrT1NUaHJlYWQAcnVudGltZS5kb2xvY2tPU1RocmVhZABydW50aW1lLm1haW4uZnVu +YzIAcnVudGltZS5pbml0LjcAcnVudGltZS5mb3JjZWdjaGVscGVyAHJ1bnRpbWUuZ29wYXJrAHJ1 +bnRpbWUuZ29yZWFkeQBydW50aW1lLmdvcmVhZHkuZnVuYzEAcnVudGltZS5hY3F1aXJlU3Vkb2cA +cnVudGltZS5yZWxlYXNlU3Vkb2cAcnVudGltZS5iYWRtY2FsbABydW50aW1lLmJhZG1jYWxsMgBy +dW50aW1lLmJhZG1vcmVzdGFja2cwAHJ1bnRpbWUuYmFkbW9yZXN0YWNrZ3NpZ25hbABydW50aW1l +LmJhZGN0eHQAcnVudGltZS5hbGxnYWRkAHJ1bnRpbWUuZm9yRWFjaEcAcnVudGltZS5mb3JFYWNo +R1JhY2UAcnVudGltZS5hdG9taWNBbGxHAHJ1bnRpbWUuYXRvbWljQWxsR0luZGV4AHJ1bnRpbWUu +Y3B1aW5pdABydW50aW1lLnNjaGVkaW5pdABydW50aW1lLm1vZHVsZWRhdGF2ZXJpZnkAcnVudGlt +ZS5mYXN0cmFuZGluaXQAcnVudGltZS5zaWdzYXZlAHJ1bnRpbWUuZ29lbnZzAHJ1bnRpbWUuY2hl +Y2ttY291bnQAcnVudGltZS5tUmVzZXJ2ZUlEAHJ1bnRpbWUubWNvbW1vbmluaXQAcnVudGltZS5p +bnQ2NEhhc2gAcnVudGltZS5yZWFkeQBydW50aW1lLmZyZWV6ZXRoZXdvcmxkAHJ1bnRpbWUuY2Fz +ZnJvbV9Hc2NhbnN0YXR1cwBydW50aW1lLmNhc3RvZ3NjYW5zdGF0dXMAcnVudGltZS5jYXNnc3Rh +dHVzAHJ1bnRpbWUuY2FzZ3N0YXR1cy5mdW5jMQBydW50aW1lLmNhc0dUb1ByZWVtcHRTY2FuAHJ1 +bnRpbWUuY2FzR0Zyb21QcmVlbXB0ZWQAcnVudGltZS5zdG9wVGhlV29ybGRXaXRoU2VtYQBydW50 +aW1lLnN0YXJ0VGhlV29ybGRXaXRoU2VtYQBydW50aW1lLm5ldHBvbGxpbml0ZWQAcnVudGltZS4o +KnB1aW50cHRyKS5zZXQAcnVudGltZS50cmFjZUdDU1RXRG9uZQBydW50aW1lLm1zdGFydDAAcnVu +dGltZS5tc3RhcnQxAHJ1bnRpbWUubXN0YXJ0bTAAcnVudGltZS5tUGFyawBydW50aW1lLm1leGl0 +AHJ1bnRpbWUudW5taW5pdABydW50aW1lLmZvckVhY2hQAHJ1bnRpbWUucnVuU2FmZVBvaW50Rm4A +cnVudGltZS5hbGxvY20AcnVudGltZS5hbGxvY20uZnVuYzEAcnVudGltZS5uZWVkbQBydW50aW1l +LnVubG9ja2V4dHJhAHJ1bnRpbWUubmV3ZXh0cmFtAHJ1bnRpbWUub25lTmV3RXh0cmFNAHJ1bnRp +bWUuZHJvcG0AcnVudGltZS5tc2lncmVzdG9yZQBydW50aW1lLmxvY2tleHRyYQBydW50aW1lLnVz +bGVlcF9ub19nAHJ1bnRpbWUub3N5aWVsZF9ub19nAHJ1bnRpbWUubmV3bQBydW50aW1lLm5ld20x +AHJ1bnRpbWUuc3RhcnRUZW1wbGF0ZVRocmVhZABydW50aW1lLm1Eb0ZpeHVwAHJ1bnRpbWUubURv +Rml4dXBBbmRPU1lpZWxkAHJ1bnRpbWUudGVtcGxhdGVUaHJlYWQAcnVudGltZS5tdWludHB0ci5w +dHIAcnVudGltZS5zdG9wbQBydW50aW1lLm1zcGlubmluZwBydW50aW1lLnN0YXJ0bQBydW50aW1l +Lm1nZXQAcnVudGltZS5ydW5xZW1wdHkAcnVudGltZS5oYW5kb2ZmcABydW50aW1lLm5vYmFycmll +cldha2VUaW1lAHJ1bnRpbWUud2FrZXAAcnVudGltZS5zdG9wbG9ja2VkbQBydW50aW1lLnN0YXJ0 +bG9ja2VkbQBydW50aW1lLmdjc3RvcG0AcnVudGltZS5leGVjdXRlAHJ1bnRpbWUuc2V0VGhyZWFk +Q1BVUHJvZmlsZXIAcnVudGltZS5maW5kcnVubmFibGUAcnVudGltZS5wb2xsV29yawBydW50aW1l +LnN0ZWFsV29yawBydW50aW1lLigqcmFuZG9tT3JkZXIpLnN0YXJ0AHJ1bnRpbWUuKCpyYW5kb21F +bnVtKS5uZXh0AHJ1bnRpbWUuKCpyYW5kb21FbnVtKS5kb25lAHJ1bnRpbWUuKCpyYW5kb21FbnVt +KS5wb3NpdGlvbgBydW50aW1lLnBNYXNrLnJlYWQAcnVudGltZS5jaGVja1J1bnFzTm9QAHJ1bnRp +bWUuY2hlY2tUaW1lcnNOb1AAcnVudGltZS5jaGVja0lkbGVHQ05vUABydW50aW1lLndha2VOZXRQ +b2xsZXIAcnVudGltZS5yZXNldHNwaW5uaW5nAHJ1bnRpbWUuaW5qZWN0Z2xpc3QAcnVudGltZS5p +bmplY3RnbGlzdC5mdW5jMQBydW50aW1lLnNjaGVkdWxlAHJ1bnRpbWUuc2NoZWRFbmFibGVkAHJ1 +bnRpbWUuY2hlY2tUaW1lcnMAcnVudGltZS5wYXJrdW5sb2NrX2MAcnVudGltZS5wYXJrX20AcnVu +dGltZS5kcm9wZwBydW50aW1lLnNldE1Ob1dCAHJ1bnRpbWUuc2V0R05vV0IAcnVudGltZS5nb3Nj +aGVkSW1wbABydW50aW1lLmdsb2JydW5xcHV0AHJ1bnRpbWUuZ29zY2hlZF9tAHJ1bnRpbWUuZ29z +Y2hlZGd1YXJkZWRfbQBydW50aW1lLnRyYWNlR29TY2hlZABydW50aW1lLmdvcHJlZW1wdF9tAHJ1 +bnRpbWUudHJhY2VHb1ByZWVtcHQAcnVudGltZS5wcmVlbXB0UGFyawBydW50aW1lLmdveWllbGRf +bQBydW50aW1lLmdvZXhpdDEAcnVudGltZS50cmFjZUdvRW5kAHJ1bnRpbWUuZ29leGl0MABydW50 +aW1lLnNhdmUAcnVudGltZS5yZWVudGVyc3lzY2FsbABydW50aW1lLnJlZW50ZXJzeXNjYWxsLmZ1 +bmMxAHJ1bnRpbWUuZW50ZXJzeXNjYWxsX3N5c21vbgBydW50aW1lLmVudGVyc3lzY2FsbF9nY3dh +aXQAcnVudGltZS5lbnRlcnN5c2NhbGxibG9jawBydW50aW1lLmVudGVyc3lzY2FsbGJsb2NrLmZ1 +bmMyAHJ1bnRpbWUuZW50ZXJzeXNjYWxsYmxvY2suZnVuYzEAcnVudGltZS5lbnRlcnN5c2NhbGxi +bG9ja19oYW5kb2ZmAHJ1bnRpbWUudHJhY2VHb1N5c0NhbGwAcnVudGltZS5leGl0c3lzY2FsbGZh +c3QAcnVudGltZS5leGl0c3lzY2FsbGZhc3QuZnVuYzEAcnVudGltZS5leGl0c3lzY2FsbGZhc3Rf +cmVhY3F1aXJlZABydW50aW1lLmV4aXRzeXNjYWxsZmFzdF9yZWFjcXVpcmVkLmZ1bmMxAHJ1bnRp +bWUuZXhpdHN5c2NhbGxmYXN0X3BpZGxlAHJ1bnRpbWUuZXhpdHN5c2NhbGwwAHJ1bnRpbWUubWFs +ZwBydW50aW1lLnJvdW5kMgBydW50aW1lLm1hbGcuZnVuYzEAcnVudGltZS5uZXdwcm9jAHJ1bnRp +bWUubmV3cHJvYy5mdW5jMQBydW50aW1lLm5ld3Byb2MxAHJ1bnRpbWUuZ29zdGFydGNhbGxmbgBy +dW50aW1lLmdvc3RhcnRjYWxsAHJ1bnRpbWUuc2F2ZUFuY2VzdG9ycwBydW50aW1lLmdmcHV0AHJ1 +bnRpbWUuKCpnUXVldWUpLnB1c2gAcnVudGltZS5nZmdldABydW50aW1lLmdmZ2V0LmZ1bmMxAHJ1 +bnRpbWUuZ2ZwdXJnZQBydW50aW1lLnVubG9ja09TVGhyZWFkAHJ1bnRpbWUuZG91bmxvY2tPU1Ro +cmVhZABydW50aW1lLmJhZHVubG9ja29zdGhyZWFkAHJ1bnRpbWUuX1N5c3RlbQBydW50aW1lLl9F +eHRlcm5hbENvZGUAcnVudGltZS5fTG9zdEV4dGVybmFsQ29kZQBydW50aW1lLl9HQwBydW50aW1l +Ll9Mb3N0U0lHUFJPRkR1cmluZ0F0b21pYzY0AHJ1bnRpbWUuX1ZEU08AcnVudGltZS5zaWdwcm9m +AHJ1bnRpbWUuc2lncHJvZk5vbkdvAHJ1bnRpbWUuc2lncHJvZk5vbkdvUEMAcnVudGltZS4oKnAp +LmluaXQAcnVudGltZS5wTWFzay5zZXQAcnVudGltZS5wTWFzay5jbGVhcgBydW50aW1lLigqcCku +ZGVzdHJveQBydW50aW1lLmdsb2JydW5xcHV0aGVhZABydW50aW1lLigqcCkuZGVzdHJveS5mdW5j +MQBydW50aW1lLnByb2NyZXNpemUAcnVudGltZS4oKnJhbmRvbU9yZGVyKS5yZXNldABydW50aW1l +LnRyYWNlR29tYXhwcm9jcwBydW50aW1lLmdjZABydW50aW1lLmFjcXVpcmVwAHJ1bnRpbWUud2ly +ZXAAcnVudGltZS5yZWxlYXNlcABydW50aW1lLmluY2lkbGVsb2NrZWQAcnVudGltZS5jaGVja2Rl +YWQAcnVudGltZS5jaGVja2RlYWQuZnVuYzEAcnVudGltZS5zeXNtb24AcnVudGltZS5yZXRha2UA +cnVudGltZS5wcmVlbXB0YWxsAHJ1bnRpbWUucHJlZW1wdG9uZQBydW50aW1lLnNjaGVkdHJhY2UA +cnVudGltZS5zY2hlZEVuYWJsZVVzZXIAcnVudGltZS5tcHV0AHJ1bnRpbWUuZ2xvYnJ1bnFnZXQA +cnVudGltZS51cGRhdGVUaW1lclBNYXNrAHJ1bnRpbWUucGlkbGVwdXQAcnVudGltZS5waWRsZWdl +dABydW50aW1lLnJ1bnFwdXQAcnVudGltZS4oKmd1aW50cHRyKS5jYXMAcnVudGltZS5ydW5xcHV0 +c2xvdwBydW50aW1lLnJ1bnFwdXRiYXRjaABydW50aW1lLnJ1bnFnZXQAcnVudGltZS5ydW5xZHJh +aW4AcnVudGltZS5ydW5xZ3JhYgBydW50aW1lLnJ1bnFzdGVhbABydW50aW1lLmRvSW5pdABydW50 +aW1lLigqcHJvZkJ1ZikuY2FuV3JpdGVSZWNvcmQAcnVudGltZS4oKnByb2ZBdG9taWMpLmxvYWQA +cnVudGltZS5wcm9mSW5kZXgudGFnQ291bnQAcnVudGltZS5jb3VudFN1YgBydW50aW1lLigqcHJv +ZkJ1ZikuY2FuV3JpdGVUd29SZWNvcmRzAHJ1bnRpbWUuKCpwcm9mQnVmKS53cml0ZQBydW50aW1l +LigqcHJvZkJ1ZikuaGFzT3ZlcmZsb3cAcnVudGltZS4oKnByb2ZCdWYpLnRha2VPdmVyZmxvdwBy +dW50aW1lLigqcHJvZkJ1ZikuaW5jcmVtZW50T3ZlcmZsb3cAcnVudGltZS5wcm9mSW5kZXguYWRk +Q291bnRzQW5kQ2xlYXJGbGFncwBydW50aW1lLigqcHJvZkF0b21pYykuY2FzAHJ1bnRpbWUuKCpw +cm9mQnVmKS53YWtldXBFeHRyYQBydW50aW1lLmFyZ3MAcnVudGltZS5nb2FyZ3MAcnVudGltZS5n +b3N0cmluZ25vY29weQBydW50aW1lLmdvZW52c191bml4AHJ1bnRpbWUudGVzdEF0b21pYzY0AHJ1 +bnRpbWUuY2hlY2sAcnVudGltZS50aW1lZGl2AHJ1bnRpbWUucGFyc2VkZWJ1Z3ZhcnMAcnVudGlt +ZS5leHRlbmRSYW5kb20AcnVudGltZS53YWl0UmVhc29uLlN0cmluZwBydW50aW1lLigqcndtdXRl +eCkucmxvY2sAcnVudGltZS4oKnJ3bXV0ZXgpLnJsb2NrLmZ1bmMxAHJ1bnRpbWUuKCpyd211dGV4 +KS5ydW5sb2NrAHJ1bnRpbWUucmVhZHlXaXRoVGltZQBydW50aW1lLnNlbWFjcXVpcmUxAHJ1bnRp +bWUuY2Fuc2VtYWNxdWlyZQBydW50aW1lLnNlbXJvb3QAcnVudGltZS5zZW1yZWxlYXNlMQBydW50 +aW1lLmdveWllbGQAcnVudGltZS4oKnNlbWFSb290KS5xdWV1ZQBydW50aW1lLigqc2VtYVJvb3Qp +LmRlcXVldWUAcnVudGltZS4oKnNlbWFSb290KS5yb3RhdGVMZWZ0AHJ1bnRpbWUuKCpzZW1hUm9v +dCkucm90YXRlUmlnaHQAcnVudGltZS5kdW1wcmVncwBydW50aW1lLigqc2lnY3R4dCkucmF4AHJ1 +bnRpbWUuKCpzaWdjdHh0KS5yZWdzAHJ1bnRpbWUuKCpzaWdjdHh0KS5yYngAcnVudGltZS4oKnNp +Z2N0eHQpLnJjeABydW50aW1lLigqc2lnY3R4dCkucmR4AHJ1bnRpbWUuKCpzaWdjdHh0KS5yZGkA +cnVudGltZS4oKnNpZ2N0eHQpLnJzaQBydW50aW1lLigqc2lnY3R4dCkucmJwAHJ1bnRpbWUuKCpz +aWdjdHh0KS5yc3AAcnVudGltZS4oKnNpZ2N0eHQpLnI4AHJ1bnRpbWUuKCpzaWdjdHh0KS5yOQBy +dW50aW1lLigqc2lnY3R4dCkucjEwAHJ1bnRpbWUuKCpzaWdjdHh0KS5yMTEAcnVudGltZS4oKnNp +Z2N0eHQpLnIxMgBydW50aW1lLigqc2lnY3R4dCkucjEzAHJ1bnRpbWUuKCpzaWdjdHh0KS5yMTQA +cnVudGltZS4oKnNpZ2N0eHQpLnIxNQBydW50aW1lLigqc2lnY3R4dCkucmlwAHJ1bnRpbWUuKCpz +aWdjdHh0KS5yZmxhZ3MAcnVudGltZS4oKnNpZ2N0eHQpLmNzAHJ1bnRpbWUuKCpzaWdjdHh0KS5m +cwBydW50aW1lLigqc2lnY3R4dCkuZ3MAcnVudGltZS4oKnNpZ2N0eHQpLnByZXBhcmVQYW5pYwBy +dW50aW1lLigqc2lnY3R4dCkucHVzaENhbGwAcnVudGltZS4oKnNpZ2N0eHQpLnNldF9yc3AAcnVu +dGltZS4oKnNpZ2N0eHQpLnNldF9yaXAAcnVudGltZS5pbml0c2lnAHJ1bnRpbWUuZ2V0c2lnAHJ1 +bnRpbWUuc2lnSW5zdGFsbEdvSGFuZGxlcgBydW50aW1lLnNpZ0luaXRJZ25vcmVkAHJ1bnRpbWUu +ZG9TaWdQcmVlbXB0AHJ1bnRpbWUud2FudEFzeW5jUHJlZW1wdABydW50aW1lLigqc2lnY3R4dCku +c2lncGMAcnVudGltZS4oKnNpZ2N0eHQpLnNpZ3NwAHJ1bnRpbWUuc2lnRmV0Y2hHAHJ1bnRpbWUu +c2lndHJhbXBnbwBydW50aW1lLnJlc3RvcmVHc2lnbmFsU3RhY2sAcnVudGltZS5hZGp1c3RTaWdu +YWxTdGFjawBydW50aW1lLnNldEdzaWduYWxTdGFjawBydW50aW1lLnNldFNpZ25hbHN0YWNrU1AA +cnVudGltZS5zaWdoYW5kbGVyAHJ1bnRpbWUuKCpzaWdjdHh0KS5zaWdjb2RlAHJ1bnRpbWUuKCpz +aWdjdHh0KS5mYXVsdABydW50aW1lLigqc2lnY3R4dCkuc2lnYWRkcgBvcy9zaWduYWwuc2lnbmFs +X2lnbm9yZWQAcnVudGltZS5zaWdwYW5pYwBydW50aW1lLmNhbnBhbmljAHJ1bnRpbWUucGFuaWNt +ZW1BZGRyAHJ1bnRpbWUucGFuaWNtZW0AcnVudGltZS5wYW5pY292ZXJmbG93AHJ1bnRpbWUucGFu +aWNmbG9hdABydW50aW1lLmRpZUZyb21TaWduYWwAcnVudGltZS5yYWlzZWJhZHNpZ25hbABydW50 +aW1lLm5vU2lnbmFsU3RhY2sAcnVudGltZS5zaWdOb3RPblN0YWNrAHJ1bnRpbWUuc2lnbmFsRHVy +aW5nRm9yawBydW50aW1lLmJhZHNpZ25hbABydW50aW1lLnNpZ2Z3ZGdvAHJ1bnRpbWUuc2lnYmxv +Y2sAcnVudGltZS51bmJsb2Nrc2lnAHJ1bnRpbWUuc2lnYWRkc2V0AHJ1bnRpbWUubWluaXRTaWdu +YWxzAHJ1bnRpbWUubWluaXRTaWduYWxTdGFjawBydW50aW1lLm1pbml0U2lnbmFsTWFzawBydW50 +aW1lLmJsb2NrYWJsZVNpZwBydW50aW1lLnVubWluaXRTaWduYWxzAHJ1bnRpbWUuc2lnbmFsc3Rh +Y2sAcnVudGltZS5zaWdzZW5kAHJ1bnRpbWUubWFrZXNsaWNlY29weQBydW50aW1lLnBhbmljbWFr +ZXNsaWNlbGVuAHJ1bnRpbWUubWFrZXNsaWNlAHJ1bnRpbWUucGFuaWNtYWtlc2xpY2VjYXAAcnVu +dGltZS5ncm93c2xpY2UAcnVudGltZS5pc1Bvd2VyT2ZUd28AcnVudGltZS5zdGFja2luaXQAcnVu +dGltZS4oKm1TcGFuTGlzdCkuaW5pdABydW50aW1lLnN0YWNrcG9vbGFsbG9jAHJ1bnRpbWUuZ2Ns +aW5rcHRyLnB0cgBydW50aW1lLnN0YWNrcG9vbGZyZWUAcnVudGltZS5zdGFja2NhY2hlcmVmaWxs +AHJ1bnRpbWUuc3RhY2tjYWNoZXJlbGVhc2UAcnVudGltZS5zdGFja2NhY2hlX2NsZWFyAHJ1bnRp +bWUuc3RhY2thbGxvYwBydW50aW1lLnN0YWNrbG9nMgBydW50aW1lLnN0YWNrZnJlZQBydW50aW1l +LmFkanVzdHBvaW50ZXJzAHJ1bnRpbWUuYWRqdXN0ZnJhbWUAcnVudGltZS5hZGp1c3Rwb2ludGVy +AHJ1bnRpbWUuYWRqdXN0ZGVmZXJzAHJ1bnRpbWUuc3luY2FkanVzdHN1ZG9ncwBydW50aW1lLmFk +anVzdHN1ZG9ncwBydW50aW1lLmNvcHlzdGFjawBydW50aW1lLmZpbmRzZ2hpAHJ1bnRpbWUuYWRq +dXN0Y3R4dABydW50aW1lLmFkanVzdHBhbmljcwBydW50aW1lLm5ld3N0YWNrAHJ1bnRpbWUuc2hy +aW5rc3RhY2sAcnVudGltZS5mcmVlU3RhY2tTcGFucwBydW50aW1lLmdldFN0YWNrTWFwAHJ1bnRp +bWUuc3RhY2ttYXBkYXRhAHJ1bnRpbWUuaW5pdC45AHJ1bnRpbWUuY29uY2F0c3RyaW5ncwBydW50 +aW1lLnN0cmluZ0RhdGFPblN0YWNrAHJ1bnRpbWUuY29uY2F0c3RyaW5nMgBydW50aW1lLmNvbmNh +dHN0cmluZzQAcnVudGltZS5zbGljZWJ5dGV0b3N0cmluZwBydW50aW1lLnJhd3N0cmluZ3RtcABy +dW50aW1lLnJhd3N0cmluZwBydW50aW1lLmF0b2kAcnVudGltZS5maW5kbnVsbABydW50aW1lLmJh +ZHN5c3RlbXN0YWNrAHJ1bnRpbWUubW9kdWxlc2luaXQAcnVudGltZS5tb2R1bGVkYXRhdmVyaWZ5 +MQBydW50aW1lLmZpbmRmdW5jAHJ1bnRpbWUuZmluZG1vZHVsZWRhdGFwAHJ1bnRpbWUucGN2YWx1 +ZQBydW50aW1lLnBjdmFsdWVDYWNoZUtleQBydW50aW1lLmZ1bmNuYW1lAHJ1bnRpbWUuY2Z1bmNu +YW1lAHJ1bnRpbWUuZnVuY3BrZ3BhdGgAcnVudGltZS5mdW5jbmFtZUZyb21OYW1lb2ZmAHJ1bnRp +bWUuY2Z1bmNuYW1lRnJvbU5hbWVvZmYAcnVudGltZS5mdW5jZmlsZQBydW50aW1lLmZ1bmNsaW5l +MQBydW50aW1lLmZ1bmNsaW5lAHJ1bnRpbWUuZnVuY3NwZGVsdGEAcnVudGltZS5mdW5jTWF4U1BE +ZWx0YQBydW50aW1lLnBjZGF0YXZhbHVlAHJ1bnRpbWUucGNkYXRhc3RhcnQAcnVudGltZS5wY2Rh +dGF2YWx1ZTIAcnVudGltZS5zdGVwAHJ1bnRpbWUucmVhZHZhcmludABydW50aW1lLmRvYWRkdGlt +ZXIAcnVudGltZS5kZWx0aW1lcgBydW50aW1lLmRvZGVsdGltZXIAcnVudGltZS51cGRhdGVUaW1l +cjBXaGVuAHJ1bnRpbWUuZG9kZWx0aW1lcjAAcnVudGltZS5tb2R0aW1lcgBydW50aW1lLnVwZGF0 +ZVRpbWVyTW9kaWZpZWRFYXJsaWVzdABydW50aW1lLm1vdmVUaW1lcnMAcnVudGltZS5hZGp1c3R0 +aW1lcnMAcnVudGltZS5hZGRBZGp1c3RlZFRpbWVycwBydW50aW1lLnJ1bnRpbWVyAHJ1bnRpbWUu +cnVuT25lVGltZXIAcnVudGltZS5jbGVhckRlbGV0ZWRUaW1lcnMAcnVudGltZS50aW1lU2xlZXBV +bnRpbABydW50aW1lLnNpZnR1cFRpbWVyAHJ1bnRpbWUuc2lmdGRvd25UaW1lcgBydW50aW1lLmJh +ZFRpbWVyAHJ1bnRpbWUudHJhY2VSZWFkZXIAcnVudGltZS50cmFjZVByb2NGcmVlAHJ1bnRpbWUu +dHJhY2VGdWxsUXVldWUAcnVudGltZS50cmFjZUJ1ZlB0ci5wdHIAcnVudGltZS50cmFjZUV2ZW50 +AHJ1bnRpbWUudHJhY2VFdmVudExvY2tlZABydW50aW1lLigqdHJhY2VCdWZQdHIpLnNldABydW50 +aW1lLigqdHJhY2VCdWYpLmJ5dGUAcnVudGltZS4oKnRyYWNlQnVmKS52YXJpbnQAcnVudGltZS50 +cmFjZVN0YWNrSUQAcnVudGltZS50cmFjZUFjcXVpcmVCdWZmZXIAcnVudGltZS50cmFjZVJlbGVh +c2VCdWZmZXIAcnVudGltZS50cmFjZUZsdXNoAHJ1bnRpbWUuKCp0cmFjZVN0YWNrVGFibGUpLnB1 +dABydW50aW1lLigqdHJhY2VTdGFjaykuc3RhY2sAcnVudGltZS4oKnRyYWNlU3RhY2tUYWJsZSku +ZmluZABydW50aW1lLigqdHJhY2VTdGFja1RhYmxlKS5uZXdTdGFjawBydW50aW1lLigqdHJhY2VB +bGxvYykuYWxsb2MAcnVudGltZS4oKnRyYWNlQWxsb2NCbG9ja1B0cikuc2V0AHJ1bnRpbWUudHJh +Y2VQcm9jU3RhcnQAcnVudGltZS50cmFjZVByb2NTdG9wAHJ1bnRpbWUudHJhY2VHQ1N3ZWVwU3Rh +cnQAcnVudGltZS50cmFjZUdDU3dlZXBTcGFuAHJ1bnRpbWUudHJhY2VHQ1N3ZWVwRG9uZQBydW50 +aW1lLnRyYWNlR29DcmVhdGUAcnVudGltZS50cmFjZUdvU3RhcnQAcnVudGltZS50cmFjZUdvUGFy +awBydW50aW1lLnRyYWNlR29VbnBhcmsAcnVudGltZS50cmFjZUdvU3lzRXhpdABydW50aW1lLnRy +YWNlR29TeXNCbG9jawBydW50aW1lLnRyYWNlSGVhcEdvYWwAcnVudGltZS50cmFjZWJhY2tkZWZl +cnMAcnVudGltZS5nZXRBcmdJbmZvRmFzdABydW50aW1lLmdlbnRyYWNlYmFjawBydW50aW1lLmVs +aWRlV3JhcHBlckNhbGxpbmcAcnVudGltZS5wcmludEFyZ3MAcnVudGltZS5wcmludEFyZ3MuZnVu +YzIAcnVudGltZS5wcmludEFyZ3MuZnVuYzEAcnVudGltZS5nZXRBcmdJbmZvAHJ1bnRpbWUudHJh +Y2ViYWNrQ2dvQ29udGV4dABydW50aW1lLnByaW50Y3JlYXRlZGJ5AHJ1bnRpbWUucHJpbnRjcmVh +dGVkYnkxAHJ1bnRpbWUudHJhY2ViYWNrdHJhcABydW50aW1lLnRyYWNlYmFjazEAcnVudGltZS5w +cmludEFuY2VzdG9yVHJhY2ViYWNrAHJ1bnRpbWUucHJpbnRBbmNlc3RvclRyYWNlYmFja0Z1bmNJ +bmZvAHJ1bnRpbWUuY2FsbGVycwBydW50aW1lLmNhbGxlcnMuZnVuYzEAcnVudGltZS5zaG93ZnJh +bWUAcnVudGltZS5zaG93ZnVuY2luZm8AcnVudGltZS5pc0V4cG9ydGVkUnVudGltZQBydW50aW1l +Lmdvcm91dGluZWhlYWRlcgBydW50aW1lLnRyYWNlYmFja290aGVycwBydW50aW1lLnRyYWNlYmFj +a290aGVycy5mdW5jMQBydW50aW1lLnRyYWNlYmFja0hleGR1bXAAcnVudGltZS50cmFjZWJhY2tI +ZXhkdW1wLmZ1bmMxAHJ1bnRpbWUuaXNTeXN0ZW1Hb3JvdXRpbmUAcnVudGltZS5wcmludENnb1Ry +YWNlYmFjawBydW50aW1lLnByaW50T25lQ2dvVHJhY2ViYWNrAHJ1bnRpbWUuY2FsbENnb1N5bWJv +bGl6ZXIAcnVudGltZS5jZ29Db250ZXh0UENzAHJ1bnRpbWUuKCpfdHlwZSkuc3RyaW5nAHJ1bnRp +bWUuKCpfdHlwZSkudW5jb21tb24AcnVudGltZS4oKl90eXBlKS5wa2dwYXRoAHJ1bnRpbWUucmVz +b2x2ZU5hbWVPZmYAcnVudGltZS5yZWZsZWN0T2Zmc0xvY2sAcnVudGltZS5yZWZsZWN0T2Zmc1Vu +bG9jawBydW50aW1lLnJlc29sdmVUeXBlT2ZmAHJ1bnRpbWUuKCpfdHlwZSkudGV4dE9mZgBydW50 +aW1lLm5hbWUubmFtZQBydW50aW1lLm5hbWUucmVhZHZhcmludABydW50aW1lLm5hbWUuZGF0YQBy +dW50aW1lLm5hbWUudGFnAHJ1bnRpbWUubmFtZS5wa2dQYXRoAHJ1bnRpbWUudHlwZWxpbmtzaW5p +dABydW50aW1lLnR5cGVzRXF1YWwAcnVudGltZS4oKmZ1bmN0eXBlKS5pbgBydW50aW1lLigqZnVu +Y3R5cGUpLm91dABydW50aW1lLnZkc29Jbml0RnJvbVN5c2luZm9FaGRyAHJ1bnRpbWUudmRzb0Zp +bmRWZXJzaW9uAHJ1bnRpbWUudmRzb1BhcnNlU3ltYm9scwBydW50aW1lLnZkc29QYXJzZVN5bWJv +bHMuZnVuYzEAcnVudGltZS5fRUxGX1NUX1RZUEUAcnVudGltZS5fRUxGX1NUX0JJTkQAcnVudGlt +ZS52ZHNvYXV4dgBydW50aW1lLmluVkRTT1BhZ2UAcnVudGltZS5kZWJ1Z0NhbGxXcmFwLmZ1bmMy +AHJ1bnRpbWUuZGVidWdDYWxsV3JhcDEuZnVuYzEAcnVudGltZS5nY1N0YXJ0LmZ1bmMxAHJ1bnRp +bWUuZ2NNYXJrRG9uZS5mdW5jMS4xAHJ1bnRpbWUuZ2NNYXJrRG9uZS5mdW5jMQBydW50aW1lLmdj +TWFya0RvbmUuZnVuYzMAcnVudGltZS5nY01hcmtUZXJtaW5hdGlvbi5mdW5jMgBydW50aW1lLmdj +TWFya1Rlcm1pbmF0aW9uLmZ1bmMzAHJ1bnRpbWUuZ2NNYXJrVGVybWluYXRpb24uZnVuYzQuMQBy +dW50aW1lLmdjTWFya1Rlcm1pbmF0aW9uLmZ1bmM0AHJ1bnRpbWUuZ2NCZ01hcmtXb3JrZXIuZnVu +YzEAcnVudGltZS5nY1Jlc2V0TWFya1N0YXRlLmZ1bmMxAHJ1bnRpbWUuYmdzY2F2ZW5nZS5mdW5j +MQBydW50aW1lLnN3ZWVwb25lLmZ1bmMxAHJ1bnRpbWUuKCpwYWdlQWxsb2MpLnN5c0dyb3cuZnVu +YzEAcnVudGltZS5ibG9ja0FsaWduU3VtbWFyeVJhbmdlAHJ1bnRpbWUud2JCdWZGbHVzaC5mdW5j +MQBydW50aW1lLnN5c1NpZ2FjdGlvbi5mdW5jMQBydW50aW1lLnByZXByaW50cGFuaWNzLmZ1bmMx +AHJ1bnRpbWUuZmF0YWxwYW5pYy5mdW5jMgBydW50aW1lLm1haW4uZnVuYzEAcnVudGltZS5zY2hl +ZHRyYWNlLmZ1bmMxAHJ1bnRpbWUuaW5pdABydW50aW1lLmRlZmF1bHRNZW1Qcm9maWxlUmF0ZQBz +eW5jLmV2ZW50AHJ1bnRpbWUuZW50ZXJzeXNjYWxsAHJ1bnRpbWUuZXhpdHN5c2NhbGwAcnVudGlt +ZS9kZWJ1Zy5TZXRUcmFjZWJhY2sAcnVudGltZS5tb3Jlc3RhY2tjAHJ1bnRpbWUuZ29zdHJpbmcA +Z29nbwBnb3NhdmVfc3lzdGVtc3RhY2tfc3dpdGNoAHNldGdfZ2NjAGFlc2hhc2hib2R5AGRlYnVn +Q2FsbDMyAGRlYnVnQ2FsbDY0AGRlYnVnQ2FsbDEyOABkZWJ1Z0NhbGwyNTYAZGVidWdDYWxsNTEy +AGRlYnVnQ2FsbDEwMjQAZGVidWdDYWxsMjA0OABkZWJ1Z0NhbGw0MDk2AGRlYnVnQ2FsbDgxOTIA +ZGVidWdDYWxsMTYzODQAZGVidWdDYWxsMzI3NjgAZGVidWdDYWxsNjU1MzYAX3J0MF9hbWQ2NABy +dW50aW1lLnJ0MF9nbwBydW50aW1lLmFzbWluaXQAcnVudGltZS5tc3RhcnQAcnVudGltZS5nb2dv +AHJ1bnRpbWUubWNhbGwAcnVudGltZS5zeXN0ZW1zdGFja19zd2l0Y2gAcnVudGltZS5zeXN0ZW1z +dGFjawBydW50aW1lLm1vcmVzdGFjawBydW50aW1lLm1vcmVzdGFja19ub2N0eHQAcnVudGltZS5w +cm9jeWllbGQAcnVudGltZS5wdWJsaWNhdGlvbkJhcnJpZXIAcnVudGltZS5qbXBkZWZlcgBydW50 +aW1lLmFzbWNnb2NhbGwAcnVudGltZS5zZXRnAHJ1bnRpbWUuYWJvcnQAcnVudGltZS5zdGFja2No +ZWNrAHJ1bnRpbWUuY3B1dGlja3MAcnVudGltZS5tZW1oYXNoAHJ1bnRpbWUubWVtaGFzaDMyAHJ1 +bnRpbWUubWVtaGFzaDY0AHJ1bnRpbWUuY2hlY2tBU00AcnVudGltZS5nb2V4aXQAcnVudGltZS5z +aWdwYW5pYzAAcnVudGltZS5nY1dyaXRlQmFycmllcgBydW50aW1lLmdjV3JpdGVCYXJyaWVyQ1gA +cnVudGltZS5nY1dyaXRlQmFycmllckRYAHJ1bnRpbWUuZ2NXcml0ZUJhcnJpZXJCWABydW50aW1l +LmdjV3JpdGVCYXJyaWVyU0kAcnVudGltZS5nY1dyaXRlQmFycmllclI4AHJ1bnRpbWUuZ2NXcml0 +ZUJhcnJpZXJSOQBydW50aW1lLmRlYnVnQ2FsbFYyAHJ1bnRpbWUuZGVidWdDYWxsUGFuaWNrZWQA +cnVudGltZS5wYW5pY0luZGV4AHJ1bnRpbWUucGFuaWNJbmRleFUAcnVudGltZS5wYW5pY1NsaWNl +QWxlbgBydW50aW1lLnBhbmljU2xpY2VBbGVuVQBydW50aW1lLnBhbmljU2xpY2VBY2FwAHJ1bnRp +bWUucGFuaWNTbGljZUFjYXBVAHJ1bnRpbWUucGFuaWNTbGljZUIAcnVudGltZS5wYW5pY1NsaWNl +QlUAcnVudGltZS5wYW5pY1NsaWNlM0FsZW4AcnVudGltZS5wYW5pY1NsaWNlM0FsZW5VAHJ1bnRp +bWUuZHVmZnplcm8AcnVudGltZS5kdWZmY29weQBydW50aW1lLm1lbWNsck5vSGVhcFBvaW50ZXJz +AHJ1bnRpbWUubWVtbW92ZQBydW50aW1lLmFzeW5jUHJlZW1wdABfcnQwX2FtZDY0X2xpbnV4AHJ1 +bnRpbWUuZXhpdABydW50aW1lLmV4aXRUaHJlYWQAcnVudGltZS5vcGVuAHJ1bnRpbWUuY2xvc2Vm +ZABydW50aW1lLndyaXRlMQBydW50aW1lLnJlYWQAcnVudGltZS5waXBlAHJ1bnRpbWUucGlwZTIA +cnVudGltZS51c2xlZXAAcnVudGltZS5nZXR0aWQAcnVudGltZS5yYWlzZQBydW50aW1lLnJhaXNl +cHJvYwBydW50aW1lLmdldHBpZABydW50aW1lLnRna2lsbABydW50aW1lLm1pbmNvcmUAcnVudGlt +ZS5uYW5vdGltZTEAcnVudGltZS5ydHNpZ3Byb2NtYXNrAHJ1bnRpbWUucnRfc2lnYWN0aW9uAHJ1 +bnRpbWUuY2FsbENnb1NpZ2FjdGlvbgBydW50aW1lLnNpZ2Z3ZABydW50aW1lLnNpZ3RyYW1wAHJ1 +bnRpbWUuY2dvU2lndHJhbXAAcnVudGltZS5zaWdyZXR1cm4AcnVudGltZS5zeXNNbWFwAHJ1bnRp +bWUuY2FsbENnb01tYXAAcnVudGltZS5zeXNNdW5tYXAAcnVudGltZS5jYWxsQ2dvTXVubWFwAHJ1 +bnRpbWUubWFkdmlzZQBydW50aW1lLmZ1dGV4AHJ1bnRpbWUuY2xvbmUAcnVudGltZS5zaWdhbHRz +dGFjawBydW50aW1lLnNldHRscwBydW50aW1lLm9zeWllbGQAcnVudGltZS5zY2hlZF9nZXRhZmZp +bml0eQBydW50aW1lLmVwb2xsY3JlYXRlAHJ1bnRpbWUuZXBvbGxjcmVhdGUxAHJ1bnRpbWUuZXBv +bGxjdGwAcnVudGltZS5lcG9sbHdhaXQAcnVudGltZS5jbG9zZW9uZXhlYwBydW50aW1lLnNldE5v +bmJsb2NrAHRpbWUubm93AHJ1bnRpbWUuKCppdGFiVGFibGVUeXBlKS5hZGQtZm0AcnVudGltZS5k +ZWJ1Z0NhbGxDaGVjawBydW50aW1lLmRlYnVnQ2FsbFdyYXAAcnVudGltZS53YkJ1ZkZsdXNoAHJ1 +bnRpbWUub3Npbml0AHJ1bnRpbWUuYXN5bmNQcmVlbXB0MgBydW50aW1lLmJhZG1jYWxsAHJ1bnRp +bWUuYmFkbWNhbGwyAHJ1bnRpbWUuYmFkbW9yZXN0YWNrZzAAcnVudGltZS5iYWRtb3Jlc3RhY2tn +c2lnbmFsAHJ1bnRpbWUuc2NoZWRpbml0AHJ1bnRpbWUubXN0YXJ0AHJ1bnRpbWUubXN0YXJ0MABy +dW50aW1lLmdvZXhpdDEAcnVudGltZS5uZXdwcm9jAHJ1bnRpbWUuc2lncHJvZk5vbkdvAHJ1bnRp +bWUuYXJncwBydW50aW1lLmNoZWNrAHJ1bnRpbWUuc2lndHJhbXBnbwBydW50aW1lLm5ld3N0YWNr +AHJ1bnRpbWUubW9yZXN0YWNrYwBydW50aW1lLmJhZHN5c3RlbXN0YWNrAHJ1bnRpbWUuYXNtY2dv +Y2FsbABydW50aW1lLigqZXJyb3JTdHJpbmcpLkVycm9yAHR5cGUuLmVxLnJ1bnRpbWUuaXRhYgB0 +eXBlLi5lcS5ydW50aW1lLm1vZHVsZWhhc2gAdHlwZS4uZXEucnVudGltZS5iaXR2ZWN0b3IAdHlw +ZS4uZXEucnVudGltZS5UeXBlQXNzZXJ0aW9uRXJyb3IAdHlwZS4uZXEucnVudGltZS5fcGFuaWMA +dHlwZS4uZXEucnVudGltZS5fZGVmZXIAdHlwZS4uZXEucnVudGltZS5ib3VuZHNFcnJvcgBydW50 +aW1lLigqYm91bmRzRXJyb3IpLkVycm9yAHR5cGUuLmVxLnJ1bnRpbWUuc3lzbW9udGljawB0eXBl +Li5lcS5ydW50aW1lLnNwZWNpYWwAdHlwZS4uZXEucnVudGltZS5tc3BhbgB0eXBlLi5lcS5ydW50 +aW1lLm1jYWNoZQB0eXBlLi5lcS5zdHJ1Y3QgeyBydW50aW1lLmdMaXN0OyBydW50aW1lLm4gaW50 +MzIgfQB0eXBlLi5lcS5ydW50aW1lLmhjaGFuAHR5cGUuLmVxLnJ1bnRpbWUuc3Vkb2cAdHlwZS4u +ZXEucnVudGltZS5nY1dvcmsAcnVudGltZS4oKmxvY2tSYW5rKS5TdHJpbmcAcnVudGltZS4oKndh +aXRSZWFzb24pLlN0cmluZwB0eXBlLi5lcS5ydW50aW1lLmVycm9yQWRkcmVzc1N0cmluZwBydW50 +aW1lLigqZXJyb3JBZGRyZXNzU3RyaW5nKS5FcnJvcgBydW50aW1lLigqcGxhaW5FcnJvcikuRXJy +b3IAbWFpbi5tYWluAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQwAAAAAAAAD/////FAAAACwA +AABDAAAA////////////////UwAAAEMAAAD//////////////////////////30AAACdAAAAvgAA +AN0AAABDAAAAQwAAAAABAAAjAQAA/////08BAABjAQAAfAEAAP////+hAQAA/AEAAP////////// +//////////9eAgAAeAIAAP//////////ogIAAP//////////TwYAALcCAADMAgAA/////68DAAD/ +////wwMAAN0CAAD/////jwQAAO8CAAAQAwAAIwMAAAwCAAA8AwAAIgIAALUBAADUAwAA9AMAAP// +////////CgQAANgBAAB6AwAALwQAAFcEAABQAwAA/////20EAABlAwAAjAMAAMgEAAAABQAAJgUA +ANsEAABDBAAApAQAAOsBAAByBQAAtQUAAIgFAACeBQAA0QUAAF8FAAD/////4wMAAPcFAAA9BQAA +PQEAAAsGAAAjBgAANgYAAGsGAAB/BgAA////////////////mwYAAP/////DBgAA/////7cEAAA6 +AgAAEAcAAP//////////////////////////mwMAAEoCAAAjBwAApwcAAH8EAAA2BwAATgcAAKwG +AABsBwAA////////////////gAcAAP/////vBAAAFAUAAMcBAADrBgAA////////////////jwEA +AP/////9BgAATwUAAIsCAAD//////////x4EAADiBQAAAAMAAA8BAAD//////////5EHAAD///// +1gYAAP////8AAAAA//////////8UAAAA/////////////////////////////////////1MAAAC5 +BwAAzQcAAOIHAAD5BwAAEQgAACkIAABDCAAAXQgAAEMAAAB4CAAAAAAAAAAAAAAAAAAAAAAAAAAA +AABpbnRlcm5hbC9jcHUvY3B1LmdvAGludGVybmFsL2NwdS9jcHVfeDg2LmdvAGludGVybmFsL2Nw +dS9jcHVfeDg2LnMAPGF1dG9nZW5lcmF0ZWQ+AHJ1bnRpbWUvaW50ZXJuYWwvc3lzL2ludHJpbnNp +Y3NfY29tbW9uLmdvAGludGVybmFsL2J5dGVhbGcvaW5kZXhfYW1kNjQuZ28AaW50ZXJuYWwvYnl0 +ZWFsZy9jb21wYXJlX2FtZDY0LnMAaW50ZXJuYWwvYnl0ZWFsZy9lcXVhbF9hbWQ2NC5zAGludGVy +bmFsL2J5dGVhbGcvaW5kZXhieXRlX2FtZDY0LnMAcnVudGltZS9hbGcuZ28AcnVudGltZS90eXBl +a2luZC5nbwBydW50aW1lL2F0b21pY19wb2ludGVyLmdvAHJ1bnRpbWUvbXdiYnVmLmdvAHJ1bnRp +bWUvY2dvX21tYXAuZ28AcnVudGltZS9jZ29fc2lnYWN0aW9uLmdvAHJ1bnRpbWUvY2dvY2FsbC5n +bwBydW50aW1lL3N5bXRhYi5nbwBydW50aW1lL2Nnb2NoZWNrLmdvAHJ1bnRpbWUvbWFsbG9jLmdv +AHJ1bnRpbWUvc3R1YnMuZ28AcnVudGltZS9tYml0bWFwLmdvAHJ1bnRpbWUvbWhlYXAuZ28AcnVu +dGltZS9jaGFuLmdvAHJ1bnRpbWUvbG9ja19mdXRleC5nbwBydW50aW1lL2xvY2tyYW5rX29mZi5n +bwBydW50aW1lL3Byb2MuZ28AcnVudGltZS9ydW50aW1lMi5nbwBydW50aW1lL2NwdWZsYWdzX2Ft +ZDY0LmdvAHJ1bnRpbWUvY3B1cHJvZi5nbwBydW50aW1lL3RpbWVfbm9mYWtlLmdvAHJ1bnRpbWUv +ZGVidWdjYWxsLmdvAHJ1bnRpbWUvZW52X3Bvc2l4LmdvAHJ1bnRpbWUvZXJyb3IuZ28AcnVudGlt +ZS9oYXNoNjQuZ28AcnVudGltZS9pZmFjZS5nbwBydW50aW1lL3R5cGUuZ28AcnVudGltZS9sZnN0 +YWNrLmdvAHJ1bnRpbWUvbGZzdGFja182NGJpdC5nbwBydW50aW1lL2xvY2tyYW5rLmdvAHJ1bnRp +bWUvbWVtX2xpbnV4LmdvAHJ1bnRpbWUvbWZpeGFsbG9jLmdvAHJ1bnRpbWUvbWNhY2hlLmdvAHJ1 +bnRpbWUvbWdjLmdvAHJ1bnRpbWUvcnVudGltZTEuZ28AcnVudGltZS9mYXN0bG9nMi5nbwBydW50 +aW1lL2Zsb2F0LmdvAHJ1bnRpbWUvbWFwLmdvAHJ1bnRpbWUvbXNpemUuZ28AcnVudGltZS9tYXBf +ZmFzdDMyLmdvAHJ1bnRpbWUvbWJhcnJpZXIuZ28AcnVudGltZS90cmFjZS5nbwBydW50aW1lL21j +ZW50cmFsLmdvAHJ1bnRpbWUvbWdjc3dlZXAuZ28AcnVudGltZS9tY2hlY2ttYXJrLmdvAHJ1bnRp +bWUvbWZpbmFsLmdvAHJ1bnRpbWUvc2VtYS5nbwBydW50aW1lL2hpc3RvZ3JhbS5nbwBydW50aW1l +L21nY3dvcmsuZ28AcnVudGltZS9wcmludC5nbwBydW50aW1lL21nY21hcmsuZ28AcnVudGltZS9t +Z2NzdGFjay5nbwBydW50aW1lL3N0YWNrLmdvAHJ1bnRpbWUvbWdjcGFjZXIuZ28AcnVudGltZS9z +dHJpbmcuZ28AcnVudGltZS9tZ2NzY2F2ZW5nZS5nbwBydW50aW1lL21zdGF0cy5nbwBydW50aW1l +L3RpbWUuZ28AcnVudGltZS9tcmFuZ2VzLmdvAHJ1bnRpbWUvbXBhZ2VhbGxvYy5nbwBydW50aW1l +L21wYWdlY2FjaGUuZ28AcnVudGltZS9tcGFsbG9jYml0cy5nbwBydW50aW1lL21wYWdlYWxsb2Nf +NjRiaXQuZ28AcnVudGltZS9tcHJvZi5nbwBydW50aW1lL3RyYWNlYmFjay5nbwBydW50aW1lL21z +cGFuc2V0LmdvAHJ1bnRpbWUvbmJwaXBlX3BpcGUyLmdvAHJ1bnRpbWUvbmV0cG9sbC5nbwBydW50 +aW1lL25ldHBvbGxfZXBvbGwuZ28AcnVudGltZS9kZWZzX2xpbnV4X2FtZDY0LmdvAHJ1bnRpbWUv +b3NfbGludXguZ28AcnVudGltZS9vc19saW51eF9nZW5lcmljLmdvAHJ1bnRpbWUvcGFuaWMuZ28A +cnVudGltZS9zaWduYWxfdW5peC5nbwBydW50aW1lL3ByZWVtcHQuZ28AcnVudGltZS93cml0ZV9l +cnIuZ28AcnVudGltZS9zdHViczIuZ28AcnVudGltZS9zeXNfeDg2LmdvAHJ1bnRpbWUvcHJvZmJ1 +Zi5nbwBydW50aW1lL3J3bXV0ZXguZ28AcnVudGltZS9zaWduYWxfYW1kNjQuZ28AcnVudGltZS9z +aWduYWxfbGludXhfYW1kNjQuZ28AcnVudGltZS9zaWdxdWV1ZS5nbwBydW50aW1lL3NsaWNlLmdv +AHJ1bnRpbWUvdmRzb19saW51eC5nbwBydW50aW1lL3NlbGVjdC5nbwBydW50aW1lL2FzbV9hbWQ2 +NC5zAHJ1bnRpbWUvZHVmZl9hbWQ2NC5zAHJ1bnRpbWUvbWVtY2xyX2FtZDY0LnMAcnVudGltZS9t +ZW1tb3ZlX2FtZDY0LnMAcnVudGltZS9wcmVlbXB0X2FtZDY0LnMAcnVudGltZS9ydDBfbGludXhf +YW1kNjQucwBydW50aW1lL3N5c19saW51eF9hbWQ2NC5zAHJ1bnRpbWUvdGltZV9saW51eF9hbWQ2 +NC5zAGV4YW1wbGUuY29tL2dvMTE3L21haW4uZ28AAAAAAAAAAAAAAAAAAAAAAgowMy8cAARZAPYB +FAQKAQcCDwIKBRsAAAQBAgIOAUMCAgAAHgIRAg8DGwACFtAC8wTPAgHQAv0GzwIeAASlDACqAi4C +AgIPBAKIAQyFAQoKAwcGBC0EDgYFdgyLAQoYCghiBgYCDwEGBhkOHgIRGwUqExdeMQo0BSMFDwUa +OxkKHAVEHCIQIRsCDgEKChsKBAdVCw8OAgYKCgwHVRUhLAwCEQEMAhRDDQIyAQUCDyUFJAUCBUsK +TAUBBQICAiYCGykJKgNPClIGCDtZCloFCQUBBQEICwUDDQIkFwQaTwEIHRgPAxALExQPHgAACQEG +Ah8B8gsCBQAAnQMEIwIgAjQCIAFGB3wMIAJcASACwwEHqwECKgjdAQ8eAABBAgwBUwQMA6IGAh0C +IAOaBAACDmC1B18BYGtfAWAJXwwACMUIAFoYJDsfHgILAgsCCwIOAg4CDgIOAg4CDgIOAg4CDgIO +Bg4fGQIQAhACEAITAhMCEwIXAhMCEwITAhMCFAITBhMhNCghBAkDBAwlBCICCwQLAgsCCwILAgsC +CwILSAc9Bj4HMQgyBCkGLwQyFSgEIwIkByMEFgIOCx0IBAkCCgoEAx4CCxAHDQ0CCwILAgsCCl0K +LwsAAAQBBgIOAacIAgYAAB8CmwcBFgJgARUAAMwGAgcBBgQHAwgCBAEfBgQFAggHBwYKCwlIDAcL +TQACGwAaBAIEAgICBAIEAgQCBAIBAAIRADAFAgMCBAIEAgEAAgpAXj8fAAKHAQAEhwEAAAQBAgIO +AW4CBQAANgIkAgUDKAACCkBbPwFADj8gAAKUAQAElAEAAAQBAgIOAXsCBQAAUQIkAR8AAm8ACm8A +ugEaAhoCFwIKAgoCCwIFAAFvAAIiAA4iABoJAg0ECwQBAAEiAAKtBAB0AwIGAgMCAwIEAgQCBgQE +AgICBwIGAgUEBAICAgQCBAIEAgQCBgICAgQCBAIEAgIGBAIEAgIEBAIEAgIEBAIEBgQCAwIDAgMC +AwIICAEIBAICAgMCAwIDAgIEBQIFAgMCAggDAgMCAwIEAgMCBAIICAEICAIDAgIGBAICAgMCAgQF +AgMEAwYEAgICAwICBAUCAwQDBAMCAwIDAgICBAIDAgQCCAgBBgMCAwIDAgMCAwIFCAEIBAIEAgQC +BAIGAgYEBQIFAgQCBAIGAgYEBQIFAgQCBAIGAgYEBQIFAgQCBAIGAgYEBAIEAgQCBAIGAgUKBAIE +AgUCBQIEAgQCBQICAgQCBAIFAgIEBAIEAgQCBAICAgIIAwIFCAMCBQgDAgUAAg4ARAMCAwIDEAUA +Ar4CAJgBBAIGAgQCBgIHAgIIBAIGAgQCBAIFAgUCBQIFAgUCBQIEAgQCBAIEAgQCBAIEAgQCBAIE +AgQCBgICBAMIAQgEAgICBAIEAgUCBQIEAgQCBAIEAgQCBAIEAgYCAgIDBAMIAQYDCAQCAgIDAgMC +BAIEAgQCAwICBAMIAQgFAgUCAwQDCAEGBAICBAgCAwQEAgIGAwICBgUCAwgEAgICAwICBAUCAwYD +AgMGAwgBAB4DAgICBwIBBAMCAwIDAgUAAhwAVgMCAgIHAgEEAwIDAgQCBQAClwIAPgUCBAIEAgUE +BAICBAMEBAIGBAUCAggEBAQEBAQDAgIEBAQDAgIIAwIEAgQCBAIDAgIGBwIBDgMCAwIDAgEIAwIC +BgQCBAICBAQCBAIEAgMCAgICAgICAwIBBgYCBAIEAgICAgIDAgMCAgIDAgEGBwIGAgUCBQIFBAQC +BAIFAgICBAIDAgICAwIEAgQCBQICAgMCBwIBBgQCAwIDAgMCAwIDAgEAAhgAIgUCBQIEAgUCBQAC +CkAdPxwAAkMABEMAAAQBAgIOAS0CAgAAGQQPAxsAPhQCFAEbAAIGAAQGAJQDBgABBgACCQAECQCa +AwkAAQkAAgoABAoAoAMKAAEKAAIIAAQIAKYDCAABCACsAwoAAhoABBoAsgMaAAEaAAIWAAQWALgD +FgABFgACFwAEFwC+AxcAARcAAjcABDcAwgMFAjIAATcAAjkABDkAyAMFAjQAATkAAgpAMT8cAARX +AM4DFAIoARsAAAQBAgIOAUECAgAALQQFAyUAAgpAMz8cANQDFAIHAgUCHgUbAAAvBAUDJQDeAxQC +BwIFAh4FGwACDmAxXwJgFF8BYA5fAWA1XysABCbwAQjvAZcBAOgDGAIFBgQCBZsDCKIDAggRBBUX +Dwg1CysAAAQBBgIOAaUBAggAAEoEUAMrAAAmAggBlwEAAg5gNV8BYBRfAWAOXwFgM18qAAQq8AEI +7wGTAQCIBBgCBQYEAgQCBb0DCMQDAgQQBBUVDwozDSoAAAQBBgIOAaYBAgcAAE0ETgMqAAAqAggB +kwEAAg5ASz8BQD0/DgAEpQEA8AQYBgoCCQIJAgEeBwQUHwoMFAIIAggCCAIIAgojDQAABAEGAg4B +hQECCAAASwJNAQ0AADUCGwFVAAIEMFcvAQAGG4YBB4UBBoYBIIUBFAAsDgQN9AEH8wEG9gEDAgQC +EgIH+wECAggECgAADgFOAABNBAUDCgAAGwIHAQYCIAEUAAIEMDgvAQAGPQBADgIJAg8EDQIKAAAO +AS8AACECHAACBOABngHfAQHgAQvfAQHgAT/fAQEACu8BADwOAg4MCQJiBg0CDwQMBEAAAA4B4QEA +AHUEegACCnBobwgACnoATDECOAIKAwcAAAQBAgIOAWQCAgAASgQpAwcAAg5gXV8BYCRfJAAKtAEA +YhgCCgJAAgoEGwIKCyMAAAQBBgIOAZcBAgUAAFAEQQMjAAIKMDYvCAAKSABmSAAABAECAg4BMgIC +AAAlAhwBBwACBMAB2gK/AQEADN8CACwSFiESCAYJAgUIBQcEFwocBQQJBgsIMRRiHSwqBwgVVQVk +CgAADgHRAgAAhwECwgECDAMKAAIKYFVfCQAMaACGASkCLQILAwcAAAQBAgIOAVICAgAAOQQoAwcA +AgRgugFfAWAnAA7mAQD2AQ4CEggJTgo9CQIHAgYGDiAFEgwCMggMAgYIBSwOiQERBxYAAA4B2AEA +AFcEdAIbAAIEIFAfASALHwEgUh8BIAsfAgAOJNIBE9EBigEAqAkOAgUBBQoM/QEHAgUGB/4BDwUP +BwwOAxoDGQkCBxgJFwcYBRcQGAcXAgIPCA0AAA4BswEAABgCDAGdAQAAJAITAS0EAwMQBAkDBwQF +AxAGBwUeAAIEUGpPAVAJTwFQCU8BUAlPAVAiTwFQTk8BABJcOgk5KDoZOVkANA44BS8FBREGDg4U +DAkOCOQVBwIC8RUKCwoNCgUKmhYDBwUCEeUVCgZFCAoAAA4B8QEAABsCOQGPAQQcAABcAgkBKAIZ +AVkAAg5QfU8KABKVAQBsJQJUAhIDCgAABAEGAg4BeAIFAAAqAmEBCgACBJABTo8BAZABIo8BAZAB +CY8BApABCY8BAQASiwEAhgEOAgcSDwUFBREGDwIKBBkCCgsLBQoAAA4BfQAAMQQOAgoFHggPBxUA +AgSAAVJ/AYABCX8BgAEJfwGAAUl/AQASmAHEAQnDARQArgEOAgcBBQ4FBQUQBQ8MBhEKBQECBQoF +CgUKEgUCFwEJBAilAQmiAQoICgAADgGnAQAALAQOAhMFNQgpBwoAAJgBAgkBFAACBIACXP8BAYAC +Cf8BAYACcP8BAYACKP8BAYACmAL/AQGAAtcB/wEBgAIpABIqzgETzQExAwMEGwMLBBADCAQXxAEE +wwElxAEEwwESZAljA2Q5YwlGCUUDRgEeCB01RZcBRgRFPEYFRQVGBUUFRgVFBUYzRUJGCh4OANoB +DgQJBgZSBEkHBwLQBQcCBQYHwwULCQ8CCg8KGAPcBwPbBw0CDtoHC9kHAgoO0AcIzwcEBwYCDdsB +BNwBCAIKBgYCDeUBBOYBCAIKrggJpQgDpAgBzQES0AEmowgJ7gIJzQIDzAIB6AMI5QMQCAUKEQII +BAflAgtFDUgJEVYGCgwW0AUEywULARMEBAIOCAQPCKQEBaMEBaQEBaMEBaAEBZkEBZoEBgIIAgWd +BQagBQUGFaUEBQoJDwoQAg8IEAUECgkRwgIKtgUOAAAOAZEGAABSBA8DbASNAwI4BSkIMQcWBB0A +ACoCEwExBAMDGwQLAxAGCAUXCAQHJQoECRIMCQsDDAECEgEmCwkQCQ8DEAECCAE1D5cBFAQTPBYF +FQUWBRUFFgUVBRYTAgYBGhVCEAoDDgACClAoTwgAEjoApAIkAgUCCgMHAAAEAQICDgEkAgIAACQC +DwEHAAIEkAGvAY8BAZABEgASKUYDfgzDARhGA0VzANgCDgIHAgcCAQIBAgMCCM8BA3EMxgINAggC +A9cBA9oBAgQCBAUCBgQGEwUUBA8FCgUIBAIMFwUWBQcFCQUGBAkFGAIMCgkSAAAOAbgBAACCAQQo +AxYGBgAAKQIDAgwDGAYDBXMAAg7wAYAB7wEB8AEJ7wEB8AEJ7wEB8AEJ7wEB8AGhAe8BAfABnwHv +ATcAEvoBxAEFwwEmxAEDwwGPAcQBBcMBH8QBBcMBRQCeAxgCDQoFBgZUBEsHBwIQAwYFJwosBSAJ +BBYzEgIKDwoJCm4KIQcCAwIDAwYCBQEKAgUUBREFAwUECAMFDAULBcsDBcwDBRYFCQsGBwkBCgYC +A98DA+IDAwkCCwQcAxsRGApBFgYDHAMZAwIDAwoCCgEFFgURBQEFAgwECAQFCwWnAwW+AwUNBwQD +Bg0CA7sDBb4DAwkCDApDNgAABAEGAg4BhAQCCQAAfQQSA1cGXwUWBEEESQdAAAD6AQIFASYCAwGP +AQQFAx8EBQNFAAIOcOICbwFwN28eABSVAcIBBMEBrQIAkAEYAgQGEQYKCwUUBgInOAUjCQoMBhAC +AqsBBKwBICcFLhECOAYDBQIVEQQCggsEgQsXGAwCGAIJDApBFAkRBRIJHgAABAEGAg4BqQMCBQAA +jgEEUAI6AWcEKQceAACVAQIEAYYBBAQDowEAAgRQIk8BABQnAJ4CDgIPAgoAAA4BGQAAGAQPAAIO +gAJN/wEBgAKBAf8BAYACvQL/AQGAAtwF/wEBgAJf/wEBgAJC/wEBgAJj/wExABSxATIBBAg1ywIy +AQQKNfkFwgEEwQEyMgEECjWWATIBBAo1RQC+AhgCEQECQgh1CQQIcgJrBmwEAgxDB2gQGw8CFlQD +TQhOBU0JrwIBMQjmAhLmCAK9CQIEAgIMBBgCEbQJBwIJAQQIBQIFCBECBAIIAwYRAxIDEQMSBwIN +CwQOBRYDGQIFCQIFAQoCCR4JgwkDhAkbKwwsC/sIDg0JDg1vDJABAgIB7QEBnwEKkAMMCAUCBQII +AgoCCAgVAggCCAMTAg0CDQYDAwQCLAIKAgcBEAILAQIBAwIIAgNPA1AHGAMaAy8B6AcX7QcD7gcD +uQgDuggH0QcDGgNLBRYFAgUCBewHDQIFDA0CBAIEAwb9BwP+BwPJCAPKCAgCCQILywcDBgPCBwIJ +EQIEAgQDBvMHA/QHA78IA8AIBwIJAgjbBwMaAyUQAiIQFwYiAgcCBAIiAQQECQIVBAUJBQohAgUL +Bw4GDA+FAQQIBMMBCNMBBJgDDQITAgYCCAQDAgHjAQGfAQqGAw8hNAIPhgEOAhEEEx0RfQG1AQGf +AQrYAhRbMQAABAEGAg4BkwwCBQAAggECMwIXAyUGkwIFBwYWAvABB40CCooBAjACIQ0cEDAJmgEF +BwYZBTEAADMCEQECAgYBZQQBAggFEggCBzkIlgEHAwgyBzMKAQIKC4ICDhcNAw4DDQMOBw0aDi0N +Aw4DDQMOHA0GDiENAw4DDQMOGA2UAhAIAgQRMhQBAgoVlgEYAQIKGUUAAgogJB8IABQeMgEEBjUR +AKYDHsMBAZ8BBuQCEQAABAECAg4BIAICAAAfAhABBwAAHgIBAgYDEQACDlDkAU89ABSvAgDKBB0k +BQEHIQUkCQIaAwkECwQDAgUBCAIIAi8CBAIHAh8EEwIKOTwAAAQBBgIOAZICAgUAADIESQJpAg8H +PAACCmBFXycAFHYAnAUUFAUFBQEJAggGFwIKFSYAAAQBAgIOAWACAgAAKgQXAg8FJgACCmBLXyYA +FHsAtgUUDAUBBQEJAg4CFwIKDSUAAAQBAgIOAWUCAgAAMAQXAg8FJQACDnCuB28BcDNvFQAUNzIB +BA01JpQBDRAFowGWA5QBDRAFowHJAjIBBAo1F5QBCJMBA5QBGJMBCzIBBAo1PADIBRgCEQEFCgnv +BAExDaQFDxYHBAkGAhwECAGIXAwCAY9eBeQBA4YGDYMGCQEFCAoCCQIYAwkEB/wFBfcFEgIfBAny +BQXxBQMCLfAFBAIJAQQIBQIFCBECBAIIAwYRAxILAggLBA4FAwIFCQIFAQ4CCR5A8wUECAHiWwwC +AY9eBeoHB9cFCQYeAgcJBQwgBAnMBQXLBQMCKMoFCAIJAQQIBAIFCBECBAIIAwYRAxILAggLBA4F +AwIFCQIFAQkCCR4z5wUB3wQBnwEKhgYCBAsCCshbCM1bAgIB8lsFAgUCDu1bCnsB8wMBnwEKlgUT +CxQDFQAABAEGAg4B5wcCBgAAOwRCAywG/AIFOgiOAgcHChwJNAosCRUAADcCAQINAyYGDQIFBwMK +DQlJCgUJOgoFCTAKxAEJBQwNAgUCBw9cEAUPKxC1AQ8BEgECChMXFggVAxgYFwsaAQIKGzwAAgRA +HT8BABQiAO4GDgIKAgoAAA4BFAAAEwQPAAIWkAKOAY8CAZACE48CAZACswGPAgGQAhmPAgGQArQD +jwIBkAIrjwIBkALWBY8CAZACSo8CAZACEo8CKQAUjwIyAQQINRUyAQQKNfoCwgEEwQEPwgEFwQEE +wgEDwQFGMgEEDTUkMgEECjXxBgCOByYQCQ8CIgZRBwIKUAJLCUwEFAdlBwIKeAJzCXQGCgUCCQQX +GRQ/B1YIOggrDgIYagNjCGQFYwmXBwExCM4HFAgBowYBnwEKxgcNAhEEF+4DAu0EAhICAhQEGAIR +1gQQAgkBBAgEAgUIEQIEAggDBhEDEgMRAxIHAggLBA4FFgMZAgUJAgUBCgIOHgmLBAOMBBsrFCwL +gwQFEQkSDQQEswYI0wEEkAgPjwgFkggEkQgDkggLBBYCEwIGAggEAwIB4QYBnwENhAgXtwEKvgEC +AgHrBgGfAQqOCBQIBQIFAggCCwIICBUCCAETAgsCHgIDAQoCCCADHQQCBAIHARUCCwECBQMGAxkD +Gg0CAwEDAggaAw4DxgID4QIDCQHqAhfzAgP0AgPtAgPuAgjRAgMOA0EFEgUGBQIF7gIIAgUMDQIF +AgQDBoMDA4QDA/0CA/4CCAIJAgjlAgMOA9QCAgkRAgQCBAMG+QID+gID8wID9AIIAgkCCNsCAw4D +GxACIAYhBiICBwIJAhUEBQIFAQgCFAIIAQ4CDAIJAhqBATQCF2oS+wEpAAAJAQYCFwGYDQIHAACJ +AQRVATUEGQUHAh4CHAMtBIwCAzEIFgIwBSsDBwQeCPIBC4gCDmgCTQuAAQMpAAA3AhEBAgIJAQsE +EQMCBAkDlQEGAQIIBxUKAQIKCzUOAg1BDp4BDQMOOg0fEAgCBBEPEgURBBIDEUYUAQINFSQYAQIK +GYUCHAMbBBwXGwMcAxsDHAgbGhwpGwMcAxsDHBkbBhwhGwMcAxsDHBkbjwMAnggeuwYBnwEG3AcR +AAIOYIYDXz0AFGvCAQTBAQXCAQXBAdgCAMAJIg4FCwcICQQOMAUQBQsFMwUNBRwE5QcJ0wEExgkF +xQkFyAkSBiICEwIGAggECAQFEAULBQMZPQM+CwQDAQgCCAIvAgQCBwIfBBMCCk88AAAEAQYCDgG0 +AwIFAAA7BEsCIgJ1AmkCDws8AABiAgkCBAMFBAUD2AIAAgogNB8cABQnMgEECDUqAJQKFAwHCAsM +AdEIAZ8BCPQJDyEbAAAEAQICDgFEAgIAACsEFAMbAAAnAgECCAMqAAJUAB5UACALBAkCDAIHAg4C +CAQWAgEAAVQAAg6wAasCrwE9ACDMAcgBEscBH8gBBMcBdQC4AR0wCh8FIAgrAgISCAUiBQMFBA0f +AwseCA0CIAIIBBKhARK6AQwBAwYQvQEEvgEkBgsCCjc8AAAEAQYCDgHZAgIFAAA2An8CawIaBTwA +AMwBAhIBHwIEAXUAAgRA8gE/AUAPACCGAgCGAg4SBQYFDQICEgYFBgUNGwgYBBsCOAIFDwcQGgQH +BgQCCg8PAAAOAfgBAAAaAuIBAgoAAhagAp0EnwIBoAImnwIVACCTAogBIIcBfYgBIIcBnwEAugIu +BBQCBwJFAhYDCwI2BgsGDgIV7gQg6wQJAgMBBQQFAwgIJgITBhECFdoEINcECQIDAQUEBQMFCCYC +EwYQMyYHFQAACQEGAhcBxAQCBQAAiQECmwMBFgQgAxUAAJMCAgkCCQEHAgcDfQYJAgkBBwIHB58B +AAIEcKEBbwFwFW8BcBVvAQAm0gEAOBMEGgYQFAYCT14UaRYNFgAADgHEAQAAewRXAAIOwAFgvwEB +wAGRBL8BAcABWr8BAcABZ78BCwAmMroBA7kBmQYAWCoCCLYKA7MKAgIuAgoGEQQQAlUBCgJWARQC +fgEgAoABHAoQIwIuAgoIEgIGBBECBQQwBApbCgAABAEGAg4BsQYCBQAALQQ4AxQGSgWqAgg4B3MG +OgUuCjoJFAAAMgIDAZkGAAIEoAHZAZ8BAQAmtQGSARGRARgAzgEOAgUCCAQFCGhcDC4HAgkCCgIG +AgGiAQpMB+sBAgYMAgoAAA4B0AEAAHYEaAAAtQECCgIHAxgAAg6wAaACrwEBsAEXrwEKACb0AZIB +CpEBAZIBB5EBDZIBA5EBCJIBB5EBKwDeAUAKFQIQAggCIwQnCA4EBAIFCAgCDQgGAgoEAfoBCvcB +AcQCB8ECDfYBA+cBBwgB4AEH3QEKMRcjCgAABAEGAg4BswICBQAASwQRAt8BAwsBCgAA9AECCgEB +BAcDDQYDBQgGBwUrAAIKMGsvCAAmVJIBDJEBHQCMAxQCBwQMAhYDCgQHBgUGAXIMbww6Ck8HAAAE +AQICDgFnAgIAAE4EGQEPAQcAAFQCDAEdAAIOcJMBbwFwDm8VACbFAQDgAykECQIOAgoEBQIrDAoC +BQIoHRQAAAQBBgIaAX4CDwEPAgUAAIIBBC8DFAACCjBBLwgAJlMA7gMYAgUCCgIbBAoJBwAABAEC +Ag4BPQICAAAiAiABEQACDpABS48BAZABrwGPAQGQARyPAR8ALsUCABodAg4CCQYFAgUoBTEHCAIK +DgkOAhUBFCoOKQoCBQEFBAoBAygFJwoBDgQ1BxQFHwAABAEGAhMBogICBgAAmgEEcwIZBR8AAD4C +BQFOAg4BIQIFAYABAAIWgATeBP8DAYAE3wH/AwGABD//AxUAMKkHAD4uBBYCCBADCwgMCgsMAhwK +DQMIAhMC2wECExAKDxsQCg8FBFYCHwQdBhAGHwFjAl4fQA0UAAAJAQYCFwH+BgIFAABHAiECMQLu +AQIyAkgCGgImAiQP1wEQWQ8UAAIKYDFfHAAwVwCaARkCIwEbAAAEAQICEwE8AgIAAC0EDwMbAAIK +YDFfJwAwYgCwARkCIwEmAAAEAQICEwFGAgMAAC0EDwMmADAGANgBBQIBAAAFAQEAAhmQBYgGjwUB +kAXUBY8FMQAwpwwA6gIpAiUCEgIaDAgNBCQIIwgOBQM4AjABCAQFFgMVDQIGFAUTBQIGAgkFBQIF +BBkFBSMIKgUFBQIIBAwpCBYFFAYCBQQEAgkBBQIJAgkdAhUHAiIBCAIFIgUCCAEFIQYCCBIFEwUC +CAMDCgMBGeMBBQIJkgIJNwUCIgIIAQUiBQIIAQUhBgEIFgUTBQIIAwMKAwEZ4wEFAgL+AQMWDQYd +KQPtAQwCA9YBC9kBA+QBA+MBBvABA+0BCuIBA+EBJQgW2gEP1wEb2AEF1wEF2AEF1wEI2AEV1wEF +2AEk1wED2AENCQgWBRUIJAUCCAEFFhmJAg0HGQID1gED2QED5AED4wEJAgriAQPhAQfuAQPtARUI +FtoBD9cBFtgBBdcBDdgBLdcBBdgBEdcBA9gBDgkIFgUVCCQFAggBBRYSAwWFAg0HDYoCCwsLDQ0D +DgExAAAMAQYCFwH5CwIFAADZAgRwAyIEfgMeBIYBAdoBBDcCTwXBAQg6AUsFQwExAADnAgIIAR4E +CAM2BCkBCAIFAxIEDgMFBCwCDgUJAjQBEgIGAggDBQIsBAUCAgcwCA8DAwEIBgMFAwYGBwMICgUD +BjsFDwYbBQUGBQUFBggFFQYFBSQGAwUNAggDBQIIASsIGgEPAQMCAwEDAhMBAwIHBQMGKwEPAhYB +BQINAS0CBQERAgMBFgMFAggBKQYaBWIAAhaQAqcKjwIfADDcCgCsAzYCPgYcAhUHETodAhc7DSIa +AhkjET4lAiU/FS4YAhgvERoYAhcbDTYcAhk3EUIlAh5DKBYYAhQXGDIYAhQzEA4ZAhQPFyoXAhQr +HR4dAhkfEUYhAh1HDxIZAhQTEyYYAhQnCgoUAhgLAkwFSwIEG0wQUR4AAAkBBgInAaEKAgUAAJAB +BKwHAhMB3wEDLgACDoAC7Q7/AR8AMJoPAIIEKAQNKAUgBUMqBgUFAwoGAloDXgYFAlsEYBMJFgUC +aARgAgUCWwRgIxImBQJpBGACBQJbBGAzCTYFAmEEXQIPBAk/BUZhQ2hASwNQDApTHgAABAEGAg4B +/Q4CBQAAMAJXAg8CSgEPAlYBDwJMAQ8CXgEPAlkBDwJWAQ8CTAEPAmgBDwJZAQ8CVwEPAkwBDwJh +AQ8CUgEPAmYCJQIbAy0GDwIgBz0BDwI8ARACIwUoAAIZ0ATFBs8EDAAw6gYA4gQpBBIMCAMqAgkG +EwIJAiMBQgEFCCkCJQIJBhICJQEkCQMUmQEBHAKBAQkKAygLFAEIAykPDAAADAEGAhcBvAYCBQAA +MQIlAlUBCgF+BksDDAHEAgKQAQEMAAL/AwA6UpwBC5sBAjUDNgE1AzYENQM2BZwBC5sBAjUCNgE1 +AjYKNQI2BJwBC5sBVjUDNg01BDYZnAEEmwEPnAEQmwEBNQM2ATUDNgs1AzYNNQQ2DzUENg01BDYP +NQQ2DTUENiOcAQSbASQAOhQEDAQGCAIGBgYCBgYKBi8DOgMCAwUFNQMwBUMLPAFcAYgEA4cEAYgE +A+EEAgcBYgGIBAO1BAMxAjcLMAFgAf4DAv0DAf4DAqMEBjcCBwFmAf4DAqMEAj0CKwsiAwIRAgpE +BUMCBwRMEBwGGw0eAwEDAgMdAbYEA8MEDcQEBMMEAwYDJANxA3IDKwQEAToCDwOHAQRqAw0DBgMH +BlkQmAEBiAQDhwQBiAQDtQQGBQW8BAPTBA3UBATTBAMWAyQDcQNyA5oEBNEEDdIEBNEEAzgMmgQE +zwQN0AQEzwQDNgw7BAgBRgIPAxACDwMQAg8DhwEEagMdAwIDAgMSAxcKDAsAAf8DAABSAgsBAQQB +AgMCAQIDCQMMAQIDDQUQCw8BEgECAgIBAgIXCRoBAgIbBB4LHTkgBh8NIAMCBiEBJgMlDSoEKQYs +AysDLAMrBSQBBAEEAwIELQ8wCAIIAgECAwIBAgM5Cz4DPQ1CBEEGRANDA0QDBARHDUwESwNEAwoD +CQMKAwQEUQ1WBFUDTgMKAwkDCgNXBTwBBAEEAwIBBAEEAwIBBAEEAwIEWSQAAkUABAI2QwCwBQKZ +BCQSBgIGAQ0TAR4BBwMTAQABRQAEAgMkBgwCDQcBAgEGAwcBAAJGAAQDNkMAwgUDoQQkCAYCBgEN +CQEcAQ8DCQEAAUYABAMDJAYMAg0HAQIBBgMHAQACDoABRX8BgAGfAn8BgAEJfwGAAckCfycAQHcG +AQQNCSYGAQQMCY4BBgEEDAnEAbIBCbEBzgEARBgCDygKHQ8LAg4GAgwPBCgHAhcKARUBMQ1KJQIB +ZgGfAQxeCCEFCCICEwIJAQ8CEwMFEAcCBQIPAgFGAZ8BDF4MWwdiAgIMBQoWlwFfAtoCCdcClQER +EgMnAAAEAQYCDgHRBQIFAABoAg4BCQI0Ai8BTwRDBQoCGAZFAjoJAgIfBkUCTAknAAB3AgECDQMm +BgECDAeOAQoBAgwLxAEOCQ3OAQBAJJYBCJUBGgDKAQaLAQaOAQgCBhgDAgMBAxUBrQEItgEDAgUG +CwIEBQMAAAYCBgEYBAgDGgACDpAB8AGPAQGQASePARQAQLoBOSQ6XAD0ARgKEQgHAhsBBS4FIR0C +DgwlAhQIAecBCQIMBA/mAQcGEAIKEREjFgsUAAAEAQYCDgGdAgIFAABoBDQCHQURCCYCNgkUAAC6 +AQIkAVwAAk4AQDCWAQiVARYAxgIHAQaFAgaIAggCBiYDAgMBAyMBAgWpAgisAggOBQoDAgQCAREB +AAFOAAANAgYBHQQIAxYAAhbQAu8DzwIB0ALiAs8CAdACK88CFABAV5YBCZUBH5YBBZUBdbIBEbEB +ELIBC7EBpwKyAQixARayAQqxAWSyAQ2xAQWyAQ6xAbABAIADLgIIAgkCCAwJAgQCA/MCCfQCDQQN +EwXjAgXwAgUCDQoCAhYSBQQFLgMlBR0JAhEBBQIIEAUyCD8FYhFfEAwLCQ04DTUNAgUCDhQmBBgC +ChwIGQUEJRYIFQUWDAIUMxgQBQ8DEAUPCQIGAQUCCQIEAwgICAEOTgpNOSgFJw0CDQIFAgcLDQwF +2gMO1QM6CQUKBRwQAh0rCBELBwoDDhUUAAAJAQYCFwH9BgIFAABCBLoBAiMCDgIaAlECnAENGxAg +AmIRDBINETIQUAQoExQAAFcCCQEfAgUBdQQRAxAGCwWnAggIBxYKCglkDA0LBQ4ODbABAAIOgAGC +AX8BgAE3fwoAQBgGAQUHCgWWAROfAUIGAQQMCUsA7AMYiQMBjgMHvwMFhAcHAgUGB8kDJQIcCAGZ +AgGfAQy8AwoJCQIFAQkBDwIRBwoAAAQBBgIOAbUBAgUAACACWQEJAhgCLgMKAAAYAgEBBwQFAhMF +QggBAgwJSwACCmBUXxwAQHoA/gQUFAUFIAYSBhQZGwAALwQXAhkFGwACCkBvPx8AQJgBAOYFGQIF +AgkBBQgFARYCKQQKDR4AAAQBAgITAXoCBQAAQgIuASgAAgpgUV8cAEB3AJAGFBAFAR0CEgYUFRsA +AAQBAgIOAWECAgAALAQXAhkFGwACClA9TwFQDU8BUA1PKQBAjAEAzAceAgUCAhAFCQoCBQoPBw4H +DgUoAAAEAQICGAFpAgUAAC8ENQMoAAIKQFs/EgD8BxkGDgIKAgUCBQMFBhAFCgYCBgoTEQAASQQT +AxsAAgowHC8HAEAtAOIIFAISAQcAAAQBAgIOARcCAgAAIAIGAQcAAg5wYG8BcLcBbx8AQiQCHwE2 +AgoBwgEANBgCCwIBJhQOCzEHCgYCAxEDFA8IChkFBgUiBQ4FL4wBAhcJHwAABAEGAg4BpwICBgAA +gwEEGwJdAisHHwAAJAIUAgsDNgIFAgUDwgEAAg5QLU8BUFBPGQBCGwISAXgAfBgCAx8KDggSBQoK +CwUEBQI0AhIHGQAABAEGAg4BgwECCgAAQQQPAjwFGQAAGwIKAggDeAACCjAaLxIARjYAYBQCBwIK +AxEAABYEDwMRAAIOgAFZfwGAAbwBfwGAASl/AYABKH8BgAESfxQARp4DAGgYAgQEDgYIBgcCBBMF +NAcKEwkCHQpaES8FAQoYBhcGKAcCBykFDBkLIhQFDBILEhMFFAsYChUJUwNWFAIKFw0/A0IPAgo7 +EgcUAAAEAQYCDgGBAwIFAAB0AqcBAWkEBgMUAAIKIBgfEgBGNADgARQCBQIKAxEAAAQBAgIOAR4C +AgAAFAQPAxEAAg4wWi8BMCMvGQBGpQEA6AEYAggCBAYFAgoICgIOBgwCCAQKCREREgUZAAAuBB0D +KgQXAxkAAgpAKD8BQE0/FABGlAEAmAIUAgcCBAgKAgoLBAQ3AhIHFAAAJARcAxQAAg5gsgFfAWAS +XxQARucBAKoCGAIEAg0DBQIFCgsMFQsCFBALBQQFAwYCCwIHAg8CIQgKHRIFFAAABAEGAg4BygEC +BQAAggEENQMWBgYFFAACBHB4bwFwywFvAXB6bwEARkmiARahAbABogEWoQGfAQDcAhMCCAQCAgsM +DgsFHgklBbECFuACDQICBw8qBgUDGRsGEAIUAg8CHwQQAgsGAfsCFv4CDgoWORALBQQFAwYCEAIM +Ag8CIQgPAAAOAbYDAABJBMUBAwEEtQEAAEkCFgGwAQQWA58BAAIKMDMvATASLxsARmsAuAMUAgQC +FwgPBRIFGwAABAECAg4BVQICAAAvBCEDGwACCmBSXwFgEl8eAEaNAQDOAxQCBAINCgoBBQIVAgUC +DwsSBR4AAAQBAgIOAXQCBQAALwQPAjEFHgACBDBDLwEwFS8BMBUvATAGAEh6AOACDgIFBggGLQMW +BRYKBgAADgFsAAB0AgYAAgogGB8cAEo+AC4UAgUCCgMbAAAEAQICDgEoAgIAABQEDwMbAEo0AEAU +AgUCCgMRAAIOQK4CPwFA0gM/CgBMmQYAxgYYAg4IBxICAhoBCQoXCBcIDQgNCBQICQoLBAoiDAIR +BgJKBTIMAxkGAwISNQnAAQqrAgsBJxk7AhYJOwIRCV4CFglbAhEHESUSAwoAAAQBBgIOAfwFAgUA +ACYCjQIBPAKgAwEKAAIWoAKSBZ8CAaACngafAgGgAqgBnwIeAEw4igETiQFBKhEpNyoRKWAqCinE +AxomGTYqDCkCKhIpGhQeExwqCik1KgopHIoBA4kBAioKKbUBKgopDyoXKQIqEikYKggpngIA7gku +AgIEAQYHqwQTrAQVAhe8ARXPAhHQAgpUCQwIXxACAwEJzwIR0AIOAgcCBQQRAgkGEg8NFgUICO0C +CuYCJQgdAgUCFgIJDBwwBS0EAgUCBwQaAgkILQIjDAgOBQ0eAhIcExsIAQUTERcRDREBDQkReQsC +AYsJFQQKAgfGCQODARUEAwIFAgYEDgUCpQEMrAECqwESsgENBQoOA+UHFwIHjAgIOQU6BRkFCQW9 +AQq+AQgeCR0IIAoECDkFOgXhAQqkAghFBRsFBgrhBAPmBALLAQqkAgMhBhcJGBcCCQENDBkCDwIS +AgoCGAISOgjRAgqkAgYCCaUCF6YCAqUCEqoCGKkCCK4CBwYpEAkMBgsFPRRAESEFAgUOCgZxAhK1 +AR4AAAkBBgIXAeMMAgUAAFoCdAE9BIEBAqACA9EBATsCUwJoATcGKwNpA5YBAhkIYgchAR4AADgC +EwFBBBEDNwYRBWAICgfEAwomCTYIDAcCCBIHGgweCxwICgc1CAoHHAIDAQIICge1AQgKBw8OFw0C +DhINGBAID54CAAIOgAHKAX8BgAERfwGAAQ5/LABMKxQdEw8UBhMUigETiQGhAQC0DB1SBUUJgwoa +AgOCCgwEA4UKBooKDgcFOAGpBxOsBw0CDgITAgUCDgQUNxIHDxErAAAEAQYCDgGFAgIIAAA9BJIB +A1YAACsCHQEPAgYBFAQTA6EBAAIOgAHkAX8BgAGAAn8hAEyUBADODRgCGhIJEQoECAITBA0FBQ4T +BBQEBRMKGgUFAhUDHAoIDAIKAgwlBi4PCQUCCAJPAhYNERcKAk8CFg0OASEAAAQBBgIOAfcDAgUA +AEgCqwMBIQACDvABlAHvAQHwAeAD7wEB8AHHC+8BAfABEO8BAfABfO8BLABMxgJqBGkBahBpJA4h +DR9qBWk7igEIiQEKigEIiQEKigEHiQEqaiZpoAKKAQiJARuKAQuJATAqA2AEXwMp0QMqBCnqAmom +aZcBHCwb2wEAkA4YAg4ICQgNAgkEBRAWBA8EBgwOCwcMGQYZBAdFB4oDCM0CCBgJBhABBQwWBAUD +BQwNNAijAQiOBAXxAggrBNcHBOIHAeEHBgIK4gcNBg0GCpcNCwQFDgYDC5QNCQoWgQgF9gcFDAXy +AgXpAg0HAwoQPAQECKcKCKoKAhIIuwoIvgoCAgcCAcEKB8YKEwQEAgQCBAIKAgHpCA8CDwQI5ggK +BgSNAxgCBgIJgAUD/QQJAgkCEIYDAv8CGQIEAgwCDGAImgIC9AEDgQUCkAMKAg3IAQUoBeEBBY8B +BoYBA54BA1oD7QEIqwIIhAMIXwQIDQQHAgaUAgPBAQpHCOUKCOgKG+cKC+wKGQQWAgGfCAO5AgS6 +AgOiCBa/AwmEBQODBQkCCgIJAgoCCAIT0gQFlQEHAwKxAxkCBQIOAg5gCOYDBZUBBgMCmgEFlQEF +AwIFBQgKAguWAQUoBTMFvQIG+AIFvwEDbANaAwUIkwQIhAMIgwMC1gIKAQXGAQVZBWcLvAEIkwQI +xgQFwQEIYgUoBTMFvQIG/gIGBQVTBVoDWQOGAQMxBTIJxQQC5gIhAggCBs8JBNIJAQIE7QII5gMF +KAUzBb0CBqQCAzAJJAiPAQjCAQUxBQYFKQUvBaMCA9oBBw4HBh8CEwcCEAQCCgEDDAkLAggEeANz +CSADVAhTBQ0SDAkCGxYMBBgCCAQUNAUzAjQFLQ8CAcEKDwIPBAjECgnvAwrwAwICDSIFGwkCCwIV +CgUDGQQHCAoGDwYJ6wkbEhHcCQgCCygPuQQRvgINAw0DDgMNqQETDxEFEXkSAywAAAQBBgIOAeMR +AgoAAJQBBIcBAuQBBa0CCG4HyQIIYQJ6AZsBAqsCAi4CTA06EC0LdAMsAADGAgIEAQECEAEkBCED +HwIFATsGCAUKCAgHCgoHCSoMJgsODhINBg4PDQMOIg0CDjUNDQ4CDYABEAgPGxILETAUAwIEAQMT +FhgJFwMYQRcOGDoXpgIaBBnqAhwmG5cBHiwd2wEAAgpQeU8iAExiXAxbNwDsEhQCBAYFAgIMAwIH +BhQTGwIK4w0M7g0KBQISCiEhAAAEAQICDgGJAQIIAAA4BCoDBwQRAysAAGICDAE3AAIKQCY/EgBM +QgCYExQCHQERAAAEAQICDgEsAgIAACIEDwMRAAIOQE0/AUAcPwFAFD8eAEyrAQC2ExgCBgYDAwME +BgIbBhcLHQgUCx4AAAQBBgIOAY4BAgUAAE0EQAMeAAIOYIABXwFgF18oAEwYDh0NmQEA1hMY1REL +BAUOBgMHzBEFHg8XAgcFCgoyCDMFAgUBAwIFAQMCDgIKBxcFKAAABAEGAg4BsQECBQAAWgQjAikF +KAAAGAIdAQUEDwMRBAgDbAACBDCpAi8BMAsvATAOAEwpigE2iQEQFwEEBQN3GE4XDgCeFA4IDQIF +AgmtEgQMDAYGAwcCEAIGAgO0EggCCJ8UATgFLwsGGgcVBAsEBgIskBQMAg0IKSMM9xMOAAAOAboC +AADCAgIGAAApAjYBEAQBAgUBdwNOBA4AAg6AAXx/KQBMswEAphUYAgkCWwYPCSgAAAQBBgIOAZYB +AgUAAGoEIQMoAAIKUDVPCQBMSACqFSkCDQILAwcAACkEGAMHAAIOgAGoAn8BgAGrAX8BgAG0AX8o +AExuagRpAWoPaR0FAQQMAgpqBSAFiQEQigETiQFkahJpB2oYaRIFAQQMAm+KARGJAQlqBWkrBQEE +DAJiALwVGAoRBgUCEgYTDgwCChYF0Q4Ewg4BwQ4GAgnEDhECCwQBkRUBMQzMFQU0BfkOBecBBagQ +BwQJqxATrBAEAhICGQITGxI+BwIIAgHvDhKWEAeTEBAECJAQBaMBDAIBxxQBnwEM8BUFAQUIBQcM +AgsCFAQSIwMBBwIDAhbHEBHQEAQGBe0OBewOAw4FEQUbDAYRAwGhFAGfAQzGFQIxEQUVBxILKAAA +BAEGAg4BogUCBQAApwEEXgIjAmEHCQomAk4LRAgFBwkIPwcoAABuAgQBAQIPAR0EAQIMBQoIBQIF +CRAKEwlkCBIEBwMYBAULDQ4BAgwPbxIREQkIBQcrFAECDBViAAIOYKEBXwFgC18zAEwbigEOiQEY +igEOiQEQigEKiQEiFAUTXgCQFxgCA8MRDsYRCgYDAgvNEQ7OEQUCBg0FwREK1BEOAhShFQWmFQUE +BQMJBA0VDAUyAAAEAQYCDgHRAQIFAAByBEoDMgAAGwIOARgEDgMQAgUCBQMiBgUFXgACcwBOCYgB +OYcBFIgBA4cBGgDOAwnRAQQMDAYGAwkCFAIGzgEUywED0gEEAgoEARcKAgEAAXMAAAkCOQEUAgMB +GgACDrABoASvASsATj6IAQuHAUiIAQSHAR4BEgK4AogBDYcBTwDsAyImBSESTQX/AguAAwZWAg4B +WxleJuUDBNoDDwsEDAcSBIwPEoEPAwUFBgUFBQIUAgEQDAIhNwk4BX0FggEKAi8VA20Fbk1tBW41 +bQmHAw2IAwJyAXEYdAoxKgAABAEGAg4BvAQCBQAAvgECFAIZAxQEmgECjAEFNAAAOQIFAgsBBgED +BhkFJggEBx4KEgknDC0LCQwFAgUBOQsDDgUNTQ4FDTUOCQINAQINAQ4YDTQAAg6AAfQCfykATmiI +AUKHAYECAOAEGAIHMAMvBgIYCgUkBRMFDQzwCwXNCwUdA/cCBAwNBgcDCQIWAgcCBOQCBQoCAgIE +A9wLBusLA/8BCuwNDuELBAYICggZBR4wAgUBBQQeAgkBFAIOCBA/KAAABAEGAg4BjgMCBQAAVgRg +A0EGOgJSBygAAFsCBQEIBEIDDAIGAQMGCgMOAdQBAAIOcNgDbwFwP28nAE5LOB5QCE8vNwKIAQtP +LzcCOAw3BTgBUBCHAa0BiAELhwEHiAEEhwEpODo3JwCyBRgCA78CCsgCCQoEAQPPAgnQAgMCCqcF +FQIJxgUIwwUvpgUCHgu/BS+iBQKbBQycBQWVBQGcBRADBQIJGQMaAxoDGQcIAw4GFwIKAw4GLwW9 +AgXuAgUNBRIFAwsCBwINBAsUBycEDAMLCxwPDAkCB90FC4gDAtQCBAQB3wUEiAMX3AINOQWlBRoD +IJAFJwAABAEGAg4BsAQCBQAA4wIEJgJeAj8HJwAAGwIKARAECQMNBh4CCAEvBQIKCwMvBQIGDAUF +BgEGEAs3AgUBcQ4DBAgBAg8FFAQDFw8SBjoFJwACDnD9AW8BcDRvAXCtAW8BcBdvKABOnQGIAQSH +ARqIAQSHASOkAQejASeIAQiHAU+IAQgcB6MBTogBDRwHowFWAIoHGBQbDBcfChgFDhICCScDlQQK +CgS2BBQCBJMHBJQHBdwJCdkJAgQDBAedBwT2AgUCCKgEAgwDxQQEAgQCA74EBAICqQEHfgICDgQT +gwQE/wIIgAMIrAQMAgIoEycJAgkCBwMFDgi7BwjoBQfWAQICAwQZEwoBBa0EBcQEAgIaxQcN7gUH +2gECAgMEEkMXISgAAAQBBgIOAZEEAgUAAGkEPQNRBpcBAhwHUQYLBSgAAHcECgEEARgGBAUFCAkH +DAoEAg0LBQ4LDQYQBw8jEgQCCAEDEUwWCAIHFy0SBREcGg0CBxtWAAIO8AGkCe8BAfABN+8BKABO +iAEBDQKQAYgBCIcBEIgBBIcBrgGIAQuHAVGIAROHATOIAQgcCaMBXKQBD6MBNYgBDYcBKogBDIcB +CKQBA6MBEqQBB6MBCwERAiekAQijAQ2kAQejAQsBEgJPpAEIowEepAEHowFxAPgIGAIRGA4gEBkc +CBIEBwIM9AkN8wkXAQwCBQgPZAgCFWUDCAgHBLkGCAoEsAYD2gcF1wcCAQUECKkGBa4GBZMJCJQJ +EJMJBOYCBwIEAgOsBhEOBVgD6gYF9wYT0AYG5QYDhQcM7A0QzwYIDgPYBgPlBgXmBgoIEO0GCxUL +MwKjBgX/AguAAwbaBgo1CQIS5wcE6gcHBAw1AzgMrQkTpgkDGQMYA1QDRQoNBYwBBRMFIwU7CbsJ +COgFCdYDAgIDBB8dCsAHCLkHCB8FGAVUBSQFFAVpBcsDD9IDAgIWBB3RCQ3UCQUoAwMFBA4CAgIB +AgyBCgygCgi3BAO2BAgSBREFtQQHogQCEgUPBOwIEekIGhAFAgixBAjCBAUTAwIFrwQHpgQCCAUF +BOIIEt8IGgQeAgwCC7EECMIEBQkSBge9BAfABAICAwQNCxK1AREXFAMoAAAEAQYCDgH1CQIFAABc +BNkBA4oDBj8FHwgQBzgEVwZFCRoMPQJnAisPKAAAiAECDQFjBggBBAMDCAUHDwoFCQUMCAsQDAQB +DgkZDgUNExAGDwMSDAEQDwsUAxMFFBoTGBYFAgsBAxUoGgQZIhwLAggdMyAIAgkhLg4IDSYkDyM1 +Jg0lKigEAggpCCwDKxIsBysLLhEtJzAILw0wBy8LMhIxTzQIMx40BzNxAAIOcJYDbwFwEm8eAE7V +AwCGEBgKClAG6w0I7A0TTwkEDA0FFAQBCQIPBA4CBgIDBhACBAISAgQBEBMEFgUCCAIGBBEECwYP +BQQGBQIfBAUCCS0FMCATCRQFBCYKChsSNR4AAAQBBgIOAbgDAgUAAHQE2gECTQUWCAYHHgAAIgIG +AggBCwEFAgMBkgMAAgpQZU8qAE6ZAQC0ERQGBCcMDAICAhoKBQO/Dgq6DgQMAxUBFgUxCjgCAg4E +ChEpAAAEAQICDgGAAQIFAABHBBoCBQUzAAAYBgwBBAMNCAoFBAEDBAEDBQoKCUMAAhbgAr4C3wIB +4AKVCN8CKABOnQGIAQSHAQWIAQSHAQqIAQiHASCIAQSHAQiIAQSHAQ2IAQiHAVmIAQiHAROIAQ6H +ASeIAQiHATeIAQaHASekAQqjAVqkAQ+jASukAQijAQmkAQqjAVCIAQWHAQukAQijAWGIAQSHARGI +AQiHAQikAQOjASmIAQWHASCkAQmjAU6kAQijASaIAQUcCaMBd4gBCocBcADkESYEBAEISQUJDQwC +AgNEA+8ODCQEAgjQDgYKKgQJ1REE1hEF1REE2BEFAgXZEQjaEQhpCW4CCA3lEQTmEQjlEQToEQgC +BekRCOoRCCcFPwTSARCBEAfKEAYCA0kDShAEEKEQBf8CCIADA/AOCgIBAgXzEQ72EQKSARUCCwYB +AgSREwiUEwsKCAENAgUJBZ0BDfURBvYRCgIHuRAEvBACAgQCAgQKmQwKogwCAgVWA2sKbAVZBZcB +CaIBChkFIBICEqUMD6YMBgEFAhg6COsMCLIMAh4FHQKxDAqyDASzDwQCBAIDyA8IAgIEDiYFJQVk +BUcIBwUTA2YFMQXrEgXiEgUVAxID9QwI5AwNAhcEDwwFCSACCdUSBNgSBQIM2RII2hII8QwD0AwF +ZAWrAgiIAQUBBUIDJgUOBesSBd4SBgQFAxX1DAn4DAICFmsDbANxA3INaQMBA2wCBBj3DAjKDAVk +BasCCIgBBQEFdgrrEgXuBQn8DAICHHUDdgN7A3wIcwMBA3YCBBcOCEUFZAWrAgiIAQUBBX4F8xIK +9BIFCgoCDUcDZgUdBTENSRJBKAAACQEGAhcB5woCBQAAuwIECgNzAl4BUAYUBS0GLAWqAQbfAQI8 +B1IIaQMXAygAADICBQINAQUBAwYMAgwHOQoECQUMBAsKDggNCBAJDw8SBBEIFAQTDRYIFQ0CBAEQ +BgUFLhgFAggBAxcQHAQCCh0nIAQCBCE3JAMCAyURKAQnEioKKR4sCSszLg8tKzAILwkwCi8EMgsx +QSQFIwswCC9hNAQzETYINQgwAy8KLAgrFyQFIyAwCS9OOAg3CiwIKxQkBRQJN2AsCCsPJgUBBSNw +AAIOELQBDykATk+IAQSHAZgBAMgTFgIMBgcCAwYEBQIIBwEMiwIJAgGrEQT2AgQCDb4QAgYFBB4I +CQIeBAoECSsoAAAEAQYCDAHQAQIFAABFAgoCBAIRBYcBAAIOMN8BLwEwMC8BMDgvATASLyYAUIIB +AQ4CGIYBBIUBBQEHAgyGAQSHAQsCBQEFAhEBBIgBCIcBBgIFAQGIAQaFARaGAQSFASCGAQiFAUIA +GiAKEwYWCAYECRsKIBQCCSED2gIKCgS/AhQCBB0EHgXSEAfPEAIEAwQHJwT2AgUCBs0CAgoDzAIF ++wIR/AIE/wIIgAMGywIF3AIBjwMGNgIMEQsDNQQ2BgISAgg5CDoKLxIRJgAABAEGAg4B8wICBQAA +dAI9AbMBBAYDJgAAggEECgEEARgGBAUFCAcHDAoEAgsLBQ4DDRMOBAIIAQUNBhIBAgYTFhYEFSAY +CBdCAAIOMOIBLwEwMS8BMEEvATASLyYAUIIBAQ4CGIYBBIUBBQEHAgyGAQSHAQwCBQEFAhMBBIgB +CIcBBQIFAQGIAQaFARiGAQSFASSGAQiFAUcAaiAKEwYWCAYECRsKIBQCCSEDigIKCgTvARQCBG0E +bgWCEAf/DwIEAwQHdwT2AgUCB/0BAgoD/AEFqwITrAIE/wIIgAMF+wEFjAIBjwMGhgECDBMLA4UB +BIYBBgIWAgiJAQiKAQ8vEhEmAAAEAQYCDgGAAwIFAAB0Aj0BwAEEBgMmAACCAQQKAQQBGAYEBQUI +BwcMCgQCDAsFDgMNFQ4EAggBBQ0FEgECBhMYFgQVJBgIF0cAAg5glAVfAWA2XyYAUHgDDQRRAQwC +AwEFAg8BBQIKhgEFhQEIhgELhQELAQUCEAEGAgMBGgILAQICBAEXAhOGAQuHAQMCJwEEAh2GAQuF +AUYBEAIOhgELhQEYhgEIhQFxALoBIAIJDg0eChcZBg8EBwIJvhENvREUAQkCBQgMVggCD1cDCAUH +BJABCAoEmQEDpA8FoQ8CAQUECKABBZsBCskBBcoBCMkBC9oBAkoDAwMKA7wOBc8OEKgOBp0OAzIL +7A0Ppw4ICgO0DgK9DgS+DgoIDcUOCxUDKgNPAtkBC4ADA30KJwRGAwoDTQMBCQIHHQQeAgIDQgcK +B0MK5QEL8AEDAgcVA0QDRQNQAzUCSAY/AiQIBA4YBQUFCwY8CQIEAgM/DqkCC64CAQQIBAMGCAIE +vQIIvgIFBgYCCgURlQERDRQDJgAABAEGAg4B4gUCBQAAUgK7AQGjAgIaAWUEKgMmAAB4Ag0BUQYI +AQQDAwgFBw8KBQkKDAULCAwLCwsOBQ0QEAYPAxILAQsPARADDwsUAhMEFBcTExgLAQMVJxoEGR0c +CxtGChAJDh4LHRggCB9xAFAYARACDQEOAgMBAQIFAQoCQwDWBRQGBLYLDAwCAgLDCwoFA+ECCroO +BNELA8gLAccLBawLCqULAgIOBAoRKQACFsACpAK/AgHAAt8EvwIrAFAyARYCAwEZAjaGAQSFAQWG +AQSFAQWGAQSFAQUBDwIPhgEEhQEFhgEEhQEIhgEEhQENAQQCEAEHAicBBYgBCIcBAwIMhgELhQEn +hgEIhQEyhgEHhQEPAQQCJwEJArcBhgEEhQEFhgEEhQGCAQEIAgyGAQqFAVQA7AUmBAQBCK4LBAkO +DAICArMLA/cCCyQEAgrYAgYKJwQJ3QUE3gUF3QUE4AUF3wUE4gUFjgsPiQsCCA3tBQTuBQXtBQTw +BQjvBQTyBQgnBbgLBNUKENkDB6IEBgIDSQNKCwQQ+QMF/wIIgAMD+AIKAgECAfsFC/4FAmIVAgsG +AQIE6QYI7AYLCggBDQIFCQVtCP0FB/4FCgIFwQQExAQCAgUCAgQKCwUCBQEFRgWsCgndCgcGEgIN +MAgqBUcIEQJcBTEFBgUlBRIGERACHwQLAh0CCa0GBLAGBa8GBLIGBRgIBQUGBRMdDgoGFgIKCgYC +EVkDHAVaBfwJCO8KBRoCXAX1Bgr+BQVaBTENGRJBKwAACQEGAhcBlwcCCAAApgIEBQNsAo0BBN8C +ARcDKwAAMgIEAg4BBAEDBgsCDgc2CgQJBQwECwUOBA0FEA8PDxIEEQUUBBMIFgQVDQIEARAGBQUp +GAUCCAEDFwwcBAIHHScgBAIEITIkBAIDJQ8oBCcnKgkptwEsBCsFLgQtggEqCCkMJAUCBSVUAAIE +YIwBXwFgCV8BAFabAQC8Ag4CBR4FAwoTEgIREiACCQIZBAofCgAADgGNAQAAPQJFAgUDFAACBHCw +AW8BcAxvAXALbwEAVs4BAOYDDgQDBgcFBgYFKAoBCQIFJwMoCxYKDxIQBwIJAQUEBwILCBoCDx0N +LQwAAA4BwAEAAEQCQwIaAi0AAgRAVj8BAFZbAOYEDgIXAQUIBQMKBBgCCgAADgFNAAA0AhgCDwAC +BFA5TwEAVj4AmgUOBAUBEgIPAgoAAA4BMAAAHQISAg8AAhAAWAoeAR0FAKwCBAwDCAPmGwHlGwQC +AQABEAAACgIBAQUAAg5gR18BYF9fAWBuXwFgEF8BYBJfFABY3AIAzgIYAgQCBAIFBgYHBSgFFwQE +DgICEQ0WBQoEBAYCCQIOFQoGAwUDHgMdBgQLAgUCBAINFAQCBQoZAggEDhEFDgUQCAIFBAoCBQEE +Ag0hBAINLxINFAAABAEGAg4BvwICBQAAXwLpAQEUAAIOkAHhA48BNgBYah4HHTIeCR35AgCsBR0s +BRcFGAUZBQIlAhMCASYHIwkCGwgO2gEJ2QFrBBMCCggLAhEHWgIgDSUfNgAALAI9ARECGwEXAsMC +ATYAAGoCBwEyBAkD+QIAAgRguAFfAWARXwFgJ18BYERfAQBYDh4BHQoeEh0CHicdFR4EHQYeBB3F +AQCCBg74AwH1AwrkAgeYAQv7AwKOBAcIBgIFBhWVBAMHCQYJHwQ6BoYBBIUBDwQGCgkCEQQXDRIU +AQIEzwIDEwvkAggCDTUYCBYEFwAADgGuAgAAoQECLgFMAiEAAA4CAQEKBAcBCwECAicBFQYEBQYI +BAddCgMCCwtaAAIEMEwvATASLwEwEi8BMA8AWIYBAP4GGAYIAgUFAxIMAgUIBwIRBxMHEwQPAAAY +AW4AAHwECgACBDCYAS8BMBMvATANAFi+AQCwBxgCBgIOAgQCCAwHAg0EJgQKAg4IEx0EAhAODQAA +GAGmAQAAuAEEBgACCmBKXzkAWI0BAOQHHgIRAgMGGQUKAzgAAAQBAgIYAWoCBQAAQQQUAzgAAgTQ +AcECzwEB0AEnzwEB0AFAzwEB0AFfzwEB0AEJzwEB0AGKAs8BAdABhQIAWDkeHR0CHisdDR4EHQwe +BB0mHgYdT2AFXyKIAROHAVaIAROHAYIBNAczBzQhMyo0BTNMNAUzTTQHMwM0JjMqNAUzRzQFMz4A +4AgOAh4GDZIBAZEBEpgBCpEBAqQBBwgGAgUGFvcCA8QBDYMDBKQDDOMBBOQBEBQNnwQJ/gQG/QQK +CAUKEgILBAeKBAIIDTMNwQQF8AQMLQcuBQkK5wEHAgUGB7gBCwkKCgwCFQIWAgrFAQcCBQYHxAEX +AhUCFwIKBgowFbUBBB4EmgEGAgQCBJ0HB6AHB50HBAIEAhICB5gHAgUTCAkGBA8IlwcFngcF5wIF +4gIF5QIFBAXmAgXpAgYCCAIFnQUGoAUFBhC9BAWcBwUMCQkFBQUQAg8IEAUGCikOoQEEHgSGAQIC +BIcHB4oHA4cHBAIJAhICB4IHAgMTBgkGBA0IgwcFhAcFzQIFAwUEBdICBdUCBgIIAgWdBQagBQUG +EL0EBYgHBQoJDQUOAg0IDgWzBAriAxIAAA4BmwgAAJwDAhMBTAIUAX0ENAMpBjEFbwQvAykGMQMX +AAA5AgECEgEKAQICKAQDBQ0IBAcMBgQFHQoJBwYIMwkcDAULIg4TDVYQEw9sFAQBBBEOFgcVBxYh +FSoWBRUFGAUXBRgKFwUYEwIGARUBBRU/HgQBBBsGIAcfAyAmHyogBR8FIg8hBSITAgYBFQEFHyIK +CgkSAAIEoAG/AZ8BAaAB+QGfAQGgAR8AWFIeER1OYAdfLjQHMwQ0JTMuNAUzSzQFM0QA6AkOAh8G +DQYN8wQKAQHoAxHlAxEIBQoRAggEB94EDg8KsQUHxAUCBwoIEvMBBB4G2AECAgTZBwfcBwTZBwgC +BAISAgfUBwIDFAYJBwoEBdkHBbYEBa4DBbEDBQQFrgMDsQMGAggCBZ0FBqAFBQYRnAMF2QcF5AcM +DQoOBQQKiQUN6gQSAAAOAc8DAACwAgQtAy0GKgUSAhcAAEcCCwIRATYBGAYHBR4KBAEGBwYMBwsE +DCULLgwFAgUNBQ4KDQMOEwIGARYNBQwFCyUCDQESAAIEcIoDbwEAWChgBV+AATQHMwM0MDMZNAUz +GzQHMwY0IDMVNAUzKACgCh0KC/MFBfQFAgcHAgQCBAUDBgyRCQWYCQYECAMJAgWZCQSeCQYEBAII +BQIODxMFFAoCBAIFnwgHoggDnwgDAggCHgIHmggGAgkFBRUFhwgFoAgFEwUDBS4GEQIGBKkIB6wI +BqkIAwIEAhICB6QIAgIJDwUVBYcIBaAIBRMFAwUuDwQKAAAOAYEDAADxAQSUAQMKAAAoAgUBIAQF +AxwGBAU7CAcHAwgwBxkIBQcbCgcJBgogCRUIBQcoAAIEoAFrnwEBoAGsAZ8BAaABmgIAWFRgBV9L +NAczBjQgMyQ0BTO9AgCECw4CCQgJAQkIDQgJFwogC+0GBe4GAgEEAgUEAgkKChACCQIElwoDmgoC +BAMEBwIEAgSVCQeYCQaVCQMCBAISAgeQCQITBQ4FCQUSCRUFEgWVCQWUCQULBQMFCgUMBQgKL00C +EQmUAQIWBxIAAA4BqQQAAOYBBC0DCgYSAiABMQMcBiQBdwAAVAIFATQEAwMUBgcFBgYgBSQGBQW9 +AgACDsABzAG/AQHAATy/ATgAWM8CAOILIgQMAhIGCAYJAgIaDRsFBBIBCgIkAgcFCQgCBAsKDwoK +EQQBAwQBxwoDxAoKDxUFEgc4AAAEAQYCGAGoAgIFAACJAQRBAgcFKwgbBzgAAOMBAgMBaQACGZAE +rAaPBAGQBDCPBAGQBFyPBAGQBA+PBAGQBIYCjwQBkATIDY8EMgBYPR4RHToeAx07HhEdEx4IHRAe +CB1mHgMd5gkeAx2JCwDuDCkiCo0ICQEB6AMR5QMSCAUKEQILBAfOAwPKBAKvCA6wCAMCBBAKSgpZ +A+4BBO8BA/ABA+8BA8kEEboGBRAGwwIIhQQIugYIRgj/Bgj+Bg9oGAoObwizAgO8AwgCAxYL2gID +8QIFRAjLCAPMCAQCAQILoxALoAwDqgUHcAePAgMFBYMEBvQFCyYIAgiRAgNvBWUH0w4E1A4dRRNm +BfcBBSILFgMECQQGBA8IBAgCBQUCBwISBAcCDBMIFgYCCAIPlQgEAQUEBZ0FA6AFAgYOlggGAwMC +BwISAgYCCAISowgFAgGfBQTIDQKhCBGmCAYBBgINBBDFAQYgGwQQGA4IBAECAgILBQIDAhIGFAIJ +AhBfEKoHBtMPFggFChkCCwQHwg8CCBMBDAIDHAoCEqkOBAEFAgcCBZ0FBqAFAogOCAEI/w0OpA4I +UAhRDO8TCPITBPETA/QTBeYBELcBCCMIBhweGB0FBgwCGAIQBAGRFBTCFAgfIiMKIgoCEQIB4Q4G +AgYCBZ0FBaAFAgYR9A4IGwulFAjCBQUEBQgQzQUIwhQIwRQDwhQWlwQF1QgD2AgEAgGFEAOSDwO3 +CwjIDQrJDQiwDAJ1A4QCBQwFuwMIqAICBAoKDHoDeRXDDAiIDQM/DSAFIAcCBwINAgUEAwIBAgQC +Ce8QCvAQAgQKIgsCBwQHAQQMBL8DA74DCAIGAgGnEQSqEQQPArMDA84DBdwBBdkBBRUEAg7uAQPF +AQPjAwjkAwQCB3YEAgECDMUSBMYSBd8ECPYDA/UDCPIDDAIHAhIGAwIBAgwGDQoGAg33EQO8EgXV +BAiUBAUMCQQFFhcCCQIGAg2lEgSoEgIECD8DNAO5BAiGBAM0AxwHKwYCCQIJKAUpBasECLIEBCAM +AgcCDbsSA9ASA+kECOoEA0ADqQUI6gQFCCQCDxADBwcIBQIDAgEEBPESBPISAhwJAg6pBQOqBQWp +BQOoBQUZBAEEBAH1EgPyEgUeAwEFGwK1AhEXVgIWCQWGAQVKCCgFgQMIcAObAgiWAgVFBwEEAhgC +AbcPA9gDCI4KEM4BBRgIOQjLCwjuCwoIBxQSEwMUAsICCBQDvwQKvAMFcA2vAgUNFAIE/QsIugsI +QAYOBSMDJCsCB7UBA7YBGE8LuQsIjgwGBQWZDA7sBzIAAAwBBgIXAbcXAgUAAPMDBLgBAw4ETAMT +BB4D8AIGNQU3CEECiwEJEwwgCxUMJAdOA5wGBCEKJgm4AwMyAAAzAgoCEQE6AgMDAgYOBSsIEQcT +CAgHEAQIA2YIAwcQCgsJSQwEC8IBDgQCCgIDARAPQhQGAgQVAhQRE80BBkYFQBgEAhECBgECGRAa +DhkcHggdBB4DHZMBIBQfUCIRAgUBEyETJggCGgEIJQgmAyUbCAMHBSoDKQMCCAEKBggFRwYIBUYs +Cis/LgQtVDAEL20yAzFTNAQzczYDNWg4BDcyOgM53wE8AzUIBSUGCAVsBggFbgYIBQsCDgEyAAIO +0AGVAs8BAdABhQTPAWAAWIkHAJAXJQINAQgSCAMFAQUOCQsDDAgQGwQZBAUIBgQFBAQBBQQKCQkL +BR8SAhR+CAIMixcNjhcOAgqFAQUESAIUQg8CBAQCAg0BBAQDAxICCwYPAgMCBwICAg0BBAQDAw8C +CAYLAgQCBnUFaAUOBQYSDiITDQMPBQ0HDQUNBw0PDQEEBAMDDAIJBgsCCAEQBQ0rBRMSBWAAAAQB +BgIbAd8GAgUAANIBBBkDKgatAgEvArgBBWAAAP8BAgkCBAP9BAACDpABkwGPAQGQASSPAR8AWOUB +AK4YGAgFBRECLQIKAQUCBAIXAhMGCgMRBQgBCwMfAAAEAQYCDgHHAQIGAABGBDcCGwUWCBgHHwAC +BDDNDC8BMBYAWOgMAPAYIBgFJAYOAzEFJAQCAQIH8xcE9hcHBAUECwYCCAYCBTYMggMS1wIEFgYM +BgIB+RgJtBcDgAQDtwIFDgGJGQm0FwOABAOnAgWoAQMCBgQIAgTnGg7oGgUCIgIBAgQCA9saBLYb +BVEMHAgCBJUbDpYbBQImAgECBAIDiRsEthsFIwwQCQIbAQMCDAIEpwMDyAMD1QMDIwn6AwP5AwwC +BgIDAgECBNMXBNQXAgQOAgMEBwILAgQCAQIE5RcE5hcCzgMGAgoCDQIFDAoCDgIF6wIExRgFzBgH +AwQCAQIT4xgD5hgMrgIQAQMCDAIBAhIFAwgBAgSbGwaiGwMqAzkKKwwBAwIMAgECBAUDCAECBO0a +BvQaBg8FEA0CGwEDAgwCCdEBBAIMAgECBJMZBowZBRgFAgrxAQPyAQkCAnoDcQkOFzYDLQIKCh8D +IAUkDAIDLAMtBQIHAQMCDwIEfwSCAQwkBQIHAgcBFwIMAgTvAgMyA8gDAwYD2wMD0AIFGxECAQIE +AgSpGgPMGwOpAQsLAwIBAgQCBJsaA5QaCxkECwMlAyYQAgQIAwsGDAWJAQOKAQXDAgPEAgkCB4sB +A4wBEbwBA6kBAw0FBQVhBAIIAgECBIMZA/wYCk0ErRgDsBgDBAcDBAIBAgf6AgP5AgzLGATOGAe1 +GAPIGAMPBUEDjRgDxhsDtwMFAgS2AwO1AwkCBAIGAgQCAQIEmRgEmhgCBA4CBAQHAgsCBAIBAgSr +GASsGAIGCQIEkgMDkQMJAgECBLcYA7gYAo4DBscDAw0DTAWQAw4CAwMEBgECBNEbA8wbBwEFDg0b +AwEEBAECBL8bA7obBwAADgHaDAAAmQoEzwIAAD8CBAFTBAkDDAYJBSAIDgcvCgQJHQwOCzMOBA13 +EAQPLhIEEUUEBQMfFAMTRhYDAgMXOBoDAgMbYB4GHYgCIAMfGiIDIYcBJAMjDgQDAyUmBCUHBAMD +CygDJysqBCkvLAQrIC4DLS0wAy8lMgMxBwACCmBqXyAAWB5+D31nAKQdGQgBAwTBFw/IFx0CBJsc +CZwcDwIPDR8AADwCJQIUAx8AAB4CBAILAyEGCQU9AAIOUJsBTwsAWnoNDw4CDQgOIQCqARgCCQI3 +AwIQAgITAQu+Eg+3EgLUEgjTEggCDxcKAABGAjQBEQIfAQoAAHoCDwECAggBIQACCjBYLwgAWh4T +AQQMECETAQQMEBEArgEdAgFNATEMggEUAgwCASwBnwEMdgoJBwAABAECAg4BVAICAAAmBC4BDwEH +AAAeAgECDAMhBgECDAcRAAIKQEU/EwBaYgDUARQCMhoKGxIAAAQBAgIOAUsCAwAANAIcARIAAgow +di8LAFowEwEEDBABDCsLARMBBAwQFADWAR0CCAIKDgGFAQExDLoBASkVBA8CByYBCQGfAQysAQoX +CgAABAECAg4BcgIFAAAdAmQBCgAAMAIBAgwDAQYrBQEIAQIMCRQAAg6QAf8DjwEBkAFfjwEdAFrp +ARwJG6wBlAEzkwG5AQCmAhgEGgQYBwkEBQoNBBIGGwgdAgkIEgkFFgwIDAIC2gUJ2QU1BgUCDwII +BAwGEQIXCBUCCAQJBAHqDjPbDgoJCQQMBhQCCjcNExEHEQ8RCREFDgMdAAAEAQYCDgHtBAIFAACN +AQKQAgEvAqEBAR0AAOkBAgkBrAEEMwO5AQACDrAB+gOvAQGwATevATEAWjIcBRswfAx7cZQBM5MB +NgIcAQocCRsMAQkeBB0BHhIdMQIwAQoCXgCqAxgCEQEFHgTwBAWFBQcEBAEHDh0EAbgCDLUCFgID +FQYUCQgMAgsCCwIMBhECCQQBlg4zkQ4LAgwKH9kCHNoCCrgDCbUDDI4BCagCBKkCAegDEuUDDAgF +ChECCAQHowEcAhSKAQqPAQolER0SAzEAAAQBBgIOAdQEAgUAAGECggEBLwKuAgExAAAyAgUBMAQM +A3EGMwU2CBwHCgoJCQwMCQIEAQEEEgMxCzAMCgteAAIOgAGYA38BgAEOfxQAWo8BHAcbswIA/gMd +BBACCAQMAhICBQINAwUCBQYSAgwCAqoEB6kELgIMAhUQIQYfAhYlAwEKEAQYBQoFBg8CDwIIAgwG +CQIMBAo9DhcUAAAEAQYCDgGsAwIFAACIAQKVAgESAgYBFAAAjwECBwGzAgACDkBRPwJACT8BQF8/ +FABa3gEA3gQYEAYCCgQHCAoCDAIRAgsPCgMEAgQGRQISGRQAAAQBBgIOAcEBAgUAADQCLQESAlcB +FAACDuAB+wHfAQHgAbAL3wEB4AEr3wEUAFwbGgIZNxYGFZQCFhIVAhYmFQMWDhU2FpsCFXAWEhUC +FiYVAxYOFW0WmAIVEhafAhV/AKABGAQDnAcCmwcOoAEFnwEPAgcGCQIF7AEGxQEJAgRVH1whfAUF +DQIFBA0CDgYEAgQEBQgaBAoVEXUbcRRyBXEMdAoCCcABErkBAsQBARYHAhACBRUJzwED0AEOXwhn +BWgHZw0HBQwPAgHiAQsKBQItAgEKDQLQAZ8BAgYFXwVrBYIBBYEBBGACIiOBAQ+CAQWBAQyEAQgC +CZgBEpEBApwBARYHAhACBRUJpwEDqAEOXwg/BUAHPw8HBQwPBAoCC4kBEpYBCgkEAgGyAQsKBQIt +AgEKDQLNAZ8BAgYFLwUcBQ0BogELCgUCLwIBCg0C0gGtAQULDwIFCgwCBQwKCQy7AQwXDDATAxQA +AAQBBgIOAd0NAgUAAEECEwEyArABASgCEAF7AkMBQALdAQEjAg4BfQIaARICTAFAAqwBAYABAosC +ARQAABsCAgE3BAYDDQYfBbABCBQHBQgMBxMKEgkCCgECHAEJCQMKDgk2Dj4C3QEPDBIFEQUSBBEl +Eg8RBRIMEREUEhMCFAECHAEJEwMUDhNMGBIXDxo+AtoBGxIeQALfAR9AEgwJDAcnAAIOQKQBPwFA +Ez8fAFzlAQCAAxgCDggGAggGAg4GEwQUAgYFpQICrgICFhDDAhPIAgevAhO4AgcRDgIMFgpREwMf +AABuAjsBFgIHAR8AAEcCAgESAhMBBwQTA10AAg6AAYsCfwGAAQt/AYABGH8UAFwbGgQZWAMMBBED +CR4EHQEeER0yBEEDDQQfANwDGAID4gQE4QQeAhAEGAIJDAkVDBgRjgEJqAIEqQIB6AMR5QMNCAUK +EQIIBAejAQ0RBRIUAg8PDJoBDaMBCwEUAAAEAQYCDgG1AgIFAABbAhMBmAECOAEUAAAbAgQBWAQM +AxEGCQIEAQEEEQMyBUEGDQUfAAIOgAG9AX8BgAE6fwoAXpACAEoYCDQCCQELAg0BBQIEAgoECQUF +ChcCBQYREgcCChcRDAUBERcTCQoAAAQBBgIOAfMBAgUAAJwBBDwBLgEKAAIKME0vATASLwcAXiAb +AyYVCTkAgAEUAgwPA4wUAgYTgRQPBgcCCgUSAwcAAAQBAgIOAVsCAgAAZAIGAQcAACAEAwEVATkA +Ag5whQFvAXAPbwFwggJvRgBeIAUDBhAYEReoAwCUAR0CA4IDA4EDBiAJAQG0BxGxBw0GCwMNBAME +AgUSBgUEDwYEAgwVDRkKAQUBBQIqAlsGGwYgBAsCFhlGAAAEAQYCEwHKAwIFAACsAQT6AQNGAAAg +AgMBEAQRA6gDAAIEkAHOAY8BAZABIo8BAQBg9gEAKg4aBRctAgkCBgMFBhsCGQQLAhsCGQQMEwUY +DwIPAAAOAegBAAAyBCICjgECFAACDoAB7QN/AYABEn8eAGA+dgl1AnYFdRF2CXXEAwBQGJABClsQ +DAzIBAnDBALEBAW/BAkECLwECbsEAgweDAIGBREFDiMEBQwKCwUDAgcmGAoXAwwKAioKDAkCCicQ +EAYGBDQGDgIxBAodEn0eAAAEAQYCDgGPBAIFAAClAQK3AgIWAxYEBgMeAAA+AgkBAgQFAxEECQPE +AwACClBxTx8AYCp2FnVaAIgCFAIRBAEEBLwDDgoIwQMFAi0GChUeAAAEAQICDgGBAQIFAABcBBYD +KAAAKgIOAggDWgACBEA5PwEAYD4AqAIOBAoBDQIPAgoAAB4EEQIPAAIOYGBfAWAoXygAYL8BAMoC +GAYFAw0EIQIGBhQGCgMWBRIJKAAABAEGAg4BogECBQAAJQRAAxYGHAUoAAIOQKIDPwFAdT8/AGRA +HQEEDBrNAh0BBAwavgEAngEYAg5SBQEFBgUBBQUFOQFRATEMhgESAgoCGgIKAg4CECAHAgsCCwIH +BBsCCgEEBAgECAIIAggHIAIKAhMCCgIKAw8CDQINAgcCARcBnwEMugEKEQoRFQERAi8pFg8/AAAE +AQYCDgHGBAIHAABIAtoCAoQBAz8AAEACAQIMA80CBgECDAe+AQACCjBpLwgAZBQdAR4HGQkaMR0B +BA4aFgCUAhSxAQG2AQfnAQnqARICBwIHAgsGBQEBQQGfAQ7kAQ8RBwAAHAJCAhYDBwAAFAIBAQcE +CQMxBgECDgcWAAIOYFlfAWB4XwFgLV8XAGalAgCCARgCEQoJBAcCBwIGBwUKCwQSGQUeCAIZAgcG +BwIJAQUEDAQFBgUFBwIGAgcCDTEbAhIFFwAAUQQvATQELQEtAxcAAgowQy8IAGhVALICFAoKCBUG +EQgKHwcAAAQBAgIOAT8CAgAAHgIwAQcAAg4wqwEvDABoxQEA5gIYBC8CDgIQAg4CEgIkAgcCChEL +AAAEAQYCDgGnAQIGAAAkApYBAQsAAg4gRR8BIFIfCwBoGIABBX8UgAEEf3wAoAQY9QMF/AMNAgf9 +AwT+AwsCBQIPBAsCEwY1FwoAAAQBBgIOAZQBAgUAABgCjwEBCgAAGAIFARQCBAF8AAIO8AGoAe8B +AfAB8ATvAQHwAbcF7wEC8AGJAe8BJgBoGE4ETQFOD00lTg9NBE4TTRFOJU3tAVYSVeMBVhZVAVYS +VQuGATiFARNWEFWbAYABFn8dhgEwhQH1AU4ETQFOD004VhBVAU4rTQhOAk0ITgJNC0ANPwFWEFW6 +AQD4CBjRAQTcAQHZAQYCCdoBJAgB1QEPzgEEywELBAimAQQdBRgHKgHPAQ8CDgQIzAEKIAZLAx0n +CAUKEQwDSgJTBgIJCAVKAksIAiACBgQTBQNKAkkISgJJBUoUQwQFBUoCXwVgCAwB8QcS/gYjCAoK +EmYCYwYCDWICWwcCHgIGBBZUCQwHCwkMCw4QDgIJA0AT0wgWoAgBnwgSpAgKAgHcBTECB9UFEjUB +/QYQgAcKNAwCGQwFBCAEEgIKBgYEDgILAgzpCRbuCQUCBwIHAgkCAbQFMK8FHQQdDAUEBgQMAg4K +CgIHAQUkAbUHDgIiAha2BwGIBwoCCocHBQ4FDg4IDO0DBPYDAfUDBgIJ+gM3FgH/CBCCCQGFBBQC +DwQIjAMIiwMCjAMIjwMCkAMIfAICAb8GDcYGAZEJEJQJC90BCAJvAhJ7JgAABAEGAg4B8wwCBQAA +9QICJAEOApsCAS0CHgEMAtIBASwCegFcAl0CawMKAgYBDAKZAQEmAAAYAgQBAQIPASUEDwMEBBMC +DgUDCCUHEAZDBQIGFAUCBkQFAgYIBQIGBQUUBgkFAgYFBQkKEgI/CwIMEwsCDEELTA4WDQEQEg8L +EjgRExQQE5sBFhYVHRgwF3YaRhkBHBQbJB4EHQEeDx04IBAfASIrIQgiAiEIIgIhCyQNIwEmECW6 +AQACDkCHAj8BQBA/CgBoWikWKgUpRioDKU4qCikQKgoApAshAhQCEQIKAgr3CQUCEfYJBeUJBgYT +AhwWEcgJA8cJJgoowAkKvwkQtgkKAAAEAQYCDgGTAgIFAAAoBOQBAw8CCwEKAABaAhYBBQJGAQMC +TgEKAhABCgACDqAB+ASfAQGgARqfAQsAaB9WC1UgVhBVHyUKJilWElUxVhFVBoABFn9ThgEwhQGs +AVYQVS1WEFUUAIwMGAYH0QoL5AoCqgEdCgGdCxDqCR+xCwqMFAIGE98IEw4B8QoS+AoKAh02CQwB +wQoRxAoFDgHhDBbkDAcCBwI7AgkCAb4CMLkCHRwFAjcSBwI7GAsIBQoBuwsQxAsKChMGBQIK6QEB +6wkQ7gkKGwoAAAQBBgIOAY8FAgUAACUCVAFAAjUBDAILAQECaAEsAu0BAQwCDwEKAAAfAgsBIAQQ +Ax8ICgEVBRQKEgkxDBELBg4WDVMQMA+sARIQES0UEBMUAAIOcJ0BbwsAaHcMIQseALgNIQIlBgUF +HQIIAgfRCCHSCAICCAgKDwoAAAQBBgIOAZkBAgUAAGsENwMUAAB3AiEBHgACGcAGuw6/BgHABvEK +vwYWAGiIAYABEjEETQGAAQkxF02XAoYBGIUBeIABG39SKRYqBSlHKgMpHyoDKTEqhQIhAQQMHhMh +AQQPHgUKBgkKChUJaQq3AgloVhNVAVYQVQFOLk2YCj4EPQE+Gl8BBAweHCkKKi0AmA4pBAXlCgkC +G+4LCe0LCAIW5goOAgH3DRL+BgT+BgH7DQn+BgYCEf4GPwIIAgsCEwIPAg8OQxYdKhMCFwQJAgFE +GDs4BA0KDgIOBhYGAYEPG4QPGwQZAgcDCgYNgQ4FAhGADgXvDQYGEwIdFhHMDQPLDR/UDQPTDQUK +LMoNDQINAhkCEgIOBg4IBwcJBhUCFwIHAg4GHAIcCAsECQIGCAGNDwExDMIPBgIMAgGTDgGfAQ++ +DwXlDAbyDAkCAcEMEAIFxAwdCgUIBQYhDCAOAesMCwoFAi0CAQoNAuwB3AwJAhkEGAIFBCeQCAKv +FhPgDgHfDhDmDgHpCRcCDwQI9AkQugcUAgoBAwInAgQCBgIIAgYIFaMIDqYIGaMIFAFqBxEIGwYH +AkkIXwcJCAUHGAIJBgcDGwQKAwUKIAJ4DlcNJwIMBB4IBQcFAgwCGwkFDgUDBQotAgwCDAUWAgwB +FgIMARYCDAEWBAkDNggJAhsED/MPBPYPAfMPEAIJAgFAAZ8BDNYQBeQHCg8N/RYKqA0XfRYAAAwB +BgIXAa4ZAgUAAIgBAqYBApkBAg4FFAZ2ArAEBwkI+AEHTAiWAgcLCEgHtwEIGQJEAc8BAhUBvQIC +FQHsAwcJCA8FMwEWAAAuAiQBCQIeAQ8EEgIEBQEECQIXBZcCCBgHeAobCVIMFgsFDEcLAwwfCwMM +MQuFAg4BAgwPExIBAg8TBRYGFQoYFRdpGj4C+QEbZh4CAhMfASIQIQEkLiMQHnUdDh4ZHewIJgQl +ASYaAgECDCkFHhcRCgstAAIKIBwfCABoLgDADhgCBRAKEQcAAAQBAgIOARgCAgAAGAIPAQcAAgow +Vy8IAGg5IQsiJQC8ERQMAgIPBBMCAb0PC8YPBg8OFAofBwAABAECAg4BUwICAAAfAjkBEQAAOQIL +ASUAAg7wAbgI7wEKAGijAVAEAQRNAU4GTQFOBAIETypOBE0BTgZNAU4EAglPLlAFTwGAARZ/LIAB +BX9HgAEWf6YBJQomOU4mTQFQBk/kAgCeEhgCCQo3Ag0COAEFBgGfDgSWAwSOCwGNCwaOCwGLCwTL +AgTaDQwqHrkLBIAMAf8LBoAMAf0LBMsCCcwOEAQNChGBDwWKDwGVExaYEwwEEgIOnRMFqBNHpxMW +9BMIAgwCBgIMAhMCBgIPAgsCBgIMChMCFRQLCAjlEwqMFAIGEysbCAmpDQ8CDwQIpg0BgxAGiBAK +KwQCBQJzBBN7BAIEAkUCExERCTkCF5MBCgAABAEGAg4BswgCBQAAXwRjAnACygIHcAYfA7sCAQoA +AKMBAgQCBAMBBAYDAQQEAgQFKggEBwEIBgcBCAQCCQkuDAULAQ4WDSwOBQ1HEBYPpgEUCgEVESQW +JhUBGAYX5AIAAg6QAeMCjwEBkAEUjwEKAGjYASEBBAweAUAPEAMPPj8BIQEEDR5LANITKhAPAg4G +BiIGBAoCFgMXIRsCEgwgAgGZEwExDM4TAahGCqAHBbldA8BdCwIMAgkEBwQHrwcKAgapRgGdEgGf +AQ3IEw8MFAIKLxQVCgAANAQ1AjICIQJTCTAKEAMUAyMBCgAA2AECAQIMAwEGCgIFAgMBLgEQBQEM +AQINDUsAAhagAvYDnwIBoALKBZ8CGABopQIMIwuCAYYBM4UB8gUAlhUmAgkOCAsFDAgFDQYHBjUK +CQYFBA4UHxwHKgVFKQwJBgoEDgYHgxEjhhELMA4KHgIEAgUGEgIKDA4CDgQJAgHTBDPYBBBbBQJi +AhUGLgQRBi4EFAIWCR0JKDcRE7kCAhYNEgsYAAAJAQYCFwHECQIFAAA3ApgBAjoCwAEFLwIFARAI +xAEFbAY0BeYCARgAAKUCAiMBggEEMwPyBQACDkCuAj8BQDY/AUASPxQAaCwhAQQMHmchAQQMHgEK +CwkOIQEEDB4MIQEEDB4DIQEEDB4pIQEEDB5nAOAWGAYOBQUOAYsWATEMwBYHAgoCCwJCBAgCAZkV +AZ8BDL4WAdkVC94VDQYBpxYBMQzcFgsCAasVAZ8BDNAWAiABzxYBMQyEFwkCBwIYBAHZFQGfAQz8 +FgopBgELCAUCDQoFAgUCCkESBxQAAAQBBgIOAf0CAgUAADQCkgEBCQIgAQkCLQEJAlgBFAAALAIB +AgwDZwYBAgwHAQoLCQ4MAQIMDQwQAQIMEQMUAQIMFSkYAQIMGWcAAg5gyQFfAWAOXwoAaCYhAQQM +HhkhAQQMHpcBANYXGAYNDgGHFwExDLwXGAIBixYBnwEMrhcTAgQCFQMYAgkBBAILDAsCDgIKDw4b +CgAABAEGAg4B0wECBQAAHwI1AowBAQYBCgAAJgIBAgwDGQYBAgwHlwEAAg4g9QEfASAsHwoAaCsh +AQQMHlwhAQQQHgEdDgMCIjchAQQMHkAAmBgYBA0CBQ4ByRcBMQyAGBICBAIUAQQCCQQkAgHXFgGf +ARCAGAGRGA4yAuIXCQgNCBMHBAgJBAH1FgGfAQyYGAoNCAIEAhMBBAIJNwoAACgCAgEJAmABCQJQ +AQkCOwEKAAArAgECDANcBgECEAcBDA4BAgk3DgECDA9AAAIEMIABLwEwzQEvATCsAS8CMB8AaKAE +AIYZEwIJGwMgCh8DIAclBwQFKAMlAyYKJQMmBwIMAggCJQEKEAMFEQIEAwY3BAIKAhACCgEDAh4C +BAIFAgoCCggMAhcoECkLCQgFCwEUAwMGCgEDAh4CBAIJAgoCBggMAhcODg8LCQgFDAAAEwGNBAAA +jQEELQOfAQTHAQAAHAIDAQoCAwEHBAwDAwIDAQoCAwFoAokBARACHgKCAQMOBB8AAg4QqAMPDwBq +HnYTdSdsC2sSdhd1JWwPa4UCAHIWEgiwBgcCBQYHsQYYAg4PAeoEC9cECQIJpAYLAgUGB6MGFgIO +HQHqBA/JBAkCCSA8AgsQDgQKAiYGCgIQAhUCFQITAglzDgAABAEGAgwBpgMCCQAAHgITASYEAQIL +BRIIFwckCgECDwuFAgACDnBSbwFwW28OAGrKAQDuARgCEBQJAiYcCjEIAkECEgUOAAAEAQYCDgGt +AQIFAABSAmoBDgACDlA0TwFQCU8BUL4BTxQAalM+Cj3CAQCGAhwCDQgJDgcCChUKCAUCAeALCuEL +DAQLA4UBBhIRFAAABAEGAg4BggICBQAAdAIbAnwDFAAAUwIKAcIBAAIO8AGXA+8BAfABtQLvARwA +ajd2FXUpdhd1iAE+Bj3dAwC6Ah80CC0Q9AQJAgUGB/kEGQgQ6gQLAgUGB+8EGQgEAgwKCQoSBAwK +GggdDgHmCgbjChUCDgpCNR1+CmMHBBEtBQIHAiADCAQIAwgJCQI6AQkCCAEFAgcBDwkJAjUBCQII +AQUCBwEPBxwAAAQBBgIOAdoFAgUAANIBBMoBAwwEPQFWBG8CMQccAAA3AhUBKQQXA4gBBgYF3QMA +Ag6AAekBfwGAARV/AYABEn8KAGowPgY99AEAnAMYCgQJCAoHAgXMCgbLCgsLCgoFAggCAgISAgwU +DRQCEQIIDgoNAw0CDAIULQcyAgIUBAoZDAIKBhIxCgAABAEGAg4BjQICBQAAZwRFAiACHQUFASwC +BgEKAAAwAgYB9AEAAgpgJF8BYDdfPwBqUGwDa1IA8AMUEgwCBQIKCgcCBgEDCgoPAwIE6QMD+AMK +CQofPgAABAECAg4BigECBwAAWAQPAz4AAFACAwFSAAIOkAFujwEBkAGHAY8BCwBqGCMBJAcfBSAY +IwEEDF4GPUhOCE0NIwEEDCAUPh89ASMBBAwgFACoBBjFAwHKAwf7AwX+AwwCCwIBzwIBnwEM1GEG +4V0CChoCBwkKCgUCDAQJBgE5CC4MFAHxAwExDKYEFLxcA5oBAgIOAgzXXQH1AgGfAQyYBAoxCgAA +BAEGAg4B8gECBQAAIAJTARYEIwMJAh4BIwIPAQoAABgCAQEHBAUDGAYBAgwCBglIDAgLDQ4BAgwP +FBQDARwRARYBAgwXFAACFpAC9wGPAgGQAqAEjwIeAGqgAyMBBAUgDx8KICojBSQWDAQLoQEMBQue +AQDkBCYUBgYpAgoBBAIRCAIFCwYzAgoRBAgFAgUGBWIHXQ4EBgIEBQUEBwICaBBhBBQDEwYLAw4R +EiYIFRsFJiMRBSIJ6wQBMQWgBQ4kAbEFCqgFBVEFAgUSBRIEEQUbBQgDFAXJAwXuAwgCBg4IygEE +yQEFDwUQDAoKBiEPBQ8FIAIPBSEGBDgCESYFMy4CEQsNIQ0BDQcRAQkZHgAACQEGAhcBoQYCBQAA +oQMEEwMGBkIFJwQyBNkBBx4AAKADAgECBQMPCAoHKgYFBRYKBAmhAQwFC54BAAIOsAFArwEBsAEJ +rwEBsAH3A68BAbABK68BFQBqigExBTIJMQUyjwGEARmDAa4BPgw9MIQBGIMBFIQBGIMBHgCGBhgG +DQYXbgdZAhEKBQq4AQqfAQcCBwIHGgcZAwIIwQUFwgUJwQUFyAURAwcmBCwKNwMZAhwFBCQGBwQV +BAMECRQJXgJdAgwFBwG8CRmzCQ9SBFFADA8CFAEFAgIBBwIHAQMEAgIFBg8QCQIBowIMpgIKgQED +qAECJQUYFw4CawICAdQJGM8JCmYJAgHoCBjjCArRARQAAAQBBgIOAfQEAgUAAL0CApYBAQICHQEI +AjQBFAQFAycEBQMeAACKAQIFAQkEBQOPAQYZBa4BCAwHMAoYCRQMGAseAAIKMCQvCABqNgCABxwC +CQYKBwcAABwCEwEHAAIOkAFOjwEBkAHCA48BApABxQGPAR8Aam1+Fn0lfgV9vQEnCiYZAit+En0S +fgR9pQIA8AcYBiYEChALAgoVBT4FCwURAesHFvAHEgIT8QcF/gcYAgwIFAIPBBQQDAIsCBICFgwC +7QcKjBQCBhejDA8KHL8IEsYIEAICxwgExAgIBA4CCQIMAgsECykEAgQCRgQWPwQCBAJFAhQvHwAA +BAEGAg4B5wUCBgAAbQT7AQNQAl4BEwa9AQUfAABtAhYBJQIFAb0BBgoBGQMrCBIHEggEB6UCAAIK +MFIvCwBqFCMBJAcfCV4PPRIjAQQMIBQAhgkUowgBpggH1wgJzGEHAgjzWAUCDAIBqwcBnwEMzggK +CQoAAAQBAgIOAVECAgAAHAJBAQoAABQCAQEHBAkCDwUSCAECDAkUAAIOYLkBXwFgMV8BYBxfDwBq +GCMBJAcfBSAiPhcQAw8KEAoPARAMTSNOAk0IIwEEDCANPiI9ECMBBAwgHQCgCRi9CAHACAfxCAX6 +CA4KBQIOAgH2VwsCDJ1dA6BdCQQBo10KqF0Bp10MsgUMAhICBbUFArgFBwQB6wcBnwEMjgkMBgHr +AyLuAw8rAckHAZ8BDOwIDw0OAAAEAQYCDgGEAgIJAAAgAj4BWQIRAR4CFAEJAhQBDgAAGAIBAQcE +BQMiBhcCAwEKBAoDAQYMCyMOAg0IEAECDBENFCITEBYBAgwXHQACDmBuXwFg8QFfFwBqNCMBBBAg +WT4KPQg+JD0ZPhgQAw8KEA0PARAJTS8jAQQMICAA8gkYAgoBBRQMBgGpCQExENYJCgIWBgIPDAIK +FgUGCw4JGwUIA/xWCoNXBwIB1FcBAgUCDgIFAgvVVxEaBwwB8lYLAg2dXQOgXQkEAaNdDahdAadd +CbQGBQQMAhECDAQB7wgBnwEMkgoKYRYAAAQBBgIOAeYCAgcAADwCYgHCAQIPARYAADQCAQIQA1kG +CgUICCQHGQoYAgMBCgQKAgMFAQYJDy8SAQIMEyAAAhnwB8ID7wcB8AegBu8HAfAHiQPvBx8AakY+ +Bj0paCNnxAI+Cj31AgYfBRQGGgUPBgcFBGgHZxdoBmcWaAhnE2gUZwJoA2cdaA9nAmgDZyoRFRJu +Bi0FGj4KPYwBPgo9sAEA7gopBAgBFYQDBvkCCRIUCAzSBiObBggtBAoJBQUOCCQIJRwCFxoLAhwQ +CAdBCDIMFwYZaQUCEPACCvUCdAIRbCUCCwYeHAgXEwYeEggHEwImBg0FBQYJBAwQAbsHH8AHCwIJ +hwcLAggCB4wHBQYEAgUIAfUJB4oKBPAIB+sIBjEECAUICJoJBuMICAIEBgVNBaYJCMkIBQ0KTQJQ +AtYICQIFAgbXCALcCAPbCB3UCAQCBQIG0wgC2AgD1wgbBg4CAbIQFY8RBW4MFggCBRsNICIGEAMR +iwgJBggCBAICBBACBAIC3AURGwMMBu4CCusCewIR6gIKgQN6AhcFHwAADAEGAhcB9gwCBgAArQEC +5QECFwEjASYGHgKkAQKhAQIFCxMMIQtbDi4NLwwHCyoMEwsNDCIDNAccCEwHFgYiAl4HFgYeAmcH +HwAARgIGASkEIwPEAgYKBfUCCB8HFAoaCQ8MBwsEDgcNFxAGDxYSCBETEhQRAhIDER0UDxMCFAMT +KhYVFW4KLQkaGAoXjAEaChmwAQACCkApPx4AalEA0gscAgkCDwMdAAAEAQICDgE7AgIAABwEGAMd +AAIO0AG0A88BAdABCc8BAdABZc8BKABqIHYDdQp2A3WqBADADRgKCLkCA7oCCrkCA7oCCAINIgU0 +BTMFHxIDAwQGAQICBlQHJAodBA4FAQQJBwIhCAQSBQ8FEAURBAIiEAoHEAgVewUCBBwJAgkCBQIT +CgUGBTEFQgVDBi4JBhQxBUIFQwo6AjcCOAIKCQoHBApMCiEJBAQCBAIEBAcCBg0KHBobBQQFAxWF +ASgAAAQBBgIOAb0EAgUAAHkEPwI5AmIDNwYvCVAMKQsoAAAgAgMBCgIDAaoEAAIOoAHfBZ8BAaAB +2AGfAR8Aar8BCgUJTAoFCRkKEwkCChoJ6gEKBQmiAQoNCeoBAKoPGAIOCAsKBAwJDwQUCRACCwd+ +EIMBBxIOMwUKBTgFZAVxEB8FiAEFZwUgCEcDmgEFGAOXDQWuDAlHBEgmagUhBTsKAgW7DAWuDAVF +BbABBRcFCQX1DAMCBQYLugwCsQwIAhKwDAQCBQIFAgUICQIMIwVFBbABBRcFCQUdAwgFBwkICwoW +AgyDAQmGAQIqBSkCAhEGBY8BB5ABBAIIAQkGBQQOCQUiBRcK/wwF5gsFmAEFAgIYCBcEAQdPBVIF +AwUEBZkBBZABBSIFLQWBAQOwAQUFBTUCPAOvAQOqAQ4CDKsBBK4BAgIPBA0ECo0NDYAMEoYBBXkF +WAVjCSsELCYCEgIQBggCInQFqQEFsAEFdwVBFwMfAAAEAQYCDgHHBwIGAACLAgQjAzgE7AIBEgES +Bn4BRwILBR8AAL8BAgUBTAIFARkCEwECAhoB6gECBQGiAQINAeoBAAIOUKoDTwFQH08eAGqKAQoF +CQ8KEwkCCh8J5wEKDQkwAI4RGAIOAQUaBQcEAwcEDAIvVAVPCgIF1w0F0g0KVAWlDgMCBQYL4A0C +1w0NAhLWDQQCBQIKAgUGCQIKIwpUBScDEAUPBQoPAhACDAIHOwVUBRUFDAsGEgIMAgQCCAEJAgUB +BQMFFhKlDg2wDRIDHgAABAEGAg4B2QMCBQAAhQEEGQM9BOYBAhcFHgAAigECBQEPAhMBAgIfAecB +Ag0BMAACDsABXb8BAcAB3wG/ATwAaocDAJ4SIiwIAwUBBxcPBAsCBAIEBwMKBQMCKAofAxgCAgQZ +EAIGBAcCBQcFBgoDBAgTBCwCBwEIAgIDGgIIBwUPBSIEFwUOCAEFDgUZChs8AAAEAQYCDgHqAgIF +AACzAQSYAQM8AAIWoAKDAp8CAaACsQKfAgGgAtoBnwIfAGomEQoeER05EgIMJgsnDAQLCwwECwIM +BQtNEQ0SehEEEhERDRIHES0SeAoWCQIKEAkdCggJJQoOCREREBIfAOoSJucNCQEB6AMR5QMRCAUK +EQILBAfcDQK9CCbACAECBAIJNAhSCH0J2QsE4AsGDAXNCgTOCgLdCAW+CBY2DREEAhAyFscLDc4L +CwUFDAoMCQgQGxIwEwIdNQX3DAT4DAU+CD0E9wwDBAqMDQUXAvcMBQIJAgWdBQagBQUGD+wMBRgF +FxA+CD0FPgQCBAIQTwsCBwwKCwUVGP8QAwIFBA78EALzEAcCCfIQCAEFBAsDBf8QCKYRESUFJgUR +BQ8F9RAOxBAR+w0Q6A0fAAAJAQYCFwGZBgIGAACLAwRTAyoCfQE7BjwCKgcfAAAmAgoCEQE5AQIG +JgUnCAQHCwoECQIGBQVNDA0Leg4EDREODQ0HDhMCBgEUDXgSFhECEhARHRIIESUSDhERAhABHwAC +DqAB1wOfAQGgARKfATwAaooCDAQdHh4EHQgeER0DEugBAKQUIqYBCiMHQxYCBQQLAgcODQYEAgIZ +Bx4EXwNgCh8DAQMkAggDKQMBBSwHBhcSEBcFKQUBBTIFNgVnBUYFGQUiDAIJCgGRDgS5AwgTDx0G +BgHuGgSdHQSyAgTsGhALAd8aA/gRAi8FKQUBBTIFNgVnBV4FCAsCGjsFKQUBBTIFNgVnBWgFOwUp +BQEFMgU2BWcFUgUaClMSVTwAAAQBBgIOAZcEAgUAAMcBBEIDiAEESwMWBgYFPAAAigIEBAEIBA8C +BwIEBAQFBAIQAgEDAwfoAQACCnBIbxIAamQA2hUUAhECFAIQBAoJEQAABAECAg4BTgICAAAZAjAB +GwACDpABogKPAQGQAQmPAQGQAWmPAQGQAQmPAQGQAYgCjwFGAGoyEQQeDgsDDA0dBBIXERQSBhEI +EgcRBB4JCwIMKgsCDA4LFAwUCw0MAws8CiEJAgoQCScKDQINC7QCAPYVGAQUDAUFAfsRBK4aDqsI +A6wIDAsBkR0E8BQODAnREgYGBAIKyhIG9xEIihIGBgH/EQTEBgnCCwKDDwTAAwGBAhSEAhHECwLB +Cw7CCxS/CxTACwUCBAoE8w0D9A0CAgwCChsKMgVPFcsTBQ0IAgUED6IUApkUBwIJmBQIAgsECk8K +zxMNsggNmgsKDHcCGwIgAgsCFR0SBUYAAAQBBgIOAeAFAgUAAMwCAhMBNwQFAxwEgAIDRgAAMgIE +Ag4DAwQMAgECBAcXChQJBgwICwcOBAIJDwISBAEBBBQDEQ8CEA4PFBAUDw0WAxU8GCEXAhgQFycY +DQcND7QCAAIO0AHrBM8BAdAB2wLPATMAajIMAQsKDBILAgwnC2wMCQudAQwIC1AMBAtCDAML3QMA +4hYgBBLrDAHqDAr7DQeYAQvkDALRDAcICAIFBhPCDAwCNwIOCBvFDwnGD50BhREIiBEWAjqJEQSO +ETkICZURA5gRCQgODAgHCyMbAgoiDwgjBAILBRAEAhsGggECEgIfBA8XCAcMIAUPAhQCAhsEClUy +AAAEAQYCDgHrBwIFAACEAQJFASQC0gECEAGGAQRiA/oBBBsFPAAAMgIBAQoEBwELAQICJwFsBgkF +nQEICAdQCAQHQgoDCd0DAAIEMN0BLwEwHABqHhEBHgQdEB4ICwMMDR0IHgkLAgwjCwIMDgsTDBIL +LAwKCxIAyhcOAg0KAQIC5xMBugMEuQMBEw4oAa4aCOcGA+gGDAsBkR0ElAMExAYJ8gwCsxAEwAMB +gQIShAIM9AwC8QwO8gwT7wwS8AwEAgMGDQIHAgcCCoENCuIMEgAADgHwAQAA5wECFwAAHgIBAgQB +AQQOAgECCAkDCgwCAQIEAgQCCRECFAQBAQQSAwwRAhIOERMSEhEsEgoREgACDsABnwG/AQsAargB +APgXGAYoAgQCDgMKAgUIDQIMAhkNDw4CBAoXCgAABAEGAg4BmwECBQAAaQQlAhYFFAACAQBsAQB0 +AQABAQACCjA1LxsAbFoAiAQUAggGDQoIBgUCCxkZAAAxBBADGQACDpAB6gSPARcAbLYDMwo0zwEA +rAQYAgUGBQQFEhECBBAZAhgCGwQcCgYECgQuBAsICQIOAgsIFwICAggDCA4NBA0CEQQMAgwCCQIM +Ag4JGLkECroEFgIKARYECgMWBgoFTw4KhQEWAAAEAQYCDgHyBAIFAAC9AgJhAtEBAyAAALYDAgoB +zwEAAusBAGzOATMFNAszBTQIAOIFAgwEAgQCBAgEFQIsDysCLCYoAycaBAUDBQQFChkGAhQGAgc4 +CR0DAgMcEQIIAQeTBgWYBgcBBJUGBZoGBz0BAAHrAQAAzgECBQELBAUDCAACDoACmwj/AQGAAg7/ +AR4AbI8CfBJ7EHwEe6EGAIQHIgIGsAVqAgz/BBH+BAP9BBv+BAOxBQgyBgIxjQcSjgcQjQcEkAcM +CgUCOQ0MFDYIEAQNHwcUIAgGCgkECQQJBC4GCQLZAQchCMMBGBB5DwsdAAAEAQYCDgG5CAIFAACP +AgLOAgLDAwM2AAAoAnYBEQIDARsCAwE/BBIDEAQEA6EGAAIOUE1PAVAJTwFQCU8BULEBTwFQCU8B +UAlPFABsfWoEaQZqOWkDagxpegCaCBgWCggJCB0GCAICBQoHCgcKFg3LBgTOBgbBBg0GBwMJAhYC +BrwGA7kGBA4IrgYEAgIEGgIKCwUSDBEFBAQOBQIKBgoPCTcUAAAEAQYCDgGsAgIFAACBAgQXAxcE +BgMUAAB9BAQDBgQ5AwMEBAEIAXoAAg5ggwFfAWBiXwFg0AFfAWAvXwFgSF8BYBJfIwBsJgMFDB8L +BwQCKQomFQQSKQIqDykQAgsBBCoDKRoqhAF8FnucAUwFS2gA6ggYAg6MDAWLEB+MEAeDDAKBCAqM +FAIGE48MEp0IAqgIDEID5wgLAgUSCwsBAgOICAOFCBrcCA83BTgCFwIHDAoLAgUIGwcDFAYIHV0K +HAXbCBawCQkCNQQRAgwGEggFBwIbCwIMBw0qBOMFBeYFDwIJAgwED4sBEgMjAAAEAQYCDgHXBAIF +AADVAgRgAmECNQEGBSMAACYCBQIfAQcBAgYKAxUBEggCBw8IEAILAQQHAwgaBxYMAgsMDC4LMg4W +DZwBEAUPaAACDqABxwSfAQGgAZACnwEgAGyUAgYaBYEBBhkFvgMAlgoYDgYCDqgBA6cBFwECAgJM +CkMCAhMGDAIGIgMBDAIGGAMXAgYMEgUOCQJhBATLBRrOBQUMCwIDCAcGBwUGDgMUBBMJIgUPCQIE +AgsCBSQFAQUZCQILGgUBBasGGZwGBAINDgQUBwIHEwMIBwIHFAkHAwIDDQIQDQQcAgcICggFAgqn +AQYsBQjuAQIXkQEgAAAEAQYCDgHpBgIFAACCAwQtA5gBBp8CBSAAAJQCAhoBgQEEGQO+AwACCkBY +PxoAbHwAzgwUCAIGCAcGCAICCgQDAxUEDQQEDwoHGQAABAECAg4BZgICAABQBBMDGQACCjA8LwEw +Gi8BMA4vCABsR2gPZyIAkA0UAhECEwIP9wYP/AYCAgoEDw8HAAAEAQICDgFiAgIAACACOAEgAABH +Ag8BIgACtQIAbs4BHAcbYAD0AQ4KYgJWEAcaAZ4IB/cIBzoKBgMFCgYOPwNkEQILAgEEBwIBRwsC +AQABtQIAAM4BBAcBBwElAgMBMQACDjCAAS8LAG4YJwEoByMFJBt4BXcXOgwQBUkLJwEEDCQUAPQC +GJECAZQCB8UCBcgCCQQLEAcsBRsHEAkCB/JeCwIBj14FYQoEAdEBAZ8BDPQCCkEKAAAEAQYCDgF8 +AgUAACACVwEJAg8BCgAAGAIBAQcEBQMbBgUFFwgMAgUJCwwBAgwNFAACDpABuQGPARUAbh0nASgH +IwUkAXoSeQd6CXkJeB13CDolQBZ5JgDKAx3nAgHqAgebAwWqAwGvAxKyAwexAwmyAwkPAegEHNEE +BwIBhgIlvwUWwAMSHxQAAAQBBgIOAb8BAgUAACUCowEBFAAAHQIBAQcEBQMBBhIFBwYJBQkIAQIc +CQgMIgsDDhYNJgACDrABrgWvAQ4Abj4nAQQMJGA6Kjn8AicBBAwkCDoyOTIA+gMYAiUGAZ8DATEM +1AMHBBwCCwEOAhAIEwIB1gEqEwaZAQkGCQhKJhIOECQPAlISDAwOAgYSEgIUBgUGFg4WDB+LAQGP +BAExDMQEBwIBdDITCF0FLAUbEo0BDgAABAEGAg4BrQUCBQAARgKKBAEJAmMBDgAAPgIBAgwDYAYq +BfwCCAECDAkIDDILMgACDnBgbwFwiwFvEABuKycBBAxABxsXJwEEDiQLehJ5B3oJeSJ6FnkaJwEE +DCQZAMoEKgIB6QMBMQyYCgf3CAeCAweBAwOEAwUCAfMCAZ8BDpYECggBtQQSuAQHtwQJuAQbAge5 +BBa8BBkEAYcDAZ8BDKoECiEPAAAEAQYCDgHtAQIFAAAzBCIDCQIRAQEERAI4Aw8BDwAAKwIBAgwE +BwEHBQcGAwUGCgECDgsLDhINBw4JDSIQFg8aEgECDBMZAAIOgAGWAn8oAG5+FB4TERQfE4ABAKAG +HRwJGQ0mBQsWAggBBQwLCwULE7cEFaEBAgoBjAEGxgQCCgUJBQIFuwQWoQECCgGMAQbIBAIIBQcF +CBAMAwsKBwUUHAIPKScAAAQBBgIOAa8CAgUAAEkCNQEqAh0BTAQUAycAAH4EFQEDBAYFEQoWAQME +BguAAQACDlDCAk8BUAZPJQBuOBwHG+ABOAQ3ATgWXwEEDCQ1ANoGJQIFBgcCB+QDB/cICpQFCQEF +AhEFJgIJARYECQM2AwcOAgIbBA/RBQTUBQHRBRACBQIBQAGfAQywBgoNBgklAAAEAQYCDgHfAgIF +AAAlApQCAQkCBQEKAgYBJQAAOAQHAQoB1gEGBAUBBhYCAQIMCTUAAg5Q4AFPFwBucRQUExpoDmdY +AIYHHQICBAkCFgQgCBOhBRS+BQ4QDPcBDvgBCwIGAgsCDgIOMwoTFgAABAEGAg4B5gECBwAAOQIg +AjUBYQEWAABxAhQBGgQOA1gAAg5gbl8BYDxfFgBuOhQdEyxoC2dBAOwHHQICBhv7BRShAQIKAYwB +BpAGAgIHAQUCHQ4BswILugINBB8pFQAABAEGAg4BsgECBQAANQQbA0YCJAEVAAA6BBQBAwQGBSwI +CwdBAAIOQEg/AUApPwFAEj8yAG4aFB0TjgEArAgYAgK1BhShAQIKAYwBBsYGAgwHCw0CEwQJBg4C +CgUSDTIAAHIEIQMyAAAaBBQBAwQGBY4BAAIWkALEA48CAZACT48CAZAC/gKPAgGQAhyPAgGQAjSP +AjwAbigUHhMCCgUJWAobCQ0KDQkCCgsKBRMoCiUJBAoZCRwUBRMNFA8JBQlpChMJGAoXChITEhQa +EyMnAQQIJEEnAQQFLgUJFgo8CXwKFwlZAOYIJgIC7wYVoQECCgGMAQaEBwKhBwWiBxMKDSYLAgQu +CkMHBAQBB9ABBI8BAQIIgQgbgggDQQZCBJIFDZEFApgFC+8MBeYHCDEINwXqAQWNAQ7XBwy6Aw3j +AwXkAweaBATBBweoAwfNAwvyBwoGBR4NjQgFwAcIsgEF5QgPHwWECAIFBQgYBiQIAQ4loQgT4AgC +wwEV/AEBlwkXgQES5AgIpAEK5QgRoQECCgGMAQbMCAIIBgcJAgGNAQMMAoABBX0H1wcBnwEI/gkh +CgGdAQoCApIBBI8BD80IATEFyAEFtAcIlAEHDgYOAeMIEboDHKcDB6gDCKwFFQIJAQUEGAYkEB3F +BReeBAtTEhU8AAAJAQYCFwGMCAIFAADZAgJZAikDzQEGZgVMBicBegM8AAAoBBUBAwQGBQIIBQdY +CBsHDQoNCQIKCwUFAygODAENAwUEBwsEEAcDBwYLERwWBRUNBAoUBQIFGWkaExkYHBcHEgoIHQoE +ChQHAwMCBhUVHgIdBR4HAgECCCEsJAIjBCQPAgECBQQFKxYsEQEcBAcDCCl8KgodDQtZAAIOcJEC +bwJwDW8BcBdvHwBuGBQFExIKEQkGChoJHgoNCQIKCwkKCgcJJQoHCRQKBQkKCgoJYwD6CRjdBwXe +BxKrCBGwCAavCBqwCAkQFdQEDdMEAtoEC9kEBQ8DEgKXCAeqCBcCCSUF8wcHmggU1QgFwAgKvwgK +sAgFJgUCFQYOCQ0TChMfAAAEAQYCDgHHAgIGAADgAQRXAQ8BHwAAGAIFARIEEQMGBhoFHggNBwII +CwcKCgcJJQwHCxQGBQUKBgoFYwACDnCzAW8BcGlvAXAObzIAbhoKFAkUChIJHwoJCREUGROGAQoO +CTIAjgsYAgKXCQe6Aw3cBQoGCuEFB6cDB6gDBOIFFgYJwQkJwgkRoQkZqAkCAgcKCQIKEwUYDggO +AgwGDAIQAgYCDAQPmQYO3AUyAAAEAQYCDgHPAgIFAABlAlMBGARqAzIAABoEBwENARQCBwQHAwQB +HwgJBxEKGQmGAQIOATIAAg4wgQEvATDfAS8BMBYvHwBupQMA4gsYKgQKBAcGBAYNIxAFBwQKBhEj +FAUHCgMEEgIDBhUkGAUPBBIGGSMcAgIGHSMgAwIGAgEjHz4qFRZJHwAABAEGAg4BhwMCBgAA/AIC +CgEfAAAsAiMBDwQjAxsGJAUPCCMHCAojCQoMHwtfAAIOQHs/AUALPwFAgQE/HgButQIA0gwYBAUB +FgYGBwoIAgwJDgoNCg4TBgYCDwYMJS8CEQcvAhIFHgAABAEGAg4BmAICBQAAcAImAoEBAx4AAg6Q +Af0CjwEBkAHuAY8BAZABkQGPATIAbktoEWeoAaoBFqkBI6oBBKkBFKoBD6kBMKoBB6kBNKoBEqkB +VmgOZwloA2ftAQCwDRgEBQEWBg0SBRUDGAIEAYEIEZAIAy0FegVTBAQVUAVPCgQgAgYFCB4FMgUx +Cg8GECUCBvMLFvgLBwEc9QsE+AsDBQMGDvcLD/4LBQcKEgQCBiMPJAiJDAeqDAUmBUUNBAUBGIsM +Eo4MDAIFGgUmBSsIBg0OFhMCFAkOBAIB/wgOiAkFBAEEA4UJA4YJBQoGAgIIAxMCFAMbAhwTeQ0Z +DSMqAhEHKgISBTIAAAQBBgIOAaEGAgUAAKwBAicBJgLRAgE5BIkBAzIAAEsCEQGoAQQWAyMEBAMU +Bg8FMAQHAzQIEgdWCg4JCQwDC+0BAAIOQNIBPwFAHD8oAHClAgCeAxgCIgYHBwIKAgIHBAUkBSMD +AgUMCgIMBA4EBQYFBgUHCwIHAgUZBQIFBAgYGQIEAgoDCi8SAygAAAQBBgIOAYcCAgYAAIIBBGQC +FwUoAAIOkAHVAY8BAZABMY8BAZABEI8BFABwugIA5AMdAj4CBQIFCAcCDA0KAgUQCAYFFwUqBQ0F +AwwCBAIDAgUKDAIdBgwCCAIQBA4NECsUAACKAQRpASgECwUUAAIOQJ8CPwFALD8rAHCFAwCmBCcC +BwIFBAUCCAIMAgcgBRtRBgYFBQoFAggCCQIMCgUDAwQDAgUJAggDAwoCCAEEBA8CBwIBiQIFjgIH +AgoNDRMRAQ4RKwAABAEGAg4B5QICCAAAMwKDAgIkAysAAJgCAgUBaAACDpABvgGPAQGQARGPAQGQ +AQ6PASwAcJkCAPgEGAIRAQUKFwIMAQoCGQIEAgYCBgICBB8CFAIFAg8ZEggOCywAAAQBBgIOAfwB +AgUAAEACrQEBLAACDkCmAj8BQDg/AUAmPxQAcn8VJhYHFSYWnAEVGBYiAMIBHQIMQQZEHgIBEQQU +GwMEBAgTBBgCQSZEB1smYA4ECQYBTwELBg4GWAFZAQsHDgICAQ8GDiBaDFcBDwZeA08GSANHFlAQ +awwYDD4OBxQAAAQBBgIOAYsDAgUAAKUBAgcBJgIXAYwBAh8BFAAAKQIGAR8EBAMnBAQDAgYmBQcI +JgcYCgECBgEGCQEOAQIHAQMEBgMgDQwKAQoGEwMKBgkDChYJEAgMAQwFIgACDnDHAW8BcBBvCgBy +UhURFgUVHRYSFQkWDBUWFhQVEBYKAPwBGBACAgYBCxAMAgcCFKsBEa4BBZUBHZgBBQUNqQEJqgEI +AgSrARa8AQUEBQIKwQEQiAEKAAAgAsYBAQoAAFICEQEFBB0DEgIJAQwCFgEUAhABCgACDlCgAk8K +AHI+KwEEDCgbNiI1AysBBAwoHjYiNQMrAQQMKAk2DzUONgw1HgC6AhgCJQYB3wEBMQyUAgcCEwIB +ogMimwMCDgH5AQExDIoFFNsCAg4HAgH8AiKbAwIYAYMBAZ8BDKYCAhcGAgG0Ag+3Ag0KAa4CDK8C +FBsKAAAEAQYCDgGbAgIFAABGAkQBCQJHAQkCKgEIAhkBCgAAPgIBAgwDGwYiBQMIAQIMAhQLCg4i +DQMQAQIMEQkUDxMOFgwVHgACDtABTs8BAdAB5wTPAQHQAYEBzwEKAHJyBAMDoQQDDgQhBAQDhwEA +/gMYAgQIBgIKCQVlBnwJCgITCgIRFAwCCcYBA70BBQgXiQESlgECiwEBFgcCEAIFFQlqA2kTiAET +BAkCEAgYUgJDAjUcegU7AYsBCwoFAi0CAQoNeAR10AF2BTgFNwQgHRABtQIOvgIFAQ8CDUoEtQEI +AmECFDkKAAAEAQYCDgGzBgIFAABkBA4DkgEEhAEDRASFAgF1AQoAADECBgE7BAMDHAYSBQIGAQIc +AQkFAwYTBWoKPgINCwQM0AELLA4ODSEEBAOHAQACDpABhQaPAQGQATKPAQGQARKPARYAcu8GAMQF +GAgEAh6zAga8AgmrAhKwAgKlAgEWBwIQAgUVCYwCA4sCDrQCCBECEgcRDRUFIAGLAg4KBQIwAgEK +DQLSAfoBBfkBBeoBDAIBhQILCgUCLQIBCg0CzQHuAQoUFwcDAg8KCicSCxYAAAQBBgIOAdIGAgUA +ALsCAvQBAUACmwEBCgIpARYCBgEWAAA6AgYBCQQSAwIEAQIcAQkDAwQOAyQIRALfAQkFCgUJDQw+ +AtoBDWUAAhagA4UMnwMBoAOoAZ8DAaADQ58DAaADpAmfAx0AcmUEBQN/BAwDGQQDAwQEEAMjBAQD +BBkBHgsDAwQLHQoaCgQTAwYEFgMvBAUDDxkLGioECgMBBGYDEhkEGhwZBBqHAQQFAwURBRIKGQR+ +FX1NGgwEBQMFEQUHFxp0BAwDQAQKAyQEBANTFQQWSBUcFiQVHBZfBAQDAREZEmgVHBaaA2QIY0cE +BQMFEQUSAhkIGicZCBolGQcaQgQFAx0ZOBoBGTkaFgQXAyYECwMTBAQDDgQTAwMEFgMyBAUDvwEA +hgYmBgQCHwgDDwISAgYHBgoCBDsFPCAfBxAFEAUKCQINBgUxCNoCBAcFnwIQBAkCCfwWDNcWBwIQ +AgJEA7YBBB8FghUL2xYGBA5TA1QFAgc+BD0DAgHpAgGuGgubGAOcGAoLAZEdBIQDBtgCAj4HDAGQ +FggCC+UWBUoBqhYLAgupFgUCE60BCNICBe0BBQcFSAUKBTsEAgUGAc0CC9ICBQkEAQUeBAkFKQNg +EPISCu8SAe4SAYcTBL4BEcwREQIDBQ0GL+sSErsDBMIDHLUFBLgFHgIFLAoOOQIIBg2PAgjaAgQf +BRgFowYF4gUJEAH6AwS5BhXEBgYMCAsFChMCCQsFDAwLCAoFrQYI2gIEHwUYBaMGBYIKEAsFDAKN +BA8CCg4FAggCCQIhCggCEgYK5QIM7gIaCB8cBzsKQAgED/0CC/4CAkMETgQCDAItAgyNAwrtBASY +CAUCBhkGDA4MFAIVmQgcmggPBBW1CBy2CAoNDAIVowMMtgMGBBAiCQIIAgG7AgS8AgGxBxm6BwID +DAQMAg4CDgIMAhUIEeUIHOYIBQQSAw1HDwMNHw0vEQdqAhEzBgieAQIWGwevAgiwAgkCGgEFBAkD +Co8CCNoCBB8FGAWjBgXmBQKvAwjqAhYCEb0DCL4DJb0DB74DAgIMAgkFBQQFBAsHCs0BCNoCBB8F +GAWAAwX1AwUBCQIC9gMC4wMBlQMTAggCCgQLBAiMAwGXAw8CCAINBBX0BggBDt4OCgENuRMaAgzG +Fgu1FhMeBBUDAgoOAaYWCAILvxYCDAHCFgsCC8EWEQMFjwEIhgEFzAEFvQEFNwVIBQ8FLwVBdQIR +GRIJHQAACQEGAhcBnxcCBQAArAEEbgOhAQY0BcYCBDYDUQScAwNZBNcFAzcEtwEEOAeSAQQ4A2UG +MwGdAQMdAABlAgUBfwQMAxkGAwUECAUCCwkjBgQFBAwBAgsNAw4KAgECBAIGEwoWExUGGBYXLwYF +BQ8aCxkqHAobARwBAgQCEQNQGxIiBCEcJAQjhwEIBQcFJgUlCigEAhUBTScMCAUHBSYFAhcndCwM +K0AICgckCAQHUy4ELUguHC0kMBwvXzIEMQEmGSVoNBwzmgM2CDVHCAUHBSYFJQI4CDcnOgg5JTwH +O0IIBQcFPgU9ED4CPQFAOD8BODkGFiEXGyZCC0ETRARDDkYTRQNIFkcyRAVDvwEAAg7wAdgF7wEU +AHK7ARkMGhMZBBoXGQUaaRkIGkoZBRpLGQUafBkhGgEZKBoqAKQKIAIFApYBmQYMnAYTjwgEkggV +AgLDBQXEBSACEQIUAiSbBgicBgYCAgIdBBsECqcGBagGBgIgBBsECrEGBbIGGAICAhwEDwcHCgIC +DAIHBhoEAYMGCgIFAgcEBgQF+gUBhQYKAgUCDgQLggYWQxQAAAQBBgIOAd0FAgUAACACiwIC2gID +VwIKARQAALsBAgwBEwQEAxcGBQVpCAgHSgoFCUsMBQt8DiENAQYoBSoAAg4wTy8BMNIBLyQActQC +AJALGAITAQoMCQIHDgUCCA8CBwooDRcHBhQCMAcECBECAgMFBAUBCAIPAgsIFQULEgkCBQQKOyMA +AAQBBgIOAbcCAgUAAD4CFgGaAQI5AS0AAgoQOg8SAHRWAMYBFwINAgUCBQIFBAkCCQ0RAAAEAQIC +DAFCAgIAABcCLgERAAIOQM4BPwFACz8eAHSGAgDeARguBR0FCwMKBQgKAgsEEgIIAgkCBQIIBwwS +CBEGEgUXAgUFAggcGAIEDA4CDAQKEwstHgAABAEGAg4B6QECBQAAVAJ6AgUDDwQGAx4AAg6gAV+f +AQGgAeABnwEBoAFlnwEyAHTmAwDMAh0CCA4FAwMCBQoPBwUCCAwDBQ8cBC8KKBoTGwIFDjINBQ4F +CgUJDwkFDgUBBQIFBgUFBQEKAgsGDgIMBAobCAIJAhUBAwoKBAoREhIIAw4nMgAABAEGAhMBxAMC +BQAAQQIjAY8BBE0CBQUNBE8CEwUyAAIOUIABTwFQJ08BUA9PFAB02gEAlAMYBgUDAwIFAgUCCAgD +AQcCDgQHAwUICgIFAQUICgIPBQwOEQImKxQAAAQBBgIOAb0BAgUAACUClwECCgMUAAIOIMEBHxYA +dOUBAPYDGAINAQUEBwIICgcJAgQIAgkEBwQEAgcCBQoFCQIEBQIJBAoEDAoMAggECQIMAggECj0V +AAA0ApIBAR8AAg4gTh8BID0fASAJHxUAdLkBAL4EGAINAQUICwgHAhECBAoCBQoNCAIJAg4QCQIM +BAofCgMUAAAEAQYCDgGcAQIFAAA/AhQBDQIsAgUDKAACCjAaLwEwEi8VAHRMAJwFFAIHBgoDEgMV +AAAEAQICDgE2AgIAADECBgEVAKgFFAIHBgoDEgMVAAIOsAGXAq8BAbABzAGvAQoAdCIxBzIiLQEE +DCovLQEEDCpTLQEEDCoSLQEEDCo8AgQBCQICAUAxDAILARsyLAC6BRgECucEB/wEDgQJAgoCAfcE +ATEMrAUMAgUCDAIRBAGDBAGfAQzaBQUxCQI3Bg0IAZcFATEMzAURAgGbBAGfAQzaBQUTBBQKFR8W +BRMFogEEoQEJogECnwEIAgUCCg4FCwIECggFBwIfEZsFBwIFEgsLAQIDAhfeBAUBBQQLDggDBRMK +AAAEAQYCDgHfAwIFAABTAjMBCQJXAQkCLQFKBBsBXAIVAwoAACICBwEiBAECDAUvCAECDAlTDAEC +DA0SEAECDBE8FAQTCRQCE0ACDBQLExsBLAACClA6TwgA7AUdAh4CCgMHAAAuBBcDBwACCjAwLxIA +qAYUAgoCEwIKBREAABkCIgERALoGFAIKAhMCCgURAAIKMGEvATAMLw0AdCAxBQILARsyOgDKBhQC +DPEFBRILCwECAwIX6gUFAQUECAIPBA0LDAAABAECAg4BagIHAABYBCEDDAAAIAIFAgsBGwE6AAIO +UH1PAVAOTxUAdK8BAN4GGBIFDQgCFgIHAgQCDQcFCBsGCgIPBw4LFQAABAEGAg4BkgECBQAAHQJ9 +ARUAAg4wUy8BMIoBLwoAdBgtAS4HKQUqGQIMAQEtAQQMKgoCcwEhAP4GGJsGAZ4GB88GBdIGDgwL +6hEKUgK3EgGvBQGfAQzSBgrCEgsBCFsOZgIICgISAhAJHBAI3xIFCxIFCgAABAEGAg4B2QECBQAA +IAIqAQkCIgFxAgYBCgAAGAIBAQcEBQMZCAoBAgUBCgECDAsKBhMIDgdSBSEAAg5QR08BUGxPFAB0 +HC0BLgcpBSoJAggBAy0BBAwqSy0BMAwrDCwIASAAnAcYDgTHBgG+BgfvBgXyBgnWEQjVEQICAcEF +AZ8BDOQGDAQ+GAHfBQGWFwy1GAy2GAi3EQMECicTAAAEAQYCDgG5AQIFAAAkAg4BEwJ+ARMAABwC +AQEHBAUDCQYIBQMIAQIMCUsMAQIMAgwBCA0gAAIOgAGaAX8LAHSzAQCqBxgCEAEKBAcCBQgMAhsL +FgMFBAYDAgQLAgcCBQwKEwoAAEUEWgMUAAIOgAHbAn8BgAEffx4AdqcDANwHGAgCBBQLBToFIw8F +GAojAgMLBhIRAgoCEwIaBBMCDwEIAhUCAgIRBggCBQEeAhACCgEIAQUZEh0eAAAEAQYCDgGKAwIF +AAB4BKcBAU4EHAUeAHZzAPoJATkKVweYAQs/AlIHCAYCBQYV9wIDlgIEAgWXAgSYAgIGAd0DA+AD +CQIIBAMLAwACAQEKBAcBCwECAicEAwUJBgQFAwgDBxcAAnkAdnkA+gkBUgrjAQeYAQtMAjkHCAYC +BQYV9wIDogMEDAXtBAPuBBGtAwSuAwgCAwQEAAF5AAIBAQoEBwELAQICJwQDBQkIAwcRBgQFDwAC +DkCDBz8VAHYbD5oBEAEPjQEQAQ+KARABD30QAQ9/EAEPfxATGQcaQAD6ChgCAgYBlQoLAhcCBwEV +AgwECAIKAgsCLAIHiAoBlwoLAgkCCAETAg4ECAIKAgsCLAIHigoBmQoLAgkCCAEQAg4ECAIKAgsC +LAIHjAoBmwoLAgkCCAEQAg8ECAIKAgsCHgIHjgoBnQoLAgoCCAEQAg8ECAIKAgsCHwIHkAoBnwoL +AgoCCAEQAg4ECAIKAgsCIAIHogoJCAICCMcKB8YKDAgWAgo1FAAABAEGAg4BiQcCBQAAgwcCDwEU +AAAbApoBAQEEjQEDAQaKAQUBCH0HAQp/CQEMfwsTDgcNQAACDtABmAHPAQHQAeYCzwEfAHY1QAQ/ +AUAPPxhABT/FAS8BBAUsdUAFPxIvAQQHbAU/DkAFPwFAJj8oAMALGA4YWAX9BAS0BAGzBAYCCbYE +CR8FIgUGBbEEBbYEDwMmBAIbCiADAwwEDwQDCA0qAykQDwQSBhEHCAIWDwIQHwYsBQ0FEgG5CwEx +BfYLKgIKAgM5BToCBhMjBRoFNQ02BTkItQQFtgQFJAcIAyACAgHXCgGfAQf0BgWKBQkCBYsFBZAF +AY8FDwIPBAiMBQqHAR4AAFcERgOPAQaQAQUCBB8DTwAANQIEAQECDwEYBAUDxQEGAQIFB3UEBQMS +CgECBwcFAw4EBQMBBCYDKAACFvACuAXvAgHwAqQE7wJGAHZIAwYE/QEDnwInCCwXLwEEBSwaLwEE +DSy+AQMYBAIDPScOLAIrDiwiLwEEBSwRLwEEDSw/KwgswQEA2Aw2GBALAskJBtIJCRMFFgJUCAID +VQwCFAIKAQ0CEQgCAQoCNAILAgwQAx8IBAUGCAIIAggSBR0LHAVOBREB7wkLCgUCLQIBCg0C1AGv +AwiODQgKBQ8JAgHpCwGfAQWODRkCAe8MATENqg0YTQNECF8IHAkCFQEDAhACBAMJBAIIBCcIVAgr +BgIRAijxCRj0CQLpCQEWBwISAgUVHvUCDtwMAtsMDtwMDgUFQgU5CQIBvwsBnwEF4gwQBgHHDAEx +DYQNCg0NNBAlGPEMCNYMCAQFDwUQCA0IDgUNCDQNJRENCxMNBw0BCRdGAAAJAQYCFwGOCgIFAADI +AwTAAQMCBIUCA34GrwEBNwNGAABIAgYB/QEEPgLhAQQICRcIAQIFCRoMAQINDb4BEBgPAhABAh4B +HgYOFQIWDhUiFAECBRURGAECDRk/FggVwQEAAg6QAcgBjwEBkAENjwEyAHaWAgCODh0ICQJaEgoG +BiEHJAIBAgIChwcEigcVAgIFBQoJAhEPDh0xAAAEAQYCDgH5AQIFAABuAncBMQAAnQECBAF1AAIK +gAFwfxAAdi4DFARIAJgOLt0IFOQIAgUKCAUEHgIKDQ8AAAQBAgIOAXECBQAATgQtAw8AAC4CFAFI +AAIKQB4/AUASPyQAdl8A5A4UAgQGEQMSAyQAAAQBAgIOAUkCAgAAGgIhASQAAgQwmwEvAjAbAHa8 +AQD2Dg4GCwEBkQYSlAYJAwcEBwICDAoLCQILAgUECQEKmwYHngYPBwMEAgwLBw0JDgAADgGuAQAA +qQECEwAAGgISAU8EBwM6AAIOMLYBLwEwVS8rAHbFAgCmDxoCAlwHAgdZAwMDYgNhCQQLAQHBBhjE +BhEECQIKAgckCzIKLxEtAzgFJgxdAzgIPQNAFwYHBAoIEUsOBSsAAAQBBgIOAaUCAggAAIcCAhMB +KwAASAIYAeUBAAIOUEhPAVAuTwFQTU8UAHY5QgdBpwEA1hAYAgIECwIFCgoRBZ8MB7IMAgUVFhsC +BwINAwoLBQIiAQMCBgEGBg0bFAAASAJXAjQDFAAAOQIHAacBAAIO0AHyAs8BAdAB0AfPAQHQAUHP +ATIAdlMGBQUCQgVBAi8BBAUsIC8BBAosigEvBAQFLEUvAQQKLEsvAQQKLNUDFQUWoAQA1hEfxgEI +YwUvBS0EFgsCE7kRBcIRAsMNBcYNApkRATEFzhEfAgGdEAGfAQqyEQUUBRMEFAUCBd8BCwYRCBkC +B9IBBAINIgO0AQPBAQ7CAQWXAQUbBQ0CvxEEMQWAEgoqA4oBBbMBAgQYAgUCEwIB1xABnwEK+hEM +BA8CCaIBBYkBBQ0KBgowCAkBlREBnwEKwBIKYgVhCmIFsAQEBAgCBAIJAgYCBAIIBAUCBAIIBAUC +Ac0SDqANAwIRAgsIBQMRjwULkgUCAggCCAIQrw0Msg0FCAzbCgjeCgUCBAIIAgcQAw8CBCMCDCcD +KAkCDggIAggCEgISChAYAfkNDMwOBYMNA8oMBfkFCOoFCukFA+oFAgYF5xEF6hEX8QUKtAYFKQPZ +DAXKDAX5BQj4BQICD/kFA/oFAggDAwIEEgYMAg0CCQIRjQYCkAYCAg0EDwIFAg0CBQILBAzxDAmE +DQ+zBgq4BgLFCQnSCQKTDQnAAwGBAhmEAgzUCQLRCQ7SCQvPCRLQCQMGFQoSBA/nCQ6+CAUBDWMT +iwIOngEyAAAEAQYCDgGnCwIGAABiBD4BrAEEHQUHAtQBBtcCBwUCQAEoAosDATIAAFMCBQECBAUD +AgYBAgUHIAoBAgoLGA48DTYQBAIFEUUUAQIKFUsYAQIKGR4cQgIOHTUgCx8iIgwhESQII6sBJgwl +BSgDJwUgCB8KIAMfByoFKRcgCh8IKAUnBSAIHxEgAx9MIAIfTCgJJw8gCh8CLAkrAi4JAQEEGQMM +KwIsDisLLBIrOSwOKyUODg0yAAIOwAHYA78BAcAB7AG/ASEAdh9gDl8WYA5flQJgDl+FAhQHGwcI +DgcDCFwA4hQYJgUjApUPDqQPBAoLAgevDw6yDxgNBRYFAgkKFAYVChYbChwFCA8EEQIMAgsCDAYa +CAUCBQEFQAU/CgILDhaFEA6GEAJHZQIMTgUoBScMAgcGIRARAg8CCwIMCB25Cwf3CAe0FA6zFAPA +FBQEBgYTBA+7ASAAAAQBBgIOAdcFAgUAAG4CZgKuAQHyAQRJAxcBIAAAHwIOARYEDgOVAgYOBYUC +CgcBBwcOCAMHXAACClBTTxwAphYUAkAWChcbAAAEAQICDgFjAgIAAEICHAEbAAIKYEpfCAB2KC8B +BAUsEi8BBAosEQCoFh0CAgEJxRUBMQWIFhECAdcUAZ8BCvoVChMHAAAEAQICDgFGAgIAACkCLAEH +AAAoAgECBQMSBgECCgcRAAIKUFBPJQB2KC8BBAUsGC8BBAosLgDYFhQGDgMEAgL5FQExBa4WFwIB +/RQBnwEKoBYKCSQAAAQBAgIOAWkCAgAAKQIyASQAACgCAQIFAxgGAQIKBy4AAg6QAaMEjwEBkAGU +Ao8BJwB2/QMPKxDFAgDmFhjDBQXGBQIEAwoJAiEIEJMMCZoMAtsPBMADAYECFIQCEZwMApkMDpoM +C5cMCPYLAvULDJgMCRkJAgsqAgEIAgUCEgEDAgIIAz0KMgUIAgQPBgwCDBECFAICEAQQAgUCEAIF +Ag4EDAYSmxAEnBAOBgHhEQkHBLoLCwQUAgoCB6oGAp8GAe0PGgQKAgeKFgqtBgrQBREaEZ8MDY4M +E88PCdAPqAECFxUnAAAEAQYCDgHQBgIFAACjAgLGAQFOAjQBHAK/AQEnAAAYAgUBPwQJAwIGBAEB +BBQDEQMCBA4DCwQIAwIEDAPoAQoECQ8MCQIECzABAgIBDisPCgIKASIEDQMTEgkR5gEAAg5gYl8B +YNoBXx4AdukCAMwYGAINCgUCCAQOBAYCCgQLBgQCCAIKJQoCBQIJAgkBpwEEEgceAAAEAQYCDgHM +AgIFAACSAQK5AQEeAAIOYFFfAmCFAV8fAHaFAgD+GBgCGwgGAgwGBgYEBAMCBAILAwUXaQIXBR8A +AAQBBgIOAecBAgYAAHgCbgEfAAIOgAH8AX8BgAHAAX8BgAErfx4AdjVABD8BQA8/Ly8BBAUsSi8B +BAosAUArP4cBLwEECiwBQCs/WADIGh1MBUkFAgkBBaETBLITAbETBgIJshMFBBMCDgQJgxoBMQW8 +GgwCDAIKAgUGEgoQBwGXGQGfAQq6GgHFExQCDwQIwhMMEgkCBgIDbwlyAXMB9xIEvgEUvBERAgIF +DQYlcgG1GQGfAQrYGgHjExQCDwQI4hMPdwoBDyoSBR4AAAQBBgIOAfgDAgUAACICJwIwAj4FFwiD +AgImCR4AADUCBAEBAg8BLwQBAgUFSggBAgoJAQwrCx4OCQ0BDgECBAIUA0UNARQBAgoVARgrFw8O +GQ0wAAIOUGlPAVASTx4AdiIvATAHKwUsEi8BBAwsWgCGHRgKBQIFrxwBphwH1xwF2hwRAgGpGwGf +AQzMHAkCCQIOBgoDEg0eAAAEAQYCDgGLAQIFAAAqBB8CHAIJBxYIBgceAAAiAgEBBwQFAxIGAQIM +B1oAAg5g8wFfAWASXygAdhgPBRAwLwEEDCwBDysfAQQMLB4vAQQMLAEPKxABLwEEDCxEAOgdGKEc +BaQcBAIFDAUMCQQJCw8CAZsdATEM0B0BvxwVBA8CBxwBnwEMwB0CERsCAY8dATEMxB0BsxwVBA8C +B7AcAZMcAZ8BDNIdCgUSJSgAAAQBBgIOAZ8CAgUAAEcEsQEDFgQGAygAABgCBQEwBAECDAUBCCsC +AQIMCx4OAQIMDwECKwEBEgECDBNEAAIOgAHWBH8BgAEzfwGAASF/AYABDH8BgAFAfxQAdqgBLwEE +DCyUAy8BBAwsJy8BBAwsFS8BBAwscACMHxgKBwcIHyAiByEFCAUiAiEMAgogAhcOAhQWCQcFAQUW +AcEeATEM/B4LPR0ICgcECAc2AjURAgo0AisPAhQqDAwFCAdRHQgKBwQICUoCSRECDUgCPwMCGz4J +USRoAl8RAgteAlUDAhVWCQEFDAsCCgQBhR4BnwEMqB8PKQUGCwIHAgHnHQGfAQyKHw8fBQIByx0B +nwEM7h4PFQ1AEVcvChQAAAQBBgIOAf8FAgUAALABApkCAYgBArcBARQAACcCIAEHAgUCBQMCAhYB +AgIiARQGAQIMBwsEKwYHCQIEGwMCBCMDGAorAgkLAgoeCQIKHgkJDCQLAgwcCwIMGAskDgECDA8n +EgECDBMVFgECDBctDA0BDQUKAQsBFAB2NAD8HxQCDwERAAAUAg8BEQACDiB1HwEgIB8KAHYYLwEw +BysFLEgvAQQMLDQAoCAYvR8BwB8H8R8F9B8MAgwCBxQOAg4CDAIB3x4BnwEMgiAKEwwEBAIQFQoA +AAQBBgIOAZEBAgUAACAChAEBCgAAGAIBAQcEBQNIBgECDAc0AAIOUIwBTwFQEk8KAHZHLwEEDCwc +LwEEDCw6AMwgGAQSEgsCCgQFAwIRAe8eAZ8BDJIgEQIFAQUIAfkfATEMuCANCAcICh8SCwoAAAQB +BgIOAZoBAgUAADoCDAEJAikCLwEGAQoAAEcCAQIMAxwGAQIMBzoAAg5AtAE/AUCDAT8oAHjuAgDe +BBgCEigFFyAMBQUSBgoGEwYoBg4CCi8FCD0CKgIXDSgAAAQBBgIOAdECAgUAAGEEZwJ+BSgAAg7Q +AfkDzwEB0AEfzwEoAHgpXiNdpQEKFQnJAgDABSAUBRECCAECAQIXCgwDCAoMAgKRBB6QBAMEBwIH +lQQcmgQJAgoKJQoMhQQVhgQCAg40BSMMXQenAweoAwR2FxcTAgMiBSMFlwQQmgQNAQUCE5kEBZoE +BQoWAgUGCHEFpwMFqAMFcgUQGAIKFRENDlEoAAAEAQYCDgGyBAIFAABPAqECAUsEPgIuBSgAACkC +FwIMAxYGHgURBgoCEgdEChUJIQwHAgcBBAs3EBAPJRAFDygMBQIFAQULbgACFvACuwTvAgHwApkB +7wIB8AKnBe8CPAB4zwsAygYuAgIGCAIChQVGjAUJiwUI/AQDIgI8FQIFNwj5BAi6Aw2YAQgoDb8B +B6cDB6gDBMYBLQhWNQs4BAwFBxIUDecBCKcDCPwEBSUV5wQMugMNrgEFrQEHpwMHqAMEsAELAgoG +KQMsUgPxBAMCA/AEEvEEDwID/AQRAhEGAYUFCQITigUCFBATGYsFCAId8AQCHAMBDhkEHGQCCAIn +AgIEDyUFIgUBCAMZaQi3AQ2uAQjnBBO6Aw+nAweoAwToASk6DTkI5wEN3gEOARINEwcNxQEN+gEM +ARQCA+UECAII4gQJsQUHugMJ+AEF9wEHpwMHqAME+gEw+QEOmAE8AAAJAQYCFwGkCwIFAAD6AQSQ +AgE4AeYCAkoEOwUxBDUCmwEFLQgjARUFPAAAOgIkAiIDCQQIAycICAENBRUGBwQHAwQFtgEMCAII +DRoSDAENDwUQBwQHAwQPbRYGFRIWEhUjFhwVKxYlFe4BEA0PCBgTCw8CBwEECz4MDQtABg0FIxYQ +FQkcBwEJGQUaBwQHAwQZMBoOGTwAAg6AAukD/wEBgALHAv8BKAB45wYA+gcYNhgzAgQQAgICArUG +KBYbpgYJDgGfBg26Aw3jAwoWBRUF5AMMpwMHqAMJ6AIk5wIN6gIXAiAPAZUGBLoDGacDB6gDCd4C +CAIDARbdAg3gAhIgCgEoAhP/Ag3sAgXrAgynAweoAwnwAhnvAg3yAggFCAQOAw0MCQsFAgGnBge6 +AxH2AgGvBgm6AxSnAweoAwn4AiL3Ag36Ag8BDgIF+QIiyAIoAAAEAQYCDgHKBgIFAAD3AQQkASoB +RQQkAS4EWQEaATYBXwQhAR0EHQUoAABGAhQCFAIQAgsHCgwNAQ0HBQIFBAUDBQYMBAcDCQkkCg0J +OBIEARkEBwMJDyEQDQ9XEA0PBRYMAgcBCRUZFg0VOhoHAxEVAR4JARQEBwMJGyIcDRsiHAoFCgsO +CSgAAg4wUS8CMBgvATAQLwEwDi8eAHgwARECdgDECBgCAggBAhUYERcSFg4TDQICAgoKEQ8OCx4A +AGcECQMjBAYDHgAAMAIRAXYAAhmwBKYFrwQCsATtDa8EAbAEcK8EArAEpQOvBB8AePkTChEJ2wMA +jAkpyAIIFgjbAgJWCQYPAg8WNSwCAhwOCw0KBBIDAwQPAgwIBgYpBQiqAQrPCRKsCBkODbkIFroI +EAIMIwgECAIFDggsFvcICLoDDYwFCIsFB6cDB6gDBNwGDQIKAQgeBNMKF9QKEgIM1QoE1goOAhUL +JAwXGSzCAgoFDbkCCboCELkCCtoCC9kCuAECMgIR3AIL3wIF0AIOFQW5AgUBBeMGDb4FAxwD8wgT +2AgJAgoCBwgCCwgCCGAIRQUFB+EIC+IIJ+EIF+IIDtIDEgUNxwMKAhTGAwwQB78DAsYDDsUDA7AD +Bq8DCRIFngMFywMFLgItCS4YngMFIALrAwk4AroDC/EDBDoYFAgWCHkIFggTCM0ICMwIAxYMPAUG +BAwIFgh5DxYIEwjNCAjMCAgWBUwICQVBCRARAQM+DAIIFggGD8sJCM4JCCUDDAgLEAwJgAMPBQ3p +AgrqAg3pAgKAAw4VBSYGjwMUigMIiQMCkAMLjwO5AQJsAmoCggECBRMZCQgCCK8JDLAJBa8JD64J +DK0JDk0R/gkXIMkBAx8CDOYCCgUN3QIK3gIN3QIC9AIOFQUmBoMDAwMIBAr+Agf9AgWEAwuDAwUE +FoUBCy0WtwEfAAAMAQYCFwG2FwIGAACPBARfARMEiAQFiwEIRgerBAq3AgKPAgVbBvUCBSsFHwAA +uwICEgEmAhYBTwYIAQ0DCAQHBAcDBAMjChcJHgwEC4oBDhcNCQ4QDQoQCw/7ARALDwUSDgMFDQoE +DQMGAhMBQBQLEycUFxMOFh8VHhYMAgcXAhgOFwMWBhUOFgUVKBYFBAIZCxoLGUQUCBNHFAgTYhwI +GyweHB0KHg0dAiAOAQUEBiEUIgghAiILIb8EHAwbBRwPGwwcDggRI4sCJhclCiYNJQIoDgEFBAYp +FSoHKQUqCylbAAIOYIABXwFg1wFfHwB4HwoUCQYKFwkFCggJBgoUCQYKBgmCAgCCChwCA/8HFIAI +Bq0IBC4TgAgCBgOzCAi2CAaHCAotBCIGlggGlQgGlggCDgoZBQgFClYCYAIXFR8AAAQBBgIOAecC +AgYAAJ4BBGcBYQEfAAAfAhQBBgQEAhMFBQgIBwYGCgQEAgYLBg4GDYICAAIOcMMCbwFwYG8BcA5v +AXDcAW8eAHiAAwoYCaQCAM4MGAICCAeJCx+KCw0wCqMLC4ILEwYY9gEN9QEC/AEL+wEF8woEugMN +4wMFngsFuQcHpwMHqAMEvAcMAgoKBKELCaILEQIKAgIKBwIFAgsMEwQPCAMKAwkFCgUJEgoMtQsY +tgsCAgcEElkPGE0CJ48LD5ALKgIWwwcNugcMHR4AAAQBBgIOAZ8FAgUAAN8BAhABKgKHAgJ+Ax4A +ACECHwEXBAsDKwYNBQIGCwUFCgQBDQUFAQUIBwQHAwQHGg4JDYQBEBgPngESDxFACA0HKgACDrAB +wAOvAQGwAeYBrwEoAHghChQJIwoTCWIKBQkuBggFRAYIBToGDgUEBhYFGQYNBUYGBQVgBgcFTwDI +DRgCAgYH1wsU2AsCAgcGEwIH4QsT4gsCAgcECiwPHQICAqEMDhYWkAwF+wsGugMNuQMFKQoWBc4D +B6cDB6gDBMwIDbUICLgIF4cMBLoDFKcDB6gDBMYICq8ICLAIBREB7QsHugMR4wMEPAeoAwS2CAOD +DAraAwG/BA1mBGUWjg0PAgqPDQ20BA3aCBHZCA3OCAXNCAenAweoAwTQCASvCAWuCBPNCAPOCAiH +DAe6Awq5Awq6Aw+nAweoAwTUCA29CAe+CAXTCCKWCCgAAAQBBgIOAcAFAgUAAIMCAhwBMAIKAWcE +YgEjAUQCDwIdAygAACECFAEjBBMDJgYHAgcCCwILCwUQBgENCQUEBQEFBgUCBwQHAwQNDRQIExcY +BAEUBAcDBBUKHAgbBiAHAREEBAIHBQQdAyYKAgECDQMEBBYpGSoNCw0dERYNFQUsBwIHAQQrBDAF +LxMsAysIMgcFCgoKAQ8EBwMEMw06BzkFNAoHCh0ODSgAAgRAzwI/AUALAHjfAgCWDxMKDB0XHgIX +CgIOAgsUBQIHDgQoAzUDDgkCBAEHKAwCBQEDFwoGBwUEHQseBx0FBAQjGSQFHQoCDgILGgVnE6YB +Ap8BBwILAQMECwEDngENQwsAABMBzAIAANkCBAYAAB8CFwECAiMBbgQZAwUEIwMFBhMFAgYjBRgA +Ag7wAbMC7wEB8AES7wEUAHqMAVwOdRkaEBkCGqMBAJABIAYsGAMXFAIiBgewBA6RAxYCA50BDQID +nAECkQEnAkwECg0SExQAAAQBBgIOAcsCAgUAAKsBAo0BARYEBgMUAACMAQIOAhkDEAQCA6MBAAIW +wAKzBr8CAcAChQG/AioAeu0CXBRbCVwIW/YBCB4HAggGBwkICgcFCAkHHAgdBwkZCBoKCAoHzAEA +ygEmBAhUElUaIB8aQBwYBh4EOwwgAg4IAVMEAgcCCeYDCQkL2wMJzwEIrAIKAhtZBQIFWDonBSwI +AgYBGAJKLQU+CAkGDQVVFaEBAgoBjAEGcAJvBnAJYwpkBWMDoQECCgGMAQN6BQkFChJtDaEBAgoB +jAENfAllCEwFJQUvCkAIMAUEEBUIBQtdCkgLa0sCEgUqAAAJAQYCFwHOBwIFAADFAQSTAQOMAQRC +AzMELwNeBBQDGgQmAxMGggEFKgAA2QICFAIJAgsDCQYEAgQJJQIKAccBDhUBAwQGDwIUBhMJDgoN +BQ4DBAMCAxMcDg0IAwINFwkaCBkKDgoNMAIKAZIBAAIKYEFfKwB6dgCGAhQCBQENAgUCIQMqAAAp +BBUBDgEqAAIEMFsvAjASAHogXBRbGFwEWwFcBFseAOwBEgYCAwsCAegDCQkL2wMYzwEE1AEB0wEE +1gEBAwsBEgAADgFlAABjAhAAACACCQILAxgGBAUBCAQHHgACDjBqLwEwDi8BMA4vHwB8tQEAThgC +DQYGAgQEBAEHAhkCBAMKBhgEDxEPAx4AAAQBBgIOAZgBAgUAAHkEHgMeAAIOQPEBPwJADj8fAHwc +AjEBAgIZAcYBAHYYAgT+BAkEAhYDBgMVAwUFAgUMCQIKkQUCiAUMIA2lBQoGHgIcCwUCBQgFAgoG +CAMFAQMCDAIEAhwLDwUeAAAEAQYCDgGRAgIFAAC+AQJSAR4AABwCMQECAhkBxgEAAg6QAV6PAQGQ +AdkCjwEBkAEsjwEhAHwwAzEEEAMPBBQDLwQHAhEBAwIVAQMDBgQOAxEEBQMSBCMDBQQFAxQEDwYK +BTsDDQYRBQ4EIQCgARgCAmkGbgIFBTAFIwMCASAfFhItAg0KDgRUAxEMQQoCCkAMugMVpwMKqAME +9wMHOxE4AzcTwAQC/wMDOgMSA0sOOgS6Aw37AwX8AwenAweoAwTxAxkHBQIF+AMF+wMFGAoqBRIF +SQUKCi4KLQICBwQXAhECCt4DDbMEEbQEDpEEIQAAxgICYQIoAiQFIQAAGgIGARAEHwISBRAKAwIM +CxQMDAMVAgoBBAcHEBEPAxATAQINAwwDAQMJDhQEAQ0RBRIHBAcDBBEjCAUHBQQKCAUBBQkPGAoX +OxINAREHDgchAAIOwAGeAr8BAcABqwK/AQHAARC/AQHAAUu/ARQAfCEDHwRLAy4EAQMPBAQDCQYY +AQtaB1cEASoDMgQJAwsEGwIbWARZBAMJBBECCQEVAgQBfAIKAREDDQYKBQ0EIADqARgCAggHJR8m +DQoRQAU/DR4QAgkMAQIBXQ0qCLoDGYEDAWEEPAeoAwT/AgRNCfwDAbMFF4YCAwIIpAMHqwUEiAIF +DxMCESMBCQS6Aw3jAwXkAwynAweoAwmtAwkfCyARAgraAwGzBRqsBQTJAwQ1CTYR4QEJ5AEDAhLl +AQSSAiQGGxQYAhRjEckBCtwBEaoDDYUFCoYFDbMDDBMUAAAEAQYCDgGsBQIFAAB1AhQB8wEEEwN+ +AmIERgUUAAAhAh8BSwQNBAgBGQUBBAQGBwMEBQQMCQIBAhcPCxIHAgQTKhgEAQ0TBRQMBAcDCRUJ +HAsbGx4BAhoCBCEEJAkjESAJHxUmBCV8IAofERYNBQoJDQUgAAIEMIsBLwEwGS8BMDsvATAgAH6G +AgA+DgIcAgYMCAINCAoEEQIGCyACCg0BDw8SChgIAQgIIgIKDw0XEwAA7gEEGAAAkQECDwFmAH4Y +AG4CAgICCgEJBgEAARgAAgQwkQEvATAcLwEwPC8BMBsAfosCAIYBDgIcAgYMCAINCAoEFAIGCyMC +Cg0BDxISChgIAQoIIQIKDw0XDgAADgH9AQAA+AEEEwAAlwECEgFiALYBAgICAgoBCQYBAAIOcIoC +bwFwI28BcK8BbwFwG28oAH6wBADGARgCEgYTAggCDAYKEQUKBQIFCCILBQwQAgoGBQcDBAUBDQcL +DQMORgkxEgMCAwQFBQUCEgEFAhABCgYFAw8ERAIKCw0JDgcoAAAEAQYCDgGTBAIFAACUAQJwAnAB +XgI2AygAAtwDAH4vmgEEmQESmgEPmQEBmgEHmQEkBRMGAgUjBlwFEwYCBSMGkAEA+gENCAsCCAIF +AgQCAgQEBwQQAwIIBgMWBC0PCgEJByAEBAYKAwYIBQQGC4QME/8LAoYMBwILAQMECwEDhwwEDQsY +CgIGEA4RAxIGAg5GBDUFAwMgAh8DEgIRAz4CmAsTjQsClAsHAgsBAwQLAQOVCwQ9AwEGBgMFAgID +BAwCDAUDKBECBgIIAgYCAwIOFwMGAyEIAgUSCQIOAAHcAwAALwIEARICDwEBAgcBJAQTAwIEIwNc +BhMFAgYjBZABAAIKQCw/AUAOPwFAOD8pAH6nAQDIAxQCBhwCAgYXBgIPBA8SCQIEAggGCyUJAhAF +KAAABAECAg4BjgECBQAAKARXAygAABoCCAEkAiABQQAC9AEAfpEBmgEWmQFNAJQEHQIGAgsDBQgC +CAYGEQgDAw4CCMYBCQQHwQECyAEMIAjlAQazAha6AgUNDhILzAEDBgMVAwUFAgUMCQIHzQEMAAH0 +AQAAZQIQAQICFAEGBBYDHgIjAQwAAqQCAH5nmgEDmQEImgEPmQEQmgEHmQEumgEPmQEQmgEHmQE4 +AOIEKgIGAggECAQGBiHtAgP2AgMDBfECD/QCAQIP9QIH3AIGFgMGBQQOAgwIBoUDD4gDAQIPiQMH +8gIDFQMwBQQEHQMVAzQFDQcSBQILBAcAAaQCAABnAgMBCAIPARACBwEuBA8DEAQHAzgAAgpgRF8o +AH52AMIGGZcBCpwBApsBBZ4BGwIKBycAACUCGAISAycAABkCCgECAgUBTAACCiAyHxIAfk4A0gYZ +BAKhAQqkAQ4CCgcRAAAEAQICDgE4AgIAABsCEwIPAxEAABsCCgEpAAIOUHJPAVASTyEAgAG0AQDG +AhgcBQEFFwQIBgIJAgoCBAYWAgwCCQIJAgoVEgchAABUAj8BIQACCjAmLwEwFS8RAIABH1YIVTAA ++AIUAgcGBN0CCOACCgUVAxEAAD0CCQERAAAfAggBMAACCjAxLwEwEi8RAIABKlYIVS0AigMUAhIG +BO8CCPICCgUSAxEAAEgCBgERAAAqAggBLQACDsABzwO/AQHAAegBvwEBwAELvwEBwAEzvwFHAIAB ++gJWBFWYAVYEVbMCAJwDIEwFGQUUBRMNLwoCMQIPLAUdBQwFEggdBRMJMAUbCQIDAgsCCgUFHAoP +AwIKAgoEBAIKHAMXChgDFxQCDAUFBAUCEgIVXwECDlwFqQMErAMYHAUbEhQFRQkyBQEFAgoCETMD +PAYKC3MBAhV0DMEDBMIDCwEFAg8CDgIJAhUCDQILAgsCCQIGAgsCCQQLAgcECicMaRRcCicVB0cA +AAQBBgIOAbAGAgUAAFIClAIBJwKnAQKJAQMbBC4DRwAA5gICDwEFBAQDdgYWBQwIBAe5AQYKAwoB +ZgACNACAATQAiAQKAgcGAgMDBAwCCQIDBgYAAAoBKgACCiBkHwgAgAEUOQE6BzUJNi45AQQRNhEA +sAQUzQMB0AMHgQQJigQmAgcCAdsCAZ8BEf4DCg8HAAAcAlMBBwAAFAIBAQcECQMuBgECEQcRAAIK +IEcfCACAARQ5AToHNQk2FjkBBAw2EQDSBBTvAwHyAwejBAmmBAkCBQIHBAH7AgGfAQyeBAoNBwAA +HAIeAQkCDwEHAAAUAgEBBwQJAxYGAQIMBxEAAg5AsgE/CwCAAcsBAOQEGAIKAhECCAgcAgHzAgXy +AgPvAgkCDQIJ8AIkDQkKAgkJEgoVCgAABAEGAg4BrgECBQAANgKBAQEUAABYAgUBAwIfAUwAAhnQ +BeACzwUB0AUMzwUfAIABcDkBBAw2gQE5AQQMNpoBAKoFKSAIFwgFGgIcAgHNBAExDIIFOQIKAgUC +JgIFAg0CAdsDAZ8BDIgFXwYQHQwHHwAADAEGAhcB9gICBgAAZQLzAQIuAx8AAHACAQIMA4EBBgEC +DAeaAQCAATYAygUcAgkCCgMHAAIOQIABPx8AgAEiOQE6BzUFNkc5AQQONigA1gUdBgX5BAH2BAen +BQWqBQoCDAIhAgUCCgIBgQQBnwEOpAUKDx4AAAQBBgIOAZABAgUAACoCZQEeAAAiAgEBBwQFA0cG +AQIOBygAAg5QwAFPHwCAAUVWN1UOVg5VVQCcBhgKBwcDCgkQAxkCDAsLBRoFuwQEDA4CCgIXBASY +BAMQC6kEDqoECg0IDgQNFwQKER4AAAQBBgIOAdABAgUAAMABAgUBKAAAKwIDARICBQI3AwMCCwIO +AQoBCAIEAT8AAhnQBaIDzwUB0AULzwU3AIABcnAubx05AQQMNtUBOQEEDDZSAMQGORQIEQUEEwIZ +uAcusQcCAxUKBQEB8wUBMQyoBjYEJQQxAh0EFQIWBAGJBQGfAQysBhAVCxM3AAAMAQYCFwHQAwIF +AACaAQKEAgEJAiABNwAAcgIuAR0EAQIMBdUBCAECDAlSAAIO8AGIBO8BKQCAASo5AToHNQU2tgJw +GG97OQEEDDYyANYNGA4IAwr9DAH2DAenDQWqDQUCCwIRBoUBA14IGhAIAhDlARjqAQITBQZUDg8C +EAIBnwwBnwEMwg0KLSgAAAQBBgIOAaIEAgUAADIEJgIPAhsCPgIdBR4GjQICDw0oAAAqAgEBBwQF +A7YCBhgFewgBAgwJMgACCmApXwgAgAEhcAlvEQDwDSACAd0BCeABCgMHAAAEAQICDgElAgIAACUC +DwEHAAAhAgkBEQACDqABowKfASMAgAEiOQE6BzUFNuwBOQEEDDYsAIgOGAgKrQ0BqA0H2Q0F3A0F +AgsCXAIKBlEGFAIQAgG/DAGfAQziDQobIgAAKgQVAhwCyAECDwkiAAAiAgEBBwQFA+wBBgECDAcs +AJgOIAIBhQIJiAIKAwcAAg5AoAE/CwCAARg5AToHNQU2cTkBBA42FACoDhjFDQHIDQf5DQX8DQUC +CwIbBAoCGwIQAhACAdkMAZ8BDvwNChUKAAAgAhUCaQERAQoAABgCAQEHBAUDcQYBAg4HFAACCjA0 +LwEwEi8gAIIBcQBCFAQhBgoDEgUgAABLAgYBIAACDjBCLwEwRy8BMBIvMwCCAd4BAHgYAgqKAQuJ +AQKKAQuJAQKKAQSJAQICDwICehB5AnoKdQJ2A3UCCgMHAoABBYEBAoIBA30CcgNxAgYNCxIHMwAA +pQECBgEzAAAiAgsBAgQLAwICBAETBhAFAggKBwIKAwkHAgUBAgIDAQIMAwtUAAIKMDIvATARLwEw +Di8pAIIBhgEAlgEUAgpsC2sCbAZlAgIKBBIJDwMoAAAEAQICDgFtAgUAAEACHgEoAAAeAgsBAgQG +A1UAAgpAdz8kAIIBpQEAzgIZCAUFCAQIAh8CIwIIAgoNIwAABAECAg4BhwECCgAAOwJHASMAAgQw +lgEvATA8LwEwDC8BMBcAggH8AQDkAg4ODAIRAhUCGXMUmQECjgEGgAECcwN+AhYIIwMkAhkSJgoC +EK0BFK4BAgINBg0HCy0MAAAOAe4BAADrAQQRAABZBBQBAgQGBQIEAwM7CBQHMwACDjBMLwEwWy8B +MB0vATANLwEwFS8eAIIBlgIAvgMYAg8CBQIvBCLDARmZAQKOAQvQAQICEgQFAhkEDgMFBwgDCAUe +AAAiAnYBSwQVAx4AAH0EGQECBAsFcwACDsABmwi/AQHAAfIBvwEpAIIBxQoA8gMYGAqRAgqhAQIK +AYwBBp4CAhAFTApbCZECCp4CDAI1hQIGhgICAgUgBx8rhwIEiAIFGAgXAwIGAwMECgEDCC8GbAIF +vQIKvgIgsQINsgIFDwMCAgMDFAIGFrkCDboCBRcDGgIZAyAOwQINwgIFIwUsDR4KAnAEFAMFRw82 +BAIKAhoGOQKDAQoz7QINoQECCgGMAQP+AgkCCgUICQgBEwgYAQglCAcFBwUBEwUOBwUBCAtdAhIb +KQAABAEGAg4BpwoCBgAAXgJgAdsBAnsBpwECTQLEAQFFARwG7wEFKQAAIgQKAQMEBgUaBAoDQQgE +BzsKBAnGAQ4KDSAEDQMlBA0DGwQNA9ADBA0IAwIDDa4CAAJ0AIIBdAD4BAwGE4UDFKEBAgoBjAEG +lAMCnwMFogMFAgGjAwamAwUCBAIHBAQCBAIHFwYAAXQAAB8EFAEDBAYFAggFBwYIBgclAAIOoAGF +AZ8BAaABkwKfAQGgAWOfAR4AggGpBACcBRgCDwIFDkkJDQIIAgoMFK8DG5kBBo4BB7wDBhUFFhQW +BekEAdgECMsDDcoDCL0DFKEBAgoBjAEGzgMCzQMFzgMFDgoLAsMDA6EBAgoBjAEG1AMCAgYBAwIk +BggCAwENAgQCCgMIBQgTBwIDAQe1AxqhAQIKAYwBA8QDBQEEAgMBBRwGGwsRHgAABAEGAg4BjAQC +BQAAIgKnAQEvAjABgwEEOgMgBAYDHgAAqAEEGwEGBAcFJAgBBwgKDQkIDhQBAwQGDwIUBRMRDgME +AwIGE3YYGgEDBAMZQAACDkCgAT8BQAY/HwCCAdQBANQFGAIOAQoICAIIAh0EGQIIAicCCgUGDx8A +AAQBBgIOAbcBAgUAAFACSwIKAwoEBgMfAAIOoAHmAp8BIACIAU5BAQQKPhhBAQQIPqABTglNED0I +PglBAkIFTgRNRQCWASIEDggJBwcCCwoCGgEiAZ8BBREFeAULBAwODgFTATEIjAEOAgoKCQQDAgMG +KAIMBwIGBQgNBg4CCRYRBgndAQniAQMCDAIBxQEIWgR2BS8CNwQCAYkBBIwBA2QCdQd2EAIKfR8A +AAQBBgIOAfcCAgUAACsCIgECAiEBBQJvAiMBLwQ/BR8AAE4CAQIFBAUHGAYBAggHoAEKCQkQDggN +CQwCCwUQBA9FAAIOQD8/AUALPwFAvgE/FwCIAaEBTgRNigEAnAIYCAQCAdYCARMHvwIEDAQCDLIC +A5UCBAECCwwXDCwBrgIEqwIBogIBEwcoA70CBgEEAgOMAgrBAgP0AhC9AgIeBAkFDAHPAgTaAgMC +AhcGGAsCAggDBwgSBhwaLQUyBQYBkAEFAg+NARKfARYAAIICBBcDFgAAHQIBAgcDFAQDAx8GBAUB +CAECBwIDCw0OCg0DDBALDBAED04SFBEoAAIOUH1PAVCFAU8UAIgBOU4ETegBANADGAIEqgEBEweT +AQgIAwIGDATHAwTKAwMCBQQHDAgtBT4FBgE0BQIMegWpAQGqAQanAQYCChcRCxGMAQUdBZEBRwIS +BxQAAAQBBgIOAYgCAgUAAGsCCgEjBHkDFAAAHAIBAgcDFQYEBSIIEQIFCQEKBgkyDAUHBQNtAAIK +QFM/AUAhPxYAiAEWRQ8CCwEERgNFFUZJALYEFAIC4QMCAggCBRILCwECA9QDA9EDFdQDBQINBCMH +FAAABAECAg4BfAIFAABxBBADFAAAFgIPAgsBBAEDAhUBSQACDkAlPwFAYz8YAIgBrwEArAUYAg4E +BAgKCwUGFksJTC0CEgkYAAA5BF4DGAAATwIJAVcAigEBAJYGAQACBEBKPwFACT8BQFcAigGwAQDW +Cg4CBQoFAxECHAgKDQoIRQISAAAOAaIBAABZBFcAAg5AgwE/AUBUPxQAigFQQwEECEChAQC2DBgC +EAIQAgoFBRIJ5QsBMQicDB8CJhMEBi0CFwsUAAAEAQYCDgHdAQIFAABUAkUCTQMUAABQAgECCAOh +AQACDkBIPwFART8UAIoBR0MBBAVAYwDkDBgCEAIPAggMCJMLAZ8BBbgMChEEBi8CEgsUAAAEAQYC +DgGTAQIFAABIBAUDDgRBAxQAAEcCAQIFA2MAAgowSC8BMBMvEQCMAXcAogEUAgcECQYKEAsGEAYK +AxMjEQAAXwQHAxEAAgQwVy8BMCYvATAgLwEAjAGkAQDSAg4WDRAOBgUCCQQNqQEOrAEKDB0GCicJ +lQEOmAEKAAAOAZYBAAApBHsAAEQCDgE6BA4DCgACDoABmQF/AYABxgN/AYABLn8UAIwBgwIVGx0K +NB4zBB4JFgIVKBYCFQ4WFBUUFg0VBBa/ARUNFh8AsAMYBAkCDgINBwUGBQoLBA8yBC0PAggBGAYL +AgomGAIJAQVABSsSAgUVD0AHJwIIAZwaGgsBoRoEFAghD0AHGwUEAQoExAYJxwYChgMEwAMBgQIS +hAIRxQYCyAYOxwYUygYUyQYFAgQGBJoEBJkEAgIXOQ1ABQMFBBwCBEEKQgUICYcBBzoHThMEFgIK +BQ0JCbYGDbMHCwcUAAAEAQYCDgGUBQIFAAB2AiMCQgEnAcgCAhECQgMUAACDAgQaAgEDBAYGBx4K +BAIJCwIOBAEBBBIDEQsCDA4LFAwUCw0SBBG/AQwNCx8AAg5g5gFfAWAQXwsAjgGQAgAWGAIoAgkC +HQIIAQwIEgIZAhkCGQQeDREJCgAAHwLnAQEKAAIKIFUfCQCQAR9JAQQMRh1JAQQMRhIA9gEUAgoE +AZkBATEMzgEJAgUCDgQBIwGfAQzIAQsTBwAAJwIhAQkCBQESAAAfAgECDAMdBgECDAcSAALqAQCQ +ATQYCxADJwYYCxADJwEYAxc6GAgXAhgDFzQYEBcFAOoFDagBCaEBEqIBBpsBBQIBqFwKAgGPXgPq +AQUCAaJcCgIBj14D8AEBnlwDgVsHAga5AQPSARsCBAYJzQECrlwIrVwCrlwDgVsEAga5AQPSARcC +BAYK0wECtFwQs1wFAAHqAQAADQICARkEAgMKBgsCAwcGCgsCAwsBBgMBDQMDBCgDAgYIBQIGAwMK +AQMCJQECBhAFBQACDpABkQKPAQKQAc4BjwEKAJIB+QMARBgCKAIEAiMCCAgVBAUCCAEIChICCAQM +AjcCBAgOAg4CCw0EBC4CEhcEBC4CERMEBDECEgsKAAAEAQYCDgHcAwIFAAAfAtADAQoAAg5gLl8B +YFZfAWBIXw4AkgFJVi1VHlYEVVIApgEYAhkEAh4KHQUCB3UteAoGBQYFAgqFAQSKATICEh0OAAAE +AQYCDgHNAQIFAABfAisBDgJEAQ4AAEkCLQEeAgQBUgACGbAZnAGvGQGwGWGvGQGwGdICrxkBsBms +Aa8ZGACSAa8GANgBKQIJCA4aAhUCAwUaAhEIBwgaAg0PFQMYEw0ICQUoBAMPAgIhEiQ7AgQCCQwI +AhIICQITAh0CDwICBQQMGgIJRwNQAggJAi4CCyEIOQxcBQwGCBQBBgYHCwYOAwIEAgkCBAQNRQg5 +DGoGFgUGFUcEDCkCEQ8NF08CEi8YAAAMAQYCFwGBBgIFAADXAQLABAEYAAIEkAFIjwEBkAF4jwEB +AJQBV2cKaANnG2hHAEwODAUCMAIKBgYCAQkDegp5A3oHAQUCD209AgoAADEElQEAAFcCCgEDAhsB +RwACBKABUJ8BAaABV58BAQCUAa0BAHAOAjkCBAIKDEMICwIKAAAOAZ8BAAAxAhoBOwQnAAIKUG5P +DQCUAYUBAIIBJgJJAgoDDAAAJgQbATgBDAACIdCAAYIBz4ABAdCAASrPgAEB0IABHs+AAQoAlAH3 +AQCQATESDwI2AgQIFQUVBg0CAgoCBgoFEAcHAgMDCQELHQoAAAMBFwIXAcEBAgUAAFsCkgEBCgAC +DrAB9QGvAQGwAaMBrwEUAJQBahQJEwgUBxOCARQTE6QBAJgCGBoFFwwUCQIB+AM3lgEJiwUIjAUH +iwU6AgH0AzXvAwgOCqJGE61GWAIJBhEDHiUUAAAEAQYCDgGeAwIFAABYBEoCWAUdBpABBRQAADMC +NwIJAwgEBwM7BjUFEggTB6QBAAIZgBHFAf8QAYARe/8QAYARLf8QAYARD/8QAYARmwH/EAGAEQj/ +EBwAlAEwIgohB0IEQZQEAIoDKQIDBgICAp0CCpwCAgoCBgP/AgSCAxYMOwIECh4CBQENAgUCEA8E +MBgCLwIZAQYEAgIQCAwCEgIQTRAgCAIIAQ4CNAICAgwOCAUKAgsEDwIQKwglHAAADAEGAhcBqwQC +BQAATQSCAQNBBDsDKQQVA0sGRwEiAxwAADACCgEHBAQDlAQAAg5A0AE/AUAXPygAlAGeAgCABBgE +DhoLGSsCCQEFAgUOBgILCwYGOQ4DDQUSGBsLAQwDKAAABAEGAg4BgQICBQAALAK5AQIRAygAAg6Q +Ae0BjwEBkAEJjwEBkAELjwEBkAELjwEBkAEIjwEKAJQBsAIArAQYAhICQgIIAQQKLwIZAggGAwIP +Ag0GCQQMBAoVDAsMAwgDCgAAVgKmAQEkAgYBCgACChBgDwgAlAE2AhABAQIRAQECCQEQANwEEgIO +AgwCCRwBuQQQvAQBuwQRvgQBvQQJxAQJKwcAAAQBAgIMAV4CAgAAEgIjAT0AADYCEAEBBBEDAQYJ +BRAAAg6QAW2PAQGQAbgBjwEBkAESjwEoAJQB7wIAkAUYBg8DEwIfAhkCCgQ6Ag8BBAItAhwCGQIK +BQcBCwsoAAAEAQYCDgHSAgIFAABUAhkCMwGGAQIhAygAAgogXh8SAJQBegDGBRQEBQEsAhoCCgUR +AAAeAkEBGwACCiA1HwkAlAFIANgFFAIFCh0CCw0HAAAUAi0BBwACBIABmwF/AgCUASQCCRIQEwUU +EBMLFBATAhQREyEA0gYOAgwCCQIBiQYJ8AYQWwVcEFcCAglWEFMCVBFPAgYIAgwCCwAADgGTAQAA +kQECEAAAJAIJAhADBQYQBQsIEAcCChEJIQACBJABOo8BAZABI48BAQCUAWMAgAcOBAQBDAIMAgsC +CgQKAhACCgAADgFVAAAlAhABHwIPAAIEYHlfAQCUAX4AvAcSAjIYEwQdCgoAAA4BcAAAKwRJAwoA +AgpQW08gAPgHGQJDAgoDHwAABAECAg4BawIGAAAeAiwCHAMfAAIOQF4/AUAfPy0AngEsNhs1cgBE +GAYKAwriBBvhBAIOGgYKAw8LEAUtAAAiAkEBFAQVAy0AACwCGwFyAAIKMDYvATAGLxsAngFiAGgZ +BgcCFwYKAwYJGwAABAECAhMBRwICAABBBAYDGwACCnBsbx4AngGUAQCyARQECgEWAkIDHgAALwJH +AR4AugEUBAoBFgJCAx4AxgEUBAoBFgJCAx4AzgEUBAoBFgJCAx4A1gEUBAoBFgJCAx4A3gEUBAoB +FgJCAx4A6gEUBAoBFgJCAx4A8gEUBAoBFgJCAx4A/gEUBAoBFgJCAx4AhgIUBAoBFgJCAx4AAgpA +QD8HAJ4BUQCAAxQCFgIgAwcAACUCJQEHAAIKMDwvBwCeAU0AjgMUAhECIQMHAAAEAQICDgE3AgIA +ACACJgEHAAIOkAH0Ao8BAZABvQGPAQoAngGAARcWUApPNRgFOAtPNxgCFxoYAhcBUBw3swEXNhgK +ANAFGAIUBAICCAENBgMrCC4CJxAqCi8EEAcmAh8JpwUNAgnGBQrDBTXEBRC/BTfABQK5BRq6BQKz +BQGcBRwaDgIEAgUECwoKGwUIBQcFFGcCEc0FGwMbrgUKAAAEAQYCDgGtBAIFAACSAwKuAQEKAABG +AggBAgIQAQoCBAIHAwIEBAMFBhYCCgE1BQUKCwM3BQIGGgUCBgEGHAuzAQY2BQoAngFnALIGFAIF +AhQCJgIKBwoAAgTAAakDvwEBwAERAJ4BdRoFGcUCAM4GDgIJAgmhAQiiAQKbAQjaAQ45CgILAiCj +AgWkAgUGQDAEGRkCDgI7Ag8GCARPCgsCCQIPFwUBDAAADgGxAwAArQEEzAECNQMRAAAgAggBAgII +AUMEBQPFAgACDlCAAk8BUCtPCwCeATYXE1AITywYAjgLTygYAhcSUA43OxcrGAsAhgcgxQELyAEC +wQEEwAEF5wYKAgnGBQjDBSzmBgKhAQu/BSjiBgLbBhEGAZwFDrwBMQIK5QYWAxXkBgsAAOMBBCID +DwImAQsAACACCwECAgQBBQQTAggBLAMCCAsDKAMCBBIGDgk7BCsDCwACDuAB5gLfAQHgARHfAQoA +ngErUwwDClidAlcBBAxUJQDgBioCAbEGDDIKggZIBBECCAECAwQEDAIIAhgHDwhJBwUILAQBjQUB +nwEMsAYKDREDCgAAMgS7AQJwBQkCIAEKAAArBAwBCgGdAgYBAgwHJQACBPAB1APvAQHwAQnvAQHw +AQnvAQEAngF1GgUZ8wIAqAcOcAhtBwIIBA8CBQQSBgOLAgiMAgKFAgiIAgoGC40DBZADKwpALgga +DxkGAgQCBAQFAggCCAIRCAgHDQgLBH8CClkKBwoAAA4B3wMAAB0CLgGIAQT8AQMeAABOAggBAgII +ARUEBQPzAgACDmDAAl8BYCFfCgCeAd0BVwEEDFRKVwEEEFQ1ANAHJAQCGgMZJwQaAhwCEwIFBicM +Aw8FDwUgBQEFAQGJBwExDL4HJAIJARICCgIBjwYBnwEQsgcKFwsDCwMLAwoAAAQBBgIOAd0CAgUA +AOUBBE4DCQI0AQoAAN0BAgECDANKBgECEAc1AJ4BLQCiCBQEEgMHAKwIFAQSAwcAAgSAAYMCfwGA +AUt/AYABCX8BgAEJfwGAARIAngFYOAQ3EDgEN4oCANIIDgIEBAkIDwsFAhEQCiIDAgUEBusIBO4I +BoEDAYIDBQQDhQMB6wUE8ggIBAUECgMEAhMBBEEDRAsCAwMFBBoCBQwHAiACCkUNAgIGLgIFAgoR +CgcKEhIAAA4B7AIAAHMEXwIaAyICNwEPASACBgAAWAQEAwYCAQEIBgECBAeKAgACDrABoAKvAQGw +AQ6vARUAngHSAgDqCikCEQoCDgQNCQINAQUCCgIRAjcIAwsCBhsGBQsFCDoEAwsFECUdFAAABAEG +AhoBiwICDwEPAgUAAGwEEQKhAQIgBxQAAg4wzwEvATAJLxUAngH8AQCQCx0CCQIIAhICGwYOBhwC +FQILAiAEDwIKDwoPFAAABAEGAg4B3wECBQAAKQJ8AjkDHgACDnB/bywAngG5AQDQCxgECgIEAgQC +BARWgAEKjQErAAByBBwDKwACCuABat8BCACeAXwA3gskBCUBInwKfQcAAGYCDwEHAAIOkAEvjwEB +kAGkAY8BAZABN48BAZAB+wKPAQGQAUCPAR4AngFLQghBAkIICQsKEAkFNwVCLUEKOAUKAQkECgRB +bjgDN70DAOILIAIPCA8JCg4D/AMI+QMC+gMEBgTPDwvSDwcCCdMPBc4LBYgELfEDBQcF2wsF2g8B +2Q8E3g8EiwQFCgYEAgsPJAQwA0cIAgQCCgYCAgoGDwoOCAEzCYQBAtMMA6gMAy4EAgECBgYVNwM4 +DAIEAhIHCYsBBRAFSAU0E1UFAgQCHg4SAggCJAYIAggCJQIKBhoDFE0JTgUKDCgFChMFBWERDxIt +HgAABAEGAg4B2AUCBQAAggEEGwIhBe0BCOwBAkAJHgAASwIIAQICCAILARACBQMFAi0BCgYFAwEE +BAMEAWwIAgIDCQMIIAcDCCsHDwgTB+wBCB0HQQACDvAB9QXvAQHwAboK7wEeAJ4BKTgENzY4AzdY +OAM34gI4BDfoATgDN0Y4CDdCOAM3OzgFN64COA03SzgDN1I4AzfpATgENyE4BDcgOAQ34wIAmg0Y +BAQGBC0CCAMFAwIB1wwE1gwDBAIGCQIFAgsHCSgELwIIBQUDAgHXDAPaDAUGGwIEAgsHECoBKQsH +AggGBQQCAdcMA9oMBgYeAgQCCwcJHAgbBCwHKwcsBCsJMA4iAxsDDAMuBT8JBgQ9BQgQOAE3DQcF +CBA6ATkNBw0IIDwJCwUSIhAKAhsjA6cHCM4HAsUHAesFBNwMCkIFEgUCBRMHGgUEBRAHAQsCAwIJ +yQcEzgcICBQGJwQPAiFTBQwFSAoMCpUBBgUEAgHXDAPaDA4GHgIEAgsHBgUEAgHXDAjaDAoGHgIE +AgsHBgUEAgHXDAPaDAoGHgIEAgtUBLcNBdwMFWYEAgcBCwkQKwVADBQFUwM+BhoKKwUSBhEJBANf +BzADNwUIFWIBYRIHCQgQZAFjFgcFCBSAAQUMCgIFXQVYBYcBCIQBBVMDNQQCAdcMDdoMCgYmAgQC +CwcHBQQCAdcMA9oMEgYmAgQCCAcDbgZzBAIB1wwD2gwKBgUDBAQdAgQCCAcGChMFCAMFChMFCAMF +ChMFBUACAgMBBQICTQIICUgCTwoICUoCUQIIB1IDGwMMBgwFTQMFAwIB1wwE1gwDBAIGCQIEAgsN +AwIB1wwE1gwEBAcGBQIEAggNAwIB1wwE1gwEBAMGBQIEAggHBQoTBQUDBQoTBQUDBQoVBQU6Fj0F +ChMFCAMFChMFCAMFChMFCAMFChMFCAMFChMFCAMFChMFBhgeAAAEAQYCDgG/EAIFAAD2BARJAjsF +kQIIUgeiAwpoCaoBCq0CCR4AACACCQIEAScBBAYLAgMBPwUBBgsEDQIDATwJCAYEBQcKBwkECgkJ +KQ4VDQEODQIVDwEQDQItEVgUCBMCFAECBAcFBAURPhgEF5EBDgsMAws7AgsMCAs3AgsMAws3EQQi +BQEVH2YkByMDJBojASQSAhklASYWBRkfHiQIIwgkBQQNAz8CDAQDA0QFAx8GIAUCAwFdBiABHSMM +LAsrAi4TLQIwCS8RLAcGBAUdAgQGBAUcAgQGBAU1AR0BHysWEiABIAEgAyADIAMeAR4AAg4QrAEP +HwCeAdkBANwOIAgFBwUKIgIMAh0EDQkIDAICCAIeBAkbHgAABAEGAgwBvgECBQAAKgJWAjIDJwAC +FvACqgfvAgHwAsUG7wIfAJ4BpQ4A/g4mCBAFBAITDg0PBRwOEg0OLQIQAgwCKQQQCBgGDAIJAQQQ +CgIOAgQEEwUEEQMYCg4DCwYKFyMDJAoCGwIIJwUqBQ4EChhBA0INSAObAQUSCkIFBgYCBwIRAQQE +DjgFEwUjAiQFJQIICgYBhwoJigoFFgoFCQYRBh8KCQIJAgQCHgEEBB0CCAUFhQEFjAIFAQV5DgIV +ZAVjDAIYEBAQBC0CMgYKBBgDEwMDAl4MBA4CCQIQTQMTDAIKEAgWBhMKBhfxAQPyAQgWAxUCAxYa +BRkC7QED7gEMCAkCCBAFFgUVBRQFQwUUBQ4FDhwGDwIcAQMGAggHAgcCDAIRCwxpCwILAgwCESsW +lwEgAhUCEAIRFyACFQIQAhsCPgIRAhEXIAIVAg8CERMbAhoCDwISCx8AAAkBBgIXAfkNAgYAAI8C +BIECAvEBApQBAg4CrgECiQEBawkwCiYEMAVRCBMFLAkwCiUJKworCx8AANgEAgkBxAkAngEGAIoS +BgACLwCeAS8ArBIEBBkCBAIJBAUAAS8AAgRQf08BAJ4BhAEAyhITBkAIDwIKBAUCCQIKAAATAXEA +AEEEQwACClBUTwkA0BImAi8CCgMIAAAmBCABGQEIAAIOYHNfAWCaAV8UAJ4BsAIAhhMYBAcCBwYF +BwUIFhAEAgQCCAIIAhoCCheIAQISDxQAAGYEtgEDFAACBGBrXwEAngFwALoTDgYGBkkaCQIKAAAO +AWIAAEsCJQACCnBpbwgAngF7AMYTLwIFBBgIBQYZAgoVBwAALwQUATEBBwACBJABuAGPAQEAngGD +ASYKJTAA8BMOBgUIaCAHCAHpBgrwBh0ICQIKAAAOAa8BAABpBBkDBgQ1AACDAQIKATAAAg6QAYcB +jwEQAJ4BpQEA/hNFAhMOEAQFBh8CChsPAABFBCMCFAIaBw8AAg5A3gE/AUAwPwFAWj8NAJ4BhQMA +0hQYAg0CAgIeDA8IDgIKBgoCChQFDAUGCgIbAhkIGQIMGQoCGwIMGQoCEAINAhICDgQFAg81DAAA +JwRtAeUBAQwAAg7gAccF3wEpAJ4BNSYLJQImFSWTAhg2FwUYBhdrGAUXPBgFFyAYBRcBVwEEDFQX +VwEEDFQBVwEEEGwHFzIAuBUYKAoBBSMO+RQL/BQC9RQV+BQFAQoELAUECiQEoQEMBRIKlRUGBBwD +BAYHBgYFA/gUAwQC+xQG/hQPFggPBQgWAh4CFggFmxUFmBUGAwIJDwIKAhYOBZsVBZgVFAIHAgWb +FQWiFQGRFAGfAQy2FRYKAZ8VATEM1BUBoRUBMRAiB7wVClUoAAAEAQYCDgHhBQIFAABmBCAB5AEB +cwKaAQQKBQkGGwUJBigFKAAANQILAQICFQGTAgQ2AwUEBgNrBAUDPAQFAyAEBQMBBgECDAcXCgEC +DAsBDgECEAsHAzIAAgowPC8BMBovATAOLwEwCy8pAJ4BM0IDQRtCA0FRANoWFAIFIg0KBQkI7QsD +7gsCBg8ECvcLA/gLAgYMCA8tDA0oAAAEAQICDgGMAQIFAAAuBE8DKAAAMwIDARsEAwNRAAIEICQf +ASALHwEAngETQgNBHwCqFw4CBZsMA54MAgYRAwwAAA4BJwAADgInAAATAgMBHwACDtABgwjPAQHQ +ARLPAQHQAbgCzwEUAKIBKAYGBVoGBwWRAkYWRRBGAyMdJAUjDSFMBgUFBQYFBWsGBQVLBgUFNEYW +RQxGG0U8RhZFUAaUAgU3ANQBGAIQngwGnQwJehJ5AiAh+gEFIQUjCSsEbAWKCgf7CxFQCjoKBlUK +HwwOAggGCQIKAiAIFA4NFQcWBgwBoQMWpAMPBAGnAwOoBR2nBQW8BQ2VAgUlCiYHDwIVCiQFIwok +BRwFBgVxBSwEJAOmCgWzCgW0CgWJCgUGBXEFLAkkBTkFSwszChAKCAYvETgTwAsFiQoFBgVxBSwJ +JAWXAQIaEC4ELRikCwWJCgUGBXEFLAkkCiMJKwRsBb8DFsIDBwQFxQMbxgMCAhkBBQIFBBfLAxbO +Aw+XAQ4CDhwRXRMRAc4JBZgCCpUCepYCD5MCfM8JETESCRQAAAQBBgIOAdQKAgUAAJMCBGgCJAUB +BiUFLgbCAQHwAgMcCBsCjgECcwsUAAAoAgYBWgQHA5ECBhYFEAYDAh0BBQINB0wEBQMFBAUDawQF +A0sEBQM0ChYJDAwbCzwOFg1QEAUCCgF6BA8DfA83AAIOkAFpjwEBkAEJjwEBkAGqAo8BJgCiAS4G +BgVPBpcCBTgAhgQdAgciBBcFAgHgCQbVCRECAgQNBgkEEQQKJQoMAcQHBZgCCpUCfZYCD5MCfMUH +EhMmAAAEAQYCEwGwAwIFAABPBBoCBQUwCB0CjwEDYgUmAAAuAgYBTwQFAgoBfQQPA3wDOAACBDBQ +LwEAogFVAN4EDgINBAoCDgQMBAwCCgAADgFHAAAsBCkAAg5Aaz8BQEI/DgCiAR8GCQUOBhAFhAEA +/AQYAgfAAgm/AgUCCb4CELsCCAILBA4CCRwKBTACEiUOAAAoAkgBCgJCAQ4AAB8CCQEOBBADhAEA +Ag7gAYgB3wEB4AGABd8BAeABFt8BAeABDd8BAeABDd8BAeABDd8BAeABDd8BAeABDd8BAeABC98B +MgCiAacBPgM9KD4UPQM0BTMCPgMJDgoPCQUKKgkFMwU0CQoCPQM+CT0wPgs9Bz4DCRAKEAkFCi89 +BTQFMwo0BAoCPQM+BD1wMiAxAjIFMQ8yIDECMgUxBDIbMawBANgFGAIICg0KEKsBLawBBgoZAg4I +EI4FA4sFBkYKJw8CCcgJFKEJA6cGBY4GAsIJA88PDtIPBgIJ0w8F1g8q1Q8FqAYFpwYJ2g8CsQkD +tgkJzwkZGg0QCpIJC5sJBQMCpgkDzw8Q0g8HAgnTDwXWDy+dCQW3BgW4BgUJBa0GBNoPAqsJA7AJ +BLMJCgQhAgQCLQYUsQEgsgECsQEFsgELAgSzASC0AQKzAQW0AQSzARu2AQIBAhgOHBdJDhkOIw4X +DgkObgt7MgAABAEGAg4BlQcCBQAAogEEIgIOBUMIOQczCiEJKAxDCy4ONAkxDBwPGhAVDxYEfwMy +AAA9Ai0BPQQDAygGFAUDCgUJAgYDBg4FDwYFBSoIBQ0FCgUEBAcCBQMGCQUwCAsHBwgDAhABEAIF +AS8HBRAFDwoQBAcCBwMIBAdwEhsCBRMCFAUTDxQbAQURAhYFFQQSBAQXFawBAAIOYElfAWDuAV8o +AKYB7gIAUhgMBQEFAgUJBQQdFAUCCgsKAiADBAMIAjIBAwI2AQUCGgEFBgwBBQEYCygAACcCIgLA +AQEqAhMDKAACCjBPLwkApgE4XwEEDFwdAIYBFAIEAgYCEAIEBQUIAS0BMQxkCwIKDwgAAEAEGgMI +AAA4AgECDAMdAAIKIDofCACmAS5fAQQMXBEAmgEUAgQCEAIFAgFAAZ8BDGQKCwcAADYCBQERAAAu +AgECDAMRAAIOcM4BbwFwOG8BcAlvAnAJbygApgHeAVgBFS1BRgCuAR0CEQYKDjsKKhUFFgoCJwIK +CwGpAQEcLZABChULHgkhKAAABAEGAhMBsAICBQAAMwJ0AjYDHQYcBQ4GBgUoAADeAQIBAi0DRgCm +ATYA1gEUAhECCgMHAAAgAg8BBwDeARQCEQIKAwcAAgowOy8RAKYBVgDmARYCAgITBBEECgsQAAAE +AQICDgFAAgIAACQCGAEaAAIOYGBfAWAfXwFgdl8BYLYCXxcApgHTBAD2ARgEFAYcBgoCEwIKBxYC +DQ4SAgUECAISAgVKB0MCBAUBDAICCAM4BGkRAgoyAwIEAxAKAwIEAxoOAwIEAQYGBAIGAgMCBAwD +FAQTBQIJAwMGDAIMBwYMCQIFBgcCBQQFAQMIFwIHAQoCDQIDASQCEQISAgqLARYAAAQBBgIOAbYE +AgUAAF4CcQEoAsYCARYAAgowaS8jAKYBlgEAhgMUAlYCCgMiAAAEAQICDgF9AgUAACACVAEiAAIW +kAK4AY8CAZACDo8CFACmAfEBAI4DJgItBAoCAwEFAi0CBgonAhABDhMUAAAJAQYCFwHGAQIFAAC6 +AQIjARQAAgowOy8SAKYBVwCoAxQCBQEFBBECCAQFAgoLEQAAKgIcAREAAhaQAtEBjwIBkAIVjwIU +AKYBkQIAvAMmBC0ECgoEBwMBBQIXAhYKCgILAgQCCwIeAhADCgMLFxQAAAkBBgIXAeYBAgUAANMB +AioBFACmATQA4AMUAgUCCgMRAOYDFAIFAgoDEQACCpABWY8BHACmAX8A7gMZAhnFAwoCCgIKwgMK +AgoDGwAABAECAhMBZAICAABVBA8DGwAAMgIeAS8AAg5QjwFPKQCmAcYBAPYDGAQKAw8EWQIFGQUc +CgcoAAAEAQYCDgGpAQIFAAAxAl4CDwMoAACPAQIFATIAAg6wAe8DrwEpAKYBxQI6AzkGOgU50wEA +nAQnAgUCBQIFAg0CMAINAgUCDwQnCQoSBREFEAoCDQIHAgUGEwINAhgCDwYMwgYDvwYGwAYFvQZ/ +BgsCEgIFAgpBKAAABAEGAg4BiQQCBQAAJwKsAgIcAiADTgYhBygAAMUCAgMBBgIFAdMBAAIOsAG/ +BK8BAbAB7QGvAQ8AqAGYARADDwsQBw8jQBY/5AQApAIpAgUIEgwLEAsGBwwOAh36RATjRAHkRAZL +BE4Bx0MD+kIHAgSvQgflARwGB9MCFtwCBwIJCAkCDgIHBhEGBQIrDAUELAIJAg4IDgYOCA4KBQIO +BgwIBwQMBAcCBQQSDBASDhcZKgoCGAYZBgobBQgBaAxvEQIQSREHEQURBxEtEQ8hVQ4AAAQBBgIa +AYkGAg8BCQIFAAB2BFoC7AQFDgAAiAECBAEBAgYCBAEBBAMBCwQHByMKFgmzAwwMC6UBAAIKECMP +CACoATUAogMWAgUCCgQJBwcAAAQBAgIMASECAgAAGwIKARAAqAE2AMwEFAIRAgoDBwAAHQISAQcA +Ag5Q2gFPCgCoAT9AFz8PYQEEDF6AAQDUBBgCJQYCsQQXyAQPjwQBMQyuBAkGDwIBhAEifwkCIAsS +CwoAAAQBBgIOAdUBAgUAAD8CJgEIAnsBCgAAPwIXAQ8EAQIMBRkIIgdFAAIEIB8fAQCoASQAggUO +BAwCCgAADgEWAAAVAg8AAg4w0QEvAjASLzsAqAEYDgQNAQ4QDXYOJw1kALwFGOoBBOEBAeQBBgIK +4wEHAgGsCAupCBMGEAIHAQkGAwUIAgwEAwEHAgcCBwIB3AEPAhAECN0BDAILExIPOwAA0QEEIgM7 +AAAYAgQBAQIQAQgECwNjBicFZACoAXkA8AUUAkAGCgcbAABCBBwDGwACCkAlPwgAqAE3APIFHAIK +AgoDBwAABAECAg4BIQICAAAhAg8BBwACBNAB/wHPAQHQAe4DAKgBDg4EDQEOCg0HDgYNCg4FAgUP +AV0MXhVhBWJoDioNrwJhAQQMXrIBDgUNCAD+BQ6oAQSVAQGYAQYCBJcBB5gBBpUBCpYBBfMCBeAB +AeUFDOQFFbEFBdAFDgIjAhwCEwIHBgF2EwIPBAh5DQURAwsDBRknBBQCCAESAgsCIAcFCHsEAcME +AZ8BDOgFEwKfAYoBBYMBCAAADgHkBQAARgSGAQNEAqQBBHEFCQhWAW4AAA4CBAEBAgoBBwIGAQoC +BQIFAwEIDAcVBgUFaAoqCa8CDAECDA2yAQoFCQgAAgTQAeoCzwEB0AHKAwCoAWsOBA0BDhANBxAF +AQYNdA4FDTIOKA3LAWEBBAxeSV0MXgUOBQ0FYQVikwEAygYOAgsGDQYLBgsGCwYLCBQyBQQELwEw +BwIJLwfDAgX0AgYtIChIJwcoBRIFETECARARAg8ECBMKCwMZIQQjAhsCHwIFBicMBA8FEAUBBQEB +twYBMQzsBhsCDAELAhYCAd0GDOIGBRIFEQXBBQWoBQsDCxERBxEFEQUWBREFEQUSAAAOAasGAADX +AQRlA/wBBk0FCAQfBI0BAABrAgQBAQIQAQcEBQEGAXQGBQUyBigFywEIAQIMCUkODA0FBgUFBQwF +C5MBAKgBLQDGBxQCEgEHAM4HFAISAQcAAgRQRE8BAKgBHEAjPwoA5gcOBA6xByO0BwoAAA4BOwAA +LQIcAAAcAiMBCgD4Bw4EDsMHI8YHCgACBDAcAKgBIACECA4CEgAADgESAAAaAgYAAg6gAaACnwEB +oAEZnwEUAKgBLmEBBAxenwGhAQmiAQWhARuiARJhAQQPXjcAyggYAhAKBQEB8QcBMQymCIIBAhwC +AZcICZYIBZMIDAQPlggRAgH9BgGfAQ+gCAoJBwkSAxQAADYCowEBFgQuAisFFAAALgIBAgwDnwEG +CQUFBhsFEggBAg8JNwACDlCAAU8XAKgBHWEBYgddBV5LYQEED14gAIQJHaEIAaQIB9UIBdgIJQIN +ARgGAasHAZ8BD84ICgsWAAAEAQYCDgGGAQIHAAAlAjUCGgMJBhIFFgAAHQIBAQcEBQNLBgECDwcg +AAIKUGlPFQCoAYgBAJwJGQIHMwUCDjQHAgECAysEJAMICAMJBAUDBScFKAUICgsUAAAEAQICDgFv +AgUAAEsEHwMeAAAgAhMBCwQEAx4EBQMjAAIOgAHNAn8BgAEOfwoAqAExDgcgBx8EDTUOByAHHwgN +HyweK6kBAL4JGAwJDAICAgEM3wgHUwdUBOAICggfAgzpCAdTB1QI7AgfuQQewAQFAQgCKAIGCgsI +DQINAg0EDQQNAgohDikKAAAEAQYCDgHXAgIFAACOAQRNAY8BAQoAADECBwIHAQQBNQYHAgcBCAUf +Ch4JqQEAAg6wAcQErwEBsAFJrwEOAKgBLjgdN14cAS8QFAQTHxQdEwYUC2EBBAyeARY/IiwLKwgs +AysELAIrHmEBBAxekwIAngoYLg8UB/sBBwIFARH+AQUCBQIBkgIUjwIbAgUCBQIFAgUCBQQLpAUB +8QkQzgQEzQQf0AQXDgUCAd0FBuAFBQIFBAGtCgExDAUW6AoHAgoCEfcEC/gECPcEA/gEBPcEAvgE +BAYZBgG/CQGfAQzwCgsCBwIHAh8KCgY3BAoGLwQKHQkCEQEdGRJ/DgAABAEGAg4BjQUCBQAANQQW +AqkBA5UBAQkCtwEBHwguBQYBDgAALgIdAQsEFAM/BgECEAcECB8HHQoGCQsMAQIMAhYPIhILEQgS +AxEEEgIRHhQBAgwVkwIAAg5AKz8BQFM/CgCoAZcBAOgLGAYG+jwO+TwECAoHBAI9AhIJCgAABAEG +Ag4BegIFAAA+Ak8BCgAAHgIOAWsAAgpAOj8BQBI/BwCoAV4AggwUBhYIBwIFAg8JEgcHAAAEAQIC +DgFIAgIAADECJgEHAAIOcLQDbyMAqAFKYQEEDF5DowELpAEwowEKpAFroQEkogEBYQEEDF5pAJwM +GCwFFQUTBAYIAhsGAckLATEMgAwOAg4EEQYW9QcL9gcLAiX3Bwr4BwsCDAIKBggCDgIOCiUIAZsM +CQIMBA+YDAH9CgGfAQykDAkCNAQKTyIAAAQBBgIOAcQDAgkAAEQCBQEJAvcBARECXwEsAABKAgEC +DANDBgsFMAgKB2sKJAkBDAECDA1pAAIOoAG8AZ8BAaABoQKfAScAqAFADgQNAQ4GDQQOBA0KDgUN +NA4rDdICAIANGCIEHQUBCQIFcAVRBRcBagbLBgTqBQHpBQbsBQTpBQTqBQUDBeUFBfYFDwIfAgUC +Ae8FFAIPBAjsBQoRAbkBBZgCCpUCdJYCD5MCfLgBEhcnAAAEAQYCDgH2AwIFAAAqAg8BMwQfAjYF +JggbAogBAmILJwAALwIFAQYCBgIEAwEEBgMEBAQDCgQFAzQGKwULCAUCCgF0BA8DfAc5AAIOMK0B +Lw8AqAHKAQDCDRgCEAgCEBkPEwQKAg4ECQwbAgUCGwIKJQ4AADECiwEBDgACDoABV38BgAGmBn8k +AKgBsAcAgg4gDAUGFwoKDwMSEQYCDAoJaQIBzQIFmAIPlQJ2lgIPkwJxzAIRHW8CAbMCBZgCD5UC +gAGWAg+TAnyyAhIPJAAABAEGAg4BkwcCBQAAZgKKAQIbAoUBAlsFkAEIIAKPAQNiByQAANABAgUC +DwF2BA8DcQGBAQgFAg8BgAEEDwN8BzYAAg4wPy8BMF4vJACoAdABALoOIAQICAoLAw4NCAwITAIS +HyQAAAQBBgIOAbMBAgUAAE4EXgMkAAIEgAHABX8BgAGBAQCoAcUBQBY/DEAdPyRAFj9AQBg/amkW +agppkAFqFWkKancA7A4TBAgBEQI1Hg0RBAICEBYhAyIcAhIGCusOFu4OBwQF8Q4d8g4SChL7Dhb+ +DgwKBQQLAgcEDwQNAgGXDxiaDwkIGQINCggFCQYPAgkIBwILrw4FAhGuDgqdDgYGEwIdFjIKKPgN +CwYK/Q0KsA0FAh4BEgEFAh8BAwIFCxYAAA4BuAYAAE8CrgIBAQK9AgEPBBEBYQIKAADFAQIWAQwE +HQMkBhYFQAgYB2oKFgkKCpABCRUKCgl3AAIOUHhPCgCoAZABAPAOKAJHAhcDCgAABAEGAg4BcwIF +AAAoAl4BCgACCjBILwEwEy8kAKgBigEAnBAUAhYIAwkDChkECgkTAyQAAF8EBwMkAAIKMDcvATAS +LyEAqAF1ALQQFAIKAQMIIQMSAyEAAAQBAgIOAV8CAgAATgQGAyEAAg6wAYAErwEBsAFMrwEPAKgB +L2EBBAxeigJhAWIKXQxeKGELYkNhAQQRXgFhAQQMXncAmBIYAgQIDQcFEAHHEQExDPwRDAIOAgUE +FwIGBDUCAwIFAwMEFAMLBAUCCQUKCAgCDgkKDgUNCw4DAgsSBwIGCQYCBQ4BgREBgBEKnxIMoBIG +CAQOBQcYAgHfEAvyEAkGJQwKEAoFAb0SATER8hIBvxIBMQz2EgUCCAEFCgoHCx0EAgQaFBsIHAYP +BXESCw8AAAQBBgIOAc0EAgUAADcEuwECSAMLARMCLAFXCCoHEAJGAQ8AAC8CAQIMA4oCBgEFCggM +BygKCwlDDAECEQ0BEAECDBF3AAIOsAGeA68BAbABEq8BEwCoARwOBA0BDg8lCBgdYQEED15VYQEE +DF4ZEAUPIhAHDyJAGz8KRhhFBg4rDTQAtBMcjQwElAwBkQwGAgmZBQisEQICEAIKBAHhEgExD5gT +BgIKBA0EDgIKAgkCCgIMBAH7EQGfAQygEwcECgQEAgS1DwW2DwUCAQIIAhMGAbsPB74PDgYTCgHf +ExuKEwdYAgIBsQQYvgQFBAHnDBQCDwQI5gwPLRI/EwAABAEGAg4BtQMCBQAAPAQYAwkEVwIFBQkG +KAUrCBoHAQQkAxQEUQEGARMAABwCBAEBAg8CCAMdBgECDwdVCgECDAsZDgUNIhAHDyISGxEKFBgT +BhYrFTQAAgRQjgFPAQCoAZMBAPgUDgIIBA0CAhIJAgUCCQQOAhQIEgYEAgUtC0AFAgoAAA4BhQEA +AHQCHwACDkDIAT8BQBI/CgCoAfMBAMoVGAIEBA0DBRgEAgkCCQQSAgUIEgIMBhUCDAYOAgwCFAQF +Ago9EgcKAAAEAQYCDgHWAQIFAABEBIQBASEBCgACCiA4HwgAqAFKAKAWFAgSAgcCBQQHAgoRBwAA +BAECAg4BNAICAAAtAhYBBwACBDBOLwEAqAFTAL4WDgIFBAUDBAQOCBYCCQIKAAAOAUUAACUEJAMK +AAIOQIMDPwFAKj8BQF8/EwCoAUATBRRDXQwDDmKGAWEBBAxeI2EBBAxeDWEBBAxeWmEBBAxeDWEB +BBFeKgDuFhgCBAYQBwQEBS4KAgGxEQW4EQ4CDAoYCwkMBwgBixcMMg7cFhMCBQIdUwtgAgoLChcC +BwEOAgwEAYUWAZ8BDKoXGAYKDAGbFwExDNAXBwIFAgGhFgGfAQysFgeyAQIGCg4hAgprETsKAgGt +FgExDOIWBwIFAgGzFQGfARHWFgUCEisTAAAEAQYCDgGSBAIFAAA6BE0DCASYAQMJBFgDGQJ7ARMA +AEACBQFDBgwBDgOGAQgBAgwJIwwBAgwNDRABAgwRWhQBAgwVDRgBAhEZKgACDsAB3gW/AQHAATm/ +ARQAqAEdDgkNAQ4GDQwODgIFcQEEDF7RAWEBYgpdEF7sAWELYjthAQQNXiVhAQQMXgEOKw1XANAY +GBIFuxEJrBEBqREGrBELBAGtEQ7zAgXTAwExDKoYDQYOAigGJAIFAhEGBRAVAhgBBQQNAgYFBwYC +CgGvFwGuFwrNGBDWGA8ILQIDAg4DAwQRAwgEAgMIBAUDBQYJBQUICAINBAgCCA8SEAUfBioEFAUH +FQIBtRcLwhcNBiACDQoBgxkBMQ24GSQCAYcYAZ8BDKoZAbUSFAIPBAiyEgoPEQcReRcLFAAABAEG +Ag4BnQYCBQAAVATEAQIRBRMEFgSBAQIdAiYDGgdPCCoHCQgxBxYOLQ0UAAAdAgkBAQIGAQwCDgIF +AgECDAfRAQoBCQoMEAvsAQ4LDTsQAQINESUUAQIMFQEYKxdXAAIOIIEBHwEgCR8MAKgBS2EBBAxe +IWEBBAxeHwCeHBgCCwgYBg8CAc0bATEMghwPAgUCDAQB1xoBnwEM+hsKEQoLCwAABAEGAg4BhwEC +BgAASAIwAQkCDwEVAABLAgECDAMhBgECDAcfAAIOoAGXA58BAaAB2AGfASgAqAExDgYNLWEBBAxe +ggIOJg3TAWEBBAxeLQDoHBgIBTwFAQU/CQIBwxUGxhUOAgVSBRUFAQUvCgIBmRwBMQzQHBQoDAIo +AQUECggJAicIAwcCBCgEIAQaAgYIBQMJkRYPAg8ECJAWDUsOAgkCBwISAQoCBQIFAgUCAgUFEDwG +ERUFFgUEHgIBvxsBnwEM5hwFPygAAAQBBgIOAYkFAgUAAEUEHgMJBFcCnAECvAECUAkJBAoDKAAA +MQIGAS0EAQIMBYICCCYH0wEKAQIMCy0AAgowKS8IAKgBOwCWHRgCEgIKAwcAAgRwgANvAXAIAKgB +OUAjPyMcAS80FNkBAJYeDgIUDhftHSPwHRkWCQIBpQ4B8Qk0mhgHDA8SDwIGAgeKAwqDAwwWGQQQ +AgoCCgYSAgUGGwIQAgpzCAAADgH/AgAASgK5AQJjAScAADkCIwEjBAECNAUyCAoHnQEAAgowTS8B +MB0vBwCoAXwAqB8UAgwCBAEGAgIMCgIBtAINsQIFAgUGChUEAgkBEAUHAAAxAh0BDgIZAQcAADcC +DQE4AAIOYMYCXwsAqAHeARAKDwEQBw8+EAwPJQDKHxgMFQIPAg4CBAIIAggCBwIIAgQCCwoPAiwC +IAIGAgHnGwrqGwGdGwegGxsKCAwQBgoCAb0bDMAbBgIBzgEKywEKVwoAAAQBBgIOAcICAgUAACMC +DwJUAogBAUcDCgAA3gECCgEBBAcDPgYMBQcICgcUAAIOgAH0AX8LAKgBbhMFFBEQDA80HAEvNBQU +ANQgGAgJBhYCEwIQDAwCBwIBhRsFihsKAgYCAZ8cDKQcGgYBZg9hCtsQAYcKNOYaCjkKAAAEAQYC +DgHwAQIFAAAyBLUBARwBCgAAbgIFAREEDAMbBg8FCggBAjQJFAACBDDNAS8BAKgBTjAeLyATEhQL +ExIUFwCwIRQIAgIFDAIVAhYECwcCBggFEwIUAgIECBAGAaMhHqYhAgQdBgGfGxKGGwUcBSMB/RoS +hhsFBQUcDQAAVQI2AQECHAEBAikAAE4CHgEgBBIDCwYSBRcAAg5Q0wFPAVAOTwFQFU8oAKgBOxAH +DzthAQQMXh0QBw8dYQEEDF5WALIiGAITAg8CAfsdB/4dDgIVBwUIEhgB8SEBMQymIgkGEwIB/x0H +gh4JAgcCDAQBhSEBnwEMqCIKBAUCChcVJSgAACYEVgMJBkACBQcJCDMHKAAAOwIHATsEAQIMBR0I +BwcdCgECDAtWAAIOcKkBbwFwMG8BcBJvFACoAUkQBQ/BAQD0IhgCDgIPAhEGA+0eBfAeDMMbEMYb +BQgMAicCDAIKBgUBDAIKAgwCCh8SBxQAAHYEGgI0AwoELQUUAABJAgUBDAQQA6UBAAIOUGdPAVBP +TwsAqAEYDgQNAQ4KDR8OJg0KDgUNFg4rDRQAqCMYgRwEkBwBjRwGAgSOHB4CAYMcDwIPBAiAHAqR +HAWWHBUCAYscFAIPBAiIHAobCgAAiwEEMQMUAAAYAgQBAQIKAR8EJgMKAgUBFgYrBRQAAgSQAcIC +jwEBkAELjwEBkAETAKgBNxwBLzQUGWEBBAheXmEBBAVeChwBLzUUNADqIw4CBAIWAQUMCQIB3xMB +8Qk01B0HAhKbIwExCNAjFQIFAg0cBxoUAgwEEN0iAZ8BBYAkCpEUAYcKNdodC0IKUwwqEwAADgHY +AgAAWgRgAjMCcgUHAAA3AgECNAMZBgECCAdeCgECBQsKDgECNQ80AKgBGADWJAUCEgIBAAIOMOYB +LwoAqAEYYQFiB10FXgxhAQQOXgphC2IBYQEEDl4RYQEEDl4DYQEEDF4REAUPDF0OXgVhAmIHEAUP +JgD2JBiTJAGWJAfHJAXKJAYCBQIBmyMBnwEOwiQCGgcCAY0jC5AjAb8jAZ8BDuIkDAIFwyQBMQ7W +JAIUAbckATEM1iQQAgGvIAWyIAsCAckkDswkBasjAq4jB7cgBbogCwIFBQwXCgAABAEGAg4B4QEC +BQAAIAJWAQgCCQEJAhUBGQQ2AwoAABgCAQEHBAUDDAYBAg4HCgoLCQEMAQIODREQAQIOEQMUAQIM +FREYBRcMHA4bBRoCGQceBR0mAAIOQJwBPwFAND8LAKgBSGEBBAxeEWEBBAxedwCyJRgCBAQNBg4G +Cw8FGAHpJAExDJ4lEAIB7SMBnwEMkCUFAhUCFAIKEREFEQUSBwsAAFAEZwEoAQsAAEgCAQIMAxEG +AQIMB3cAAgwAqAEMAOAlCwIBAAEMAAIOgAH2AX8BgAG0AX8BgAFmfwGAAWl/HQCoASEOBA0BDgYN +AQ4JbwEEDl5RYQEEDl4rDhcNAg4SDQoQBQ8BYQEEDF47YQEEEV4gDisNIRAMDw0OLQ2QAQD8JSHV +HgT4HgH1Hgb4HgH1HgnHBgExDvIlCgIFAglqA08BmjMHAgUCDgIGAwPLMgVNAhoKAgH7JAGfAQ6q +JioGAbsfFAIDrB8Cqx8KBAi4HwqbIgWgIgGVJQGfAQy4JhIGDoMBCooBAso1DMk1Al0BxSQBnwER +uiUHLgIGFggBgR8UAg8ECP4eCgcRXAbRIgzWIgwGAeEfFgIPBAjeHwq4NQYCBwIHAhACCNM1AggC +BwUCEQURBRJ/HQAABAEGAg4BigUCBQAAPgQRAhEFKwg/ATEFGAoxCRcGKQVCDB4FMwVLDCgLHQAA +IQIEAQECBgEBAgkCAQIOBRwIIwcSCgECDgsrDhcNAg4SDQoQBQ8BEgECDBMsFgwVAxgBAhEZIBwr +GyEeDB0NIC0fChYsFVoAAg5AwAE/AUBLPwFAWj8BQLIBPwFAVz8BQCM/AUAQPxoAqAFfMyELB0AC +ZQomGkBvYQEEDF42YQEEDF4vPgU9Mz4FPS5hAQQMXhY+IT0RYQEEDF4aYQEEDF5AAKQnGAoGpjUG +AgcCCgIOqzUTCgnLIiGMEAfAEgLFJgqMFAIGGLQSDwIJAgoLBRQSEwMUGwINAgoEAekmATEMnicJ +AgwCDwIFAgwEAfclAZ8BDJonCgQOLwMwFJscBZwcBAIPAg8CBQIMoxwFqhwODh8CAZsmAZ8BDL4n +DAIKvRwHAgcCE8QcCAIIAgGtJgGfAQzSJwoCBQQKKQGNJgGfAQywJwwCCkMHAgoNGQAABAEGAg4B +sgUCBQAAvgEEXQMJAjUCBQMJBEoBVAEJAgwCDwMsBCIDEwIMAiADGQAAHgIlARwGIQEHAwIICgMa +A28KAQIMCzYOAQIMDy8SBREzEgURLhQBAgwVFhIhEREYAQIMGRocAQIMHUAAAgowQy8BMBUvATAJ +LwgAoigUAgoIJgIKBAwCCg0KAwcAAFUCDwERAAIOkAGlAY8BAZABiQOPAQ0AqAH/ARAFD8YCALwo +GAIEBCADBQoKBAUCBQQLBAUCEOMaBuYaCwoQAhQCCvMaBOgaNwIQ0SQF0AcFmAIKlQJ8lgIPkwJ8 +gB0RGxIHDQAASwTUAQIcAo8BBXMBDQAAdQIGATkCBAFHBAUCBQIKAXwEDwN8BTAAAg5AYD8BQCg/ +GACoATAQBQ8REAwPXQDyKBgCBwQBAgYGCqckBbAkCwIFAgHPJAzSJA4CBQIKDxYFEgkYAAA6BB8C +PgUYAAAwAgUBEQQMA10AAg5ApQE/AUAjPw4AqAFbYQEEDF4tYQEED15AAJopGAIIBAYGDQIHBhYI +CgIB1SgBMQyKKQwCDwIFAgwEAeEnAZ8BD4QpBQIKFxENEgcOAAAEAQYCDgHEAQIJAABQAhMCLAEF +AQkCOgEOAABbAgECDAMtBgECDwdAAAIOQLcCPyAAqAHoARwEGwEcBhtyAN4pGAIJCCQJBAwjAhkC +EAIHAg4TCBYCDAULAgITCgQBDgIIhyUEiiUBiSUGkCUJBhACDAQFBh8CCj0fAAAEAQYCDgHGAgIH +AAB7BJgBARcEHAUfAADoAQIEAQECBgFyAAIW0APrDM8DAdADbc8DAdADEc8DAdADD88DAdADkwHP +AwHQA3DPAwHQA4kBzwMB0APhAc8DCwCoAVEQCA+rAWEBBAxeHWEBBAxeERcIGPMBMycLB0ACZQwm +FUAoZQVmfGEBBAxeG2EBBAxeMWEBBAxe/QEXCBh3QBY/XkAWPyxhAQQMXg5hAQQMXnYQCA+cAhAI +D3wXCxhDYQEEDF5gZQwCCwEkZhgQCA9LAKQqJgIIDggNBA4HAgkCBQICgyYIiCYLAgUGGQQSAgoC +DLgBCrEBEQIfCBYKCQIBhyoBMQy8KhwCAYspAZ8BDK4qEd8oCPgoKQINjDcDizcGIgwBBgItWgoh +AjUNAgcCEAYKBAkDAwwIChY4CCEJ9yYnjBAH7BYC8SoMjBQCBhPgFg0GChwKIQQCA48rBdArLQYn +AicGAdErATEMhiwaAgHVKgGfAQz4KwUEDQoWBggCAe0qAZ8BDLosGQIGAgcCGxYwAgUCBQITAhAC +BQgJAgkoNAwIYwVYAnIFtSwIzCszAg0CEgYNCAUCClYIUwHHLRa0LQdhBXoLwQMIwgMDAgMQCQcT +CBECDN8tFuItCgIIDQKWNAaHNAIGCgIFBAG3LQExDOwtDQIBuywBnwEM3i0RBgbwMwjtMwarAQe+ +AQICEwIWiQQIhgQF+QIQMwGwNwGvXgiyXgUCDLE3CgIXAgkCDwQaJRIREMoCEAQFAhMCEAYTAhcC +CQIPBBpoAZI0Aa9eCLJeBQIMkzQKAhoCCQIPBBoTDjACCwW1LAu2LAgCBwIKAgWXBAueBAUCBZMC +DwIB3yoBnwEMgiwapAERBRNdETkR8SsHAgUSCwsBAgMCIIQrCQILAgTfJwjiJw8CCQIPBBrRAQoA +AAkBBgIXAeISAgUAAEoEGgKfAQUJBioCFgczBsgBBYICCh8JCQQXBhkCfgILCS0KQQkZA38EngED +CAQPAwkEGwwWC1MDMBJgESAUTQI3FSASVg0QAyQEHAEWFisVnwEQNxEKAABRAggBqwEEAQIMBR0I +AQIMCREMCAs2DgMNugESJwEHDwIUDAMVDygWBRV8GAECDBkbHAECDB0xIAECDCH9ASQII3cmFiVe +KBYnFCoGKRIsAQIMLQ4wAQIMMRc0CDNWNgECCAERNYoCOgECCAEROWs+Cz1DQAECDEFgFgwuCy0k +FRhGCEVLAAIOMC8vATB5LwEwCy8BMA4vCwCoAWUXCBhwAOIuGAIKBgvmLQLpLQ/sLQYCBgIHAgoC +CO0tAtssCOIsIQIMojMDoTMCAgoCDwYMDw8LCgAABAEGAg4BwAECBQAAkAECQwEKAAAtAgIBDwIl +AQIECAMtBgMFQAACDvABkALvAQHwAcwD7wEB8AEY7wEB8AEY7wEB8AEy7wEVAKgBIxALDxwuRS0k +EAMPrwUAkC8YAgvbKgvmKgUSA1QKZQqfLQQMDgYHAwoCFwIHAgSQYgcEDgEJnzQDoDQD618D7F8K +AgwBAwIQhzUFiDUEgTUWAwsEAnIUdQOaNQm3NAOuNAqLNQYCDQgOlDUHkzUNAgkPBDIQqCsGBBir +Kw4jBVYKLwsEFywILwIKAiYFJQIJBTAFExMUDwEKUQVUBWMFYA7CNATHCQ8EFP0qAgQPARJhBWIF +Ag9TBYg1BAIYtTQbERlHGZA1B7sJEM8rCPg0DQEGkTUVAAAEAQYCDgHHBgIGAADLAwTCAQIoBWkC +MgEVAAAjAgsBHARFAh4FAwYDAwMEKQUFBgQFOggJBwMKCgkhDAcLKg4eDaUBEAQCIxFBCBwHTQgH +CggDCA0IBhMFFQACDkB1PwFAXD8BQAs/RgCoAagBYQEEDF4LYQEEDF5lAKQwItgqAtUqD9IqDQQU +0yoGqCwGCg+xLAkaDJAsBwIHAgcCDq0sAccvATEM/C8KAgHLLgGfAQzuLwoCCsoqC9cqRgAABAEG +AhgBiwICBQAAsAEEGAIPBQ8EBgNGAAAiAgIBDwIhAQYEFQMVBCMDAQYBAgwHCwoBAgwLFAILAUYA +AgQwigEvATALAKgBRD4hPTUAzjAYrioCqyoGBgMFCagqBgQQqSoCjSUJAgcCEYwlAwIYAwIQDZoq +CwAAGAGCAQAAlAEEBgAAGAICARICFgECBCEDKgILAAIOQJIBPwFAGz8BQA0/AUANPwFAcj8BQCY/ +CwCoAShlCiYbQBBhAQQMXhNlAmYHYQEEDF4PYQEEDF4tZQwCCwEEZgNlGGYVYQEEDF4hYQEED14Y +APAwGAYQgzAKjBQCBhn4Gw8qAcMwATEM+DAFAgUOCeEwAuQwBgIB2S8BnwEM/DAOEQHJLwGfAQzs +MA4vDgUOUAPvMAcCBRILCwECA8YwA8MwGOQwBQEFHQUcBQ4B7y8BnwEMlDEYDQgCAecvAZ8BD4ox +DlsKAAAEAQYCDgHgAgIFAABlAhgBAgIiAQkChQECIAEkAQoAACgECgEbARAGAQIMBxMKAgkHDAEC +DA0PEAECDBEtCgwKCwkECQMKGAkVFgECDBchGgECDxsYAAIKEDkPEgCoAVUA4jESAgwKBwIKAgcM +BQYJIREAAAQBAgIMAUECAgAALwIMARoAAgowRC8BMCgvBwCoAX4AiDIUAgQCCgYHAhICBAwGAgoL +FgkSBQcAAAQBAgIOAWgCAgAAPwI4AQcAAg7wATbvAQHwAbgC7wEB8AGSAu8BAfABZe8BGQCoAUgQ +DA81EAUPERAFDwEQBQ8xYQEEDF4QEAMPQGEBBAxeNxAFD0kQAw8JEAsPCxAFDwZhAQQMXhAQAw9A +YQEEDF6oAQC4MhgCCNwvA9svAgEICAkCBQUKFAPDLgzKLgcGEgUMBAgQCNcuBZovBVUFDgYCAdMu +BdYuAdUuBdguBxAPAgUQBgIGAQQGBRMBmzIBMQzQMgGmJwqgBwW5XQPAXQsCDAIJBAcEB68HCwIG +pycBnzEBnwEMwjIBEwwCCQEWFgq0LgGnXQWALwiALgj/LQICAdAuAQIFAgwCBQIJ1S4BmC4LAgqd +XQOgXQmfXQukXQKbLgnjLgXcLgUKAbkyATEM7jIBiCcKoAcFuV0DwF0LAgwCCQQHBAevBwsCBokn +Ab0xAZ8BDOAyATEFIgMhAqIuBuctAgINBAo/BQIJAQg0BTMONAgIBQcCYQUCCgEMDggNCgkZAAAE +AQYCDgHyBQIFAABoBHIDCQYvBTEGMQWOAQgvBzEIEAcQBgUFEwg5AiMJGQAAIAIDASUEDAM1BgUF +EQYFBQEIBQcxCgECDAsBDgoCBQIDAS4BEQ0BFAECDBUBGCsXChoBAgUbCB4IHQMgIB8BGhUIAwcJ +CggHAwECGQkmBSUGKAECDCkBLAoCBQIDAS4BESsBMgECDDMBNgU1AzYCAgY3GTYWNQU2DjVVAAIO +gAGGB38BgAFGfw8AqAGiARAFD5UCYQEEDF4iYQEEDF60AWEBBAxeQBADDwoQDQ8BEAcPB2EBBAxe +C2EBBAxeowEAuDMYAgQEDQMFDAoCBQIaChfCAQUCBbkBCwIJBA0ICbUvBbgvBQxACAcSEgIFAgUB +BQQRAgwDCAwDCwoMDmYEYxwCBmILWQUsBSsCCCoCAdczATEMjDQhAgHbMgGfAQzINAUtBRcOAhVE +BS0FDQUCBToFLQVCBA0FMw0CBTIFBQWrAQXAAQU5DcQkCsMkAyYFqwEFwAEFOQIsBSsJCAGJNAEx +DPZYDrckArokD7kkBAoBtiwQAgydXQOgXQkEAaNdDahdAaddB+gwBgIBmTMBnwEMvDQKCQGRMwGf +AQzONAUFBasBBcABBRMFAgUEBbEBBcABBQ0SDgUCCrkBBQIFCQMKBRQRJREVEgcPAAAEAQYCDgHN +BwIFAAA4BG8CRwEwBFEDTAMJBC8GKAU+BiYJBQonCQkKKAlOBA8DCQpEBw8CHgEoAQ8AAKIBAgUB +lQIEAQIMBSIIAQIMCYcBDAoLIw4BAgwCDhECEg8RBRQcAgMBCgQKAgMFAQYHGQccAQIMHQsgAQIM +IaMBAAIOgAGuAX8BgAFJfwGAAacBfx8AqAFZQBY/WmEBBAheuQFhAQQKXjcA0jUYBgkCBwIPCBIq +BR8FCQUMAcU1FtQ1BQkRCiYCEggMpTUBMQjcNRQkAiEMAgInFigPLAQlEiYRCwURBBIFDAsLKgIF +BgHVNAGfAQr6NRlpHgAABAEGAg4BsAMCBQAAWQJkARAEuAECBQUHCB4HHgAAWQIWAVoEAQIIBbkB +CAECCgk3AAIKICEfHACoARRhAWIDXQVeKgDCNhTfNAHiNAOBNgWENg8DGwAABAECAg4BMQICAAAY +BBQDGwAAFAIBAQMEBQMqAAIOUMgCTxkAqAFlEAMPCBADD/wBAM42GA4FCwwEAgIaBhSjAQumAQHx +MQENAtowBAIEjzEBFwKoMQeoARkCFAIvAi8DAgYCAgkCEwQUAg8GBQIKLRgAAD0E3wECJwIUBxgA +AFkCCwEBBAECAgMIBgECAgcHAfUBAAIOYMABXwFgnQJfGQCoAUsQAw8IEAMPCGEBBAxeHhADDwoQ +Cg8BEAcPB2EBBA1exQIAgDcYoSsFpCsGAg0ID9MBC9YBAaEyAQ0C2jAEAgSPMQEXAqgxB9YBAa02 +ATEM4jYB5iIBtgcQAgydXQOgXQkEAaNdCqhdAaddB+hVBuUiAbE1AZ8BDdY2BQIKEwGpKwWYAgqV +AnuWAg+TAnGoKxIHGQAABAEGAg4B4wMCCgAAOgJSAS4EFQMcBh8ChgEDXAMZAAAYAgUBIgQLAwEG +AQICAwgGAQICBwcDAQ4BAgwPARIBAhwCAwEKBAoDAQYHBwYRARwBAg0dEAIFHgodeyAPH3EBKwCo +AUwAoDcUAgkGBQMFBAoCCgkRAAAiBAoCDwURAAIOYKABXxcAqAEhBScGKkYERQFGJEUqALA3GBIF +DQTxMifyMgICGgwFBQnBJgTEJgHDJg4CFsYmCgIKExYAAFICTgIPAxYAACECJwEqBAQDAQQkAyoA +AgpwVG8SAKgBIkYERQFGJEUlAMg3FAIJBgXJJgTGJgHFJg4CFsgmCgIKCREAAAQBAgIOAVoCAgAA +RgQKAg8FEQAAIgIEAQECJAElAAIOgAHFAX8BgAGYA38ZAKgBZzgDNy8QAw8IEAMP3gMA3DcY/SsF +gCwJAgzrKQXwKQHvKQbyKQ0IBwQJCAzpLAPsLAYGCRgU4QIL5AIBrzMBDQLaMAQCBI8xARcCqDEH +5AIXAgUCCh1KAhEHFhUBiywFmAIKlQJ6lgIPkwJ8iiwSDRkAAAQBBgIOAeMEAgoAAC0CCgErAlsC +JgIgAUIDHAgbAo4BBWIDGQAAGAIFARUEBQMBBAYDKQYDBSMICwcBCgECAgMIBgECAgcHB5gBAgUQ +Cg96Eg8RfAErAAIOcKUBbxUAqAEmRgRFAUYkRRAQBQ8bEAMPCBADDzsAtjgYCgUHCbEnBLQnAbMn +DgIWticQhzQFijQPhwMLigMB1TMBDQLaMAQCBI8xARcCqDEHigMRAgUCChEUAAAEAQYCDgGrAQIF +AABKAiQCMgIUBRQAACYCBAEBAiQBEAQFAw8GCwUBCAECAgMIBgECAgcHBTQAAgpgRF8IAKgBHkYY +RSAAzjgUCAkCAecnGOwnDwIKDwcAADECHgEHAAAeAhgBIAACDmCZBF8BYEJfFACoAbMDEAMPCBAD +D70BAOQ4GAYFAwoECgITAhAEJQIHAgsCFAIHAgcCEQIIAQ0CCwIsAgcCFAQIAxACDAIOBBUIDAIY +AgwCC+MDC+oDAbU0AQ0C2jAEAgSPMQEXAqgxB/YDDjsFRAxDCEYCECgOBQIKKQQCLAISTxQAAAQB +BgIOAeEEAgUAACwEtgMCJQJjBxQAAKcDAgsBAQQBAgIDCAYBAgIHBwG2AQACBDBPLwEwEgCoAWYA +7jkOAgQEFBIEAgQCCAIICAcCBQQKGRIAAA4BWAAARQIFARYCBgACBGCPBF8BAKgBpQMQAw9sAOg6 +Dh4KGwkIBgwIAggGBQIOAgkCEgIUAjImBRkJAh0IDxAFCQoCHgITBh0EHQIPBhkCBwILAggCBI03 +A443BwIPAggCCQIdAg8GDwIKAAAOAYYEAAA3BN0DAAClAwIDAWwAAg5grgFfDgCQOxwCjgECEgMO +AAA2AoYBAQ4AAgogUx8KAKgBFGEBYgddCV4iYQEEDF4TAPQ7FJE7AZQ7B8U7Ccg7CgILAgwEAZ06 +AZ8BDMA7Cg0JAAAcAioBCQIPAQkAABQCAQEHBAkDIgYBAgwHEwACDjCwAS8MAKgBJBAFcQEEDF5y +YQEEDF4VAIY8GAIEAgcEAdc3BdMDATEM4DsiCggJBAIJAggCCgQIAhQCDAYBwToBnwEM5DsKHwsA +ADEEcQEFAQkCDwELAAAkAgUCAQIMBXIIAQIMCRUAAgTAAZkDvwEBAKgBngMArjwOAgkEBgIIAggC +FAIIAg4KDwINAg0CCwQEBFsKFAIUAkUMHQYRBA8CCgAADgGQAwAAYQS9AgACDoAB3wF/CgCoAfcB +AOQ8JQK2AQISAwoAAAQBBgIOAdoBAgUAAEgCpQEBCgDWPDcCpAECEgMKAAIKYE5fCgCoAR5GFkUu +AIA9FAIJAgHPKxbSKxEECgIKCwkAAAQBAgIOAUoCBAAALwIqAQkAAB4CFgEuAAIEgAFbfwKAAXB/ +AYABDn8BgAELfwEAqAHtAQC8Pg4CCgYGBQUQCwQDAxMECAIFAhAMAwMKAgUCTBoHCAwFDzcMAAAO +Ad8BAABHBKYBAAIOUFRPAVA2TwwAqAGlAQDiPjMCDwILAgwSCgMJBRIBGwsMAAAzBCYDDAIHBC0F +DAACBFCAAU8BAKgBhQEAlD8OAgQCEwEFBAkINQ4TBAoAAA4BdwAAVgQlAwoAAgogLh8KAKgBQgCi +PxgEEAQHAgoJCQAABAECAg4BKgIEAAAjAhYBCQACDjB0LwEwCy8LAKgBGGEBYgddBV4xYQEEDl40 +ALg/GNU+Adg+B4k/BYw/CgIPAgsCDAQB4z0BnwEO/D4ICgICBQIPBAwXCgAAIAIwAgUDCQQRASAB +CgAAGAIBAQcEBQMxBgECDgc0AAIOUNYCTxUAqAE4EAMPCBADDwhhAQQMXncQAw8KEA0PARAKDxVh +AQQMXmAA4j8YAhSrCguuCgH5OgENAtowBAIEjzEBFwKoMQeuCgGFPwExDPZYCQIRuxkCAgkGChIK +AgsCDBUFGgQXAYIaAbYHEAIMnV0DoF0JBAGjXQ2oXQGnXQroVQb1GQsUBLU+AZ8BDLI/ChwCAgUC +DAQKCgUCDAQFAgUCCkkUAAAEAQYCDgHcAgIFAAAnAn8CDgNgBBEBJwQZBRQAACwCCwEBBAECAgMI +BgECAgcHAQEMAQIMAhoPQBIBAhwCAwEKBAoCAwUBBgoHBhEPHAECDB1gAAIOcC9vAXCXAW8SAKgB +MioCKQoqDCkCKhApiwEA+EEcAgwCCAICtzMCzDMKyTMEAQiyMwKxMxC0MwUGSgYTAggGDxUSAAAj +Ag0BaQQ8AxIAADICAgEKAgwBAgIQAYsBAAIKMDAvDQCoAUcAgEIgAhECCgMMAAAgBBsDDAACBIAB +gAF/AQC2QhMCAQZnFAoAAGkEHAACCmBXXwgAqAFpAL5CJwIIBAsCEAQJAgUEChEHAAAqAi4BEQAC +DpABmAaPAQGQAWSPAToAqAExDgQNAQ4LDSA8BTsQEAUPuAE8CzsBKgkSLzulAS48LZ8BDiYNqAEA +6EIYAhAUCdU7BN47Ad07BgIF4DsGDAwbBRAEegXTQwXSQwVjC+c+Beo+CgIFAg8CDAIKBBC3NQbA +NQkHBRQJNhE5CwIKAgw4CQIHAg4CB9dDBAIEAgPWQwHHMggIAZURBwIEAgQCINJDDAIyAg8CFQIl +BA4CEJNCBAwMBgYDCQIUAgYCA4ZCCwIFAgcEDwQVCBECCwILBBMCBwgJAgwIBQMJiz0PAg8ECIo9 +Co8BEQcRFREXDgIRDRILOgAABAEGAg4BqAcCBQAAewQlAh8FKQagAQJ7AuEBAjkLFg5YDToAADEC +BAEBAgsBIAYFBRAIBQdECgYJbgYLBQEECQIvBaUBDDwLnwEOJg2oAQACGaAPNZ8PAaAPzwSfDwGg +DwufDxQAqAFPSAhHvQFIOUfxAgDSRCkEFAIS0zYI2jYOAg0EEQIDBh8CCQcGBggCMgQlAgHvNjny +NhICHwEIAgkEDQEkAhgCFwIPBVIMDAIzAhAbCxsUAAAMAQYCFwGQBQIFAACcAQRNAqEBAs0BAjgJ +FQwGCxQAAE8CCAG9AQI5AfECAAIOwAGhAb8BAcABrAK/AR4AqAF8EAcPbRAIDwoQBQ8QEAgPCBAF +DwdhAQQRXnFhAQQOXjUAlEUYAhEIDQQJDQoSCwQJAggGBe4cA+0cB+4cBwIBj14HokEOAgUGBgII +BAIgCg8CDwkCAfgcBwIFAg75HAYCBgIB0hsMAgGLXQiOXQqNXQWQXQLTGwHOGwwCAYtdCI5dCI1d +BZBdAsUbBAUB80QBMRGoRRS6GwOaAQICDgIM1RwUuBsDmgECAg4CDNMcCgIB+0MBnwEOoEUFSRID +HgAABAEGAg4B3QMCBQAATwIeAc4BBCMDYAQeAx4AAGoCAwEHAggCBwM5BhoFDQgNAggBCgQFAwIH +AQ4NAggBCAQFAwINBRQBAhEVFBoDARwXFB4DARwbCyABAg4hNQACDlDeAU8BUAtPAVBvTxQAqAEg +YQEEDF4tXQwDCmIjEAUPdxAHD2YA6kUdAgIiAatEAZ8BDM5FBSEOqBwDpxwCqBwIpxwCqBwIpxwC +AgG/RQwyCrJFAagcAQIFAg6pHAIGBgIGkUIFlkIyBhMSECEMEQYCAZgcDgIBj14H+EEGFQ0EAcQc +BwIFAg7FHAICAcAcBwIFAg7BHAcTFAAAKAINASwCDwFIBHEDUwAAIAIBAgwDEwYDBQIICAcCCggJ +Aw4MAQoLARAUDw4SBRFoFA8CBxUUGBoXAxoaGRsAAgowMi8LAKJGHQIWAgoDCgAAIgQbAwoAAg6w +AbICrwEVAKgBZRAIDwgQBQ8QEAgPCBAGDwdhAQQMXnZhAQQMXh4AwkYYBgYCCAQCEAIPB8gbA8cb +AgIB7BsBAgUCDu0bBgIGAgHGGgwCAYtdCI5dCI1dBZBdAscaAcIaDAIBi10Ijl0IjV0GkF0CuRoE +BQH/RQExDLRGFK4aA5oBAgIOAgzJGxmsGgOaAQICDgIMxxsKAgGHRQGfAQyqRgopFAAABAEGAg4B +uAICBQAArwEEHgNlBA8DFAAAMQIDAQMEFAMNBg0CCAEIBAUDAgUBDA0CCAEIBAYDAgsFEgECDBMU +GAMBHBUZHAMBHBkLHgECDB8eAAIEMIABLwEAsEgOAgkCCwIdBA9HBEoBSRIGCwILRAoAAC0ETgMK +AABOAgQBAQIoAQoAwkgUAhIBBwACChAVDwkAqAEoAPhIKAAABAECAgwBFAICAAASAg8BBwD6SCgA +/EgoAP5IKACASSgAgkkoAAIZ0Ak6zwkB0AnFAc8JAdAJ5gLPCQHQCeYBzwk+AKgBpQcAjEkpAg0O +DgIQEQiCARAxDDIDLRgEJQ4tHCV9FoQBCXcLigERAjIYCRcEBAkGEfNCEfZCAwoDCQICEvdCEfxC +CAQIAhKBQxGEQwqDQxGIQwgKEAInBAoCEAUNTUMECAEDDgQNBQENDQMBFAYMBQUGDDgILQVOCFUI +GwUcFAELbT4AAAwBBgIXAfUGAgcAAIMCBAcDaATwAQJqASECHAUYBEACBgU+AACeAwIRARoEEQMi +BhEFCggRB/0CAAIEUCtPAVBMAPRKDgINEAsCCgsDASMGJgAADgFuAABnAhUAAgRwVm8BAKgBWwCW +Sw4CCdlDENxDDgQIBBQECgAATAIFAQoAABcCEAE0AAIOMPICLwEwJS8fAKgBxQMA+ksYAgICBwI/ +AgwCRwkFAgQMDwIM/g4E/Q4CAggCEAoLBA7sDgTNDg7ODgoBAwIcAhDJDg7YDhDVDgrWDgUNCAEF +9w4TFR8AAAQBBgIOAaUDAggAAMYBArsBAiUDHwAA1wECBAEzAgQBDgI5AQ4EEAMKBAUBDQEyAAIO +wAGhBr8BFgCoAVIQCg8LEAcPIBAKDwsQBw8vEAUPDmEBBAVeEWEBBAVecWEBBApeAWEBBApetAMA +zkwYBAK+DQa3DQ4ECAILBAGsDQGOBw4CAYtdCo5dCgIBj10H/lUCqQ0MAgKmDQGOBw4CAYtdCo5d +CgIBj10H/lUGpQ0LKAUjDgILuUgFxEgOl0wBMQXMTBGZTAExBc5MJwIjEQcSCwQLAgkCAaVLAZ8B +CshMAadLAZ8BCtZMBQcKAgUCEQQZBkICBgIJAQ8CIQYgBRAGGwUDBg0EMhQRAhICBQIKKBACBwIK +owEVAAAEAQYCDgGnBgIGAADtAQQXAicCVQIMB6QDARUAABoCBgEiAgECDwIKAQsEBwUCAQ4KAQIP +AgoBCwQHBQYJKRIFEQ4UAQIFFREYAQIFGXEcAQIKHQEgAQIKIbQDAAIOQKYBPwFACz8KAKgBKkEm +QithAQQMXhZhAQQMXh8ArE0gAgIECPFLFQQKAgfoSxAECgUFCgsCAdVMATEMik0VAgHZSwGfAQz8 +TAoLCwUKAACDAQQVASgBCgAAKgImASsEAQIMBRYIAQIMCR8AAhbQAtEOzwIB0AK/Ac8CEgCoAVFG +K0UBQBU/B0AHP2phAQQMXpoDYQEEDF4VoQELogGvAaEBDaIBCKEBGqIBHBAIDxhGHUWqAmEBBAxe +ZWEBBAxeyQEQBA8BEAQPqQMAhk4mCAYCEAEE/BUH8xUJAgH7PiuEPwH1TRX4TQf3TQf4TQUCGkwE +RwoEJQYXBgHVTQExDIpOEgIcBB8GIgJBKAchHwITAhwOCAkPBjIEGgQxBAGFTQGfAQyuTgxsBc0B +AmICp04LqE4RWAQUDWsJAiABBAIJAgUCDAQeAii5Tg26Tgi3Tg0ECAMFvk4IAhTNSgjOSggUBgIJ +CAGnPgcCFqg+GCQMFAUzHR8HAhYiFwIdAggCDAIFAgkCBQoLBgvCFALBFAYCBAIFAwsOBw0IAhsM +DwIB9U4BMQyqTyQCIAIgAgH9TQGfAQymTxgiCiEIAiECBAEEAgweAxsCBAjeDAUKCOUMAgsEDgiU +FAehFAQECB4IEwUEAeAJBwIFAg4CBuUJAZtLBJ5LAb9LBMJLBQYBhhQGAhACEQMDPgU3BQIeAwQE +OQMHBAaHFAWIFBiJFAkCE8AUBr8UA4YUBTgQ9QcHAgcCBwIR7QwLCwsBCAELDw0bBzEFCQgfDAEI +EwgpEgsSAAAJAQYCFwGOEAIFAAB3AgUBAQKMAQEJAm8CrQEBggEBCQKiAQQuBSEITQcZCq4CCQkK ++QECJQuUAQ68AQtyARIAAEACBwEKBCsDAQYVBQcGBwVqCAECDAmaAwwBAgwNFRALD68BEA0PCBAa +DxwSAxEdFB0T1AECAgFUFgECDBdlGgECDBtsHg0dDgIHARogIB8BIgQhASQEIwYCKiQFI20BBQIY +ARwmBiUDAgUkEAcmHYoBAAIKIDkfEgCcUBQECgwOBAkCBQQKGREAABkCDgITAxsAAgRgWV8BYMAB +AKgBOxAKDwQQBA/RAQDGUA4CBAQUBhQQAaVMCqhMBIVMBIhMBwIMFQMEBgIHBH8CGAcFBxIAAA4B +kAIAAH0EoQEAADsCCgEEBAQD0QEAAg6AAYEBfwGAAawBfw4AqAExEAUPlAIA7lAYAgQEFb1MBcRM +AQIUCwUUCQIFBgUBFAIIAgcCChGJAQIRCRIHDgAABAEGAg4BrQICBQAAWQQ/AhsCGwVuAQ4AADEC +BQGUAgACCiBFHxAAqAEYYQFiB10FXhRhAQQMXhkAllEYs1ABtlAH51AF6lAKAgQCBQQBv08BnwEM +4lAKDQ8AACACGAEJAg8BDwAAGAIBAQcEBQMUBgECDAcZAAIOkAEmjwEBkAGBAY8BAZABggGPAQGQ +ASSPAQGQAY8BjwEBkAHeAY8BCgCoAe4CYQEEDF5SEAcPMmEBBAxexAEArlEYDBMCCg4REhQSAg8K +BAEBB/EvCvIvAxADAQaFCQ6GCRcCBAIKBAgKCQImIAwMCgIFAgUCBwIJJhKPAQqQAQ8CDAIKCA4C +AYdRAZ8BDKpSETUOAgUCBwgBmAcHAgUCDgIGAwOXBwIKAZFOB5ROEQIKBxYlAcFQAZ8BDORREr0J +BYYJDQqEAQISSwoAAAQBBgIOAboFAgUAAGECHQFrAkcBRgIkAUECKgEJAr8BAQoAAG4CCgEMBA4D +3AEGAQIMBywKIwkDDAcLMg4BAgwPEgQFA60BAAIOYFhfAWAJXwFgdV8UAKgB+gEAglIYAgUBCQIL +BgGVRAuYRAYICAUKBAgOChkKkUQEpkRaAhcZFAAAKAQJA1AGZQUUAAAyAgsBNAIEAYUBAAIOwAHh +CL8BCgCoARhhAWIHXQVeDGEBBAxeGWEBBA1eYUAWP0BABT8BYQEEDF5qYQEEDF5AYQEEDF4MYQti +JGEBBAxeEmEBBAxeBkASPwpACT8wFwgYiQE/B0ACPxxACT8oQAc/AkAbYQEEDF4mEAUPDmEBBAxe +ZgCGUxijUgGmUgfXUgXaUgYCBQIBq1EBnwEM1FIRDAKGAgUIAc1TAZ8BDeJSBQYE/gEKgwIUBgQB +BAYJBiMCBSIBrVMWsFMHKwIsKS8E6AEFcQX1UwWyUwH5UgExDK5TKDME6AEFdwU3BQIKOQToAQit +AQUBBQQOAgGFUgGfAQysUxUCEAYECBECBQgBpVMBMQzaUwsCAflRC+xRBxIJCgJvBOgBCucBBMdR +AZ8BDOhSBOgBCucBBMdSATEM9lMG+1MSglQKgVQJglQFAh8IBQEHo1IIplIUAhICDNwNA9kNAhAK +AgoCCgYFKg0EBQgKBggFAucBCeVKB85MAs1MExYJ1EwHDwK9TAwCHMxMB8tMAsxMB4MCBfQBDgIB +t1QBMQzsVAoCCQIHgA0LAgGPXgWQUQ0CAcFTAZ8BDOJSB4QCLwQPAwgEBQMKoQIKAAAEAQYCDgHc +CAIFAAAgAp8HAQkCpwEBCgAAGAIBAQcEBQMMBgECDAcZCgECDQthDhYNQA4FDQEQAQIMEWoUAQIM +FUAYAQIMGQwcCxskHgECDB8SIgECDCMGJhIlCiYJJTAoCCcyKgMpVCwHKwIsHCsJLCgrBywCKxsu +AQIMLxoyDAIFMw42AQIMN2YAAg5wlwJvAXCBAm8UAKgBHWEBYgddBV7gAWEBBAxec2EBBAxecF0M +Xg9hBWIUAM5VGC4FmVUB9FQHpVUFrlUOdAVzFAIEAgUDBQIFDgcECgQDAg0CAwIIAhoCCAENHwV0 +BUEFAwQNDg4JBAQCDQIEAgQCBbgGAv8FAe1UAZ8BDJBWD4AGBgIHAgcCDAIItwY7CAHHVAGfAQzy +VQ4CFwIJAggCCgYIAggDCQQCDgoJCgIBmVYMrlUFHgVWBe9VBexUFAAABAEGAg4BngQCBQAAJQKI +AQJaAwsCdwEJBFABRQEUAAAdAgEBBwQFA90BBgIFAQgBAgwJDwYoBTwMAQIMDXASDBEPEAUPFAAC +CmBuXw0A4lYUBCoCBgEFEAQHCAgHDw8IAggMEwwAAFAEKQMMAAIOMH8vATALLwEwCy8VAKgBXBwj +GzsAkFcYAggCCwYHAgoIBwwIBgkCBwIB51EWFA3aUQ8hDAcMBRQAAAQBBgIOAZ0BAgUAAHoELAMU +AABcAiMBOwACFvACpArvAgHwApEF7wIZAKgBLUAFPw1ACT8SYQEEDF7+DgDKVy2fVwWiVw2hVwmk +VwoCBwYB81YBMQyoV0PvDhbwDooCDQoQBgLCAREIHCMbCBwjAgQCDAIMIQIkBgQFAhAExAIRBRIF +CBECCgIbBBsCHwIcJQUmBUECTAIKDAcMAhAGCCDsAx8WAgcCBwIHBAUCCQYFAhAMCQUFAhUBBQwM +GgwCEJcBGAAACQEGAhcBtQ8CCgAALQIsAQkChwYCvQMBlgEExwECqQIFIQEYAAAtAgUBDQIJARIE +AQIMBUMIFgelDgACDkDgAT8BQAk/AUAiPxQAqAEcYQFiB10FXkQQAw9BXQwDBmIBYQEEDF4DYQEE +EF5KAPBYHI1YAZBYB8FYBcRYFggGCwIOAgIGAgoCAaIBDqAHBbldA8BdCwIMAgkEBwQH1wgEqAEG +AgijAQHDWAygAQauVwGtVwGfAQzSWAIdAZNXAZ8BELZYChwKCwQCCQEVFxQAACQCYwExAgsBCQIH +AQkCEwESAhoBFAAAHAIBAQcEBQMxBg4CBQIDAS4HBAYOBQEODAEGCwEQAQIMEQMUAQIQFUoAAgow +Ji8BMA4vEwCoAVIAoFkUBgkCFAQQCxEAAAQBAgIOATwCAgAAIgQfAxEAAgoQMQ8SAKgBIRAHDyUA +ulkSBg4CAeNUB+ZUBgIFAgkNEQAABAECAgwBOQICAAAuBA4DEQAAIQIHASUAAg5wrgFvAXBYbwFw +Bm8cAKgBowEQBQ+QAQC0WhgGEAgOIgMhAgcFCBECAgwDBQIFAgYEBggIEAQBiAcJAgOVBwKYBw4C +BQIL310F2FYFBQICAiMMKAoDBggFAwUDCAIBggcHAgUCDgIFAg2DBwolBg0cAADCAQRKAwoGBgUc +AAB3AgcBAgIDAQICHgIFAzgGLAUsAAIOQKYBPwFACT8BQA4/GACoATlhAQQLXlthAQQKXjoA0FsY +AhABBRAM/VoBMQuyWw8CEE0LAQMCHAIRUAGFWgGfAQqoWwoVCjsIAQY6GAAAPQQXA1IGDwUNBgsF +GAAAOQIBAgsDHwY7BQEIAQIKCRQGDgUYAAIOMMsBLwEwIi8UAKgBuQEQBw9QAIBcGAYGTgYCBwIK +Ag5TBgYKAhWHAQgCAgEDAhSGAQOFAQ4CE4YBCwIB1VcH2FcQAgqLAQsBBYABEgcUAABOApIBAhwD +FAAAHgIlASUEIQMDBCEDDAYHBRoEEAMmAAIEMNsBLwIwJQCoAYYCAKZcDgYHAgkEEKkBCwEDAiAC +FKoBEJ8BCwIfAhGeAQsCEAQLowELAQULDwEGAADnAQIfAAAuAkIBEAQ7AyYEEAEVAAIOUFJPAVAe +TwJQQU8oAKgBZRAIDycQEw9DAIRdGgoCKgUHBwUFDgUNBgIGAgwKDQIKCQSdWQigWQkCCyEDAgMB +BwIDJAOdWRP6WAIGBRADBgMNBQUKFycAAE4ECQOTAQAAZQIIAScEEwNDAAIZ8BCmAe8QAvAQ2QHv +EAHwEBzvEC4AqAHGARAHDy8QBQ8FEAUPAWEBBAxeEBADD01hAQQMXl8Ayl0pAgkBAgIfBgICCwIQ +CBYBBgYDFwMYEwYHFAIXExoF61kH6lkEAhwpBC4GAgXxWQX0WQXzWQX6WQGpXQExDN5dAecDCqAH +BbldA8BdCwIOAgkEBwQHrwcWAgbmAwGtXAGfAQzQXRUTCiUSDS4AAAwBBgIXAbcDAgUAAJQCBDED +PAQ2Ay4AAMYBAgcBLwQFAwUGBQUBCAECDAkBDAoCBQIDATABHAsBEgECDBNfAAIOMPgBLykAqAEs +EAgPXGEBBAxeEhADDz5hAQQMXjIAoF4YAgYCCAQCBASZWgicWgQCAuICCekCDwIBugMBAgUCCgIF +AgqfAwoXA94CBMMCAi0FFAUcAe1dATEMol4BqwQMoAcFuV0DwF0LAgwCCQQHBAevBwsCBKoEAfFc +AZ8BDJZeCjcoAACYAQQxAy8GBQUyAAAsAggBBgQJAxAGHwUNCAQHDQoBAgwLAQ4MAgUCAwEuAQ8N +ARQBAgwVMgCoARQQFQ9LAOheAgIKAgUHA8laFdhaAgIJCgkCBgIEBgsCAyUDJhECBgcFAAAUAhUB +SwACBDDcAS8BMGQvAQCoASIQEg8oEAMPChAKDwEQDA9iEAUPLRADDwsQCg8aAJpfEwIHAgUDA/9a +EoRbAgwKCwUCAo4CCwIKnV0DoF0JBAGjXQqoXQGnXQyQWwQDAhQCBwYCBgICAgoGCAgDHAIbHAsY +iAIBp10FwlsCBgIFBAIPAgHWAQsCCp1dA6BdC59dCqRdAtcBGAAADgG4AgAAIgISARMEFQIDAQoE +BwIDBQEGDAlhDAECBQ0YDBUEAwMLBgcDAwECCxgAAg4wkAIvATALLwEwMy8xAKgBkgEQBQ8EEAUP +DRAWD8wBAOxfKwICAgkCBgICAggCCQsDDgYEEQIGAQUaHv1bBapcBKlcBapcBVcI0VsWjlwIEglJ +BkwHAQIOBFsDXBJXBlgFGQkCDwYMDgIBEAIMAwkKDV0wAAAEAQYCDgHyAgIFAAB7Ai8B5QEAAJIB +AgUBBAIFAQ0CFgHMAQACDlCBAU8BUAxPAVALTwFQEk8qAKgB5QEA2mAdAgoCGQIECBwCBQYHAhEG +BgINDQ0JDBISGSoAAAQBBgIOAcUBAggAADgEfQIGBSoAAhngBEXfBAHgBLMB3wQB4AShBN8EAeAE +8QPfBBQAqAGgAUASPw5ABD8jLhMtIy4ELUZAEj8iQAk/8wJ3B3gXd3N4V3d0eGl3NngoAIBlKQID +BgoDBgMIDgkEAgsQDAUEBQIIBRYMBxQRAgGJZRKOZQ6NZQS4ZQMjCCQKAwUfCZtlE55lAh8HAhAe +BQIBBASjZQSkZQIFESIFAwUGBaVeBYheCAUGDBACAaNlEpQHCpReGKdlCZQHEpheDwYNARUCQwJx +Am8CDeVkBQIC7mQHBBDvZAkCAwMMAgoBAwImCBICFthkSgIN52QFAgICCQIEAwYKAwcKAQMCIggS +AhbaZE0CDwYIBQXdZAoHDQgSBw2KZBQJFAAADAEGAhcBjAoCBQAAbAIzAQECgQECRQMBBFIBLgQj +A2EGEgVdBhIF2QEGEgW7AQYSBT0IRQkUAACgAQISAQ4CBAEjBBMDIwYEBSIIBQcfChIBCgcYCgkB +EgfhAgwHCxcMcwtXDAUCbw1pDhcBHwsoAAIEEIIBDwEQCg8BEAYAqgGYAQDeAwwCAwIB8wEE+gEE +2QEOFAvGAQkKBM8BDNABAwIEAgcDAwQGAg0GBgkEDg8XCwwGAAAMAYwBAACSAQQGAAAQAgQBBAQH +AgcCCwcNCgwJTgACBBC9AQ8BEAoPARAGAKoB0wEAkgQMAgMCAacCBK4CBI0CDhQL+gENCgSDAg6E +AgMGBwIRAg0GCwICDgMJAwIDBgwCDQYGAgIPBBQPMwsQBgAADAHHAQAAzQEEBgAAEAIEAQQEBwIH +AgsHEQoOCYMBAAIOgALAAv8BAYACswT/AQGAAsAB/wEBgALZAf8BaACqAYULAOIEKAIJBgoHCAgI +fggICGkIXghxCLsCB7wCCLsCBbwCDXIIIwgoEAgIGwidAwW8AgKtAgiuAgStAgQCBQIFEQPQAhRe +CCMIKBAICBsITQKVAgiWAgQGBp0CBfgBCjIBqQMDrAMBqwME0AMRrwMH5AIDTANLA0wHAgUBAgQQ +DBwGFQIRAgkEBQQsIAMfBBADDSMCDwRCBwogBRcUGAUVBQYIBQgGCAMKAwIECQEJAhMEAgIEAQwC +Ew4DAQGPBASSBAzTAxPUAwOBBA6EBAIIBwIMCAohCwULAxgDDQEKAQgJCAcFCwgDBc0CBwgEFAUG +BOABA98BEvwBA/8BAhEEAhOQAgsCCrMCBwIEnAIDrwIDDAMJBQoOnAIDmwIXCQMMAmkDkgMJAwIC +CQIIAiJiCPUDCNIDCCgQCAgbCOUDCP4CEgloAAAEAQYCHgHUCgIJAACPAQJIAR0CRgH1AgSAAQNF +BgUFEAaNAgUwAlEEBgVoAABzAgcBCAIFAT0CBQECBAgDBAQOAQMBRgYIBQoGBQULCAMHAQoECREM +BwvAAw4EDQwQEw8DEg4RhgEGFAUDBhIFAwYZBRUECwMDBBkDAwQcBAMHRggIBygIAwd/AAIKIEof +EgCqAWYAsgYUAgICBwIIBQOzBA68BAIGBwIMCAoXEQAABAECAg4BUAICAABGBAUDGwAAKAIOATAA +AgowPC8cALYBYgB8FAIGAh4CBQIKBxsAADgEDwMbAAIOgAH4AX8BgAEMfwoAtgGDASAHHwgeHR0V +HgUdJB4DHS0AiAEYCE8CFRsHUwdUCKgGHYkGFYwGBYsGJIwGA4sGDQQKAwwLCgAABAEGAg4BgAIC +BQAAKQJsAmgDEAIGAQoAAHwCBwIHAQgEHQUVBgUFJAYDBS0AAg5gggJfAWAMXw0AtgExIAcfeCAH +H3MAnAEaCgICAgEMLwdTB1QEMAYISgIRAgw7B1MHVAM8TQQKAwwVDQAAVAKzAQEQAgYBDQAAKgIH +AgcBBAFtBgcCBwEDBXAAAg4wqwIvATCwAS8KALYB9AMAygEYAgsCCwIgBg4GCwIVBhwGEgYNAhoG +KwYMBhIGFgYKAxEFEQURBREFEQcWBREFEQcRBRIHCgAABAEGAg4B1wMCBQAAxgICpAEBCgACDoAB +pgV/AYABpgJ/DwC2AfIEHB8b2QIAlAIYDAgIDwIPBgh0EdIDDgINAgMGCwYJBA3lAwLwAwjvAxkI +CAIIAh8GCwgIAhkGCwgIAiEGCwgQAg0CLAgQAggCMggJAhcGCAgJAhgGCAgIAhEOCAITDgUEAowK +AgIDARqLCgIIGAYKAxEHERERDxEJEQURCREFEQkRCxELEQUVCREFEQkRBRELEpEBDwAA6wQCJgEC +AhgBFgKaAgEPAABXAj8BAgIIAdIDBB8D2QIAAg6AAvwD/wEBgALtAf8BDQC2AckEHgUdBR4FHSAe +GR0JHgUdBR4FHVwAvAUYBAoWEQYxBCUCBQoOBwIENgsKECoCCQY3CiYCEAIWBhcbBQgKFAoUGwQW +AgwCChsVAgkBDwIRFAUTBRQFEw8BDwICFBkRAgIHEAUTBRQFEw8BIBMSCw4rDQAABAEGAg4B5QUC +CAAAPwIzAn8CgAECegVTCDoJCgpZBx0BDQAAyQQCBQEFAgUBIAIZAQkCBQEFAgUBXAACDlCXAU8B +UEBPMgC4AUAwEi8YMAQvqgEAqg4YCAUMBREDBgkFAhIDCw2HDhKODgkGD5MOBJQOCAUCBhECEwwK +CQMBAwQDAgQFFgINAxATMgAABAEGAg4B+wECBQAAQAJcATUEFQMyAABAAhIBGAIEAaoBAAIpALgB +KQD6EAQCDQQYAAEpAAIKQGU/FgC6ARQDBAQBAwsEYQBEFOIGBNcGAdoGBgIF2QYQBDIkCjMVAABU +BBIDHwAAFAIEAQECCwFhAAIOQIQBPwsAugEjbwVwEnMBBApwFwEEAgFzAQQFcBJzEHQUAFQhAgIl +BSgMBAUCAYQBAZ8BCjICDQkCCwIB9AME8QMBdgGfAQUsEQIBogEQnQEKHwoAAAQBBgIOAYABAgUA +ACMEGAEnBCcFFAAAIwIFARIEAQIKBRcIBAcBCgECBQsSDhANFAACDjCbAS8BMBIvFAC6AUtvBXAa +bwVwAXMBBAVwBAMmBDAAfhgCEAINCBALBRABXQVgCQIFAgxRBVYBSgGfAQVcBJgGDwIPBAibBgob +EgUUAABLAiQBAgQvAxYEBgMUAABLAgUBGgYFBQEEAQIFBQQIJgcwAAIKQFM/HAC+AXkAoAEUAgcB +BQgFAx8EEAIKCRsAACUCKgIPAxsAAg7AAUi/AQHAAesEvwEBwAE5vwExAL4BkQNzBXRbFRsWbncB +BAp0qAEAxgEYAgQCEQoB6gECAgUCBAYDpwEDqAEO8QEKDw1aCTcKAgHMAQjHAQgCCAIHxAEEzQED +zgEKlQEDlgEcwQEVAhoCDb4BBbEBBbwBBZEBAzEFAwQEEAIFAha2AQWxAQW8AQW7AQQqBYgBG6sB +FdkBBd4BHbABDQQDrwEGDhwCDMgDG8UDC5wBBZMBCAcCmgEKBAKdAQIIAwcTnAEDAgkGGrMBCQIB +MwGfAQrqAQ4CHAQKAgoHA5QBAgIIBgPfAQPgART7ARIFMQAABAEGAg4BkAYCBQAAbQJvAqwBAwkG +MgUYBiwCHwdaCjQCSAsxAAAuAg4BAwIOASsECAMXBAQDAwQKAwMEHAM8BAUDBQYFBTcEBQMFBgUF +CQQZAxcIBQcdBhAFLgobCQsGBQUKDAwLGAYmBQoOAQIKD0EMDQsDDBQLQwACDpABwQKPAQGQAQmP +AQGQAQmPAQGQAYIBjwEnAL4BjwFzBXRPdwEECnRNdwEECnRFFQwWdgDAAhgCAwIJchFPA1AbZxsN +CQ4FaAVPBQ0JpwIFqgIUXBBPFwIFAg4EAYsBAZ8BCqYCCAgCAggCBQIWBA85B0ACPgddAX0BnwEK +oAIKEQpgCikOAhgiAYY1DP80Ai0DPgQCBAYDYQNiDwUCQQICAwECBgMFAgIHBAMDAgUTOycAAAQB +BgIOAfADAgUAAI8BBFMDBwZKBQ4IDwcdCh0JCAguBx8IBwcnAAAkAhEBAwIbASkCBQEOBAUDFAIQ +ASsGAQIKB0UKBwkBDAECCg1FEAwPBQoLCQMKEQlSAAIOMIAFLwEwuQEvATAULzIAvgGPBRhBF78B +ANYDGAICBAQCBAUPAgQEBQUCAQMCCwIMAg8CCAcDDgsCBQIDdgMBA3UMAglOCAIGBAZlAxYGBBYC +BwIIAhwCGwIbAgUCGgQJAhgGGQIcAgUCGQQRAggCCAIIBQUdBB4HGwQeBRsEHgULBA4FBQIMCgYa +AxgIDwIIAQIJBAoFAgwECrMCBAwNBgcDCQIWAgYCBMoCBgINAgMBDgIIbQV4BQMfAgYGBgYHCQcQ +CgkUlwEyAAAEAQYCDgHyBgIFAACxBgQOAxgGBgUyAACPBQJBAb8BAAIOcNEDbwJwYG8BcNgBbx4A +vgG4BgCEBRgCCwQFAgMkAyUMAgsGCAIGBAYRBQQFVAU3CwoFBxYEBUwFRwUgAw4FMQkkBSMFBBQC +BgIcAh0CBQIYBB0CBQIbBAcCHgQeBAQCCQIFAQIlBCYHAgkBAjMPYBECCAQGBQ1xBHQHAg0CCQIH +Ag0tCh4FEAUtEwITBgsDCgwPAgYCFgEEAgwEKQYlcx4AAAQBBgIOAZsGAgUAAHgEhwQCmwEFHgAC +DjDoAS8BMBIvHgC+AacCAJIGGAQIBAQEDQIEAgQDAgMFBAUHBAoFBwQKBQIFAhgGGAIFBAYCEQEE +AgcEBgYPBQQGBw0aEgoHEiceAAAEAQYCDgGKAgIFAACDAgQGAx4AygYYBAgEBAQNAgQCBAMCAwUE +BQcECgUHBAoFAgUCGAYYAgUEBgIRAQQCBwQGBg8FBAYHDRoSCgcSJx4AAhbgAsML3wIWAMABLgIU +AS0CGQEqAhkBLgIZASoCFgEqAhkBMAIbAS8CGQEqAhYBKgIWASoCFgEqAhYBKgIWASoCGQExAhYB +KgIWASoCGQEqAhkBLQIcAS4CHAExAhwBVwAiLggEBgEFDwUtBgwIAQcMAyoEDAoBCQwBLgIMDAEL +Qg4BDQkCKgEMEAEPDAQwAwwSAREOBi8FDBQBEwwIKgcMFgEVCQoqCQwYARcJDCoLDBoBGQkOKg0M +HAEbCRAqDwweAR0JEioRDCABHwwUMRMMIgEhCRYqFQwkASMJGCoXDCwBKwwaKhkMMAEvDBwtGwwy +ATEPHi4dDDQBMw8gMR8MNgE1DyIyAhArFQAACQEGAhcBxAsCBQAAQgLWCgJCAxUAAC4EBAEBAg8D +LQgMAQECDAcqDAwBAQIMCy4QDAEBAgwPKhQMAQECCRMqGAwBAQIMFzAcDAEBAg4bLyAMAQECDB8q +JAwBAQIJIyooDAEBAgknKiwMAQECCSsqMAwBAQIJLyo0DAEBAgkzKjgMAQECDDcxPAwBAQIJOypA +DAEBAgk/KkQMAQECDEMqSAwBAQIMRy1MDAEBAg9LLlAMAQECD08xVAwBAQIPU1cAAg5QuQFPJwDA +AR0CDQEFAg0BDxcJGhABBwICAQIXBxgJAgQBAQIMAQECCAECFwkaCQECFwcaCAEwAGQdJQsTAlwB +AgEKAzsBKwxoD6wGCZMHCRQHagQCA38CagKqBgeXBgEECH8EggEBQQE/C4QBAUUBPQd0AqAGCZMH +CXACpAYH1QYBPQd0CjkmAABCBHwDMAAAHQILBAIFBQQBAgwFDwgJBgkBBwEHBAINAggHAgkIBAcB +BgECBwQECwEKAQIHFQIYCQQJGwIYBwIBAgcbMAACBKABOJ8BAaAB8QKfAQGgAQsAxAFuLx0wiQEC +MQE7GxAcHwILAOIBEAICBAcKCQ8CEAsPAhACAg0BAggLAgQCFAMEAgWeBQwCDAIFkwUPMgsxBDQS +LwI8FzsCSB9HDwYPBgGiAiECDQIDoQICBwUbFBwCGwskBQYOmgUQlwUFKwssBQQKkgILAAAOAawD +AACBAQKSAQE0Al4BDwIGAABuAh0BDwQLAwQEEgMCBBcDAgQfAx8GMQU7CBAHHwYLAAIOUO8BTx8A +xAEgIR0GBgUSIg4BBAEBAggBAQIHAhEbCRoQAQcCAgICGwcYBAIEAQECDAEBAggCSgCOBRgGCB4d +wggGwQgSHQQFBQgF6wQELAIrBzABGwfYBBGoAgmTBwkUB2oEAgN/AvAEAqQCB5cGAQQDfwSCAQFB +AT8LhAEBRQE9B/oEFgIMCgohHgAABAEGAg4B/wECBQAAfQKBAQEeAAAgAh0CBgESAQ4KBAMBAgEC +BwIBAgcNERAJBgkBBwEHBAIVAhAHAgQIBAcBBgECBwQECwEKAQIHHUoAAgQAxAEEAKYGBAABBAAC +BOABmgHfAQHgARXfAQHgARzfAQHgAZgC3wEB4AEJ3wEBAMQBtQEBBQEBAggCsgIAwgYOAiEGIAIP +AhYCCgIJCA4WCgYMAgrNBgUsAisHqgYFAgooIAYbAhoCAgIbBgUJBAoKAg4IIAIbFQcYAtALCM0L +AdALEQIJAgkCDNELCl0KAAAOAecDAAAiAjICDgIzBREIOAKgAQIQA1MHFAAAtQEGBQMBAgECBwXm +AQgIBwEILwcUAAIEwAE8vwEBwAGXAr8BAcAB+wG/AQHAAVG/AQEAxAGgAy8FMIICALQHHAQEAQoC +CwIMBhICJAIFAi0CAYALBQIKAhYCEAIQAhMGDQEFAgMCFAITAgyTCw8GIBYhAgY9BUAB3goFAgoC +FgIQAhACEwYNAQUCAwIUAggCBwEEAgzxCg8IHQIFAgsCCwQJBAUCDAAADgGZBQAAZQREA7QDBikC +IQAAqgECoAEBVgQFAwEGoAEFYQACFtACkgHPAgHQAj3PAgHQAtECzwIB0AJnzwIB0AK4Bc8CAdAC +D88CAdACK88CAdAC2wPPAgHQAocBzwIwAMQBtwIBDAIdAQgBAQIIAgUBCAIWAQwCPQEMAgcBCAEB +AgQCBwEIAQECCAI8AQ8CBQIqAVQLCwzAAQEQAgwDAQIWAvYBAQgCCAMBAggCCQEFAnADAQIIAQEC +BwIrDRgOBQ0KDg0BCAEBAggBAQIHAjobBhxmLBMrDQ0IDhQbDhyHAgILATAAtAgmAgkCEQQJBQUI +CGAKVygIDwcCAhAGHQgHGAgfAgIQBg4EEA4HBggFBQIehwgMjAgZTASPCQgsAisHzggFlQgI2AgW +1wgMoAgDCQYKCSQWERWxCAy0CAexCAgFAQYEtAgH7QgILAIrB/AIBwQNAhAIFBIE1wgP0AgF5QQn +AgPkBAICEEsHUgYCBw4EYQdiDAgTAgTBBQvCBQcECW8DcAICBXEKhAEFCQcMCA0CAkkENAYJwQkI +OAiKCQyVCQIrDDgJigluAikCGwIQPAM7AjwIPwJAEDcVywkI5gkBAge7CQIrB+gJCecJBegJFgoF +AxsCCgIMTxAhEGsE5wcCKwcwARsHgAgHAhDmARTVCQYECwgFBQLUCQMCAtUJBgUE3gkNgwoILAIr +BzABGwfyCQ8CK8IDBsEDFAoIBQcDBAgJAgUCEQcQAhDcARPRAQ3vCQj2CQYCDp4+Dps+ChYbAhkC +GQQB0gIKxwIZAhBZBQIiARMCAwEKAg8EFAMNCwXFBQvoAzAAAAkBBgIXAf4PAgUAAIIBBBcDJwQX +AykEpAICHAElA0sEdgMmCBIDmgIGoQECMQs3BhUGmwELNg4hAyMEDQeGAQUGBjwKSAMcBRUFMAAA +twICDAEdCAgDAQIBAgcHBQoICRYKDAk9DAwLBxAIAQECBA8HFggDAQIBAgcVPBgPFwUaKhlUHAsb +wAEiCAIIIwweAQIBAgwCCSP2ASoIKQgmAQIBAgcpCSoFKXAsAQIBAgcCAQIHMys2GDUFNgo1DTwI +AwECAQIHAgECBz86QgZBZkQTQw02CDUURg5FWEgKR6UBGgsZMAACDpAB6AePAQ8AxAEcJRgmRyVV +JlYlMCYdJTImASUwJgElMCZIJVsmTSUwJm8lCyYFJSYKBgkOJgUlDCYUAJoLGAIElAsYkQsICAYC +CQIbCAkCDO8HEQJE8gc/AhYNAfEHEQIflAgJAgcCBgQGAgG7CBICIL4IAbEIEQIfqAgBxQgRAh+q +CAkCKggJAgyDCBYCRYYIOwIRDQGFCBECH4IIHiYFBBEEKksRlgsLlwsFmAslBgHHCAbKCA6fCwWg +CwyfCwUDDwAABAEGAg4B4wcCCgAAhwECngEBDQJAAQ0CJgENAiQBDQJgARgCmwEBDQLIAQE6AAAc +AhgBRwRVA1YGMAUdCDIHAQowCQEMMAtIDlsNTRAwD28CCwEFAiYQBg8OAQUCDAEUAAIEMIICLwEw +CwDEAZICAPwLDgIJBCICGQ4SAhICEgYLAhkEFAISAhIGGQIKKwsAAA4BhAIAABICgAIAAg5AkQE/ +AkBUPwFACT8eAMQBhQEBDAJCGxAcOgC4DBgCEQEFDgkGEBgFAQkCEBggmQwMmgwFAgsGGQoZzwUQ +4AUJAgphCwUcAABQAkYBEgROAycAAIUBAgwBQgQQAzoAAgQgHR8BAMQBIgCmDQ4WCgIKAAATAg8A +AgowXC8PAMQBdQCcDhgCNwIXAw8AABgCTgEPAKwOGAI3AhcDDwDADhgCNwIXAw8AAgRgrwFfAQDE +ATkkIyNYANYOExwFGRMKDqkOI6wOGQIMBAUCDgYRBAUCCgAADgGmAQAASgRUAhYAADkCIwFYAAIE +gAGjAX8BgAE7fwGAAUt/AYABC38BgAEjfwKAASB/AYABMn8BgAELfwEAxAGIAQEEArUCAI4PDgIJ +KAQhEAIPBiYyCTkFIgogEPUOBIAPEAIMDAoCJgIMCAsCMgYPMRAvEQIQAQMKAgIHAgkCDAYkAg8n +DAAADgGzAwAArQECYgKyAQAAiAECBAG1AgACBFBOTwFQQE8BAMQBEy82MAsvNjAKANQQEAICAgGv +CjayCgoEAbUKNrgKCgAADgGGAQAANwIcASUCHAAAEwI2AQsENgMKAAIEYH9fAWAOAMQBGS0sLgEv +NDAKLQ4A8BAOAgkCArcQLLoQAc0KNNAKCrsQDgAADgGEAQAAaAIqAAAZAiwBAQQ0AwoCDgACChAa +DwgAxAEsAIAREgIFAgUCCQUHAAAEAQICDAEYAgIAABICEwEHAAIOcLUCbwsAxAHOAgCeERgCDgIJ +AiQCIAgJngEPnQEEngEEAhACEAITBggBBQIDAhQCEwIMrQENBxQCEAoKFQoAAAQBBgIOAbECAgUA +AEEE5AECFQUUAAB8Ag8BBAJ6AUUAAg5g2AFfAWAfXwoAxAF4LS4uAy80MAotDS4cAMgRGAISAgtG +IwIGBhIGBlECiREujBECBgGtCzSwCwqTEQ3OERJJCgAAywECOwEKAAA1AkEBAgQuAwMGNAUKBA0B +EgEKAAIEYI8BXwEAxAGUAQDiEQ4CDYoBBHUBeA8CCAIIAgt5AhUaAiQUCgAAeAISAQoAABsCBAEB +AioBSgACBGBdXwEAxAEvLwUwLgCAEw4CHgID2QsF3AskAgoAAA4BVAAARgQcAAAvAgUBLgACDjBU +LwEwCy8BMEYvATCXAS8BMBwvEgDGAfwCAJQBGAQFCBAGIQIJAgwPDBoMAgcBBAoGFwIYEAUJAg8U +DwYEAh0OIAcFGAUEClcHWAIJDE0HYAkCDy8RKwsREgAAkAICWgESAAIOgAGFAn8BgAEVfzUAzgGg +AQgEB7oBAEoYBAUECCoDKQYCJAYLCgcKAw4DIgUfBQ8HDgULCQIPAQUEA1EEUggqCikCCA0CEwEF +CAoaHgQPSwEdFBI1AAAEAQYCDgHBAgIFAACEAQQjAhkBIgIZAhkHDwoGCTUAAKABAgQBcQQUAzUA +Ag5ATT8BQFU/KADOAdkBAKgBGAIDHAMbCQIeGhcNBgICAgGDAROCARQGAX8ibAIDKAAABAEGAg4B +vAECBQAATQQXAw8EGQMPBBYDKAAAZQITARUEIgMqAAIOwAFfvwEBwAHeCb8BAcABuQK/AT8AzgGA +AUcWUApPNkgCCAtPM0gCRw1IBUcBUA4HLUcWUAhPMkgCCAtPM0gCRw5QDgdsRxZQCk82SAIIC08z +SAJHDlAOB1lHG1AITzBIAggKTy5IAkcNUAwHiQEIAwekAUfNAUiTAQDGAh0SDAgIDgQCCwYSDwMM +AgsXPhKFAw0CCcYFCsMFNogDArwCC78FM4QDAv0CDf4CBfcCAZwFDqECEXQDCgN7BQIKBgeXAw0C +CcYFCMMFMpQDArACC78FM5ADAokDDQYBnAUOlQIRBQcCBAYHcAMRAxINbwV4B3UGCARcA1MJAgMC +EbEDDQIJxgUKwwU2rgMClgILvwUzqgMCowMNBgGcBQ77ARQCBkQDAg0QBikDKgNVBXMFuAEDOwYD +DAIEuwMTAgjGBQjDBTC8AwKIAgq/BS64AwKxAwwGAZwFDO0BAwEDAgoDCS4DIA0pAxgDEgUpEiAF +BQUGBQoFHwcOBQUFBREGDe8DA/ADCBQKEwIGFQITAQUIGQYZBBknGNsDFAMWBBoDHAQaAxoEGgMf +rAMDDghZCgENCgJEBg4EUQIjA3YDDQVzGRM/AAAEAQYCEwGhDQIHAACnCQQdAiECMQEZBCwCqwIL +PwAAgAECFgIKATYBAgYLAzMBAgINAQUCAQYOBy0KFgIIATIJAg4LAzMJAgoOBg4POxIHESoUFgIK +ATYTAhgLAzMTAhQOBg4ZWRwbAggBMBsCIAoDLhsCHA0GDCGJASQDI6QBHCoHNgk0BzkBkwEAAlwA +0gELWxlcFVsYXAsA0gICCAICB+gVEAIJ6xUOCgfgFQ8CCeMVCggBAAFcAAALAhkBFQQYAwsAAg5w +2AFvAXCfAW8TANIBogF3A3hTWwRcA3cDeJcBAIYDGAIlAgUEBQcEAgUKFgIJBgsGDwgTAgIOBKMC +A6YCBQYHAgoCCgsFEAsEEg8R7gME/QMDlwIDmgIHAgQFEBcFIg0UBQ8FAwUREQUWBRENEAETAAAE +AQYCDgH8AgIFAABhAqUCARMAAKIBAgMBUwQEAwMGAwWXAQACDmCqAl8BYDpfIQDSARhbCVwBW0Zc +DFsFXEJ3BXgFdwN4nQFbDlwhANQDGOQGCeEGAeAGAc0BHdABKN8GDOAGBdkGDAQNDQQUBQUbKgX3 +AgXSAgXRAgPSAgcCBAILAg8gJAINBBEECgkMKQoHFt4GDuMGIQAAsQECfgEPAjUBIQAAGAIJAQEC +AQIdASgBDAIFAUIGBQUFBgMFnQECDgEhAAIOcNkBbwFwDm8eANIBTIcBCQMCjAEOdwN4QYsBAQQK +iAFgAKIEGBISEQUSBQQEAxSDBAkyAtYDDqMDA6YDCAEDBhUGAw0IDg0NCAwB3wIBnwEKggQdAg0C +ChMOER4AAAQBBgIOAfcBAgUAAEwEXAMHAkcBHgAATAQJAQIBDgYDBUEIAQIKCWAAAg5w7gFvAXAO +bx0A0gFshwEIiAEMiwECFAN4P4sBAQQKiAFZAM4EGAgSBwUSBAkZAg0CE6kECKwEBQ4HhwQCMgPK +AwgCCgQVBgUNDQ4FAQGFAwGfAQqoBBcCDQIKFw4HHQAABAEGAg4BiwICBQAAbwRbAUEBHQAAbAQI +AwwCAgQDBT8IAQIKCVkAAg5wlwFvAXAubxQA0gE7iwEBBAqIATGLAQEEBYgBSncDeB4A9AQgCAIQ +DAIMAgGtAwGfAQq6BAcEBQMJAhybBAExBdAEHQICEgoPCAIKBgoJBe0DA+4DCg0UAAAEAQYCDgHL +AQIFAABBAjcCJAMXBBQDIQAAOwIBAgoDMQYBAgUHSgoDCR4AAg7gAZUB3wEB4AHYAd8BAeABwgLf +ARIA0gFrBA4DXYsBAQQOiAETWw5cBlsFXBOLAQEEDIgBMVsEXHuLAQEEBYgBEIcBDogBCYsBAowB +G3cDeDZ3A3hqAKQFGAgEAg0GCw4JHQQ4BzcEPgU0D6EDAtACCQoOBxMCBQYTAxHTAgMCCAMGogMK +AgUGAbsFATEO8AUT2BIO1xIG2BIF1xIDBAUMCgcBxQQBnwEM7gUKBBYCBQgJBANoBE8XIREXCjsC +AgYDBwoWCg0KBAkTkwUBMQXIBQ8CAbcFDpgGBEkFrQQCnAQEAhfrBAPuBAUDBQIIBAUCEgQJAgT3 +BAP2BAsCCk4ESQUPDQkKPxEFEgsSAAAEAQYCDgG0BQIFAACHAQLbAgIaAcMBARIAAGACAgEJBA4D +PAIRARAGAQIOBxMKDgkGCgUJEwwBAgwNMRAED3sSAQIFExAYDhcJFgIVGxoDGTYcAxtqAAIOgALn +Af8BAYACCf8BAoAC5wP/AR4A0gFlWwlcAVs4XDFxFHIhWwVcBosBAQQMiAEkiwEBBAyIARRbCVw8 +Ww1cV4sBAQQFiAEQiwEBBAqIAUd3BXgKdwN4eQDsBhgCAwQGAg0GDQ4NFgkvBQQKMgWUAwntAgHs +AgHNARHQASbrAgwICRYG9QQC6AQTTwHXBBTgBApiC/8EAwIDAwbEBwXNAgUCAYkHATEMvgcjAgGN +BgGfAQy0BwIHDX0FOgkkKwIR6AINkQMCAgMDCTMFPiACDQoECRPNBgExBYIHDwIB0QUBnwEKtAcF +PQQCIEcFRgUCBQIKCAWvBgWsBgqrBgOsBgsCCAINCQ0JCzMRBRIJHgAABAEGAg4B6QUCBQAA0AEC +BwEQAg8BKgJTAhwBhQEEGwO5AQEeAABlAgkBAQIBAhEBJgEbBgIFFAgUBxUGDAMFAQYKAQIMCyQO +AQIMDxQSCRE8Ag0BVxQBAgUVEBgBAgoZRxwFGwocAxt5AAIOwAFvvwEBwAHTAr8BPADSAWl5BHo1 +BAQDLw4DDV0EBQPTAQCaCSg4BTUDAgQCBAIIDBICEAwHpwgEqAgFAgI4CjcFDQUOBQIEAg0CBKEJ +BKQJAhcFJAUTAi4JBQUTBQ0FBAnIAQPFARoLBgwCCwYaEikDMgIEHsEJBa4JCwJ6AhI5PAAABAEG +Ah4B4AMCBQAAywIEDwIgARsEPAc8AABpAgQBNQQEAy8GAwVdBAUD0wEAAhbQAuACzwIB0AIUzwIB +0AIUzwIB0ALvAs8CHgDSAfMEeRR6pwEA/AkmBAsIBwgOGAgJCBUKFhNMCksVBggCBwIrQgo3GRwB +7wEDCBgCCgIC9AEIBwcIEvcBCIICCAIKAQcCCVEVExWmARU/NAIHAgQCBAIEBAYCCtYKCAIFAgkB +A8sKAtIKA+sKCiAHEwUOBQoIAgSpAgiIAggDBS4FLQUqBQoDMwU2BQEFmhIUzRIFgwIIiAIIAxwq +BK0CBK4CBQIXbQNuDAIFuQIDCAu4AQO1AQqGAgODAgKSAR4AAAkBBgIXAYMGAgUAAG4EWQIoBUQI +xQEHPAo/CQ8KjgEJHgAA8AECJwEhBAgDuAEGGQUCBgMFJwQIAy4IFAcFBAgDKAQEAzAEDgMDBAoD +AwQCAx4AAg5AjwI/HwDSAbwCANALGAgI8QILAgvyAhMCAf0CBAgLAgv2AgH/AgQICwIL+AIBgQME +CAsCC/oCAYMDBQgLAgv8AgGFAwkICwIL/gIBhwMJCBMCDQIFiAMPAgojHgAAjwIEDwMeAAAgAhYB +FAQaAwEGGgUBCBoHAQobCQEMHwsBDi4NNwACDnA1bwFwggJvKQDSAWiHAQWIAbwBhwEFiAEKiwEC +jAE1ALwMGAINAREOAgkMCgQaBBkIAgkBBRYGrwwF2gsFowMF5AMFMAUZAgwBYRICAa0DCQgLAgsC +Ao4EEQIHAggCCAILCgUOCg0PCAQHCAIJAQUEBssMBdYMBQ0FpwsCtgsNWygAAGgEGwNjBkMCHgco +AABoAgUCBQIFBQ0EEwIhBXEKBQkKCAIHNQACGdAGmwXPBgHQBpQBzwYeANIB5wYAog0pAgsGDAIJ +CQheCFUKCAkGCGoKVx8CGgIOGxAkEbEBD7QBIw4B2wESAgGtAwQIGgIPAgKeBRQKAd8CAdMCDwgU +Ag/iAgHrAgQIFAIPrAUNAgGXAgGfAwwIFAIPsAUNAhMIEQILAgwCDwYzDBICEFkRvQEJAhECEgIC +zgEICBsICgcFUREHEgMeAAAMAQYCFwG5BgIFAABvAncBhAECOQEwArsBAnwBKAIXAx4AAOYBAg8B +JAQTAi8FFQgBAjIBAQQnCw4OAQIvD70BAi4BcwACGdAGqgnPBgHQBuQNzwYKANIBMRkDGrkBLycw +3AUOAw3DBCkKKuACDgMN4wQeDx3lAh4LHS0A6g4pAgQEBOMKA+QKEgMIChsSDSowAgwCDAIMAgwK +ERwCAgSfCyegCwIGDgIaCAgtBS4MBgQGCQQJDwUGBQwJAjACJAPWAggIRwVIEkcHVAICFwYdCAkG +BwIFBghKBUkJAgUIDwgUDAwJC7kFA8QFAgkLDAgCBwIQAg0KEA4ODQIODhgPCBIIFwIiAhApDQI5 +CJEBAhEFQx0DARMYCBcFORMFEacCCpoCigECVQIRHRVnMAIMAgwEE4MEA4YEFAgSAiICMAIkBZwD +ChMCGJUDD5gDERMNAhcEBgMFKQ0IugECPgIg5wIL6gIRCxIHCgAADAEGAhcBhBcCBQAArAIEqAEC +uQICiQECjAIHLwhSB4kBCHIHGAEZCosBB7oBCqsBAiQC2wIDUgkeEEsCOQIeAh4CHwJvGSwBCgAA +MQIDAbkBBCcD3AUGAwXDBAgKB+ACCgMJ4wQMDwvlAg4LDS0AAg4wzwEvATBBLwIwDi8BMAkvATAJ +LwEwRS8UANIBJikGKp0BDgMN0QEA9BEYAg6BBAaIBAYILB0jKAgMHwgNPgU3D5cHA5oHCAYKBhIC +BgYJEhcCCw4FAgohChsKBxELEQcRDxIDFAAAxAECEAFNBA8DIAQ5AxQAACYCBgEyBCMDSAYDBdEB +AAIO0AHRAc8BAtABpAHPAQoA0gEdiwEBBAqIAS2LAQEECIgBKIcBDgMCjAEziwEBBAyIAbkBAOgS +GgYCGgGlEQGfAQqsEhQCGY0SATEIwhIZAg4cAd8SDjICsBIVAh0QAcERAZ8BDOQSCxEFAggCCwQW +CQUGBQMRHwwCAwIJAwUCBQQIAg0EEQUMBQUMAhcKAAAjAjYCLAMIAjsBCQJ+AjYDCgAAHQIBAgoD +LQYBAggHKAwOAQIJMw4BAgwPuQEAAhagA9wInwMBoAMmnwMBoAORBp8DJwDSAZABDggNQw4NDQUO +CAkMChEJCAowDR4ECAoICQQKCQ03DhCHAQh6Cg4JDUAOAw0FDhENBQ4ICQsKEQkFCi4NDAQFAxoO +CAkECg0NOg4PhwEIegkOCA08DgcNCg4EDQMOBAkLCg4JBQMNDi0NCAQFAxIOAQkECgQNCQQEA6sH +AK4TOwIFAgkDCA4XBAwKCQIT3wMIjAQIJwJ0BWMIZAVzA3AOXwgQAx8GEAXzAw2IBAWHBAQGBM8P +DNIPBwIK0w8I1g8wqAQIHQQEBQ0N1RMIyg8IyQ8E2g8CDwMUBPYDFQoLCwUOEQoBmQMQ3Q8I+hII +DQKTBAmUBQ1fFSQDIQkIAwIEAgNSBFUEvQQDwgQFwQQRygQFyQQEBgTPDwvSDwcCCtMPBdYPLuAE +CBcEnRQFohQFPAhFCEYFkwUIyQ8E2g8CDwMUCLgEFQELChEKCdcDD90PCLgTBwEC3QQI7AQHCg4e +Ax0k9QQHlAUFGQX5BASUBQONBQTPDwvSDwcCB9MPBd4UDYcFLYgFCN0UBd4UEoMFAdkPBN4PBOgE +BQIDAgHJFATMFBgSNMkBJ44BwQECEQujAQIWNcUBAhEPngECEksnAAAJAQYCFwGnDwIFAACLAQSP +AQJeBfYBCGEH9gEKHgIuC48BDiICYQJPAiwDJQJoBB8FaAJPAiMDIwJkEScAAJABAggBQwIHAgYD +BQIIBAwDEQQIAzABHggIAwgEBAUCAgMBBAE3ChACCAsKBAkDQA4DDQUEBwoKDQUECAwLCxEMBQsu +AwwSBREaDggEBA0CCgMJCAM6FA8CCBUJDggNPA4HDQoOBA0DDgQKCwkOCgUXDQ4tDQgaBRkSDgEM +BAsEDQkcBBurBwACCjAkLwEwFy8HANIBTQCgFRQCEQYKAxcDBwAAOwILAQcAAg7gAfYB3wEB4AEN +3wEB4AER3wEB4AGyAt8BMgDUAYkFADIgCCAYCQ4LAggBJ5QBBQIHAg2XAQYCBQEFJRAsCSsLLgII +CgcNBRwNDg4IDgoHBAIGAQMCAwEEAgoBAwIPAQoCCgEFAhsBCAgFAwUDCgQGBAUDBwMEBBcDEwQL +CwglCiwDAwMnBAQFAwYoAwQDJQIECQgDEQsmAw0DFAQrBQwSEzIAAAQBBgIWAeQEAgUAAL8BBFcC +BQVcCD4HNgZsBTIAAIMBAhkB7QMAAgqQAVOPAT0A1AGaAQB2HgJAATwAAAQBAgIYAXcCBQAATwQP +AzwAAg7QAY8BzwFoANQBhQIAhgE1AmkBZwAABAEGAisByAECCAAAjwEEDwNnAAIOYG5fAWAvXwFg +E18pANQB6QEAogEgAgkeBiQFDQsGEQYFAgUDBQIFAgUCFCMOCAUCCQIUIxQJKAAABAEGAg4BzAEC +BQAASwQZAl0FKAACCjBNLwEwGy8hANQBlAEA/gEUAgwCAgIKCAYHAgQIBBwHDQEOAyEAADcEPAMh +AAIKkAFkjwEXANQBhQEAjgQUAgUBFQIJBAUCCgQPBBoPFgAAMgQ9AxYAAtEBANQB0QEAvgUFAgUK +CAQWCAkTBRQDMgcxBQIEAgoGEggKAggCBQQFCwUHBREDMBIvAzYRAgUKBkEDSAkHAQ0FAAAFAcwB +AAIEoAExnwEBoAGBAZ8BAQDUAbgBAKwGDgIFNhUEAjcMRg4CDgcFBR8EMAISAAAOAaoBAACEAQQ0 +ANYBHBIjEQoAggEOBA5NI1AKANYBNwD8AQQMDAYGAwcCEAIGAgQAAg6wAYgErwEBsAEPrwEKAOAB +5gPZASfaASMA5gcYAhkCGAIJAQUIbgIhAjwBBwIFAkMFBQYFGhcCCQITAjcKAd8HCQIPBA/cBwoL +DzEKAAAfAlYC8AIDFAYkAwkBCgAA5gMCJwEjAAIWoAPfBZ8DAaADmQefAxYA4AGlDQDyCCYuCCkD +AkAUFAIOBCgCGQIVBwUEDwQEBBMFCAgLBPIBAigCLQQPAhQUIQIYCBwMEAsMBAgDLgIhAQ0CDwEb +BJoBAhELEQMHEV0BHwIxAQUCVgoRFRkJCAEIAQgXdwIbBg8CEQcNAjMLFgAACQEGAhcB+AwCBwAA +ogIEKwJgAi8BMwMyCCMHggEBhgEMZAIjAiMCWAczB5oBEIYBD24QLQInAQwRFgACBDDUAS8BME4v +ATANLwEwcADgAWUJBwq6AgCqCw4CBzcQAhs4CQoNAgcGCJ0LB5oLCwYbDBICBAQjIw48HAgQAgYO +HQMOBAgPCAcIDQIBGwYEAhYHCAcIEREAAA4BmAMAALkCAm0AABUCKwElBAcDugIAAhawAlOvAgGw +Ap8BrwIBsALPAq8CAbAC2QKvAgGwApoCrwIBsAIIrwJFAOABtgMJBAoBCTUKEwkCCpcGALYMNgIE +FAUCASEOJAUVF6kBA9oBAhAaQxCUARBPBVAIIwcpCAQIBRYIAksDNgsIFwUyAhEWBi4DLQsuBS0n +AgYGEkELUAafCwSiCwGVCwwGBgMJAhQCBp4KC3QIjwsCkgsDAiAIBAUVDhkODYsBCowBBjsPRL8B +BCQCEAIIAxcGBQ8XFA8IVgc+AgYMERcIUw8pBQwDCwYMEgITAhcuAz0FGUUAAAkBBgInAeEJAgUA +AJICBHMCkQICGQIgAWUEmQECggECGgtuA0UAAEACDgEcBAMDyQIGBAUBBjUCCwcIBgIFlwYAAg5g +bl8BYAlfHwDgAUsLIgw4APwNIgIDDQkGFAgJ3wYdAgXeBhAHCQYfAAAEAQYCGAF9AgYAAE4ELwIJ +BR8AACUCHQEJBCIDEAIJAR8AAgowWi8BMBQvHgDgAZcBAIQOHgIHAgQCCgIMCAgCDAEDChcHDBEe +AAAEAQICGAF0AgUAAB4EKQMhBBEDHgACCmBpXwFgCV8oAOABQgsiDEEAsg4eAgUNAgYUCAmVBx0C +BZQHEAcJBigAAAQBAgIYAYABAgcAAEUEMgIGBSgAACMCFgEJBCIDEAIJASgAAg5gdF8BYBVfAWAV +XwFgEF8mAOABVAsfDHIAug4iAgMCAggYAhWpBxoCBagHEAYWDRYIBQELCyYAAAQBBgIYAb4BAgUA +AFQEWwIQBSYAAFQCHwFyAAIOwAGhAb8BAcABG78BAcABF78BMwDgAZYCANQOIgQJBioCIgIhBBgF +BAoKAg4RGAUyAAAEAQYCGAHvAQIFAABMBG0CKwUyAAIKUCdPJgDgAVcA8g4eAhQBJQAABAECAhgB +NwICAAAjBA8DJQACDvABhALvATMA4AHFAgD6DigCJQILAQQEqQEEDgkyAAAEAQYCHgGYAgIFAABI +BB4CJAIlAloJPAACDqABuAGfAQGgAQmfAR4A4AHuAQCMDyICBAIaAggCCAMYCAIMDQMEAwkEAwMF +BAQDJwICAg4PCQMeAAAEAQYCGAHHAQIFAACvAQQbAgYFHgACCnApbwFwJW87AOABOwkIClEAuA8e +AgcCDwkHjw8IkA8CDgsCCgk6AAAEAQICGAFxAgUAAEsEDwM6AAA0AgcCCAECAU8AAgpwK28BcCdv +MQDgAT0JCApJANwPHgIHAhEtB48PCJAPAjIXBzAAAAQBAgIYAWsCBQAATwQPAzAAADYCBwIIAQIB +RwACBDA3LwEwmAEvATCxAQDgAYYDAIgQEwYNAgQHAwgCAhMGBgIIJAsfEAIgBA4EBwIHFAcPBQID +ARYEDwMIHgQHAgMNAgQCEgIGFQMBAwMCFggbCgMLLAQrAyAKAgICByMDJAkCBiUDAQYCAwUFJgox +DgAAEwHzAgAA2AEErgEAAEoCCwFMBAcDNQQvAwgECAMVAgQBAwITAQMCDwERAgoBDgACDqABwgKf +AQGgARmfAR4A5gEwLQUuEi0GLrsCALYEHRgFEQkCBQEFCBEGAQ0GEBcCCQEDAgIBBQI0AQcCCwIF +ASsCIAIiAgsEDQIKBwcLEg8eAAAEAQYCDgHrAgIFAAArAhsBQQLMAQIXAx4AADACBQESAgYBuwIA +Ag5A/AI/AUALPwFAcT8BQHM/FQDmAVwvBDABLwowHi8vMCYvBDABLwowHC88MGgvBAIFLgovBDAB +LyYwNC8FAgUuCi8FMAEvJjAyAOYEHQICAhBaEQcOUQliBdwBBI8CAZACBgIEwwIDNBoYAYQCDwIT +BA3pARJnCmgFZQW8AgS1AgG2AgYCBMMCAw4YGgGoAhcCFwQO/wEFBhJXClgFFAVrCmwFCwwPDBkD +AhOIAgTzAgVuBQYFjAIEjQIBjgIPAg8ECJECDwQPMQMCE6wCBfMCBUoFBgWwAgWxAgGyAg8CDwQI +tQIPBA8jFAAAuAECUAFCAmwCDgNlBg8FWQAAXAIEAQECCgEeBC8DJgYEBQEGCgUcCDwHaAIECAUJ +CgwECwEMJgs0BgUIBQ0KEAUPARAmDzIAAg5QgwNPAVA9Tx4A5gHtAwDoBRgCMQYHBBICBQIvBDoC +IAcDCgIXBR4hAiSoCQWjCQoCAaIJEwILBA6jCQ0CChUIAQsDEAsRAQkBHgAABAEGAg4B0AMCBQAA +lgICOAFHBDoDHgAAvwICBQELAiwBcgACDlCsAk8BUDJPGADmAYUDAKAGGAIxBgcEEgIFAhgEOgIT +BwMKAhcFGh72CAXxCAHyCBECCwQO9QgNAgoLCgELDxUBCAEYAAAEAQYCDgHkAgIJAADvAQIKAUIE +MgMYAAD0AQIFAQECKgFhAAIOgAHIB38BgAHRAX9QAOYBfC8EMAEvCzAdLzAwEC8EMAEvCzAdLzYw +Di8EMAEvCzAfLzww+QItBS4OnwEBBAWcARCfAQEECpwBIS8rMEIvLTADLwIwAy8CMMEBAMwGIgIJ +BglyBQEFHgULBQsFAQpzAhQRFAkuCQEECwEMBwIELAM3GQoBDhcCDwQKUwdCCQEEJQEmBwIELANL +GQoBIhICFwQNCQlHBT4ENwE4BwIELANjGwgBPBcCFwQOEQUGEhQFLAUnBQEFAgUCBWELSAUMBQgF +LAUnBQEFAgUCBWELVAgDEgwFLAUnBQEFAgUCBWEQUAUXGAQJEwk0BAIZXANbCFwDWRxKBRAEVwQE +BCIEBgQMDQMEBAMLBg4BwgcF9QcEAgubAwWeAw7xBgExBaYHDwIB9QUBnwEKmAcbAgUEASkUAg8E +CCYKQA8VEgkEDAULBCAFBwVjFgIPBAhGA0UCRgNJAkoDFgICCQgFBwWuBwkCEQYc+QgRBRIDUAAA +BAEGAhgB0QkCBQAAygME3wIDKQYPAgwCIAIFCzEOKgEYCzoOaQ1QAAB8AgQBAQILAR0EMAMQBgQF +AQYLBR0INgcOCgQJAQoLCR8MPAvlAg4FDQ8QBQ8OEgECBRMQFgECChchGisZQhwtGwMcAhsDHAIb +GA42DXMAAg5giAJfAWD5AV81AOYBxQQAqgkYAgoSBxEFKAMnFwIFDgcCCwIbAgoVDygNDQIgBwQC +GQgCBwILAh8CCisPKAsIBS4KNQMhEEAIMwMHHEIJOQMiIwICLwU6BSMFCQMMIC4FCgkRFAgKBgcH +BUkPBgUiECk1AAAEAQYCDgGlBAIIAABaBCACRgEkAikFrAEEVwM1AAIO4AE+3wEB4AGIBd8BAeAB +Hd8BIQDmAZQGAJgKGAwJAgomBRURBgINCg4cAgoCDAMFAgUwDycYMBopAwEYAgsCGwIKBBIECRYP +MQwOBQMQFgoPAxIcBAgKCwJkBA8DCTEIIgUYDgIFOwoyDwoFMQQ4BgoFSwoyERoFBRICCUcFMg8W +BQwFAg4MClsRAwwlIQAABAEGAg4B9wUCBQAA6wEEIAKMAQErBHEBlAEEBQkWChEJIQACDmCLAV8z +AOYBzAEApgsYAgoCBwEKAgUBIAINAh8CDAYKDTIAAAQBBgIOAa8BAgUAAFsEKQIMBTwAAg5QwgRP +AVAMTwFQEE8BUB5PHgDmAasFAOILGCQKIQICKAIMAQUIEVYIURMUAwkiTglDAwgaAgUEDAIfAgoE +EgITLw1OCRUFHwMiGgIFBAgCDQIPAhsCCk8KTgIWBQoKbQVuBxMXWQVaBAoFBgppBWoCBwphCAwP +FA4CDBENKhE1FgMIAx4AAAQBBgIOAY4FAgUAANwBBCgBdwI0AZEBBE0FHgACDqAB4QKfASoA5gG4 +Ap8BAQQFnAEdnwEBBAqcATMA5gwdNgUjCQISAgkEDQQEAgMWAxUYAgUCDgQeAh4CBQQBmgIYAgsE +EJkCCAIYAggaE9cLAZ8BBfwMHAQB3wwBMQqeDQpnKQAAoQEEIwIFBTQEIAIcAh8CCQIPCykAAMoB +AjMBOwQBAgUFHQgBAgoJMwACDsABkQW/AQLAAc0DvwEXAOYBhQkA5A0dBgsKLgQYbAVRBREFBwkC +BQYFAicCEWAFawUcBRsKCAUIBVwFWwMCBR4HBBgCBTUFHAUbChoFGAs6CzkDOggvBSMIAiQCEQYb +AgoFCU4FawUcBRsPbAVDBQ0DSgoCIksDTAcMAwsCBg8CCQQFAiJ3B3gFAgFCBQILBA49Cw0ICwgG +A2cSQg4GCl8DQh8pCjwGKwYnAyosNgoKBTsFAgUbBRoFEQUSBRgFLQVoCxUFERIpBQIFGwUaBREF +EgUYBS0FaAsnBQgKBgU3BQIFGwUaBREFEgUYBS0FaAsZCAcFLwUCBRsFGgURBRIFGAUtBWgLIQUv +CA8JHRcAAAQBBgIOAeUIAggAAMMBBNABAiABSAOpAQirAQGRAgIOBxcAAPgEAh4B7wMAAg5AqgE/ +DQDmARifAQGgAQebAQWcAXOfAQEEDJwBIADiDxj/DgGKDwe7DwW+DyoCBQwMAgoaBg8HAg4OBgkC +CgoDAbEOAZ8BDNYPFDUMAAAgAoABAhkDDAAAGAIBAQcEBQNzBgECDAcgAAIOQOwBPwFALT82AOYB +3gIArhAdAgUGBQUFAgUBDQYFDgMNEwIFAQUEBQQFBgoFCQIKAggCFAYiBhICFwQKBQsFCwULDQwH +NgAALAIfARgCjgEBEAQnAzYAAg5A9gI/AUBZPzIA5gGQBADaEB0CBQgFBwMCAgIFAQ0GBRADDxMC +BQEFBAUEBS4FJQUHCQICAg4ECQYPAh4UAxADLQQSBQIQAiAIAwQDAwUKAwYDDwIKAwYFBQUGMAYS +AhQECgULBRYXCwELCQsBCxUMCTIAAAQBBgIOAfMDAgUAACwCHwEYAjYB8gEEUwMyAOYBLQC4ERQC +EgEHAAIOMDIvATBGLwEwLC8LAO4BQqcBAQQMpAEgpwEBBAykAQ41DjYBpwEBBAykARkAmgcYAh0C +DAQBvwYBMQz0Bh8CAcMFAZ8BDOYGDAQBAgGhAwUGCZ4DAc0FAZ8BDPAGDxcKAABKAiQBCQIRARkE +FAMKAABCAgECDAMgBgECDAcOCgUCCQsBDgECDA8ZAAIOMIIBLwEwCS8VAO4BN6cBAQQNpAE1pwEB +BAykASgAugcYAgkCCwIFAwUKAeMGATENqgII7gQBDAcCCgIJBAoEBxUB5wUBnwEMigcKCQoHFAAA +PwRSAx4AADcCAQINBAgHAQYrBQEKAQIMCygAAg6wAZgBrwEBsAEQrwE8AO4B8wEAjAgdLhYrBRgT +CgoEBAEKCAQrBCwpAgkCChUHAgodOwAAMwJcAikDOwACDrABmQSvAQGwAd4DrwFiAO4B6AgAwggl +GAgwDQsIOQMECe0FA/AFGAIHAgHzBQMCCPoFBQEaAgkCBAEDBg0CDiAIFw8GBAIf7AIVAgvpAgYE +As4CB8cCAcYCBAICAg4BBAQDAwsCDwYVAgMCBNECDM4CCAIEAgTRAgwCAcICBAIFCBLvAgMoAr4C +BgIFuwICGwUqBQsWuAIJAgK1AgcCCgYFBAYECgsRtAIOAQQEAwMSAgsGEQIEAgTBAgMIBQcFAwXC +Ag8FGgEEBAMDCwIPBhACAwIEyQIFxgINBRwBBAQDAxQCCwYNBQ0GDQUbAQQEAwMVAgsGEQIEAgQD +C9MCBdQCDQUNFA+XA2IAAAQBBgIOAcsIAgUAAGsEFgKlAQXMAQgsBxYK0gMJYgAATgIDASAEAwII +BYQBCCAHCAoHCQEMUQsMDhANDQ4bDQUQCw8iEgsRNxJLERISHAFFDwUQGgFPARoBWQkFChoBDwdi +AAIOcLMBbwFwCW88AO4BhwIAogkYCgoHBwIFBAgECQEDBgMFAwYDBQMGCxAKCwMGBQkCBQkWCgsD +BgUDBAEDBgcFAgYKAgMEEQIMAQkfPAAABAEGAg4B6gECBQAAVARdAhoFPAACCjA8LwEwLS8IAO4B +FDcEOAE3DzgfNwU4AacBAQQMpAEiAMwJFKUCBKgCAaUCBgIJpgIMAhOnAgWsAgHzCAExDKgJGwsH +AABVBCADBwAAFAIEAQECDwEfAgUBAQQBAgwFIgACCiBQHxEA7gEapwEBBAykAQQ3JjgaAOAJFAIF +AgGBCAGfAQymCQSxAg8CDwQIrgIKCRAAACICLwEaAAAaAgECDAMEBiYFGgACDmDlA18BYFdfHADu +AUynAQEEDKQB/gKnAQEEDaQBggEA8AkYAggCGgMEBAYCAp0CBaACAZUJATEMzAkFBwUwBCcFoQcD +pAcBqQIHAgsCCQQKBAeiAgynBwOsBwwEEQIJCAW5BwUCAroHAwIIBhoCCQIBuAEZAgu3AQegAQIC +DQEEBAMDEwIOBhQCAwIEqQEBngECAg0BBAQDAwsCCwYQAgMCBNsBBzYCAgHJCAGfAQ3uCQ+cAREF +DQYNBQ0UDcsBEh0cAAAEAQYCDgHKBAIFAABUApQBAu4BAwkCbAEcAABGAgUBAQQBAgwFEwgDBwEC +LAEMCgMJKwwFAgINLxAkDwcSUhEBFEMTChYBAg0XDxQeARoBDQ8uAAIOkAGFAo8BAZABCY8BAZAB +C48BAZABWI8BAZABDo8BNADuAXunAQUEBaQBdqcBAQQKpAFgpwEBBAqkAVQAnAwdAhEKDwMTBCQI +B80LBTEFggwkCggCCgIJAgsCCQIIMw02CRUEAgHRCgGfAQr0Cw4LCgkMKAQCBQEICQUQBwIQAg4C +AfEKAZ8BCpQMEkMOCjQAAAQBBgITAaEDAgcAAEYCuwECEwNJBg8CHwMGAzQAAHsCBQIFA1sGDQUO +CAECCglgDAECCg0SBg4FNAACBDBULwEwKC8BMA4A7gGQAQDeDBMGAgMKBA8CElMLVgIQDA8RAgsI +DV8OAAATAX0AAIoBBAYAAEACCwE3Ag4AAgowJi8hAO4BUQCADRQCHQEgAAAiBA8DIAACDkCpAT8B +QDA/HgDuARwXBBgyFwUYrwEA3g4YAgSRCQSUCSACDQUFjwkFnAkRAgUGCBkDHAEbAx4IBAMCBQES +AggBCAQKAw0LEQcSBx4AAGMCCgFTBCgDHgAAHAIEATICBQEeBAMDAQYDBYoBAAIKcEpvCADuAVwA +og8UAjcCCgMHAABGAg8BBwACDoABlAF/FQDuARg3BDgBNxU4DTUHNio3KTgeAKoPGIMIBIoIAYcI +BgIPiAgMAgH3Cgf6ChgCEQIBgwgSAg8ECIAIChEUAABZBEADHgAAGAIEAQECFQENBAcDKgYpBR4A +AgowOC8BMBIvBwDsDxQGCwILBg8CCgUSCQcAAE8CBgEHAAIKcGRvFwDuATk1BTZHAIgQFAILAgsC +CgUF0QsF2gsWBBEECg8WAABPBBYDIAAAOQIFAUcAAg6QAX6PAQGQARKPAQsA7gE4NQU2bQCcEBgC +CwILBgrvCwXyCzoEDAIKCxIFCwAABAEGAg4BjQECBQAAcgQnAQYBCwAAOAIFAW0AAg6AAaABfx8A +7gHNAQDCECICCwISBC4COAIKCx4AAAQBBgIOAbABAgUAAEYCWAIRAx4AAg7AAboCvwEBwAELvwEK +AO4BNTUDNqYCANIQGAILAgsCB6EMA6QMDQJ7AgkCNQQHAjoECg0LCQoAALMBAowBAQ8CBgEKAAA1 +AgMBpgIAAgpgO18IAO4BTQD2EBQCBAIOAhYCCgcHAAA3Ag8BBwACCnBRbxsA7gF2AI4RFAIEBgUH +BAQYBBkCCgkaAAA0AigBGgACDpABlwGPASAA7gHFAQCcERgCCwIHAgkCLwQHAjMEChEfAABaBEID +KQDuATsAshEUAhYCCgMHAAIOkAGgAY8BFwC6ERgCExgLAgcCEgJWAgohFgAAmwECFAEWAOARGLkK +BMAKAb0KBgIPvgoMAgGtDQewDRgCEQIBuQoSAg8ECLYKChEUAAIOcIABbwsA7gGZAQD+ERgCDgQy +BC0ECg0KAABRAjQBFAACFtAC8QLPAgHQAg/PAgHQAk7PAigA8AHGAQ8DEBBRDlICUQE4BBqgAgA2 +LjYFMyUCDzINMQ4CBAIFBAkCBgQJAhENBRIIAgXACgO9CgYICqwFDqkFArIFAesFBDoI2gkH1QkU +1gkH0wkCAiopBTINAQoCGAIQBhAfNwIXHygAAAkBBgIXAeMDAgUAAMEBBCgDRgZJBSoIRAcoAADG +AQIDARAEDgMCBAECBAUICAcHFAgHB/YBAAIZ0A2kBc8NAdAN8RjPDQHQDUTPDQHQDfMUzw1kAPAB +UDkWOgc5AjqkAg8DEOMKDwMQfg8IEKgFDwgQyAEPCBCAAQ8IECQPCgkMChkJCBoFDzIQJA8ICQga +GxkECgIQAw8EEIoBDwoQCA8KCQsKHQkIGgkPLhAqDwgJCBodGQQKAhADDwQQ0wEPCBBJDwMQ3gcP +CBCvAQ8IEJAGDwgQugMPCBDFAwCIASkCDgoZQwYECwgDBQJeAwgEZQJmBgIJAgcEAggEAgQIBg4G +AQMBDJIFA5EFJgIIAggKDgIQZxBaCAIIDAXOAwiUAhBfCIMFEI4DCMcDBEQFCgsCFAgNhAkDgQkG +FBAEJwhSHQ0fByACAkECHgMLBAIDAwgCBhIDEdIFGAIQAgwCEAIMAgwbCKsFCC4DLQbsAgO9Aggt +CaIFCNMCA78CEAIMARAQBgIRCgQEIggENQXsBQWPAQSNAggeBKsCC+YDIOUDBQYwBAYYBgYMAhkC +BAYQBQUGECkFDgwCFQYTAQUCDAIZjAIF6wII7AUFnQMIagjFAgYWBdQDBfsDAjEIGAIXCEQQpAQQ +owQQDgUNLgYMEAVYCIwBBfABBdsECOwFBZ0DCJ8CEFIGpgMIcCBvCKUDBgQPsAQOrQQFAhskBggM +swEItAUU+wEIgQIFOAglCwIRAg3LAQjyAQcfDZAHA40HAqUBCGsFxggIwwUNbwWRAgmeAiedAgae +AgIQCA8CsQEIwgEIrQIDoAkIrQMIxQMbAooBAiEDC8YDCIUFCGsF8gUI1AIIwwUITQZ2BesCCOwF +BZ0DCJ8CENoHCPECCAEIIBD7AQWkAQVYCLMFCLQFEPsBCLUBEAYMKAoCDIIDCO8CDYEDB4IDAoED +CYIDCQIQwgUZvQUUvgUDuwUCBwcKK2oFiQMIvgIF8AEF2wQI7AUFpwMIgQMG9gMI3AEIzgMIywMI +HgPJAhACBAIUAhMrAywCFgxBA9ICCtECA0oCywMLzAMFAiDNAwv2AwWJAwi+AgXwAQXbBAjsBQX2 +AgidBgiBAwa6BAhDCN4BCAEI+QEGAgocFEsCTAJLArIBCGUIAgYEBD4DLwNSCKAJCL0NCMYDCjQI +MwUGBfIJBQYFzw8M0g8LAg7TDwiIBgXOCTLPCQXLAwi+AgWGAQUICPIBBfYCCO0KCM4BBroECAkI +CgWHBgTaDwLbCQPgCQTfCQMsCFcIBgUFGdwBCNsBBYMEA8gEBhIIAhSGAQiFAQaxAQKyAQICBYQB +CIUBCKAJCpcJBX4DmggFBgXPDwvSDwsCEtMPCPQGBQsE7gguuQgIMwhICEcF2wQI7AUF9gII7QoI +9AYFJAgKCAEInwcE2g8CwQgDxggErQgDFwhlA+EBCeIBEQQkBAXrBAPOBQQEFRADDQ0MEd0CA94C +AkoIjwEESBL1BAiYBgXpAQXbBAjsBQj2AgifAgifAQgKCBUQsAEDgwcI1AUF0wUD4AUICga2AwOt +AwuhAhW0Aw8DC1cCWBkMEAsQBAUBewKqAQIWAQOPAQVfCAMNBB4lCLwBCI8BBCsiAggMMAIUAQwG +RagBDacBBQIVBA0BHgQQAwQCMQJbAhFCCDMI2wQI4ggInQYI4AIIFwgKCAEINwUMCIQBEIMBCws/ +rQQIugQI5wQI7AUI9gIInQYI/QEI3gQIFwgKCAEINwI0CDMIChAKDQcdEBAPBAJFBkgCCAIVAh4C +UQIqAjMQCAswAp8BBA8CDAgI4QQILQjiCAidBgj9AQjeBAgXCAoIAQgHBhgGFwVNClUIAwUEDwIE +HQjWAQilAQgBCAcEJSACCAYu+Aga9wgCCAgCCBwIqAEIzQEFBA0CAwIIAggcCB8FAhUCIQIIAQoB +Ba0DCL4CBY4BCPIBBfYCCJ8JBroECAkNCAgCCKYBCK8BCNoIDdkIAgQFAgoCBQIMAgMEBgIDkgEI +AQgOCN0CC8ABBQcKHQoJDNsBEUlZAhatAhEjEgNkAAAMAQYCFwGeNAIFAACOAwS9AQJKAhUHIArr +AwLjAQKYBAIlAiYBjQEPywIUxgETKBRvE7oBFoEBFewBGIUBF7kBGnIZkgEciwERjQIUmQECVQIa +AhICSgXmAQupARR1AhICNwJJE+ADD2QmOy+AAwpMKCYnTAlkAABQAhYBBwICAaQCBAMD4woGAwUP +CAgHZwYIBfYBCAgHOAgIB8QBCBkHFAgDB3YKCAnIAQoICYABDAgLJAwKAgwBGQIIDQUMMgskCggG +CA8bEAQDAgsDDAQLigESChEIEgoCCwEdAggTCRIuESoKCAwIFR0WBAMCEQMSBBHTAQoICUkKAwne +BwoICa8BCggJkAYKCAnlARgaF7sBCggJOxoNGf0CAAIOsAHHAa8BAbAB+AOvAQGwARivASgA8AEm +DwQQAg8HCQsKFAkFGgUPMxAFGQUKAQkECgQQ7QQAhAkiJgTCBgTBBgLCBgQGA88PC9IPBgIJCQXJ +DwWSCQXEBjPDBgWRCQXaDwHZDwTeDwTTBgUBBQoXHAcOAi8PNBUCEAIKGgYCAS0HAhsuGyEFIg0d +BBQGAgEnBwIbKB0bBRwFBwwJBgIBGQcCHBogAgUPBRIMCwoFBSYBMwcOCQsbNAoEBQMTBA4BCQIC +FRsVBTAPBAoLDScLXSgAAAQBBgIYAegFAgUAAFwEHwIeBX8IUgcICFQHCAj3AQcSChAJKAAAJgIE +AQICBwILARQCBQMFAjMBBQYFAwEEBAMEAW8IIgc4CiIJOgwjC0YOBw0JDhsNtAEAAgowWy8aAPAB +FxkE0QEF7AFfALQJFAIDkwkEoAUF+AMFAggIFgYZAgoXGQAASAIeARkAABcCBAIFA18AAg6QAf0B +jwEBkAFajwE/APABpQMAzAoiAgYBAwQSAwoCBQYlAi0KDAgCDgkCAwgEBwMIAxACCxAIBAIIAgQC +CgoHRwJIFRVGAhQ7PwAABAEGAhgB/AICBwAAXAS4AQIPAiABIwU/AAIZkAqjBI8KAZAKDo8KOwDw +AYYFAKgLSAIrAhUCMARJIAMMBSsXAhIGDgIREwIYAg0FJAQTEQYWAg0HCAoCGQUSLgEIEAUPAxQH +IwUJCBIIBQ0iBAIMAg0EGCEOEzsAAAwBBgIXAdgEAgUAAIMBAv0BAqABAisFOwACDoABeX8VAPAB +MQ8DEGgA5AsYCAUDDAIIWQNcIQIFAQICBQETAgoECg0UAAAEAQYCDgF/AgUAACwCJAIpAgUFHgAA +MQIDAWgAAg7QAcoCzwEtAPABigEPDxAKDwQQ3gEA9gsqAkgGDAEIBgTyAg/vAgrwAgTvAkcCFQkC +CgUCNgQPAgoXLAAABAEGAhgB2gICCQAANAQPAiABMQQTAiABQwRPCywAAIoBAg8BCgIEAd4BAAIK +YBpfMADwAVQAkgwUAgcCCgMvAAAEAQICDgE+AgIAABYEDwMvAAIKYD5fAWATXzMA8AGPAQCqDBQC +EQQaAgoECgIKDTIAAAQBAgIOAXYCBQAAOgQjAzIAAhnwBpkE7wYB8AYP7wYB8AaPAe8GPADwAbsC +Rw5IxQMAvAwpNA0vXgMYEgsCDSADHywCEgIMBA0YFQcIjAEOkwEJBAQCBAIECCkCEAI1BAYCGwQN +BBQGFQMQChAFRAIwARtLPAAADAEGAhcB4AUCBQAAmQICKgE/AqEBAY8BBCADPAAAuwICDgHFAwAC +DrAB4AKvAT0A8AGgAg8DEIgBAJYNOQI2Aj4CCAIFAgUBGgIWBggCGwYOnQIDoAIbAgUBAgIFAQwC +DwQKHzwAADkCdwIgAhgBDwFCBiUCBwlGAACgAgIDAYgBAAIOgAKPBf8BAYACC/8BKADwAUkPEBAC +DwgJCwoQCQUKLBAFGQUaBQ8FCQQKCRBkDwUQCw8aEAoPBBDlAgDCDSgCCgYIBQUSCpYCC3cFrQEC +pgIEBgTPDwvSDwcCCdMPBdYPLKECBbMNBbQNBZ4BBdEOBNoPAocBA4wBBLkCCgQcAgQCMAgKngEF +nQEKAQGgARqVAQqWAQSdAUIGPAJMAhoCNQQPAgoZCwsoAAAEAQYCHgGkBQIFAAA6BEkCQAUuCDID +NgZmAhACQAJBAlkRKAAASQILAgUDAgIIBAsDEAQFAywBBQgFBwUEBQQEBQICAwEEAWQEBQMLBBoD +CgQEA+UCAAIOwAG0Ab8BMwDwAfUBAOwNHQYJBI4BBg8PMgAABAEGAhMB0wECBQAAogEEIQMyAAIK +wAFZvwEBwAEIvwEHAPABcwD2DTACBQEFAiACCgEIAQcAAAQBAgIOAV0CAgAATQQZAQYBBwACCrAB +SK8BAbABC68BPADwAZoBAIIOGQJFATwAAAQBAgITAXwCBQAARAQaAzwAAgpAST8BQBo/SADwAbYB +AI4OHgIEAiMCDwQbCUcAAAQBAgIYAZMBAgUAAFsEFANHAAIOcG9vAXBKbwFwqAFvAXALbwFwDm88 +APABKDkSOgk5BioDEK8BGy8cBBsEHJYBAKIOIggG1Q0MCAMFA9QNBgID1Q0GugoDpgMGEwQcCjwQ +OwICDAYHDiwxBzICAg8GMrMJL7QJBLMJBMIJMA0QIwwHDw07AAAEAQYCGAGhAwIFAAB+BGMCGgUg +BhMFmgEAACgCEgEJAgYCAwMUBhAFiwEILwcECAQCMAlmAAIOgAGFBH8VAPABYDcGOBE3FDgNRwRI +GwcWCNsCAJYPGAIGBgYIBQIlDBLGAQbFAQ8CAsoBFMsBAwICAQMdBaEBBNIBCgURlQ8WmA8qBAMI +AwsEDAUHUykILAICGwQLAjYEDwIcBCACCkUUAAAEAQYCDgGLBAIFAAC3AQSYAQF/BEYFFAAAYAIG +ARECFAENBAQDGwYWBdsCAAIO0AHWAc8BFQDwARg5BToKORU6DTkCOq4BAOAPGJEPBZQPBgYEkw8Q +CAMFAowPAwYHAgOTDwKUDwUaBRsFBQQKEQIKAgHbAxjwA0kiCkUUAAAEAQYCDgHcAQIFAABeBHgC +DwUUAAAYAgUBCgIVAQ0CAgEvBBgDZwACDnBhbwFwfm8XAPABOUcGSFlHBkhnAIQQJAIVkQIGkgIP +AQQCCwEHAgICCgQPAgoKD6MCBqQCCQIeAgwEAYsEE5AECh8WAABUAhIBCgIoAQ8CIwIHAw8EBQMg +AAA5AgYBWQQGAzQGEwUgAAIO0AGUA88BMwDwAdUDALAQGAgEBBEGAwgDCwIBAwMIBAUgBQYFDBAj +BQcHCAgilwECNSMLBA8OBBEMCgoODAwEEQgSDh8KHTIAAHMCoQICDwMyAAIuAPABLgDuEAgEBgQG +BAUCBgQDBwYDBgABLgACDkA4PwFAGT8BQBM/AUAvPwFACz8eAPABKA8DEFUbGxwzAJgRHAQMiwYD +jgYCBg4CDAQFEQcYAgYMBBQEC5UMG5YMCh0MBx0AACMEeAMzAAAoAgMBVQQbAzMAAg7QAawBzwEB +0AFVzwEVAPABpQIA7BQdAgoUJgIHEwwILwcdAgUKCggFCBIHHQIFCgkCCgIKJxQAAGACUQEcBDUC +DwUUAAIO0AHGA88BKQDwAWwbKRxsGyIcERsKHL8BAJ4VHQgIGgUdBQIKHAcbDgIIAhaJDh8CCpAO +IAQbBBsCFpsOIp4OEZsOCpwOOwQsAggCGwgNLSgAAAQBBgIOAeADAgUAAFECGwIpAg8DYAI6BBEF +fAEyAABsAikBbAQiAxEECgO/AQACCjBRLxIA8AFtANQVFAQpEBUCChURAAAEAQICDgFXAgIAAFAE +DAMRAAIOcIUBbwFwCW8BcAhvMgDwAdgBAPAVHQIKCDIOBQMMAgUCBQIFChECCiEKFAgXMgAABAEG +AhMBtgECBQAAiAEEDAMMBAYDMgACCjBULwEwCS8BMA4vGADyAY8BAGYUzAIFyQIDygIJyQIFAg4C +JwQKAw4FGAAAHAJDARIEBgMYAAAUAgUBAwIJAWoAAnAA8gFwAHgJBhEyBjEDPgIKBQsFIQUKBTwH +XwVKBQoFOwUOBSMFagVfBSQFMwMAAXAAAg4wSS8BMA0vATASLwEwGi8VAPIBqAEAkgIYAhIGDAgK +BQUEEwoOAxMPApwBCpsBDwMUAAAdAiwCMAMHBBQDFAAAewIKASMAAg6gASafAQGgAUGfAQGgAc0B +nwEBoAGhAp8BHwDyAX6rAQEEDKgBKKsBAQQRqAHAAwD0AhgCBCIEGwcCAgUMBhACGAIHAgkICg8F +GgE7AfcBATEM6AImAgErAYsBAZ8BEdYCBwQCAmMCBQoPCQUCZQEWBhElCgZvAhcTHwAABAEGAg4B +5QQCCAAAhgEENQITAZQBBG8DlQEDHwAAfQIBAgECDAUnCAECAQIRC8ADAAIOoAErnwEBoAGDAZ8B +AaABCZ8BAaABtAGfAQGgAY4CnwEcAPIBzwGrAQEEDKgBIasBAQQMqAGdAwC4AxgCDQoHBAIHDAgM +AhYkBBkKGiMGFwIQCAoNCisFEgF7AfcBATEMqAMfAgFrAYsBAZ8BDJoDCgJdAgIKCgkFAl0BEQYV +DgUCBQJqAhJBHAAAfwQ1AyMGLgIPAV8FJwpiA48BBRwAAM4BAgECAQIMBSAIAQIBAgwLnQMAAg6Q +ATGPAQKQAY8CjwEBkAGiAY8BAZAB0QGPASAA8gEmSRBKbasBAQQMqAEhqwEBBAyoAYcEAIwEGAIF +CgcEAqIDEKkDCwgMAhcOBAMFLhsCBRQPUwUSAc8BAfcBATEM/AMfAgG/AQGLAQGfAQzuAwoCWgIF +CgoJBQJjAREGETgMCA0HBQJ2AhYdCAIIAgUCEwILAggJBUsgAAAEAQYCDgHEBQIJAACrAQQuAg8B +XwMnCGgD7wEDIAAAJgIQAWwEAQIBAgwHIAoBAgECDA2HBAACBDAsLwEwXC8BMCIvATAGAPIBNRsO +HEwbCBwgALQHEwIFBgQZAhYTFQT/Bg6CBwMCCBUBFhIBAwIMAgYWBQITLwHtBgikBwQCBQIRIwYA +AA4BqQEAALEBBAYAABwCAgETAgQEDgMLAgEBJwEYCAECCAkaAgYAAgQwMC8BMLQBLwEwCwDyATkb +DhxIGwgcNBsIHCIA0AcOPwVCCAYFNQIyEzEE/wYOggcDAggVARYWAQMCDAIGMgc3BgIEgQcIggcD +AggVARYSAQMCDAIGFwHtBgi8BwQCBQIOOwsAAA4B5wEAAOoBBAsAAA4CBQENBAIDEwQEBA4DCwIB +ASsDBwoKBAgDCwIBAScGAQIIERcKBQUGAAIOQCo/AUChAT8BQF8/FQDyAT0bAxwDGwscXRsEHDQb +CBxkAOgHGAINBgRNAkoOSQT/BgOEBwODBwuCBwMCCBUHFhYBAwIMAgZKDAIFAgRTAloIawHtBgTc +BxECDAIPXwQCBIEHCIIHAwIIFQEWEgEDAgwCBlARUQtCFQAABAEGAg4BsgICBQAAxwEEcwMVAAAp +AgIBDgIEBAMDAwQLAwsCBwErARUIAgcICgECBAssCAgICAcLBgEFJwcRCAUFBgEVAAIZ8Ar+Ae8K +AfAK8gLvCgHwCr8G7woKAPIBrwERFxKOCgCcCCkCDgZ4bwcCBQYLbAgCCQUIBgMCJgsQDAMGBQUO +BggFDAYjIgoIFgIgAQgCWwIPAQcJDwwFAiIfCBUSBggwBR4QTQkQCBUSKAgdCAMfBAwCDAQVBi4C +DwoJCwUMtQEUCQIIASICEgI2DggFAgZfDSwCXwIsJRMCBxsLDQoAAAwBBgIXAaYLAgUAAKoBAl4B +dwQ1AscBBYcBCCYClAECpgECVwK0AQIkAjMRBgEKAACvAQIXAY4KAAIZwASmAr8EAsAEWr8EAcAE +zgK/BAHABJgBvwQBwAQRvwQBwARhvwQBwARWvwQBwARZvwQBwARAvwQBwAQ0vwQBwASrAb8EAcAE +Eb8EAcAEHb8EAcAEFL8EAcAEEb8EAcAEEb8EAcAEFL8EAcAEFL8EAcAE3AO/BAHABBG/BAHABBG/ +BAHABBG/BAHABBS/BAHABNMDvwQBwAQRvwQBwAQRvwQBwAQUvwQBwASCAb8EAcAEpAK/BAHABBS/ +BAHABC6/BCgA8gGDBRsEHC4bCBwBGwwc9xAbAxxPGwMckAIApgkpJBAfNw4+BBsGCQIXAQgIPQIT +BBUCDwIeAgcCEgUICgKpBg2qBgUEDQEKqwYRrAYFAigIEwYUFgoGLQYBiQULAgzlBATqBA2EBQGJ +BQoCDIIFCucJCOoEAekEDPAJDwUSHgoGVAYkBgUDEgUSOGJ1CAUEDgYGRQdaCwZ+BgY1CDaJAQqM +AQoGGgZLBiADEgUVkwEGAhUHFQcSGxIHFREVhgIxAhYlJwIVAhEBDQIIAwUGXAY0Bl0GMAISBRIF +EgUSGBUbBQEFUycCFgIZAQgBBQIICAwECAkZCAsCWgZQBiYCIgIjAhIJEgUSFhUhBQEFKyv5BBD6 +BAUCDQEFAhUCEgYBgwUODwUSBwQKAgb9BAP+BAz8BAH7BBkHCw8FEgcECwIH/QQD/gRB/gQuAgkB +BQIWAhIGFQcI/wQU9AQHhwULmgQoAAAMAQYCFwH4GAIFAABjBKgBAhoBOQRQAisCJQdtA5YBBCEC +GgGnAQoFDVIOBQ0wEGYBfQkhAh0BzgEKsgEEJgIeAX8CHQGWAQN8CCwCFQIaAiEBFwQaAiUCIgIm +AhwhWwpmGh0nhwIqNhsrDSgAAKYDAg0BHAQRA4wBBhcCBAENBQEKCgMFBAcJCggIAgECDAv+Dw4I +AggPPw4OBwUIFwQDAwwNAQ4ZAgsJBQoZBAMDEwEKAiQPgQEQDQEHDQcGCwUoAAIOMPEBLwIwmAYv +ATAJLwEwRi8eAPoBiAkA0gEYAgMCBwQNDAoCCAIEAgYCBAQSCwUIAgoGAgsLAhQKDgQCBAIEAgQF +EQIIBAUFAggDBwMGAwUIAgsCCwILAgMeAx8HAgITCxQsBBMCCgoGOgM3AgkGAg8JAwoDDwMQCAQD +DQMKBQMEBgYCFQ0DDgMPAxAIDQMeBw8FBgkiAx8FAgkCEhkDGgMfAyALCwMNAxoFAg0CEh0DHgMj +AyQNDwMNAx4FCCQIBwIlBgkEAgIGAgMCQQIGARgCSgIJBgMCBAJAAgYBGAJCCAMCCjEKKBABEQsH +ARI/DEUeAAAEAQYCDgHrCAIFAACoCARCAx4AAg6AASp/AYABwAF/AYABDn8fAPoBOiMDJDAlHSYM +JQomhwEA9gIYAgUBCgoEAgIHDB4B9QID4gIPAgMCCAMFBBGWBB2VBAyYBAqXBCkKBQkFCg8HGBQP +Jx4AAHAEQgJXBR4AADoCAwEwBB0DDAQKA4cBAAIWsAKdAa8CAbACD68CAbACpAKvAgGwAv0CrwIf +APoBhQcApAMmAgUINCgGGiMVHDEQQBANBAIDARYCES0DLgItAy4QASECBwEPAggBEgIEAQUEEgMK +BAUBEgIXASs6ECUEAgMBGQIRQwNEAkMDRBUCBgMGBgIFIQQCAw8CCAESCggEAw0GDAwLAwwECwMM +FQISAQUGJBEPEg4IBQcFAwUBCA4QDQsJD00fAAAJAQYCFwHXBgIIAACnAwQgAhIFnwIIWgEUBR8A +Ag7wAXjvAQHwAagB7wEB8AEp7wEB8AEL7wFBAPoBwQElHybGAQCuAywCBgEDAgcCDAIBqQIHrgIL +rwIFsAIbAgwLCQIKDgoQBR0FDhPgAxoCBeEDGAgsAgwGGwIPDwwRQAAABAEGAh4B+QICBQAAwQEE +LgJ3BUAAAEkCBwELBAUDYQYfBcYBAAIWkAJ1jwIBkAIPjwIfAPoBugEAmgQmBAYCBQgiCAoCHw0Q +EhAbHgAACQEGAhcBjwECBQAAWAQdAQcBPgACWgD6AVoAwAQXDAMLCAIIAQoEEAIWAAFaAAIOQJIB +PxUAJh+SAQWRAUOCAQuBAQGSAQMPCBADDweBAS0AugIYBAfOAQXLAQsGBwIJBgUDDwQU6jIL5zIB +nAIBDQLaMAQCBI8xARcCqDEH3TIPAgojFAAATgQUAjACDwcUAAAfAgUBQwQLAwEGAQICAwgGAQIC +BwcDLQACDnDSAm8VACZOkgEFkQEPyAEExwEByAEkxwEUggELgQEBkgEDDwgQAw8HgQEBIAEEDCMB +ggEdEAMPChAKDwEQBw8GgQEBIAEEDCNWAKIDGAIHAgsICgILAg9aBUcFBQq8DQS5DQG6DQ4CFrcN +FPQxC/ExAaYBAQ0C2jAEAgSPMQEXAqgxB/ExAeUCATEMmgMBrlYBtgcQAgydXQOgXQkEAaNdCqhd +AaddB+hVBq1WAekBAZ8BDI4DCQIMBBQCDwIKNxQAAAQBBgIOAdgCAgUAAIYBBGYDLgY4Ag8HFAAA +TgIFAQ8EBAMBBCQDFAYLBQEIAQICAwgGAQICBwcFARABAgwRARQBAhwCAwEKBAoDAQYHBwYTAR4B +AgwfVgBoKACoChICBQIKAwcAAgogVR8TAGhyAMgMFAYKDhMEDgIQAgcECx8RAAAEAQICDgFcAgIA +ABkCPQEcAAIKQFI/CwBoZwC6DBQCEAoPAgwiFAIKMQoAAC4EIAEPAQoAAg4w6QEvATAOLwoAaEMp +FioHKY4BKgopDioKAM4NGAINAhQCCp8MBQIRngwHjQwGBhMCHBYxCijoCwrnCw7gCwoAAB0C0QEB +DwIJAQoAAEMCFgEHAo4BAQoCDgEKAAIOQMABPwsAaNkBANYOGAIOAgkKBQIFAhgCBwIQAgoCBQgB +wQsLAiACFsALDAIKJQoAAC8CJAIaAQoBSQIPAQoAAHgCQQEgAAIKIB0fCABoLwCSEC8AAAQBAgIO +ARkCAgAAGQIPAQcAAgogHB8SAGg4ALgQFAIJAgoDEQAABAECAg4BIgICAAAYBA8DEQBoNgC2EBQC +EQYKBwcAABsCFAEHAAIKMFQvHABoHk4mTTYA4hIUBgkgAdELDwIPBAjUCwwGDzMbAABLBBQDGwAA +HgImATYAAhMAaBMA3hcHAgsCAQABEwBuKACKBBICBQIKAwcAAgogPh8IAHIUKwEsBycJKA0rAQQM +KBEAiAUUpQQBqAQH2QQJ3AQMAgGrAwGfAQzOBAoHBwAABAECAg4BOgICAAAcAi0BBwAAFAIBAQcE +CQMNBgECDAcRAAIEMI8BLwEwCwB6EQgECTMKAwksXhBbDQELANwBDgIDPgQqDQIHEBIRDSkDKgkC +ExAOAgL+AgkJB+0DDWYLAAAOAZEBAACZAQIGAAARAgQCFAISAQ0EAwMcAhAECQIHCw0ECwCMATYA +kgMUAhECCgMHAJQBLQDaBxQCEgEHAOwKFAIRBgoDFwMHAAAZAgwBFgILAQcAAgogMR8MAJ4BRwCs +FBQCHgIKAwsAABsCIQELAAIKQCg/CACoAToA3gIUAhUCCgMHAAIOkAHKAo8BFwCoAWgQEg8CEBQP +3wEAxlgYAgQCBwQFAhAGBQIQBArnSgroSgfhRxLiRwLbRxTcR78BAgoXFgAApgEEWgJZBRYAAFcC +CgEHBBIDAgQUA98BAAIOENMDDwsAOCYIvwFAHxAlQigECANSKRAUBxMQFAc7CUcKABQmzgQlAiUC +JwIlAimSAwcLGI4BJYwMJQYDwQ8Iwg8IAhACEQIp1w0QiQcHigcQhwcHxgcJ5wcKAAAEAQYCDAHR +AwIFAADlAQIHAWUECANSBhAFBwgQBwcCCQEKAAIOUJoBTx8AgAEwVjdVDlYOVUQAmAcYCAcFAw4J +AQMLAp0FBAwOAgoCFwQEmAUDAQuXBQ6YBQUCFwQKEx4AAAQBBgIOAaoBAgUAAJoBAgUBKAAAMAI3 +AQ4CDgFEAAIEMCIvAQCoAScA7DsOAg8CCgAAGAIPAAIEUNYCTwFQQU8BUHQAqAFFEAUPxwMAqD0O +AgQEBgISBQUOCwQEAQeDOQWGOQsCCUAQAhc/CQIlAh8IBQcCCBMEDwgNAgoCCQQKBg4EBwQSBAUG +CiIKBgwQDQIOAgcCCi0VASYOJ2ESAAAOAYMEAABVBGkChQEDDgEbBjEBOwIzAwYAAEUCBQHHAwAC +DjDxAS8gALYBnwIAqgYdBBgKEgIFCwYCCgIEAwYSEwIHEwYGFQICBhUCAgoRAg0KEgIDBgYECQIL +Nx4AAAQBBgITAf0BAgUAALQBBE0DHgDSASAA5BUOAhIAAgqAAVl/AYABDX8BgAELfxQA1AGRAQCA +BRQCDAIFAQUIBQIFAgoBEgIUBw4GCwsUAAAEAQICDgF4AgUAABkEMgIsAgYHFAACQACUBAkCAwID +AgQCBAIEAgcCCAIIAggCBAICAOoKBwIEAgUCBAIIAgQEBAIDAgICBQIBAAINAJwPCQIDAgEAAtoK +ANAQBQgFAgUCBAIIAgUEBAICAgYCBAIGAgQCBgIHAgYCBQYDAgIEBAIEAgIIBQIDAgcCBQQEAgUC +BQIFBAUIAQwGAgMCBwIGAgIIBQQFCAEGBAICCAgCBQYEAgYGBAIEBgUCBQIFAgUCBQIFBgQEBQgB +CAQCBAIIAggCCAIFAgUCBQQEAgUCBgIGBAQCBAIEAgQEBQIFAgUCBQQFAgUCBQIFBAUCBQIFAgUE +BAIEAgQEBQgBCAQCBAIEAgQCBAIEAggCCAIIAggCCAIIAggCBQIFAgUCBQIFAgUCBQYFAgYCBgIG +AgcCBwIHAgcGBQIFAgUCBQIFAgUCBQIFBgYCBgIGAgYCBgIGAgYCBgQGAgYCBgIGAgYCBgIGAgYE +BgIGAgYCBgIGAgYCBgIGBgUCBQIFAgUCBQIFAgUGBQYFCAEIBAIEAgQCBAIEAgQCCAIIAggCCAII +AggCCAIFAgUCBQIFAgUCBQIFBgcCBwIHAgcCBwIHAgcCBwYFAgUCBQIFAgUCBQIFAgUGAwIECAYC +BgIGAgYCBgIGAgYCBgYEAgUCBQIFAgYCBgIGAgYCBQIFAgUCBQIGAgYCBgIGBAYCAwIGBgYCBgIG +AgYCBgIGAgYCBgIGAgYCBgIGAgYCBgIGAgYCBgIGAgYCBgIGAgYCBgIGBAUCBQIFAgUCBQIFAgUG +BQYFCAEAAg1QLE8BUBEAAksAkh5LAAINkAEsjwEBkAERAJQeSwACG5ACOI8CCJACFACWHm8AABIB +AgJAAQUCFgACHpAEOI8ECJAEFAACcgCYHnIAABUBAgJAAQUCFgACHpAIOI8ICJAIFACaHnIAAh6Q +EDiPEAiQEBQAnB5yAAIekCA4jyAIkCAUAJ4ecgACIpBAOI9ACJBAFAACdgCgHnYAAAwBDwJAAQUC +FgACIpCAATiPgAEIkIABFACiHnYAAiKQgAI4j4ACCJCAAhQApB52AAIikIAEOI+ABAiQgAQUAKYe +dgACIpCACDiPgAgIkIAIFACoHnYAIgQCBQIFAAKtAhACEAYPAQ8TAALJAgCoAQMCAwIEAgQCBQIF +CAcCCAIEAgQCAwIEBgUCAgICAgMCAgoGAgICBgICAgYCAgIHAgcIBQICAgYIBwIDAgIEBw4HAgcS +AgYHAgMCBgIEAgQGAjAHAgUIDQIHAgYCAgIFCAcCCQIHBgMEBAQBAgUEBAIDAgUCBQIFAgUCBQYH +AgECAgIFAgECAQYFBAUCAQgHAgEA7AMBAPIDBQIBAIYEBQIEAgMCBQACPRAHDwYAAkoAugQDBgQC +BAIFAgQCBAYEAgMCAwICAgUEAwIDBAkCBAIBAgMCAwIBAgUCAQC+BQEAAoYBAMYFBQQJAgQEBAIC +BAMCAwICBAcCAgoFBgkCAwIEAgMGAwIDAgIICQIEAgcCCQIEAggCAQwDAgMCAggHAgICAgACkQEA +0AYJAgQCAwIJAgICBQIFBgQCCQICAgUCBQoFAgQCBQIEBAkCBAYEAgQCBQIEAgQCBAYDAgkCBAIF +AgUCAQCoBwUCBQCeCgQEAgIDAgICAQC0CgEAAh0AxgoFAgUCBAIFAgUCAwICAAK6AQCwCwUCBQQD +DAkCBAICAgQCAwIJAgMCAgIEAgMCAgYDAgUCCQIECgQCBAIFAgQCAwIFAgMCAwICCAUCBAIFAgkC +AwQEAgEaBAIEAgkCBQIDAgMCAgIFAgMCBAIBAAIPAIwPBQQJAgEApg8CBAIAAh8AtA8JAgQCAgIF +AgMCAgIFAgEAAiAAyg8HAgICAwICBAMEAgIEAgMCBQIBAPQPBwICDAUEBQACOgCwFgcCAgQFCgYC +CQIJAgkEBQgBBAUAAjsA6hYHAgIEBQoHAgkCCQIJBAUIAQQFAOAXBwIHAgMCBgIFAgEAAgcA3BgB +AgUEAQACEgCGGQkCBAQFAAIE8AFe7wEB8AGAAQAC4wEAoBkOBgUCBQgEDAcCBwQFAgcCBwQFDgMC +BQQCBAUCBQQDAgoeBAIFAgUCBQIFBAUCBQIFAgUCBQIFCAUGBQQEAgUCBQIFAgUCBQIFAgUCBQIF +AgUCBQIFANIaAgIFAgICAQDiGgICBQICAgEA8hoCAgUCAgIBAJIbAgIFAgICAQCiGwICBQICAgEA +shsCAgUCAgIBAAIHwAKBBb8CAQACiQUAiBwXDAUCBQIFAgUCBQIFAgUCBQIFAgUCBQIFAggGCAII +AggGBQYIAgQCBQIFAgMCAgYEAgUCBQgHAgECBTgFAhsCGwIdAh0CHQIdAh0CHQIaAhoCGgIaBAcC +BAIJAgcCAQICDgcCAQoIAggCCAIFAgUCBQIFAgUCBQIFAgUCBQIFAgUCBQQQAAIEMC4vAQACMwCu +Hg4EBQIEAgUCBQIHAgECCgDSHgMKBQDiHgMKBQACCwDyHgMCAwoFAIQfAwIDCgUAlh8DAgMKBQCo +HwMCAwoFALofAwoFAMofAwoFANofAwoFAOofAwoFAALxAgASBAIFAgUCBQIEBAQCBQIFAgUCBAQE +AgUCBQIFAgQEBAIFAgUCBQIEBAQCBQIFAgUCBAQEAgUCBQIFAgQEBAIFAgUCBQIEBAQCBQIFAgUC +BAQEAgUCBQIFAgQEBAIFAgUCBQIEBAQCBQIFAgUCBAQEAgUCBQIFAgQEBAIFAgUCBQIEBAQCBQIF +AgUCBAQEAgUCBQIFAgQEBAIFAgUCBQIEBAEAAoEHANgBAwIEAgMCBAQDAgQCAwIEBAMCBAIDAgQE +AwIEAgMCBAQDAgQCAwIEBAMCBAIDAgQEAwIEAgMCBAQDAgQCAwIEBAMCBAIDAgQEAwIEAgMCBAQD +AgQCAwIEBAMCBAIDAgQEAwIEAgMCBAQDAgQCAwIEBAMCBAIDAgQEAwIEAgMCBAQDAgQCAwIEBAMC +BAIDAgQEAwIEAgMCBAQDAgQCAwIEBAMCBAIDAgQEAwIEAgMCBAQDAgQCAwIEBAMCBAIDAgQEAwIE +AgMCBAQDAgQCAwIEBAMCBAIDAgQEAwIEAgMCBAQDAgQCAwIEBAMCBAIDAgQEAwIEAgMCBAQDAgQC +AwIEBAMCBAIDAgQEAwIEAgMCBAQDAgQCAwIEBAMCBAIDAgQEAwIEAgMCBAQDAgQCAwIEBAMCBAID +AgQEAwIEAgMCBAQDAgQCAwIEBAMCBAIDAgQEAwIEAgMCBAQDAgQCAwIEBAMCBAIDAgQEAwIEAgMC +BAQDAgQCAwIEBAMCBAIDAgQEAwIEAgMCBAQDAgQCAwIEBAMCBAIDAgQEAwIEAgMCBAQDAgQCAwIE +BAMCBAIDAgQEAwIEAgMCBAQDAgQCAwIEBAMCBAIDAgQEAwIEAgMCBAQDAgQCAwIEBAMCBAIDAgQE +AwIEAgMCBAQDAgQCAwIEBAMCBAIDAgQEAwIEAgMCBAQBAAKoBQAoAwoDCgMCBgIEAgYCBAIGAgQC +BgIGAgQCBggEAgYCBAIGAgcCBgIHAgYCBwIGCAUCBgIGAgYCBgIGAgYCBgIJAgkCCQIJAgkCCQIJ +AgkCBwIHAgcCBgIFBgQIBwICBAQCBQIFAgUCBwIHAgcCAgIGAgYCBgIGAgMCAQYEAgMCBAIEAgMC +AwQEAgUCBQIFAgcCBwIHAgIKAwIGAgYCBgIGAgMCAQYCAgQCAQQBBAMCBQIBBAICBAIBBgMCAQQD +AgUCAQQFAgcCAQQFAgYCBwIHAgEEBQIGAgYCBgIHAgcCBwIHAgEEBQIGAgYCBgIGAgYCBgIGAgcC +BwIHAgcCBwIHAgcCBwIBAAK/DQBUAwIDAgMkAwIGAgQCBgIEAgYCBgIEAgYCBgIEAgYCBAIGAgQC +BgIHAgYCBwIGBAcCBgoDAgIMBwIGBgcCAgYCAgICBQICBgMCAgIBCAMCBAIEAgMCBQwDAgMCAwIC +CgMCAwIBCgMCBAIEBAQCBAIDBAECBAIEAgMCAwIFBgICBAICAgQCAQQBBAICAgIBBAMCAwIDAgMC +AQQCAgQCAgIEAgEGAwIDAgEEAwIFAgMCBQIBBAQCBgIEAgYCAQQEAgUCBgIGAgQCBQIGAgYCAQQE +AgUCBQIFAgYCBgIGAgYCBAIFAgUCBQIGAgYCBgIGAgEEBAIFAgUCBQIFAgUCBQIFAgcCBwIHAgcC +BwIHAgcCBwIEAgUCBQIFAgUCBQIFAgUCBwIHAgcCBwIHAgcCBwIHBgUEAQQHAgQCBQIFAgUCBQIF +AgUCBQIJAgkCCQIJAgkCCQIJAgkCBAIFAgUCBQIFAgUCBQIFAgkCCQIJAgkCCQIJAgkCCQIHAgcC +BwIGBgUEBQwDAgMEAwQGBgcCBjgEAgMEBQIFAgcEBAIEBAUCBgQDAgMEBgIGBAMEBgIGBgQEAwID +BgQCBQIFAgUCAwIEAgUCBQIFAgMCAwICBAMCAwIFAgMCBQIFAgUCBgIGAgYCBgIGAgEMBAIGAgUC +BQIGAgYCBgIGAgYCBAIDAgQCBAIDAgMCAwIDAgQCBwQHAgcKBAIFAgUCBQIHAgQCBQIFAgUCBwIH +AgIGAwIFAgMCBQIFAgUCBgIGAgYCBgIGAgEGAwYEAgUCAwIFAgYCBAIDAgYCBgIEAgYCBgIDBAME +BQIDAgMEBwICAgcEBQIFAgUCBQIHAgUCBQIFAgUCBwIHAgIEBQIDAgQCBQIFAgYCBgIGAgYCBgIB +BgcEBwIHAgUCBQIFAgUCBwIFAgUCBQIFAgcCBwICAgMCBQIDAgQCBQIFAgYCBgIGAgYCBgIBAAIB +EAQQB+AFoAPfBQEPAQ8BAAKvAwASAQIDBAEEBwYEAgUCBQIFAgUCBQIFAgUCBQIFAgUCBQIFAgUM +BQIIAggCCAIIAggCCAIIAgkCCQIJAgkCCQIJAgkCCQIFAgkCCQIJAgkCCQIJAgkCCQIIAggCCAII +AggCCAIIAgUCBQIFAgUCBQIFAgUCBQIFAgUCBQIFAgUCBQIEAgcCAQIBAgEAAgUAEgUAcAQCBQIC +AgEAfgUEBgIFAgUCAgQCAgIAAiwAlgEFAgUCBAIFAgUCAgIGAgICBQIEAgEAsAEEAgUCAgIGAgIC +BQIEAgEAxAEFAgUCBAIFAgICBAIBAAIZANYBBAIFAgQCBQICAgQCAQDqAQUCBQICAgQCAQACFQD6 +AQUCBAIFAgICBAIBAAIEMEIvAQACRwCIAg4CBQIEAgUCAgIEAgUCAgIFBgMCBQIFAgICCgCsAgUC +AgIEAgEAuAIFAgICAwIFAgICAgIDAgQCBQICAgEA0gIFAgICAgIEAgUCAgIBAOQCBQICAgUCAQDw +AgUCBQIFAgUCAgIBAJADBQIFAgUCBQICAgQCAQACBDCqAS8BMAsAogMOEAMGBBQHAgcCBAIFBAUC +BAIHAgcGBwgCBAMCBAYEAgQEBQIEAgcCBAICAgIEBAIFAgMMBQIHAgQCBwYHAgMCBQIKBAcCAgIC +ALoEBAIFAgUCBQIFAgICBgICAgsCAQDSBAUCBQIFAgUCBQICAgQCAQACBDA5LwEAAj4A5gQOAgUC +BQIFAgcCAwIEAgICAwIEAgoAAhQQDQ8BAIAFBQIEAgUCBQIBAgMCBAICAgMCAQIBAAIEYCYwFy8h +XwEAAmMAoAUmCAQCBAIFAgUCBQIEBCECAQACuwEAxgUHAgMCAggHAgMCAgoJAgMCAgIEAgMCAgIG +AgICAgIHAgMCAgIEAgMCAgIHAgMCAgIGAgICAg4HAgcCBwICBgUKAwICBgUCBQIHAgECBAICDAcC +BwIHAgcCAgDkBgcCAgICAAJQAO4GBQIFAgQCBQIFAgUEBQICAgYCAgIDAgMCCQIFAgEEBQIJAgEA +AgQwTC8BAAJRAJwHDgIFAgUCBAIEAgUCBQIHAgMCBAIEAgICBAIFAgoAAicAvgcFAgUCBwICAgYC +AgILAgEAAgQwNS8BANQHDgIFAgUCBwIDAgQCBAICAgQCCgDsBwUCBQIEAgcCAgIEAgEAAigAgggF +AgQCBAIFAgUCBQIFAgICBAIBAAKdAQCcCAQCBQIHAgcCBwYFAgUCBQIEAgICBAICAgcKBAQHBAUC +AgYEAgICBAIBBgMGBAICAgQCAgYFAgICBAgEAgkCAwIFCAMGBQIFAgICAgCcCQUCBQIHAgICBgIC +AgsCAQACBFA9TwEAAkIAsAkOCgQEAwIHAgcCAgIGAgICCwIKANIJBQICAgEA3AkFAgUCBQIFAgIC +BAIBAPAJBAIFAgICBAIBAIAKBAIFAgICBAIBAJAKBAIEAgQCBQIFAgICBAIBAAIlAKgKBAIFAgQC +BQIHAgUCAgIEAgEAwAoEAgcCBwIFAgICAQACNQDSCgQCBwIHAgUCAgIEAgcCBwICAgUCAgIBAAIE +MLgBLwEwHQAC2gEAIA4CAwYEEAUKBwIHAgQCBQQEAgcCBwYHCAIEAwIEBgQCBAQFAgcCBAICAgIE +BQIEAgcCAgYEAgUEAwwFAgcCBAIHCAcCAwIFAgoGBwICBAUCBAIHAgIEAgACCjArLxcwEQBAXQC+ +Al0AAAQBAgIXAS0CEwAAJwQPAycAAgQgNB8BAAAOASsAABMCJgACBCAqHwEABC8AAA4BIQAAEwIc +AAIEMC8vAQAENAAADgEmAAAYBBwABBIAARIAABMEHAAEEwAEMwAADgElAAAXBBwAAgRAMz8BAAI4 +AAQ4AAAOASoAABwEHAACBEAyPwEAAA4BKQAAFwQgAAIKYERfAWAGXxFgEQACLi4XLTIABC6YAReX +ATIAAAQBAgIXAUcCEwAAPAQZAyIAAC4CFwEyAAIKQCs/HAAEUQAAJwQFAyUAAgpAcT8fAAKaAQAE +mgEAAEoCHwIJAygABBUAARUAAgpAVD8cAAJ6AAR6AAAtAiMCBQMlAAIKQGg/IgAAPgIiAgkDKwAC +CkBLPxwAAnEABHEAACMCJAIFAyUAAgpAPT8BQAY/EUARAARwAAAEAQICFwFAAhMAADkEFQMiAAQn +AAEnAAAZAg8BGwAAIwIpASUAAgQwgQEvATAtAAIoRhFFB0YURQJGIEUQRgVFKAAEKN4CEd0CB+QC +FOMCAuoCIOkCEOoCBekCKAAAFwGcAQAAhgEELQAAKAIRAQcCFAECAiABEAIFASgAAgQQUw8BEBcA +AiK2ARG1AQK2ARS1ASYABCL2EBH1EAL8EBT7ECYAABUBWgAAWAQXAAAiAhEBAgIUASYAAgpAPj8c +AAJkAARkAAA2BAkDJQAELq4BF60BMgACBBAtDwEQGgACTAAETAAAFQE3AAAyBBoABAEAEAEAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAABBAAAAAAAAoPQAAAAAAAGAQQAAAAAAAkD0AAAAAAACgFkAAAAAA +APg9AAAAAAAAABtAAAAAAABQPgAAAAAAACAbQAAAAAAAsD4AAAAAAABAG0AAAAAAABA/AAAAAAAA +4BtAAAAAAAB4PwAAAAAAAIAcQAAAAAAA4D8AAAAAAAAAHUAAAAAAAEBAAAAAAAAAQB1AAAAAAACA +QAAAAAAAAIAfQAAAAAAAsEAAAAAAAACgH0AAAAAAAOBAAAAAAAAA4CBAAAAAAAAQQQAAAAAAAAAh +QAAAAAAAQEEAAAAAAAAgIUAAAAAAAHBBAAAAAAAAQCJAAAAAAACgQQAAAAAAAGAiQAAAAAAAAEIA +AAAAAADAIkAAAAAAAGhCAAAAAAAAICNAAAAAAADQQgAAAAAAAEAjQAAAAAAAMEMAAAAAAABgI0AA +AAAAAJBDAAAAAAAAgCNAAAAAAADwQwAAAAAAAKAjQAAAAAAAUEQAAAAAAADAI0AAAAAAALBEAAAA +AAAA4CNAAAAAAAAQRQAAAAAAAAAkQAAAAAAAcEUAAAAAAAAgJEAAAAAAANBFAAAAAAAAYCRAAAAA +AAAwRgAAAAAAAKAkQAAAAAAAkEYAAAAAAAAAJUAAAAAAAPhGAAAAAAAAYCVAAAAAAABgRwAAAAAA +AMAlQAAAAAAAyEcAAAAAAACgJkAAAAAAADBIAAAAAAAAgCdAAAAAAACYSAAAAAAAAEAoQAAAAAAA +8EgAAAAAAACgKEAAAAAAAFhJAAAAAAAA4ChAAAAAAADASQAAAAAAAOApQAAAAAAAKEoAAAAAAABg +KkAAAAAAAHBKAAAAAAAAICtAAAAAAADYSgAAAAAAAIArQAAAAAAAIEsAAAAAAADgLEAAAAAAAIhL +AAAAAAAAYC1AAAAAAADQSwAAAAAAAGAuQAAAAAAAOEwAAAAAAABAL0AAAAAAAKBMAAAAAAAAQDBA +AAAAAAAITQAAAAAAAOAwQAAAAAAAUE0AAAAAAACAMUAAAAAAALhNAAAAAAAAQDJAAAAAAAAgTgAA +AAAAAGA1QAAAAAAAiE4AAAAAAACgNUAAAAAAANBOAAAAAAAAgDZAAAAAAAA4TwAAAAAAAMA4QAAA +AAAAoE8AAAAAAACgOkAAAAAAAAhQAAAAAAAA4DpAAAAAAABwUAAAAAAAACBBQAAAAAAA2FAAAAAA +AABgQUAAAAAAADBRAAAAAAAAoEJAAAAAAACYUQAAAAAAACBDQAAAAAAAAFIAAAAAAACgQ0AAAAAA +AGhSAAAAAAAAwEdAAAAAAADQUgAAAAAAAABIQAAAAAAAOFMAAAAAAADgTkAAAAAAAKBTAAAAAAAA +IE9AAAAAAAD4UwAAAAAAAABRQAAAAAAAYFQAAAAAAABgUUAAAAAAAMhUAAAAAAAAwFFAAAAAAAAI +VQAAAAAAAEBTQAAAAAAAcFUAAAAAAABgVEAAAAAAANhVAAAAAAAA4FZAAAAAAABAVgAAAAAAAMBX +QAAAAAAAqFYAAAAAAAAgW0AAAAAAAABXAAAAAAAAAFxAAAAAAABoVwAAAAAAAGBdQAAAAAAAwFcA +AAAAAADgXUAAAAAAABhYAAAAAAAAwF5AAAAAAACAWAAAAAAAACBfQAAAAAAAyFgAAAAAAACAYEAA +AAAAADBZAAAAAAAAQGRAAAAAAACYWQAAAAAAAKBkQAAAAAAAAFoAAAAAAAAgZUAAAAAAAGhaAAAA +AAAAQGVAAAAAAADIWgAAAAAAAIBrQAAAAAAAMFsAAAAAAADgcEAAAAAAAJhbAAAAAAAAgHhAAAAA +AAAAXAAAAAAAAAB8QAAAAAAAUFwAAAAAAAAAfkAAAAAAALhcAAAAAAAAYH5AAAAAAAAgXQAAAAAA +AMB+QAAAAAAAiF0AAAAAAADAgUAAAAAAAPBdAAAAAAAAIIJAAAAAAABYXgAAAAAAAGCDQAAAAAAA +wF4AAAAAAADAg0AAAAAAAChfAAAAAAAAgIdAAAAAAACQXwAAAAAAAGCIQAAAAAAA6F8AAAAAAADg +iEAAAAAAAFBgAAAAAAAAgIlAAAAAAAC4YAAAAAAAAACKQAAAAAAAIGEAAAAAAACgikAAAAAAAIhh +AAAAAAAAIItAAAAAAADwYQAAAAAAAGCLQAAAAAAAOGIAAAAAAADAjEAAAAAAAKBiAAAAAAAAgI1A +AAAAAAAIYwAAAAAAAMCNQAAAAAAAcGMAAAAAAABgj0AAAAAAANhjAAAAAAAAoI9AAAAAAABAZAAA +AAAAAGCQQAAAAAAAqGQAAAAAAAAAkUAAAAAAABBlAAAAAAAAAJJAAAAAAAB4ZQAAAAAAAOCTQAAA +AAAA4GUAAAAAAABglEAAAAAAAEhmAAAAAAAAAJVAAAAAAACwZgAAAAAAAICVQAAAAAAAGGcAAAAA +AADAlUAAAAAAAIBnAAAAAAAAAJZAAAAAAADoZwAAAAAAACCZQAAAAAAAMGgAAAAAAADAn0AAAAAA +AJhoAAAAAAAAAKFAAAAAAAAAaQAAAAAAACCjQAAAAAAAaGkAAAAAAABArEAAAAAAANBpAAAAAAAA +AK1AAAAAAAA4agAAAAAAAGCtQAAAAAAAoGoAAAAAAAAgrkAAAAAAAAhrAAAAAAAAAK9AAAAAAABw +awAAAAAAAGCwQAAAAAAA2GsAAAAAAAAgsUAAAAAAAEBsAAAAAAAAgLFAAAAAAACIbAAAAAAAAEC0 +QAAAAAAA8GwAAAAAAABAtUAAAAAAAFhtAAAAAAAAwLVAAAAAAADAbQAAAAAAACC4QAAAAAAAKG4A +AAAAAADguUAAAAAAAJBuAAAAAAAAQLxAAAAAAAD4bgAAAAAAAIC+QAAAAAAAYG8AAAAAAACgw0AA +AAAAAMhvAAAAAAAAgMVAAAAAAAAwcAAAAAAAACDGQAAAAAAAmHAAAAAAAADAy0AAAAAAAABxAAAA +AAAAwMxAAAAAAABocQAAAAAAAGDOQAAAAAAA0HEAAAAAAAAA0EAAAAAAADhyAAAAAAAAANNAAAAA +AACgcgAAAAAAAKDTQAAAAAAACHMAAAAAAACA10AAAAAAAHBzAAAAAAAAINhAAAAAAADYcwAAAAAA +AADZQAAAAAAAQHQAAAAAAABg2UAAAAAAAKh0AAAAAAAAoNlAAAAAAAAQdQAAAAAAAMDZQAAAAAAA +eHUAAAAAAAAg20AAAAAAAOB1AAAAAAAAYN1AAAAAAABIdgAAAAAAAKDeQAAAAAAAsHYAAAAAAABA +30AAAAAAABh3AAAAAAAAAOBAAAAAAACAdwAAAAAAAKDgQAAAAAAA6HcAAAAAAADg5EAAAAAAAFB4 +AAAAAAAAwOZAAAAAAAC4eAAAAAAAAGDoQAAAAAAAIHkAAAAAAACg6kAAAAAAAIh5AAAAAAAAAOxA +AAAAAADweQAAAAAAAAD4QAAAAAAAWHoAAAAAAACg+0AAAAAAAMB6AAAAAAAAoPxAAAAAAAAoewAA +AAAAACADQQAAAAAAkHsAAAAAAADAA0EAAAAAAPh7AAAAAAAAgARBAAAAAABQfAAAAAAAAAAFQQAA +AAAAqHwAAAAAAACABUEAAAAAABB9AAAAAAAAIAZBAAAAAABofQAAAAAAAMAIQQAAAAAA0H0AAAAA +AABAC0EAAAAAADh+AAAAAAAAIA1BAAAAAACgfgAAAAAAAAAOQQAAAAAACH8AAAAAAAAAFUEAAAAA +AHB/AAAAAAAAABZBAAAAAADYfwAAAAAAAGAXQQAAAAAAQIAAAAAAAACAGEEAAAAAAIiAAAAAAAAA +ABlBAAAAAADggAAAAAAAAAAbQQAAAAAASIEAAAAAAAAAHEEAAAAAALCBAAAAAAAAQB5BAAAAAAAY +ggAAAAAAAOAeQQAAAAAAgIIAAAAAAAAgH0EAAAAAAOiCAAAAAAAA4B9BAAAAAABQgwAAAAAAAGAi +QQAAAAAAuIMAAAAAAADgIkEAAAAAABCEAAAAAAAAICRBAAAAAAB4hAAAAAAAAIAkQQAAAAAAwIQA +AAAAAABgJUEAAAAAAAiFAAAAAAAAICZBAAAAAABghQAAAAAAAMAsQQAAAAAAyIUAAAAAAAAALkEA +AAAAACCGAAAAAAAAwDBBAAAAAAB4hgAAAAAAAIAxQQAAAAAA0IYAAAAAAABgPkEAAAAAADiHAAAA +AAAAoD5BAAAAAACAhwAAAAAAACA/QQAAAAAA2IcAAAAAAACAQ0EAAAAAADCIAAAAAAAAIEVBAAAA +AACIiAAAAAAAACBKQQAAAAAA8IgAAAAAAADAS0EAAAAAAFiJAAAAAAAAwExBAAAAAACwiQAAAAAA +AABOQQAAAAAACIoAAAAAAAAgUEEAAAAAAHCKAAAAAAAAAFJBAAAAAADIigAAAAAAAOBSQQAAAAAA +GIsAAAAAAAAAVEEAAAAAAICLAAAAAAAAAFdBAAAAAADoiwAAAAAAAEBYQQAAAAAAQIwAAAAAAAAA +WUEAAAAAAKiMAAAAAAAAIFpBAAAAAAAAjQAAAAAAAIBdQQAAAAAAaI0AAAAAAAAgYEEAAAAAANCN +AAAAAAAAYGBBAAAAAAAYjgAAAAAAAIBjQQAAAAAAgI4AAAAAAAAAZEEAAAAAANiOAAAAAAAAQGVB +AAAAAAAwjwAAAAAAAOBmQQAAAAAAmI8AAAAAAACgbUEAAAAAAACQAAAAAAAAAG5BAAAAAABokAAA +AAAAAGBwQQAAAAAA0JAAAAAAAABgdEEAAAAAADiRAAAAAAAAYHZBAAAAAACgkQAAAAAAAAB4QQAA +AAAACJIAAAAAAABge0EAAAAAAHCSAAAAAAAAoH1BAAAAAADYkgAAAAAAACB+QQAAAAAAQJMAAAAA +AAAggUEAAAAAAKiTAAAAAAAAQIVBAAAAAAAQlAAAAAAAAECGQQAAAAAAeJQAAAAAAAAAh0EAAAAA +AMCUAAAAAAAAIIdBAAAAAAAAlQAAAAAAAICHQQAAAAAAaJUAAAAAAAAgikEAAAAAANCVAAAAAAAA +IItBAAAAAAA4lgAAAAAAAICPQQAAAAAAoJYAAAAAAADgkEEAAAAAAAiXAAAAAAAAYJNBAAAAAABw +lwAAAAAAAACXQQAAAAAA2JcAAAAAAACAl0EAAAAAAECYAAAAAAAAAJhBAAAAAACYmAAAAAAAAECZ +QQAAAAAA8JgAAAAAAADgmUEAAAAAAEiZAAAAAAAAwJpBAAAAAACwmQAAAAAAAKCdQQAAAAAACJoA +AAAAAADAnkEAAAAAAGCaAAAAAAAAIKBBAAAAAADImgAAAAAAAKChQQAAAAAAMJsAAAAAAADAokEA +AAAAAJibAAAAAAAAoKNBAAAAAAAAnAAAAAAAAICkQQAAAAAAaJwAAAAAAADAqEEAAAAAANCcAAAA +AAAAQKpBAAAAAAA4nQAAAAAAAMCrQQAAAAAAoJ0AAAAAAACArUEAAAAAAAieAAAAAAAAwK5BAAAA +AABwngAAAAAAAACyQQAAAAAA2J4AAAAAAABAs0EAAAAAAECfAAAAAAAAgLRBAAAAAAConwAAAAAA +ACC2QQAAAAAAEKAAAAAAAABAt0EAAAAAAHigAAAAAAAAALlBAAAAAADgoAAAAAAAAAC6QQAAAAAA +OKEAAAAAAABAu0EAAAAAAJChAAAAAAAAoL5BAAAAAADooQAAAAAAACDCQQAAAAAAUKIAAAAAAAAA +zkEAAAAAALiiAAAAAAAAANFBAAAAAAAgowAAAAAAAGDSQQAAAAAAiKMAAAAAAADA0kEAAAAAAPCj +AAAAAAAA4NNBAAAAAABYpAAAAAAAAODVQQAAAAAAwKQAAAAAAADA1kEAAAAAACilAAAAAAAAwNdB +AAAAAACQpQAAAAAAAIDYQQAAAAAA+KUAAAAAAADg2EEAAAAAAGCmAAAAAAAAQNlBAAAAAADIpgAA +AAAAAEDbQQAAAAAAIKcAAAAAAACg20EAAAAAAGinAAAAAAAAANxBAAAAAADQpwAAAAAAAGDcQQAA +AAAAOKgAAAAAAAAA3UEAAAAAAJCoAAAAAAAAwN1BAAAAAAD4qAAAAAAAAMDeQQAAAAAAUKkAAAAA +AACg30EAAAAAALipAAAAAAAAYOBBAAAAAAAAqgAAAAAAACDiQQAAAAAAaKoAAAAAAACg4kEAAAAA +ANCqAAAAAAAAIONBAAAAAAA4qwAAAAAAAODmQQAAAAAAoKsAAAAAAAAg6UEAAAAAAAisAAAAAAAA +YO5BAAAAAABwrAAAAAAAAIDvQQAAAAAA2KwAAAAAAAAg8EEAAAAAADCtAAAAAAAAgPBBAAAAAACY +rQAAAAAAAEDxQQAAAAAAAK4AAAAAAACg8kEAAAAAAGiuAAAAAAAAoPNBAAAAAADQrgAAAAAAAID5 +QQAAAAAAOK8AAAAAAACA/EEAAAAAAKCvAAAAAAAAAP1BAAAAAAAIsAAAAAAAAGD9QQAAAAAAYLAA +AAAAAADg/UEAAAAAAMiwAAAAAAAAYAFCAAAAAAAwsQAAAAAAAOACQgAAAAAAmLEAAAAAAAAABEIA +AAAAAACyAAAAAAAAIAZCAAAAAABosgAAAAAAAOAGQgAAAAAA0LIAAAAAAAAgCEIAAAAAADizAAAA +AAAAQAtCAAAAAACgswAAAAAAAIALQgAAAAAACLQAAAAAAABADEIAAAAAAGC0AAAAAAAAAA1CAAAA +AAC4tAAAAAAAAIAOQgAAAAAAILUAAAAAAADgEEIAAAAAAIi1AAAAAAAAwBZCAAAAAADwtQAAAAAA +AEAaQgAAAAAAWLYAAAAAAAAAG0IAAAAAAMC2AAAAAAAAACdCAAAAAAAotwAAAAAAAKAoQgAAAAAA +kLcAAAAAAABgK0IAAAAAAPi3AAAAAAAAQC5CAAAAAABguAAAAAAAAKAvQgAAAAAAyLgAAAAAAAAg +MUIAAAAAADC5AAAAAAAAIDVCAAAAAACYuQAAAAAAAKA1QgAAAAAAALoAAAAAAAAgNkIAAAAAAGi6 +AAAAAAAA4DZCAAAAAADQugAAAAAAACA4QgAAAAAAOLsAAAAAAABAOkIAAAAAAKC7AAAAAAAAID1C +AAAAAAAIvAAAAAAAAEA+QgAAAAAAcLwAAAAAAABgPkIAAAAAANC8AAAAAAAAgD9CAAAAAAA4vQAA +AAAAAKA/QgAAAAAAmL0AAAAAAADgQUIAAAAAAAC+AAAAAAAAwENCAAAAAABovgAAAAAAAIBEQgAA +AAAA0L4AAAAAAACARUIAAAAAADi/AAAAAAAAwEZCAAAAAACgvwAAAAAAAEBHQgAAAAAACMAAAAAA +AACgR0IAAAAAAHDAAAAAAAAAYEhCAAAAAADYwAAAAAAAAMBIQgAAAAAAQMEAAAAAAAAgSUIAAAAA +AKjBAAAAAAAAgExCAAAAAAAQwgAAAAAAAMBMQgAAAAAAcMIAAAAAAABATUIAAAAAAMjCAAAAAAAA +oE1CAAAAAAAgwwAAAAAAAIBOQgAAAAAAeMMAAAAAAABAUEIAAAAAAODDAAAAAAAAgFBCAAAAAAAo +xAAAAAAAAEBRQgAAAAAAkMQAAAAAAABAUkIAAAAAAPjEAAAAAAAAQFRCAAAAAABgxQAAAAAAAIBW +QgAAAAAAyMUAAAAAAADAVkIAAAAAACDGAAAAAAAAIFhCAAAAAACIxgAAAAAAAGBYQgAAAAAA4MYA +AAAAAAAgWUIAAAAAADjHAAAAAAAAoFlCAAAAAACgxwAAAAAAAIBaQgAAAAAACMgAAAAAAAAgW0IA +AAAAAHDIAAAAAAAA4FtCAAAAAADYyAAAAAAAAOBcQgAAAAAAQMkAAAAAAAAAXkIAAAAAAKjJAAAA +AAAAYGNCAAAAAAAQygAAAAAAAOBjQgAAAAAAeMoAAAAAAAAgZkIAAAAAAODKAAAAAAAAAGdCAAAA +AABIywAAAAAAAKBoQgAAAAAAsMsAAAAAAADgaUIAAAAAABjMAAAAAAAAIGtCAAAAAACAzAAAAAAA +AMBrQgAAAAAA6MwAAAAAAACAbEIAAAAAAFDNAAAAAAAAoGxCAAAAAACQzQAAAAAAAGBtQgAAAAAA ++M0AAAAAAABgbkIAAAAAAGDOAAAAAAAAIG9CAAAAAADIzgAAAAAAAKBvQgAAAAAAMM8AAAAAAABg +cEIAAAAAAJjPAAAAAAAAIHNCAAAAAAAA0AAAAAAAAEB0QgAAAAAASNAAAAAAAADAdEIAAAAAAKDQ +AAAAAAAAwHVCAAAAAAAI0QAAAAAAAMB3QgAAAAAAUNEAAAAAAADAeEIAAAAAAKjRAAAAAAAAAHxC +AAAAAAAQ0gAAAAAAAOB8QgAAAAAAeNIAAAAAAACgfUIAAAAAAODSAAAAAAAAQH5CAAAAAAAo0wAA +AAAAAEB/QgAAAAAAcNMAAAAAAAAAgUIAAAAAANjTAAAAAAAAYINCAAAAAABA1AAAAAAAAICEQgAA +AAAAqNQAAAAAAADAhUIAAAAAAPDUAAAAAAAAQIZCAAAAAABI1QAAAAAAAMCHQgAAAAAAsNUAAAAA +AABAiEIAAAAAABjWAAAAAAAAoIhCAAAAAABg1gAAAAAAAGCJQgAAAAAAyNYAAAAAAADgiUIAAAAA +ADDXAAAAAAAAYIpCAAAAAACY1wAAAAAAAACLQgAAAAAAANgAAAAAAADAi0IAAAAAAGjYAAAAAAAA +QIxCAAAAAADQ2AAAAAAAAOCMQgAAAAAAONkAAAAAAACAjUIAAAAAAKDZAAAAAAAAII5CAAAAAAAI +2gAAAAAAAMCOQgAAAAAAcNoAAAAAAABgj0IAAAAAANjaAAAAAAAAAJBCAAAAAABA2wAAAAAAAKCQ +QgAAAAAAqNsAAAAAAABAkUIAAAAAABDcAAAAAAAA4JFCAAAAAAB43AAAAAAAAICSQgAAAAAA4NwA +AAAAAADgkkIAAAAAACjdAAAAAAAAQJNCAAAAAABw3QAAAAAAAKCVQgAAAAAAyN0AAAAAAAAglkIA +AAAAABDeAAAAAAAA4JdCAAAAAAB43gAAAAAAAECZQgAAAAAA0N4AAAAAAADgmkIAAAAAACjfAAAA +AAAA4JxCAAAAAACQ3wAAAAAAAGCeQgAAAAAA6N8AAAAAAACgnkIAAAAAADDgAAAAAAAA4J5CAAAA +AAB44AAAAAAAAGCgQgAAAAAA0OAAAAAAAADAoUIAAAAAADjhAAAAAAAAwKJCAAAAAACg4QAAAAAA +AICjQgAAAAAACOIAAAAAAAAApEIAAAAAAFjiAAAAAAAAAKdCAAAAAADA4gAAAAAAAGCvQgAAAAAA +KOMAAAAAAABAsEIAAAAAAJDjAAAAAAAAgLdCAAAAAAD44wAAAAAAAKC3QgAAAAAAOOQAAAAAAADg +t0IAAAAAAJjkAAAAAAAAgLhCAAAAAAAA5QAAAAAAAAC5QgAAAAAASOUAAAAAAABAukIAAAAAALDl +AAAAAAAAwLpCAAAAAAAA5gAAAAAAAEC7QgAAAAAASOYAAAAAAAAAvEIAAAAAALDmAAAAAAAAwLxC +AAAAAAD45gAAAAAAAGC+QgAAAAAAQOcAAAAAAABgwUIAAAAAAKjnAAAAAAAAIMJCAAAAAAAQ6AAA +AAAAAGDCQgAAAAAAeOgAAAAAAADgx0IAAAAAAODoAAAAAAAAwMlCAAAAAABI6QAAAAAAACDKQgAA +AAAAkOkAAAAAAAAAy0IAAAAAAOjpAAAAAAAAwM5CAAAAAABQ6gAAAAAAAEDQQgAAAAAAuOoAAAAA +AADA0EIAAAAAABDrAAAAAAAAINFCAAAAAABo6wAAAAAAAIDSQgAAAAAA0OsAAAAAAADA0kIAAAAA +ABjsAAAAAAAAANNCAAAAAABg7AAAAAAAAGDTQgAAAAAAyOwAAAAAAADA1UIAAAAAADDtAAAAAAAA +YNZCAAAAAACY7QAAAAAAAGDXQgAAAAAAAO4AAAAAAADA10IAAAAAAGjuAAAAAAAA4NhCAAAAAADQ +7gAAAAAAACDZQgAAAAAAOO8AAAAAAABg2UIAAAAAAKDvAAAAAAAA4NlCAAAAAAAI8AAAAAAAAMDa +QgAAAAAAcPAAAAAAAAAA3UIAAAAAANjwAAAAAAAAYOBCAAAAAAA48QAAAAAAAKDgQgAAAAAAgPEA +AAAAAADg4EIAAAAAAMjxAAAAAAAA4OFCAAAAAAAg8gAAAAAAACDiQgAAAAAAaPIAAAAAAABg40IA +AAAAANDyAAAAAAAA4ONCAAAAAAA48wAAAAAAACDkQgAAAAAAgPMAAAAAAAAg50IAAAAAANjzAAAA +AAAAYOpCAAAAAABA9AAAAAAAAKDqQgAAAAAAqPQAAAAAAADg6kIAAAAAABD1AAAAAAAAQOtCAAAA +AABo9QAAAAAAAKDrQgAAAAAAwPUAAAAAAADA60IAAAAAAAj2AAAAAAAAIO1CAAAAAABw9gAAAAAA +AODtQgAAAAAA2PYAAAAAAACA7kIAAAAAAED3AAAAAAAAAPBCAAAAAACY9wAAAAAAAMDyQgAAAAAA +8PcAAAAAAABg80IAAAAAAEj4AAAAAAAAwPNCAAAAAACQ+AAAAAAAAMD1QgAAAAAA+PgAAAAAAADg +90IAAAAAAGD5AAAAAAAAwPhCAAAAAACo+QAAAAAAAID8QgAAAAAAEPoAAAAAAABg/UIAAAAAAHj6 +AAAAAAAAwABDAAAAAADg+gAAAAAAAGABQwAAAAAAKPsAAAAAAAAAAkMAAAAAAJD7AAAAAAAAgAJD +AAAAAAD4+wAAAAAAAAAFQwAAAAAAUPwAAAAAAADgBkMAAAAAALj8AAAAAAAAgAdDAAAAAAAA/QAA +AAAAAIAIQwAAAAAASP0AAAAAAADgCEMAAAAAAJD9AAAAAAAAQAlDAAAAAADY/QAAAAAAAIALQwAA +AAAAQP4AAAAAAADADkMAAAAAAKj+AAAAAAAAgA9DAAAAAAAA/wAAAAAAAEASQwAAAAAAaP8AAAAA +AACAEkMAAAAAALD/AAAAAAAAIBRDAAAAAAAIAAEAAAAAAKAUQwAAAAAAYAABAAAAAAAAFkMAAAAA +ALgAAQAAAAAAIBdDAAAAAAAQAQEAAAAAAAAYQwAAAAAAeAEBAAAAAABAGUMAAAAAAOABAQAAAAAA +YBpDAAAAAABIAgEAAAAAAEAbQwAAAAAAoAIBAAAAAADAHEMAAAAAAPgCAQAAAAAA4BxDAAAAAABA +AwEAAAAAAOAdQwAAAAAAmAMBAAAAAADgHkMAAAAAAPADAQAAAAAAAB9DAAAAAAAwBAEAAAAAAMAh +QwAAAAAAmAQBAAAAAACgJEMAAAAAAAAFAQAAAAAAICVDAAAAAABIBQEAAAAAAIAnQwAAAAAAoAUB +AAAAAABAKEMAAAAAAAgGAQAAAAAAQClDAAAAAABgBgEAAAAAAMAqQwAAAAAAyAYBAAAAAABgNEMA +AAAAACAHAQAAAAAAQDVDAAAAAAB4BwEAAAAAAMA4QwAAAAAA4AcBAAAAAAAAOkMAAAAAAEgIAQAA +AAAAoDpDAAAAAACwCAEAAAAAACA8QwAAAAAACAkBAAAAAACAPEMAAAAAAHAJAQAAAAAAAD1DAAAA +AAC4CQEAAAAAACBAQwAAAAAAIAoBAAAAAAAgREMAAAAAAHgKAQAAAAAAAEZDAAAAAADgCgEAAAAA +AGBGQwAAAAAASAsBAAAAAADgR0MAAAAAALALAQAAAAAAAEpDAAAAAAAYDAEAAAAAAGBKQwAAAAAA +gAwBAAAAAABAS0MAAAAAAOgMAQAAAAAAwEtDAAAAAABQDQEAAAAAAGBOQwAAAAAAuA0BAAAAAABA +T0MAAAAAACAOAQAAAAAAoE9DAAAAAAB4DgEAAAAAACBSQwAAAAAA4A4BAAAAAACgUkMAAAAAAEgP +AQAAAAAAwFRDAAAAAACwDwEAAAAAAKBVQwAAAAAA+A8BAAAAAAAgVkMAAAAAAFAQAQAAAAAAAFdD +AAAAAACoEAEAAAAAAKBYQwAAAAAA+BABAAAAAACgWUMAAAAAAEARAQAAAAAAoFpDAAAAAACIEQEA +AAAAACBbQwAAAAAA4BEBAAAAAAAgXEMAAAAAAEgSAQAAAAAA4FxDAAAAAACQEgEAAAAAAIBdQwAA +AAAA4BIBAAAAAADgXUMAAAAAACgTAQAAAAAAgF5DAAAAAACAEwEAAAAAAABgQwAAAAAA6BMBAAAA +AAAAYUMAAAAAAFAUAQAAAAAAYGFDAAAAAACYFAEAAAAAAABiQwAAAAAAABUBAAAAAACAYkMAAAAA +AEgVAQAAAAAAYGZDAAAAAACwFQEAAAAAACBpQwAAAAAAGBYBAAAAAAAga0MAAAAAAIAWAQAAAAAA +oGxDAAAAAADoFgEAAAAAAABtQwAAAAAAMBcBAAAAAABgbkMAAAAAAJgXAQAAAAAAAG9DAAAAAADw +FwEAAAAAAEBvQwAAAAAAOBgBAAAAAACAb0MAAAAAAIAYAQAAAAAAwG9DAAAAAADIGAEAAAAAAABw +QwAAAAAAEBkBAAAAAABAcEMAAAAAAFgZAQAAAAAAgHBDAAAAAACgGQEAAAAAAMBwQwAAAAAA6BkB +AAAAAACAdEMAAAAAAFAaAQAAAAAAAHVDAAAAAACYGgEAAAAAAGB1QwAAAAAAABsBAAAAAABAd0MA +AAAAAGgbAQAAAAAAoHpDAAAAAADQGwEAAAAAAIB7QwAAAAAAKBwBAAAAAADAg0MAAAAAAJAcAQAA +AAAAIIRDAAAAAAD4HAEAAAAAAECFQwAAAAAAYB0BAAAAAACghkMAAAAAALgdAQAAAAAAAIdDAAAA +AAAgHgEAAAAAAOCJQwAAAAAAeB4BAAAAAADgikMAAAAAAOAeAQAAAAAAYI9DAAAAAAA4HwEAAAAA +AKCRQwAAAAAAoB8BAAAAAABAkkMAAAAAAOgfAQAAAAAAAJNDAAAAAABQIAEAAAAAAACbQwAAAAAA +uCABAAAAAABAnEMAAAAAACAhAQAAAAAAoJxDAAAAAACIIQEAAAAAAACdQwAAAAAA8CEBAAAAAABA +nkMAAAAAAFgiAQAAAAAAQJ9DAAAAAADAIgEAAAAAAGCgQwAAAAAAKCMBAAAAAACAoUMAAAAAAIAj +AQAAAAAAgKJDAAAAAADoIwEAAAAAAICkQwAAAAAAUCQBAAAAAADApUMAAAAAALgkAQAAAAAAQKZD +AAAAAAAgJQEAAAAAAKCnQwAAAAAAiCUBAAAAAABAqUMAAAAAAPAlAQAAAAAAQKpDAAAAAABYJgEA +AAAAAICvQwAAAAAAwCYBAAAAAAAgsEMAAAAAACgnAQAAAAAAALFDAAAAAACQJwEAAAAAAKC2QwAA +AAAA+CcBAAAAAAAgt0MAAAAAAGAoAQAAAAAAoLdDAAAAAADIKAEAAAAAAMC4QwAAAAAAICkBAAAA +AAAAukMAAAAAAHgpAQAAAAAAALxDAAAAAADAKQEAAAAAAADAQwAAAAAAGCoBAAAAAAAgw0MAAAAA +AHAqAQAAAAAAQMRDAAAAAADYKgEAAAAAAIDEQwAAAAAAOCsBAAAAAAAgxUMAAAAAAKArAQAAAAAA +wMVDAAAAAAD4KwEAAAAAAKDGQwAAAAAAYCwBAAAAAAAgx0MAAAAAAMgsAQAAAAAAYMpDAAAAAAAw +LQEAAAAAAIDMQwAAAAAAmC0BAAAAAAAg0EMAAAAAAAAuAQAAAAAAYNNDAAAAAABoLgEAAAAAAKDU +QwAAAAAA0C4BAAAAAADg1UMAAAAAADgvAQAAAAAA4NtDAAAAAACgLwEAAAAAAODcQwAAAAAACDAB +AAAAAACg3kMAAAAAAHAwAQAAAAAAwN9DAAAAAADYMAEAAAAAAODfQwAAAAAAODEBAAAAAADg4UMA +AAAAAKAxAQAAAAAAoORDAAAAAAAIMgEAAAAAAODsQwAAAAAAcDIBAAAAAAAA8UMAAAAAAMgyAQAA +AAAAIPJDAAAAAAAwMwEAAAAAAEDzQwAAAAAAmDMBAAAAAACA80MAAAAAAOAzAQAAAAAAAPRDAAAA +AABINAEAAAAAAID0QwAAAAAAsDQBAAAAAAAA9UMAAAAAABg1AQAAAAAAwPVDAAAAAACANQEAAAAA +AKD3QwAAAAAA6DUBAAAAAABA+EMAAAAAAFA2AQAAAAAA4PhDAAAAAAC4NgEAAAAAACD5QwAAAAAA +ADcBAAAAAACA+kMAAAAAAFg3AQAAAAAAoPtDAAAAAACwNwEAAAAAAED8QwAAAAAACDgBAAAAAADA +/EMAAAAAAHA4AQAAAAAAQP5DAAAAAADYOAEAAAAAAKD/QwAAAAAAQDkBAAAAAACAAEQAAAAAAKg5 +AQAAAAAAYAdEAAAAAAAQOgEAAAAAAMAHRAAAAAAAaDoBAAAAAABgCUQAAAAAANA6AQAAAAAAAAtE +AAAAAAA4OwEAAAAAACAMRAAAAAAAoDsBAAAAAABgDUQAAAAAAAg8AQAAAAAAYA5EAAAAAABwPAEA +AAAAAEARRAAAAAAA2DwBAAAAAABgFEQAAAAAAEA9AQAAAAAAgBZEAAAAAACoPQEAAAAAAMAZRAAA +AAAAED4BAAAAAAAAG0QAAAAAAHg+AQAAAAAAgBxEAAAAAADgPgEAAAAAAAAgRAAAAAAASD8BAAAA +AADAK0QAAAAAAKA/AQAAAAAAYC1EAAAAAAAIQAEAAAAAAAAvRAAAAAAAYEABAAAAAADgNkQAAAAA +AMhAAQAAAAAAQDdEAAAAAAAQQQEAAAAAAOA5RAAAAAAAeEEBAAAAAACAOkQAAAAAAOBBAQAAAAAA +oDtEAAAAAABIQgEAAAAAAKA8RAAAAAAAsEIBAAAAAABAPUQAAAAAABhDAQAAAAAA4D1EAAAAAACA +QwEAAAAAAMA+RAAAAAAA4EMBAAAAAACAP0QAAAAAAEhEAQAAAAAA4D9EAAAAAACgRAEAAAAAACBA +RAAAAAAA4EQBAAAAAABgQkQAAAAAADhFAQAAAAAAIElEAAAAAACgRQEAAAAAAOBKRAAAAAAACEYB +AAAAAAAAUEQAAAAAAHBGAQAAAAAAwFBEAAAAAADYRgEAAAAAAGBRRAAAAAAAQEcBAAAAAAAgUkQA +AAAAAKhHAQAAAAAAIFNEAAAAAAAQSAEAAAAAAEBURAAAAAAAeEgBAAAAAACgVEQAAAAAAOBIAQAA +AAAAAFZEAAAAAABISQEAAAAAAABXRAAAAAAAsEkBAAAAAACgV0QAAAAAABhKAQAAAAAAQFhEAAAA +AACASgEAAAAAAOBZRAAAAAAA6EoBAAAAAACAW0QAAAAAAFBLAQAAAAAAIF5EAAAAAAC4SwEAAAAA +ACBgRAAAAAAAIEwBAAAAAADAYUQAAAAAAIhMAQAAAAAAwGZEAAAAAADwTAEAAAAAACBpRAAAAAAA +WE0BAAAAAABAbEQAAAAAAMBNAQAAAAAAIG1EAAAAAAAoTgEAAAAAAOBvRAAAAAAAkE4BAAAAAACA +cUQAAAAAAPhOAQAAAAAAIHZEAAAAAABgTwEAAAAAAAB3RAAAAAAAuE8BAAAAAABgeEQAAAAAACBQ +AQAAAAAAgHpEAAAAAACIUAEAAAAAAMB6RAAAAAAA0FABAAAAAACAe0QAAAAAAChRAQAAAAAAQHxE +AAAAAACQUQEAAAAAAEB9RAAAAAAA+FEBAAAAAADAgUQAAAAAAGBSAQAAAAAA4IJEAAAAAADIUgEA +AAAAAGCDRAAAAAAAIFMBAAAAAADgg0QAAAAAAIhTAQAAAAAAYIZEAAAAAADwUwEAAAAAAECIRAAA +AAAAWFQBAAAAAADgiEQAAAAAAMBUAQAAAAAAQIlEAAAAAAAoVQEAAAAAAGCKRAAAAAAAkFUBAAAA +AADAikQAAAAAANhVAQAAAAAAgItEAAAAAABAVgEAAAAAAOCLRAAAAAAAiFYBAAAAAACAjEQAAAAA +APBWAQAAAAAAQI1EAAAAAABIVwEAAAAAACCORAAAAAAAsFcBAAAAAACAj0QAAAAAAAhYAQAAAAAA +4I9EAAAAAABQWAEAAAAAAGCQRAAAAAAAuFgBAAAAAABAkUQAAAAAACBZAQAAAAAAgJFEAAAAAABo +WQEAAAAAAGCSRAAAAAAA0FkBAAAAAAAgk0QAAAAAADhaAQAAAAAAwJNEAAAAAACAWgEAAAAAAOCV +RAAAAAAA6FoBAAAAAABAsEQAAAAAAFBbAQAAAAAAYLNEAAAAAAC4WwEAAAAAAOCzRAAAAAAAIFwB +AAAAAACgtUQAAAAAAIhcAQAAAAAAQLhEAAAAAADwXAEAAAAAAOC4RAAAAAAAWF0BAAAAAACAukQA +AAAAAMBdAQAAAAAA4LpEAAAAAAAoXgEAAAAAAIC7RAAAAAAAkF4BAAAAAACgvkQAAAAAAPheAQAA +AAAAYMBEAAAAAABgXwEAAAAAAEDDRAAAAAAAyF8BAAAAAABAxEQAAAAAADBgAQAAAAAAwMREAAAA +AAB4YAEAAAAAAGDFRAAAAAAA4GABAAAAAAAgxkQAAAAAAEhhAQAAAAAAAMhEAAAAAACwYQEAAAAA +AEDKRAAAAAAAGGIBAAAAAABAy0QAAAAAAIBiAQAAAAAAYMxEAAAAAADoYgEAAAAAAEDORAAAAAAA +UGMBAAAAAACAzkQAAAAAALBjAQAAAAAAYM9EAAAAAAAYZAEAAAAAAKDQRAAAAAAAgGQBAAAAAACg +0kQAAAAAAOhkAQAAAAAAINNEAAAAAABQZQEAAAAAAADURAAAAAAAuGUBAAAAAACg1EQAAAAAACBm +AQAAAAAAINVEAAAAAACAZgEAAAAAAODVRAAAAAAA6GYBAAAAAACA2EQAAAAAAFBnAQAAAAAAQNtE +AAAAAAC4ZwEAAAAAAEDeRAAAAAAAIGgBAAAAAAAA30QAAAAAAIhoAQAAAAAAAOBEAAAAAADwaAEA +AAAAAGDhRAAAAAAAWGkBAAAAAABA50QAAAAAALBpAQAAAAAAAPREAAAAAAAYagEAAAAAAKD4RAAA +AAAAgGoBAAAAAADg+UQAAAAAAOhqAQAAAAAAgP1EAAAAAABQawEAAAAAAED/RAAAAAAAuGsBAAAA +AAAAAEUAAAAAACBsAQAAAAAAYABFAAAAAACAbAEAAAAAACABRQAAAAAA6GwBAAAAAACgAkUAAAAA +AFBtAQAAAAAA4AJFAAAAAACYbQEAAAAAAGADRQAAAAAAAG4BAAAAAADgA0UAAAAAAEhuAQAAAAAA +AAVFAAAAAACgbgEAAAAAAOAFRQAAAAAA+G4BAAAAAAAgBkUAAAAAAEBvAQAAAAAAYAZFAAAAAACo +bwEAAAAAAKAGRQAAAAAA8G8BAAAAAAAgB0UAAAAAAFhwAQAAAAAAQAdFAAAAAAC4cAEAAAAAAIAH +RQAAAAAAIHEBAAAAAADgB0UAAAAAAHhxAQAAAAAAgAhFAAAAAADgcQEAAAAAAMAIRQAAAAAAKHIB +AAAAAAAACUUAAAAAAHByAQAAAAAAYAlFAAAAAAC4cgEAAAAAAMAJRQAAAAAAAHMBAAAAAAAACkUA +AAAAAEhzAQAAAAAAgAtFAAAAAACwcwEAAAAAAIANRQAAAAAACHQBAAAAAABgDkUAAAAAAHB0AQAA +AAAAoA5FAAAAAAC4dAEAAAAAAMAQRQAAAAAAEHUBAAAAAADgEUUAAAAAAHh1AQAAAAAAABJFAAAA +AADAdQEAAAAAAKASRQAAAAAAKHYBAAAAAADgEkUAAAAAAFh2AQAAAAAAIBNFAAAAAACIdgEAAAAA +AEATRQAAAAAAuHYBAAAAAACgGEUAAAAAAOh2AQAAAAAAABlFAAAAAAAodwEAAAAAAGAZRQAAAAAA +aHcBAAAAAADgGUUAAAAAAKh3AQAAAAAAYBpFAAAAAADodwEAAAAAAOAaRQAAAAAAKHgBAAAAAABg +G0UAAAAAAGh4AQAAAAAA4BtFAAAAAACoeAEAAAAAAGAcRQAAAAAA6HgBAAAAAADgHEUAAAAAACh5 +AQAAAAAAYB1FAAAAAABoeQEAAAAAAOAdRQAAAAAAqHkBAAAAAABgHkUAAAAAAOh5AQAAAAAAgB5F +AAAAAAAYegEAAAAAAOAfRQAAAAAASHoBAAAAAAAAIEUAAAAAAHh6AQAAAAAAICBFAAAAAACoegEA +AAAAAEAgRQAAAAAA2HoBAAAAAACgIEUAAAAAAAh7AQAAAAAAwCBFAAAAAAA4ewEAAAAAAGAhRQAA +AAAAaHsBAAAAAAAAIkUAAAAAAJh7AQAAAAAAICJFAAAAAADIewEAAAAAAEAiRQAAAAAA+HsBAAAA +AABgIkUAAAAAAFh8AQAAAAAAgCJFAAAAAACIfAEAAAAAAEAjRQAAAAAA6HwBAAAAAABgI0UAAAAA +ABh9AQAAAAAAgCNFAAAAAABIfQEAAAAAAKAjRQAAAAAAeH0BAAAAAADAI0UAAAAAAKh9AQAAAAAA +4CNFAAAAAADYfQEAAAAAACAkRQAAAAAACH4BAAAAAABgJEUAAAAAADh+AQAAAAAAgCRFAAAAAACY +fgEAAAAAAKAkRQAAAAAAyH4BAAAAAADAJEUAAAAAAPh+AQAAAAAAwCVFAAAAAAAofwEAAAAAAOAl +RQAAAAAAWH8BAAAAAAAAJkUAAAAAAIh/AQAAAAAAICZFAAAAAAC4fwEAAAAAAEAmRQAAAAAA6H8B +AAAAAABgJkUAAAAAABiAAQAAAAAAgCZFAAAAAABIgAEAAAAAACApRQAAAAAAeIABAAAAAABgKUUA +AAAAAKiAAQAAAAAAgClFAAAAAADYgAEAAAAAAKApRQAAAAAACIEBAAAAAADAKUUAAAAAADiBAQAA +AAAA4ClFAAAAAABogQEAAAAAAAAqRQAAAAAAmIEBAAAAAAAgKkUAAAAAAMiBAQAAAAAAQCpFAAAA +AAD4gQEAAAAAAGAqRQAAAAAAKIIBAAAAAACAKkUAAAAAAFiCAQAAAAAAoCpFAAAAAACIggEAAAAA +ACAsRQAAAAAAuIIBAAAAAADAL0UAAAAAAOiCAQAAAAAAgDJFAAAAAAAYgwEAAAAAAEA5RQAAAAAA +SIMBAAAAAAAAO0UAAAAAAHiDAQAAAAAAIDtFAAAAAACogwEAAAAAAEA7RQAAAAAA2IMBAAAAAABg +O0UAAAAAAAiEAQAAAAAAoDtFAAAAAAA4hAEAAAAAAMA7RQAAAAAAaIQBAAAAAADgO0UAAAAAAJiE +AQAAAAAAADxFAAAAAADIhAEAAAAAACA8RQAAAAAA+IQBAAAAAABAPEUAAAAAACiFAQAAAAAAoDxF +AAAAAABYhQEAAAAAAMA8RQAAAAAAiIUBAAAAAAAAPUUAAAAAALiFAQAAAAAAID1FAAAAAADohQEA +AAAAAEA9RQAAAAAASIYBAAAAAABgPUUAAAAAAKiGAQAAAAAAgD1FAAAAAADYhgEAAAAAAEA+RQAA +AAAACIcBAAAAAACAPkUAAAAAADiHAQAAAAAAoD5FAAAAAABohwEAAAAAAOA+RQAAAAAAmIcBAAAA +AAAgP0UAAAAAAMiHAQAAAAAAoD9FAAAAAAD4hwEAAAAAAGBARQAAAAAAKIgBAAAAAACAQEUAAAAA +AFiIAQAAAAAA4EBFAAAAAACIiAEAAAAAAEBBRQAAAAAAuIgBAAAAAACAQUUAAAAAAOiIAQAAAAAA +wEFFAAAAAAAYiQEAAAAAAOBBRQAAAAAASIkBAAAAAAAgQkUAAAAAAHiJAQAAAAAAwEJFAAAAAACo +iQEAAAAAAABDRQAAAAAA2IkBAAAAAABgQ0UAAAAAAAiKAQAAAAAAgENFAAAAAAA4igEAAAAAAKBD +RQAAAAAAaIoBAAAAAADAQ0UAAAAAAJiKAQAAAAAA4ENFAAAAAADIigEAAAAAAABERQAAAAAA+IoB +AAAAAABAREUAAAAAACiLAQAAAAAAYERFAAAAAABYiwEAAAAAAKBERQAAAAAAiIsBAAAAAACARUUA +AAAAALiLAQAAAAAA4EVFAAAAAAAgjAEAAAAAACBGRQAAAAAAiIwBAAAAAABgRkUAAAAAAPCMAQAA +AAAAoEZFAAAAAABYjQEAAAAAAMBGRQAAAAAAmI0BAAAAAADgRkUAAAAAANiNAQAAAAAAIEdFAAAA +AABAjgEAAAAAAGBHRQAAAAAAqI4BAAAAAACAR0UAAAAAAOiOAQAAAAAAoEdFAAAAAAAojwEAAAAA +AMBHRQAAAAAAaI8BAAAAAADgR0UAAAAAALCPAQAAAAAAAEhFAAAAAADwjwEAAAAAACBIRQAAAAAA +MJABAAAAAABgSEUAAAAAAJiQAQAAAAAAgEhFAAAAAADYkAEAAAAAAMBIRQAAAAAAQJEBAAAAAADg +SEUAAAAAAICRAQAAAAAAIElFAAAAAADokQEAAAAAAEBJRQAAAAAAKJIBAAAAAABgSUUAAAAAAGiS +AQAAAAAAgElFAAAAAACokgEAAAAAAMBJRQAAAAAAEJMBAAAAAABASkUAAAAAAHiTAQAAAAAAoEpF +AAAAAADgkwEAAAAAAEBLRQAAAAAASJQBAAAAAABgS0UAAAAAAKiUAQAAAAAA4EtFAAAAAAAQlQEA +AAAAAIBMRQAAAAAAeJUBAAAAAAAATUUAAAAAAOCVAQAAAAAAYE1FAAAAAABIlgEAAAAAAOBNRQAA +AAAAsJYBAAAAAAAgTkUAAAAAABCXAQAAAAAAgE5FAAAAAAB4lwEAAAAAAABPRQAAAAAA4JcBAAAA +AABgT0UAAAAAAEiYAQAAAAAAgE9FAAAAAAComAEAAAAAAABQRQAAAAAAEJkBAAAAAACAUEUAAAAA +AHiZAQAAAAAA4FBFAAAAAADgmQEAAAAAAKBRRQAAAAAASJoBAAAAAAAgUkUAAAAAALCaAQAAAAAA +oFJFAAAAAAAYmwEAAAAAACBTRQAAAAAAgJsBAAAAAACAU0UAAAAAAOibAQAAAAAAgVNFAAAAAAAA +EEAAAAAAAAAAAAAQAAAAAAAAAAEAAAAIAAAACwAAAAIAAAAAAAAAAAAABhkAAAAkAAAAAAAAADhs +RgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACsZ0YAAAAAAGAQQAAAAAAAGAAA +ABAAAAAAAAAALQAAAD4AAABCAAAAAwAAAAAAAAAAAAAG7QAAAPkAAAAcAQAACHxGAAAAAACYhkYA +AAAAAAAAAAAAAAAAuIdGAAAAAAAAAAAAAAAAAKxnRgAAAAAAoBZAAAAAAABLAAAAAAAAAAAAAAAv +AQAAPwEAAEMBAAADAAAAAAAAAAAAAATYAQAA5AEAAPABAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAA +AAC0vUYAAAAAAAAbQAAAAAAAcgAAABgAAAAAAAAAEAIAABACAAATAgAAAAAAAAQAAAAAAAAGAAAA +AEBpRwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA0Z0YAAAAAACAbQAAAAAAA +hQAAAAgAAAAAAAAAJAIAACQCAAAnAgAAAAAAAAQAAAAAAAAGAAAAAFBpRwAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYZ0YAAAAAAEAbQAAAAAAAmQAAABAAAAAAAAAAMgIAADkC +AAA9AgAAAgAAAAAAAAAAAAAGQQIAAEwCAAAAAAAAfG1GAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAHxgRQAAAAAA4BtAAAAAAAC2AAAAEAAAAAAAAABVAgAAYAIAAGQCAAACAAAA +AAAAAAAAAAZoAgAAcwIAAAAAAAB8bUYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAfGBFAAAAAACAHEAAAAAAANcAAAAIAAAAAAAAAHoCAAB9AgAAgAIAAAEAAAAFAAAAAAAABpAC +AAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAAAAAAHUAAAAAA +APgAAAAAAAAAAAAAAJMCAACWAgAAmQIAAAEAAAAKAAAAAAAAAqICAAD4Z0YAAAAAAPhnRgAAAAAA +QB1AAAAAAAAQAQAAAAAAAAAAAAClAgAApQIAAKkCAAAAAAAAEQAAAAAAAAAAAAAAgB9AAAAAAAAY +AQAAKAAAAAAAAADKAwAAygMAAM0DAAAAAAAAEQAAAAAAAAAAAAAAoB9AAAAAAAAqAQAAAAAAAAAA +AADWAwAA1gMAANoDAAAAAAAAEgAAAAAAAAAAAAAA4CBAAAAAAAA0AQAAGQAAAAAAAAAQAgAAEAIA +AIgEAAAAAAAAEgAAAAAAAAAAAAAAACFAAAAAAABFAQAAEQAAAAAAAACZBAAAmQQAAJwEAAAAAAAA +EgAAAAAAAAAAAAAAICFAAAAAAABdAQAAAAAAgAAAAACtBAAArQQAALEEAAAAAAAAEwAAAAAAAAAA +AAAAQCJAAAAAAABrAQAAIAAAAAAAAABWBQAAVgUAAFkFAAAAAAAAEwAAAAAAAAYAAAAAMGlHAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFRrRgAAAAAAYCJAAAAAAACMAQAAEAAA +AAAAAABkBQAAawUAAG4FAAACAAAAFAAAAAAAAAZxBQAAfAUAAAAAAAB8bUYAAAAAAAhoRgAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAfGBFAAAAAADAIkAAAAAAAKoBAAAQAAAAAAAAAGQFAABu +BQAAgwUAAAIAAAAVAAAAAAAABnEFAAB8BQAAAAAAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAABkZ0YAAAAAACAjQAAAAAAAvQEAABAAAAAAAAAAigUAAI0FAACQBQAAAQAA +ABUAAAAAAAAGlAUAADxrRgAAAAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YA +AAAAAEAjQAAAAAAAzwEAABAAAAAAAAAAlwUAAJoFAACdBQAAAQAAABUAAAAAAAAGoQUAAHxtRgAA +AAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAAGAjQAAAAAAA4QEAABAA +AAAAAAAApAUAAKcFAACqBQAAAQAAABUAAAAAAAAGrgUAAHxtRgAAAAAACGhGAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAAIAjQAAAAAAA9AEAABAAAAAAAAAAsQUAALQFAAC3BQAA +AQAAABUAAAAAAAAGuwUAAHxtRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABk +Z0YAAAAAAKAjQAAAAAAABwIAABAAAAAAAAAApAUAAKcFAAC+BQAAAQAAABUAAAAAAAAGrgUAAHxt +RgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAAMAjQAAAAAAAGgIA +ABAAAAAAAAAAwgUAAMUFAADIBQAAAQAAABUAAAAAAAAGzAUAAHxtRgAAAAAACGhGAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAAOAjQAAAAAAALgIAABAAAAAAAAAAzwUAANIFAADV +BQAAAQAAABUAAAAAAAAG2QUAAHxtRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AABkZ0YAAAAAAAAkQAAAAAAAPwIAABAAAAAAAAAA3AUAAN8FAADiBQAAAQAAABUAAAAAAAAG5gUA +AHxtRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAACAkQAAAAAAA +UAIAABAAAAAAAAAA6QUAAOwFAADvBQAAAQAAABUAAAAAAAAG9QUAAHxtRgAAAAAACGhGAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAAGAkQAAAAAAAYQIAABAAAAAAAAAA+AUAAPsF +AAD+BQAAAQAAABUAAAAAAAAGBAYAAHxtRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAABkZ0YAAAAAAKAkQAAAAAAAcwIAABAAAAAAAAAABwYAAA4GAAARBgAAAgAAABUAAAAAAAAG +GQYAACQGAAAAAAAAfG1GAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRnRgAA +AAAAACVAAAAAAACEAgAAEAAAAAAAAAArBgAACAAAADIGAAACAAAAFQAAAAAAAAYZAAAAPgYAAAAA +AAB8bUYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZGdGAAAAAABgJUAAAAAA +AJcCAAAQAAAAAAAAACsGAAAIAAAARQYAAAIAAAAVAAAAAAAABhkAAAA+BgAAAAAAAHxtRgAAAAAA +CGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAAMAlQAAAAAAArQIAABgAAAAA +AAAAUQYAAGQGAABuBgAAAwAAABUAAAAAAAAGiAYAAJQGAACbBgAA+GxGAAAAAAAIaEYAAAAAAAAA +AAAAAAAAOH1GAAAAAAAAAAAAAAAAAOxnRgAAAAAAoCZAAAAAAADTAgAAGAAAAAAAAACjBgAAtgYA +AMAGAAADAAAAFQAAAAAAAAbcBgAA6AYAAO8GAAD4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAABMfUYA +AAAAAAAAAAAAAAAA7GdGAAAAAACAJ0AAAAAAAOMCAAAAAAAAAAAAAPcGAAACBwAABgcAAAMAAAAV +AAAAAAAABCYHAAAyBwAAOQcAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAGB9RgAAAAAAQChAAAAA +AAAGAwAAEAAAAAAAAABABwAARwcAAFYHAAADAAAAFQAAAAAAAAZxBwAAdgcAAH0HAAB8bUYAAAAA +AAhoRgAAAAAAAAAAAAAAAAB0fUYAAAAAAAAAAAAAAAAAZGdGAAAAAACgKEAAAAAAADADAAAQAAAA +AAAAAIgHAACPBwAAkgcAAAIAAAAVAAAAAAAABp0HAACiBwAAAAAAAHxtRgAAAAAACGhGAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAAOAoQAAAAAAARQMAACAAAAAAAAAApwcAAL0H +AADBBwAAAgAAABUAAAAAAAAG0gcAANgHAAAAAAAAOGxGAAAAAAAUbEYAAAAAAIBxRwAAAAAAAAAA +AAAAAAAAAAAAAAAAAIR3RgAAAAAA4ClAAAAAAABSAwAAAAAAAAAAAADdBwAA5AcAAOcHAAACAAAA +FQAAAAAAAALwBwAA+wcAAAAAAAAIaEYAAAAAAARtRgAAAAAAYCpAAAAAAABlAwAAEAAAAAAAAAAC +CAAADQgAABEIAAACAAAAFQAAAAAAAAYgCAAALAgAAAAAAAA4bEYAAAAAACxsRgAAAAAAoHFHAAAA +AAAAAAAAAAAAAAAAAAAAAAAAZGdGAAAAAAAgK0AAAAAAAHQDAAAAAAAAAAAAADMIAAA6CAAAPQgA +AAIAAAAVAAAAAAAAAkAIAABLCAAAAAAAAPhnRgAAAAAA+GdGAAAAAACAK0AAAAAAAIkDAAAYAAAA +AAAAAFIIAABcCAAAYAgAAAIAAAAVAAAAAAAABoUIAACLCAAAAAAAAExtRgAAAAAAFGxGAAAAAAAg +c0cAAAAAAAAAAAAAAAAAAAAAAAAAAADcZ0YAAAAAAOAsQAAAAAAAmwMAAAAAAAAAAAAAlggAAJ0I +AACgCAAAAgAAABUAAAAAAAACqggAALUIAAAAAAAACGhGAAAAAAAEbUYAAAAAAGAtQAAAAAAAswMA +ABAAAAAAAAAAvAgAAMYIAADKCAAAAgAAABUAAAAAAAAG7wgAAPUIAAAAAAAAnG5GAAAAAAC0bkYA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRnRgAAAAAAYC5AAAAAAADDAwAACAAAAAAAAAD8 +CAAADwkAABkJAAADAAAAFQAAAAAAAAZJCQAATwkAAFcJAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAA +AADIlEYAAAAAAAAAAAAAAAAAMGdGAAAAAABAL0AAAAAAAAMEAAAQAAAAAAAAAG4JAACJCQAAlAkA +AAMAAAAVAAAAAAAABsEJAADHCQAA0QkAADhsRgAAAAAALGxGAAAAAABgbEcAAAAAAIh9RgAAAAAA +AAAAAAAAAABkZ0YAAAAAAEAwQAAAAAAAOgQAAAAAAAAAAAAA3AkAAOMJAADnCQAAAgAAABUAAAAA +AAAC8AkAAPsJAAAAAAAA+GdGAAAAAAD4Z0YAAAAAAOAwQAAAAAAAXQQAACgAAAAAAAAAAgoAAB0K +AAAhCgAAAgAAABUAAAAAAAAGOQoAAD4KAAAAAAAAuHNGAAAAAAD4ckYAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAADRwRgAAAAAAgDFAAAAAAAB1BAAAIAAAAAAAAABLCgAAYgoAAGwKAAADAAAA +FQAAAAAAAAaYCgAAngoAAKsKAACUdEYAAAAAAAR2RgAAAAAAAAAAAAAAAACcfUYAAAAAAAAAAAAA +AAAAhGtGAAAAAABAMkAAAAAAAJsEAAAgAAAAAAAAALMKAADfCgAALQsAAAMAAAAVAAAAAAAABvkL +AAD/CwAAEwwAAJxxRgAAAAAAkHFGAAAAAABAbEcAAAAAAKj8RgAAAAAAAAAAAAAAAACEa0YAAAAA +AGA1QAAAAAAAMgUAAAAAAAAAAAAAYwwAAGoMAABtDAAAAgAAABUAAAAAAAACdwwAAIIMAAAAAAAA ++GdGAAAAAAD4Z0YAAAAAAKA1QAAAAAAAUwUAACAAAAAAAAAAiQwAAJYMAACkDAAAAwAAABUAAAAA +AAAG7AwAAPIMAAD8DAAAuG1GAAAAAACYb0YAAAAAAAAAAAAAAAAABJVGAAAAAAAAAAAAAAAAAIRr +RgAAAAAAgDZAAAAAAAB1BQAAIAAAAAAAAAAJDQAAMw0AAFANAAADAAAAFQAAAAAAAAbsDQAA+A0A +AAkOAAC0cUYAAAAAAPxxRgAAAAAAAAAAAAAAAADgh0YAAAAAAAAAAAAAAAAAhGtGAAAAAADAOEAA +AAAAAI8FAAAQAAAAAAAAAB4OAAAqDgAANQ4AAAMAAAAVAAAAAAAABnkOAACFDgAAkw4AAGxxRgAA +AAAAyHJGAAAAAAAAAAAAAAAAAAiIRgAAAAAAAAAAAAAAAABkZ0YAAAAAAKA6QAAAAAAAugUAABAA +AAAAAAAAoQ4AAKgOAACrDgAAAgAAABUAAAAAAAAGsw4AALgOAAAAAAAAfG1GAAAAAAAIaEYAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRnRgAAAAAA4DpAAAAAAADMBQAAIAAAAAAAAAC9DgAA +7Q4AABIPAAADAAAAFQAAAAAAAAaPEAAAmxAAAMYQAADIe0YAAAAAACh8RgAAAAAAwGxHAAAAAADo +B0cAAAAAAAAAAAAAAAAAnGtGAAAAAAAgQUAAAAAAAHQGAAAAAAAAAAAAACMRAAAqEQAAMxEAAAMA +AAAVAAAAAAAABEARAABLEQAAUhEAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAADCIRgAAAAAAYEFA +AAAAAACLBgAAKAAAAAAAAABbEQAAYxEAAGcRAAACAAAAFQAAAAAAAAaPEQAAmxEAAAAAAABcckYA +AAAAAGhyRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANHBGAAAAAACgQkAAAAAAAJgGAAAY +AAAAAAAAAKYRAACtEQAAsBEAAAIAAAAVAAAAAAAABsIRAADNEQAAAAAAAPRtRgAAAAAAtG5GAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADsZ0YAAAAAACBDQAAAAAAAqwYAABgAAAAAAAAA1hEA +AN0RAADgEQAAAgAAABUAAAAAAAAG8hEAAP0RAAAAAAAA9G1GAAAAAAC0bkYAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAOxnRgAAAAAAoENAAAAAAAC+BgAACAAAAAAAAAAGEgAAEhIAAEUSAAAD +AAAAFQAAAAAAAAY4EwAARBMAAF0TAAC0dEYAAAAAAGR1RgAAAAAAAAAAAAAAAAAMEUcAAAAAAAAA +AAAAAAAAMGdGAAAAAADAR0AAAAAAACoHAAAQAAAAAAAAAKITAACpEwAArBMAAAIAAAAVAAAAAAAA +BrQTAAC5EwAAAAAAAHxtRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YA +AAAAAABIQAAAAAAAPAcAABgAAAAAAAAAvhMAAPsTAAArFAAAAwAAABUAAAAAAAAG0RUAAN0VAAAN +FgAAWHtGAAAAAACwhkYAAAAAAKBsRwAAAAAAJBJHAAAAAAAAAAAAAAAAALRnRgAAAAAA4E5AAAAA +AABbBwAAAAAAAAAAAAAjEQAAKhEAAH4WAAADAAAAFQAAAAAAAARAEQAASxEAAFIRAAD4Z0YAAAAA +APhnRgAAAAAAAAAAAAAAAABYiEYAAAAAACBPQAAAAAAAcgcAACgAAAAAAAAAixYAAJMWAACjFgAA +AwAAABUAAAAAAAAG9BYAAAAXAAAPFwAA9HdGAAAAAADkd0YAAAAAAAAAAAAAAAAAgIhGAAAAAAAA +AAAAAAAAADRwRgAAAAAAAFFAAAAAAAB/BwAAEAAAAAAAAAAdFwAAJBcAAC0XAAADAAAAFQAAAAAA +AAZCFwAATRcAAFQXAAB8bUYAAAAAAAhoRgAAAAAAAAAAAAAAAACoiEYAAAAAAAAAAAAAAAAAZGdG +AAAAAABgUUAAAAAAAJYHAAAAAAAAAAAAAF0XAABgFwAAYxcAAAEAAAAVAAAAAAAAAnQXAAD4Z0YA +AAAAAPhnRgAAAAAAwFFAAAAAAAClBwAAKAAAAAAAAAB3FwAAgRcAAJEXAAADAAAAFQAAAAAAAAbJ +FwAA1RcAAOAXAACQbkYAAAAAABhoRgAAAAAAAAAAAAAAAACwfUYAAAAAAAAAAAAAAAAABHdGAAAA +AABAU0AAAAAAANAHAAAgAAAAAAAAAOwXAAD2FwAA+hcAAAIAAAAVAAAAAAAABiAYAAAmGAAAAAAA +AAhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgcEYAAAAAAGBUQAAAAAAA +7wcAAAgAAAAAAAAALhgAAD4YAABPGAAAAwAAABUAAAAAAAAGkxgAAJ8YAACsGAAAOGxGAAAAAAAI +aEYAAAAAAAAAAAAAAAAAwJ1GAAAAAAAAAAAAAAAAADBnRgAAAAAA4FZAAAAAAAAdCAAACAAAAAAA +AADFGAAA1RgAANkYAAACAAAAFQAAAAAAAAbqGAAA8BgAAAAAAAAIaEYAAAAAAFhtRgAAAAAAoHZH +AAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAAAADAV0AAAAAAADQIAAAAAAAAAAAAAPUYAAARGQAA +GxkAAAMAAAAVAAAAAAAABFMZAABfGQAAdxkAAABoRgAAAAAA5HRGAAAAAAAAAAAAAAAAAMR9RgAA +AAAAIFtAAAAAAABoCAAACAAAAAAAAAB/GQAAiRkAAJMZAAADAAAAFQAAAAAAAAa3GQAAvRkAAMIZ +AAAIaEYAAAAAAFxsRgAAAAAAgG1HAAAAAADQiEYAAAAAAAAAAAAAAAAAMGdGAAAAAAAAXEAAAAAA +AJYIAAAAAAAAAAAAAMwZAADcGQAA+BkAAAMAAAAVAAAAAAAABDQaAABAGgAATBoAABhoRgAAAAAA +FG9GAAAAAAAAAAAAAAAAAECVRgAAAAAAYF1AAAAAAACyCAAAAAAAAAAAAABgGgAAZxoAAHAaAAAD +AAAAFQAAAAAAAASKGgAAlRoAAJ4aAAAIaEYAAAAAAARtRgAAAAAAAAAAAAAAAADYfUYAAAAAAOBd +QAAAAAAAyQgAAAgAAACiAAAApRoAALEaAAC1GgAAAgAAABUAAAAAAAAGyxoAANoaAAAAAAAACGhG +AAAAAABkbUYAAAAAAOB2RwAAAAAAAAAAAAAAAAB8Z0YAAAAAADBnRgAAAAAAwF5AAAAAAADgCAAA +AAAAAAAAAADiGgAA6RoAAOwaAAACAAAAFQAAAAAAAAL6GgAABRsAAAAAAAD4Z0YAAAAAAPhnRgAA +AAAAIF9AAAAAAAD9CAAAEAAAAAAAAAAMGwAAIhsAACYbAAADAAAAFQAAAAAAAAZXGwAAYxsAAG0b +AACMb0YAAAAAAJhvRgAAAAAAAAAAAAAAAADsfUYAAAAAAAAAAAAAAAAArGdGAAAAAACAYEAAAAAA +ACIJAAAIAAAAAAAAAH0bAACUGwAAmBsAAAIAAAAVAAAAAAAABs4bAADaGwAAAAAAABh8RgAAAAAA +mIdGAAAAAADAakcAAAAAAAAAAAAAAAAAAAAAAAAAAAAwZ0YAAAAAAEBkQAAAAAAARgkAABAAAAAA +AAAA9RsAAPwbAAD/GwAAAgAAABUAAAAAAAAGBxwAABIcAAAAAAAAOGxGAAAAAAAIaEYAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAKxnRgAAAAAAoGRAAAAAAABgCQAAGAAAAAAAAAAZHAAAIBwA +ACMcAAACAAAAFQAAAAAAAAYrHAAANhwAAAAAAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAQHBGAAAAAAAgZUAAAAAAAIEJAAAQAAAAAAAAAIoFAAA9HAAAQBwAAAEAAAAV +AAAAAAAABkYcAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAArGdGAAAA +AABAZUAAAAAAAJoJAAAYAAAAAAAAAEscAABcHAAAYBwAAAMAAAAVAAAAAAAABq4dAAC6HQAA2x0A +AABoRgAAAAAABHVGAAAAAAAAAAAAAAAAABCeRgAAAAAAAAAAAAAAAAAccEYAAAAAAIBrQAAAAAAA +1gkAABAAAAAAAAAAbR4AAHceAAB7HgAAAgAAABUAAAAAAAAG8R4AAP0eAAAAAAAAdG9GAAAAAAAs +b0YAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKxnRgAAAAAA4HBAAAAAAADnCQAAEAAAAAAA +AAALHwAAFR8AABkfAAACAAAAFQAAAAAAAAZrHwAAdx8AAAAAAACkekYAAAAAAKR5RgAAAAAAoHJH +AAAAAAAAAAAAAAAAAAAAAAAAAAAArGdGAAAAAACAeEAAAAAAAAIKAAAAAAAAAAAAAMofAADUHwAA +2B8AAAIAAAAVAAAADwAAAxAgAAAcIAAAAAAAABhoRgAAAAAASHxGAAAAAAAgckcAAAAAAAB8QAAA +AAAAFAoAABgAAAAAAAAAMyAAADcgAACeIAAAAwAAABUAAAAAAAAGoCEAAAAAAACkIQAAOGxGAAAA +AAAIaEYAAAAAAAAAAAAAAAAAqFtHAAAAAAAAAAAAAAAAAOxnRgAAAAAAAH5AAAAAAAB+CgAAEAAA +AAAAAABLIgAATiIAAFMiAAADAAAAFQAAAAAAAAZoIgAAAAAAAGsiAAA4bEYAAAAAAAhoRgAAAAAA +AAAAAAAAAABgnkYAAAAAAAAAAAAAAAAAZGdGAAAAAABgfkAAAAAAAJgKAAAQAAAAAAAAAHwiAAB/ +IgAAhCIAAAMAAAAVAAAAAAAABpkiAAAAAAAAnCIAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAALCe +RgAAAAAAAAAAAAAAAABkZ0YAAAAAAMB+QAAAAAAAsgoAABgAAAAAAAAArSIAAMYiAADkIgAAAwAA +ABUAAAAAAAAGPyMAAEsjAABsIwAAVHRGAAAAAAA0dEYAAAAAAAAAAAAAAAAAbMpGAAAAAAAAAAAA +AAAAALRnRgAAAAAAwIFAAAAAAADbCgAAGAAAAAAAAAB8IgAAiCMAAJEjAAADAAAAFQAAAAAAAAaZ +IgAAAAAAALMjAAD4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAD4iEYAAAAAAAAAAAAAAAAA7GdGAAAA +AAAggkAAAAAAAA4LAAAIAAAAAAAAAL4jAADOIwAA1iMAAAMAAAAVAAAAAAAABgIkAAAOJAAAHSQA +ANR0RgAAAAAAlHVGAAAAAADAcEcAAAAAAAB+RgAAAAAAAAAAAAAAAAAwZ0YAAAAAAGCDQAAAAAAA +HgsAABAAAAAAAAAAJSQAACgkAAAxJAAAAwAAABUAAAAAAAAGWSQAAAAAAABcJAAAfG1GAAAAAAAI +aEYAAAAAAAAAAAAAAAAAIIlGAAAAAAAAAAAAAAAAAGRnRgAAAAAAwINAAAAAAAA7CwAACAAAAAAA +AABnJAAAfiQAALMkAAADAAAAFQAAAAAAAAZVJQAAYSUAAIQlAADofEYAAAAAAACHRgAAAAAAAAAA +AAAAAAD4ykYAAAAAAAAAAAAAAAAAMGdGAAAAAACAh0AAAAAAAIELAAAAAAAAAAAAAKklAAC3JQAA +zCUAAAMAAAAVAAAAAAAABPwlAAAIJgAAFSYAAAhoRgAAAAAAiG1GAAAAAAAAAAAAAAAAAKSvRgAA +AAAAYIhAAAAAAACTCwAAEAAAAAAAAAAoJgAALyYAADImAAACAAAAFQAAAAAAAAbwBwAAQCYAAAAA +AAC8ckYAAAAAAPRwRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZGdGAAAAAADgiEAAAAAA +AKMLAAAQAAAAAAAAAEkmAABQJgAAVCYAAAIAAAAVAAAAAAAABmgmAABzJgAAAAAAADhsRgAAAAAA +CGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACsZ0YAAAAAAICJQAAAAAAAtwsAABAAAAAA +AAAAeiYAAIEmAACEJgAAAgAAABUAAAAAAAAGkiYAAJ0mAAAAAAAAvHJGAAAAAAD0cEYAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRnRgAAAAAAAIpAAAAAAADMCwAAGAAAAAAAAACmJgAAtSYA +ALkmAAACAAAAFQAAAAAAAAbPJgAA2iYAAAAAAACkb0YAAAAAALRuRgAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAkGtGAAAAAACgikAAAAAAAN8LAAAIAAAAAAAAAOEmAACBJgAA6CYAAAIAAAAV +AAAAAAAABpImAAAAJwAAAAAAAIxvRgAAAAAAtG5GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAwZ0YAAAAAACCLQAAAAAAA9QsAAAAAAAAAAAAABycAAA4nAAARJwAAAgAAABUAAAAAAAACGScA +ACQnAAAAAAAA+GdGAAAAAAD4Z0YAAAAAAGCLQAAAAAAADwwAABAAAAAAAAAAKycAADcnAABDJwAA +AwAAABUAAAAAAAAGaScAAHUnAACBJwAAnHFGAAAAAAD4ckYAAAAAAAAAAAAAAAAASIlGAAAAAAAA +AAAAAAAAAGRnRgAAAAAAwIxAAAAAAABRDAAACAAAAAAAAACRJwAAnCcAAKMnAAADAAAAFQAAAAAA +AAa6JwAAxicAAM8nAAAAbkYAAAAAALRuRgAAAAAAAAAAAAAAAABwiUYAAAAAAAAAAAAAAAAAMGdG +AAAAAACAjUAAAAAAAOoFAAAIAAAAAAAAANgnAADfJwAA4icAAAIAAAAVAAAAAAAABkARAADrJwAA +AAAAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwZ0YAAAAAAMCNQAAA +AAAAaAwAAAgAAAAAAAAA8icAAA8oAAATKAAAAgAAABUAAAAAAAAGXigAAGooAAAAAAAAOGxGAAAA +AAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAAAAAAYI9AAAAAAAAlBgAACAAA +AAAAAAB2KAAAfSgAAIAoAAACAAAAFQAAAAAAAAaKKAAAlSgAAAAAAAA4bEYAAAAAAAhoRgAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAAAACgj0AAAAAAAHYMAAAIAAAAAAAAAJwoAACn +KAAAqygAAAIAAAAVAAAAAAAABronAADHKAAAAAAAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAwZ0YAAAAAAGCQQAAAAAAAhgwAAAgAAAAAAAAA0igAAN0oAADhKAAAAgAA +ABUAAAAAAAAGaAIAAPUoAAAAAAAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAADBnRgAAAAAAAJFAAAAAAACZDAAACAAAAAAAAAD8KAAACCkAAAwpAAACAAAAFQAAAAAAAAY0 +KQAAQCkAAAAAAACMb0YAAAAAALRuRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAA +AAAAkkAAAAAAAKsMAAAQAAAAAAAAAEwpAABcKQAAbSkAAAMAAAAVAAAAAAAABrcpAAC9KQAAyCkA +AIxvRgAAAAAAtG5GAAAAAAAAAAAAAAAAAJiJRgAAAAAAAAAAAAAAAABkZ0YAAAAAAOCTQAAAAAAA +xwwAABAAAAAAAAAA1SkAAOApAADjKQAAAgAAABUAAAAAAAAG8SkAAPwpAAAAAAAAOGxGAAAAAAAI +aEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRnRgAAAAAAYJRAAAAAAADaDAAAEAAAAAAA +AAADKgAADioAABIqAAACAAAAFQAAAAAAAAYoKgAAMyoAAAAAAAAAbkYAAAAAALRuRgAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAZGdGAAAAAAAAlUAAAAAAAO4MAAAIAAAAAAAAADwqAABNKgAA +UCoAAAIAAAAVAAAAAAAABmAqAABlKgAAAAAAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAwZ0YAAAAAAICVQAAAAAAA9wUAABAAAAAAAAAAaioAAHEqAAB0KgAAAgAAABUA +AAAAAAAGfSoAAIgqAAAAAAAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AGRnRgAAAAAAwJVAAAAAAAA0BgAACAAAAAAAAAB2KAAAjyoAAJIqAAACAAAAFQAAAAAAAAaKKAAA +lSgAAAAAAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAAAAAA +lkAAAAAAAAYNAAAAAAAAAAAAAJsqAACoKgAArCoAAAIAAAAVAAAAAAAAAvoqAAAGKwAAAAAAAPhn +RgAAAAAA+GdGAAAAAAAgmUAAAAAAABkNAAAQAAAAAAAAABMrAAArKwAAdSsAAAMAAAAVAAAAAAAA +BpgsAACkLAAAyywAAABoRgAAAAAAZHRGAAAAAAAAAAAAAAAAABjYRgAAAAAAAAAAAAAAAABkZ0YA +AAAAAMCfQAAAAAAAbw0AABgAAAAAAAAAES0AACQtAAA2LQAAAwAAABUAAAAAAAAGaC0AAHQtAAB8 +LQAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAwIlGAAAAAAAAAAAAAAAAAOxnRgAAAAAAAKFAAAAA +AACJDQAAEAAAAAAAAACMLQAAmy0AAJ8tAAACAAAAFQAAAAAAAAbfLQAA6y0AAAAAAAD4Z0YAAAAA +APhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPGdGAAAAAAAgo0AAAAAAAKQNAAAYAAAA +AAAAAPMtAAAXLgAAZi4AAAMAAAAVAAAAAAAABscwAADTMAAA+zAAANh7RgAAAAAAiHtGAAAAAAAA +AAAAAAAAAOQaRwAAAAAAAAAAAAAAAAC0Z0YAAAAAAECsQAAAAAAAZQ4AABAAAAAAAAAAYDEAAGcx +AABuMQAAAwAAABUAAAAAAAAGjjEAAJoxAAClMQAACGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAFH5G +AAAAAAAAAAAAAAAAAGRnRgAAAAAAAK1AAAAAAACgDgAACAAAAAAAAACsMQAAszEAALYxAAACAAAA +FQAAAAAAAAa+MQAAyTEAAAAAAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAMGdGAAAAAABgrUAAAAAAALIOAAAQAAAAAAAAANAxAADfMQAA4zEAAAIAAAAVAAAAAAAABvkx +AAAFMgAAAAAAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAA +ACCuQAAAAAAAww4AABgAAAAAAAAADDIAABgyAAAgMgAAAwAAABUAAAAAAAAGTDIAAFgyAABhMgAA +dG9GAAAAAAC0bkYAAAAAAAAAAAAAAAAA6IlGAAAAAAAAAAAAAAAAAOxnRgAAAAAAAK9AAAAAAADr +DgAACAAAAAAAAABwMgAAfjIAAJEyAAADAAAAFQAAAAAAAAbMMgAA0jIAANgyAAD4Z0YAAAAAAPhn +RgAAAAAAAAAAAAAAAAB8lUYAAAAAAAAAAAAAAAAAMGdGAAAAAABgsEAAAAAAADUPAAAYAAAAAAAA +AOkyAADxMgAA9TIAAAIAAAAVAAAAAAAABgEzAAANMwAAAAAAABBtRgAAAAAAUGxGAAAAAABAckcA +AAAAAAAAAAAAAAAAAAAAAAAAAADsZ0YAAAAAACCxQAAAAAAATQ8AAAAAAAAAAAAAFDMAABszAAAe +MwAAAgAAABUAAAAAAAACQAgAACgzAAAAAAAACGhGAAAAAAAEbUYAAAAAAICxQAAAAAAAaw8AABgA +AAAAAAAALzMAAEQzAACAMwAAAwAAABUAAAAAAAAGJjQAADI0AABMNAAA5HhGAAAAAAAUeEYAAAAA +AAAAAAAAAAAA7PRGAAAAAAAAAAAAAAAAAOxnRgAAAAAAQLRAAAAAAACdDwAAIAAAAAAAAACFNAAA +kTQAAKo0AAADAAAAFQAAAAAAAAbcNAAA6DQAAO80AAA8bkYAAAAAABhoRgAAAAAAAAAAAAAAAAC4 +lUYAAAAAAAAAAAAAAAAAhGtGAAAAAABAtUAAAAAAAMoPAAAIAAAAAAAAAAQ1AAAHNQAAFjUAAAMA +AAAVAAAAAAAABjg1AAAAAAAAOzUAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAACh+RgAAAAAAAAAA +AAAAAAAwZ0YAAAAAAMC1QAAAAAAA6A8AABgAAAAAAAAARjUAAFA1AABqNQAAAwAAABUAAAAAAAAG +xzUAANM1AADlNQAANHNGAAAAAADIckYAAAAAAAAAAAAAAAAAuNhGAAAAAAAAAAAAAAAAAOxnRgAA +AAAAILhAAAAAAABYEAAAGAAAAAAAAAAcNgAAJTYAAC82AAADAAAAFQAAAAAAAAaKNgAAljYAAKM2 +AAAUdkYAAAAAADR2RgAAAAAAAAAAAAAAAAD0lUYAAAAAAAAAAAAAAAAA7GdGAAAAAADguUAAAAAA +AJMQAAAYAAAAAAAAALk2AADFNgAA8zYAAAMAAAAVAAAAAAAABoM3AACPNwAAmzcAANxzRgAAAAAA +2HFGAAAAAAAAAAAAAAAAABztRgAAAAAAAAAAAAAAAACUZ0YAAAAAAEC8QAAAAAAAvxAAABgAAAAA +AAAA1DcAAOk3AAAVOAAAAwAAABUAAAAAAAAGpzgAALM4AADFOAAAfHNGAAAAAABUcUYAAAAAAAAA +AAAAAAAAPBNHAAAAAAAAAAAAAAAAAOxnRgAAAAAAgL5AAAAAAACFEQAAGAAAAAAAAAD8OAAADDkA +AHQ5AAADAAAAFQAAAAAAAAbHOgAA0zoAAPI6AACoe0YAAAAAAEh7RgAAAAAAAAAAAAAAAAC0QEcA +AAAAAAAAAAAAAAAA7GdGAAAAAACgw0AAAAAAAP8RAAAQAAAAAAAAAHI7AAB+OwAAgjsAAAMAAAAV +AAAAAAAABtQ7AADgOwAA7jsAAEBzRgAAAAAAqHFGAAAAAAAAAAAAAAAAABCKRgAAAAAAAAAAAAAA +AABkZ0YAAAAAAIDFQAAAAAAAEBIAABgAAAAAAAAA/jsAAAU8AAAJPAAAAwAAABUAAAAAAAAGLzwA +ADs8AABEPAAA7G9GAAAAAAC0bkYAAAAAAAAAAAAAAAAACLBGAAAAAAAAAAAAAAAAAOxnRgAAAAAA +IMZAAAAAAABbEgAAGAAAAAAAAABbPAAAbDwAAAI9AAADAAAAFQAAAAAAAAayPgAAvj4AAOA+AAAQ +c0YAAAAAAOh7RgAAAAAAoG1HAAAAAADYREcAAAAAAAAAAAAAAAAA7GdGAAAAAADAy0AAAAAAAIwS +AAAYAAAAAAAAAIM/AACLPwAAlT8AAAMAAAAVAAAAAAAABsU/AAAAAAAA0T8AAHxtRgAAAAAACGhG +AAAAAAAAAAAAAAAAADCWRgAAAAAAAAAAAAAAAADsZ0YAAAAAAMDMQAAAAAAAwhIAABgAAAAAAAAA +3T8AAPE/AAAtQAAAAwAAABUAAAAAAAAGkkAAAJ5AAACqQAAAfG1GAAAAAAAIaEYAAAAAAAAAAAAA +AAAAmP1GAAAAAAAAAAAAAAAAAMRnRgAAAAAAYM5AAAAAAADxEgAAGAAAAAAAAADcQAAA8EAAACxB +AAADAAAAFQAAAAAAAAaWQQAAokEAAK5BAAB8bUYAAAAAAAhoRgAAAAAAAAAAAAAAAACI/kYAAAAA +AAAAAAAAAAAAxGdGAAAAAAAA0EAAAAAAAAsTAAAYAAAAAAAAAOBBAADsQQAAQUIAAAMAAAAVAAAA +AAAABiZDAAAyQwAAQ0MAAHxtRgAAAAAACGhGAAAAAAAAAAAAAAAAAOwhRwAAAAAAAAAAAAAAAADE +Z0YAAAAAAADTQAAAAAAAJBMAABgAAAAAAAAA/jsAAJRDAACnQwAAAwAAABUAAAAAAAAGLzwAADs8 +AABEPAAA7G9GAAAAAAC0bkYAAAAAAAAAAAAAAAAAbLBGAAAAAAAAAAAAAAAAAOxnRgAAAAAAoNNA +AAAAAAA8EwAAGAAAAAAAAADUQwAA5UMAAFxEAAADAAAAFQAAAAAAAAZsRQAAeEUAAIpFAACEbkYA +AAAAAKR4RgAAAAAAwG1HAAAAAABIOUcAAAAAAAAAAAAAAAAA7GdGAAAAAACA10AAAAAAAFQTAAAY +AAAAAAAAAPVFAAABRgAABUYAAAIAAAAVAAAAAAAABh1GAAAjRgAAAAAAAPhsRgAAAAAACGhGAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADsZ0YAAAAAACDYQAAAAAAAaRMAACgAAAAAAAAALEYA +ADxGAABARgAAAgAAABUAAAAAAAAGbEYAAHJGAAAAAAAAeG5GAAAAAAAYaEYAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAADRwRgAAAAAAANlAAAAAAACAEwAAEAAAAAAAAAB7RgAAgkYAAIVGAAAC +AAAAFQAAAAAAAAaVRgAAmkYAAAAAAAB8bUYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAZGdGAAAAAABg2UAAAAAAAJQTAAAQAAAAAAAAAKFGAACoRgAAq0YAAAIAAAAVAAAAAAAA +BrdGAAC8RgAAAAAAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YA +AAAAAKDZQAAAAAAArhMAABAAAAAAAAAAw0YAAMZGAADNRgAAAwAAABUAAAAAAAAG3UYAAAAAAADg +RgAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAPH5GAAAAAAAAAAAAAAAAAGRnRgAAAAAAwNlAAAAA +AADoEwAACAAAAAAAAADnRgAA/kYAAAJHAAACAAAAFQAAAAAAAAZYRwAAZEcAAAAAAAD4Z0YAAAAA +APhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAAAAAg20AAAAAAAAcUAAAgAAAA +AAAAAGxHAAB2RwAAgkcAAAMAAAAVAAAAAAAABuwNAACyRwAAwkcAAPhnRgAAAAAA+GdGAAAAAAAA +AAAAAAAAADiKRgAAAAAAAAAAAAAAAACEa0YAAAAAAGDdQAAAAAAANxQAABgAAAAAAAAAzkcAAOJH +AAD6RwAAAwAAABUAAAAAAAAGRUgAAEtIAABVSAAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAALL5G +AAAAAAAAAAAAAAAAAOxnRgAAAAAAoN5AAAAAAACVFAAAGAAAAAAAAAB0SAAAhUgAAIlIAAACAAAA +FQAAAAAAAAahSAAApkgAAAAAAACAbEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAcHBGAAAAAABA30AAAAAAALAUAAAgAAAAAAAAAKtIAAC5SAAAvUgAAAIAAAAVAAAAAAAABttI +AADhSAAAAAAAAIBsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB0d0YAAAAA +AADgQAAAAAAAyRQAACAAAAAAAAAA50gAAO5IAADySAAAAgAAABUAAAAAAAAGAEkAAAtJAAAAAAAA +gGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHR3RgAAAAAAoOBAAAAAAADs +FAAAGAAAAAAAAAASSQAAP0kAAIdJAAADAAAAFQAAAAAAAAa8SgAAwkoAAN5KAAAYaEYAAAAAAPhv +RgAAAAAAAAAAAAAAAAC8LkcAAAAAAAAAAAAAAAAA7GdGAAAAAADg5EAAAAAAADgVAAAYAAAAAAAA +AFNLAABnSwAAgksAAAMAAAAVAAAAAAAABgdMAAANTAAAG0wAABhoRgAAAAAA+G9GAAAAAAAAAAAA +AAAAAFjZRgAAAAAAAAAAAAAAAADsZ0YAAAAAAMDmQAAAAAAAWxUAACgAAAAAAAAAUkwAAFpMAAB6 +TAAAAwAAABUAAAAAAAAGAE0AAAZNAAAPTQAAsG9GAAAAAACYb0YAAAAAAAAAAAAAAAAA0LBGAAAA +AAAAAAAAAAAAADRwRgAAAAAAYOhAAAAAAAB1FQAAIAAAAAAAAAA2TQAASk0AAF5NAAADAAAAFQAA +AAAAAAbLTQAA0U0AAOVNAAAcc0YAAAAAAKBzRgAAAAAAAAAAAAAAAABslkYAAAAAAAAAAAAAAAAA +hGtGAAAAAACg6kAAAAAAAJEVAAAgAAAAAAAAAP1NAAANTgAAEU4AAAMAAAAVAAAAAAAABkdOAABT +TgAAYU4AANxzRgAAAAAAUHJGAAAAAAAAAAAAAAAAAFB+RgAAAAAAAAAAAAAAAAB0d0YAAAAAAADs +QAAAAAAAqxUAACAAAAAAAAAAaU4AAJNOAAC0TgAAAwAAABUAAAAAAAAGHlIAACpSAABbUgAARHpG +AAAAAAD0eUYAAAAAAAAAAAAAAAAArE1HAAAAAAAAAAAAAAAAAIRrRgAAAAAAAPhAAAAAAADDFQAA +QAAAAAAAAAAYUwAAKVMAAC1TAAADAAAAFQAAAAAAAAbCUwAAzlMAAOBTAABcb0YAAAAAALRuRgAA +AAAAAAAAAAAAAABgikYAAAAAAAAAAAAAAAAAUIZGAAAAAACg+0AAAAAAAOEVAAAQAAAAAAAAAOtT +AAD7UwAA/1MAAAIAAAAVAAAAAAAABh1UAAApVAAAAAAAAGxxRgAAAAAA9HBGAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAAKD8QAAAAAAA+xUAACAAAAAAAAAANlQAAEBUAABEVAAA +AwAAABUAAAAAAAAGuFYAAL5WAADFVgAA+GxGAAAAAAB0bEYAAAAAAAAAAAAAAAAAwD5HAAAAAAAA +AAAAAAAAAIRrRgAAAAAAIANBAAAAAAAxFgAAEAAAAAAAAAA2VwAAPVcAAERXAAADAAAAFQAAAAAA +AAZoAgAAXlcAAGdXAAAIbEYAAAAAAAhoRgAAAAAAAAAAAAAAAAColkYAAAAAAAAAAAAAAAAAZGdG +AAAAAADAA0EAAAAAAEsWAAAAAAAAAAAAAHRXAAB8VwAAh1cAAAMAAAAVAAAAAAAABCAIAACnVwAA +slcAAPhnRgAAAAAAPGtGAAAAAADga0cAAAAAAGR+RgAAAAAAgARBAAAAAABfFgAAAAAAAAAAAAC9 +VwAAxFcAANNXAAADAAAAFQAAAAAAAATtVwAA+FcAAAFYAAAIaEYAAAAAAARtRgAAAAAAAAAAAAAA +AAAAn0YAAAAAAAAFQQAAAAAAeRYAAAgAAAAAAAAAEFgAABdYAAAaWAAAAgAAABUAAAAAAAAGJFgA +AC9YAAAAAAAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAAAAAA +gAVBAAAAAACMFgAAAAAAAAAAAAA2WAAAPVgAAFBYAAADAAAAFQAAAAAAAAR0WAAAf1gAAIZYAAD4 +Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAA0sUYAAAAAACAGQQAAAAAApRYAABAAAAAAAAAAmVgAAKlY +AAC5WAAAAwAAABUAAAAAAAAGD1kAABtZAAApWQAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAiIpG +AAAAAAAAAAAAAAAAADxnRgAAAAAAwAhBAAAAAADxFgAAGAAAAAAAAAA3WQAAR1kAAHBZAAADAAAA +FQAAAAAAAAbgWQAA7FkAAPlZAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAD42UYAAAAAAAAAAAAA +AAAAbGtGAAAAAABAC0EAAAAAACwXAAAIAAAAAAAAACBaAAAuWgAAN1oAAAMAAAAVAAAAAAAABn1a +AACJWgAAlloAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAHh+RgAAAAAAAAAAAAAAAAAwZ0YAAAAA +ACANQQAAAAAASRcAAAgAAAAAAAAAn1oAAK5aAACyWgAAAgAAABUAAAAAAAAG0FoAANxaAAAAAAAA ++GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAAAAAAAA5BAAAAAABr +FwAACAAAAAAAAADnWgAA/loAADFbAAADAAAAFQAAAAAAAAY7XAAAR1wAAHdcAAD4Z0YAAAAAAPhn +RgAAAAAAAAAAAAAAAAAsI0cAAAAAAAAAAAAAAAAAMGdGAAAAAAAAFUEAAAAAAI8YAAAQAAAAAAAA +AN5cAADqXAAA7lwAAAMAAAAVAAAAAAAABh1UAAAgXQAAK10AAPhnRgAAAAAA+GdGAAAAAAAAAAAA +AAAAALCKRgAAAAAAAAAAAAAAAABkZ0YAAAAAAAAWQQAAAAAArxgAAAgAAAAAAAAAOl0AAE1dAABo +XQAAAwAAABUAAAAAAAAGql0AALZdAADCXQAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAmLFGAAAA +AAAAAAAAAAAAADBnRgAAAAAAYBdBAAAAAADIGAAAAAAAAAAAAADdXQAA610AAO9dAAACAAAAFQAA +AAAAAAIYXgAAJF4AAAAAAAAIaEYAAAAAAARtRgAAAAAAgBhBAAAAAADgGAAAAAAAAAAAAAAuXgAA +OV4AAEJeAAADAAAAFQAAAAAAAARaXgAAZV4AAGxeAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAADY +ikYAAAAAAAAZQQAAAAAAKxkAADAAAAAAAAAAdV4AAIZeAACSXgAAAwAAABUAAAAAAAAG0F4AANxe +AADlXgAAHG1GAAAAAAAIaEYAAAAAAAAAAAAAAAAAAItGAAAAAAAAAAAAAAAAAAR7RgAAAAAAABtB +AAAAAABaGQAAEAAAAAAAAADxXgAAAV8AAAVfAAACAAAAFQAAAAAAAAYkXwAAKl8AAAAAAACIc0YA +AAAAABh0RgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZGdGAAAAAAAAHEEAAAAAAGsZAAAQ +AAAAAAAAADRfAABCXwAAUl8AAAMAAAAVAAAAAAAABqJfAACuXwAAvV8AADhsRgAAAAAACGhGAAAA +AAAAAAAAAAAAACiLRgAAAAAAAAAAAAAAAABkZ0YAAAAAAEAeQQAAAAAAjxkAABAAAAAAAAAAzV8A +ANRfAADbXwAAAwAAABUAAAAAAAAG818AAP9fAAAGYAAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAA +UItGAAAAAAAAAAAAAAAAAGRnRgAAAAAA4B5BAAAAAACjGQAAGAAAAAAAAAAPYAAAFmAAABlgAAAC +AAAAFQAAAAAAAAa3RgAAJWAAAAAAAADAbkYAAAAAALRuRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAA7GdGAAAAAAAgH0EAAAAAALMZAAAYAAAAAAAAACxgAAA3YAAAO2AAAAIAAAAVAAAAAAAA +BlFgAABdYAAAAAAAAMRtRgAAAAAAGGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADsZ0YA +AAAAAOAfQQAAAAAAwhkAACgAAAAAAAAAaGAAAHRgAACFYAAAAwAAABUAAAAAAAAG5mAAAPJgAAD9 +YAAA4GxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAUJ9GAAAAAAAAAAAAAAAAADRwRgAAAAAAYCJBAAAA +AADZGQAAAAAAAAAAAAAOYQAAFWEAACZhAAADAAAAFQAAAAAAAATyEQAASmEAAFNhAAAIaEYAAAAA +AARtRgAAAAAAAAAAAAAAAACgn0YAAAAAAOAiQQAAAAAA6hkAAAgAAAAAAAAAZGEAAHNhAAB3YQAA +AgAAABUAAAAAAAAGaC0AAK1hAAAAAAAA3G1GAAAAAAAsb0YAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAADBnRgAAAAAAICRBAAAAAAAEGgAAAAAAAAAAAAC6YQAAwWEAAMRhAAACAAAAFQAAAAAA +AALSYQAA3WEAAAAAAAD4Z0YAAAAAAPhnRgAAAAAAgCRBAAAAAAATGgAAAAAAAAAAAADkYQAA7GEA +APBhAAACAAAAFQAAAAAAAAIGYgAAEmIAAAAAAAD4Z0YAAAAAAPhnRgAAAAAAYCVBAAAAAAAkGgAA +AAAAAAAAAAAaYgAAJWIAADJiAAADAAAAFQAAAAAAAARQYgAAXGIAAGRiAAD4Z0YAAAAAAPhnRgAA +AAAAAAAAAAAAAACMfkYAAAAAACAmQQAAAAAARRoAABgAAAAAAAAAb2IAAI5iAADrYgAAAwAAABUA +AAAAAAAGMmQAAD5kAABnZAAACGhGAAAAAADIbEYAAAAAAKBvRwAAAAAAjDFHAAAAAAAAAAAAAAAA +ALRrRgAAAAAAwCxBAAAAAADjGgAAAAAAAAAAAADsZAAA+GQAAAtlAAADAAAAFQAAAAAAAAQ5ZQAA +RWUAAFFlAAAIaEYAAAAAAARtRgAAAAAAAAAAAAAAAACgfkYAAAAAAAAuQQAAAAAAGRsAAAAAAAAA +AAAAZGUAAHRlAACfZQAAAwAAABUAAAAAAAAEEGYAABxmAAA4ZgAA+GdGAAAAAAA8a0YAAAAAAGBv +RwAAAAAA5O1GAAAAAADAMEEAAAAAACwbAAAAAAAAAAAAAGJmAABqZgAAcWYAAAMAAAAVAAAAAAAA +BItmAACXZgAAnmYAAAhoRgAAAAAAQG1GAAAAAAAAAAAAAAAAALR+RgAAAAAAgDFBAAAAAABdGwAA +CAAAAAAAAAClZgAAtmYAABtnAAADAAAAFQAAAAAAAAawaAAAvGgAAPloAAAAaEYAAAAAACR1RgAA +AAAAAAAAAAAAAAAANkcAAAAAAAAAAAAAAAAAMGdGAAAAAABgPkEAAAAAAK8bAAAAAAAAAAAAAG5p +AAB1aQAAeGkAAAIAAAAVAAAAAAAAAoJpAACNaQAAAAAAAPhnRgAAAAAA+GdGAAAAAACgPkEAAAAA +AM8bAAAAAAAAAAAAAJRpAACbaQAAomkAAAMAAAAVAAAAAAAABLppAADFaQAAzGkAAPhnRgAAAAAA ++GdGAAAAAAAAAAAAAAAAAMh+RgAAAAAAID9BAAAAAAD+GwAAAAAAAAAAAADTaQAA3WkAACBqAAAD +AAAAFQAAAAYAAATOagAA2moAAO1qAABAaEYAAAAAACxyRgAAAAAAQG9HAAAAAAB4/0YAAAAAAIBD +QQAAAAAAFRwAAAAAAAAAAAAAL2sAAD9rAABXawAAAwAAABUAAAAAAAAEkkAAAKZrAAC7awAAAGhG +AAAAAABEdUYAAAAAAAAAAAAAAAAAhMtGAAAAAAAgRUEAAAAAAH4cAAAIAAAAAAAAANdrAADoawAA ++GsAAAMAAAAVAAAAAAAABl9sAABrbAAAiGwAAEBoRgAAAAAAdHJGAAAAAAAAAAAAAAAAAHiLRgAA +AAAAAAAAAAAAAAAwZ0YAAAAAACBKQQAAAAAAjRwAAAgAAAAAAAAAlmwAAKZsAADRbAAAAwAAABUA +AAAAAAAGPG0AAEhtAABcbQAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAA7AhHAAAAAAAAAAAAAAAA +ADBnRgAAAAAAwEtBAAAAAAC5HAAAAAAAAAAAAACHbQAAk20AAKNtAAADAAAAFQAAAAAAAATUbQAA +4G0AAOxtAAAIaEYAAAAAAARtRgAAAAAAAAAAAAAAAADwn0YAAAAAAMBMQQAAAAAA0hwAAAAAAAAA +AAAA/G0AAAhuAAAjbgAAAwAAABUAAAAAAAAEAiQAAHFuAACEbgAA+GdGAAAAAAD4Z0YAAAAAAAAA +AAAAAAAAmNpGAAAAAAAATkEAAAAAAOUcAAAgAAAAAAAAAJ9uAACzbgAAt24AAAMAAAAVAAAAAAAA +BiFvAAAnbwAAM28AADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAKCLRgAAAAAAAAAAAAAAAAAocEYA +AAAAACBQQQAAAAAA9xwAAAAAAAAAAAAAWG8AAGBvAAB0bwAAAwAAABUAAAAAAAAEvW8AAAAAAADJ +bwAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAApL5GAAAAAAAAUkEAAAAAADEdAAAAAAAAAAAAAOFv +AADsbwAA8G8AAAIAAAAVAAAAAAAAAwRwAAAQcAAAAAAAAPhnRgAAAAAAPGtGAAAAAACAb0cAAAAA +AOBSQQAAAAAASR0AAAgAAAAAAAAAF3AAACdwAAAvcAAAAwAAABUAAAAAAAAGUHAAAFxwAABlcAAA +OGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAA3H5GAAAAAAAAAAAAAAAAADBnRgAAAAAAAFRBAAAAAAB7 +HQAAEAAAAAAAAABtcAAAfnAAAI9wAAADAAAAFQAAAAAAAAb5cAAABXEAABhxAADwcUYAAAAAAKRy +RgAAAAAAQHFHAAAAAADklkYAAAAAAAAAAAAAAAAARGdGAAAAAAAAV0EAAAAAAIwdAAAAAAAAAAAA +AClxAAA8cQAARHEAAAMAAAAVAAAAAAAABIJxAACOcQAAn3EAAEBoRgAAAAAAeHFGAAAAAAAAAAAA +AAAAAPB+RgAAAAAAQFhBAAAAAACjHQAAKAAAAAAAAACncQAAsnEAALlxAAADAAAAFQAAAAAAAAbZ +cQAA5XEAAOxxAAAobUYAAAAAAAhoRgAAAAAAAAAAAAAAAAAEf0YAAAAAAAAAAAAAAAAANHBGAAAA +AAAAWUEAAAAAALkdAAAAAAAAAAAAAPNxAAADcgAAKnIAAAMAAAAVAAAAAAAABH5yAACKcgAAnXIA +AAhoRgAAAAAABG1GAAAAAAAAAAAAAAAAAGgARwAAAAAAIFpBAAAAAAAGHgAAEAAAAAAAAADGcgAA +13IAAPNyAAADAAAAFQAAAAAAAAaPcwAAm3MAAK5zAABscUYAAAAAAAB0RgAAAAAAAAAAAAAAAAAc +v0YAAAAAAAAAAAAAAAAAZGdGAAAAAACAXUEAAAAAABweAAAIAAAAAAAAAMpzAADmcwAACnQAAAMA +AAAVAAAAAAAABqF0AACtdAAAxnQAADhsRgAAAAAALGxGAAAAAAAgb0cAAAAAAJS/RgAAAAAAAAAA +AAAAAAAwZ0YAAAAAACBgQQAAAAAAhx4AAAAAAAAAAAAA5HQAAOt0AADudAAAAgAAABUAAAAAAAAC +QBEAAPh0AAAAAAAA+GdGAAAAAAD4Z0YAAAAAAGBgQQAAAAAAox4AABAAAAAAAAAA/3QAABZ1AAAx +dQAAAwAAABUAAAAAAAAGnHUAAKh1AAC5dQAAjG9GAAAAAAC0bkYAAAAAAAAAAAAAAAAAQKBGAAAA +AAAAAAAAAAAAAGRnRgAAAAAAgGNBAAAAAAC6HgAAAAAAAAAAAADUdQAA23UAAO51AAADAAAAFQAA +AAAAAAQSdgAAHXYAACR2AAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAD8sUYAAAAAAABkQQAAAAAA +7R4AAAAAAAAAAAAAN3YAAEd2AAB0dgAAAwAAABUAAAAAAAAEzXYAANl2AADsdgAA+GdGAAAAAAAk +a0YAAAAAAAAAAAAAAAAAWAFHAAAAAABAZUEAAAAAADIfAAAIAAAAAAAAABl3AAAldwAASncAAAMA +AAAVAAAAAAAABrZ3AADCdwAAzncAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAKzuRgAAAAAAAAAA +AAAAAAAwZ0YAAAAAAOBmQQAAAAAAYB8AABAAAAAAAAAA9XcAAA14AABYeAAAAwAAABUAAAAAAAAG +VnkAAGJ5AACheQAANHpGAAAAAADEeUYAAAAAAGB4RwAAAAAA8AlHAAAAAAAAAAAAAAAAAGRnRgAA +AAAAoG1BAAAAAABfIAAAEAAAAAAAAADseQAA83kAAPZ5AAACAAAAFQAAAAAAAAYAegAAC3oAAAAA +AACkbEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZGdGAAAAAAAAbkEAAAAA +AHcgAAAYAAAAAAAAABJ6AAAoegAANHoAAAMAAAAVAAAAAAAABr96AADLegAA3noAANR5RgAAAAAA +VHpGAAAAAAAAAAAAAAAAABh/RgAAAAAAAAAAAAAAAADsZ0YAAAAAAGBwQQAAAAAAjyAAABAAAAAA +AAAA6noAAPt6AAAaewAAAwAAABUAAAAAAAAGMXwAAD18AABUfAAAjG9GAAAAAACYb0YAAAAAAAAA +AAAAAAAALH9GAAAAAAAAAAAAAAAAAGRnRgAAAAAAYHRBAAAAAAC8IAAAEAAAAAAAAABzfAAAf3wA +AJR8AAADAAAAFQAAAAAAAAYEfQAAEH0AAB99AACMb0YAAAAAALRuRgAAAAAAAAAAAAAAAABAf0YA +AAAAAAAAAAAAAAAAZGdGAAAAAABgdkEAAAAAAM0gAAAoAAAAAAAAADR9AABEfQAASH0AAAIAAAAV +AAAAAAAABpZ9AACifQAAAAAAAKBtRgAAAAAAtG5GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAA0cEYAAAAAAAB4QQAAAAAA3yAAABAAAAAAAAAAq30AAMN9AAACfgAAAwAAABUAAAAAAAAGzn4A +ANp+AADsfgAACHJGAAAAAAAMdEYAAAAAAAAAAAAAAAAA+ONGAAAAAAAAAAAAAAAAAGRnRgAAAAAA +YHtBAAAAAAAlIQAAKAAAAAAAAAAvfwAAP38AAFJ/AAADAAAAFQAAAAAAAAbtfwAA+X8AAAqAAACg +bUYAAAAAALRuRgAAAAAAAAAAAAAAAAAQzEYAAAAAAAAAAAAAAAAANHBGAAAAAACgfUEAAAAAAG0h +AAAIAAAAAAAAACWAAAAsgAAAL4AAAAIAAAAVAAAAAAAABj2AAABIgAAAAAAAAPhnRgAAAAAA+GdG +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwZ0YAAAAAACB+QQAAAAAAeyEAADAAAAAAAAAA +T4AAAHKAAACugAAAAwAAABUAAAAAAAAGSYEAAFWBAABmgQAARGxGAAAAAAAIaEYAAAAAAAAAAAAA +AAAASAJHAAAAAAAAAAAAAAAAAER3RgAAAAAAIIFBAAAAAADfIQAAIAAAAAAAAACqgQAAu4EAANyB +AAADAAAAFQAAAAAAAAZPggAAW4IAAHSCAACMb0YAAAAAALRuRgAAAAAAAAAAAAAAAABgskYAAAAA +AAAAAAAAAAAAWHBGAAAAAABAhUEAAAAAAPQhAAAgAAAAAAAAAJeCAAChggAAxoIAAAMAAAAVAAAA +AAAABh+DAAAlgwAAK4MAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAMj1RgAAAAAAAAAAAAAAAACE +a0YAAAAAAECGQQAAAAAADCIAAAAAAAAAAAAAXoMAAGiDAABsgwAAAgAAABUAAAAAAAACiIMAAJSD +AAAAAAAAGGhGAAAAAACobkYAAAAAAACHQQAAAAAAJSIAAAAAAAAAAAAAnYMAAKCDAACjgwAAAQAA +ABUAAAAAAAACpoMAAPhnRgAAAAAA+GdGAAAAAAAgh0EAAAAAADQiAAAQAAAAAAAAAKmDAACwgwAA +s4MAAAIAAAAVAAAAAAAABkIXAADDgwAAAAAAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAABEZ0YAAAAAAICHQQAAAAAAViIAAAgAAAAAAAAAyoMAANSDAADdgwAAAwAAABUA +AAAAAAAGMoQAAD6EAABJhAAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAVH9GAAAAAAAAAAAAAAAA +ADBnRgAAAAAAIIpBAAAAAAB+IgAACAAAAAAAAABShAAAVoQAAGKEAAADAAAAFQAAAAAAAAakhAAA +AAAAAKiEAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAADIi0YAAAAAAAAAAAAAAAAAMGdGAAAAAAAg +i0EAAAAAAKIiAAAQAAAAAAAAALSEAADEhAAA0YQAAAMAAAAVAAAAAAAABiWFAAAxhQAAPYUAADhs +RgAAAAAACGhGAAAAAAAAAAAAAAAAAPCLRgAAAAAAAAAAAAAAAAA8Z0YAAAAAAICPQQAAAAAA+iIA +AAgAAAAAAAAAVYUAAHGFAACAhQAAAwAAABUAAAAAAAAGyoUAANaFAADihQAAOGxGAAAAAAAIaEYA +AAAAAAAAAAAAAAAAGIxGAAAAAAAAAAAAAAAAADBnRgAAAAAA4JBBAAAAAAA2IwAAEAAAAAAAAADz +hQAAEIYAADuGAAADAAAAFQAAAAAAAAa3hgAAw4YAANGGAABMc0YAAAAAAKBzRgAAAAAAAAAAAAAA +AAA420YAAAAAAAAAAAAAAAAAZGdGAAAAAABgk0EAAAAAALcjAAAQAAAAAAAAAAOHAAAUhwAAIocA +AAMAAAAVAAAAAAAABrOHAAC/hwAAzYcAAABuRgAAAAAAtG5GAAAAAAAAAAAAAAAAAECMRgAAAAAA +AAAAAAAAAABkZ0YAAAAAAACXQQAAAAAA7yMAABAAAAAAAAAA24cAAOKHAADlhwAAAgAAABUAAAAA +AAAG/4cAAAqIAAAAAAAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAERn +RgAAAAAAgJdBAAAAAAAZJAAAAAAAAAAAAAARiAAAIIgAACeIAAADAAAAFQAAAAAAAAQ9iAAASIgA +AE+IAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAABof0YAAAAAAACYQQAAAAAAOSQAAAAAAAAAAAAA +VogAAFqIAABiiAAAAwAAABUAAAAAAAAEjIgAAAAAAACQiAAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAA +AAAAaIxGAAAAAABAmUEAAAAAAIEkAAAAAAAAAAAAAJ6IAACmiAAAwYgAAAMAAAAVAAAAAAAABPSI +AAD/iAAACokAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAJzMRgAAAAAA4JlBAAAAAACmJAAACAAA +AAAAAAAliQAAL4kAAEyJAAADAAAAFQAAAAAAAAZ9iQAAiYkAAJGJAAD4Z0YAAAAAAPhnRgAAAAAA +AAAAAAAAAAAozUYAAAAAAAAAAAAAAAAAMGdGAAAAAADAmkEAAAAAAN8kAAAAAAAAAAAAALKJAAC8 +iQAA1IkAAAMAAAAVAAAAAAAABDCKAAA8igAASIoAAPhnRgAAAAAAMGtGAAAAAAAAbEcAAAAAAAzA +RgAAAAAAoJ1BAAAAAADyJAAAAAAAAAAAAABgigAAbIoAAI+KAAADAAAAFQAAAAAAAATXigAA44oA +APaKAAAYaEYAAAAAAKhuRgAAAAAAAAAAAAAAAAB070YAAAAAAMCeQQAAAAAACyUAABgAAAAAAAAA +H4sAACiLAAA0iwAAAwAAABUAAAAAAAAGfosAAIqLAACZiwAAOGxGAAAAAAAIaEYAAAAAAAAAAAAA +AAAAhMBGAAAAAAAAAAAAAAAAALRnRgAAAAAAIKBBAAAAAABuJQAAGAAAAAAAAACtiwAAuYsAAM2L +AAADAAAAFQAAAAAAAAYPjAAAG4wAACuMAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAADEskYAAAAA +AAAAAAAAAAAA1GdGAAAAAACgoUEAAAAAAIUlAAAIAAAAAAAAAEGMAABJjAAAVIwAAAMAAAAVAAAA +AAAABnyMAACIjAAAk4wAAIxvRgAAAAAAtG5GAAAAAAAAAAAAAAAAAJCMRgAAAAAAAAAAAAAAAAAw +Z0YAAAAAAMCiQQAAAAAAqyUAAAgAAAAAAAAAnowAAKmMAAC0jAAAAwAAABUAAAAAAAAG3IwAAOiM +AADzjAAAjG9GAAAAAAC0bkYAAAAAAAAAAAAAAAAAkKBGAAAAAAAAAAAAAAAAADBnRgAAAAAAoKNB +AAAAAADQJQAAIAAAAAAAAAACjQAAEY0AABmNAAADAAAAFQAAAAAAAAbcBgAAPY0AAESNAAA4bEYA +AAAAAAhoRgAAAAAAAAAAAAAAAAAgl0YAAAAAAAAAAAAAAAAAxHpGAAAAAACApEEAAAAAAPclAAAo +AAAAAAAAAFCNAABzjQAAwo0AAAMAAAAVAAAAAAAABtCOAADcjgAA8Y4AANxtRgAAAAAAvG9GAAAA +AABAa0cAAAAAABQ7RwAAAAAAAAAAAAAAAACYfEYAAAAAAMCoQQAAAAAA/SYAABAAAAAAAAAAbI8A +AHyPAACjjwAAAwAAABUAAAAAAAAG848AAP+PAAAJkAAACGhGAAAAAAAEbUYAAAAAAAAAAAAAAAAA +/MBGAAAAAAAAAAAAAAAAAARwRgAAAAAAQKpBAAAAAAA5JwAAIAAAAAAAAAAwkAAAQJAAAFiQAAAD +AAAAFQAAAAAAAAahkAAArZAAALiQAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAos0YAAAAAAAAA +AAAAAAAAhGtGAAAAAADAq0EAAAAAAGInAAAQAAAAAAAAANaQAADnkAAA65AAAAMAAAAVAAAAAAAA +BimRAAA1kQAAPZEAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAHTBRgAAAAAAAAAAAAAAAABkZ0YA +AAAAAICtQQAAAAAAkCcAABAAAAAAAAAAWJEAAGiRAABskQAAAgAAABUAAAAAAAAGkpEAAJ6RAAAA +AAAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRnRgAAAAAAwK5BAAAA +AAC7JwAAIAAAAAAAAACokQAAwJEAAO+RAAADAAAAFQAAAAAAAAabkgAAp5IAALmSAAA4bEYAAAAA +AAhoRgAAAAAAAAAAAAAAAADswUYAAAAAAAAAAAAAAAAAhGtGAAAAAAAAskEAAAAAAAsoAAAYAAAA +AAAAAN6SAADqkgAA7pIAAAIAAAAVAAAAAAAABiiTAAA0kwAAAAAAAABuRgAAAAAAtG5GAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC0Z0YAAAAAAECzQQAAAAAALCgAAAgAAAAAAAAAPpMAAFST +AABYkwAAAgAAABUAAAAAAAAGAiQAAIyTAAAAAAAAjG9GAAAAAADMbkYAAAAAAIBrRwAAAAAAAAAA +AAAAAAAAAAAAAAAAADBnRgAAAAAAgLRBAAAAAABNKAAAGAAAAAAAAACYkwAApJMAAKiTAAADAAAA +FQAAAAAAAAbwkwAA/JMAAAaUAACAbEYAAAAAAAhoRgAAAAAAAAAAAAAAAAB8f0YAAAAAAAAAAAAA +AAAA7GdGAAAAAAAgtkEAAAAAAHEoAAAYAAAAAAAAAA6UAAAklAAAKJQAAAIAAAAVAAAAAAAABk6U +AABalAAAAAAAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADsZ0YAAAAA +AEC3QQAAAAAAiigAAAgAAAAAAAAAYpQAAHKUAACClAAAAwAAABUAAAAAAAAG0pQAAN6UAADvlAAA ++GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAPPBGAAAAAAAAAAAAAAAAADBnRgAAAAAAALlBAAAAAAD9 +KAAAAAAAAAAAAAAulQAAOpUAAFGVAAADAAAAFQAAAAAAAATUbQAAh5UAAI+VAAD4Z0YAAAAAAPhn +RgAAAAAAAAAAAAAAAAC4jEYAAAAAAAC6QQAAAAAAEykAAAAAAAAAAAAAppUAAK6VAADTlQAAAwAA +ABUAAAAAAAAEI5YAAC+WAABClgAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAApPZGAAAAAABAu0EA +AAAAACMpAAAAAAAAAAAAAGmWAACAlgAAkZYAAAMAAAAVAAAAAAAABAqXAAAWlwAAKpcAAAhoRgAA +AAAABG1GAAAAAAAAAAAAAAAAALTNRgAAAAAAoL5BAAAAAABOKQAACAAAAAAAAABZlwAAb5cAAHOX +AAADAAAAFQAAAAAAAAbblwAA55cAAP2XAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAABAzkYAAAAA +AAAAAAAAAAAAMGdGAAAAAAAgwkEAAAAAAGspAAAQAAAAAAAAACaYAABEmAAABJkAAAMAAAAVAAAA +AAAABnybAACImwAAt5sAAGxxRgAAAAAA5HFGAAAAAACgeEcAAAAAAMBSRwAAAAAAAAAAAAAAAAA8 +Z0YAAAAAAADOQQAAAAAA4CoAAAgAAAAAAAAAi5wAAJWcAAC5nAAAAwAAABUAAAAAAAAGKp0AADad +AABFnQAACGhGAAAAAAA0bUYAAAAAAKB1RwAAAAAAzM5GAAAAAAAAAAAAAAAAADBnRgAAAAAAANFB +AAAAAAD/KgAAEAAAAAAAAABpnQAAdZ0AAHmdAAACAAAAFQAAAAAAAAaxnQAAvZ0AAAAAAAD4Z0YA +AAAAAPhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZGdGAAAAAABg0kEAAAAAABkrAAAI +AAAAAAAAAMmdAADQnQAA050AAAIAAAAVAAAAAAAABuWdAADwnQAAAAAAADhsRgAAAAAACGhGAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwZ0YAAAAAAMDSQQAAAAAAMCsAABAAAAAAAAAA950A +AAOeAAAHngAAAgAAABUAAAAAAAAGPZ4AAEmeAAAAAAAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAGRnRgAAAAAA4NNBAAAAAABGKwAAIAAAAAAAAABWngAAbJ4AAHCeAAAC +AAAAFQAAAAAAAAbEngAA0J4AAAAAAAAMbkYAAAAAALRuRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAoHBGAAAAAADg1UEAAAAAAGErAAAIAAAAAAAAAOSeAAD0ngAA+J4AAAIAAAAVAAAAAAAA +BiKfAAAunwAAAAAAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwZ0YA +AAAAAMDWQQAAAAAAeisAAAgAAAAAAAAAOJ8AAECfAABEnwAAAgAAABUAAAAAAAAGHVQAAHqfAAAA +AAAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAAAAAAwNdBAAAA +AACUKwAACAAAAAAAAACCnwAAkZ8AAJWfAAACAAAAFQAAAAAAAAa5nwAAxZ8AAAAAAAA4bEYAAAAA +AAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAAAACA2EEAAAAAAK4rAAAIAAAA +AAAAANKfAADdnwAA4J8AAAIAAAAVAAAAAAAABuyfAAD3nwAAAAAAAPhnRgAAAAAA+GdGAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwZ0YAAAAAAODYQQAAAAAAzysAAAgAAAAAAAAA0p8AAN2f +AAD+nwAAAgAAABUAAAAAAAAG7J8AAPefAAAAAAAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAADBnRgAAAAAAQNlBAAAAAADtKwAAAAAAAAAAAAAKoAAAG6AAAEqgAAADAAAA +FQAAAAAAAATQoAAA3KAAAPOgAAAIaEYAAAAAAMhsRgAAAAAAAHBHAAAAAACA90YAAAAAAEDbQQAA +AAAA/isAAAAAAAAAAAAAIqEAAN2fAAApoQAAAgAAABUAAAAAAAAC7J8AADOhAAAAAAAACGhGAAAA +AAAEbUYAAAAAAKDbQQAAAAAAFSwAAAgAAAAAAAAAOqEAAN2fAABBoQAAAgAAABUAAAAAAAAG7J8A +AE2hAAAAAAAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAAAAAA +ANxBAAAAAAAmLAAACAAAAAAAAAA6oQAA3Z8AAFShAAACAAAAFQAAAAAAAAbsnwAATaEAAAAAAAD4 +Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAAAABg3EEAAAAAADYs +AAAAAAAAAAAAAGChAABroQAAdqEAAAMAAAAVAAAAAAAABJShAACfoQAApqEAAAhoRgAAAAAABG1G +AAAAAAAAAAAAAAAAAOCMRgAAAAAAAN1BAAAAAABJLAAACAAAAAAAAACxoQAAvKEAAMChAAACAAAA +FQAAAAAAAAbcoQAA6KEAAAAAAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAMGdGAAAAAADA3UEAAAAAAFksAAAAAAAAAAAAAO+hAAD7oQAAFKIAAAMAAAAVAAAAAAAABFGi +AABdogAAbKIAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAFjPRgAAAAAAwN5BAAAAAACvLAAACAAA +AAAAAACLogAAlqIAALWiAAADAAAAFQAAAAAAAAbvogAA+6IAAAajAAD4Z0YAAAAAAPhnRgAAAAAA +AAAAAAAAAADY20YAAAAAAAAAAAAAAAAALGdGAAAAAACg30EAAAAAAMUsAAAAAAAAAAAAACWjAAAu +owAAMqMAAAIAAAAVAAAAAAAAAgEzAABUowAAAAAAAAhoRgAAAAAABG1GAAAAAABg4EEAAAAAAOEs +AAAQAAAAAAAAAFujAABpowAAbaMAAAIAAAAVAAAAAAAABqmjAAC1owAAAAAAAJxuRgAAAAAAYG5G +AAAAAADgckcAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAACDiQQAAAAAA9CwAAAgAAAAAAAAA +BDUAAMGjAADEowAAAwAAABUAAAAAAAAGODUAAAAAAAD1owAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAA +AAAA4KBGAAAAAAAAAAAAAAAAADBnRgAAAAAAoOJBAAAAAAAKLQAACAAAAAAAAAAOpAAAEaQAABSk +AAADAAAAFQAAAAAAAAZCpAAAAAAAAEWkAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAAwoUYAAAAA +AAAAAAAAAAAAMGdGAAAAAAAg40EAAAAAAB0tAAAIAAAAAAAAAF6kAABmpAAAiKQAAAMAAAAVAAAA +AAAABjClAAA8pQAARKUAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAOTPRgAAAAAAAAAAAAAAAAAw +Z0YAAAAAAODmQQAAAAAAZS0AABAAAAAAAAAAZqUAAHelAAChpQAAAwAAABUAAAAAAAAGol8AADam +AABHpgAAGGhGAAAAAACAb0YAAAAAAAAAAAAAAAAAZMJGAAAAAAAAAAAAAAAAAGRnRgAAAAAAIOlB +AAAAAAB+LQAAMAAAAAAAAABxpgAAgqYAAL2mAAADAAAAFQAAAAAAAAaopwAAtKcAAMmnAABQb0YA +AAAAAORuRgAAAAAAAAAAAAAAAAD0CkcAAAAAAAAAAAAAAAAAtHpGAAAAAABg7kEAAAAAAJwtAAAY +AAAAAAAAAAqoAAAaqAAAHqgAAAMAAAAVAAAAAAAABkSoAABQqAAAV6gAAPhnRgAAAAAAGGtGAAAA +AADgakcAAAAAAJB/RgAAAAAAAAAAAAAAAABsa0YAAAAAAIDvQQAAAAAAsy0AAAAAAAAAAAAAX6gA +AGeoAABuqAAAAwAAABUAAAAAAAAEgqgAAI2oAACUqAAACGhGAAAAAAAEbUYAAAAAAAAAAAAAAAAA +pH9GAAAAAAAg8EEAAAAAANAtAAAYAAAAAAAAAJuoAACmqAAAqagAAAIAAAAVAAAAAAAABrWoAADA +qAAAAAAAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC0Z0YAAAAAAIDw +QQAAAAAA7S0AACAAAAAAAAAAx6gAANGoAADVqAAAAwAAABUAAAAAAAAGBakAAAupAAARqQAA+GdG +AAAAAAD4Z0YAAAAAAAAAAAAAAAAACI1GAAAAAAAAAAAAAAAAAIRrRgAAAAAAQPFBAAAAAAAHLgAA +GAAAAAAAAAAcqQAAKKkAACypAAADAAAAFQAAAAAAAAZsqQAAeKkAAICpAAD4Z0YAAAAAAPhnRgAA +AAAAAAAAAAAAAAC4f0YAAAAAAAAAAAAAAAAA7GdGAAAAAACg8kEAAAAAACcuAAAIAAAAAAAAAIip +AACXqQAAn6kAAAMAAAAVAAAAAAAABjQpAADLqQAA1KkAAAhoRgAAAAAABG1GAAAAAAAAAAAAAAAA +AMx/RgAAAAAAAAAAAAAAAAAwZ0YAAAAAAKDzQQAAAAAASS4AABgAAAAAAAAA3KkAAPOpAAAjqgAA +AwAAABUAAAAAAAAGv6sAAMurAADoqwAAQGhGAAAAAAAMcUYAAAAAAAAAAAAAAAAA4DxHAAAAAAAA +AAAAAAAAAGxrRgAAAAAAgPlBAAAAAADuLgAAEAAAAAAAAAB2rAAAh6wAAKKsAAADAAAAFQAAAAAA +AAYarQAAJq0AADetAAAYaEYAAAAAADhvRgAAAAAAAAAAAAAAAACMs0YAAAAAAAAAAAAAAAAAZGdG +AAAAAACA/EEAAAAAAAQvAAAQAAAAAAAAAFKtAAARpAAAWa0AAAIAAAAVAAAAAAAABmOtAABurQAA +AAAAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAAAD9QQAA +AAAAHi8AAAAAAAAAAAAAda0AAHytAACLrQAAAwAAABUAAAAAAAAEqK0AALOtAAC6rQAA+GdGAAAA +AAD4Z0YAAAAAAAAAAAAAAAAAgKFGAAAAAABg/UEAAAAAAD4vAAAYAAAAAAAAAMmtAADQrQAA360A +AAMAAAAVAAAAAAAABv6tAAAJrgAAEK4AAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAANChRgAAAAAA +AAAAAAAAAAC0Z0YAAAAAAOD9QQAAAAAAWi8AABgAAAAAAAAAH64AADCuAAA5rgAAAwAAABUAAAAA +AAAG464AAO+uAAABrwAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAArORGAAAAAAAAAAAAAAAAALRn +RgAAAAAAYAFCAAAAAACbLwAAEAAAAAAAAABCrwAATq8AAFKvAAACAAAAFQAAAAAAAAZ5rwAAha8A +AAAAAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZGdGAAAAAADgAkIA +AAAAALcvAAAQAAAAAAAAAI6vAACarwAAnq8AAAIAAAAVAAAAAAAABrqvAADGrwAAAAAAAPhnRgAA +AAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAAAAEQgAAAAAA0y8AABAA +AAAAAAAAza8AAOGvAAAHsAAAAwAAABUAAAAAAAAGhrAAAJKwAACksAAAhHRGAAAAAADEdkYAAAAA +AAAAAAAAAAAAOANHAAAAAAAAAAAAAAAAAGRnRgAAAAAAIAZCAAAAAAD+LwAAEAAAAAAAAADbsAAA +5rAAAPewAAADAAAAFQAAAAAAAAYgsQAALLEAADuxAABscUYAAAAAADBxRgAAAAAAAAAAAAAAAAAg +okYAAAAAAAAAAAAAAAAAZGdGAAAAAADgBkIAAAAAABcwAAAYAAAAAAAAAEyxAABYsQAAfbEAAAMA +AAAVAAAAAAAABs2xAADZsQAA5bEAAAhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAATxRgAAAAAAAAAA +AAAAAADsZ0YAAAAAACAIQgAAAAAAKzAAAAgAAAAAAAAACrIAACeyAABEsgAAAwAAABUAAAAAAAAG +57IAAPOyAAACswAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAKARHAAAAAAAAAAAAAAAAADBnRgAA +AAAAQAtCAAAAAABfMAAACAAAAAAAAAB2KAAAYbMAAGSzAAACAAAAFQAAAAAAAAaKKAAAbLMAAAAA +AAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAAAACAC0IAAAAA +AHQwAAAAAAAAAAAAAHOzAAB+swAAj7MAAAMAAAAVAAAAAAAABLyzAADIswAA0LMAAPhnRgAAAAAA ++GdGAAAAAAAAAAAAAAAAAHCiRgAAAAAAQAxCAAAAAACSMAAAAAAAAAAAAADhswAA7bMAAPyzAAAD +AAAAFQAAAAAAAAQrtAAAN7QAAEa0AAAIaEYAAAAAAARtRgAAAAAAAAAAAAAAAADAokYAAAAAAAAN +QgAAAAAArDAAABgAAAAAAAAAVbQAAGK0AABmtAAAAgAAABUAAAAAAAAGiLQAAJS0AAAAAAAA1G9G +AAAAAAAYaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOxnRgAAAAAAgA5CAAAAAADGMAAA +GAAAAAAAAACdtAAArbQAALq0AAADAAAAFQAAAAAAAAYutQAAOrUAAEi1AACMb0YAAAAAALRuRgAA +AAAAAAAAAAAAAAB43EYAAAAAAAAAAAAAAAAA7GdGAAAAAADgEEIAAAAAAOAwAAAgAAAAAAAAAHe1 +AACPtQAAk7UAAAMAAAAVAAAAAAAABpC2AACctgAAu7YAAKxwRgAAAAAAJHFGAAAAAAAAAAAAAAAA +ABAcRwAAAAAAAAAAAAAAAAAQcEYAAAAAAMAWQgAAAAAAGDEAABgAAAAAAAAAKLcAADm3AAA9twAA +AwAAABUAAAAAAAAG47cAAO+3AAANuAAAjG9GAAAAAAC0bkYAAAAAAAAAAAAAAAAAbCRHAAAAAAAA +AAAAAAAAAOxnRgAAAAAAQBpCAAAAAAA4MQAAEAAAAAAAAABsuAAAf7gAAIa4AAADAAAAFQAAAAAA +AAYrtAAAorgAAK24AAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAADgf0YAAAAAAAAAAAAAAAAAzGdG +AAAAAAAAG0IAAAAAAFwxAAAQAAAAAAAAALS4AADSuAAA27gAAAMAAAAVAAAAAAAABq26AAC5ugAA +27oAADR4RgAAAAAAZHlGAAAAAAAga0cAAAAAAKQ3RwAAAAAAAAAAAAAAAABkZ0YAAAAAAAAnQgAA +AAAA8DEAABAAAAAAAAAAkbsAAJ67AAC2uwAAAwAAABUAAAAAAAAG8LsAAPy7AAAGvAAACGhGAAAA +AAAEbUYAAAAAAAAAAAAAAAAAcNBGAAAAAAAAAAAAAAAAAJxnRgAAAAAAoChCAAAAAAAqMgAAEAAA +AAAAAAAkvAAAObwAAEK8AAADAAAAFQAAAAAAAAbCvAAAzrwAAN28AAA4bEYAAAAAAAhoRgAAAAAA +AAAAAAAAAABg5UYAAAAAAAAAAAAAAAAAZGdGAAAAAABgK0IAAAAAAEUyAAAYAAAAAAAAABK9AAAj +vQAATr0AAAMAAAAVAAAAAAAABhu+AAAnvgAAP74AADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAGhL +RwAAAAAAAAAAAAAAAADsZ0YAAAAAAEAuQgAAAAAAzjIAACAAAAAAAAAAxr4AANC+AADUvgAAAwAA +ABUAAAAAAAAGKb8AAC+/AAA1vwAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAXJdGAAAAAAAAAAAA +AAAAAChwRgAAAAAAoC9CAAAAAAAUMwAACAAAAAAAAABQvwAAYL8AAG+/AAADAAAAFQAAAAAAAAaY +vwAApL8AALG/AAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAwjUYAAAAAAAAAAAAAAAAAMGdGAAAA +AAAgMUIAAAAAADEzAAAYAAAAAAAAAMC/AADRvwAA+78AAAMAAAAVAAAAAAAABo/AAACbwAAAusAA +AIxvRgAAAAAA4G9GAAAAAADgdUcAAAAAAPgLRwAAAAAAAAAAAAAAAADsZ0YAAAAAACA1QgAAAAAA +cTMAABgAAAAAAAAA/sAAAAXBAAAIwQAAAgAAABUAAAAAAAAGwhEAABbBAAAAAAAACGhGAAAAAAAE +bUYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABR3RgAAAAAAoDVCAAAAAABOMwAAGAAAAAAA +AAAfwQAAKMEAADfBAAADAAAAFQAAAAAAAAZZwQAAXsEAAGPBAAD4Z0YAAAAAAPhnRgAAAAAAAAAA +AAAAAAAQo0YAAAAAAAAAAAAAAAAA7GdGAAAAAAAgNkIAAAAAAJQzAAAQAAAAAAAAAHTBAACDwQAA +h8EAAAIAAAAVAAAAAAAABqLBAACuwQAAAAAAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAABkZ0YAAAAAAOA2QgAAAAAArzMAABAAAAAAAAAAtcEAAMHBAADNwQAAAwAAABUA +AAAAAAAGDMIAABjCAAAgwgAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAA9H9GAAAAAAAAAAAAAAAA +AGRnRgAAAAAAIDhCAAAAAADiMwAAEAAAAAAAAAAswgAAQsIAAHnCAAADAAAAFQAAAAAAAAbfLQAA +/cIAAAnDAAAMbkYAAAAAABhoRgAAAAAAAAAAAAAAAAAYBUcAAAAAAAAAAAAAAAAAZGdGAAAAAABA +OkIAAAAAAP0zAAAIAAAAAAAAAFzDAAB5wwAAusMAAAMAAAAVAAAAAAAABljEAABkxAAAdsQAAIxv +RgAAAAAAmG9GAAAAAAAAAAAAAAAAAAgzRwAAAAAAAAAAAAAAAAAwZ0YAAAAAACA9QgAAAAAAWTQA +ABgAAAAAAAAAzcQAAN/EAADjxAAAAwAAABUAAAAAAAAGIBgAAArFAAAQxQAAOGxGAAAAAAAIaEYA +AAAAAAAAAAAAAAAACIBGAAAAAAAAAAAAAAAAAOxnRgAAAAAAQD5CAAAAAACONAAACAAAAAAAAABW +BQAAGMUAABvFAAABAAAAFQAAAAAAAAYmxQAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAADBnRgAAAAAAYD5CAAAAAACpNAAAGAAAAAAAAAApxQAAO8UAAD/FAAADAAAAFQAA +AAAAAAZnxQAAbcUAAHPFAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAcgEYAAAAAAAAAAAAAAAAA +7GdGAAAAAACAP0IAAAAAAMg0AAAIAAAAAAAAAFYFAAAYxQAAe8UAAAEAAAAVAAAAAAAABibFAAA4 +bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAAAACgP0IAAAAAAOU0 +AAAYAAAAAAAAAIfFAACcxQAAoMUAAAIAAAAVAAAAAAAABurFAAD2xQAAAAAAADhsRgAAAAAACGhG +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADsZ0YAAAAAAOBBQgAAAAAABTUAAAgAAAAAAAAA +BMYAAAjGAAAuxgAAAwAAABUAAAAAAAAGyMYAAAAAAADMxgAAOGxGAAAAAAAIaEYAAAAAAAAAAAAA +AAAAmJdGAAAAAAAAAAAAAAAAADBnRgAAAAAAwENCAAAAAAAlNQAAGAAAAAAAAADsxgAA+8YAAP/G +AAADAAAAFQAAAAAAAAYdxwAAKccAADDHAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAwgEYAAAAA +AAAAAAAAAAAA7GdGAAAAAACAREIAAAAAAFw1AAAYAAAAAAAAADvHAAA/xwAASccAAAMAAAAVAAAA +AAAABovHAAAAAAAAj8cAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAFiNRgAAAAAAAAAAAAAAAADs +Z0YAAAAAAIBFQgAAAAAAfTUAABgAAAAAAAAAoscAAKbHAADHxwAAAwAAABUAAAAAAAAGF8gAAAAA +AAAbyAAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAgI1GAAAAAAAAAAAAAAAAAOxnRgAAAAAAwEZC +AAAAAACeNQAAGAAAAAAAAAAyyAAAOcgAADzIAAADAAAAFQAAAAAAAAbCEQAAUMgAAFnIAAA4bEYA +AAAAAAhoRgAAAAAAAAAAAAAAAABEgEYAAAAAAAAAAAAAAAAA7GdGAAAAAABAR0IAAAAAAOA1AAAI +AAAAAAAAAGTIAABryAAAbsgAAAMAAAAVAAAAAAAABn7IAACJyAAAksgAADhsRgAAAAAACGhGAAAA +AAAAAAAAAAAAAFiARgAAAAAAAAAAAAAAAAAwZ0YAAAAAAKBHQgAAAAAAHjYAABAAAAAAAAAAmcgA +AKTIAACpyAAAAgAAABUAAAAAAAAGIAgAAMnIAAAAAAAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAGRnRgAAAAAAYEhCAAAAAAAwNgAACAAAAAAAAADQyAAA28gAAOPIAAAD +AAAAFQAAAAAAAAYZBgAA9cgAAPzIAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAABsgEYAAAAAAAAA +AAAAAAAAMGdGAAAAAADASEIAAAAAAEU2AAAIAAAAAAAAAAPJAAAOyQAAFskAAAMAAAAVAAAAAAAA +BrWoAAAoyQAAL8kAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAICARgAAAAAAAAAAAAAAAAAwZ0YA +AAAAACBJQgAAAAAAWjYAADAAAAAAAAAANskAAFPJAABiyQAAAwAAABUAAAAAAAAGAMoAAAzKAAAg +ygAAEG1GAAAAAAAIaEYAAAAAAAAAAAAAAAAAYKNGAAAAAAAAAAAAAAAAADR7RgAAAAAAgExCAAAA +AACCNgAAMAAAAAAAAAA7ygAAPsoAAELKAAABAAAAFQAAAAAAAAZUygAAmGxGAAAAAAAIaEYAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKx8RgAAAAAAwExCAAAAAACSNgAAAAAAAAAAAABZygAA +YMoAAHLKAAADAAAAFQAAAAAAAATCEQAAkcoAAJjKAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAACw +o0YAAAAAAEBNQgAAAAAAqjYAAAAAAAAAAAAAqcoAALDKAADCygAAAwAAABUAAAAAAAAEGQAAAOPK +AADuygAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAKRGAAAAAACgTUIAAAAAAL42AAAAAAAAAAAA +AP/KAAAHywAADMsAAAMAAAAVAAAAAAAABDTLAABAywAASMsAAPhnRgAAAAAA+GdGAAAAAAAAAAAA +AAAAAJSARgAAAAAAgE5CAAAAAAD2NgAAEAAAAAAAAABTywAAY8sAAHXLAAADAAAAFQAAAAAAAAam +ywAAsssAALzLAAA4bEYAAAAAACxsRgAAAAAA4HBHAAAAAABQpEYAAAAAAAAAAAAAAAAAZGdGAAAA +AABAUEIAAAAAAAs3AAAAAAAAAAAAAOR0AADNywAA0csAAAIAAAAVAAAAAAAAAkARAAD4dAAAAAAA +APhnRgAAAAAA+GdGAAAAAACAUEIAAAAAACY3AAAQAAAAAAAAANvLAADjywAA9csAAAMAAAAVAAAA +AAAABhzMAAAozAAAL8wAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAKCkRgAAAAAAAAAAAAAAAABk +Z0YAAAAAAEBRQgAAAAAAOTcAABAAAAAAAAAAQMwAAEjMAABUzAAAAwAAABUAAAAAAAAGiMwAAJTM +AACczAAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAqI1GAAAAAAAAAAAAAAAAAGRnRgAAAAAAQFJC +AAAAAABhNwAAIAAAAAAAAAC1zAAAxcwAANrMAAADAAAAFQAAAAAAAAYTzQAAH80AACzNAAD4Z0YA +AAAAAPhnRgAAAAAAAAAAAAAAAADws0YAAAAAAAAAAAAAAAAAhGtGAAAAAABAVEIAAAAAAIk3AAAY +AAAAAAAAAEDNAABKzQAAYc0AAAMAAAAVAAAAAAAABp/NAACrzQAAwc0AAOR5RgAAAAAAFHpGAAAA +AAAgdEcAAAAAAFS0RgAAAAAAAAAAAAAAAADsZ0YAAAAAAIBWQgAAAAAArjcAAAAAAAAAAAAA180A +AN7NAADmzQAAAwAAABUAAAAAAAAE9M0AAP/NAAAGzgAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAA +qIBGAAAAAADAVkIAAAAAAMc3AAAQAAAAAAAAAA3OAAAXzgAAKs4AAAMAAAAVAAAAAAAABrGdAABV +zgAAY84AANR0RgAAAAAAFHVGAAAAAADAdEcAAAAAAPCkRgAAAAAAAAAAAAAAAABkZ0YAAAAAACBY +QgAAAAAA2TcAAAAAAAAAAAAA180AAN7NAAB1zgAAAwAAABUAAAAAAAAE9M0AAP/NAAAGzgAA+GdG +AAAAAAD4Z0YAAAAAAAAAAAAAAAAAvIBGAAAAAABgWEIAAAAAAPE3AAAAAAAAAAAAAIPOAACLzgAA +nc4AAAMAAAAVAAAAAAAABLmfAADGzgAA0c4AAAhoRgAAAAAABG1GAAAAAAAAAAAAAAAAAEClRgAA +AAAAIFlCAAAAAAABOAAAEAAAAAAAAADizgAA7c4AAPHOAAACAAAAFQAAAAAAAAZaXgAA/M4AAAAA +AAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZGdGAAAAAACgWUIAAAAA +ABc4AAAgAAAAAAAAAAPPAAASzwAAF88AAAMAAAAVAAAAAAAABtBaAABXzwAAX88AAPhnRgAAAAAA ++GdGAAAAAAAAAAAAAAAAANzCRgAAAAAAAAAAAAAAAAAghkYAAAAAAIBaQgAAAAAAMjgAABgAAAAA +AAAAhs8AAJXPAACazwAAAwAAABUAAAAAAAAGsM8AALvPAADCzwAA+GdGAAAAAAD4Z0YAAAAAAAAA +AAAAAAAA0I1GAAAAAAAAAAAAAAAAACR3RgAAAAAAIFtCAAAAAABXOAAAEAAAAAAAAADNzwAA1M8A +ANnPAAACAAAAFQAAAAAAAAbtzwAA+c8AAAAAAAB8bUYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAZGdGAAAAAADgW0IAAAAAAHI4AAAQAAAAAAAAAADQAAAS0AAAF9AAAAMAAAAV +AAAAAAAABkzQAABS0AAAWNAAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAJClRgAAAAAAAAAAAAAA +AABkZ0YAAAAAAOBcQgAAAAAArDgAABAAAAAAAAAAa9AAAILQAACH0AAAAwAAABUAAAAAAAAGRKgA +AK/QAAC60AAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAA1JdGAAAAAAAAAAAAAAAAAGRnRgAAAAAA +AF5CAAAAAADXOAAAGAAAAAAAAADF0AAA1tAAANvQAAADAAAAFQAAAAAAAAaU0QAAoNEAALvRAACM +b0YAAAAAALRuRgAAAAAAAAAAAAAAAAD80EYAAAAAAAAAAAAAAAAAFHdGAAAAAABgY0IAAAAAAAc5 +AAAQAAAAAAAAAO3RAADw0QAA9NEAAAMAAAAVAAAAAAAABiLSAAAAAAAAJdIAADhsRgAAAAAACGhG +AAAAAAAAAAAAAAAAAOClRgAAAAAAAAAAAAAAAABkZ0YAAAAAAOBjQgAAAAAAPDkAABAAAAAAAAAA +ONIAAE/SAABU0gAAAwAAABUAAAAAAAAG3tIAAOrSAAD/0gAAOGxGAAAAAAAIaEYAAAAAAAAAAAAA +AAAA/AxHAAAAAAAAAAAAAAAAAGRnRgAAAAAAIGZCAAAAAABlOQAAEAAAAAAAAAAv0wAAO9MAAEDT +AAACAAAAFQAAAAAAAAZa0wAAZtMAAAAAAAB8bUYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAZGdGAAAAAAAAZ0IAAAAAAIU5AAAQAAAAAAAAAHPTAAB90wAAntMAAAMAAAAVAAAA +AAAABgXUAAAR1AAAJtQAABxzRgAAAAAAqHFGAAAAAAAAAAAAAAAAABjdRgAAAAAAAAAAAAAAAABk +Z0YAAAAAAKBoQgAAAAAAnTkAAAgAAAAAAAAASNQAAFjUAABi1AAAAwAAABUAAAAAAAAGjxEAANHU +AADZ1AAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAFOZGAAAAAAAAAAAAAAAAADBnRgAAAAAA4GlC +AAAAAABiOgAACAAAAAAAAAAC1QAADtUAABfVAAADAAAAFQAAAAAAAAZe1QAAatUAAHXVAAA4bEYA +AAAAAAhoRgAAAAAAAAAAAAAAAABUw0YAAAAAAAAAAAAAAAAAMGdGAAAAAAAga0IAAAAAAJo6AAAI +AAAAAAAAAJLVAACd1QAArdUAAAMAAAAVAAAAAAAABs/VAADa1QAA4dUAADhsRgAAAAAACGhGAAAA +AAAAAAAAAAAAAPiNRgAAAAAAAAAAAAAAAAAwZ0YAAAAAAMBrQgAAAAAAvToAAAgAAAAAAAAA8NUA +APvVAAAA1gAAAwAAABUAAAAAAAAG3KEAABbWAAAd1gAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAA +0IBGAAAAAAAAAAAAAAAAADBnRgAAAAAAgGxCAAAAAADeOgAAAAAAAAAAAACdgwAAJNYAACjWAAAB +AAAAFQAAAAAAAAKmgwAA+GdGAAAAAAD4Z0YAAAAAAKBsQgAAAAAA7ToAABAAAAAAAAAALNYAADnW +AAA+1gAAAgAAABUAAAAAAAAGUtYAAFjWAAAAAAAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAGRnRgAAAAAAYG1CAAAAAAAHOwAACAAAAAAAAABd1gAAadYAAHTWAAADAAAA +FQAAAAAAAAaU1gAAoNYAAKnWAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAgjkYAAAAAAAAAAAAA +AAAAMGdGAAAAAABgbkIAAAAAAC47AAAIAAAAAAAAALPWAAC+1gAAyNYAAAMAAAAVAAAAAAAABuXW +AADx1gAA/NYAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAEiORgAAAAAAAAAAAAAAAAAwZ0YAAAAA +ACBvQgAAAAAAVTsAAAgAAAAAAAAABdcAABDXAAAU1wAAAgAAABUAAAAAAAAGkiYAACjXAAAAAAAA +OGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAAAAAAoG9CAAAAAABs +OwAAEAAAAAAAAAAv1wAAPtcAAEPXAAADAAAAFQAAAAAAAAZj1wAAadcAAG7XAAA4bEYAAAAAAAho +RgAAAAAAAAAAAAAAAABwjkYAAAAAAAAAAAAAAAAAZGdGAAAAAABgcEIAAAAAAJg7AAAIAAAAAAAA +AHnXAACN1wAAs9cAAAMAAAAVAAAAAAAABk3YAABZ2AAAa9gAADhsRgAAAAAACGhGAAAAAAAAAAAA +AAAAAMjmRgAAAAAAAAAAAAAAAAAwZ0YAAAAAACBzQgAAAAAArDsAAAAAAAAAAAAAmtgAAKbYAACr +2AAAAgAAABUAAAAAAAACGF4AAMbYAAAAAAAA+GdGAAAAAAD4Z0YAAAAAAEB0QgAAAAAAxDsAAAAA +AAAAAAAAztgAANXYAADl2AAAAwAAABUAAAAAAAAEqggAAAXZAAAQ2QAA+GdGAAAAAAD4Z0YAAAAA +AAAAAAAAAAAAMKZGAAAAAADAdEIAAAAAAN87AAAYAAAAAAAAAB/ZAAAj2QAAQ9kAAAMAAAAVAAAA +AAAABqPZAAAAAAAAp9kAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAMzDRgAAAAAAAAAAAAAAAADE +Z0YAAAAAAMB1QgAAAAAACzwAAAAAAAAAAAAA2tkAAOvZAADw2QAAAgAAABUAAAAAAAACJ9oAADPa +AAAAAAAA+GdGAAAAAAD4Z0YAAAAAAMB3QgAAAAAAHzwAAAAAAAAAAAAAO9oAAEraAABW2gAAAwAA +ABUAAAAAAAAEeNoAAITaAACP2gAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAA5IBGAAAAAADAeEIA +AAAAAEI8AAAIAAAAAAAAAJraAAC42gAAvdoAAAIAAAAVAAAAAAAABjvbAABH2wAAAAAAAPhnRgAA +AAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwZ0YAAAAAAAB8QgAAAAAAUjwAABgA +AAAAAAAAUNsAAF/bAABr2wAAAwAAABUAAAAAAAAG7AwAAIjbAACO2wAAOGxGAAAAAAAIaEYAAAAA +AAAAAAAAAAAA+IBGAAAAAAAAAAAAAAAAALxnRgAAAAAA4HxCAAAAAACBPAAAEAAAAAAAAACZ2wAA +qNsAAK3bAAACAAAAFQAAAAAAAAa82wAAwtsAAAAAAAA4bEYAAAAAACxsRgAAAAAAAG9HAAAAAAAA +AAAAAAAAAAAAAAAAAAAARGdGAAAAAACgfUIAAAAAAJU8AAAAAAAAAAAAAMvbAADS2wAA19sAAAIA +AAAVAAAAAAAAApShAADh2wAAAAAAAAhoRgAAAAAABG1GAAAAAABAfkIAAAAAAK88AAAAAAAAAAAA +AOrbAAAG3AAAC9wAAAIAAAAVAAAAAAAAAi3cAAA53AAAAAAAAPhnRgAAAAAA+GdGAAAAAABAf0IA +AAAAAMQ8AAAIAAAAAAAAAEHcAABS3AAAZNwAAAMAAAAVAAAAAAAABpfcAACj3AAAsdwAAIxvRgAA +AAAAtG5GAAAAAAAAAAAAAAAAAICmRgAAAAAAAAAAAAAAAAAwZ0YAAAAAAACBQgAAAAAA+TwAABAA +AAAAAAAAx9wAAPDcAAD93AAAAwAAABUAAAAAAAAGVd0AAGHdAAB33QAAdG9GAAAAAAAsb0YAAAAA +AAAAAAAAAAAAmI5GAAAAAAAAAAAAAAAAAFRnRgAAAAAAYINCAAAAAAAcPQAAGAAAAAAAAACD3QAA +j90AAJTdAAACAAAAFQAAAAAAAAa43QAAxN0AAAAAAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAqGtGAAAAAACAhEIAAAAAACw9AAAAAAAAAAAAAM7dAADw3QAA9d0AAAIA +AAAVAAAAAAAAAjllAAAb3gAAAAAAAPhnRgAAAAAA+GdGAAAAAADAhUIAAAAAAEQ9AAAAAAAAAAAA +ACfeAAAu3gAAPt4AAAMAAAAVAAAAAAAABF7eAABp3gAAcN4AAPhnRgAAAAAA+GdGAAAAAAAAAAAA +AAAAABCYRgAAAAAAQIZCAAAAAABlPQAAGAAAAAAAAAB/3gAAld4AAJreAAACAAAAFQAAAAAAAAa8 +3gAAyN4AAAAAAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAqGtGAAAA +AADAh0IAAAAAAHs9AAAIAAAAAAAAANbeAADd3gAA4d4AAAIAAAAVAAAAAAAABvAHAADv3gAAAAAA +ADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwZ0YAAAAAAECIQgAAAAAA +jD0AAAAAAAAAAAAA9t4AAP3eAAAB3wAAAgAAABUAAAAAAAACQAgAAA3fAAAAAAAA+GdGAAAAAAD4 +Z0YAAAAAAKCIQgAAAAAAmj0AABAAAAAAAAAAFN8AAB3fAAAz3wAAAwAAABUAAAAAAAAGWd8AAF/f +AABl3wAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAuLRGAAAAAAAAAAAAAAAAAFRnRgAAAAAAYIlC +AAAAAAC8PQAACAAAAAAAAAB63wAAid8AAI3fAAACAAAAFQAAAAAAAAah3wAApt8AAAAAAAD4Z0YA +AAAAAPhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKGdGAAAAAADgiUIAAAAAANA9AAAY +AAAAAAAAAK/fAAC23wAAut8AAAIAAAAVAAAAAAAABsbfAADL3wAAAAAAAExtRgAAAAAACGhGAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADcZ0YAAAAAAGCKQgAAAAAA5T0AABAAAAAAAAAA0t8A +ANLbAADZ3wAAAgAAABUAAAAAAAAG498AAO7fAAAAAAAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAGRnRgAAAAAAAItCAAAAAAD1PQAAGAAAAAAAAAD33wAAAuAAAArgAAAD +AAAAFQAAAAAAAAa5nwAAIeAAACzgAAAIbEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAMgUYAAAAAAAAA +AAAAAAAAkGtGAAAAAADAi0IAAAAAABs+AAAQAAAAAAAAADPgAAA+4AAAQuAAAAIAAAAVAAAAAAAA +Bk/gAABa4AAAAAAAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACsZ0YA +AAAAAECMQgAAAAAALz4AABAAAAAAAAAAYeAAAGjgAABt4AAAAgAAABUAAAAAAAAGaAIAAHngAAAA +AAAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRnRgAAAAAA4IxCAAAA +AABEPgAAEAAAAAAAAABh4AAAaOAAAIDgAAACAAAAFQAAAAAAAAZoAgAAeeAAAAAAAAD4Z0YAAAAA +APhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZGdGAAAAAACAjUIAAAAAAFo+AAAQAAAA +AAAAAGHgAABo4AAAjOAAAAIAAAAVAAAAAAAABmgCAAB54AAAAAAAAPhnRgAAAAAA+GdGAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAACCOQgAAAAAAcz4AABAAAAAAAAAAYeAAAGjg +AACY4AAAAgAAABUAAAAAAAAGaAIAAHngAAAAAAAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAGRnRgAAAAAAwI5CAAAAAACNPgAAEAAAAAAAAABh4AAAaOAAAKTgAAACAAAA +FQAAAAAAAAZoAgAAeeAAAAAAAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAZGdGAAAAAABgj0IAAAAAAKY+AAAQAAAAAAAAAGHgAABo4AAAsOAAAAIAAAAVAAAAAAAABmgC +AAB54AAAAAAAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAA +AACQQgAAAAAAwD4AABAAAAAAAAAAYeAAAGjgAAC84AAAAgAAABUAAAAAAAAGaAIAAHngAAAAAAAA ++GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRnRgAAAAAAoJBCAAAAAADW +PgAAEAAAAAAAAABh4AAAaOAAAMjgAAACAAAAFQAAAAAAAAZoAgAAeeAAAAAAAAD4Z0YAAAAAAPhn +RgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZGdGAAAAAABAkUIAAAAAAO0+AAAQAAAAAAAA +AGHgAABo4AAA1OAAAAIAAAAVAAAAAAAABmgCAAB54AAAAAAAAPhnRgAAAAAA+GdGAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAAOCRQgAAAAAABz8AABAAAAAAAAAAYeAAAGjgAADg +4AAAAgAAABUAAAAAAAAGaAIAAHngAAAAAAAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAGRnRgAAAAAAgJJCAAAAAAAiPwAAAAAAAAAAAADs4AAA8+AAAPfgAAACAAAAFQAA +AAAAAAIAegAAAeEAAAAAAAD4Z0YAAAAAAPhnRgAAAAAA4JJCAAAAAAA1PwAAAAAAAAAAAAAI4QAA +D+EAABPhAAACAAAAFQAAAAAAAAId4QAAKOEAAAAAAAD4Z0YAAAAAAPhnRgAAAAAAQJNCAAAAAABJ +PwAAAAAAAAAAAAAv4QAAQOEAAGLhAAADAAAAFQAAAAAAAAS84QAAyOEAANHhAAD4Z0YAAAAAAPhn +RgAAAAAAAAAAAAAAAABExEYAAAAAAKCVQgAAAAAAij8AAAAAAAAAAAAA1HUAAAPiAAAH4gAAAQAA +ABUAAAAAAAADEnYAAPhnRgAAAAAAPGtGAAAAAACgcEcAAAAAACCWQgAAAAAAmT8AAAgAAAAAAAAA +FeIAACLiAAAr4gAAAwAAABUAAAAAAAAGZeIAAGviAAB24gAAGGhGAAAAAABsbkYAAAAAAGB5RwAA +AAAAwI5GAAAAAAAAAAAAAAAAAChnRgAAAAAA4JdCAAAAAACqPwAAAAAAAAAAAACG4gAAkuIAAK7i +AAADAAAAFQAAAAAAAARpJwAA6uIAAPbiAAAIaEYAAAAAAARtRgAAAAAAAAAAAAAAAAActUYAAAAA +AECZQgAAAAAAwT8AAAAAAAAAAAAAGeMAACnjAAA64wAAAwAAABUAAAAAAAAEkkAAAG3jAAB74wAA +GGhGAAAAAAAgb0YAAAAAAAAAAAAAAAAA0KZGAAAAAADgmkIAAAAAANg/AAAIAAAAAAAAAIvjAACh +4wAAquMAAAMAAAAVAAAAAAAABvLjAAD44wAABeQAAIxvRgAAAAAAGG5GAAAAAACAbkcAAAAAAOiO +RgAAAAAAAAAAAAAAAAAwZ0YAAAAAAOCcQgAAAAAA6j8AAAAAAAAAAAAAFeQAACHkAAAy5AAAAwAA +ABUAAAAAAAAEc+QAAH/kAACL5AAACGhGAAAAAACIbUYAAAAAAAAAAAAAAAAAIKdGAAAAAABgnkIA +AAAAAAJAAAAAAAAAAAAAAAcnAACb5AAAn+QAAAIAAAAVAAAAAAAAAhknAAAkJwAAAAAAAPhnRgAA +AAAA+GdGAAAAAACgnkIAAAAAABlAAAAAAAAAAAAAAAcnAACb5AAAp+QAAAIAAAAVAAAAAAAAAhkn +AAAkJwAAAAAAAPhnRgAAAAAA+GdGAAAAAADgnkIAAAAAAC1AAAAAAAAAAAAAAK/kAADK5AAA1+QA +AAMAAAAVAAAAFgAABDDlAAA25QAAR+UAABhoRgAAAAAAVG5GAAAAAAAAAAAAAAAAAHCnRgAAAAAA +YKBCAAAAAABTQAAACAAAAC8BAABZ5QAAaeUAAG7lAAACAAAAFQAAAAAAAAaY5QAAqOUAAAAAAABs +cUYAAAAAAIxyRgAAAAAAAAAAAAAAAAAAAAAAAAAAAIxnRgAAAAAAMGdGAAAAAADAoUIAAAAAAGpA +AAAIAAAAAAAAALTlAADA5QAAxeUAAAIAAAAVAAAAAAAABuPlAADv5QAAAAAAADhsRgAAAAAACGhG +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwZ0YAAAAAAMCiQgAAAAAAfkAAABgAAAAAAAAA ++OUAAP/lAAAE5gAAAgAAABUAAAAAAAAGuZ8AABjmAAAAAAAAgGxGAAAAAABQbEYAAAAAAMBrRwAA +AAAAAAAAAAAAAAAAAAAAAAAAAOxnRgAAAAAAgKNCAAAAAACbQAAAAAAAAAAAAAAf5gAAKOYAACzm +AAACAAAAFQAAAAAAAAP/hwAAOOYAAAAAAAD4Z0YAAAAAADBrRgAAAAAAoGtHAAAAAAAApEIAAAAA +AL5AAAAQAAAAAAAAAD/mAABi5gAAh+YAAAMAAAAVAAAAAAAABi3nAAA55wAAS+cAAPR1RgAAAAAA +JHZGAAAAAAAAAAAAAAAAAIC1RgAAAAAAAAAAAAAAAABkZ0YAAAAAAACnQgAAAAAADUEAABAAAAAA +AAAAgecAAJLnAADT5wAAAwAAABUAAAAWAAAG/OkAAAjqAAAi6gAANHVGAAAAAADkdUYAAAAAAAAA +AAAAAAAAvEJHAAAAAAAAAAAAAAAAAGRnRgAAAAAAYK9CAAAAAAA5QQAAEAAAAAAAAADu6gAA9uoA +APvqAAACAAAAFQAAABYAAAYX6wAAI+sAAAAAAAAMbkYAAAAAABhoRgAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAZGdGAAAAAABAsEIAAAAAAE9BAAAQAAAAAAAAACzrAAA96wAAQusAAAMAAAAV +AAAACQAABnTsAACA7AAArewAAIR8RgAAAAAAyIZGAAAAAABgcEcAAAAAACCBRgAAAAAAAAAAAAAA +AACsZ0YAAAAAAIC3QgAAAAAAX0EAAAAAAAAAAAAAigUAALbsAAC67AAAAQAAABUAAAAAAAAClAUA +APhnRgAAAAAA+GdGAAAAAACgt0IAAAAAAG9BAAAIAAAAAAAAAL7sAADB7AAAxewAAAEAAAAVAAAA +AAAABtHsAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAAAADg +t0IAAAAAAIFBAAAQAAAAAAAAANTsAADb7AAA4OwAAAIAAAAVAAAAAAAABvDsAAD17AAAAAAAADhs +RgAAAAAALGxGAAAAAAAAdEcAAAAAAAAAAAAAAAAAAAAAAAAAAACsZ0YAAAAAAIC4QgAAAAAAj0EA +AAAAAAAAAAAA+uwAAAPiAAAB7QAAAgAAABUAAAAAAAACEnYAAAvtAAAAAAAACGhGAAAAAAAEbUYA +AAAAAAC5QgAAAAAAo0EAAAgAAAAAAAAAFO0AACDtAAAl7QAAAgAAABUAAAAAAAAGOWUAAEbtAAAA +AAAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAAAAAAQLpCAAAA +AAC0QQAAAAAAAAAAAABO7QAAVe0AAFntAAACAAAAFQAAAAAAAANl7QAAau0AAAAAAAD4Z0YAAAAA +AEhrRgAAAAAAQG5HAAAAAADAukIAAAAAAMdBAAAAAAAAAAAAAG/tAAB27QAAeu0AAAIAAAAVAAAA +AAAAAvIRAACK7QAAAAAAAAhoRgAAAAAABG1GAAAAAABAu0IAAAAAAOBBAAAIAAAAAAAAAJPtAACd +7QAApu0AAAMAAAAVAAAAAAAABrztAADC7QAAy+0AADhsRgAAAAAAFGxGAAAAAAAgbkcAAAAAADSB +RgAAAAAAAAAAAAAAAAAwZ0YAAAAAAAC8QgAAAAAAAUIAAAAAAAAAAAAA0+0AAN3tAADi7QAAAgAA +ABUAAAAAAAACuicAAPLtAAAAAAAAQGhGAAAAAACMckYAAAAAAMC8QgAAAAAAGkIAAAAAAAAAAAAA +/e0AAA3uAAAS7gAAAgAAABUAAAAAAAACtncAAEruAAAAAAAACGhGAAAAAAAEbUYAAAAAAGC+QgAA +AAAAL0IAABgAAAAAAAAAVO4AAF7uAACT7gAAAwAAABUAAAAAAAAGG+8AACfvAABA7wAAjG9GAAAA +AAC0bkYAAAAAAAAAAAAAAAAAuN1GAAAAAAAAAAAAAAAAAOxnRgAAAAAAYMFCAAAAAABlQgAAGAAA +AAAAAAB07wAAh+8AAJPvAAADAAAAFQAAAAAAAAa37wAAw+8AAMrvAAA4bEYAAAAAAAhoRgAAAAAA +AAAAAAAAAAAQj0YAAAAAAAAAAAAAAAAA7GdGAAAAAAAgwkIAAAAAAIBCAAAIAAAAAAAAANXvAADg +7wAA6O8AAAMAAAAVAAAAAAAABvjvAAD97wAAAvAAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAEiB +RgAAAAAAAAAAAAAAAAAwZ0YAAAAAAGDCQgAAAAAAkkIAAAgAAAAAAAAACfAAACDwAABc8AAAAwAA +ABUAAAAAAAAGVPEAAGDxAAB/8QAAdHlGAAAAAADEd0YAAAAAAAAAAAAAAAAAzPFGAAAAAAAAAAAA +AAAAADBnRgAAAAAA4MdCAAAAAADIQgAAEAAAAAAAAADB8QAA1/EAAOTxAAADAAAAFQAAAAAAAAYY +8gAAJPIAADbyAADUdEYAAAAAAER0RgAAAAAAAAAAAAAAAADAp0YAAAAAAAAAAAAAAAAAwGtGAAAA +AADAyUIAAAAAANhCAAAAAAAAAAAAAEnyAABQ8gAAVPIAAAIAAAAVAAAAAAAAAmTyAABp8gAAAAAA +AAhoRgAAAAAABG1GAAAAAAAgykIAAAAAAO5CAAAAAAAAAAAAAG7yAAB58gAAhvIAAAMAAAAVAAAA +AAAABARwAACo8gAAs/IAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAADiPRgAAAAAAAMtCAAAAAAD9 +QgAAIAAAAAAAAAC/8gAA+vIAAFbzAAADAAAAFQAAAAAAAAYd9AAAKfQAAE/0AAAIfEYAAAAAALh7 +RgAAAAAAAAAAAAAAAABc+EYAAAAAAAAAAAAAAAAAhGtGAAAAAADAzkIAAAAAACpDAAAYAAAAAAAA +ALX0AADB9AAAxvQAAAIAAAAVAAAAAAAABoi0AADz9AAAAAAAADhsRgAAAAAACGhGAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAACoa0YAAAAAAEDQQgAAAAAAQUMAAAAAAAAAAAAAAfUAAAj1AAAS +9QAAAwAAABUAAAAAAAAEJFgAACz1AAAz9QAACGhGAAAAAAAEbUYAAAAAAAAAAAAAAAAAYI9GAAAA +AADA0EIAAAAAAJsbAAAAAAAAAAAAADz1AABD9QAATfUAAAMAAAAVAAAAAAAABOyfAABi9QAAafUA +APhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAIiPRgAAAAAAINFCAAAAAABTQwAAGAAAAAAAAABy9QAA +hvUAAJH1AAADAAAAFQAAAAAAAAa19QAAwfUAANL1AAAAbkYAAAAAALRuRgAAAAAAAAAAAAAAAACw +j0YAAAAAAAAAAAAAAAAAqGtGAAAAAACA0kIAAAAAAHNDAAAAAAAAAAAAAOR0AADc9QAA4PUAAAIA +AAAVAAAAAAAAAkARAADq9QAAAAAAAPhnRgAAAAAA+GdGAAAAAADA0kIAAAAAAINDAAAAAAAAAAAA +AOR0AADc9QAA8fUAAAIAAAAVAAAAAAAAAkARAADq9QAAAAAAAPhnRgAAAAAA+GdGAAAAAAAA00IA +AAAAAJNDAAAIAAAAAAAAAPv1AAAC9gAABvYAAAIAAAAVAAAAAAAABhT2AAAf9gAAAAAAAPhnRgAA +AAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAsZ0YAAAAAAGDTQgAAAAAApUMAAAgA +AAAAAAAAJvYAADr2AAA/9gAAAgAAABUAAAAAAAAGuPYAAMT2AAAAAAAA+GdGAAAAAAD4Z0YAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAAAAAAwNVCAAAAAAC4QwAAEAAAAAAAAADQ9gAA +1/YAANz2AAACAAAAFQAAAAAAAAbm9gAA8fYAAAAAAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAArGdGAAAAAABg1kIAAAAAAM1DAAAIAAAAAAAAAPj2AAAI9wAADfcAAAIA +AAAVAAAAAAAABiX3AAAx9wAAAAAAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAwZ0YAAAAAAGDXQgAAAAAA30MAAAgAAAAAAAAAOfcAAED3AABE9wAAAgAAABUAAAAAAAAG +GQYAAFb3AAAAAAAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAA +AAAAwNdCAAAAAADwQwAACAAAAAAAAABd9wAAbfcAAHL3AAACAAAAFQAAAAAAAAaW9wAAovcAAAAA +AAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAAAADg2EIAAAAA +AAFEAAAIAAAAAAAAAHYoAACq9wAArvcAAAIAAAAVAAAAAAAABoooAACVKAAAAAAAADhsRgAAAAAA +CGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwZ0YAAAAAACDZQgAAAAAAFkQAAAgAAAAA +AAAAdigAAKr3AAC49wAAAgAAABUAAAAAAAAGiigAAGyzAAAAAAAA+GdGAAAAAAD4Z0YAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAAAAAAYNlCAAAAAAArRAAAEAAAAAAAAADC9wAAy/cA +AM/3AAADAAAAFQAAAAAAAAbj9wAA7vcAAPX3AAA4bEYAAAAAAFBsRgAAAAAA4HdHAAAAAABcgUYA +AAAAAAAAAAAAAAAArGdGAAAAAADg2UIAAAAAAE1EAAAYAAAAAAAAAPz3AAAE+AAACfgAAAMAAAAV +AAAAAAAABhv4AAAn+AAAMPgAADhsRgAAAAAACGhGAAAAAADAckcAAAAAAHCBRgAAAAAAAAAAAAAA +AACoa0YAAAAAAMDaQgAAAAAAYEQAABgAAAAAAAAAOPgAAEL4AABQ+AAAAwAAABUAAAAAAAAGlPgA +AKD4AACw+AAAuHBGAAAAAAD0cEYAAAAAAAAAAAAAAAAAhIFGAAAAAAAAAAAAAAAAAOxnRgAAAAAA +AN1CAAAAAAB1RAAAAAAAAC0DAAC9+AAAzvgAAOD4AAADAAAAFQAAABIAAAVn+QAAd/kAAIH5AAAY +aEYAAAAAAOhtRgAAAAAAAHFHAAAAAAC8xEYAAAAAAIRnRgAAAAAAYOBCAAAAAACuRAAAAAAAAAAA +AACh+QAAqPkAAKz5AAACAAAAFQAAAAAAAAK4+QAAw/kAAAAAAAD4Z0YAAAAAAPhnRgAAAAAAoOBC +AAAAAADBRAAAAAAAAAAAAADkdAAAyvkAAM75AAACAAAAFQAAAAAAAAJAEQAA2PkAAAAAAAD4Z0YA +AAAAAPhnRgAAAAAA4OBCAAAAAADQRAAAAAAAAAAAAADf+QAA5/kAAPb5AAADAAAAFQAAAAAAAAQb ++gAAJ/oAADL6AAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAAQqEYAAAAAAODhQgAAAAAA0xoAAAAA +AAAAAAAAQ/oAAEr6AABO+gAAAgAAABUAAAAAAAACVvoAAFv6AAAAAAAA+GdGAAAAAAD4Z0YAAAAA +ACDiQgAAAAAA5kQAACAAAAAAAAAAYPoAAGz6AAB8+gAAAwAAABUAAAAAAAAGDMIAALz6AADE+gAA +fG1GAAAAAAAIaEYAAAAAAAAAAAAAAAAATJhGAAAAAAAAAAAAAAAAAGRwRgAAAAAAYONCAAAAAAD1 +RAAAEAAAAAAAAABSrQAA1/oAANv6AAACAAAAFQAAAAAAAAZjrQAA5foAAAAAAAA4bEYAAAAAACxs +RgAAAAAAgHBHAAAAAAAAAAAAAAAAAAAAAAAAAAAAZGdGAAAAAADg40IAAAAAAAVFAAAAAAAAAAAA +AOz6AADz+gAA9/oAAAIAAAAVAAAAAAAAAgH7AAAM+wAAAAAAAPhnRgAAAAAA+GdGAAAAAAAg5EIA +AAAAABtFAAAAAAAAAAAAABP7AAAh+wAAT/sAAAMAAAAVAAAAAAAABLj7AAC++wAA0fsAAEBoRgAA +AAAAwHFGAAAAAAAAAAAAAAAAAIjRRgAAAAAAIOdCAAAAAAAwRQAACAAAAAAAAAD++wAADPwAADr8 +AAADAAAAFQAAAAAAAAa8/AAAwvwAANT8AADQcEYAAAAAAIRxRgAAAAAAAAAAAAAAAAAU0kYAAAAA +AAAAAAAAAAAAMGdGAAAAAABg6kIAAAAAAEVFAAAIAAAAAAAAAAcnAAAB/QAABf0AAAIAAAAVAAAA +AAAABhknAAAkJwAAAAAAACRrRgAAAAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAw +Z0YAAAAAAKDqQgAAAAAAVkUAAAgAAAAAAAAABycAAAH9AAAN/QAAAgAAABUAAAAAAAAGGScAACQn +AAAAAAAAJGtGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAAAAAA4OpC +AAAAAABoRQAAAAAAAAAAAAAV/QAAHP0AACT9AAADAAAAFQAAAAAAAAQw/QAANf0AADr9AAD4Z0YA +AAAAAPhnRgAAAAAAAAAAAAAAAACYgUYAAAAAAEDrQgAAAAAAf0UAAAAAAAAAAAAAFf0AABz9AABB +/QAAAwAAABUAAAAAAAAEMP0AADX9AAA6/QAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAArIFGAAAA +AACg60IAAAAAAJtFAAAAAAAAAAAAAE39AABS/QAAVv0AAAIAAAAVAAAAAAAAAlz9AABh/QAAAAAA +APhnRgAAAAAA+GdGAAAAAADA60IAAAAAAKtFAAAIAAAAAAAAAGb9AAB2/QAAk/0AAAMAAAAVAAAA +AAAABlhHAADJ/QAA1/0AAABuRgAAAAAAtG5GAAAAAAAAAAAAAAAAAOS1RgAAAAAAAAAAAAAAAAAw +Z0YAAAAAACDtQgAAAAAAu0UAAAgAAAAAAAAA7/0AAPf9AAAJ/gAAAwAAABUAAAAAAAAGKv4AADb+ +AABD/gAAjG9GAAAAAAC0bkYAAAAAAAAAAAAAAAAAYKhGAAAAAAAAAAAAAAAAADBnRgAAAAAA4O1C +AAAAAADMRQAACAAAAAAAAABU/gAAW/4AAGD+AAADAAAAFQAAAAAAAAaE/gAAj/4AAJb+AACMb0YA +AAAAALRuRgAAAAAAAAAAAAAAAADYj0YAAAAAAAAAAAAAAAAAMGdGAAAAAACA7kIAAAAAAAxGAAAA +AAAAAAAAAKX+AACz/gAAzP4AAAMAAAAVAAAAAAAABA7/AAAa/wAAJf8AAAhoRgAAAAAAyGxGAAAA +AABgbUcAAAAAAEi2RgAAAAAAAPBCAAAAAAAcRgAAAAAAAAAAAAA9/wAATf8AAH//AAADAAAAFQAA +AAAAAAQLAAEAFwABAC8AAQBAaEYAAAAAAJhyRgAAAAAAAAAAAAAAAAA4+UYAAAAAAMDyQgAAAAAA +e0YAAAAAAAAAAAAAYwABAG4AAQBzAAEAAwAAABUAAAAAAAAEiQABAJQAAQCbAAEA+GdGAAAAAAD4 +Z0YAAAAAAAAAAAAAAAAAwIFGAAAAAABg80IAAAAAAI9GAAAAAAAAAAAAAKIAAQCtAAEAsQABAAIA +AAAVAAAAAAAAAsEAAQDMAAEAAAAAAPhnRgAAAAAA+GdGAAAAAADA80IAAAAAAKJGAAAQAAAAAAAA +ANMAAQDbAAEA/QABAAMAAAAVAAAAAAAABlABAQBcAQEAbAEBADhsRgAAAAAACGhGAAAAAAAAAAAA +AAAAAKDSRgAAAAAAAAAAAAAAAABkZ0YAAAAAAMD1QgAAAAAAyEYAABgAAAAAAAAAhwEBAJgBAQCx +AQEAAwAAABUAAAAAAAAGBQIBABECAQAnAgEABHlGAAAAAABUeUYAAAAAAAAAAAAAAAAANMVGAAAA +AAAAAAAAAAAAALRnRgAAAAAA4PdCAAAAAADWRgAAAAAAAAAAAABQAgEAWAIBAF0CAQACAAAAFQAA +AAAAAAIEcAAAeQIBAAAAAAD4Z0YAAAAAAPhnRgAAAAAAwPhCAAAAAADtRgAAEAAAAAAAAACBAgEA +jwIBAJQCAQADAAAAFQAAAAAAAAbVAgEA4QIBAPoCAQBkeEYAAAAAAFR4RgAAAAAAAAAAAAAAAACs +xUYAAAAAAAAAAAAAAAAApGdGAAAAAACA/EIAAAAAAAlHAAAQAAAAAAAAABgDAQAjAwEAKAMBAAIA +AAAVAAAAAAAABjwDAQBIAwEAAAAAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAACkZ0YAAAAAAGD9QgAAAAAAIkcAABAAAAAAAAAATwMBAFwDAQB+AwEAAwAAABUAAAAAAAAG +/AMBAAIEAQAVBAEAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAArLZGAAAAAAAAAAAAAAAAAKRnRgAA +AAAAwABDAAAAAAA1RwAAAAAAAAAAAAA2BAEAPQQBAEIEAQACAAAAFQAAAAAAAAJMBAEAVwQBAAAA +AAD4Z0YAAAAAAPhnRgAAAAAAYAFDAAAAAABORwAAEAAAAAAAAABeBAEAaQQBAG4EAQACAAAAFQAA +AAAAAAaCqAAAgAQBAAAAAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +pGdGAAAAAAAAAkMAAAAAAGhHAAAQAAAAAAAAAIcEAQCSBAEAlgQBAAIAAAAVAAAAAAAABqQEAQCv +BAEAAAAAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACkZ0YAAAAAAIAC +QwAAAAAAgkcAAAAAAAAAAAAAtgQBAMYEAQDpBAEAAwAAABUAAAAAAAAEeQUBAIUFAQCdBQEAQGhG +AAAAAADccEYAAAAAAAAAAAAAAAAAfOdGAAAAAAAABUMAAAAAAJ9HAAAIAAAAAAAAAL8FAQDPBQEA +/QUBAAMAAAAVAAAAAAAABncGAQCDBgEAogYBAEBoRgAAAAAASHFGAAAAAAAAAAAAAAAAABT6RgAA +AAAAAAAAAAAAAAAsZ0YAAAAAAOAGQwAAAAAAAkgAAAAAAAAAAAAAzwYBANcGAQDcBgEAAgAAABUA +AAAAAAAC/AYBAAIHAQAAAAAA+GdGAAAAAAAka0YAAAAAAIAHQwAAAAAAEkgAAAAAAAAAAAAABwcB +ABMHAQAYBwEAAgAAABUAAAAAAAACQgcBAE4HAQAAAAAACGhGAAAAAAAEbUYAAAAAAIAIQwAAAAAA +IkgAAAAAAAAAAAAAWAcBAF8HAQBjBwEAAgAAABUAAAAAAAACcwcBAH4HAQAAAAAA+GdGAAAAAAD4 +Z0YAAAAAAOAIQwAAAAAAM0gAAAAAAAAAAAAAhQcBAIwHAQCQBwEAAgAAABUAAAAAAAACogcBAKcH +AQAAAAAACGhGAAAAAAAEbUYAAAAAAEAJQwAAAAAAQUgAAAgAAAAAAAAArgcBAL4HAQDrBwEAAwAA +ABUAAAAAAAAGcQgBAH0IAQCRCAEACGhGAAAAAAAEbUYAAAAAAAAAAAAAAAAAAA5HAAAAAAAAAAAA +AAAAACxnRgAAAAAAgAtDAAAAAABfSAAACAAAAAAAAAC9CAEAzQgBAAEJAQADAAAAFQAAAAAAAAaq +CQEAtgkBANkJAQCUekYAAAAAAAR6RgAAAAAAAAAAAAAAAAAIBkcAAAAAAAAAAAAAAAAAMGdGAAAA +AADADkMAAAAAAHBIAAAAAAAAAAAAAAwKAQAYCgEAKAoBAAMAAAAVAAAAAAAABE8KAQBbCgEAZgoB +APhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAALCoRgAAAAAAgA9DAAAAAACHSAAAGAAAAAAAAAB1CgEA +hgoBAKAKAQADAAAAFQAAAAAAAAYZCwEAJQsBADwLAQDEeEYAAAAAABR5RgAAAAAAYHZHAAAAAAAk +xkYAAAAAAAAAAAAAAAAA7GdGAAAAAABAEkMAAAAAAJZIAAAAAAAAAAAAAFULAQBcCwEAYAsBAAIA +AAAVAAAAAAAAAvTNAAD/zQAAAAAAAPhnRgAAAAAA+GdGAAAAAACAEkMAAAAAAKtIAAAAAAAAAAAA +AGoLAQB0CwEAgwsBAAMAAAAVAAAAAAAABMALAQDGCwEA0AsBAAhoRgAAAAAABG1GAAAAAAAAAAAA +AAAAAACpRgAAAAAAIBRDAAAAAADNSAAAAAAAAAAAAADiCwEA7QsBAPELAQADAAAAFQAAAAAAAAT/ +hwAAEwwBAB4MAQD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAADUgUYAAAAAAKAUQwAAAAAA30gAAAAA +AAAAAAAAJQwBAC0MAQA+DAEAAwAAABUAAAAAAAAEhAwBAJAMAQCeDAEAGGhGAAAAAACAb0YAAAAA +AAAAAAAAAAAAUKlGAAAAAAAAFkMAAAAAAPRIAAAAAAAAAAAAALIMAQC7DAEAzQwBAAMAAAAVAAAA +AAAABAANAQAMDQEAFg0BAAhoRgAAAAAABG1GAAAAAAAAAAAAAAAAABC3RgAAAAAAIBdDAAAAAAAW +SQAACAAAAAAAAAArDQEAMw0BAEMNAQADAAAAFQAAAAAAAAbqGAAAfw0BAIwNAQD4Z0YAAAAAAPhn +RgAAAAAAAAAAAAAAAACImEYAAAAAAAAAAAAAAAAALGdGAAAAAAAAGEMAAAAAAFFJAAAYAAAAAAAA +AJsNAQCrDQEAww0BAAMAAAAVAAAAAAAABgzCAAAEDgEAFQ4BAExzRgAAAAAAaHJGAAAAAAAAAAAA +AAAAAJzGRgAAAAAAAAAAAAAAAADsZ0YAAAAAAEAZQwAAAAAAXkkAAAgAAAAAAAAALA4BADwOAQBF +DgEAAwAAABUAAAAAAAAGfnIAAHMOAQCADgEAAG5GAAAAAADIb0YAAAAAAAByRwAAAAAAAJBGAAAA +AAAAAAAAAAAAADBnRgAAAAAAYBpDAAAAAABsSQAAAAAAAAAAAACMDgEAlw4BAK8OAQADAAAAFQAA +AAAAAAQ8AwEA4Q4BAOkOAQAIaEYAAAAAAARtRgAAAAAAAAAAAAAAAADEmEYAAAAAAEAbQwAAAAAA +iEkAAAAAAAAAAAAAAA8BABMPAQAvDwEAAwAAABUAAAAAAAAEdA8BAHoPAQCFDwEAQGhGAAAAAAB0 +ckYAAAAAAAAAAAAAAAAAWN5GAAAAAADAHEMAAAAAAJlJAAAAAAAAAAAAAFYFAACgDwEApA8BAAIA +AAAVAAAAAAAAAibFAABWBQAAAAAAAPhnRgAAAAAA+GdGAAAAAADgHEMAAAAAALRJAAAAAAAAAAAA +AKwPAQC0DwEA7A8BAAMAAAAVAAAAAAAABFIQAQBeEAEAcRABAAhoRgAAAAAABG1GAAAAAAAAAAAA +AAAAADwdRwAAAAAA4B1DAAAAAADgSQAAAAAAAAAAAACoEAEAtBABAMQQAQADAAAAFQAAAAAAAAR4 +2gAA9RABAP4QAQAIaEYAAAAAAARtRgAAAAAAAAAAAAAAAACgqUYAAAAAAOAeQwAAAAAA7kkAAAAA +AAAAAAAADREBABARAQAUEQEAAQAAABUAAAAAAAACGhEBAPhnRgAAAAAA+GdGAAAAAAAAH0MAAAAA +AABKAAAQAAAAAAAAAB0RAQA2EQEAdREBAAMAAAAVAAAAAAAABi8SAQA7EgEAXBIBANR3RgAAAAAA +RHlGAAAAAAAAAAAAAAAAAKwlRwAAAAAAAAAAAAAAAAA8Z0YAAAAAAMAhQwAAAAAALkoAAAgAAAAA +AAAApRIBAMYSAQAAEwEAAwAAABUAAAAAAAAGrBMBALgTAQDcEwEAOGxGAAAAAAAIaEYAAAAAAAAA +AAAAAAAAaB5HAAAAAAAAAAAAAAAAADBnRgAAAAAAoCRDAAAAAABZSgAAAAAAAAAAAAAZFAEAkgQB +ACgUAQACAAAAFQAAAAAAAAKkBAEAOhQBAAAAAAD4Z0YAAAAAAPhnRgAAAAAAICVDAAAAAABnSgAA +AAAAAAAAAABBFAEAUhQBAFwUAQADAAAAFQAAAAAAAAS84QAAnxQBAK4UAQBAaEYAAAAAAARzRgAA +AAAAAAAAAAAAAAB0t0YAAAAAAIAnQwAAAAAAe0oAAAgAAAAAAAAAxxQBANIUAQDeFAEAAwAAABUA +AAAAAAAG3KEAAAQVAQANFQEAAG5GAAAAAAC0bkYAAAAAAAAAAAAAAAAAKJBGAAAAAAAAAAAAAAAA +ADBnRgAAAAAAQChDAAAAAACQSgAAAAAAAAAAAAAYFQEAJBUBADQVAQADAAAAFQAAAAAAAARnFQEA +cxUBAIIVAQAIaEYAAAAAAARtRgAAAAAAAAAAAAAAAADwqUYAAAAAAEApQwAAAAAAoEoAABAAAAAA +AAAAkRUBAJkVAQCmFQEAAwAAABUAAAAAAAAG4hUBAO4VAQD6FQEAjG9GAAAAAAC0bkYAAAAAAAAA +AAAAAAAA6IFGAAAAAAAAAAAAAAAAADxnRgAAAAAAwCpDAAAAAADNSgAAAAAAAAAAAAAGFgEAPRYB +ALEWAQADAAAAFQAAAAAAAASZGAEApRgBAP0YAQAgaEYAAAAAAHiHRgAAAAAAAAAAAAAAAAAEUEcA +AAAAAGA0QwAAAAAA4koAAAAAAAAAAAAAhBkBAJcZAQCfGQEAAwAAABUAAAAAAAAE0xkBAN8ZAQDn +GQEA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAJlGAAAAAABANUMAAAAAAPNKAAAIAAAAAAAAAPoZ +AQAdGgEALhoBAAMAAAAVAAAAAAAABvUaAQABGwEAEBsBABhoRgAAAAAAqG5GAAAAAAAAAAAAAAAA +ADDoRgAAAAAAAAAAAAAAAAAwZ0YAAAAAAMA4QwAAAAAAiksAADAAAAAAAAAAUBsBAF8bAQBwGwEA +AwAAABUAAAAAAAAGsxsBAL8bAQDNGwEA8G5GAAAAAAAsb0YAAAAAAAAAAAAAAAAAFMdGAAAAAAAA +AAAAAAAAAKx8RgAAAAAAADpDAAAAAACgSwAAOAAAAAAAAADwGwEA+hsBAAIcAQADAAAAFQAAAAAA +AAYrHAEAMRwBADccAQCYbEYAAAAAAAhoRgAAAAAAAAAAAAAAAABQkEYAAAAAAAAAAAAAAAAAJH1G +AAAAAACgOkMAAAAAALdLAAAAAAAAAAAAAEgcAQBkHAEAnBwBAAMAAAAVAAAAAAAABBEdAQAdHQEA +MR0BAAhoRgAAAAAAiG1GAAAAAAAAAAAAAAAAAFQURwAAAAAAIDxDAAAAAADOSwAACAAAAAAAAABo +HQEAbx0BAHMdAQACAAAAFQAAAAAAAAaFHQEAkB0BAAAAAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAAAACAPEMAAAAAAORLAAAAAAAAAAAAAJcdAQCiHQEAph0B +AAIAAAAVAAAAAAAAAr4dAQDJHQEAAAAAAPhnRgAAAAAA+GdGAAAAAAAAPUMAAAAAAPpLAAAIAAAA +AAAAANAdAQDtHQEAMh4BAAMAAAAVAAAAAAAABkUfAQBRHwEAcx8BAPR2RgAAAAAAhHVGAAAAAAAA +AAAAAAAAAAhHRwAAAAAAAAAAAAAAAAAwZ0YAAAAAACBAQwAAAAAAKEwAAAAAAAAAAAAA8B8BAP4f +AQA0IAEAAwAAABUAAAAAAAAEQCEBAEwhAQB7IQEAAGhGAAAAAAAkdEYAAAAAAAAAAAAAAAAAbClH +AAAAAAAgREMAAAAAAE5MAAAQAAAAAAAAAL4hAQDSIQEA5yEBAAMAAAAVAAAAAAAABjoiAQBGIgEA +WCIBABxzRgAAAAAAZHNGAAAAAAAAAAAAAAAAANi3RgAAAAAAAAAAAAAAAABkZ0YAAAAAAABGQwAA +AAAAYkwAABAAAAAAAAAAbCIBAHMiAQB/IgEAAwAAABUAAAAAAAAGkSIBAJwiAQCjIgEACGxGAAAA +AAAIaEYAAAAAAAAAAAAAAAAAeJBGAAAAAAAAAAAAAAAAAGRnRgAAAAAAYEZDAAAAAAB3TAAACAAA +AAAAAACuIgEAtiIBAMMiAQADAAAAFQAAAAAAAAa83gAABCMBABAjAQBscUYAAAAAANhxRgAAAAAA +AAAAAAAAAAA8uEYAAAAAAAAAAAAAAAAAMGdGAAAAAADgR0MAAAAAALZMAAAIAAAAAAAAACYjAQAz +IwEAWCMBAAMAAAAVAAAAAAAABskjAQDVIwEA6SMBAMRwRgAAAAAA1HJGAAAAAAAAAAAAAAAAAMAq +RwAAAAAAAAAAAAAAAAAwZ0YAAAAAAABKQwAAAAAA3kwAAAgAAAAAAAAAOqEAACwkAQAwJAEAAgAA +ABUAAAAAAAAG7J8AAEAkAQAAAAAAAG5GAAAAAAC0bkYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAADBnRgAAAAAAYEpDAAAAAADwTAAACAAAAAAAAABJJAEAUSQBAGEkAQADAAAAFQAAAAAAAAbc +BgAAhyQBAJAkAQA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAACgkEYAAAAAAAAAAAAAAAAAMGdGAAAA +AABAS0MAAAAAAB5NAAAIAAAAAAAAAJ8kAQCmJAEAsiQBAAMAAAAVAAAAAAAABswkAQDXJAEA4CQB +AABuRgAAAAAAtG5GAAAAAAAAAAAAAAAAAPyBRgAAAAAAAAAAAAAAAAAwZ0YAAAAAAMBLQwAAAAAA +SU0AAAgAAAAAAAAA6yQBAPokAQALJQEAAwAAABUAAAAAAAAGbiUBAHolAQCUJQEARHZGAAAAAAC0 +dkYAAAAAAAAAAAAAAAAAlPJGAAAAAAAAAAAAAAAAADBnRgAAAAAAYE5DAAAAAABdTQAACAAAAAAA +AADGJQEAziUBAOYlAQADAAAAFQAAAAAAAAYjJgEALyYBADomAQCMb0YAAAAAALRuRgAAAAAAAAAA +AAAAAAAs00YAAAAAAAAAAAAAAAAAMGdGAAAAAABAT0MAAAAAAG9NAAAAAAAAAAAAAFsmAQBiJgEA +aiYBAAMAAAAVAAAAAAAABBT2AAB8JgEAgyYBAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAABCCRgAA +AAAAoE9DAAAAAACSTQAACAAAAAAAAACKJgEAliYBAKQmAQADAAAAFQAAAAAAAAYPJwEAGycBACcn +AQAcc0YAAAAAAGRzRgAAAAAAAAAAAAAAAACguEYAAAAAAAAAAAAAAAAAMGdGAAAAAAAgUkMAAAAA +AKJNAAAQAAAAAAAAAD4nAQBHJwEASycBAAIAAAAVAAAAAAAABmMnAQBoJwEAAAAAAPhnRgAAAAAA ++GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAAKBSQwAAAAAAr00AABAAAAAA +AAAAcScBAHknAQCCJwEAAwAAABUAAAAAAAAG0CcBANYnAQDcJwEACGhGAAAAAADIbEYAAAAAAABz +RwAAAAAAJIJGAAAAAAAAAAAAAAAAAGRnRgAAAAAAwFRDAAAAAADGTQAAAAAAAAAAAADkJwEAWAIB +AOwnAQACAAAAFQAAAAAAAAIEcAAA9ycBAAAAAAD4Z0YAAAAAAPhnRgAAAAAAoFVDAAAAAADjTQAA +AAAAAAAAAAD/JwEABigBABgoAQADAAAAFQAAAAAAAAQSdgAAOSgBAEQoAQD4Z0YAAAAAAPhnRgAA +AAAAAAAAAAAAAABAqkYAAAAAACBWQwAAAAAA/00AAAAAAAAAAAAAVSgBAF0oAQBvKAEAAwAAABUA +AAAAAAAEBHAAAKEoAQCuKAEACGhGAAAAAAAEbUYAAAAAAAAAAAAAAAAABLlGAAAAAAAAV0MAAAAA +ABtOAAAAAAAAAAAAAL8oAQDJKAEAzigBAAIAAAAVAAAAAAAAA/ooAQAAKQEAAAAAAAhoRgAAAAAA +PHFGAAAAAAAgd0cAAAAAAKBYQwAAAAAANU4AAAAAAAAAAAAABikBAA8pAQAUKQEAAgAAABUAAAAA +AAACHykBACspAQAAAAAA+GdGAAAAAAD4Z0YAAAAAAKBZQwAAAAAAVU4AAAAAAAAAAAAABikBAA8p +AQAzKQEAAgAAABUAAAAAAAACHykBACspAQAAAAAA+GdGAAAAAAD4Z0YAAAAAAKBaQwAAAAAAdU4A +AAAAAAAAAAAAPikBAEUpAQBNKQEAAwAAABUAAAAAAAAEYSkBAGwpAQBzKQEA+GdGAAAAAAD4Z0YA +AAAAAAAAAAAAAAAAOIJGAAAAAAAgW0MAAAAAAK5OAAAIAAAAAAAAAHopAQCRKQEAlikBAAIAAAAV +AAAAAAAABrwpAQDCKQEAAAAAADhsRgAAAAAAUGxGAAAAAADgbUcAAAAAAAAAAAAAAAAAAAAAAAAA +AAAwZ0YAAAAAACBcQwAAAAAAxk4AAAAAAAAAAAAAyCkBANMpAQDYKQEAAgAAABUAAAAAAAACKv4A +AOwpAQAAAAAAGGhGAAAAAABob0YAAAAAAOBcQwAAAAAA5E4AAAAAAAAAAAAA+SkBAAEqAQAGKgEA +AgAAABUAAAAAAAADGCoBAB0qAQAAAAAACGhGAAAAAADIbEYAAAAAAABuRwAAAAAAgF1DAAAAAAAH +TwAAAAAAAAAAAAAkKgEAKyoBAC8qAQACAAAAFQAAAAAAAAI7KgEARioBAAAAAAD4Z0YAAAAAAPhn +RgAAAAAA4F1DAAAAAAAwTwAAAAAAAAAAAABNKgEAWCoBAGoqAQADAAAAFQAAAAAAAAT0iAAAlSoB +AKQqAQAIaEYAAAAAAARtRgAAAAAAAAAAAAAAAACQqkYAAAAAAIBeQwAAAAAATk8AAAgAAAAAAAAA +tSoBAL0qAQDhKgEAAwAAABUAAAAAAAAGXSsBAGkrAQB6KwEAjG9GAAAAAAC0bkYAAAAAAAAAAAAA +AAAAlB9HAAAAAAAAAAAAAAAAADBnRgAAAAAAAGBDAAAAAABjTwAACAAAAAAAAACxKwEAvSsBAM4r +AQADAAAAFQAAAAAAAAY0KQAA9isBAAEsAQAIaEYAAAAAAHBtRgAAAAAAIHFHAAAAAABMgkYAAAAA +AAAAAAAAAAAAKGdGAAAAAAAAYUMAAAAAAH9PAAAAAAAAAAAAABEsAQAYLAEAHCwBAAIAAAAVAAAA +AAAAApEiAQAmLAEAAAAAAAhoRgAAAAAABG1GAAAAAABgYUMAAAAAAJJPAAAQAAAAAAAAAC0sAQAB +KgEANiwBAAIAAAAVAAAAAAAABhgqAQBALAEAAAAAAAhsRgAAAAAAFGxGAAAAAABgd0cAAAAAAAAA +AAAAAAAAAAAAAAAAAABUZ0YAAAAAAABiQwAAAAAAok8AAAAAAAAAAAAARSwBAEwsAQBQLAEAAgAA +ABUAAAAAAAACumkAAGIsAQAAAAAA+GdGAAAAAAD4Z0YAAAAAAIBiQwAAAAAAuE8AACgAAAAAAAAA +aSwBAHksAQCjLAEAAwAAABUAAAAAAAAGXy0BAGstAQCELQEAhHpGAAAAAAC0eUYAAAAAAAAAAAAA +AAAAuNNGAAAAAAAAAAAAAAAAAHxwRgAAAAAAYGZDAAAAAADzTwAACAAAAAAAAACwLQEAxi0BANQt +AQADAAAAFQAAAAAAAAYYLgEAJC4BADguAQB0eUYAAAAAAHR4RgAAAAAAAAAAAAAAAABggkYAAAAA +AAAAAAAAAAAAMGdGAAAAAAAgaUMAAAAAAAlQAAAQAAAAAAAAAEUuAQBWLgEAei4BAAMAAAAVAAAA +AAAABhgvAQAkLwEANC8BAHxtRgAAAAAACGhGAAAAAAAAAAAAAAAAABQsRwAAAAAAAAAAAAAAAABk +Z0YAAAAAACBrQwAAAAAALlAAAAgAAAAAAAAAdS8BAIUvAQCdLwEAAwAAABUAAAAAAAAGD4wAABEw +AQAgMAEAAG5GAAAAAABgbkYAAAAAACBwRwAAAAAABA9HAAAAAAAAAAAAAAAAADBnRgAAAAAAoGxD +AAAAAAA8UAAAAAAAAAAAAABRMAEAGCwBAFgwAQACAAAAFQAAAAAAAAKRIgEAYjABAAAAAAAIaEYA +AAAAAARtRgAAAAAAAG1DAAAAAABQUAAACAAAAAAAAABpMAEAczABAJMwAQADAAAAFQAAAAAAAAYQ +MQEAHDEBACgxAQA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAADsJkcAAAAAAAAAAAAAAAAAMGdGAAAA +AABgbkMAAAAAAGBQAAAAAAAAAAAAAGMxAQABKgEAazEBAAMAAAAVAAAAAAAABBgqAQCDMQEAijEB +AAhoRgAAAAAABG1GAAAAAAAAAAAAAAAAAHSCRgAAAAAAAG9DAAAAAACQUAAAAAAAAAAAAAAHJwAA +Af0AAJUxAQACAAAAFQAAAAAAAAIZJwAAJCcAAAAAAAD4Z0YAAAAAAPhnRgAAAAAAQG9DAAAAAACq +UAAAAAAAAAAAAACdMQEApDEBAKgxAQACAAAAFQAAAAAAAAKsMQEAtzEBAAAAAAD4Z0YAAAAAAPhn +RgAAAAAAgG9DAAAAAAC6UAAAAAAAAAAAAACdMQEApDEBAL4xAQACAAAAFQAAAAAAAAKsMQEAtzEB +AAAAAAD4Z0YAAAAAAPhnRgAAAAAAwG9DAAAAAADQUAAAAAAAAAAAAACdMQEApDEBAMIxAQACAAAA +FQAAAAAAAAKsMQEAtzEBAAAAAAD4Z0YAAAAAAPhnRgAAAAAAAHBDAAAAAADqUAAAAAAAAAAAAACd +MQEApDEBAMYxAQACAAAAFQAAAAAAAAKsMQEAtzEBAAAAAAD4Z0YAAAAAAPhnRgAAAAAAQHBDAAAA +AAD2UAAAAAAAAAAAAACdMQEApDEBAMoxAQACAAAAFQAAAAAAAAKsMQEAtzEBAAAAAAD4Z0YAAAAA +APhnRgAAAAAAgHBDAAAAAAAZUQAAAAAAAAAAAACdMQEApDEBAM4xAQACAAAAFQAAAAAAAAKsMQEA +tzEBAAAAAAD4Z0YAAAAAAPhnRgAAAAAAwHBDAAAAAAAnUQAAKAAAAAAAAADSMQEA8DEBAPUxAQAD +AAAAFQAAAAAAAAZ2MgEAgjIBAJsyAQBIbkYAAAAAALRuRgAAAAAAAAAAAAAAAADgqkYAAAAAAAAA +AAAAAAAANHBGAAAAAACAdEMAAAAAADdRAAAAAAAAAAAAALAyAQDtCwEAuTIBAAIAAAAVAAAAAAAA +AskyAQDOMgEAAAAAAPhnRgAAAAAA+GdGAAAAAAAAdUMAAAAAAExRAAAIAAAAAAAAANMyAQDaMgEA +3jIBAAMAAAAVAAAAAAAABpVGAADwMgEA9zIBAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAIiCRgAA +AAAAAAAAAAAAAAAwZ0YAAAAAAGB1QwAAAAAAY1EAABAAAAAAAAAA/jIBAAozAQAPMwEAAwAAABUA +AAAAAAAGVzMBAGMzAQBuMwEAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAyJBGAAAAAAAAAAAAAAAA +AERnRgAAAAAAQHdDAAAAAACbUQAACAAAAAAAAACIMwEAkjMBAMMzAQADAAAAFQAAAAAAAAZyNAEA +fjQBAI80AQC0eEYAAAAAAJR5RgAAAAAAAGtHAAAAAABoLUcAAAAAAAAAAAAAAAAAMGdGAAAAAACg +ekMAAAAAAMhRAAAAAAAAAAAAAM80AQDbNAEA7zQBAAMAAAAVAAAAAAAABARwAAAgNQEAKjUBAAho +RgAAAAAABG1GAAAAAAAAAAAAAAAAAGi5RgAAAAAAgHtDAAAAAADjUQAACAAAAAAAAAA9NQEATjUB +AJ01AQADAAAAFQAAAAAAAAYhNwEALTcBAGI3AQA4aEYAAAAAACR6RgAAAAAAAAAAAAAAAACENEcA +AAAAAAAAAAAAAAAAKGdGAAAAAADAg0MAAAAAADdSAAAIAAAAAAAAANU3AQBvHQEA3DcBAAIAAAAV +AAAAAAAABtJhAADsNwEAAAAAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAwZ0YAAAAAACCEQwAAAAAASFIAAAgAAAAAAAAA9TcBAP83AQAMOAEAAwAAABUAAAAAAAAGNDgB +ADo4AQBAOAEAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAA8JBGAAAAAAAAAAAAAAAAADBnRgAAAAAA +QIVDAAAAAABWUgAAAAAAAAAAAABMOAEAWzgBAGQ4AQADAAAAFQAAAAAAAASNOAEAmTgBAKY4AQBA +aEYAAAAAADhyRgAAAAAAAAAAAAAAAACcgkYAAAAAAKCGQwAAAAAAZ1IAAAgAAAAAAAAArjgBALU4 +AQDHOAEAAwAAABUAAAAAAAAGtagAAOg4AQDzOAEA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAMKtG +AAAAAAAAAAAAAAAAAChnRgAAAAAAAIdDAAAAAAB9UgAAAAAAAAAAAAAEOQEALzkBAEU5AQADAAAA +FQAAAAAAAATSOQEA3jkBAPY5AQD4Z0YAAAAAADxrRgAAAAAA4GxHAAAAAAD43kYAAAAAAOCJQwAA +AAAAj1IAAAgAAAAAAAAAGzoBACo6AQAvOgEAAwAAABUAAAAAAAAGlNYAAFU6AQBgOgEAjG9GAAAA +AAC0bkYAAAAAAAAAAAAAAAAAsIJGAAAAAAAAAAAAAAAAADBnRgAAAAAA4IpDAAAAAACnUgAAAAAA +AAAAAABsOgEAdjoBAOU6AQADAAAAFQAAAAAAAAQxPAEAPTwBAEo8AQD4Z0YAAAAAAPhnRgAAAAAA +AAAAAAAAAAA4SUcAAAAAAGCPQwAAAAAAtlIAAAgAAAAAAAAAvTwBAMo8AQDrPAEAAwAAABUAAAAA +AAAGfz0BAIs9AQCfPQEACGhGAAAAAAAEbUYAAAAAAAAAAAAAAAAA5OhGAAAAAAAAAAAAAAAAADBn +RgAAAAAAoJFDAAAAAADFUgAAAAAAAAAAAADHPQEAASoBAM49AQACAAAAFQAAAAAAAAKUoQAA5j0B +AAAAAAAIaEYAAAAAAARtRgAAAAAAQJJDAAAAAADYUgAACAAAAAAAAADtPQEA/D0BAAQ+AQADAAAA +FQAAAAAAAAYoPgEAND4BADs+AQA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAADEgkYAAAAAAAAAAAAA +AAAAMGdGAAAAAAAAk0MAAAAAAOtSAAAIAAAAAAAAAEI+AQBTPgEAZj4BAAMAAAAVAAAAAAAABu0+ +AQD5PgEAEz8BAEBoRgAAAAAAzHFGAAAAAAAAAAAAAAAAAICrRgAAAAAAAAAAAAAAAAAsZ0YAAAAA +AACbQwAAAAAA/lIAAAgAAAAAAAAAKT8BADk/AQBbPwEAAwAAABUAAAAAAAAGjxEAAL0/AQDUPwEA ++GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAA8PpGAAAAAAAAAAAAAAAAACxnRgAAAAAAQJxDAAAAAAA5 +TAAACAAAAAAAAAD/PwEACkABAA5AAQACAAAAFQAAAAAAAAYaQAEAJUABAAAAAAA4bEYAAAAAAAho +RgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAAAACgnEMAAAAAABZTAAAIAAAAAAAA +ACxAAQAzQAEAO0ABAAMAAAAVAAAAAAAABk9AAQBaQAEAYUABADhsRgAAAAAACGhGAAAAAAAAAAAA +AAAAANiCRgAAAAAAAAAAAAAAAAAwZ0YAAAAAAACdQwAAAAAAI1MAABAAAAAAAAAAaEABAHhAAQCC +QAEAAwAAABUAAAAAAAAGI5YAANtAAQDnQAEAAG5GAAAAAACYb0YAAAAAAAAAAAAAAAAAPJlGAAAA +AAAAAAAAAAAAAERnRgAAAAAAQJ5DAAAAAAA3UwAACAAAAAAAAAD8QAEADEEBABxBAQADAAAAFQAA +AAAAAAZnFQEAS0EBAFpBAQCMb0YAAAAAALRuRgAAAAAAAAAAAAAAAADMuUYAAAAAAAAAAAAAAAAA +MGdGAAAAAABAn0MAAAAAAFBTAAAIAAAAAAAAAHFBAQB9QQEAhkEBAAMAAAAVAAAAAAAABhheAADC +QQEAzEEBADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAHiZRgAAAAAAAAAAAAAAAAAwZ0YAAAAAAGCg +QwAAAAAAYVMAAAAAAAAAAAAA40EBAO1BAQDyQQEAAwAAABUAAAAAAAAEIBgAAB9CAQAlQgEA+GdG +AAAAAAD4Z0YAAAAAAAAAAAAAAAAAGJFGAAAAAACAoUMAAAAAAHJTAAAYAAAAAAAAADRCAQBDQgEA +T0IBAAMAAAAVAAAAAAAABnjaAACNQgEAlUIBAAxuRgAAAAAAtG5GAAAAAAAAAAAAAAAAAECRRgAA +AAAAAAAAAAAAAAC0Z0YAAAAAAICiQwAAAAAAmlMAABgAAAAAAAAAoEIBALdCAQDYQgEAAwAAABUA +AAAAAAAGR0MBAFNDAQBfQwEAfG1GAAAAAABglEYAAAAAAAAAAAAAAAAAXPNGAAAAAAAAAAAAAAAA +ANhrRgAAAAAAgKRDAAAAAACuUwAAGAAAAAAAAACHQwEAj0MBAKdDAQADAAAAFQAAAAAAAAaPEQAA +DkQBABpEAQDsb0YAAAAAABhoRgAAAAAAAAAAAAAAAADM+0YAAAAAAAAAAAAAAAAA7GdGAAAAAADA +pUMAAAAAAMNTAAAIAAAAAAAAAO3RAABFRAEATUQBAAMAAAAVAAAAAAAABiLSAAAAAAAAcUQBADhs +RgAAAAAACGhGAAAAAAAAAAAAAAAAAOyCRgAAAAAAAAAAAAAAAAAwZ0YAAAAAAECmQwAAAAAA01MA +AAgAAAAAAAAAeEQBAIREAQCkRAEAAwAAABUAAAAAAAAGDkUBAAAAAAAURQEAOGxGAAAAAAAIaEYA +AAAAAAAAAAAAAAAAmOlGAAAAAAAAAAAAAAAAADBnRgAAAAAAoKdDAAAAAADlUwAAGAAAAAAAAAA/ +RQEAT0UBAGFFAQADAAAAFQAAAAAAAAa1RQEAwUUBAMlFAQB8bUYAAAAAAAhoRgAAAAAAAAAAAAAA +AAAAg0YAAAAAAAAAAAAAAAAAzGtGAAAAAABAqUMAAAAAAPZTAAAYAAAAAAAAANpFAQDuRQEA80UB +AAIAAAAVAAAAAAAABhFGAQAdRgEAAAAAAAxuRgAAAAAAGGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAC0Z0YAAAAAAECqQwAAAAAACFQAAAgAAAAAAAAAJkYBAERGAQByRgEAAwAAABUAAAAA +AAAGPEcBAEhHAQB0RwEAtHRGAAAAAAB0dEYAAAAAAAAAAAAAAAAARNRGAAAAAAAAAAAAAAAAADBn +RgAAAAAAgK9DAAAAAAAXVAAAEAAAAAAAAACtRwEAu0cBAMBHAQADAAAAFQAAAAAAAAbyRwEA+EcB +AP5HAQA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAwukYAAAAAAAAAAAAAAAAAZGdGAAAAAAAgsEMA +AAAAAIBUAAAYAAAAAAAAABFIAQAfSAEAJEgBAAMAAAAVAAAAAAAABmJIAQBoSAEAbkgBADhsRgAA +AAAACGhGAAAAAAAAAAAAAAAAAJS6RgAAAAAAAAAAAAAAAADsZ0YAAAAAAACxQwAAAAAAplQAAEgA +AAAAAAAAgkgBAKFIAQCmSAEAAwAAABUAAAAAAAAG6kkBAPZJAQAXSgEARG9GAAAAAAC0bkYAAAAA +AAAAAAAAAAAATOpGAAAAAAAAAAAAAAAAAICGRgAAAAAAoLZDAAAAAABmVQAACAAAAAAAAAB0SgEA +e0oBAH9KAQADAAAAFQAAAAAAAAaZSgEApEoBAKtKAQA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAU +g0YAAAAAAAAAAAAAAAAAMGdGAAAAAAAgt0MAAAAAAIVVAAAQAAAAAAAAALJKAQC5SgEAvUoBAAIA +AAAVAAAAAAAABmEpAQDKSgEAAAAAAAhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAABUZ0YAAAAAAKC3QwAAAAAAklUAAAAAAAAAAAAA0UoBAN9KAQD0SgEAAwAAABUAAAAAAAAE +GksBACZLAQAzSwEACGhGAAAAAADIbEYAAAAAAEBwRwAAAAAAtJlGAAAAAADAuEMAAAAAALhVAAAA +AAAAAAAAAEhLAQBUSwEAYEsBAAMAAAAVAAAAAAAABIJxAACGSwEAkksBAPhnRgAAAAAA+GdGAAAA +AAAAAAAAAAAAANCrRgAAAAAAALpDAAAAAADMVQAAAAAAAAAAAAClSwEAsksBALdLAQACAAAAFQAA +AAAAAALvSwEA+0sBAAAAAAD4Z0YAAAAAAPhnRgAAAAAAALxDAAAAAADhVQAAAAAAAAAAAAAETAEA +E0wBAB1MAQADAAAAFQAAAAAAAARAIQEArEwBAL1MAQD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAABo +kUYAAAAAAADAQwAAAAAA/1UAAAAAAAAAAAAAzkwBAN9MAQD4TAEAAwAAABUAAAAAAAAEVE0BAGBN +AQB4TQEAAGhGAAAAAADkdkYAAAAAAAAAAAAAAAAAKINGAAAAAAAgw0MAAAAAABZWAAAgAAAAAAAA +AJBNAQCcTQEAqU0BAAMAAAAVAAAAAAAABuFNAQDtTQEA+E0BADhsRgAAAAAACGhGAAAAAAAAAAAA +AAAAADyDRgAAAAAAAAAAAAAAAAAocEYAAAAAAEDEQwAAAAAAK1YAAAgAAAAAAAAABE4BAAdOAQAL +TgEAAQAAABUAAAAAAAAGE04BAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAsZ0YAAAAAAIDEQwAAAAAARVYAAAgAAAAAAAAAFk4BAB1OAQApTgEAAwAAABUAAAAAAAAG498A +AEBOAQBHTgEAOGxGAAAAAABobEYAAAAAAGBrRwAAAAAAUINGAAAAAAAAAAAAAAAAADBnRgAAAAAA +IMVDAAAAAABeVgAAAAAAAAAAAABSTgEAWk4BAHZOAQADAAAAFQAAAAAAAASoTgEAtE4BAL9OAQAY +aEYAAAAAANBtRgAAAAAAAAAAAAAAAADQ1EYAAAAAAMDFQwAAAAAAfVYAAAgAAAAAAAAA2k4BAOZO +AQD8TgEAAwAAABUAAAAAAAAGPAMBACpPAQA5TwEAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAIKxG +AAAAAAAAAAAAAAAAADBnRgAAAAAAoMZDAAAAAACYVgAAEAAAAAAAAABOTwEAVU8BAFlPAQACAAAA +FQAAAAAAAAZjrQAAa08BAAAAAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAZGdGAAAAAAAgx0MAAAAAAK5WAAAgAAAAAAAAAHRPAQCKTwEAnk8BAAMAAAAVAAAAAAAABmZQ +AQByUAEAjFABAHR5RgAAAAAAlHhGAAAAAAAAAAAAAAAAAJjfRgAAAAAAAAAAAAAAAADka0YAAAAA +AGDKQwAAAAAA6FYAABgAAAAAAAAA4VABAP5QAQAXUQEAAwAAABUAAAAAAAAGllEBAKJRAQC+UQEA +1HRGAAAAAADUdkYAAAAAAAAAAAAAAAAAOOBGAAAAAAAAAAAAAAAAAJRnRgAAAAAAgMxDAAAAAAAM +VwAAIAAAAAAAAADxUQEAAlIBAAxSAQADAAAAFQAAAAAAAAa9UgEAyVIBANVSAQD0bUYAAAAAABho +RgAAAAAAAAAAAAAAAABkg0YAAAAAAAAAAAAAAAAAYGtGAAAAAAAg0EMAAAAAACZXAAAQAAAAAAAA +AN5SAQDvUgEA9FIBAAIAAAAVAAAAAAAABoJTAQCOUwEAAAAAALxyRgAAAAAAAHRGAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAAGDTQwAAAAAAQlcAABAAAAAAAAAAmVMBAKVTAQCq +UwEAAgAAABUAAAAAAAAG5lMBAPJTAQAAAAAAfG1GAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAGRnRgAAAAAAoNRDAAAAAABhVwAAEAAAAAAAAACZUwEApVMBAPpTAQACAAAAFQAA +AAAAAAbmUwEA8lMBAAAAAAB8bUYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +ZGdGAAAAAADg1UMAAAAAAIFXAAAIAAAAAAAAADZUAQBAVAEAmFQBAAMAAAAVAAAAAAAABkNVAQBP +VQEAWVUBADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAGBYRwAAAAAAAAAAAAAAAAAwZ0YAAAAAAODb +QwAAAAAAi1kAABgAAAAAAAAABFYBAAxWAQA+VgEAAwAAABUAAAAAAAAG3DQAAI9WAQCWVgEAxG1G +AAAAAAAYaEYAAAAAAAAAAAAAAAAAbBVHAAAAAAAAAAAAAAAAALxnRgAAAAAA4NxDAAAAAAD9WQAA +CAAAAAAAAADVVgEA6FYBAPtWAQADAAAAFQAAAAAAAAZeVwEAZFcBAHNXAQD4Z0YAAAAAAPhnRgAA +AAAAAAAAAAAAAABwrEYAAAAAAAAAAAAAAAAALGdGAAAAAACg3kMAAAAAAE9aAAAQAAAAAAAAAJRX +AQCcVwEAzlcBAAMAAAAVAAAAAAAABiJYAQAuWAEANlgBAHxtRgAAAAAACGhGAAAAAAAAAAAAAAAA +AMAgRwAAAAAAAAAAAAAAAABkZ0YAAAAAAMDfQwAAAAAAr1oAAAgAAAAAAAAAcVgBAHRYAQB4WAEA +AQAAABUAAAAAAAAGfFgBACRrRgAAAAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAw +Z0YAAAAAAODfQwAAAAAAwVoAABgAAAAAAAAAf1gBAKJYAQCwWAEAAwAAABUAAAAAAAAGAlkBAAhZ +AQAeWQEAJHhGAAAAAACEeEYAAAAAAMBzRwAAAAAAwKxGAAAAAAAAAAAAAAAAANxnRgAAAAAA4OFD +AAAAAADwWgAAGAAAAAAAAAA1WQEAUlkBAFxZAQADAAAAFQAAAAAAAAbGWQEAzFkBANhZAQDocEYA +AAAAALByRgAAAAAAIHZHAAAAAADwmUYAAAAAAAAAAAAAAAAA3GdGAAAAAACg5EMAAAAAADtbAAAg +AAAAAAAAAOpZAQAoWgEArloBAAMAAAAVAAAAAAAABjRcAQBAXAEAgVwBAPh7RgAAAAAAOHxGAAAA +AABgc0cAAAAAAJBVRwAAAAAAAAAAAAAAAADwa0YAAAAAAODsQwAAAAAAtlsAAAAAAAAAAAAAGl0B +ACRdAQBYXQEAAwAAABUAAAATAAAE4F0BAOxdAQAPXgEA+GdGAAAAAAAwa0YAAAAAAKBzRwAAAAAA +AOtGAAAAAAAA8UMAAAAAACdcAAAIAAAAAAAAAEJeAQBMXgEAUV4BAAIAAAAVAAAAAAAABnFeAQB3 +XgEAAAAAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoZ0YAAAAAACDy +QwAAAAAAPVwAABAAAAAAAAAAfV4BAI1eAQCaXgEAAwAAABUAAAAAAAAGGksBAMZeAQDRXgEACGxG +AAAAAAAIaEYAAAAAAAAAAAAAAAAAkJFGAAAAAAAAAAAAAAAAAFRnRgAAAAAAQPNDAAAAAADzQQAA +AAAAAAAAAADdXgEA5F4BAOheAQACAAAAFQAAAAAAAAK0EwAA8F4BAAAAAAD4Z0YAAAAAAPhnRgAA +AAAAgPNDAAAAAABUXAAACAAAAAAAAAD1XgEA/F4BAABfAQACAAAAFQAAAAAAAAakBAEACl8BAAAA +AAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKGdGAAAAAAAA9EMAAAAA +AGpcAAAIAAAAAAAAAPVeAQD8XgEAEV8BAAIAAAAVAAAAAAAABqQEAQAKXwEAAAAAAPhnRgAAAAAA ++GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoZ0YAAAAAAID0QwAAAAAAgFwAAAgAAAAA +AAAA9V4BAPxeAQAbXwEAAgAAABUAAAAAAAAGpAQBAApfAQAAAAAA+GdGAAAAAAD4Z0YAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAChnRgAAAAAAAPVDAAAAAACZXAAAEAAAAAAAAAAlXwEALV8B +ADVfAQADAAAAFQAAAAAAAAZRXwEAV18BAF5fAQB0b0YAAAAAALRuRgAAAAAAAAAAAAAAAAB4g0YA +AAAAAAAAAAAAAAAAZGdGAAAAAADA9UMAAAAAAKtcAAAYAAAAAAAAAGVfAQCRXwEAm18BAAMAAAAV +AAAAAAAABttfAQDhXwEA6l8BAExtRgAAAAAAaGxGAAAAAABAc0cAAAAAAIyDRgAAAAAAAAAAAAAA +AADcZ0YAAAAAAKD3QwAAAAAAvFwAAAgAAAAAAAAA818BAP5fAQAKYAEAAwAAABUAAAAAAAAGIGAB +ACZgAQAvYAEA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAuJFGAAAAAAAAAAAAAAAAACxnRgAAAAAA +QPhDAAAAAADNXAAACAAAAAAAAAA6YAEAQ2ABAFFgAQADAAAAFQAAAAAAAAZoYAEAbmABAHNgAQD4 +Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAADgkUYAAAAAAAAAAAAAAAAAKGdGAAAAAADg+EMAAAAAAPJc +AAAAAAAAAAAAAIBgAQCHYAEAi2ABAAIAAAAVAAAAAAAAApdgAQCiYAEAAAAAAPhnRgAAAAAA+GdG +AAAAAAAg+UMAAAAAAAddAAAAAAAAAAAAAKlgAQCxYAEAtmABAAMAAAAVAAAAAAAABOpgAQD2YAEA +AGEBABhoRgAAAAAACG9GAAAAAABgcUcAAAAAAKCDRgAAAAAAgPpDAAAAAAAgXQAAAAAAAAAAAAAL +YQEAF2EBACdhAQADAAAAFQAAAAAAAAQYXgAAT2EBAFdhAQD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAA +AAAsmkYAAAAAAKD7QwAAAAAATV0AAAAAAAAAAAAAbGEBAHRhAQB5YQEAAwAAABUAAAAAAAAEIGAB +AJRhAQCbYQEA+GdGAAAAAAAwa0YAAAAAAAB1RwAAAAAAtINGAAAAAABA/EMAAAAAAGRdAAAIAAAA +AAAAAKZhAQCtYQEAtWEBAAMAAAAVAAAAAAAABsVhAQDKYQEAz2EBADhsRgAAAAAALGxGAAAAAACA +c0cAAAAAAMiDRgAAAAAAAAAAAAAAAAAwZ0YAAAAAAMD8QwAAAAAAeF0AAAgAAAAAAAAA1mEBAO5h +AQDzYQEAAgAAABUAAAAAAAAGD4wAADNiAQAAAAAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAChnRgAAAAAAQP5DAAAAAACIXQAAIAAAAAAAAAA7YgEASWIBAFNiAQADAAAA +FQAAAAAAAAaSYgEAnmIBALJiAQDEdUYAAAAAAGR2RgAAAAAAAAAAAAAAAAAIkkYAAAAAAAAAAAAA +AAAAhGtGAAAAAACg/0MAAAAAALhdAAAYAAAAAAAAAL5iAQDJYgEAzmIBAAMAAAAVAAAAAAAABvBi +AQD8YgEAC2MBADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAADCSRgAAAAAAAAAAAAAAAADsZ0YAAAAA +AIAARAAAAAAA5F0AACgAAAAAAAAAFmMBAC1jAQCSYwEAAwAAABUAAAAAAAAG22QBAOdkAQD6ZAEA +BHhGAAAAAAAkeUYAAAAAAAAAAAAAAAAAJDBHAAAAAAAAAAAAAAAAAGR3RgAAAAAAYAdEAAAAAAAL +XgAAAAAAAAAAAABnZQEAamUBAHZlAQADAAAAFQAAAAAAAASSZQEAAAAAAJVlAQD4Z0YAAAAAAPhn +RgAAAAAAAAAAAAAAAABYkkYAAAAAAMAHRAAAAAAAN14AAAgAAAAAAAAAoGUBAK1lAQC/ZQEAAwAA +ABUAAAAAAAAGE2YBAB9mAQAnZgEA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAaJpGAAAAAAAAAAAA +AAAAACxnRgAAAAAAYAlEAAAAAABkXgAAEAAAAAAAAAA4ZgEARGYBAGFmAQADAAAAFQAAAAAAAAYF +1AAArWYBALlmAQD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAACkmkYAAAAAAAAAAAAAAAAAPGdGAAAA +AAAAC0QAAAAAAHpeAAAQAAAAAAAAANlmAQDlZgEA/WYBAAMAAAAVAAAAAAAABjhnAQBEZwEAT2cB +AAhoRgAAAAAABG1GAAAAAAAAAAAAAAAAAPi6RgAAAAAAAAAAAAAAAAA8Z0YAAAAAACAMRAAAAAAA +k14AABAAAAAAAAAAYmcBAG5nAQCHZwEAAwAAABUAAAAAAAAGxGcBANBnAQDZZwEACGhGAAAAAAAE +bUYAAAAAAAAAAAAAAAAAXLtGAAAAAAAAAAAAAAAAADxnRgAAAAAAYA1EAAAAAACtXgAACAAAAAAA +AADsZwEA+GcBABBoAQADAAAAFQAAAAAAAAZHaAEAU2gBAGBoAQAIaEYAAAAAAARtRgAAAAAAAAAA +AAAAAADAu0YAAAAAAAAAAAAAAAAAMGdGAAAAAABgDkQAAAAAAMZeAAAIAAAAAAAAAHNoAQCLaAEA +y2gBAAMAAAAVAAAAAAAABoJpAQCOaQEAnGkBAAhoRgAAAAAABG1GAAAAAAAAAAAAAAAAAIQWRwAA +AAAAAAAAAAAAAAAoZ0YAAAAAAEARRAAAAAAA614AABAAAAAAAAAA2WkBAPBpAQA0agEAAwAAABUA +AAAAAAAG62oBAPdqAQARawEAGGhGAAAAAACsbUYAAAAAAAAAAAAAAAAAnBdHAAAAAAAAAAAAAAAA +AKxnRgAAAAAAYBREAAAAAAD9XgAAKAAAAAAAAABWawEAZmsBAHtrAQADAAAAFQAAAAAAAAbbawEA +52sBAPVrAQDgckYAAAAAAMRzRgAAAAAAAAAAAAAAAADgmkYAAAAAAAAAAAAAAAAANHdGAAAAAACA +FkQAAAAAABRfAAAQAAAAAAAAAAlsAQAmbAEAMGwBAAMAAAAVAAAAAAAABudsAQDzbAEACm0BAIR5 +RgAAAAAADIZGAAAAAAAAeUcAAAAAABCtRgAAAAAAAAAAAAAAAABkZ0YAAAAAAMAZRAAAAAAAPl8A +ABAAAAAAAAAAO20BAENtAQBIbQEAAwAAABUAAAAAAAAGzbEAAJptAQCibQEAfG1GAAAAAAAIaEYA +AAAAAAAAAAAAAAAAXNVGAAAAAAAAAAAAAAAAAGRnRgAAAAAAABtEAAAAAABTXwAAGAAAAAAAAADB +bQEAzW0BAORtAQADAAAAFQAAAAAAAAa83gAARm4BAFNuAQDUdUYAAAAAAHR2RgAAAAAAAAAAAAAA +AAAkvEYAAAAAAAAAAAAAAAAA7GdGAAAAAACAHEQAAAAAAIFfAAAQAAAAAAAAAGxuAQB9bgEAgm4B +AAMAAAAVAAAAAAAABg1vAQAZbwEAMG8BADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAANjgRgAAAAAA +AAAAAAAAAABkZ0YAAAAAAAAgRAAAAAAAzF8AAAAAAAAAAAAAU28BAGRvAQCKbwEAAwAAABUAAAAA +AAAEeXABAIVwAQDEcAEAKGhGAAAAAAA4nUYAAAAAAAAAAAAAAAAA6NVGAAAAAADAK0QAAAAAAN1f +AAAIAAAAAAAAAOlwAQAFcQEAE3EBAAMAAAAVAAAAAAAABpZBAABRcQEAYXEBADhsRgAAAAAACGhG +AAAAAAAAAAAAAAAAABybRgAAAAAAAAAAAAAAAAAwZ0YAAAAAAGAtRAAAAAAA8V8AAAAAAAAAAAAA +cXEBAIJxAQCncQEAAwAAABUAAAAAAAAEtUUBAAFyAQAUcgEACGhGAAAAAAAEbUYAAAAAAAAAAAAA +AAAAeOFGAAAAAAAAL0QAAAAAAAhgAAAYAAAAAAAAADByAQBHcgEAx3IBAAMAAAAVAAAAAAAABgt0 +AQAXdAEASXQBABB9RgAAAAAAWIdGAAAAAADgb0cAAAAAALQYRwAAAAAAAAAAAAAAAAC0Z0YAAAAA +AOA2RAAAAAAAMWAAAAAAAAAAAAAA0nQBAN10AQDhdAEAAgAAABUAAAAAAAACHeEAAO10AQAAAAAA ++GdGAAAAAAD4Z0YAAAAAAEA3RAAAAAAAQGAAACAAAAAAAAAA9HQBABF1AQAWdQEAAwAAABUAAAAA +AAAGmXUBAKV1AQC3dQEA0HNGAAAAAABgcUYAAAAAAEBtRwAAAAAA3INGAAAAAAAAAAAAAAAAAKBw +RgAAAAAA4DlEAAAAAABwYAAAKAAAAAAAAADAdQEAyXUBAM51AQACAAAAFQAAAAAAAAbVdQEA4HUB +AAAAAACUbUYAAAAAAOxsRgAAAAAAAG1HAAAAAAAAAAAAAAAAAAAAAAAAAAAA1HpGAAAAAACAOkQA +AAAAAIZgAABIAAAAAAAAAOd1AQDxdQEA9nUBAAIAAAAVAAAAAAAABv51AQAKdgEAAAAAACBsRgAA +AAAAvGxGAAAAAAAgbUcAAAAAAAAAAAAAAAAAAAAAAAAAAADkhkYAAAAAAKA7RAAAAAAAnGAAABgA +AAAAAAAAEnYBACF2AQAmdgEAAgAAABUAAAAAAAAGTHYBAFh2AQAAAAAATHNGAAAAAABwc0YAAAAA +AOBzRwAAAAAAAAAAAAAAAAAAAAAAAAAAAOxnRgAAAAAAoDxEAAAAAAC2YAAAEAAAAAAAAABhdgEA +bHYBAHF2AQACAAAAFQAAAAAAAAZoAgAAiXYBAAAAAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAZGdGAAAAAABAPUQAAAAAAMtgAAAIAAAAAAAAAJB2AQCZdgEAnnYBAAIA +AAAVAAAAAAAABpShAACydgEAAAAAAAhoRgAAAAAA1GxGAAAAAAAgeEcAAAAAAAAAAAAAAAAAAAAA +AAAAAAAwZ0YAAAAAAOA9RAAAAAAA3WAAABAAAAAAAAAAuXYBAL12AQDCdgEAAQAAABUAAAAAAAAG +/HYBADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACsZ0YAAAAAAMA+RAAA +AAAA6mAAAAgAAAAAAAAAAncBABJ3AQAXdwEAAgAAABUAAAAAAAAGL3cBADV3AQAAAAAAAG5GAAAA +AABgbkYAAAAAAGBuRwAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAAAAAAgD9EAAAAAAD7YAAAAAAA +AAAAAAAV/QAAO3cBAEN3AQADAAAAFQAAAAAAAAQw/QAANf0AADr9AAD4Z0YAAAAAAPhnRgAAAAAA +AAAAAAAAAADwg0YAAAAAAOA/RAAAAAAA/w4AAAAAAAAAAAAA6QUAAE13AQBRdwEAAQAAABUAAAAA +AAAC9QUAAPhnRgAAAAAA+GdGAAAAAAAgQEQAAAAAABJhAAAAAAAAAAAAAGF3AQBxdwEAfHcBAAMA +AAAVAAAAAAAABOrFAACwdwEAwHcBABhoRgAAAAAA2G5GAAAAAAAAAAAAAAAAAASERgAAAAAAYEJE +AAAAAAAmYQAACAAAAAAAAADIdwEA2XcBAN53AQACAAAAFQAAAAAAAAZKeAEAVngBAAAAAAD8fEYA +AAAAAByHRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAAAAAgSUQAAAAAAEBhAAAI +AAAAAAAAAIZ4AQCYeAEAoXgBAAMAAAAVAAAAAAAABuF4AQDneAEA7XgBAPhnRgAAAAAA+GdGAAAA +AAAAAAAAAAAAAICSRgAAAAAAAAAAAAAAAAAwZ0YAAAAAAOBKRAAAAAAAaWEAADAAAAAAAAAA+XgB +ACR5AQA2eQEAAwAAABUAAAAAAAAG2nkBAOZ5AQABegEAmHtGAAAAAAB4e0YAAAAAAAAAAAAAAAAA +YK1GAAAAAAAAAAAAAAAAAOR6RgAAAAAAAFBEAAAAAACRYQAAEAAAAAAAAAAcegEAJ3oBAC96AQAD +AAAAFQAAAAAAAAZHegEAUnoBAFt6AQC4bUYAAAAAAGBuRgAAAAAAwG5HAAAAAACokkYAAAAAAAAA +AAAAAAAArGdGAAAAAADAUEQAAAAAALRhAAAQAAAAAAAAAGp6AQB1egEAenoBAAIAAAAVAAAAAAAA +BpJ6AQCdegEAAAAAAHxtRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACsZ0YA +AAAAAGBRRAAAAAAAyGEAABgAAAAAAAAAqHoBALN6AQC7egEAAwAAABUAAAAAAAAG03oBAN96AQDo +egEAuG1GAAAAAABgbkYAAAAAAOBuRwAAAAAA0JJGAAAAAAAAAAAAAAAAAHhrRgAAAAAAIFJEAAAA +AAABYgAAGAAAAAAAAAD3egEACnsBABJ7AQADAAAAFQAAAAAAAAYwewEAPHsBAEV7AQC4bUYAAAAA +AGBuRgAAAAAAoG5HAAAAAAAYhEYAAAAAAAAAAAAAAAAAeGtGAAAAAAAgU0QAAAAAABJiAAAgAAAA +AAAAAEx7AQBiewEAZ3sBAAIAAAAVAAAAAAAABn97AQCLewEAAAAAALhtRgAAAAAAmG9GAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAABMcEYAAAAAAEBURAAAAAAAJGIAABgAAAAAAAAAlHsBAJt7 +AQCfewEAAgAAABUAAAAAAAAGp3sBALJ7AQAAAAAAfG1GAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAPxrRgAAAAAAoFREAAAAAAA1YgAAIAAAAAAAAAC5ewEAw3sBAMh7AQACAAAA +FQAAAAAAAAbZewEA5XsBAAAAAAB0dUYAAAAAAIR2RgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAWHBGAAAAAAAAVkQAAAAAAEliAAAQAAAAAAAAAPJ7AQACfAEAB3wBAAIAAAAVAAAAAAAABi18 +AQA5fAEAAAAAALhtRgAAAAAAtG5GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACsZ0YAAAAA +AABXRAAAAAAAYGIAACgAAAAAAAAAQ3wBAE58AQBWfAEAAwAAABUAAAAAAAAGbHwBAHd8AQB+fAEA +jGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAA+JJGAAAAAAAAAAAAAAAAAJR3RgAAAAAAoFdEAAAAAACI +YgAAIAAAAAAAAACJfAEAlHwBAJx8AQADAAAAFQAAAAAAAAawfAEAu3wBAMJ8AQB8bUYAAAAAAAho +RgAAAAAAAAAAAAAAAAAgk0YAAAAAAAAAAAAAAAAAiHBGAAAAAABAWEQAAAAAAJ1iAAAwAAAAAAAA +AM18AQDcfAEA4XwBAAMAAAAVAAAAAAAABj99AQBFfQEATH0BALBsRgAAAAAACGhGAAAAAAAAAAAA +AAAAAEiTRgAAAAAAAAAAAAAAAAAke0YAAAAAAOBZRAAAAAAAvWIAABAAAAAAAAAAb30BAH99AQCM +fQEAAwAAABUAAAAAAAAGwn0BAM59AQDcfQEAfG1GAAAAAAAIaEYAAAAAAAAAAAAAAAAALIRGAAAA +AAAAAAAAAAAAAGRnRgAAAAAAgFtEAAAAAADQYgAACAAAAAAAAADofQEA/H0BADR+AQADAAAAFQAA +AAAAAAahdAAA1H4BAOZ+AQBscUYAAAAAAOhzRgAAAAAAAAAAAAAAAAAY4kYAAAAAAAAAAAAAAAAA +MGdGAAAAAAAgXkQAAAAAAOFiAAAQAAAAAAAAAB1/AQApfwEALn8BAAMAAAAVAAAAAAAABmp/AQB2 +fwEAgn8BADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAECERgAAAAAAAAAAAAAAAABkZ0YAAAAAACBg +RAAAAAAADWMAAAgAAAAAAAAAjn8BAJp/AQCffwEAAwAAABUAAAAAAAAG1X8BAOF/AQDtfwEAOGxG +AAAAAAAIaEYAAAAAAAAAAAAAAAAAVIRGAAAAAAAAAAAAAAAAADBnRgAAAAAAwGFEAAAAAAAhYwAA +OAAAAAAAAAD5fwEACIABAFaAAQADAAAAFQAAAAAAAAZ3gQEAg4EBAKCBAQB0ekYAAAAAAGR6RgAA +AAAAAAAAAAAAAADMGUcAAAAAAAAAAAAAAAAA1HxGAAAAAADAZkQAAAAAAFZjAAAgAAAAAAAAAPCB +AQD9gQEAAoIBAAIAAAAVAAAAAAAABmaCAQByggEAAAAAAEBzRgAAAAAA2HFGAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAACgcEYAAAAAACBpRAAAAAAAaWMAABAAAAAAAAAAhIIBAJqCAQCfggEA +AgAAABUAAAAAAAAGFYMBACGDAQAAAAAAtHRGAAAAAADEdEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAGRnRgAAAAAAQGxEAAAAAAB+YwAAIAAAAAAAAAA5gwEAQYMBAEaDAQACAAAAFQAAAAAA +AAZegwEAaoMBAAAAAABAc0YAAAAAANhxRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoHBG +AAAAAAAgbUQAAAAAAJhjAAAQAAAAAAAAAHODAQCHgwEAjIMBAAIAAAAVAAAAAAAABvqDAQAGhAEA +AAAAAIxvRgAAAAAAtG5GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkZ0YAAAAAAOBvRAAA +AAAAqWMAABgAAAAAAAAAF4QBACGEAQA2hAEAAwAAABUAAAAAAAAGE2YBAH2EAQCThAEARHhGAAAA +AAA0eUYAAAAAAAAAAAAAAAAAiLxGAAAAAAAAAAAAAAAAAOxnRgAAAAAAgHFEAAAAAAC9YwAACAAA +AAAAAACnhAEAuIQBAL2EAQADAAAAFQAAAAAAAAa5hQEAxYUBAN2FAQDQcEYAAAAAAMhyRgAAAAAA +AAAAAAAAAABohEYAAAAAAAAAAAAAAAAAMGdGAAAAAAAgdkQAAAAAANhjAAAAAAAAAAAAAOaFAQDu +hQEABoYBAAMAAAAVAAAAAAAABNwGAAA1hgEAP4YBAAhoRgAAAAAABG1GAAAAAAAAAAAAAAAAALCt +RgAAAAAAAHdEAAAAAADvYwAAIAAAAAAAAABQhgEAXIYBAGGGAQACAAAAFQAAAAAAAAaSYgEAmYYB +AAAAAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKHBGAAAAAABgeEQA +AAAAAANkAAAgAAAAAAAAAKmGAQC1hgEAuoYBAAIAAAAVAAAAAAAABh6HAQAqhwEAAAAAADhsRgAA +AAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAocEYAAAAAAIB6RAAAAAAAGWQAAAAA +AAAAAAAABycAADqHAQA+hwEAAgAAABUAAAAAAAACGScAACQnAAAAAAAA+GdGAAAAAAD4Z0YAAAAA +AMB6RAAAAAAAKmQAAAAAAAAAAAAARocBAFWHAQB1hwEAAwAAABUAAAAAAAAEUWAAAKmHAQC4hwEA +CGhGAAAAAAAEbUYAAAAAAAAAAAAAAAAAuOJGAAAAAACAe0QAAAAAAD5kAAAIAAAAAAAAANOHAQDf +hwEA84cBAAMAAAAVAAAAAAAABtyhAAAliAEALIgBADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAIzH +RgAAAAAAAAAAAAAAAAAwZ0YAAAAAAEB8RAAAAAAAg2QAACgAAAAAAAAAQYgBAFGIAQBWiAEAAgAA +ABUAAAAAAAAGQgcBAHaIAQAAAAAAEG1GAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAALR3RgAAAAAAQH1EAAAAAACWZAAASAAAAAAAAAB/iAEAkIgBAJWIAQADAAAAFQAAAAAAAAaL +iQEAl4kBAKuJAQC0dUYAAAAAAPR0RgAAAAAAAAAAAAAAAAC060YAAAAAAAAAAAAAAAAAaIZGAAAA +AADAgUQAAAAAAP5kAAAoAAAAAAAAAO+JAQD7iQEAAIoBAAIAAAAVAAAAAAAABj6KAQBKigEAAAAA +AOxvRgAAAAAAtG5GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABkd0YAAAAAAOCCRAAAAAAA +E2UAAAAAAAAAAAAAU4oBAF6KAQB2igEAAwAAABUAAAAAAAAE/4cAAJqKAQChigEACGhGAAAAAAAE +bUYAAAAAAAAAAAAAAAAAWJtGAAAAAABgg0QAAAAAAC5lAAAIAAAAAAAAALaKAQC9igEAzYoBAAMA +AAAVAAAAAAAABvEpAADqigEA8YoBAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAJSbRgAAAAAAAAAA +AAAAAAAoZ0YAAAAAAOCDRAAAAAAASWUAABAAAAAAAAAA/ooBAAqLAQAgiwEAAwAAABUAAAAAAAAG +w4sBAM+LAQDeiwEACGhGAAAAAAAEbUYAAAAAAAAAAAAAAAAA+AZHAAAAAAAAAAAAAAAAAERnRgAA +AAAAYIZEAAAAAABcZQAAIAAAAAAAAAAXjAEAOYwBAFWMAQADAAAAFQAAAAAAAAapjAEAtYwBAMeM +AQCUc0YAAAAAANRyRgAAAAAAAAAAAAAAAAB01kYAAAAAAAAAAAAAAAAAoHBGAAAAAABAiEQAAAAA +AJdlAAAoAAAAAAAAAOSMAQDxjAEA9owBAAMAAAAVAAAAAAAABhCNAQAVjQEAG40BAHxtRgAAAAAA +CGhGAAAAAAAAAAAAAAAAAHyERgAAAAAAAAAAAAAAAABkd0YAAAAAAOCIRAAAAAAAt2UAABAAAAAA +AAAAJI0BACuNAQAvjQEAAgAAABUAAAAAAAAGAHoAADeNAQAAAAAAOGxGAAAAAAAIaEYAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRnRgAAAAAAQIlEAAAAAADbZQAAEAAAAAAAAAA+jQEASo0B +AFeNAQADAAAAFQAAAAAAAAY9ngAAjY0BAJiNAQA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAADQm0YA +AAAAAAAAAAAAAAAAZGdGAAAAAABgikQAAAAAABlmAAAAAAAAAAAAAKyNAQCzjQEAt40BAAIAAAAV +AAAAAAAAAqitAADBjQEAAAAAAPhnRgAAAAAA+GdGAAAAAADAikQAAAAAADBmAAAIAAAAAAAAAMiN +AQDRjQEA5Y0BAAMAAAAVAAAAAAAABiu0AAAPjgEAFo4BAABuRgAAAAAAtG5GAAAAAAAAAAAAAAAA +AAycRgAAAAAAAAAAAAAAAAAwZ0YAAAAAAICLRAAAAAAARmYAAAAAAAAAAAAAKY4BALONAQA0jgEA +AgAAABUAAAAAAAACqK0AAESOAQAAAAAA+GdGAAAAAAD4Z0YAAAAAAOCLRAAAAAAAYGYAAAgAAAAA +AAAAS44BAFKOAQBajgEAAwAAABUAAAAAAAAGlKEAAHKOAQB5jgEACGhGAAAAAAAEbUYAAAAAAAAA +AAAAAAAAkIRGAAAAAAAAAAAAAAAAADBnRgAAAAAAgIxEAAAAAAB5ZgAAAAAAAAAAAACAjgEAj44B +AJeOAQADAAAAFQAAAAAAAASvjgEAu44BAMSOAQAIaEYAAAAAAARtRgAAAAAAAAAAAAAAAACkhEYA +AAAAAECNRAAAAAAAkmYAABAAAAAAAAAAy44BANSOAQDZjgEAAgAAABUAAAAAAAAG6Y4BAPWOAQAA +AAAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRnRgAAAAAAII5EAAAA +AACoZgAAAAAAAAAAAAD+jgEADo8BABePAQADAAAAFQAAAAAAAASSYgEAN48BAESPAQD4Z0YAAAAA +APhnRgAAAAAAAAAAAAAAAAC4hEYAAAAAAICPRAAAAAAACU0AAAAAAAAAAAAATI8BAFOPAQBXjwEA +AgAAABUAAAAAAAACHeEAAGWPAQAAAAAA+GdGAAAAAAD4Z0YAAAAAAOCPRAAAAAAAvWYAABAAAAAA +AAAAbI8BAHOPAQB3jwEAAgAAABUAAAAAAAAGwhEAAImPAQAAAAAA+GdGAAAAAAD4Z0YAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAHRnRgAAAAAAYJBEAAAAAADRZgAAEAAAAAAAAACQjwEAmo8B +AJ+PAQACAAAAFQAAAAAAAAYGYgAAs48BAAAAAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAZGdGAAAAAABAkUQAAAAAAJdOAAAAAAAAAAAAANfNAAC6jwEAvo8BAAIAAAAV +AAAAAAAAAvTNAAD/zQAAAAAAAPhnRgAAAAAA+GdGAAAAAACAkUQAAAAAAOdmAAAIAAAAAAAAAMiP +AQCajwEA0o8BAAIAAAAVAAAAAAAABtwGAADkjwEAAAAAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAwZ0YAAAAAAGCSRAAAAAAA/mYAAAgAAAAAAAAAyI0BANGNAQDsjwEA +AwAAABUAAAAAAAAGK7QAAA+OAQAWjgEAAG5GAAAAAAC0bkYAAAAAAAAAAAAAAAAASJxGAAAAAAAA +AAAAAAAAADBnRgAAAAAAIJNEAAAAAAAWZwAAAAAAAAAAAAAWkAEAHpABACOQAQACAAAAFQAAAAAA +AAL0iAAAMZABAAAAAAD4Z0YAAAAAAPhnRgAAAAAAwJNEAAAAAAAsZwAAGAAAAAAAAAA4kAEATpAB +AGKQAQADAAAAFQAAAAAAAAa3kAEAw5ABANOQAQAYcUYAAAAAAGh7RgAAAAAAgHRHAAAAAAAArkYA +AAAAAAAAAAAAAAAA7GdGAAAAAADglUQAAAAAAFtnAABQAAAAAAAAAO6QAQAMkQEAlJEBAAMAAAAV +AAAAAAAABr6WAQDKlgEANpcBAIyURgAAAAAA5MlGAAAAAADAb0cAAAAAAAgQRwAAAAAAAAAAAAAA +AAA4hkYAAAAAAECwRAAAAAAAjGcAABgAAAAAAAAA25cBAPKXAQATmAEAAwAAABUAAAAAAAAGr5gB +ALuYAQDVmAEApHZGAAAAAACkdUYAAAAAAGByRwAAAAAAANdGAAAAAAAAAAAAAAAAAPxrRgAAAAAA +YLNEAAAAAAC2ZwAACAAAAAAAAAAJmQEAEJkBAByZAQADAAAAFQAAAAAAAAb+rQAANZkBADyZAQD4 +Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAABwk0YAAAAAAAAAAAAAAAAAXGdGAAAAAADgs0QAAAAAAM5n +AAAoAAAAAAAAAEWZAQBVmQEAWpkBAAIAAAAVAAAAAAAABpKZAQCemQEAAAAAAOxyRgAAAAAARHJG +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUd0YAAAAAAKC1RAAAAAAA4WcAACgAAAAAAAAA +rJkBALyZAQDBmQEAAgAAABUAAAAAAAAGC5oBABeaAQAAAAAAjG9GAAAAAAAwbkYAAAAAAEB0RwAA +AAAAAAAAAAAAAAAAAAAAAAAAAJRwRgAAAAAAQLhEAAAAAAD9ZwAACAAAAAAAAAAlmgEALZoBADWa +AQADAAAAFQAAAAAAAAZRmgEAXJoBAGeaAQCMb0YAAAAAAJhvRgAAAAAAAAAAAAAAAADMhEYAAAAA +AAAAAAAAAAAAMGdGAAAAAADguEQAAAAAABRoAAAYAAAAAAAAAG6aAQB4mgEAhpoBAAMAAAAVAAAA +AAAABqyaAQC4mgEAy5oBAPR4RgAAAAAA1HhGAAAAAAAAAAAAAAAAAOCERgAAAAAAAAAAAAAAAAD8 +a0YAAAAAAIC6RAAAAAAAnDcAACAAAAAAAAAA2JoBAN+aAQDjmgEAAgAAABUAAAAAAAAG7ZoBAPia +AQAAAAAAHG1GAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIRrRgAAAAAA4LpE +AAAAAAAsaAAAIAAAAAAAAAD/mgEACpsBAA+bAQACAAAAFQAAAAAAAAYfmwEAKpsBAAAAAAAcbUYA +AAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAhGtGAAAAAACAu0QAAAAAAEJoAAAo +AAAAAAAAADGbAQBImwEAUpsBAAMAAAAVAAAAAAAABpabAQCimwEAtJsBAPxuRgAAAAAAJG5GAAAA +AAAAAAAAAAAAAPSERgAAAAAAAAAAAAAAAAA0cEYAAAAAAKC+RAAAAAAAVWgAACgAAAAAAAAAvZsB +AMebAQDRmwEAAwAAABUAAAAAAAAGijYAAP2bAQAQnAEAtHRGAAAAAABUdkYAAAAAAAAAAAAAAAAA +CIVGAAAAAAAAAAAAAAAAAPR6RgAAAAAAYMBEAAAAAAB0aAAAGAAAAAAAAAAZnAEAKZwBAFacAQAD +AAAAFQAAAAAAAAbEnAEA0JwBAOmcAQBwfEYAAAAAAFx8RgAAAAAAAAAAAAAAAABQrkYAAAAAAAAA +AAAAAAAA/GtGAAAAAABAw0QAAAAAAJtoAAAgAAAAAAAAABudAQAlnQEAKp0BAAIAAAAVAAAAAAAA +BjedAQBDnQEAAAAAAAhsRgAAAAAAKHNGAAAAAAAgbEcAAAAAAAAAAAAAAAAAAAAAAAAAAACgcEYA +AAAAAEDERAAAAAAAq2gAAAAAAAAAAAAAS50BAFqdAQBenQEAAgAAABUAAAAAAAACbp0BAHmdAQAA +AAAACGhGAAAAAAAEbUYAAAAAAMDERAAAAAAAeDcAACgAAAAAAAAAgp0BAJGdAQCWnQEAAgAAABUA +AAAAAAAGnp0BAKmdAQAAAAAAgGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAR3RgAAAAAAYMVEAAAAAADBaAAAIAAAAAAAAACwnQEAu50BAMCdAQACAAAAFQAAAAAAAAbOnQEA +2p0BAAAAAAD4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFHtGAAAAAAAg +xkQAAAAAANNoAAAYAAAAAAAAAOGdAQD5nQEAEZ4BAAMAAAAVAAAAAAAABlaeAQBingEAcJ4BALht +RgAAAAAALG9GAAAAAAAAAAAAAAAAAOy8RgAAAAAAAAAAAAAAAACkd0YAAAAAAADIRAAAAAAAAmkA +AAgAAAAAAAAAjJ4BAJWeAQCqngEAAwAAABUAAAAAAAAG+J4BAASfAQARnwEAjG9GAAAAAAC0bkYA +AAAAAAAAAAAAAAAAhJxGAAAAAAAAAAAAAAAAADBnRgAAAAAAQMpEAAAAAAAaaQAACAAAAAAAAAAl +nwEAL58BAECfAQADAAAAFQAAAAAAAAZ2nwEAgp8BAIufAQCMb0YAAAAAACRuRgAAAAAAoHRHAAAA +AACYk0YAAAAAAAAAAAAAAAAAMGdGAAAAAABAy0QAAAAAADJpAAAIAAAAAAAAAJ6fAQCpnwEAtZ8B +AAMAAAAVAAAAAAAABnyMAADlnwEA+p8BADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAMCcRgAAAAAA +AAAAAAAAAAAwZ0YAAAAAAGDMRAAAAAAAUGkAACAAAAAAAAAACaABABOgAQAYoAEAAgAAABUAAAAA +AAAG1DsAAFOgAQAAAAAAEG1GAAAAAAAsbEYAAAAAAGB0RwAAAAAAAAAAAAAAAAAAAAAAAAAAAFhw +RgAAAAAAQM5EAAAAAABpaQAACAAAAAAAAABdoAEAYKABAGSgAQABAAAAFQAAAAAAAAZ2oAEA+GdG +AAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAAAAAAgM5EAAAAAACIaQAA +EAAAAAAAAAB5oAEAkKABAJygAQADAAAAFQAAAAAAAAZMMgAAwqABAMmgAQA4bEYAAAAAAAhoRgAA +AAAAAAAAAAAAAADAk0YAAAAAAAAAAAAAAAAAPGdGAAAAAABgz0QAAAAAAKJpAAAIAAAAAAAAANSg +AQDkoAEA6aABAAIAAAAVAAAAAAAABl7VAAANoQEAAAAAAIxvRgAAAAAAMG5GAAAAAACAckcAAAAA +AAAAAAAAAAAAAAAAAAAAAAAwZ0YAAAAAAKDQRAAAAAAAvGkAABgAAAAAAAAAGqEBACShAQA1oQEA +AwAAABUAAAAAAAAGbaEBAHmhAQCMoQEApHRGAAAAAACUdkYAAAAAAKB3RwAAAAAA6JNGAAAAAAAA +AAAAAAAAAOxnRgAAAAAAoNJEAAAAAADZaQAACAAAAAAAAACcoQEAo6EBAKehAQACAAAAFQAAAAAA +AAazoQEAvqEBAAAAAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdG +AAAAAAAg00QAAAAAAPNpAAAgAAAAAAAAAMWhAQDVoQEA2qEBAAIAAAAVAAAAAAAABvahAQACogEA +AAAAAAhsRgAAAAAA7GxGAAAAAACAbEcAAAAAAAAAAAAAAAAAAAAAAAAAAACgcEYAAAAAAADURAAA +AAAACWoAAAgAAAAAAAAADqIBAB2iAQAiogEAAwAAABUAAAAAAAAGH5sBADuiAQBGogEAOGxGAAAA +AAAIaEYAAAAAAAAAAAAAAAAAHIVGAAAAAAAAAAAAAAAAADBnRgAAAAAAoNREAAAAAAAhagAACAAA +AAAAAABRogEAVKIBAFiiAQABAAAAFQAAAAAAAAaBogEAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAADBnRgAAAAAAINVEAAAAAAA7agAACAAAAAAAAACEogEAl6IBAJyiAQAD +AAAAFQAAAAAAAAYgsQAAuKIBAMWiAQA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAwhUYAAAAAAAAA +AAAAAAAAMGdGAAAAAADg1UQAAAAAAFRqAAAQAAAAAAAAAMyiAQDpogEA/qIBAAMAAAAVAAAAAAAA +BkmjAQBVowEAZ6MBAGxxRgAAAAAA9HNGAAAAAAAAAAAAAAAAAATIRgAAAAAAAAAAAAAAAABEZ0YA +AAAAAIDYRAAAAAAAnWoAABAAAAAAAAAAe6MBAJ+jAQC1owEAAwAAABUAAAAAAAAGLxIBAAakAQAc +pAEA1HRGAAAAAABUdUYAAAAAAAAAAAAAAAAAfMhGAAAAAAAAAAAAAAAAAERnRgAAAAAAQNtEAAAA +AAC0agAAEAAAAAAAAAAxpAEAT6QBAGikAQADAAAAFQAAAAAAAAbLpAEA16QBAOqkAQBscUYAAAAA +APRzRgAAAAAAAAAAAAAAAACM10YAAAAAAAAAAAAAAAAARGdGAAAAAABA3kQAAAAAAM1qAAAIAAAA +AAAAAAKlAQATpQEAH6UBAAMAAAAVAAAAAAAABlGlAQBXpQEAXaUBADhsRgAAAAAAaGxGAAAAAADA +cUcAAAAAAFC9RgAAAAAAAAAAAAAAAABsZ0YAAAAAAADfRAAAAAAACWsAAAgAAAAAAAAAeKUBAIal +AQCWpQEAAwAAABUAAAAAAAAG3qUBAOSlAQDqpQEAOGxGAAAAAABobEYAAAAAAOBxRwAAAAAAaOxG +AAAAAAAAAAAAAAAAAGxnRgAAAAAAAOBEAAAAAAAaawAACAAAAAAAAAAVpgEAJaYBADmmAQADAAAA +FQAAAAAAAAaRpgEAnaYBAKWmAQA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAABY40YAAAAAAAAAAAAA +AAAAbGdGAAAAAABg4UQAAAAAAC9rAAAAAAAAAAAAANimAQDwpgEA+qYBAAMAAAAVAAAAAAAABHmn +AQCFpwEAqqcBABBoRgAAAAAAQK9GAAAAAADAeUcAAAAAAESFRgAAAAAAQOdEAAAAAABFawAAGAAA +AAAAAACzpwEAeKgBAJOoAQADAAAAFQAAAAAAAAbbqQEA56kBAEmqAQA4h0YAAAAAAHidRgAAAAAA +4HRHAAAAAAAk9EYAAAAAAAAAAAAAAAAA7GdGAAAAAAAA9EQAAAAAAIdrAAAQAAAAAAAAAJ6qAQCz +qgEAuKoBAAIAAAAVAAAAAAAABpSrAQCgqwEAAAAAAHxtRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAABkZ0YAAAAAAKD4RAAAAAAAp2sAABAAAAAAAAAAqKsBALurAQDMqwEAAwAA +ABUAAAAAAAAG5lMBAAKsAQALrAEAWHNGAAAAAAAUckYAAAAAACB1RwAAAAAAEJRGAAAAAAAAAAAA +AAAAAGRnRgAAAAAA4PlEAAAAAAC/awAAEAAAAAAAAAAbrAEAOawBAD6sAQACAAAAFQAAAAAAAAbG +rAEA0qwBAAAAAACscEYAAAAAAIByRgAAAAAAYHVHAAAAAAAAAAAAAAAAAAAAAAAAAAAARGdGAAAA +AACA/UQAAAAAANhrAAAoAAAAAAAAAOOsAQD/rAEACa0BAAMAAAAVAAAAAAAABkWtAQBRrQEAW60B +AKxzRgAAAAAAIHJGAAAAAABAdUcAAAAAAPycRgAAAAAAAAAAAAAAAADAfEYAAAAAAED/RAAAAAAA +IWwAABAAAAAAAAAAa60BAHqtAQB/rQEAAgAAABUAAAAAAAAGk60BAJ+tAQAAAAAACGhGAAAAAAAA +cUYAAAAAAIB1RwAAAAAAAAAAAAAAAAAAAAAAAAAAAGRnRgAAAAAAAABFAAAAAAAybAAACAAAAAAA +AACorQEAq60BAK+tAQABAAAAFQAAAAAAAAa/rQEA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAADBnRgAAAAAAYABFAAAAAABFbAAACAAAAAAAAADCrQEAyq0BAOWtAQADAAAA +FQAAAAAAAAaiwQAAGq4BACWuAQAcc0YAAAAAAGRzRgAAAAAAAAAAAAAAAAD0yEYAAAAAAAAAAAAA +AAAAMGdGAAAAAAAgAUUAAAAAAGFsAAAIAAAAAAAAAD6uAQBGrgEAi64BAAMAAAAVAAAAAAAABgKv +AQAOrwEAHK8BABxzRgAAAAAAZHNGAAAAAAAAAAAAAAAAACwoRwAAAAAAAAAAAAAAAAAwZ0YAAAAA +AKACRQAAAAAAfmwAAAAAAAAAAAAAnTEBAFuvAQBerwEAAgAAABUAAAAAAAACrDEBALcxAQAAAAAA ++GdGAAAAAAD4Z0YAAAAAAOACRQAAAAAAlGwAAAgAAAAAAAAAaK8BAG+vAQByrwEAAgAAABUAAAAA +AAAGhK8BAI+vAQAAAAAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBn +RgAAAAAAYANFAAAAAACvbAAAAAAAAAAAAACWrwEAna8BAKCvAQACAAAAFQAAAAAAAAISdgAAsK8B +AAAAAAAIaEYAAAAAAARtRgAAAAAA4ANFAAAAAADIbAAAAAAAAAAAAAC5rwEAxa8BANWvAQADAAAA +FQAAAAAAAAQYXgAA+68BAAewAQD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAABYhUYAAAAAAAAFRQAA +AAAA4WwAAAAAAAAAAAAAF7ABAB+wAQAjsAEAAwAAABUAAAAAAAAE8GIBAEmwAQBYsAEACGhGAAAA +AAAEbUYAAAAAAAAAAAAAAAAAbIVGAAAAAADgBUUAAAAAAAFtAAAAAAAAAAAAAF+wAQBmsAEAabAB +AAIAAAAVAAAAAAAAAm2wAQB4sAEAAAAAAPhnRgAAAAAA+GdGAAAAAAAgBkUAAAAAACFtAAAIAAAA +AAAAAH+wAQCGsAEAibABAAIAAAAVAAAAAAAABpOwAQCesAEAAAAAADhsRgAAAAAACGhGAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwZ0YAAAAAAGAGRQAAAAAAQ20AAAAAAAAAAAAAIxEAAKWw +AQCosAEAAgAAABUAAAAAAAACQBEAALKwAQAAAAAA+GdGAAAAAAD4Z0YAAAAAAKAGRQAAAAAAY20A +ABAAAAAAAAAAubABAMCwAQDHsAEAAwAAABUAAAAAAAAG8AcAAN2wAQDksAEACGxGAAAAAAAIaEYA +AAAAAAAAAAAAAAAAgIVGAAAAAAAAAAAAAAAAAGRnRgAAAAAAIAdFAAAAAACAbQAACAAAAAAAAADr +sAEA7rABAPGwAQABAAAAFQAAAAAAAAb5sAEAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAADBnRgAAAAAAQAdFAAAAAACfbQAAGAAAAAAAAACdMQEA/LABAP+wAQACAAAAFQAA +AAAAAAasMQEAtzEBAAAAAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +/GtGAAAAAACAB0UAAAAAALhtAAAAAAAAAAAAAAmxAQAQsQEAIbEBAAMAAAAVAAAAAAAABD6xAQBJ +sQEAULEBAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAKCuRgAAAAAA4AdFAAAAAADPbQAAGAAAAAAA +AABhsQEAa7EBAHyxAQADAAAAFQAAAAAAAAagsQEAprEBAKyxAQD4Z0YAAAAAAPhnRgAAAAAAAAAA +AAAAAABsyUYAAAAAAAAAAAAAAAAAFHdGAAAAAACACEUAAAAAABFuAAAAAAAAAAAAACMRAADFsQEA +ybEBAAIAAAAVAAAAAAAAAkARAABLEQAAAAAAAPhnRgAAAAAA+GdGAAAAAADACEUAAAAAACpuAAAA +AAAAAAAAAAcnAADTsQEA17EBAAIAAAAVAAAAAAAAAhknAAAkJwAAAAAAAPhnRgAAAAAA+GdGAAAA +AAAACUUAAAAAAEVuAAAAAAAAAAAAANJ0AQAP4QAA37EBAAIAAAAVAAAAAAAAAh3hAADrsQEAAAAA +APhnRgAAAAAA+GdGAAAAAABgCUUAAAAAAGJuAAAAAAAAAAAAAPaxAQD9sQEAAbIBAAIAAAAVAAAA +AAAAApEiAQALsgEAAAAAAPhnRgAAAAAA+GdGAAAAAADACUUAAAAAAHtuAAAAAAAAAAAAABKyAQAZ +sgEAHbIBAAIAAAAVAAAAAAAAAncMAACCDAAAAAAAAPhnRgAAAAAA+GdGAAAAAAAACkUAAAAAAI5u +AAAIAAAAAAAAACeyAQAxsgEAPrIBAAMAAAAVAAAAAAAABrzeAABnsgEAcbIBAABuRgAAAAAAtG5G +AAAAAAAAAAAAAAAAADiURgAAAAAAAAAAAAAAAAAwZ0YAAAAAAIALRQAAAAAAp24AAAAAAAAAAAAA +gbIBAImyAQClsgEAAwAAABUAAAAAAAAE3rIBAAAAAADqsgEA+GdGAAAAAAD4Z0YAAAAAAAAAAAAA +AAAA8K5GAAAAAACADUUAAAAAANJuAAAQAAAAAAAAAAKzAQAKswEAFrMBAAMAAAAVAAAAAAAABkCz +AQBMswEAVLMBAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAJSFRgAAAAAAAAAAAAAAAABkZ0YAAAAA +AGAORQAAAAAA3W4AAAAAAAAAAAAAX7MBAGazAQBqswEAAgAAABUAAAAAAAACsw4AAHKzAQAAAAAA ++GdGAAAAAAD4Z0YAAAAAAKAORQAAAAAA8m4AAAAAAAAAAAAAd7MBAIWzAQCOswEAAwAAABUAAAAA +AAAE4LMBAOazAQD6swEAGGhGAAAAAADYbkYAAAAAAAAAAAAAAAAAqIVGAAAAAADAEEUAAAAAAAZv +AAAQAAAAAAAAAAK0AQAKtAEAD7QBAAIAAAAVAAAAAAAABj+0AQBLtAEAAAAAADhsRgAAAAAACGhG +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACsZ0YAAAAAAOARRQAAAAAAIW8AAAAAAAAAAAAA +Tf0AAFO0AQBXtAEAAgAAABUAAAAAAAACXP0AAGH9AAAAAAAA+GdGAAAAAAD4Z0YAAAAAAAASRQAA +AAAANG8AAAgAAAAAAAAAXbQBAG+0AQB0tAEAAgAAABUAAAAAAAAGjrQBAJm0AQAAAAAAbHFGAAAA +AAD0cEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAAAAAAoBJFAAAAAABFbwAAAAAA +gAAAAACktAEApLQBAKe0AQAAAAAAoQAAAAACAAAAAAAA4BJFAAAAAABKbwAAAAAAgAAAAAC+7AAA +vuwAAMG0AQAAAAAAoQAAAAAAAAAAAAAAIBNFAAAAAABkbwAAAAAAgAAAAADZtAEA2bQBANy0AQAA +AAAAoQAAAAAAAAAAAAAAQBNFAAAAAABtbwAAAAAAAAAAAADktAEA5LQBAOi0AQAAAAAAoQAAAAAA +AAAAAAAAoBhFAAAAAAB5bwAAAAAAAAAAAADutgEA97YBAPq2AQAAAAAAoQAAABYAAAIAAAAAAAAA +AAAAAAAQaUcAAAAAAAAZRQAAAAAAhW8AAAAAAAAAAAAA/rYBAPe2AQAKtwEAAAAAAKEAAAAWAAAC +AAAAAAAAAAAAAAAAEGlHAAAAAABgGUUAAAAAAJFvAAAAAAAAAAAAAA63AQB6AgAAGrcBAAEAAACh +AAAAFgAAAh63AQAAAAAAAAAAABBpRwAAAAAA4BlFAAAAAACebwAAAAAAAAAAAAAptwEANbcBADi3 +AQABAAAAoQAAABYAAAI8twEAAAAAAAAAAAAQaUcAAAAAAGAaRQAAAAAAq28AAAAAAAAAAAAAR7cB +ADW3AQBTtwEAAQAAAKEAAAAWAAACPLcBAAAAAAAAAAAAEGlHAAAAAADgGkUAAAAAALhvAAAAAAAA +AAAAAFe3AQA1twEAY7cBAAEAAAChAAAAFgAAAjy3AQAAAAAAAAAAABBpRwAAAAAAYBtFAAAAAADG +bwAAAAAAAAAAAABntwEANbcBAHO3AQABAAAAoQAAABYAAAI8twEAAAAAAAAAAAAQaUcAAAAAAOAb +RQAAAAAA1G8AAAAAAAAAAAAAd7cBAIO3AQCGtwEAAQAAAKEAAAAWAAACircBAAAAAAAAAAAAEGlH +AAAAAABgHEUAAAAAAOJvAAAAAAAAAAAAAJW3AQCDtwEApLcBAAEAAAChAAAAFgAAAoq3AQAAAAAA +AAAAABBpRwAAAAAA4BxFAAAAAADwbwAAAAAAAAAAAACotwEAg7cBALe3AQABAAAAoQAAABYAAAKK +twEAAAAAAAAAAAAQaUcAAAAAAGAdRQAAAAAA/28AAAAAAAAAAAAAu7cBAIO3AQDKtwEAAQAAAKEA +AAAWAAACircBAAAAAAAAAAAAEGlHAAAAAADgHUUAAAAAAA5wAAAAAAAAAAAAAM63AQCDtwEA3bcB +AAEAAAChAAAAFgAAAoq3AQAAAAAAAAAAABBpRwAAAAAAYB5FAAAAAAAdcAAAAAAAgAAAAADKAwAA +ygMAAOG3AQAAAAAAoQAAAAAAAAAAAAAAgB5FAAAAAAAocAAAAAAAgAAAAADotwEA9LcBAPi3AQAA +AAAAoQAAAAADAAAAAAAA4B9FAAAAAAA3cAAAAAAAAAAAAACdgwAAnYMAAIy4AQAAAAAAoQAAAAAA +AAAAAAAAACBFAAAAAABHcAAAAAAAgAAAAACKBQAAigUAAJC4AQAAAAAAoQAAAA4BAAAAAAAAICBF +AAAAAABWcAAACAAAAAAAAAAkAgAAJAIAAJa4AQAAAAAAoQAAAAgAAAAAAAAAQCBFAAAAAABjcAAA +CAAAAAAAAACguAEAp7gBAKq4AQAAAAAAoQAAAAwCAAAAAAAAoCBFAAAAAABxcAAAAAAAAAAAAACd +gwAAnYMAANa4AQAAAAAAoQAAAAAAAAAAAAAAwCBFAAAAAACMcAAACAAAAAAAAADauAEA2rgBAN64 +AQAAAAAAoQAAABQCAAAAAAAAYCFFAAAAAACgcAAAAAAAAAAAAAAeuQEAHrkBACK5AQAAAAAAoQAA +AA0CAAAAAAAAACJFAAAAAACycAAAAAAAgAAAAACkBQAApAUAAGC5AQAAAAAAoQAAAAAAAAAAAAAA +ICJFAAAAAADLcAAAAAAAAAAAAAANEQEADREBAGa5AQAAAAAAoQAAAAAAAAAAAAAAQCJFAAAAAADd +cAAAAAAAAAAAAACdgwAAnYMAAHK5AQAAAAAAoQAAAAAAAAYAAAAAGGlHAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAACRnRgAAAAAAYCJFAAAAAAD4cAAAEAAAAAAAAAB2uQEAdrkB +AHm5AQAAAAAAoQAAAAsCAAAAAAAAgCJFAAAAAAAJcQAAFAAAAAAAAACJuQEAibkBAI25AQAAAAAA +oQAAAAICAAYAAAAAYGlHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAExnRgAA +AAAAQCNFAAAAAAAccQAACAAAAAAAAADpuQEA6bkBAOy5AQAAAAAAoQAAAAAAAAAAAAAAYCNFAAAA +AAApcQAAAAAAAAAAAABxWAEAcVgBAPS5AQAAAAAAoQAAAAEAAAAAAAAAgCNFAAAAAAA3cQAAAAAA +AAAAAAD6uQEA+rkBAP25AQAAAAAAoQAAAAAAAAAAAAAAoCNFAAAAAABKcQAAAAAAAAAAAAAPugEA +D7oBABK6AQAAAAAAoQAAAAAAAAAAAAAAwCNFAAAAAABbcQAAIAAAAAAAAADrsAEA67ABACi6AQAA +AAAAoQAAAAAAAAAAAAAA4CNFAAAAAABrcQAAGAAAAAAAAAAyugEAMroBADW6AQAAAAAAoQAAAAAA +AAAAAAAAICRFAAAAAAB9cQAAGAAAAAAAAABLugEAS7oBAE66AQAAAAAAoQAAAAAAAAAAAAAAYCRF +AAAAAACPcQAAAQAAAAAAAAB2uQEAdrkBAGS6AQAAAAAAoQAAAAAAAAYAAAAAcGlHAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABxnRgAAAAAAgCRFAAAAAACgcQAAAAAAAAAAAABy +ugEAcroBAHW6AQAAAAAAoQAAAAcBAAAAAAAAoCRFAAAAAACvcQAAAAAAAAAAAAB9ugEAfboBAIC6 +AQAAAAAAoQAAAAAAAAAAAAAAwCRFAAAAAADBcQAAAAAAgAAAAACIugEAlboBAJm6AQAAAAAAoQAA +AAAAAAAAAAAAwCVFAAAAAADYcQAAAAAAgAAAAACkBQAApAUAAPG6AQAAAAAAoQAAAAAAAAAAAAAA +4CVFAAAAAADxcQAAAAAAgAAAAACkBQAApAUAAPu6AQAAAAAAoQAAAAAAAAAAAAAAACZFAAAAAAAK +cgAAAAAAgAAAAACkBQAApAUAAAW7AQAAAAAAoQAAAAAAAAAAAAAAICZFAAAAAAAjcgAAAAAAgAAA +AACkBQAApAUAAA+7AQAAAAAAoQAAAAAAAAAAAAAAQCZFAAAAAAA8cgAAAAAAgAAAAACkBQAApAUA +ABm7AQAAAAAAoQAAAAAAAAAAAAAAYCZFAAAAAABVcgAAAAAAgAAAAACkBQAApAUAACO7AQAAAAAA +oQAAAAAAAAAAAAAAgCZFAAAAAABucgAAAAAAAAAAAAAtuwEAN7sBADu7AQAAAAAAoQAAAAUAAAAA +AAAAIClFAAAAAACCcgAAEAAAAAAAAADDuwEAyrsBAM27AQAAAAAAoQAAAAAAAAAAAAAAYClFAAAA +AACccgAAEAAAAAAAAACxBQAAsQUAAN+7AQAAAAAAoQAAAAAAAAAAAAAAgClFAAAAAACvcgAAEAAA +AAAAAACxBQAAsQUAAOW7AQAAAAAAoQAAAAAAAAAAAAAAoClFAAAAAADDcgAAEAAAAAAAAADruwEA +67sBAO67AQAAAAAAoQAAAAAAAAAAAAAAwClFAAAAAADacgAAEAAAAAAAAADruwEA67sBAPa7AQAA +AAAAoQAAAAAAAAAAAAAA4ClFAAAAAADycgAAEAAAAAAAAADruwEA67sBAP67AQAAAAAAoQAAAAAA +AAAAAAAAACpFAAAAAAAJcwAAEAAAAAAAAADruwEA67sBAAa8AQAAAAAAoQAAAAAAAAAAAAAAICpF +AAAAAAAhcwAAEAAAAAAAAACxBQAAsQUAAA68AQAAAAAAoQAAAAAAAAAAAAAAQCpFAAAAAAA1cwAA +EAAAAAAAAACxBQAAsQUAABS8AQAAAAAAoQAAAAAAAAAAAAAAYCpFAAAAAABKcwAAEAAAAAAAAACx +BQAAsQUAABq8AQAAAAAAoQAAAAAAAAAAAAAAgCpFAAAAAABicwAAEAAAAAAAAACxBQAAsQUAACC8 +AQAAAAAAoQAAAAAAAAAAAAAAoCpFAAAAAAB7cwAAAAAAAAAAAAAmvAEAJrwBACq8AQAAAAAAogAA +AAAAAAAAAAAAICxFAAAAAACMcwAAAAAAAAAAAADNvAEAzbwBANG8AQAAAAAAogAAAAAAAAAAAAAA +wC9FAAAAAACdcwAAEAAAAAAAAADVvgEA1b4BANm+AQAAAAAAowAAAAAAAAAAAAAAgDJFAAAAAAC6 +cwAAGAAAAAAAAADgvwEA4L8BAOS/AQAAAAAApAAAAAAAAAAAAAAAQDlFAAAAAADKcwAAAAAAAAAA +AACzwgEAxcIBAMnCAQAAAAAApQAAAAMAAAAAAAAAADtFAAAAAADfcwAAAAAAgAAAAABUwwEAVMMB +AFfDAQAAAAAApgAAAAAAAAAAAAAAIDtFAAAAAADwcwAABAAAAAAAAAANEQEADREBAFrDAQAAAAAA +pwAAAAAAAAAAAAAAQDtFAAAAAAD9cwAACAAAAAAAAAAQAgAAEAIAAGPDAQAAAAAApwAAAAAAAAAA +AAAAYDtFAAAAAAAQdAAAFAAAAAAAAABywwEAcsMBAHXDAQAAAAAApwAAAAAAAAAAAAAAoDtFAAAA +AAAddAAADAAAAAAAAAB2uQEAdrkBAI3DAQAAAAAApwAAAAAAAAAAAAAAwDtFAAAAAAAtdAAAHAAA +AAAAAADCBQAAwgUAAJ/DAQAAAAAApwAAAAAAAAAAAAAA4DtFAAAAAAA8dAAAHAAAAAAAAACvwwEA +r8MBALLDAQAAAAAApwAAAAAAAAAAAAAAADxFAAAAAABJdAAADAAAAAAAAAAkAgAAJAIAAMLDAQAA +AAAApwAAAAAAAAAAAAAAIDxFAAAAAABWdAAAFAAAAAAAAADOwwEAzsMBANHDAQAAAAAApwAAAAAA +AAAAAAAAQDxFAAAAAABkdAAAAAAAgAAAAADfwwEA5sMBAOnDAQAAAAAApwAAAAAAAAAAAAAAoDxF +AAAAAABzdAAABAAAAAAAAAANEQEADREBAAfEAQAAAAAApwAAAAAAAAAAAAAAwDxFAAAAAACCdAAA +AAAAgAAAAACTAgAAkwIAABHEAQAAAAAApwAAAAAAAAAAAAAAAD1FAAAAAACQdAAAAAAAgAAAAADO +wwEAzsMBACnEAQAAAAAApwAAAAAAAAAAAAAAID1FAAAAAACidAAACAAAAAAAAADZtAEA2bQBADnE +AQAAAAAApwAAAAAAAAYAAAAAgGlHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +ACBnRgAAAAAAQD1FAAAAAACxdAAAAAAAgAAAAADcBQAA3AUAAEPEAQAAAAAApwAAAAAAAAYAAAAA +IGlHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAORnRgAAAAAAYD1FAAAAAADA +dAAAHAAAAAAAAAAQAgAAEAIAAFHEAQAAAAAApwAAAAAAAAAAAAAAgD1FAAAAAADQdAAACAAAAAAA +AABhxAEAibkBAGvEAQAAAAAApwAAAAACAAAAAAAAQD5FAAAAAADidAAAHAAAAAAAAABdoAEAXaAB +ALfEAQAAAAAApwAAAAAAAAAAAAAAgD5FAAAAAAD4dAAAJAAAAAAAAAAPugEAD7oBAM3EAQAAAAAA +pwAAAAAAAAAAAAAAoD5FAAAAAAANdQAAAAAAgAAAAADfxAEA5sQBAOnEAQAAAAAApwAAAAACAAAA +AAAA4D5FAAAAAAAmdQAAIAAAAAAAAAABxQEAkwIAAAjFAQAAAAAApwAAAAACAAAAAAAAID9FAAAA +AAA1dQAAAAAAgAAAAAAgxQEAK8UBAC7FAQAAAAAApwAAAAAAAAAAAAAAoD9FAAAAAABGdQAAAAAA +gAAAAABCxQEAQsUBAEbFAQAAAAAApwAAAAAAAAAAAAAAYEBFAAAAAABadQAAAAAAgAAAAADruwEA +67sBAKLFAQAAAAAApwAAAAAAAAAAAAAAgEBFAAAAAABsdQAAAAAAgAAAAACqxQEAqsUBAK3FAQAA +AAAApwAAAAAAAAAAAAAA4EBFAAAAAAB8dQAAAAAAgAAAAADTxQEA2sUBAN3FAQAAAAAApwAAAAAC +AAAAAAAAQEFFAAAAAACQdQAAAAAAgAAAAAD9xQEA/cUBAADGAQAAAAAApwAAAAAAAAAAAAAAgEFF +AAAAAACidQAAEAAAAAAAAAASxgEAMroBABnGAQAAAAAApwAAAAACAAAAAAAAwEFFAAAAAAC4dQAA +AAAAgAAAAACZBAAAmQQAAC/GAQAAAAAApwAAAAAAAAAAAAAA4EFFAAAAAADIdQAAAAAAgAAAAAA/ +xgEAP8YBAELGAQAAAAAApwAAAAAAAAAAAAAAIEJFAAAAAADWdQAAAAAAgAAAAABYxgEAWMYBAFzG +AQAAAAAApwAAAAACAAAAAAAAwEJFAAAAAADkdQAAAAAAgAAAAAD9xQEA/cUBAKrGAQAAAAAApwAA +AAAAAAAAAAAAAENFAAAAAAD4dQAAAAAAgAAAAAC8xgEAw8YBAMbGAQAAAAAApwAAAAAAAAAAAAAA +YENFAAAAAAAHdgAAAAAAgAAAAACxBQAAsQUAANzGAQAAAAAApwAAAAAAAAAAAAAAgENFAAAAAAAX +dgAAAAAAgAAAAAAQAgAAEAIAAOTGAQAAAAAApwAAAAAAAAAAAAAAoENFAAAAAAAxdgAAAAAAgAAA +AADDRgAAw0YAAPTGAQAAAAAApwAAAAAAAAAAAAAAwENFAAAAAABFdgAAAAAAgAAAAADDRgAAw0YA +AADHAQAAAAAApwAAAAAAAAAAAAAA4ENFAAAAAABadgAAAAAAgAAAAAB2uQEAdrkBAAzHAQAAAAAA +pwAAAAAAAAAAAAAAAERFAAAAAABrdgAAAAAAgAAAAAAexwEAHscBACHHAQAAAAAApwAAAAAAAAAA +AAAAQERFAAAAAAB9dgAAAAAAgAAAAADCBQAAwgUAADXHAQAAAAAApwAAAAAAAAAAAAAAYERFAAAA +AACRdgAABAAAAAAAAABDxwEAQ8cBAEbHAQAAAAAApwAAAAAAAAAAAAAAoERFAAAAAACldgAAGAAA +AAAAAABgxwEAascBAG7HAQAAAAAAqAAAAAACAAAAAAAAgEVFAAAAAACudgAACAAAAAAAAADHxwEA +0McBANPHAQACAAAAFQAAABYAAAbXxwEA4scBAAAAAAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAMGdGAAAAAADgRUUAAAAAAM52AAAYAAAAAAAAAOnHAQD4BQAA+wUAAAIA +AAAVAAAAFgAABvDHAQD1xwEAAAAAADxrRgAAAAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAwZ0YAAAAAACBGRQAAAAAA5XYAAAgAAAAAAAAA+scBAL7sAAAByAEAAgAAABUAAAAWAAAG +BMgBAAnIAQAAAAAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAA +AAAAYEZFAAAAAAD7dgAAEAAAAAAAAAAOyAEAO8oAABXIAQACAAAAFQAAABYAAAYYyAEAHcgBAAAA +AAA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZGdGAAAAAACgRkUAAAAA +AA53AAAAAAAAAAAAAH26AQB9ugEAIsgBAAEAAAAVAAAAFgAAAiXIAQD4Z0YAAAAAAPhnRgAAAAAA +wEZFAAAAAAAddwAAAAAAAAAAAAB9ugEAfboBACLIAQABAAAAFQAAABYAAAIlyAEA+GdGAAAAAAD4 +Z0YAAAAAAOBGRQAAAAAAM3cAAAgAAAAAAAAA+scBAL7sAAAByAEAAgAAABUAAAAWAAAGBMgBACjI +AQAAAAAAOGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAAAAAAIEdF +AAAAAABEdwAACAAAAAAAAAD6xwEAvuwAAAHIAQACAAAAFQAAABYAAAYEyAEAKMgBAAAAAAA4bEYA +AAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMGdGAAAAAABgR0UAAAAAAFZ3AAAA +AAAAAAAAAH26AQB9ugEAIsgBAAEAAAAVAAAAFgAAAiXIAQD4Z0YAAAAAAPhnRgAAAAAAgEdFAAAA +AABtdwAAAAAAAAAAAAB9ugEAfboBACLIAQABAAAAFQAAABYAAAIlyAEA+GdGAAAAAAD4Z0YAAAAA +AKBHRQAAAAAAiXcAAAAAAAAAAAAAfboBAH26AQAiyAEAAQAAABUAAAAWAAACJcgBAPhnRgAAAAAA ++GdGAAAAAADAR0UAAAAAAJt3AAAAAAAAAAAAAOuwAQDrsAEALcgBAAIAAAAVAAAAFgAAAvmwAQDr +sAEAAAAAAPhnRgAAAAAA+GdGAAAAAADgR0UAAAAAAKp3AAAAAAAAAAAAAH26AQB9ugEAIsgBAAEA +AAAVAAAAFgAAAiXIAQD4Z0YAAAAAAPhnRgAAAAAAAEhFAAAAAAC6dwAAAAAAAAAAAAB9ugEAfboB +ACLIAQABAAAAFQAAABYAAAIlyAEA+GdGAAAAAAD4Z0YAAAAAACBIRQAAAAAAyncAABAAAAAAAAAA +w7sBAMq7AQAwyAEAAgAAABUAAAAWAAAGM8gBADjIAQAAAAAACGxGAAAAAAAIaEYAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAFRnRgAAAAAAYEhFAAAAAADadwAAAAAAAAAAAAB9ugEAfboBACLI +AQABAAAAFQAAABYAAAIlyAEA+GdGAAAAAAD4Z0YAAAAAAIBIRQAAAAAA73cAABAAAAAAAAAAw7sB +AMq7AQAwyAEAAgAAABUAAAAWAAAGM8gBADjIAQAAAAAACGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAFRnRgAAAAAAwEhFAAAAAAD8dwAAAAAAAAAAAAB9ugEAfboBACLIAQAB +AAAAFQAAABYAAAIlyAEA+GdGAAAAAAD4Z0YAAAAAAOBIRQAAAAAACngAABgAAAAAAAAAPcgBAETI +AQBHyAEAAgAAABUAAAAWAAAGSsgBAE/IAQAAAAAATG1GAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAANxnRgAAAAAAIElFAAAAAAAdeAAAAAAAAAAAAAB9ugEAfboBACLIAQABAAAA +FQAAABYAAAIlyAEA+GdGAAAAAAD4Z0YAAAAAAEBJRQAAAAAALngAAAAAAAAAAAAAfboBAH26AQAi +yAEAAQAAABUAAAAWAAACJcgBAPhnRgAAAAAA+GdGAAAAAABgSUUAAAAAAEF4AAAAAAAAAAAAAH26 +AQB9ugEAIsgBAAEAAAAVAAAAFgAAAiXIAQD4Z0YAAAAAAPhnRgAAAAAAgElFAAAAAABYeAAAEAAA +AAAAAABUyAEA6QUAAOwFAAACAAAAFQAAABYAAAZbyAEAYMgBAAAAAAB8bUYAAAAAAAhoRgAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZGdGAAAAAADASUUAAAAAAGt4AAAIAAAAAAAAAGXIAQBy +yAEAecgBAAMAAAAVAAAAFgAABoLIAQCNyAEAlMgBADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAALyF +RgAAAAAAAAAAAAAAAAAwZ0YAAAAAAEBKRQAAAAAAiHgAABAAAAAAAAAAm8gBANrFAQCiyAEAAgAA +ABUAAAAAAAAGAHoAAKXIAQAAAAAAfG1GAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAHxgRQAAAAAAoEpFAAAAAACeeAAAEAAAAAAAAACsyAEAs8gBALfIAQACAAAAFQAAAAAAAAbz +XwAAu8gBAAAAAAB8bUYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAfGBFAAAA +AABAS0UAAAAAALp4AAAQAAAAAAAAAM7DAQDOwwEAxMgBAAEAAAAVAAAAAAAABsfIAQB8bUYAAAAA +AAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAfGBFAAAAAABgS0UAAAAAANV4AAAQAAAA +AAAAAMrIAQDRyAEA1MgBAAIAAAAVAAAAAAAABvAHAADXyAEAAAAAAHxtRgAAAAAACGhGAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB8YEUAAAAAAOBLRQAAAAAA+XgAABAAAAAAAAAA4MgBAGAC +AABkAgAAAgAAABUAAAAAAAAGaAIAAOfIAQAAAAAAfG1GAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAHxgRQAAAAAAgExFAAAAAAAReQAAEAAAAAAAAADwyAEA98gBAPrIAQACAAAA +FQAAAAAAAAZaXgAA/cgBAAAAAAB8bUYAAAAAAAhoRgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAfGBFAAAAAAAATUUAAAAAACl5AAAQAAAAAAAAAGQFAABrBQAAbgUAAAIAAAAVAAAAAAAABnEF +AAB8BQAAAAAAAHxtRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB8YEUAAAAA +AGBNRQAAAAAARnkAAAgAAAAAAAAABskBAFGiAQATyQEAAgAAABUAAAAWAAAGFskBACHJAQAAAAAA +OGxGAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADBnRgAAAAAA4E1FAAAAAABj +eQAAEAAAAAAAAAD9xQEA/cUBACjJAQABAAAAFQAAAAAAAAYryQEAfG1GAAAAAAAIaEYAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAHxgRQAAAAAAIE5FAAAAAAB/eQAAEAAAAAAAAABkBQAAawUA +AG4FAAACAAAAFQAAAAAAAAZxBQAALskBAAAAAAD4Z0YAAAAAAPhnRgAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAfGBFAAAAAACATkUAAAAAAJh5AAAQAAAAAAAAAPDIAQD3yAEA+sgBAAIAAAAV +AAAAAAAABlpeAAA1yQEAAAAAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAB8YEUAAAAAAABPRQAAAAAAr3kAABAAAAAAAAAAZAUAAGsFAABuBQAAAgAAABUAAAAAAAAGcQUA +AC7JAQAAAAAA+GdGAAAAAAD4Z0YAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHxgRQAAAAAA +YE9FAAAAAADHeQAAEAAAAAAAAADOwwEAzsMBAMTIAQABAAAAFQAAAAAAAAbHyAEAfG1GAAAAAAAI +aEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHxgRQAAAAAAgE9FAAAAAAD6eQAAEAAAAAAA +AADwyAEA98gBAPrIAQACAAAAFQAAAAAAAAZaXgAA/cgBAAAAAAB8bUYAAAAAAAhoRgAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAfGBFAAAAAAAAUEUAAAAAABF6AAAQAAAAAAAAAPDIAQD3yAEA ++sgBAAIAAAAVAAAAAAAABlpeAAD9yAEAAAAAAHxtRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAB8YEUAAAAAAIBQRQAAAAAAKHoAABAAAAAAAAAAZAUAAGsFAABuBQAAAgAAABUA +AAAAAAAGcQUAAHwFAAAAAAAAfG1GAAAAAAAIaEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AHxgRQAAAAAA4FBFAAAAAABAegAACAAAAAAAAAA8yQEARskBAFnJAQADAAAAFQAAABYAAAZ0yQEA +eskBAIDJAQA4bEYAAAAAAAhoRgAAAAAAAAAAAAAAAADQhUYAAAAAAAAAAAAAAAAAMGdGAAAAAACg +UUUAAAAAAFt6AAAIAAAAAAAAAJPJAQCcyQEAq8kBAAMAAAAVAAAAFgAABrrJAQC/yQEAxMkBADhs +RgAAAAAACGhGAAAAAAAAAAAAAAAAAOSFRgAAAAAAAAAAAAAAAAAwZ0YAAAAAACBSRQAAAAAAeHoA +ABAAAAAAAAAAz8kBANbJAQDZyQEAAgAAABUAAAAAAAAGPYAAANzJAQAAAAAAfG1GAAAAAAAIaEYA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHxgRQAAAAAAoFJFAAAAAACcegAACAAAAAAAAABl +yAEAcsgBAOPJAQADAAAAFQAAABYAAAaCyAEAjcgBAJTIAQA4bEYAAAAAAAhoRgAAAAAAAAAAAAAA +AAD4hUYAAAAAAAAAAAAAAAAAMGdGAAAAAAAgU0UAAAAAAMB6AAAIAAAAAAAAAOzJAQD1yQEA+MkB +AAIAAAAVAAAAFgAABvvJAQAAygEAAAAAADhsRgAAAAAACGhGAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAwZ0YAAAAAAIBTRQAAAAAA3HoAAAAAAAAAAAAAnYMAAAXKAQAIygEAAAAAAKkAAAAA +AAACAAAAAPhnRgAAAAAA+GdGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAP8gR28gYnVpbGRpbmY6CABgkksAAAAAAKCSSwAAAAAAAQAAAAgAAAD/////AQAA +AAgAAAABAAAAe++9994AAABAAAAAAAAAAP//////////ALCO8BsAAACAh0sAAAAAAP//////fwAA +//////9/AAAAABAAAAAAAAAAEAAAAAAAAAAAAACA////////////////////////L2Rldi91cmFu +ZG9tAAAAAAAAAAAAAAAAL3Byb2Mvc2VsZi9hdXh2AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAB1AAAAAAAAOAAAAAAAAAAMAAAAAAAAAAwAA +AAAAAAADAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFQAAAAAAAAASAAAA +AAAAAA8AAAAAAAAADAAAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACIAAAAA +AAAAHwAAAAAAAAAcAAAAAAAAABkAAAAAAAAAFgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AABmYXRhbCBlcnJvcjogY2dvIGNhbGxiYWNrIGJlZm9yZSBjZ28gY2FsbAoAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAL3N5cy9rZXJuZWwvbW0vdHJhbnNwYXJlbnRfaHVnZXBhZ2UvaHBhZ2VfcG1kX3Np +emUAAAAAAAAAAAAAAAAAAAABAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAgECAQIB +AwIDAQMCAwQFBgEHBgUEAwUHAgkHBQgDCgcEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAQAAAAAAAAAIAAAAAAAAAOCASwAAAAAAgAtFAAAAAABgUUAAAAAAAACHQQAAAAAAgGxC +AAAAAACglUIAAAAAACDKQgAAAAAAoOBCAAAAAADgNkQAAAAAAAABAgMEBQUGBgcHCAgJCQoKCwsM +DA0NDg4PDxAQERESEhMTExMUFBQUFRUVFRYWFhYXFxcXGBgYGBkZGRkaGhoaGxsbGxsbGxscHBwc +HBwcHB0dHR0dHR0dHh4eHh4eHh4fHx8fHx8fHx8fHx8fHx8fICAgICAgICAgICAgICAgIAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAEAAYACAAMABAAFAAYABwAIAAkACgALAAwADQ +AOAA8AAAASABQAFgAYABoAHAAeABAAJAAoACwAIAA4ADAASABAAFgAUABgAHAAgACYAKAAyADIAN +ABAAEwAVABiAGYAaABsAIAAlACYAKIAqADAANQA4AEAASIBKAFAAVQBggGoAcACAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAICEiIyQlJSYmJycoKCgpKSkqKyssLCwsLC0tLS0tLS4uLi4vLy8vLy8w +MDAxMTIzMzMzMzMzMzMzNDQ0NDQ0NDQ0NDU1NjY2Njc3Nzc3ODg4ODg4ODg4ODg5OTk5OTk5OTk5 +Ojo6Ojo6Ozs7Ozs7Ozs7Ozs7Ozs7Ozw8PDw8PDw8PDw8PDw8PDw9PT09PT4+Pj4+Pj4+Pj4+Pz8/ +Pz8/Pz8/P0BAQEBAQEBAQEBAQEBAQEBAQEBAQEBBQUFBQUFBQUFBQUFBQUFBQUFBQUFCQkJCQkJC +QkJCQkNDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDAAAAAAAAAAAAAAAAAAAAkP2OddO6 +pj8YE8n69mO2P+R52oxYjMA/jNb7ORrAxT+Y7LQtXs/KPwxoArkWvM8/OgcOqwdE0j+KG81LeJrU +P94Mnc0h4tY/Chcfibob2T+ggjj360fbPxDqMuBTZ90/0AbLaIV63z+mOtYABcHgPw7QlR4xv+E/ +0no/RwO44j9nIaD6s6vjP4sbzUt4muQ/NJ2YJoKE5T/MiEeOAGrmP1QHTtYfS+c/c3C+1Qko6D/N +AgAW5gDpP7QQUP3Z1ek/1BSA9Qin6j/aMlWPlHTrP1Wg4aKcPuw/lggmbT8F7T9s9T+rmcjtP3Nq +YrPGiO4/VQbPi+BF7z8AAAAAAADwPwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAA +EKuqqgoAAAAIVlVVBQAAAAQ0MzMDq6qqApMkSQIAAAACchzHAZqZmQEYXXQBVlVVAbITOwFKkiQB +EhERAQAAAAE5juMAzczMAIwuugCrqqoA2YmdACVJkgCJiIgAAACAAB3HcQBnZmYARhddAFZVVQCT +JEkAAABAAI/jOAA0MzMAo4suAKuqKgBKkiQAAAAgAMhxHACHYRgAVlUVAOJ6FACF9hIAAAAQAER5 +DQDEMAwAq6oKAAsKCgAPqQkAQ3sJAAAACAA/6wYAorwGAGdmBgAHBgYAVlUFAIjUBABKkgQAAAAE +ADmOAwCubwMANDMDAAQDAwCrqgIAXWcCACVJAgAAAAIAAAAAAAAAAAAAAAAAAAAAACCJRwAAAAAA +YIlHAAAAAADoegAAAAAAAOh6AAAAAAAAYARIAAAAAACwAgAAAAAAALACAAAAAAAAIAdIAAAAAACY +CAAAAAAAAJgIAAAAAAAAwA9IAAAAAAAQygEAAAAAABDKAQAAAAAA4NlJAAAAAAAonAEAAAAAACic +AQAAAAAA4NlJAAAAAADTAwAAAAAAANMDAAAAAAAAIH5HAAAAAAAAEEAAAAAAAIFTRQAAAAAAABBA +AAAAAACQU0UAAAAAACCASwAAAAAAoJFLAAAAAACgkUsAAAAAAJCzSwAAAAAAoLNLAAAAAADInk4A +AAAAAOCeTgAAAAAAAPJOAAAAAAAA8k4AAAAAAK1nRwAAAAAAWGBHAAAAAAAAYEUAAAAAALiERwAA +AAAAcGpHAAAAAAABAAAAAAAAAAEAAAAAAAAAQIZHAAAAAACwAAAAAAAAALAAAAAAAAAAAIlHAAAA +AAABAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAgAAAAAAAAADAAAAAAAAAAQA +AAAAAAAABQAAAAAAAAAGAAAAAAAAAAcAAAAAAAAACAAAAAAAAAAJAAAAAAAAAAoAAAAAAAAACwAA +AAAAAAAMAAAAAAAAAA0AAAAAAAAADgAAAAAAAAAPAAAAAAAAABAAAAAAAAAAEQAAAAAAAAASAAAA +AAAAABMAAAAAAAAAFAAAAAAAAAAVAAAAAAAAABYAAAAAAAAAFwAAAAAAAAAYAAAAAAAAABkAAAAA +AAAAGgAAAAAAAAAbAAAAAAAAABwAAAAAAAAAHQAAAAAAAAAeAAAAAAAAAB8AAAAAAAAAIAAAAAAA +AAAhAAAAAAAAACIAAAAAAAAAIwAAAAAAAAAkAAAAAAAAACUAAAAAAAAAJgAAAAAAAAAnAAAAAAAA +ACgAAAAAAAAAKQAAAAAAAAAqAAAAAAAAACsAAAAAAAAALAAAAAAAAAAtAAAAAAAAAC4AAAAAAAAA +LwAAAAAAAAAwAAAAAAAAADEAAAAAAAAAMgAAAAAAAAAzAAAAAAAAADQAAAAAAAAANQAAAAAAAAA2 +AAAAAAAAADcAAAAAAAAAOAAAAAAAAAA5AAAAAAAAADoAAAAAAAAAOwAAAAAAAAA8AAAAAAAAAD0A +AAAAAAAAPgAAAAAAAAA/AAAAAAAAAEAAAAAAAAAAQQAAAAAAAABCAAAAAAAAAEMAAAAAAAAARAAA +AAAAAABFAAAAAAAAAEYAAAAAAAAARwAAAAAAAABIAAAAAAAAAEkAAAAAAAAASgAAAAAAAABLAAAA +AAAAAEwAAAAAAAAATQAAAAAAAABOAAAAAAAAAE8AAAAAAAAAUAAAAAAAAABRAAAAAAAAAFIAAAAA +AAAAUwAAAAAAAABUAAAAAAAAAFUAAAAAAAAAVgAAAAAAAABXAAAAAAAAAFgAAAAAAAAAWQAAAAAA +AABaAAAAAAAAAFsAAAAAAAAAXAAAAAAAAABdAAAAAAAAAF4AAAAAAAAAXwAAAAAAAABgAAAAAAAA +AGEAAAAAAAAAYgAAAAAAAABjAAAAAAAAAGQAAAAAAAAAZQAAAAAAAABmAAAAAAAAAGcAAAAAAAAA +aAAAAAAAAABpAAAAAAAAAGoAAAAAAAAAawAAAAAAAABsAAAAAAAAAG0AAAAAAAAAbgAAAAAAAABv +AAAAAAAAAHAAAAAAAAAAcQAAAAAAAAByAAAAAAAAAHMAAAAAAAAAdAAAAAAAAAB1AAAAAAAAAHYA +AAAAAAAAdwAAAAAAAAB4AAAAAAAAAHkAAAAAAAAAegAAAAAAAAB7AAAAAAAAAHwAAAAAAAAAfQAA +AAAAAAB+AAAAAAAAAH8AAAAAAAAAgAAAAAAAAACBAAAAAAAAAIIAAAAAAAAAgwAAAAAAAACEAAAA +AAAAAIUAAAAAAAAAhgAAAAAAAACHAAAAAAAAAIgAAAAAAAAAiQAAAAAAAACKAAAAAAAAAIsAAAAA +AAAAjAAAAAAAAACNAAAAAAAAAI4AAAAAAAAAjwAAAAAAAACQAAAAAAAAAJEAAAAAAAAAkgAAAAAA +AACTAAAAAAAAAJQAAAAAAAAAlQAAAAAAAACWAAAAAAAAAJcAAAAAAAAAmAAAAAAAAACZAAAAAAAA +AJoAAAAAAAAAmwAAAAAAAACcAAAAAAAAAJ0AAAAAAAAAngAAAAAAAACfAAAAAAAAAKAAAAAAAAAA +oQAAAAAAAACiAAAAAAAAAKMAAAAAAAAApAAAAAAAAAClAAAAAAAAAKYAAAAAAAAApwAAAAAAAACo +AAAAAAAAAKkAAAAAAAAAqgAAAAAAAACrAAAAAAAAAKwAAAAAAAAArQAAAAAAAACuAAAAAAAAAK8A +AAAAAAAAsAAAAAAAAACxAAAAAAAAALIAAAAAAAAAswAAAAAAAAC0AAAAAAAAALUAAAAAAAAAtgAA +AAAAAAC3AAAAAAAAALgAAAAAAAAAuQAAAAAAAAC6AAAAAAAAALsAAAAAAAAAvAAAAAAAAAC9AAAA +AAAAAL4AAAAAAAAAvwAAAAAAAADAAAAAAAAAAMEAAAAAAAAAwgAAAAAAAADDAAAAAAAAAMQAAAAA +AAAAxQAAAAAAAADGAAAAAAAAAMcAAAAAAAAAyAAAAAAAAADJAAAAAAAAAMoAAAAAAAAAywAAAAAA +AADMAAAAAAAAAM0AAAAAAAAAzgAAAAAAAADPAAAAAAAAANAAAAAAAAAA0QAAAAAAAADSAAAAAAAA +ANMAAAAAAAAA1AAAAAAAAADVAAAAAAAAANYAAAAAAAAA1wAAAAAAAADYAAAAAAAAANkAAAAAAAAA +2gAAAAAAAADbAAAAAAAAANwAAAAAAAAA3QAAAAAAAADeAAAAAAAAAN8AAAAAAAAA4AAAAAAAAADh +AAAAAAAAAOIAAAAAAAAA4wAAAAAAAADkAAAAAAAAAOUAAAAAAAAA5gAAAAAAAADnAAAAAAAAAOgA +AAAAAAAA6QAAAAAAAADqAAAAAAAAAOsAAAAAAAAA7AAAAAAAAADtAAAAAAAAAO4AAAAAAAAA7wAA +AAAAAADwAAAAAAAAAPEAAAAAAAAA8gAAAAAAAADzAAAAAAAAAPQAAAAAAAAA9QAAAAAAAAD2AAAA +AAAAAPcAAAAAAAAA+AAAAAAAAAD5AAAAAAAAAPoAAAAAAAAA+wAAAAAAAAD8AAAAAAAAAP0AAAAA +AAAA/gAAAAAAAAD/AAAAAAAAAAEAAAAAAAAA2LNLAAAAAACAo0sAAAAAAAAAAAAAAAAAyjBGAAAA +AAAVAAAAAAAAALgyRgAAAAAAFgAAAAAAAAD+JkYAAAAAABAAAAAAAAAAzy5GAAAAAAAUAAAAAAAA +AEFfRgAAAAAAMQAAAAAAAACg40UAAAAAAOC2SwAAAAAAakdGAAAAAAAfAAAAAAAAAPw0RgAAAAAA +FwAAAAAAAAAYQEYAAAAAABwAAAAAAAAAzmBGAAAAAAAzAAAAAAAAAEBoRwAAAAAABgAAAAAAAACI +akcAAAAAANCRSwAAAAAAiGpHAAAAAADwkUsAAAAAAIhqRwAAAAAAAJJLAAAAAADCZkYAAAAAAFYA +AAAAAAAAiGpHAAAAAADgkUsAAAAAAKC1RQAAAAAAaJ9OAAAAAACIakcAAAAAAMCRSwAAAAAAQK9F +AAAAAADQtEsAAAAAAMCfRQAAAAAAoLRLAAAAAACAoEUAAAAAAPieTgAAAAAAwKBFAAAAAAAAn04A +AAAAAAChRQAAAAAAYJ9OAAAAAADAlksAAAAAABIAAAAAAAAAEgAAAAAAAAAAAAAAAAAAAMCBSwAA +AAAAKgAAAAAAAAAqAAAAAAAAAAAAAAAAAAAAQJpLAAAAAAAyAAAAAAAAADIAAAAAAAAAAAAAAAAA +AACAlEsAAAAAAAQAAAAAAAAABAAAAAAAAAAAAAAAAAAAAPC0SwAAAAAAAQAAAAAAAAABAAAAAAAA +AAAAAAAAAAAAsIBLAAAAAAAQAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAgksAAAAAADMAAAAAAAAA +MwAAAAAAAAAAAAAAAAAAAJiASwAAAAAADQAAAAAAAAANAAAAAAAAAAAAAAAAAAAAkxpGAAAAAAAJ +AAAAAAAAAPZ1rgMAAAAAAAAAAAAAAADAlEsAAAAAAAIAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAADtGkYAAAAAAAkAAAAAAAAAkB1GAAAAAAAKAAAAAAAAAEgfRgAAAAAACwAA +AAAAAAD2GkYAAAAAAAkAAAAAAAAA3itGAAAAAAATAAAAAAAAAFnKFQMAyhuw4KBOAAAAAAAvLkYA +AAAAABQAAAAAAAAAdew1DRijQ27YoE4AAAAAAFg1RgAAAAAAFwAAAAAAAABiSEYAAAAAAB8AAAAA +AAAAYkhGAAAAAAAfAAAAAAAAAENIRgAAAAAAHwAAAAAAAAChSkYAAAAAACAAAAAAAAAAoUpGAAAA +AAAgAAAAAAAAAIFKRgAAAAAAIAAAAAAAAABhSkYAAAAAACAAAAAAAAAAZVNGAAAAAAAmAAAAAAAA +AGhdRgAAAAAALgAAAAAAAACvXkYAAAAAADAAAAAAAAAAVE1GAAAAAAAhAAAAAAAAAFBeRgAAAAAA +LwAAAAAAAADUX0YAAAAAADEAAAAAAAAALE9GAAAAAAAiAAAAAAAAAApPRgAAAAAAIgAAAAAAAAB8 +ZkYAAAAAAEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOgVRgAAAAAABAAAAAAAAABxGUYAAAAAAAgA +AAAAAAAAphhGAAAAAAAHAAAAAAAAALQYRgAAAAAABwAAAAAAAADCGEYAAAAAAAcAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAOAVRgAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAtxpGAAAAAAAJAAAA +AAAAABobRgAAAAAACQAAAAAAAAAAJEYAAAAAAA4AAAAAAAAAIKNOAAAAAAD7HkYAAAAAAAsAAAAA +AAAA5KJOAAAAAABRGUYAAAAAAAgAAAAAAAAA4KJOAAAAAABIF0YAAAAAAAYAAAAAAAAA6KJOAAAA +AAA9H0YAAAAAAAsAAAAAAAAA7KJOAAAAAAAgIUYAAAAAAAwAAAAAAAAA8KJOAAAAAADuJkYAAAAA +ABAAAAAAAAAA9KJOAAAAAABGJEYAAAAAAA4AAAAAAAAA+KJOAAAAAAAMGEYAAAAAAAcAAAAAAAAA +/KJOAAAAAACGHUYAAAAAAAoAAAAAAAAAAKNOAAAAAAA4IUYAAAAAAAwAAAAAAAAABKNOAAAAAAD4 +FUYAAAAAAAQAAAAAAAAAKKNOAAAAAAAsG0YAAAAAAAkAAAAAAAAACKNOAAAAAAB/H0YAAAAAAAsA +AAAAAAAADKNOAAAAAAC4HUYAAAAAAAoAAAAAAAAAEKNOAAAAAABtK0YAAAAAABIAAAAAAAAAFKNO +AAAAAAD+JEYAAAAAAA8AAAAAAAAAGKNOAAAAAADbGkYAAAAAAAkAAAAAAAAAJKNOAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHIoRgAAAAAAEQAAAAAAAADNF0YAAAAAAAcAAAAA +AAAAzjRGAAAAAAAXAAAAAAAAAJMuRgAAAAAAFAAAAAAAAADwIEYAAAAAAAwAAAAAAAAA7ypGAAAA +AAASAAAAAAAAABM1RgAAAAAAFwAAAAAAAAAIG0YAAAAAAAkAAAAAAAAAWhdGAAAAAAAGAAAAAAAA +AOgpRgAAAAAAEQAAAAAAAADII0YAAAAAAA4AAAAAAAAAiyJGAAAAAAANAAAAAAAAAD4mRgAAAAAA +EAAAAAAAAADkIEYAAAAAAAwAAAAAAAAArhpGAAAAAAAJAAAAAAAAADgkRgAAAAAADgAAAAAAAAAN +JUYAAAAAAA8AAAAAAAAAwh1GAAAAAAAKAAAAAAAAAK0WRgAAAAAABQAAAAAAAADEJEYAAAAAAA4A +AAAAAAAA7DNGAAAAAAAWAAAAAAAAAAI0RgAAAAAAFgAAAAAAAABOKkYAAAAAABEAAAAAAAAATiZG +AAAAAAAQAAAAAAAAABobRgAAAAAACQAAAAAAAABeHUYAAAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAABmF0YAAAAAAAYAAAAAAAAAkRlGAAAAAAAIAAAAAAAAAP4XRgAA +AAAABwAAAAAAAACYIUYAAAAAAAwAAAAAAAAAzx5GAAAAAAALAAAAAAAAAPAXRgAAAAAABwAAAAAA +AADGFkYAAAAAAAUAAAAAAAAAYRlGAAAAAAAIAAAAAAAAAKgWRgAAAAAABQAAAAAAAABZGUYAAAAA +AAgAAAAAAAAAsBVGAAAAAAAEAAAAAAAAALQVRgAAAAAABAAAAAAAAABsF0YAAAAAAAYAAAAAAAAA +7BVGAAAAAAAEAAAAAAAAAF4fRgAAAAAACwAAAAAAAACPFkYAAAAAAAUAAAAAAAAATxVGAAAAAAAD +AAAAAAAAAJodRgAAAAAACgAAAAAAAACZGUYAAAAAAAgAAAAAAAAApCFGAAAAAAAMAAAAAAAAAFAh +RgAAAAAADAAAAAAAAADwFUYAAAAAAAQAAAAAAAAAFCFGAAAAAAAMAAAAAAAAAPQVRgAAAAAABAAA +AAAAAADLFkYAAAAAAAUAAAAAAAAANCNGAAAAAAANAAAAAAAAAFMfRgAAAAAACwAAAAAAAACJGUYA +AAAAAAgAAAAAAAAAgRlGAAAAAAAIAAAAAAAAAIwhRgAAAAAADAAAAAAAAACKFkYAAAAAAAUAAAAA +AAAAXhxGAAAAAAAJAAAAAAAAAMwdRgAAAAAACgAAAAAAAABxFkYAAAAAAAUAAAAAAAAAwRZGAAAA +AAAFAAAAAAAAAHAcRgAAAAAACQAAAAAAAACZFkYAAAAAAAUAAAAAAAAARCFGAAAAAAAMAAAAAAAA +AAspRgAAAAAAEQAAAAAAAACAFkYAAAAAAAUAAAAAAAAA0hpGAAAAAAAJAAAAAAAAAJ4WRgAAAAAA +BQAAAAAAAABeJ0YAAAAAABAAAAAAAAAAxyhGAAAAAAARAAAAAAAAAMsqRgAAAAAAEgAAAAAAAADW +HUYAAAAAAAoAAAAAAAAAXCFGAAAAAAAMAAAAAAAAAGIkRgAAAAAADgAAAAAAAADJGkYAAAAAAAkA +AAAAAAAAAAAAAAAAAABuJkYAAAAAABAAAAAAAAAAAwAAAAAAAACoP0YAAAAAABwAAAAAAAAAAwAA +AAAAAAClKEYAAAAAABEAAAAAAAAABQAAAAAAAAClIkYAAAAAAA0AAAAAAAAAhAAAAAAAAACyPEYA +AAAAABsAAAAAAAAAhAAAAAAAAADLK0YAAAAAABMAAAAAAAAABQAAAAAAAADyI0YAAAAAAA4AAAAA +AAAAiAAAAAAAAACDKEYAAAAAABEAAAAAAAAAiAAAAAAAAAChSEYAAAAAACAAAAAAAAAAAAAAAAAA +AACYIkYAAAAAAA0AAAAAAAAAAQAAAAAAAACwREYAAAAAAB4AAAAAAAAAiAAAAAAAAACwRkYAAAAA +AB8AAAAAAAAAAQAAAAAAAADOREYAAAAAAB4AAAAAAAAAAQAAAAAAAAAQQkYAAAAAAB0AAAAAAAAA +AQAAAAAAAAAHLkYAAAAAABQAAAAAAAAAAwAAAAAAAAAbLkYAAAAAABQAAAAAAAAAhAAAAAAAAADc +MUYAAAAAABYAAAAAAAAAgQEAAAAAAAACS0YAAAAAACEAAAAAAAAAEQEAAAAAAACUKEYAAAAAABEA +AAAAAAAAAAAAAAAAAAARO0YAAAAAABoAAAAAAAAAEQEAAAAAAADyMUYAAAAAABYAAAAAAAAAEQEA +AAAAAAAjS0YAAAAAACEAAAAAAAAAEQEAAAAAAADBSEYAAAAAACAAAAAAAAAAAQEAAAAAAADYTUYA +AAAAACIAAAAAAAAAAQAAAAAAAADNPEYAAAAAABsAAAAAAAAAAQAAAAAAAABES0YAAAAAACEAAAAA +AAAAAQAAAAAAAADsREYAAAAAAB4AAAAAAAAAgQAAAAAAAACSREYAAAAAAB4AAAAAAAAAAQEAAAAA +AADEP0YAAAAAABwAAAAAAAAAAQAAAAAAAACJNEYAAAAAABcAAAAAAAAAAQAAAAAAAAAtQkYAAAAA +AB0AAAAAAAAABAAAAAAAAACgNEYAAAAAABcAAAAAAAAAwAAAAAAAAAA1G0YAAAAAAAkAAAAAAAAA +wAAAAAAAAAA+G0YAAAAAAAkAAAAAAAAAwAAAAAAAAABHG0YAAAAAAAkAAAAAAAAAAQAAAAAAAABQ +G0YAAAAAAAkAAAAAAAAAAQAAAAAAAABZG0YAAAAAAAkAAAAAAAAAAQAAAAAAAABiG0YAAAAAAAkA +AAAAAAAAAQAAAAAAAABrG0YAAAAAAAkAAAAAAAAAAQAAAAAAAAB0G0YAAAAAAAkAAAAAAAAAAQAA +AAAAAAB9G0YAAAAAAAkAAAAAAAAAAQAAAAAAAACGG0YAAAAAAAkAAAAAAAAAAQAAAAAAAACPG0YA +AAAAAAkAAAAAAAAAAQAAAAAAAACYG0YAAAAAAAkAAAAAAAAAAQAAAAAAAAChG0YAAAAAAAkAAAAA +AAAAAQAAAAAAAACqG0YAAAAAAAkAAAAAAAAAAQAAAAAAAACzG0YAAAAAAAkAAAAAAAAAAQAAAAAA +AAC8G0YAAAAAAAkAAAAAAAAAAQAAAAAAAADFG0YAAAAAAAkAAAAAAAAAAQAAAAAAAADOG0YAAAAA +AAkAAAAAAAAAAQAAAAAAAADXG0YAAAAAAAkAAAAAAAAAAQAAAAAAAADgG0YAAAAAAAkAAAAAAAAA +AQAAAAAAAADpG0YAAAAAAAkAAAAAAAAAAQAAAAAAAADyG0YAAAAAAAkAAAAAAAAAAQAAAAAAAAD7 +G0YAAAAAAAkAAAAAAAAAAQAAAAAAAAAEHEYAAAAAAAkAAAAAAAAAAQAAAAAAAAANHEYAAAAAAAkA +AAAAAAAAAQAAAAAAAAAWHEYAAAAAAAkAAAAAAAAAAQAAAAAAAAAfHEYAAAAAAAkAAAAAAAAAAQAA +AAAAAAAoHEYAAAAAAAkAAAAAAAAAAQAAAAAAAAAxHEYAAAAAAAkAAAAAAAAAAQAAAAAAAAA6HEYA +AAAAAAkAAAAAAAAAAQAAAAAAAABDHEYAAAAAAAkAAAAAAAAAAQAAAAAAAABMHEYAAAAAAAkAAAAA +AAAAAQAAAAAAAABVHEYAAAAAAAkAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABaTElCAAAAAAAAAeZ4AVSPX0/iQBRH +z2W27N1O2U5h6bACCgr6RkJCopFEfZ341oREv41/v7eZFrC+3fOb37mZK4UYHVjnC9n5sS71s1Lo +NOmRzUqMFtKXh/S2e5/BrxjMbByTlaxd8wbd6R6eus85/I648w3oBqOdNLho/bnCaHCTHqQx3+bB +dVKwR/CQHQB6G9Z14W89eMgTjD5mB9G10EPRYHDQT/bqoB48/LOy8zC0Ev8NZQ+jN3lw71UGPiFq +oxHwf05w1zmcbDEK4wVG76y1L5X9qAQmIvEOmx+CaVkf1uBb5eB0KEa/C2elRKMpwKzdf61crczz +1hI4jwgXg9aiurdoJ7D8iZclRoMDvgIAAP//+Hwx41pMSUIAAAAAAAIIW3gBjL0JeJTV2T4+894n +N0Nwa5tUwUrQCZAoE3WCTpAZyBtgUg1V0jJYEzVBEjUEMkACTEhmBqEV1KCCFVQQg4pWsEqrtnUF +t6qtu61b1eK+1L2KCm39X/eZJdCv3+/7e12Sed/3rM95znOe/bzyd6/H8fxosMfj9f6zcIjH6/V6 +PR6P17NgYUdX29zW3N+j2zq6Whd0zJxzdGd3pyf/MGvewoGHmWe1DTycE29NzGtd0Da3taPrfzYy +sys+t23W/3w/d2bXuQNtnNXd1TpzzjkeT3jmwq74Oa0drQtmdrW2TPB4PJ6Zc86pOCfu8Xo8M21b +TfPidlTZl7POiQ/8apo7d+a8vR47286ZOaurLd4x8G7WzDlz9n06a+as9r3enNs68HjuzHxNvZ7X +le81Pnde25zWvR/ntCZyrcxbePacmed0/sdj08y5LcePHXg5b0H87IGnrrZZ7bkaLa1nLcxN2v7e +a9D2eU58n89z4uc0xc/ONdbSenZn05y2joWJfXps7VjUNC/e2ZYbZeuCBfHc+FsTWvPsYM6e2dk1 +J35O8D8eu2aeNac1925OfGZX9ve5MzvPzU/r3NaZ81oWzs2twbltnV3xcxbMnJst23b2zFm5Nuac +3dk1APnsU9PxY89qy7U8Jz6rvenshV15wOrFgpkdudXKPe4197kz58yJz8r2NoALc2fOa9K0KnOT +yr3IDzz3ojO/xHPPmrlgQVt+ieee1da1V4OzZs46NzeTubNaO7oWzMyh1VyLK3NnLsiNc27r3Mx6 +5MbV2rWgbVZuseee3dYxUPfstsQ+UzgnP5lzZu3d5Dmz5s2cNTC4c2Z1zpq5qLXjnPyYzpm1N3j1 +uLi1Nbcuc8+ZtTi+ID8+rVpubPNmntO6zwjyL/ZZG/t6HyDMs7XOauvKT2wvBJ+7YGbHOa35L50z +O47JddjZtiQ/6M55Mzs6W3PrP7eza+ZAa4vPOmthDsU7zprXNq+1Sf/kVrSjtWtePL+1s09NrXu9 +i2d3Rbbj3GOTJTdtOTjnX3fEZy5MLPrPwomq4wdedcQ74vNaO87qbMm+mzezI9/QvDkLz2nL7al5 +C1pb587LTSz7pPqL2zpa4otzgJm3oK1joFAekQXIgcnrac7Ms1pz+LZg5qzWHDQXWPKQHc2C1jkz +E02dXQvPyr3IkPp9n47d9zEHzwWL5+618zpb57TOyo2ss3XuzGylzrZzOmbO2YfOZF/9TwKU/bCw +I0+COtvOmb+wdWFuQLnHpo54117vumaelW1t35US4syaM7OzM49XnXPa8uSlM352lyVT+S2+937o +7FrQ1pGDkyCUWwH7OwcD+1CZm2rXwrP2peGZN3aiuTLdc7tm5qDd2d2pFZ43b9bxYwcKdO6FQl1t +c3Pz1M+mjvjZM9v3fjOzM0c5u+Z07r2SXVr0bKf2916HWFf3vPyn7nmt7W0dOfRc2HV2VbbSopbO +eFPrnLPz0LEv9p7LwIt9lnfxgrau1qbWBbmjY+aCWeeqTcfjmTVvoX7B41lyTjy3j/TGsW9UcqCp +TPkcMODxzDyrTWWN/TVQzng8rYl5TQtaz5l5VtvMBed0NmVO8wJPjh+YmTtW6fGIrVArgzyZ1dFv +ejxtHS2tCTEZTR0zu9oWWfD49Fpo0Nk2q7NpVnzu3EzDzt7vVd/ZewQtrWe3LhgYgscz2NlxRLX4 +FI9xcEhRYd0GurzeOdTj9XoGOyP8+W8PeovoOt7M++qB93/wFjHgMPO+eeD9Y/b9oMz7pQPvH/cW +0Z8r3z/w/qm9y+8YeP+s3vMF+nN97xz49hdvEUPOdzN9eErzY33Rvv9e5v2I/HviZW8RaxhyKjKf +mvf69Gr20zGZT/35T9jptWAJ0edfkAOMZ2S+szcznwOsYojDGOCd83KlmgdKvfe/l9oxUOojb1Fh +5wYGOKqKAbOE+G2qhCHjEI97w6MZ4NUMEIntdLmYLn1fIlVMX5IBpyA77FH5cf1roK1sY5tzjb3q +DY9kgJcxQHSrsQRd+l5H0jbmzzW2dHS+sV87RYWdGxljBWMs1f8oHH4qY2xCVwTehH7VNDc3N2NB +nO7AeKrLsk2gEAcWXc2QiRJfejoMiAvT9Xra5ZlWUcUYUwY4P13PUs50CrOz2av20KIN6pYuXU5i +Z0WuzM5cmYJCHFy0gU2Ed9psunSiX9NlMwO8gH4uZIh/p+scnml5Z3l2XAUoKSqs43r6ahji64JD +6fwc2jYfmS91pF2YJpayjr5b6XKNWtRg1iAVZyddpyjT9Iij8pXGZJtOOt/JfFua+0aBA3yUTfTB +DD+LsdIpESQqMTjBSQzwZAY4iyGup0sfCsrYRsfFwj3ZVzjoHboczRBx2h0IRPPAGJPtmwhn+n6D +ftpGDmIbhx6cn1kgW3BQIb7wWNCexwCOmIbWOGfPOJkxYkgDTJQuZ9DlE3T5Nl3OOAXfiTJEbE/H +GaMbYYxtTm73Vww0+oQp2sDAEVEzj/h5evkM4TQDZhCx2xOmnxfTZZx+wkkQviCxuJwBTiUGNSDR +ikENRCKi74P69W8iopVfTL8j0c8z2Kk+OtvX4EJ8r4g4uB9zIpzAADeyk3DCgo33ds3DKaNrRhEf +FQY5wwwmLlpYglQcyfg8vV45JEH1XcqAPq5euG6R1liocHkOrtXH5HrD0UWFnVy/hk5klwBwYq7I +znyRQoSLNtCFKbuWE7QZ6PIEuhxKl1UL6XJzjVCmjUO/n6u89Nhc+4V4WsvhHgEnOgGIEr0RNvEE +djawesS8eYRJsIYuH6DLcXRNO/HrdDldM5h43BOmyym5VquDA62+49Hu42QGUBokmioZKtfavOmU +5dZmMHFtslLQTrcyxBR64xiU0JK9j6C+ru0VnLoYM4PwHhr0al1vd5meiTfRkAHght4I/SxhgBUM +aHhr03EVfd8TZvOigderM68/1mszHmtMgjGV+6ynxIzHKhPE1eliduJNT4sZTB8+71nHkBlD/MYb +pS35aqrEjMEmb9CMxyfe4fqK1alK3sYAyxjgBobGqcVnUvX0Y2YrUbKWLtFTqe10/NeiW01q8U2n +3wzGxmQlAzyTLq+uosu5RG+lGYNVTosZjM+TEaPf0fzvi+zvr5IRtbDS0a4oZYA1Iqp3JDV//MkJ +TmaAqzWIz5KRKHpbgQR6Swi0TEJmC6n6Bm/UjMfzToKrc8dc83G5xcNFOju4nttq8pSpP/+1EJu9 +2mqs0hE0bJiAPobYkI6YdtSVa02w0dtBP0P0M2AJWKkZgyvS5WYw3vMER+ooKGWIIxnCkhIURNFb +GeWUKjaJIDEaQToyGelKDIoS3rLssHNH+NLj8+N8NHPihNiR2e0YGSTOFKYRKDuGAcwrwXcSxJAo +A/yRdujs7QxxA3rjSMcRTBCd8RkVPIohwfKILg2mJzKRk3gGvFGkNKeNqYgZjNe9UTCKdMRusGME +9HQlh3OqaccvUprah94gkS4nUiUYHCYWlROD7sOsOGPsnIzBluZk+prK0ElqrSdSbdcvNIkzM51t +s5297I0SyQhDPFo9dZl23JCyoCVe9pZluvGW4aw4fa8zlDs/d1RlQVOI+z06P06w5CmEVC0KopzK +Oh8O7iAWRBjgNLr8KUNcadpxS7rEFBLPeMI+zGskvhvlCmrTr6DrW8Qj0V1Cl7FJxD0FZXhwSSUx +aLel1DFm1jlkaQS84cx8mwSoGN0J+gfpWmd45jzqPyE3wkKsFBq57MpRj535b7hCC0udrYGTiB+U +iUacWMsQJ2k7jQ3i5BIGcGyQVfTjrDiObiAWbWWTjg6MvI84o5w4ogVnRIyf+DDVakaivdgU4hZv +C5uIW0yC2NxTwpGl9HMh/byzhjG6xF9MlIEqdTs+y12N1hybq86gHz2VMEH0lBMmSL9WDqk4RfvB +qE6UEeIYqulSZzsKowzhgZ4I7jVBHzrK8Z0+/KGnm0iuE/o40xjA1DjxoKev1PixKl1sRuL2tMb5 +oDdB/2LW1DSrsVkaAptYpaejWMc1DHE2A4t4lHjANjaxjk16M2M6t7H3iIpmnF5O+IP0Y1DUHndz +yvEJhmcawqreWnyOIN7VXK7QZLC+NwJfEGeWg9GaGuIziMDM1kC9ZapWVXUc/fh5byV2I4i/qWa/ +CVoCpMUPYGNv4zHEKwgiXaIKzbbChb2V+BJBvKUK69WVBU4NA1hruyTmlzPG2Yzx7zp/XP5BtZmk +Xxg2Q6xa9mk6x9m+3F8LvaKMMcSmTDEiFSGcBniF13Dj43FtupIhnQdXpSuF3Y94w1zDJqa02WdW +GD/W7AXx+7ziSdEZ11l8p3AfExJs4lTG6E5ijDEsFPPhYkrE+HHTXlVvtOxoDDNqnWMzaD7CzaG5 +xI/COj4oPLw2XWtGUivMoaYQv/JGc2xa80D5jY4977F/lNV00VlC/RSENVt3Ptvpb69gtQW8qL9L +30M6Pkrp+kZaBnE2zok74cxI+mvyI3nE0UhQ0D+B1QzV8Cy6fAjdcW7mUOf7meIjJuWL/zlT3Ntf +nSm+wRZPxp2RmaL9A0VftWPObtcoQ5rs48mI3XQM0DWFeMIRZwcmTp2DwWXjzXR8Gxiu9TWnE/+c +VqwF+hQt2t7Lko0M8KQpDIlOUhvTn504vnGiWJmMMEk3M2F84XQQFyUjFfTTh384z1b9r5vXbtuA +OPQMsoQnZvdriF06G7Rf6ZTiwuQeDfDD/zJAO4kr7GErllWbUvRPHKuwHt6ouC7fSLyuYW3MDWun +08+QDsyTOF7H5j405f8xrDwZoQ+XJj8zfvw5mScSxL+dBEemzHT8LZAQlImrf1xeaqbji8BwDlV/ +Ep0E2lU/rkc0bvy4MJWvjz84EqNiAkaMsRMtHhRkFndHbQ4PCrEN4i5Zk6PTnh/mvmEHigo3A7sZ +QG8tsV+ZwNios0LnO/zTcKYI9eHTiIKwSEl9pV24EBGahpP07bhpnGa5hFCGuC/eTj9atmNUAw5p +QUfjBi0+jrmP+HE5EWjBjy1N/31vnqa/AdH0i4wfn/cOTO41aPfWsIntQqrPvY8STyRLCFNWSj8f +3ovm3+CIPbAU9b+tz/+k+aI04f+F5icYwr+TEZznZGn+K1iW6iZ6dObCTGOI7figd2/ShOUmzCbT +jn/21GuxfmaiIlt7eipNIX5mgmmutgRgb9L1XG6qYkdMIf4l1MQv09uJlz0JHSdP5kqYQiwzCfof +yR4nU/PHyZnZo0To6xvGMexiQLwSFpazqorTLR1uYlUvRxwjeUkHSVPNGPHZ8EXhBLHRG8RzqXIx +JNjmzR0vf0zV4g5vEE/oy2+8QQbw11RE5WeVw0Qn12R4arsjxOGpn6oaHS8PpCpxrzeIh1XzLtWk +9oBOi2dsC8Rm774HzCOpSvzeG8QfVeWOfao8n62yKHfAvDBZ5PIE9Zc/Xyxo3V/XiMrRHjapMxkT +TGYwZM+BSa8OnAnHPKrzAOena+nrHNBc7Dw5vydWmqIcwX+ndx+C/z7yBH/EKfnyL5kMwZdcmhJ3 +KbXLjxjC9xKAWFDMjogXxOAoVvdWZiQwe6ThNROWxINNPSXCm7+ZKEMbuZkh3kBX0ghLZzI0LtMM +J9KP1s8wLJjmGHtyCLb/7fQ4PX96eOrzw1xeUFRYB2//aZJwvWHjx197Bnbbl8aS7s54XhOVq3kg +BhexjjpBJVEL1fRXZ7v+Hs0Yb3CyWr4duUoH4S/iX7me+F6UrlQ+Q6JER2Mp3cuxX3TiJHStIlLl +IvrjGeNU/Zg6k22mi7gt3UrXHIQXPWt5Po8xXdic3moO0s4Ia8I/pZ+zcW5+ltU/yc7yoEL801NE +mP7JYr9L6f7C/lDjhzF2N1dITGXMGbZLUjLQwBifUnOlRG+c63MTaZ6eaxFrvUWFm/klZ7ItNYNO +xS66/B39s+lLMsRfLrUsRpuZTvzG6WPMHEQfnkxqW1eNmTRxlrSAbg2bljIwUPKX+ZIv5kvquLQl +uZSYvZW+1WpjqHNghqDvnJEd0fcKUVzEp+gbJbJYsGwq6+gEcaTYeZyySaub3MRYrt6OU3P1MNLi +KR9iQHL82oKo+R4+WFKuVbxTm4Ix1iyyDIgr0W6lE/l3lzPh33yRAWfWv33NdNoweDcBMVoh+vap +KO3iGgZ4tBWjAxQn80S21UmYEc8dPiMac+MpxE5P0YYazuQUKd+monw3vttnjxeXVWIepoq3eNwz +HIcnzPeIi9P1HEV3Rq4pz+m5pvBeFtt8t2e4LMt02M0v9aNwVcyXVCm+xYzx1MwZ7rKCdTxt6kL1 +87VnuDpZmY7r6WqvfXozVc+kOR27PGX6eGm6/DR9/Xum7OXperRb6RALanOjaj4jP6rtXumBT2XA +MseiWIczwFPp52RONqdjrO1j6nIc0qc9m1V97xxo4H2JUb5TZ7PNSmeTakTH4JVOokaUrdOHhcU5 +NeKOM/Mdf6qOuT7LQAgaIbuulyEdz3Uzoilb/GAwgxhi1KTakSqxlDgwivbIaSyjH+nlFn4VDDxA +HPQKA6wj4pGu6kkLcGCQmKcddiZDVmnqMkDfHeJIfkjfUA7NsSRLm7MdHiKzwWa7pQJ8UpoNpMrh +baGEsRCb2fQKL2m2p6VLZ7t3F12gH71xHxBEb5yT2EQntIt+KWg5Un8n/4ZN/IIxOlOkZXOO/Fa7 +o2uryMV6/UxuzcGpuiU/jj94ROrXW+D6TsqRvv6BAs/sU0BbfqjjeDKWjtZcM8TbniLW5GhHdf4D +bvEWFT64S3tlNv3cxhC8nyLZSOwfJhY0wmnhRK6g70V2+q6aST/BINGdUZqhYBqWROzxWcrQhJpJ +vu9jQQQHtUyomYn4crAF0glMEr9cI0EPC2spQd+vOr6TmrFftOZMdEamwrRkWgpMZHAMV2BNelUN +8TOv1C44tzbDvvmmnpmrcZIat33vXQO7PWUa4C/oZx1dxLdiVbqcK4l3PEGsS3fjI0+QWJMuZwj/ +8JRhVbocH3oSxGXpcgZ8i9kpxvE9T4s4ySvSyxnCu57duCIdwbueaPZvC65Iv6pn+mwZvOsJIr1K +I+qOYP+WmjMt4+bi29Rzq8T1ZHsv36t31hADHeNzT5h2qKF7/+sQHs12vVY9ZvrGFek7sl2v26tr +MMr7ifPSkVVY6l2LVDeS3Zdw9SE5wXFpew4rZJ+yWL6NATqlXzPG9VK40aWwEof0MWbPo6lCz7Nl +VMF+LYwxmT3XseA5fRmmcm10uU24jJZGvfyu6FxbruCcx/TOlymYYQqwyDb4/UyDEsRc4lzb4A8y +5XINnqVjxgcuY2ygxcQmlS96du93s+07J1NOIHaJXvvuIPFUbRbvXKLDvjvw2+yERafQZXsenuk5 +JO2HSzTbl9Lisi3X4BJb+YB+e3KIbrrEvE30UxMRZbHNpR7D4WU4M6JDBod/isatOf3Uznl5+O8W +8eTfOZNNomXiP86SZty3iIERDE4VETnhX/p34td2BzTRCX/FJqcaS/YQVidZqpcnfqV/3V3/UWjB +HuKAR3MvT/rvhWbvIb6vbWZbqrGFJqilC7QHWmr17/Ay2z7O2qO/J+LQ3fp7lB3asfDuRkM8N72l +ifz0tjhFhQ9+xSbppM+gS1G5h6whSNyj1sfPL+lWO0f+m1NEkjcwJKtTvpAEM99QutU+0XcUBNVr +97/p7nLakN7D+cK3oXSRijxve9CD1qO3mP7caefpzQ7oMHy/iDFrIWxniKNHMcRR+qeMIcwpIQ4J +VvBItEXwvaBpx1kRcxgxdBnKhxsH33qHm8OwNFVuHOJb73BzGPFcsrwEY4bnv76fLEd7ueSqc0tU +93vLcJTq7rZ1f54qUd3Mw+cSE8+tLUFFvgAzL1tL1ECHbWC/ZRUZXgWHNRBnRawC/vDhMgYNlkQr +qaOTKAyOqcDCcuNgrTPcHIY3kuXsVF9XO0GN441kJY8h9g+irpxSbXaUj8Mhw9FbYtpRXa4ix4SR +LD+G5Qyp91NLGNLbRzx9+ckt3Wvqmil/aBxc6UTNYXg5199m9YeXk5UEgwP9GQcr7Ze/JiszA7s+ +94iDgwxoqCudDnMYXskVuM4WeCVZie9L/4ezW8djuEYMBIleGRvsX402P4OF5RUVUpsiUS4q7p3G +gOPLHIWeVBYJHDzjFJnD8FWyhJ2yU8qoIiiPESSXlKNweDGWlOdOyuaBei/bev/QXOmTAJWvlS7H +4OHoLUY6X29Hrt4IHFok83cIBy4jFsYZgkR1p2wq/eiNEweGxdDU0WWH8aNb8k57MWMcakZghPSP +vqnGLzTJ6G1HEEf2EZ2N4kh9NfTzLrrH8waGriMQnSHi49YYP4YP36eGSxczSogxDVJNzJOazDKd +TgLT95hu/NIri+II4sWU9CdOAj17ROFOdEZnILjjvCwER+A1T5GIwn06MokHPFqhGgyaVsExRHe5 +acev0sUMqbXnPWGBaVRGJ8bvD7g8jFiWb+8fnqLCTmmvLp/B0BeyDKBThgnfMLqEt4N19Pk1sZsM +cGla0CnlBbIWqIuPPUH+1BLeQA16xOu6sj7E2IZUbY6jal6e722Ddb44XjM4LzODb7MzOEAzwIJy +DjPt+CQVyUxhtdeqlawCp6yiCj3FOeTYMdDonVkRTKZ5q00Qs+iyqsq0W61BwIzAHd5pvIwxLKk1 +7Xg2FTEjcJs3WsO1pQzBPMpJxHejUlVLCyy7cejSGqAFRVGipdF0Y0xGh+Rq1tMjdE034V0mK6wZ +IW01Dl5LtK2SjCPBBwUdoo/STT1JF4NlPmicxCafzELdjWIUVsrqMXu7uNdn0RvJbC5pkabypBrO +Nt3HmRGcxTMzfR3RYUbgjDhd5zAcuFss9ZpsXxpJSjo5zdMbNd14zxtlkwZ6ZaqWMedQpPdEpV9g +E5vZhI5X0dPYPZ+r0Z2XDpauyK/RJzoYjR/bU+vMCDzgFS5fn6o383Avotovo9QTcU2qkn4+yDqm +GOPbxo+v8srHEXjJ20dZ9mZX0z3+BnYywMrcqdC8Mt/Z7zNa6IJ+a9n2yRrWKSUOFm/K8ds7B0p/ +aEuLIbeazyOmsFr6BUH6Aqmuh2e2y9IL8u2fB7HwQL+07fQdRJe/EGJ355v3XJgv/DdbeH0X/awA +ZIaLdFnTr2tdC5bTz585ZZk++geqfZWpxsmMzbC8b2kNA4dzMgMXL+IoLInnzAAjLsp3tUVapA2y +HB6Sk7iac18Px/cyZIszzBHEAcMJ5xUGzOFEeyMDYQYqaK1UBdr8FxGL45PNEdgvCieqQnMiEozm +SdDphBRZe0lZO/qyQzgcIXWiDrrKrddESHWlcWKZFbUmsYbOYWrFreVmunlBbVW2CT/KJcDSz5r5 +uSnuGPgYUPvjWMez6OfJVnD3Y0kJURxtoIv0OmJhCcqkLYgRcyLVGJRAopY4UCaeUvoRj1iN+woi +tYouX2dAIg28VkuvV1iyinoVkpsHZesIMISGEmKUDHgPEYm4fdVSS/xAr9bgbEulBojTJfnZvC9p +bsPAXPoHPn0kKqmZShElglq6mOKaQ1xEP7tmCtNjdKUFjtH94d7gar4038HPJXfLzqJjYHSXhfI4 +rdWhGSjnCKZndb7KxdqMG1h1SY3sYbK/pl4lCsrEd3VPks6P0oXMmMEYmxmyksBNOuo0jHx7a3Lt +FeJGb5HdQO7Rsh3YBtdZU6Yf8+tNF/F3KSll4dnqvU/+TqIsOCjBwJGygDLGl+kyRZedUgjk+rws ++/Jk04U/pIppW7hLLYgO+olzNu0zZDURkApmr2HTzRGInZflBiyXPPkQVvEbupyXmVbWmbD5F/lS +WZe6qrVcoLMqJvKvH1IZWEhkKYPn8myV0kLcJPt0gKNYx1FZWet9xngtY1sGHBOX5iqMxEEaSM38 +XO87Br4MzX75ToY0eNZmOxmFDdAmYBNPsf47ktT5AWOUSvw2xuz/pzDEuYzxcr1Py3y9X9lKuryQ +GFFGFEV/TpcvV2iTtHXr3796yvBxKs6Q1eD7sFgyn1NicUi0zfowZJ+zDzX/8XGifc7YO+fbPyfm +YD/iyvzoPzUZEbV5JH8gZu/jZD1jcnFa7UgR10mX8k/ES55X6JrxxCuprexUgeu8DWzCyDLLQbjE +9HIG1ozJFGkdKBJgSgeK5LRYDQMno3c7uHaKGY8nUo1q5xbZyek7jzFZ8EX0Xa6wHjHeaWzC914h +5kakrSs9VX5Q2QGdIi+4NYz5hN5OC5HUFDVlDF7GABZtPZPSX2jHJvTJzZRwWU2kV9n3tWKCZphK +3NcjnudquSpqRK+ZBqJG++DqCg3kaAaWmPH4dbrEjMIznj7N8ZZ0SWaOf/a0MPZDUVdzDFb3Sp29 +2qoVA2YUzjNhnNOIQ6JYLI8YMx6b0xEzCi97EkRhCzMKmyHShvg5KVPUjMdlttAqbwIzG4lFkcnW +RPBcsl6ju8UJagAb7WhUprOcoqkBYpCEFm2MZsrpwruWTVzBGt4gV0l26WyM4ZBlZjz+Ktirtuyz +Zjw+S2lYF3t3q+nPUtm5XeptYUxvPkndwZA6X+MN86f0Se8gQ39MHiypWqJV6j8c+qjtHZ2RGvFz +0iQ7I6SG82lhXNw04EWy45o8Cl6WUT6iNErMFFN4DLGsRwfcrTXq858myAv08ryecj0vLZhGt5U4 +OGracWtvcWZkz2CL7NSiCLOt5SRQejU7ca4Oh00CTs9yKe8CuSPOsyk/gJ0awAb6Mfhb9LRyoVw0 +rmbobGoSLtsm0eWNwib4EhOkXpMjUMhuatHGqQxgfivxXTmDpJfXUPWdSk25M4d0wsEu8TBy7PM9 +JKbAouzqQ6T4yBHxEdflx3TeIEtVYjyNAZ5izSMxyutKTEsdf806thPeaRBp9zYwYCG/iDE+TMys +xw5vA15NNRKDWhjjKIbOz5ypMS4U46IHDQXFUfwm3YhlTgv+lIqgo9E0Elf0VppR+BBhPVzWW2yR +ZBTxIaYxZpXDi0wAv+8WfZJuW+6SIR6eIRwPU/z8NAZMI5b3PqZ6u+ST6I3idm+UeCoVYV09g1Ig +D02MY53IzqdLJBKPIi4tCMOUZV6+k3t5VUFYxugK0443lxRrZxEbC8IM8caqqqou04ineopZJw0o +LzCjcItpWYR7U5UM8UWGRJFxtxMlHk5KcBWKyEiI+7wdeMiy1S9KJzeDIRzVQqyUHvpLTxkeTJYT +D3rLGLBKcZf4myeKA6Pan7gnHcGEafijJ4iyoFSL+DjViHtVd3rjeIbabXs/95bhOjmUNcczc3om +N6dbC8KMaepPL3kuA7hf6w0fZogPmfH4oqfYtOPdVLn23xXesIDTZ4Lig65PSaK41wnjgWQ5Z1su +RfaSjJa/ijFc7/ThznSEP8FW+W/zDZkNmuniqA587G2x304kHvWU6efEEVY9JDIdIP6QmePsCN5O +RfA77zRcn47gWE3zSOxO2jniAaHWDd4y+s/Ea54oTo0Q0yTYBHRURG25jU4LrktHMKZFMMu2d5d3 +2g/REEFZFKeWY0IQ7eUoiuL1ZCtxi9qjb0azGY8NPbVmFN4y4WHEE6lKO+oDo7lWVnmn4XtB7N+B +e9KNeMXTgvbIZMQjmBPB57Ltv+cJar2cMlGrGWzq1r6pYmA6vEGkWjmaCIWJk8rpYkgYI4NoaLQb +UgLxWsZKpQ8NMsZJFocCIgH24C22Yygu06EkzzfTaFFyAPeuLmihi75UJf7hTVDenS6xX1RknBk3 +eOFaklFO1GY9jS4lDC2ia8YSF/dU6vg3r2RQ4n0Z/S1DGsKBliV+NxlnDFd5E5dcciVxqz3JYuhV +21lPvVr7+0T65F+e9QWpvjlPWG4dLE6GJ9TIal/BnxJLpIRdwcAMWTDXdMeFaZ+zj/NgytAZl9Jv +gGr+Kt/Qw5mGAvQd4WS5ouaBr38ebOlXiKN41NF2Br4fWIrjWoYtIBWPHLJSeb/CEbfkm35LlU0A +Vy5ex9DeFMaMwus+ETsUir1CorGEUaIoQRezy2uqK6opdUhdjQSAzh86WUHOc2uu6UKsLCy6hnWU +FCe313Z8tKiW8IZZx5NPlJxD/wZONaNw+WCRqFIGzDDii4UR8wMcPNwMk+gV4FmM8TEGtFtP1QFg +RtGH5YX3ydWCAS7WBLFgHf1mGFYuiu/lorstNxQ8XigQzWYbnfEKO5BlLjvgEb/Ol3qyUNLter4h +Rp17++MvHSj0gppiP5us9lc8VIiKvIDTx7qpUkaZRvw1oTN+wyBL1d9M5Kn61YOk95Dk0278uHth +RkUmLHi9sA/FUdOIRxIR045LLVtyiS/KUzMuDB8vlg/MKKz2bbGGRknpGfYjRCzaJD53WMJaPEKs +Mo14PFGcwevzfcsY0uGye7Eo8gXqbLmvhfhNWthbY/zYacchV8pRxD8K+6qn0H5igIEaujdajRQW +xMdwRmYhiZ2LH6Or4tf5+niCacQTdsa/HhSV30pPJLsHJ2uME40f/xjoAq8V9qG9RF/k79qmrlwh +Z1mG36++Pb8g9w/Rspl2XKuxq7vXfAmGeHJFhp5fnQXKG76EVFVIlph2XLdYsH/bl+AGuuJlblu4 +SVWfLIxOmkh3nOhmd5x+znOC2R7vyPY4Gju9RQyZdqxPS1jOYmpAm+ohM5p4w6PDtp3YmL7DjMZb +HgnBOi9/hESxDgcn55sx0OQn3iKFJOAAuU+LDcYZ69TEQ6mImrzbu0wC3agwnDLMEHPstw43fuLI +qMZCH1an4mY03vOG5X7WqB0E02BG4f5BYQbMaDyY2HTkZE7QaXsDYB0FpXali3Ofsy7MUyIIy0FV +7pkBuj7pzH5UTgSjDDnH4kd7EIw6h+FHezLItiNVqaFt967F2DKirpg44VsZ3zfT7wzNAG3E7/NA +u0kOjRvkgxPEnHIBi58ygP3LLCOXjOAfBX3E1UsixMKIpv6onI8sdH/6EwvdwJOcoR5/5Q1HWcqR +xPsFQYZw8ZIS4mdpcfr4jOLdcdmSbvpH0Q3Sj8W1GVnbnhe+lQwcwUlcwYekRn6RIZGGANESz6HW +zjvzY37WjhlvpWzb13jV9rTj8UqqXMzSdd4gK8wkYpvOAG7IoNtrThlDeqs4jJgZTR+edG437ZZC +6v31ycrs+8ed280k/DZZYp/xmFSuKnGnRdHHnOH2/TmWfgSwuFbOxL4gFpefpv2TqC3Tnzm1EF/w +G7FyyQgD4tYmMpT1CgpoJSk+VXoczKun5DNBdvAykX7roavQsnbicsVOCMDbxTRt9pZxnBzxk8WZ +d2HiJu9uBnBO3EzCr5MR3+H0xYRYzvCc0rz6njzs7oMwWlpGEUGF8YTQ9RgGJSwXH9K+w7xK4puk +gNuCtY7Y1LtMO+7qtYj1MMLE+zZ6Cet6KkceLaGLuENf8VtvWGpnXOKEJfK8niyvop+luMoJVplu +3JouEc4TJ1hGdaHO8GWp5ZrlQxm0vdvbgtVOCwOziQPt2TVvueUHQvKMOy4zSNOOG3orpez9REUF +h3c84dKjFWTx6gJ8lIzgake46Jp2bOp9VSU/d9Zmun7PlgwQZ8kDqBOdtTkMW3pfHkp3WE81aSsy +4mIIQxLEYpmjv2GTdY8pZZMo05dSE2oEF1qdv59NJ3HlFAS2sBn1r/LYRrwrTzYGcRnCeA9BXNEb +oXAdn/Q2Yg0knd3VEzGj8bBpYZOlGrirp9KMxi1W5Z+B4GpYntZZS4yOEtsQxKO9EYvvWKtz5Jne +Etxvo4jwmbXI4pXeVawTZboRQTzTW07cAh1hEpXwhP3YHcHsbuJzmeC4F1XDLzFNU2+rt8D3E5f2 +bsWnCEv+pqv3ODioseJXPY1mNJ40z2JxOQYFbWDKzHIGkKzH/Z4olkRkBsHZMhS044ae/LqFBLV3 +vNHSo0XUFpbgr0jgxt5GdDXKk39kFPtHpdHEKRHTrdAUNqnGW94GEe2HuUKkvucOvXvbrJUXDo+y +7lQB1tWUDmOT0Gp5j47e0biooEXWgKTGutYJys+jJyK8eF+bU230yc/VlCkozIzCr3PL+lTPOpkl +3qkx3fi7rb7BkY9IhSp/mKt8sRPmaE4wo3BzruKfezbRfdP6s66R9zebbAvlZrRaoLRRTVzEgNp5 +O7fF1ztC44Dl/WKsw032nHclFstvOYep/Q/mMfWfEoD7VcXfIDS4uNtqKT+jFOI4fasVh0NINgqU +x9ttFGKarpWHSo9hiCsl5IkMVx3POiyJEwzXNCuESJrMk+2BFhohTjhEn5zWSulibqX1V1yBJc+h +oENuRiH6yrBIDKWLtjy3uvTh/Eif8xUV1m0UcQkR3pZSwlc2j3grIUKDKwa1iAR2lxMcTrRLasDB +US31QpnTNnNo/vT6Q77Nv/uKCjd/pWmhqVIglVyipg8LEo8n5B6E/TukFsZ1g8LTpFm4TFSBuHKx +pWbv+MJCkysXawO+6wvXmHasWyztymi85wsTp0ijiGB4tJCdWL24Vcjycaba6sXCp0984WbTjkuy +1T71hW1kUUuEeMsTxvLEVqLf18BQKZrKiRFamaZJ+Mfireo6z0q04LZB+jRDZr8Ls43t8iWIhzzy +Nr+LIf5U3+5eZHGaeHiwaHOA0TPXaO64Ix3JEOMLFsczxG6XT1RjKt5dXI8rfeHSqpEMCeG+6q7N +7L8+TitlAKPKcHI5jk1YAfX3VmuKxz1hulgcsTFlOhiaVPOz7nhmH17MBoaOqMFh35puPNktQPyC +LSgsE5/Qjb93l5jR+AUTxJZEZYaG/W2QpM2KkazDAS2YH0EgiuZIo0hERRDt4p2fXKSFuHlwVKTJ +BE03HuvWlvkV5XUrr+UQn6ILb8LuIJc+aTteWtRqRuPGweGazK5/IQMh/HJw2HTjT3Zw1zGo/j5O +RTCxpRo/iOKAoKxE7dk+twyO/ogTMxbGx7q3ZsDzO0aF7SffIcG+rhxjgzhl1RiFY1lA4tHu58xo +9LNl76aPF0l/clFc4PrcER84mrhTLTXhxOdwfBDRcowL4qTGMTg+qJbgPIuZjXu3cYLa+KOAQdwx +OCGZtoYumhsFqYUR7Q80ZdSzs/IH2I4/5TfGqxINVzLEs2Ycw7GUn4DdJf9ONUoHYtpxy+JiYw9s +uzuu8nXIljPS7vYmG+lxmiUWinrLPJ/KEH+EjtYcEdrxeLa3soxb5IMUNv6GIT7CTMTjIuGOeP5t +Cu+x++2XXnHdLaYMb6UUEirGYmGlyG5hWGC63id4lRGvLC7PsRbzdBZaziKgL6+mysvMaBubWoY/ +pCLixnCnN2Ha92Ii6FfR19SHmr3JW2bacU9KStAyfF/KeMtcpOKmHW3leic1mYxRdRIFtNlxjh3X +IS2Yk6FqPbXOKI/1aGx+Mj/1CrFT/7+mvjQ/9b50buo/sl0E7dT/OSg79Z8l8lP/4b5T/3m6UlP/ +xhM0sntkpr7a2/dfpn6+fAo09fP2mfqEgakvN+1wNfUJe0+9y059e9oObIdHu+5/zt/zdH7+z8sc +yAfFrFJxfiEUdlhbY4hv0c1ySlbafNkngaeM2LxY8PTTPYmzp2BUH5vx0008Vhv/QdEcana/S2vq +TwzS79sSlmPSbP7kEe781kbgXpOOq7mdHsGtnbgmLbpYRrznaZDB6CgGzWhcaZt4JRFhnRq4ZFCC +IVOGLxMleiwSOmYYJtX8NLGcdQLx5bbW25lCV0lU42kZ9Hs9ERfX1R0hDgridLmZrEvbtu72LjNl +mGIPnXn1xPBwkArPHMOA1a761eHCiClDYbhURlg5U9WLYmN1ul7d/8OzrIIuuktwWBAzyxXViLZy +lEi1ipZyYn/9CljBoY4YFJ2gUNrRGDqNriWxpkzFAtZvpNuUYY13GQ+ly6N10FcJSG+nKk0ZrvK2 +yCjPCmtGhHSgsgBZ41DsxByae57LL/Nbjg5vMxrPW30yvC2lGvC1S0rmaVYKjPHbQ9yUYf0SLd3f +CoIq0b/EyinPFETp6vnXSyL/7UzvH+jqEyd7pqvhZwvCCj7Wsd5he9yyxMJaUqg/e7ar2Ru906bp +NFiYOd0vT1r8/cCxp/vlSQH973rAL+zvD52wGY0HcoPaviSePd9XJVvV3OcqS6yyhf+hB/QlG00Z +vhCXOK9RQ3uhIKqiL9nZrktt1UOfs+8xH5hkRmNF6j8OelOG33nzR/35yQxV2u00EGP2OemfssFT +ZcSvIdo6cNL/KHvQ/zwpVqWM2O0ERbfMaLxpx3y1Ey6tGSkvr9qaagyPasCrc9P9yxL5Vo2yh7qL +77XgoBY0R9DTiPIyqyGeH4GJoqdW4gh1PviOpmvdb30HSRXY22rKJM+YdnzYK9j+QkLhoOjpk2VL +bNNyOVusxjYkgZ7RqWzCqa0YHERZ1IzGNyZqyvD4kkbFVpWLND9mm9ntJKTxbo8Qi3OH2z7WqKUv +ZDHyqEKss0ZrFJZNJJbEGUvB3MfN1lFRCmSXL6FrIJjixYGKH3qLrj6GoSNPlq54AWOlciHCgVqQ +gDym0Rvnav2EdlsVY5LWAlXn2calVLMOdAFibnwv986XBjq42VHWj1umTFxonX1SOXty815l/uAU +8WrriSPpwK3JlenPlRmDS7xF8IaPNePx8f5BM4ZYvaCYVXkt1UDBfq9UbUpFw1Gs1ayQiIxjqIvo +KWEATDDKcRwqHVMVQ+iOLzoShUEb/GZPamvTKJVGFgUJ/REWvsEQxzGGnnrZ92CDpKs4qYvVKoHO +uP2TPRuTeY+NEX/NAmIMnpKehuvxnSgSEZviYHFEY/DJ8rfBjJc/T0ATW6DzUN7dn9CVHec9TwdD +ZgzWpSN0pn9FV2qJKda5ypVgIMfgmDxLKJLl0oeuxn3SAzS/mhtEIXY7RWY8vvL2mzG4IFVsxuNX +Xmlb8YhHpqUxxPmpSjMey5yy0ygfEkExYMbgm+R2Sjb5KQMaVFetGYNDwgzpYUKZGQNXw76AIevD +IBNGmw3s9BPJrfq34PZheCVViSUSh3/lbWGzsKZNKW2uYYDrM6WdgzJMRf9rA2P+PeSaIjOlvApG +wEjHPpWB0VYhLy/vO+i/m/korOq/DdR9xtbtor+KG1glUm1VywGrblnNGNMy4GdNpli4tYoBqRVW +5BDLszPXFv6edefq0q7SEZKkywnznarsiPMlC3GTTUojtzVrCX/IMw13p4vlzPqIp4V1AvPdaYH5 +Cc/eYMajqcf07VVPkH4B9oFUtxmDJ7xZOK9JN5ox+MgT1nlXIF4SXyRb1QfucrQGDydrrRlBW/UG +BmYSS50GjsbsVeZ0vJlcpcZvkwYPc1vx3QZJDWYeLk3HZcaT+94Y4gWbNgfp1gTrrJpVaaKeNPOw +aZ9ir9pibXuVeXERvqcxSZ2Hm9LdKAzjUU94apWJWoXYGPwJSt0StfqxKgnaKTMGj6CBI/gM/fRP +pjKhRPH7Xs3zj4ji6VQEN3ujCgKdhFukPpQlOGTxS1p97JsApUmNv61MKXgMUVmP061KbpNxTm6Z +jLiYLdyWrtfgJoRLTRQ7bPKVHWiwv8urlJBA47oHDTycT2fHlaw0UTxgx3Uvong1FZk4CddrQC/J +J/X/x4C2IxpFshVOFPIzl/vJB8kI8aE3IXYjl+Zk51s5hCvER1k0UuDqyVIU/FGrzTEmk4am04zB +6lQ977W+iJk0NMQWp0FezMvN6diajJsxeNVpYInC13ChXRQ19K6nTHNfZ+f+vp37ut7K5tyavG3n +/rLmzshCE8W6XmWlWSt7+LOa9NZUZBLP+N8Xoo14zBvNV/wIUSSXy99lErE9GSe2OWv3nvWOdwZm +fUGBtvsWhszpuDBZL5z80omWaN9WcSzlCoq7e1oZwCOmjP5xDHBiRvdO/K6nBH80muLIKvp/CEc7 +IYAlrVL4dEmHHCW+6CkxY9BntOJf9JRXNDPGXWYMVpoGufH5OQHdERPFUjvpi4zUdhPxXcsDjDJR +fNJTKcibLDJ/0lNekQPcKmNduf1SnP33RlinlDS5A675vYF5312g9Dqn8FzrvGF9v5AoISQ0mtPx +RK8FxW8QPYJVk4h0ScL67cZYiq+XVGJlQXCkGJNS7bc77RAfyQ3xzr2G+KAclZaUCA9lGjJRu8pW +w6AFu7NHK/2wUQTyRG0hTrEyjxw/FMUqzjgdYSedTuusYo1eOeTt/yA3HXxQUFSoQF5lLxOAl1pn +z1CNFF10sahxEh+gfCRc2/QuBsZy7ki6Nj+NOitDT/z4o7FzSTGutg6t6JX+tU2mNrc2R3U9H+Z7 +3Maiwgex327h9796ahnjEf+xUR7Jb5TATDMeb3gbhFz9TpD4sKcEl5l++fcKezBiGkM4tZb4XLFp +klHxj8wv61vxt6SiOcbgCjQAUbyRqjXjcV5KZOsKtBDDW0S+HxK9IO7K5Lc4WufsOwxhahwfpBrx +kjfYrJCMX3qXEX9KleMwiW7j8Zpdu00myKphDOHzxfW4iMvwtBNEZQP+mao/EstSEXzpTMMBy5qx +J9WIqeWojxAvJ+qJzYPCPrlEoeJZrExH5EYQItggZeWPspG2F40fYQ02IcryfaP96xffCLbYTS1G +N0B807NcPvkyzYjSKs+UFCN1xPkmyhMrbEVpw0QAs3U+79mktnF7shh/Yxj/ENclgnCxifKkTFou +LsSm9FaNa4iYq68w6HaGFh9fSqRKaljBPzDEbfRzBqZvxdO4HVclahnjTXb9mniXPdpeGdTA0Xyc +TVVZ36qpavCTxDoNWWKV5qZhayZ/0iJ/y7IziZ91N9JGoq4aFIW/hWhsZAB9i1qJLwa34KJFlfhy +cBAfe6LEL9NbsaCRZzHgQ+FwLNzDyQxRQaZJpZiYiT0Li9mEDYV9lunCgrjGsJ+yPWg0a/CctaZq +NGKJBkZTJji2EXco19WiRvrx2cJW4hWZcCy8paRqIq5ORrChsIV4c2EjsUF86tfJd8WD4SWRMeJG +E6zCeYli4qFBQfxySTn+xhZc0x0nLmMLcWIJ/uUJwhvFDakIcYm3ZSRDnMYAqjpQW443ul/FdhVs +sbL6EGGpliXAKuVImD2pYpicT/D4YjF4uM3XMDI7R5tSheOmU7rq61OVylElwUuscAW2LS7G01Ia +91ZiS6oEb3iDeF8wXZcWaayii889HdiULsca2fo+8rbYfo4vqxqmmb++qJLYODiMPycj2OVtwQUq +9YVN5GRHtrFC5U5gE6NZ/MPKxa3ElYODeEkn68+dFrmshWyiQ1g2/knp+3KlP1vUiksGh0cyhJZK +FAfxbrIc//a26OfgINFRb0vKaO7iwLBchT5Wu79wWjAkLK8NAQjvCHOuGxzEU8lybHNacGgQt8oL +bJtdTC2kbzYDQB8F4fHaHRWH4k+a3+2Dw8Rg7YEahnBrMm63+a3JCJ4Swi6OHD8Ojy0qxu2Dg5R0 +N8aqEd9PWacDGUKkTHGZdRwUP4cJUYyI4sAWPJiKoLIFu1LlOF/GUrlJcgyr6DsIvxS+vOsNYllB +C97w3C4b0zmN2Likkfi8QHquKdo1Um9gyToUBLHBW4aCFryVLMdL3gQqo/gwtcpKHAHeRfx8SSVe +LGghDhUBsecuFpRg/yB9TzKEm+TO9rxHkzj9Vdy4pJtYJiX29/FVd4Sr8RmGO1kfBs/X1R79Z8Zk +vCa4nmsYOwmm5QxWa1A605GqhdKuoYVtfJgBfqQ8CCMn8Xhlu5OzFroiCp23TOIJ91ni1ETc11VJ +Fw8MaeEGJYuyMoczRgdZE34gNEpxnN2pC8ZZ+1msmUuV+6GCoYqB93WMlfIshjhDAuSJJThaIRac +zAA7JR66PiyJXHLtqotXEb2rGMi3Vso5qsVA9c3oWEXpB6wXenNWhNiTn/ql++n4vIYTGKLiXn13 +0a2uofRat0pq3DhfNTnhTHYeLGeroTmvrf5/5tooxK/3K+J6DHpUtkNnGtrLif2jVRKicGFXJXYN +CduUDRmCc9ZohkqJ70cZwD3eDgWdbQftYegVgpbikc5iuvjLkDD2eIN4T9vRiWL7Pm8/1dunPFEd +DaITVcTTCypx6/79wnD6pg6rwvMLyonnFcveKO8/DO6wy4M7F1TiD7mCXcOqsD1fcIEtCBWsYeiL +CkWPb0tV4m/eIK5MlWNBudQsX8v+vyIl7yD52JZWDMNVXZV4e0hYJ1hKJ6A2/4yqYVjdVYmPh4Tx +huZWyi4GukQruhhIEU+KUL7eWYlN+0X5kwzAnu3cilv2k8kGa7wNDJ1fwZWC5LXHDsNvJbLj8f2D +OChInKNdbonHmSoz9thheMIWuGN/hUfa2F0IorOzMwlRXjoifjOG4a5OS2bv308aGAwS/yqb1EXp +enzhCXI2UVWmkFFtfG2FEL7/LDG3XPGyXbUYElTasDoGKvCdoN6L2jA0xh6GK4nJlUTYyo/7y6at +g0lLJa3sizZwGo+nKlEUFGSPwVqt5nXeoKhOgBuGVeGTBZXE65rIrYrXJ1L1FcOqsGtBJX6+vxxd +FtcTOgrmlxMHaA5rON5mbJGds4wBbE+XYIcniDdSEVzt1YxCuE/k4X65X+IlT59tGslyTiB+YuFS +MQwPdFbizgxQzqgnPuqsxGX7BY/OLA9+7w3jEQ32tc7luHY/2ere7yzBNfsJ4uWYFcG/5PH+mgb+ +S2VUVKaK54UuN9mnqRxq4VBaNQxfdVZi5X4ZlBmnY1BbtmoYPu6sxOr9wqOsJQgPqfLdmcrD8F5n +JbFuvyAe8JahP1Wr0ePwIB70yscgddQJw7A+B7npOSJGLIqPGTcM1y+oxIv7BxlwTIYOjIArCigS ++KcDigqVkmPKwXII65JOfwzxtyEKp19BlzXYNV8k7YIDwtaTDkkpVIY6RZmWduRaCuBJ2SY2zGad +M/Rfwj9FI17FNrltXDW4jDETIN5b1GhGYfNghZ0G8PKicvn/yoCRk1eWmuzIAkofa0c22/hxRrEZ +qUSLAYQVq79GWgHjx3cUIihvvACO70M6n0PEU5Bv5U2P9Syto09Gd/jWYlHcmrez6TKWDhR9S0W5 +frZCSp2E8cPN9/qip89UIi6fMwVkNJsADmgwfuST9QUUgoFEfh4jmB9Bn/SFVyvVz3cYeq5d4VT8 +Eet4IUMKLZI3uJzpQzYwjZLfTllpxuM9JwOkdclKOsMzmQBcds28kC5lanyGLrfNlMOBWYxtg4az +zQR8eD6xXS6bes0LbaBMDXFunApYi0mdJ/Eq45grKSsmDRNXH+wcll1OX37YT2rYGxhbgMKoZI3b +nLAJ4H75YHDc8XDk/+A7VAZ873C6Wtz3U5KfOmsUDoyCaZyrYCLpGadmxhyjhnp+Zqj0YU9iO2Os +6qNrKujDC2I61M4vve+Y8XhBXll6vEmxyULKBzwNUkWJxZQu6hOrR1S63IyeopEXWfY9o6dQzd+k +6wX2UrpmDHZ4tpgA7knHifmCj6uA2xjdE51jM3OvHpKf+9eau9UYhrhyJusktXdyag1jJ9ZQDPoG +rdClmRUiPsmEla/UsgD3SUWqNMnqGPPvGAMkMCjM6GTGMgrtruYLJVVfQOWoXo1EcW43jdg/P4Rf +SpsL9CuNkojEeAVq+OUxn5K/LXqLOb+K6kb5nX2K6HSxuDbXkueAbEsVeEz78mozHo8haCrw294S +hvAnTwK3pyO84acaiRmHzzxRU4FNaSVDrcFplQIcyqNiNcaFKZl4qmVmMkLY0qpxU+jyTXRnQHli +hG5Gi19XywDG3oe6CHFcHwMac8CMI/5qHUIriBvTm8w4vOopA+V/KTKRQrcsKxU22e84vOnRQKev +I6Y1MoCjyzCtfpWNDnEV5j4OH3pUc5r4FN82JYebypipcH7txe3pPaOoOGmcLEdOTIoQR4WVQNDl +66KLkzS+cJ/9EOlrzoztkdzY7rZje+y/jm1bOmLG4an82Fo0tkPL0FK/KTe2SSKYYhS2p2v15w8e +pURCOlJlxuHP/33Qt+QGPSdiQ6WRVDeP71X4oYEZ3ubFr9J7bE5SxS9GBRAd3xQL5aon+nFDOo7E +cnyvjw8enMeH4jw+3CnkVjDo3KkMcLLO+ACWlIsXH0U/PtMpt8ZblpFn8EZqubRN1yiikvgwVaKn +dZmnnhJmEkD5FVzeKFSoO3EfZtHz/Xy379puzXjc7Ej5XEE8lyyRH1MpY2xXHKRlpZVmVZu9WydR +BTGoRbtXCQeUxOx/3fz1vHzvzV/BzOZHT7cUUC8KNr1x+s0YPOpJmAr8Jr2KnYfkoNN8cHaYR2N4 +UWHnNYzZXaW8OKWMHZG6tMYyxjMZ+KGF+WoN1eY5M30zkNDEXXij1exEop4PIl2cP88Oybc8zh5E +MXM4eiNmLHFeYXAU68zRxPJFqxji0hnWPHxiLizbMzRf9UFtYa5njSBzjzeoSg+myjFkWlYJ9RcM +Z6c5Glt7620apHMYWMijRyhrn3BykeU+/OiOY/GqBawWjvpkCXMmfi3TjijTC5xkTT5toiLRDEn0 +HJodwjGFOKCI8PXL1iX6puRBUxk63IaJfMgVcl6TjsbVAy+jH3JtG2QRtGYgvtbzg1yDOFzyBx70 +9BMz4ozB2yGD+TVOHwNyRX8jWavnq50yPb0ly6vCXvTuSjnJ6u07SQVJlFkNH7yfchR6InCivrQU +86n4xBIfBvVreSTKBWb4HkV6eRl9wBbGOEc+d6s1l5Otf56b4RAuw8LMck7Je9VVD88P+l9iETYw +tpA6+V6S3YSJzICvS0UYKKXvLXTn0+LszNcsxCqvTJALGLAhdwMmyBEl+dY3aotwo/yiOJMuv6yx +oSoufylhrHOCjiuk8w5IO3M1g3hUCLKBMZQ2VE+unkCco1zCjVn+JYgJ4ppS9tYH11qM5fUuq5T+ +YkiH9Az6WaWbMF6kWz1uZnMzG3VgKeNGlqzVUP5JCujK/D9RDImdUIBdM4wfCXUofiwojonoicuZ +cCeVQlp5V+IyM7eht9YSCfrqBhJdNh+RBUMQy71FhXXGj63pdSaI5z3DzUji0nQ9QyaITz1Kj5JS +3PkJrDZ+zMr2qbGZID7yCOfQk+cFd+YarsQJMtwyQGUd0d0U0iNVMaRonYDgHcDiOFEYtbmqQnxD +tsgqhqpVuH0yJ1he3d4GYKuiPW7xXRK2rNU5CXlEaXYqY7FGU+nnYvnfcCM7eRq68iNbOlDuZq28 +OEuftJIxZXhpE7S1JinNjEn6GeOMGQKoOyCNN4/M93WBOAbThQtS68xYfOOVtmG26SJ+nqo3Y7Hb +G2bKCmvyE5IR2s97lAJsGBJ7cuRqxKh8c9Yp2zRifVrNve+xATrr0pkAnbF4zxOl7yLTiCvTcqgY +izc8yxQ7szEt1qFdBO5JTwsWVooI4KAG04gN6Vw4ZNqMxeueFukCcWattYzH6JtlKXgM3oS8NpVN +QRQlNloZcZTvN0bJw/Z6k5GsuxgjgmgqwYgWNJVQZfw4a/tYjIhiegmOUob/FtOEi3qFGXVmLO6C +MMc3nq5YcFQsI36sA6iUMYyZRswomcEAfZJWRzKGkVu0aDpXkaoX5+CV0lV8b53sTDYr86FB00S8 +3POYQmfNWJly9OKLnmKuNmOxyqjHCrMY7zvD2UbfNjMWVybFJaOt2zThYTu4kBmLuxFFcUJijZVj +31FaU227y1jHlazjCT/S6qfoakEv6anPBOZ9bu6r0U5gqVmMv2T60Dhutq5Fyq14mU2Q6BLetZN+ +tJIxNti9O12KGtzUqxxrs1lHn+xLeB5bsB3Du7CjNy7Ni6OtNJeu1nWVRaJdXosGF6SyaEB87ZXE +4gwDd5smPNQjaGtC9xiFaJ2njAyyFPMCYcT93hbTiPtSjWYs7veWmUbcY38/qKsHxrWUmul4y4Jk +qBmLjYiaJtydb/FhExXnS1yeLra2fSe2SzBCtDbnQrVjTB59/5Q1D81mTOo9ik7NYMCUECt7tJ6S +CsYSu0yfKcHqHoUbSfK+QTgA6g6OsfibWasK/T3lDPiWMaCZ0Ye/mg69vr4nzs1mLJ43w3P7x1OR +H8C3GsBG04S/LpEUElJnvyqIEk975AypxVlrmnDHkhzEPhW2KBXK1iV7GFPxDIdA/LlH/AGcLaYJ +f7Llhc+/K4h25Vb3+zJ2wltmmqBY5JiaVwtXFwyHEzVdeK5bqTHGEr/ifTM4g06pYCe0ucprUVNo +824qkqlKDF5bo/ytw2B20y1V8vJBu+nnXC3JDXYIWuRXC6IJxnhhDS5Jx00TLs4v1ucmikV5aXjH +MXmw/EVh7Vxv1+V1uidMsE4725rpmhOIv/aU2Kn7cL0pk2RwLHrzrSw9Nt/KzwcVFT74FZ5ZogGX +sg6fF7yDVUtaKUsgbinQAT/DdGGlnXajTGRrLQkIa/+c311sGnEXOsTCCCh3YS2TzmG7rFZPZ6Fi +BpUd1+73Nvp0TxIoIGvdcNBamwW0iZ3ap4LiYRJI1puxirxoojP0K8Yo5kwEXCJW52TThQcT2UW4 +d9CndPkRY3zoJ4zxGMaET8u7pbrL4uUeCi/7JBftjZdlFu6v0+Llxu5yBsxYH95m2JTgqu5yHV2a +kQ877WinMmb/V2x/irGf8hqZHMTuSWDcZs1rMTvAN4T3h8lZpfQU48e99ihvLzZj8cqgvqkrjR+7 +7avb08VcIZi9PChoxuGZ7ogZi9sp2lgj0U0AZ8qMxdWishZca6eow6YMFVJkhX45a804PGlL/4cc +N5bOk17cxN2STNnEu7mCKycpqcZy0yLy+mGXrBMxMxZbB1l6++dEMZNmLG4ZJJpzSU8xm+w+vsCM +hRJI40Xcrs3LNVxAGe18K6VzK21maHQm8ebnPfWsIy421rjwec+rbGKsmk0atRPZcy99x8zgCjqj +vtGJOXkP3fEUxXiRgZP5E7ofzbdyga9ChWJwdmeq0jkSyT10GcoIOiGxdb7DlDw1oIQQSL8LJzrR +QsqK8DrzJ9C96/wPz//w/I/OoVPyjYWCkMU1P8GFibhgv2tQ2D6IdMyW/cj4cWhOFzaWuMHXZ10i +euOmBA93rdK7p5jQKSW+9vsZ7r55Qn4/XTmoyOraqmRsHuCd+gdKvOBTCXj7KbftJuF9TBGxmxPC +kdcGLRNXV0eZcAY4nBET8z2sHGyZLzdMZ4RIj9zjnEP/Pdmcji8Sxdqif/CGzVg8UGBPmHuXqNkH +Cqbpy31LipXd+LVUpSbyHsM4SPzsg6YRNy4R/u3JVPqZKhF7VAs/36fSpkEddK0v1jQGzOl4MLFO +be0YJOJ456LsvnxksJQ3p9KVhtF04Y5FjSr11GCdcbpIS9Rj26JNmdP3S1u6woYbWU6Fkqs2ZH83 +c6jcu8VeWNzQhkNB/3msE41fvShD6PI0vkMk/hvas/bW7sp87P5YPMkWzX6ZAE18O2gaXXREqiyu +KPrfqROxiRHRSBWrZKwbquwmOhti6BGBtKq+AJ0SAb4Td6XzSaGXTsov0BtaIK4n8Kia1VRkS5Lq +JOO1KFqi/pQhoE5b2LpmbFZBJXURkrt8yPjxu0ViyS3teGNwn5mOS/yijviT4uHM6bixvkRB/X+t +CIq1FcqPPBU394oyTTVj8cnptcaPL20jojbKijgWfxssZkRSts0AlfGEjKE7P5URU/JTeaiwqHDz +LsbgKKFYo9XOnK4cIaLEvk+VemQSm5TImDEc8C1xUIOMly7vtHyectiOYhOVwPnCzLn0wOLMcvlw +n88aN74nzk5Z2pPi2SRIKJWTJCxxjMryH6NkKZ0NLyeL2WbEONzoPIoZWwUxJ7qLrg3f8D1MpVwK +ZX4oKY6ALOWsi/QeSv7y4VyJYq7MeXV0pmp5ndu8mdW0rJ6rfDcuQ7ns6iNOygPjpiFFhZ0bGfux +ojmNH//uyq/O+iF9M7q0Bk7pV81auHzo+VjiqiF9ZhyxbmEx/Xr+oFA76Czjx6cDLdw0RCo+1b1l +77rXDemTid/48cpA2XuHaNOW6kS82/jVsMlktxyLO4f0IR4/iS5PETpMq6nBLCuF5/WZO+ryE3or +OyEFQBg/dgz08JLt4a78JCWJjsWzQ/royjfTzz9R+Whlm4kdIV/NZZr8DKI3ztVYlFfa7Jia7+zn ++xUVWsFMujI4mtI1+Q6JPUP6rNqzVPyzn02yfvvtwuclNuPHi7ZGBpd1OmPPkKhk2j2dOkv9RFGZ +GYsPh5QxNtpm49L1LyXj1ZLx47FcbTMWK/brsxYYuUDm+jzC6qZwbn2eCT05N/xC3CMLrfYYFsnT +K0h0xYkhspthfiVDR9M6MGeNadKMZ4xqwmidLNJA+BnwHU8gcSQ7OeM4CSjnllQxMEMYNyN7olao +hlxQZjB0ij5QBy0OfIUBxFch3X16r4J95vOY3BlsawisM3QMi3zYGo5q9HarBpM5L9kR07JTOg7B +osJOmD4zD5ehRRKb3EqPI/7eK8FTYda6HE7ECr31ph3bHEWQH0e8qJBzkWMzj1gFMeEac7b659nq +GZdULNyn6t+S2n9SVEKLRGf4LkEBIgCnitD6fqp/JWuIzOv/lTipVZPKKQU8P87P4NOMskaHZWK2 +hC8MbrHKBDlqi790fqCt7fKHDOXWdOdA9X+qejaJvvW6dGTrrZAqfjFrzHT8vWA46zTjy5ZUMkYd +1cfpWGiTrs9FT35Teabnx6TbJDq5HsUNxNxG1glGd9hrfLSmWRg9Yi9V/CbzdZv9uhcEH89+9VtA +iROdxjpeQD8l1/6V2rCfmum4pWC4OQ7PLhGzK760QSCwDHbbppz4ccJa4sdxu5FcYq4YplAFfWco +e+AZDIg6CtjOof+mW3VmzXSpAJGuty99Hwy8Y73zgwyz4zk1P9dHpGxhPxXqEgpbqXA6samgzByH +1zQspBoTNWzWzDtr6KKgwZ4WUTn4j6bbwUtElkXGtQU67QBR1CB5pXOSdEguZ8ib3EV7/niq/mm+ +/8+kFIJRGLG2WGliBgNjZEeOK7ryklSlOQ5febVF/UhH0BWnU/LtgKbMc1q+pVVOUWGn8WNTap05 +DjttxtTzU/XmOOz26oCqSYkE55OjTsfbx/SZ43BZfW0Z66Tnk5cnbUYQcaY8QYzGOcXCne+2aGIF +IngX2Qbai81xuNiRtW46cWXFFjzqCWpLzKCrClfU14tY2bIivMdhhdOH9rz0NqIhP+ytjm53Kth9 +MmP0dTE2pupejlC8LQN0hu5iCD2142smY0wUMcnOonYZHqSU/hGTZlAmoZ9stXc/yd0/hFQJ4Qhi +AmuIvsUMYdEdxAFRyQ09lTbZkb0+7MlkJGNKPg5/svoN7N9hRmJr0s76KSeBxkiN4nFMFD2VWBSZ +goKgaPQ1qYg5Dp9bdYe8Guy2k0Fipyes+V+TroSS6UbZx0U+XZDZG8EQeYaLUfqRtT+5ym4cy0Rd +HBBVrdlWm36XXsqqg85Vl676PZuwuNjJZo5cekYebv/Ucm9kSIwZgs/i1Hph5Dy51Cd0/Vt8xgz4 +GigXiQoGzA+I65IR9fKKE808tZrj8DdH+XjPIQ5owHfCGevw4nLKSyxgPQN9c6j8aQHeKIcWFIhL +OrIMsRKxNhGzGC8Z8XOb1fKWnghKtxCN4lgzu9heAyJy6Gcva3iEgiiqeXSpNRU0KeWVRBozHZ+n +LMwv9vZhUlyGNWJ+idSFLpxHiZNLxDy9bftab47Dhp4IKhLyrzOLcWv+9bM9EaxPx52sQ8KI5jy4 +7szEUzQzNDmvc20e+PyZcnNspJ++C0SLEU4pkSuGyTGZylfQhYulINM0/wEJBmsooeCi3lV6tVsX +Y3G9qMBp1pu005paOgG787TWN2UECeX9MIdjVW+Gh8yy/Grj8+47GKJcLaTc9We0e2/1KHnDccRG +s5ZuynRhZ0+Jnq82cqfSxUbiobT1atIZjVi8OMv5HylRoJNo+S+iwJxaq3XPHUhLZ+UBdY8psgr3 +z3vWaY+b4WJOXuyRwn06Xj8mOsMchz/+pFFz8kvmuyV/A9hxuNb0UUloA3mzVUu+3ceU18P48bZt +d73axZ96RJ5+b6KsaxFfwhlmOnYdNTzFmDkdd9RHRF3+VBG03gKnE7+pL9abpyoSetpWb1Hmz06f +LgWyCSQlUdyUtAB6wSkTw3TxXsN7wPTRz0VmOi5OSVr+e3YP6z6IuPHjmr3K/toouagcXMRJ55KU +tean87mms4FNR1DO/jFONn7cau+qsrnbQ1qjd0yUXXTvIbTarFGWK9lYlbnIr1RPUzSbO48KMqbi +D0wvV3YvMx1vHzVc2m2tqpiaUnMcrpzerUPTl4C3wUzH7QNAqlUj/w1Ia/cC0uXosBtao5BPRIyT +jB+P2wFnCPTPCvrwk7hNVZSd7c6z87PdViAV25msU1/3ereY4/BISt4BMqjqzqjAKaaNuB46EY7z +YZcu6pvbSL+NzxohaQP7txDzG+nSsku+Z+lO9QHhRjYpv9jQb9hE32ooabiSbzcRp9wh7txMx23e +2zM8zGMp8TDODMsR4ViRrhXWnuSe7NN9CEg0smmpD2YaG9mU8m3kxZN17S3QZ6pwQ28xmwTov2K3 +zCY1DJgqYkevgi0rpHQ8jrjfRh7qonqO1tcLeiySLS1oMG24TcEeKvVkj66PaS/heE7RxB81LQzo +w7M95c08AWdETBueNFFR2J5GSi7z44xGwl9m2nCDEZNWo9llar3YU27acJMpUxPP64E+bDZRMVTZ +Ii/rLW7MFHmxp1zhXi5nmzH4hU8uAa4OXtyzuJI2O5V865vlSNdkc0i00feDX+lcMVX4d08km4Ot +pkpOkyurzHF43xEhdoWUMpCa6fiNd1oG6E9mgH6GBbqZjm069TTQBzMfTteHzh/uQ0/65+ZR5w1d +6UYRXpGM+eyk38km4/N05Eu9JE1wv5mHd1ON5jhcoYD9fZ6a6cd+y2CTG1LutrGeo08qJUZOI/Zv +sGGhfjhl1ZaRC0j14RsPE9a5q9SePfLPLZUDC6KM06XuhY9ur6EuUVaCI/0vWTCgxIeT0R5Rmoqz +IjZnqi4YD6F9a+7VIQni3BIcXGaxu+xUHJ5QUCZjCgNi6Tjbv9KqEYN1U+OidydNzKgW3Xonm66t +eV5+7k8qj8wGxk7jMmX3iqG3xHIzfgINlps5Qdl+txKmAbPXRXFwFIEo5VUTQ1crJkYxzXIXzY1i +wzAoak7A77vLtUZrpE5EcwlKwj4c0y/uA2dHZPuVhv2dbrExaxjFzAie89hY0XREXx7rjkgAG5Uh +ZFewwZygvCLW48cSq8soKiauuFSIPJp+Ofe4GvSCyD7NYWHkWArlThHvJo3NXYyhJkJM6CAOKiPi +ikv/Be5RhNCDnoYqnsk5llq5SLdiPw3LnICHuuPmOHxcEMUjnmgNBpVptx9PnF6PUt3V97t0BA96 +oio/N4JJmlBwAtEWIYaW4eZ0OZ72yKNWKx3DLersWY+SYaXpztSc77Ud7ClYiwmPojYCJXv6cYQx +dJSIN/JtYB1lNZvTLDw6Lopb0+okSvwwglnSrXWeuM8uaO7Kr/EOX5HuAIlNhy8a4XEMHb2SDzKG +kZ8Sp+fX4wq7Hm8xipEJjeizxN6r8He7Cld0axVmZ1bhndwqlNLVMozKLAMWxDEygXmRgescxOPP +phKeS6jF2fUYGbVxWF4JZwEl+Fyj06GKZ86unqhYbIbMCfgssU5o9Jw1ILgn5vjQ5kX5mVldriKN +E2dUy+FFdyzIAbrZyn6VDNJ/FHFgGbriGBQshXIPWp9F0Z4KBkomz7dRCtIiShHnW8npzTycAT7I +mkbtXyFAJZzomTJDLo5QtqRwVpBL5Mdxy+Ciws1mDC7t/oz/MxhrIGoxMFMTep8NYtff7imRY4/e +rDfvDFM8bvE0lLYQUxoz7oXrlpSY4/D3gjC808x4vLpEO+bagrBq/0WsGK5SlN/3PxWl7YlUoeh2 +cQyMyTunf4ly/RxHvFHQoYT8AnaIsMZjG/JgfX3wPUl/mr+W8HWhrFyTLC78jmExoa2N9k3l0eY4 +3M+E2r6nO2O4FDoH9or1sXfz2y36a0YxSJsHw25HayMxLFpVpbxUIam5vA3QteLdESaFag8tXqUN +r9w6xxF3+hLmBPxxcYkYkgwNuN0Xpf8keT8wwInaj4sFiid9LZh/Bw6IoiciX6YT8JjeE1f77P1Y +Gur53REJCPiEw/fdHz351buoUB5A6/GkJyHkPUuSXsh6nEgplMMnGaFjPLkKi22YVWGYmF6Jo4Lo +io/R7UYcjXj5GOI7cnewoZvmFcueIBFBPFKNoii+L2pJDI/S4tJ+YiJ0zmp19ySKzRjcKB/TjCxJ +LOyW240Zj9/sP9yMwR3zK/ELb9iMx6/3DwvuKRvm/OcFlkv4yhck5pT8R1dHRhUuIu3ArSK++3xV +MMfZ+74iSqPwRdFglZ44rTaH7P3JPLjeKbT+Y37eYKWNgJJapeNO9lwdkcoX/MLC9Wrd3S60m2DG +4PzFCsRfo+GLnvZ0C5VXGyswp1rNGNw9X9L9bYVhMZxeudmOIf66SGRnVJXYDKJ/cFQhrItrM49L +BwdPNmPws8W6c3w8cTWCGRR8paDMjMcDiRL81RvGckWl41+ehFDs1YL7zHhct0RpCY/DKwXLVPH6 +Jc8J2/DXgttZw6EMmvG4sLvSHIcvGZSg3SyPbWXTFdVUoJ9rTsDGRZXisNy9ELivcJk5AV8t3AuB +VxRGqUzFAQYm4fS4kH7DorgZj1+YteY4PFqwlspt5dJpFEezWRLFC6yh1MJtWf/gHzpZZ7gR5+VB +vEUqaK6fqfE/2b3dHIdtHG7G4/xuJXbPpjo4TsZ2oc1fu0uU6mCqTXVgeckN3JIxANI5VHaAubSa +hE2Jz/Qdrw2Sx7Xzcy/uX7wnA+7tPi3WkyrnDMf2xXvU/AM+CYTOYTIbjSfuXCyoPeJTkp3T2ATv +tEZiQbd8lLCfAuelGFXWdt84Wh4Dgk4p3Y/pymh95i42SXihrntHIt6IIX3ojRAF1hT/EU6LO8EM +Ea7+eQ4WhXhtSJEUntOGiRIvKjHj8aJX++ZG5WLQNe04UMn91+wvn4rj8HRvpQZ7KzShPPbhYW+Y +Q/Uhj0i3U4j0weJ9EalFE7+DfWY8/tSdQaTbaRHp8e7HLPRuYz+rOFSMDGfPmKE2H9QxT9xsTaD5 +YOLq83OTwJda0I3W7bOOU6sY6Omy1Fp+bit1S8NzcLZw5F4O/p4V2crHo7Ior6YeePkvKfk2nMUm +Rs+qkZQulc7QDPSWDhS7RCqdDZMnSagKcQ0g69KTbKIcLH3fYRNfsaqvgNWF+m5kgLplOycm6ve2 +0jOkv5AzulKCy+Dv8gId7PZ/HRYSAZt0I49UDD8g3k3eQdccjyuc6IcLPpr/8Tw6VXB2wxWb7HcO +zAxzxIX5KX7gFMFbVlU1Ht2V+E4Z69DRiu+U+XBgP+aVC2ECh0+yOt+PhS72oqCDxaK3Vih29i6e +mdFH7e6tNMdjBaLTzQ/wjX1YiSha6/MQvCjf593Sz3A9u/CcM40hOqfsEvV9KNnKEO5y1uLhZLkP +dzlBbFUwZFELp5ouOkNxb682zfF4AGtNF+7t3WqOJx6EAipFv0sZcg6TWR4FkvedYd+wiTPFV9ei +oME3y/oyTLJRnSF79vmeVlh5gK5zp3ePb65zsxeDdrMOtYIWUnHnsAy8ll6SH/v5UhWAyn09mjHa +EM1R9E/liHgcwkLTjtd6azWwaxE27Xizt5Z1/Gkux685Hldimmkn3u0thxNmSEXX2IygpToapSRN +1VpZvAk9lZNUpmaRtWQ14dzinDZs56X5Id2iISk9r1yP2cCAqOG/s3ZNnO/7ln71sWxJxByO5dJe +2JIZJ+XjiX92r2JIpusve9aZ43GJUiMQP0iYw/F0j64lqWJAlzrpbj0VSzUOU7VvEdTTeb1yVTke +N5ndxLkK4MbBZUiUlFjzQEyVOZqhz4iC4ClsQuxVCVKmC8/3SBt2PPGcuU+6PwboHCotl1tD/yIr +fZxHMYhiDwJynFTCahy0TJqQNclKVf3YKnpd7Qfp6P1yicZ9aXvYnqggmRy4mi/Lg+tnct+C029z +1ITwlCeBbenyL0pxeBlxVlxS0G8UocuQLORjsj8loZRTodcwHWJdAnIxohP9yozDxmQl6zSeNxyx +KNKmii07liHlR6LyuaEgiCWV2K+DIry6Ss7CBvtHqy3vIvuVmDH4otWyzn7cG8k0eQGetcy30sIl +UBCegSX1x86gTBfagtogVsuxUG4+w05jjMfWENvScXxnGsWrOM9l7cl3p2udERmE9qzLg6OfRYV1 +13AUpeAZz9KFPIqdlBillCrxPGUbqHG/auhg6LIXf7pmNvFob1x0h7gTidKFdIWveb3hFdnOdEdv +EZso9ylnss5lpwrYDV+DmYILzXATwqM99TikgYt4JAM/Rcs6IhCUjit16Aw0F6dyxKT6ynyTt8nm +Y/zYk1pnQvi5tZS8nqo3IVxjLSUjJcWiw3TiOw0mhEURoGG2lG5HN6SESf+cVmxCihaka9Vo69M5 +63UId3j7MCtv59g50Olr6pTrZT14OhUxIWyz/f7W9vu0V64nN8tBUi3/2btWziVb7bc/eztMJ9Hb +ilsdnZoh4q/JPdKU/tsjzc/fTReuTzbS1vxrJhija1UOi3dclZ/2Lo1Ai+A72fhxuzXOtNuZvO3t +46lcsa2Xcli5iDGuNNPxTqZ5NQyFav8uXSux/BTW0QngAOnQlL0oRuzfkXVRcarkIygl9zQ2caoU +cZKZ67hxst46t7PJug0pODr9mfHjn0mBTqPA1U6fJnWcnVOm061ErNGy81JlWVexexWU5BRmcLL/ +6vzkNurs5Hrrc3JJbmpTsKKgL4si93gbiAdT9cS91pxWavz4oy2YscX7TQjnO9HFpgvvJ+UCFCLW +OXYd3ksKN9Y5HUp52oV3k/K4DRFXOqJCL8nvINeQCeFSp4/ojA/EOezYmB/lBciIOYMlfbTSjwOW +EYoVUK6AMfBGdVlzSnjlGpd4L9kqBf5t6e4y3O8ZzoB63egEWSajZ41x8U5S2VFwW3r5XiWucYIc +HZkC7+1Ilceln1bwSp30b4jHnex1niP688O6ITMsrpk8w0zBR15h/aM99YzhYSd4LmX3v1g2jNl0 +j+dHdI9zhv9L3pAioB9ZX7T1ybjG9rYjX7T1SWnXrTlzb1+0EC5DxmPBVVDMyOwabsoP457sMGpG +WtpW92u6CkiZkgHGbcl17FQnLzviX13I5qJTcWW6njF9+MrTICI6VRy7rCKYXY8lcW4eMJfuuDbf +2Se5zkYxJlifn3wuA+v4XpBcjiBLp1tXLgz61rTjg1Qx69TXFVZvYoO8fIgr8fPQHKvUf12+k0vs +OWtcvN9b8j9b36gbbL1lcilfw062M8ZRiqaumSdcR7KWvicGxr70+nyzNxv5NBkXj+ea3RtB7kXQ +hInbektMCI8jyme1Rw8QCa3Cd8VT4qxKBjjueOJN5eu8J105jCHVWd6r63lD+Cfuw6xy/GCaXj7k +hLUdibuTjSaMR+RioL67tSs+RhTtjTi3UcEYuDcdUY0retdJz/Git4UxrEnV4iNvVEBb6TSYMK7r +rSSe0na8WkGi+vCFZchGUVknjIt/9awyIfwVUfzZ02JcPNMTMWF0bTIh/AvRzF1At0pl5TsNjvwb +wvThHz2yUYWw1Wwhrk2XiIh/43nWhPH7Hu1nl5i0PTORX5m1xBPZphXCHNIWKt9r4ZUQZ3QtZxP3 +poulESfGlxFTIsaPK3rrs85KIWK3kW3c5aPEHelidmVSRgukbBKtf7VHhduLBZa/eaImhL8gioc9 +Ldo/cKaZMJ7tsYLRi95+1hHbUu9a6KjxZ5wWObhlLpTL6F40bR8gJwS/CWNT8jETwoXObqJBBiJX +TnBZ5diOX+YRZpfwUBC4vHedGv5AKQU1pI977KbCZabBtvdhj0UA4jIjL1lgWgYxNovUqebLFgd+ +aXEghBeS9eJNlC46jK97JIKFiEtNmDEJLZhfq04u7I3o9Uqj1X+p5zkTwnoTNWG81RMXVTAJ2eZF +nn4jL4flik9uw5LifZRF1Vvys9khdmyDXVgxenhRN3yYEN53ptlbisK4TJlTtonYPyViRnwotpmr +J9PVgK7rqRQi4BUTFlBu67kjQ1meMltK5cZTqrSWMj5fIxuLrHuKXk5xKGsuUAv4qjfO1bbH4U5W +Ct6xNT++DzQ+HNSvlP0lVmILEUkRAJ+uGYkwrASfioEqGB7hfMKED1eR7kyRQWWW4ZP1YnGtLTZ4 +eKm+L8x8HzKcfg6TZykWCHN8GBaWnwO61zHGNjTV7gO4pb/KD+xaWVls7ssQ/8BQyWLLd7qSF2Tp +FL/YYCOB5Knq2ltFMV+Cjb0Pzo3oDJCXjvtDJ+uQsOOWfOPP7Nu4mF4GNCcMCo407Vjeq9igEJab +tZhbjqoo0RBRlkt/v9XmuJwk+/tUhkagUHceoz0inlA46KcvxYDp8xIXLilnSOj0VcGjVIoZ176+ +YMlnJoSvCsI8Upo/+27lku5M0d0FyyQWmIQ8vbHAkob919qa+La7XIc2G6TDnCk7RYCnVttvxL+t +GiGk/DH9cuRhAL4yISs6BAyrYpTrE5Zs0nWocDqU3/uhHjtTYrtJyI6h8MD7e8rVzn0maCXjCo62 +YbZlWFSecbYIoHaVUq9iwSqBWcr4DMDtrx/m0MzzmyzAq/Co+Dl7U0bs6XqWsfRUZePAgoj0YBIo +mqxqYBKTPJqTJ0cYaLaWqQCxuKRGd0kM0nX6Ia6pEhJm0onaW7nyfgHVt+V7+7vtTe6DLn2bM9Z2 +SvIZbS/VjSEZ6bKpSs+SsvWgKJKRiXRnVGkwMnc2s47SOf1BCgg5t6XpV5q+zejIS6hLb89315fl +526QAXykrTCDGNIgRnC1FADlXzM0glH7pYpuw8SjKyiuaDSPYkiC1vFKoWpjg6qIb1PFxDJvdIZF +dBEYcZPVXI2e4n3S3I74bX4Id9ghbJAb4Ul20V6kf0EXfcOERdqQpZT07VJauxtEvuTuFbiJq5Eq +5maGcoJQ9e+ybY7Dyx7LAFKyqdxacdg0+hniCtZxqsbiz6hM/jXOstyDwqYCs3Tx4Dj6cFjCZkfQ +Le4h++IoecC0y5qL04uRKpbUGCJGl+FUPbJOp2Op0gPoi6DvQ5k0gvhp3L49chpOl9OBHyVlaC6H +/1NJlKYCJ8cxqM+Mw7A+rs4vj+fO/Dy+svMA+lkqT13t0bqpct40FbglvVXi6LOe+0wF7tSNBnp8 +3PPtBA0ft6Yb9fy0p4N1paaCuCVdSb9eveR5ZVKN3lyXVjDVOEwc7hRmuETPXfmurXdiXoz5hZW9 +2otV/+fePsv3rTTT8bjXyhF6/Vsl2LHeuivyhbHD6WMd8XxqHZwyyR13/keFQfJ2q5F0Iv+IcdYz +KVEy1UzH5d7hXKGG30spEVCLmY413uHKRJBaLr8CibKLWnMjr747P3KbSWIDx3Mq6+RviPoIxkmb +FdWN1ELamLIAr2RAesrXPNPMOGxWMnmrM8L29HPau3/wbMF96RJQflfiwDMJE3BMFKeUExMl+99t +NX2yxrocTxelzxJnlaCthH4fqsqsGeK+tAL+BuJtEJpGnFjJmPMbL47fjYOjxLkym2fcXOWLCqzV +oWfF1HG4ytthsyh3abwYkTDjcFO60jquOlMkqsdwTt7lsf++PBj+6iisXsoMfJD6DFdac5huh9Ds +LvNu0e2u/2N253mj+HeqHJd7+wm5VJgy9NSjexUxRF80lw7tQd+TWvipjDm3e/G1ZzdWpCMUcypw +1OC8/5i1StK51YvPPbuV5dbl9gq5YWFeXCQCqVrVy7H2Ix7Iz0L+d5u/1ix83yN6ywlTNuqnrGOV +Fu/sSiHI0ChDPnRVyo6PqEzxohtz0B7BL70trMQXKQUOKY+xrGzjiTuHRBkz4/BiV4Qz2NRsxuOw +MjNO2czMeHRVEvcU6lr9ccTzXeU/oouGVowMynKo6sPC+tJayQrlgLYZl6VqD5hx2NXZqBIX7BcU +mFP1GUMP7tVFR6p0ozeIYQqfV+TgeCwfMrxZ9YhnuhoxImrG49Ihw6ksfE0q/klXOWvwq/RjuN7b +glXpSMSMR0nUjIsKuVjFyWYMXk3pKtRxVBlxduXHozBqMT8ewVFB1mlEtw9OaMrE44vKZWcbPBw7 +FhXjZU+YndBdincdydAcS++eXPQZrWH1hlS3GYdXvGE2yTh1U/I5Mw4vOGE2fciM9U8ELyTf7LPY +hH+mLMAtBM04OUgcg897IxrZh45shYDsWV97h5t2vNurbBNjZMW6z9pqkVIKR51hN9CPVL0V05e3 +mHH4qPe/tzsG/zRRpLbCG1Uf/7K5LVIizc60GkUl6vTQdRvUUbJRiVHMeDwjwV/lf2Nvy/1GB63l +DQSls8WxjiMOlZlG6/5wKo67vDKmnFWiD0eEq1aKT3qSuDe13FTgiWTcjMMKE2YNFtfK+ufai7CV +QS8mKvtVz7oM4Fead+hyYebt+7m3a+1bJCOyefwmtZV43tunXEXYT8FjpWzWuN5JFZtxuNIbNsdg +qzwkuVnoi+jhF4hnlsvBbP24LdP6M7qqS8O9xbwzif5foXnVdehaRcxRNgBXJMCZJtrhErMj9gJj +Z3lWGbkooqT3KUl544jLhLg65z+wevmhnIh/puLaV0YbZZ9lltJXK3pzqsSMwzZvizUdiuBa9Nmc +arUHKl72hllnxtCHzbluXvZm7I6MjZg0qRtXp/+jC9ztjRIXpCLVjJVil2KX16RKzBjcmSpmG7uU +vVNcwkwzzkY8j8Hv7Yep+jAps8PuwKNnMmbGY8ughgWa242LVmVcsgXhjUMaMlvu/a7yzI4pVLhk +d6XolCTI8bg4u1nxVpcCnGQq2zCwWfFmV/nvYG8DbsHrnig2pyM4N7df0RuBss3MsrBvtLDHYnmz +2PQSWXOG50k3kyRsHC6SdMP1bK6x0HMW4avePZkRXgBpCsYQ/+xZZ8bhZyZMd5uer00K8K9a1fsK +huxeUMqRGygHAR929dYLFLgAUU7Vmeb7TjbMcQzxaW9EH4lLUGY18C7tpQjSPI1RrEf28y/s5w9V +/Su6cTYoYLTqp3T5phmDZUkdETU1Ngmhn+PMOHztaNefN/ChPcOH/NPpsILUWbVO1pHY80x++vdr ++hvY1GutxTEFQMKJ1qC3kSXEgQkiseqnVJx5k2/r+LsOn1Rt45DF8crceoWor2mRgmsSzpbuYcDg +8Gy2kxPwilKLcIWViZtr5JgmM0UVkq05G/+OgcLvZXix7/bLSI1URAR+tGX+dTNtiBvUd1UN3XKY +PvQ0wkxDIsIRimYYLN9a2RrtbkpdIMGXnZhbnBPwdj6XH9Q1Xsu7ajgBWRgnVjBQxk1qxhs8EoO3 +EHMqBQKf2JkAA28IMqzzQRbTaFTxC+lNcKLVNZMnVFsvGXm5nsUAUxdgUdwOk+4twYVUYosQTP8Z +XI15xTnZZ+df8mN50I5lox0DS2voHssxVg4J6d6bw6mEVDUMHMEpSpU1T2y11RoqWZxyv327l9Sx +4/l8s1+o2Q0MLLDu26VsURzJSA1s3KmKpiylZsGRwqJxbNLGDtljvWam5bhirJmJVqkc8yLUjhfy +rV8hxse2PoUh6l5lyU/iF5vDI5DMAYCA8K/0AnTF5+cwcOmL+WYecazzh8tRygTjnihIOvt57PVI +OweKPfH/KFb9Ur615zWoq9kkw9zTyRI5VBB3OkoFmZ4tLDJ+PJrT0p+AZ7Le1gGWiqxfYPxYlld+ +n0C85/RRgnsdK2QnGPY163ia8eP3A028rrgRrjR+7LIvxV/nai6JKPWUwpdcxsBniSUREcHHPVGV +uS2ts3k8M8+4PV3SLPRTpvhUZCTlHNZVIr7NHI7fJiPK8XCEGDPmrJAVjKmdLY60W0co74uUHlGe +gSWRPMf+1zxwPhEMmUlQ0Knrjec7gzKA7h8otEqhBRuEZxvm5ban55V8G2v/y+fmgc8b7WfaZA8X +KaHefx/xpxrxEZwkXJTdDinpZfw5X23Pq/n+fmkV2zD98omxd6bcxQCrqhhqnULfMV0ijVZCVeaY +rHV/x0D136u61qZ3nTkBF0KR5K/01psTcD3kK5Xxg/vUp0AFlAbNCXhvUYnx4+aBtcT1kEvGBjF2 +PF5i1gwtCioaRmfvv0hJ0km36l+8Y07Anxc1yuyh5Dm5VdjxWn5Gj2hIhNMvherO3kpzAq6y43rc +jut2iGceT7zjKzMn4MrFjcLZ6/Yaz68gx/+FcoNOirMYtNb4oWtcx+MPQxQROJJ4v9Mi2tr9+oQg +v1zceDTmxZ3vZVf7b/nBPGkHs4G3MPSjGZyykjFu1tjd38lP4PCRTEljIbX+knhuLs07s9XH411V +v0Y5ZVKrcEhU6hq2KbiPr+hcm2qzl8LoRiXtrxrZ4hjQPSwxyoVsAZv5RilDv2hmNWPUcT2bC87M +GoRdLMxnqhrxRr7TXaYI5xTjp3Fhu717hWipRGmZXJFkRZfm/dHUduJ8OV5bsoSvUyVY4Q2PwuWp +Yvzdvlc2jHGipali+nOG1f59+ykdjjXpvTs6PIxzrM/TPh09bhuUlipAPJEswW8cnVv4VWodnvVK +czqVfh7KQK6bEW/mp/NbewSPMpX4ZkmxUEkBMzOms826505qtmHMetcm6Jnx+NY0mEp8nSlceqb4 +MsZ46v8sKDf/Sny1V0HZkf5LwfMK1OKu/yg4c6BFdV+jq21swS8zBSvUtVr8j4JTJXDZgl/8R8G9 +ulaLM8x49BU0KDS4lCEx8TuX6GaXjQUN7JTPBFfXoC2eOzR2vpOH2UuCmXVhwOhPTSPxcFIc4OVO +WA8fJDM5fMbTXrsRExWVa+Rk04i/24K6jeM6xnynEosUPJw4Sic8f8QYZx7LAA5KIF6OXjmphxlA +T/lpNgl6ncyAlsH/zjQGpASHUeD5jUuKjU3iOp7YVRCVwjrANh2rNcQ55TgkanFywVZ5QrZF8P21 +OCdiGvF+0kqxax1dUcvTFL2szf6imlPUWzH9ms8dkNMcdjgKvyrVm/s0C+JBR1lB7knmkk9dYMbj +fqcF1fk0bSPez8NsM4sKN/NLDLqdSG0143BNjzza3jLTGNJJxSgxUnJhFXGach9LHrHi1Zg19/II +Bmz0jjNMAS9SHaMDXeUzJzc36w7chREMXoYjolR02Tj0yUa1lzZGegkz0rnDi5tTe+RDyZYWaxaI +Wcv1JZqwPTnH4wGKLt+leJ7uAag+zbXUbRc4YytxWBDTG6nQYg0x9PsKbqP/8HaOJuZVYmQL0dFo +lf4BeQBYWfuinuVyMCS+NK+YcbikJ6+TqeNKuQ27ZqQPCvEcT9zEfjOSeL5b8LmJ/VfgB1HlnRNT +ZPy4fq/R3kxpYpbq/vy9Rnsv11rNrIviFszfakbiT92a9g1smSkJBVHdo+u06Fp0lViF5B1Y0r3q +UjYpZWrWK7P54/zaLVVQBddzymy20TlBbgmusrWis8QGjBxRSuzoLcEOyHZyF0fQr+CAKh0PvTbL +5cLluWSwSz/JN/vzQUWFdVxvIfSXnpKMPPYr088RXMxJ3GwzccqW4uZo/4hP85W3qvIG+jmKMWZ4 +N6c0c8AszZcqxO2DihSJeC729BazEz9Hi1I4jpthqaXfqiOsHkk4RZnKhWkZEQoXIrzpaFuwjUNZ +jPQqrs5R0OrP8iN5PAOdUqJcjJ4cl04tPwoKPP6JEPlH9PNcvGn7vxr9vFvXY2TxGfIcRlGLvagl +hrlxHBy0wbV+xOzvAANoLSEOFUF/WdGjcjFyiTMjXP39fbTt/Z/nh3S9jZAYKQM+AyyTJ1uJOR1f +dItM/YK6tilJ6y7HF2cwMF8MkE5H6SUfYAVPonIjJUWecuSv/x/51t9S61yP1zwN+HGjFXjnRsCw +rEoTiBsXbzXj8YovKveea4XFxGsUofrd4pIsocIbPmGu+tNe+3Kx9pp41/HEn30JFOjmHgXuSEDC +L9Ky/CmI+RplD8RbHlnWlFx9vwT+ostdrlMa8lc8/fh9OiKrhnK9+nH94hIO1V5qlzJsj7qUMUU5 +zLekI9SdNVYCk5TryilMKSUGhoKvfQn6JQNRASF43DMN26UATcpeNVl3ZgVkJjF+3DlQi1g/OIEx +CfwkgjEJ4jInOq5ZW6FqatXMmqqZNXh1UTFD+BMTNsWN8pBU0T3FJn3EPcnlbCIecoKlzcqEj5u7 +pQfHn9kwRR+eXyJvA2wpUKRJaVUzgQYpTLXWPtHFea04UKnZYzzNNiAnJJnbMGgZVnRLbMemQQ3E +A90R9DOIFbLufirbOC5JRvCpE+QocwwuXSxU+dSC7KLkdlyOhPSML3SXo5/RTK2vnBal+FuRbMTX +TnA0j5II90txWs2WNDbhX72txHKUCcAzszkt9Vvf9fdH+FNmevdSDtcu8UFvhFiHBDb1WDb3LVOG +uzJ3ge/t9b6FeNv0YX2P7gV8yywjNvR0M4Q3TT+Pocvb6fIyuri0J46rTH818eNVNkEW/p2KcDX+ +6BlO3wsDguXSb/LYvaVQkjkO6CcWREw77kyUSJ9EPDGoQaZ4GSVvTkhxJlwdlJC17g6C0RoGpvFl +hngfY1i0CYMTU3gjXa4URKZeSPftmmocGkWLbHvOMsbkb3NeYlPmZPj3oASaNWWndhf9OCJh04fM +i0gM5k9l7O5QW8xmup56EV0+YaZAeZqs354Zj8sLG6wp/oOFrRrc+kIRjbsZoO6865TWQymMbdRg +Vhez9J/5efcNsZJwXTbYnGuRiDuHZiiq51/5YqtUzEpxRxg/sWlha25Lf1uoLW1zV72Yz1kzHl8X +JsS9Z0W95oGWNtuWNvJKbbsVC0ty7awZonZW66x7eK92LhqSQDqeo/E7B9q5WS7mxLXJdcRrTnSk +Jq1kjz4pBrC7e7kko5Usw9pEyT5IlCAu4H34St7uWMllxNfdQqJdBWHiAgVTECvYoU2+WDk1ptLF +QdN4BKVqDoj+XF8xGq3lCqGYMlIrM1nq3Z56mR2F3mkbI6aSihXL/G1XynbsWryKuMAX5kbissXF +bNVZ9GKB+B/R+LvoZzMuRQO+6C0xlVil/HZ8UwKfGY+lQxqIj3rjxJuyl9OFT6yZSxF4YhnXasgb +ElsZwpuDonRWeyXadMqdJeuv1uytyaggx+Oa/Ww0XchOVDoUeWgvtAxMiD9igGeJysyPd7FGHAhD +1k7lXOS1vsJ5CWmnk2/x95kWAzLmZJK6Kt/VqHEVVfYgPZjSmIXonGxH9UPxq3kNj8fkm7l6f23E +9Tg8SjRFahg4SbznbYtEmP482LLaWxcVa6mF7H8eHOUk6pagdoaEmcs6s/koxhNX7h+VBKjDMJAV +nVyupZ/rpE3pzB8+uGL/hGnEJqUtyya6G49XBrdodYzswxK7KjC1XCg1tgwnFePRvHnlJV+Cd41g +aA59GCuh+Ka9Gr7RNrxmn4Y/HNxCFz+MLyAO2pKR17OeICOYh8I9BwgK/TWa0ofzNfeLDxhuRuLF ++drxv1QeDnYpXn7BwCw2HSD/e3vmsJMnWO4IyYGlGmj9I7VuKol/zF8nQK06wMZJZ0VJ3fpBZXOW +TB5iyvjx2/nqpr3YjMebB+jqhrfmFzPGq8WrGT8GN5gcJ/3XA/qUxzzEXcaPewaqvWqrfbBXtfG4 +7ADd+bDbFrJcAN46oC/juDe/NhflO8KXB8r2A4sKO/uV10xmvAB9kq8C6KycQkU4dsqTSQmAMCSM +hB5SrAP7W6tFp40fb87Lz+LOA/vQ2s2Ab3R8PPH9Btl1XSozLIrD6FDllazDQf2Et28KGutzRUd0 +oEmuiiLJKJmGlpIKxjDsFaKx1Q7p6A5MV3WZ13FkGKepADGqn4GTqOjFGGXxPNr4cd5eM3/qwD66 +iMep3I5VynJ16V5fd9ivXXGKPG7c68Nv9MEq+wLEoQmhE/HTCH1/H54j9tVD8gD86kBL7F0uyCnZ +lg58vOCgTPKbzfPWmfH464FCt4vmCd2+PFCERpxijRzgY5T8PYN+i4F/tkDNLN8/D+yz2u8AQ5yk +WaJ7wIi/X34Y16gn4Q7MMsusLpyoxCuSvvZq7GcHCZ1XWg208WOF/dReLHxdp0+zThNARAdTeRO5 +Z/9sJ+HMhQ9K4X3gMkt5/D7Z7yYwxrrJSiakKyOkxpaCQ88zLDMupbdz+C7pkeQ01Kn9kw03X3pA +vu2XpPDleuj+rJ7I6GGmXb5kzzJmwpzYQpczsCdZohDn5c5wHk0/a/CNXJ6Jlc5uK3rEKFnJp2z8 +LnFeqp6dcqv7t3dZKV0Ud4gNEW2P1evfbUrIY/09j5estV6k1NWVWLovJ3PX0tdJYR1W6p7Ok2rw +VfIxrHSiVrN+g42Uc1FXjm2pCJ7yRjE2SpnTm7gNmSsnxDtzNRqLc9PdeVB+utKvbuaXbB6lMxD3 +Wz88ZyTud3YzgPuStbjTaSH2C2JoGUN3WZ+HEG16qxAzroV4xDNNd5xdn15F5LwKidc8iiAK0PUt +FgMS8qFtO3HwMvpFhWROQ8EWAeDSdKPqNWZ6mi+pGE8nle0l6yL7rGcafiUufaDx5zKNSyMyWZkx +zvGh9BXiT8nyCuK2jK7sT0nlyGrGnU6QSmoc4yLi/mSx1bnscG7XmU4s2erDfcniZp8u45U9eQwD +WNzNTOYD58RdUin7ukYyhPndokWsPsOHU4rxy3QjnvWs1Vgb2YQXvcs0i03WO5X4xKPF9AtPDpmG +c1ZlZvmBZy07pfNFcvlj2CSAfe1ZyyYez9A2PJMsbyZudmR/xdakGJQshL/ylOn+SHVQn+3gawuB +JnWwIt3I1XjEMzzv2vr9/PrukkKV+H4/MXuPRIqVpVlvax9s2Aguho2wcabhC7kgEi96O5jxS2YC +3RF8JBPuttTybMe3eju0mE+ntGpeaVFWaxl86C3HN73FmRFrpNttBWzxdjRzDEP4i95stTWa5JbC +BVo5rSEeSq3Dr71rM7d9uaqcm+Zt6o1Jung8FSFu9649ne7RV0KFU6LKpfi0V26ett864rfeZfjQ +Wtpzb7al1tmREL9TYz48qmiw33vFLTSxegMeSUXoa8M13uG5DVI9LA/A9UbM1OhhNrO8vUF6UDSz +nxM2OTmV6xpLulnHGaUSHXrj1n7gZA9+z6H5prabosIHiWu8u6E7Hp9PNWJU2Ryds4wpI4kU8PjO +K6VsgvMKkVRs9Y+UyYM3MGZjLI9hyHex0mKLzdjQE8Fq8yh9cOTTg3d7auVEf4VRnsk1DGFB48xm +a/z5Tosg/mjPOp0kLyqIB/c703BfUkuYB/VN9ip/t4ahk4jCFry8pFLVNhdskZVUP19aUkncUBC2 +Xn9NYiGa6MvYRrH4Ofuwwavef6yBvCv/L1zhDeNduUf80ptgHV5IRehs8loq/Ial8DfZ7Pv4c6oe +13l14V13N2167Wo82xshXnbWUvfz/7GnXBdsPmsF0TrLt40nUiXNxFEtxMxGez8fnuspF2W62TSw +yYfneupxs9lNvOVVgOE8jas/VW7ru3jNu4W6CtJPXJ+K4DVxR9iU6sZz3iBuFpYYOen+0VuGjakI +vhvF3BJ8twNPeb+1CIyLRbsP7sCL3jKGcJFAeYj0uwXvgC2obdS/Sxrxak8jNpkWMEgM7ZtMeMP6 +ou9b9vnegsOjSDfK20BWzgCvIoYFV4ItZ2Fjzya8YVowv1FXdGS8brQNAsTdXhlE5rETT6XKsc2b +IJ5KrcLddj7d3Tggins0mx3eVzTX7alam0Y+hHtS9XSutavhTFNI8xN0qQssm+jM2oUlSu+R3s5m +3NwjBSieMy3EzT0R4lmzhU34VU+ljrgz1mXc4HcbK6cMka3Gudn7L/mGsokrhTu3WBP6j9mJv6TK +sUVj/EtqFe7Ya4wPpGqPmknc5ZWOBg+nIrg3gxsPpOonKSEz4UYwt5uv45fp4n2FjNH5XbZbG1Y5 +H42Ln/XqauAwsQcJ+jXUj3oqFeRhD+Q96BhlXHwgjFGhtTZ/kYulvSWU4m6/MAN6f74JZip/01OZ +fyMTPxbHp1CirZgNgc3lnQJghRWD0L2vp3l/WX6MChHs3DAa25fUsxM7Ct7B99YSv1kipzrcUyCl +xIXC4l0egRIXpBvxjUcodn66nvjjkoilbRki/ceCKG7URhnYyS/ZnTybIU7DbUt0Jd3jBWHexRCn +llk3gIzgKqYppqyTlwsx0+VRqb7osolRgeqmJcszcHq+oL9UMixO3ST0GSSa7xyrLaxb73DPkjvo +4sGCBK33uwT33ghexFomsTBPB5uPys/+VYqkCvRXaP4C8bsFidKfMsBtxsVlS+TtEiY+LLi99FSG +8J1pxMJNpfiHxHqC1ra63OYFwPlLpEiENmDMuLgxVWvCUmsxgTcy51buJNkky4uZSLzfEzdhrDNi +5rB0yfYMLHXorMqeExtzRT/sqVe4hGlBTwSrluTPmRixJhXH0HC2wloTxrIlCtudSLznbcBbSbn2 +hVGRMBNRX6nZPEB5D8dqVOSebg3hcmctfmnW4oWeO3CtYm5eVQCvBlKSbfc6EzYTFXViwlhjonTx +as86XGukEAlo8JYvVIBHHXFjJjGH6LsNlGnNNrLZovVE4u2eVhPGVbpz/OUee+SdbwaOvIrc+hTi +oUFFXG99urGwVedx4aMZoUvidoohFDTU2ACdA8NEWzlhGa6pSjd0oFKUTmU1kRAdPpmhI4mFQsFC +rWK9HP6xoFKN7v+KPHIyViDr/edrwDRpx/Ckp0/OCxJFjlcOv+PxR3td7VlxdubU9f1HZ8cbwf3W +w2c9z1KOzDZiUDjb+zbFfZxs2nGZ08CQrkG6O6WQ5801JoI70uVyWdDbxzxBEyHuTpdIWSa3Difw +lcwUkkCW5N2cRhyb7/I1j2xfX1nb0Gm6mg9DnhWV8+2WPWUEBu+m/2PKpbyTIaZtEjdXZ6X2jk9O +/+ehobhqBm2I3seWNkKxmxrH2pQ2QDuxIfdmtVfKVElpOPW5GmJ01DkC8/bgO9Hmk+QKwpWsc8Zg +7h4qHU5at60ly7WHT4toP942EAAXISqD+jK1xERQOc24+HGriSCQ0K/IQKCXiaAqSL/IgnHxo+36 +sioTAuaqlXEt9At29fV6OrwF8yPGxZyIiUhWxfdaJhHdW09HrBslCeLkbhHGo0UY3R9SMYdDM4rI +EcflgfpP69a1oZprsgTKdxw3083l4O/PlyzEJx5h6AmHM1YlQH3ile9tFoCCn15uSO8xEbztCevh +qnSxxvmOJ3ywVVkdwjtzmDTi+OwIJmCECBNDVrtVOu74rGrpQx7HRQzxJWFtjxDokhx3t3Og7ljV +NRNxl2KpJsm8e4xud1TctVXCSeqReTFgJihWsN5MxHbrR4QhHQzp5a+TrQxpy+g6xBBSe2qm1Cw6 +o7rCGirloUKk9tDPi3O9j6jKj/wx7YENOEX6CwSHc4TSrivYxYQVEdVJ8B3GzAQdn0iWwEmIJo0M +o6dcXc9aTvl72mPw0GmiLVYKoGWeAxJPVWplSuffRMrdNmAm4OB+MxHnlhPrnWf1/WepEhPGIUEF +vE0gzlQoUpg4JGFJlHo/PyX6rAuXkDuuVPKiVCVDr9aYML7yRs0EnJ9SROOgqOzmRi+aIiJHwtvv +aidPIObW2jNJ19WtVCcfeEVqP0iuMmFc7tzH1WYC9iSL2clAXltyQh5cH1twjTZhzJd0PgEHvMO1 +DOHQtWiJEIVr2WmT0MA7bZgJI6Z2dKFwQEdNVFNRtPR2dfyTYuYOAo3roZSI9RlZQH3jnYYtScU7 +TSD+7Xm2giFh4xNe0fVbkvVmAr71hBUkqD3sJxZsMmGks1JULjhUwMqIX2rmV9YfN5xJ82cm6Prz +uxjSsryX7S2gPp5xdpsJuMYOp03u0RPwffV6WfoOtfKRp6OCrmqtTq/Si59ZGuPydUWO3qmwaNcq +X/HjTfSbibgmvUnF3vdYTZT6f9obNRNxvwX5fU7CTMCW1ComsSQfEjMikoX4xEIcUiTbrq9UfI3Y +ZUrzujirkdamU56dEH1VSNromwBh+jAocSbrOFofptpwt8QqPpjbudUTcs0TxxSVspR+nqkw4VyB +5nyBQoT/X/2LN8r2P2Ov/rfY/gMUIbdjqMmNIW82XjpxYAx/9vy3QfTnS0COq5mYNO1zv+1WYJhj +8zhHa+T56EOqFt7otkWRo6k71uMMIVmvI9Y5/FtBJS0KpCAvm8mz4mh72VopQ1fV3LJoIm+UHr58 +lzzgrHl7QOnjDgx0l6focusxI4gp2VJp1UjT5600E4FwBUM8gWDQRgnOlUvhRC6zD+liIcwQ2Wql +Syg7hqGxJwgV/lUg69dE4mdLlpsw/l1QNnB4m4lYKt6IsNsRg4azyXYMORfo4jsTlM4fwxMUpk6U +2wqtj40JSnkoGciE8UhBGWPq417bx8P/0ced6gOtxWNOPrKUysY8m+6PZTkNyIPWmi0yOFehCSIR +PwadMuZbe7QysWgMObeEHZNywMKrNjuRsHZwmUXehDhOGYNsVg6j7VCqRdRltyInfssmXezkdIuT +800td4ooZUaT9d6qYuAMThFajWLochuXp5u38JonarOyXpeux988ckCwJwSx2rp3Ksrzg5SC3iYS +V3gfle1a4aQaB173hP+XU4e4WtUdX+bYXTolP6otdlQPMsCYPQT8cniwjmwyFIxiwPR5fXgiVWsm +4rfeII+yb4g/5QZxh3fZsOy7x1Jyi5mIm7xBqZeOkeWpiyHOYsgO/P5cne0DdbZn69zuDaK7nPhe +cJJYRnSXozhodZciBVImKpXTOAZyV3PviObncJXcRLkev5ea8DFPAr9Nl+CP9n7DO2vkcOzPcRHV +tflK12YqcZJ4mxK6eMJWCNXEcyr0/lzhamzwFhV2EoP7xdpGxltrWsbPZOoJdLlCOjB5IFobDTKK +V+lj8qdP8w+zHVfjjgxKuRwlQv2XVKWpJrZ6W2QfaTox1/mOgQr3ZSqEuFEVHs9UuD1Twc1XGHFi +vodH7GjX4/AW6/qIrPfFnFV267libOlrZhO+0yBXK29UOdwXNspt63sdcrfwM6SNooCmAH+sFRis +FLjoLRfzOoOBU4W/GNxgWW0MSti/gbt+NqmGdYrWFE5Ov2MYQyhPaNC3piKmGs9YTncFXQwOK3NM +waOYGUFRR3WNsqlh9nJiaFD8RwAnRaR4x3H3ZXjk61L1gtIr3nBvjb1yLANpxihrzFS6cgmz5hdt +UmGKvo2USeZUzIvTj1PiKH+F+h1jU61Tm90IU3NAI/7lLdIWPIGHW+dKpQINUbGoP6CbO1J25Mtj +tZPJBX51ap2pxpve4WYksTwlR4JqLHWiEiaaZxg/nk/JOHV7ulhwN9W6cQC9+Ww+1T/KjQDXZlvs +sy1+kWnx82SmxYsz7no1ytv9h1yLphoXOn1I5lvrH2htq0wZSssqSHRxls7WP6aKcbs3iselnLMb +XTFIuM0bttrHZDndMJG2cqWZhu68k+7Sk/OjfNKxeswhu5Fo5Ol0+QZ1o7VrLVivWsfidg3sHkdW +Ht0VV6cACz99K2vYZEut3Wv8250+CU1jvqXLXxALtzK5r3Knelq+69cEoEy65Lyv+dKBz29pxlw/ +2fiJR5JCuN86snVtS9abajxtl0Qb1K9F9c3QnpXVbXluNFLymmo84dj1yWpwq+vz3a+QNr2/PihB +AGFicTm8ZUh1wxc17TbbjLShMqJlxNqXhLIrnDCBYIYr+0TXo5hqXOo0iM+QbSVZyRCfQk883+GP +8x2uh0D9JQpuV1jnpCcZMgniBbwjZWsTXbW/tbdY+WBXM0pd8nrhwAJcooTjdIZ9pdJyFvGJXXbp +8mq63MZOG6Cw3VaQla8aq9EnsY2dyi/xAt38kKbnh/SWtDeE03+GoPxwd9xU47e0UO62W/RpRq1j +wQw2cQ5d59x/yQpEZ6JC9oB3mulqEqvTysixQjP4VFmCl5QoCZJTKjRQ4iRXFMFaqL+wnpCZAb7C +PsyN58/WGflhfaYcWlzPlSZhb730Uyv5d090fs5RZcdA4X/ZOVyjCfzKTuApO4F1mQm8n5nADBt7 +4kzctYh1lvcSesqJIDOqt/Ya1YfswyK54YRyssaIn+ZH9vNB1inSjkzxtZmRfbvXyJoHCq+S66Gu +KekWTfmbHdaKzLC+yQzLnjIfKZ7iQ41E/xs/XsiNJovC3zC61+YdcVp2NC6GFRXWXc1Ofj1fxOiH +itvIHTj9A6WOkzCLHR6lEsL2dD12SP1of9L+xvZ0pZjHNYqXDhG1y7M/TqzF8WU4sZxlLM9khZkU +IcLKDlOOsWWosznf5J4Yo1tLeWxl99nShvwgH5ImYAOOtd2fXI9jLRdUX166UEoWywBfksOBEY35 +av06Jrl+CifMbGZAugMbXCbtbIwhdjUjGZfa2M3FhezM1y3EVm+RHMBDsvfpxsEAH9Kxd9K7Ep5/ +l47jgQwvNj5InFRCnBCEol6xoBL7B4nF4gpxQD9DHGrVFwHiV+lVqvysJyEvnEUMxCV+529q2nl6 +fuQv6rzWRfNyS6SYS7H68jXz4dq0VeL8Lr0pN4Q3PS1q8EbbTx2fov/nkTMY+34Okp4z8w1/roZ1 +tfeeVFzrdFu6NqPECRAXeoM20x0OL8OMOPakWvVOTjF+McXOMJyX3oN/e6axFEOiRJfcM3xYmt4k +oZV+DArqBKjgiUcvOJZ1ktF+wZBTvwtvpyLEVd4EQz78PaWcwGvRG6k4nCgIjmII76a24wrvWpm6 +4csAD0OCrMObqUZs8K6V3TsbqFzYodODAWfGV39Bc4T4/hYx24q1+JghZ/guvsCQM323riQP4InU +u2LmMooq3GWjKGZwHtKNZz7zb8xvbIxU/3FdNWU9cem4Vh2FRbXyuh2aQ4rmWXno3eEUMWCzreP+ +1Fat5W35BFjEw94gcbuUtY/LcMVS/EYR3cST3mCNBjmuSv8iUUl/jmfdOdD4Izqs8msue4JLZwzS +ugSzuRmCHnFzptvfpe/AAx4lQ8BL6rW7FoyCrxBPJSO40RmOjalGnUo7vVatvTlV/p+jfUP1rk6V +SC8OGxIWJ67V6G/L53Yi3vYGWXqS3Fh9P7VmW86u4vzTlW5jxhgh0E25lrdaPMKrFsZjWDUGg6MC +ZSd68kr/EWfnQflmZrahBVwsL9gqhnyHsknBlVYsCymFzTy6yrvRyaE5Muo5J9vAZEsTZNdZSD/l +Kimv2Sr62QGfclHspxPodmWj6omYyQjpahodxtbJ39Htdw3FxsZUTMb9sp9axkfR7ZMq5F/qS9Gt +HyUBfA47xSOAYa7QRTziXi9PS4MymfjQE5Y+boYZiU3pYr0pa8HxYZ0kc9TvuSVyJxV5LLYdhREI +4sQS4vgGbsMUnRH5yfWfW5Nx2ZuMFdqm11ThBu9wnbwB4q1UpfRB3rIqBvALbwfxTaqcgUOsmsIv ++VX/fieh5Br4OFWJvmxVHBjG26lynqQklzjPG5TPoKx/d3hlNsDbqVUoLEO6Uo4YQU3ulZTG+ktv +uIJIlGDQtBRHU6Kx5C5LeQcH6WLuq0RRlH4egwc9SrJUwZiK3KscJvpxezq+dzaInW356b2u6W3A +055wdnZ3qXueYiVxb5lpR3+q0kzGTo1BXvsBy0jVYa6E/0yfxZkuUdVHe7XA03YM2GZbqkJCtjMX +9lpml7jfE5V1AXfbwTlH2Z0+oMlvzw/tF478pszhuCxVYvW0Ir//GWWodX7bG1QQ6HqtQub50UxZ +pQagrzKHsjsG2n4qg/OuIqc5WioZ4RNxpgxmh4eZtYCnlUQjfx/l0jnZsU3Bz1CUEzv6828LcX1B +0QYGyjiRAW6QY9phSMYXi3qU22k6zIg2zXPzLf1CDI+UGPJcTjH0QwUa0S+0Xb0km4JtipxnRT3q ++CJDUg/5tFqSbn3bZH6VM1bWy7i5I9/01r2b7hKJUHyaWr5rSa3J+A5OwTpFA/XE1ZhLn/yv3L2a +GxHPNhfFH+3RzxCPZmyUblJkjCcwxAvFr9RhQW2eW8nXKcS13iJlEA5Zq3mMP63RlUgM4dZ0K/GM +p1/ENMaPreyHO9Mlyjven9P1NM/Ld/6M8PRqq8EREx+rncxomjEUtUgPdsOZdG2uyFEpHKCL+h5i +7AilnEBHhAHdf6eEXjh0CzGvfrQZjwv2t2aJ1GdmDP6xoBgFMijLzR+dmWKL6ieZMcP3TuYdJXb2 +KkzoPm/ZPq7NLfr0eu8d8u+FInaj2Nlrk9P34xWGTBR/691KZYU/xozHSidsovgqWSmPs3O249C1 +nCgBYBcxqxI/SHAKY6guwSENOKKBT2UUGPOQXmXGYy06TBRbelexc0CR25mFUS1kXpfI7ZttzRku +fQfRf5IlySLlCn74j7+XcYVoVXKrNmZ6eQ7u1V3ZNn+ILzKLXsdxxo/fpbOE+od4w9MnJYvktHbj +R94h9of4m0dC41DpmgdlsH3HQHMf22PbJiK+Ld0qYvK4Zwt+m5ZHlW5MuqRC6zJD0SxKVvtD4lPZ +F+23vv/x7R+eoO51urBiuEo++OPyGQzhRiHRW7aW7yCeQFydblWpNRVlKvVRfeNeTw/+uLxq3zov +qspV6bgKXZ6p8kH91hwHsmNhFjIn4lAx4owJ2n6WcgV30c9T2MTLagT1FrZx9kwbUVBnQ+lEZ0J2 +G3ZZO5d0ruO0SHfJO9J6S+YfXYWPZx5fFDeuDVmYBebi/Aget6YS+r+ijo2F8RrThRPFCT5B15xI +hO5jwEbyjBI5NF2EG9f7HR7ZLDMkjlhYywHlaiLf+NueosIHRbSU9hu+Zegu4eFK2SxH47oSwtei +qMhORdRhUFBOm61ykDkwehxvkFewH/vfTr8m/wNBRmyx3Bqk6fXzttkK/rdKzUDG8pDiCHuySGcR +QHMt4Q+OVpyQQKz8NZ2tOKcR5X1y51Uq36tOliVOJu5pFcpGUyPoBSit1TmNOKZPWs7JdNG8Ndvu +61ZwjQm4cspIKF2dXENidDFnH0Lv6c2C4aRClBTBq2tOpLvySV5Zs2gqpW7/PqqC5iTUxs33KYNs +gMfXMGZOIn4kjxzfZrpck1u1nXu1eGyRWEspX/wsFf2rU7pU93KmcqX7k7n+cUJGQn1IZ0hg13x7 +imevOatO5UvJzu7zZ2IFu9hESeqKa9Da7aCuaK6bwTKO4rytcovlURJ1q9kpL5AsXlWn8639XLQW +vn4JAQGioGM2vvLKkc1XR9dMxx88azXN+5KteiTud4KZ53o6/l34MlNW8WboS+lGcDhhMx19/kQG +Ph+dvlzTCSu8VI7X7kRtA8zP0zTPefmxSL/64C4eo2BYfJkqYUBdXeBtKBtl2vFQqtychHu8wmeR +N8hBMgvaMvr5pVJprFFobu9W4nsq9iR1QY+wLKQql/JHpbfp6G2qwRlbMeR23cYhS5XEGGfoLlFp +p4GnSzurEBO3hujYyphG+4PMjmxenh/tN4KcNDjwdci6oXw0Ll9ggDcwVC3PDLG1UoFCdtVRzhFQ +BtRkOXT4YNGrvUwenBOfl/4s3+4lGXalTdcn0SndJQIjtfuDAxOxpP0EqYBX0s+l8gVXPJ303Qq9 +jNM/n7GDc6u94+f5tm8UmyUIaEvKNoTBZaYO7yrUWLC+0kmYOuKdZLGernLu09PbmaernQ50tObo +fPX5+Ubv14C5kddZ62nAV8UXahYyRtECd0GViE+IddohdHVjybzp7EQqbxHbsSLf0isaHq+2gTCi +A/QdNC/HaFSvzBd7W8U2yOPLd5DIK5LFuWH150sVyrCwQdrj2aYO3/YWm+m4DR0ZrJzRaqajXBbk +k4iftpYypiNglCxbJxEN+joy+3B6XCRVZ5+/Rp053iwmXJAbTyHuRBGv5vHqTGoOFZ66UJpYXQ9U +k1uHnXvVeAHynJDg5MN+3zLEG61uxOXaXOnmC3Pt40tkCIPaFVVyRuyaLwXK0Bx18FyULXuKPam4 +kfNp09ffLBw0pxDPJT+TaBuQwskV0ycPM+sC4jpZF5AdA40EMv1NznpevDwf6VonG7JW3Zfv61nx +C7yaC+jyLhH/oaJF7sFOVpW0c6Do8/9H0aWr8q3KOvv/anXExfmiO/+PVncMFH3r/yjafEm+1ff+ +j6KeS/NFP/4/ivYPFP38/yhavTrf6tf/R9GdA0Uzzjr/+xIsXZNv9U6p5XQaiQf59uDcjtk5UGJH +pkTGS983NF+k+rJ8I69rr29gW5qlIpwno0P+ZfhuGPOs5u2gPoZ0wEov71fANbpKzCRsSyqNytRM +aOZrThlDZhJxbbKWMXMKfXjV+dS0u4zp7fW69dy+3el8aybhxmSJLYUXnWfNJGxJlph2POYoI8op +hO8dZT+uUpKcFmJRfEpEZjYdOXViaFy1eHUy4juKPr85Bc86+Wyd/ZfnZ7VGG2yDxBXxzL4uxgdu +hx2xNlesEP0o2iB+W/o7iTmPe8IycTKE9zxh4px1lsmRbthvTvfh81StOQWrvFFKp667hvG9tVhc +SWXpkepYVNgpkQunlNsj2eYcvdvGbkv1WEUXiXj1XorKdbmRZJPQP0hskjWZ+JunAdelS4i/esKa +8dO98uCr+B/wrmSdhewt6DftuC0tyE6iBXnmw81ImEl4vrdEcRp7w/klj3JO+x5S8Wd64+/zBXMK +fgWl98yToeor8wO8SSYarudDZiT+3lts/Cjqo2tOIf7pSJHgHP2VVTqdefg4BSCJtD12k3Psnin0 +vWT8uDdvDDkln4//1oNzpHHnVbl+CvEEiojKfsuIueI7dT2fy1L7N8STlY9TS3WeN4zz0hZHv7VB +ISczZE7Hval6DepBqzn03S727Fa6+O5a+GSz3TS1alxNswID/LMaWcfTbLYy17plnxrPW+V2bsgN +CK9nJn4BfGXUJdV++i5jiB/KIMA268IiMi6FwSgqYmqtrLlEbwRDotXWdLS9tzijIjsFL6NP7NIM +XWkgtlBmsn/az9KxxRgyp+DPujo0Ee/pYWceQs0b8wO60FhTQ0xfB3ls/q3+ga/K/V939T5fd26s +8Xg8Ho85pRDXmqIN/x95fx4eZXX+AeMz83n4MERxTVQWSWAGSJQZNFFn0IxmBphUE4HUTNAESICg +JiwBEmBCkkmEVlCDCipUQQ0uaMFdtG5F3Oq+17pUW6TWpWqrteJW5Xd9zjPzJGi//X5/1/Ve1/vH +iyaZec55zrnPOfe5z72fcQxwso6MJgO5PPVDxP5TxH6MtB2pXmjLtSbjLqtYe/6xVBHxkHsYx+OR +VIE1Fxd35GiOt1vFM8dzkXh8H2tHYHYRcWQhA3VH2zm1D4Jb+1a8t/bHOsxsElulxRylSXgghram +JtvTI+ooV3qcgd7WLzur2VvFRJLw5MueZ3xJzo2xxSSpC3mGfqPERcpr0czoBAVfTmTIk/uVnkIX +NWhBljjqqZ2bnbYfUdvcSB3oJpnzOkb/bvSmAm0Po+zQqsr+bbL+taC9SbbltJ5m53VOS++qpasZ +YFU4PE4G1xNw0rf4rXIJM5A5iruud+p/aN9MyztZ5q2kF+OSmOhYmV03OPX+pnaV8JABKQFknZF3 +4UJ8I9Wj5n9VqshaSHztzueJ0jduSprv7/VfwajfmourknOsydjdL65ae91xbVcvLkptU7D++uUR +Pf6wvyysG5I51kL8rX+xNRk/dEgDnM/QSKMSjRIDk+K+UWerLv0S1yZpUScxSmUaP49ozCEiC6y5 +eKs1V6BdwkLikMLwYIZGsYEhnFujfB2r9e4GBcB1lgJx5MaREyfOejqWRiLjHyGW3eaI7adRsy5L +jfApATRheCbZRcR/SgbxGb1aiGfYZZheSoxYEKcPByapQMwETybOEAmNSnyU0DjfifJ13eRM+u+k +dLv6RGH9/FyOPopBay7eaM2xJmMLi8dKFJcHhjwo5HKNA5VHG/NyOZon91a9mYUMmHFzk0Zsqh60 +nmjKYStH82dhp1VVtaeID6SrYsAULNX9M8UMYUGBXEkWFBGHSibxMarwCSwtINyFOKSQ8qsMTCLm +FJmrPQ9fT5ybwzXmyxHriXNyKFF5tiR1ZQYOIFVqlJQhZUqQ5UMDajATGMBixcLpnixiN+LY3K6j +Ob3df81iDE3CisfkzuKt5nhtMeXCD9QxKBJybXsEu1E8m1HebFaiVvtyKrew1ovqHLnD5MYpoRn3 +dzb9ZBqWGjA0DRVOGQNCp5tlUJj0H4qtyfYM9pZhZLU1Fze3FlmT8WspR4Pva17HiPIoUXsIC1Yq +t0S/blRHMLgQ3ilY0CQJTrcjBE5HZQEOrSbOLVBcK8ZspzxRHzYZrQNEYD3x8xx+YOb3pB02hYsR +iQhGx1ESQaQe1RF576fXODOAJEOTuJEhvhFmqII/TxdnBnCNGZxJ8l6EQplCS39UY9IpHKdFUx74 +IhNimODJ0zPY7XxQrl7/GAb8xPRcaRCQOwXTcukzKIC8KajRY/iLsbTARDy5pxDTI/8R1aVd66jY +dxMoE5vkCNEB0eRetN9CUzhU+qowQ0UZ2E7Z90OD9uIR2pWSW+ZEMqXOEP4vHxhF/2EZsaTudmcD +39Q/O6uZq7iMgVyFEHlj0hMwzKhXuV0NCsBd7zccAG16MxVL52SaKrnDaeq+/opF+Ia3hLXwnu26 +IruUGBCXrdyaomAVzxR59ygOS7Y3/Ww01CpqEnmex1rexSirTBg7FLiqvdYyzt4beWgrMsq3gAge +UuJq8u0iLC6lrN8BBYaE+AaxsIY4SPZFtdLAKGaXEkMLGUCunPIK5Gc2NGl8JcVq+um7wWggQ8Th +yhCk67EH5hOLIlLLTUBXm0lw0yXVueztC+cxmo7v1qREWU25aOlH+lGRXD0PVaaRkNMQjeA8ty6F +K40Yz8taBNbj5xHD/QHi4oEVlIIf2fkMMZ/oX4zlBUcTR+m70tw0GpuAdDfL7dNFE6S0/iHKThii +crraGbQu6nwa/3IVKsCvtYDDGYL3KSyLYE1nLv7pKuRFBligHowrvLZfHLMicreGR/PGWpNxJsXQ +DzyGo1lVtdocAynjBDBZfKSURumnC3RUNBu3xyi9QdayVuxASq8YZa+cfT7pfRylnEY8I8Q4mHZ/ +Wkm5/6IK/VPdWprgXvtZqnQf2b/rNw4K/uDN7vWQ6+l9fsmAbDFxn4iyjafPA5sN3OXUyMI1A6SP +mCD8XMsor1BWiiVyz6rKMN1d9zn9bB4g/cBjlDDpfZyLOCjDWbrudyo9NCA7q/nqqjMZyJ1I4LNw +2KRNkqZwncyEIXrm79EEod3huEoeyLyehT8MyL5mkqNUgHe91j4DzE6nIt42wGykNyjLijjHkyZw +FdfJ7ybN75Y82Nvs5wOyrxGp9pRjv28Z5WhrFl5sF2G9GzJyCT45ZTtduR7KvIyvNSR1JU87ealE +2Sk3fe3iKnkeRRnNqGR29r52XZaxHkTrzF7zwi2hSS73LWJSpDFSmoMA4ZaVd5QEw0+4ildoFGIt +ljXZj3yKdNRH4dUgSatyOwqNF6qhwTb6ptXFdTscoF9T78SBPYsViMIyaxbx5JIa+6C5P6vemoWn +l8iPeTLxm6wdDIyQR7X3OkqBKjX6eEY9v3TD/S2suBLF3L+k6XSGvueJZzJwlP1W4VgV/G6JYeYe +yFLmQa4g2msY4GgGpIYVGke5ihOtGty7JGJNxpNZ8VMpH8coWzSqVdp6fUtZJvNflZo2bxDPZcUt +H75zkhdNxh+yJCDt0FWB5ulcreRfs7otH+7WA2JuDjusGuyvq+InE29mFepW4LRXeN0jzjxtUdYk +iWW6HVNJe9BpWN41y2usyfinPGaiXGItxOrlTdZkfKUHPsrei/nGCaDD8YfLezTTahZe2C97k7UQ +e1o15AtZOIohXkJfBpHrMlUr8BfbwhOSJvJCdlsV+LK1lAGEp6D0YXO/RZRek3AAzYZAL5iDY4tV +u6tf3KrA+ctlDoJvvejgEWLVJjFBT0hYLvoD93apSmYZwqfI7Si9x+rmLrmhJIwvS7WsNlYL7k7l +MGpVEM+bmxqXWC2KfckR6rzoURRuixd3dBTo+5seWVJN5W9SEeWNwAFJtFcQrbkYmDQHSRID4hNQ +FsHBcQyUdeDifoVq/W/LIwL/CgP+3uURp84CpWXEjIhG0mhGZVjPyfo+z8RWHa5lj8okgT/2K7Yq +fvz6ffJA9+bjkO3mSAlYlXi+3/s/rofFTVgwhzhWFuEWhZ/n2kN5272DU/VksxSlgvVP7s/8XHVp +TNtBBdekHrar7nbv0Nl+X2cN5cwSIAas0FZNmCjdtns0wt3WMGKVWxiNb1JFWpZpON+tlcG3qSYm +rAp6pmNT23eGIM7VcRKVzjUtP+56Mo1QFfi3WxzLRvEpPuyfj2QRY1YlPraGWRW4sq2UZcYOFxxp +DANRc6onxmNRE9xJgfIOfgJK7X8AZXO7DcryUs8o+7TY+XQGhixlfhHtnylDCVdx5AViWzIonfdM +piJ+Y7SFViU2uqvp0zS+l6qIadXkl1Zsxo33U3Pk7KeMWCJsGJC/UKj5gqaCWNTHYul61mlZJgDj +SaEgbcNc3WYyNXvxsqtHnuufGP+kWgashXjUo/2xo6OAGDbFuJ77dEAHJNH+WVmMrQrc3bbZmovP +2gsE5QrErYV428q35eQCVXyvfw9DGTl5IfF+f0nIc724MrnGqsAlkGSsWwCi1kJ82L9Y7Wxry1XV +P1iFDFkVShVBL7yyekchfYMKH1dAsVWBe9tKeYq1EC/8qNftvdK502uFF/e0PWzNxYftEasCG6CE +74L10/ZIb++32b2/apnNtrWtyOTk9g7Sse5nwPs7q5HY1qnL7Cu8+L2r2GrEts5cAf57l5R0unvb +VCkV+PTiNdUhttqVXlMlVdjaKX5IDrUVxB9cr4xAfxnkMECbFLUPazssW6nfi83vpPnduZI+FHaz +GXEnef7OF9IrfAZGCM/h7pElrTMynmVEv29jtseqVE/eU3i8pDRZ2LzPBT3Dfoh5R0w4v4Rfote2 +VPKS016xHIzC5UyIi8zjRMuH+Wk92xlEoJvRcrTrEB3k8aQxvvfl30kjv0n6Bal/lcx3SNrRzToD +gW6kHJVM3stOjy+JoNtbNcAupfYP0DP8Wwa8aN0MxlvYzAQ/QarJ6sTtnTlWCwYMY62x6J9BPOZa +L4OeUqQiaxhr0ZyTOba6XnE6MXYNuSa09HpY95Z+aOBmok+p61Xn3U9Uyo30y0ipI0JxGtHxGs1B +LqMprOutLOeQ5qsZ4l3SDQw1mu4EEzE5fDXHKF/0QhktFWsUNtfyIy9OzKoYWWKumh4tjQ4mi/hF +Y8hOBo9nLevNb3ieCp6gzKhiUP1BmfDRr5BYXiR92ym6wK6RtZSDvnikKMMTJOmMLDmWAb4hY7z0 +aWxmUL6RXglWUvXUxrCjsykTcbTzNWfcD9l2jjJjlPQE9yzyjEgP9w9OHdlCbtwjfljEzceg0QlI +wex9nJJOjKvkbVwr35c0ma573Xn9ebsLJcMXHROLNlbIm7Fm7uyt+ro7O+vGr5yeClnkdPUGQ9T9 +gRJzqjQ5kjm8ft66iM1IOib0XW84/X5k+o0tyvST96ZT9MmPiup6iz5TkVFvRonNqVeN6TJqq3vf +dsdbFnnSxr9dve/8yxxEV8vSKIaRH0stPmKPHeg0IO4wQzvfciBYpeOAG7klFouxQxvXsNte3bqn +BZUoKrOp9G2dH8BKoi1ikmgJFGU88ynaiWHx3A2MmUVWYsKl8pSwFmIT4tYZeK/d0OuNmKJd9Nd2 +WTo9s7+mgg+jbInp5ut8e71d76Rhq8RFZjjeKrmeiuwAK7rgrjZnugET941YT7wyvYi4dcQwYkYp +RhTLd6lnhkzL9w0vtF2ZPplRpHN2bafSin1tteCtlG5iriR+7d6hUU7WfSybdPtKdEJMBge41wtJ +ZNn1ij7pRhj5Fk1irfk7Raz+mRdNZYIScfyqDLdJv2HJLVve31HioLcNV+fj/ZJ/LqdPasGJGDsM +5XLBkAneLypaqx0bNSzGyZpq8wnz7T/GbhQlznqn11GobpczS88YRFmlZfAzOn5ORsLq6a1yrUeE +lj5ucQjSrt7S64UCIkjeSeKiPu7IVXiHZudaj+6pjlq6bjudYLKSWOfpVvQKWxCRU8rHmCA4hShe +qUCEKb256Xa9mwE0Cw94smXWcIzvebszZXhdMFi6C7jHqsRlnUWSMD51DdM2M312llZhraX+8I82 +UR2jnqyqpO9MHkUZqxLjOaaqqkq53l3DWGZOhcusSlzRWbNaFFsem4KtvdRzko1qdX9x+v/cnqF0 +QtjLm5zN1Vvln6oibX4vVc97L9NAFr72ZFs1eNHAf78N/5OuYeUC/j59xZOuagb09f7OAtZaNfid +K9+ahmdShuH+2BW3RmJtR45qfOrplpfoto53rEr8xSNFm6Ey8g7xORYrxcUrULIGJ2i8Sg1xgVWJ +Ukl4CzVUntJmtn6C3pkM8Q7OMIEbdUx4cybSU/+V5cNux6pXiX94xBxs4iqrxtOOna5vrUrEIh6v +PVt5H/QO9hoYZnOSSQgRpcwImZ/JrEV2t1XD8VYl5pl1PHRYlUY1b4625u5Uhdb2Hln39HS+8ot6 +ceAwJjwFytxxYLVVI7G2UpTGM3iPVQNo3VsYMONbXmPchWXIS9A7UkS/d1581Mb0HmRy/2VMeUUi +WLYd70NjqJtrZvlOdEsxFK00PO4WayRek0cKcQu6e0HQQlzfYdboRU+8jWhTcJa9Q0/WLOtXs36Z +T/YvxxT2t8yc4W1jz5ZV7whnh/aW7vpp6S6nNAufQDtHu3NbewVrpT7U3L0GxzWl5ONMT1n40ql9 +Xd/af+qt3dOn9ndaywSbM040O50yXGN7tJYljQ30ImfucLEl71NPvZRFuy3g67YcuevgfCvfAvFt +W4Q6oo0Hz/mWSTJo+fCEacC2kgZE3+W429lE3ZWUULRu+szO+zQzFjwoCLhRi/BFW8SqxBpLTN6b +bRVWJW6ytDO85YzS8T+2fLjtR91cZ8XFot5I59KbXb0dPGlJbyYroqTaGBPIqWbVYMpsM7D4WE5n +dNwpSk2tFNUBjhYjlgaz6+8OmG8ITPkpJsQN+UewTDj++7ZSO7SxusiqwchqqxLbjD12NkMChy12 +vRf71qOpSNypLExWI734oEOqoEovNnryWUKfSUsg1tAbpM+4oQUm0tz7IG8O5cUzP+VEcxPXoikn +IwW6PnMAXi3TJA4xydL7JSXFnN9ZwaoSRhW5INz61pUfi+EXbm09cxGi1EWxWEz3/1qziOc9U+SX +dVGbTtrrPNv15Z2OIlt5rgau9yijDzxPxazTcEWHtBeVxMeeYbHxlg+fmpsd5uZYLb+wZc/b++2o +MoelXE4a8VRHRM/xG88U863G0udC87nIfM6X73uC1yj3fZukBoVBVuLufkLNcJVogjebtRzFDjYw +YX4uSH/3XkAsaZLUwRvSnutYViEkJCb0kW53/tOZsU81Y9fgvmFTmMCTMyt4EpvkKxalvEV0sHSW +Zia67gvntW9lK76GCf7axNQK3RXdlyEBO3trrpLhkji4x3j0Lo/wRNbJVWGo5cO1mdmqxHf9RKTl +pTLX6PubLQ9xQbvYiUFWJfZIhpXR2/LQi9Xtxtn6KySZYhnLl0qqb2TAJArzTraAPa058iOns11X +sVBb1XRoz+gaakalVPDeKdc77yCGMkrNnV86I71f8GP/HmUoMFcM+YmiOGoiUqTjmGFWDf7UXmpV +4jNPsXbHJfZZ97lnilWDS7SoVNEpVg3W2vR2j0fOB4ZRfzcWs2rQLSyiErvqoPvcU6+cQ7ipM+fH +mpyqfdQn9MzA665vNfFIlmbSgu/6ygH9LYG+idHzqKi/VUS7DK7ibqdKC6qJVuiGTChRrtMJI8Zw +HbGoqZcl2/m10963ao8b4a43XkY76EN7jS67p483w50vLIjSc/oemptOjRoqSnwjtRM9Z/4I+DOw +2W0Dv0Sb0HPGHuJvqfSYe1+b9aPXZuDy9GvtpZlM8l3/dmBcb8xsGzmTAU7U9aQ1JlmZj+elVaie +Y0Tdo2hx3q77PvN2Fm5XIrTxnGE8hsrolSEwaz3a1jCsOYyi+R7lxqWPXgloN9No6F5Iml1N3NV/ +x1SrBc9mvt/ffy993G7X+p1cw0UsHugvPZ6UmSNNtgImKI/+ch2E9yVztAqq9lT/HQxRFoQ32JHP +aQwZlwK5XUyuYsLKJT5JyiXCL5JCrOsvv6dc4suk1JmjGDI45YV8JryfaeGtSnrxFRd0qdovW5us +SrzKpM4HQ3kvZfTSGBsYejQolxUm2OBJhxbtdI23Q9cqcWd6iu+gLYp5lay0+VSHSridmqu8hkEP +vIzOAhYyyjeFIgmWOfe8uDxO5cvSlSspVbH3eOH1qZ7hNpfW1VvvlwNk4ziJiVLxP5YPPebiDjE/ ++MarW6x1cjZKfnlc1zG9y5CUc2wJMjB87GudGm0el4lkhIjOCDxJZVSBZ0VQMosVr7Ljs3QXI8w1 +2Y8t02HwoHcYy5Tw4V9Ljdh10YBuTpamyZOvmXrA8uH3puJ2rwSmUUwoSTWQ5ELLh78uFS2fK6ok +X/ftAwT5e3po7AaGpbh3QDzGGbrtxXZH0sYIE95iYumrLMOAfKK9FGXqLkpZ55xbdVz9nHl8VfPD +jaJF37fqTPkli/Xlu1Y70f7PiN90yDpTSbzszfcbH5UoY0ssH14VQDaUAasSTw6IlytGh81jFR3G +mMyC/pYLLB/ON5NuU9JnpX0bSYUEjgxb04hfJ9XvH/rHGRtcx1p26eGWpGFP3+pfzVVySDMGtHyr +BZcvzTFzsnPAZ8Toar+1DB8birjRqsRVnvh4qwYvmWtPMiTyNhO1JmOIDqj6Us+hNpbsYmYWsnBP +luHlxyrlHyyZdaXI9ovoNTL6awPt0rCE0SgxZIojvXX1z7SBP2UJ0zjuEmUADksZlNZyrxNjkmJU +d6mGZFaWjTrEmNzdb4phjhF7nWOzy+u0eIEsP9cywVTV+Cq0Oaq8XU6VLPxqv2xuGk85VVxrvJ2j +DGf2VskAp6k79hNwJzIxZxxxSLzFOg3bDPq9rgtfTC7Qepu0i0tUBIOPE62R2NHiuEhW4tf7LRBj +ZM5OPxOo/oA4eoVShkrRfQcrKdZejM15MnGl1/ul/XRWv6SJcwuN722xcVtS3X7dVfIc/MI8MuhB +rN2vm/i9q3qUFJNRs+7rMMlMEs4W7dcbn5o3zC6+Jd3I1kyfWmarElv2i+uQimKBs+Jd+zkTsllX +VhBWj5B9y7I0RX7Za5D/dk2N8F3GqqlWDW5dJqSbhmdSc8SFX5XKKAJw6f7iC2QPtl0xoobFUlb5 +21mWVh7pvNMti/auYPQ5oy6p48/OpEKTb2nRXLyyX5pgvNNcJMWORrC/jv5lTPBdJgxFahv7m5nD +Z54iyp7HiSJKAbUVlEkTeAotEeKgKSIz/QyZ2WT58GvT/Nv7iczI8NaRTlgm++4Ky4dHzaUVhtYM +sirx+v6SJ3zsMNBWWT50mbk2q4OX9pdU+JH2oLO/Lh9QLwV1Ii0Pyo8ioUAKx3q+8wBn2lfr+olN +5v5WH/+u3NaWD1ebwD2zmP8eKJIs5swvuvGGKbG7/npgt5KBmZs7B2VYr64DnaZvVdO8FoPky1Ev +RAlrba9bkiPa9ScZf9kCf5xojZRQl1jVGoX5V4tybK/WSlx/gBYTB67nan7JxGKO1+T72eydWMCR +bPbKFhSQvB3dzUGcSPFp37XMSStaxI+t3M+kZ5/RlFbQ+cWC+5gYwRYpF/N0Csl5IPdrRtG6EtQl +JjZm6IpC4fYNfQZ92wHdqI9k1IIlhzijffwAW2STU0+vQqvLqZCFXxyYre0I3QogXmHNom29QvMX +B6wXi8f1jdYsfJ7MsU6TECNdQkDUlVi9/3ppGZXZLmr87GpZVU7sdFcTj5o1AKsZ4KdKMMKEua9W +0bvrpAOVE0mKOK2JiYxAnZftgP68fRWFJD/fiHLJii3EFe5hxMepNXK6mqBbLcIMlRjdoI/HZQSF +nt5G/nRgdpYRNn18QWp+46AYZp3REZlkmIIuIQogNrWDUUr35y0v1+ly6WKDFJ8NTOrbPfa3ZwdW +M8rPxLBG2dWikhvtkj8O7C4ndruGEdd0NuHMpgyFdR3mjOr6g7KzmnVbmRw5lYFqkybOaENjVdZp +eERNYefAeIuB4F+LDARrDujmWl5OBIcRZ2ymkSr/sCxinWZcmxLMrMdtA9fjHKffvMMz/Wbh5YPk +ITOZfvkltuZCDkqpIqnKRjOA5bnol08d23KY6rLm4a6mHK7SlnjhoKd0f5d3lHUati8WgavEjoHr +5RqGxYLgiT4PT5bDWMAZ9xGZ/rHuYJ0sZnqjNJN2vj20bw+oZqLqnYlE/6csHy5qcrbZxQdrm8nd +ebUm+Rd2/X+rPv1V2vePmMr2vr/k4O4lMSR1ANRiUamzFwY5INxtg6CQ7AaWK62Qmn1+YYFGebe5 +wQTXLRYfhD8NXGAunfXWreZSlqxuWY0Wtdwb+1832Gn3ebVr1WDl4g1WJVYeYM6G7xflsNyqxIoD +5FTuDVs1WLE4pw8F+G5gvejOd4tM9O75B4icNkhw+2FRjskPZGvw9g6sx1JnQUuGZHrNwt8Pzt7E +8N8UYqrdefnSn+7Ovx6wXrmBfcS9C9ZkqNdfDq43aZuM5ZEYIGuz2EFR6pBkBtV/bYGu77Jn9ruD +4zbCfbdEyy1fugS9IZsAfHiA0VwMnYJq40VykJyQyhwGf2gGXtxwSLasSOsyO31Xb9H2Q7KzlK3o +2/mawPMPkdZq13xprXoOiZeL1t27QFghbZhByU2H5MObr0Pntwt0KD52sE6oG5xKCfqsStxySHy1 +tLbz7VPYfkpsOSSuOzssHz7MGFArseUQ4VqjEOKTJrPlLj8onrJG4s/znRc3HxLX1Ny2QLq1Vw7O +V+UPmyqEPb86KJ7iyYb9Q7Iog/67jnRG/4qGuEmukLKKTGIC7fKGFKPzYi8UzxwiHihl+bAmMxar +Eo8d0q2Z69CvLZLiE2nNqVGkspSDnA6HOR1+dIiSRFZlZtuV65R8JVBsVv7NhYoU/M2BBmOfXmjG +/ZsDpWN4emEOzW1Zu9r7zh7+dYh4jZi4XhwzBdMK8NSgYbIKyQRbxVPxp7ML5F8Lr/g9Ld3vzPC0 +dPaqXHqoWvDW6tbbLQsNqluVePnAQu2G2xbmcolVidcPrNfs7llgNsdFB2ue7s80ZFViw6GaqMtZ +i1ipMkklsWuwOsS1cyqQyLV8eDdTW7OlJXpDy35IteIdfmvV4DG76/Que+DAerO9UWAup/zjglK9 +suXg7gutGlxmqup+b53cfzuwHq8PTpYbpZk4EFw1J8LRzNUZYi+JWRdMKPUcaUsQO4c7k//sobqg +Ci8MnlJua/B8xG1zVrKM6+1L0/+0IJcJ6zh6cZ43X2isayPj1nC8O88uwHlZhaNYJviuGCDjakqe +CzEi1SQn0OUPU7ZnH70SNaMxLhHOvjhP+8ewTs8cKkyXBUoAaACGhVqdmS+rEk8d2r2EUcybg0ML +rWbsWfSqVYkLDoibAAUZO81bVjPxz0UrBcilhnrJr7eM3sGWD3tMfzb5eMuslS9mNeOvi5qqdF18 +JXH1AdtliRJf92mfyu+ZyrHxaGqyfPi+T8mrpsQXw6wm5daVasFzoD2/PX5nfi/MNsqBspPYrDnw +K81PNIaOpkUZ5iBvpFN5Q7bOJG3KBibOoZxmR8bGs5lDtNXwyuJW3DZQbPIChsQjiELimcVNxO0D +5V3oGbKHUWsa/ji/VVKK4pj2dZKagb5OUlKtPXiArZ1aqmNGe9c5K7tGO3DdZeDiVCrwYhRrrWn4 +bH6uZu2yQ+Js5uCYNsfb881+veGQbq6iP2bz1rZxxdY1hDLzU5LvNP07u+nEEGOPapSs+VvJ6ZYP +b851cGSngt32xY/LMksh0c6qxG+z4+nTNipFh/S0jPbqOksKnC4/tLuMnuRF50qjG/QpD4UJpY6a +i9x17w1YLxZPGiI/oz0mGLlO3p7YaQ995yHFUqU/bI/6sUMkS2ozp5jgeCVPYMhY49dhThP39tov +dh7lQHJRjlzc99BccSwVXrGJIfeb6Vxhd7P3kHr62MgJrGXC5OSRk2rMcId43RWXx5FIRQtDsizx +Cq6lxF351HT1zuCVOd2mgSiF4TtMgb0d1qlE+pLfz51jb+PfZMvbyMSLDxIz+MiQOHbUFyhG7XxZ +KbzI70FzE3UDXYiNVWI7lzBwqmJiWMZ8BnD4K0RzjU1BnpqXy9o0Bam2Kcid+4mCPNvoUJB4moJs +20/elj5GR2FAUn2EOUjpiR5qzBVG54gp8Uoj75XY6p1l+bCx0UGTH3JESl4Q7piH9gC/yumWhX8K +o3xIg3++T9mvDtO0PFCuMYCfKaFkh84SvzJ6aqJT0kxKWFGsUa0SU8Ad12XCo+PEcfnWz/BRQ1rN +9G5OPn0yelK2nBAF5mTD1UQxKk5U11gt6GmwGdbXDtuBMyoQjuP6VCvxtrs+qGyac9gh+1JAW4A+ +iT+XyxFZW+2Tw6RPDRLClPMH56Pr7FLhm7HI3NzgTMLnh+ko0kDfNw/tSfibBjqPg/DFIB13WKNj +8bbOUuItV7UcJFKMYm/HPcQP7npdHYXGbXh1yDBrGvHJFLOxLx8rEVHnKrbVR5Qpo1lHjF72KUgU +i4xjWts2ose9nXdmprCLtUyx9se1xz6FXwz+H9r/do5pf+2+7S9XGFCIODRuQuMCQtqPGnJtpF1/ +mDzsynX0VxFfd0SICz0rYvjWpM7SdPy+z7r/W0hPPO96SpRC52MCCUMCcY75Yw3HRY0RnQ0jKMVa +QFiYqhkcNEpML36bI6/D8pTm5/J5RlX6yaHiY6P7wryx8+EMM1R3nLPxtx4uQs8Ej9F4/n5YNS5p +MG64nx+WLO89iNc0lFo/wx8yQ7zlsHyRKknmeG5QHHefXeCXiWOpDm3jaicDZ5Z9BpUc7/T2iN1b +yJqGPzSKi70tx3jUWoUYWUhUNDFkzcX2hlJDDBM88wyaPK9vsEryw7HF0t1XMoB/zSnFRYMLrWm4 +eEqBVYnPxxbKsn9uEDvnbMPDg+PEY3OKWMsrGZqq7cEATr0HTw0uxIdz5hDrBks9jh1zIuVMyG+H +eGKwIl3wwBzFn7QwAMX4ZCtrL17MqWaASnmO2xrfoQ+LV5p9NK+JAQbGM4CD38a8GsmNB5jQGjm/ +G+RYHmEz+i24ATVNGBlXUAyOFuXApsHV6GmTNeECORY/0XgPR/IGs92i5hrseAzVm6mhyDsMH815 +VTMOO1qn7lX68Njgb7GlrQKbGrfhN4PrL2UUD87J4Y29up2dYWfmLzoiW8n9cFNOEm825hpVCW7I +kYSzRAC81WjuA7zELKxykuDjhpUq8OqUx8rDklWGrcOnDRWWDz3nOrv8m8O1y3VM/sk8tHf5P+2n +RE9Oks14ozEnc0m+68QMUFnoOSKbG7EuJ4m/G5hOpJxwcImBy8z3PxrXWM347XxBnGGQHjmkR26i +f5aFKIDV6iDDzPc4jeNOjfgalmm7rDhXntjH9eUQvxr4ijD+IIlHu89xRvObI7pFdZVRI6E3Xz0n +VyTv1iPyOUhj3NBnjPcf0S18b1sp02qf53faz+etpI+/oA++brX03DkFP93EavuaI0zeu93nFBgj +wK1HvM8SBrAk1+gRPGLwu/u0/sQRmm90PC1KcmmmIM177DxCtAdVKzObL684M9v42xHZWWWKSYrb +pCkqrmICmud4DrY3aldv3e9V13jFHqMDaR19cs8cFJPyUKrJTkOafpbhHF0Rp5crBhmCEsBHh8eJ +9ecWcARnyy/uZHkITCN2n2u0Cj2Ha/IUP9gs6XAqo4D28xvnykb/lF4+S0nf4Mk36p63zjWkf8vh +Rgx50/520+FxQNtWHoBAfjlPO1E5t8Rnn+2s6fWDuvHCoQrNwbOIM2BNw00VZln/GCwUkyRCOXIq +bmmXY1857phXavmw3SCFjc2bBnXL70uMn4++EiPNmouycMXhRgF2R4WB7sHDpZrEd4fla3Z/0VDg +x/WHD5O4ocNKiOXHm+fOwarDh6UMf3xPhXEJejZYyIQ1DdttsJ4PrsCec3MNpK+cK2P3rw+PU4jw +Qe+o7hrUjYf//xjVw2ZUN/YZ1bZB3TKGYf84mguI7sPzcWlDqeSfwHiiXkkB7NFC1wQ0mtXIWcHo +sRGckWtGo6mQbCNOIGAccKJ0ZI+8EgcpXjNIMY4JcWYSJkzGudvN+f+26ynGZH6XEVUX44rGTzbT +8etzzSq9LmRwi70P0TPkG1jKV5FSTijK6piQSOLjKVI7QskjCXc1AwiuIH4uT5LHYgxgTCExtVWk +Oh8LJJVG6V1n+fCr3hk9b7B2lpIxaaf/wRSY9SeuHdyN+RHIzmznHpDu98g45kfS9yf4iSP0FUPj +6CiiZivERuXzPBm3dUbwR/HH5kDH1YdMwe75TcQBhZkinBOBDdd8y4e9cxzUvXeM+Eh8OL8CvzpE +SBUlPpgfYTHWHyaETpl5uqcPFnGkNtldNj6+GFyvb2nsvP2I6nKl/+sdGK40Q1Y8lA8P9j4nrhvc +3Qfw5RHjJz0e2zJD6VCoJzzJGG7qbGJMBKFQMZYyEC4yxEH+JzJzl9KxmuZNcDBi9+DsrOZrFPhX +xcBFbGBAZheZXwJQthN33GrB5jmGRSX+OniHgBR2mXhUr7z4fk2jj7tzjq2GsSrx0WDxPRi4Fc25 +bKZfmWsCxIGvMIqDk8QinbdyMTggjtQ9lGegD1a+cAupVrkAbpoglt3Mv1l5fD04qTmf/GomLtgV +d8Zw1ZDsrDKZoedIOXfLYCnnVs4RO/L9YKH2Is8Qm7DW9b7To3yDVIShAk6jU+ln6O+CiVHcZbYA +XnBVW9Nw0Tk5yi1oVeLmI/IZtqbhoXOKiKSevHhEvpIvStkuH/kHhMuPM+R9lqNEZ7SDG7QzmrUz +ohNistxKAk0rBXaVOkO4X+Dg7sYNeD4nLl/0+YIFV9mQvG8g+eLs/wqJqeUjPq13sPaWId247eBh ++INJIvO/4mhfCvqPo6rx8gLR322ZVbAVm1uGDIthuRQMSjGxrzJhOvZVJpyBF3OMMqGUut4sHTnp +Os0Z9tNauU3Kf3qKSEgMnY77fl1vrZc1ObJwao9bFXj27KI0z719UL3fFqp1QejJTFjLiL/Mk2R4 +9aHF+rJ7Xo6xX6wyJ7wUewb6Pt6DJWUOOO9kwNGbf54nJeo1djPv9m3mWruZXlh39jbxgWDFLxs3 +4DsjmgoHfMTFcyPEF9lxdM/Nwb+yZY2dL/o6miF5673YoMP2zsMKlYRRUU9m9V9NiRTjVrfw8Jb/ +vvqmmm6niMWwRKon+25O49hqJ5X8oSHHdk0q43+OkJPyp+sws17o2GfF6iY5U3T5UPGvPYzYc73N +zPVr9iRt7Z0k/OHQOBusaeg622ifuwZrI0ax0x7STjOkK/+3IclGF2UiprQ0GTfNkskOLD1Ds7PK +rpW/m7UMl86TolauE58fKs4+IHVThi/q6X1pmxnANTxFOQEkK8moJt5K6Qi9dVUS8bcYFttXpxu5 +ZH6qkpy/WlKWr3M6G4xKRryKVlD96/MDTDCGdYZ84TMzvo/m/NcNS+xxVzOEs5uIIxXHs5IYVi9m +OkovPCti1FX68r1Qzje5oEQxzTFF5/08Mw9Z+Gxo9iaOI6y3w+E6Jfvwj8cyqYAcz+WePrVvPTKb +yOohljYxehLhnlI3SwRKTojqrVZftkhPqqnBAeuVdlE3AJkgDV4h3DTGXx02inLV6aL0MPIYkNE0 +xDfTdRUjFJ3ABt5hprFOh+Bv64u0Fx8dUqhnmm6l1xa9DKVbUXoB6crKM8zzzsrMWPHIkdmSnyhh +zNO8x/R9aq/XXk/CqfnFkfap8OAsnQpPHGkurdk8S1fMVBJ/PjLOFvrYKCPdl7NEMu2D5o9HdhMd +TfRlcCevymny3+p8U3iqhvGr+hxR26t6HSg+PrKbCW8hOmuMFtpfp0gb+VZtutDy4S99+vhUfbQ0 +9VpFXFMzfWTh0mFy4BnH8jpFWd2hX0lNuVZFX6TZnjmODZ6TvtVjqRek9S2jV6HkooNmnS7ngzJj +Zqaw58xMD9g2zJ7CdWLjNrrNHJ7q1DvLqXdDup5XYornwh9XrHYq3j/MaLQT1D7c1Kh9uMqqxHs5 +ys5ZqwPQyCtR+jJxf3k1mZezsCE3exOX/sxgVGDm0XyxLsaozM1Hs1b8aRk9Y+HVWEfTRwWu4lwt +T5Z9pudNyzSFrbn2wLxBjVyLK5bBql7EBPMYvdRMlxgOQZI+jHY5r2fhsdxsiVrnU8rB0ZpR0WF4 +ZD68OtNf13Snv2dys7MUQqIsMInxVfLxU8vp2MZdvRVfE2BiaOqEijflikF5uE56l0dyRRclkKwW +vELGdTPTyGhbo4j7c+OsN4y6TAVIOqbWrhkOKH9TD7Jd4+/LN0ueXtdPh8aTs3L+G/NiavmIR+rU +pbG93GsYXZ1aZuxbmnP1Wf4Qysij3S7VM1pXKnexTrZkLv69//+ZxejLBv/jqCQuaC5lLfoVCoqP +6oqstKsV8fvcbmJA0gzbJxSKC/9942Mx1DpngavOmYAv8+y1D4sMM8Sw1UDcfLaRW94dlG814Nqz +m3gCfYr+x+/1hNh2tmS7Srw2aD1Fpbxfpq++2TRL/GrrSs8hNpK5ZjodfZNnL7p83lZzEaMZOlHn +1MnC5cOzXxgqn+Ay9Ps2gzmuWU4jvxpuN6K0DYrm1A6rFdb5x6PXO66r94Vbh2t4upveR/SbwoBM +kVfOkkj6tyONbfSKWXJzNL7+Hx+ZL/DdhSzD0lKl9QoXW4tx1QwxNO8Nj1P6L+n+XjI4eIdwUCjo +ZzONGSEmyqW9c/OcUvxx8BQGSk3KxxBxmNnTkqBCgrqKMAqmMKs2WYuJX82oCDMsAvv+8EKrAbvO +FjU4VkX3L8thFcNmhWQHMStBvDd8hZHnm5V7374KSTn4vXJQ0vkT1q1o+HZGq1FFlnHSqUbw8z1O ++S78kCf9Ybl06lOFscKUuxnlagNeyJzQq88SaWJios58zduOmUV9vBx2DqvHQ8ZgEu0rL0nGVH5x +CVIddkGq1HOijQ91ZztLed2I7KzHvmIZN9FnJfFW/TaN/uYh9caLI2BmcrWUuqJKRZPq6BnylQ74 +OiuJ5+pzxFncaPspbB8i12jJCt4RdXN5KVfRexBreQdrKVfjaSIIqr+WCd4Od1wo6P2EWLqNzVjq +xKrmNTjAPTbCKIHKeB5DnMuEUowrvZ/RlkYZnkXcdUCceH5RAe46oLpKB5sSxVcSz+Xpgms5XY8q +Z9lIZccwGfdyNZn4zQGOSR1PLqqQxOLHewMlCptGdlTkqJFHg/UsUw7ICnPmPxaM46rFudj9Xyqa +HftYME5sXpzL4ywfnpluUycthVWJF0eIZOKNgVvxlKtQfIO4N7y5SEojvP6jx28tUnKLcnnuzlAz +jpWfeHZEnKg1S3tqRqrMm+vM3d81d9zIGPaXJ8bfDRSGRyB2j+gmmk2ioksOKMZniwpwyQH5+GxR +Teav3xqJG6b3upwSvx8RxyUHOEoo/GNRxahysw2/m2EGff5wKa0M03U5o7B6tEwm6sdHtDWVY9UB +it4x85tepK/yqnX5foKn4+o+K/Luogo/NvZ58JdFFWpkeSTDRffMc8Z5oc+mRY1sNgQtkCForvlO +nat80hLIJ3okkI/2XPx5YDV6Fhf48fnA3iGtWZzGhGv+DwuMXYtzsel/rziF+CCNCb8wa2DOqSt8 +0hFh3Y+W+9rFBgvWDnx7H+ToWZzBgp2mCbOMuNTXjSbnMKlbkBluFtb6xYMZUqUDRYzpZaKHdU3I +KyZqc/i7FI5c4VD2psyruMqf5oV64wJLekvvUakibgZ5vDYl6eotvPcnhTt7C3/zk0LXQqfT+35S +WNJbeP9PCrt6Cx/4SeHO3sKH/dlZ5s4CeGSfSBFt21Bcj7nblGpWoZ5RvihHK5Un6MWAHtZSUeqY +mMvbicgUBvAz4/4i89IW1nrR/ykk51DJsiZxLfHMDPkI497hSRYqWkK2Qzw1Q8IL7h+eZGgcAzIb +4jH72W+HK8ElHp2h8C48rG8iB7oQM+TtZAtvIaavUfqXkSKzYxQFg46iEqJjjVIX3EXfwgk4vBvz +azAijrNqsLRG+n0mLmUzTs+hJ7ePZb6rJTPHWbh7pO2FChliGpFqUnPeM+mjt8OTzljuWtJb//F0 +/Q0zchVH+NHwBYxyrnFzTrB2fAZ36pxXsGekqLWfVWJEEo1KoSK1lOdodH4XM8dfQpMRxQP9q/FE +skiFBj/l/CHF/Z39FxDvJM2MXtdflxfPUEyhCdnBn5OaFtzUXzY0vJlcQ2zprxCQGcRTSZHBMtGS +LxnI8MYlyzKjQc8opRrDW8zHza2lLONM+syxOxMvcxgehUOZHMqfFPtuTSMe0lmAb4+qJm5v1VzE +8ALz/w/vUC/hztZS4sAVxGKpOm2K+UGN0Spc7U9f7fP1WSszvoObR8kV0Ed81fvsOj2jVyKUF22l +Ulx3pqTztHy4szp9LFiVuHZU0vLhDj2haIR9Lm8ZFZe6g0FxoPLkUFBlLSfQM3QvE9KUh3kuQynv +PHqnyfOBoX8ew8skP4iBQT/JDzqr18WU/VQOJbirs6mvgbBnuTPTL2qmubGOI60iXFRdwwQVZR9U +QPdIaXg6WXYc4/LAe/MsgW7I4c5RIocd0pdvEPiC3qrE/aO6ZQFm82EZdOtqc3r696jsrBu/Kafv +KnyTVUxcuER7SjLc7+py2GApZuq3uUm5HFxwVo5MojrSvxq1Xk9WnyXx5atR1UrYi+Gfoa6Igani +TiYwwbssH67sha5rtLTnsg15h9C+rnCWCEaQUc9IHPgt0ayuvYrzvoMJeN6npOgEA1IneP/CBDeK +s/MqcfUjpl1DwYl1o7txbDeu7oxgjIlV/uxMuZZUEpeMLmQUo7vRr5iYJG/scga8JWEjPytAMpF2 +QLBALz44c45UivjV6CmKeP7wzBpeNl1hOKPjrDOH9DvV26xKXDeynuhXL2P1aGsZPqvLEfMhafeS +XNlTOoiBcfTrNoaHWqLlHd2hs/9TJmGuTFyfyEslJoWRP84ubMpagb8syS1n+BSG4KmGrsA1CR4D +BPcyYfwUnj7TWeiXzFR6JeJ4dTQZX7Zf9p2SF0yNAFeZNGvymLF3qrecJ4haICuudLUjsZX1Yjjx +cmvEeE9A17f+MqseXyypGIeseoXmj8fHlsPpVWFtW6l29L3Vhrt7ZqTJznGD/e3tkcUT8MOAYeVy +A5W3vY++6RhcSJwri5WfiS/GeYZ/PxVdS0uxd0Cc2LvkOzGTOik2ZhXSh/eWFOGHAYU4XAo8n+Jg +K4y1G8+3NiHV2j4D7a1YugZzK9AZQUvN+QIRTdojUXHnaWlt50oHx1flKzW3EtyK9iYkXy1zVMx5 +v8jUy8Km/OxN41jLCyXkYLtG+flZZpSXjErq23nVhoXdO7K4SgrVXI4tl8TRISnvjD0yxjLKXJHR +aMa1uOSXmfZxR75I+zhjJfrrWRuEoFePiuvS6NQ2I1pJvzaBsmhuqasSu0fPYrUahW3iZWkfJdf5 +TruPa3w0AfV/m6rgs/X5Ui68NFW78458CWvl9NGvUKYbDBaZjYNb8s1VygfaTIhrldPgSwIU7h6l +o1DYUsLIC/q9Hu25S44gLjq3gvjX4Uk5J3EMHmkrZQI7rGruUUdSUUxpFO3DfvUsk3ZR54AtXlYR +B8gTDQ91tuIxl07ygC3NsYXLTIzPzqlCdRvISwtENqJokNUrJF9MyXkM8Vg8iV6s5KnEfe0Fxtn5 +uzPNmv1ytEyT0ZjYiGYsikj/1qfhT/PVcBlxY1sEO6w4RaE959rzPaM0Qyt3XeBMy7eaFm4qMTfG +nIhvpkusxaoR4gVOYoAnM6AT2HAby5qI76Y3ERePKBYjUqx2m7HMUd3uutBp96GCbONX/s8qaYYu +LtDivVGlxbu5wPYrv60P3DcWyFYP91biB1c+J1o+XF2Vni8mFPRE3FIQn4BvXcXEik7ZAW6mjyN4 +IgOT0iFoW7C/bHq3VqWPt1cLdhALKnBwvhfz7sE/9GpZRDT+gqqIEPX3BXHLhydNR+aweaWgm2Ve +NDbZlX8WwbkR6ezOiegBbu2sKZE1h42WD+f1Af9ls56bGO3iSVzFmOC/s7fZD83o9NKqqWYVP89f +QZ3VPlYT5bmm8Us7m3S490ztoyu6vaC7b9muKpUZSPcUdHOQhnJ+VTou7t8F8mJuwcoq2Qf/XbBA +55ftJlli/Qz/rKqwnccuLsinj48zYERv3ZSMDa0FQH5Ly1IcJEFt6sSljB6FSztzreOIl6vnaK6u +G1VtHYd3zsol+Io5UW4cpejVCus4/PUsw/xIR+PF5lHDrOPwJ3OKbh6VjxWdFfjSFTd74MOE1tQM +4M6julsa8SCrjVtZWup7LF8sr9xOrjezZ3aLVr4b3xtjZBm9FzAxfeYpxBcpR11Yd4mDdruOkjoJ +h/RoLn+TkF7onqOEe3cmhHuvHKUOyojjlfdaNG1dmIHhMaUPZcAw71ODsqvrlzTxn3AGWtbg+Dhm +R04xdnePpNfT5SbZEg6b9Hm/6Mi1fHjejM1AjJVHi2nBd560q7q2Nz5s127fK7b6LMuHXX3q/+so +1Q/p7HtBBLNOaamCSukBT75yMbRYI/FVpcGdV47qRlYco0w8RLcZ4Z1HxR3xd60zFVuP1gnBBL+k +uQZ/BBBnEWejveLXRuMsY2e011m3ZJ3z6iNHi6hLRTKrigGeBXf1LMFVZc3CpVMVz45+b9t8Ez40 +a2afxopCWZgBxXWZ096fjpbz75fC1+2VFVYlnjs6zpCShVYa5Hr+6AWspf9M6UQqHRR58WgRM48f +zyW/YwJ399/q+acbHd8RSEpi9wzay6hnAdq+IwZIK7WXWFqkj/cLu8VEKE9Kc4QYmC+tm1Qqcxn1 +3OQx71hxObyKfxX/+6nyhoh3O4hRLPpcFvzZEWJ4fj4TjKzmXRTdj9LzItDynQnTC9BfVTVVf+jj +meZvQnG5Z9sfrzL+s97u1bivcxs30snv69rozMydY4Sv5pj7/gwdcyvHCFv/coZm6eoxwtbfmsBw +ePL95S14ziV5BM9COZr+s5cR8UZbBfGiq7gcd3XWWCOx+wypa551dUtFTLxytNQ+t5qJNvhK3DCm +m1giNH4g81jGFqsS68fEMSBpeG1lEfEp4M8nFia5hiEOyhy0JZucIT09JjvLZBKR1nWJlAup1owD +WE9vtT+o2oVM2GqfVxIGuW87qro8hg7HYcx1tdPsZQFpaljGZSzD/vVoLsidyAcIdz6PYt6oE+mb +zCg78TCGNRL3txcQT0JrJww24trlFQoVuOnoOE+WN83iXA3jQJ3VvlMYGn0KTxFDiYf6nsAB+sX6 +Ptqea5JoRHVd2FCbtyi5xoHtXcG2ib6rCAURi/Y88XPbaUR8gFWJ3xthXqZOTJ9DH8+Vf6EJE4Gv +R6fJGUJ7sxp4MZBEaxNxdtMMRnHEMN7Yx7f/WqfTb9TpNRjRJ5qleuUk00XJBo1kRJABnLwD0QKa +3MQnb5c2QSzflTYp+eBoiV0pYkdnUztxkpjTqHOY1/VkuspCdzD7Gs3j6WKYbu/sHYGhbC+55K6N +7ea5rHhekUc849JV6hIjOpXXjLirM6cdqcgNvYnGuzZn+sCDQbMTmE8clM8Ifw5PXG4R5WLzkgbn +ogyjs0kDuEd7hXh2jNy0y+3Wm3MYHMMqDCy0axhnR+K+MdtHMQDkH0N0FJyMZFMmsUrXdU7nfwoa +hdC5xj7kHaX9LFZvGX38g5JLEskC9C98Cf2qdcGyEMe7Q91cfIbhJD4bI52En1E0fU4cUmwcCi+y +y74cYxwKL7S/fT1GPMftFVpucxD+LdiNS5UzWbv6P/kOVukm3KnGd7CMd+Dv7imWD5+aBmx8+Uuw +W+netxkMNVr9Uk/QxtGuG5whXjBWNN1ovEcK8q9+btj+CwL1YdxisofjxVQpfVyHN9yO5lNo6met +uSM9gGQFtroLsVu39MzPlTcWTNqLNQYaM5zusd34u0n28F+JlBlOlOW41FNs+fCsacAezvlju3Fu +bwrIG50R3Do2W0k0qxjI1QD+/PP0Ct8UeMrPwBQGDKro4pQ6YlEBDig0gWMFdLLWl2zJNJaF343N +fqGKgeFq6vFMU78NbBVFQUoeZyvc+amf+FDak+d4vxkJKu399ligm77hiuXNLS8XCs1iGUcTAwqx +NFc5B+UqiL2uV9Rl+p23A/FReRIvGOBqvO8a9tMejdemWavng0n8qjOX0Ux6jJ6bMuPBP7S83MgX +NEXqPiCuQylF4CnUtm1haIQxbRqnBmsafvnzNVYlLg3GrWn4hzrQZ5wZEXQbf14jbo5YEXxF7JKV +zwRqc4nhUxgdocOIqCpCQSHOfBijpxDLxNtoqy+lT9LVCCqNRojTGaVx9k40eY6xcbLk1w7QW46x +97xSWeq4lqQpvd5oecpJZzJKROcaWcHkuSAHULEYMvQIz5eV0tvcR6u31Wn4/kLtZxmnTmVoOOGp +Z2KkrQtwb+f4mLRZHRKI3XJ51JIl2GK14OHT07LDo4XrmbBa8NvTjT7u9cJiLC4gBtYvIloiDClh +8r2n60MlnincSrTkVhFZ2xkbYs3FXafb61VYLPM89u8+hROIz6dEsDcYN3l/1HGCjVYL7rK7xBNY +T/xybNJ0e8fpJvf3E5BqFavGrmftmbyUDYZb8YyW3CXZPGr/to7Ay6fXxMSm31WYP9U6gnju9G1s +5NEMKIsJF2sfe89js+zHNhNwU6FUNwE14VMk3RF46fSamJ+NxzOUX2C/cFbfF242L8h4qntFo0un +WUfgtdMjTVc8rH67NF9MoNM5PrpuT69GIgsfurNXB3FhZw6+dhVjQ6e8JNFvBX34k2sBF2CTnuAv +rgVBMRmnjJUylPPMehdIeRMglPdZvFU0o5HOu6O3/RUetf9ZKgfr3MX4PtXb/teuBVyPFab9710r +TBCKMGwL7Q7qbFzuLAgKWeSFom6DMgTpvhKiUZmmtQyZfl13ZvrFVo+NvncwoJC+0tk4sX42Jtfg +qHqirAbvp2oYIq50FzKE91NFuNIdx3HrsbgGB9bfjaU1uCdVgyvdcXyaysU6dz3xaapIYsdgynUU +17sLmcBRfd+YXoNvUrk4313PEzmI+DIVwVeuOLKTuD2Vg5fdhcRtSrr1F3exOr9F+Rpwq7sYwwsx +owBKNavUwkorALci0LyHMGQIpXJ+jKePY7X7dMMtWgqAYh2belLFEDsZ8oz6DvslkYyg/9tojUyY +Nmv6opE8V+IBQzf5TfBguVR+WzkG/+7IwS88BpCrUjUM4K/u7cTKVIGW9Hv3CiU88F8k6WF52/eX +LkZnTQ1Skel4JxVh4CQds/1k8R9FfOwqxuWdBfjG9RTxi84CtOWGGcJK91YG6vC3VBNDDOICdzHx +SapALgIyugSouJwL3CuwJ1WAzs9N2B4m5+JxdyHP8YtXEQfj9RFj4rggVYP33PWZxVmbqiEecEuy +iCqq01O6t5cR67nXQQP7Bkh/FQMnE3/rMGO70rNCAWv+i8RvpKMf8n6TfqUGIdnSpGtVQhH9xGJN +/JK+jItNT2/ViMlZkRBt9IyQJqqBWNrEgCiQ8jVSeRgbiUtQr2ytxN72dxjSk/Oww6rBD+0RBqxG +L87DAvO1gJdxETuQysmog3be5wD2pMtGaZMbkphX2tsRRlbrZmSfUQsp6VoDFY+Ms/vUETDEoEJx +vYzSc/RX6opozaETbOt6wOntT+ptUyolmdjE5qfMHdS1fIsJdikpNxN8SR4U85jgBDYb7frNcnsy +gWw//iWBvvlknQ3ml77anxzNl+shp/OVytoMd4/uK1vNAC/E8fX04T33VkrJK7WcPITqjmHAdDtS +au+1qQi+cOczhO5UEb1H1inBj1dLPVmf3mLU/uCVY9Fk1lJP9eP9dfqLXLi8fxV3+dBs/f7dbDOL +nhX2n/Xj5d9zXr98+scwJKeOH5Z/oG3ykhm1GZJog86/dvubrUc1I/4Pn+UFrsomNtDUavgPtey3 +7TrEg519fOpdO50pM9mPrwaTbKaSUJcxylUM8E6GhJ/SKuVzw0hGPYOQMnK6SVppCkziXc+Qb0VK +FFApc8HNTGDptonol6R+yiiDSNQEhEplITyXQ806Bk7bYjWin/B5eUSfdNnL8sgNW4Tn/XSGR2k8 +YNqalDuvEci3avatyhuVamUDUg9zLRZIy+mYA/MeS49wGi7uL/y/1kjubREec7a89w25ClgtRFer +8nFPI1b0X8+ovMesFnybbBI71b942ijKU15KZrnMjRkbPI4hPk5fG2/EcueigpLHne7u9WYb5i/B +0xnw5NgMUlemfLpuZC6zanALeqzpeKO9SPhwM5RtOmBNJ15vVyRlJ8tEwXR94no0NvU1v+U9ke5q +Onwa2EY2WiOxKNeajgOlW5iqqAvLh8gwyzEMTke2BK9ck3HM+GR86R6m3i5KFVk+nDAs7cs2HcpN +wkTM8hm1RFR17ulsRYujmNvZ231Y0odMhAxwGaNQJDDi1kgkcqzpOFrASGj0C46alWpKGi7k9fZW +oDVfhWLHw2k69rTXCEcSjP4so3Xq+V16xLV4WlcUbGJ0hPGP8mr/BhRI0pSJmsh70qmri6ObNwmC +HnzmGiYmWhz8KOKOzgK86CokLurcQDkK5a8nzp1jkBTvKt9QT2dNXR3xjqsaV3cWYberEPd3NuF3 +rkJ6cX1nEaN3GEeLBxjFW656E+LypiuOBzsjmFtwivGyFAGS+PgH145MObFFF9EcVIxT4vTits4C +RjU7Gzq3WbX4m0tThjdcexmdSdwiRklh34/FWCuG7M9uqehvtGrta2jwtGsKHuiM1BBPufIZHU/c +05nL0QXE065XJOUpRza2dn4gKdvywQkgrsVnLm3jRka/kktuE/GEvJN8nERzIn+syTm/81V7GY61 +cbjuGWdqr3Rr6YOM4rhiTLsHPk3LlAL5bL0KzwKMjWOaHeQU0vA+S0WsWlzqjjOKgq3EtG1URFRU +EbOaIs61fNjRKflZ8mItcbE7ydl+w9qaGnIPDuGcUmJINQaLOMCkJZnovIOL3EklKjs6KUd05daL +Evu/ouRKg4epycfnrMT8XMrIpzvpfEKEMwowpvA6YnYu/RzDUTzKz6lHM0/pL3G2k6616zln5O+Z +kXNk+DhuGceTRwSZYohSq65mICLBjCFZ2YM4oXAMThXjYK6dxQhjm+VIhlAhhbsR4HQo6Uda4ox6 +2nsQQ9xjfjR0+UfLyBoyvpeyxfrS32W71mdbHgjpRfsFPUzIQdNXV1c3zWSoRFaxLgCImA/KlqEH +OhvgKT5ZgkuIk6h8aHZXZ/T5rDSKofAMRSezWIDOFJvyt1STcfIoowm3LtetXuVWLXrcEg11xvvC +Y1immcHpEYTip3CVyXUa4ijzEzUNyjc/X4Q1wQTu6yz1FNq4lveSM+MPiiPfxJChaxhYiEXOpJ5p +uHtNaVvueIyMG+ttdAojjD6I4XHU6RJleAqpK5RCMnYvjhh/aRnyfOISNelmwo3eTspkTZ5+5On6 +k4czdfuwplo5YeXeqJpTJeb6S4hFuYRJT6VZKkGj0SkdvV5zWac3D2MZyiqI45XCSnUaJclVMYD5 +TchKAnFivqJotrCMlzPAQYxyuFgUcScG0WwIa0skU5lsGgKiliHn8kXtK5y2MuMiWPeqM48rdBO1 +LmwZpwwtvnD4eA7neEZYrB9Nht24jQEZ5NIAVTaSIVMeksohzKh5oFxiPobydGhS7po+YsBe2TGj +WFiagaHn9w4M1/y/BcOu19Iw1CHLtlvM5OwTeZhVx+sYtWbSexdreRJzrDr+0n5wGWtZTHTq0fn2 +owfMWhQTKT1cpYfetYRHQRh60qgn9F6qahKy2vXwAvvhs3poFRNteniN/XCnHsrnY7keXmQ/XKOH +6rpVDxfYD9FpHvcvpqL56mjkZsGNlClQ/tllqg9P5o0OUzCgmFhqCiTxmjfaTYG8aJaYAivzRpsp +2K+YaDEFSvdt3lj+qmDav5hoNgUOVK3mjYHFxGJT0D/TVNIUHFxMLDAFXqU9nkkvlpmCQ4uJeaZA +aWlNwVLlSUR2saKLNcCsremRL7lHnecU655oFeyXKWgxUB1WTDSYgv23MkrvbizYxmivGWPXH52l +lwmMG62ZxCJtMyyvsOpQkD+Yw/Ts7AIbWIzttiqxFlutmXigfQ7Rb4FVh2OqGVatCQX6ki81G/Kt +OnzdXsEhpqDGqkNxsTUTs3OIegNRZIU1EzNziDnm6/HVOsYvQr7dsgZTtMCqxBrz5M32ClMPx1cr +52D6nrVdb6fBn5WFv7mUc9A/slr5znKNc2sroDyDzWMogaI8zMQlYaXDf8Yjs6/07NGYNQsPdJgB +Z69gAHNrqNkMEPM2Y5A5S89+GLmyq6DuO/p4OUPssmaj2z1MUlwVyygr4RepglydHRjwtpSfGLhi +AhMa0FOeaiaUTOi+jjnEgggOXsHx1mziF+4kmzNKlp4/ZQaCnSLp19jXvWWpj9vtvDqbrAW6+LCt +lCE1t2QDzc1cM/FCR8Sqw8HF1sxB9hoRh1ZbM4XYs3CTR4r6Mvm/4RK3FuA/rtos3O3Rsq1x77ts +Ws//smyzcJ9HJpWQCUGqR3NThq7t/LMzoG5kM+Bx22fXLud5Fq5CtlQwtzNqNMde2c+UvFuxZjoB +JmoZuGKp0nJaM/FOe5PGGc+McxZxKapjsmiFJH82mFv/d/sZ/bsEvmJZWA0fKqtnSIxiVF+1LdG/ +mvhLv3zi6uU1+Es/ebuvFiOY/sHG5RXS4GalgX43M5gsvIpsPs+5PKHFJEPTxdq6b0ua3HUC2vsA +8WC/YcbEqu+PpxXDimO4mVHOYMwcrHekD3w8sLyGUd3KFCX4GRt05AaEpXMKrFkYEie2/u/N3cG6 +SoaJZ5evZJTLTLrwqA2Q1FhTxSHF6Nyk2fNeZkC4zMrOunGPhjCXgZ9xInF0PXGm8rBsZOCPm+m/ +iwHvaHvKWmrMV3YyILLjiZ/JaRRf5TNUqqu9iLVal/OsYgYw/DNt/YfairRs9X2W7TpL+PlNW0R1 +b7LqrZl4o+0DfbnI6pYeawox0NzW1LaNCTXycdsGu+UrLOHwx20VVt00ayaH6iX72fN6hn36uduq +nrrubq6CuiDeaEuDd70la/Js4uUOOXJ466xZuNVT6CdmbCP81eY4h6caZxQRY6pZRplaN0kOvr6j +SF2+7VkxlQmU56KoULbVsggRSRKdESgbbUeNhCnPMV8x6v3amonr2+YQh9frzZ1WvUjb/W0rrZmD +JJcncHjcmoWd1gJ6puqSY68w8RNGMWgFDd0dlI/6GgzJJ85NT8c1bRusWfi7VT/YqlPGGmdu8Q8z +tZe2zbH0Of6ZImU+oaJjMa0CzduISdvU/6XtRXb/Vl0vtSC+QzWr8HfXdlu7cGXb5tPTiZGPsmbh +r1Z+UBL5lW25P9NY/mYWY0Xbmn0WeB/68y9L9Kc2ZoihF2+7u61ZuD5VytD2aWfwJGlR5Yyw0uyS +qJUkPnKtEMNrzcLGzoh6625bo872WPXltkfPhbn1NkndU1fBMnObddTcAryOq3gZbnAPw65Uk7kR +GFO3xehdzOimMYb707Jcxuj1SK2xZmNzR4TNmij3sIyxZtcnzs74nS4W2WRNxhPSjAuEB5e/Qx8n +lxuppZYvC3omrMlEt62akl1HvsRR1f6XTOP0DJfxohkdutP2ovYcSY2ajVnEhdZetaQNajhIazLe +1309LLMreHGVNcyajI/Nw8xbl1t7rcl4d59nV9gtMUHveb1ArTBANRugjtxrA/WDDdSRAkrJ9KzJ +OK8PUDjP2ksvDlCUgNj4qOH0MWODwLYm43f94vbU37e8SON+sl+c3st1zu1qy9GDHmuKfT5t182/ +xLP94hrq9uUrT5HKypMO1nR9npnmLHzVL3vTXJFwzasCNOw4TZ0C+tGB6xMllYL1MhMha4coX26v +8IImOsdo3j8zzeIKSv+zSZTwtLS30yT60L+HJZIsEuivkYjp+kqZ+QXkapOf2S/ruq5sVH/r5RFM +bOpYYw/7Y8+CRpzRpCqHOX4RX2Q6zcJzzAZ7pCvJnB1dTilW9c/OKuNGIyF4hmgFmk/N+JO4/pVp +BJf8l2pdvdWu/C/VXF9mWsvCDboQRJmSZbNMsFv2NuIfycx1VbOItf3Xc9U6XhBjs5FWEuMd8Hf2 +aWln/+xNDChZHpqL6DP2ky7hHLKVHeacUuwfJ/ZLskwk++lkkzULD/aXyLtOwo87qRwjXim/dRO0 +Is2U2FchUsTcJlWZt8GwEtF2FvqrFDahS783ptcbC5v6nGBf9Y7weq/h/PzWafhkeY4AknJ0FnF5 +v/VLrNPw0T4PN/Rbn1mbkq97G7nTm72p0SrDh8tyDQJ78Svvdus0vGO/HLC3z/X91ltlxPvLmrjW +YWmcVnC/VwmhYrGzM5FXed+Md+mfNQtPeaUWSitfwnJZN+MK0XuSjv5nXMUMGXWiPskmttrQxBBv +Zogn/JhruLVTpjZp5ZTXe4UucM+ymZWub50u37G7DG0yJhcMkyOkV9gv2bQLNaVWGR52xvuI96lR +1mm4dZ8Bv2oP+IFlWszd3vW8H7U5ma56vst0lYW/eQ0zh52uBTi1CKFhuhWfimoxPyitIFpz2cAH +GOVajbjPOvy7t537B2RvYgN3W6dhw3JxQB/1k+JPb6zL9LszU382HpNF5Wrq6vQou2yOPGoUAGhv +og9eedIoFXzgBIonkVtLPm8xdaJpJfy8tM7LK21IlK/L/VusZ/98BjiSUZwZ0bUe6Mwlpq8kjBeR +WEWlq5N3gdIfNaMtJ+O2XvJDejznQJH3Am+k0vHE0VRAefJ55cI7hYFieAqxf6Es7DJuRI33Y0qu +Bb4Yo4s53pqLmgLrHMJfjIPqpbwbLboUY/SqiRqbcKhZARWY0YQlwv0ErCl8nYEMI96z14Hlec3V +Jts/o4DaXl6JxqJzUbkblPsRM00QxXG+hQn6skrTdGxyuCc/jV6uCUJol3UONkndx42w5EoaMPkw +4YlLpzxLMpjifAuREoeFwdsZssYTj6R0kpQzYc0l/uTJt59e11HKhHUOvXjY/a01F1e685lQ9Rs6 +lB9GBb91v22NxxNqTdPxgHsKA6rxVCrXmounPcNMA/giVcFmHF9dSMQLdGhMsqvdl1Io1U/7zTT/ +hPsVay4u+2m/j5l+H1a/nNS3p2/UE3/OAMMnMoCiYrQVwCokytcgJG99nFowmSgoJqoLdMwHbUi2 +/S8z8IqZgfN/CslLBpK7fgqJLohrpvIrHlkYBNajqRQHFRJnFRCnRfC6Ky4u8lTFZlvjsSWlkOl1 +LPvRChSxzEz0m+6nrLnY68pnmeZXK2AXvO7ebo3HNtP/hPRMlFnnYE1n6Ri5GEzHIQuwXyGa1qB/ +oZmJpggD2G/ryRPixockKuw+Wf71OuQnKBqc6E7VaD2/cBeiXz2WN5k0ux3Cb4ltbxkEN7HgWCiF ++yrBtD4Vee1T73DvOO9w70jvcOscL152V2vzYmoNA6xkgEniyHxFM6KmiFicixMKoTvNjQ/0Kiwv +NbGb9Jb1+gPX9U9jdgNxnTt7tGwi1knEP7JkWWi0GnDxkjms1b7V44/N4zqrAevM49GZ82Cn0465 +PLlZvojelvEM5IYZl2E7ZtwndDtnGeczuom6pClA/DNVpL4vdsucLLcunQLlsh3ZN/tKMau9HjC0 +LcGodRL+6Y5bDfhlqkgNfJ0qUAOr5X/BoGxxASaQzKUOZm1z9IujPXKKcbZIJ3a3zcdyPOSNR3gO +tTd53YDMVOAv2uTWSbgVPUyo9Vfbc62TsA3DcHVnDpvxF9crVgO9+H37NgZMyQJVe90wn8rk2dGE +fsUne3FQoRXAU6mISm9wxwXwH/UNN7gLMzc2J4zTcUCnVJQzBeN4mRbRtJLSnDZr4rcizrVWA15p +z2EDQxlDmGs/B+YLpUIxYddo2wwrPv54GVpW6K5muapmF1sNeN94QoUExV2CgnjBXcjoGB7D6F3o +KJALlbSECnyR+/DvUxlrSgP+7ZaBSI7TrDNe5WuVRbSBgzKw5O3vwNIjWK6lbxusODqLpDavlWJA +EP2jY7MmY62n3iSlJLpSFQIoO1+Pdxk8iC1h9C6zPFGBVGL58EgfUC7xJI252UCSbOLanEwi+LqB +Dgx3CwZunK2EjqtXWz7c2qeFX6mFkTxFNIoBy0cv9pi7Z+fmWA243hNn1HsQZRZul1IgxqXySQ1g +jT1r/zRQoqMiM/a6A5x+n1W/1zJs9PWCPmFycB5ezCjOzcXgYiInTh/uTq2R70++ORZ0JjQYz/oH +zGWxUV4pPMBbqYjMdoque86dz5YJlg9b0oDKimA14AsBKxS5aUAaWd9YqixBJ+GmAUoc8sbSCMfK +1M0my4d3zcuyqTXgn+aG3fGZ3Tla12RZJ2GsMl9MK5JfUfPTgkJXJhJnN0mOPJ5RSi+4lYEkOouE +WQ90PG1Hrzbgzx6lV4xZI3Gz8Q4NWQ3EKsRRa2p+2hGxGnCBJz9MnxXACx3vqPwuT6GJzpgpmfgy +Bsq1YzqKrAbc6ymUvgkj6lFbhPaIVN6w7xE4WYD1CZorOdhZgNtlUtiktK+y5czTODwLtMk4hQnM +yx2P7KTm6wm3hKEG4vmUNved7t7NXW029wupDQxwvnlVziBPd+QSt3s0fLkvYFqudQze6Mihz2zN +bZ44xiS1I/B4R07waIask/AOtAbydJIF/eF27SfhGPEWuinvxDeUht88N6tCvIbuXExcIxCPUABF +g4IypcdWfmWz0reL/hEPptb4j2ZCZiUMNjWCvavyTruzKg+ZWzijAmBj7/KroyQxJxKmz4u6UgyL +xygqrQvHAvik3dCstXrXCuDv+oq1KGSqjrVS+XSuIRpbtQz2WnQ65p26bGctfmdJIN2Iw7cTDZFg +OMwA3PHHbdZc6TWtMcRnbVrsS6xiJfA8XBqfJuUsgUcsll88Z501F3ebWs/LYJLMleVueYQ4JJ+B +n2F5RPXH0ySj+7NLh8PmzgJdI/kd3FPEjSn/cABNm4nsQo605npxfVupyKkVN4G9Oic8Z0o6jeKc +Uk+xy+12DfB0HeYM5CtpDL+S4w8DPIkh9E+iswatNYT81jF8h66aes/w/QFJOl4MXUGc28RaDNhB +dHfmHEslQwmxnPjSlc8aRuUCi9WdG/BvV30dFxtTYoCoUaJ/LN8mJvQFaaVQv81Yfb0PVIXFtiv9 +gHAbW/pNkaHZT4n7f1pegBuk3n1heYGRSKO6dMgTZ7yO+EeqCHtd9dN16ZxsijgoHkdjgTLilNCs +wuoBQuxaods7SyNY0RmxX5A7KQ7qIT6VQ/g6Y831M4DTX0VhIVfjg1TkWAbwuque+HVnAUP4wTVF +4XsFeNyVn+HIdw5ypvJdqZiu1bZ81vhwyh8T25VLzqBFOZ5I5Sgo0Ty/112Nx/t8/427Go/1+X6f +uxqPpnIY4pEM2K/c767WaUFdwpl+9IC7GjtNrYkMGC10Ix7Wnd9MxLDY0d+7hjhAbpAmZRMTp8Gj +nKG59DEpuQHLigxL4meohKAC+EcSn7QaJubDfnGdq/jUHTdpu5dFdL3qGYZSS0OlI4i9YFVhhSEm +uulAyxlmILZCCya5JMHQSezT13j61NOfWk1PWxiXcs1H3LFcNPUPxoMSVY5HfddQZyjPUG7ShvUy +fgnoLMVR9TirhssY4gTiGFv3XUtV2s0AlrYSA7ezVlcxzBH5e82lEAyvrnVY37nPcE5iLR/XnWR4 +xiPJcYd4BryVymEtz2ItV6uVW9zVeEGLlv7+sqd6Hb5O5WC7qac2TL1XPdWGT6jleYxytQJZxMCz +lmeylt6NkL8Ia2OYqv0QxY3GKU72Dl03vwW7XfmzGWAHo7ObiBrjOGd+nZpBQ1eeMy239de0jJN+ ++5XlZlJv7VfI2kXm/kVYb2vcBynTpfcNM2+aO/unXrg7YL3fqsD25RKmxAk+269e3Kuk/xCjV2B+ +jdlqAc7TzdBiCdtLJ5hjOERlBPHKFuKdzqjnAc83qKtBbtykr60rJUZW+1nLm1nLvzLBUVWcgLx6 +2sl+/JzIZk6kTPxKVRyicg1ElYbzpg6RtHc8hcaRQjnoogxq9upYa+5eETcSEmuqeMP+0thF1ZD5 +1SygaqWdQdM2c+FDlJ4jv2EtPYsNXVR6bc8oLPmOY7QD0H89kjWESUacMEeAVYnPUhGWiYSsNp5J +nnl71MAJpgEs0Vo0YIoMj943WCsyKb8owb4itdlqwA9K+nWtQSivpkCT5J1mvB48ez3fspYp1kpf +fn9KtmGxP0/qFBepi6xj7U2noilyBT3Xu02/uXvs2i+lZMQMWQ24I13b9oF0HEZ3Hu2gxRdp9ZHU +uytTTVYD9rqnMDSaXix5VaewVIE+uKfQexYTGDzFuJgkNIb72musBjyFQvo4ScYtYeUxwiNzkekL +hjXwDIuhuQlLm8TaZ/w2NTFRKUMSjJ7q8JQBB6hb7cthRxHefCX9WCY2wrALWDsgaUx8Idm8RBC+ +X/pOhh3bNECJjN/QOTJggTUSv1+aw1Wqky2m5LIB+QyxzLwZoFfelD5c36fpWwd0C2Q5KBjGQMxB +wLCLsvlMOIUJBtBSIWxgeZDR42McX8e0CMOTf6HtimRuRoHlGuuM56UBcrnwdlB6ucHmphWPKMyb +DJhsMOqpQ1N6ufhtnOepx8IaHJ+Pn0cQqJeETxQVEpUbhP9yrJAT+xRrIS7UC8Re93prIbpSkRMl +8W9KFqmx9/p3M+q35uKqZJO+v9+/nlGrwYtLPK9gSc04UdS5uHLpGmshdvWv15vXJmushfhQThbt +uVSmvwbsMUfgUwwhVWGYFB+3Wgvxm5SiSsYQVyRbhWvEfQPkYtVRai3Etx25VgOe8CwgKtfI8WgH +0SyH+4nBMOEvxFlF1kL81XC9mzyF1kL8Rbwy0ePZYS3Enzt+NJAVPxqIksY0eLHNswPJGmsuvlwi +lqo7K45BccyswbB6a6FM/v9xTAxox24Qu/Uoleloq81J/Sql0Je+I3pdIyqxFuI1jYe40yMEy2cC +WdXEkgLtAy+eFuh4wlNNLItYC/GkvhNPeIo167/tKBAUzpo4Q6mwGrA1K646OzvW2CvzsGe7NRev +LdHGSpc92vGwNRcbkjnWQvytv6Ta2z1St/uD1lz8eYnkt81ZsoPD0hHmxbWd8grx4lGXUgcZTu9a +OxuoyEDAfnKeedKMBiX7MHVEQ0RY5FlvP2m3ydgMxwtw1wkORj+dJWd/JrhUHtzNp2Z04SWhdJVG +2LfCUSeUV4aYAKWqCVXNoBdPuLYyRrkJ4r7OVgaCJXJIwcG6wksODmJ1V3B0Ey82l8aGiI6nFYPV +bWh6U/i0IMdIB1kZZOD8lsUx75jZp5yy5PRTGB3OcV+8iFQrFtXg4OTRxMKCcQwMx4FTxip+Tja4 +iUSbtHUBHBCXEqBfIRYUUbtX41/uuODtGueM5TjbqCMTq8f3g8Mvn+hUKDZ8vzK9ROkp/8GEVTiu +2D0nOfWekXqWr4ujJHQ1LAbmG8E/wFkmU9GEGE/Rha1RYvFmBnhezKRFXbxZSorODRkNbU+x0+TX +tkdwgI1+Bs4q5UjO1Mm67VKuPZyKJMu2ufuSiPNKl0fj2cQSlnELfUrUqfzKjNqhze3v9M24vct5 +j3jDk22IlVif8Qy8TCxX2KwuiJOkgbmijn6ptCaxmbMp7l+mAanYfeaaAZ9CFzX/OFQ5ZOZtIw4r +VHoJ+ljIAGYXILeQcyAx0wMb8p0nZyDPwiXyJwngsHz+BeeKTcUwHQiNjM6hjmyvVlgac2+MUc9A +u4GuU9INzM3CidlX29ax6Rnj2FzC55iLdjlV8ZUrG255Zk86lT6l2tzE8l4nl7ySdJsL8YmRwJmw +5bwyo2RJjOBEZew1yWa8fiY8R/5QJe9Vj//fE0zIorwrFBCgXDJyPA3IHhCg58hvLeByY6/1S7dn +1IHmwP0IhSZmfSPmOuaZupgDxl2S2YhDviWaaoK6ymo0+n3LhAysAfni06skwrsZ3Tb+JqWFuMVs +Tt3k/DGjYiPpucNwEl4da947jX+yDChbZW/zM8Hf6nR3v0+ktl0wnbrR2VvHWu8R63gDo/xUzE+e +0l97TrZph8138BS0VdAz8QdGeb+ZnZB3V4yeU77VjtO9yF505syYQXFYIXpGqwmvWJqXDAgyv0QE +ujJtpx7ui5x5p2aGn4UX+mVvIs5V7kR7PgUxDldOhymGwwvwRIa03PcsL7IW4ql+1UzwRBEmk7sp +NU6mJSyeQx8GdRtK+Q/6KPclcZAh7SpeQt8FaK+pITpqRqj+UEmo0ikxUoO2Gt6eURHvOi0DGi6n +zv8veYc2gRFd/IrsDmDgCo1ec3x758P40FUoZ7YQZn2AJ1zbMW0bfHEsriGatK/Gcxr/QczINVvG +x/mMcqdYGTPXks5wyBTML8AhKyTyarZaJjGK2WsI3wqNe/vyUh03z/Urtubi/uU5fRO0quDCfvIJ +wPH19jQ9ZKbp8X75kta8J0nXLPEwRKX8SmBHZykecSUVK4Ujt7LWsNOey9xfSXiR8bOO0/m1DpQo ++j3FXzPqGfcdvxRcyrQWFFOppVXYXbUGudkQYUWmhEwsacDYPTcxikOewpwCHa+eifaOdk1xJvcr +ia7cyBbp/pWyIsS76VMMze/btcV3WwshdXJgDWuH0juoz0VPvY18R1HEjS3ag9I5lBkxa7b561vM +QkYfZgfacjzD7P7rKpz+1/U3r3I1OgpG9XZ+v9P5E4qAtztv6LVE5P3caWG9ic/ZyJagYoTfoAl0 +u739HkYN8C/qfV6urdexmYlfl7BZcW6jbEjyznDauT7dzpK07cGrnW8nHxvPxL0sluRt3/d6b8bP +vKTSef2u9EB0yt3gGWk339Nbfm+6+Tu0gnWMnsZxRjEY9ax0f8WQblhIO6+7qpxGH06/1MKwyGCA +MU6n91A5dEenHz/36PFGxept8TMk3XPt4ZkZdk112njNBkw5wuWHJsf5JmEoHkoWCG0f618PT35K +AJyYhrr35Q9+8vL8lT9+2dyXqLiv3njuMzO9Z+F7OVdQdy610Jd7LKUXfogh8xcDpwiellIGjPjj +HU95zkXtwn5xbfh28Vh+juGnjC4isupZNNxQlyqxMFsYQmvuGB5NMB9LaxZR3qOGI9lPNqIqKZ8W +F2FgoaQHLModU86jKNZlblMf/4Sz0vA243Ip5LlRLJeivRImT0iJ1Wx7SM6SAqAjx2omTGrM3QyV +BEfEPEN/qEqV0KtMPa+LGTBoo5RpdQqOSVVop6Ld0Xl21Tj93WL3x5g5rgJEvszxmxlATSsxZgED +cvD6yJVvNePfqSJ9WekexnKBYJIu5EiAxP57rRp87xqWjoS6k7WqcElnhFFMWUOMTaJCQZ81lOnY +asZHdlvr92nLY9ryfmbV4BK32pJ94U27rV90apVGyda/hhid1PJgUoQ4NqmDykchoI9Y1hSkz6rB +XtcOaxqeSRUJkMI4J1g1OM8d17fv1TdWmB4Ub3uB1YzvUjX08TKhw9x0C2vddgsRvZNnt3CZ3cLH +qYhVg8v3aeGj3hYWO4lu82Y4M/2pZlp+Hl8yxLemMmF0uWISvG8yQc/QPUx4c1goKZTIkor8S7OW +PuJFK4672nQoBOjHc5Yy7qymTw6hxF1trbZxqbVpRpsOd602S1h7RGY35tU5UFwA4+Pi45cM8L1e +KKImu6wBYmUGBq9soY8R91px4tm2CLabjrWTlVsVT2c6TjUtJtqlXm9gCRNH9E0Ps3Om0/e16luU +0lC4AI6bUqK8UakIUpESnNZkH64BOw6fOEQuoDU8zWqGFoxwD2OV1kJu+ecW0MdvrRp4NBcKd79A +eILqXGJo74uLDMYe0OfFGZkX99eLiv1LI1hnRF3XldoYP1hIeq55+/A+b/9cb3uPtGowSK+r3/Tr +bRn07LceB8exMFISi+GMCJG7QsLJxH0fjlQi9qN198KPavu2Kxl6gH+hTw7++p1TrQgdH0Mo6Caq +ChhAQZyYWqCDduRek+AHZxaJ/8H++SdSbO8l48xJDM8UvI1huKA9oiQL03BvqkIz+IxbumAf8Wz7 +SmukbDzNeEHGE9vwcoFjeGnGkxCH/sl4qwZNClG178huxgH1GKIStDbp+iqd/oosiWC/Yu2NnwtZ +E9RSX6Aeh9VbNZhZYDVjmOJga/UpL59goXbAIE6MEK9AFNfk1xBjhCdtD0qjrjvVM8U+HHae7eDT +C8ZeswlWUhyqjziwUBIjA3wDnrjaCWgDyVG3NYIDt8pajP3yp2JZruQap4LXz6j8/g/cyvOge8Eb +miiFR4AYsp4+NJZK7KEXQ/MZwDmlxP4iVN5GYl6u7kBTzcW5moZcKs5fdQ8RM2aiE9qUZOXXDFDX +K+dKM3oz5IbXtI3ed+nImXkNzsB+KaMD3D1KmytBt9rwmb6bFaSkVCbNfNxcDxE4J2ZU9wvNNRSH +ildeyxCaSo3Wc7eOqfQEGDHkMkbBYszdTMX8nEJwu87tUsoXJKB8sSFKOWFe8vzsK0YXYe4cYv96 +HUcKe9axibMjyE5yq4lKk6XYx2JihnkI7w6JcesYRV2RCdrlVmG/Dv4Ay+COo8qoEnkKO7Aoh30l +3HnO+J/T+LmppZI+RoMlXEvZQQJysI2a1eKZ6GjK8Ct5850X/2ZexMAe5W8wtNVHb4qBcylTCkZp +RmWxCqHfVnmxqxgj6hnArFIeq28HiQUIadaEFWba1rEW81qJEeu1zq2l0iPKYcPgwEFyvxUnII8v ++5fwTeGjI5IwonY+ozg9Qh1CPl3UyCimPkzkC9nzbQM6JkeIY+K59j2o7noKvoC8jE06evV7UFIs +ZINmr5mnMOGwWruanOHfYjPTujgCiyNGdx8yEpCAnBvWbkNrgdwitCDNFMnwymrhHTweq92KsXjB +PlXwlZJ0xBh9gQFzDohY/Gq5iIU0uc24nzqgvtJNcK3S787NsZrxDJOYYkd7D7J3a9ciB7iPbeA4 +gT5dke29yYSORhnAsStQE0FbJMJG8Wl+4eJqBtigi2MxKYIBcSyNLJtoyF5VJIis+FQGuFtWWrsM +3iT6x3HECuKcVuKIQmJejT0FWhuvTvkG5Tf+hFEpGjCi2GkXx24lZplwX+RpTcv8jLIMMz5XfhJ3 +tSQaGUtWcxBnmolLePGzAoSqGcNQ3UMsltIciHNa2SANWEcOUjWzqJDiEDG0hwF2MYTKUmJoIQNe +NBURg5IqXJDLkK7sw0FiF72DpCg6TCVTtNFiWBqBN462iHxT+sd1H+PYONHaKuYwiqY1/1ONha1S +YEwizv0fG1kmtqGAmPc/1mhuVSnaa1iLJTlKV5DxDs9b6qzr3yQjWD48ntxgNWNnf4XHb0lWWM14 +Sw7c/NLoSQJGJkwwJnyqJtpLqxiAZwWVxiSfiywfPk4KkWz0el/ab6JhM6O9QpdrmdPppV6jsotN +MNlg/Sy5HB6pA7VyCtrVLZnjJ+gCI6G3sN1WZD0k3DrTyKbSMsiIKw3KwuXL0dHKjhy+zlCGa6pL +Ot1tSXc3kREG7e4QR/u+3eGQOBZFIoyGDQoHDLZMMhxxyIt++UChEvlDkTf0s4wtDHlPGsujGcg/ +Bu0F+ceYwQjXvO/1BXE8Fy5H8oN2dLa2o7WVHeh0Mjd0LXeAfE2uzdLqNh+RUejuzJQuw+/EfG1i +LXfTt0S7+T3jxjE3x1qGe9AtHdFBlg/XOUf/MtyGbvq4ZLCEjrtTOROsZcQz7gWWD1f3qXU3lCkU +fa5KakuDtAx/sDv1sSrFwHAqz9wr6W6pHLrLaMLqcZenHu5hsv75lVqOCY5Fa4Xlw6V9OtqJbnQ2 +Oax1SbvTzV51w43m7F7wAQzaCajxPGGZkcUO1rHmbYg1orWpCkuaPCfZ5Kmkw2njKsNMbET/9cSh +9ayVUIFzasYyhDUdulBQjj/cXUVc2JGjlCn42lNYYp8SYZ7AY2xjMPabIj1YgCHOMqQHdxrj4iZt +9hdTFQxxpJwX84ldqQpc4x5GG13axwkRRHxuZoKbgvhjqoi1uM5dzLqw+VLGcmwy9ScygeeUruLA +Qlyl4Go/8b5cEJfm4iBdzy7avxYbZIEOEx+lxO8EzV1v0uWGOBIpHSRRXOEW85Lpu9EEC7SEib+k +io6NiEXY5I5fZ87fxfvA1xjEn+VhcY07frEpPt8uTrR0XXy5/cQ0uooJf6bR44m3BPSN7vjF6Lee +G3FfKsczy16Jnec5K/GMWYkeNrGKOo3HM3DaMUcdcxSV9iZA7bxU0zHi1/MYAuP6xlNOOIaho0ax +Ra5Um6T1qUVLjiftG7NrhdP6n63sLCUw+Ty1wVoGBVzWEJ+ncqjbDpYpdYOknpiRIT9L3aNHa91C +3GfaRJ7MfnnREnYh9UFmn9WtdNr/SO3LodRv+bDSILtI2jK8YHVbNfjYOPfqPLrAWoZ17npIG8JB +nv3sedjV29Lnmgft58EMfDHOeOKkcomLDDT3O9AQ71rdEd3Lj4VFDODu9lzieRTiH6kc7aigs6O6 +3Yq8vaM9QrwMHTdzZbSQ+XIycVN7JEa8gfygTBm75X6P112KEdgt1L25s4IhPOtSLF0ggwyJRl7D +BNcFcXtnkbJrDjOUrxdVupRRLkisV/Fh1ZqSPxvAtxvHjEHWMvzTUlqNp13Jey83WIFrOtdIgMWS +0kz4Tt35zuyuspkIwRxYwesZ4k2MmuuLZ9N4jl69PJeD5Hsyt8hahn+byISPTca/KIPSjek8KNfo +dFA3SjOCw6YwitYiKsnIH5drle0V+7KfDmKkmrRGPkWxhktYBjH5iFelM8eUxSwfnu3z1i+ZZBSz +1nDtMPH3vsyRWbI6M4os3EzFw0yQIklaGp/RVIaI6g1MYFSxvEDF+Rl3TmV9UgDaWQygsq/SyGkP +90v/KdWCd4ysJ5nsXiUXZHrEzTqkN1lziQfbioTTT1hKgab0CJ8w8TNrLh5qi+j501oOJQORqoVY +VoABhcTSHCKrcCaDLDEHU+gEzuQiKng/wahxCHCUS3UXOr0+I+3dJjbQ+8mizFbZ2Vv8goCyarCi +XVuxyzJ3q3W1O1txL1YwwZiOn6eW5wi6+/oplnmj2PPV5kUdI5LhtZm+Qz3anOOh6yIHjH8IjGso +pwMf5zHBuoUCW06A/dO7rrfy96osai37ZiNr86xp2LjczM2HBptWcTybp2ptbuxd3K5up7dV4hUy +DcxjrTUN3ctzBf2/9L5n2FeMVsGmv/wZB2U8DkrWOE1cpSaMoiTFJ7iKXiGuZ+gepBzvr7yLndrX +qbbdYcjOjvRFa4X6WyObHFvYQE+NrAx+rcSbQlwBIb+tZnESWfYcdF3itLjd5iTCVDDUqphzteuu +3io77Sq+URGjR96oSTEJQzubPIPtBusudRp80gGxSipNwXG14PB+KoV2X61+yVrnpVfsLqJ8YFFm +nbp6S/9gNxlA/+1MaX8Izzwn7EGzBBBn19Wtc9r7SG9YNfhgudBtYz+Dbn9d7qDbVf0cdLu11aDb +q9wH3T4wcVcZdFvfry+65V3m9PO1+hHdnqpxypo5Uh8S4yWzpid7Z6Z6EtnGuoTD6onGiCrKAOtj +1M4lsYYBpsRIjGw8vZGr5DoKE80TsxZiY7+4lcRflxdMZlShnS974jQRurd3zLEm4/eefGsuLu7I +kRPSEcXEh1a9qMyVbQWsxftWN3FVW4XErcckS2yTwK6LO2MSru5Ci3w6PXYUJFpKPenwol1XpEea +VCLAx/YwsMKqoRezckwg8lFWEkflH6tTtaLISiKYr5NV/vQcOzosZ1RP/mlj8LinGw92FNC7qa5e +GbcZwJQ1yN9eB3c9drvj+KW7HpXb8GhHDR51b0dPqoY4spiBfKZYy4tYay0kfmvFrSQeaytAvyms +5UzW0nsWa+kplOtXLVoi2K+acs7yjNjD2o86h/PYUfTxVrP/3/PoGpk7lAVPHn11q7F0JUO4raMI +h9TLFHenewr+miqg/OMCeM4zBW8JjPdSBXqHZacR2K5osRXjZDUbTTQU4XNPHA91RqRllrUAf08V +SZL9uzuftVib+o4TVFf56dBfPk2j0o56u3kib+EEommb3CpDJkef0rq0M8bVDOE89/t4IRXB5556 +3JOK4Jh6vJuqAafgr+44cX4qQszNRf96uRwfWY81nu34Z8c2eLYbH4Vaeq9hCKjH6DgecdXjC1cc +f+jobe9fqRr4t+MF9xQU1XtRXkPcmaoRt6j7bS4z/C8+7WhV0BYGfjtZublW4jVPPc7vfJVY2VlD +HB4nFtWwQXPeqKMc/5KrbUsFzXo9BiHsb5UJ/c7OXEyqwTVWHNd11Cgoi/hOPOyF7gX0efFlag0u +dE+BrsCrwbbOXLztqsenKbk4PWsVWkncqrXYnKphLV+g7PFRembvYa33cPzJvQKPdNSog3954rjR +XW93Mr2GOGHKPrOoUX+I7Xg2VTMD+2ks2/C8K65J2tpRkx7eNnkLq6NZDBEH1tNcFoaH3PXEc4IA +j6UieMi4++biIfcCnRzE2556HeM3dnzOWrwl/5ctHQUKXJTHz0ZGccx2PJyK4Cr3vqtxRyqCx1I1 +2OOego9c8oba7a63krgmVaO7gv6cKjDGaOH1agawYBsOLpQBipi9Eje4C/FHpbfKKySuEcf4rntF +jAEvrk5FsMe94j91aS3EvW6tzZftNabrX3mmYFYN/uaqn0Gs6qwpl2vsgHwGMOMeDM/HjFz83pWP +ihrjxjySGFyMeQXwF8qk96ylqyiTuLE1l/iTu54+5Ii81XCCtRD/suIKTyM2JSOqbTzdyo2n2wYr +ib2eej3dLUbESnqxqe0dXGNPEnFynBOFUml45+L55TVWErtd9QbizhrjCNfVEbEWyhFOhG99W67a ++1g4Q1zeVoSDC3FmzayTvfjINQzrO2uwQnh3racQ1pS6dIJ6oVyphiiOFpF8RGvUSnpkxH1tRcSJ +cWshVluFPxrOCqaH87CVxD2oR8qM8y6rWNTZiw/a7kFjDXLrUVeT2YE/GpDeGhivnyGObXu7cP7D +/mY0f7FHc409mk/bihQvW1GDyzoL8IlrAR/UPh9HvCCZdbencBzLgK1QYJ17PWvxWmoOtroLkZeP +h7Rhc4vp03UCT3dEsAr1MtntA5LQTltke2dNPVpr8KprOx7V/v2rp564qiOCjz2FFPh/cE8xm7KA +ZYrylJrzNfd2Xi/EObOAGJ1Pr0w4SG27U1YieqbLScRzke3MxhLioc4CVEams3a8F4tK8aar3gZy +QRFr+a6cjOR1EKB3N0O38ULWekbhRde3uKNzDWPGhWE8L2Ut7+f4Ww2j75U7qeeUPazF0AWa8z90 +aC5vsTQsofo2dz2SNfUzcG6NlNOResS2UZkgaz25+LTjO/zKU4+XOmpEdM0SZV78ZWcNtnp6X87J +R2MN5RYq72UlibrdSKrt2wwgnoKvTa6zKGulwn6jTtCKjfUOFtk6XjMhLs8LfEaIosiduJaeLrfi +OFpYexJe7ajB/H3B7wsFa4j+U5AsxfQIcVbTdI7nFmJJEd531ePOzpoZNcSiIkb5sWZykuYRv7FM +MvWn2l7l6DoMqMeLrnpc3VnD8UadETTPWmrM7K5TUs8gAyWLcau9H/GoK95nNyaxva0mvQ+vqMML +ovy3dNYIH262pOnG621NyvIslkPWFynByseWE4+7CmfIX/+1VA3+6FoRwRIjc2F950pzUtfKNqeZ +8xSKkW3GpykjdEjFa7yQ622Ws+QOhzd5UNIzNxIHJLVxN8HePu3GuXYT8nmi9tYm2wfhvf4rGPKj +c441F9cmI1YSj1va21dDccKGDO1ulyfng21mK3/YP27N7ePWmcRb7UUMjKCyVimEJkp48o11KEdZ +vYyuT4c44Z0iKwrmy5YmRTMO+YxRNH9A2YSiWFJD9Csux0Kn/ECVL9WFMnGpdeRAH6Ify5wK3m+F +UkHKVUG8YZRIVTD0M8rNoDGHJqoXqTmMIuezJadI5jdS57mSghXxH0XrB9IfGA9CibsL17AZiZwM +x1d3tzOrrxrthjUXD7QVWB48K2dWEdmvzHUNWlcZWpQfeYs0goPsZdnV28C1xkKzUYbBMRzLsl+2 +UHr5G2kLnbJKb2eUZ8lnKxrmsSw6FihESqdmgMvk3VNufLZCU4i6Cirxl0ka2CpOWZFu4vSUjvHs +UiNYsG8Acc89zkie75ed9Ri/pNfIIRJ0OmJcRe8F2nWj9sp9Q6Y8LNCki7+4WZw4E1jeZBg3dCiu +i1BMBZcywBcYFQGQcdGsgvc+oN5elH6FWFyhMDSrfhqx/3oNYDXlbBBA8zauxfIcRyN5nwPhx5qq +a+W4Ll5/qs65gkKGvLESfwlfWCJpIu0cuqv3pX/pJW40ibbs2fiatTyBUR5jLcR2d7GeasWe06GE +u91J+4tslJ6j9zCKzm35FGXyDDaUB8km+b6HMmqMrgccAFdk1A6xRZ6x6ZXuLb3YLg3xMib4wpIW +wZv2MOp60GlDoUuPETnfEvPWMOE55XvlYNQ9HAxhvwVoKRARDxn96Uyi//p3rEpxr4qVwAOdpQxx +fDis/Bxeo8mSo6Egl6Uyyk+VBfFChvib7bxFUo6Ojlp6O3T5H2Z8kNEP9uxwAHrcaJiepzcsSmnS +FkX5vEkaLSNkXcxcHLibUWlaOMpobqL8u83ffpkqtZK4SNHde7RKEiumoqUp00/dTqeft0w/G+9Y +ymnCF9Go893SHP2sVCtya6chNBe7HUJlCJcIVToqYI5eed840ye9+CK12ZqLra2iXa9RtOsZV1wU +7o9issQ9ECW5xCOuQjX/QarIOJq9rqPAz4D3r9ZCPONKimHpjHCxtRDPuoaxlmtV+34Bg9+pWEl6 +Da6M0oEmDJPnlfFV8xxjh2RF6RlkUEfOOLa3W1p0LHnMGf4/7OFzwlw20LPga1Gt9qYMgpQ87lT8 +l9CIG7UegRExeawuZqLXwNFb8dt0RUmvPm3i3L1HOPP+hNNel/Q7IkByqrhd+iVD9nxmOaW9KLUx +Oe93zisX9jfGJi6zanBlq1Bu9lGalr+ykOUSdK9sNZ5sv+onDkhB6lheSqWuDBGDRO9x9ko/feYi +kriMQbrVWXJplIq8eZ5ROTxubXtHRw29eM16Sj6Q2NqWq37etsSkedHxAXXPlY9YrFw8TZqzDseW +4XrKAfhxAbxpEn1UaKLMk14FWAWkSfuTSwETSWzuzOUWaxp2ufIZkmzRuY1BjsCSAqsSd3cKj95z +FUtRVmu1EPcncw1weLL/Dka9lx2LVIEcTSJMeI5D03fEwfU6znRnmdSr2oNKkBylUMEbxMymDLUq +ecYB9CMBqhl8UKw/8UT/OCfp+wNJIdl5Zpof6Z9/DKvspwVWEk/0z5dNdGnkZE1LlC0tWJ+SInAV +8YU7zo3iU/Zi/3jfPp91+vxOffIxZXtX2NR3KQkdv3BPYeD8cHsV/UvUZFLpGJXvoclKYo1iqgxN +9A5mM0fihw51l8BKTxwt+6qbnnP6WSs10LXjGLr8WOOLM+LomdgvPkFs5uJtjNA3UZ4V2i1tewQx +Fn/OKE/XcZ2Uz0i37P2eQoaMP0Cb5kN+GiFGpdZY0rtXXsh0SdzhzeZMI7HLzBQwDMYQ+ihe4XA6 +F4Z0Oa/gCUHJjRxnO6n/0K6FX4FCaSXk6LWEAbTlEv2rpTdZwgQvsxrxTauihpLE+fwWSwwyeo60 +903dixlwcHuWvW/Emsu3q8Ebw/IKXiR25WFGTUZlRQTnc5lhG5rlM7TKaD4CzG9swUIHZ3peclp9 +ym51Irz1Jjd0wG9ug1sqJ2uFe2sr7U1uUCdKvJ8kVopI0jtWB4Ih6KbOnqQSFmq7Xdg/P/NojpAc +F/SP0+M3iLtFdgPWvmyyHGfoWM8rDjR/USiNrjT0Arq37XHleTrYnoq8V51qn9tAm6kYTd3f7nmF +AQ6XUS8Wkw1ukxSXunIggWSOJ31Qun6fbqEVY+SnjM3uHuLPEld7dMjoo+iXItOUiNt7Ohdx7WGK +scukLevpbeL4bIY4gSEie70SqRYQgwulgkJKkb5o1tVw5vugYXJXKKDcHgP2o7NNFc8wWeDk60Qc +Y6pWmucDZSgSs+nLONDnveaAvsJtfA7FpkUpbu8sBqi8PIr20NUWvxRM7ZvxtCsf93aKri3KEItd +vc3cbTfjw/75WFpUxTz6uFShxwbRE8ZZVJrSycYVVCTibF0yuNokZp+bY7Xidrc2lbcKC1SwwxRs +N5aoqNVKbHHHOdXPqKd0j7xb0NmkkA964gp90KUSSsoVNdl3Rbbt6JBmLC8V2+rpZy961xvOuF9L +A7xiKhXrsozNMm9j4FbCBGbDqxAGr+7leJwJzpdGXsny5OsRF4R9QH/RBv0gTFKBA7rViqfcMkYq +Xia6h/k6wmyoTxbUCeNflJAC0mMyJkTpmSQRKorZjn2t5I8OyHttkAP5U0UblcUjoZm5PTVHni1H +pkfg2c7jhQeH5DPxhlLWMIDqlQbuTZ2Crw/g32QAv8GU9IX8nw7kJ2vCnfme0BfyKMwN1RMZSI/r +TIHfjCURw3qFGNaBL6925eHpLCBmRORim1mPkj9lBpeFV5EtTl6UDMty5Z0VJVo3iE16vL1IA30I +PQwSrcqUIHeo0LlBW2wznlo64DBvg9RO7TkM6YXnURyOMYTFa3rJq+vPvV2+i2zUb8BQbXF5PqGh +QA7ryqf3P/fb0NtvtpGbzy0qF5CPZYCs/89AJnP7gvZmGrSzm9ic2ZWuXRnQ8CnMrgwYwmFAm54r +l0wfUbtSvZkpwW9RaM2VykejfQgr0j0PUoKSPnDmiRiImIvbQHUu8vMxScGdz7cLST+CBI/VhrZH +icqmsHr438bjy2cINRJGh2oxBmVoWt27zihWWQqMZy0nyYf69L32MVYekXphFAMC+lgpYcsZ+Nks +Kj2NV/GF7u003jZd00zqeS/2l0p2NBO6gth7RB09E/awthKdFYR7a0wuXy0FaK35nXxDkBXH9G2M +AsVSiR34FJbUTDdeEDcZfjLAubqBgVH5hNTqMvhaVisCNyJfSM81bgV2RcMzzCv2tcchzxB4vuU0 +jpJiiJ6QLpWNmnirF8TLlmiISsOMRZtZpmyNWBphs+7+zLLpTskHzpw8pzmRip5o2SYA1HQtvV9S +ee9r+ZLtGficbOYm+6mPD3ItpZPy6ayWk5yfvltYj3+05+JSrE/r1fD39lzW0qclXIt8nTze8xjl +NaLf3G5uYail93I8rIQ6BmVf7lhpteIOT7G4md96pthFIld4yFMtJLi0rcnUMJrbFztytGRfWN2s +5c+Fi1mZZj3Hf81a74nUlai3qORwRcx6Cvfi1G0MHUd4iiep93c1ZE/+XvrMfe8niv/WrHr/qlI8 +3PmBoH3Eped64XcKWjtGUvL39NktP9D5dFqAlAlsmzpbqt/6hjat/A+u7WYavCLczVgkzsU7iOau +DD7A0KXrplMnrScqycarX+mPcrAU+Jkf2S+jWLqthKjep5XHGdo1I6YwzJnS+XiPpPKDpOO0gl9z +LT1j1K63UayOiKDwIyVgO7dlRsLtUi6p8QURRr14smMDcZ9nuzkyQ+fG5A3Gu/BoRw5NBI80Fz7j +Lip3mLnEEx25WpAHlKfhflkwfteRQ6+USsZvWD6sijwtZ3Tf2p6j8IDnW3uOZBhWCpIYP6GPdzG6 +DY931OAez9vY2VHBZqz2DMuwVq5/pdG4A7tdhkAFOVU88ALHSyIsA6nIjBI/ptZwokiPn1Fzs2S/ +fPrgqQ+HwzxOLn51M0/m6IJjeJQJXtIMmp86ZBVKt1CF5cp4OQj9hvFGejUUWdKnMIQjlNunQN4e +0gxuLkR7AVA4Ex0FhafwRJaj4x140k/RUaMG2yt08QT2z9c9dTEm/Fj2MLzqSGn3Uq2KV/YW6ngu +x5KH9cbiAgwsRPsalWQVUlE+3rNMAJLcE7XVAkbxdLpUa8EgA57Re06TA28zPQW6ZWVCMKyw01F7 +GBjjPYieo78JKrQIByZvQ+qeStQr8m2mE/nW87UzvdvTR/0IJsNiT5CVtObi6VSO1YFbxFwyFWTg +FCGDWOrzPetZa3UQ33S8Y8urq5RY3urA1x0R7w65Y+mMU3Jz9HtFFGKh46yw6xun12fdSmZOZQOt +pRcHvq3N6M03LHGrMkBi7jsYIovltvAY40anEKv5a3BIIebn4pDC27yNlVs4g6FNJ4x/ZEmJsgdT +KrxHTXvyAF6kN5UpQm/OWoMjCzErF0cWypFpOCcxzzt3uGlBaYDHauYl/Al/FEeuIF37rzZy4AZO +0L5pq4C19drIKSUmhD2KjlZ0tnIjpufsw/t974zzJc0uNyoecETQ+O/UySFLXNM6t1xv8ImUtLjM +PYzeoGhVUyPl2x+Vf9sSXcUTp5n4j93SIO+2OnCZjCp8Wdtdcd/KvCuNC9olgKGtyVOSPgr2OmCc +78nOuvErQ8w05VHvRMqXxARudmyAtR1Lt2X8XV2uiXbe0A5c6ckWvXZvF2A5DAnH5PmjeCH5rt3e +y+fXZd46FOs15qulZNjtqrYOlStXpKuKiVFKcZ4OUVfe30rcMUiDO5R46ewKaxoucA+jwtoqcfMR ++Qxb0/DQOUXUTSGVePGIfFV87ewKvRvDYkcmzHOn4T0Uz6jnTVXnsZb+FEPWNOSvsA5FXYQnM2Et +I95koXUobmot0pc3OIxS+K5Swze3iuraYH1pUpgdSlx0jsB6QVdpdeT8z2CZej7iZZOsb26Odai4 +casS644ehj9APEuKui33noqI+Olng4VyxtMM/cfrev9xVLV1KD6tLLV8uCQjnViH4lHDQ/vlCy0K +1ayDNxrDuc5U9HjSU3Ec1vQzCY5Zy0GHezLuTb3FV0tLdo1BNF6dTnWckqPKILlLpgXXOjjNrVd9 +kUQJGdEY9tff5phJBx1w2u+tv0t6PU2uLtNbx4SVS/yrVTE5EsaPIy5US3r6fau8UqVnUEiRFxzG +kPd9bXnrOHrxj34LVOvS5U1cax2Hj/sNy3A6rn4OdDdIlyNXG7kV5cfkmfCGfIESsZaJuH55Dn1s +lG5FXsX4o4mEj2GRcxncrt6WVsjZSJq4zBzk0enlCpVpDmKLMqV1vaWX26UJLm3KzEdPb+nWAebq ++RCxX9yqwcblOcYhPMALrOPw1371F4pSNjZ5hthbN6+/0+uH+2VXpTJ8e0nmeQg/uH+6wF2Z4nH4 +rbJDWD583rHBGodLPOYavdc7lGRxHH7tiYtDtXy4zaCs/BzH4TpPtwP6rkxLkSx84MreZEUxuFBp +7A4qXkdAAivaWtlsRdFUYBUrbKGRzcACGquQMQNBankrQqxJFTBgFSPY5x6VLm96hHE8qpFoXmtj +izJTtzNTfBreRzpLSa8/u2tA+uXJuF14LkS7cJHi90/NNFDXW+US6WaEHXu4iKFMhZ2ZCpW42QYg +wU1O/66sdA+V2DRG/EdYoSbCn3mGZ0oYp1m8VC3xE7ePLLam4bJpmt1K4hNfjzUN66flMmRV4iPf +dnpudu9BUhoNX0Z46dov3cNQSALj61YecZ2ug5akkPml7eNVNuQSevG4OymTXsD6GfGo630qy6sX +55nECF7lPL4+WcCQ8uK83V86UV0z0Gii76Ti9OJXrU1MWDOIh9vnWJXYqWXU153tc6wS/AnV1lBc +2j4ns8G6BqYhLMETAvHan1OKyHOlxrTm4p6OXEr1mzA2XAlab1olxJOeYo5hQGzgfR01TFgleNJT +bRy/XsRyhxeoOyDdeGUW/nFktlawJdNxT5+y54dlb6K8nqMXK29R5wmimo/OrrAqsWNonLqWFUMk +VKZ0UpmcgyKQ77JM/JWxSlzByGipwkxelKh4K2Xp0+SMZK2kQ3bqQOucgP3l6DsJJpJBdVqqiPlN +DDB6KR/Tme3ZoVNX9wokGPXQ3qw7D0oPpQYXa564Ue7qIaruOiqAQL5hYfqGUym/bbFtecTEhGyR +5WiL4gNXm1zICaaOOT3tGzkI5+ZkDuVdB6f7aMjCx1nZwuZeYf6QdFkj7teu32TUM4ERnBhjwL/Y +2AA303ehbYNJ2wq7JqZfysOvhP78kjfyRkYz+ttdmXJ31oF83slTEE+/ZZ7yscyK1e37/MvM8x7n ++YGeQ9Kz9dNHu5xH6qu31bzS/6E357nTatdPH/X89NHO3keM9om7732cgdL1s0zXzqM851HWgXys +d4x1znOnapfzaN+qO53nTtVdziNT1RvKTF3eqT+BoOSnj+p++qjLeaQWMykgdjpPD8wqozdkHUE8 +6VJgz+2WG/d1RmT48vjsNSo5zek7q4yPsYw+rsq01POjQk9srydj2Ctz3qN8ykMSXTJnV11voWlU +Bnl6o5lmd+1TfD89J+7l6ww4+FruNJ2lA/p5RWmWZV52nb5v6Y2Zg7RunwKv8pkH6O3IDHRXbzF9 +/HhfgPMm/Q+Ndv2o4EewTP4fXqvrLRBJMWmBnNnp6i38T+Obsm+b+/bY9aNSZ/S7nIKsA7XmfmKr +u5sBy028ltpAn57c6o6zTE9eSW2w/LjDnc+E5aYXL6ZyGDBP1ltuvGikjjI2ew61kaSnIgOTGveJ +Lt/rrbcbf3rZBmsa7vfKpugmfrdsGwO8P7O58n6eeVNjfYzeGzPr2LNPiY2lrzhYeuu+WJp3RqYZ +A4AxLIjuBzz7u9zui+ByeVxnuVwut/vfWQqrcbv1xdWwoGXO4gUz542dvXCJy1U8c0lL0zlzFsxZ +PLNlTv3JknxmL1wSPKfJ5XbpU+3M+fUnHNfnezJ8QvqbyzXA4zqoRK+4LA/+pRRevHqxlGcKAV7o +4GBvnYsl9vJL1lE+T8rnhFPyiehKBqqpMORxQs8nGKKMxCcUYkfnSqK/LEoylnh+9rWR4aKLqXsp +4Hlbt5L4dVd1ExXylUBypXDLczLmfkfkJBUAbByCvHFGPcOwTA5ESRwp/g/7v080b6MMJ2+w9jQm +WctX6KOUe4+Z6qnvCM/WUaxlPmtZxNr089bvFAVKHLxVliuGeGn6L2ZtY0Jnq/e5GBZtxsB4DDMi +GJ4sUWaRhNxsEmh6FQctwLQCwp9klEearqZ+h/wkOiMlJfJ7k7tLudStyQiVwrbWM/YHLKlB+3dA +IZa38nm0O4djz2HpRbAwKjurGQN7pAQeipbvmFDoGcMmK2vvX4kJ/9vPmeInWirYQCU9lcra/kmY +HK+r03/vSP/NfNffdUwYbTKaV7JDyo0dXKsskacadslcbcevuFZ6COm0jFZA0sqPf5BbLYFhToES +doaIuQX0IXsKFTYcMk5l2WJ8cOB2LC7CQKm/sFhIBNl/dGWH3lqiJ3LJjzHJ47lWaknDwYcod1Cl +o+8DAhJNuitFF0/2s7d5yZD05Dqb1aa8aZK/a5/i50ViQ/Q2pB0pmtmczrV/bD9tx6B24L7bcfGS +BS0N8+e4XM2tzbXzGhYsSaa3XLN2oPZY3kmZfY5js8Mxf+ZAKel9Hs4eXx6LjfR7PDbQdb1Fj7uy +Y7FwLOYv98fCmVd7estfdqnNvoU7ewv/aL/sd17c1Vv2nl4M9ylzFTtw/l3v9b6V11uyx5W9z0sl +TlEWvndlbzKFMf+EYCzmz7ACPU4dXOLO7tPuzt6CK1QQjPn9wT4z5Io4EF2ncn+fsrzesm0qi2Xm +pqS34E53diwmQNJ3Qtb1Fj2SLnJmpsspy5IeBug5Bv2Kq6rC42PhqjPPGhk8MTwuFq4K+0eGY3AX +xqrCZ1YFY0uqnDUrOdkBdr0nO2w6Lvc7om9Xb/F1HgNWn4nocQqzcIsne1MsFqsKhv3BcGYKdzkV +xCMLHwarfHBmZHmnOJ0/7cn2XhtW/+O8jZnynt7ytz3ZVcHRVcH8xqA/HPSX+/1V5kOVPsOTX1VV +NSoeHBWLVQ0Ow62vVQ5alji9bEN2lTP0rt7nd8BG1/FC2GCwMTYk5gCxy6mWhSehUQrMqmA47A/H +MiMtiTp9vKG2qvaZRKcwC38xLdhvZ17e6ZTjH3o5XNW7wLt6y75Ftum67xLkxZx+V1nZ4VhV1Zmx +WCzsD/urx02NjQr7w0OPCfvDo2L+k8KNwYnHCK3Su3Vn76tPWtmxfUF2jc+0m4WXreyeccEqe0hp +g1edU47dVnYvdejqff6BGu0La09v2eeCtXeQO3tLvtm3ZFdvyXn9ssOa+d7XXBMyQErxFo6FY1V9 +Skt6S6/slx2uquqFsq636Lp0Ubiqyp+ucLZLRPPonxLNfXmY2QuXGLbEoZguhyiD2cKOcNjZp3m9 +ZQdnx4Lpgi9MT7/9aU9p8jzW6bG5tfk/ck0zF88+N80Y6WOaiNt8VMOClsUNC5obZjf/5EHt7Kb5 +85sWpJ83t2ZqLD+n6SfNLD+nqSl9RqTrL186Z3Fzg/O6Touuoenzqh/+4MpmgLIy6OcNykJg/9Qx +ZHb2qWbY/+1UWtxyzH8+lVy9J4En2xyWa0xjY/7HOXS5Fi6eM2f+wpb03DjLVTLOwR5P9uCxQ84U +8XH+wR2P1dn/GtP/Yum/jfbjOqeu+ReuGmzTtfvdQp//BtD8OfPnNy2d82OAugodgHzZQV1XHywP +l4fL7f/1qaq86vSqKUHRt0lVo/z+2Kig/8hgOByEOx4MBvODwWHB8LhweOzgcDgYnOAP+8NDhvj9 +Q4LBYHCIX8QgGBwSjAVjQ9TakHCsvFw/QzTy8nJ9iZm/5oH+VaX/qYL+pb+eOX6I8ygzKZkqme9V +VVWTxguusWMnVeHwKeGxsdjU8LjY6cGx5aePLZ80LniMXgmaX8GRQYFl914O97BwefpzWMMLBoPh +qVXVqmr6raoa1af60GPCsWCsPBw0MzV2bCw4tso/1YBrflWNNL1ogOVDp1bZD+0Sf7C3yKDmALN4 +f/0pNjk7cVZry5yZ887pffA/CTTpiuk9M7tp/sKZi+fULpjZ0rB0jvNwyYKWfR/NWbRk5rzacyQa +NcxOV7Of7fNmw4L6ORmWLbPb9eSnlQTGvo/TkpZHItQAj+vI9M7tD8ru5GcHR3aayfjGbK3j/w+T +4QxvppHbevdYpnE3irKD5cFgOHx62F9VPj7sF3Us1+9R4fBI/X9iWOs8+8iwPxgMjozFgqO17MHw +7CPrgqPC/uDI2Nhj7D9jg0Gd/eHZQ/VGcOKRpq3TY7FweN/f5nlch0ZYLGM4NlJP/KODE4MTg4Z6 +6OqW3PQEuDFE2w6FpuQpM/j/cAqkabNL23j2vMU/HvLOsc4uHphdmN7C2sSTteeq9N9kGxfLy/vs +lfIJJ1WNNAiug7a8vDw41KD+2PSz/KD90B8eMiQYG+IPD9VOHhKrGhIrr9Kv8nJtziGZpvVF/8xC +vm3GUvR/WUgb1360jD29U/SEK9uMY3S4XPOqf5rT9L/ykcceqaXVUph/5f6xxw49VksaDoeDI489 +MhYbe+zQ8Mi6seFRwVExlYyMHRM85lgDp2uAZ1eesxre7KC/akgw6KyUa7hT5k+Xhc1abTW7tuCn +43PWqn7J2Wf/ZFRHOStlc77jNNf/D/3KjCdvjNPJX13ZwXBw3P+Xf8ys7DDYGPrpav2ExtpkblZr +y0/OyrxeVDhCXPH48Mix4dNjo8eNGxccOS442qDbqCp4hgWDwSODNo0JB/1+f3BoeTjs9weD/uDQ +qvJYbLwhDUF/UH+DVUMNFQoaQF0DPCUjHJTLMiKCwbdGM4Ky/zKCmbMa/jO/NqshTa9nzmpII6RN +v8VF1Tm9HWjUkbZKMeQVQ5H/094c7J7ZPD/dlkN5ezLmCTdWerIbRXnDVXV1deEM+7+rt8Jt/bKr +wrFw3bhwsFcgzTvUQdwn+2c39s5I7/PN3uyJsVi4bnzYn2ZPJgRHhsOjY8Gq8eFYbPxEuIeVB6vK +R4+fODQ8um5C+P/HltmEyFFEcXy6y2UmnQ0jWKB2knXU7tn14JuNF9+Me1hez8yps7lVTDykE13C +tg7rQSR4GvyMX+AHeIoy5iLk4CFXBRfxorgHFdSDhDkJXhQUL6Igryf9Xje71ezX719Vr/pj/vX6 +7RYO2DBo2GfPzjiFSpBS9uk+ESXlF2LRgVst1xqyu7nFOw63IXtimqan+Vvl5xaLRMMhh+FWn4V9 +sToLPMaD0zTdYgPgX3h8MR9s8Z+HtNPcpQhRPiqze+SCvdi2wY5Pi6pI417hL1V5pvzltg1+8p9Y +9J8rf6Vtg79LnoUyz6tVPlf+WpVnx6X/1SqfK3+9bYPWwxLghAx4oybMVXizJmQnZcRbNWGuwts1 +IVuREYHFxaeJ92DF+w3LDyuNXIYIA66trKURH+4cRWccxMY/78wd57sOUhyYu9bc6D2XRiOXP97r +hwkC8X1xYUThiYROLnZA3lnuk9B/eLa8a42O0D89K+WYjuJ/PUsoe9CmCh/4dh2RkBOXKIFejiFA +KFPMtOfPvsTbU/qLb5O8jzFA7NaGOeA6wGqOLsfMeCsAq1qqyO6XVX5sbI6QRzTEPKIxIfXzAS8D +B8BJuJzYAzJk31i50h3FV5dsPwIt7Kny/pKsN1N6balwQILSQqaqfbdkKTHeIzlGCEWJpguUb/C6 +gABWCSnvoWmtIOYEEBHIq/Hmg7LQL5o2kbpLpvzbpo0lqOIfmjbHiCqmNVPxVtO6COJehCDXZE/l +v5rWGW8tKfycnw6V3j1qXTxO8/zp0fFS7kSyyE8L2dXkTOXfjlrnIJWYU5U+WbZhUt6fmfLPl23e +LyPtCQ/M18v2IxoZb8OdJXfW+GNIYkrABFfYqNguiUZD9lppMk0sC75+zJalDT5RFW7UhEZXRtys +CR0VvqwJmyp8UxMyFb6vCVMRAvPfMWu8f2ThRHQpyy4NMySCLn/+Q3P3mFrQghZVjqg4Yoe5C7nu +t1JsI9IS/86FjXYeKk8nMO+07YdcfXSh1K9ENdfbdpHxsw8pvlHBM8U32xak+57yz6p8rvyrKm9o +xrlf5R3lP1bCbiq+VcGZ4l8reKr49xIvEpUzB1OH7SsXJ88+sw1P7k56l3dPnXr00GxlcnGnrBFd +uLw72X3qAicrRTviT8t/BPrmtp2+0OBEpXcwmiQqz+1Mtg8v7MykJBeYI/baujEbYy5jaxWbi9iE +rovsM+TwnAN63sWELvb9huf9HwAA//8yJm1PWkxJQgAAAAAAAKWUeAGMfXmcXEW1/51bBHiiCJai +CMpVQUQR3AVUvB2aLBPoTJJhsk0ml2GYDJOm02maTjNM7lwfsqss4gJurQYX3BA1ouZB+xRU3Fhc +UBGu76GiP8UNFXwov8/3LHVrZjJO+o9ebn3r1KlTVadOnTpVfUAQBE888cQTJuiZPODJ++y74IAw +ODyQ1wExfVlL70Fg017bPsem6+X30fKZCO6IvfmBTQdsvzkntGnJ9pvtxqYbKEHxnYOY7sl70ePA +phXbMF8wwDfCzfSx1KbLDuXkIDiE8YfIb5uuo2+aHkn6gfQU9Prom9YjlvSL6SnSe20rHLLpED9Q +XC64a/gx1bcVrgc/rYpNN2p52aHMzxaHC2v4qunBczj9+XhI5Z2OD5ceS/pLQ3oc2NQcja+aPzuM +8+/PyYFNT6Vvmt6R9BN76DHyvwFfNT2POP9c8gqex+mK1xbV35GkPxNEiX9zEL5qevx8zv8sJFL6 +anw4OSaSfhI9BX+9trVBSylw3XlwWl70Ai5Puldg05OJsqbHkv4f9BTpS+ibpieS/iR6ivSl9E3T +M0nfl54ifTF90/SOpM+Vvyvpz6ZcyM8jRvPnkn6QSz+Nvml6cDjX75n0FPkH6JumR5L+WnqK9BZ9 +0/RE0o+np0hv0zft1x1J51ZCeq9tbZ3dHsERzAdzLzhvvCu9ZB7cq6j0IOgK7jb5TeN8q00X2kY/ +BlWjQu/n2nScEJqv80LmY1q+cwnr5zvHpmdRPuUrO5LzXU9PwX/FtsJlyNjabtNTFRe/iHGDDle2 +7XC1Tfmxo9cR3Bt8XNvBnD7LBfcHDzccvh/lDlPpw1OUScvPj+KCJhy+19bDMSvdzuGSFzPu8w5X +sY1wA+g2Rmx6htKLXsI47vWod69ttxw5Ry8T3EYMZsKV7ahZ10PMeeMyP5rpjeEh4XptIzzNyrBy +9JKXMu5XABGubBvhl4m/0SBw7Zkcw7ivAiS4ibAPuAmSz0S40v0IgqAMUBAE8bGc7wn8kHzNcBzQ +5tJZ78P0JKxobQo+45cxnXeACNGp2Ga43YoacHzmgrscIMKVbRUES7ZK1KtL0XnxTgS1v2Yvp5/B +F5BJ8tW4SjXiU97Dpcybq98rON9hBrnQbmU7FA6iHkOUbyis048xejfP6qHP8Mf0MVa0w6uZzglM +huatZtGdHK4juNsdDvX7DKhVV3hyz17D9I7ApER8VWwSZsAlxNe09/CDlBB+wIpacuOiexzTud31 +t4qtm2GqRX2bWgUefycw/kgUSuWWLZqVnwbBKXgYBEEuuJfJhAC9MhSeCjaGwjfRh3kDlTJk7iMr +ZCjcSI/Dhfxxhk3rRblRiUt4DYhTub12YmT2+EkEd6xXn2b4oE0nkaug11nI9FhLc3s0wpNtikHh +4aKTGHc+HjrcMpueiV+FHDuCW+DZS3XzabKX6ufYtJ/hrp27i5luYX+Uaf7lp0Gg+GAJP7nNyXHA +DoZvg4gGV9F7+Dn6WEvv5vMk08HN/OtHJNnBsJd+rrTpmOqjvMJ0XyN8wW7cnTyj5Yy7y5Nnw1zU +owJVesEKxrHVx/KcaBbzl5sHBaf1s2m4Ciwona6kt7zyxsxLZpUXr+Ly9na4sm2Ff0Y9WxWvXZJ+ +xm1x42TArjPPIbmsGy3aRe3efIDxd4MpqkfZ1nn41E8DdbzHhT7qrmZ8xemHih0NEyBHzYFUzGi4 +jn6Gm2zaq/WM1nO++1GIlDMe/oSVj8d/MMi4PlfPih0zEbXy2GqnHgOlm2xg/HYQIbq9tg17nc0l +h8sF58/f9fDtYLResWm/0usOMT1qJEev5Aaeyi3ayLiTPT6rGCclWxX1NQhj3unjhPEfd3IbsLdM +mSGS2C1T4QPg5JapKW0h5Sce4Xy+nQR7Rdc7iusIbhhMC9/AyfB2/TE6k+nJ8HJ2q9YrlvQjBWDT +tfaTU2ZfGtifnDI/CElNuXKzzUxvcAFKBb0BO2iOXIC646X85XXGXbcPngJXsUPm7H1m4rI248ad +nNbaz0+Zk41NlyGnjqtgknFPOPmbf+GrS0enDYKArXKUF9IUp+mJpC8CiNMJqu3VlfSH3Tiq2Br0 +ZcnWzGHUHWUGNUt7bDqqdLtvYr5m0lU5RP/J6SeAWSq3YqvhH9H61U02XaV0kgsYh7GNl01DWnA5 +/iT9Y05OWN/+gRjrl/msf7NNtR9r+dnFTLcY76jXf6L82jabLlVccgnjfHuwEfY7Bae4XHDvBpNU +H6xjt2izOlx2KdPbBpDgGuEKR0/7X3AZ494CkOCa28FekyZTvMMG1PI7gvfp0vp4BXIX5UeXM92X +8mPS/+1Bm2o7afmJ4PxxXWf9Vg+/6Krlyu++hen69n3zbOIX1uQa5TN7K+P8+ae9ziqbweuFr67g +3u/1j1pIk10t3AW6tbPoneyv2kptYS0nuYLL0X5C89yaohzFdQTn891mxd0+wxZ8J1cyvWn+Byq7 +BetmldILrmLcr6UesH8a4U1gtQEmj5bnwdWM+7qrH+abNnD18Nv8MWHTWOnmb2f8ZsmP+sDfU7Lt +lfC5KC65hnFv9XANtndR/gaVR/AOxhX9Guusk1Bwm+Y7eSdl4PWfTPKdiIfSLyHXmXq4KzgtD/wC +J93R9bPgnczHwW78VmzLHEPjt2Xu7tGZTulE1zL+bU4vY16/mqaPdea5pJ7XhR9zBqzKu/sBzneE +kzfG+x2ob20FvVdsutnj64OMf/o0vfcgIU3QY9MEtS/6a/ThmB4sEEcH2j0JrwE+MTlVJzHfWEC/ +aXWVhJMoUNst/iTnL/phr22G51lxTzlc8CnGLaTSePy2Rot+rfVNBPcJh8N6m1RHi1YxrX6SkZYf +fZrpfs/DN3hd0Vht001KN/gM45Y4OZZt2xxFFWzTgrFN+lnpJjcy/nMe3RqqDuXl9avos4zz18nN +c92y1rV/Jrhb/XYxL6Lya+EnIN9a+HmbbgJxj378eab/MB6S3LBep+VRY5lNGzrfxF9g3DkAES5s +4FPr05X0tV75YyYK1U+iuOhmpvMJJ6eKrZlmj5WKqzzzLzHuVI9e3dxL9alPuWWJq3+8i/HH+Pjw +n6h3vUnv4afoA43m2ZPZLZzvac4+qdjE7NgL2GRbIS/lq/M1xv/Q479u3sl8raSuQy+tb3Yb46+l +p9wvm+GZ6gZydKPbGXe142PArjQnEt2V5hl7KSdKt/ttxv8/R7diJ9HA0n9UL3S/w7grPX7bbE22 +aaEO5xArS1rBab7ke5zvfdPy/Q/E0t5K7zB6i4wqn+BOzveE1w4Ncw3poUbLLRNdPYJ7GD+ffDqC +u83pwwE7YA4n+QyYjbP6WfZjpvseJx/4ed4CzuGnj5Xf6F7GFeMb882NwNVPofdltAZRuQc/Ybzq +dUvrrAHC4KW4RHDFfFC2zfMczNmZHcFhTwMvmy7HRzHfS/qg1w6NcDE4a7CDoBHW6BdpT8xlZNIp +H9HPmF/fL1I1v+ix6blUjmvvnzPu9a4cyOtmokzzQGMTfQ/LXAfN17mf811E1NC/MV+S5DABcwcp +2bbaqSr3+AHOd8u0fO9GGW1ax7fZc6L1CHLGT5+/e624jZ2fpSO4o5zfoWxHTIn6yQhVYSSMUcoI +S2+EnEMj5kmMMHtJ/ZXP/EEu9+fyHO09Ht4KCuPhY/RBOzLKZ/dXjL/Cw9fNZc5fqXSThxj3WjdO +ynYc69KSHQ8/SXTNwT3FOOn8lvHLHV2sq78D4OikTbextIIg+B3jfrkffqA91tpdU+ZoMgB2TdGQ +3zXFDrtdU8uRf9eU2Ztqv2vKfHNvm25FTuUzeJTpXerGXcWOmGcQfsQsCG0abgRe69/5P8b/Eg+p +fKxfaB1dHbHpkMM9zrgxZ69gvjx1bzDUPq2od2TgXQ2m2ZcN+FOp1ALXFZzvd26G77FpL7gocNle +TI+995AP/AFr7Ex/QLCAcf56uYX13coZ9ARX+F9Br+XKVTlGezO9y1x7V2zVPEFyrKIrimNf8d19 +Gb/Vw4+ZfxB+bBuvsfFSfLwf47/l9Y+aeTfhYcKRUvDmvegpjC/8LbCDVsBX3KIh0QJTqxz9YH/G +/8O1V8UOh/+H5ho2N5L9NozlpLZv8AzG+/2gFe4AvrXKpkPKd3AQ41ZO43s/5pvsptoarGGUbvJM +xh/g48P/At3ahLc+zZ7FOOrMrp3JzoNeWqT6KziYcb/z6NXDC0GvTqOjbsIem/Li3+tHwSGc7294 +SPThl/4e8sHv7uaX4FDG0WYp4WDv/J7rtxt7IX4u468DmPCwd7e6haXKIRecP7+0vPlFcdFhTO9W +ECN6sO/IH9cY8+zXXHBnev2tZT5EfLbCc22aIncxjpLnM13fD9wOz3LjSNs3F5y/vmjwerXBy7rG +2TYdKPg9nOnybqOOz5McXa1XJjjfX9eGnUteoMKuSo5gep9l9mn9MYFxXOIdHFEPbr6NXsj4K71+ +nmDdVLKJqZMZk5gvhhgnSXhpYc9ofbtHc/6Xef2pZfZhOfbpqq3gLziG8Z/05D5u2lTQ+DqnRly9 +uy9j/E6vPvXwfQ6ofGQvZ9x9T2Yg/HTvnDKv2wc99J1TZusCSzu6Ks/kRMYfw3Belw46sTtcR3Bn +eTjah5zhT43eyPT63DwMu/q4vVQCjs+TGOfbp1VzBsmr2q/oQl7RyYz/g6M7YNeZvxF+nbkjtLKv +r/SjXsZfN609ZB3dgjBacD+4+nWXMf6Prn4YL98CEFu/zv/VPYVxvp9uIvwrcBNVmy51+qXCuMi1 +L+wj2pdqh98FvI3pfqFtkxnh+Ij6ON9tHt+T5mM9NmVLSusXrGTcHR6/dfZr1BEGQotOT3654A/z +6DZpgSTbk+EtULcF/X6m/3fHP9rxXSTvxLy6Rx10jp/VjH+xR78W/gYVrZH/BEpvaVHPeA3j/fU9 +NopLtrENSsfRXcu4p3p0q2z5VsOLXfd3+Ggd409xdgv2if4OPgZpWcrhPc6OzTYw/iqvnmNsKY7R +fDhm/k6VHttMLngdN1HC+ab57bx9OO0HieAWOH5gF1EwQdX8dw/0STW8zal5Rz87g+lvQiM6fbjS +VdjJZ4Rx0/Xsl1BfuPlLtjFIXUfx8ZmMv9CTp64kwj9BI/NL+c9HGX+904trbTZFnJfwJSIDM5sy +l/ZQJVQ+nbM5n/q3LPy/ZxXrZOUnqDFunSf/Edm9Exsdq2bnb0nqjP+lJ89x8wDrzfBe4gEvpZ80 +Gf83r75NcwM1aXPjbH6SFuMv9vCjHMcwiuWzznsqn6DN+JMdP9BLhuivMy/nz/Cns/k6n/O9yqv3 +uPkx4cdXWleOk+d2xo+gctIf6uES5+hSP3GUMu6fHt2qeT7RRTAAtqHIUSw/zL5s5ai8ojdx/n2d +noXdv5nkO2LW99iU3D+Or/hCxv8LTBFf0HM/R/9rY62s/McXMe5mgAiHdcwHXH9Wf1NwMeN6ACIc +rxeUTiTp/v5q+1x4fPmluExwV0yTw3ioPVzLiy7j8gp/SvhbUHLyuJzTBzx5DJlDyE80VCn6D20i +BUGQXcH4pVIB+D2bbH+KoiWt0gw/CxHJj6WkeumldPIrmU7T478RXoBMjXCYP+6kjy30jq5QIjcA +r1yU/+TtTOci1z8rdtwspvYcNwf0WFlQqdyCdzFewgR43GKdJIFWSjcTXJu4Zlx7Eqxg9wILf7Yh +VM7Bu5nuq51czKvwVcuNr+X0wh9UsW3Ml2KoKS4X3H2uXOjTL5Fuxktx3euY3h2e/MbMp0ONT9R6 +dN7HuCeBGapHpdjXuVy9MkV/776f8ad4+Jo5qEf9dEo36jBu0sM1zUIah81FNtVxrPx2Psj4BT4+ +zLW7Olz3Q4z7PpgVflV/T9r0NC2/82HG+XZyizsJ1HCJzJ+thf2b7WD8a10/H7CD5uvE72BIK6JB +k/JP8pcPNklJaXndj3H+Mz3+6+ZAmuDqtPyCB3bI4eMbGF/18RzCUWcFhe33rQ7f/QTjr/fwbQ6g +UbfmAJSs8pN9ivFFf8M693x00BYtL1vIvMHp8e6nGX+iN06qZjtVuBo+jHzV8Cpxknj9IbiJ8/nz +byu8G3jsK7h5K/4c407w+K+GPwSuSn7Vah8pAcf/5xnvz9MtwxvMrap61z27Zyfj4TvCC3qnCpuu +ZKvkx6tO38+Jv8j4j3n8tMwLqb4t2nbDLkphHwdfYrxvH9fDb4D/OpY32o+DLzPuOI9uk+I0OI5V +5834K4zr8+Q9wnbRiNlE6mkELcS5inp2buF8W7x8VbOArJBqA+xUV2KF4+bD/2b8Hc4vNmB7zQLy +N/Xy+q43pG2+XvL+izvGjbfge5z/H54eScy9ITWWx1dwF+P6vXq3edy00R3OANjrN8ndjOfoWdaf +kxNuOnfldwVXxPegH5O/Af14g/aX/B6m9yuv/HG4cuBXzCGXcdhRWx3+h4z/KZii8hFnSOuTJrlL +m8upitqu3R8x3vfrTIRfhajppXS7P2bcl+gp6lWxE9yuE7RdP7GUdLXSze5lvPoxYSfKDLISQMXl +8+C0/PgnTO9xTw5j5pnUr8fgN5UOpXTjnzHeL785wXMXXorr7AaHjRS1bxUX3Deb3u5wieAuRCEk +J/gPT0dDtU+x6SmO3s+Z3mcAIhzaiSTZRHyS1rsruMIvhfmTPerhZbPX0937me6PQNTRXYzym2GV +mp5eykfnAcZP37e8ztlvykeSM+6jnvxrZDeWbM2zt9UuiP6H8TP3FzW9I+nnETfoTyGZHMpX9L+c +/yOePmiZCwz6Dl7KV/4rxvn+uJHw3ajviBnltQteio9+w/jjvX25PnMzKaY+czj83xQKoHwkDzOe +Dh8Qn7BLvgn61VNtulVx2R8YdxlAhOu1NYTO8vaPoxf9kXHi7qZx0RoCOfgtznC4THC7QIzowf5+ +D9aVbdjIWp/4T0zPj19phztBsI1wafWLK77zZ8b7cRtNth+atO9ERu0qp887f2H8bU5eWDc0aNyN +mDtJP49g63AruCz0efYY53vE06+jst87Gv7OalyBk9/jjC/6A/ykq9Qd4XDBPxmn+hx6pYF9oxn+ +qkRwU8wW4ZqI0BEDTeWRC+5sr59VzQuoP1TN0xHqRRQUn/RwRMJZ3jhwhvtPi3WD4vOQ8b7912C/ +fANuafYSFXIL9mL8Mzy51cLH0Z61sEsfm1GKyi3am/EfIy5RT+gRWgk24Y5WXC64Yj8QfrC/gCD2 +GDe5eXVfpneoJ4+aGSB51M4BvEbKrEZbgjV4VGgrQsuJ9+P8ag+gffx4GMVlgvu0x/cE/NIlO5F4 +fv/4yUzvix6uyfu+qN9SlXPwFMbVvHZpYf+8ZFvhBWrFF3LO9mf8ck/OI+Yx7tcbC7zWIz+A8d93 +42DA9pnXkVz6wmshmD7z0ZBsZ7yUr+5BnG+TJ88h8xsqZ8gsdusLLSd+NuP9+ra5v7TXoJR2H73r +ElXbLTiE8/n7hDdPmY+QZ/vmKfN7soxunoJKWogH17NuVD6DIzm/b5dqXAyC/bWfKp+dFzF+lye/ +ujmJ6qXxc+QNq4f3ob9qOclLON/PvXxjpkv5xsJfOcFrP4mPYfxG165l2zLLCd/qLdqzcyzjxhwO +fs1XEi4hjzC/lI/o5Yx/zLUL4g9eSfpsAANTHOLKR/Rqxk+Ppy85g11xHcEV8x709nrS27ApXPmv +YXrFeQ3M63SuBG3LC+2hQu/lgj/Gq18Ljr2FdC7Q7b9FxzFdP56rinOGJVs1a0kc1apNE23H+ATG +f8trj1HzXAKOUlx6yY5SlxtFtPMqzRe9gfMV9ivqeTn1TYpebNOarK3znM778Ymcb+a5PaWbSPp+ +rp6gexXRpfWpOiO4OR3d7I3/nm5H0l/m1bMuUQL1M0C+zmHc9fXUB5XfvMR073P8mJ/hq7Zj9yRO +/6jwA32HdVyJmqVEay6E8Ci9rMz4Rxw+/Au+uvSTOf1wFEL63LwAX7V/dRdx+vR+WCae8VJcvJhx +vLuAeaHXTjRmr0M6gpu+z04LLujXROklS5ieH5feHoXcEEDZ58rtCk7bHfKQAKOVNu1z7RwtZXqn +uvFXsaPm+9zvwodAeJTW86Pn2DRcVMjnFM6n/cWmPNKUz66k+/KZCAsHvuLiU5mOuL+IzwnYyTPs +iI7gvg3hkhyxTrtpln2cVZhe4cdYa3dMmYQqtGPqFPgw8HLl9zF+rn0GxWWC+xQyu/IzJedw8Qqm +93uABNeEwSCKW/trvJJx/0KnIhzxyRvDO6aW6QEqLT/uZ/wbvXGTmH1Dje9TXDbAuNeBKNHttY1i ++9jhuoLz19Pj5gU9VvybSi9as2f0EsH569JW2HHt4+itZXpiplJ7t2mfDP1yo5NPR3B+HEWbj2y1 +0T04AKrh8Nk6plvES/XaNg2fNrlnoMwx/h0f6xnv74+1wm1WHQnar3PBPQ5hkjyhB9+JcSEakAyw +tvoZ1A7IB5m+v15pS/wWpqGSbW8gIuS7w9jljTPtH8EQ5z/Z2TkVOyp+8VGcP0JlgkLvJmcwvomH +xGdIA9LVQ9KP8vrPOObXkh03lkbHOI7CMRdFP4lGme4PQJToYtx1wHprkRd3FGxi3DWuP8MPjoUf +ayCtV2eMcf65udYUkSM104JPbaCwJ/KzGO/7/+i8Zsk2wT4ZvT6/mxn/DjwkfnEugtbprdNtepq2 +f1dw/rq/dSbxATGsdvNAlelJtQKO0iv460j6jSiMyivbFtZBJRfvQ11D65+czfT+ATDhIU/ad2jh +mIfWx+FrjC/oA7+EGT15th8h2sJ4/1xGm/HtVTbtU7odwe0AE8QH+vVq0JXpJKGqKj6pM93Pev2n +xvZ2zdxO/adWs8W5hOgcxvvxRA3YlSWLiEp3Pi1uMk66M+mDCSyYJTBb26ErONo8In7D3+BT+eue +y3Qek4Yi/6z4x6vw54lfyuG3Mb6gV7ENWng3EHVasrTjc6obz90244918+Ra+9Bk+F4I7KFJdiw9 +NGlWkiQemkTsAU3RWl4wyfmnn7eldqwiXidGZbx+lQu+mG8Qx0gG4DiaSfHanzvbmb7v72piH18c +b6oH4pRxf0dhJMfT7SNb+CDSI1vI7n1ky9Bsf1Y8xfm+4sl3jOMyxsIPF/1Q99eCNzG+iFtfax/e +jp34Ej7JKH14+9kQ38Pbafvj4e1wqyB1iU3Xa72SC5jOc71yW7wz10LIuerb7M2Me5mHq4a/B33Z +iSX1Iu9LXAW1nO6FnJ93X7kfTiKeS+wQbcf4IsZtmVYO1aMaUlgv/OObCroXM74499JrJ7AOwCLE +01vxJYyTbs/jwDuPoHx2BPdBZCY+y7ZGgsOvQm8nlzI99YsjvrY6QeI4AwNb6eWCkzA0wjXCIuBN +ccllTG96/1qv25sOF1zOuOl2ADvWNtq0qvS6ghtmtqm+bYobai/CbrPi4rcwPf8cVB0OVJmoFJfv +IS57657Ri962Z7juHuKSK/aMXnDlnuE6e4iLr9ozevke4rKrmZ50X2q3FrxPcqDPtYfgNE6K1h+T +s8/9xW9ner3evFKlcyuIpPivWeO08w7GzxW3o+VH72TcrW6clu2o+Tip59G+Yv7O38U432/bpANX +Jdvc7MzXYvxfy3g/rm6Y7JySHV7h2HX4/DrG/97jIzE/ID6KqxPiAv9exk84PO5DignfOL2gr/VM +3s/4uc5XKq4zD071aC64ovyyrUm8TQ2eh5Kev6U4i5r2A9WPSYf58eOO3TlybKjIeFe+uh9kfGFf +w//yfSiqNnayHO5DjPP1Sj2cwpE2eiku+zDj/PiKYfhzRI9rPYMdjPur3+8o1KPK0dTVbeChas4n +0VfRxaFytJ7BRzn/oFwUBHsjMX8lP2RiTlgAXxteylfyGcb/BA9Jb+Nc0SftrHXhjYw7wp3XHrB9 +5gbyh/XRcWxeSB4KIoiH+SLjJXzMnSvX9I6kHwswlUvXUzm+ckl/MxIpvWybkKsYGMp/djOX44+7 +JgxWmbCcXL7EOH8eboTksG6E1xV2gtKNv8z4OgqX8unerxnldwWn8WHQJxTIIO3q6H2F6f0XiAk9 +mphn0At2Ma5Y/1U44FH2ibQ+XcH5/thWeD/6RosOfbTCdTY9xZWf3MJ0/+X1q2HzDbdOVzss+Srj +ivJ7dUFJtzhgvVqyOOHIlrHWL/pvzvdqVE7qNzECdiboQJWjL7it3vpxxBiKqxqhG3ZGzHE9RXto +ffNvMP27Pf1T5X0MsaPMi3tseqby0/0W44u4cqwj+mBi0EvpRncwzo+nbfG5nBaW9TpPqH4Ivs34 +m5y9XbHD4SWo6LDJaJgND9AvWkDP+b5MQkqUj+73mK6/P9HAZTyw+B927jsn7/hOxk+zUxB3IP3e +yUFwYm5T/5zw9rm1XaK7mN5KT751rE9Ktk5hIXW6VwjvNNSUfnY355vPT9XdQ1xwD9OjxSn1o15y +FffhRxA4fhPBrfL6cwN2cck2NkL+DRzKKdmGeU0PgqmU3+4Pmf47QUzpox+JIBWX/IhxfwKIcIgn +JkN6sOKtJ5MfM44WVYRjfnX+UXpdwR3o5At6d4HTwdM8evm9TE/bBfpkd/tj0U/2DJcIzt/nrIZr +Z+2b5oK7HZWgemCf8Ap14xX1+CmX+0JP7mMG90ywPtD+HPyccb3eOBkzb6QBMob1jzieVT5JzvhC +f/TayTN2E48huPn6Wy64P6Myrj4/dYaKlpv/gsvV31iPTGx0asI9j/6Hcf7+Shvtt5AchBOaP/lf +xp2HQqnc3e9T54Ir7svrJT+PTBtFuQ8yvT87eZftSPgE+s2I+X1Pgfs144538sY5yHEyEkbMt3oc +f79l3Lx22R7icsGJG4Pk18R+w4x5Lf5/XO5cOGcXCC6C8Eh+ZQoo1n7VlXT/fOw4xZmV7PjaYt7Q ++ka/53KL+Qz9OoNKoJficsFdQk9RD7TbWRpO4srPHmZ6TTeOcb7rVJIzzt1p/ZTf4I+Mv8G1X8WO +mbKMg6UalhooH90/M/5djo+KbdFtVi14C128XfIXxrHVxfy2aB3dWonVjKMnuF96/NYpzn6YSlA+ +u39letbjc5wvGBk3z3NhDo5u/nfG+/NmG0fuZVw7uo8y7kbXL3HfBPnzauaFxvm1ld/sccbfSdyh +XhXbJj9sG6t1d59g8k/GLXJ0sR7oklxrYebodQO+4fNKj14Tji1pKC036WGcH5/j+SdduUHIOD5N +Df56bXsbhiP2CRJX70xw4548x0xA/I3B3z5DTsECpuvbN2Pm/dSvxlZaPYbq+M33ZvzbXb1gH7/N +KS6tV7YP42gTkfiFn5bik9BRXH/K9mWchF3zvNOerX9zwak/GPqyjTg/Ge/a7vF/ML1jvfq3zAVU +nxa581qIUlyp+Gw/xp/g2rNiR839JK/R82aP6+7+jPfH9QQCR0sUabjM0X0q4z7i8TEu55no3AhZ +M4Vc46cx3t+nau9mnyoT3FvduCrbusmofvXFBb3o6UzPX/e1+VwANloWaTt1BFfEMWB9vahHV/qK +Cw5iek9x5WI8PaIwh4ueybi70ejU7mU+gDajnYJnMc6f15rhPejQiDMs2eZpZHyqPOODGe+fy6qH +Hwe+TusOXIqxqtCXybMZ74/nJm6Cwz7Jcsd2Qf8Qxo+6+pVt1XD8vDhJ6US2zhfdQxk/M15A5ZVL +enFOBev4X822A57DdH4LYZG8ME7Ijd5q2rRXy8ufyzg5ZhnINWTOXgsO4/SPev2tFv4N8qH7WEq2 +FtK2eA2Ty6Czy7vP43zf9/K1sD+L+CRyEdOCDr9uArEW69AWhSa0+uzMcxqdw5neND3FAc/Y1CtZ +eDEWF3KPjmD8NH9XeD2KqtIZL1lAqFyzIxk/bZ+K7W9cB7VacfGLGOfHD7eYLngv7rs9inH+OYKW +eW2POnJ1/dV9MePe6u4XGrBls5mil8rs7y7TxmqZnPflkK4iKCOsqWTL4aWoUDn8het4amcnr2K6 +P0fjS/u3ye/ShteZl7wcdUMqUv018as535me3krofA0OEH+H9EFCgVny3mCTQsdT9zjO/wrX39Hv +6CLhFt/agP3H8GSVZ3AC44vz4hjXl6FabTCm+xwdwW2fRvdq4Fp8WzitT+U9pB3p1qgTi2vn6PVc +XrGO7rWT582OE8kEV/jrEddcRnntMeudv38D03uqLy/yrye07ofUnsZSQ9z6GpVTFHO+33r5auYi +8j/VwkV25nwalRn/Xa/+tfDT4IfuuMU4/Ki/T7yI8Wp/W+yHnG5n3eeVCM7fZ2maJT02XYNu49o1 +X8z0FnjjucHjuWGe4+BOzkEv42WYUfnt+mw5J4Lzz/c0sO8tC2btJ/Eyplf4r3B/c78jqHLtCu5C +j89aeBvJia6T4Go5ukmF6X6Tags+cW7qeu7U/jy6nHHCFtWHLpMRAWv5HcGlXvkNczDN+w04YMTQ +0HpFK5kuR92CLvo/jXWKn1NcR3BFnFvZNsxTQzWQFNftZ3q+HdtAPxHHj+I6pzFOzAWqzwRZ5Kh1 +IZ9ogHE+vTbuF5J7CpResJpxxb1+ZTtqrnX3xiius4Zxxb4o5vsHZs1fnbV7iFvHOLb+uR4N2G9S +D9WH0XrGFfFC2M+jOEUKWy9ZrHZKFnd2wKRQfqNBzufbZeSPLdlmy6bLFJcLrti3g/2aafM4etkG +psfWPvM7UXPVd7hccNPPEXIAzzLv3tlsiOmd5+mFphno0Q6s/AUJ4/w4+DrNB/XwXTZd4XCnM066 +NfULjFsdx4pLBOfXt7ab+gbDTO8s7lZEr4G5XgrQcZMJrrgfC/fLPIv0YRXxbOKA0PZMRpnuLW6c +rbWXpTTPXJaa5TTeLkuXOUNby4nO4nz/8OQ1GtJ9caPm6B7dTijwmxk/7fxjSH7pJpkxTVwGWpwv +61QZL9Wj+iI+CnMZXiq/4GzGnebxMWZeMavdkhrjfDnTeVJ0Um+cBlsYN58fJBbci5BZ+t9kcc28 +4y/bQ1x3D3FBnfmbr9x4D3HZPDi1Z7qCu97Ns2vtgyk5Fh5MeV54MJV9xQdTGNJ807K2U9Zkvov1 +TtlivxLmPlSg4oJzGbceQiW5lm0dES7STtr/EsH592K05XxJ+0wru05FO8TbmG5xLh/ryOcZPbCm +5XcmGOfr6Vb4Ecx7MLjcuYHsfMYd7/bTBmy/WUHXqvTjXjBx9Crd7gWM9+2lid3YS9GbGVfEj5SL +e3JuLcZTfCHjer1+X+N1bg3bdDJNufI7FzGeo9t4PNH9N7J/pPZhcDHjfuz0QcVWaWFSZepVjr3B +JV7Y8b4CkqliIPNEoe2TX8p0/LixBs6Nl+jePexenAulovLJL2P8ee68Mdrn9Fn3xCRXMO51Hn91 +Oc9XNz1uh0jpdq5ifDHu8X8QNWyB0Uv5ja9m3JfpKeSD9SA7mEg/4b2/kH/wdsb/ctq+5+F0AVaf +uSqcafcF72H8tDhCPogp92wNQxyO7/i9jKdFLfHTayXQrQKvj6uf4LS9YZ9ObnXmnKtf8D6m55/D +rcOeKdk6N0t9EaxaJ4/3M376+ozHAe2k41+FVnv8foDxfvxcO/w2ekcbNVN+kw7jpsf//4zOOeCI +lZaffZBx/rq7iftOSrY5BHQThyg2uXks+xDjC3/mWvvH7WB4IT5/Akb+uH3QpmdDmgXf2Q7O57dL +O3zU7W/o+r57PeNmxqsqv/FHOH2RG4+wP+9DsW1Mu6K+HL7zUcYX9wNWbFvWh+hqJYvbi+uufvHH +Ge+3R5NHZJMiemUWXSk3iqrejm/gfCe4fdW19qYpRGKW8En/kXDTlHkeTfQ3TZk/yQjSemWf4fzv +ZrGRX2+SVcAk/XvCJB17dPjoRsbfMw1P97JNwyt/wWcZf6HjD/e9x7TSGzKvJLaGeH00hFYMxdDW +/tTZyfkLux/xar1uACgu+iLjfPsW8T06zyiuI7jnuHaEXf0o2hFH15z+797M9Pz7jRp0RVzJNnBg +V+qvcgy+zPhHPLptg/9vKPE6gNVngd/F+OnrWhzLwsXyOJfPjmJHv8t4/16hIfM4yxH7asKQ4qOv +Mf5ZHj9NWLCYkGPZhPbiteOvM/4oPCR9xPt6KrdM0n0928JKXSY8xUW3MZ33gQjRgV/jzc7BoLiu +4Hz/YJviBsmh7vRJ53amJ2Yt6T/aEZGGdfX9BuNe6uwX3DdD8ayjZjN1M7iT668CU0EQJHcw/ine +PFM1fH+oBBaQGqTpcLRot+w7nO8Kr5y2BAC0cS9Jybb7bbpVy4nuZPxxHr5u7uqBitPzcLjGCLE8 +eKl8kns435E9eIp6Y76i8y/YcHDnNzs/2DNc/kPG/cGNQ9w7/DW5Ha7Ql/nPGOffE91EvWQAqrzz ++xj3Zcdf2Y5TwMC42Uq9cpxMea1P537GH+rw2I/7gzPknB5+gHFiJgQ25d+qT3JJ/6ujU7bDCOQp +2WG6d2x4EGN5WP6QaHiprCy1PfJfML3CT1umi1VLdlSsjNGQbrscxeHK2J0n7jzI+Y7y7UH2q/Vv +R4n9ZgVVu5+d3P1ydryfXBT9tJPXb34mkIttKn/Qp/LJf8f0/f/RqZqHjDq4FBf8iXFP8+rfFr0E +R6C2T/Rnxvl6roXzcgs5rp+PA2xwdOO/MN7/X56JDSI8r39kgpvuX+Y7x/ByfD7C9ObDZXuIC/7K +9IrzaLCbP+MYFPUXdAX3VU8+NQSEl2yNgrw1aJCMvtoZGIc1uiazRr5QPMdcrnLs/J3LLeJ94a+g +E6xN6FFg8dJ6x48y/mo8pHELPsnObMCGU1wuOC2H7Lp1s/fdoseYXsXVp2LrdB8fDS9HL/sH43z7 +rAHFwnExbr+v83+M8+vTgJUvFVH+4scZV8ynqEfRIXQ8dgXnr/vb8Dc6zzmctJgD34VRgrOatHLS +esf/4nKGvPrVzAIaJ7hIUO0xxXeeYLwft0j/QIEdEt6yopeO90xWPrdJQCLiIkexA4kBfz+Fn42a +43twBSG9VA8l+/CKqbhPIKQZR/noSvq1Ht+kyGFvf8C6fVvFJ//B9K7x8A2D0LWSbUxYvfbAyT/Y +j/FP9/B1GEgluijJ6f/oyYx70TQc3V+Ec9aD2p7JUxj3ENUSehX+W2qTOs7JqLyS/Rl3lzcvDofv +QNsNc3capl05akTvfEh8IOfb25vnhsJfIx/+L26hHTK/htfIyePpjPfn31G4iNAuq3rceXnlKzuI +8cc4+liP8zHKfvrnRXk3W1CMk2P3YM73ek8+LfNUPuTu8R8cwjjfv18nNxlfDMzWkGcHHMr44vz+ +WnvNlHkPWRrXTOGfL2hK1/oGEeNvcvcSEf4G6oHXTJkH92bfJqOCoPsS/jb9Xn6OQ8bKaKFGZJKV +Iu8w+8VvruUmL2U6/jpkhOOXRhBYLGapk3NwLOPv9ta9vWaQ2OwlP0GvOdBYNQecnPPXcD7oOLyg +z9oUB9pejbgUbcf4tYy7dFr/on1SiZmkXbxhcvFt9egfz/mK/a9eW4UfXhxO2s+zExjn36sxAv0m +CwuVS+d1jKNDWzIe9F41cvc2VtEaUel2Xs/4Qm9in4v20trrbZooLn4D43w7mfyysiGi+iUX3F0o +nMoPyRRWOt0TmU5xf2LZjm/FcBrHqjBWXPZGxmk7Ii7Dj8Ny5QmuiDvjk9AqjyhmOv45+DE6z1uy +Y8u5b+Kl+GQh44t7QnHP1gPU/XvNtc7vpu0eLWX8R9w4xPr5B6gQhZGUrJw8DdkwYnQQ5L387VDP +bl2Ddi/ZNdCzJbtGbn1YI9cYrDHPptljzRKbhmWVU9DHdPz9CW3vpd5+YFdwRRwW2plDoPGPMUov +WTGTHvxOdL8K6G1y9V7JOH9d3+D9iQbZq/LeJ1dMq3yjVZzPP6+r/nfaxx1dTVdzKj9xP+P9eFRc +7EV/9uHZbx3B+edLErPXrHs+gwGm568DxjkOdnypLeafYDXj/HFRJzHUz7TphOOvI7i3gRnq7722 +TlZxfRNuudd6x2uY3kV+P9nG3YRub2qHdPRU8flaxl/i8Djny3EjiAskJRwU+i1bz3j//t+WmaQe +AwcwzAgaZi0cEuh35URDnK84N1OxTYlram532yEFPmG8H6ffpI2Tkm029PbqAt89nfGPysIHdkrN +fJP2dWrQ1yE7crXe3VHGn+zp0QZd7grX62NWDSfFR2cx/ulu/qzY4RRyHTaX0Kgd3gArxPWnKuO/ +A+FRe+FeoMupz+Hl+vfZjPuEx0fTLCSK4riiQ6Jw6qEtlH6+hfP59tO4Ob1H3FuO72wr4y705DJq +4H9daEfNdynGGy+lG7UY7++Tt8IbnD9YccE2xvl2Zwv3T5Vs62ybnqtyS9qMO8CrX8u0uL8gnlou +DlC62QTj59pfUrpdwfn39bdpBaIGc5V8Jo7u+UzXj7tps58Ok787lxRPMs6P+xvjkTPWZ9Pztfx4 +O+PGPLmOyf77mLkflwSf7su1+ybG7+ONszq7F3E24HzlM/9Pxsm0zHYA3xN7NlY3iksuYNxmFEL9 +C369PlwFQi/lMxfcJk/+DfNLkj/tj5JVXtxrmlzEdP19o6qsJ8SjQyu/KjvFqxW5KETLiy/h/EXc +KOYpMjARz4JpivjTeuSCV3uY7J5RjCR+KS6+lOkWfnH4xSh+Dud2innlMsYV6w7Eb/RCdPRSel3B ++fdB1WB1SsCU4rLLZ9Mr/gpqkcPlgvPtljoc22K3OHpvYXo3EDdo34qthqRIcNC4V3HxWxn3XQ9X +Q7yKBC6rvKO3Mc4fh6PmBGrf0WWF3aF0sysYz62A8ntt43wnHofLBcdWDuPqcOnIBUFKL7mS6fn6 +oop1kWykKS6+inF+fOju7jHJBDedXmdW3HlyNdObr/2itzPOj2+oI36Al/9Ojl3B7e/GCdZH7IkT +HxS8Fu58ev4OpnvKs7mB8L8RO1KzgxT3jtRsfTLmhh0ph3LtSM05T7JpeIYrL76R8/vxXGN8P+yY +eZSab2yNTTep/JLPMd4/j9JGXIlctKP9IRdccV8O+tc/wUw1XOeW6Q7f+QLTvcjVe629L5WL+u5L +K7BB8FI+4psZ/148pH4B//95GgbicLng/Pl7xPT22LSKbA6XfZnpybCj8d9Y59x7DpcL7grKLTga +NlhoNNx8mn2F6e3v5um19sopUcxX8h85XTmFBc35REnr1bmF8/n/tzdG62PWxCrfpMu4u5y8sB9z +BTX70DIK+aWX0o2/xvi/0lPwXbGjuD9PLiZwuK8zji55IVyvHcX8WLKjS2wR79wVnL+OG2M//dgy +b35KbiuDjPf/J7g3hRa7rXU2DTVuRP1Q0e2M/+a0+YlWcnWoz5Kt8+2/dVL/8I2QAJX/4Juc/0We +XGoG/3vFLau4+A7GPUbcsTxGwh+5/qNyjr/NOH8dWg/pisQ61KWoVUc3+Q7jf+jxP2KucXa5ruPi +7zGu+P8W3vBUOWSSXvzPXYXuGivZFl2/hj9lKtkWCQb2LmwyvJTv5PtM37/fZgQBByU7Av8aS6Pg +u3Mn4/85je/be2jx7OPuZhxzC7lhXbXC7eNp+dE9jLsXmQkHPxX/TxO1fn0JbV4pPvgB4/1xpf5H +wmM7fXVxL1FH8EV8JC84nT37Q6bn35vZ5n4nC1Q6rNpeJy2u+XLJ58dTjpO/ZBzuGtzKTP8CM25w +qGaoaK97ubxiP6Jix2ldNM638GC1hdyfhwocN/vLP7FoufHPOP8vvX6LC8YX2ip5uRDD8SFkrWIR +sbHIdz/nK+afsoXJWbJtePNLtKtfsu7/b528H+B8hT4oU9wb8hGL2ITw1jVBznj//+z0HAhFk9Fx +kH7HV/ILxv/ArffX2jvT8F+owp2pBADdmZo3GrYN9H/84l9zvo9IgD7ms51TtJG60O6cwtRRwoMK +TU07p9gk3jlF3qadUzwyd06FdIZ35xRrrp1TcPEjHwUU7pzCH0/hp+SiG7N2TpF16EhJmjyUD3Of +lCup0z8UdM9uQZLKxv7OKfkzTFdsjZS29ofgEZbDJZ5d3+b7INsSJijjAn75Da5dO49yPn+/sUbn +dWq8O4+wz6FiHZ0/xvgL3Tw1YNeE70MzraEtrzXC5xq6VH9I+cv+yfkKP1DFJjiXVXJ/v0Xeg2SZ +TcO40DPxE5yviN/BOVpy0A4ut+mQ6kf9p3cx8/B/6WRhqP5O5J/g/fs1W9jHE8WmuKiH4x7O9fRa +na5TYqDiOiHj5ovXywUn0ybZCXSvlAQAKb3EMD0JgyBcC3ExYu8pLhfcAR5/bf7/JWg8t98TLGB6 +0/dLurS2xMvRE9yxeEh6t5f2/3SeVVy0N9M7DiDFDTo17nCJ4PzzurgXYCa9juD8+4hIf8u6Tts1 +2ofLtSiUymXBKV+xpM/XDpng9Fw51msT6W7WD4L7AAqj8rAOpGg40nBabrYv8zVfPbuCm8svoPSC +/2B6/vrG96MrLhGcypPqsVVPJRXt0BXcCagE1aOXDupoPqUXPInL9fc/qgb2Llsqisv2Y5z/P4qT +pLi4gzrcUxj3DRRK5WK//zq336+4ZH/GHQkQ4co09cT44c0jHcEd6Oa5ssRvI6ClBL3TDM/x6n0A +0/X//4z0oKzrtF/lBzIuQmFa/vpifRE8jdOvQiKl4/52MqVq5P3HhvEqz74QvPJvU17xuPIk/VgQ +I3p8/4mmR5bLewoSKZ13tjU9lvR1bn40a/FV5dl5FudHzC1eNj3VNqskHuryigsOno3DBcAlW52G +SwTHXnnoo/V2sG3TJXaQdpqUXi64Qr8N2V1TBNw1tapol+TZXK6Pezwl3OOQo6O3O9x2xm336R2y +G3rnM+58D5cLrvAHDNvHtzFum4dLDmV603BbGLfFw+W7w21k3EYPlzxnN/RuZcE8fuuUi3PKd4d7 +QgTzRLpK2z95LtPbH41L7cGaQu2zTNKL/6MyL+2xkwvtxCI7WSr2pZVefhjTk2nE3Sep6UHE6bJ9 +Pus+n0jSDwQzxA91H9eOsaSzdkD+7XZysU2ZqjduBDcXH11JvwiFEJ2QXFXKZ/I8pliM05BceJoe +PJ/Tn4TMlJ93zDQ9knQxH935Tk2PJX0u/hJJfw6IE3324Gn+TNIL+yWkq6M0PX4B8/dUZKb8HP+n +6Ymki1nn4qI0PZP0w5CZ8rOjTdM7kl7oOVaEmt6V9Jnzq6bnkl7MIxyRp+nR4cx/4R/jGV3TE0mf +Uz6SLtvOAfaM8dL8HUl/Gh5S/djjqvqiK+n/i0RKL9sENwfArpwq+ln3COZzrn6Qz5MevPDf54/m +SY/nSU/mSNd6ZpJe7J8vtgPmTbPOt0VHMZ+vhDBEHu2aC5dyck0EJ2FtgVpCKvdsnvSOpO+HQqgc +HumavztPej5PevBirsdc9CNJn4v/eJ70ZJ70bJ70jqRvFcVgU1PHV61/dDTz/ya3TjIZvmp691hO +/5izawy5AjQ9ewWn3yoK2KbmFnxVfR8fz+mfQaEk/5KdLNuJxfb9U6bTYydKpPZjpRe8jvGwGfCy +Ka+rND2S9Dn1oKQfgsxUHq+0NH8i6cqfXoin6R1Jn0sPdCVd3NJYx6MYlz+X9IPpKdL5oKrSD17P +9Zs5H2l6JOnPcPmnr+diSVf7GPEUdM+3GHRKpyO4ueTUlfQinpK201z+4A3M50w+lH4k6XPagZL+ +TFcPttM1fyLpM9vJ6RFJL+YjrG9ugKFI6xulE5/IfM70N2p6Jukz5xUtpyPp/v+Xt4vrXB0uF1wh +r347eWqhr5Re9Ebmx79HtjFqq6vpNsrY8dUR3FecfEKqqvKdxExnLr2SSTpbM+hnvAGvfOSSzrMQ +0iG/UxzDWk5c4nKK9Q0HUiudTNKLebVM/28q3c3R6QpO6eIsD176O5f0Yv3LHgRNjxYyH0Vcfkhb +IJrelfS5+AxO4vzFedwyxT3M5DMR3Fz6OJP0mf1S+ehI+gGoHMl1OT5cenee9FzS59IvQZnrcQRR +RbvhVFJBP5b0ufRPIunFfWUcoart2ZH0nzr66Bd0f1wbFqHispOZD/W/Y/3e3mzT1bZNXU5xueCO +9+hNNF03c7hoEdMr1pVlOzE+G5cITv1xpN+2FDjXDoKbaW9peneOdMe3pM/HT7T43/Ot5SWCm4uf +bJ70zjzpXUmf0/6V9LnKD5ZwPWamqzwiSZ/XHhPcXHQySZ+PTldwc9HJJb3w45Vt65zZ/SBauvt6 +abvE86Qnc6SrXDJJL+ISy7bVKPhQO6IrOP9/BOheiZJtLLLpCtugYePoxr3Md6Gfey3uUxO3osN1 +BOfvR7YQAcbLp6Key5jezPla65FIOq96oFewX9mvatrhcsH58XKtcMyKeeBw2SlcnoQXkX5o4UIg +qYCWG5zKOI6KF9wGV6yTXyK4Yt+r17a2Y7rH32assC0Wn6tvLvi55oOo8u/LVf4ywc1bj+X/np72 +t0Rwc7VDJunzltfH5c2HywQ3l3y1nrngiv/LLLv/K0bEgOI6K7jcwp9WtpMczzoJC05x0UrG8ewk +7ZrObteO4PZ0XESrmC57FUG3bCfp7wImydBWOWeCk2VFYNM4CIIg+P8BAAD//6NGIKcBcnVudGlt +ZS9ydW50aW1lLWdkYi5weQBaTElCAAAAAAAE+aF4AZR9B3RU1fb+d5PJpMwQEibYS2xI0QioiO/5 +lCIgSpMitvfgZuZmMmYyM86dFHgWxC5IsRdU7F3RZy8Ye1fsChZsWFHsXf/r22ffmTsB9ffPylp7 +zjnf2WefffbZp957n7mnFAHwr8LKtqVyiVYHGzHYte0wEvlrwJh0fbQ1tks03ZpJJJ36eHpQw6A9 +/lmfdeJ2YwJexoD+aEi4+yayTjQ3tsmOOrCs2hxwxqel8CFSidzwZHz4qCmwrEJ03/4djSPamvo1 +ZNpyo203x7yNwE/flaI2nYwBgz8uRW3K6TC/whkstuZ972ds59KtiaibS2edDHNncllg23wu/ioU +Z0dziXZnfDrWlnRcWFY4gzdK6zcJ+CDReHpsarKdiks9MsrLzdnZnJGh1kmpXAW+idQkJ+sm3JyT +yg1PJtNRkcTgw9HmtlQLLg+xKoUsdiymIIpY22nAhXQ3Y6cmNk1LRZudaIvjYckjbCdwQVmPrfxS +21knZY9NxZxOZUpggVmzY2dGJHLu6HR2eCyWJcYmFVhtM6wftgwgLEzwQAlZh5ttdJX0LyrE49LQ +mMhRe7XNADOuX05DyumUtlwPYcdig1h8Bnj3k6J2jMUafQm1KSNcgXff/s3RZjvVryFrRx2R3rJq +o8CUar8imtqSyQ0mJNPRFiYkgTHb+HMwYXoi1zzZTvkBtVmGMeAbv5B9+3fYidyR/RpizpFtTpuY +yJFAbJsAwm48g9/LJvcIIDwLc4P8URC+LcViNli+SdqABIXc+WKdVPdia1ksWFi4E+uk+EI+6qux +rYnFqqJqE0D7d8VVio9LuDn2QLeZyCRwzzYB1MYzwNxfukHbEqlcJpft1+A60r4EvSboDYANX6c1 +k5vlY1wQr2++6LR0XS05HM/gokHFReeZaD0KTFJ2Ki1ezO9WmtpS0UkjWWgTcCAdXt5VMWlsqind +0G4nE9KtmoAFffwW0bd/a/d6tmaAPfsGUNsKfGD5wU6q/QBn1qgj22yxOxsYta4UdGKkhXKdbDad +nZLLJlLxhlH8TekcYHZ/Pzc7k3FSsbGp3JScABqBcDiA2nbgu/IAat1EPOXEgPs+KUWYTXuz9eQA +P4NELm2TM9M0q50ELviqFOEE/rX7On+DZocSqj6uIGrWsWPTUnZSyhqymw8TPhKrSo7dyV9g1p/+ +Jzx2HVzM46WSO4p4tCbEbdmmy7PSdF/h5gSmiODJtPlRYN+3/4zcrIzTryFltzoTm8TEdcypTTc1 +Ad8XOZdEzm7cz3abR7elxDUnUjknC6z4shS1uVmZ7oNVnjvL2BD3XkVjG2VoSLijOjPpbM646hRw +/1q/qpNNbs6OtkyyjRdKpWMOMGpgALXRlA4shdopeFoqo/B8GxYw0jzTpN+oq/T0V8D07d+U6LQ5 +HvVraMo64q6agJGDAvBaPdyOmS2D/Q3qznInO66TbRd4uw6AWkY4g4ssDlhhJ5vFRVaxPdnRI9sS +WaeVI+uM+Ay8WFrci+NObvxIO9pMzuFMBsc2jAggHMUca/owvwhZJ+nYrvCRnsceFybDN7ox5DAz +Ous43tTBBRZuFkA41+yMSOTwTJDyhbOO25bM4bmgWBUVkYh1YoUEC8qKJdonp9tSsb/Raavd4kzJ +2KmRSduVQdBNzHaiEsCQTzhZSbtRO2W6aIF7YzqdHJxIidPs7J7Yt38rh/t+DY22S9149Shkj0en +ZhPxuJNtyDlmlpQDNqb2krabi0dxQ4D+wZch7UabnVi8zc7GxCQLSSmnI914hBMVWTZg/VTqFLs1 +k6QohWxNtpvL2im6zHBrBkdLo7iDcLxVSWfkDtRfxTmS6bh0/k7gtwEBhDs5DUEv6dadozoz6E25 +w53j7ZSZvmxkksbbqSlRO+lgYwkn0x3YVBg0J+LN5qevoGTazg3ZzZuWNJmyCul9+6fSubGp/Rw7 +048zEPVG1rAAahtn5RzXOJ5CBneWO80VtXXvAAVM3/6NreSXbneyTZTPsui59pwQABtn4QS/RXtg +18n9Bb423d4E4eAvplmKiWYdO+dM9BXWDJw0zl8IOY9L27HRdjSXluEjmm6jd2EvqB1h7LPAubEt +2uLkpjQnmsQSGrunZ9kb2jK0b9ZNqDjm7izG265Mbdbj0Fdld+1WZ0pitjMmm+4gq/Ukd9rtaJut +vrPRqCDcjKct9qhCcbl0ptk2sxShIk04l87g4W7Ivv1b7Qy9d78GIscn4s25SXYqEWXxrTms1zx5 +eCIVk6VMiyOTlv8T1kmqr9oA42ZpvXg23ZFIxVn6epXPpdPj7dSsidq0I6RZxLWk2KJiW7A/K91A +EybcUd7sqrN78/lUkHKcWIszqy0Ts3PSln8uaDoZM3bRqo26nrheo6byUJFVceF0MjYCz1YUN5xP +lqzTlHQ6E+3OnyrYlD/KbxKqV6+QWgPR9m/EhxXsdQU78XpbizNLZFOL8gPiUTqifg3s/jKINgL/ +2yKwwXVHK/39lJydc0akO/txFGM7NgI7TfR3P7rwiTIT8VZ+2QR+DchqKjkYp5Td8c+ArKvmlxWv +q/LuP914hHGCVn4I8Hj5ZdfBIpZoT8ScEbNGJZ1W9i5/Lm+4PhJrLY56hewZndo2cKFsWbWZDPB6 +ub8e+WVewp2UNjMlawNrPbetMZe1o7kNL+e81L9e01FlMpA20LuYUZR1jwLTvvVPn1iSQ//tra4L +FerbvzXqpHJZO9lPVn9TOpyMNFAUiE7mnLnDcTJxJwVwoCpkTDkdU5g0Lh1tceguC0l+nhk7m0vY +/3e2G8g7LeX+X4Xq21/ENTL1a8hlZw03kyrqJQlsO4U1MtMcv7xFuRq52ByZ5hCeS6RTvqx/miWW +cDNpM/nQUv4UKkWNdfdNp/xC+eHFDfL/Uft4dLydbZmezrYMb7cTSbtR5iDsA5wp+ovQKXK/wsqw +2bFjQNVAvym32tkW9vOGhEvGZkRvBa6Z4kfJVHrfdIcoSnvO+pNp12nV+S31KRsQuOfn9Yx0zMgp +Oe4V+ZejrtOaNVPav8s6dbqXu7YlwX0mDt2FertObszISc06S+zsbtHx6Ig46zkp62TsLAsrZB1j +JoRFcvXtn0u0Ovsl3Fw6nrVb+zVknWg6Sww97bHTAqiNtWVtsSFZf4bdtoyTNSMUbG4khN22Rn+4 +UGDf/vEoW9LXRB3Acwf5NS/desxINaVCXq5i9020U1sbWMjWxpwoRDPhBG4NU0XhBOP+J78LbDLZ +RCpn9le4JGnNYDxnrQVAKp1zoknHzrKkFHBjkfXEk+nGbFvqyExbrtHORWV3xPzAQYeYoWLhj34D +6Ns/fiC3hcxmygg72jLc7EYdqTmOHAyMOMSvAW9HxXPH3FB5JOhHqEcQN9mvIS+tC+SKOUnfmZxO +57T5ZatDvDMHufXmuFzlx92cnWuTMZIFF6/WCtszbW6zVkS3aFih4noUqp7fqtFKF7St0/SmbLrV +m6o3mq2JAkYtglY83OWOqtcf/gKynvkUhMmkM9yFYvOqOGFZh+O3gVcX7QL6srS50nS+PBtQTjyd +sbMteduqpcc1O4u1Wcd20ymgjpvYUp9R7WZyZkJuSyJjzLdQJ1/xZi/MkzaewVcDi9sl4U5pziZS +LVO4oTDFbmI/34CEfftLTadE7ZTMXfo1NLYlkjH/BOP9f3czNOFYyNCUSMUm5peKLiSD5xnD3GdZ +UcIxNZxuPAIvlNxz2AbYmfyyYTg5712IB+G1WWDi4X+azeTo19DmOmNGTsqmZQL9f8uSyWVjdk72 +wjRDuBNrqoo7bMxptXNONmEnuTrJl7CB8dXzZbnsrDFO/qhCHVq4g17qI2vRWH9NCpMb3RLY4Mym +b5617whE+Ro9cf5mSji7Wwn5iWPC5S6IFGCmBrUJOQswedn9M3jX4oZ7WOb171rd5+f5+XAit6Hp +sLiQDIRFrbBYj4POSr0hd3Q66zc1bsuwOibOL9dxJQW5jisplstjxoGPLuFPBvCMHXeEcdH0u9ac +Z1icbNcKJNYJmQ1LaLwscFgeD1RuK+M8HfltYjbLyHQql00nk05WO5DT1OTICRIXsjwxyCXSVHoU ++CYaQNiJp+NRXBDkRkWBlbdx4k0vKou2JvsWpjLa9QsTGS6mMEA2QGTDcBduGIa5P4OBjC2UsUFx +2X0nt6VSnEaNGcmx2MkWBgXO/3HEtAC3AF/utnuUcGVmvJ5jtXPphNnMdc0+dziFBaUyAKdbsKCU +29IFobiQmOzk7AS3rP3zob793VnueKeVau3XkEzbMuVwgQdlJ5KTkgY3l85MTbTKxJzrv9k92bck +Keu4Tq57Wm1HM6f43E0qSCBI5pEBPgfhskEk53Ny6icrETaqDfy2J8v0DjnTTU08PGtIOq47tdmW +5kwOAj4aGkBtcrD5USjbg8cSZnv6L6ByQigGzIJ19VjglJEt3CltrQ2ttoc4u2h51Lc/7VvWRv0M +t3xPyMW5x5wAfh3qr4wpMtbZkBzEMv8iXbbvNpw+yet3ZLGe2FLECJ2rri9Bkcxu1G53UnFnYspM +W1hoQQF/CaX8BaindbYnpUp2b5amRDI53BxtFPpCp5mGsCMX9atdZPmbspO7uLPchnGOHUuk4oc6 +2bRrjkU0X6H44smaz7KL52qyhJIZXYObSSZkfuICG39bilo3E4XF1W+tOcss7lPFBRR2dHQ6WOtO +4Fk5GYXdickYRvNnQT5O+maNTmenqL6po0IqV8QZJ5qwk+5YHpBwwMpwL5veuwDr29/1ofoVTtIS +wN5H+K2sO1Jcl2X9PbAtlUykWoanYhP0IFlZh6NtWXxWPXhLfynrjxIT0qnhci+Aha233mOd9rPd +CWmvskTpwGkOwWnYOLeHjLx2Auf14ODAI/Hze3A08ati/WHPM3nluAGw9GeuSzc0SHqJxh/4R8r5 +MvyaEXy+jFwF3u4sd7TNUw7rL7esVVrZFxZ3QbwKGpZYLDCHJpw0uFgohyThBM4yP1rNehrnB4v6 +CWciutTmBnu7nTLXQFqBxiKDiCbTjY1OlmcwLLlTT5e4+6NVTeCzMuq9ULW+ZgOO8/h+DTm7hX6O +mZMJNwf04aZdOtfM8zz+Drv4JNzdXmUHzzDwbZkWGPgLKxyZ8c4KC2rSI7OClLVNiSwLHz44wGlG +3KtGzs55g5g3avi3gzyGUW9nKrPeRpdxdXJI5ls4R4FJrX6T79u/laOr2SDiAd947lFS2GZg+10D +4PHaWRXcNQm7uLBiPYWwD/TLC8SQThzlCEqsrjbFQWW9daM2B8dusx+qVwEazX4orWlI0eBE5iLj +1FkZp6HVTulRuQu4RcB8pWgeUiHZBpPJgVaLmVgXVu9ZqV6h6VjOfrb71516Yb5TL8p36sXrdWoz +Gx7OSzUFFVO7jcBH6eJTozDvA93TkzbLtWU2hwckUBCMY5E7NT2lrbXVzs6SWQZ5JZ12J2kWgn6t +JxOtCd3ark2mIUestc0J86PAVAe6qelxZGOchWUVMWW53Ye/wlzCXGniDjC6zSYKGFZtQwgRXcqc +mp5opkXESbTWiEesskeznsjUh6CNbLK0WM/KtHoy2cpfrviLKRT7DfsBvWrhsLsR2MgNgKMHN6u4 +xUNaEGn9bJxOdM9XjI/zbNm/FdMIDPWVUowuFmq42Q9SuQrIgsrb8of+G2gXXiAgwyltrdSgaUKp +Ui2nhuYXG01+FdizmnEWno7241DB9YvMfjjPKmo2t611bKxThjBpPY0YZ4xSZvr0EBObmtjt28Xq +xWI1pkNiCEEn7wkUZOBqZEQiJ+ZvZk9RnXWljLjhDH4uYfOEW/CL/ChkZgVYcdO84pcMD9Wk18LF +ObyWkv3wPP7P28rDs2rmNPfPsQVpWLEiq3EdOxttHhvr1Hol8I0l9erEt1b3cdO0vKmXMBUFafH/ +P8Yrmf/Mvvr2N0dm/RrcnOxsNQJ7pXhrLdeCO6zrO7oNLk6r2XoZOSuadDikS4+1gY876f0MLShb +FOzKnQEBRpnNNXvGtVk755ifhQzxqM11tcy8vJ3H2sLGWG0myh0V9O7hl0v20hr1SgyHTtparasL +j9pkVsctj2GhOHpC0WtDNJ3iglQK1hWe+kkyK+Tw/I/jXez6C9/jYd02Oe36S7/Gxf1UO2GcZ2Hy +3wz8vHcAZu1vcYegNmcnkuZnQari3AzRUDRzAda3P5PyxRSW2M3Akv/6VboeMmqLahRo7uCKZHIH +l78KxbTaLTyTKxQkwsgxTKEG8rOQhysCOzXFyY3gjqlMCwruOgM8+F/aF9OAj2b/paiyuJci16+V +3iqOJdyobQ409FZxQZKUk8ukk8m2lCmNw1wMGHg0L/nJ7SxuFdYm0rJkgrlxF89k8ETp6Z+Xmq2Z +50ppNGGq5jX5WeDekU2Yo/YmvSzMytEh0tuRcwHaV05huKSSXdIJriNTjZwLbH8sjxm0IxVyuIl4 +JpuOmn04q7Y53QHhaRpp2zkBbTj+KuRqNfN7/0aMnY23zzBbhZZVyxAwjNkThmEhs5uIx5ykOkYp +WLgTyEGhCMgldhHyuTn+hmy23UlZp8nc+dN9pNqMiel2aTLDKxsxOecukjvmNDnZwnFxYrbp+QUp +cumcnRSUTNq5kF0fJOnDs3Gx+Bhw7XoXRb0dZL0iytrKhmEGP5azMQsF0krabZ43TUu5uifPpicq +nMVdQfaIsCuXbe7WFRbuDZJbgYkINFrvKKpA4aYUFpc+U3RCH82aWzCFnLwXardSSbwi2r3TxdNF +rtNM0yxaIVebkC2GWmXK7YYw79r14alDOIcdKXmhpEzW4QnPeOqUt2KLD9Riba2ZOJdBxYdKwnBt +GTkWOEXt1KS/ZCZ9aFRWppZ6EbaQW872MoX7CRmzBisAzAKWLW+sqjbLK8u8TRvOZtBr19M4+mUQ +WXqa3zjpeSZOmdrM9iw2uXS3JKnTI9sV18mWXYfhyeQYHjgmnVQ814zbAuImuOF6e6Dn6f7iCniZ +TVNcwkAUrY0ZC1VqTfPBBRplu5NNNPGCUpihDG4JdNb4+Xp7zrriDLu4J8iqF3i5ibhrmzueVF1R +D42nnVQ7e0UBnkjlhuzGK7QUMaETN9fhXeRiGdWtsmCZCxRY9O3v3T0RP0dGvHxStcDcRuVatQAW +gx0zcsrU6ettQrelWsm9qHXMQZzTmcvKic/6htnqJuJZx+VjIizYTcSNEzuhqGO1uUnHycxIpWfI +SVMbXXG3ASztzko4yZiHKUjs3RnnyROLoAwPVvgbpTVOC5Tbk5/vVNxveMzMbsUmrZ2RmQFRR1gG +02sa2APDMh+41vzOtqVkK+26hm7KTzfa2WzCyU63WxzukFMQKpnKNQcH75Zxi1x+D48dgfckWKgF +d9bF+EdOmjYpm25KJGWPsLZ5dvcRoW9/3gZNt07Mxhy2qN56qOUFAsQWmVUXJS8w97KMSrW19stv +BjqptlbgrUV+VRUjY3rV5O+RmbSbkOsKlrVBdIanPuKsicgAXyykoDHTyuEOCn/vzpTabLPdJ78L +NUikeHE2zp0jWT/JvD9llkoFlFy9HSXnLlJOfL1HKWLZdIYmJk7kjc2KnYjr5MZPSE8fQRFpRf3Z +QzjJ6OZseRvEw7EIcRqCK+YXL9xjIEtCiwGms6Wn8MYw/Rbd/70VG8Sox/ZQD24YNSpFPgWFeAXM +ckfasgIuJMktU1mGdhoLC7u4tJzLpQImnhbr4pqhSTa54mlZHtxxTgC1Te3AaVZABso3KjjebjAj +a+7LlDLDRW0015kzP8NuBr2Ku5PvVL7NXDs9Ui9yrK/DWNr4IN/gIXr8odsIYSxQp0d+A+RwbAzw +oZ3ltzioh7sZoMke9e6sbJjBkz4GT3Vj4DMG8S7Whu8OFPXt/Eyb3cP0bTOXlP4dTuCkwewyPr1H +2f5cYDGegzdpIV3todXu5DyWI02t+SFaKOD69vfux/TzVibUvDxxJI8H8qqMMXn+8mfMZNNN5gyg +sPzpBHpc4HczBMnA25Cz4yPlsrRlEfZi0cakVHaKWd3ppavaWaby3YuUhxmbbde7zUt+jcDt2/qL +7SvSCZT72RvAqn4tqq02x4c0ZeMgnL8QfKmEN1h6IhXNOq1OKrchvimuFWQEyLO6sxurglLsWEyU +4g5PxUbS5EYnbTNb7oSoqJazD+P9anN23PzqLpTXCrq01EYw7Uc9m/bjr0JO9ng+KpVKR9MZGRRd +PtcpNwVcF8+V/iLTNzxfWvyUFVUVMxfHvEemGBSzquWTKWj9pRR8IgR3l0ova0zkcI8cYRdK50N+ +k+UWT4N5Yout2GFu9BRQUTv1Z3cCw+14R9qugHad1mw6nSMn7kZ0vzsYT8u0omhS07e/m4jTQfGZ +R5kbRgF7SbElFSCOaZq/xDT+PZvo30N4j0KevfxzYWIJ1vQvZXH/HtIod1H+msvfQ+Qpt79ksuff +Cjto4N9DZDLwl+UMkoHuryG7/n1B8sjdX3PZ/W+5JP5ecU1ed/+rsoz7/itEkzj4v0L8veVyANa5 +g8epNmdn405u0kiz+K/lY16tjhfkcD558MfF1zPzncV1cjOyRaZDv9R9b7YYXqSw9eFxJ+cmxFMk +zNAQdm08Uzqt6BFaNxEfm3JzdjI5Jr2fnYrp7JoZZYgM53C5Nf0ifxeXLInc2HgqnTXTSVexCVwk +1+8KTqbDTuWGu7NS0cI0TQbM4qmcz69wJ0l2m/6qdbhU3LCx6IpqjGw/JOVCIu3OzQFdFwXkSd47 +KosL56R1A/BJF3ubViYjLpd8YTfnZnBdZXFDuk5uSiKespOuXFicJGXC8PC2gwtKKa5ulHt7f+O9 +muy2pHjqv9GKOHI/r7S7i6kbdZaykzMSG2q1K7u1WtROyUYXWXF6Q4WFOQdfHDI/W2fgjBAXjDxg +5NbKRSFODApVlOytTivvJ5GLCNZtYe5hioYYiczPBPxbg5IiN3aLot0Ea63TV5mh/sk+oOyq8ubZ +FNMpCiZuvMoplRx+u1fBbnHcZCLqJB3O8/8sNWrTHAupfFClw8lObJraIffx1nuZgp5Z6w2AlLlw +w0WcuT9Q4BSP8t5JJpdfxfMQboui7kiT855q1IcFwhK+S45qC7zs2BFtrn+Lyo4dkUg1pYEraOzt +XJdzucKz7BsDsqmcwU2yV9SdidsWSxsf6dlHMbOwixO6vXGAp1JunOfGBRdQy9MeZP4o5dsKmhNY +ZC51uHEsltzhDM6QqO7lcxJCu9pw6d3RYjri8/8OL8pstflcmuzbuDmGgKmXcN/bTNsK3F2ZEu5r +5+yJvH5gnqR2ze5eOJPLYhbdRJiVnM1KFnJSGYXNMxpP/vxIN9DuLiveQMtE2+1km7mIcYAjM1Az +3HjnTgXmUb5SwNt91Z3iDaSOzqZbJ9itTrpJnhFTYC0zMkqmqIVs5JlMmCdsPGixALXcm4HF+W+t +QXbrULJ7LQtnNl2BCR+qkcGmUBp3O83uNaEZfXMAY3UJwgfQZUIbbsdFFXQ+upetgZT3oxGXdHv2 +z9xR42ZUduB03tHkSUvGbHAVJPChxqdjiaaEExtlZ5MJfdrZy1DLba8OctFFTDKG98uLx21ZXI5u +SyblWQwWJ+v+8/fzd2MBjWhrmlTo6rkMuoH69vfB8puWxE28lMdUQPZSP9MC3jxYmC+bMC5Iirf7 +/fCC9kVYLwNrFs6kXawuKz5m0bwy5PJQV3sDD43WXL4BmeSkTc7cJvle35EB7CsDMpm56wp/Lqn3 +pGw6OkW39QotFXdyw7PxsammtPfUvWdafMx0eDY+3s6Y4zIeKIkjZ7dkoJVdb1qUI32LOXUosHWS +iZgzPcsXYGQ50dOnZbk1d3LRC0tk45+nNtJHOKEuMCm8hGGyvtvIsqSHGSdRAMpTqFG5tMA7RkVD +nC9tmvfalkJO9tdu/UVf9mBePMFmCrfjeVlShhNYYX504oXS4tYXRrJ+pozmfREFFoUC+/anLzDP +Mic4KtbmgENvDSDcxncIPVhC7W4Ynm6T/lyEf0rw4XRbTpb2eK6EjxYX8s8YNW70jClTZ0w9ZNIo +Ssae381uPciIsRP2/ROIjP5yzfavbj0l02bHgNebjOIcXC/WUpAn5shsbLzTqtvRk21zsNpusqDU +u2nX6rTyWe9Bg4fCe5lV6XbmpVbWkhIAVk0G2IpDLv9qmoGtqLtdGfJzcXjmPxD1mrmPUmXSMwM8 +YQmXmiO9n00b5DEUwzTvWKXKg4I8LTzOB4SNCT2zQTaDhmCm5j9CqY/Pc8LnD+VjQoMtFCtGqrTr +YMzR/Kco9fF5Qfg4lpHHhM7bIJ8hu2Gp5r9OqY/Py8LnaeVjQr9vkI+0lDJYqdTH6DVhNKjECGRC +MbZjvq2adh0sFcNqzf29Uh+XlcLlXOViQk8Vcxmym+GC7Y21bKzUx+Ut4fKbcjGhgaV+JUc9LvWa +e7pSH5fVwiVaampkQucUcxk0eKgRZqZmP1+pj837wuZJZWNCvxaxcXNZw2Wp5v5BqY/LGuGyS8AI +Y0KNAX+VEnxLjuGDHYxqDlHq4/OJ8Dlb+ZjQE0V8Uomkj9VMZXG3Uh+rL4TVL8rKhBrK/CI5fMGc +cyS6NPOJfYxcHpMc8I3FN8/ZZUBNJ0PsqWcxNMsLPV4GhJwj8YN1+iel+LkMCI4HsEZ5fqGURvKL +FWHX3jlY1DETKsVSLd3ZsZsUdiNwfMkD35ViZtDIcXwJ5TiToVlMY+ixIBDK4eQSCvwTA86ROKWE +Qu1UrkI9pUW8qJRCzS8RoWaUFwllJ+OyspmjwuzQt0ioYBrA7Zq2Qim53V/icxXmPLqjEcM09xKl +1hIamFXDaTZ6r/quFGeUAzVy2tSblXm0HAi5yXQOG3Ep82M5UH4SgIFsPgAbR64DMKACiNwG4D8V +QPVyAIsrWIuyJwAs1bLeUmotiawG8AjzfAqgqpK88u6es4nVCn2+n1dZmotVYxbAW1KwAyspJ7Al +ff0JDGSy6RywJafLdzJsFqQm4lOJiGloyyqgRmbnW3LOO6kK6P2rFrlQiwzxXsH2ZH58FatSJKDM +UQZhtWIP7V8kZkjE7EMp76oCQin0IZ/P+FuE7EMZtwoBISOjhA+UcMwETmCAEvahgHcy1IcC7cBG ++DRUpLC2FFU2U2WoG1Aki6psIIUZFDYqG0hpYuH1mWi16pVFs1LLaN9UazA5nRuWag0mo6eKGHEj +ISpHtXM09zs7eQKxt1k1smPQm9X6TeThgy29V/xcioE92CrJGEywowdQlwEQimMAN0yW96ApZrAL +Sy2rBqSJ+lGXQ6uLmigvhFZotYqw385FooT6UJYRfNq/lewoykhKspQhSiKhlxgS7Y/gKUOwp19x +0Xia55eYqaxHN3hFsBtbNTyRnE2Vje8J1Nh8pkCCZ/YEQq0ZnGxxA2gtQ042m0rjKlnv7VbTrZCx +7pi099qOmVrGNrsUl5UBXgqwrP/WAHXLqDpORDN4U66RvEyewe8AnKAMzlVKUd8MVFdbQHktgPJN +LGAgQwDeCkQaLOAftUBkhAWcSUSQiF81e6nK4UM/5qG37lXUMtF4eiTffDmd9+hGmEsNGKb59xno +1YcewaqJccNxU1r85F5AjZuNApuy8c/rJVYhR7DP9ALKx1vAHAtgvkGRIyygVwSozlrAwZG/FUCt +ZJgW/4pSy4gRYrlDWOzFESBEoYZQpg/JOO8VvIqNd1pb0+0OViuTlkHFdZIXc43gyLBJHWCqOIJt +diSDLMoEn2aQDgAjWPQOvamAxGxHw8f27mYfotUp3GMbyZO1OVrs7oM3UPxBLP4NcmRlcBCLH7UR +C6CGJfg/BlPAQVz3bLMxEMpgJnELNwbquiwglIDNxJpNAASp/E20rG2UsjGikYUWMH0TIHKuBVxM +8PpK4xMkMVnFYphmnrH7BgRPUvAXNvEkTVKgjTel1+C7AZNU1P4MGkVJ+M5N6TRwisXHS/6zGRBq +lpcUnC/vRl28GVBXX8LqmefKp1DFR5LR6s0B1M1kmulE/+VOUmQLoG6dRLLMo4kcvQWRnDaGGhk5 +V5Y6sxiLukypqOoCibuBuesDQKhd37c3aEtap3TKI7TmRyul+v4rfTJGkPRJXudi/FHSJc/dEogM +sYCn+IN98zcipW9+pFy+VMpcx1mSbeBWmi3KH8x2zlZeGwZ3NWqvUspsx0gbPkkw2/DXPHgXBe2q +lODjLUHvsrW2eOPW9CdLLaArAEGcYkXutYCztwaCz1nAZM0dU/pPABeURT6wgCeYGeVruegoM7nP +tyIVJcAv5B8pARrqgeq+JcDl9cpvqfK5RulOAB4oEX6v1ou2dy0BcruZinYqpewXWpFxJUDlNhR5 +WgkwUwu9zIqkSoB/bAsEO0qADzXTZ0pZxI2cDa3nDqbOyqhlq5OZqXZ9gVLPydAvnCFT3LrtaLHZ +KM6QOe4YBmlWZ4gJzWZQ7NuEb9xuQ46Azx1gqRbRNMRUVYsyTuZ6Yb56O6AmHpUegestnsPvuL3X +n8yS/VCGpUCNWLW96S5ySnCT5Nl2ByAkTG6V3cRFOwBVx0weiJulFNR1cWqewP+kCrV9gLoazovb +sVwAx/aRdllUAjypQj+jlO1ykxVZWgLcSpR4Gi/xdR/oZksM70MBkdM6TfxGKTndYThtsiPLW6+x +prmJVJwthjmqsiuHFquOrYTnpJkO2NFzQ2ZtvphhthOek1r+py/T6U40YrN+dAestt/XvCbgV/vR +hSxjop3D+9Zm9QFU9qdv4vQ7gQ8ENZExcxjDRyoyTgyfSPy5/YGQ90Jp82Kwr/uzhqhbJ+gcvrP+ +WR/ADgPo0ipoXy0JYfCLMGgZ4GPwh0Q9NoAMyunX6ytN1/vZaPgnpkjCTE342CRMYTcoaJWHPtFm +O4UuVeJJ//CUyQm1VZMDxvy7PoALdspragxHlGd34jqO797bj/5+x505U3JaIce+xzEUxWFTqgN4 +YmcYe5i2p+F8iFLybxJz2L0B6EWvBQRvKQHeUcT7Som0I8+VAM0NrHDBJprtlOukYoOwVMV+XKml +4keBBRbFuLABqBF5sUDM+TmyWo8TViuDmuFGWh+ji4WRtQsZZYCLhU0rg7Ljxhhe4F7FmKg8YsPz +ioulpcYO5HJzIB62eP9nIUPxDD63OGX+nKHWWW4ca63JPQLYbRBQt7SSBhDH8xK1eBBNbxjXJkdm +8LqU/Ajjyl8rARZVQXT0gBX5qAT4gQlBjn+L/2kqcYFSKvIJK4JSIDoYCFaWAmdomoehqxoY2aYU +6BpMZZfvXgqs0BKet6oPKQW23BWojpUCk7h/F0yVAh/uZYpap5RFfWQFTy0F1miUl8QSMkD5DaVA +fciI/osVWVEKHE/GK0uB1cL4g1Lg3X0M47VKyfh1K/JDKdBnNyDyB+uwm9rYJwryg2fLgPjobqyM +iLr1MMOxv1JyfM+IWq9RXpKKKtm6NO1ppcz2tMn2oEZ5SZptffPS8aVejesgpWpkoSheFBv7kfWh +VvdWwCilLPNFKzKnFBiwOyDq3UfTPIwUHrm8FPjP7qxzQQp2FcxU+MIRRgtaNnvKFSXsKYt3Z0+P +A1eU0BofYZD2foXsm4SGADVtsjXfRMgz5QHszTg6K0bQOZw4hFP/DO4uoX3fPcQ/+lGIfeUtn1iq +Imw0sliUHPCs7MtgDxXlWRFltAQ5/X1WZJm1h071X5LgjXv4y8k60XYtp175X6jUq3IOWCnlfCuM +4wyyyv8Y6k36Vwrj1qHsi9ko3pbgpUP95USTade40KXK/soxxdWJAu+LZj8mH7m/jO9Lrt4mgIY9 +gboVIXKP42cpu3FPo7oT5W3L1+6pY7RJBOoGclHvxrGolJK+qejzBL3dPxRtEoG6OUTHM7hC0g/6 +Bz0sHcNrKuqHSmlXH5eIYzj1H+oYXtc0D0O7Mo5hBfkgWME58r6mrlsqJaN5pZF+AWDjfwLBXQPA +JprmYaYCWDQoMj0A7M95pHExXWHjCX4uERdz9D/VxSwjRMr63yhT1qNKWdbFpqz3vbK6NM3D+Mra +aC/2BnFn9T1MWYtKpayxe2lZRxEiHa9a23BTpSxraan0t54a5SVRLxkgODMADNS0oUqZ7fJSBI8P +AP/UuClKmXZFaTUPDG/Ol/sfTUwoJejTEil3hkZ5SVpuoXtzNKTVD0KXQt9T6hl8FLijlH38vb04 +iHH0xh2lXKD1/leRTTfbwgnYT1U+wVAfo8eE0X7/IqMM8JiwuY1BHQ0fkzcSbbu3Gfh+KOXAt4ih +eAbXB+gY1u7NSQMHvhsCtObd92FPaItGHdfF0wEOpmfsoyY9m1KiLiPbS3GcKhlW7gN2IMYdmcFZ +sq3Sg5KW3xQAhlWbVn6tNPJAAJiWT5ijCZ+ZhAeZIN2i51j+BDZRSu3/WirdYufh2i1qNM3DsBlM +tzhtOA1MzGdnBQ1VSkZzA0GOhg0a5SUxP0dDDrNdKtipAbHLj4arXW42gosdjoY1PU2dzgrICDh+ +hI6Ad4+ATvcUYEa9z5nRiLTxAaZuOyilSJcakTbRKC9JRZKajNW0qUqZ7SqTbX+N8pI0m4zry1SM +ewIyrm89EpBx/aiR1FHRtIsmq+PiajWzjSYaWdXcQlEsCNBsbx6plfldgWUKpFQLAjIuvkcMFf1H +N4yIJ+Ni730pREEKSoB6ZfXVpKKy2WfekcL321cHo3fE9hYwSMt/R8zuFQbz4+I7AY6LFaOYQ944 ++E6A4+Jeo4xz/0PM//hR3Wy7BqAdrxWGT43y2ryrxrT5WtPmtaO1zaeP1jZHrQGYNj9tdHHt2Jn5 +vsRourU1kQMONPU7VKnquCaeAU6VpwTvHw3UMJeczuPUMna9daNV9TtqvoFKqfrzy0TjfTXKSxKN ++/0Kz0gaBmKm4m5Tai3hbq3FbXd2/XQWPbnzvN0Y7sq4Y1M5Jzkim4jFndF2ayI5C7X0DAeN8XPu +2z+aadNDavPIfJcyHzLF1Ndawp1VOf097IdtAjh1DFfVGeAwuqLnGXJzLcBhfO69dD+gblgtXZcd +n5TLIr3qu1Icvh8Qao5l4czcPoDL9mMD8bRtWS1A5q7PptYTaEI6NSaNYSrN6KnrSTXXoliv7keb +oSBzLUpSOfYvqzlKnrqbqdyeP2g9rkuEqz1WJb/YouhnjeWqk9VL4BKLlvkEi6kbyC1bVvBqgdXs +D4SSaTc3JdeCa6wrvy/FvvvT42Y83C2C6/TjbhXcjcSVvxUAlnFXGMC1lgmv0/B1Gh4YMcr7n4Yz +Gr7N8mkz5jS2xXkzRXZQsVrreet0r77c5rVquN7anLt971KirJNDf17RqjuAO+0RoMrNoJ7pfp0W +81ZP1KWcqw8rKiHUJxPFAG4s73cAYM4bBhy/cQALGWzLYCoPOl5lgFdMMJjFV44Dgj8GgK+V6XdK +KfTOG6olr+KgXov+XamlleTbouWNuGhiVe4ZB1Tll5sJRoFD7BE067XjpAWjLU6sFXF2m23H52NG +debQzI42fTzNgfvmrRncJmcd83koK7O3V7T4d5RS6Dstmb09IKDNy5BPXOMD3WVF9ikDviWo4Gvz +2mYdVdk43Ch50r8N1ZqyFi5rseME7nx4K2qXVUwxKq+JNkZdyqg+Sa1s6tlPSvGxL4q1Td3zcyl2 +mchjMacjbpi3M2jzQfTcIX0DuH8iJ9l1nJhkcAyPfdZN5KyZE9lvVc7flVIV84wqtptEEFXxhyYG +tTIEzTeqOIKgcnJaVAdJOM/kvogJf6KjQZipnN5S6lPPY7KB8MEkwFThcYt12PhArosSrjGTJ2QD +Yn/GUYeJVHwMnpB8Rx+oM/TblHOXUsr8vBHtkQM33FnYfIOxWjNc+Z+ilvPZ6JdSemiyT6LR+Mri +8Lh3UeRB+Np63gogy9h0C76TzZSrJm+wq0rpaj1dWnZkRpEMoT7pFnxv0QxWTgZ6v6WwEoXxJA8/ +WgcigB5TiuoYT8ednJNqR71CnZke543lJlOLMwsIs2sfPoW7YKl29GjqF8CqKZAd1QZevOtBt4q6 +FTQlF7VEHzmVI8Z+ZcCw3gB59YrMKAMmTQMi8TLg02lFgvTtP3VWxhnuuk6Wh7Xy5ap+DUIwR2Xq +F/Nk46mNVeMAWz3UL4DEQRw8eYkEW7PsJQzbLrZjYOh0digXOzLw4nSgblFvWrwbRz9GJQ+mHGUZ +Sqr8z1VqLYmcXgZ8eTDQ85jsQFjjmMFnvPKtLd6Fd1zXPMClEi9VDiXOehJP/6J/AHsf4meTSdoJ +U2PNXq/Z+ii1ChVuqR4QwOKi7I3ptlTMFVVp/mGaL96yXvG3WcsGBPDkITzqb83hdrnXu+RQbpfj +XouP7m91mLYsr2Ass7gFb1raF65bRx0mcL8MqIf9G6hbvRH1jOXyvsDgf6jU8nN4SWIT8J97cTeV +ARP+A0TuKAOOmAlUP10GPDoTKH+tDFityGVWr4+YMfJVGfAzYb+UATvbeZZdmxqWj1nC0rWVJXdL +hOU9UWW5dDODLGIZiAHCcv8YWRZ8UYav1bBTszBH9fZmplh/CeAV6UWLeCfgTE29SWmoHa9J6spm +AL2/VSY7JA2TUDvekH6+fSuT/6O5TlMaascq0WZSkg9rNblOUxpqx/uyn/o/SZ6a1mSloXZ8aPFh +5l8keTPNtb/SUDvWytOXI1Ise37K5L5Laahdfdepknyu5npGaagdX1mfDAjgBUkeohU6QmmoHd9Y +fBXxprz60/tejS4r5P7Oqt0pAFuS39Lo+kLZP1g77hTAc5J8oVboDaWhdvwkNvpPzq57z1dtLVYa +ascvovOrmbx+U0bb3Fy6lZdasVqzzO40lfc6VQKYU0LX+FUOCBFq7v9jbgm7+7C2oq7KF8V0ZO0M +5iibT/9bxK4qE8UtcvWVB9J3yBbh9W0che1WB7cKy0A7D6xa4rhHgtcyyIOcxyR4dgfdk5NrxuMS +PqbTL4BePh1tJ5PyhiocZYr/42hDrSU8cZWp/2ZcyzzTyTm2vD9jM84c/jOLYZjAW7OAkG2mkdvN +Fh8gU8rEbO6nbc7ZEwYx1xJGLN2CWxaOExuEXRn5HXMwPBi7MTzhv+xOch42WaX6t1LKtEuEJ+9X +ExQMBIH/aFqjUmIaIr2CwNr/AsHNg8AMTfJDDo8MCQK7HgVU7x0EZh0lhZKhh2rSXGS4Sy8yNNyi +Gu9PP7wXuaEHWXFWQz4JxbUqJZ8BItgDRymrIzTJDzGClR2tgh1wtDDkeVVW0UcrJcN+oo3FBAX3 +CwLHaNpxSonpGzkoCLx2NBCcEQSO1SQ/5ODI7CCw5TFA9dwgED1GCiVDD3WC5iLDfsLw2mOU4VxN +8kMMwzc9hj2PzTM8XdGLlJLhDsJwxLHKcIEm+SGG4SnHqoTPGYZUy1mKvkQpGW4XoZHUzeHkYX4Q +mLklJH5U5MIgcOgcIHJ5EDh9DlB9fRDo4o/bgsA3+SxLi7L0OU6zHHKcZpl/HNCDWYByNvdqxe9t +mvg4beLvVKoflVI608RfkxMNZoe5VI4wqd/KyLm32G2ZdsZKpZIXMDXykFKjg+eqePPmqnjL+YPi +fUXuoqhtlU0fpWS3jyhq++NptFTm7pr2L6XEDBfM9DzGSxvlw4wQzGmCoTa8tLE+zHBfJxqt8f70 +DXUiL3284inPCNHw/cfD9Mf9NckPMRped7x2ou1OyGt4meptiDA56ARlcrAyOUwpyzFMTj1BmdxX +YLLOYyLN1Kx5kkolrzbTwK1Ngw6RZvryBG2mbU/UZprGH2ymU070DGmpZtlDmP9XmR6r1GNOJa8u +Qs5XxCKlHpL2X19vxNhDxLj3RBXjC5ZO+9/mJEDEmMo7sMJ8kWYZKmJcokwvV+oxpxgripDLFHGb +Ug9JMWq2MWIMFTFOPknFuIelU4y1/EFt1J/sGeRrymaVUrLbU4xtCjGFoVpHtF0HF8a0Y8xYNkqp +jmk9M8B+HNNQw1EH2I+jzkknA+V0eDNVxrG96DWNy/Q4lSgnCnFwL7pM9KC/pLCsXx9NH6yUuP2l +rnefrHX9/GRt+a1PKfgP5l2q5e7fiy7K+KfJpyj4xAJ4vRoP2S1f45la7lVKfTUe76/xeNb4rlM8 +z6UlT/D1T49TVDmxJt37J6U+WdPPVErcRKnxZ6doJbY6VStx4KnFNa7f1ljDRF+NT/DAdxbAhRrH +nRw/jYsuLe2m40wLW0t4AcqqMYs3bMMP5X56KmDutWzD2xYDT+MOrZ1q4msqsQ33dU47jTMk7PLA +d6X4gL9z2P25nQKYNw+Qs409jjXsRyllIf/sBV6IknsAQzXeS+fm7cBevAdAg+BhdIsiOpWSw7+C +3GpPaoyXwrw8EmOu4BxTbkQpc42XXOUa46V4uZZzgNe01UqZa0DkvSCwfB4Fyp8g9O1PJU7lg5Vc +LPeTx03RpboMzjWFezrNATOpla/mAZ6CZ1LB28/3FDyTCp4+n9rkR0pitK7TGGyGw9/3z+f1nG1l +KisbcOsYsZQRGcRP2DmAsaczM1rYFEedTmmDa4PAYyrTM0pZJSdSVg7cfDoQqS0H3iNahq4PFPS5 +UoLjPuNhpYfHYqifayp46PGGehVtBWax/N4LOH/HXIu1PnwB16SUNDcYp0jUUwvYunwAYKVy+l0p +SzzLkrv/tQtZiT9XOR+Fm6kC3KTUEyQHXCwlHbwQoFgXW5Rr3kLqiAq+VDaIljPcjMsk8NVC7sHx +Dl0Cl0vM9ot49seYDK6wqOSjF5HBYFwp7JYtooSi5lYVwFXKelxmiZ7fX6R63mgxa80pwiJFnaeU +6Cv8O8/GvvRt7V2Kaj55GMvLV7IVeEDkGLuYdiG7Ll0W7eooRnAd86DcR3uYwU48ZI1vCCB0BpdA +CTxh7b6uFE0MHYGnJfAgA01tqYF41qKvO+RMINTZyq35p6zPGwJYxQguiNIxF89Y328XwJCzzJ4E +npIsqFtGfbXgeeF42VlA3VLeCkxwOYcXRJ5/nC3ychn2kqwoz5UIrsNelvCW5yhTg0fdMt4izLTE +J9m5ZrwpmI/P4QZSH3JqSuE9KX3luaIgcya5ug9EsU+ZK2g9zqP2tygH7j7B6PEppdT+C1ZkWDnQ +LCC6gpWa+KFSgl60xBdcmAeFTzKc6pQStNKKrAwCz50HCHojPu8n5fZX1G5KiV5lRYaWA2PPB0SA +xYKmAOcp6kqlRL9teD9CNL1S+ALW6YBy4DpF3aWU6NV+i2LXdXkShjlqRpNPMcJrn6mr35G2HcOP +Fm8z73OBaQLZa/rRunaXgIa5y/ijNC7qZjJLAj+JFWYpDMp50WPZjkb1P1g+V79Uy71SKR1vwdXz +2QMv5S5FsBI/WnLP+UoyF8/+syaWq/gE/WaJa/+lWxJL4GZEqfcAUTSdap86eBRmat6VSlUFdBvL +S+iJX78A3r295XL7ZqsL2RXwqAQmX+jfBxCmulGxWhl2nlqk2xp5pPZr2UQ4f/3Mg0el0tzSm6O5 +flDqE+vkUoq12RJ4Yp0sNx9mLhGxFkjgrCV+sWyXG7ijBo8dDJxmpJmv1ONrXAY+LKXPWCW8GToH +AVRfJB58jRR7lQTsRnxaSi/6xkX+chI5h6/7niEGhqVaxMbzTJFeUU0p4LfSr3YJIHyxcJ4T4Niw +z8VA78c1z69KQwkcF+Cgd9LFPCGijbViboBFr75YOni+QdtSWceONnMcHu/kmtMx1GvB45WqAIUs +fX1f32pzmzFTgVfO9yTeRrbb5VV52JRfF41cwoej0vx+/6ajBgZwyCV0oU4HNr/gq1LMv4Sntn0Z +lY45g7AFIQ9cwhPLRX2BKj4stA2BQHm6HFjRF2ARm0dOKgeCSxm/hE/W9TPxW0RuLwf2ZHxB6mQT +mR9kJxMxPgzdpcI6pxcLTRDwL0qQXAoEWd4qxX6olGXvLWVfwjKCLNtLW1uEoRwvEoMy9uo5Wtgd +SnkiVAqUXeo3B56EDUaXQv6z0JOPfdGqSQK7jtkmgKGXmuPC3XjQdgwD7diLh4Kf8neHnchhPwYH +XsaNrkwihXEcrTouA+om9aPPx2RGLL+Mal5tYg5nTNnltJAyzgFnauFnKbWW9OIVRp9Wzc2JwViq +iFGL1hPXpbjjLgf3N3MU6Vj+jmdwFEW/laXlrZHfFu2wW5y2DGYqp2+VWqqAFDDP4jdHPyQbmsZ8 +i1w3ucIvFxm5fC8tsNhI9JFSH5+zhM+MK4wmz5aztSsZSrk4XzaoX1+PaU64zpCOn7KTwBmG/SNn +Gupjf6Ow3/pKmr4L3CgcJ19pSrtJSnuZoZhjx+RlJI8IouIqNlF/6Q0deFHiJl4FIMg7C2O1vBlK +aRWPWCatp8qwg1KmvegfwqgVUwGsVswBZ60n9yqRe+5VKvcqkeD2q4zcb4rcH1OeolYzbOOYqfzW +KvXpY43w3fxq5btG+DZebfh+JHzvYyjdgrVyuPDl1f4mZb+YbKdavNcL4mwj+Wyl1pLtxe1k7VQL +cLM14JtSDLkGQBmvS89R1HKl1hKzRurVjysplJ3Ki9Oa9q1Sa0kvXsf1mXurfDBCJgE4xxS/6fmG +Wkv4cITVe2eNn6w0lMCyUvarBGXp/ZlG73quyRZK4DS5Y7TkGqD3Ok3t76VmME8c+fvXAKHmRCqH +ywNPDgxgo2vZSQtNkP8MkDvLldeWoF7luukSU5DKV9MMfB/g147GXsuGYIgjxVHXmtkJfpIg6hbR +AjM4sYypr7G4uoF8kkGEuKeMQhzDp5Pr5jA2g0cF+Mh13Hm3Y3ikjAcO4evpXep34uOkCawq67FV +AAuv53SUUcnBeLOMH3ivvYEAfFLGD4VNv4Gjwc50W3zA41vh+ygRKadjeDZrz8KPZdawAHrcKDoo +7yoH5uwMUP8/BSIvlQPDbgR6reIjk+YxqBtUGfcrJXSVeQzKJRd5VupbTSy7YJgwNiCU8zmqLuU/ +vwzBD8qBXZcYzCFKib2yDMFNK4AnNe4jpUxbVNZrFz60HdmjArjqRqB6nwrgC/7YrwLY5iaOYSxo +UoOpyBNGuqlMEOlOudiUuEgpuT5V5jMBd5Y72XGdbLsj7xBxYujSpv9wqcnqmUA7cH6Qy5MlN3H/ +iVrG+fIOq+cZIZ/29mI2upnnO7hEkve/mYbRwBivjHslYTFR/IbO/RJ8XYPjnBSWS0xomZqXwQPl +1NRqrewlQdHL3stULyfyB/Vy1zL6PTbwIK3DvkpZ/XuD0tifLQMibOzBt7Dd8m6JXxGzo82OeWu2 +fOwVlxpN1F1uqKeRKLAmOH1YAE23UCO8j7QmOO3bUtxwCw+Xs44jb7zFJ1KZ4K3r+4R4FPXKdIcb +ipkbBZ9czo4081bohtDJ5ZyPPssw3wE028mmgZPLuSc04X9AyHb5JecxWFbOYfJ5RrVm8Eg5r7bM +u429ozndlow1O8lMPIpnJeNPjOcDlfKk5XNS4u23A6Eoni9n9ba7gxkzdgovlfOZzcUMd+Jl+TDD ++XfKmONG7RReEX5Nd3Gi4B7qZNNODK9L3MOMI0bKeLCCtdrqbm1dIx7q6ndhDfjFSJwhUsy/m45g +JqPTTU34USJfvFsmBWdV8G1xe99DxLCBFDAx24kmbdfFFfJerBPuYWQmiusr2CZPMdiOGyXbb5Kt +ZhAdCOd1X1fQ1468l0fD+19vGuLfSqtyOLty4xF8z0b51ApgziDT1x4pr26uANqZKZirAC65wmTs +Ukpje768+tQK4Np7geozKoA3Bd1VDmx9pUHvqJToX8rFNPe/r+CHCB2kkD2VEnpihUCP7gYdqZD9 +lRJ6soEu80GXVAA5hVyglNCFFZFlFcD79wHVd1cAB97PTvdoBbBCa31WReStCuCE+4HqDyuAO/nj +mwrgU/6wKoGBfNdEsLYSuOcqU8WHlLKEqyoiO1YC0eVAr134/J+BvqWQj5USek0FgkMrgU2uNmy2 +Vcq06ysifNvEfcuBiF0JfLmcs+5KYHMFba2UzxNcGACkEsMGm6a70VRi3ANaiWP5g5W4lT9YiQ8f +oAuZXwl8c40p/QelLP3+isiVFKyLIOrSvs6ATlZK0C9Glwd0qS7vFvRNlcBWalmepRF9dmWvJ0Qd +BU/U6rRGk9kJ6f0cO6PvGXBHNrel+G3mYeotPlTquSR2AuB2eYvn511Az06G6LU5kb5D4rd+kAFB +3i0Rkx+kgx7MWHazeyTyfEauYGQKXRKz2UN0lMF3KoHLtNiblLIG91cCZT+zDjcaZYxQai2J9KwC +xj8EXiIaBOtpI1Chpimnw5YheqZmCd1kWHjV4p4aVlbS9815yExAVlayy/7vIW6eOK14W0RcQxHz +rpyv104kHZl9oV5ZfqHUY83X/n9aSf+46cPmBTKfinjjGBIl4VPhfebD4hI/q6RLXPkwTKcfquym +KaUmPquUTr/NI9rppz7CXrRZFVC/qzHALyt9tecXRJzOTNZOxYCbTcX3W2ZoXkqHH4ddUMU6X/gI +b47j0iouYN6VQDIdx2VVvw0IoO5RoHxQFTBTi7q0qnpUFXDoo0D1hCrgdP44uApY8yililYBSxV6 +WVWkowrY9DGg+tgqYBx/zKsCjuWPc6qAW/njqirgQ/64owrY5HGguqsKOOBxIPhMFZDRKrQp3QwA +3zhRaBYny0HKSeVMy8zUukZuKa6zUf6LVRwrjnmcr+qQoUFjbmGMO8udIl8CfbHqwcEBfPA4Jxl4 +qYoTvY25z/1nhTY0taWig1CvRTYrVXWbp7lfkZIPeCI/KJmIYxjhFfyKlHsLo/pk8HLVZuMC+OCv +Ch6EOVrWPrdtqLpvSaEDnixU18ScyxivVLwlxW7yFI0/g2+raL7nM1TQLb6r2mtcAN9KLBaFqJKJ +T+uIa3Kgd/R/RoaTlFbJl6RdnBKi0mEGPD6dSbP+tkoGvHOeptfjxsTWt5rcOykl6KeqXtycMKdM +9ZrgAbi0HGhWUEA5HWf9bqZDnBGSgefJp9VZbs/3s5k5uiJ+rZIBb/ozOm07TRCfVwFdilgUilSE +gA+JqAkBmzxLQbkjUaXV21gpBT0rZHYlZAczpAkegIJmdBkHGYM9DZ2rSLJYEOrFZYOZRh6QL22F +Qt5SSuhx/tJe0AQPUFRawWT79uf63s7Kyqyf+eoghqnVNNxudK8Wyx2ex0PRcQFc9SznojI7f1xa +8XNGaNcxMVs/R4jXdR4PsetMfk66zhOS5cTn1E5MEHXYnan81sizAnjGA5ggTFNldjeN+URImqrP +86qaQ573EMsU8WxIpt7zmRDcMgQ8rdVaoZQ6eyEUGRgCHiCboSHga6L96mlutTM8q4lmU/kXKw9T +vfxPqbVkR1nfNwPvWSeNC2CHFew0PItaa9GBHrxCHea63QFi11niMOetUIe5nD/oMDd+geNf3qP0 +1fJTTke+9C4tddO7vNYhR1k+fyel7/8CwO3u76yFEwJYxFAj8J2154QAHnkBCKXbm/C9BEMvAsHh +IeAnZdrzDsOULH+xIk4ImEwMT7Z+V0ylD7PC4kM8IaCXRg5QSgZzSyJnh4CTXgR6XcSXV5mTyAMV +Mk0pob9ZCHJcdzXuRKVMO6lExvZ7XmQLXxMChg0xWjytJPJUCFgrCakQsEgTTi+JnB4Cdn0JEAlm +8wcluOklleH+O009n1LKcl625OLEey+xCQpt0Gq3OHzTVv1dJsvTdxvqtXoOuLmEmu73MlAjmxC4 +WW4kHi4RDNEonnyZRuG0YlkJHd+2rwChEbi/ZMgnpZj2CjsAHz7mG3EntjvZpmS6A4+X7DkhgFNe +YcVfCgEz9zAVX14SWRsCnn8FiPwUAvq9CgRLwsA7KuMnSlmtmysim4aBw19ltWTYXqZs7i4RK1z4 +qlrhQ/xBK9z8NWLzVkgNjGiLtjg5s8mxWjWw833raeJ10cSE1wDa3OtSueMYiiWyuVlmLMbrctLy ++Gtc1DUKXxcrRSk/v8ZTAerBnY33JCr+OsDnYN6X0I2vg3s0fOxZ/M86iV0tsQP35FVK283hD9Fb +5A3qjWqZs6fR2xsloorRTBB99b7H1GArpdTXuwY0n6DybcNAl+Z+vySyZxh44A1dT4zVTAcq5YOI +NbLyKFvJUYHLk7madrpSYnoZb79Co1YrZdIm4tQOYHbTV06/1wi4SCkF/KNE7HTxSn3XziOEi/0P +/Iep55xSsf8fiKDZ91+l7Dw25/nYqdn/exXbXDr6JZp6uVIW+rspdNEqLfRhgftMJGNH+ajdYAxT +q0gtN6L7+skjpewn369iPwEeKWW3aHwTMM8/PCInXe+/yf1EfhHRfIQz9ha7DV6XL3Key0Aj3ihl +v3iLgVw6gw/kXbxHvg3UZf4hE+iVwupuxqxgTDoZa8Q7kuuPt1nPupp/cgMBHwnbwe/wJi5jWvCF +ZJ3FmHWMcfCNxNzwDvMhOCoM3KlVfFQpFfR6aeQw9sB3tCt6mAd9mOWW2F8vvgfRqPotTX1XKTm9 +USrvlxhFVJDv61ujiV8qJWhVaeSEMNApIHrnXzTxD6UEvVMqlvLTam20fd6lXZ4eRh4Vvt80EtGr +SyOXhYHsu0D1dWHgSkHfFgZqFbWFUqI/KI08HgZeJ/rFMBB6j7zfDgPbKqqfUqKfL42U9gAmE1TO +AWfgXsZU15RG/h0Clr8HyGiz6fuqmUma+2Cl5LLCeOgT32dTSK2v1tSblRL1han1aqKCPXogn3iH +D/RlaWSXHsA/P6DU1F9AzbVWKTl9YzhdLaA9eyCfuIkP9G1p5PAeQMmHFKqoQ7huIp7CHMWe8bBR +ta8/3Btgf0h9aPrDvQH2h9sYkueB7pWndqvXeP3hadn/bmbYeE28KjHLJAYrA+wUPzLATrEqwIFl +n49o5ikeFyfwVuDdT0qRK0S14G0po+JjIOTw4f13JPzwx+xKe+lU4VdhPP0TLuD/RXZ4T8o9jTFL ++cR9C76QfC9+Qh2gLsPH7lNORyMWlFGozfi+z7oVjG3hCHiOPAA8XmJrhnFPgLEXSuwcxspcoOIB +o7DeStkiLwZkLvA/AbErdmriCUoJejUgXXHNp9oVPcwcH2a56YqbfkaJg009gJM1db5ScnotEDmu +BzCOqHL2BD7KzYRVAbH+qz4DxPq/EATtiIeCRKwMSC8+4HMOQuS/Ypix+JPLhOcxksBRvWZ4PoGj ++uufw4zqobUqf2WXKkIp2euovvdayl9+Wg9gUoHNDT2AI9cCve7rwWSZ4R2hmY9WSia/BqTzXbFW +O9/rLJGVSCtotlKCtfNt9YWwfLQHcKWmXquUqA8CpmM+rXGvKmXamoC4o8nkIMV4iW/7QB8Z0El5 +UO2DpvabKiWnLwzoHgGxi3uJ2/pAXwaki38hoHd7AMdo4ulKyen7QITvFN3tS88PPKSJzysl6CdT +3Ow86CdN/E0pQYvKpHY35UFeYtlDpgIELTagdwkqp+CLtNXOLhNh69ZRDs6Ed9Nco5Qy9zllYv2H +ElROf7RCc19QJj7odCZI7jM115VKmftCk/slgiR3zQhjeFeb3MGv2LJ5D9Zsu81jsukOLFXH1fao +qYfPgc2toAMb95VxYHMr6MCO/QoINSbicSeLk2SX/DHGcAjWud68Cm4W9via8z+nw4udL7FTTWxn +YRI8v4Iu5GTGm9fmLhCm734NSPeZpLU4uUK6T9032n3GfKPd5wmVf4VS6kK7z+xvWOVCnePZdMf0 +dLYFc7Sumz62Xp3fkDrf+I2p8xtS528ZMhUB3pCTh39+C5T3rAYWqXxvVkQGVgMpxu9ZDazW+Fcr +IgdXA7d/C1TPrAZ+/RZm2D9PZbhCKeV+riLCYX8XXqmSGeztmviAUoJeMDP+doJgnNszmvyqUsLe +qRBHdK3A8s3utNvRNp5X1GvVb316PRV8JSpY951RwVeigiHf8wWCyZinha9EC0d/T2PA1xVsw2Xf +8xbmSF5rmoWfKm6aEED9D5zO8yUWLThV9kSn/cCxCKdJ4GEmd42RQWeebMYO+hGo6+LLBzjIza/k +IDf/RyDUMhhnSZ5tf+KCwXUOwXmSevRPQF09H1unRWOJbLS++pM0POowXgR8RbIO+plbELksXpdg +jMEU3pAc5/7MHNJyk8abfvONabmnftaWq/1FW266qi6mlNrWljviF44HXJQsUy7aWrcwATI569Rs +Jyhl9m8rZHJW9atOzv71K5uWvvp+RT2olOhfKmTwyeRBj2niU0oJ+rXCeOGXNe5NpUz7rcLY4Aca +t1Yp0/4wNnh5nvmvmojHjaUQdHylSPBpHlShiWGlBJ1QaSTYVOO2Ucq0EyvNuNWmcScqZdoplTJu +bfmbjluzf9NJ42wFzVVKsI5bXb+x/URjZ2vq+UqJOrUywmeitvzdU6uXeLkPdJoBzc6DntLE55SS +0yum5jfmQV7iiz7Q64bTaoJk2rBOrWGeSYjwCxCSMHOCMTZNSDFBRt/6J4y2t1fKsk+vjLzeA3hS +QBxehmjiMKUEnV0pQ00t+KZnri+WahHnV4pzmc6EIF+NfYHmulYpc19UGVlUDVxMkIxgmGgE7DJs +P2CCrBq+0Vy/K2Xu6ypl3tTfssy86USLG0g047ueNNW5TynR96l9PKtxryhl2v2VZhDLaPGPVsoQ +eBcZFryZHWu3U1FnlHFqiXRqvJ1tQZe6tOufMWV6I1oz8H4lx7DfLUu20N6v5Pg2qMSSd+M38mU3 +74s3iJVYCLl0QR9J+NwSC8ELqoEyZbm9Uor6WWXk9mrgrRILkfurgZpS1r4aGFlqGYvsqeiIUub6 +0PTh9lJqiGs2L9HPeo1xC9eSI9dsbxLtq3+rty4fNINHQLsORpcWMek5r+59ZfMyB4RZ2Z4BSz6m +EqYepjEgi5Ew9zFvDlgINWJjOvP3AhbqlvFVF+JWt+Cell1mIdRq3s1/VpmFunq+RyKKrTner2TE +HEZwlMD2ZNIjyE+JoK6L0QkMIJNhQQuhFgxgHpfpMs8f8awRd5JSCr2lTPPvDlpmKPQg4xRCJeo0 +/3PyMapu1tSkUjLaSnzV1uXUNDvEkZrWqZSYeukbkwVDez1B005RSsz24kNuKLfMa6/fETCbzgOd +6QPvIA69V4VlVtujKtgduTSuPxDCrb84uc4KC7Iyvr5CjeVWZXKPUtZTndxXZILgm9XAo5r6jFLK +NyDybTWwfaUyekyTPAgZvW6W2NMr2TBS0fcUtUapYURneRpBggmqMVUrJWZnwdxPTKE/5u1xsGeP +MzXLb8+bBrYK9rgb7XFdpbHH3WiPQ6os3Szajfa4qErscR+a0sNVFupmHugtlvelKY0IiT2O4u+2 +kIW6FUyPYgxt615GDJR3g3B3aByZfBFitVGXYXQCU5hxm7DY4xTmmRqmiXDd+Z6KvU4pazxK7PGC +sNqjB/lcIVSv2uOz5GNUV6H1Dislo9Fij+jBwmiPtZq2iVJi9hN73FUwtMftvTSPAhgnTXB6D7XH +LgHTHj3Qbj7weLHHb3qoPfaRzwLQHldMNvY4WezxkGq1x/nVakbTlMm/lbKeao8vkomxx7imppRS +vilij2U9lVGzJnkQMlJ7HNqTDSMVPUlRpyk1jGiPLQQJ5iZNu0MpMdMEczExxfYo2zSeOcoyHcAf +L65njofTHF/oaczxcJrjxjWeOR5OczyyxoLxhnEazhUM6nRY3i3/mUSgjZY2uNZCyGzMNKKDMRcX +YhLoJIOFvfKYFsyi/Y2JWLpRc5M8FDM7YqGuhq+M4ZnOIjnTuZFxqxmXMC+qX82IpVM5R8Ypch61 +Zx21aVaKndyXAHCiUqqqVZaZSaLE2F/VtPeUEnOkGPsldWrsHuRthbDt1NhfJB+zDPlQUz9VSkZZ +WYWU9ab74xbLwGnG3K60ZKowozcHyzBwlSBo6nMU0Sbd5A3Gyw7OgBdMmw1USu4XWsI+vBHZc7el +SzNfaMlysWkjC3KGcv5GWpOxmnuqUtbkZrOieoZczG5LzUFGygst2W35YyOraLflYs18g1KKssiS +DjR4Y+1AzsZq9x74WgWzRO1A12/MlpL53kua+ppSsjxGpntvEyR2/62m/aGUmFOMR6/dhN6E6ouq +aaeUEnS1Ae2bB52kiQuUEnSTAXUQVOhC3po534NeMq1wvlKfQ/9UToeu28R0oU/lWOwrhrSX4FM5 +zhm6qWWWzpNUyWtLZOncwngunZceZKacunS+dVMLsnT+aVPLLFs20bK3V0qN6gJsp82oBy7AdtfE +fZQSpIsxlyC12XGafJBSamJdiVjVVQJbb+nsaWKp5nBeMxrxaeI70cTazYwmvhNN7Lq5hRqORbJN +Anwnypi9uYxx38uR1o2bW6hbd5BZQf9WwhX0lltYqJtzsPTu00vpJCZvQSehDw51STJfRJTAwlJe +7WnY0kJd5lA+qZPOYJGcnZy6pYWQWTNL+APBHOYNpkvklGTjrWiLfAMVExrxoJS2/1YWQlwzm+Ob +oxlM4VHJsUxySJPVH276yw8lstvx/lbaZP231iZ7VZX1gVK2hjbZ8Vuz87LJ5hxuGl6b6SkmmDXz +95ot8LLRNVvpx5IIDzS2q9ch7aB6Nj27wD8UtbdSov8oEXdyah40UhPHKCVoTqnpaOM0bpJSph1X +aoxvmsbFlDLteHNcc1+euauJHUoJOtkc/JRu44l5lCbOUUrQKSrBKRo3XynTTi01a+ZVGveZUqbN +NwctQ7ZR73PmNup9VivoY6VUvHqfNyiJqfAPmvqLUrI83RyPDNnWk9dLLHml0AgLDOjMPKifJu6s +lJweNDV/LA/yEgf5QI8YTj8RJONAl9rUQpOw03Y0EzYv/m2MTRNOZkLw0R7ALGV3tFKWvbhUnOh7 +BMlyeJjmPt+02b+296oXe9VUq1kpc1+jzdGqcUcqZdq1pT43yWdkY61Oays/EzRHPcLmrxuO1hJ+ +XcEydxAvkgdpT9jegvlQz0Uy1L/FsJuNQsN77OB/fka4u8lE1InyO0D1yvcLpUX8vxb+i3cw/Cfl +ssDXUsSnGjXOSTGKzmLfPpZ8gcmH6tAoH+qFPhZCKXwjWQI7cmmcmO3gHHFg/9nRQu8LVJAblIYy +HQI5TyBX7kjPkvejUhlzMxV4w2joMKVFNbnVPNC5o4UaeiDcKhcLturr14zhs5/tetdbMVNZXazU +YyksnhEWnX254geeEfGuL2LYt3+rm7FT/fhl2UQyKTenRtrRZgdLleEypdYSfhrDqnGBs+RDRF+T +bUdzIto8YlbOYSyneTv0o4eflXNcnG0du1MAB/ezEPytGrhOGV2vlOzOtiL1PYF5/YpUlheK90f4 +QIF5KqBLM26xymjRJ9FVItFylu3yMYJEKuZ04mqRKNCfTZjiiaCLayTqPEbZpp7L5CVnXzOmMZEz +Jd0qbT90gMXPs7ltyRweknyLB/gbo9GOaTOgXkUa9dZ6or1SwicAPhvAZgVekTbYaicLNVmnaYTt +Ol6co3ETm5q8qBt24mj4b97Kz9k5B++WDPm2FN/sZMnL9fCR8AKC/XoCp6kEi5VSu++WQG5kP6xx +zyhl2kclciN7x52LNN+USMUmNh7hRHOYqVVZ8nZxlTLAHBkPD93ZXwkTd7rGSSVMVNfOKu5xpVQE +6lYXKrS4VCpEGcon9wTq/wMR7rjSSKwn0KfBQvWcnsAhDRbk6ZxZKtQcpbys/mtAvos0v4E1EV08 +qanPKmV9F5caXbykca8pNWmiiwfIITivJ7Cb1nmEUoIuLo1c0xP4psFCZFlPYMddLATv7gnsoSA/ ++Acr8nZP4NBdLETW9AQW7ELhCk6h2bEz/L6RWPjwrJOysVTZ9HmnWN/NwPLSH7YM4MFdZEGGJ0r5 ++NZSftou72Ty/JrS2Q47G8Mw5fK7Uq+vNAOvCbeXBhqnYO6lfD2IPq8xkcvgDWncoYM5EeL1o3cl +3DLYb/ndi5uYHZFuS8Xs7CxgtRH/dKW+gr+Sgi8ebAr+Svj+WMS3sS3Zoh/Um5R15Pt6WKqMnvzA +MPYYxvhhvZvl7H6fXY1fh4azEqbT1ohvdqXq8EKAehy/G6eM/2Gvwh0B2mTrEAt1A2cA8lBPBncH ++GKBpfwqWe/G90yxNytVyAMCeYkQ1C1i1gReEmGCe5DZTKAq5uY68UqAr0PmRLNrppz8vCUom6iZ +tod6x6BCbjbaidUSeH4Pmox0CzSabnFHQLpFyVDtFrsP1W6xUnX0uVJft2geKmz4YMYkZfNYQMz9 +QqZIfxn4rqnjbkpp7o8FUM5vli3STC8EIhUlwHNDLUQiJYC1p4XqviVA+56cUfzYE3hAsz+ilGxW +BCI1NcC1AuLZ3BpN/EkpQXcH5NUJb+ZBE1XZnvIJesCAev6DxW1WAzynoFeVEvR2IDK0BhjxDwvy +hTQP86IPszgQGVcCTPmnKIbvwFqtdXw3IB+Cje1lIcIPwZ67lwX5EOxTe3Eyxq+pDYuapvgwIF9T ++20vy3xNzX3fqHCWUgrDr6mJpC9q5EqlTHzZSPrKv1RSD/OqD6OSrtk7L+kiLf5VI+kP+1joRUmN +mP2HeWKuU9xKI+a/h6mYLdqJjlRKSShmwZNsoAtOyUYnppKzsFoz3b/G1LWoJ34jZr37cK8nmnCz +hKUnmogLh0tP/EV64nfD2VVi0i1+lfxlI7mvGQOqpCf8Lj2BZ4w0xUmO0f0vxhTdkWqKV41UU9x8 +X+0Pgz80Eo5Uymrq5/Mm7Etlir0u1tTzlBL1s7HX44iStrtDE7uUEvSbabvbWNyuJchj7vFhtO2u +GsXixMoWqfh/BHrxgV5jYueNVhN7enS+7RR3fJmY2O+jte2mqNoPUUpJ/rTtRiRyvObbpdiJHxuF +FDXZwjJO2O4c4zWZCX8qYWkyEzFoPws1vP89sanJdXLQ6E5GNyZyLiN4Fev6/SyEiMMZZUM+KcXe +Yy3UDWuS1j1bijqPMasZI65xaRld49eMnBOnO6bzu0oid9ifajPN9LHWYa1S1vusMnErBxMmH8rr +8ZGpYC+lBC0uk0/uzcuDGjRxsFKCzjWg5QRJM62IGyu7rEycwVf7F3Wx7Q9gM9FpDGw2uKsNbvoB +6jROO0Bb9H5CCx2L6wAO+SMKYxxmarOc+6mR3mseeXrrwTI+vbXuALOyAR4UHQ4Z5zWXCR8tYWku +E/HIOI7cOfNpwxXSMluN55ycTfViGbd2543nCNgsLfOScP2JMeuaveFolTSCDkdvSmDKBLaI9Jvb +VOq7lVKNL5gGOYkoaZC1mrhOKUGvGl3fTZDocE7C6PAto8PPJ6gOt55oGY82eSJLXX+aw1cDTMnY +KSxVzX35WbEGm4F1ZRztT5xIfTHEkf6JSZzndOBrqfUvk+htEkDITnXgF4k7+kALdZOO4D0KzoZ+ +FwV+zsiljEzgD4FtPZmCGd9/vJZ9klJW9bhg5OQSYLLA1pN/ipObOivjAJ8bqT/8wVCv/TuBO+QR +6BMnU3ppXRNxFyPyjx5r5LgpPOCdlWGYNvPOFJ8JvBFk5zxyKqPwdjkDbzHQiHfK2f/vnWYhlGrE +agnFDuJ2WyqWwbsCPWK6Caca8Z4AXmdERozpfYlIHWwh1IE1Euh9CJl14CMJvcAQP1WbwcfCLXAo +vX8jPpHkcySE14NspwmHWeg9+kujh5eVhpob8UeQJnvPvy1epmcTNOLkcvqXtf82zC6XUP1/CJjU +IkZ9p1zXWSBRXYxK6T2mXxnVe6PvTDFjlRqvdWoV1bHLDPKpTwKhaKoDyySycQaXkNkobqmi/lYw +2Iwbqyj43jMt1K0mviOdjbl4WnJ02bQPGbbqW42Rvx6UGVRFow5b/2zUYeuTqIVyvpRgjiJ1qNoi +luexQlNurBIeE2PKY25MeZzq0C+RSSZliltRbjri8WtNZRcopXWurBDPeJ9jIXJNCdC3iZOqRSXA +a4papZTob8p7EQQEOQO66AvD71KlRNxdLkPVYcKGIC9x+fqgr+M6nl2vaTcpJSOOZ+VksEJr8aDh +PDChmYJqGVVKvUxSamadES2nlIkPVolo7x5B/ZBzTdroRxM6W5TziZppnlJmFnGolkma6eEqUVxl +0oKnEyYnvjIFH62UeV800GiSqmXBz2ri90oJetNId25ShXhb095XSkxeiEUqxCrDeVxrQQjyr/3a +CDFAKfMq/zmtFIJf/bU1MaaUoNUV4quW50EbfWM41Ssl6MxK4+xmaFxUKdNu0bQ7Ne5epUz7o1KY +f5Vn3vdbw3yAUoJWGtD2KU/MYzRxrlKCvjKgI/Kg7zTxR6UEPW9Ar+ZBM7WPx5QSdFaVyFSZ9oo7 +RxPPV0rQeQa0Vx7U+b0R/L9KCbrbqC5N0J+6+TEjJ2XTceBHk//UXwz1nH0z8Hjohy0DuCzNPaJs +Oi6vfMDj8uDglkda5k1q/sgORhZGAYO8jpHyUJIf+hZjyZQM6bz2yFoI5dI5O8nZCJ6RUhYx0knF +RNAFYR68vKpRsiWIhRJX6VqoW52mb0y3pXJ4WTLv5fJkF2+Fdl9XirMZSOFbSfmCgVzWTiSdLN4M +HTgxgNYcZ4UZ3pbF2wK6PUc3J/3oD1VRxU9GRVTxwrBJ8+LCvrQFYemOh7aRQ36UZW2npnVrbrzt +tmCpqvzEXw1fT/VEAueEqZdL2/JD7TlhjgSvtHEsw7kSqGi3EOrEeWF+7uKf7f7NkGxbSlt4qbKv +KB3ORUu3Yu6UYn5u51itKtG4nTu8OZ7BvMiwGfXvDFOpizot+Wj/lJydzeEuYfQJ42Qcvlck3HIW +xZWI+ySierYM+csF/QYDhVa/o5o1bDyK+4umLV6TPPceZSEUxbweTJ55tI56F/WggjY5xoK8VeR4 +yVx5LK9EuXzeAU9K5r2O5dA5k1EJfCBRcwlazZh2fC4xtxOEunp+OIHvTpjTQz5Bwdi6RV7kfIls +4OlH3TpGZuxczsmmcKkIZs9hVb24yyRup+Ms1IGfRWjE8xKz5VyiGrFCQh1zaSN1S4loytpxnCy1 +WMHoutWMTWChRJUez2oM5CcVBHixxA6R2AxjE7hKohKMQl0X49rxsMQtYZx42nfUGN5XSlt+NSyO +53mCyofXADM7zWB0VY/IQTVA7xPokZhwwG/GUqcqZe6bDOgeAc2oAXKaOPc3AybopOpIrgaYcSI5 +0ePfoom3KyXo1GqRI3ASQeT0oiauVkrQRYbTHgIip8jvppiNlRK01HB6Mw9aoIlnKCXoMVPtniez +OHLq0sRHlBL0fNjobec/TCkDlTJtQQ+R97Y8g3ZNnKWUoGuMKB/lQV7iUT7QdQa02SmeKLdp4p1K +yWmxAY3Pg7zEe3ygMw1ojoDYZJvB9Pp6peR0q2my/wmINe9rGdAApQStM7VbkweN0sT9lBL0iQFt +eioFZ3HPa+KLSgm6zhQ3TkAsrqLEFBdWStBcw+nYPGgbTdxeKUFrTJPdmgdN1MQpSgn6xIA+zIPm +auJJSgn6Wtv1Ko27VinTfjQMNjmNlaK8r2riG0oJetQo+oA86HdNtNTNEnSfAR1DUGEkaLVzTjZh +JxOzvUG4XvPcpjQ/EuSyHE+BF6Qb33KaDsWMoPP74DT6QrxazVXkJvMs8xKevZXLCKWU5KVqeWp2 +xjzWiU/NeonjfaBXqs2gltK4DqVk8Fp1ZGkJcAYZFKoiA3urvGgLXYrOBUzrWkt25q3XUBTT+KqV +x+ZZ6L2Zpv1DaShhXjv5M9nKq1W6OsHP62HGhkvRF37M0fyfKvXK6hPFQaWTAth5vgV5t8XFCrhe +KUWa7nu1xSUa76VvBxRebSEvnXhGEW8oJYdDg6eWAs9qjJfCvBn/zWCe+alyUGaU0qhUBea7xxLU +Tm6+f/AuZPTqq9lCQcNGs4eiaGHuq73qLlPc/UoprOur7i0a76VT5PybPIJ8t91yRXyilBxykQq+ +r26+heqaCmD302lEqVLgUwX9oJTgNtHNZxrjpbCgIt307W80453xol6rdl1FURWpoTMsVvKo0zkV +ykSBMyy+Ceyh08X4zbHvFgsYyNk5F7dYt04KoJ0Rba4T4zGwi/vkpPTaBRaCJ9YAleWmkF5KKfit +VuSKGuDNBazdTTXAck38SilBj/mbqVAH6Qfj7GzcQZfKPyhkitCmYj1elnoMWch6yO7JyyJVghGF +18C9LK+/fFgizfvYNGrLRZw6ZOy44+J1ydnBGBfvyznzCwzk39f2vnDZZHFeLR+JWrKLLZTzbVyY +ZfrZe5a8eOvKxRaC6Upgx0oj9q5KeWjDF2+JRq7QyNeUUiPrLASfrAF6VJmM/ZQy7Ucr8nUN8Mdi +qnR+JbCrJu6llKCfLDn4GXwG18LcUR+mov1sybbCrDN0W+GGM3RbYe8zdUf9IGXTrJSOSrcpjjyT +Mysp9EBNnaaUhf5sCr1CUPk5eqFBs07SsV1neDKJYdqQlT1MBX0N+rs06GdnUslxHF/CPaGtzmJI +LHFpCS3ROctC3SJ+oCqBE0o4cT6PMesY4+LEEvrurxmTmc3NTcl5quTc82zWwVjs8yrFq0pZidNK +fE6yIHsm62TsrDM6nZ3S4TgZ1Kvgvyv1VeCmEvas5NkUOY47pAKPstQilTipXNZO9msQPy8bnKg2 +uvi5zlBrSQN9PY18YnRyAOFzLHCJFXX2TaccTOWLDSczzs3YqRFtsbiTQxNVcaNEoplKGHyuEeMI +qvF1BugH2X/lRsLNYvKh8whKIvHw5AD2Po8rP6ot3YIOFuJKMjoqpwRw1XmcMk/6L5+xbsE86RCr +TPo8i4Dq87nTynQWNFZuZyyUUkaeT82Lq9hcq7qlUlb0QPEU7QQFS2qBqZp2qFJiEghuWQvENeok +pUxqjQyvBa4930JkbC3wJfmUH1IL1B8FAbiRY2uBbS+wEDm5Fmi+gF3j3Fogo4COyCO1wIUXWAg+ +Wwsc39M0wyVK9wTwBvscegFfamRVrQFRgtnBAb2AnWtMjJfCbF+zqEN7AV1a1MlW5OhewLsU5sRe +wI4XEkFhBh5tpJ1niTSHXqjSLNSCrldKtiIN3URtL1PmdkopzTlW5NMaYOmFFsRfrGERIvtgRe0f +MbmIPsMS4TPdklgKhZd8hyv+v2qfzHeBybekW5KXb8Mm35byGX1vI8SHSn1G/4BFq990CT07PytB +cx63hJYaxyMW7fnYJRZ6L9Sc1yoNuS7eFlO8dQm1SoOZo1p93xIrWbNErWTTi4igClco4iOjtnEX +qdqOJWLD1eAFYmAjI//kjQ31yb9W5L/1IkqMr0T8Dy/SsXJzzba1UuryCzNW7nQxjYzXTTo08Xil +BP1orpvMIEgce80xxmDUsV91sTr2Ny5Wxz7hEnXslymb/yn1OfbjLpGuydHkIk29VCkLVcd+m6Dy +XszlFoV8hrrVzra4mKkqyGziqWIX+q+6mcfw2CWBvnxX8S+XWKhbKjFyF6Yfb7g0LDWnVjzH7H/y +1gEsWWqh97+UzxSloQR2p3t7bymF9TWKk4r5xJij8K+UWkaM4Nm9gKUad5dSCjgseL8v5XJN2Q3A +cn7RvVBdp1BZYFNTx883M1QLqUk3HgGM5ZZK70st1DTKrS8JH8Ywt0FM8DIGW2UPB2OvmRLAJ5da +CBmdHEqdHHKZF56ezsZwGHnOZxxv3GXwby5VHrjMQvDDXkC9irOdUtZrf3OPaleN2lcpkw6Ra1Rf +X0Y9Furnvecam5s6fa/UWjKQ7cj3xNdRiqGXs0d6r0ire3BwAMdeLhtQvbmb+D/+drJZ9GZj/Xy5 +f3LnznKnpTiBBLYwhey0laFeIe3AjuQy/Apzd2hHlpi7grqItSdcBwl2/Huv4BbQsUBIPokwmqCS +Kzk88jtQEjz8SnajUATYW4sao5S1GYtyJi06FhIcH6mPAHcwS5FC9muLO5PsuINhKuZKpT5xj5c7 +qJ9caeQ9Xoa6La7iIEj5Gp04TpK4iVeZ8zWcIsG5V1G+rnLgCOXZqZTynWTJG5VuFxAl9RLn+ECn +WCL2xwQVic2LlFityK22Xk/DZ4rIm19tRD5T5GlkKN+qZ1ps1nOu7tZ44+0M6pXfO0p9qrhC+D5J +TingCuH7K0N5vlcI35HXiLlcLfB2BmgvV8tVzGuv8Zd5ZJvT5jQlUrKwz2K1ljlqW69Og8U0M8AE +Ws2b11io4Xc9JpxmBbDttawgP36PCbSQoxhuSqRywAQenr7KcJqh9/YJYOB1fCZpDhBqTKajLUgv +nxLAaddZ6H3hNqa0dUpDCRxB277/OnYgWRLX1xvIzkop1iG+NeI2Gu+lc+lWWCNy2TdbmZ+slByO +kmXffzXGS2HeomUfv+/QlEjFMVPVskqpZdQTyjouTrP41ut111mQz+FMmgMpYp7/czhHab4TlbKk +Yinv0JRHlVLKxeZLN3dqlJfEzEVi9u3flOiURZ33wsbVmqVse6M9a8mu0pxNwLCRgwLY43qLr+ad +yqY9+nq/XcTNJwPqNWObUmsJ/bbvyyrxqJPil1gwRxGjdvCK6obMpJPJ0Vk7yoet7eT0dLbFyY7q +TOQwU3PU9inKGUqlO3BGyXflATxCMWNOMmfjTIn4gREZnFvSMCKA/jdYCLlOsmlqotXBeQL49w0W +yvnxiUVzIEPNGf5VRzxqdv/rtcRlOxWVXJPLJuQtJbg3sPGIABaxgNYMugIfWAFseSPv5vDDMGsC +d39TitMYpqzzyijrRzfSM9HK4xk8GJj7Sym2uIlT+oHH8R0S+D5AmRtv4nEBY5ri+CFAl3s2Ueal +p13HGaG7AvLS07eYIm8sxVyT8Gggwldl19zM2dVNlcAwTXgxUL2iEjiICcElFcA1WsVnlbJRHjK5 +TxXQoAhwz46m/s8oJej1QK8xER69CP91yn+l4f98PuvNfU3WR5Uy6xdeVmH+mKa8rpSIL/OIKRHg +fU3ZvJ/hRcTXAQSTEaBB4/ZWyrRVAfPxkDc07kulTJtXhuCCCBDub3jtpZRpC8t6XStVCt4eAW7V +lB+VEnF7Wa+nDeLlCPCTpvQYYHgRcWeZaaJhx5uWeK5MmqhkGQcbCny0gk9Xykxvlpnmm6OZ3iqT +5tt9mWXedH6U5F4XAV7XXB8qZe73yowq1mjc90qZ9qH/cwpq1rL7NhhdatTrdjbSa9cN9aGtvlB2 +xLQAbl5moTxQB6xQwV4pi+xSB3xPwfauA/rdYqF6Uh1w+C30xPmpTDzK1yXIKhkNhv0NAw31imEp +jwa/Kw9g1S3cAXI6c1NNt5ps5xJpHF/ON0ZX32qeC+BMFyuD931SiuG3WsZwtlbO/ZWyvqcE1SpF +1/tryqFKifgsCJmJztK485QybXFQpqJe0vG+JE5FxVxv0MjlSpnvvKJSv9QU7GJqTMT9QWOSQY3b +QinTHg0ak0xr3CVKmfZU0DNJ2s4BqsPDlBLxc9DU9RSNO08p084I+qYlhTbxDECR7YOMnF7L9Mk6 +Rt+rgs9+UopTb7VQN/AEcU5vBumcVtxKY7brgD0068FKWejb6xc61cm2JlJs1hTmKPTwvYtKrVnP +ADBPLGDj/9F7s/3FfS8qp8nMYGRrBmfItyOuZmhGfAbOkg9LfMFgPINzJLTtbbSudAdekZwJBl0n +ilcleAuDbalE54R0B16TqA8Y5XKPiWKPzLThPYnf+Hb6dTvbwqgPJWqGF+Uh10j0GYyOzoomHUI/ +kriVjJNDYcZ9JnE97mDVkjixgts+w+7QhzdMpcxzJ6a65repAOoybIu2XCKJq+XLEG1kk8k67bip +gsq5hmG3sa0J11ZssU8ApXda6L3tP4y6+//T0FACN0vu8XdSPy5ulrzX3WkBvfdX0HV75cF3Crj8 +LgO+U8Cxu9jpy+ksV5xg3N3p5eogZXCtOdHELio3TnGShs8oF6f4IPMHT6wDDtrVFJRSSjt6uNx0 +mTUaZ+1mMEx7pdz4pUXKcGW5+KVv77IgfmnHu9UvHXo3TZXvA68ZYrJvqZRsfij3zRBrNcEDcPpU +PPcaqIhhSsni53KZIg7SKC+JmfnZWe6jjdK0A5Qy24kVZo9rmsYdoZRpJ1WY7Z6XNK5qqJGcaZdW +yDZR7R4mykvytnvKL6oDcJLR+g0VvZYxGLmnDnjsbgu9HmGw+rk64CfRC31KVrmfqJSlPF3Ra64Z +5og4SVPOUUrEM3kEJxHnasqjSol4vkLGMOceHcNuuIdzkQ/qgJkq3xMV1b/UAe/cY5kzqV//ZSpV +qZ5hCoBJvveg/9YtXXTcy3zOyr8bYRwdu6T6HPF3gzBT+S5Q6vm7gnNZKt2y173FE90RcQ5mMiM0 +s1IXS5VF5T5GZGUVLOsNvKFpHyilMj6qKHLEhqNhhnrlkRlZxIsebGHl3F9KMepedjpOKM+uPPbg +AGbdyx2ck+iQM3i5kh65i5BCPd6ppB8ov483WZxoqoOfznuvkpPIRolryxpfvKCKuOsYl0gpbkkV +cW/dp87IsAKCuwaAh1XUx5WyaudVRqYHgB3uZ/NOrQAyJxvzu6BSOvkhTAhuXga8prlWKWXuCyoj ++5QB8wmS3F2a+yWT+wEmSO7ew4x6tlLK3C+Z3F8LiI/FjNPEA5US9HKl3F/fYTkF5JS/5hQj4DuV +xsGsVvD3SplpQZWZJSwZbkq9VynTrq+SWYKXdJUvSWYJ7BHvaGTZCJOf+W6ukh5x8HLtEfMokdQt +qKiwUqKXVfWiZooN2284YtWDMUfNpmaUKUlNkeazTsxn+XKu9TP4Siyl7AHeuT5F7oT9KiYx7gGa +SdZOpA7Er5UjDgngzAc8p3nwvoapo5SC/Vbpc5qHaIIHYJ8c6H1EIbhdb6BJEXcoJYvfKyMH9AYe +f8BCcGpvIKNpVymdCuCMnYMn9wbaNKpTKZM+5AaNrNnv1NhHlZL5H5Xike/SKC+JkhWtho2bQL3q +ref+pqqqv5pCb8JT0k2qu9jtqLkMPqlir4szZh1j4tEO/FD13EEBvNrFMbH3caMNt0eUhjI4O8Q8 +lQ9a6H2JxuZTozgnxDOrvR5kbplajVJMs1JW7ceqyFF1QJooObr8VBO3HGOKI+jKUJGv0dMyrd09 +BxigV0tZp+LmEBeqlz2o1wvGKfgQpeR6R8jX6uM1wQNQt4VW52ZKlyKeVUoW94ekYR7UKC+JmTlU +XtsbeE7T3lDKbA+Eej3Rm/1AhvGPNOVbpUQ8EpLPxn+sUV4SGQ/kVwb4OUStdw+lzPaYkcfSKC+J +2SiPTBo0bUulzPa6XxO1muABmLlYE6MVMUUpWbxlSh6jUV4SM3czUX7+LqfDT85Bl+ZYNq6oGc22 +tYvvQqfsFcDHD7KLn2r2/H8I9dgqgMkP8bSWUc02fgxxe/u8h8TUWMs1yvUbpRTxW38tP9IED0BB +i2sZVoE2VUoW35ta9tAoL4mZi2oZTTp2NpNOJ110Kfbn8cX1c+M4Jzy5RwBPP8Qpc5yLBY3Z/2EL +detYtQQulRucRzNm4Gl8xBRXhK9FAMsepotLJlItGlH5SL7yn2mRPyql5GeHfcb+uSZ4AMpfXPmp +Kq2jlCyWhMXYp2mUl8TMnnE1aVq7UmZb6i/ZA+QUwMzFJd+oKfcoJYvrTck3aZSXtB2ADIDSbFsq +9/8Y++74qIrv7Wfv3exCVkJkg1JEAtJBCPZuEAEbEgXsSjbJkqwkm3V3Q7FGUUCKNCkqYlR6kSKK +ImIERVFQ7CgiUUQQEVBUQATez3Pm3GRX8Pt7+YOTOW3OOXOm3LlzZ0MlwY79S+LX9+4S69kbuN5E +O5M3QXP/z2zNpfMRAqvlGO4l77iQzp98XX3S07/ZGPSui0d035fCPBYKgvnYIOHf8q7LrEMjI808 +++lJfq5D09e6zFL02vdc8HMpOv49l1mNvsM/vsoAQu9zkuYidp8Kv2uEp72vwl+tS1rHpn7AhvzX +vsONpaXxnKgcF6j2afsNxkfXtHPpYUb2KOZHIB6I4Ep+gbyIijLKa7DXEvsDsR5+S3uZxuYGhdRy +ZVoan5Y+5Gz5SX1gkNKGKiRPN/8f9YGrPtRTbIOVlMhyqRxiu49qTF2TlGmBQuq5Vup6iUxS10al +VSkkz3VS1/dOXZ8oKZHF1JWxnlE7UdjkNR5wownWBwo1aPwNbnlF0IPyJwi6SMu6pDOqVPZYb6NL +daQXRoBHZCv8vvUu+FqF8Igr/TY33l6vK87H5c0yPK5TgLNV+FKF9PJxl7/BKcAftKHGCD6YR0v5 +LqGPqe/ATQZW15s/CJjm4hx90QYX0kMscalbvsGFjMqahq+Qn7pftsGY05GXe1S4kn/qvkKiwEuA +R1dn0UwR25EoNvNfYjMdsWyK9c/DPNeq3m40+oh7vUTlh+OYLy9tyvh5ETIqieXhmbIY1oq1cz5y +gWu7FRLCzWST/Dxf3b5OISNVYX4mP+1jJiiTuEyJoxWSaaZh6iJMjHlGXxO4RgrJtNbEPE6m42Ou +LY6bjWC7WwzUyNPaz8Ta2R/T9vxB2CnNsIfFslgw2gOfCv3cjRzcg8X9e/P3Oj9zcbfv/o0ukwpZ +qv08hTTrM2PWmo1M5+p8dFLhCnmLlK3WbFeoVqXnZQEHJNapn3Bsqy5eyqJ+1ClIvuiNEEnTcUBs +f5HlWBGvf8ABadYvP+ErdDwqF4XU/pRr7rj5MPQxixouISqMYUIv/VTvUBqnVk1USK8eM7+Y8sqn +J/SKrxZ79I4H8gfEgFtNpOveZqD65isOxeKYbM1u5sZOVnsPpltc2J/2Gd+wjYFk0HMWHy4LPuOI +y1VA7hhI9ZOshMkvU2toqfD4KehSpVyjkB5MtWTyu0xRDonCnPxy3ajG5SgPxZ6yzGPmVMVVKCRt +piWPmatorixaFipxhUIyzbVkIfiSohwS6+VCsPkpwEdK+1UhxeZZ/u6nACmfu+DplcDzifLwmWNS +Z11J7lHkYYVUMN/4u1dRDon1Jq10nMzkabMYMrXViu/swlnXaT2TZkssjlQXfs43/SbNlsgpu4mf +s48U4g051baWpUAIqyyu8pp94eI1knjL4gKvD0tynkFeoFda/PjjJSJjkWB+KFAck8MPayx+ZvP9 +F3y4YWaE8K5UVPElt32f4OE9Zcd71nk/29hJwj4S7saHojTrKwqPJSvMFTz5xETGESO36Xwtt+nM +/4rvuRYLOoId1llN3NhPzszx3MzAXukb12+ih5H++NW65HI3Ht7EbsB/0uyVt5tgrVXI8P9k+WED +6za5zC/7va00h4ftkOVvZgPpPLEFb9gGKsZDhA+ZtvtCZbYqpAxzlSdyWtxh6myjkHXuteR8581U +6OGlQOuU+JFCMn2d/FKxi/z+sfnIq1wb/eRco1u7rpkebfbM57526WnDAoy2ORZuJyYgSrh3lBOM +8hghJth8U1L4jQtKJDaWE4ySCROF+jSpBcE8Q8OTNjd+NhDJ3z0WxkmCa7WZw1ghR+Cu0WBBKI7n +BV9OfCxeWhwM4wXBvLyZxxTZciURjLD56vPQZs40sVOA1LuMW3UVMhoTbAitgeKaKCRtog3PlFOA +BYp7VyFpb9iQd1+xfkbnQwpJ+9yGZ8UpwCLFvaeQtJeVtllx+xWS9nPiDWuFiS2jk1mmtstNCrV9 +OJlVSvN0+JZpmh8wwauUkPT7NnnnMKHBOyNXNQXzjRuqUVr8gKic8C17u6MSB0TnJqmn+v3HSDdb +rukWPtA4W3uj5f3wEOI4RU11c+ioZHFQaXSAvIV7SsT2E1e90zdHxC76jqqcHcFXhe9h4vpF+mG5 +vIxe9p1ZC2nlZm8/c4LpPyPdZqfsg4Bx61eFjPIit+yUOaTPlcSnFO6UyW5crup51Z2wsijMvzkw +gOeUTQRjKNeg7VCowTMz3Tvu2c3c2PGdy5ysqFCNa9wJE9k0FZypkP275lnq21NQTVmmHHTgHXea +61Sg8VbmNUeNtUr8TCGZ1rpltntPUQ6JNSSN/oX5OYHoAOMRUGDC1TJooONQYQRfy6GA67eqO1Xq +zsZEdzIdcYWsrMaduqcCPZTyiEJautntP/dU4JGtLjPR3qW0gEJOdJuyZKJ9VVw+2w0MUGKZQjJ9 +Y5h2VTMNVKJTG5m+NUynV3EAF1UzlWu2Qtr0o7se9491dl2klJUKyfGTie9iRTkkuswR+vJTgXVK ++0UhxXa56+Wdyp/3LZX/H+D/ZtfwsHLV0tiT+wtTyT//ImklNYvLwvzuxWWxoisKdXTMVh3v9k9q +x/SabnxQutSNVTpcyOCMoXIWZFUVv+ucaNZjw1M46p/6PVON67G4an5MIa18LCUhpcuU4DDQ1poc +4ELmO+XYqZAqhvEAxKnAPsVlqOGkDU9JW3QqcI3YwCQ6U4khhWR6NkWS6IHvNYm6Ke0qhX0AJ4kW +iyIm0Y1KvEMhmTSJtpHJS6byiRCCJs4pP0jisNtNVbGZCmnHnBTpdk8pyiExCkndji0R43oZVcr6 +WElyW/GR9FWJ/tU/uGDWX6+mcBC9/wcXUqmgfzRQEsTOlNvy3NWLmi9S3rjLjSNkefDGLLyWwqdJ +86p2TcLfq8zfGZVs5wL8mTIXbvTYxqVQ1pOy7pko94+M3cbvgUv5k2sDA/HQwCAmejjrryaex1qn +eFbc7sZNP/JEBJ7x9LrDjeksFOZzHwPz5PKR7cTEMN/Dw9qXbedkjhWif9h2HcWNHn3CblVoYtFB +ISO7MkWesNdvZyquOBXoq8TBCsm0IcX/1anAMWFynQKsLDKa1iok0yqj6ayfqOn7U4GL7zZMtykk +03gPvPUaAJhkZpQpHv8VDYDgTy6k9W0ATBXpQANgikpNV0jpCo//sQbAdz+5UG8stcAztQHwgrLM +VUjWBR7/6gZAqx205sMGQKUS1yok06se//4GQAmZvGTKVrNWGsKyHS6kHW4AHK7mGKccqwxHx53K +UbaTFaU0BAYMMG6XK2RFqz3+Fg2BOcLEAJ5bbJguV0imNSaAW6qZPlTiFwrJ9JphSv+Z/aX6MZhp +K3mvK5oKzfrfFOqMk25SGztTRue50fVnF9LLwuaE8E75Xb2BP3Obxqzjd6Zsu9ONucRwpfGzdJJv +WW9StaKSa49gFAgbr26LGPivard6WG3dXVz5yKMCtnpYR19iWAe2etgTp+1ywReKdYkNCefnRIPB +kkgcP0r/2GYoVwbzygq7BoqLsV3Q7X/hJ/R5d8cwzjsw343YLzx3N4m9l1+p/SF94sAv7IaZk+Um +mxe8PFPabjcfZoiRs+Mz5Pqce3ab53nMkeKM3Yy096Ab2DfZJO2PyaderuT7NOSqz93iyb6LX597 +6ddXu3lgtjhQGAM+9/6130aTX7lLEsHXXs4Hg1iSK5icdeF38pJ6FfH5RcF8eY2sR1Se38MhhEhs +8/6Q78ade/kQN4VHajFCLupZvpfP+2c2BMZNNZaPqOXv2RA4Yx+7W0MgtI/OVbdnofHleuSqEzcN +PIEza2vRmWn72IyOoVgrp0R+2MdLi0qjA2TiDBZgg6Av+c24+XEtujn5N+5MPCWGbhZDt/zmGFrx +lDF0szH06t/V0LH8MKbGUNabJ1s+uWrhjkHJlnLL54naZ+20sfp33fIxxVr7+QmybtfgidqyX0Oc +NNQTtenbUJZj8QGkM0E/2C9j6wTRd/IffEWPiVK4+Q8XMvC05NQkwYwiJpsYeeSeIu9dT/+TiUbk +3ZgmP1EWJGYfMRE8J4ILiMl9ht965d2N5wW3/0+uJHgh1/O1Oci3+sskuvnSy/Dc+hfbkP+q25Hh +Kc2T62Yx2MSl230GOl0yD1gmVYz+y5kGl4nvb7GKGN6Q+jwHxNeVwhlgIYTtUljJAu+gimFFbd6y +0u6gTjlGEhmLjSvFwTi+EJFmh9gBs6eph78L8oVD7KoOrjyVLfbFIcfr8lR6XevvRK8Nz8V/02sv +v7ZMf9bkzIra8rVl+G/9KOf5v/WjnFsOu8znGVkajC4KEz7KGXVY9FW4gBzV90ZtP0+HrCJF9gbW +qdgGhRyP36ktewO/CVNeQ2CTEr9VSKYPavsfbAhc9w/nCN5smDXEtMalCsm0s7b/Ogt4iExyo9I4 +teOn2nLV0jn8+XN+zL9fhQ4ppBcLbcA7oiFQpUJf1vbPbAh8dMQF/8KGQP2jLqQtbwgMPUpHk1Kl +a+JCJFcT5ckHjIkJCTNeWufVo+xP0NI/LFV3p/Gp7E5Zx5yUGp/K7jSIZR3xx6eyQ606xttqpku3 +MVeBpcBCRgUxAwPFWCs1XQsLPsn+ryQPJrDMzrFNyO9SZPFzHEd4B9/LqdyxOgiLXUGuG777fuPC +YwoZ5S2pct1we5dlrhu+y2WZxnWYw8rMqB5wSePOdFmAfCQ2UImORmXincSbqJF3EvssWuB5pyGq +2RaoGA34LtW/tSFwmWUhbX9D4B7LgrdOIyC9wtxZ8r3Lf34jYAbxvN87pwK4GMDUunKXzzjV9aRC +kibUpc8JjVoUKAiiQlvQ/aAJQ0JL7pYAfmVZqL9MuX5WKAH+VeipthP/XyX+l7Jcmqejj+GJ2Bbq +v6uy1To4lu6Rtn/RZjRqTCuMBofo0JSpdjV5ONk+tjD+Fgu+tC3nuy6DOM1twXzYZcoDWZYUwd9i +41wiWD3+lvr3slxts0rdkWLBZ74HO5bK78HGpljIGFfB1+hRXho9WV6bryYbv1a/umAwJsulSn+m +WIC3VyMg+3kz6hxL9ccbAW08Fvz3NwLCHgtpoxoBz3u0YcufNw37sOVv3QjYQc5OjYBGXgvS1D29 +lrnKPaYRGayQbTu1rr9BXWAiecZZQFxpiTwT6srVFmu9EmuTe2OV70WFzL3hPsm9Q6x4R0Pg8lpM +bH7TNle5ligk9zif3/IDUWGa2Qh4TYkrFZJpos+/phEwk0ze9Y2ASo3LZJ9/WyNgUy0L/l8aAb7a +FvwHGgGX1bbMaLxe1VQppLfLUuRruXtq0xMZdN9T6ocKWelkn/TLGeSSQfe8h0wKXaiQTM/6ZNDd +RSYZGrNeMC021ydDY5NUywyNI1MtMzRuTGWt1UNjYf6VZSWRXnkyj2ZqkrZ5zNTkdKbiQF6wGFjs +67bPhu1jdvKrxMWSLX2kzIvnTfkZHzsUlvg4q30shQGhSCRYgFU+Pv65TyKdS9a3RP78kyzzjGkk +zN/LhYSMcS9wfSvr6Nd85/1h46GTLCBjI9EhrBWu94jy8gr49BeN80t8ctd1szqWuQK+qI62xp3q +YIFCtsY/5gr4Z+owLtIa7keM+3UUMtDLTWv8QC4vt8dzta7XfGa7/KqhRuhahRSqTD65xLcl4eAg +HRmyNciXDTOCTrBNR/9RwndqmhPrH8XVa1iW2CniAUHkB8Lm2jbDtTjNcj783CN9fBsRTh/fI5pO +qWuZ+eMuNSOqkHbv8sn8cXVdy8wf95N5dG3eDGVszVeoU8PM2sCiuuxmr9cFCpToaFQmzh8/UCPn +j/rpjLaMMRUayF98MsZcla5Dy33pOrRUvVg9tHDOeCldB5KRWs9YhWxMnT9GKSqRZOYP6eMOeqKy +0edffKZrZ84wObTHdO3v07VrZ5ysXbvHyZpML6r0KwpZv3bte0+mexKy6Up9QSHr2mOSaaFwJXRF +novqEwoPkTcqMWRrZhxUqBlS/xYt/6jQF8FROYtXdTJH+Bm8Yg/H5Pxd63rsbFxbDz2J/THMcvVM +MVRuVXueOM4mj57Ehczn9Wh7jVl8VuvYGRhumt7lwGnn8dxJDV+bdoX5XUvD8WhpcXEw2pt9tq0I +I1NF7lWooryGYqi1P98Nr9/iLlVOMMpLLICh1tiDNi7yJ75tOqF6eY3QdUh+cRDlqrzuSLXT2MdK +JkklJX5L7y7sGw8Vh+6VE8M9SgPFWGjxfdPLpJfFQ8XdotHSKBYJ8ie/hfpjRxiVryn0RbBWPsRr +mMHBqIoBD8hbrBupFBtF9DoS5f3QeY8b+WyFDNzG/9O3aFC+Ts5Uf3yjjJIEv/aKXw9lWPAVVgdv +n8RuKZHF3HP7y+L5pu0sc5zAASk3qG9BXqbgoHw+eC3LRcFARALyj+AeJI4yfJvWbXAkmB8PFmC4 +vMRZkki7MVgSCIX5uegzQjxAIpXVEOYKod0pHJgkUtTpvO97Td7o3VlDlC1l533f60Idd4oFz/pU +YI/GY79CBnOFbWhHFWdrrEh7I/HF2AmTiB/7SwplqtjAsceF+m2bebrmFAvpPODRvTSaHywA3pbX +mKedyhYoDRTzR+LjRSYJvhGze5EUyI+XJRM3C3GKECUcVzrvq76VQH1ACnNR0xRbReAI0fqtpsnS +7YLv2oCpnfix2c+CH9jAQv32Y4w3F6lXvqv6lfSTr2v22EyNVRS+ql8f7JOip6EFX1G/AH4TFdez +eFW/AH4X6sMsFvUrxH6hrmPxqn6F+EPMPrmRBV9ZvwD+FOqVjSyklvUrxF9ShO/mfgF9CTioEZeY +TRsDmTMh7fSN7e/TGHirkYW0OxoDDRqT4303EFGOb/8/WpIHRZhawSjK1d9144z/CR1ntJutGWvM +ZotguryemsVSyZCrr8QL7rEHbXzd2ELG4pngK+tQMIYX3dxMyzzNQkbWLD7TFWCGMN57msURYY6b +p6EXnsbB03t3Y2DxLOPYDLf/ocZA1WkWvJ15/8NsYACAJ1xp3VIBfxMLadenAt35xy2pwHNNqOP/ +GFj7h8IFN5aF5VPkHl3V5Sp1tc/E41xeJi5/2sRCOt+DYpmYm3K6BZ98ibHazS8xrmO5MIIDJiSn +68pstZvfSyMjZzZPh8lnybvlJdA3p7OReA/GuNnG29fd/mX1gDpNLXPq2zfemNJOIV+VvpEKD2/I +OFtxlyokTV6jjmgMbFSFq931XmgMIG1hY6BPU65iGwPD+cfqxsAKVjTNCzymSsYpbAagk/8VL/Br +U8bT+xGTbY6xcqs7zX8akJnJNQtz7NwJxsxrFXLk2O2Ghyf2dyhut0LSDrj9z9UHelPB/9FS+aUl +JfzoW9vkp8mmpoR0HJnCdByWacH5ENsMIRiZwonp9UzmKSeqcSnssP2a1fR2rBLUd80sZOTO4Vo5 +P1AcChd2D+THS6OYIgoCzfkkGBjcJ3GEeFpITwopFE4izRfSe805uVVQaQ0DVkt9fze3UH/ik8aT +5xWabwx7hsJYJ0xXnMG4Z2Au5FKNyHWhgbwiKhTD70IvO4MjTTAQuTIUi/O3ybFfXiLOIZ6r1Vjv +QcFIHP8I9+Zq7NXhvrEgjgg2rQXXN4OCwWolPBsUw1HR1KUF27fJaUBcbXxCIdtwTQqEVneS8aON +QtJ2JX5m3OZEK5tYMN6ja/WKZYpRcb/ChMad5WHjxltYSA+FgVkeDi6zaXZpWRxzpfgN7axeB0aD +gYIevXp0RblqO6hQtfoieN/DJ7FmLS3U/1KJPyjkHVvr5YXB8Jb0/tzTgPeU5vDSw/Uef4/TgBUt +LaT1Yl9oZSHtltOA3q3YbNXW8CV/frB3fmBgMFzI1x9TjavnPmWga9r5XAf6mKAmbR880t6NYa0s +/v5ZPBAKBwtkUfEQc/f1RDQ/UX1SfkHtWCsL3vzTgJy5APU96fLcfxrwkVb2hUKSChO/IOJFFTW2 +ZatNXytU2zLGMQU5NeALF09WnNWaw9Y+C6jS+lYkXlqRq+JBhXwDW/Me+sXTgHKlPKaQdr3r8r9y +GtCfuj213MA0JS5USKYvXf62buCp1pY5QDBHaQ4PXyOP6ywHFtZTEeRt8WvK9a5Cavra3JHxuqIc +Em3999tiabnexcFgBFXKvvLppNaTk/A/uv70unGsNXtUnHdF7xbEWW10FjBFcyQlc55pp+2JcUtR +pakKaUt13GQqz1Wx3S54Vp0GnKecvRXSsV9d/k9OA4a0seD/+jRgQRsLnu2nAecrk8McA7Dd5T98 +GrCVzHYToF5bJq8cobhd2R9QSN2/uU50hIITQLlyTVZI7j8SDz/lFca0D6BSeUY+mxRErpUl1N1u +7tkFT8tDQLe23PGdxz23grvLYnHsEvSQtgwyW0S+Af9F1tsLFBmJY48gthLRX28QwW8iWa+d9Kvi +YCAWLMCzcsDwFuLyo6E4KoRndDt2fKb3UbXU94yxlF6VJx6HPfYvhqQmkzAOUNFJCqnicetEYWSF +J2tITldI7tfMIdZ6inJIUhVMJa2UdotCiq04USU141JNe3TsXxbOPwsVKjtvuvFVOz+//tdozbDG +7LZRyWi1knDNsO4sdMPbXkJqhirMkeej64njkKblh1k2vWKRtM2y9torTBHi/YdqwiaFdGNmYrjX +K8FhkBg4XxfKALhPe8gcSwZAh3GrClJjoY4KfykuRR0mbZ45/HngXyTWE9FzblnzTdddZEEyv4+K +362QapZaZuB5VHHjFZK2zFTxmKIcklZR00Bt2nEOl62MtnxZJ10DlSp2UkVSK/G+pbF2vNCNw+0t +pId5JRl/HEp+569LBwvpJYEhfcPyNhRj5amrrAOXRcEwnrR5TmUli1GnqSeLYNsz+bxZUBCN8dTo +hW7ceaaFjMh8wBfFs8KS2pF9pW4T4LLnjEV3KKSvU21/2ybApR0teM5K4OmtPJRoB0+gCXCTohxx +kjoBorpEaaMVUvVTtmh0SEOVRDHVOExRjhRJnRInv0g0FI5z9usTDeQHkakhnfY8OXn8mvW40hkj +LJAgje5oIb06SFggIXiLyP76RLtAYuvpZMHD+fgW1RlSSI1LbElNh9QvgVQIyBfx+zTHlttp/CK+ +J9Xx2NNtalqhwn9/EX+74h26JFW947+IP2Fu9eak1SMYRoVqmflCciAiwGrJsfJOHH7J3qWgIIoP +7R0XuPEyjWQjn6xiLRXS5/U25Ga1yxXXUyFpP9hys9pPnTj1VK+bTmgiv36MDgyiUsXHvnichX+K +hQ2z+GAWHHQFr/t7zM0XxtcRFcUhm4k8MUuHH0Mzafa3aj1VtdK4IybNHFKaklitptnJinKkSGLi ++vxAV6Vdr5AaH3P/X528bzgajImfFSoXnEGtNSkZASa52du/yWJCskSv6nTmPiB/c3mSHOW9o7Nl +PFuher5SSDumuqW1HNJHSmJF6tkninKkSErqQCdso17hIMrV3uVzKJNk91Kxeywt5RkQYKmYvpOI +ksBgltla0bOTx6yl7pU/25h5tnkSMw8p7wpnq3OIC4UN7mPBzRVcYLCk53ZBNTzXQiqf+rsGwgWh +An48+497S5EbyMhaAJjDPbvcA/+0cd25TB6OvjHFPHQun+XKyZfviHctKgsP4Eu/51L+ucCN9yhU +OgDPpdDQFudZ8OWTAwtTzg658RARcoDopRTWsZRlrcNgtp/HRTVH0pyFZoZZ7vayPy1eCAm9tkql +xnatQka4E+Ad3QRIf8lI/uT2v9AEOPN8qpzfBMhVwg63/x0WhbChCVChhF1u/69NgInnW/AcawI8 +OZN6gQqFlwDYZMHT9HTgTcW9o1Bp8Fx+OvChIr9UyFTb5fbfejqwW2qlh1hk7HwmReaHphdY8HIS +yF1kXO0E8bxCi+0Ab97pQJWK/emGh+UFs4yRaxSyruUp/kGnAzdeQNcfPh3IXGzqmpYiQ2iL2Ubm +TIWU+dbtL7eBRRdY8DxuAy2V5vDIMOrnMPoD1QLeBacDEVU8I0UWL2EViiuk4q/cfn6IUv9CC/Ih +UqnSHB4qNh+iXHUhRz9ImyxWxQtTpE3uu9CC/88mwOcXWvCycbAEGnT/KacDnS/iHMyWWan61yg0 +LeM/73TgCTL9XyNPr7BZDZ6NSu25LeeZcOlqUPvsP9Jnf7lIdu6GpXAkWn5xYjcclsJOfPRiCxnZ +S7jbh6Nu9pJul1jI2EdM8VkYlzL2fDcGX0LPPV+dDmzSSrcoZAiPuuFhau9VHOYag5QmaV+x1DTx +UZP286lR0n6fEoalSNpvJUHSO0+1hBRS27gUiWa9S51oPqHESQrJND5FotmNTP/XZBUIFwavK80f +ECxAtoZxwXxjvRPOCLBWwjf4Ugvp+SEWGadVLMpwgbUyXvxGhI4Xirr2Ml2dYb1EeybLfNcbQ1UK +r2TffZmF+j9pzXsV+kplUPw4hXN208vZSzgOZL1sQrjO5Fz/yzXnFlyuA8JtKh9QqKnF/NtKLZJ/ +jytxnEJlYv7Vy5ZmZrd9WakrFTKu60237UYuD0e9DUp07CfTxyn+65sAg8lUE/z+oeLiLsWhwnCw +AJUa3+DC5DgP5o9QcgPlrWzOKywN/NNG+y4WPAc5NKlc3QVGjrU97vHXbwrc1cVCvSZNeXhGWE9X +li4KyTrKsM74F+sdylKukKxjDOuuf7GOVZalCsk61rBmXZFswApl2aKQrOMN68B/se5QFo+Gg6wT +DevKGtaaUHJCLy4uzb8yEA+07VgUiHF1zH2rmkmzXFW98pKJlJPJJcAcD2c6qysjzL26OXKk9qqu +PPyjUqkq5QthqYcvBO7vaiGj/GXANxive9hCu7oyUf7LJs7gxxtVqWqPLjnOqK/EqCZXWkiPBQPR +/CJO2fjKw+bPJ1ZM/UpMnSxlLkJM+YMrLVq6Syxt1c2ikYfFyNtYuLcz/hE9Y1iKloVxRIqVLAbD +BVrM7s7VMg9sTPSy1qelzNUzpgji++4WMipNDPaI+tY9uN5IXwb47sZwOYZ8Ww8+ehEzGCO8DNT7 +ZELGYuKK9FrkLnmlA4OYJWoPUyL7FaCaekWwuHQQFgi141UMsyzKfYtM1E5TyCzZ4fF/5gUCV1nw +b/YCs8nubdUUGPeKGSn+8fh7NgU2VxOqlPCwFx4yjl1s1D6vkGof94pQ2tUcYyu9wAElpmjLkWmW +F7J6TlPcyQpJW+BNSg05z86T0Pq+PFIWz4lHgaWm6pYvG+iadgFXoekx4CMXzxZ2udoCB9+P5IPv +OEtJX1rgIxeXcR9dbcneewE+dfUNueG6xoKPt2B86drXz42rrrH+pzWFQbEmW624d9lx1vwm1rx3 +jYWMzFfZUsFAAX6Xuv4mMpdIVrhfKuxyLRstsW/8OwCBggI9jFSutaW8elytEyzG4JlrLaTz6R6Y +IPtgHxEhpV53uFH/Osn3idZfuW7ccR1TlMYMQYVgvr2OTRjg5xivmAqeVchQz7P8/Paibk/aW/1M +lxcKB6JDeks37BMNBpGptl243Khw2mkw8IbUclNP7vsXSJnDxRqWw6Sy1Pd6C77iYP843rL4Kcw0 +lqOhwiIHse365AYq4euSth15Www/ue5eGu3Ndx/I1vp3vubYwQv0XOlFQG6Ls904pRe7cCEC3Cm5 +upeFDCyXl0XIO/UPG+OIWSwYBHlMZE0v+u25qSkQUtWlCqk4r15c5hPvA02ByGsQZL7/qaZA7RwL +8N5WD1is+EL5PYpLBM9fz9in+JB/+MnAUMEvbwpkvW70hP1bmwIf5FhI29UUOJJjQQxpr751VkhD +uhvSQEXdr5CkqwAP1V6guJBC0mL1WIXRH7pB9V+oDNkKyaj6uyrqaoUkXZW4GO0fCodiRTG2Rb8S +4HXTCvsUukxr8NKfMS62wOobeE7ndVlXPuFiJtS+kWGT8GgYxrkkPjkkeBjPItX2mEIaMd4lwZ1C +poQ0LRRLgBXGjgveMFDtkNX+ZUq7ViG1VSRu5l+uBIdBlvrOVqWH3yj2V44RCqnixRNusHNneJRy +TVVI7oUueaoYrSiHJFUBHr64c3ALlIdiL7mMAa8qbpNC0l49oQHce9qsXDsVknupeYnyraIcEg3g +l5i8oPg3pUGDSLE5vFfMD6QprpFC0uYnvjSQduBP62Qrx7hVSU0hl7wek7tCvrvRgvNAPUaGs5a9 +iSntWRoN8v0+nrA4pJcTGyvGSGvNjW6839tCRs4K9mWMs9h16/WxkFElmDg3CSbJTQXd+nBCznyD +nyUMwHOiaUwfC74YnrNq93ajkgzmZ3LuUVsHKaRXIy1z6jHyhumkkyzIj8ss1vJzlv+dk4HUvpa5 +BvQnlbVWGn+pgzcLeFEPwEqjZInl5e/bZGuRX794/moKfPmmEfpRIYU//NdAyDNtbTsGw7GyaNC8 +Jq7Q0NZdbaQ12zlnfi+RubSvBQn4dku+0GExVoxdEsh1fbmOW2nis1vic0QYsFvik3UTA7iPDKya +m1R/yo0VBSTItajfqAFVCmn2Lo1TzpvG5d0mTlNu0jgdVd5GbxmjKVMdpwoV+s3EqUqLEifGcUKl +EZr3toEU3mPJfaobT0CiXM0g0aadJKd5vmtrMhWZGrtdG4zG6hgWA4/Y80Ju7LuJCw9nfw+PyHbx +GTdrZB+Tz/4fYDGGkTbzcbEUODIyZuNlG9p3CzMvko9n7b5/2OgtRS4wp8uG9DMsh/ljTwWY6A78 +YmMbMbFQPBjFEtt9txvtb7XAj1RjWGLz1g1kZK5iJ5CUn2DzpHDurWyxcqKrDzwuE/2TbpXn/lek +8D4LJfJxxas2j8mn3caDP29xMROIdQ+FsUJ87HKbBS6Ke/XvHwvG8YYIP07eSCUPCZVEsFJM+eQ2 +1ltJZATrhO3U2zmdIiP7bUDr2iF1XXM7H4Gl8p2CmHA7ey/ZQvhZZL8hJrIaXLv/Iphz7zDaKokM +YaRsDz5AZEbWmuogzHczCJsEXeWgY3jPzSfrOnfSxsx3lDuGvwSdTbRc/vGYJsJIhUwtXneRkQnU +XmOSo75C0pbY/k6ZQIwKvDwpHHnHJPwrtpwO/kgI/E2/SiW85ZZf52p7F5dhF2cCF6m6bgqpdqlt +zgjnvGu0vWr7442Ae+7S7xDW36WHhY/dZZkPTMa9Czlw/LDl5+cH3frp5weD++mp4cdV/RMKeWpX +Tw2PVFQiaYJ8dSKfDzjoJ5WNBi635fOBVf3oRTATmK7EJQrJ9K0NzxOZwHLFbVBI2lc2PB9mAunv +mKA2VkjaW7b/90zgNyr3Hs4ENmoYvrf9DZoBLXItpGU2Ax7iHxc3A5bm6oc+N6iavgrp5ZQ60hTb +c2kr79lyiP0TmKbWATzdmwHvK3K9Qhq0w/bf3QzIDlBDrBmwU4m7FZJpp+1/qhnwTMBCved5Cgve +j5sBmWtNE/7u9m9rBvwV4LnqZkDbPK58/mgGlCvHBLff3xy4J89CWuPmwHr+cW5zwJVvIe2y5sAd ++Rbk51HHvWuiNlNhB4C7Dq1rA2Pz2UskmaeuNVwzFNLK+W5416UDlVrpJrf8ptlqSskvwv35nhE6 ++X0DKfSNW34JLrWA/kuWKzGukEy/uyXMNwoTq/hLifXW1Wg6aqpbKEz8odcFHxjiMoXUNNItnvxJ +Ju/1zQG8Z2L4t20y8uIPjVBXhRTapbQixZUqVJp3YHMgW/Xss/1jmgNtghb8k5oDtwctpFU0B14M +0kNW+ZmK+9abqqjmkA3psVMV96JC0t61Tf8fp1WsM/3/S1HIDvK+Mm9WSKFPtIN8p7g/FJL2YfKp +T534o8FIaTR+W2lJHs9m4iNj3N8bDayetoDxKZyEOvfn5q6Ms0+mcJAvIMIMvJMEMbW/hYyN78mS +fLJsR64nJud9ObpRwKN03BG+pJADdnFxaT6ekhcljxJxr9iBGYJ5vdBC/VvUkscU+oqD4cJ4EeaJ +lj2FzE3Tz95Q299TSJ+fTJF+1qyIrcB+9o0Sv1NIpkkp0s/6FNX0MzZY5fsmR5akmBzJ/9jEZIBC +ij6ltFcUt0Ihac8pLUUtT1VI2owUSFIMU9xshaQtTjwxVxAsKMuPy7OpXiuCT4wZfT41sLqJ+EuT +Q+J82f+exGZ4ETeSA/INwaBgMGJeiylxfxEXDoOCkfgVgVgohk9lTzQQ4kI5OOiqoDlkiC9E0xyi +5S1YH/7kaxxfyqHAzaHkpWRhPlfX+qlCrpr3qkLXtItk42UQkMePI5rfbcE3KK+s/1koGHe1G33v +PrGySFkclapj7WeOy9W6wtT1zN3Oxz1hptb3VN2/uCxWFCxA6cqfbdQfoJUhwrpuH/CfdV0RiOcX +oUorWvvFcRXOlEv+vhng1DjTVb+OG92LLVRXOc/FOq8pceqc72Kly0u4WbAO8IXxujyqXhNm8lbv +ifBwpAlgPDqkRzCOKq19xZfHWbFOrBgfdqr4QKp4hwpPoK8gFIuUxoKoVEUzvzpO4d+i8MpSCxnl +tJEtg8Oi9dnS/zAzL1Ash00rVd2QTcepnW6xfbZRbWW12hcsxuOSyL/U8iVwXln/th3lmoJwaThY +EokPQblqXa/QSaQ8YIMoKqWiBK+T9RglVSrc4etkE/OAz0TJC0lKCoNxI5et/JdsTpLz5WGLiH0R +YaN+wNUfdsmz0YR7OPwRE8Ijssr8lpjcD9nwwUF5GGrT+xZReg85oF31IcCE3mql8Wj2LVE9mj0q +aqHOah7ZlnPZu74xNvypkCsCcy57lSgzmwNq8FSF1Ls78eDUaCU4DHxCr9mN4OP9e8rxuUKq+N0c +VXpfUQ6JwvJ4bwGt1axzFFLsn8Sa2yjBYaBwcs3dlaO3Qqo4amruoSiHRGGew+LaIXO9id9QM0P+ +JtGoTginLeVwW2dka0uOVqjp5GsVw59WvQFutIglDg6RMs2ECuXfrVDlmEETpUUH/Fuuf1lxMfCt +abTrFCaITROx6Uli8eiQwqCRzFWJ7QpV0peHmSL4Scw8uGGmLd8UeEY0BsqVeb5CRnCmXa/m3H/7 +uCbXXfGk5HpMBcYprEmu8XHJ1Op4FgXCBaX8SHWL8WyRQrWPAXlJDHwnzi3yzlgspQMshbHE5ibd +ZWVJMY7Khcy8vpTjX15Z/xgqVeur35laVLscNc7V9n7Lrsd36uaFepUKbFfIBKnOLu+85kCFir1n +1/ugOb+D+J5r4zILnl+aA0dVrpZWyHX+jycZ2rmK66qQtD0n1fOxDj1C6FNSA4UM/Pu2h2cITlKU +Q6JU0sFiPp/3Li0J3mw8V/bJW5M8T4+Yq21CecVBYIM8Sc8o42KsNBrEVil/VWaZQ8VV6urHiRH6 +URX/rJB2VEdIYvC7Uv5WSCc22pBR4YjiUtUs0j4xDh79F4mK2Tc5mGRuMH2zyrDmqXRYocPKFshV +1q2J69Sk2GgXrlDh3lVJEfIVRvCFbJukDuRZ4Q2yBP1SEu4GYvYRE+MnnF/JhsrUgcmZHQ3ml0YL +hCFXVXt/cKpgm7vSBxYBv9rsb1sHcvvGKV0wiK+HsMfmO4JZLMSwVyo5NojfOX3E0R9/iC0lgy34 +SosLuhQXs64YhrovudiN94kOBwfhoM1ixyE0rrrXhcJck/Xi16D5A5CpZk1U6FLz8oDvZBNj4BAu +77DVzTX73CEW5KvuyEfmboitbn9BXeDbIfpVd9179UPc9qrvLIV0Wr/qvuJe2iPjbaWqqTLPZ2Wk +yOPhEBW7XyHFtyUecqO/vfrTE1QoT8NtyRGOADPMovNecWGmPHZsvlddwMfGhZkp4kLafepCl/vU +hfmqd6lC2qAuxO+rdiFH1cxPkUfM2aSIC7+r2J8KKT4/cUHepp2+OuLXvMhU69f+lOxFEbAyhbnw +zX18QfGxpOKnKRz76tzPZ5FTzwAuVuEvFLKuypR63dkp/DeeAWTfb6HeHSxCBL5UxvwfTW0UeFsF +6lEgibtAufYoJPfq/+Teq1zF22t0r0niNjaUKH23Qmp95wR8vyq9WCNDvnf/xVd0BvCQ0ocpJN9n +Jwx4NJhfHAiVoEo5T/rZWOokfxGwVWIeu58Hn/m8QgyfCLbez4Eygl9TeBftsAf4stAcx/lDnjS/ +J0bu8ohhf8qIS9zIeFBnVyOCjI1sw3D/0rJwASbIGYEXH+R7+I38Rjwa5HW4B6SuL4nGJ9zJDAwI +4qAgaz/E1DO/Y5T7iUnhX1PkV5AeJ0l+orNCCS945NeAVj5kmd94PLWcGcM1lm+ncflkhQzWGI+n +tg2cpCiHxKzhqx8Ows8qbaZCik3zyOQ0XVEOiWJJk1N1vmv45bAjMjX2t+4xBiW0wXwP8/6acr5S +5u0oMWC+hyF9gCi2ipy7mC8x/Jy4MDnYTI0e5uNnFl6WUkBKXBUUYJlg5hATK8YrHr6xaf+IhYz0 +TwFfIITlHl5znfsIh+EAXvO0O92NiSyJDXz0xRsenrmwhzIVAlGexXrTwwuuzyMmFOancCsFcWQo +X8J/Bp74eFdOfDz2KAfxz4khX99wiVGw1sNLhKpIzflCjmh8KJW0esxCRvaX8liwwcMx+FZiMr8y +b0U+8vAB8bnHZID7yFO7txufkr6YdEncGD4Wj73DJG/4T96MFGncwwrZkq94zI/CDdhl2mLkLwaS +ts0jLzHm/ovEbQa+xJApOn2TycftHi+XKzmb4GSB5Nw6VfeVQqrdaXLuA0U5JCZPFsz7rHGq9SOP +vM/qNUzf0yzebaz7SCFNkfc0TNSNKrTRmJL1dbIpXX81sjcopCmfG1OuVJRDUlNqJtDqPDZ7Trma +uAP3GpUJCTzMywSeNMxC9TG7YV6m5/tEcQrLLw7EYsAwL9+5HCY2HAwW3BuMlhLJtu04nOeFYrcF +o6XBAkwRXB5xMYz0Mh8mDdf95SNqiEsNoU9PJR9t0QlHDNclULlyp+5LMt9XhLFi/TrWpZk0VoxP +H8FsiwTU+PFi+81EtophlJdPPtNHWOZV8UuqfbVC2vTEf9vUMxAuCxQjU215SGFCSJeIUdtHJIR0 +iVjV4HF+qjskAizxxv6wce3jiQ8G1W0WC8opjRjKVfclvyX5zbF/hdTxIBWa45IrpIYlREgkoJgf +iYmxxIY4daQZMt70csi4ZiSn669ltbZK5McRU0VMCG8JZs1I9knZ939azVmgkIFa6ZU7iWqPsmCY +DivRUpvJ9LZhuoRM/5Wj1weDBZJByFbJx34/zutPxevSUZZz8dWnYuMrRKjXBrNzlIWMnG8gPwPw +hTjbeLQFH7M2WCBfR3wlogFiZdQU5CZBbqhGXhfiN9i/CBZjHB9HqIUVCunj58bH7mT6Lx978vCN +eXOKCvVu+P7jvJxXi11yyBi+bYxgYS1eEPAWSzG8UYut2PAJC/WzVLC/Ql8Ir9Tikuu6J2jnwbrA +11pHlULaubCW/ArgRDL9l500ExWqN3joOAN3iYFrn0hI8F21OGacNNZJ8F21mOC9Wa7phthVi2PI +8LH8AiuC3XLj5UcsSQ7/Kjpc49h18wMDtfwIy5EIDkgcUsaTGg/wKvRUvpK8bjx3oNjS+ThUK6fE +jYfG8yVlzmZOldxxl4ZdIBcqLqW0IDlHxhR7SATGiQDncExPZe/InsAGCBTKDD49lf7FJjC0U84A +9mt0DipkaP+p5X/lDGCWMDH+fyvxmEIyHTDx/1qYuMpx/WHi61VIpiO1ZJVjKcoh6TAvq5zWSjtL +IcWO1ZJVThtFOSSKRQDvqjOAfZvN9PdYbX/VGcBJEy2k7TwDuJF//H4G8NhEukjDVqqadxWyhnG1 +xbBKRTkk1uAsv1L/NP7UV0ixaeanCH2KckgU44YaJ8NcpQ1QSLE3jFhAUQ7JEfvnDOBxpX2tkGKV +tf31WwCvTbTgz2wB/DrRgufCFsCHypTIPOWkeoEWXHHAe3cLIOtbE593a/tHtgDOeZLRoOzFfxm3 +eihkRetqq6yHb6hvUUquQnJ8VFveVhdVq/EdMGpOVUimXx018ry5US2YnCoPa4tEtIkPyFaRbgop ++kftelk+Wu/l9XKZW4zx01P9vF7uxyf5rrQRcOoky1wvd80kfW4sPGjMKFdIXXoH1QOTZMDnVuOd +Su2vkFzTjVmLhcuOloXj/A216rmrMFo6COU6ZMT/MbUkzI2LUzm2bZ/kDB2AueKywWReTlEaD+hl +OFghHe5aonmk+E0pTmQxLB16lSDWEjEQ5rrFplOqx4a/ZWwonML5bQsHgoFYk8qNjKfJE5BDI2tE +wwby7COPID8RZOupFupf87exfrZC/cZgk6gOT2WUUL+ButhNoS/qfBA83MfDys9TUwclVjPFSwtK +McLHIeXzRIazlNFXOjAYDRQG8bjwNHpKKpODw89pbBcqZJu8liqHlnuSTZjWKPEThWRaZZjKn9KT +zS9Xc/+kDv6hkNw/G+6fHO6GT1swXxpnfWeSbLhPPuc8dNhECWo6DysXJp5xrE4N2dziflS5sh5S +mJAeE3xMj+uetsA1ywS5zO4hVn18pjnqdI2II8aOOxSqUl8RnhSdS59mcmCKqDz0tAUZ49ood0eF +dP1J8yuabRXlkDjqOGNcttKuVUixaeZXHrsoyiFR7MSPmHRAF5O5KrRVoRrPhd58sT77GROR+WL+ +MJZkHTnfx2l2wzPqzgiVf0Ih7Vpo3HlcUQ6JdjnuzFLaEoUUW2Tcma0oh0Sx/3aHzavrmyoV7Om6 +guNToktLxSXXNOPSUnGpB0vi0lJx6b5pbC6Z5z/3cZ5/aRr3H75jX5Y5erXcUPg92Zw5erV0loxn +dS9jpRQB70SO71tN1v7k8y9vAfR41oL/rRbAmGctpL3XAnj7WQueH73AlmNdxOBaMIYzGF/W8teq +BfxB5vRaQJvp7JAy5uaq4tXm3r87puuYO5Z/HGgErJ6uY+51R02G5iqkYh1z/xR9Mub2UOq1Csm1 +2tz71+Y5TkYcmEeqjU8oJNMPhukOYeJNk88qcZZCMr3iMzOhg5uXQNvuc2YzVnGXul+okNIrTRXj +WEVSj5R2D8XibTtGgyWlA4PI1VZfZ5kgOgldHIrFgZkntTrNjXeeYwJwQMBMuWQwteJfj0M1akPh +WDAaR5Wq226fSO0qUXtDRbXaVaL20SS1gYIC/R0vwG201PcY6BgZAV6sw8niQ1HFEs8KHq2w9Mbl +GXW4Cu/2PPdWIlhQh9tso1kqNYf8FtXhyP4ZMQNC4QIsrsONk0YvcIrDK3VOv9uNwAuapUYaGVVb +5ajecqlq9gscbfkj4llVJm8X1EkrqgV8Q4KHq7PBavyDCtk+S+vIZ5d1XtTPLocozeFh583y8/e/ ++rzIBJaF1/vK9JlCKlpZx8PNkXWKckiU53qNP+z9udKOKaTYm3VkK++ZF3Ur7wfW4v20BRBRNz6o +4/+1BVB/hoW0P1sAd/CP9JbA2Bm6Q9AmxTRGB4XUO9acQFs9g/nPE2gOMTuBaRxPoHElOUyR4xVS +w4fGoeGKckiJDk1Q2psKKbbeOFR7pjqUM5Nhq56LYsF4JFraP1QczCvLHxCMI1Mz6X2FCRm1Oq35 +ThtDZ/LRFViddknYjQ9mMqPwTtpnYTeOztTXWZUaqzVpCS/8rlGNvRTS8prXWXQ7oJSwQtr/bpq0 +Y56iHBKFkwZxTkW9I/JjeKhS5tNqmYZwfIgB+9PYD7rNMu+C9otHg1mSpRPJO23Mn8VDAt9zN7A/ +/kq75HI3vpvFJ7JcwUXwj7h78mwLqO/zmkqaKPTFInikbk6pG1eS7uWAXPm96QKH0mQQHjTbggzC +88ghfeFMlb5IIV0/mhi9jkpwGBiAmuixkouVo79CqjhmKtziVJg+h/nHWDtcsQTu8roS60JFOSRW +xT7DPjteadMVspKDiXZOUILDQOEaO1nzh8qxSSFV/G1aeb2iHBKFk1o5HBzEm2Kv4IXwmdq6l/hM +AzitHA4WB0tiwLt12ZZd53C7iN9mfVB3R6kbA1nuHw3GirBZEHOJiGBX3ZdPc+PbOTqmmSIy8IP8 +rM+HQj1jriSBQX0hqJvmqoApchQ05K1CftYhmyLgbdwSyN1m0uHDuv7LWgI/zrWQdmNL4NR5FtLu +bAlcM4+DJ1k3KusXhvUBcpB1Mf8g6zayStO8r/H4QiHj+kndhA64TgkOA6Nb3TRSW/aPxrCtprZT +5qthV/MP1nb/fMewxcq6y7AuIgcN+4F/kLX+AifVQqmmgQYqpGH7TardrSiHRJOYasyTZ5U2TyHF +fjRi0xXlkBLF3lLaeoUU+9KIVSrKIalYzWgYDg6SW3skv7I1r+IKNb/qOvl1RPIrSXpwXLOzCxd4 +3SKl+UUoV/GOJ5koqJr6bi23UOgrDvCXXtOZpFcxdnJ0InO7aZFx6QkNWaEqZyqkH9UNKbH7VSl/ +K2QQKtKld+9RlEOi8L/7mJjfs/raoWw18YBCdcEXDcbKiuOYJTbft0BvThquXE8qZN1z0uvxmgZz +R8MIJTgMYkE93tHAwxkcZRYrx5sKqWJhuuzbLFGUQ6JwVmIrtGnHxbQ0o55sRB0T+XCaga5p/Ljd +xUXSS1a80I0vFlhIl017rvyBl6wezdxovJCrsCExfp9K3NtnuZG3MHl9l1CV2TbQKj5OP66qrVLV +JCqVrUJslU+dNhMR4xcgijjjJb7slE3bH4Xj5pd0iDFF+fgD24WEjNztvAAFRy3eCfDcSxaQUUFU +PobbvI1gx0t8EfYTeO3TGDkH0WgRFwDyjD1Xra1UyLD8aEG+4HVw6xNo2y1/ph/oSQ1ye0P5T+Y2 +i72WXFpSToKHVz40rmv8P1MhNSvT+8LEewLuUeIwhWQ6avl5T0D6Yo4zvN2gUquYaMuNGrcs1tsN +pi+24OXFGdhhbNhkLsv4hJLmro1spYyw5a4D9xI6Xr3yScqTsohcK1OlzbboFGN+QqbMsJkp5y/h +4oc3E2GGnBW8mwjZt485qMVE5ZeG46FCovh2aRtRsmnrYNoulVtmeM9EAb4Rpk+W8iznDllDL7In +/WGjy8sWfEOwWArTXmbj7iM9VlZSEogOwWt23fPd+Jhcg4pKi4N4SxDDl3HPaCdffmKtnGA56RW6 +nrGROEf4M+G9/BW++PwZ8OXjc8mY+YZ3MXHF2Czy3wnXLsBXXFrYLRzntcA5waj5Ve7vbb6mvfhV +pm1pYc/AYLMx/YOghxo0toueD1kqCmnp6Kvc96bWEHYK/fblfOfxCxdeZSXYJX5/uFwiFSouiAbD ++FmsPrqc/sDkWqOTTWNdppBpNNeWhHz0NaYRE3LxLyZL5trmopZ9Wn7DpNUHr1nw8H6WqaqkQiGV +OZfnrFPcRoVKMwqzd5sK1huFR17T9ByneCphBXXqG2szFSYp2ajMi42SrNfVqmX1jFClQhXyMP+/ +VlyVQqXBO6AlkPmrsWq77R/ZEsh/3YJ/ckugkn8sbAl4Vljwv9oSuHAFQ8UeN05lvjBGTCYH77BZ +t0KtSTvFWFNfoalR+tg/1CJmtVNiZ4XKxPtEOr0hrXfinii9RO5fQaWKtm5k6kvojf9Ib8x7o7o3 +/iO9cQ4RTm80qL1vMC9lOD0mPGes5BMLX84MdTNxb1ppIaPqV/aAorLwAAxz8yqMipXsbVl7qtGj +BP2ZMAsWT8i1OA3fZMrureYbK3zxN8VFk3p7TRuUmytuZpMkhH3JhD0kyEVMHU81Dl+mkLF72FzE +1GwVm4m3NWXvM2qVUCQEtl+5EkaZe6GeWWWZ1O6t2nIVUiuzkkk0RnFPKlSaSe0qVTjMKPzeUehp +YMxMVahCovAMxbVTqDSjMOs3Y/pYo7D1Wzqml7yl11R9qVJbFBppSZ5X3uKykj2pe0NTfU+FysRr +bY6QydQ1TuuaaOrqVqnR6KdSRQqNtDH+fsUNVai0hKVe0uTBi016Bvg7L134uUy2pusBhQlp+7zc +oDaqkkdtyIrn3Zy3V1VaqP+8sn+o0DdQtM2R44G/VfII4gDMkbvUWrzNGHAXIaLMDyukpbPMr7rc +TKb/6GS0GGhs4re9lYEJdr4rdo5823nFwacpsePNt3lMAj+4+ZbWvZr3spWWhQt45gflKb9H3TDr +2N5lJZiYwjns6dXsf4EYUTxH9KScqeu2xoIvP4THPFykvMLS3RjmYX/cyULNfS8G2fUdCz4J2Xg5 +ZjOQZcMkUZok2LnE9g9FY3Ex6Df3DVE3vn2H/fs3GoYpUnn6uxYy8Af4UWvyVPaU0Lu+S5MTprKn +U2jY40TfnYXZwvQjS7ISeFWol6/lsMJ1nCkvYjlo9OP5FE7U3d7Thdx+kTALuRP8nZHzF2dAuWuD +IZsjFX7zHkekcSTdjeWCqvM+XTtA5rISvCbxvu992oF3pIo173MYysg6JEedDojQXyJEDIUOitBl +69SyQyJmLDvB3/yXMe5vrXCkhw18jyM7SprPyJ7gb3grWwLph03Xn5Pi39oSmLGOwxlHrVwlDPPI +Ou+rdTompH5gmeFr02kmTasUMtU3mVnn0g/YHziO/aNEu4lhVibOOhEyQW6xaqfULgrJNd4jl8+9 +SC5h6qfEfIVkmmyYviST90BLYLFaPcrjr9cKqP0h3WncCsA/xs9RHv85rYBLSPDwOr9LmxrLrlVI +taM8csdZKZm8l7YCslV6dYr/jlbACx9a8Be1Ar4QDlZcoRzvpEjFZ61nxdSfecRUvDFFdN4rBFqU +q4TvUsSi1UJgZRVK2GkqS92glV26gTpZGY4anftNZQtJEGcua26cuU4hndlvKs76iNKsOKdGmqF4 +mQRPrBVwqIWRTm1pIKV3ppjAVqjQIVPleR+zhelfNx2teiuk0CFTZYhMUuW+GmlWOY2E/zEUOu/f +Whs7HmtjoDMgyriDcrl27eOP+VDIbo5yOSh6ykYLvlY1g055yuF73LhzowXPc62ApqrzbIU09+EU +4/+FirtUodK8lMs+ZkLu8PZVnjsVkvcx1VOiuHsUkjYsBeYZS3GDFCbSHlDcUIVK+49AycIMFRqb +O9onxygCzPLwCWnGxoRJY5YMy0cYIjOcyWC9xMNJb8AnfNw5xmsBAoPxigwe736i45ApIiMLLt6j +9apQz/6UQ1v1sGiQ/T/lECfrqXFwiQ/zPWYZdUBttdoaW+nfUg+km2xU3ldM13uKWmQcSncZJa+a +cWj9p5YZfq5TJTkKqcxZPYUVV6ZQaWYcmajIaQpJXOmBtE5mO2PZBQpJ+9Ljv7kJ0PIzZjwXel8q +cbtCMr3u+a9m4j44crV1Lu1o9DuZHAG2SSsVf1a9ct4mbbSJCGflbFC3fs5pUFbOvwjP6M/5Uxsu ++a3cHdKGn37OOamCqNL+/c1Js91C6f0FKVWk5OOYzPJPESVuP6zmTVJIj3Z45MF/P5nkBtEcyzTE +bhOP+JccT/goN04Jf3nkGW9FNWFjMuHir5wIHtN6UjuYcLC+Ax652PRhMsl6Ot029SnB2sT6ODPl +KOGoyYir2OpcfzZSbZkKqZU0zkNXKa6XQqXB83ErIFeRRQpJpPJWwLJNFvxHWgE7qquv0OoPm+ob +fW2ZhHxIpYcqpBan+imKm65Qaab6BYpcppBEKm8FDPtaq3/9a8f7Kq1+v6l+t1P9CpV+WyG1ONV/ +qbgtCpVmHtc/VaTDROLvpilv+Ya1Muez3KYx9ptW+loIntZApIbQsjVw22YL3vNaA5VuFy4H8J3l +790aGLNZRgU2X3qKUVXulYVF5WZdWHT4Vlf7sTNNVtyrkAbpZlLsW6YQG3umEhcoVCY+bKwgEzwF +rYFHlDpcIbnKvf7HWgOuLfSNBkXUoEeMQeds0QZ9VYUqFVLYiejHivtSodJMg+5Q5B6FJFJ5K6Bi +i4V6f3I9Iln1GY2oGd9LgtHCYG/ZTOJdAtk6WjzXyQTEGTViZSUx4AMv17Ce7yykJ6yNc4JRrvXx +gZdL5NnfcYSOB6JxfCyIb4jg+G6KdbZaXHQXaDF7K++bSeGvwWOjXJX4FDHw8PehQ/hUVFRRpsQp ++auowSl1r5JmfqI1UO4xzfyx1z+vNTCEFCFUKuFTQ1gghJdb812MkfjN61/fGthaZcG/qTVQ73sL +/h9aA92+p/YTb1PEhsSuDvNnJzRWtTo7MbtUvpqPAD04HQ7+nvtrXhd8xbiGD03zv2eIikL947iG +IWv8Az99I4PzsHAt2a7/gd9i4nq+wprPv6Po1Xynje/4d6wYfVfG3Dh5Gw2UrdwOWab+ixTSiuvl ++sfuZPI2rAWk13IJvle9TrUA+M+vBQzZZiHt8lrAAv5xVS1gK/n/22v+0Bsy1duG55paXTVeF9Dr +lj9WTy8F9GAAy2YqMYilP1pI5bpKdnr6lJokHCIllGwqcwOpulEpOOHg4kFKKLt2IDmSFdSQH3EN +GsQnUfn24+pwQXAwxskH/D/9yM2aWtIaEwXTcTv3x2vzZ9CDwYKrCwbLybxJQnt5O79uMWgztxn8 +Tw4eFa4jF7rR4SezaskLxIIBVMi9lPIYJB5XYwDvr62BrFTTChUuv78N0HqHBX/TNsAC/pHVBti6 +w0LaFW2AljstpPVuA9yyUzflx5xl4j1ZIVt5oEkAB/V8AmmQJMConRzEyl3APKW9pJDi9/qnuIBV +1TwObWkCz33C8xt5IJeAl6sPC13+tk2AFj9bZs7el+oCTwi1g4d3gleebcxdq5Ak3nfOe8RzfSYM +S42K0VRBmQqfUcF7+MnX4Byjo41C2vyykfnjZ/7SWhOgsdIcHq1Hjpx2VtqFCo28OXLqv8AH3LGL +neg/eznzXZ4PzkamZvtkhU7WFwcH8jeCH5GkGbvLXOv/iKTGml26byF5K1lcEnDyfKiLiX6YLP8j +1x91Mdk7/sJho4xbKJKhQ6WyshqsSVCDnvOLBaQw2SrUVP95JoquaZJ0m3/RpEvbbUGSrstuC3Uf +jJ4NVxlT2iRgfLcm4IrdFuoE20DehQ9mD/91t6ZkpdawTiHjO9Dfxs+9Z+ZdpRfVtM8TeAZBUnK3 +ovYppLhJyd4izrT9TWl/KiSPScmnyPPvtusayC8KttXnlUx1/AuFrmmXyficD7TiSfv1vyY8rrTi +WHXsVwv1r1X2pxT6QmhP4ll72AzcPO7Ao7HBPSfIncT6r0eVaki7wGmBagMuogFT9yQYcBHr+I51 +hHAxZ4eT91rwlQRiA3A567uSRak+m5PEqL0WvPe0Aap8LlDtxf5RbYBVey34J7YBcvZZSHumDTCU +f8xoA7zKt6v/ETC5gAWZauc5Fx1nby/a+/M+S67f7cUs6PQbP7w7SSbw22jvwN+47N9HTB4K+Ji3 +khj53uAi1ZytkAbfKJ8b7CWPrO2z6hhH+srS/tzfmUFcD5aqzMMKKXuTeWCLqMgdsrwr+l3fu+yr +Y2b3TZZ5GZiTVl2GrCQXp5mq7pCF5KLfLfjPbA3U2q8rStRNWlFevN+C//bWwJP72eayksupa1T0 +k6rf268LuXYXmtB1VMj1mLOQ66G46xUqDfJc9IIiFyiklwXyWJT2Bys98SAlD+Z9SiXtkK3NVvtS +Y4OT7xEgzhbr8gc3XnH/ld3dePoPLj3qyhP2wy42X9mfbL6sdJduto6RuWwOsdI8FenG4SGmWKXF +J13SXJuFj0vczJMN3ySXRCbtL8s8s2xS66oUqvfyyHRUcSkXG8uVZh7l6yjyFIWMzFSXnz9NUfgX +F9YvtQEiWutUl//jNsDTf1nwVrUBFp9sWnKv5T/WBthAAfOS+0JVl62Qaqe45CU3DjD3KO4Qr0xg +esolqs4mk3i8T+suNx73P6Aeh1UoqlC9Eo/HKW6KQqUZj+cqcplCmvawC+Jodj0T3uEu/1dtgLcO +qKPj6lU76m0L/E7rjKO7VMsehdQ2zDja8qCFehyvYfYrfleWwwoNKyQU7S4xbdNRIWkjTChuOWhB +ah118ASpygM1bXnYUOZBZGp2trrM6HNN47ObXEVy+gUxN1Yd5N3RwOlMyt9YCGuhxSGLG0KtSLj5 +ELe/67ngG4CORIw8xDZLbwss0QoqFVJ9c3/7tsDGQxb8Z7cF7L//l51diouRrdbdojDByvNo5Xl/ +W6h/hRJvVugL4XyOzaH/Vp9fHAxETSByVWrA5ccF4gpWMe1vE4gr6N7HLIRhCtZhCYQ8Q5x7mMOw +XwJxIzmLDjMQfC4+oBXU1goYiO7yiPzSYR3Qvifzv8cW02BiKUNRruJTFSaE4jbamfGPhfqPKHGK +Ql8ItzMUPf75z0hHSiP5YU2KChXb2OW4WOSxjnv/MbHIo4efsRCGKXiOSCwKSbjwCDepTCxKiCg+ +clzt8kldKM6MLCspCUS5nVulta7selzt958Sc2Mt6zCPtg9QbdOjMjMPNqXBLOWXRU0p6xgPNtCG +EMplobaImKwMF68yfVh+9/PPYxZS4xgm4y6fDofLX23kgpyMcWQN4UURvh02MjYSMxgzRPh52Pzh +5ZdFpLvLZg9YJoXRLhsZmfVlTF8vmLdcNl/beHmv/7j6ZoAY7vLzBwB+J0mehauUMN3l57NwS8s2 +z8Lj+Qefhd+xbBjWzFOMjq9c8rScZVN90sxUE155z1ipES288rjIbnMxtPm2jfTqvUeZcuYSVbPV +i23iyl7bRv1WquZyhea14HZhaO6mlZe0BXLVyu0u/+1tgb5uG/78tsAzbhtpsbbAR/zj/raAK+X/ +cKB3SaC4+HqUa33xbse5MdyiG+ekJLgxXM5n3Ztio26CG8Mtpo7ZAhkhf1em2PCFg4PMzfp822fw +DT02MhafIiurx4XzOmIyT+Xuawgj5acaH/LY0JycLDxLibgbz0hhu4eOyfqw4lTTZs9YskBs4LXN +AvEer20WiMv5BxeIR7yMIHNln8o8Z0muZNWitv/RztfxpsHrUa7hmdH9uDC9JGHKr5UQppckTJNr +JYfpJbHfcW2RlNbVoq/sqab8D8tJgTP4J2rbyMhqIIFbLJJvE7OYmMFYInHzplIXKoV8USodE5dz +GpowrTQul5AihAolrDWEChKOj8WV8tMrkv5mdK/UGJzXIzkWJUCFzVMmn6XaSA+xxLzw+BgZp9TT +Z8M7rC1QpZW/kHjNEg8gsJ8l1MlxOltrekKhM07zJ16kxid9Njxr2gK3K0dQISeGeYk18Cf4yuST +gApl6XOV40a2PEPJt04vuF7+w8Z7YnosPgB4QcasOicxxGyuF6VDd2E5Dwtd/GjgmZMSK2rTznx6 +0LZjSQS5WsdyhS6tKw94Q2Q/oqKCQDyAt1zcEHPVsc2T5EMqMVwhbXzL5R/rAs6pwyZOyF2nwrwI +KpW96dXJzuUB70iFhXVsmArflwqfdircpZK/K2SF75sKNyRXGIsPMDUiU2spuS65NgnlBgkl0mzo +gdANErrugmBoN7h4deUYlqXxgQ0uHi/8I82GrwhfCnubujZn3h+lGT5iIQ/7xJM26ZwbGrngi+Tj +K2F+JN0GMiLE5WG7cL1ClHcPx9DGLnHqJ5ff2w5ofrKNtLrtgLEna8ifvsZ4MV0hI/CqicDqkxly +UbNY1fxm1HSqp2oer6dqzrvWqLlIYYKalfWopqbpgvfEikP5QZRr/JYpdNJkMDDUYpBO8dtIH+KU +XvPbyMBpMigMl98faZNhwzc4pOP07RmspjpBSnKipf37XR8cHO86JL84iEqt5byexlCtzcuH6ezT +TJCmJt6c+K0KbFd4BvCvA9p+VdVSIX2ebm5OzFCUQ6JwJPHwkTGvO+9LRbYyP6Uw0bRyNW1uoml3 +KGNQIbUnnx2/TykjFNK0l4xp9yvKIVH4v0y7rjSfV/ZUqMiA65Ni58vHUos/CPJEho2MSjZNHl62 +ODp8QUx6Exd8JREss9YOcqNTfbaPt3E7INLExPt1KyEtTER6mk5RrjW1zEmqkY/An1scMvLq204P ++1zmn0n1OQmzi30pyfEty3nYJOa0OMXm4divxdpiFkoi+EbMWsYSxb6wvqvlxo5TbPMlxj614IhC +RvCrxEb4TQkOA+OY3AgP9jLGj1JIFd+ZRnhIUQ6JwidoBBONjvyF6M7I1lgMVKhp4otgu4TktFNt +cAQQj3udmjg8m9jKgapylT7zBmOcauFYuUskHzm1OrK7JLLrqDcfuyV4R1jgpV4SvCsb2OYTh8Xa +or8kRmieVrVIIZ1MjtBfSklRYxih30yEDvyLROGkCOXJZWIDg+E4slX8zBuTfcpnz48BY+w/vW4M +akDHBoQiRPCZZl4DG75oIB7EBGHYQm96tgP2qTcTbX+wHZDe0IY/2g7o2tCGt3MqkHO6SeBn7LRu +qcDAhjbSrk8F5vKPW1IBqxGTvWY0igUGBhMNVkMv63Nig58Xe85tZCNd7FNEERHGg+fFg0VEDCoK +5ReRhZP4D41s8E7GF+ROxraNnU7xovDfwXIe5tvso8tZYObPsJn5xxrb8DzYDliqxlUpZKPMsevN +asdHEPm044hSUnsb88kxL/HWy6P/YmDjJbf8VhXdrZAqXjHXVVYpyiFROKnl49FAftCME9kawIdu +MpY42RwBdnk4TpxzGhudaxjskgMQ9xEhU/Uuz4RdNtacZgK228NfVTlwmo367/U1yjJVaWokH3+K +MFJjEfxl/jSJP04z4RdPwrc9UbVqkEJ6UOP+y+2AhVrFuwrp/j+eeu8zxvU+kf+/4f/+be2ADk1s +vfA0Wy3qqZBi5V75JKiLohwS6/yPqOl4Uq4SbyjU2HFhcVBc7NeE67+IliawVMgSA/VuExviyfMq +PU8hTTro8dOTg01s+OlM+9NtJPhz1+nsHNUztTSnHHmpVB0Zt5gGUIvYmkO9bM3xp1e35lC5DGnT +6ab1HvPSqDpNbWmsx4UmjTXS/CnT/EZtq0e9CW21S+vcq5Bxq2krfsO2+WZjzU6F9HCcCfq3inJI +FD4+6PRNY56pnt2uUD1kzEeLpV2ampibUpylwghGi3uzm2rMu6j0NQpp0WivxPybphrzOplJMc/O +PD7mhfnIVQ0NbzU+OvYURjBJ6oxl6gif3tQMeU8mBq9cxR9TSP+Tg/eXUlK0Apr6rAnegX+RKJwU +vJLAgGCXgoKoeRDLVA3jFbqm8RWhSz/EyuRbl9mZNpxX1IL4JpMjPJrxdVidZokzYvVr546xsjxm +YBwVqvjB25xYmAoCwMVUcEczG5wmpZDZ3IYc8stpat51XmLOCn6sSr5RSBWXQN4M7FeUrfpJusyQ +UhRVVyFJ2YZ0sqIyFCaQWiuqnUKSrkiYdGp8NNch9IgGA/FgtNs9ZYFilKuQ5/bjvL2G3t7S3HZ+ +nvAaxnZVc9t4OF0F5ypkrdf6V7YCfmvOcYonH+crbYlC8lzvf7cV0OKMpDxs067ayJh+6ZepBr2r +0GnnADDLNanQjZvP4CDgfNM3y8Vv+kZSbfWIkqyVG1i9y/LzUaUav73jOJeXiOZPqJkGAUvk0crd +gkuq0jhekyex81mMl0a0eDeLPDCApS6+JlvTgvs9TV18dFshAn+14KNZZqagPhFU25bcnPmVq+BM +kzlvuCSRFmvxHNPuvdTIvgoZwnNgaJMUN1chaZ+65CVTKdX/r0iwQyUlQpUq+evO46LynUTlhZbV +mfCdROWLlvJ0ulX8Oa2VDQ/duVPlhyqkUdtcfn97oBd5mBcOz8AEnnMkLya3Yu7wi8LBSkvUc474 +toU8/+lcoKAAuMv40C3fwITcOSi+tGjNJRVwUN7s391aHBkpDwyfsJBfGigOxvKDsStLB4UxSn5I +zd2GDw8OoW8EowV9fhvbHKTYpw33qCVnHiPNTLPq+YlOatK5ChkWnoto2h54WnHTFZI2iifa2gNf +K26LQtJGW1qjVrHMkuMWWc1NjZ3+OzhmALiOXyfnamw+V5gQoxUW+9cDbbiFpb9qskIeARYzBLKK +ekvK29rYyIg0lxM3g7qFC/C2xQ5wSls2Ik98PKTKpynsAuAtS9rXIU1MIGmsJisqUYpnTa5tDyxW +J9+2/He3B65mVf+V58bb5CxX1bX7X8FlbKLXG8Tr+9tWZ/kG8XFRW+7PhwaWxvGRJMhfRBjVBfhC +eHq10+QfUWDULlZIf7+2JPm3kYfJ7/C8mMBjkr9de8aNyT9XaYl6TPLfQx5tfg3FN5ac0NmiMrsU +sm5Gje2QdYZJjW81N7Wo8a4dNFbXVZgouVhZt1qSY/u0yNSl4rkqsk5hFwBfmQZ2SGsSSFrhWkUl +Sv2PrM0vLg0Hrw7HS5GpzfaHQlfN3PyzNN9r7c3kbEond7CTkoPX+/QOxtt2jJTFioBC4/aTRQa6 +pl0pe6F5wDXv3+vGLR04wQDX8HqdUR3MFBAvLUFPToSfEhGLhMLB64Jh5BCVciZ5ikvzB+CGHfe6 +MexMGxlZLVxcgYbCQfTh2hU+4YigL0tVZ3JmiLTgqwAKRvAQ0f6ONuo/o/ZB7eMWeddABEHW1b0j +H+mCg3qL5hCFxnTknCpr1etVtI9COtZPng96Kcah6GrLLIxbuITzNm9tG0hv6RJqFsymbCM14wyF +1PmQucU4p6URHGp+iPRs5bhYodYhB4CuVVwvhdTStx53dpNHrZqmKo2gQpk/Dh3XUmNcbKrPGQ25 +GeQJF3eivJ1s+OKBULGWL2I5r1Rab6lwlBAjzYflshdcQYS2wWuC+awagzddbE9PFhflWOViQlyY +ZSNjXEuX+WVqqWu86+/L3HhduNgsTIynZKo8lsXZflB7IL2VCdUEV73J7QHUmy7/z+X/3lfaAznK +8KZV7zMiAc+37YHDGoJjChm3+S54qBMalloKSVvg8lP/WZ3tpCo8rMKlfIn8b1p+VhfszBzyejoA +49SQOS5/xw7A1M42/Bd0AD4ki6drB6CB6mmukPXOcflzOwBHyT2gA9D5LI5p5S6gi3J1U0ju18wO +c4EwDeoATFTibIVk2uzyz+gATCFT4lBfnSLRYCwYR5WKpAw4Lkl2SJJ8cBbfvbaq6Wp/uC653430 +s6t77Z/SzF3Pdhorq7VprJ3/1VgRZUhoLLra8G5jQhOF9OIPNlYH4HLF5Sgk7XFLPBwoFS/pACxW +vaMt0/pPKPNEhRTamfgTvW2caMh3/HpjhtkbydR4vKLQGeYiwNPW2/e5MfdsGxn7WvOgNp6xmOh7 +xZARjYGsNiYCz1jy80jNz7GRtrwx0Jd/8OeRRpxjQ34hqZuqz1FY8yM2b5zDnEpsOfbUPoFQsZzn +5Xo/nyVUqmhpsQmfY2kR8IU17T439pzDlyNxfGmxozU71zb5XFf5GyhkdL42+Vx0LuuufiIIhUPx +juegXBkfVuia1o0jfw1fm3axIbGewRLeF9JWlpQVyppT4tgmIryHal4KHz0WncvlEjBPfp7th3Nt ++AYGivFSytO/2cg5L3ki4q/ch2LxYDjO3+NgLbG2HQP595SFokHkahW3hZOrKgGe9gTud2PKedzt +CIYx01P7ZxtbzuMqrA3fBUXwjIf3M6efz514omLBezBN2Lqez1DIxt1ireF1hXTlRY/c1jbyfNv8 +ps8SpTk8HMTNbW1viqLEYJ3QnWiwOBiIBZGrbqSXHufOEnFnH41FW2P/UrH/jAtsZGQTRftfFvtv +ukDs51bMbFW5UCHtX+Hx81qaZy+wIRPdHKU5PLQ/4n/RBjaKokT7B+VdUdaftwVyHMlUO6codE3r +zuTgsiDn0J827As5C8iHETdwJj7vwsS2FVXmfU6FKrgy4jhuFBXw0sE5rjG7bYQu5BIjmg/MkXli +2oU2PN92AH5U2d8VUvJtl6F1UH0XKSRteeJgUGNDZ+Qq08mxZCP6RfoBX7mYLB/XOPS1WNHmIuZX +/iAclB+mu53FSGkMh+RJ7wkpxqMxbHbx9dw/F9mo/45Ws1+hLxKPwpz/OPtiLnHK2ZhE/i1V9L+Y +S6N2LvBXjfGorKKfupiB5Q2Mj8qvuW1luTTvbhkllOeCS2yYC9Ix3mp3uhsDiIjotdjjRc90okrk +ZzJHWvyZzE8ukczp1QjIv8eE4RGFjN1Iy8+f43VfasN/fyPgzkttpI1qBIy71IanToLUgyrVAYD+ +NO/PlDq/EXDaZTY8R9KAB5TJYb4YwFT+/q5nez1UVzxCuWjAKMtv+YGyyzhVzmwEzFLiPIVkesLy +r2kEzCGTXPWZ2d4lhPGWXK+8+TIbfl6vnHa5ba5X7nK5bS7BX6Fq1iukRctS5Kb8+OUSmtG1gVeU ++rpCVjreklufZ5PLk9cQaB418WulkEyTLf+DDQFkU1X1UBsuDctqKhQuzAlFgsjUBMyKGw2uaT3Y +rRIEgvFIaXFxj2A4GA3lyxc52co7R6Fr2lWUkTGsn+KKFZLyQMKGbq7iHTr7f/Ke5OPKMVkhNTxi +VrAjFeWQKJy0J6nGRoOBgiGoVO55ZY5rVOVKj5feWBYGfrdWNHPj7Gwb6ZECFrMecKM/iyWlBUEi +xh60sSCbe5SF2G9xC/t3lgY5pWu72PC6zgTGtXeJmX9a/sZnAuO72Ei76kzgqy426vQ+k2to4atS +voOWkFOvqCHXcgNZHYyaw5bnbDdwTO231P4+AMZ1BjzkraXIUxTStaOWv60buPQKG6LArzSHxyjw +3+wGIlcwK45rZU7EqFSx1wc5cbuazeuLoj0j8uIVNnyDzN9H+HcwGg2XmvLNXW34ggORVfygG0u6 +Jo7A2jRXRIOBAahU1fMGJ1WRUdGBmxa4ifVsp6489OWtqB2utOVzH0MBvP0Y1DNdoGU3+SNnArlX +2qg3hOjj3UKl1rP3vqT60guCxYEhwAC+gJx4Jf0KhOIlMZTSgLVEhPEQ/07rJo4Fw/EYHnj0QTe6 +EmEyabRrdjM3xnTj3Him7CWOcVFmNTGLiZGEelGQ3bpzhO3ogo8/6j7NtexBNwZ351AcITJSgPku +JuJC4hJd6V8WDw6OFQeDEeB+48Vaha5p17CB0s3eaMsVf9uo6m4jnQsdtORaxN+Dq6AY0JKedu/B +x7AYstIecmNIDxvekWcCizu6QD2d/UvPBBb2YIJUDxtS+6DAgGBZBFVa7ZgHjBnJ1Z8v1bO+/HAc +OF+qv4qdKBjHBYxL96sS8yJBc8f+ZeH8zqhQvS0fTNLvE/e6UP2QJA2FcsFqfn5pGV82q9D5DyUL +R3ED666kJWH05d/7+XdeWX/kHH3IjVZXc7nNNhiIm5hzd1ydFIFwcFBpLBItzUe2qj5cnlRFekkE +GOXiNb8vXM0pMz4Ao+WJ9QsW6f5USYHa19jwlXJNM9n1qNeNnGtseFaeCcRU8TyFbI0prnpbmNT1 +dvB/eLe4gexOpqWmuozcEeU/Sw2i3FMueH4/E7hScTcrJG1a4qokNiQWiBbGgIeNN7cONbC6VaOF ++TyoxYg9eg134aKFA4nILndjOV0J412h7mIhUDZ4IDa6bip3o8m1Nnz9C/CVUHNYYrT3uuaWuzH1 +WhsZ5Z24usQWidIGMgSjUWyR9UzGddxJwfeyLulxHefglI5Alpp5kUL6877L37gjsPw6G/5WHYFd +ws0nvi7K1U0huTeah9smPZOal2Gg6bnq/oOP/isMJOKYLK1yesrGdLnsO87qyTU9HWF3e0SWOr8m +6y4Mxq8qKwzy0rbefNddrroXP5ZUB2M1RSadzOvF92lSKGRhIF6Q2l5ioXQAXrDkVBpL4bIShnWy +ta69G617Jfat0pgZ0LWes4Yl1efp1BH4S2kepTFCvH2KNK/iGigk7S0LQmuouOYKSatMPDBUGOSh +84LSkit5kjBb2RYNTzIhPQq8Z510khu39zLJskF8ruxlIyMzSyaDdeK4NyepuUoi0aDxTRVeNCJZ +MTvjDxY74/WUrB7ISkQqW7lHKNRkr2GLBeOxUCEqlN7l8WTtIWCOzZHt4Rwb6f3DLPKhY1kOO34A +c+2+h21k3mDDc0tH4B1Vs0EhgzXfNp05N8t05sVartDyUi1XaXmZljM7G/5XExvamBuL84cwc9XW +NQrVN9r8ltjc5wZjZKUYOeKGJEVDYr1DhYH8eKg0jCrVcNfIZO8ZGWwRXStvsJHOX+jEFnvj3zb2 +slxaXOCUm9+YpD1UGA4U90SuKtyu0DGRjXbYZqP1vdHmAbJC4LAcjRmRpCcSCIfyuxYF8wd0BkYZ +415X6JrWSybDSD7QjG3yMVWVxAqBZt322bB623IEpw3Xc3172/Dc2xHooNI9FFJHc//YjsCI3kmJ +V1P1WahU5uajjQlO1RzEcDYr26OVncfKWvVJjEVhaQ69ME9R2arhc4WOpsHALefus3FrH3PWUgqj +/1NPX1SpgviYZJMGA3fwAPRnqugOavX0PYFBvXn6s0txMIxy1ZHxxHG68ijes68xSgrl/1NXX2Sq +kmUKEzzsT8PeU2X9qfnv/1aWH4igUnX0GXucYXdTvMtNxjApxG/6TyfzA5G+yFUlfyhMMCxMw95Q +ZWFq3vOfyq4AxhlrnlSYoChO2XNvNlZJoejm/7Lqir6oUA3njDcaEzQNpkmLVNNgqv3hPzWdLa2Y +rUo+V5ig7EHKt73FmCWFO275L7NEWV9UqZb4hONMe8RF215UdeZT7S+T9EnficnFDOUq/4lCtQop +MzsCVYo8c6JTSc3AHA/G4gXB/sEo33fGkK0sIyc5rOy5Ll8Jdlo/cg1yq436xco0VKEvhF0yqdxw +KxfeuZ1l0f6rTN2P3mojI/0sF3ymlnzsEfyHt3LQDN2L36XY8jZ9IDFU83BiSHw8ebMjsPgslwwj +eyz/Zx2Bh27j9vV3HQGcbQi/W/6DHYG1Qmh+EpBdQ7jwJKDO7TY8J9cGvlS7qxR2BZDub10byL6d +6yLyNHrSBKClQvLUM/cW3KeoKQpJauD/zAvEKJ7wuMS5seO5qNBgeicbndo4vsF4yr4BbsyiWPWM +Gg4OklAhU9m3TkkS40l7YJbNBeQ3tzOK+Zgjs2XmHWYoniunIXuzVIDZ9ly4MewOHtY4m8vDCObb +3A97g6j0c2RNsEImhVPu5NAswc45x8R0ji3BvpoUD39dbZIa9bRCpsd8W37d7n4yHe+HPIWchSr1 +ouVTSd74WhVgtf3wI24supNPUfwxKqwRd/6604Y08Dg1Zo0NadeN1WVp17Z3abu2nWpUd1bIhjHt +esddTrv2U9rdCsmj7TpLUcsUkmTadSzF/8u1zshWlzY+bep32jcSwRIJ9eq7TDMtEb9q99ND1wNU +7mGFjOXixIOlDsMgZeD+TPLmznStcr5CqlhuDpY+pyiHROGkzR0e1DOZVqWsG6YlOZBeAHwm6ZPT +zziwRRwYylIkgu/FuVf72SZnss41ObPFNqnS9xmj7TaFtO17kyq7+jHTqlO+2hLJlc6oUjtum240 +JAT0Z6mzSa6x52exJ58luc8OO8XcN4ngDZ5azg/w1cG5HITwi3BsCzAh+DXEm8+aOj5QSCv/TGyE +VUpwGBjH5EbIUDNbKqSKg6YR6ivKIVH4xI0ggzlylX+MQvX8BKHqH0aFMm1QeByztG40GC+LhlGl +TLdVGH+VmWcf57q5nDolj18NYZ6b48WjeTZf5+MlN5d88PUP40P3SJcbv+fZyKiUUJaGg3hZbne9 +MJ9jsc2zDeeZHHjP7W/QCSjOt8378AueM7VeopAhGmb7+UL8GwrDQ+lrlZqjkFzrjKarClTTdUpL +5FFNrxYwq2piFYkGI9FQOC6hjSFXPV/6vLFFI8A3hMtShh6z0SJoo/67ylVHuXwDsTLly0fcGBDk +5JZ9nouoVSndhrqxk6jE+moqq1TxlS8cV9l7Utn1/ROXBoGCgl7hYK9IMHwlJ+Pu0UBJEJUqO+TF +ZB2FEWBHCtvs4f7cbc1nke20jMWYUHlSYkd/numJBgeKTuxMYcueWfh/1audsFxr/UehhssXycde +qS23kJ0wgr1S2USWClmiYWtZqql6vodVH/r/rLpjZ2CGcfnPWQZq5en9JTDYlzI6z40ORTbSy8Jl +sWABUXQ5WpTk8m/i8kwiCyM4KqZtYql/AQ6KQNOQZP1RYbyRBVqNY1L+lOWSwGBu6HCjYYK8oRt8 +N0U6Y6I4Nf9uXbkclDPbyChnhhTEInhYjpV/dzf7RrNOQJX2jYMpad06AScPsOHlpkr2+S4ZEw56 +Tblcy0e80j8GDGBWe3t1AirPN71rgsc/oBOwdICNtHgnYDv/eKAT0KDYRtrITsC1xdpdzp1pAniB +QnaplR5R/GAxFdekb7Qs/K8ExGwjfMciA51WYAKul4PxS4ptcJpYL7H4sZiR4biwwcP9lPYltuxY +fSQfJ9zDkgxJ/JK9V//+3LL7VEL6SCk3aCRNY/hMUB8QVc2Nz+Vn0I+UarCNGDKyLpC15pceLr23 +RXhW5UJeBRItvDlUEC/CJtE1/R6eaSwujZVFg1qvIaRFWW8X7tcZxL1EKCd2eDji7SJKDBG+n8WX +JjHmGQ54OG4MjqlVhoaMrIvla/YqqfytGMeNcoPaLaiUOBfFlxhD1aBfhZIblxd/hdcFw4qZpZiu +geLiJNbdcWk+kxnll5jM+MRkRtMyG3WYGZC0mF6m2dBWG/RMhQnZcKCM6kyaqbJPjbJ2AzXN7uQf +TLNX+QfT7J+BqjhfFfZXmKC406BqxZmXGis/M4rzBqniSfyDivfwDyo+Z7AqXqUK31aYoLhwcLXi +clX8tVH89GBVXMU/qPjCIdoxHhqiiuvOMzldT2GC4qVDqhXvU8XfGMXbqYZxbX+vbUL7KP+gxR/c +q4pvUYW5ChMUH7m3WnHOZSYUm43irPvU4oH8gxb/wD+ouM39qvgFVThTYYLi2+8XxZxDF6vinz0e +Di5nzDVutlZIoWFc+zzZCShVZEwhiYc8Jgn2qZ59xsAn7lcDd/IPGnjbAxrSMQ/oQJZzufFJx5fK +B8QoDlzjlLLHKNtPUYax54Maxtf5B73NeEi99S+4go9+yFRI21Rxj4eqFadnmyr3GsX3PqRWVvIP +WplarlZeWq6KB6jCsMIExREerTDuR1Tx90bxi+VJHar5w6pswUJj5WKFCcqef7jaykpV9oNR9vnD +ScqyH1Fln6iSLxQmKIs9Uq0svYtxeZtRNuuRJGUth6qyP1XJIYUJym4ZSmU1A78MbhxfegcGBpGr +g/2Fi41rzqAfARZ6OdyNGqp7tAu9671urKK26qV8YWmE23DIVung8mQtQWCV9wa4kfKoLfuGb8mH +NzexFMEWbzO48fijNjJyushy/WcvVw6rSJaJZXQtTiyNH+Pwm4958uPlS1iKRTC/FpcAacN4/s1I +r6lF6VuGkTsaHIh3pLybHDlX8Mk3OKgAmwVZMlyCIv1i6lJj9PMKGbvJtfxzOwGvCFuNu8F4IFoY +Qbm6OUyhBi2BsTQazC8dGIyiQlk+Vqis6aIHT9TmQq7eCBOcSbW5mOrGUgSTazP6g0ckrt/iRfzp +5ypVVfCasdtRyR/4qc190/nUUBjBPNH39/EqnDWfKtihUBX5Ypgjes58PLFy9WgI8LqpN2uFgSom +64TVUuWUx81a8R1x7wOWIvl4V0onj0xU2j8QDxQbt7JV3WKFqjY1ko8tIimPKN/pn4URbJW6EjK7 +Rpk6WKmqLn4jyVLGZrsIXzmSuZKP7aJ1FEuxiJY+HXmcoZrsqu2IQjU0vSTGF4EHpN0ajjJf6x0U +xWL5IfMnK/9bKr9ulKyf8qOBWBEO12amPzSKLzM7AWtU9waFzMkJqf/2VexRX7HS+DhToVrlE6uO +iVHLWGFhBKNSmWeHWYrkY1Qqc/CK0abJTOlxlloVqG1Ha2/42cbG0YnxkLNiUn+/ElRqnd1WGRuc +uvsV9sNcqezUMYnCBWbY6FeCXJW46+0kScmlzSJ57Rh96Nksdo5nkQ89prhpjA2fud/yp9SxB22c +9ASXVMXF+CmVAb2cRccPgxpOFC3bIfpXPME311e44JP3KyVBbEllJ/r1CbPM68jzXVtSv//ZNvuU +Hbm5vyX13H0cQLwfdQKyu5oheksqvLs7AeVa/inVb2cB2WNt+OtmATH+0TALmDXWRlrLLOBr/tEp +CzhpHCdpngL8odIEYq9CNvyh1HoyVckRwG1KcBhkj6Hei7QG5iCkR2PpV0gV5b6EU0ReJTgMVFGz +x8FtkgzlaKmQKh72ye/b1VeUQxLhxF4YKyotKy7IKYsV9Q4VSoYgV2VSVhvnND+klRf4mIx9xmkr +L5BflB7OYnEUWlwxjptMbmCiKpqskHat8sFD2hLFLVNI2hpfQpcJxbrklUbjOV2RqXb0Vaj21OWz +9edigFF5ttLPU0iVXyaqjJXFIsFwQQ/kKscn7zke3iDv0fjsFKKDbcbz2SM4OH5rKFhcgNifXjfC +xMXipfxlHgxkur5ITCA2JJzfE4P4Jg8TmM5E9AiGMbj2zzbmEUVNOdFgsCQS74khVHbrRJOv2JtC +LrOnvsf8nYErXeDNBndT57KJfETJJSqEe5nIOyZyCu7GPoD7KH3Fkzxn052PLKy751n41EXRZ55M +MOcsfOYi819EhoPBgi7kxecuepI9yUZGFTWESwdhu4smjpykWeo6BRihEXtCIeN1N7wkLe7hkuJ9 +/ganAG9Syvu+G8i8yuC3u+C9IAuIaHmny39dFmBPZici4+x3TSu8qZC6/3RBaOsU96NC0g4oLXOt +kTtPIWmHXfDclAUMU9zT2sKkDXP5S7KAuybbSBuYBcyYbEOcG6XMkxTy7NieFHFnk1hJN+uqptMV +kmmvYTp9ikSqeuERDcbKSoI9UKWsS9cZQ12aZvwV9yDwiLV3hBs3TjHLiRFy6K7R1P+RF4uvkjZ/ +3GI7jpjK+NGytPeN+gyF9PVxS8x/Q5gYkHwl3qu2kGmUJQFp+5QGZMpTGpAiZY4qpK8akA+fcqrd +q8SjCsmkAbnw6eSASFpqBzgLlWpC/Q+M3RoWznWLJAglFK+OpryoOQ+Zyr1LoSMVl7cSb8lRiuef +1oMD6Veb3FtlQU4R5Wg56dRGKCZdoHegfzCnNMQDXB8ag5Z+ZKBWIcPCL2LZzqd14PtF3ow1foZv +7iPQYhmLHAcN9Y1nbOnGu+VohmuaDV9ZBOPk5VAPlsKcv2bYnL/unabtrjtVqQ/emOX8nTHuapds +1UyxuYbePo0DQhVxoXCx3F01UwgNnlUdpoiMnGtcIE88Ggxilp36uBv9nuVxnsGYLVbMepbt5H0k +C6i8xkTsL8v/VBbwNSkyUM/WoMxTyMR52IbsmuFaIzTFll2zptP1YTP32uRdswot667ZkOlSbbNO +QJVqmGk0VDoaItcla9ioZdXQ+DnRwNMM6T2NDQtsOcKQT4oQcpWw0BBeqyYsVsJLhlCngrqq040L +52hB99KoHFlApeZCx41OTvSWqSIPaMOzPHkVXJD0ZKBxJkfnFURkXc8HGHQk4lfRX1OBbHvzQztk +q84XFLqMbg79XTl+n/O8vowarAyPKaQJPRKWCEMU79Bllq/XjH4BSPkxA6hUlpM+cfxIO5wB3Mcq +uJLZo/TDCllFTsJCZq/iHTqrqF7I1DhXOCgaigeRqbVEP3Vqoz4Xo3YTo/bS82bQu52TbZ8XONma +0xDDX7DheTELOKoaOqgGyuf5l2cBH5GFZ2KP/YvlIQB1Etc2EulYBOWq41WFGucaq4UxXIxKZfhL +4YkZ80pLi4HPjGc3KVTW9IFAlDNqnGu86qSSCvoXlwbiyFWBl78wChIE7z3S3o3ZL/L0LUbKwcBW +M/hhWFl/Xos70o1bZ/AFD3OriD8W0N6N52fYqD/9c6NqvkJfCM+L+OczmAIZ6CXpuUBw3pk8u0tM +DAsFc9FMcv3L2PzSkkhxcDAq1cybvjR1OObmA6tdrTu4UULpZEfLOJjmqsANXyULDgTWyq2jq2Zy +KML7YoN7ljr6nqtrezfOn5X4ACLRE52qa7lCx5iBwGeyZik+Xq6ITij/J5uOs2WL2LKU1YdQJbYc +ZIFB3yq23DDbBlIqs4AqFc/42lFTbxMX0ydwn48hmcrWR6FaWzcC/C4nPf8tGItHQ+FC5Cr/VoUq +xx2Dv1ycLIbNtuH5PguYoByLFfYGcMDl/z0L+GK2Df/fWUDtOWzd5AYyt95VqdSsbxx3KC/1/O1i +L710jg0PHS9VjrhC8h1x+en8pGT1RcHBBWUlkZtLowUxVCr7T98mVxABxsj8+f4cG+nBcIFTPsxy +SSA6gIjLR7lxxVxOoYHoADbHWOuCVm48PpcvEpi/IUwQLauIqSKGp1KfF1xDXj4JL580cnJcYvDs +xBObJYFQGNhi7LqmykDXNK5hXL5CTHBxYMqdZyO1fxhHXdxL48fPwYK+YRm6P5Zl85PzaEuOize6 +masb3me99R/fahROVJg6GFOsksNsCO+pnYGNOS5ZLr3u8mZ1BtJvMMXdZ6Rd0xmoO1+/aZiq9j2j +kNa9e4af3zRcMd8GPI1TgFlKXKiQTGvP8F+eApSRCWYhvVHJPykk22oXPPv8wFNq5kyFpD3x72jJ +xsVZyNVYTVLoxKwVHyk0OBtd3IGYw9qr805WceejQsW+VKjiNXz9S6P5wcL8omBxJBhFlbJ9+r0J +qLLLo0FTxZ2nkFa/ZpmH6m6Ku1EhaQuthBmzuxIcBk5n1TOm5/JTgQHKMUYhVSy16uWdyi+mS+X/ +B/g/kMIYVimb+wfH0hqfuO8aHYBMJV2/zWGhTld6maRUf2CLdUVtNzbP59VBpfkDiOCCL20BbwoJ +BmKlTFkr408bXYjiXUHBbgPJxjP88WpcbEAoQixXH7MXsAdF8L0sQr9hqTCCH2Q1W2chd5HigXhZ +DNvkoSJ7oQ1vn1pAzg0mJb+30opqAbGFNsyzUl91IlchXdhmHjZmCdO0WsAaJe5SSKbdln9xLeDr +hTbSXq8FnPQSu0N1fhSWRuVDplwNztcKtcXTCyO8Q5+98vKXbBjfjZ8HxM8o1f1bm+RsZ1SprlN/ +TAo8A3FIVM58icebeJebqDwkGjcladQvY3uXFZQWIlMV1duRrLAkguFypNe3iDuVEYyQEzFDWIrh +KfvGOm58vkhX6YbThHvcDSbcw20Jt3exzT2NusCtWtNdChnJEeaczkVkkksT9t1gpEcmHo3ppxID +FCZluJetlHWjEXvOlpYpWawts5mKZVHY/SfjX2+FrH2COTzTQ1EOieqTDs9Ezbe3GjAN1K27jEKn +VbkJLlFJW2IWhMvleFwflkoieEOCuZClSAQrJZh/LNH4GSIyIjfy6Yg/04u3RFebpdxl5AUzpvzG +Uq6ayBXBauG4+GXmniR6em8ThDe4Z1UXmK2WLlBIl1eagIcpJZHLVaFPTeSef1kjt5McsiWX87Nx +9E6FVLM+sXluUILDwPhVD0BersgXayUfm4DPUoGlCimQFPC8QEFJPvdSczXIYxRqsOvy6P4P9qHR +7sSFR16goCQ/UFx8Fip2XQH8P8beOz6q4vsffm/27i4QDCGb0NVIlUgJXVEgdDtRUKwfEpJNiCSb +NbsQwBZFEBABFUWKEAsoFgSpIkIA6QgRRFFAYwEElF5V8Hm9z5yb7Arf3+vhD07m9Dlzpt65dwF8 +dQWxA1cSyy/wyS387GSUqUzTI6baatLNZfo/SrtWaQzFaad3eHOg7iK2Q3mvZQ3Klcr5bi5SVGqs +wnDNjyjuGYXUfNFovvsyzRmhYSEUK2eJQtVW4UJ6bm42P99kM9T7M6JCMgy9K1eynl2km7Ojqutv +hfRirhU21xxTgs3Alqto6o0AjmtTf2SZfrfrD2N1v0Kq/MSSA+VvFWWTqCwiDbLyC3qmZwzqjUR1 +fZ1Crau8wbHaYhpsWOREwu1KnqyQo+JaqeFfjKEMMMn9TCdZE16tnipgK6An5dWS4eM15ZitkNVY +b6rxuqJsEoWvWI170zN8KFPuQUdNXMKr8pVUpfliDrihAmyzqr1kIY3FXJ8/OzQI2+Sa3OTFvGbY +TzZC2wWzkZgSYrIDKJUaV+OLJnDf3wqAmvIqpO/brJjcVkC3JU7EjGgFDCW7ZzRXcPeZAJVa3rda +AR8sccL7YSvgR3JUZFdGYAjXQShSpaePRdQn2ucfijKLS/xGS3nGe5+cIhyy+KjkRWJKiMnBUcHs +ICbxft4bxjGLjzy6LuOEjuOiYv4yHSgNjf/krbIFanytQtbssCUvk/26TB9QL1PaKoUPA+gII39e +cTHqPOWPGfkan6n8v8rjVh5bngc2zRV3h0LKn7Tk8Oa2zyLGg2DGIF+mBAzHTaA+OGmgJoA8GJot +N8ee/IwJUJCfEcQmF+M1/zO+dMfo+LFZMOfIkj8Ym13coCctl2M0MgSwx8Vb148S5V7UCmih9u5Q +SB8/ccWUtgImCdOPrYD7lfiYQjJ96rrqLLMHHrQGEvubtFju8npbA98td8LDlx/T+jswEMD4Sl6+ +6ljlc1bbXa818JfqwglTUepc6zJzSrTiaikkbYMrbKCpqgSbgZ2qokfyqN2mtFBOqtjogqddPaBY +nd3s8vauB3T63ImYPvWA8fzjgXrAKropHXucik9RSDWlLhmfXlSUTaIPER07g68y5ZkXWEu0Ofuf +MrXVZpWXOX9U2h8KaeOsK6wz5d3rC/oKhvpu64E0lV+pUPVUdLy8jPy8vHy/5FKJMs0+E2FUXsh6 +yc1zt5Of83cUuCt180FIoxVOSKZNkGt1uSt04K9y2iioqZAevhb+8d1oJdgMjEVFe7RsDTykHD6F +VPGm29ujNTBrhRPeO1sDO1c4AeF+TblmKST3TMPt/kK5O35Bbk4qVbR+LRSSe57bexhAnjBxkdFS +ibcoJNMnbmnJZEXZJHof0ZJmwV6ibDvPmWho7GWmXCUBK/7CGbFgXyUXAg8QywdjwCq5kFh7Ja81 +m93Il24+4jBR3yBK7lrplMP0jdJCk8nLI3JTNMflV3iUJg+ojmtif+mWZyJHVzp11Z38gOmeG92Q +bcv9Z00VnlXIYOxwy+K47Spd4k1Yxfjyicp45ZqmFSf3Zrc8UVlNbj5i8pToE5WXlXm6QjIfdYk/ +N5dQo6MGUE01XaOQTPpE5VkyhSU/b+WP8IUG+QrzC3IzUaYSH5w3NdBGSKik5f4Ko3OwX6K/WPSV +r7wy0oNZBfl5A3oHM9L92gYlKhP4J0KnNGyRXAyKXe0EX4DksQuKPGyy+4ny+wrDUONWc1IakpHh +CwbxrIdj79erdWrSJ57SlOF/hzel/i0xn3TB+DLtLwMZockeibm1Rp9i3b5GY/6aMr+rkMwa86fX +2DGvo5qaKiSTxvxLMhm7df829jpqLMj1grF73rbbea3aTVTmlgrJrHYL1tp2P1HiSoVkUrvLyBTW +1hnpwVB+dljLFKkXqReNV9ra0jIzpGX+WBvWMjOkZa790ry9Ki1jUH2/dCJhsSrbpjC6AO9II037 +krNSeIrwpVueFaSp3TnyGg4i7C8W+1/RWH6uyYzFYh/rwuwbVO91PPj32U/Z13g42k5YxzO1B2Sh +tl5+xbn9er4g/aAD0cOwTTBL19O1+ABxfGBdJpKXiJVprtElE5ibFTK4WzyG1ltxmQpJ+0ppYxQ3 +UyFpO5X2o+LOKiStzAOPFQ+UPmgGkhMeb8t4oNUGJ2I6xwOZ/CM1HpiygQ5fIZZ6QlGikUx18MWm +8ohGSPTul6+PUPtmpPuRpqyHFYYnwUuVeFBSY6NJAuClSuybt7Hs9xXa5Ykbww/ZM9KDvXsV5Oep +DV7bjzLODFEYbmGqWNhFjfIG9FSxUGmTaWNo+eZN4RZ4d6LfIF9/jlj9c0KD+vry0lGkyg9bxpga +kfl2vFy88m/io4H0TJTINbfFm5xIOK9SHzuNVHQAMypzCddnM3PlIXlO/mZl1vr1zQx9wsfKuU5h +dABzRWIT6fHFFAngS0H9Q5Rs4z9ROysUstEnVw5bcs1Xgs3ASbJ8ipdt/PGHTG4sqCyz6tdqf59C +CgS4sk4AXBqDWIW0tqyy8aSv4nwKSdse7kk/JdgMVFzuiVQmSzmCCqmitLJcE8pWlE0S4fAxSC6Q +XdZ4cJnw73AbqI0X68vLCfXjcVrPofxRDHxfmaN+yy1mCt9bmYusgSyZBfuvlblgnyuIVvhd2mAv +S8ZqTp4Pz1fh2DB6q84bRoWZNwwJCR3VmX4Ko3NzgiH8WJkfrPlqKxf7yQ/LXumUGEj4iuMMMXkB +nBWXbv2KqSLHQmUPm1bbW1kO5J4gxf1AayBZtd+okEHcp210n+J8Ckn7ObyN7leCzSBhtp/Tynps +uXJsUEgVR03mfK4om0RhZg5PrcqUtl8hxU5Xljc/54nrBa0Bj7ZSjEIy/V3Z+3pr4Gdh4g4hWYl3 +KCTT81XgntUauE9xeQpJG13FrJ2CipupkLQJVWTtFL9N104Pb2N8y0fBPGneZJSpSNATkUUyBKyQ +W38TtjkRnR/sG+I3H1ZWYTKt2eZEgkslOis0H4beWIVXJM9su4KxVihS1mCly43tF2NNtzuRcEHZ +6ihbNB8DVeFjoEe2X0FvXjKKlHOzQu0LYdVNTS8YjDIlt6kcaT8bs+Rq3USqrwiRb1hOCCnKmx8d +IRNrRwRLohmStdt5hoqloqdyKTsblkWzr91S6kTCL6qlZhWjJTog165/jE6aYGFkKRcmy1sDRco2 +SiEbsjTajCAbFWerIm1f+F1Fm+EbZWSORib4ILU+RCFVnIqWoTFHUTaJwkxwXnF8Q2mzFVLskrni +OFVRNoli/GAwtzefKm2VQoo9W1WsLVSUTaKYba22BrqxQoptCK9nHSXYDBSOrGcb5eipkCo2m3q2 +VZRNonDExkqP8FJRpJzLY0yTaU7JEd57VbMmWFgizRzA+1XZzodY4mcV51blZJj8tQ6XhoqEelWN +nvsVRgewUDgzv3YCCQ8req7C6ABKhDxFyN8oevxVRkt0ANuFvO9r3j98RCbd0qqcdBvuYCeJLyEu +gD+Eqz9xMrjiUTO4vl9VBtdxJMgwtkHru1Uhgza3qgxjK4WJqWATvw9j+rBq2Jz8lRJsBsa3vHFk +Tk5VBzaaVFisFVunkAJMBc7JC7SyKxTSpV+rmu5wTnEubR7STlaVCfX8f0hUaSdlkrJ3UEix08aT +GxRlkyhGT/i45kalZSik2JmqMsKe2KEj7I07Gffy4aNgiL/8dlsvP0pU0lfNNKCmU3QAE2OYL4N3 +6slGtDLUUkhTU2LCQlxVCTYD/SwPsfS8m5Sjl0KqmBEjPa+jomwShSPyXz4anIci5WscF+Gu+W7o +QvF45k6n+STRwhiOzV+zyIObhTFcLtT6Rg9uFsVwPZzGYl4AO2PYVyZ/w/n/UZ6w+grvzAmGsFLw +e4gvI5577DysEmy7XVxX/k+4h4WwWpA5uxhtuPnAdIE6u1Qh67s4xuTJb4o7oZC0L8LDuV8JNgMj +UhFONv6pWBOD2tUNpIpfY6TxZ+zSxt9OdyT4fyjXBYXkLjXB/1NRNommrhB83ZekaOifUGhnTGMT +m80xnDuivg1f5fMeQB6KVOCqeOOvLZgXwJxqDH+7b52ownv9JdXYNpy6s/PSg4Mxsxq/E/fkt064 ++fToM1W0VyGr8ko1uHm8+q/i2nqNEdJmVYvjUSvcPGeFEmwGc+aqn5eDe1trIEtZAgqpY0G1uP2t +eV5e0ZP8vkLfsFBBeh4StUYzFNo1y8A31Tj0zf/WiYRhSnxNYXQOvhXqOcYqPo2JlBfAHglF5+84 +99KZ/sqeqZDO7K3mpTOPk6nCn3y/725fYU+6dBeKlf3fBBMH26W8AA6LhXe/48oggCMS6yMs5XHf +jfGxbIvWu/XCydOqYIxC2v+nmlw48e2mk7xwMlaJkxSS6WI1Ly+cfFTOtE2JPygk00uxwnRKmFjd +n5V4RCGZJsZKdRt/z65VPpBlFuQH8oAapn5X1TQwrJ4fS0XyvueiR2q2QhCLiAhqZi2LZWYd/N4J +Wej4VVtQIa0vjzWXap5X3CSFpK007tf9gYGg+x8qcblCMq017t8tTMdaA18q8YJCMq2L9brbAK/9 +wNeHmgPrlGbzME2nVDLfQgzfhPFSiGQhEjUCJxVqJGL9Obn5g/mVz+9iuSLc9AOv2PkzCvCDlP/5 +wYn4YiYfN+17Y7lOTt7D+iS0AfyqbLxCunooFu4b2wAfKW6lQtKOKW2T4vYoJO3H2LBk9fsK84Ba +ptFa1DbQdplPpd+oztE7Y48TZnB/ozqno9dZ5mj+RnWO5pv3sHEDmFqdOXtxDwfkAQ55XvlmdY4h +PfayJtzp3Ky2uimkR9Ory06nUJi4jnhKieMUkmlu9bBJ7mkl2AwcKitGZXaF95RjnkKqmFddsvxD +scM16BYl7lZIpoXVZSbcqiibRAsRgzEj1wopGrI+dSJDlxcAPpNonNzLhmU0QkF8Xv2eiRZu3Mdo +tLGAsSo+QSEdWFVdenWuMPE7m9OU+IFCMq2uHtaIslnr58sL5KaHfP0G8fAfaepQct0Ixzjw/CB+ +zdqnK1BT1OVfmkPU/1Bdln8HxAdOcnNU3VaF9GFvdZnkmv2ok1zwR1aL3CeUq54aJ3eZ4Z5jc/9J +7ophJK9Hfq+cYUMCSFGZOfUiHeeEdEaSKfEnfh7Bj3+rz65iYQJL9jjyd3WOI4d+4hPaNAei5RsX +Z6tz8G9VRu84NxWqhY8U0rt/qofNTcOUYDOw04+vmJuYoeuUZZtC6rhUXb6Rn1nmNN/IX680m4cp +lOzl1e8p9MWsB5ppNVsrpKJZcfKx+s1UNNYJNFeazUNFAflY/UWjiKPZLcpUoJCKiuNkNEv+WUez +TkqzeVixK41mdmN09Wf26WvesypR0d0KdYyoaMBQZP6VKd+PV0c0Y/wkNgv7DtbGcbDI/JnfdEi3 +V28bBPnGz5xi5H5DarpDqvJ5XFj3hyp1KWRAKro/e3Z1pSQqZDRWxkk04xRlkyjMhTy3FElKa6eQ +Ytvj4uokcLAXxe2V0lMhOUqN4g6Kskm2YubL/UrzKaTY6vAq9VeCzUDhiipRxSDlGKKQKrbGyZYm +R1E2SYQB9z1tgNFKe1EhxdbGmc3WJA3uOlOBKcoyWyHVMDJUs1BxyxRSzYY4b04bYKu0VvmCgGfJ +eShTvs3XRLS/dOPv4zgn/MusZMWuV5Z2Cql6f3hsmirBZqBfFbFhc/dSjr4KqeKgqVRvRdkkCkcM +5Hn8NR4/b3eXKesfCi/LcRlo84BrTZ3eu85A5TMT5MU4TpC9fpHXj3KMYlyM45y/6BczTU7wMvP/ +ZsmfF8AsKXb7VcdjQ0V8KTtFTiY+9nKKffNXM+jN93I+/u1Xp47YyQNN/5jglRG75m8c5RjV9upk +T4UMyURvWCfqoASbgYEpj6on2AYoUt2zvDEj2wC3/+Y0KeVKNPWuopCq/2gmuTCb9k1PeV31z1ZI +rnle6YJTFGWTaDoAc4GvRI0u88rssvs3nV2u3c+asbGj1WwthVT8uVFcVVE2yVY8oQ2ADBOpNV7v +O22Ae/c7EfNxG+Bj/vFZG6CMf6xrA3gP2Kb6qj6fQpp61ZjqpyibpKZk/hultPkKKTbd1KfXAa3P +S2KES6KvlWufQnJv8MqSqESYOKUeVmJNzTkybTEq6x5UlQMPctgs74iD0v2Z+VlZAZSozMSGpt0i +8vVrL/P1tYNORBcO8vkxKp7ptuegE25G7UeVddY3srT7rYlgtd81gvfxD0ZwDP9gBD//nfn5WhyQ +olHf4/UuigOO/u6EOz0euFb1pShsC+DzKnB/EQfcpri+CklbWQVmx75XkYcU0qH94Xm9Twk2Axum +PK8lga5tYOrSTCFV/GFaNVFRNonCzMxd7A5alVHx3pNtgLaHnIj5uw0wgn942gKr+Wq7WFivanYq +pIVLxsIGRdkktSBidbWBrldIsRfipcvUU5RNChfrprQ+Cil2zljrriibpGIVScKP6QdQrGwJjUxk +NEMq2Dioyx4nMw+JyvRYkwhmGdpnxnNoP32o4r7Ih/FcfZnnQXpfIPxOQcJg1TZEoRySzo5nUt58 +mFnkqAGUaug/jI/jm8yA+4UE4NrGxoGGClnvefHeWQmAn6JyL8EmdlRvlYmvG791WO8l7Dqs9wOa +qqY2Csms9wMqH+GgQGc+UeJKhWTS+wGdyBQWXZkw7LgVqQctrzdua5DlbsAyidvjR/RU7DMpvssi +71rHc774jiWeIH8pwWn1h47G81XtYoV05/N4GY0z/6DPHGPWK3GHQjKti5cxZgqZKlo6O4NtnYcU +9bJR0whvpZW/Evc2/0FvAwPwgzh0kQ5x5pmuku8rpK098WEzzwwl2AxMysgeuks59iukip9NR/hW +UTaJwhHzuW+YL2NIyIcUdXx2UkQFJNxHpQI9/nQiNsc/yFeQE5JHmTgaz3l63J+sWPYAHBO2r1kc +NAKX4vks1HXUCU/NtkBspplPihK8HdoCNx6NGHyzcvyZBUP8/vSBuT6UqAt3tY1wRWL5SgJ7zOCj +NBkYgKkJzPxZLPL6xNsJHI3bH+MNyvzc3Pv8oZxcRT4tSLlW+U0C+9j6Y05UyQ8N8hXcK+9rPFeD +lUF0YXqQvwMpS5wvBHeBoum5uYG+/vRAcFB+CK/UyJtoYfBxHkpk5voCd6UHB5cTX6txdKKFq0/w +TYycPF9BJPV1oe484URCvxtM/R5TyOu7s6V+Y0/yKW9qpjkUmCe4FSdpLSz88xLo8DFhnaSsi4W1 +/imKHydOHh1/mcBHx8+c4j7PZ3SuE8ZNZET8JEXuEWT10/8xtEcM9SA6xCjvkSgXslyohQ9Y8PsK +++cXDFb+fafpRClV+/nTPCcTnnnAQuwZJ+KTs4wTZ8Re9zNMhfgiRc6qwRYuJDK+hLhMX276cJyp +waZdeYZvRPCMe2RN1sk6y889ZBt1k2tS8u6zVJfwb2sT3IQ2BkYzIXwFqeVp8U5NKnyO3G4+ea6u +rVBbITvR1AR5ZLNYmNhZv1Ti1wrJtCghrLOuU4LN0AD4T2c9qhx/K6SKJQkyax1TlE2iMHcUfFxf +tZmpR7xCiq1NgDvNAm5X3F0KSfsyAbKiSBxk+t2JBC9XFL+f1RWFp7nR10IhVw32iqKD4rorJI0r +Cs+YukCaKjyZEPd2XQDyW4YtzznNbxkOPOfEVWuIlx8yXKjRX6mw4ocMXzvHZpKgOloYV2IU0v2p +NcKCGqUEm4FxiRwBmypHe4VUMb2GBDVJUTaJwgwq18bZSgsppNh7RmyQomySLca2eLylcXi4Qor9 +WcPcyjquOHey4SHtvNIyFTdCIWmjapo2nKC4yQpJG13TBGiO4hYqJG18TdnLvqcom0Q3uZdl7VYr +bZtCik2oKUFZoyibRDEGhdm0X2l/KqTYqzXhHmkB3VuZWj2qkLR1CTHvWMDGc06zzuihtDsVkudN +/gIWFWRrt3xWIYmTa4qC2POqYJDSAgrJIwoY+z8UeU4hiW/XNAvCjpplvRWSVmya82ZF2SS7wlwY +/aW0KJ1vKHYmQRZGPc4zS8v3CBxDZIhLU9ZO7Uw8dHESHcCimpyRCs87EV88yKF3dz6TweoLKpPN +VZn2ocU1ZXN14rxuDTpc4N6kDfAY/+DW4M0LXJOw4uvU5FcK6eUyTZ6/FHdRIWmf1YwbwRekwhYr +wZAv3fifon7PvjHC/1gO79ggI2PpBc6iAWyUCh37iz8Gle7nxF+AzTU59TT424n4xBy54PhVzXbH +nThLDB7j83OaEuZgn4J7h/jv9g0L9cY2kev7D7+dRy6ff0geSmveOcnCfEEO5r2m1tglNs8TNT/X +ATPt/C5u+S5y66WF71goSPfjd1Fc5RIdyjOzwVmZDTpd+s9kdlY4A5fYrvFpyvuv8L4jSPBt62pA +it8MnBtryhzwLYmeVlWAIiWU1ozpWQWo/K8TMXdXAW7hHw9UAV77lwvxrm2B0nJW7+C2wEYhhNoC +KflGeWlNuGe2BfztTTsMU8gGJG1jW2Cq4ooVkrarJjxn2wKTVM/Bmt6q7YC/aUCEXupgFL6skEKX +VKhUhS4ZoXRY8Ma3AzaA+VKe7PJ+w71D/I8H785PRYlmysmbjGLN+NiIdRFerpU30cJfsBB7+cII +L9fiyqjIYSE+NsC0ycQrtZg3xxwWogOt8Uotdp76URbiJ5EhEMBrgroviq5JlcsCJnSv1jJ9Kflx +uyx9aUaUZQ4qtvMP9qUEpwXpSw87Ld0VD9JKDFHI6EyuFTbj5CjBZuBwETnjFCnHSwqp4vVaMrg+ +qyibROGINbdE1nQOhhYdTUinKbxyaOdIaN9yWoi9wrIScyS2p0nmKGWWvphTi4ucpy0L8YHHTcTf +k4hftEzE35PwJrssxJeQoRBzRSTDVR5wFJgAv19Lcux1Ujzc4acqYW4tL3f421yW2eE73BZkh5/n +ppLIhLotM9fXuzurXazVvfoWU32tNht9p3hV7LYQLWvHn2tx7bjTbZnN8M+16vPExM0Tk42q5U+F +bIcvasVxrWOOR2yGHcpgL2Zk6XFRkVVuNj5Qekd4Ilz6DwPbsjwRZD00SaPwc624ivVQHY+FGP62 +8138g+uhIo9llkRj1NSrCiuWRAs9DJdM3Y2V2EYh/dpnsquJomwSPeLUzRm/m9L6KKTYLiPWXVE2 +KVxsg9K+UUix343YRkXZpHCxM0pzaQNS7DcjdvY/JBWryAUepNztC3E57itAomoYolBTIVaO2HBc +MvKAx0JCZ2W4X+Flq/qzwlubP7ZZYUx+07z83LpIZS8pVGOyvXy+9nN/O3FnJSZexUn32NrcMD5D +peXJnON/zJcRypa9CDqZ7KmWYqBqjFXq9NrLr7PwKZUO8qVn4j0xcp5l+an796XcrLKF6MeD/AL8 +3NocFINE8G3x2hwU32fJH8jJzPVhndCbVqGX2CCFfBYex4La3R60sJWF7Nz8gY9jvSAc0RbiywrM +xFgs5m6LtvgN8aDBfSC4dwQXEBw2CSquKsdMLgsTtZINFLK1Z9T2co3RU5heSAButYk2BPBebVlK +vUgmTxsLmB80A8qnteXR7UoS3CTMVqkPFFJ6oWE6KUzcgu1R4u8KybSydtjYvVcJNgNzr7zLuhsk +AIeUo2Nn015Usaq2u18CcEpJTcNIrzSXup1X0t8KKbU/madr7Ho3q8CtCkktqS3zwS2Kskl0iD22 +fTvgNqWFFFJsdW1v33ZAo6sswFOtJnBcA7a5trddTWDKVZYJZHLIBHJvsgRyMwUkkF+ptlKF1Lrb +MF0sZ/peiT8rJNMPhik5RkaiPjWA8UqdrJBcG2vDPYGOKG6pQtI21Y75pCaQQQ2yrz6mxNMKybSi +tqypXhcmNupfSvR0qWiRr8Ib9e//MDCGkY1aSUUfVEg720yjxinqVoUkaaPWVFQdhSRVNOpDis1W +SOp206gPK8om0SEefbNRA9oupaYhN0k1GcwCFRqmkPq+qR33WA2uOivGFnldeEiuD4k6oFTpYcKi +A4sMVXulf95ZjfN4AEdlkHiVpewAnqvDYexPlsJPjEbW4bL9l1gL0aGC4f3TB/tSMUZwqdUt01pr +1OIGhfTwqGmtldU5FLC19nY17hxSSKYZdcK64D4l2AwMTkVrsbtEdzMqaimkipl1pLtUVZRNojC7 +yyPtgFlKm6uQYgvqeEPtgBPl7m1V4vcKybQ43L2vlGAz0EK5ex7aKdUWXGJ03xnHqrMrXlTRm7sb +/6n78zrSLd+Ls0zvq620egrJo70vyktFHOsaKzFZIZm09z1KJhkpY4dEdPBlJJgFQjcV66OQ4l+Y ++HVXlE1i7Rg/hv0hpWUrpNhnRuxhRdkkFavIyrBVKxI1If/sacKgiRkbCAC/1+F85Y23YLaTv9fh +GjQ3nrMVr5wekfKfdrlr5mP4Q1C9EyzEpw7hfq8Q4+pSakQC48XXOYarxYkK6fm5OmYVd0hxJxWS +NrKuXC/5OMEy10sOK83mYfXM9ZJKNWSwY3g+1OosUUhF0+tKVn6kKJtE+YhFfSC9YPAQP59eDMgA +epm49FZox4cf/ymuy86ZUcNCLLmB4rpczr5ew9JrBtrobxnDtVTF1QqvaJiftFb69t7GdLjJOWJy +Uw3LvIj8nhQv1rCQ8INKva9SfD1kft1ulS20qmkh4dR/yfmDsaAuR5HMmoya5+l2QKk6PK9uzMR2 +wJSaFtzT2wHrVHijQkZzT524Ze04Z/LW2Xol2AyPAFgcFccLmGRZS93KslMhdeytE1cmOtiPvlaC +zUAdk6Pi+tM7/itfsGXny7B6W14gF2Va2/23XR6rVRKczTUtVLHfH67L5Z/Zelzh3V4P36GOHWp6 +6pq68qzsYk3LfJo3uZZlPje1+FZjapVCVkSflWXWYpLzWVlH9ec2hWTSZ2VTyGTinaq21te9ivE2 +wb5B9bZQSNGIYDdTgs3AQP0n2O2V5SaF1BER7A5KsBmoIyzYMjl0VZ67FVLJhrphk0M3JdgMTOjy +0dd9pB3QRzleVkgVG+t6o9oDWxhPDsSpSntRIXn+biYD8SXyMDdylZavkDw6ELeqzZiTabgSRyok +kw7EmeVMzysx3JquuqaQSUflV5StWCF1bTLd+FVF2STWOmL80PwckAfcblLlToXhPXm7yU7a/G9m +Zw9JL8j0UUGaCja6wygKV/CdKKhex4L7uWTgWWV9XyE9/r6ufDm3Rx2uP6u2ByZpwv1YN6Zhe6CQ +hHDzAfNF8AF5SFGL8xWGW/5VLH9Qx4InuT1Qqkr31425vT1wIlKpqpQ3zUpUWc+7L6/On6K0Yd2K +3nri/09vLTS99ZTprf3ram8dV1d7a/RdxlSCQoZFe+vKuswc9taJSpypkEzaW0+QyXy0IVVtnagr +H0loVI/i5y2gyZ3GRpJCil+sa7r4JBV6qZ4MqQ/U0yH1QeV9RCFlInr5Q0qwGdhD/9PLM5UlWyF1 +RPRynxJsBuoI6+XhLT88x5crGadNc30fU6fwdp9Wj/Pdi/XMQnVmPa4OVtazs0ArOqOeZMFJE51q +wCrV+KVCejmznmwbGl3NxOTMEzvMtGOxhulqDdNPKvSLQgpHhKlMCTYDq/ifMB1Wlj8VUkdEmI4o +wWagjv8jTHz5sRVSNDivK9Qgue9vD/xPcUMV0tqH9SJiTSXJKFaGa+69PNILTKSv1mn+UymuvNpC +QtV7DHdvhdH8CEP3Al9mTghb6nGldbI8qKka1NJ6knuNrtGg9lXZ+xXSxYig9lOCzcCA/Ceo/1OW +dIXUERHUAUqwGajjikEN8leEEjUIoxVqRGMDGcDZenwp4IFrLMQGA3bxxWs0NudMbK6xwkJc4PP5 +Q76C4PCgfH2uWJUu6mdiF6584dVUfsJWbooNr1Xli65mxg++1kKVQAC/XM2Mh1ws6am6blXI+v92 +dRzfXA5/UBPpib7CVKIy4+6L8Ec2gxvF4qxrw6sTXpkBweHBvHw/ilXWc3+EDrk7PEnb/a+rwybr +FSqwTiGnrYrJmmvmg0o5rZBV+vdqWTP/riibROGIOS/CxeyMwvScEBLVtcMKNexSzZHXMLA7JM6B +AXj+GkbWlah71x4qcZtCevL8NTJk3JTIUfd4FMqJ94cxjb4mrMa3K8FmoNORNd6oHLsU0s7L10iN +NynKJlH4/6zxQLP872+aYuYDBobXd4rUNzeRiZSBt65h0smP+rxt/kx4UYUPK6wSDLTCe4YYHQy0 +xvvy91usfsW4HR52cUIyrDWK1YUpD0a6EgzgU9HzTaJm+KfiWN3r/q98q9DaCsWqbcpD/9XaCh+K +2ruvs4y3pviaKbbR4l4Web/oQzHaoP7/0+gAvceJYjUX9XCEWXdee2Cl0r5RyBZcfU14hIblhHQc +yOL3NxNVyVWPRCiLzc/NDABvXss8vL++BmfmtczSsfUtxJcOc4B7pk+u5Z7pV7oe1gyRRqQRWiFR +TaxTaOdD4/zBmH8tv0dcs4GFaLG8UAz/j2UGaIkYnt0gIkCRRgYU+PQDsJkoUwu+RyMqJao2i6o/ +/n+qUteLVNEahbbr9G2HKGzT8P/lmzndLlPp4f/7j1uBAdgnFc5qaJnXHRKHm/l/77VhHRgq51LI +PhjZgdsrpadCNv8v10oH7qAom0ThyA5cEc9kFCn3hQERrso2/7hU+OOGTIrAAJwWz8tYZLf3ZeK8 +5ESTRpZZ8qVpXU5eG7arW6YGPldITyPm3M+UYDNwvvzPnLtWWdYrpI6IOfdLJdgM1BE258qwuU15 +9iikklPhUd+uBJuBgauIOs/UbMoZ5aSKM9fK2Z2/EYdnbv3aaSSfUkimv66Vrd9bjSzI1q+90goV +kke3ft+Qh7u6e5V2n0Ly6NbP05jGyDRAiTkKyaRbv45kkjO4Ym0X3e7lkaDbvfEqN1Uh5UcnSh69 +pCibxHBE5FFeem42kGby5qBC7TD8XZ2MwfIoCH8n8hplcWM5PSvMxj+JHFx2NrbgGdMeKFPvLiXG +vN0eqNOE5x7lO0Ma0b6JdGOqt0I1FS0fvhVTRdfR0l1NbEtF19FSETWWK/T7CnmHEmmqZb9C1RYb +zBkBTBNFG5pY5o32adeNc1j4q4mFKvLrg9Ov43GX3HacISaaX2/mtzevk/ntMmt2DQaaGuQoVJvR +jbP8mHndS6MspF1vIVpszBIbr7JMl2aJR+tZzA5glli9wFIgA7PEarOmdrUNdQDL7LdvXcex/ZWm +4eOWRqEVitSVnlnGNXVJXgVdKNVe19RCrLiEheJTdBLPRdMLsolgwDsTwQWnr4CngwvFuccrkFzL +LhQftyRxLMkegBXC0+gGzpo5I7BW9DzAIj3eJh6vY9HPjNku3BdYDgbwk6ga0Yyyga4F2SgTxMfN +LMgnLZJHmCF1zXXwfNQeCGj53+u8XzPfmlm46of2vCSzvz1QMsKBHAA/VYpzdwDgrdYBaNzcQkyd +DkBuc4tdhdu5sxqpvxWyq2y7zhsbC8wil2you2SYMPZUSKbvr5Nt8w4yeXj9JvYJ4+C4+nL9pnYL +y1y/uZN/8PrN7Bbs3zMqAYU+o+9NhdRXXN+8KENufr09umVEjwmmD/V19Wf4gqH8giDSsoyCa3IM +tNu3orner89O0rklswcL6rNBn5dCICOIz+rzOe5SltNtnQGsqd+7qoV/iTV6Kux9XD+vqoXWyUxj +WwCLBbmKyBxqXV4/4SoLoVbcT2cEsaz+udctJLXWuxHGJjxPJwGpGqrP6ntfSQIGtLYQNz2Jn/2f +w//Delp2VmBICIlazTqDI6vLrMKm+uwIs1tbkOltk1S8ehvmUcgMINvqs/8ObsPbO0/wsok/A3sl +JDMNW3rG4Huwr363By0cIMaf39fgfhRc3bbc1/Mpe/KTpo131/ey4/Vpq09WZqiD7yhkg05q5eVh +7+sUNpcU9yr1rEJy/Vxf7hjuI5e7YQfA+ZipY1OFZPqtvrdXByC2nZqLVVotheTZlizmurdjknES +qa/EJgrJVGqYhpLJmEtS6t0KyXXAmJtrm+ugtC6PGdfIo+b2UpOY663EOxWSSc1Va89Ulum6r1J9 +Csl1uH7YIqmfEmyGBhH3lOvXAIYrxxsKqeJIfW+vGkC39hbkeabN82QYz2u86kkFHylyi0Iq+MMo +GGorsHk+CeMRBdxZfqvI/Qqp4Gh9mV6/U5RNovsR02t2VrYvhERN5Bm5JqJ2/5WEPi0J/UF7S6ai +sQ3Ykfe1502pJ82liL8lxTt0YEvTnxTVdpdC+jO6gfjTVVE2Sf2RixNpSstQSLGz9Q3Np7g8heG0 +fMUVKrRp3MWOUNwkhaSdC29fm2G0MtCh8uWYhzdxy7SPjW0gPWNSB8tc312oEssUUrXcvmXHTNRA +tldI4sgG0kkPUwE7REulhfNoJ736RslQmr9duTIVUtPfppOmkkuYspX4rEIyXTRMI8lUsVSQ9taF +QrFyH1Sorc6p/2Vp5SWULV/TcOwryPYBeSZJ7vMbqGImWYobcPS7eKPF36HLwNsNONIn3ySjnwxr +7zTgsDaMmPJh7V3BfXyThfjEp0xOvScO/HwTc4q3alqp0bYKWcM5DUztOyjuDoWkvWdaK74jFXAc +e0CJTyok0wcNZBzr3VHHMZ/S8hSSRweWEaKIzTZEiSMUkkkHlo/JZMaxp5T6tkJyfWTMldnmxint +FYXkUXPem+k3zU1TYrFCMqm5XmQy49h7Sl2kkFzzG4SNY+8rwWZocNk4tlE5DiqkigUNZBwbfrOO +YzbPljAeGYY4jkHz4VqFVPCpUfCxrcDmcYbxiAKOG0mK7KCQChaZceMGRdkkuh8xjpknyn362t+p +UP79CjVHZYf9RkOOYD/fbMGT2gEoesrMoW82jMntACTcwq5XnvQD0zON6vxgyHwCA/km7ZsrVNUV +IgP6Dg+GfHlIUYZBCi9n7Dks5Cvwp+d253s/Rcr2vsLL2e/MD4YiREqU9XeFl4vw+W3A+Hu9wsuZ +qLfvbb1T7+3Tq8eQghx/dtdQfl5ORvu2SFGZQQovl72/R98+KFLy+wovYwvmZAcK8rNQogy+kPFJ +GeUcel1DropuvcUy59CmuIhF/sCmKR5kkav/ddKGdTpZiM0LAOsa8lXGMZ1kdXm8IVeTZ1kIhgbj +WMN3p1gY05k3dzm4ZGTn98nKwknh2tGZS6l9FpDytEmDqY0g5SItz9ByiZaLtYxnDP/bjcJGV63n +3fn+3vko0kr+o1ArmzBfyzY+2o/PG9HlQV0iUi9cWWp3YIiJ2cMKVZ3EbnMjxm56FwsJTZR8j8Iq +jMHWRlwKw9QsRT3fFu759UmBpi3ka/lpKtio0NgrtxMA/mnE0X1bF7m2ziJH9/gUCwklKuUdaqSi +c1DUmJV6OIURHsaOpnZnNPa+1AGYkGLB+1oHYF+KhZiZHYCGXS3EfNAB6N/VAjxLOwBlKjOrsXdb +B2BcVwvebzsA35OjosuJ95m+YKggfzhS1PFRI4wr4RWY05gVSOxmIcGvbM8p5E/Wz2vMoQGITyzi +Cxa5+RnpuVgqMlndLCDhnuFG5xGF0TnYLtWcRrLnfAeguMgkxvzGcd4bua3j7DNSjcxTyKFtdPM4 +rqLNcvUVJcxQSIZtyXo9QyaC95QSrqLU5uCPR3UAVirPcYXUsjDcj1VKOKyQDBF+bFPCboVkiPTj +V6WEq4jwoxpQe5gJ0zUKqWVpYy93secYKFmbNVNia4VkWtHYCyeQ1F1vIzVXms3Dcd/cRnq8OzuL +KOquTLcppKIvjKJltqIeSrN5KhT9axRxAlquTOsUUtHaxrJw/VxRNony9mWx9Ur7WiHFvjRiGxRl +k1TsyplrL8s0bx97wgRR85dXnL+VVOzVg7eUmaE5+Fayr7gHlwu/eYDNKnxcIX3Z3dhbqRKws4eF +mNhKgLsnubk8rqsmrldI7j2Nw5YN9ZRgM9D/8uWxxKyzctyhkCr2mcp3UZRNonDEpM0jsgKfOVRT +3s9GRlQ61k+eIPBcEw41N/W05EQfo6X4DIt8FWtikzMeCz+wyE8K9s/nj1G+JjxX9dLDoJIm7Nv3 +sWi/QZ0axMkmHBDG9rKQsO5JY7r3UwZG+/nODmY1yZto4Yte7P4zlJT8tM2SY7/WjA+a3BpvoV9v +zkEVL57gE0Ef6i23yJ/lbh9LxbG0W/ncg5hAAMvEjQ63Mafjk5/j4INvBTebuPhionLwq0juvo0L +5ZHC9JswPXi7CAaIy8Ep4Rp/u4X45OeF67Rw/SVcnqtvBI4/bwapZ6+P6Xsj0PwOC55HbwSKRhn8 +C629uTcCaXdY8IZuBH66gwbcT90IfK0t9Y9CNviLTbxv3Ag0upNjPW8hlqqeiU1MFx2nsZ2ikELT +m4Ql2otKsBmYK5GJ1kpj3lkhVSxsIp2ztaJsEoUDgPzSVuxoU6MvmnjLAAy+04L8Isqn9FWO3B54 +xjTmwwqpeHUTuHmjZ5jiXlNIWqnWaVWRkduqkLQjTeRt0xJF2SQ6ZL9t2upZI9ZZIcWOaT0UZZMo +xnrwiylpWo8LTbz8zsyBO/X1rdp3WZDXt+7kH3wV8pm72L35pZ0Dqs/9nDFJU/80ka/ufCpMvFzo +UWK0QsPk5eXC/cJU0BqoqsRYhWS62EQ+9FDrbqZG+YJZn5oFUKI9OeF5Y1yHMbNNfOl6drs7KVou +WZhT4AsgUdk7j7qC2Nsi9szd2qXfuZ5d+tO7LSSMVLFmKsZPHM29nqPCfhqRSjysPD6FrMT866US +zfowZgxHjhLzFJJpwfUSjgFkqvBXf+8wgBS1OW50hMuy2VgqLr7Shx4HBmCFVOD7Pvok/nGVHKqQ +xlZcH2Yjx5/BEUafhBWrgX8V2iEdCmy+nuPjVan6wK9Ys2XL9WF9rETF1ilkekX2se+VckAh/Sm9 +XvrYD4qySRSOGMzl2nMm39vBCyYO3401UN3ksJuMP8TRlFQLLOOsFIMsZnNY5gez/r2eC8c5qZzl +RjvAb5UcvZ4L/KNkG5Q+1KefQT12/YpDTlx3D8fWxBcc+qmhKU3Z7P3u0ZBPacpUe+EeC/FF5MkL +oLgptX1FVBlRgQCmNa0ywQLuZS4jPnGMA/KM4GMRbkO0m18A9WvVhilkgI5fLx8wzSKT52RzIG2M +GXLOXm+mSP8YE4cnFFJoUdO4Ir5aK7HNV4LNILGNe4d0QHrye8qxWSFVFDeVnjyVdiW/v1LiNwrJ +NKep5PdWYeIax62N4lVIppfDnfEowWaIcKYi+8ubW1ctZSr2wzhTWW102a0VNWU/jetrIbpxeSsX +NY19yELPvryRiOebytVh+SpPsYbv+abyRGN8X7ZJ+SgRNBd0ytTMgxMjzEXnpgdDIf4QBbZKGqyi +UXYhfNWUWeXpZ0G/lLFNbA7sxzRje3MhcUSEdpCJmviGKKYmMZ/uu08fHBgWxKeO4+vb5g4UXxHB +sabMxjH3cQHA+/vHRVfpfdx4Cm+uzxfAWcHWvJ+Vik980X7zfUYSP9NxO9HxacSGsCWpZjcLT99P +eWJyc4IhbBO+/eSDebYfO95k29qmYV0dL5qwuBSyESO7enWlXKOQebC+qaRjnKJsEoXttW4HpfVU +SLHvkkTsRkXZJBWT9UCqOnmkqVkPRI03/lVTSDV/NpW506kom0Q19tw5XGmjFVLslHF6hKJsEsXo +NJe565W2UyHFRiaJtQ2KskkUozV+tvEnpR1WSLFRSaZPx7xkKlBXIWnjTRyqKcomUaXtSXOldVRI +sYlJYQ3XQgk2A4XLG05COUlD+UqS+X2S11TiTYVUOS3JHJtvV9wOhaS9mQTPvMpAqerZkuTdWBmo +3d9CTGllIMSnRBK24gmmjh8rpPTWcGffUoLNEOGsnINvUI7dCqlie5Kcg7/XX09ddyotnEfPwX+g +LybgB5TrlEJq+toE/KCibBLdiJiaCnyh9ME+pOlYMf1lUy97iGLHxwHp5Vc9wN6Lg0kcK1Ie4Ar6 +JVmzHxVMiJgyYjiZH0vizLKdIkGcTeIIlvSghejg8KBaPJfEIWHUg+zs5u7gBNNZD4eHcZT6NU4h +/a9ocy7+SlVs0g2y+NvyoC7+Gj+ki78H+QcXf7Me4kKGA/1jk0wtCxUyYO/dID11sKJsEg0yQyl2 +WGnnFFJsshE7oiibpGJSsdiJpmLv3iDdKltDHFRI1uTwB6l6bz09NxfFytPoFeOwNkt0gS+IBTcw +gDse4gtPE3VO/vQGhr3Zwwxq+bSg+vL9PqSonr8Uqj6z8NwgwgMetmRZsfEGLgReYSk7gK03cJra +/bAF943JwAyVX6WQkdh7g/fOZCD6kQjj8sKMmXXwqqnE/mkG2sYzfaH0nFz+xtZvUqV7HmGi5Rdi +/w2cXZ5/RGcXU0R8MWubg5M3MBG3kpkZd1K8v+VRC/HHyZAXwCmpQv6jFqIH4fQNTMK3WQjhjBS+ +I3PqJCZxJs6Jsar/o/eIn0RsXgATmzEK/f7H/ShRtDSpGYM873+WPPR7uRkj8zNLZimajVcEFT+A +D3oyW+HVZhPPO9HbFFtjSjPW6m1TbIPpUvx3AO3KIBb7skmX/TdAhpqHNGzZChnrwzeEjYsPK8Fm +kISK41ePuUTiqusD5ShRSBVHbghbsUgr9ZRPp90X9BUA000TtZxhoN1UPmEBljZj7j2Uxtx7mV8S +xiqp5Et8HiVJP19rsaxZmKcNVW1ThRGeehokAMdVbHUz7x0JwO9pFtx8p3qQSsxUyBro67d5igoo +JKni9dsFil2rkNQ1zaSr26QvlERv7K7+peK+VkixDUZsnaJsUrjYN0rbr5Biy43YLkXZJBWDi/fa +UjTOryh0zJA7bd3TLVR7uqAVHNsl4GEtlsfLFsXK/qdCu5l47r9XMndougXZu61Wlo0K6VlZM9m7 +zU1n7pUPF3zpv2CI/3E+/Mabpv1vnGmgbYDdAFObsx/sTeeDhvRhLDPPqw1k70WxFLqxkB3Ax83Z +SX4fyEnjFXmK2QrzBdcyg+ciE2oCxa+YvP+4ubz9PTBDHya/oz68r5Cen0j2zkoA5lLYvEdeptSq +6im55htNe4WrvH5DApnpIV8/flYv9S7+yEaKijSaFVlJnuoflDpWy7RMXxyurKMU0sqx5nIu+wR5 +KjuBEUqzedjQ5lz220wGWg7vj2tlTzSXw/tWPj28L/JZiPmpA7CQfxzpABzw2bNWmSo+opDGTzWX +VP5ZUTaJNiOmebkWy5RJ0UqmFkdWVlr0X6lt7SzLfIn2f8o7UiENPtdCptkBWTrNzuYf/ETObv7B +aTY6my3KpxrJr5oWHd1Cnmp0ztanGu9m61ON7/gHn2pUGcRK8jTlgtqC+kebY1rIRq0TmSraUarE +JE1T1qy3IqokY/VrLZijIwdZEI+K1KM3jEdLBqlH9XLUoz78gx49l8NaLO0AlKjM1BbSVItztKna +PGZBWijrMTZreX6x7zDSRerOY29HuGUivVr8mvqY3oRa3YL94ycWZX+E1S04wN40WKaub1pw6spl +IYRdUpg12EI8JvNmeG6miGxs8aXbwo7BDGQbC9iv5v9UyEB+18Lb3wJcuWSafSMQr741UUimTS28 +m24E7sy14N15I/AMuS+rXjA3vxBFKuZ79wpV/E2q+GmuVvE3qeJ+FgcBv0ktauXxS0R26Y48GTl+ +F9LTLAxMD2UMwv4WP71hYQERj6OoJe9IHMizkHBcrVd9x1iPzsGfIlvXz1OPVEYnBxcFdTdREpde +yn2XQlb5konLs+VMTyrxWYVkeralBG9ROdNoJb6okEzPGaaDwsT91XglTlVIplEtw+bEl5RgM7Dr +VszenBOnKcd5hVQxuqVMi+8qaq9CknRanKuojxSSVDEtXlCsW1uO1Bdayljy139IdChiLNEEN41T +pAo+mW0awZ4hOHhObMnOVyffQuzjwMSW9z9o4S4pBfmJGkxsyRXc5HxJ8kktmeR7WAjhZSnEBCQh +XpFC14CF+Pls0+wAJrdkfwkFmMdM9o/Ui0UKWZ3XTFO8J0z8csZyJa5VSKZXW3r55YwfhImz0E4l +nlRIpsktve/VBK573IJMTYMep12q/Fe5nFp7cs8xKqcLE1MgVolXKyTTe+EpUF0JNgMjHpkC1yhH +vkKqeL+lt0sC8MvjukRKUlqaQvJoLrRUVGuFJFXkQkCxTykkda7JhccVZZPo2WW5wDG4RBnbzolM +A5lWFkgeNC3gkPUaV4s8AlrUkuPVIwXsqinEDsJKaeiJBfy2CFZJYRcL2QGslQavFGTgOXB9o/YO +KaTPy1rKwHVz0EIcBy78Z9TKLEjP8SNFPRz93hU83Sqe+oMWYoX7Hji2yoDzVtA8XeM3CPGVuL6L +qEH4RvysGxKnd0khnYXH/fhWSpNDFuKLWMMcHBLMBmJKiMkO4LDU7K8Q5xCp2kB1MKSQVdtmqtZ8 +SHnV5C2GF5SlRCFZt7eUF9iDQ/QIYYbSZikkj77FMGcIw8n+M1eJ8xWS6QdzAfV7MslbDLGvm+lc +32KoOpQuy8eFUpVyxNjuMtQyEpMUrxIFlBB7/TT4/RXSnjo1u5xpoBJzFJJJndpNpv80b3ZB+kAU +K2/LuVdo3TPSutGFFmLN4IUzLT1TLXQux9wqZ/RnpJVGESvfoCz/9iTOtOS0vLWQ28HXHdxWnhNe +7zALVUI4LwVE+3FB/npwGDdIU+yUvyR58+kwpnwxsTmYlcxRr+pwC/HHiclGcTL7RZfhDC48zPWU +N0zcX02W/C4YXp4EkbOy+IoUrXmjD64QgY+SOSJvHW4hNtAaWvyXRRGuqOhHyaxo6xGS1R+Lkz4W +/JgnhY9YyA5gYTIH4p9YGoSlQop7Ivw9h8z82/z81Vb1ZsQnkV6FgFWtm0210PMJXvl+g+fGofSC +EL5pzV3yi09YiM7KKQiGeg3xZ+DH1nztYyeRA31Z+QU+7Gp9dqqFOk/qEYFhQHxgKjf12Niad39C +T1pIeFcdWKAwOtQam8XyiicZ6vj5RuQnETlGXHzsNAeiff5MHBBnOjzF6/KDs3G4dc/jTjzDYnDg +kCz80bpeFwuHn1InDDfPoPm4t2Saab3vW8Nd5ACafGgi0FEhs/rH1t6JDuDap9kfyfSoEtMVkqnM +MPnIJNewMN1oPtzaHBykavlAa3i+cwGTtHy8ddxBF1+sOOECPnqaX1h0ASdFD/mOK9+JcL5Gzyjf +4GcYnvLV7fVJvK3WbUhW0xYZ6f7+BTkh372+jPyCTBRp09aabyromHEff7w9diBw1LG4voWZz/C1 +Fd65wlEHZ/6vn7FQZWABjjl28EPf0QMLcVz+tIo48WfivLB1YKkw3R/CBSk/xXIO/pLC/CIL7h9v +BCap+ckKafy4A27XTcAbipulkLRTjrgGN7GVhMOmzIng8JLj1yK2SrObUE5cEMEEoa1U3GaFNHHe +EdZDrxC4foX5JnZBJGrQTikMC96YKAavxrMavFbAmChG7zYb09rGPPmsiefYqPJ4jpM/P3nWxPMV +EfyFJYnna1K+/jkeVOF1KTz8nMYzWT1pq5AVGhdlonWj4rooJO2lqPB42pReERwSzwnP2fG0ifdE +MJl4DlRcQCFNvBJ15XgWMguBBSbvRi0xMCyE8yWEq5/jbiM9OzVUAMyP2nPGieYjGdX8QpY53sxj +eVCm0Jc3sNDyeV65DA0mndcEdz9vSZpulbA+O8pCdGEIP0rkFkkpEwekdJCl4OCcAH6XcspoCfKp +KI5Io1nITA+l45iYKRvNg5EZDkTzdgxWRE074cSSF/geAZF0YGVU/HEnDr3Azhg/n9gcnBPNV4/h +sJSBc6I6dQxnl7Q3zc4Ql8TRkWTx+wrxrxSXkMdzz03A8TcdEtllUd7gTcAhITx5E5A60xBWRHln +3QTcN9aCd+FNwOqxFmJW3gQ0H8f98fabgBJlXRflPXcTEBxnwfvvTcBP4yzEVO4IpL1IVvbR5FlG +65Yo02f7fWqa6gGFbOStmmM5igsoJO1Hk0FzqFA6ffIio6CtQjJdioK7TkcgRXF9FZL2b5S3V0fg ++xcteO/uCFQdbyHuwY4yDGR2BO5X3kyFlClyesd2BLqMt+B9tSNQMJ5tcKUBsTB9sG9IQB7Ao1iz +0L3UeBiWjW842aFnj+eEN8uBKvz5yWlO02XZSNPl7900I05tVFXfKqRTM5xxdMp4FP1ShEfpBdlB +JKrd5xQ6Zjwso3EG0GnieSc6v2QhdijQKaXIwuNUUF6j7HzRUKySXyyzayAa4ktnydx6K7W8+5LO +ebf9zF/igdvVAqinAq0VUuw2b+MWQOMJlpkEGygpSSFZOgKemWy3YoeUb/Mu6wi8P8FCzJqOQM2J +FmK2dgQGTIyobXa+zz80OGCIP2cYSlTd4c8iXI724x56+8pEhrxY3O9PxPdUJT4fVsl/FNKhe8Xn +hyepzyeUdF4hWTpqlXPU4AiFpD0g4hNs8YCSChWShYlXHvaQLxgqv12O5aYGpz83UJuvglvuGAAr +DPnwKgOVLdqHiQ5WcPUkGXVedRy6wcIZKbTSUtOXLUQ/hsmOizdYeFQKrbT0Mkt5mOI4c8yJdSyM +wC4HF6oXXrbg+bYjUKaN9LXDe6gj0PwVCzGum4E0/hF7M/DqKxzl+XpxwUrj28sKWfEZUfKq8Xoy +VdQpkF4Q9GX6Bg7JHppeEARKjKBrrYFaufjEtxyIDuDPKK7CLrzCYwFicnBMxsNLr8qa0ZebiaPC +4p9sIXqwbzguSLHpaxaih6bnDvEpovHrvPK82hiZrDDaj39F3buvc+c3GP9GcUX83eu86/jyGsM8 +TSE/zzDKSXqVKUxOeNrVA1LfNok8yuntXQ/oNMVCTJ96wOf844F6wFFhLk8A37CQz595L39DLA+J +WukbvzSmHDP4Sr0jtgCY7Kla1cJ1b3DqYomLgWFv8CrE27zng2mCOfCGHKvM9HDC6TeVE8w7kvmz +hPzxVDoqK9TAOw55X3+mB3DVuxlIUZs5Ch0zvDfdDJRNlacRyXB8XomhD2u565MKCvOGhHzDmrYo +4FMwFKmoa53t/qMy/BQUAtclTbfgnaYvC89/xwHSGscMqgT0mka3ykPyX716oSZRtX6h0GG0RxcU +IonKh0/jforVzUNrPsAroVo5ne+uIrcqpGm9t/CQonwKSWohR2IPK8am6DGIPFh5UmlFCinVVp6r +eKazC/DB8bNKe0EhedqJ5ucUY1NszbxjMVVpcxVSqsP/EXTzIg5KlHXS+svD3pmRuXu6hfiUdx2I +LkAXjhCTiZhPRCF6M1Qb6LWJVbRqiVVIB3qZD0xitmm1OzxjnUDKbEe553xpuqMKpCmk4N3yzvRf +or28ffnDucP754QG8XEMipW99gbb+wGSNEGgz71XWWg+gws43mCShRX6MPODM8K3m0FfXnrG40Ny +CnytkKhq7tocqS49kwu8gcv/cmI5NebmZOUDA9l98SZ3xgX5WTm5PmDgPWeceJgo2ssqSM/zBYGB +tLr4TfP0N4Pb32ozLUQHEaSP3fh3QX5+CKHfp1sYy2IoGUO4uiydqbOlkODpdjNQNNsB1jLH+/DN +QKdZFmKybgZGzrIAT+BmoEQZQt6RNwObBE/B1DlGcIxDBKoVM9n4/GnBRlPdJQqp/XmH/ApuN2Hq +UhOovskwNVNIpkmOuIE1AcTl83/vUzWBsRQQcyVq7mVjbgUJktvTVMNshdQ0ziHJPV1RNkmTu6L5 +g768Al+uLz3oa4U0baW8rcYzhza+aa1ZDjbXsWILsfbHeTDLwSar/xaX6INzyttnluwLJ71lwbRD +sYMNcYnlID51sJFavc1zjWR86mCzjH/bQnzsew5Ea+rIInyZ0A6+bbdD0Xsm3sUOaYg6/L0aifcx +dfy0QgbgQxPvu4SJI0D3LaZWdysk0xITpR6KskkaJYl6mRpd5ZD0eP8dTY895Yqnq/QchVT8iVE8 +Q1E2SRW7J94MHFHaOYUU+yZyu8qudG9+fqhpi8eH+Ib4UKQNU63UVMVuIEl2HJYox7zLT1VI/zos +LZZNRJBkxn0nS6a7HZbGqzPbgtwmxHFhuIvlQAgnHNMTLcyZzVnrfW6LcFLoN85hc/ATDpPeN83x +UZR8wuHlOfoJhy/5xwNVgOT3/jOTVNQn02dqlKg1eWDnlWq0Oop5k/FeeY1WRy3/y4kV73G7FcSa +KLp4jKUg1kaxep+/z3NB4+8uwfSZ+386UZAfSg/57vRlhZCm9l/45kp+jHHSj+fmWogdBoxx0tLi +uXQC46TwOwvD8aIU6n5gIXogxkvh7g/Cx8brw0JgrN+bkz0ohGI1u2HXlcy/K+af/cBC7HDgXdG7 +iEYCmCOFgywMw3tSqPOhmH9fCnd9GG4+c0heoMCXHUSZ2tm+x7aXJsN8BhCbPsNC0Yc6TNbgNhjy +25XXVPxZveLPmhV/xlX8GV/x59UVf9ar+NNb8Wfdij+vq/jz2oo/61T8Wb/iz9oVfyZW/JlQ8Wct ++dP9zs1AzLemmvUVsrLVvV/cDCz80IJn/c1A6lwH0gHU9P58M3CAYYP78M1AH5UZpJCycV7rFqD2 +Ryo7SWVrieydHzHh3FfdAryiMh8qpKzX2+QW4BlbtlRla4vsp0Y2+Rbge5U5qZCy8d7bbwH227Kx +Hxif64hsrY/Fbt9bgHrfmfq2VkjZBG/eLcAdH6vPqSpbV2SfNrJDbwEeUZkhCilbwzvpFmCBLTtJ +ZeuJ7G9GduotwGyVKVFI2ZrehbcANeep3VKVvVpkb58nPq+4BTimMpV2G98pW8v77S3AU7Zs7Iem +vteI7Hwj+9MtQGuVuVUhZWt7z98C/GrLpqrstSJb4xOx6+gEFKjMGIWUreOt0wm47RP1eZLKJors +k0a2QSfgU5XZqJCydb2dOgGf2LKlKnudyP5iZHt1Ak6qTKXvK+pbz5vWCUiYr3ZjPzL1rS+yt84X +n3M6Aa1V5laFtHu1d2Qn4AlbNlVlG4jsPCM7nvVVmRcVUvYa75xOwM+27CSVbSiy8QvE7iedgBUq +s0MhZa/1buoE9F6gPpeqbCORHWFkd3QC/lGZuB8q6pvo/aMT8LEtG/uxqW9jkS0zsmc6AbeozH0K +afc6b7XOgPdTtZuqskki2+tT8blWZ+AZlXlDIWXre9t1BobbspNUtpnIfmRkO3cG1qnMzwop28Db +vzPwky1bqrLNRTZuodhN6wxU06H1BoWUbegd3hnouVB9jp1n6ttCZIcZ2ec6A4+ozDCFlG3knd4Z ++NCWTVXZliL7o8hWrOquTwrmZGeEhoWatggU+ALpBb7UdH9OBspU4ba9phEcFeN+K4771RdxJZeT +DbTiWUMPFvlCeyuusQsXccbLQDduZz9gIRhAdxZOLNLpopsZc9mfL6qpSmqKdegu/bnhYq4g2K6T +5jkE303asv9ijUup1s205Tiym8ctsZ8Y/tvgebczkGoXvWs7A18stuDd0hk4zj9+6Aw0WGIh5lBn +4LElFjx0apJK3CuOvEk0B//ST0xDmIGldAkb0XOqMxA73xi8z1u9C+BcasFbuwvQfql6mjrfCLaV +VshZKoL1uwCTVPB+b0oXYAYFb+0CbLcFS1WwjQhGLaOg1nGBMXknRFGqXRRF7ZZZEEWDlqkHkxYY +D4yi6UaRs2CIvOTZgq/MsznLtBGm/Wi3+0CZ7wMFPrIAAa7jty3j4mmBHFE8yfZ3fMYdzT1dgLv3 +GcHBCin+vMOb0QXI/8xCzOAuwD5ye4Z2AQKfOoRjtMP7Wheg0XI2+NwuQKkSxju8a7sAry63ELOl +C+D4nBz80EHsQiP6SvhKODO/b052qvk8OYq1Dit+Mi45tC7M0/VRTNS2n1uIZfYD66OY1iM+50Pg +hTyQH4zNcoq0+nM+j/EVBjKwOYo5fJouuHd1AUrUwBmFrOumKO+JLkCTFZb5xtk+pf2q8B4Au6Pk +jaC8FWxM999MGHUxViE1bY7y1kwBFq6wIH2geKFp7yTpA38RzZQsW2ja1fSBrl9QJ9yJKShXVitS +6S0pwJgvLEjnq640m4edr5nk/HbRJKFOXGRCvTXKdKe08rL0pxortT89yj/Ynyat1P60dqX2p+JF +xnnTn84SLc4vMs6b/tRpFZ2X/pS42PCb/vT8Ku1PW1ZpNqctNoKmP10yguxPxSpo+lPPEu0G40tU +sEwFTTdYVUKLYeNhMCe7ly+UMag3SjQyXyrU7KmWAfwruRIpFSpIzwtk56NM2Xf9HJl07FwodrKz +uFdb/GHtrHwiNs2wcBMRGaFhLPOhedFqC1Uy8K6TKYnobMx2Mls3rObRhS/Ul18gwhone2LMGp5t +BHOy/em5Br/a2dNj4f41lsms18qMH+8oZGbNdcYxs0xaJS4xkU6K4zRp1rhpS0x0k+K4xuU/T88U +oHiJyYPtzpi8FODFNQxe+RCSnvnYkGCob5grZRqDF34zPmgITSy+l1isXGMhllegv3fyQOsEi6yO +qQu+d5a8aaHRWlY8gB+c7H+5UgoFA/hZym+ZMn5y3jLTwsG1fA9uidxNwElB1f2Sg9OwFGDcL8aR +pF8NZDB+dXpnpAB9vrQQMycFeP1LC1d9ksJKu1ekAEXKO0ohZU45vWUpwJZyxaOV2FhrSqbTRvEl +W3HPdeWKK4IWzMnmGQl/FK9YZTf9YZyLiNZUi5kzYl155ky1mDk7iTAD2FSLqeNer1c4p1pMmZvW +8+3R7AGYJsVf10tiTbdMYmXlpmcH8anFs8V7NliIzvUN9eVisouI1Rt0qk6SqVp2dubP+OSlvCWD +Iheb4zQFAxl4QUqdN8qecrzLP9PC9I18hkPmHLwk5F95ywKewynA8aUmmZZYcZW6SrQ5EOKAqXx1 +hQzk51ZYuiYvuzxdA8suS1damL/MWCixLQj2uGI321h39a7AJLX3pkLa3WLFNadnnvZdgeTPjN0W +cf2Ig+ld7yr7YoUU2xrubkDFwnvX/M+u6O7xz4y7u23HPI/S7nIb6x3eFbhhk4WYZ7sCz29iUvM+ +2MiDJmQTFNKHQ5bcf11KJg/jGlA1Z8Kdm7/c1CncuePLr+hc8ufGjTPlzolWxb7gCmuhks8v1xq7 +4r9apbVjDhvXayqk6zPVQzfXfdWUYDNwelJv3Zw/Yv9D58a8YtCSWdAWvVZZ1QZnQTMF1lKCzUAb +zeK4n+S1oD9aAmkrHCI22eV1JgONNluIaZQMPLDZwlUt+T6Yqc7TqmmiQpqaEh6a+SsuD83xy0PD +ydvWMSVSV4Xbk5RgM0S4LZ9X/Vo5vlNId95wwb0wCeh0xEQ+VSFp011xG5N4pvy1/P8D//f+mgS8 +uJm5drI5MFzZX1BIsdkuuFe0BH5S3CGFpC1yRY51AbO50BFu/3HjhI500dn4ysVRa+VmC54Xmfxf +mMBvc3lndQWsLRZiFnYFbt9i4aqV7IdS0Y9Uz6cK+wB4M1rWWE9t4Rzl/qor8PCfxtp8hfTve5d3 +f1dgLbncJ7oC7qOGqb1CMu1ywe3uBvRV3CiFpP3hgjuxGzBacUsUknbUBffsFsBSxR1USNphl3Gq +5TFjb5xC0n41TlXZyqjTqe+U6NH6kemn8Mhm5vh6FeTnmbkXOGF0xp00UMNrpt2LLk4k91B3+cxd +kJ4T9A1MzzTLCCSqXKdTV5Cf7qb881u5cAamuzmT7NtqIdqext5yc1bo8JVlhttJ2oRfuL2VugJP +f2Xp9qVUCdvcgIv5k6LmBip0zKjw0M91ffkSp0g5ziiMqOBLHjq4m3bKKxjMyb47P9THr6uK06Ze +QxRGiL8m4tHbwk9DTVzMN+V65RcMRpFKnlEYoWGWaOgcoaEiuDhjjC9SGCH6gTwAfnybCe4HHgb3 +3W0W3AOaAzepxCMKmQXzPd7hzYEjNBZR3azCzOx8lCjrs+eM0Qhj68XPa7aXLynWe7ikyCRCFqPr +PVxRrNzOR/SFmb382Czu1SolQpYOWzxcKRSUyqriiLjL5eo5D3vxslILbs7Dt5411u9USMePe3TW +j/B6oDyPLlZv2543crbX/BQ/v6uBuZW49D1C/SuaAwuV/5BC6v+wUty+5hzAheO0UpJUIznm2RwV +HgzxiwNcqqco5w6Ftg+kYWklptg1X3NV6gthWaXnPRbu+dqCp0M3IHalGbQ+q+Tt3w0Y9TW7Mf0c +rbqWK6QXyy/3Io8bbdOVgyhT3qoXImJR4XMYt8nuROX0/xUhEc1V4PZK0jDieAillbhg/oN+c3Gc +qn5/W0nWwYk7rrBcDbN2F1/HK1Ijk/6ONObPI/U3CUzfHRbii1fKYcH+Snwg+8IODgJp3YDjavNA +pZjnugHLhdCyBZC8ygTxYCVvnxbAnyRIEJuqpUcVMoiHLw/iEH+Yq0EUK3fbfyL8TPhD8UmKjw6G +cFLCkriTPnLXE1BX/q4ku55+JFTE34wNQdmSpaiWtxWWJw2wqvILMy28sJMpE0JJZUZ++U5uwlOA +fOUPKWSdVleWLcbRy6wFecm5RDlvuWTXJ0OOaYLAbczN677hTZtVcsXkXiL6fcOrhijhZ2jYNA8S ++QKRFVXJSx/sC+bmZPgy8gPDkaKqZ/5rm8gWE74Q0PCVw04s/4ZP+PNzfX6gIZv1TyKyCvLzKlCJ +u/ijAAX5eUBDDiZ9d/Fhbn6eLw+NOE18zDLNlWOafyscaEf2F761EJ9S4kCV/KG+giy+xtaEXR/m +ZlqZuvibQjp4k9zOPvOtBe8UB3DLd+x9k7oB92k9chSSOelKtUexcgxHD44gcITVvC9rPvI7C7FS +yb6s9xIWM9IDMMV/vuNL+b489GMNW+7mHkji7svDw0QN3G375FILtRTSp0fhntcNaK6oPgpJGhDm +bnZBfqE0FoqUo6fncnffdtDf13Zb4E/XAG87XhpnwfE9Jxg6/LY8fy/Ya464iFosmGXEZOYPGZjr +I3KJIM/v4+0quyE2y0Phzj/KttHPym1xsHZfEOP3FeaGI2v+ZCE6Iz1QwfYhMQGMjGI7Z5ZZ8NSv +CpSWOKSm3zi8N1UFdpRZcFevDNRwmLolKmRKxnqbVAY6/8xgkucBpWUoJE8c3CUe4ENFfaaQpFre +nR7gA4ob2ymrje0fHKLuH+W1ooxpysQab260UTYEbEujFTVZIUm1APfWbsB2xX2rkE26z2GMF6vx +g8Z4DacxmqiQatT4A4rKUEiSVnOOohYpJKmW1g5rTO2OmsjW+EUje71lLCUrpIyJ7G2/2JEdqLRc +heRRk3MVtVghSSayT1Lc9NPOLmOjq0JWfEyUdNRPftGOelbYK1ZsHFBz+DZKmqb1CoWOGTkch+LT +1siM8pGk5j2/cngrNqj5gnqeKPef3YDBKvm8Qsp/7DC0qYpboJC0BeEnyuJIID8/Nz03Nz8DJcr3 +YGVTJfUmNr8gk59iWONof8iJpb+yU/AzR2sdjetZiP2Nwz6+dEysY+FpFoZhj+O1ehaO/MbTKuP2 +N9J9rtnPexRrHYgehl3Cc89+2cLU6A4MrGSs+hTS270Ob8vuwPPkco+vDCxT4gqFZNrl8M6uDCwV +JmpapcQ1Csn0rdF0iUz/aQkGIKvA50Oa1vt0FeOJXf9hwCHxttUBjjUmGockGpkHpPaHpfYrD+ip +kinCU+wAAmvNr4ocdng/dwC1Dlpwb3MAI9VYscKbAUxzeX9zAHccZFA8rEqpSp807j8tlMhEykjP +GOQr8GXl5OYC0cbz+KoG2jXIACZF9U+xsOBgRQ0mRbE9zx2023NKFFvtnt9ZI74W+4Y8Efjod15K ++lLabLpwlP3OvsMXSu9Uc/0VMtBTo7xwAt5DlrmVZPPcpzy8/GO+QvDQIVZTWsynxEEKqWhGlLT9 +W+SSq0tzlbhIIZmKo9y89/eBomwSjQQiDrHZ5TRScs8LiRqiW6+6LFRzJFTfHKoI1RwJVZ3DFjP3 +A4lCiAWJ04cSp88PW4hPY5yGY55wRB3hKotxKv7S5MBHJja3HdHYjFcfJiuk2yY2kyhrYvO2Emcr +ZLXnmdh8Ry6JzW9KPKaQTAtNbPYryibRyP8RmwEZub70AqRpSG6NuSw0n0lorvmDPZt1NZ3hCwnP +PcSmrpNMWSUR2PmHhQS3aqmlMHo4Vgs15U+2v/F/upqco5D+bzD+z1CUTVL/JQG/U9o+hRRbaYL8 +wp8a5N1Ks3kob4J8VhyQBGyiziUppKISE2TfUboZ2efMeJmmzK2qR8bJD3wdxTXoG0e53huUE8zG +Lnkm9yMRQ/GHLAw6HLMQX8p4DcUvgnnqGEf62PV8RCejzFEJ7PxjXMK0xjFRes1xycLTEsPM41wL +UyADl6Rt9h+nu/GJG/iUAGOdHJabnWAfz89u7Q+kZ/vwopMru/QTTM8buwPzN5j0fNHpvas7MPeE +hRj+2uVecsgaY4vW9HuFDM/PUbLIaHjSgnePB+h/0h4W2lcz8eipkNwTnGGfGuigBJtBmsT+UJDn +SH3guLo00emNbgCME928v/i8ir6skLpfdcogMEpRNola+RkdzhtrlLZeIcWKnXGcN/TXGbrGGqd7 +KyTHXyaVvjipqdRNaTYPLZhUsk4x6h56mLzRRPMfk779VcankDL0iqP7i4qboJA2i5wy7N1BhcL0 +pRI3KCTT84bpaTL9JzVlJkvRjOwVbyplzwN8gQuLnYF/nVhwig/XAlgij98OsTQUS+URXffTzDcs +kydQb57m218by1Nyu7P9ISdKhaU1SoXnhjOcIzY5EJ2BPU7OMx+cYTzii4gL4jfJQtdZnR4PiJB5 +6mJIiI/d7EB0RYqelhRNO0stZhrdbKL6m1Nmz/lawdUKzewJSD6XKu9pk8/vn9V8Pkp17u3XAbXi +TFyaKWRM1zm9v14HtD9nwXvkOmDSOTufH1Aun0JynwnP5weVYDOwjcvzWdJ2rHK8rpAqzpm0Haco +m0RhJgjT9gOlLVRIsQNOWe6sLXevr9dU5kGFZPrOKbPwNec1c/spzeahEZO5vvOMsTiZo0whhVT0 +vXHyMUXZJMoHYNYosVtM0/zkDMtF88A0kJ/jD/kKgkjTRMyrYXwtT8iMdH8A2CrP9z46zxfih7J4 +X4aFn1hMz3wsR54mb7XenWkh7gL33OSY0NhCzwvcjeb4A/jK4vbsdSmnDwtgm5Q3sZzpyw2lY7sg +/rnAKxBbZHH9nWBa/sWPQWx18K7qPoupvYiYEmIew0/C0/BvC9GBAH62XvrDif5Swq9Cmvm35vTP +UgPEF33lIO8QHLf4XufffzO6gHtSFLBJg/CVQsZ3nwUPX7Qu+coE8WdLtg7d/mHysWnaJZiI3aiQ +Qr9Zl0VaXgpAkYZ3Xm0jZIfZUHHRGj/QwpB/eJO4IJtlblC3/cOfaNM4X5Iw33LRQnT+wMeCmOAa +mmHhA5Zz8zPSc4kprWmhzSUK8QW/CVJ+6hLjuI3DxMDHsMi18xEL6wS3nbisLCyWJ68XKDcwPejD +Enlo2uxfxhbLpTBACqECeSN1taC+Iio7QzBrXHy572a4+JLDWhdntmfhMuOI4Ub8cZrLwSYR/gF8 +3Rzugd2BQE0TkikKGcc5Li8/kH6Vw2W+pPuEw4UYfppuHv94szvws4ODCuXTSk0DbTEy8VEq81yU +yizmH5T5PYoyW2oBx1VmtSvm71pAXacLcLtqA6dqGWc82k50ZrvL27A2cDeZKsZ0048yfVnsRSXK +/k8dI243Ly/9HJSnS5OdLlT0mYMu9pkNThfik792IDoTf7jmwsJftCFRKVONZxXSkSOmhs0tF6Q+ +aRbdZgyqqN2GCsn9p+F+1eZeX87dSLm6KCT3UcN9weZu5rJ1pyjXQwrJfcxwD3CpJ6+Ucz+sXEMV +kvu44V5nc58v5y5UrtcVkvuE4b7Brbr/57Y9maJc6xSS+6Thftnm/pLcFa0UHO7PMC0VHJKZnx0E +6vaQ47X8egaGt9Sbbh5Un3O7EDskyK9NvilPlTp5wpvuTTeb7nmPy7yWkIF33H1jLGwhIpgt+47l +Ila9kgvx89m6wWy86+al/AcquTij7mDHy83slh/CCmFdV8kFnl4R84VgzgtnKjmD2fhSpG+ozEDw +BZ5BWok8hQzEJ27vdU7gQzJ5+I2T4h2mXyx2e/nu54+VXfBu7A40rOIyN84HV3FBelCZco5yS6+b +SQb2l/1V2Etl7ZS40yhb7/YWOYFa0S7z8e2mGsYWCmX68fLj23dEU7iiJXjeKps9FCnvwWsvb4JS +aYKno12I9fsKJZoolYh8Ge1CtDTLbilfU5Uxo1YcEkQWEfZo+Zu71iwL86q64OEVpDR1/2+3lw/P +fq3qQgzPVJpe5UIMZ9Dn+MfVPYCNVzHIjN9UFgHsUcggnzSx/JvcvPjeNcZluutHyhTOPMotUQzF +MBDuJj2A48p1xzWm5lQ5xuPt1AN4z9Z0QnkahPHsdommP6lJ+ntDJYYr+s0wJVYTc/f1AB5UrrEK +aW6cx5vXA+hbTR1/SGnhPJeMpqmiqaIF2SDcrQOJxv1PbzBQ+1C02VAtludwW6txMgjgb3l6Fx/r +kp8ekQZ9pxKXBO8TRZU80XhXUMeJyssv8PGTKbs9C9wWBlbXaWS0PP1CfPFOPkxQnhXCM6U6e1UZ +CYEMTmgbxebW6i5EBzL86Xk+bPTwddQmcS6U618nsmvjXEB88jdUmj4MH4gf58nn9/kyfZn4UDA3 +eO3k+0jK//MyS15IAFppLNoqZIyXeuTbky8L03PJwJ9KjL/OBIxMxz3eqcnAl8LE9cTxBoZ4SiGZ +5lYydw3aNza0FIWkja4EkStmZgF4RyFpmz3mwoU7ycglKCTtG4+XFy7OeV3w8s5FUrwL3h+SgEfj +WSne0xihDTtKIcW+MGKTyE2xtfyDYmfjXYjjhY2IY51BBTn+wSZdSlTLo82NM5ouMjlekGeDTRPC +0mNaZabHcwkuxBexXYK4KG2/OMEFJKxqZpT8rNAMCe+JzO/kcDtqAAfU5B8KWYGLleRuRt0arOXy +msA1qqKrQjKNruz9riYQEia2ylgljldIpsmVw3pFVoHP15fdom8g3R9EmlbyWAvjp1Y2voRVMWcH +Sypzl/9VDRfiE3dxc8Uz22WVeWabVJO1JjKIzypzHTWKmBJi/L5hISwX5MmaLkYb8bHfyop5o2i8 +qZYL8anEBLFJ+CYTU0yMCG8WpLO2CMvG4in1doxC1m5tZdm0P60omyQju27JFyttpUKKLa0sG5vb +a7vM8eISpdk8lDcbm9nigBwTHVCmswqpaEPlsAMJm+G0MogW+0BCqtBUA91eIVV8baqQpCibROFA +eJZm67XWu9IDQEvTYDvaG6gNZ6/Qv63MFTrquBArp5XAt5WTZ1l4kBj5IgExfEw4nph8LtEd31bm +Gn1VHReiQ+kF2b5QIAPfSaqeIi5gVs8/VuZ9hoy6HDA5GB4Tjp11deQ77+FewCymr/B3fBnbV3pa +TibOi67ketQVGpyXHsCFyv1mWXipnmq7UFm0IT7tO55DMXHJNa0KuS7aXNOqKFfybu6a8KmU865m +dhLjx6Iq7KXFV7sAeK5rCZTsNiuEC5VjerYEdl7tMvunlO8dEvTzHlMu0vJFj+ynWlxToeF7o2Fa +FdEw9BrVEPghUsN8LauGr0WDe1QPYEWyabtvFDIXRlXxvtkDqH2tC94PegBjr3WZTd9WZbKZmRnL +KnnfjwIOXFvu1PEfjFOfGqdqJ6pTJXsincJeU1anQonUIE7VaW2caqWQTs02Tq1IVKeaXadONVUm +mznMqbHXiUruSRu0MSqbKKTKxVUkniuEq/xQlA+zWtyEMk3q5h2MoCZ3BV9Gvj8jPRQM8Qcsg0hR +tk032eyP8elXLOdl1No6y8Kx67gmBmplNbXQrr6LPxk0DLU5sDVswKUx6vDvSfw7I3+IP4S6LNdv +yMSEfB1tEP8eiNb8VMWOhi7Ep+yVwaweGXs3dvFEvx6n7RGNOUeXkjoMbYj5mBj3wh7AWvV0r0L6 +2dJb2gNIaOJCzO4ewG1NGLfyiITXtDXKtIYjOl6hph1Z0yebsKbJQEda/kRKrbR0iaqvqLktilTj +qJuvoLkbNd91vWruRs1FUmoFU9oupdZauqYpOdto6f6m4bfw5HH8wOEhXyjfNB+K1eLYW65gOZWW +J1BfIFQApHLjvppFP5DKyNdNciE2GCqAI7XncScGJnEVhcc4Hrye5ELC1ar1LoXRAaSTuDcpIswF +6YXGnRB/Mkh523Kly0wKz6en6VG1G1yIzQWepgvdbgivYLkmpKj4OoW2Ghk1MVo+xjKGioJwjJYv +uXzO0kCWmGXHbpC6vCBH+g2ahRtJD+XnoEz1LugcGbgg8KOou7+ZC1X8vmz8GsWRHtFD/DgUNfSM +E7c3524EI+V09J3mLsSn7JNsPixeVWvhQkIjVTtYYXQGjkTxYOu+Flxd+lvhtOia3oKRrMirrBx/ +pn9Ibi5KVHBYl8v8e83JhtxGRflZWbwf9pG44mjJKocK8LEcILdjMZie5bvT58cC4XinpQsJZ/+j +OTqERU42f/VkF+InmaosEf4eyZHe8ZKh/GSvzCUoUt82KXTMGMwGl7uEMxS3SiEp3eQqYaEodfG1 +/TIl1kyxK1ntaX5R5snK/H6UvMr/QbILMXdXAfbxjweqAO1bhTdmXn7mkFxfkGMfElVNajdbHYdU +R7Ty4JQzsZaFnFYuJNyTYlimdzUwOi8Tp53DYi3Mp/r4UhOGMRZz9Fwr7iUyMcYiQ1JrLio38jxR +7TRXSGuTrDj+ep756bxHyVrRuMYPnqIN9RXkZA1vhTSVdN1m3HAYj2PJEwCWicF3W7sQzc/QLbdu +8lr4jkV/Vih9INaJe1XauBAf+6Ok4HrBpBKTRkxWK2ySU9rX27gQndUam6X0SFspyX5pi8XGP9HW +hfgSijyGb0TJ3HbS+PH4iRsmCfOg9OAg/GoNrmlhr1DLx0PmbRY/ypmoNVl/e2SNAhnANy6uI2Lb +uxBtKrjLxYB2J2IY9gh1LAtZWQPxk2tUsYUVLOZkDkOZ3Co+xiLtcOt3QQT8HVxw/9kDaKeW0xSy +NXa5vJV6Am91cCGmek/gmw5sO06ro5RrvEJy/+SSG12eG1nx8qoFMsznocq0Rv/0iaxZFjDNzYPw +jje6EEvHME2uMN96kwux5WtBTJOTi8VE6qpympurypiOZhDOyQiRiYPNoI7lIfrczRCtISJQ4Bsa +yMBK0dP9ZnZ2rHBzsBsmhQyY86QenVzySSuscnOxOaETRyg24jDMEdndxBQTk4P33Mzw6M6scvxx +4vIHY7V4cU9njgdl8ih5mwg+39mFaB+2uz1vWVjKQkYOSqWyh0VDwuS7TXA6aZCo7bBou5q/k+i+ +tieQeIfhaaGQoZ/j9nbsCaQKE/dityvxLoVkWuSGh8PG8TKz/ip1yxAxt4sOEXv5B4eIdilsZtoq +vdPY2qOQarYZW4PIVNHMzCo/zw+QakRmK7T7ZBZwUdp5dYqury+5ORDDc3tPIOVn49MlN9z8bt0d +Kp2nkIYvub3LOgKxXV2Qj9j14B/8iN2wrgy/035Vlq4EBmcH0kODUKLy/e8xXqk31bKAZz3MOkTn +YKR8xOvDrpyXWIXn5AjkR2qNUEoir/HfnZ7nY6KmqVLXvRHKWdU3RHmwGw/nlBtveJhPa7pp9ad6 +TPUH9QSKtPpTPd5JPYEz3dgAjMPbauJLhYzDVI/E4fruGodH+AfjMKn7ZXGQby8lqn+uvpf5OUP8 +XNvdhViy+vOBGeJmSo/yPvSmh30o2IMj3M8OfkI4V6r/ttxPn9PD9jWkZl5VSF/fMb5+30N9Tezp +gvjar+dlvubm+PmhKXWyfb/LnJ0rzr7QM2JcmCsnWMeJ5CqKo8BcD0eB23uV1+ADqcG7RGgt50kl +L/VyAa5ZzD+1NkWhY4Z3YU+gZ28XvJ/3BJ7q7ULc5p68R7iL/0cmRjBgHtAVq/R+HilyLtf5KAtY +Ib7P7x3h+wrxPerWip3yCg/HtPa3urik/0KczLk1fMJmct+VPqxvag8fHwnifhOlowrDLK4Tiwuo +ykwX6yUKtW7jbJw+DJtF+x0sBrBBPoP3jBQy9FTw+O06EG4SzoZ3uJDwmNrZoJAD1FYJ9wN3RDRo +QDbsZuhHf+PkZIVhTu4RJ1+8Q8IykN8J2yNZddJg9CAAeyRQje6sCNQeCdQDd7r48bWfxMEX73TB +/XNPoK3a6amQifiTx4tewErycAK7WWndFZJnt9lnn7iTNalo4bCqtEax8rd4wNQprC5/Sl0a3lVR +lz+lLkGDsevyp9Rlzl3q7Leq8KBCOnLCOPs9eejsPqXtV0gedbbq3ZHOBkP8oVh1bv2DkU4GgJGV +OON1udsFWVOMrMTHwW1SXYgdmp5Lct7fToxnWb4UTgw7045UrruHmjQ3B8yuezheYpycON7BQiDD +0F8W1NJ7XPBc1QuI/cWM7eMreRuweK8L3ht6Ab6+Lnjb9wLe6OtCTEovYAv/uK0XcIl/3N8LaNXP +hZgBvYDMfnwgSXVFqm6yUTeln6pLvU/VjbxP1S3hH1T3D/+gupb3q7ox9zNsTnvKyMxPz8zkl8oK +UKYxG/SwHbsge7L8Bse0qBbdLHx+P1sYmBY1opqFo/e74OGP7ZT+4gBZ342SH9hp1z/Sgi/X6C9S +vdX/F6k/BHwqCgf153r0Vx45YZHclppODH7jOjKAZVF8D/t0f51FTBHxk0gOBQL4Qny8+wFajz9O +bF4AW0To/QdUyBQRH9jvgAhtF6GoBykET79KQOl+U5stUfIpyUdIcs+oBFR+xPh9o0JWeXeUfI9w +4oMuxHxWCThEblGTeMCoWWbU1HuIkwXVjFXxdQqpZr1R0+chVTNXuPk7/60eNUbbKyT39ij5nf+9 +ZPJQZUBt7TRqqj2sanIeplGqeVfF5yqkmi+MmhlkEjXzVc1qo2a7rabpI4xOWMqUN2miNuVVacZL +R1jKnJLIPvKIC7E5wCnZ4i5+hMc+6cEQLki52qMuxB8/wLbAacmB+x5ltuf0AlIOmgCOdnpH9QLG +kHAlJ5KRqNZnp1/uxUQnE/fnR227r8n+tOP/eEh5UOxOcjKZ8/5n2y1VuzON3WISKuzm5WtvKVFj +y7IijYaA2aLxwv9ciC0cxHdQZjv5gcAuAzj0+Apy8jPJQ9RUorJY6h9r4RRLcrVjtvMeWLgrjQt8 +3+OkcwO0NM3F14DSQ0OC+Fi+UVA93YXowvTgvb68/KG+TMxzcsh6itiAz5/JV9I+EdR8ovICmC/f +Mqg3kB3td57aBvC7RGjGQBcQX0Kc31fY1xi5IEZKB/JoOhDAJeFMymCk2FOKD5kW+sopPeVREiTF +cwaamExVyGzb6ZSeMilDc3MtuUVNmaopMWrOkiBqvlXxqhlGHdVsMmpuyVQ1z2Xa3iQeNt4sNWoW +kyBqblPxUQqpZoVR87utpq6Paib2AtJUzbOWd0kv4G6fC95VvYBnfS7EbOsF7CGrdKpLmcYvp89A +Kv7dKX0zJos9j7eHqyuxpkIyHTL3u7pm6WOQOKXZPHwGYR6DjKEe8wyjiTK1VkhFR5zyGOZ6Rdkk +yvMxDMeGvkp7WiHFjpnab8/SINbIZu3JXay1H2PJcHYbCeHJP9T84l0QJZr2rhxT9/Ce/43FPvdk +NicL/j5eEPjGKrjdwtpsF+JxRLrdLovd7qVB3GMSE8Rui6c3vw/iYFNh01zMUD2Jaq1t7uVWX3TR +ar8cbjz4Kf4XXexh03JcXLcXBEN42cXPRXxFhOktM1x0Co+5EF9KF3Lwpoub2vXEJP8hbs500c22 +g12ILyMmiLflQOGJweJmuJ+ZXTMfGxIM+TL7mUqnqJNV+SydM2nY4LhDnJ032IVY4wt2iDNnBruQ +8IAKliqMDmGnuPFOLo2WD8Q6hxcgUS1syb88LMfF0uFcOyzHJSxX57kQH/hTqnhSdGcSU0JMEGel +iivzaC3CXB+/TyqHMrU0/PHLLX7qZkNYfjY/8KmbAbydJWmWT91slqf9bBZ87ua4t4AFjnPmGOI8 +ixwGV7g5CnbJd8HNCeGDgLG1TyEzebNbJoe5+exufl46UYeuUUimvW65j3KMisY6gdpKs3mku8h9 +lPYBVln67c3K1EshFZW55WbNLYqySZTnpxbKG0Yu4vfw5foqkqFIZVxDTSXCk+GEBOypgAvRXEfi +rBy/zGc5lI9zctDS7nFSB6X7s32Zt/rSAzgvhyNLidbOccHNdB5Y4EJ82lFp2b8l9oOCnOyOybLq +opt9bG2Q1YyPPSFHfx/LIUC9EEPIOJ8PGhdrhAxkzZd4whKB9vryp9bv84dycpGoVdqvUKtmHief +8LC5x4V46FTgC+Gkp0U3C1cPYfrRfCCAs4J6k6gSogpxUYRKh9BLDwfR2JNmbD/jCXvsm6L2eiqU +Zoh47LtMKesUsiZjK0kOfKYom0Rhbi7K2zCYkxUaEjDZjkITif8NM1BryLn+9UoMunOoWeW8Li8R +PziUszJn/mmVWPuXWOYjjpmV2Bf+GspH8Cc5+6JYBJoXsqYRpjPzC/3GeJoaDYy4zPhHYjxYaIx/ +JLo2Fcrm5GMpXGRBPFkgnvQYxqk8L4DF4snMYbx9Rk8ysFQEviZDRhssk1LGcFYEK0R2xXAuGk7x +xd02WCOohBH0u8Lxgelm7EOR+vqxQg1YBWeoID3Dd68vna+mlSjTM0/YFSwE4ODvQu2S6+0Pj3CZ +PhmlHNUUkq/UGZYSTiXYDGzVyJRoqRy3KKSKHWYWTVaUTaIwZ1He4LldaX0UUmyX0/wKmY17IIz2 +rTOuv9wdlkHpQaVkKaT0d8boQ4qySWr0P5FKLcjP6MWX3IqUveWTEbGSPdpeWZ5NHOFCNJ/B7nNO +vdXCGjt2S1VynUL68Et47JYpwWagJ+Wx8/zQCwiccojYr07v6V5AvSdccP/bC+USX6kGqj7k9Nbp +DQx9ghkiQfhLqR51nVy/mSD8/R8STUd0RkmXnkN9/hBSVLzNU5ER8A0FRskV8NInuGzmb+ZglDyQ +qPkkn4zytjNGWcsbWLjjSZ4EBTDa4o7yXZYCOZkYLZ+b+o7FgUOySO7zloVWT4WfQlV4cmd+xmBf +JlLUj/ef/Y8/w0IF6d2Gh/it92JxI/MpzvYBoFjMTmGRZlEsdn9kWeyiWAxXf9oFqVaxVKsni6Za +Rt94Iky1iqVaO5/Wpn/LKnjLQtIzrGX6sL68L/KOuPA8UaGcjME9crKy8IHFBdEW4vycbz8SO9WL +zC6jIJSaH8RikXuMuFyfP4ClFs9uj7DsGyqazZOj3Gc5lHB0SMdK0TvrWU4ozI8JGqKXFbLl37Ik +P3aUMy1Q4iKFZJptmFzPUVPD3ignrghjmmN5u/UGbnzOBe8dvYFccnv69QZiT5t8XWJ5A72BWeQY +0RvYKRwjewOpyrHc8r7VG3CPdMH7YW+g40gXYhb2BopGcmVM1mJl/cKwfm+zXvW8smY9b7OWKesq +w/rR86q1zGZtPMpmTT5jXFxjWJ8ZpawLR6nWA+WsRcr6pWFtNlpZ00Yr65zR7Gvls4hkqlw6u60H +SjQ7NzwXmaVMx1JJx7gXXGD6AaUWf6OqJ8sm3UolBzqPcSGan+r42uIN6MdZzA5gh5TeZcnPF5R2 +CvMRlnMyUSZr+h5jL+tCXTPk5xW6DcnK4tGTuvXIyAj3qjx9bzIOiHtIqKS0JgqjAzgoG41Caped +ZKmG6IAlG9IPSHBz/dBcZToqZHIdtsImjhZKsBk4ApUPfv+J6b3mHVR1PU0ltyh0mNnL9Oxj0rP3 +jXWZIXCE8rygkG6csOJ4WdrclH5CCTYD3QjE8aY0nxpzg/aicixQSBWnLNlax467vP175Q4JDkKZ +MvceHRFe09znLM4T3cfxcCInEzgnPs8fx9uWhX5fAc5LC58jIjM/Nz9jMC5YPGZIetEMNwGMdXG8 +efxFl5knEs+apL5kyTwx+0WdJ+583pjvo5C+6zxx5EX6Lm3VTKk3KSTX3+Ft1VwJNgODVN5WMuCM +UY4XFVLFc+rEEsV9rpC0sS4ZjFqNt8cZm/hFGNM4VxyHF54S9gYCWsdJLhlbfBSVcWK+El52yZAy +dbx206/Gaze96SW78x9X1lcM6zMvKevCl5T1AFll8lzKm2sA1imk16+5wjJnmRJsBgalPHMqEvj6 +pIphoR8fOzRtwR8ZT9PEaDTGtJCdxKH0gcBL7pFvW2g2gQmSEWSZ48OoCbxckB4chJflEW+biTJt +DMZM94F3LExgMcCf4Jwj+5ffWQ6GBgcy8LbI153kQnzyOe5AMvGqbEumTuIZVJHi3hDcVsGVEhfg +axQ8B/O+zKtkeEcU93qZjcZOvkcrsV8hYzTF7NjKFGWTGJ1kwLOKs8R5k61vu70/9gaGiz5uJR9+ +wQQjSyH1TXXL7uERRdkk6uNSkWLrlfa1QorNNWIbFGWTVOz/2Ty8IIEUbZdShRHtM0/aZ+fLdvvM +k/i6X3EhVtoH8yRqoVe4B2OLfCKBe+8VbkPOc1sYGowF0mo/EDefuBwsFK7EV02zBjKwUNT0fZU9 +VSJ3XCO30ERutFDK558rpJo5WcwYjDKtR6uxJsQR9Vkl9fnsVZ5YAKvEjT+o+7+qu/K7F01bmNe5 +U1RVr3GRKtOBeZ7X37Vw7WSjcJ48hfJNZjSwQh5FX5jsMldnPxEa4pMv8JVCGesWeZa9a6HZa+y0 +JR6g+IJJl0888gL1ABLcv/cGFqsDy8b2AAAUAljmgfv/o+5fwCMpqv5x/EwmmSQbXHbZJbCAGBRw +uS17Y13kFZLNbQObi5nsLsr3ddKZ6UzanekeunuyCb7oqKDoK4KKiooYL+AVXUBRxEsUFAVEBOQm +YkTkIgLLVW7K//mcOt3VnZmwfH/P7/k/zy/K1nSdT506derUqUtXVYN2tdB+IiFs4urGZftuJvok +GFDDwZuJRkX0n0iYuEgXmZstpgFp3ynRrCB2/m+8sPhc+u08nf7VpzH4K9GdjRjiPo8np5Ar0V2N +dzXW00mfkQIrsuo956RkdzZy7/mhz6Bh4f3OK5JbSnKD9Hc38rueqxmEjmm1EN8hIUD3NnLH9M/P +yDrjus/CdsJq5DL1dqZ3mmYp7cMyK5L4XgnFLlrw/e/nuVzjYFGbQ8mwaU4Smh+LKwZ7H730TrPE +7/XgRD732QZivmc1YUXiqc82EC/qrpCUB0uIcpzVxIu6Wy5cUPwuxzapIkmOPDeWOefycc7lfRdK +Lj8S6E8lRC4fV7n8ulYuTqdrGr5J7cL6dR+PZYFzUHmiGT4o8cKF7AnwiIIe/Tl4yxx9jd9Nep+r +Gor1OmlWfpvwfO95Md485vsmc74UrFAX32qCFd39OSnNVkl5qoQozXdUafb6PHTWsNcGoorw/Z2E +iYted+gGdGrzqtMZMtwdNCeobefHpFnCRtM9SfTrJmyLPOnz4WTv101YvD0bGc7nuNUugeeo8Drs +E3GeOIH5Oy7hTZrd75jdK58XM7mZi7z2C9g3UdxA1C483iNhVWPtddLTXveU5VNFIN2fjGfre0T3 +8kKK+QXkks/QIyzFt5HL/DKkp71N7JFGhc1+n4qzQ+N/gs3sPrArluipJjT+pRdhKIfG/zSX4dSL +pPErsmr8bS8qt/ZUEzf+L10UNP4vSW6XSoi6faaJG/+tDELjv0WIL0gI0L+auPE3fFEa/5YvwhLi +pcJKZq9jFKhNynK1hNLw990oz5dI2DJhGirFK02YvL6vBtMxI7sjOHIryW7+dKCqKSwtcYUfgFnM +FV/kjRWFAhIRHfDOsXr6O+ImiQ7AjuUVF2Oh2sXmqANPytaTcTGWV19M4DjuQd+kevoMIuilBLWM +2/T6jybqadWXUFDejPahC1S+H5EQua+kxiSOQryU4Mdjlu1/LNHklxrUhVTfEuR3JcSFsR9K8q79 +l5hv4/GbiehllfjYfU7D07LCZqLKDLIN9Zs3bW4qXLA5Kf/Ru5Q8CdFDKbuaqHPtw0m6cgbtqBQ8 +Lv4ytlq7weN5eEQr6YTSfoknNTXsRJN7Fs+lLBadqBO7Kr78Fax0GFOk6C/hMRsquRNK3vpVbLwg +6oSSd+FhnO/tpE7slG75Gt7XmpNmgd5xHj4zi+esP+XTadDwh/Bol1zL9smGBH9FxE7DU5f4ljE7 +ueoSdPCA4C3oTkStvBRcDc/HV+D7uuiCxIefTVIJsTZ9mq/m+unXUd10duLjh9fTc9/Emnve6UTG +kxj03vcdkNkaHFhD32VAGNkJkz6ZaMrU03svk7ZV6xBO+8s401t289QHNf7mMh7uIhJlp0sSX3k2 +SS+B5XjBpT+yDNZ3G2j5DNVRS8HFF6efTqC2jvse1s8TddSSc4ZQRjqbN5t/8nuwgeVtdXW0yNlB +P+ZIouVzdXWE8fMDfNnRPd/DKjBWYEtZepCjSrvwEqy+jlosu4BNU/QCb4B/cpeURz3S8pkGhfGx +Gvli3aKP1NNbLsdAMoX4KXq5DvV15uWQg2g5NdZRS5DV1/gt9i7AZ5sBl6y+ydvOz75CslKPtHxm +kcJwVt9KIqvfAWPZBa5AXCzxnSQqiqVGHH07edjh9bTvlZhbtCD5FH03CYkyVzbQvr/6ehf6G7rq +GypswcY7+jEDPnkl1s3Rzq/mTe334Bl75ujH/Nz7fSnSLvDlhDdwwu9/v4HwuqpEv+dyvIxnZvRb +TnjsD2Az+BLkDfx80Q/AaHn7XnWkTFp9K6D5KkTzpd6VxXXsGt6xLLma6O1XNdA+e+P90j4r8G9w +rTc2u3qfVQWZlBD+5UMJagRtVrhU6sRn7K24XlO3LL2Z6HNgC79B+3wIPkQlKglod3IZDr32X41h +7yHHEq1eolK/ULfsuGOJbrm6gfgA1lk/aVBHtnYtqcMUiIIjXLvlWc46/eUnXDxmtVSx+maSWb35 +p8LqTz8PWC2dx0qehdWGWbBKnb+Z6KOXqfJ/TUKUv5Jc9r3NRO8FikFf+a4CfU9CgM5ToN0AaZ/J +7qIDC8Xt4igHcX4IXYY4zHGiX/BG+Df9Qq0plxABH+b+Av6Gbq8/+9J6uukXDbSIua2hu+sfvbSe +qKVk0WO8Nrbslzylwnj3IV5IefCXYvm3Myc1FREHsnz1PnXU4tBuXptdfS22oiLGO4POasAA6H+v +hTZYsUuWKcXeXs9V87NrRZ+lZXF97pJn0ef+14FD6urNRG+/XKnKkxCqqjQo2rcl7ncSgvbvekVr +vEKlO1JC0J6rp0bw3C1SYSU17J1YN9D0KmzrXEOjoue/SBjoG5uN6W4ue+Y69FBnBI+fvI53it7D +a82/uq6B9j1b0n5VwhZvwhr35V6YvX6FAQ22FO4U8nskhKz31PMG/pMYdGCKQmIlDtqQInJ/1UCL +T0wRXQK0LlLeRIHYK81JqrN/oNQSlEb1HHRJw8fG6umuXzUQ7OmSBriwrl9jdmqauQ43zydNL2lA +l/UbRHPHR5c0oOt76ddwbcvrqKU4SZc1JL9eT5uux1ZQ099mFKwcfZfTfRFxY5P0qwbcxfO361HD +SdkvoOZdGBd05p1Ox/bNKZ9mRNTDfxQXWbr12xrQr7f+BjML9DfoU+k2zqoPkUrE2xrWPpyk8xFh +g4yu+Vo88VjgNn5+w2/hLu3p9HRxzClYZ5g5upP5jIOQzTtDnR7d3nBfUz39AjF4MXFHw37fqKfn +fttAy0soeilLd3FWR92ADSX7wpPS35i9dwOKqsvK0mZd0/DN3Ng0tUvx7pUwqBm04ycb0CtfegPa +cZae5gzuvgFTnHqi0yWBLyGs5tnoVCqe0RqaE6D547hCx4le5BpvuxHKzOIRWnNvjHTKL3PUpYji +nqbCOwLuwTP3JJUUjt688aYGauQ93Puqhl9J8Zbtc29qoGXYsj2LH9i1fdTvoJWGK48kqog8t0uY +uGgZTvp7v2tQJ/0vxY97jiS6Gz9wZH+vm6Pl5M4cxuO7RonmhItzTbyUKNcFvF500s1otiWSRxeP +BTd4vASPUP4FKSj/rtpZraGK8P/4z6rzURewtPxe8lGP03hEPurxYTwiH3V9y+pb0PbUgPOLKYw4 +c7eg+6fv8mrTbbdgxwSbVd7pNAoFXN/z9ZTXVE8r/oBB2yxohp01Pd9x6drUS3vV0wAojbhYgFpV +dXwvxZcJfACEeRbZIWlHMAqDNmlGSnbKbLyEYS50A2dz1R8aaN+9f65A50nYMo6jTjex8K23KgO+ +idV/1q3we7DgWcFeKyEs+M7UHgXDsIqd2qgItuZalXfQdMaJ5vh4y023ikHPcc5vuA2jbcs26RG2 +3nE82xhg/ZWt97t4Zmt+hJ9X3C69YNDztbdiACdjxPtT6GTPCjDqkZbPCIbHiH9LYYx44+0N1GJN +0QOc6z5/hOVz97hbquX+FHeP3X+U7nF0v3j3OCPP0j1+SXFAO5vbT1XsI6qdPfhHaWfH3NFA3M4u +vAO5hY4Wcx6YTruo7Ljr4qpTc6hnudrm7mDt8STq2RQmG8vubKBFXomeY3XSolKW/qV+4r3b89xg +Bu5ks32BWbz/zmhDlcylg22XrK+UUGoPrvQlZnoVOOEuFn56BE/5Er3EuRx0F0YseKP+Emc0iOfD +bXoxteSd9fQBPKl+4iUW/N67sEjy3iOJZiW3d/4qKPiyTx5JtOnuBlr2hSOJyvhx6ZFE37i7gfZ+ +r7uOEhXejRQxS2/C2ak6zlHh0vrrgBtsmE8dfagRnejqexqIW/mHGuFNPovHccv1/B6estGH+LzG +DRxdtrN9XYjCHPDfiMpOWIWcjlv9pwZ8MeLDzCr7p6hqWaSynbUwBWkTaR6+vkqq81mqz/wJziYi +xvksxsscLWKcz2JsuxerEYEYKu6ie9GMeCp8AR/2uB8R3I6+0oheYOWfG9S0YXR/ZZ0XNPK84Rd/ +js8bFuO7QI33NcQmEJYIX5AQCv18oxo3VyTufAlBu7iRB9f99zUQNZ6ximhGMr2scdl5q4g+BULq +j5uJhkUdH5AQqS9rXPbUZqKj8YUY3UryjuuUfcs2J9R+IPqNUuTKG1QopsoVewPXxq/+gqopeWqX +8o2N2NPWMoe9Sobljzsu3ccbyE5ElID+wLo6e66BGg86gWhOxL6nUd0Fc6HkeZGEkPbGRnbfW/4K +D/qbeqIfCvF3EgI01xgpSdg3Ov4Et3spwfE3xktSNIme4JJ88q9h7e7m2r0LEdmymy/R04xYd79U +b9sKVb27VfWee3+DqtKH8QOfejrobxAUffyPJd9bJYSg/2pc4BNKg0ima2NeGQL3ISXovileEnSp +7+c1y2/+DUshLPgHOKLugQal22lJ+z4JIc0Hmli3xz0AkdFn3iXE+yQE6H8V6OMMQrn2kdwPkxCg +85oWKNdDSFajXJvNqVy5WKJRYXLczfEief4Oos824ZbV9r+j5Srf8dkmDNo/hZgxIwfE2oeTdM/f +UX8OfYGfDnkQ70QtefqfB6M+I9Sr5B8oVnK3JQyMvUR0FfO8HDyVDD9iER5ABET4CdP3eyiai+Wl ++Zx9b9CmqCKMB34fLyZq7kauqJMfQimtKTNHdCOfeTrzIRn1fkMSf1tCKPzmJkqh5ZMwPFBC0O5r +Yi9wOWTSqi9hxtCZd0bCwc6opDkbl3BEptfSZxH9YNGbm+pp1cOYK7h5ml2ECYDxcAPt+31JulSS +YusiXzPz7YcxMiut4MnALzjqaUTNk2PQNmOizAifL94a1w9GsDczl8MfCVYzb1503O4kFfAMqejm +Red/u55mHoHl0++Z+NAjDeoc715/UPyOkxDquXMRH9999z/kKOjF/4AbxWnXXdK4/6wQDwaIAx6t +GlB05p1w7uTSjIj9uttUdoH9sHyPsHzGo5DPKBToH4v+8+16ugA8k8FkECNcNQfExKtN2DwvYcBO +ze/+xfq4/tFgp9C/FmGc8mLI/yXmP/FPqbX3t3zkO/X0hX9GDXTlkRl/umQesQpHVu080e1K7vMk +TFw0DXvA1tq1+EDI78HMUze61D3WQI0/SxHtXlFHgK1bdk+K6H2PNdCy+1NEVzwWU1aYVdnOOsWi +Y9OMZLLkjyrTSGZvRWZ/B4NQM2H6khypbpNkv5Ewkvwc/prJ/o9jcn5AHbWU6SOJ/lX1dMrjuI9s +VBJMSYhv+Xw8cUJbPb0X9OUVJLF8+kTilieSdDniUijnFwR/iYQo80cTXOgHHpdC7/dErNCu6TmF +SXPAKJqD4+M0JylNXHIDxYp6S77bZ/c7uAKD6OoExtYnPxHcQnB14rlHknTmE9jeg7tef8prztfh +2TU9upHhB+/GsqZTtnN0Y+InjyTJ3I21KZSkmKOfJXBy+juIWnJgHXG6WWbzl92Qd/kQYm2sPPye +ofs8iZ7gP5uJ/vsOVTk7JEShb0jwDo93SVSU9J1EZBtXZh4AGyzCrUGUOqCP6JMCuURCsL8pwTs3 +PiVRUdJVCd7wccE8Ehhjh6q2FlH8yHSJFV8Rhb/5T6o0NRX/J9bkO54MFP+nxD7/SNLHngwU/1fW +2G14LuboflbUMU/xao9HLzDx80/hroVQmX9jzO+egiu8BbGosEc5G3q6gZa3HVRHSu1PMLL3aa6N +UUT79Bxb8ccRx1UxcLcS/v9ICF39Q1XFoERFSbGqGJoHgMbmVcX/COSjEoL9P1VVnClRUZJUxXvn +kcA4XhVhw8XKFlpBu1TCYfep8gSV4ROdXYeG/4ungyo4u+7cY+vpmaeDKvhffj1ywjPYTZGjj9XB +rD+NJyh2homPP9NAy3dBg2zP5zKm91lUwW7EAvnpOjSxM55toOVDrw+q4HOM/AWQtPx8RFv0bT4X +2fRcA+37uz8raek+FbZ4ZtY3cjmXvsP5nvAcZmVm1sf3ni7jKPs5VGcKHyEflzJPSgjlfqRO1ev9 +ErdbQtAuqOMm9jeJipJi9frAPADUP69el9yrBD5YQrD/TB03saUSFSVJve4zjwTG8XrFpIf/oXZR +yb8kDCrUJro7+dPHkvTl57AaRIm7+ZXL7dBUgf6cRPfd9K8GatzYR3TL61Uv8ufkYrOP6IR/NdBi +u4/Ixo+dfURfBvL9fURtByvkH5K8glyWXD8oIcp3fZKIUgDfLLF3Sgjq/Srh7yUqSuKEYZ/DRfSN +PNFflA6flTBSwoe4hLf/S5XwIS5h4/MYh9KjXMIT+GEt/ZOf7OcblGCLhdUyCSHYw8l9Lu5Dx8Aa +GZVyPqo08uXnRSO34wc00vhCA7FGdglSNHK8sOyVEKy5YMx2t4D/qdi+9QVhW8QPsJ0J2K5+Q0zR +5wm7z0oYsmVF3yCxt0kI6uNK0TdKVJTE8sQVXdqRH8JdKTSnlD34VxVGlP00K/u2F1i9/2KFpl7E +forxcXqeH4/HI+oNnubfSXSd73sRK9VvwEvbtfQio658EUM9mN0uKeO/lDYefLFBmd3RLzUQayPz +kiiZ2gJtNOLFxZA8ckGY1fkS82Jy2Wno2sABpvwr/IApP48f0PBRLwvPOUkiFfc3KfmjEkKHzD8F +U/6WxH5PQlD/k1wGi3nXy7Jh4dtCi2KuT/LrlE+8DG+kdY6BX8Gyd6hLt0ZF2esf6oIBBkpvAWqC +95XWP3V5PV33MpYzXXOSLqiH893r33DFPHLx6FP13fvX09X/Rs92CN6J5OjTjHrzf1ADiPILdGH9 +ec8n6SxELXkj3kfT5+rh9X/4Hyy5F3Cz6kz9Wd+rp31egcDLKwwq0mX1D+1XT99/BWc/OapAu5jV +3pSi5avfxKwuZ1bbEVNCTNawc1YOe7WuYMqiRIqLR6lniWjv+1VhD5cQGv1U/eLFCaJPMDA0UKjB +6z69bBSoXTT04NMqcWCePtGtnMd1iRRhx4p6WlSXoiWeadogH3pFPZ1Yl6KWHZaN7fh4ZXg2nss+ +PVSPIeqz/DRJD/PTcDJFLX6Jbqt//LJ6+nwypd5G/or7r9jv5bMorrODbueXmP9MpvC+/1BsNFDj +ZZ8e4xvC3lCfohaJm6THOe5j9UCXgDZ8erH+gLZ6ego4Y5Je4qeNDSmifbseVUU+T8KWrE//qf/v +tnoqNKSoJTtJr/DTFUAv3wV+4z59oOGdV9TTg0CMT9IH+enoFApm2fThBlT1KB4nw8fr8eg7ZZ/O +ZfJ+jSlqmdTPlzcqRazCNXnnNlz6nXqljKpn9MXn8psoVlb0efnoYVjQpnOYfHATNDCroj7OUR9H +FC2nwwHz6dMNmA082pSiFmuSPsNPa5pTtLydAXJY2UTM+Yjxi3Rxw6pV9fTLZghfpC/x06JFXHB/ +ukTfaoDZDyNikiO+zRGfX8Qmuu8d/1DK3hgou+jTFQ3nDdTT75CkOElX8tOyFlTNE4I68p8qVUvJ +p6sa7j+pnnpbUOGT9EN+OpfRAWo0QHuTdE3DGVfU0y+A9nz6MT89C/TyOZTH8+lnDZgknbBXioD/ +OT+9f68ULW97M3REv2a9/WAvyM8zpgcfVsI8LSHa12P1PGPa53UpNU089XUpUui9H1HoNgmBflyh +/zdA/xzoxm/2EZXeXMeIDzcsu7aP6KnXpWjxjX1EGxen1BaI3QL4VR3v19qxOKW+9Hn5YsjHPFav +jPF4YLHw2H/vlNr/9R8RZZHUBUQSfqfsLfzetzfzu6uPqCL8zm1Y9nQf0RV7p2jxS31Ez+NH48lE +Ry0Rxu9/SpX1LAnB+PdK0HctEcZfWxIypiOUoML4ziXCuGVpipjxiUuF8W3C8A4JI4xPX5pSGrhk +KRhr5zaZ85w+2/Jxp1h62sM6ejcuKqRnlJDnPK/CxEVnYtK6BACiiS9cUU93LU3REsZOLL+qnlr2 +ga35VMD4+kQ88Jx0G4+TS5iXno7I3LRNp3f/sJ5+iCfuXj6U+M8P6+nfeM7b5UjUtmUp2vdYkeQ9 +ErZY5BqPJumXy9BsS0ew+f0v7xl7dhna5JFwfHQuT8pWLkdxl48ehZH+WDm7w/TpGwmsUl+wPEUt +YwUHX0Q/w6RvceQ9gC+fiaJ3MaFt3xS12NkJw7Lpco4Z3xecw04Ceuyx7Nw20/UsLHCI3q5/oZb+ +fpyAAr+zLzoLfBLyx4nRH9XTU8gkZ47TzxMjP6in41ulXu8RXn+WEDVxY4It+9LWFBGvW+0+qo4J +1yb4ZjbaL0W8xNWLH7iZ7T37VQk8ZLieqZawPJoTSc1/15L4Dyzxd/dTEnMR6Q8J9ObP7JeiRUap +VJimOxIX/hhfXlt9NJb+WFmPsbLevD/qag7R3nSxz86ZU/Q8Uz67f4qWzx5Th2/DexP0b478C+Dz +jDQqq6zYVkTSw1+JSxxmQXcwu31WpGjJDjxlrq6n7hVw4/a4Q3dyma7H86RU2/1cpMUHwFFPl+iu +BHrpbQekKLX/yUS/lQxvkRAVcVdi2eqTiT5yAFzZCScT/UmIcxICdHdCLUV2/EdJOiohaPepCvvJ +ASlVYfUHptT1dBsPrKowozw1Se1S3BclDJomT1Y+xrPPHQeinnCZlnq++MAU7Tsu+JskbGElfKru +zCvr6Q/xvCx7W1d6cAgfuSPq5lHTOyUMcsOq7MWcWeqgFO27XsgBrIW/kPdlBhx/UKwgfP16p1Eo +bHcNtfi+lkYl+b4JlVviomXsbrA2flHiAy8lqXAQWqC5M08X8/MVB6Uo9eF9id4nSc+SEAkvTvAn +HR5Exo3vPY5o9ao6Jnw9sfi844hWvD5FqS8cR3SbJLpDwhF8QeSAfX50HOaQBzYQ3S6EAPB/iOgH +dfuchOEpUera44juE8hfJQSPew/YZ455rKsn+osQAgB4XFC3jzqWHrXzmGbWiJW3iUo+XVetmltY +FVtejwGYUShYdr6X/sBxn0QJoR5bknsSQj1/UOq5+/VwHjgQUBL1/CWxGAcB9joYBOhtlxDuT7wO +elNKu1FY3SzhyHyl3SSEAIACz1PaHQK5W0LwiCntTiEEAPCIKI2XSP4qmMclRNn+Fl2FvF8IAQAr +GeESSerR44ieEMQa0S5YPJBYVreB6KSDU5Taez+i3YI5SjCQ9aWjlx23H9HZwKCO9xLa3hICc9fq +ZajjG6FNBq0Q4pskBOgeBVr6BvgPcDpUiNHc7lWg7QDJJURrBXaShBD872qJdJ1EBSSUGus3YWeV +z6ax6VEMbEbgv5EwcdF6tL0ovN9wd+BgjUqxag3NCXY4GRglp1mCIyn0Vb5F5YsQNpJlnAeNSsrr +JZRcsdVB7Qy65dXSr6M5Sbe7PiZBC662eSiFyz72a0tRY/1yot2r6gjiPZJaduxyopPbUrT4xOVE +Z+LH0HKiXW0xB5VnWUdMt2jZhm85Nhd7LVGDyupuCUXkfY+V550StuSzO+nKxpu31dOxh6Baf7CM +aEqIV0sIga5p3OdG1Fukq6ud9zqak2R7p5QMkvd8Bc8Xev2qNdQmSU6VUJLuzXX1hSacudqjAOtp +VFJ/RkLhEhVgUx51vN1xd5guq2wNzQj6gMaY2EvyRLua4dYnD0nREtvJmSXEYOj4zUPg6J2cSZc3 +v+/Uerr3kBTt+33h8zMJcabmymacqdn7jdAwjr4EtAcEAw1f28xHXza9MaVuBSwDHZV52PRMH2Kn +fcMX+6Y2kXadhFJW7o0ea4HY3wCf0LjH8l7WmDTtfMCgXRJOSJi46C3xJuXhpJ1jB/iK4G6RMHHR +8cDzjTvtx9YRnq6r24d4abM5iQtylT4vkRBNXDs2HH69Qig/lxAsfl3Hr1CulKiAhMQx/7DySPTe +cqjTm/Z6XWenVOicpH0fLuWGjBe9Df8uUftiqIh13T+9MUVLXKLiv4+vp8VvwpSxXOzLTW3CKyw+ +JeK/KUWpOw8melLYPSshmNnUuOMwosqxdepx2UcPI/r6m1K07DOHEd2DH18+jOh1h6Zo2WWHEbUf +it7qXScTzUkSZ1nlZCIPgI+dTHTpoSlafPHJRHcfmqJUyzKi7SL9qIT/RUTfSSxrW0a012EpWrZy +GdFJh8GwZhspROXjaHys1wVIFWWlUI+SUBVFG8nOsU3lcb4FQFRZEeBVEiYu6oEqdQpv2ktbeSMb +uqA1NCvYxyRMXHRyPE3JNfklf8mwrWwwUKZmZS39EiYuwmGmSE7jhm8UOAnLtpZGBfl1CatSFA1L +ucU1NCuYFyVMXIR+LcLdy06YOd51wdzXEC1S8jgtKpQU3L6+cjTa1yWHpbiNf/VotPG78IQTd2au +SF/jqJbDMYrPraFLjoavP1E9rqVv8uOFh0cWiS47+q/4FsGi9w6rRaDLjoaR8jNddjTmgmp3zHaR +ypAQpbjsaN4d85fDYWLYrtS2ui4gHH8C0T5vjvUc/A2DipSq8LqgdAehjhq/cTLR6Oo6aieifyX3 ++S1qTn0W+g97KeTdEuIDMdcv4lPN3cigEa+RZlbX0buIaIV6nJPHA4iS3rSdXWXypTwVybR3sWKZ +uAjZJZZkp7MFXIBzI992OfVmLExiGyHdmIQ2vv3mFLW4WCv9PQPue7Nc1N62Rgl8a3Jx9yKipStT +6lsOXfiBi9q/uDKmAZO/UzvtYThKoyLEhyWUetYWbk5ZfgCeEdDlSwLJUQHq0OvtB8MoblmZInV2 ++t6D0Wvtd0SK+KTw3yXpPyREwnsP5pPCpxwRFe9YHmGvSpu+3gEzK/n9ZWmQ72morsCjXcBvdd57 +BGaE9OkkzGX2yBTt6wv8kxK22PRVVmXqKEi5g77K130ef1Q0/1X4FJ6HrzhlaU4S0j5Bvqj2iHLy +TrAzQxCfkDD4kEqJ6Od1sO0CsizQLL+xnMGDR9fVde9O0kN4GKPr+IMq+aOjm4xXHmn5xtgIrp7A +q/IjVhm53DHjRap0K3n+LGHiojbWR5Ho4sTPn03SAzE2rFHM5jonTNwnIKlW9CguiYsSnBrTxQOx +T2u/Y1K093vdNZQ4EAKSNoaQE+aF1CYMBiUMGOUsr2T42QmicbA7+ZhombSPpVFJd4GEYXrPJ/pG +AocozjwGrcDN4hm8dsV4OerlyIykv0JC4aPFNtD6hlzTLJb8tTQrsFslrIKPGbkiN445QezdG9fU +uE10f/KFj9XTAzF5goRrqU1SDEooeXDKBznlfquiWkFKx1V2l19No5LMlVCS6xLF8J6Vt40CVQT8 +SQmrErGTZ/83I5ArJKyC4tszrk+zQr9NwgVwq2lOAE9JWAXMO3Aka4g2K1XuI2EV0DZ3llwnS20C +SEsowCWedQbR5w857/kknbwqRazRzx+C0z5nxjTqWfmS64wPOHavQ6PCxJVQmGl98s1gFaFeKaGg +lmSJ3ob8Lkd+k0Rva6/U0wOx3LLcsmYl4a0SCgOdjWflfdcolvIOzQlm/z6lEMEu8aw80Qw7sf2P +TZGs5s4kf3tRPZ2CiKw/BQDG4e87NmpDtrmT/Ra1Ccu1EgprLUZoallqF8ywhFXYeV8jGhWcK2EV +3vCK2bzDzacimB9KKFiusqv4HdmVKBDvvlPPD8YKtPJI03UdN+3jG2pHrOrGA80Ks7fgeyDwW8p7 +pXAZ6eNCawhooC/DvUArVrOPx3vCVavM01cFG/rgX6ld4J+QUFjCebM3nVidoiWnk3r4AjhVM1Iv +WXlhfEbYvPWUeM2WiBLZmXq6OWCHh+VrUIVVco1Z/qSZ9R2X2oXLVgmjwuG42mlrRDg8fLw2N3Qe +HZ5nuhimKjWOCrt7JIyy/eUR9fSLgC0eGtfWFDLDw1GaExb+luryfvCVJA2sFQnxcMECrPgOAqoI +iycljEqF+wP+FLDCw+J1NaUawzY7T5WS+pVInRJG+T3y43rauk5Ew8M5zC+wjJVHRhgFxjcqfG6R +UPjVsAdv2is6Nm4PpDkBNw4oYSQR28Ne19TTNYEQeHichaiyB69kZi2jQG3CIythlNfa19fTIeul +QHgYWV9TQUWc/KeKcHhSwignfCb1wwEnPPx4AU7qND8NqoJ1Shhltb29nhLHiVB4WH9cTCjPd8tZ +v+09bYHe81sszz8hfLTbLNtft7btTBoV7mUJo7msuqae8kEuePh8PJeA+UR2wrCpIhyelDDKKb24 +nn4XcMIDbYjJG3Dij58TDamSny5hlBM+T967QUqOhzNqc8pnsRxDFWFxrYRRVjdvq6dfBKzw8Ayz +CmRZeSQmXsOGveOIVcpZ0pxw+dTblYDCrcU17B2UOOrpJL35LSiXZoHTIMOm4Tl2yGRGEu89HGey +kxLLn03SO5lDlamyz+7I5VzT80SaNmFwloQiDbeAb15TT+e+RdSEh1lmqwWr5he0xhlht186Jp/u +51YeWSoYlvJ4QaI2ARckFFlaTEosPqqenkbuzUHu2dWEUeeXb7rn1s/c/qe36vg1HP/z63967zWP +XXNEGA//P2zYOae4aRqXmWLeVkmE5KIx1VGw8jbHN4XRbJV8PSgym9PZqw8Pnrpxw2bD69h2KpN/ +uSB5LdOvrU3vHu5PM/3XtenpdLdKf+N8esdw/2bD6+vq29bBDH4xH9DfN5TesP7UzYbXn1aQWQ0p +GlOdQ1uHXGc8jTkVlzxCzplj5fwWJx9RV0RfAVWZ0RaraPnMoLIk1F0A2WrvsJ2dSrNa4QF1k+MU +RtyyyanrqhKD3GMUPEVPVtH7bJVvfRVlqyWkhirSZnOKs0tVUXA1CSyjsYqiCspEbR5BITod2/Mj +iOYFkg9Omu54wdnJbBZVgYY6mdBSRRgJT5xAuL2q6Jv5xBnbKQDVakxP29mQvE+YPtPdNzAyzGm0 +AjPdHb0dfQMcq0XJdA8M9nf3c6wWALHpd6Q59nDNdmh4cCQzMDjQzQRtcoow3N3RxQRtDYqwfbhv +RCXRJVCU7lO7lW4iYvZ3DGU6BgaVoG06c8QPDfdt66jmBVJP36ndKnttqpn+jq5tma7BgZGBbiHG +MuralukZ7lai6dpXqTZv7e0e6uhVxMVRMbq2ZQYGY+S9NTndkRnuTo90DI+wKiqVaOsCdXAgPdLR +eUpAjbRMUJF2cLhb1V2lUomUP92RSff19g30DHLaSEHSfb2btw5xbET1DFZSRNSe7ut9+9Y+Fa2b +HaL7tmxhFnHGI8MdirNubgB3bJIC6raG6E1blc3odobYniGlxYiK0329pwQZ6laF6K3p4TUsh25H +iE53927j6IjppvuAXsvREdtN9/UO9UmWr4vUTF9vx5ZhZemR+kz39aZHTunZolQSsZ10X2/n5i1d +zH1pjE3n4IBCR1ocsxlUqloWQ4+kR1T08nj0iLTFfedFD27lLFtj0VuHezl2v1jsqZ1DCrx/PLon +/U5Gr4hFbxsJy39AjDA0PNjD+ANj0dv7Bjo3c/xBsfg+ZYCvj0UObVc2e3AsNvAhb9CxPUPdmb6B +ka4+VZ0RgxXK4DYlTMRoQenZEqaJ2K1QgjQR2xXK1oEuLkLEfIUy3K1sNWLBQukbULJFrFgo6a2b +mFvEkjdtTWc6uoY7tiiHFSmPULqHlWYi5QFlcNPJASVSHpg53FlAirBjUkdnZ0CK8Osb6evvHoYD +Vk044pmFtK1veGSrUCM8hRoaQIRn99Dgli1iopEUHD24Vdl/RN0cH4gWUQ/HB94p0ro4frgroMT8 +INO6VRZwoPPZZTq3DIY9R8xHcspM58iWTEeXqvf5ojOxq1vpaX5xmdg/qFImtc129GS2DvSdylUf +YZge7Dwl09U73KGcSoTbYGa4a3Bgyzs4RaQ2BtF7btoyGPj/iEIGFyyVmhWr/WGNTycj/BQlXbCy +Zgd29oKq5YtSs0aJqVrGCHUTk3R5I6R1IV9d1zFywFi3ryhZcdYtLErr5Fx1G1O0TseeNF2fabre +xw3PLzj5gXJxk+V7rFWdYXE9R1z7uT9/8/qr777pLq3v4nFM+f3Xnvnqy9d96nrtscYts5A7xbJz +3U6BITpNSArGjFqhIalv3MiqkavWZ0jsDolao76R7xYfq3Pyjfzg2LvNrBrp6mwQ70+Y7rDjKJLO +xDfyWFlikWPce8MD1Rgo6rryjTzPAnr4lDhoWm++kR8yXKOo9KkryTfyPZZt4PIjl5PoOvKNfB8u +3QYjXTsQOD0y4ZpGjim6Q/eNfL9ZxC4ClYnu030j//ayWTZz8ax07+4b+S7DN5ih7tx9I78prfy2 +7toBNXFPP8TSXTuXz7ayHK2HaEomzI+YoJugb+T57X7aKJYKSse63x8rj4cD7cioDHPxTlywuIm3 +0apSViIVNl1S9A7Pc5QkkcqxiuZmy/PT5TGVPDTuasxAuRjCVC4RwYUPY0qmq5gp1DGhFwtQI45v +FKKQP2iTGO+zlVbgdiuVyvOvaHPFwm2fbflaDVpKbGcOhV8dZpm1dZH0oMiwprBYEeKPD/GGNdUZ +SaKtpVj2zalM2caqi6mMTAumiBGSbkqK5OFjUJatZnm6brCHYNLMeCVLTWIjxcn6ASWTlemmHtiU +DM8LyGxCOj8IgVWhrnKxOE1Y9NFiBrQ0r1MysTphWrbMMFmLGqTtcdysmc8yVbf/gJrG/pnthuXj +cg9krksUQDo8z/J8bnmM0FUfIDpLZbzFYar2CQGVs2CadgoBbcgpFLpMT0mnvUNATmPDAyfV/iGg +dZlGDr+ZrKsdUVBnR6GQZ5J2DxGSkkf7iIDE36fyON3rQhMLiOzGoCPtLwLSsDleMLM4masSa9cR +IDbzoiZS60YY0HosVbfadQSUAce3xqex2MpJ9eQhAPBywKay0r1uLzGyWo1QgukJRQDpx0JzWlau +IZ+eXAQQdnwg6QlGQOrNok12uKZtqAz0bCOAcJ+E1HrGEZBYeqbpaUeMxh3RiDHGGD0DCTADpl9y +CgW4GEboyUiAGN7J7Xk7k/WcZB55mMm6rw/IuIA9bfrpEu78QhF0iw4gvV7WULWnJywBjaUvOU6B +0+p5eYy+xXDzSgeHVFmc6qKQ8RuraOlyzlEW/qYq2nZ0PSXDVlVyaBW9H3dOM9/DatOi9qAXcwK5 +ewvOmFHgjo+ZvLmKSS9/2guCr6yicUvYYhrKZvWybMB9iDtgpD2yKu2AubO4GUu34yr1UVWILqxV +Dvlu0fCUazi6CtJj7DDRs2GMoTR/TBVmxMruUNpbVUUbNrJmj2VbLOOxVWT4NO7fma57tqB8LCHT +1lSlhVrgvZj8sPaXRWNqxLKnw35U+xBfojsLhufR5qeTkS6gaEyli0ahECarRIYZ2JmY5utEMQTS +3o7jcZIH0dE5Fgj90CoIr5wUyl40pgbH3u0NmS5MjqnR6RcOP9Tkx5RaDDMb1o/JGrLu7TILlD+M +r6WATI81xWbaOVG21bJ2JbJyneEGynWlJYyoKIPRERbEB90c+kcUXPePaEOxQYmuan6FUJDBGwZF +lagiOSE8ZnQ9PTJALDhYxS3Ng2jvp9NbftEoxbhohaG6UCnMhkWPymAg/y1r4LyZpsccirI2pGif +ryhrtNHMIy2UZJPh4SS5Z/pYiMdxU7xBUWoZDc0o029M9WZ5Vwn0rL1l0bK3mHmjMORY2BOnEmty +CSeOPN+0fa7kSEXqRqIOrEXHiXokFNI42+gYxMj1GHi7P1AuMk23kUJI6zKVxetxV9GYOsXULTVS +I0VjqrtgFrWEurJyhm8MjkNFXDwtBTZDTQ+bnpJO1xLHD9pqyqH5mJNGtoyLYdXCg5YqJKj1BV38 +kNBdLPnTXE5t4kXLHnFKm3GgDHWiR34YLxrY+IBonb1TyPX5pqbo/PHObbtr8S27SKPz8Iwitz7s +jWaSLnzRmHqn6TocG3EpY5Yf2AJY6ezHLD+N3hixuvLRXGCa2m51ip2Om0Mj0S2JE2vphGVHQc35 +d+ukWoqAqAd92YIzNma6OdPgJQGY+2U//eGzf/7jv+54lzb4bqzMzZ+Y4j3MNll10+0LZUB35XWZ +JXpHbEEHm5t9Q1O0hHZZTaO7zJLHiXR1FE3ftbJYzdhk5KgcY6hpaK4b1jNZc9XknoJjBPRarIW+ +2fJ8J+8aRWakLS/DHWGvepmjLTvTY9n8VRDdTnRj5+3+/ZaNr2x0WZ6Pq2S5xmIONtPbiQECzCDC +t7ezaLiqE9ClkVhfn1HhZLo46hwJXv6b7oDjqx90QUxjUUyXmbOyaIH9OHkBnM4siutxMWN0bKMQ +Amtn2pcrmCFEay+f3WRkd+Rd7KNh8tXzROJ5X0iJCsEUVnFIjmY94lr5vOlCxXT7PKZCwzSJaVG2 +QuuczhYUUTPlq/gwDwhXbub3OhrhmmYvd7oKonMIIZ1OWSbYOgfXcXwuUrQ31D6AxygF049StZ/J +uYZl87ZbrEh5E05BrRfMnKtns0F3ijEOSsJmElnOyGe7wIW/Hy2bWOm5p5MR+QXRUyh7E5vyna6Z +sxREl0IgqHJOrP2QULTVMF0XIZ9VDNMFfIwMln+Ldkf5rJrDo9o04Bw9bM9nBydNV4Fg4cygEild +zhw3ygUfNtFv2VZResRKdLTkyTrEkOlm8VlbCKFrzzV9bPLAJ2SjAD1xLxpTGCrCGw9NTHv4zSwi +bzuDHHDWyJ3kE2XpCcPNKUuJAOH5sLDBY0JTkc/XamYvwjScxaP9nknSK6+88opWWAZKwBwqHAdr +U9rpuCDxsJIFjOqgaNmB8Dqtdl8oo5RNk6vpm8t5E+WPYCKj0sASh81swbCKpsvjHiWKNpgiDBVr +JLThmej6P8f32Vs9kwm6gpjQb9hlo8AUrS5os2TYrDBR5jk6HU9AAjJtfSYZGSZgaYEVBcMhLyZI +SOPGzkTNNCQO+W46axQM9NAeg7RcIQi1hTUQZKA9ZAbdm0xkQ7/DatL5RDFY5cA3TWC3OpMoYtg0 +shPY588YnVOeF0G4GpSDQYcfbT4KoLZmaIS2qZKB2QIzQL0rg41UZoSukkPGWOMrOPmhWkyi62ZR +hGajBxleuVg03OktuJkZAybORZcyIK8OaXodLCKgF5k7vC4c7MQAkSnEfES6LJuyoELt3tBwjOwO +M7fNKMgGodguiYKT74f/iEP0Sta4a5qs3nS5SJ9+JkmVsyr4v24vQelQeFVyPdpFi2MzXodpEatF +jytC4ob1m6pmrfFih4qLljqvFvtVvaPUET9WNPmlA6zyymeifckYFhMCgwVJGyyvd0VJugYx1cFg +PvQrX/pPzP+qlgjbirgssOI+nU+T4NhOpVLRU100wrSp+t5u23etausVCFbpeCWt0yhxJhGfxudN +Ysl1gULatIz6VfVoAJYABxy1lxjCR+rGKRS6XbezgDMoai1ft30holPErUhIGefZ7WLEh+WcGm2+ +lBs2jdw011iEZw5r6RwZ58Vjk1DvEZef6dk60n1qsEsp07OlQ+3aiEwOBbK9o28kwLG0tSCndMcg +79eiZTq3DA50Z7apF88R/QqhR81EIk0iIPRtkVlKZMgkNGwj6pDtEpE+UqhDI8MdnSJrxKqFuq1n +cPgUVZBoSZSYQx3D3QPB63ytSqGObA43jkXdrFAHurcPSGFqSPyO9La0bF6rVIuc7h4Z2RIk1q1A +OCupGNTXJaJXF6xzc9+Wrkznlu6OYQ2rLuLWAVZPwKda8YoPJAozq1GekcGhIc1E9yki8kD39q0j +QYGqSzTQvb1P9hrGF6myBcc2ewpGXrW2ysX6zUGmYyQzsFW2funGhmjsfpONRHroDMLm7Z2yKSwi +YcdIZrhjoGtQmaR+BQACJ1D7tPTKVyadznT1pTs2bVFWFTGMgXSfajiRna+ZdF9ma1p2xkUETff1 +ZjaFOyrmEbYOaFKk9SBNunukvyOtbFZnzYciVN/ODRw+/DeaadGyeRmfOxAm3lKD6OY9pukuL9Pr +WDlem9zEx/Tgn7Tu0KGdYeLLuexwQMQQVq8Yjztu1pSJSNAaLrtf93d4YQcPZrrp4FXnregXK9FG +ARBWbUwet4cA3S4A2FQeZ++Gt6W7Y0M8oQ44akIfALRWHW/YLBhT/ZYtImrN+ME+1043WH7S6UJi +sPSiayMk6eUeTcz0Wjl5Ta9zyvS6ZdsOXbzORREgNdQb5RIctUW87l4zvTuNWitcmd6i41pjZTuX +KdtlT15G6+FFphcLRcws0mh6Tft07HaIptHVm+nNOqVpdZIMUkTtpqTOUEo+egSYUe+qAI9UoYqN +qeD9uoI1NdDDB+YTo8r44HxiVCNnzSfGBf1IhDxUq56GIGQgRqSahqISRKppKJ/1fEeNNSK1NBRq +W7cG3lYNAx0yXctRdaFV6he8dMGR3RG6ihAdTBD1gCiTtvLqtS1rOiJn2sqfYslCYkTMtJUfmXCd +nQzXIiFevQpDhWlhEN+lpuNM0T4BlF6Hv8+LJBF1pq08XmPiVQaTon1W2spvtXk8yaRoP5W28n1y +DiLaB/Nb2+Gy7VtFkzcJqS4iUlJGjOBrWhAkUlYmnFwuYjEzqzKMlJip+HRUvzEVYfyAbqswJ3X+ +hReGccClFrG3U61l4D4SGAxwWjzNpG+QvSfIWkhN7pww7GEza1qT5oBVwBNDtS3FoWnTzkVxumQa +11UuYlsJz4eRrfYCGtNruGOYFjgF7CnAJXhAart7FSQvfwOtPYVGszWFBdYWpRFpEzkyA+025pMH +nE4D6wDIRk8fNCpQfpiT3oERBfECTYjRWzFiGFlSCmF64qRhqBepJZZJz0rjGFQPA/Q6jwb0BLvX +wqx0u4qg0Kn2dvL6HIqvN2xoTNosGtnTy5arpNFbNiIQ9LjMQO/YiFCn7WynY6uZBHLRIy8NwqzF +DXfwhQLpGW8Eip5UdfXcT5tKC3rer6EofI/j9nbyhI8z1/s5NKy3EysuJi9NM0jv7NAgGXtIbnpw +pxG8/o8bD5iHHuW5as9GvzGlpFbuBSMT3aI8tlSu0nXPRmfGijBsZieZoJu1IgSeE4l0O/bM4ogx +Fjrzl7Sv8MyiwVrDNBjz8LfPy61o9GOjWZSs8+RRY1fPFh7aaUfF0X29AxwdycvKi9LYbWrNe1ae +axjOVHPxrLwye3g4kGKcoJqAoAXyrHyPNVWu6hP7X22ngBfsIeiyJjkj7TpCUr+hzlFFJigFbGyB +ToNk0a4Fb9atM9RLe+k/rtNF460BegCnW33RsP0xy/c2rKfJZ5O0PlxfMqdKkXjtccYsA2Co5/1T +Ido27A3rCV9bh1VVKpWXo5sW7fE4Mbqj0TbzcWKl8r6QbSDdurUsna5AkU7itfog3To1yXm/ZmMb +9rq1wUrLf7Qnt+xxHf+yjrfNvI6PVECGO3z19STWQETBTOqXXYzRWWOPNWXmmLqa09SmqeM20RVz +nU6VpzZtHfOsTVNbsmvTjnuVdBtehcakaBG4ZJsstRYUHQUzobdsuGrwN6PVq0i8kUax0+1MkfSR +xxndSfPQnD0cJ9KqZ4LcEuyb8ytGUfGuZdDuwQfJYbnzEg85lufYnU5JjTDnUdE9ZyfUErUm+Rhy +4YaGTZFNyXqEWDSm8LKXzVYbYtGYCg5VYnLJf5plZqizq2OkI7PV9oxxk9fm5kkbIFi3/UZJnUdA +iSIqFC59dmHENU0N0U4r07N1QGWFaTJnZLr9Rkn1CxGBQtwWXG1RhYxkGiJZNLWlXrGrla3IxpLr +TkMzGSyZdqeTM3M80e/DlbgopO6tNLTDVd9TBl2P/RDtwR9GD8hWKpXKVOhbAj1FtJ02xpVMusZq +oFQCzvC9Vdyws8RwfdWcz1yIrFr0/yxE7vCxcquM8T0hSH0HMmM7btEo0IefTUZqSmjGmOP6TNJV +E5C88IISJNW1EtKn7eCyIEboehFEVt1vgukEA+rnS8YndjH82LaWAbo6hEM+G72IkTG6fQcYvjGH +adpjhLS8wxTt8ENKiTc5omh6oC3ECcPOFcwOFLCbryIDSg+0BfXuYokXnpiq+zuhqpuRkFAPrANS +cIMRk3XXGpA9GATT9EBaaCz0TkzrwFkPo4Xs+qszUmI9eA5oZXvcsk/nhHrUrIm+VTQzRdwKDtZ6 +0CwI3BMUakwPmgMqfyCQXScn1wPmakDG22n52QnG6dGy4FC2kql0qsfHIGIlNDMyONQz3NHfTV+J +DQJDenpIHVsGWVts0bKB4Bao9YKvy+IWdS9YQYh0RlCGO+BgW09ZeSXt5JiGcXowuNNNh0nDZTtc +I9EyMKnLLJgYj8P36ObCpGGz6ITjSN1SNE0Wk3QbYVK/k7PGpwNJdOPQRMvMdRtuwao+FoQ315ze +MnNbDF8AuqkIYDLgrttJ0ZjaPiH77rhPwl9EQ5jvdE8OOLI7roqi11MjulOJelzz9LJpZ5Uzi+hP +kbm3mK8/Rep1isYUdi+qCovoUNGHXAdvul3VR0b0GCXLYlVEkYra25nmVoma024mJPIuBtAi2gsT +jmzXabUOo/QwuXYymoxtFJqBdjRxRMhCu5sA4HS6+FQ9y6e9TUjVvLW7CYndthqNaVcTkrD0zTx1 +k9K07ITYq3Y1ITE6xdJ+JiSruTmUqd1MSOSZIBO1iwmJ0VU07VlCMqfFrIzTa+8SB/DEFbnryUMc +IEs0gOhZ9zzItK28jZ50xwFYXWAOes4dBwyYykz1eYqQnp720Gdycn2eIkoOFyD1eYoomfXAyfVh +ipAedWxvCDvskNxnp6fVjZgovx6+CgCrauo1C8iHzE8Pcq9jKNn1gQpJHF9RYQ76YIVgesq+VTC3 +GztMmULroxWCUGdtOLU+WiG0XmXsPDhlhD5ZESLEiDRGb9IKMelp1rHG6DMWGgN/scUYM1VO+pRF +iOB6kN2X+qBFQO7E0qlaRGVWLK8+bVEDFroAfeBCUFs90x0xvB0RR6APXMzDBC1en7qIAIbNPFZD +Ubn62EWEvsVRmteL8ULUOwb1kQsm4ZhHsDwRWRRnGjt7DMo5w8isj6m9BWcMPp2JegDONN5cM2S6 +A+XimPRr81xrh5tnmfQaxzyfH7M0fDsjmv94wchvDb4Ae9Aj0eUvpvHbOlQbEyO9GBLiO6o5JkT6 +KBCGzXy5YLj9ZtFxpxmguxN8agv373Bpda+J6GCGqLOR2I0M1gMNiV6j5us6c4mXdRDdM0r8BrUu +oPWDeJ6swg50bxhEq2zjogMu+eouMEggGesaCgiSs+7zAgKOMCBz3dmBwtudhZnu6EKKcNO9HCid +Ds4NTwlNd3MR2pq1qkS6nwOxw3UNNUjRXRzisRjOsumuDdE9ZekSdKeG6D4cqAjPnus+DbR+2aWj +uzLEBifcdf+FWL5OgLPV3RbH85IDE3RnJYSyHGLXnRQIarYqc3xOqPso0Lss18z6faHMug8Atbdz +yBUfEGnNoPQHB5d0r+KWbVNvGvr3K9pSQUmbBTnWrM29aEwNl2VU+cor1+qBpVd2XSdv+Ga4orZL +c9NEWSN95eeaKFNvzcqXdVSdq0y/ZyMxahltLhKjGknkeIKvlsye1xiciFPMo2e3vB1qdq/1ApzK +QNsj4lQWusFBS2vCMukSIH5tGK95IH5dGB9Rd8HJjnFVRwo9YUncz3UBsC0k/Y5030DPYKZ7c9cw +p9G9fGZoJLNlsMZ9UiOZrncMdPT3qZMG2k9lumrvY+kayWzuSKtrbLSbQnR6ZHikYxPnq/0UE97R +HxC0owKhd2CrZvbsv195JU7e1j2cfofa/rL7lRrErm51sc3LceK27uFMz5bezKaOtNoIE1FSevNA +ZutAkFBXSya9mRUR5KfdWiY9MsIrVFywiH4QPzA48g65GCnKa2RTpnfL4CY5qBLNfmRTZnt3x/xN +MpnuvsxAX1ewp0s7s8mc56Snw9cv2DzzrT/if9ofA9I1bXNXDDqme9Hd2qCnp4vK2fAyGhdEwV7R +sgG3zXQBDSbdgtElAya2RVIhdIsBYFNBvnaGrckmy6S1BgB7Z23q/JdoCu4rLHtmh+nhRBQ1J5+6 +eAAX5BK+66YhhuntMKf5KmhqTrZ/VWGevaY+ggGDHeY06F9W9P1/EqVbXjbvUHPynzVzyOZRTh4p +9FNz8oGFQJlp3HtCzclvfOIUFvRPWPsIy2LLMhs1JzfPKCnwWkUDpvhexKHBoc6BEWpOPlszHwVK +p7vXr6Hm5HOvgunp76Dm5DM1EYZb3Gx423qGJtdTc/JvC2E2rMdFjSOD/X2daWpO3l8TV/bMjm2n +Fs1i0Zk0qTn5ZE1UVi4+aE7O/FFp54tvjFYBFiEznumb9iQ1J+euVJg3PpyMKIgxZTtEPVwTNeaU +bbkvt6foe9ScrHxacXvX0dEcFW7AzHPPJlAS6PEx+xgrW4XwM3vNydFPKn64TF5Xn9yis5avt4ep +na1Q74yxsuxxak42iQH8+6ioRLgDZIuT3UHNyf0FcMyi+YCA/S4xsptjhQILRmALMYr+VSXEicdE ++ZQt21+zQV2l05ykTynM22k+Zt3aALNkQcyG9QGmrSZGfVAgwOwW1cXz8nDHUgCZqwlRIvP1PM3J +yveVxPigqa4AQNatFcg5C0I2rBfI+TUhSl6BTNSGQFxBjNZG+IZvZSHQhvWwwJmPKIHPjNkCdqZj +jgFESZQ3fkS0EkrRIzDNyXPEKuDadbkBip2DaU5W9gzk8+zNySlB4h2xZnmG6Tpj+HROc3K3AOJ5 +5iP3GjQnl4hX/XKsePOOG3MphVkCtyiHnhG5bTPgGNuvUI4x+4aoFvgEr7rluTk5+jUFWXZQFFL2 +TD6VxucWm5NP1/RBRm7S8sytamNhc/LQiqoUbNTXwoxbfA8OSv4llVO8DY5jlbU5uVrqHR8JiSU+ +nZqTz0vKn6WjMo5bdpaakw8vRCzJVQzNyfavqJwL4/PS57ElhJqTD9UsH0TbaeyA83m4JsAoFMYt +G7W+kAzqXip0LRtFO8mfzpchWDdvTj5YM5d81uQ9onDn5RI1JzeKrt4YY5XPliaUiR0q0sQrYqdr ++eYmw3WxIt6cJLGxt81jsgnH+Ey7m7NE73tQbW6Oizpt/5bS7Bt+Fi2VOm2KZRk+cshs2mqywRUl +2YJp2FywU6VgNzVGuTm2qauyTdS48fAoJJ/tdGzfxbY1F/V9qZLqFTMKCs7woc+8SpnqJ6+PAbDi +jNYu1Cd/E6UWJ0yjlKHm5Oz6fh6OvHxcjIyTaHhvYgYu6PyaLkgdyOKrEdCCK1INTb+NMTOm0qbh +ZidwywM1Jyek1A9tjKL4g118sqc5Se9XJTrVq0IErqmyMGQLTmjl2XG210RhPzXcK3QnPifejIs4 +4WPy8GClAP7LjooypgEXiinEAVMa8HBtDuXsDow84SDFUJbcEMuCX2sVzSLGjJJH3M0Wh1wH44U2 +ycCKp0cJUVKrgJsE2FCETXxkyQed4sB24RgH9oenqIYN5lcSfljD0W4uZ3nYBN7P618Q0cJ3QKFr +GZTEh+i80gdZqTl5i+Q7rzYse3B8XIynUtN4cC45hIzWhMjpKV6uHcJdQs3JXZLd6/8nqviiiVeg +XPntV6mmV/ldFGCb4W1JMiAbFUZxuSM4E75ntagr7skEhRV73I3SnFxfE1aSyw2bk7M1h0ClnIw5 +hsSa4sMfszQOEVaKas57Pjp0FhE2uaaxYxiwISlP3N6isO1oyKe+Ggyr+2kL9b6xdoFcJ9tRnsKA +/hbxLXvtFVUzLrbzSkbWzEya6Br/Kv1J3Fnya/ZySd1Dz7cWNid3/0C5jzg/b9ob2TyEKeuQ4aPd +7a6ZbdnFvUjFTM6EaEsuqMnKynumnzEKGJacLzo9K+bnPQyhePIAu5Yqu+MD0QLmrEkrJ0tnzclS +TYwj15sHnHbVRI3jBokAUqkJKXJzDDDn18TkzHHTFTNaWdOM5OQA74DmTTrw+ptr1y/2GfCu8eZk ++8KIAtr9hBhSvP3krJzjT6hG8Xep/LjvwAkEcR2zkkWcheHp7S38HgLza6mvuHHzRwpxFUTBgc3O +fVu1/C/eHK2xKEjtsGpOni+yz3OCuD5LRPtNTdGKlj1hTuWsPG5VbU5urMmmuBp+82plg+2pqCx5 +kOhHirTulWh7Vl8qAX21cMW3SLSHxv6QjGVbfiaHl/vNyXap7NlYaQFL4/0Xe68nataArMVgQokX +f+irJEt8BVFnidzSVh7rn+jxBBNvMmNGLtxUk1/d76EeVouhxmfTMaTHHwRT8PaacKNQCOrinJp1 +AQAseYk4jntjegAVV/w2JyuSOm46oONVRnPyeVlz2PvcaE1h7g+n4pmsyLmaTPggmpo9LBdAvJ/Y +6biFHLZRo4sQE44j8lkhH12TbBpuYVqWmXg3V3NyqKYHNKfw2S64LREkXlwm93fy5SHNyb0EExdF +YXSftmQBlJmVHpRkUnPbF6KasyPX1DUnZ2V0edDvoxjL7nHcHWauc8IqoO96pKadYnxDzckLxfAW +3xJlgV1RrjOO9/HK27R9XbV+r2lhGG5+aE5uqVkwPjSYz5bUIajm5JBUSLxNcNeF7TnUnPyRCDYf +YRqFQVyahrmtDOSvOz8qFBoWD6GAEAU997kowrQnYdslse34GgI+BYcFCwhZE8C8YS4Z9fmj5uRq +KUy8xjVOLdIN1VSM4ebRldcLMT4MMdw8utwmcUb47px2IL7p+ZkzNmBF8jeiq/jolAFTDJitCWCX +jOVDMbVizAZyY/lJw4WiVkujmLstqkXMrdWJKVkfhzu4UHnfQ2+PIo1CAY2HpBj4XqwuRl5vb2pO +rqypBjtbKlNz8uiaRLEs9DtSXefEMg+Wumd+qkTbfGdUNFt9bhCy/1dN9kahUJIWeb4A5nWohQIm +lasl8+J5UfY4bFjiN4TNyV2CeDyGgLW7AWSJGHQcEt8tKiN1kho9YnU0vzg08Ej7i+Bx48Jw0/Q8 +x92Ge+4cm3cXNye7BBw3ZsvDG10M7v5R05sUxk07a24yxx3XHM75Hurj8ZpIyytYY67h4vXCYwsg +MDW2eHX80ZqI7IRhe6adKyGbH4nAcZ8MiGtmJxmyqybEM4s+5mXUnGx7VhnHD+6KanN8Z06N1+cu +U96vfHeUzNtpLTuvMG3fVJgLYxjVFXuDp2AQXrssOIWsurkmETNeT2jGaSvvY3dsc7IkjWj5PVFR +BFP2eC1qSjDvi2HGjFzespVA/TyUaKs5NlBD+Y5CAdutlGDniIOLj048nstUxHl8PpYZvnQpuh39 +otJt3b1RibO4VTTjOxneptqcnPmgQh1TG8U30Ni4EQVNtf0DCjxYG5yzJotG3oJxjH5IIZ0YEnki +bxZiI5ZXJPdPLAzDbonmZPtZit/340Ccf8VaF0zpG8oO7qxGqAt5m5O0S0Ee/XNUI0VjivlAOKw5 +is7jZh2AsmawhlCqiTPGrGEz3+HmveA9wRKp7PirBI2TWc5uGbDFJ8tF059w1BU5GBPwEVpczQq3 +9imlkclstDDxz4YqcxsSCeIj15Jl2yZP1Ys4eNGcvEX85IfvizIct1zPV1/ZzBk+hnyVD6uMD1kW +xSmI+gwFpjbSFNr2j4LG5Z5eak7eJW0uPszgvhs947otvBb4l5gs+bRv+GX5uh100PYZJcptf4vm +ks07I9gsi7ECDENEib+ky+ad9HRxzFEfOGhO7l8T5UZuAW9Otks38Wwsu8mc52yx7PKU+HMMTC5Q +cl3/o6hck+rF+JhTOMWchvhDFyjY0r/Ph/WaPBxzxnPGdHoanfic9Dtxs5zMeU4nJnd5lUBh74ph ++XUkBmeYEomVKV0wxedPNmR4CAgrqEbYfGV5RiZYzCqYrs1Wo7NGQcauM9XE4NXzXdWkYtFAf76r +BqVsK9qPqmmelVe3LlJz8jdRMm1pJKpnI2pK8DWztlE4FgOa1rlHE0RL2pk2uqaOVlGv05Yt5o7N +OsWSVTDb8s6aVWveckKba+aNMYuypTLVR1mssuycOYU9gZRILPWIW9bSLPG2vr0sujeBKfj8NF7a +9IGf2Jkl7uWXThq4+op/UzKWAWaxFr/kCCV9h0icuKgOn9ddgsEtnYE2vWkjPikZSy9jjMESDpt7 +NCppz25VpY7xuCABJp/amKLl56+poxaLLmT5m9+aokXqFQl9I4HlDmrh75vQZznF2FtT1ILtCpfy +460npKhFlUhFWP+VouVDa5nj95ljy9tSRMtnVNQ1HHUiomj56nV11OLQrxMv7U7S6YhrPPiaJJXW +1RFKe2Fi2cZrknTJ21K0ePM1SXr3iSlSiF2C+LJCfPFEQbzvJP7wcEwpOYdbwYwo4e8HBMqAmSRa +cDauizow/rripBS1mNmpNdSHx73a8ZgLHtN4dLx0uVRyXJ+/EJmBej6EeHNs6i1URKq59hQt342C +mcYUjSFqWQfkHr4mSe3r69g6ty+buCZJPYhPIf6FFUooOkCFEO2/GXNuiDlKaMdKCEyOMbeHmDVC +2yihxjRuSpHK60Sh9UkITJ75vDXEfEJoF0oIjMOY9wND+jukMVU7bHjUfqAqxfUSykc/l5SIEvbu +JH1/k3yAFA8vgqHmd9qa4/67Fs854eUfpHhHeS5/MkkdncITD35niqg5xoYv4Nli2eaQkcMSLC+X +zc4DZUtlK5fhj3OiKVcqlbY4lxCg9tUlalGHOrf0b93y9q63M4+6WpB0Oi0cKvW16D39HZx4wezX +r1H0hQFqM2AFt1rF9KBKINt4pJA1ZegI7grDPUI1WAymT013bFO71tSNUzVA+IyqZFKpqaxN/X2q +KE0LJFflqFkNm/r7FLVSU8n8DVbOvHb5utTV2pVKW1NMcj7rO8iWjN56LjaNiCFDm2KDak62x8aG +MeipGzdg1H2JGon+H+75Y4COYezempUNYtc8hKHBfACvLZDwmHy4CqKaH4TeJcOaRf+oAuHy8inf +tHMmbyZGT4GpqFvka1ipOfmyFBieK0EPpXR/KkMBanWeSFDbCeozwJX1e+5Kg4Thd4bNKcsPORwt +nKRJx1G4yNfIUbtgTpOwCuuUTJtGhfoxCatQ2YLjmeM5mhHATyWsAu7E6/01NCv0eySswvF3r+aE ++oKEVaiSVTKJ/kspbKmENVFrqU3Ix0lYBSt7fOtIu9A/LGEVjseGOZoR+nckrMK5huWZNCvk+yWs +DcMAg+htqiD7SlgFzZt+ycpRm9CPkbAK5+d34BqldqFvl7AKV7TsrOOaNCqA90hYBbQNDFqL5hqq +COStJypZq6CurxZds/xGoF1gtoQ14JExZ0VQMxJWoTEY7sw76XCUGiDvWyiFZ+XHd+ZoTuh1Jy0g +Nib6rlEsUZsgPihhtQycvwLPCOi09oXZuqZfdm0aFcgOCav4etNeP8blFQHcImEVUJTA4DkBrelY +SIBpr1+N99sFkpdwQb4KXhHYjyWsghcN3ldFswK4V8Iq4HjZN6doTshNmxaQNIv7FKlNyD+VsIob +ZigFn1cYaFZAD0lYDTZ9v+ARdao8N0lYhXM8NZMaFcCEhFVAXoXN5E3fGB/H6HeaKgL9goRVSUxc +o5p1+YjnjIB2Sfhq4DU0K6hbJFwA7RdoThD/lrA2EivdRF1KF4dJWAVlV+7Y5pSZpXYBvVPCKrBn ++gOOXI42KqBPSxiA6Wuku7po10utX3smQSQD0JnX0N9hvhmfhPCwK+RxoPAKso5jp/KmPzZJbQJa +I2EApne1aDmlZz02lNeb9qh18bMJqshAefY1yItEQd8cY7Vq0DY9XtfesD7k+IRwTlzUwFPSKaJe +vAq5hkfdtSRa1dHftWE9vfvJyN0R8Xww/AE53ORfRZb04f78OKBv3cYNzCA8kROn46v5TA/PgVTT +JYfwQEgcMTTUKYDw5EccMNyX7twmkPBwTRySXnf86lNZjKbaetreke5neni6Lc4At6hbZ6jJywIs +eJcB33/TXy74Vik8sr+Qat3sRI9RtArTnHF4giKecZe6qLD60vvw3FYcP9T59rJh+/J5gQWy7rP9 +Des7ClZenXRboED9ls0LoGHBFxCRC75HbpnXVt7M/22BM6+pxJnXWhh1n80eS5MzN7ll6932hvVZ +/wye7Pz8mmcf+c5T3/jYAu0kSLBubZDgL89eEB6PjtdhUV0cxYd4cJBnAb0X1QTuygdv+hz+W1fb +sotqqnbexT95fvdfr/1weJosnmUgnpyj/MhnvnXpF7770EsLNOpex3CzE2j5mOYtIJ4CdRRzwnMB +W4zASnL289UZuvNufIoXRNi5xTHVWPfES4TbM+y18BsqZV8LP4bNv0W3VjG2OI6N29L2qOV+a/5d +TrXYAfVasgXutZRD4V47x9dUv4ppad3a18J3qJTds3KGLS+rrt171Vpm2GspNjqTPecKlDos+qq5 +pkuG+xrKwLDXItx2w9tD+3C8DmsPgjleh51zHUvdcbGg/I7XZbg75Qq8V0O5Rt6xxwvquPWrAHFf +25i3x1w3l+WOuVdh1VcolIuOukrm1WB7hpy8Ryb8NootYkEf53gDRrbAmFeRZsD0X0PxcVnaa4AN +FQz7+D3lmHYKhmvtsYTbLTvn7Nwj7J2vrk0eKMwbIUl3z3ImyI/MBGQ8S60NzyWIZIVn7jUMqoOE +yQzu0zLQAWUKeGEYcmkQbuHIfqpWvsPPJaj9eDUbouP+Hyy2RfeghpyeeKviGObt1cr7b88lqLJW +Idv+n+RdlIOXAZOTpCBhtk8m9GQmnMSMTWPfWZ5aZ55PEL1erfi3v4b8g4R6QiUxvE1t1eqQ2RuF +aeKiRsxhSK8+Bwkc/orjqXzqld8I4BXsjXsGrl/LyN/vAdmx7VQFvHYPQFmqR+43LAhlL48zs6cy +8JYFgTyL2Wx4Q4Pbu4ePZ3AtAYZcq2jKRz0+9cFKZHE+UE+/MbXJLfsm36TMzWaW9IJ1BLXFxKLy +qfLeH0OIBH0pYmvVlX7ZCwlql/oZ/b+p9GyxNObkpsPExTcoy1HGRuEUd1W2WFIHKakikI9LqKAJ +okpExKAlt37rxQTNHquaQ+U1SBYkDObWaA7Zgjvg4AajIYeL7oUcJ6ShhTL8ISIDY/EuW1RLrZMv +JWhGxJ55DcIECZNFs2iezooKkt/XtoCiGFo2CjQniJckDGSMlswEMjNpuNirTIcolgdJGCSgiyOF +CvTTuu3lBM0cqRQ7+xrKEiQMs8+Vx8dxUDPksvRoxS3IN4bEJxeoTRBflgoNkPT9iITVaq//T4La +pFBzr0HUUO28ewBPrPqAxVveqPSkco9uCADUKORX8T5/7DpQ+z6pXVKcKqFKmSBK1nKlxphFrb/+ +T4JGBU4b9tyFIJF+JRvqwBizVsk2p5DdV4WtSMHveC97MkmjXSlacjoRP3yqK/Y+Fnz6bL/DxVVF +qjdv1g4Y1B6cJ4nS947Tu8fHzaxvTZoMHDbz4eJAE3UuIsL7aqImbWDrX0nQzD7KJNpegwZCA8vj +7s8g5ZxwUIUlSuYdz5g0M5F7K4NrKQPo3stUpmESz/TzmXw2S21COUbCEGGoOyXYTNqF+Pn957Hh +zcTYHLZuLc0I8VEJQ04haMN6ohWKwykSVoOw5W5UqL+XsBq19rgNNCfU4QMUz2rUcWvW0qhQb5Ww +GrVm9dr1NCfk4QMXYrZ29fqNNCrkWyWs5rZ+9fEbaE7I2w5aiNvGNcevpVEh3yFhNbc1G9ZtXE9z +Qt/2+oXYrVv7lg0baVTod0hYzW/Dccet20BzQt928Dx+epBIo0KzJVS8Iu9FMZ7MO1QR+m/fEOUV +wRleEYv/NCeAv0pYxRDnDV2fqE0xSklYhePG0CbUNRJWodSdte1C/qiEVbAajWZGsF+W8NXS0KyA +eg5RUleBi45rqhcwowJ5UsKFoRnbyfpTPtEbFdNFElalwJtQ9TKmTSBHS1gNLY8V8Clhy7GDA+Pt +gu2QsCpNeCvwqCDOlLAKGblguSKYt75JCV+Fhe+hdqEOSliFMvg251Eh5ySsgrFqsxNmdgdVBPMl +Cauw2VLZt7I7PJoRxKyEVciiWeRTyQHgtj0A162lOYEccOgCxRaeG9ZTm0BOk7Aqey4PFuFHBXGm +hFVIuTC6IvQPS1iFw2tmw7ayq2lGIFdIWAXNZ/kbXYGRzAruq4ctUK44vvNUmhXkDRLuIYeuU2lO +kP+QcA8pNp1KdLiSZpGEe0iR7qM2QR4p4R5SDG+kdkGeLOGeUhxPo4J8t4RVKcKucNtaqgio+QhV +lIXBQ6i5HWaO2gSalrAqCdcxD9VoVDATEr4KditVBHSOhLXBaRyR6sC4ekZw35ZwD/itNCvAGyV8 +tQRZo0RzgntUwj3gtxIdqbTYIuGrJNhEbQJaKeGrgbdSu6A2S/gq6HWsnVEBTki4pwRbqSLIcyQM +UtCHI7MAHESlVqexjippVdj21zCG5FRJ/MsnWcO07xcesh2XmmSwic30Fr450Jycie7dpxeWJqpH +tH9urKMZeYU9+hqkkUwo6VtFc5Xt7AxTn9GtyhQU/YD/Ktteycxa45aZO5HeWObvQqyS+SodiIuD +cNy0sWmv2a+opIcJ91UZf7pk0uoVsxfwPmtq5eMVvM5ArSXf5cMETVjMoFb280uwzYxa+eJSWo47 +UKnV4FeC++K2VGrlvcfqRVirisE1kLSf+s1zTtr/3EeSRK35LLNv+ys/eb5LK5/jnyXfHXFGJiyP +jt7nH0kiOjwub8/DSVWsdWupsX4RbVLCHxiglHSNiSZSkzUF3oiYimAPHi/b2ZVxVR3dFn8+om0M +9z40LaMhlcF+2JCv/vkdBD38tTPZ/EiSDmR2dYlE+wmK3yFHcgx2JdPsIUEc6msjb1JvnV2rIsOS +2UbRHBwfp4b6hspJ82ioSqG1S7nDejZc15gGgDavUMlQidMlouceRlWYBbNIq3GqhVo9OC/aqB7g +w9q5/nUlaGar/pGkQ44M1K6MCQxbR9+vcgkFwIEzzr99j/nnLJc2zssyTP69fyQ1U6ifmW6sydSy +eSsErTYeRRmdsq+e1/KzLk/Ipv5Rsas1G6ixrnm2Y14hikaJszu1ZnbYZC8qZHWKBtW9LNSu9IlG +ZLq0eeyfEAl3DWJ/wJC0joJZ5Oe3q2eVlGOGWeZW3Bfs0Qg3Ql2AQC7/0eRChm3Z8AFHtMkPGLX8 +KXuGvvc7F0ItYNTzGRzyT1S9RCuHQesUy7DSS77L+qpdPSitKEwXJUjytX9G6pkt8v+Wk070zxgv +HzfgMrOhmtVY2pHn+z5W//SxZODPPNr4+8dijijCZuNjEVHRQKlpxez/KlW0YlXII2IXp0vJqM8+ +lqQ3nfbfQetRHNl90v77qeRExuNJauUGR/SvN9QTcYtswrostaL/X8I/tc5jbHQBOUdSRUK7b1Lm +qFbNO2zHpiUwgYiMMUaPR4rI60u4CFiU2DathGW+3PwJSizh0hRRYnEi59LGPz8eU2GczxmPx7Rh +FflMHTSxUbHf9oTWxKpVr0ETAYumFbO2YiFaUJ2MBTXUz+tfgjRvfCJaYN8Yo7YV9FHhwoIT3fJE +kqhVOT1RJ1p30EVmaPmzjBgv27T/8t3xsoPljifQhIL6Z66hWn+Jem8/QuV40Gnr/xstTfUK9Usr +m1X8ClRZPR102homo5NHTNPSOdXNK0CCDrRsnxqa6trfptIdKqvpS/ZvFxfHva+y0oiB0SFHslRY +x86Wyqsc3mVPOP3TOnuo4nVYLUTbivZzFLl1wCiaxEevqLXHNPyya9IS7j1b08GghfbnLrBVXbtG +K9TTsMlfc8zRAfxMh9fK6RO7ocOFTqGc8GSSWpUgBDXVPquC8swll8r6hFLa3mGfG98Sobcj8YnQ +hqa6gP2BqKEN66mxqaVdBhkR9WGVNFiLXfVUklorVfqLQs6vWzHzQcW5tc/2PaLNT8HaeP3Uo80W +Pw35rkeziY/ywzBvCO7zcFP4OXV/R1xEZVHmf3gySQeddnzMZjYvnT1Z5cdG1ay0JUXCbsGppZWo +Ue2tOMSHmhgebV7apo5LhozCaoMM4TryJsvHtmR0dHVLR7dF8q6bJ/b8JFc8lQwrR+5d5WuK8CUw +HuKdqJiFgyacvBw27B2orcrbhWjZ/kZqSCTb56Nx22aXWaLGpkYFJc2paPqulT3FsnPgVUVWN/+p +b3T2OzmTQfOHaflsSJK/UNB8dsS18nnTfZUculzDsvFVJI+5y1/Igm/x68TpcR4Zy19I1vf0QVPt +82XDnWeSONHUfppKHSYGlT/zwaaP4bX8hYgSH0RPl4toBULUVDWc4cTRdhM4QFypombkjU3ViUHd +VB4fNo3cgvrDTmR15wZLl1YShBnIl0gtt2b2uAtnyHVw/V21etEQYBEV8ZchT4wd+7o4O/mLkcAp +StSDZe4BfoY+rTKiUh5yZI3m1NougyqdcufYpvI4pZ9LUuvou1XScMSlaOe3rZg5U1FabRPLo+gW +qNW0c8Fkbqw8TkvGn4t1Sipx63PwDsetWcutP+hTKm1L56K9TqWeDjnyNI25BOKoLIkOihCQ85Kl +AYWdSx0dZjt4VaaGGG3vaesJxqUntAXWYORy7rz5mCba1fiS6/htlu2vW6thPFquiszNj3HGx5lf +NKlr+m3B6LbtTFqtR1KreoL5MUSkJjg9arVl/NQKOWj/856Ht+b86SB5yFGb+uWMj9OhPIRvdU2f +VvKwmw7//5pOfv1ckkcW69ZSQ33DrIwt/l+r2rYzKTJ+fXWt/7+nvLYz6cvPJ19LKTwrrywkaja2 +ubMtHNHhTAkf2PO1STqF3KsD2PDYjtvOpJW1zM6z8tT04xdhYLa5k5bcwj+dQo72Vz9hVG3Fl7ht +79mo/v9RjvwLGKXBicoHc1vbZWVDuzWtLdr6UmTYFvq2CKBN68Uz5MONbtAsPSOjGl4Txi2EZ7xY +clzTDRqpZ2T4HNn+jNDzskgWT70Ikbki+MaZ1tk3KSemJc7zNXqtlTcrQihons5Jrmj/popt5fcu +RKVXUGH8kMfXbleHsuB2FI5aQ/vDW1Jrhlenqe2DnCbD1yzSym8SJj1FWo0boqiVzwzRxstTiPWm ++fNcXolKioVElLI0JRH+Dt8peSWqJFREyXCNIp2TUP7L8J2ilcXYp+zR+QnlnVg0vt7pAonJY3fn +hQlcByISFCx7B80krmMx0AF7lp3FVQQKgxjXNDzHpl0JfFQdHtLEHdx0ubqtoLWknvnDclfMi5tw +wf5KieWdcfj6Kq9t0vclmrXl2OP4ei/9QCLzWS+Lizltk66SKH/CdXZ6pYLl0w8lCu1zUn1xCl/w +8ehHQigZLu6iHOSv3NPVCYxTqRX3lVh5G4cbf5zY/DTX6LTHN+fynSg5ukbS4xEM6CfzItLm6fTT +gF3Z5oum0z6OCM6KyoJYvh/xNxLpTXs4hKveyt0ikcjE9MzT6a4EG7KKKBieX6K5xF2NsAwMfs1c +kR5O/KIJz/Aeu6U2+dQsRgLPJ3D1KVZ/81knZ66m8+uUkXgqYg1dqCNKWZqRp7xTytI35MGws6bn +O65Hu+p69+K8cBlkKUs/EgTUl/WnaFaeYRy4Quk3dcOvAz6bdzrxNveWun35uWCMmQWPHq5TNoqV +cJd2152xN8AymkMFP1+n7DWf7fA8y/Ox+8ajSpJNULfuPH3hpci0nq2blqyouNJSC07gQiYsGSfp +xAq96RV4hWCMIu30EKqn1vbDFJfQCwhx44q5jytKq+HmS8STJP5NTbhsiFq5Be2v2nopS22i+RKt +fOPDSaJW18w6k6Zr5v5/zL0LfFxXdS+8JI+dyaOJHJKx1IgyJoTrGAckWbZlSI3s2IkMfgjLjg2O +Kx/NnBlNNHNmfM4ZWU4wTB5Q8fZtC5hHi6BQQnn5XgIYyuWKfnzFlJabFr4f7qXt50K/D5NS6gKh +5gPK9/uvtffZ+5yZscc8eq/y+zlz9nPttdZee++111qbBvggnOELXzdPg/KpbjaH+MsArPp/98+W +0DP4CAbdQvbB4/QWStFNh5VKW521dN46SpkRuEilHiNrJ1njw8GPKMO6e9n9mE6lSo6wO4ywJPJr +titFGWrCkmSO9S2+VWEpKD1ALHFZdPoYaIqHlkEIcloqv+E/zgFlaZkkBDVFMuBQdFmZgke9r+0C +syhkKInKGFfytJCnAWGvWcevKcVzpuA7FbeWa1J9K3C/EBsgzgCzTpn+B0Y4mlwNdG66L1tQQyx4 +itEM4nSpD3TZqKsQYrhmRldLxYi3KtRI9zU+IqmZ4gARXg6gDEwoMJ/TsirkS7OVap5GZXbABKKU +p7G3/yvklgqDSuNSk72P2dKaDmxjwRGo6KuNLsQ1oww8Xue7TnKeGKAUPDrdhWj1lMnV/SItdklb +OadenIa/Np3pknWhRk8ocYTjSI3Oqq9qOW8kVSlP55Vgq/A5EqLhQpfsmVl2I+EHKkGtGthLX+za +Bu0ai7qA5rulRp7fx329+sLxsVR2px+gEyolQGgttPh73cJBLMbdPL1ZfXvu0YlSUeb9W1QaB/SF +SKW3dov8L+ElHjqp8hGVdb9TCumdSiTpWK600H0Vy1/PdfMc7bRCj6lKLMY5DNYHumWdQZvwx6ZT +3UItJNBpBTnEpIRvQkzRz6ieTCotdv8n7itfdYNxx5+hM6onrGr0RPdHOdcpl3kinO22NhSccq5b +1goM1M0X6Xy3kFHcjQUjF7r/nluRMtvmQrqQEj6TFDxP+AOVAqIfdUphhS6mpGV81flpjAI1lm65 +GnMUaahL80tlQiKBcePO0omlgpkoLZgp1ejkUtZ+Y6PiYyXh0rSgZITaA2HNpPcuFeBAIB7jY0tl +1JW7S3P1Gp1aWmEgyqUpRvyZpTddC6DUdy1HF5eKYFYp2Ekti6UUaX6Z4El1TCeWSSOzCCk2Touq +PD7H76Iz6lNx8j2uR08sEyglDOG46+XBoF9VqflytTju+jvp7LK7r01RV6aye4LOLgv5NzAXjLnl +PAy4zy4TvESJdG7ZqzAeI3Aq9NMuS9YXq5AbI32Lb1BCJagRn91Znqob4CL1yADZ5qpXqMSnDYUI +n1bJr6kaDfAv06H0sHtZymhIijgRQMNwVfq6rFLMRTJOCSiRSav6zr1KAxY6uSvcT0d7X4HTgkkQ +LX38w7IU9etFS8I8clCV9PKFA9K30l5Et9Gq0ORV0MGvj+lLBpaTrU9dpu7lbsfFm7r9VTdtt9On +rrJwUrNwMrpH+oWycv0wLU0vI6Wt1BoWdZxKL6d7pKgC0RxSLLEQpLFPebYUjEZqFQDFGt3LR+3h +Zs1wrZLb0xbveNUQd14LRWkZt5qKd0xdLvO6tDXQijXQc4r4/QfXGpWSgmbRHllW4THaXBTbXtov +7BZwVmBRkktOvnaQ+83LNtB1dYpuixRUmi0woaIPfgjDHO3rgZtv0iMVvCyW9lVT1WpZ7AlYrzE6 +K8DxDCWOmUsZNEBpkQAFj3redw3soqMJwd1Rum+hZtWFXnuCrzSJcnjeqyX2per3MCR7JHblvsWX +SrOGZhisVWT0mpTCvhkMpZ/WUOr+FVhFbRRbpV55jcUuSoDC6uT10iVGy+SmjKdlDQJNk9rH+dHJ +2B9Sczjj+lrYWAAr4f1pu7tIalIfNQ0xyrz9WgtCyFXqy75MoDPtI/2+azHZBweie9tpyNxqbgZR +meh716ZooWv5wl1SlWfjNYaEsbI9fVl9LwlMb87nIxWGj9uK9HNwwDS9xyp//NoULo8xh0aod8Xo +86VDLJL6upjvnK3bPPu6eOXqg4c0J+hzGw+gcl2KMg1li2LdTscK9a6gDdLfj1FcXU9/5S28YLre +pa6nY+2s6ls4Ku1karmASE59RegYevnsJgfMpNSOtfHJ6yJEYCXpXdFQqID40KCxhrUtKjQignq+ +WqTR63FOeZ7AFU0XyTvQt/h2ycgU9Z4bmxtKywG25ruz1CO/2bxBrZFOjq8ycYClrIzMd8uuE0jS +KknCRsUNaUAkACJRlt1cSMOYWJQJ6jkEpKZ18llzfNcLaUQ6w7aItzWj5jt0SmUak+8cjU9cD2li +1i4ebPXX7OPGNMxsaOUNKcpkf0vGGSFA8g73jb5FMjJHchzRn996Iz4NHglKD1Aab79RBruJHtkg +ABFsw9IrNiwcCSZP/TJM5PKxMisWCYHr5edolbSCENVzNGA+jtBIUZ3+vfwRGpMPTB86cAeLy2iA +AnDtemteA0tHqKdvEc6TsIAqIGwukeAIehNFSNOKVBm+IWUtqSCjTzf2gE8G0I5lfiF5Y33Z/ywZ +mVqNSDQxR6cxNYTUKtJ/j3wVqHd/DyaP4xcpK1oBqHVkC8UcxpVHpLjS0smxzoAqXb/1Bi2l7UP/ +Gl6YaqGPDcg5LbHR0wrME7UotqlxW489egkhzKd/xBbOjD5HRhpxilXgZKpvYUGyM7XcmOvkoabd ++DSMFcui51RcWEKkRf2Uq+MjO3YT55fKnDciebUc8sb1R9mTANrTklBAZqPrUVXTy6N1pJ1U2s5K +yYPeSn85c1Bcqa8QV2Sn1IfLX6fVl1fVRpCLKsU1SWdUEuPiCfUhmDmrvqaCgM6p3y4+zqsPbgUJ +F1SCtIuUi1FKnhpKU6ZMJeejTxTUSjpMn4BOqjw2ywkiFR2GE7g5vmx/rDtzM1CLCpAWOCl+mlNw +GykpZ7uf4pQasHeheySDCrVyvVjy2KJnfokctmszRVi7oOMlX+JCQneQlE6pQpKkyi2qctNOUHFK +Hp1dgtVKm4FWnGCGzi15YgX6K2J8nHJBpQDkilOjRupbXGLKydO80guxBD6RmsMUMtNB+mbavD3G +wBEjDt2EtW6N8GfEvlH2aN/o2yQvU+F46XwaoEzNyQ9SSmCvOfkhUkfTSsnbcRctUxmhzz5uV8mn +B37U66IH1g6oR8QacoC03YVC4Ib6CiBXV9+y+GVQJXSmVKI6ZfGcUElKVtRyZTx+hpZGeGYbhEQD +e+nToiVz7RD1rljcKMOEQNYrptwvtV7MrW0BgAej9K4YVTuCrptTUSP3/ROI2bqRCN+6iZ6+RkXg +yLhe6OsjBOMHWh45g5rh6HpfvIlHoxdxze/YpKhxvdcC6c5OQDJt9J17uYJplm97eVfBO4liOK1A +ykw5gcu5sl01IEbt3Hczwyi3Sb0rRtdLo1BUaYTzfV0bXN168FC0ScFMRRPD0sR/f8pYxT16R3t0 +WzTDzN7GGO5d0VDGdY9nDNVu6QRFppF037kjAkvMvK6VdZ2p9MoMI0QTTeYq5ASGpmCaWWFgyi20 +H1rESVYrq/oWHlBASSpmmDZDg6DDYsn99Yg4U5BwUvYRiBhDRmmBs562AkuhMp9DzUxjUPqJoJgq +hbNuLqz61NM3WpbMjKdV2rAHZZmU5u256cRUe9uKFGUrTu2ggkkbdhseED38+59K0c2N/dLDjU8u +Yb08NvYRrwjg3N22XuxW1J/NT1aZ3hUNxVUQpZoxP/C8S6C+g+tlP6f3HuaonA9i1hUt7/kDP0cy +5zP5IKQe3sN3ctHfSY9v7E11cscfHqtlI2wy1s0YuJuY/ZvJ0xYltdBKxB5Yo6Ld3T5mjbIlxfjV +DhriT10NoxG1JnSCi1/lAF7ch7mgmVReVMY7ypnR3xE+i6aE5M139Y0imgA23rxk05t+HazFh6a0 +/C6XQOpn9yOdbw/5VKzGrh44UStixfHqThmxf3ag0qo3cyXod0te3p3TC6KHA0agFkPovKs5jgNN +o6JZ55sGfkZ27GPcQjFXcfwZThmXFLapK7oeHcAKSbhW2Vkv033yxQ2IY8FhOeDATo7fTqGpfT/A +XQs27C7lVjwdw8IFAIc+yMvWgOGDE8A0r9eZcqlSCklf48NElnuYoTk+4GRUUoCLfDRoRIig+UO/ +HiMLHuZlBL2iP0WZc0rvZEgT5ff0kVYsqWORIokThIo8VldRtXv6LZ1eMQfZinM4nI3UFePKiEeK +gui/AiQLh8AIlnWgyryqK72olJxxICdCJ3S3VOeoqy+rNY0B0XpgOAFYVPQ7/TYuFObod5+eoszi +HdJ/1InO7enL6jVfuJSxjDmITVVaSMw+TdcwAU3fuoWXPN06c+LSkpnJEnJTpbCmra0DvPuptX6O +73oOXSeMVQbiexLLRNTcY0+38M71tufnYKIowyIL7aizmVv+/m9Yq0A0cpN/4uSjXX26AYBZcWpE +17DfAbg6QOj4rnX8XXOK7nYPN1CNxqNduShtp+PPBNSYt9MmhI0DapzQyey5i5kWUOPko12/zfUx +L9z8FjzkPX/yUTkHGfwaOF/8G1DCDQ1s3DC4ThTGYhrfaDS64raGSKH+gyODG4cORXwoEwXs3Wik +EuVHYfk8MGQb3DfSiTJpC7lmHDudGv3tMywEGxU7vK55rCgDrmk0RhNtNkbNQOPlP/SMGBOzslfc +RNg5RJPLTDPWx+HARA8+aRlSmQLSBpd45T+3LAFFEOd/N95ChELRrkCnZAEQ5QZ1aNKgVolyO9gq +5Mxay81nj0Moqb+MsQDMURodd7QAtmjzU1kbn6Iah2jNLCpNhkGUaH+gZMpkXQEkmjVFlqrpvgV9 +qz/tOnkiviAzpJRSD620u5S096+Emq0krZouozsxNJRZyCfzc7W6slKm33+mxWwRXFaBC5vMZGYd +GY+TMlWP0qJFLFeL1PPxZ2Jd4ktp6n3+s/Dh1Svb5kLfobOb9E1iEErKuU2ycJarQbiZrcbovCSZ +UVsw9D3THjqu4WG9/OpbW4Guc+ctIeQTveNWgHSU0vJDP3FMPQKI/mZ7KbHoy7jVAmVFjE7nfV5e +1ZmZd8MDn+YmQ6cY0NgP+be/y50L6fBfYzmJmgSo01IWrECNLr46NwPVEH/1mZY0RqLCjGXJfutB +1tavH8bJVOmoMQC9z/5yA6NsfVhG3dhWE6d21QaUrLqNrz+F7UbrNvoPDg4MDDAMWKCx2VjcFBdC +569CKS6yfpiBS0i+rk6mcS0ndrGxzS+btcq5iS/AhK3Jnta1HKV5k89Xyj3qEHb5E0YH/TWeZTPh +JFQHdNuzW7Gg5Fnm3jFdBE6R2IyncXIH1osBXSe/83gLWwXlV07StVxQo5uEC2s5aHCoV3+VPa0A +92rKDVqyIsWPshAHrNu30qrfZtLCsJVufw//nqTVeNqcMqxcYr5+DtYWiz1lNPtvw2opdJV1MrFI +dhnpwd3Bqd02+0Ii0fNvS1EXz58apXFCtDpCCa62/9kdHayK1rmqaM5NML1w/VqrE2MpqDlhblof +oEwdCDY3X8myL3UyeducOmiuHWpnxV+sURpXxLgfK3PvSo+T0X3q84fqibJ8j6w+YfmyirVlnZzH +/teM+73Pttk/unj/XBor0KRMRbMCRaE39vtODW8k0plVUJQqw4FooWku19N3Tis8ItyxoBHclrzi +PQrXRoI2t/Lwqo5YqDojJG+3S6jOUJoJ1QldrMa+tYpVUyKrIKvVH2t7eGtFxLKprZxdryujyuG4 +iF1mIRqeUJuDwPXDUtVjFzZ62WqQ5D7pMsJ0i4Kr+g4/KqUyk9G1jTbNzFW9nO/CKEPu1BzuxM1T +j3xXSkFQ8oo7lYstwLQo0qK379yeomfpk77r+1VfBbfr6T38UgUG4jc03TmbMXAtnObdIFCVe/uy +2qinEhS1dq6lJrVF7VOrLZhqZaekUNjTe05d3LcBiRdCXuREGObjFMobmKeqdS8vzoXU23dO+//O +Eb+HSZljWqMPMyKgVzZVObj+Lb8qfntvt5V/DgTykA3ETXEgbqJbCuWqA/18KvVrjTsFx5K2fphS +6esXdRqeQyy7eMh6SfqG0UQqYsQt6ekhlRzZ18jIs/FOs+zTqRgfTHEisU5cDfXiiELdW9dY62d/ +lIwVKL38sG38hOPSQX2ggk90dCgwQgdq7b14EhfcR5vusNqOpkG8zIms2UTwBk9LGr6WVppDrN4l +3LW8FLpxI3XiTX1/DegBl7foBIMCrFxvJJDEjm/NxaIhXeKcs8ecb9B8NNy4BIvcy/ZQ+str+M6+ +/R7oEk2++w5b6GvHdvaiJ+VXbtBf93LVSqUqZ76/fm4r7MeK9Bjc12aKNbj6k7jVVwT/KdGTzMnX +Mvmq8N5J9jmTSs1haBLr4MhzWQprmWOiAai/Y88zlwM/fbL9zjliHtWCBTg2cxHUULxKMIBMqeBR ++g2s9YaJ83X804CpGhp4nrXjx0WPCr2i4KOVhkksFTu2TRGrSAyaGC/hel63sAJ3O+pmPlHoL55n +07ZcEJtIPHYeNR65oepc6yhi6F4ueBBWbxtIUaahuCJCmcrs6TtcFZiUQuztbEBeqwfTOS9UM83g +R9XKDdggsvkfzk6ZUXXfamBg5dVYyQtpzWArxjP5vYbreJVQMz5fPerpAy2r7Hq+OMDzRnOPaeHJ +GFSF0hzrVulCy46j7DHTry1pRE2a3jwE9oPpRg/OYtDgBiH1znBybrruzWiVtSdfq2QGlLx64Go1 +NVTENPI5rvOA61dplNcSg9YImNygNvCIHwmbDCFhaqr+xPyRoRPzx8tVXTZkU68CrS59ZKgVdSQv +bTDEFCAevYFeSs3HWsUbYm4Fulr6pt10xLlWgRjzRpNh2sUGFSrT1bZi07BWBSXoJ+taAs55819d +akDHBl/rSKDeDCj9pfUgbnDUdfkG4LGHlUk7p7AvvpunD8VSXT+gUzrFKZdFcXr64dSdL0Bbk/SE +zuQ+RId69uGUmlZO0Q0mjrq1kM41p21xglJA53UGQ4Hg1DtKs67kXYjljaO1cdeHRxRdfDj10+cA +hCDnzLpe0b2n6pSp8Yjq2XdzZadUERf8+UTqXb6bL4V04pEUFAc4+rqeE9DJR1INHhWQyTpmDhBA +C4+kqneiq2jqBbT4SIrnpaTRmahIucw1Azr/SGqea/G4VGLjUZUIBapKO6nTcnWfk+jUo6kvc9VJ +WnxU0SiH1d8p05lHU8/+TQADSsiEP/PEUsx5nEty066knfgrlab0+IWSx287+5J9NpENLU+prOou +/HW87h7XyU1jOyPoOJ/IZkY79ZWlUDsqfEAASuHTX1Ft1SEg8tT46tI7XxiTZ8K6m9da1wxgG6m+ +8HDKcHRQr1Qc2FJ8g/mY5U9Ac/duYHS4jp+bxp6cLjZS3xrhtNDxQ2o8lPoJf7penub1R4m1/Sce +ShU3ctGcM0unH0rd9Hx8MUw7qrkZmn84dc/KFCwWjwU8u088nBLBFrpBSCcfFrMdIxwM7Des543Y +IWOcWmPkI6LEDRtSNBffsC6FYkwLeVPSRE968w+MdcSf1gBTO92YXBDI/+ItbnVCh960IdXyrmBl ++yqf3GBJHXUD0aLlNSNoOXkbMGpWYRkXQ9Gw9KKSzDdLVBmBZgb0wKzO0+iHQDwbweA4buLbGywn +B6sNCJ8EEOlkA9zZ1pFYA0WXUy9XXRV71YjFstVCgXnPWjsctaSbnnWhPx2xtlvMxonrrmjXgp3B +HscrugFZ7vO+pNArmXXDauiUIRMDrdvRvJplTjX9W631bUzZ/BblWPz2041mVzo7257fmoG1N6aw +JyKS+Si3wWn+aAEV/dnGVm4KPFMjP4UI0sCox4quxNCwgxf4buD6s/yuMOOmWdfmu2UYLYubQ0y/ +C+bbsX9n1Keim+kQF/NjzQWyx+0wbAw3kcgX3LirrVISMr1r0vDo630FBY0yvtgYYGz/ThrjTxBv +tZ6BcgXYu2JxWLZIuAzU2p0bZ9pTDrG2Vh8cHtw4vHZg2DQXLX/0iRekmrXmKy9Z5R9eEJMUl2gc +25xGo9ETF4ONRqPbsFS55LmOL2uBdUSWbZks3hWnVnOjUDCQ8WK5hoydbqXqH1MRuQzH2a323Rmf +CNgKYDYauftr2I4pXdm2ifbYbHawAe83853rRSwXPyyjuJIZPA459oNKa1kLV8+F2QcjpqyoHUFT +gmHTmpPPHtS3v9njNP+bKWr8RUIPMt/VAvKfo/FFS57r6kQv3ASE1Zw8PdbVzwt/JC6iQo9ZNbGr +EXMT2se3VzXHD0tOmdL7uKVCvVymA/zb0DNq6anfxJI7FC1NaG3CDen/2ZSi8TifWUymS1kSNqiV +PJc3AOqWkROUh7h87HCjMDmceZdT0/JXbHay/x8gNlDqbl6xyZL+066T3+uUyrJRtU4G/RHhWAvV +G4e+1zoXJHd49PkXWlMwQndTMeteSOURW5OwB1OPuJp7vhvqURVwqlWODdWQVn2DyRmdS1Ub0UaT +8i+0j101FT7za99peUfvVcPtHjb/NDoKx759Iski6E0+9TWUStT0bXKfOWp3Wsk5uWmXbt6cosxC +PdGkyjx1dR+dkCyWKxNOpVaO5iHiboj4kdmYCUveMS1h8FuCTmocIYWLB/qYzLsVWnX7ZswDVm0I +TOfT+zipUK4H03CXfexqPkmbQSn46qPgaYiAhLyHlF9ML1/cKsCze9Y87EOGDf/DvRSLFZuifXZz +ikaXZ8es8tYOJl62py8bSDkxYyOxSGONgWDCABqv+ehmq9EoCJyt7mB7I/oqVMoZxIJWbiSeumo0 +DZvaf7PZJmuNgm2g6KRAGDFJjU7s6lvEU8q4jC3ltZ0q9BH1gNRJis//aXFj4bgzcFVSfWPrhHs7 +TrqJCYIkKBNLuRnq/fY2kLFCI7huoowi0ej+USTXhLJjW+/GF+KDIEpBLpyjw7JU8bVqrVot0/RB +LhMlwMPoXNeXOBEuY9LSuWuxFSWTgtXtvEr0694RSA+6cK3AiQR2k/qBlUAXr33WPQxO3eNV8+It +n+foNkUYG1Kjfw/nBrBxkU5P9P+BSQJcj/V/gxN4lyFlHnvW9WNolF3UUebUmreZhAkoGOj0Glb6 +SBnRASyuETxwtT1yQnfzdEYlyw6enljzA25rks6tkZHBvtkf2A+Xo/NrBCWctLOa5zjh2xy/XMJx +7ILK1bFT2Iri4hrxNCrm7vYRIqfqOeWdjj/DmY07dCaSrGiH83f8HjOoHQZxAudJrnYiqnaUTt7x +8u1AhkScW7wDwewI1IiC+9zt0RM7ZShgxWDCPUJfVQk8kICXmrM75QgtSXRup/8itOvVK+jTD+ii +qpN3y27o5lXqT1QqV9vj5Ny7wjlq7BJUK398mt/1GQT5NnOrRvdssSZqcCxi8mzfwqsxgXDozU27 +Mj3Yn4JkvrD7lpq0KjoAzw5lKaCSuJR4PpperW4e3mZ1j1OzWLL29h1+UPVu74eE79JCfOyMlcGM +ado00XU3JCYO3pHEnOSZRq++G8fuhe3SAUtMPnYny/WuaKhIhR9AOBy1+XvoYZCj3aF73aGDa4ea +WvrR3SlqXLN8NN5lf6ui6KrRvfxw3EW8/+DQuvWRQI8ZVDV6li/cbQ0FG2exhrO2iWyfZbaEnsQL +5I1ndloqZ7gMsRFXF5yW2QgkdsJg+YA98TqpwtZwCi3vyF4KLYNDIwYp0gwqN9LLD9sLV6PVLrTs +MrgG+ql6IRtvkYUStrbpvgU9HFCIH/sTz810BeIE22gbFK7IaygsIhOgRFpbllRb6gXEob0qfd3C +Hhm+WXFcHzGaXE+pu3r6RrX/iHBvF68MfFXEE9Li10TVT41Z86GYgyiiVX3Z10iPmaNT9cIg0Qkl +auqFIUrLB7wwAkgvN68mBSaIxw3I/JNNhpvnpGxCEKiulm+3V9ijVX8Gcv3b23GFPyUwRKPWuY2e +vsOh5GVU2jT8vte9CCqc6tT91LsTIswMWpWij263BqsSUbW3j9Q2LcN3OXyZg99T9+vg4NGG06r2 +79sx44fWrT0EH3JtEXb+huWj9vz4aVeMp1lewkVH8TQicWmeHnx3e56+xKWoPpBjm9R88OOnJZqT +lbbG3KeaqwLD+bWsuSmIdrvZtkEVH3DVLk29Z6GOxKor6mXFUKZG2V/fwcpY77IhTP/jhvSJF4ET +I5EcjZbnUnQlaO5FkhMJu4fMYtPNnH3ohzq/RUvTsBB/wU6w/MPC1hHLc9ZAX+N3JT0jV8FKysCQ +LaA0rAUos4Wulh+etuWka+TGGE5XA+qmWD+HoK/ZquW8RBQOtJu95846uboTag8ZZdO6audOpphm +NQbsPTuAMp1UcWpi//o5DIWUIXA0lCi7t++cjnYfgUpfQ/MAJ0pKSxJ2j7tnXb9Qrh6lno27YlBE +bVZ2AhLLxW8KOP0hABldKbizHbY4t3cFDUsWWtVT8NwlLsIN8bmB47twGshJG9E4OSvdt6BsnTNh +tQYKEBt7WDKJy+3bBbgtDLLB+HX2YRVhanQB0GZrENKK3ThSSccqSLilWNClLM3VFPEgKVPS5jYz +mgVcRXkj4XT9r8WBm6g5Hu4iXDiQwA8ns6ACHPQfHLbtcFbFIVvViejKWYJGjje8V9BDjFuuLxln +LvAuKz1atnpkN3BukC57PJylornZgay1jOKlhUvAi7Y7MqFr1egzxwGu5oBp16mBCMFWtxw6lNrT +SueSKLRwtcEjTFRKIezYlP1VpP5VO+uSqEN6nuIgdSVQHI/Y9epvLOZb6oWAVAyOkjce+hM5p+z4 +fIsRxeHwRCMhXl3Ks7rs+EV1xSae1VaKFJR3cjJBBTeauHGR5LHsHl4ZUZ9PkOfVHWqUIuUuqGSu +j4KSfDEl9Sfp5NV3oSXD7AlcnR3Hkr5eDLT0ir6QcHDZSv0HheFV7CiKczwsVzTBIiUhVJKGwXQ2 +39LiGOb69iUOJ0M9Q3J+47h6Ja+owjga8Lmgqv/IHmt3YzLydsOy/+QNaFerVvL0t3tsfjPN+PR/ +2MxmZGGku2aNtZ6x0Qhx5cyc0Wu4EHvHyHuLYxSkZfESFaZsGwx4URPuhA1boeQxXmj93lazIMo+ +9wzTs6MC9tFnWaWO1YXS8jvnhfosOUk38WEEykfq3bGXxY0mWdTu30+AVwYHBiP5bLSQ/2Nvis48 +I84WrkGKKWhpfxHSUvSfMISRsKCiChWEAJqQesX+tBpSNqEKNY0+sLcj81+vejSL8Mzrh9sJL9aT +3b+Px395cRtr7//dC1qVPNhaQnhk6FnCG4ZzsBUeKwVhteg7FfrWvlZUjJeZv8OQkvdCAVH3vRAN +dS+vNgmNO/i4btgn3sR794FqG5TlqPLRaNwRJ9UTSztZAxAS2/HDLJvNt0OhKnQFhtSqRtTss+8F +JiOJUqPnbrEwZdCpjlRQC2UO4y0/fkZgLUImrB9mGiTU+fxIgZWdWLg7wgEbO0AlhBN+e0aKiql1 +hjpgp6hO1PTq/R3xdRsPgZrlMIFDjCWQjGOccWWo1agHmO4I1g67/Ox+kFJLkmJuSxEnaKyprr8L +VoT/ciBFmcVk/NsWBbN9h+dAYD6m5l19ZC3WdHzJCvWytpgiwdWilQ8csJaM4kvqbj12i86KXvai +owzreNOsxbWalCo3vDQ2LEnc8tKWXMorijyQsgKmJTII6mS+lVhatZtoJUr3vKxDSRW19BaGvOSF +fKbKjKoA9R1sANtQvJizXi+QGdnOWcuwWjF3lHq+DDnWyby4op5vfdkvMmVwDwXujGbgZeaMLq/2 +j7/AaHRLUc//+jKbx/jyZ/fU/Qjutv4+i9GifYddwjoJVQsFvamCbkHfzviU3n0f1pGyWwip59MH +8dsvFadD6uUPw/N2w393sA1Qe9xc1c/TVy4HmipnoZUBlN2HACi/J2uheklUvlU0q56Eb7sFnGp6 +7r6OGCBAaIBsJJq4nYmcOvEZhdAVsDe3SOlvHgIqr4zDuerPC8z37muiSTQQ+pPfassrptAT3dYu +g69yKT2JUTBWqNEtD1Hkqp4y9pl16YQKDw315cnuC1wcV6Bwf11Q3zlkPqY+WLae6v7hYTTMwvW0 ++vCqU/cjrJh4DfvVakhnulsxoYH46t+y5HgNkdTr6mKjYY0Ftpzw5KBnAjzD0bEKX/st7I+GDh0c +OaTXKTt/mxfS/skUrgy05OYLjW7Y57Wt8RXUSFiApM1cTXbQY/Af4siHBxCU3Q2mh3J/QLx48Rxt +PRSA+prJlLVDYvpBnoEqBw63ZQVdpGEBYtcdg7r58cNa3Zz9KxDRAGEXpZsOW6Sxc9CIJZgsvbLS +ZPPhpJdZqXXbaME/zORSIRz0ifXcDXHq/KQriQWRncBD3mmLB1OoCRNRFoCgzzsRLr7p8BqseYdH +HBWmfieJjSgPDXWAD54wCXzE2mg4wMj6tREzWiDwWpFEzguTXiRBOMOPFzS5A2QRj9f2CXj9VIpW +wCfADtprCdFLtnPjVJw3pTC9MNeaHirbejgXB8Y3PhvyoxY53OaqXshxGnv54idT9rVJCF7DkLSC +jiyesd9r4Kc1lP7F8Yu4zhqT8o5fhDp1fF8uQVoF0odiAzFRuBCIMLOonpCzoqdZBFGLlDF74xVT +XfHpFblVNFxx/GmmCLurioOGyvw/czaai7m7ql7oV+GmPIE1j37mtsJ3c7lzlr1YMTfu+rCE07Ye +k3ojAX3SzpJXqtQr+t44lAcJ98AqlHrEfUClUS+fVbnWPfAkyMo34tbALInTVkkaWoaPAg2Yb6wA +NGK+cZqA8bKk6D0UjeEATpmpIsorN4RxSXP4tRcc4OiApOTdfCnnhG4ejXHGYckoNNsxTEtOKV/m +mDVcuiZpUNoYy4U5SYy1DSnr+sEuvOaQp4Z6t0IAQp72uZhXL/NIDpvVjrs+StAJlWVA2xeWyqUH +gGqPkXeySxA+SQtdOwsx/m0mcX8+RbexK7DnlPm5VrYO2FHy3HEnT9a725NEZWmtffHfKLAg0l6b +0HAlzMNHOzloaA1ti4gQOb8UZlcXylVHjt7WNtKcaHV9UpEhUIl6DhUZF5dX6ujqHXT/+gJmmgKH +gPcMqQOV0VLAYII1uwS/oOj4ZwoYU2s2Yo5KdHAmsw/4jtI0m71rpeSxL4++C213LqtROgR2MrqC +0r91cpq5Ygg+WdSuaHrFjMZ/ezZKy03XPVjrr+EVKOaWBnv5FU8rpWJL0JW09Z+nQTVdQ/dE7Lyi +/wx1xFqK/SDgdhFRx/LVZPGO6YndBbYvUSHcZRlFVqzcvlKKEjbZ3Va3rDVgPXeerp5oJbCVIWiw +PcQbdWYDWePYePQb92OdDCjNocis/YMKq8XV/rIEVES4UHkcvSwag0GFypaqqfstoEwZSMEtpTCg +x2JAX37aBVkDByvt26k9AkrfCGP7TrizVaMP3x8bM/fF9wPRiDuAVtl7labKLnNoO2CtcsrbtBOw +rVq69YGZjk610+YQWcH6aYSBBwERiQKTDnNptgQ3VhU1Rx70tQq1II6lyDeyd5rSz1oLxpPulCCx +zM1797G5eUDZjon4HzimMzPgDS0azA0wIWJgZlE9qWW4HTjlFZNgjJpZVJG5OmCg9oMKLBKCFO2k +doTqgHpgo9wRZ11Rt0cqNjbEboL+u2fN+0j5pDItW3c+zNGdHpjBKZf5My2f8Jvv+RjzAVsK9Mqu +m7U/WfntBeGMir5lRJfq5GWeDZYSSsqzkoarrcBLFLK2DSpHm+RPUQ/DaDpVBXT7P2rVeeS5SV+8 +VPemWHsA8nibUMUE8bVDKC1PGJMpoEyDc1UbJxIWU5xcx2utEGKXaDQs50+ocLTqQWj2rSpIOAWZ +3lOCP6TBjd3KNTXe/K1bN2SeOZF8jt154WdL4if0C/wqYntXSHhPRhLZnnG4JIdv55uvUJnOnve4 +Is8ipljcyQdGhJH8a3JD4zAsyezs8XYyP+qI0j8+0uGmM6rzSwTumzXwRJOl6s81VnqJb7PRL6lR +a9cCqLT/IuNb+y/KuVeLZEWbNXpLG9sZfmskFXuaon2l/+YDNTpftni8a4FXbEuuU86ocJKN8vsP +rmNDSH7OpPkuUTcfWdZNlEs515776tgvxqvQP6RFBZpzak1mmIlWZoOO9gPR3npvdULcutnNVd6X +Knnhmoito5K3ZyWj5IW3WzsAqza3BTfcZFvcIOo1t9puFYu65Ubhd86NUvpsHWKnfbfU82I4y3ay +nWrTx/8SJJwN9PGHkaV5JALRxr6teANjyNEHd378j9L6dNDOiXqsU67RmkxNXXK//A/7JouiSTew +pl0rS2ft+RUNjnB2i/462CW1Ol22E7v6FNsJR7Rp962zMTy1IE5TUhPCMMQmXDVVazFD6I6jQBpc +FrR1NV7LjKSNWQMrbkWpEZ85Z0tl3YvJt53/4ZMz6xK9cw4Tq1AP675L2dP4Miu6qfqto5biOkq+ +61iu7Npaa16OA71jwO4BR87zCCnKWZMwfYrOAchXKbLpa9GzdLFxDtsJo9NOQIBBJKLk2Y8stygd +4bEjtovHujFicMpszmU32k6u1bQ9k9lRXv4MXLuifq8/1tESUMvpxdIMI6g1p8Vu2bPHbSKb8x3i +rQp1g5o+3BVr1MvBMDuaeT8nNF8/hsmhORznIu1wHBHWTBCVvYVDptz7YKs5EityMmvUJyoUFXtE +dGVqVfZ61y5XNccLqLf2IO/qEsBIb9c+CL6NhYXjUxxrGFrGhdPN4H7UuCjDqfkyIxOl4tNffrnh +STkrYgbroYhDf1kTEIObcGUYUuXJB4HzCEC7AH3rAatbg3lcEZcCOOpAm4/7hoD+viWErUpeuMXQ +AVfiAdHPXg5pVXQ9OneLmIZ61XHoxej8LezGZyRIqxZf/3KQw4gR6EQYKmNOfO6W+MHEliO1arm8 +1Q1y9PRXpCizkPSWjrLPd/UdfqcsbOL3SgPHATczYJrhxKPhPTJ18JgdbFt7+VSXcWfxLmWffNQD +19dRfn2856YOxH5RHYczfkgDN+LZqoyf19cMR1GyIbHvM0eLNK9/hnSiSwofzdNZdeMQuOUCneti +EA36osGMH7ckv4tkdxbPBl5niMMJgbYr4TDCKcR2tJqzKq59BYgAtzDN61Zm+RUpalwXJ0GjC25k +8jiu+OUnNKU9RvMQlioujsP2YSKcnQzcnLb4DmcnPXyKwbcZcFTzq68An2vYouTrX2mxeAerBrY5 +cmkQD5ASYpe0Fr6A1rHfiFRUozRHA5Yg1m+6uASIvPxygZqX7nDrK4F5jlOEPYXgspG4lGmMWhNY +PU+M59szo+q5i5WrV0tt2N5YQung0PqR4bXrhtcJpXAXP9ewcNbfogBmQKPRgDOg+mNrCiQx0Yfi +luhN5XCvD2CUHWumMSTN4CwmT2UBfYnIqB0ZdsasJa2lMlqw2i71NUqz3WQm0LfSHa2E9ia0TX9L +HupoiQ9KDwiPmWbylsZWvHLbgc8PcAJplMlTz0Nwvu0E/E77/OxDmFzR7BJY2AnX5iTju6oKwIU4 +KvAMDuVLNz6SopsWlJckPVsSP/ewJa1KBScXM+jEO3MckpVESomRuZEBUuEfHk7RM/T8l1i2eKwT +3WWVu5OpEuXf9khHxKlFDGSog/1XLDa/yYptw0wyLnW2ugXXNxtQwVT2eGtNfHynBpMNMjs13EtI +c5SFP3RHFP/VD+Rghxj1FfBJXBh0xbDYjvOjdijNSAB+ejrfyV45FB9+NDYXEAAfNl3wgYh43bpX +nKw5XilHj/zMCuXSwToUZIVF2w07oPS2Cx0uMVZbf/Gqjtg9hnlDkHbMU/KwbrQ7bhiLYWPwE+j4 +P51IqZ8Xmhe9uqPBVoKidV0j9DJj/rk6N9Xz1ZzvBNNZ7W1hXa6Y3QMgoDRYxGJfYEvO18CW2kGq +5mgVWwx1gjy0bU0x5kYD3i95dBt+25LjQT2ouV7+HmzVY/I8ekE7j4dz1IM0QSintKt5C21J6lgr +H5jH5Ds4ODA8sm6DCfVQ8uCxnL/LKZdp/Wtiu5dLFT3/GoRgbFyT2Lg2rCCMdss3mb2zeoSbul6L +EwJkwPat1C1PdUySet8WUUJJWUIDPv16CIpPkhiBqpbGc6S3jHoJs3v2XpOiW0peOLielnanGr+p +t0uDtpnO9fFhXG9vCFlp/YbXWpiJFE2SZ+1qlUJbVpvLKLSl8vhrQRUNuAiuCX6uin7cus/QL3m6 +iHWBwE8JJDu2eMGu9vnXaj2fkj+3szpkxFbmYaO6AhtepfNMlLz5dR0JCDzct8/DMTCrp7EFs5nG +ptwV+E2ZSlHjBzoDq83c5TAcwUypht1ku+XDyOSoeHRZcfkzS4c9/8vrwBbWtlBCmiBkUrRSys1Q +xDvFmL6cl3HRVtslvvZ6tGtSCOWiFjtYXKvl/KxTZnaxT3mee9Qkt9NQS11tzS1VVJyCTsSxVL9s +17e/IRb3Du/7XSuT3nbTrSl+5nfmMwuHpYS1+ajQP3ZZcz6B7FoM2Xz6SSC7djttfWNHcwRK4Yp1 +XKm0wx8XpPTqN7L67fK8xuUTDb8cMGlxkytWw2nfdfLseUeWICtGXmDlgNJfbrCo9lQIASNVEg30 +vImDXEbsVbPsfIEjLR8fh+dTG/l4qwauthOvmxtDYWi/dAOiLbiMgJUGPvwma131HS9frWzz6hVb +DJW0Ioc9TLVhb60aaFYtebmITzV4VlNdJzCp4hm7/bzr0+dPWCwUoV2qSglL8yq98zgR+KPmQ7ND +6TG8z29Qblf+7RYd8+B2xPq9PKdMFifNRgcb55ZCerI4GT1F9HO0+bMTHU0HnA1lV2z2Wk3wWaxq +FpGgpqPRANLOTzId9fi6/9wh9IMsoeKBfmtDrRLXNic2DbSllVlQG9QjDWpDWq0a1NbqXS/Gn4Vo +70iwBr9kmLf/Tkeoqqpl25C5Ws63Vgo1ocWaNob+0etVWGm0My/jovP7mV8EqPB3Oxo3Xz/A2Au7 +jMQiWoxPw9ZsjmibqK83xZ57tHgl7K7rX77/8u91NKKCZ60y2KLPOhZV4WfS9mKvhRqtzRbJnKDb +vUrHb9C8CksVulSLFev3ekW/V6ypWYEDovjGdLLx+I8f3qY32ytKsTpVL9B/WYaAtav0RkUvN46X +c4Ow6vNLgj++Dt7kt0kZ82YXNNPYzy8si59z8p3s+GLkaLc1iTbFneAz2eItb4WWXvlNaFiTjwx3 +5DoR0ym3gzXSV3cCa7LFL7y1oxlRRBzUkle8pAO7LkSd+7HrKrrd8ZNA3eDQRrPpkj39fOIV0odw +cdA6xiMCt0ZHAHOVWfJK4V5swd70tlabmCjbklG4szTWk3m3FtlCeAVPG0GYrUzUxI63WXs0OVWF +TmgLeW28wRoOhNyVyyyxsQh0PEKxtRAXK9ONafB9b7OnFUxa1ZOz77jVGqJBQX6qeK/j00Nvt3Kj +XZzKtIbvORVXv8g365QRWqHyY1Y06smq6ux8e0dM5FuRBfyj/OhyO572j1J69Ts6PBe0bvfNb7eR +o/qjr7yj1dh1rqWS8/lSWoUxx5kCkWTTHJFCfY47QaAtCY5y6V65GT7ql+DDkbULSxyn6CkDJ+/6 ++51SSBKq1xBXQ7L9HTb0gVtx9sCP+sXvbAV+lG0RDzoKDX7ou3ivFQFEKePxK8ZiAWH6jZpIvzPW +camYC+dCerx1vyrX2l2X8G4sffGdWLW4ZvKGRtdpJPvhmlt+v+X4SpJrDS8oTeLBxar2XwxKk67v +e1WtZAtKkzmEFVE3YUFp0snnfTWxrGGrls8nodnrTO2lD7aDhnMtaCSSIV9VkkwbrZLXMyUoSaXZ +37fRW8QYnPIEdjC07So8wKYWxGha8u4mdpYNgskgCj4VBJPSuRpozclP5orVyQG6Dq8sEgrw3kps +FayRo8+Q/jEGj0q88w8sKqxcfTC6un8cZhOyGtvKB65GCCqQoedKthE6Tv7+ehAydXvfZbUbjdAq +sGA5+1fLeeI2CYGbQ0eNQIU07lUBDYrTJTrZjc2ANTarxT/7Axvftp8+h0RoNxi4C3+iJbQ8VmRb +5hNeRPqpUqg5EMKbLSnS/EJzAvNoIXyXDVtYqcHf7IYFC0f9mn1UJlwwE29RZk3DqtCT78IaOmQ/ +4pmog1gHw3Z+woLABkseLmS/jpkVYFAVF9MQuFDy8tgkiwEd/ZM9gIjIiUKW/ryUn9NKgqA+JY0E +lAKrWUhL1P/gAmA82D5gA0I8WMSNtpewBvElZq0wqs3HvLKqsMVvG4OFkCOFzGB1EfrKuy06RcOM +smNOGDp1jOU+0eh7EG3A8X06mb7vPbzKRYRG/Hvwwe+8O7mH2FLXDZy0jL84LD9Di2CEQbi3lJsJ +tOc2dD49sPalDBxwej8AxZfFMKozBde33w3GWT88PCJGG2Jecv7fEp4e5/9tSRIhLMD2OlNll376 +nrZ4sUot2D4q9lIFwycVHqPiVqhn/A+xkuDaP/sm/EwAbzX5x+9JIkys3qzFCSZ5RNNoJ8NxOBJy +g/EhtX79Dy2tq0lno0AV1VpYgxA3ZHCjeTyFCzNcKPf5P8S9UiL+YmM0ESo7Kn5VOmo2wXNchA69 +99LYtfUrwhkAgMQvS4aL3aas/hmPeg1zZOvvbeZE6TX9XjCGRIzU55lLhIxkDAgaQb52E41LMELp +6e9rOyyrVGxOsTOToqWI2ee/rxl+q/b/zYOAQ5PN3ReTfkwX2Y8pNh1NI6Ao2CcaEhyCod1T5ks9 +8fMo5KzE1lb5TfHrDJHtOz5cRkZddKDuhHogoZFot7FG0WhPdnmlJ4o3t/zd97H0HdwwyIMfoXf9 +kUW/fisDC9YHE3FzPtilbh+AuBFx/RmJ3T+gFv+jLiHiBf/0j9C7ppDvFspuLtzphtPV/L18Sjn2 +fguaSDS3KGjv4Dxt884LPKU5VgjQZR4sMrKnRVt3vN8SP7lideJYZapaLj3g+pv9Io1Y98U53RNf +Cqf5ThKT1fWqeo+D5XQXTl+9kovAR8f0xX+l6rtK2yOcP5AQY029vw2wXZ7YRiNlVJitdMkxfYdV +VF00mpQaAkZlDzKTxBTInj7vW6/qGr1n3NSJBweFjCg9M+iGsiK3uANadfOvYZHwaBS+NJ2pQdoY +cSUV9b/Eoa57LNXJXK64lsrUoDJX99vAAmvesqg+25ndVFx1z4G3UhFhkVXpGa6pdImd6I5+QdA+ +/YGOEMCRiwwOAhWkx6BiyrEfgrPmsOEgboPSiG5EmSknr+ZVJ4Pkqp13/yd/bAujppm34jFLFP0G +pvXlnl0WStqWCzCCkH9Y+yL3snFzw+ZAT2I7e/qDcaG0Fxu+KSc3A5lk3Tkg6JI7F2rBFJSKCDKE +FLVngMK25w3fgRtQxZlT1zJGHuaK8aaf+6GUffc8yVHhEfc3uapVnJpyvDmoBToXHndKPg4o7NZ7 +nE509WVfKfstxKOfgvMysYF6ZsY9FlD6nR+GBHDLbiWg+a7nfgRfVR11f74LKIyo32Gfn/9QivpF +aFrnjAg6+u6HU9ToWn54p8DFZs9pzWEta9z/YYsapqGevoUZaSMTDhKjiTLhEKWBMAvJpsZnPmyD +pp13j9PdGPfy7C5prQkiU3DpR6wXQ48T9R2ekDpaQlk4gkxcO3TIst1gQOhCV99CQ2q1pskzPgoq +CE1WufwRkeR8W5K06+6+j9hjllJ8t5xd3mg3YFXqR7GqTQN59KMpWuxavrBDBtOEtqYKwx+NmwUI +PnpXLKrHj0A3nO6cY0R/9CEgofXNvlqSgevpvH/JCWApJtW1uiyASveDDdOVP6Hx4VOAzX5CQ1I8 +t/UTGk2z6HJQn/koxADGJ8f6Sw5x5MOWqEygRgjZTIlfCVoW/0sSLZJyRWhpB/E3TyVQ0q7glz5i +oSNbcWptS375+ym6+dx+4V5YlYChucalROo130vRzVlVC7IMwsMcR7BUsdR+8T9Z1somn639uMDK +f25ZYDYfVPkO7/jHrIFEu/Eou2Z2xrNOGY8Pyp1Iuerk4ZOuVyB87y4UAjdUi3kmOFaBXqD3dz8G +egXHKmITGFD2TY8jJTftlDxaNQZ7E/VuDI3KV3CssrtQoAPAFmVKwT279o05wTTdJ33Pun5wrEKH +v8ftzLp+3i3Q9N6Px86X0Qj+5L+CpAeH128cHBreuH5wcGR42KgD3HJh4liFPmdjof8yhR98PEUX +fsZ/ibPTH/9f+O8ag0fVvLULCsJJubiRwQXhJOteUyIjgnCyGk67Pi2NvoNpLz9Hy+SxnSCclKse +/WZcqNTHHJHPrPiq3yWP89gHB4fWjWzcuHFg/cjw0PohOfOO0Ccet0jf364UAOGx/iw5Wp0KHfS6 +9UMbhzduXLd2eGhw7aAcPAfXU9/HY120KYWx/bu0lkCoJL7S4my3XLhXKP5Tu+2Ic02+pcSczU+C +aUpVj7gzQooI5250zt9Acyr6ynmhRvpsfpIVrErxNZufdOpzykILX6zrEJ1NjAIKzvrHQQQ92d1y +YRveBBv4pIUZG3rOtUIBupOlPPwBWbtPGVeWsx4B1J2sOLnpkufSjTpBD/QmYTB3Uk6nck2ZcSdr +09CtqUCQ7mTAnyoGpKuQMhDVnebbiWHdeG3a9UJOWm+SYN82oj+DqMTzTRJK3Gk+g9AHsjdxSgxl +PPi1nwDCDm4YWLt+ZMPw8OCGDeutcKtuubD1mEeBjb/+yxR+8pPtZqww2A0xCqB5Sy2ZnwydovZu +zGP+qdkXgxyVPv5JBnxoZHB4w/DGDes3DA6sXydBS9YO0W2nLYr3tysFzP9EoGo5F15ozQXIuHtd +H3z9YvcYfcHuIGKpRCFLFGlOIfZPwSdL2SQnJxp49WkMUrMzMidYlwIIJj9lDTEGgSljnWtEEErv +wbGK1Xum6NX5U3Ex1Jq9fMAxSI93nf5USm9L9EY6e3ezxRuErTk+ogmsgub0qnAiB7R2NtlohNLv ++K9YxlSFyClAjiS/TBAe+FRKn0352Lgmim8B8Cc08lU42vTTZKdBtAKstAIksaPWdtLG3k/bJGbH +vwk26KcHVuMZuUPShdluTFXrXj7Y5vtVn049Bzcz9yWLWG+k4rnhzOjzdIkIV5qnijtKgaWmMsTA +q74ZmtEVdXlcUOxxvBl6zveXUObc3mQ+btT3uE5Q9eimp7AbujdZgoeI7YwbBGqk3+WR5pMla2Wn +5MlAr8dAs8qUu58droFcuZZJRiNnd1vxAQEvQ2uWjWujU7ipwCEQvuqYE8km0tR/cO3aQwUJx8rx +T+e7422shFxR/sFKp91IKL1bPqfKxh5mEtQgUzisAOvW49OAy4rpQlcmUZLSeKYQnsUi9QQXS+NA +LqWn56Ydpiq9+kcpunHxBYJlnJmaY125Hi6u8hyZ0QLRyWeV8n7EpMJFo6lorlhNVOYnKyvylmXy +TRnVHal9bs3JUxd9FjNdGldPZGVyxSotlf2oak6vCv0aLt46JW7nlrQYIV7/j5i/XGANtxmTW6mF +x9pn15z8AKSV2zKmsGnm6FS9gCf22MWQ9TamzWpuJvpg8ppqMOo3eWggPjmn6oEBLsrPHjctTLIQ +tg1PcfWunnblvPXDpjQCmuJJgl3uXMiZdkWd+aLqlITWtDM9PEveVCVkXwMgyO6FjWyaynoIOws7 +ngDlDUzelomJVsnARst02G60yEDQNXTR1DEytkxMtExHJ82DRQ3upnXWNk8wa+OHETHhVpq7B1q3 +Vj23ZeZUEXG697hO3tDZq4a2wpdLoIGmAVSqecM9xdzOat6qh9gbd1f9XNOEDauhU4YlAMhgkw0W +fCWnjFgniFTO3dn5EsGbnweCU27nXH4kYnF5XMjmXw4NvN+BzdgVzp1yKQhNw5g3drs5xHZqJl8Q +HgUjMwva5Ks4cy3Twwm8S7/X9StJbIUgXMvElqXBM/ycmmH8mlMP3F0TLZMn+GmyRAXEexloIgtS +xU3BJhZSxU8hmYrA6qqR7HFaWGJUEJCUKpJOhuUipW8fgGyGFKQeDsDOb0rzrKHxdX+GzEmaw1aI +2OZHCR5qqNDuYH4tcGi+SwrqRAgaer1KZAFDJ9SXCBY62SWR51mi0ILKNJKEHuvCskaZSIjQKZ0C +IFlM0OkoKRIdtKjSMNUhNeiMahwJWyYm6C+sbzQV0BN2CloK6KtW0jYPQWlkhAHIhwlPf6tSMGZM +Yk48pxJlbvPsp/NdH00DnZKGonRBFcM8p4tdn/o+biDMvKZGt6yQ0Yym+W7BV2Iq04lu1mCoIPw8 +helk95eYfvYMpFPda74AKGT60GK3Ho9MG/p8txjf6/lCZ1RCGE0UekIBwTOEztpfmBp0TqcAY+fV +h5oMdMH+5llAF1USWHqAGktkKPgapHnra4hOWF9gczopCdS87/lVrMox+5NqLrJAxSpP6Vf0A7FY +0amXf18hVM1S1D5t2v0doTSeb7vSDloJVPukbneBspR+/0ooBa8MubZTjC0L/ZnmlapUsUOE8Kep +Ikee/c6MrFQck8hysLKhLeob4Bp3o0MSc4PUizfcoVANKlVujrLM9OaEzTOEzWyyRlh20H4AKeDm +SQIeZ7ypIrekT90e87wk9XKXGTxK4jvl7V7enaN+vLBnnci5JINx/xnrBitaAwFR9NFqoxkd6SSU +MYdUNviEnu0yRXJ133e9y5VCLJFStW5FW7D6i0d4Achy2CE1SyRAM2ChHvmtOqVe+dStU5a/oR0d +3LBx40bLaULuW2jDn1vqEtjltCyFCNkX/3V//BD1rn+/pgVbs1DkpdPeOhTK9WBa8W72uB2ajstr +805VTB1wDHNVgtAJA1o8aliLY00SiZxj4T4pSVotfSxQttwZr1ytztRrWv/o4eXlak7rHz2WPEr7 +CIE5iaoD3xyCJOLvklcPXPsNmknfLbtOYJ6h4WJVfqsvoDGBic9Qgao8HktDBwekAw4VqAodlkKS +hDLTqgy/0KYK1VQhSUOpOSkFgk47wTSnNbokUZ5n3FIvbPf2BS7pd2aKuXG/Why3n7WWAvq1mWJu +ZynITRwL6KRqie8EuO0FlYIXfCaLucm6V5qjx9RehqfrpNDDw0ZDwJVkD9uM458BYiXB9fJ0ermk +ePVKMUenV8ok9+qVQhUb82KOPqPSirnJXK0+qV/CocWVeICF1Hm5mKMzK2Wtz7tT9WIxR19U31PH +5IriiZW/95fofJJOPff4l/FLj8FzvCokHZ1+rgDMGUxWof6iSkcSYpQEdOa5CHNIwCX2qFtLQUin +X4Qnfi1xpBj3C38O1cp61q3EjwRQZDfNFsWhnG7vTJlVVWr2OP3kL1N0LhFIbys1rzS/SB+WrhbN +6ImqINSTjQFT0w1ShLVQ6uXwhGUra2YGh7iEqCeabF9TSjvDT5G9799SdGNDqWdA3lut2/ai5S3/ +8I+XRHfqN7yBieu2elarGT8QsO3WBM89WrHyktZf0OvB6Q2gmlXiKNZbrRqMn1SnnVl3r1uplZ3Q +3TsNryGm56UWZoCg/YpUf3pxRkfUK3vi5pajNbpJvSlOJfBIUr0b2KdVYLm4lDYVNAdMP6DcGFog +NFes5qbdHJ/87FUgV65OTbk+BC6ObXaWW3C9XFNqMcftVBy/qa1irubkXJ8tiZONFXPBtF/yZlgE +VwuFFvlhtRZOu0erfjnfnNuy0ZI3i9vnWugnK1Sc/Gy+6oXQziXzgpwz27K5IDft5vMunsVsrjPt +5ltW4kRYZ2kf2KYzuhMc83LqNZkWI5elL8GuvG6CKNx8Ehock1pmBFOaKtnjtMOszJr62uXForpi +mYxQW3skWVTWgY5s6qp7jUySqvoF+2LOpqayVswUhYrUL2cxQz3KSopNNXqWpEXUolU6IaISrbGS +GCE0ICn8EaMKDUtOgho0IslCBboT4kx5dEbop1EpE6GdtkoC0E1j/BsyUIsX5WBpooaw76YKy/S8 +t7eXg/0HhzYcEoMI1vWfSqiJbzHbL16JgWYnjL1mAMGpd6bR8QEv5TVd4MVauPurlpUbz4OQLjzP +MFCxWsoj/q/a3GEpRkxavdjg9z4vLJXVgiNgKM/KCnevHCs9+VK0lC/A7OY1MSu8iR4QhUDFmauI +7ZSik1fB/upOxnnGq4BGeRqVwl4ReWOyW6lxr+NnrwK6PflSpiNeJaiVPHhG031S2K97R+gwHz75 +N6+rNekkXwpw1UBz930NTRXxegmd7Ppb/grq+WoR8NPprjuuQT6n8E6QFrvEeTOPkJFc6IwqxCm1 +arVMT3S9/iyqYSAVutiFeD+YKXpVaUSajGoNafSoUlxggmEdo/luWWzkXMllTuhKfDbkUidjpVhB +gRulBV3SKbjj1ZIX3u3RY90FBPfJRGlQc9Ip3a8uuasaunRaNQs35lLZnX6AFlU5qCV9F4jkTdwZ +pQzhbSinaHWLAMnoOdstOJykcwoulNxb3VP36Hx3Yh+nuPQ94Nym1bQeuH5CpMIZHoSMdgGi1DXL +LN898apvHZfRkL710S0oNQU8cZhFWiy3GE7UUeJMy0tglMkaYAuIKqvo2mbLUhC3GUdnes5z40rD +kVGNUQ8rPAy8iNMbHaolgikHg216PqX/4NC6QUik+A4ZmtEIQu2EbMbAt3DD4nk0kj1ON/5NihaP +x0+qP+5qgbUrbNcyh0FNYkdrROTLU+9LdkPF039wvVw56tvV+cTWd7MpovbGjyYuTzerW/YIX8r7 +eE3WToHXtJVSbL52h3f1CuedKVqBXfGKv/z2Eu0oY7cDGwKrHeV3baXolj/9N/r6P6qus6y7/rZd +6bIT/7OVRgYSI6IvPgxtK04wo69Y7c3iUccL3ejy1c4pFb2q3zrLd3OzrRqDakEOVXZDebdcmnVx +Ed60Q+YDIc/37HHbPQWwE4l8BOSU/sbXIW0FWrpJvhSElJVPQEVr5DdDQiOyShgA6E5J4X5plDcN +0Q2vYqTEKXCJcJreI2jXcnirn0wUZaYUf7fB9WyZNp8w+tmKxqSEHNi2xmeXla+gOZFga5RAxA4c +n6SJhxJ9PIRj4dDwRqvIjxJFfoQi7JMclw+l0DUnNJZIWOu2h27FMNJk1hYRw3/X/Lh7K8688pYt +IYHKRE/8HThgUouIyB4pBqd9yK7UI+FaczylEKfI+Tde7+TftQIbEjqaUInlACt/9uDadRFrWPfl +dNp6FQ6NaEhQidIv+3sRcq0qE3T1pxLi7lZsU2GWrNkwPFZzdxcK1mqAFGvr+i24j6ut60eeBOZ+ +GUd4DGX3Uc+1gkdbSnWX1+mkbUcwXQ/z1aMeT3PDR9Muh+XwS2HoJvMK1Wro+q3zdHO42GqSJ4F7 +hLdInGGre0I4aktW4pKTc7Z5fNqM16jIfVBzhQru3pLJgXvknrua+oVWAkqNiG7wCRKnd4MKN2YT +0rIIrkrhOH7phlAKr6BcuhRgcv2oTBGipBZa8PDM2OtMRUUYJN7fsNu5gVzOPAFrP/QQE9NEFWET +efnNgslGtSRPuEea0AdtBd4Gdv0dzpRbDrIHxejKrj1VL1yq/6l6IQKNxyHYzx6nsw+1maQRl+tr +HMXYWlWkWVBftMRYmW7kVSUT42F6miTqmmBefS2jeVbr1JkhmVUpK+cjTsHl5Sr9rVhTn7aw20b+ +iOQzK5J61h3UxolhDDEW9D37uHyAX8BVdMB8g3/osHyjruvT9OeXQYBotqAawg4QggaXPCE+nXgo +JUcAoWVAJx9Kve6bUgulJtwjtPBQShTCSarSYw+lPsaFFS3pjG4OrnBPPJRicKiZh+iuCym6WezN +xPAVPfRHXMKfLSzzBgewOQZaYJq3kFgeW93FgCciPkqwOA6+EAf2pqfCwLKF56G4+6CZPZWSN8vF +4gXkKY72RwV0p/U9Fer52D8CzWiLer+B34KnVl3Th7+bopsXlcEkzkDwMmoLAv3sOxZ2URRVbj14 +SE/1mMWqtfjASlUvPt88DfAut/jE3sWfjH/yG8HqVf0XZMecYPO2CbWY4GPrAevj3vjHkMnasnP7 +YOzLytu2Z6fV4t07N5uCuycOTGy+d5tJGL9rx859O16y9SVW0u7xu3btNd8TE9usxicmtq218xKf +24YtsCYmtg3rqp2iJMsem3oOZCaJxHAGuNk2oTa5/LX1AG0WOYS8ew/QFvtriO6KPoEs2mp/DtG2 +6BPoorujz7t3bqZ7oi+FMBqLUiKM0XaTxiijF0UJwBm92P5cSzusz4lta2mn9b1teJB2xb6HaLd8 +T9I4Y8A6qHaKS2ave+8enx02FNu+dfu9miE6bSd7nB7uak0Tbt2iCjev6TJJW34h0GMzY3znvh07 +zDgmxjbbnDa2WTPamBPcteeutdbn5r27d26/y5oSd43v275VNbU92OXCtTRwd+n2tgcvc+uByr8C +HL2mDY7ifMvj0Bgac3ggFutiJBbr8lAs3lVjsdiXB6P51x6N5mIZjubgSdouNLn14KHY2Kq1sFT1 +qHdFY73Mvh9fMNdmHh4JaSf2tPyUxVIi9m/dv3nP3bb5C8fkl4gURnraxxesX3bdiSdTtHJ1B6cE +nA2iPytq+RTiZG3chbOJ+jM+ARUO+UHHntcytxY6U9u80D9Gj2dQIHr7So8U7u2BmwvpvTeniBqR +t4DOZw9IZ4q6kE3/SbrvP3hoxj12Z3x13MQL0aiUICLlQdx/8NCsU76TV71NvE5lkct/qshtuNZ3 +w0R7a1SVGhfFHxyscfsfd3p/8J+wjrG7XkBj7+cvTINCuXqUptd8B8NeufqSXfw5Kum/29BFG1gG +dCH6JTogM4gxB2RJ8do4IO/cKUO6BJwHeNTqT1FLoRM7hSYKKCJ1RFBNLW5vTbLK5ah1+z9b1Fq1 +g78Mtf4nvg21Wncxx4XUn1CrdcFfCbUYxBi1JKUjarWGc9l3QVD1p6glEmgTb4MbXSovOalwQFw/ +vIk3000TT9NJWlqjyz5hGms9n74GaPR8mu/6CX9GJDrT9Yp/AbTRjEo0n+Vc9SfESZT4lVCFgYpR +RVI6okoCwD+0h6DIocUha3MQlmMTe4u3pYy+Pzm+iT3KzV9C5DW1u8aqeeJypCpcsCbTfNdb+DMi +1XzX7f8aI9UlO/tvqKz/hHCXLP8rISMDHCOjpHRExkuCex+jQv0pomJKrB3axL6g7USiZYjDtN/E +cQUWDWUSBJU21zRXu2CqtJ53L/ieRcxVk/wV0fJ811NIMNOubT9v5HLqT+jYtuyvhIYMaIyGktIR +DduC+qzvg5XVn6KfJjg0rLsLhU1045NL2hIy2tEoMiI8SVuZmWh5TbLy5Va5HwNaLUNXrfoBKBvR +cnoR34aUl+tsLxdXf0LRy1X5lRCWwY4RVlI6IuzlIP5hbJB8538nDM420QE1ctziE1vAkdK0iDEA +7ICpR34jzA718lVgxskdqZd8uSxX+jpfTFrDUsU1KrvcjBtqT/xSMOEijh4Ny4k1qOdybhDQOvms +ObB6phHpDJYBHF1y1HzD1InG5DtH4xPXg2tvQ8kjajg9ZjiFkh+ERF//IdgDFiCU5t+0crWNgK8/ +hUbU323TuWnHU20pp1ig5ogYddDsU/ANgXE4bC4oLd/Q1vVAT0SMIhgRUC+HBYDlEox8++X+DQjE +dKIs5gdlAhcRBFZJK7jHm6MB83GERv6YgUexIzQmH9DI0QHWOtLK1Ta8P0Rh/aeIXPLCn4fGPS8D +0v63ozGPppnEn70IaIXE/DsiMVdY/LdmCnPG/wYEfiODrggsH60IzOCevZgiIiIiIqL/PwAA//+Q +OrHMWkxJQgAAAAAABkTGeAHsnWmUZEd1529U9ZLdEkYYsWMoFmH2RewYjxsbsNhfZlZBdXWrVbJZ +BIJhGVYbVTdmk0CAkoexJNqmMRYIWUgCZCMPGDceyTBjVgNjDdi4gQE0WLYZziA0B7DmROb/F/Ey +MiMya+kyPtP3Q8Z98eKucWN58eJF3iQ4fNdd1oQ76cJZJSyAKAok3VBWAMnc3YalbNN9lyfZlZC0 +AklWsZRkRyDJKracSIkkWSkpyc7JUg4kUrYHkqyUlCSan7XlUCJlCltSkilsOZxIud1kW1KS2weS +rC1HEim3DCRZj6Uktw4kWSl29+GwPDGQZKWkJLcJJFkpc4mUhweSg8IChPaSJclKWU5IHimmLi8l +T5KVciiRcocgJeuxlOR+gSQrxU4arpdfDCRZKSnJAwJJVsry6qWkJFNIOZxIeVhQLGtLSvLEQJK1 +JSV5ciDpCAtAjEFyF92JJG3lBEhJgClsOXSP4apEWmF8SUmeInGFwSIleVogyZoPCQo9KpBk6wWS +k1Q0Ssl6DJJVSNl1z2GPYUHBYylJrJdswKQkq5CC+ZFkXg4JQMAgBYWmqEpIkAKpKzg58dg9pIez +CvSXQ1a2z81z6aZcZqzHeBwAk48kyjxfJQrVl5K8MZBkqy9PQq0EyCkWpSyEsoLJJNkazyu2KOYB +Uil46msqMWu9q/YLDxCI7jVoJQ/Snb1KC2ECyW1VNJJkWzwkDHORJO9mKcZEOpLkK1MkdBKRJO9m +kdx9xJZ8ZYrkniMkiwj8Ld2asd4ZQgPg+OV7DxwPxGEj68WU5DdEW+jQ5+4zkHJHFY1zpqzjIaF6 +I0lWsQOSAjxVSKGtpiQXBJIu6IIb5JV6iTybzhg2JwwYRqAyYPME3cI5M9bbBf4q3ZvCqGeqKCoU +GhOSf1UkGD1rvY/nG+19B7V6LxHdR2lBDiTU6pMCSbZWISF2IgkuCYAfIZnTnUjSBaVBFNvG/Qb2 +AY0QBP0r3StNyxMutE5nXVC4+BAD/+vAOdvRwBllYOesAoXbNGa+UxI/pNRZBdpgQ1AFwOnL9x/2 +1l1VwlkF+oyQdZDh6pMxS1iAlPGLdOdKpS7fn6LL/VUUD7l8zEACoPOGqf8Kcf5dpS7ftx91XdDh +osm67HrAcLUydjqrQG8KXLJ1mHKBxEe8qANQ79D8mu7Enm/fecr6oVJn+4QFSLn8pu48S6mzCi5E +d6FLRZdRLm24RF2yjw+7HjjwJf1SfN7sgFIzM9ZblqYBgkliQ48IbUH/IyIBmGc5q0Bfo3vODoK+ +OWTle+eEcZzfd0Hh5vsg8Mg527vlOXfGcF4e5ZwfHrI6L4zhvDjKOT8py3KeH8O5GuWcnSEeeNAg +dID7CnFWgZ4esg6C0mU6q0D/JJYSFoD4SmURIc66oAjwdQqOBGddUIT5YkGMYLK0zhhpJ4xK64yR +lp9iJX5EhLMFUCTMWO+24IhwtgDasI3ZTIB12vY7YuSsA/oeZfl+ARwNCqMa5qI0pDPWWwSPbLKt +HDYvkxaQrlGb88UmSs7G/a6Th+M+LtJUfyYu/01poftLudA1FB5ZUpI4d+kgOI4bnffpIeHEmYEy +zjqnCD1+dpA1Y/moTIXFJfUFUBZQnLVBo0oLqPQ3A1k+TkdVWkhVmrXexfvR787SszAvQc1TZO1p +Sp3tltwAtABILlbRf1V6vM3XrcV6S1VvadetO8lXS0qH75L7dt090a8k1K3e+fvrLb2D/ueK/XWL +u+9SqWEe5B7S3eO8fMSTebFuDpOS+8e6e5w1NCfzI7q50+rAlryP6d4wW3Kv0d2+Rov1lr5Dgh8F +qTvHRIVKBkhJVnQnhsjESiOqPivSKZrLN1X0BqVTkNxVDiCdtd6h/Vws66azLuj1yuqX4+IGZTrr +ghLY/XJSJ0DqHWIMIVN0Jg8Rs7gYObEOgFX0P68XzSGlzpaEBUhtOVd33q/U2R5hAVISYBWKoRBB +MoViKBRJjoZiH5cxMdAnegxYhfnfFs2PlU5h/n3U9z1S6RRtYw0kzxT31yktPCbTOb9VRX9f6Y5B +51hvqVtkMZrs5Fa7bpX7dPklQBpyr5QwJBRWjdBzDabRj9C+h/tgchk7jo0sG+owBsdhp5PL8Ngf ++BiKyWR0HCYld1Vj5vJDB1NIFtdZrix08ZAwsY8k2cdVSHgojyR0KgFoCZCwHBBJsg+Yh2QLwFOl +swqUWamfm1MMQHLKhkfIQo+UktxKPJ11eNHBIOusDXqJSs1YrwKP63dt0IbO2WeQVAFWEJ0tgDbY +ZF9PwQaToXXWBUVR/9QKjqbOuqANadmHW6QNItDCE5azeWbvSHU2D4rQGes9FhypzuZBGwo8Vn4O +kNY0CvA86awCRcKM9U4BR4SzCrQh7ZQgRpBKo/oxqNC34yGAellFKBL5hQaNFNxAMy5IOfAwCg/g +xYPES3mThq6TwqQzv5aScDkrcOn+Z6GkzpZA/1G3nHVB7ymZ/skdfI/ynHVfLJTU2W7Qi3TLWRf0 +iLJ8swSPxuCcAFRvagxzU2edUc07aH4HCes/bZ6ki6crdbYb9FkhqzOqegfV/0al+rP6r+ki6p5t +96nucd/SAugjZLCzNmi0cGHUwgUsfIjU6L/i5iKas3Cm7mODswVQNO+TSn6A1O9jFA1lBSkJsL7o +PWPEgPYYA66YaMC09Uol453SCpca2LVSMZJk15QgAdbnm9G4GB/5NAPCfQqb7iqbIsnE8B7VJvQg +jxO3GestgkfOExv9aN2F7gSf++4EpwIE5OFHDHenv6IChU47JTknkHRBeYjw75HA98vOQt+ecua1 +t7M26Lni4qdR4O9UXmE4gzNs4ghYfULq30JcfD0oK0DqrVp3oHVWsdz5dd1yVoE2OJ/xC5KCV6bw +8xhhkhEg1Q8pj5K0gpRDjxoOgF8Q0zWRZKP1SCKFybez6j9J4OeUOjsI+uWQVd1fljxQm9oLb/ZT +We8IXLqgCPDxCc4SmbNuKswXE5MAuDwvrTNG2glI+64YOeuMkZZ9T5KXtnBEHK9XWlgwhgsL/u8N +JBUoevZnB6mG/UzRBEgdchfNwn66ZVDE75b53NYB/h2lPm+QEyHlcw/x2Sk+zipQWM9a74r9Ke9+ +ZmQ7gJQ5s6zrFV6FLgSn3Uy6315poT+DBNiMqL9UwhhxGpPARp1eNa5OC/sA1XyPKvOfUx10lU5R +F3Mq+galLj8SY8JRDadlvaz5jFJnpz5OYftBpc5OVQ0FSIMS2IxwoQntmD6aKUpDKDSAuccMmjcL +NrwiK0yyIGFnZyTJjnprIFmWYsDJQgqjXkoSNy91QR+gMCyNF3k2VHcA4iJPsoBknl/9y/FAL8ix +aWjLUleAlIYX1chxVoE22JwR6AWwOfSrgzgAGltjQF+ue4WBPeUS94Jl5x0pCVJKb32hgT0aOuuC +NthkbZ77tWGbHxoMrEB/O2RlFypSLvcOJFmbUxKklGyGBpsfGMS0QRtssjYfSmz+pcCmAv1W6Bqz +NsOFPY1Xi6T0HAPNWyTxMqXO2qBPja0zO8GHDRNoaApvayC5r+YpL1HqrA36qmhAfu1XrjtP5FvD +eJGtaEjeK/YfUVroxCChv3tccFMHlAryy6y6GyC0Zym7QWzW4O01kFCXPBYX5hfYt6yQOax0ijBY +Q/2tgeSwouSlocqzS07Y8kWR3DeQbGRgHf714d6OUagQi3mSrGJ2yrAU1hucVV9QkL4wVNVB0JeF +rOofhF4z+TEylfVWCXDW/bRQBPjxHvwVkuCsmwrzxUQZgBaVl9Y5rMJI8P04+HlBWmeMtOxjJNLC +dpDApgJFQn9BN+Xdz5RaAVJTXqBAe6Z87R/23iUc//u8wECQ8nmJ+HTDFLUChXX/Wa+X8O5nimeA +lPk3xfQkETvrgDaYv3siH9rwZWplhc4F5/+pin5ZqbMu6A+UVXzQhg+wmY1hM2TtVFDeVqmzDmgj +OK8aF5z5J1mcdlSZM1xsUTVOEQxflJEnB5LsHAUTgM2oi0uk1oeVFp75UI+ihPkUJEe1pV+iCdWJ +avEufJgTgM7hyFMHnRKDWHwGzT6jQbKKx9Y1kMw9baAYwKTaWcWXFt/XvcJzVMqFmZ+zLijhWBqy +8myyXkpJGmLwdQAqI6VBRWfzoA022SdZ2Py8GtqdlTqrQBtssjNf2LxQesYvJtqgH9Qt/4pCaICc +UeurSD71PFNyCv0N+lMUOwok1h4Ouvg9YXZFJiXhGbLQByxLynGy4cFKCxNISNjpyfeZBZLDkgKw +ZcJZBfryEBjZB9SUCySl1pLSIG6HderWQt2ar1tkwe54651Qt3q3rVu9uTo7c4Tx92QTEzhnHVDe +7hfcD5dviQs6lJ4Cd3WHAyPuEckGRo5kp3Xrlrd2oW7BZpsqYqf1WoObeScszw+r8sIwTmUrERJe +CULi99vICQFottAQaX+kEv2pJhe7pXdhkRM+FH241C1Nhws02W4GmtuJP3KK2xpyRKXllyNPH/Y/ +e9gLTRESija6zaz/Dz9jIIYBmfUHZ23Qp+ppoRS4sGFTILTOOqANNlnvwmarRN5LacFoSA4ocEi3 +WlW36haXP1AIbfPZ3brF9StUjy3rnVK3ervqFjlv0B0x4rJWdiQgh48rRCB9AhDxc3sGHgf40rJg +IyQMxbwGLwYdRECD6CPZhy+Ixki6ciIRzZUNdIXmWpCTf9KAaBVybO/A2cwG6EadLYBWiozSLgXY +EN7QFlavIMGTkWQRtCE52z5hw5ZZaJ3tBm2wyTYs2NAYoXW2BNpgk52rwQYavr5vfIjfYEOpADQB +2PCJ8pdUotAEIPmvKsoOkMI7eUhQY53Kbr7k5VMHwcvn5yziFT4CT0l4+1BoipDwnBWl4LgA1B8k +cI8kHVDeZZUGjQ1ic0RuAniGKUQTJBS9RrTFAVlixtBkWy80bAtpyHm8RAbAuYf3DWqdx7d4iFz2 +IRASipJqJOKSeRljYBAtQIO50wYaABwvAx3XxMdWq/2Q+kqVv5061O12sG75iSYZ7C1QeRUPkBPP +09FWa/sRnUu02Oaz5+ug1bvF0EuZr1tcosQ2q1t+qOeamTxsRB0g1YpPrOi7ttqi14pLxG3z2QtR +/KfEsGW95brVq+oWOXR+UHD9DVFIAJf/S9kU55rlNxXnEjMpLuoAqXlAUrdYjXnbvXm+bsnAGtnX +ivahQSAhY43hEOu96ocd14nGGxWNB5aHG8MHwpQ0+xwECUdMkao5csl3ETQrrkl9AHfrkP0CVU3L +P972Pr1St8hBpZ3W+/5KPbinwgGoZ3SDllS6cclJSujG9ZKs32m9A/7L5GtX6hZ5vJOFhusDopE9 +QSVBqtkfKJ9UmnHJplekcP1Nzct3Wu9G74QvrNQt8nhBBw3XD1mdZuyOJZVmXH5I/R5SuH5W1Ow6 +r9nhlbpFHhpCI9sD4JxdzxyOQk72g45rNs5P8PWhDLuW2lQ9wnD7oI/PrxfMPWtYQ739CRoGkwTB +soTuct0/Xqp0ardQO3K/rqXmW1jvCu/Lq/zPj86qXe249S0V8V+vd2vRk/mPuum/Xu/WrT5v8r6v +e0OEZP6i7GkSStMAqUmsadBDbbUlP1RwuU8Mtvns3XWLawzZMRgqDqzULbJQFZIgWpBqcDOJuKXS +4/rd9tkrtR+ayGTJZIe/Od+/RRYqDdGReaGY7vAjqudZt8j6vm4N0ZH5YLWHBh1Z5+rWEB2Zn9fN +Bh1Z29WQh+jI/HndbNhHl7RFbzAaLMk6SbeOt94Zdatv30LdIvcM3f25eLd33krdIv9JiqUGY7Je +oFvDjMm9THdTxuR/WvcbjMn6km4NOYHMr+tmwwmKmQC54KG+nbVBP6D6nbXejWdxwTs6Z21QqsC/ +dV3hgvpw1qYeMKFfjgtULiyk03k8UPqgn7M9oGg3a70beCgJkJq83iaBsEbtkEX0D9UOmcR/g44s +on+Ijsy/HG0SZP1Qt4boyKQqGvLIonYagbJPDeh6pUMsyaTdNFiSRaMZjnhyaTJpxJP/VjW1Ialk +EisNqWQR+UN0ZBJbDStDUAjS2ADiZuouKNz8etwKF4wYhVcEhO83VFdYNWu9a1eQB6T6fEVEz1et +TCGH+U7c1jb/W/IuogsrWmj7b9VI2GfeqGuyaA9DdU0mg0SDjizawxAdmbSIBh1ZYxoJXRmObFCR +9WD1wUPSyGR4aNCRxdgw3H7IZWRI2w/5hPyQVDIJ1jW0Axb6nFWgcPMvbybG7+MUvxxEM2u9q1c+ +o3CkEfczaQFA2hIemXByVv2e+LBFobAWRFhTSzeJ1Nn8YVUY2hQax4Ezh6e/50rZR/tp7OBBtdt/ +XF3VL1w4LPFR/tCl1bEI8o7XGHlrpevh9UHx+KjS9fD6Z/G4Uel6eH1NkfBtpevhxTFhvINaD6+9 +6qOfrXQ9vP5APC5Wuh5eDN43bACvB6jd8HnOevR6tXi9Qel6eH1UPK5Ruh5e/yQeP1K6Hl7qIALQ +qx3rSdbawx3rSWJvH7r9AnKsJyl5Z9Rx/157knQW4axiQsExOLPWe232/T49Utr7OavoCG+hmdKM +5d/6wiZtps4qxv6ba9z2D1IT1UkHL2cV49jN1T+Xdh6hTjqXcVYxrfm8piSlTUSwSacLzipmDrs0 +uvpp8USr7icXPEKps9NB28qatd65E/lwlOW1GmCcnQ76L8ryqy0T+TC3X5Ivtlq95fR6C5cc/tKy +36i39M7eX28hh7ODdlrvdf7oUn9PggOkYx6fd71alvpl7NPrFpdvVLYX1uod2l+3yLlId3Za7zz/ +XsLfC1IEqTAmLA+XWl5YVbe4ZB4zeN9y/v66Rc6yCHYMFv/8LYkIkMoCaIrHHgsm9b1pL7Geqdax +x4Ixo1mhBtKedT2+p5XReNbD6//nx4IjLxpeYGADOG8AueZLaN+bdfOv62BHcf5GTu80uaS/2m69 +xX4QkXGh/gt0R/9GuQ985UBz3i1znq6zzk/UL56gwcVZ54lCn6O0sHIDYzb38f8ZsoFLdrO1rP9S +uFu3yGHvx87Bnf6rNvLmNGjv9Gux9eAeec+L9/xr5j6dLAkQRgBZj5LIlpJc8hdW2weq1C0yOPJG +2ld1ixwOlQ8kZLTkukhCDgZg8I1n1cGox8uoSEUO5gZBwUhBaiufEnP4lmzlkirePnBt3SKDOm8N +blR1ixwmE4GEjE9o4I8k5HxXdwIJGRybHUnIwT2BRPYFSA19umTsVeob3XzderYuz1K6tX+WcJab +vXrQQIBnC3FW0dwujln3UO1ySoSzedAn6JazigMDvxeyFkFvCllsmQmAfWjEH8tepxLOuhzVhkxn +FehLAuPuZ4X+T6XO5kfFs1cwQCoeiNZ3+FA4GpHdFYcRHxEbtig5a8Ml2pX92BsukDCJLqwFQwKM +UZ9ZtbPOp+SiVVhEtaCTsy4HCX5X3Jwt4PAfhazsQYFoPIbx34oabs4q0Khx9hhfGAPl0KYUkIuH +Y1wGcMy7wzDwyk0/k355k1SlRy9tXNd4sDkkwLFGNQC8/xU5Zqs6wCkqjLEhkkzscRmB/o+kFTa7 +oBgk/xxIJnacnN8bFZs49sI9KjZRCkWjlImjKlOHxsxFVgUY1Eps1JtDgg3MtqYY7nnBeqsQMPkD +B9TC76ii91I6Re1Dwjys8HyElM3x2BqkUOuculw4bBBbNodkdIYzMSw3hwRPset5Co9xskc8H5fN +/AHSJgb3904flpBcEEgmeox901HKxPZSi/tBpYUTJwgYuDPNnqKJQfLnQcpEW65QUR67pqgXXjKw +V9LZaaFCBGm9YMPXJW0KWyBhGj9Fd7GaSF4ZfnLl26jCXjRIbi4bT1Za+EINEja5c2ZgyXwpRlH+ +NmIVJI8OiuVrX1LgHs3PDnzLIgH4DKzgMUg4eyqe45B99IWEL79YHCyYDwlFV2ELJJwaUYgxpMA9 +mp/12OHEY0+U65xVoB/XOkvhhJaUC99rOeuCwsWfOQH+ycA5O2FKOXN0iLMO6G3ExZ8zBj6nvELc +w5mFU97VOatYQ2Vtw1kFyrHp/ntm8P8RfcYx2Q8LfUgFuqgs/80m+POUV6hT1MRa1C0EGySjzsi7 ++TXDXQ0fM5YUS0hisE0tJZJk2xpS9IFIgyQf0lKMPjBKyT4vQML/Id4t1OgCKKmzRdDIOLtCA2PO +CDk5MM6OxpBQNEqZSAJEko2sirnfGY4RzoV2VoFymJNvHSgDMNzChn6tEdCgHA3jNxiAN/4Xm/cZ +t1Hr8cWQAqTSYM1nhc4q0Aab7FiE0utks/zaYRfysVKhmeVJsnULyc3kjCgl2wAgufUISbZlQsLf +dsWTftugpM4WQKMu2fYLY/48J5JkYwoSYiqSZFsmJADjf2HEgEQv58LJAaWuOKlwtpWWxlKR8D/S +vy4NnXVA4VIKfNjwrhFaZ21Q2Mz6P8KVlAA0IPhAxMf2zirQV+vdVn8HDhcwL4Q2rGl18HPWBoXb +rPWuW+HiHMmbwu9/LHs4k9R/UrXCBasQzhZB2RfRL8fFcWEysQjKObb9clygX+HxBJOxD1V8hwmO +Ls52g6KJL/Yr0gapPg88asApRQHS+uQwEzRxNg8Kl/6pqVzE+sx2IhiX1uc2f4ZAu26l/Hf6Sq1b +/e92EUPdQhP0F6Rm0InjvFn/lTYXuM/ZAiie7Jfjglp1tgCKR/vluEDJwmIlPkAeZ/U1ju1Dan+D +GN/YIKKfKVsDpEafrIGPv8vRG2Aun6zb282fhNCtW2QgeUf/ht8z1kqjqXEriBekWsCVVFpwyR62 +bf4bjW7d4pp/+/Zvc7t1i0tSceHybbIFLlw3TNnl9w/sr1t3SxrGDgu3ZEGA1JTPiJRUSnB5rW6j +BNd/r3wV/54uSZPsLeq24ML1ccqXQ7jkKyiKc33H4eJcnhSy+ydzBVMFqcX8Xe435N8puumjOAL8 +X2lJqyk8r9PAbi/FT1I6BclGDYVPVj3PK91E0c+TyJcq3UTRBD1f/xREH3jjYJrLYwH/xjEFyYWK +BE4+KcQlUliFYr5YmsGdPVAM4LHMWQXKzqDCDC3lAol/soczQLuDhn97j49oFWiDDYNzANgcSQzg +YEE6Ca6xRn0Kl6whbfedhD9GhgyG5nCqUJAsQIED5wx7kJPT/IZsFX2F0oIHUy6QlDwIDfPiU4KY +cBRqg03Wg/amYQOokEKgpSQE3DZ/CGinbnENq+0+359rQAbPLf3Tkc6oW1zzqgBGXPMW29dep7Cz +KWtK9tEQEjRjja9g/aFECk4uVC8knM7HXkFnHVC4lGocNkQvDcVZG5RT5/zZxeCwLjwewJm//H1S +iKWs55bfPBw3Dc+9VtRnKy04M+US+6yJgika//FyIgleirp20ZXdgt5xUjsAbX1zlD187rBj6cIK +XlwHCdzprApSDrxlWDEOCiyQHE5IXiWfOqtA94f5Co8uAfB7nsu+t6swTzjO9rHkzBzK2T7+I/ac +IIvT1gOksthAywKXs/bvq/D7lRZGbzSGC8Y6a4Ni94z18g+Qch9ndKG1s3n0arChFw2Q2oTiX1UJ +H+vgeK0wWUAbirLs6Lss8KcFD2fXyQ69bTiM4gsmnpMDYAAknAAZSSZK4VQkJhOl0yzPG1ZsiiYB +CX+6enep7uJ/yrF/esZ6z9fdANhnvWHJjX+3BY2bL7PNJOXyPMkpmAwJ64ORpAOKYB8sYhgg6P/2 +Yf2vDlGQVzYh2SOmzrqg/LGsf0MG/p5YDJS/hfLFwFFgGqU5+bJBk521ovVzpAZqOavGqJNtkLDh +WQ21Cz3pkXrYxUxY1kSSHSjzUuh6AlD3y+/IKoaO/A+c7ygCvWAym+4YNsuwfK7YFPrivIITbeK9 +Mc3DWQWKAjPWy2+k+N1h1/AGq1Bn9s4BCac2cjznFCS8a+LPfr23wX85NMhsH4tk9pPFhbpsHwsJ +39UgraDs3AUD+4C36j1mYfKckjTme8zn6JicdUFhPGu9s7Prh7Ce11rRbyt1VoGeFbL2gF4YsirQ +94SsPaA/VFZ/0ZiLE/Qtp7M9fKHG550uPP0EoG2g5uWi/opSZ9VXhWLuFK6/mz6evlKps1M5SQcu +xUVXtHmj6M9X6h/OluoWl7DfZntqn3+div1A6WAxdqlukYFwrbhesT//oIcKl8pV/DGy/3IanBmJ +s3nQd2qRyBcDv1R5zuZBH62a88UkIEBaJbB+cGxgTIGeHbKyb/awo6ei71LqrLpUKOu6zipQvveZ +td75+7nAnikC4DtibcH0RVC4zfoPm4PRgtR2iJ4U+GQ7FwwFGm2YLCCVcpFuUNdTGLhR3cJhBcKn +lE4hGpIvB5KJBq63PR5+93CUMmgVtE1JeExx1gVlauJnVaqBAFRRyqZBw9vjACkNjzQs7TS+Pvs7 +EZUmb4gmSJkmO+uCNthkJ2KwYXU1/ul6ts1C8mFpySHrhS3bkLCjMNZQG7ThuuwUCTbQoOIUFQ0J +m6YK0yWkbFAFAQ0Ds7Fh7xmO5DErmtep7yrMF1IuT5EKzrqgcPHBDX5i6MWyLRbOsIkLaxUoJwf7 +k0nArw4K5PvH9w5bzgHYfoQX9Z3jTAm0q5bptz0sC/+c0oKDUmF8+OO/65OwBucW+KHAuQuKMO9H +8C/FYmIWgD4gVYBHZGcdthAi1O9hBH9W4NwBRagvBn5tLBYkC1IF/rfyf6rU2T5Q0hnrLYMTw87a +oI9VPPrJ9uOFnxoiqQ2KCZ7bC6TfBUp9HvhFyiss1eK8BUl7UBhp5kEb0qox0qpRadkVKKTB+rQg +rfMcoW9X6mzPdZqN3kFzPGcdRrcnhKwl0J6yZq130f4LdPGHSp3tuZUifp9Sf8yuUCzsk6ryAqS1 +DIxpVE8RP2cHQRusr9q/hlZ1aKReFnEerP3Ecky97Bqtl+zARb2crVhDhLMlUOrHWRc0VtXSbVVV +VJmzvaBUmbMuKFXmbC9oo/Yu3s/FO0Lt7R2t0KVrdPdmwevVaB0vjanj/LlB+GFJfsAfzqozFZev +Veqs/XqhZyv1DVk6YZazNihG+Q1F4FjlrA0aI3bv34lZFQzcC9qo+4mV+hcjIbSbSoXNrPXetz+N +oX4m0Q6krYFKwPPOuqB43tmpoA1575/I+jUy+y1KnXVBLwpZFegnQ9Zu0Ia0/NFVVDlvbl4vS521 +GcliS2eOFCB1B5+rzyqAnC1tF3orpc6W+P89dic5W7q37saI421lgFTW7RV2VKazNugzdKswl8Pu +j6nonyt1tvQNoT9WWlpL0ExjRu3/a0qnIPmcitJPOGuD3lKBP4X616vaf6J0iqGOAzv4tqywoI15 +b1ET4hOuwhZESChKlbij2nGPibsQN4I0fDaHZHPnFaP9dnZRlnqiwcV6mui5C9VW+f+Twod8SLlc +JLcKTeq00eabXfmFy9UjXE5V5QZIaxmIfVjFIwDdo7ODa5iYrKGTOKoD0ZgZCLYDqW/upx5oRWnh +I0ZqYJeKPk9p4TEdko6KvljpFD0Ug2pjZoMNQGrL5razrvbon6O00IXuunT4WZSXroW+PSVhza6w +zJCSRCkdULj4ByS8CODNlM0jVMBZF5Q3Nv4RDfw/xmKgvCD2xcDRYAo7UJoNZIX36PbBVTt4OSHh +S8tCnaQkcTEl+1wOCd+m8PxckDJ32bAtjwmOzTeAhIRO1FkXlH9O8XUhhgGo+TybDmyYGZYCCDac +TAetszZog01+6i6jWOJji1ap7xAJRRG3Ssl2+XANNHblsIHh7poMFeoRLrzYfpn8XdAfkhtVNO5V +/k1QTmZz4W+HA1CNcOGvxBpeCGUFKQm7QPge2ll4IcgKnbPu00V9ntLCtwro8kgVjdvEsusjkPA3 +/HE5+3QxCZCqj4tfrRJTuPpyFeUPiAr9Eorxj9SRZKJfWQS/SwgbOoIAqS2YzZE0brL57Nj6gpi6 +sDgdACnLVwyHOPOEQjynJNHJ+d4vkRJJsuanJHwU7Rd4ghUCjJn70OAWjfP+ul8wBhJ2qkSSrDGQ +8BfSkSRrDCRsUo0k86CMIaV++YDsA9h05qwCvWcIrIOg9wtZ1YuE8i6ysJKdysIdzrqg8bVhFxSZ +ftk8FebzEHymJu+FVpYqgAhnHdCGtBPAo7WdMQqcgAIvCgpkKw0FGDGR6qwNitAZ6y0uJ771eVQU +QKDCmi1c1J6zec42bLBeHmNIdtYGa9T5PanlrA06I9unCbVR47MPkUgeJdkwf40aNVEb+sDYfWY/ +08WAdcQ3FQ2kFc5nc68LtTJRm/+gotReYbnlKBqw68pB1wqw39JZBfo+3St0KimXOKPH5wHwXErC +LLDwEJCS8AKxMD2BhJEjkrRBMa/UamDzNJnB1MVZF7TBZhmc992F3hDOTEHvJgmFmQ0kwCo8R72w +VW4KxfggIc4Fs08pKPZmaRZJ8lHwJ8Ph19gcKiYBQuCIhEkqTzslj4mEwwyoxpL5Irml5LPn39n8 +PZXFR0TOFjmJKOqSnf3C+HbiwiS+EMSHpQvwp0KcVaC71fUX2mjKBRL/IloMA+DrlOZjKuGsA9pg +k92kABv2ffDPF84q0Aab7BAIG6BBM7UFq6FhaorahQhLVaNe1ldF+HiN7gZWY/N66mON6s5dNdwH +xKaW7QNSEhg464KyI76xOZ769BtNwH8gJxW6glRY5NwBje2+AkVAYTyDMR0KPVrpsUa+gvuPgvoT +fcWOY86SctYFfbmmIqWpLZKPnn9pM6uwaQ0kzGouCZ7LPtxj8ktU9A1KnS2cI5RvqZwtsB8OPzvL +np8BY75repu4FaaAR/4Mxw/gRFVZIVZSklsEku5PhD4lDBvZETrlwmc7hbiGhIXN+GXkImejxnWS +ioMoeAZwVoFeLzV9gx2YHYEhCmEI4ZBTZ4ujVmYf3OBiksh5TlNY+XSRcPBnYSBHCmdRfkKkU0gB +1leJrNi+S+ymELwxfv1Z4vJWeZ3omqIJ8Q9kVNwUfqPoKup4DYptjhRs+Ko8V3ghuOvjw/1U48xy +0G+G4DsIGg9mqx4qGTcoLcxoU1mND5tAEeAnueBxfOmmwnwxBP84KJDtG1MFzgyWdUAR6p8uweNX +tp0xCpyBAltD55xdykIBpHEieiGkIeENDQ9rzjpXSv2PKp2imvmC74uBpANK6mweFAc46/B58Zy8 +7N83gTMdcdYBxSe+mEQFYCTAMjaxI23Ges8Y42c2awdI+fDtMnyc7YYNW1gLYzza/IX4Ry75R0O1 +Hbi/Vr5x1gZt+CE7lsEG350X2EyMoy+rKGIKTzxIeansY51giqBZA8kFkkK8FTpgFAPoCpxtcq+w +MfHDQ1XcET+x4tcQcnDny+XCTBT/rqHPSVumG+3ufWOl5oC0YQIMKM6O3thyFPsp4N+5Gcc8REUC +/xbxeuATw/Munj2dVaCfCXuzeD0VAI1TLvGtcRcULn6qBP7FwDk7VUo5837HWQcUbn6qBP7fA+fs +2AVn2LxZVjmrQOFW/EYXPts1BhLXjckKn4T3DymVmACpDx8jPvzbrrMOKPqU1ltQ5/MSAM2s9d49 +UTbPopzM6KzNqW9XhlllG5RPuf2LXHD+UdCfo6rtnNcoddYGRSlPKT0DpP5AJ/6Rz9niPunyl0qd +LYKih98WAB51WuS7chRxthhQhYyfKoKfpDxni6CnKssLCDoLcqqjr7PduBN9ne0GRV2/agEeVd/N +BxVBX9sN2nBndr2DuPipwmtn8F12Sy8kN1PRRyt1tvQqoejZ//JeXgiQuuMPRYTBzjqgDT6FvwdX +X4VTCLDCUwskeIqKnYKECv+AKnwKEuay/Ad7YXEYxdjRtUXVUpigroMEoDN2G9Yvb1TvSXfFF/39 +7ooL/rPBWQVKyPfLYR6QBh79L13rFE4GNt5jr1JNX6XUTW6AB1T0r5W68C4hQGryGkhQ6O+DlIk9 +AyTUxjQDE8D0Ym0zjbVI/lmmAdbnFWB9XNbipzX0yWPGYvQH0qgG1mfflZqQhEFh+unn+gTfUkMJ +Q8tRF3zovwzPUXhxVBjLUhJeYjnrgn5FHYSfylMjADUGG6qUtRtnHdAGG3YiB0jZ8E8+0YBwuF+D +TfaJCm0gn+IQsJSkK6P9fkeh/M2Zf/AA/5DuFaoWzhye8U+yurACBsmdVJRZXmGdDRI+logbrLNP +RGsgOXDNcIDFDUIVKEtw3ktSPwDVnGfTHcMmG3Qpm4dLUGm8zxqwLslo7dbGht3HkU32ZfMGG8Ce +gs2TPPdXwyH0iNCCKs4QfXv45DW7ApFy+aXApZtyKXVbKZu44yvbaiDhxLn4j5/zPLeT9tcRuEAt +Z/O31oMRdvbLKXYD0FaQxzJxg6jwIbeczOQzCt/D68wzpISzPaCnye/906+4eK4yC5NQpL1GlRCl +Lf2LshDhbAkUAf1nSS6itOxkFGn/MMJ6YQzr61ZGWWf33ML6X0dY74U1J5jNWu/bK1zwrO5sL+jt +NPbPWu87K1ycrExne0e14tEjQBoBo77csJr7joLhx0qdzYNio7N50GVNqErfiCoAcdsfBcYV6GeV +1f/bEy7+VpmFGQusL1MlfVhpodeHhKLx/0AnNvNLxD2SZN/LIQVYZ5/EaBY7mEUaVlRmkb4ERxfe +yGywfhz/u+H6jelZcSmQNg3aDjPCwkM2XqA/ja8KJ/Y4wDorltdbR8RvCmX5QxsGk/6IwQVRsdaR +BVjfSHxUVTyqzMcMkrgESOPtIvUK9EFT9D0UpQ+aguQodvijrDdsXC4Mns/damZmNmu9/8fem0dJ +dlR3/je6Wih7QYDYjM1SIMAg+CFhY4wBQwMS8EMsLzfIqu5GJbOZkWHAiE1CUpsBC0YMKEnDIBC4 +WeSjsc02NvviBh1W2xgQwzIYaMZICNCRYTgWOtgez4nM7yfivaiKyKWqG4sh/sh38753l7hx40a8 +eLFceR62JaU2Jm3OLUmbrLOkrVFmc1wI9TSkbuva1G+q+aXxn6EZJi2Wp0N/3Xwb+IHYFQTnSbJf +FVOSlwQp2fa/QAJ11DX7tpZyiSQz6xpJGAUIiRqDFHgyM6LQF4CEXRsgnYGER+eQwqOxeWacKKSQ +l79pOsSz9YSzCvA7AZV9PUy5MAblrAsIF/96CDzDLP+UM9oWwjkkGI59amorWePqxA4gV2dtQNR0 +1vk32SCObOU9WRZ9nki+rmupiomElVmvDSSdjwhkjXlhFAsul4kkqs+wakhp6TO/KJLkq5h0vZOa +5IfrWhiRg2SnHj1OV2dtuDw0ooKSSqmuvLmPdH8GiyxA8jZxZ+nxDP5GwuO3zvkXKFFmjsUSnRoC +FnACliWx6ncGJ2Aq/xw1iURAWjA2ycXmEXxUSO4qKXPUgTkUW/scIXOSNjDilZM7pTndKZcNfBwu +PsADc75OwS9SzmhbqHCQFAJ8PLe/A8jVWRsQNZ11tqkQ9uo6g3gCPO+XhQCPxhfJ0osFeLgQDqL6 +2QAPCXU7kmTjLCS4Zawv2DskQjMku2U8jhJw1oYLGx8WPqnAZY5ovQmSOQI8Ukhb7/wLlCgBPpZo +NsCj/wJOsD7AZ8eGkLJATSJtLjYtIPjokJyoajFHHZhDMfs8IXOS4jeVCvBDes0sLNpJudC9ddYF +hIsP8MCHAudscEg5x5X1MA6JeAIJvnaOnvCH2QhEAT+dUKiQUjZUkdoCH0COCCp9PUWb26gY6bM6 +6wBerFvOVgE59cS/dejuP+haGIBE1j306Gm6OusUGL83PNUHvDygOp8UGMX3AdmaxjeAKsnjdXXW +Z1/gkwOq8xsCH6Wrswrw7IAaAA4DampRk+H/X8o6GwC+NqBWAA8F1ACQXDpbAbxTED8ARH9nK+h9 +enhqBb1fHFB4YEipb0HCtbDMlLLFKn8apGRHGiDBg/4tkGQHzyH5oEyEMZytAn5ft5ytUMTRLlMZ +4yi7gy7Z4Vt0eYHMx+K2QncQki+LJK7PmupCC5CQ4L110Q7HIV4tGLr6MvIcvkJII1wVerIYm7T1 +ZnigXO0sXQsv6yizR4/+oa4z1KgFSFCIiDKDYhsEYAxHSqMDiYbYLdQmLyD4ChmPjbZnaHIWkEII +jcEv2weldHmUIFjyziuaHZs4Z2auyU95NnNNQUrZ8Ll4oQxsSnK0w0Js5pj8lOZ5k5LnmPy0RZIP +Ji50gqqFs4p1L3GWanZ0O+XCO62zbsrF942p9CSCQsom7uyajSOQ8Ik+bi/So2fIdfyJmj+o5az3 +T8oy+Rw/h2qkVMXX60aNKD/5CSUZOonCVz4l4XdVM+ZsBfBVmtQynvzEn4uFLCzgRtrpYh2lrX5W +KEQ4WwVEwHjyE3+itGynB2lstQA/Z3sB4bZkw+vP409kne0cwfpr67TetwHr6zZgzSyMkNJi/D2Z +/YCuLnwLCSklQfQDAkm2c0oGOKCCox8LERESHr1EeZ+BhFkXkYQuVkhpXkibrK/3FZ9Y+fo4XVSm +Tz3DgIVuC1YgbVI/lvhtuX4bRB1UJqUmpyLxTj5DRSbWvEk8ZyAhbdJwnEj8FfGbQTKvqQTacTTl +D16xaNQlba6VOqIqHlHmGzQgmISU+hsbuxKDZggkPEoMmoFkfcOyZc3YetZb1mYdwYaFtGAFPPCV +Zs8+9qUrwKFEOLsEMO6OlW2SUsb3D1y6gHDzPTXgyDk7ipnn3NmA803Xc8728Za/2jQG8zydVYBs +jufsEkAmxRQ+OuUZd2HMRwRvDGBOnS4M0qScbxnMPHMuTwkkfUAU2GbDXwKOyuQnkWftl22tUhJ2 +xHPWAUQBPzAMHJWZmk2Kic25nXUA2YW4NHKNgn8gK3EoQSFW2deabkTj7awCpNRLnwESLveRAs66 +gHDxPgNcezEUQUhE7LWE84P0REGZlCTucdgFhIsvJjEMCckHE8nsh1uoOimJ1hF4MwT2SqkUHo1S +ppKQIC3UvENJXqLhK9bwsRq7YNiUC6TO2oDsfem9FPiY0FtvA7Ibh38M+J7hsS7gIwIq+9kZnZZl +DRRx1gMkZ9tsOACOnLO1Hc6chfIISXDW58SkGQ7IgAvU0fJ9qLGTsw4gdiq8DcCYs0oiyQAQy3ov +B8YAhRmEy19vhoS7hHe67FBHShI30+4CwsUrI0uGRGXIs+lswIaqEtJ0Nv0N2PDOEVLK5tm6U8sB +MyVDSmlI0BRq1do3mubm7P5CnElJ4jtQNmikJLHvkm2UICH/tECFMW9ISHPkhe4Q9bE042NxizGs +wgeGkjciZhEa0hwGWEQMr6Dfl7yCzxz8ZtPNGFXdadWoNdreGW0ftcDx5A7ztyp/CxRxbCe3+qMW +OJxvQjbyt0BRYSa3xsLYKO0/SPfaLSIjWzRPbo310MMhUfHIHt0NuBfcaM+3yNQksYx7t7dId2yS +/qgF9s6Th2yXv8tNkIz1TKzSGWcdHJmZZGKcdVAc4LvTBt7Q43vg+Oi5K9xrj1ogOea3xhMUC6Ua +dCDfr0zUC12okLBozjzOetgEfyn0PODCGW8PkJxC+wPJg/VorEPZXoAdbhYltm8WJVgY+tKalPMo +a4WUM7TOeoD/RXqWYsnBRMFbiKZUYRMSxlBc3Oi2FsVawKx5ctblw33cerb7CrXld9aIue+NS5WQ +cIBUgcg5zA2JnDt8mTte3dFtlm+g4Xy8+nfovttWRq3ByNf03qjFkdP/WTo374L9jO4eb8O1UWv4 +/vNH24cf9j+DUYt7n9Mzuzx/2IP8sm422YP9mu7usppmIL+tm96RYAvue7rXZAv2Wt0dazQYbR/5 +DFMoF6hwmqRgP6a7G2WYe5/UM2P2aAbyc7rZZA/2C7rbyDDIr+lmPcPgOF23yRbsVaIca0SGg8sp +pZ7HXhyErUIwx6XOlktFZ80PBqqCMeAVnXkqyd2UG65+2zngl+uesy5grU4045TZujwvyzO4+s34 +gKljzvqAHMnsHwPGjwpvD+T9sZLW0bVwLjQkpDlCGG9iHMU4QzmS5pCygLeQ5pDyI9EsydF2TGKK +7yOBuolu7eRWb9QqhzrxDCmtCJx/jYRCF5xiYtvguMYi23pCUknvFV1nKKYP6NGP6zoDCc5MAG/G +C7CE7o2iHPeI4OOYQpQDSQBvsgdLBG9EOZCE8HqUA0cAb7IFu2FYD6WqlBbuLwyihB8SlM4IQYl3 +o5BEEUiIX4TFGeIYUiIJK/5CSqVsXUnhak03AouzNRwbJL7WJAW7kAfSdNHFaLIGS+dioyrJPfoY +Dc1B0sVosgdLH6NRJUHSyahXSXB0MZpswV6l5nis0awdj18YRIkq+ShZ8Ym6FrYZSkleGUjyE0/U +E9s6s+M3TZ8Ai+eMfYKGAySO0yQFe5VyMyad5k6Hv9Ps7sX+T/ZTZEoSeyYMpYSkIlpHwrEWLh5I +8jE1z/6lDJi3w8LHIZTh4FvGmnfb3lFrdbR9ZbR9MApfAJHbvAv26VLhppP3tD2j7cNTRtv9Wxp3 +nqkndnnuMAfJ8RdN5mDZG8THjqAXSPa7msSOic7gXiWZTbbBxEqppVkRxdV3voHvJY7OKsC/FMo/ +BkwpFAYAsD5pDu9hMOhi0e6YmNR3UkGxkG0ntwajVrmcxSuk1Cqsb0NCYYhnE1k7VfIZNix0hZFC +0eCKzcIGixv+PDkoefv5zzGvZdtV1wodQNwCEoxUmOYIyVZ5EqGpWS5gCU6NQAhyvoh16KpmCwR1 +4XtUSoKZnHUB4VIaPUzZMF3DWQewxib7LS9lwxhuYWwjJYlflHqAfPNz1gfkDEhnPcDrFGqc9QFf +KvfyH66ByUYhEqES8uM0nNX1wlYRxm7hpVN0Us7QOusAoqf//g0cdc5+/oPzn8kKM5wjBgkrV+Mp +/9kFHpDwKSGWb3a4AhISfakZ7L9BYcOFRKuGFLL9GT0wgxRIKIBCQ4gULBb3v5qqGMYdyCXnkBJJ +si/Wa1c3A0fcEKQCvGd4Pc/OC0i5XBWM2AW8i9QvxZKUTdxsqQNYY5ONJSmbM4I2faoaefKTyYDv +G/KZnUwGZ3bMjl8fe/SKUNRZD/DmMesDYPJRcDOEwZnvT84GHHGBBGcDQAQs2fDD5/MnSstW0FQa +/Jz1AeHmR8GBI+epRnuHiuHduhYWnaEMlQN7F4YdIcFa7KUxg4FRjKHUwkRvpJAHTDNDXpASSbKV +8uD3m5WSQmt+5gT7fNnzxjb83ZE+dPZHLdCv1O0mMVgOedw5JvYfqEYtcCyubVKC5UDOOiU4TlP2 +72mTL6/9UQsk05Ym72mTj+hsDcJBuQ1CkLdSPaoTKnMhEdZTE9L8FtwBktSu3t1TYzprpxb0j6WW +c9ZOzeUfC9oqpUozQHlQ9/0LJXDccaoHiFT/2AbiWAASUiqOEE3pFfwfG6ViCv4PyRxS7JqmiZgu +W3iJTkl+KF9x1gW8peK7b4GA3yycs+7DNeITh766D9U37st2TWznKYGvFK4wJyLVKS6s7zxact+r +q7POYYG3lB7OOqi0ElFP3TlRBTX8SA/w24NK2c5eqtK3g5n6qPQB6eGsj0q/FOT3Uen1EXXhjSYq +ve3YydVZ/3RpeWnQKNs6oNFbJsTGjnbO9qPRR4NG+9HodkH8fjT6UkSJVUg4/JGQ1VPuH69Q5Gzl +xQJfoauz9qsFHtTVWft/CDxxx0RPv92CzPY4XZ21zxH4CV0LdZPcvU4afVhXZ+03qGjep6uzASAy +x+sTz5KUy3R11gG8Qqjxc/w5LKSzlQtU1LHIs1/XURTT7QuK7r9IIHlYsuFXzzteWt9JV2ftlwgk +Y872AT5XdnXWBowFsR/wVfEpQMrI2X7AS8NTnb8SuCMUV/sk5R5vd7b/KeussGpJWueMqoJsbe5s +hRpF2fgz6YAfIaHOVgCR7x9LhK2TxpHEUdpqKm3Jhlefh7hTgrhVQMSNn8MD7q+MO1tdwAaoQLE7 +axNN/kJ2d9b/jEC81Fkf8PlBzQ6++dOAWgG8p5RcsuHnz+PPHA6Lix2S9znrAF4h1JINP30ef+4k +hf1WQAKX5D7j5/hzMyGd7QO8bUB11gQ+R1dn7dcI/BNdC/1iKtsVqlknB5Kpnnkb5SnWu9OfJNTz +dK3FlicHxqcDPjWgBoBo7ux0wLfFp6Z57zXKBBHBWftB0uRUXZ21AauA6gDuDaj2twRuV8k4G3xb +IIXgbABIIThrk5Wzg97ZIQyMzyxsGrnddvqotX+0fd/Iz/YEe7myv8vf5SZI5nfsnNwcE4L7lgh3 +cG8Ueul01Da4xXho4Lh3FAbJ6RfUyEB9V/GqTgbuet2rkdGIv0ONeCNzIOkGTXr2E6uAo/VvEH5K +3OgP1ASCoovQoDtBnQQ6DTsms3z9BxNQj9cjDTqQ5+imMr/PE4K7UPcahCDpndQJiXrUs1omiLEE +p9otlXVIaXvyGqnxEV0Lbzy4J71h5j3MQHKCCuDuujrrMY4SN4vqPUR3n6mrsx6bDWDuJRu+J7v5 +PApijgtDTG8DviugeoDvEWrJhofP489HhJwhVD5fXsxg0Qwk91cseIiuM3TTrtDLxdd1dbb3xwJv +HQ/5AfxjocbHs/Dnk0IWvtJjwYFYH9TVWQX4d0L5ec3AUakKpXjhKb3AIu1UsTxX1922OmqNv8j6 +eAf2gO7u8nf9J2Z/E+TLdLNJCvZC3fXjBIEvyNfo5iSaDMZswf2Z7jXZptlu3sUClMWNbfjIWnZA +Uyo7x7cnQyngvq2SajIGe7XuNnID8jrdbJKG6q+URoGTRHU/XZ31ACnIJRsemlrzSHO8grPxLefx +1mIXqNeJraLhuISOfht5ZJqmYvtD1KM9abQVIGlQZJ3NNTKydEipo/CphKKZIdiR5vAJ0hwkt1dN +PVnXHZMo4VtcUIzP7ORWe9S6oYUeLENKy4dlv+R5hvKhP3k0hlIWaCt/dhWd0b5GrQNJX3TcN+/S +/QZLb9SH6HATJF3UJukNLBBQLLxwNIwEkjcOhaZx4Aa3wVsIERYbN3iC/LmyHnVv7ApH9A0P2+Kh +DduCxEMnXaEZXqxw2p16e2hwBXl8uDleUzphC3LD965/f+0e7cRZal6a3Suw9DfHd7v0UcHSqVRQ +KPQ0AylhnpSG+1StRt8YrRqd2MCabupudfn8otHupBvKi8IP1t9CEVKq0JwvDl+WOWf4VLNDytAf +neXFgjzS516gK/4ViR0XaXh3APu1+t0eBQ52sb74Ihm9mUa67qHrWNugD4VFSguNkfUni3qG19Hr +xZTSKJwmylsew5z3EKmzlYcLrH0NQUdSqisR82j0VtCYzM6QSQYAGfWbgeRiDQLMMbDJ0MUsn3mu +b34cZdC0+ZUe7Ntl9t3j7/vjcNAftcB+Snd3je/6WNEftUB+VjcnpCyFB/t3utsgBcn0s/rcAHAP +1VhQgxDk03WzTgju1qHV8fMcpCzI2+tmnRAcg34K05MZByBZizBpISf3lLWQgssmpq9NKgBEpp9Y +BIyI0rBawvnFkl2Y7JWSsGdAqa4nUthWwtkqIDpvs+HJwFH//CeDLOe9b9Ro2iNUtM72vkUghVca +vxJj9PtmMEzFRJp/Dqj8OLy4MM3kl6VTqZUSSUuPQlr63i8SCo+K5KzCCN8RN2fVD2WE63WdQReq +0G8Hki5gtG5vvXWzS0rQ+N1SizF+Z/sAr9Ut74kSG4eJB4BR/D7Ev1NPF2ZoIz41mO8JYDF8xONU +0CGl9ZJtlTgvx9kAkLmaznqAsXgG31AmybSzAdn4cMjGAPBjAdX7vEC0nKOKM/vLWQVIhfMjsSGT +SuT14D83zcDpJfPQ3FE8C76MGKLKfwsk2U/MkLCf3GkiOdZGrd6oNWqB4NqyUWvo3yDCHTbBG3+W +mdwC9V5x2zlZVzQmA/ch3YsMwWCdGkM9G1Iw7L80Dfu7oe+Uz3FCchcxddYFvEdAheN2Pi1nc9b9 +ksCT5EbOum8V+PTQke6+WeDt1Td11t0j8NZByWwdzynpZw6h5XOClh3Av5Vq3q+AUdeTAsdZsx1A +Gmz/GDAKe3bAWNg/JwVCypXKb8k8zvpYKpqlD+cTglmy845Ss8yjDGarxcbTpNflujob/C+Bt1DX +xNlgWeDTdfVPCXyuCtnZ/lcKpNyd7b9MqPfp6mz/eXKBd+rqbOCU7Zj9wR6hnqirs8EZAsmxj+xJ +Sq1Phh+oLDnrPUzgQFdnvTWBMXc9cnetcumsB3jTkJXTAZcDqgcYzXI6YLRBDxvErDBjOKQ0KySy +5KwCvJfuOesCsjeRs72A9IGcVYCMoTvrAv5O4FUBPi2guoBsMOisAqxFB0Cqn7O9gFQrZxVgLWAA +YjFnewFjZdkohhBOogPtBXykvMZZBbgSUHsBY0FMDZpEC2qOszar+8mSn8shGZFxtpNHjaagKApn +KwS1j4ciWEEsUcTZyuVy0r/X1dkKAQVHc7ZyjeobhnG2ChiVzL71oiTvC3dUmHVWPUcgU7ScVa8Q +qnZKPZ/So97V3aVvV1c/a09gzEp1pVBroZJVBJjvRpQMFFJafTiqBos66zBwHI27ygmaHDbkbB9d +blbAONv3m8od3xCc7XuRUG/V1dm+DRoabMA+Cs46gKcol872HRAYXX7111R2FKsLtTmkNMPk6kt6 +wk0vW47AaikeOhvcWmDc/p/YEVIq+A+k6/W6Ft4PcKrL9ehndXU2YOQKry0MX8AF40Sfn+oVVNjo +lFOzhyPtCCU9oKQ3aFYj48FzVaxv1NXZ4KBAKrCzAY5/o+Dag0cJrHR1NjhOhoouMjhRqJh7hlJC +SkuKrFCna0sLWbrgrHqWMko1d1aR4Vqzeraygv7OKkAaTmcrgHwcdNb5deXqvro66zBYGnPXOUm5 +o4ydVYA3UqQtvMLgIUSXmr1plL8k8X7hJHCUP9WUcCbPzla+II7f19XZygYZC4WjlJYR1TeGn6kh ++iYqikrXGSrgr6mMX6ermy7lC4oMP9Z1BiknqqTo283QgcNFqZY156PuziAYLrMFf969apH+YcEu +pbCOE9TasehC5CCktKR/oaSFBnJzlqQAa614sLrSDdL4eGGt36HshHSDzBfdootCKGFgKaQ0X7yj +xugz1RTrSbLrbWgveO2tdc6CRkqpYpDEd8mpUujvxV74VCnr37Snhpf1b+JTSdb3Oqcq9nC10y/U +tTBCjpEv0KM71DrMQfJakRa2CNqEFJqpWC5Tx6rWD1FkSZaXOJNjktgYutCcpSR/NKH0fZbs6CA0 +bEsT15RXgDU22dFb2JCOkLaHZBRc8y8lz28NITAkah40v687HC5SMOSB7RPbc2ZkHOzsALJ9jx/3 +E+OQkGzHTNiQThZQkiwSJDOEUxvNuVBcSpL/fbFZvlHTDuzu5KwC5MDfwr4uKZe4bUAXEC5+PSvw +amg08lVN+jHA/2pZ2FnnnQJZbuKsAuRs7ZLvwfm7YhOPws++ekKypi7l7+tamB8JCQlzuE1b5lCr +WXIcclDw4JSEKOKsC8hCCl9MwPefXkwp54cqv846gHDbZsPHAT84cM6+TKWc4/YXfUC4bbPhzYAj +5+xoPJxRsHb4KSDva84qQASUXAvOvJwz3cRZBXhn5by0fjJlwxZqzrqANTaPAY7LY7K1Cs6w4Tip +OXwH47stK4fHqDY9UddCrUL/LTLwaySRg/tnkMwheWzwvcPa/pgFP1Mc1D+I605uDUatK4Vjf4Dd +/t7KeJvIwagFloM7d+munxKW7SVgC3b/Q3zhDR2SBfK9SYvv2dUMWp+RPQptCyQPUEipDQQAwmWb +DTkDKSQafNg8RRJrI97hWaWU5OHCc7SGszYgLYGPBsBIKHgRytAx2qBVgFspQMAG8r+WooVqvHzj +ZgH8ViQBpClwdgngC+JTgkLCWHnGnf+oh5lHU+od2XFN/Sh1ZxXgl8PLRnZvKLgQ3z4qDZx1LhD1 +H+nqrAP4EaGWbPiG7HnUsF6WH7FV1vgYZ/50ddNZB3CPRvTGz/HnDCGddQCfpvei8XNSOySMjRL0 +Ny/WE84qsox+fg4LHvn2oFZ1M0n+FV2dVc8W+AFdnVVtKbNXV7+CT0Z6t64FX0NLvjw+PWiZbe0h ++YYe/bauhVkjkCAF9xifgyvykFID8m0zDv9PVe2LMiKWW7Lh+6fKYQPgfwm2nSrnsB7F9ks2zB94 +jhH49HBCGLjOVlhISLcTUAhbKQk1crHKSYHN4Rakoy54thqEeqTU20hHXf3ZqjbqkX726t+y2Rgw +X6EUcxISugilACISjpHiBNHCyBQkb5GpmKlWbNkkhjRHZhbQjHSEpNyqWTAnSFypYETCeS03Uhh1 +VgHeVqhtNqRjFFLwRbEh3UFAqbBEskWSOe/sb4Lkqcp+WY/ycWPH5Nw1/7IA6h/1iI5rGy9y+9/C +7ZJddnuyyRbk7fz7ANk9JGoklOK6LLRA1jZp1LVbTxyJeTNxH/KsUSFhyfelyqezPiAzMX3fmRP1 +MYPHAf84kgoKCX9DGoohwtkAEGkF54fLesH5bsBtJpYhPUmAs4qSvUZ+UXh5Srn8duDSZSIrXIqx +K1Em7o3ZIfc1NhRmSBgzZXOOnthh/VGrN3KDUQsU3diWvzW+A+YT64mECSmVx/QthqZLVUE5pVPC +IGihcCFhnTmKziCFtGDh/krTRU4SO2cVINsEF0NqwqZGkx8+Fw3Tlxkdc1Z9VVrU2OQdIsumC0e2 +bfD1VpxDCuV826Yh2BWmVDESEiboO+sCwsUPhQaRSqnk81UV2VjNj/QJxYysbTbMfoJDG047rXVB +HiuJaOWsA0j4KtZc5ZOS5NuIsx6v5bCrLVOocc6XnTgTC5BQmMuz5/bNcjpReStUrpSEL0WF/hwk +9xN3JmfMIAV3jiSEjpAoeaSQBz5QzCAFEqQVAsXhxGKcjuqsApzlvPmUTY0m69opTfzclW2zlu/Q +LN+4uXF2pCYlwTbOuoBwKbl5ns1UZVkZQn+iUIBIgYTGquCMhxOTsDGun9Isn2KeZiFapVz4PuGs +CwgXH62AWSk1h36wcxY+gsBtmw1vChzPHs/aN69zHyFw8wsXgSPnbJ8sz7m3Aefl9ZyzE/bh/N9V +NKwtLdVQFTDzKn+gsF9wo7U7TaoJY9WEeGcdmjg8q+Tzh8WGRKhyVj1DOPY/LrmWuLxVJHQLXewh +wqWkzNpdJnn6DbGJsS3rIJDwKPubF17lIKEViCTZT115EuwVEqF9+a6TvJDYdXu8oKs7anVG20EN +9YxfBTa5A4ZW1a9E6/pPQyDYbnayjcBwz6g1/PPzRxqviylV5ld1CzbO2n8rFK21s/ayBmr36eqs +ArxSqFJXkJxzUual+krprA3IesHxsOdjNEZJnsZIaRVSmpPviQiFigObaETzGrP/RLLPsbzOnhhk +KqWi7yPR6F0IjAhmF5FI0kPxWq4vm5prEhmZQdvZihzGpDTPJyggsfVmIZaR560RvFvOc0ddCx1D +BK9336lFup5kIY9/jmrHJ3QtFM+he07Cw21l8f9PV2cdwJNl81LEhA0vpWdGNoJCokghIfpwJEfB +jSHZQMph8WfSTO3gAObEOOsAxrXobc5EIJcFj7J7yVYyCMfWFVpHSF4gtzlXVxc+TIaEXSBhnJTM ++sON/lWSLxAbjwsMlOBz6GRpK3x8Le8A0jwWS1ZsoHmW2DnrANbYZAe6Dtx7og3p9QKcnQH497sn +OL9BOouWQyJX8KGFj2XQAazxedlUPnSOKFNnFSB8/KkQQQ+lVB1GfWs0U02BnF9XoTqrAGts6ESF +lIp+kXxhpKuzDuAnokUPTLXEfxV9XICbrboUwidF8gC1RIXODiQ/0aNvVB/BWQfw0qjtRVO1vZno +T9G1EI8RzWdE5DnrA9ZEv2uq6JMkMi4UyvbXEL0iEuQVggwkA8Xummqvm6raAb0Av1TXQmRCzlv0 +6DHac8dZeyDwpbo6awO+Uyi/AfxUbX6or+p30EbhSza86Lz36E8tXxdlN1FFycdJLJr5rtm5/EE3 +ZxVgTc33T1XzWlngTAnxkxXO5Q8cnfUBa8wPTmX+Ju2a/BZd/Z6x5/LnC0I6qwDvHXfYPzSV+RNE +/wFd/SyBc/kDR2d9wBrzy6YyJ8IS2gqdCIrpCsWDB6mSF84ehORYPXqcrjPU4w2kPEHU+3R11gb8 +nYDqA54bUNn+JuoN5K0v1HWGitvXo5A66wE+R7ecVW8WSHVw1gO8SrdmqLxb1HpRQVF9hqI+Q8H6 +JboWlF170KTpZ6p5nJuRbduWHzwh4VFWnJaaYmj4iMe6Y2cVYI0N7+kh0azChjVbp4TWeQXwOvlP +aRQfNo8UeTxHsgJ8udiMXxn582ohC/aE9VPF+mJdnbXhgoY+IORf5mTkb4r+Daq9M4ju6dE/1NVZ ++6cCUWHJhm+bKnqR8uWlOe4+lH+tUg4hwQ8K/RRI/kmOQZkXqkRK8lORFmIFJMfJ9DhbYbkrJCx3 +vVr2niEvkPwokGQtduBhk4rH+DjTKJ2FGZUXSWe/QFk5DYlKBBvePSKbHmCNTb6nLG34SsoXZ2cd +wJWoDaflhpRqwzFvNZp8x35rRC+fOrEn6YECnFWAtEHOLgFkQ4hSTRRjlmVfL8a+VIDxLmcdQCZw ++seOlzs8RSHH48QlJEyIOMaQETuDhnxNjHOU+RoaUiqFR9+mwnXTSeZR7FHNMqktIQM8JwjOfus5 +dNqECytVmFd9jPkx1RZ/yeuxfva7P8QfxA+V+9Z4M0Z/R4iQsAqS/lR33qOrP19OIMwKHS648NLO +vmSF+HHo0ZMs0laytqS2DobVfH6jGekSP512APk466wNyKjPDOL5ogBpIa6i8QekC1G84KaQsC1n +JGkfKy+4u67O2oD3D6jsux+Mb6pHZyCxx04MTrq7gDhW3wJFWYSx+hYYPvfWiEDxISYSgWHSvR/g +H/svCCYD+ht+5L8FglU2kReYk5RhTzLm9TkhvqVrJAHzY93xJGMpxyo2LesabsgiIVFJUtP9pp4o +1AhIWNuJBxT8K0+S7cpCQqIAa8UDihKM9gFDAdaIQFGCkQgMRehtNy4HEBShvzG2NghKMPIC8zMt +0w0KiM8lV8usR6bMWGkRPaMPiALOKkDChbM+IOsWnFWAZ8jVnfUBfy+gKsBnB1RfOQwp9fhP604k +oYkJKSUh4Xg1twKF50VXAIPj1YhA4XmRCAyu513uBu2LbHkRN6ih0xZSam1CEQZ01qes4OasAnxf +LHnAzwdUBXgTxUVnfcBfDqgKkOhZCILL1aTJYZIWE6AKX2EgYb74+5XzGUjupkfvrWuhXUYK3zf+ +UyDJTnjYBAncXxGkZAM6Ut6lR7+oa2HkagES60zKheyzeK30wfuwaEj3FeCsYij/DsGVsh3blMsL +AxeYhISnpySLCbbeJMskdqx2VgH+JPh4Vv2UCyR+KhGcSegPzdli/2pdS62KlN2lRznObwYSHCf2 +49uXq1h+qqsf3z6fP4iYgTULeeNB2dl3azJwO4m8r64zSLm5Hj1X10IfHikPk8mZJTSDlPVVMh9p +VRjzKPaEibPdWYqxOY6zCpCdzkpVDjbLYsO0aWd9wBqb7Os1bNj0A1pnbcAam2xogs0ka1Zj09uA +DdPLQgrVQbbZpDZ7BugxSbHssxlISRgjKTQtKQnvYwWnTEko8EIzmZJQ3oWgDwnZjlKmZp88RJJs +6EUKlnr8xNbeYoCRSzYYwIWEAgUj2sqkdMnMTRQM/BJd2JCCY4nm2brBejZfw9jCvsYnq26Oj7MO +bFi34Kc2AH9QYmfI1RaxYcSxlqnsiOPy6sSgpDfIoM4qQMbYfaaAOVKq1JVKOCOnUKtSkj8JyvQA +UcB/Lj2fP4/WWGKhXsCaCSVnKcMzaMOjqOCsC4gC3v2A45kC+HNIeCTKsFdTPMQx63yQMF2Tt9Nj +/YI2v0s/CAbnxgvkVvx5py1Q/0eKtMwP/HkiMH0Zeuf4zoQKHPmKVGBwgpooSQgpzTGqHC+BzjqA +cB33QgIDpZTPtcKTAWftDfi8aSofiCjQ0piv/Bm7RJK9gE9Rt2z8bU0ahpRm4ckyQTyueB/Hv3HG +yJIN/3gqHzaZ/ivxc9b+kcC7SR9nbUD2iyp/o1NWby96iGeo7VslmuCDaZ1VgFi5+OVAOdgiNpst +8wP7HxL8wKd7q4CcVYBD2bowzxwuxDCufjso4PsFzp2ewBPF2T8GzPkzzjqP120U8I95HesJ30UD +Tmxh/aYLG+eHlJK8UXdOlVaFuYhIoTl8h0idrQCyoMXZAJCtUQpVGMYsiIu6tAExmv/KDvzioHL2 +XQDOqBwPN2IiZUipYSDhrE5nq4AM4hfeWxDMCBpFVyBZXps4I59J+epaqt0iYTJFJMm2cWsiIRHH +nFWAx4RmO/tenefSTbmUwgFseLtjBouzCvDlqga+4IHjGzn9zZAoRTgzj5iFxc46cCGbpWkksOED +M2vZC2UCCY8uBS+tANHA93SD5kppBjoi52XeWZt1RDU22a/daHO12PyjroVeLyRs6IW4Qp4PPGni +unz1Y/8T3zVVvkIig9Awsxnagmb25IkYlj8yt6Ik5ueNZs9TJjYg8apXm0OA9Z1dAsjSHWcVIKef +FRo2ZDGJCm7OOm9CfvQnvoaeFlE8RKLsYUznmDeigov9P05y+KnNYr9Us2kLpZeSEFGddQE/paLy +w6HAPxHOWZedrx+q2cf+MeBHCFd4UUMBqikS/JT+F6W8x0j8hIS/wOnj0g1OvlE4Xw0Einkc9KSU +z0vVwl2sa8HvEA0JS7ycdQE5S37Jhle8iD+1+eBnaL7303R11uVcX86yddY9QQYlHzMYlv3orw3Z +aAPukqwlGx6eatZ3ix5iZwNADo4qNZKY6OPr2Kwwwf80qeNsBWu8KKIoJlJaXCjDoRx+bYSoP6rr +kg2vnprPm+vhqE17vTbZviTZJC1QBY8sCS46h1e/WUWGJzlrc4hL9MOpFoGacnI/Wz/cInfeIjbU +oLvJ+Qqbl+JhWyS5LYmsN5lB8naRzFWtSEfGuw8+o9nyxXecbJhPSXijKoRUSHg0Ssm+SUFCiiTZ +fo89s5kXlro4qwAZCCo07Hku3ZSLb7HRj0R0TdnUPu0Cokyxg6080SlHBWcDvlKxgcCSDT/E62FI +qTq8B0Q+bdby1fh8cCof6Hm3PMZ6fmiVv2+WAseaP/x0MGqBQOnW+IZfxH6cehhYI94Rj5DSvOxR +f2Wga2EMBDOSxx0S6qwDiPxSWwyb/ymJ6D6D5O+tI8m6/tqzmn7857KAswrwvWpZCn6c59JNuXg/ +huOHAuep+jGzBHbO2oBwKw5PKJvs/ACtsx4gU+KcVYBxdKgHyLCef+EHPlMF7EsTmL6sH4zRbTYn +8KTAl+mes97bBX5RV9+TlX3IYXHgniyeqRJ8hq7OBgQBMuFsAHg/SfPfV4DJg+9/A5MJZwNA8uBJ +gdHYN1CJ8v4xqRQStQzdn6w7Ufc+ulNMzvqAmKUU0+DMnLwzVTWcdQDjAQkVb9IMhNfGxD8YCaVk +SGkuSOi5WF1i2Bo1C/V+7axmJWbSl7MKkD2ES5U4y6WbcvGVGI5fDIbJV2JxfoAsE/er6wG+U1x8 +JQZGQuGjI5yZgwc7Z21AuBS9RAqme2DWFqRcI92dVf8q8BbS2VkFyGzQwtAXskjYdrHCoq1jWLMw +12HteU0v+Qtp4KwCvCDkKD9enHCBxLuEGIYUKkZCw0wpZx3AGpvskdcpmzhBsQ9YY5Mdrsyz6W3A +hhkhIeUyhQ2PnjnppxFWnQ0AmZziV31L8zhdpM3By7WVF4BYwNkAsGbT7EQnbPpcCYPWWRuwxgbt +Qkptul6blQ3YkLOQUjacwwOts1XAmjbZlw8yBQ3L1XzLnKRUMiQsXyq8r6RSfiDWhRBiL2jWZJb9 +FRRLSWjpnHUB2W2/VJNhc1BKcnixs9WL1ZOAzfj7N39upc6Ay3+tgzWzelyIRW2kPEEixv0g/kTW +2dENWP+qWHL1XRLgC3XPWQV4H4nzjwGfGjKSdRuk0QCxInK3rYxag9H23mj73lELLH31m9hwze/A +Nbzk/NF2P3uEG9dJsV2eHGqQTiruNE874QzuxrrXIDxLSArm+Mkytdd5oQf9TzVqcQ/LNhVXyYeE +65Ntyi3uL9Dm0Bzy7KyNHp+QPs7agMzsHc/x4A86Fb66ogCHkpwpw83gcyjGTI8ZKhIklwUp2e4P +ikEST7jIdiUhIc1Rw2n3zhbtjonf+H28QTG8upNbe0ctvJWduJqFTnFRDFvvNpgiVP/p1Zk0h3FI +c5AcwVDHLsQxqNBIh5TWLsIVvtQsJrDEjqMUVIK2SkdIac4/aeYZ7IbhTgqFlGpGog1csDk8ugHu +ZYqZBEyXd5u1c5o9BabyOKsAadlKb4YJF0hKPYWUhpEXZx3AGpt8nz8RHQ806APW2BD7QqK882x6 +G7BZNjMzM7P/y965QFl2lXX+212k+3aCQIKMYXCGkrV4DBFBBZUEsXAciCJyH1Va/TLdgmBwwdIR +ZVhId2XkEeRhaq5DQmxIySBkeAiCQFxknB7XBHkNBAMmQhhKRiQwCC2P4b161jn3/9v73F21v3vu +7apKJ4tv3bXOd/Y932N/+9vvlxjlbIgBm/iC9Vj7/DyROOUXyrC2Cw2C9b4i6ltVq1RjRQqKkCvD +xD1nADiVF5J/V+xfrmcLZZnaI54tSGZQ7PYzyQzKcsTgv45WpBKLsAVp9XBxv0hPp19A8k5Bsvbc +8QKC7q7jRJDwaWMAI8ZbQPRzEmQ63aKchHVRTvQheaSkJ8XKWwuPosoI0j66LugbVdo65WPOpReT +anCTULhU41vgfxY5F5uOOeefG6npWSEneaw0CLaEMrekIPGLQIrlXH42kiyC/oqCdtnqPDirgYMt +slo6CWudCukCkGLhl+s3BQmbNxonQsToC3IrcMMbhYTjhLliW2P+KaKHyT6nuATbtzHdKc0j5DFm +q+fL9IVT3OcxviPoehrR2/EEBaawKzAFyRtE8zE9W7j4zmSkGRQDTi+pqN6ZiJkiC1BEhp0oLamo +wkx11saSu1waXjZeWXLN7m7rDqsDR3jvaOnJHjs+7Kx2yreA5QwvVcIF64PCq7rXQf9GiGVWpleD +huX0EXIaFh9AUw86xa8FORH7yanInWYMurHBh5n/YH3WvjdEXzdRNDOTjenIf1Q93ODzuol8VtT2 +uF7PYD3QN2mIsxr2BGeq1csDWSqQgjMmJlMNnDSyc5KxXcOe5TtNF54/nidS96jYGM5J0r2ExWZg +ToJMp5DOSZi0dCYIIWHT5Gvk/k5fABKWor85khS7QpAAO2OxGaRMQUISXqBIOQXCaUSfnUdseHKy +BFJmIGFykAHhFj62Mw4zg2JAm6R8wShTMYOa7iRdAiU/VH04OANUEbChuwptsGXQBpvipnHYQMMd +2p5jKQKQpJZ7OSuKhHmBoWLjLLiYf+HITAANaaeMKJMUy7uVTAp7B4N1Qf8gVlzFpQpwoZObTpbl +1J0IJB8k7Bdnn2CI3hchJ3mAquGf1NMxIlJO6NMb9HR2bM7//rjd0yxxsZ7JSZhbcDI0JAztJCnl +RqEUgyQ5RDF1c5IkpTxCkEmhU+DUZjlJQzFuZrpCidnCImxoTbpOjB46JhIcMQI+hK4f1D+JhEIv +Qk7CptxEUnTu+ReP+9APRLfrgj5YTfdgx0F/MQaVPS1jzMJfp7SChN1nKBDsCCgKVNcL0CaLEM0g +0czZ/1mM00QzPEafHtLTK8Ik5bf1KYe6BXvSMQX9Vz1D3DQeIdf1f+jTH41FGNvyIuQkHxYJO/OD +HQFtmOnyiWbiULlHRdFd0AaftYl8zhc9SgTrgTb4lJvNGHSGZOOmJuQFG4A2RDvHoystt8rzNt5R +MDGfzOAzyxsM3t0k1ldNTDiATBa2Ku+z1gqzBjsCmmRNl7dX/mC8vMKuTkaFhOXiXGbrHPMOCUPr +tJaC9TjHiOGcXbbKCuUI5FTYQMMymGBd0PspE1crUcE5gLVFnD4qmdx06DUwZDl2rqe1OMVaCxJa +PKxNcMpwSGj+Q9qChHNDWb4abIlmMQdQBNvHwU5J/WINii5/KCNxqHa1ED0DUmz9inH3os3nJEVO +8lSlaLABKHvtvblx2HBWD7TBeqANNsvg6fS/YmsMzlxr91gVG16XBZpnKjJv0XPn7MCVNGnB/MQI +4hqY0OkLE7+NJEVvgmQGxTgZgoR0cilSOBSbRHMMf3g47rNMdU5Bwm6qYEdAU1f/SaCpK3gElAM4 +gj0JNGXfI6AsrA125KCc7wVpQLHYbyZinK9PaRysR6fg56Nf9kB/OQb1QZP83jMk/1l6Oj6CeLbF +wC1Y/1clIzHugxLBej0qL0gN1gfFAPV3WWkUiyNUuFQfYONQ7gNAwqpr0tNpg0KySTpLbgTKSUjY +45BsMzgiy04VwUfLoBA5fThEs8brZySvhbtDQrI4fbxcSiIpFkM5CffnO1UfJHyapBTrZLtyPLen +6TyaQhFIrJzkR/WFoxgk9OApFYP1QVkwustWOXA0ApJXpCybZ3meVc0ODTu8EgcmjXhnT0B1z8ag +PGWEFD7/D9LDiR8kaMBzexVLJ6wU0wrFpoiLXTXyCG77YIzBGTmC5N2yVDo4q0/DlPNenJwIl7so +86abVoveu/CKka60wV8hBarmr9AIOBE0jMKmkYSiFSFBzLPE1MnwM5CsKzLArUKcVUGQMMjLwGxV +MYgaLp5J1iWZEeXYwSibBBJOik0kxcSCZHulnLh65BJ0Zd6n8ryaapRJIuAS0HxenvclPR2Xh8TE +/iw9HZNBwqdr7Unuo09/WM9gR0DZIdxC1x9XtBomwXEi5CbZhKaYTdZeOTI9pR+bx4L1QM+WCl5n +ATacrgBtsD5og00xn8MGdRkAczItJBxvnUiKTg0JvU2W8gfrgjaUZeFUhGjuV41MB3BxUbAuKA1S +pzjIuUDidRRzGjJJsD5ogw0nVEbII0BjgNqjGs8Af5Oogi2DpgVy/Y5cgzUxTrcanXPTeGUcNCjI +sfK7bWnY2T/s8M6KzrOq8GEH3ZkBqj9fHHZ4JyL6nEicq8jwOe9ErmqCLJabIChLBA9Ey+HNEfIE +mFLhyEcAu/m1cYdkObfjfTnJg8Uy2AAULp5Dltn0N2FTXrGjCLDZ6KeVIk4BvfbH43E+K5KQ2hEw +EySsS6Ij67QVIblGzF6rp9N1PPHqccXSls+iYpCg2MujlC4oXHaZUy5J8mmyWfsv4xFg862XGBkJ +V+HstkF9CxDvtMXOqsLLOSpnB5nTHl15zbjS3P0YrAvKHEWw44dUNf+9RiKcjJIzZkCCqPHONK+i +xit3++y21YurQ2p4f7/cdU8d3h12CPheqQYB72g/Ijgw7BBA20KCef23GR/eL1N45EMA3W8E8361 +bDQi6A47BLxVf0DA+wcUPiI4OOwQgLUh4P2rIlAM5PwRyL55SlCLT+EVbAUNtgiKGatV6+DYMNgi +KLpWn4F/U3o7Q3rozHooioBgB0FRJNgSKHrUiw+fqgRDah3Iy8moAl3zCLnZXqB/EBKsD9qQd+2x +X9ko79oo7x+iPKZFIuTyyAEcc+uUslgJfZ6uzOGUODlJmoEdXCNqcs6cOROY8CGd36vY10NzvHxB +gcG6oB+RIervog0EuSk+KPqv6BlsADqf+Fx5jJeuAluY7PH69LCebUymT39XzylILoskA9A3K2jO +nNlGrDyDT3DrXlqHTvsqQm5ugIJ/y+qAGdT/TSU5pWyYrP62uuJrpA+nSzkdLBJtBhJyCV7ewsVm +IAG2PJ23NQXueMxPXDvetqLJs6c+omKQLkFmvmF3/cewwzuTcpGAANr7EPB+fxXhkYCAH9EfEPDO +pEwkwDcAygjiQu8knZPUA4WX20GVSX5A/KENtgjaYFPuB4oNFuWCVqdJAwmT64hrQQI0NCvOvyGG +JEwLTotbxSCZITJTkCy8ftwZ07BPFzStoaCNFQE3yLmQjk6Nm5OwEi9YHxTB1eYUcPbeOgVtzvlp +UjfYEijcdtnq+eDcIeukPJxpKqed5V1QuHkdS9gwVXC+cqFTrEOyRZIBkjjsdGrP4CDbS3Kahj3x +hvFsxEhasC57BJhnD7YIirME67KX7peiL/BRBDJbListsCyO1OYkDfXoxzR0iRIFJcHf5TKCrbHu +ypvGXSgt0eiC/pESxBtfybg8Qv7kjTpCgxhG6qsaG5w1EMH6oFz6W30GnjQsd2ilIWxWYqTKjven +46bhQrtgXdCr1RB3TJNzSaeglvNNJvjR6p1VNZLUjjDyhVMbxFyqLyoacLStTAc+De881tVRshIT +IdeHm+aYYnZqG6LNwXBphWlxWAQSLoV7hvwuWJ+xBJblButzqtf16SvQbyuomrMET1f5Fr0K+fh7 +Erb/T8Tx38lFqqoZ/KjCnEPM4HyJLIvtg/VBScIqOfVVhDwZepLI2p5gSxwCnJSZaGZuUbxeclok +5nNkhmSZ3qqCSI7qhGXwD+m/YD3QhoeW506VY3KaFgqSJr8mCzkn8iJlE2PKIBFy+wOkXbAl1tZh +VKcFiGCgYZKJBQIZ73HRshPz7BRWtLeOl5EvlIqO4cskA6i/JC5V9QH+NYU5jfsy52IGhoRYoEGw +HigKeNkMNnSr0mzLImxYwxysC8pMZnVOuGL3V3oG64Ii3xmgXngb+o/gwpjW5b6TSBiH42rCYIug +/zBiVkW86DKwoQsKbbAlUI7QrYpWcFg7Pr/y5+NxSoe5FeOUk7B2IdgAFC6VZyl6EcizZTZxNUSD +TTEDwuaHJAAVGmszNukI/YW+DtYDvU1BlQOCs0DAmcJEPmw47sbJmpAASeWpYj7/jvGka6z3AuX2 +xmAD0BsltVqoAs6SIkfnsrABwujZB+uzsjAd6LbERCunlAVbAkWNYH3Qm6SlUwzlGnGuXOOQPLhV +rTMxjIAbwoZYcBNosC5og02xaoQNcWLIyTHpwjvH04/K2WnlQuIsRIJL5cYxsgLiDJuHKpxy27E2 +JOwNwd2DdRmgtFgclssxRZmITyF4i6K8RWxmiAArgpKZ+uyi5BlsCXR/NGZcnc81DN7omOyLjHMS +F1CewQag905fySEi4DH2F6MYszfs0foiWA/0pIJ22WpxdAM20LCSrGq1g6ed5kU3gs0WaXOabBbe +NbINw+T0A4P9MiiTlG6mFJuvy457U6qAXqCgqo5nLz/zrlWYKCOQerCGpnGIMuh/Vrt8zlZfW5yj +hQ+CuDminmbP41oHRlUEuUYQsVnIKy5lH+oWSudgfWyAmefMOcUHPp+RSrtk1BaiuceCdKgntH9T +9ChRB4p3hNOI99pfjtwLaGy1W5dorjUP1r1Agxi/oKcTq5wx3uHs94SEjhhSPOeDhpqR2tQpxiCh +smmIKVbA0DxcEYemhRhgyyw7/1fjSXatJDiJkZOk7uoA9PxzRmyqRjX4/RXm1NxlzhzkHQEvhYTN +FWgQrAeKAnO2evIoL0mb4uYfWHOgBvyCLYK+WnGas9UbmVKLkKv4cf1zL2WFYEugaDVnq++ZyIeu +UjpOZB9og89tE/ncTSc+LOsZrA96ubpCc7b6gYl82CPSEO6cCCB/YwasYcH3T5T0M7LcxXq2cNIn +6tPr9Ax2cJfqj7voGYw9JRHypJtBMNzPi1K6V589EnCv5DXvnRhndgMnPku/t5GPc3KGDL4kIoiD +HXiRgtDLWcQCl1eK5GY9nSEISD4gA3xRz6oA/vG9I0tcJDZV2CgkQZ4GnxN9g8/yJnyYKIiQ8zHJ +/i09g3VB36Kg6mrjo7ygYwtnOyT65+gZbD8o3KrrfSem+VNUMfyGnsH65MgblDOD9UGJ0Jx5J4PI +DZ6nzP7RPSMDzVmLfLdbNx1fqGcLU1yu667foGcLkm0swreR9VaV6tuo4gylF9MmL4rFZu+LQlNB +NLHmvCMJnkHXbUwyVgczJdMi+3BR0n9UOelMJVEYbGME7pis96u0eoaeLcy+Va2xbTTYy1SNUIe4 +a3rxjReL6Ho9g3VBaTjO2erKMV4Oq2JpYbI1sWRnQYt2BzXhM0XqdJWIwFY127eKzzYm8Fap+BI1 +Lq7Qs0URQrvm46nJM2pYJMgbYFuVY74lkamBVhwTxCe2IQ3sg+Md6G/G4rc4KwUJZ/pB4rWCofmy +7NqgKbZ4oTklGgZR650TvDxM1XqwPujNikL9HS+fVqCT85DHEZnwC9YDhVt9a6DUioCjwOfXpVqD +6JqJRGx6aRC9aiLRkxW55+vpdGtQDiAhnKmQWUg+PO5VTPlUJ6NKMiPXwY6D4hzBuqCLsqGnXlFW +nC9DQDWeA8425GADUIRVn0nLCDFxJY31figarA/aYMMscoQSG44Vc6Y/kfxuMfsbPYP1QBuSizNS +h/9mPGk4fcGxMCT/RhLZ/xysC5q4FKcSTmSCSYhgXdB0glqx8Mm5QOKlGTSsXGCCtHGcW5oFZcA1 +AkkGF+a12O1dHYemj7nrP1gXNMlaBE1HJvUYWU/iMUUExK/fNJ5uLE6vFr3q4yfrGew4aONIaVAO +4ncSvCRrb7VBtj/sLKXNFQi6q1VXKKzeY9hZPX/YIZQjmxuEBLF0/66jqz2Xh53qZlFCGf9vEBLE +lMA4IaEMrTcIWQtMxMcJZbMIub3RiMmiYIugyNxlq48DR36wxZeo7EKuUypj8IdIDVI0WI/9+5i0 +cYkbegTrgaJGtQQNPKnU26hSsROMSqzaQUKwfaAI8JZNrn903G3ThrliFoeEk56Y7QvW+xHZh9G2 +YD2K2LRCqQv6fH1dLcBg8U6SX4z5ib8dV/nhYhOsC8o63WDHQXk6HYiccToWr88EAePKHpebx9X7 +qaQeaFo+fBy0sYpA30fA42HM+h5qaqdOWrllXJcfFFNH/fXpSRb+biSF+meQpIB+v/Ja1QYFJ98H +64IeTJ9dDP7SFLYM3ljasXCj/ud8y+q2fQWxsHaXc/wByjOdhnb1pO1lYoS8OlCxi0D6wIlsR/sy +WB8U/f2ZUZlzMwNEoYJcNoqmA6yLnRbEvE9xxI5OCxySGaQwUYIlHKdd+PjInbh1hzZMsB4oHS1v +FcGa2ADQOPWq3TqSDLQgOSySx4uGvnKwPihn3VXKglP0OcaGM7eCpLxbTFJIKBlQamul2CfGrZRu +aylWGZDQNvt1WStYF5QKoLKS/o2Al8PmIv3Dh44znZCynJmTtmEUrXjif4/H728lzXEcSCjFE0mx +NF/75EgKycuulGB9UBaveiaBDXUV8avqVHD4OKU+bFiOdLXiXJWc4IkNpXoEEujw+ihSQGoUdUHZ +exvsODsmXqcSyDFwzpilL8EGoDCuehrgSAg2AEVY9RlqAnfSaJBN6GEEi50NDFU1ETexUNF7SY9P +ynScJq7TMHlly+fu6vTOwbDDOxuz9bmYRMjTIfedyis30bZYO85/atwt0zLjYgQh4Zy4dPhhsdyA +BJhCCmV76ldsh5Qp4rI2vcUgQcqzZQen8oEEaGExSKawGCTAFFKmiMv8/xn3sd+LUwllHxPJpfp0 +RU/HYpAALaSs/+O4YqxGCdYFZZgj2HHQ10lCsC7oD8bRy2I9X5Y1yGVV5e9GYQOEsT8l2CHQdIPm +AJRD5qtuBTiLlZyFYKjJMaXoFqwHimpVPYpOGKAKk3kiUGTBGho09QYXlEAcj8ziuz1Wn4I27BDA +Np69tn/YWa0PBiMIdc8ZjdZcXI8BoQN6a2Dl4mFntVs+Rg11DqhW/vd6Oq0HSFjqz7FzzswbJCRu +2nQ0iGgUfAA0WXP7XICufoQ8abfRa04qyqSYswB1/rPj2fqRcfFMMXfmJPh6YyyOILidHYcNyw6T +s6XZGWxxRfH5X3o6uSDnkvbQLIOi1C5bfVBMHQGJtMVs6K+lCCyBfkjFYb3uk5fz4jT1Eug1Cqov +oOKFqDj70YhJniRztnpLeYmXvIItBKwkbWF4rAxJvd6Ll59LCXh31VI3pXhdfoyXdQW2kPfflHL/ +U09nHyyxeoo+/Wc9z7F6XPrrR6szEgn8tv7s2LBzqP4RwpzkXttXhx8adghaUPzqv5brfwki6mPC +ciOM/SkFIuS++QX9g8Z77GAVgWGMAqL3Vn+M4kAQ2nR0WNJwgyrpH4mJkOvxKXkwnhrsEOh7lI71 +GgxeSOQQRzki5KzvKSeBX7Deb4gl3JxOO+n9gA1cij1PSH5bJL+vZ7BLOH3wMyko6i3I1f+cPk3q +s9E2Qk7yRZG8U9H0V2Iqm+7Wx0lOH7TBx1lrLT6nF+vDXxivSmjcODV+TpLK/GIKlUmKfZ2chKOV +d9nquTEhBKQHNBQvN+t/p1ENCfFuiGEiIwJiVr44shmD8UQ7WBzzZkeP11yEDWOdic0AtMGm2MGF +DeepQxtsCZSrWnbZ6i/F2AjySDFMB22wRdAGG4YVIsBm/uTINsB9hTj+VCZBcITJUor+tJIplmb1 +i320nORR0sPpXeQk6Xi21oql8y6K091IwUOZVAvWYyg1Ra84lwYXrMxgmnMyOySsyuCs6WD7EUir +wVngBRdgCrtylzuNIEfXhX8ueiKDYe+SBo5z5lzSXm7MFgHnzEn+k75wSiFINiq2uEetE7bCOK0r +uKBjit7y+6UDz2A90K/or2DLoElW0XnKsibaBadJcwcH0SUJpuUbIbcu00FE1smUa18a9wSm+4J1 +rxF/tkE6w9BwofvB+FCwHguO2NwXrAcKY3ccX/pxqTDsHIeBhPUqJxSPqr4BR3QLNhT6xM1pn619 +edyYrK8J1gW9YXJ3tMxlkHOpBosUuwg4Q84Gyd5RBzkN8oItgTbYFNsasKHwgtbptkOCbzT23NIX +YXA8WA/0YyoAqnEu8Meo9R6sC3q5jF59Fu0kyM2FAizQauyCR0I9S81LujRtAIq4+jtesJuXF+U+ +rNBKZuuBop2bZcTm3mpEv0hPpySHhO4xYlqQvEJJ8FY9g3VBiXIbswNEtLrzU2ENNhOd/fOK6x6l +eIsIfFsk50WSYnPkNMxEViBtd6wUuU5W5HCJFoKZvOKMG6daxSLkhmfLiO6CkJyIbNrIsfCZxnUa +flI8ywXRAD42YwE3U7TfLYf7qp5eHaSsvDNuPYNiUxlg5Rvj1WM6z7w4NJuTPEAJ55SikFykAum5 +egbrcskZgusCWhwjUCHAh5ybZjqLa10hoWWYSJafKS2SNstoc3assMr9WFnu6+KSSHqgjTiVt4HD +h7P22LzmFJOQEJfUpiz3DaUtJFw6EGwAihmq9Z7g3PvaIm1pHv+dUi3YEih28EoOFPx/IufqqhZm +uEqJ9Ud6OsUpUjg45c2RZAB6QwzCZSLkbjgDCTBFppnB2anRWPDrDN1jkafIjZ+hZwsjshKSs1Sc +zu3Kt8eLGdbueKmbkZDDHVe074ykMELFgFewLij9NK+xBhuWO0EbrA/aYMNIXAS85LC0AaBtaENB +53TjylwGMISL1/PI2TRoGI2JQATWswjMR9fognLnkhOBnMvTNfjsaZvTsFbNqZYh4TT6YVI2RkyQ +x4+ri96o/x3XRwrnk74jkuCeEXIpAKYLO2bFW2SKvZNLNuIHzJJY09CwopJ1e06zFtV2JrVmUAyY +Jv7AdtEcDpz2NQIGu4J1f2cUYpfKO5wsnHO5UqTBBqBwcXP1rnFlWmxQyEmemySDci2iJ9nuMi75 +XjEnFFu5OckDk2RQuLSRDE2qkeNNqrQ5q3WNEhKBQgRtWM+R2PRBG2yKVRJsqL7SsHQftMGm2PSF +DQkIbbBF0AYbbluIkEeKeZpX6ItgPdC0ma3YGEMZxkkSSXE+ApKN1iwuqIGE9jPxDNYFbUS5ON20 +xWw2RmAmd6Cdfo9YFEy0wxb5M/fhcUa90x7EdD0pyaJap7aGZIuU3SJrX6kIfEBPZwR2fvd4uZW2 +r3VBOaXOa9DmbBo0xdE7aC5SpmzcLALaYFMscdazGDwqRrpY8uYkmN1p+OckaTNMcdwwJ5lBse+S +1OA0HOY74/7Loignm0OCf7+9llGdjVusASBhcypXELSQcn81BC7UM1gf9GkK8qplJLO0NSlb9DtI +6KegdLD+qyTxXXrucrYCwuaE8hOHigbroTjsqvEcGTECVTBsZrA2aZmW0xRbL0jh0+9XF7BFArG6 +IC60nWxZGoSNjlaMtyCPPp9+R5ZvoRiHzP+TEqCFYnRJ1yLJRJfmohSS2ansMDIONUUuMEX7Hno6 +vTCk8OlnJ5OsnDMqBDgygW3EjsUW7joi4Q6AtNSjC8rJ+FW9B/5qJa+TfPY9I87AESHBuqCMhzrl +Ws6F5lKwAShc3M5BpgxZyrHMyt3G9WfYoXHyxRNjkhSr2DKXAQzh4s4a5Xwg8uazoaFhgcBgPVDY ++HMBssTjlIIQhxnjwB2rrAP3RjMlmu0bN8cMXSxpIAHQ2mnUQMKnpxvBM83gQIpXPLwFB3CyINZh +k/YUyba9JOv3GM+hH9L0qxOXnIT54WCDB2lC8gl6Ov6Sc+GACac0yUk4HivY0kbBxe50zqUxmMcd +Jf9dOWSXrd4b/O6xnFpE2ENiLBdBX6ygivIdwm/TswoD/6bCnLoLPZ8lwX+oZ7D+54U+VC2TOVv9 +zFFe9ikwWB/0npJWf4cjA7QukPcRRT5FuAfal9w5W73t6H3EFGs4lT2scz5OzQfJn0tNnOwsG3Z6 +ww6vuMDu6nye3rDDO6m2t17MX+9jwCdJosZfa4oKqdP4iyASDEFSK0JuRY6YuETWrLacDav9D9yl +z4DAnmoDb7VJgYCrRNGp/hhU/xDySv0TSaJ0Qa4EHthiYwD2JjdwWUJl78Vhh1eSo1Ptn1g9eXTY +wTAUHHtH++Sqv6RVhFy9XSprHqGns08F9bi+nUxRb/3hhWxRzdPLUxueXz5pG+avE9GtegY7tAmf +y5mMipDH6zrRJz5Lm/BZmcjnrzfwKY5xEgVMcZNI65MAeUn67NtEn7VZ9IEPhU99QikvB2NJtAza +SI+rJsp7k+jX9WxRXLxUn16h5yjbHRx2CHi5/qj2MO0bVv8Q8if6pyLZV2U7AlCjzpD1PzHlBbkD +AJRSp1czkrOmqJK3l2RbMyFwerY707IOMEXC3F+l4hSl4/aSnPgX403FdI1jF/SpykOOq+ZcuMU3 +2AAULl5nNGfToCku3YOGaKSjgoqNRUi4heJbqoLrco6xN66DqQNJaYCyAU5/LA5v09NpNkFyN33K +muVgvW+pfD87Grw4OAOXh4gLIy4hrfyi1qhrC17eLhEtFPx5fbqoZwsSgDSfMflnSErg9pNMbgmn +lXHOBC4sHcRnZnWgM40PMJuNF/4lbjmCFkfE5iTsjnc60JDw6WUjYZVXCYtAGZSTvExftJACIM0h +sfuMR//TujXHqRRyku+NBckAFC5epQAb9uh8R1pX5ZFKv66ewXqgH1dQvS2fl8YMxVC9w7cpHvV3 +vJxQYCUiA6yea8Vq1WBLoEitTyDghZgHW0KFN0qV+jte7pZuhLn2GC/nKzDYEqpiQWd5Za7qT8s0 +wRZB0a6+OQu9EFEH8pLkMWYWoWSaZoJlkJNgnUF0lS7oexVU3wbEy7oCnU4EsYf1gyMJC5Ei5Nq8 +Xp8izRnFQgra0ppuodiVko9lvbkyxLAR51OibaEZpyj8Vkz9g7g5kp2rtxD8TQnEYYJdsom7vGcT +dykeIgBrpqRenRRk6ilJO7iJtPINTrB+qnLYzXoGOwR6UneL1GMOvFwQ89mhd+pagwt0N1b9nYwQ +IXcclmYid85WTxzlZSUqsQSaTorYD4oqNSkv50a99qPXjdIvWPGmB6yAB7DMuhoFOjjs8Ipj7K6O ++zg47PCO8c+qwoedPAF214eKDDt5ASHu0UaC3FRcaZSOU50Yj0284mmy6fepexXsICgH8dTDm7xc +Jb1aZE/ie7VEBFuijL5W1YR/fYIqzgWp9hk9nUFiSPC5rybfO3mUl5dLeIsocMPdfSPJxKLvn1T0 +fU3PFgUM9fMUZRKQCqDidCE2QcoUlcoZS3KJjPskPVs0v2ZIl50hwchTpD4rVSiK68NzePl6zG6H +QBvZrTzMip9sVbY90/jMYOe2rWSiSkl+OzSJqSMbKtyySVuiOLJD6mMlmuJTtMrvRCrMUO5vM8kD +xzuRHPkdrMtq6+er9VedykD1AMS2Q8aG1nCwAWiDjar6BJPZ9DdhU1xPvfag8UixTTpYF/TtilSw +43RDro9lfrHKyxl/WlEINgCFcdV/BmdUs6pNJARh1WdiEgFrIO37pCvr0YL1lhXEZtVgPVCEeskF +Z87XRPlgy6AIq5Yb0km/XMpXYeDsmXQWxiAN1lxj5LSTcpJzFd/Kghnk1nqp/r9azxaKAThH2Co/ +Oc1kOnzBuB/PRSt0QVmfFOw4KOuPgnW/IIJPKOGcoaFc1mtllGADUARUPguehA0+KWEIDTYARX5F +Kb4RSL9cAa4AcNq5kDAmn05tOrBRmQNRoqAkmOjueMy3Jhps17ivkiNY765K/gU9qx3X4E9XmDOw +hpmBO7iBgO9GI3rITBl1a9wVoEAL21i2bTxFgDhE2FAoyEbp4IHiGv48m+xEjJhk5F6+FrmYJeqJ +5IyKEXC75U7gdlPgeXK5VLUWlyPhcjOQADvhpVtU/5xZbO6nivOBejqtWZIJ2AmbH5VaL9OzhXov +1KeJpFgyLPzQeMuUjuFeLabsDzsEpfOeu8PROktCflGOXq0EqxdgEkDjslo8NqivECCkucyzJsGk +AOV3riCEzqQcJDQUuQXNKVNzEi6jcUaScxLGRYItg/IMdgD0fjKV05mBMQdlQVpthQHHwE6bGjYc +ps4GamfRaE6SbF2cOMhJuP3BuW8BEs7ivUBJ3iJ53qtP06VWxcOwkYLFpnACSKZwAkhaJO/aQ8dz +3K9FjyiPVoiErfbPlBmC9UF/QlzqBSW8/KwCg/VBkVZ/JzYRyHPI41xdPNFJIUjYA/gWMQ22HxSt +6imdTfS5dqI+V4spHL1BBJmMhEG4U3pCwkGlkMzZavl4L4heItWYcW6h2gyxIeLEqkVsICE2wQbY +/heiexTnrOZ/eNxZb1A0g3VBnxVX1BX9N+fyvshlAPoNcfEGFnI2DZriEkZokLPJAiUi4GYJ+KxL +dZ71EkZe2JoarA96oeJVf8fLExQYrA/aUKK8zhslHqyEO6Sn4wY5yYcjyeASaYFox2XhwtkG3FNc +TUQvDzu83lPtjqodUG8hIeDx+mO3LdV7Onhn7PFsW73i2LCzevmxYYew12Y0vP9fhZ9V8Rp2eMW2 +e2z12mPVanQsu6RoIpt3oq84KF0jUBYSddKYZ52gvJDad9qE/x25DQ7QwuO4sPb1InVWUWBjjglh +eYTTxoNkZxTbKp+M3iXInezOJudc5b176+k0V+cfMV7TcH6VN/uR09CSrPec9VmQkiCaW7I4pRE6 +p30Dyb8St0SyCNrQuDihBRtoWhyUmJM8Uho4xfWa4gewvtXJtTkJjfMppHAqpZPISOF6biabgw2e +LmU5gamFYHbAJhL4RSDFEQyw6dpJcUjuIxqcxekSQrLj0Zv/sfHMw5mPToLnJD+maDp2h4RP2Rnr +WAQSDl9osRp6PYvLX0bFuqBp3rHY5My50Fp1ogcJmTeRFAeXIblCOiaSRdBP6C+vHIPNx/TtST2d ++g8SrtnhhJdgPdCHJS6gHMUfrEduZWQo2BIoo5TBegyhYBMnv6ARgAFaWHxeNGRfx6GQQozQ2Sl0 +5n9iPHNcGBskXVBWDjjTqjmXx0YuA1C4uH0YKcPVKn+vqAfr57uWW0TpOVICUid5EAwQ9TCTFR4q +NmmnRjGP5IKxVpjNcO+W5FQG7DtbVni8nsH2cWTHbSlIdBGoGtCPHuKH9YXXXlUinkm6AKdp3ik8 +auHC8XzF0E+wLugB9dWCHadp+8YUhMoAKQJjLiVk6q7qVi5XXTsC7qLU7VTLp+t/CLmX/uESxd6w +w3l8HJe7pyKquBHALGriRsh50rnBjSBi+D22Wl1muzDsrF53rNzyJGqsoWJhvlNRQ4JN09gnzboI +uQFnIAEg3bLEhOEU6nPGEovrvTFz+eIMJDMo9g1ZCW9zil0UA5C2ZXadwZN2huRzyoJf1tOp1DHS +FIrN/+R44fNZGdjJSDkJrQ6ngZKTtGltSDGudk9NruKmLkierDgwDh6sxzpuJj4cT4MLsD0WWXj0 +uN05i9+xOyTcXZVIMGcECrDDmZRU2RfLvJykRermJKjjNLwgmSJ1IWEShdPsgvUYxv+IDOCkLlyA +7bHIyk9NnbqQTJG6a5kUjtcM1gW9UfEMdvxXVX5wIbnTRs8Zcymfk8NzEtrljg9AwrrGdBptMYdD +QoKn6A2eoOg9W89gA9A/VZB3ROn6Y8ZTLF2y2GWl0xPFJliXAdZbU5AMHYEsCOO76x9aa8F6f62g +h0UuvY2Mi/2BnPF54hZsEZTxk2ADUMYcgi2C0qIPNgBlqUKwRdC0lHSA3qy1CjbYqHdxKizXG5kN +8Zy7HmwZdMvFY6N0GkOfpEmpSkkW4dSpU6dOnTp16v9zdy5Qll1lnd+7y+6uDuQhj8hLpmDMBARk +HCeE8BhKIQGC4LmPkuquplOJCySggBKCA0xXMTAQIJHclDAISWggQsAAkckMGcKjeDg4BiGKYIBR +a6EsGRyGjmPGwDg4a9/z/+197r61v3vu7eqE8Vvd63z31Pme+9vvF0ZQOpL81kiDIgyLcKyRqSDh +Hs2/lh7edYkZjvsy8hlcuDY5NQOX6PGdHCPwIMeEkXm8Owi6Fr9aAqWl4d1B1vTfnL76vNDkUEYm +IuTZBFPJdN713zbGpVg+YCq6JMETU/IySWG3QIuUnCHxibo0Xt15qXxB9O1YwcBI8IYEGA1H/IZ6 +lBbe9VBvxzPgNiktRSMQHJtPHC2cH6EvvKsYoeWSVaP9VObSg0sqDIrFV86FIDAyICRIYZNJCxJg +ZyyeQlcOr2fpSgtdcVmLsaz1J40maLofqPozmZzGbilbIxAWOZd0/VEPLql5h3YRci5U0Kmx06dF +cqGIvOuTdr+XXgmLUGLMNcxG6xSL7ilm5+hpeB8SbuFKWbdY1kLyLHFPNRF1cYTcljfoL7jBKFOQ +8jKRpB770qv0ilTybonx25TsE3UBji3ZSduU9fukbdJlYhp/XsokkmKFg1+mSbCzR7PLPtVVRmGX +k6QruMsZQVJOlS0/FqWUIykjoTvn07W/3D/kXectYny1nuFAWaGNc+v/j175KL57otCkUbFVnGuU +SMppKCMoHZlj867Lnr/EpZys4sKECmWHMdgGCWebppGG4yFlCsU2zxmNN2bkjS5jTsLkmXc9ULhY +M3hlNt1t2DCbFoHiavHJowZw+ZNhwJGM5Gli6l0FSjpNwQUSayoLyVQ8Z0TJfSYXKQu964Neoq/C +amahEXADnHMao9xYfcqo57gTqAUJn6ZCoFhzzyBlM1PsX8haQzFI+BRvtiABIDVIFp466jGipRE4 +LKEzAifnQgngXQ8ULlYs5WyeKUNOdt3htoGljfCvvzHPH2B6Lzc4ZWN++P+5G/PD/wvl6ZdcDFx2 +ucFDJDECwQgNV5VQZXq3n6nCb4nIu/3fFpoaTftZcpru19/PesErVEF4tx+UA4LDUVXgt2n+yccc +ESHX8qHieFjPcIGZULiFcgw8cS7GPPb/rdj8ZFSmAh3oVRioAk8He03kTCsaD1szrwraJ0oZvObd +AVD2uO9yg7jfHd29O/B16YoHWgh7yp0pjNTCnBY5+E7VD1eS9n7HwuB+Spmz9DTa6ITBziuz/rOj +pSLD2o39ULy6tzLh8Jjr4R4m3tw3/mW4Uaq7Mc8b+sOJhjcsZguT3ENm4hGBnJ5rCOt9YT10f7i8 +mlfUzPPhT8O/8AZpDaIoSJDLgymkVt9OTnywWCHV6EEeeXrt9weKhNtMvOuCMvCyyw2YKYuAsrCh +Nwmtdx3QD4goNEDAYd1CQdgkkmL5tv6M2iaAw9WM6jQn4YYk73qgcLGqU9hwgye3Kvl0wRJs5tzg +9+jrRMCd8GEQLa3pits/XqbCMZxkOZEPJXzi0wFt8PnMRD4Q7ZFwo5DEBAYLGnbfWpSzWNUpx+q6 +0+QZQw4kzG4mknLHVVIAego+Xf/4fv0tBKvQCDGRMjYNmnKmFg1tduayvOuCNtgUsxtsoEljJcVM +sdipXQuk6dDWJI2T7uEC4BOksM6aXGtk8JxkCsVmkMI45fukuVHZoRj+4Qoy75bp8nOdtXfLZPXE +uLx3U0lBm5oZaqOZudgdTb3vxwzINECEmBQZyav1hXc9ULhYRVrOht1zRjWUk6RRySXQhmQyboTc +AGjSRlpSIkJOwqeceeVd9WF9TBHW2EmHMnNusFUumuROhpvYCxYKDSXGR/UMx3JO5INVDeFHy0T9 +0eRvdDNBn6DWW+i/yNQI0T0ZmzP1hXc9UNiYmy9zPhBZ99NB8zjJhMa8jA4iJhnT9GF5iPHna1cx +Otco3CU5Am6BhEhMJMVyDhIqqkTy/59im/IYkGyJtyFz1IkVXDkbaMwEhojGLUThgCX0AUiuheWp +UxiSKVIYkilSGJIpQg+S46vYqjxGyjIDYrVuFvbXXgYor4ymWE6SRraK+SgnQcoenXXBb1iFTaO9 +8jjTkUxp7riGHb/ZhiR2/1pGvlxPvebnG/UaLvzmuTsouxFfU/nwOb+pjMRdTCMQ3rkNNCj2hCG5 +pbLt6wdGE4z8gxr8ps0orZkZYbpjgna5FLgZzYE7h2RhZdR8vG10+nISFjh51wOFi9VC2sokk5eN +nJKT/L2iwFrbIymUR0SV0YDNSdLewgOgqbfUJT6oJo3WKIyBKdSfQQq60mr1ro/x9I9bOAFoo+vB +0VA6Ve06I5RyEuY2vOuBwsUMJUmGJk0KkioRKC4gASCdUTJw17Jxh0ZTgDEY7yrQD8XDtIs9oJwL +nW62KfMblrvD3Zwb8/w8W6facxsArgFIAcRAx14II/9DwjgYR04ai8ggYc6DqWbvVriS/MXqAhhn +AK3+wqhj2Wdo6JqT0PkyyipImEB7kVxmmAcJR+0wgmaUQpAALRTbysx/aPRYMYhykm0yBlzMnC3J +kDNJ5131WhUvrCj0rgK9Xn8yFgXAmBXcd8gd3lWgcAuNavDEuQL9lISFz8B5encAdHf0GU2HCOQI +dAIwulEmTeMz4K5ls/Cc0axDk7Qxb0sJEFbPS+l0K3+xBM8Z86F3PVC4WRHmfnFUv99Qchq1Vk7C +kg/veqBwaSN5RRKhMWplJONEWp+N2+oZFQuTm+CsDfOuw3oGVtmHz8C7UsQYdUQB1jk9KpJUoHAL +vSQ4coyX0dyFM+SQtiD5p1IinZlGMRGBLIYUegdpo1IfFAWsTjtsoElLiirQBhvG8iOgzZHnjsYe +/SyjWoGEjR806Qw35STpWsxi/xISTuNKQ9hEXITcFhSC1AgmpNBbuF1MW9gC971K+hZS2Gry6Shl +CZSnd8ugifESKDEfAhv8kVF+cQALK5mVIOW864OimlHRw4UBR4o4o7CABMFJ14mJOAPJDLYwZfGm +mCITFWO64BXtSVBsirCfIYanCMitC0dzfZoNqUDTXoJyA0tcHi9PMLdiZB5I6Eym9mWxIJiBZOH5 +o+Y1+vUPl66P0dMo5+BCV45VjUZGh4QVv0hrQYJCSJuJpFgCzKDYZubEdJoPGT8CBTAkfJr2vlSg +HxRRKMGERthhNkd+aTQKfkyCjCQvk/SgppsTmlXgjf4LKJkxfCa5EbATaTlr76pt2BTdtfrLo3ZS +gnlXgdLkMlqUcKF4e6fUNeIQEpI7LQct9i8g4VOGc73rIJBLHXe5AWcfRcBzsCESWfZqlDyzkLxg +1LEsPDICCBJc8nSpbikmKVSrySUVffxGMJVdIjaQJ8k4KUL04vQkCy8cdQkz5YZLIGFpE8tGDZcc +yaT8hFQ3pJRJihXL6otGbUnlewV6RewxXwn6O/FV2a9izFA0sxFGQwmSx4n7z+nZguRaOedWPb3r +gZ4mLmFhJ/iT9c67HgfMYFr4TEwixFiRTWeIHAV3u044ESZ+L8jJOHAuuaLYP0ISO7+5OqmFK/j0 +O9LCu4qxj6/HVvLEZIMLZ+Z414PLbZHLmPOsQTX5DsZppfNEJ3BlHFf6t3AC40TXRV0nSplBMUh+ +EJzEEeJnRYs7JNj/SK/yaLcqttkT7AdJF9JoinjjsoHXRL9NDJ47J0RnUAzzdzxECa4/iE6inRWB +8m/zotE6Jk17FYuhnIR2gXe9K8SfmQSjzMm5pCq0y6WoiUsxjeFyqgRjjHcdlsClLX6dccbFdiCM +USvtJO5y92JiPFE94NhcxRjQF8XOKCBQnx4vCWO0aSDZxmLJi5AHzzQkL6mT6HVilnZbVqAXKmpD +bQ/+Ar0zGlhlzr1tOPfHORfbYTnnj0h377rcWvX9+Ko6Tapeq+cuNziov0aIDpQ34IhS1nr4zZfW +HqQJmK7nqEDJOFYPBTasWm3QMMISIaor0dAQGFaq/FqtLcB6mBYkrxdNuuypw36rR+rw6XDdOy23 +CFFbiWb/VoPoMxOJ6OwhfM4Njr54G/FHJ3J6iOLgJjVMw6r6iUQPF1G6Z6oLyvGEYZX/i/lxkZiH +60OEcirj8Dt+pM4BaRchd9s/SIUzxc/YVYWnN0QCaVj1OtHU+4p/klPMiO4Vo8HEJLARTGWSGaQU +fYYUiltOEQ378qKDBfgZGiAZUxSzldnPen5j1CQnSVKK9kNCR50haMPLR/5NnTAPkjGsnveuC8rQ +sjWutni4ZkPRxnWxRkUHCadAcNNLCxKksJnzh9zg6Cc4qC8CyYUgGkgf0xfe9b4mNMku+nZxrbYQ +uEk5xkhBSNj/BInpSIlhTooK0KdatsGm2DiEDSsmGjTFfrh75aiFaRVfsUGZk5wr7xitx5zkpyNJ +sUUHCQqRy4ywhuTN4s5Eg3fVX+jV65SAc25wK+cTRSB24EPfq0H0zYlEpCESvaveK6GMdVihgOxx +NsUgheRqiUFcKM5fyg8uiPGuC3o3leTD7/jBsI7R8kQeMEX6s5aA0Xsj288gZfVVo7HMOPg+dyW7 +QHn1QPlq+KfB8BBizAGIBrj+pP6AuSEVwdOOdMI0Qs6Go5wYZjNGYNZfXdvDGO3bxdS7CvResiM0 +vvXXCEiGzbv0FwbCvKtAG2yKpQtsGAxMxWcF2ii3JCtCrgxc0mltxfhGMHUhB9hZLfXF19SuAyhm +Tg6hUEdDfRsif3ihHHkPfTA8xyCcZ3BKeZ01Qtg9z5E03lWgZ4rtLld2LGxYKpEmH5ZAOaPMSubN +146anFbIUA5GIDGOXDJKQsFplLE5CUfF+XR/SGqAdEBpTXjXA02ySIIIJfXeoS/CAh+hEaBZfd2o +SZxCtssNmAmNkNOwKaJBU5SzlclJN3EUvZ2TMM1t1JtlknI58/pRB+Bvo8mSk7CRzLseKFxCoRf9 +J8CN7g2jktMivaIXSyR7w/rbkPXgwT6VeRcK6/AXCY+Qa8FcalpWwRxohJyE2XWe2hfBz7NEuCds +8uhtzPObFoo+11cRkLKZuYcNLbCLBIJJdL5s0OKloynBilAjBnISrpfwrgcKF2sKemt6yTkJ4qaU +vH7Z1DbnJDNKXvj1qSXnJDNK3pxeck4yo+TVN05tc04yo2R3+dSSc5IZJR+ZXnJOMqPkxcHUNuck +M0reml5yTjKj5MWN2mZK33PVWzFbfaKhZE7dzGINBAmlceN6RdATk+SJbBg+gta7DmiDzVPAMcro +B6HgM9WS5EjlMN+uV0jwrgeKgLCCX/VJBCoWOO+Q6TvEhhn9dLpAsWOAAceX5Mib6kBkuzor0K3B +woU31zRAWg5agX5CiWfUyXChEfYJsTMajJDQl0gkxdYPJACKmfv+IaKpnOT0Qf9IDIe7vvnxhmh1 +sZ8Ja0jS9qZiIEDCduu0JGgiCSscMcTojm/9+zpRkfJNGehdB5Rz7a1mMmy4cgLaFpIZgmAt811O +svibtUvYPJomTDuglyvFra4rbNjRCq13XdAGm+KhFltvrbUBvi3RRhaDhHKbOX6jQIaE/vVXJMXq +lULDUcHQWru5rqyNIdqY8POux6XTHNhklAdwwbPJvgqUXSkhgcCRYAwEwJmViRx97F0HlBzs3Qpo +sqIDisyQZ8CX5FPvOqBJpRXQZH9xCAEtGY3n6d0KKHp41wVFjVDEgyPUJ/mrUctibxz5Fygm8YOx +MXPr6jrhOVGfBbTeHQJtnI1RgTOA5d0hUPxjRmYmjfk/7ypQJARvgCMinKMvLzSkkZsi0ORAGoeB +c7GEd13Qk8QuhGOkF+Rs7qlv2S1lhetdYidpiCuPr1f/MUpbPVLnBoCxcO8qUFZwhPjkM4CIgQ07 +Z3hqzIafHLvEmAy/OVVNR2IwGvVxidFr/YqQC6dc5Cnh/KRsQDi/WQY1QcrmO0Y9lY43KI5Ibr5z +lOQFUt3IRzlJOoqh2NTKSVh+5F0XNHEplqRwob5JJMXCf/1dtXlM+rBQxjAPEurLRFI0DxI2BSUS +2pYRCAhIqNsTSXHpkbumtgXgmCTvKuZFblc/1Wju5FxeK3ZGCwISdKXaNFpIkABJSh8UXa2qCTZc +GvVolfYtJHO711ekgndLzFGxvMi7JVCe3i2DshPeuyXQK+Veq2JCZRoSsGsIS1Z0Qdkt7d0S6PnR +1i7oeny1BJpWcXdBG1oW28lo+dviiIVGvoCETy+TM1qQsObu/pJmtHmRwqQS0eLdEiibWcPABjjB +ZOw1nZKzwiYCGRc2hCRT7i1C8rEyHw+2cMM4CfOBEXLFniApxJGxCgxbkNLGlveMFkKUCVYUZCSc +jh4ulpOu39hbm2PNK8Dme4q7vVpR6F31HR3/8l09p1CGbqt3HdDzal1C1/o5Qqntvet/Uq9Q3bs+ +6KKs8a4CfbmU9a4Piv6hEAHnGa7fk1k9PcMtekKv0tO7/m8JPSVa3Aft61VoC4F/Ve+864Piq6AG ++Ek65Ma7/oLQC/T0rv8CoTfp6V3/00J376m94l3/AqHP19O7Puh6fHUA9LL4agX0rfHVAdBr46s+ +6C3x1QHQv42vVkBPVVxZeUDRyco8EtO7DijROecGR4qrzeDDbS+nx2Ao9mEgoVhEnnf7QYkj7zqg +xJF3+0EJnzk3ePsaP3h6t/+MsWjZT7QQDnNucPUaP4iHIT9+nCf3Dr/kR3JwuVySg3EIpnm3DIpp +oUQHPyXmnGVQLAqfgfP0bhkrt2Std8ugv6I4CJTg63pnlcJS/ZeUlvjbuwOgyJ9zgxvW1iQZsd4d +AMVhw+/qzJIgFuGS90bJ+56e3q18Zoz1yni2WhnPHez3jJDL2pQMDPLuIGjDtuvWCJiGIdet8SOF +Qbn1KtuwaSGm70QVT9en6OXdCigqhmNNcg3Du2i2ILf+eWOsq21YF+d8MAo/UGCFXYLKLsk1E7kQ +K5Taw+F3fpA3vavOGyt1J7K+QhVA4rJ0+RiXYmcFMz81xmX5y2NcJhZ331Mo7xI3q+WkoAGoohu1 +dRro74PSdguFpig5Wsi7PiiR2kI8YhHgXQc07TKqHq7e/6P1bMEYLqjZgoQd46w4MyYA8B6WpoZN +B5SzE7yr7i6176NnC13gwtO7ZVAWXVuLSJW4DIxSLE4hmPZZCyfwKZV5CxJcnFp/y6BPU6kZ1vmp +HrlDzxb1CVx4ercflJuoW/iN9ez/TTE+hWAasS2cQH38kmhx/wRl4ErPFlxorEPq3QHQxKX6nFx4 +q54tmm1w4endCmhiXBxRIgRPkiU0IaYQTHu9hRNw4ZHkSqjfKgVacMGs1EFYAU1cquvkwhv1NGYb +cAJceHp3EDQxnujKZ8oS2kJTCKaf08IJuJDTLhu9JGrRFlzuo9o/HWBQYWnqax0ETYyrS+XVt+jp +YxkbIW9pwIWnd88CTYwnevcd8i7NwSkE02Vs4ZdvSMrnVEOHriR4akHEDmGj9xfNF+ReoO/Ic84N +rl3jxzfUlvCuCwrr4Xf3UZNqQc/hSwmKUJIIpxZZ+xPS4+t6+ljBRShJoUPcwsmEbqMJu0xj74gc +710/dr+iMhOdDPX1kct5XtSph78MWulP3p0HmmRNbMxdJBk0ML1b2iOGj9RzuPqDH4n1xNbmPUT/ +JD1b1G/YxOhFi2TAWcR3Y+yDuGzB5Qa5gdwx5wbXrDF2cnaK12smxitESfjENHieHPQmPVuEOFIY +02lhIgM7F8sa7/qgjOZ4d4Ae6G/Hrw6A0iPyrg96S/oqZi1BnsOQkbpYS+Oy+uOylsZl9RGb+mYT +g3Gc8cRsiMYMbHl34HrZiwI+jV6lVyugjGK1SM9xkonqjZNMTAMsSWlw4EuyKOkaB9/SqwOg39XX +3q2ApjSYKB4uiWSikeMkxdGG1Q+OjmyniaPi5GlOss06V7a/WCPbi9ePSj5Z7cM9YSNSb2Oe36ep +tbE3vB/MG7uSxJD9wOztsvYLiISTKs5VWTIsvJUlI5A3IWJ93sekuHc9DoamPTXnBpesfVXtigbz +SyYyv7caadvcdfk0/WnODf7sMD+6euld9SJQyfWuAn2IXs25wZ8f5se6XvrygAomv1+s90SSPmjD +5K3DmPyo5M8tNgpFyP3JwQMNTke343R0IqeXSb1L9fSuAwpz73qf1F8/q6d3PdCnqmLzrgP68/FV +F/SVejXnBusTE3QbncYV6KBAR6y9q0Ab0u44zA9q3hZpt1OGHJewP/Lh0cKAsSfDrpwkFUPFFb9l +kmIXZCtTjL1re7QXit+skthd37sag1xArMOO9TqsvzHKqPUba9ewzIKzC7SIhp+cX7DXhd2b3Y15 +XrBse374h8FquQB1/7mWBDBEZSRCTsJCPe+6l4tNY9dquU0nyaxxSWyKa1w2P1Iry97FtC6kA8ry ++LDAUspEIEFgg3eh9a4CbbAh2SLkbEgNaL3rgjbYFP2weFNtFMAvIwVyEsQYEZWTMKQ25wblcyXc +R2tlGLRlB7h3HVDOfQqTq1gA4CjYsM4IWu8q0Aabor9hQzkBrXdd0Aab8kSBjGI5FrTe9UAbbMrz +KmLDmm1oveuDNtiUY1FsflRO46o/K35XP14nC/B8IT4dQP19VdrG8qgylx4M4WIlbs6mQVNMSWje +pVbULXoa4+GQXKpPv6Gnd0ugvyWb59zgg8WqGT6f08eoa+Q2SLjKF3lGboOEc95w6B5X35jN77cp +5fbV5fSNa+WCGpY74zPJjUBWRcoM7nm5UgQ3tXAPJFeJ1LsOKOkynBmMagqitp8czQhkaSsxM5J0 +Vzi1eYSSlETSBeVE5pBr2XnPMujwDpwSwZjzWP3UqE1M4jXuW+mqhW1lbnGBGhKzkwMRa415zrnB +u9f4wTJ077qgDebvLqeVNKJpAr9QtoDTzPSuA/pkhUb4DLynvBs8C/7P1aAP34Hv1zvvOqCoGj6L +CS2I6S1NaQqh3TAU+cHKfe96oB2pOvyOH+g3XA3BDxQcfskPVPSuB4q2w++kZYRcXVrAaBhWgIDj +Te/6oPAOn0Wegpw1wMn8Vjkt51H9okFYogCOCt4tgTa0Kda4sOZyhb+UWt7t/xtFBKWGd/sfpt7U +OXp6V1y3A+OfUQJSB3vXASXlQsidrrC6lziHd1IlQu7BbcPhYRmjYYxEFoKcEyt00GjODd6zdtM4 +p/dM5PSfZO7NelqlplIVoFiZrVA6jn4Gjk1BYqcRleW2zI66ZhrJW58drSk4+XJf3U313Q3PK84H +uduwndHb8IOf2/C8pON1wvCP9d9495tyaIMnr6jLRnjy8o8UU02eYhWByM7tYKizcQUD6oSsBo4o +7ypQpIbPohhBLo0rQ5K0YrfzrlEQSArGGylwQbATHB/M6A4uuWi4sBjyd40/ptFw87/WWeOeciLt +3eAwvYpAYCz8fk3DBnTyjncV6O+LyDq71t1cswHY/WeUrjlJOlqyx9FnKyrgw+A6+LP0ztrPkinD +1i7vuheK+iV6Gk3SnAvnWXq3NM4FV0fAvzkX2sTedca5FPMiXLi5jZ2ULdzLsexsy9dgGj+/oAJq +rxtUwxP0eMHZvPuGfxhcv1bupKEdm2UR2cK591MT5mw9veuAfkhJFNpQ0a2C3Ls7xAbRPxu1oUEX +IZcMHFu67lYq3E9P77qgXb0KLVZwDgpp4eCWnDEDyM38iLQgOrzrgP6wnDXsZPHjVL002sxETVvW +qAbkKq5LJAMi86FBMIxb3swQ0j/4TDc/P1rycpjO8NjL+gBEXnFk+92YX7QOWYPva+RvzhIwihxI +viQSuoje7Qf9KcVRWHGkryKQoIu31BYxUM3ha951QemkWjXbZsbmsRJkZJqFP6wlAwxwnBxCqS4e +a4/yB675PiV49MqN4aRtmLjdmOcvD5LBYyz4A3l6nAVaAPgHLdkUep0+MIaeIOGMv0TSBWW/m5Uy +sAFww5hx/AEvjBvHX3DDGAv+ML1/3BdHU5Er5Y3IXc1IGOHy7ny2Zqd+9/nsM03br88/Ucl8np7e +nQ/6UyqXfIzaCKQo4veJ+pN6ttCYbU3osssNLmB4gc5yeBdlCnLRHGvDBbA+HpcTISf5aWmJGCNj +YeBFIvmwni1IuIQ3SSm2kja/NJrwTJkM11lcsDHPb84lnbjOYvXLowyfJV94V1F6cWyrlVTiAjU5 +zjJeJMBtQubc4NJiRbj6J6PaYq53FSjTNsYY6mbGBa29qziTKN3AWWwewQXqIzLAuy4oZyJ4V7GH ++odjWExkDEBie+Yro57h0Lg9oVzvbczzm1a82sjIAGIGyNiRIlZLFRpENWiKQ7JbmRwuhzshqK25 +b95xv8EJbmP4J2v2G75vlGGc+mnE8OZXRz34z2KhVlw8lZPQCvGuBwqX0L+TLhFwdplNdxs2p0R6 +Qc6GShPR5kwwspnafIaYhoQG504U7ypQpsrCZ6KIkOtDxfEjMfQ7oA025fb5n9bpwhzQuanTVMxC +0HCqD/spdrkBB/xFQF1o3qJ0/4CewUTw/6J3RhjBhjtcqFyNpssOkHxI5hiKrW7VfuTWKgpMg2Th +6zUJ8IdCpiBJNUFxGi6X8t4opbiQJydJF/xQvkUgeXOSL+sLY9kwJBzfnEiK9TMkrBNJJMUiEBIO +O0zd7OK8BiRA8hhzRRFy82dQbAbzUSiZX1Rs9S9GYyxdqlDM3TlJGkgtxtiWpLBShbVJRiRDQh8t +kRSlLPxlbQtrPtJkRdyy+WylTDgyBvzesWDsczMj817hM1FEiEkqaYxrJmnLoEgIncgbJKXBujxk +Kta/LqFJw+IKZ0jGbS8mPSScbIQhxi4JSMalHE8PcwtKckPZpr+qQwBoDGOCfjpWIsWGBVxogkPq +3aEPKBkfH7kcukHoH+hpVDUwZmFZY+MlKOqZoScrL5WZDZpyTImGOjntnu+A/ng0YAX0vXq1yw1W +wblXzLsOKC7wbgUUnYzxOVT6VfmUp3croMnNXVDUCEsNwBHakP9haW70gZBPmmK0seEQEk4SIgWM +WXBIdijNnyRnkY5GtG19q84M91aYPEFPo9Rd/OupSTZFAtBf8K4Cfbx0NjqFC9+uBQNp2q2cS0XC +fCub1I15my2RAMx9GYpBwrG5kIQyHTYAVQM0tPEaNBQpEXIa5k8bdccyOHyMPIVovJluXKeajpBL +nkXbWbwyC80xemX9f9b+4A7u2+WEsMK8uzHPZbx7FKe7XTfcH85PRgD2hNed8iwVUnAkzdSw8UAC +GZ6ySnfYQEPIhOt/wDfEz7suKNqGzw7KkIa4YvIjjg+4pTWUr+DI8K4CbYhb3kZcsb2NOIYVb46W +YG8EAhQSAodGtFG0Q8IsPoWvkXMgmUIxd7QOK5pD71OdE2rLaIUAY6DhxLTGORCgDTb0SiLkbH4h +S2nDJ0hmvPNikRrVASQHZdiz9TTcCMkZUpkOfwgncNrULSRzOuqfil0L+2YgqeQKnhqe4yfHR+8J +w2K9cgGA7Xz/HLFtYScknMpoVGKb/6uOOoBRHKMSy0nOFK13PVC4mCNlksywGOs3jDYIJNTnDTHF +LAIN3R9oW/iEicGGmGItjRho/jw5RVgEch0kAGIs198+mlppa1UFes+Yq4ptnTKXsq6ZYKTscgN2 +dUaI9omGJlVjSxUog1BW+wc26Aatd33QBptlcO67M8aE4AzgQj+TN4+ffmxAwDQrh8jnwLGZhMBG +apfz2V0meevv6kzB4DUDJiGowGn0ehdHahrrxMq5+o6aNQC7xpLa6zWEbeXajAskZumY0bAZxbsu +aIPN3VEQiHlQbBjqgdbKFSIBjs1mysOjqr2s1sw/FsmrKoIbCXQnRdg0ktf/fjS2qbuN9kVOwqHW +ViDnNDRCveuCNtgUAxk2OY3RhFv9v6MGnjM5p+YkXMfuXQ8ULpbNZTbdbdgUbYbNRcqKTNiF/h54 +Gk3sMzv1XX0eTusRem7KfRX4FXrnXZ9h5MfJQ0EA+Nl6Z5QY6MnMDbqFVjp40rMDiiLhM/CkVGdc +gWKfDgX+WBmPY8zCdB847Ix2H2zeLM/8jp5GjoDESTIHeLaQcpZIFvU0zgo3pLxK1P9OT+/6oG+I +r3qg18dX/d8Vis+s9PVMQ9fAzm7LL0WSHlURRzNaripy6cIlVezFCUa3a1T9NPlD1zEC1WZOwuy5 +oWtOwoZzo4Bal2Ls2OI+N+8OgR5QKlnVJmw+rnD9mp7DEaZqY57fxOg+N1jemK/3sYo9h5pAwm8W +KmsM62H6HK2GO2WNLiyaMQryd3K0dz3QR0jZUOTorxFIDdhcr7/QcjNCFhKOi1yWGCNkIXmgjOSY +XyNfut2jkcW+DKMtCAmHQLBEtLFaFC6hXJTFEXAJbGjPpQ0M5WwgZfn0BjH1rgLltJ0QbeA8Q2sa +nMXejXXfbBcLn4ETkIEdeMO4YpsITc9Roh3Q07sKFHahhJcdEXIfMRaJr0KocVdLQ51i+KHONtZH +mYJcNGPByWEoEyEnGQ+M4vwkijHbl9Y8F2tKSGawZVyxuL654cWJEftrSkxS0OjMbu2psxcbhBgt +DjEW/SfAjdCQwK/T342MDwkjwly8aOW/9b21ahTd6dzlYmpBgkZYYwx/bs3XUnAAQwyWAxb31TQA +kx3eVaBfVBIYxVTOJc32Fe0rk3ShRnAb/Vl2wik1RgKu3y0zOTZZi0NgkNBD2C93hV3oQuESdrLo +VQSiDTYcJ9CgKUYoNJvi9jU9jXoGElr46YLOCpRJqBZueqWS/1Y9veuD3q5XVrGKMq/Wt9fo2UL/ +C1W1XqynVYErTR8s7zQ8WyyooWE5P6liNcEkZofi4GMyLG1xL2aXzZPqoD1LBrJo01L25JoEeLek +Gfk4J0lS4jk6cLE6tGU28VQf2Az3VaEhEPOLLGDMHyKzqBXNQMwaC7FB360QNMNWbPh2LjqvAn26 +XoUcD36R3ln5SpzZt9kwqhyqovmMNP/fehqVASQv1qcY0oKEAxISSTEsF+9Rxxgr5e4vr1thKZLf +1adpZVUFigJWwZ+zadAUB50X7lVrSynBGhHvunQMXyGtrO3AsGHRxwsTzXPB4WMEAmzQgpmRnSU5 +Ipu5rCGttKtA2csTGt4yJQJZETYM+12hL7yrQBtsirUZbOjHsWPOsBmSHZK8Q36YwYBjlOxOrYOX +daz4w6erqk9UXrfyDWy4GZT2pZECkHCrXUNMMV6gmUHMDhnIYGZD22JvY+FHat8C3FxqVJWQsKYe +kl1uQBkQgUwEDfsZ0hx2uQd2n1HN6EFYyZWR/ESKiuI+lJyG3ptRjkPCp+y+9K4CZSuhGY/Sli4R +I8BGRw8SYBoDj1HbxfuOJsfF0sG7itMXj8ZXxe4EXOgoQGq1a6AhZpLk4rDC6v1qZVnFz0me3nVA +G3MpxTI7Z0Mu8u4QS5tgZyRaziWRzKQMSqDBnSeZ/ZW4zpC8eP86AWhvcFiWd4dAOf8sZBFwRq28 +OwTKap7wmeIrAqUL0q7WXz6qp3cdUCSEmn4b1sXSHNbbaB3H2BoqFtux8Bl3SDGIIaFMYWGOUTId +AwnLQP4m+m6iLTOQnKkSuZEcxHMEUnXrAXUMkV++ry+sSIDmVn07DQ1DfdAaft760Vo16iQ2YHl3 +CJTAs7R1/6RmA6QWyZWg3AYb4hY8HXF2Jeh71PsKn8ENwJ+5tDSi3gNtsClXmFKaUUBoGwPzDTbF +1IUNg82ksnedExQk2Btc+N/1rsG6WBzAGpbsTPauAkVEGOLGT0DuL8afvqAPrAZI0TfFfiQkx9EP +OAoRRkcYbXYocVnUgruNGgPJhC+J1ziHuxEPq18ejwcoI+QJ+URlkl/U0xiIQ5txkokBPU7CQGmE +XLEd8vbr5RNO3rlrvb1DRn1HRuFWw6jNB40Wp4yzeFf9sXzP1imjewMXBrZ+Q9ESSiFxiUBCQgMc +m2ROKeHqQ6MiQjAkSXCxX1UmKU65QQIkKbM4Fjg2Lo9RsjDF7l0HdJoUgw2pbSxcWDitDi+WRVBx +eHcI9HFSyurOwAYaGgxGtQLJP8h1P6Q80YKETxuaFaMYMRxX06Aplq3QHG8566fXzgeYojNckJPQ +5zRKEEjYnJtIuqD0PkNxAE4L3ciocKa11FgsikkAhQokAErtrJTFh9SOXZCYFI7FJgskfMo8sHcV +KHk7eEmMI2Df4kNryUDyYhxGeXYM9PKIQsaFNPGuBwoXa+ImZ9OgmWjAliy4T9SW+ecIuc1TkGz+ +eO0mhs4u21dztcoYaFjywc4C76rYGNGVXsMUEv6reoZWMfj9dHlV+A78dHUOwnfgDbWKDkMt9odx ++K53FShs5tzgm8VbkuDDECvE3vVAG3zK9zbBh0P+PxsTkNCOQAJCwjVHu3Vsdrg6Syiem3ODLxzm +xtFfkcuGL9+vH3+l5/BlFCXIJXKJwddUx+wN+yLDhklefEt/2OcGT9kIOyY35nn1DE303334p8Hl +a8MjQXmLwielvw7esrYxj+63Sc3871I0Qq7xd6XQvKTvDfdFBI15cS/94QQ3PM9xKfyNdyh3Yv23 +wfraxvzyxpjOJzf+PLhkG6XHPojqCnKtF6QUp0rudfuDXhvzvHisPjgh3IG6MX8g/JV3Da2vXduY +H9xyeGN+ZVut458Htx4ed/XJNe8h/fADKRsh15q4f5++MOpFQpjDx+4eo75YFeUk7M01WkuQnCbu +j9IzbFAXuktFy3AmnLKlkXFvLE6Pw/wycXqbnnNucMfhd6oUO6rn8KXcEiH335+IA0TN4g6VwrvI +QJDzubtiHpoW6TBOUqw+sHucpDiKmJP8KylodH5zkmRLuYegSurBqpw6ehoLVyD5GX2KlDZepjT9 +t8oX3lWg1+pVWJugNIqQJ9ZzdfXIxXp6118TuqFnCwM+p08pRufc4OjhmxXbVI/Dl1ERQa7P9SJ6 +QLJhYsA9Vd8u6WkM7uBxzqqjddAiRvl0m4bENOlGTfK9qOzEiLq3woPIapEgM0jZqew/Swo+RhaS +EYzkWP2XdVvwEQqgtGuxS7uOqWqjlwCXq8TlRj2tnAfNzkhG2TdJshG0CB4nKZaSZZJitC2eUTsW +YDzFuwqU0sS7JdD/oM+9q5iJTZd/LYHuiyVuBfoEvZpzgyNr/LhCL71bAv2PejW8Mocft+qld0ug +t+nV8Dt+nKB2ylAIP07US++WuKpnMb2SPREooHAPCzmeLRIrYqDBWbjDuz4ols+5wdXbuKG/jRve +vo0b+tu44e3bueHq6AY2O3nXxw0csWgUMJiUh8TwQnvs5Ehs77ocLYGHh99F5wpyH7MoihPe59zg +qjV+JOYdIq7B/KqJzHM+oQeXaxneSbcIuZK/rOYSKendMiiJGqaiwElI7+J1Vo9WCBnnnuFuOJN7 +fCEjbRv4p0kOdwUPc0O0S5Cbh0QGn412bq7kHfKMUf5CcpJy7QP0NAr+nORBkaRYCkKC7/FOCylM +a5LdjeIZKWz84uzZFslKafM8JYJ3+6m8XppesT/hmviqeBM4urA6MR0qySa4CHmCo3aa4xuf5/l/ +7J0NuGVXWd/fda+ZOROTEG2ekIDAladQKSny2RIoemnDR1HCPvvc29yZyQwXmkACkqINhATI3IqQ +4COGk4MQcSRXNIIYTQIIFvyYVgJWiCUPwdaIMPKAUExpI0YRsPRZe/9/a+2z7l3rnnPuzGSs8z73 +ufs9a+/3c71r7fW11xINUl6njGbWxe9qtHvU4ydH/+2wPSOfzu/3iOwkazqOQSOBhAQpz1RGMz4t +Kfx8mW4jhd9k+kk2kZS+2PBZdCFGDj15/E3JWE5zLMPA94V/UabwFVq8Q8pv6Imd7VHDox57DT5e +zgk39GAA3JNqwZj8Llse9dpTKUmKCra3RkFB1OiFO0GOIBXHB53U9YVCjoa8K4inQlmC5EOSzqpt +Z2EBN44qvXhhM4Nk1u/F3c93p7lZ2HMPwRu57NnIJVsk81z2buSSrQvsKeNhyuByDEZS3i5vbxlz +KUtWa3ZijiR4x8gihcX5HSKSUCQSSbMAhGNOE2e70YAq3FkflK6ks92gV4m1sz4oWhTyGfEwZjV1 +oTRAwvcMHCVbKA2QbKKrtA6Q8wu27FABDwSClG4G7TAELZ31Qbk62wOKkzv+5mMIly8ReGITWTIk +ABatK/wpioRfoWKH5IGqg5+oq7MadElJvmkXRApSyX+udDb/KwQHkv9OJGgwE0m2PYSUKRRb+Jfj +lUjcAi/bQIaEplMkyY7QrUkKM3BMEZbmKUXC2TeRJCvFnjZuC7N7zirQB6ipXFgcARc2uPxDNRxK +7yNoqKtZx+gnsMB5FZfqAxlAGxHR8za8Idugh+j3FFq8P51VjBnPyQRnFeiPKclbBX6N0pxVHxOK +Bv4x8Q9AcUABFhdGBfoosFPsJrD9GhXASFKDomfTv2GZCxo2iUE3QaoiedAhyo/A5+zyX5ZhGIx8 +mmQGSIXT6yWinfUhv0f+8bUOeAzVbBWIhoQq4edzC5zwy4Ri6hFPGiwQpIZgPCOKhXF2NAQoh7MV +SVyMvROEE4/i6Unsw2X4ptA1xj5KLLstNiWWHyyeboau+EGcO6tB0bEhxmPAEciEGfLtuCVh7TG1 +XuGtTx7hZqoTl6lZqGZ+XIXSWf1WoVQ6zSgXP8i2wrt8YbF9M9GcfJpy1dl+0Lv1ZvLlVHcDkPew +YbuAeAhHBYo/fJyDf02MCj6CM+MJrGb1FRI405z+UCZV0uzK4cfTwOP+4dXleowvW51VbGz5Nd1y +VoGSL14m+IPleC9TKHslOatA9+uWr1WEvj26M9uOS41GwpwNzwdHhLMBKCKcDUA70hglCpBmHtvQ +sDOKsxrX7QtOqUH5QH/Ohs8F70hjsipAKu11ukPITBEDLEf1+QGOwdv0NOxOsFb2/D1xyHltNQbE +1fYVKB9mOjvICoo7QlRnV7jlGQ/o3t0moc4GoMjyUQT+zfjYP5Fc5PvHdDdAKC6yjC8IMcdZH/Qj +IvJ1HTjfhRYmVFLOcVwtO1MHyTky4Ad1LZVd6b9RsRod8ZB/M/wbcey4JtvqO4qs7Znj8cRGAwVD +UxJ2SC+00VYTKbygvCOUpwGIB2j4lq1Dk40haPjItXPqduAvSMXwaBwWyPa+kUIDEdJCV/rws8bd +fJLekYVO8DZI+CAGKSU3H/9iAKw5Oj4Djo4Ue854/vPJsLMKlC9ZSpm1mrBhAXTBJZB8rwxkx9ZC +xw0SShEbXhfie/GHxu3rLJwHpQSW7IMNNHH9Or3WAJTfQ4lkttssuGQbJMxiI6X0Qjv+xbCyJIZR +1s2r549ncJw1yrYoIOHgV0hKAQANR9XRuC6EHiRsYIq4UoDLGJazxkMIsvavP2/c/qu3rsAh4QUG +Scl+aN6s1sFduhZaN5Dw6JcCSdaYhXrcmLgNVfadlyfJDoXnSbKKrSeKnTGBlxMStmp1NgCFS+nL +DNjQmOnQZMcCDi21bmQ2jBq80HZa+LctCaMQDEgW4huSR6nqY7vVUhhBgxhoC+MjdkGrGZsbvkNh +1IyzSHQAKl+IWNjwFT1RKHvrK60c5iEYHXRWg7IZbWlEFzb4vmXqd/jMhvD67vYhJhIR56wP2pGc +zXXYTGbAltqQNajgbADa0YZh1wDkANpM4YeFva0fgBiDWWVTkriV335Qphx8poFzMncpuqXMjdIG +0tJJJgsXtgbwrRIbDDrrg1KjFwuJ2OA6aJ1VoB022XiADSeDQeusBu2wyXZdDkubs+UI+kXO9oMy +vVoan1zb1/qGfhzGFUr+YZGw7IQocFaB8qIsuXNtfysZuFIVSKEhBgmbubF03VnFhDVcJpHck8Sz +dXXWB+2weTEKAhQltAGgmcAA4B8Yia2OZzn7URU8Bgkz178q1zmrQOHiR3l0NwB5BRs2CoW28OKx +F7TKUm9TXgtva0goTJGkBmVTwVJ8rkoywFZcBckpSdyGKdvcggTukSTb3LIXti6hCxoX+ufnO0WC +SyJJVjFI2LsykuSXEUjKw+SySEIvMkCICJHQ5Ikk2UE+SHg0rm/Ix52k8GhsOWc9tiYSgO+FnVWg +lWZKCuUm5cK7oDDolpLEnn/W7ylJlLIMiq6+5wu+J+ifzZ4856VNOC9s5JzNRTgzbgy7QqMDEqDj +GcozY97OalB0mrPheeDR8i19SnmEtPgWV8xgC+1rZ9VbpDRtCmcVKOcG+IoTnA2pXdyb+jHKLf+Y +mAWgNKEAXwIw21hoRmyDBKA8bK9obI8Le4dxFnPhjYLFCCSHC/U6JEcoezjdeyX0k7eshh6pR5+s +q7Olpwo9T1dnK88TyphFYfwBk84RyQQH6EDyNyIhIieIrl0K3kiSfbutvqh9uwFse++sYo3j70iB +QsULF2ZxKWyFTw8gebbaoewE2Dkw45ck2H81zNLxABRC+HBCK0SF+IKELgMbCu70a1P9snkSLpO0 +uAyY1dgYeLInGa5d6alIw1+n8D18c1esAqT6v0l34NKzkT+ZuR4Fvtw5xUa94bVX6h+prKU81Srd +aZ4hGbecZsPF7n3So9Ldu9IpQKp0mnle6Wrk/8hO9mNulP76q0a95h+p1ynzW6XDbZIpcw+w5uSf ++AA30H6ce1BXkGr9WEmFet6Gb+YcsAApETBbAVm4dLyYMSDgrAKNX9UcBP1lCXVWvVfoU6V7IbxT +WXF4eACKAD/Y91Fx5tyXQl0OZwZDaG4UqiRI0P8ZQf8a9E4lzdvw5uwgGnzQvEN069REvlmG1QD5 +jSAAl7nNvcdjwFFiM0MmnSBpQdE1QVhzijuxVWr7ESZUVR0aVk4ECDHx0vFK4DVBtez0ECSvEDPG +tZ3VoOxfNGfDFVb4R87ZFjecWW7xPUGZPt8YnB+T+kIj42wbCsbHVGW5J0Dq8k2sDM8KUpJNvKAn +A6QkU3hp/UfaWGCIgNEFZzUovazSqIm9rGUD3CKk8HqAhJFMWtnOalAWBBdrZPgAHaJ8jQwRXwRC +VKqRoWE9OmvRCiX6BEmTK8fGYx9vZFk3L7NVIBkzr/qkk//ZbvbqZa01TLrR3y+VC2j4tBGawnjH +oZe3YgAGyp1VoKwwdnYQdEGGOKtAb1Lvq9BfSmV9QkILA1UpyR8HkmwVn5IwXeBsGRQzvC/BscPZ +Migm+cckNwCVYCot7l6+BIqEORsugCPC2RJoRxrj0AFSaRuzJuTDc2PWXCGU87+cVaA/o1vOqiep +v/lCXZ1Vdwr9sq6FehX7U9f67fNenZrWJAajBKltPy2hbDfnrAZFn3kbHn41P/6XnndW3yf0oSEW +a9ALldSQ8gOXF1rya5e35YNvmZkiKrjEXtGSAHEIMV/YExJasc4q0NtDjmWbTSkXqihnA1C4+C4Q ++gFkBGzY/ZfPgwtegoQ5NTLebxYwGPX4OZINW51GCzsA7Wc0hAOD+Li/8ApFMktwOcKrNKSjvFuV +cS/W1dtej3r8fIOSd1i/GeLANiD1PjCj7YtXjAchM2fOKtB4Xng2ouDyWWnzVhkxZ/lJb2gYgOTa +bLLHjyeopPrtjYT+pq7Nc5IXAO/AnPcAihViExLMplp2VoF2zMqupjjCbDgXgq2O/ZYycm5Hm2yF +gTbbZLN+VRslLAW4Vg531gflMzv/EgSnE1YoSHBm6o8x0kK9CQlN9CgFJgEIhxlI1l7dmgzEnV4q +UHa/KzRlUi77xM5ZHxQufpztAD8eqXJf8BysOSmCMaRCiEPCopq4UH03KArM2/At2dIFn4/oZXmP +tgMsFXdo7tImhIciTbYsQcNAKkMQfiB10OwMSwrXZqjzVr+hqf9HKp9ctQOp/k7zj+Rz5ezTbHjJ +qL3V3FdeBSCaUOthqomomaJapHBt1Go2UvX/SL1K9K1a/k7zj2TUGicO2gimV+qAxI7zJRWpiVKS +FiAVCxDVnQBn298mwPlhCp32g2/9eJauzvqghFjzHD+It4YjP+5VQDVP8qMTzjMVLtT959LMD3qB +o6KzGpRddZ1VoOjsKXESkDrxZm3MfLuuhUqQGASok1ysnu4NXA6Cnq7CN2/Dgwf48SwlOqtAcdsE +Ndt9kvINXSfQ+fcl8Bna99ZZH/RlSmo+nubHzUp01n+vMpnaY4L68btEvaCrCzs0Bkjz4TY9eruu +E0h5nx7tkIDeqVvOatBPh6T+FbJoTVdnNei7Y1JQVZBqfGS4AJsE0xQR8SaVlVt0dVaBdgpDdoqB +yKbodWi2LEAz5DVOJrJcGBUJkDp7I8ne8KxgaxL2fguQkrxcMfJmXSdQbAaSP1ZZpED5tbdBJQGa +HXrdeJOIlTqFWmLhJ8ZJOBqxUEukJBMMTkHycikcSfqg7CdZaqLABphCWR6Nc37ZWF18fesSBnlZ +fOBsPyj73ZaUhQ2zjUx9lsaK19/Qigb43qWQgXZNS8K6Wr4LdJatkyDhvP+vSlxh+A4SuDMH7V+a +T1F3p3PIS9a38GFIgkGkQrhBwqMYOoGBdMTi5NCWikHyUBnlbHCj0D/RdQI33aZH2YZhAl2vF0mU +sqWujG2wHGECxWbw+wy2bM+Jh9/YRjTwDSGFQgAJH0Cyt7SzGhQuvvsrhgGoP2EDQFOS/JOtskyu +dT4aXAG/Ui2vouiED7NEvrYAf6f0claBdlhn39c51vM2fP8BeHcOyXubxHBaobMBKOIaUj0WIHgx +MeUzesKbD/7dinZn1fcKZWVaM2y0pi7Qr+jqrA+KDp6dOAdIVfhtkdNUKZRDtKb8xaIbNlNhs5J5 +G75vS+tfIrNepauzig1d4VM6WAB1Pixy1CqUcUj+UkazM7Wvo4OHBKmjCF/8vqPpaY++g984/yQ/ +6Dn6Dn6SFzwu7gFSMV9Si/NherLUmMGcVDWf76laM4bH66QOO71M4FwsOEsWTBBRU5CsXtfWJgAL +xQoVECRPEg0kpVyHhm+Xfk20zipQCqhv4fBBdId19kspWNP0jINeA9AO60s2YZ0t1LBm5gZ+zpZA +O6x/eBPWrFULQHzC+o90Jx7cu8z6lPt0q7ALG1yOou2sWGZ7swm02UiyQt00yaSIIhIub1N9VOik +rw/bIOYbTNzhbADKPm5zNrwAnI5AoRDCmS/HYVdYubx4fasMQAu+0OCEhK9MqOqd1cyvsK9yofTD +BS9wnl1hqAKS7Qm20bjFXwyDFGgfgPCHBMIX6glnFShc/LQrLfAAKZ//pjfQYV3nbbh2gB9PUKXr +Z4yEPkB93OY5fjxSiYWcQm/eeBeJXyEeIJnTow/XtVCQUhIUnLfhZ7Z0xWni3yG6a0silHq0iJ0t +g67pveMXlW/J50FyYXRl/a+VtKio8PU7+G1KK0Q1vtinR1+g6wSZNNCjl+taKAiplOsDSXYCCZLn +yT4Um7fh/7iaH5cEPhXozTEphLMgjer9Yo0BzUFM/Hht4NMHhXXznHgGSJk/Vcxv19XZABTr/aKL +q/mxHuTlXfJzlOcWvj9U3fmKQCSsGv7tltJHBCi7Hc7Z8Ef+Trfh7NOUFCAYevRYLx4cN5QViIWg +XE9IWBVyZEns58cVY0FbQUpKQs+r8FZMSWgwOxuA0jd01gfl6mwAylYxzvqgHfGg/11ZO2fDC8Ef +HEILiQFC/ssXzB8jwVkFZ7j4Fn6gF6Rs2IUiHjmYLwdHVjJFGUMmyMwplF18x3jIsG+pswqUyXtn +B0F/V15yVoF+MSRlSzuy2FmcD16d1aAI8FmScvZpEhKAbDoGrBnSp4Xil+n0Rz1+8vHJDhs0R5gG +FQWppmwpdKnuF16CGMfXVHzcVWhDQMKex+RJoaWyeON4JBDfzipQzt10gV+AYN4R5UI1iQIlJ0nw +RpJsFZEnYdQ+QGoeC2VfoCd8MNSjHj9xO2u2+E3BbIc18jEiWwh3uiwT5B6fETB6PQEJZ02gXKGr +hWJAzJdpYmR1vY00FvN/QewKNRskbJHAeRfe7/1Rj584ikLIb+rQk3zhzPsdKWwCQywVwg4S9oVi +84hC2YSECb8fD+Znww4S5mGjPU1dw0/2BMV8fuNgmc9PijOPS48ABD3SWaobe87ZKR9IONSXgxoL +4bj2C21UACwtc1Yxph4jNNt4TrlA6mwAeo0k+PcJn3D8J6UVOgkp56hf1guQMH4Qi0u2TkpJJogn +SKhksGkCW7AB12S8JOcEIDC2IZn9TaOyOCdAkPLO8cBgt4JCdZGSsBa60KpNSb4lPQplPyXhM45C +2YeEkSSawqW8SsxnosB3z6gPAqQ+w4p1NZh993RLIlZwXie2pUIr5e7Qo4xeTpA35AmkE0jBmh2y +ZoK84dEzJiD5pfE4u0E2lWxJSJindzYA7fRVsxVsnk29CZt8cZQ2/1iKQ1sKroQk7j2TfRdBwn6q +UQpBHSBEo6SwRQIN2cLi+G2QAFGxI5QZM7GZwWbg/jXg0E3jxUFjq6XiAAmPTtB+ggRZTLY4q39U +frhc10JpT7nEMM6+aCGhJY+0QmGBZArzFn8Zw1qg/VqoU1ISTpHwM/MtkwgUMGgepFsY5axP6/IN +ulWwDy68YGk+OuuzZ/0TQjWadWzK5a1BcLZKgYRHMbnwSoDkf4o7W+xM4Fgm7fi4qPC6RsqR8Su5 +wqKYCbLiuCWZwYnHHcn6u8YLJ5sTFIY3UpLrFX7OBqBwKX0pl2dTb8LmdAkJQLHPs1nehM0ZgV6Q +sqGbQjXjrP85PRpXJmXLPcrABd86W3qmuKCUs5WnqSLBW4WmAIxRK3Lpg/Ktm+/Ssb4HCYVSBmem +7mspVah3IIF71J9j/gOk7j3y+i/+Ck5uYZIx2ISEJRmFnlFKwgxz4X0MCTPLsWfUB0XX0hwObKCJ +77Bs/x8Spr/ZA7iQpZCwJ188Uym76mTxPeOOZxbKWQV6dphYY+w1AIEBF053oXvjrH+WIhEufk7v +tYGBIOXDdiaRz3LKx0+/ijpAyoaTxGFzcnvW9fDNV496Kb/mk6Ebrh61dwNLQcqZMe//qPvzNrz2 +an78nBJLYSW3M2Y+QZsNEhSv5Fg/9X81P+IWu1s6h8W5h8THWR+0k1df3jKv+J6ao3pO8vuQjXr8 +vEfsd9pwtfmujQSE7GpuDL/+2vxoJqanpPM2PMSIYYA0qwDCebbIPkIR+UCtS3i6rs6q84ReoGup +3aeogXo9kGQH7iD5th79R1oF4awPek5MwldA6ktIHhlIstXX2i3jFQvxWDAvJUGdQo0OCVvh8uVB +4VUJCfvIM9PhrE//7I2yv1DVwoUZrEiSzQpIEBhJMDQAfodkCvMWbh33O9IKfk9JaKoX/J6SMPNS +qPIg2WhLtjcFCQOqjAs7W2ISnDdOIavggo6RJJtVkLxUORJnrLNZBckU5tlt41lFb7Wzjy1b5Tk7 +CPrvVKE6q0C/e+s3dCrrp2SZswEoAnxzH/whGiFwNgBFmH8M/IFBAbIqAKGcKhCHZGtQhM7Z8HTw +QVCgBkWofwz8e4IC2TxNFcDbu/xOnkuj3sqoRxLCT7bhWaPecMFv8knay+X+DhlJKNMlIw0FO2TB +RYKcp9Bql+0e9faMens3VfSxo97wyZsrGsg2UTSQbVQ0kEm/AKmihCHsCz0gMoGjbuOi2OyLBJL3 +yfGf19Vv4yH0lBAk1af0nvtTXQvVHowPi8sXdHW2F8YPCoxZMxEgdQKftrPA31kN+lVx8QvLwXcG +/bYM2G9LL05G32mj3j6f0yT0xN9/pL+vmUYlBf0DCQnvDyR7xYwUFDy5+Y5h+GwviLQrpXQvUL1F +KR/XNd4JjhKk/nqrmhAf19WF7+sDpCS/rkcJ1dJXIGTu6VLsHF0LLyhIztajkSRb6UOyKBI+lndb +5+pAJHHl7ZaBAFAhHIv3xLGQxVQbHWln+2pl9Yquzp4PSuA42wc6RQxtlHUhjC8KsljPECCNROBY +eAfgLemO9Qvza4rUBwT/9M8UuqBroblNEZnhJTFDqYKEYwQmUAySo1N21z8w3sY7U9lZeC9BwgwL +n0G7fA0xA8nCB8cVY41kQbE8SbbVty4pdL65ztlwCfxnoktAWR7tHwM/Va9BZxXoQ5TkHxOTABTZ +408DNnz3I1jg9E2c1aDY7R/bxGD60wFyBnf4rG7Ch7HcACmf/6o78ZD+7Oz4od9sY+qhIiFTnFWg +/zRmWVYybNh1j+ucDS8CZ+VzZxH0zsj6InCO+HZWgXY0YBQvALajAftwxVVF2aYAJGwNcLOYFkqt +/VbrLibWv6KK1q8PF3UANCvQZCOiQEO2BJhATnbVRkFONq8LNPkegfz2KWl9d/Rb9jtFaJjbiN8p +ZnM0JWFOt1BDpiRoOEEQXKEW+U26lqQcauOGpeZcNfLKT3yxww/IDkY9fmPwrraFf8OBUY+kn5BH +IeE3ewzusqE/ZMKTkPTnKnUdbiT9hW75pa2DUe8ZsozdSjsUJJ2knOzIkUYBQoDKCYyiMTfhrP4t +PfxKyXdW3yoU1ZqdtH5ACvGNe5Mo2gCpvK1cxRGNHRNIQrhu+V3RSNraVd7vnmIbrkpVV7yQyeyo +u6PJ5FGP37hxZ5Nej3ok8FU+BPzGqECQOloOWD8w6uH7Lyov4MVvzA28SCBUIAgZJkjzjfUe5yoS +FJTsv4dN8smH9BSm7LC2J8yH3KhNuoQGSKXDHhdNULo3kmRbWZSFa+TEX9e1MIALyf3rFhS+MSic +r/RV3iGZwMaF32vrSYC3+cm+Qhz2mj1oSWNWbZcdHDX38rNRKVv67c6WQamM/Cw+OD1PF7+MR6h/ +DCUBIghp9AIQsYW02PxYBp1G2sZatYINI+/OKtA/lNKFqMYMuNwZSDAoAJYvfmQ8+75TTxSkrCUk +RNMUJHFJEx+8B0CxhdtbxVhzzmmdzipQFjyWlrrBhmP8OwsGQTts8gEibbbJ5pDYAB3Xgd6me4VV +RXBh4e5TAkk2myGhyfuSyUmu0qNRMVQNQJ6tf7TNM+BfCHFWgXISVMG8PJcBXGhRF8v0x8aVeWxU +hu987g01IvVHAExKucS1yANQuMzb8HcP8OOvA+vsCyXPuk5Z+1kYOH8zcM7mdp7z8iacz9rIOdv5 +zHPOF2TlA5LfLR87q0BRoFiQxeYVajSw+a6zFdAOm2w3DTZ/LS0eJHbOKvZUYxLKWZ8tzJDgrA/a +EcZ6owC54NleCBL81EHO+hRnfFsYicNwYHu6VApC9hArzEYh+KfkabYdd1aBdnyZr4GV//eKTZyO +ojIIkLr/D6QsYgpeWv2D8UoDdQrvtpQkDlcPQFk45GdzwTnCz9kA9FTp6R8LxgiwKS+t3kTa6Rul +1ZtIy64YRRrf490ubSZwB4+y4tpZBfp0ZaGfquPskI7t2dcL2rBhe2djSNAO68WflZgO62xJhTXn +mbH4uDCmEEgEU4TKT4qEHOvMMZFhfkByk5zK1mvb0Oad8tPduhaKx9onxosH+206q5xGFdjd2lnY +6Po63Sq881PGLIhzNmBdIdsvOBs8RgUl1j6DkZJODbKy791UViSpQdHYv3eVVwEoiSmbST69lPtY +DLIupZ3tBkUDZ3seIVOiR1kzFCDVhbVV0X0roHQ2nO1B/C+Ij7OV9wh9r67O9oBGjVY2apSNR0xl +6RYyne0G5epsLyheKHxTDmMy/JPBh4y8BEi987N69Nd0dbZ0q9AP6eps70eFRruXNtqdfeegHpZ8 +QNwKFi18crxQUTv4LRlly12haB7k0OQnKzoKhSplzFxXYcQiT7IM9eOkk7MaFM87WwZlQbWzGhRj +nC2D/hfx8g0/cN4Szpb/TGZ/RVdnyy+SP/+9roWvcbCFU64wwFkNigHOlkAxoKM43vd1Mvizgvwa +lBzxj8muAIQiOjGBAbvCyrrVT42HB0exlN7BCQnnwRUyHhKcxbhk4WUACRBJsh0TSOi5RpJsgVq/ +a2rzU5IJzIdkCvMhAaItWfMhmcL8w58eN/9/hzV42S7rCRKypIETHmug8J44ETCNh4ATAdPAiYBp +3NABXuInykvHKXaivLTe+IdSXv5kPPcZeSmZn5AwkFlojqYksXOdbVxBAncO+HY2AEXX0rQBbJhk +gl2hhQ4JgJgpXHLckbxSxtBRKQyEYT6ewtszOp71YmhQ6IEgGTjuvMiY4HFnCwpNkb3A0XXyByQm +StmywAORJP/m/sx45cV4cKEvDQmnK3c+Kge9ScMTczY8D/wWpfnDy4Sy3MU/hs5AaFpIQR5AhLOj +KQ0pKO8H5jfROjswf/9ofWx99K+Uiz+sa2FAb/2z41EWN0CsQO9RxhdeESkXBnKdDUA5OKo0YQQb +JMeTdLLZmZIgpvC+hgRAmtuWyUeGC96a0XHACTYWqqk0u7fpG3Z1jIGWrfSRTGhEkux8DyTvVE4i +rdCwgATYpn3A8cGGraf/3voBxWPebxkuwIw5cPhz4zX6feElkG9qJCRx1/8BKFxK1XeeTb0Jm+xk +JWx2yxFMFDqrQFkdXZq+gM0jZP+qrs4qUNjM2/CW7Dpv+DDN3iG6dUui2yQUolLj7c/G840uyhQk +79JETzGTEjFsmeqsBu2wyWeS2LBcFpp5G7477xURscD8efKOs4rvv39IJkxgNSdrILr09S+SEfNj +EjNn+WUK0AC4Z0ZP3SI++2X0vA0L8SNPsbktnyuUurYiAbap7q9KTTzVnCUIb4CuwPoXxmM3yq5A +r4hDT4/TKb+P0sqhUsMyYYwGpeZdQsIukQXXpSTEp7Nl0A/pw9Y5G54F/hyZ5Lsg4JjpnwN/g55z +tswRxF/d2brQk4LjDk/a3o2QczTqOdsPilRfHMCjBvvPSJzvH4tyWkilPUqx8GhdndVsSnlhTAJF +aHOuUprXTWIrJUIqj+N68FahMiDrzpCPyYfmuPxXy8t4uEmMYltIhf8f5fRDxdFZDdphfu2Bz21k +fu2WzH9ATOHkT218fcLIp7WqRUiV5Az094u2OUOeH3cosdRoTAoJpbRzpD7Z6GyWAnudKth367q9 +0ocuvmiAx5heToPMPxZ910LqwRli7JAy77O6uvxCYvz78yptt+paWAJy+J62DuUjAtYZOdt/j9bP +nKTQLAg+Olye2PqwlItHVjBjftEJNSibQpSKCco8R9F3sa7OatAPK6lU/9m9bZYANAELL6yFvxwn +YXW6swqU5dzODoLy3XqhosszDkdBws03/8Aj52yfE86MhaOnsxoUbqXJCNgA0HbOquywmUbBta+N ++5SDYArZYH81Ncna9CR237gUOnslxRISVsU7G4DCpeRs2LBhKbSFADqUSH6HMspZBfrB8DLPdhVT +Lqy8dPFQ2seLiw9D8EuUVmi35TnXCIGbX+oK/rLAOduxhjPfHsGuc9ot3OZsuAIeOWcXfOU5Ly9K +r8iFAhGAVxJc0OtuPVHIzPW/GY88TmsvkCx+fZzkB7eWkpKwJ3EpMqE5W+wn2Clh4W9bzR4ikvPl +Ot84VlIAfAYNtVqHJr+gUXL+VtziKu3sB0Fr32hVo5t7elSN+bIAqAbN9+sOB78V8gYSVH+NSAst +OEimkLL+zdYYVrRy9UEvgQEwBhp2MYSm0J5b/FYrBkDDgv0pCSfPlTSDBvbnSl5BzKFEM2rPCUjY +K4F4K9iPFM4po+FSaPst/t9xlz16a1tSktjNRcUA5GVKwhZ2BVtSkrgyfxmUkU3f6Ab/PhWTCUzm +QDI2BHFWMave+fATFKHOKtAPy05fW4DzQU8hY7EMJ0T5NfLpUTurQTGwGJjKTBTkPChnFSh6+ooU +XT8vO5xVoB1p2dpmSmkSEiCNDRyNwVPavv7t8UCe4L2UktD3dfGbtLh/dja2Uy5s5luIbUgYm67l +lELQQEL9xI5VBZK1MPfbAq8aZxV7PV2l7p2z6hR1LR+sq7PlRXUdL9d1Clm0OHb5swHrUW951CMJ +NXq6NeqRwv7AHSKSKBh+X8mWHSmXtcY1+0o2ckjAxg43ks5RDyyqQAobu+70yvn9Jkngi4pIQgo+ +DCQkvFHOjSSk3KTedCAhYa+c32v2w/HyL1fK7+saSEi4QzeiFFI+rTuBhIS7dSOSkPJ53QkkJHxJ +NyIJKawDDiQk/NUGElK+pTuBhASnQItSSNmlO4GEhNN0I5KQcqbuBBISFnQjkpDyfboTSEj4Z7oR +SUh5vO4EEhIoNJGElDWRdAKTpNfrVgxzRXcAas20cBP1zpb6YrKqa2FNXsrl3PD2XAJdDkl90BeF +pCXQG0LSCiVmTYXM2dLNKgnPUNw7WwK9VEn+tQZ+s9Kc7QGlYDRzAfy4WHHkbA/F5PqQtERQkSHO +lkCp2JytgHacdbH89lJdJ/Dfk5Q/1GTOVsiF3YFLtqVLLjxHTsTPznY/Qk7Eqc52P1vOfImuzvac +L4fFbUP7G33JpGaANJSond4lbs5qUDzuh6zAo/d5fQVIOQOdIH2TtCeDnS2D/qnkN9NU/PiKEp0t +b8zrZZzdyUVkAqlOm8T3GXL2oq7Olp4oNc/X1dkSbiKonVVPknIEtbNqo//7qX1+GHGjeXxZFyBV +HDYfkVBnFSjc/IeCOGxnKBEVKDnoHwtiBKk0suy5is1OaV4KCixtzJIlKujbQ/xvmSX/v8raJGLk +7QCp248NyYShFLQUpMpeqkCA3RRlWRwDwHp1Z7tukfXlnX04wrOClASekwxMSgrA7pfOKlAGKDun +HsSDbfJjlWLMTrZxJ6dlUBj7QggeNwpYZmtQhPnHUBPAcqSx/B8RzvqgSOgMuUZh/YercLOni38Z +IwRIhbFOjN1pCp0D9OO4WE5vLfSQIPm2xENaGJaCZAbFNvHSw2dwyRTKru5q4xvA3QUvpiRxp6kB +6JfEzo9/g58iQwrj33DGDawud7YMyrolH4bgFwfOFLkABAucgRnMPDok9p3j/mcGylkF+hdSujSl +knBhBMLZABQuPkvAYzBnBxXgfLGU4LgZXzLBma1xVoMiwT8mygDkCaypTjo02boVGtRha7BCuEKC +O2nXOKtAO5Kzo0NHmM0UBiycMh4inGTnrAJlH5xCiKRccEfnsx+4+BABj8OY2RCBMxxZoehsAAo3 +P0AZ4kBAOMCG1hm0zpZAO2yyUQWba8X+bbo6q0GxaRJt2HUQFZzVoB1tslGDNkeIDfUrh/g5q0E7 +2mRdvHrqeCR9Ofgm24CAhF3M6GLN2fAC8LhKtwaFn38MvPPxEigK+MekSwDiAg3YAx52zgawYb8r +ZzXox8THV0Dgdyit9AaShwqcOzpngxA2T5dENnzzg93guMpZBYpp/jE06IjLRtn9JG4av542HnlM +DDmr2AP2F0NnFzcECKGQcInTSvs5CYlTipztv14M4/5BjIoESBkzisTVWc3o0bXiVhiEQT3UImIL +LydI3q4mTFR/AMpgwLwN33WAH+wb7myAmdF/2ZoaacD2coGt/b4l3Z1VoKjZLIV+tDz3OF2dLV0g +9CJdnS2BviYk5RtzCgOkRGdUoOSWHyfCWiDNdDaS/GnZ0Qz38DCQEnHWAkuxnC2BkpfNECH0wNZ8 +qFUDbE0ym2gO8vsPklTo0uDxS/Uoa9MKHSdIaGOeLe9OUBg+o0cpAZ1TzchyX0NKlwCpn44Qm03C +N4gUpJKniOXF7xqvGJkt3OH34x7kN9xO6WInev/vyH2fUEnyvgL/I6UVlk/CGU0Yru2cKsLeX86q +l0jYe3X171t5JQDuSTnfpCcK72NIsOk7g/416NOUNG/DGw/w45VKLAQorLGzow3S/rOMcjYAPTsw +HrxY6I/q6mxwSOiduk5g2UaNl1D+N8SlqUj4gYjCewjLsCjGxoDlm2Rbc7wDP14VzVXOBEizkLNk +FgJJBQq3YjWKilSjHPHurIY1Hy1NkIWQoIKbURtC4bgI+aOYe6cpsGIwV6CEox9fCbkvSINgm2xW +zxiv+viYqfSGSEg42r9QzlKSKCX/ok2ksMnrTn8q6vCSUdgUmK1qd9ioSZefAuAwGDJ4DMPSG1c6 +sOw8kmSrV0g49D+SEM8BUsXgybWpcfjBS9/ZhaB0gZvn+BEPBL4QFAc1zwXpglQJOkbwc5EPnfnm +K0V+0HFwdiFoR17+a0b8NJQefGDr7PlKCZCqyKPRtc8HRXTpBYjkD0rAJ3Vtqkp+sOLJ2T5QeDfP +iSbAUdIRiOUlO6ywduZ4QY4Nygr0xeE9cfBMVT6PCTOZ2f5eyph5OWcDUBj7MStwti735zhLLsL8 +Y1gG4MD7RxoeYufdk2zJL7fhJ8MAPRv6E6ZWRj1SsPbk9s7w2gOjXurb7j3MBVKzb9AN+DZlNmXY +JOrBAFtx8vPLKSOfFhgIUj404DuZR50UIEcTvzbOdiPJb846NgVmYasQSL6qsDo3kCyDXqakeRuu +HeAHB/Q4W+aMJowqrA9FGr5DhLPd586ZmZldNmf/j72zgbK0KO/8U9Nneu4MKESN4tc6GyVqFMX4 +iTHawSUbWQ7ej27sjwHbuKuQzUr8yLozw8y0GgyRGLm5uAEZ2XFDTmZjNHhERURtowgoMX6wLhui +tmxQjrsejVnUxUT21Hv/v6r3VnfVfbt7Bs051Dn3vE89730+6qmn6q3vqkxR7SlcLW1utbTstwBp +dKH3KSHOequ5ZBuIcEHX8wKXNmA0xvxqxvkzs1XJXC+G9GhcWL4aQuoTBOqMo1t9cCfkY+Qjbnya +NkFCg76B0xLuIzMon6Kfs8w4hDSfjggJs+0hpFIo5vQfG1huc567/PDRzyMjIoV2bkpyXPClbMGD +hB0+UQqzXCFgEUgI65DCwctx+8DRkHKmNKMnUujzbiIt50lKHA3NDvBsQgqryeONmUdDyiGlhX1U +R8di9Nq/KGlHRwqN+nvGS7FHjhYxCoGzNuvUzw9FKNvcTLnEyeUeIFxK7cg8m+4abLLHi8CGbTfQ +OpvmymIOJSjUJHC5XUZkqttZB7CWpmzzCja02WKvoAPI09kMYFS5A3ix9CiNLyCMHhF3hRbcDBKu +sI0k2V4LJE6Owb42Zx1Ans56gE/Rv511AH9ZKD/OCoxNCz185DOOjYUaZCbhSLv5hvyTHKVSaJBk +QkzyhiSvo7LD2JQY8q6BsddBsvSo0WqI5Rm1lRonNej1JlzieEhYTwEXXw0B/2LgnG0jwJmCtSDP +ddYBhJsfGGZaOgQaDimfmrcDbohPjejzY4UTNmdkro/5npqrzuYAtwSDzgMeF1D5bpRy79vSL95m +McvFZpFLtr6Fy4rUQoFSZ0KCCZuzC0Pp+IUr9KMl+GR5U23XALoQUgcibE7XJQn+lJ6FGghdD+uv +N+pZmGSAhLA5Xb8hga9TxhYE285hbcJVN88XqbNzAP9U979OWP+i7JFC8CErSbuzacAXya/9AOM+ +IlcI6WzmsMAP6un35Gk5/2l6OmvfrRX29+hZqlyVtlvF8E49nXUBTxQXf65PtjJowGdei8IrPkTY +qFBVc0TWJZFlrZfrpKEJ6980Ngt+XflVy7drs8NppO1DknCahvCcdQCfvmPolT7/952uyH49C8Ug +x7pl3UGrf8m+QSuVsMP6l+zXO6Ty1P3gRNuSP6kpbOLopVvGh5rHQN2AbhgWsmpALf59GFKiljL7 +OD0bFC/++uTmJI/RXyNJtvdLaijErEZpkDl8QWh6NiD5/vahWX5WWdCgBP54FUm2+UBaJsU9SslO +pUGC41+oAtBAsd/QXyPJWMVes4okq9jyk4aVK+FTAvwmJoHPDfXtQcA435jtVsD4u+LCgT+FKUpI +mB78LZEWnBcSeoI/H3TNeiIkBJJ5xFJMOGKMF08a5tFjxZkyVE1CEIkFqg3I7urqf6INgRojZc7s +o18xBgwjZ+1pGfhtevq/BZ4KOdbsHfKrCfXXyLl7kjgiwVkXsCYs62+kg22E0DqbBlwPm9V9qjZj +SVxP76duGF2ssWZoIITUHJ/WG/g463xDKO4CqNYLEqEJ6pdpAyOvUBdiEC4m5KgbZ11AuHjO0iCE +VGucjz/6Xj4w60md9QDJWP+3wFPhPmBtTx0WGQL+5qwNuE8fL2cH3ynwi2phFLYPpIzp9ReqtZSE +UlWo1lISFCtNqkNDcbxEaXfWASTJE9Zf2U+ExDvrACKu+p/YhEDuIQ/mHGrrr47Qn+PUTxcQqRPW +v3U/pyzU5N06Vh6r1+HobBfg36kKmbD+Xft/rHZzjfldY5l/4mhqfqHUu1VPZ/OATPH6dviBk6U5 +KaiQRCaDj2anCMkZOs1PC52KDuCpQvmOzgGy4Xx1YSokEc7YLNQ0yHuEihF53IDk51aRnK0cCCH1 +t0lZj0x3djbgBXpVVZ9EyH935FgH3RRSFc9Wp/a1ejrrAH5JqOr8WSK3CdnAYE1ZS7MQUhXva188 +qgXrFuU7pahBVv+ZCsBn9HTWAXyofNJ3lA8Q4dkgi5qyDnmjkGbRfV18j2oWPVF25rbhwiKztDp5 +vPKjYPqpZ49+7jlIy1n7sSpbZ+hZ6HGlXJ6sjKmdCwOKj2w8TAYMorfZwB+MNGiB+Dtx80fJ9I/3 +a5vA0ELbbrMDvQLFuYSBG4h/kMtHbmDOUkIDieSGgJ+RXDRnxYCz6ZZ4PEJPZ9OAjw6oWUyLxMK8 +VCoLozibYdHyk5UiZzNbJAM1nM0CPkyvnM1xnX7M2VnAqFF26DXViH13zjqIj8eOZNdOwYX08NRQ +EFGyc1JDQa9WWsmz7db/tUGrf/hAYaODXByWPLf5oSrvTSB4VmNYM/U3aLHdk8wMWtODFkbnBICW +X3W34InAPFOqeneqXoB4oV7UuIF6vV5VKlRywLCj4Bjrt72U/tUHBi2Q7C2IZGAYnq5kLVQcQd1V +l1UpCIalb5EdGMzu01QxC2VDIS0ieB+FoIGnHytHZaeOy7fXyFq6rr+qJPkBwvlBiygNmknbNfB4 +4pjPp6Z/qIELvU2pZGV9aQ4a3aBhdWTBAiunjFbGrO8o1LwpCVuoJ/2pb91BizhrALxluvmykrJj +Hn7SL1yfHrSIo5lnN51nt/Tc0QRdKvsVEpSS/K5ISBBxtkh6DQoJStmRGySIOJp5dhtI0KT/AM2t +3xCTNj/wZU+JDIFidOiXRg1IdhYMmJK8SEyd9QDpyPvJX+A/jn8TFEJOGXZ0O+sCshLKHy4d6BVy +bKhcS4UpTVTNDgylILqBaRh6iIcKZ0d5Dz1vNAMaXCiUkjxP6XfW48wGvK0w9jH1y6OCuf+sMMIC +CTpC4ofbpEMIZEaBJju1nNKw6aFg+ZUkNY+UInzWiTPQqVYAUY7y5e/En15n0xm0iHOInS/NvUEL +12AQo2LTGbSI4w/+751Biyi7r/k7cU6SkZLSIQRMS5ppJfL0U5/6M4o5m2ZS+ON6VfpMyJhrcGFn +CyOlzqYBPxcYtwHjOCr1eQhpIk7RG6oLZz0+ghyYUHBmNKYCer24OZtBY4xeWCYAF6jZI1QgWX7+ +aCn6+SCYaiMEUgwJupK80lAvNISaGDKJGsYPRALHlZxdwJq0bPlDGlLoWjnr/JI0QIIv+sCIcNYB +rEnL1hBIY8c2tIXshoSpgPdKq0IFsfSCYUb9rP7a4GD9FZGokxuqvcLoMCSafgzVkLMO5+Y/Tgo4 +63DqPvVpqUhKFz567AMtGGnn1DDFJ0ogjuLiNNTJagmXvoywgTze69EF/B21ikszCrD5Pf2XLS0F +Y0LyhyL5gJ4FM0GyyTQvnjo0HeFPNOxb+DZCwqQfu6MKQyOQUM08PqbvEo33vkvPQpLhwpn/fESc +9b6i7P2GngVngQvhVgH+i3Vw0CKKipO2MPB44hx5uc36U9ULEF2lCQLiFylh4k+UdPN34mSA/i7l +QqBqTVPBwlmRET0mKNWtlCXOZkb9nSiJnNTfia9TKch46utOlHW5VWOgN2gR5+mVqqExZMs6g1b/ +pv2DYNq+cnuHVV3O6t3fqw7DzvV3wYoKqTE/LFp4NPDol2hI8VSR+q+EuIeAmJ0vGi1rZ+kfhboc +EqqkU5Sf/uIogQyil6oXCUYg0xsNBLNT4vKga3YYCincyhLPgOwAMm3mrPO3Yhj3uXRetCpFnUuE +4t7JQo4gfqtIGKRskEgC9mlAwp4bMqaB9emxxEP6s6ZcOX3UU2i/UGKIM/umEvMuJYTDb7b5kT6/ +Rx7ED/UH+BBnV4f4sJDlYTKl0NwUulu+3rL+4nCvPZgL9UYERD8qNGKJf1x4/V3KhUC5Sa2BWv7a +R9UAKOA7y8BL4u6si8BHqS4u5FYqLHaPZ3DCH8gszuYeoOL/ID0LnpMyjqmYfo8YovkW6+8Ejirn +OxaJt0TOs/Or7fOE1ZxxpxBylsffnPVSX/MtItwMd3LWW0OBc1GA7C98rEnayxILVdNiQV2FVGtW +1n9J7/3YZDVoCeJ/6oWfv+hUb8CwZcWTdPwYMQiS2LK5wfANpQIVPbNeNe0C5i1Sfrtozh60QPGs +SqqXA4JnyxfhXv0NzrJdb7qDFih2C0ciMFw86NNTceNkrz+QgxwjbtVYOMj/Kq9+YDVQXs0MVO/B +X6X3MclgPqw3Xl5lCxDkfcpSORFCLi+xf7UcIXWuChlYKDTgdOlanC4dy4mROopPofOMH6c1q7O5 +t6qaYgGEs+x0EVxwDqYfGgj+kXzwXj0bSJmQa2BlFxIaQmpbpiiYdyg05kkL58W/SmZokBbCEa+a +r5NxmJoptDZQn8mhL4nU2fSXBX5NT2ez3xK4VSZtYBfy9naR/iTsYu3RZsizZPlJVRbEGdFSO1// +CgEvgd2v6g0bCfw4jlAhpDQs6avRsBE5hJSGHW3QFL76y0rpE8WMg0AakNAB5ykjEGWjPTYjTmtQ +fyfKcmP+Tpzjcsc0lRY7oznGuAvsiLO6WdKV6hCwZMqOjC6YZRMknIvPoGTJLxBDE7pGk/ULaLgk +CppCanZ2R+3JJk/sSZwqW9nDaB3jukIz2s1HTOhgdgWsnwqP+/sZhwwhR4JehSYWUkgK85KFpiwk +/DVKaQMyRlPKQ9iQHzWabB5CQwGFppSHvdE8ZFH+pGox4rQKh9ds+a4NGDYyj8uvRBATxc66gHD0 +ZgGO16vmszXhfLoy3tkMINy2WP/FwA8d/+1IOWMMZz1AuPmhU2AsUvIr6QybuOK4DQi30nzA1PRo +7jGPRe7JEiFQEnJ0BUc5lIjizj5EEedCjNj/DdIVUCLlCGUpwSkNX8tCAwASFOR+PGdtQCRPWP+a +bMPWzhq1NYvmMQDxzyiZk7qQjjibBcaUk5yYgjNBwvZrJG6xPivGQsD40DAJz+jOsE8yN2iBgIPv +xnQH/g0Y7r7wJF3fCQOBHtWYZPUGzBukR617Bor1FrF7BoYVGKEbCIJF7Nv92Ge3Wt8AitsdKhWq +N2B4Bm4geEYFwHx0tdagMLW3zrBfq/+GkDM4lL7iCH9WaECTbxPKTVEeOYUSYi8Z9WzuT8aziXNc +ozyYaFta83dFQwipScScpn8UKp2UBFHrSEwkyQ4RIYW/xlUTYEI4AmnBbDH5fARCQMqhxGJkBJYm +zsSsMibwUcixi13E7Lc11QAbFjIgJVmHFFpZkSRrmqnZUZ9lpUatZgFFsz6WUTA09H0NVg3E0GDm +WeMGiqosVhNg9sjcNSJQHOQTicC8X0TVss5q8AnM9Xqz3S/47FXjaaA+rVeRCAwHKEdBYDgcJNKA +Ybmot0KlAYjfUgspMgPz2vBm4Em6gxaYPXoTmIHg6IOYC2DenJIodSHgv2mexyb/DLkTT2eeYaAX +9gWXTRmzoMnZNLaIXCgCIeTUi16crawRjPr0f511XiKrnKdnYYAp5cIn1dksc0uflLbO2v9XIPZx +Nk8i42USY8sdjL8ubs5mAdkq6azNTqB49l+2RiURBEpvrTSBovhGVwKzkQL9akmMXg6GMhqaCv9s +S8y/khOtI4M34H/PkZR1uCye93KRFkZecRCkrCMtp4p73BTPXE0IaSHGB6jcQ30Ggqo9OiEYao9A +AoIRjlgFg6GS3ubret+UBoHHRZKgsUKq+BrFW/8MISXZQCmljHOXk7PuBrhQY8Rp7rE1JYL/Salx +1gasVWiAXJhSGK7BrTgF4UeBMaM1IWC3lbnRtsdFcq/CsqCUhIrKWQ8QLn6xdBCpsAnJSwujyrIs +pKAsJIwmvFFaOOvSU94Vktx9q0Aukyk052HMeSFwc9Z5nAb//42evlsEzLXxhRkHOLP3iZ1HzrqA +cPNzscDRGNkWMJz/vYwQ+1Od9ynl/0NPP24F/D3hGuj8bv31I3o66wLCzesMHKePGut8hXQv3KVy +9JOJEtGATFuEgJujDH7y2GCZxiRH3rVQInIeqwxFghxuUDSOnput9p+x+uNqlKoG+pNl0UpHqmiv +LrNZ/XeeM1rnxaHeNuCZwaUOAr4zoLKN1JTxgwNJDxBuvhoHZh60MIYHZw6BjQu0Zjn67PogLPuV +hAv7ErBQYUoVEgIGcpu21eJoJnxHEpy1AVlw6ewg4K+FROYzQYwx1bcC42yFCAl3lMStpNneHCQE +dN60+ssvG7ULzQxnYX7o98N0P12QEKgiUy5xY30PEC7eE4H7gTMDvyGM59xdg/PxcP7PgXM2F/I6 +z6zB+YTVnOkmhzBO50nfnp4dtFL2msV6wiAYBvWhCCIUxklyNpcK2WL9k1engewOIeX8cr3hUm5n +HRZexXZy5wyVkzfq6axzg0D0KDQ9yIm/UGvrDK1IKrSTU5IvitRZ569E/W09nbV/JPDhwSs6pwtc +1LOBegz0k35n8xyMRPqdzXONF+l3Nv9fpBwLkJ3NA5IjpVkm0vpCpYIE+kapcieENPseJZrn6+ms +i10wxhbrnwF8WrDG2GLzWHGMnLOVFwnAgHH4hy9CCKn+kNRsHv6rkJLAPZKMlUI2Pkk+6+ycv1Ge +xeQ15lJzBriQ3YVN3RgJHSKXsYJxuT+O6ss2IaRGgqTmpeG/CikJ3CPJWCkb8BFCrDaO1BdkQb59 +gZ6FEVjygsyn5BVanJBQd8UcbwPWym22yXSE2VCjosIGa57LVNZv0rPQcyMBq2uUvLe8YrQJwsCq +szYga2N9zxMPIQQ/Tdiw52mHPxPAH6AyM2iB43L/HdZvcYQKOArCMeHdzKAFkkWhdUJwDPOMEIIk +BXVCUkBIU4JGcafxNCAy/TcgFeFxsCSkrOk+nh6qjJmOwNfo6WyGz/l/i6hrBC7rWShFO88bzVeW +PDhrAz4kOFS2aZlyiVsks21GSNjUGGdmshMJkKAYzf1jq2UTfX/YSnvQAssNENVkWKeadAL1YX03 +RgnB3qi3NUJQ3MtUja1W69jv0J+/qWd8Q64SyN1cKvxQELozsuisC4h2/m/An5DQQpWHNNZ3wc7Z +LCDcSi4Jm6uUGpbvlwbff3PUs1iR6awNeIu4lWqMlA1z4fUaAxzOVi+/4JA5UvBBctxhnRAcSo4Q +SvEQyFy0TaX6/WX6M3yL5v4Po7Z7d8jofBFMSFgT6awHCBffu5MyIZAA2LDADlpnHUBu8PW5Bnxh +UDBfeqUgnZV4qsc0m7DjdrZsM3Xl/FHLUIk6awNeHmq9rLFSLpCULJPS8LUpFD5I2A0fz1ToANYk +o38I5AlsfkVvuKjT2RxgjU12uyRsnio2lBpdykgUltuqTSqzgxYIZLSqF34bnBiFkCpMIGUby6Sn +Kz+5F8BZGxCVfKWIMEKqzCbZLL161PHYi1eaFBEJqmGGCetff4AIBxA7mwPkAsbqf0SeGkrY3PsE +okL1P5JNIPk5JfwZz+hwk4iczQIi1v8N+PvKCGez3CyCCv5vYhLCOA0mrP+RYAZW4TmbAURs9T8i +UYeZX5cZXqOns5k/FzgdmiwzgGhacQs6KqSqUkQ4syJMsoJgocb2astmtWARFLl9TPXKt0o6g7DB +FIsfO3w5NWj5JgtYbhkakvrbCtqDFkgsMEoKFp/QW7WFwJJ4vZVYJT6E1AosCsEnnM0DItZ/A9bw +hWz7H2/kossfKMP8RZfKs6fq6WzhNIHkof9XElKdMSWKjnHpXwzyZ1NblVx6538cVgY0/tkZ4+Jl +amw0cTYLyHkyznqA7O12NgvIHofaMTac/ldsMiQqIdTZNCBPZzOA8WibaUD0cDYDWJOfzdlUPrSl +vodUpkUQSShDIZDN9vqh4Qn0FWtLfkCxj3S7HRy0fA8yfwRayhXSLdY/AUmEVBMGZzGMsw4gk+C+ +kACzSsNZZ7uqUw46LoxuoiBHHEWSbEMJEoxaKxCcbx+5ZO29+J8a2/t4FaUG9k65Qlqyd0pDNjvr +AdbYZOs22Kw325RXhTyCM3kUSbJ5BAldAroazuYA411uc4DxCqoFwEFQbw7wyoBaAIwakfoQ8Go0 +Yqs3c2zOFgDjbN8C4wwfj7ICR4WUMYsAb9Z7Z/MklcQ4m0fjzwTGtLZDSBnTO2kFkl1ozDyrM/Yh +hACX5T1DP+fMRY4680U3/FkBmqW9QxpCHPSgGx9CSsIZTHFBdxeQi/u9ZGBaRIWuBsr8c+I8dcGo +AemBOGsDcnn0FuuzzCEEjJqyqdFkq4GUBnnVWbTDU3RBwe5Y8+OT/RMGrf7O/KcExlDjUIVpspSE +Wzac9QC5Fn+L9X8TmE5zYYVAyrnB5r6pfaN5Qne+pH9CQkY56wGeq+F879Qh/xRCNiZsaKIUnD4l +qYnJfrdTmigmtJNqbDiWJoTx2obmXo0Ny+JCyLHhZL/qaIvh+augYLfD+icPWv3n+DWfgZ9CypYv +T0xkZ7f+OtDTWQcQCaXeNOZDq8h5AbDGJlsRwubd0uJ5odZu36xG/4qeDTyPS5k4PNLZ/BfE+Ct6 +OpsHvCuiBIWQ2o/VpvGEmlnA7Wru+AOqgE8RztnsksAP6On/BoyBSuvzVRg4sPvOYJ8eIEJ9pwD4 +GZLmrLeGAosocGEwLeOUIaQWgHNMWnbSGZ2ZiHh/0Hmsnfn44a+FZhZS8LYLpHoDNyFg/1JzEzEU +lyiGGjGE1GT8tSYm275GDIuiH97cZhsRc0gZf5ueznqsNLgnokLKFNIEXqq/UlKdzQPGQjsHGBmP +dYONJGmNGoQ1dBcHY26kUnmGqDl20tncbyvhsfxkG9Jk7EaSRAGKk/jzDIqcKw0KaxSQzNgo3JzN +v1dJiowXniyGkXG2CQ/jjSTpZlVM9+rZoLR+TsrGXkuHkaZHS+kG1cRGlKWKf0FUtiPwcj0b6G/6 +KzWos9kzhaJurn0nIuNpQGrrwmdi+Q2jrTWOWymol5JwSMjP+Bn44X7kqo17wqDFK2rnBw0n26vd +Jv69//GOs5seHP7jhy99Y/mEQYuX7JBZixHvOMt3TUa85HqRtRip0gqB2itNePzWAIWQkmBWSntp +aYVyZDVJWKqJ8qXGMGyQSEPaWRsbMITorA3I3qBCkxnG/JVjlgses/LGoZMxQ081Xih9kDCOHkmy +C0wW3zSUQvg5AQXFIGEBSs1IIg6BHF1JpJD1ztqAjN84O8jW0NvFx1kbkGtkfOtLb0PICSMznfUA +X6kaoTjxmOjMSfKFXIaE3dGIc9YFrElmvUoIaQLWSHS2G4doAmZd28L8i5BKJmyOC52B9WTaRtK8 +EZpn6zP3Aj2d9doCucnTWe9moWj8O+txNlytzY+1CKk1OVo5Mu48Xg4YuWTbt+TsRlLJusR4htvY +IsrpoDHJHZIcb+zJKrv45tHqhJjj0DQLV7A6O8jZv7FhlPXMlDGT8sfF72d34IeQeMHtrg8JX8Zq +HOn4QfX95C2Du6vY8ALV1mZDjhPIebTlC/xK/cFZGxANfE32r+VmCPM4UYSQsv6I3sRDSttTYsMO +eP+lA4Z1qV5X3qEN+4oLFR4kBIy/1R+8PViVF5PVWN5glfX199Tm/B3uhNQUnERMp7FBGpnti6N5 +bQzJNE3pTB7SnWpcaDNC8ixl0ryehXFESI58Vk9dRMEcBvy0YLmUhJG+7b4ADoseKHZRxwv9wDDj +ss0TFUfSEgUfM9TTT6sKCgFvgIS/ssKxsKMKEv5K69HZNCAaN+BCkY0XCWQrWwT/hlIRSWgnhJAm +bx2KrSRGfISYFnI5JfkXgST7lZv6vVFfYrLHWRswjsZn6/eUCyPAznqAtH1qU70w9jUdMPNazjqA +DEH6vwF/M6Qs+zFDJyqJBo12SNCVPRoFk0NCYFq+UDFAsg4py28ZzSU6BAXFIHm8NIskHcAP6ZU3 +rMAQcFzYnKI30DqbBayxoRCFkLIhQFvqFP7E0/x6aUsCC8ouXTzMIOYsqOcKX98VkRBYFuOsDcjq +V9/Y0t+oiws5D+PVunRQC24+54Ej52yRgjOVNBIKndql3x8ahqOzsKmzDuCVSppXRmAI+A9suHsf +dgXJU28dSibEdbL5aiwhiR/WHiBcSn1P2LDAiamiQp7t/IOhsqSPT0shfZA8RQnEaQqtmA2QrLxt +qBiBVbnO2oDxbMisYVMunCPkrAfIDEnJCxYvGVWGPQ0Fw6Yk1FCF+hkS8iIaNlswlvqjijG44ixs +9InnP2fbFXDhRGuOwCk4weIfjgqO9WpWCiSb/DDAZpMfBtgQYgKAQqAuyJPcV2mm5mrwYdg5GGYQ +jVSO+yzk6bJICLFJkU1fSvJADVCUaqmUJorpAsKmuqIfdQhkB3xoLtaILmtMhERn04B4/4T1LzpA +hPP3avcgcp5P9T8il4X+2TRgTa+Lxur1ZqUyypsBrPFZGsvnpyIxPxVKcHoUM4SFybiVtw/LDOE9 +Apy1AVmw56wHeMvDh3/z92/sHYIx4Kwpa1aYOuvRA6E54qwLiAhnXcCatMv2EvmCVCh8XFIFGGwu +NRQTc8RbbrPnNKQkaO1sGvBrsYp4AvCLNUXoy/teIq8V0tkC4MnHDS1b/e/unxlGMEKFJPLlYBGa +niHksgQFnc0Cws2f9AAcOWfXCeStEBZ8wm2L9Z8DHDlnu/N5zvOpzhPWXw5iImsmd0IYb46FNVhf +cmC11tm56FTrMzSkUNiNA8l50nN/qFc7gCgwYf1bGMMLIU0VTWFKlrNdgKTO2S7AGutl7vYNIWWN +v16nZXPOZgCf+8Ah1YT179pN5O1COpsB/IFQE9b/7h4iO+XszmYAa/5/124iV8b//ZWuDT3+YVHu +HiK1RN011l5PV+EjcRPW/9huIiTPWQcQZar/rVEyPzZWIpc2kgfOZgBrmn98LJ/VSvZQ8iUPCGZZ +3k3kjoj85G4id0bk8m4ib1EuOesBklcT1v/L3UTILT8IpbypmWc5ZNyiXjrr3aGMa6lO84V3N5Gn +COmsB/j0B9dSQmReSGe9Hz10+H5SjuCsB1gz5nJjY74n+HYb8GLdlTph/aU9RK6JyOU9RO6MyKU9 +RP5fRN60504N1m/ZPlTbWRvwXKEmrH/tHiI3COms/W2BH9RVvH7WV+ClxwRugO8VasL6V+8h8h0h +nbX/5bFDkl3Kfz8aKZDi65t7e4nMBZdoA74joOYBcQxnbUAcw9k8YD+4QxvwBrlFZWMiFHJn84BT +yuZKNyKvFNLPHgm8TmPt1f+InBI+lW3As4Sq/jenSM1xljgKNYS0Ujxbdw2+U89Jf/vbrkGL+EOV +/Tusf+3+Qat/eP+gBQ7X2Gq7/GA/0au3DaVtG5IMWiA+oBet4YveoAXm83KOQALiNr0QSXfQAkMJ +CyQgnqVSKJLeoAXmer0JJCD+Wi8iCZi79SaQgLhHLyIJmMc8KEk+iBP1IpKA2a0KIUgBcaFeRBIw +9+hNIAFxr15EEjCPe0iiGIgn6kUkAXOH3gQpIL6pF5EEzPly4UAC4nV6EUnAvEyOG0hAvEIvIgkY +fDyQDJMWQ+rn39Krv9dzq53tfZYoX7BJj37poEUcOTusura3f3hPfpMCTaE/kwhoJ6zQz4WIC9RO +Uqt7wvq37V3Wx73G6baxPRc6DCShMCiN8LdLDt9fZwt36ENCPeBsAZDiP2H9T+wm8kLVE4XNpEij +M/EdJbVl/Uv2D/yeeDDfU3ZuG74ZtEDwsdnqr7setIg+SLVEdUt1e9AizrPl/96/en98c4EoxIgo +9WkQrKwMIXWrzyrZt+vpbB6QD6ivm/c8SVUfdXSFDFwVUuZHqqU1pe/mHj2dzb1RYPykvhTw1fq0 +Tlh/ZTcRiqqzlwLyWfLfvPCNeoBKrAsn94SQpm4NrVAQ7ZydDXitFHY2B/i+qOjndz9ZtQvaFY5Q +wQ8Pif4ykTo7G/A1+v4WhsqPLBfUXgq6ZIfyEHyj1Kfd42wekNaxr0RCU5mmkLN5mkC0aKosnFOz +htZshSTCJ9TZPCCt2ep/RM7U18fZPCDdmup/wRsUUqcwNcq+pwaVsw4gnVHf92KsPISjxeeZ0qdm +qcPBUrWUHR6bMsz2K2o4FKYkyOFnqsHwfD2ddQG/KtSE9W8a2zmgZbJFRLpOkSjtk5b1f7ivOtEJ +DFJ2DN/0f7h7/NePNs1Jo8KIkoCW9Q/tr4SBqQnzb/qXNPjUnqJmyJV6+iE5gXerZzVh/U+NtdDf +iIhGTIPBNlpV20QqoxKlbdWy/tIwnWBQbMfwTf+7DYxKe+xpo8KIkuaW9S8bCgNTE+bf9C9rYNRz +5POv0tPZAuC9Qvkh1LGfryM1JEQTg/7F/d0N36mhUFNWQ0sYBAUxNp7B0LsIJCDoXEQSMPQuAgkI +ikEkAYO/BxIQOHMkAUPvIpCAoFxGEjD0LgIJCL6okQQMvYtAAoLORSQBQwM8kIQPj0L6/RnfxqVt +GpqaIGiLbrZpen+xsfDlizl6f7GpAqUkODQICkm0GBhKSSABQSGJJGDWXWxuViufvoizOcDz1dz1 +bdp9RF4nZIPWflPWlXlqIS3Z9xer+4uVWmHR4cFQikIZAUEhiiRgKEWBBASFKJKAWXexeoqGxukf +OpsDXNErP/O1jwgTKQ2KVVPWtRJVhbRY/S+V5FpHa3k3XVJGyqvZFiKY1lkPkOGb6n+VmFpIJTJ+ +cU/o6+fXQF89ut6AtX6TupubOCslx1z/mbKDrHacKUubt1j/eOBHh6lVlsKFQNrgzJUFkWQa8DiN +9ZWO3YfNlyVgRU9ns4Cwc9YG5CxFPwN0gMgVGuTz574Bf1C46o9EbhDS2SwgNxtU/yPydf3PMwQm +TYU+NUni0iOMOmH9Dx4gnfCpkEpzCKmNOeiX5DubYSfcPuk4Yf0PHSBysZCFo9tQcjXJWNc8VXpG +X5oGZP7cH5YBzLO070FO35Cz5IeAuRbfP1p2NMBVyquUJK4izpoBEpZi1hZ+ApKhxfWjUpY1p3FT +aHZlJySk65GyQMEXIeGv7IA4siQrSguBtSq1623YE+XsIGC82aQN+Ceh3skv301ksdRv0m8E6g7C +TeVIiXfSg0EYFMQRHilIDwFHS3XA/QvlLU8SCg/6bbH8qUXrZMNpJCStCeu/VGpv1NNZF5Dr7Ave +g4K3iTqSZN360AdGi+2bQ+2VdYKUJG757gHCpbTsNGXzJildGJNLSeIa7hnAH4vLFuufAIw2fpFB +dvw25c2pAM6mAeFTykjYIJs7l511AdmHO2H9P9pL5MRg9+xHH9YkFXX8Z3Nssvie14moKUOgiCGJ +za0Xhpph5l0CubzET3qOFQ49xAUXnrp21B/xwsLSNEhYWc2Bls66gHDZYvnDhFYSyaxnn1TjjziJ +aVBTJRyh9I29YHMFTL8ZGs6E/3PxdNYFRPSE9a/JZjuyLxc9jQcNeROF5/CI896gBQIh24dHnF97 +oDCLINNwph4iC4UfEv6KOkdRu80adOm6UWemIVDw/zxJtk2UJ8lWJXmS/EriJC1x41V+BjMhYTuL +y6clJeEgyJJfJFJiWy6blqWPjOYLn1hnbcAXqp5zdhDwdwOqDfhAdbL8lyXb9U2lxTuSe4CI8N9L +YGQ46wHWxB1uLO59Ks3OuoA1Ptc25gOR/7KKZQjUXmlSac44mwYkef4jCswlPM6mAZHmZ6DGqoh9 +roo5dItALmEtFTl5Q5r1fnRhrOgb9d1GX9/5CmZRyFmnRpO16KGPjbrq88XzeH2V/DlR7UFretDi +DXffHmt+4dPwiPx8RQx/dji/Q/x9txs47jPoAHI5p/8b8D+KtLCjCWmcYww7Z11AuPlWTfZLtbg8 +ahUOD95W3WXUG7RAfEHZo/snF/OGyLF01k2Zlb7fsDlX/hcP7OgCXimdJqx/1dgUIpv06mtHlKZp ++BaDIOH6Fh8ufItRmXPbXy7VJYoo6m+z/mw1MwmC9GyvXvSvbiCKA7K/H10GEMW9Z+ltCBQkND5C +bBDNMZKFugLJXAiFuZ11AKlGfE0AzChUoWisk3OwiUJqGgL+s0E3vg99YrMlZuqTFIphuHT4KOVm +SnKNSGqHoQgTAnbOkRZWYaYkd4up/8Rl64GpTw1TdbL+TJOu4KOQPGoVSbYJCQkjVFFKtg0FCb2u +SJLd4AUJTcdIsgvwFdLZD94BU9YKB9DDmc8oI4l+gL47aBFliHDSZgceTxzpWz0+/4lYVl4QhjlT +9LCU5HmiLTSBUxJOiqkdrg6KT37Ln9M04z/3YDjBaZtVp2nFK3T4preGL6bzqV28gfQNw+nDR+gP +E2cjs7d1b9AiyrAc3WdRh0BBSsVAX2jxpyTs+yuMAKYkbI0rjIunJCSosFY0T5Ldw77z06NWZn8z +ZiPOoRyy8uv0jb5JT6GDdRUwcioFbs66gHD0jRtguBfyIs955k3SLXLJViZ5LtOruWTrF7jQbiBp +zjqruWRHROECNU81h4hyxwUZRZxT1JUjRLkHYZtvt/t2KAhsPb6FimJHJnnPlouQHmfnrDZSY7eN +XDaSYVDz/KkxNQrx/IkrZjeNVhfxIpg24BPUvSiMmMKFbzekhXIOyQ/kNuxZcNYBRHBpxhk2bF7B +l7da9fkiylzkpEf3Bq0zJfVVeoIn/tvCiw2b+Tm8QWj2HJNcuBAnASqKs/lvI8kgwMGtKxumPjOa +mfELE+54Zca6kJlwoaUfuXQB4VJddUiEg0gLmQ5rDvaBn7MOINx8dw04cs5WsnA+iko/WpnDzfe+ +PyZUCHwfUYeWKe75k7HNZ0fdYo0JXZrGzg7+jhKzVV/bkqckjOPhRDSAQwiGSUiOmi4//YwXbxnN +Fg57nPRf9N6gRZwT97dbdVNkaRYkx9JZN+Xmm2UhdxTIpDybmTXY0EcKIccmDqR3uLgTds46gCR3 +wvqHs71YNKS+rhFdnSWa+tyoweMhltkZh5SEI9FK5VhSaDtHkmzltSQSAueDFfrkKQnNQj/gDxsC ++QENGzB4qhFClHEn/JA415+pIUr0CokRF8VCQPjiX48a/8Tx1Qskaa+6GsllWBc+FTJIVUiFM9PN +BGPlY0Q+KaLSmJoSkQovkBz6/Gi645xVG/Cfxpsi5QJJKbtTmlpHGLDGJlsZwIamFM9qlIkIDJ11 +AD+rVFX/I8LTWQcQHQrOjgocunudcqrUMExpGKdw1gWETdF14PMkJQe1S2dxQ3OKaE7T01kPsMYG +G4aA1x764qjrUEuVLCUS7grhO1yor3Z+aVRKXDaDvBBQLCUhBbVhTlBc+BLPfAZDtTL2zGek8UVn +ir9Q6CBBFkO3JYexW0ftwKldhfYPJL8gC/FUVUiUsbNJfdiJkzn6O98yBKueFe8QyASEEyBroC8D +uSeKtjB4eCixCiOFBR/c+d9HDcncXEExSFCI6+yOjhTCOhQjrIOERUx09QtGXkksxgGaheSnJFwX +7awHeKH67aVvBGyQGFvwbVor/1Zpd9amOHEHRKkHBGe+rlxa7awLWGPDhzkEvBw2HMwPrbMeYI0N +l76HkLKhWkJeoV5EMunnzN0GmXK66vtlPf3kAzDrCApslm4bLUB0ItdBwjIDZz3AP23gDkhGIpNE +ztovlk1ppjtrc2LxQyJn7BsCGQBn6j4W+jrrAtJ6c9YFRII3IPDfinch6xB2lf7KNFwDA66hn5iE +kCaJ2d4dym1nXcCaZZhTCCFlw0KV/y02voEBzBVoDRLAZ5W0F0imbh/1M0bv1kFytZLjrAd4Z/SG +7LcLyUikJeysTfORdDhrv0tCOOqmVO3AmZEZSq+zLiCKOut+VLqic8GnYPxHUgalnHXQ61vKOWed +a8UYAYW2CozX0FiiQsBlIGFPylf1D2ddQFLkvUhvQ0jZnCW9WX/hSxvwF/SugU9cpDQjugHJ20US +zZQtJqT5HSI5rGdhq9nUV0f9m9ZFSbGEhHOYi2YUDVegcpdbKc9F8hzlyTpInikS7gsquaykMNr6 +MpEWpjXta6Mma9DcS0nulpSSyaDhY/IG0RRMBgmXC9MFLzSrIGGOfB1STpFC5+tZsDJSmL4+L5Bk +ZychoaW2jrRAwt0/DRSju8uCuFL2r4xmP9uZCiUGEpq51I8NpPDXuJhp5hdU3cSznPNGlK6cmvY1 +2d1fyi4wBGo8aE5YJYbWbggpCRvEeDqbAWTTZMkVpSz9clrSpewTCZpx0eQ6DBsVG2tFrBcNj9wQ +UotAwo6rBmlh/9B22b9BWkhDzUmCRgqpYusgWfz60OH/QqxaQbE2PvKJrcN3fvUufboQEA2fO/Qm +8pmFD3tf/HFWFxB5bpA3C3iqUP4Ytb1E9uprVxETOSiks1nAmrJL+FgIqbKMGd+pf2y36WpxT2fQ +AnWvXrX0atAKGKkZFgTpnyGk0kgfmzqczQCSoCp1G0rKV2SK/6Nn4TRBsuof9Nd/1NPZwg6tvj9J +T2cLgA/WsYb+FImxXvA00T9DT2+jrl9NBeI5ehFHxcD8O72pDaWBuiK8GrT6F+31/EC9R6+CoJAL +CmlmYPlrlIvO2oDHyx7+hLC9x28xMzN78RYzs/9P3tmF6HGVcfycbrM73dg2BhRMKl1LW8Eb6wdq +8Ga9iE212vedyeo2DWa9kGJRxIpeGLp5pX5ETMTXQaoQcSlaaItetCIKIkFvClLcithWUF5ENCKU +IKEkFKucmf/vzMxJztl5d7cr4nNzPmaez/Mx5/OZ5Mk07HpAlrpXoTVLRH/QNKUvbmjE/cKH3owZ +rx+/S5mnFTqfbsdJPNRkrh8nQVi9SeLqRpD143uV2KewepNES+T1DUWmknREvqJ0f5Go8K6UI3G/ +BLEmJzqVGD9TofNficSoavFvdf8HMPy2ZkD0ETlYSCzfhVRASa08hTj8SsI2PwlpkUEYD1RoyPxW +T5iMW5MTvVkuNmfM+J8PkDigTOc6QVHYVe+JnIeQHz+15zyzNQOisJgx44urj11O/GLU4TzK8P/T +U03r5CbK3qZ1nt+QDoD16rNircObqDxnzpSZewAGEKrNb7S+pxcSQx6U+amUYAFMq90kn9fjzIy/ +caK6BkAOqs7XT8YnV+PnZ2DGWiChmJF8UcxYiSf9svL1upTzEFoBwK4731r+nzh/RVX+eYW7nKfx +MntKTuA/o/5pzownq+6jSAaVW+9TaEBYqL8U+YbN0FE7ezmb5yo20AFCeod310/uU+i2c/IyI/kn +Zc86NnmZkT4idebNeN35Tb/4QJmRt6Jn4JC+V/m7HK2NG8lPxHpdoZPscJmRRJLMnZWrnFCSgxw4 +tXYOKmslGwjN8LDs95xCt+W+SuKSMq0ZEF2QUNV7JN6kzMQ8lB4AWY1s4u6Tr5J4vTKd8xlFFxXW +7wUQKgPsfPP7lgz9I4VuV32VBO6sW39VeFCWrd5DbCBU62kRnZNza2vuJtqiM2JV3kNIZ0H4b1No +zWGiLTrnNpTnJeHfICWsGRA9oiznB3tDeW7Ry+9S2KP6ADtfwsB/jzMd5jTDtu2RdnuoTCM/47Jf +aXzm/i21SmKPnK5bMyD6OTWR6j2KCgjbAbA1tR7RoORJhT3GWsA0lviNyE8UJpZx6GOBren3oD64 +31e4Y61zC4ynMezj0ut3CqfQb4fYbK383qKPMs73e+j3bqG8R+GOzUw3ISswTVl8QXo9q7CHSU7p +1VJh4gJZ2PxeIcnMhe6awK1aq0goE6K8WcZLdFmg8OpX+6PghKc5tD7EaeMvPJX48rXUg0pzG3SJ +v682myvxtWZR4a5K4XvPI0SXfNYxoie9KaN/6QsJNxuwQ6K/bwgTHWkB0JrhR8Wj4bWhKTiY3fzI +Prr1jHizkoGF7MRXA5RFoWAfa459WVkPKbTm2OUbz8dYkbzZd6THVMwe+ALDi753wdsl/5DscrfC +HhL/WAxfUmhNARWs3KOGf1wyPKnQmmJRMtyusAeVTwp7n0c5TMNsqERvMq5d7LZpWp37/ZmM+EZf +Bji38oB1QyqcLLCmIAqV1BgSMhw2a4TJ2R/j/pU1OVE2Hq4y42XicEuUJcxeOcqsiSGoNUOiyJk6 +d4qAuLpr4WBUD2FBrOkJxxQSH4/JpW4NaJ2JIvolVbHESm+cStQVACi3qX7dodCao0RPK8sNrk+Q +eEyZ1hwlytm56j2p7gHbwO+darQnFFpzlOiHpeiMGZ/akM7Twm+GcRxE9gDr0b+6NmYVOfHlDFE4 +cXedHGAulVeXVy+XGflPiOm1eq7HZDNneHXtMc15aXFvjA/FV4VCCZpdg8NEoeru5Im/h1B3zqc0 +35IhUcjMmPGZqNkRh8Pn3FdP7OiBckhCgdoD5edCQVNrBkRb0sb9BsH6GdEBKXX4BRw8KF4QbuIj +sPjvbsVqDdyF7IHyiKNEWyooUGcD2W3LRsvLmIOetwP9x8tZ0iXbgGigoBNjoUTvtRJwaR0QJ9oc +iIp+v6DCsUNOxLgDb21BHSAsOAC4iakLKKeFwzkXm7j+brtW5C6WNQOin/X9YVQ/qLxVjFsDZKJQ +SakMGQByCZVBgQ3HlHZMZQBrbc5wwP8+lclV3Qp1v1RLfNtXZmqUG/UqswJrBkRpY6naMxEZTls2 +tSfaIYDCgapHJUGiwoHCxI/RU6IPAeUKXPAlyOFCawqizU38IVFOMVlTXJSke3zjjPauZldt3juF +Qi/b2jz8uh6lzLs2W5MB+IVAYowRolAOiQ/OFlD2S7KGS3QyCZdNoDAeatSPuxaZ61pMfxpMWWwS +oHBdOWHkEAXB8EtOGs8TbtMtL7PvyFpsqO9y47r4YA0uTPSaak918sAHDBRgC7okKkzIBTUTMzNQ +dkaXTXBhFtdcOKROewiNvDNcphDMzHcrf3NtIapLiNKannu9BagPyq3KZ2SX6MBBgTpH4a3JiT4u +atbkRD+lftYtAuiph1AYhOD6uDVDoi0yvc0wheUYTDZLi9QLDwg72l2Xz1CKsVKY6GlG19YoQI+L +aSHKN4WbaNCgsEiD65tEkYLydlGnT0t8k0GhU2Iwb80y30kuBFmzzBmxhjBTBw/erjISjn+ouonr +ImvX1XZlhA9taz5ClJXI1PQOMtwHaOFQLzwgLTi1AMbjJmrB2T28XANXH1/lLhkPqoNURZmRizHn +Kx++RfWYPBYAu5jksgbVxiSPwVcXk1y+TW1M8nAa0cUk97u1Sh1M8ljp6GKSy9HHNk8R84DRseCv +9YQjhNYMiN6oVtmjHOD8lKhNgdJwiQ8e93ZLG+cbqZEiOPjgpKe1ZkCU6VqyOxVrcAh1Vo4kFydn +Xf0rSn/JHRb6u8Gg9B4I0cGNhIr4kCfOvzpSDX/GV7Mue6n04y36d8dlqczY4mkzX4ozHx3s2p17 +0onSnQQoB3yFiM/b39vlgk6pyVqAQp+7CZTUUv0kYNNDsoXbp1Zm4VAXhelmQpkYSkqZ0fRsYigp +NpOADcOZhDYxlM2wSTl2Gt3RNXSP631xlGhndVZc8H+BByR5zSOJKJXXvITf2MX38WYN9GeJNhii +cJ89Md5ZC7jw5bRmQJQLnomCDKkwCLCmIAqVVNkuvr+rsn797oSpbdAAX7MQ5TV6JaHySsCltVJD +lP3vhMpxKkVIJaXyJBCGEaE1A6KsPiWECangvtiagihUUsKM7uza/9PemNE+PETBRbQ1BVGopDib +D3Q59/BRFkeJttBJwKWHw7EQpfkSRrksfHBqXeIoUS6jgAtWTtSSEIUCmras7ppavzhKVL+VgEuP +viBE6dEXjAIufawYoGzWioOuFXtwDlE2yXkUcO7R1uIo0fKbiAtC9piurw1rk9ykbqf5fDGN9UD3 +v5DXKEAzhIp+MUKUxglHVJeFouaCYKz37HKD7jIjSb9fjcVTQ/s93e1ktgxm3XWkosxIs+G+4TWl +lYAgq8SzmpSQRtVZd611ucxIM3XlfdJs5GvOQxJXadXrS2VGmo0/vU6S//VVry+XGWm8xFxTO/hc +O1FmZH1Mhw/mTeX3s3pG3uf1zM1sijJ7QYcaduuAEFxIv6hbhdc0XMh6WY9EieQtOhtdUUoUI1Zn +dY3Vktbv4sZ+Ij38hJi9oDCxoARhigcGCEQahlV+XmakWaeo8ovSr1vMSJoqPy8z0hSIyu16Gfh1 +CjNXi8bnV8uMHIqC4nHPyOtTPHmZrcsME4WuDPIyO6fkPxQqmxK71M0mScm5t4vyygWZx2e72PtZ +9SJ/VOjILZXZ35Xk/6PaqyCJueZM9cueMnuHaiIq+gdk/MFrUbirViLvgb4NseZUbHsVOrHuKTOS +/E/ZXYIen1stMzIQMDP3lPUTchB53vma1bNQ6vYzL5oglPCgFKKOJ6YoKHW9jATbHig/VH18Ri0+ +dboMNjQherSq6hel7+GCqh/2P+4n8ydcGfWs3WpBVNc+9bJHB4PCdHWJzaVQb/pgCUaSll+ZY7t7 +AsSkB4YJ1b9fk1dV8xDWuZBLoj9de239vV1TBXpCoTUDorepOrqFReIHlZeomlA+L4onVTWtGRB9 +VFkp/yyQuSAyhXASBQ3KGb36sEJrBkT/7LNyoqh2lRkzH/SAgaH8V6GD08MMXxPKtxX2QNkmy20T +mU0osAmUbRJ2igJa3F+3AKA1TCbK2fLE/DGkQudqTUEUKqm5/iQQBi+1Cc4hCgNaawqiUElxHt3Q +NYMGeq6e7pNp3qDQmpy1ZdZhrMlZbWc+Ys2QpSI2jqzJMQJ7ANbk94kszm4TLWPlpq6QPaZncZTL +pjT/CQAA//8jRAd2WkxJQgAAAAAAAjaAeAHsnWeU3VXV/y8YaqjS+1ACCJFQIxAgI0jvvQUYREpA +eu+DdJCW0BLaEEQRCAESpIQyRkMvISCCtBEEIRTpRUH/a3k+n3nx/a+7fvdOgk8eHveLfHPu/PY+ ++/S2zz416Om/9fr3/zqXmO7fePOKBZP+BbUu2/rvPy30739rtSXAZcGNwCTY/9W+fOFfhQ+2AQeB +W03Df4Lkb1uh8P+Av28A7gn+BNwXPBpsB88Efw5eCV4HjgGTMv5m40lSXmu/kh7Tvzcfngf+EkyS +v32lwr8UH3wPNF82J3wAeDl4LXgTmKT81pWL/Pn4YBHQ8l6OcFLyr80H1g/1OYjfDwUtl8sI/woc +BT4IPgr+DfwYnK6i/pie0Xyf8l7i96RMj9/1NH7zcXEiugO97wcXnbb8YVVwdfAI8FjwJPBU8Czw +PHAIOB58F/wMnOE7JZ6kTO8OfL8fWE/+o/y9WflV9SAp9bPcZyP/5gQ3AecknYuBSSlvW/gGgXuA +V4Hmm3KTUt4R5EtPyy1J+R2rlfa5Oh9sBW4L3gjar1nfk1Le8XxwLngheAl4A9is/Pb+RV/7jRWR +MwC0n0hSP/mLlFptfT7cEtwO3JtyuhO8F3wR7AL/BiZlfN+0/P3Q41DwePB08BzwdnBy02O+r0rC +zbfDCY8E/w7+C+xD/MuCW4JJmX8Zn/XxHhg7wfHgU+Ar4OvgO+AH4KdgL/RI/b7P7yuBq4Lrg43q +X5UPSZn+qnqYJH/XmqWm78UHlo/9UAfpsD6/TXga+rleYD9wZfAM8GxwIvgHcGemgVeCSanfl2Ua +WZuG72cFB4KbgMpNSnnOD3/Eh85nNiOclPyP88GT4AzkS+bbRH5PSnnfJV++B/YFM1/X4PdtwXbw +VDDz/Xp+vwl8HHwKzHJ5j9+T1LdjvVJf9uED51nOJ5/j9xfBeUj/gqDjZVLKn43xbC6wL5jzj7v4 +PSnlHU+6LgSHg1eAScnv/L7ZdJqOpJTfh3RUpfPACn1rG5XyeZgIJ4BbIH9r8EJwGDhw+vLhOWCS ++ir/l/DdCP4GdB7oPO1Jfn8anA/9+4BrgUkZn+sx5/ebwOA8JCn5d+eDg0HXA8cRTkr+A6m/B4O3 +gpmv55LOpJTXbL4lpbzbibfRckhKeTvT3w4C28CjwePAE8F28CxwBPgY+CT4NPgs+B74MTg3/fh8 +4ILgImB/MEn9W7Ys9d/y/jEfWu5HED4WtL9KSnlVcpKS336ynh5J8nduXdLjes75lO3AfYghCHBc +cl7zNb9PQ311/pKU8VXJSZK/bbui77p88AboOnMX6mlS8v+TD+ZC71VA56lJyX8F39tOJxBeqsH4 +jXdh+My3voSTMv56fKYjKfnV+27iGwc+AiYlfz0+8yFJ/q4dSvl9nw9cL7lPdC+/J8nfsmvhv4IP +7gBdFzouJclfG1T4nad/wofWB9cJ05MPllM/whuASSlfvnn5flHQck6Sv2WPop/tcB0+dDxy/rgL +v18EXgPmOuS7xJuU8RnPenxYJTdJee1tRX/1dN25Iwzuc55N+FLwNvDIivbTuneR77j6M/jOAM8H +lWs96U0+7As6zp5HeDho/Z8GPZwfthEeDF4FjgI/BWvMN9ZkvNkb/Bnjyk3gePBFMMn8NL2ZvjnQ +13rWQngg+CPwx6DpPYLwsWC99E9PepxfZvpH8HfnGV8QNv0DSfeG4Gag+XEm6b4AvBjsaf5kuX5C ++r4C65XbHeg9Dnwf/Bi0XJ1Prkr5/gDcFNwC7EM6VwazHpzM78eRXutFUpZ/lT4DiX9dcH0w9duJ +33cFLwVdx3USHg8+Crqe+yPhd8EPwXlI1/yg5Wx6kzJ91sNLKK/HwCzHbJeLU05Lg9ZTy/U+fs9y +zXK03L6P/uo9hvCd4JGUW9bTpEzfNuihfqmP8Z1PPBeBF4OXgTeCI8HU7zV+/xCcAX1nARcANwA3 +BQeBe4CNpq92QumP+5MBa4EDQcctz3E8L/uCv7vvPRPlPTs4P7gI+D1wNdD97Z0I7wq6D/lTwoeB +J4M/By8Eh4LXgc6LxhB23+t3hB8GXwH/BSZZ/uZP5kc9fe2nvyl9/4q+k8BG9X+eBDrvdx/zfX5P +yvT73Yd8KDr/Skp+90mtN9YL51VJyZ/1TL7FyIek5LdeOp9y38l6qpwlGpSX9dZ65fwjKfXJei7f +ow3Gb7s4ku+PAd23t11cyu/DwCtB43mCcFLqq1zrtXxPNsif7Va+pxrkr5eOa+C3nd9D+D6wEzSe +qv1W27ty7VfkewZ5SZlf2Q/J92yD/PXSYT+mnD9WyTup9O8Sw1j3+dUP/UNQd3rgX5m/2w/K5/54 +kvxt8EtMN7r53QdISn73402NfJ6/JcnfeUrhmJsPtFPoS9h9/aTkX5APmuVv+VmJf3v4PW+yH3b/ +w3Wr64Ik9Wk7tcjT/sVzLc9ft0hGKPk992+Wv5P4teO4C/ljwXdBx4UvCc9MPXWf33X0YH73nPFX +hEeBL4B/Bt8EXd9sRoXeDtwf9Lz7ecJvgJNA1zsnWCGDzC/Tewt/z/S6Xn+Ev/8JfBX8tuaH5W36 +tU/oaXlqb7Ik5eN6YHnClu8gwg+AVeXreaPrvarzBMu72fqYlPXH/ZM2PjwZPA1Mkr/9zNLebd/u +S7u/U7VfJv9uRJD82lHcyd8blTc/3y8AtoB9wCTT03FOSc9KfDAALL/Wuu1H3B/TLi1JeW3nFk73 +vVyv5D7+qQhYmv4jKeUpx/OjHWDQPi4p+R0nstxOSkYo+eUzXfKZjiT5O88r+bExH5h/lvtZ/O4+ +28WER4Cvge63OR5pp5D2SM47k1KfjLdePAtRPo4PKxDWfkI9koyvNqSk/0Q+cN6vvKr99ZZhhV/7 +qrkYH7YGDwY9Lz6R8AXgr8GHwNdBz82XYF2/KrgRuCm4OZhk+tRvF77z3G0w4V6s/8eAD4NJKW8N +9LR/7Is89dyRsPHOi9w+4FrgOuBW4H7gIWAH+AJ4SMV5sunVjlb7zKtI0LOg86i0q7mUceLXoOVq +Prmv8mv0GQUmZX5pF+p8wH7zMRi1B12e+rs62FP9klKf55C/M+n0nPchwknJb3+zKR9mvUxK/kxX +VTup1y6sd7YD653lZL1LSn2q6mWS/J3XlPbvear25YfAoF2A4/YL/J6U8pbnA89lnQ84LifJ33Jd +0cd1jnbiE2FwvpmU/B9TP9wvmoF6MSe4AOj8aznCzruOI6x9zbOEnwPtJ+ehH1kQXAkcALaCG4Ab +g1uCO4L7gqeDSZm+IXyn3c4IwveBD4NfgNPSn7nfvT/hw8F7wHHgQ+Bb4LT0E9OB1rdVCdsPbkx4 +txlLCoaASZmej4jnc9D+0/3r95H7MfgG/WifGYrk5cC+YFLG57mo9lebUl/c53yA8O/BpJRnPRlO +/egAPW/THiXrUW/KZ37QepSU8VmOi5NfS4GeY1iu7rsnKa/15tLebP/adWgf7rllUvIvxgeej+9E +uFl5LbcUfdaE33md82PlJalPbVTh916G5/biHskIJb92Ka5n16QebAQmyd9+W4nf/DuMD7Uj9fw3 +Kfl/wweuOzck3p3BNtD5aAv1Tbu5pJSvXNez2sVp95uU/PX0OBG9kuRvHVPyx/MX99U8d7ecByOg +nj38cP7u+JCU8bl/ZL2wHnivKCn5U0/51DOpm/+Okt6Z+GDWQPf1XG/tzN+TlNeJPOvlAuS3+/jL +ENYuR7v8pJSXfO7Xa+d3D3LtFz3nnIV6NytYVf9a7ir5YX/hfMPx3nWo+wauP7U/Pxw9kkyP8nfl +A+XYnrTXeYC//xFMSnn78cFPA72PkSR/190lvSvwwWqg9ivOPzv43XWp953cN53E373/VCMf8v7N +3Pw+H+i5keXruaTnau/yXVLqn3IXh0+51reV+X0t8Ieg463nN+38fiqYlPHLdxzfa1c+knDWz6SU +Z3/g/s0PkKPeSck/ueVmvrvvl5Txeb5iOj0X+RN6J8nfem+pf8P4wHWk9xG1YxvF3x0XtHfUnj8p +5U8uv/XH/Q/rjXYyR5FOz6duIOz87HnCVfeLzA/nfbvA535uxnc1f5/c+G5ETk/1nQV+94syH5Is +n/b7S/k7X3A8OAN5Z4Mvg56ze3Bmf/49+ve1wIHgpuDu4J7gT8B9Qc8pvI/nfPki/j4CnAC6z91F +OM8zvuT3f4KzMn/Wnsj11Xb8nlSVP29FfsxMPP9X8kM7wp6m1/L2XKrZ8p7S5Wu9d17juGG/aj1x +PT6IenMsmJT151a+GwNqj/Yq4aTkn53121yg9l3aJSYlv/2CdpX2n1X7x/YP7hd7f29t6r9ykjJ+ +89X5YrbjpOR3PpD90vXokZT8luNovr8L1I4hKfnPo5y8DzWCsPvbScmvfeEn8E1DOc4IJiX/EXx3 +AjgcvBZMkr/jd6V/dx6d50uepyTJ3z6+8Hu+pZ8B56nyazewYQqCGpWnPUWj8loeLPrtTzz15i/O +w73n6jz/9/A9ATqv7iL8Fuh6yvN774do56cdless57na99m/OE91Xuu47z1i7XS0Y/H+qPOWf1Jv +ezHezAu6vl+H8Iag+5RnEx4D3gt6/+wpws+Ar4AaqExHvc39TO/vL8XfvX/p+fLl/H4lOBK8A3Q/ +Mu10J/L3t8H3wdmo7/OB7mcdSPgwUPvToYRtJ18Rnol9wvlB3K3UfkhYe9L9CSdZn61/VeWvHWSW +v+fqL1Gu/y3vktP1yns1ym91sNnyHw/fw+D/VH2w/f+3/Et5N9revy3l/yXtvao/99zJ+4Dftv7d ++xcvUQ+q+vt+tFvvo2R9yHPaqv7hNOSdDeZ48QC/e+6V/ccz/P05MPuT2Rg/5gKn1Hij/58c329m +3G50fNcv0D/gc7zXv4LjfY7Plpfrlxx/PaczPzz/c3xNyvHU+2tV8yPHzaSUp72u50Xe5/Mcyn3S +HMfdv0xK+SnPeWCj/I777vvlvCrbhfX0cuqd972dxySlvsbjvNJ6qZyk5M9+yHtV9uPee3ceph8S +511JKb9qnprzzJxH5jwxKePLfsJ8dd6YJH/n42X94bmjdoz6Z9Ke8GYE3AZqJ+P64xl+/wOYlPH9 +gg++KfmTq5/7kbarvE+i3aH7/lswHnqPJCnT7/0d/aZ4H0e78ST5256cMuWlnVWVPwTja7a8mpWf +5dUsv+WkfeOSlIfltBxhy2kHwq5rkzK/tQtyfe19qWmRkyR/bUIpL/et7Ffr3X9xP/5m5Hoe4r0X +7524vnb8cF/bc0vtgjy/nJ3x8bvgPOB84CLgYmBSVXq8h6T+v5nK9Tc/PWd6HH17mr+Tm5/uy+j3 +YH3KYZPA7QknZfnMSHpmBb0fZz1MSv5W+OzfNibsOZLnLXvxe1LKO4jvDgGPAM8Fk7r5nyntZ0rt +31XJa3b/rgP93L/TbuNqEqT9hvaljqO38nf9ZeQ46rm9/U29/Tzv5ej3xfL2Pqblrf3XjuS3fgg8 +H7uI323Htl/bh/cLvb/jeZr7eNpHDKB+toLW3y0J/x58GHwC1F+U9/a9D/9j1lWHgtoDJllfLI+q +/Nie9GZ+/ITfx4Lf1vRb/v9X0uv9yqy/H1DOn4J/Bx0/p3T9TjvZevX9qCbr+9y0o1XAjUDbn3a5 +xteGfNtXUrYn/ZNU9Re2m6SU53pW+2ntTbQTczzMduz6Ninlpzz7xUb5bfcPUR+0X/4p+ZaU8ctn +vymf/VhS8me90/+j/WKS/C0vlvFSuz3HJe3cHY8ch7SjUk/X00nKb3u5yD+QD7T79D6HflaTkn82 +Ppgd9N6ZdvXas+pHNkl5HV1FH+uN+yHeq3Id670ox2XXM56nWS9c561IuafdlPMh7UPtP53feM9V +/13a4Sel/lXxV8WXlPL1t2A8nnd5n3JN6rX2/BsS3gQ8BHT8vYbwL8DR4F3gO+Cs7O8kVennvaUq +fQ4jHu8VnE+4nj4f8Pek1MdyW4F+tD/4I1C7IP3IHM7v+qHS3udX/O6+5m2Ex4KdoP6Equzjre/a +KVXpo/+uevYp6ldPH+9V6/93+gbzbw0yONulflX1t69ddlKWh3y2a+1JbcdJyf9bPsh4v0M7t556 +LpuU8qryPetB+hdKSvmWS1W9UW5SyqtqR9mu36ScbcdJKb9eP6CcJPm7/lL6b/d/9Kfgesj9Kdc/ +byLoPdB9GP366WfNfR79El1AOV8MDgP1K5SU+nm/tSp+/c9o76n9RPqBS32SMn7tUrRz9V0H/XIl +JX+V3uab6/zMp6SUL1/m+27ks/sFpjtJeZ1vlvqgX6q8ZzISRsdt77n/ld+1G3e8c/3rOYb2X5uj +l+sg7WI9z05K/ex3qvT5B4LUJ/1mVOln/qmffrSSUj/9kVbJr9yvpjyq0pn5nvmalPpW6Zkkf9tb +pb54X0U/5d7v1H+380DLzXdqPM94mgj0a6V/fv05JmX8vvtTJd9zEvsT/XZ5/yPj9zzA/e1G9anS +I+NJMn0dk0r+6jfB+ybeW3HefwoCXFe4r2W+es7h/Sv7I9OdlPHbzsfx4dQir/ZuyZ85mMc57t9P ++GtwZsbTWcBFQPcDkky/8rXnd9/KfVftUrebuUg4BhwKXg7eCt4DJmV8ynW/1ntJ2v2MoP/Uv8+W +yFWPpJS/IuuCQ8DzwKvAm0Dv6T5FOO0VXuT3d8DB2CmcCD4BfgbOxL3aVcAB4DrgdmBS6v8P4tNv +eT/kG39S8uv/yvt/niM5PuyLHgeD1/cuEkeDSSnfe6r6j9JfR734klKe/aj+eydRfy3HY9HzFPBh +cAJ4PPXjVPAG0Hq5MOlaChwCDgeTUj/9Tbtv7n2yj2DMerwzeg0Ck1K+diX12nFS8mv/XK+eJyV/ +VTtISn7r/Zykd3mwXjtI6pb3RenvtBfRD5r7Lvrxdh7l/S/9PvkeRFLK970o5d1NP6pdT5L8HV8V +/fRf5HmOdt7uV52OAP1S66/b+4nX83fnCTnfdJ3iObXjue+E6RdNO2rP0+xHT6D/dJ9oPOGnQfer +3yf8EaidtH7tzd+9yR/9Kh9E+DzQ9z/07/ASv78KvgbOTbt2H+I4wr4700X4NVB/NfvQH/oehH5z +r+f3G8C/gpNA7cvmpP+0H12ZcCu4Huh7P/rH1R/NT/n74eBR4Cmg/q0vJDwU1D/MaMKvgW+Cn4Jf +gNPTbmxPvQnbrhYmnJT103P6euWZlPza6+zLhyeh3+Sm9y3kJGX8d/Bd5lej/PoX6mn5Hkv8pjsp +9W20fVxJO7D+GE9Syr+AD+xPsv+wv/Be9t60n2xfF/P7ZWC2N/016R+qqr29SjvTD3W2v7/z96/B +qvZouSVlfmhP9yzpSDvZ0cSnX5b7CPdUX9+RWZR6kZT6ue9Yr/9OSn77d+2LfAfT/rOFdOunPinl +eS+43niRlPyOJ2cxTliP7e/t39UzKeXlOOj6y3qc48ZelJ/jwLmEHQfsF79P/7gS2B9cA2wFk1I/ +25njdo572a/kuJCk/M7vFE8Y+oXxXQznzUOTEermn7Hwe77pO1b2084/kpLfctQuQz/8nl/PRD/l +PdYk5bX2LvqUf2v/Xz+lvYr76VtTf1zX+Y5sUsp3f8B9Ac8ds1/M+NYhPv3O9DR+/dlUxTdgMuOr +zVZy0vucP0Peb8EHQf1p67/gVX73fNV3Xueln0gyf43v53zn+dYThCeCL4BvgfonnJN2uCTovYOt +6Sd3AJMy/iktrz/54b6G93j1T/UQ/cAzYFLq5/lrlot+DW4iPt9JSkp5lstC5Kf9ueem2lP4fsdu +fDcYHIvepiMp43MekOXsfes7keu966RuefOU+ul5gfcF9BejfzrR/fWklHckH7jfJ7qvkNTNP2/R +x/P3vN+qn0fH0Sp/nm3zFXnaA6Y/D9+JSlIf+XP96r6mdg36tWpWfm3hop/9v35wsjzcl9FPk/53 +vNfrfoZ+J7WD8b1Z15PrUq+TTK/6WA/qxZskf+uiJT36kfGd5bLKrnXnU5L8XYsVftOpfYh2Qu6b +JsnftkTh19/7PHy4MKheSd38fQq/9UU/cPor9h54kvwtSxd+/cv7Hrb1xXOHMxFgexhJ2HlTUsr3 +/tKstPNlwc3BP9NP70+/kpTyfE+np/zml/lsvpkPSRn/1nyQ+WS/kZT8fuc+zX0w6M9N9NzJ/VPP +gT2H1b/x7rSTfUD3t/XncAq/+96g71s4/0pKfbXXfwc574H6mZmW+dp6YFLKcz9kHspdP6n6pUxK +/oPgGwZeC14H+p7QSMJJyuvsW+q/fus8V3M8sH+8HwHem7Ec7NeTuuWvWOT7foHl7nsErkOS5G9f +qfAPpV18AD6Kn8znwQ/B3WYpkvYEfwwmpXz9m3ku7XiqH+6k5LdfcP7rusX7T0nJL5/2a/K5n5CU +/PpFeJv6p93IavQvq4P6h9mF8Lng9eTTKDAp47McPqI8tif/65XL+cg1nqSUX68cz0ZOkvwtA0t9 +eYgPnH+8TdhzA88ltS/xXoZ2Hfovd73u+sV7DdrLaqehP+cTyP8rQN890P/Tm/zuOn0D5u1JmR7v +F7hf7Tns7TCOBivHI/Kn0XzxHDkp9avKpyT529cr5aXe3ptTP8+RXU9or5mkvJb1i7xf84F+gSw/ +7YW8z+A+XVLK01+4+0nO1yZXfuemRV/9Znqubj6Y/iT1k1+/jo6nzsP1j6d/mablb1b00/+kfmZ6 +Mf5pR5vUrR/89mPN8te2KPE7T9Hvpn5Qna/oh1Q/nPpjtT93X2s99LYeHEo4Sf2N3/Ncz5vk00+3 +7yS635+kvJatSnqcn+pXfC8YHHeTkl+/+t4zcr6kn/Ek+WvblfhdH7mfcyj90otgUvIbv/vgzn+U +k5T8tk/ro/2ocpKSX/t970V6fub9jqTk996VfkHd7/W9tF9SL9QrqVveziU/bX/u//mehfMc99/s +t5NSnvNz7eY976xaD7XsUvTRLsdxTfuiDtJl+tz/0y+u9/w9l/SewozUC99luIXw38Ek05P6ZL/m +OOw4k/rqF04/sFOL/uZns/p5n9f3sf5T+e29AO+Jmp/7Ux/0/1SVv/r/9P1D0/MX5HxT6dEuSn2t +f+/1sP5Z/7UPtv6bvnr1v+q9+3r1PeV/Rn5l+3Je4j3uLUjftmCj8Ve16yTba/tupf+wn/F92RxX +k+Rvayv87rNpz+96xn2TpOTX/6H3lqxnXeRbkvzte5X4e7qfVPtJ4e9p/9u6d+G3//ccS7sq13f6 +G3b8dl6p3XuS6VO+9r/eZ9K+Vv/YSfJ37VP0u4V89B7bAdQv7XB+S/gj0H5+FdaL3jfw3eukjM/9 +b883bkCO+973Ex4HPgJ6Pu49g6/4vRfrJu9zLE/Y96S1W/NdmkH8fXdwT9BzzA7C2rMkZXoyHZ6r +D6jYbzH/m01HUupTlc4k+WsHlfpg+9Z/vfNn7+e57qqyJ1ee75+5L+L9HPe9fXfJc/rPVJB6qZ8P +11eet61MfUzK9Lge9zxOf2aep86PHP1iJKU8519L8qH3F32PM0n+tkNK/pqP7iebL9pdug/mOt/1 +sOcFXxGB+bIU+aRdgX5uXd94r0A7Pe972I++AX/60Xc86k3+rA6uDW4Ibg3uBPrOuv3IIfx+Mnga +qJ/P8wkPBX1vZgRh7QgXpr0nZf66vvRekfnkfKcqn0yX75O6f+a+fVJ3/EeU8vVeq/XhKBjOAYeB +9v+3EXbfxnWb+wozUD76ddiKsOf2SamP8XpeXhWv56aebyd1yz+qpFd/vdqpuf7x/mxSPX7fUbH/ +0c7U/dcpJV/9fPfBfeue6qu9yGUk1Pshud9ifP0ov6r3GTJ/zR/3bxz3G5VXO6aUl/MM7+F5/97z +1ANpd6eD7l9632Ekvz8Kvg6+BTpP+JxwkuWvPurhvsXa5I/7dN7f0u+wdjGv8J3zY/0bTSTe1Ktq +fVBPH+0+jD8p07M0H2hXpH3B0hX9l/GfAL/jgO+meJ8mKeP3PXXPXfV3ZL45b3Z9kZTyGq0H5rvl +rh/mJOW3HV/qo/2H8wTvF3k/YjPK2fHKd6S/4PekevLdt/N8POPzXVXf581+2vup3kdyX9pzhNTX ++ZHjb97rc32tX3j9CbgOdV/rYdI5EUx/YM3mh3YBmX77MdOv/Vuz8j030X7B9zO9N5qU5eU+svz2 +A1X8LSeV+uS9QO3EXHdq32f63I9z39L5lvtl3t/7Mwq7L6Ufdc89fbdKezP9il1PP5RketVX+xHf +49d+xPEvKfkdF3rK73hlPniu1Gg+3Eu9dB8oKfV1/0b/YqLv4vgOnHaRtgvjSeqWf0op/yHo8xpo +/a3ydyf/upSb+yz6rR1V0X/Lb3na33g/y3WU9hLHZEKgTI92CvXSlSR/6+klP9RnAB9qp6Q+Sclv +OnrKr72o71k7j7Oft33aLr3P6jmT9208L0rq1veMkl7tlTzvMV77M8dV40lSXvuZRd421CP9pOlP +8GDqSZL8LecWft+hcN7tfo/ntb7n4Po3KeX5LrbzSc+n9T+QJH/7BUUf7ZrMH+083KeyXLRzcl72 +OwRr16V/S/v7RuNvVL7zUs8BHF9Nf0/jbzadSZmfjcrTDiFJeV0XlvI5gPrmO2AvE9auOCn5nX/a +z9j+tHNMSn7fRz6DDy8B816c40NSytNfg3YXjqP6bU2Sv21oyQ/9yniv33asXknJb3vzfM/1v/Mz +7cb0DzuIdr1PRX+vfmdSPtqP5XzOebf7CknqW7u8pNd36jyX9P0Wz9mT5G8dVvi1X7IfdD1tP+h+ +vPZXSSmvp/vfXeijPeGiROR5gvalScYvv/spjqPOczZPRkj+zuElP/RT7T6V77vtQzlfDuqHPkl5 +HVcVeb637rs/6mG/mpT89re+n+u8dEgyQvJ3Xl3iNz9t5+rjPonz3wPgd37neYvvIfpekH4p9IOR +ZPwtHSV+zxlsv+4/+q5vkvztvyj8juP6M03/zJfTnl4Ek1Ke83jTaXtxP8Nz/NOQ5/3kvpT7bqDv ++51K2P3JIYTdn3T/w3eItJPwnpT+s2an/9gUvBFMyvQ4X7B+2X+ab6bjfNKj3lX2Fua/8m1PU1q+ +fh5tF64rnEfol0c7saTMD+dHvpdvv6jeSclvv5/xrkH+ud/k/urTlNMrYFLKPww5R4OWy1DC2gHr +z9lyOoV6lVQl/3j4GuVPfYz/GOQkZfymw3apfZN+SJOS33Tbzi4lXvvd3FdMSnkvwJ/t7hN+T0p+ +v/NdxmynD1Hu1oOkRuV5jyhJ/s4bSn/oOG17147e/Rn3ZTwfsr9zHaP9lPMGxzvnW7438AWKaD/v ++sH946r9b/UdjJwq/bQ71N4p9X0EOVX6ev7RU30zX6vyJSnLqyrdSfK33ljKe0E+cJ6mvaLngq6n +fKc/SXntI4s89/PcL3J97blPkvydtxR+54Xe23Ed7rmj/iQvqWhfyrMf9P1871d47uj8+Cn6R+2f +tNtLSn3dh+8pf+67fo4eX4GeK/+A9OqfX/ubnEf6TvBefO+8cj/CSZke/TY3Kj8p5blO0I7R8c1z +YcvhCtKblPLcr3MfW9QfbpL8raNL/XK++hc+dN/Uc7ek5Hde636Z82bvGbpP36z89jFFP9e17u9r +Z+38OKlbvzsLfz8+sB26r+J9EfcHfG/H+ZD2n573eo7guaB+JNennNwH936T92B9Z9ZyTUp9nZel +vq4L9R89teqfeqeeSZn+LAff4Tcfk5I/y81yUE5S8lvOn/Oh+2aNlnuS8mt3l/qoHwLrofbk3jv1 +XoDn1Y7Ppisp5afcenL035ftKkn5rWOL/t5Xdr9KO3jHJf2a2460H3L+43mN9kL6A7UdPUK/XOWP +T30y/mb5U996+lXZ66lP5ovv4qpXUuZvs/mYlPJcN/nOke/h+W56UvL7zkCz/B33l/ri+GC9cR/A +/Xbrifvt2js539DPm/5cktTX+Bx36sXjvoXn7Mpdm3VFUsr3fDrT5TmJ+7LqkZTytK9IvcyHpOTX +3qxZ/pZxpXz2JALvl+gvxPlp2sXOgx/AK8Ak9VP+8oyPruftRx0vtR/Yiu+OAE8CrXeXEfY9ff20 +TeR3z92dr/qeyD/4u3Yo2+DvUP+c+kG8lN+Hg/pFHEv4fvBZcBL4Efg5+BU4DfnTGzTfkjK/PA+d +2vSfg3Qkpf7e80n9tVfM/E9KefrX0R4my+c+8rte+dQrj0bTo99330l6nPHpb+AnM5YULDRTwSXB +jcAtwdPBDvBW8HZwNPgO+C6YlPkzCT2mFn22Ru99wAPA48FMf1Kmz/cwfAfDd3IOoL8+nfz/OXgx ++BQ4CXwXTMr4nBd5Hqg980T6kfQz8zp66O/Hd4Cvxk59K+5bPwZ2occb4KXkyxVg9iNJqe9nxDMQ +e/aViWcgmJT8+tvdivvh+t3dnvBB4HgwKeV530r/m9nPaydmP52U8rQLdr3s+zuWg/vAlsPvyY8n +wY9A8+lF8t1yuIh8txySUh/HKc8ltY9xnZGU/PrV6il/H9p7P3BF0H3bAfSH9rNJqY/+kmxX1mP9 +TGU9sN7arpJSvu8tTaSdKNd7IUnJvzH1enPwEvA2MCn5bff20/arfcinpOS3XWa/Zb+tHPM9KeXl +PGN6xlXnCUny154s8zX9EHlO77mgfsi1T/Beme+WTkc9mRk8A/R9kxsIJ2X8tsP/VHyZTvWcUKFv +1zMlv9yP0N5CP1ve63D95/6U+8/6T7Efcx6rfzTtQJPMr87nSvyez+e5uufKSd38fyz8nsOrv/Yi +2iEnyd/6QuF3v8F32NyH1n5oGAI8r/Pc1/NZ/fdY7u4Lul5Pv3He17d/cz/sSMZP92Hcz/HdUd+r +Tcr0/Kf0OQ99k6r0mVL50Wj82mfrZ8F8nNCg/tbneuWWZPo7Xy71y/rd7P13+bV39D6b5yOeOyV1 +x/9KiV9/E+5jLwWD+ZGU/Nopuc71HOyWZISSvx6f9pFJ8ne8WvTX/lM/676v5T6C/ZR+ipJS3hp8 +4Hpee6wq/rauoo/2Ku7PeV7kPlCS8be8Vvglqx/DW3f56BfEc7+klOd37ru4v6ndWZL8HVNYH+Wl +PtrzuG+SpD4tr5f82Z550B6gdmyHEU6Sv+vNwu+9Nf3laRfvvSPrnecj2qX4/oJ2m/bn3kd2fNNe +1X7kVgrSeURS6ldPH9uzfiWq9PO9pvQP53nlMPRKSn18t6Cn8SelfMvBdrcmDNbXpOR3/816pL8D +/aS6z+X9+rQ79Z6V+ytJxtfydqk/3kewHu8Ow2DQ/fTe5K/vHDnea1c1kfnXn8Ajqb/ngReAfwJf +An2v5GvC07I+mwX8Lqif3JUIbwTuBPpezTGER4FjwI/BpMwP32c3vSuSbs999O9ZL/368/b+WH/S +9T+dH78i/ebHBMJJzebHO5S377B8h/R6D1i7mqktP04n/eeAPc2fJUjv1J6+IaRzOHgNaH0YTzgp +64N+S5utz1+QT5PbvtvRM8tvSqWv0f5qSqUnKfN7cvMrKeV77ncY7Vg72gsIJ1XxnwNfo/z16tHj +1JekjN/v6o0rOY7kOJHUrHzfp3WcalR++wdl/PX9I+f72su4jrgWBavWH8rz/oD3xVzfp3zPlT2X +bDa+1g+L/q639Kuh3yHtirWPVL72C3eTLudj3qfyHqfz0STLp+PjEr9+BPXLZ3r1i6udnPMq33tw +v0M7fufL2hNp75GU8WuPqX208+TrmDckJf9Y+jPfWXmX8PvgV+AM7G8uAK4EJqX8TNfp8PnOUlLy ++17VsbTr02iX+sO/lPAw8F7wQfB5UH9AHxL+BGwlfeuCvwRvBZNSvxOQczL4JPgM6DsnlzUo73a+ +s1weJNxouWxD/raBSerf/mWpv+6XuF/hfqR+U72nrH9sz8P1/5KU8rWn0B+07U07m6Tkt9zPpPy1 +K9UePEn+rq9L+tyP0m7VfQjbof2Q+xL2d9o3ef6m3dAB6HESqF5V9kHqk/ntOkr/k9qVLot87Ut9 +1z0p05vp9F5PVTq1a3S9brq9/2S6k4y/1qtYJtvP6vfW/lQ/Zb5b6/omKeX1dD2rPovQD7pPbL3T +v5H+LqrsW1qmL+mzPunPyf1w7TqTTE8X/Poz9z6W+3369dFfun6JtP/z3lVSyldu3oPTjtNxSXsh +6716JKX8rL+pZ5L8LTOW/Mvx0ftt2qk5H/A+wFjagfYOSSlfv7D6v3KfyXME7wN7jqLfCNfVw4jv +GlB7C/VIyvhdl2tHlO/OaE8+gnrp/XD9Ifge3YnE73xWvZIy/mX4QL8Qnjd5TmP518t33yWekfEr +KeNTruXqvS3ra1W+f0w6jTcp41Ou5Wo+uy+s/516+awfdeNNyviUazlaTr7joJ+43qRDuyDPxb1/ +ZLxJxtfeu7QP21ez5xftsxV+/czon95zM+3ok7rjh1+/tPaTa1BPHf89z/XdXPPFexo38v1EULsK +7cj0B7wM+ZWU+ujPo0reAg3K8zzHcw3HRf1sJqU+mS+pV1Lye5/c/TztBLVDSUr+qnxPkr9rjlI/ +9CfrO7LWY+1b3bfdhvycg/nn/GAfUH+ZSRlf+g9z31s/HfYPtjPbr/XIfX719p3LU9HjQvByUD+c +bxBOSv3y3Ff+qv0o89P5junUvsf3WXyXZQXm4+uAvlu7PWHXRUmpr/52/lPxuf7yHujR1Av3afT3 +5rv27r/7jrDvVejfaznGFeuV5ag/1aRMf7Px9SU+7al/SPgN0P3/nupTld5vOn7l9yM9Uzp9nsfY +X2onYr+ZlOWlPbn3srPd205tN0kpL/sF+yH7hewHklKe9wPtZ7ak39gBTEr+qn40+82klNdsO8t2 +lf1OUsbXaL9UtV/TOW8ZX5z/ej/Yd829b3Yt84LRYJL6Kc/9MdcpL8Og3wbtO4037cp8B30g/ZZ+ +WrU3u5rfne9rT51+XBehfSWlvtopNKuP/Wo7+qhfUsZXL33bIUf/NO6z65+xUfmey7pu0C+I92OS +Uj/53FeSz/vCrtP085yU8rxPbT+UcpKS33pkOoy3Xr1aiXIfCCal/Kp6rx9K62lSytM+sl45W47W +42XQU72TUn6j9V65ScqrLVTav3ZOa/DhMHAA7V3/rPYL2vFoR+B89FO+35d++EDQ87dOwo+Bjh8z +MZ+aD+wPbgfuCw4GjwaPA9vB80H7vbGEx4FPgM+DvmO9FHbpB4CHg38AXwZnwh6+D9gPXAtcB9wR +PAA8HDwSbAcfwG78MXAC+CroPY6ZsTP2/k9fDKL6g+uB64O7g4eDx4Eng+eAD4JJVfXD/Rr3DR5F +gH5bXiVcVX9cf3of67/1qWTct70+uS+R/Yn1ISnr4zj6GetLvf7mZvoZ7RWmtv7nE/qhXvQv2R+t +xe+bgdk/tfP7WeCN4PW061vBpMzP/ckn++uzCV8J/gL835Kfs5AP5ucyhM3PXQmbn8cTnlL5+SDy +cvx4g9/zvtDi31D/fkGD5e/4pn7q4/3UmZDT7PizJ3yDwarxqFF9fa9Gf++9mD/53rLzfuc/K/N3 +152rU5/XBrcCXY8mZXupGtdsR857klKe/VjOo2ZgPaBe9nNJKa8qXzYlvaY7KeVV5WNS8lf1u7bT +evOwpJRfb964C/2r6Pl6o/PGTviTMn7l1punJiW/8dSbpyYlf/Yzjc4rv2SemZTyq+a9mzFv2RpM +Snl+1+g8OSnlVfVL29L/7AQmpTy/a3QendQtb5myvtIPo/5D9deiP54k+TuWLfzaU+sfzvej9U+h +vXzaP3j/+AUi8N6f/h70az9umlqtVqvV+k9bq9Vqtf/H3Z2Ha1uVZQN/1E/LGGQSRCl2A6WgQooh +ELRDoMScY4qgLWn6RjiSolK9SqJQRoWimOiLAQ5FSoMRqW0H1MTMIbXPRLd9EpKUiKYCJt8f6/zt +7+jseI77eV9ezOM7/7mOZ+/7utZwr7Xuta5prefDoneQH9N5tdH1vb3lNVo+e7y8eu6T5//QaP6p +fmw0v35Wbvdz91uDvJV9xvuVL067+E+Ia2o0v7g+9lz+I6c1Y9D8/NflffXd2Tbfn+8L/YFQdsCD +8tt9UfKoNbq8ric9szxZjeanx5Yn8ZNhcP5sND8+8UzKJafR/PzjnIOVK+9Po/ndT6zd+MxP8bR/ +kf5ttDzv7bCci/gleY+N5v+58Mlz6p6GF+Tv7Jzu+2y0vM5f++rIYd9vNL98ps0nP7+8gOrVaHnn +pvytJa/nxbx5cEzen/yn3muj64vPvRTyEpPTwL/8wLGe8J+Vf5P9Rnxu59lvtDx+d/TB8sfxL+aX +Sx/EX3BrlS/eT9yR+EL+UI2u/1Q9Gs3P/8h9BIvmw96073gf7hN0j7Z74Ny3oL/co2Me/2zGbUP9 +yHe/N/9t+4s7qjz7j3ntaKjv0o+O/nCvwPFpn3tP3Gv1zvxdXtJLM4/E7bsP+C/zd+fB9+a3+DJx +Rvyidsw5cPfQHwx9QKg8MUflt7wWT87vZ4Y2un1fygNT7ZNXapZ6u0fxV/P7jm7/96U9W7v9U+3u +fm10f3Y/zHvvf5t+896/kd/9nhvzyrPe3tHl9bjWDuO40fVtfvbQKf61HxvzUTwGf1PriHySjfXy +Hzr4xUlYd5xrfCf4IfATte6xG7hvzT149kHuiXVe8d3i7ybuWByFPLb2M/Lq0DO7d6rR7fEcvybx +wi/PutRofvHV+lE9NizIP9Vv3S/dbuVoNz/pr2xm+c4P9rn8hsV/ey/yFimv0f1Dbo8LcT7apxzt +UU6j5U+NK/4f+qWxqDz+mpsrb2ocu6eb3Ib6LR805p97D/gpruQ96zfxH/bB7gPnR9P3SH48/PLw +nZZ11Lm20fXhpyyvvf2hc4/9mPOP/PXet/zW2tXo8pyb6QPkVT487Wg0v3o6F6onv/5G8+PTTnza ++er03xtDGy1PPl39Qv9CTqP5nWO3lN846n40rowD9Wp0fYw7fmDiN4wzfi7GWaPl/W7e6+2Vt3TI +mD/OS/InON8bBw31WTl08PvubciD9sX29Q38m35i8Itf4ifa/PIByRtnvZf3SH+8Nv3SUN7aT47y +tssD4l/c/zeir2br9xQcnefEJ+mnRsvnD39Hyadvc++LfpnSB2i/8we/e3lEtlSe76CENfRp1vnt +8l6cI3fK7y0tr/tV/K77Jfq9Nfp9+Q7zsxJXxA+00fzawc9enmtxHg38qw8b45H+0z4UFbfFn838 +bJC38fAhD8QN0p+7D816T49If26+Gw/83BtdnjhM932YT9+u8qfKsQ5pV6Pbo1/sx1D91FjnP+K/ +9j+9kbxG4sXcD6kc38/3RrB4A/NT/iHzqNHli+9btHx5xKfKd64Xx+TeSn6ti9Zv0Xrpl0a3d6re +DfyrR473JW5JfGTnGWzgX3r44Def3N8oDlAeNHEP9jWNltflW+/kCZDXTLnWK+X6jna5aynY+iSe +xz6HX1LnGew84u5V4bdm/ywf/HXZ5305tNHtnSr3msght9HyvprvCz/UXcN/79CPhZLbIG/jz4z3 +u2Me2Dl011DrXQP/avh3yQP3DLW+e88N/MuPHOVvKf+m8PuOsDu5R8B+xn0h/JIb6kOe9X1Knryg +7IX05Y2WPyW3gX/jY0Z/aa/5bD00jxr4lx87+MUhe7/OVfKF6i/z2355qn3k26+Q556AqfVu+WdH +/bRHPeU/lKe0oX2rxwz+Le2fleMGv/2tdZN+f2o8bzp+8Cu/85/oZ3lX7Mvkd3GvS0P71k4Y8vXL +8Xmw9z/Py9/dAyyviXjKG/P/9X1tocvDR0/k3CtPn+84uY2WN1Xvxjx+ekj7Pu1u4J+dNPrP+2GH +p8dTL/ncP5V19rZQdkH28f2zzspTvWP8rhrzypen5ttdvjw0vm/qvduC9W/+XcI3xb/yi6P/5QGS +/1D75eVp6D/8/CbkSWo5DfxrKd++X97o/wyD81yj+Z3v5Plxb65zfGMev/wXm8sv/6Z8MMp9fMZp +o8tvfvEfU/yzJ4/3J57GvUHyA7tnv6F8/OwQ8iDQJ9KLm8fyPbmn+MET7SPfOdl8brmNrl/zO9eo +V6P51ZMfKT2lfmvg3/SU0b++P86dvjvid+yT2cfNG/udU1KA+/bsn1+Rvze6/KlyG83f94R3vRvN +zy6lHb6L2tHAv7Zh9J91Xbn88/q96jd29Kn9Dfn2t/JI0+dN6lNSP/27pfzmi+/+s9Mh9lsN/bN0 +yugf+237fP4F4oobzY9v/zxoX0ZOA//KqaN843v8mq3nReE/6bwu71qj5bkPm7yF+Z86ONRHfpZj +UyD/y8Z6+eFX/k/kwYX5nz7KN2+fGP43hN4l6919Q60f1ml6Yve0uiddXnhxve55EFfz3siTD+WM +7GPo/Rvr7U19+QHZB7q3T14kdp6u/94p98DQrd2ez0buDaHOxUtp35a2t7+v/B8W7Xf9OmVPWT1t +jIft8gLuEdr2A+uUPF4N74s849O5Svxv66nEdTdannk/ajtbH+/k8gMwf8xH7Wj9X2O9vOeMEuRx +dC63ntv3Wf+U65zJb7PR8qfkNNb5Tx/1c6+1uFn+BOI+Pp/xKN79ERmP/ADlcXtr/t7o8uTHOjly +6Ze6fPc7KN99yLe3fP6bLX/Kf1R/mZfq4z5m/dHQ/tkZo7+tr/xSrJv0Fg38y78++O3jzs2DF4f6 +jv99fv9baGNd3m8MefTfz8qDzvfO9Q38sxcMfv524mrlrX/IxHhYOXPwyy9nnrDnsRuoH7uacqzX +DfVbPmvI932Ux0z++qn9xqaXDH56EP6dqHncUD7+IWW2fj+Ne2rc89BofvpA9iWU/qDR/E/LA/bT +KP1NA//yeaPmzqnsXM499ju+/96TfR29hvXyUyloj8z77wvlV+q++WPz9+eGnhlKX9zo+n6n12/R +fuTP3Oj2eq773TxuNL9xsKXvyXtZnpjvS+eP8eQeHPW1nvHf4n/nO8EfSL6uhvaQf3v56Ut730GP +21D+2itG+/DTc1o/rG+N5re+9L3t8pzIqyr/Kbu9+dNYl/+qUT/zw/0y9PnmcWOd/9WD37xnt5WX +lb8UfcqTMm8b6/JeM+TxR+BP7j4U92g1ml/9fTfk4+P30sC/smmUr1/Vn3+SfJ4N/LOLB/+b84D3 +8n/yWzwDeyy/UPdlf3JivpDveyEvJv8J48N3Wj34efleNrr+vi/0Fr4P5DeaX3n8ynoeN5qfvsl3 +gN7phIwf96vZH74yf3d/zCvSj38e2lgv7/XjfYmPoDe0D3Xfnzy+8vjJp3qvyHceE2/2o/n700Kf +FXpLqHth5Au/JX7tK9E7i5uW58D9qu6DkDf88jy/Fvqt0LslDnj70L8IlV/mxMT1bwg9LfTZoWeH +/lboy0IvCf3lxM3KC9Po/uXvLP/tw/O+Nre//zZ8Hwq9Z/pza/X/UvpP/skfy+8DQ72fRd/HV8N3 +c2i/n23yXv6n38/mjodGv++fyntxDpo33vX3w9M/jwr92dBF+7vR9ZGPTX3MN/kYT9zK5fne088u +5f7dQ0MbXV/fL3ntLw2D/N13zfi/e+gscpXTaPnksBPtEDm7hfouWRfXMt+V02j58+a1vLLsjffP +OJEPw707qylvvdxCl0dur9PnpT3kviLr7KbIV06j5ZNj3Vdv6zy5xrl1UTmNlj/vO6KfGs3f35H3 +ZTz7LvzpnHVmKet5Y558+Uzlf3h35Daa3zye+q65v6LXSd+xX0h5y6m3davR5ZPb3039RK5+0i/K +abT8/o76Tv5d6vmIjDffzUbL6+8wOV+OvAdGHrmNltff8ZbTwL/8lrE/4i/7tcwncYDHZf4+IfR3 +Mr/eGXr3jEP3E/X39Ii8z0aXv0Pk3Tv0vqHPC3U/yuaWv++C5Z+Vcs4O1b735Lf7w7u97Uewb/pj +/9DuD/lgn5b/Pze0oX9W3jHeDz+c8ev/6a/pv+WPtn9utDz2xnlyG83/yxkn4uflSbM/py+9T8bN +saHyiLLfvCB/dz4SJ9hYL/9dowfo95zX+YWx69GPO0c9IuPgxNBGy58nx7219LDaLw8AO5X8nIen +fQ8PfVTo40Plu353fm+f8bZz6E+Fik/dkN/uL/rr/P5E6A9nPB2Udmp3Y6q9/A/ODOOi7X1g2vHQ +0HntPzH/1/6/y2/tly/rsLTLd2lL26c99O38A6fen/bIJzKvPSen/uKkP5rfa6HuVzw07Wn0+1Bf +diJxNFP1/c/MS3k2tk/5/Gz3zO/7hWrfT+a39rHviP/+p/xfe/ZJOx4S2uj29LxRzpbOi0aXN6+f +nC/dI/HF9Fej5c3jM88b+NfeN9Yr+kDrFHuk8wN9jvn2qtTrj0Mb5C99YMiXj0LcIWp9bDQ//SN/ +AJTeuoF/dvUoX1wff0pxefT58srLn99oefzZX5f2u+/DvSg7Zzz6zhjfz87f3ce4cWJ8qr/yjJup +8n4o5WxpeexI7Dooe0qj+0d+IP4m+p3f1qPSb/Sr7CveS6Pli3cXR+L8eM/IbTS/+IWuz6S/XcbT +9SnAuKJP0i7xhd6bcXJz6vet0LvkPU2Nm0a3Z6o+9j/8Vvp+60bLn1ffHVP/RvO73+mSPN/rtfwT +X8r/Gy2PnDflefdXXZ7fq6HKabQ890mLF1GPu03Mz9UPjfXFfo89l/6EP05D+c3Pj40/dcsTx/M7 +EchviH2Jn5A4q3tlnLkfbqo+Kx8b7WGvZy9wv9LPR15De/B7Th4Hemx5axqL8j8j5YvP5xfSIG/1 +U6M97MXuK/jpyDFf+X99NH9vtDx+CX3fozwK+m9rlTclp6G+s2tG++XDuiDzwvyR/6fR/PZH/Drl +A7ysGYPmFyfLX5Y/0RT/0j+P+ptf/ETNE/uWhvKXPj/4t88DO4WKyyKH3yh/LvufA/N8o+XbjzjP +sVvyF2k0Pzuf+3CvzXv619DGOv91o31Py7jdL887D7g3QT6ot+X/jZZH37hHnmff4O+3ufL4HfP7 +4G/Lzm9c0Rc0un7WI/drHpL2s++yv03lf9B/+ok/4UVpt3Y21Gf1i6P/+ZGws4ob5q/ZwL/0b4Pf +OGw/b+OxgX8t/Pz4xHGIF+txOLV/I48/UssTZ8BOz1++0fUjZ7R29t/q1cC/8d8Hh/nPX8F6zi7s +e2g9cY+tceJ7RC/U6PK0846S/yupwNaWP7tx9Jf9rTwP9G6+w/wb9cevZP6wYzT0D/lvyfPyStIX +2ec1mv+NeUD+AHZ699M0mp9+ZEv5N7dfGuqz+h+jv8WJ+m6YP/SXjeb33OF50Pz1PRHPxd+Cf8TJ +eb5B/sp/jvrx6+SXJZ+UOFLfc/4z8ovZ/8kvxG/+0Lx/ftzypk3dD60+nmd3u3v22fTq9Df75e+N +bh+/8Y47lyfQOdx37C1Z36fuu1XfKTn0w9rVUN+Ndx4rl3VK/Kc4lw+Gkb+POFB5FNhd+Vexi8hn +O5X/SfkX5P29I5R+gv/Ij6V/Dg1tdHucK8TdPTZyTwxt4J/ddfSHuFf5BrT7u8Lv/jPxgeIsp/zv +yOeX2/L4fW6uPPtQdnb6LOc1VF67hvZv3Ga0/wVpp/ORvHmX5u/8Da27V+Tvbw8Vh3J1fvMj/4/8 +pm8Vb9vo+tB70h+5H+FbU+Mh7bGf4UetP+zX7avFBzW6PuJc9kl75KFwLmqs899j9C+/O37t9Nfi +4PTzr0y1L/Lm8T9yQX7vjT3zYeH76dCG9iztMtojzpkdyHeDH2VjnX+3wS/ezHuin2V3I5c8+xX7 +dPEnvuPOwfR+DeXP7jXK56/pfGRfyr/Pd0n+HPE59nXWC3ma5RW032sof3n3Ub5xyQ6nneJt1Me6 +TL92awsOyF+595Bvv2qd0M/mAbvJpD/0fYY851b6cussvUVDfdb2GPzyFTjHt1+P/C/yedgH0Gf4 +LomP0v/ivhtd/jx+56RG8+vPzc0fof1T7Woof7bn6D95c9ix9d+Hw2if1OPE97rR8n0/xCUbn+a5 +dZMdpNHy7Pfkq1CPqXh87Z1qV6PLp1e3z/7+rGtT6+Pq94/+Fkds3LoH076poXz89O7sJR/Ld0P8 +GPviUalXg7ylvUZ9xAWYr/yqrQ8N/Mv3G/zGubxc4qDYMRr4V/ce/J/PA7eE3jn+NzuFNpqfHsI+ +nh+5vJX8m+WfuzF+PspptHz773PSz+Jl6ZFeGHlnhl4YqpxGy7+9/J9Jvexrn5nyyW10+fSlb854 +eXsoO8MP5F6gg0IPCz0ztDElf5/wbam8X44/5vNCXxx6UWij63Nh/DTcE39Dfv9I/Db4Le2X30eG +PjL0caH8WZ6U3xeHXhn6odB/CP2X0K+E3invafvQvUIfEPrg0INCnxR6aqj3fGnmyetD3xm6V/pD +v1yV343un3nydluQn38yfwz3Ir8y46rR5Rt3zp3XpVzjsIF/Zf+xntCXiZOUB25qPcPve0TvQb9n +n91Q/vJDRvn8DMRdWdflnxNPad2zftL3Ou+6X8K5wHombkNcObuy9ekNWQ98j+TRdX/MR/Ie5Pvn +RzLlH6B99LH0Ot2+/1/bw+7IL9z7WvT9uEdHnFW/L/fGiPvu99d6/2/X++z20Y/1+PtObR9/gOsy +L74UKv89PyJ5u/5X5sduoTeEuqe/0fP/9vLbB8+bX/0+5Ju8ve1kXxNHbjzSk/T4o2e0nuinRveP +PITWSXH0b23GoPmtq861qHMMym+y0fI81+tY12veum3dfMyEPtf62ev6vHm814Lyeh1xf4t1vdHt +F7fjO6Fcchr4Vx86vnfsy74L/OroG9hF2YPoPeStoSfm78Fe7B60Rpe/teXxT5nXLvVy3ml0/bof +nFvVu4F/dtDoX/mwnZeH1WO2rhejZ+LH2Gh59LrmIX726kbz01fMq5f5ZP/VaHn0e/wdUfVq4N+U +/qFX418sjwt/A+dKefga5M0OGf1t/T04D/5t1v/Ph1rnvpbfjZZnPshzzV+En0ij+c/P9+oVoe7F +oidvND/9gn2xew7lMWk0P7uTe/yUy4+30fzyTOH/XNohz0yj+Z+Sfj41lF/0M/P790Ldp8df7sr8 +vUH+0vJ433+V594X6n3/Y343ml/8jLiBXbJu3ye0sc5/+Cjffl+eRfpP+RIb+GdHDn752eRTNj/p +ZRv4V8Iv7yH7gXknT0QD/9pPjfK7PPo1+VAbzc+/yHedXZh/RwP/ylGjfPkW6Z3lQZGHtoF/+RGD +33pPL0mPrR4N/Kvh7/azF/7cxPjBL88Fu4T7DJ07G10++7b15IIwsCs08K88arRfu/njyKPBLt9o +fv6Cv54HrcPyazXwb3r0KP/1WQ/cP/Rn+f3+0MY6/9GDv8c/uwR9cwP/0gmDnz7Z+OdHII8+u4/v +foO8tROHPONhc/X5G08a/L7zPR747TSUj5+e3v6EPtb3sIF/tjLKNx75exiH5qe8jfJ/NMjbePKQ +x6+J/c331f0yjeb3fvjpy1/hHMof6DkT40V9puTxn11UXvsR8sMQv9nQvpVfGv1jnslTxu9BnHcD +/9KGwc9uI46dnultYXTPAv89+gv2Enol/cru7Hzv/jv+yOIMxd01un7sjfJn8c8Rb9NY5z9ltI/f +t/t0ds175t/g/gZ2dPYZ39En5Hl5bhpdXtu/F96vnTrqy45pPTXezWffFX708qLZxzbUb+WpQ77z +rnxm/KbYbxr4V58++OlB+aH8Tr5Tnw4VZ/bD2b88MPTBoQeHPiz0p0Pdi9ro8l+T90FPib45f280 +v3gwdi90ar+h/d9MOXdLe1FxKo0un16I3Q+9b+Q18M9+dfT/bnnAOQy172jgXwk//yl+bPyHnG/c +g2n+yqvDXtVo+U9O/1j/+O+Q08C//JzRPvld3fvvHk/fI/mZ2O2sR+z97uukJ+28E40u37lUvzov +88tu4N/0vFF/fOYvfw7rWOsrGuRtfP6Q1/mZ3Ocoz4443A9k/NCnuZflM/m7uJ1Z5ttLQ18V+rrQ +G2L/uTm00fXjn3i/8NP7bC354ve/Efn3jL3tAaGHhp4Qenrox0KvTTu0q9Ht0U9fSb8pR56hz0Tu +9aFfDL1r7HLbhrID/mPKV49Gl//lyLsldJfbKe8d4f9E6Fro98YOKJ/G3vl9QOgRoT8TelzoSugv +hZ4V+uJQ+Sx+O79/N/TloW9Of1wZ2uj++MKC9T0w8g8JVf/H5Pf/VP3p4fnz9bhRT/18fur7oPTP +AaGPDj0l9Bmhp4WeHdro/lyN/KtCW95zI2dRefYd4lMuSAU2hTa6Pr1OzVs33pb532h5PY61w7hr +4N901lhvt/T8NXvx4HeucQ6kt57aX+HnhyLfqn2zc+4H8n11b4x7fe+a9aqhfeT73t9R8sUZt3x+ +pI2uH7790k55MsVLNpqfftx4FJe190T/rJ093p/z6yEpyH6en1ND+WvnDP7H5gHnMfow+gD6ZecP ++bzZdfhxX5D26zf6UnZH9kb5XOWDODjtZP+lR1rJ3+k95XGRL54dif+O7+BN4RO/IF/XozMfTwk9 +LfTVoX8YeknoaujfhV4X6rt+t3zvdg6VZ6fR/S2uTdzrxvTb5vYfP/9Gl8detrnyp+LBjR/7t+7/ +ndJfja7fM/Oc93F5fi/a/42W/zdbWd6Wrrf6a2peNbo9U/Oi0fy3d140yF85d6wn+oc+3rmCntc6 +9Z4Ick8GvY24EfEd4rqNY3ahmzJvnG/FRcjX8b1ZBxpT9bUesmPyX3Wu2dr1la9v0frq182NL/V+ +9KtzqHu29HOj+6v7WZzRVH7BTb83xgf9NX99el55h+1H6M8a6jM7b8i7OO+ZPUvclvxO7FX3yL68 +0fJ8P5rf+n5a1nvnp++J3EXl75r1qOVN5WfT3p1T3p6hB4dekn1wo9v3sDy/b54/InRRfv6C9h3s +We6ta6yX/wfjfXnv5pf4E/dp0ufKK0ofKo9oo+Wzh5gn/Bru14xB80/Vq7HOf+FoH/3ngXmQ3wd/ +m7fn7/Jns7fQc/G7sQ42ujz2P/Yofg1nhlFclHo0yFt+zag/uxC7uXWPn0pjnf+1g1+eb3Z4cUba +L75eHF+DvJVNQx6/KfqsReOD8OsPcU3sWuQ0lL/xolE+Pwb3dfNjYW8UPyBvS2Nd3uuGvL4P951h +sC/Qz+xY4h/k/Wmsy794yKePZ1dj1xLH3Hr0Bnmrlwx5WV7X70E2j8htNL/nrO/sAOwiDfzLl47y +xTMZP+JzbmvGYIqfvfCLeV487qLyjEN2YO0wzhpdH+Ov28Eu1Wh+cXL42YPun/1QA/+m14/+tH6z +wxkPzp/yuvAnoBcRJ2WcK988njrvKt/7Vx554gHtT/iJNLo9U/XdJ/0ylQ9M/RaVJ09Yo+unnfPk +Gtf6oUHe8pvG++M/5v3Zn3pP7pvgX2if7X3xD5NHynn+Yekn9mB2EffjWIfYQXfLPoZfPT/KN+Tv +t4U+K/ulT4c2un32g90+8WON5tc+9zfvnHZ1e93/or3H5LlGy9efvt/yQ7H7kKs/5edYtD/n9d/f +p/8+Gbpof06Ni0a3Vzv0kzyLxsXF6TfjotHy9AM5xhU5jebvccfvt/vNvdNfSX/tlH1vo+Xj63FL +TgP/7PIxPx+d8h4X+tLQq0Mbze/58/P8K0MX5WcfPyEF2Z/wj6DHk4+fHsn5k31OPrEPZQPwidDv +yry+f+iZofJl66dbU+9Gt9d5XV6PP0g5m0Ldu8JvUbnyieyf8pdDHx96TOgTQ58b2uj6+K7zB+Rn +bP0x/+kB+BWIA2q0fN858df2/eQcnn7rcXRK/v7U0HNCG12efYX7sN2bwu/lX9Mv3wq9U+TSa+6a +3/KNN7o8ccLs2+ppXLNz/lPk3iXz0j0zDfI3vXXML/kO+Ls5b/Ezaazz/+Xgd68nPxT3GvV7tf+W +79T9FvYnviuNLs/3f9FyGuStXjHq7zwlX4O8SeolHtj3rkHe7G1DHr0/vyp+lb4bjXX+1cEvDuI9 ++Q5sk3kr70jnIecndnqe48/8qvx+bSi9zhvz+xuhja6P86v7y3yXv1Pqt7X7paE/lt493g89CL9f +eWy8N/t//h7sgO6n5Wcn/2+jy3Pub3763Ubzsz/Pq1cD/8b3jPbSL9JD80ulB2rgX7pq8LN700Py +y+ZHaf14bgsKyFuNPPWQB4xfqfWq/W3EDzlnyo/m+93o8jZXfqPlGRddL/7njSl+9skp/qX3jfdB +/+Nef/pG54SG8vGzZzuf9zmM/pF+jL7B/p7flnsEnfvoy/lb8t9qdH22yzrJf8138rj8nT/4u/Ob +/x2/W3kuxIU0urwfzrr5oFD7On4M9nVT+UlXPjDexzdSIP9TcRz8aZ+Yev9W6AWhrw7961D24Yb6 +K8+85U/NjmSf6ztOT+c98vtstHz1ZrdXT/4Lb0p91btB3sYPjv7hN0H/aj10LudPd27exzdD75T9 +V6Pl45MHGP1w5DTwL3141E+8g3klzp/e1v7XvBFHJZ/yYSlHHKV7qd2n01D+2sdH+cplP/Ad4rfY +aH7rkH72/uVDajS/9yBPJf22fXmj+dkj5OGRp8p8bjT/j2c8OU9bT6b0U/rvjPC/JNS9lfLMNbp8 +cdMfCb/3upbfjea/R96/+3asS1P+wOr/uPCfFPoLoeLeGl2+vL6vCZ/94hvyu9H8H81z/Ew/m99f +CG00P78Geq89M2/lu2w0/+Py/EmhvxD6lNBG8/9+nntN6GtD/yi00fzulftCnr8+9KbQRvPvkXPb +fUPvF/qg0Ebz/3yee0rohtBfDW00/2vz3B+F/nHoFaGN5r82z/FbvSm/75RzaKP5+cmyu/54+Kbu +PzL+z8jzLwk9O/SVoY318v9prJ/sO/wq5G2xP2pM8TsPLMpv3yrO13o8tZ9Tf3zs+/gu6IoHXX/8 +znnykZPXmMcv7+7m8ov3833nv+dc3ujym599dlF+z/1uCmKXXLT/mp9+YlF+diHx1PQaU/ksvH98 +l6X++HyPG91/zS8f4CT/p8f8sc9wzpU/Q/w+Oxf9Crsxuz29gvPsf6TC4tjo1fj1uT9myn6rfre3 +Ho31/kv7u57ikK6c+v6HX7vsO1DtbCh/02dG/5+UB+w77TPtk52/jI9GyyPHudw+1vm5McW/IQyL +8qu3eaTe9O6NLh+fduPjF9KYx89/xrglrzGP37zeXH7nLPoJ80Te0UaX3/zm8aL8nqOPl+d00f5r +fueAKf61z47x7DxHD2u9mLLnzuPXj4vyvzHz9vJQceXyTjT0/9rnRv3Z4eTXtE7x4z4y++KjQxvk +bfr8kPfZ7KeuC70xtO8tPyr7nWfG/+zfQxstX3zR7uE/NLTlvT/yPh16fWij5R+QfhRnjbKDNppf +fjrxtxsiz31HjSl+9zosyi8vmTwmL0v53m+jy78mz19b9Pr8bjT/jXlOnjH3pLkvstH88onJby5O +1f38jSl+91Iuyk+vcXzGO8rvutHlPyl89DTG9VR8ovnzJzmHvTX0A6H/GNro8pv/6vBtKf8HN5Nf +vd9RfJ/L70bXn3/DoutIY0reF7IezVuXGlPyxMdtqbw3Zf16V+j7Qj8S+vbEvYl/a3T9PhM++Vev +y+8vhTam+MUzTvGv3TDWf/63+6eg8/PeLwq9NNQ98+7b/XL+3lC/pZuGfOcleWBQ56BG87O7i2tC +5Yto4F/+xiifPYk9x/nPPq6Bf+nmwU9fiQ91Djk3AvjnN8jbeMuQRw9AP+48bV/Nfkcv3mh5LUde +eHrORvN3ufzOyGng33TraM998oB7DMSns2s18K9+c/Dzb3QucN6lx+an6LvaaHn0Gy3HubWBf/lb +oz7sr/wV6G/oH5zHxCE2yNt025DHLilOT3+bBw38G+M5yA7m/Cr/vzwj7pncK99xdp4D8/vmrCMP +yrrU6PL4v/EPk0/NfRn2Cbvku3lV5Cun0fI3ho8dSx4zcRb3yrrCT+jcyFdOo+WL77Ve/XPk3Tvf +EfrQh+S3/NjKaZC/8l3DkzPVn22TB9m72Ecaze+5A/Og+CX+Xw38qymfnZ4dlN8kf+g7571P+ZOQ +N4+f3bOhPit3H/3BXu4eBX4LzvX8rdnfxVM+K/VstHznM/6xzn/vb8YA/2ybUT/fNXZI/jPmo3pO +2aOWth3y6G/ZwXzfxAk01Ae/+1z0i/nNb62Bf2W7Ub719eA8KB+luBLx8saJ7wn7uPM2vVijy/M+ +lceOz9/DuN3q5W8/2uueNXpf48g5vrFe//DLB2aevTLjznlb3uWv5+93zwR3Dtkhv/cJFQf9xPx+ +cqh4Z+vb1P2X6ifeXr6VzS1PfDU7W6P7Qz/wkxHH4vvCr5HfgH3Etukf/mr8UfkTnJz/u49CPze6 +PurhPXuv6tVo/ql6Nprfe5ffoN97A//yjmN8qr/15G3pB+/VPY38ZnfPeHEPyoH5fWLoyaHPDzWe +3O/40vzd/QONrp94Wx+uLS3vypTb6PLo6e3DxF+9K4z8NvnRssP3PoadnZ9NfzfowRtdH/o55W9u +eY2W773Pa7fvqHLV2zhptHz11m/u3Se30fxT46iBf2WXMb7tf+3TrfsbwsiO53smrpF+2/fHeYa+ +/ePht5/VP/bV7pnj58P/oTFVX+vKVH3ZB+0DvlPq75wwVR9+MuwR3b/ymlkvxd3IW7i1+p89wbjd +Leuh99no99ftxGccNPBv3HWMV/5Q9kHuP/N948fDf0ceOt+vi1Jf5xx5Nazf4h7c831E1kX3Fz41 +v58V+uLQ3w59Waj7wN0nI78h/5Yb89w2Ob+Ib290+z8+UX/5P7v+8rnw9zk25f98qP3Ohvye117t +s/+5Is/fUe2dV98npNzNLb/R/et+G3EZ3Y9T79l7dZ8nP8J+z/IgNLo+U+O0gX919zFfxGPQ1/DT +8F3j/+Ccwt7N/kivwq5vnvm+N5Q/u89/Ld/5bPx1tn4uc66gh+M3ar1utPzmY3cUT9xofs+JK7Fu +iuPlj2pcNFqe76V20SeyqzaaXz5x/PyjF+VXb3FG6j25vu4x3oy4An4U/JO/mvnWWK//0uA/+XvG +E8/bZtAXhZ4f2mh++S8uiH3wD0PvFrn3DD06tNHyZtED3T+U3vyu0ZdtF7pTaIO81X1G+8Tx85ug +T5OvtIF/5QGDX35Qekj7L/H45NI70PfJQ9po+fT95PMj21ryjcstrZ+4t8PyHZMnmX1UHOui7RUn +RV8knl+/Nbq/+GU0f98Dvqg886f5d097G+oz23eMj473Esco/s2+hj1bPzlv/EPKcS+je/3+KPOo +0eXbnznX8Sez7/QdoKejT2EfXs46Yd90cX7zB74sv/809MrQd4XK/8x//UezPzoo9FWZx9rV6Pac +E75zQ+mJfzByfiT0J0IbLW/PPLel/PwEvWd5lOWR8J7d/+89Pz3vVT5080V+ZOOg0fW3rjtnOhfc +O/IbzT+vPvzgG82vncb1VLuMY3q8fTNO3DdqnNmP9jiTf+rj4WOPMM4aXV/7ZnGSGyLHOaDR/PPq +I96n0fzyeMk/qd3mWefZ6naaN4dl/P966ItCzYvz8pt9+Ir8viqUP/ot+S2fxI6ZD+ZVo9sjX7S8 +jn8WecprNH+Xv1PK3zW0gX/5gLG+0q+La6TXp0eyrrlHxv77znnv3x0qDkycfr8X56tG10c5H82D +7N/y4rD7/V7mp/uY3IPwlvydn09DeWuHjPbbb7BzOB8cGEb9wv7dIG/10CGPvojdhb17ar+6tDz4 +n5/682/Sr+LkGspfOWzwuz9GHJN9iTjmx2dcXBg7Y2NKHvuU9yHuvOW/fEH5v5bx0/wnLMgvT/u/ +RI57wA5OOxvat3bU6C/xmt6z+xzE770ocuQ3+Wh+8wNqtPytLc/5k97JvST8Io1/9mx+0+IDb834 +ukv6ix1AXlv3A5+V/3e7G93ert+i5f9kymv8N/mpf8s137tdDfJmjxnvn98Dv3t6tLOaMcC/8tjB +794+6wW9MDkN/EuPG/zumRA3Qp/nvG+/KQ6wQd7ycUMevYbzDXsVvaLzr/vMxAmfmH59aqh9VaPL +uygP0Hdfk9/sZ/xH5HVpkLd2/Ki/eB9+PuLd2WEb+DeeMPj55fBDYa8SH89uwS/euiwPg/0Yfyt6 +EP5O4icuSz8Zj6v5LY+DOMYv5u97Znyzgz04vx8aemCoeEPrYqPb2/d6TNXf+eSOqs8dLX/qfcq/ +u7nvR70b3d93tPx545Y/gHbxp290feeNa+cq4+CHMv4aLW9qnDeaf2rcN/AvnTTmN/8b+lPfbXkq +/iLzjR260fL4zx8evkeH2oc11vlX/mt93CfV9ZFv6NTIbbQ8fojiuOlf+R808K8+YdTH/XDy3fIL +5LdJLj8ofobqt2/OH4eGulfqsfl9bKj7peRHd17iV/3+PNfo+vJLkW+Bnx2/AecGdhn7u1szXuX7 +Eue8lHL3Dv1k6NdCbw2lB3UfzX7Z14kT5i93QP5+ROhRofJfNbp99PHsCfT53kuj+e2zzdd/D4P3 +2Gh+9zedlPG3sH9pxpN+NC6027640eXr57um39BF+72xLv+Xxni3HvAPYH+V56nR/Py95I11bnJP +on0Hu4X757wH558u/4b0t/32VH2WNoz2yH9hH+RexuFdO1uvj3xz/Erdr9TQ3nny+Qn8Rhi107po +nygPPHuqfYt8G+JKnpZ5KT/eJfm9tepr33ZH1a+x3n+njvdjvBk36/6waecBofyWGlPy+F3/VcaP +PDv02P87f+fHxd9J3gvrp/ck70XXz3tqqN/GZ4z2HpMH7JudT9gH5INuuyF7sfHjnCGP6XvTjqtD +3T8tjuvL+Tt9kvbItyg/xufS3/ypd8/6fkboi0PfG/qR0O2yHt0r9JGhx4XKY9jo/tEfznv0RPqH +Hot9mD2m+8v5SL5b9x64t2fR/vxw+o1fxNbqT/lAfjH91/0rv+ll+X/395fy96+Gdv/L37h/+n9L +38e8cdN64/tn3Mjfd3l+N7zvpWeP+UBfIO+YOGx6Nu+fH1Oj5ckjvLXkWZ/EAdHvqndDfWbPGe3z +nTkyD7b9/pz83fjmL2v8NshfOX0x+fzCj844FrcvD1Cj5fOj5ffAL2yHyOs4EnHE7IUN8jc+f9Rf +XJV9Af0eP+0G/uVfG/zy3NM3qa97OtmvXpr6Xhgqz1ej5fNvc+/o1zPf3AfVaP77pjz3Zv1G+MVn +XpPf5DZanvVLu54UfnIbzT+vH+hXdo08chstj5+y9ZHd9euZ//Kdkdtoeb7D7CH0OL8ZeeeHKqdB +3toLx/iYlx+SnYI+jJ9/Y13emUOeeSwvjngM/srWL/ETjZZHr+e9Wke1u9H84tfl5dgp/UNOo/nd +s+S+vq9mvJLTwL9y1ugP+3v3PNEb0NvSV7o3r9HyrPvyUnwtDLeGWh/JbbQ85zxy+FkcnHY2mt+9 +LOTwH2Eva+Bffsnon53zwD1D6VP1l3Fk/Ts+z1m3Gy1/Hv/UfYDqR79CjnL5ETTWyz97tA+/+rPT +KV+8ovXc+d25iD2Uvb7R5fE/8h1u+2AD/+o5o76+5+TwYxJX18C/8vu3j3/1vMFvvoj/cj+G75bv +LD20eFp6C+dk49l5WVyNOAj5AsXR0Ac1tG9e/eTZMZ+Nj61dX/pCcRtbWv+pfmt0+6f6tYF/7eXj +/dJbWgevysbxE6GN5pf/97I86NxyZNYr5z36J/4nfX89fePh+Z67D/O4/P5w6KdC3Qstz8rHYi92 +H/OZ8V/8rdBrQq8NvTn0m6F3iz/YzqF7hT4w9OjQE0KfE/rC0FO+e3TA00NPD90Yel7optArQldD +bwq9JfT4uw95K6Enh54S+puhF4ZeHvrnobeFbhu/0F1C7x26d+iDQl8Wen7oH4ReGnp16CdC/y30 +ztuMen5P6A6hLwz9/dA/Cb029F9D99h28B8cekzoKaHPCD0t9PTQt4b+Veidtxtytg3dJfRRoceG +vij0daF/E/rp0H8OvTb0+tCDth/yDwk9I/Q3Q18X+vrQPw79YOgNoV8PvSX0ttDt7jHk7xC6X+j7 +d5jNZrPZ7Lt3nM1ms//L3rmG5nmWcfyth3YkVNPNtWAhZK0Tt7EpMsU6JxFxbCjasVKGxRLU1TFb +inVrIMIWGLrRGp2Hgh0bC9rg6mEHu+7gjNWRuSpmK7OtczHpq9KDPYJtbY1Y/HD/fgH/8PC8aat+ +2f/Ln+dJrvt8X+91H67raTQ65pT/+x3chPddWN7/Hb6CH/Jr4A3wj+GtfKjgefgH84r8CNzHD88A +/Bz8EqyeSKR++CXz8tfwj5hvdfPxDP/Xxfyqm4+f4/9Ww87PdTwvY17dC38N/gh6bimcaLU+T1Be +9cthnqv0x/+qvKl/e9Cf7uNM8LwPVp9eQn+pT42zov56mPa7HT1Tp4+e5P+G4QNwItvbclxGeR6i +Xb8PNynHXriNdC+EzecIz+q/Begv9dkYz+qvLvTTe+El8FL4ZvjTsPf7W9V3l6C3roKnq/++hNzd +8Gb4cVj92MF8fh/cDed8n+D9PvgW5vkaOPXBT3j/Uzj1w79438W8cp4lsr97K9rzAd5vgv09mW79 +l1C/rP9u3lv/dsp/EWx7vPMc67M5yr+e/roPzv56rbz/iRwvaX9+ELtT+1N7UT13Ej13MfvMV6NX +FsEvwtqTZ3ieib55O3wVrN45iP6ZRM9ofx1Ar2gvfZL+T72hvZTzehvjYgz+C/xX+BD8N3gSTnto +DnZM2kMDvN8APwinffRb3qd9dJj3p+AG9oz2kvbRvbxP+2i0o/TvBdgxVfbNbuyaCVh7Zyb2zJvg +Tjjtny/zfgDWHnqe51F4HN4D74eHsJPG4b3wSfif8Az0iPM4Md3xm+O1bjxq/9rfJ+iXqv7J9n8L +/ZD2pe0/8zzXbw7jwvHy/y7Pfzt/599N9IvrjxGeFzLecvxex/scv8d4v4UNu2dhx2six59+kepN +19vaiZ9C/62CUw+6ft2O/nM9+X703ldh15faVe9AD6bdknou9Zrj2vVd6i311E7a0/Wbemop460X +7oNdz6mn7ue967edPP8RrlvPzWL9dgZ9lXpqLv02DKce2sX7XeiV1DtHeJ96J+28tOvSbtNOeyN2 +zWz4YjiR48d9oxX8o+eFni8kUj73YfpYr7mvkkj5dv7ffZvcd8l9lrp1SiLzy32TScb36xjP3fz+ +Xg8/yjh8Cv4Z42cEznX/UcaL6/ghxoHr9mH63XX7Xdin98CJLL92xjrKqx1dtS9jPRZTn7p9mUTm +/zT5jsDjsPknUr6qPKspXyLl0y5Sj6iXc5/IflNP1+0TJTL/TsbBZfB1sPknUr6qPE3SSSjf/VjZ +79W/0v13/fk/j93seesLPB+E38U5nfeb9LdbxnvjM7m/u5/3iSyP/oumb3qem5qe8UISmZ7+T8Yd +1T/dcx7vV3sf9tskqJ9vItP3HMR7tq2e59j+5reRjIwvYXzDROZvO1X1Q0L55pbS/37fznMZz3P1 +208oP7i1yOsv8QbGhX4RN/Ls9+Ly3uke/u65jn4Rx3mv3861jJuPwoksz2rkvwnfD/udtITyzadL +fbzPYn98DwH9vRPK9z9b5PWXNF7VCwjUnR8rb/y1hdT3w/DLcCLz91zY+O+y51IJ5Xt+UcrvOaL3 +UjzPNc6h/pieQ/tdK+9NJerSfxWBs03vBuT1l7Kcj/Jev/xnePY7BZ5TGh/JeiWy/Mbp9f6j8b70 +z0qkfFV5HCeJlLee9ovfn6iql993X8Q88JzS8zf91Vfy94T5d/2qjA/vnTg+vB9mPyRS3nsQ9rfr +jXfX5N/YXvLX30f7spcMvWeXMP/u3xR5zSLPzfW3Mw6I9fA+QcL0el4q6XlvwH7RH1a/jYTygzuK +vHEGFlJ/+8v1V0L5xs4i7/0gzXPvBdTd7+vaVeRtD38fvefg9wq9z+LvVMLymJ7+0t6rsJ9MN6F8 +k/LYH96/1+/Pft5NO62o0Yem5/1p5f1uYKvy+ksal0V/P++bJaxP/yulfbU3vDfhvNHuSKS896m9 +D6j/u/46bbSHfruJTM/vchh/2Hil+ssmUt5yOw+9p+r9VvWR90ETptc9VtrH+/Laafo3GIcrkfL+ +n3GM9T/VfnuMBNTv+r+N0W6JTD/j8GjXeG8wkfLGU7R9jFerfZyYkp8o7eN87uQf9UMxjsDjvDfu +TCLTcz6pt72v6r3jhPKNPaU83lvS38H0/B08RgLGUUlkeov4B+8xaj/rb59IefWTcZ0PImA5Eimv +3W8cMceNdl9iSr5Z2sN7kn5/XjvP+7KJlFfP6edje/h75u9BXXwyy+O9PPVv9rN2g/M3keWrSy+R +8raL7ep6M9vJeiYyPf2w9ZfwvtOpFATK9/yp9Jf+0H4f2/tmxoldz+/KNvY3E5nefPSI96u1Z/y9 +UH8Yv3cJ6eu/aJyctbw3f+P1PMn7Ns6nhihXq+XL/DI+pen/nHzG4bPNz/VXJ+lcCVsO62l7Zz6J +bG/9ls5XehfRrvPha2HbOZHlSf/HAeS/Dh+F69Lr3l/Gp/Pfe9g3cK64HE5YHuW93+t6dxPjU78G +v9euP7T20Gz6qYN8FsCJzO8056f6ddoem6l3IuWvJl/jqTo+N/D+h/BW2Pguh3h+kP3W7fAr8Dz2 +ERfDN8K/h29mf3Al/EU4keU1/z/U5D+XfKabvuXz/of7rJY3keWzXYxTdR/nMLZTIuVX0Z9rYOXv +pF2/Bbea3lPkfxQ+Dp9tepej/7yHNMQ4tZyJrN9G5J+Blf8C5foK3Gp6OW79vvcmxofjdpjnSxkX +l8N3wN4Xchze2eJ4vJt0jbOc+SWyPZxnWW7nneV23CcyPefhDsaJ8zDrnfPyNtrB+zBbeL6FdnAe +JTJ/81UPmG62c+bjvHOemW/C/Bonir72ew3ug3rfXX8L9wG0U1zvGwfC/VHj/LkuM37ZI+hvv49h +HHj9jCb5+3sYB/phL+Z5OXr4NjiR9dF+tvxXkn6WN+NEZPkz/qLxqB4mvayfcXP0c8766sepf0/W +37hz+tV7jqC9kO1zPe3jvrLtlcj2cZ9zBfrROBC2cyLlsx1NR//Xt9FPppswvcHTZfx5vuq+tfFe +tbsTyjf/UeRdp/sdWf2RjZfvOk772vgj7jtcQX+6f/chnt1vXsazcSY+y7Pf/fT7zPqxJ7K8R5B3 +v8I4PW+lP/UPN47NAt57juB+unEx3Qe6lf8z3tldPBuHTnv8u7x/BNZPWL1rvPULGB/zYPs5kfXb +SLrnmp9xWRLm14Pnqv4xxtXWf919i4TyXTOK56vfl3Cfz/RcvyZS/iH+wXt7xhv2e1cJ5ZuvL/m7 +vs11redZfu9Bv7uE6fXPKum5n8q1man9l7r91Cbyyrmuv4YMjb/jvrLt5L6q7ZCwfKaf6ZrOPSkI +lG+0lfq5T9bH3/Vnq/OX628v8n4f6TDyzts/My8T5q+8ev8J/l82HmEi5Z9DzjgR8su8T0zJzy7l +N46K7aafX9X5jdiLPdONXaL/ScL8ejrKDozxJN3X9LxA/Wx8ROPCuI/7ZvSA8Ts+wbPf913Oczu/ +F6vgXngH/Co8H3v3UngWdnMiy3877Wr85A8gvwZeC4/CiUxPf1392C2/v5+fobxZnxHej8JZvybv +XR9YzkSWx3bsoz3XwS+itxMpn/2q3KEW5R0HnhsrZ7skzH9wbhlfa8lnPWy84m/w/B34ATiR6X2M +/7sJvhVeCVfhDu45V8F96ypU3Q8TFcNrCn5XqgrbKsa7cD+7Ch9nXFTBONZVqIr/LWbUVNA4nVXw +d7AKB7inWwWmeSU8p0/8OwAA//9qBDInAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAQA +8f8AAAAAAAAAAAAAAAAAAAAABwAAAAIAAQAAEEAAAAAAAAAAAAAAAAAAFAAAAAIAAQBAHUAAAAAA +AC0CAAAAAAAAHAAAAAIAAQCgH0AAAAAAAD4BAAAAAAAAJgAAAAIAAQAgIUAAAAAAABcBAAAAAAAA +NAAAAAIAAQCgEkUAAAAAAEAAAAAAAAAAOQAAAAIAAQDgEkUAAAAAAC8AAAAAAAAAUwAAAAIAAQAg +E0UAAAAAAA0AAAAAAAAAXAAAAAIAAQBAE0UAAAAAAFoFAAAAAAAAaAAAAAIAAQCgGEUAAAAAAEsA +AAAAAAAAdAAAAAIAAQAAGUUAAAAAAEsAAAAAAAAAgAAAAAIAAQBgGUUAAAAAAG8AAAAAAAAAjQAA +AAIAAQDgGUUAAAAAAHIAAAAAAAAAmgAAAAIAAQBgGkUAAAAAAHIAAAAAAAAApwAAAAIAAQDgGkUA +AAAAAHIAAAAAAAAAtQAAAAIAAQBgG0UAAAAAAHIAAAAAAAAAwwAAAAIAAQDgG0UAAAAAAHYAAAAA +AAAA0QAAAAIAAQBgHEUAAAAAAHYAAAAAAAAA3wAAAAIAAQDgHEUAAAAAAHYAAAAAAAAA7gAAAAIA +AQBgHUUAAAAAAHYAAAAAAAAA/QAAAAIAAQDgHUUAAAAAAHYAAAAAAAAADAEAAAIAAQCQU0UAAAAA +AAAAAAAAAAAAGgEAAAEACQCwgEsAAAAAABAAAAAAAAAALQEAAAEACQAAgksAAAAAADMAAAAAAAAA +QAEAAAEACQCYgEsAAAAAAA0AAAAAAAAAUwEAAAEACQDAgUsAAAAAACoAAAAAAAAAZgEAAAEAAgCA +aUcAAAAAAAoAAAAAAAAAgwEAAAEAAgAgaUcAAAAAAAkAAAAAAAAAoAEAAAEAAgAYaUcAAAAAAAgA +AAAAAAAAyQEAAAEAAgBgaUcAAAAAAAoAAAAAAAAA6gEAAAEAAgBwaUcAAAAAAAoAAAAAAAAACQIA +AAEAAgBAekcAAAAAAAABAAAAAAAADwIAAAEAAgBAe0cAAAAAAAABAAAAAAAAFgIAAAEAAgBQakcA +AAAAABQAAAAAAAAALQIAAAEAAgBAaUcAAAAAAAoAAAAAAAAATgIAAAEAAgBQaUcAAAAAAAoAAAAA +AAAAcAIAAAEAAgAwaUcAAAAAAAoAAAAAAAAAnwIAAAEAAgBIaEcAAAAAAAgAAAAAAAAAtQIAAAEA +AgBQaEcAAAAAAAgAAAAAAAAAywIAAAEAAgBYaEcAAAAAAAgAAAAAAAAA4QIAAAEAAgBgaEcAAAAA +AAgAAAAAAAAA9wIAAAEAAgBoaEcAAAAAAAgAAAAAAAAADQMAAAEAAgBwaEcAAAAAAAgAAAAAAAAA +IwMAAAEAAgB4aEcAAAAAAAgAAAAAAAAAOQMAAAEAAgCAaEcAAAAAAAgAAAAAAAAATwMAAAEAAgCI +aEcAAAAAAAgAAAAAAAAAZQMAAAEAAgCQaEcAAAAAAAgAAAAAAAAAewMAAAEAAgCYaEcAAAAAAAgA +AAAAAAAAkQMAAAEAAgCgaEcAAAAAAAgAAAAAAAAApwMAAAEAAgCoaEcAAAAAAAgAAAAAAAAAvQMA +AAEAAgCwaEcAAAAAAAgAAAAAAAAA0wMAAAEAAgC4aEcAAAAAAAgAAAAAAAAA6QMAAAEAAgDAaEcA +AAAAAAgAAAAAAAAA/wMAAAEAAgDIaEcAAAAAAAgAAAAAAAAAFQQAAAEAAgDQaEcAAAAAAAgAAAAA +AAAAKwQAAAEAAgDYaEcAAAAAAAgAAAAAAAAAQQQAAAEAAgDgaEcAAAAAAAgAAAAAAAAAVwQAAAEA +AgDoaEcAAAAAAAgAAAAAAAAAbQQAAAEAAgDwaEcAAAAAAAgAAAAAAAAAgwQAAAEAAgD4aEcAAAAA +AAgAAAAAAAAAmQQAAAEAAgAAaUcAAAAAAAgAAAAAAAAArwQAAAEABABAhkcAAAAAAMACAAAAAAAA +wAQAAAEABQAAiUcAAAAAAAgAAAAAAAAA0QQAAAEABwAgiUcAAAAAAAAAAAAAAAAA4QQAAAEAAgAg +fkcAAAAAAJgGAAAAAAAA9QQAAAEAAgAAYEUAAAAAAAAAAAAAAAAABAUAAAEAAgC4hEcAAAAAAAAA +AAAAAAAAFAUAAAEAAgAAYEUAAAAAAAAAAAAAAAAAIgUAAAEAAgC4hEcAAAAAAAAAAAAAAAAAMQUA +AAEACQAggEsAAAAAAAAAAAAAAAAAQwUAAAEACQCgkUsAAAAAAAAAAAAAAAAAVgUAAAEACgCgkUsA +AAAAAAAAAAAAAAAAYwUAAAEACgCQs0sAAAAAAAAAAAAAAAAAcQUAAAEACwCgs0sAAAAAAAAAAAAA +AAAAfQUAAAEACwDInk4AAAAAAAAAAAAAAAAAigUAAAEADADgnk4AAAAAAAAAAAAAAAAAmwUAAAEA +DAAA8k4AAAAAAAAAAAAAAAAArQUAAAEADAAA8k4AAAAAAAAAAAAAAAAAuQUAAAEABwAIdksAAAAA +AAAAAAAAAAAAygUAAAEABgAIiUcAAAAAAAAAAAAAAAAA2gUAAAEAAgCtZ0cAAAAAAJMAAAAAAAAA +6QUAAAEAAgBAaEcAAAAAAAAAAAAAAAAA+QUAAAEAAgBYYEcAAAAAAFUHAAAAAAAABwYAAAEAAgCt +Z0cAAAAAAAAAAAAAAAAAFgYAAAEAAgDgFEYAAAAAAAAAAAAAAAAAIgYAAAEAAgAYZ0YAAAAAAAAA +AAAAAAAALAYAAAEAAgAsX0cAAAAAAAAAAAAAAAAAPQYAAAEABgAIiUcAAAAAAAAAAAAAAAAATAYA +ABIAAQAAEEAAAAAAAFkAAAAAAAAAZAYAABIAAQBgEEAAAAAAACUGAAAAAAAAgAYAABIAAQCgFkAA +AAAAAEUEAAAAAAAAlAYAABIAAQAAG0AAAAAAABsAAAAAAAAArAYAABIAAQAgG0AAAAAAABEAAAAA +AAAAxQYAABIAAQBAG0AAAAAAAIcAAAAAAAAA4gYAABIAAQDgG0AAAAAAAJQAAAAAAAAAAwcAABIA +AQCAHEAAAAAAAG8AAAAAAAAAJAcAABIAAQAAHUAAAAAAACIAAAAAAAAAPAcAABIAAQCAH0AAAAAA +AA4AAAAAAAAATgcAABIAAQDgIEAAAAAAABsAAAAAAAAAXwcAABIAAQAAIUAAAAAAABwAAAAAAAAA +dwcAABIAAQBAIkAAAAAAABgAAAAAAAAAnQcAABIAAQBgIkAAAAAAAEMAAAAAAAAAuwcAABIAAQDA +IkAAAAAAAEMAAAAAAAAAzgcAABIAAQAgI0AAAAAAAAYAAAAAAAAA4AcAABIAAQBAI0AAAAAAAAkA +AAAAAAAA8gcAABIAAQBgI0AAAAAAAAoAAAAAAAAABQgAABIAAQCAI0AAAAAAAAgAAAAAAAAAGAgA +ABIAAQCgI0AAAAAAAAoAAAAAAAAAKwgAABIAAQDAI0AAAAAAABoAAAAAAAAAPwgAABIAAQDgI0AA +AAAAABYAAAAAAAAAUAgAABIAAQAAJEAAAAAAABcAAAAAAAAAYQgAABIAAQAgJEAAAAAAADcAAAAA +AAAAcggAABIAAQBgJEAAAAAAADkAAAAAAAAAhAgAABIAAQCgJEAAAAAAAFcAAAAAAAAAlQgAABIA +AQAAJUAAAAAAAFkAAAAAAAAAqAgAABIAAQBgJUAAAAAAAFkAAAAAAAAAvggAABIAAQDAJUAAAAAA +AMUAAAAAAAAAzggAABIAAQCgJkAAAAAAAMUAAAAAAAAA3ggAABIAAQCAJ0AAAAAAAKUAAAAAAAAA +7ggAABIAAQBAKEAAAAAAAFwAAAAAAAAA/wgAABIAAQCgKEAAAAAAAD0AAAAAAAAAFAkAABIAAQDg +KEAAAAAAAO8AAAAAAAAAIQkAABIAAQDgKUAAAAAAAHoAAAAAAAAANAkAABIAAQBgKkAAAAAAALQA +AAAAAAAAQwkAABIAAQAgK0AAAAAAAEgAAAAAAAAAWAkAABIAAQCAK0AAAAAAAF8BAAAAAAAAagkA +ABIAAQDgLEAAAAAAAGgAAAAAAAAAggkAABIAAQBgLUAAAAAAAOYAAAAAAAAAkgkAABIAAQBgLkAA +AAAAAMEAAAAAAAAAqQkAABIAAQBAL0AAAAAAAP8AAAAAAAAAxgkAABIAAQBAMEAAAAAAAJUAAAAA +AAAA6QkAABIAAQDgMEAAAAAAAIsAAAAAAAAAAQoAABIAAQCAMUAAAAAAALUAAAAAAAAAGwoAABIA +AQBAMkAAAAAAAB8DAAAAAAAANgoAABIAAQBgNUAAAAAAADoAAAAAAAAAVwoAABIAAQCgNUAAAAAA +AMYAAAAAAAAAbAoAABIAAQCANkAAAAAAACUCAAAAAAAAhgoAABIAAQDAOEAAAAAAAMYBAAAAAAAA +lwoAABIAAQCgOkAAAAAAACcAAAAAAAAAqQoAABIAAQDgOkAAAAAAADAGAAAAAAAAugoAABIAAQAg +QUAAAAAAADYAAAAAAAAA0QoAABIAAQBgQUAAAAAAAC8BAAAAAAAA3goAABIAAQCgQkAAAAAAAHYA +AAAAAAAA8QoAABIAAQAgQ0AAAAAAAHsAAAAAAAAABAsAABIAAQCgQ0AAAAAAAAUEAAAAAAAAFgsA +ABIAAQDAR0AAAAAAACIAAAAAAAAAKAsAABIAAQAASEAAAAAAAMUGAAAAAAAAOQsAABIAAQDgTkAA +AAAAADYAAAAAAAAAUAsAABIAAQAgT0AAAAAAANEBAAAAAAAAXQsAABIAAQAAUUAAAAAAAFoAAAAA +AAAAdAsAABIAAQBgUUAAAAAAAFQAAAAAAAAAgwsAABIAAQDAUUAAAAAAAHYBAAAAAAAAnQsAABIA +AQBAU0AAAAAAAAYBAAAAAAAAvAsAABIAAQBgVEAAAAAAAG8CAAAAAAAA2wsAABIAAQDgVkAAAAAA +ANIAAAAAAAAA8gsAABIAAQDAV0AAAAAAAE4DAAAAAAAADwwAABIAAQAgW0AAAAAAAN4AAAAAAAAA +JQwAABIAAQAAXEAAAAAAAFABAAAAAAAAQQwAABIAAQBgXUAAAAAAAH0AAAAAAAAAWAwAABIAAQDg +XUAAAAAAAMUAAAAAAAAAbwwAABIAAQDAXkAAAAAAAFMAAAAAAAAAjAwAABIAAQAgX0AAAAAAAEUB +AAAAAAAAnQwAABIAAQCAYEAAAAAAAKkDAAAAAAAAwQwAABIAAQBAZEAAAAAAAFcAAAAAAAAA2wwA +ABIAAQCgZEAAAAAAAGIAAAAAAAAA/AwAABIAAQAgZUAAAAAAAAYAAAAAAAAAFQ0AABIAAQBAZUAA +AAAAACcGAAAAAAAALw0AABIAAQCAa0AAAAAAAFwFAAAAAAAAQA0AABIAAQDgcEAAAAAAAJoHAAAA +AAAAWw0AABIAAQCAeEAAAAAAAGoDAAAAAAAAbQ0AABIAAQAAfEAAAAAAAP8BAAAAAAAAhQ0AABIA +AQAAfkAAAAAAAEUAAAAAAAAAnw0AABIAAQBgfkAAAAAAAEYAAAAAAAAAuQ0AABIAAQDAfkAAAAAA +AO4CAAAAAAAAyQ0AABIAAQDAgUAAAAAAAEYAAAAAAAAA5w0AABIAAQAggkAAAAAAADoBAAAAAAAA +9w0AABIAAQBgg0AAAAAAAE4AAAAAAAAAFA4AABIAAQDAg0AAAAAAAKgDAAAAAAAAKQ4AABIAAQCA +h0AAAAAAANIAAAAAAAAAOw4AABIAAQBgiEAAAAAAAHoAAAAAAAAASw4AABIAAQDgiEAAAAAAAJgA +AAAAAAAAXw4AABIAAQCAiUAAAAAAAHcAAAAAAAAAdA4AABIAAQAAikAAAAAAAIwAAAAAAAAAhw4A +ABIAAQCgikAAAAAAAHcAAAAAAAAAnQ4AABIAAQAgi0AAAAAAAC0AAAAAAAAAtw4AABIAAQBgi0AA +AAAAAEUBAAAAAAAAzw4AABIAAQDAjEAAAAAAAKUAAAAAAAAA5g4AABIAAQCAjUAAAAAAADYAAAAA +AAAA8w4AABIAAQDAjUAAAAAAAJ4BAAAAAAAAAQ8AABIAAQBgj0AAAAAAADQAAAAAAAAAEA8AABIA +AQCgj0AAAAAAAKUAAAAAAAAAIA8AABIAAQBgkEAAAAAAAJQAAAAAAAAAMw8AABIAAQAAkUAAAAAA +AOcAAAAAAAAARQ8AABIAAQAAkkAAAAAAAMQBAAAAAAAAYQ8AABIAAQDgk0AAAAAAAGsAAAAAAAAA +dA8AABIAAQBglEAAAAAAAI0AAAAAAAAAiA8AABIAAQAAlUAAAAAAAHoAAAAAAAAAoA8AABIAAQCA +lUAAAAAAAD4AAAAAAAAAtQ8AABIAAQDAlUAAAAAAADQAAAAAAAAAzA8AABIAAQAAlkAAAAAAABkD +AAAAAAAA3w8AABIAAQAgmUAAAAAAAI4GAAAAAAAA+Q8AABIAAQDAn0AAAAAAACUBAAAAAAAAExAA +ABIAAQAAoUAAAAAAABQCAAAAAAAALhAAABIAAQAgo0AAAAAAAAUJAAAAAAAAPxAAABIAAQBArEAA +AAAAAKUAAAAAAAAAYxAAABIAAQAArUAAAAAAAEIAAAAAAAAAdRAAABIAAQBgrUAAAAAAAKsAAAAA +AAAAhhAAABIAAQAgrkAAAAAAAM4AAAAAAAAAmxAAABIAAQAAr0AAAAAAAEgBAAAAAAAArxAAABIA +AQBgsEAAAAAAALMAAAAAAAAAxxAAABIAAQAgsUAAAAAAAEgAAAAAAAAA5RAAABIAAQCAsUAAAAAA +AL8CAAAAAAAA/hAAABIAAQBAtEAAAAAAAO4AAAAAAAAAGxEAABIAAQBAtUAAAAAAAHMAAAAAAAAA +OREAABIAAQDAtUAAAAAAAFkCAAAAAAAAVREAABIAAQAguEAAAAAAAKsBAAAAAAAAZREAABIAAQDg +uUAAAAAAAE0CAAAAAAAAfREAABIAAQBAvEAAAAAAAC4CAAAAAAAAkBEAABIAAQCAvkAAAAAAABIF +AAAAAAAAohEAABIAAQCgw0AAAAAAANUBAAAAAAAAsxEAABIAAQCAxUAAAAAAAJkAAAAAAAAAxBEA +ABIAAQAgxkAAAAAAAJIFAAAAAAAA1REAABIAAQDAy0AAAAAAAOsAAAAAAAAA8xEAABIAAQDAzEAA +AAAAAJABAAAAAAAADRIAABIAAQBgzkAAAAAAAJ0BAAAAAAAAJxIAABIAAQAA0EAAAAAAAP8CAAAA +AAAAQBIAABIAAQAA00AAAAAAAJkAAAAAAAAAWBIAABIAAQCg00AAAAAAAMUDAAAAAAAAcBIAABIA +AQCA10AAAAAAAJsAAAAAAAAAhRIAABIAAQAg2EAAAAAAAM4AAAAAAAAAnBIAABIAAQAA2UAAAAAA +AFsAAAAAAAAAsBIAABIAAQBg2UAAAAAAAD4AAAAAAAAAyhIAABIAAQCg2UAAAAAAABAAAAAAAAAA +7BIAABIAAQDA2UAAAAAAAFwBAAAAAAAACxMAABIAAQAg20AAAAAAACUCAAAAAAAAHhMAABIAAQBg +3UAAAAAAADwBAAAAAAAAMRMAABIAAQCg3kAAAAAAAIYAAAAAAAAATBMAABIAAQBA30AAAAAAAL4A +AAAAAAAAZRMAABIAAQAA4EAAAAAAAI0AAAAAAAAAiBMAABIAAQCg4EAAAAAAACkEAAAAAAAApBMA +ABIAAQDg5EAAAAAAAN0BAAAAAAAAxxMAABIAAQDA5kAAAAAAAI8BAAAAAAAA4RMAABIAAQBg6EAA +AAAAADcCAAAAAAAA/RMAABIAAQCg6kAAAAAAAE8BAAAAAAAAFxQAABIAAQAA7EAAAAAAAOULAAAA +AAAALxQAABIAAQAA+EAAAAAAAIkDAAAAAAAATRQAABIAAQCg+0AAAAAAAOUAAAAAAAAAZxQAABIA +AQCg/EAAAAAAAGgGAAAAAAAAeRQAABIAAQAgA0EAAAAAAJQAAAAAAAAAkxQAABIAAQDAA0EAAAAA +ALQAAAAAAAAApxQAABIAAQCABEEAAAAAAGoAAAAAAAAAwRQAABIAAQAABUEAAAAAAGIAAAAAAAAA +1BQAABIAAQCABUEAAAAAAIsAAAAAAAAA7RQAABIAAQAgBkEAAAAAAIoCAAAAAAAABhUAABIAAQDA +CEEAAAAAAHECAAAAAAAAIxUAABIAAQBAC0EAAAAAAMkBAAAAAAAAQBUAABIAAQAgDUEAAAAAAN4A +AAAAAAAAYhUAABIAAQAADkEAAAAAAPoGAAAAAAAAgBUAABIAAQAAFUEAAAAAAOUAAAAAAAAAoBUA +ABIAAQAAFkEAAAAAAFIBAAAAAAAAuRUAABIAAQBgF0EAAAAAABABAAAAAAAA0RUAABIAAQCAGEEA +AAAAAHEAAAAAAAAA5xUAABIAAQAAGUEAAAAAAOwBAAAAAAAA/BUAABIAAQAAG0EAAAAAAPYAAAAA +AAAADRYAABIAAQAAHEEAAAAAACwCAAAAAAAAHxYAABIAAQBAHkEAAAAAAJoAAAAAAAAAMxYAABIA +AQDgHkEAAAAAAD4AAAAAAAAAQxYAABIAAQAgH0EAAAAAAL8AAAAAAAAAUhYAABIAAQDgH0EAAAAA +AGUCAAAAAAAAaRYAABIAAQBgIkEAAAAAAHsAAAAAAAAAehYAABIAAQDgIkEAAAAAACUBAAAAAAAA +lBYAABIAAQAgJEEAAAAAAFUAAAAAAAAAoxYAABIAAQCAJEEAAAAAAMUAAAAAAAAAtBYAABIAAQBg +JUEAAAAAALEAAAAAAAAA1RYAABIAAQAgJkEAAAAAAJAGAAAAAAAA5RYAABIAAQDALEEAAAAAADAB +AAAAAAAA+xYAABIAAQAALkEAAAAAAKwCAAAAAAAADhcAABIAAQDAMEEAAAAAALYAAAAAAAAAJxcA +ABIAAQCAMUEAAAAAANwMAAAAAAAAQRcAABIAAQBgPkEAAAAAAC4AAAAAAAAAYRcAABIAAQCgPkEA +AAAAAGkAAAAAAAAAfhcAABIAAQAgP0EAAAAAAFAEAAAAAAAAlRcAABIAAQCAQ0EAAAAAAJABAAAA +AAAAshcAABIAAQAgRUEAAAAAAO8EAAAAAAAAwRcAABIAAQAgSkEAAAAAAJoBAAAAAAAA0RcAABIA +AQDAS0EAAAAAAPAAAAAAAAAA6hcAABIAAQDATEEAAAAAADoBAAAAAAAA/RcAABIAAQAATkEAAAAA +ACACAAAAAAAADxgAABIAAQAgUEEAAAAAAMUBAAAAAAAAKRgAABIAAQAAUkEAAAAAAMoAAAAAAAAA +QRgAABIAAQDgUkEAAAAAAB8BAAAAAAAAXxgAABIAAQAAVEEAAAAAAPcCAAAAAAAAcBgAABIAAQAA +V0EAAAAAACoBAAAAAAAAhxgAABIAAQBAWEEAAAAAAKUAAAAAAAAAnRgAABIAAQAAWUEAAAAAAA8B +AAAAAAAAuRgAABIAAQAgWkEAAAAAAEwDAAAAAAAAzxgAABIAAQCAXUEAAAAAAJECAAAAAAAA5RgA +ABIAAQAgYEEAAAAAADYAAAAAAAAAARkAABIAAQBgYEEAAAAAAAUDAAAAAAAAGBkAABIAAQCAY0EA +AAAAAGcAAAAAAAAAMRkAABIAAQAAZEEAAAAAACUBAAAAAAAARhkAABIAAQBAZUEAAAAAAIUBAAAA +AAAAXhkAABIAAQDgZkEAAAAAAKUGAAAAAAAAcBkAABIAAQCgbUEAAAAAAFEAAAAAAAAAiBkAABIA +AQAAbkEAAAAAAFoCAAAAAAAAoBkAABIAAQBgcEEAAAAAAOUDAAAAAAAAsBkAABIAAQBgdEEAAAAA +APYBAAAAAAAAwRkAABIAAQBgdkEAAAAAAIcBAAAAAAAA0xkAABIAAQAAeEEAAAAAAEUDAAAAAAAA +5hkAABIAAQBge0EAAAAAADQCAAAAAAAA/xkAABIAAQCgfUEAAAAAAGQAAAAAAAAADRoAABIAAQAg +fkEAAAAAAP0CAAAAAAAAIBoAABIAAQAggUEAAAAAAAgEAAAAAAAANRoAABIAAQBAhUEAAAAAAP4A +AAAAAAAATRoAABIAAQBAhkEAAAAAALgAAAAAAAAAZhoAABIAAQAAh0EAAAAAAAEAAAAAAAAAdRoA +ABIAAQAgh0EAAAAAAFoAAAAAAAAAlxoAABIAAQCAh0EAAAAAAI8CAAAAAAAAvxoAABIAAQAgikEA +AAAAAOsAAAAAAAAA4xoAABIAAQAgi0EAAAAAAFYEAAAAAAAACRsAABIAAQCAj0EAAAAAAEkBAAAA +AAAAMxsAABIAAQDgkEEAAAAAAHQCAAAAAAAAZRsAABIAAQBgk0EAAAAAAIYDAAAAAAAAiRsAABIA +AQAAl0EAAAAAAHwAAAAAAAAAsxsAABIAAQCAl0EAAAAAAHgAAAAAAAAAxBsAABIAAQAAmEEAAAAA +ADUBAAAAAAAA3BsAABIAAQBAmUEAAAAAAJkAAAAAAAAA8hsAABIAAQDgmUEAAAAAANwAAAAAAAAA +CBwAABIAAQDAmkEAAAAAAMoCAAAAAAAAGxwAABIAAQCgnUEAAAAAAAoBAAAAAAAANBwAABIAAQDA +nkEAAAAAAEwBAAAAAAAAUhwAABIAAQAgoEEAAAAAAHwBAAAAAAAAaRwAABIAAQCgoUEAAAAAAAUB +AAAAAAAAjxwAABIAAQDAokEAAAAAAM8AAAAAAAAAtBwAABIAAQCgo0EAAAAAAMUAAAAAAAAA2xwA +ABIAAQCApEEAAAAAADcEAAAAAAAA/BwAABIAAQDAqEEAAAAAAGUBAAAAAAAAIx0AABIAAQBAqkEA +AAAAAGwBAAAAAAAATB0AABIAAQDAq0EAAAAAAKUBAAAAAAAAYB0AABIAAQCArUEAAAAAADUBAAAA +AAAAix0AABIAAQDArkEAAAAAAD4DAAAAAAAAtx0AABIAAQAAskEAAAAAACUBAAAAAAAA2B0AABIA +AQBAs0EAAAAAADoBAAAAAAAA+R0AABIAAQCAtEEAAAAAAIUBAAAAAAAAHR4AABIAAQAgtkEAAAAA +ABkBAAAAAAAANh4AABIAAQBAt0EAAAAAAKgBAAAAAAAAWB4AABIAAQAAuUEAAAAAAPAAAAAAAAAA +bh4AABIAAQAAukEAAAAAADgBAAAAAAAAfh4AABIAAQBAu0EAAAAAAFADAAAAAAAAjx4AABIAAQCg +vkEAAAAAAG8DAAAAAAAArB4AABIAAQAgwkEAAAAAAMoLAAAAAAAAyR4AABIAAQAAzkEAAAAAAPoC +AAAAAAAA6B4AABIAAQAA0UEAAAAAAFQBAAAAAAAAAh8AABIAAQBg0kEAAAAAAFYAAAAAAAAAGR8A +ABIAAQDA0kEAAAAAAAYBAAAAAAAALx8AABIAAQDg00EAAAAAAOYBAAAAAAAASh8AABIAAQDg1UEA +AAAAANoAAAAAAAAAYx8AABIAAQDA1kEAAAAAAOUAAAAAAAAAfR8AABIAAQDA10EAAAAAALkAAAAA +AAAAlx8AABIAAQCA2EEAAAAAAEwAAAAAAAAAuB8AABIAAQDg2EEAAAAAAEwAAAAAAAAA1h8AABIA +AQBA2UEAAAAAAPwBAAAAAAAA5x8AABIAAQBA20EAAAAAAEwAAAAAAAAA/h8AABIAAQCg20EAAAAA +AEwAAAAAAAAADyAAABIAAQAA3EEAAAAAAEwAAAAAAAAAHyAAABIAAQBg3EEAAAAAAIUAAAAAAAAA +MiAAABIAAQAA3UEAAAAAAK8AAAAAAAAAQiAAABIAAQDA3UEAAAAAAPYAAAAAAAAAXiAAABIAAQDA +3kEAAAAAANYAAAAAAAAAdCAAABIAAQCg30EAAAAAALMAAAAAAAAAkCAAABIAAQBg4EEAAAAAAKcB +AAAAAAAAoyAAABIAAQAg4kEAAAAAAHMAAAAAAAAAuSAAABIAAQCg4kEAAAAAAHkAAAAAAAAAzCAA +ABIAAQAg40EAAAAAAKYDAAAAAAAA4iAAABIAAQDg5kEAAAAAACwCAAAAAAAA+yAAABIAAQAg6UEA +AAAAADkFAAAAAAAAGSEAABIAAQBg7kEAAAAAABYBAAAAAAAAMCEAABIAAQCA70EAAAAAAIoAAAAA +AAAATSEAABIAAQAg8EEAAAAAAF8AAAAAAAAAaiEAABIAAQCA8EEAAAAAALwAAAAAAAAAhCEAABIA +AQBA8UEAAAAAAEUBAAAAAAAApCEAABIAAQCg8kEAAAAAAOcAAAAAAAAAxiEAABIAAQCg80EAAAAA +AMUFAAAAAAAA4SEAABIAAQCA+UEAAAAAAPQCAAAAAAAA9yEAABIAAQCA/EEAAAAAAHkAAAAAAAAA +ESIAABIAAQAA/UEAAAAAAFwAAAAAAAAAMSIAABIAAQBg/UEAAAAAAH8AAAAAAAAATSIAABIAAQDg +/UEAAAAAAG0DAAAAAAAAbSIAABIAAQBgAUIAAAAAAGkBAAAAAAAAiSIAABIAAQDgAkIAAAAAAAUB +AAAAAAAApSIAABIAAQAABEIAAAAAABUCAAAAAAAAuCIAABIAAQAgBkIAAAAAAKgAAAAAAAAA0SIA +ABIAAQDgBkIAAAAAADwBAAAAAAAA5SIAABIAAQAgCEIAAAAAABwDAAAAAAAA+SIAABIAAQBAC0IA +AAAAADQAAAAAAAAADiMAABIAAQCAC0IAAAAAAK4AAAAAAAAALCMAABIAAQBADEIAAAAAALcAAAAA +AAAARiMAABIAAQAADUIAAAAAAG4BAAAAAAAAYCMAABIAAQCADkIAAAAAAE8CAAAAAAAAeiMAABIA +AQDgEEIAAAAAAM8FAAAAAAAAliMAABIAAQDAFkIAAAAAAGcDAAAAAAAAtiMAABIAAQBAGkIAAAAA +ALcAAAAAAAAA2iMAABIAAQAAG0IAAAAAAOULAAAAAAAA9CMAABIAAQAAJ0IAAAAAAIUBAAAAAAAA +FCQAABIAAQCgKEIAAAAAALwCAAAAAAAALyQAABIAAQBgK0IAAAAAAN0CAAAAAAAASSQAABIAAQBA +LkIAAAAAAF8BAAAAAAAAYCQAABIAAQCgL0IAAAAAAGgBAAAAAAAAfSQAABIAAQAgMUIAAAAAAPkD +AAAAAAAAmiQAABIAAQAgNUIAAAAAAHYAAAAAAAAAvSQAABIAAQCgNUIAAAAAAHMAAAAAAAAA4CQA +ABIAAQAgNkIAAAAAALUAAAAAAAAA+yQAABIAAQDgNkIAAAAAAC4BAAAAAAAAFyUAABIAAQAgOEIA +AAAAABQCAAAAAAAAMiUAABIAAQBAOkIAAAAAAMkCAAAAAAAAVCUAABIAAQAgPUIAAAAAAAYBAAAA +AAAAcSUAABIAAQBAPkIAAAAAABgAAAAAAAAAjCUAABIAAQBgPkIAAAAAAAsBAAAAAAAAqyUAABIA +AQCAP0IAAAAAABgAAAAAAAAAyCUAABIAAQCgP0IAAAAAADACAAAAAAAA6CUAABIAAQDgQUIAAAAA +ANwBAAAAAAAACCYAABIAAQDAQ0IAAAAAAKcAAAAAAAAAIyYAABIAAQCAREIAAAAAAPQAAAAAAAAA +RCYAABIAAQCARUIAAAAAACQBAAAAAAAAZSYAABIAAQDARkIAAAAAAHYAAAAAAAAAhiYAABIAAQBA +R0IAAAAAAE4AAAAAAAAApSYAABIAAQCgR0IAAAAAALQAAAAAAAAAtyYAABIAAQBgSEIAAAAAAFcA +AAAAAAAAzCYAABIAAQDASEIAAAAAAF8AAAAAAAAA4SYAABIAAQAgSUIAAAAAAE0DAAAAAAAA8yYA +ABIAAQCATEIAAAAAADQAAAAAAAAAAycAABIAAQDATEIAAAAAAHYAAAAAAAAAGycAABIAAQBATUIA +AAAAAFkAAAAAAAAALycAABIAAQCgTUIAAAAAAMsAAAAAAAAASScAABIAAQCATkIAAAAAAKUBAAAA +AAAAXicAABIAAQBAUEIAAAAAADYAAAAAAAAAeScAABIAAQCAUEIAAAAAAK0AAAAAAAAAjCcAABIA +AQBAUUIAAAAAAO0AAAAAAAAAnycAABIAAQBAUkIAAAAAAP4BAAAAAAAAticAABIAAQBAVEIAAAAA +AD8CAAAAAAAAyScAABIAAQCAVkIAAAAAADsAAAAAAAAA4icAABIAAQDAVkIAAAAAAFQBAAAAAAAA +9CcAABIAAQAgWEIAAAAAADsAAAAAAAAADCgAABIAAQBgWEIAAAAAALkAAAAAAAAAHCgAABIAAQAg +WUIAAAAAAHEAAAAAAAAAMigAABIAAQCgWUIAAAAAAN4AAAAAAAAATSgAABIAAQCAWkIAAAAAAIYA +AAAAAAAAcigAABIAAQAgW0IAAAAAAKUAAAAAAAAAjSgAABIAAQDgW0IAAAAAAPwAAAAAAAAArCgA +ABIAAQDgXEIAAAAAABYBAAAAAAAA1ygAABIAAQAAXkIAAAAAAEUFAAAAAAAA8SgAABIAAQBgY0IA +AAAAAHQAAAAAAAAAEikAABIAAQDgY0IAAAAAACkCAAAAAAAAOykAABIAAQAgZkIAAAAAANQAAAAA +AAAAWykAABIAAQAAZ0IAAAAAAJQBAAAAAAAAcykAABIAAQCgaEIAAAAAAC8BAAAAAAAAiikAABIA +AQDgaUIAAAAAACUBAAAAAAAAoykAABIAAQAga0IAAAAAAJUAAAAAAAAAxikAABIAAQDAa0IAAAAA +AK8AAAAAAAAA5ykAABIAAQCAbEIAAAAAAAEAAAAAAAAA9ikAABIAAQCgbEIAAAAAALAAAAAAAAAA +ECoAABIAAQBgbUIAAAAAAPoAAAAAAAAANyoAABIAAQBgbkIAAAAAALAAAAAAAAAAXioAABIAAQAg +b0IAAAAAAHcAAAAAAAAAdSoAABIAAQCgb0IAAAAAAKQAAAAAAAAAiCoAABIAAQBgcEIAAAAAALEC +AAAAAAAAnCoAABIAAQAgc0IAAAAAABABAAAAAAAAtCoAABIAAQBAdEIAAAAAAGgAAAAAAAAAzyoA +ABIAAQDAdEIAAAAAAOoAAAAAAAAA5CoAABIAAQDAdUIAAAAAAPkBAAAAAAAA+CoAABIAAQDAd0IA +AAAAAOoAAAAAAAAADSsAABIAAQDAeEIAAAAAAC8DAAAAAAAAHSsAABIAAQAAfEIAAAAAAMYAAAAA +AAAAMCsAABIAAQDgfEIAAAAAAK0AAAAAAAAARCsAABIAAQCgfUIAAAAAAIUAAAAAAAAAXisAABIA +AQBAfkIAAAAAAPcAAAAAAAAAcysAABIAAQBAf0IAAAAAALsBAAAAAAAAhSsAABIAAQAAgUIAAAAA +AFkCAAAAAAAAlSsAABIAAQBgg0IAAAAAAB4BAAAAAAAApSsAABIAAQCAhEIAAAAAADABAAAAAAAA +vSsAABIAAQDAhUIAAAAAAHIAAAAAAAAAzCsAABIAAQBAhkIAAAAAAG8BAAAAAAAA4isAABIAAQDA +h0IAAAAAAHoAAAAAAAAA8ysAABIAAQBAiEIAAAAAAEgAAAAAAAAAASwAABIAAQCgiEIAAAAAAKEA +AAAAAAAAECwAABIAAQBgiUIAAAAAAGMAAAAAAAAAJCwAABIAAQDgiUIAAAAAAH4AAAAAAAAAOSwA +ABIAAQBgikIAAAAAAIUAAAAAAAAASSwAABIAAQAAi0IAAAAAALkAAAAAAAAAXSwAABIAAQDAi0IA +AAAAAGIAAAAAAAAAcSwAABIAAQBAjEIAAAAAAJQAAAAAAAAAhiwAABIAAQDgjEIAAAAAAJQAAAAA +AAAAnCwAABIAAQCAjUIAAAAAAJQAAAAAAAAAtSwAABIAAQAgjkIAAAAAAJQAAAAAAAAAzywAABIA +AQDAjkIAAAAAAJQAAAAAAAAA6CwAABIAAQBgj0IAAAAAAJQAAAAAAAAAAi0AABIAAQAAkEIAAAAA +AJQAAAAAAAAAGC0AABIAAQCgkEIAAAAAAJQAAAAAAAAALy0AABIAAQBAkUIAAAAAAJQAAAAAAAAA +SS0AABIAAQDgkUIAAAAAAJQAAAAAAAAAZC0AABIAAQCAkkIAAAAAAFEAAAAAAAAAdy0AABIAAQDg +kkIAAAAAAE0AAAAAAAAAiy0AABIAAQBAk0IAAAAAAEoCAAAAAAAAoi0AABIAAQCglUIAAAAAAGcA +AAAAAAAAsS0AABIAAQAglkIAAAAAAL8BAAAAAAAAwi0AABIAAQDgl0IAAAAAAEUBAAAAAAAA2S0A +ABIAAQBAmUIAAAAAAJABAAAAAAAA8C0AABIAAQDgmkIAAAAAAO0BAAAAAAAAAi4AABIAAQDgnEIA +AAAAAHoBAAAAAAAAGi4AABIAAQBgnkIAAAAAAC0AAAAAAAAAMS4AABIAAQCgnkIAAAAAAC0AAAAA +AAAARS4AABIAAQDgnkIAAAAAAHoBAAAAAAAAWS4AABIAAQBgoEIAAAAAAFIBAAAAAAAAcC4AABIA +AQDAoUIAAAAAAPwAAAAAAAAAhC4AABIAAQDAokIAAAAAALkAAAAAAAAAoS4AABIAAQCAo0IAAAAA +AHwAAAAAAAAAxC4AABIAAQAApEIAAAAAAPUCAAAAAAAA6S4AABIAAQAAp0IAAAAAAFwIAAAAAAAA +Ay8AABIAAQBgr0IAAAAAANkAAAAAAAAAGS8AABIAAQBAsEIAAAAAACUHAAAAAAAAKS8AABIAAQCA +t0IAAAAAAAYAAAAAAAAAOS8AABIAAQCgt0IAAAAAAC8AAAAAAAAASy8AABIAAQDgt0IAAAAAAIQA +AAAAAAAAWS8AABIAAQCAuEIAAAAAAGcAAAAAAAAAbS8AABIAAQAAuUIAAAAAADABAAAAAAAAfi8A +ABIAAQBAukIAAAAAAHAAAAAAAAAAkS8AABIAAQDAukIAAAAAAHsAAAAAAAAAqi8AABIAAQBAu0IA +AAAAAL0AAAAAAAAAvS8AABIAAQAAvEIAAAAAAKUAAAAAAAAA1i8AABIAAQDAvEIAAAAAAIUBAAAA +AAAA6y8AABIAAQBgvkIAAAAAAP4CAAAAAAAA/S8AABIAAQBgwUIAAAAAAKUAAAAAAAAAGDAAABIA +AQAgwkIAAAAAADUAAAAAAAAAKjAAABIAAQBgwkIAAAAAAHEFAAAAAAAAOzAAABIAAQDgx0IAAAAA +ANIBAAAAAAAASzAAABIAAQDAyUIAAAAAAFUAAAAAAAAAYTAAABIAAQAgykIAAAAAAMoAAAAAAAAA +cDAAABIAAQAAy0IAAAAAALIDAAAAAAAAiTAAABIAAQDAzkIAAAAAAG4BAAAAAAAAoDAAABIAAQBA +0EIAAAAAAGIAAAAAAAAAsjAAABIAAQDA0EIAAAAAAEwAAAAAAAAAxjAAABIAAQAg0UIAAAAAAFIB +AAAAAAAA1TAAABIAAQCA0kIAAAAAADYAAAAAAAAA5TAAABIAAQDA0kIAAAAAADYAAAAAAAAA9TAA +ABIAAQAA00IAAAAAAFYAAAAAAAAABzEAABIAAQBg00IAAAAAAFMCAAAAAAAAGjEAABIAAQDA1UIA +AAAAAJYAAAAAAAAALzEAABIAAQBg1kIAAAAAAPEAAAAAAAAAQTEAABIAAQBg10IAAAAAAFcAAAAA +AAAAUjEAABIAAQDA10IAAAAAABEBAAAAAAAAYzEAABIAAQDg2EIAAAAAADQAAAAAAAAAeDEAABIA +AQAg2UIAAAAAADQAAAAAAAAAjTEAABIAAQBg2UIAAAAAAH8AAAAAAAAAoTEAABIAAQDg2UIAAAAA +AMYAAAAAAAAAtDEAABIAAQDA2kIAAAAAACYCAAAAAAAAyTEAABIAAQAA3UIAAAAAAEoDAAAAAAAA +1jEAABIAAQBg4EIAAAAAADUAAAAAAAAA6TEAABIAAQCg4EIAAAAAADYAAAAAAAAA+DEAABIAAQDg +4EIAAAAAAPIAAAAAAAAADjIAABIAAQDg4UIAAAAAACQAAAAAAAAAHjIAABIAAQAg4kIAAAAAAC4B +AAAAAAAALTIAABIAAQBg40IAAAAAAHkAAAAAAAAAPTIAABIAAQDg40IAAAAAADcAAAAAAAAAUzIA +ABIAAQAg5EIAAAAAAPICAAAAAAAAaDIAABIAAQAg50IAAAAAADkDAAAAAAAAfTIAABIAAQBg6kIA +AAAAAC0AAAAAAAAAjjIAABIAAQCg6kIAAAAAAC0AAAAAAAAAoDIAABIAAQDg6kIAAAAAAEkAAAAA +AAAAtzIAABIAAQBA60IAAAAAAEkAAAAAAAAA0zIAABIAAQCg60IAAAAAACAAAAAAAAAA4zIAABIA +AQDA60IAAAAAAFwBAAAAAAAA8zIAABIAAQAg7UIAAAAAAKUAAAAAAAAABDMAABIAAQDg7UIAAAAA +AIgAAAAAAAAAGTMAABIAAQCA7kIAAAAAAHQBAAAAAAAAKTMAABIAAQAA8EIAAAAAAKoCAAAAAAAA +OzMAABIAAQDA8kIAAAAAAJcAAAAAAAAATzMAABIAAQBg80IAAAAAAF4AAAAAAAAAYjMAABIAAQDA +80IAAAAAAOUBAAAAAAAAdjMAABIAAQDA9UIAAAAAABMCAAAAAAAAhDMAABIAAQDg90IAAAAAAMoA +AAAAAAAAmzMAABIAAQDA+EIAAAAAALADAAAAAAAAtzMAABIAAQCA/EIAAAAAANAAAAAAAAAA0DMA +ABIAAQBg/UIAAAAAAEYDAAAAAAAA4zMAABIAAQDAAEMAAAAAAJAAAAAAAAAA/DMAABIAAQBgAUMA +AAAAAIoAAAAAAAAAFjQAABIAAQAAAkMAAAAAAHUAAAAAAAAAMDQAABIAAQCAAkMAAAAAAGoCAAAA +AAAATTQAABIAAQAABUMAAAAAANIBAAAAAAAAazQAABIAAQDgBkMAAAAAAJMAAAAAAAAAezQAABIA +AQCAB0MAAAAAAPMAAAAAAAAAizQAABIAAQCACEMAAAAAAEoAAAAAAAAAnDQAABIAAQDgCEMAAAAA +AFMAAAAAAAAAqjQAABIAAQBACUMAAAAAAC8CAAAAAAAAuDQAABIAAQCAC0MAAAAAADoDAAAAAAAA +yTQAABIAAQDADkMAAAAAAKUAAAAAAAAA4DQAABIAAQCAD0MAAAAAAKYCAAAAAAAA7zQAABIAAQBA +EkMAAAAAADsAAAAAAAAABDUAABIAAQCAEkMAAAAAAI0BAAAAAAAAEjUAABIAAQAgFEMAAAAAAHwA +AAAAAAAAJDUAABIAAQCgFEMAAAAAAF8BAAAAAAAAOTUAABIAAQAAFkMAAAAAAA0BAAAAAAAARzUA +ABIAAQAgF0MAAAAAANIAAAAAAAAAWTUAABIAAQAAGEMAAAAAAC4BAAAAAAAAZjUAABIAAQBAGUMA +AAAAAA8BAAAAAAAAdDUAABIAAQBgGkMAAAAAANAAAAAAAAAAkDUAABIAAQBAG0MAAAAAAGYBAAAA +AAAAoTUAABIAAQDAHEMAAAAAABgAAAAAAAAAvDUAABIAAQDgHEMAAAAAAP4AAAAAAAAA0zUAABIA +AQDgHUMAAAAAAOoAAAAAAAAA4TUAABIAAQDgHkMAAAAAAAwAAAAAAAAA8zUAABIAAQAAH0MAAAAA +AKcCAAAAAAAAAjYAABIAAQDAIUMAAAAAAM8CAAAAAAAAEzYAABIAAQCgJEMAAAAAAHUAAAAAAAAA +ITYAABIAAQAgJUMAAAAAAEoCAAAAAAAANTYAABIAAQCAJ0MAAAAAAK8AAAAAAAAASjYAABIAAQBA +KEMAAAAAAOUAAAAAAAAAWjYAABIAAQBAKUMAAAAAAGUBAAAAAAAAajYAABIAAQDAKkMAAAAAAI0J +AAAAAAAAfzYAABIAAQBgNEMAAAAAAN0AAAAAAAAAkDYAABIAAQBANUMAAAAAAGUDAAAAAAAAojYA +ABIAAQDAOEMAAAAAADIBAAAAAAAAuDYAABIAAQAAOkMAAAAAAJoAAAAAAAAAzzYAABIAAQCgOkMA +AAAAAH0BAAAAAAAA5jYAABIAAQAgPEMAAAAAAFUAAAAAAAAA/DYAABIAAQCAPEMAAAAAAH4AAAAA +AAAAEjcAABIAAQAAPUMAAAAAAA8DAAAAAAAAJjcAABIAAQAgQEMAAAAAAOoDAAAAAAAANzcAABIA +AQAgREMAAAAAAM0BAAAAAAAASzcAABIAAQAARkMAAAAAAEcAAAAAAAAAYDcAABIAAQBgRkMAAAAA +AG8BAAAAAAAAbzcAABIAAQDgR0MAAAAAAAUCAAAAAAAAgzcAABIAAQAASkMAAAAAAEwAAAAAAAAA +lTcAABIAAQBgSkMAAAAAAMUAAAAAAAAArjcAABIAAQBAS0MAAAAAAHAAAAAAAAAAwjcAABIAAQDA +S0MAAAAAAIUCAAAAAAAA1jcAABIAAQBgTkMAAAAAAMgAAAAAAAAA6DcAABIAAQBAT0MAAAAAAFYA +AAAAAAAA+DcAABIAAQCgT0MAAAAAAH4CAAAAAAAACDgAABIAAQAgUkMAAAAAAGYAAAAAAAAAFTgA +ABIAAQCgUkMAAAAAABQCAAAAAAAALDgAABIAAQDAVEMAAAAAAMoAAAAAAAAASTgAABIAAQCgVUMA +AAAAAGcAAAAAAAAAZTgAABIAAQAgVkMAAAAAAMoAAAAAAAAAgTgAABIAAQAAV0MAAAAAAJ4BAAAA +AAAAmzgAABIAAQCgWEMAAAAAAPcAAAAAAAAAuzgAABIAAQCgWUMAAAAAAPcAAAAAAAAA2zgAABIA +AQCgWkMAAAAAAGIAAAAAAAAA/TgAABIAAQAgW0MAAAAAAO0AAAAAAAAAFTkAABIAAQAgXEMAAAAA +AKUAAAAAAAAAMzkAABIAAQDgXEMAAAAAAIUAAAAAAAAAVjkAABIAAQCAXUMAAAAAAEIAAAAAAAAA +fzkAABIAAQDgXUMAAAAAAJkAAAAAAAAAnTkAABIAAQCAXkMAAAAAAHkBAAAAAAAAsjkAABIAAQAA +YEMAAAAAAOcAAAAAAAAAvzkAABIAAQAAYUMAAAAAAEcAAAAAAAAA0jkAABIAAQBgYUMAAAAAAIUA +AAAAAAAA4jkAABIAAQAAYkMAAAAAAGkAAAAAAAAA+DkAABIAAQCAYkMAAAAAAMUDAAAAAAAACToA +ABIAAQBgZkMAAAAAAL4CAAAAAAAAHzoAABIAAQAgaUMAAAAAAPoBAAAAAAAALToAABIAAQAga0MA +AAAAAHwBAAAAAAAAOzoAABIAAQCgbEMAAAAAAEcAAAAAAAAATzoAABIAAQAAbUMAAAAAAFUBAAAA +AAAAXzoAABIAAQBgbkMAAAAAAIUAAAAAAAAAdjoAABIAAQAAb0MAAAAAAC0AAAAAAAAAkDoAABIA +AQBAb0MAAAAAACgAAAAAAAAAoDoAABIAAQCAb0MAAAAAACgAAAAAAAAAtjoAABIAAQDAb0MAAAAA +ACgAAAAAAAAA0DoAABIAAQAAcEMAAAAAACgAAAAAAAAA3DoAABIAAQBAcEMAAAAAACgAAAAAAAAA +/zoAABIAAQCAcEMAAAAAACgAAAAAAAAADTsAABIAAQDAcEMAAAAAAKUDAAAAAAAAHTsAABIAAQCA +dEMAAAAAAHwAAAAAAAAAMjsAABIAAQAAdUMAAAAAAFsAAAAAAAAASTsAABIAAQBgdUMAAAAAAMUB +AAAAAAAAWzsAABIAAQBAd0MAAAAAAEUDAAAAAAAAcDsAABIAAQCgekMAAAAAAMoAAAAAAAAAizsA +ABIAAQCAe0MAAAAAADkIAAAAAAAAnjsAABIAAQDAg0MAAAAAAFUAAAAAAAAArzsAABIAAQAghEMA +AAAAAB4BAAAAAAAAvTsAABIAAQBAhUMAAAAAAEoBAAAAAAAAzjsAABIAAQCghkMAAAAAAF8AAAAA +AAAA5DsAABIAAQAAh0MAAAAAANcCAAAAAAAA9jsAABIAAQDgiUMAAAAAAPoAAAAAAAAADjwAABIA +AQDgikMAAAAAAHkEAAAAAAAAHTwAABIAAQBgj0MAAAAAADsCAAAAAAAALDwAABIAAQCgkUMAAAAA +AIUAAAAAAAAAPzwAABIAAQBAkkMAAAAAALoAAAAAAAAAUjwAABIAAQAAk0MAAAAAAOUHAAAAAAAA +ZTwAABIAAQAAm0MAAAAAAC8BAAAAAAAAfTwAABIAAQBAnEMAAAAAAFIAAAAAAAAAkjwAABIAAQCg +nEMAAAAAAE0AAAAAAAAAnzwAABIAAQAAnUMAAAAAADgBAAAAAAAAszwAABIAAQBAnkMAAAAAAOUA +AAAAAAAAzDwAABIAAQBAn0MAAAAAABABAAAAAAAA3TwAABIAAQBgoEMAAAAAAAYBAAAAAAAA7jwA +ABIAAQCAoUMAAAAAAOoAAAAAAAAA/jwAABIAAQCAokMAAAAAAOUBAAAAAAAAEj0AABIAAQCApEMA +AAAAAC8BAAAAAAAAJz0AABIAAQDApUMAAAAAAHQAAAAAAAAANz0AABIAAQBApkMAAAAAAEYBAAAA +AAAAST0AABIAAQCgp0MAAAAAAI8BAAAAAAAAWj0AABIAAQBAqUMAAAAAAOUAAAAAAAAAbD0AABIA +AQBAqkMAAAAAADoFAAAAAAAAez0AABIAAQCAr0MAAAAAAJgAAAAAAAAAnT0AABIAAQAgsEMAAAAA +ANMAAAAAAAAAwz0AABIAAQAAsUMAAAAAAIUFAAAAAAAA3D0AABIAAQCgtkMAAAAAAGYAAAAAAAAA ++z0AABIAAQAgt0MAAAAAAGIAAAAAAAAACD4AABIAAQCgt0MAAAAAAB0BAAAAAAAAFz4AABIAAQDA +uEMAAAAAACoBAAAAAAAAKz4AABIAAQAAukMAAAAAAPQBAAAAAAAAQD4AABIAAQAAvEMAAAAAAOoD +AAAAAAAATj4AABIAAQAAwEMAAAAAAAUDAAAAAAAAZT4AABIAAQAgw0MAAAAAABgBAAAAAAAAej4A +ABIAAQBAxEMAAAAAACkAAAAAAAAAlD4AABIAAQCAxEMAAAAAAIUAAAAAAAAArT4AABIAAQAgxUMA +AAAAAJ0AAAAAAAAAzD4AABIAAQDAxUMAAAAAANAAAAAAAAAA5z4AABIAAQCgxkMAAAAAAHkAAAAA +AAAA/T4AABIAAQAgx0MAAAAAAC0DAAAAAAAAET8AABIAAQBgykMAAAAAAA0CAAAAAAAAJT8AABIA +AQCAzEMAAAAAAI8DAAAAAAAAPz8AABIAAQAg0EMAAAAAADgDAAAAAAAAWz8AABIAAQBg00MAAAAA +ACcBAAAAAAAAej8AABIAAQCg1EMAAAAAACcBAAAAAAAAmj8AABIAAQDg1UMAAAAAAO8FAAAAAAAA +qz8AABIAAQDg20MAAAAAAO4AAAAAAAAAyz8AABIAAQDg3EMAAAAAALoBAAAAAAAA2z8AABIAAQCg +3kMAAAAAABwBAAAAAAAA8D8AABIAAQDA30MAAAAAAAQAAAAAAAAAAkAAABIAAQDg30MAAAAAAPUB +AAAAAAAAFUAAABIAAQDg4UMAAAAAAKcCAAAAAAAAL0AAABIAAQCg5EMAAAAAACkIAAAAAAAAQkAA +ABIAAQDg7EMAAAAAAAUEAAAAAAAAU0AAABIAAQAA8UMAAAAAABIBAAAAAAAAaUAAABIAAQAg8kMA +AAAAAB0BAAAAAAAAgEAAABIAAQBA80MAAAAAACIAAAAAAAAAjkAAABIAAQCA80MAAAAAAHUAAAAA +AAAApEAAABIAAQAA9EMAAAAAAHUAAAAAAAAAukAAABIAAQCA9EMAAAAAAHUAAAAAAAAA00AAABIA +AQAA9UMAAAAAALQAAAAAAAAA5UAAABIAAQDA9UMAAAAAAMEBAAAAAAAA9kAAABIAAQCg90MAAAAA +AJQAAAAAAAAAB0EAABIAAQBA+EMAAAAAAJIAAAAAAAAAGkEAABIAAQDg+EMAAAAAACwAAAAAAAAA +L0EAABIAAQAg+UMAAAAAAE4BAAAAAAAASEEAABIAAQCA+kMAAAAAABABAAAAAAAAYEEAABIAAQCg ++0MAAAAAAJQAAAAAAAAAd0EAABIAAQBA/EMAAAAAAGIAAAAAAAAAi0EAABIAAQDA/EMAAAAAAHwB +AAAAAAAAm0EAABIAAQBA/kMAAAAAAF4BAAAAAAAAsUEAABIAAQCg/0MAAAAAANkAAAAAAAAAw0EA +ABIAAQCAAEQAAAAAAMUGAAAAAAAA1UEAABIAAQBgB0QAAAAAAFwAAAAAAAAA50EAABIAAQDAB0QA +AAAAAJkBAAAAAAAA/kEAABIAAQBgCUQAAAAAAJQBAAAAAAAAFEIAABIAAQAAC0QAAAAAABQBAAAA +AAAALUIAABIAAQAgDEQAAAAAACgBAAAAAAAAR0IAABIAAQBgDUQAAAAAAOgAAAAAAAAAYEIAABIA +AQBgDkQAAAAAANECAAAAAAAAc0IAABIAAQBAEUQAAAAAAAYDAAAAAAAAhUIAABIAAQBgFEQAAAAA +AA0CAAAAAAAAnEIAABIAAQCAFkQAAAAAAC4DAAAAAAAAsEIAABIAAQDAGUQAAAAAADwBAAAAAAAA +xUIAABIAAQAAG0QAAAAAAG8BAAAAAAAA3kIAABIAAQCAHEQAAAAAAGcDAAAAAAAA8EIAABIAAQAA +IEQAAAAAALILAAAAAAAAAUMAABIAAQDAK0QAAAAAAJ0BAAAAAAAAFUMAABIAAQBgLUQAAAAAAI8B +AAAAAAAALEMAABIAAQAAL0QAAAAAANIHAAAAAAAAQEMAABIAAQDgNkQAAAAAAE0AAAAAAAAAT0MA +ABIAAQBAN0QAAAAAAIkCAAAAAAAAZUMAABIAAQDgOUQAAAAAAJoAAAAAAAAAe0MAABIAAQCAOkQA +AAAAAAUBAAAAAAAAkUMAABIAAQCgO0QAAAAAAOkAAAAAAAAAq0MAABIAAQCgPEQAAAAAAJQAAAAA +AAAAwEMAABIAAQBAPUQAAAAAAIUAAAAAAAAA0kMAABIAAQDgPUQAAAAAANEAAAAAAAAA30MAABIA +AQDAPkQAAAAAALgAAAAAAAAA8EMAABIAAQCAP0QAAAAAAEkAAAAAAAAAB0QAABIAAQDgP0QAAAAA +ADcAAAAAAAAAGEQAABIAAQAgQEQAAAAAADACAAAAAAAALEQAABIAAQBgQkQAAAAAAKUGAAAAAAAA +RkQAABIAAQAgSUQAAAAAAKYBAAAAAAAAV0QAABIAAQDgSkQAAAAAABwFAAAAAAAAZ0QAABIAAQAA +UEQAAAAAAKUAAAAAAAAAeEQAABIAAQDAUEQAAAAAAJcAAAAAAAAAjEQAABIAAQBgUUQAAAAAAKUA +AAAAAAAAqEQAABIAAQAgUkQAAAAAAOUAAAAAAAAAuUQAABIAAQAgU0QAAAAAABYBAAAAAAAAy0QA +ABIAAQBAVEQAAAAAAFcAAAAAAAAA3EQAABIAAQCgVEQAAAAAAEUBAAAAAAAA8EQAABIAAQAAVkQA +AAAAAO4AAAAAAAAAB0UAABIAAQAAV0QAAAAAAJQAAAAAAAAAG0UAABIAAQCgV0QAAAAAAI4AAAAA +AAAAMEUAABIAAQBAWEQAAAAAAIYBAAAAAAAAPUUAABIAAQDgWUQAAAAAAIgBAAAAAAAAUEUAABIA +AQCAW0QAAAAAAJECAAAAAAAAYUUAABIAAQAgXkQAAAAAAO0BAAAAAAAAdEUAABIAAQAgYEQAAAAA +AIUBAAAAAAAAiEUAABIAAQDAYUQAAAAAAPgEAAAAAAAAmUUAABIAAQDAZkQAAAAAAEUCAAAAAAAA +rEUAABIAAQAgaUQAAAAAABQDAAAAAAAAwUUAABIAAQBAbEQAAAAAAMwAAAAAAAAA20UAABIAAQAg +bUQAAAAAAKsCAAAAAAAA7EUAABIAAQDgb0QAAAAAAJkBAAAAAAAAAEYAABIAAQCAcUQAAAAAAIUE +AAAAAAAAG0YAABIAAQAgdkQAAAAAAMUAAAAAAAAAMkYAABIAAQAAd0QAAAAAAF4BAAAAAAAARkYA +ABIAAQBgeEQAAAAAABACAAAAAAAAXEYAABIAAQCAekQAAAAAAC0AAAAAAAAAbUYAABIAAQDAekQA +AAAAAL8AAAAAAAAAgUYAABIAAQCAe0QAAAAAAK8AAAAAAAAAl0YAABIAAQBAfEQAAAAAAPMAAAAA +AAAAqkYAABIAAQBAfUQAAAAAAGgEAAAAAAAAw0YAABIAAQDAgUQAAAAAAAcBAAAAAAAA2EYAABIA +AQDggkQAAAAAAHwAAAAAAAAA80YAABIAAQBgg0QAAAAAAGsAAAAAAAAADkcAABIAAQDgg0QAAAAA +AGcCAAAAAAAAIUcAABIAAQBghkQAAAAAAMUBAAAAAAAAQEcAABIAAQBAiEQAAAAAAJAAAAAAAAAA +YEcAABIAAQDgiEQAAAAAAFEAAAAAAAAAhEcAABIAAQBAiUQAAAAAAAYBAAAAAAAAoEcAABIAAQBg +ikQAAAAAAFwAAAAAAAAAt0cAABIAAQDAikQAAAAAALcAAAAAAAAAzUcAABIAAQCAi0QAAAAAAFwA +AAAAAAAA50cAABIAAQDgi0QAAAAAAIUAAAAAAAAAAEgAABIAAQCAjEQAAAAAAKoAAAAAAAAAGUgA +ABIAAQBAjUQAAAAAAM0AAAAAAAAAL0gAABIAAQAgjkQAAAAAAF4BAAAAAAAAREgAABIAAQCAj0QA +AAAAAE0AAAAAAAAAWUgAABIAAQDgj0QAAAAAAHYAAAAAAAAAbUgAABIAAQBgkEQAAAAAAMUAAAAA +AAAAg0gAABIAAQBAkUQAAAAAADsAAAAAAAAAmkgAABIAAQCAkUQAAAAAAMUAAAAAAAAAsUgAABIA +AQBgkkQAAAAAALcAAAAAAAAAyUgAABIAAQAgk0QAAAAAAJkAAAAAAAAA30gAABIAAQDAk0QAAAAA +AA4CAAAAAAAA90gAABIAAQDglUQAAAAAAEwaAAAAAAAADEkAABIAAQBAsEQAAAAAAA8DAAAAAAAA +HkkAABIAAQBgs0QAAAAAAH8AAAAAAAAANkkAABIAAQDgs0QAAAAAAKUBAAAAAAAASUkAABIAAQCg +tUQAAAAAAIYCAAAAAAAAZUkAABIAAQBAuEQAAAAAAJwAAAAAAAAAfEkAABIAAQDguEQAAAAAAIUB +AAAAAAAAlEkAABIAAQCAukQAAAAAAFQAAAAAAAAApkkAABIAAQDgukQAAAAAAI8AAAAAAAAAvEkA +ABIAAQCAu0QAAAAAAA4DAAAAAAAAz0kAABIAAQCgvkQAAAAAAKsBAAAAAAAA7kkAABIAAQBgwEQA +AAAAANECAAAAAAAAFUoAABIAAQBAw0QAAAAAAPUAAAAAAAAAJUoAABIAAQBAxEQAAAAAAHMAAAAA +AAAAO0oAABIAAQDAxEQAAAAAAJoAAAAAAAAATEoAABIAAQBgxUQAAAAAALYAAAAAAAAAXkoAABIA +AQAgxkQAAAAAAMgBAAAAAAAAc0oAABIAAQAAyEQAAAAAACgCAAAAAAAAi0oAABIAAQBAykQAAAAA +APkAAAAAAAAAo0oAABIAAQBAy0QAAAAAAAUBAAAAAAAAwUoAABIAAQBgzEQAAAAAANUBAAAAAAAA +2koAABIAAQBAzkQAAAAAAC4AAAAAAAAA+UoAABIAAQCAzkQAAAAAAM4AAAAAAAAAE0sAABIAAQBg +z0QAAAAAACUBAAAAAAAALUsAABIAAQCg0EQAAAAAAP0BAAAAAAAASksAABIAAQCg0kQAAAAAAG0A +AAAAAAAAZEsAABIAAQAg00QAAAAAANgAAAAAAAAAeksAABIAAQAA1EQAAAAAAI8AAAAAAAAAkksA +ABIAAQCg1EQAAAAAAHAAAAAAAAAArEsAABIAAQAg1UQAAAAAAKgAAAAAAAAAxUsAABIAAQDg1UQA +AAAAAIUCAAAAAAAA3EsAABIAAQCA2EQAAAAAAKcCAAAAAAAA80sAABIAAQBA20QAAAAAAOUCAAAA +AAAADEwAABIAAQBA3kQAAAAAALcAAAAAAAAAHkwAABIAAQAA30QAAAAAAPUAAAAAAAAAL0wAABIA +AQAA4EQAAAAAAE8BAAAAAAAAREwAABIAAQBg4UQAAAAAANQFAAAAAAAAWkwAABIAAQBA50QAAAAA +AKYMAAAAAAAAbUwAABIAAQAA9EQAAAAAAIgEAAAAAAAAjUwAABIAAQCg+EQAAAAAACcBAAAAAAAA +pUwAABIAAQDg+UQAAAAAAIUDAAAAAAAAvkwAABIAAQCA/UQAAAAAAKYBAAAAAAAA3UwAABIAAQBA +/0QAAAAAALoAAAAAAAAA7kwAABIAAQAAAEUAAAAAAFoAAAAAAAAAAU0AABIAAQBgAEUAAAAAALUA +AAAAAAAAHU0AABIAAQAgAUUAAAAAAHUBAAAAAAAAOk0AABIAAQCgAkUAAAAAACgAAAAAAAAAUE0A +ABIAAQDgAkUAAAAAAHIAAAAAAAAAa00AABIAAQBgA0UAAAAAAGcAAAAAAAAAhE0AABIAAQDgA0UA +AAAAABABAAAAAAAAnU0AABIAAQAABUUAAAAAANkAAAAAAAAAvU0AABIAAQDgBUUAAAAAAC8AAAAA +AAAA3U0AABIAAQAgBkUAAAAAADgAAAAAAAAA/00AABIAAQBgBkUAAAAAADYAAAAAAAAAH04AABIA +AQCgBkUAAAAAAHoAAAAAAAAAPE4AABIAAQAgB0UAAAAAABMAAAAAAAAAW04AABIAAQBAB0UAAAAA +ACgAAAAAAAAAdE4AABIAAQCAB0UAAAAAAFAAAAAAAAAAi04AABIAAQDgB0UAAAAAAJ8AAAAAAAAA +rk4AABIAAQCACEUAAAAAADYAAAAAAAAAx04AABIAAQDACEUAAAAAAC0AAAAAAAAA4k4AABIAAQAA +CUUAAAAAAE0AAAAAAAAA/04AABIAAQBgCUUAAAAAAEcAAAAAAAAAGE8AABIAAQDACUUAAAAAADoA +AAAAAAAAK08AABIAAQAACkUAAAAAAG8BAAAAAAAARE8AABIAAQCAC0UAAAAAAOwBAAAAAAAAUU8A +ABIAAQCADUUAAAAAAMcAAAAAAAAAXE8AABIAAQBgDkUAAAAAACcAAAAAAAAAcU8AABIAAQCgDkUA +AAAAABECAAAAAAAAhU8AABIAAQDAEEUAAAAAAB8BAAAAAAAAoE8AABIAAQDgEUUAAAAAACAAAAAA +AAAAs08AABIAAQAAEkUAAAAAAJEAAAAAAAAAxE8AABIAAQBgHkUAAAAAAA4AAAAAAAAAz08AABIA +AQCAHkUAAAAAAEkBAAAAAAAA408AABIAAQDgH0UAAAAAAAEAAAAAAAAA+E8AABIAAQAAIEUAAAAA +AAYAAAAAAAAADFAAABIAAQAgIEUAAAAAABEAAAAAAAAAHlAAABIAAQBAIEUAAAAAAEoAAAAAAAAA +LFAAABIAAQCgIEUAAAAAAAEAAAAAAAAATFAAABIAAQDAIEUAAAAAAIYAAAAAAAAAZVAAABIAAQBg +IUUAAAAAAJEAAAAAAAAAfFAAABIAAQAAIkUAAAAAAAoAAAAAAAAAmlAAABIAAQAgIkUAAAAAAAwA +AAAAAAAAsVAAABIAAQBAIkUAAAAAAAEAAAAAAAAA0VAAABIAAQBgIkUAAAAAAB0AAAAAAAAA51AA +ABIAAQCAIkUAAAAAALoAAAAAAAAA/1AAABIAAQBAI0UAAAAAAA8AAAAAAAAAEVEAABIAAQBgI0UA +AAAAAAQAAAAAAAAAJFEAABIAAQCAI0UAAAAAAB8AAAAAAAAAPFEAABIAAQCgI0UAAAAAACAAAAAA +AAAAUlEAABIAAQDAI0UAAAAAABMAAAAAAAAAYlEAABIAAQDgI0UAAAAAADoAAAAAAAAAdFEAABIA +AQAgJEUAAAAAADsAAAAAAAAAhlEAABIAAQBgJEUAAAAAAB0AAAAAAAAAnFEAABIAAQCAJEUAAAAA +AAcAAAAAAAAAsFEAABIAAQCgJEUAAAAAABIAAAAAAAAAwlEAABIAAQDAJEUAAAAAAOMAAAAAAAAA +2VEAABIAAQDAJUUAAAAAAAoAAAAAAAAA8lEAABIAAQDgJUUAAAAAAAoAAAAAAAAAC1IAABIAAQAA +JkUAAAAAAAoAAAAAAAAAJFIAABIAAQAgJkUAAAAAAAoAAAAAAAAAPVIAABIAAQBAJkUAAAAAAAoA +AAAAAAAAVlIAABIAAQBgJkUAAAAAAAoAAAAAAAAAb1IAABIAAQCAJkUAAAAAAIkCAAAAAAAAg1IA +ABIAAQAgKUUAAAAAADMAAAAAAAAAolIAABIAAQBgKUUAAAAAAAgAAAAAAAAAtVIAABIAAQCAKUUA +AAAAAAgAAAAAAAAAyVIAABIAAQCgKUUAAAAAAAsAAAAAAAAA4FIAABIAAQDAKUUAAAAAAAsAAAAA +AAAA+FIAABIAAQDgKUUAAAAAAAsAAAAAAAAAD1MAABIAAQAAKkUAAAAAAAsAAAAAAAAAJ1MAABIA +AQAgKkUAAAAAAAgAAAAAAAAAO1MAABIAAQBAKkUAAAAAAAgAAAAAAAAAUFMAABIAAQBgKkUAAAAA +AAgAAAAAAAAAaFMAABIAAQCAKkUAAAAAAAgAAAAAAAAAgVMAABIAAQCgKkUAAAAAAHEBAAAAAAAA +klMAABIAAQAgLEUAAAAAAIEDAAAAAAAAo1MAABIAAQDAL0UAAAAAAKgCAAAAAAAAwFMAABIAAQCA +MkUAAAAAAL8GAAAAAAAA0FMAABIAAQBAOUUAAAAAAK8BAAAAAAAA5VMAABIAAQAAO0UAAAAAAAUA +AAAAAAAA9lMAABIAAQAgO0UAAAAAAAwAAAAAAAAACFQAABIAAQBAO0UAAAAAABsAAAAAAAAAIFQA +ABIAAQBgO0UAAAAAACwAAAAAAAAAMlQAABIAAQCgO0UAAAAAAB0AAAAAAAAAR1QAABIAAQDAO0UA +AAAAABoAAAAAAAAAW1QAABIAAQDgO0UAAAAAABkAAAAAAAAAbVQAABIAAQAAPEUAAAAAABEAAAAA +AAAAf1QAABIAAQAgPEUAAAAAABUAAAAAAAAAklQAABIAAQBAPEUAAAAAAEcAAAAAAAAAplQAABIA +AQCgPEUAAAAAAAwAAAAAAAAAulQAABIAAQDAPEUAAAAAACIAAAAAAAAAzVQAABIAAQAAPUUAAAAA +ABUAAAAAAAAA5FQAABIAAQAgPUUAAAAAAA0AAAAAAAAA+FQAABIAAQBAPUUAAAAAABcAAAAAAAAA +DFUAABIAAQBgPUUAAAAAABsAAAAAAAAAIVUAABIAAQCAPUUAAAAAALoAAAAAAAAAOFUAABIAAQBA +PkUAAAAAAC4AAAAAAAAAU1UAABIAAQCAPkUAAAAAACAAAAAAAAAAbVUAABIAAQCgPkUAAAAAAD4A +AAAAAAAAi1UAABIAAQDgPkUAAAAAACIAAAAAAAAAn1UAABIAAQAgP0UAAAAAAGMAAAAAAAAAsFUA +ABIAAQCgP0UAAAAAALsAAAAAAAAAxFUAABIAAQBgQEUAAAAAAAsAAAAAAAAA1lUAABIAAQCAQEUA +AAAAAFAAAAAAAAAA61UAABIAAQDgQEUAAAAAAFEAAAAAAAAABFYAABIAAQBAQUUAAAAAACcAAAAA +AAAAG1YAABIAAQCAQUUAAAAAADoAAAAAAAAANlYAABIAAQDAQUUAAAAAABwAAAAAAAAAS1YAABIA +AQDgQUUAAAAAACgAAAAAAAAAXlYAABIAAQAgQkUAAAAAAJ0AAAAAAAAAcVYAABIAAQDAQkUAAAAA +ACcAAAAAAAAAilYAABIAAQAAQ0UAAAAAAEIAAAAAAAAAnlYAABIAAQBgQ0UAAAAAAAgAAAAAAAAA +s1YAABIAAQCAQ0UAAAAAABsAAAAAAAAA0lYAABIAAQCgQ0UAAAAAABAAAAAAAAAA61YAABIAAQDA +Q0UAAAAAABAAAAAAAAAABVcAABIAAQDgQ0UAAAAAAB0AAAAAAAAAG1cAABIAAQAAREUAAAAAACUA +AAAAAAAAMlcAABIAAQBAREUAAAAAABoAAAAAAAAAS1cAABIAAQBgREUAAAAAADUAAAAAAAAAZFcA +ABIAAQCgREUAAAAAANoAAAAAAAAAclcAABIAAQCARUUAAAAAAF0AAAAAAAAAklcAABIAAQDgRUUA +AAAAADkAAAAAAAAArlcAABIAAQAgRkUAAAAAAC8AAAAAAAAAyVcAABIAAQBgRkUAAAAAADQAAAAA +AAAA4VcAABIAAQCgRkUAAAAAABIAAAAAAAAA9VcAABIAAQDARkUAAAAAABIAAAAAAAAAEFgAABIA +AQDgRkUAAAAAAC8AAAAAAAAAJlgAABIAAQAgR0UAAAAAAC8AAAAAAAAAPVgAABIAAQBgR0UAAAAA +ABIAAAAAAAAAWVgAABIAAQCAR0UAAAAAABIAAAAAAAAAelgAABIAAQCgR0UAAAAAABIAAAAAAAAA +kVgAABIAAQDAR0UAAAAAABMAAAAAAAAAoFgAABIAAQDgR0UAAAAAABIAAAAAAAAAtVgAABIAAQAA +SEUAAAAAABIAAAAAAAAAylgAABIAAQAgSEUAAAAAADMAAAAAAAAA31gAABIAAQBgSEUAAAAAABIA +AAAAAAAA+VgAABIAAQCASEUAAAAAADMAAAAAAAAAC1kAABIAAQDASEUAAAAAABIAAAAAAAAAHlkA +ABIAAQDgSEUAAAAAADgAAAAAAAAANlkAABIAAQAgSUUAAAAAABIAAAAAAAAATFkAABIAAQBASUUA +AAAAABIAAAAAAAAAZFkAABIAAQBgSUUAAAAAABIAAAAAAAAAgFkAABIAAQCASUUAAAAAADcAAAAA +AAAAk1kAABIAAQDASUUAAAAAAHcAAAAAAAAAsFkAABIAAQBASkUAAAAAAFEAAAAAAAAAxlkAABIA +AQCgSkUAAAAAAJoAAAAAAAAA4lkAABIAAQBAS0UAAAAAABUAAAAAAAAA/VkAABIAAQBgS0UAAAAA +AHoAAAAAAAAAIVoAABIAAQDgS0UAAAAAAJQAAAAAAAAAOVoAABIAAQCATEUAAAAAAHEAAAAAAAAA +UVoAABIAAQAATUUAAAAAAEMAAAAAAAAAbloAABIAAQBgTUUAAAAAAHAAAAAAAAAAi1oAABIAAQDg +TUUAAAAAACcAAAAAAAAAp1oAABIAAQAgTkUAAAAAAEMAAAAAAAAAwFoAABIAAQCATkUAAAAAAHEA +AAAAAAAA11oAABIAAQAAT0UAAAAAAEMAAAAAAAAA71oAABIAAQBgT0UAAAAAABUAAAAAAAAAIlsA +ABIAAQCAT0UAAAAAAHEAAAAAAAAAOVsAABIAAQAAUEUAAAAAAHEAAAAAAAAAUFsAABIAAQCAUEUA +AAAAAEMAAAAAAAAAaFsAABIAAQDgUEUAAAAAALMAAAAAAAAAg1sAABIAAQCgUUUAAAAAAG8AAAAA +AAAAoFsAABIAAQAgUkUAAAAAAGQAAAAAAAAAxFsAABIAAQCgUkUAAAAAAHcAAAAAAAAA6FsAABIA +AQAgU0UAAAAAAEwAAAAAAAAABFwAABIAAQCAU0UAAAAAAAEAAAAAAAAADlwAABEACQDAgEsAAAAA +ABgAAAAAAAAAHVwAABEACQCggksAAAAAAGAAAAAAAAAAL1wAABEADADynk4AAAAAAAEAAAAAAAAA +QlwAABEADABAo04AAAAAAIAAAAAAAAAAVlwAABEADABAoU4AAAAAACAAAAAAAAAAZlwAABEADADs +nk4AAAAAAAEAAAAAAAAAdFwAABEADADknk4AAAAAAAEAAAAAAAAAiVwAABEACgCokUsAAAAAAAgA +AAAAAAAAm1wAABEADABIoE4AAAAAAAgAAAAAAAAArFwAABEADAD1nk4AAAAAAAEAAAAAAAAAwVwA +ABEADAD2nk4AAAAAAAEAAAAAAAAA1VwAABEADAD0nk4AAAAAAAEAAAAAAAAA51wAABEADADjnk4A +AAAAAAEAAAAAAAAA+1wAABEADADink4AAAAAAAEAAAAAAAAAE10AABEADADxnk4AAAAAAAEAAAAA +AAAAKV0AABEACwCg1UsAAAAAAHAfAAAAAAAAOV0AABEACwDgs0sAAAAAAAgAAAAAAAAATV0AABEA +CwDos0sAAAAAAAgAAAAAAAAAY10AABEACgCAlUsAAAAAAJAAAAAAAAAAe10AABEACgAAlUsAAAAA +AIAAAAAAAAAAll0AABEACgBgkksAAAAAABAAAAAAAAAAq10AABEACQBAhUsAAAAAAAgBAAAAAAAA +wV0AABEADAAIoE4AAAAAAAgAAAAAAAAAzV0AABEADAAYoE4AAAAAAAgAAAAAAAAA3l0AABEACgCw +kUsAAAAAAAgAAAAAAAAA8F0AABEACgCAo0sAAAAAABAQAAAAAAAABl4AABEACgAAk0sAAAAAABAA +AAAAAAAAGl4AABEACgAQk0sAAAAAABAAAAAAAAAALl4AABEACgAgk0sAAAAAABAAAAAAAAAAQl4A +ABEACgDwkksAAAAAABAAAAAAAAAAVl4AABEACgDgkksAAAAAABAAAAAAAAAAaV4AABEACwCAtEsA +AAAAAAgAAAAAAAAAfF4AABEACwCItEsAAAAAAAgAAAAAAAAAj14AABEACwCQtEsAAAAAAAgAAAAA +AAAAol4AABEACwBotEsAAAAAAAgAAAAAAAAAtV4AABEACwBgtEsAAAAAAAgAAAAAAAAAx14AABEA +CQCgiUsAAAAAAAAIAAAAAAAA3V4AABEACgBwk0sAAAAAABgAAAAAAAAA714AABEADACIoE4AAAAA +AAgAAAAAAAAABF8AABEADACAoE4AAAAAAAgAAAAAAAAAHV8AABEADAB4oE4AAAAAAAgAAAAAAAAA +N18AABEADADwoE4AAAAAAAgAAAAAAAAASF8AABEADAAQoU4AAAAAABgAAAAAAAAAXF8AABEADABw +oE4AAAAAAAgAAAAAAAAAdV8AABEADABAsk4AAAAAAAAEAAAAAAAAhV8AABEADABgpE4AAAAAAIgA +AAAAAAAAmF8AABEADADznk4AAAAAAAEAAAAAAAAArV8AABEACQAkgEsAAAAAAAQAAAAAAAAAwl8A +ABEADADwn04AAAAAAAgAAAAAAAAA0l8AABEACwAwtEsAAAAAAAgAAAAAAAAA318AABEADAD4n04A +AAAAAAgAAAAAAAAA7F8AABEADADon04AAAAAAAgAAAAAAAAA+V8AABEADABAok4AAAAAAEAAAAAA +AAAADGAAABEADADnnk4AAAAAAAEAAAAAAAAAHWAAABEADADonk4AAAAAAAEAAAAAAAAALmAAABEA +DAB4n04AAAAAAAgAAAAAAAAAPWAAABEACQA4gEsAAAAAAAUAAAAAAAAAUGAAABEADADmnk4AAAAA +AAEAAAAAAAAAZGAAABEACwA4tEsAAAAAAAgAAAAAAAAAe2AAABEADAAkn04AAAAAAAQAAAAAAAAA +i2AAABEADAAAoU4AAAAAABAAAAAAAAAAoGAAABEADAAcn04AAAAAAAQAAAAAAAAAuWAAABEADABA +qk4AAAAAAKABAAAAAAAAxmAAABEADAAgn04AAAAAAAQAAAAAAAAA4GAAABEACwBYtEsAAAAAAAgA +AAAAAAAA9GAAABEACQAggEsAAAAAAAEAAAAAAAAAB2EAABEADABApk4AAAAAAOAAAAAAAAAAHGEA +ABEACwCgtksAAAAAACgAAAAAAAAALWEAABEACwCAtksAAAAAACAAAAAAAAAAO2EAABEACwDANE0A +AAAAAAhqAQAAAAAASmEAABEACgCQk0sAAAAAABgAAAAAAAAAYmEAABEADACAoU4AAAAAACgAAAAA +AAAAd2EAABEACQBogEsAAAAAAAgAAAAAAAAAjWEAABEACQAAgUsAAAAAACgAAAAAAAAAn2EAABEA +CQCAgUsAAAAAACgAAAAAAAAAsmEAABEACQBAgUsAAAAAACgAAAAAAAAAyGEAABEADACgoE4AAAAA +AAgAAAAAAAAA2WEAABEADAAooE4AAAAAAAgAAAAAAAAA6mEAABEADACYn04AAAAAAAgAAAAAAAAA ++2EAABEADADooE4AAAAAAAgAAAAAAAAADGIAABEACwAQtEsAAAAAAAgAAAAAAAAAHWIAABEADACo +n04AAAAAAAgAAAAAAAAAL2IAABEADAAgoE4AAAAAAAgAAAAAAAAAPWIAABEADACgn04AAAAAAAgA +AAAAAAAAVmIAABEADABAoE4AAAAAAAgAAAAAAAAAb2IAABEADABwn04AAAAAAAgAAAAAAAAAhmIA +ABEACgCgkUsAAAAAAAEAAAAAAAAApWIAABEADADQoE4AAAAAAAgAAAAAAAAAt2IAABEACQCAgEsA +AAAAAAgAAAAAAAAAymIAABEACQBggEsAAAAAAAgAAAAAAAAA3WIAABEADACwoE4AAAAAAAgAAAAA +AAAA9mIAABEADABAtk4AAAAAAMA7AAAAAAAAB2MAABEADABgoE4AAAAAAAgAAAAAAAAAH2MAABEA +DAAwn04AAAAAAAQAAAAAAAAANWMAABEADAA0n04AAAAAAAQAAAAAAAAATGMAABEACgDAkksAAAAA +ABAAAAAAAAAAXGMAABEACwBQtEsAAAAAAAgAAAAAAAAAa2MAABEACQAogEsAAAAAAAQAAAAAAAAA +eGMAABEADABQoE4AAAAAAAgAAAAAAAAAj2MAABEADABYoE4AAAAAAAgAAAAAAAAApmMAABEADAA4 +n04AAAAAAAQAAAAAAAAAvWMAABEACgDQk0sAAAAAABgAAAAAAAAAzmMAABEADADhnk4AAAAAAAEA +AAAAAAAA5GMAABEACwDwtUsAAAAAABgAAAAAAAAA/mMAABEACgDwk0sAAAAAABgAAAAAAAAAFWQA +ABEACgAQlEsAAAAAABgAAAAAAAAAKWQAABEACQCQgEsAAAAAAAgAAAAAAAAAPGQAABEACgDQkksA +AAAAABAAAAAAAAAAT2QAABEACgBwkksAAAAAABAAAAAAAAAAY2QAABEACgCwkksAAAAAABAAAAAA +AAAAeWQAABEACgCAkksAAAAAABAAAAAAAAAAjGQAABEACgCQkksAAAAAABAAAAAAAAAAoGQAABEA +CwAotEsAAAAAAAgAAAAAAAAAsmQAABEADABIn04AAAAAAAQAAAAAAAAAzWQAABEADABAn04AAAAA +AAQAAAAAAAAA32QAABEADABooE4AAAAAAAgAAAAAAAAA72QAABEADADlnk4AAAAAAAEAAAAAAAAA +AWUAABEADADAn04AAAAAAAgAAAAAAAAAEmUAABEACQBIgEsAAAAAAAgAAAAAAAAALGUAABEAAgAQ +aUcAAAAAAAgAAAAAAAAASWUAABEADADgq04AAAAAAAACAAAAAAAAXmUAABEADACQoE4AAAAAAAgA +AAAAAAAAeGUAABEADADIn04AAAAAAAgAAAAAAAAAimUAABEADAA4oE4AAAAAAAgAAAAAAAAAn2UA +ABEACgCgkksAAAAAABAAAAAAAAAAr2UAABEACwCguUsAAAAAAAAEAAAAAAAAumUAABEACwAAuEsA +AAAAAIgBAAAAAAAAxWUAABEADAAwoE4AAAAAAAgAAAAAAAAA1WUAABEACwBAtEsAAAAAAAgAAAAA +AAAA7GUAABEADADvnk4AAAAAAAEAAAAAAAAAAGYAABEADACooE4AAAAAAAgAAAAAAAAAGGYAABEA +DAAQoE4AAAAAAAgAAAAAAAAALGYAABEACgAwkksAAAAAABAAAAAAAAAARmYAABEACgBAkksAAAAA +ABAAAAAAAAAAZWYAABEADACIn04AAAAAAAgAAAAAAAAAdmYAABEACwAQtUsAAAAAABgAAAAAAAAA +hGYAABEADACAn04AAAAAAAgAAAAAAAAAlGYAABEACwD4s0sAAAAAAAgAAAAAAAAApGYAABEADADg +n04AAAAAAAgAAAAAAAAAuWYAABEADAAUn04AAAAAAAQAAAAAAAAAymYAABEACQA0gEsAAAAAAAQA +AAAAAAAA3GYAABEACQAsgEsAAAAAAAQAAAAAAAAA62YAABEACgBQk0sAAAAAABgAAAAAAAAABGcA +ABEADADQn04AAAAAAAgAAAAAAAAAE2cAABEADAAMn04AAAAAAAQAAAAAAAAAJ2cAABEADAAQn04A +AAAAAAQAAAAAAAAAPWcAABEADAAAok4AAAAAADAAAAAAAAAATmcAABEADADAoU4AAAAAACgAAAAA +AAAAYmcAABEADADpnk4AAAAAAAEAAAAAAAAAeGcAABEADACYoE4AAAAAAAgAAAAAAAAAhWcAABEA +DAAgp04AAAAAAAABAAAAAAAAnGcAABEADABMn04AAAAAAAQAAAAAAAAAtmcAABEACQBQgEsAAAAA +AAgAAAAAAAAAzGcAABEADAC4oE4AAAAAAAgAAAAAAAAA3mcAABEACwBgtksAAAAAACAAAAAAAAAA +8WcAABEADABgoU4AAAAAACAAAAAAAAAAA2gAABEACwBwtUsAAAAAABgAAAAAAAAAEGgAABEACwBQ +tUsAAAAAABgAAAAAAAAAIWgAABEACQAwgEsAAAAAAAQAAAAAAAAAOWgAABEADABQn04AAAAAAAQA +AAAAAAAAT2gAABEADAAEn04AAAAAAAQAAAAAAAAAXGgAABEACwAItEsAAAAAAAgAAAAAAAAAaWgA +ABEADADIoE4AAAAAAAgAAAAAAAAAemgAABEADADAoE4AAAAAAAgAAAAAAAAAi2gAABEADADgok4A +AAAAAEwAAAAAAAAAmWgAABEACgAwk0sAAAAAABgAAAAAAAAAqWgAABEACgCAmEsAAAAAALABAAAA +AAAAw2gAABEACwAAtEsAAAAAAAgAAAAAAAAA0GgAABEADAAon04AAAAAAAQAAAAAAAAA42gAABEA +DAAsn04AAAAAAAQAAAAAAAAA8GgAABEACwCQtUsAAAAAABgAAAAAAAAAAGkAABEACwCgvUsAAAAA +APAXAAAAAAAADmkAABEADAA8n04AAAAAAAQAAAAAAAAAH2kAABEADACQn04AAAAAAAgAAAAAAAAA +MGkAABEACwAwtUsAAAAAABgAAAAAAAAAPWkAABEACwCwtUsAAAAAABgAAAAAAAAAT2kAABEACwAQ +tksAAAAAABgAAAAAAAAAYmkAABEADAAAoE4AAAAAAAgAAAAAAAAAfWkAABEADAAYn04AAAAAAAQA +AAAAAAAAmWkAABEADABEn04AAAAAAAQAAAAAAAAAtmkAABEADADqnk4AAAAAAAEAAAAAAAAAxmkA +ABEADADunk4AAAAAAAEAAAAAAAAA4GkAABEADADtnk4AAAAAAAEAAAAAAAAA8mkAABEADADrnk4A +AAAAAAEAAAAAAAAABGoAABEADAC4n04AAAAAAAgAAAAAAAAAF2oAABEADACwn04AAAAAAAgAAAAA +AAAAKmoAABEACwAg9UsAAAAAAMA+AAAAAAAAO2oAABEADADgrU4AAAAAAAgCAAAAAAAASmoAABEA +DAAgqU4AAAAAAAQBAAAAAAAAXmoAABEADADwnk4AAAAAAAEAAAAAAAAAcGoAABEADAAIn04AAAAA +AAQAAAAAAAAAgWoAABEACwBwtEsAAAAAAAgAAAAAAAAAlWoAABEACwB4tEsAAAAAAAgAAAAAAAAA +qWoAABEACgAgkksAAAAAABAAAAAAAAAAwWoAABEACQCIgEsAAAAAAAgAAAAAAAAA2moAABEADACA +ok4AAAAAAEgAAAAAAAAA5moAABEACgBgnUsAAAAAABgGAAAAAAAA92oAABEACQCgg0sAAAAAAIgA +AAAAAAAADWsAABEACQBAgksAAAAAAEQAAAAAAAAAKmsAABEACQBghksAAAAAABABAAAAAAAARGsA +ABEACQAAg0sAAAAAAIEAAAAAAAAAW2sAABEACQBAhEsAAAAAAPkAAAAAAAAAdGsAABEADAAgqE4A +AAAAAAABAAAAAAAAhmsAABEADAAAsE4AAAAAADgCAAAAAAAAmWsAABEACQB4gEsAAAAAAAgAAAAA +AAAArmsAABEACQBwgEsAAAAAAAgAAAAAAAAAxmsAABEACgAQkksAAAAAABAAAAAAAAAA3msAABEA +CwDws0sAAAAAAAgAAAAAAAAA9WsAABEACgCwk0sAAAAAABgAAAAAAAAAFmwAABEACgBQkksAAAAA +ABAAAAAAAAAAMGwAABEACwDQtUsAAAAAABgAAAAAAAAAR2wAABEACQCAh0sAAAAAABgCAAAAAAAA +X2wAABEACQBYgEsAAAAAAAgAAAAAAAAAd2wAABEACwBItEsAAAAAAAgAAAAAAAAAjGwAABEADADY +n04AAAAAAAgAAAAAAAAAnWwAABEACwDgM0wAAAAAANgAAQAAAAAAq2wAABEACgAglksAAAAAAKAA +AAAAAAAAwmwAABEACwAgtEsAAAAAAAgAAAAAAAAA12wAABEACwAYtEsAAAAAAAgAAAAAAAAA7WwA +ABEACwBAtksAAAAAACAAAAAAAAAAAW0AABEACgAwlEsAAAAAABgAAAAAAAAAGm0AABEACgBQlEsA +AAAAABgAAAAAAAAAMW0AABEADADgoE4AAAAAAAgAAAAAAAAATW0AABEADADYoE4AAAAAAAgAAAAA +AAAAaW0AABEADADgnk4AAAAAAAEAAAAAAAAAg20AABEACQBAgEsAAAAAAAgAAAAAAAAAnm0AABEA +DACgpU4AAAAAAJAAAAAAAAAAr20AABEADADAo04AAAAAAIIAAAAAAAAAwG0AABEADAAApU4AAAAA +AIkAAAAAAAAA020AABEACwCwtEsAAAAAABgAAAAAAAAA6G0AABEADAD8nk4AAAAAAAQAAAAAAAAA +FG4AABEACQDggEsAAAAAACAAAAAAAAAAL24AABEADABYn04AAAAAAAgAAAAAAAAAR24AABEAAgCI +akcAAAAAACAAAAAAAAAAaW4AABEACwCos0sAAAAAAAgAAAAAAAAAc24AABEACwDQs0sAAAAAAAgA +AAAAAAAAhW4AABEACwDAs0sAAAAAAAgAAAAAAAAAo24AABEACwCgs0sAAAAAAAgAAAAAAAAAsG4A +ABEACwDYs0sAAAAAAAgAAAAAAAAAu24AABEACwCws0sAAAAAAAgAAAAAAAAAxW4AABEACwC4s0sA +AAAAAAgAAAAAAAAA0W4AABEACwDIs0sAAAAAAAgAAAAAAAAA4G4AABEAAgAIaUcAAAAAAAgAAAAA +AAAA724AABEAAgBAaEcAAAAAAAcAAAAAAAAACG8AABEAAgAAYEUAAAAAAAAAAAAAAAAAD28AABEA +AgBwakcAAAAAABgAAAAAAAAAAGdvLmdvAHJ1bnRpbWUudGV4dABjbXBib2R5AG1lbWVxYm9keQBp +bmRleGJ5dGVib2R5AGdvZ28AZ29zYXZlX3N5c3RlbXN0YWNrX3N3aXRjaABzZXRnX2djYwBhZXNo +YXNoYm9keQBkZWJ1Z0NhbGwzMgBkZWJ1Z0NhbGw2NABkZWJ1Z0NhbGwxMjgAZGVidWdDYWxsMjU2 +AGRlYnVnQ2FsbDUxMgBkZWJ1Z0NhbGwxMDI0AGRlYnVnQ2FsbDIwNDgAZGVidWdDYWxsNDA5NgBk +ZWJ1Z0NhbGw4MTkyAGRlYnVnQ2FsbDE2Mzg0AGRlYnVnQ2FsbDMyNzY4AGRlYnVnQ2FsbDY1NTM2 +AHJ1bnRpbWUuZXRleHQAcnVudGltZS4uZ29ieXRlcy40AHJ1bnRpbWUuLmdvYnl0ZXMuNQBydW50 +aW1lLi5nb2J5dGVzLjYAcnVudGltZS4uZ29ieXRlcy43AHJ1bnRpbWUuZ2V0cGlkLmFyZ3Nfc3Rh +Y2ttYXAAcnVudGltZS50Z2tpbGwuYXJnc19zdGFja21hcABydW50aW1lLnB1YmxpY2F0aW9uQmFy +cmllci5hcmdzX3N0YWNrbWFwAHJ1bnRpbWUuYXNtY2dvY2FsbC5hcmdzX3N0YWNrbWFwAHJ1bnRp +bWUuY2hlY2tBU00uYXJnc19zdGFja21hcABtYXNrcwBzaGlmdHMAZGVidWdDYWxsRnJhbWVUb29M +YXJnZQBpbnRlcm5hbC9jcHUuY3B1aWQuYXJnc19zdGFja21hcABpbnRlcm5hbC9jcHUueGdldGJ2 +LmFyZ3Nfc3RhY2ttYXAAaW50ZXJuYWwvYnl0ZWFsZy5JbmRleEJ5dGVTdHJpbmcuYXJnc19zdGFj +a21hcAAkZjY0LjNlYjAwMDAwMDAwMDAwMDAAJGY2NC4zZjUwNjI0ZGQyZjFhOWZjACRmNjQuM2Y4 +NDdhZTE0N2FlMTQ3YgAkZjY0LjNmZDAwMDAwMDAwMDAwMDAAJGY2NC4zZmQzMzMzMzMzMzMzMzMz +ACRmNjQuM2ZlMDAwMDAwMDAwMDAwMAAkZjY0LjNmZTMzMzMzMzMzMzMzMzMAJGY2NC4zZmVjMDAw +MDAwMDAwMDAwACRmNjQuM2ZlZTY2NjY2NjY2NjY2NgAkZjY0LjNmZjAwMDAwMDAwMDAwMDAAJGY2 +NC4zZmYxOTk5OTk5OTk5OTlhACRmNjQuM2ZmMzMzMzMzMzMzMzMzMwAkZjY0LjQwMTQwMDAwMDAw +MDAwMDAAJGY2NC40MDI0MDAwMDAwMDAwMDAwACRmNjQuNDAzYTAwMDAwMDAwMDAwMAAkZjY0LjQw +NTkwMDAwMDAwMDAwMDAAJGY2NC40MGMzODgwMDAwMDAwMDAwACRmNjQuNDBmMDAwMDAwMDAwMDAw +MAAkZjY0LjQxNjMxMmQwMDAwMDAwMDAAJGY2NC40M2UwMDAwMDAwMDAwMDAwACRmNjQuN2ZmMDAw +MDAwMDAwMDAwMAAkZjY0LjgwMDAwMDAwMDAwMDAwMDAAJGY2NC5iZmQzMzMzMzMzMzMzMzMzACRm +NjQuYmZlNjJlNDJmZWZhMzllZgBydW50aW1lLnR5cGVsaW5rAHJ1bnRpbWUuaXRhYmxpbmsAcnVu +dGltZS5wY2xudGFiAHJ1bnRpbWUuZmluZGZ1bmN0YWIAcnVudGltZS5yb2RhdGEAcnVudGltZS5l +cm9kYXRhAHJ1bnRpbWUudHlwZXMAcnVudGltZS5ldHlwZXMAcnVudGltZS5ub3B0cmRhdGEAcnVu +dGltZS5lbm9wdHJkYXRhAHJ1bnRpbWUuZGF0YQBydW50aW1lLmVkYXRhAHJ1bnRpbWUuYnNzAHJ1 +bnRpbWUuZWJzcwBydW50aW1lLm5vcHRyYnNzAHJ1bnRpbWUuZW5vcHRyYnNzAHJ1bnRpbWUuZW5k +AHJ1bnRpbWUuZXBjbG50YWIAcnVudGltZS5lc3ltdGFiAHJ1bnRpbWUuZ2NkYXRhAHJ1bnRpbWUu +ZWdjZGF0YQBydW50aW1lLmdjYnNzAHJ1bnRpbWUuZWdjYnNzAGdvLnN0cmluZy4qAGdvLmZ1bmMu +KgBydW50aW1lLmdjYml0cy4qAHJ1bnRpbWUuc3ltdGFiAGludGVybmFsL2NwdS5Jbml0aWFsaXpl +AGludGVybmFsL2NwdS5wcm9jZXNzT3B0aW9ucwBpbnRlcm5hbC9jcHUuZG9pbml0AGludGVybmFs +L2NwdS5jcHVpZC5hYmkwAGludGVybmFsL2NwdS54Z2V0YnYuYWJpMAB0eXBlLi5lcS5pbnRlcm5h +bC9jcHUub3B0aW9uAHR5cGUuLmVxLlsxNV1pbnRlcm5hbC9jcHUub3B0aW9uAHJ1bnRpbWUvaW50 +ZXJuYWwvc3lzLk9uZXNDb3VudDY0AGludGVybmFsL2J5dGVhbGcuaW5pdC4wAHJ1bnRpbWUuY21w +c3RyaW5nAHJ1bnRpbWUubWVtZXF1YWwAcnVudGltZS5tZW1lcXVhbF92YXJsZW4AaW50ZXJuYWwv +Ynl0ZWFsZy5JbmRleEJ5dGVTdHJpbmcuYWJpMAB0eXBlLi5lcS5pbnRlcm5hbC9hYmkuUmVnQXJn +cwBydW50aW1lLm1lbWhhc2gxMjgAcnVudGltZS5tZW1lcXVhbDAAcnVudGltZS5tZW1lcXVhbDgA +cnVudGltZS5tZW1lcXVhbDE2AHJ1bnRpbWUubWVtZXF1YWwzMgBydW50aW1lLm1lbWVxdWFsNjQA +cnVudGltZS5tZW1lcXVhbDEyOABydW50aW1lLmYzMmVxdWFsAHJ1bnRpbWUuZjY0ZXF1YWwAcnVu +dGltZS5jNjRlcXVhbABydW50aW1lLmMxMjhlcXVhbABydW50aW1lLnN0cmVxdWFsAHJ1bnRpbWUu +aW50ZXJlcXVhbABydW50aW1lLm5pbGludGVyZXF1YWwAcnVudGltZS5lZmFjZWVxAHJ1bnRpbWUu +aWZhY2VlcQBydW50aW1lLmFsZ2luaXQAcnVudGltZS5hdG9taWN3YgBydW50aW1lLmF0b21pY3N0 +b3JlcABydW50aW1lLm1tYXAAcnVudGltZS5tbWFwLmZ1bmMxAHJ1bnRpbWUubXVubWFwAHJ1bnRp +bWUubXVubWFwLmZ1bmMxAHJ1bnRpbWUuc2lnYWN0aW9uAHJ1bnRpbWUuc2lnYWN0aW9uLmZ1bmMx +AHJ1bnRpbWUuY2dvY2FsbABydW50aW1lLmNnb0lzR29Qb2ludGVyAHJ1bnRpbWUuY2dvQ2hlY2tX +cml0ZUJhcnJpZXIAcnVudGltZS5jZ29DaGVja1dyaXRlQmFycmllci5mdW5jMQBydW50aW1lLmNn +b0NoZWNrTWVtbW92ZQBydW50aW1lLmNnb0NoZWNrU2xpY2VDb3B5AHJ1bnRpbWUuY2dvQ2hlY2tU +eXBlZEJsb2NrAHJ1bnRpbWUuY2dvQ2hlY2tUeXBlZEJsb2NrLmZ1bmMxAHJ1bnRpbWUuY2dvQ2hl +Y2tCaXRzAHJ1bnRpbWUuY2dvQ2hlY2tVc2luZ1R5cGUAcnVudGltZS5tYWtlY2hhbgBydW50aW1l +LmNoYW5zZW5kMQBydW50aW1lLmNoYW5zZW5kAHJ1bnRpbWUuY2hhbnNlbmQuZnVuYzEAcnVudGlt +ZS5zZW5kAHJ1bnRpbWUuc2VuZERpcmVjdABydW50aW1lLnJlY3ZEaXJlY3QAcnVudGltZS5jbG9z +ZWNoYW4AcnVudGltZS5jaGFucmVjdjEAcnVudGltZS5jaGFucmVjdgBydW50aW1lLmNoYW5yZWN2 +LmZ1bmMxAHJ1bnRpbWUucmVjdgBydW50aW1lLmNoYW5wYXJrY29tbWl0AHJ1bnRpbWUuaW5pdC4w +AHJ1bnRpbWUuKCpjcHVQcm9maWxlKS5hZGQAcnVudGltZS4oKmNwdVByb2ZpbGUpLmFkZE5vbkdv +AHJ1bnRpbWUuKCpjcHVQcm9maWxlKS5hZGRFeHRyYQBydW50aW1lLmRlYnVnQ2FsbENoZWNrAHJ1 +bnRpbWUuZGVidWdDYWxsQ2hlY2suZnVuYzEAcnVudGltZS5kZWJ1Z0NhbGxXcmFwAHJ1bnRpbWUu +ZGVidWdDYWxsV3JhcC5mdW5jMQBydW50aW1lLmRlYnVnQ2FsbFdyYXAxAHJ1bnRpbWUuZGVidWdD +YWxsV3JhcDIAcnVudGltZS5kZWJ1Z0NhbGxXcmFwMi5mdW5jMQBydW50aW1lLmdvZ2V0ZW52AHJ1 +bnRpbWUuKCpUeXBlQXNzZXJ0aW9uRXJyb3IpLkVycm9yAHJ1bnRpbWUuZXJyb3JTdHJpbmcuRXJy +b3IAcnVudGltZS5lcnJvckFkZHJlc3NTdHJpbmcuRXJyb3IAcnVudGltZS5wbGFpbkVycm9yLkVy +cm9yAHJ1bnRpbWUuYm91bmRzRXJyb3IuRXJyb3IAcnVudGltZS5wcmludGFueQBydW50aW1lLnBy +aW50YW55Y3VzdG9tdHlwZQBydW50aW1lLnBhbmljd3JhcABydW50aW1lLm1lbWhhc2hGYWxsYmFj +awBydW50aW1lLm1lbWhhc2gzMkZhbGxiYWNrAHJ1bnRpbWUubWVtaGFzaDY0RmFsbGJhY2sAcnVu +dGltZS5nZXRpdGFiAHJ1bnRpbWUuKCppdGFiVGFibGVUeXBlKS5maW5kAHJ1bnRpbWUuaXRhYkFk +ZABydW50aW1lLigqaXRhYlRhYmxlVHlwZSkuYWRkAHJ1bnRpbWUuKCppdGFiKS5pbml0AHJ1bnRp +bWUuaXRhYnNpbml0AHJ1bnRpbWUuY29udlQyRQBydW50aW1lLmNvbnZUc3RyaW5nAHJ1bnRpbWUu +Y29udlQyRW5vcHRyAHJ1bnRpbWUuYXNzZXJ0RTJJMgBydW50aW1lLml0ZXJhdGVfaXRhYnMAcnVu +dGltZS51bnJlYWNoYWJsZU1ldGhvZABydW50aW1lLigqbGZzdGFjaykucHVzaABydW50aW1lLmxm +bm9kZVZhbGlkYXRlAHJ1bnRpbWUubG9jawBydW50aW1lLmxvY2syAHJ1bnRpbWUudW5sb2NrAHJ1 +bnRpbWUudW5sb2NrMgBydW50aW1lLm5vdGV3YWtldXAAcnVudGltZS5ub3Rlc2xlZXAAcnVudGlt +ZS5ub3RldHNsZWVwX2ludGVybmFsAHJ1bnRpbWUubm90ZXRzbGVlcABydW50aW1lLm5vdGV0c2xl +ZXBnAHJ1bnRpbWUubG9ja1JhbmsuU3RyaW5nAHJ1bnRpbWUubG9ja1dpdGhSYW5rAHJ1bnRpbWUu +dW5sb2NrV2l0aFJhbmsAcnVudGltZS5tYWxsb2Npbml0AHJ1bnRpbWUuKCptaGVhcCkuc3lzQWxs +b2MAcnVudGltZS5zeXNSZXNlcnZlQWxpZ25lZABydW50aW1lLigqbWNhY2hlKS5uZXh0RnJlZQBy +dW50aW1lLm1hbGxvY2djAHJ1bnRpbWUubWVtY2xyTm9IZWFwUG9pbnRlcnNDaHVua2VkAHJ1bnRp +bWUubmV3b2JqZWN0AHJ1bnRpbWUubmV3YXJyYXkAcnVudGltZS5wcm9maWxlYWxsb2MAcnVudGlt +ZS5mYXN0ZXhwcmFuZABydW50aW1lLnBlcnNpc3RlbnRhbGxvYwBydW50aW1lLnBlcnNpc3RlbnRh +bGxvYy5mdW5jMQBydW50aW1lLnBlcnNpc3RlbnRhbGxvYzEAcnVudGltZS4oKmxpbmVhckFsbG9j +KS5hbGxvYwBydW50aW1lLigqaG1hcCkuaW5jcm5vdmVyZmxvdwBydW50aW1lLigqaG1hcCkubmV3 +b3ZlcmZsb3cAcnVudGltZS5tYWtlbWFwAHJ1bnRpbWUubWFrZUJ1Y2tldEFycmF5AHJ1bnRpbWUu +bWFwYWNjZXNzMgBydW50aW1lLm1hcGFzc2lnbgBydW50aW1lLmhhc2hHcm93AHJ1bnRpbWUuZ3Jv +d1dvcmsAcnVudGltZS5ldmFjdWF0ZQBydW50aW1lLmFkdmFuY2VFdmFjdWF0aW9uTWFyawBydW50 +aW1lLm1hcGFjY2VzczFfZmFzdDMyAHJ1bnRpbWUubWFwYWNjZXNzMl9mYXN0MzIAcnVudGltZS5t +YXBhc3NpZ25fZmFzdDMyAHJ1bnRpbWUuZ3Jvd1dvcmtfZmFzdDMyAHJ1bnRpbWUuZXZhY3VhdGVf +ZmFzdDMyAHJ1bnRpbWUudHlwZWRtZW1tb3ZlAHJ1bnRpbWUudHlwZWRzbGljZWNvcHkAcnVudGlt +ZS50eXBlZG1lbWNscgBydW50aW1lLm1lbWNsckhhc1BvaW50ZXJzAHJ1bnRpbWUuKCptc3Bhbiku +cmVmaWxsQWxsb2NDYWNoZQBydW50aW1lLigqbXNwYW4pLm5leHRGcmVlSW5kZXgAcnVudGltZS5i +YWRQb2ludGVyAHJ1bnRpbWUuZmluZE9iamVjdABydW50aW1lLmhlYXBCaXRzLm5leHRBcmVuYQBy +dW50aW1lLmhlYXBCaXRzLmZvcndhcmQAcnVudGltZS5oZWFwQml0cy5mb3J3YXJkT3JCb3VuZGFy +eQBydW50aW1lLmJ1bGtCYXJyaWVyUHJlV3JpdGUAcnVudGltZS5idWxrQmFycmllclByZVdyaXRl +U3JjT25seQBydW50aW1lLmJ1bGtCYXJyaWVyQml0bWFwAHJ1bnRpbWUudHlwZUJpdHNCdWxrQmFy +cmllcgBydW50aW1lLmhlYXBCaXRzLmluaXRTcGFuAHJ1bnRpbWUuaGVhcEJpdHNTZXRUeXBlAHJ1 +bnRpbWUuaGVhcEJpdHNTZXRUeXBlR0NQcm9nAHJ1bnRpbWUucHJvZ1RvUG9pbnRlck1hc2sAcnVu +dGltZS5ydW5HQ1Byb2cAcnVudGltZS5tYXRlcmlhbGl6ZUdDUHJvZwBydW50aW1lLmFsbG9jbWNh +Y2hlAHJ1bnRpbWUuYWxsb2NtY2FjaGUuZnVuYzEAcnVudGltZS5mcmVlbWNhY2hlAHJ1bnRpbWUu +ZnJlZW1jYWNoZS5mdW5jMQBydW50aW1lLigqbWNhY2hlKS5yZWZpbGwAcnVudGltZS4oKm1jYWNo +ZSkuYWxsb2NMYXJnZQBydW50aW1lLigqbWNhY2hlKS5yZWxlYXNlQWxsAHJ1bnRpbWUuKCptY2Fj +aGUpLnByZXBhcmVGb3JTd2VlcABydW50aW1lLigqbWNlbnRyYWwpLmNhY2hlU3BhbgBydW50aW1l +LigqbWNlbnRyYWwpLnVuY2FjaGVTcGFuAHJ1bnRpbWUuKCptY2VudHJhbCkuZ3JvdwBydW50aW1l +LnN0YXJ0Q2hlY2ttYXJrcwBydW50aW1lLmVuZENoZWNrbWFya3MAcnVudGltZS5zZXRDaGVja21h +cmsAcnVudGltZS5zeXNBbGxvYwBydW50aW1lLnN5c1VudXNlZABydW50aW1lLnN5c0h1Z2VQYWdl +AHJ1bnRpbWUuc3lzRnJlZQBydW50aW1lLnN5c01hcABydW50aW1lLnF1ZXVlZmluYWxpemVyAHJ1 +bnRpbWUud2FrZWZpbmcAcnVudGltZS4oKmZpeGFsbG9jKS5hbGxvYwBydW50aW1lLmdjaW5pdABy +dW50aW1lLmdjZW5hYmxlAHJ1bnRpbWUucG9sbEZyYWN0aW9uYWxXb3JrZXJFeGl0AHJ1bnRpbWUu +Z2NTdGFydABydW50aW1lLmdjU3RhcnQuZnVuYzIAcnVudGltZS5nY01hcmtEb25lAHJ1bnRpbWUu +Z2NNYXJrRG9uZS5mdW5jMgBydW50aW1lLmdjTWFya1Rlcm1pbmF0aW9uAHJ1bnRpbWUuZ2NNYXJr +VGVybWluYXRpb24uZnVuYzEAcnVudGltZS5nY0JnTWFya1N0YXJ0V29ya2VycwBydW50aW1lLmdj +QmdNYXJrV29ya2VyAHJ1bnRpbWUuZ2NCZ01hcmtXb3JrZXIuZnVuYzIAcnVudGltZS5nY01hcmsA +cnVudGltZS5nY1N3ZWVwAHJ1bnRpbWUuZ2NSZXNldE1hcmtTdGF0ZQBydW50aW1lLmNsZWFycG9v +bHMAcnVudGltZS5mbXROU0FzTVMAcnVudGltZS5nY01hcmtSb290UHJlcGFyZQBydW50aW1lLmdj +TWFya1Jvb3RDaGVjawBydW50aW1lLmdjTWFya1Jvb3RDaGVjay5mdW5jMQBydW50aW1lLm1hcmty +b290AHJ1bnRpbWUubWFya3Jvb3QuZnVuYzEAcnVudGltZS5tYXJrcm9vdEJsb2NrAHJ1bnRpbWUu +bWFya3Jvb3RGcmVlR1N0YWNrcwBydW50aW1lLm1hcmtyb290U3BhbnMAcnVudGltZS5nY0Fzc2lz +dEFsbG9jAHJ1bnRpbWUuZ2NBc3Npc3RBbGxvYy5mdW5jMQBydW50aW1lLmdjQXNzaXN0QWxsb2Mx +AHJ1bnRpbWUuZ2NXYWtlQWxsQXNzaXN0cwBydW50aW1lLmdjUGFya0Fzc2lzdABydW50aW1lLmdj +Rmx1c2hCZ0NyZWRpdABydW50aW1lLnNjYW5zdGFjawBydW50aW1lLnNjYW5zdGFjay5mdW5jMQBy +dW50aW1lLnNjYW5mcmFtZXdvcmtlcgBydW50aW1lLmdjRHJhaW4AcnVudGltZS5nY0RyYWluTgBy +dW50aW1lLnNjYW5ibG9jawBydW50aW1lLnNjYW5vYmplY3QAcnVudGltZS5zY2FuQ29uc2VydmF0 +aXZlAHJ1bnRpbWUuc2hhZGUAcnVudGltZS5ncmV5b2JqZWN0AHJ1bnRpbWUuZ2NEdW1wT2JqZWN0 +AHJ1bnRpbWUuZ2NtYXJrbmV3b2JqZWN0AHJ1bnRpbWUuZ2NNYXJrVGlueUFsbG9jcwBydW50aW1l +LmluaXQuMQBydW50aW1lLigqZ2NDb250cm9sbGVyU3RhdGUpLmluaXQAcnVudGltZS4oKmdjQ29u +dHJvbGxlclN0YXRlKS5zdGFydEN5Y2xlAHJ1bnRpbWUuKCpnY0NvbnRyb2xsZXJTdGF0ZSkucmV2 +aXNlAHJ1bnRpbWUuKCpnY0NvbnRyb2xsZXJTdGF0ZSkuZW5kQ3ljbGUAcnVudGltZS4oKmdjQ29u +dHJvbGxlclN0YXRlKS5lbmxpc3RXb3JrZXIAcnVudGltZS4oKmdjQ29udHJvbGxlclN0YXRlKS5m +aW5kUnVubmFibGVHQ1dvcmtlcgBydW50aW1lLigqZ2NDb250cm9sbGVyU3RhdGUpLmNvbW1pdABy +dW50aW1lLigqZ2NDb250cm9sbGVyU3RhdGUpLnNldEdDUGVyY2VudABydW50aW1lLnJlYWRHT0dD +AHJ1bnRpbWUuZ2NQYWNlU2NhdmVuZ2VyAHJ1bnRpbWUud2FrZVNjYXZlbmdlcgBydW50aW1lLnNj +YXZlbmdlU2xlZXAAcnVudGltZS5iZ3NjYXZlbmdlAHJ1bnRpbWUuYmdzY2F2ZW5nZS5mdW5jMgBy +dW50aW1lLigqcGFnZUFsbG9jKS5zY2F2ZW5nZQBydW50aW1lLnByaW50U2NhdlRyYWNlAHJ1bnRp +bWUuKCpwYWdlQWxsb2MpLnNjYXZlbmdlU3RhcnRHZW4AcnVudGltZS4oKnBhZ2VBbGxvYykuc2Nh +dmVuZ2VSZXNlcnZlAHJ1bnRpbWUuKCpwYWdlQWxsb2MpLnNjYXZlbmdlVW5yZXNlcnZlAHJ1bnRp +bWUuKCpwYWdlQWxsb2MpLnNjYXZlbmdlT25lAHJ1bnRpbWUuKCpwYWdlQWxsb2MpLnNjYXZlbmdl +T25lLmZ1bmMzAHJ1bnRpbWUuKCpwYWdlQWxsb2MpLnNjYXZlbmdlUmFuZ2VMb2NrZWQAcnVudGlt +ZS5maWxsQWxpZ25lZABydW50aW1lLigqcGFsbG9jRGF0YSkuaGFzU2NhdmVuZ2VDYW5kaWRhdGUA +cnVudGltZS4oKnBhbGxvY0RhdGEpLmZpbmRTY2F2ZW5nZUNhbmRpZGF0ZQBydW50aW1lLigqc3Rh +Y2tTY2FuU3RhdGUpLnB1dFB0cgBydW50aW1lLigqc3RhY2tTY2FuU3RhdGUpLmdldFB0cgBydW50 +aW1lLigqc3RhY2tTY2FuU3RhdGUpLmFkZE9iamVjdABydW50aW1lLmJpbmFyeVNlYXJjaFRyZWUA +cnVudGltZS4oKm1oZWFwKS5uZXh0U3BhbkZvclN3ZWVwAHJ1bnRpbWUuZmluaXNoc3dlZXBfbQBy +dW50aW1lLmJnc3dlZXAAcnVudGltZS5zd2VlcG9uZQBydW50aW1lLigqbXNwYW4pLmVuc3VyZVN3 +ZXB0AHJ1bnRpbWUuKCpzd2VlcExvY2tlZCkuc3dlZXAAcnVudGltZS4oKm1zcGFuKS5yZXBvcnRa +b21iaWVzAHJ1bnRpbWUuZGVkdWN0U3dlZXBDcmVkaXQAcnVudGltZS4oKmdjV29yaykuaW5pdABy +dW50aW1lLigqZ2NXb3JrKS5wdXQAcnVudGltZS4oKmdjV29yaykucHV0QmF0Y2gAcnVudGltZS4o +KmdjV29yaykudHJ5R2V0AHJ1bnRpbWUuKCpnY1dvcmspLmRpc3Bvc2UAcnVudGltZS4oKmdjV29y +aykuYmFsYW5jZQBydW50aW1lLigqd29ya2J1ZikuY2hlY2tub25lbXB0eQBydW50aW1lLigqd29y +a2J1ZikuY2hlY2tlbXB0eQBydW50aW1lLmdldGVtcHR5AHJ1bnRpbWUuZ2V0ZW1wdHkuZnVuYzEA +cnVudGltZS5wdXRlbXB0eQBydW50aW1lLnB1dGZ1bGwAcnVudGltZS50cnlnZXRmdWxsAHJ1bnRp +bWUuaGFuZG9mZgBydW50aW1lLnByZXBhcmVGcmVlV29ya2J1ZnMAcnVudGltZS5mcmVlU29tZVdi +dWZzAHJ1bnRpbWUuZnJlZVNvbWVXYnVmcy5mdW5jMQBydW50aW1lLnJlY29yZHNwYW4AcnVudGlt +ZS5pbkhlYXBPclN0YWNrAHJ1bnRpbWUuc3Bhbk9mSGVhcABydW50aW1lLigqbWhlYXApLmluaXQA +cnVudGltZS4oKm1oZWFwKS5yZWNsYWltAHJ1bnRpbWUuKCptaGVhcCkucmVjbGFpbUNodW5rAHJ1 +bnRpbWUuKCptaGVhcCkuYWxsb2MAcnVudGltZS4oKm1oZWFwKS5hbGxvYy5mdW5jMQBydW50aW1l +LigqbWhlYXApLmFsbG9jTWFudWFsAHJ1bnRpbWUuKCptaGVhcCkuc2V0U3BhbnMAcnVudGltZS4o +Km1oZWFwKS5hbGxvY05lZWRzWmVybwBydW50aW1lLigqbWhlYXApLmFsbG9jTVNwYW5Mb2NrZWQA +cnVudGltZS4oKm1oZWFwKS5hbGxvY1NwYW4AcnVudGltZS4oKm1oZWFwKS5ncm93AHJ1bnRpbWUu +KCptaGVhcCkuZnJlZVNwYW4AcnVudGltZS4oKm1oZWFwKS5mcmVlU3Bhbi5mdW5jMQBydW50aW1l +LigqbWhlYXApLmZyZWVNYW51YWwAcnVudGltZS4oKm1oZWFwKS5mcmVlU3BhbkxvY2tlZABydW50 +aW1lLigqbVNwYW5MaXN0KS5yZW1vdmUAcnVudGltZS4oKm1TcGFuTGlzdCkuaW5zZXJ0AHJ1bnRp +bWUuYWRkc3BlY2lhbABydW50aW1lLnNldHByb2ZpbGVidWNrZXQAcnVudGltZS5mcmVlU3BlY2lh +bABydW50aW1lLm5ld01hcmtCaXRzAHJ1bnRpbWUubmV3QWxsb2NCaXRzAHJ1bnRpbWUubmV4dE1h +cmtCaXRBcmVuYUVwb2NoAHJ1bnRpbWUubmV3QXJlbmFNYXlVbmxvY2sAcnVudGltZS4oKnBhZ2VB +bGxvYykuaW5pdABydW50aW1lLigqcGFnZUFsbG9jKS5ncm93AHJ1bnRpbWUuKCpwYWdlQWxsb2Mp +LnVwZGF0ZQBydW50aW1lLigqcGFnZUFsbG9jKS5hbGxvY1JhbmdlAHJ1bnRpbWUuKCpwYWdlQWxs +b2MpLmZpbmRNYXBwZWRBZGRyAHJ1bnRpbWUuKCpwYWdlQWxsb2MpLmZpbmQAcnVudGltZS4oKnBh +Z2VBbGxvYykuZmluZC5mdW5jMQBydW50aW1lLigqcGFnZUFsbG9jKS5hbGxvYwBydW50aW1lLigq +cGFnZUFsbG9jKS5mcmVlAHJ1bnRpbWUubWVyZ2VTdW1tYXJpZXMAcnVudGltZS4oKnBhZ2VBbGxv +Yykuc3lzSW5pdABydW50aW1lLigqcGFnZUFsbG9jKS5zeXNHcm93AHJ1bnRpbWUuKCpwYWdlQWxs +b2MpLnN5c0dyb3cuZnVuYzMAcnVudGltZS4oKnBhZ2VBbGxvYykuc3lzR3Jvdy5mdW5jMgBydW50 +aW1lLigqcGFnZUNhY2hlKS5hbGxvYwBydW50aW1lLigqcGFnZUNhY2hlKS5hbGxvY04AcnVudGlt +ZS4oKnBhZ2VDYWNoZSkuZmx1c2gAcnVudGltZS4oKnBhZ2VBbGxvYykuYWxsb2NUb0NhY2hlAHJ1 +bnRpbWUuKCpwYWdlQml0cykuc2V0UmFuZ2UAcnVudGltZS4oKnBhZ2VCaXRzKS5zZXRBbGwAcnVu +dGltZS4oKnBhZ2VCaXRzKS5jbGVhclJhbmdlAHJ1bnRpbWUuKCpwYWdlQml0cykuY2xlYXJBbGwA +cnVudGltZS4oKnBhZ2VCaXRzKS5wb3BjbnRSYW5nZQBydW50aW1lLigqcGFsbG9jQml0cykuc3Vt +bWFyaXplAHJ1bnRpbWUuKCpwYWxsb2NCaXRzKS5maW5kAHJ1bnRpbWUuKCpwYWxsb2NCaXRzKS5m +aW5kU21hbGxOAHJ1bnRpbWUuKCpwYWxsb2NCaXRzKS5maW5kTGFyZ2VOAHJ1bnRpbWUuKCpwYWxs +b2NEYXRhKS5hbGxvY1JhbmdlAHJ1bnRpbWUuKCpwYWxsb2NEYXRhKS5hbGxvY0FsbABydW50aW1l +Lm5ld0J1Y2tldABydW50aW1lLigqYnVja2V0KS5tcABydW50aW1lLigqYnVja2V0KS5icABydW50 +aW1lLnN0a2J1Y2tldABydW50aW1lLmVxc2xpY2UAcnVudGltZS5tUHJvZl9OZXh0Q3ljbGUAcnVu +dGltZS5tUHJvZl9GbHVzaABydW50aW1lLm1Qcm9mX0ZsdXNoTG9ja2VkAHJ1bnRpbWUubVByb2Zf +TWFsbG9jAHJ1bnRpbWUubVByb2ZfTWFsbG9jLmZ1bmMxAHJ1bnRpbWUubVByb2ZfRnJlZQBydW50 +aW1lLmJsb2NrZXZlbnQAcnVudGltZS5zYXZlYmxvY2tldmVudABydW50aW1lLnRyYWNlYWxsb2MA +cnVudGltZS50cmFjZWFsbG9jLmZ1bmMxAHJ1bnRpbWUudHJhY2VmcmVlAHJ1bnRpbWUudHJhY2Vm +cmVlLmZ1bmMxAHJ1bnRpbWUudHJhY2VnYwBydW50aW1lLm1ha2VBZGRyUmFuZ2UAcnVudGltZS5h +ZGRyUmFuZ2Uuc3VidHJhY3QAcnVudGltZS5hZGRyUmFuZ2UucmVtb3ZlR3JlYXRlckVxdWFsAHJ1 +bnRpbWUuKCphZGRyUmFuZ2VzKS5pbml0AHJ1bnRpbWUuKCphZGRyUmFuZ2VzKS5maW5kU3VjYwBy +dW50aW1lLigqYWRkclJhbmdlcykuZmluZEFkZHJHcmVhdGVyRXF1YWwAcnVudGltZS4oKmFkZHJS +YW5nZXMpLmFkZABydW50aW1lLigqYWRkclJhbmdlcykucmVtb3ZlTGFzdABydW50aW1lLigqYWRk +clJhbmdlcykucmVtb3ZlR3JlYXRlckVxdWFsAHJ1bnRpbWUuKCphZGRyUmFuZ2VzKS5jbG9uZUlu +dG8AcnVudGltZS4oKnNwYW5TZXQpLnB1c2gAcnVudGltZS4oKnNwYW5TZXQpLnBvcABydW50aW1l +Ligqc3BhblNldCkucmVzZXQAcnVudGltZS4oKnNwYW5TZXRCbG9ja0FsbG9jKS5hbGxvYwBydW50 +aW1lLigqaGVhZFRhaWxJbmRleCkuaW5jVGFpbABydW50aW1lLmluaXQuNABydW50aW1lLigqc3lz +TWVtU3RhdCkuYWRkAHJ1bnRpbWUuKCpjb25zaXN0ZW50SGVhcFN0YXRzKS5hY3F1aXJlAHJ1bnRp +bWUuKCpjb25zaXN0ZW50SGVhcFN0YXRzKS5yZWxlYXNlAHJ1bnRpbWUuKCp3YkJ1ZikucmVzZXQA +cnVudGltZS53YkJ1ZkZsdXNoAHJ1bnRpbWUud2JCdWZGbHVzaDEAcnVudGltZS5ub25ibG9ja2lu +Z1BpcGUAcnVudGltZS5uZXRwb2xsR2VuZXJpY0luaXQAcnVudGltZS5uZXRwb2xscmVhZHkAcnVu +dGltZS5uZXRwb2xsaW5pdABydW50aW1lLm5ldHBvbGxCcmVhawBydW50aW1lLm5ldHBvbGwAcnVu +dGltZS5mdXRleHNsZWVwAHJ1bnRpbWUuZnV0ZXh3YWtldXAAcnVudGltZS5mdXRleHdha2V1cC5m +dW5jMQBydW50aW1lLmdldHByb2Njb3VudABydW50aW1lLm5ld29zcHJvYwBydW50aW1lLnN5c2Fy +Z3MAcnVudGltZS5zeXNhdXh2AHJ1bnRpbWUuZ2V0SHVnZVBhZ2VTaXplAHJ1bnRpbWUub3Npbml0 +AHJ1bnRpbWUuZ2V0UmFuZG9tRGF0YQBydW50aW1lLm1wcmVpbml0AHJ1bnRpbWUubWluaXQAcnVu +dGltZS5zZXRzaWcAcnVudGltZS5zZXRzaWdzdGFjawBydW50aW1lLnN5c1NpZ2FjdGlvbgBydW50 +aW1lLnNpZ25hbE0AcnVudGltZS5wYW5pY0NoZWNrMQBydW50aW1lLnBhbmljQ2hlY2syAHJ1bnRp +bWUuZ29QYW5pY0luZGV4AHJ1bnRpbWUuZ29QYW5pY0luZGV4VQBydW50aW1lLmdvUGFuaWNTbGlj +ZUFsZW4AcnVudGltZS5nb1BhbmljU2xpY2VBbGVuVQBydW50aW1lLmdvUGFuaWNTbGljZUFjYXAA +cnVudGltZS5nb1BhbmljU2xpY2VBY2FwVQBydW50aW1lLmdvUGFuaWNTbGljZUIAcnVudGltZS5n +b1BhbmljU2xpY2VCVQBydW50aW1lLmdvUGFuaWNTbGljZTNBbGVuAHJ1bnRpbWUuZ29QYW5pY1Ns +aWNlM0FsZW5VAHJ1bnRpbWUucGFuaWNzaGlmdABydW50aW1lLnBhbmljZGl2aWRlAHJ1bnRpbWUu +dGVzdGRlZmVyc2l6ZXMAcnVudGltZS5pbml0LjUAcnVudGltZS5uZXdkZWZlcgBydW50aW1lLm5l +d2RlZmVyLmZ1bmMyAHJ1bnRpbWUubmV3ZGVmZXIuZnVuYzEAcnVudGltZS5mcmVlZGVmZXIAcnVu +dGltZS5mcmVlZGVmZXIuZnVuYzEAcnVudGltZS5mcmVlZGVmZXJwYW5pYwBydW50aW1lLmZyZWVk +ZWZlcmZuAHJ1bnRpbWUuZGVmZXJyZXR1cm4AcnVudGltZS5wcmVwcmludHBhbmljcwBydW50aW1l +LnByaW50cGFuaWNzAHJ1bnRpbWUuYWRkT25lT3BlbkRlZmVyRnJhbWUAcnVudGltZS5hZGRPbmVP +cGVuRGVmZXJGcmFtZS5mdW5jMQBydW50aW1lLmFkZE9uZU9wZW5EZWZlckZyYW1lLmZ1bmMxLjEA +cnVudGltZS5ydW5PcGVuRGVmZXJGcmFtZQBydW50aW1lLmRlZmVyQ2FsbFNhdmUAcnVudGltZS5n +b3BhbmljAHJ1bnRpbWUuZ2V0YXJncABydW50aW1lLmdvcmVjb3ZlcgBydW50aW1lLnRocm93AHJ1 +bnRpbWUudGhyb3cuZnVuYzEAcnVudGltZS5yZWNvdmVyeQBydW50aW1lLmZhdGFsdGhyb3cAcnVu +dGltZS5mYXRhbHRocm93LmZ1bmMxAHJ1bnRpbWUuZmF0YWxwYW5pYwBydW50aW1lLmZhdGFscGFu +aWMuZnVuYzEAcnVudGltZS5zdGFydHBhbmljX20AcnVudGltZS5kb3BhbmljX20AcnVudGltZS5z +aG91bGRQdXNoU2lncGFuaWMAcnVudGltZS5pc0Fib3J0UEMAcnVudGltZS5zdXNwZW5kRwBydW50 +aW1lLnJlc3VtZUcAcnVudGltZS5hc3luY1ByZWVtcHQyAHJ1bnRpbWUuaW5pdC42AHJ1bnRpbWUu +aXNBc3luY1NhZmVQb2ludABydW50aW1lLnJlY29yZEZvclBhbmljAHJ1bnRpbWUucHJpbnRsb2Nr +AHJ1bnRpbWUucHJpbnR1bmxvY2sAcnVudGltZS5nd3JpdGUAcnVudGltZS5wcmludHNwAHJ1bnRp +bWUucHJpbnRubABydW50aW1lLnByaW50Ym9vbABydW50aW1lLnByaW50ZmxvYXQAcnVudGltZS5w +cmludGNvbXBsZXgAcnVudGltZS5wcmludHVpbnQAcnVudGltZS5wcmludGludABydW50aW1lLnBy +aW50aGV4AHJ1bnRpbWUucHJpbnRwb2ludGVyAHJ1bnRpbWUucHJpbnR1aW50cHRyAHJ1bnRpbWUu +cHJpbnRzdHJpbmcAcnVudGltZS5wcmludHNsaWNlAHJ1bnRpbWUuaGV4ZHVtcFdvcmRzAHJ1bnRp +bWUubWFpbgBydW50aW1lLm1haW4uZnVuYzIAcnVudGltZS5pbml0LjcAcnVudGltZS5mb3JjZWdj +aGVscGVyAHJ1bnRpbWUuR29zY2hlZABydW50aW1lLmdvcGFyawBydW50aW1lLmdvcmVhZHkAcnVu +dGltZS5nb3JlYWR5LmZ1bmMxAHJ1bnRpbWUuYWNxdWlyZVN1ZG9nAHJ1bnRpbWUucmVsZWFzZVN1 +ZG9nAHJ1bnRpbWUuYmFkbWNhbGwAcnVudGltZS5iYWRtY2FsbDIAcnVudGltZS5iYWRtb3Jlc3Rh +Y2tnMABydW50aW1lLmJhZG1vcmVzdGFja2dzaWduYWwAcnVudGltZS5iYWRjdHh0AHJ1bnRpbWUu +YWxsZ2FkZABydW50aW1lLmZvckVhY2hHAHJ1bnRpbWUuZm9yRWFjaEdSYWNlAHJ1bnRpbWUuY3B1 +aW5pdABydW50aW1lLnNjaGVkaW5pdABydW50aW1lLmNoZWNrbWNvdW50AHJ1bnRpbWUubVJlc2Vy +dmVJRABydW50aW1lLm1jb21tb25pbml0AHJ1bnRpbWUucmVhZHkAcnVudGltZS5mcmVlemV0aGV3 +b3JsZABydW50aW1lLmNhc2Zyb21fR3NjYW5zdGF0dXMAcnVudGltZS5jYXN0b2dzY2Fuc3RhdHVz +AHJ1bnRpbWUuY2FzZ3N0YXR1cwBydW50aW1lLmNhc2dzdGF0dXMuZnVuYzEAcnVudGltZS5jYXNH +VG9QcmVlbXB0U2NhbgBydW50aW1lLmNhc0dGcm9tUHJlZW1wdGVkAHJ1bnRpbWUuc3RvcFRoZVdv +cmxkV2l0aFNlbWEAcnVudGltZS5zdGFydFRoZVdvcmxkV2l0aFNlbWEAcnVudGltZS5tc3RhcnQw +AHJ1bnRpbWUubXN0YXJ0MQBydW50aW1lLm1zdGFydG0wAHJ1bnRpbWUubVBhcmsAcnVudGltZS5t +ZXhpdABydW50aW1lLmZvckVhY2hQAHJ1bnRpbWUucnVuU2FmZVBvaW50Rm4AcnVudGltZS5hbGxv +Y20AcnVudGltZS5hbGxvY20uZnVuYzEAcnVudGltZS5uZWVkbQBydW50aW1lLm5ld2V4dHJhbQBy +dW50aW1lLm9uZU5ld0V4dHJhTQBydW50aW1lLmRyb3BtAHJ1bnRpbWUubG9ja2V4dHJhAHJ1bnRp +bWUubmV3bQBydW50aW1lLm5ld20xAHJ1bnRpbWUuc3RhcnRUZW1wbGF0ZVRocmVhZABydW50aW1l +Lm1Eb0ZpeHVwAHJ1bnRpbWUubURvRml4dXBBbmRPU1lpZWxkAHJ1bnRpbWUudGVtcGxhdGVUaHJl +YWQAcnVudGltZS5zdG9wbQBydW50aW1lLm1zcGlubmluZwBydW50aW1lLnN0YXJ0bQBydW50aW1l +LmhhbmRvZmZwAHJ1bnRpbWUud2FrZXAAcnVudGltZS5zdG9wbG9ja2VkbQBydW50aW1lLnN0YXJ0 +bG9ja2VkbQBydW50aW1lLmdjc3RvcG0AcnVudGltZS5leGVjdXRlAHJ1bnRpbWUuZmluZHJ1bm5h +YmxlAHJ1bnRpbWUucG9sbFdvcmsAcnVudGltZS5zdGVhbFdvcmsAcnVudGltZS5jaGVja1J1bnFz +Tm9QAHJ1bnRpbWUuY2hlY2tUaW1lcnNOb1AAcnVudGltZS5jaGVja0lkbGVHQ05vUABydW50aW1l +Lndha2VOZXRQb2xsZXIAcnVudGltZS5yZXNldHNwaW5uaW5nAHJ1bnRpbWUuaW5qZWN0Z2xpc3QA +cnVudGltZS5zY2hlZHVsZQBydW50aW1lLmNoZWNrVGltZXJzAHJ1bnRpbWUucGFya3VubG9ja19j +AHJ1bnRpbWUucGFya19tAHJ1bnRpbWUuZ29zY2hlZEltcGwAcnVudGltZS5nb3NjaGVkX20AcnVu +dGltZS5nb3NjaGVkZ3VhcmRlZF9tAHJ1bnRpbWUuZ29wcmVlbXB0X20AcnVudGltZS5wcmVlbXB0 +UGFyawBydW50aW1lLmdveWllbGRfbQBydW50aW1lLmdvZXhpdDEAcnVudGltZS5nb2V4aXQwAHJ1 +bnRpbWUuc2F2ZQBydW50aW1lLnJlZW50ZXJzeXNjYWxsAHJ1bnRpbWUucmVlbnRlcnN5c2NhbGwu +ZnVuYzEAcnVudGltZS5lbnRlcnN5c2NhbGxfc3lzbW9uAHJ1bnRpbWUuZW50ZXJzeXNjYWxsX2dj +d2FpdABydW50aW1lLmVudGVyc3lzY2FsbGJsb2NrAHJ1bnRpbWUuZW50ZXJzeXNjYWxsYmxvY2su +ZnVuYzIAcnVudGltZS5lbnRlcnN5c2NhbGxibG9jay5mdW5jMQBydW50aW1lLmVudGVyc3lzY2Fs +bGJsb2NrX2hhbmRvZmYAcnVudGltZS5leGl0c3lzY2FsbGZhc3QAcnVudGltZS5leGl0c3lzY2Fs +bGZhc3QuZnVuYzEAcnVudGltZS5leGl0c3lzY2FsbGZhc3RfcmVhY3F1aXJlZABydW50aW1lLmV4 +aXRzeXNjYWxsZmFzdF9yZWFjcXVpcmVkLmZ1bmMxAHJ1bnRpbWUuZXhpdHN5c2NhbGxmYXN0X3Bp +ZGxlAHJ1bnRpbWUuZXhpdHN5c2NhbGwwAHJ1bnRpbWUubWFsZwBydW50aW1lLm1hbGcuZnVuYzEA +cnVudGltZS5uZXdwcm9jAHJ1bnRpbWUubmV3cHJvYy5mdW5jMQBydW50aW1lLm5ld3Byb2MxAHJ1 +bnRpbWUuc2F2ZUFuY2VzdG9ycwBydW50aW1lLmdmcHV0AHJ1bnRpbWUuZ2ZnZXQAcnVudGltZS5n +ZmdldC5mdW5jMQBydW50aW1lLmdmcHVyZ2UAcnVudGltZS51bmxvY2tPU1RocmVhZABydW50aW1l +LmJhZHVubG9ja29zdGhyZWFkAHJ1bnRpbWUuX1N5c3RlbQBydW50aW1lLl9FeHRlcm5hbENvZGUA +cnVudGltZS5fTG9zdEV4dGVybmFsQ29kZQBydW50aW1lLl9HQwBydW50aW1lLl9Mb3N0U0lHUFJP +RkR1cmluZ0F0b21pYzY0AHJ1bnRpbWUuX1ZEU08AcnVudGltZS5zaWdwcm9mAHJ1bnRpbWUuc2ln +cHJvZk5vbkdvAHJ1bnRpbWUuc2lncHJvZk5vbkdvUEMAcnVudGltZS4oKnApLmluaXQAcnVudGlt +ZS4oKnApLmRlc3Ryb3kAcnVudGltZS4oKnApLmRlc3Ryb3kuZnVuYzEAcnVudGltZS5wcm9jcmVz +aXplAHJ1bnRpbWUuYWNxdWlyZXAAcnVudGltZS53aXJlcABydW50aW1lLnJlbGVhc2VwAHJ1bnRp +bWUuaW5jaWRsZWxvY2tlZABydW50aW1lLmNoZWNrZGVhZABydW50aW1lLmNoZWNrZGVhZC5mdW5j +MQBydW50aW1lLnN5c21vbgBydW50aW1lLnJldGFrZQBydW50aW1lLnByZWVtcHRhbGwAcnVudGlt +ZS5wcmVlbXB0b25lAHJ1bnRpbWUuc2NoZWR0cmFjZQBydW50aW1lLnNjaGVkRW5hYmxlVXNlcgBy +dW50aW1lLnNjaGVkRW5hYmxlZABydW50aW1lLm1wdXQAcnVudGltZS5nbG9icnVucWdldABydW50 +aW1lLnVwZGF0ZVRpbWVyUE1hc2sAcnVudGltZS5waWRsZXB1dABydW50aW1lLnBpZGxlZ2V0AHJ1 +bnRpbWUucnVucXB1dABydW50aW1lLnJ1bnFwdXRzbG93AHJ1bnRpbWUucnVucXB1dGJhdGNoAHJ1 +bnRpbWUucnVucWdldABydW50aW1lLnJ1bnFkcmFpbgBydW50aW1lLnJ1bnFncmFiAHJ1bnRpbWUu +cnVucXN0ZWFsAHJ1bnRpbWUuZG9Jbml0AHJ1bnRpbWUuKCpwcm9mQnVmKS5jYW5Xcml0ZVJlY29y +ZABydW50aW1lLigqcHJvZkJ1ZikuY2FuV3JpdGVUd29SZWNvcmRzAHJ1bnRpbWUuKCpwcm9mQnVm +KS53cml0ZQBydW50aW1lLigqcHJvZkJ1Zikud2FrZXVwRXh0cmEAcnVudGltZS5hcmdzAHJ1bnRp +bWUuZ29hcmdzAHJ1bnRpbWUuZ29lbnZzX3VuaXgAcnVudGltZS50ZXN0QXRvbWljNjQAcnVudGlt +ZS5jaGVjawBydW50aW1lLnBhcnNlZGVidWd2YXJzAHJ1bnRpbWUuZXh0ZW5kUmFuZG9tAHJ1bnRp +bWUud2FpdFJlYXNvbi5TdHJpbmcAcnVudGltZS4oKnJ3bXV0ZXgpLnJsb2NrAHJ1bnRpbWUuKCpy +d211dGV4KS5ybG9jay5mdW5jMQBydW50aW1lLigqcndtdXRleCkucnVubG9jawBydW50aW1lLnJl +YWR5V2l0aFRpbWUAcnVudGltZS5zZW1hY3F1aXJlMQBydW50aW1lLnNlbXJlbGVhc2UxAHJ1bnRp +bWUuKCpzZW1hUm9vdCkucXVldWUAcnVudGltZS4oKnNlbWFSb290KS5kZXF1ZXVlAHJ1bnRpbWUu +KCpzZW1hUm9vdCkucm90YXRlTGVmdABydW50aW1lLigqc2VtYVJvb3QpLnJvdGF0ZVJpZ2h0AHJ1 +bnRpbWUuZHVtcHJlZ3MAcnVudGltZS4oKnNpZ2N0eHQpLnByZXBhcmVQYW5pYwBydW50aW1lLmlu +aXRzaWcAcnVudGltZS5kb1NpZ1ByZWVtcHQAcnVudGltZS5zaWdGZXRjaEcAcnVudGltZS5zaWd0 +cmFtcGdvAHJ1bnRpbWUuYWRqdXN0U2lnbmFsU3RhY2sAcnVudGltZS5zaWdoYW5kbGVyAHJ1bnRp +bWUuc2lncGFuaWMAcnVudGltZS5kaWVGcm9tU2lnbmFsAHJ1bnRpbWUucmFpc2ViYWRzaWduYWwA +cnVudGltZS5jcmFzaABydW50aW1lLm5vU2lnbmFsU3RhY2sAcnVudGltZS5zaWdOb3RPblN0YWNr +AHJ1bnRpbWUuc2lnbmFsRHVyaW5nRm9yawBydW50aW1lLmJhZHNpZ25hbABydW50aW1lLnNpZ2Z3 +ZGdvAHJ1bnRpbWUuc2lnYmxvY2sAcnVudGltZS51bmJsb2Nrc2lnAHJ1bnRpbWUubWluaXRTaWdu +YWxzAHJ1bnRpbWUubWluaXRTaWduYWxTdGFjawBydW50aW1lLm1pbml0U2lnbmFsTWFzawBydW50 +aW1lLnVubWluaXRTaWduYWxzAHJ1bnRpbWUuc2lnbmFsc3RhY2sAcnVudGltZS5zaWdzZW5kAHJ1 +bnRpbWUubWFrZXNsaWNlY29weQBydW50aW1lLm1ha2VzbGljZQBydW50aW1lLmdyb3dzbGljZQBy +dW50aW1lLnN0YWNraW5pdABydW50aW1lLnN0YWNrcG9vbGFsbG9jAHJ1bnRpbWUuc3RhY2twb29s +ZnJlZQBydW50aW1lLnN0YWNrY2FjaGVyZWZpbGwAcnVudGltZS5zdGFja2NhY2hlcmVsZWFzZQBy +dW50aW1lLnN0YWNrY2FjaGVfY2xlYXIAcnVudGltZS5zdGFja2FsbG9jAHJ1bnRpbWUuc3RhY2tm +cmVlAHJ1bnRpbWUuYWRqdXN0cG9pbnRlcnMAcnVudGltZS5hZGp1c3RmcmFtZQBydW50aW1lLmFk +anVzdGRlZmVycwBydW50aW1lLnN5bmNhZGp1c3RzdWRvZ3MAcnVudGltZS5jb3B5c3RhY2sAcnVu +dGltZS5uZXdzdGFjawBydW50aW1lLnNocmlua3N0YWNrAHJ1bnRpbWUuZnJlZVN0YWNrU3BhbnMA +cnVudGltZS5nZXRTdGFja01hcABydW50aW1lLmluaXQuOQBydW50aW1lLmNvbmNhdHN0cmluZ3MA +cnVudGltZS5jb25jYXRzdHJpbmcyAHJ1bnRpbWUuY29uY2F0c3RyaW5nNABydW50aW1lLnNsaWNl +Ynl0ZXRvc3RyaW5nAHJ1bnRpbWUucmF3c3RyaW5ndG1wAHJ1bnRpbWUucmF3c3RyaW5nAHJ1bnRp +bWUuYXRvaQBydW50aW1lLmZpbmRudWxsAHJ1bnRpbWUuYmFkc3lzdGVtc3RhY2sAcnVudGltZS5m +YXN0cmFuZABydW50aW1lLm1vZHVsZXNpbml0AHJ1bnRpbWUubW9kdWxlZGF0YXZlcmlmeTEAcnVu +dGltZS5maW5kZnVuYwBydW50aW1lLnBjdmFsdWUAcnVudGltZS5mdW5jbmFtZQBydW50aW1lLmZ1 +bmNwa2dwYXRoAHJ1bnRpbWUuZnVuY25hbWVGcm9tTmFtZW9mZgBydW50aW1lLmZ1bmNmaWxlAHJ1 +bnRpbWUuZnVuY2xpbmUxAHJ1bnRpbWUuZnVuY2xpbmUAcnVudGltZS5mdW5jc3BkZWx0YQBydW50 +aW1lLmZ1bmNNYXhTUERlbHRhAHJ1bnRpbWUucGNkYXRhdmFsdWUAcnVudGltZS5wY2RhdGF2YWx1 +ZTIAcnVudGltZS5zdGVwAHJ1bnRpbWUuZG9hZGR0aW1lcgBydW50aW1lLmRlbHRpbWVyAHJ1bnRp +bWUuZG9kZWx0aW1lcgBydW50aW1lLmRvZGVsdGltZXIwAHJ1bnRpbWUubW9kdGltZXIAcnVudGlt +ZS5tb3ZlVGltZXJzAHJ1bnRpbWUuYWRqdXN0dGltZXJzAHJ1bnRpbWUuYWRkQWRqdXN0ZWRUaW1l +cnMAcnVudGltZS5ydW50aW1lcgBydW50aW1lLnJ1bk9uZVRpbWVyAHJ1bnRpbWUuY2xlYXJEZWxl +dGVkVGltZXJzAHJ1bnRpbWUudGltZVNsZWVwVW50aWwAcnVudGltZS5zaWZ0dXBUaW1lcgBydW50 +aW1lLnNpZnRkb3duVGltZXIAcnVudGltZS5iYWRUaW1lcgBydW50aW1lLnRyYWNlUmVhZGVyAHJ1 +bnRpbWUudHJhY2VQcm9jRnJlZQBydW50aW1lLnRyYWNlRXZlbnQAcnVudGltZS50cmFjZUV2ZW50 +TG9ja2VkAHJ1bnRpbWUudHJhY2VTdGFja0lEAHJ1bnRpbWUudHJhY2VBY3F1aXJlQnVmZmVyAHJ1 +bnRpbWUudHJhY2VSZWxlYXNlQnVmZmVyAHJ1bnRpbWUudHJhY2VGbHVzaABydW50aW1lLigqdHJh +Y2VTdGFja1RhYmxlKS5wdXQAcnVudGltZS4oKnRyYWNlU3RhY2tUYWJsZSkuZmluZABydW50aW1l +LigqdHJhY2VTdGFja1RhYmxlKS5uZXdTdGFjawBydW50aW1lLigqdHJhY2VBbGxvYykuYWxsb2MA +cnVudGltZS50cmFjZVByb2NTdGFydABydW50aW1lLnRyYWNlUHJvY1N0b3AAcnVudGltZS50cmFj +ZUdDU3dlZXBTdGFydABydW50aW1lLnRyYWNlR0NTd2VlcFNwYW4AcnVudGltZS50cmFjZUdDU3dl +ZXBEb25lAHJ1bnRpbWUudHJhY2VHb0NyZWF0ZQBydW50aW1lLnRyYWNlR29TdGFydABydW50aW1l +LnRyYWNlR29TY2hlZABydW50aW1lLnRyYWNlR29QYXJrAHJ1bnRpbWUudHJhY2VHb1VucGFyawBy +dW50aW1lLnRyYWNlR29TeXNDYWxsAHJ1bnRpbWUudHJhY2VHb1N5c0V4aXQAcnVudGltZS50cmFj +ZUdvU3lzQmxvY2sAcnVudGltZS50cmFjZUhlYXBHb2FsAHJ1bnRpbWUudHJhY2ViYWNrZGVmZXJz +AHJ1bnRpbWUuZ2VudHJhY2ViYWNrAHJ1bnRpbWUucHJpbnRBcmdzAHJ1bnRpbWUucHJpbnRBcmdz +LmZ1bmMxAHJ1bnRpbWUuZ2V0QXJnSW5mbwBydW50aW1lLnRyYWNlYmFja0Nnb0NvbnRleHQAcnVu +dGltZS5wcmludGNyZWF0ZWRieQBydW50aW1lLnByaW50Y3JlYXRlZGJ5MQBydW50aW1lLnRyYWNl +YmFjawBydW50aW1lLnRyYWNlYmFja3RyYXAAcnVudGltZS50cmFjZWJhY2sxAHJ1bnRpbWUucHJp +bnRBbmNlc3RvclRyYWNlYmFjawBydW50aW1lLnByaW50QW5jZXN0b3JUcmFjZWJhY2tGdW5jSW5m +bwBydW50aW1lLmNhbGxlcnMAcnVudGltZS5jYWxsZXJzLmZ1bmMxAHJ1bnRpbWUuZ2NhbGxlcnMA +cnVudGltZS5zaG93ZnJhbWUAcnVudGltZS5zaG93ZnVuY2luZm8AcnVudGltZS5nb3JvdXRpbmVo +ZWFkZXIAcnVudGltZS50cmFjZWJhY2tvdGhlcnMAcnVudGltZS50cmFjZWJhY2tvdGhlcnMuZnVu +YzEAcnVudGltZS50cmFjZWJhY2tIZXhkdW1wAHJ1bnRpbWUudHJhY2ViYWNrSGV4ZHVtcC5mdW5j +MQBydW50aW1lLmlzU3lzdGVtR29yb3V0aW5lAHJ1bnRpbWUucHJpbnRDZ29UcmFjZWJhY2sAcnVu +dGltZS5wcmludE9uZUNnb1RyYWNlYmFjawBydW50aW1lLmNhbGxDZ29TeW1ib2xpemVyAHJ1bnRp +bWUuY2dvQ29udGV4dFBDcwBydW50aW1lLigqX3R5cGUpLnN0cmluZwBydW50aW1lLigqX3R5cGUp +LnVuY29tbW9uAHJ1bnRpbWUuKCpfdHlwZSkucGtncGF0aABydW50aW1lLnJlc29sdmVOYW1lT2Zm +AHJ1bnRpbWUucmVzb2x2ZVR5cGVPZmYAcnVudGltZS4oKl90eXBlKS50ZXh0T2ZmAHJ1bnRpbWUu +bmFtZS5uYW1lAHJ1bnRpbWUubmFtZS50YWcAcnVudGltZS5uYW1lLnBrZ1BhdGgAcnVudGltZS50 +eXBlbGlua3Npbml0AHJ1bnRpbWUudHlwZXNFcXVhbABydW50aW1lLnZkc29Jbml0RnJvbVN5c2lu +Zm9FaGRyAHJ1bnRpbWUudmRzb0ZpbmRWZXJzaW9uAHJ1bnRpbWUudmRzb1BhcnNlU3ltYm9scwBy +dW50aW1lLnZkc29QYXJzZVN5bWJvbHMuZnVuYzEAcnVudGltZS52ZHNvYXV4dgBydW50aW1lLmlu +VkRTT1BhZ2UAcnVudGltZS5kZWJ1Z0NhbGxXcmFwLmZ1bmMyAHJ1bnRpbWUuZGVidWdDYWxsV3Jh +cDEuZnVuYzEAcnVudGltZS5nY1N0YXJ0LmZ1bmMxAHJ1bnRpbWUuZ2NNYXJrRG9uZS5mdW5jMS4x +AHJ1bnRpbWUuZ2NNYXJrRG9uZS5mdW5jMQBydW50aW1lLmdjTWFya0RvbmUuZnVuYzMAcnVudGlt +ZS5nY01hcmtUZXJtaW5hdGlvbi5mdW5jMgBydW50aW1lLmdjTWFya1Rlcm1pbmF0aW9uLmZ1bmMz +AHJ1bnRpbWUuZ2NNYXJrVGVybWluYXRpb24uZnVuYzQuMQBydW50aW1lLmdjTWFya1Rlcm1pbmF0 +aW9uLmZ1bmM0AHJ1bnRpbWUuZ2NCZ01hcmtXb3JrZXIuZnVuYzEAcnVudGltZS5nY1Jlc2V0TWFy +a1N0YXRlLmZ1bmMxAHJ1bnRpbWUuYmdzY2F2ZW5nZS5mdW5jMQBydW50aW1lLnN3ZWVwb25lLmZ1 +bmMxAHJ1bnRpbWUuKCpwYWdlQWxsb2MpLnN5c0dyb3cuZnVuYzEAcnVudGltZS53YkJ1ZkZsdXNo +LmZ1bmMxAHJ1bnRpbWUuc3lzU2lnYWN0aW9uLmZ1bmMxAHJ1bnRpbWUucHJlcHJpbnRwYW5pY3Mu +ZnVuYzEAcnVudGltZS5mYXRhbHBhbmljLmZ1bmMyAHJ1bnRpbWUubWFpbi5mdW5jMQBydW50aW1l +LnNjaGVkdHJhY2UuZnVuYzEAcnVudGltZS5pbml0AHN5bmMuZXZlbnQAcnVudGltZS5lbnRlcnN5 +c2NhbGwAcnVudGltZS5leGl0c3lzY2FsbABydW50aW1lL2RlYnVnLlNldFRyYWNlYmFjawBydW50 +aW1lLm1vcmVzdGFja2MAcnVudGltZS5nb3N0cmluZwBfcnQwX2FtZDY0AHJ1bnRpbWUucnQwX2dv +LmFiaTAAcnVudGltZS5hc21pbml0LmFiaTAAcnVudGltZS5tc3RhcnQuYWJpMABydW50aW1lLmdv +Z28uYWJpMABydW50aW1lLm1jYWxsAHJ1bnRpbWUuc3lzdGVtc3RhY2tfc3dpdGNoLmFiaTAAcnVu +dGltZS5zeXN0ZW1zdGFjay5hYmkwAHJ1bnRpbWUubW9yZXN0YWNrLmFiaTAAcnVudGltZS5tb3Jl +c3RhY2tfbm9jdHh0LmFiaTAAcnVudGltZS5wcm9jeWllbGQuYWJpMABydW50aW1lLnB1YmxpY2F0 +aW9uQmFycmllci5hYmkwAHJ1bnRpbWUuam1wZGVmZXIuYWJpMABydW50aW1lLmFzbWNnb2NhbGwu +YWJpMABydW50aW1lLnNldGcuYWJpMABydW50aW1lLmFib3J0LmFiaTAAcnVudGltZS5zdGFja2No +ZWNrLmFiaTAAcnVudGltZS5jcHV0aWNrcy5hYmkwAHJ1bnRpbWUubWVtaGFzaABydW50aW1lLm1l +bWhhc2gzMgBydW50aW1lLm1lbWhhc2g2NABydW50aW1lLmNoZWNrQVNNLmFiaTAAcnVudGltZS5n +b2V4aXQuYWJpMABydW50aW1lLnNpZ3BhbmljMABydW50aW1lLmdjV3JpdGVCYXJyaWVyAHJ1bnRp +bWUuZ2NXcml0ZUJhcnJpZXJDWABydW50aW1lLmdjV3JpdGVCYXJyaWVyRFgAcnVudGltZS5nY1dy +aXRlQmFycmllckJYAHJ1bnRpbWUuZ2NXcml0ZUJhcnJpZXJTSQBydW50aW1lLmdjV3JpdGVCYXJy +aWVyUjgAcnVudGltZS5nY1dyaXRlQmFycmllclI5AHJ1bnRpbWUuZGVidWdDYWxsVjIAcnVudGlt +ZS5kZWJ1Z0NhbGxQYW5pY2tlZC5hYmkwAHJ1bnRpbWUucGFuaWNJbmRleABydW50aW1lLnBhbmlj +SW5kZXhVAHJ1bnRpbWUucGFuaWNTbGljZUFsZW4AcnVudGltZS5wYW5pY1NsaWNlQWxlblUAcnVu +dGltZS5wYW5pY1NsaWNlQWNhcABydW50aW1lLnBhbmljU2xpY2VBY2FwVQBydW50aW1lLnBhbmlj +U2xpY2VCAHJ1bnRpbWUucGFuaWNTbGljZUJVAHJ1bnRpbWUucGFuaWNTbGljZTNBbGVuAHJ1bnRp +bWUucGFuaWNTbGljZTNBbGVuVQBydW50aW1lLmR1ZmZ6ZXJvAHJ1bnRpbWUuZHVmZmNvcHkAcnVu +dGltZS5tZW1jbHJOb0hlYXBQb2ludGVycwBydW50aW1lLm1lbW1vdmUAcnVudGltZS5hc3luY1By +ZWVtcHQAX3J0MF9hbWQ2NF9saW51eABydW50aW1lLmV4aXQuYWJpMABydW50aW1lLmV4aXRUaHJl +YWQuYWJpMABydW50aW1lLm9wZW4uYWJpMABydW50aW1lLmNsb3NlZmQuYWJpMABydW50aW1lLndy +aXRlMS5hYmkwAHJ1bnRpbWUucmVhZC5hYmkwAHJ1bnRpbWUucGlwZS5hYmkwAHJ1bnRpbWUucGlw +ZTIuYWJpMABydW50aW1lLnVzbGVlcC5hYmkwAHJ1bnRpbWUuZ2V0dGlkLmFiaTAAcnVudGltZS5y +YWlzZS5hYmkwAHJ1bnRpbWUucmFpc2Vwcm9jLmFiaTAAcnVudGltZS5nZXRwaWQuYWJpMABydW50 +aW1lLnRna2lsbC5hYmkwAHJ1bnRpbWUubWluY29yZS5hYmkwAHJ1bnRpbWUubmFub3RpbWUxLmFi +aTAAcnVudGltZS5ydHNpZ3Byb2NtYXNrLmFiaTAAcnVudGltZS5ydF9zaWdhY3Rpb24uYWJpMABy +dW50aW1lLmNhbGxDZ29TaWdhY3Rpb24uYWJpMABydW50aW1lLnNpZ2Z3ZC5hYmkwAHJ1bnRpbWUu +c2lndHJhbXAAcnVudGltZS5jZ29TaWd0cmFtcABydW50aW1lLnNpZ3JldHVybgBydW50aW1lLnN5 +c01tYXAuYWJpMABydW50aW1lLmNhbGxDZ29NbWFwLmFiaTAAcnVudGltZS5zeXNNdW5tYXAuYWJp +MABydW50aW1lLmNhbGxDZ29NdW5tYXAuYWJpMABydW50aW1lLm1hZHZpc2UuYWJpMABydW50aW1l +LmZ1dGV4LmFiaTAAcnVudGltZS5jbG9uZS5hYmkwAHJ1bnRpbWUuc2lnYWx0c3RhY2suYWJpMABy +dW50aW1lLnNldHRscy5hYmkwAHJ1bnRpbWUub3N5aWVsZC5hYmkwAHJ1bnRpbWUuc2NoZWRfZ2V0 +YWZmaW5pdHkuYWJpMABydW50aW1lLmVwb2xsY3JlYXRlLmFiaTAAcnVudGltZS5lcG9sbGNyZWF0 +ZTEuYWJpMABydW50aW1lLmVwb2xsY3RsLmFiaTAAcnVudGltZS5lcG9sbHdhaXQuYWJpMABydW50 +aW1lLmNsb3Nlb25leGVjLmFiaTAAcnVudGltZS5zZXROb25ibG9jay5hYmkwAHRpbWUubm93LmFi +aTAAcnVudGltZS4oKml0YWJUYWJsZVR5cGUpLmFkZC1mbQBydW50aW1lLmRlYnVnQ2FsbENoZWNr +LmFiaTAAcnVudGltZS5kZWJ1Z0NhbGxXcmFwLmFiaTAAcnVudGltZS53YkJ1ZkZsdXNoLmFiaTAA +cnVudGltZS5vc2luaXQuYWJpMABydW50aW1lLmFzeW5jUHJlZW1wdDIuYWJpMABydW50aW1lLmJh +ZG1jYWxsLmFiaTAAcnVudGltZS5iYWRtY2FsbDIuYWJpMABydW50aW1lLmJhZG1vcmVzdGFja2cw +LmFiaTAAcnVudGltZS5iYWRtb3Jlc3RhY2tnc2lnbmFsLmFiaTAAcnVudGltZS5zY2hlZGluaXQu +YWJpMABydW50aW1lLm1zdGFydABydW50aW1lLm1zdGFydDAuYWJpMABydW50aW1lLmdvZXhpdDEu +YWJpMABydW50aW1lLm5ld3Byb2MuYWJpMABydW50aW1lLnNpZ3Byb2ZOb25Hby5hYmkwAHJ1bnRp +bWUuYXJncy5hYmkwAHJ1bnRpbWUuY2hlY2suYWJpMABydW50aW1lLnNpZ3RyYW1wZ28uYWJpMABy +dW50aW1lLm5ld3N0YWNrLmFiaTAAcnVudGltZS5tb3Jlc3RhY2tjLmFiaTAAcnVudGltZS5iYWRz +eXN0ZW1zdGFjay5hYmkwAHJ1bnRpbWUuYXNtY2dvY2FsbABydW50aW1lLigqZXJyb3JTdHJpbmcp +LkVycm9yAHR5cGUuLmVxLnJ1bnRpbWUuaXRhYgB0eXBlLi5lcS5ydW50aW1lLm1vZHVsZWhhc2gA +dHlwZS4uZXEucnVudGltZS5iaXR2ZWN0b3IAdHlwZS4uZXEucnVudGltZS5UeXBlQXNzZXJ0aW9u +RXJyb3IAdHlwZS4uZXEucnVudGltZS5fcGFuaWMAdHlwZS4uZXEucnVudGltZS5fZGVmZXIAdHlw +ZS4uZXEucnVudGltZS5ib3VuZHNFcnJvcgBydW50aW1lLigqYm91bmRzRXJyb3IpLkVycm9yAHR5 +cGUuLmVxLnJ1bnRpbWUuc3lzbW9udGljawB0eXBlLi5lcS5ydW50aW1lLnNwZWNpYWwAdHlwZS4u +ZXEucnVudGltZS5tc3BhbgB0eXBlLi5lcS5ydW50aW1lLm1jYWNoZQB0eXBlLi5lcS5zdHJ1Y3Qg +eyBydW50aW1lLmdMaXN0OyBydW50aW1lLm4gaW50MzIgfQB0eXBlLi5lcS5ydW50aW1lLmhjaGFu +AHR5cGUuLmVxLnJ1bnRpbWUuc3Vkb2cAdHlwZS4uZXEucnVudGltZS5nY1dvcmsAcnVudGltZS4o +KmxvY2tSYW5rKS5TdHJpbmcAcnVudGltZS4oKndhaXRSZWFzb24pLlN0cmluZwB0eXBlLi5lcS5y +dW50aW1lLmVycm9yQWRkcmVzc1N0cmluZwBydW50aW1lLigqZXJyb3JBZGRyZXNzU3RyaW5nKS5F +cnJvcgBydW50aW1lLigqcGxhaW5FcnJvcikuRXJyb3IAbWFpbi5tYWluAG1haW4uLmluaXR0YXNr +AHJ1bnRpbWUuLmluaXR0YXNrAHJ1bnRpbWUudXNlQWVzaGFzaABydW50aW1lLmFlc2tleXNjaGVk +AHJ1bnRpbWUuaGFzaGtleQBydW50aW1lLmlzY2dvAHJ1bnRpbWUuY2dvSGFzRXh0cmFNAHJ1bnRp +bWUuY2dvX3lpZWxkAHJ1bnRpbWUubmNnb2NhbGwAcnVudGltZS54ODZIYXNQT1BDTlQAcnVudGlt +ZS54ODZIYXNTU0U0MQBydW50aW1lLng4Nkhhc0ZNQQBydW50aW1lLmFybUhhc1ZGUHY0AHJ1bnRp +bWUuYXJtNjRIYXNBVE9NSUNTAHJ1bnRpbWUudXNlQVZYbWVtbW92ZQBydW50aW1lLmNwdXByb2YA +cnVudGltZS5fY2dvX3NldGVudgBydW50aW1lLl9jZ29fdW5zZXRlbnYAcnVudGltZS5ib3VuZHNF +cnJvckZtdHMAcnVudGltZS5ib3VuZHNOZWdFcnJvckZtdHMAcnVudGltZS5idWlsZFZlcnNpb24A +cnVudGltZS5mYXN0bG9nMlRhYmxlAHJ1bnRpbWUuaW5mAHJ1bnRpbWUuaXRhYkxvY2sAcnVudGlt +ZS5pdGFiVGFibGUAcnVudGltZS5pdGFiVGFibGVJbml0AHJ1bnRpbWUudWludDE2RWZhY2UAcnVu +dGltZS51aW50MzJFZmFjZQBydW50aW1lLnVpbnQ2NEVmYWNlAHJ1bnRpbWUuc3RyaW5nRWZhY2UA +cnVudGltZS5zbGljZUVmYWNlAHJ1bnRpbWUudWludDE2VHlwZQBydW50aW1lLnVpbnQzMlR5cGUA +cnVudGltZS51aW50NjRUeXBlAHJ1bnRpbWUuc3RyaW5nVHlwZQBydW50aW1lLnNsaWNlVHlwZQBy +dW50aW1lLnN0YXRpY3VpbnQ2NHMAcnVudGltZS5sb2NrTmFtZXMAcnVudGltZS5waHlzUGFnZVNp +emUAcnVudGltZS5waHlzSHVnZVBhZ2VTaXplAHJ1bnRpbWUucGh5c0h1Z2VQYWdlU2hpZnQAcnVu +dGltZS56ZXJvYmFzZQBydW50aW1lLmdsb2JhbEFsbG9jAHJ1bnRpbWUucGVyc2lzdGVudENodW5r +cwBydW50aW1lLnplcm9WYWwAcnVudGltZS5lbXB0eW1zcGFuAHJ1bnRpbWUudXNlQ2hlY2ttYXJr +AHJ1bnRpbWUuYWR2aXNlVW51c2VkAHJ1bnRpbWUuZmlubG9jawBydW50aW1lLmZpbmcAcnVudGlt +ZS5maW5xAHJ1bnRpbWUuZmluYwBydW50aW1lLmZpbnB0cm1hc2sAcnVudGltZS5maW5nd2FpdABy +dW50aW1lLmZpbmd3YWtlAHJ1bnRpbWUuYWxsZmluAHJ1bnRpbWUuZmluYWxpemVyMQBydW50aW1l +LmZpbmdSdW5uaW5nAHJ1bnRpbWUuZ2NlbmFibGVfc2V0dXAAcnVudGltZS5nY3BoYXNlAHJ1bnRp +bWUud3JpdGVCYXJyaWVyAHJ1bnRpbWUuZ2NCbGFja2VuRW5hYmxlZABydW50aW1lLndvcmsAcnVu +dGltZS5nY01hcmtEb25lRmx1c2hlZABydW50aW1lLnBvb2xjbGVhbnVwAHJ1bnRpbWUub25lcHRy +bWFzawBydW50aW1lLmdjQ29udHJvbGxlcgBydW50aW1lLnNjYXZlbmdlAHJ1bnRpbWUuc3dlZXAA +cnVudGltZS5taGVhcF8AcnVudGltZS5tU3BhblN0YXRlTmFtZXMAcnVudGltZS5nY0JpdHNBcmVu +YXMAcnVudGltZS5tYXhTZWFyY2hBZGRyAHJ1bnRpbWUubGV2ZWxCaXRzAHJ1bnRpbWUubGV2ZWxT +aGlmdABydW50aW1lLmxldmVsTG9nUGFnZXMAcnVudGltZS5wcm9mbG9jawBydW50aW1lLm1idWNr +ZXRzAHJ1bnRpbWUuYmJ1Y2tldHMAcnVudGltZS54YnVja2V0cwBydW50aW1lLmJ1Y2toYXNoAHJ1 +bnRpbWUuYnVja2V0bWVtAHJ1bnRpbWUubVByb2YAcnVudGltZS5ibG9ja3Byb2ZpbGVyYXRlAHJ1 +bnRpbWUubXV0ZXhwcm9maWxlcmF0ZQBydW50aW1lLk1lbVByb2ZpbGVSYXRlAHJ1bnRpbWUuZGlz +YWJsZU1lbW9yeVByb2ZpbGluZwBydW50aW1lLnRyYWNlbG9jawBydW50aW1lLm1pbk9mZkFkZHIA +cnVudGltZS5tYXhPZmZBZGRyAHJ1bnRpbWUuc3BhblNldEJsb2NrUG9vbABydW50aW1lLm1lbXN0 +YXRzAHJ1bnRpbWUubmV0cG9sbEluaXRMb2NrAHJ1bnRpbWUubmV0cG9sbEluaXRlZABydW50aW1l +Lm5ldHBvbGxXYWl0ZXJzAHJ1bnRpbWUucGRFZmFjZQBydW50aW1lLnBkVHlwZQBydW50aW1lLmVw +ZmQAcnVudGltZS5uZXRwb2xsQnJlYWtSZABydW50aW1lLm5ldHBvbGxCcmVha1dyAHJ1bnRpbWUu +bmV0cG9sbFdha2VTaWcAcnVudGltZS5wcm9jQXV4dgBydW50aW1lLmFkZHJzcGFjZV92ZWMAcnVu +dGltZS5zdGFydHVwUmFuZG9tRGF0YQBydW50aW1lLnN5c1RIUFNpemVQYXRoAHJ1bnRpbWUudXJh +bmRvbV9kZXYAcnVudGltZS5zaWdzZXRfYWxsAHJ1bnRpbWUuc2hpZnRFcnJvcgBydW50aW1lLmRp +dmlkZUVycm9yAHJ1bnRpbWUub3ZlcmZsb3dFcnJvcgBydW50aW1lLmZsb2F0RXJyb3IAcnVudGlt +ZS5tZW1vcnlFcnJvcgBydW50aW1lLmRlZmVyVHlwZQBydW50aW1lLnJ1bm5pbmdQYW5pY0RlZmVy +cwBydW50aW1lLnBhbmlja2luZwBydW50aW1lLnBhbmljbGsAcnVudGltZS5kaWRvdGhlcnMAcnVu +dGltZS5kZWFkbG9jawBydW50aW1lLmFzeW5jUHJlZW1wdFN0YWNrAHJ1bnRpbWUubm9fcG9pbnRl +cnNfc3RhY2ttYXAAcnVudGltZS5wcmludEJhY2tsb2cAcnVudGltZS5wcmludEJhY2tsb2dJbmRl +eABydW50aW1lLmRlYnVnbG9jawBydW50aW1lLm1pbmhleGRpZ2l0cwBydW50aW1lLm1vZGluZm8A +cnVudGltZS5tMABydW50aW1lLmcwAHJ1bnRpbWUubWNhY2hlMABydW50aW1lLm1haW5faW5pdF9k +b25lAHJ1bnRpbWUubWFpblN0YXJ0ZWQAcnVudGltZS5ydW50aW1lSW5pdFRpbWUAcnVudGltZS5p +bml0U2lnbWFzawBydW50aW1lLmJhZG1vcmVzdGFja2cwTXNnAHJ1bnRpbWUuYmFkbW9yZXN0YWNr +Z3NpZ25hbE1zZwBydW50aW1lLmFsbGdsb2NrAHJ1bnRpbWUuYWxsZ3MAcnVudGltZS5hbGxnbGVu +AHJ1bnRpbWUuYWxsZ3B0cgBydW50aW1lLmZhc3RyYW5kc2VlZABydW50aW1lLmZyZWV6aW5nAHJ1 +bnRpbWUud29ybGRzZW1hAHJ1bnRpbWUuZ2NzZW1hAHJ1bnRpbWUuZWFybHljZ29jYWxsYmFjawBy +dW50aW1lLmV4dHJhbQBydW50aW1lLmV4dHJhTUNvdW50AHJ1bnRpbWUuZXh0cmFNV2FpdGVycwBy +dW50aW1lLmV4ZWNMb2NrAHJ1bnRpbWUubmV3bUhhbmRvZmYAcnVudGltZS5pbkZvcmtlZENoaWxk +AHJ1bnRpbWUucHJvZgBydW50aW1lLnNpZ3Byb2ZDYWxsZXJzAHJ1bnRpbWUuc2lncHJvZkNhbGxl +cnNVc2UAcnVudGltZS5mb3JjZWdjcGVyaW9kAHJ1bnRpbWUuc3RhcnR0aW1lAHJ1bnRpbWUuc3Rl +YWxPcmRlcgBydW50aW1lLmluaXR0cmFjZQBydW50aW1lLmVudnMAcnVudGltZS5hcmdzbGljZQBy +dW50aW1lLnRyYWNlYmFja19jYWNoZQBydW50aW1lLnRyYWNlYmFja19lbnYAcnVudGltZS5hcmdj +AHJ1bnRpbWUuYXJndgBydW50aW1lLnRlc3RfejY0AHJ1bnRpbWUudGVzdF94NjQAcnVudGltZS5k +ZWJ1ZwBydW50aW1lLmRiZ3ZhcnMAcnVudGltZS53YWl0UmVhc29uU3RyaW5ncwBydW50aW1lLmFs +bG0AcnVudGltZS5nb21heHByb2NzAHJ1bnRpbWUubmNwdQBydW50aW1lLmZvcmNlZ2MAcnVudGlt +ZS5zY2hlZABydW50aW1lLm5ld3Byb2NzAHJ1bnRpbWUuYWxscExvY2sAcnVudGltZS5hbGxwAHJ1 +bnRpbWUuaWRsZXBNYXNrAHJ1bnRpbWUudGltZXJwTWFzawBydW50aW1lLmdjQmdNYXJrV29ya2Vy +UG9vbABydW50aW1lLmdjQmdNYXJrV29ya2VyQ291bnQAcnVudGltZS5wcm9jZXNzb3JWZXJzaW9u +SW5mbwBydW50aW1lLmlzSW50ZWwAcnVudGltZS5sZmVuY2VCZWZvcmVSZHRzYwBydW50aW1lLmlz +bGlicmFyeQBydW50aW1lLmlzYXJjaGl2ZQBydW50aW1lLmNoYW5zZW5kcGMAcnVudGltZS5jaGFu +cmVjdnBjAHJ1bnRpbWUuc2VtdGFibGUAcnVudGltZS5md2RTaWcAcnVudGltZS5oYW5kbGluZ1Np +ZwBydW50aW1lLnNpZ25hbHNPSwBydW50aW1lLmNyYXNoaW5nAHJ1bnRpbWUudGVzdFNpZ3RyYXAA +cnVudGltZS50ZXN0U2lndXNyMQBydW50aW1lLmJhZGdpbnNpZ25hbE1zZwBydW50aW1lLnNpZ3Nl +dEFsbEV4aXRpbmcAcnVudGltZS5zaWcAcnVudGltZS5zaWd0YWJsZQBydW50aW1lLmNsYXNzX3Rv +X3NpemUAcnVudGltZS5jbGFzc190b19hbGxvY25wYWdlcwBydW50aW1lLmNsYXNzX3RvX2Rpdm1h +Z2ljAHJ1bnRpbWUuc2l6ZV90b19jbGFzczgAcnVudGltZS5zaXplX3RvX2NsYXNzMTI4AHJ1bnRp +bWUuc3RhY2twb29sAHJ1bnRpbWUuc3RhY2tMYXJnZQBydW50aW1lLm1heHN0YWNrc2l6ZQBydW50 +aW1lLm1heHN0YWNrY2VpbGluZwBydW50aW1lLmFiaVJlZ0FyZ3NFZmFjZQBydW50aW1lLmFiaVJl +Z0FyZ3NUeXBlAHJ1bnRpbWUubWV0aG9kVmFsdWVDYWxsRnJhbWVPYmpzAHJ1bnRpbWUuYmFkc3lz +dGVtc3RhY2tNc2cAcnVudGltZS5waW5uZWRUeXBlbWFwcwBydW50aW1lLmZpcnN0bW9kdWxlZGF0 +YQBydW50aW1lLmxhc3Rtb2R1bGVkYXRhcABydW50aW1lLm1vZHVsZXNTbGljZQBydW50aW1lLmZh +a2V0aW1lAHJ1bnRpbWUudHJhY2UAcnVudGltZS5nU3RhdHVzU3RyaW5ncwBydW50aW1lLmNnb1Ry +YWNlYmFjawBydW50aW1lLmNnb1N5bWJvbGl6ZXIAcnVudGltZS5yZWZsZWN0T2ZmcwBydW50aW1l +LnZkc29MaW51eFZlcnNpb24AcnVudGltZS52ZHNvU3ltYm9sS2V5cwBydW50aW1lLnZkc29HZXR0 +aW1lb2ZkYXlTeW0AcnVudGltZS52ZHNvQ2xvY2tnZXR0aW1lU3ltAGludGVybmFsL2NwdS5EZWJ1 +Z09wdGlvbnMAaW50ZXJuYWwvY3B1LkNhY2hlTGluZVNpemUAaW50ZXJuYWwvY3B1Llg4NgBpbnRl +cm5hbC9jcHUuQVJNAGludGVybmFsL2NwdS5BUk02NABpbnRlcm5hbC9jcHUub3B0aW9ucwBpbnRl +cm5hbC9jcHUubWF4RXh0ZW5kZWRGdW5jdGlvbkluZm9ybWF0aW9uAGludGVybmFsL2J5dGVhbGcu +LmluaXR0YXNrAGludGVybmFsL2J5dGVhbGcuTWF4TGVuAGdvLml0YWIucnVudGltZS5lcnJvclN0 +cmluZyxlcnJvcgBfY2dvX2luaXQAX2Nnb190aHJlYWRfc3RhcnQAX2Nnb19ub3RpZnlfcnVudGlt +ZV9pbml0X2RvbmUAX2Nnb19jYWxsZXJzAF9jZ29feWllbGQAX2Nnb19tbWFwAF9jZ29fbXVubWFw +AF9jZ29fc2lnYWN0aW9uAHJ1bnRpbWUubWFpblBDAHJ1bnRpbWUuYnVpbGRWZXJzaW9uLnN0cgB0 +eXBlLioAcnVudGltZS50ZXh0c2VjdGlvbm1hcAA= diff --git a/src/debug/buildinfo/testdata/go117/main.go b/src/debug/buildinfo/testdata/go117/main.go new file mode 100644 index 00000000000000..78cd9120d5cc5f --- /dev/null +++ b/src/debug/buildinfo/testdata/go117/main.go @@ -0,0 +1,7 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func main() {} diff --git a/src/debug/buildinfo/testdata/notgo b/src/debug/buildinfo/testdata/notgo deleted file mode 100755 index bc19ec2d3eb27d..00000000000000 Binary files a/src/debug/buildinfo/testdata/notgo and /dev/null differ diff --git a/src/debug/buildinfo/testdata/notgo/README.md b/src/debug/buildinfo/testdata/notgo/README.md new file mode 100644 index 00000000000000..ebbe104beda0b3 --- /dev/null +++ b/src/debug/buildinfo/testdata/notgo/README.md @@ -0,0 +1,17 @@ +notgo.base64 is a base64-encoded C hello world binary used to test +debug/buildinfo errors on non-Go binaries. + +The binary is base64 encoded to hide it from security scanners that might not +like it. + +Generate notgo.base64 on linux-amd64 with: + +$ cc -o notgo main.c +$ base64 notgo > notgo.base64 +$ rm notgo + +The current binary was built with "gcc version 14.2.0 (Debian 14.2.0-3+build4)". + +TODO(prattmic): Ideally this would be built on the fly to better cover all +executable formats, but then we need to encode the intricacies of calling each +platform's C compiler. diff --git a/src/debug/buildinfo/testdata/notgo/main.c b/src/debug/buildinfo/testdata/notgo/main.c new file mode 100644 index 00000000000000..d554d4a4ce3187 --- /dev/null +++ b/src/debug/buildinfo/testdata/notgo/main.c @@ -0,0 +1,7 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +int main(void) { + return 0; +} diff --git a/src/debug/buildinfo/testdata/notgo/notgo.base64 b/src/debug/buildinfo/testdata/notgo/notgo.base64 new file mode 100644 index 00000000000000..74a7fe1d02e988 --- /dev/null +++ b/src/debug/buildinfo/testdata/notgo/notgo.base64 @@ -0,0 +1,278 @@ +f0VMRgIBAQAAAAAAAAAAAAMAPgABAAAAQBAAAAAAAABAAAAAAAAAAGA2AAAAAAAAAAAAAEAAOAAN +AEAAHgAdAAYAAAAEAAAAQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAA2AIAAAAAAADYAgAAAAAAAAgA +AAAAAAAAAwAAAAQAAAAYAwAAAAAAABgDAAAAAAAAGAMAAAAAAAAcAAAAAAAAABwAAAAAAAAAAQAA +AAAAAAABAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOAFAAAAAAAA4AUAAAAAAAAAEAAA +AAAAAAEAAAAFAAAAABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAPQEAAAAAAAA9AQAAAAAAAAAQAAAA +AAAAAQAAAAQAAAAAIAAAAAAAAAAgAAAAAAAAACAAAAAAAADcAAAAAAAAANwAAAAAAAAAABAAAAAA +AAABAAAABgAAAAAuAAAAAAAAAD4AAAAAAAAAPgAAAAAAABACAAAAAAAAGAIAAAAAAAAAEAAAAAAA +AAIAAAAGAAAAEC4AAAAAAAAQPgAAAAAAABA+AAAAAAAAsAEAAAAAAACwAQAAAAAAAAgAAAAAAAAA +BAAAAAQAAAA4AwAAAAAAADgDAAAAAAAAOAMAAAAAAAAgAAAAAAAAACAAAAAAAAAACAAAAAAAAAAE +AAAABAAAAFgDAAAAAAAAWAMAAAAAAABYAwAAAAAAAEQAAAAAAAAARAAAAAAAAAAEAAAAAAAAAFPl +dGQEAAAAOAMAAAAAAAA4AwAAAAAAADgDAAAAAAAAIAAAAAAAAAAgAAAAAAAAAAgAAAAAAAAAUOV0 +ZAQAAAAEIAAAAAAAAAQgAAAAAAAABCAAAAAAAAAsAAAAAAAAACwAAAAAAAAABAAAAAAAAABR5XRk +BgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAFLldGQE +AAAAAC4AAAAAAAAAPgAAAAAAAAA+AAAAAAAAAAIAAAAAAAAAAgAAAAAAAAEAAAAAAAAAL2xpYjY0 +L2xkLWxpbnV4LXg4Ni02NC5zby4yAAAAAAAEAAAAEAAAAAUAAABHTlUAAoAAwAQAAAABAAAAAAAA +AAQAAAAUAAAAAwAAAEdOVQB5fSf98eWTEBscIbS6nrsfguPLDgQAAAAQAAAAAQAAAEdOVQAAAAAA +AwAAAAIAAAAAAAAAAAAAAAIAAAAFAAAAAQAAAAYAAAAAAIEAAAAAAAUAAAAAAAAA0WXObQAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAEgAAAAAAAAAAAAAAAAAAAAAAAABDAAAAIAAAAAAA +AAAAAAAAAAAAAAAAAABfAAAAIAAAAAAAAAAAAAAAAAAAAAAAAABuAAAAIAAAAAAAAAAAAAAAAAAA +AAAAAAATAAAAIgAAAAAAAAAAAAAAAAAAAAAAAAAAX19saWJjX3N0YXJ0X21haW4AX19jeGFfZmlu +YWxpemUAbGliYy5zby42AEdMSUJDXzIuMi41AEdMSUJDXzIuMzQAX0lUTV9kZXJlZ2lzdGVyVE1D +bG9uZVRhYmxlAF9fZ21vbl9zdGFydF9fAF9JVE1fcmVnaXN0ZXJUTUNsb25lVGFibGUAAAACAAEA +AQABAAMAAAAAAAEAAgAiAAAAEAAAAAAAAAB1GmkJAAADACwAAAAQAAAAtJGWBgAAAgA4AAAAAAAA +AAA+AAAAAAAACAAAAAAAAAAgEQAAAAAAAAg+AAAAAAAACAAAAAAAAADgEAAAAAAAAAhAAAAAAAAA +CAAAAAAAAAAIQAAAAAAAAMA/AAAAAAAABgAAAAEAAAAAAAAAAAAAAMg/AAAAAAAABgAAAAIAAAAA +AAAAAAAAANA/AAAAAAAABgAAAAMAAAAAAAAAAAAAANg/AAAAAAAABgAAAAQAAAAAAAAAAAAAAOA/ +AAAAAAAABgAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEiD7AhIiwXF +LwAASIXAdAL/0EiDxAjDAAAAAAAAAAAA/zXKLwAA/yXMLwAADx9AAP8lqi8AAGaQAAAAAAAAAAAx +7UmJ0V5IieJIg+TwUFRFMcAxyUiNPc4AAAD/FV8vAAD0Zi4PH4QAAAAAAA8fQABIjT2ZLwAASI0F +ki8AAEg5+HQVSIsFPi8AAEiFwHQJ/+APH4AAAAAAww8fgAAAAABIjT1pLwAASI01Yi8AAEgp/kiJ +8EjB7j9IwfgDSAHGSNH+dBRIiwUNLwAASIXAdAj/4GYPH0QAAMMPH4AAAAAA8w8e+oA9JS8AAAB1 +K1VIgz3qLgAAAEiJ5XQMSIs9Bi8AAOgp////6GT////GBf0uAAABXcMPHwDDDx+AAAAAAPMPHvrp +d////1VIieW4AAAAAF3DSIPsCEiDxAjDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAIAARsDOygAAAAEAAAA +HPD//3QAAAAs8P//nAAAADzw//9EAAAAJfH//7QAAAAUAAAAAAAAAAF6UgABeBABGwwHCJABBxAU +AAAAHAAAAPDv//8iAAAAAAAAAAAAAAAUAAAAAAAAAAF6UgABeBABGwwHCJABAAAkAAAAHAAAAKDv +//8QAAAAAA4QRg4YSg8LdwiAAD8aOyozJCIAAAAAFAAAAEQAAACI7///CAAAAAAAAAAAAAAAHAAA +AFwAAABp8P//CwAAAABBDhCGAkMNBkYMBwgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACARAAAAAAAA4BAAAAAAAAABAAAAAAAA +ACIAAAAAAAAADAAAAAAAAAAAEAAAAAAAAA0AAAAAAAAANBEAAAAAAAAZAAAAAAAAAAA+AAAAAAAA +GwAAAAAAAAAIAAAAAAAAABoAAAAAAAAACD4AAAAAAAAcAAAAAAAAAAgAAAAAAAAA9f7/bwAAAACg +AwAAAAAAAAUAAAAAAAAAWAQAAAAAAAAGAAAAAAAAAMgDAAAAAAAACgAAAAAAAACIAAAAAAAAAAsA +AAAAAAAAGAAAAAAAAAAVAAAAAAAAAAAAAAAAAAAAAwAAAAAAAADoPwAAAAAAAAcAAAAAAAAAIAUA +AAAAAAAIAAAAAAAAAMAAAAAAAAAACQAAAAAAAAAYAAAAAAAAAPv//28AAAAAAAAACAAAAAD+//9v +AAAAAPAEAAAAAAAA////bwAAAAABAAAAAAAAAPD//28AAAAA4AQAAAAAAAD5//9vAAAAAAMAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAED4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIQAAAAAAAAEdDQzogKERl +YmlhbiAxNC4yLjAtMytidWlsZDQpIDE0LjIuMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB +AAAABADx/wAAAAAAAAAAAAAAAAAAAAAJAAAAAQAEAHwDAAAAAAAAIAAAAAAAAAATAAAABADx/wAA +AAAAAAAAAAAAAAAAAAAeAAAAAgAOAHAQAAAAAAAAAAAAAAAAAAAgAAAAAgAOAKAQAAAAAAAAAAAA +AAAAAAAzAAAAAgAOAOAQAAAAAAAAAAAAAAAAAABJAAAAAQAZABBAAAAAAAAAAQAAAAAAAABVAAAA +AQAUAAg+AAAAAAAAAAAAAAAAAAB8AAAAAgAOACARAAAAAAAAAAAAAAAAAACIAAAAAQATAAA+AAAA +AAAAAAAAAAAAAACnAAAABADx/wAAAAAAAAAAAAAAAAAAAAATAAAABADx/wAAAAAAAAAAAAAAAAAA +AACuAAAAAQASANggAAAAAAAAAAAAAAAAAAAAAAAABADx/wAAAAAAAAAAAAAAAAAAAAC8AAAAAQAV +ABA+AAAAAAAAAAAAAAAAAADFAAAAAAARAAQgAAAAAAAAAAAAAAAAAADYAAAAAQAXAOg/AAAAAAAA +AAAAAAAAAADuAAAAEgAAAAAAAAAAAAAAAAAAAAAAAAALAQAAIAAAAAAAAAAAAAAAAAAAAAAAAAA2 +AQAAIAAYAABAAAAAAAAAAAAAAAAAAAAnAQAAEAAYABBAAAAAAAAAAAAAAAAAAAAuAQAAEgIPADQR +AAAAAAAAAAAAAAAAAAA0AQAAEAAYAABAAAAAAAAAAAAAAAAAAABBAQAAIAAAAAAAAAAAAAAAAAAA +AAAAAABQAQAAEQIYAAhAAAAAAAAAAAAAAAAAAABdAQAAEQAQAAAgAAAAAAAABAAAAAAAAABsAQAA +EAAZABhAAAAAAAAAAAAAAAAAAAA6AQAAEgAOAEAQAAAAAAAAIgAAAAAAAABxAQAAEAAZABBAAAAA +AAAAAAAAAAAAAAB9AQAAEgAOACkRAAAAAAAACwAAAAAAAACCAQAAEQIYABBAAAAAAAAAAAAAAAAA +AACOAQAAIAAAAAAAAAAAAAAAAAAAAAAAAACoAQAAIgAAAAAAAAAAAAAAAAAAAAAAAADDAQAAEgIL +AAAQAAAAAAAAAAAAAAAAAAAAU2NydDEubwBfX2FiaV90YWcAY3J0c3R1ZmYuYwBkZXJlZ2lzdGVy +X3RtX2Nsb25lcwBfX2RvX2dsb2JhbF9kdG9yc19hdXgAY29tcGxldGVkLjAAX19kb19nbG9iYWxf +ZHRvcnNfYXV4X2ZpbmlfYXJyYXlfZW50cnkAZnJhbWVfZHVtbXkAX19mcmFtZV9kdW1teV9pbml0 +X2FycmF5X2VudHJ5AG1haW4uYwBfX0ZSQU1FX0VORF9fAF9EWU5BTUlDAF9fR05VX0VIX0ZSQU1F +X0hEUgBfR0xPQkFMX09GRlNFVF9UQUJMRV8AX19saWJjX3N0YXJ0X21haW5AR0xJQkNfMi4zNABf +SVRNX2RlcmVnaXN0ZXJUTUNsb25lVGFibGUAX2VkYXRhAF9maW5pAF9fZGF0YV9zdGFydABfX2dt +b25fc3RhcnRfXwBfX2Rzb19oYW5kbGUAX0lPX3N0ZGluX3VzZWQAX2VuZABfX2Jzc19zdGFydABt +YWluAF9fVE1DX0VORF9fAF9JVE1fcmVnaXN0ZXJUTUNsb25lVGFibGUAX19jeGFfZmluYWxpemVA +R0xJQkNfMi4yLjUAX2luaXQAAC5zeW10YWIALnN0cnRhYgAuc2hzdHJ0YWIALmludGVycAAubm90 +ZS5nbnUucHJvcGVydHkALm5vdGUuZ251LmJ1aWxkLWlkAC5ub3RlLkFCSS10YWcALmdudS5oYXNo +AC5keW5zeW0ALmR5bnN0cgAuZ251LnZlcnNpb24ALmdudS52ZXJzaW9uX3IALnJlbGEuZHluAC5p +bml0AC5wbHQuZ290AC50ZXh0AC5maW5pAC5yb2RhdGEALmVoX2ZyYW1lX2hkcgAuZWhfZnJhbWUA +LmluaXRfYXJyYXkALmZpbmlfYXJyYXkALmR5bmFtaWMALmdvdC5wbHQALmRhdGEALmJzcwAuY29t +bWVudAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAABsAAAABAAAAAgAAAAAAAAAYAwAAAAAAABgDAAAAAAAAHAAAAAAA +AAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAjAAAABwAAAAIAAAAAAAAAOAMAAAAAAAA4AwAAAAAA +ACAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAANgAAAAcAAAACAAAAAAAAAFgDAAAAAAAA +WAMAAAAAAAAkAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAEkAAAAHAAAAAgAAAAAAAAB8 +AwAAAAAAAHwDAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAABXAAAA9v//bwIA +AAAAAAAAoAMAAAAAAACgAwAAAAAAACQAAAAAAAAABgAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAYQAA +AAsAAAACAAAAAAAAAMgDAAAAAAAAyAMAAAAAAACQAAAAAAAAAAcAAAABAAAACAAAAAAAAAAYAAAA +AAAAAGkAAAADAAAAAgAAAAAAAABYBAAAAAAAAFgEAAAAAAAAiAAAAAAAAAAAAAAAAAAAAAEAAAAA +AAAAAAAAAAAAAABxAAAA////bwIAAAAAAAAA4AQAAAAAAADgBAAAAAAAAAwAAAAAAAAABgAAAAAA +AAACAAAAAAAAAAIAAAAAAAAAfgAAAP7//28CAAAAAAAAAPAEAAAAAAAA8AQAAAAAAAAwAAAAAAAA +AAcAAAABAAAACAAAAAAAAAAAAAAAAAAAAI0AAAAEAAAAAgAAAAAAAAAgBQAAAAAAACAFAAAAAAAA +wAAAAAAAAAAGAAAAAAAAAAgAAAAAAAAAGAAAAAAAAACXAAAAAQAAAAYAAAAAAAAAABAAAAAAAAAA +EAAAAAAAABcAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAA9wAAAAEAAAAGAAAAAAAAACAQ +AAAAAAAAIBAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAJ0AAAABAAAABgAA +AAAAAAAwEAAAAAAAADAQAAAAAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAACmAAAA +AQAAAAYAAAAAAAAAQBAAAAAAAABAEAAAAAAAAPQAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAA +AAAArAAAAAEAAAAGAAAAAAAAADQRAAAAAAAANBEAAAAAAAAJAAAAAAAAAAAAAAAAAAAABAAAAAAA +AAAAAAAAAAAAALIAAAABAAAAEgAAAAAAAAAAIAAAAAAAAAAgAAAAAAAABAAAAAAAAAAAAAAAAAAA +AAQAAAAAAAAABAAAAAAAAAC6AAAAAQAAAAIAAAAAAAAABCAAAAAAAAAEIAAAAAAAACwAAAAAAAAA +AAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAyAAAAAEAAAACAAAAAAAAADAgAAAAAAAAMCAAAAAAAACs +AAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAANIAAAAOAAAAAwAAAAAAAAAAPgAAAAAAAAAu +AAAAAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAADeAAAADwAAAAMAAAAAAAAACD4A +AAAAAAAILgAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAA6gAAAAYAAAADAAAA +AAAAABA+AAAAAAAAEC4AAAAAAACwAQAAAAAAAAcAAAAAAAAACAAAAAAAAAAQAAAAAAAAAKEAAAAB +AAAAAwAAAAAAAADAPwAAAAAAAMAvAAAAAAAAKAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAA +AADzAAAAAQAAAAMAAAAAAAAA6D8AAAAAAADoLwAAAAAAABgAAAAAAAAAAAAAAAAAAAAIAAAAAAAA +AAgAAAAAAAAA/AAAAAEAAAADAAAAAAAAAABAAAAAAAAAADAAAAAAAAAQAAAAAAAAAAAAAAAAAAAA +CAAAAAAAAAAAAAAAAAAAAAIBAAAIAAAAAwAAAAAAAAAQQAAAAAAAABAwAAAAAAAACAAAAAAAAAAA +AAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAHAQAAAQAAADAAAAAAAAAAAAAAAAAAAAAQMAAAAAAAACUA +AAAAAAAAAAAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAQAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAODAA +AAAAAABIAwAAAAAAABwAAAASAAAACAAAAAAAAAAYAAAAAAAAAAkAAAADAAAAAAAAAAAAAAAAAAAA +AAAAAIAzAAAAAAAAyQEAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAARAAAAAwAAAAAAAAAA +AAAAAAAAAAAAAABJNQAAAAAAABABAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAA diff --git a/src/debug/dwarf/entry.go b/src/debug/dwarf/entry.go index 4541d74d4db408..30fad93e7937d2 100644 --- a/src/debug/dwarf/entry.go +++ b/src/debug/dwarf/entry.go @@ -229,7 +229,7 @@ func formToClass(form format, attr Attr, vers int, b *buf) Class { } } -// An entry is a sequence of attribute/value pairs. +// An Entry is a sequence of attribute/value pairs. type Entry struct { Offset Offset // offset of Entry in DWARF info Tag Tag // tag (kind of Entry) @@ -407,7 +407,8 @@ type Offset uint32 // Entry reads a single entry from buf, decoding // according to the given abbreviation table. -func (b *buf) entry(cu *Entry, atab abbrevTable, ubase Offset, vers int) *Entry { +func (b *buf) entry(cu *Entry, u *unit) *Entry { + atab, ubase, vers := u.atable, u.base, u.vers off := b.off id := uint32(b.uint()) if id == 0 { @@ -425,16 +426,6 @@ func (b *buf) entry(cu *Entry, atab abbrevTable, ubase Offset, vers int) *Entry Field: make([]Field, len(a.field)), } - // If we are currently parsing the compilation unit, - // we can't evaluate Addrx or Strx until we've seen the - // relevant base entry. - type delayed struct { - idx int - off uint64 - fmt format - } - var delay []delayed - resolveStrx := func(strBase, off uint64) string { off += strBase if uint64(int(off)) != off { @@ -531,18 +522,7 @@ func (b *buf) entry(cu *Entry, atab abbrevTable, ubase Offset, vers int) *Entry return nil } - // We have to adjust by the offset of the - // compilation unit. This won't work if the - // program uses Reader.Seek to skip over the - // unit. Not much we can do about that. - var addrBase int64 - if cu != nil { - addrBase, _ = cu.Val(AttrAddrBase).(int64) - } else if a.tag == TagCompileUnit { - delay = append(delay, delayed{i, off, formAddrx}) - break - } - + addrBase := int64(u.addrBase()) var err error val, err = b.dwarf.debugAddr(b.format, uint64(addrBase), off) if err != nil { @@ -574,7 +554,7 @@ func (b *buf) entry(cu *Entry, atab abbrevTable, ubase Offset, vers int) *Entry case formData16: val = b.bytes(16) case formSdata: - val = int64(b.int()) + val = b.int() case formUdata: val = int64(b.uint()) case formImplicitConst: @@ -682,18 +662,7 @@ func (b *buf) entry(cu *Entry, atab abbrevTable, ubase Offset, vers int) *Entry off *= 4 } - // We have to adjust by the offset of the - // compilation unit. This won't work if the - // program uses Reader.Seek to skip over the - // unit. Not much we can do about that. - var strBase int64 - if cu != nil { - strBase, _ = cu.Val(AttrStrOffsetsBase).(int64) - } else if a.tag == TagCompileUnit { - delay = append(delay, delayed{i, off, formStrx}) - break - } - + strBase := int64(u.strOffsetsBase()) val = resolveStrx(uint64(strBase), off) case formStrpSup: @@ -742,18 +711,7 @@ func (b *buf) entry(cu *Entry, atab abbrevTable, ubase Offset, vers int) *Entry case formRnglistx: off := b.uint() - // We have to adjust by the rnglists_base of - // the compilation unit. This won't work if - // the program uses Reader.Seek to skip over - // the unit. Not much we can do about that. - var rnglistsBase int64 - if cu != nil { - rnglistsBase, _ = cu.Val(AttrRnglistsBase).(int64) - } else if a.tag == TagCompileUnit { - delay = append(delay, delayed{i, off, formRnglistx}) - break - } - + rnglistsBase := int64(u.rngListsBase()) val = resolveRnglistx(uint64(rnglistsBase), off) } @@ -762,32 +720,6 @@ func (b *buf) entry(cu *Entry, atab abbrevTable, ubase Offset, vers int) *Entry if b.err != nil { return nil } - - for _, del := range delay { - switch del.fmt { - case formAddrx: - addrBase, _ := e.Val(AttrAddrBase).(int64) - val, err := b.dwarf.debugAddr(b.format, uint64(addrBase), del.off) - if err != nil { - b.err = err - return nil - } - e.Field[del.idx].Val = val - case formStrx: - strBase, _ := e.Val(AttrStrOffsetsBase).(int64) - e.Field[del.idx].Val = resolveStrx(uint64(strBase), del.off) - if b.err != nil { - return nil - } - case formRnglistx: - rnglistsBase, _ := e.Val(AttrRnglistsBase).(int64) - e.Field[del.idx].Val = resolveRnglistx(uint64(rnglistsBase), del.off) - if b.err != nil { - return nil - } - } - } - return e } @@ -839,6 +771,7 @@ func (r *Reader) Seek(off Offset) { u := &d.unit[0] r.unit = 0 r.b = makeBuf(r.d, u, "info", u.off, u.data) + r.collectDwarf5BaseOffsets(u) r.cu = nil return } @@ -854,6 +787,7 @@ func (r *Reader) Seek(off Offset) { u := &d.unit[i] r.unit = i r.b = makeBuf(r.d, u, "info", off, u.data[off-u.off:]) + r.collectDwarf5BaseOffsets(u) } // maybeNextUnit advances to the next unit if this one is finished. @@ -869,6 +803,17 @@ func (r *Reader) nextUnit() { u := &r.d.unit[r.unit] r.b = makeBuf(r.d, u, "info", u.off, u.data) r.cu = nil + r.collectDwarf5BaseOffsets(u) +} + +func (r *Reader) collectDwarf5BaseOffsets(u *unit) { + if u.vers < 5 || u.unit5 != nil { + return + } + u.unit5 = new(unit5) + if err := r.d.collectDwarf5BaseOffsets(u); err != nil { + r.err = err + } } // Next reads the next entry from the encoded entry stream. @@ -884,7 +829,7 @@ func (r *Reader) Next() (*Entry, error) { return nil, nil } u := &r.d.unit[r.unit] - e := r.b.entry(r.cu, u.atable, u.base, u.vers) + e := r.b.entry(r.cu, u) if r.b.err != nil { r.err = r.b.err return nil, r.err @@ -974,6 +919,7 @@ func (r *Reader) SeekPC(pc uint64) (*Entry, error) { r.cu = nil u := &r.d.unit[unit] r.b = makeBuf(r.d, u, "info", u.off, u.data) + r.collectDwarf5BaseOffsets(u) e, err := r.Next() if err != nil { return nil, err @@ -1091,7 +1037,7 @@ func (d *Data) baseAddressForEntry(e *Entry) (*Entry, uint64, error) { } u := &d.unit[i] b := makeBuf(d, u, "info", u.off, u.data) - cu = b.entry(nil, u.atable, u.base, u.vers) + cu = b.entry(nil, u) if b.err != nil { return nil, 0, b.err } diff --git a/src/debug/dwarf/entry_test.go b/src/debug/dwarf/entry_test.go index 1ce1c98f60a6ad..ee0c80a5038abf 100644 --- a/src/debug/dwarf/entry_test.go +++ b/src/debug/dwarf/entry_test.go @@ -6,6 +6,7 @@ package dwarf_test import ( . "debug/dwarf" + "debug/elf" "encoding/binary" "path/filepath" "reflect" @@ -457,3 +458,62 @@ func TestIssue52045(t *testing.T) { t.Errorf("got non-nil entry0, wanted nil") } } + +func TestIssue57046(t *testing.T) { + f, err := elf.Open("testdata/issue57046-clang.elf5") + if err != nil { + t.Fatalf("elf.Open returns err: %v", err) + } + d, err := f.DWARF() + if err != nil { + t.Fatalf("f.DWARF returns err: %v", err) + } + // Write down all the subprogram DIEs. + spdies := []Offset{} + lopcs := []uint64{} + r := d.Reader() + for { + e, err := r.Next() + if err != nil { + t.Fatalf("r.Next() returns err: %v", err) + } + if e == nil { + break + } + if e.Tag != TagSubprogram { + continue + } + var name string + var lopc uint64 + if n, ok := e.Val(AttrName).(string); ok { + name = n + } + if lo, ok := e.Val(AttrLowpc).(uint64); ok { + lopc = lo + } + if name == "" || lopc == 0 { + continue + } + spdies = append(spdies, e.Offset) + lopcs = append(lopcs, lopc) + } + + // Seek to the second entry in spdies (corresponding to mom() in + // issue57046_part2.c) and take a look at it. + r2 := d.Reader() + r2.Seek(spdies[1]) + e, err := r2.Next() + if err != nil { + t.Fatalf("r2.Next() returns err: %v", err) + } + if e == nil { + t.Fatalf("r2.Next() returned nil") + } + + // Verify that the lopc we see matches what we saw before. + got := e.Val(AttrLowpc).(uint64) + if got != lopcs[1] { + t.Errorf("bad lopc for fn2 following seek: want %x got %x\n", + lopcs[1], got) + } +} diff --git a/src/debug/dwarf/line.go b/src/debug/dwarf/line.go index 3a02c8e307b82c..3811416b921a43 100644 --- a/src/debug/dwarf/line.go +++ b/src/debug/dwarf/line.go @@ -837,7 +837,7 @@ func splitDrive(path string) (drive, rest string) { } if len(path) > 3 && (path[0] == '\\' || path[0] == '/') && (path[1] == '\\' || path[1] == '/') { // Normalize the path so we can search for just \ below. - npath := strings.Replace(path, "/", `\`, -1) + npath := strings.ReplaceAll(path, "/", `\`) // Get the host part, which must be non-empty. slash1 := strings.IndexByte(npath[2:], '\\') + 2 if slash1 > 2 { diff --git a/src/debug/dwarf/line_test.go b/src/debug/dwarf/line_test.go index e947d99ebb872f..0a7ade934af15e 100644 --- a/src/debug/dwarf/line_test.go +++ b/src/debug/dwarf/line_test.go @@ -94,8 +94,8 @@ func TestLineGCCWindows(t *testing.T) { toWindows := func(lf *LineFile) *LineFile { lf2 := *lf - lf2.Name = strings.Replace(lf2.Name, "/home/austin/go.dev/", "C:\\workdir\\go\\", -1) - lf2.Name = strings.Replace(lf2.Name, "/", "\\", -1) + lf2.Name = strings.ReplaceAll(lf2.Name, "/home/austin/go.dev/", "C:\\workdir\\go\\") + lf2.Name = strings.ReplaceAll(lf2.Name, "/", "\\") return &lf2 } file1C := toWindows(file1C) diff --git a/src/debug/dwarf/testdata/issue57046-clang.elf5 b/src/debug/dwarf/testdata/issue57046-clang.elf5 new file mode 100755 index 00000000000000..009af83135a100 Binary files /dev/null and b/src/debug/dwarf/testdata/issue57046-clang.elf5 differ diff --git a/src/debug/dwarf/testdata/issue57046_part1.c b/src/debug/dwarf/testdata/issue57046_part1.c new file mode 100644 index 00000000000000..8866ca66e141b1 --- /dev/null +++ b/src/debug/dwarf/testdata/issue57046_part1.c @@ -0,0 +1,42 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Part 1 of the sources for issue 57046 test case. + +// Build instructions: +// +// clang-16 -O -g -gdwarf-5 -c issue57046_part1.c +// clang-16 -O -g -gdwarf-5 -c issue57046_part2.c +// clang-16 -o issue57046-clang.elf5 issue57046_part1.o issue57046_part2.o + +#include +#include +#include + +extern const char *mom(); + +int gadgety() { + const char *ev = getenv("PATH"); + int n = strlen(ev); + int s1 = (int)ev[0]; + int s2 = (int)ev[1]; + int s3 = (int)ev[2]; + for (int i = 0; i < strlen(ev); i++) { + if (s1 == 101) { + int t = s1; + s1 = s3; + s3 = t; + } + if (ev[i] == 99) { + printf("%d\n", i); + } + } + s2 *= 2; + return n + s1 + s2; +} + +int main(int argc, char **argv) { + printf("Hi %s %d\n", mom(), gadgety()); + return 0; +} diff --git a/src/debug/dwarf/testdata/issue57046_part2.c b/src/debug/dwarf/testdata/issue57046_part2.c new file mode 100644 index 00000000000000..2f4e9f0d246912 --- /dev/null +++ b/src/debug/dwarf/testdata/issue57046_part2.c @@ -0,0 +1,10 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Part 2 of the sources for issue 57046 test case. +// See part 1 for build instructions. + +const char *mom() { + return "mom"; +} diff --git a/src/debug/dwarf/typeunit.go b/src/debug/dwarf/typeunit.go index 8ecf876416ccdf..e5b8973ac9c4f2 100644 --- a/src/debug/dwarf/typeunit.go +++ b/src/debug/dwarf/typeunit.go @@ -137,7 +137,7 @@ func (tur *typeUnitReader) Next() (*Entry, error) { if len(tur.tu.data) == 0 { return nil, nil } - e := tur.b.entry(nil, tur.tu.atable, tur.tu.base, tur.tu.vers) + e := tur.b.entry(nil, &tur.tu.unit) if tur.b.err != nil { tur.err = tur.b.err return nil, tur.err diff --git a/src/debug/dwarf/unit.go b/src/debug/dwarf/unit.go index 8b810d0a00b955..7a384f32c3f3a3 100644 --- a/src/debug/dwarf/unit.go +++ b/src/debug/dwarf/unit.go @@ -17,10 +17,18 @@ type unit struct { off Offset // byte offset of data within the aggregate info data []byte atable abbrevTable + *unit5 // info specific to DWARF 5 units asize int vers int - utype uint8 // DWARF 5 unit type is64 bool // True for 64-bit DWARF format + utype uint8 // DWARF 5 unit type +} + +type unit5 struct { + addrBase uint64 + strOffsetsBase uint64 + rngListsBase uint64 + locListsBase uint64 } // Implement the dataFormat interface. @@ -37,6 +45,34 @@ func (u *unit) addrsize() int { return u.asize } +func (u *unit) addrBase() uint64 { + if u.unit5 != nil { + return u.unit5.addrBase + } + return 0 +} + +func (u *unit) strOffsetsBase() uint64 { + if u.unit5 != nil { + return u.unit5.strOffsetsBase + } + return 0 +} + +func (u *unit) rngListsBase() uint64 { + if u.unit5 != nil { + return u.unit5.rngListsBase + } + return 0 +} + +func (u *unit) locListsBase() uint64 { + if u.unit5 != nil { + return u.unit5.locListsBase + } + return 0 +} + func (d *Data) parseUnits() ([]unit, error) { // Count units. nunit := 0 @@ -135,3 +171,30 @@ func (d *Data) offsetToUnit(off Offset) int { } return -1 } + +func (d *Data) collectDwarf5BaseOffsets(u *unit) error { + if u.unit5 == nil { + panic("expected unit5 to be set up already") + } + b := makeBuf(d, u, "info", u.off, u.data) + cu := b.entry(nil, u) + if cu == nil { + // Unknown abbreviation table entry or some other fatal + // problem; bail early on the assumption that this will be + // detected at some later point. + return b.err + } + if iAddrBase, ok := cu.Val(AttrAddrBase).(int64); ok { + u.unit5.addrBase = uint64(iAddrBase) + } + if iStrOffsetsBase, ok := cu.Val(AttrStrOffsetsBase).(int64); ok { + u.unit5.strOffsetsBase = uint64(iStrOffsetsBase) + } + if iRngListsBase, ok := cu.Val(AttrRnglistsBase).(int64); ok { + u.unit5.rngListsBase = uint64(iRngListsBase) + } + if iLocListsBase, ok := cu.Val(AttrLoclistsBase).(int64); ok { + u.unit5.locListsBase = uint64(iLocListsBase) + } + return nil +} diff --git a/src/debug/elf/elf.go b/src/debug/elf/elf.go index cecda61ed6e833..58e37daed2c081 100644 --- a/src/debug/elf/elf.go +++ b/src/debug/elf/elf.go @@ -620,36 +620,37 @@ func (i SectionIndex) GoString() string { return stringName(uint32(i), shnString type SectionType uint32 const ( - SHT_NULL SectionType = 0 /* inactive */ - SHT_PROGBITS SectionType = 1 /* program defined information */ - SHT_SYMTAB SectionType = 2 /* symbol table section */ - SHT_STRTAB SectionType = 3 /* string table section */ - SHT_RELA SectionType = 4 /* relocation section with addends */ - SHT_HASH SectionType = 5 /* symbol hash table section */ - SHT_DYNAMIC SectionType = 6 /* dynamic section */ - SHT_NOTE SectionType = 7 /* note section */ - SHT_NOBITS SectionType = 8 /* no space section */ - SHT_REL SectionType = 9 /* relocation section - no addends */ - SHT_SHLIB SectionType = 10 /* reserved - purpose unknown */ - SHT_DYNSYM SectionType = 11 /* dynamic symbol table section */ - SHT_INIT_ARRAY SectionType = 14 /* Initialization function pointers. */ - SHT_FINI_ARRAY SectionType = 15 /* Termination function pointers. */ - SHT_PREINIT_ARRAY SectionType = 16 /* Pre-initialization function ptrs. */ - SHT_GROUP SectionType = 17 /* Section group. */ - SHT_SYMTAB_SHNDX SectionType = 18 /* Section indexes (see SHN_XINDEX). */ - SHT_LOOS SectionType = 0x60000000 /* First of OS specific semantics */ - SHT_GNU_ATTRIBUTES SectionType = 0x6ffffff5 /* GNU object attributes */ - SHT_GNU_HASH SectionType = 0x6ffffff6 /* GNU hash table */ - SHT_GNU_LIBLIST SectionType = 0x6ffffff7 /* GNU prelink library list */ - SHT_GNU_VERDEF SectionType = 0x6ffffffd /* GNU version definition section */ - SHT_GNU_VERNEED SectionType = 0x6ffffffe /* GNU version needs section */ - SHT_GNU_VERSYM SectionType = 0x6fffffff /* GNU version symbol table */ - SHT_HIOS SectionType = 0x6fffffff /* Last of OS specific semantics */ - SHT_LOPROC SectionType = 0x70000000 /* reserved range for processor */ - SHT_MIPS_ABIFLAGS SectionType = 0x7000002a /* .MIPS.abiflags */ - SHT_HIPROC SectionType = 0x7fffffff /* specific section header types */ - SHT_LOUSER SectionType = 0x80000000 /* reserved range for application */ - SHT_HIUSER SectionType = 0xffffffff /* specific indexes */ + SHT_NULL SectionType = 0 /* inactive */ + SHT_PROGBITS SectionType = 1 /* program defined information */ + SHT_SYMTAB SectionType = 2 /* symbol table section */ + SHT_STRTAB SectionType = 3 /* string table section */ + SHT_RELA SectionType = 4 /* relocation section with addends */ + SHT_HASH SectionType = 5 /* symbol hash table section */ + SHT_DYNAMIC SectionType = 6 /* dynamic section */ + SHT_NOTE SectionType = 7 /* note section */ + SHT_NOBITS SectionType = 8 /* no space section */ + SHT_REL SectionType = 9 /* relocation section - no addends */ + SHT_SHLIB SectionType = 10 /* reserved - purpose unknown */ + SHT_DYNSYM SectionType = 11 /* dynamic symbol table section */ + SHT_INIT_ARRAY SectionType = 14 /* Initialization function pointers. */ + SHT_FINI_ARRAY SectionType = 15 /* Termination function pointers. */ + SHT_PREINIT_ARRAY SectionType = 16 /* Pre-initialization function ptrs. */ + SHT_GROUP SectionType = 17 /* Section group. */ + SHT_SYMTAB_SHNDX SectionType = 18 /* Section indexes (see SHN_XINDEX). */ + SHT_LOOS SectionType = 0x60000000 /* First of OS specific semantics */ + SHT_GNU_ATTRIBUTES SectionType = 0x6ffffff5 /* GNU object attributes */ + SHT_GNU_HASH SectionType = 0x6ffffff6 /* GNU hash table */ + SHT_GNU_LIBLIST SectionType = 0x6ffffff7 /* GNU prelink library list */ + SHT_GNU_VERDEF SectionType = 0x6ffffffd /* GNU version definition section */ + SHT_GNU_VERNEED SectionType = 0x6ffffffe /* GNU version needs section */ + SHT_GNU_VERSYM SectionType = 0x6fffffff /* GNU version symbol table */ + SHT_HIOS SectionType = 0x6fffffff /* Last of OS specific semantics */ + SHT_LOPROC SectionType = 0x70000000 /* reserved range for processor */ + SHT_RISCV_ATTRIBUTES SectionType = 0x70000003 /* RISCV object attributes */ + SHT_MIPS_ABIFLAGS SectionType = 0x7000002a /* .MIPS.abiflags */ + SHT_HIPROC SectionType = 0x7fffffff /* specific section header types */ + SHT_LOUSER SectionType = 0x80000000 /* reserved range for application */ + SHT_HIUSER SectionType = 0xffffffff /* specific indexes */ ) var shtStrings = []intName{ @@ -678,6 +679,8 @@ var shtStrings = []intName{ {0x6ffffffe, "SHT_GNU_VERNEED"}, {0x6fffffff, "SHT_GNU_VERSYM"}, {0x70000000, "SHT_LOPROC"}, + // We don't list the processor-dependent SectionType, + // as the values overlap. {0x7000002a, "SHT_MIPS_ABIFLAGS"}, {0x7fffffff, "SHT_HIPROC"}, {0x80000000, "SHT_LOUSER"}, @@ -794,6 +797,8 @@ const ( PT_MIPS_OPTIONS ProgType = 0x70000002 /* Options */ PT_MIPS_ABIFLAGS ProgType = 0x70000003 /* ABI flags */ + PT_RISCV_ATTRIBUTES ProgType = 0x70000003 /* RISC-V ELF attribute section. */ + PT_S390_PGSTE ProgType = 0x70000000 /* 4k page table size */ PT_HIPROC ProgType = 0x7fffffff /* Last processor-specific type. */ @@ -3579,6 +3584,15 @@ type intName struct { s string } +// Dynamic version flags. +type DynamicVersionFlag uint16 + +const ( + VER_FLG_BASE DynamicVersionFlag = 0x1 /* Version definition of the file. */ + VER_FLG_WEAK DynamicVersionFlag = 0x2 /* Weak version identifier. */ + VER_FLG_INFO DynamicVersionFlag = 0x4 /* Reference exists for informational purposes. */ +) + func stringName(i uint32, names []intName, goSyntax bool) string { for _, n := range names { if n.i == i { diff --git a/src/debug/elf/file.go b/src/debug/elf/file.go index 398439dcce6655..1d56a06c3fb221 100644 --- a/src/debug/elf/file.go +++ b/src/debug/elf/file.go @@ -25,6 +25,7 @@ import ( "internal/saferio" "internal/zstd" "io" + "math" "os" "strings" "unsafe" @@ -52,11 +53,12 @@ type FileHeader struct { // A File represents an open ELF file. type File struct { FileHeader - Sections []*Section - Progs []*Prog - closer io.Closer - gnuNeed []verneed - gnuVersym []byte + Sections []*Section + Progs []*Prog + closer io.Closer + dynVers []DynamicVersion + dynVerNeeds []DynamicVersionNeed + gnuVersym []byte } // A SectionHeader represents a single ELF section header. @@ -207,11 +209,19 @@ func (p *Prog) Open() io.ReadSeeker { return io.NewSectionReader(p.sr, 0, 1<<63- type Symbol struct { Name string Info, Other byte + + // HasVersion reports whether the symbol has any version information. + // This will only be true for the dynamic symbol table. + HasVersion bool + // VersionIndex is the symbol's version index. + // Use the methods of the [VersionIndex] type to access it. + // This field is only meaningful if HasVersion is true. + VersionIndex VersionIndex + Section SectionIndex Value, Size uint64 - // Version and Library are present only for the dynamic symbol - // table. + // These fields are present only for the dynamic symbol table. Version string Library string } @@ -491,6 +501,9 @@ func NewFile(r io.ReaderAt) (*File, error) { if c < 0 { return nil, &FormatError{0, "too many sections", shnum} } + if shnum > 0 && ((1<<64)-1)/uint64(shnum) < uint64(shentsize) { + return nil, &FormatError{0, "section header overflow", shnum} + } f.Sections = make([]*Section, 0, c) names := make([]uint32, 0, c) shdata, err := saferio.ReadDataAt(sr, uint64(shnum)*uint64(shentsize), shoff) @@ -680,6 +693,9 @@ func (f *File) getSymbols64(typ SectionType) ([]Symbol, []byte, error) { if len(data)%Sym64Size != 0 { return nil, nil, errors.New("length of symbol section is not a multiple of Sym64Size") } + if len(data) == 0 { + return nil, nil, ErrNoSymbols + } strdata, err := f.stringTable(symtabSection.Link) if err != nil { @@ -815,17 +831,9 @@ func (f *File) applyRelocationsAMD64(dst []byte, rels []byte) error { switch t { case R_X86_64_64: - if rela.Off+8 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val64 := sym.Value + uint64(rela.Addend) - f.ByteOrder.PutUint64(dst[rela.Off:rela.Off+8], val64) + putUint(f.ByteOrder, dst, rela.Off, 8, sym.Value, rela.Addend, false) case R_X86_64_32: - if rela.Off+4 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val32 := uint32(sym.Value) + uint32(rela.Addend) - f.ByteOrder.PutUint32(dst[rela.Off:rela.Off+4], val32) + putUint(f.ByteOrder, dst, rela.Off, 4, sym.Value, rela.Addend, false) } } @@ -857,12 +865,7 @@ func (f *File) applyRelocations386(dst []byte, rels []byte) error { sym := &symbols[symNo-1] if t == R_386_32 { - if rel.Off+4 >= uint32(len(dst)) { - continue - } - val := f.ByteOrder.Uint32(dst[rel.Off : rel.Off+4]) - val += uint32(sym.Value) - f.ByteOrder.PutUint32(dst[rel.Off:rel.Off+4], val) + putUint(f.ByteOrder, dst, uint64(rel.Off), 4, sym.Value, 0, true) } } @@ -895,12 +898,7 @@ func (f *File) applyRelocationsARM(dst []byte, rels []byte) error { switch t { case R_ARM_ABS32: - if rel.Off+4 >= uint32(len(dst)) { - continue - } - val := f.ByteOrder.Uint32(dst[rel.Off : rel.Off+4]) - val += uint32(sym.Value) - f.ByteOrder.PutUint32(dst[rel.Off:rel.Off+4], val) + putUint(f.ByteOrder, dst, uint64(rel.Off), 4, sym.Value, 0, true) } } @@ -940,17 +938,9 @@ func (f *File) applyRelocationsARM64(dst []byte, rels []byte) error { switch t { case R_AARCH64_ABS64: - if rela.Off+8 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val64 := sym.Value + uint64(rela.Addend) - f.ByteOrder.PutUint64(dst[rela.Off:rela.Off+8], val64) + putUint(f.ByteOrder, dst, rela.Off, 8, sym.Value, rela.Addend, false) case R_AARCH64_ABS32: - if rela.Off+4 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val32 := uint32(sym.Value) + uint32(rela.Addend) - f.ByteOrder.PutUint32(dst[rela.Off:rela.Off+4], val32) + putUint(f.ByteOrder, dst, rela.Off, 4, sym.Value, rela.Addend, false) } } @@ -986,11 +976,7 @@ func (f *File) applyRelocationsPPC(dst []byte, rels []byte) error { switch t { case R_PPC_ADDR32: - if rela.Off+4 >= uint32(len(dst)) || rela.Addend < 0 { - continue - } - val32 := uint32(sym.Value) + uint32(rela.Addend) - f.ByteOrder.PutUint32(dst[rela.Off:rela.Off+4], val32) + putUint(f.ByteOrder, dst, uint64(rela.Off), 4, sym.Value, 0, false) } } @@ -1026,17 +1012,9 @@ func (f *File) applyRelocationsPPC64(dst []byte, rels []byte) error { switch t { case R_PPC64_ADDR64: - if rela.Off+8 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val64 := sym.Value + uint64(rela.Addend) - f.ByteOrder.PutUint64(dst[rela.Off:rela.Off+8], val64) + putUint(f.ByteOrder, dst, rela.Off, 8, sym.Value, rela.Addend, false) case R_PPC64_ADDR32: - if rela.Off+4 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val32 := uint32(sym.Value) + uint32(rela.Addend) - f.ByteOrder.PutUint32(dst[rela.Off:rela.Off+4], val32) + putUint(f.ByteOrder, dst, rela.Off, 4, sym.Value, rela.Addend, false) } } @@ -1069,12 +1047,7 @@ func (f *File) applyRelocationsMIPS(dst []byte, rels []byte) error { switch t { case R_MIPS_32: - if rel.Off+4 >= uint32(len(dst)) { - continue - } - val := f.ByteOrder.Uint32(dst[rel.Off : rel.Off+4]) - val += uint32(sym.Value) - f.ByteOrder.PutUint32(dst[rel.Off:rel.Off+4], val) + putUint(f.ByteOrder, dst, uint64(rel.Off), 4, sym.Value, 0, true) } } @@ -1117,17 +1090,9 @@ func (f *File) applyRelocationsMIPS64(dst []byte, rels []byte) error { switch t { case R_MIPS_64: - if rela.Off+8 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val64 := sym.Value + uint64(rela.Addend) - f.ByteOrder.PutUint64(dst[rela.Off:rela.Off+8], val64) + putUint(f.ByteOrder, dst, rela.Off, 8, sym.Value, rela.Addend, false) case R_MIPS_32: - if rela.Off+4 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val32 := uint32(sym.Value) + uint32(rela.Addend) - f.ByteOrder.PutUint32(dst[rela.Off:rela.Off+4], val32) + putUint(f.ByteOrder, dst, rela.Off, 4, sym.Value, rela.Addend, false) } } @@ -1165,17 +1130,9 @@ func (f *File) applyRelocationsLOONG64(dst []byte, rels []byte) error { switch t { case R_LARCH_64: - if rela.Off+8 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val64 := sym.Value + uint64(rela.Addend) - f.ByteOrder.PutUint64(dst[rela.Off:rela.Off+8], val64) + putUint(f.ByteOrder, dst, rela.Off, 8, sym.Value, rela.Addend, false) case R_LARCH_32: - if rela.Off+4 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val32 := uint32(sym.Value) + uint32(rela.Addend) - f.ByteOrder.PutUint32(dst[rela.Off:rela.Off+4], val32) + putUint(f.ByteOrder, dst, rela.Off, 4, sym.Value, rela.Addend, false) } } @@ -1211,17 +1168,9 @@ func (f *File) applyRelocationsRISCV64(dst []byte, rels []byte) error { switch t { case R_RISCV_64: - if rela.Off+8 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val64 := sym.Value + uint64(rela.Addend) - f.ByteOrder.PutUint64(dst[rela.Off:rela.Off+8], val64) + putUint(f.ByteOrder, dst, rela.Off, 8, sym.Value, rela.Addend, false) case R_RISCV_32: - if rela.Off+4 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val32 := uint32(sym.Value) + uint32(rela.Addend) - f.ByteOrder.PutUint32(dst[rela.Off:rela.Off+4], val32) + putUint(f.ByteOrder, dst, rela.Off, 4, sym.Value, rela.Addend, false) } } @@ -1257,17 +1206,9 @@ func (f *File) applyRelocationss390x(dst []byte, rels []byte) error { switch t { case R_390_64: - if rela.Off+8 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val64 := sym.Value + uint64(rela.Addend) - f.ByteOrder.PutUint64(dst[rela.Off:rela.Off+8], val64) + putUint(f.ByteOrder, dst, rela.Off, 8, sym.Value, rela.Addend, false) case R_390_32: - if rela.Off+4 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val32 := uint32(sym.Value) + uint32(rela.Addend) - f.ByteOrder.PutUint32(dst[rela.Off:rela.Off+4], val32) + putUint(f.ByteOrder, dst, rela.Off, 4, sym.Value, rela.Addend, false) } } @@ -1303,17 +1244,10 @@ func (f *File) applyRelocationsSPARC64(dst []byte, rels []byte) error { switch t { case R_SPARC_64, R_SPARC_UA64: - if rela.Off+8 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val64 := sym.Value + uint64(rela.Addend) - f.ByteOrder.PutUint64(dst[rela.Off:rela.Off+8], val64) + putUint(f.ByteOrder, dst, rela.Off, 8, sym.Value, rela.Addend, false) + case R_SPARC_32, R_SPARC_UA32: - if rela.Off+4 >= uint64(len(dst)) || rela.Addend < 0 { - continue - } - val32 := uint32(sym.Value) + uint32(rela.Addend) - f.ByteOrder.PutUint32(dst[rela.Off:rela.Off+4], val32) + putUint(f.ByteOrder, dst, rela.Off, 4, sym.Value, rela.Addend, false) } } @@ -1444,9 +1378,13 @@ func (f *File) DynamicSymbols() ([]Symbol, error) { if err != nil { return nil, err } - if f.gnuVersionInit(str) { + hasVersions, err := f.gnuVersionInit(str) + if err != nil { + return nil, err + } + if hasVersions { for i := range sym { - sym[i].Library, sym[i].Version = f.gnuVersion(i) + sym[i].HasVersion, sym[i].VersionIndex, sym[i].Version, sym[i].Library = f.gnuVersion(i) } } return sym, nil @@ -1467,39 +1405,181 @@ func (f *File) ImportedSymbols() ([]ImportedSymbol, error) { if err != nil { return nil, err } - f.gnuVersionInit(str) + if _, err := f.gnuVersionInit(str); err != nil { + return nil, err + } var all []ImportedSymbol for i, s := range sym { if ST_BIND(s.Info) == STB_GLOBAL && s.Section == SHN_UNDEF { all = append(all, ImportedSymbol{Name: s.Name}) sym := &all[len(all)-1] - sym.Library, sym.Version = f.gnuVersion(i) + _, _, sym.Version, sym.Library = f.gnuVersion(i) } } return all, nil } -type verneed struct { - File string - Name string +// VersionIndex is the type of a [Symbol] version index. +type VersionIndex uint16 + +// IsHidden reports whether the symbol is hidden within the version. +// This means that the symbol can only be seen by specifying the exact version. +func (vi VersionIndex) IsHidden() bool { + return vi&0x8000 != 0 } -// gnuVersionInit parses the GNU version tables -// for use by calls to gnuVersion. -func (f *File) gnuVersionInit(str []byte) bool { - if f.gnuNeed != nil { - // Already initialized - return true +// Index returns the version index. +// If this is the value 0, it means that the symbol is local, +// and is not visible externally. +// If this is the value 1, it means that the symbol is in the base version, +// and has no specific version; it may or may not match a +// [DynamicVersion.Index] in the slice returned by [File.DynamicVersions]. +// Other values will match either [DynamicVersion.Index] +// in the slice returned by [File.DynamicVersions], +// or [DynamicVersionDep.Index] in the Needs field +// of the elements of the slice returned by [File.DynamicVersionNeeds]. +// In general, a defined symbol will have an index referring +// to DynamicVersions, and an undefined symbol will have an index +// referring to some version in DynamicVersionNeeds. +func (vi VersionIndex) Index() uint16 { + return uint16(vi & 0x7fff) +} + +// DynamicVersion is a version defined by a dynamic object. +// This describes entries in the ELF SHT_GNU_verdef section. +// We assume that the vd_version field is 1. +// Note that the name of the version appears here; +// it is not in the first Deps entry as it is in the ELF file. +type DynamicVersion struct { + Name string // Name of version defined by this index. + Index uint16 // Version index. + Flags DynamicVersionFlag + Deps []string // Names of versions that this version depends upon. +} + +// DynamicVersionNeed describes a shared library needed by a dynamic object, +// with a list of the versions needed from that shared library. +// This describes entries in the ELF SHT_GNU_verneed section. +// We assume that the vn_version field is 1. +type DynamicVersionNeed struct { + Name string // Shared library name. + Needs []DynamicVersionDep // Dependencies. +} + +// DynamicVersionDep is a version needed from some shared library. +type DynamicVersionDep struct { + Flags DynamicVersionFlag + Index uint16 // Version index. + Dep string // Name of required version. +} + +// dynamicVersions returns version information for a dynamic object. +func (f *File) dynamicVersions(str []byte) error { + if f.dynVers != nil { + // Already initialized. + return nil + } + + // Accumulate verdef information. + vd := f.SectionByType(SHT_GNU_VERDEF) + if vd == nil { + return nil + } + d, _ := vd.Data() + + var dynVers []DynamicVersion + i := 0 + for { + if i+20 > len(d) { + break + } + version := f.ByteOrder.Uint16(d[i : i+2]) + if version != 1 { + return &FormatError{int64(vd.Offset + uint64(i)), "unexpected dynamic version", version} + } + flags := DynamicVersionFlag(f.ByteOrder.Uint16(d[i+2 : i+4])) + ndx := f.ByteOrder.Uint16(d[i+4 : i+6]) + cnt := f.ByteOrder.Uint16(d[i+6 : i+8]) + aux := f.ByteOrder.Uint32(d[i+12 : i+16]) + next := f.ByteOrder.Uint32(d[i+16 : i+20]) + + if cnt == 0 { + return &FormatError{int64(vd.Offset + uint64(i)), "dynamic version has no name", nil} + } + + var name string + var depName string + var deps []string + j := i + int(aux) + for c := 0; c < int(cnt); c++ { + if j+8 > len(d) { + break + } + vname := f.ByteOrder.Uint32(d[j : j+4]) + vnext := f.ByteOrder.Uint32(d[j+4 : j+8]) + depName, _ = getString(str, int(vname)) + + if c == 0 { + name = depName + } else { + deps = append(deps, depName) + } + + j += int(vnext) + } + + dynVers = append(dynVers, DynamicVersion{ + Name: name, + Index: ndx, + Flags: flags, + Deps: deps, + }) + + if next == 0 { + break + } + i += int(next) + } + + f.dynVers = dynVers + + return nil +} + +// DynamicVersions returns version information for a dynamic object. +func (f *File) DynamicVersions() ([]DynamicVersion, error) { + if f.dynVers == nil { + _, str, err := f.getSymbols(SHT_DYNSYM) + if err != nil { + return nil, err + } + hasVersions, err := f.gnuVersionInit(str) + if err != nil { + return nil, err + } + if !hasVersions { + return nil, errors.New("DynamicVersions: missing version table") + } + } + + return f.dynVers, nil +} + +// dynamicVersionNeeds returns version dependencies for a dynamic object. +func (f *File) dynamicVersionNeeds(str []byte) error { + if f.dynVerNeeds != nil { + // Already initialized. + return nil } // Accumulate verneed information. vn := f.SectionByType(SHT_GNU_VERNEED) if vn == nil { - return false + return nil } d, _ := vn.Data() - var need []verneed + var dynVerNeeds []DynamicVersionNeed i := 0 for { if i+16 > len(d) { @@ -1507,7 +1587,7 @@ func (f *File) gnuVersionInit(str []byte) bool { } vers := f.ByteOrder.Uint16(d[i : i+2]) if vers != 1 { - break + return &FormatError{int64(vn.Offset + uint64(i)), "unexpected dynamic need version", vers} } cnt := f.ByteOrder.Uint16(d[i+2 : i+4]) fileoff := f.ByteOrder.Uint32(d[i+4 : i+8]) @@ -1515,68 +1595,120 @@ func (f *File) gnuVersionInit(str []byte) bool { next := f.ByteOrder.Uint32(d[i+12 : i+16]) file, _ := getString(str, int(fileoff)) - var name string + var deps []DynamicVersionDep j := i + int(aux) for c := 0; c < int(cnt); c++ { if j+16 > len(d) { break } - // hash := f.ByteOrder.Uint32(d[j:j+4]) - // flags := f.ByteOrder.Uint16(d[j+4:j+6]) - other := f.ByteOrder.Uint16(d[j+6 : j+8]) + flags := DynamicVersionFlag(f.ByteOrder.Uint16(d[j+4 : j+6])) + index := f.ByteOrder.Uint16(d[j+6 : j+8]) nameoff := f.ByteOrder.Uint32(d[j+8 : j+12]) next := f.ByteOrder.Uint32(d[j+12 : j+16]) - name, _ = getString(str, int(nameoff)) - ndx := int(other) - if ndx >= len(need) { - a := make([]verneed, 2*(ndx+1)) - copy(a, need) - need = a - } + depName, _ := getString(str, int(nameoff)) + + deps = append(deps, DynamicVersionDep{ + Flags: flags, + Index: index, + Dep: depName, + }) - need[ndx] = verneed{file, name} if next == 0 { break } j += int(next) } + dynVerNeeds = append(dynVerNeeds, DynamicVersionNeed{ + Name: file, + Needs: deps, + }) + if next == 0 { break } i += int(next) } + f.dynVerNeeds = dynVerNeeds + + return nil +} + +// DynamicVersionNeeds returns version dependencies for a dynamic object. +func (f *File) DynamicVersionNeeds() ([]DynamicVersionNeed, error) { + if f.dynVerNeeds == nil { + _, str, err := f.getSymbols(SHT_DYNSYM) + if err != nil { + return nil, err + } + hasVersions, err := f.gnuVersionInit(str) + if err != nil { + return nil, err + } + if !hasVersions { + return nil, errors.New("DynamicVersionNeeds: missing version table") + } + } + + return f.dynVerNeeds, nil +} + +// gnuVersionInit parses the GNU version tables +// for use by calls to gnuVersion. +// It reports whether any version tables were found. +func (f *File) gnuVersionInit(str []byte) (bool, error) { // Versym parallels symbol table, indexing into verneed. vs := f.SectionByType(SHT_GNU_VERSYM) if vs == nil { - return false + return false, nil } - d, _ = vs.Data() + d, _ := vs.Data() - f.gnuNeed = need f.gnuVersym = d - return true + if err := f.dynamicVersions(str); err != nil { + return false, err + } + if err := f.dynamicVersionNeeds(str); err != nil { + return false, err + } + return true, nil } // gnuVersion adds Library and Version information to sym, // which came from offset i of the symbol table. -func (f *File) gnuVersion(i int) (library string, version string) { +func (f *File) gnuVersion(i int) (hasVersion bool, versionIndex VersionIndex, version string, library string) { // Each entry is two bytes; skip undef entry at beginning. i = (i + 1) * 2 if i >= len(f.gnuVersym) { - return + return false, 0, "", "" } s := f.gnuVersym[i:] if len(s) < 2 { - return + return false, 0, "", "" } - j := int(f.ByteOrder.Uint16(s)) - if j < 2 || j >= len(f.gnuNeed) { - return + vi := VersionIndex(f.ByteOrder.Uint16(s)) + ndx := vi.Index() + + if ndx == 0 || ndx == 1 { + return true, vi, "", "" + } + + for _, v := range f.dynVerNeeds { + for _, n := range v.Needs { + if ndx == n.Index { + return true, vi, n.Dep, v.Name + } + } + } + + for _, v := range f.dynVers { + if ndx == v.Index { + return true, vi, v.Name, "" + } } - n := &f.gnuNeed[j] - return n.File, n.Name + + return false, 0, "", "" } // ImportedLibraries returns the names of all libraries @@ -1690,3 +1822,38 @@ type nobitsSectionReader struct{} func (*nobitsSectionReader) ReadAt(p []byte, off int64) (n int, err error) { return 0, errors.New("unexpected read from SHT_NOBITS section") } + +// putUint writes a relocation to slice +// at offset start of length length (4 or 8 bytes), +// adding sym+addend to the existing value if readUint is true, +// or just writing sym+addend if readUint is false. +// If the write would extend beyond the end of slice, putUint does nothing. +// If the addend is negative, putUint does nothing. +// If the addition would overflow, putUint does nothing. +func putUint(byteOrder binary.ByteOrder, slice []byte, start, length, sym uint64, addend int64, readUint bool) { + if start+length > uint64(len(slice)) || math.MaxUint64-start < length { + return + } + if addend < 0 { + return + } + + s := slice[start : start+length] + + switch length { + case 4: + ae := uint32(addend) + if readUint { + ae += byteOrder.Uint32(s) + } + byteOrder.PutUint32(s, uint32(sym)+ae) + case 8: + ae := uint64(addend) + if readUint { + ae += byteOrder.Uint64(s) + } + byteOrder.PutUint64(s, sym+ae) + default: + panic("can't happen") + } +} diff --git a/src/debug/elf/file_test.go b/src/debug/elf/file_test.go index 5dd83a2917618d..b796cdb95b65c6 100644 --- a/src/debug/elf/file_test.go +++ b/src/debug/elf/file_test.go @@ -78,80 +78,80 @@ var fileTests = []fileTest{ }, []string{"libc.so.6"}, []Symbol{ - {"", 3, 0, 1, 134512852, 0, "", ""}, - {"", 3, 0, 2, 134512876, 0, "", ""}, - {"", 3, 0, 3, 134513020, 0, "", ""}, - {"", 3, 0, 4, 134513292, 0, "", ""}, - {"", 3, 0, 5, 134513480, 0, "", ""}, - {"", 3, 0, 6, 134513512, 0, "", ""}, - {"", 3, 0, 7, 134513532, 0, "", ""}, - {"", 3, 0, 8, 134513612, 0, "", ""}, - {"", 3, 0, 9, 134513996, 0, "", ""}, - {"", 3, 0, 10, 134514008, 0, "", ""}, - {"", 3, 0, 11, 134518268, 0, "", ""}, - {"", 3, 0, 12, 134518280, 0, "", ""}, - {"", 3, 0, 13, 134518284, 0, "", ""}, - {"", 3, 0, 14, 134518436, 0, "", ""}, - {"", 3, 0, 15, 134518444, 0, "", ""}, - {"", 3, 0, 16, 134518452, 0, "", ""}, - {"", 3, 0, 17, 134518456, 0, "", ""}, - {"", 3, 0, 18, 134518484, 0, "", ""}, - {"", 3, 0, 19, 0, 0, "", ""}, - {"", 3, 0, 20, 0, 0, "", ""}, - {"", 3, 0, 21, 0, 0, "", ""}, - {"", 3, 0, 22, 0, 0, "", ""}, - {"", 3, 0, 23, 0, 0, "", ""}, - {"", 3, 0, 24, 0, 0, "", ""}, - {"", 3, 0, 25, 0, 0, "", ""}, - {"", 3, 0, 26, 0, 0, "", ""}, - {"", 3, 0, 27, 0, 0, "", ""}, - {"", 3, 0, 28, 0, 0, "", ""}, - {"", 3, 0, 29, 0, 0, "", ""}, - {"crt1.c", 4, 0, 65521, 0, 0, "", ""}, - {"/usr/src/lib/csu/i386-elf/crti.S", 4, 0, 65521, 0, 0, "", ""}, - {"", 4, 0, 65521, 0, 0, "", ""}, - {"", 4, 0, 65521, 0, 0, "", ""}, - {"/usr/src/lib/csu/i386-elf/crti.S", 4, 0, 65521, 0, 0, "", ""}, - {"crtstuff.c", 4, 0, 65521, 0, 0, "", ""}, - {"__CTOR_LIST__", 1, 0, 14, 134518436, 0, "", ""}, - {"__DTOR_LIST__", 1, 0, 15, 134518444, 0, "", ""}, - {"__EH_FRAME_BEGIN__", 1, 0, 12, 134518280, 0, "", ""}, - {"__JCR_LIST__", 1, 0, 16, 134518452, 0, "", ""}, - {"p.0", 1, 0, 11, 134518276, 0, "", ""}, - {"completed.1", 1, 0, 18, 134518484, 1, "", ""}, - {"__do_global_dtors_aux", 2, 0, 8, 134513760, 0, "", ""}, - {"object.2", 1, 0, 18, 134518488, 24, "", ""}, - {"frame_dummy", 2, 0, 8, 134513836, 0, "", ""}, - {"crtstuff.c", 4, 0, 65521, 0, 0, "", ""}, - {"__CTOR_END__", 1, 0, 14, 134518440, 0, "", ""}, - {"__DTOR_END__", 1, 0, 15, 134518448, 0, "", ""}, - {"__FRAME_END__", 1, 0, 12, 134518280, 0, "", ""}, - {"__JCR_END__", 1, 0, 16, 134518452, 0, "", ""}, - {"__do_global_ctors_aux", 2, 0, 8, 134513960, 0, "", ""}, - {"/usr/src/lib/csu/i386-elf/crtn.S", 4, 0, 65521, 0, 0, "", ""}, - {"", 4, 0, 65521, 0, 0, "", ""}, - {"", 4, 0, 65521, 0, 0, "", ""}, - {"/usr/src/lib/csu/i386-elf/crtn.S", 4, 0, 65521, 0, 0, "", ""}, - {"hello.c", 4, 0, 65521, 0, 0, "", ""}, - {"printf", 18, 0, 0, 0, 44, "", ""}, - {"_DYNAMIC", 17, 0, 65521, 134518284, 0, "", ""}, - {"__dso_handle", 17, 2, 11, 134518272, 0, "", ""}, - {"_init", 18, 0, 6, 134513512, 0, "", ""}, - {"environ", 17, 0, 18, 134518512, 4, "", ""}, - {"__deregister_frame_info", 32, 0, 0, 0, 0, "", ""}, - {"__progname", 17, 0, 11, 134518268, 4, "", ""}, - {"_start", 18, 0, 8, 134513612, 145, "", ""}, - {"__bss_start", 16, 0, 65521, 134518484, 0, "", ""}, - {"main", 18, 0, 8, 134513912, 46, "", ""}, - {"_init_tls", 18, 0, 0, 0, 5, "", ""}, - {"_fini", 18, 0, 9, 134513996, 0, "", ""}, - {"atexit", 18, 0, 0, 0, 43, "", ""}, - {"_edata", 16, 0, 65521, 134518484, 0, "", ""}, - {"_GLOBAL_OFFSET_TABLE_", 17, 0, 65521, 134518456, 0, "", ""}, - {"_end", 16, 0, 65521, 134518516, 0, "", ""}, - {"exit", 18, 0, 0, 0, 68, "", ""}, - {"_Jv_RegisterClasses", 32, 0, 0, 0, 0, "", ""}, - {"__register_frame_info", 32, 0, 0, 0, 0, "", ""}, + {"", 3, 0, false, 0, 1, 134512852, 0, "", ""}, + {"", 3, 0, false, 0, 2, 134512876, 0, "", ""}, + {"", 3, 0, false, 0, 3, 134513020, 0, "", ""}, + {"", 3, 0, false, 0, 4, 134513292, 0, "", ""}, + {"", 3, 0, false, 0, 5, 134513480, 0, "", ""}, + {"", 3, 0, false, 0, 6, 134513512, 0, "", ""}, + {"", 3, 0, false, 0, 7, 134513532, 0, "", ""}, + {"", 3, 0, false, 0, 8, 134513612, 0, "", ""}, + {"", 3, 0, false, 0, 9, 134513996, 0, "", ""}, + {"", 3, 0, false, 0, 10, 134514008, 0, "", ""}, + {"", 3, 0, false, 0, 11, 134518268, 0, "", ""}, + {"", 3, 0, false, 0, 12, 134518280, 0, "", ""}, + {"", 3, 0, false, 0, 13, 134518284, 0, "", ""}, + {"", 3, 0, false, 0, 14, 134518436, 0, "", ""}, + {"", 3, 0, false, 0, 15, 134518444, 0, "", ""}, + {"", 3, 0, false, 0, 16, 134518452, 0, "", ""}, + {"", 3, 0, false, 0, 17, 134518456, 0, "", ""}, + {"", 3, 0, false, 0, 18, 134518484, 0, "", ""}, + {"", 3, 0, false, 0, 19, 0, 0, "", ""}, + {"", 3, 0, false, 0, 20, 0, 0, "", ""}, + {"", 3, 0, false, 0, 21, 0, 0, "", ""}, + {"", 3, 0, false, 0, 22, 0, 0, "", ""}, + {"", 3, 0, false, 0, 23, 0, 0, "", ""}, + {"", 3, 0, false, 0, 24, 0, 0, "", ""}, + {"", 3, 0, false, 0, 25, 0, 0, "", ""}, + {"", 3, 0, false, 0, 26, 0, 0, "", ""}, + {"", 3, 0, false, 0, 27, 0, 0, "", ""}, + {"", 3, 0, false, 0, 28, 0, 0, "", ""}, + {"", 3, 0, false, 0, 29, 0, 0, "", ""}, + {"crt1.c", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"/usr/src/lib/csu/i386-elf/crti.S", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"/usr/src/lib/csu/i386-elf/crti.S", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"crtstuff.c", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"__CTOR_LIST__", 1, 0, false, 0, 14, 134518436, 0, "", ""}, + {"__DTOR_LIST__", 1, 0, false, 0, 15, 134518444, 0, "", ""}, + {"__EH_FRAME_BEGIN__", 1, 0, false, 0, 12, 134518280, 0, "", ""}, + {"__JCR_LIST__", 1, 0, false, 0, 16, 134518452, 0, "", ""}, + {"p.0", 1, 0, false, 0, 11, 134518276, 0, "", ""}, + {"completed.1", 1, 0, false, 0, 18, 134518484, 1, "", ""}, + {"__do_global_dtors_aux", 2, 0, false, 0, 8, 134513760, 0, "", ""}, + {"object.2", 1, 0, false, 0, 18, 134518488, 24, "", ""}, + {"frame_dummy", 2, 0, false, 0, 8, 134513836, 0, "", ""}, + {"crtstuff.c", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"__CTOR_END__", 1, 0, false, 0, 14, 134518440, 0, "", ""}, + {"__DTOR_END__", 1, 0, false, 0, 15, 134518448, 0, "", ""}, + {"__FRAME_END__", 1, 0, false, 0, 12, 134518280, 0, "", ""}, + {"__JCR_END__", 1, 0, false, 0, 16, 134518452, 0, "", ""}, + {"__do_global_ctors_aux", 2, 0, false, 0, 8, 134513960, 0, "", ""}, + {"/usr/src/lib/csu/i386-elf/crtn.S", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"/usr/src/lib/csu/i386-elf/crtn.S", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"hello.c", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"printf", 18, 0, false, 0, 0, 0, 44, "", ""}, + {"_DYNAMIC", 17, 0, false, 0, 65521, 134518284, 0, "", ""}, + {"__dso_handle", 17, 2, false, 0, 11, 134518272, 0, "", ""}, + {"_init", 18, 0, false, 0, 6, 134513512, 0, "", ""}, + {"environ", 17, 0, false, 0, 18, 134518512, 4, "", ""}, + {"__deregister_frame_info", 32, 0, false, 0, 0, 0, 0, "", ""}, + {"__progname", 17, 0, false, 0, 11, 134518268, 4, "", ""}, + {"_start", 18, 0, false, 0, 8, 134513612, 145, "", ""}, + {"__bss_start", 16, 0, false, 0, 65521, 134518484, 0, "", ""}, + {"main", 18, 0, false, 0, 8, 134513912, 46, "", ""}, + {"_init_tls", 18, 0, false, 0, 0, 0, 5, "", ""}, + {"_fini", 18, 0, false, 0, 9, 134513996, 0, "", ""}, + {"atexit", 18, 0, false, 0, 0, 0, 43, "", ""}, + {"_edata", 16, 0, false, 0, 65521, 134518484, 0, "", ""}, + {"_GLOBAL_OFFSET_TABLE_", 17, 0, false, 0, 65521, 134518456, 0, "", ""}, + {"_end", 16, 0, false, 0, 65521, 134518516, 0, "", ""}, + {"exit", 18, 0, false, 0, 0, 0, 68, "", ""}, + {"_Jv_RegisterClasses", 32, 0, false, 0, 0, 0, 0, "", ""}, + {"__register_frame_info", 32, 0, false, 0, 0, 0, 0, "", ""}, }, }, { @@ -208,79 +208,79 @@ var fileTests = []fileTest{ }, []string{"libc.so.6"}, []Symbol{ - {"", 3, 0, 1, 4194816, 0, "", ""}, - {"", 3, 0, 2, 4194844, 0, "", ""}, - {"", 3, 0, 3, 4194880, 0, "", ""}, - {"", 3, 0, 4, 4194920, 0, "", ""}, - {"", 3, 0, 5, 4194952, 0, "", ""}, - {"", 3, 0, 6, 4195048, 0, "", ""}, - {"", 3, 0, 7, 4195110, 0, "", ""}, - {"", 3, 0, 8, 4195120, 0, "", ""}, - {"", 3, 0, 9, 4195152, 0, "", ""}, - {"", 3, 0, 10, 4195176, 0, "", ""}, - {"", 3, 0, 11, 4195224, 0, "", ""}, - {"", 3, 0, 12, 4195248, 0, "", ""}, - {"", 3, 0, 13, 4195296, 0, "", ""}, - {"", 3, 0, 14, 4195732, 0, "", ""}, - {"", 3, 0, 15, 4195748, 0, "", ""}, - {"", 3, 0, 16, 4195768, 0, "", ""}, - {"", 3, 0, 17, 4195808, 0, "", ""}, - {"", 3, 0, 18, 6293128, 0, "", ""}, - {"", 3, 0, 19, 6293144, 0, "", ""}, - {"", 3, 0, 20, 6293160, 0, "", ""}, - {"", 3, 0, 21, 6293168, 0, "", ""}, - {"", 3, 0, 22, 6293584, 0, "", ""}, - {"", 3, 0, 23, 6293592, 0, "", ""}, - {"", 3, 0, 24, 6293632, 0, "", ""}, - {"", 3, 0, 25, 6293656, 0, "", ""}, - {"", 3, 0, 26, 0, 0, "", ""}, - {"", 3, 0, 27, 0, 0, "", ""}, - {"", 3, 0, 28, 0, 0, "", ""}, - {"", 3, 0, 29, 0, 0, "", ""}, - {"", 3, 0, 30, 0, 0, "", ""}, - {"", 3, 0, 31, 0, 0, "", ""}, - {"", 3, 0, 32, 0, 0, "", ""}, - {"", 3, 0, 33, 0, 0, "", ""}, - {"init.c", 4, 0, 65521, 0, 0, "", ""}, - {"initfini.c", 4, 0, 65521, 0, 0, "", ""}, - {"call_gmon_start", 2, 0, 13, 4195340, 0, "", ""}, - {"crtstuff.c", 4, 0, 65521, 0, 0, "", ""}, - {"__CTOR_LIST__", 1, 0, 18, 6293128, 0, "", ""}, - {"__DTOR_LIST__", 1, 0, 19, 6293144, 0, "", ""}, - {"__JCR_LIST__", 1, 0, 20, 6293160, 0, "", ""}, - {"__do_global_dtors_aux", 2, 0, 13, 4195376, 0, "", ""}, - {"completed.6183", 1, 0, 25, 6293656, 1, "", ""}, - {"p.6181", 1, 0, 24, 6293648, 0, "", ""}, - {"frame_dummy", 2, 0, 13, 4195440, 0, "", ""}, - {"crtstuff.c", 4, 0, 65521, 0, 0, "", ""}, - {"__CTOR_END__", 1, 0, 18, 6293136, 0, "", ""}, - {"__DTOR_END__", 1, 0, 19, 6293152, 0, "", ""}, - {"__FRAME_END__", 1, 0, 17, 4195968, 0, "", ""}, - {"__JCR_END__", 1, 0, 20, 6293160, 0, "", ""}, - {"__do_global_ctors_aux", 2, 0, 13, 4195680, 0, "", ""}, - {"initfini.c", 4, 0, 65521, 0, 0, "", ""}, - {"hello.c", 4, 0, 65521, 0, 0, "", ""}, - {"_GLOBAL_OFFSET_TABLE_", 1, 2, 23, 6293592, 0, "", ""}, - {"__init_array_end", 0, 2, 18, 6293124, 0, "", ""}, - {"__init_array_start", 0, 2, 18, 6293124, 0, "", ""}, - {"_DYNAMIC", 1, 2, 21, 6293168, 0, "", ""}, - {"data_start", 32, 0, 24, 6293632, 0, "", ""}, - {"__libc_csu_fini", 18, 0, 13, 4195520, 2, "", ""}, - {"_start", 18, 0, 13, 4195296, 0, "", ""}, - {"__gmon_start__", 32, 0, 0, 0, 0, "", ""}, - {"_Jv_RegisterClasses", 32, 0, 0, 0, 0, "", ""}, - {"puts@@GLIBC_2.2.5", 18, 0, 0, 0, 396, "", ""}, - {"_fini", 18, 0, 14, 4195732, 0, "", ""}, - {"__libc_start_main@@GLIBC_2.2.5", 18, 0, 0, 0, 450, "", ""}, - {"_IO_stdin_used", 17, 0, 15, 4195748, 4, "", ""}, - {"__data_start", 16, 0, 24, 6293632, 0, "", ""}, - {"__dso_handle", 17, 2, 24, 6293640, 0, "", ""}, - {"__libc_csu_init", 18, 0, 13, 4195536, 137, "", ""}, - {"__bss_start", 16, 0, 65521, 6293656, 0, "", ""}, - {"_end", 16, 0, 65521, 6293664, 0, "", ""}, - {"_edata", 16, 0, 65521, 6293656, 0, "", ""}, - {"main", 18, 0, 13, 4195480, 27, "", ""}, - {"_init", 18, 0, 11, 4195224, 0, "", ""}, + {"", 3, 0, false, 0, 1, 4194816, 0, "", ""}, + {"", 3, 0, false, 0, 2, 4194844, 0, "", ""}, + {"", 3, 0, false, 0, 3, 4194880, 0, "", ""}, + {"", 3, 0, false, 0, 4, 4194920, 0, "", ""}, + {"", 3, 0, false, 0, 5, 4194952, 0, "", ""}, + {"", 3, 0, false, 0, 6, 4195048, 0, "", ""}, + {"", 3, 0, false, 0, 7, 4195110, 0, "", ""}, + {"", 3, 0, false, 0, 8, 4195120, 0, "", ""}, + {"", 3, 0, false, 0, 9, 4195152, 0, "", ""}, + {"", 3, 0, false, 0, 10, 4195176, 0, "", ""}, + {"", 3, 0, false, 0, 11, 4195224, 0, "", ""}, + {"", 3, 0, false, 0, 12, 4195248, 0, "", ""}, + {"", 3, 0, false, 0, 13, 4195296, 0, "", ""}, + {"", 3, 0, false, 0, 14, 4195732, 0, "", ""}, + {"", 3, 0, false, 0, 15, 4195748, 0, "", ""}, + {"", 3, 0, false, 0, 16, 4195768, 0, "", ""}, + {"", 3, 0, false, 0, 17, 4195808, 0, "", ""}, + {"", 3, 0, false, 0, 18, 6293128, 0, "", ""}, + {"", 3, 0, false, 0, 19, 6293144, 0, "", ""}, + {"", 3, 0, false, 0, 20, 6293160, 0, "", ""}, + {"", 3, 0, false, 0, 21, 6293168, 0, "", ""}, + {"", 3, 0, false, 0, 22, 6293584, 0, "", ""}, + {"", 3, 0, false, 0, 23, 6293592, 0, "", ""}, + {"", 3, 0, false, 0, 24, 6293632, 0, "", ""}, + {"", 3, 0, false, 0, 25, 6293656, 0, "", ""}, + {"", 3, 0, false, 0, 26, 0, 0, "", ""}, + {"", 3, 0, false, 0, 27, 0, 0, "", ""}, + {"", 3, 0, false, 0, 28, 0, 0, "", ""}, + {"", 3, 0, false, 0, 29, 0, 0, "", ""}, + {"", 3, 0, false, 0, 30, 0, 0, "", ""}, + {"", 3, 0, false, 0, 31, 0, 0, "", ""}, + {"", 3, 0, false, 0, 32, 0, 0, "", ""}, + {"", 3, 0, false, 0, 33, 0, 0, "", ""}, + {"init.c", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"initfini.c", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"call_gmon_start", 2, 0, false, 0, 13, 4195340, 0, "", ""}, + {"crtstuff.c", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"__CTOR_LIST__", 1, 0, false, 0, 18, 6293128, 0, "", ""}, + {"__DTOR_LIST__", 1, 0, false, 0, 19, 6293144, 0, "", ""}, + {"__JCR_LIST__", 1, 0, false, 0, 20, 6293160, 0, "", ""}, + {"__do_global_dtors_aux", 2, 0, false, 0, 13, 4195376, 0, "", ""}, + {"completed.6183", 1, 0, false, 0, 25, 6293656, 1, "", ""}, + {"p.6181", 1, 0, false, 0, 24, 6293648, 0, "", ""}, + {"frame_dummy", 2, 0, false, 0, 13, 4195440, 0, "", ""}, + {"crtstuff.c", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"__CTOR_END__", 1, 0, false, 0, 18, 6293136, 0, "", ""}, + {"__DTOR_END__", 1, 0, false, 0, 19, 6293152, 0, "", ""}, + {"__FRAME_END__", 1, 0, false, 0, 17, 4195968, 0, "", ""}, + {"__JCR_END__", 1, 0, false, 0, 20, 6293160, 0, "", ""}, + {"__do_global_ctors_aux", 2, 0, false, 0, 13, 4195680, 0, "", ""}, + {"initfini.c", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"hello.c", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"_GLOBAL_OFFSET_TABLE_", 1, 2, false, 0, 23, 6293592, 0, "", ""}, + {"__init_array_end", 0, 2, false, 0, 18, 6293124, 0, "", ""}, + {"__init_array_start", 0, 2, false, 0, 18, 6293124, 0, "", ""}, + {"_DYNAMIC", 1, 2, false, 0, 21, 6293168, 0, "", ""}, + {"data_start", 32, 0, false, 0, 24, 6293632, 0, "", ""}, + {"__libc_csu_fini", 18, 0, false, 0, 13, 4195520, 2, "", ""}, + {"_start", 18, 0, false, 0, 13, 4195296, 0, "", ""}, + {"__gmon_start__", 32, 0, false, 0, 0, 0, 0, "", ""}, + {"_Jv_RegisterClasses", 32, 0, false, 0, 0, 0, 0, "", ""}, + {"puts@@GLIBC_2.2.5", 18, 0, false, 0, 0, 0, 396, "", ""}, + {"_fini", 18, 0, false, 0, 14, 4195732, 0, "", ""}, + {"__libc_start_main@@GLIBC_2.2.5", 18, 0, false, 0, 0, 0, 450, "", ""}, + {"_IO_stdin_used", 17, 0, false, 0, 15, 4195748, 4, "", ""}, + {"__data_start", 16, 0, false, 0, 24, 6293632, 0, "", ""}, + {"__dso_handle", 17, 2, false, 0, 24, 6293640, 0, "", ""}, + {"__libc_csu_init", 18, 0, false, 0, 13, 4195536, 137, "", ""}, + {"__bss_start", 16, 0, false, 0, 65521, 6293656, 0, "", ""}, + {"_end", 16, 0, false, 0, 65521, 6293664, 0, "", ""}, + {"_edata", 16, 0, false, 0, 65521, 6293656, 0, "", ""}, + {"main", 18, 0, false, 0, 13, 4195480, 27, "", ""}, + {"_init", 18, 0, false, 0, 11, 4195224, 0, "", ""}, }, }, { @@ -338,21 +338,21 @@ var fileTests = []fileTest{ []ProgHeader{}, nil, []Symbol{ - {"hello.c", 4, 0, 65521, 0, 0, "", ""}, - {"", 3, 0, 1, 0, 0, "", ""}, - {"", 3, 0, 3, 0, 0, "", ""}, - {"", 3, 0, 4, 0, 0, "", ""}, - {"", 3, 0, 5, 0, 0, "", ""}, - {"", 3, 0, 6, 0, 0, "", ""}, - {"", 3, 0, 8, 0, 0, "", ""}, - {"", 3, 0, 9, 0, 0, "", ""}, - {"", 3, 0, 11, 0, 0, "", ""}, - {"", 3, 0, 13, 0, 0, "", ""}, - {"", 3, 0, 15, 0, 0, "", ""}, - {"", 3, 0, 16, 0, 0, "", ""}, - {"", 3, 0, 14, 0, 0, "", ""}, - {"main", 18, 0, 1, 0, 23, "", ""}, - {"puts", 16, 0, 0, 0, 0, "", ""}, + {"hello.c", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"", 3, 0, false, 0, 1, 0, 0, "", ""}, + {"", 3, 0, false, 0, 3, 0, 0, "", ""}, + {"", 3, 0, false, 0, 4, 0, 0, "", ""}, + {"", 3, 0, false, 0, 5, 0, 0, "", ""}, + {"", 3, 0, false, 0, 6, 0, 0, "", ""}, + {"", 3, 0, false, 0, 8, 0, 0, "", ""}, + {"", 3, 0, false, 0, 9, 0, 0, "", ""}, + {"", 3, 0, false, 0, 11, 0, 0, "", ""}, + {"", 3, 0, false, 0, 13, 0, 0, "", ""}, + {"", 3, 0, false, 0, 15, 0, 0, "", ""}, + {"", 3, 0, false, 0, 16, 0, 0, "", ""}, + {"", 3, 0, false, 0, 14, 0, 0, "", ""}, + {"main", 18, 0, false, 0, 1, 0, 23, "", ""}, + {"puts", 16, 0, false, 0, 0, 0, 0, "", ""}, }, }, { @@ -384,21 +384,21 @@ var fileTests = []fileTest{ []ProgHeader{}, nil, []Symbol{ - {"hello.c", 4, 0, 65521, 0, 0, "", ""}, - {"", 3, 0, 1, 0, 0, "", ""}, - {"", 3, 0, 3, 0, 0, "", ""}, - {"", 3, 0, 4, 0, 0, "", ""}, - {"", 3, 0, 5, 0, 0, "", ""}, - {"", 3, 0, 6, 0, 0, "", ""}, - {"", 3, 0, 8, 0, 0, "", ""}, - {"", 3, 0, 9, 0, 0, "", ""}, - {"", 3, 0, 11, 0, 0, "", ""}, - {"", 3, 0, 13, 0, 0, "", ""}, - {"", 3, 0, 15, 0, 0, "", ""}, - {"", 3, 0, 16, 0, 0, "", ""}, - {"", 3, 0, 14, 0, 0, "", ""}, - {"main", 18, 0, 1, 0, 27, "", ""}, - {"puts", 16, 0, 0, 0, 0, "", ""}, + {"hello.c", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"", 3, 0, false, 0, 1, 0, 0, "", ""}, + {"", 3, 0, false, 0, 3, 0, 0, "", ""}, + {"", 3, 0, false, 0, 4, 0, 0, "", ""}, + {"", 3, 0, false, 0, 5, 0, 0, "", ""}, + {"", 3, 0, false, 0, 6, 0, 0, "", ""}, + {"", 3, 0, false, 0, 8, 0, 0, "", ""}, + {"", 3, 0, false, 0, 9, 0, 0, "", ""}, + {"", 3, 0, false, 0, 11, 0, 0, "", ""}, + {"", 3, 0, false, 0, 13, 0, 0, "", ""}, + {"", 3, 0, false, 0, 15, 0, 0, "", ""}, + {"", 3, 0, false, 0, 16, 0, 0, "", ""}, + {"", 3, 0, false, 0, 14, 0, 0, "", ""}, + {"main", 18, 0, false, 0, 1, 0, 27, "", ""}, + {"puts", 16, 0, false, 0, 0, 0, 0, "", ""}, }, }, { @@ -430,23 +430,69 @@ var fileTests = []fileTest{ []ProgHeader{}, nil, []Symbol{ - {"hello.c", 4, 0, 65521, 0, 0, "", ""}, - {"", 3, 0, 1, 0, 0, "", ""}, - {"", 3, 0, 3, 0, 0, "", ""}, - {"", 3, 0, 4, 0, 0, "", ""}, - {"", 3, 0, 5, 0, 0, "", ""}, - {"", 3, 0, 6, 0, 0, "", ""}, - {"", 3, 0, 8, 0, 0, "", ""}, - {"", 3, 0, 9, 0, 0, "", ""}, - {"", 3, 0, 11, 0, 0, "", ""}, - {"", 3, 0, 13, 0, 0, "", ""}, - {"", 3, 0, 15, 0, 0, "", ""}, - {"", 3, 0, 16, 0, 0, "", ""}, - {"", 3, 0, 14, 0, 0, "", ""}, - {"main", 18, 0, 1, 0, 44, "", ""}, - {"puts", 16, 0, 0, 0, 0, "", ""}, + {"hello.c", 4, 0, false, 0, 65521, 0, 0, "", ""}, + {"", 3, 0, false, 0, 1, 0, 0, "", ""}, + {"", 3, 0, false, 0, 3, 0, 0, "", ""}, + {"", 3, 0, false, 0, 4, 0, 0, "", ""}, + {"", 3, 0, false, 0, 5, 0, 0, "", ""}, + {"", 3, 0, false, 0, 6, 0, 0, "", ""}, + {"", 3, 0, false, 0, 8, 0, 0, "", ""}, + {"", 3, 0, false, 0, 9, 0, 0, "", ""}, + {"", 3, 0, false, 0, 11, 0, 0, "", ""}, + {"", 3, 0, false, 0, 13, 0, 0, "", ""}, + {"", 3, 0, false, 0, 15, 0, 0, "", ""}, + {"", 3, 0, false, 0, 16, 0, 0, "", ""}, + {"", 3, 0, false, 0, 14, 0, 0, "", ""}, + {"main", 18, 0, false, 0, 1, 0, 44, "", ""}, + {"puts", 16, 0, false, 0, 0, 0, 0, "", ""}, }, }, + { + "testdata/gcc-riscv64-linux-exec", + FileHeader{ELFCLASS64, ELFDATA2LSB, EV_CURRENT, ELFOSABI_NONE, 0, binary.LittleEndian, ET_EXEC, EM_RISCV, 0x10460}, + []SectionHeader{ + {"", SHT_NULL, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + {".interp", SHT_PROGBITS, SHF_ALLOC, 0x10270, 0x270, 0x21, 0x0, 0x0, 0x1, 0x0, 0x21}, + {".note.gnu.build-id", SHT_NOTE, SHF_ALLOC, 0x10294, 0x294, 0x24, 0x0, 0x0, 0x4, 0x0, 0x24}, + {".note.ABI-tag", SHT_NOTE, SHF_ALLOC, 0x102b8, 0x2b8, 0x20, 0x0, 0x0, 0x4, 0x0, 0x20}, + {".gnu.hash", SHT_GNU_HASH, SHF_ALLOC, 0x102d8, 0x2d8, 0x30, 0x5, 0x0, 0x8, 0x0, 0x30}, + {".dynsym", SHT_DYNSYM, SHF_ALLOC, 0x10308, 0x308, 0x60, 0x6, 0x1, 0x8, 0x18, 0x60}, + {".dynstr", SHT_STRTAB, SHF_ALLOC, 0x10368, 0x368, 0x4a, 0x0, 0x0, 0x1, 0x0, 0x4a}, + {".gnu.version", SHT_GNU_VERSYM, SHF_ALLOC, 0x103b2, 0x3b2, 0x8, 0x5, 0x0, 0x2, 0x2, 0x8}, + {".gnu.version_r", SHT_GNU_VERNEED, SHF_ALLOC, 0x103c0, 0x3c0, 0x30, 0x6, 0x1, 0x8, 0x0, 0x30}, + {".rela.plt", SHT_RELA, SHF_ALLOC + SHF_INFO_LINK, 0x103f0, 0x3f0, 0x30, 0x5, 0x14, 0x8, 0x18, 0x30}, + {".plt", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x10420, 0x420, 0x40, 0x0, 0x0, 0x10, 0x10, 0x40}, + {".text", SHT_PROGBITS, SHF_ALLOC + SHF_EXECINSTR, 0x10460, 0x460, 0xd8, 0x0, 0x0, 0x4, 0x0, 0xd8}, + {".rodata", SHT_PROGBITS, SHF_ALLOC, 0x10538, 0x538, 0x15, 0x0, 0x0, 0x8, 0x0, 0x15}, + {".eh_frame_hdr", SHT_PROGBITS, SHF_ALLOC, 0x10550, 0x550, 0x24, 0x0, 0x0, 0x4, 0x0, 0x24}, + {".eh_frame", SHT_PROGBITS, SHF_ALLOC, 0x10578, 0x578, 0x6c, 0x0, 0x0, 0x8, 0x0, 0x6c}, + {".preinit_array", SHT_PREINIT_ARRAY, SHF_WRITE + SHF_ALLOC, 0x11e00, 0xe00, 0x8, 0x0, 0x0, 0x1, 0x8, 0x8}, + {".init_array", SHT_INIT_ARRAY, SHF_WRITE + SHF_ALLOC, 0x11e08, 0xe08, 0x8, 0x0, 0x0, 0x8, 0x8, 0x8}, + {".fini_array", SHT_FINI_ARRAY, SHF_WRITE + SHF_ALLOC, 0x11e10, 0xe10, 0x8, 0x0, 0x0, 0x8, 0x8, 0x8}, + {".dynamic", SHT_DYNAMIC, SHF_WRITE + SHF_ALLOC, 0x11e18, 0xe18, 0x1d0, 0x6, 0x0, 0x8, 0x10, 0x1d0}, + {".got", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x11fe8, 0xfe8, 0x8, 0x0, 0x0, 0x8, 0x8, 0x8}, + {".got.plt", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x11ff0, 0xff0, 0x20, 0x0, 0x0, 0x8, 0x8, 0x20}, + {".sdata", SHT_PROGBITS, SHF_WRITE + SHF_ALLOC, 0x12010, 0x1010, 0x8, 0x0, 0x0, 0x8, 0x0, 0x8}, + {".bss", SHT_NOBITS, SHF_WRITE + SHF_ALLOC, 0x12018, 0x1018, 0x8, 0x0, 0x0, 0x1, 0x0, 0x8}, + {".comment", SHT_PROGBITS, SHF_MERGE + SHF_STRINGS, 0x0, 0x1018, 0x26, 0x0, 0x0, 0x1, 0x1, 0x26}, + {".riscv.attributes", SHT_RISCV_ATTRIBUTES, 0x0, 0x0, 0x103e, 0x66, 0x0, 0x0, 0x1, 0x0, 0x66}, + {".shstrtab", SHT_STRTAB, 0x0, 0x0, 0x10a4, 0xff, 0x0, 0x0, 0x1, 0x0, 0xff}, + }, + []ProgHeader{ + {PT_PHDR, PF_R, 0x40, 0x10040, 0x10040, 0x230, 0x230, 0x8}, + {PT_INTERP, PF_R, 0x270, 0x10270, 0x10270, 0x21, 0x21, 0x1}, + {PT_RISCV_ATTRIBUTES, PF_R, 0x103e, 0x0, 0x0, 0x66, 0x0, 0x1}, + {PT_LOAD, PF_X + PF_R, 0x0, 0x10000, 0x10000, 0x5e4, 0x5e4, 0x1000}, + {PT_LOAD, PF_W + PF_R, 0xe00, 0x11e00, 0x11e00, 0x218, 0x220, 0x1000}, + {PT_DYNAMIC, PF_W + PF_R, 0xe18, 0x11e18, 0x11e18, 0x1d0, 0x1d0, 0x8}, + {PT_NOTE, PF_R, 0x294, 0x10294, 0x10294, 0x44, 0x44, 0x4}, + {PT_GNU_EH_FRAME, PF_R, 0x550, 0x10550, 0x10550, 0x24, 0x24, 0x4}, + {PT_GNU_STACK, PF_W + PF_R, 0x0, 0x0, 0x0, 0x0, 0x0, 0x10}, + {PT_GNU_RELRO, PF_R, 0xe00, 0x11e00, 0x11e00, 0x200, 0x200, 0x1}, + }, + []string{"libc.so.6"}, + nil, + }, } func TestOpen(t *testing.T) { @@ -994,7 +1040,6 @@ var relocationTests = []relocationTest{ func TestDWARFRelocations(t *testing.T) { for _, test := range relocationTests { - test := test t.Run(test.file, func(t *testing.T) { t.Parallel() f, err := Open(test.file) diff --git a/src/debug/elf/symbols_test.go b/src/debug/elf/symbols_test.go index 42f02312e871dd..6053d99acc1a0b 100644 --- a/src/debug/elf/symbols_test.go +++ b/src/debug/elf/symbols_test.go @@ -37,8 +37,21 @@ func TestSymbols(t *testing.T) { fs = []Symbol{} } if !reflect.DeepEqual(ts, fs) { - t.Errorf("%s: Symbols = %v, want %v", file, ts, fs) + t.Errorf("%s: Symbols = %v, want %v", file, fs, ts) } + + for i, s := range fs { + if s.HasVersion { + // No hidden versions here. + if s.VersionIndex.IsHidden() { + t.Errorf("%s: symbol %d: unexpected hidden version", file, i) + } + if got, want := s.VersionIndex.Index(), uint16(s.VersionIndex); got != want { + t.Errorf("%s: symbol %d: VersionIndex.Index() == %d, want %d", file, i, got, want) + } + } + } + } for file, ts := range symbolsGolden { do(file, ts, (*File).Symbols) @@ -53,750 +66,936 @@ func TestSymbols(t *testing.T) { var symbolsGolden = map[string][]Symbol{ "testdata/gcc-amd64-linux-exec": { Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x1, - Value: 0x400200, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x2, - Value: 0x40021C, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x3, - Value: 0x400240, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x4, - Value: 0x400268, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x5, - Value: 0x400288, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x6, - Value: 0x4002E8, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x7, - Value: 0x400326, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x8, - Value: 0x400330, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x9, - Value: 0x400350, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0xA, - Value: 0x400368, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0xB, - Value: 0x400398, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0xC, - Value: 0x4003B0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0xD, - Value: 0x4003E0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0xE, - Value: 0x400594, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0xF, - Value: 0x4005A4, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x10, - Value: 0x4005B8, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x11, - Value: 0x4005E0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x12, - Value: 0x600688, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x13, - Value: 0x600698, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x14, - Value: 0x6006A8, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x15, - Value: 0x6006B0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x16, - Value: 0x600850, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x17, - Value: 0x600858, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x18, - Value: 0x600880, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x19, - Value: 0x600898, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x1A, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x1B, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x1C, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x1D, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x1E, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x1F, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x20, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x21, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "init.c", - Info: 0x4, - Other: 0x0, - Section: 0xFFF1, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "initfini.c", - Info: 0x4, - Other: 0x0, - Section: 0xFFF1, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "call_gmon_start", - Info: 0x2, - Other: 0x0, - Section: 0xD, - Value: 0x40040C, - Size: 0x0, - }, - Symbol{ - Name: "crtstuff.c", - Info: 0x4, - Other: 0x0, - Section: 0xFFF1, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "__CTOR_LIST__", - Info: 0x1, - Other: 0x0, - Section: 0x12, - Value: 0x600688, - Size: 0x0, - }, - Symbol{ - Name: "__DTOR_LIST__", - Info: 0x1, - Other: 0x0, - Section: 0x13, - Value: 0x600698, - Size: 0x0, - }, - Symbol{ - Name: "__JCR_LIST__", - Info: 0x1, - Other: 0x0, - Section: 0x14, - Value: 0x6006A8, - Size: 0x0, - }, - Symbol{ - Name: "__do_global_dtors_aux", - Info: 0x2, - Other: 0x0, - Section: 0xD, - Value: 0x400430, - Size: 0x0, - }, - Symbol{ - Name: "completed.6183", - Info: 0x1, - Other: 0x0, - Section: 0x19, - Value: 0x600898, - Size: 0x1, - }, - Symbol{ - Name: "p.6181", - Info: 0x1, - Other: 0x0, - Section: 0x18, - Value: 0x600890, - Size: 0x0, - }, - Symbol{ - Name: "frame_dummy", - Info: 0x2, - Other: 0x0, - Section: 0xD, - Value: 0x400470, - Size: 0x0, - }, - Symbol{ - Name: "crtstuff.c", - Info: 0x4, - Other: 0x0, - Section: 0xFFF1, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "__CTOR_END__", - Info: 0x1, - Other: 0x0, - Section: 0x12, - Value: 0x600690, - Size: 0x0, - }, - Symbol{ - Name: "__DTOR_END__", - Info: 0x1, - Other: 0x0, - Section: 0x13, - Value: 0x6006A0, - Size: 0x0, - }, - Symbol{ - Name: "__FRAME_END__", - Info: 0x1, - Other: 0x0, - Section: 0x11, - Value: 0x400680, - Size: 0x0, - }, - Symbol{ - Name: "__JCR_END__", - Info: 0x1, - Other: 0x0, - Section: 0x14, - Value: 0x6006A8, - Size: 0x0, - }, - Symbol{ - Name: "__do_global_ctors_aux", - Info: 0x2, - Other: 0x0, - Section: 0xD, - Value: 0x400560, - Size: 0x0, - }, - Symbol{ - Name: "initfini.c", - Info: 0x4, - Other: 0x0, - Section: 0xFFF1, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "hello.c", - Info: 0x4, - Other: 0x0, - Section: 0xFFF1, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "_GLOBAL_OFFSET_TABLE_", - Info: 0x1, - Other: 0x2, - Section: 0x17, - Value: 0x600858, - Size: 0x0, - }, - Symbol{ - Name: "__init_array_end", - Info: 0x0, - Other: 0x2, - Section: 0x12, - Value: 0x600684, - Size: 0x0, - }, - Symbol{ - Name: "__init_array_start", - Info: 0x0, - Other: 0x2, - Section: 0x12, - Value: 0x600684, - Size: 0x0, - }, - Symbol{ - Name: "_DYNAMIC", - Info: 0x1, - Other: 0x2, - Section: 0x15, - Value: 0x6006B0, - Size: 0x0, - }, - Symbol{ - Name: "data_start", - Info: 0x20, - Other: 0x0, - Section: 0x18, - Value: 0x600880, - Size: 0x0, - }, - Symbol{ - Name: "__libc_csu_fini", - Info: 0x12, - Other: 0x0, - Section: 0xD, - Value: 0x4004C0, - Size: 0x2, - }, - Symbol{ - Name: "_start", - Info: 0x12, - Other: 0x0, - Section: 0xD, - Value: 0x4003E0, - Size: 0x0, - }, - Symbol{ - Name: "__gmon_start__", - Info: 0x20, - Other: 0x0, - Section: 0x0, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "_Jv_RegisterClasses", - Info: 0x20, - Other: 0x0, - Section: 0x0, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "puts@@GLIBC_2.2.5", - Info: 0x12, - Other: 0x0, - Section: 0x0, - Value: 0x0, - Size: 0x18C, - }, - Symbol{ - Name: "_fini", - Info: 0x12, - Other: 0x0, - Section: 0xE, - Value: 0x400594, - Size: 0x0, - }, - Symbol{ - Name: "__libc_start_main@@GLIBC_2.2.5", - Info: 0x12, - Other: 0x0, - Section: 0x0, - Value: 0x0, - Size: 0x1C2, - }, - Symbol{ - Name: "_IO_stdin_used", - Info: 0x11, - Other: 0x0, - Section: 0xF, - Value: 0x4005A4, - Size: 0x4, - }, - Symbol{ - Name: "__data_start", - Info: 0x10, - Other: 0x0, - Section: 0x18, - Value: 0x600880, - Size: 0x0, - }, - Symbol{ - Name: "__dso_handle", - Info: 0x11, - Other: 0x2, - Section: 0x18, - Value: 0x600888, - Size: 0x0, - }, - Symbol{ - Name: "__libc_csu_init", - Info: 0x12, - Other: 0x0, - Section: 0xD, - Value: 0x4004D0, - Size: 0x89, - }, - Symbol{ - Name: "__bss_start", - Info: 0x10, - Other: 0x0, - Section: 0xFFF1, - Value: 0x600898, - Size: 0x0, - }, - Symbol{ - Name: "_end", - Info: 0x10, - Other: 0x0, - Section: 0xFFF1, - Value: 0x6008A0, - Size: 0x0, - }, - Symbol{ - Name: "_edata", - Info: 0x10, - Other: 0x0, - Section: 0xFFF1, - Value: 0x600898, - Size: 0x0, - }, - Symbol{ - Name: "main", - Info: 0x12, - Other: 0x0, - Section: 0xD, - Value: 0x400498, - Size: 0x1B, - }, - Symbol{ - Name: "_init", - Info: 0x12, - Other: 0x0, - Section: 0xB, - Value: 0x400398, - Size: 0x0, + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x1, + Value: 0x400200, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x2, + Value: 0x40021C, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x3, + Value: 0x400240, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x4, + Value: 0x400268, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x5, + Value: 0x400288, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x6, + Value: 0x4002E8, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x7, + Value: 0x400326, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x8, + Value: 0x400330, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x9, + Value: 0x400350, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xA, + Value: 0x400368, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xB, + Value: 0x400398, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xC, + Value: 0x4003B0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xD, + Value: 0x4003E0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xE, + Value: 0x400594, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xF, + Value: 0x4005A4, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x10, + Value: 0x4005B8, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x11, + Value: 0x4005E0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x12, + Value: 0x600688, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x13, + Value: 0x600698, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x14, + Value: 0x6006A8, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x15, + Value: 0x6006B0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x16, + Value: 0x600850, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x17, + Value: 0x600858, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x18, + Value: 0x600880, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x19, + Value: 0x600898, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x1A, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x1B, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x1C, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x1D, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x1E, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x1F, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x20, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x21, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "init.c", + Info: 0x4, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xFFF1, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "initfini.c", + Info: 0x4, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xFFF1, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "call_gmon_start", + Info: 0x2, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xD, + Value: 0x40040C, + Size: 0x0, + }, + Symbol{ + Name: "crtstuff.c", + Info: 0x4, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xFFF1, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "__CTOR_LIST__", + Info: 0x1, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x12, + Value: 0x600688, + Size: 0x0, + }, + Symbol{ + Name: "__DTOR_LIST__", + Info: 0x1, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x13, + Value: 0x600698, + Size: 0x0, + }, + Symbol{ + Name: "__JCR_LIST__", + Info: 0x1, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x14, + Value: 0x6006A8, + Size: 0x0, + }, + Symbol{ + Name: "__do_global_dtors_aux", + Info: 0x2, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xD, + Value: 0x400430, + Size: 0x0, + }, + Symbol{ + Name: "completed.6183", + Info: 0x1, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x19, + Value: 0x600898, + Size: 0x1, + }, + Symbol{ + Name: "p.6181", + Info: 0x1, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x18, + Value: 0x600890, + Size: 0x0, + }, + Symbol{ + Name: "frame_dummy", + Info: 0x2, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xD, + Value: 0x400470, + Size: 0x0, + }, + Symbol{ + Name: "crtstuff.c", + Info: 0x4, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xFFF1, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "__CTOR_END__", + Info: 0x1, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x12, + Value: 0x600690, + Size: 0x0, + }, + Symbol{ + Name: "__DTOR_END__", + Info: 0x1, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x13, + Value: 0x6006A0, + Size: 0x0, + }, + Symbol{ + Name: "__FRAME_END__", + Info: 0x1, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x11, + Value: 0x400680, + Size: 0x0, + }, + Symbol{ + Name: "__JCR_END__", + Info: 0x1, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x14, + Value: 0x6006A8, + Size: 0x0, + }, + Symbol{ + Name: "__do_global_ctors_aux", + Info: 0x2, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xD, + Value: 0x400560, + Size: 0x0, + }, + Symbol{ + Name: "initfini.c", + Info: 0x4, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xFFF1, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "hello.c", + Info: 0x4, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xFFF1, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "_GLOBAL_OFFSET_TABLE_", + Info: 0x1, + Other: 0x2, + HasVersion: false, + VersionIndex: 0, + Section: 0x17, + Value: 0x600858, + Size: 0x0, + }, + Symbol{ + Name: "__init_array_end", + Info: 0x0, + Other: 0x2, + HasVersion: false, + VersionIndex: 0, + Section: 0x12, + Value: 0x600684, + Size: 0x0, + }, + Symbol{ + Name: "__init_array_start", + Info: 0x0, + Other: 0x2, + HasVersion: false, + VersionIndex: 0, + Section: 0x12, + Value: 0x600684, + Size: 0x0, + }, + Symbol{ + Name: "_DYNAMIC", + Info: 0x1, + Other: 0x2, + HasVersion: false, + VersionIndex: 0, + Section: 0x15, + Value: 0x6006B0, + Size: 0x0, + }, + Symbol{ + Name: "data_start", + Info: 0x20, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x18, + Value: 0x600880, + Size: 0x0, + }, + Symbol{ + Name: "__libc_csu_fini", + Info: 0x12, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xD, + Value: 0x4004C0, + Size: 0x2, + }, + Symbol{ + Name: "_start", + Info: 0x12, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xD, + Value: 0x4003E0, + Size: 0x0, + }, + Symbol{ + Name: "__gmon_start__", + Info: 0x20, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x0, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "_Jv_RegisterClasses", + Info: 0x20, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x0, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "puts@@GLIBC_2.2.5", + Info: 0x12, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x0, + Value: 0x0, + Size: 0x18C, + }, + Symbol{ + Name: "_fini", + Info: 0x12, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xE, + Value: 0x400594, + Size: 0x0, + }, + Symbol{ + Name: "__libc_start_main@@GLIBC_2.2.5", + Info: 0x12, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x0, + Value: 0x0, + Size: 0x1C2, + }, + Symbol{ + Name: "_IO_stdin_used", + Info: 0x11, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xF, + Value: 0x4005A4, + Size: 0x4, + }, + Symbol{ + Name: "__data_start", + Info: 0x10, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x18, + Value: 0x600880, + Size: 0x0, + }, + Symbol{ + Name: "__dso_handle", + Info: 0x11, + Other: 0x2, + HasVersion: false, + VersionIndex: 0, + Section: 0x18, + Value: 0x600888, + Size: 0x0, + }, + Symbol{ + Name: "__libc_csu_init", + Info: 0x12, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xD, + Value: 0x4004D0, + Size: 0x89, + }, + Symbol{ + Name: "__bss_start", + Info: 0x10, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xFFF1, + Value: 0x600898, + Size: 0x0, + }, + Symbol{ + Name: "_end", + Info: 0x10, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xFFF1, + Value: 0x6008A0, + Size: 0x0, + }, + Symbol{ + Name: "_edata", + Info: 0x10, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xFFF1, + Value: 0x600898, + Size: 0x0, + }, + Symbol{ + Name: "main", + Info: 0x12, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xD, + Value: 0x400498, + Size: 0x1B, + }, + Symbol{ + Name: "_init", + Info: 0x12, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xB, + Value: 0x400398, + Size: 0x0, }, }, "testdata/go-relocation-test-clang-x86.obj": { Symbol{ - Name: "go-relocation-test-clang.c", - Info: 0x4, - Other: 0x0, - Section: 0xFFF1, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: ".Linfo_string0", - Info: 0x0, - Other: 0x0, - Section: 0xC, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: ".Linfo_string1", - Info: 0x0, - Other: 0x0, - Section: 0xC, - Value: 0x2C, - Size: 0x0, - }, - Symbol{ - Name: ".Linfo_string2", - Info: 0x0, - Other: 0x0, - Section: 0xC, - Value: 0x47, - Size: 0x0, - }, - Symbol{ - Name: ".Linfo_string3", - Info: 0x0, - Other: 0x0, - Section: 0xC, - Value: 0x4C, - Size: 0x0, - }, - Symbol{ - Name: ".Linfo_string4", - Info: 0x0, - Other: 0x0, - Section: 0xC, - Value: 0x4E, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x1, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x2, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x3, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x4, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x6, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x7, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x8, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0xA, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0xC, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0xD, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0xE, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0xF, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "", - Info: 0x3, - Other: 0x0, - Section: 0x10, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "v", - Info: 0x11, - Other: 0x0, - Section: 0xFFF2, - Value: 0x4, - Size: 0x4, + Name: "go-relocation-test-clang.c", + Info: 0x4, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xFFF1, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: ".Linfo_string0", + Info: 0x0, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xC, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: ".Linfo_string1", + Info: 0x0, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xC, + Value: 0x2C, + Size: 0x0, + }, + Symbol{ + Name: ".Linfo_string2", + Info: 0x0, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xC, + Value: 0x47, + Size: 0x0, + }, + Symbol{ + Name: ".Linfo_string3", + Info: 0x0, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xC, + Value: 0x4C, + Size: 0x0, + }, + Symbol{ + Name: ".Linfo_string4", + Info: 0x0, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xC, + Value: 0x4E, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x1, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x2, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x3, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x4, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x6, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x7, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x8, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xA, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xC, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xD, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xE, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xF, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "", + Info: 0x3, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0x10, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "v", + Info: 0x11, + Other: 0x0, + HasVersion: false, + VersionIndex: 0, + Section: 0xFFF2, + Value: 0x4, + Size: 0x4, }, }, "testdata/hello-world-core.gz": {}, @@ -805,34 +1004,324 @@ var symbolsGolden = map[string][]Symbol{ var dynamicSymbolsGolden = map[string][]Symbol{ "testdata/gcc-amd64-linux-exec": { Symbol{ - Name: "__gmon_start__", - Info: 0x20, - Other: 0x0, - Section: 0x0, - Value: 0x0, - Size: 0x0, - }, - Symbol{ - Name: "puts", - Info: 0x12, - Other: 0x0, - Section: 0x0, - Value: 0x0, - Size: 0x18C, - Version: "GLIBC_2.2.5", - Library: "libc.so.6", - }, - Symbol{ - Name: "__libc_start_main", - Info: 0x12, - Other: 0x0, - Section: 0x0, - Value: 0x0, - Size: 0x1C2, - Version: "GLIBC_2.2.5", - Library: "libc.so.6", + Name: "__gmon_start__", + Info: 0x20, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x0, + Section: 0x0, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "puts", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x2, + Section: 0x0, + Value: 0x0, + Size: 0x18C, + Version: "GLIBC_2.2.5", + Library: "libc.so.6", + }, + Symbol{ + Name: "__libc_start_main", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x2, + Section: 0x0, + Value: 0x0, + Size: 0x1C2, + Version: "GLIBC_2.2.5", + Library: "libc.so.6", }, }, "testdata/go-relocation-test-clang-x86.obj": {}, "testdata/hello-world-core.gz": {}, + "testdata/libtiffxx.so_": { + Symbol{ + Name: "_ZNSo3putEc", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x3, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBCXX_3.4", + Library: "libstdc++.so.6", + }, + Symbol{ + Name: "strchr", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x4, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBC_2.2.5", + Library: "libc.so.6", + }, + Symbol{ + Name: "__cxa_finalize", + Info: 0x22, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x4, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBC_2.2.5", + Library: "libc.so.6", + }, + Symbol{ + Name: "_ZNSo5tellpEv", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x3, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBCXX_3.4", + Library: "libstdc++.so.6", + }, + Symbol{ + Name: "_ZNSo5seekpElSt12_Ios_Seekdir", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x3, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBCXX_3.4", + Library: "libstdc++.so.6", + }, + Symbol{ + Name: "_Znwm", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x3, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBCXX_3.4", + Library: "libstdc++.so.6", + }, + Symbol{ + Name: "_ZdlPvm", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x5, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "CXXABI_1.3.9", + Library: "libstdc++.so.6", + }, + Symbol{ + Name: "__stack_chk_fail", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x6, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBC_2.4", + Library: "libc.so.6", + }, + Symbol{ + Name: "_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x7, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBCXX_3.4.9", + Library: "libstdc++.so.6", + }, + Symbol{ + Name: "_ZNSo5seekpESt4fposI11__mbstate_tE", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x3, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBCXX_3.4", + Library: "libstdc++.so.6", + }, + Symbol{ + Name: "_ZNSi4readEPcl", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x3, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBCXX_3.4", + Library: "libstdc++.so.6", + }, + Symbol{ + Name: "_ZNSi5seekgESt4fposI11__mbstate_tE", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x3, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBCXX_3.4", + Library: "libstdc++.so.6", + }, + Symbol{ + Name: "_ZNSo5writeEPKcl", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x3, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBCXX_3.4", + Library: "libstdc++.so.6", + }, + Symbol{ + Name: "_ZNSi5seekgElSt12_Ios_Seekdir", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x3, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBCXX_3.4", + Library: "libstdc++.so.6", + }, + Symbol{ + Name: "_ZSt21ios_base_library_initv", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x8, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBCXX_3.4.32", + Library: "libstdc++.so.6", + }, + Symbol{ + Name: "TIFFClientOpen", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x9, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "LIBTIFF_4.0", + Library: "libtiff.so.6", + }, + Symbol{ + Name: "_ZNSt9basic_iosIcSt11char_traitsIcEE5clearESt12_Ios_Iostate", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x3, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBCXX_3.4", + Library: "libstdc++.so.6", + }, + Symbol{ + Name: "_ZNSi5tellgEv", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x3, + Section: 0x0, + Value: 0x0, + Size: 0x0, + Version: "GLIBCXX_3.4", + Library: "libstdc++.so.6", + }, + Symbol{ + Name: "_ITM_deregisterTMCloneTable", + Info: 0x20, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x1, + Section: 0x0, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "__gmon_start__", + Info: 0x20, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x1, + Section: 0x0, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "_ITM_registerTMCloneTable", + Info: 0x20, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x1, + Section: 0x0, + Value: 0x0, + Size: 0x0, + }, + Symbol{ + Name: "LIBTIFFXX_4.0", + Info: 0x11, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x2, + Section: 0xFFF1, + Value: 0x0, + Size: 0x0, + Version: "LIBTIFFXX_4.0", + Library: "", + }, + Symbol{ + Name: "_Z14TIFFStreamOpenPKcPSo", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x2, + Section: 0xF, + Value: 0x1860, + Size: 0xB8, + Version: "LIBTIFFXX_4.0", + Library: "", + }, + Symbol{ + Name: "_Z14TIFFStreamOpenPKcPSi", + Info: 0x12, + Other: 0x0, + HasVersion: true, + VersionIndex: 0x2, + Section: 0xF, + Value: 0x1920, + Size: 0x13, + Version: "LIBTIFFXX_4.0", + Library: "", + }, + }, } diff --git a/src/debug/elf/testdata/gcc-riscv64-linux-exec b/src/debug/elf/testdata/gcc-riscv64-linux-exec new file mode 100644 index 00000000000000..e01f6f292c9406 Binary files /dev/null and b/src/debug/elf/testdata/gcc-riscv64-linux-exec differ diff --git a/src/debug/elf/testdata/libtiffxx.so_ b/src/debug/elf/testdata/libtiffxx.so_ new file mode 100644 index 00000000000000..403525715183c5 Binary files /dev/null and b/src/debug/elf/testdata/libtiffxx.so_ differ diff --git a/src/debug/macho/file.go b/src/debug/macho/file.go index 7b11bb29543543..8a4088828415b1 100644 --- a/src/debug/macho/file.go +++ b/src/debug/macho/file.go @@ -610,15 +610,33 @@ func (f *File) Section(name string) *Section { // DWARF returns the DWARF debug information for the Mach-O file. func (f *File) DWARF() (*dwarf.Data, error) { dwarfSuffix := func(s *Section) string { + sectname := s.Name + var pfx int switch { - case strings.HasPrefix(s.Name, "__debug_"): - return s.Name[8:] - case strings.HasPrefix(s.Name, "__zdebug_"): - return s.Name[9:] + case strings.HasPrefix(sectname, "__debug_"): + pfx = 8 + case strings.HasPrefix(sectname, "__zdebug_"): + pfx = 9 default: return "" } - + // Mach-O executables truncate section names to 16 characters, mangling some DWARF sections. + // As of DWARFv5 these are the only problematic section names (see DWARFv5 Appendix G). + for _, longname := range []string{ + "__debug_str_offsets", + "__zdebug_line_str", + "__zdebug_loclists", + "__zdebug_pubnames", + "__zdebug_pubtypes", + "__zdebug_rnglists", + "__zdebug_str_offsets", + } { + if sectname == longname[:16] { + sectname = longname + break + } + } + return sectname[pfx:] } sectionData := func(s *Section) ([]byte, error) { b, err := s.Data() @@ -701,15 +719,29 @@ func (f *File) DWARF() (*dwarf.Data, error) { // referred to by the binary f that are expected to be // satisfied by other libraries at dynamic load time. func (f *File) ImportedSymbols() ([]string, error) { - if f.Dysymtab == nil || f.Symtab == nil { + if f.Symtab == nil { return nil, &FormatError{0, "missing symbol table", nil} } st := f.Symtab dt := f.Dysymtab var all []string - for _, s := range st.Syms[dt.Iundefsym : dt.Iundefsym+dt.Nundefsym] { - all = append(all, s.Name) + if dt != nil { + for _, s := range st.Syms[dt.Iundefsym : dt.Iundefsym+dt.Nundefsym] { + all = append(all, s.Name) + } + } else { + // From Darwin's include/mach-o/nlist.h + const ( + N_TYPE = 0x0e + N_UNDF = 0x0 + N_EXT = 0x01 + ) + for _, s := range st.Syms { + if s.Type&N_TYPE == N_UNDF && s.Type&N_EXT != 0 { + all = append(all, s.Name) + } + } } return all, nil } diff --git a/src/debug/macho/file_test.go b/src/debug/macho/file_test.go index 313c376c54a27f..fbcc7bdcb01e96 100644 --- a/src/debug/macho/file_test.go +++ b/src/debug/macho/file_test.go @@ -9,15 +9,17 @@ import ( "internal/obscuretestdata" "io" "reflect" + "slices" "testing" ) type fileTest struct { - file string - hdr FileHeader - loads []any - sections []*SectionHeader - relocations map[string][]Reloc + file string + hdr FileHeader + loads []any + sections []*SectionHeader + relocations map[string][]Reloc + importedSyms []string } var fileTests = []fileTest{ @@ -46,6 +48,7 @@ var fileTests = []fileTest{ {"__jump_table", "__IMPORT", 0x3000, 0xa, 0x2000, 0x6, 0x0, 0x0, 0x4000008}, }, nil, + nil, }, { "testdata/gcc-amd64-darwin-exec.base64", @@ -74,6 +77,7 @@ var fileTests = []fileTest{ {"__la_symbol_ptr", "__DATA", 0x100001058, 0x10, 0x1058, 0x2, 0x0, 0x0, 0x7}, }, nil, + nil, }, { "testdata/gcc-amd64-darwin-exec-debug.base64", @@ -102,6 +106,7 @@ var fileTests = []fileTest{ {"__debug_str", "__DWARF", 0x10000215c, 0x60, 0x115c, 0x0, 0x0, 0x0, 0x0}, }, nil, + nil, }, { "testdata/clang-386-darwin-exec-with-rpath.base64", @@ -126,6 +131,7 @@ var fileTests = []fileTest{ }, nil, nil, + nil, }, { "testdata/clang-amd64-darwin-exec-with-rpath.base64", @@ -150,6 +156,7 @@ var fileTests = []fileTest{ }, nil, nil, + nil, }, { "testdata/clang-386-darwin.obj.base64", @@ -185,6 +192,7 @@ var fileTests = []fileTest{ }, }, }, + nil, }, { "testdata/clang-amd64-darwin.obj.base64", @@ -221,6 +229,15 @@ var fileTests = []fileTest{ }, }, }, + []string{"_printf"}, + }, + { + "testdata/clang-amd64-darwin-ld-r.obj.base64", + FileHeader{0xfeedfacf, CpuAmd64, 0x3, 0x1, 0x4, 0x1c0, 0x2000}, + nil, + nil, + nil, + []string{"_printf"}, }, } @@ -345,6 +362,17 @@ func TestOpen(t *testing.T) { } } } + + if tt.importedSyms != nil { + ss, err := f.ImportedSymbols() + if err != nil { + t.Errorf("open %s: fail to read imported symbols: %v", tt.file, err) + } + want := tt.importedSyms + if !slices.Equal(ss, want) { + t.Errorf("open %s: imported symbols differ:\n\thave %v\n\twant %v", tt.file, ss, want) + } + } } } diff --git a/src/debug/macho/testdata/clang-amd64-darwin-ld-r.obj.base64 b/src/debug/macho/testdata/clang-amd64-darwin-ld-r.obj.base64 new file mode 100644 index 00000000000000..036b5746abe351 --- /dev/null +++ b/src/debug/macho/testdata/clang-amd64-darwin-ld-r.obj.base64 @@ -0,0 +1 @@ +z/rt/gcAAAEDAAAAAQAAAAQAAADAAQAAACAAAAAAAAAZAAAAiAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJgAAAAAAAAAAAIAAAAAAACYAAAAAAAAAAcAAAAHAAAABAAAAAAAAABfX3RleHQAAAAAAAAAAAAAX19URVhUAAAAAAAAAAAAAAAAAAAAAAAAKgAAAAAAAAAAAgAABAAAAJgCAAACAAAAAAQAgAAAAAAAAAAAAAAAAF9fY3N0cmluZwAAAAAAAABfX1RFWFQAAAAAAAAAAAAAKgAAAAAAAAAOAAAAAAAAACoCAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAX19laF9mcmFtZQAAAAAAAF9fVEVYVAAAAAAAAAAAAAA4AAAAAAAAAEAAAAAAAAAAOAIAAAMAAACoAgAABAAAAAAAAAAAAAAAAAAAAAAAAABfX2NvbXBhY3RfdW53aW5kX19MRAAAAAAAAAAAAAAAAHgAAAAAAAAAIAAAAAAAAAB4AgAAAwAAAMgCAAABAAAAAAAAAgAAAAAAAAAAAAAAAAIAAAAYAAAA0AIAAAUAAAAgAwAAKAAAACQAAAAQAAAAAAwKAAAAAAApAAAAEAAAANACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABVSInlSIPsEEiNPQAAAADHRfwAAAAAsADoAAAAADHJiUX4ichIg8QQXcNoZWxsbywgd29ybGQKABQAAAAAAAAAAXpSAAF4EAEQDAcIkAEAACQAAAAEAAAA+P////////8qAAAAAAAAAABBDhCGAkMNBgAAAAAAAAAAAAAAAAAAACoAAAAAAAABAAAAAAAAAAAAAAAAAAAAABkAAAAEAAAtCwAAAAAAAB0cAAAAAQAAXBwAAAACAAAMIAAAAAIAAF4gAAAAAwAADgAAAAADAAAOEAAAAB4CAAAqAAAAAAAAABQAAAAOAwAAOAAAAAAAAAAeAAAADgMAAFAAAAAAAAAAAgAAAA8BAAAAAAAAAAAAAAgAAAABAAAAAAAAAAAAAAAgAF9tYWluAF9wcmludGYATEMxAEVIX0ZyYW1lMQBmdW5jLmVoAAAA diff --git a/src/debug/pe/file_test.go b/src/debug/pe/file_test.go index 3d960ab7f36b84..acea0455d88c19 100644 --- a/src/debug/pe/file_test.go +++ b/src/debug/pe/file_test.go @@ -413,7 +413,7 @@ func testDWARF(t *testing.T, linktype int) { var foundDebugGDBScriptsSection bool for _, sect := range f.Sections { - if sect.Name == ".debug_gdb_scripts" { + if sect.Name == ".debug_gdb_scripts" || sect.Name == ".zdebug_gdb_scripts" { foundDebugGDBScriptsSection = true } } diff --git a/src/debug/pe/symbol.go b/src/debug/pe/symbol.go index 6e8d9d16c22540..80acebe9f1f40b 100644 --- a/src/debug/pe/symbol.go +++ b/src/debug/pe/symbol.go @@ -98,7 +98,12 @@ func readCOFFSymbols(fh *FileHeader, r io.ReadSeeker) ([]COFFSymbol, error) { // isSymNameOffset checks symbol name if it is encoded as offset into string table. func isSymNameOffset(name [8]byte) (bool, uint32) { if name[0] == 0 && name[1] == 0 && name[2] == 0 && name[3] == 0 { - return true, binary.LittleEndian.Uint32(name[4:]) + offset := binary.LittleEndian.Uint32(name[4:]) + if offset == 0 { + // symbol has no name + return false, 0 + } + return true, offset } return false, 0 } diff --git a/src/embed/embed.go b/src/embed/embed.go index f6c0ef9b975160..4d5e418c9024b3 100644 --- a/src/embed/embed.go +++ b/src/embed/embed.go @@ -90,7 +90,8 @@ // depending on whether the package wants to make the data available to other packages. // It can only be used with variables at package scope, not with local variables. // -// Patterns must not match files outside the package's module, such as ‘.git/*’ or symbolic links. +// Patterns must not match files outside the package's module, such as ‘.git/*’, symbolic links, +// 'vendor/', or any directories containing go.mod (these are separate modules). // Patterns must not match files whose names include the special punctuation characters " * < > ? ` ' | / \ and :. // Matches for empty directories are ignored. After that, each pattern in a //go:embed line // must match at least one file or non-empty directory. @@ -170,7 +171,7 @@ type FS struct { // // p # dir=. elem=p // q/ # dir=. elem=q - // w/ # dir=. elem=w + // w # dir=. elem=w // q/r # dir=q elem=r // q/s/ # dir=q elem=s // q/v # dir=q elem=v diff --git a/src/encoding/asn1/asn1.go b/src/encoding/asn1/asn1.go index 781ab8769164b0..f4be515b98ef1c 100644 --- a/src/encoding/asn1/asn1.go +++ b/src/encoding/asn1/asn1.go @@ -22,9 +22,11 @@ package asn1 import ( "errors" "fmt" + "internal/saferio" "math" "math/big" "reflect" + "slices" "strconv" "strings" "time" @@ -224,16 +226,7 @@ type ObjectIdentifier []int // Equal reports whether oi and other represent the same identifier. func (oi ObjectIdentifier) Equal(other ObjectIdentifier) bool { - if len(oi) != len(other) { - return false - } - for i := 0; i < len(oi); i++ { - if oi[i] != other[i] { - return false - } - } - - return true + return slices.Equal(oi, other) } func (oi ObjectIdentifier) String() string { @@ -471,7 +464,21 @@ func parseIA5String(bytes []byte) (ret string, err error) { // parseT61String parses an ASN.1 T61String (8-bit clean string) from the given // byte slice and returns it. func parseT61String(bytes []byte) (ret string, err error) { - return string(bytes), nil + // T.61 is a defunct ITU 8-bit character encoding which preceded Unicode. + // T.61 uses a code page layout that _almost_ exactly maps to the code + // page layout of the ISO 8859-1 (Latin-1) character encoding, with the + // exception that a number of characters in Latin-1 are not present + // in T.61. + // + // Instead of mapping which characters are present in Latin-1 but not T.61, + // we just treat these strings as being encoded using Latin-1. This matches + // what most of the world does, including BoringSSL. + buf := make([]byte, 0, len(bytes)) + for _, v := range bytes { + // All the 1-byte UTF-8 runes map 1-1 with Latin-1. + buf = utf8.AppendRune(buf, rune(v)) + } + return string(buf), nil } // UTF8String @@ -490,8 +497,16 @@ func parseUTF8String(bytes []byte) (ret string, err error) { // parseBMPString parses an ASN.1 BMPString (Basic Multilingual Plane of // ISO/IEC/ITU 10646-1) from the given byte slice and returns it. func parseBMPString(bmpString []byte) (string, error) { + // BMPString uses the defunct UCS-2 16-bit character encoding, which + // covers the Basic Multilingual Plane (BMP). UTF-16 was an extension of + // UCS-2, containing all of the same code points, but also including + // multi-code point characters (by using surrogate code points). We can + // treat a UCS-2 encoded string as a UTF-16 encoded string, as long as + // we reject out the UTF-16 specific code points. This matches the + // BoringSSL behavior. + if len(bmpString)%2 != 0 { - return "", errors.New("pkcs12: odd-length BMP string") + return "", errors.New("invalid BMPString") } // Strip terminator if present. @@ -501,7 +516,16 @@ func parseBMPString(bmpString []byte) (string, error) { s := make([]uint16, 0, len(bmpString)/2) for len(bmpString) > 0 { - s = append(s, uint16(bmpString[0])<<8+uint16(bmpString[1])) + point := uint16(bmpString[0])<<8 + uint16(bmpString[1]) + // Reject UTF-16 code points that are permanently reserved + // noncharacters (0xfffe, 0xffff, and 0xfdd0-0xfdef) and surrogates + // (0xd800-0xdfff). + if point == 0xfffe || point == 0xffff || + (point >= 0xfdd0 && point <= 0xfdef) || + (point >= 0xd800 && point <= 0xdfff) { + return "", errors.New("invalid BMPString") + } + s = append(s, point) bmpString = bmpString[2:] } @@ -643,10 +667,17 @@ func parseSequenceOf(bytes []byte, sliceType reflect.Type, elemType reflect.Type offset += t.length numElements++ } - ret = reflect.MakeSlice(sliceType, numElements, numElements) + elemSize := uint64(elemType.Size()) + safeCap := saferio.SliceCapWithSize(elemSize, uint64(numElements)) + if safeCap < 0 { + err = SyntaxError{fmt.Sprintf("%s slice too big: %d elements of %d bytes", elemType.Kind(), numElements, elemSize)} + return + } + ret = reflect.MakeSlice(sliceType, 0, safeCap) params := fieldParameters{} offset := 0 for i := 0; i < numElements; i++ { + ret = reflect.Append(ret, reflect.Zero(elemType)) offset, err = parseField(ret.Index(i), bytes, offset, params) if err != nil { return @@ -702,6 +733,8 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam if !t.isCompound && t.class == ClassUniversal { innerBytes := bytes[offset : offset+t.length] switch t.tag { + case TagBoolean: + result, err = parseBool(innerBytes) case TagPrintableString: result, err = parsePrintableString(innerBytes) case TagNumericString: @@ -803,9 +836,18 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam } // Special case for time: UTCTime and GeneralizedTime both map to the - // Go type time.Time. - if universalTag == TagUTCTime && t.tag == TagGeneralizedTime && t.class == ClassUniversal { - universalTag = TagGeneralizedTime + // Go type time.Time. getUniversalType returns the tag for UTCTime when + // it sees a time.Time, so if we see a different time type on the wire, + // or the field is tagged with a different type, we change the universal + // type to match. + if universalTag == TagUTCTime { + if t.class == ClassUniversal { + if t.tag == TagGeneralizedTime { + universalTag = t.tag + } + } else if params.timeType != 0 { + universalTag = params.timeType + } } if params.set { @@ -1078,6 +1120,13 @@ func setDefaultValue(v reflect.Value, params fieldParameters) (ok bool) { // numeric causes strings to be unmarshaled as ASN.1 NumericString values // utf8 causes strings to be unmarshaled as ASN.1 UTF8String values // +// When decoding an ASN.1 value with an IMPLICIT tag into a time.Time field, +// Unmarshal will default to a UTCTime, which doesn't support time zones or +// fractional seconds. To force usage of GeneralizedTime, use the following +// tag: +// +// generalized causes time.Times to be unmarshaled as ASN.1 GeneralizedTime values +// // If the type of the first field of a structure is RawContent then the raw // ASN1 contents of the struct will be stored in it. // diff --git a/src/encoding/asn1/asn1_test.go b/src/encoding/asn1/asn1_test.go index 9a605e245c1369..41cc0ba50ec304 100644 --- a/src/encoding/asn1/asn1_test.go +++ b/src/encoding/asn1/asn1_test.go @@ -7,10 +7,12 @@ package asn1 import ( "bytes" "encoding/hex" + "errors" "fmt" "math" "math/big" "reflect" + "runtime" "strings" "testing" "time" @@ -506,6 +508,7 @@ var unmarshalTestData = []struct { {[]byte{0x30, 0x05, 0x02, 0x03, 0x12, 0x34, 0x56}, &TestBigInt{big.NewInt(0x123456)}}, {[]byte{0x30, 0x0b, 0x31, 0x09, 0x02, 0x01, 0x01, 0x02, 0x01, 0x02, 0x02, 0x01, 0x03}, &TestSet{Ints: []int{1, 2, 3}}}, {[]byte{0x12, 0x0b, '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ' '}, newString("0123456789 ")}, + {[]byte{0x14, 0x03, 0xbf, 0x61, 0x3f}, newString("¿a?")}, } func TestUnmarshal(t *testing.T) { @@ -1126,34 +1129,43 @@ func TestTaggedRawValue(t *testing.T) { } var bmpStringTests = []struct { + name string decoded string encodedHex string + invalid bool }{ - {"", "0000"}, + {"empty string", "", "0000", false}, // Example from https://tools.ietf.org/html/rfc7292#appendix-B. - {"Beavis", "0042006500610076006900730000"}, + {"rfc7292 example", "Beavis", "0042006500610076006900730000", false}, // Some characters from the "Letterlike Symbols Unicode block". - {"\u2115 - Double-struck N", "21150020002d00200044006f00750062006c0065002d00730074007200750063006b0020004e0000"}, + {"letterlike symbols", "\u2115 - Double-struck N", "21150020002d00200044006f00750062006c0065002d00730074007200750063006b0020004e0000", false}, + {"invalid length", "", "ff", true}, + {"invalid surrogate", "", "5051d801", true}, + {"invalid noncharacter 0xfdd1", "", "5051fdd1", true}, + {"invalid noncharacter 0xffff", "", "5051ffff", true}, + {"invalid noncharacter 0xfffe", "", "5051fffe", true}, } func TestBMPString(t *testing.T) { - for i, test := range bmpStringTests { - encoded, err := hex.DecodeString(test.encodedHex) - if err != nil { - t.Fatalf("#%d: failed to decode from hex string", i) - } + for _, test := range bmpStringTests { + t.Run(test.name, func(t *testing.T) { + encoded, err := hex.DecodeString(test.encodedHex) + if err != nil { + t.Fatalf("failed to decode from hex string: %s", err) + } - decoded, err := parseBMPString(encoded) + decoded, err := parseBMPString(encoded) - if err != nil { - t.Errorf("#%d: decoding output gave an error: %s", i, err) - continue - } + if err != nil && !test.invalid { + t.Errorf("parseBMPString failed: %s", err) + } else if test.invalid && err == nil { + t.Error("parseBMPString didn't fail as expected") + } - if decoded != test.decoded { - t.Errorf("#%d: decoding output resulted in %q, but it should have been %q", i, decoded, test.decoded) - continue - } + if decoded != test.decoded { + t.Errorf("parseBMPString(%q): got %q, want %q", encoded, decoded, test.decoded) + } + }) } } @@ -1175,3 +1187,70 @@ func BenchmarkObjectIdentifierString(b *testing.B) { _ = oidPublicKeyRSA.String() } } + +func TestImplicitTypeRoundtrip(t *testing.T) { + type tagged struct { + IA5 string `asn1:"tag:1,ia5"` + Printable string `asn1:"tag:2,printable"` + UTF8 string `asn1:"tag:3,utf8"` + Numeric string `asn1:"tag:4,numeric"` + UTC time.Time `asn1:"tag:5,utc"` + Generalized time.Time `asn1:"tag:6,generalized"` + } + a := tagged{ + IA5: "ia5", + Printable: "printable", + UTF8: "utf8", + Numeric: "123 456", + UTC: time.Now().UTC().Truncate(time.Second), + Generalized: time.Now().UTC().Truncate(time.Second), + } + enc, err := Marshal(a) + if err != nil { + t.Fatalf("Marshal failed: %s", err) + } + var b tagged + if _, err := Unmarshal(enc, &b); err != nil { + t.Fatalf("Unmarshal failed: %s", err) + } + + if !reflect.DeepEqual(a, b) { + t.Fatalf("Unexpected diff after roundtripping struct\na: %#v\nb: %#v", a, b) + } +} + +func TestParsingMemoryConsumption(t *testing.T) { + // Craft a syntatically valid, but empty, ~10 MB DER bomb. A successful + // unmarshal of this bomb should yield ~280 MB. However, the parsing should + // fail due to the empty content; and, in such cases, we want to make sure + // that we do not unnecessarily allocate memories. + derBomb := make([]byte, 10_000_000) + for i := range derBomb { + derBomb[i] = 0x30 + } + derBomb = append([]byte{0x30, 0x83, 0x98, 0x96, 0x80}, derBomb...) + + var m runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&m) + memBefore := m.TotalAlloc + + var out []struct { + Id []int + Critical bool `asn1:"optional"` + Value []byte + } + _, err := Unmarshal(derBomb, &out) + if !errors.As(err, &SyntaxError{}) { + t.Fatalf("Incorrect error result: want (%v), but got (%v) instead", &SyntaxError{}, err) + } + + runtime.ReadMemStats(&m) + memDiff := m.TotalAlloc - memBefore + + // Ensure that the memory allocated does not exceed 10<<21 (~20 MB) when + // the parsing fails. + if memDiff > 10<<21 { + t.Errorf("Too much memory allocated while parsing DER: %v MiB", memDiff/1024/1024) + } +} diff --git a/src/encoding/asn1/marshal.go b/src/encoding/asn1/marshal.go index b9c0b8bce054f5..ddb3bd85dd618d 100644 --- a/src/encoding/asn1/marshal.go +++ b/src/encoding/asn1/marshal.go @@ -460,17 +460,20 @@ func makeBody(value reflect.Value, params fieldParameters) (e encoder, err error case flagType: return bytesEncoder(nil), nil case timeType: - t := value.Interface().(time.Time) + t, _ := reflect.TypeAssert[time.Time](value) if params.timeType == TagGeneralizedTime || outsideUTCRange(t) { return makeGeneralizedTime(t) } return makeUTCTime(t) case bitStringType: - return bitStringEncoder(value.Interface().(BitString)), nil + v, _ := reflect.TypeAssert[BitString](value) + return bitStringEncoder(v), nil case objectIdentifierType: - return makeObjectIdentifier(value.Interface().(ObjectIdentifier)) + v, _ := reflect.TypeAssert[ObjectIdentifier](value) + return makeObjectIdentifier(v) case bigIntType: - return makeBigInt(value.Interface().(*big.Int)) + v, _ := reflect.TypeAssert[*big.Int](value) + return makeBigInt(v) } switch v := value; v.Kind() { @@ -605,7 +608,7 @@ func makeField(v reflect.Value, params fieldParameters) (e encoder, err error) { } if v.Type() == rawValueType { - rv := v.Interface().(RawValue) + rv, _ := reflect.TypeAssert[RawValue](v) if len(rv.FullBytes) != 0 { return bytesEncoder(rv.FullBytes), nil } @@ -650,7 +653,8 @@ func makeField(v reflect.Value, params fieldParameters) (e encoder, err error) { tag = params.stringType } case TagUTCTime: - if params.timeType == TagGeneralizedTime || outsideUTCRange(v.Interface().(time.Time)) { + t, _ := reflect.TypeAssert[time.Time](v) + if params.timeType == TagGeneralizedTime || outsideUTCRange(t) { tag = TagGeneralizedTime } } @@ -725,6 +729,7 @@ func makeField(v reflect.Value, params fieldParameters) (e encoder, err error) { // omitempty: causes empty slices to be skipped // printable: causes strings to be marshaled as ASN.1, PrintableString values // utf8: causes strings to be marshaled as ASN.1, UTF8String values +// numeric: causes strings to be marshaled as ASN.1, NumericString values // utc: causes time.Time to be marshaled as ASN.1, UTCTime values // generalized: causes time.Time to be marshaled as ASN.1, GeneralizedTime values func Marshal(val any) ([]byte, error) { diff --git a/src/encoding/asn1/marshal_test.go b/src/encoding/asn1/marshal_test.go index 64ce47640034be..dfd6d4e40b9868 100644 --- a/src/encoding/asn1/marshal_test.go +++ b/src/encoding/asn1/marshal_test.go @@ -311,6 +311,26 @@ func TestIssue11130(t *testing.T) { } } +func TestIssue68241(t *testing.T) { + for i, want := range []any{false, true} { + data, err := Marshal(want) + if err != nil { + t.Errorf("cannot Marshal: %v", err) + return + } + + var got any + _, err = Unmarshal(data, &got) + if err != nil { + t.Errorf("cannot Unmarshal: %v", err) + return + } + if !reflect.DeepEqual(got, want) { + t.Errorf("#%d Unmarshal, got: %v, want: %v", i, got, want) + } + } +} + func BenchmarkMarshal(b *testing.B) { b.ReportAllocs() diff --git a/src/encoding/base32/base32.go b/src/encoding/base32/base32.go index 9e988ef39b4f4d..8bda6c6799ef72 100644 --- a/src/encoding/base32/base32.go +++ b/src/encoding/base32/base32.go @@ -387,7 +387,8 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { // Decode decodes src using the encoding enc. It writes at most // [Encoding.DecodedLen](len(src)) bytes to dst and returns the number of bytes -// written. If src contains invalid base32 data, it will return the +// written. The caller must ensure that dst is large enough to hold all +// the decoded data. If src contains invalid base32 data, it will return the // number of bytes successfully written and [CorruptInputError]. // Newline characters (\r and \n) are ignored. func (enc *Encoding) Decode(dst, src []byte) (n int, err error) { @@ -400,6 +401,7 @@ func (enc *Encoding) Decode(dst, src []byte) (n int, err error) { // AppendDecode appends the base32 decoded src to dst // and returns the extended buffer. // If the input is malformed, it returns the partially decoded src and an error. +// New line characters (\r and \n) are ignored. func (enc *Encoding) AppendDecode(dst, src []byte) ([]byte, error) { // Compute the output size without padding to avoid over allocating. n := len(src) @@ -414,6 +416,8 @@ func (enc *Encoding) AppendDecode(dst, src []byte) ([]byte, error) { } // DecodeString returns the bytes represented by the base32 string s. +// If the input is malformed, it returns the partially decoded data and +// [CorruptInputError]. New line characters (\r and \n) are ignored. func (enc *Encoding) DecodeString(s string) ([]byte, error) { buf := []byte(s) l := stripNewlines(buf, buf) diff --git a/src/encoding/base32/base32_test.go b/src/encoding/base32/base32_test.go index f5d3c49e38fbef..6f8d564def3c86 100644 --- a/src/encoding/base32/base32_test.go +++ b/src/encoding/base32/base32_test.go @@ -709,7 +709,6 @@ func TestBufferedDecodingPadding(t *testing.T) { } for _, testcase := range testcases { - testcase := testcase pr, pw := io.Pipe() go func() { for _, chunk := range testcase.chunks { diff --git a/src/encoding/base64/base64.go b/src/encoding/base64/base64.go index 87f45863bd7b4d..57aa1a697fc7bd 100644 --- a/src/encoding/base64/base64.go +++ b/src/encoding/base64/base64.go @@ -6,7 +6,7 @@ package base64 import ( - "encoding/binary" + "internal/byteorder" "io" "slices" "strconv" @@ -409,6 +409,7 @@ func (enc *Encoding) decodeQuantum(dst, src []byte, si int) (nsi, n int, err err // AppendDecode appends the base64 decoded src to dst // and returns the extended buffer. // If the input is malformed, it returns the partially decoded src and an error. +// New line characters (\r and \n) are ignored. func (enc *Encoding) AppendDecode(dst, src []byte) ([]byte, error) { // Compute the output size without padding to avoid over allocating. n := len(src) @@ -423,6 +424,8 @@ func (enc *Encoding) AppendDecode(dst, src []byte) ([]byte, error) { } // DecodeString returns the bytes represented by the base64 string s. +// If the input is malformed, it returns the partially decoded data and +// [CorruptInputError]. New line characters (\r and \n) are ignored. func (enc *Encoding) DecodeString(s string) ([]byte, error) { dbuf := make([]byte, enc.DecodedLen(len(s))) n, err := enc.Decode(dbuf, []byte(s)) @@ -508,7 +511,8 @@ func (d *decoder) Read(p []byte) (n int, err error) { // Decode decodes src using the encoding enc. It writes at most // [Encoding.DecodedLen](len(src)) bytes to dst and returns the number of bytes -// written. If src contains invalid base64 data, it will return the +// written. The caller must ensure that dst is large enough to hold all +// the decoded data. If src contains invalid base64 data, it will return the // number of bytes successfully written and [CorruptInputError]. // New line characters (\r and \n) are ignored. func (enc *Encoding) Decode(dst, src []byte) (n int, err error) { @@ -534,7 +538,7 @@ func (enc *Encoding) Decode(dst, src []byte) (n int, err error) { enc.decodeMap[src2[6]], enc.decodeMap[src2[7]], ); ok { - binary.BigEndian.PutUint64(dst[n:], dn) + byteorder.BEPutUint64(dst[n:], dn) n += 6 si += 8 } else { @@ -555,7 +559,7 @@ func (enc *Encoding) Decode(dst, src []byte) (n int, err error) { enc.decodeMap[src2[2]], enc.decodeMap[src2[3]], ); ok { - binary.BigEndian.PutUint32(dst[n:], dn) + byteorder.BEPutUint32(dst[n:], dn) n += 3 si += 4 } else { diff --git a/src/encoding/binary/binary.go b/src/encoding/binary/binary.go index a150c0bf05be47..c92dfded272798 100644 --- a/src/encoding/binary/binary.go +++ b/src/encoding/binary/binary.go @@ -65,17 +65,20 @@ var BigEndian bigEndian type littleEndian struct{} +// Uint16 returns the uint16 representation of b[0:2]. func (littleEndian) Uint16(b []byte) uint16 { _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 return uint16(b[0]) | uint16(b[1])<<8 } +// PutUint16 stores v into b[0:2]. func (littleEndian) PutUint16(b []byte, v uint16) { _ = b[1] // early bounds check to guarantee safety of writes below b[0] = byte(v) b[1] = byte(v >> 8) } +// AppendUint16 appends the bytes of v to b and returns the appended slice. func (littleEndian) AppendUint16(b []byte, v uint16) []byte { return append(b, byte(v), @@ -83,11 +86,13 @@ func (littleEndian) AppendUint16(b []byte, v uint16) []byte { ) } +// Uint32 returns the uint32 representation of b[0:4]. func (littleEndian) Uint32(b []byte) uint32 { _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 } +// PutUint32 stores v into b[0:4]. func (littleEndian) PutUint32(b []byte, v uint32) { _ = b[3] // early bounds check to guarantee safety of writes below b[0] = byte(v) @@ -96,6 +101,7 @@ func (littleEndian) PutUint32(b []byte, v uint32) { b[3] = byte(v >> 24) } +// AppendUint32 appends the bytes of v to b and returns the appended slice. func (littleEndian) AppendUint32(b []byte, v uint32) []byte { return append(b, byte(v), @@ -105,12 +111,14 @@ func (littleEndian) AppendUint32(b []byte, v uint32) []byte { ) } +// Uint64 returns the uint64 representation of b[0:8]. func (littleEndian) Uint64(b []byte) uint64 { _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 } +// PutUint64 stores v into b[0:8]. func (littleEndian) PutUint64(b []byte, v uint64) { _ = b[7] // early bounds check to guarantee safety of writes below b[0] = byte(v) @@ -123,6 +131,7 @@ func (littleEndian) PutUint64(b []byte, v uint64) { b[7] = byte(v >> 56) } +// AppendUint64 appends the bytes of v to b and returns the appended slice. func (littleEndian) AppendUint64(b []byte, v uint64) []byte { return append(b, byte(v), @@ -142,17 +151,20 @@ func (littleEndian) GoString() string { return "binary.LittleEndian" } type bigEndian struct{} +// Uint16 returns the uint16 representation of b[0:2]. func (bigEndian) Uint16(b []byte) uint16 { _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 return uint16(b[1]) | uint16(b[0])<<8 } +// PutUint16 stores v into b[0:2]. func (bigEndian) PutUint16(b []byte, v uint16) { _ = b[1] // early bounds check to guarantee safety of writes below b[0] = byte(v >> 8) b[1] = byte(v) } +// AppendUint16 appends the bytes of v to b and returns the appended slice. func (bigEndian) AppendUint16(b []byte, v uint16) []byte { return append(b, byte(v>>8), @@ -160,11 +172,13 @@ func (bigEndian) AppendUint16(b []byte, v uint16) []byte { ) } +// Uint32 returns the uint32 representation of b[0:4]. func (bigEndian) Uint32(b []byte) uint32 { _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 } +// PutUint32 stores v into b[0:4]. func (bigEndian) PutUint32(b []byte, v uint32) { _ = b[3] // early bounds check to guarantee safety of writes below b[0] = byte(v >> 24) @@ -173,6 +187,7 @@ func (bigEndian) PutUint32(b []byte, v uint32) { b[3] = byte(v) } +// AppendUint32 appends the bytes of v to b and returns the appended slice. func (bigEndian) AppendUint32(b []byte, v uint32) []byte { return append(b, byte(v>>24), @@ -182,12 +197,14 @@ func (bigEndian) AppendUint32(b []byte, v uint32) []byte { ) } +// Uint64 returns the uint64 representation of b[0:8]. func (bigEndian) Uint64(b []byte) uint64 { _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 } +// PutUint64 stores v into b[0:8]. func (bigEndian) PutUint64(b []byte, v uint64) { _ = b[7] // early bounds check to guarantee safety of writes below b[0] = byte(v >> 56) @@ -200,6 +217,7 @@ func (bigEndian) PutUint64(b []byte, v uint64) { b[7] = byte(v) } +// AppendUint64 appends the bytes of v to b and returns the appended slice. func (bigEndian) AppendUint64(b []byte, v uint64) []byte { return append(b, byte(v>>56), @@ -448,7 +466,7 @@ func Encode(buf []byte, order ByteOrder, data any) (int, error) { // Append appends the binary representation of data to buf. // buf may be nil, in which case a new buffer will be allocated. // See [Write] on which data are acceptable. -// It returns the (possibily extended) buffer containing data or an error. +// It returns the (possibly extended) buffer containing data or an error. func Append(buf []byte, order ByteOrder, data any) ([]byte, error) { // Fast path for basic types and slices. if n, _ := intDataSize(data); n != 0 { diff --git a/src/encoding/binary/binary_test.go b/src/encoding/binary/binary_test.go index 9e5fed53b72d52..e0c5c0d9e0d7ee 100644 --- a/src/encoding/binary/binary_test.go +++ b/src/encoding/binary/binary_test.go @@ -7,6 +7,7 @@ package binary import ( "bytes" "fmt" + "internal/asan" "io" "math" "reflect" @@ -710,6 +711,9 @@ func TestNoFixedSize(t *testing.T) { } func TestAppendAllocs(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } buf := make([]byte, 0, Size(&s)) var err error allocs := testing.AllocsPerRun(1, func() { @@ -745,6 +749,9 @@ var sizableTypes = []any{ } func TestSizeAllocs(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } for _, data := range sizableTypes { t.Run(fmt.Sprintf("%T", data), func(t *testing.T) { // Size uses a sync.Map behind the scenes. The slow lookup path of diff --git a/src/encoding/binary/varint_test.go b/src/encoding/binary/varint_test.go index 5c3ea318c397bd..dbb615070bf448 100644 --- a/src/encoding/binary/varint_test.go +++ b/src/encoding/binary/varint_test.go @@ -181,7 +181,6 @@ func TestBufferTooBigWithOverflow(t *testing.T) { } for _, tt := range tests { - tt := tt t.Run(tt.name, func(t *testing.T) { value, n := Uvarint(tt.in) if g, w := n, tt.wantN; g != w { diff --git a/src/encoding/encoding.go b/src/encoding/encoding.go index 50acf3c23a1013..4d288b6d3b9b3e 100644 --- a/src/encoding/encoding.go +++ b/src/encoding/encoding.go @@ -35,6 +35,18 @@ type BinaryUnmarshaler interface { UnmarshalBinary(data []byte) error } +// BinaryAppender is the interface implemented by an object +// that can append the binary representation of itself. +// If a type implements both [BinaryAppender] and [BinaryMarshaler], +// then v.MarshalBinary() must be semantically identical to v.AppendBinary(nil). +type BinaryAppender interface { + // AppendBinary appends the binary representation of itself to the end of b + // (allocating a larger slice if necessary) and returns the updated slice. + // + // Implementations must not retain b, nor mutate any bytes within b[:len(b)]. + AppendBinary(b []byte) ([]byte, error) +} + // TextMarshaler is the interface implemented by an object that can // marshal itself into a textual form. // @@ -52,3 +64,15 @@ type TextMarshaler interface { type TextUnmarshaler interface { UnmarshalText(text []byte) error } + +// TextAppender is the interface implemented by an object +// that can append the textual representation of itself. +// If a type implements both [TextAppender] and [TextMarshaler], +// then v.MarshalText() must be semantically identical to v.AppendText(nil). +type TextAppender interface { + // AppendText appends the textual representation of itself to the end of b + // (allocating a larger slice if necessary) and returns the updated slice. + // + // Implementations must not retain b, nor mutate any bytes within b[:len(b)]. + AppendText(b []byte) ([]byte, error) +} diff --git a/src/encoding/gob/dec_helpers.go b/src/encoding/gob/dec_helpers.go index 44a74e24427987..e8696830e55486 100644 --- a/src/encoding/gob/dec_helpers.go +++ b/src/encoding/gob/dec_helpers.go @@ -58,7 +58,7 @@ func decBoolArray(state *decoderState, v reflect.Value, length int, ovfl error) } func decBoolSlice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]bool) + slice, ok := reflect.TypeAssert[[]bool](v) if !ok { // It is kind bool but not type bool. TODO: We can handle this unsafely. return false @@ -85,7 +85,7 @@ func decComplex64Array(state *decoderState, v reflect.Value, length int, ovfl er } func decComplex64Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]complex64) + slice, ok := reflect.TypeAssert[[]complex64](v) if !ok { // It is kind complex64 but not type complex64. TODO: We can handle this unsafely. return false @@ -114,7 +114,7 @@ func decComplex128Array(state *decoderState, v reflect.Value, length int, ovfl e } func decComplex128Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]complex128) + slice, ok := reflect.TypeAssert[[]complex128](v) if !ok { // It is kind complex128 but not type complex128. TODO: We can handle this unsafely. return false @@ -143,7 +143,7 @@ func decFloat32Array(state *decoderState, v reflect.Value, length int, ovfl erro } func decFloat32Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]float32) + slice, ok := reflect.TypeAssert[[]float32](v) if !ok { // It is kind float32 but not type float32. TODO: We can handle this unsafely. return false @@ -170,7 +170,7 @@ func decFloat64Array(state *decoderState, v reflect.Value, length int, ovfl erro } func decFloat64Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]float64) + slice, ok := reflect.TypeAssert[[]float64](v) if !ok { // It is kind float64 but not type float64. TODO: We can handle this unsafely. return false @@ -197,7 +197,7 @@ func decIntArray(state *decoderState, v reflect.Value, length int, ovfl error) b } func decIntSlice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]int) + slice, ok := reflect.TypeAssert[[]int](v) if !ok { // It is kind int but not type int. TODO: We can handle this unsafely. return false @@ -229,7 +229,7 @@ func decInt16Array(state *decoderState, v reflect.Value, length int, ovfl error) } func decInt16Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]int16) + slice, ok := reflect.TypeAssert[[]int16](v) if !ok { // It is kind int16 but not type int16. TODO: We can handle this unsafely. return false @@ -260,7 +260,7 @@ func decInt32Array(state *decoderState, v reflect.Value, length int, ovfl error) } func decInt32Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]int32) + slice, ok := reflect.TypeAssert[[]int32](v) if !ok { // It is kind int32 but not type int32. TODO: We can handle this unsafely. return false @@ -291,7 +291,7 @@ func decInt64Array(state *decoderState, v reflect.Value, length int, ovfl error) } func decInt64Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]int64) + slice, ok := reflect.TypeAssert[[]int64](v) if !ok { // It is kind int64 but not type int64. TODO: We can handle this unsafely. return false @@ -318,7 +318,7 @@ func decInt8Array(state *decoderState, v reflect.Value, length int, ovfl error) } func decInt8Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]int8) + slice, ok := reflect.TypeAssert[[]int8](v) if !ok { // It is kind int8 but not type int8. TODO: We can handle this unsafely. return false @@ -349,7 +349,7 @@ func decStringArray(state *decoderState, v reflect.Value, length int, ovfl error } func decStringSlice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]string) + slice, ok := reflect.TypeAssert[[]string](v) if !ok { // It is kind string but not type string. TODO: We can handle this unsafely. return false @@ -390,7 +390,7 @@ func decUintArray(state *decoderState, v reflect.Value, length int, ovfl error) } func decUintSlice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]uint) + slice, ok := reflect.TypeAssert[[]uint](v) if !ok { // It is kind uint but not type uint. TODO: We can handle this unsafely. return false @@ -421,7 +421,7 @@ func decUint16Array(state *decoderState, v reflect.Value, length int, ovfl error } func decUint16Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]uint16) + slice, ok := reflect.TypeAssert[[]uint16](v) if !ok { // It is kind uint16 but not type uint16. TODO: We can handle this unsafely. return false @@ -452,7 +452,7 @@ func decUint32Array(state *decoderState, v reflect.Value, length int, ovfl error } func decUint32Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]uint32) + slice, ok := reflect.TypeAssert[[]uint32](v) if !ok { // It is kind uint32 but not type uint32. TODO: We can handle this unsafely. return false @@ -483,7 +483,7 @@ func decUint64Array(state *decoderState, v reflect.Value, length int, ovfl error } func decUint64Slice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]uint64) + slice, ok := reflect.TypeAssert[[]uint64](v) if !ok { // It is kind uint64 but not type uint64. TODO: We can handle this unsafely. return false @@ -510,7 +510,7 @@ func decUintptrArray(state *decoderState, v reflect.Value, length int, ovfl erro } func decUintptrSlice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]uintptr) + slice, ok := reflect.TypeAssert[[]uintptr](v) if !ok { // It is kind uintptr but not type uintptr. TODO: We can handle this unsafely. return false diff --git a/src/encoding/gob/decgen.go b/src/encoding/gob/decgen.go index af4cdbee9dfd11..420c86207c200e 100644 --- a/src/encoding/gob/decgen.go +++ b/src/encoding/gob/decgen.go @@ -231,7 +231,7 @@ func dec%[2]sArray(state *decoderState, v reflect.Value, length int, ovfl error) const sliceHelper = ` func dec%[2]sSlice(state *decoderState, v reflect.Value, length int, ovfl error) bool { - slice, ok := v.Interface().([]%[1]s) + slice, ok := reflect.TypeAssert[[]%[1]s](v) if !ok { // It is kind %[1]s but not type %[1]s. TODO: We can handle this unsafely. return false diff --git a/src/encoding/gob/decode.go b/src/encoding/gob/decode.go index d178b2b2fb6467..fb7837b11b2bb2 100644 --- a/src/encoding/gob/decode.go +++ b/src/encoding/gob/decode.go @@ -768,11 +768,14 @@ func (dec *Decoder) decodeGobDecoder(ut *userTypeInfo, state *decoderState, valu // We know it's one of these. switch ut.externalDec { case xGob: - err = value.Interface().(GobDecoder).GobDecode(b) + gobDecoder, _ := reflect.TypeAssert[GobDecoder](value) + err = gobDecoder.GobDecode(b) case xBinary: - err = value.Interface().(encoding.BinaryUnmarshaler).UnmarshalBinary(b) + binaryUnmarshaler, _ := reflect.TypeAssert[encoding.BinaryUnmarshaler](value) + err = binaryUnmarshaler.UnmarshalBinary(b) case xText: - err = value.Interface().(encoding.TextUnmarshaler).UnmarshalText(b) + textUnmarshaler, _ := reflect.TypeAssert[encoding.TextUnmarshaler](value) + err = textUnmarshaler.UnmarshalText(b) } if err != nil { error_(err) @@ -911,8 +914,11 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProg var maxIgnoreNestingDepth = 10000 // decIgnoreOpFor returns the decoding op for a field that has no destination. -func (dec *Decoder) decIgnoreOpFor(wireId typeId, inProgress map[typeId]*decOp, depth int) *decOp { - if depth > maxIgnoreNestingDepth { +func (dec *Decoder) decIgnoreOpFor(wireId typeId, inProgress map[typeId]*decOp) *decOp { + // Track how deep we've recursed trying to skip nested ignored fields. + dec.ignoreDepth++ + defer func() { dec.ignoreDepth-- }() + if dec.ignoreDepth > maxIgnoreNestingDepth { error_(errors.New("invalid nesting depth")) } // If this type is already in progress, it's a recursive type (e.g. map[string]*T). @@ -938,7 +944,7 @@ func (dec *Decoder) decIgnoreOpFor(wireId typeId, inProgress map[typeId]*decOp, errorf("bad data: undefined type %s", wireId.string()) case wire.ArrayT != nil: elemId := wire.ArrayT.Elem - elemOp := dec.decIgnoreOpFor(elemId, inProgress, depth+1) + elemOp := dec.decIgnoreOpFor(elemId, inProgress) op = func(i *decInstr, state *decoderState, value reflect.Value) { state.dec.ignoreArray(state, *elemOp, wire.ArrayT.Len) } @@ -946,15 +952,15 @@ func (dec *Decoder) decIgnoreOpFor(wireId typeId, inProgress map[typeId]*decOp, case wire.MapT != nil: keyId := dec.wireType[wireId].MapT.Key elemId := dec.wireType[wireId].MapT.Elem - keyOp := dec.decIgnoreOpFor(keyId, inProgress, depth+1) - elemOp := dec.decIgnoreOpFor(elemId, inProgress, depth+1) + keyOp := dec.decIgnoreOpFor(keyId, inProgress) + elemOp := dec.decIgnoreOpFor(elemId, inProgress) op = func(i *decInstr, state *decoderState, value reflect.Value) { state.dec.ignoreMap(state, *keyOp, *elemOp) } case wire.SliceT != nil: elemId := wire.SliceT.Elem - elemOp := dec.decIgnoreOpFor(elemId, inProgress, depth+1) + elemOp := dec.decIgnoreOpFor(elemId, inProgress) op = func(i *decInstr, state *decoderState, value reflect.Value) { state.dec.ignoreSlice(state, *elemOp) } @@ -1115,7 +1121,7 @@ func (dec *Decoder) compileSingle(remoteId typeId, ut *userTypeInfo) (engine *de func (dec *Decoder) compileIgnoreSingle(remoteId typeId) *decEngine { engine := new(decEngine) engine.instr = make([]decInstr, 1) // one item - op := dec.decIgnoreOpFor(remoteId, make(map[typeId]*decOp), 0) + op := dec.decIgnoreOpFor(remoteId, make(map[typeId]*decOp)) ovfl := overflow(dec.typeString(remoteId)) engine.instr[0] = decInstr{*op, 0, nil, ovfl} engine.numInstr = 1 @@ -1160,7 +1166,7 @@ func (dec *Decoder) compileDec(remoteId typeId, ut *userTypeInfo) (engine *decEn localField, present := srt.FieldByName(wireField.Name) // TODO(r): anonymous names if !present || !isExported(wireField.Name) { - op := dec.decIgnoreOpFor(wireField.Id, make(map[typeId]*decOp), 0) + op := dec.decIgnoreOpFor(wireField.Id, make(map[typeId]*decOp)) engine.instr[fieldnum] = decInstr{*op, fieldnum, nil, ovfl} continue } diff --git a/src/encoding/gob/decoder.go b/src/encoding/gob/decoder.go index c4b60880130787..c35398d1054e9e 100644 --- a/src/encoding/gob/decoder.go +++ b/src/encoding/gob/decoder.go @@ -35,6 +35,8 @@ type Decoder struct { freeList *decoderState // list of free decoderStates; avoids reallocation countBuf []byte // used for decoding integers while parsing messages err error + // ignoreDepth tracks the depth of recursively parsed ignored fields + ignoreDepth int } // NewDecoder returns a new decoder that reads from the [io.Reader]. @@ -197,7 +199,7 @@ func (dec *Decoder) Decode(e any) error { value := reflect.ValueOf(e) // If e represents a value as opposed to a pointer, the answer won't // get back to the caller. Make sure it's a pointer. - if value.Type().Kind() != reflect.Pointer { + if value.Kind() != reflect.Pointer { dec.err = errors.New("gob: attempt to decode into a non-pointer") return dec.err } diff --git a/src/encoding/gob/doc.go b/src/encoding/gob/doc.go index 0866ba1544666d..c746806887ab3d 100644 --- a/src/encoding/gob/doc.go +++ b/src/encoding/gob/doc.go @@ -274,7 +274,7 @@ released version, subject to issues such as security fixes. See the Go compatibi document for background: https://golang.org/doc/go1compat See "Gobs of data" for a design discussion of the gob wire format: -https://blog.golang.org/gobs-of-data +https://go.dev/blog/gob # Security diff --git a/src/encoding/gob/enc_helpers.go b/src/encoding/gob/enc_helpers.go index c3b4ca8972a686..a9c45a46690e5e 100644 --- a/src/encoding/gob/enc_helpers.go +++ b/src/encoding/gob/enc_helpers.go @@ -57,7 +57,7 @@ func encBoolArray(state *encoderState, v reflect.Value) bool { } func encBoolSlice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]bool) + slice, ok := reflect.TypeAssert[[]bool](v) if !ok { // It is kind bool but not type bool. TODO: We can handle this unsafely. return false @@ -83,7 +83,7 @@ func encComplex64Array(state *encoderState, v reflect.Value) bool { } func encComplex64Slice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]complex64) + slice, ok := reflect.TypeAssert[[]complex64](v) if !ok { // It is kind complex64 but not type complex64. TODO: We can handle this unsafely. return false @@ -108,7 +108,7 @@ func encComplex128Array(state *encoderState, v reflect.Value) bool { } func encComplex128Slice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]complex128) + slice, ok := reflect.TypeAssert[[]complex128](v) if !ok { // It is kind complex128 but not type complex128. TODO: We can handle this unsafely. return false @@ -133,7 +133,7 @@ func encFloat32Array(state *encoderState, v reflect.Value) bool { } func encFloat32Slice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]float32) + slice, ok := reflect.TypeAssert[[]float32](v) if !ok { // It is kind float32 but not type float32. TODO: We can handle this unsafely. return false @@ -156,7 +156,7 @@ func encFloat64Array(state *encoderState, v reflect.Value) bool { } func encFloat64Slice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]float64) + slice, ok := reflect.TypeAssert[[]float64](v) if !ok { // It is kind float64 but not type float64. TODO: We can handle this unsafely. return false @@ -179,7 +179,7 @@ func encIntArray(state *encoderState, v reflect.Value) bool { } func encIntSlice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]int) + slice, ok := reflect.TypeAssert[[]int](v) if !ok { // It is kind int but not type int. TODO: We can handle this unsafely. return false @@ -201,7 +201,7 @@ func encInt16Array(state *encoderState, v reflect.Value) bool { } func encInt16Slice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]int16) + slice, ok := reflect.TypeAssert[[]int16](v) if !ok { // It is kind int16 but not type int16. TODO: We can handle this unsafely. return false @@ -223,7 +223,7 @@ func encInt32Array(state *encoderState, v reflect.Value) bool { } func encInt32Slice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]int32) + slice, ok := reflect.TypeAssert[[]int32](v) if !ok { // It is kind int32 but not type int32. TODO: We can handle this unsafely. return false @@ -245,7 +245,7 @@ func encInt64Array(state *encoderState, v reflect.Value) bool { } func encInt64Slice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]int64) + slice, ok := reflect.TypeAssert[[]int64](v) if !ok { // It is kind int64 but not type int64. TODO: We can handle this unsafely. return false @@ -267,7 +267,7 @@ func encInt8Array(state *encoderState, v reflect.Value) bool { } func encInt8Slice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]int8) + slice, ok := reflect.TypeAssert[[]int8](v) if !ok { // It is kind int8 but not type int8. TODO: We can handle this unsafely. return false @@ -289,7 +289,7 @@ func encStringArray(state *encoderState, v reflect.Value) bool { } func encStringSlice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]string) + slice, ok := reflect.TypeAssert[[]string](v) if !ok { // It is kind string but not type string. TODO: We can handle this unsafely. return false @@ -312,7 +312,7 @@ func encUintArray(state *encoderState, v reflect.Value) bool { } func encUintSlice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]uint) + slice, ok := reflect.TypeAssert[[]uint](v) if !ok { // It is kind uint but not type uint. TODO: We can handle this unsafely. return false @@ -334,7 +334,7 @@ func encUint16Array(state *encoderState, v reflect.Value) bool { } func encUint16Slice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]uint16) + slice, ok := reflect.TypeAssert[[]uint16](v) if !ok { // It is kind uint16 but not type uint16. TODO: We can handle this unsafely. return false @@ -356,7 +356,7 @@ func encUint32Array(state *encoderState, v reflect.Value) bool { } func encUint32Slice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]uint32) + slice, ok := reflect.TypeAssert[[]uint32](v) if !ok { // It is kind uint32 but not type uint32. TODO: We can handle this unsafely. return false @@ -378,7 +378,7 @@ func encUint64Array(state *encoderState, v reflect.Value) bool { } func encUint64Slice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]uint64) + slice, ok := reflect.TypeAssert[[]uint64](v) if !ok { // It is kind uint64 but not type uint64. TODO: We can handle this unsafely. return false @@ -400,7 +400,7 @@ func encUintptrArray(state *encoderState, v reflect.Value) bool { } func encUintptrSlice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]uintptr) + slice, ok := reflect.TypeAssert[[]uintptr](v) if !ok { // It is kind uintptr but not type uintptr. TODO: We can handle this unsafely. return false diff --git a/src/encoding/gob/encgen.go b/src/encoding/gob/encgen.go index 64f5c69bd44dfe..b89fa47ef6c6a6 100644 --- a/src/encoding/gob/encgen.go +++ b/src/encoding/gob/encgen.go @@ -208,7 +208,7 @@ func enc%[2]sArray(state *encoderState, v reflect.Value) bool { const sliceHelper = ` func enc%[2]sSlice(state *encoderState, v reflect.Value) bool { - slice, ok := v.Interface().([]%[1]s) + slice, ok := reflect.TypeAssert[[]%[1]s](v) if !ok { // It is kind %[1]s but not type %[1]s. TODO: We can handle this unsafely. return false diff --git a/src/encoding/gob/encode.go b/src/encoding/gob/encode.go index 5f4d2539faafe3..15932aabe0b990 100644 --- a/src/encoding/gob/encode.go +++ b/src/encoding/gob/encode.go @@ -440,11 +440,14 @@ func (enc *Encoder) encodeGobEncoder(b *encBuffer, ut *userTypeInfo, v reflect.V // We know it's one of these. switch ut.externalEnc { case xGob: - data, err = v.Interface().(GobEncoder).GobEncode() + gobEncoder, _ := reflect.TypeAssert[GobEncoder](v) + data, err = gobEncoder.GobEncode() case xBinary: - data, err = v.Interface().(encoding.BinaryMarshaler).MarshalBinary() + binaryMarshaler, _ := reflect.TypeAssert[encoding.BinaryMarshaler](v) + data, err = binaryMarshaler.MarshalBinary() case xText: - data, err = v.Interface().(encoding.TextMarshaler).MarshalText() + textMarshaler, _ := reflect.TypeAssert[encoding.TextMarshaler](v) + data, err = textMarshaler.MarshalText() } if err != nil { error_(err) @@ -662,7 +665,7 @@ func (enc *Encoder) encode(b *encBuffer, value reflect.Value, ut *userTypeInfo) for i := 0; i < indir; i++ { value = reflect.Indirect(value) } - if ut.externalEnc == 0 && value.Type().Kind() == reflect.Struct { + if ut.externalEnc == 0 && value.Kind() == reflect.Struct { enc.encodeStruct(b, engine, value) } else { enc.encodeSingle(b, engine, value) diff --git a/src/encoding/gob/gobencdec_test.go b/src/encoding/gob/gobencdec_test.go index ae806fc39a21fc..d30e622aa2cbe7 100644 --- a/src/encoding/gob/gobencdec_test.go +++ b/src/encoding/gob/gobencdec_test.go @@ -806,6 +806,8 @@ func TestIgnoreDepthLimit(t *testing.T) { defer func() { maxIgnoreNestingDepth = oldNestingDepth }() b := new(bytes.Buffer) enc := NewEncoder(b) + + // Nested slice typ := reflect.TypeFor[int]() nested := reflect.ArrayOf(1, typ) for i := 0; i < 100; i++ { @@ -819,4 +821,16 @@ func TestIgnoreDepthLimit(t *testing.T) { if err := dec.Decode(&output); err == nil || err.Error() != expectedErr { t.Errorf("Decode didn't fail with depth limit of 100: want %q, got %q", expectedErr, err) } + + // Nested struct + nested = reflect.StructOf([]reflect.StructField{{Name: "F", Type: typ}}) + for i := 0; i < 100; i++ { + nested = reflect.StructOf([]reflect.StructField{{Name: "F", Type: nested}}) + } + badStruct = reflect.New(reflect.StructOf([]reflect.StructField{{Name: "F", Type: nested}})) + enc.Encode(badStruct.Interface()) + dec = NewDecoder(b) + if err := dec.Decode(&output); err == nil || err.Error() != expectedErr { + t.Errorf("Decode didn't fail with depth limit of 100: want %q, got %q", expectedErr, err) + } } diff --git a/src/encoding/gob/type.go b/src/encoding/gob/type.go index c3ac1dbd61fd51..a26070713f8be8 100644 --- a/src/encoding/gob/type.go +++ b/src/encoding/gob/type.go @@ -8,6 +8,7 @@ import ( "encoding" "errors" "fmt" + "maps" "os" "reflect" "sync" @@ -779,10 +780,7 @@ func buildTypeInfo(ut *userTypeInfo, rt reflect.Type) (*typeInfo, error) { // Create new map with old contents plus new entry. m, _ := typeInfoMap.Load().(map[reflect.Type]*typeInfo) - newm := make(map[reflect.Type]*typeInfo, len(m)) - for k, v := range m { - newm[k] = v - } + newm := maps.Clone(m) newm[rt] = info typeInfoMap.Store(newm) return info, nil diff --git a/src/encoding/json/bench_test.go b/src/encoding/json/bench_test.go index 032114cac14b8e..047188131cecf2 100644 --- a/src/encoding/json/bench_test.go +++ b/src/encoding/json/bench_test.go @@ -8,13 +8,15 @@ // We benchmark converting between the JSON form // and in-memory data structures. +//go:build !goexperiment.jsonv2 + package json import ( "bytes" - "compress/gzip" "fmt" "internal/testenv" + "internal/zstd" "io" "os" "reflect" @@ -44,15 +46,12 @@ var codeJSON []byte var codeStruct codeResponse func codeInit() { - f, err := os.Open("testdata/code.json.gz") + f, err := os.Open("internal/jsontest/testdata/golang_source.json.zst") if err != nil { panic(err) } defer f.Close() - gz, err := gzip.NewReader(f) - if err != nil { - panic(err) - } + gz := zstd.NewReader(f) data, err := io.ReadAll(gz) if err != nil { panic(err) diff --git a/src/encoding/json/decode.go b/src/encoding/json/decode.go index 69a1013b8598fe..46f9ed2da9bc75 100644 --- a/src/encoding/json/decode.go +++ b/src/encoding/json/decode.go @@ -5,6 +5,8 @@ // Represents JSON data structure using native Go types: booleans, floats, // strings, arrays, and maps. +//go:build !goexperiment.jsonv2 + package json import ( @@ -41,11 +43,14 @@ import ( // and the input is a JSON quoted string, Unmarshal calls // [encoding.TextUnmarshaler.UnmarshalText] with the unquoted form of the string. // -// To unmarshal JSON into a struct, Unmarshal matches incoming object -// keys to the keys used by [Marshal] (either the struct field name or its tag), -// preferring an exact match but also accepting a case-insensitive match. By -// default, object keys which don't have a corresponding struct field are -// ignored (see [Decoder.DisallowUnknownFields] for an alternative). +// To unmarshal JSON into a struct, Unmarshal matches incoming object keys to +// the keys used by [Marshal] (either the struct field name or its tag), +// ignoring case. If multiple struct fields match an object key, an exact case +// match is preferred over a case-insensitive one. +// +// Incoming object members are processed in the order observed. If an object +// includes duplicate keys, later duplicates will replace or be merged into +// prior values. // // To unmarshal JSON into an interface value, // Unmarshal stores one of these in the interface value: @@ -113,9 +118,6 @@ func Unmarshal(data []byte, v any) error { // The input can be assumed to be a valid encoding of // a JSON value. UnmarshalJSON must copy the JSON data // if it wishes to retain the data after returning. -// -// By convention, to approximate the behavior of [Unmarshal] itself, -// Unmarshalers implement UnmarshalJSON([]byte("null")) as a no-op. type Unmarshaler interface { UnmarshalJSON([]byte) error } @@ -127,7 +129,7 @@ type UnmarshalTypeError struct { Type reflect.Type // type of Go value it could not be assigned to Offset int64 // error occurred after reading Offset bytes Struct string // name of the struct type containing the field - Field string // the full path from root node to the field + Field string // the full path from root node to the field, include embedded struct } func (e *UnmarshalTypeError) Error() string { @@ -255,7 +257,11 @@ func (d *decodeState) addErrorContext(err error) error { switch err := err.(type) { case *UnmarshalTypeError: err.Struct = d.errorContext.Struct.Name() - err.Field = strings.Join(d.errorContext.FieldStack, ".") + fieldStack := d.errorContext.FieldStack + if err.Field != "" { + fieldStack = append(fieldStack, err.Field) + } + err.Field = strings.Join(fieldStack, ".") } } return err @@ -468,7 +474,7 @@ func indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnm // Prevent infinite loop if v is an interface pointing to its own address: // var v any // v = &v - if v.Elem().Kind() == reflect.Interface && v.Elem().Elem() == v { + if v.Elem().Kind() == reflect.Interface && v.Elem().Elem().Equal(v) { v = v.Elem() break } @@ -476,11 +482,11 @@ func indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnm v.Set(reflect.New(v.Type().Elem())) } if v.Type().NumMethod() > 0 && v.CanInterface() { - if u, ok := v.Interface().(Unmarshaler); ok { + if u, ok := reflect.TypeAssert[Unmarshaler](v); ok { return u, nil, reflect.Value{} } if !decodingNull { - if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { + if u, ok := reflect.TypeAssert[encoding.TextUnmarshaler](v); ok { return nil, u, reflect.Value{} } } @@ -697,7 +703,10 @@ func (d *decodeState) object(v reflect.Value) error { if f != nil { subv = v destring = f.quoted - for _, i := range f.index { + if d.errorContext == nil { + d.errorContext = new(errorContext) + } + for i, ind := range f.index { if subv.Kind() == reflect.Pointer { if subv.IsNil() { // If a struct embeds a pointer to an unexported type, @@ -717,13 +726,16 @@ func (d *decodeState) object(v reflect.Value) error { } subv = subv.Elem() } - subv = subv.Field(i) - } - if d.errorContext == nil { - d.errorContext = new(errorContext) + if i < len(f.index)-1 { + d.errorContext.FieldStack = append( + d.errorContext.FieldStack, + subv.Type().Field(ind).Name, + ) + } + subv = subv.Field(ind) } - d.errorContext.FieldStack = append(d.errorContext.FieldStack, f.name) d.errorContext.Struct = t + d.errorContext.FieldStack = append(d.errorContext.FieldStack, f.name) } else if d.disallowUnknownFields { d.saveError(fmt.Errorf("json: unknown field %q", key)) } @@ -1202,10 +1214,6 @@ func unquoteBytes(s []byte) (t []byte, ok bool) { if c == '\\' || c == '"' || c < ' ' { break } - if c < utf8.RuneSelf { - r++ - continue - } rr, size := utf8.DecodeRune(s[r:]) if rr == utf8.RuneError && size == 1 { break diff --git a/src/encoding/json/decode_test.go b/src/encoding/json/decode_test.go index ed906950394d05..0b26b8eb91813b 100644 --- a/src/encoding/json/decode_test.go +++ b/src/encoding/json/decode_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json import ( @@ -10,6 +12,7 @@ import ( "errors" "fmt" "image" + "io" "maps" "math" "math/big" @@ -62,6 +65,21 @@ func (*SS) UnmarshalJSON(data []byte) error { return &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[SS]()} } +type TAlias T + +func (tt *TAlias) UnmarshalJSON(data []byte) error { + t := T{} + if err := Unmarshal(data, &t); err != nil { + return err + } + *tt = TAlias(t) + return nil +} + +type TOuter struct { + T TAlias +} + // ifaceNumAsFloat64/ifaceNumAsNumber are used to test unmarshaling with and // without UseNumber var ifaceNumAsFloat64 = map[string]any{ @@ -398,6 +416,8 @@ type DoublePtr struct { J **int } +type NestedUnamed struct{ F struct{ V int } } + var unmarshalTests = []struct { CaseName in string @@ -428,6 +448,7 @@ var unmarshalTests = []struct { {CaseName: Name(""), in: `{"x": 1}`, ptr: new(tx), out: tx{}}, {CaseName: Name(""), in: `{"x": 1}`, ptr: new(tx), err: fmt.Errorf("json: unknown field \"x\""), disallowUnknownFields: true}, {CaseName: Name(""), in: `{"S": 23}`, ptr: new(W), out: W{}, err: &UnmarshalTypeError{"number", reflect.TypeFor[SS](), 0, "W", "S"}}, + {CaseName: Name(""), in: `{"T": {"X": 23}}`, ptr: new(TOuter), out: TOuter{}, err: &UnmarshalTypeError{"number", reflect.TypeFor[string](), 8, "TOuter", "T.X"}}, {CaseName: Name(""), in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: Number("3")}}, {CaseName: Name(""), in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: Number("1"), F2: int32(2), F3: Number("3")}, useNumber: true}, {CaseName: Name(""), in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(any), out: ifaceNumAsFloat64}, @@ -442,20 +463,22 @@ var unmarshalTests = []struct { // Z has a "-" tag. {CaseName: Name(""), in: `{"Y": 1, "Z": 2}`, ptr: new(T), out: T{Y: 1}}, - {CaseName: Name(""), in: `{"Y": 1, "Z": 2}`, ptr: new(T), err: fmt.Errorf("json: unknown field \"Z\""), disallowUnknownFields: true}, + {CaseName: Name(""), in: `{"Y": 1, "Z": 2}`, ptr: new(T), out: T{Y: 1}, err: fmt.Errorf("json: unknown field \"Z\""), disallowUnknownFields: true}, {CaseName: Name(""), in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), out: U{Alphabet: "abc"}}, - {CaseName: Name(""), in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, + {CaseName: Name(""), in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), out: U{Alphabet: "abc"}, err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, {CaseName: Name(""), in: `{"alpha": "abc"}`, ptr: new(U), out: U{Alphabet: "abc"}}, {CaseName: Name(""), in: `{"alphabet": "xyz"}`, ptr: new(U), out: U{}}, {CaseName: Name(""), in: `{"alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, // syntax errors + {CaseName: Name(""), in: ``, ptr: new(any), err: &SyntaxError{"unexpected end of JSON input", 0}}, + {CaseName: Name(""), in: " \n\r\t", ptr: new(any), err: &SyntaxError{"unexpected end of JSON input", 4}}, + {CaseName: Name(""), in: `[2, 3`, ptr: new(any), err: &SyntaxError{"unexpected end of JSON input", 5}}, {CaseName: Name(""), in: `{"X": "foo", "Y"}`, err: &SyntaxError{"invalid character '}' after object key", 17}}, {CaseName: Name(""), in: `[1, 2, 3+]`, err: &SyntaxError{"invalid character '+' after array element", 9}}, {CaseName: Name(""), in: `{"X":12x}`, err: &SyntaxError{"invalid character 'x' after object key:value pair", 8}, useNumber: true}, - {CaseName: Name(""), in: `[2, 3`, err: &SyntaxError{msg: "unexpected end of JSON input", Offset: 5}}, - {CaseName: Name(""), in: `{"F3": -}`, ptr: new(V), out: V{F3: Number("-")}, err: &SyntaxError{msg: "invalid character '}' in numeric literal", Offset: 9}}, + {CaseName: Name(""), in: `{"F3": -}`, ptr: new(V), err: &SyntaxError{"invalid character '}' in numeric literal", 9}}, // raw value errors {CaseName: Name(""), in: "\x01 42", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, @@ -547,6 +570,7 @@ var unmarshalTests = []struct { CaseName: Name(""), in: `{"2":4}`, ptr: new(map[u8marshal]int), + out: map[u8marshal]int{}, err: errMissingU8Prefix, }, @@ -555,36 +579,42 @@ var unmarshalTests = []struct { CaseName: Name(""), in: `{"abc":"abc"}`, ptr: new(map[int]string), + out: map[int]string{}, err: &UnmarshalTypeError{Value: "number abc", Type: reflect.TypeFor[int](), Offset: 2}, }, { CaseName: Name(""), in: `{"256":"abc"}`, ptr: new(map[uint8]string), + out: map[uint8]string{}, err: &UnmarshalTypeError{Value: "number 256", Type: reflect.TypeFor[uint8](), Offset: 2}, }, { CaseName: Name(""), in: `{"128":"abc"}`, ptr: new(map[int8]string), + out: map[int8]string{}, err: &UnmarshalTypeError{Value: "number 128", Type: reflect.TypeFor[int8](), Offset: 2}, }, { CaseName: Name(""), in: `{"-1":"abc"}`, ptr: new(map[uint8]string), + out: map[uint8]string{}, err: &UnmarshalTypeError{Value: "number -1", Type: reflect.TypeFor[uint8](), Offset: 2}, }, { CaseName: Name(""), in: `{"F":{"a":2,"3":4}}`, ptr: new(map[string]map[int]int), + out: map[string]map[int]int{"F": {3: 4}}, err: &UnmarshalTypeError{Value: "number a", Type: reflect.TypeFor[int](), Offset: 7}, }, { CaseName: Name(""), in: `{"F":{"a":2,"3":4}}`, ptr: new(map[string]map[uint]int), + out: map[string]map[uint]int{"F": {3: 4}}, err: &UnmarshalTypeError{Value: "number a", Type: reflect.TypeFor[uint](), Offset: 7}, }, @@ -666,6 +696,7 @@ var unmarshalTests = []struct { CaseName: Name(""), in: `{"X": 1,"Y":2}`, ptr: new(S5), + out: S5{S8: S8{S9{Y: 2}}}, err: fmt.Errorf("json: unknown field \"X\""), disallowUnknownFields: true, }, @@ -679,6 +710,7 @@ var unmarshalTests = []struct { CaseName: Name(""), in: `{"X": 1,"Y":2}`, ptr: new(S10), + out: S10{S13: S13{S8{S9{Y: 2}}}}, err: fmt.Errorf("json: unknown field \"X\""), disallowUnknownFields: true, }, @@ -873,6 +905,7 @@ var unmarshalTests = []struct { CaseName: Name(""), in: `{"V": {"F4": {}, "F2": "hello"}}`, ptr: new(VOuter), + out: VOuter{V: V{F4: &VOuter{}}}, err: &UnmarshalTypeError{ Value: "string", Struct: "V", @@ -882,6 +915,20 @@ var unmarshalTests = []struct { }, }, + { + CaseName: Name(""), + in: `{"Level1a": "hello"}`, + ptr: new(Top), + out: Top{Embed0a: &Embed0a{}}, + err: &UnmarshalTypeError{ + Value: "string", + Struct: "Top", + Field: "Embed0a.Level1a", + Type: reflect.TypeFor[int](), + Offset: 19, + }, + }, + // issue 15146. // invalid inputs in wrongStringTests below. {CaseName: Name(""), in: `{"B":"true"}`, ptr: new(B), out: B{true}, golden: true}, @@ -918,7 +965,29 @@ var unmarshalTests = []struct { "Q": 18, "extra": true }`, - ptr: new(Top), + ptr: new(Top), + out: Top{ + Level0: 1, + Embed0: Embed0{ + Level1b: 2, + Level1c: 3, + }, + Embed0a: &Embed0a{Level1a: 5, Level1b: 6}, + Embed0b: &Embed0b{Level1a: 8, Level1b: 9, Level1c: 10, Level1d: 11, Level1e: 12}, + Loop: Loop{ + Loop1: 13, + Loop2: 14, + Loop: nil, + }, + Embed0p: Embed0p{ + Point: image.Point{ + X: 15, + Y: 16, + }, + }, + Embed0q: Embed0q{Point: Point{Z: 17}}, + embed: embed{Q: 18}, + }, err: fmt.Errorf("json: unknown field \"extra\""), disallowUnknownFields: true, }, @@ -946,7 +1015,29 @@ var unmarshalTests = []struct { "Z": 17, "Q": 18 }`, - ptr: new(Top), + ptr: new(Top), + out: Top{ + Level0: 1, + Embed0: Embed0{ + Level1b: 2, + Level1c: 3, + }, + Embed0a: &Embed0a{Level1a: 5, Level1b: 6}, + Embed0b: &Embed0b{Level1a: 8, Level1b: 9, Level1c: 10, Level1d: 11, Level1e: 12}, + Loop: Loop{ + Loop1: 13, + Loop2: 14, + Loop: nil, + }, + Embed0p: Embed0p{ + Point: image.Point{ + X: 15, + Y: 16, + }, + }, + Embed0q: Embed0q{Point: Point{Z: 17}}, + embed: embed{Q: 18}, + }, err: fmt.Errorf("json: unknown field \"extra\""), disallowUnknownFields: true, }, @@ -956,12 +1047,14 @@ var unmarshalTests = []struct { CaseName: Name(""), in: `{"data":{"test1": "bob", "test2": 123}}`, ptr: new(mapStringToStringData), + out: mapStringToStringData{map[string]string{"test1": "bob", "test2": ""}}, err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[string](), Offset: 37, Struct: "mapStringToStringData", Field: "data"}, }, { CaseName: Name(""), in: `{"data":{"test1": 123, "test2": "bob"}}`, ptr: new(mapStringToStringData), + out: mapStringToStringData{Data: map[string]string{"test1": "", "test2": "bob"}}, err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[string](), Offset: 21, Struct: "mapStringToStringData", Field: "data"}, }, @@ -995,12 +1088,13 @@ var unmarshalTests = []struct { CaseName: Name(""), in: `{"Ts": [{"Y": 1}, {"Y": 2}, {"Y": "bad-type"}]}`, ptr: new(PP), + out: PP{Ts: []T{{Y: 1}, {Y: 2}, {Y: 0}}}, err: &UnmarshalTypeError{ Value: "string", Struct: "T", Field: "Ts.Y", Type: reflect.TypeFor[int](), - Offset: 29, + Offset: 44, }, }, // #14702 @@ -1037,8 +1131,168 @@ var unmarshalTests = []struct { CaseName: Name(""), in: `{"A":"invalid"}`, ptr: new(map[string]Number), + out: map[string]Number{}, err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), }, + + { + CaseName: Name(""), + in: `5`, + ptr: new(Number), + out: Number("5"), + }, + { + CaseName: Name(""), + in: `"5"`, + ptr: new(Number), + out: Number("5"), + }, + { + CaseName: Name(""), + in: `{"N":5}`, + ptr: new(struct{ N Number }), + out: struct{ N Number }{"5"}, + }, + { + CaseName: Name(""), + in: `{"N":"5"}`, + ptr: new(struct{ N Number }), + out: struct{ N Number }{"5"}, + }, + { + CaseName: Name(""), + in: `{"N":5}`, + ptr: new(struct { + N Number `json:",string"` + }), + err: fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into json.Number"), + }, + { + CaseName: Name(""), + in: `{"N":"5"}`, + ptr: new(struct { + N Number `json:",string"` + }), + out: struct { + N Number `json:",string"` + }{"5"}, + }, + + // Verify that syntactic errors are immediately fatal, + // while semantic errors are lazily reported + // (i.e., allow processing to continue). + { + CaseName: Name(""), + in: `[1,2,true,4,5}`, + ptr: new([]int), + err: &SyntaxError{msg: "invalid character '}' after array element", Offset: 14}, + }, + { + CaseName: Name(""), + in: `[1,2,true,4,5]`, + ptr: new([]int), + out: []int{1, 2, 0, 4, 5}, + err: &UnmarshalTypeError{Value: "bool", Type: reflect.TypeFor[int](), Offset: 9}, + }, + + { + CaseName: Name("DashComma"), + in: `{"-":"hello"}`, + ptr: new(struct { + F string `json:"-,"` + }), + out: struct { + F string `json:"-,"` + }{"hello"}, + }, + { + CaseName: Name("DashCommaOmitEmpty"), + in: `{"-":"hello"}`, + ptr: new(struct { + F string `json:"-,omitempty"` + }), + out: struct { + F string `json:"-,omitempty"` + }{"hello"}, + }, + + { + CaseName: Name("ErrorForNestedUnamed"), + in: `{"F":{"V":"s"}}`, + ptr: new(NestedUnamed), + out: NestedUnamed{}, + err: &UnmarshalTypeError{Value: "string", Type: reflect.TypeFor[int](), Offset: 13, Field: "F.V"}, + }, + { + CaseName: Name("ErrorInterface"), + in: `1`, + ptr: new(error), + out: error(nil), + err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[error](), Offset: 1}, + }, + { + CaseName: Name("ErrorChan"), + in: `1`, + ptr: new(chan int), + out: (chan int)(nil), + err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[chan int](), Offset: 1}, + }, + + // #75619 + { + CaseName: Name("QuotedInt/GoSyntax"), + in: `{"X": "-0000123"}`, + ptr: new(struct { + X int64 `json:",string"` + }), + out: struct { + X int64 `json:",string"` + }{-123}, + }, + { + CaseName: Name("QuotedInt/Invalid"), + in: `{"X": "123 "}`, + ptr: new(struct { + X int64 `json:",string"` + }), + err: &UnmarshalTypeError{Value: "number 123 ", Type: reflect.TypeFor[int64](), Field: "X", Offset: int64(len(`{"X": "123 "`))}, + }, + { + CaseName: Name("QuotedUint/GoSyntax"), + in: `{"X": "0000123"}`, + ptr: new(struct { + X uint64 `json:",string"` + }), + out: struct { + X uint64 `json:",string"` + }{123}, + }, + { + CaseName: Name("QuotedUint/Invalid"), + in: `{"X": "0x123"}`, + ptr: new(struct { + X uint64 `json:",string"` + }), + err: &UnmarshalTypeError{Value: "number 0x123", Type: reflect.TypeFor[uint64](), Field: "X", Offset: int64(len(`{"X": "0x123"`))}, + }, + { + CaseName: Name("QuotedFloat/GoSyntax"), + in: `{"X": "0x1_4p-2"}`, + ptr: new(struct { + X float64 `json:",string"` + }), + out: struct { + X float64 `json:",string"` + }{0x1_4p-2}, + }, + { + CaseName: Name("QuotedFloat/Invalid"), + in: `{"X": "1.5e1_"}`, + ptr: new(struct { + X float64 `json:",string"` + }), + err: &UnmarshalTypeError{Value: "number 1.5e1_", Type: reflect.TypeFor[float64](), Field: "X", Offset: int64(len(`{"X": "1.5e1_"`))}, + }, } func TestMarshal(t *testing.T) { @@ -1141,9 +1395,28 @@ func TestMarshalEmbeds(t *testing.T) { } func equalError(a, b error) bool { + isJSONError := func(err error) bool { + switch err.(type) { + case + *InvalidUTF8Error, + *InvalidUnmarshalError, + *MarshalerError, + *SyntaxError, + *UnmarshalFieldError, + *UnmarshalTypeError, + *UnsupportedTypeError, + *UnsupportedValueError: + return true + } + return false + } + if a == nil || b == nil { return a == nil && b == nil } + if isJSONError(a) || isJSONError(b) { + return reflect.DeepEqual(a, b) // safe for locally defined error types + } return a.Error() == b.Error() } @@ -1154,7 +1427,7 @@ func TestUnmarshal(t *testing.T) { var scan scanner if err := checkValid(in, &scan); err != nil { if !equalError(err, tt.err) { - t.Fatalf("%s: checkValid error: %#v", tt.Where, err) + t.Fatalf("%s: checkValid error:\n\tgot %#v\n\twant %#v", tt.Where, err, tt.err) } } if tt.ptr == nil { @@ -1187,10 +1460,20 @@ func TestUnmarshal(t *testing.T) { if tt.disallowUnknownFields { dec.DisallowUnknownFields() } + if tt.err != nil && strings.Contains(tt.err.Error(), "unexpected end of JSON input") { + // In streaming mode, we expect EOF or ErrUnexpectedEOF instead. + if strings.TrimSpace(tt.in) == "" { + tt.err = io.EOF + } else { + tt.err = io.ErrUnexpectedEOF + } + } if err := dec.Decode(v.Interface()); !equalError(err, tt.err) { - t.Fatalf("%s: Decode error:\n\tgot: %v\n\twant: %v", tt.Where, err, tt.err) - } else if err != nil { - return + t.Fatalf("%s: Decode error:\n\tgot: %v\n\twant: %v\n\n\tgot: %#v\n\twant: %#v", tt.Where, err, tt.err, err, tt.err) + } else if err != nil && tt.out == nil { + // Initialize tt.out during an error where there are no mutations, + // so the output is just the zero value of the input type. + tt.out = reflect.Zero(v.Elem().Type()).Interface() } if got := v.Elem().Interface(); !reflect.DeepEqual(got, tt.out) { gotJSON, _ := Marshal(got) @@ -1749,19 +2032,12 @@ func TestNullString(t *testing.T) { } } -func intp(x int) *int { - p := new(int) - *p = x - return p -} - -func intpp(x *int) **int { - pp := new(*int) - *pp = x - return pp +func addr[T any](v T) *T { + return &v } func TestInterfaceSet(t *testing.T) { + errUnmarshal := &UnmarshalTypeError{Value: "object", Offset: 6, Type: reflect.TypeFor[int](), Field: "X"} tests := []struct { CaseName pre any @@ -1772,21 +2048,55 @@ func TestInterfaceSet(t *testing.T) { {Name(""), "foo", `2`, 2.0}, {Name(""), "foo", `true`, true}, {Name(""), "foo", `null`, nil}, - - {Name(""), nil, `null`, nil}, - {Name(""), new(int), `null`, nil}, - {Name(""), (*int)(nil), `null`, nil}, - {Name(""), new(*int), `null`, new(*int)}, - {Name(""), (**int)(nil), `null`, nil}, - {Name(""), intp(1), `null`, nil}, - {Name(""), intpp(nil), `null`, intpp(nil)}, - {Name(""), intpp(intp(1)), `null`, intpp(nil)}, + {Name(""), map[string]any{}, `true`, true}, + {Name(""), []string{}, `true`, true}, + + {Name(""), any(nil), `null`, any(nil)}, + {Name(""), (*int)(nil), `null`, any(nil)}, + {Name(""), (*int)(addr(0)), `null`, any(nil)}, + {Name(""), (*int)(addr(1)), `null`, any(nil)}, + {Name(""), (**int)(nil), `null`, any(nil)}, + {Name(""), (**int)(addr[*int](nil)), `null`, (**int)(addr[*int](nil))}, + {Name(""), (**int)(addr(addr(1))), `null`, (**int)(addr[*int](nil))}, + {Name(""), (***int)(nil), `null`, any(nil)}, + {Name(""), (***int)(addr[**int](nil)), `null`, (***int)(addr[**int](nil))}, + {Name(""), (***int)(addr(addr[*int](nil))), `null`, (***int)(addr[**int](nil))}, + {Name(""), (***int)(addr(addr(addr(1)))), `null`, (***int)(addr[**int](nil))}, + + {Name(""), any(nil), `2`, float64(2)}, + {Name(""), (int)(1), `2`, float64(2)}, + {Name(""), (*int)(nil), `2`, float64(2)}, + {Name(""), (*int)(addr(0)), `2`, (*int)(addr(2))}, + {Name(""), (*int)(addr(1)), `2`, (*int)(addr(2))}, + {Name(""), (**int)(nil), `2`, float64(2)}, + {Name(""), (**int)(addr[*int](nil)), `2`, (**int)(addr(addr(2)))}, + {Name(""), (**int)(addr(addr(1))), `2`, (**int)(addr(addr(2)))}, + {Name(""), (***int)(nil), `2`, float64(2)}, + {Name(""), (***int)(addr[**int](nil)), `2`, (***int)(addr(addr(addr(2))))}, + {Name(""), (***int)(addr(addr[*int](nil))), `2`, (***int)(addr(addr(addr(2))))}, + {Name(""), (***int)(addr(addr(addr(1)))), `2`, (***int)(addr(addr(addr(2))))}, + + {Name(""), any(nil), `{}`, map[string]any{}}, + {Name(""), (int)(1), `{}`, map[string]any{}}, + {Name(""), (*int)(nil), `{}`, map[string]any{}}, + {Name(""), (*int)(addr(0)), `{}`, errUnmarshal}, + {Name(""), (*int)(addr(1)), `{}`, errUnmarshal}, + {Name(""), (**int)(nil), `{}`, map[string]any{}}, + {Name(""), (**int)(addr[*int](nil)), `{}`, errUnmarshal}, + {Name(""), (**int)(addr(addr(1))), `{}`, errUnmarshal}, + {Name(""), (***int)(nil), `{}`, map[string]any{}}, + {Name(""), (***int)(addr[**int](nil)), `{}`, errUnmarshal}, + {Name(""), (***int)(addr(addr[*int](nil))), `{}`, errUnmarshal}, + {Name(""), (***int)(addr(addr(addr(1)))), `{}`, errUnmarshal}, } for _, tt := range tests { t.Run(tt.Name, func(t *testing.T) { b := struct{ X any }{tt.pre} blob := `{"X":` + tt.json + `}` if err := Unmarshal([]byte(blob), &b); err != nil { + if wantErr, _ := tt.post.(error); equalError(err, wantErr) { + return + } t.Fatalf("%s: Unmarshal(%#q) error: %v", tt.Where, blob, err) } if !reflect.DeepEqual(b.X, tt.post) { @@ -2193,49 +2503,27 @@ func TestPrefilled(t *testing.T) { } func TestInvalidUnmarshal(t *testing.T) { - buf := []byte(`{"a":"1"}`) - tests := []struct { - CaseName - v any - want string - }{ - {Name(""), nil, "json: Unmarshal(nil)"}, - {Name(""), struct{}{}, "json: Unmarshal(non-pointer struct {})"}, - {Name(""), (*int)(nil), "json: Unmarshal(nil *int)"}, - } - for _, tt := range tests { - t.Run(tt.Name, func(t *testing.T) { - err := Unmarshal(buf, tt.v) - if err == nil { - t.Fatalf("%s: Unmarshal error: got nil, want non-nil", tt.Where) - } - if got := err.Error(); got != tt.want { - t.Errorf("%s: Unmarshal error:\n\tgot: %s\n\twant: %s", tt.Where, got, tt.want) - } - }) - } -} - -func TestInvalidUnmarshalText(t *testing.T) { - buf := []byte(`123`) tests := []struct { CaseName - v any - want string + in string + v any + wantErr error }{ - {Name(""), nil, "json: Unmarshal(nil)"}, - {Name(""), struct{}{}, "json: Unmarshal(non-pointer struct {})"}, - {Name(""), (*int)(nil), "json: Unmarshal(nil *int)"}, - {Name(""), new(net.IP), "json: cannot unmarshal number into Go value of type *net.IP"}, + {Name(""), `{"a":"1"}`, nil, &InvalidUnmarshalError{}}, + {Name(""), `{"a":"1"}`, struct{}{}, &InvalidUnmarshalError{reflect.TypeFor[struct{}]()}}, + {Name(""), `{"a":"1"}`, (*int)(nil), &InvalidUnmarshalError{reflect.TypeFor[*int]()}}, + {Name(""), `123`, nil, &InvalidUnmarshalError{}}, + {Name(""), `123`, struct{}{}, &InvalidUnmarshalError{reflect.TypeFor[struct{}]()}}, + {Name(""), `123`, (*int)(nil), &InvalidUnmarshalError{reflect.TypeFor[*int]()}}, + {Name(""), `123`, new(net.IP), &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[*net.IP](), Offset: 3}}, } for _, tt := range tests { t.Run(tt.Name, func(t *testing.T) { - err := Unmarshal(buf, tt.v) - if err == nil { + switch gotErr := Unmarshal([]byte(tt.in), tt.v); { + case gotErr == nil: t.Fatalf("%s: Unmarshal error: got nil, want non-nil", tt.Where) - } - if got := err.Error(); got != tt.want { - t.Errorf("%s: Unmarshal error:\n\tgot: %s\n\twant: %s", tt.Where, got, tt.want) + case !reflect.DeepEqual(gotErr, tt.wantErr): + t.Errorf("%s: Unmarshal error:\n\tgot: %#v\n\twant: %#v", tt.Where, gotErr, tt.wantErr) } }) } diff --git a/src/encoding/json/encode.go b/src/encoding/json/encode.go index 988de716124862..1b7942dd3ad3f4 100644 --- a/src/encoding/json/encode.go +++ b/src/encoding/json/encode.go @@ -2,12 +2,46 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package json implements encoding and decoding of JSON as defined in -// RFC 7159. The mapping between JSON and Go values is described -// in the documentation for the Marshal and Unmarshal functions. +//go:build !goexperiment.jsonv2 + +// Package json implements encoding and decoding of JSON as defined in RFC 7159. +// The mapping between JSON and Go values is described in the documentation for +// the Marshal and Unmarshal functions. // // See "JSON and Go" for an introduction to this package: // https://golang.org/doc/articles/json_and_go.html +// +// # Security Considerations +// +// The JSON standard (RFC 7159) is lax in its definition of a number of parser +// behaviors. As such, many JSON parsers behave differently in various +// scenarios. These differences in parsers mean that systems that use multiple +// independent JSON parser implementations may parse the same JSON object in +// differing ways. +// +// Systems that rely on a JSON object being parsed consistently for security +// purposes should be careful to understand the behaviors of this parser, as +// well as how these behaviors may cause interoperability issues with other +// parser implementations. +// +// Due to the Go Backwards Compatibility promise (https://go.dev/doc/go1compat) +// there are a number of behaviors this package exhibits that may cause +// interopability issues, but cannot be changed. In particular the following +// parsing behaviors may cause issues: +// +// - If a JSON object contains duplicate keys, keys are processed in the order +// they are observed, meaning later values will replace or be merged into +// prior values, depending on the field type (in particular maps and structs +// will have values merged, while other types have values replaced). +// - When parsing a JSON object into a Go struct, keys are considered in a +// case-insensitive fashion. +// - When parsing a JSON object into a Go struct, unknown keys in the JSON +// object are ignored (unless a [Decoder] is used and +// [Decoder.DisallowUnknownFields] has been called). +// - Invalid UTF-8 bytes in JSON strings are replaced by the Unicode +// replacement character. +// - Large JSON number integers will lose precision when unmarshaled into +// floating-point types. package json import ( @@ -72,8 +106,8 @@ import ( // // The "omitempty" option specifies that the field should be omitted // from the encoding if the field has an empty value, defined as -// false, 0, a nil pointer, a nil interface value, and any empty array, -// slice, map, or string. +// false, 0, a nil pointer, a nil interface value, and any array, +// slice, map, or string of length zero. // // As a special case, if the field tag is "-", the field is always omitted. // Note that a field with name "-" can still be generated using the tag "-,". @@ -99,6 +133,17 @@ import ( // // Field appears in JSON as key "-". // Field int `json:"-,"` // +// The "omitzero" option specifies that the field should be omitted +// from the encoding if the field has a zero value, according to rules: +// +// 1) If the field type has an "IsZero() bool" method, that will be used to +// determine whether the value is zero. +// +// 2) Otherwise, the value is zero if it is the zero value for its type. +// +// If both "omitempty" and "omitzero" are specified, the field will be omitted +// if the value is either empty or zero (or both). +// // The "string" option signals that a field is stored as JSON inside a // JSON-encoded string. It applies only to fields of string, floating point, // integer, or boolean types. This extra level of encoding is sometimes used @@ -346,25 +391,22 @@ func typeEncoder(t reflect.Type) encoderFunc { } // To deal with recursive types, populate the map with an - // indirect func before we build it. This type waits on the - // real func (f) to be ready and then calls it. This indirect - // func is only used for recursive types. - var ( - wg sync.WaitGroup - f encoderFunc - ) - wg.Add(1) + // indirect func before we build it. If the type is recursive, + // the second lookup for the type will return the indirect func. + // + // This indirect func is only used for recursive types, + // and briefly during racing calls to typeEncoder. + indirect := sync.OnceValue(func() encoderFunc { + return newTypeEncoder(t, true) + }) fi, loaded := encoderCache.LoadOrStore(t, encoderFunc(func(e *encodeState, v reflect.Value, opts encOpts) { - wg.Wait() - f(e, v, opts) + indirect()(e, v, opts) })) if loaded { return fi.(encoderFunc) } - // Compute the real encoder and replace the indirect func with it. - f = newTypeEncoder(t, true) - wg.Done() + f := indirect() encoderCache.Store(t, f) return f } @@ -433,7 +475,7 @@ func marshalerEncoder(e *encodeState, v reflect.Value, opts encOpts) { e.WriteString("null") return } - m, ok := v.Interface().(Marshaler) + m, ok := reflect.TypeAssert[Marshaler](v) if !ok { e.WriteString("null") return @@ -456,7 +498,7 @@ func addrMarshalerEncoder(e *encodeState, v reflect.Value, opts encOpts) { e.WriteString("null") return } - m := va.Interface().(Marshaler) + m, _ := reflect.TypeAssert[Marshaler](va) b, err := m.MarshalJSON() if err == nil { e.Grow(len(b)) @@ -474,7 +516,7 @@ func textMarshalerEncoder(e *encodeState, v reflect.Value, opts encOpts) { e.WriteString("null") return } - m, ok := v.Interface().(encoding.TextMarshaler) + m, ok := reflect.TypeAssert[encoding.TextMarshaler](v) if !ok { e.WriteString("null") return @@ -492,7 +534,7 @@ func addrTextMarshalerEncoder(e *encodeState, v reflect.Value, opts encOpts) { e.WriteString("null") return } - m := va.Interface().(encoding.TextMarshaler) + m, _ := reflect.TypeAssert[encoding.TextMarshaler](va) b, err := m.MarshalText() if err != nil { e.error(&MarshalerError{v.Type(), err, "MarshalText"}) @@ -701,7 +743,8 @@ FieldLoop: fv = fv.Field(i) } - if f.omitEmpty && isEmptyValue(fv) { + if (f.omitEmpty && isEmptyValue(fv)) || + (f.omitZero && (f.isZero == nil && fv.IsZero() || (f.isZero != nil && f.isZero(fv)))) { continue } e.WriteByte(next) @@ -948,7 +991,7 @@ func resolveKeyName(k reflect.Value) (string, error) { if k.Kind() == reflect.String { return k.String(), nil } - if tm, ok := k.Interface().(encoding.TextMarshaler); ok { + if tm, ok := reflect.TypeAssert[encoding.TextMarshaler](k); ok { if k.Kind() == reflect.Pointer && k.IsNil() { return "", nil } @@ -1003,10 +1046,7 @@ func appendString[Bytes []byte | string](dst []byte, src Bytes, escapeHTML bool) // For now, cast only a small portion of byte slices to a string // so that it can be stack allocated. This slows down []byte slightly // due to the extra copy, but keeps string performance roughly the same. - n := len(src) - i - if n > utf8.UTFMax { - n = utf8.UTFMax - } + n := min(len(src)-i, utf8.UTFMax) c, size := utf8.DecodeRuneInString(string(src[i : i+n])) if c == utf8.RuneError && size == 1 { dst = append(dst, src[start:i]...) @@ -1048,11 +1088,19 @@ type field struct { index []int typ reflect.Type omitEmpty bool + omitZero bool + isZero func(reflect.Value) bool quoted bool encoder encoderFunc } +type isZeroer interface { + IsZero() bool +} + +var isZeroerType = reflect.TypeFor[isZeroer]() + // typeFields returns a list of fields that JSON should recognize for the given type. // The algorithm is breadth-first search over the set of structs to include - the top struct // and then any reachable anonymous structs. @@ -1154,6 +1202,7 @@ func typeFields(t reflect.Type) structFields { index: index, typ: ft, omitEmpty: opts.Contains("omitempty"), + omitZero: opts.Contains("omitzero"), quoted: quoted, } field.nameBytes = []byte(field.name) @@ -1163,6 +1212,40 @@ func typeFields(t reflect.Type) structFields { field.nameEscHTML = `"` + string(nameEscBuf) + `":` field.nameNonEsc = `"` + field.name + `":` + if field.omitZero { + t := sf.Type + // Provide a function that uses a type's IsZero method. + switch { + case t.Kind() == reflect.Interface && t.Implements(isZeroerType): + field.isZero = func(v reflect.Value) bool { + // Avoid panics calling IsZero on a nil interface or + // non-nil interface with nil pointer. + return v.IsNil() || + (v.Elem().Kind() == reflect.Pointer && v.Elem().IsNil()) || + v.Interface().(isZeroer).IsZero() + } + case t.Kind() == reflect.Pointer && t.Implements(isZeroerType): + field.isZero = func(v reflect.Value) bool { + // Avoid panics calling IsZero on nil pointer. + return v.IsNil() || v.Interface().(isZeroer).IsZero() + } + case t.Implements(isZeroerType): + field.isZero = func(v reflect.Value) bool { + return v.Interface().(isZeroer).IsZero() + } + case reflect.PointerTo(t).Implements(isZeroerType): + field.isZero = func(v reflect.Value) bool { + if !v.CanAddr() { + // Temporarily box v so we can take the address. + v2 := reflect.New(v.Type()).Elem() + v2.Set(v) + v = v2 + } + return v.Addr().Interface().(isZeroer).IsZero() + } + } + } + fields = append(fields, field) if count[f.typ] > 1 { // If there were multiple instances, add a second, diff --git a/src/encoding/json/encode_test.go b/src/encoding/json/encode_test.go index 23a14d0b172927..87074eabd46399 100644 --- a/src/encoding/json/encode_test.go +++ b/src/encoding/json/encode_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json import ( @@ -14,10 +16,13 @@ import ( "regexp" "runtime/debug" "strconv" + "sync" "testing" + "testing/synctest" + "time" ) -type Optionals struct { +type OptionalsEmpty struct { Sr string `json:"sr"` So string `json:"so,omitempty"` Sw string `json:"-"` @@ -45,7 +50,7 @@ type Optionals struct { } func TestOmitEmpty(t *testing.T) { - var want = `{ + const want = `{ "sr": "", "omitempty": 0, "slr": null, @@ -56,8 +61,189 @@ func TestOmitEmpty(t *testing.T) { "str": {}, "sto": {} }` - var o Optionals + var o OptionalsEmpty + o.Sw = "something" + o.Mr = map[string]any{} + o.Mo = map[string]any{} + + got, err := MarshalIndent(&o, "", " ") + if err != nil { + t.Fatalf("MarshalIndent error: %v", err) + } + if got := string(got); got != want { + t.Errorf("MarshalIndent:\n\tgot: %s\n\twant: %s\n", indentNewlines(got), indentNewlines(want)) + } +} + +type NonZeroStruct struct{} + +func (nzs NonZeroStruct) IsZero() bool { + return false +} + +type NoPanicStruct struct { + Int int `json:"int,omitzero"` +} + +func (nps *NoPanicStruct) IsZero() bool { + return nps.Int != 0 +} + +type OptionalsZero struct { + Sr string `json:"sr"` + So string `json:"so,omitzero"` + Sw string `json:"-"` + + Ir int `json:"omitzero"` // actually named omitzero, not an option + Io int `json:"io,omitzero"` + + Slr []string `json:"slr,random"` + Slo []string `json:"slo,omitzero"` + SloNonNil []string `json:"slononnil,omitzero"` + + Mr map[string]any `json:"mr"` + Mo map[string]any `json:",omitzero"` + Moo map[string]any `json:"moo,omitzero"` + + Fr float64 `json:"fr"` + Fo float64 `json:"fo,omitzero"` + Foo float64 `json:"foo,omitzero"` + Foo2 [2]float64 `json:"foo2,omitzero"` + + Br bool `json:"br"` + Bo bool `json:"bo,omitzero"` + + Ur uint `json:"ur"` + Uo uint `json:"uo,omitzero"` + + Str struct{} `json:"str"` + Sto struct{} `json:"sto,omitzero"` + + Time time.Time `json:"time,omitzero"` + TimeLocal time.Time `json:"timelocal,omitzero"` + Nzs NonZeroStruct `json:"nzs,omitzero"` + + NilIsZeroer isZeroer `json:"niliszeroer,omitzero"` // nil interface + NonNilIsZeroer isZeroer `json:"nonniliszeroer,omitzero"` // non-nil interface + NoPanicStruct0 isZeroer `json:"nps0,omitzero"` // non-nil interface with nil pointer + NoPanicStruct1 isZeroer `json:"nps1,omitzero"` // non-nil interface with non-nil pointer + NoPanicStruct2 *NoPanicStruct `json:"nps2,omitzero"` // nil pointer + NoPanicStruct3 *NoPanicStruct `json:"nps3,omitzero"` // non-nil pointer + NoPanicStruct4 NoPanicStruct `json:"nps4,omitzero"` // concrete type +} + +func TestOmitZero(t *testing.T) { + const want = `{ + "sr": "", + "omitzero": 0, + "slr": null, + "slononnil": [], + "mr": {}, + "Mo": {}, + "fr": 0, + "br": false, + "ur": 0, + "str": {}, + "nzs": {}, + "nps1": {}, + "nps3": {}, + "nps4": {} +}` + var o OptionalsZero o.Sw = "something" + o.SloNonNil = make([]string, 0) + o.Mr = map[string]any{} + o.Mo = map[string]any{} + + o.Foo = -0 + o.Foo2 = [2]float64{+0, -0} + + o.TimeLocal = time.Time{}.Local() + + o.NonNilIsZeroer = time.Time{} + o.NoPanicStruct0 = (*NoPanicStruct)(nil) + o.NoPanicStruct1 = &NoPanicStruct{} + o.NoPanicStruct3 = &NoPanicStruct{} + + got, err := MarshalIndent(&o, "", " ") + if err != nil { + t.Fatalf("MarshalIndent error: %v", err) + } + if got := string(got); got != want { + t.Errorf("MarshalIndent:\n\tgot: %s\n\twant: %s\n", indentNewlines(got), indentNewlines(want)) + } +} + +func TestOmitZeroMap(t *testing.T) { + const want = `{ + "foo": { + "sr": "", + "omitzero": 0, + "slr": null, + "mr": null, + "fr": 0, + "br": false, + "ur": 0, + "str": {}, + "nzs": {}, + "nps4": {} + } +}` + m := map[string]OptionalsZero{"foo": {}} + got, err := MarshalIndent(m, "", " ") + if err != nil { + t.Fatalf("MarshalIndent error: %v", err) + } + if got := string(got); got != want { + fmt.Println(got) + t.Errorf("MarshalIndent:\n\tgot: %s\n\twant: %s\n", indentNewlines(got), indentNewlines(want)) + } +} + +type OptionalsEmptyZero struct { + Sr string `json:"sr"` + So string `json:"so,omitempty,omitzero"` + Sw string `json:"-"` + + Io int `json:"io,omitempty,omitzero"` + + Slr []string `json:"slr,random"` + Slo []string `json:"slo,omitempty,omitzero"` + SloNonNil []string `json:"slononnil,omitempty,omitzero"` + + Mr map[string]any `json:"mr"` + Mo map[string]any `json:",omitempty,omitzero"` + + Fr float64 `json:"fr"` + Fo float64 `json:"fo,omitempty,omitzero"` + + Br bool `json:"br"` + Bo bool `json:"bo,omitempty,omitzero"` + + Ur uint `json:"ur"` + Uo uint `json:"uo,omitempty,omitzero"` + + Str struct{} `json:"str"` + Sto struct{} `json:"sto,omitempty,omitzero"` + + Time time.Time `json:"time,omitempty,omitzero"` + Nzs NonZeroStruct `json:"nzs,omitempty,omitzero"` +} + +func TestOmitEmptyZero(t *testing.T) { + const want = `{ + "sr": "", + "slr": null, + "mr": {}, + "fr": 0, + "br": false, + "ur": 0, + "str": {}, + "nzs": {} +}` + var o OptionalsEmptyZero + o.Sw = "something" + o.SloNonNil = make([]string, 0) o.Mr = map[string]any{} o.Mo = map[string]any{} @@ -1219,3 +1405,21 @@ func TestIssue63379(t *testing.T) { } } } + +// Issue #73733: encoding/json used a WaitGroup to coordinate access to cache entries. +// Since WaitGroup.Wait is durably blocking, this caused apparent deadlocks when +// multiple bubbles called json.Marshal at the same time. +func TestSynctestMarshal(t *testing.T) { + var wg sync.WaitGroup + for range 5 { + wg.Go(func() { + synctest.Test(t, func(t *testing.T) { + _, err := Marshal([]string{}) + if err != nil { + t.Errorf("Marshal: %v", err) + } + }) + }) + } + wg.Wait() +} diff --git a/src/encoding/json/example_marshaling_test.go b/src/encoding/json/example_marshaling_test.go index 7f15c742b8ef86..72f4cca8ad8326 100644 --- a/src/encoding/json/example_marshaling_test.go +++ b/src/encoding/json/example_marshaling_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json_test import ( diff --git a/src/encoding/json/example_test.go b/src/encoding/json/example_test.go index 2261c770c01f27..15c2538349444a 100644 --- a/src/encoding/json/example_test.go +++ b/src/encoding/json/example_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json_test import ( diff --git a/src/encoding/json/example_text_marshaling_test.go b/src/encoding/json/example_text_marshaling_test.go index 04c7813b267835..178c7bafd2079c 100644 --- a/src/encoding/json/example_text_marshaling_test.go +++ b/src/encoding/json/example_text_marshaling_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json_test import ( diff --git a/src/encoding/json/fold.go b/src/encoding/json/fold.go index c4c671b52765a4..f096ed60543b26 100644 --- a/src/encoding/json/fold.go +++ b/src/encoding/json/fold.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json import ( diff --git a/src/encoding/json/fold_test.go b/src/encoding/json/fold_test.go index 9d6fd0559d69ef..4d03e3d1c2f736 100644 --- a/src/encoding/json/fold_test.go +++ b/src/encoding/json/fold_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json import ( diff --git a/src/encoding/json/fuzz_test.go b/src/encoding/json/fuzz_test.go index f01960398a0e88..37dc436fcdb9dc 100644 --- a/src/encoding/json/fuzz_test.go +++ b/src/encoding/json/fuzz_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json import ( diff --git a/src/encoding/json/indent.go b/src/encoding/json/indent.go index 01bfdf65e7d5e1..b6f31fb5103fec 100644 --- a/src/encoding/json/indent.go +++ b/src/encoding/json/indent.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json import "bytes" diff --git a/src/encoding/json/internal/internal.go b/src/encoding/json/internal/internal.go new file mode 100644 index 00000000000000..c95f83fe440913 --- /dev/null +++ b/src/encoding/json/internal/internal.go @@ -0,0 +1,42 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package internal + +import "errors" + +// NotForPublicUse is a marker type that an API is for internal use only. +// It does not perfectly prevent usage of that API, but helps to restrict usage. +// Anything with this marker is not covered by the Go compatibility agreement. +type NotForPublicUse struct{} + +// AllowInternalUse is passed from "json" to "jsontext" to authenticate +// that the caller can have access to internal functionality. +var AllowInternalUse NotForPublicUse + +// Sentinel error values internally shared between jsonv1 and jsonv2. +var ( + ErrCycle = errors.New("encountered a cycle") + ErrNonNilReference = errors.New("value must be passed as a non-nil pointer reference") + ErrNilInterface = errors.New("cannot derive concrete type for nil interface with finite type set") +) + +var ( + // TransformMarshalError converts a v2 error into a v1 error. + // It is called only at the top-level of a Marshal function. + TransformMarshalError func(any, error) error + // NewMarshalerError constructs a jsonv1.MarshalerError. + // It is called after a user-defined Marshal method/function fails. + NewMarshalerError func(any, error, string) error + // TransformUnmarshalError converts a v2 error into a v1 error. + // It is called only at the top-level of a Unmarshal function. + TransformUnmarshalError func(any, error) error + + // NewRawNumber returns new(jsonv1.Number). + NewRawNumber func() any + // RawNumberOf returns jsonv1.Number(b). + RawNumberOf func(b []byte) any +) diff --git a/src/encoding/json/internal/jsonflags/flags.go b/src/encoding/json/internal/jsonflags/flags.go new file mode 100644 index 00000000000000..a53dfe7985601d --- /dev/null +++ b/src/encoding/json/internal/jsonflags/flags.go @@ -0,0 +1,215 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +// jsonflags implements all the optional boolean flags. +// These flags are shared across both "json", "jsontext", and "jsonopts". +package jsonflags + +import "encoding/json/internal" + +// Bools represents zero or more boolean flags, all set to true or false. +// The least-significant bit is the boolean value of all flags in the set. +// The remaining bits identify which particular flags. +// +// In common usage, this is OR'd with 0 or 1. For example: +// - (AllowInvalidUTF8 | 0) means "AllowInvalidUTF8 is false" +// - (Multiline | Indent | 1) means "Multiline and Indent are true" +type Bools uint64 + +func (Bools) JSONOptions(internal.NotForPublicUse) {} + +const ( + // AllFlags is the set of all flags. + AllFlags = AllCoderFlags | AllArshalV2Flags | AllArshalV1Flags + + // AllCoderFlags is the set of all encoder/decoder flags. + AllCoderFlags = (maxCoderFlag - 1) - initFlag + + // AllArshalV2Flags is the set of all v2 marshal/unmarshal flags. + AllArshalV2Flags = (maxArshalV2Flag - 1) - (maxCoderFlag - 1) + + // AllArshalV1Flags is the set of all v1 marshal/unmarshal flags. + AllArshalV1Flags = (maxArshalV1Flag - 1) - (maxArshalV2Flag - 1) + + // NonBooleanFlags is the set of non-boolean flags, + // where the value is some other concrete Go type. + // The value of the flag is stored within jsonopts.Struct. + NonBooleanFlags = 0 | + Indent | + IndentPrefix | + ByteLimit | + DepthLimit | + Marshalers | + Unmarshalers + + // DefaultV1Flags is the set of booleans flags that default to true under + // v1 semantics. None of the non-boolean flags differ between v1 and v2. + DefaultV1Flags = 0 | + AllowDuplicateNames | + AllowInvalidUTF8 | + EscapeForHTML | + EscapeForJS | + PreserveRawStrings | + Deterministic | + FormatNilMapAsNull | + FormatNilSliceAsNull | + MatchCaseInsensitiveNames | + CallMethodsWithLegacySemantics | + FormatByteArrayAsArray | + FormatBytesWithLegacySemantics | + FormatDurationAsNano | + MatchCaseSensitiveDelimiter | + MergeWithLegacySemantics | + OmitEmptyWithLegacySemantics | + ParseBytesWithLooseRFC4648 | + ParseTimeWithLooseRFC3339 | + ReportErrorsWithLegacySemantics | + StringifyWithLegacySemantics | + UnmarshalArrayFromAnyLength + + // AnyWhitespace reports whether the encoded output might have any whitespace. + AnyWhitespace = Multiline | SpaceAfterColon | SpaceAfterComma + + // WhitespaceFlags is the set of flags related to whitespace formatting. + // In contrast to AnyWhitespace, this includes Indent and IndentPrefix + // as those settings take no effect if Multiline is false. + WhitespaceFlags = AnyWhitespace | Indent | IndentPrefix + + // AnyEscape is the set of flags related to escaping in a JSON string. + AnyEscape = EscapeForHTML | EscapeForJS + + // CanonicalizeNumbers is the set of flags related to raw number canonicalization. + CanonicalizeNumbers = CanonicalizeRawInts | CanonicalizeRawFloats +) + +// Encoder and decoder flags. +const ( + initFlag Bools = 1 << iota // reserved for the boolean value itself + + AllowDuplicateNames // encode or decode + AllowInvalidUTF8 // encode or decode + WithinArshalCall // encode or decode; for internal use by json.Marshal and json.Unmarshal + OmitTopLevelNewline // encode only; for internal use by json.Marshal and json.MarshalWrite + PreserveRawStrings // encode only + CanonicalizeRawInts // encode only + CanonicalizeRawFloats // encode only + ReorderRawObjects // encode only + EscapeForHTML // encode only + EscapeForJS // encode only + Multiline // encode only + SpaceAfterColon // encode only + SpaceAfterComma // encode only + Indent // encode only; non-boolean flag + IndentPrefix // encode only; non-boolean flag + ByteLimit // encode or decode; non-boolean flag + DepthLimit // encode or decode; non-boolean flag + + maxCoderFlag +) + +// Marshal and Unmarshal flags (for v2). +const ( + _ Bools = (maxCoderFlag >> 1) << iota + + StringifyNumbers // marshal or unmarshal + Deterministic // marshal only + FormatNilMapAsNull // marshal only + FormatNilSliceAsNull // marshal only + OmitZeroStructFields // marshal only + MatchCaseInsensitiveNames // marshal or unmarshal + DiscardUnknownMembers // marshal only + RejectUnknownMembers // unmarshal only + Marshalers // marshal only; non-boolean flag + Unmarshalers // unmarshal only; non-boolean flag + + maxArshalV2Flag +) + +// Marshal and Unmarshal flags (for v1). +const ( + _ Bools = (maxArshalV2Flag >> 1) << iota + + CallMethodsWithLegacySemantics // marshal or unmarshal + FormatByteArrayAsArray // marshal or unmarshal + FormatBytesWithLegacySemantics // marshal or unmarshal + FormatDurationAsNano // marshal or unmarshal + MatchCaseSensitiveDelimiter // marshal or unmarshal + MergeWithLegacySemantics // unmarshal + OmitEmptyWithLegacySemantics // marshal + ParseBytesWithLooseRFC4648 // unmarshal + ParseTimeWithLooseRFC3339 // unmarshal + ReportErrorsWithLegacySemantics // marshal or unmarshal + StringifyWithLegacySemantics // marshal or unmarshal + StringifyBoolsAndStrings // marshal or unmarshal; for internal use by jsonv2.makeStructArshaler + UnmarshalAnyWithRawNumber // unmarshal; for internal use by jsonv1.Decoder.UseNumber + UnmarshalArrayFromAnyLength // unmarshal + + maxArshalV1Flag +) + +// bitsUsed is the number of bits used in the 64-bit boolean flags +const bitsUsed = 42 + +// Static compile check that bitsUsed and maxArshalV1Flag are in sync. +const _ = uint64((1< 0b_110_11011 + dst.Values &= ^src.Presence // e.g., 0b_1000_0011 & 0b_1010_0101 -> 0b_100_00001 + dst.Values |= src.Values // e.g., 0b_1000_0001 | 0b_1001_0010 -> 0b_100_10011 +} + +// Set sets both the presence and value for the provided bool (or set of bools). +func (fs *Flags) Set(f Bools) { + // Select out the bits for the flag identifiers (everything except LSB), + // then set the presence for all the identifier bits (using OR), + // then invert the identifier bits to clear out the values (using AND-NOT), + // then copy over all the identifier bits to the value if LSB is 1. + // e.g., fs := Flags{Presence: 0b_0101_0010, Values: 0b_0001_0010} + // e.g., f := 0b_1001_0001 + id := uint64(f) &^ uint64(1) // e.g., 0b_1001_0001 & 0b_1111_1110 -> 0b_1001_0000 + fs.Presence |= id // e.g., 0b_0101_0010 | 0b_1001_0000 -> 0b_1101_0011 + fs.Values &= ^id // e.g., 0b_0001_0010 & 0b_0110_1111 -> 0b_0000_0010 + fs.Values |= uint64(f&1) * id // e.g., 0b_0000_0010 | 0b_1001_0000 -> 0b_1001_0010 +} + +// Get reports whether the bool (or any of the bools) is true. +// This is generally only used with a singular bool. +// The value bit of f (i.e., the LSB) is ignored. +func (fs Flags) Get(f Bools) bool { + return fs.Values&uint64(f) > 0 +} + +// Has reports whether the bool (or any of the bools) is set. +// The value bit of f (i.e., the LSB) is ignored. +func (fs Flags) Has(f Bools) bool { + return fs.Presence&uint64(f) > 0 +} + +// Clear clears both the presence and value for the provided bool or bools. +// The value bit of f (i.e., the LSB) is ignored. +func (fs *Flags) Clear(f Bools) { + // Invert f to produce a mask to clear all bits in f (using AND). + // e.g., fs := Flags{Presence: 0b_0101_0010, Values: 0b_0001_0010} + // e.g., f := 0b_0001_1000 + mask := uint64(^f) // e.g., 0b_0001_1000 -> 0b_1110_0111 + fs.Presence &= mask // e.g., 0b_0101_0010 & 0b_1110_0111 -> 0b_0100_0010 + fs.Values &= mask // e.g., 0b_0001_0010 & 0b_1110_0111 -> 0b_0000_0010 +} diff --git a/src/encoding/json/internal/jsonflags/flags_test.go b/src/encoding/json/internal/jsonflags/flags_test.go new file mode 100644 index 00000000000000..e4d3358bffdde1 --- /dev/null +++ b/src/encoding/json/internal/jsonflags/flags_test.go @@ -0,0 +1,75 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsonflags + +import "testing" + +func TestFlags(t *testing.T) { + type Check struct{ want Flags } + type Join struct{ in Flags } + type Set struct{ in Bools } + type Clear struct{ in Bools } + type Get struct { + in Bools + want bool + wantOk bool + } + + calls := []any{ + Get{in: AllowDuplicateNames, want: false, wantOk: false}, + Set{in: AllowDuplicateNames | 0}, + Get{in: AllowDuplicateNames, want: false, wantOk: true}, + Set{in: AllowDuplicateNames | 1}, + Get{in: AllowDuplicateNames, want: true, wantOk: true}, + Check{want: Flags{Presence: uint64(AllowDuplicateNames), Values: uint64(AllowDuplicateNames)}}, + Get{in: AllowInvalidUTF8, want: false, wantOk: false}, + Set{in: AllowInvalidUTF8 | 1}, + Get{in: AllowInvalidUTF8, want: true, wantOk: true}, + Set{in: AllowInvalidUTF8 | 0}, + Get{in: AllowInvalidUTF8, want: false, wantOk: true}, + Get{in: AllowDuplicateNames, want: true, wantOk: true}, + Check{want: Flags{Presence: uint64(AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(AllowDuplicateNames)}}, + Set{in: AllowDuplicateNames | AllowInvalidUTF8 | 0}, + Check{want: Flags{Presence: uint64(AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(0)}}, + Set{in: AllowDuplicateNames | AllowInvalidUTF8 | 0}, + Check{want: Flags{Presence: uint64(AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(0)}}, + Set{in: AllowDuplicateNames | AllowInvalidUTF8 | 1}, + Check{want: Flags{Presence: uint64(AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(AllowDuplicateNames | AllowInvalidUTF8)}}, + Join{in: Flags{Presence: 0, Values: 0}}, + Check{want: Flags{Presence: uint64(AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(AllowDuplicateNames | AllowInvalidUTF8)}}, + Join{in: Flags{Presence: uint64(Multiline | AllowInvalidUTF8), Values: uint64(AllowDuplicateNames)}}, + Check{want: Flags{Presence: uint64(Multiline | AllowDuplicateNames | AllowInvalidUTF8), Values: uint64(AllowDuplicateNames)}}, + Clear{in: AllowDuplicateNames | AllowInvalidUTF8}, + Check{want: Flags{Presence: uint64(Multiline), Values: uint64(0)}}, + Set{in: AllowInvalidUTF8 | Deterministic | ReportErrorsWithLegacySemantics | 1}, + Set{in: Multiline | StringifyNumbers | 0}, + Check{want: Flags{Presence: uint64(AllowInvalidUTF8 | Deterministic | ReportErrorsWithLegacySemantics | Multiline | StringifyNumbers), Values: uint64(AllowInvalidUTF8 | Deterministic | ReportErrorsWithLegacySemantics)}}, + Clear{in: ^AllCoderFlags}, + Check{want: Flags{Presence: uint64(AllowInvalidUTF8 | Multiline), Values: uint64(AllowInvalidUTF8)}}, + } + var fs Flags + for i, call := range calls { + switch call := call.(type) { + case Join: + fs.Join(call.in) + case Set: + fs.Set(call.in) + case Clear: + fs.Clear(call.in) + case Get: + got := fs.Get(call.in) + gotOk := fs.Has(call.in) + if got != call.want || gotOk != call.wantOk { + t.Fatalf("%d: GetOk = (%v, %v), want (%v, %v)", i, got, gotOk, call.want, call.wantOk) + } + case Check: + if fs != call.want { + t.Fatalf("%d: got %x, want %x", i, fs, call.want) + } + } + } +} diff --git a/src/encoding/json/internal/jsonopts/options.go b/src/encoding/json/internal/jsonopts/options.go new file mode 100644 index 00000000000000..39da81b34549ec --- /dev/null +++ b/src/encoding/json/internal/jsonopts/options.go @@ -0,0 +1,202 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsonopts + +import ( + "encoding/json/internal" + "encoding/json/internal/jsonflags" +) + +// Options is the common options type shared across json packages. +type Options interface { + // JSONOptions is exported so related json packages can implement Options. + JSONOptions(internal.NotForPublicUse) +} + +// Struct is the combination of all options in struct form. +// This is efficient to pass down the call stack and to query. +type Struct struct { + Flags jsonflags.Flags + + CoderValues + ArshalValues +} + +type CoderValues struct { + Indent string // jsonflags.Indent + IndentPrefix string // jsonflags.IndentPrefix + ByteLimit int64 // jsonflags.ByteLimit + DepthLimit int // jsonflags.DepthLimit +} + +type ArshalValues struct { + // The Marshalers and Unmarshalers fields use the any type to avoid a + // concrete dependency on *json.Marshalers and *json.Unmarshalers, + // which would in turn create a dependency on the "reflect" package. + + Marshalers any // jsonflags.Marshalers + Unmarshalers any // jsonflags.Unmarshalers + + Format string + FormatDepth int +} + +// DefaultOptionsV2 is the set of all options that define default v2 behavior. +var DefaultOptionsV2 = Struct{ + Flags: jsonflags.Flags{ + Presence: uint64(jsonflags.DefaultV1Flags), + Values: uint64(0), // all flags in DefaultV1Flags are false + }, +} + +// DefaultOptionsV1 is the set of all options that define default v1 behavior. +var DefaultOptionsV1 = Struct{ + Flags: jsonflags.Flags{ + Presence: uint64(jsonflags.DefaultV1Flags), + Values: uint64(jsonflags.DefaultV1Flags), // all flags in DefaultV1Flags are true + }, +} + +func (*Struct) JSONOptions(internal.NotForPublicUse) {} + +// GetUnknownOption is injected by the "json" package to handle Options +// declared in that package so that "jsonopts" can handle them. +var GetUnknownOption = func(Struct, Options) (any, bool) { panic("unknown option") } + +func GetOption[T any](opts Options, setter func(T) Options) (T, bool) { + // Collapse the options to *Struct to simplify lookup. + structOpts, ok := opts.(*Struct) + if !ok { + var structOpts2 Struct + structOpts2.Join(opts) + structOpts = &structOpts2 + } + + // Lookup the option based on the return value of the setter. + var zero T + switch opt := setter(zero).(type) { + case jsonflags.Bools: + v := structOpts.Flags.Get(opt) + ok := structOpts.Flags.Has(opt) + return any(v).(T), ok + case Indent: + if !structOpts.Flags.Has(jsonflags.Indent) { + return zero, false + } + return any(structOpts.Indent).(T), true + case IndentPrefix: + if !structOpts.Flags.Has(jsonflags.IndentPrefix) { + return zero, false + } + return any(structOpts.IndentPrefix).(T), true + case ByteLimit: + if !structOpts.Flags.Has(jsonflags.ByteLimit) { + return zero, false + } + return any(structOpts.ByteLimit).(T), true + case DepthLimit: + if !structOpts.Flags.Has(jsonflags.DepthLimit) { + return zero, false + } + return any(structOpts.DepthLimit).(T), true + default: + v, ok := GetUnknownOption(*structOpts, opt) + return v.(T), ok + } +} + +// JoinUnknownOption is injected by the "json" package to handle Options +// declared in that package so that "jsonopts" can handle them. +var JoinUnknownOption = func(Struct, Options) Struct { panic("unknown option") } + +func (dst *Struct) Join(srcs ...Options) { + dst.join(false, srcs...) +} + +func (dst *Struct) JoinWithoutCoderOptions(srcs ...Options) { + dst.join(true, srcs...) +} + +func (dst *Struct) join(excludeCoderOptions bool, srcs ...Options) { + for _, src := range srcs { + switch src := src.(type) { + case nil: + continue + case jsonflags.Bools: + if excludeCoderOptions { + src &= ^jsonflags.AllCoderFlags + } + dst.Flags.Set(src) + case Indent: + if excludeCoderOptions { + continue + } + dst.Flags.Set(jsonflags.Multiline | jsonflags.Indent | 1) + dst.Indent = string(src) + case IndentPrefix: + if excludeCoderOptions { + continue + } + dst.Flags.Set(jsonflags.Multiline | jsonflags.IndentPrefix | 1) + dst.IndentPrefix = string(src) + case ByteLimit: + if excludeCoderOptions { + continue + } + dst.Flags.Set(jsonflags.ByteLimit | 1) + dst.ByteLimit = int64(src) + case DepthLimit: + if excludeCoderOptions { + continue + } + dst.Flags.Set(jsonflags.DepthLimit | 1) + dst.DepthLimit = int(src) + case *Struct: + srcFlags := src.Flags // shallow copy the flags + if excludeCoderOptions { + srcFlags.Clear(jsonflags.AllCoderFlags) + } + dst.Flags.Join(srcFlags) + if srcFlags.Has(jsonflags.NonBooleanFlags) { + if srcFlags.Has(jsonflags.Indent) { + dst.Indent = src.Indent + } + if srcFlags.Has(jsonflags.IndentPrefix) { + dst.IndentPrefix = src.IndentPrefix + } + if srcFlags.Has(jsonflags.ByteLimit) { + dst.ByteLimit = src.ByteLimit + } + if srcFlags.Has(jsonflags.DepthLimit) { + dst.DepthLimit = src.DepthLimit + } + if srcFlags.Has(jsonflags.Marshalers) { + dst.Marshalers = src.Marshalers + } + if srcFlags.Has(jsonflags.Unmarshalers) { + dst.Unmarshalers = src.Unmarshalers + } + } + default: + *dst = JoinUnknownOption(*dst, src) + } + } +} + +type ( + Indent string // jsontext.WithIndent + IndentPrefix string // jsontext.WithIndentPrefix + ByteLimit int64 // jsontext.WithByteLimit + DepthLimit int // jsontext.WithDepthLimit + // type for jsonflags.Marshalers declared in "json" package + // type for jsonflags.Unmarshalers declared in "json" package +) + +func (Indent) JSONOptions(internal.NotForPublicUse) {} +func (IndentPrefix) JSONOptions(internal.NotForPublicUse) {} +func (ByteLimit) JSONOptions(internal.NotForPublicUse) {} +func (DepthLimit) JSONOptions(internal.NotForPublicUse) {} diff --git a/src/encoding/json/internal/jsonopts/options_test.go b/src/encoding/json/internal/jsonopts/options_test.go new file mode 100644 index 00000000000000..caa686e4f0d579 --- /dev/null +++ b/src/encoding/json/internal/jsonopts/options_test.go @@ -0,0 +1,236 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsonopts_test + +import ( + "reflect" + "testing" + + "encoding/json/internal/jsonflags" + . "encoding/json/internal/jsonopts" + "encoding/json/jsontext" + "encoding/json/v2" +) + +func makeFlags(f ...jsonflags.Bools) (fs jsonflags.Flags) { + for _, f := range f { + fs.Set(f) + } + return fs +} + +func TestJoin(t *testing.T) { + tests := []struct { + in Options + excludeCoders bool + want *Struct + }{{ + in: jsonflags.AllowInvalidUTF8 | 1, + want: &Struct{Flags: makeFlags(jsonflags.AllowInvalidUTF8 | 1)}, + }, { + in: jsonflags.Multiline | 0, + want: &Struct{ + Flags: makeFlags(jsonflags.AllowInvalidUTF8|1, jsonflags.Multiline|0)}, + }, { + in: Indent("\t"), // implicitly sets Multiline=true + want: &Struct{ + Flags: makeFlags(jsonflags.AllowInvalidUTF8 | jsonflags.Multiline | jsonflags.Indent | 1), + CoderValues: CoderValues{Indent: "\t"}, + }, + }, { + in: &Struct{ + Flags: makeFlags(jsonflags.Multiline|jsonflags.EscapeForJS|0, jsonflags.AllowInvalidUTF8|1), + }, + want: &Struct{ + Flags: makeFlags(jsonflags.AllowInvalidUTF8|jsonflags.Indent|1, jsonflags.Multiline|jsonflags.EscapeForJS|0), + CoderValues: CoderValues{Indent: "\t"}, + }, + }, { + in: &DefaultOptionsV1, + want: func() *Struct { + v1 := DefaultOptionsV1 + v1.Flags.Set(jsonflags.Indent | 1) + v1.Flags.Set(jsonflags.Multiline | 0) + v1.Indent = "\t" + return &v1 + }(), // v1 fully replaces before (except for whitespace related flags) + }, { + in: &DefaultOptionsV2, + want: func() *Struct { + v2 := DefaultOptionsV2 + v2.Flags.Set(jsonflags.Indent | 1) + v2.Flags.Set(jsonflags.Multiline | 0) + v2.Indent = "\t" + return &v2 + }(), // v2 fully replaces before (except for whitespace related flags) + }, { + in: jsonflags.Deterministic | jsonflags.AllowInvalidUTF8 | 1, excludeCoders: true, + want: func() *Struct { + v2 := DefaultOptionsV2 + v2.Flags.Set(jsonflags.Deterministic | 1) + v2.Flags.Set(jsonflags.Indent | 1) + v2.Flags.Set(jsonflags.Multiline | 0) + v2.Indent = "\t" + return &v2 + }(), + }, { + in: jsontext.WithIndentPrefix(" "), excludeCoders: true, + want: func() *Struct { + v2 := DefaultOptionsV2 + v2.Flags.Set(jsonflags.Deterministic | 1) + v2.Flags.Set(jsonflags.Indent | 1) + v2.Flags.Set(jsonflags.Multiline | 0) + v2.Indent = "\t" + return &v2 + }(), + }, { + in: jsontext.WithIndentPrefix(" "), excludeCoders: false, + want: func() *Struct { + v2 := DefaultOptionsV2 + v2.Flags.Set(jsonflags.Deterministic | 1) + v2.Flags.Set(jsonflags.Indent | 1) + v2.Flags.Set(jsonflags.IndentPrefix | 1) + v2.Flags.Set(jsonflags.Multiline | 1) + v2.Indent = "\t" + v2.IndentPrefix = " " + return &v2 + }(), + }, { + in: &Struct{ + Flags: jsonflags.Flags{ + Presence: uint64(jsonflags.Deterministic | jsonflags.Indent | jsonflags.IndentPrefix), + Values: uint64(jsonflags.Indent | jsonflags.IndentPrefix), + }, + CoderValues: CoderValues{Indent: " ", IndentPrefix: " "}, + }, + excludeCoders: true, + want: func() *Struct { + v2 := DefaultOptionsV2 + v2.Flags.Set(jsonflags.Indent | 1) + v2.Flags.Set(jsonflags.IndentPrefix | 1) + v2.Flags.Set(jsonflags.Multiline | 1) + v2.Indent = "\t" + v2.IndentPrefix = " " + return &v2 + }(), + }, { + in: &Struct{ + Flags: jsonflags.Flags{ + Presence: uint64(jsonflags.Deterministic | jsonflags.Indent | jsonflags.IndentPrefix), + Values: uint64(jsonflags.Indent | jsonflags.IndentPrefix), + }, + CoderValues: CoderValues{Indent: " ", IndentPrefix: " "}, + }, + excludeCoders: false, + want: func() *Struct { + v2 := DefaultOptionsV2 + v2.Flags.Set(jsonflags.Indent | 1) + v2.Flags.Set(jsonflags.IndentPrefix | 1) + v2.Flags.Set(jsonflags.Multiline | 1) + v2.Indent = " " + v2.IndentPrefix = " " + return &v2 + }(), + }} + got := new(Struct) + for i, tt := range tests { + if tt.excludeCoders { + got.JoinWithoutCoderOptions(tt.in) + } else { + got.Join(tt.in) + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("%d: Join:\n\tgot: %+v\n\twant: %+v", i, got, tt.want) + } + } +} + +func TestGet(t *testing.T) { + opts := &Struct{ + Flags: makeFlags(jsonflags.Indent|jsonflags.Deterministic|jsonflags.Marshalers|1, jsonflags.Multiline|0), + CoderValues: CoderValues{Indent: "\t"}, + ArshalValues: ArshalValues{Marshalers: new(json.Marshalers)}, + } + if v, ok := json.GetOption(nil, jsontext.AllowDuplicateNames); v || ok { + t.Errorf("GetOption(..., AllowDuplicateNames) = (%v, %v), want (false, false)", v, ok) + } + if v, ok := json.GetOption(jsonflags.AllowInvalidUTF8|0, jsontext.AllowDuplicateNames); v || ok { + t.Errorf("GetOption(..., AllowDuplicateNames) = (%v, %v), want (false, false)", v, ok) + } + if v, ok := json.GetOption(jsonflags.AllowDuplicateNames|0, jsontext.AllowDuplicateNames); v || !ok { + t.Errorf("GetOption(..., AllowDuplicateNames) = (%v, %v), want (false, true)", v, ok) + } + if v, ok := json.GetOption(jsonflags.AllowDuplicateNames|1, jsontext.AllowDuplicateNames); !v || !ok { + t.Errorf("GetOption(..., AllowDuplicateNames) = (%v, %v), want (true, true)", v, ok) + } + if v, ok := json.GetOption(Indent(""), jsontext.AllowDuplicateNames); v || ok { + t.Errorf("GetOption(..., AllowDuplicateNames) = (%v, %v), want (false, false)", v, ok) + } + if v, ok := json.GetOption(Indent(" "), jsontext.WithIndent); v != " " || !ok { + t.Errorf(`GetOption(..., WithIndent) = (%q, %v), want (" ", true)`, v, ok) + } + if v, ok := json.GetOption(jsonflags.AllowDuplicateNames|1, jsontext.WithIndent); v != "" || ok { + t.Errorf(`GetOption(..., WithIndent) = (%q, %v), want ("", false)`, v, ok) + } + if v, ok := json.GetOption(opts, jsontext.AllowDuplicateNames); v || ok { + t.Errorf("GetOption(..., AllowDuplicateNames) = (%v, %v), want (false, false)", v, ok) + } + if v, ok := json.GetOption(opts, json.Deterministic); !v || !ok { + t.Errorf("GetOption(..., Deterministic) = (%v, %v), want (true, true)", v, ok) + } + if v, ok := json.GetOption(opts, jsontext.Multiline); v || !ok { + t.Errorf("GetOption(..., Multiline) = (%v, %v), want (false, true)", v, ok) + } + if v, ok := json.GetOption(opts, jsontext.AllowInvalidUTF8); v || ok { + t.Errorf("GetOption(..., AllowInvalidUTF8) = (%v, %v), want (false, false)", v, ok) + } + if v, ok := json.GetOption(opts, jsontext.WithIndent); v != "\t" || !ok { + t.Errorf(`GetOption(..., WithIndent) = (%q, %v), want ("\t", true)`, v, ok) + } + if v, ok := json.GetOption(opts, jsontext.WithIndentPrefix); v != "" || ok { + t.Errorf(`GetOption(..., WithIndentPrefix) = (%q, %v), want ("", false)`, v, ok) + } + if v, ok := json.GetOption(opts, json.WithMarshalers); v == nil || !ok { + t.Errorf(`GetOption(..., WithMarshalers) = (%v, %v), want (non-nil, true)`, v, ok) + } + if v, ok := json.GetOption(opts, json.WithUnmarshalers); v != nil || ok { + t.Errorf(`GetOption(..., WithUnmarshalers) = (%v, %v), want (nil, false)`, v, ok) + } + if v, ok := json.GetOption(json.DefaultOptionsV2(), json.WithMarshalers); v != nil || ok { + t.Errorf(`GetOption(..., WithMarshalers) = (%v, %v), want (nil, false)`, v, ok) + } +} + +var sink struct { + Bool bool + String string + Marshalers *json.Marshalers +} + +func BenchmarkGetBool(b *testing.B) { + b.ReportAllocs() + opts := json.DefaultOptionsV2() + for range b.N { + sink.Bool, sink.Bool = json.GetOption(opts, jsontext.AllowDuplicateNames) + } +} + +func BenchmarkGetIndent(b *testing.B) { + b.ReportAllocs() + opts := json.DefaultOptionsV2() + for range b.N { + sink.String, sink.Bool = json.GetOption(opts, jsontext.WithIndent) + } +} + +func BenchmarkGetMarshalers(b *testing.B) { + b.ReportAllocs() + opts := json.JoinOptions(json.DefaultOptionsV2(), json.WithMarshalers(nil)) + for range b.N { + sink.Marshalers, sink.Bool = json.GetOption(opts, json.WithMarshalers) + } +} diff --git a/src/encoding/json/internal/jsontest/testcase.go b/src/encoding/json/internal/jsontest/testcase.go new file mode 100644 index 00000000000000..73a64c8cfada3b --- /dev/null +++ b/src/encoding/json/internal/jsontest/testcase.go @@ -0,0 +1,37 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontest + +import ( + "fmt" + "path" + "runtime" +) + +// TODO(https://go.dev/issue/52751): Replace with native testing support. + +// CaseName is a case name annotated with a file and line. +type CaseName struct { + Name string + Where CasePos +} + +// Name annotates a case name with the file and line of the caller. +func Name(s string) (c CaseName) { + c.Name = s + runtime.Callers(2, c.Where.pc[:]) + return c +} + +// CasePos represents a file and line number. +type CasePos struct{ pc [1]uintptr } + +func (pos CasePos) String() string { + frames := runtime.CallersFrames(pos.pc[:]) + frame, _ := frames.Next() + return fmt.Sprintf("%s:%d", path.Base(frame.File), frame.Line) +} diff --git a/src/encoding/json/internal/jsontest/testdata.go b/src/encoding/json/internal/jsontest/testdata.go new file mode 100644 index 00000000000000..74de366136ce97 --- /dev/null +++ b/src/encoding/json/internal/jsontest/testdata.go @@ -0,0 +1,607 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +// Package jsontest contains functionality to assist in testing JSON. +package jsontest + +import ( + "bytes" + "embed" + "errors" + "internal/zstd" + "io" + "io/fs" + "path" + "slices" + "strings" + "sync" + "time" +) + +// Embed the testdata directory as a fs.FS because this package is imported +// by other packages such that the location of testdata may change relative +// to the working directory of the test itself. +// +//go:embed testdata/*.json.zst +var testdataFS embed.FS + +type Entry struct { + Name string + Data func() []byte + New func() any // nil if there is no concrete type for this +} + +func mustGet[T any](v T, err error) T { + if err != nil { + panic(err) + } + return v +} + +// Data is a list of JSON testdata. +var Data = func() (entries []Entry) { + fis := mustGet(fs.ReadDir(testdataFS, "testdata")) + slices.SortFunc(fis, func(x, y fs.DirEntry) int { return strings.Compare(x.Name(), y.Name()) }) + for _, fi := range fis { + var entry Entry + + // Convert snake_case file name to CamelCase. + words := strings.Split(strings.TrimSuffix(fi.Name(), ".json.zst"), "_") + for i := range words { + words[i] = strings.Title(words[i]) + } + entry.Name = strings.Join(words, "") + + // Lazily read and decompress the test data. + entry.Data = sync.OnceValue(func() []byte { + filePath := path.Join("testdata", fi.Name()) + b := mustGet(fs.ReadFile(testdataFS, filePath)) + zr := zstd.NewReader(bytes.NewReader(b)) + return mustGet(io.ReadAll(zr)) + }) + + // Check whether there is a concrete type for this data. + switch entry.Name { + case "CanadaGeometry": + entry.New = func() any { return new(canadaRoot) } + case "CitmCatalog": + entry.New = func() any { return new(citmRoot) } + case "GolangSource": + entry.New = func() any { return new(golangRoot) } + case "StringEscaped": + entry.New = func() any { return new(stringRoot) } + case "StringUnicode": + entry.New = func() any { return new(stringRoot) } + case "SyntheaFhir": + entry.New = func() any { return new(syntheaRoot) } + case "TwitterStatus": + entry.New = func() any { return new(twitterRoot) } + } + + entries = append(entries, entry) + } + return entries +}() + +type ( + canadaRoot struct { + Type string `json:"type"` + Features []struct { + Type string `json:"type"` + Properties struct { + Name string `json:"name"` + } `json:"properties"` + Geometry struct { + Type string `json:"type"` + Coordinates [][][2]float64 `json:"coordinates"` + } `json:"geometry"` + } `json:"features"` + } +) + +type ( + citmRoot struct { + AreaNames map[int64]string `json:"areaNames"` + AudienceSubCategoryNames map[int64]string `json:"audienceSubCategoryNames"` + BlockNames map[int64]string `json:"blockNames"` + Events map[int64]struct { + Description string `json:"description"` + ID int `json:"id"` + Logo string `json:"logo"` + Name string `json:"name"` + SubTopicIds []int `json:"subTopicIds"` + SubjectCode any `json:"subjectCode"` + Subtitle any `json:"subtitle"` + TopicIds []int `json:"topicIds"` + } `json:"events"` + Performances []struct { + EventID int `json:"eventId"` + ID int `json:"id"` + Logo any `json:"logo"` + Name any `json:"name"` + Prices []struct { + Amount int `json:"amount"` + AudienceSubCategoryID int64 `json:"audienceSubCategoryId"` + SeatCategoryID int64 `json:"seatCategoryId"` + } `json:"prices"` + SeatCategories []struct { + Areas []struct { + AreaID int `json:"areaId"` + BlockIds []any `json:"blockIds"` + } `json:"areas"` + SeatCategoryID int `json:"seatCategoryId"` + } `json:"seatCategories"` + SeatMapImage any `json:"seatMapImage"` + Start int64 `json:"start"` + VenueCode string `json:"venueCode"` + } `json:"performances"` + SeatCategoryNames map[uint64]string `json:"seatCategoryNames"` + SubTopicNames map[uint64]string `json:"subTopicNames"` + SubjectNames map[uint64]string `json:"subjectNames"` + TopicNames map[uint64]string `json:"topicNames"` + TopicSubTopics map[uint64][]uint64 `json:"topicSubTopics"` + VenueNames map[string]string `json:"venueNames"` + } +) + +type ( + golangRoot struct { + Tree *golangNode `json:"tree"` + Username string `json:"username"` + } + golangNode struct { + Name string `json:"name"` + Kids []golangNode `json:"kids"` + CLWeight float64 `json:"cl_weight"` + Touches int `json:"touches"` + MinT uint64 `json:"min_t"` + MaxT uint64 `json:"max_t"` + MeanT uint64 `json:"mean_t"` + } +) + +type ( + stringRoot struct { + Arabic string `json:"Arabic"` + ArabicPresentationFormsA string `json:"Arabic Presentation Forms-A"` + ArabicPresentationFormsB string `json:"Arabic Presentation Forms-B"` + Armenian string `json:"Armenian"` + Arrows string `json:"Arrows"` + Bengali string `json:"Bengali"` + Bopomofo string `json:"Bopomofo"` + BoxDrawing string `json:"Box Drawing"` + CJKCompatibility string `json:"CJK Compatibility"` + CJKCompatibilityForms string `json:"CJK Compatibility Forms"` + CJKCompatibilityIdeographs string `json:"CJK Compatibility Ideographs"` + CJKSymbolsAndPunctuation string `json:"CJK Symbols and Punctuation"` + CJKUnifiedIdeographs string `json:"CJK Unified Ideographs"` + CJKUnifiedIdeographsExtensionA string `json:"CJK Unified Ideographs Extension A"` + CJKUnifiedIdeographsExtensionB string `json:"CJK Unified Ideographs Extension B"` + Cherokee string `json:"Cherokee"` + CurrencySymbols string `json:"Currency Symbols"` + Cyrillic string `json:"Cyrillic"` + CyrillicSupplementary string `json:"Cyrillic Supplementary"` + Devanagari string `json:"Devanagari"` + EnclosedAlphanumerics string `json:"Enclosed Alphanumerics"` + EnclosedCJKLettersAndMonths string `json:"Enclosed CJK Letters and Months"` + Ethiopic string `json:"Ethiopic"` + GeometricShapes string `json:"Geometric Shapes"` + Georgian string `json:"Georgian"` + GreekAndCoptic string `json:"Greek and Coptic"` + Gujarati string `json:"Gujarati"` + Gurmukhi string `json:"Gurmukhi"` + HangulCompatibilityJamo string `json:"Hangul Compatibility Jamo"` + HangulJamo string `json:"Hangul Jamo"` + HangulSyllables string `json:"Hangul Syllables"` + Hebrew string `json:"Hebrew"` + Hiragana string `json:"Hiragana"` + IPAExtentions string `json:"IPA Extentions"` + KangxiRadicals string `json:"Kangxi Radicals"` + Katakana string `json:"Katakana"` + Khmer string `json:"Khmer"` + KhmerSymbols string `json:"Khmer Symbols"` + Latin string `json:"Latin"` + LatinExtendedAdditional string `json:"Latin Extended Additional"` + Latin1Supplement string `json:"Latin-1 Supplement"` + LatinExtendedA string `json:"Latin-Extended A"` + LatinExtendedB string `json:"Latin-Extended B"` + LetterlikeSymbols string `json:"Letterlike Symbols"` + Malayalam string `json:"Malayalam"` + MathematicalAlphanumericSymbols string `json:"Mathematical Alphanumeric Symbols"` + MathematicalOperators string `json:"Mathematical Operators"` + MiscellaneousSymbols string `json:"Miscellaneous Symbols"` + Mongolian string `json:"Mongolian"` + NumberForms string `json:"Number Forms"` + Oriya string `json:"Oriya"` + PhoneticExtensions string `json:"Phonetic Extensions"` + SupplementalArrowsB string `json:"Supplemental Arrows-B"` + Syriac string `json:"Syriac"` + Tamil string `json:"Tamil"` + Thaana string `json:"Thaana"` + Thai string `json:"Thai"` + UnifiedCanadianAboriginalSyllabics string `json:"Unified Canadian Aboriginal Syllabics"` + YiRadicals string `json:"Yi Radicals"` + YiSyllables string `json:"Yi Syllables"` + } +) + +type ( + syntheaRoot struct { + Entry []struct { + FullURL string `json:"fullUrl"` + Request *struct { + Method string `json:"method"` + URL string `json:"url"` + } `json:"request"` + Resource *struct { + AbatementDateTime time.Time `json:"abatementDateTime"` + AchievementStatus syntheaCode `json:"achievementStatus"` + Active bool `json:"active"` + Activity []struct { + Detail *struct { + Code syntheaCode `json:"code"` + Location syntheaReference `json:"location"` + Status string `json:"status"` + } `json:"detail"` + } `json:"activity"` + Address []syntheaAddress `json:"address"` + Addresses []syntheaReference `json:"addresses"` + AuthoredOn time.Time `json:"authoredOn"` + BillablePeriod syntheaRange `json:"billablePeriod"` + BirthDate string `json:"birthDate"` + CareTeam []struct { + Provider syntheaReference `json:"provider"` + Reference string `json:"reference"` + Role syntheaCode `json:"role"` + Sequence int64 `json:"sequence"` + } `json:"careTeam"` + Category []syntheaCode `json:"category"` + Claim syntheaReference `json:"claim"` + Class syntheaCoding `json:"class"` + ClinicalStatus syntheaCode `json:"clinicalStatus"` + Code syntheaCode `json:"code"` + Communication []struct { + Language syntheaCode `json:"language"` + } `json:"communication"` + Component []struct { + Code syntheaCode `json:"code"` + ValueQuantity syntheaCoding `json:"valueQuantity"` + } `json:"component"` + Contained []struct { + Beneficiary syntheaReference `json:"beneficiary"` + ID string `json:"id"` + Intent string `json:"intent"` + Payor []syntheaReference `json:"payor"` + Performer []syntheaReference `json:"performer"` + Requester syntheaReference `json:"requester"` + ResourceType string `json:"resourceType"` + Status string `json:"status"` + Subject syntheaReference `json:"subject"` + Type syntheaCode `json:"type"` + } `json:"contained"` + Created time.Time `json:"created"` + DeceasedDateTime time.Time `json:"deceasedDateTime"` + Description syntheaCode `json:"description"` + Diagnosis []struct { + DiagnosisReference syntheaReference `json:"diagnosisReference"` + Sequence int64 `json:"sequence"` + Type []syntheaCode `json:"type"` + } `json:"diagnosis"` + DosageInstruction []struct { + AsNeededBoolean bool `json:"asNeededBoolean"` + DoseAndRate []struct { + DoseQuantity *struct { + Value float64 `json:"value"` + } `json:"doseQuantity"` + Type syntheaCode `json:"type"` + } `json:"doseAndRate"` + Sequence int64 `json:"sequence"` + Timing *struct { + Repeat *struct { + Frequency int64 `json:"frequency"` + Period float64 `json:"period"` + PeriodUnit string `json:"periodUnit"` + } `json:"repeat"` + } `json:"timing"` + } `json:"dosageInstruction"` + EffectiveDateTime time.Time `json:"effectiveDateTime"` + Encounter syntheaReference `json:"encounter"` + Extension []syntheaExtension `json:"extension"` + Gender string `json:"gender"` + Goal []syntheaReference `json:"goal"` + ID string `json:"id"` + Identifier []struct { + System string `json:"system"` + Type syntheaCode `json:"type"` + Use string `json:"use"` + Value string `json:"value"` + } `json:"identifier"` + Insurance []struct { + Coverage syntheaReference `json:"coverage"` + Focal bool `json:"focal"` + Sequence int64 `json:"sequence"` + } `json:"insurance"` + Insurer syntheaReference `json:"insurer"` + Intent string `json:"intent"` + Issued time.Time `json:"issued"` + Item []struct { + Adjudication []struct { + Amount syntheaCurrency `json:"amount"` + Category syntheaCode `json:"category"` + } `json:"adjudication"` + Category syntheaCode `json:"category"` + DiagnosisSequence []int64 `json:"diagnosisSequence"` + Encounter []syntheaReference `json:"encounter"` + InformationSequence []int64 `json:"informationSequence"` + LocationCodeableConcept syntheaCode `json:"locationCodeableConcept"` + Net syntheaCurrency `json:"net"` + ProcedureSequence []int64 `json:"procedureSequence"` + ProductOrService syntheaCode `json:"productOrService"` + Sequence int64 `json:"sequence"` + ServicedPeriod syntheaRange `json:"servicedPeriod"` + } `json:"item"` + LifecycleStatus string `json:"lifecycleStatus"` + ManagingOrganization []syntheaReference `json:"managingOrganization"` + MaritalStatus syntheaCode `json:"maritalStatus"` + MedicationCodeableConcept syntheaCode `json:"medicationCodeableConcept"` + MultipleBirthBoolean bool `json:"multipleBirthBoolean"` + Name rawValue `json:"name"` + NumberOfInstances int64 `json:"numberOfInstances"` + NumberOfSeries int64 `json:"numberOfSeries"` + OccurrenceDateTime time.Time `json:"occurrenceDateTime"` + OnsetDateTime time.Time `json:"onsetDateTime"` + Outcome string `json:"outcome"` + Participant []struct { + Individual syntheaReference `json:"individual"` + Member syntheaReference `json:"member"` + Role []syntheaCode `json:"role"` + } `json:"participant"` + Patient syntheaReference `json:"patient"` + Payment *struct { + Amount syntheaCurrency `json:"amount"` + } `json:"payment"` + PerformedPeriod syntheaRange `json:"performedPeriod"` + Period syntheaRange `json:"period"` + Prescription syntheaReference `json:"prescription"` + PrimarySource bool `json:"primarySource"` + Priority syntheaCode `json:"priority"` + Procedure []struct { + ProcedureReference syntheaReference `json:"procedureReference"` + Sequence int64 `json:"sequence"` + } `json:"procedure"` + Provider syntheaReference `json:"provider"` + ReasonCode []syntheaCode `json:"reasonCode"` + ReasonReference []syntheaReference `json:"reasonReference"` + RecordedDate time.Time `json:"recordedDate"` + Referral syntheaReference `json:"referral"` + Requester syntheaReference `json:"requester"` + ResourceType string `json:"resourceType"` + Result []syntheaReference `json:"result"` + Series []struct { + BodySite syntheaCoding `json:"bodySite"` + Instance []struct { + Number int64 `json:"number"` + SopClass syntheaCoding `json:"sopClass"` + Title string `json:"title"` + UID string `json:"uid"` + } `json:"instance"` + Modality syntheaCoding `json:"modality"` + Number int64 `json:"number"` + NumberOfInstances int64 `json:"numberOfInstances"` + Started string `json:"started"` + UID string `json:"uid"` + } `json:"series"` + ServiceProvider syntheaReference `json:"serviceProvider"` + Started time.Time `json:"started"` + Status string `json:"status"` + Subject syntheaReference `json:"subject"` + SupportingInfo []struct { + Category syntheaCode `json:"category"` + Sequence int64 `json:"sequence"` + ValueReference syntheaReference `json:"valueReference"` + } `json:"supportingInfo"` + Telecom []map[string]string `json:"telecom"` + Text map[string]string `json:"text"` + Total rawValue `json:"total"` + Type rawValue `json:"type"` + Use string `json:"use"` + VaccineCode syntheaCode `json:"vaccineCode"` + ValueCodeableConcept syntheaCode `json:"valueCodeableConcept"` + ValueQuantity syntheaCoding `json:"valueQuantity"` + VerificationStatus syntheaCode `json:"verificationStatus"` + } `json:"resource"` + } `json:"entry"` + ResourceType string `json:"resourceType"` + Type string `json:"type"` + } + syntheaCode struct { + Coding []syntheaCoding `json:"coding"` + Text string `json:"text"` + } + syntheaCoding struct { + Code string `json:"code"` + Display string `json:"display"` + System string `json:"system"` + Unit string `json:"unit"` + Value float64 `json:"value"` + } + syntheaReference struct { + Display string `json:"display"` + Reference string `json:"reference"` + } + syntheaAddress struct { + City string `json:"city"` + Country string `json:"country"` + Extension []syntheaExtension `json:"extension"` + Line []string `json:"line"` + PostalCode string `json:"postalCode"` + State string `json:"state"` + } + syntheaExtension struct { + URL string `json:"url"` + ValueAddress syntheaAddress `json:"valueAddress"` + ValueCode string `json:"valueCode"` + ValueDecimal float64 `json:"valueDecimal"` + ValueString string `json:"valueString"` + Extension []syntheaExtension `json:"extension"` + } + syntheaRange struct { + End time.Time `json:"end"` + Start time.Time `json:"start"` + } + syntheaCurrency struct { + Currency string `json:"currency"` + Value float64 `json:"value"` + } +) + +type ( + twitterRoot struct { + Statuses []twitterStatus `json:"statuses"` + SearchMetadata struct { + CompletedIn float64 `json:"completed_in"` + MaxID int64 `json:"max_id"` + MaxIDStr int64 `json:"max_id_str,string"` + NextResults string `json:"next_results"` + Query string `json:"query"` + RefreshURL string `json:"refresh_url"` + Count int `json:"count"` + SinceID int `json:"since_id"` + SinceIDStr int `json:"since_id_str,string"` + } `json:"search_metadata"` + } + twitterStatus struct { + Metadata struct { + ResultType string `json:"result_type"` + IsoLanguageCode string `json:"iso_language_code"` + } `json:"metadata"` + CreatedAt string `json:"created_at"` + ID int64 `json:"id"` + IDStr int64 `json:"id_str,string"` + Text string `json:"text"` + Source string `json:"source"` + Truncated bool `json:"truncated"` + InReplyToStatusID int64 `json:"in_reply_to_status_id"` + InReplyToStatusIDStr int64 `json:"in_reply_to_status_id_str,string"` + InReplyToUserID int64 `json:"in_reply_to_user_id"` + InReplyToUserIDStr int64 `json:"in_reply_to_user_id_str,string"` + InReplyToScreenName string `json:"in_reply_to_screen_name"` + User twitterUser `json:"user,omitempty"` + Geo any `json:"geo"` + Coordinates any `json:"coordinates"` + Place any `json:"place"` + Contributors any `json:"contributors"` + RetweeetedStatus *twitterStatus `json:"retweeted_status"` + RetweetCount int `json:"retweet_count"` + FavoriteCount int `json:"favorite_count"` + Entities twitterEntities `json:"entities,omitempty"` + Favorited bool `json:"favorited"` + Retweeted bool `json:"retweeted"` + PossiblySensitive bool `json:"possibly_sensitive"` + Lang string `json:"lang"` + } + twitterUser struct { + ID int64 `json:"id"` + IDStr string `json:"id_str"` + Name string `json:"name"` + ScreenName string `json:"screen_name"` + Location string `json:"location"` + Description string `json:"description"` + URL any `json:"url"` + Entities twitterEntities `json:"entities"` + Protected bool `json:"protected"` + FollowersCount int `json:"followers_count"` + FriendsCount int `json:"friends_count"` + ListedCount int `json:"listed_count"` + CreatedAt string `json:"created_at"` + FavouritesCount int `json:"favourites_count"` + UtcOffset int `json:"utc_offset"` + TimeZone string `json:"time_zone"` + GeoEnabled bool `json:"geo_enabled"` + Verified bool `json:"verified"` + StatusesCount int `json:"statuses_count"` + Lang string `json:"lang"` + ContributorsEnabled bool `json:"contributors_enabled"` + IsTranslator bool `json:"is_translator"` + IsTranslationEnabled bool `json:"is_translation_enabled"` + ProfileBackgroundColor string `json:"profile_background_color"` + ProfileBackgroundImageURL string `json:"profile_background_image_url"` + ProfileBackgroundImageURLHTTPS string `json:"profile_background_image_url_https"` + ProfileBackgroundTile bool `json:"profile_background_tile"` + ProfileImageURL string `json:"profile_image_url"` + ProfileImageURLHTTPS string `json:"profile_image_url_https"` + ProfileBannerURL string `json:"profile_banner_url"` + ProfileLinkColor string `json:"profile_link_color"` + ProfileSidebarBorderColor string `json:"profile_sidebar_border_color"` + ProfileSidebarFillColor string `json:"profile_sidebar_fill_color"` + ProfileTextColor string `json:"profile_text_color"` + ProfileUseBackgroundImage bool `json:"profile_use_background_image"` + DefaultProfile bool `json:"default_profile"` + DefaultProfileImage bool `json:"default_profile_image"` + Following bool `json:"following"` + FollowRequestSent bool `json:"follow_request_sent"` + Notifications bool `json:"notifications"` + } + twitterEntities struct { + Hashtags []any `json:"hashtags"` + Symbols []any `json:"symbols"` + URL *twitterURL `json:"url"` + URLs []twitterURL `json:"urls"` + UserMentions []struct { + ScreenName string `json:"screen_name"` + Name string `json:"name"` + ID int64 `json:"id"` + IDStr int64 `json:"id_str,string"` + Indices []int `json:"indices"` + } `json:"user_mentions"` + Description struct { + URLs []twitterURL `json:"urls"` + } `json:"description"` + Media []struct { + ID int64 `json:"id"` + IDStr string `json:"id_str"` + Indices []int `json:"indices"` + MediaURL string `json:"media_url"` + MediaURLHTTPS string `json:"media_url_https"` + URL string `json:"url"` + DisplayURL string `json:"display_url"` + ExpandedURL string `json:"expanded_url"` + Type string `json:"type"` + Sizes map[string]struct { + W int `json:"w"` + H int `json:"h"` + Resize string `json:"resize"` + } `json:"sizes"` + SourceStatusID int64 `json:"source_status_id"` + SourceStatusIDStr int64 `json:"source_status_id_str,string"` + } `json:"media"` + } + twitterURL struct { + URL string `json:"url"` + URLs []twitterURL `json:"urls"` + ExpandedURL string `json:"expanded_url"` + DisplayURL string `json:"display_url"` + Indices []int `json:"indices"` + } +) + +// rawValue is the raw encoded JSON value. +type rawValue []byte + +func (v rawValue) MarshalJSON() ([]byte, error) { + if v == nil { + return []byte("null"), nil + } + return v, nil +} + +func (v *rawValue) UnmarshalJSON(b []byte) error { + if v == nil { + return errors.New("jsontest.rawValue: UnmarshalJSON on nil pointer") + } + *v = append((*v)[:0], b...) + return nil +} diff --git a/src/encoding/json/internal/jsontest/testdata/canada_geometry.json.zst b/src/encoding/json/internal/jsontest/testdata/canada_geometry.json.zst new file mode 100644 index 00000000000000..e0f56291c4d79a Binary files /dev/null and b/src/encoding/json/internal/jsontest/testdata/canada_geometry.json.zst differ diff --git a/src/encoding/json/internal/jsontest/testdata/citm_catalog.json.zst b/src/encoding/json/internal/jsontest/testdata/citm_catalog.json.zst new file mode 100644 index 00000000000000..3a74ae7af7cac2 Binary files /dev/null and b/src/encoding/json/internal/jsontest/testdata/citm_catalog.json.zst differ diff --git a/src/encoding/json/internal/jsontest/testdata/golang_source.json.zst b/src/encoding/json/internal/jsontest/testdata/golang_source.json.zst new file mode 100644 index 00000000000000..7ff5f04767bfff Binary files /dev/null and b/src/encoding/json/internal/jsontest/testdata/golang_source.json.zst differ diff --git a/src/encoding/json/internal/jsontest/testdata/string_escaped.json.zst b/src/encoding/json/internal/jsontest/testdata/string_escaped.json.zst new file mode 100644 index 00000000000000..932b1e2513712b Binary files /dev/null and b/src/encoding/json/internal/jsontest/testdata/string_escaped.json.zst differ diff --git a/src/encoding/json/internal/jsontest/testdata/string_unicode.json.zst b/src/encoding/json/internal/jsontest/testdata/string_unicode.json.zst new file mode 100644 index 00000000000000..dc13c4f47e4e9f Binary files /dev/null and b/src/encoding/json/internal/jsontest/testdata/string_unicode.json.zst differ diff --git a/src/encoding/json/internal/jsontest/testdata/synthea_fhir.json.zst b/src/encoding/json/internal/jsontest/testdata/synthea_fhir.json.zst new file mode 100644 index 00000000000000..3f869a2713a171 Binary files /dev/null and b/src/encoding/json/internal/jsontest/testdata/synthea_fhir.json.zst differ diff --git a/src/encoding/json/internal/jsontest/testdata/twitter_status.json.zst b/src/encoding/json/internal/jsontest/testdata/twitter_status.json.zst new file mode 100644 index 00000000000000..f5a456be899031 Binary files /dev/null and b/src/encoding/json/internal/jsontest/testdata/twitter_status.json.zst differ diff --git a/src/encoding/json/internal/jsonwire/decode.go b/src/encoding/json/internal/jsonwire/decode.go new file mode 100644 index 00000000000000..42eeedcddf94cf --- /dev/null +++ b/src/encoding/json/internal/jsonwire/decode.go @@ -0,0 +1,629 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsonwire + +import ( + "io" + "math" + "slices" + "strconv" + "unicode/utf16" + "unicode/utf8" +) + +type ValueFlags uint + +const ( + _ ValueFlags = (1 << iota) / 2 // powers of two starting with zero + + stringNonVerbatim // string cannot be naively treated as valid UTF-8 + stringNonCanonical // string not formatted according to RFC 8785, section 3.2.2.2. + // TODO: Track whether a number is a non-integer? +) + +func (f *ValueFlags) Join(f2 ValueFlags) { *f |= f2 } +func (f ValueFlags) IsVerbatim() bool { return f&stringNonVerbatim == 0 } +func (f ValueFlags) IsCanonical() bool { return f&stringNonCanonical == 0 } + +// ConsumeWhitespace consumes leading JSON whitespace per RFC 7159, section 2. +func ConsumeWhitespace(b []byte) (n int) { + // NOTE: The arguments and logic are kept simple to keep this inlinable. + for len(b) > n && (b[n] == ' ' || b[n] == '\t' || b[n] == '\r' || b[n] == '\n') { + n++ + } + return n +} + +// ConsumeNull consumes the next JSON null literal per RFC 7159, section 3. +// It returns 0 if it is invalid, in which case consumeLiteral should be used. +func ConsumeNull(b []byte) int { + // NOTE: The arguments and logic are kept simple to keep this inlinable. + const literal = "null" + if len(b) >= len(literal) && string(b[:len(literal)]) == literal { + return len(literal) + } + return 0 +} + +// ConsumeFalse consumes the next JSON false literal per RFC 7159, section 3. +// It returns 0 if it is invalid, in which case consumeLiteral should be used. +func ConsumeFalse(b []byte) int { + // NOTE: The arguments and logic are kept simple to keep this inlinable. + const literal = "false" + if len(b) >= len(literal) && string(b[:len(literal)]) == literal { + return len(literal) + } + return 0 +} + +// ConsumeTrue consumes the next JSON true literal per RFC 7159, section 3. +// It returns 0 if it is invalid, in which case consumeLiteral should be used. +func ConsumeTrue(b []byte) int { + // NOTE: The arguments and logic are kept simple to keep this inlinable. + const literal = "true" + if len(b) >= len(literal) && string(b[:len(literal)]) == literal { + return len(literal) + } + return 0 +} + +// ConsumeLiteral consumes the next JSON literal per RFC 7159, section 3. +// If the input appears truncated, it returns io.ErrUnexpectedEOF. +func ConsumeLiteral(b []byte, lit string) (n int, err error) { + for i := 0; i < len(b) && i < len(lit); i++ { + if b[i] != lit[i] { + return i, NewInvalidCharacterError(b[i:], "in literal "+lit+" (expecting "+strconv.QuoteRune(rune(lit[i]))+")") + } + } + if len(b) < len(lit) { + return len(b), io.ErrUnexpectedEOF + } + return len(lit), nil +} + +// ConsumeSimpleString consumes the next JSON string per RFC 7159, section 7 +// but is limited to the grammar for an ASCII string without escape sequences. +// It returns 0 if it is invalid or more complicated than a simple string, +// in which case consumeString should be called. +// +// It rejects '<', '>', and '&' for compatibility reasons since these were +// always escaped in the v1 implementation. Thus, if this function reports +// non-zero then we know that the string would be encoded the same way +// under both v1 or v2 escape semantics. +func ConsumeSimpleString(b []byte) (n int) { + // NOTE: The arguments and logic are kept simple to keep this inlinable. + if len(b) > 0 && b[0] == '"' { + n++ + for len(b) > n && b[n] < utf8.RuneSelf && escapeASCII[b[n]] == 0 { + n++ + } + if uint(len(b)) > uint(n) && b[n] == '"' { + n++ + return n + } + } + return 0 +} + +// ConsumeString consumes the next JSON string per RFC 7159, section 7. +// If validateUTF8 is false, then this allows the presence of invalid UTF-8 +// characters within the string itself. +// It reports the number of bytes consumed and whether an error was encountered. +// If the input appears truncated, it returns io.ErrUnexpectedEOF. +func ConsumeString(flags *ValueFlags, b []byte, validateUTF8 bool) (n int, err error) { + return ConsumeStringResumable(flags, b, 0, validateUTF8) +} + +// ConsumeStringResumable is identical to consumeString but supports resuming +// from a previous call that returned io.ErrUnexpectedEOF. +func ConsumeStringResumable(flags *ValueFlags, b []byte, resumeOffset int, validateUTF8 bool) (n int, err error) { + // Consume the leading double quote. + switch { + case resumeOffset > 0: + n = resumeOffset // already handled the leading quote + case uint(len(b)) == 0: + return n, io.ErrUnexpectedEOF + case b[0] == '"': + n++ + default: + return n, NewInvalidCharacterError(b[n:], `at start of string (expecting '"')`) + } + + // Consume every character in the string. + for uint(len(b)) > uint(n) { + // Optimize for long sequences of unescaped characters. + noEscape := func(c byte) bool { + return c < utf8.RuneSelf && ' ' <= c && c != '\\' && c != '"' + } + for uint(len(b)) > uint(n) && noEscape(b[n]) { + n++ + } + if uint(len(b)) <= uint(n) { + return n, io.ErrUnexpectedEOF + } + + // Check for terminating double quote. + if b[n] == '"' { + n++ + return n, nil + } + + switch r, rn := utf8.DecodeRune(b[n:]); { + // Handle UTF-8 encoded byte sequence. + // Due to specialized handling of ASCII above, we know that + // all normal sequences at this point must be 2 bytes or larger. + case rn > 1: + n += rn + // Handle escape sequence. + case r == '\\': + flags.Join(stringNonVerbatim) + resumeOffset = n + if uint(len(b)) < uint(n+2) { + return resumeOffset, io.ErrUnexpectedEOF + } + switch r := b[n+1]; r { + case '/': + // Forward slash is the only character with 3 representations. + // Per RFC 8785, section 3.2.2.2., this must not be escaped. + flags.Join(stringNonCanonical) + n += 2 + case '"', '\\', 'b', 'f', 'n', 'r', 't': + n += 2 + case 'u': + if uint(len(b)) < uint(n+6) { + if hasEscapedUTF16Prefix(b[n:], false) { + return resumeOffset, io.ErrUnexpectedEOF + } + flags.Join(stringNonCanonical) + return n, NewInvalidEscapeSequenceError(b[n:]) + } + v1, ok := parseHexUint16(b[n+2 : n+6]) + if !ok { + flags.Join(stringNonCanonical) + return n, NewInvalidEscapeSequenceError(b[n : n+6]) + } + // Only certain control characters can use the \uFFFF notation + // for canonical formatting (per RFC 8785, section 3.2.2.2.). + switch v1 { + // \uFFFF notation not permitted for these characters. + case '\b', '\f', '\n', '\r', '\t': + flags.Join(stringNonCanonical) + default: + // \uFFFF notation only permitted for control characters. + if v1 >= ' ' { + flags.Join(stringNonCanonical) + } else { + // \uFFFF notation must be lower case. + for _, c := range b[n+2 : n+6] { + if 'A' <= c && c <= 'F' { + flags.Join(stringNonCanonical) + } + } + } + } + n += 6 + + r := rune(v1) + if validateUTF8 && utf16.IsSurrogate(r) { + if uint(len(b)) < uint(n+6) { + if hasEscapedUTF16Prefix(b[n:], true) { + return resumeOffset, io.ErrUnexpectedEOF + } + flags.Join(stringNonCanonical) + return n - 6, NewInvalidEscapeSequenceError(b[n-6:]) + } else if v2, ok := parseHexUint16(b[n+2 : n+6]); b[n] != '\\' || b[n+1] != 'u' || !ok { + flags.Join(stringNonCanonical) + return n - 6, NewInvalidEscapeSequenceError(b[n-6 : n+6]) + } else if r = utf16.DecodeRune(rune(v1), rune(v2)); r == utf8.RuneError { + flags.Join(stringNonCanonical) + return n - 6, NewInvalidEscapeSequenceError(b[n-6 : n+6]) + } else { + n += 6 + } + } + default: + flags.Join(stringNonCanonical) + return n, NewInvalidEscapeSequenceError(b[n : n+2]) + } + // Handle invalid UTF-8. + case r == utf8.RuneError: + if !utf8.FullRune(b[n:]) { + return n, io.ErrUnexpectedEOF + } + flags.Join(stringNonVerbatim | stringNonCanonical) + if validateUTF8 { + return n, ErrInvalidUTF8 + } + n++ + // Handle invalid control characters. + case r < ' ': + flags.Join(stringNonVerbatim | stringNonCanonical) + return n, NewInvalidCharacterError(b[n:], "in string (expecting non-control character)") + default: + panic("BUG: unhandled character " + QuoteRune(b[n:])) + } + } + return n, io.ErrUnexpectedEOF +} + +// AppendUnquote appends the unescaped form of a JSON string in src to dst. +// Any invalid UTF-8 within the string will be replaced with utf8.RuneError, +// but the error will be specified as having encountered such an error. +// The input must be an entire JSON string with no surrounding whitespace. +func AppendUnquote[Bytes ~[]byte | ~string](dst []byte, src Bytes) (v []byte, err error) { + dst = slices.Grow(dst, len(src)) + + // Consume the leading double quote. + var i, n int + switch { + case uint(len(src)) == 0: + return dst, io.ErrUnexpectedEOF + case src[0] == '"': + i, n = 1, 1 + default: + return dst, NewInvalidCharacterError(src, `at start of string (expecting '"')`) + } + + // Consume every character in the string. + for uint(len(src)) > uint(n) { + // Optimize for long sequences of unescaped characters. + noEscape := func(c byte) bool { + return c < utf8.RuneSelf && ' ' <= c && c != '\\' && c != '"' + } + for uint(len(src)) > uint(n) && noEscape(src[n]) { + n++ + } + if uint(len(src)) <= uint(n) { + dst = append(dst, src[i:n]...) + return dst, io.ErrUnexpectedEOF + } + + // Check for terminating double quote. + if src[n] == '"' { + dst = append(dst, src[i:n]...) + n++ + if n < len(src) { + err = NewInvalidCharacterError(src[n:], "after string value") + } + return dst, err + } + + switch r, rn := utf8.DecodeRuneInString(string(truncateMaxUTF8(src[n:]))); { + // Handle UTF-8 encoded byte sequence. + // Due to specialized handling of ASCII above, we know that + // all normal sequences at this point must be 2 bytes or larger. + case rn > 1: + n += rn + // Handle escape sequence. + case r == '\\': + dst = append(dst, src[i:n]...) + + // Handle escape sequence. + if uint(len(src)) < uint(n+2) { + return dst, io.ErrUnexpectedEOF + } + switch r := src[n+1]; r { + case '"', '\\', '/': + dst = append(dst, r) + n += 2 + case 'b': + dst = append(dst, '\b') + n += 2 + case 'f': + dst = append(dst, '\f') + n += 2 + case 'n': + dst = append(dst, '\n') + n += 2 + case 'r': + dst = append(dst, '\r') + n += 2 + case 't': + dst = append(dst, '\t') + n += 2 + case 'u': + if uint(len(src)) < uint(n+6) { + if hasEscapedUTF16Prefix(src[n:], false) { + return dst, io.ErrUnexpectedEOF + } + return dst, NewInvalidEscapeSequenceError(src[n:]) + } + v1, ok := parseHexUint16(src[n+2 : n+6]) + if !ok { + return dst, NewInvalidEscapeSequenceError(src[n : n+6]) + } + n += 6 + + // Check whether this is a surrogate half. + r := rune(v1) + if utf16.IsSurrogate(r) { + r = utf8.RuneError // assume failure unless the following succeeds + if uint(len(src)) < uint(n+6) { + if hasEscapedUTF16Prefix(src[n:], true) { + return utf8.AppendRune(dst, r), io.ErrUnexpectedEOF + } + err = NewInvalidEscapeSequenceError(src[n-6:]) + } else if v2, ok := parseHexUint16(src[n+2 : n+6]); src[n] != '\\' || src[n+1] != 'u' || !ok { + err = NewInvalidEscapeSequenceError(src[n-6 : n+6]) + } else if r = utf16.DecodeRune(rune(v1), rune(v2)); r == utf8.RuneError { + err = NewInvalidEscapeSequenceError(src[n-6 : n+6]) + } else { + n += 6 + } + } + + dst = utf8.AppendRune(dst, r) + default: + return dst, NewInvalidEscapeSequenceError(src[n : n+2]) + } + i = n + // Handle invalid UTF-8. + case r == utf8.RuneError: + dst = append(dst, src[i:n]...) + if !utf8.FullRuneInString(string(truncateMaxUTF8(src[n:]))) { + return dst, io.ErrUnexpectedEOF + } + // NOTE: An unescaped string may be longer than the escaped string + // because invalid UTF-8 bytes are being replaced. + dst = append(dst, "\uFFFD"...) + n += rn + i = n + err = ErrInvalidUTF8 + // Handle invalid control characters. + case r < ' ': + dst = append(dst, src[i:n]...) + return dst, NewInvalidCharacterError(src[n:], "in string (expecting non-control character)") + default: + panic("BUG: unhandled character " + QuoteRune(src[n:])) + } + } + dst = append(dst, src[i:n]...) + return dst, io.ErrUnexpectedEOF +} + +// hasEscapedUTF16Prefix reports whether b is possibly +// the truncated prefix of a \uFFFF escape sequence. +func hasEscapedUTF16Prefix[Bytes ~[]byte | ~string](b Bytes, lowerSurrogateHalf bool) bool { + for i := range len(b) { + switch c := b[i]; { + case i == 0 && c != '\\': + return false + case i == 1 && c != 'u': + return false + case i == 2 && lowerSurrogateHalf && c != 'd' && c != 'D': + return false // not within ['\uDC00':'\uDFFF'] + case i == 3 && lowerSurrogateHalf && !('c' <= c && c <= 'f') && !('C' <= c && c <= 'F'): + return false // not within ['\uDC00':'\uDFFF'] + case i >= 2 && i < 6 && !('0' <= c && c <= '9') && !('a' <= c && c <= 'f') && !('A' <= c && c <= 'F'): + return false + } + } + return true +} + +// UnquoteMayCopy returns the unescaped form of b. +// If there are no escaped characters, the output is simply a subslice of +// the input with the surrounding quotes removed. +// Otherwise, a new buffer is allocated for the output. +// It assumes the input is valid. +func UnquoteMayCopy(b []byte, isVerbatim bool) []byte { + // NOTE: The arguments and logic are kept simple to keep this inlinable. + if isVerbatim { + return b[len(`"`) : len(b)-len(`"`)] + } + b, _ = AppendUnquote(nil, b) + return b +} + +// ConsumeSimpleNumber consumes the next JSON number per RFC 7159, section 6 +// but is limited to the grammar for a positive integer. +// It returns 0 if it is invalid or more complicated than a simple integer, +// in which case consumeNumber should be called. +func ConsumeSimpleNumber(b []byte) (n int) { + // NOTE: The arguments and logic are kept simple to keep this inlinable. + if len(b) > 0 { + if b[0] == '0' { + n++ + } else if '1' <= b[0] && b[0] <= '9' { + n++ + for len(b) > n && ('0' <= b[n] && b[n] <= '9') { + n++ + } + } else { + return 0 + } + if uint(len(b)) <= uint(n) || (b[n] != '.' && b[n] != 'e' && b[n] != 'E') { + return n + } + } + return 0 +} + +type ConsumeNumberState uint + +const ( + consumeNumberInit ConsumeNumberState = iota + beforeIntegerDigits + withinIntegerDigits + beforeFractionalDigits + withinFractionalDigits + beforeExponentDigits + withinExponentDigits +) + +// ConsumeNumber consumes the next JSON number per RFC 7159, section 6. +// It reports the number of bytes consumed and whether an error was encountered. +// If the input appears truncated, it returns io.ErrUnexpectedEOF. +// +// Note that JSON numbers are not self-terminating. +// If the entire input is consumed, then the caller needs to consider whether +// there may be subsequent unread data that may still be part of this number. +func ConsumeNumber(b []byte) (n int, err error) { + n, _, err = ConsumeNumberResumable(b, 0, consumeNumberInit) + return n, err +} + +// ConsumeNumberResumable is identical to consumeNumber but supports resuming +// from a previous call that returned io.ErrUnexpectedEOF. +func ConsumeNumberResumable(b []byte, resumeOffset int, state ConsumeNumberState) (n int, _ ConsumeNumberState, err error) { + // Jump to the right state when resuming from a partial consumption. + n = resumeOffset + if state > consumeNumberInit { + switch state { + case withinIntegerDigits, withinFractionalDigits, withinExponentDigits: + // Consume leading digits. + for uint(len(b)) > uint(n) && ('0' <= b[n] && b[n] <= '9') { + n++ + } + if uint(len(b)) <= uint(n) { + return n, state, nil // still within the same state + } + state++ // switches "withinX" to "beforeY" where Y is the state after X + } + switch state { + case beforeIntegerDigits: + goto beforeInteger + case beforeFractionalDigits: + goto beforeFractional + case beforeExponentDigits: + goto beforeExponent + default: + return n, state, nil + } + } + + // Consume required integer component (with optional minus sign). +beforeInteger: + resumeOffset = n + if uint(len(b)) > 0 && b[0] == '-' { + n++ + } + switch { + case uint(len(b)) <= uint(n): + return resumeOffset, beforeIntegerDigits, io.ErrUnexpectedEOF + case b[n] == '0': + n++ + state = beforeFractionalDigits + case '1' <= b[n] && b[n] <= '9': + n++ + for uint(len(b)) > uint(n) && ('0' <= b[n] && b[n] <= '9') { + n++ + } + state = withinIntegerDigits + default: + return n, state, NewInvalidCharacterError(b[n:], "in number (expecting digit)") + } + + // Consume optional fractional component. +beforeFractional: + if uint(len(b)) > uint(n) && b[n] == '.' { + resumeOffset = n + n++ + switch { + case uint(len(b)) <= uint(n): + return resumeOffset, beforeFractionalDigits, io.ErrUnexpectedEOF + case '0' <= b[n] && b[n] <= '9': + n++ + default: + return n, state, NewInvalidCharacterError(b[n:], "in number (expecting digit)") + } + for uint(len(b)) > uint(n) && ('0' <= b[n] && b[n] <= '9') { + n++ + } + state = withinFractionalDigits + } + + // Consume optional exponent component. +beforeExponent: + if uint(len(b)) > uint(n) && (b[n] == 'e' || b[n] == 'E') { + resumeOffset = n + n++ + if uint(len(b)) > uint(n) && (b[n] == '-' || b[n] == '+') { + n++ + } + switch { + case uint(len(b)) <= uint(n): + return resumeOffset, beforeExponentDigits, io.ErrUnexpectedEOF + case '0' <= b[n] && b[n] <= '9': + n++ + default: + return n, state, NewInvalidCharacterError(b[n:], "in number (expecting digit)") + } + for uint(len(b)) > uint(n) && ('0' <= b[n] && b[n] <= '9') { + n++ + } + state = withinExponentDigits + } + + return n, state, nil +} + +// parseHexUint16 is similar to strconv.ParseUint, +// but operates directly on []byte and is optimized for base-16. +// See https://go.dev/issue/42429. +func parseHexUint16[Bytes ~[]byte | ~string](b Bytes) (v uint16, ok bool) { + if len(b) != 4 { + return 0, false + } + for i := range 4 { + c := b[i] + switch { + case '0' <= c && c <= '9': + c = c - '0' + case 'a' <= c && c <= 'f': + c = 10 + c - 'a' + case 'A' <= c && c <= 'F': + c = 10 + c - 'A' + default: + return 0, false + } + v = v*16 + uint16(c) + } + return v, true +} + +// ParseUint parses b as a decimal unsigned integer according to +// a strict subset of the JSON number grammar, returning the value if valid. +// It returns (0, false) if there is a syntax error and +// returns (math.MaxUint64, false) if there is an overflow. +func ParseUint(b []byte) (v uint64, ok bool) { + const unsafeWidth = 20 // len(fmt.Sprint(uint64(math.MaxUint64))) + var n int + for ; len(b) > n && ('0' <= b[n] && b[n] <= '9'); n++ { + v = 10*v + uint64(b[n]-'0') + } + switch { + case n == 0 || len(b) != n || (b[0] == '0' && string(b) != "0"): + return 0, false + case n >= unsafeWidth && (b[0] != '1' || v < 1e19 || n > unsafeWidth): + return math.MaxUint64, false + } + return v, true +} + +// ParseFloat parses a floating point number according to the Go float grammar. +// Note that the JSON number grammar is a strict subset. +// +// If the number overflows the finite representation of a float, +// then we return MaxFloat since any finite value will always be infinitely +// more accurate at representing another finite value than an infinite value. +func ParseFloat(b []byte, bits int) (v float64, ok bool) { + fv, err := strconv.ParseFloat(string(b), bits) + if math.IsInf(fv, 0) { + switch { + case bits == 32 && math.IsInf(fv, +1): + fv = +math.MaxFloat32 + case bits == 64 && math.IsInf(fv, +1): + fv = +math.MaxFloat64 + case bits == 32 && math.IsInf(fv, -1): + fv = -math.MaxFloat32 + case bits == 64 && math.IsInf(fv, -1): + fv = -math.MaxFloat64 + } + } + return fv, err == nil +} diff --git a/src/encoding/json/internal/jsonwire/decode_test.go b/src/encoding/json/internal/jsonwire/decode_test.go new file mode 100644 index 00000000000000..549c1a1f62b283 --- /dev/null +++ b/src/encoding/json/internal/jsonwire/decode_test.go @@ -0,0 +1,443 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsonwire + +import ( + "errors" + "io" + "math" + "reflect" + "strings" + "testing" +) + +func TestConsumeWhitespace(t *testing.T) { + tests := []struct { + in string + want int + }{ + {"", 0}, + {"a", 0}, + {" a", 1}, + {" a ", 1}, + {" \n\r\ta", 4}, + {" \n\r\t \n\r\t \n\r\t \n\r\t", 16}, + {"\u00a0", 0}, // non-breaking space is not JSON whitespace + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + if got := ConsumeWhitespace([]byte(tt.in)); got != tt.want { + t.Errorf("ConsumeWhitespace(%q) = %v, want %v", tt.in, got, tt.want) + } + }) + } +} + +func TestConsumeLiteral(t *testing.T) { + tests := []struct { + literal string + in string + want int + wantErr error + }{ + {"null", "", 0, io.ErrUnexpectedEOF}, + {"null", "n", 1, io.ErrUnexpectedEOF}, + {"null", "nu", 2, io.ErrUnexpectedEOF}, + {"null", "nul", 3, io.ErrUnexpectedEOF}, + {"null", "null", 4, nil}, + {"null", "nullx", 4, nil}, + {"null", "x", 0, NewInvalidCharacterError("x", "in literal null (expecting 'n')")}, + {"null", "nuxx", 2, NewInvalidCharacterError("x", "in literal null (expecting 'l')")}, + + {"false", "", 0, io.ErrUnexpectedEOF}, + {"false", "f", 1, io.ErrUnexpectedEOF}, + {"false", "fa", 2, io.ErrUnexpectedEOF}, + {"false", "fal", 3, io.ErrUnexpectedEOF}, + {"false", "fals", 4, io.ErrUnexpectedEOF}, + {"false", "false", 5, nil}, + {"false", "falsex", 5, nil}, + {"false", "x", 0, NewInvalidCharacterError("x", "in literal false (expecting 'f')")}, + {"false", "falsx", 4, NewInvalidCharacterError("x", "in literal false (expecting 'e')")}, + + {"true", "", 0, io.ErrUnexpectedEOF}, + {"true", "t", 1, io.ErrUnexpectedEOF}, + {"true", "tr", 2, io.ErrUnexpectedEOF}, + {"true", "tru", 3, io.ErrUnexpectedEOF}, + {"true", "true", 4, nil}, + {"true", "truex", 4, nil}, + {"true", "x", 0, NewInvalidCharacterError("x", "in literal true (expecting 't')")}, + {"true", "trux", 3, NewInvalidCharacterError("x", "in literal true (expecting 'e')")}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + var got int + switch tt.literal { + case "null": + got = ConsumeNull([]byte(tt.in)) + case "false": + got = ConsumeFalse([]byte(tt.in)) + case "true": + got = ConsumeTrue([]byte(tt.in)) + default: + t.Errorf("invalid literal: %v", tt.literal) + } + switch { + case tt.wantErr == nil && got != tt.want: + t.Errorf("Consume%v(%q) = %v, want %v", strings.Title(tt.literal), tt.in, got, tt.want) + case tt.wantErr != nil && got != 0: + t.Errorf("Consume%v(%q) = %v, want %v", strings.Title(tt.literal), tt.in, got, 0) + } + + got, gotErr := ConsumeLiteral([]byte(tt.in), tt.literal) + if got != tt.want || !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("ConsumeLiteral(%q, %q) = (%v, %v), want (%v, %v)", tt.in, tt.literal, got, gotErr, tt.want, tt.wantErr) + } + }) + } +} + +func TestConsumeString(t *testing.T) { + var errPrev = errors.New("same as previous error") + tests := []struct { + in string + simple bool + want int + wantUTF8 int // consumed bytes if validateUTF8 is specified + wantFlags ValueFlags + wantUnquote string + wantErr error + wantErrUTF8 error // error if validateUTF8 is specified + wantErrUnquote error + }{ + {``, false, 0, 0, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"`, false, 1, 1, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`""`, true, 2, 2, 0, "", nil, nil, nil}, + {`""x`, true, 2, 2, 0, "", nil, nil, NewInvalidCharacterError("x", "after string value")}, + {` ""x`, false, 0, 0, 0, "", NewInvalidCharacterError(" ", "at start of string (expecting '\"')"), errPrev, errPrev}, + {`"hello`, false, 6, 6, 0, "hello", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"hello"`, true, 7, 7, 0, "hello", nil, nil, nil}, + {"\"\x00\"", false, 1, 1, stringNonVerbatim | stringNonCanonical, "", NewInvalidCharacterError("\x00", "in string (expecting non-control character)"), errPrev, errPrev}, + {`"\u0000"`, false, 8, 8, stringNonVerbatim, "\x00", nil, nil, nil}, + {"\"\x1f\"", false, 1, 1, stringNonVerbatim | stringNonCanonical, "", NewInvalidCharacterError("\x1f", "in string (expecting non-control character)"), errPrev, errPrev}, + {`"\u001f"`, false, 8, 8, stringNonVerbatim, "\x1f", nil, nil, nil}, + {`"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"`, true, 54, 54, 0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", nil, nil, nil}, + {"\" !#$%'()*+,-./0123456789:;=?@[]^_`{|}~\x7f\"", true, 41, 41, 0, " !#$%'()*+,-./0123456789:;=?@[]^_`{|}~\x7f", nil, nil, nil}, + {`"&"`, false, 3, 3, 0, "&", nil, nil, nil}, + {`"<"`, false, 3, 3, 0, "<", nil, nil, nil}, + {`">"`, false, 3, 3, 0, ">", nil, nil, nil}, + {"\"x\x80\"", false, 4, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd", nil, ErrInvalidUTF8, errPrev}, + {"\"x\xff\"", false, 4, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd", nil, ErrInvalidUTF8, errPrev}, + {"\"x\xc0", false, 3, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd", io.ErrUnexpectedEOF, ErrInvalidUTF8, io.ErrUnexpectedEOF}, + {"\"x\xc0\x80\"", false, 5, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd\ufffd", nil, ErrInvalidUTF8, errPrev}, + {"\"x\xe0", false, 2, 2, 0, "x", io.ErrUnexpectedEOF, errPrev, errPrev}, + {"\"x\xe0\x80", false, 4, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd\ufffd", io.ErrUnexpectedEOF, ErrInvalidUTF8, io.ErrUnexpectedEOF}, + {"\"x\xe0\x80\x80\"", false, 6, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd\ufffd\ufffd", nil, ErrInvalidUTF8, errPrev}, + {"\"x\xf0", false, 2, 2, 0, "x", io.ErrUnexpectedEOF, errPrev, errPrev}, + {"\"x\xf0\x80", false, 4, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd\ufffd", io.ErrUnexpectedEOF, ErrInvalidUTF8, io.ErrUnexpectedEOF}, + {"\"x\xf0\x80\x80", false, 5, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd\ufffd\ufffd", io.ErrUnexpectedEOF, ErrInvalidUTF8, io.ErrUnexpectedEOF}, + {"\"x\xf0\x80\x80\x80\"", false, 7, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd\ufffd\ufffd\ufffd", nil, ErrInvalidUTF8, errPrev}, + {"\"x\xed\xba\xad\"", false, 6, 2, stringNonVerbatim | stringNonCanonical, "x\ufffd\ufffd\ufffd", nil, ErrInvalidUTF8, errPrev}, + {"\"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602\"", false, 25, 25, 0, "\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602", nil, nil, nil}, + {`"¢"`[:2], false, 1, 1, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"¢"`[:3], false, 3, 3, 0, "¢", io.ErrUnexpectedEOF, errPrev, errPrev}, // missing terminating quote + {`"¢"`[:4], false, 4, 4, 0, "¢", nil, nil, nil}, + {`"€"`[:2], false, 1, 1, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"€"`[:3], false, 1, 1, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"€"`[:4], false, 4, 4, 0, "€", io.ErrUnexpectedEOF, errPrev, errPrev}, // missing terminating quote + {`"€"`[:5], false, 5, 5, 0, "€", nil, nil, nil}, + {`"𐍈"`[:2], false, 1, 1, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"𐍈"`[:3], false, 1, 1, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"𐍈"`[:4], false, 1, 1, 0, "", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"𐍈"`[:5], false, 5, 5, 0, "𐍈", io.ErrUnexpectedEOF, errPrev, errPrev}, // missing terminating quote + {`"𐍈"`[:6], false, 6, 6, 0, "𐍈", nil, nil, nil}, + {`"x\`, false, 2, 2, stringNonVerbatim, "x", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"x\"`, false, 4, 4, stringNonVerbatim, "x\"", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"x\x"`, false, 2, 2, stringNonVerbatim | stringNonCanonical, "x", NewInvalidEscapeSequenceError(`\x`), errPrev, errPrev}, + {`"\"\\\b\f\n\r\t"`, false, 16, 16, stringNonVerbatim, "\"\\\b\f\n\r\t", nil, nil, nil}, + {`"/"`, true, 3, 3, 0, "/", nil, nil, nil}, + {`"\/"`, false, 4, 4, stringNonVerbatim | stringNonCanonical, "/", nil, nil, nil}, + {`"\u002f"`, false, 8, 8, stringNonVerbatim | stringNonCanonical, "/", nil, nil, nil}, + {`"\u`, false, 1, 1, stringNonVerbatim, "", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"\uf`, false, 1, 1, stringNonVerbatim, "", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"\uff`, false, 1, 1, stringNonVerbatim, "", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"\ufff`, false, 1, 1, stringNonVerbatim, "", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"\ufffd`, false, 7, 7, stringNonVerbatim | stringNonCanonical, "\ufffd", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"\ufffd"`, false, 8, 8, stringNonVerbatim | stringNonCanonical, "\ufffd", nil, nil, nil}, + {`"\uABCD"`, false, 8, 8, stringNonVerbatim | stringNonCanonical, "\uabcd", nil, nil, nil}, + {`"\uefX0"`, false, 1, 1, stringNonVerbatim | stringNonCanonical, "", NewInvalidEscapeSequenceError(`\uefX0`), errPrev, errPrev}, + {`"\uDEAD`, false, 7, 1, stringNonVerbatim | stringNonCanonical, "\ufffd", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"\uDEAD"`, false, 8, 1, stringNonVerbatim | stringNonCanonical, "\ufffd", nil, NewInvalidEscapeSequenceError(`\uDEAD"`), errPrev}, + {`"\uDEAD______"`, false, 14, 1, stringNonVerbatim | stringNonCanonical, "\ufffd______", nil, NewInvalidEscapeSequenceError(`\uDEAD______`), errPrev}, + {`"\uDEAD\uXXXX"`, false, 7, 1, stringNonVerbatim | stringNonCanonical, "\ufffd", NewInvalidEscapeSequenceError(`\uXXXX`), NewInvalidEscapeSequenceError(`\uDEAD\uXXXX`), NewInvalidEscapeSequenceError(`\uXXXX`)}, + {`"\uDEAD\uBEEF"`, false, 14, 1, stringNonVerbatim | stringNonCanonical, "\ufffd\ubeef", nil, NewInvalidEscapeSequenceError(`\uDEAD\uBEEF`), errPrev}, + {`"\uD800\udea`, false, 7, 1, stringNonVerbatim | stringNonCanonical, "\ufffd", io.ErrUnexpectedEOF, errPrev, errPrev}, + {`"\uD800\udb`, false, 7, 1, stringNonVerbatim | stringNonCanonical, "\ufffd", io.ErrUnexpectedEOF, NewInvalidEscapeSequenceError(`\uD800\udb`), io.ErrUnexpectedEOF}, + {`"\uD800\udead"`, false, 14, 14, stringNonVerbatim | stringNonCanonical, "\U000102ad", nil, nil, nil}, + {`"\u0022\u005c\u002f\u0008\u000c\u000a\u000d\u0009"`, false, 50, 50, stringNonVerbatim | stringNonCanonical, "\"\\/\b\f\n\r\t", nil, nil, nil}, + {`"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\ud83d\ude02"`, false, 56, 56, stringNonVerbatim | stringNonCanonical, "\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602", nil, nil, nil}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + if tt.wantErrUTF8 == errPrev { + tt.wantErrUTF8 = tt.wantErr + } + if tt.wantErrUnquote == errPrev { + tt.wantErrUnquote = tt.wantErrUTF8 + } + + switch got := ConsumeSimpleString([]byte(tt.in)); { + case tt.simple && got != tt.want: + t.Errorf("consumeSimpleString(%q) = %v, want %v", tt.in, got, tt.want) + case !tt.simple && got != 0: + t.Errorf("consumeSimpleString(%q) = %v, want %v", tt.in, got, 0) + } + + var gotFlags ValueFlags + got, gotErr := ConsumeString(&gotFlags, []byte(tt.in), false) + if gotFlags != tt.wantFlags { + t.Errorf("consumeString(%q, false) flags = %v, want %v", tt.in, gotFlags, tt.wantFlags) + } + if got != tt.want || !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("consumeString(%q, false) = (%v, %v), want (%v, %v)", tt.in, got, gotErr, tt.want, tt.wantErr) + } + + got, gotErr = ConsumeString(&gotFlags, []byte(tt.in), true) + if got != tt.wantUTF8 || !reflect.DeepEqual(gotErr, tt.wantErrUTF8) { + t.Errorf("consumeString(%q, false) = (%v, %v), want (%v, %v)", tt.in, got, gotErr, tt.wantUTF8, tt.wantErrUTF8) + } + + gotUnquote, gotErr := AppendUnquote(nil, tt.in) + if string(gotUnquote) != tt.wantUnquote || !reflect.DeepEqual(gotErr, tt.wantErrUnquote) { + t.Errorf("AppendUnquote(nil, %q) = (%q, %v), want (%q, %v)", tt.in[:got], gotUnquote, gotErr, tt.wantUnquote, tt.wantErrUnquote) + } + }) + } +} + +func TestConsumeNumber(t *testing.T) { + tests := []struct { + in string + simple bool + want int + wantErr error + }{ + {"", false, 0, io.ErrUnexpectedEOF}, + {`"NaN"`, false, 0, NewInvalidCharacterError("\"", "in number (expecting digit)")}, + {`"Infinity"`, false, 0, NewInvalidCharacterError("\"", "in number (expecting digit)")}, + {`"-Infinity"`, false, 0, NewInvalidCharacterError("\"", "in number (expecting digit)")}, + {".0", false, 0, NewInvalidCharacterError(".", "in number (expecting digit)")}, + {"0", true, 1, nil}, + {"-0", false, 2, nil}, + {"+0", false, 0, NewInvalidCharacterError("+", "in number (expecting digit)")}, + {"1", true, 1, nil}, + {"-1", false, 2, nil}, + {"00", true, 1, nil}, + {"-00", false, 2, nil}, + {"01", true, 1, nil}, + {"-01", false, 2, nil}, + {"0i", true, 1, nil}, + {"-0i", false, 2, nil}, + {"0f", true, 1, nil}, + {"-0f", false, 2, nil}, + {"9876543210", true, 10, nil}, + {"-9876543210", false, 11, nil}, + {"9876543210x", true, 10, nil}, + {"-9876543210x", false, 11, nil}, + {" 9876543210", true, 0, NewInvalidCharacterError(" ", "in number (expecting digit)")}, + {"- 9876543210", false, 1, NewInvalidCharacterError(" ", "in number (expecting digit)")}, + {strings.Repeat("9876543210", 1000), true, 10000, nil}, + {"-" + strings.Repeat("9876543210", 1000), false, 1 + 10000, nil}, + {"0.", false, 1, io.ErrUnexpectedEOF}, + {"-0.", false, 2, io.ErrUnexpectedEOF}, + {"0e", false, 1, io.ErrUnexpectedEOF}, + {"-0e", false, 2, io.ErrUnexpectedEOF}, + {"0E", false, 1, io.ErrUnexpectedEOF}, + {"-0E", false, 2, io.ErrUnexpectedEOF}, + {"0.0", false, 3, nil}, + {"-0.0", false, 4, nil}, + {"0e0", false, 3, nil}, + {"-0e0", false, 4, nil}, + {"0E0", false, 3, nil}, + {"-0E0", false, 4, nil}, + {"0.0123456789", false, 12, nil}, + {"-0.0123456789", false, 13, nil}, + {"1.f", false, 2, NewInvalidCharacterError("f", "in number (expecting digit)")}, + {"-1.f", false, 3, NewInvalidCharacterError("f", "in number (expecting digit)")}, + {"1.e", false, 2, NewInvalidCharacterError("e", "in number (expecting digit)")}, + {"-1.e", false, 3, NewInvalidCharacterError("e", "in number (expecting digit)")}, + {"1e0", false, 3, nil}, + {"-1e0", false, 4, nil}, + {"1E0", false, 3, nil}, + {"-1E0", false, 4, nil}, + {"1Ex", false, 2, NewInvalidCharacterError("x", "in number (expecting digit)")}, + {"-1Ex", false, 3, NewInvalidCharacterError("x", "in number (expecting digit)")}, + {"1e-0", false, 4, nil}, + {"-1e-0", false, 5, nil}, + {"1e+0", false, 4, nil}, + {"-1e+0", false, 5, nil}, + {"1E-0", false, 4, nil}, + {"-1E-0", false, 5, nil}, + {"1E+0", false, 4, nil}, + {"-1E+0", false, 5, nil}, + {"1E+00500", false, 8, nil}, + {"-1E+00500", false, 9, nil}, + {"1E+00500x", false, 8, nil}, + {"-1E+00500x", false, 9, nil}, + {"9876543210.0123456789e+01234589x", false, 31, nil}, + {"-9876543210.0123456789e+01234589x", false, 32, nil}, + {"1_000_000", true, 1, nil}, + {"0x12ef", true, 1, nil}, + {"0x1p-2", true, 1, nil}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + switch got := ConsumeSimpleNumber([]byte(tt.in)); { + case tt.simple && got != tt.want: + t.Errorf("ConsumeSimpleNumber(%q) = %v, want %v", tt.in, got, tt.want) + case !tt.simple && got != 0: + t.Errorf("ConsumeSimpleNumber(%q) = %v, want %v", tt.in, got, 0) + } + + got, gotErr := ConsumeNumber([]byte(tt.in)) + if got != tt.want || !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("ConsumeNumber(%q) = (%v, %v), want (%v, %v)", tt.in, got, gotErr, tt.want, tt.wantErr) + } + }) + } +} + +func TestParseHexUint16(t *testing.T) { + tests := []struct { + in string + want uint16 + wantOk bool + }{ + {"", 0, false}, + {"a", 0, false}, + {"ab", 0, false}, + {"abc", 0, false}, + {"abcd", 0xabcd, true}, + {"abcde", 0, false}, + {"9eA1", 0x9ea1, true}, + {"gggg", 0, false}, + {"0000", 0x0000, true}, + {"1234", 0x1234, true}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + got, gotOk := parseHexUint16([]byte(tt.in)) + if got != tt.want || gotOk != tt.wantOk { + t.Errorf("parseHexUint16(%q) = (0x%04x, %v), want (0x%04x, %v)", tt.in, got, gotOk, tt.want, tt.wantOk) + } + }) + } +} + +func TestParseUint(t *testing.T) { + tests := []struct { + in string + want uint64 + wantOk bool + }{ + {"", 0, false}, + {"0", 0, true}, + {"1", 1, true}, + {"-1", 0, false}, + {"1f", 0, false}, + {"00", 0, false}, + {"01", 0, false}, + {"10", 10, true}, + {"10.9", 0, false}, + {" 10", 0, false}, + {"10 ", 0, false}, + {"123456789", 123456789, true}, + {"123456789d", 0, false}, + {"18446744073709551614", math.MaxUint64 - 1, true}, + {"18446744073709551615", math.MaxUint64, true}, + {"18446744073709551616", math.MaxUint64, false}, + {"18446744073709551620", math.MaxUint64, false}, + {"18446744073709551700", math.MaxUint64, false}, + {"18446744073709552000", math.MaxUint64, false}, + {"18446744073709560000", math.MaxUint64, false}, + {"18446744073709600000", math.MaxUint64, false}, + {"18446744073710000000", math.MaxUint64, false}, + {"18446744073800000000", math.MaxUint64, false}, + {"18446744074000000000", math.MaxUint64, false}, + {"18446744080000000000", math.MaxUint64, false}, + {"18446744100000000000", math.MaxUint64, false}, + {"18446745000000000000", math.MaxUint64, false}, + {"18446750000000000000", math.MaxUint64, false}, + {"18446800000000000000", math.MaxUint64, false}, + {"18447000000000000000", math.MaxUint64, false}, + {"18450000000000000000", math.MaxUint64, false}, + {"18500000000000000000", math.MaxUint64, false}, + {"19000000000000000000", math.MaxUint64, false}, + {"19999999999999999999", math.MaxUint64, false}, + {"20000000000000000000", math.MaxUint64, false}, + {"100000000000000000000", math.MaxUint64, false}, + {"99999999999999999999999999999999", math.MaxUint64, false}, + {"99999999999999999999999999999999f", 0, false}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + got, gotOk := ParseUint([]byte(tt.in)) + if got != tt.want || gotOk != tt.wantOk { + t.Errorf("ParseUint(%q) = (%v, %v), want (%v, %v)", tt.in, got, gotOk, tt.want, tt.wantOk) + } + }) + } +} + +func TestParseFloat(t *testing.T) { + tests := []struct { + in string + want32 float64 + want64 float64 + wantOk bool + }{ + {"0", 0, 0, true}, + {"-1", -1, -1, true}, + {"1", 1, 1, true}, + + {"-16777215", -16777215, -16777215, true}, // -(1<<24 - 1) + {"16777215", 16777215, 16777215, true}, // +(1<<24 - 1) + {"-16777216", -16777216, -16777216, true}, // -(1<<24) + {"16777216", 16777216, 16777216, true}, // +(1<<24) + {"-16777217", -16777216, -16777217, true}, // -(1<<24 + 1) + {"16777217", 16777216, 16777217, true}, // +(1<<24 + 1) + + {"-9007199254740991", -9007199254740992, -9007199254740991, true}, // -(1<<53 - 1) + {"9007199254740991", 9007199254740992, 9007199254740991, true}, // +(1<<53 - 1) + {"-9007199254740992", -9007199254740992, -9007199254740992, true}, // -(1<<53) + {"9007199254740992", 9007199254740992, 9007199254740992, true}, // +(1<<53) + {"-9007199254740993", -9007199254740992, -9007199254740992, true}, // -(1<<53 + 1) + {"9007199254740993", 9007199254740992, 9007199254740992, true}, // +(1<<53 + 1) + + {"-1e1000", -math.MaxFloat32, -math.MaxFloat64, false}, + {"1e1000", +math.MaxFloat32, +math.MaxFloat64, false}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + got32, gotOk32 := ParseFloat([]byte(tt.in), 32) + if got32 != tt.want32 || gotOk32 != tt.wantOk { + t.Errorf("ParseFloat(%q, 32) = (%v, %v), want (%v, %v)", tt.in, got32, gotOk32, tt.want32, tt.wantOk) + } + + got64, gotOk64 := ParseFloat([]byte(tt.in), 64) + if got64 != tt.want64 || gotOk64 != tt.wantOk { + t.Errorf("ParseFloat(%q, 64) = (%v, %v), want (%v, %v)", tt.in, got64, gotOk64, tt.want64, tt.wantOk) + } + }) + } +} diff --git a/src/encoding/json/internal/jsonwire/encode.go b/src/encoding/json/internal/jsonwire/encode.go new file mode 100644 index 00000000000000..8f9b8ab09e64c4 --- /dev/null +++ b/src/encoding/json/internal/jsonwire/encode.go @@ -0,0 +1,290 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsonwire + +import ( + "math" + "slices" + "strconv" + "unicode/utf16" + "unicode/utf8" + + "encoding/json/internal/jsonflags" +) + +// escapeASCII reports whether the ASCII character needs to be escaped. +// It conservatively assumes EscapeForHTML. +var escapeASCII = [...]uint8{ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // escape control characters + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // escape control characters + 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, // escape '"' and '&' + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, // escape '<' and '>' + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, // escape '\\' + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +} + +// NeedEscape reports whether src needs escaping of any characters. +// It conservatively assumes EscapeForHTML and EscapeForJS. +// It reports true for inputs with invalid UTF-8. +func NeedEscape[Bytes ~[]byte | ~string](src Bytes) bool { + var i int + for uint(len(src)) > uint(i) { + if c := src[i]; c < utf8.RuneSelf { + if escapeASCII[c] > 0 { + return true + } + i++ + } else { + r, rn := utf8.DecodeRuneInString(string(truncateMaxUTF8(src[i:]))) + if r == utf8.RuneError || r == '\u2028' || r == '\u2029' { + return true + } + i += rn + } + } + return false +} + +// AppendQuote appends src to dst as a JSON string per RFC 7159, section 7. +// +// It takes in flags and respects the following: +// - EscapeForHTML escapes '<', '>', and '&'. +// - EscapeForJS escapes '\u2028' and '\u2029'. +// - AllowInvalidUTF8 avoids reporting an error for invalid UTF-8. +// +// Regardless of whether AllowInvalidUTF8 is specified, +// invalid bytes are replaced with the Unicode replacement character ('\ufffd'). +// If no escape flags are set, then the shortest representable form is used, +// which is also the canonical form for strings (RFC 8785, section 3.2.2.2). +func AppendQuote[Bytes ~[]byte | ~string](dst []byte, src Bytes, flags *jsonflags.Flags) ([]byte, error) { + var i, n int + var hasInvalidUTF8 bool + dst = slices.Grow(dst, len(`"`)+len(src)+len(`"`)) + dst = append(dst, '"') + for uint(len(src)) > uint(n) { + if c := src[n]; c < utf8.RuneSelf { + // Handle single-byte ASCII. + n++ + if escapeASCII[c] == 0 { + continue // no escaping possibly needed + } + // Handle escaping of single-byte ASCII. + if !(c == '<' || c == '>' || c == '&') || flags.Get(jsonflags.EscapeForHTML) { + dst = append(dst, src[i:n-1]...) + dst = appendEscapedASCII(dst, c) + i = n + } + } else { + // Handle multi-byte Unicode. + r, rn := utf8.DecodeRuneInString(string(truncateMaxUTF8(src[n:]))) + n += rn + if r != utf8.RuneError && r != '\u2028' && r != '\u2029' { + continue // no escaping possibly needed + } + // Handle escaping of multi-byte Unicode. + switch { + case isInvalidUTF8(r, rn): + hasInvalidUTF8 = true + dst = append(dst, src[i:n-rn]...) + dst = append(dst, "\ufffd"...) + i = n + case (r == '\u2028' || r == '\u2029') && flags.Get(jsonflags.EscapeForJS): + dst = append(dst, src[i:n-rn]...) + dst = appendEscapedUnicode(dst, r) + i = n + } + } + } + dst = append(dst, src[i:n]...) + dst = append(dst, '"') + if hasInvalidUTF8 && !flags.Get(jsonflags.AllowInvalidUTF8) { + return dst, ErrInvalidUTF8 + } + return dst, nil +} + +func appendEscapedASCII(dst []byte, c byte) []byte { + switch c { + case '"', '\\': + dst = append(dst, '\\', c) + case '\b': + dst = append(dst, "\\b"...) + case '\f': + dst = append(dst, "\\f"...) + case '\n': + dst = append(dst, "\\n"...) + case '\r': + dst = append(dst, "\\r"...) + case '\t': + dst = append(dst, "\\t"...) + default: + dst = appendEscapedUTF16(dst, uint16(c)) + } + return dst +} + +func appendEscapedUnicode(dst []byte, r rune) []byte { + if r1, r2 := utf16.EncodeRune(r); r1 != '\ufffd' && r2 != '\ufffd' { + dst = appendEscapedUTF16(dst, uint16(r1)) + dst = appendEscapedUTF16(dst, uint16(r2)) + } else { + dst = appendEscapedUTF16(dst, uint16(r)) + } + return dst +} + +func appendEscapedUTF16(dst []byte, x uint16) []byte { + const hex = "0123456789abcdef" + return append(dst, '\\', 'u', hex[(x>>12)&0xf], hex[(x>>8)&0xf], hex[(x>>4)&0xf], hex[(x>>0)&0xf]) +} + +// ReformatString consumes a JSON string from src and appends it to dst, +// reformatting it if necessary according to the specified flags. +// It returns the appended output and the number of consumed input bytes. +func ReformatString(dst, src []byte, flags *jsonflags.Flags) ([]byte, int, error) { + // TODO: Should this update ValueFlags as input? + var valFlags ValueFlags + n, err := ConsumeString(&valFlags, src, !flags.Get(jsonflags.AllowInvalidUTF8)) + if err != nil { + return dst, n, err + } + + // If the output requires no special escapes, and the input + // is already in canonical form or should be preserved verbatim, + // then directly copy the input to the output. + if !flags.Get(jsonflags.AnyEscape) && + (valFlags.IsCanonical() || flags.Get(jsonflags.PreserveRawStrings)) { + dst = append(dst, src[:n]...) // copy the string verbatim + return dst, n, nil + } + + // Under [jsonflags.PreserveRawStrings], any pre-escaped sequences + // remain escaped, however we still need to respect the + // [jsonflags.EscapeForHTML] and [jsonflags.EscapeForJS] options. + if flags.Get(jsonflags.PreserveRawStrings) { + var i, lastAppendIndex int + for i < n { + if c := src[i]; c < utf8.RuneSelf { + if (c == '<' || c == '>' || c == '&') && flags.Get(jsonflags.EscapeForHTML) { + dst = append(dst, src[lastAppendIndex:i]...) + dst = appendEscapedASCII(dst, c) + lastAppendIndex = i + 1 + } + i++ + } else { + r, rn := utf8.DecodeRune(truncateMaxUTF8(src[i:])) + if (r == '\u2028' || r == '\u2029') && flags.Get(jsonflags.EscapeForJS) { + dst = append(dst, src[lastAppendIndex:i]...) + dst = appendEscapedUnicode(dst, r) + lastAppendIndex = i + rn + } + i += rn + } + } + return append(dst, src[lastAppendIndex:n]...), n, nil + } + + // The input contains characters that might need escaping, + // unnecessary escape sequences, or invalid UTF-8. + // Perform a round-trip unquote and quote to properly reformat + // these sequences according the current flags. + b, _ := AppendUnquote(nil, src[:n]) + dst, _ = AppendQuote(dst, b, flags) + return dst, n, nil +} + +// AppendFloat appends src to dst as a JSON number per RFC 7159, section 6. +// It formats numbers similar to the ES6 number-to-string conversion. +// See https://go.dev/issue/14135. +// +// The output is identical to ECMA-262, 6th edition, section 7.1.12.1 and with +// RFC 8785, section 3.2.2.3 for 64-bit floating-point numbers except for -0, +// which is formatted as -0 instead of just 0. +// +// For 32-bit floating-point numbers, +// the output is a 32-bit equivalent of the algorithm. +// Note that ECMA-262 specifies no algorithm for 32-bit numbers. +func AppendFloat(dst []byte, src float64, bits int) []byte { + if bits == 32 { + src = float64(float32(src)) + } + + abs := math.Abs(src) + fmt := byte('f') + if abs != 0 { + if bits == 64 && (float64(abs) < 1e-6 || float64(abs) >= 1e21) || + bits == 32 && (float32(abs) < 1e-6 || float32(abs) >= 1e21) { + fmt = 'e' + } + } + dst = strconv.AppendFloat(dst, src, fmt, -1, bits) + if fmt == 'e' { + // Clean up e-09 to e-9. + n := len(dst) + if n >= 4 && dst[n-4] == 'e' && dst[n-3] == '-' && dst[n-2] == '0' { + dst[n-2] = dst[n-1] + dst = dst[:n-1] + } + } + return dst +} + +// ReformatNumber consumes a JSON string from src and appends it to dst, +// canonicalizing it if specified. +// It returns the appended output and the number of consumed input bytes. +func ReformatNumber(dst, src []byte, flags *jsonflags.Flags) ([]byte, int, error) { + n, err := ConsumeNumber(src) + if err != nil { + return dst, n, err + } + if !flags.Get(jsonflags.CanonicalizeNumbers) { + dst = append(dst, src[:n]...) // copy the number verbatim + return dst, n, nil + } + + // Identify the kind of number. + var isFloat bool + for _, c := range src[:n] { + if c == '.' || c == 'e' || c == 'E' { + isFloat = true // has fraction or exponent + break + } + } + + // Check if need to canonicalize this kind of number. + switch { + case string(src[:n]) == "-0": + break // canonicalize -0 as 0 regardless of kind + case isFloat: + if !flags.Get(jsonflags.CanonicalizeRawFloats) { + dst = append(dst, src[:n]...) // copy the number verbatim + return dst, n, nil + } + default: + // As an optimization, we can copy integer numbers below 2⁵³ verbatim + // since the canonical form is always identical. + const maxExactIntegerDigits = 16 // len(strconv.AppendUint(nil, 1<<53, 10)) + if !flags.Get(jsonflags.CanonicalizeRawInts) || n < maxExactIntegerDigits { + dst = append(dst, src[:n]...) // copy the number verbatim + return dst, n, nil + } + } + + // Parse and reformat the number (which uses a canonical format). + fv, _ := strconv.ParseFloat(string(src[:n]), 64) + switch { + case fv == 0: + fv = 0 // normalize negative zero as just zero + case math.IsInf(fv, +1): + fv = +math.MaxFloat64 + case math.IsInf(fv, -1): + fv = -math.MaxFloat64 + } + return AppendFloat(dst, fv, 64), n, nil +} diff --git a/src/encoding/json/internal/jsonwire/encode_test.go b/src/encoding/json/internal/jsonwire/encode_test.go new file mode 100644 index 00000000000000..6459d20e0951ea --- /dev/null +++ b/src/encoding/json/internal/jsonwire/encode_test.go @@ -0,0 +1,332 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsonwire + +import ( + "bufio" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "flag" + "math" + "net/http" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "encoding/json/internal/jsonflags" +) + +func TestAppendQuote(t *testing.T) { + tests := []struct { + in string + flags jsonflags.Bools + want string + wantErr error + wantErrUTF8 error + }{ + {"", 0, `""`, nil, nil}, + {"hello", 0, `"hello"`, nil, nil}, + {"\x00", 0, `"\u0000"`, nil, nil}, + {"\x1f", 0, `"\u001f"`, nil, nil}, + {"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", 0, `"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"`, nil, nil}, + {" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f", 0, "\" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f\"", nil, nil}, + {" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f", jsonflags.EscapeForHTML, "\" !#$%\\u0026'()*+,-./0123456789:;\\u003c=\\u003e?@[]^_`{|}~\x7f\"", nil, nil}, + {" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f", jsonflags.EscapeForJS, "\" !#$%&'()*+,-./0123456789:;<=>?@[]^_`{|}~\x7f\"", nil, nil}, + {"\u2027\u2028\u2029\u2030", 0, "\"\u2027\u2028\u2029\u2030\"", nil, nil}, + {"\u2027\u2028\u2029\u2030", jsonflags.EscapeForHTML, "\"\u2027\u2028\u2029\u2030\"", nil, nil}, + {"\u2027\u2028\u2029\u2030", jsonflags.EscapeForJS, "\"\u2027\\u2028\\u2029\u2030\"", nil, nil}, + {"x\x80\ufffd", 0, "\"x\ufffd\ufffd\"", nil, ErrInvalidUTF8}, + {"x\xff\ufffd", 0, "\"x\ufffd\ufffd\"", nil, ErrInvalidUTF8}, + {"x\xc0", 0, "\"x\ufffd\"", nil, ErrInvalidUTF8}, + {"x\xc0\x80", 0, "\"x\ufffd\ufffd\"", nil, ErrInvalidUTF8}, + {"x\xe0", 0, "\"x\ufffd\"", nil, ErrInvalidUTF8}, + {"x\xe0\x80", 0, "\"x\ufffd\ufffd\"", nil, ErrInvalidUTF8}, + {"x\xe0\x80\x80", 0, "\"x\ufffd\ufffd\ufffd\"", nil, ErrInvalidUTF8}, + {"x\xf0", 0, "\"x\ufffd\"", nil, ErrInvalidUTF8}, + {"x\xf0\x80", 0, "\"x\ufffd\ufffd\"", nil, ErrInvalidUTF8}, + {"x\xf0\x80\x80", 0, "\"x\ufffd\ufffd\ufffd\"", nil, ErrInvalidUTF8}, + {"x\xf0\x80\x80\x80", 0, "\"x\ufffd\ufffd\ufffd\ufffd\"", nil, ErrInvalidUTF8}, + {"x\xed\xba\xad", 0, "\"x\ufffd\ufffd\ufffd\"", nil, ErrInvalidUTF8}, + {"\"\\/\b\f\n\r\t", 0, `"\"\\/\b\f\n\r\t"`, nil, nil}, + {"٩(-̮̮̃-̃)۶ ٩(●̮̮̃•̃)۶ ٩(͡๏̯͡๏)۶ ٩(-̮̮̃•̃).", 0, `"٩(-̮̮̃-̃)۶ ٩(●̮̮̃•̃)۶ ٩(͡๏̯͡๏)۶ ٩(-̮̮̃•̃)."`, nil, nil}, + {"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602", 0, "\"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602\"", nil, nil}, + {"\u0000\u001f\u0020\u0022\u0026\u003c\u003e\u005c\u007f\u0080\u2028\u2029\ufffd\U0001f602", 0, "\"\\u0000\\u001f\u0020\\\"\u0026\u003c\u003e\\\\\u007f\u0080\u2028\u2029\ufffd\U0001f602\"", nil, nil}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + var flags jsonflags.Flags + flags.Set(tt.flags | 1) + + flags.Set(jsonflags.AllowInvalidUTF8 | 1) + got, gotErr := AppendQuote(nil, tt.in, &flags) + if string(got) != tt.want || !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("AppendQuote(nil, %q, ...) = (%s, %v), want (%s, %v)", tt.in, got, gotErr, tt.want, tt.wantErr) + } + flags.Set(jsonflags.AllowInvalidUTF8 | 0) + switch got, gotErr := AppendQuote(nil, tt.in, &flags); { + case tt.wantErrUTF8 == nil && (string(got) != tt.want || !reflect.DeepEqual(gotErr, tt.wantErr)): + t.Errorf("AppendQuote(nil, %q, ...) = (%s, %v), want (%s, %v)", tt.in, got, gotErr, tt.want, tt.wantErr) + case tt.wantErrUTF8 != nil && (!strings.HasPrefix(tt.want, string(got)) || !reflect.DeepEqual(gotErr, tt.wantErrUTF8)): + t.Errorf("AppendQuote(nil, %q, ...) = (%s, %v), want (%s, %v)", tt.in, got, gotErr, tt.want, tt.wantErrUTF8) + } + }) + } +} + +func TestAppendNumber(t *testing.T) { + tests := []struct { + in float64 + want32 string + want64 string + }{ + {math.E, "2.7182817", "2.718281828459045"}, + {math.Pi, "3.1415927", "3.141592653589793"}, + {math.SmallestNonzeroFloat32, "1e-45", "1.401298464324817e-45"}, + {math.SmallestNonzeroFloat64, "0", "5e-324"}, + {math.MaxFloat32, "3.4028235e+38", "3.4028234663852886e+38"}, + {math.MaxFloat64, "", "1.7976931348623157e+308"}, + {0.1111111111111111, "0.11111111", "0.1111111111111111"}, + {0.2222222222222222, "0.22222222", "0.2222222222222222"}, + {0.3333333333333333, "0.33333334", "0.3333333333333333"}, + {0.4444444444444444, "0.44444445", "0.4444444444444444"}, + {0.5555555555555555, "0.5555556", "0.5555555555555555"}, + {0.6666666666666666, "0.6666667", "0.6666666666666666"}, + {0.7777777777777777, "0.7777778", "0.7777777777777777"}, + {0.8888888888888888, "0.8888889", "0.8888888888888888"}, + {0.9999999999999999, "1", "0.9999999999999999"}, + + // The following entries are from RFC 8785, appendix B + // which are designed to ensure repeatable formatting of 64-bit floats. + {math.Float64frombits(0x0000000000000000), "0", "0"}, + {math.Float64frombits(0x8000000000000000), "-0", "-0"}, // differs from RFC 8785 + {math.Float64frombits(0x0000000000000001), "0", "5e-324"}, + {math.Float64frombits(0x8000000000000001), "-0", "-5e-324"}, + {math.Float64frombits(0x7fefffffffffffff), "", "1.7976931348623157e+308"}, + {math.Float64frombits(0xffefffffffffffff), "", "-1.7976931348623157e+308"}, + {math.Float64frombits(0x4340000000000000), "9007199000000000", "9007199254740992"}, + {math.Float64frombits(0xc340000000000000), "-9007199000000000", "-9007199254740992"}, + {math.Float64frombits(0x4430000000000000), "295147900000000000000", "295147905179352830000"}, + {math.Float64frombits(0x44b52d02c7e14af5), "1e+23", "9.999999999999997e+22"}, + {math.Float64frombits(0x44b52d02c7e14af6), "1e+23", "1e+23"}, + {math.Float64frombits(0x44b52d02c7e14af7), "1e+23", "1.0000000000000001e+23"}, + {math.Float64frombits(0x444b1ae4d6e2ef4e), "1e+21", "999999999999999700000"}, + {math.Float64frombits(0x444b1ae4d6e2ef4f), "1e+21", "999999999999999900000"}, + {math.Float64frombits(0x444b1ae4d6e2ef50), "1e+21", "1e+21"}, + {math.Float64frombits(0x3eb0c6f7a0b5ed8c), "0.000001", "9.999999999999997e-7"}, + {math.Float64frombits(0x3eb0c6f7a0b5ed8d), "0.000001", "0.000001"}, + {math.Float64frombits(0x41b3de4355555553), "333333340", "333333333.3333332"}, + {math.Float64frombits(0x41b3de4355555554), "333333340", "333333333.33333325"}, + {math.Float64frombits(0x41b3de4355555555), "333333340", "333333333.3333333"}, + {math.Float64frombits(0x41b3de4355555556), "333333340", "333333333.3333334"}, + {math.Float64frombits(0x41b3de4355555557), "333333340", "333333333.33333343"}, + {math.Float64frombits(0xbecbf647612f3696), "-0.0000033333333", "-0.0000033333333333333333"}, + {math.Float64frombits(0x43143ff3c1cb0959), "1424953900000000", "1424953923781206.2"}, + + // The following are select entries from RFC 8785, appendix B, + // but modified for equivalent 32-bit behavior. + {float64(math.Float32frombits(0x65a96815)), "9.999999e+22", "9.999998877476383e+22"}, + {float64(math.Float32frombits(0x65a96816)), "1e+23", "9.999999778196308e+22"}, + {float64(math.Float32frombits(0x65a96817)), "1.0000001e+23", "1.0000000678916234e+23"}, + {float64(math.Float32frombits(0x6258d725)), "999999900000000000000", "999999879303389000000"}, + {float64(math.Float32frombits(0x6258d726)), "999999950000000000000", "999999949672133200000"}, + {float64(math.Float32frombits(0x6258d727)), "1e+21", "1.0000000200408773e+21"}, + {float64(math.Float32frombits(0x6258d728)), "1.0000001e+21", "1.0000000904096215e+21"}, + {float64(math.Float32frombits(0x358637bc)), "9.999999e-7", "9.99999883788405e-7"}, + {float64(math.Float32frombits(0x358637bd)), "0.000001", "9.999999974752427e-7"}, + {float64(math.Float32frombits(0x358637be)), "0.0000010000001", "0.0000010000001111620804"}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + if got32 := string(AppendFloat(nil, tt.in, 32)); got32 != tt.want32 && tt.want32 != "" { + t.Errorf("AppendFloat(nil, %v, 32) = %v, want %v", tt.in, got32, tt.want32) + } + if got64 := string(AppendFloat(nil, tt.in, 64)); got64 != tt.want64 && tt.want64 != "" { + t.Errorf("AppendFloat(nil, %v, 64) = %v, want %v", tt.in, got64, tt.want64) + } + }) + } +} + +// The default of 1e4 lines was chosen since it is sufficiently large to include +// test numbers from all three categories (i.e., static, series, and random). +// Yet, it is sufficiently low to execute quickly relative to other tests. +// +// Processing 1e8 lines takes a minute and processes about 4GiB worth of text. +var testCanonicalNumberLines = flag.Float64("canonical-number-lines", 1e4, "specify the number of lines to check from the canonical numbers testdata") + +// TestCanonicalNumber verifies that appendNumber complies with RFC 8785 +// according to the testdata provided by the reference implementation. +// See https://github.com/cyberphone/json-canonicalization/tree/master/testdata#es6-numbers. +func TestCanonicalNumber(t *testing.T) { + const testfileURL = "https://github.com/cyberphone/json-canonicalization/releases/download/es6testfile/es6testfile100m.txt.gz" + hashes := map[float64]string{ + 1e3: "be18b62b6f69cdab33a7e0dae0d9cfa869fda80ddc712221570f9f40a5878687", + 1e4: "b9f7a8e75ef22a835685a52ccba7f7d6bdc99e34b010992cbc5864cd12be6892", + 1e5: "22776e6d4b49fa294a0d0f349268e5c28808fe7e0cb2bcbe28f63894e494d4c7", + 1e6: "49415fee2c56c77864931bd3624faad425c3c577d6d74e89a83bc725506dad16", + 1e7: "b9f8a44a91d46813b21b9602e72f112613c91408db0b8341fb94603d9db135e0", + 1e8: "0f7dda6b0837dde083c5d6b896f7d62340c8a2415b0c7121d83145e08a755272", + } + wantHash := hashes[*testCanonicalNumberLines] + if wantHash == "" { + t.Fatalf("canonical-number-lines must be one of the following values: 1e3, 1e4, 1e5, 1e6, 1e7, 1e8") + } + numLines := int(*testCanonicalNumberLines) + + // generator returns a function that generates the next float64 to format. + // This implements the algorithm specified in the reference implementation. + generator := func() func() float64 { + static := [...]uint64{ + 0x0000000000000000, 0x8000000000000000, 0x0000000000000001, 0x8000000000000001, + 0xc46696695dbd1cc3, 0xc43211ede4974a35, 0xc3fce97ca0f21056, 0xc3c7213080c1a6ac, + 0xc39280f39a348556, 0xc35d9b1f5d20d557, 0xc327af4c4a80aaac, 0xc2f2f2a36ecd5556, + 0xc2be51057e155558, 0xc28840d131aaaaac, 0xc253670dc1555557, 0xc21f0b4935555557, + 0xc1e8d5d42aaaaaac, 0xc1b3de4355555556, 0xc17fca0555555556, 0xc1496e6aaaaaaaab, + 0xc114585555555555, 0xc0e046aaaaaaaaab, 0xc0aa0aaaaaaaaaaa, 0xc074d55555555555, + 0xc040aaaaaaaaaaab, 0xc00aaaaaaaaaaaab, 0xbfd5555555555555, 0xbfa1111111111111, + 0xbf6b4e81b4e81b4f, 0xbf35d867c3ece2a5, 0xbf0179ec9cbd821e, 0xbecbf647612f3696, + 0xbe965e9f80f29212, 0xbe61e54c672874db, 0xbe2ca213d840baf8, 0xbdf6e80fe033c8c6, + 0xbdc2533fe68fd3d2, 0xbd8d51ffd74c861c, 0xbd5774ccac3d3817, 0xbd22c3d6f030f9ac, + 0xbcee0624b3818f79, 0xbcb804ea293472c7, 0xbc833721ba905bd3, 0xbc4ebe9c5db3c61e, + 0xbc18987d17c304e5, 0xbbe3ad30dfcf371d, 0xbbaf7b816618582f, 0xbb792f9ab81379bf, + 0xbb442615600f9499, 0xbb101e77800c76e1, 0xbad9ca58cce0be35, 0xbaa4a1e0a3e6fe90, + 0xba708180831f320d, 0xba3a68cd9e985016, 0x446696695dbd1cc3, 0x443211ede4974a35, + 0x43fce97ca0f21056, 0x43c7213080c1a6ac, 0x439280f39a348556, 0x435d9b1f5d20d557, + 0x4327af4c4a80aaac, 0x42f2f2a36ecd5556, 0x42be51057e155558, 0x428840d131aaaaac, + 0x4253670dc1555557, 0x421f0b4935555557, 0x41e8d5d42aaaaaac, 0x41b3de4355555556, + 0x417fca0555555556, 0x41496e6aaaaaaaab, 0x4114585555555555, 0x40e046aaaaaaaaab, + 0x40aa0aaaaaaaaaaa, 0x4074d55555555555, 0x4040aaaaaaaaaaab, 0x400aaaaaaaaaaaab, + 0x3fd5555555555555, 0x3fa1111111111111, 0x3f6b4e81b4e81b4f, 0x3f35d867c3ece2a5, + 0x3f0179ec9cbd821e, 0x3ecbf647612f3696, 0x3e965e9f80f29212, 0x3e61e54c672874db, + 0x3e2ca213d840baf8, 0x3df6e80fe033c8c6, 0x3dc2533fe68fd3d2, 0x3d8d51ffd74c861c, + 0x3d5774ccac3d3817, 0x3d22c3d6f030f9ac, 0x3cee0624b3818f79, 0x3cb804ea293472c7, + 0x3c833721ba905bd3, 0x3c4ebe9c5db3c61e, 0x3c18987d17c304e5, 0x3be3ad30dfcf371d, + 0x3baf7b816618582f, 0x3b792f9ab81379bf, 0x3b442615600f9499, 0x3b101e77800c76e1, + 0x3ad9ca58cce0be35, 0x3aa4a1e0a3e6fe90, 0x3a708180831f320d, 0x3a3a68cd9e985016, + 0x4024000000000000, 0x4014000000000000, 0x3fe0000000000000, 0x3fa999999999999a, + 0x3f747ae147ae147b, 0x3f40624dd2f1a9fc, 0x3f0a36e2eb1c432d, 0x3ed4f8b588e368f1, + 0x3ea0c6f7a0b5ed8d, 0x3e6ad7f29abcaf48, 0x3e35798ee2308c3a, 0x3ed539223589fa95, + 0x3ed4ff26cd5a7781, 0x3ed4f95a762283ff, 0x3ed4f8c60703520c, 0x3ed4f8b72f19cd0d, + 0x3ed4f8b5b31c0c8d, 0x3ed4f8b58d1c461a, 0x3ed4f8b5894f7f0e, 0x3ed4f8b588ee37f3, + 0x3ed4f8b588e47da4, 0x3ed4f8b588e3849c, 0x3ed4f8b588e36bb5, 0x3ed4f8b588e36937, + 0x3ed4f8b588e368f8, 0x3ed4f8b588e368f1, 0x3ff0000000000000, 0xbff0000000000000, + 0xbfeffffffffffffa, 0xbfeffffffffffffb, 0x3feffffffffffffa, 0x3feffffffffffffb, + 0x3feffffffffffffc, 0x3feffffffffffffe, 0xbfefffffffffffff, 0xbfefffffffffffff, + 0x3fefffffffffffff, 0x3fefffffffffffff, 0x3fd3333333333332, 0x3fd3333333333333, + 0x3fd3333333333334, 0x0010000000000000, 0x000ffffffffffffd, 0x000fffffffffffff, + 0x7fefffffffffffff, 0xffefffffffffffff, 0x4340000000000000, 0xc340000000000000, + 0x4430000000000000, 0x44b52d02c7e14af5, 0x44b52d02c7e14af6, 0x44b52d02c7e14af7, + 0x444b1ae4d6e2ef4e, 0x444b1ae4d6e2ef4f, 0x444b1ae4d6e2ef50, 0x3eb0c6f7a0b5ed8c, + 0x3eb0c6f7a0b5ed8d, 0x41b3de4355555553, 0x41b3de4355555554, 0x41b3de4355555555, + 0x41b3de4355555556, 0x41b3de4355555557, 0xbecbf647612f3696, 0x43143ff3c1cb0959, + } + var state struct { + idx int + data []byte + block [sha256.Size]byte + } + return func() float64 { + const numSerial = 2000 + var f float64 + switch { + case state.idx < len(static): + f = math.Float64frombits(static[state.idx]) + case state.idx < len(static)+numSerial: + f = math.Float64frombits(0x0010000000000000 + uint64(state.idx-len(static))) + default: + for f == 0 || math.IsNaN(f) || math.IsInf(f, 0) { + if len(state.data) == 0 { + state.block = sha256.Sum256(state.block[:]) + state.data = state.block[:] + } + f = math.Float64frombits(binary.LittleEndian.Uint64(state.data)) + state.data = state.data[8:] + } + } + state.idx++ + return f + } + } + + // Pass through the test twice. In the first pass we only hash the output, + // while in the second pass we check every line against the golden testdata. + // If the hashes match in the first pass, then we skip the second pass. + for _, checkGolden := range []bool{false, true} { + var br *bufio.Reader // for line-by-line reading of es6testfile100m.txt + if checkGolden { + resp, err := http.Get(testfileURL) + if err != nil { + t.Fatalf("http.Get error: %v", err) + } + defer resp.Body.Close() + + zr, err := gzip.NewReader(resp.Body) + if err != nil { + t.Fatalf("gzip.NewReader error: %v", err) + } + + br = bufio.NewReader(zr) + } + + // appendNumberJCS differs from appendNumber only for -0. + appendNumberJCS := func(b []byte, f float64) []byte { + if math.Signbit(f) && f == 0 { + return append(b, '0') + } + return AppendFloat(b, f, 64) + } + + var gotLine []byte + next := generator() + hash := sha256.New() + start := time.Now() + lastPrint := start + for n := 1; n <= numLines; n++ { + // Generate the formatted line for this number. + f := next() + gotLine = gotLine[:0] // reset from previous usage + gotLine = strconv.AppendUint(gotLine, math.Float64bits(f), 16) + gotLine = append(gotLine, ',') + gotLine = appendNumberJCS(gotLine, f) + gotLine = append(gotLine, '\n') + hash.Write(gotLine) + + // Check that the formatted line matches. + if checkGolden { + wantLine, err := br.ReadBytes('\n') + if err != nil { + t.Fatalf("bufio.Reader.ReadBytes error: %v", err) + } + if !bytes.Equal(gotLine, wantLine) { + t.Errorf("mismatch on line %d:\n\tgot %v\n\twant %v", + n, strings.TrimSpace(string(gotLine)), strings.TrimSpace(string(wantLine))) + } + } + + // Print progress. + if now := time.Now(); now.Sub(lastPrint) > time.Second || n == numLines { + remaining := float64(now.Sub(start)) * float64(numLines-n) / float64(n) + t.Logf("%0.3f%% (%v remaining)", + 100.0*float64(n)/float64(numLines), + time.Duration(remaining).Round(time.Second)) + lastPrint = now + } + } + + gotHash := hex.EncodeToString(hash.Sum(nil)) + if gotHash == wantHash { + return // hashes match, no need to check golden testdata + } + } +} diff --git a/src/encoding/json/internal/jsonwire/wire.go b/src/encoding/json/internal/jsonwire/wire.go new file mode 100644 index 00000000000000..6cf19c5cfe62bb --- /dev/null +++ b/src/encoding/json/internal/jsonwire/wire.go @@ -0,0 +1,217 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +// Package jsonwire implements stateless functionality for handling JSON text. +package jsonwire + +import ( + "cmp" + "errors" + "strconv" + "strings" + "unicode" + "unicode/utf16" + "unicode/utf8" +) + +// TrimSuffixWhitespace trims JSON from the end of b. +func TrimSuffixWhitespace(b []byte) []byte { + // NOTE: The arguments and logic are kept simple to keep this inlinable. + n := len(b) - 1 + for n >= 0 && (b[n] == ' ' || b[n] == '\t' || b[n] == '\r' || b[n] == '\n') { + n-- + } + return b[:n+1] +} + +// TrimSuffixString trims a valid JSON string at the end of b. +// The behavior is undefined if there is not a valid JSON string present. +func TrimSuffixString(b []byte) []byte { + // NOTE: The arguments and logic are kept simple to keep this inlinable. + if len(b) > 0 && b[len(b)-1] == '"' { + b = b[:len(b)-1] + } + for len(b) >= 2 && !(b[len(b)-1] == '"' && b[len(b)-2] != '\\') { + b = b[:len(b)-1] // trim all characters except an unescaped quote + } + if len(b) > 0 && b[len(b)-1] == '"' { + b = b[:len(b)-1] + } + return b +} + +// HasSuffixByte reports whether b ends with c. +func HasSuffixByte(b []byte, c byte) bool { + // NOTE: The arguments and logic are kept simple to keep this inlinable. + return len(b) > 0 && b[len(b)-1] == c +} + +// TrimSuffixByte removes c from the end of b if it is present. +func TrimSuffixByte(b []byte, c byte) []byte { + // NOTE: The arguments and logic are kept simple to keep this inlinable. + if len(b) > 0 && b[len(b)-1] == c { + return b[:len(b)-1] + } + return b +} + +// QuoteRune quotes the first rune in the input. +func QuoteRune[Bytes ~[]byte | ~string](b Bytes) string { + r, n := utf8.DecodeRuneInString(string(truncateMaxUTF8(b))) + if r == utf8.RuneError && n == 1 { + return `'\x` + strconv.FormatUint(uint64(b[0]), 16) + `'` + } + return strconv.QuoteRune(r) +} + +// CompareUTF16 lexicographically compares x to y according +// to the UTF-16 codepoints of the UTF-8 encoded input strings. +// This implements the ordering specified in RFC 8785, section 3.2.3. +func CompareUTF16[Bytes ~[]byte | ~string](x, y Bytes) int { + // NOTE: This is an optimized, mostly allocation-free implementation + // of CompareUTF16Simple in wire_test.go. FuzzCompareUTF16 verifies that the + // two implementations agree on the result of comparing any two strings. + isUTF16Self := func(r rune) bool { + return ('\u0000' <= r && r <= '\uD7FF') || ('\uE000' <= r && r <= '\uFFFF') + } + + for { + if len(x) == 0 || len(y) == 0 { + return cmp.Compare(len(x), len(y)) + } + + // ASCII fast-path. + if x[0] < utf8.RuneSelf || y[0] < utf8.RuneSelf { + if x[0] != y[0] { + return cmp.Compare(x[0], y[0]) + } + x, y = x[1:], y[1:] + continue + } + + // Decode next pair of runes as UTF-8. + rx, nx := utf8.DecodeRuneInString(string(truncateMaxUTF8(x))) + ry, ny := utf8.DecodeRuneInString(string(truncateMaxUTF8(y))) + + selfx := isUTF16Self(rx) + selfy := isUTF16Self(ry) + switch { + // The x rune is a single UTF-16 codepoint, while + // the y rune is a surrogate pair of UTF-16 codepoints. + case selfx && !selfy: + ry, _ = utf16.EncodeRune(ry) + // The y rune is a single UTF-16 codepoint, while + // the x rune is a surrogate pair of UTF-16 codepoints. + case selfy && !selfx: + rx, _ = utf16.EncodeRune(rx) + } + if rx != ry { + return cmp.Compare(rx, ry) + } + + // Check for invalid UTF-8, in which case, + // we just perform a byte-for-byte comparison. + if isInvalidUTF8(rx, nx) || isInvalidUTF8(ry, ny) { + if x[0] != y[0] { + return cmp.Compare(x[0], y[0]) + } + } + x, y = x[nx:], y[ny:] + } +} + +// truncateMaxUTF8 truncates b such it contains at least one rune. +// +// The utf8 package currently lacks generic variants, which complicates +// generic functions that operates on either []byte or string. +// As a hack, we always call the utf8 function operating on strings, +// but always truncate the input such that the result is identical. +// +// Example usage: +// +// utf8.DecodeRuneInString(string(truncateMaxUTF8(b))) +// +// Converting a []byte to a string is stack allocated since +// truncateMaxUTF8 guarantees that the []byte is short. +func truncateMaxUTF8[Bytes ~[]byte | ~string](b Bytes) Bytes { + // TODO(https://go.dev/issue/56948): Remove this function and + // instead directly call generic utf8 functions wherever used. + if len(b) > utf8.UTFMax { + return b[:utf8.UTFMax] + } + return b +} + +// TODO(https://go.dev/issue/70547): Use utf8.ErrInvalid instead. +var ErrInvalidUTF8 = errors.New("invalid UTF-8") + +func NewInvalidCharacterError[Bytes ~[]byte | ~string](prefix Bytes, where string) error { + what := QuoteRune(prefix) + return errors.New("invalid character " + what + " " + where) +} + +func NewInvalidEscapeSequenceError[Bytes ~[]byte | ~string](what Bytes) error { + label := "escape sequence" + if len(what) > 6 { + label = "surrogate pair" + } + needEscape := strings.IndexFunc(string(what), func(r rune) bool { + return r == '`' || r == utf8.RuneError || unicode.IsSpace(r) || !unicode.IsPrint(r) + }) >= 0 + if needEscape { + return errors.New("invalid " + label + " " + strconv.Quote(string(what)) + " in string") + } else { + return errors.New("invalid " + label + " `" + string(what) + "` in string") + } +} + +// TruncatePointer optionally truncates the JSON pointer, +// enforcing that the length roughly does not exceed n. +func TruncatePointer(s string, n int) string { + if len(s) <= n { + return s + } + i := n / 2 + j := len(s) - n/2 + + // Avoid truncating a name if there are multiple names present. + if k := strings.LastIndexByte(s[:i], '/'); k > 0 { + i = k + } + if k := strings.IndexByte(s[j:], '/'); k >= 0 { + j += k + len("/") + } + + // Avoid truncation in the middle of a UTF-8 rune. + for i > 0 && isInvalidUTF8(utf8.DecodeLastRuneInString(s[:i])) { + i-- + } + for j < len(s) && isInvalidUTF8(utf8.DecodeRuneInString(s[j:])) { + j++ + } + + // Determine the right middle fragment to use. + var middle string + switch strings.Count(s[i:j], "/") { + case 0: + middle = "…" + case 1: + middle = "…/…" + default: + middle = "…/…/…" + } + if strings.HasPrefix(s[i:j], "/") && middle != "…" { + middle = strings.TrimPrefix(middle, "…") + } + if strings.HasSuffix(s[i:j], "/") && middle != "…" { + middle = strings.TrimSuffix(middle, "…") + } + return s[:i] + middle + s[j:] +} + +func isInvalidUTF8(r rune, rn int) bool { + return r == utf8.RuneError && rn == 1 +} diff --git a/src/encoding/json/internal/jsonwire/wire_test.go b/src/encoding/json/internal/jsonwire/wire_test.go new file mode 100644 index 00000000000000..a0bf1d1368e686 --- /dev/null +++ b/src/encoding/json/internal/jsonwire/wire_test.go @@ -0,0 +1,98 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsonwire + +import ( + "cmp" + "slices" + "testing" + "unicode/utf16" + "unicode/utf8" +) + +func TestQuoteRune(t *testing.T) { + tests := []struct{ in, want string }{ + {"x", `'x'`}, + {"\n", `'\n'`}, + {"'", `'\''`}, + {"\xff", `'\xff'`}, + {"💩", `'💩'`}, + {"💩"[:1], `'\xf0'`}, + {"\uffff", `'\uffff'`}, + {"\U00101234", `'\U00101234'`}, + } + for _, tt := range tests { + got := QuoteRune([]byte(tt.in)) + if got != tt.want { + t.Errorf("quoteRune(%q) = %s, want %s", tt.in, got, tt.want) + } + } +} + +var compareUTF16Testdata = []string{"", "\r", "1", "f\xfe", "f\xfe\xff", "f\xff", "\u0080", "\u00f6", "\u20ac", "\U0001f600", "\ufb33"} + +func TestCompareUTF16(t *testing.T) { + for i, si := range compareUTF16Testdata { + for j, sj := range compareUTF16Testdata { + got := CompareUTF16([]byte(si), []byte(sj)) + want := cmp.Compare(i, j) + if got != want { + t.Errorf("CompareUTF16(%q, %q) = %v, want %v", si, sj, got, want) + } + } + } +} + +func FuzzCompareUTF16(f *testing.F) { + for _, td1 := range compareUTF16Testdata { + for _, td2 := range compareUTF16Testdata { + f.Add([]byte(td1), []byte(td2)) + } + } + + // CompareUTF16Simple is identical to CompareUTF16, + // but relies on naively converting a string to a []uint16 codepoints. + // It is easy to verify as correct, but is slow. + CompareUTF16Simple := func(x, y []byte) int { + ux := utf16.Encode([]rune(string(x))) + uy := utf16.Encode([]rune(string(y))) + return slices.Compare(ux, uy) + } + + f.Fuzz(func(t *testing.T, s1, s2 []byte) { + // Compare the optimized and simplified implementations. + got := CompareUTF16(s1, s2) + want := CompareUTF16Simple(s1, s2) + if got != want && utf8.Valid(s1) && utf8.Valid(s2) { + t.Errorf("CompareUTF16(%q, %q) = %v, want %v", s1, s2, got, want) + } + }) +} + +func TestTruncatePointer(t *testing.T) { + tests := []struct{ in, want string }{ + {"hello", "hello"}, + {"/a/b/c", "/a/b/c"}, + {"/a/b/c/d/e/f/g", "/a/b/…/f/g"}, + {"supercalifragilisticexpialidocious", "super…cious"}, + {"/supercalifragilisticexpialidocious/supercalifragilisticexpialidocious", "/supe…/…cious"}, + {"/supercalifragilisticexpialidocious/supercalifragilisticexpialidocious/supercalifragilisticexpialidocious", "/supe…/…/…cious"}, + {"/a/supercalifragilisticexpialidocious/supercalifragilisticexpialidocious", "/a/…/…cious"}, + {"/supercalifragilisticexpialidocious/supercalifragilisticexpialidocious/b", "/supe…/…/b"}, + {"/fizz/buzz/bazz", "/fizz/…/bazz"}, + {"/fizz/buzz/bazz/razz", "/fizz/…/razz"}, + {"/////////////////////////////", "/////…/////"}, + {"/🎄❤️✨/🎁✅😊/🎅🔥⭐", "/🎄…/…/…⭐"}, + } + for _, tt := range tests { + got := TruncatePointer(tt.in, 10) + if got != tt.want { + t.Errorf("TruncatePointer(%q) = %q, want %q", tt.in, got, tt.want) + } + } + +} diff --git a/src/encoding/json/jsontext/coder_test.go b/src/encoding/json/jsontext/coder_test.go new file mode 100644 index 00000000000000..8602e3e7fff286 --- /dev/null +++ b/src/encoding/json/jsontext/coder_test.go @@ -0,0 +1,856 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "bytes" + "errors" + "io" + "math" + "math/rand" + "path" + "reflect" + "strings" + "testing" + + "encoding/json/internal/jsontest" + "encoding/json/internal/jsonwire" +) + +func E(err error) *SyntacticError { + return &SyntacticError{Err: err} +} + +func newInvalidCharacterError(prefix, where string) *SyntacticError { + return E(jsonwire.NewInvalidCharacterError(prefix, where)) +} + +func newInvalidEscapeSequenceError(what string) *SyntacticError { + return E(jsonwire.NewInvalidEscapeSequenceError(what)) +} + +func (e *SyntacticError) withPos(prefix string, pointer Pointer) *SyntacticError { + e.ByteOffset = int64(len(prefix)) + e.JSONPointer = pointer + return e +} + +func equalError(x, y error) bool { + return reflect.DeepEqual(x, y) +} + +var ( + zeroToken Token + zeroValue Value +) + +// tokOrVal is either a Token or a Value. +type tokOrVal interface{ Kind() Kind } + +type coderTestdataEntry struct { + name jsontest.CaseName + in string + outCompacted string + outEscaped string // outCompacted if empty; escapes all runes in a string + outIndented string // outCompacted if empty; uses " " for indent prefix and "\t" for indent + outCanonicalized string // outCompacted if empty + tokens []Token + pointers []Pointer +} + +var coderTestdata = []coderTestdataEntry{{ + name: jsontest.Name("Null"), + in: ` null `, + outCompacted: `null`, + tokens: []Token{Null}, + pointers: []Pointer{""}, +}, { + name: jsontest.Name("False"), + in: ` false `, + outCompacted: `false`, + tokens: []Token{False}, +}, { + name: jsontest.Name("True"), + in: ` true `, + outCompacted: `true`, + tokens: []Token{True}, +}, { + name: jsontest.Name("EmptyString"), + in: ` "" `, + outCompacted: `""`, + tokens: []Token{String("")}, +}, { + name: jsontest.Name("SimpleString"), + in: ` "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" `, + outCompacted: `"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"`, + outEscaped: `"\u0061\u0062\u0063\u0064\u0065\u0066\u0067\u0068\u0069\u006a\u006b\u006c\u006d\u006e\u006f\u0070\u0071\u0072\u0073\u0074\u0075\u0076\u0077\u0078\u0079\u007a\u0041\u0042\u0043\u0044\u0045\u0046\u0047\u0048\u0049\u004a\u004b\u004c\u004d\u004e\u004f\u0050\u0051\u0052\u0053\u0054\u0055\u0056\u0057\u0058\u0059\u005a"`, + tokens: []Token{String("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")}, +}, { + name: jsontest.Name("ComplicatedString"), + in: " \"Hello, 世界 🌟★☆✩🌠 " + "\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602" + ` \ud800\udead \"\\\/\b\f\n\r\t \u0022\u005c\u002f\u0008\u000c\u000a\u000d\u0009" `, + outCompacted: "\"Hello, 世界 🌟★☆✩🌠 " + "\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602" + " 𐊭 \\\"\\\\/\\b\\f\\n\\r\\t \\\"\\\\/\\b\\f\\n\\r\\t\"", + outEscaped: `"\u0048\u0065\u006c\u006c\u006f\u002c\u0020\u4e16\u754c\u0020\ud83c\udf1f\u2605\u2606\u2729\ud83c\udf20\u0020\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\ud83d\ude02\u0020\ud800\udead\u0020\u0022\u005c\u002f\u0008\u000c\u000a\u000d\u0009\u0020\u0022\u005c\u002f\u0008\u000c\u000a\u000d\u0009"`, + outCanonicalized: `"Hello, 世界 🌟★☆✩🌠 €ö€힙דּ�😂 𐊭 \"\\/\b\f\n\r\t \"\\/\b\f\n\r\t"`, + tokens: []Token{rawToken("\"Hello, 世界 🌟★☆✩🌠 " + "\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602" + " 𐊭 \\\"\\\\/\\b\\f\\n\\r\\t \\\"\\\\/\\b\\f\\n\\r\\t\"")}, +}, { + name: jsontest.Name("ZeroNumber"), + in: ` 0 `, + outCompacted: `0`, + tokens: []Token{Uint(0)}, +}, { + name: jsontest.Name("SimpleNumber"), + in: ` 123456789 `, + outCompacted: `123456789`, + tokens: []Token{Uint(123456789)}, +}, { + name: jsontest.Name("NegativeNumber"), + in: ` -123456789 `, + outCompacted: `-123456789`, + tokens: []Token{Int(-123456789)}, +}, { + name: jsontest.Name("FractionalNumber"), + in: " 0.123456789 ", + outCompacted: `0.123456789`, + tokens: []Token{Float(0.123456789)}, +}, { + name: jsontest.Name("ExponentNumber"), + in: " 0e12456789 ", + outCompacted: `0e12456789`, + outCanonicalized: `0`, + tokens: []Token{rawToken(`0e12456789`)}, +}, { + name: jsontest.Name("ExponentNumberP"), + in: " 0e+12456789 ", + outCompacted: `0e+12456789`, + outCanonicalized: `0`, + tokens: []Token{rawToken(`0e+12456789`)}, +}, { + name: jsontest.Name("ExponentNumberN"), + in: " 0e-12456789 ", + outCompacted: `0e-12456789`, + outCanonicalized: `0`, + tokens: []Token{rawToken(`0e-12456789`)}, +}, { + name: jsontest.Name("ComplicatedNumber"), + in: ` -123456789.987654321E+0123456789 `, + outCompacted: `-123456789.987654321E+0123456789`, + outCanonicalized: `-1.7976931348623157e+308`, + tokens: []Token{rawToken(`-123456789.987654321E+0123456789`)}, +}, { + name: jsontest.Name("Numbers"), + in: ` [ + 0, -0, 0.0, -0.0, 1.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, 1e1000, + -5e-324, 1e+100, 1.7976931348623157e+308, + 9007199254740990, 9007199254740991, 9007199254740992, 9007199254740993, 9007199254740994, + -9223372036854775808, 9223372036854775807, 0, 18446744073709551615 + ] `, + outCompacted: "[0,-0,0.0,-0.0,1.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001,1e1000,-5e-324,1e+100,1.7976931348623157e+308,9007199254740990,9007199254740991,9007199254740992,9007199254740993,9007199254740994,-9223372036854775808,9223372036854775807,0,18446744073709551615]", + outIndented: `[ + 0, + -0, + 0.0, + -0.0, + 1.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, + 1e1000, + -5e-324, + 1e+100, + 1.7976931348623157e+308, + 9007199254740990, + 9007199254740991, + 9007199254740992, + 9007199254740993, + 9007199254740994, + -9223372036854775808, + 9223372036854775807, + 0, + 18446744073709551615 + ]`, + outCanonicalized: `[0,0,0,0,1,1.7976931348623157e+308,-5e-324,1e+100,1.7976931348623157e+308,9007199254740990,9007199254740991,9007199254740992,9007199254740992,9007199254740994,-9223372036854776000,9223372036854776000,0,18446744073709552000]`, + tokens: []Token{ + BeginArray, + Float(0), Float(math.Copysign(0, -1)), rawToken(`0.0`), rawToken(`-0.0`), rawToken(`1.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001`), rawToken(`1e1000`), + Float(-5e-324), Float(1e100), Float(1.7976931348623157e+308), + Float(9007199254740990), Float(9007199254740991), Float(9007199254740992), rawToken(`9007199254740993`), rawToken(`9007199254740994`), + Int(minInt64), Int(maxInt64), Uint(minUint64), Uint(maxUint64), + EndArray, + }, + pointers: []Pointer{ + "", "/0", "/1", "/2", "/3", "/4", "/5", "/6", "/7", "/8", "/9", "/10", "/11", "/12", "/13", "/14", "/15", "/16", "/17", "", + }, +}, { + name: jsontest.Name("ObjectN0"), + in: ` { } `, + outCompacted: `{}`, + tokens: []Token{BeginObject, EndObject}, + pointers: []Pointer{"", ""}, +}, { + name: jsontest.Name("ObjectN1"), + in: ` { "0" : 0 } `, + outCompacted: `{"0":0}`, + outEscaped: `{"\u0030":0}`, + outIndented: `{ + "0": 0 + }`, + tokens: []Token{BeginObject, String("0"), Uint(0), EndObject}, + pointers: []Pointer{"", "/0", "/0", ""}, +}, { + name: jsontest.Name("ObjectN2"), + in: ` { "0" : 0 , "1" : 1 } `, + outCompacted: `{"0":0,"1":1}`, + outEscaped: `{"\u0030":0,"\u0031":1}`, + outIndented: `{ + "0": 0, + "1": 1 + }`, + tokens: []Token{BeginObject, String("0"), Uint(0), String("1"), Uint(1), EndObject}, + pointers: []Pointer{"", "/0", "/0", "/1", "/1", ""}, +}, { + name: jsontest.Name("ObjectNested"), + in: ` { "0" : { "1" : { "2" : { "3" : { "4" : { } } } } } } `, + outCompacted: `{"0":{"1":{"2":{"3":{"4":{}}}}}}`, + outEscaped: `{"\u0030":{"\u0031":{"\u0032":{"\u0033":{"\u0034":{}}}}}}`, + outIndented: `{ + "0": { + "1": { + "2": { + "3": { + "4": {} + } + } + } + } + }`, + tokens: []Token{BeginObject, String("0"), BeginObject, String("1"), BeginObject, String("2"), BeginObject, String("3"), BeginObject, String("4"), BeginObject, EndObject, EndObject, EndObject, EndObject, EndObject, EndObject}, + pointers: []Pointer{ + "", + "/0", "/0", + "/0/1", "/0/1", + "/0/1/2", "/0/1/2", + "/0/1/2/3", "/0/1/2/3", + "/0/1/2/3/4", "/0/1/2/3/4", + "/0/1/2/3/4", + "/0/1/2/3", + "/0/1/2", + "/0/1", + "/0", + "", + }, +}, { + name: jsontest.Name("ObjectSuperNested"), + in: `{"": { + "44444": { + "6666666": "ccccccc", + "77777777": "bb", + "555555": "aaaa" + }, + "0": { + "3333": "bbb", + "11": "", + "222": "aaaaa" + } + }}`, + outCompacted: `{"":{"44444":{"6666666":"ccccccc","77777777":"bb","555555":"aaaa"},"0":{"3333":"bbb","11":"","222":"aaaaa"}}}`, + outEscaped: `{"":{"\u0034\u0034\u0034\u0034\u0034":{"\u0036\u0036\u0036\u0036\u0036\u0036\u0036":"\u0063\u0063\u0063\u0063\u0063\u0063\u0063","\u0037\u0037\u0037\u0037\u0037\u0037\u0037\u0037":"\u0062\u0062","\u0035\u0035\u0035\u0035\u0035\u0035":"\u0061\u0061\u0061\u0061"},"\u0030":{"\u0033\u0033\u0033\u0033":"\u0062\u0062\u0062","\u0031\u0031":"","\u0032\u0032\u0032":"\u0061\u0061\u0061\u0061\u0061"}}}`, + outIndented: `{ + "": { + "44444": { + "6666666": "ccccccc", + "77777777": "bb", + "555555": "aaaa" + }, + "0": { + "3333": "bbb", + "11": "", + "222": "aaaaa" + } + } + }`, + outCanonicalized: `{"":{"0":{"11":"","222":"aaaaa","3333":"bbb"},"44444":{"555555":"aaaa","6666666":"ccccccc","77777777":"bb"}}}`, + tokens: []Token{ + BeginObject, + String(""), + BeginObject, + String("44444"), + BeginObject, + String("6666666"), String("ccccccc"), + String("77777777"), String("bb"), + String("555555"), String("aaaa"), + EndObject, + String("0"), + BeginObject, + String("3333"), String("bbb"), + String("11"), String(""), + String("222"), String("aaaaa"), + EndObject, + EndObject, + EndObject, + }, + pointers: []Pointer{ + "", + "/", "/", + "//44444", "//44444", + "//44444/6666666", "//44444/6666666", + "//44444/77777777", "//44444/77777777", + "//44444/555555", "//44444/555555", + "//44444", + "//0", "//0", + "//0/3333", "//0/3333", + "//0/11", "//0/11", + "//0/222", "//0/222", + "//0", + "/", + "", + }, +}, { + name: jsontest.Name("ArrayN0"), + in: ` [ ] `, + outCompacted: `[]`, + tokens: []Token{BeginArray, EndArray}, + pointers: []Pointer{"", ""}, +}, { + name: jsontest.Name("ArrayN1"), + in: ` [ 0 ] `, + outCompacted: `[0]`, + outIndented: `[ + 0 + ]`, + tokens: []Token{BeginArray, Uint(0), EndArray}, + pointers: []Pointer{"", "/0", ""}, +}, { + name: jsontest.Name("ArrayN2"), + in: ` [ 0 , 1 ] `, + outCompacted: `[0,1]`, + outIndented: `[ + 0, + 1 + ]`, + tokens: []Token{BeginArray, Uint(0), Uint(1), EndArray}, +}, { + name: jsontest.Name("ArrayNested"), + in: ` [ [ [ [ [ ] ] ] ] ] `, + outCompacted: `[[[[[]]]]]`, + outIndented: `[ + [ + [ + [ + [] + ] + ] + ] + ]`, + tokens: []Token{BeginArray, BeginArray, BeginArray, BeginArray, BeginArray, EndArray, EndArray, EndArray, EndArray, EndArray}, + pointers: []Pointer{ + "", + "/0", + "/0/0", + "/0/0/0", + "/0/0/0/0", + "/0/0/0/0", + "/0/0/0", + "/0/0", + "/0", + "", + }, +}, { + name: jsontest.Name("Everything"), + in: ` { + "literals" : [ null , false , true ], + "string" : "Hello, 世界" , + "number" : 3.14159 , + "arrayN0" : [ ] , + "arrayN1" : [ 0 ] , + "arrayN2" : [ 0 , 1 ] , + "objectN0" : { } , + "objectN1" : { "0" : 0 } , + "objectN2" : { "0" : 0 , "1" : 1 } + } `, + outCompacted: `{"literals":[null,false,true],"string":"Hello, 世界","number":3.14159,"arrayN0":[],"arrayN1":[0],"arrayN2":[0,1],"objectN0":{},"objectN1":{"0":0},"objectN2":{"0":0,"1":1}}`, + outEscaped: `{"\u006c\u0069\u0074\u0065\u0072\u0061\u006c\u0073":[null,false,true],"\u0073\u0074\u0072\u0069\u006e\u0067":"\u0048\u0065\u006c\u006c\u006f\u002c\u0020\u4e16\u754c","\u006e\u0075\u006d\u0062\u0065\u0072":3.14159,"\u0061\u0072\u0072\u0061\u0079\u004e\u0030":[],"\u0061\u0072\u0072\u0061\u0079\u004e\u0031":[0],"\u0061\u0072\u0072\u0061\u0079\u004e\u0032":[0,1],"\u006f\u0062\u006a\u0065\u0063\u0074\u004e\u0030":{},"\u006f\u0062\u006a\u0065\u0063\u0074\u004e\u0031":{"\u0030":0},"\u006f\u0062\u006a\u0065\u0063\u0074\u004e\u0032":{"\u0030":0,"\u0031":1}}`, + outIndented: `{ + "literals": [ + null, + false, + true + ], + "string": "Hello, 世界", + "number": 3.14159, + "arrayN0": [], + "arrayN1": [ + 0 + ], + "arrayN2": [ + 0, + 1 + ], + "objectN0": {}, + "objectN1": { + "0": 0 + }, + "objectN2": { + "0": 0, + "1": 1 + } + }`, + outCanonicalized: `{"arrayN0":[],"arrayN1":[0],"arrayN2":[0,1],"literals":[null,false,true],"number":3.14159,"objectN0":{},"objectN1":{"0":0},"objectN2":{"0":0,"1":1},"string":"Hello, 世界"}`, + tokens: []Token{ + BeginObject, + String("literals"), BeginArray, Null, False, True, EndArray, + String("string"), String("Hello, 世界"), + String("number"), Float(3.14159), + String("arrayN0"), BeginArray, EndArray, + String("arrayN1"), BeginArray, Uint(0), EndArray, + String("arrayN2"), BeginArray, Uint(0), Uint(1), EndArray, + String("objectN0"), BeginObject, EndObject, + String("objectN1"), BeginObject, String("0"), Uint(0), EndObject, + String("objectN2"), BeginObject, String("0"), Uint(0), String("1"), Uint(1), EndObject, + EndObject, + }, + pointers: []Pointer{ + "", + "/literals", "/literals", + "/literals/0", + "/literals/1", + "/literals/2", + "/literals", + "/string", "/string", + "/number", "/number", + "/arrayN0", "/arrayN0", "/arrayN0", + "/arrayN1", "/arrayN1", + "/arrayN1/0", + "/arrayN1", + "/arrayN2", "/arrayN2", + "/arrayN2/0", + "/arrayN2/1", + "/arrayN2", + "/objectN0", "/objectN0", "/objectN0", + "/objectN1", "/objectN1", + "/objectN1/0", "/objectN1/0", + "/objectN1", + "/objectN2", "/objectN2", + "/objectN2/0", "/objectN2/0", + "/objectN2/1", "/objectN2/1", + "/objectN2", + "", + }, +}} + +// TestCoderInterleaved tests that we can interleave calls that operate on +// tokens and raw values. The only error condition is trying to operate on a +// raw value when the next token is an end of object or array. +func TestCoderInterleaved(t *testing.T) { + for _, td := range coderTestdata { + // In TokenFirst and ValueFirst, alternate between tokens and values. + // In TokenDelims, only use tokens for object and array delimiters. + for _, modeName := range []string{"TokenFirst", "ValueFirst", "TokenDelims"} { + t.Run(path.Join(td.name.Name, modeName), func(t *testing.T) { + testCoderInterleaved(t, td.name.Where, modeName, td) + }) + } + } +} +func testCoderInterleaved(t *testing.T, where jsontest.CasePos, modeName string, td coderTestdataEntry) { + src := strings.NewReader(td.in) + dst := new(bytes.Buffer) + dec := NewDecoder(src) + enc := NewEncoder(dst) + tickTock := modeName == "TokenFirst" + for { + if modeName == "TokenDelims" { + switch dec.PeekKind() { + case '{', '}', '[', ']': + tickTock = true // as token + default: + tickTock = false // as value + } + } + if tickTock { + tok, err := dec.ReadToken() + if err != nil { + if err == io.EOF { + break + } + t.Fatalf("%s: Decoder.ReadToken error: %v", where, err) + } + if err := enc.WriteToken(tok); err != nil { + t.Fatalf("%s: Encoder.WriteToken error: %v", where, err) + } + } else { + val, err := dec.ReadValue() + if err != nil { + // It is a syntactic error to call ReadValue + // at the end of an object or array. + // Retry as a ReadToken call. + expectError := dec.PeekKind() == '}' || dec.PeekKind() == ']' + if expectError { + if _, ok := errors.AsType[*SyntacticError](err); !ok { + t.Fatalf("%s: Decoder.ReadToken error is %T, want %T", where, err, new(SyntacticError)) + } + tickTock = !tickTock + continue + } + + if err == io.EOF { + break + } + t.Fatalf("%s: Decoder.ReadValue error: %v", where, err) + } + if err := enc.WriteValue(val); err != nil { + t.Fatalf("%s: Encoder.WriteValue error: %v", where, err) + } + } + tickTock = !tickTock + } + + got := dst.String() + want := td.outCompacted + "\n" + if got != want { + t.Fatalf("%s: output mismatch:\ngot %q\nwant %q", where, got, want) + } +} + +func TestCoderStackPointer(t *testing.T) { + tests := []struct { + token Token + want Pointer + }{ + {Null, ""}, + + {BeginArray, ""}, + {EndArray, ""}, + + {BeginArray, ""}, + {Bool(true), "/0"}, + {EndArray, ""}, + + {BeginArray, ""}, + {String("hello"), "/0"}, + {String("goodbye"), "/1"}, + {EndArray, ""}, + + {BeginObject, ""}, + {EndObject, ""}, + + {BeginObject, ""}, + {String("hello"), "/hello"}, + {String("goodbye"), "/hello"}, + {EndObject, ""}, + + {BeginObject, ""}, + {String(""), "/"}, + {Null, "/"}, + {String("0"), "/0"}, + {Null, "/0"}, + {String("~"), "/~0"}, + {Null, "/~0"}, + {String("/"), "/~1"}, + {Null, "/~1"}, + {String("a//b~/c/~d~~e"), "/a~1~1b~0~1c~1~0d~0~0e"}, + {Null, "/a~1~1b~0~1c~1~0d~0~0e"}, + {String(" \r\n\t"), "/ \r\n\t"}, + {Null, "/ \r\n\t"}, + {EndObject, ""}, + + {BeginArray, ""}, + {BeginObject, "/0"}, + {String(""), "/0/"}, + {BeginArray, "/0/"}, + {BeginObject, "/0//0"}, + {String("#"), "/0//0/#"}, + {Null, "/0//0/#"}, + {EndObject, "/0//0"}, + {EndArray, "/0/"}, + {EndObject, "/0"}, + {EndArray, ""}, + } + + for _, allowDupes := range []bool{false, true} { + var name string + switch allowDupes { + case false: + name = "RejectDuplicateNames" + case true: + name = "AllowDuplicateNames" + } + + t.Run(name, func(t *testing.T) { + bb := new(bytes.Buffer) + + enc := NewEncoder(bb, AllowDuplicateNames(allowDupes)) + for i, tt := range tests { + if err := enc.WriteToken(tt.token); err != nil { + t.Fatalf("%d: Encoder.WriteToken error: %v", i, err) + } + if got := enc.StackPointer(); got != tests[i].want { + t.Fatalf("%d: Encoder.StackPointer = %v, want %v", i, got, tests[i].want) + } + } + + dec := NewDecoder(bb, AllowDuplicateNames(allowDupes)) + for i := range tests { + if _, err := dec.ReadToken(); err != nil { + t.Fatalf("%d: Decoder.ReadToken error: %v", i, err) + } + if got := dec.StackPointer(); got != tests[i].want { + t.Fatalf("%d: Decoder.StackPointer = %v, want %v", i, got, tests[i].want) + } + } + }) + } +} + +func TestCoderMaxDepth(t *testing.T) { + trimArray := func(b []byte) []byte { return b[len(`[`) : len(b)-len(`]`)] } + maxArrays := []byte(strings.Repeat(`[`, maxNestingDepth+1) + strings.Repeat(`]`, maxNestingDepth+1)) + trimObject := func(b []byte) []byte { return b[len(`{"":`) : len(b)-len(`}`)] } + maxObjects := []byte(strings.Repeat(`{"":`, maxNestingDepth+1) + `""` + strings.Repeat(`}`, maxNestingDepth+1)) + + t.Run("Decoder", func(t *testing.T) { + var dec Decoder + checkReadToken := func(t *testing.T, wantKind Kind, wantErr error) { + t.Helper() + if tok, err := dec.ReadToken(); tok.Kind() != wantKind || !equalError(err, wantErr) { + t.Fatalf("Decoder.ReadToken = (%q, %v), want (%q, %v)", byte(tok.Kind()), err, byte(wantKind), wantErr) + } + } + checkReadValue := func(t *testing.T, wantLen int, wantErr error) { + t.Helper() + if val, err := dec.ReadValue(); len(val) != wantLen || !equalError(err, wantErr) { + t.Fatalf("Decoder.ReadValue = (%d, %v), want (%d, %v)", len(val), err, wantLen, wantErr) + } + } + + t.Run("ArraysValid/SingleValue", func(t *testing.T) { + dec.s.reset(trimArray(maxArrays), nil) + checkReadValue(t, maxNestingDepth*len(`[]`), nil) + }) + t.Run("ArraysValid/TokenThenValue", func(t *testing.T) { + dec.s.reset(trimArray(maxArrays), nil) + checkReadToken(t, '[', nil) + checkReadValue(t, (maxNestingDepth-1)*len(`[]`), nil) + checkReadToken(t, ']', nil) + }) + t.Run("ArraysValid/AllTokens", func(t *testing.T) { + dec.s.reset(trimArray(maxArrays), nil) + for range maxNestingDepth { + checkReadToken(t, '[', nil) + } + for range maxNestingDepth { + checkReadToken(t, ']', nil) + } + }) + + wantErr := &SyntacticError{ + ByteOffset: maxNestingDepth, + JSONPointer: Pointer(strings.Repeat("/0", maxNestingDepth)), + Err: errMaxDepth, + } + t.Run("ArraysInvalid/SingleValue", func(t *testing.T) { + dec.s.reset(maxArrays, nil) + checkReadValue(t, 0, wantErr) + }) + t.Run("ArraysInvalid/TokenThenValue", func(t *testing.T) { + dec.s.reset(maxArrays, nil) + checkReadToken(t, '[', nil) + checkReadValue(t, 0, wantErr) + }) + t.Run("ArraysInvalid/AllTokens", func(t *testing.T) { + dec.s.reset(maxArrays, nil) + for range maxNestingDepth { + checkReadToken(t, '[', nil) + } + checkReadValue(t, 0, wantErr) + }) + + t.Run("ObjectsValid/SingleValue", func(t *testing.T) { + dec.s.reset(trimObject(maxObjects), nil) + checkReadValue(t, maxNestingDepth*len(`{"":}`)+len(`""`), nil) + }) + t.Run("ObjectsValid/TokenThenValue", func(t *testing.T) { + dec.s.reset(trimObject(maxObjects), nil) + checkReadToken(t, '{', nil) + checkReadToken(t, '"', nil) + checkReadValue(t, (maxNestingDepth-1)*len(`{"":}`)+len(`""`), nil) + checkReadToken(t, '}', nil) + }) + t.Run("ObjectsValid/AllTokens", func(t *testing.T) { + dec.s.reset(trimObject(maxObjects), nil) + for range maxNestingDepth { + checkReadToken(t, '{', nil) + checkReadToken(t, '"', nil) + } + checkReadToken(t, '"', nil) + for range maxNestingDepth { + checkReadToken(t, '}', nil) + } + }) + + wantErr = &SyntacticError{ + ByteOffset: maxNestingDepth * int64(len(`{"":`)), + JSONPointer: Pointer(strings.Repeat("/", maxNestingDepth)), + Err: errMaxDepth, + } + t.Run("ObjectsInvalid/SingleValue", func(t *testing.T) { + dec.s.reset(maxObjects, nil) + checkReadValue(t, 0, wantErr) + }) + t.Run("ObjectsInvalid/TokenThenValue", func(t *testing.T) { + dec.s.reset(maxObjects, nil) + checkReadToken(t, '{', nil) + checkReadToken(t, '"', nil) + checkReadValue(t, 0, wantErr) + }) + t.Run("ObjectsInvalid/AllTokens", func(t *testing.T) { + dec.s.reset(maxObjects, nil) + for range maxNestingDepth { + checkReadToken(t, '{', nil) + checkReadToken(t, '"', nil) + } + checkReadToken(t, 0, wantErr) + }) + }) + + t.Run("Encoder", func(t *testing.T) { + var enc Encoder + checkWriteToken := func(t *testing.T, tok Token, wantErr error) { + t.Helper() + if err := enc.WriteToken(tok); !equalError(err, wantErr) { + t.Fatalf("Encoder.WriteToken = %v, want %v", err, wantErr) + } + } + checkWriteValue := func(t *testing.T, val Value, wantErr error) { + t.Helper() + if err := enc.WriteValue(val); !equalError(err, wantErr) { + t.Fatalf("Encoder.WriteValue = %v, want %v", err, wantErr) + } + } + + wantErr := &SyntacticError{ + ByteOffset: maxNestingDepth, + JSONPointer: Pointer(strings.Repeat("/0", maxNestingDepth)), + Err: errMaxDepth, + } + t.Run("Arrays/SingleValue", func(t *testing.T) { + enc.s.reset(enc.s.Buf[:0], nil) + checkWriteValue(t, maxArrays, wantErr) + checkWriteValue(t, trimArray(maxArrays), nil) + }) + t.Run("Arrays/TokenThenValue", func(t *testing.T) { + enc.s.reset(enc.s.Buf[:0], nil) + checkWriteToken(t, BeginArray, nil) + checkWriteValue(t, trimArray(maxArrays), wantErr) + checkWriteValue(t, trimArray(trimArray(maxArrays)), nil) + checkWriteToken(t, EndArray, nil) + }) + t.Run("Arrays/AllTokens", func(t *testing.T) { + enc.s.reset(enc.s.Buf[:0], nil) + for range maxNestingDepth { + checkWriteToken(t, BeginArray, nil) + } + checkWriteToken(t, BeginArray, wantErr) + for range maxNestingDepth { + checkWriteToken(t, EndArray, nil) + } + }) + + wantErr = &SyntacticError{ + ByteOffset: maxNestingDepth * int64(len(`{"":`)), + JSONPointer: Pointer(strings.Repeat("/", maxNestingDepth)), + Err: errMaxDepth, + } + t.Run("Objects/SingleValue", func(t *testing.T) { + enc.s.reset(enc.s.Buf[:0], nil) + checkWriteValue(t, maxObjects, wantErr) + checkWriteValue(t, trimObject(maxObjects), nil) + }) + t.Run("Objects/TokenThenValue", func(t *testing.T) { + enc.s.reset(enc.s.Buf[:0], nil) + checkWriteToken(t, BeginObject, nil) + checkWriteToken(t, String(""), nil) + checkWriteValue(t, trimObject(maxObjects), wantErr) + checkWriteValue(t, trimObject(trimObject(maxObjects)), nil) + checkWriteToken(t, EndObject, nil) + }) + t.Run("Objects/AllTokens", func(t *testing.T) { + enc.s.reset(enc.s.Buf[:0], nil) + for range maxNestingDepth - 1 { + checkWriteToken(t, BeginObject, nil) + checkWriteToken(t, String(""), nil) + } + checkWriteToken(t, BeginObject, nil) + checkWriteToken(t, String(""), nil) + checkWriteToken(t, BeginObject, wantErr) + checkWriteToken(t, String(""), nil) + for range maxNestingDepth { + checkWriteToken(t, EndObject, nil) + } + }) + }) +} + +// FaultyBuffer implements io.Reader and io.Writer. +// It may process fewer bytes than the provided buffer +// and may randomly return an error. +type FaultyBuffer struct { + B []byte + + // MaxBytes is the maximum number of bytes read/written. + // A random number of bytes within [0, MaxBytes] are processed. + // A non-positive value is treated as infinity. + MaxBytes int + + // MayError specifies whether to randomly provide this error. + // Even if an error is returned, no bytes are dropped. + MayError error + + // Rand to use for pseudo-random behavior. + // If nil, it will be initialized with rand.NewSource(0). + Rand rand.Source +} + +func (p *FaultyBuffer) Read(b []byte) (int, error) { + b = b[:copy(b[:p.mayTruncate(len(b))], p.B)] + p.B = p.B[len(b):] + if len(p.B) == 0 && (len(b) == 0 || p.randN(2) == 0) { + return len(b), io.EOF + } + return len(b), p.mayError() +} + +func (p *FaultyBuffer) Write(b []byte) (int, error) { + b2 := b[:p.mayTruncate(len(b))] + p.B = append(p.B, b2...) + if len(b2) < len(b) { + return len(b2), io.ErrShortWrite + } + return len(b2), p.mayError() +} + +// mayTruncate may return a value between [0, n]. +func (p *FaultyBuffer) mayTruncate(n int) int { + if p.MaxBytes > 0 { + if n > p.MaxBytes { + n = p.MaxBytes + } + return p.randN(n + 1) + } + return n +} + +// mayError may return a non-nil error. +func (p *FaultyBuffer) mayError() error { + if p.MayError != nil && p.randN(2) == 0 { + return p.MayError + } + return nil +} + +func (p *FaultyBuffer) randN(n int) int { + if p.Rand == nil { + p.Rand = rand.NewSource(0) + } + return int(p.Rand.Int63() % int64(n)) +} diff --git a/src/encoding/json/jsontext/decode.go b/src/encoding/json/jsontext/decode.go new file mode 100644 index 00000000000000..511832f2ae0879 --- /dev/null +++ b/src/encoding/json/jsontext/decode.go @@ -0,0 +1,1185 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "bytes" + "errors" + "io" + + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonopts" + "encoding/json/internal/jsonwire" +) + +// NOTE: The logic for decoding is complicated by the fact that reading from +// an io.Reader into a temporary buffer means that the buffer may contain a +// truncated portion of some valid input, requiring the need to fetch more data. +// +// This file is structured in the following way: +// +// - consumeXXX functions parse an exact JSON token from a []byte. +// If the buffer appears truncated, then it returns io.ErrUnexpectedEOF. +// The consumeSimpleXXX functions are so named because they only handle +// a subset of the grammar for the JSON token being parsed. +// They do not handle the full grammar to keep these functions inlinable. +// +// - Decoder.consumeXXX methods parse the next JSON token from Decoder.buf, +// automatically fetching more input if necessary. These methods take +// a position relative to the start of Decoder.buf as an argument and +// return the end of the consumed JSON token as a position, +// also relative to the start of Decoder.buf. +// +// - In the event of an I/O errors or state machine violations, +// the implementation avoids mutating the state of Decoder +// (aside from the book-keeping needed to implement Decoder.fetch). +// For this reason, only Decoder.ReadToken and Decoder.ReadValue are +// responsible for updated Decoder.prevStart and Decoder.prevEnd. +// +// - For performance, much of the implementation uses the pattern of calling +// the inlinable consumeXXX functions first, and if more work is necessary, +// then it calls the slower Decoder.consumeXXX methods. +// TODO: Revisit this pattern if the Go compiler provides finer control +// over exactly which calls are inlined or not. + +// Decoder is a streaming decoder for raw JSON tokens and values. +// It is used to read a stream of top-level JSON values, +// each separated by optional whitespace characters. +// +// [Decoder.ReadToken] and [Decoder.ReadValue] calls may be interleaved. +// For example, the following JSON value: +// +// {"name":"value","array":[null,false,true,3.14159],"object":{"k":"v"}} +// +// can be parsed with the following calls (ignoring errors for brevity): +// +// d.ReadToken() // { +// d.ReadToken() // "name" +// d.ReadToken() // "value" +// d.ReadValue() // "array" +// d.ReadToken() // [ +// d.ReadToken() // null +// d.ReadToken() // false +// d.ReadValue() // true +// d.ReadToken() // 3.14159 +// d.ReadToken() // ] +// d.ReadValue() // "object" +// d.ReadValue() // {"k":"v"} +// d.ReadToken() // } +// +// The above is one of many possible sequence of calls and +// may not represent the most sensible method to call for any given token/value. +// For example, it is probably more common to call [Decoder.ReadToken] to obtain a +// string token for object names. +type Decoder struct { + s decoderState +} + +// decoderState is the low-level state of Decoder. +// It has exported fields and method for use by the "json" package. +type decoderState struct { + state + decodeBuffer + jsonopts.Struct + + StringCache *[256]string // only used when unmarshaling; identical to json.stringCache +} + +// decodeBuffer is a buffer split into 4 segments: +// +// - buf[0:prevEnd] // already read portion of the buffer +// - buf[prevStart:prevEnd] // previously read value +// - buf[prevEnd:len(buf)] // unread portion of the buffer +// - buf[len(buf):cap(buf)] // unused portion of the buffer +// +// Invariants: +// +// 0 ≤ prevStart ≤ prevEnd ≤ len(buf) ≤ cap(buf) +type decodeBuffer struct { + peekPos int // non-zero if valid offset into buf for start of next token + peekErr error // implies peekPos is -1 + + buf []byte // may alias rd if it is a bytes.Buffer + prevStart int + prevEnd int + + // baseOffset is added to prevStart and prevEnd to obtain + // the absolute offset relative to the start of io.Reader stream. + baseOffset int64 + + rd io.Reader +} + +// NewDecoder constructs a new streaming decoder reading from r. +// +// If r is a [bytes.Buffer], then the decoder parses directly from the buffer +// without first copying the contents to an intermediate buffer. +// Additional writes to the buffer must not occur while the decoder is in use. +func NewDecoder(r io.Reader, opts ...Options) *Decoder { + d := new(Decoder) + d.Reset(r, opts...) + return d +} + +// Reset resets a decoder such that it is reading afresh from r and +// configured with the provided options. Reset must not be called on an +// a Decoder passed to the [encoding/json/v2.UnmarshalerFrom.UnmarshalJSONFrom] method +// or the [encoding/json/v2.UnmarshalFromFunc] function. +func (d *Decoder) Reset(r io.Reader, opts ...Options) { + switch { + case d == nil: + panic("jsontext: invalid nil Decoder") + case r == nil: + panic("jsontext: invalid nil io.Reader") + case d.s.Flags.Get(jsonflags.WithinArshalCall): + panic("jsontext: cannot reset Decoder passed to json.UnmarshalerFrom") + } + // Reuse the buffer if it does not alias a previous [bytes.Buffer]. + b := d.s.buf[:0] + if _, ok := d.s.rd.(*bytes.Buffer); ok { + b = nil + } + d.s.reset(b, r, opts...) +} + +func (d *decoderState) reset(b []byte, r io.Reader, opts ...Options) { + d.state.reset() + d.decodeBuffer = decodeBuffer{buf: b, rd: r} + opts2 := jsonopts.Struct{} // avoid mutating d.Struct in case it is part of opts + opts2.Join(opts...) + d.Struct = opts2 +} + +// Options returns the options used to construct the encoder and +// may additionally contain semantic options passed to a +// [encoding/json/v2.UnmarshalDecode] call. +// +// If operating within +// a [encoding/json/v2.UnmarshalerFrom.UnmarshalJSONFrom] method call or +// a [encoding/json/v2.UnmarshalFromFunc] function call, +// then the returned options are only valid within the call. +func (d *Decoder) Options() Options { + return &d.s.Struct +} + +var errBufferWriteAfterNext = errors.New("invalid bytes.Buffer.Write call after calling bytes.Buffer.Next") + +// fetch reads at least 1 byte from the underlying io.Reader. +// It returns io.ErrUnexpectedEOF if zero bytes were read and io.EOF was seen. +func (d *decoderState) fetch() error { + if d.rd == nil { + return io.ErrUnexpectedEOF + } + + // Inform objectNameStack that we are about to fetch new buffer content. + d.Names.copyQuotedBuffer(d.buf) + + // Specialize bytes.Buffer for better performance. + if bb, ok := d.rd.(*bytes.Buffer); ok { + switch { + case bb.Len() == 0: + return io.ErrUnexpectedEOF + case len(d.buf) == 0: + d.buf = bb.Next(bb.Len()) // "read" all data in the buffer + return nil + default: + // This only occurs if a partially filled bytes.Buffer was provided + // and more data is written to it while Decoder is reading from it. + // This practice will lead to data corruption since future writes + // may overwrite the contents of the current buffer. + // + // The user is trying to use a bytes.Buffer as a pipe, + // but a bytes.Buffer is poor implementation of a pipe, + // the purpose-built io.Pipe should be used instead. + return &ioError{action: "read", err: errBufferWriteAfterNext} + } + } + + // Allocate initial buffer if empty. + if cap(d.buf) == 0 { + d.buf = make([]byte, 0, 64) + } + + // Check whether to grow the buffer. + const maxBufferSize = 4 << 10 + const growthSizeFactor = 2 // higher value is faster + const growthRateFactor = 2 // higher value is slower + // By default, grow if below the maximum buffer size. + grow := cap(d.buf) <= maxBufferSize/growthSizeFactor + // Growing can be expensive, so only grow + // if a sufficient number of bytes have been processed. + grow = grow && int64(cap(d.buf)) < d.previousOffsetEnd()/growthRateFactor + // If prevStart==0, then fetch was called in order to fetch more data + // to finish consuming a large JSON value contiguously. + // Grow if less than 25% of the remaining capacity is available. + // Note that this may cause the input buffer to exceed maxBufferSize. + grow = grow || (d.prevStart == 0 && len(d.buf) >= 3*cap(d.buf)/4) + + if grow { + // Allocate a new buffer and copy the contents of the old buffer over. + // TODO: Provide a hard limit on the maximum internal buffer size? + buf := make([]byte, 0, cap(d.buf)*growthSizeFactor) + d.buf = append(buf, d.buf[d.prevStart:]...) + } else { + // Move unread portion of the data to the front. + n := copy(d.buf[:cap(d.buf)], d.buf[d.prevStart:]) + d.buf = d.buf[:n] + } + d.baseOffset += int64(d.prevStart) + d.prevEnd -= d.prevStart + d.prevStart = 0 + + // Read more data into the internal buffer. + for { + n, err := d.rd.Read(d.buf[len(d.buf):cap(d.buf)]) + switch { + case n > 0: + d.buf = d.buf[:len(d.buf)+n] + return nil // ignore errors if any bytes are read + case err == io.EOF: + return io.ErrUnexpectedEOF + case err != nil: + return &ioError{action: "read", err: err} + default: + continue // Read returned (0, nil) + } + } +} + +const invalidateBufferByte = '#' // invalid starting character for JSON grammar + +// invalidatePreviousRead invalidates buffers returned by Peek and Read calls +// so that the first byte is an invalid character. +// This Hyrum-proofs the API against faulty application code that assumes +// values returned by ReadValue remain valid past subsequent Read calls. +func (d *decodeBuffer) invalidatePreviousRead() { + // Avoid mutating the buffer if d.rd is nil which implies that d.buf + // is provided by the user code and may not expect mutations. + isBytesBuffer := func(r io.Reader) bool { + _, ok := r.(*bytes.Buffer) + return ok + } + if d.rd != nil && !isBytesBuffer(d.rd) && d.prevStart < d.prevEnd && uint(d.prevStart) < uint(len(d.buf)) { + d.buf[d.prevStart] = invalidateBufferByte + d.prevStart = d.prevEnd + } +} + +// needMore reports whether there are no more unread bytes. +func (d *decodeBuffer) needMore(pos int) bool { + // NOTE: The arguments and logic are kept simple to keep this inlinable. + return pos == len(d.buf) +} + +func (d *decodeBuffer) offsetAt(pos int) int64 { return d.baseOffset + int64(pos) } +func (d *decodeBuffer) previousOffsetStart() int64 { return d.baseOffset + int64(d.prevStart) } +func (d *decodeBuffer) previousOffsetEnd() int64 { return d.baseOffset + int64(d.prevEnd) } +func (d *decodeBuffer) previousBuffer() []byte { return d.buf[d.prevStart:d.prevEnd] } +func (d *decodeBuffer) unreadBuffer() []byte { return d.buf[d.prevEnd:len(d.buf)] } + +// PreviousTokenOrValue returns the previously read token or value +// unless it has been invalidated by a call to PeekKind. +// If a token is just a delimiter, then this returns a 1-byte buffer. +// This method is used for error reporting at the semantic layer. +func (d *decodeBuffer) PreviousTokenOrValue() []byte { + b := d.previousBuffer() + // If peek was called, then the previous token or buffer is invalidated. + if d.peekPos > 0 || len(b) > 0 && b[0] == invalidateBufferByte { + return nil + } + // ReadToken does not preserve the buffer for null, bools, or delimiters. + // Manually re-construct that buffer. + if len(b) == 0 { + b = d.buf[:d.prevEnd] // entirety of the previous buffer + for _, tok := range []string{"null", "false", "true", "{", "}", "[", "]"} { + if len(b) >= len(tok) && string(b[len(b)-len(tok):]) == tok { + return b[len(b)-len(tok):] + } + } + } + return b +} + +// PeekKind retrieves the next token kind, but does not advance the read offset. +// +// It returns 0 if an error occurs. Any such error is cached until +// the next read call and it is the caller's responsibility to eventually +// follow up a PeekKind call with a read call. +func (d *Decoder) PeekKind() Kind { + return d.s.PeekKind() +} +func (d *decoderState) PeekKind() Kind { + // Check whether we have a cached peek result. + if d.peekPos > 0 { + return Kind(d.buf[d.peekPos]).normalize() + } + + var err error + d.invalidatePreviousRead() + pos := d.prevEnd + + // Consume leading whitespace. + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + if d.needMore(pos) { + if pos, err = d.consumeWhitespace(pos); err != nil { + if err == io.ErrUnexpectedEOF && d.Tokens.Depth() == 1 { + err = io.EOF // EOF possibly if no Tokens present after top-level value + } + d.peekPos, d.peekErr = -1, wrapSyntacticError(d, err, pos, 0) + return invalidKind + } + } + + // Consume colon or comma. + var delim byte + if c := d.buf[pos]; c == ':' || c == ',' { + delim = c + pos += 1 + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + if d.needMore(pos) { + if pos, err = d.consumeWhitespace(pos); err != nil { + err = wrapSyntacticError(d, err, pos, 0) + d.peekPos, d.peekErr = -1, d.checkDelimBeforeIOError(delim, err) + return invalidKind + } + } + } + next := Kind(d.buf[pos]).normalize() + if d.Tokens.needDelim(next) != delim { + d.peekPos, d.peekErr = -1, d.checkDelim(delim, next) + return invalidKind + } + + // This may set peekPos to zero, which is indistinguishable from + // the uninitialized state. While a small hit to performance, it is correct + // since ReadValue and ReadToken will disregard the cached result and + // recompute the next kind. + d.peekPos, d.peekErr = pos, nil + return next +} + +// checkDelimBeforeIOError checks whether the delim is even valid +// before returning an IO error, which occurs after the delim. +func (d *decoderState) checkDelimBeforeIOError(delim byte, err error) error { + // Since an IO error occurred, we do not know what the next kind is. + // However, knowing the next kind is necessary to validate + // whether the current delim is at least potentially valid. + // Since a JSON string is always valid as the next token, + // conservatively assume that is the next kind for validation. + const next = Kind('"') + if d.Tokens.needDelim(next) != delim { + err = d.checkDelim(delim, next) + } + return err +} + +// CountNextDelimWhitespace counts the number of upcoming bytes of +// delimiter or whitespace characters. +// This method is used for error reporting at the semantic layer. +func (d *decoderState) CountNextDelimWhitespace() int { + d.PeekKind() // populate unreadBuffer + return len(d.unreadBuffer()) - len(bytes.TrimLeft(d.unreadBuffer(), ",: \n\r\t")) +} + +// checkDelim checks whether delim is valid for the given next kind. +func (d *decoderState) checkDelim(delim byte, next Kind) error { + where := "at start of value" + switch d.Tokens.needDelim(next) { + case delim: + return nil + case ':': + where = "after object name (expecting ':')" + case ',': + if d.Tokens.Last.isObject() { + where = "after object value (expecting ',' or '}')" + } else { + where = "after array element (expecting ',' or ']')" + } + } + pos := d.prevEnd // restore position to right after leading whitespace + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + err := jsonwire.NewInvalidCharacterError(d.buf[pos:], where) + return wrapSyntacticError(d, err, pos, 0) +} + +// SkipValue is semantically equivalent to calling [Decoder.ReadValue] and discarding +// the result except that memory is not wasted trying to hold the entire result. +func (d *Decoder) SkipValue() error { + return d.s.SkipValue() +} +func (d *decoderState) SkipValue() error { + switch d.PeekKind() { + case '{', '[': + // For JSON objects and arrays, keep skipping all tokens + // until the depth matches the starting depth. + depth := d.Tokens.Depth() + for { + if _, err := d.ReadToken(); err != nil { + return err + } + if depth >= d.Tokens.Depth() { + return nil + } + } + default: + // Trying to skip a value when the next token is a '}' or ']' + // will result in an error being returned here. + var flags jsonwire.ValueFlags + if _, err := d.ReadValue(&flags); err != nil { + return err + } + return nil + } +} + +// SkipValueRemainder skips the remainder of a value +// after reading a '{' or '[' token. +func (d *decoderState) SkipValueRemainder() error { + if d.Tokens.Depth()-1 > 0 && d.Tokens.Last.Length() == 0 { + for n := d.Tokens.Depth(); d.Tokens.Depth() >= n; { + if _, err := d.ReadToken(); err != nil { + return err + } + } + } + return nil +} + +// SkipUntil skips all tokens until the state machine +// is at or past the specified depth and length. +func (d *decoderState) SkipUntil(depth int, length int64) error { + for d.Tokens.Depth() > depth || (d.Tokens.Depth() == depth && d.Tokens.Last.Length() < length) { + if _, err := d.ReadToken(); err != nil { + return err + } + } + return nil +} + +// ReadToken reads the next [Token], advancing the read offset. +// The returned token is only valid until the next Peek, Read, or Skip call. +// It returns [io.EOF] if there are no more tokens. +func (d *Decoder) ReadToken() (Token, error) { + return d.s.ReadToken() +} +func (d *decoderState) ReadToken() (Token, error) { + // Determine the next kind. + var err error + var next Kind + pos := d.peekPos + if pos != 0 { + // Use cached peek result. + if d.peekErr != nil { + err := d.peekErr + d.peekPos, d.peekErr = 0, nil // possibly a transient I/O error + return Token{}, err + } + next = Kind(d.buf[pos]).normalize() + d.peekPos = 0 // reset cache + } else { + d.invalidatePreviousRead() + pos = d.prevEnd + + // Consume leading whitespace. + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + if d.needMore(pos) { + if pos, err = d.consumeWhitespace(pos); err != nil { + if err == io.ErrUnexpectedEOF && d.Tokens.Depth() == 1 { + err = io.EOF // EOF possibly if no Tokens present after top-level value + } + return Token{}, wrapSyntacticError(d, err, pos, 0) + } + } + + // Consume colon or comma. + var delim byte + if c := d.buf[pos]; c == ':' || c == ',' { + delim = c + pos += 1 + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + if d.needMore(pos) { + if pos, err = d.consumeWhitespace(pos); err != nil { + err = wrapSyntacticError(d, err, pos, 0) + return Token{}, d.checkDelimBeforeIOError(delim, err) + } + } + } + next = Kind(d.buf[pos]).normalize() + if d.Tokens.needDelim(next) != delim { + return Token{}, d.checkDelim(delim, next) + } + } + + // Handle the next token. + var n int + switch next { + case 'n': + if jsonwire.ConsumeNull(d.buf[pos:]) == 0 { + pos, err = d.consumeLiteral(pos, "null") + if err != nil { + return Token{}, wrapSyntacticError(d, err, pos, +1) + } + } else { + pos += len("null") + } + if err = d.Tokens.appendLiteral(); err != nil { + return Token{}, wrapSyntacticError(d, err, pos-len("null"), +1) // report position at start of literal + } + d.prevStart, d.prevEnd = pos, pos + return Null, nil + + case 'f': + if jsonwire.ConsumeFalse(d.buf[pos:]) == 0 { + pos, err = d.consumeLiteral(pos, "false") + if err != nil { + return Token{}, wrapSyntacticError(d, err, pos, +1) + } + } else { + pos += len("false") + } + if err = d.Tokens.appendLiteral(); err != nil { + return Token{}, wrapSyntacticError(d, err, pos-len("false"), +1) // report position at start of literal + } + d.prevStart, d.prevEnd = pos, pos + return False, nil + + case 't': + if jsonwire.ConsumeTrue(d.buf[pos:]) == 0 { + pos, err = d.consumeLiteral(pos, "true") + if err != nil { + return Token{}, wrapSyntacticError(d, err, pos, +1) + } + } else { + pos += len("true") + } + if err = d.Tokens.appendLiteral(); err != nil { + return Token{}, wrapSyntacticError(d, err, pos-len("true"), +1) // report position at start of literal + } + d.prevStart, d.prevEnd = pos, pos + return True, nil + + case '"': + var flags jsonwire.ValueFlags // TODO: Preserve this in Token? + if n = jsonwire.ConsumeSimpleString(d.buf[pos:]); n == 0 { + oldAbsPos := d.baseOffset + int64(pos) + pos, err = d.consumeString(&flags, pos) + newAbsPos := d.baseOffset + int64(pos) + n = int(newAbsPos - oldAbsPos) + if err != nil { + return Token{}, wrapSyntacticError(d, err, pos, +1) + } + } else { + pos += n + } + if d.Tokens.Last.NeedObjectName() { + if !d.Flags.Get(jsonflags.AllowDuplicateNames) { + if !d.Tokens.Last.isValidNamespace() { + return Token{}, wrapSyntacticError(d, errInvalidNamespace, pos-n, +1) + } + if d.Tokens.Last.isActiveNamespace() && !d.Namespaces.Last().insertQuoted(d.buf[pos-n:pos], flags.IsVerbatim()) { + err = wrapWithObjectName(ErrDuplicateName, d.buf[pos-n:pos]) + return Token{}, wrapSyntacticError(d, err, pos-n, +1) // report position at start of string + } + } + d.Names.ReplaceLastQuotedOffset(pos - n) // only replace if insertQuoted succeeds + } + if err = d.Tokens.appendString(); err != nil { + return Token{}, wrapSyntacticError(d, err, pos-n, +1) // report position at start of string + } + d.prevStart, d.prevEnd = pos-n, pos + return Token{raw: &d.decodeBuffer, num: uint64(d.previousOffsetStart())}, nil + + case '0': + // NOTE: Since JSON numbers are not self-terminating, + // we need to make sure that the next byte is not part of a number. + if n = jsonwire.ConsumeSimpleNumber(d.buf[pos:]); n == 0 || d.needMore(pos+n) { + oldAbsPos := d.baseOffset + int64(pos) + pos, err = d.consumeNumber(pos) + newAbsPos := d.baseOffset + int64(pos) + n = int(newAbsPos - oldAbsPos) + if err != nil { + return Token{}, wrapSyntacticError(d, err, pos, +1) + } + } else { + pos += n + } + if err = d.Tokens.appendNumber(); err != nil { + return Token{}, wrapSyntacticError(d, err, pos-n, +1) // report position at start of number + } + d.prevStart, d.prevEnd = pos-n, pos + return Token{raw: &d.decodeBuffer, num: uint64(d.previousOffsetStart())}, nil + + case '{': + if err = d.Tokens.pushObject(); err != nil { + return Token{}, wrapSyntacticError(d, err, pos, +1) + } + d.Names.push() + if !d.Flags.Get(jsonflags.AllowDuplicateNames) { + d.Namespaces.push() + } + pos += 1 + d.prevStart, d.prevEnd = pos, pos + return BeginObject, nil + + case '}': + if err = d.Tokens.popObject(); err != nil { + return Token{}, wrapSyntacticError(d, err, pos, +1) + } + d.Names.pop() + if !d.Flags.Get(jsonflags.AllowDuplicateNames) { + d.Namespaces.pop() + } + pos += 1 + d.prevStart, d.prevEnd = pos, pos + return EndObject, nil + + case '[': + if err = d.Tokens.pushArray(); err != nil { + return Token{}, wrapSyntacticError(d, err, pos, +1) + } + pos += 1 + d.prevStart, d.prevEnd = pos, pos + return BeginArray, nil + + case ']': + if err = d.Tokens.popArray(); err != nil { + return Token{}, wrapSyntacticError(d, err, pos, +1) + } + pos += 1 + d.prevStart, d.prevEnd = pos, pos + return EndArray, nil + + default: + err = jsonwire.NewInvalidCharacterError(d.buf[pos:], "at start of value") + return Token{}, wrapSyntacticError(d, err, pos, +1) + } +} + +// ReadValue returns the next raw JSON value, advancing the read offset. +// The value is stripped of any leading or trailing whitespace and +// contains the exact bytes of the input, which may contain invalid UTF-8 +// if [AllowInvalidUTF8] is specified. +// +// The returned value is only valid until the next Peek, Read, or Skip call and +// may not be mutated while the Decoder remains in use. +// If the decoder is currently at the end token for an object or array, +// then it reports a [SyntacticError] and the internal state remains unchanged. +// It returns [io.EOF] if there are no more values. +func (d *Decoder) ReadValue() (Value, error) { + var flags jsonwire.ValueFlags + return d.s.ReadValue(&flags) +} +func (d *decoderState) ReadValue(flags *jsonwire.ValueFlags) (Value, error) { + // Determine the next kind. + var err error + var next Kind + pos := d.peekPos + if pos != 0 { + // Use cached peek result. + if d.peekErr != nil { + err := d.peekErr + d.peekPos, d.peekErr = 0, nil // possibly a transient I/O error + return nil, err + } + next = Kind(d.buf[pos]).normalize() + d.peekPos = 0 // reset cache + } else { + d.invalidatePreviousRead() + pos = d.prevEnd + + // Consume leading whitespace. + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + if d.needMore(pos) { + if pos, err = d.consumeWhitespace(pos); err != nil { + if err == io.ErrUnexpectedEOF && d.Tokens.Depth() == 1 { + err = io.EOF // EOF possibly if no Tokens present after top-level value + } + return nil, wrapSyntacticError(d, err, pos, 0) + } + } + + // Consume colon or comma. + var delim byte + if c := d.buf[pos]; c == ':' || c == ',' { + delim = c + pos += 1 + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + if d.needMore(pos) { + if pos, err = d.consumeWhitespace(pos); err != nil { + err = wrapSyntacticError(d, err, pos, 0) + return nil, d.checkDelimBeforeIOError(delim, err) + } + } + } + next = Kind(d.buf[pos]).normalize() + if d.Tokens.needDelim(next) != delim { + return nil, d.checkDelim(delim, next) + } + } + + // Handle the next value. + oldAbsPos := d.baseOffset + int64(pos) + pos, err = d.consumeValue(flags, pos, d.Tokens.Depth()) + newAbsPos := d.baseOffset + int64(pos) + n := int(newAbsPos - oldAbsPos) + if err != nil { + return nil, wrapSyntacticError(d, err, pos, +1) + } + switch next { + case 'n', 't', 'f': + err = d.Tokens.appendLiteral() + case '"': + if d.Tokens.Last.NeedObjectName() { + if !d.Flags.Get(jsonflags.AllowDuplicateNames) { + if !d.Tokens.Last.isValidNamespace() { + err = errInvalidNamespace + break + } + if d.Tokens.Last.isActiveNamespace() && !d.Namespaces.Last().insertQuoted(d.buf[pos-n:pos], flags.IsVerbatim()) { + err = wrapWithObjectName(ErrDuplicateName, d.buf[pos-n:pos]) + break + } + } + d.Names.ReplaceLastQuotedOffset(pos - n) // only replace if insertQuoted succeeds + } + err = d.Tokens.appendString() + case '0': + err = d.Tokens.appendNumber() + case '{': + if err = d.Tokens.pushObject(); err != nil { + break + } + if err = d.Tokens.popObject(); err != nil { + panic("BUG: popObject should never fail immediately after pushObject: " + err.Error()) + } + case '[': + if err = d.Tokens.pushArray(); err != nil { + break + } + if err = d.Tokens.popArray(); err != nil { + panic("BUG: popArray should never fail immediately after pushArray: " + err.Error()) + } + } + if err != nil { + return nil, wrapSyntacticError(d, err, pos-n, +1) // report position at start of value + } + d.prevEnd = pos + d.prevStart = pos - n + return d.buf[pos-n : pos : pos], nil +} + +// CheckNextValue checks whether the next value is syntactically valid, +// but does not advance the read offset. +// If last, it verifies that the stream cleanly terminates with [io.EOF]. +func (d *decoderState) CheckNextValue(last bool) error { + d.PeekKind() // populates d.peekPos and d.peekErr + pos, err := d.peekPos, d.peekErr + d.peekPos, d.peekErr = 0, nil + if err != nil { + return err + } + + var flags jsonwire.ValueFlags + if pos, err := d.consumeValue(&flags, pos, d.Tokens.Depth()); err != nil { + return wrapSyntacticError(d, err, pos, +1) + } else if last { + return d.checkEOF(pos) + } + return nil +} + +// AtEOF reports whether the decoder is at EOF. +func (d *decoderState) AtEOF() bool { + _, err := d.consumeWhitespace(d.prevEnd) + return err == io.ErrUnexpectedEOF +} + +// CheckEOF verifies that the input has no more data. +func (d *decoderState) CheckEOF() error { + return d.checkEOF(d.prevEnd) +} +func (d *decoderState) checkEOF(pos int) error { + switch pos, err := d.consumeWhitespace(pos); err { + case nil: + err := jsonwire.NewInvalidCharacterError(d.buf[pos:], "after top-level value") + return wrapSyntacticError(d, err, pos, 0) + case io.ErrUnexpectedEOF: + return nil + default: + return err + } +} + +// consumeWhitespace consumes all whitespace starting at d.buf[pos:]. +// It returns the new position in d.buf immediately after the last whitespace. +// If it returns nil, there is guaranteed to at least be one unread byte. +// +// The following pattern is common in this implementation: +// +// pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) +// if d.needMore(pos) { +// if pos, err = d.consumeWhitespace(pos); err != nil { +// return ... +// } +// } +// +// It is difficult to simplify this without sacrificing performance since +// consumeWhitespace must be inlined. The body of the if statement is +// executed only in rare situations where we need to fetch more data. +// Since fetching may return an error, we also need to check the error. +func (d *decoderState) consumeWhitespace(pos int) (newPos int, err error) { + for { + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + if d.needMore(pos) { + absPos := d.baseOffset + int64(pos) + err = d.fetch() // will mutate d.buf and invalidate pos + pos = int(absPos - d.baseOffset) + if err != nil { + return pos, err + } + continue + } + return pos, nil + } +} + +// consumeValue consumes a single JSON value starting at d.buf[pos:]. +// It returns the new position in d.buf immediately after the value. +func (d *decoderState) consumeValue(flags *jsonwire.ValueFlags, pos, depth int) (newPos int, err error) { + for { + var n int + var err error + switch next := Kind(d.buf[pos]).normalize(); next { + case 'n': + if n = jsonwire.ConsumeNull(d.buf[pos:]); n == 0 { + n, err = jsonwire.ConsumeLiteral(d.buf[pos:], "null") + } + case 'f': + if n = jsonwire.ConsumeFalse(d.buf[pos:]); n == 0 { + n, err = jsonwire.ConsumeLiteral(d.buf[pos:], "false") + } + case 't': + if n = jsonwire.ConsumeTrue(d.buf[pos:]); n == 0 { + n, err = jsonwire.ConsumeLiteral(d.buf[pos:], "true") + } + case '"': + if n = jsonwire.ConsumeSimpleString(d.buf[pos:]); n == 0 { + return d.consumeString(flags, pos) + } + case '0': + // NOTE: Since JSON numbers are not self-terminating, + // we need to make sure that the next byte is not part of a number. + if n = jsonwire.ConsumeSimpleNumber(d.buf[pos:]); n == 0 || d.needMore(pos+n) { + return d.consumeNumber(pos) + } + case '{': + return d.consumeObject(flags, pos, depth) + case '[': + return d.consumeArray(flags, pos, depth) + default: + if (d.Tokens.Last.isObject() && next == ']') || (d.Tokens.Last.isArray() && next == '}') { + return pos, errMismatchDelim + } + return pos, jsonwire.NewInvalidCharacterError(d.buf[pos:], "at start of value") + } + if err == io.ErrUnexpectedEOF { + absPos := d.baseOffset + int64(pos) + err = d.fetch() // will mutate d.buf and invalidate pos + pos = int(absPos - d.baseOffset) + if err != nil { + return pos + n, err + } + continue + } + return pos + n, err + } +} + +// consumeLiteral consumes a single JSON literal starting at d.buf[pos:]. +// It returns the new position in d.buf immediately after the literal. +func (d *decoderState) consumeLiteral(pos int, lit string) (newPos int, err error) { + for { + n, err := jsonwire.ConsumeLiteral(d.buf[pos:], lit) + if err == io.ErrUnexpectedEOF { + absPos := d.baseOffset + int64(pos) + err = d.fetch() // will mutate d.buf and invalidate pos + pos = int(absPos - d.baseOffset) + if err != nil { + return pos + n, err + } + continue + } + return pos + n, err + } +} + +// consumeString consumes a single JSON string starting at d.buf[pos:]. +// It returns the new position in d.buf immediately after the string. +func (d *decoderState) consumeString(flags *jsonwire.ValueFlags, pos int) (newPos int, err error) { + var n int + for { + n, err = jsonwire.ConsumeStringResumable(flags, d.buf[pos:], n, !d.Flags.Get(jsonflags.AllowInvalidUTF8)) + if err == io.ErrUnexpectedEOF { + absPos := d.baseOffset + int64(pos) + err = d.fetch() // will mutate d.buf and invalidate pos + pos = int(absPos - d.baseOffset) + if err != nil { + return pos + n, err + } + continue + } + return pos + n, err + } +} + +// consumeNumber consumes a single JSON number starting at d.buf[pos:]. +// It returns the new position in d.buf immediately after the number. +func (d *decoderState) consumeNumber(pos int) (newPos int, err error) { + var n int + var state jsonwire.ConsumeNumberState + for { + n, state, err = jsonwire.ConsumeNumberResumable(d.buf[pos:], n, state) + // NOTE: Since JSON numbers are not self-terminating, + // we need to make sure that the next byte is not part of a number. + if err == io.ErrUnexpectedEOF || d.needMore(pos+n) { + mayTerminate := err == nil + absPos := d.baseOffset + int64(pos) + err = d.fetch() // will mutate d.buf and invalidate pos + pos = int(absPos - d.baseOffset) + if err != nil { + if mayTerminate && err == io.ErrUnexpectedEOF { + return pos + n, nil + } + return pos, err + } + continue + } + return pos + n, err + } +} + +// consumeObject consumes a single JSON object starting at d.buf[pos:]. +// It returns the new position in d.buf immediately after the object. +func (d *decoderState) consumeObject(flags *jsonwire.ValueFlags, pos, depth int) (newPos int, err error) { + var n int + var names *objectNamespace + if !d.Flags.Get(jsonflags.AllowDuplicateNames) { + d.Namespaces.push() + defer d.Namespaces.pop() + names = d.Namespaces.Last() + } + + // Handle before start. + if uint(pos) >= uint(len(d.buf)) || d.buf[pos] != '{' { + panic("BUG: consumeObject must be called with a buffer that starts with '{'") + } else if depth == maxNestingDepth+1 { + return pos, errMaxDepth + } + pos++ + + // Handle after start. + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + if d.needMore(pos) { + if pos, err = d.consumeWhitespace(pos); err != nil { + return pos, err + } + } + if d.buf[pos] == '}' { + pos++ + return pos, nil + } + + depth++ + for { + // Handle before name. + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + if d.needMore(pos) { + if pos, err = d.consumeWhitespace(pos); err != nil { + return pos, err + } + } + var flags2 jsonwire.ValueFlags + if n = jsonwire.ConsumeSimpleString(d.buf[pos:]); n == 0 { + oldAbsPos := d.baseOffset + int64(pos) + pos, err = d.consumeString(&flags2, pos) + newAbsPos := d.baseOffset + int64(pos) + n = int(newAbsPos - oldAbsPos) + flags.Join(flags2) + if err != nil { + return pos, err + } + } else { + pos += n + } + quotedName := d.buf[pos-n : pos] + if !d.Flags.Get(jsonflags.AllowDuplicateNames) && !names.insertQuoted(quotedName, flags2.IsVerbatim()) { + return pos - n, wrapWithObjectName(ErrDuplicateName, quotedName) + } + + // Handle after name. + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + if d.needMore(pos) { + if pos, err = d.consumeWhitespace(pos); err != nil { + return pos, wrapWithObjectName(err, quotedName) + } + } + if d.buf[pos] != ':' { + err := jsonwire.NewInvalidCharacterError(d.buf[pos:], "after object name (expecting ':')") + return pos, wrapWithObjectName(err, quotedName) + } + pos++ + + // Handle before value. + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + if d.needMore(pos) { + if pos, err = d.consumeWhitespace(pos); err != nil { + return pos, wrapWithObjectName(err, quotedName) + } + } + pos, err = d.consumeValue(flags, pos, depth) + if err != nil { + return pos, wrapWithObjectName(err, quotedName) + } + + // Handle after value. + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + if d.needMore(pos) { + if pos, err = d.consumeWhitespace(pos); err != nil { + return pos, err + } + } + switch d.buf[pos] { + case ',': + pos++ + continue + case '}': + pos++ + return pos, nil + default: + return pos, jsonwire.NewInvalidCharacterError(d.buf[pos:], "after object value (expecting ',' or '}')") + } + } +} + +// consumeArray consumes a single JSON array starting at d.buf[pos:]. +// It returns the new position in d.buf immediately after the array. +func (d *decoderState) consumeArray(flags *jsonwire.ValueFlags, pos, depth int) (newPos int, err error) { + // Handle before start. + if uint(pos) >= uint(len(d.buf)) || d.buf[pos] != '[' { + panic("BUG: consumeArray must be called with a buffer that starts with '['") + } else if depth == maxNestingDepth+1 { + return pos, errMaxDepth + } + pos++ + + // Handle after start. + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + if d.needMore(pos) { + if pos, err = d.consumeWhitespace(pos); err != nil { + return pos, err + } + } + if d.buf[pos] == ']' { + pos++ + return pos, nil + } + + var idx int64 + depth++ + for { + // Handle before value. + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + if d.needMore(pos) { + if pos, err = d.consumeWhitespace(pos); err != nil { + return pos, err + } + } + pos, err = d.consumeValue(flags, pos, depth) + if err != nil { + return pos, wrapWithArrayIndex(err, idx) + } + + // Handle after value. + pos += jsonwire.ConsumeWhitespace(d.buf[pos:]) + if d.needMore(pos) { + if pos, err = d.consumeWhitespace(pos); err != nil { + return pos, err + } + } + switch d.buf[pos] { + case ',': + pos++ + idx++ + continue + case ']': + pos++ + return pos, nil + default: + return pos, jsonwire.NewInvalidCharacterError(d.buf[pos:], "after array element (expecting ',' or ']')") + } + } +} + +// InputOffset returns the current input byte offset. It gives the location +// of the next byte immediately after the most recently returned token or value. +// The number of bytes actually read from the underlying [io.Reader] may be more +// than this offset due to internal buffering effects. +func (d *Decoder) InputOffset() int64 { + return d.s.previousOffsetEnd() +} + +// UnreadBuffer returns the data remaining in the unread buffer, +// which may contain zero or more bytes. +// The returned buffer must not be mutated while Decoder continues to be used. +// The buffer contents are valid until the next Peek, Read, or Skip call. +func (d *Decoder) UnreadBuffer() []byte { + return d.s.unreadBuffer() +} + +// StackDepth returns the depth of the state machine for read JSON data. +// Each level on the stack represents a nested JSON object or array. +// It is incremented whenever an [BeginObject] or [BeginArray] token is encountered +// and decremented whenever an [EndObject] or [EndArray] token is encountered. +// The depth is zero-indexed, where zero represents the top-level JSON value. +func (d *Decoder) StackDepth() int { + // NOTE: Keep in sync with Encoder.StackDepth. + return d.s.Tokens.Depth() - 1 +} + +// StackIndex returns information about the specified stack level. +// It must be a number between 0 and [Decoder.StackDepth], inclusive. +// For each level, it reports the kind: +// +// - 0 for a level of zero, +// - '{' for a level representing a JSON object, and +// - '[' for a level representing a JSON array. +// +// It also reports the length of that JSON object or array. +// Each name and value in a JSON object is counted separately, +// so the effective number of members would be half the length. +// A complete JSON object must have an even length. +func (d *Decoder) StackIndex(i int) (Kind, int64) { + // NOTE: Keep in sync with Encoder.StackIndex. + switch s := d.s.Tokens.index(i); { + case i > 0 && s.isObject(): + return '{', s.Length() + case i > 0 && s.isArray(): + return '[', s.Length() + default: + return 0, s.Length() + } +} + +// StackPointer returns a JSON Pointer (RFC 6901) to the most recently read value. +func (d *Decoder) StackPointer() Pointer { + return Pointer(d.s.AppendStackPointer(nil, -1)) +} + +func (d *decoderState) AppendStackPointer(b []byte, where int) []byte { + d.Names.copyQuotedBuffer(d.buf) + return d.state.appendStackPointer(b, where) +} diff --git a/src/encoding/json/jsontext/decode_test.go b/src/encoding/json/jsontext/decode_test.go new file mode 100644 index 00000000000000..209ff65ec8191b --- /dev/null +++ b/src/encoding/json/jsontext/decode_test.go @@ -0,0 +1,1350 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "bytes" + "errors" + "fmt" + "io" + "net" + "path" + "reflect" + "slices" + "strings" + "testing" + "testing/iotest" + + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsontest" + "encoding/json/internal/jsonwire" +) + +// equalTokens reports whether to sequences of tokens formats the same way. +func equalTokens(xs, ys []Token) bool { + if len(xs) != len(ys) { + return false + } + for i := range xs { + if !(reflect.DeepEqual(xs[i], ys[i]) || xs[i].String() == ys[i].String()) { + return false + } + } + return true +} + +// TestDecoder tests whether we can parse JSON with either tokens or raw values. +func TestDecoder(t *testing.T) { + for _, td := range coderTestdata { + for _, typeName := range []string{"Token", "Value", "TokenDelims"} { + t.Run(path.Join(td.name.Name, typeName), func(t *testing.T) { + testDecoder(t, td.name.Where, typeName, td) + }) + } + } +} +func testDecoder(t *testing.T, where jsontest.CasePos, typeName string, td coderTestdataEntry) { + dec := NewDecoder(bytes.NewBufferString(td.in)) + switch typeName { + case "Token": + var tokens []Token + var pointers []Pointer + for { + tok, err := dec.ReadToken() + if err != nil { + if err == io.EOF { + break + } + t.Fatalf("%s: Decoder.ReadToken error: %v", where, err) + } + tokens = append(tokens, tok.Clone()) + if td.pointers != nil { + pointers = append(pointers, dec.StackPointer()) + } + } + if !equalTokens(tokens, td.tokens) { + t.Fatalf("%s: tokens mismatch:\ngot %v\nwant %v", where, tokens, td.tokens) + } + if !slices.Equal(pointers, td.pointers) { + t.Fatalf("%s: pointers mismatch:\ngot %q\nwant %q", where, pointers, td.pointers) + } + case "Value": + val, err := dec.ReadValue() + if err != nil { + t.Fatalf("%s: Decoder.ReadValue error: %v", where, err) + } + got := string(val) + want := strings.TrimSpace(td.in) + if got != want { + t.Fatalf("%s: Decoder.ReadValue = %s, want %s", where, got, want) + } + case "TokenDelims": + // Use ReadToken for object/array delimiters, ReadValue otherwise. + var tokens []Token + loop: + for { + switch dec.PeekKind() { + case '{', '}', '[', ']': + tok, err := dec.ReadToken() + if err != nil { + if err == io.EOF { + break loop + } + t.Fatalf("%s: Decoder.ReadToken error: %v", where, err) + } + tokens = append(tokens, tok.Clone()) + default: + val, err := dec.ReadValue() + if err != nil { + if err == io.EOF { + break loop + } + t.Fatalf("%s: Decoder.ReadValue error: %v", where, err) + } + tokens = append(tokens, rawToken(string(val))) + } + } + if !equalTokens(tokens, td.tokens) { + t.Fatalf("%s: tokens mismatch:\ngot %v\nwant %v", where, tokens, td.tokens) + } + } +} + +// TestFaultyDecoder tests that temporary I/O errors are not fatal. +func TestFaultyDecoder(t *testing.T) { + for _, td := range coderTestdata { + for _, typeName := range []string{"Token", "Value"} { + t.Run(path.Join(td.name.Name, typeName), func(t *testing.T) { + testFaultyDecoder(t, td.name.Where, typeName, td) + }) + } + } +} +func testFaultyDecoder(t *testing.T, where jsontest.CasePos, typeName string, td coderTestdataEntry) { + b := &FaultyBuffer{ + B: []byte(td.in), + MaxBytes: 1, + MayError: io.ErrNoProgress, + } + + // Read all the tokens. + // If the underlying io.Reader is faulty, then Read may return + // an error without changing the internal state machine. + // In other words, I/O errors occur before syntactic errors. + dec := NewDecoder(b) + switch typeName { + case "Token": + var tokens []Token + for { + tok, err := dec.ReadToken() + if err != nil { + if err == io.EOF { + break + } + if !errors.Is(err, io.ErrNoProgress) { + t.Fatalf("%s: %d: Decoder.ReadToken error: %v", where, len(tokens), err) + } + continue + } + tokens = append(tokens, tok.Clone()) + } + if !equalTokens(tokens, td.tokens) { + t.Fatalf("%s: tokens mismatch:\ngot %s\nwant %s", where, tokens, td.tokens) + } + case "Value": + for { + val, err := dec.ReadValue() + if err != nil { + if err == io.EOF { + break + } + if !errors.Is(err, io.ErrNoProgress) { + t.Fatalf("%s: Decoder.ReadValue error: %v", where, err) + } + continue + } + got := string(val) + want := strings.TrimSpace(td.in) + if got != want { + t.Fatalf("%s: Decoder.ReadValue = %s, want %s", where, got, want) + } + } + } +} + +type decoderMethodCall struct { + wantKind Kind + wantOut tokOrVal + wantErr error + wantPointer Pointer +} + +var decoderErrorTestdata = []struct { + name jsontest.CaseName + opts []Options + in string + calls []decoderMethodCall + wantOffset int +}{{ + name: jsontest.Name("InvalidStart"), + in: ` #`, + calls: []decoderMethodCall{ + {'#', zeroToken, newInvalidCharacterError("#", "at start of value").withPos(" ", ""), ""}, + {'#', zeroValue, newInvalidCharacterError("#", "at start of value").withPos(" ", ""), ""}, + }, +}, { + name: jsontest.Name("StreamN0"), + in: ` `, + calls: []decoderMethodCall{ + {0, zeroToken, io.EOF, ""}, + {0, zeroValue, io.EOF, ""}, + }, +}, { + name: jsontest.Name("StreamN1"), + in: ` null `, + calls: []decoderMethodCall{ + {'n', Null, nil, ""}, + {0, zeroToken, io.EOF, ""}, + {0, zeroValue, io.EOF, ""}, + }, + wantOffset: len(` null`), +}, { + name: jsontest.Name("StreamN2"), + in: ` nullnull `, + calls: []decoderMethodCall{ + {'n', Null, nil, ""}, + {'n', Null, nil, ""}, + {0, zeroToken, io.EOF, ""}, + {0, zeroValue, io.EOF, ""}, + }, + wantOffset: len(` nullnull`), +}, { + name: jsontest.Name("StreamN2/ExtraComma"), // stream is whitespace delimited, not comma delimited + in: ` null , null `, + calls: []decoderMethodCall{ + {'n', Null, nil, ""}, + {0, zeroToken, newInvalidCharacterError(",", `at start of value`).withPos(` null `, ""), ""}, + {0, zeroValue, newInvalidCharacterError(",", `at start of value`).withPos(` null `, ""), ""}, + }, + wantOffset: len(` null`), +}, { + name: jsontest.Name("TruncatedNull"), + in: `nul`, + calls: []decoderMethodCall{ + {'n', zeroToken, E(io.ErrUnexpectedEOF).withPos(`nul`, ""), ""}, + {'n', zeroValue, E(io.ErrUnexpectedEOF).withPos(`nul`, ""), ""}, + }, +}, { + name: jsontest.Name("InvalidNull"), + in: `nulL`, + calls: []decoderMethodCall{ + {'n', zeroToken, newInvalidCharacterError("L", `in literal null (expecting 'l')`).withPos(`nul`, ""), ""}, + {'n', zeroValue, newInvalidCharacterError("L", `in literal null (expecting 'l')`).withPos(`nul`, ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedFalse"), + in: `fals`, + calls: []decoderMethodCall{ + {'f', zeroToken, E(io.ErrUnexpectedEOF).withPos(`fals`, ""), ""}, + {'f', zeroValue, E(io.ErrUnexpectedEOF).withPos(`fals`, ""), ""}, + }, +}, { + name: jsontest.Name("InvalidFalse"), + in: `falsE`, + calls: []decoderMethodCall{ + {'f', zeroToken, newInvalidCharacterError("E", `in literal false (expecting 'e')`).withPos(`fals`, ""), ""}, + {'f', zeroValue, newInvalidCharacterError("E", `in literal false (expecting 'e')`).withPos(`fals`, ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedTrue"), + in: `tru`, + calls: []decoderMethodCall{ + {'t', zeroToken, E(io.ErrUnexpectedEOF).withPos(`tru`, ""), ""}, + {'t', zeroValue, E(io.ErrUnexpectedEOF).withPos(`tru`, ""), ""}, + }, +}, { + name: jsontest.Name("InvalidTrue"), + in: `truE`, + calls: []decoderMethodCall{ + {'t', zeroToken, newInvalidCharacterError("E", `in literal true (expecting 'e')`).withPos(`tru`, ""), ""}, + {'t', zeroValue, newInvalidCharacterError("E", `in literal true (expecting 'e')`).withPos(`tru`, ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedString"), + in: `"start`, + calls: []decoderMethodCall{ + {'"', zeroToken, E(io.ErrUnexpectedEOF).withPos(`"start`, ""), ""}, + {'"', zeroValue, E(io.ErrUnexpectedEOF).withPos(`"start`, ""), ""}, + }, +}, { + name: jsontest.Name("InvalidString"), + in: `"ok` + "\x00", + calls: []decoderMethodCall{ + {'"', zeroToken, newInvalidCharacterError("\x00", `in string (expecting non-control character)`).withPos(`"ok`, ""), ""}, + {'"', zeroValue, newInvalidCharacterError("\x00", `in string (expecting non-control character)`).withPos(`"ok`, ""), ""}, + }, +}, { + name: jsontest.Name("ValidString/AllowInvalidUTF8/Token"), + opts: []Options{AllowInvalidUTF8(true)}, + in: "\"living\xde\xad\xbe\xef\"", + calls: []decoderMethodCall{ + {'"', rawToken("\"living\xde\xad\xbe\xef\""), nil, ""}, + }, + wantOffset: len("\"living\xde\xad\xbe\xef\""), +}, { + name: jsontest.Name("ValidString/AllowInvalidUTF8/Value"), + opts: []Options{AllowInvalidUTF8(true)}, + in: "\"living\xde\xad\xbe\xef\"", + calls: []decoderMethodCall{ + {'"', Value("\"living\xde\xad\xbe\xef\""), nil, ""}, + }, + wantOffset: len("\"living\xde\xad\xbe\xef\""), +}, { + name: jsontest.Name("InvalidString/RejectInvalidUTF8"), + opts: []Options{AllowInvalidUTF8(false)}, + in: "\"living\xde\xad\xbe\xef\"", + calls: []decoderMethodCall{ + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos("\"living\xde\xad", ""), ""}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos("\"living\xde\xad", ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedNumber"), + in: `0.`, + calls: []decoderMethodCall{ + {'0', zeroToken, E(io.ErrUnexpectedEOF), ""}, + {'0', zeroValue, E(io.ErrUnexpectedEOF), ""}, + }, +}, { + name: jsontest.Name("InvalidNumber"), + in: `0.e`, + calls: []decoderMethodCall{ + {'0', zeroToken, newInvalidCharacterError("e", "in number (expecting digit)").withPos(`0.`, ""), ""}, + {'0', zeroValue, newInvalidCharacterError("e", "in number (expecting digit)").withPos(`0.`, ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedObject/AfterStart"), + in: `{`, + calls: []decoderMethodCall{ + {'{', zeroValue, E(io.ErrUnexpectedEOF).withPos("{", ""), ""}, + {'{', BeginObject, nil, ""}, + {0, zeroToken, E(io.ErrUnexpectedEOF).withPos("{", ""), ""}, + {0, zeroValue, E(io.ErrUnexpectedEOF).withPos("{", ""), ""}, + }, + wantOffset: len(`{`), +}, { + name: jsontest.Name("TruncatedObject/AfterName"), + in: `{"0"`, + calls: []decoderMethodCall{ + {'{', zeroValue, E(io.ErrUnexpectedEOF).withPos(`{"0"`, "/0"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("0"), nil, ""}, + {0, zeroToken, E(io.ErrUnexpectedEOF).withPos(`{"0"`, "/0"), ""}, + {0, zeroValue, E(io.ErrUnexpectedEOF).withPos(`{"0"`, "/0"), ""}, + }, + wantOffset: len(`{"0"`), +}, { + name: jsontest.Name("TruncatedObject/AfterColon"), + in: `{"0":`, + calls: []decoderMethodCall{ + {'{', zeroValue, E(io.ErrUnexpectedEOF).withPos(`{"0":`, "/0"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("0"), nil, ""}, + {0, zeroToken, E(io.ErrUnexpectedEOF).withPos(`{"0":`, "/0"), ""}, + {0, zeroValue, E(io.ErrUnexpectedEOF).withPos(`{"0":`, "/0"), ""}, + }, + wantOffset: len(`{"0"`), +}, { + name: jsontest.Name("TruncatedObject/AfterValue"), + in: `{"0":0`, + calls: []decoderMethodCall{ + {'{', zeroValue, E(io.ErrUnexpectedEOF).withPos(`{"0":0`, ""), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("0"), nil, ""}, + {'0', Uint(0), nil, ""}, + {0, zeroToken, E(io.ErrUnexpectedEOF).withPos(`{"0":0`, ""), ""}, + {0, zeroValue, E(io.ErrUnexpectedEOF).withPos(`{"0":0`, ""), ""}, + }, + wantOffset: len(`{"0":0`), +}, { + name: jsontest.Name("TruncatedObject/AfterComma"), + in: `{"0":0,`, + calls: []decoderMethodCall{ + {'{', zeroValue, E(io.ErrUnexpectedEOF).withPos(`{"0":0,`, ""), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("0"), nil, ""}, + {'0', Uint(0), nil, ""}, + {0, zeroToken, E(io.ErrUnexpectedEOF).withPos(`{"0":0,`, ""), ""}, + {0, zeroValue, E(io.ErrUnexpectedEOF).withPos(`{"0":0,`, ""), ""}, + }, + wantOffset: len(`{"0":0`), +}, { + name: jsontest.Name("InvalidObject/MissingColon"), + in: ` { "fizz" "buzz" } `, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError("\"", "after object name (expecting ':')").withPos(` { "fizz" `, "/fizz"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("fizz"), nil, ""}, + {0, zeroToken, newInvalidCharacterError("\"", "after object name (expecting ':')").withPos(` { "fizz" `, "/fizz"), ""}, + {0, zeroValue, newInvalidCharacterError("\"", "after object name (expecting ':')").withPos(` { "fizz" `, "/fizz"), ""}, + }, + wantOffset: len(` { "fizz"`), +}, { + name: jsontest.Name("InvalidObject/MissingColon/GotComma"), + in: ` { "fizz" , "buzz" } `, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError(",", "after object name (expecting ':')").withPos(` { "fizz" `, "/fizz"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("fizz"), nil, ""}, + {0, zeroToken, newInvalidCharacterError(",", "after object name (expecting ':')").withPos(` { "fizz" `, "/fizz"), ""}, + {0, zeroValue, newInvalidCharacterError(",", "after object name (expecting ':')").withPos(` { "fizz" `, "/fizz"), ""}, + }, + wantOffset: len(` { "fizz"`), +}, { + name: jsontest.Name("InvalidObject/MissingColon/GotHash"), + in: ` { "fizz" # "buzz" } `, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError("#", "after object name (expecting ':')").withPos(` { "fizz" `, "/fizz"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("fizz"), nil, ""}, + {0, zeroToken, newInvalidCharacterError("#", "after object name (expecting ':')").withPos(` { "fizz" `, "/fizz"), ""}, + {0, zeroValue, newInvalidCharacterError("#", "after object name (expecting ':')").withPos(` { "fizz" `, "/fizz"), ""}, + }, + wantOffset: len(` { "fizz"`), +}, { + name: jsontest.Name("InvalidObject/MissingComma"), + in: ` { "fizz" : "buzz" "gazz" } `, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError("\"", "after object value (expecting ',' or '}')").withPos(` { "fizz" : "buzz" `, ""), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("fizz"), nil, ""}, + {'"', String("buzz"), nil, ""}, + {0, zeroToken, newInvalidCharacterError("\"", "after object value (expecting ',' or '}')").withPos(` { "fizz" : "buzz" `, ""), ""}, + {0, zeroValue, newInvalidCharacterError("\"", "after object value (expecting ',' or '}')").withPos(` { "fizz" : "buzz" `, ""), ""}, + }, + wantOffset: len(` { "fizz" : "buzz"`), +}, { + name: jsontest.Name("InvalidObject/MissingComma/GotColon"), + in: ` { "fizz" : "buzz" : "gazz" } `, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError(":", "after object value (expecting ',' or '}')").withPos(` { "fizz" : "buzz" `, ""), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("fizz"), nil, ""}, + {'"', String("buzz"), nil, ""}, + {0, zeroToken, newInvalidCharacterError(":", "after object value (expecting ',' or '}')").withPos(` { "fizz" : "buzz" `, ""), ""}, + {0, zeroValue, newInvalidCharacterError(":", "after object value (expecting ',' or '}')").withPos(` { "fizz" : "buzz" `, ""), ""}, + }, + wantOffset: len(` { "fizz" : "buzz"`), +}, { + name: jsontest.Name("InvalidObject/MissingComma/GotHash"), + in: ` { "fizz" : "buzz" # "gazz" } `, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError("#", "after object value (expecting ',' or '}')").withPos(` { "fizz" : "buzz" `, ""), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("fizz"), nil, ""}, + {'"', String("buzz"), nil, ""}, + {0, zeroToken, newInvalidCharacterError("#", "after object value (expecting ',' or '}')").withPos(` { "fizz" : "buzz" `, ""), ""}, + {0, zeroValue, newInvalidCharacterError("#", "after object value (expecting ',' or '}')").withPos(` { "fizz" : "buzz" `, ""), ""}, + }, + wantOffset: len(` { "fizz" : "buzz"`), +}, { + name: jsontest.Name("InvalidObject/ExtraComma/AfterStart"), + in: ` { , } `, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError(",", `at start of string (expecting '"')`).withPos(` { `, ""), ""}, + {'{', BeginObject, nil, ""}, + {0, zeroToken, newInvalidCharacterError(",", `at start of value`).withPos(` { `, ""), ""}, + {0, zeroValue, newInvalidCharacterError(",", `at start of value`).withPos(` { `, ""), ""}, + }, + wantOffset: len(` {`), +}, { + name: jsontest.Name("InvalidObject/ExtraComma/AfterValue"), + in: ` { "fizz" : "buzz" , } `, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError("}", `at start of string (expecting '"')`).withPos(` { "fizz" : "buzz" , `, ""), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("fizz"), nil, ""}, + {'"', String("buzz"), nil, ""}, + {0, zeroToken, newInvalidCharacterError(",", `at start of value`).withPos(` { "fizz" : "buzz" `, ""), ""}, + {0, zeroValue, newInvalidCharacterError(",", `at start of value`).withPos(` { "fizz" : "buzz" `, ""), ""}, + }, + wantOffset: len(` { "fizz" : "buzz"`), +}, { + name: jsontest.Name("InvalidObject/InvalidName/GotNull"), + in: ` { null : null } `, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError("n", "at start of string (expecting '\"')").withPos(` { `, ""), ""}, + {'{', BeginObject, nil, ""}, + {'n', zeroToken, E(ErrNonStringName).withPos(` { `, ""), ""}, + {'n', zeroValue, E(ErrNonStringName).withPos(` { `, ""), ""}, + }, + wantOffset: len(` {`), +}, { + name: jsontest.Name("InvalidObject/InvalidName/GotFalse"), + in: ` { false : false } `, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError("f", "at start of string (expecting '\"')").withPos(` { `, ""), ""}, + {'{', BeginObject, nil, ""}, + {'f', zeroToken, E(ErrNonStringName).withPos(` { `, ""), ""}, + {'f', zeroValue, E(ErrNonStringName).withPos(` { `, ""), ""}, + }, + wantOffset: len(` {`), +}, { + name: jsontest.Name("InvalidObject/InvalidName/GotTrue"), + in: ` { true : true } `, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError("t", "at start of string (expecting '\"')").withPos(` { `, ""), ""}, + {'{', BeginObject, nil, ""}, + {'t', zeroToken, E(ErrNonStringName).withPos(` { `, ""), ""}, + {'t', zeroValue, E(ErrNonStringName).withPos(` { `, ""), ""}, + }, + wantOffset: len(` {`), +}, { + name: jsontest.Name("InvalidObject/InvalidName/GotNumber"), + in: ` { 0 : 0 } `, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError("0", "at start of string (expecting '\"')").withPos(` { `, ""), ""}, + {'{', BeginObject, nil, ""}, + {'0', zeroToken, E(ErrNonStringName).withPos(` { `, ""), ""}, + {'0', zeroValue, E(ErrNonStringName).withPos(` { `, ""), ""}, + }, + wantOffset: len(` {`), +}, { + name: jsontest.Name("InvalidObject/InvalidName/GotObject"), + in: ` { {} : {} } `, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError("{", "at start of string (expecting '\"')").withPos(` { `, ""), ""}, + {'{', BeginObject, nil, ""}, + {'{', zeroToken, E(ErrNonStringName).withPos(` { `, ""), ""}, + {'{', zeroValue, E(ErrNonStringName).withPos(` { `, ""), ""}, + }, + wantOffset: len(` {`), +}, { + name: jsontest.Name("InvalidObject/InvalidName/GotArray"), + in: ` { [] : [] } `, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError("[", "at start of string (expecting '\"')").withPos(` { `, ""), ""}, + {'{', BeginObject, nil, ""}, + {'[', zeroToken, E(ErrNonStringName).withPos(` { `, ""), ""}, + {'[', zeroValue, E(ErrNonStringName).withPos(` { `, ""), ""}, + }, + wantOffset: len(` {`), +}, { + name: jsontest.Name("InvalidObject/MismatchingDelim"), + in: ` { ] `, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError("]", "at start of string (expecting '\"')").withPos(` { `, ""), ""}, + {'{', BeginObject, nil, ""}, + {']', zeroToken, newInvalidCharacterError("]", "at start of value").withPos(` { `, ""), ""}, + {']', zeroValue, newInvalidCharacterError("]", "at start of value").withPos(` { `, ""), ""}, + }, + wantOffset: len(` {`), +}, { + name: jsontest.Name("ValidObject/InvalidValue"), + in: ` { } `, + calls: []decoderMethodCall{ + {'{', BeginObject, nil, ""}, + {'}', zeroValue, newInvalidCharacterError("}", "at start of value").withPos(" { ", ""), ""}, + }, + wantOffset: len(` {`), +}, { + name: jsontest.Name("ValidObject/UniqueNames"), + in: `{"0":0,"1":1} `, + calls: []decoderMethodCall{ + {'{', BeginObject, nil, ""}, + {'"', String("0"), nil, ""}, + {'0', Uint(0), nil, ""}, + {'"', String("1"), nil, ""}, + {'0', Uint(1), nil, ""}, + {'}', EndObject, nil, ""}, + }, + wantOffset: len(`{"0":0,"1":1}`), +}, { + name: jsontest.Name("ValidObject/DuplicateNames"), + opts: []Options{AllowDuplicateNames(true)}, + in: `{"0":0,"0":0} `, + calls: []decoderMethodCall{ + {'{', BeginObject, nil, ""}, + {'"', String("0"), nil, ""}, + {'0', Uint(0), nil, ""}, + {'"', String("0"), nil, ""}, + {'0', Uint(0), nil, ""}, + {'}', EndObject, nil, ""}, + }, + wantOffset: len(`{"0":0,"0":0}`), +}, { + name: jsontest.Name("InvalidObject/DuplicateNames"), + in: `{"X":{},"Y":{},"X":{}} `, + calls: []decoderMethodCall{ + {'{', zeroValue, E(ErrDuplicateName).withPos(`{"X":{},"Y":{},`, "/X"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("X"), nil, ""}, + {'{', BeginObject, nil, ""}, + {'}', EndObject, nil, ""}, + {'"', String("Y"), nil, ""}, + {'{', BeginObject, nil, ""}, + {'}', EndObject, nil, ""}, + {'"', zeroToken, E(ErrDuplicateName).withPos(`{"X":{},"Y":{},`, "/X"), "/Y"}, + {'"', zeroValue, E(ErrDuplicateName).withPos(`{"0":{},"Y":{},`, "/X"), "/Y"}, + }, + wantOffset: len(`{"0":{},"1":{}`), +}, { + name: jsontest.Name("TruncatedArray/AfterStart"), + in: `[`, + calls: []decoderMethodCall{ + {'[', zeroValue, E(io.ErrUnexpectedEOF).withPos("[", ""), ""}, + {'[', BeginArray, nil, ""}, + {0, zeroToken, E(io.ErrUnexpectedEOF).withPos("[", ""), ""}, + {0, zeroValue, E(io.ErrUnexpectedEOF).withPos("[", ""), ""}, + }, + wantOffset: len(`[`), +}, { + name: jsontest.Name("TruncatedArray/AfterValue"), + in: `[0`, + calls: []decoderMethodCall{ + {'[', zeroValue, E(io.ErrUnexpectedEOF).withPos("[0", ""), ""}, + {'[', BeginArray, nil, ""}, + {'0', Uint(0), nil, ""}, + {0, zeroToken, E(io.ErrUnexpectedEOF).withPos("[0", ""), ""}, + {0, zeroValue, E(io.ErrUnexpectedEOF).withPos("[0", ""), ""}, + }, + wantOffset: len(`[0`), +}, { + name: jsontest.Name("TruncatedArray/AfterComma"), + in: `[0,`, + calls: []decoderMethodCall{ + {'[', zeroValue, E(io.ErrUnexpectedEOF).withPos("[0,", ""), ""}, + {'[', BeginArray, nil, ""}, + {'0', Uint(0), nil, ""}, + {0, zeroToken, E(io.ErrUnexpectedEOF).withPos("[0,", ""), ""}, + {0, zeroValue, E(io.ErrUnexpectedEOF).withPos("[0,", ""), ""}, + }, + wantOffset: len(`[0`), +}, { + name: jsontest.Name("InvalidArray/MissingComma"), + in: ` [ "fizz" "buzz" ] `, + calls: []decoderMethodCall{ + {'[', zeroValue, newInvalidCharacterError("\"", "after array element (expecting ',' or ']')").withPos(` [ "fizz" `, ""), ""}, + {'[', BeginArray, nil, ""}, + {'"', String("fizz"), nil, ""}, + {0, zeroToken, newInvalidCharacterError("\"", "after array element (expecting ',' or ']')").withPos(` [ "fizz" `, ""), ""}, + {0, zeroValue, newInvalidCharacterError("\"", "after array element (expecting ',' or ']')").withPos(` [ "fizz" `, ""), ""}, + }, + wantOffset: len(` [ "fizz"`), +}, { + name: jsontest.Name("InvalidArray/MismatchingDelim"), + in: ` [ } `, + calls: []decoderMethodCall{ + {'[', zeroValue, newInvalidCharacterError("}", "at start of value").withPos(` [ `, "/0"), ""}, + {'[', BeginArray, nil, ""}, + {'}', zeroToken, newInvalidCharacterError("}", "at start of value").withPos(` [ `, "/0"), ""}, + {'}', zeroValue, newInvalidCharacterError("}", "at start of value").withPos(` [ `, "/0"), ""}, + }, + wantOffset: len(` [`), +}, { + name: jsontest.Name("ValidArray/InvalidValue"), + in: ` [ ] `, + calls: []decoderMethodCall{ + {'[', BeginArray, nil, ""}, + {']', zeroValue, newInvalidCharacterError("]", "at start of value").withPos(" [ ", "/0"), ""}, + }, + wantOffset: len(` [`), +}, { + name: jsontest.Name("InvalidDelim/AfterTopLevel"), + in: `"",`, + calls: []decoderMethodCall{ + {'"', String(""), nil, ""}, + {0, zeroToken, newInvalidCharacterError(",", "at start of value").withPos(`""`, ""), ""}, + {0, zeroValue, newInvalidCharacterError(",", "at start of value").withPos(`""`, ""), ""}, + }, + wantOffset: len(`""`), +}, { + name: jsontest.Name("InvalidDelim/AfterBeginObject"), + in: `{:`, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError(":", `at start of string (expecting '"')`).withPos(`{`, ""), ""}, + {'{', BeginObject, nil, ""}, + {0, zeroToken, newInvalidCharacterError(":", "at start of value").withPos(`{`, ""), ""}, + {0, zeroValue, newInvalidCharacterError(":", "at start of value").withPos(`{`, ""), ""}, + }, + wantOffset: len(`{`), +}, { + name: jsontest.Name("InvalidDelim/AfterObjectName"), + in: `{"",`, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError(",", "after object name (expecting ':')").withPos(`{""`, "/"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String(""), nil, ""}, + {0, zeroToken, newInvalidCharacterError(",", "after object name (expecting ':')").withPos(`{""`, "/"), ""}, + {0, zeroValue, newInvalidCharacterError(",", "after object name (expecting ':')").withPos(`{""`, "/"), ""}, + }, + wantOffset: len(`{""`), +}, { + name: jsontest.Name("ValidDelim/AfterObjectName"), + in: `{"":`, + calls: []decoderMethodCall{ + {'{', zeroValue, E(io.ErrUnexpectedEOF).withPos(`{"":`, "/"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String(""), nil, ""}, + {0, zeroToken, E(io.ErrUnexpectedEOF).withPos(`{"":`, "/"), ""}, + {0, zeroValue, E(io.ErrUnexpectedEOF).withPos(`{"":`, "/"), ""}, + }, + wantOffset: len(`{""`), +}, { + name: jsontest.Name("InvalidDelim/AfterObjectValue"), + in: `{"":"":`, + calls: []decoderMethodCall{ + {'{', zeroValue, newInvalidCharacterError(":", "after object value (expecting ',' or '}')").withPos(`{"":""`, ""), ""}, + {'{', BeginObject, nil, ""}, + {'"', String(""), nil, ""}, + {'"', String(""), nil, ""}, + {0, zeroToken, newInvalidCharacterError(":", "after object value (expecting ',' or '}')").withPos(`{"":""`, ""), ""}, + {0, zeroValue, newInvalidCharacterError(":", "after object value (expecting ',' or '}')").withPos(`{"":""`, ""), ""}, + }, + wantOffset: len(`{"":""`), +}, { + name: jsontest.Name("ValidDelim/AfterObjectValue"), + in: `{"":"",`, + calls: []decoderMethodCall{ + {'{', zeroValue, E(io.ErrUnexpectedEOF).withPos(`{"":"",`, ""), ""}, + {'{', BeginObject, nil, ""}, + {'"', String(""), nil, ""}, + {'"', String(""), nil, ""}, + {0, zeroToken, E(io.ErrUnexpectedEOF).withPos(`{"":"",`, ""), ""}, + {0, zeroValue, E(io.ErrUnexpectedEOF).withPos(`{"":"",`, ""), ""}, + }, + wantOffset: len(`{"":""`), +}, { + name: jsontest.Name("InvalidDelim/AfterBeginArray"), + in: `[,`, + calls: []decoderMethodCall{ + {'[', zeroValue, newInvalidCharacterError(",", "at start of value").withPos(`[`, "/0"), ""}, + {'[', BeginArray, nil, ""}, + {0, zeroToken, newInvalidCharacterError(",", "at start of value").withPos(`[`, ""), ""}, + {0, zeroValue, newInvalidCharacterError(",", "at start of value").withPos(`[`, ""), ""}, + }, + wantOffset: len(`[`), +}, { + name: jsontest.Name("InvalidDelim/AfterArrayValue"), + in: `["":`, + calls: []decoderMethodCall{ + {'[', zeroValue, newInvalidCharacterError(":", "after array element (expecting ',' or ']')").withPos(`[""`, ""), ""}, + {'[', BeginArray, nil, ""}, + {'"', String(""), nil, ""}, + {0, zeroToken, newInvalidCharacterError(":", "after array element (expecting ',' or ']')").withPos(`[""`, ""), ""}, + {0, zeroValue, newInvalidCharacterError(":", "after array element (expecting ',' or ']')").withPos(`[""`, ""), ""}, + }, + wantOffset: len(`[""`), +}, { + name: jsontest.Name("ValidDelim/AfterArrayValue"), + in: `["",`, + calls: []decoderMethodCall{ + {'[', zeroValue, E(io.ErrUnexpectedEOF).withPos(`["",`, ""), ""}, + {'[', BeginArray, nil, ""}, + {'"', String(""), nil, ""}, + {0, zeroToken, E(io.ErrUnexpectedEOF).withPos(`["",`, ""), ""}, + {0, zeroValue, E(io.ErrUnexpectedEOF).withPos(`["",`, ""), ""}, + }, + wantOffset: len(`[""`), +}, { + name: jsontest.Name("ErrorPosition"), + in: ` "a` + "\xff" + `0" `, + calls: []decoderMethodCall{ + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` "a`, ""), ""}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` "a`, ""), ""}, + }, +}, { + name: jsontest.Name("ErrorPosition/0"), + in: ` [ "a` + "\xff" + `1" ] `, + calls: []decoderMethodCall{ + {'[', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a`, "/0"), ""}, + {'[', BeginArray, nil, ""}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a`, "/0"), ""}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a`, "/0"), ""}, + }, + wantOffset: len(` [`), +}, { + name: jsontest.Name("ErrorPosition/1"), + in: ` [ "a1" , "b` + "\xff" + `1" ] `, + calls: []decoderMethodCall{ + {'[', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , "b`, "/1"), ""}, + {'[', BeginArray, nil, ""}, + {'"', String("a1"), nil, ""}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , "b`, "/1"), ""}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , "b`, "/1"), ""}, + }, + wantOffset: len(` [ "a1"`), +}, { + name: jsontest.Name("ErrorPosition/0/0"), + in: ` [ [ "a` + "\xff" + `2" ] ] `, + calls: []decoderMethodCall{ + {'[', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ [ "a`, "/0/0"), ""}, + {'[', BeginArray, nil, ""}, + {'[', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ [ "a`, "/0/0"), ""}, + {'[', BeginArray, nil, "/0"}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ [ "a`, "/0/0"), ""}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` [ [ "a`, "/0/0"), ""}, + }, + wantOffset: len(` [ [`), +}, { + name: jsontest.Name("ErrorPosition/1/0"), + in: ` [ "a1" , [ "a` + "\xff" + `2" ] ] `, + calls: []decoderMethodCall{ + {'[', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , [ "a`, "/1/0"), ""}, + {'[', BeginArray, nil, ""}, + {'"', String("a1"), nil, "/0"}, + {'[', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , [ "a`, "/1/0"), "/0"}, + {'[', BeginArray, nil, "/1"}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , [ "a`, "/1/0"), "/1"}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , [ "a`, "/1/0"), "/1"}, + }, + wantOffset: len(` [ "a1" , [`), +}, { + name: jsontest.Name("ErrorPosition/0/1"), + in: ` [ [ "a2" , "b` + "\xff" + `2" ] ] `, + calls: []decoderMethodCall{ + {'[', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ [ "a2" , "b`, "/0/1"), ""}, + {'[', BeginArray, nil, ""}, + {'[', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ [ "a2" , "b`, "/0/1"), ""}, + {'[', BeginArray, nil, "/0"}, + {'"', String("a2"), nil, "/0/0"}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ [ "a2" , "b`, "/0/1"), "/0/0"}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` [ [ "a2" , "b`, "/0/1"), "/0/0"}, + }, + wantOffset: len(` [ [ "a2"`), +}, { + name: jsontest.Name("ErrorPosition/1/1"), + in: ` [ "a1" , [ "a2" , "b` + "\xff" + `2" ] ] `, + calls: []decoderMethodCall{ + {'[', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , [ "a2" , "b`, "/1/1"), ""}, + {'[', BeginArray, nil, ""}, + {'"', String("a1"), nil, "/0"}, + {'[', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , [ "a2" , "b`, "/1/1"), ""}, + {'[', BeginArray, nil, "/1"}, + {'"', String("a2"), nil, "/1/0"}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , [ "a2" , "b`, "/1/1"), "/1/0"}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , [ "a2" , "b`, "/1/1"), "/1/0"}, + }, + wantOffset: len(` [ "a1" , [ "a2"`), +}, { + name: jsontest.Name("ErrorPosition/a1-"), + in: ` { "a` + "\xff" + `1" : "b1" } `, + calls: []decoderMethodCall{ + {'{', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a`, ""), ""}, + {'{', BeginObject, nil, ""}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a`, ""), ""}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` { "a`, ""), ""}, + }, + wantOffset: len(` {`), +}, { + name: jsontest.Name("ErrorPosition/a1"), + in: ` { "a1" : "b` + "\xff" + `1" } `, + calls: []decoderMethodCall{ + {'{', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b`, "/a1"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("a1"), nil, "/a1"}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b`, "/a1"), ""}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b`, "/a1"), ""}, + }, + wantOffset: len(` { "a1"`), +}, { + name: jsontest.Name("ErrorPosition/c1-"), + in: ` { "a1" : "b1" , "c` + "\xff" + `1" : "d1" } `, + calls: []decoderMethodCall{ + {'{', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b1" , "c`, ""), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("a1"), nil, "/a1"}, + {'"', String("b1"), nil, "/a1"}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b1" : "c`, ""), "/a1"}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b1" : "c`, ""), "/a1"}, + }, + wantOffset: len(` { "a1" : "b1"`), +}, { + name: jsontest.Name("ErrorPosition/c1"), + in: ` { "a1" : "b1" , "c1" : "d` + "\xff" + `1" } `, + calls: []decoderMethodCall{ + {'{', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b1" , "c1" : "d`, "/c1"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("a1"), nil, "/a1"}, + {'"', String("b1"), nil, "/a1"}, + {'"', String("c1"), nil, "/c1"}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b1" : "c1" : "d`, "/c1"), "/c1"}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b1" : "c1" : "d`, "/c1"), "/c1"}, + }, + wantOffset: len(` { "a1" : "b1" , "c1"`), +}, { + name: jsontest.Name("ErrorPosition/a1/a2-"), + in: ` { "a1" : { "a` + "\xff" + `2" : "b2" } } `, + calls: []decoderMethodCall{ + {'{', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a`, "/a1"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("a1"), nil, "/a1"}, + {'{', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a`, "/a1"), ""}, + {'{', BeginObject, nil, "/a1"}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a`, "/a1"), "/a1"}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a`, "/a1"), "/a1"}, + }, + wantOffset: len(` { "a1" : {`), +}, { + name: jsontest.Name("ErrorPosition/a1/a2"), + in: ` { "a1" : { "a2" : "b` + "\xff" + `2" } } `, + calls: []decoderMethodCall{ + {'{', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a2" : "b`, "/a1/a2"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("a1"), nil, "/a1"}, + {'{', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a2" : "b`, "/a1/a2"), ""}, + {'{', BeginObject, nil, "/a1"}, + {'"', String("a2"), nil, "/a1/a2"}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a2" : "b`, "/a1/a2"), "/a1/a2"}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a2" : "b`, "/a1/a2"), "/a1/a2"}, + }, + wantOffset: len(` { "a1" : { "a2"`), +}, { + name: jsontest.Name("ErrorPosition/a1/c2-"), + in: ` { "a1" : { "a2" : "b2" , "c` + "\xff" + `2" : "d2" } } `, + calls: []decoderMethodCall{ + {'{', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a2" : "b2" , "c`, "/a1"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("a1"), nil, "/a1"}, + {'{', BeginObject, nil, "/a1"}, + {'"', String("a2"), nil, "/a1/a2"}, + {'"', String("b2"), nil, "/a1/a2"}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a2" : "b2" , "c`, "/a1"), "/a1/a2"}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a2" : "b2" , "c`, "/a1"), "/a1/a2"}, + }, + wantOffset: len(` { "a1" : { "a2" : "b2"`), +}, { + name: jsontest.Name("ErrorPosition/a1/c2"), + in: ` { "a1" : { "a2" : "b2" , "c2" : "d` + "\xff" + `2" } } `, + calls: []decoderMethodCall{ + {'{', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a2" : "b2" , "c2" : "d`, "/a1/c2"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("a1"), nil, "/a1"}, + {'{', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a2" : "b2" , "c2" : "d`, "/a1/c2"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("a2"), nil, "/a1/a2"}, + {'"', String("b2"), nil, "/a1/a2"}, + {'"', String("c2"), nil, "/a1/c2"}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a2" : "b2" , "c2" : "d`, "/a1/c2"), "/a1/c2"}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a2" : "b2" , "c2" : "d`, "/a1/c2"), "/a1/c2"}, + }, + wantOffset: len(` { "a1" : { "a2" : "b2" , "c2"`), +}, { + name: jsontest.Name("ErrorPosition/1/a2"), + in: ` [ "a1" , { "a2" : "b` + "\xff" + `2" } ] `, + calls: []decoderMethodCall{ + {'[', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , { "a2" : "b`, "/1/a2"), ""}, + {'[', BeginArray, nil, ""}, + {'"', String("a1"), nil, "/0"}, + {'{', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , { "a2" : "b`, "/1/a2"), ""}, + {'{', BeginObject, nil, "/1"}, + {'"', String("a2"), nil, "/1/a2"}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , { "a2" : "b`, "/1/a2"), "/1/a2"}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , { "a2" : "b`, "/1/a2"), "/1/a2"}, + }, + wantOffset: len(` [ "a1" , { "a2"`), +}, { + name: jsontest.Name("ErrorPosition/c1/1"), + in: ` { "a1" : "b1" , "c1" : [ "a2" , "b` + "\xff" + `2" ] } `, + calls: []decoderMethodCall{ + {'{', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b1" , "c1" : [ "a2" , "b`, "/c1/1"), ""}, + {'{', BeginObject, nil, ""}, + {'"', String("a1"), nil, "/a1"}, + {'"', String("b1"), nil, "/a1"}, + {'"', String("c1"), nil, "/c1"}, + {'[', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b1" , "c1" : [ "a2" , "b`, "/c1/1"), ""}, + {'[', BeginArray, nil, "/c1"}, + {'"', String("a2"), nil, "/c1/0"}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b1" , "c1" : [ "a2" , "b`, "/c1/1"), "/c1/0"}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b1" , "c1" : [ "a2" , "b`, "/c1/1"), "/c1/0"}, + }, + wantOffset: len(` { "a1" : "b1" , "c1" : [ "a2"`), +}, { + name: jsontest.Name("ErrorPosition/0/a1/1/c3/1"), + in: ` [ { "a1" : [ "a2" , { "a3" : "b3" , "c3" : [ "a4" , "b` + "\xff" + `4" ] } ] } ] `, + calls: []decoderMethodCall{ + {'[', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ { "a1" : [ "a2" , { "a3" : "b3" , "c3" : [ "a4" , "b`, "/0/a1/1/c3/1"), ""}, + {'[', BeginArray, nil, ""}, + {'{', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ { "a1" : [ "a2" , { "a3" : "b3" , "c3" : [ "a4" , "b`, "/0/a1/1/c3/1"), ""}, + {'{', BeginObject, nil, "/0"}, + {'"', String("a1"), nil, "/0/a1"}, + {'[', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ { "a1" : [ "a2" , { "a3" : "b3" , "c3" : [ "a4" , "b`, "/0/a1/1/c3/1"), ""}, + {'[', BeginArray, nil, ""}, + {'"', String("a2"), nil, "/0/a1/0"}, + {'{', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ { "a1" : [ "a2" , { "a3" : "b3" , "c3" : [ "a4" , "b`, "/0/a1/1/c3/1"), ""}, + {'{', BeginObject, nil, "/0/a1/1"}, + {'"', String("a3"), nil, "/0/a1/1/a3"}, + {'"', String("b3"), nil, "/0/a1/1/a3"}, + {'"', String("c3"), nil, "/0/a1/1/c3"}, + {'[', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ { "a1" : [ "a2" , { "a3" : "b3" , "c3" : [ "a4" , "b`, "/0/a1/1/c3/1"), ""}, + {'[', BeginArray, nil, "/0/a1/1/c3"}, + {'"', String("a4"), nil, "/0/a1/1/c3/0"}, + {'"', zeroValue, E(jsonwire.ErrInvalidUTF8).withPos(` [ { "a1" : [ "a2" , { "a3" : "b3" , "c3" : [ "a4" , "b`, "/0/a1/1/c3/1"), "/0/a1/1/c3/0"}, + {'"', zeroToken, E(jsonwire.ErrInvalidUTF8).withPos(` [ { "a1" : [ "a2" , { "a3" : "b3" , "c3" : [ "a4" , "b`, "/0/a1/1/c3/1"), "/0/a1/1/c3/0"}, + }, + wantOffset: len(` [ { "a1" : [ "a2" , { "a3" : "b3" , "c3" : [ "a4"`), +}} + +// TestDecoderErrors test that Decoder errors occur when we expect and +// leaves the Decoder in a consistent state. +func TestDecoderErrors(t *testing.T) { + for _, td := range decoderErrorTestdata { + t.Run(path.Join(td.name.Name), func(t *testing.T) { + testDecoderErrors(t, td.name.Where, td.opts, td.in, td.calls, td.wantOffset) + }) + } +} +func testDecoderErrors(t *testing.T, where jsontest.CasePos, opts []Options, in string, calls []decoderMethodCall, wantOffset int) { + src := bytes.NewBufferString(in) + dec := NewDecoder(src, opts...) + for i, call := range calls { + gotKind := dec.PeekKind() + if gotKind != call.wantKind { + t.Fatalf("%s: %d: Decoder.PeekKind = %v, want %v", where, i, gotKind, call.wantKind) + } + + var gotErr error + switch wantOut := call.wantOut.(type) { + case Token: + var gotOut Token + gotOut, gotErr = dec.ReadToken() + if gotOut.String() != wantOut.String() { + t.Fatalf("%s: %d: Decoder.ReadToken = %v, want %v", where, i, gotOut, wantOut) + } + case Value: + var gotOut Value + gotOut, gotErr = dec.ReadValue() + if string(gotOut) != string(wantOut) { + t.Fatalf("%s: %d: Decoder.ReadValue = %s, want %s", where, i, gotOut, wantOut) + } + } + if !equalError(gotErr, call.wantErr) { + t.Fatalf("%s: %d: error mismatch:\ngot %v\nwant %v", where, i, gotErr, call.wantErr) + } + if call.wantPointer != "" { + gotPointer := dec.StackPointer() + if gotPointer != call.wantPointer { + t.Fatalf("%s: %d: Decoder.StackPointer = %s, want %s", where, i, gotPointer, call.wantPointer) + } + } + } + gotOffset := int(dec.InputOffset()) + if gotOffset != wantOffset { + t.Fatalf("%s: Decoder.InputOffset = %v, want %v", where, gotOffset, wantOffset) + } + gotUnread := string(dec.s.unreadBuffer()) // should be a prefix of wantUnread + wantUnread := in[wantOffset:] + if !strings.HasPrefix(wantUnread, gotUnread) { + t.Fatalf("%s: Decoder.UnreadBuffer = %v, want %v", where, gotUnread, wantUnread) + } +} + +// TestBufferDecoder tests that we detect misuses of bytes.Buffer with Decoder. +func TestBufferDecoder(t *testing.T) { + bb := bytes.NewBufferString("[null, false, true]") + dec := NewDecoder(bb) + var err error + for { + if _, err = dec.ReadToken(); err != nil { + break + } + bb.WriteByte(' ') // not allowed to write to the buffer while reading + } + want := &ioError{action: "read", err: errBufferWriteAfterNext} + if !equalError(err, want) { + t.Fatalf("error mismatch: got %v, want %v", err, want) + } +} + +var resumableDecoderTestdata = []string{ + `0`, + `123456789`, + `0.0`, + `0.123456789`, + `0e0`, + `0e+0`, + `0e123456789`, + `0e+123456789`, + `123456789.123456789e+123456789`, + `-0`, + `-123456789`, + `-0.0`, + `-0.123456789`, + `-0e0`, + `-0e-0`, + `-0e123456789`, + `-0e-123456789`, + `-123456789.123456789e-123456789`, + + `""`, + `"a"`, + `"ab"`, + `"abc"`, + `"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"`, + `"\"\\\/\b\f\n\r\t"`, + `"\u0022\u005c\u002f\u0008\u000c\u000a\u000d\u0009"`, + `"\ud800\udead"`, + "\"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\U0001f602\"", + `"\u0080\u00f6\u20ac\ud799\ue000\ufb33\ufffd\ud83d\ude02"`, +} + +// TestResumableDecoder tests that resume logic for parsing a +// JSON string and number properly works across every possible split point. +func TestResumableDecoder(t *testing.T) { + for _, want := range resumableDecoderTestdata { + t.Run("", func(t *testing.T) { + dec := NewDecoder(iotest.OneByteReader(strings.NewReader(want))) + got, err := dec.ReadValue() + if err != nil { + t.Fatalf("Decoder.ReadValue error: %v", err) + } + if string(got) != want { + t.Fatalf("Decoder.ReadValue = %s, want %s", got, want) + } + }) + } +} + +// TestBlockingDecoder verifies that JSON values except numbers can be +// synchronously sent and received on a blocking pipe without a deadlock. +// Numbers are the exception since termination cannot be determined until +// either the pipe ends or a non-numeric character is encountered. +func TestBlockingDecoder(t *testing.T) { + values := []string{"null", "false", "true", `""`, `{}`, `[]`} + + r, w := net.Pipe() + defer r.Close() + defer w.Close() + + enc := NewEncoder(w, jsonflags.OmitTopLevelNewline|1) + dec := NewDecoder(r) + + errCh := make(chan error) + + // Test synchronous ReadToken calls. + for _, want := range values { + go func() { + errCh <- enc.WriteValue(Value(want)) + }() + + tok, err := dec.ReadToken() + if err != nil { + t.Fatalf("Decoder.ReadToken error: %v", err) + } + got := tok.String() + switch tok.Kind() { + case '"': + got = `"` + got + `"` + case '{', '[': + tok, err := dec.ReadToken() + if err != nil { + t.Fatalf("Decoder.ReadToken error: %v", err) + } + got += tok.String() + } + if got != want { + t.Fatalf("ReadTokens = %s, want %s", got, want) + } + + if err := <-errCh; err != nil { + t.Fatalf("Encoder.WriteValue error: %v", err) + } + } + + // Test synchronous ReadValue calls. + for _, want := range values { + go func() { + errCh <- enc.WriteValue(Value(want)) + }() + + got, err := dec.ReadValue() + if err != nil { + t.Fatalf("Decoder.ReadValue error: %v", err) + } + if string(got) != want { + t.Fatalf("ReadValue = %s, want %s", got, want) + } + + if err := <-errCh; err != nil { + t.Fatalf("Encoder.WriteValue error: %v", err) + } + } +} + +func TestPeekableDecoder(t *testing.T) { + type operation any // PeekKind | ReadToken | ReadValue | BufferWrite + type PeekKind struct { + want Kind + } + type ReadToken struct { + wantKind Kind + wantErr error + } + type ReadValue struct { + wantKind Kind + wantErr error + } + type WriteString struct { + in string + } + ops := []operation{ + PeekKind{0}, + WriteString{"[ "}, + ReadToken{0, io.EOF}, // previous error from PeekKind is cached once + ReadToken{'[', nil}, + + PeekKind{0}, + WriteString{"] "}, + ReadValue{0, E(io.ErrUnexpectedEOF).withPos("[ ", "")}, // previous error from PeekKind is cached once + ReadValue{0, newInvalidCharacterError("]", "at start of value").withPos("[ ", "/0")}, + ReadToken{']', nil}, + + WriteString{"[ "}, + ReadToken{'[', nil}, + + WriteString{" null "}, + PeekKind{'n'}, + PeekKind{'n'}, + ReadToken{'n', nil}, + + WriteString{", "}, + PeekKind{0}, + WriteString{"fal"}, + PeekKind{'f'}, + ReadValue{0, E(io.ErrUnexpectedEOF).withPos("[ ] [ null , fal", "/1")}, + WriteString{"se "}, + ReadValue{'f', nil}, + + PeekKind{0}, + WriteString{" , "}, + PeekKind{0}, + WriteString{` "" `}, + ReadValue{0, E(io.ErrUnexpectedEOF).withPos("[ ] [ null , false , ", "")}, // previous error from PeekKind is cached once + ReadValue{'"', nil}, + + WriteString{" , 0"}, + PeekKind{'0'}, + ReadToken{'0', nil}, + + WriteString{" , {} , []"}, + PeekKind{'{'}, + ReadValue{'{', nil}, + ReadValue{'[', nil}, + + WriteString{"]"}, + ReadToken{']', nil}, + } + + bb := struct{ *bytes.Buffer }{new(bytes.Buffer)} + d := NewDecoder(bb) + for i, op := range ops { + switch op := op.(type) { + case PeekKind: + if got := d.PeekKind(); got != op.want { + t.Fatalf("%d: Decoder.PeekKind() = %v, want %v", i, got, op.want) + } + case ReadToken: + gotTok, gotErr := d.ReadToken() + gotKind := gotTok.Kind() + if gotKind != op.wantKind || !equalError(gotErr, op.wantErr) { + t.Fatalf("%d: Decoder.ReadToken() = (%v, %v), want (%v, %v)", i, gotKind, gotErr, op.wantKind, op.wantErr) + } + case ReadValue: + gotVal, gotErr := d.ReadValue() + gotKind := gotVal.Kind() + if gotKind != op.wantKind || !equalError(gotErr, op.wantErr) { + t.Fatalf("%d: Decoder.ReadValue() = (%v, %v), want (%v, %v)", i, gotKind, gotErr, op.wantKind, op.wantErr) + } + case WriteString: + bb.WriteString(op.in) + default: + panic(fmt.Sprintf("unknown operation: %T", op)) + } + } +} + +// TestDecoderReset tests that the decoder preserves its internal +// buffer between Reset calls to avoid frequent allocations when reusing the decoder. +// It ensures that the buffer capacity is maintained while avoiding aliasing +// issues with [bytes.Buffer]. +func TestDecoderReset(t *testing.T) { + // Create a decoder with a reasonably large JSON input to ensure buffer growth. + largeJSON := `{"key1":"value1","key2":"value2","key3":"value3","key4":"value4","key5":"value5"}` + dec := NewDecoder(strings.NewReader(largeJSON)) + + t.Run("Test capacity preservation", func(t *testing.T) { + // Read the first JSON value to grow the internal buffer. + val1, err := dec.ReadValue() + if err != nil { + t.Fatalf("first ReadValue failed: %v", err) + } + if string(val1) != largeJSON { + t.Fatalf("first ReadValue = %q, want %q", val1, largeJSON) + } + + // Get the buffer capacity after first use. + initialCapacity := cap(dec.s.buf) + if initialCapacity == 0 { + t.Fatalf("expected non-zero buffer capacity after first use") + } + + // Reset with a new reader - this should preserve the buffer capacity. + dec.Reset(strings.NewReader(largeJSON)) + + // Verify the buffer capacity is preserved (or at least not smaller). + preservedCapacity := cap(dec.s.buf) + if preservedCapacity < initialCapacity { + t.Fatalf("buffer capacity reduced after Reset: got %d, want at least %d", preservedCapacity, initialCapacity) + } + + // Read the second JSON value to ensure the decoder still works correctly. + val2, err := dec.ReadValue() + if err != nil { + t.Fatalf("second ReadValue failed: %v", err) + } + if string(val2) != largeJSON { + t.Fatalf("second ReadValue = %q, want %q", val2, largeJSON) + } + }) + + var bbBuf []byte + t.Run("Test aliasing with bytes.Buffer", func(t *testing.T) { + // Test with bytes.Buffer to verify proper aliasing behavior. + bb := bytes.NewBufferString(largeJSON) + dec.Reset(bb) + bbBuf = bb.Bytes() + + // Read the third JSON value to ensure functionality with bytes.Buffer. + val3, err := dec.ReadValue() + if err != nil { + t.Fatalf("fourth ReadValue failed: %v", err) + } + if string(val3) != largeJSON { + t.Fatalf("fourth ReadValue = %q, want %q", val3, largeJSON) + } + // The decoder buffer should alias bytes.Buffer's internal buffer. + if len(dec.s.buf) == 0 || len(bbBuf) == 0 || &dec.s.buf[0] != &bbBuf[0] { + t.Fatalf("decoder buffer does not alias bytes.Buffer") + } + }) + + t.Run("Test aliasing removed after Reset", func(t *testing.T) { + // Reset with a new reader and verify the buffer is not aliased. + dec.Reset(strings.NewReader(largeJSON)) + val4, err := dec.ReadValue() + if err != nil { + t.Fatalf("fifth ReadValue failed: %v", err) + } + if string(val4) != largeJSON { + t.Fatalf("fourth ReadValue = %q, want %q", val4, largeJSON) + } + + // The decoder buffer should not alias the bytes.Buffer's internal buffer. + if len(dec.s.buf) == 0 || len(bbBuf) == 0 || &dec.s.buf[0] == &bbBuf[0] { + t.Fatalf("decoder buffer aliases bytes.Buffer") + } + }) +} diff --git a/src/encoding/json/jsontext/doc.go b/src/encoding/json/jsontext/doc.go new file mode 100644 index 00000000000000..d8906926863a86 --- /dev/null +++ b/src/encoding/json/jsontext/doc.go @@ -0,0 +1,116 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +// Package jsontext implements syntactic processing of JSON +// as specified in RFC 4627, RFC 7159, RFC 7493, RFC 8259, and RFC 8785. +// JSON is a simple data interchange format that can represent +// primitive data types such as booleans, strings, and numbers, +// in addition to structured data types such as objects and arrays. +// +// This package (encoding/json/jsontext) is experimental, +// and not subject to the Go 1 compatibility promise. +// It only exists when building with the GOEXPERIMENT=jsonv2 environment variable set. +// Most users should use [encoding/json]. +// +// The [Encoder] and [Decoder] types are used to encode or decode +// a stream of JSON tokens or values. +// +// # Tokens and Values +// +// A JSON token refers to the basic structural elements of JSON: +// +// - a JSON literal (i.e., null, true, or false) +// - a JSON string (e.g., "hello, world!") +// - a JSON number (e.g., 123.456) +// - a begin or end delimiter for a JSON object (i.e., '{' or '}') +// - a begin or end delimiter for a JSON array (i.e., '[' or ']') +// +// A JSON token is represented by the [Token] type in Go. Technically, +// there are two additional structural characters (i.e., ':' and ','), +// but there is no [Token] representation for them since their presence +// can be inferred by the structure of the JSON grammar itself. +// For example, there must always be an implicit colon between +// the name and value of a JSON object member. +// +// A JSON value refers to a complete unit of JSON data: +// +// - a JSON literal, string, or number +// - a JSON object (e.g., `{"name":"value"}`) +// - a JSON array (e.g., `[1,2,3,]`) +// +// A JSON value is represented by the [Value] type in Go and is a []byte +// containing the raw textual representation of the value. There is some overlap +// between tokens and values as both contain literals, strings, and numbers. +// However, only a value can represent the entirety of a JSON object or array. +// +// The [Encoder] and [Decoder] types contain methods to read or write the next +// [Token] or [Value] in a sequence. They maintain a state machine to validate +// whether the sequence of JSON tokens and/or values produces a valid JSON. +// [Options] may be passed to the [NewEncoder] or [NewDecoder] constructors +// to configure the syntactic behavior of encoding and decoding. +// +// # Terminology +// +// The terms "encode" and "decode" are used for syntactic functionality +// that is concerned with processing JSON based on its grammar, and +// the terms "marshal" and "unmarshal" are used for semantic functionality +// that determines the meaning of JSON values as Go values and vice-versa. +// This package (i.e., [jsontext]) deals with JSON at a syntactic layer, +// while [encoding/json/v2] deals with JSON at a semantic layer. +// The goal is to provide a clear distinction between functionality that +// is purely concerned with encoding versus that of marshaling. +// For example, one can directly encode a stream of JSON tokens without +// needing to marshal a concrete Go value representing them. +// Similarly, one can decode a stream of JSON tokens without +// needing to unmarshal them into a concrete Go value. +// +// This package uses JSON terminology when discussing JSON, which may differ +// from related concepts in Go or elsewhere in computing literature. +// +// - a JSON "object" refers to an unordered collection of name/value members. +// - a JSON "array" refers to an ordered sequence of elements. +// - a JSON "value" refers to either a literal (i.e., null, false, or true), +// string, number, object, or array. +// +// See RFC 8259 for more information. +// +// # Specifications +// +// Relevant specifications include RFC 4627, RFC 7159, RFC 7493, RFC 8259, +// and RFC 8785. Each RFC is generally a stricter subset of another RFC. +// In increasing order of strictness: +// +// - RFC 4627 and RFC 7159 do not require (but recommend) the use of UTF-8 +// and also do not require (but recommend) that object names be unique. +// - RFC 8259 requires the use of UTF-8, +// but does not require (but recommends) that object names be unique. +// - RFC 7493 requires the use of UTF-8 +// and also requires that object names be unique. +// - RFC 8785 defines a canonical representation. It requires the use of UTF-8 +// and also requires that object names be unique and in a specific ordering. +// It specifies exactly how strings and numbers must be formatted. +// +// The primary difference between RFC 4627 and RFC 7159 is that the former +// restricted top-level values to only JSON objects and arrays, while +// RFC 7159 and subsequent RFCs permit top-level values to additionally be +// JSON nulls, booleans, strings, or numbers. +// +// By default, this package operates on RFC 7493, but can be configured +// to operate according to the other RFC specifications. +// RFC 7493 is a stricter subset of RFC 8259 and fully compliant with it. +// In particular, it makes specific choices about behavior that RFC 8259 +// leaves as undefined in order to ensure greater interoperability. +// +// # Security Considerations +// +// See the "Security Considerations" section in [encoding/json/v2]. +package jsontext + +// requireKeyedLiterals can be embedded in a struct to require keyed literals. +type requireKeyedLiterals struct{} + +// nonComparable can be embedded in a struct to prevent comparability. +type nonComparable [0]func() diff --git a/src/encoding/json/jsontext/encode.go b/src/encoding/json/jsontext/encode.go new file mode 100644 index 00000000000000..e3b9c04ca615a0 --- /dev/null +++ b/src/encoding/json/jsontext/encode.go @@ -0,0 +1,977 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "bytes" + "io" + "math/bits" + + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonopts" + "encoding/json/internal/jsonwire" +) + +// Encoder is a streaming encoder from raw JSON tokens and values. +// It is used to write a stream of top-level JSON values, +// each terminated with a newline character. +// +// [Encoder.WriteToken] and [Encoder.WriteValue] calls may be interleaved. +// For example, the following JSON value: +// +// {"name":"value","array":[null,false,true,3.14159],"object":{"k":"v"}} +// +// can be composed with the following calls (ignoring errors for brevity): +// +// e.WriteToken(BeginObject) // { +// e.WriteToken(String("name")) // "name" +// e.WriteToken(String("value")) // "value" +// e.WriteValue(Value(`"array"`)) // "array" +// e.WriteToken(BeginArray) // [ +// e.WriteToken(Null) // null +// e.WriteToken(False) // false +// e.WriteValue(Value("true")) // true +// e.WriteToken(Float(3.14159)) // 3.14159 +// e.WriteToken(EndArray) // ] +// e.WriteValue(Value(`"object"`)) // "object" +// e.WriteValue(Value(`{"k":"v"}`)) // {"k":"v"} +// e.WriteToken(EndObject) // } +// +// The above is one of many possible sequence of calls and +// may not represent the most sensible method to call for any given token/value. +// For example, it is probably more common to call [Encoder.WriteToken] with a string +// for object names. +type Encoder struct { + s encoderState +} + +// encoderState is the low-level state of Encoder. +// It has exported fields and method for use by the "json" package. +type encoderState struct { + state + encodeBuffer + jsonopts.Struct + + SeenPointers map[any]struct{} // only used when marshaling; identical to json.seenPointers +} + +// encodeBuffer is a buffer split into 2 segments: +// +// - buf[0:len(buf)] // written (but unflushed) portion of the buffer +// - buf[len(buf):cap(buf)] // unused portion of the buffer +type encodeBuffer struct { + Buf []byte // may alias wr if it is a bytes.Buffer + + // baseOffset is added to len(buf) to obtain the absolute offset + // relative to the start of io.Writer stream. + baseOffset int64 + + wr io.Writer + + // maxValue is the approximate maximum Value size passed to WriteValue. + maxValue int + // availBuffer is the buffer returned by the AvailableBuffer method. + availBuffer []byte // always has zero length + // bufStats is statistics about buffer utilization. + // It is only used with pooled encoders in pools.go. + bufStats bufferStatistics +} + +// NewEncoder constructs a new streaming encoder writing to w +// configured with the provided options. +// It flushes the internal buffer when the buffer is sufficiently full or +// when a top-level value has been written. +// +// If w is a [bytes.Buffer], then the encoder appends directly into the buffer +// without copying the contents from an intermediate buffer. +func NewEncoder(w io.Writer, opts ...Options) *Encoder { + e := new(Encoder) + e.Reset(w, opts...) + return e +} + +// Reset resets an encoder such that it is writing afresh to w and +// configured with the provided options. Reset must not be called on +// a Encoder passed to the [encoding/json/v2.MarshalerTo.MarshalJSONTo] method +// or the [encoding/json/v2.MarshalToFunc] function. +func (e *Encoder) Reset(w io.Writer, opts ...Options) { + switch { + case e == nil: + panic("jsontext: invalid nil Encoder") + case w == nil: + panic("jsontext: invalid nil io.Writer") + case e.s.Flags.Get(jsonflags.WithinArshalCall): + panic("jsontext: cannot reset Encoder passed to json.MarshalerTo") + } + // Reuse the buffer if it does not alias a previous [bytes.Buffer]. + b := e.s.Buf[:0] + if _, ok := e.s.wr.(*bytes.Buffer); ok { + b = nil + } + e.s.reset(b, w, opts...) +} + +func (e *encoderState) reset(b []byte, w io.Writer, opts ...Options) { + e.state.reset() + e.encodeBuffer = encodeBuffer{Buf: b, wr: w, availBuffer: e.availBuffer, bufStats: e.bufStats} + if bb, ok := w.(*bytes.Buffer); ok && bb != nil { + e.Buf = bb.AvailableBuffer() // alias the unused buffer of bb + } + opts2 := jsonopts.Struct{} // avoid mutating e.Struct in case it is part of opts + opts2.Join(opts...) + e.Struct = opts2 + if e.Flags.Get(jsonflags.Multiline) { + if !e.Flags.Has(jsonflags.SpaceAfterColon) { + e.Flags.Set(jsonflags.SpaceAfterColon | 1) + } + if !e.Flags.Has(jsonflags.SpaceAfterComma) { + e.Flags.Set(jsonflags.SpaceAfterComma | 0) + } + if !e.Flags.Has(jsonflags.Indent) { + e.Flags.Set(jsonflags.Indent | 1) + e.Indent = "\t" + } + } +} + +// Options returns the options used to construct the decoder and +// may additionally contain semantic options passed to a +// [encoding/json/v2.MarshalEncode] call. +// +// If operating within +// a [encoding/json/v2.MarshalerTo.MarshalJSONTo] method call or +// a [encoding/json/v2.MarshalToFunc] function call, +// then the returned options are only valid within the call. +func (e *Encoder) Options() Options { + return &e.s.Struct +} + +// NeedFlush determines whether to flush at this point. +func (e *encoderState) NeedFlush() bool { + // NOTE: This function is carefully written to be inlinable. + + // Avoid flushing if e.wr is nil since there is no underlying writer. + // Flush if less than 25% of the capacity remains. + // Flushing at some constant fraction ensures that the buffer stops growing + // so long as the largest Token or Value fits within that unused capacity. + return e.wr != nil && (e.Tokens.Depth() == 1 || len(e.Buf) > 3*cap(e.Buf)/4) +} + +// Flush flushes the buffer to the underlying io.Writer. +// It may append a trailing newline after the top-level value. +func (e *encoderState) Flush() error { + if e.wr == nil || e.avoidFlush() { + return nil + } + + // In streaming mode, always emit a newline after the top-level value. + if e.Tokens.Depth() == 1 && !e.Flags.Get(jsonflags.OmitTopLevelNewline) { + e.Buf = append(e.Buf, '\n') + } + + // Inform objectNameStack that we are about to flush the buffer content. + e.Names.copyQuotedBuffer(e.Buf) + + // Specialize bytes.Buffer for better performance. + if bb, ok := e.wr.(*bytes.Buffer); ok { + // If e.buf already aliases the internal buffer of bb, + // then the Write call simply increments the internal offset, + // otherwise Write operates as expected. + // See https://go.dev/issue/42986. + n, _ := bb.Write(e.Buf) // never fails unless bb is nil + e.baseOffset += int64(n) + + // If the internal buffer of bytes.Buffer is too small, + // append operations elsewhere in the Encoder may grow the buffer. + // This would be semantically correct, but hurts performance. + // As such, ensure 25% of the current length is always available + // to reduce the probability that other appends must allocate. + if avail := bb.Available(); avail < bb.Len()/4 { + bb.Grow(avail + 1) + } + + e.Buf = bb.AvailableBuffer() + return nil + } + + // Flush the internal buffer to the underlying io.Writer. + n, err := e.wr.Write(e.Buf) + e.baseOffset += int64(n) + if err != nil { + // In the event of an error, preserve the unflushed portion. + // Thus, write errors aren't fatal so long as the io.Writer + // maintains consistent state after errors. + if n > 0 { + e.Buf = e.Buf[:copy(e.Buf, e.Buf[n:])] + } + return &ioError{action: "write", err: err} + } + e.Buf = e.Buf[:0] + + // Check whether to grow the buffer. + // Note that cap(e.buf) may already exceed maxBufferSize since + // an append elsewhere already grew it to store a large token. + const maxBufferSize = 4 << 10 + const growthSizeFactor = 2 // higher value is faster + const growthRateFactor = 2 // higher value is slower + // By default, grow if below the maximum buffer size. + grow := cap(e.Buf) <= maxBufferSize/growthSizeFactor + // Growing can be expensive, so only grow + // if a sufficient number of bytes have been processed. + grow = grow && int64(cap(e.Buf)) < e.previousOffsetEnd()/growthRateFactor + if grow { + e.Buf = make([]byte, 0, cap(e.Buf)*growthSizeFactor) + } + + return nil +} +func (d *encodeBuffer) offsetAt(pos int) int64 { return d.baseOffset + int64(pos) } +func (e *encodeBuffer) previousOffsetEnd() int64 { return e.baseOffset + int64(len(e.Buf)) } +func (e *encodeBuffer) unflushedBuffer() []byte { return e.Buf } + +// avoidFlush indicates whether to avoid flushing to ensure there is always +// enough in the buffer to unwrite the last object member if it were empty. +func (e *encoderState) avoidFlush() bool { + switch { + case e.Tokens.Last.Length() == 0: + // Never flush after BeginObject or BeginArray since we don't know yet + // if the object or array will end up being empty. + return true + case e.Tokens.Last.needObjectValue(): + // Never flush before the object value since we don't know yet + // if the object value will end up being empty. + return true + case e.Tokens.Last.NeedObjectName() && len(e.Buf) >= 2: + // Never flush after the object value if it does turn out to be empty. + switch string(e.Buf[len(e.Buf)-2:]) { + case `ll`, `""`, `{}`, `[]`: // last two bytes of every empty value + return true + } + } + return false +} + +// UnwriteEmptyObjectMember unwrites the last object member if it is empty +// and reports whether it performed an unwrite operation. +func (e *encoderState) UnwriteEmptyObjectMember(prevName *string) bool { + if last := e.Tokens.Last; !last.isObject() || !last.NeedObjectName() || last.Length() == 0 { + panic("BUG: must be called on an object after writing a value") + } + + // The flushing logic is modified to never flush a trailing empty value. + // The encoder never writes trailing whitespace eagerly. + b := e.unflushedBuffer() + + // Detect whether the last value was empty. + var n int + if len(b) >= 3 { + switch string(b[len(b)-2:]) { + case "ll": // last two bytes of `null` + n = len(`null`) + case `""`: + // It is possible for a non-empty string to have `""` as a suffix + // if the second to the last quote was escaped. + if b[len(b)-3] == '\\' { + return false // e.g., `"\""` is not empty + } + n = len(`""`) + case `{}`: + n = len(`{}`) + case `[]`: + n = len(`[]`) + } + } + if n == 0 { + return false + } + + // Unwrite the value, whitespace, colon, name, whitespace, and comma. + b = b[:len(b)-n] + b = jsonwire.TrimSuffixWhitespace(b) + b = jsonwire.TrimSuffixByte(b, ':') + b = jsonwire.TrimSuffixString(b) + b = jsonwire.TrimSuffixWhitespace(b) + b = jsonwire.TrimSuffixByte(b, ',') + e.Buf = b // store back truncated unflushed buffer + + // Undo state changes. + e.Tokens.Last.decrement() // for object member value + e.Tokens.Last.decrement() // for object member name + if !e.Flags.Get(jsonflags.AllowDuplicateNames) { + if e.Tokens.Last.isActiveNamespace() { + e.Namespaces.Last().removeLast() + } + } + e.Names.clearLast() + if prevName != nil { + e.Names.copyQuotedBuffer(e.Buf) // required by objectNameStack.replaceLastUnquotedName + e.Names.replaceLastUnquotedName(*prevName) + } + return true +} + +// UnwriteOnlyObjectMemberName unwrites the only object member name +// and returns the unquoted name. +func (e *encoderState) UnwriteOnlyObjectMemberName() string { + if last := e.Tokens.Last; !last.isObject() || last.Length() != 1 { + panic("BUG: must be called on an object after writing first name") + } + + // Unwrite the name and whitespace. + b := jsonwire.TrimSuffixString(e.Buf) + isVerbatim := bytes.IndexByte(e.Buf[len(b):], '\\') < 0 + name := string(jsonwire.UnquoteMayCopy(e.Buf[len(b):], isVerbatim)) + e.Buf = jsonwire.TrimSuffixWhitespace(b) + + // Undo state changes. + e.Tokens.Last.decrement() + if !e.Flags.Get(jsonflags.AllowDuplicateNames) { + if e.Tokens.Last.isActiveNamespace() { + e.Namespaces.Last().removeLast() + } + } + e.Names.clearLast() + return name +} + +// WriteToken writes the next token and advances the internal write offset. +// +// The provided token kind must be consistent with the JSON grammar. +// For example, it is an error to provide a number when the encoder +// is expecting an object name (which is always a string), or +// to provide an end object delimiter when the encoder is finishing an array. +// If the provided token is invalid, then it reports a [SyntacticError] and +// the internal state remains unchanged. The offset reported +// in [SyntacticError] will be relative to the [Encoder.OutputOffset]. +func (e *Encoder) WriteToken(t Token) error { + return e.s.WriteToken(t) +} +func (e *encoderState) WriteToken(t Token) error { + k := t.Kind() + b := e.Buf // use local variable to avoid mutating e in case of error + + // Append any delimiters or optional whitespace. + b = e.Tokens.MayAppendDelim(b, k) + if e.Flags.Get(jsonflags.AnyWhitespace) { + b = e.appendWhitespace(b, k) + } + pos := len(b) // offset before the token + + // Append the token to the output and to the state machine. + var err error + switch k { + case 'n': + b = append(b, "null"...) + err = e.Tokens.appendLiteral() + case 'f': + b = append(b, "false"...) + err = e.Tokens.appendLiteral() + case 't': + b = append(b, "true"...) + err = e.Tokens.appendLiteral() + case '"': + if b, err = t.appendString(b, &e.Flags); err != nil { + break + } + if e.Tokens.Last.NeedObjectName() { + if !e.Flags.Get(jsonflags.AllowDuplicateNames) { + if !e.Tokens.Last.isValidNamespace() { + err = errInvalidNamespace + break + } + if e.Tokens.Last.isActiveNamespace() && !e.Namespaces.Last().insertQuoted(b[pos:], false) { + err = wrapWithObjectName(ErrDuplicateName, b[pos:]) + break + } + } + e.Names.ReplaceLastQuotedOffset(pos) // only replace if insertQuoted succeeds + } + err = e.Tokens.appendString() + case '0': + if b, err = t.appendNumber(b, &e.Flags); err != nil { + break + } + err = e.Tokens.appendNumber() + case '{': + b = append(b, '{') + if err = e.Tokens.pushObject(); err != nil { + break + } + e.Names.push() + if !e.Flags.Get(jsonflags.AllowDuplicateNames) { + e.Namespaces.push() + } + case '}': + b = append(b, '}') + if err = e.Tokens.popObject(); err != nil { + break + } + e.Names.pop() + if !e.Flags.Get(jsonflags.AllowDuplicateNames) { + e.Namespaces.pop() + } + case '[': + b = append(b, '[') + err = e.Tokens.pushArray() + case ']': + b = append(b, ']') + err = e.Tokens.popArray() + default: + err = errInvalidToken + } + if err != nil { + return wrapSyntacticError(e, err, pos, +1) + } + + // Finish off the buffer and store it back into e. + e.Buf = b + if e.NeedFlush() { + return e.Flush() + } + return nil +} + +// AppendRaw appends either a raw string (without double quotes) or number. +// Specify safeASCII if the string output is guaranteed to be ASCII +// without any characters (including '<', '>', and '&') that need escaping, +// otherwise this will validate whether the string needs escaping. +// The appended bytes for a JSON number must be valid. +// +// This is a specialized implementation of Encoder.WriteValue +// that allows appending directly into the buffer. +// It is only called from marshal logic in the "json" package. +func (e *encoderState) AppendRaw(k Kind, safeASCII bool, appendFn func([]byte) ([]byte, error)) error { + b := e.Buf // use local variable to avoid mutating e in case of error + + // Append any delimiters or optional whitespace. + b = e.Tokens.MayAppendDelim(b, k) + if e.Flags.Get(jsonflags.AnyWhitespace) { + b = e.appendWhitespace(b, k) + } + pos := len(b) // offset before the token + + var err error + switch k { + case '"': + // Append directly into the encoder buffer by assuming that + // most of the time none of the characters need escaping. + b = append(b, '"') + if b, err = appendFn(b); err != nil { + return err + } + b = append(b, '"') + + // Check whether we need to escape the string and if necessary + // copy it to a scratch buffer and then escape it back. + isVerbatim := safeASCII || !jsonwire.NeedEscape(b[pos+len(`"`):len(b)-len(`"`)]) + if !isVerbatim { + var err error + b2 := append(e.availBuffer, b[pos+len(`"`):len(b)-len(`"`)]...) + b, err = jsonwire.AppendQuote(b[:pos], string(b2), &e.Flags) + e.availBuffer = b2[:0] + if err != nil { + return wrapSyntacticError(e, err, pos, +1) + } + } + + // Update the state machine. + if e.Tokens.Last.NeedObjectName() { + if !e.Flags.Get(jsonflags.AllowDuplicateNames) { + if !e.Tokens.Last.isValidNamespace() { + return wrapSyntacticError(e, err, pos, +1) + } + if e.Tokens.Last.isActiveNamespace() && !e.Namespaces.Last().insertQuoted(b[pos:], isVerbatim) { + err = wrapWithObjectName(ErrDuplicateName, b[pos:]) + return wrapSyntacticError(e, err, pos, +1) + } + } + e.Names.ReplaceLastQuotedOffset(pos) // only replace if insertQuoted succeeds + } + if err := e.Tokens.appendString(); err != nil { + return wrapSyntacticError(e, err, pos, +1) + } + case '0': + if b, err = appendFn(b); err != nil { + return err + } + if err := e.Tokens.appendNumber(); err != nil { + return wrapSyntacticError(e, err, pos, +1) + } + default: + panic("BUG: invalid kind") + } + + // Finish off the buffer and store it back into e. + e.Buf = b + if e.NeedFlush() { + return e.Flush() + } + return nil +} + +// WriteValue writes the next raw value and advances the internal write offset. +// The Encoder does not simply copy the provided value verbatim, but +// parses it to ensure that it is syntactically valid and reformats it +// according to how the Encoder is configured to format whitespace and strings. +// If [AllowInvalidUTF8] is specified, then any invalid UTF-8 is mangled +// as the Unicode replacement character, U+FFFD. +// +// The provided value kind must be consistent with the JSON grammar +// (see examples on [Encoder.WriteToken]). If the provided value is invalid, +// then it reports a [SyntacticError] and the internal state remains unchanged. +// The offset reported in [SyntacticError] will be relative to the +// [Encoder.OutputOffset] plus the offset into v of any encountered syntax error. +func (e *Encoder) WriteValue(v Value) error { + return e.s.WriteValue(v) +} +func (e *encoderState) WriteValue(v Value) error { + e.maxValue |= len(v) // bitwise OR is a fast approximation of max + + k := v.Kind() + b := e.Buf // use local variable to avoid mutating e in case of error + + // Append any delimiters or optional whitespace. + b = e.Tokens.MayAppendDelim(b, k) + if e.Flags.Get(jsonflags.AnyWhitespace) { + b = e.appendWhitespace(b, k) + } + pos := len(b) // offset before the value + + // Append the value the output. + var n int + n += jsonwire.ConsumeWhitespace(v[n:]) + b, m, err := e.reformatValue(b, v[n:], e.Tokens.Depth()) + if err != nil { + return wrapSyntacticError(e, err, pos+n+m, +1) + } + n += m + n += jsonwire.ConsumeWhitespace(v[n:]) + if len(v) > n { + err = jsonwire.NewInvalidCharacterError(v[n:], "after top-level value") + return wrapSyntacticError(e, err, pos+n, 0) + } + + // Append the kind to the state machine. + switch k { + case 'n', 'f', 't': + err = e.Tokens.appendLiteral() + case '"': + if e.Tokens.Last.NeedObjectName() { + if !e.Flags.Get(jsonflags.AllowDuplicateNames) { + if !e.Tokens.Last.isValidNamespace() { + err = errInvalidNamespace + break + } + if e.Tokens.Last.isActiveNamespace() && !e.Namespaces.Last().insertQuoted(b[pos:], false) { + err = wrapWithObjectName(ErrDuplicateName, b[pos:]) + break + } + } + e.Names.ReplaceLastQuotedOffset(pos) // only replace if insertQuoted succeeds + } + err = e.Tokens.appendString() + case '0': + err = e.Tokens.appendNumber() + case '{': + if err = e.Tokens.pushObject(); err != nil { + break + } + if err = e.Tokens.popObject(); err != nil { + panic("BUG: popObject should never fail immediately after pushObject: " + err.Error()) + } + if e.Flags.Get(jsonflags.ReorderRawObjects) { + mustReorderObjects(b[pos:]) + } + case '[': + if err = e.Tokens.pushArray(); err != nil { + break + } + if err = e.Tokens.popArray(); err != nil { + panic("BUG: popArray should never fail immediately after pushArray: " + err.Error()) + } + if e.Flags.Get(jsonflags.ReorderRawObjects) { + mustReorderObjects(b[pos:]) + } + } + if err != nil { + return wrapSyntacticError(e, err, pos, +1) + } + + // Finish off the buffer and store it back into e. + e.Buf = b + if e.NeedFlush() { + return e.Flush() + } + return nil +} + +// CountNextDelimWhitespace counts the number of bytes of delimiter and +// whitespace bytes assuming the upcoming token is a JSON value. +// This method is used for error reporting at the semantic layer. +func (e *encoderState) CountNextDelimWhitespace() (n int) { + const next = Kind('"') // arbitrary kind as next JSON value + delim := e.Tokens.needDelim(next) + if delim > 0 { + n += len(",") | len(":") + } + if delim == ':' { + if e.Flags.Get(jsonflags.SpaceAfterColon) { + n += len(" ") + } + } else { + if delim == ',' && e.Flags.Get(jsonflags.SpaceAfterComma) { + n += len(" ") + } + if e.Flags.Get(jsonflags.Multiline) { + if m := e.Tokens.NeedIndent(next); m > 0 { + n += len("\n") + len(e.IndentPrefix) + (m-1)*len(e.Indent) + } + } + } + return n +} + +// appendWhitespace appends whitespace that immediately precedes the next token. +func (e *encoderState) appendWhitespace(b []byte, next Kind) []byte { + if delim := e.Tokens.needDelim(next); delim == ':' { + if e.Flags.Get(jsonflags.SpaceAfterColon) { + b = append(b, ' ') + } + } else { + if delim == ',' && e.Flags.Get(jsonflags.SpaceAfterComma) { + b = append(b, ' ') + } + if e.Flags.Get(jsonflags.Multiline) { + b = e.AppendIndent(b, e.Tokens.NeedIndent(next)) + } + } + return b +} + +// AppendIndent appends the appropriate number of indentation characters +// for the current nested level, n. +func (e *encoderState) AppendIndent(b []byte, n int) []byte { + if n == 0 { + return b + } + b = append(b, '\n') + b = append(b, e.IndentPrefix...) + for ; n > 1; n-- { + b = append(b, e.Indent...) + } + return b +} + +// reformatValue parses a JSON value from the start of src and +// appends it to the end of dst, reformatting whitespace and strings as needed. +// It returns the extended dst buffer and the number of consumed input bytes. +func (e *encoderState) reformatValue(dst []byte, src Value, depth int) ([]byte, int, error) { + // TODO: Should this update ValueFlags as input? + if len(src) == 0 { + return dst, 0, io.ErrUnexpectedEOF + } + switch k := Kind(src[0]).normalize(); k { + case 'n': + if jsonwire.ConsumeNull(src) == 0 { + n, err := jsonwire.ConsumeLiteral(src, "null") + return dst, n, err + } + return append(dst, "null"...), len("null"), nil + case 'f': + if jsonwire.ConsumeFalse(src) == 0 { + n, err := jsonwire.ConsumeLiteral(src, "false") + return dst, n, err + } + return append(dst, "false"...), len("false"), nil + case 't': + if jsonwire.ConsumeTrue(src) == 0 { + n, err := jsonwire.ConsumeLiteral(src, "true") + return dst, n, err + } + return append(dst, "true"...), len("true"), nil + case '"': + if n := jsonwire.ConsumeSimpleString(src); n > 0 { + dst = append(dst, src[:n]...) // copy simple strings verbatim + return dst, n, nil + } + return jsonwire.ReformatString(dst, src, &e.Flags) + case '0': + if n := jsonwire.ConsumeSimpleNumber(src); n > 0 && !e.Flags.Get(jsonflags.CanonicalizeNumbers) { + dst = append(dst, src[:n]...) // copy simple numbers verbatim + return dst, n, nil + } + return jsonwire.ReformatNumber(dst, src, &e.Flags) + case '{': + return e.reformatObject(dst, src, depth) + case '[': + return e.reformatArray(dst, src, depth) + default: + return dst, 0, jsonwire.NewInvalidCharacterError(src, "at start of value") + } +} + +// reformatObject parses a JSON object from the start of src and +// appends it to the end of src, reformatting whitespace and strings as needed. +// It returns the extended dst buffer and the number of consumed input bytes. +func (e *encoderState) reformatObject(dst []byte, src Value, depth int) ([]byte, int, error) { + // Append object begin. + if len(src) == 0 || src[0] != '{' { + panic("BUG: reformatObject must be called with a buffer that starts with '{'") + } else if depth == maxNestingDepth+1 { + return dst, 0, errMaxDepth + } + dst = append(dst, '{') + n := len("{") + + // Append (possible) object end. + n += jsonwire.ConsumeWhitespace(src[n:]) + if uint(len(src)) <= uint(n) { + return dst, n, io.ErrUnexpectedEOF + } + if src[n] == '}' { + dst = append(dst, '}') + n += len("}") + return dst, n, nil + } + + var err error + var names *objectNamespace + if !e.Flags.Get(jsonflags.AllowDuplicateNames) { + e.Namespaces.push() + defer e.Namespaces.pop() + names = e.Namespaces.Last() + } + depth++ + for { + // Append optional newline and indentation. + if e.Flags.Get(jsonflags.Multiline) { + dst = e.AppendIndent(dst, depth) + } + + // Append object name. + n += jsonwire.ConsumeWhitespace(src[n:]) + if uint(len(src)) <= uint(n) { + return dst, n, io.ErrUnexpectedEOF + } + m := jsonwire.ConsumeSimpleString(src[n:]) + isVerbatim := m > 0 + if isVerbatim { + dst = append(dst, src[n:n+m]...) + } else { + dst, m, err = jsonwire.ReformatString(dst, src[n:], &e.Flags) + if err != nil { + return dst, n + m, err + } + } + quotedName := src[n : n+m] + if !e.Flags.Get(jsonflags.AllowDuplicateNames) && !names.insertQuoted(quotedName, isVerbatim) { + return dst, n, wrapWithObjectName(ErrDuplicateName, quotedName) + } + n += m + + // Append colon. + n += jsonwire.ConsumeWhitespace(src[n:]) + if uint(len(src)) <= uint(n) { + return dst, n, wrapWithObjectName(io.ErrUnexpectedEOF, quotedName) + } + if src[n] != ':' { + err = jsonwire.NewInvalidCharacterError(src[n:], "after object name (expecting ':')") + return dst, n, wrapWithObjectName(err, quotedName) + } + dst = append(dst, ':') + n += len(":") + if e.Flags.Get(jsonflags.SpaceAfterColon) { + dst = append(dst, ' ') + } + + // Append object value. + n += jsonwire.ConsumeWhitespace(src[n:]) + if uint(len(src)) <= uint(n) { + return dst, n, wrapWithObjectName(io.ErrUnexpectedEOF, quotedName) + } + dst, m, err = e.reformatValue(dst, src[n:], depth) + if err != nil { + return dst, n + m, wrapWithObjectName(err, quotedName) + } + n += m + + // Append comma or object end. + n += jsonwire.ConsumeWhitespace(src[n:]) + if uint(len(src)) <= uint(n) { + return dst, n, io.ErrUnexpectedEOF + } + switch src[n] { + case ',': + dst = append(dst, ',') + if e.Flags.Get(jsonflags.SpaceAfterComma) { + dst = append(dst, ' ') + } + n += len(",") + continue + case '}': + if e.Flags.Get(jsonflags.Multiline) { + dst = e.AppendIndent(dst, depth-1) + } + dst = append(dst, '}') + n += len("}") + return dst, n, nil + default: + return dst, n, jsonwire.NewInvalidCharacterError(src[n:], "after object value (expecting ',' or '}')") + } + } +} + +// reformatArray parses a JSON array from the start of src and +// appends it to the end of dst, reformatting whitespace and strings as needed. +// It returns the extended dst buffer and the number of consumed input bytes. +func (e *encoderState) reformatArray(dst []byte, src Value, depth int) ([]byte, int, error) { + // Append array begin. + if len(src) == 0 || src[0] != '[' { + panic("BUG: reformatArray must be called with a buffer that starts with '['") + } else if depth == maxNestingDepth+1 { + return dst, 0, errMaxDepth + } + dst = append(dst, '[') + n := len("[") + + // Append (possible) array end. + n += jsonwire.ConsumeWhitespace(src[n:]) + if uint(len(src)) <= uint(n) { + return dst, n, io.ErrUnexpectedEOF + } + if src[n] == ']' { + dst = append(dst, ']') + n += len("]") + return dst, n, nil + } + + var idx int64 + var err error + depth++ + for { + // Append optional newline and indentation. + if e.Flags.Get(jsonflags.Multiline) { + dst = e.AppendIndent(dst, depth) + } + + // Append array value. + n += jsonwire.ConsumeWhitespace(src[n:]) + if uint(len(src)) <= uint(n) { + return dst, n, io.ErrUnexpectedEOF + } + var m int + dst, m, err = e.reformatValue(dst, src[n:], depth) + if err != nil { + return dst, n + m, wrapWithArrayIndex(err, idx) + } + n += m + + // Append comma or array end. + n += jsonwire.ConsumeWhitespace(src[n:]) + if uint(len(src)) <= uint(n) { + return dst, n, io.ErrUnexpectedEOF + } + switch src[n] { + case ',': + dst = append(dst, ',') + if e.Flags.Get(jsonflags.SpaceAfterComma) { + dst = append(dst, ' ') + } + n += len(",") + idx++ + continue + case ']': + if e.Flags.Get(jsonflags.Multiline) { + dst = e.AppendIndent(dst, depth-1) + } + dst = append(dst, ']') + n += len("]") + return dst, n, nil + default: + return dst, n, jsonwire.NewInvalidCharacterError(src[n:], "after array value (expecting ',' or ']')") + } + } +} + +// OutputOffset returns the current output byte offset. It gives the location +// of the next byte immediately after the most recently written token or value. +// The number of bytes actually written to the underlying [io.Writer] may be less +// than this offset due to internal buffering effects. +func (e *Encoder) OutputOffset() int64 { + return e.s.previousOffsetEnd() +} + +// AvailableBuffer returns a zero-length buffer with a possible non-zero capacity. +// This buffer is intended to be used to populate a [Value] +// being passed to an immediately succeeding [Encoder.WriteValue] call. +// +// Example usage: +// +// b := d.AvailableBuffer() +// b = append(b, '"') +// b = appendString(b, v) // append the string formatting of v +// b = append(b, '"') +// ... := d.WriteValue(b) +// +// It is the user's responsibility to ensure that the value is valid JSON. +func (e *Encoder) AvailableBuffer() []byte { + // NOTE: We don't return e.buf[len(e.buf):cap(e.buf)] since WriteValue would + // need to take special care to avoid mangling the data while reformatting. + // WriteValue can't easily identify whether the input Value aliases e.buf + // without using unsafe.Pointer. Thus, we just return a different buffer. + // Should this ever alias e.buf, we need to consider how it operates with + // the specialized performance optimization for bytes.Buffer. + n := 1 << bits.Len(uint(e.s.maxValue|63)) // fast approximation for max length + if cap(e.s.availBuffer) < n { + e.s.availBuffer = make([]byte, 0, n) + } + return e.s.availBuffer +} + +// StackDepth returns the depth of the state machine for written JSON data. +// Each level on the stack represents a nested JSON object or array. +// It is incremented whenever an [BeginObject] or [BeginArray] token is encountered +// and decremented whenever an [EndObject] or [EndArray] token is encountered. +// The depth is zero-indexed, where zero represents the top-level JSON value. +func (e *Encoder) StackDepth() int { + // NOTE: Keep in sync with Decoder.StackDepth. + return e.s.Tokens.Depth() - 1 +} + +// StackIndex returns information about the specified stack level. +// It must be a number between 0 and [Encoder.StackDepth], inclusive. +// For each level, it reports the kind: +// +// - 0 for a level of zero, +// - '{' for a level representing a JSON object, and +// - '[' for a level representing a JSON array. +// +// It also reports the length of that JSON object or array. +// Each name and value in a JSON object is counted separately, +// so the effective number of members would be half the length. +// A complete JSON object must have an even length. +func (e *Encoder) StackIndex(i int) (Kind, int64) { + // NOTE: Keep in sync with Decoder.StackIndex. + switch s := e.s.Tokens.index(i); { + case i > 0 && s.isObject(): + return '{', s.Length() + case i > 0 && s.isArray(): + return '[', s.Length() + default: + return 0, s.Length() + } +} + +// StackPointer returns a JSON Pointer (RFC 6901) to the most recently written value. +func (e *Encoder) StackPointer() Pointer { + return Pointer(e.s.AppendStackPointer(nil, -1)) +} + +func (e *encoderState) AppendStackPointer(b []byte, where int) []byte { + e.Names.copyQuotedBuffer(e.Buf) + return e.state.appendStackPointer(b, where) +} diff --git a/src/encoding/json/jsontext/encode_test.go b/src/encoding/json/jsontext/encode_test.go new file mode 100644 index 00000000000000..a9505f5258cbfe --- /dev/null +++ b/src/encoding/json/jsontext/encode_test.go @@ -0,0 +1,829 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "bytes" + "errors" + "io" + "path" + "slices" + "testing" + + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsontest" + "encoding/json/internal/jsonwire" +) + +// TestEncoder tests whether we can produce JSON with either tokens or raw values. +func TestEncoder(t *testing.T) { + for _, td := range coderTestdata { + for _, formatName := range []string{"Compact", "Indented"} { + for _, typeName := range []string{"Token", "Value", "TokenDelims"} { + t.Run(path.Join(td.name.Name, typeName, formatName), func(t *testing.T) { + testEncoder(t, td.name.Where, formatName, typeName, td) + }) + } + } + } +} +func testEncoder(t *testing.T, where jsontest.CasePos, formatName, typeName string, td coderTestdataEntry) { + var want string + var opts []Options + dst := new(bytes.Buffer) + opts = append(opts, jsonflags.OmitTopLevelNewline|1) + want = td.outCompacted + switch formatName { + case "Indented": + opts = append(opts, Multiline(true)) + opts = append(opts, WithIndentPrefix("\t")) + opts = append(opts, WithIndent(" ")) + if td.outIndented != "" { + want = td.outIndented + } + } + enc := NewEncoder(dst, opts...) + + switch typeName { + case "Token": + var pointers []Pointer + for _, tok := range td.tokens { + if err := enc.WriteToken(tok); err != nil { + t.Fatalf("%s: Encoder.WriteToken error: %v", where, err) + } + if td.pointers != nil { + pointers = append(pointers, enc.StackPointer()) + } + } + if !slices.Equal(pointers, td.pointers) { + t.Fatalf("%s: pointers mismatch:\ngot %q\nwant %q", where, pointers, td.pointers) + } + case "Value": + if err := enc.WriteValue(Value(td.in)); err != nil { + t.Fatalf("%s: Encoder.WriteValue error: %v", where, err) + } + case "TokenDelims": + // Use WriteToken for object/array delimiters, WriteValue otherwise. + for _, tok := range td.tokens { + switch tok.Kind() { + case '{', '}', '[', ']': + if err := enc.WriteToken(tok); err != nil { + t.Fatalf("%s: Encoder.WriteToken error: %v", where, err) + } + default: + val := Value(tok.String()) + if tok.Kind() == '"' { + val, _ = jsonwire.AppendQuote(nil, tok.String(), &jsonflags.Flags{}) + } + if err := enc.WriteValue(val); err != nil { + t.Fatalf("%s: Encoder.WriteValue error: %v", where, err) + } + } + } + } + + got := dst.String() + if got != want { + t.Errorf("%s: output mismatch:\ngot %q\nwant %q", where, got, want) + } +} + +// TestFaultyEncoder tests that temporary I/O errors are not fatal. +func TestFaultyEncoder(t *testing.T) { + for _, td := range coderTestdata { + for _, typeName := range []string{"Token", "Value"} { + t.Run(path.Join(td.name.Name, typeName), func(t *testing.T) { + testFaultyEncoder(t, td.name.Where, typeName, td) + }) + } + } +} +func testFaultyEncoder(t *testing.T, where jsontest.CasePos, typeName string, td coderTestdataEntry) { + b := &FaultyBuffer{ + MaxBytes: 1, + MayError: io.ErrShortWrite, + } + + // Write all the tokens. + // Even if the underlying io.Writer may be faulty, + // writing a valid token or value is guaranteed to at least + // be appended to the internal buffer. + // In other words, syntactic errors occur before I/O errors. + enc := NewEncoder(b) + switch typeName { + case "Token": + for i, tok := range td.tokens { + err := enc.WriteToken(tok) + if err != nil && !errors.Is(err, io.ErrShortWrite) { + t.Fatalf("%s: %d: Encoder.WriteToken error: %v", where, i, err) + } + } + case "Value": + err := enc.WriteValue(Value(td.in)) + if err != nil && !errors.Is(err, io.ErrShortWrite) { + t.Fatalf("%s: Encoder.WriteValue error: %v", where, err) + } + } + gotOutput := string(append(b.B, enc.s.unflushedBuffer()...)) + wantOutput := td.outCompacted + "\n" + if gotOutput != wantOutput { + t.Fatalf("%s: output mismatch:\ngot %s\nwant %s", where, gotOutput, wantOutput) + } +} + +type encoderMethodCall struct { + in tokOrVal + wantErr error + wantPointer Pointer +} + +var encoderErrorTestdata = []struct { + name jsontest.CaseName + opts []Options + calls []encoderMethodCall + wantOut string +}{{ + name: jsontest.Name("InvalidToken"), + calls: []encoderMethodCall{ + {zeroToken, E(errInvalidToken), ""}, + }, +}, { + name: jsontest.Name("InvalidValue"), + calls: []encoderMethodCall{ + {Value(`#`), newInvalidCharacterError("#", "at start of value"), ""}, + }, +}, { + name: jsontest.Name("InvalidValue/DoubleZero"), + calls: []encoderMethodCall{ + {Value(`00`), newInvalidCharacterError("0", "after top-level value").withPos(`0`, ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedValue"), + calls: []encoderMethodCall{ + {zeroValue, E(io.ErrUnexpectedEOF).withPos("", ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedNull"), + calls: []encoderMethodCall{ + {Value(`nul`), E(io.ErrUnexpectedEOF).withPos("nul", ""), ""}, + }, +}, { + name: jsontest.Name("InvalidNull"), + calls: []encoderMethodCall{ + {Value(`nulL`), newInvalidCharacterError("L", "in literal null (expecting 'l')").withPos(`nul`, ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedFalse"), + calls: []encoderMethodCall{ + {Value(`fals`), E(io.ErrUnexpectedEOF).withPos("fals", ""), ""}, + }, +}, { + name: jsontest.Name("InvalidFalse"), + calls: []encoderMethodCall{ + {Value(`falsE`), newInvalidCharacterError("E", "in literal false (expecting 'e')").withPos(`fals`, ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedTrue"), + calls: []encoderMethodCall{ + {Value(`tru`), E(io.ErrUnexpectedEOF).withPos(`tru`, ""), ""}, + }, +}, { + name: jsontest.Name("InvalidTrue"), + calls: []encoderMethodCall{ + {Value(`truE`), newInvalidCharacterError("E", "in literal true (expecting 'e')").withPos(`tru`, ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedString"), + calls: []encoderMethodCall{ + {Value(`"star`), E(io.ErrUnexpectedEOF).withPos(`"star`, ""), ""}, + }, +}, { + name: jsontest.Name("InvalidString"), + calls: []encoderMethodCall{ + {Value(`"ok` + "\x00"), newInvalidCharacterError("\x00", `in string (expecting non-control character)`).withPos(`"ok`, ""), ""}, + }, +}, { + name: jsontest.Name("ValidString/AllowInvalidUTF8/Token"), + opts: []Options{AllowInvalidUTF8(true)}, + calls: []encoderMethodCall{ + {String("living\xde\xad\xbe\xef"), nil, ""}, + }, + wantOut: "\"living\xde\xad\ufffd\ufffd\"\n", +}, { + name: jsontest.Name("ValidString/AllowInvalidUTF8/Value"), + opts: []Options{AllowInvalidUTF8(true)}, + calls: []encoderMethodCall{ + {Value("\"living\xde\xad\xbe\xef\""), nil, ""}, + }, + wantOut: "\"living\xde\xad\ufffd\ufffd\"\n", +}, { + name: jsontest.Name("InvalidString/RejectInvalidUTF8"), + opts: []Options{AllowInvalidUTF8(false)}, + calls: []encoderMethodCall{ + {String("living\xde\xad\xbe\xef"), E(jsonwire.ErrInvalidUTF8), ""}, + {Value("\"living\xde\xad\xbe\xef\""), E(jsonwire.ErrInvalidUTF8).withPos("\"living\xde\xad", ""), ""}, + {BeginObject, nil, ""}, + {String("name"), nil, ""}, + {BeginArray, nil, ""}, + {String("living\xde\xad\xbe\xef"), E(jsonwire.ErrInvalidUTF8).withPos(`{"name":[`, "/name/0"), ""}, + {Value("\"living\xde\xad\xbe\xef\""), E(jsonwire.ErrInvalidUTF8).withPos("{\"name\":[\"living\xde\xad", "/name/0"), ""}, + }, + wantOut: `{"name":[`, +}, { + name: jsontest.Name("TruncatedNumber"), + calls: []encoderMethodCall{ + {Value(`0.`), E(io.ErrUnexpectedEOF).withPos("0", ""), ""}, + }, +}, { + name: jsontest.Name("InvalidNumber"), + calls: []encoderMethodCall{ + {Value(`0.e`), newInvalidCharacterError("e", "in number (expecting digit)").withPos(`0.`, ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedObject/AfterStart"), + calls: []encoderMethodCall{ + {Value(`{`), E(io.ErrUnexpectedEOF).withPos("{", ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedObject/AfterName"), + calls: []encoderMethodCall{ + {Value(`{"X"`), E(io.ErrUnexpectedEOF).withPos(`{"X"`, "/X"), ""}, + }, +}, { + name: jsontest.Name("TruncatedObject/AfterColon"), + calls: []encoderMethodCall{ + {Value(`{"X":`), E(io.ErrUnexpectedEOF).withPos(`{"X":`, "/X"), ""}, + }, +}, { + name: jsontest.Name("TruncatedObject/AfterValue"), + calls: []encoderMethodCall{ + {Value(`{"0":0`), E(io.ErrUnexpectedEOF).withPos(`{"0":0`, ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedObject/AfterComma"), + calls: []encoderMethodCall{ + {Value(`{"0":0,`), E(io.ErrUnexpectedEOF).withPos(`{"0":0,`, ""), ""}, + }, +}, { + name: jsontest.Name("InvalidObject/MissingColon"), + calls: []encoderMethodCall{ + {Value(` { "fizz" "buzz" } `), newInvalidCharacterError("\"", "after object name (expecting ':')").withPos(` { "fizz" `, "/fizz"), ""}, + {Value(` { "fizz" , "buzz" } `), newInvalidCharacterError(",", "after object name (expecting ':')").withPos(` { "fizz" `, "/fizz"), ""}, + }, +}, { + name: jsontest.Name("InvalidObject/MissingComma"), + calls: []encoderMethodCall{ + {Value(` { "fizz" : "buzz" "gazz" } `), newInvalidCharacterError("\"", "after object value (expecting ',' or '}')").withPos(` { "fizz" : "buzz" `, ""), ""}, + {Value(` { "fizz" : "buzz" : "gazz" } `), newInvalidCharacterError(":", "after object value (expecting ',' or '}')").withPos(` { "fizz" : "buzz" `, ""), ""}, + }, +}, { + name: jsontest.Name("InvalidObject/ExtraComma"), + calls: []encoderMethodCall{ + {Value(` { , } `), newInvalidCharacterError(",", `at start of string (expecting '"')`).withPos(` { `, ""), ""}, + {Value(` { "fizz" : "buzz" , } `), newInvalidCharacterError("}", `at start of string (expecting '"')`).withPos(` { "fizz" : "buzz" , `, ""), ""}, + }, +}, { + name: jsontest.Name("InvalidObject/InvalidName"), + calls: []encoderMethodCall{ + {Value(`{ null }`), newInvalidCharacterError("n", `at start of string (expecting '"')`).withPos(`{ `, ""), ""}, + {Value(`{ false }`), newInvalidCharacterError("f", `at start of string (expecting '"')`).withPos(`{ `, ""), ""}, + {Value(`{ true }`), newInvalidCharacterError("t", `at start of string (expecting '"')`).withPos(`{ `, ""), ""}, + {Value(`{ 0 }`), newInvalidCharacterError("0", `at start of string (expecting '"')`).withPos(`{ `, ""), ""}, + {Value(`{ {} }`), newInvalidCharacterError("{", `at start of string (expecting '"')`).withPos(`{ `, ""), ""}, + {Value(`{ [] }`), newInvalidCharacterError("[", `at start of string (expecting '"')`).withPos(`{ `, ""), ""}, + {BeginObject, nil, ""}, + {Null, E(ErrNonStringName).withPos(`{`, ""), ""}, + {Value(`null`), E(ErrNonStringName).withPos(`{`, ""), ""}, + {False, E(ErrNonStringName).withPos(`{`, ""), ""}, + {Value(`false`), E(ErrNonStringName).withPos(`{`, ""), ""}, + {True, E(ErrNonStringName).withPos(`{`, ""), ""}, + {Value(`true`), E(ErrNonStringName).withPos(`{`, ""), ""}, + {Uint(0), E(ErrNonStringName).withPos(`{`, ""), ""}, + {Value(`0`), E(ErrNonStringName).withPos(`{`, ""), ""}, + {BeginObject, E(ErrNonStringName).withPos(`{`, ""), ""}, + {Value(`{}`), E(ErrNonStringName).withPos(`{`, ""), ""}, + {BeginArray, E(ErrNonStringName).withPos(`{`, ""), ""}, + {Value(`[]`), E(ErrNonStringName).withPos(`{`, ""), ""}, + {EndObject, nil, ""}, + }, + wantOut: "{}\n", +}, { + name: jsontest.Name("InvalidObject/InvalidValue"), + calls: []encoderMethodCall{ + {Value(`{ "0": x }`), newInvalidCharacterError("x", `at start of value`).withPos(`{ "0": `, "/0"), ""}, + }, +}, { + name: jsontest.Name("InvalidObject/MismatchingDelim"), + calls: []encoderMethodCall{ + {Value(` { ] `), newInvalidCharacterError("]", `at start of string (expecting '"')`).withPos(` { `, ""), ""}, + {Value(` { "0":0 ] `), newInvalidCharacterError("]", `after object value (expecting ',' or '}')`).withPos(` { "0":0 `, ""), ""}, + {BeginObject, nil, ""}, + {EndArray, E(errMismatchDelim).withPos(`{`, ""), ""}, + {Value(`]`), newInvalidCharacterError("]", "at start of value").withPos(`{`, ""), ""}, + {EndObject, nil, ""}, + }, + wantOut: "{}\n", +}, { + name: jsontest.Name("ValidObject/UniqueNames"), + calls: []encoderMethodCall{ + {BeginObject, nil, ""}, + {String("0"), nil, ""}, + {Uint(0), nil, ""}, + {String("1"), nil, ""}, + {Uint(1), nil, ""}, + {EndObject, nil, ""}, + {Value(` { "0" : 0 , "1" : 1 } `), nil, ""}, + }, + wantOut: `{"0":0,"1":1}` + "\n" + `{"0":0,"1":1}` + "\n", +}, { + name: jsontest.Name("ValidObject/DuplicateNames"), + opts: []Options{AllowDuplicateNames(true)}, + calls: []encoderMethodCall{ + {BeginObject, nil, ""}, + {String("0"), nil, ""}, + {Uint(0), nil, ""}, + {String("0"), nil, ""}, + {Uint(0), nil, ""}, + {EndObject, nil, ""}, + {Value(` { "0" : 0 , "0" : 0 } `), nil, ""}, + }, + wantOut: `{"0":0,"0":0}` + "\n" + `{"0":0,"0":0}` + "\n", +}, { + name: jsontest.Name("InvalidObject/DuplicateNames"), + calls: []encoderMethodCall{ + {BeginObject, nil, ""}, + {String("X"), nil, ""}, + {BeginObject, nil, ""}, + {EndObject, nil, ""}, + {String("X"), E(ErrDuplicateName).withPos(`{"X":{},`, "/X"), "/X"}, + {Value(`"X"`), E(ErrDuplicateName).withPos(`{"X":{},`, "/X"), "/X"}, + {String("Y"), nil, ""}, + {BeginObject, nil, ""}, + {EndObject, nil, ""}, + {String("X"), E(ErrDuplicateName).withPos(`{"X":{},"Y":{},`, "/X"), "/Y"}, + {Value(`"X"`), E(ErrDuplicateName).withPos(`{"X":{},"Y":{},`, "/X"), "/Y"}, + {String("Y"), E(ErrDuplicateName).withPos(`{"X":{},"Y":{},`, "/Y"), "/Y"}, + {Value(`"Y"`), E(ErrDuplicateName).withPos(`{"X":{},"Y":{},`, "/Y"), "/Y"}, + {EndObject, nil, ""}, + {Value(` { "X" : 0 , "Y" : 1 , "X" : 0 } `), E(ErrDuplicateName).withPos(`{"X":{},"Y":{}}`+"\n"+` { "X" : 0 , "Y" : 1 , `, "/X"), ""}, + }, + wantOut: `{"X":{},"Y":{}}` + "\n", +}, { + name: jsontest.Name("TruncatedArray/AfterStart"), + calls: []encoderMethodCall{ + {Value(`[`), E(io.ErrUnexpectedEOF).withPos(`[`, ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedArray/AfterValue"), + calls: []encoderMethodCall{ + {Value(`[0`), E(io.ErrUnexpectedEOF).withPos(`[0`, ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedArray/AfterComma"), + calls: []encoderMethodCall{ + {Value(`[0,`), E(io.ErrUnexpectedEOF).withPos(`[0,`, ""), ""}, + }, +}, { + name: jsontest.Name("TruncatedArray/MissingComma"), + calls: []encoderMethodCall{ + {Value(` [ "fizz" "buzz" ] `), newInvalidCharacterError("\"", "after array value (expecting ',' or ']')").withPos(` [ "fizz" `, ""), ""}, + }, +}, { + name: jsontest.Name("InvalidArray/MismatchingDelim"), + calls: []encoderMethodCall{ + {Value(` [ } `), newInvalidCharacterError("}", `at start of value`).withPos(` [ `, "/0"), ""}, + {BeginArray, nil, ""}, + {EndObject, E(errMismatchDelim).withPos(`[`, "/0"), ""}, + {Value(`}`), newInvalidCharacterError("}", "at start of value").withPos(`[`, "/0"), ""}, + {EndArray, nil, ""}, + }, + wantOut: "[]\n", +}, { + name: jsontest.Name("Format/Object/SpaceAfterColon"), + opts: []Options{SpaceAfterColon(true)}, + calls: []encoderMethodCall{{Value(`{"fizz":"buzz","wizz":"wuzz"}`), nil, ""}}, + wantOut: "{\"fizz\": \"buzz\",\"wizz\": \"wuzz\"}\n", +}, { + name: jsontest.Name("Format/Object/SpaceAfterComma"), + opts: []Options{SpaceAfterComma(true)}, + calls: []encoderMethodCall{{Value(`{"fizz":"buzz","wizz":"wuzz"}`), nil, ""}}, + wantOut: "{\"fizz\":\"buzz\", \"wizz\":\"wuzz\"}\n", +}, { + name: jsontest.Name("Format/Object/SpaceAfterColonAndComma"), + opts: []Options{SpaceAfterColon(true), SpaceAfterComma(true)}, + calls: []encoderMethodCall{{Value(`{"fizz":"buzz","wizz":"wuzz"}`), nil, ""}}, + wantOut: "{\"fizz\": \"buzz\", \"wizz\": \"wuzz\"}\n", +}, { + name: jsontest.Name("Format/Object/NoSpaceAfterColon+SpaceAfterComma+Multiline"), + opts: []Options{SpaceAfterColon(false), SpaceAfterComma(true), Multiline(true)}, + calls: []encoderMethodCall{{Value(`{"fizz":"buzz","wizz":"wuzz"}`), nil, ""}}, + wantOut: "{\n\t\"fizz\":\"buzz\", \n\t\"wizz\":\"wuzz\"\n}\n", +}, { + name: jsontest.Name("Format/Array/SpaceAfterComma"), + opts: []Options{SpaceAfterComma(true)}, + calls: []encoderMethodCall{{Value(`["fizz","buzz"]`), nil, ""}}, + wantOut: "[\"fizz\", \"buzz\"]\n", +}, { + name: jsontest.Name("Format/Array/NoSpaceAfterComma+Multiline"), + opts: []Options{SpaceAfterComma(false), Multiline(true)}, + calls: []encoderMethodCall{{Value(`["fizz","buzz"]`), nil, ""}}, + wantOut: "[\n\t\"fizz\",\n\t\"buzz\"\n]\n", +}, { + name: jsontest.Name("Format/ReorderWithWhitespace"), + opts: []Options{ + AllowDuplicateNames(true), + AllowInvalidUTF8(true), + ReorderRawObjects(true), + SpaceAfterComma(true), + SpaceAfterColon(false), + Multiline(true), + WithIndentPrefix(" "), + WithIndent("\t"), + PreserveRawStrings(true), + }, + calls: []encoderMethodCall{ + {BeginArray, nil, ""}, + {BeginArray, nil, ""}, + {Value(` { "fizz" : "buzz" , + "zip" : { + "x` + "\xfd" + `x" : 123 , "x` + "\xff" + `x" : 123, "x` + "\xfe" + `x" : 123 + }, + "zap" : { + "xxx" : 333, "xxx": 1, "xxx": 22 + }, + "alpha" : "bravo" } `), nil, ""}, + {EndArray, nil, ""}, + {EndArray, nil, ""}, + }, + wantOut: "[\n \t[\n \t\t{\n \t\t\t\"alpha\":\"bravo\", \n \t\t\t\"fizz\":\"buzz\", \n \t\t\t\"zap\":{\n \t\t\t\t\"xxx\":1, \n \t\t\t\t\"xxx\":22, \n \t\t\t\t\"xxx\":333\n \t\t\t}, \n \t\t\t\"zip\":{\n \t\t\t\t\"x\xfdx\":123, \n \t\t\t\t\"x\xfex\":123, \n \t\t\t\t\"x\xffx\":123\n \t\t\t}\n \t\t}\n \t]\n ]\n", +}, { + name: jsontest.Name("Format/CanonicalizeRawInts"), + opts: []Options{CanonicalizeRawInts(true), SpaceAfterComma(true)}, + calls: []encoderMethodCall{ + {Value(`[0.100,5.0,1E6,-9223372036854775808,-10,-1,-0,0,1,10,9223372036854775807]`), nil, ""}, + }, + wantOut: "[0.100, 5.0, 1E6, -9223372036854776000, -10, -1, 0, 0, 1, 10, 9223372036854776000]\n", +}, { + name: jsontest.Name("Format/CanonicalizeRawFloats"), + opts: []Options{CanonicalizeRawFloats(true), SpaceAfterComma(true)}, + calls: []encoderMethodCall{ + {Value(`[0.100,5.0,1E6,-9223372036854775808,-10,-1,-0,0,1,10,9223372036854775807]`), nil, ""}, + }, + wantOut: "[0.1, 5, 1000000, -9223372036854775808, -10, -1, 0, 0, 1, 10, 9223372036854775807]\n", +}, { + name: jsontest.Name("ErrorPosition"), + calls: []encoderMethodCall{ + {Value(` "a` + "\xff" + `0" `), E(jsonwire.ErrInvalidUTF8).withPos(` "a`, ""), ""}, + {String(`a` + "\xff" + `0`), E(jsonwire.ErrInvalidUTF8).withPos(``, ""), ""}, + }, +}, { + name: jsontest.Name("ErrorPosition/0"), + calls: []encoderMethodCall{ + {Value(` [ "a` + "\xff" + `1" ] `), E(jsonwire.ErrInvalidUTF8).withPos(` [ "a`, "/0"), ""}, + {BeginArray, nil, ""}, + {Value(` "a` + "\xff" + `1" `), E(jsonwire.ErrInvalidUTF8).withPos(`[ "a`, "/0"), ""}, + {String(`a` + "\xff" + `1`), E(jsonwire.ErrInvalidUTF8).withPos(`[`, "/0"), ""}, + }, + wantOut: `[`, +}, { + name: jsontest.Name("ErrorPosition/1"), + calls: []encoderMethodCall{ + {Value(` [ "a1" , "b` + "\xff" + `1" ] `), E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , "b`, "/1"), ""}, + {BeginArray, nil, ""}, + {String("a1"), nil, ""}, + {Value(` "b` + "\xff" + `1" `), E(jsonwire.ErrInvalidUTF8).withPos(`["a1", "b`, "/1"), ""}, + {String(`b` + "\xff" + `1`), E(jsonwire.ErrInvalidUTF8).withPos(`["a1",`, "/1"), ""}, + }, + wantOut: `["a1"`, +}, { + name: jsontest.Name("ErrorPosition/0/0"), + calls: []encoderMethodCall{ + {Value(` [ [ "a` + "\xff" + `2" ] ] `), E(jsonwire.ErrInvalidUTF8).withPos(` [ [ "a`, "/0/0"), ""}, + {BeginArray, nil, ""}, + {Value(` [ "a` + "\xff" + `2" ] `), E(jsonwire.ErrInvalidUTF8).withPos(`[ [ "a`, "/0/0"), ""}, + {BeginArray, nil, "/0"}, + {Value(` "a` + "\xff" + `2" `), E(jsonwire.ErrInvalidUTF8).withPos(`[[ "a`, "/0/0"), "/0"}, + {String(`a` + "\xff" + `2`), E(jsonwire.ErrInvalidUTF8).withPos(`[[`, "/0/0"), "/0"}, + }, + wantOut: `[[`, +}, { + name: jsontest.Name("ErrorPosition/1/0"), + calls: []encoderMethodCall{ + {Value(` [ "a1" , [ "a` + "\xff" + `2" ] ] `), E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , [ "a`, "/1/0"), ""}, + {BeginArray, nil, ""}, + {String("a1"), nil, "/0"}, + {Value(` [ "a` + "\xff" + `2" ] `), E(jsonwire.ErrInvalidUTF8).withPos(`["a1", [ "a`, "/1/0"), ""}, + {BeginArray, nil, "/1"}, + {Value(` "a` + "\xff" + `2" `), E(jsonwire.ErrInvalidUTF8).withPos(`["a1",[ "a`, "/1/0"), "/1"}, + {String(`a` + "\xff" + `2`), E(jsonwire.ErrInvalidUTF8).withPos(`["a1",[`, "/1/0"), "/1"}, + }, + wantOut: `["a1",[`, +}, { + name: jsontest.Name("ErrorPosition/0/1"), + calls: []encoderMethodCall{ + {Value(` [ [ "a2" , "b` + "\xff" + `2" ] ] `), E(jsonwire.ErrInvalidUTF8).withPos(` [ [ "a2" , "b`, "/0/1"), ""}, + {BeginArray, nil, ""}, + {Value(` [ "a2" , "b` + "\xff" + `2" ] `), E(jsonwire.ErrInvalidUTF8).withPos(`[ [ "a2" , "b`, "/0/1"), ""}, + {BeginArray, nil, "/0"}, + {String("a2"), nil, "/0/0"}, + {Value(` "b` + "\xff" + `2" `), E(jsonwire.ErrInvalidUTF8).withPos(`[["a2", "b`, "/0/1"), "/0/0"}, + {String(`b` + "\xff" + `2`), E(jsonwire.ErrInvalidUTF8).withPos(`[["a2",`, "/0/1"), "/0/0"}, + }, + wantOut: `[["a2"`, +}, { + name: jsontest.Name("ErrorPosition/1/1"), + calls: []encoderMethodCall{ + {Value(` [ "a1" , [ "a2" , "b` + "\xff" + `2" ] ] `), E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , [ "a2" , "b`, "/1/1"), ""}, + {BeginArray, nil, ""}, + {String("a1"), nil, "/0"}, + {Value(` [ "a2" , "b` + "\xff" + `2" ] `), E(jsonwire.ErrInvalidUTF8).withPos(`["a1", [ "a2" , "b`, "/1/1"), ""}, + {BeginArray, nil, "/1"}, + {String("a2"), nil, "/1/0"}, + {Value(` "b` + "\xff" + `2" `), E(jsonwire.ErrInvalidUTF8).withPos(`["a1",["a2", "b`, "/1/1"), "/1/0"}, + {String(`b` + "\xff" + `2`), E(jsonwire.ErrInvalidUTF8).withPos(`["a1",["a2",`, "/1/1"), "/1/0"}, + }, + wantOut: `["a1",["a2"`, +}, { + name: jsontest.Name("ErrorPosition/a1-"), + calls: []encoderMethodCall{ + {Value(` { "a` + "\xff" + `1" : "b1" } `), E(jsonwire.ErrInvalidUTF8).withPos(` { "a`, ""), ""}, + {BeginObject, nil, ""}, + {Value(` "a` + "\xff" + `1" `), E(jsonwire.ErrInvalidUTF8).withPos(`{ "a`, ""), ""}, + {String(`a` + "\xff" + `1`), E(jsonwire.ErrInvalidUTF8).withPos(`{`, ""), ""}, + }, + wantOut: `{`, +}, { + name: jsontest.Name("ErrorPosition/a1"), + calls: []encoderMethodCall{ + {Value(` { "a1" : "b` + "\xff" + `1" } `), E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b`, "/a1"), ""}, + {BeginObject, nil, ""}, + {String("a1"), nil, "/a1"}, + {Value(` "b` + "\xff" + `1" `), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1": "b`, "/a1"), ""}, + {String(`b` + "\xff" + `1`), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":`, "/a1"), ""}, + }, + wantOut: `{"a1"`, +}, { + name: jsontest.Name("ErrorPosition/c1-"), + calls: []encoderMethodCall{ + {Value(` { "a1" : "b1" , "c` + "\xff" + `1" : "d1" } `), E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b1" , "c`, ""), ""}, + {BeginObject, nil, ""}, + {String("a1"), nil, "/a1"}, + {String("b1"), nil, "/a1"}, + {Value(` "c` + "\xff" + `1" `), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":"b1": "c`, ""), "/a1"}, + {String(`c` + "\xff" + `1`), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":"b1":`, ""), "/a1"}, + }, + wantOut: `{"a1":"b1"`, +}, { + name: jsontest.Name("ErrorPosition/c1"), + calls: []encoderMethodCall{ + {Value(` { "a1" : "b1" , "c1" : "d` + "\xff" + `1" } `), E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b1" , "c1" : "d`, "/c1"), ""}, + {BeginObject, nil, ""}, + {String("a1"), nil, "/a1"}, + {String("b1"), nil, "/a1"}, + {String("c1"), nil, "/c1"}, + {Value(` "d` + "\xff" + `1" `), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":"b1":"c1": "d`, "/c1"), "/c1"}, + {String(`d` + "\xff" + `1`), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":"b1":"c1":`, "/c1"), "/c1"}, + }, + wantOut: `{"a1":"b1","c1"`, +}, { + name: jsontest.Name("ErrorPosition/a1/a2-"), + calls: []encoderMethodCall{ + {Value(` { "a1" : { "a` + "\xff" + `2" : "b2" } } `), E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a`, "/a1"), ""}, + {BeginObject, nil, ""}, + {String("a1"), nil, "/a1"}, + {Value(` { "a` + "\xff" + `2" : "b2" } `), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1": { "a`, "/a1"), ""}, + {BeginObject, nil, "/a1"}, + {Value(` "a` + "\xff" + `2" `), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":{ "a`, "/a1"), "/a1"}, + {String(`a` + "\xff" + `2`), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":{`, "/a1"), "/a1"}, + }, + wantOut: `{"a1":{`, +}, { + name: jsontest.Name("ErrorPosition/a1/a2"), + calls: []encoderMethodCall{ + {Value(` { "a1" : { "a2" : "b` + "\xff" + `2" } } `), E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a2" : "b`, "/a1/a2"), ""}, + {BeginObject, nil, ""}, + {String("a1"), nil, "/a1"}, + {Value(` { "a2" : "b` + "\xff" + `2" } `), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1": { "a2" : "b`, "/a1/a2"), ""}, + {BeginObject, nil, "/a1"}, + {String("a2"), nil, "/a1/a2"}, + {Value(` "b` + "\xff" + `2" `), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":{"a2": "b`, "/a1/a2"), "/a1/a2"}, + {String(`b` + "\xff" + `2`), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":{"a2":`, "/a1/a2"), "/a1/a2"}, + }, + wantOut: `{"a1":{"a2"`, +}, { + name: jsontest.Name("ErrorPosition/a1/c2-"), + calls: []encoderMethodCall{ + {Value(` { "a1" : { "a2" : "b2" , "c` + "\xff" + `2" : "d2" } } `), E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a2" : "b2" , "c`, "/a1"), ""}, + {BeginObject, nil, ""}, + {String("a1"), nil, "/a1"}, + {BeginObject, nil, "/a1"}, + {String("a2"), nil, "/a1/a2"}, + {String("b2"), nil, "/a1/a2"}, + {Value(` "c` + "\xff" + `2" `), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":{"a2":"b2", "c`, "/a1"), "/a1/a2"}, + {String(`c` + "\xff" + `2`), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":{"a2":"b2",`, "/a1"), "/a1/a2"}, + }, + wantOut: `{"a1":{"a2":"b2"`, +}, { + name: jsontest.Name("ErrorPosition/a1/c2"), + calls: []encoderMethodCall{ + {Value(` { "a1" : { "a2" : "b2" , "c2" : "d` + "\xff" + `2" } } `), E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : { "a2" : "b2" , "c2" : "d`, "/a1/c2"), ""}, + {BeginObject, nil, ""}, + {String("a1"), nil, "/a1"}, + {Value(` { "a2" : "b2" , "c2" : "d` + "\xff" + `2" } `), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1": { "a2" : "b2" , "c2" : "d`, "/a1/c2"), ""}, + {BeginObject, nil, ""}, + {String("a2"), nil, "/a1/a2"}, + {String("b2"), nil, "/a1/a2"}, + {String("c2"), nil, "/a1/c2"}, + {Value(` "d` + "\xff" + `2" `), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":{"a2":"b2","c2": "d`, "/a1/c2"), "/a1/c2"}, + {String(`d` + "\xff" + `2`), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":{"a2":"b2","c2":`, "/a1/c2"), "/a1/c2"}, + }, + wantOut: `{"a1":{"a2":"b2","c2"`, +}, { + name: jsontest.Name("ErrorPosition/1/a2"), + calls: []encoderMethodCall{ + {Value(` [ "a1" , { "a2" : "b` + "\xff" + `2" } ] `), E(jsonwire.ErrInvalidUTF8).withPos(` [ "a1" , { "a2" : "b`, "/1/a2"), ""}, + {BeginArray, nil, ""}, + {String("a1"), nil, "/0"}, + {Value(` { "a2" : "b` + "\xff" + `2" } `), E(jsonwire.ErrInvalidUTF8).withPos(`["a1", { "a2" : "b`, "/1/a2"), ""}, + {BeginObject, nil, "/1"}, + {String("a2"), nil, "/1/a2"}, + {Value(` "b` + "\xff" + `2" `), E(jsonwire.ErrInvalidUTF8).withPos(`["a1",{"a2": "b`, "/1/a2"), "/1/a2"}, + {String(`b` + "\xff" + `2`), E(jsonwire.ErrInvalidUTF8).withPos(`["a1",{"a2":`, "/1/a2"), "/1/a2"}, + }, + wantOut: `["a1",{"a2"`, +}, { + name: jsontest.Name("ErrorPosition/c1/1"), + calls: []encoderMethodCall{ + {Value(` { "a1" : "b1" , "c1" : [ "a2" , "b` + "\xff" + `2" ] } `), E(jsonwire.ErrInvalidUTF8).withPos(` { "a1" : "b1" , "c1" : [ "a2" , "b`, "/c1/1"), ""}, + {BeginObject, nil, ""}, + {String("a1"), nil, "/a1"}, + {String("b1"), nil, "/a1"}, + {String("c1"), nil, "/c1"}, + {Value(` [ "a2" , "b` + "\xff" + `2" ] `), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":"b1","c1": [ "a2" , "b`, "/c1/1"), ""}, + {BeginArray, nil, "/c1"}, + {String("a2"), nil, "/c1/0"}, + {Value(` "b` + "\xff" + `2" `), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":"b1","c1":["a2", "b`, "/c1/1"), "/c1/0"}, + {String(`b` + "\xff" + `2`), E(jsonwire.ErrInvalidUTF8).withPos(`{"a1":"b1","c1":["a2",`, "/c1/1"), "/c1/0"}, + }, + wantOut: `{"a1":"b1","c1":["a2"`, +}, { + name: jsontest.Name("ErrorPosition/0/a1/1/c3/1"), + calls: []encoderMethodCall{ + {Value(` [ { "a1" : [ "a2" , { "a3" : "b3" , "c3" : [ "a4" , "b` + "\xff" + `4" ] } ] } ] `), E(jsonwire.ErrInvalidUTF8).withPos(` [ { "a1" : [ "a2" , { "a3" : "b3" , "c3" : [ "a4" , "b`, "/0/a1/1/c3/1"), ""}, + {BeginArray, nil, ""}, + {Value(` { "a1" : [ "a2" , { "a3" : "b3" , "c3" : [ "a4" , "b` + "\xff" + `4" ] } ] } `), E(jsonwire.ErrInvalidUTF8).withPos(`[ { "a1" : [ "a2" , { "a3" : "b3" , "c3" : [ "a4" , "b`, "/0/a1/1/c3/1"), ""}, + {BeginObject, nil, "/0"}, + {String("a1"), nil, "/0/a1"}, + {Value(` [ "a2" , { "a3" : "b3" , "c3" : [ "a4" , "b` + "\xff" + `4" ] } ] `), E(jsonwire.ErrInvalidUTF8).withPos(`[{"a1": [ "a2" , { "a3" : "b3" , "c3" : [ "a4" , "b`, "/0/a1/1/c3/1"), ""}, + {BeginArray, nil, ""}, + {String("a2"), nil, "/0/a1/0"}, + {Value(` { "a3" : "b3" , "c3" : [ "a4" , "b` + "\xff" + `4" ] } `), E(jsonwire.ErrInvalidUTF8).withPos(`[{"a1":["a2", { "a3" : "b3" , "c3" : [ "a4" , "b`, "/0/a1/1/c3/1"), ""}, + {BeginObject, nil, "/0/a1/1"}, + {String("a3"), nil, "/0/a1/1/a3"}, + {String("b3"), nil, "/0/a1/1/a3"}, + {String("c3"), nil, "/0/a1/1/c3"}, + {Value(` [ "a4" , "b` + "\xff" + `4" ] `), E(jsonwire.ErrInvalidUTF8).withPos(`[{"a1":["a2",{"a3":"b3","c3": [ "a4" , "b`, "/0/a1/1/c3/1"), ""}, + {BeginArray, nil, "/0/a1/1/c3"}, + {String("a4"), nil, "/0/a1/1/c3/0"}, + {Value(` "b` + "\xff" + `4" `), E(jsonwire.ErrInvalidUTF8).withPos(`[{"a1":["a2",{"a3":"b3","c3":["a4", "b`, "/0/a1/1/c3/1"), "/0/a1/1/c3/0"}, + {String(`b` + "\xff" + `4`), E(jsonwire.ErrInvalidUTF8).withPos(`[{"a1":["a2",{"a3":"b3","c3":["a4",`, "/0/a1/1/c3/1"), "/0/a1/1/c3/0"}, + }, + wantOut: `[{"a1":["a2",{"a3":"b3","c3":["a4"`, +}} + +// TestEncoderErrors test that Encoder errors occur when we expect and +// leaves the Encoder in a consistent state. +func TestEncoderErrors(t *testing.T) { + for _, td := range encoderErrorTestdata { + t.Run(path.Join(td.name.Name), func(t *testing.T) { + testEncoderErrors(t, td.name.Where, td.opts, td.calls, td.wantOut) + }) + } +} +func testEncoderErrors(t *testing.T, where jsontest.CasePos, opts []Options, calls []encoderMethodCall, wantOut string) { + dst := new(bytes.Buffer) + enc := NewEncoder(dst, opts...) + for i, call := range calls { + var gotErr error + switch tokVal := call.in.(type) { + case Token: + gotErr = enc.WriteToken(tokVal) + case Value: + gotErr = enc.WriteValue(tokVal) + } + if !equalError(gotErr, call.wantErr) { + t.Fatalf("%s: %d: error mismatch:\ngot %v\nwant %v", where, i, gotErr, call.wantErr) + } + if call.wantPointer != "" { + gotPointer := enc.StackPointer() + if gotPointer != call.wantPointer { + t.Fatalf("%s: %d: Encoder.StackPointer = %s, want %s", where, i, gotPointer, call.wantPointer) + } + } + } + gotOut := dst.String() + string(enc.s.unflushedBuffer()) + if gotOut != wantOut { + t.Fatalf("%s: output mismatch:\ngot %q\nwant %q", where, gotOut, wantOut) + } + gotOffset := int(enc.OutputOffset()) + wantOffset := len(wantOut) + if gotOffset != wantOffset { + t.Fatalf("%s: Encoder.OutputOffset = %v, want %v", where, gotOffset, wantOffset) + } +} + +// TestEncoderReset tests that the encoder preserves its internal +// buffer between Reset calls to avoid frequent allocations when reusing the encoder. +// It ensures that the buffer capacity is maintained while avoiding aliasing +// issues with [bytes.Buffer]. +func TestEncoderReset(t *testing.T) { + // Create an encoder with a reasonably large JSON input to ensure buffer growth. + largeJSON := `{"key1":"value1","key2":"value2","key3":"value3","key4":"value4","key5":"value5"}` + "\n" + bb := new(bytes.Buffer) + enc := NewEncoder(struct{ io.Writer }{bb}) // mask out underlying [bytes.Buffer] + + t.Run("Test capacity preservation", func(t *testing.T) { + // Write the first JSON value to grow the internal buffer. + err := enc.WriteValue(append(enc.AvailableBuffer(), largeJSON...)) + if err != nil { + t.Fatalf("first WriteValue failed: %v", err) + } + if bb.String() != largeJSON { + t.Fatalf("first WriteValue = %q, want %q", bb.String(), largeJSON) + } + + // Get the buffer capacity after first use. + initialCapacity := cap(enc.s.Buf) + initialCacheCapacity := cap(enc.s.availBuffer) + if initialCapacity == 0 { + t.Fatalf("expected non-zero buffer capacity after first use") + } + if initialCacheCapacity == 0 { + t.Fatalf("expected non-zero cache capacity after first use") + } + + // Reset with a new writer - this should preserve the buffer capacity. + bb.Reset() + enc.Reset(struct{ io.Writer }{bb}) + + // Verify the buffer capacity is preserved (or at least not smaller). + preservedCapacity := cap(enc.s.Buf) + if preservedCapacity < initialCapacity { + t.Fatalf("buffer capacity reduced after Reset: got %d, want at least %d", preservedCapacity, initialCapacity) + } + preservedCacheCapacity := cap(enc.s.availBuffer) + if preservedCacheCapacity < initialCacheCapacity { + t.Fatalf("cache capacity reduced after Reset: got %d, want at least %d", preservedCapacity, initialCapacity) + } + + // Write the second JSON value to ensure the encoder still works correctly. + err = enc.WriteValue(append(enc.AvailableBuffer(), largeJSON...)) + if err != nil { + t.Fatalf("second WriteValue failed: %v", err) + } + if bb.String() != largeJSON { + t.Fatalf("second WriteValue = %q, want %q", bb.String(), largeJSON) + } + }) + + t.Run("Test aliasing with bytes.Buffer", func(t *testing.T) { + // Test with bytes.Buffer to verify proper aliasing behavior. + bb.Reset() + enc.Reset(bb) + + // Write the third JSON value to ensure functionality with bytes.Buffer. + err := enc.WriteValue([]byte(largeJSON)) + if err != nil { + t.Fatalf("fourth WriteValue failed: %v", err) + } + if bb.String() != largeJSON { + t.Fatalf("fourth WriteValue = %q, want %q", bb.String(), largeJSON) + } + // The encoder buffer should alias bytes.Buffer's internal buffer. + if cap(enc.s.Buf) == 0 || cap(bb.AvailableBuffer()) == 0 || &enc.s.Buf[:1][0] != &bb.AvailableBuffer()[:1][0] { + t.Fatalf("encoder buffer does not alias bytes.Buffer") + } + }) + + t.Run("Test aliasing removed after Reset", func(t *testing.T) { + // Reset with a new reader and verify the buffer is not aliased. + bb.Reset() + enc.Reset(struct{ io.Writer }{bb}) + err := enc.WriteValue([]byte(largeJSON)) + if err != nil { + t.Fatalf("fifth WriteValue failed: %v", err) + } + if bb.String() != largeJSON { + t.Fatalf("fourth WriteValue = %q, want %q", bb.String(), largeJSON) + } + + // The encoder buffer should not alias the bytes.Buffer's internal buffer. + if cap(enc.s.Buf) == 0 || cap(bb.AvailableBuffer()) == 0 || &enc.s.Buf[:1][0] == &bb.AvailableBuffer()[:1][0] { + t.Fatalf("encoder buffer aliases bytes.Buffer") + } + }) +} diff --git a/src/encoding/json/jsontext/errors.go b/src/encoding/json/jsontext/errors.go new file mode 100644 index 00000000000000..4b95d03f409e2e --- /dev/null +++ b/src/encoding/json/jsontext/errors.go @@ -0,0 +1,182 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "bytes" + "io" + "strconv" + + "encoding/json/internal/jsonwire" +) + +const errorPrefix = "jsontext: " + +type ioError struct { + action string // either "read" or "write" + err error +} + +func (e *ioError) Error() string { + return errorPrefix + e.action + " error: " + e.err.Error() +} +func (e *ioError) Unwrap() error { + return e.err +} + +// SyntacticError is a description of a syntactic error that occurred when +// encoding or decoding JSON according to the grammar. +// +// The contents of this error as produced by this package may change over time. +type SyntacticError struct { + requireKeyedLiterals + nonComparable + + // ByteOffset indicates that an error occurred after this byte offset. + ByteOffset int64 + // JSONPointer indicates that an error occurred within this JSON value + // as indicated using the JSON Pointer notation (see RFC 6901). + JSONPointer Pointer + + // Err is the underlying error. + Err error +} + +// wrapSyntacticError wraps an error and annotates it with a precise location +// using the provided [encoderState] or [decoderState]. +// If err is an [ioError] or [io.EOF], then it is not wrapped. +// +// It takes a relative offset pos that can be resolved into +// an absolute offset using state.offsetAt. +// +// It takes a where that specify how the JSON pointer is derived. +// If the underlying error is a [pointerSuffixError], +// then the suffix is appended to the derived pointer. +func wrapSyntacticError(state interface { + offsetAt(pos int) int64 + AppendStackPointer(b []byte, where int) []byte +}, err error, pos, where int) error { + if _, ok := err.(*ioError); err == io.EOF || ok { + return err + } + offset := state.offsetAt(pos) + ptr := state.AppendStackPointer(nil, where) + if serr, ok := err.(*pointerSuffixError); ok { + ptr = serr.appendPointer(ptr) + err = serr.error + } + if d, ok := state.(*decoderState); ok && err == errMismatchDelim { + where := "at start of value" + if len(d.Tokens.Stack) > 0 && d.Tokens.Last.Length() > 0 { + switch { + case d.Tokens.Last.isArray(): + where = "after array element (expecting ',' or ']')" + ptr = []byte(Pointer(ptr).Parent()) // problem is with parent array + case d.Tokens.Last.isObject(): + where = "after object value (expecting ',' or '}')" + ptr = []byte(Pointer(ptr).Parent()) // problem is with parent object + } + } + err = jsonwire.NewInvalidCharacterError(d.buf[pos:], where) + } + return &SyntacticError{ByteOffset: offset, JSONPointer: Pointer(ptr), Err: err} +} + +func (e *SyntacticError) Error() string { + pointer := e.JSONPointer + offset := e.ByteOffset + b := []byte(errorPrefix) + if e.Err != nil { + b = append(b, e.Err.Error()...) + if e.Err == ErrDuplicateName { + b = strconv.AppendQuote(append(b, ' '), pointer.LastToken()) + pointer = pointer.Parent() + offset = 0 // not useful to print offset for duplicate names + } + } else { + b = append(b, "syntactic error"...) + } + if pointer != "" { + b = strconv.AppendQuote(append(b, " within "...), jsonwire.TruncatePointer(string(pointer), 100)) + } + if offset > 0 { + b = strconv.AppendInt(append(b, " after offset "...), offset, 10) + } + return string(b) +} + +func (e *SyntacticError) Unwrap() error { + return e.Err +} + +// pointerSuffixError represents a JSON pointer suffix to be appended +// to [SyntacticError.JSONPointer]. It is an internal error type +// used within this package and does not appear in the public API. +// +// This type is primarily used to annotate errors in Encoder.WriteValue +// and Decoder.ReadValue with precise positions. +// At the time WriteValue or ReadValue is called, a JSON pointer to the +// upcoming value can be constructed using the Encoder/Decoder state. +// However, tracking pointers within values during normal operation +// would incur a performance penalty in the error-free case. +// +// To provide precise error locations without this overhead, +// the error is wrapped with object names or array indices +// as the call stack is popped when an error occurs. +// Since this happens in reverse order, pointerSuffixError holds +// the pointer in reverse and is only later reversed when appending to +// the pointer prefix. +// +// For example, if the encoder is at "/alpha/bravo/charlie" +// and an error occurs in WriteValue at "/xray/yankee/zulu", then +// the final pointer should be "/alpha/bravo/charlie/xray/yankee/zulu". +// +// As pointerSuffixError is populated during the error return path, +// it first contains "/zulu", then "/zulu/yankee", +// and finally "/zulu/yankee/xray". +// These tokens are reversed and concatenated to "/alpha/bravo/charlie" +// to form the full pointer. +type pointerSuffixError struct { + error + + // reversePointer is a JSON pointer, but with each token in reverse order. + reversePointer []byte +} + +// wrapWithObjectName wraps err with a JSON object name access, +// which must be a valid quoted JSON string. +func wrapWithObjectName(err error, quotedName []byte) error { + serr, _ := err.(*pointerSuffixError) + if serr == nil { + serr = &pointerSuffixError{error: err} + } + name := jsonwire.UnquoteMayCopy(quotedName, false) + serr.reversePointer = appendEscapePointerName(append(serr.reversePointer, '/'), name) + return serr +} + +// wrapWithArrayIndex wraps err with a JSON array index access. +func wrapWithArrayIndex(err error, index int64) error { + serr, _ := err.(*pointerSuffixError) + if serr == nil { + serr = &pointerSuffixError{error: err} + } + serr.reversePointer = strconv.AppendUint(append(serr.reversePointer, '/'), uint64(index), 10) + return serr +} + +// appendPointer appends the path encoded in e to the end of pointer. +func (e *pointerSuffixError) appendPointer(pointer []byte) []byte { + // Copy each token in reversePointer to the end of pointer in reverse order. + // Double reversal means that the appended suffix is now in forward order. + bi, bo := e.reversePointer, pointer + for len(bi) > 0 { + i := bytes.LastIndexByte(bi, '/') + bi, bo = bi[:i], append(bo, bi[i:]...) + } + return bo +} diff --git a/src/encoding/json/jsontext/example_test.go b/src/encoding/json/jsontext/example_test.go new file mode 100644 index 00000000000000..4bf6a7ae5aef0b --- /dev/null +++ b/src/encoding/json/jsontext/example_test.go @@ -0,0 +1,130 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext_test + +import ( + "bytes" + "fmt" + "io" + "log" + "strings" + + "encoding/json/jsontext" + "encoding/json/v2" +) + +// This example demonstrates the use of the [Encoder] and [Decoder] to +// parse and modify JSON without unmarshaling it into a concrete Go type. +func Example_stringReplace() { + // Example input with non-idiomatic use of "Golang" instead of "Go". + const input = `{ + "title": "Golang version 1 is released", + "author": "Andrew Gerrand", + "date": "2012-03-28", + "text": "Today marks a major milestone in the development of the Golang programming language.", + "otherArticles": [ + "Twelve Years of Golang", + "The Laws of Reflection", + "Learn Golang from your browser" + ] + }` + + // Using a Decoder and Encoder, we can parse through every token, + // check and modify the token if necessary, and + // write the token to the output. + var replacements []jsontext.Pointer + in := strings.NewReader(input) + dec := jsontext.NewDecoder(in) + out := new(bytes.Buffer) + enc := jsontext.NewEncoder(out, jsontext.Multiline(true)) // expand for readability + for { + // Read a token from the input. + tok, err := dec.ReadToken() + if err != nil { + if err == io.EOF { + break + } + log.Fatal(err) + } + + // Check whether the token contains the string "Golang" and + // replace each occurrence with "Go" instead. + if tok.Kind() == '"' && strings.Contains(tok.String(), "Golang") { + replacements = append(replacements, dec.StackPointer()) + tok = jsontext.String(strings.ReplaceAll(tok.String(), "Golang", "Go")) + } + + // Write the (possibly modified) token to the output. + if err := enc.WriteToken(tok); err != nil { + log.Fatal(err) + } + } + + // Print the list of replacements and the adjusted JSON output. + if len(replacements) > 0 { + fmt.Println(`Replaced "Golang" with "Go" in:`) + for _, where := range replacements { + fmt.Println("\t" + where) + } + fmt.Println() + } + fmt.Println("Result:", out.String()) + + // Output: + // Replaced "Golang" with "Go" in: + // /title + // /text + // /otherArticles/0 + // /otherArticles/2 + // + // Result: { + // "title": "Go version 1 is released", + // "author": "Andrew Gerrand", + // "date": "2012-03-28", + // "text": "Today marks a major milestone in the development of the Go programming language.", + // "otherArticles": [ + // "Twelve Years of Go", + // "The Laws of Reflection", + // "Learn Go from your browser" + // ] + // } +} + +// Directly embedding JSON within HTML requires special handling for safety. +// Escape certain runes to prevent JSON directly treated as HTML +// from being able to perform `, + } + + b, err := json.Marshal(&page, + // Escape certain runes within a JSON string so that + // JSON will be safe to directly embed inside HTML. + jsontext.EscapeForHTML(true), + jsontext.EscapeForJS(true), + jsontext.Multiline(true)) // expand for readability + if err != nil { + log.Fatal(err) + } + fmt.Println(string(b)) + + // Output: + // { + // "Title": "Example Embedded Javascript", + // "Body": "\u003cscript\u003e console.log(\"Hello, world!\"); \u003c/script\u003e" + // } +} diff --git a/src/encoding/json/jsontext/export.go b/src/encoding/json/jsontext/export.go new file mode 100644 index 00000000000000..0ecccad5b3c671 --- /dev/null +++ b/src/encoding/json/jsontext/export.go @@ -0,0 +1,77 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "io" + + "encoding/json/internal" +) + +// Internal is for internal use only. +// This is exempt from the Go compatibility agreement. +var Internal exporter + +type exporter struct{} + +// Export exposes internal functionality from "jsontext" to "json". +// This cannot be dynamically called by other packages since +// they cannot obtain a reference to the internal.AllowInternalUse value. +func (exporter) Export(p *internal.NotForPublicUse) export { + if p != &internal.AllowInternalUse { + panic("unauthorized call to Export") + } + return export{} +} + +// The export type exposes functionality to packages with visibility to +// the internal.AllowInternalUse variable. The "json" package uses this +// to modify low-level state in the Encoder and Decoder types. +// It mutates the state directly instead of calling ReadToken or WriteToken +// since this is more performant. The public APIs need to track state to ensure +// that users are constructing a valid JSON value, but the "json" implementation +// guarantees that it emits valid JSON by the structure of the code itself. +type export struct{} + +// Encoder returns a pointer to the underlying encoderState. +func (export) Encoder(e *Encoder) *encoderState { return &e.s } + +// Decoder returns a pointer to the underlying decoderState. +func (export) Decoder(d *Decoder) *decoderState { return &d.s } + +func (export) GetBufferedEncoder(o ...Options) *Encoder { + return getBufferedEncoder(o...) +} +func (export) PutBufferedEncoder(e *Encoder) { + putBufferedEncoder(e) +} + +func (export) GetStreamingEncoder(w io.Writer, o ...Options) *Encoder { + return getStreamingEncoder(w, o...) +} +func (export) PutStreamingEncoder(e *Encoder) { + putStreamingEncoder(e) +} + +func (export) GetBufferedDecoder(b []byte, o ...Options) *Decoder { + return getBufferedDecoder(b, o...) +} +func (export) PutBufferedDecoder(d *Decoder) { + putBufferedDecoder(d) +} + +func (export) GetStreamingDecoder(r io.Reader, o ...Options) *Decoder { + return getStreamingDecoder(r, o...) +} +func (export) PutStreamingDecoder(d *Decoder) { + putStreamingDecoder(d) +} + +func (export) IsIOError(err error) bool { + _, ok := err.(*ioError) + return ok +} diff --git a/src/encoding/json/jsontext/fuzz_test.go b/src/encoding/json/jsontext/fuzz_test.go new file mode 100644 index 00000000000000..3ad181d43416b8 --- /dev/null +++ b/src/encoding/json/jsontext/fuzz_test.go @@ -0,0 +1,237 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "bytes" + "errors" + "io" + "math/rand" + "slices" + "testing" + + "encoding/json/internal/jsontest" +) + +func FuzzCoder(f *testing.F) { + // Add a number of inputs to the corpus including valid and invalid data. + for _, td := range coderTestdata { + f.Add(int64(0), []byte(td.in)) + } + for _, td := range decoderErrorTestdata { + f.Add(int64(0), []byte(td.in)) + } + for _, td := range encoderErrorTestdata { + f.Add(int64(0), []byte(td.wantOut)) + } + for _, td := range jsontest.Data { + f.Add(int64(0), td.Data()) + } + + f.Fuzz(func(t *testing.T, seed int64, b []byte) { + var tokVals []tokOrVal + rn := rand.NewSource(seed) + + // Read a sequence of tokens or values. Skip the test for any errors + // since we expect this with randomly generated fuzz inputs. + src := bytes.NewReader(b) + dec := NewDecoder(src) + for { + if rn.Int63()%8 > 0 { + tok, err := dec.ReadToken() + if err != nil { + if err == io.EOF { + break + } + t.Skipf("Decoder.ReadToken error: %v", err) + } + tokVals = append(tokVals, tok.Clone()) + } else { + val, err := dec.ReadValue() + if err != nil { + if expectError := dec.PeekKind() == '}' || dec.PeekKind() == ']'; expectError { + if _, ok := errors.AsType[*SyntacticError](err); ok { + continue + } + } + if err == io.EOF { + break + } + t.Skipf("Decoder.ReadValue error: %v", err) + } + tokVals = append(tokVals, append(zeroValue, val...)) + } + } + + // Write a sequence of tokens or values. Fail the test for any errors + // since the previous stage guarantees that the input is valid. + dst := new(bytes.Buffer) + enc := NewEncoder(dst) + for _, tokVal := range tokVals { + switch tokVal := tokVal.(type) { + case Token: + if err := enc.WriteToken(tokVal); err != nil { + t.Fatalf("Encoder.WriteToken error: %v", err) + } + case Value: + if err := enc.WriteValue(tokVal); err != nil { + t.Fatalf("Encoder.WriteValue error: %v", err) + } + } + } + + // Encoded output and original input must decode to the same thing. + var got, want []Token + for dec := NewDecoder(bytes.NewReader(b)); dec.PeekKind() > 0; { + tok, err := dec.ReadToken() + if err != nil { + t.Fatalf("Decoder.ReadToken error: %v", err) + } + got = append(got, tok.Clone()) + } + for dec := NewDecoder(dst); dec.PeekKind() > 0; { + tok, err := dec.ReadToken() + if err != nil { + t.Fatalf("Decoder.ReadToken error: %v", err) + } + want = append(want, tok.Clone()) + } + if !equalTokens(got, want) { + t.Fatalf("mismatching output:\ngot %v\nwant %v", got, want) + } + }) +} + +func FuzzResumableDecoder(f *testing.F) { + for _, td := range resumableDecoderTestdata { + f.Add(int64(0), []byte(td)) + } + + f.Fuzz(func(t *testing.T, seed int64, b []byte) { + rn := rand.NewSource(seed) + + // Regardless of how many bytes the underlying io.Reader produces, + // the provided tokens, values, and errors should always be identical. + t.Run("ReadToken", func(t *testing.T) { + decGot := NewDecoder(&FaultyBuffer{B: b, MaxBytes: 8, Rand: rn}) + decWant := NewDecoder(bytes.NewReader(b)) + gotTok, gotErr := decGot.ReadToken() + wantTok, wantErr := decWant.ReadToken() + if gotTok.String() != wantTok.String() || !equalError(gotErr, wantErr) { + t.Errorf("Decoder.ReadToken = (%v, %v), want (%v, %v)", gotTok, gotErr, wantTok, wantErr) + } + }) + t.Run("ReadValue", func(t *testing.T) { + decGot := NewDecoder(&FaultyBuffer{B: b, MaxBytes: 8, Rand: rn}) + decWant := NewDecoder(bytes.NewReader(b)) + gotVal, gotErr := decGot.ReadValue() + wantVal, wantErr := decWant.ReadValue() + if !slices.Equal(gotVal, wantVal) || !equalError(gotErr, wantErr) { + t.Errorf("Decoder.ReadValue = (%s, %v), want (%s, %v)", gotVal, gotErr, wantVal, wantErr) + } + }) + }) +} + +func FuzzValueFormat(f *testing.F) { + for _, td := range valueTestdata { + f.Add(int64(0), []byte(td.in)) + } + + // isValid reports whether b is valid according to the specified options. + isValid := func(b []byte, opts ...Options) bool { + d := NewDecoder(bytes.NewReader(b), opts...) + _, errVal := d.ReadValue() + _, errEOF := d.ReadToken() + return errVal == nil && errEOF == io.EOF + } + + // stripWhitespace removes all JSON whitespace characters from the input. + stripWhitespace := func(in []byte) (out []byte) { + out = make([]byte, 0, len(in)) + for _, c := range in { + switch c { + case ' ', '\n', '\r', '\t': + default: + out = append(out, c) + } + } + return out + } + + allOptions := []Options{ + AllowDuplicateNames(true), + AllowInvalidUTF8(true), + EscapeForHTML(true), + EscapeForJS(true), + PreserveRawStrings(true), + CanonicalizeRawInts(true), + CanonicalizeRawFloats(true), + ReorderRawObjects(true), + SpaceAfterColon(true), + SpaceAfterComma(true), + Multiline(true), + WithIndent("\t"), + WithIndentPrefix(" "), + } + + f.Fuzz(func(t *testing.T, seed int64, b []byte) { + validRFC7159 := isValid(b, AllowInvalidUTF8(true), AllowDuplicateNames(true)) + validRFC8259 := isValid(b, AllowInvalidUTF8(false), AllowDuplicateNames(true)) + validRFC7493 := isValid(b, AllowInvalidUTF8(false), AllowDuplicateNames(false)) + switch { + case !validRFC7159 && validRFC8259: + t.Errorf("invalid input per RFC 7159 implies invalid per RFC 8259") + case !validRFC8259 && validRFC7493: + t.Errorf("invalid input per RFC 8259 implies invalid per RFC 7493") + } + + gotValid := Value(b).IsValid() + wantValid := validRFC7493 + if gotValid != wantValid { + t.Errorf("Value.IsValid = %v, want %v", gotValid, wantValid) + } + + gotCompacted := Value(string(b)) + gotCompactOk := gotCompacted.Compact() == nil + wantCompactOk := validRFC7159 + if !bytes.Equal(stripWhitespace(gotCompacted), stripWhitespace(b)) { + t.Errorf("stripWhitespace(Value.Compact) = %s, want %s", stripWhitespace(gotCompacted), stripWhitespace(b)) + } + if gotCompactOk != wantCompactOk { + t.Errorf("Value.Compact success mismatch: got %v, want %v", gotCompactOk, wantCompactOk) + } + + gotIndented := Value(string(b)) + gotIndentOk := gotIndented.Indent() == nil + wantIndentOk := validRFC7159 + if !bytes.Equal(stripWhitespace(gotIndented), stripWhitespace(b)) { + t.Errorf("stripWhitespace(Value.Indent) = %s, want %s", stripWhitespace(gotIndented), stripWhitespace(b)) + } + if gotIndentOk != wantIndentOk { + t.Errorf("Value.Indent success mismatch: got %v, want %v", gotIndentOk, wantIndentOk) + } + + gotCanonicalized := Value(string(b)) + gotCanonicalizeOk := gotCanonicalized.Canonicalize() == nil + wantCanonicalizeOk := validRFC7493 + if gotCanonicalizeOk != wantCanonicalizeOk { + t.Errorf("Value.Canonicalize success mismatch: got %v, want %v", gotCanonicalizeOk, wantCanonicalizeOk) + } + + // Random options should not result in a panic. + var opts []Options + rn := rand.New(rand.NewSource(seed)) + for _, opt := range allOptions { + if rn.Intn(len(allOptions)/4) == 0 { + opts = append(opts, opt) + } + } + v := Value(b) + v.Format(opts...) // should not panic + }) +} diff --git a/src/encoding/json/jsontext/options.go b/src/encoding/json/jsontext/options.go new file mode 100644 index 00000000000000..7eb4f9b9e048a8 --- /dev/null +++ b/src/encoding/json/jsontext/options.go @@ -0,0 +1,304 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "strings" + + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonopts" + "encoding/json/internal/jsonwire" +) + +// Options configures [NewEncoder], [Encoder.Reset], [NewDecoder], +// and [Decoder.Reset] with specific features. +// Each function takes in a variadic list of options, where properties +// set in latter options override the value of previously set properties. +// +// There is a single Options type, which is used with both encoding and decoding. +// Some options affect both operations, while others only affect one operation: +// +// - [AllowDuplicateNames] affects encoding and decoding +// - [AllowInvalidUTF8] affects encoding and decoding +// - [EscapeForHTML] affects encoding only +// - [EscapeForJS] affects encoding only +// - [PreserveRawStrings] affects encoding only +// - [CanonicalizeRawInts] affects encoding only +// - [CanonicalizeRawFloats] affects encoding only +// - [ReorderRawObjects] affects encoding only +// - [SpaceAfterColon] affects encoding only +// - [SpaceAfterComma] affects encoding only +// - [Multiline] affects encoding only +// - [WithIndent] affects encoding only +// - [WithIndentPrefix] affects encoding only +// +// Options that do not affect a particular operation are ignored. +// +// The Options type is identical to [encoding/json.Options] and +// [encoding/json/v2.Options]. Options from the other packages may +// be passed to functionality in this package, but are ignored. +// Options from this package may be used with the other packages. +type Options = jsonopts.Options + +// AllowDuplicateNames specifies that JSON objects may contain +// duplicate member names. Disabling the duplicate name check may provide +// performance benefits, but breaks compliance with RFC 7493, section 2.3. +// The input or output will still be compliant with RFC 8259, +// which leaves the handling of duplicate names as unspecified behavior. +// +// This affects either encoding or decoding. +func AllowDuplicateNames(v bool) Options { + if v { + return jsonflags.AllowDuplicateNames | 1 + } else { + return jsonflags.AllowDuplicateNames | 0 + } +} + +// AllowInvalidUTF8 specifies that JSON strings may contain invalid UTF-8, +// which will be mangled as the Unicode replacement character, U+FFFD. +// This causes the encoder or decoder to break compliance with +// RFC 7493, section 2.1, and RFC 8259, section 8.1. +// +// This affects either encoding or decoding. +func AllowInvalidUTF8(v bool) Options { + if v { + return jsonflags.AllowInvalidUTF8 | 1 + } else { + return jsonflags.AllowInvalidUTF8 | 0 + } +} + +// EscapeForHTML specifies that '<', '>', and '&' characters within JSON strings +// should be escaped as a hexadecimal Unicode codepoint (e.g., \u003c) so that +// the output is safe to embed within HTML. +// +// This only affects encoding and is ignored when decoding. +func EscapeForHTML(v bool) Options { + if v { + return jsonflags.EscapeForHTML | 1 + } else { + return jsonflags.EscapeForHTML | 0 + } +} + +// EscapeForJS specifies that U+2028 and U+2029 characters within JSON strings +// should be escaped as a hexadecimal Unicode codepoint (e.g., \u2028) so that +// the output is valid to embed within JavaScript. See RFC 8259, section 12. +// +// This only affects encoding and is ignored when decoding. +func EscapeForJS(v bool) Options { + if v { + return jsonflags.EscapeForJS | 1 + } else { + return jsonflags.EscapeForJS | 0 + } +} + +// PreserveRawStrings specifies that when encoding a raw JSON string in a +// [Token] or [Value], pre-escaped sequences +// in a JSON string are preserved to the output. +// However, raw strings still respect [EscapeForHTML] and [EscapeForJS] +// such that the relevant characters are escaped. +// If [AllowInvalidUTF8] is enabled, bytes of invalid UTF-8 +// are preserved to the output. +// +// This only affects encoding and is ignored when decoding. +func PreserveRawStrings(v bool) Options { + if v { + return jsonflags.PreserveRawStrings | 1 + } else { + return jsonflags.PreserveRawStrings | 0 + } +} + +// CanonicalizeRawInts specifies that when encoding a raw JSON +// integer number (i.e., a number without a fraction and exponent) in a +// [Token] or [Value], the number is canonicalized +// according to RFC 8785, section 3.2.2.3. As a special case, +// the number -0 is canonicalized as 0. +// +// JSON numbers are treated as IEEE 754 double precision numbers. +// Any numbers with precision beyond what is representable by that form +// will lose their precision when canonicalized. For example, +// integer values beyond ±2⁵³ will lose their precision. +// For example, 1234567890123456789 is formatted as 1234567890123456800. +// +// This only affects encoding and is ignored when decoding. +func CanonicalizeRawInts(v bool) Options { + if v { + return jsonflags.CanonicalizeRawInts | 1 + } else { + return jsonflags.CanonicalizeRawInts | 0 + } +} + +// CanonicalizeRawFloats specifies that when encoding a raw JSON +// floating-point number (i.e., a number with a fraction or exponent) in a +// [Token] or [Value], the number is canonicalized +// according to RFC 8785, section 3.2.2.3. As a special case, +// the number -0 is canonicalized as 0. +// +// JSON numbers are treated as IEEE 754 double precision numbers. +// It is safe to canonicalize a serialized single precision number and +// parse it back as a single precision number and expect the same value. +// If a number exceeds ±1.7976931348623157e+308, which is the maximum +// finite number, then it saturated at that value and formatted as such. +// +// This only affects encoding and is ignored when decoding. +func CanonicalizeRawFloats(v bool) Options { + if v { + return jsonflags.CanonicalizeRawFloats | 1 + } else { + return jsonflags.CanonicalizeRawFloats | 0 + } +} + +// ReorderRawObjects specifies that when encoding a raw JSON object in a +// [Value], the object members are reordered according to +// RFC 8785, section 3.2.3. +// +// This only affects encoding and is ignored when decoding. +func ReorderRawObjects(v bool) Options { + if v { + return jsonflags.ReorderRawObjects | 1 + } else { + return jsonflags.ReorderRawObjects | 0 + } +} + +// SpaceAfterColon specifies that the JSON output should emit a space character +// after each colon separator following a JSON object name. +// If false, then no space character appears after the colon separator. +// +// This only affects encoding and is ignored when decoding. +func SpaceAfterColon(v bool) Options { + if v { + return jsonflags.SpaceAfterColon | 1 + } else { + return jsonflags.SpaceAfterColon | 0 + } +} + +// SpaceAfterComma specifies that the JSON output should emit a space character +// after each comma separator following a JSON object value or array element. +// If false, then no space character appears after the comma separator. +// +// This only affects encoding and is ignored when decoding. +func SpaceAfterComma(v bool) Options { + if v { + return jsonflags.SpaceAfterComma | 1 + } else { + return jsonflags.SpaceAfterComma | 0 + } +} + +// Multiline specifies that the JSON output should expand to multiple lines, +// where every JSON object member or JSON array element appears on +// a new, indented line according to the nesting depth. +// +// If [SpaceAfterColon] is not specified, then the default is true. +// If [SpaceAfterComma] is not specified, then the default is false. +// If [WithIndent] is not specified, then the default is "\t". +// +// If set to false, then the output is a single-line, +// where the only whitespace emitted is determined by the current +// values of [SpaceAfterColon] and [SpaceAfterComma]. +// +// This only affects encoding and is ignored when decoding. +func Multiline(v bool) Options { + if v { + return jsonflags.Multiline | 1 + } else { + return jsonflags.Multiline | 0 + } +} + +// WithIndent specifies that the encoder should emit multiline output +// where each element in a JSON object or array begins on a new, indented line +// beginning with the indent prefix (see [WithIndentPrefix]) +// followed by one or more copies of indent according to the nesting depth. +// The indent must only be composed of space or tab characters. +// +// If the intent to emit indented output without a preference for +// the particular indent string, then use [Multiline] instead. +// +// This only affects encoding and is ignored when decoding. +// Use of this option implies [Multiline] being set to true. +func WithIndent(indent string) Options { + // Fast-path: Return a constant for common indents, which avoids allocating. + // These are derived from analyzing the Go module proxy on 2023-07-01. + switch indent { + case "\t": + return jsonopts.Indent("\t") // ~14k usages + case " ": + return jsonopts.Indent(" ") // ~18k usages + case " ": + return jsonopts.Indent(" ") // ~1.7k usages + case " ": + return jsonopts.Indent(" ") // ~52k usages + case " ": + return jsonopts.Indent(" ") // ~12k usages + case "": + return jsonopts.Indent("") // ~1.5k usages + } + + // Otherwise, allocate for this unique value. + if s := strings.Trim(indent, " \t"); len(s) > 0 { + panic("json: invalid character " + jsonwire.QuoteRune(s) + " in indent") + } + return jsonopts.Indent(indent) +} + +// WithIndentPrefix specifies that the encoder should emit multiline output +// where each element in a JSON object or array begins on a new, indented line +// beginning with the indent prefix followed by one or more copies of indent +// (see [WithIndent]) according to the nesting depth. +// The prefix must only be composed of space or tab characters. +// +// This only affects encoding and is ignored when decoding. +// Use of this option implies [Multiline] being set to true. +func WithIndentPrefix(prefix string) Options { + if s := strings.Trim(prefix, " \t"); len(s) > 0 { + panic("json: invalid character " + jsonwire.QuoteRune(s) + " in indent prefix") + } + return jsonopts.IndentPrefix(prefix) +} + +/* +// TODO(https://go.dev/issue/56733): Implement WithByteLimit and WithDepthLimit. +// Remember to also update the "Security Considerations" section. + +// WithByteLimit sets a limit on the number of bytes of input or output bytes +// that may be consumed or produced for each top-level JSON value. +// If a [Decoder] or [Encoder] method call would need to consume/produce +// more than a total of n bytes to make progress on the top-level JSON value, +// then the call will report an error. +// Whitespace before and within the top-level value are counted against the limit. +// Whitespace after a top-level value are counted against the limit +// for the next top-level value. +// +// A non-positive limit is equivalent to no limit at all. +// If unspecified, the default limit is no limit at all. +// This affects either encoding or decoding. +func WithByteLimit(n int64) Options { + return jsonopts.ByteLimit(max(n, 0)) +} + +// WithDepthLimit sets a limit on the maximum depth of JSON nesting +// that may be consumed or produced for each top-level JSON value. +// If a [Decoder] or [Encoder] method call would need to consume or produce +// a depth greater than n to make progress on the top-level JSON value, +// then the call will report an error. +// +// A non-positive limit is equivalent to no limit at all. +// If unspecified, the default limit is 10000. +// This affects either encoding or decoding. +func WithDepthLimit(n int) Options { + return jsonopts.DepthLimit(max(n, 0)) +} +*/ diff --git a/src/encoding/json/jsontext/pools.go b/src/encoding/json/jsontext/pools.go new file mode 100644 index 00000000000000..3066ab4a1d2590 --- /dev/null +++ b/src/encoding/json/jsontext/pools.go @@ -0,0 +1,164 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "bytes" + "io" + "math/bits" + "sync" +) + +// TODO(https://go.dev/issue/47657): Use sync.PoolOf. + +var ( + // This owns the internal buffer since there is no io.Writer to output to. + // Since the buffer can get arbitrarily large in normal usage, + // there is statistical tracking logic to determine whether to recycle + // the internal buffer or not based on a history of utilization. + bufferedEncoderPool = &sync.Pool{New: func() any { return new(Encoder) }} + + // This owns the internal buffer, but it is only used to temporarily store + // buffered JSON before flushing it to the underlying io.Writer. + // In a sufficiently efficient streaming mode, we do not expect the buffer + // to grow arbitrarily large. Thus, we avoid recycling large buffers. + streamingEncoderPool = &sync.Pool{New: func() any { return new(Encoder) }} + + // This does not own the internal buffer since + // it is taken directly from the provided bytes.Buffer. + bytesBufferEncoderPool = &sync.Pool{New: func() any { return new(Encoder) }} +) + +// bufferStatistics is statistics to track buffer utilization. +// It is used to determine whether to recycle a buffer or not +// to avoid https://go.dev/issue/23199. +type bufferStatistics struct { + strikes int // number of times the buffer was under-utilized + prevLen int // length of previous buffer +} + +func getBufferedEncoder(opts ...Options) *Encoder { + e := bufferedEncoderPool.Get().(*Encoder) + if e.s.Buf == nil { + // Round up to nearest 2ⁿ to make best use of malloc size classes. + // See runtime/sizeclasses.go on Go1.15. + // Logical OR with 63 to ensure 64 as the minimum buffer size. + n := 1 << bits.Len(uint(e.s.bufStats.prevLen|63)) + e.s.Buf = make([]byte, 0, n) + } + e.s.reset(e.s.Buf[:0], nil, opts...) + return e +} +func putBufferedEncoder(e *Encoder) { + if cap(e.s.availBuffer) > 64<<10 { + e.s.availBuffer = nil // avoid pinning arbitrarily large amounts of memory + } + + // Recycle large buffers only if sufficiently utilized. + // If a buffer is under-utilized enough times sequentially, + // then it is discarded, ensuring that a single large buffer + // won't be kept alive by a continuous stream of small usages. + // + // The worst case utilization is computed as: + // MIN_UTILIZATION_THRESHOLD / (1 + MAX_NUM_STRIKES) + // + // For the constants chosen below, this is (25%)/(1+4) ⇒ 5%. + // This may seem low, but it ensures a lower bound on + // the absolute worst-case utilization. Without this check, + // this would be theoretically 0%, which is infinitely worse. + // + // See https://go.dev/issue/27735. + switch { + case cap(e.s.Buf) <= 4<<10: // always recycle buffers smaller than 4KiB + e.s.bufStats.strikes = 0 + case cap(e.s.Buf)/4 <= len(e.s.Buf): // at least 25% utilization + e.s.bufStats.strikes = 0 + case e.s.bufStats.strikes < 4: // at most 4 strikes + e.s.bufStats.strikes++ + default: // discard the buffer; too large and too often under-utilized + e.s.bufStats.strikes = 0 + e.s.bufStats.prevLen = len(e.s.Buf) // heuristic for size to allocate next time + e.s.Buf = nil + } + bufferedEncoderPool.Put(e) +} + +func getStreamingEncoder(w io.Writer, opts ...Options) *Encoder { + if _, ok := w.(*bytes.Buffer); ok { + e := bytesBufferEncoderPool.Get().(*Encoder) + e.s.reset(nil, w, opts...) // buffer taken from bytes.Buffer + return e + } else { + e := streamingEncoderPool.Get().(*Encoder) + e.s.reset(e.s.Buf[:0], w, opts...) // preserve existing buffer + return e + } +} +func putStreamingEncoder(e *Encoder) { + if cap(e.s.availBuffer) > 64<<10 { + e.s.availBuffer = nil // avoid pinning arbitrarily large amounts of memory + } + if _, ok := e.s.wr.(*bytes.Buffer); ok { + e.s.wr, e.s.Buf = nil, nil // avoid pinning the provided bytes.Buffer + bytesBufferEncoderPool.Put(e) + } else { + e.s.wr = nil // avoid pinning the provided io.Writer + if cap(e.s.Buf) > 64<<10 { + e.s.Buf = nil // avoid pinning arbitrarily large amounts of memory + } + streamingEncoderPool.Put(e) + } +} + +var ( + // This does not own the internal buffer since it is externally provided. + bufferedDecoderPool = &sync.Pool{New: func() any { return new(Decoder) }} + + // This owns the internal buffer, but it is only used to temporarily store + // buffered JSON fetched from the underlying io.Reader. + // In a sufficiently efficient streaming mode, we do not expect the buffer + // to grow arbitrarily large. Thus, we avoid recycling large buffers. + streamingDecoderPool = &sync.Pool{New: func() any { return new(Decoder) }} + + // This does not own the internal buffer since + // it is taken directly from the provided bytes.Buffer. + bytesBufferDecoderPool = bufferedDecoderPool +) + +func getBufferedDecoder(b []byte, opts ...Options) *Decoder { + d := bufferedDecoderPool.Get().(*Decoder) + d.s.reset(b, nil, opts...) + return d +} +func putBufferedDecoder(d *Decoder) { + d.s.buf = nil // avoid pinning the provided buffer + bufferedDecoderPool.Put(d) +} + +func getStreamingDecoder(r io.Reader, opts ...Options) *Decoder { + if _, ok := r.(*bytes.Buffer); ok { + d := bytesBufferDecoderPool.Get().(*Decoder) + d.s.reset(nil, r, opts...) // buffer taken from bytes.Buffer + return d + } else { + d := streamingDecoderPool.Get().(*Decoder) + d.s.reset(d.s.buf[:0], r, opts...) // preserve existing buffer + return d + } +} +func putStreamingDecoder(d *Decoder) { + if _, ok := d.s.rd.(*bytes.Buffer); ok { + d.s.rd, d.s.buf = nil, nil // avoid pinning the provided bytes.Buffer + bytesBufferDecoderPool.Put(d) + } else { + d.s.rd = nil // avoid pinning the provided io.Reader + if cap(d.s.buf) > 64<<10 { + d.s.buf = nil // avoid pinning arbitrarily large amounts of memory + } + streamingDecoderPool.Put(d) + } +} diff --git a/src/encoding/json/jsontext/quote.go b/src/encoding/json/jsontext/quote.go new file mode 100644 index 00000000000000..5ecfdbc21156e3 --- /dev/null +++ b/src/encoding/json/jsontext/quote.go @@ -0,0 +1,41 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonwire" +) + +// AppendQuote appends a double-quoted JSON string literal representing src +// to dst and returns the extended buffer. +// It uses the minimal string representation per RFC 8785, section 3.2.2.2. +// Invalid UTF-8 bytes are replaced with the Unicode replacement character +// and an error is returned at the end indicating the presence of invalid UTF-8. +// The dst must not overlap with the src. +func AppendQuote[Bytes ~[]byte | ~string](dst []byte, src Bytes) ([]byte, error) { + dst, err := jsonwire.AppendQuote(dst, src, &jsonflags.Flags{}) + if err != nil { + err = &SyntacticError{Err: err} + } + return dst, err +} + +// AppendUnquote appends the decoded interpretation of src as a +// double-quoted JSON string literal to dst and returns the extended buffer. +// The input src must be a JSON string without any surrounding whitespace. +// Invalid UTF-8 bytes are replaced with the Unicode replacement character +// and an error is returned at the end indicating the presence of invalid UTF-8. +// Any trailing bytes after the JSON string literal results in an error. +// The dst must not overlap with the src. +func AppendUnquote[Bytes ~[]byte | ~string](dst []byte, src Bytes) ([]byte, error) { + dst, err := jsonwire.AppendUnquote(dst, src) + if err != nil { + err = &SyntacticError{Err: err} + } + return dst, err +} diff --git a/src/encoding/json/jsontext/state.go b/src/encoding/json/jsontext/state.go new file mode 100644 index 00000000000000..e93057a34c577d --- /dev/null +++ b/src/encoding/json/jsontext/state.go @@ -0,0 +1,828 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "errors" + "iter" + "math" + "strconv" + "strings" + "unicode/utf8" + + "encoding/json/internal/jsonwire" +) + +// ErrDuplicateName indicates that a JSON token could not be +// encoded or decoded because it results in a duplicate JSON object name. +// This error is directly wrapped within a [SyntacticError] when produced. +// +// The name of a duplicate JSON object member can be extracted as: +// +// err := ... +// serr, ok := errors.AsType[*jsontext.SyntacticError](err) +// if ok && serr.Err == jsontext.ErrDuplicateName { +// ptr := serr.JSONPointer // JSON pointer to duplicate name +// name := ptr.LastToken() // duplicate name itself +// ... +// } +// +// This error is only returned if [AllowDuplicateNames] is false. +var ErrDuplicateName = errors.New("duplicate object member name") + +// ErrNonStringName indicates that a JSON token could not be +// encoded or decoded because it is not a string, +// as required for JSON object names according to RFC 8259, section 4. +// This error is directly wrapped within a [SyntacticError] when produced. +var ErrNonStringName = errors.New("object member name must be a string") + +var ( + errMissingValue = errors.New("missing value after object name") + errMismatchDelim = errors.New("mismatching structural token for object or array") + errMaxDepth = errors.New("exceeded max depth") + + errInvalidNamespace = errors.New("object namespace is in an invalid state") +) + +// Per RFC 8259, section 9, implementations may enforce a maximum depth. +// Such a limit is necessary to prevent stack overflows. +const maxNestingDepth = 10000 + +type state struct { + // Tokens validates whether the next token kind is valid. + Tokens stateMachine + + // Names is a stack of object names. + Names objectNameStack + + // Namespaces is a stack of object namespaces. + // For performance reasons, Encoder or Decoder may not update this + // if Marshal or Unmarshal is able to track names in a more efficient way. + // See makeMapArshaler and makeStructArshaler. + // Not used if AllowDuplicateNames is true. + Namespaces objectNamespaceStack +} + +// needObjectValue reports whether the next token should be an object value. +// This method is used by [wrapSyntacticError]. +func (s *state) needObjectValue() bool { + return s.Tokens.Last.needObjectValue() +} + +func (s *state) reset() { + s.Tokens.reset() + s.Names.reset() + s.Namespaces.reset() +} + +// Pointer is a JSON Pointer (RFC 6901) that references a particular JSON value +// relative to the root of the top-level JSON value. +// +// A Pointer is a slash-separated list of tokens, where each token is +// either a JSON object name or an index to a JSON array element +// encoded as a base-10 integer value. +// It is impossible to distinguish between an array index and an object name +// (that happens to be an base-10 encoded integer) without also knowing +// the structure of the top-level JSON value that the pointer refers to. +// +// There is exactly one representation of a pointer to a particular value, +// so comparability of Pointer values is equivalent to checking whether +// they both point to the exact same value. +type Pointer string + +// IsValid reports whether p is a valid JSON Pointer according to RFC 6901. +// Note that the concatenation of two valid pointers produces a valid pointer. +func (p Pointer) IsValid() bool { + for i, r := range p { + switch { + case r == '~' && (i+1 == len(p) || (p[i+1] != '0' && p[i+1] != '1')): + return false // invalid escape + case r == '\ufffd' && !strings.HasPrefix(string(p[i:]), "\ufffd"): + return false // invalid UTF-8 + } + } + return len(p) == 0 || p[0] == '/' +} + +// Contains reports whether the JSON value that p points to +// is equal to or contains the JSON value that pc points to. +func (p Pointer) Contains(pc Pointer) bool { + // Invariant: len(p) <= len(pc) if p.Contains(pc) + suffix, ok := strings.CutPrefix(string(pc), string(p)) + return ok && (suffix == "" || suffix[0] == '/') +} + +// Parent strips off the last token and returns the remaining pointer. +// The parent of an empty p is an empty string. +func (p Pointer) Parent() Pointer { + return p[:max(strings.LastIndexByte(string(p), '/'), 0)] +} + +// LastToken returns the last token in the pointer. +// The last token of an empty p is an empty string. +func (p Pointer) LastToken() string { + last := p[max(strings.LastIndexByte(string(p), '/'), 0):] + return unescapePointerToken(strings.TrimPrefix(string(last), "/")) +} + +// AppendToken appends a token to the end of p and returns the full pointer. +func (p Pointer) AppendToken(tok string) Pointer { + return Pointer(appendEscapePointerName([]byte(p+"/"), tok)) +} + +// TODO: Add Pointer.AppendTokens, +// but should this take in a ...string or an iter.Seq[string]? + +// Tokens returns an iterator over the reference tokens in the JSON pointer, +// starting from the first token until the last token (unless stopped early). +func (p Pointer) Tokens() iter.Seq[string] { + return func(yield func(string) bool) { + for len(p) > 0 { + p = Pointer(strings.TrimPrefix(string(p), "/")) + i := min(uint(strings.IndexByte(string(p), '/')), uint(len(p))) + if !yield(unescapePointerToken(string(p)[:i])) { + return + } + p = p[i:] + } + } +} + +func unescapePointerToken(token string) string { + if strings.Contains(token, "~") { + // Per RFC 6901, section 3, unescape '~' and '/' characters. + token = strings.ReplaceAll(token, "~1", "/") + token = strings.ReplaceAll(token, "~0", "~") + } + return token +} + +// appendStackPointer appends a JSON Pointer (RFC 6901) to the current value. +// +// - If where is -1, then it points to the previously processed token. +// +// - If where is 0, then it points to the parent JSON object or array, +// or an object member if in-between an object member key and value. +// This is useful when the position is ambiguous whether +// we are interested in the previous or next token, or +// when we are uncertain whether the next token +// continues or terminates the current object or array. +// +// - If where is +1, then it points to the next expected value, +// assuming that it continues the current JSON object or array. +// As a special case, if the next token is a JSON object name, +// then it points to the parent JSON object. +// +// Invariant: Must call s.names.copyQuotedBuffer beforehand. +func (s state) appendStackPointer(b []byte, where int) []byte { + var objectDepth int + for i := 1; i < s.Tokens.Depth(); i++ { + e := s.Tokens.index(i) + arrayDelta := -1 // by default point to previous array element + if isLast := i == s.Tokens.Depth()-1; isLast { + switch { + case where < 0 && e.Length() == 0 || where == 0 && !e.needObjectValue() || where > 0 && e.NeedObjectName(): + return b + case where > 0 && e.isArray(): + arrayDelta = 0 // point to next array element + } + } + switch { + case e.isObject(): + b = appendEscapePointerName(append(b, '/'), s.Names.getUnquoted(objectDepth)) + objectDepth++ + case e.isArray(): + b = strconv.AppendUint(append(b, '/'), uint64(e.Length()+int64(arrayDelta)), 10) + } + } + return b +} + +func appendEscapePointerName[Bytes ~[]byte | ~string](b []byte, name Bytes) []byte { + for _, r := range string(name) { + // Per RFC 6901, section 3, escape '~' and '/' characters. + switch r { + case '~': + b = append(b, "~0"...) + case '/': + b = append(b, "~1"...) + default: + b = utf8.AppendRune(b, r) + } + } + return b +} + +// stateMachine is a push-down automaton that validates whether +// a sequence of tokens is valid or not according to the JSON grammar. +// It is useful for both encoding and decoding. +// +// It is a stack where each entry represents a nested JSON object or array. +// The stack has a minimum depth of 1 where the first level is a +// virtual JSON array to handle a stream of top-level JSON values. +// The top-level virtual JSON array is special in that it doesn't require commas +// between each JSON value. +// +// For performance, most methods are carefully written to be inlinable. +// The zero value is a valid state machine ready for use. +type stateMachine struct { + Stack []stateEntry + Last stateEntry +} + +// reset resets the state machine. +// The machine always starts with a minimum depth of 1. +func (m *stateMachine) reset() { + m.Stack = m.Stack[:0] + if cap(m.Stack) > 1<<10 { + m.Stack = nil + } + m.Last = stateTypeArray +} + +// Depth is the current nested depth of JSON objects and arrays. +// It is one-indexed (i.e., top-level values have a depth of 1). +func (m stateMachine) Depth() int { + return len(m.Stack) + 1 +} + +// index returns a reference to the ith entry. +// It is only valid until the next push method call. +func (m *stateMachine) index(i int) *stateEntry { + if i == len(m.Stack) { + return &m.Last + } + return &m.Stack[i] +} + +// DepthLength reports the current nested depth and +// the length of the last JSON object or array. +func (m stateMachine) DepthLength() (int, int64) { + return m.Depth(), m.Last.Length() +} + +// appendLiteral appends a JSON literal as the next token in the sequence. +// If an error is returned, the state is not mutated. +func (m *stateMachine) appendLiteral() error { + switch { + case m.Last.NeedObjectName(): + return ErrNonStringName + case !m.Last.isValidNamespace(): + return errInvalidNamespace + default: + m.Last.Increment() + return nil + } +} + +// appendString appends a JSON string as the next token in the sequence. +// If an error is returned, the state is not mutated. +func (m *stateMachine) appendString() error { + switch { + case !m.Last.isValidNamespace(): + return errInvalidNamespace + default: + m.Last.Increment() + return nil + } +} + +// appendNumber appends a JSON number as the next token in the sequence. +// If an error is returned, the state is not mutated. +func (m *stateMachine) appendNumber() error { + return m.appendLiteral() +} + +// pushObject appends a JSON begin object token as next in the sequence. +// If an error is returned, the state is not mutated. +func (m *stateMachine) pushObject() error { + switch { + case m.Last.NeedObjectName(): + return ErrNonStringName + case !m.Last.isValidNamespace(): + return errInvalidNamespace + case len(m.Stack) == maxNestingDepth: + return errMaxDepth + default: + m.Last.Increment() + m.Stack = append(m.Stack, m.Last) + m.Last = stateTypeObject + return nil + } +} + +// popObject appends a JSON end object token as next in the sequence. +// If an error is returned, the state is not mutated. +func (m *stateMachine) popObject() error { + switch { + case !m.Last.isObject(): + return errMismatchDelim + case m.Last.needObjectValue(): + return errMissingValue + case !m.Last.isValidNamespace(): + return errInvalidNamespace + default: + m.Last = m.Stack[len(m.Stack)-1] + m.Stack = m.Stack[:len(m.Stack)-1] + return nil + } +} + +// pushArray appends a JSON begin array token as next in the sequence. +// If an error is returned, the state is not mutated. +func (m *stateMachine) pushArray() error { + switch { + case m.Last.NeedObjectName(): + return ErrNonStringName + case !m.Last.isValidNamespace(): + return errInvalidNamespace + case len(m.Stack) == maxNestingDepth: + return errMaxDepth + default: + m.Last.Increment() + m.Stack = append(m.Stack, m.Last) + m.Last = stateTypeArray + return nil + } +} + +// popArray appends a JSON end array token as next in the sequence. +// If an error is returned, the state is not mutated. +func (m *stateMachine) popArray() error { + switch { + case !m.Last.isArray() || len(m.Stack) == 0: // forbid popping top-level virtual JSON array + return errMismatchDelim + case !m.Last.isValidNamespace(): + return errInvalidNamespace + default: + m.Last = m.Stack[len(m.Stack)-1] + m.Stack = m.Stack[:len(m.Stack)-1] + return nil + } +} + +// NeedIndent reports whether indent whitespace should be injected. +// A zero value means that no whitespace should be injected. +// A positive value means '\n', indentPrefix, and (n-1) copies of indentBody +// should be appended to the output immediately before the next token. +func (m stateMachine) NeedIndent(next Kind) (n int) { + willEnd := next == '}' || next == ']' + switch { + case m.Depth() == 1: + return 0 // top-level values are never indented + case m.Last.Length() == 0 && willEnd: + return 0 // an empty object or array is never indented + case m.Last.Length() == 0 || m.Last.needImplicitComma(next): + return m.Depth() + case willEnd: + return m.Depth() - 1 + default: + return 0 + } +} + +// MayAppendDelim appends a colon or comma that may precede the next token. +func (m stateMachine) MayAppendDelim(b []byte, next Kind) []byte { + switch { + case m.Last.needImplicitColon(): + return append(b, ':') + case m.Last.needImplicitComma(next) && len(m.Stack) != 0: // comma not needed for top-level values + return append(b, ',') + default: + return b + } +} + +// needDelim reports whether a colon or comma token should be implicitly emitted +// before the next token of the specified kind. +// A zero value means no delimiter should be emitted. +func (m stateMachine) needDelim(next Kind) (delim byte) { + switch { + case m.Last.needImplicitColon(): + return ':' + case m.Last.needImplicitComma(next) && len(m.Stack) != 0: // comma not needed for top-level values + return ',' + default: + return 0 + } +} + +// InvalidateDisabledNamespaces marks all disabled namespaces as invalid. +// +// For efficiency, Marshal and Unmarshal may disable namespaces since there are +// more efficient ways to track duplicate names. However, if an error occurs, +// the namespaces in Encoder or Decoder will be left in an inconsistent state. +// Mark the namespaces as invalid so that future method calls on +// Encoder or Decoder will return an error. +func (m *stateMachine) InvalidateDisabledNamespaces() { + for i := range m.Depth() { + e := m.index(i) + if !e.isActiveNamespace() { + e.invalidateNamespace() + } + } +} + +// stateEntry encodes several artifacts within a single unsigned integer: +// - whether this represents a JSON object or array, +// - whether this object should check for duplicate names, and +// - how many elements are in this JSON object or array. +type stateEntry uint64 + +const ( + // The type mask (1 bit) records whether this is a JSON object or array. + stateTypeMask stateEntry = 0x8000_0000_0000_0000 + stateTypeObject stateEntry = 0x8000_0000_0000_0000 + stateTypeArray stateEntry = 0x0000_0000_0000_0000 + + // The name check mask (2 bit) records whether to update + // the namespaces for the current JSON object and + // whether the namespace is valid. + stateNamespaceMask stateEntry = 0x6000_0000_0000_0000 + stateDisableNamespace stateEntry = 0x4000_0000_0000_0000 + stateInvalidNamespace stateEntry = 0x2000_0000_0000_0000 + + // The count mask (61 bits) records the number of elements. + stateCountMask stateEntry = 0x1fff_ffff_ffff_ffff + stateCountLSBMask stateEntry = 0x0000_0000_0000_0001 + stateCountOdd stateEntry = 0x0000_0000_0000_0001 + stateCountEven stateEntry = 0x0000_0000_0000_0000 +) + +// Length reports the number of elements in the JSON object or array. +// Each name and value in an object entry is treated as a separate element. +func (e stateEntry) Length() int64 { + return int64(e & stateCountMask) +} + +// isObject reports whether this is a JSON object. +func (e stateEntry) isObject() bool { + return e&stateTypeMask == stateTypeObject +} + +// isArray reports whether this is a JSON array. +func (e stateEntry) isArray() bool { + return e&stateTypeMask == stateTypeArray +} + +// NeedObjectName reports whether the next token must be a JSON string, +// which is necessary for JSON object names. +func (e stateEntry) NeedObjectName() bool { + return e&(stateTypeMask|stateCountLSBMask) == stateTypeObject|stateCountEven +} + +// needImplicitColon reports whether an colon should occur next, +// which always occurs after JSON object names. +func (e stateEntry) needImplicitColon() bool { + return e.needObjectValue() +} + +// needObjectValue reports whether the next token must be a JSON value, +// which is necessary after every JSON object name. +func (e stateEntry) needObjectValue() bool { + return e&(stateTypeMask|stateCountLSBMask) == stateTypeObject|stateCountOdd +} + +// needImplicitComma reports whether an comma should occur next, +// which always occurs after a value in a JSON object or array +// before the next value (or name). +func (e stateEntry) needImplicitComma(next Kind) bool { + return !e.needObjectValue() && e.Length() > 0 && next != '}' && next != ']' +} + +// Increment increments the number of elements for the current object or array. +// This assumes that overflow won't practically be an issue since +// 1< 0. +func (e *stateEntry) decrement() { + (*e)-- +} + +// DisableNamespace disables the JSON object namespace such that the +// Encoder or Decoder no longer updates the namespace. +func (e *stateEntry) DisableNamespace() { + *e |= stateDisableNamespace +} + +// isActiveNamespace reports whether the JSON object namespace is actively +// being updated and used for duplicate name checks. +func (e stateEntry) isActiveNamespace() bool { + return e&(stateDisableNamespace) == 0 +} + +// invalidateNamespace marks the JSON object namespace as being invalid. +func (e *stateEntry) invalidateNamespace() { + *e |= stateInvalidNamespace +} + +// isValidNamespace reports whether the JSON object namespace is valid. +func (e stateEntry) isValidNamespace() bool { + return e&(stateInvalidNamespace) == 0 +} + +// objectNameStack is a stack of names when descending into a JSON object. +// In contrast to objectNamespaceStack, this only has to remember a single name +// per JSON object. +// +// This data structure may contain offsets to encodeBuffer or decodeBuffer. +// It violates clean abstraction of layers, but is significantly more efficient. +// This ensures that popping and pushing in the common case is a trivial +// push/pop of an offset integer. +// +// The zero value is an empty names stack ready for use. +type objectNameStack struct { + // offsets is a stack of offsets for each name. + // A non-negative offset is the ending offset into the local names buffer. + // A negative offset is the bit-wise inverse of a starting offset into + // a remote buffer (e.g., encodeBuffer or decodeBuffer). + // A math.MinInt offset at the end implies that the last object is empty. + // Invariant: Positive offsets always occur before negative offsets. + offsets []int + // unquotedNames is a back-to-back concatenation of names. + unquotedNames []byte +} + +func (ns *objectNameStack) reset() { + ns.offsets = ns.offsets[:0] + ns.unquotedNames = ns.unquotedNames[:0] + if cap(ns.offsets) > 1<<6 { + ns.offsets = nil // avoid pinning arbitrarily large amounts of memory + } + if cap(ns.unquotedNames) > 1<<10 { + ns.unquotedNames = nil // avoid pinning arbitrarily large amounts of memory + } +} + +func (ns *objectNameStack) length() int { + return len(ns.offsets) +} + +// getUnquoted retrieves the ith unquoted name in the stack. +// It returns an empty string if the last object is empty. +// +// Invariant: Must call copyQuotedBuffer beforehand. +func (ns *objectNameStack) getUnquoted(i int) []byte { + ns.ensureCopiedBuffer() + if i == 0 { + return ns.unquotedNames[:ns.offsets[0]] + } else { + return ns.unquotedNames[ns.offsets[i-1]:ns.offsets[i-0]] + } +} + +// invalidOffset indicates that the last JSON object currently has no name. +const invalidOffset = math.MinInt + +// push descends into a nested JSON object. +func (ns *objectNameStack) push() { + ns.offsets = append(ns.offsets, invalidOffset) +} + +// ReplaceLastQuotedOffset replaces the last name with the starting offset +// to the quoted name in some remote buffer. All offsets provided must be +// relative to the same buffer until copyQuotedBuffer is called. +func (ns *objectNameStack) ReplaceLastQuotedOffset(i int) { + // Use bit-wise inversion instead of naive multiplication by -1 to avoid + // ambiguity regarding zero (which is a valid offset into the names field). + // Bit-wise inversion is mathematically equivalent to -i-1, + // such that 0 becomes -1, 1 becomes -2, and so forth. + // This ensures that remote offsets are always negative. + ns.offsets[len(ns.offsets)-1] = ^i +} + +// replaceLastUnquotedName replaces the last name with the provided name. +// +// Invariant: Must call copyQuotedBuffer beforehand. +func (ns *objectNameStack) replaceLastUnquotedName(s string) { + ns.ensureCopiedBuffer() + var startOffset int + if len(ns.offsets) > 1 { + startOffset = ns.offsets[len(ns.offsets)-2] + } + ns.unquotedNames = append(ns.unquotedNames[:startOffset], s...) + ns.offsets[len(ns.offsets)-1] = len(ns.unquotedNames) +} + +// clearLast removes any name in the last JSON object. +// It is semantically equivalent to ns.push followed by ns.pop. +func (ns *objectNameStack) clearLast() { + ns.offsets[len(ns.offsets)-1] = invalidOffset +} + +// pop ascends out of a nested JSON object. +func (ns *objectNameStack) pop() { + ns.offsets = ns.offsets[:len(ns.offsets)-1] +} + +// copyQuotedBuffer copies names from the remote buffer into the local names +// buffer so that there are no more offset references into the remote buffer. +// This allows the remote buffer to change contents without affecting +// the names that this data structure is trying to remember. +func (ns *objectNameStack) copyQuotedBuffer(b []byte) { + // Find the first negative offset. + var i int + for i = len(ns.offsets) - 1; i >= 0 && ns.offsets[i] < 0; i-- { + continue + } + + // Copy each name from the remote buffer into the local buffer. + for i = i + 1; i < len(ns.offsets); i++ { + if i == len(ns.offsets)-1 && ns.offsets[i] == invalidOffset { + if i == 0 { + ns.offsets[i] = 0 + } else { + ns.offsets[i] = ns.offsets[i-1] + } + break // last JSON object had a push without any names + } + + // As a form of Hyrum proofing, we write an invalid character into the + // buffer to make misuse of Decoder.ReadToken more obvious. + // We need to undo that mutation here. + quotedName := b[^ns.offsets[i]:] + if quotedName[0] == invalidateBufferByte { + quotedName[0] = '"' + } + + // Append the unquoted name to the local buffer. + var startOffset int + if i > 0 { + startOffset = ns.offsets[i-1] + } + if n := jsonwire.ConsumeSimpleString(quotedName); n > 0 { + ns.unquotedNames = append(ns.unquotedNames[:startOffset], quotedName[len(`"`):n-len(`"`)]...) + } else { + ns.unquotedNames, _ = jsonwire.AppendUnquote(ns.unquotedNames[:startOffset], quotedName) + } + ns.offsets[i] = len(ns.unquotedNames) + } +} + +func (ns *objectNameStack) ensureCopiedBuffer() { + if len(ns.offsets) > 0 && ns.offsets[len(ns.offsets)-1] < 0 { + panic("BUG: copyQuotedBuffer not called beforehand") + } +} + +// objectNamespaceStack is a stack of object namespaces. +// This data structure assists in detecting duplicate names. +type objectNamespaceStack []objectNamespace + +// reset resets the object namespace stack. +func (nss *objectNamespaceStack) reset() { + if cap(*nss) > 1<<10 { + *nss = nil + } + *nss = (*nss)[:0] +} + +// push starts a new namespace for a nested JSON object. +func (nss *objectNamespaceStack) push() { + if cap(*nss) > len(*nss) { + *nss = (*nss)[:len(*nss)+1] + nss.Last().reset() + } else { + *nss = append(*nss, objectNamespace{}) + } +} + +// Last returns a pointer to the last JSON object namespace. +func (nss objectNamespaceStack) Last() *objectNamespace { + return &nss[len(nss)-1] +} + +// pop terminates the namespace for a nested JSON object. +func (nss *objectNamespaceStack) pop() { + *nss = (*nss)[:len(*nss)-1] +} + +// objectNamespace is the namespace for a JSON object. +// In contrast to objectNameStack, this needs to remember a all names +// per JSON object. +// +// The zero value is an empty namespace ready for use. +type objectNamespace struct { + // It relies on a linear search over all the names before switching + // to use a Go map for direct lookup. + + // endOffsets is a list of offsets to the end of each name in buffers. + // The length of offsets is the number of names in the namespace. + endOffsets []uint + // allUnquotedNames is a back-to-back concatenation of every name in the namespace. + allUnquotedNames []byte + // mapNames is a Go map containing every name in the namespace. + // Only valid if non-nil. + mapNames map[string]struct{} +} + +// reset resets the namespace to be empty. +func (ns *objectNamespace) reset() { + ns.endOffsets = ns.endOffsets[:0] + ns.allUnquotedNames = ns.allUnquotedNames[:0] + ns.mapNames = nil + if cap(ns.endOffsets) > 1<<6 { + ns.endOffsets = nil // avoid pinning arbitrarily large amounts of memory + } + if cap(ns.allUnquotedNames) > 1<<10 { + ns.allUnquotedNames = nil // avoid pinning arbitrarily large amounts of memory + } +} + +// length reports the number of names in the namespace. +func (ns *objectNamespace) length() int { + return len(ns.endOffsets) +} + +// getUnquoted retrieves the ith unquoted name in the namespace. +func (ns *objectNamespace) getUnquoted(i int) []byte { + if i == 0 { + return ns.allUnquotedNames[:ns.endOffsets[0]] + } else { + return ns.allUnquotedNames[ns.endOffsets[i-1]:ns.endOffsets[i-0]] + } +} + +// lastUnquoted retrieves the last name in the namespace. +func (ns *objectNamespace) lastUnquoted() []byte { + return ns.getUnquoted(ns.length() - 1) +} + +// insertQuoted inserts a name and reports whether it was inserted, +// which only occurs if name is not already in the namespace. +// The provided name must be a valid JSON string. +func (ns *objectNamespace) insertQuoted(name []byte, isVerbatim bool) bool { + if isVerbatim { + name = name[len(`"`) : len(name)-len(`"`)] + } + return ns.insert(name, !isVerbatim) +} +func (ns *objectNamespace) InsertUnquoted(name []byte) bool { + return ns.insert(name, false) +} +func (ns *objectNamespace) insert(name []byte, quoted bool) bool { + var allNames []byte + if quoted { + allNames, _ = jsonwire.AppendUnquote(ns.allUnquotedNames, name) + } else { + allNames = append(ns.allUnquotedNames, name...) + } + name = allNames[len(ns.allUnquotedNames):] + + // Switch to a map if the buffer is too large for linear search. + // This does not add the current name to the map. + if ns.mapNames == nil && (ns.length() > 64 || len(ns.allUnquotedNames) > 1024) { + ns.mapNames = make(map[string]struct{}) + var startOffset uint + for _, endOffset := range ns.endOffsets { + name := ns.allUnquotedNames[startOffset:endOffset] + ns.mapNames[string(name)] = struct{}{} // allocates a new string + startOffset = endOffset + } + } + + if ns.mapNames == nil { + // Perform linear search over the buffer to find matching names. + // It provides O(n) lookup, but does not require any allocations. + var startOffset uint + for _, endOffset := range ns.endOffsets { + if string(ns.allUnquotedNames[startOffset:endOffset]) == string(name) { + return false + } + startOffset = endOffset + } + } else { + // Use the map if it is populated. + // It provides O(1) lookup, but requires a string allocation per name. + if _, ok := ns.mapNames[string(name)]; ok { + return false + } + ns.mapNames[string(name)] = struct{}{} // allocates a new string + } + + ns.allUnquotedNames = allNames + ns.endOffsets = append(ns.endOffsets, uint(len(ns.allUnquotedNames))) + return true +} + +// removeLast removes the last name in the namespace. +func (ns *objectNamespace) removeLast() { + if ns.mapNames != nil { + delete(ns.mapNames, string(ns.lastUnquoted())) + } + if ns.length()-1 == 0 { + ns.endOffsets = ns.endOffsets[:0] + ns.allUnquotedNames = ns.allUnquotedNames[:0] + } else { + ns.endOffsets = ns.endOffsets[:ns.length()-1] + ns.allUnquotedNames = ns.allUnquotedNames[:ns.endOffsets[ns.length()-1]] + } +} diff --git a/src/encoding/json/jsontext/state_test.go b/src/encoding/json/jsontext/state_test.go new file mode 100644 index 00000000000000..c227600945e122 --- /dev/null +++ b/src/encoding/json/jsontext/state_test.go @@ -0,0 +1,396 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "fmt" + "slices" + "strings" + "testing" + "unicode/utf8" +) + +func TestPointer(t *testing.T) { + tests := []struct { + in Pointer + wantParent Pointer + wantLast string + wantTokens []string + wantValid bool + }{ + {"", "", "", nil, true}, + {"a", "", "a", []string{"a"}, false}, + {"~", "", "~", []string{"~"}, false}, + {"/a", "", "a", []string{"a"}, true}, + {"/foo/bar", "/foo", "bar", []string{"foo", "bar"}, true}, + {"///", "//", "", []string{"", "", ""}, true}, + {"/~0~1", "", "~/", []string{"~/"}, true}, + {"/\xde\xad\xbe\xef", "", "\xde\xad\xbe\xef", []string{"\xde\xad\xbe\xef"}, false}, + } + for _, tt := range tests { + if got := tt.in.Parent(); got != tt.wantParent { + t.Errorf("Pointer(%q).Parent = %q, want %q", tt.in, got, tt.wantParent) + } + if got := tt.in.LastToken(); got != tt.wantLast { + t.Errorf("Pointer(%q).Last = %q, want %q", tt.in, got, tt.wantLast) + } + if strings.HasPrefix(string(tt.in), "/") { + wantRoundtrip := tt.in + if !utf8.ValidString(string(wantRoundtrip)) { + // Replace bytes of invalid UTF-8 with Unicode replacement character. + wantRoundtrip = Pointer([]rune(wantRoundtrip)) + } + if got := tt.in.Parent().AppendToken(tt.in.LastToken()); got != wantRoundtrip { + t.Errorf("Pointer(%q).Parent().AppendToken(LastToken()) = %q, want %q", tt.in, got, tt.in) + } + in := tt.in + for { + if (in + "x").Contains(tt.in) { + t.Errorf("Pointer(%q).Contains(%q) = true, want false", in+"x", tt.in) + } + if !in.Contains(tt.in) { + t.Errorf("Pointer(%q).Contains(%q) = false, want true", in, tt.in) + } + if in == in.Parent() { + break + } + in = in.Parent() + } + } + if got := slices.Collect(tt.in.Tokens()); !slices.Equal(got, tt.wantTokens) { + t.Errorf("Pointer(%q).Tokens = %q, want %q", tt.in, got, tt.wantTokens) + } + if got := tt.in.IsValid(); got != tt.wantValid { + t.Errorf("Pointer(%q).IsValid = %v, want %v", tt.in, got, tt.wantValid) + } + } +} + +func TestStateMachine(t *testing.T) { + // To test a state machine, we pass an ordered sequence of operations and + // check whether the current state is as expected. + // The operation type is a union type of various possible operations, + // which either call mutating methods on the state machine or + // call accessor methods on state machine and verify the results. + type operation any + type ( + // stackLengths checks the results of stateEntry.length accessors. + stackLengths []int64 + + // appendTokens is sequence of token kinds to append where + // none of them are expected to fail. + // + // For example: `[nft]` is equivalent to the following sequence: + // + // pushArray() + // appendLiteral() + // appendString() + // appendNumber() + // popArray() + // + appendTokens string + + // appendToken is a single token kind to append with the expected error. + appendToken struct { + kind Kind + want error + } + + // needDelim checks the result of the needDelim accessor. + needDelim struct { + next Kind + want byte + } + ) + + // Each entry is a sequence of tokens to pass to the state machine. + tests := []struct { + label string + ops []operation + }{{ + "TopLevelValues", + []operation{ + stackLengths{0}, + needDelim{'n', 0}, + appendTokens(`nft`), + stackLengths{3}, + needDelim{'"', 0}, + appendTokens(`"0[]{}`), + stackLengths{7}, + }, + }, { + "ArrayValues", + []operation{ + stackLengths{0}, + needDelim{'[', 0}, + appendTokens(`[`), + stackLengths{1, 0}, + needDelim{'n', 0}, + appendTokens(`nft`), + stackLengths{1, 3}, + needDelim{'"', ','}, + appendTokens(`"0[]{}`), + stackLengths{1, 7}, + needDelim{']', 0}, + appendTokens(`]`), + stackLengths{1}, + }, + }, { + "ObjectValues", + []operation{ + stackLengths{0}, + needDelim{'{', 0}, + appendTokens(`{`), + stackLengths{1, 0}, + needDelim{'"', 0}, + appendTokens(`"`), + stackLengths{1, 1}, + needDelim{'n', ':'}, + appendTokens(`n`), + stackLengths{1, 2}, + needDelim{'"', ','}, + appendTokens(`"f"t`), + stackLengths{1, 6}, + appendTokens(`"""0"[]"{}`), + stackLengths{1, 14}, + needDelim{'}', 0}, + appendTokens(`}`), + stackLengths{1}, + }, + }, { + "ObjectCardinality", + []operation{ + appendTokens(`{`), + + // Appending any kind other than string for object name is an error. + appendToken{'n', ErrNonStringName}, + appendToken{'f', ErrNonStringName}, + appendToken{'t', ErrNonStringName}, + appendToken{'0', ErrNonStringName}, + appendToken{'{', ErrNonStringName}, + appendToken{'[', ErrNonStringName}, + appendTokens(`"`), + + // Appending '}' without first appending any value is an error. + appendToken{'}', errMissingValue}, + appendTokens(`"`), + + appendTokens(`}`), + }, + }, { + "MismatchingDelims", + []operation{ + appendToken{'}', errMismatchDelim}, // appending '}' without preceding '{' + appendTokens(`[[{`), + appendToken{']', errMismatchDelim}, // appending ']' that mismatches preceding '{' + appendTokens(`}]`), + appendToken{'}', errMismatchDelim}, // appending '}' that mismatches preceding '[' + appendTokens(`]`), + appendToken{']', errMismatchDelim}, // appending ']' without preceding '[' + }, + }} + + for _, tt := range tests { + t.Run(tt.label, func(t *testing.T) { + // Flatten appendTokens to sequence of appendToken entries. + var ops []operation + for _, op := range tt.ops { + if toks, ok := op.(appendTokens); ok { + for _, k := range []byte(toks) { + ops = append(ops, appendToken{Kind(k), nil}) + } + continue + } + ops = append(ops, op) + } + + // Append each token to the state machine and check the output. + var state stateMachine + state.reset() + var sequence []Kind + for _, op := range ops { + switch op := op.(type) { + case stackLengths: + var got []int64 + for i := range state.Depth() { + e := state.index(i) + got = append(got, e.Length()) + } + want := []int64(op) + if !slices.Equal(got, want) { + t.Fatalf("%s: stack lengths mismatch:\ngot %v\nwant %v", sequence, got, want) + } + case appendToken: + got := state.append(op.kind) + if !equalError(got, op.want) { + t.Fatalf("%s: append('%c') = %v, want %v", sequence, op.kind, got, op.want) + } + if got == nil { + sequence = append(sequence, op.kind) + } + case needDelim: + if got := state.needDelim(op.next); got != op.want { + t.Fatalf("%s: needDelim('%c') = '%c', want '%c'", sequence, op.next, got, op.want) + } + default: + panic(fmt.Sprintf("unknown operation: %T", op)) + } + } + }) + } +} + +// append is a thin wrapper over the other append, pop, or push methods +// based on the token kind. +func (s *stateMachine) append(k Kind) error { + switch k { + case 'n', 'f', 't': + return s.appendLiteral() + case '"': + return s.appendString() + case '0': + return s.appendNumber() + case '{': + return s.pushObject() + case '}': + return s.popObject() + case '[': + return s.pushArray() + case ']': + return s.popArray() + default: + panic(fmt.Sprintf("invalid token kind: '%c'", k)) + } +} + +func TestObjectNamespace(t *testing.T) { + type operation any + type ( + insert struct { + name string + wantInserted bool + } + removeLast struct{} + ) + + // Sequence of insert operations to perform (order matters). + ops := []operation{ + insert{`""`, true}, + removeLast{}, + insert{`""`, true}, + insert{`""`, false}, + + // Test insertion of the same name with different formatting. + insert{`"alpha"`, true}, + insert{`"ALPHA"`, true}, // case-sensitive matching + insert{`"alpha"`, false}, + insert{`"\u0061\u006c\u0070\u0068\u0061"`, false}, // unescapes to "alpha" + removeLast{}, // removes "ALPHA" + insert{`"alpha"`, false}, + removeLast{}, // removes "alpha" + insert{`"alpha"`, true}, + removeLast{}, + + // Bulk insert simple names. + insert{`"alpha"`, true}, + insert{`"bravo"`, true}, + insert{`"charlie"`, true}, + insert{`"delta"`, true}, + insert{`"echo"`, true}, + insert{`"foxtrot"`, true}, + insert{`"golf"`, true}, + insert{`"hotel"`, true}, + insert{`"india"`, true}, + insert{`"juliet"`, true}, + insert{`"kilo"`, true}, + insert{`"lima"`, true}, + insert{`"mike"`, true}, + insert{`"november"`, true}, + insert{`"oscar"`, true}, + insert{`"papa"`, true}, + insert{`"quebec"`, true}, + insert{`"romeo"`, true}, + insert{`"sierra"`, true}, + insert{`"tango"`, true}, + insert{`"uniform"`, true}, + insert{`"victor"`, true}, + insert{`"whiskey"`, true}, + insert{`"xray"`, true}, + insert{`"yankee"`, true}, + insert{`"zulu"`, true}, + + // Test insertion of invalid UTF-8. + insert{`"` + "\ufffd" + `"`, true}, + insert{`"` + "\ufffd" + `"`, false}, + insert{`"\ufffd"`, false}, // unescapes to Unicode replacement character + insert{`"\uFFFD"`, false}, // unescapes to Unicode replacement character + insert{`"` + "\xff" + `"`, false}, // mangles as Unicode replacement character + removeLast{}, + insert{`"` + "\ufffd" + `"`, true}, + + // Test insertion of unicode characters. + insert{`"☺☻☹"`, true}, + insert{`"☺☻☹"`, false}, + removeLast{}, + insert{`"☺☻☹"`, true}, + } + + // Execute the sequence of operations twice: + // 1) on a fresh namespace and 2) on a namespace that has been reset. + var ns objectNamespace + wantNames := []string{} + for _, reset := range []bool{false, true} { + if reset { + ns.reset() + wantNames = nil + } + + // Execute the operations and ensure the state is consistent. + for i, op := range ops { + switch op := op.(type) { + case insert: + gotInserted := ns.insertQuoted([]byte(op.name), false) + if gotInserted != op.wantInserted { + t.Fatalf("%d: objectNamespace{%v}.insert(%v) = %v, want %v", i, strings.Join(wantNames, " "), op.name, gotInserted, op.wantInserted) + } + if gotInserted { + b, _ := AppendUnquote(nil, []byte(op.name)) + wantNames = append(wantNames, string(b)) + } + case removeLast: + ns.removeLast() + wantNames = wantNames[:len(wantNames)-1] + default: + panic(fmt.Sprintf("unknown operation: %T", op)) + } + + // Check that the namespace is consistent. + gotNames := []string{} + for i := range ns.length() { + gotNames = append(gotNames, string(ns.getUnquoted(i))) + } + if !slices.Equal(gotNames, wantNames) { + t.Fatalf("%d: objectNamespace = {%v}, want {%v}", i, strings.Join(gotNames, " "), strings.Join(wantNames, " ")) + } + } + + // Verify that we have not switched to using a Go map. + if ns.mapNames != nil { + t.Errorf("objectNamespace.mapNames = non-nil, want nil") + } + + // Insert a large number of names. + for i := range 64 { + ns.InsertUnquoted([]byte(fmt.Sprintf(`name%d`, i))) + } + + // Verify that we did switch to using a Go map. + if ns.mapNames == nil { + t.Errorf("objectNamespace.mapNames = nil, want non-nil") + } + } +} diff --git a/src/encoding/json/jsontext/token.go b/src/encoding/json/jsontext/token.go new file mode 100644 index 00000000000000..e78c3f84d8650f --- /dev/null +++ b/src/encoding/json/jsontext/token.go @@ -0,0 +1,527 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "bytes" + "errors" + "math" + "strconv" + + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonwire" +) + +// NOTE: Token is analogous to v1 json.Token. + +const ( + maxInt64 = math.MaxInt64 + minInt64 = math.MinInt64 + maxUint64 = math.MaxUint64 + minUint64 = 0 // for consistency and readability purposes + + invalidTokenPanic = "invalid jsontext.Token; it has been voided by a subsequent json.Decoder call" +) + +var errInvalidToken = errors.New("invalid jsontext.Token") + +// Token represents a lexical JSON token, which may be one of the following: +// - a JSON literal (i.e., null, true, or false) +// - a JSON string (e.g., "hello, world!") +// - a JSON number (e.g., 123.456) +// - a begin or end delimiter for a JSON object (i.e., { or } ) +// - a begin or end delimiter for a JSON array (i.e., [ or ] ) +// +// A Token cannot represent entire array or object values, while a [Value] can. +// There is no Token to represent commas and colons since +// these structural tokens can be inferred from the surrounding context. +type Token struct { + nonComparable + + // Tokens can exist in either a "raw" or an "exact" form. + // Tokens produced by the Decoder are in the "raw" form. + // Tokens returned by constructors are usually in the "exact" form. + // The Encoder accepts Tokens in either the "raw" or "exact" form. + // + // The following chart shows the possible values for each Token type: + // ╔═════════════════╦════════════╤════════════╤════════════╗ + // ║ Token type ║ raw field │ str field │ num field ║ + // ╠═════════════════╬════════════╪════════════╪════════════╣ + // ║ null (raw) ║ "null" │ "" │ 0 ║ + // ║ false (raw) ║ "false" │ "" │ 0 ║ + // ║ true (raw) ║ "true" │ "" │ 0 ║ + // ║ string (raw) ║ non-empty │ "" │ offset ║ + // ║ string (string) ║ nil │ non-empty │ 0 ║ + // ║ number (raw) ║ non-empty │ "" │ offset ║ + // ║ number (float) ║ nil │ "f" │ non-zero ║ + // ║ number (int64) ║ nil │ "i" │ non-zero ║ + // ║ number (uint64) ║ nil │ "u" │ non-zero ║ + // ║ object (delim) ║ "{" or "}" │ "" │ 0 ║ + // ║ array (delim) ║ "[" or "]" │ "" │ 0 ║ + // ╚═════════════════╩════════════╧════════════╧════════════╝ + // + // Notes: + // - For tokens stored in "raw" form, the num field contains the + // absolute offset determined by raw.previousOffsetStart(). + // The buffer itself is stored in raw.previousBuffer(). + // - JSON literals and structural characters are always in the "raw" form. + // - JSON strings and numbers can be in either "raw" or "exact" forms. + // - The exact zero value of JSON strings and numbers in the "exact" forms + // have ambiguous representation. Thus, they are always represented + // in the "raw" form. + + // raw contains a reference to the raw decode buffer. + // If non-nil, then its value takes precedence over str and num. + // It is only valid if num == raw.previousOffsetStart(). + raw *decodeBuffer + + // str is the unescaped JSON string if num is zero. + // Otherwise, it is "f", "i", or "u" if num should be interpreted + // as a float64, int64, or uint64, respectively. + str string + + // num is a float64, int64, or uint64 stored as a uint64 value. + // It is non-zero for any JSON number in the "exact" form. + num uint64 +} + +// TODO: Does representing 1-byte delimiters as *decodeBuffer cause performance issues? + +var ( + Null Token = rawToken("null") + False Token = rawToken("false") + True Token = rawToken("true") + + BeginObject Token = rawToken("{") + EndObject Token = rawToken("}") + BeginArray Token = rawToken("[") + EndArray Token = rawToken("]") + + zeroString Token = rawToken(`""`) + zeroNumber Token = rawToken(`0`) + + nanString Token = String("NaN") + pinfString Token = String("Infinity") + ninfString Token = String("-Infinity") +) + +func rawToken(s string) Token { + return Token{raw: &decodeBuffer{buf: []byte(s), prevStart: 0, prevEnd: len(s)}} +} + +// Bool constructs a Token representing a JSON boolean. +func Bool(b bool) Token { + if b { + return True + } + return False +} + +// String constructs a Token representing a JSON string. +// The provided string should contain valid UTF-8, otherwise invalid characters +// may be mangled as the Unicode replacement character. +func String(s string) Token { + if len(s) == 0 { + return zeroString + } + return Token{str: s} +} + +// Float constructs a Token representing a JSON number. +// The values NaN, +Inf, and -Inf will be represented +// as a JSON string with the values "NaN", "Infinity", and "-Infinity". +func Float(n float64) Token { + switch { + case math.Float64bits(n) == 0: + return zeroNumber + case math.IsNaN(n): + return nanString + case math.IsInf(n, +1): + return pinfString + case math.IsInf(n, -1): + return ninfString + } + return Token{str: "f", num: math.Float64bits(n)} +} + +// Int constructs a Token representing a JSON number from an int64. +func Int(n int64) Token { + if n == 0 { + return zeroNumber + } + return Token{str: "i", num: uint64(n)} +} + +// Uint constructs a Token representing a JSON number from a uint64. +func Uint(n uint64) Token { + if n == 0 { + return zeroNumber + } + return Token{str: "u", num: uint64(n)} +} + +// Clone makes a copy of the Token such that its value remains valid +// even after a subsequent [Decoder.Read] call. +func (t Token) Clone() Token { + // TODO: Allow caller to avoid any allocations? + if raw := t.raw; raw != nil { + // Avoid copying globals. + if t.raw.prevStart == 0 { + switch t.raw { + case Null.raw: + return Null + case False.raw: + return False + case True.raw: + return True + case BeginObject.raw: + return BeginObject + case EndObject.raw: + return EndObject + case BeginArray.raw: + return BeginArray + case EndArray.raw: + return EndArray + } + } + + if uint64(raw.previousOffsetStart()) != t.num { + panic(invalidTokenPanic) + } + buf := bytes.Clone(raw.previousBuffer()) + return Token{raw: &decodeBuffer{buf: buf, prevStart: 0, prevEnd: len(buf)}} + } + return t +} + +// Bool returns the value for a JSON boolean. +// It panics if the token kind is not a JSON boolean. +func (t Token) Bool() bool { + switch t.raw { + case True.raw: + return true + case False.raw: + return false + default: + panic("invalid JSON token kind: " + t.Kind().String()) + } +} + +// appendString appends a JSON string to dst and returns it. +// It panics if t is not a JSON string. +func (t Token) appendString(dst []byte, flags *jsonflags.Flags) ([]byte, error) { + if raw := t.raw; raw != nil { + // Handle raw string value. + buf := raw.previousBuffer() + if Kind(buf[0]) == '"' { + if jsonwire.ConsumeSimpleString(buf) == len(buf) { + return append(dst, buf...), nil + } + dst, _, err := jsonwire.ReformatString(dst, buf, flags) + return dst, err + } + } else if len(t.str) != 0 && t.num == 0 { + // Handle exact string value. + return jsonwire.AppendQuote(dst, t.str, flags) + } + + panic("invalid JSON token kind: " + t.Kind().String()) +} + +// String returns the unescaped string value for a JSON string. +// For other JSON kinds, this returns the raw JSON representation. +func (t Token) String() string { + // This is inlinable to take advantage of "function outlining". + // This avoids an allocation for the string(b) conversion + // if the caller does not use the string in an escaping manner. + // See https://blog.filippo.io/efficient-go-apis-with-the-inliner/ + s, b := t.string() + if len(b) > 0 { + return string(b) + } + return s +} +func (t Token) string() (string, []byte) { + if raw := t.raw; raw != nil { + if uint64(raw.previousOffsetStart()) != t.num { + panic(invalidTokenPanic) + } + buf := raw.previousBuffer() + if buf[0] == '"' { + // TODO: Preserve ValueFlags in Token? + isVerbatim := jsonwire.ConsumeSimpleString(buf) == len(buf) + return "", jsonwire.UnquoteMayCopy(buf, isVerbatim) + } + // Handle tokens that are not JSON strings for fmt.Stringer. + return "", buf + } + if len(t.str) != 0 && t.num == 0 { + return t.str, nil + } + // Handle tokens that are not JSON strings for fmt.Stringer. + if t.num > 0 { + switch t.str[0] { + case 'f': + return string(jsonwire.AppendFloat(nil, math.Float64frombits(t.num), 64)), nil + case 'i': + return strconv.FormatInt(int64(t.num), 10), nil + case 'u': + return strconv.FormatUint(uint64(t.num), 10), nil + } + } + return "", nil +} + +// appendNumber appends a JSON number to dst and returns it. +// It panics if t is not a JSON number. +func (t Token) appendNumber(dst []byte, flags *jsonflags.Flags) ([]byte, error) { + if raw := t.raw; raw != nil { + // Handle raw number value. + buf := raw.previousBuffer() + if Kind(buf[0]).normalize() == '0' { + dst, _, err := jsonwire.ReformatNumber(dst, buf, flags) + return dst, err + } + } else if t.num != 0 { + // Handle exact number value. + switch t.str[0] { + case 'f': + return jsonwire.AppendFloat(dst, math.Float64frombits(t.num), 64), nil + case 'i': + return strconv.AppendInt(dst, int64(t.num), 10), nil + case 'u': + return strconv.AppendUint(dst, uint64(t.num), 10), nil + } + } + + panic("invalid JSON token kind: " + t.Kind().String()) +} + +// Float returns the floating-point value for a JSON number. +// It returns a NaN, +Inf, or -Inf value for any JSON string +// with the values "NaN", "Infinity", or "-Infinity". +// It panics for all other cases. +func (t Token) Float() float64 { + if raw := t.raw; raw != nil { + // Handle raw number value. + if uint64(raw.previousOffsetStart()) != t.num { + panic(invalidTokenPanic) + } + buf := raw.previousBuffer() + if Kind(buf[0]).normalize() == '0' { + fv, _ := jsonwire.ParseFloat(buf, 64) + return fv + } + } else if t.num != 0 { + // Handle exact number value. + switch t.str[0] { + case 'f': + return math.Float64frombits(t.num) + case 'i': + return float64(int64(t.num)) + case 'u': + return float64(uint64(t.num)) + } + } + + // Handle string values with "NaN", "Infinity", or "-Infinity". + if t.Kind() == '"' { + switch t.String() { + case "NaN": + return math.NaN() + case "Infinity": + return math.Inf(+1) + case "-Infinity": + return math.Inf(-1) + } + } + + panic("invalid JSON token kind: " + t.Kind().String()) +} + +// Int returns the signed integer value for a JSON number. +// The fractional component of any number is ignored (truncation toward zero). +// Any number beyond the representation of an int64 will be saturated +// to the closest representable value. +// It panics if the token kind is not a JSON number. +func (t Token) Int() int64 { + if raw := t.raw; raw != nil { + // Handle raw integer value. + if uint64(raw.previousOffsetStart()) != t.num { + panic(invalidTokenPanic) + } + neg := false + buf := raw.previousBuffer() + if len(buf) > 0 && buf[0] == '-' { + neg, buf = true, buf[1:] + } + if numAbs, ok := jsonwire.ParseUint(buf); ok { + if neg { + if numAbs > -minInt64 { + return minInt64 + } + return -1 * int64(numAbs) + } else { + if numAbs > +maxInt64 { + return maxInt64 + } + return +1 * int64(numAbs) + } + } + } else if t.num != 0 { + // Handle exact integer value. + switch t.str[0] { + case 'i': + return int64(t.num) + case 'u': + if t.num > maxInt64 { + return maxInt64 + } + return int64(t.num) + } + } + + // Handle JSON number that is a floating-point value. + if t.Kind() == '0' { + switch fv := t.Float(); { + case fv >= maxInt64: + return maxInt64 + case fv <= minInt64: + return minInt64 + default: + return int64(fv) // truncation toward zero + } + } + + panic("invalid JSON token kind: " + t.Kind().String()) +} + +// Uint returns the unsigned integer value for a JSON number. +// The fractional component of any number is ignored (truncation toward zero). +// Any number beyond the representation of an uint64 will be saturated +// to the closest representable value. +// It panics if the token kind is not a JSON number. +func (t Token) Uint() uint64 { + // NOTE: This accessor returns 0 for any negative JSON number, + // which might be surprising, but is at least consistent with the behavior + // of saturating out-of-bounds numbers to the closest representable number. + + if raw := t.raw; raw != nil { + // Handle raw integer value. + if uint64(raw.previousOffsetStart()) != t.num { + panic(invalidTokenPanic) + } + neg := false + buf := raw.previousBuffer() + if len(buf) > 0 && buf[0] == '-' { + neg, buf = true, buf[1:] + } + if num, ok := jsonwire.ParseUint(buf); ok { + if neg { + return minUint64 + } + return num + } + } else if t.num != 0 { + // Handle exact integer value. + switch t.str[0] { + case 'u': + return t.num + case 'i': + if int64(t.num) < minUint64 { + return minUint64 + } + return uint64(int64(t.num)) + } + } + + // Handle JSON number that is a floating-point value. + if t.Kind() == '0' { + switch fv := t.Float(); { + case fv >= maxUint64: + return maxUint64 + case fv <= minUint64: + return minUint64 + default: + return uint64(fv) // truncation toward zero + } + } + + panic("invalid JSON token kind: " + t.Kind().String()) +} + +// Kind returns the token kind. +func (t Token) Kind() Kind { + switch { + case t.raw != nil: + raw := t.raw + if uint64(raw.previousOffsetStart()) != t.num { + panic(invalidTokenPanic) + } + return Kind(t.raw.buf[raw.prevStart]).normalize() + case t.num != 0: + return '0' + case len(t.str) != 0: + return '"' + default: + return invalidKind + } +} + +// Kind represents each possible JSON token kind with a single byte, +// which is conveniently the first byte of that kind's grammar +// with the restriction that numbers always be represented with '0': +// +// - 'n': null +// - 'f': false +// - 't': true +// - '"': string +// - '0': number +// - '{': object begin +// - '}': object end +// - '[': array begin +// - ']': array end +// +// An invalid kind is usually represented using 0, +// but may be non-zero due to invalid JSON data. +type Kind byte + +const invalidKind Kind = 0 + +// String prints the kind in a humanly readable fashion. +func (k Kind) String() string { + switch k { + case 'n': + return "null" + case 'f': + return "false" + case 't': + return "true" + case '"': + return "string" + case '0': + return "number" + case '{': + return "{" + case '}': + return "}" + case '[': + return "[" + case ']': + return "]" + default: + return "" + } +} + +// normalize coalesces all possible starting characters of a number as just '0'. +func (k Kind) normalize() Kind { + if k == '-' || ('0' <= k && k <= '9') { + return '0' + } + return k +} diff --git a/src/encoding/json/jsontext/token_test.go b/src/encoding/json/jsontext/token_test.go new file mode 100644 index 00000000000000..ebe324e0dbcce6 --- /dev/null +++ b/src/encoding/json/jsontext/token_test.go @@ -0,0 +1,168 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "math" + "reflect" + "testing" +) + +func TestTokenStringAllocations(t *testing.T) { + if testing.CoverMode() != "" { + t.Skip("coverage mode breaks the compiler optimization this depends on") + } + + tok := rawToken(`"hello"`) + var m map[string]bool + got := int(testing.AllocsPerRun(10, func() { + // This function uses tok.String() is a non-escaping manner + // (i.e., looking it up in a Go map). It should not allocate. + if m[tok.String()] { + panic("never executed") + } + })) + if got > 0 { + t.Errorf("Token.String allocated %d times, want 0", got) + } +} + +func TestTokenAccessors(t *testing.T) { + type token struct { + Bool bool + String string + Float float64 + Int int64 + Uint uint64 + Kind Kind + } + + tests := []struct { + in Token + want token + }{ + {Token{}, token{String: ""}}, + {Null, token{String: "null", Kind: 'n'}}, + {False, token{Bool: false, String: "false", Kind: 'f'}}, + {True, token{Bool: true, String: "true", Kind: 't'}}, + {Bool(false), token{Bool: false, String: "false", Kind: 'f'}}, + {Bool(true), token{Bool: true, String: "true", Kind: 't'}}, + {BeginObject, token{String: "{", Kind: '{'}}, + {EndObject, token{String: "}", Kind: '}'}}, + {BeginArray, token{String: "[", Kind: '['}}, + {EndArray, token{String: "]", Kind: ']'}}, + {String(""), token{String: "", Kind: '"'}}, + {String("hello, world!"), token{String: "hello, world!", Kind: '"'}}, + {rawToken(`"hello, world!"`), token{String: "hello, world!", Kind: '"'}}, + {Float(0), token{String: "0", Float: 0, Int: 0, Uint: 0, Kind: '0'}}, + {Float(math.Copysign(0, -1)), token{String: "-0", Float: math.Copysign(0, -1), Int: 0, Uint: 0, Kind: '0'}}, + {Float(math.NaN()), token{String: "NaN", Float: math.NaN(), Int: 0, Uint: 0, Kind: '"'}}, + {Float(math.Inf(+1)), token{String: "Infinity", Float: math.Inf(+1), Kind: '"'}}, + {Float(math.Inf(-1)), token{String: "-Infinity", Float: math.Inf(-1), Kind: '"'}}, + {Int(minInt64), token{String: "-9223372036854775808", Float: minInt64, Int: minInt64, Uint: minUint64, Kind: '0'}}, + {Int(minInt64 + 1), token{String: "-9223372036854775807", Float: minInt64 + 1, Int: minInt64 + 1, Uint: minUint64, Kind: '0'}}, + {Int(-1), token{String: "-1", Float: -1, Int: -1, Uint: minUint64, Kind: '0'}}, + {Int(0), token{String: "0", Float: 0, Int: 0, Uint: 0, Kind: '0'}}, + {Int(+1), token{String: "1", Float: +1, Int: +1, Uint: +1, Kind: '0'}}, + {Int(maxInt64 - 1), token{String: "9223372036854775806", Float: maxInt64 - 1, Int: maxInt64 - 1, Uint: maxInt64 - 1, Kind: '0'}}, + {Int(maxInt64), token{String: "9223372036854775807", Float: maxInt64, Int: maxInt64, Uint: maxInt64, Kind: '0'}}, + {Uint(minUint64), token{String: "0", Kind: '0'}}, + {Uint(minUint64 + 1), token{String: "1", Float: minUint64 + 1, Int: minUint64 + 1, Uint: minUint64 + 1, Kind: '0'}}, + {Uint(maxUint64 - 1), token{String: "18446744073709551614", Float: maxUint64 - 1, Int: maxInt64, Uint: maxUint64 - 1, Kind: '0'}}, + {Uint(maxUint64), token{String: "18446744073709551615", Float: maxUint64, Int: maxInt64, Uint: maxUint64, Kind: '0'}}, + {rawToken(`-0`), token{String: "-0", Float: math.Copysign(0, -1), Int: 0, Uint: 0, Kind: '0'}}, + {rawToken(`1e1000`), token{String: "1e1000", Float: math.MaxFloat64, Int: maxInt64, Uint: maxUint64, Kind: '0'}}, + {rawToken(`-1e1000`), token{String: "-1e1000", Float: -math.MaxFloat64, Int: minInt64, Uint: minUint64, Kind: '0'}}, + {rawToken(`0.1`), token{String: "0.1", Float: 0.1, Int: 0, Uint: 0, Kind: '0'}}, + {rawToken(`0.5`), token{String: "0.5", Float: 0.5, Int: 0, Uint: 0, Kind: '0'}}, + {rawToken(`0.9`), token{String: "0.9", Float: 0.9, Int: 0, Uint: 0, Kind: '0'}}, + {rawToken(`1.1`), token{String: "1.1", Float: 1.1, Int: 1, Uint: 1, Kind: '0'}}, + {rawToken(`-0.1`), token{String: "-0.1", Float: -0.1, Int: 0, Uint: 0, Kind: '0'}}, + {rawToken(`-0.5`), token{String: "-0.5", Float: -0.5, Int: 0, Uint: 0, Kind: '0'}}, + {rawToken(`-0.9`), token{String: "-0.9", Float: -0.9, Int: 0, Uint: 0, Kind: '0'}}, + {rawToken(`-1.1`), token{String: "-1.1", Float: -1.1, Int: -1, Uint: 0, Kind: '0'}}, + {rawToken(`99999999999999999999`), token{String: "99999999999999999999", Float: 1e20 - 1, Int: maxInt64, Uint: maxUint64, Kind: '0'}}, + {rawToken(`-99999999999999999999`), token{String: "-99999999999999999999", Float: -1e20 - 1, Int: minInt64, Uint: minUint64, Kind: '0'}}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + got := token{ + Bool: func() bool { + defer func() { recover() }() + return tt.in.Bool() + }(), + String: tt.in.String(), + Float: func() float64 { + defer func() { recover() }() + return tt.in.Float() + }(), + Int: func() int64 { + defer func() { recover() }() + return tt.in.Int() + }(), + Uint: func() uint64 { + defer func() { recover() }() + return tt.in.Uint() + }(), + Kind: tt.in.Kind(), + } + + if got.Bool != tt.want.Bool { + t.Errorf("Token(%s).Bool() = %v, want %v", tt.in, got.Bool, tt.want.Bool) + } + if got.String != tt.want.String { + t.Errorf("Token(%s).String() = %v, want %v", tt.in, got.String, tt.want.String) + } + if math.Float64bits(got.Float) != math.Float64bits(tt.want.Float) { + t.Errorf("Token(%s).Float() = %v, want %v", tt.in, got.Float, tt.want.Float) + } + if got.Int != tt.want.Int { + t.Errorf("Token(%s).Int() = %v, want %v", tt.in, got.Int, tt.want.Int) + } + if got.Uint != tt.want.Uint { + t.Errorf("Token(%s).Uint() = %v, want %v", tt.in, got.Uint, tt.want.Uint) + } + if got.Kind != tt.want.Kind { + t.Errorf("Token(%s).Kind() = %v, want %v", tt.in, got.Kind, tt.want.Kind) + } + }) + } +} + +func TestTokenClone(t *testing.T) { + tests := []struct { + in Token + wantExactRaw bool + }{ + {Token{}, true}, + {Null, true}, + {False, true}, + {True, true}, + {BeginObject, true}, + {EndObject, true}, + {BeginArray, true}, + {EndArray, true}, + {String("hello, world!"), true}, + {rawToken(`"hello, world!"`), false}, + {Float(3.14159), true}, + {rawToken(`3.14159`), false}, + } + + for _, tt := range tests { + t.Run("", func(t *testing.T) { + got := tt.in.Clone() + if !reflect.DeepEqual(got, tt.in) { + t.Errorf("Token(%s) == Token(%s).Clone() = false, want true", tt.in, tt.in) + } + gotExactRaw := got.raw == tt.in.raw + if gotExactRaw != tt.wantExactRaw { + t.Errorf("Token(%s).raw == Token(%s).Clone().raw = %v, want %v", tt.in, tt.in, gotExactRaw, tt.wantExactRaw) + } + }) + } +} diff --git a/src/encoding/json/jsontext/value.go b/src/encoding/json/jsontext/value.go new file mode 100644 index 00000000000000..a4b06b2a947de7 --- /dev/null +++ b/src/encoding/json/jsontext/value.go @@ -0,0 +1,395 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "bytes" + "errors" + "io" + "slices" + "sync" + + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonwire" +) + +// NOTE: Value is analogous to v1 json.RawMessage. + +// AppendFormat formats the JSON value in src and appends it to dst +// according to the specified options. +// See [Value.Format] for more details about the formatting behavior. +// +// The dst and src may overlap. +// If an error is reported, then the entirety of src is appended to dst. +func AppendFormat(dst, src []byte, opts ...Options) ([]byte, error) { + e := getBufferedEncoder(opts...) + defer putBufferedEncoder(e) + e.s.Flags.Set(jsonflags.OmitTopLevelNewline | 1) + if err := e.s.WriteValue(src); err != nil { + return append(dst, src...), err + } + return append(dst, e.s.Buf...), nil +} + +// Value represents a single raw JSON value, which may be one of the following: +// - a JSON literal (i.e., null, true, or false) +// - a JSON string (e.g., "hello, world!") +// - a JSON number (e.g., 123.456) +// - an entire JSON object (e.g., {"fizz":"buzz"} ) +// - an entire JSON array (e.g., [1,2,3] ) +// +// Value can represent entire array or object values, while [Token] cannot. +// Value may contain leading and/or trailing whitespace. +type Value []byte + +// Clone returns a copy of v. +func (v Value) Clone() Value { + return bytes.Clone(v) +} + +// String returns the string formatting of v. +func (v Value) String() string { + if v == nil { + return "null" + } + return string(v) +} + +// IsValid reports whether the raw JSON value is syntactically valid +// according to the specified options. +// +// By default (if no options are specified), it validates according to RFC 7493. +// It verifies whether the input is properly encoded as UTF-8, +// that escape sequences within strings decode to valid Unicode codepoints, and +// that all names in each object are unique. +// It does not verify whether numbers are representable within the limits +// of any common numeric type (e.g., float64, int64, or uint64). +// +// Relevant options include: +// - [AllowDuplicateNames] +// - [AllowInvalidUTF8] +// +// All other options are ignored. +func (v Value) IsValid(opts ...Options) bool { + // TODO: Document support for [WithByteLimit] and [WithDepthLimit]. + d := getBufferedDecoder(v, opts...) + defer putBufferedDecoder(d) + _, errVal := d.ReadValue() + _, errEOF := d.ReadToken() + return errVal == nil && errEOF == io.EOF +} + +// Format formats the raw JSON value in place. +// +// By default (if no options are specified), it validates according to RFC 7493 +// and produces the minimal JSON representation, where +// all whitespace is elided and JSON strings use the shortest encoding. +// +// Relevant options include: +// - [AllowDuplicateNames] +// - [AllowInvalidUTF8] +// - [EscapeForHTML] +// - [EscapeForJS] +// - [PreserveRawStrings] +// - [CanonicalizeRawInts] +// - [CanonicalizeRawFloats] +// - [ReorderRawObjects] +// - [SpaceAfterColon] +// - [SpaceAfterComma] +// - [Multiline] +// - [WithIndent] +// - [WithIndentPrefix] +// +// All other options are ignored. +// +// It is guaranteed to succeed if the value is valid according to the same options. +// If the value is already formatted, then the buffer is not mutated. +func (v *Value) Format(opts ...Options) error { + // TODO: Document support for [WithByteLimit] and [WithDepthLimit]. + return v.format(opts, nil) +} + +// format accepts two []Options to avoid the allocation appending them together. +// It is equivalent to v.Format(append(opts1, opts2...)...). +func (v *Value) format(opts1, opts2 []Options) error { + e := getBufferedEncoder(opts1...) + defer putBufferedEncoder(e) + e.s.Join(opts2...) + e.s.Flags.Set(jsonflags.OmitTopLevelNewline | 1) + if err := e.s.WriteValue(*v); err != nil { + return err + } + if !bytes.Equal(*v, e.s.Buf) { + *v = append((*v)[:0], e.s.Buf...) + } + return nil +} + +// Compact removes all whitespace from the raw JSON value. +// +// It does not reformat JSON strings or numbers to use any other representation. +// To maximize the set of JSON values that can be formatted, +// this permits values with duplicate names and invalid UTF-8. +// +// Compact is equivalent to calling [Value.Format] with the following options: +// - [AllowDuplicateNames](true) +// - [AllowInvalidUTF8](true) +// - [PreserveRawStrings](true) +// +// Any options specified by the caller are applied after the initial set +// and may deliberately override prior options. +func (v *Value) Compact(opts ...Options) error { + return v.format([]Options{ + AllowDuplicateNames(true), + AllowInvalidUTF8(true), + PreserveRawStrings(true), + }, opts) +} + +// Indent reformats the whitespace in the raw JSON value so that each element +// in a JSON object or array begins on a indented line according to the nesting. +// +// It does not reformat JSON strings or numbers to use any other representation. +// To maximize the set of JSON values that can be formatted, +// this permits values with duplicate names and invalid UTF-8. +// +// Indent is equivalent to calling [Value.Format] with the following options: +// - [AllowDuplicateNames](true) +// - [AllowInvalidUTF8](true) +// - [PreserveRawStrings](true) +// - [Multiline](true) +// +// Any options specified by the caller are applied after the initial set +// and may deliberately override prior options. +func (v *Value) Indent(opts ...Options) error { + return v.format([]Options{ + AllowDuplicateNames(true), + AllowInvalidUTF8(true), + PreserveRawStrings(true), + Multiline(true), + }, opts) +} + +// Canonicalize canonicalizes the raw JSON value according to the +// JSON Canonicalization Scheme (JCS) as defined by RFC 8785 +// where it produces a stable representation of a JSON value. +// +// JSON strings are formatted to use their minimal representation, +// JSON numbers are formatted as double precision numbers according +// to some stable serialization algorithm. +// JSON object members are sorted in ascending order by name. +// All whitespace is removed. +// +// The output stability is dependent on the stability of the application data +// (see RFC 8785, Appendix E). It cannot produce stable output from +// fundamentally unstable input. For example, if the JSON value +// contains ephemeral data (e.g., a frequently changing timestamp), +// then the value is still unstable regardless of whether this is called. +// +// Canonicalize is equivalent to calling [Value.Format] with the following options: +// - [CanonicalizeRawInts](true) +// - [CanonicalizeRawFloats](true) +// - [ReorderRawObjects](true) +// +// Any options specified by the caller are applied after the initial set +// and may deliberately override prior options. +// +// Note that JCS treats all JSON numbers as IEEE 754 double precision numbers. +// Any numbers with precision beyond what is representable by that form +// will lose their precision when canonicalized. For example, integer values +// beyond ±2⁵³ will lose their precision. To preserve the original representation +// of JSON integers, additionally set [CanonicalizeRawInts] to false: +// +// v.Canonicalize(jsontext.CanonicalizeRawInts(false)) +func (v *Value) Canonicalize(opts ...Options) error { + return v.format([]Options{ + CanonicalizeRawInts(true), + CanonicalizeRawFloats(true), + ReorderRawObjects(true), + }, opts) +} + +// MarshalJSON returns v as the JSON encoding of v. +// It returns the stored value as the raw JSON output without any validation. +// If v is nil, then this returns a JSON null. +func (v Value) MarshalJSON() ([]byte, error) { + // NOTE: This matches the behavior of v1 json.RawMessage.MarshalJSON. + if v == nil { + return []byte("null"), nil + } + return v, nil +} + +// UnmarshalJSON sets v as the JSON encoding of b. +// It stores a copy of the provided raw JSON input without any validation. +func (v *Value) UnmarshalJSON(b []byte) error { + // NOTE: This matches the behavior of v1 json.RawMessage.UnmarshalJSON. + if v == nil { + return errors.New("jsontext.Value: UnmarshalJSON on nil pointer") + } + *v = append((*v)[:0], b...) + return nil +} + +// Kind returns the starting token kind. +// For a valid value, this will never include '}' or ']'. +func (v Value) Kind() Kind { + if v := v[jsonwire.ConsumeWhitespace(v):]; len(v) > 0 { + return Kind(v[0]).normalize() + } + return invalidKind +} + +const commaAndWhitespace = ", \n\r\t" + +type objectMember struct { + // name is the unquoted name. + name []byte // e.g., "name" + // buffer is the entirety of the raw JSON object member + // starting from right after the previous member (or opening '{') + // until right after the member value. + buffer []byte // e.g., `, \n\r\t"name": "value"` +} + +func (x objectMember) Compare(y objectMember) int { + if c := jsonwire.CompareUTF16(x.name, y.name); c != 0 { + return c + } + // With [AllowDuplicateNames] or [AllowInvalidUTF8], + // names could be identical, so also sort using the member value. + return jsonwire.CompareUTF16( + bytes.TrimLeft(x.buffer, commaAndWhitespace), + bytes.TrimLeft(y.buffer, commaAndWhitespace)) +} + +var objectMemberPool = sync.Pool{New: func() any { return new([]objectMember) }} + +func getObjectMembers() *[]objectMember { + ns := objectMemberPool.Get().(*[]objectMember) + *ns = (*ns)[:0] + return ns +} +func putObjectMembers(ns *[]objectMember) { + if cap(*ns) < 1<<10 { + clear(*ns) // avoid pinning name and buffer + objectMemberPool.Put(ns) + } +} + +// mustReorderObjects reorders in-place all object members in a JSON value, +// which must be valid otherwise it panics. +func mustReorderObjects(b []byte) { + // Obtain a buffered encoder just to use its internal buffer as + // a scratch buffer for reordering object members. + e2 := getBufferedEncoder() + defer putBufferedEncoder(e2) + + // Disable unnecessary checks to syntactically parse the JSON value. + d := getBufferedDecoder(b) + defer putBufferedDecoder(d) + d.s.Flags.Set(jsonflags.AllowDuplicateNames | jsonflags.AllowInvalidUTF8 | 1) + mustReorderObjectsFromDecoder(d, &e2.s.Buf) // per RFC 8785, section 3.2.3 +} + +// mustReorderObjectsFromDecoder recursively reorders all object members in place +// according to the ordering specified in RFC 8785, section 3.2.3. +// +// Pre-conditions: +// - The value is valid (i.e., no decoder errors should ever occur). +// - Initial call is provided a Decoder reading from the start of v. +// +// Post-conditions: +// - Exactly one JSON value is read from the Decoder. +// - All fully-parsed JSON objects are reordered by directly moving +// the members in the value buffer. +// +// The runtime is approximately O(n·log(n)) + O(m·log(m)), +// where n is len(v) and m is the total number of object members. +func mustReorderObjectsFromDecoder(d *Decoder, scratch *[]byte) { + switch tok, err := d.ReadToken(); tok.Kind() { + case '{': + // Iterate and collect the name and offsets for every object member. + members := getObjectMembers() + defer putObjectMembers(members) + var prevMember objectMember + isSorted := true + + beforeBody := d.InputOffset() // offset after '{' + for d.PeekKind() != '}' { + beforeName := d.InputOffset() + var flags jsonwire.ValueFlags + name, _ := d.s.ReadValue(&flags) + name = jsonwire.UnquoteMayCopy(name, flags.IsVerbatim()) + mustReorderObjectsFromDecoder(d, scratch) + afterValue := d.InputOffset() + + currMember := objectMember{name, d.s.buf[beforeName:afterValue]} + if isSorted && len(*members) > 0 { + isSorted = objectMember.Compare(prevMember, currMember) < 0 + } + *members = append(*members, currMember) + prevMember = currMember + } + afterBody := d.InputOffset() // offset before '}' + d.ReadToken() + + // Sort the members; return early if it's already sorted. + if isSorted { + return + } + firstBufferBeforeSorting := (*members)[0].buffer + slices.SortFunc(*members, objectMember.Compare) + firstBufferAfterSorting := (*members)[0].buffer + + // Append the reordered members to a new buffer, + // then copy the reordered members back over the original members. + // Avoid swapping in place since each member may be a different size + // where moving a member over a smaller member may corrupt the data + // for subsequent members before they have been moved. + // + // The following invariant must hold: + // sum([m.after-m.before for m in members]) == afterBody-beforeBody + commaAndWhitespacePrefix := func(b []byte) []byte { + return b[:len(b)-len(bytes.TrimLeft(b, commaAndWhitespace))] + } + sorted := (*scratch)[:0] + for i, member := range *members { + switch { + case i == 0 && &member.buffer[0] != &firstBufferBeforeSorting[0]: + // First member after sorting is not the first member before sorting, + // so use the prefix of the first member before sorting. + sorted = append(sorted, commaAndWhitespacePrefix(firstBufferBeforeSorting)...) + sorted = append(sorted, bytes.TrimLeft(member.buffer, commaAndWhitespace)...) + case i != 0 && &member.buffer[0] == &firstBufferBeforeSorting[0]: + // Later member after sorting is the first member before sorting, + // so use the prefix of the first member after sorting. + sorted = append(sorted, commaAndWhitespacePrefix(firstBufferAfterSorting)...) + sorted = append(sorted, bytes.TrimLeft(member.buffer, commaAndWhitespace)...) + default: + sorted = append(sorted, member.buffer...) + } + } + if int(afterBody-beforeBody) != len(sorted) { + panic("BUG: length invariant violated") + } + copy(d.s.buf[beforeBody:afterBody], sorted) + + // Update scratch buffer to the largest amount ever used. + if len(sorted) > len(*scratch) { + *scratch = sorted + } + case '[': + for d.PeekKind() != ']' { + mustReorderObjectsFromDecoder(d, scratch) + } + d.ReadToken() + default: + if err != nil { + panic("BUG: " + err.Error()) + } + } +} diff --git a/src/encoding/json/jsontext/value_test.go b/src/encoding/json/jsontext/value_test.go new file mode 100644 index 00000000000000..184a27d88ea7f7 --- /dev/null +++ b/src/encoding/json/jsontext/value_test.go @@ -0,0 +1,200 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package jsontext + +import ( + "io" + "strings" + "testing" + + "encoding/json/internal/jsontest" + "encoding/json/internal/jsonwire" +) + +type valueTestdataEntry struct { + name jsontest.CaseName + in string + wantValid bool + wantCompacted string + wantCompactErr error // implies wantCompacted is in + wantIndented string // wantCompacted if empty; uses "\t" for indent prefix and " " for indent + wantIndentErr error // implies wantCompacted is in + wantCanonicalized string // wantCompacted if empty + wantCanonicalizeErr error // implies wantCompacted is in +} + +var valueTestdata = append(func() (out []valueTestdataEntry) { + // Initialize valueTestdata from coderTestdata. + for _, td := range coderTestdata { + // NOTE: The Compact method preserves the raw formatting of strings, + // while the Encoder (by default) does not. + if td.name.Name == "ComplicatedString" { + td.outCompacted = strings.TrimSpace(td.in) + } + out = append(out, valueTestdataEntry{ + name: td.name, + in: td.in, + wantValid: true, + wantCompacted: td.outCompacted, + wantIndented: td.outIndented, + wantCanonicalized: td.outCanonicalized, + }) + } + return out +}(), []valueTestdataEntry{{ + name: jsontest.Name("RFC8785/Primitives"), + in: `{ + "numbers": [333333333.33333329, 1E30, 4.50, + 2e-3, 0.000000000000000000000000001, -0], + "string": "\u20ac$\u000F\u000aA'\u0042\u0022\u005c\\\"\/", + "literals": [null, true, false] + }`, + wantValid: true, + wantCompacted: `{"numbers":[333333333.33333329,1E30,4.50,2e-3,0.000000000000000000000000001,-0],"string":"\u20ac$\u000F\u000aA'\u0042\u0022\u005c\\\"\/","literals":[null,true,false]}`, + wantIndented: `{ + "numbers": [ + 333333333.33333329, + 1E30, + 4.50, + 2e-3, + 0.000000000000000000000000001, + -0 + ], + "string": "\u20ac$\u000F\u000aA'\u0042\u0022\u005c\\\"\/", + "literals": [ + null, + true, + false + ] + }`, + wantCanonicalized: `{"literals":[null,true,false],"numbers":[333333333.3333333,1e+30,4.5,0.002,1e-27,0],"string":"€$\u000f\nA'B\"\\\\\"/"}`, +}, { + name: jsontest.Name("RFC8785/ObjectOrdering"), + in: `{ + "\u20ac": "Euro Sign", + "\r": "Carriage Return", + "\ufb33": "Hebrew Letter Dalet With Dagesh", + "1": "One", + "\ud83d\ude00": "Emoji: Grinning Face", + "\u0080": "Control", + "\u00f6": "Latin Small Letter O With Diaeresis" + }`, + wantValid: true, + wantCompacted: `{"\u20ac":"Euro Sign","\r":"Carriage Return","\ufb33":"Hebrew Letter Dalet With Dagesh","1":"One","\ud83d\ude00":"Emoji: Grinning Face","\u0080":"Control","\u00f6":"Latin Small Letter O With Diaeresis"}`, + wantIndented: `{ + "\u20ac": "Euro Sign", + "\r": "Carriage Return", + "\ufb33": "Hebrew Letter Dalet With Dagesh", + "1": "One", + "\ud83d\ude00": "Emoji: Grinning Face", + "\u0080": "Control", + "\u00f6": "Latin Small Letter O With Diaeresis" + }`, + wantCanonicalized: `{"\r":"Carriage Return","1":"One","€":"Control","ö":"Latin Small Letter O With Diaeresis","€":"Euro Sign","😀":"Emoji: Grinning Face","דּ":"Hebrew Letter Dalet With Dagesh"}`, +}, { + name: jsontest.Name("LargeIntegers"), + in: ` [ -9223372036854775808 , 9223372036854775807 ] `, + wantValid: true, + wantCompacted: `[-9223372036854775808,9223372036854775807]`, + wantIndented: `[ + -9223372036854775808, + 9223372036854775807 + ]`, + wantCanonicalized: `[-9223372036854776000,9223372036854776000]`, // NOTE: Loss of precision due to numbers being treated as floats. +}, { + name: jsontest.Name("InvalidUTF8"), + in: ` "living` + "\xde\xad\xbe\xef" + `\ufffd�" `, + wantValid: false, // uses RFC 7493 as the definition; which validates UTF-8 + wantCompacted: `"living` + "\xde\xad\xbe\xef" + `\ufffd�"`, + wantCanonicalizeErr: E(jsonwire.ErrInvalidUTF8).withPos(` "living`+"\xde\xad", ""), +}, { + name: jsontest.Name("InvalidUTF8/SurrogateHalf"), + in: `"\ud800"`, + wantValid: false, // uses RFC 7493 as the definition; which validates UTF-8 + wantCompacted: `"\ud800"`, + wantCanonicalizeErr: newInvalidEscapeSequenceError(`\ud800"`).withPos(`"`, ""), +}, { + name: jsontest.Name("UppercaseEscaped"), + in: `"\u000B"`, + wantValid: true, + wantCompacted: `"\u000B"`, + wantCanonicalized: `"\u000b"`, +}, { + name: jsontest.Name("DuplicateNames"), + in: ` { "0" : 0 , "1" : 1 , "0" : 0 }`, + wantValid: false, // uses RFC 7493 as the definition; which does check for object uniqueness + wantCompacted: `{"0":0,"1":1,"0":0}`, + wantIndented: `{ + "0": 0, + "1": 1, + "0": 0 + }`, + wantCanonicalizeErr: E(ErrDuplicateName).withPos(` { "0" : 0 , "1" : 1 , `, "/0"), +}, { + name: jsontest.Name("Whitespace"), + in: " \n\r\t", + wantValid: false, + wantCompacted: " \n\r\t", + wantCompactErr: E(io.ErrUnexpectedEOF).withPos(" \n\r\t", ""), + wantIndentErr: E(io.ErrUnexpectedEOF).withPos(" \n\r\t", ""), + wantCanonicalizeErr: E(io.ErrUnexpectedEOF).withPos(" \n\r\t", ""), +}}...) + +func TestValueMethods(t *testing.T) { + for _, td := range valueTestdata { + t.Run(td.name.Name, func(t *testing.T) { + if td.wantIndented == "" { + td.wantIndented = td.wantCompacted + } + if td.wantCanonicalized == "" { + td.wantCanonicalized = td.wantCompacted + } + if td.wantCompactErr != nil { + td.wantCompacted = td.in + } + if td.wantIndentErr != nil { + td.wantIndented = td.in + } + if td.wantCanonicalizeErr != nil { + td.wantCanonicalized = td.in + } + + v := Value(td.in) + gotValid := v.IsValid() + if gotValid != td.wantValid { + t.Errorf("%s: Value.IsValid = %v, want %v", td.name.Where, gotValid, td.wantValid) + } + + gotCompacted := Value(td.in) + gotCompactErr := gotCompacted.Compact() + if string(gotCompacted) != td.wantCompacted { + t.Errorf("%s: Value.Compact = %s, want %s", td.name.Where, gotCompacted, td.wantCompacted) + } + if !equalError(gotCompactErr, td.wantCompactErr) { + t.Errorf("%s: Value.Compact error mismatch:\ngot %v\nwant %v", td.name.Where, gotCompactErr, td.wantCompactErr) + } + + gotIndented := Value(td.in) + gotIndentErr := gotIndented.Indent(WithIndentPrefix("\t"), WithIndent(" ")) + if string(gotIndented) != td.wantIndented { + t.Errorf("%s: Value.Indent = %s, want %s", td.name.Where, gotIndented, td.wantIndented) + } + if !equalError(gotIndentErr, td.wantIndentErr) { + t.Errorf("%s: Value.Indent error mismatch:\ngot %v\nwant %v", td.name.Where, gotIndentErr, td.wantIndentErr) + } + + gotCanonicalized := Value(td.in) + gotCanonicalizeErr := gotCanonicalized.Canonicalize() + if string(gotCanonicalized) != td.wantCanonicalized { + t.Errorf("%s: Value.Canonicalize = %s, want %s", td.name.Where, gotCanonicalized, td.wantCanonicalized) + } + if !equalError(gotCanonicalizeErr, td.wantCanonicalizeErr) { + t.Errorf("%s: Value.Canonicalize error mismatch:\ngot %v\nwant %v", td.name.Where, gotCanonicalizeErr, td.wantCanonicalizeErr) + } + }) + } +} diff --git a/src/encoding/json/number_test.go b/src/encoding/json/number_test.go index c82e6deb832626..69eccaaffd5706 100644 --- a/src/encoding/json/number_test.go +++ b/src/encoding/json/number_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json import ( diff --git a/src/encoding/json/scanner.go b/src/encoding/json/scanner.go index da6ea2ac8f843f..f4086186e29b0b 100644 --- a/src/encoding/json/scanner.go +++ b/src/encoding/json/scanner.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json // JSON value parser state machine. @@ -173,7 +175,7 @@ func (s *scanner) eof() int { return scanError } -// pushParseState pushes a new parse state p onto the parse stack. +// pushParseState pushes a new parse state newParseState onto the parse stack. // an error state is returned if maxNestingDepth was exceeded, otherwise successState is returned. func (s *scanner) pushParseState(c byte, newParseState int, successState int) int { s.parseState = append(s.parseState, newParseState) diff --git a/src/encoding/json/scanner_test.go b/src/encoding/json/scanner_test.go index 068439dcaca858..a062e91243e477 100644 --- a/src/encoding/json/scanner_test.go +++ b/src/encoding/json/scanner_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json import ( @@ -72,6 +74,7 @@ func TestCompactAndIndent(t *testing.T) { -5e+2 ]`}, {Name(""), "{\"\":\"<>&\u2028\u2029\"}", "{\n\t\"\": \"<>&\u2028\u2029\"\n}"}, // See golang.org/issue/34070 + {Name(""), `null`, "null \n\r\t"}, // See golang.org/issue/13520 and golang.org/issue/74806 } var buf bytes.Buffer for _, tt := range tests { @@ -100,7 +103,7 @@ func TestCompactAndIndent(t *testing.T) { buf.Reset() if err := Indent(&buf, []byte(tt.compact), "", "\t"); err != nil { t.Errorf("%s: Indent error: %v", tt.Where, err) - } else if got := buf.String(); got != tt.indent { + } else if got := buf.String(); got != strings.TrimRight(tt.indent, " \n\r\t") { t.Errorf("%s: Compact:\n\tgot: %s\n\twant: %s", tt.Where, indentNewlines(got), indentNewlines(tt.indent)) } }) diff --git a/src/encoding/json/stream.go b/src/encoding/json/stream.go index e2d9470bcc7fca..fc480c994651d2 100644 --- a/src/encoding/json/stream.go +++ b/src/encoding/json/stream.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json import ( diff --git a/src/encoding/json/stream_test.go b/src/encoding/json/stream_test.go index 32ede8cc7e6271..0e937cfaa13c78 100644 --- a/src/encoding/json/stream_test.go +++ b/src/encoding/json/stream_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json import ( @@ -79,9 +81,9 @@ func TestEncoder(t *testing.T) { t.Fatalf("#%d.%d Encode error: %v", i, j, err) } } - if have, want := buf.String(), nlines(streamEncoded, i); have != want { + if got, want := buf.String(), nlines(streamEncoded, i); got != want { t.Errorf("encoding %d items: mismatch:", i) - diff(t, []byte(have), []byte(want)) + diff(t, []byte(got), []byte(want)) break } } @@ -148,9 +150,9 @@ func TestEncoderIndent(t *testing.T) { for _, v := range streamTest { enc.Encode(v) } - if have, want := buf.String(), streamEncodedIndent; have != want { - t.Error("Encode mismatch:") - diff(t, []byte(have), []byte(want)) + if got, want := buf.String(), streamEncodedIndent; got != want { + t.Errorf("Encode mismatch:\ngot:\n%s\n\nwant:\n%s", got, want) + diff(t, []byte(got), []byte(want)) } } @@ -520,3 +522,102 @@ func TestHTTPDecoding(t *testing.T) { t.Errorf("Decode error:\n\tgot: %v\n\twant: io.EOF", err) } } + +func TestTokenTruncation(t *testing.T) { + tests := []struct { + in string + err error + }{ + {in: ``, err: io.EOF}, + {in: `{`, err: io.EOF}, + {in: `{"`, err: io.ErrUnexpectedEOF}, + {in: `{"k"`, err: io.EOF}, + {in: `{"k":`, err: io.EOF}, + {in: `{"k",`, err: &SyntaxError{"invalid character ',' after object key", int64(len(`{"k"`))}}, + {in: `{"k"}`, err: &SyntaxError{"invalid character '}' after object key", int64(len(`{"k"`))}}, + {in: ` [0`, err: io.EOF}, + {in: `[0.`, err: io.ErrUnexpectedEOF}, + {in: `[0. `, err: &SyntaxError{"invalid character ' ' after decimal point in numeric literal", int64(len(`[0.`))}}, + {in: `[0,`, err: io.EOF}, + {in: `[0:`, err: &SyntaxError{"invalid character ':' after array element", int64(len(`[0`))}}, + {in: `n`, err: io.ErrUnexpectedEOF}, + {in: `nul`, err: io.ErrUnexpectedEOF}, + {in: `fal `, err: &SyntaxError{"invalid character ' ' in literal false (expecting 's')", int64(len(`fal `))}}, + {in: `false`, err: io.EOF}, + } + for _, tt := range tests { + d := NewDecoder(strings.NewReader(tt.in)) + for i := 0; true; i++ { + if _, err := d.Token(); err != nil { + if !reflect.DeepEqual(err, tt.err) { + t.Errorf("`%s`: %d.Token error = %#v, want %v", tt.in, i, err, tt.err) + } + break + } + } + } +} + +func TestDecoderInputOffset(t *testing.T) { + const input = ` [ + [ ] , [ "one" ] , [ "one" , "two" ] , + { } , { "alpha" : "bravo" } , { "alpha" : "bravo" , "fizz" : "buzz" } + ] ` + wantOffsets := []int64{ + 0, 1, 2, 5, 6, 7, 8, 9, 12, 13, 18, 19, 20, 21, 24, 25, 30, 31, + 38, 39, 40, 41, 46, 47, 48, 49, 52, 53, 60, 61, 70, 71, 72, 73, + 76, 77, 84, 85, 94, 95, 103, 104, 112, 113, 114, 116, 117, 117, + 117, 117, + } + wantMores := []bool{ + true, true, false, true, true, false, true, true, true, false, + true, false, true, true, true, false, true, true, true, true, + true, false, false, false, false, + } + + d := NewDecoder(strings.NewReader(input)) + checkOffset := func() { + t.Helper() + got := d.InputOffset() + if len(wantOffsets) == 0 { + t.Fatalf("InputOffset = %d, want nil", got) + } + want := wantOffsets[0] + if got != want { + t.Fatalf("InputOffset = %d, want %d", got, want) + } + wantOffsets = wantOffsets[1:] + } + checkMore := func() { + t.Helper() + got := d.More() + if len(wantMores) == 0 { + t.Fatalf("More = %v, want nil", got) + } + want := wantMores[0] + if got != want { + t.Fatalf("More = %v, want %v", got, want) + } + wantMores = wantMores[1:] + } + checkOffset() + checkMore() + checkOffset() + for { + if _, err := d.Token(); err == io.EOF { + break + } else if err != nil { + t.Fatalf("Token error: %v", err) + } + checkOffset() + checkMore() + checkOffset() + } + checkOffset() + checkMore() + checkOffset() + + if len(wantOffsets)+len(wantMores) > 0 { + t.Fatal("unconsumed testdata") + } +} diff --git a/src/encoding/json/tables.go b/src/encoding/json/tables.go index 10acdc18c6b0b4..e8841cfc681f66 100644 --- a/src/encoding/json/tables.go +++ b/src/encoding/json/tables.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json import "unicode/utf8" diff --git a/src/encoding/json/tagkey_test.go b/src/encoding/json/tagkey_test.go index d432cd7d8bdfc8..8e4d360e94dcfd 100644 --- a/src/encoding/json/tagkey_test.go +++ b/src/encoding/json/tagkey_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json import ( diff --git a/src/encoding/json/tags.go b/src/encoding/json/tags.go index b490328f4c46e7..5ebd700fa69fed 100644 --- a/src/encoding/json/tags.go +++ b/src/encoding/json/tags.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json import ( diff --git a/src/encoding/json/tags_test.go b/src/encoding/json/tags_test.go index 1d2323dcee6014..6bb621c1284209 100644 --- a/src/encoding/json/tags_test.go +++ b/src/encoding/json/tags_test.go @@ -2,11 +2,11 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !goexperiment.jsonv2 + package json -import ( - "testing" -) +import "testing" func TestTagParsing(t *testing.T) { name, opts := parseTag("field,foobar,foo") diff --git a/src/encoding/json/testdata/code.json.gz b/src/encoding/json/testdata/code.json.gz deleted file mode 100644 index 1572a92bfbdfe9..00000000000000 Binary files a/src/encoding/json/testdata/code.json.gz and /dev/null differ diff --git a/src/encoding/json/v2/arshal.go b/src/encoding/json/v2/arshal.go new file mode 100644 index 00000000000000..5537a467d836e3 --- /dev/null +++ b/src/encoding/json/v2/arshal.go @@ -0,0 +1,576 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "bytes" + "encoding" + "io" + "reflect" + "sync" + "time" + + "encoding/json/internal" + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonopts" + "encoding/json/jsontext" +) + +// Reference encoding and time packages to assist pkgsite +// in being able to hotlink references to those packages. +var ( + _ encoding.TextMarshaler + _ encoding.TextAppender + _ encoding.TextUnmarshaler + _ time.Time + _ time.Duration +) + +// export exposes internal functionality of the "jsontext" package. +var export = jsontext.Internal.Export(&internal.AllowInternalUse) + +// Marshal serializes a Go value as a []byte according to the provided +// marshal and encode options (while ignoring unmarshal or decode options). +// It does not terminate the output with a newline. +// +// Type-specific marshal functions and methods take precedence +// over the default representation of a value. +// Functions or methods that operate on *T are only called when encoding +// a value of type T (by taking its address) or a non-nil value of *T. +// Marshal ensures that a value is always addressable +// (by boxing it on the heap if necessary) so that +// these functions and methods can be consistently called. For performance, +// it is recommended that Marshal be passed a non-nil pointer to the value. +// +// The input value is encoded as JSON according the following rules: +// +// - If any type-specific functions in a [WithMarshalers] option match +// the value type, then those functions are called to encode the value. +// If all applicable functions return [SkipFunc], +// then the value is encoded according to subsequent rules. +// +// - If the value type implements [MarshalerTo], +// then the MarshalJSONTo method is called to encode the value. +// +// - If the value type implements [Marshaler], +// then the MarshalJSON method is called to encode the value. +// +// - If the value type implements [encoding.TextAppender], +// then the AppendText method is called to encode the value and +// subsequently encode its result as a JSON string. +// +// - If the value type implements [encoding.TextMarshaler], +// then the MarshalText method is called to encode the value and +// subsequently encode its result as a JSON string. +// +// - Otherwise, the value is encoded according to the value's type +// as described in detail below. +// +// Most Go types have a default JSON representation. +// Certain types support specialized formatting according to +// a format flag optionally specified in the Go struct tag +// for the struct field that contains the current value +// (see the “JSON Representation of Go structs” section for more details). +// +// The representation of each type is as follows: +// +// - A Go boolean is encoded as a JSON boolean (e.g., true or false). +// It does not support any custom format flags. +// +// - A Go string is encoded as a JSON string. +// It does not support any custom format flags. +// +// - A Go []byte or [N]byte is encoded as a JSON string containing +// the binary value encoded using RFC 4648. +// If the format is "base64" or unspecified, then this uses RFC 4648, section 4. +// If the format is "base64url", then this uses RFC 4648, section 5. +// If the format is "base32", then this uses RFC 4648, section 6. +// If the format is "base32hex", then this uses RFC 4648, section 7. +// If the format is "base16" or "hex", then this uses RFC 4648, section 8. +// If the format is "array", then the bytes value is encoded as a JSON array +// where each byte is recursively JSON-encoded as each JSON array element. +// +// - A Go integer is encoded as a JSON number without fractions or exponents. +// If [StringifyNumbers] is specified or encoding a JSON object name, +// then the JSON number is encoded within a JSON string. +// It does not support any custom format flags. +// +// - A Go float is encoded as a JSON number. +// If [StringifyNumbers] is specified or encoding a JSON object name, +// then the JSON number is encoded within a JSON string. +// If the format is "nonfinite", then NaN, +Inf, and -Inf are encoded as +// the JSON strings "NaN", "Infinity", and "-Infinity", respectively. +// Otherwise, the presence of non-finite numbers results in a [SemanticError]. +// +// - A Go map is encoded as a JSON object, where each Go map key and value +// is recursively encoded as a name and value pair in the JSON object. +// The Go map key must encode as a JSON string, otherwise this results +// in a [SemanticError]. The Go map is traversed in a non-deterministic order. +// For deterministic encoding, consider using the [Deterministic] option. +// If the format is "emitnull", then a nil map is encoded as a JSON null. +// If the format is "emitempty", then a nil map is encoded as an empty JSON object, +// regardless of whether [FormatNilMapAsNull] is specified. +// Otherwise by default, a nil map is encoded as an empty JSON object. +// +// - A Go struct is encoded as a JSON object. +// See the “JSON Representation of Go structs” section +// in the package-level documentation for more details. +// +// - A Go slice is encoded as a JSON array, where each Go slice element +// is recursively JSON-encoded as the elements of the JSON array. +// If the format is "emitnull", then a nil slice is encoded as a JSON null. +// If the format is "emitempty", then a nil slice is encoded as an empty JSON array, +// regardless of whether [FormatNilSliceAsNull] is specified. +// Otherwise by default, a nil slice is encoded as an empty JSON array. +// +// - A Go array is encoded as a JSON array, where each Go array element +// is recursively JSON-encoded as the elements of the JSON array. +// The JSON array length is always identical to the Go array length. +// It does not support any custom format flags. +// +// - A Go pointer is encoded as a JSON null if nil, otherwise it is +// the recursively JSON-encoded representation of the underlying value. +// Format flags are forwarded to the encoding of the underlying value. +// +// - A Go interface is encoded as a JSON null if nil, otherwise it is +// the recursively JSON-encoded representation of the underlying value. +// It does not support any custom format flags. +// +// - A Go [time.Time] is encoded as a JSON string containing the timestamp +// formatted in RFC 3339 with nanosecond precision. +// If the format matches one of the format constants declared +// in the time package (e.g., RFC1123), then that format is used. +// If the format is "unix", "unixmilli", "unixmicro", or "unixnano", +// then the timestamp is encoded as a possibly fractional JSON number +// of the number of seconds (or milliseconds, microseconds, or nanoseconds) +// since the Unix epoch, which is January 1st, 1970 at 00:00:00 UTC. +// To avoid a fractional component, round the timestamp to the relevant unit. +// Otherwise, the format is used as-is with [time.Time.Format] if non-empty. +// +// - A Go [time.Duration] currently has no default representation and +// requires an explicit format to be specified. +// If the format is "sec", "milli", "micro", or "nano", +// then the duration is encoded as a possibly fractional JSON number +// of the number of seconds (or milliseconds, microseconds, or nanoseconds). +// To avoid a fractional component, round the duration to the relevant unit. +// If the format is "units", it is encoded as a JSON string formatted using +// [time.Duration.String] (e.g., "1h30m" for 1 hour 30 minutes). +// If the format is "iso8601", it is encoded as a JSON string using the +// ISO 8601 standard for durations (e.g., "PT1H30M" for 1 hour 30 minutes) +// using only accurate units of hours, minutes, and seconds. +// +// - All other Go types (e.g., complex numbers, channels, and functions) +// have no default representation and result in a [SemanticError]. +// +// JSON cannot represent cyclic data structures and Marshal does not handle them. +// Passing cyclic structures will result in an error. +func Marshal(in any, opts ...Options) (out []byte, err error) { + enc := export.GetBufferedEncoder(opts...) + defer export.PutBufferedEncoder(enc) + xe := export.Encoder(enc) + xe.Flags.Set(jsonflags.OmitTopLevelNewline | 1) + err = marshalEncode(enc, in, &xe.Struct) + if err != nil && xe.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return nil, internal.TransformMarshalError(in, err) + } + return bytes.Clone(xe.Buf), err +} + +// MarshalWrite serializes a Go value into an [io.Writer] according to the provided +// marshal and encode options (while ignoring unmarshal or decode options). +// It does not terminate the output with a newline. +// See [Marshal] for details about the conversion of a Go value into JSON. +func MarshalWrite(out io.Writer, in any, opts ...Options) (err error) { + enc := export.GetStreamingEncoder(out, opts...) + defer export.PutStreamingEncoder(enc) + xe := export.Encoder(enc) + xe.Flags.Set(jsonflags.OmitTopLevelNewline | 1) + err = marshalEncode(enc, in, &xe.Struct) + if err != nil && xe.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return internal.TransformMarshalError(in, err) + } + return err +} + +// MarshalEncode serializes a Go value into an [jsontext.Encoder] according to +// the provided marshal options (while ignoring unmarshal, encode, or decode options). +// Any marshal-relevant options already specified on the [jsontext.Encoder] +// take lower precedence than the set of options provided by the caller. +// Unlike [Marshal] and [MarshalWrite], encode options are ignored because +// they must have already been specified on the provided [jsontext.Encoder]. +// +// See [Marshal] for details about the conversion of a Go value into JSON. +func MarshalEncode(out *jsontext.Encoder, in any, opts ...Options) (err error) { + xe := export.Encoder(out) + if len(opts) > 0 { + optsOriginal := xe.Struct + defer func() { xe.Struct = optsOriginal }() + xe.Struct.JoinWithoutCoderOptions(opts...) + } + err = marshalEncode(out, in, &xe.Struct) + if err != nil && xe.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return internal.TransformMarshalError(in, err) + } + return err +} + +func marshalEncode(out *jsontext.Encoder, in any, mo *jsonopts.Struct) (err error) { + v := reflect.ValueOf(in) + if !v.IsValid() || (v.Kind() == reflect.Pointer && v.IsNil()) { + return out.WriteToken(jsontext.Null) + } + // Shallow copy non-pointer values to obtain an addressable value. + // It is beneficial to performance to always pass pointers to avoid this. + forceAddr := v.Kind() != reflect.Pointer + if forceAddr { + v2 := reflect.New(v.Type()) + v2.Elem().Set(v) + v = v2 + } + va := addressableValue{v.Elem(), forceAddr} // dereferenced pointer is always addressable + t := va.Type() + + // Lookup and call the marshal function for this type. + marshal := lookupArshaler(t).marshal + if mo.Marshalers != nil { + marshal, _ = mo.Marshalers.(*Marshalers).lookup(marshal, t) + } + if err := marshal(out, va, mo); err != nil { + if !mo.Flags.Get(jsonflags.AllowDuplicateNames) { + export.Encoder(out).Tokens.InvalidateDisabledNamespaces() + } + return err + } + return nil +} + +// Unmarshal decodes a []byte input into a Go value according to the provided +// unmarshal and decode options (while ignoring marshal or encode options). +// The input must be a single JSON value with optional whitespace interspersed. +// The output must be a non-nil pointer. +// +// Type-specific unmarshal functions and methods take precedence +// over the default representation of a value. +// Functions or methods that operate on *T are only called when decoding +// a value of type T (by taking its address) or a non-nil value of *T. +// Unmarshal ensures that a value is always addressable +// (by boxing it on the heap if necessary) so that +// these functions and methods can be consistently called. +// +// The input is decoded into the output according the following rules: +// +// - If any type-specific functions in a [WithUnmarshalers] option match +// the value type, then those functions are called to decode the JSON +// value. If all applicable functions return [SkipFunc], +// then the input is decoded according to subsequent rules. +// +// - If the value type implements [UnmarshalerFrom], +// then the UnmarshalJSONFrom method is called to decode the JSON value. +// +// - If the value type implements [Unmarshaler], +// then the UnmarshalJSON method is called to decode the JSON value. +// +// - If the value type implements [encoding.TextUnmarshaler], +// then the input is decoded as a JSON string and +// the UnmarshalText method is called with the decoded string value. +// This fails with a [SemanticError] if the input is not a JSON string. +// +// - Otherwise, the JSON value is decoded according to the value's type +// as described in detail below. +// +// Most Go types have a default JSON representation. +// Certain types support specialized formatting according to +// a format flag optionally specified in the Go struct tag +// for the struct field that contains the current value +// (see the “JSON Representation of Go structs” section for more details). +// A JSON null may be decoded into every supported Go value where +// it is equivalent to storing the zero value of the Go value. +// If the input JSON kind is not handled by the current Go value type, +// then this fails with a [SemanticError]. Unless otherwise specified, +// the decoded value replaces any pre-existing value. +// +// The representation of each type is as follows: +// +// - A Go boolean is decoded from a JSON boolean (e.g., true or false). +// It does not support any custom format flags. +// +// - A Go string is decoded from a JSON string. +// It does not support any custom format flags. +// +// - A Go []byte or [N]byte is decoded from a JSON string +// containing the binary value encoded using RFC 4648. +// If the format is "base64" or unspecified, then this uses RFC 4648, section 4. +// If the format is "base64url", then this uses RFC 4648, section 5. +// If the format is "base32", then this uses RFC 4648, section 6. +// If the format is "base32hex", then this uses RFC 4648, section 7. +// If the format is "base16" or "hex", then this uses RFC 4648, section 8. +// If the format is "array", then the Go slice or array is decoded from a +// JSON array where each JSON element is recursively decoded for each byte. +// When decoding into a non-nil []byte, the slice length is reset to zero +// and the decoded input is appended to it. +// When decoding into a [N]byte, the input must decode to exactly N bytes, +// otherwise it fails with a [SemanticError]. +// +// - A Go integer is decoded from a JSON number. +// It must be decoded from a JSON string containing a JSON number +// if [StringifyNumbers] is specified or decoding a JSON object name. +// It fails with a [SemanticError] if the JSON number +// has a fractional or exponent component. +// It also fails if it overflows the representation of the Go integer type. +// It does not support any custom format flags. +// +// - A Go float is decoded from a JSON number. +// It must be decoded from a JSON string containing a JSON number +// if [StringifyNumbers] is specified or decoding a JSON object name. +// It fails if it overflows the representation of the Go float type. +// If the format is "nonfinite", then the JSON strings +// "NaN", "Infinity", and "-Infinity" are decoded as NaN, +Inf, and -Inf. +// Otherwise, the presence of such strings results in a [SemanticError]. +// +// - A Go map is decoded from a JSON object, +// where each JSON object name and value pair is recursively decoded +// as the Go map key and value. Maps are not cleared. +// If the Go map is nil, then a new map is allocated to decode into. +// If the decoded key matches an existing Go map entry, the entry value +// is reused by decoding the JSON object value into it. +// The formats "emitnull" and "emitempty" have no effect when decoding. +// +// - A Go struct is decoded from a JSON object. +// See the “JSON Representation of Go structs” section +// in the package-level documentation for more details. +// +// - A Go slice is decoded from a JSON array, where each JSON element +// is recursively decoded and appended to the Go slice. +// Before appending into a Go slice, a new slice is allocated if it is nil, +// otherwise the slice length is reset to zero. +// The formats "emitnull" and "emitempty" have no effect when decoding. +// +// - A Go array is decoded from a JSON array, where each JSON array element +// is recursively decoded as each corresponding Go array element. +// Each Go array element is zeroed before decoding into it. +// It fails with a [SemanticError] if the JSON array does not contain +// the exact same number of elements as the Go array. +// It does not support any custom format flags. +// +// - A Go pointer is decoded based on the JSON kind and underlying Go type. +// If the input is a JSON null, then this stores a nil pointer. +// Otherwise, it allocates a new underlying value if the pointer is nil, +// and recursively JSON decodes into the underlying value. +// Format flags are forwarded to the decoding of the underlying type. +// +// - A Go interface is decoded based on the JSON kind and underlying Go type. +// If the input is a JSON null, then this stores a nil interface value. +// Otherwise, a nil interface value of an empty interface type is initialized +// with a zero Go bool, string, float64, map[string]any, or []any if the +// input is a JSON boolean, string, number, object, or array, respectively. +// If the interface value is still nil, then this fails with a [SemanticError] +// since decoding could not determine an appropriate Go type to decode into. +// For example, unmarshaling into a nil io.Reader fails since +// there is no concrete type to populate the interface value with. +// Otherwise an underlying value exists and it recursively decodes +// the JSON input into it. It does not support any custom format flags. +// +// - A Go [time.Time] is decoded from a JSON string containing the time +// formatted in RFC 3339 with nanosecond precision. +// If the format matches one of the format constants declared in +// the time package (e.g., RFC1123), then that format is used for parsing. +// If the format is "unix", "unixmilli", "unixmicro", or "unixnano", +// then the timestamp is decoded from an optionally fractional JSON number +// of the number of seconds (or milliseconds, microseconds, or nanoseconds) +// since the Unix epoch, which is January 1st, 1970 at 00:00:00 UTC. +// Otherwise, the format is used as-is with [time.Time.Parse] if non-empty. +// +// - A Go [time.Duration] currently has no default representation and +// requires an explicit format to be specified. +// If the format is "sec", "milli", "micro", or "nano", +// then the duration is decoded from an optionally fractional JSON number +// of the number of seconds (or milliseconds, microseconds, or nanoseconds). +// If the format is "units", it is decoded from a JSON string parsed using +// [time.ParseDuration] (e.g., "1h30m" for 1 hour 30 minutes). +// If the format is "iso8601", it is decoded from a JSON string using the +// ISO 8601 standard for durations (e.g., "PT1H30M" for 1 hour 30 minutes) +// accepting only accurate units of hours, minutes, or seconds. +// +// - All other Go types (e.g., complex numbers, channels, and functions) +// have no default representation and result in a [SemanticError]. +// +// In general, unmarshaling follows merge semantics (similar to RFC 7396) +// where the decoded Go value replaces the destination value +// for any JSON kind other than an object. +// For JSON objects, the input object is merged into the destination value +// where matching object members recursively apply merge semantics. +func Unmarshal(in []byte, out any, opts ...Options) (err error) { + dec := export.GetBufferedDecoder(in, opts...) + defer export.PutBufferedDecoder(dec) + xd := export.Decoder(dec) + err = unmarshalDecode(dec, out, &xd.Struct, true) + if err != nil && xd.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return internal.TransformUnmarshalError(out, err) + } + return err +} + +// UnmarshalRead deserializes a Go value from an [io.Reader] according to the +// provided unmarshal and decode options (while ignoring marshal or encode options). +// The input must be a single JSON value with optional whitespace interspersed. +// It consumes the entirety of [io.Reader] until [io.EOF] is encountered, +// without reporting an error for EOF. The output must be a non-nil pointer. +// See [Unmarshal] for details about the conversion of JSON into a Go value. +func UnmarshalRead(in io.Reader, out any, opts ...Options) (err error) { + dec := export.GetStreamingDecoder(in, opts...) + defer export.PutStreamingDecoder(dec) + xd := export.Decoder(dec) + err = unmarshalDecode(dec, out, &xd.Struct, true) + if err != nil && xd.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return internal.TransformUnmarshalError(out, err) + } + return err +} + +// UnmarshalDecode deserializes a Go value from a [jsontext.Decoder] according to +// the provided unmarshal options (while ignoring marshal, encode, or decode options). +// Any unmarshal options already specified on the [jsontext.Decoder] +// take lower precedence than the set of options provided by the caller. +// Unlike [Unmarshal] and [UnmarshalRead], decode options are ignored because +// they must have already been specified on the provided [jsontext.Decoder]. +// +// The input may be a stream of zero or more JSON values, +// where this only unmarshals the next JSON value in the stream. +// If there are no more top-level JSON values, it reports [io.EOF]. +// The output must be a non-nil pointer. +// See [Unmarshal] for details about the conversion of JSON into a Go value. +func UnmarshalDecode(in *jsontext.Decoder, out any, opts ...Options) (err error) { + xd := export.Decoder(in) + if len(opts) > 0 { + optsOriginal := xd.Struct + defer func() { xd.Struct = optsOriginal }() + xd.Struct.JoinWithoutCoderOptions(opts...) + } + err = unmarshalDecode(in, out, &xd.Struct, false) + if err != nil && xd.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return internal.TransformUnmarshalError(out, err) + } + return err +} + +func unmarshalDecode(in *jsontext.Decoder, out any, uo *jsonopts.Struct, last bool) (err error) { + v := reflect.ValueOf(out) + if v.Kind() != reflect.Pointer || v.IsNil() { + return &SemanticError{action: "unmarshal", GoType: reflect.TypeOf(out), Err: internal.ErrNonNilReference} + } + va := addressableValue{v.Elem(), false} // dereferenced pointer is always addressable + t := va.Type() + + // In legacy semantics, the entirety of the next JSON value + // was validated before attempting to unmarshal it. + if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + if err := export.Decoder(in).CheckNextValue(last); err != nil { + if err == io.EOF && last { + offset := in.InputOffset() + int64(len(in.UnreadBuffer())) + return &jsontext.SyntacticError{ByteOffset: offset, Err: io.ErrUnexpectedEOF} + } + return err + } + } + + // Lookup and call the unmarshal function for this type. + unmarshal := lookupArshaler(t).unmarshal + if uo.Unmarshalers != nil { + unmarshal, _ = uo.Unmarshalers.(*Unmarshalers).lookup(unmarshal, t) + } + if err := unmarshal(in, va, uo); err != nil { + if !uo.Flags.Get(jsonflags.AllowDuplicateNames) { + export.Decoder(in).Tokens.InvalidateDisabledNamespaces() + } + if err == io.EOF && last { + offset := in.InputOffset() + int64(len(in.UnreadBuffer())) + return &jsontext.SyntacticError{ByteOffset: offset, Err: io.ErrUnexpectedEOF} + } + return err + } + if last { + return export.Decoder(in).CheckEOF() + } + return nil +} + +// addressableValue is a reflect.Value that is guaranteed to be addressable +// such that calling the Addr and Set methods do not panic. +// +// There is no compile magic that enforces this property, +// but rather the need to construct this type makes it easier to examine each +// construction site to ensure that this property is upheld. +type addressableValue struct { + reflect.Value + + // forcedAddr reports whether this value is addressable + // only through the use of [newAddressableValue]. + // This is only used for [jsonflags.CallMethodsWithLegacySemantics]. + forcedAddr bool +} + +// newAddressableValue constructs a new addressable value of type t. +func newAddressableValue(t reflect.Type) addressableValue { + return addressableValue{reflect.New(t).Elem(), true} +} + +// TODO: Remove *jsonopts.Struct argument from [marshaler] and [unmarshaler]. +// This can be directly accessed on the encoder or decoder. + +// All marshal and unmarshal behavior is implemented using these signatures. +// The *jsonopts.Struct argument is guaranteed to identical to or at least +// a strict super-set of the options in Encoder.Struct or Decoder.Struct. +// It is identical for Marshal, Unmarshal, MarshalWrite, and UnmarshalRead. +// It is a super-set for MarshalEncode and UnmarshalDecode. +type ( + marshaler = func(*jsontext.Encoder, addressableValue, *jsonopts.Struct) error + unmarshaler = func(*jsontext.Decoder, addressableValue, *jsonopts.Struct) error +) + +type arshaler struct { + marshal marshaler + unmarshal unmarshaler + nonDefault bool +} + +var lookupArshalerCache sync.Map // map[reflect.Type]*arshaler + +func lookupArshaler(t reflect.Type) *arshaler { + if v, ok := lookupArshalerCache.Load(t); ok { + return v.(*arshaler) + } + + fncs := makeDefaultArshaler(t) + fncs = makeMethodArshaler(fncs, t) + fncs = makeTimeArshaler(fncs, t) + + // Use the last stored so that duplicate arshalers can be garbage collected. + v, _ := lookupArshalerCache.LoadOrStore(t, fncs) + return v.(*arshaler) +} + +var stringsPools = &sync.Pool{New: func() any { return new(stringSlice) }} + +type stringSlice []string + +// getStrings returns a non-nil pointer to a slice with length n. +func getStrings(n int) *stringSlice { + s := stringsPools.Get().(*stringSlice) + if cap(*s) < n { + *s = make([]string, n) + } + *s = (*s)[:n] + return s +} + +func putStrings(s *stringSlice) { + if cap(*s) > 1<<10 { + *s = nil // avoid pinning arbitrarily large amounts of memory + } + clear(*s) // avoid pinning a reference to each string + stringsPools.Put(s) +} diff --git a/src/encoding/json/v2/arshal_any.go b/src/encoding/json/v2/arshal_any.go new file mode 100644 index 00000000000000..8c0c445404d883 --- /dev/null +++ b/src/encoding/json/v2/arshal_any.go @@ -0,0 +1,288 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "cmp" + "math" + "reflect" + "slices" + "strconv" + + "encoding/json/internal" + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonopts" + "encoding/json/internal/jsonwire" + "encoding/json/jsontext" +) + +// This file contains an optimized marshal and unmarshal implementation +// for the any type. This type is often used when the Go program has +// no knowledge of the JSON schema. This is a common enough occurrence +// to justify the complexity of adding logic for this. + +// marshalValueAny marshals a Go any as a JSON value. +// This assumes that there are no special formatting directives +// for any possible nested value. +func marshalValueAny(enc *jsontext.Encoder, val any, mo *jsonopts.Struct) error { + switch val := val.(type) { + case nil: + return enc.WriteToken(jsontext.Null) + case bool: + return enc.WriteToken(jsontext.Bool(val)) + case string: + return enc.WriteToken(jsontext.String(val)) + case float64: + if math.IsNaN(val) || math.IsInf(val, 0) { + break // use default logic below + } + return enc.WriteToken(jsontext.Float(val)) + case map[string]any: + return marshalObjectAny(enc, val, mo) + case []any: + return marshalArrayAny(enc, val, mo) + } + + v := newAddressableValue(reflect.TypeOf(val)) + v.Set(reflect.ValueOf(val)) + marshal := lookupArshaler(v.Type()).marshal + if mo.Marshalers != nil { + marshal, _ = mo.Marshalers.(*Marshalers).lookup(marshal, v.Type()) + } + return marshal(enc, v, mo) +} + +// unmarshalValueAny unmarshals a JSON value as a Go any. +// This assumes that there are no special formatting directives +// for any possible nested value. +// Duplicate names must be rejected since this does not implement merging. +func unmarshalValueAny(dec *jsontext.Decoder, uo *jsonopts.Struct) (any, error) { + switch k := dec.PeekKind(); k { + case '{': + return unmarshalObjectAny(dec, uo) + case '[': + return unmarshalArrayAny(dec, uo) + default: + xd := export.Decoder(dec) + var flags jsonwire.ValueFlags + val, err := xd.ReadValue(&flags) + if err != nil { + return nil, err + } + switch val.Kind() { + case 'n': + return nil, nil + case 'f': + return false, nil + case 't': + return true, nil + case '"': + val = jsonwire.UnquoteMayCopy(val, flags.IsVerbatim()) + if xd.StringCache == nil { + xd.StringCache = new(stringCache) + } + return makeString(xd.StringCache, val), nil + case '0': + if uo.Flags.Get(jsonflags.UnmarshalAnyWithRawNumber) { + return internal.RawNumberOf(val), nil + } + fv, ok := jsonwire.ParseFloat(val, 64) + if !ok { + return fv, newUnmarshalErrorAfterWithValue(dec, float64Type, strconv.ErrRange) + } + return fv, nil + default: + panic("BUG: invalid kind: " + k.String()) + } + } +} + +// marshalObjectAny marshals a Go map[string]any as a JSON object +// (or as a JSON null if nil and [jsonflags.FormatNilMapAsNull]). +func marshalObjectAny(enc *jsontext.Encoder, obj map[string]any, mo *jsonopts.Struct) error { + // Check for cycles. + xe := export.Encoder(enc) + if xe.Tokens.Depth() > startDetectingCyclesAfter { + v := reflect.ValueOf(obj) + if err := visitPointer(&xe.SeenPointers, v); err != nil { + return newMarshalErrorBefore(enc, mapStringAnyType, err) + } + defer leavePointer(&xe.SeenPointers, v) + } + + // Handle empty maps. + if len(obj) == 0 { + if mo.Flags.Get(jsonflags.FormatNilMapAsNull) && obj == nil { + return enc.WriteToken(jsontext.Null) + } + // Optimize for marshaling an empty map without any preceding whitespace. + if !mo.Flags.Get(jsonflags.AnyWhitespace) && !xe.Tokens.Last.NeedObjectName() { + xe.Buf = append(xe.Tokens.MayAppendDelim(xe.Buf, '{'), "{}"...) + xe.Tokens.Last.Increment() + if xe.NeedFlush() { + return xe.Flush() + } + return nil + } + } + + if err := enc.WriteToken(jsontext.BeginObject); err != nil { + return err + } + // A Go map guarantees that each entry has a unique key + // The only possibility of duplicates is due to invalid UTF-8. + if !mo.Flags.Get(jsonflags.AllowInvalidUTF8) { + xe.Tokens.Last.DisableNamespace() + } + if !mo.Flags.Get(jsonflags.Deterministic) || len(obj) <= 1 { + for name, val := range obj { + if err := enc.WriteToken(jsontext.String(name)); err != nil { + return err + } + if err := marshalValueAny(enc, val, mo); err != nil { + return err + } + } + } else { + names := getStrings(len(obj)) + var i int + for name := range obj { + (*names)[i] = name + i++ + } + slices.Sort(*names) + for _, name := range *names { + if err := enc.WriteToken(jsontext.String(name)); err != nil { + return err + } + if err := marshalValueAny(enc, obj[name], mo); err != nil { + return err + } + } + putStrings(names) + } + if err := enc.WriteToken(jsontext.EndObject); err != nil { + return err + } + return nil +} + +// unmarshalObjectAny unmarshals a JSON object as a Go map[string]any. +// It panics if not decoding a JSON object. +func unmarshalObjectAny(dec *jsontext.Decoder, uo *jsonopts.Struct) (map[string]any, error) { + switch tok, err := dec.ReadToken(); { + case err != nil: + return nil, err + case tok.Kind() != '{': + panic("BUG: invalid kind: " + tok.Kind().String()) + } + obj := make(map[string]any) + // A Go map guarantees that each entry has a unique key + // The only possibility of duplicates is due to invalid UTF-8. + if !uo.Flags.Get(jsonflags.AllowInvalidUTF8) { + export.Decoder(dec).Tokens.Last.DisableNamespace() + } + var errUnmarshal error + for dec.PeekKind() != '}' { + tok, err := dec.ReadToken() + if err != nil { + return obj, err + } + name := tok.String() + + // Manually check for duplicate names. + if _, ok := obj[name]; ok { + // TODO: Unread the object name. + name := export.Decoder(dec).PreviousTokenOrValue() + err := newDuplicateNameError(dec.StackPointer(), nil, dec.InputOffset()-len64(name)) + return obj, err + } + + val, err := unmarshalValueAny(dec, uo) + obj[name] = val + if err != nil { + if isFatalError(err, uo.Flags) { + return obj, err + } + errUnmarshal = cmp.Or(err, errUnmarshal) + } + } + if _, err := dec.ReadToken(); err != nil { + return obj, err + } + return obj, errUnmarshal +} + +// marshalArrayAny marshals a Go []any as a JSON array +// (or as a JSON null if nil and [jsonflags.FormatNilSliceAsNull]). +func marshalArrayAny(enc *jsontext.Encoder, arr []any, mo *jsonopts.Struct) error { + // Check for cycles. + xe := export.Encoder(enc) + if xe.Tokens.Depth() > startDetectingCyclesAfter { + v := reflect.ValueOf(arr) + if err := visitPointer(&xe.SeenPointers, v); err != nil { + return newMarshalErrorBefore(enc, sliceAnyType, err) + } + defer leavePointer(&xe.SeenPointers, v) + } + + // Handle empty slices. + if len(arr) == 0 { + if mo.Flags.Get(jsonflags.FormatNilSliceAsNull) && arr == nil { + return enc.WriteToken(jsontext.Null) + } + // Optimize for marshaling an empty slice without any preceding whitespace. + if !mo.Flags.Get(jsonflags.AnyWhitespace) && !xe.Tokens.Last.NeedObjectName() { + xe.Buf = append(xe.Tokens.MayAppendDelim(xe.Buf, '['), "[]"...) + xe.Tokens.Last.Increment() + if xe.NeedFlush() { + return xe.Flush() + } + return nil + } + } + + if err := enc.WriteToken(jsontext.BeginArray); err != nil { + return err + } + for _, val := range arr { + if err := marshalValueAny(enc, val, mo); err != nil { + return err + } + } + if err := enc.WriteToken(jsontext.EndArray); err != nil { + return err + } + return nil +} + +// unmarshalArrayAny unmarshals a JSON array as a Go []any. +// It panics if not decoding a JSON array. +func unmarshalArrayAny(dec *jsontext.Decoder, uo *jsonopts.Struct) ([]any, error) { + switch tok, err := dec.ReadToken(); { + case err != nil: + return nil, err + case tok.Kind() != '[': + panic("BUG: invalid kind: " + tok.Kind().String()) + } + arr := []any{} + var errUnmarshal error + for dec.PeekKind() != ']' { + val, err := unmarshalValueAny(dec, uo) + arr = append(arr, val) + if err != nil { + if isFatalError(err, uo.Flags) { + return arr, err + } + errUnmarshal = cmp.Or(errUnmarshal, err) + } + } + if _, err := dec.ReadToken(); err != nil { + return arr, err + } + return arr, errUnmarshal +} diff --git a/src/encoding/json/v2/arshal_default.go b/src/encoding/json/v2/arshal_default.go new file mode 100644 index 00000000000000..33931af17e6423 --- /dev/null +++ b/src/encoding/json/v2/arshal_default.go @@ -0,0 +1,1943 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "bytes" + "cmp" + "encoding" + "encoding/base32" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "math" + "reflect" + "slices" + "strconv" + "strings" + "sync" + + "encoding/json/internal" + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonopts" + "encoding/json/internal/jsonwire" + "encoding/json/jsontext" +) + +// optimizeCommon specifies whether to use optimizations targeted for certain +// common patterns, rather than using the slower, but more general logic. +// All tests should pass regardless of whether this is true or not. +const optimizeCommon = true + +var ( + // Most natural Go type that correspond with each JSON type. + anyType = reflect.TypeFor[any]() // JSON value + boolType = reflect.TypeFor[bool]() // JSON bool + stringType = reflect.TypeFor[string]() // JSON string + float64Type = reflect.TypeFor[float64]() // JSON number + mapStringAnyType = reflect.TypeFor[map[string]any]() // JSON object + sliceAnyType = reflect.TypeFor[[]any]() // JSON array + + bytesType = reflect.TypeFor[[]byte]() + emptyStructType = reflect.TypeFor[struct{}]() +) + +const startDetectingCyclesAfter = 1000 + +type seenPointers = map[any]struct{} + +type typedPointer struct { + typ reflect.Type + ptr any // always stores unsafe.Pointer, but avoids depending on unsafe + len int // remember slice length to avoid false positives +} + +// visitPointer visits pointer p of type t, reporting an error if seen before. +// If successfully visited, then the caller must eventually call leave. +func visitPointer(m *seenPointers, v reflect.Value) error { + p := typedPointer{v.Type(), v.UnsafePointer(), sliceLen(v)} + if _, ok := (*m)[p]; ok { + return internal.ErrCycle + } + if *m == nil { + *m = make(seenPointers) + } + (*m)[p] = struct{}{} + return nil +} +func leavePointer(m *seenPointers, v reflect.Value) { + p := typedPointer{v.Type(), v.UnsafePointer(), sliceLen(v)} + delete(*m, p) +} + +func sliceLen(v reflect.Value) int { + if v.Kind() == reflect.Slice { + return v.Len() + } + return 0 +} + +func len64[Bytes ~[]byte | ~string](in Bytes) int64 { + return int64(len(in)) +} + +func makeDefaultArshaler(t reflect.Type) *arshaler { + switch t.Kind() { + case reflect.Bool: + return makeBoolArshaler(t) + case reflect.String: + return makeStringArshaler(t) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return makeIntArshaler(t) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return makeUintArshaler(t) + case reflect.Float32, reflect.Float64: + return makeFloatArshaler(t) + case reflect.Map: + return makeMapArshaler(t) + case reflect.Struct: + return makeStructArshaler(t) + case reflect.Slice: + fncs := makeSliceArshaler(t) + if t.Elem().Kind() == reflect.Uint8 { + return makeBytesArshaler(t, fncs) + } + return fncs + case reflect.Array: + fncs := makeArrayArshaler(t) + if t.Elem().Kind() == reflect.Uint8 { + return makeBytesArshaler(t, fncs) + } + return fncs + case reflect.Pointer: + return makePointerArshaler(t) + case reflect.Interface: + return makeInterfaceArshaler(t) + default: + return makeInvalidArshaler(t) + } +} + +func makeBoolArshaler(t reflect.Type) *arshaler { + var fncs arshaler + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + xe := export.Encoder(enc) + if mo.Format != "" && mo.FormatDepth == xe.Tokens.Depth() { + return newInvalidFormatError(enc, t) + } + + // Optimize for marshaling without preceding whitespace. + if optimizeCommon && !mo.Flags.Get(jsonflags.AnyWhitespace|jsonflags.StringifyBoolsAndStrings) && !xe.Tokens.Last.NeedObjectName() { + xe.Buf = strconv.AppendBool(xe.Tokens.MayAppendDelim(xe.Buf, 't'), va.Bool()) + xe.Tokens.Last.Increment() + if xe.NeedFlush() { + return xe.Flush() + } + return nil + } + + if mo.Flags.Get(jsonflags.StringifyBoolsAndStrings) { + if va.Bool() { + return enc.WriteToken(jsontext.String("true")) + } else { + return enc.WriteToken(jsontext.String("false")) + } + } + return enc.WriteToken(jsontext.Bool(va.Bool())) + } + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + xd := export.Decoder(dec) + if uo.Format != "" && uo.FormatDepth == xd.Tokens.Depth() { + return newInvalidFormatError(dec, t) + } + tok, err := dec.ReadToken() + if err != nil { + return err + } + k := tok.Kind() + switch k { + case 'n': + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + va.SetBool(false) + } + return nil + case 't', 'f': + if !uo.Flags.Get(jsonflags.StringifyBoolsAndStrings) { + va.SetBool(tok.Bool()) + return nil + } + case '"': + if uo.Flags.Get(jsonflags.StringifyBoolsAndStrings) { + switch tok.String() { + case "true": + va.SetBool(true) + case "false": + va.SetBool(false) + default: + if uo.Flags.Get(jsonflags.StringifyWithLegacySemantics) && tok.String() == "null" { + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + va.SetBool(false) + } + return nil + } + return newUnmarshalErrorAfterWithValue(dec, t, strconv.ErrSyntax) + } + return nil + } + } + return newUnmarshalErrorAfterWithSkipping(dec, t, nil) + } + return &fncs +} + +func makeStringArshaler(t reflect.Type) *arshaler { + var fncs arshaler + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + xe := export.Encoder(enc) + if mo.Format != "" && mo.FormatDepth == xe.Tokens.Depth() { + return newInvalidFormatError(enc, t) + } + + // Optimize for marshaling without preceding whitespace. + s := va.String() + if optimizeCommon && !mo.Flags.Get(jsonflags.AnyWhitespace|jsonflags.StringifyBoolsAndStrings) && !xe.Tokens.Last.NeedObjectName() { + b := xe.Buf + b = xe.Tokens.MayAppendDelim(b, '"') + b, err := jsonwire.AppendQuote(b, s, &mo.Flags) + if err == nil { + xe.Buf = b + xe.Tokens.Last.Increment() + if xe.NeedFlush() { + return xe.Flush() + } + return nil + } + // Otherwise, the string contains invalid UTF-8, + // so let the logic below construct the proper error. + } + + if mo.Flags.Get(jsonflags.StringifyBoolsAndStrings) { + b, err := jsonwire.AppendQuote(nil, s, &mo.Flags) + if err != nil { + return newMarshalErrorBefore(enc, t, &jsontext.SyntacticError{Err: err}) + } + q, err := jsontext.AppendQuote(nil, b) + if err != nil { + panic("BUG: second AppendQuote should never fail: " + err.Error()) + } + return enc.WriteValue(q) + } + return enc.WriteToken(jsontext.String(s)) + } + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + xd := export.Decoder(dec) + if uo.Format != "" && uo.FormatDepth == xd.Tokens.Depth() { + return newInvalidFormatError(dec, t) + } + var flags jsonwire.ValueFlags + val, err := xd.ReadValue(&flags) + if err != nil { + return err + } + k := val.Kind() + switch k { + case 'n': + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + va.SetString("") + } + return nil + case '"': + val = jsonwire.UnquoteMayCopy(val, flags.IsVerbatim()) + if uo.Flags.Get(jsonflags.StringifyBoolsAndStrings) { + val, err = jsontext.AppendUnquote(nil, val) + if err != nil { + return newUnmarshalErrorAfter(dec, t, err) + } + if uo.Flags.Get(jsonflags.StringifyWithLegacySemantics) && string(val) == "null" { + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + va.SetString("") + } + return nil + } + } + if xd.StringCache == nil { + xd.StringCache = new(stringCache) + } + str := makeString(xd.StringCache, val) + va.SetString(str) + return nil + } + return newUnmarshalErrorAfter(dec, t, nil) + } + return &fncs +} + +var ( + appendEncodeBase16 = hex.AppendEncode + appendEncodeBase32 = base32.StdEncoding.AppendEncode + appendEncodeBase32Hex = base32.HexEncoding.AppendEncode + appendEncodeBase64 = base64.StdEncoding.AppendEncode + appendEncodeBase64URL = base64.URLEncoding.AppendEncode + encodedLenBase16 = hex.EncodedLen + encodedLenBase32 = base32.StdEncoding.EncodedLen + encodedLenBase32Hex = base32.HexEncoding.EncodedLen + encodedLenBase64 = base64.StdEncoding.EncodedLen + encodedLenBase64URL = base64.URLEncoding.EncodedLen + appendDecodeBase16 = hex.AppendDecode + appendDecodeBase32 = base32.StdEncoding.AppendDecode + appendDecodeBase32Hex = base32.HexEncoding.AppendDecode + appendDecodeBase64 = base64.StdEncoding.AppendDecode + appendDecodeBase64URL = base64.URLEncoding.AppendDecode +) + +func makeBytesArshaler(t reflect.Type, fncs *arshaler) *arshaler { + // NOTE: This handles both []~byte and [N]~byte. + // The v2 default is to treat a []namedByte as equivalent to []T + // since being able to convert []namedByte to []byte relies on + // dubious Go reflection behavior (see https://go.dev/issue/24746). + // For v1 emulation, we use jsonflags.FormatBytesWithLegacySemantics + // to forcibly treat []namedByte as a []byte. + marshalArray := fncs.marshal + isNamedByte := t.Elem().PkgPath() != "" + hasMarshaler := implementsAny(t.Elem(), allMarshalerTypes...) + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + if !mo.Flags.Get(jsonflags.FormatBytesWithLegacySemantics) && isNamedByte { + return marshalArray(enc, va, mo) // treat as []T or [N]T + } + xe := export.Encoder(enc) + appendEncode := appendEncodeBase64 + if mo.Format != "" && mo.FormatDepth == xe.Tokens.Depth() { + switch mo.Format { + case "base64": + appendEncode = appendEncodeBase64 + case "base64url": + appendEncode = appendEncodeBase64URL + case "base32": + appendEncode = appendEncodeBase32 + case "base32hex": + appendEncode = appendEncodeBase32Hex + case "base16", "hex": + appendEncode = appendEncodeBase16 + case "array": + mo.Format = "" + return marshalArray(enc, va, mo) + default: + return newInvalidFormatError(enc, t) + } + } else if mo.Flags.Get(jsonflags.FormatByteArrayAsArray) && va.Kind() == reflect.Array { + return marshalArray(enc, va, mo) + } else if mo.Flags.Get(jsonflags.FormatBytesWithLegacySemantics) && hasMarshaler { + return marshalArray(enc, va, mo) + } + if mo.Flags.Get(jsonflags.FormatNilSliceAsNull) && va.Kind() == reflect.Slice && va.IsNil() { + // TODO: Provide a "emitempty" format override? + return enc.WriteToken(jsontext.Null) + } + return xe.AppendRaw('"', true, func(b []byte) ([]byte, error) { + return appendEncode(b, va.Bytes()), nil + }) + } + unmarshalArray := fncs.unmarshal + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + if !uo.Flags.Get(jsonflags.FormatBytesWithLegacySemantics) && isNamedByte { + return unmarshalArray(dec, va, uo) // treat as []T or [N]T + } + xd := export.Decoder(dec) + appendDecode, encodedLen := appendDecodeBase64, encodedLenBase64 + if uo.Format != "" && uo.FormatDepth == xd.Tokens.Depth() { + switch uo.Format { + case "base64": + appendDecode, encodedLen = appendDecodeBase64, encodedLenBase64 + case "base64url": + appendDecode, encodedLen = appendDecodeBase64URL, encodedLenBase64URL + case "base32": + appendDecode, encodedLen = appendDecodeBase32, encodedLenBase32 + case "base32hex": + appendDecode, encodedLen = appendDecodeBase32Hex, encodedLenBase32Hex + case "base16", "hex": + appendDecode, encodedLen = appendDecodeBase16, encodedLenBase16 + case "array": + uo.Format = "" + return unmarshalArray(dec, va, uo) + default: + return newInvalidFormatError(dec, t) + } + } else if uo.Flags.Get(jsonflags.FormatByteArrayAsArray) && va.Kind() == reflect.Array { + return unmarshalArray(dec, va, uo) + } else if uo.Flags.Get(jsonflags.FormatBytesWithLegacySemantics) && dec.PeekKind() == '[' { + return unmarshalArray(dec, va, uo) + } + var flags jsonwire.ValueFlags + val, err := xd.ReadValue(&flags) + if err != nil { + return err + } + k := val.Kind() + switch k { + case 'n': + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) || va.Kind() != reflect.Array { + va.SetZero() + } + return nil + case '"': + // NOTE: The v2 default is to strictly comply with RFC 4648. + // Section 3.2 specifies that padding is required. + // Section 3.3 specifies that non-alphabet characters + // (e.g., '\r' or '\n') must be rejected. + // Section 3.5 specifies that unnecessary non-zero bits in + // the last quantum may be rejected. Since this is optional, + // we do not reject such inputs. + val = jsonwire.UnquoteMayCopy(val, flags.IsVerbatim()) + b, err := appendDecode(va.Bytes()[:0], val) + if err != nil { + return newUnmarshalErrorAfter(dec, t, err) + } + if len(val) != encodedLen(len(b)) && !uo.Flags.Get(jsonflags.ParseBytesWithLooseRFC4648) { + // TODO(https://go.dev/issue/53845): RFC 4648, section 3.3, + // specifies that non-alphabet characters must be rejected. + // Unfortunately, the "base32" and "base64" packages allow + // '\r' and '\n' characters by default. + i := bytes.IndexAny(val, "\r\n") + err := fmt.Errorf("illegal character %s at offset %d", jsonwire.QuoteRune(val[i:]), i) + return newUnmarshalErrorAfter(dec, t, err) + } + + if va.Kind() == reflect.Array { + dst := va.Bytes() + clear(dst[copy(dst, b):]) // noop if len(b) <= len(dst) + if len(b) != len(dst) && !uo.Flags.Get(jsonflags.UnmarshalArrayFromAnyLength) { + err := fmt.Errorf("decoded length of %d mismatches array length of %d", len(b), len(dst)) + return newUnmarshalErrorAfter(dec, t, err) + } + } else { + if b == nil { + b = []byte{} + } + va.SetBytes(b) + } + return nil + } + return newUnmarshalErrorAfter(dec, t, nil) + } + return fncs +} + +func makeIntArshaler(t reflect.Type) *arshaler { + var fncs arshaler + bits := t.Bits() + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + xe := export.Encoder(enc) + if mo.Format != "" && mo.FormatDepth == xe.Tokens.Depth() { + return newInvalidFormatError(enc, t) + } + + // Optimize for marshaling without preceding whitespace or string escaping. + if optimizeCommon && !mo.Flags.Get(jsonflags.AnyWhitespace|jsonflags.StringifyNumbers) && !xe.Tokens.Last.NeedObjectName() { + xe.Buf = strconv.AppendInt(xe.Tokens.MayAppendDelim(xe.Buf, '0'), va.Int(), 10) + xe.Tokens.Last.Increment() + if xe.NeedFlush() { + return xe.Flush() + } + return nil + } + + k := stringOrNumberKind(xe.Tokens.Last.NeedObjectName() || mo.Flags.Get(jsonflags.StringifyNumbers)) + return xe.AppendRaw(k, true, func(b []byte) ([]byte, error) { + return strconv.AppendInt(b, va.Int(), 10), nil + }) + } + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + xd := export.Decoder(dec) + if uo.Format != "" && uo.FormatDepth == xd.Tokens.Depth() { + return newInvalidFormatError(dec, t) + } + stringify := xd.Tokens.Last.NeedObjectName() || uo.Flags.Get(jsonflags.StringifyNumbers) + var flags jsonwire.ValueFlags + val, err := xd.ReadValue(&flags) + if err != nil { + return err + } + k := val.Kind() + switch k { + case 'n': + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + va.SetInt(0) + } + return nil + case '"': + if !stringify { + break + } + val = jsonwire.UnquoteMayCopy(val, flags.IsVerbatim()) + if uo.Flags.Get(jsonflags.StringifyWithLegacySemantics) { + // For historical reasons, v1 parsed a quoted number + // according to the Go syntax and permitted a quoted null. + // See https://go.dev/issue/75619 + n, err := strconv.ParseInt(string(val), 10, bits) + if err != nil { + if string(val) == "null" { + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + va.SetInt(0) + } + return nil + } + return newUnmarshalErrorAfterWithValue(dec, t, errors.Unwrap(err)) + } + va.SetInt(n) + return nil + } + fallthrough + case '0': + if stringify && k == '0' { + break + } + var negOffset int + neg := len(val) > 0 && val[0] == '-' + if neg { + negOffset = 1 + } + n, ok := jsonwire.ParseUint(val[negOffset:]) + maxInt := uint64(1) << (bits - 1) + overflow := (neg && n > maxInt) || (!neg && n > maxInt-1) + if !ok { + if n != math.MaxUint64 { + return newUnmarshalErrorAfterWithValue(dec, t, strconv.ErrSyntax) + } + overflow = true + } + if overflow { + return newUnmarshalErrorAfterWithValue(dec, t, strconv.ErrRange) + } + if neg { + va.SetInt(int64(-n)) + } else { + va.SetInt(int64(+n)) + } + return nil + } + return newUnmarshalErrorAfter(dec, t, nil) + } + return &fncs +} + +func makeUintArshaler(t reflect.Type) *arshaler { + var fncs arshaler + bits := t.Bits() + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + xe := export.Encoder(enc) + if mo.Format != "" && mo.FormatDepth == xe.Tokens.Depth() { + return newInvalidFormatError(enc, t) + } + + // Optimize for marshaling without preceding whitespace or string escaping. + if optimizeCommon && !mo.Flags.Get(jsonflags.AnyWhitespace|jsonflags.StringifyNumbers) && !xe.Tokens.Last.NeedObjectName() { + xe.Buf = strconv.AppendUint(xe.Tokens.MayAppendDelim(xe.Buf, '0'), va.Uint(), 10) + xe.Tokens.Last.Increment() + if xe.NeedFlush() { + return xe.Flush() + } + return nil + } + + k := stringOrNumberKind(xe.Tokens.Last.NeedObjectName() || mo.Flags.Get(jsonflags.StringifyNumbers)) + return xe.AppendRaw(k, true, func(b []byte) ([]byte, error) { + return strconv.AppendUint(b, va.Uint(), 10), nil + }) + } + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + xd := export.Decoder(dec) + if uo.Format != "" && uo.FormatDepth == xd.Tokens.Depth() { + return newInvalidFormatError(dec, t) + } + stringify := xd.Tokens.Last.NeedObjectName() || uo.Flags.Get(jsonflags.StringifyNumbers) + var flags jsonwire.ValueFlags + val, err := xd.ReadValue(&flags) + if err != nil { + return err + } + k := val.Kind() + switch k { + case 'n': + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + va.SetUint(0) + } + return nil + case '"': + if !stringify { + break + } + val = jsonwire.UnquoteMayCopy(val, flags.IsVerbatim()) + if uo.Flags.Get(jsonflags.StringifyWithLegacySemantics) { + // For historical reasons, v1 parsed a quoted number + // according to the Go syntax and permitted a quoted null. + // See https://go.dev/issue/75619 + n, err := strconv.ParseUint(string(val), 10, bits) + if err != nil { + if string(val) == "null" { + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + va.SetUint(0) + } + return nil + } + return newUnmarshalErrorAfterWithValue(dec, t, errors.Unwrap(err)) + } + va.SetUint(n) + return nil + } + fallthrough + case '0': + if stringify && k == '0' { + break + } + n, ok := jsonwire.ParseUint(val) + maxUint := uint64(1) << bits + overflow := n > maxUint-1 + if !ok { + if n != math.MaxUint64 { + return newUnmarshalErrorAfterWithValue(dec, t, strconv.ErrSyntax) + } + overflow = true + } + if overflow { + return newUnmarshalErrorAfterWithValue(dec, t, strconv.ErrRange) + } + va.SetUint(n) + return nil + } + return newUnmarshalErrorAfter(dec, t, nil) + } + return &fncs +} + +func makeFloatArshaler(t reflect.Type) *arshaler { + var fncs arshaler + bits := t.Bits() + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + xe := export.Encoder(enc) + var allowNonFinite bool + if mo.Format != "" && mo.FormatDepth == xe.Tokens.Depth() { + if mo.Format == "nonfinite" { + allowNonFinite = true + } else { + return newInvalidFormatError(enc, t) + } + } + + fv := va.Float() + if math.IsNaN(fv) || math.IsInf(fv, 0) { + if !allowNonFinite { + err := fmt.Errorf("unsupported value: %v", fv) + return newMarshalErrorBefore(enc, t, err) + } + return enc.WriteToken(jsontext.Float(fv)) + } + + // Optimize for marshaling without preceding whitespace or string escaping. + if optimizeCommon && !mo.Flags.Get(jsonflags.AnyWhitespace|jsonflags.StringifyNumbers) && !xe.Tokens.Last.NeedObjectName() { + xe.Buf = jsonwire.AppendFloat(xe.Tokens.MayAppendDelim(xe.Buf, '0'), fv, bits) + xe.Tokens.Last.Increment() + if xe.NeedFlush() { + return xe.Flush() + } + return nil + } + + k := stringOrNumberKind(xe.Tokens.Last.NeedObjectName() || mo.Flags.Get(jsonflags.StringifyNumbers)) + return xe.AppendRaw(k, true, func(b []byte) ([]byte, error) { + return jsonwire.AppendFloat(b, va.Float(), bits), nil + }) + } + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + xd := export.Decoder(dec) + var allowNonFinite bool + if uo.Format != "" && uo.FormatDepth == xd.Tokens.Depth() { + if uo.Format == "nonfinite" { + allowNonFinite = true + } else { + return newInvalidFormatError(dec, t) + } + } + stringify := xd.Tokens.Last.NeedObjectName() || uo.Flags.Get(jsonflags.StringifyNumbers) + var flags jsonwire.ValueFlags + val, err := xd.ReadValue(&flags) + if err != nil { + return err + } + k := val.Kind() + switch k { + case 'n': + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + va.SetFloat(0) + } + return nil + case '"': + val = jsonwire.UnquoteMayCopy(val, flags.IsVerbatim()) + if allowNonFinite { + switch string(val) { + case "NaN": + va.SetFloat(math.NaN()) + return nil + case "Infinity": + va.SetFloat(math.Inf(+1)) + return nil + case "-Infinity": + va.SetFloat(math.Inf(-1)) + return nil + } + } + if !stringify { + break + } + if uo.Flags.Get(jsonflags.StringifyWithLegacySemantics) { + // For historical reasons, v1 parsed a quoted number + // according to the Go syntax and permitted a quoted null. + // See https://go.dev/issue/75619 + n, err := strconv.ParseFloat(string(val), bits) + if err != nil { + if string(val) == "null" { + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + va.SetFloat(0) + } + return nil + } + return newUnmarshalErrorAfterWithValue(dec, t, errors.Unwrap(err)) + } + va.SetFloat(n) + return nil + } + if n, err := jsonwire.ConsumeNumber(val); n != len(val) || err != nil { + return newUnmarshalErrorAfterWithValue(dec, t, strconv.ErrSyntax) + } + fallthrough + case '0': + if stringify && k == '0' { + break + } + fv, ok := jsonwire.ParseFloat(val, bits) + va.SetFloat(fv) + if !ok { + return newUnmarshalErrorAfterWithValue(dec, t, strconv.ErrRange) + } + return nil + } + return newUnmarshalErrorAfter(dec, t, nil) + } + return &fncs +} + +func makeMapArshaler(t reflect.Type) *arshaler { + // NOTE: The logic below disables namespaces for tracking duplicate names + // when handling map keys with a unique representation. + + // NOTE: Values retrieved from a map are not addressable, + // so we shallow copy the values to make them addressable and + // store them back into the map afterwards. + + var fncs arshaler + var ( + once sync.Once + keyFncs *arshaler + valFncs *arshaler + ) + init := func() { + keyFncs = lookupArshaler(t.Key()) + valFncs = lookupArshaler(t.Elem()) + } + nillableLegacyKey := t.Key().Kind() == reflect.Pointer && + implementsAny(t.Key(), textMarshalerType, textAppenderType) + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + // Check for cycles. + xe := export.Encoder(enc) + if xe.Tokens.Depth() > startDetectingCyclesAfter { + if err := visitPointer(&xe.SeenPointers, va.Value); err != nil { + return newMarshalErrorBefore(enc, t, err) + } + defer leavePointer(&xe.SeenPointers, va.Value) + } + + emitNull := mo.Flags.Get(jsonflags.FormatNilMapAsNull) + if mo.Format != "" && mo.FormatDepth == xe.Tokens.Depth() { + switch mo.Format { + case "emitnull": + emitNull = true + mo.Format = "" + case "emitempty": + emitNull = false + mo.Format = "" + default: + return newInvalidFormatError(enc, t) + } + } + + // Handle empty maps. + n := va.Len() + if n == 0 { + if emitNull && va.IsNil() { + return enc.WriteToken(jsontext.Null) + } + // Optimize for marshaling an empty map without any preceding whitespace. + if optimizeCommon && !mo.Flags.Get(jsonflags.AnyWhitespace) && !xe.Tokens.Last.NeedObjectName() { + xe.Buf = append(xe.Tokens.MayAppendDelim(xe.Buf, '{'), "{}"...) + xe.Tokens.Last.Increment() + if xe.NeedFlush() { + return xe.Flush() + } + return nil + } + } + + once.Do(init) + if err := enc.WriteToken(jsontext.BeginObject); err != nil { + return err + } + if n > 0 { + nonDefaultKey := keyFncs.nonDefault + marshalKey := keyFncs.marshal + marshalVal := valFncs.marshal + if mo.Marshalers != nil { + var ok bool + marshalKey, ok = mo.Marshalers.(*Marshalers).lookup(marshalKey, t.Key()) + marshalVal, _ = mo.Marshalers.(*Marshalers).lookup(marshalVal, t.Elem()) + nonDefaultKey = nonDefaultKey || ok + } + k := newAddressableValue(t.Key()) + v := newAddressableValue(t.Elem()) + + // A Go map guarantees that each entry has a unique key. + // As such, disable the expensive duplicate name check if we know + // that every Go key will serialize as a unique JSON string. + if !nonDefaultKey && mapKeyWithUniqueRepresentation(k.Kind(), mo.Flags.Get(jsonflags.AllowInvalidUTF8)) { + xe.Tokens.Last.DisableNamespace() + } + + switch { + case !mo.Flags.Get(jsonflags.Deterministic) || n <= 1: + for iter := va.Value.MapRange(); iter.Next(); { + k.SetIterKey(iter) + err := marshalKey(enc, k, mo) + if err != nil { + if mo.Flags.Get(jsonflags.CallMethodsWithLegacySemantics) && + errors.Is(err, jsontext.ErrNonStringName) && nillableLegacyKey && k.IsNil() { + err = enc.WriteToken(jsontext.String("")) + } + if err != nil { + if serr, ok := err.(*jsontext.SyntacticError); ok && serr.Err == jsontext.ErrNonStringName { + err = newMarshalErrorBefore(enc, k.Type(), err) + } + return err + } + } + v.SetIterValue(iter) + if err := marshalVal(enc, v, mo); err != nil { + return err + } + } + case !nonDefaultKey && t.Key().Kind() == reflect.String: + names := getStrings(n) + for i, iter := 0, va.Value.MapRange(); i < n && iter.Next(); i++ { + k.SetIterKey(iter) + (*names)[i] = k.String() + } + slices.Sort(*names) + for _, name := range *names { + if err := enc.WriteToken(jsontext.String(name)); err != nil { + return err + } + // TODO(https://go.dev/issue/57061): Use v.SetMapIndexOf. + k.SetString(name) + v.Set(va.MapIndex(k.Value)) + if err := marshalVal(enc, v, mo); err != nil { + return err + } + } + putStrings(names) + default: + type member struct { + name string // unquoted name + key addressableValue + val addressableValue + } + members := make([]member, n) + keys := reflect.MakeSlice(reflect.SliceOf(t.Key()), n, n) + vals := reflect.MakeSlice(reflect.SliceOf(t.Elem()), n, n) + for i, iter := 0, va.Value.MapRange(); i < n && iter.Next(); i++ { + // Marshal the member name. + k := addressableValue{keys.Index(i), true} // indexed slice element is always addressable + k.SetIterKey(iter) + v := addressableValue{vals.Index(i), true} // indexed slice element is always addressable + v.SetIterValue(iter) + err := marshalKey(enc, k, mo) + if err != nil { + if mo.Flags.Get(jsonflags.CallMethodsWithLegacySemantics) && + errors.Is(err, jsontext.ErrNonStringName) && nillableLegacyKey && k.IsNil() { + err = enc.WriteToken(jsontext.String("")) + } + if err != nil { + if serr, ok := err.(*jsontext.SyntacticError); ok && serr.Err == jsontext.ErrNonStringName { + err = newMarshalErrorBefore(enc, k.Type(), err) + } + return err + } + } + name := xe.UnwriteOnlyObjectMemberName() + members[i] = member{name, k, v} + } + // TODO: If AllowDuplicateNames is enabled, then sort according + // to reflect.Value as well if the names are equal. + // See internal/fmtsort. + slices.SortFunc(members, func(x, y member) int { + return strings.Compare(x.name, y.name) + }) + for _, member := range members { + if err := enc.WriteToken(jsontext.String(member.name)); err != nil { + return err + } + if err := marshalVal(enc, member.val, mo); err != nil { + return err + } + } + } + } + if err := enc.WriteToken(jsontext.EndObject); err != nil { + return err + } + return nil + } + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + xd := export.Decoder(dec) + if uo.Format != "" && uo.FormatDepth == xd.Tokens.Depth() { + switch uo.Format { + case "emitnull", "emitempty": + uo.Format = "" // only relevant for marshaling + default: + return newInvalidFormatError(dec, t) + } + } + tok, err := dec.ReadToken() + if err != nil { + return err + } + k := tok.Kind() + switch k { + case 'n': + va.SetZero() + return nil + case '{': + once.Do(init) + if va.IsNil() { + va.Set(reflect.MakeMap(t)) + } + + nonDefaultKey := keyFncs.nonDefault + unmarshalKey := keyFncs.unmarshal + unmarshalVal := valFncs.unmarshal + if uo.Unmarshalers != nil { + var ok bool + unmarshalKey, ok = uo.Unmarshalers.(*Unmarshalers).lookup(unmarshalKey, t.Key()) + unmarshalVal, _ = uo.Unmarshalers.(*Unmarshalers).lookup(unmarshalVal, t.Elem()) + nonDefaultKey = nonDefaultKey || ok + } + k := newAddressableValue(t.Key()) + v := newAddressableValue(t.Elem()) + + // Manually check for duplicate entries by virtue of whether the + // unmarshaled key already exists in the destination Go map. + // Consequently, syntactically different names (e.g., "0" and "-0") + // will be rejected as duplicates since they semantically refer + // to the same Go value. This is an unusual interaction + // between syntax and semantics, but is more correct. + if !nonDefaultKey && mapKeyWithUniqueRepresentation(k.Kind(), uo.Flags.Get(jsonflags.AllowInvalidUTF8)) { + xd.Tokens.Last.DisableNamespace() + } + + // In the rare case where the map is not already empty, + // then we need to manually track which keys we already saw + // since existing presence alone is insufficient to indicate + // whether the input had a duplicate name. + var seen reflect.Value + if !uo.Flags.Get(jsonflags.AllowDuplicateNames) && va.Len() > 0 { + seen = reflect.MakeMap(reflect.MapOf(k.Type(), emptyStructType)) + } + + var errUnmarshal error + for dec.PeekKind() != '}' { + // Unmarshal the map entry key. + k.SetZero() + err := unmarshalKey(dec, k, uo) + if err != nil { + if isFatalError(err, uo.Flags) { + return err + } + if err := dec.SkipValue(); err != nil { + return err + } + errUnmarshal = cmp.Or(errUnmarshal, err) + continue + } + if k.Kind() == reflect.Interface && !k.IsNil() && !k.Elem().Type().Comparable() { + err := newUnmarshalErrorAfter(dec, t, fmt.Errorf("invalid incomparable key type %v", k.Elem().Type())) + if !uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return err + } + if err2 := dec.SkipValue(); err2 != nil { + return err2 + } + errUnmarshal = cmp.Or(errUnmarshal, err) + continue + } + + // Check if a pre-existing map entry value exists for this key. + if v2 := va.MapIndex(k.Value); v2.IsValid() { + if !uo.Flags.Get(jsonflags.AllowDuplicateNames) && (!seen.IsValid() || seen.MapIndex(k.Value).IsValid()) { + // TODO: Unread the object name. + name := xd.PreviousTokenOrValue() + return newDuplicateNameError(dec.StackPointer(), nil, dec.InputOffset()-len64(name)) + } + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + v.Set(v2) + } else { + v.SetZero() + } + } else { + v.SetZero() + } + + // Unmarshal the map entry value. + err = unmarshalVal(dec, v, uo) + va.SetMapIndex(k.Value, v.Value) + if seen.IsValid() { + seen.SetMapIndex(k.Value, reflect.Zero(emptyStructType)) + } + if err != nil { + if isFatalError(err, uo.Flags) { + return err + } + errUnmarshal = cmp.Or(errUnmarshal, err) + } + } + if _, err := dec.ReadToken(); err != nil { + return err + } + return errUnmarshal + } + return newUnmarshalErrorAfterWithSkipping(dec, t, nil) + } + return &fncs +} + +// mapKeyWithUniqueRepresentation reports whether all possible values of k +// marshal to a different JSON value, and whether all possible JSON values +// that can unmarshal into k unmarshal to different Go values. +// In other words, the representation must be a bijective. +func mapKeyWithUniqueRepresentation(k reflect.Kind, allowInvalidUTF8 bool) bool { + switch k { + case reflect.Bool, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return true + case reflect.String: + // For strings, we have to be careful since names with invalid UTF-8 + // maybe unescape to the same Go string value. + return !allowInvalidUTF8 + default: + // Floating-point kinds are not listed above since NaNs + // can appear multiple times and all serialize as "NaN". + return false + } +} + +var errNilField = errors.New("cannot set embedded pointer to unexported struct type") + +func makeStructArshaler(t reflect.Type) *arshaler { + // NOTE: The logic below disables namespaces for tracking duplicate names + // and does the tracking locally with an efficient bit-set based on which + // Go struct fields were seen. + + var fncs arshaler + var ( + once sync.Once + fields structFields + errInit *SemanticError + ) + init := func() { + fields, errInit = makeStructFields(t) + } + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + xe := export.Encoder(enc) + if mo.Format != "" && mo.FormatDepth == xe.Tokens.Depth() { + return newInvalidFormatError(enc, t) + } + once.Do(init) + if errInit != nil && !mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return newMarshalErrorBefore(enc, errInit.GoType, errInit.Err) + } + if err := enc.WriteToken(jsontext.BeginObject); err != nil { + return err + } + var seenIdxs uintSet + prevIdx := -1 + xe.Tokens.Last.DisableNamespace() // we manually ensure unique names below + for i := range fields.flattened { + f := &fields.flattened[i] + v := addressableValue{va.Field(f.index0), va.forcedAddr} // addressable if struct value is addressable + if len(f.index) > 0 { + v = v.fieldByIndex(f.index, false) + if !v.IsValid() { + continue // implies a nil inlined field + } + } + + // OmitZero skips the field if the Go value is zero, + // which we can determine up front without calling the marshaler. + if (f.omitzero || mo.Flags.Get(jsonflags.OmitZeroStructFields)) && + ((f.isZero == nil && v.IsZero()) || (f.isZero != nil && f.isZero(v))) { + continue + } + + // Check for the legacy definition of omitempty. + if f.omitempty && mo.Flags.Get(jsonflags.OmitEmptyWithLegacySemantics) && isLegacyEmpty(v) { + continue + } + + marshal := f.fncs.marshal + nonDefault := f.fncs.nonDefault + if mo.Marshalers != nil { + var ok bool + marshal, ok = mo.Marshalers.(*Marshalers).lookup(marshal, f.typ) + nonDefault = nonDefault || ok + } + + // OmitEmpty skips the field if the marshaled JSON value is empty, + // which we can know up front if there are no custom marshalers, + // otherwise we must marshal the value and unwrite it if empty. + if f.omitempty && !mo.Flags.Get(jsonflags.OmitEmptyWithLegacySemantics) && + !nonDefault && f.isEmpty != nil && f.isEmpty(v) { + continue // fast path for omitempty + } + + // Write the object member name. + // + // The logic below is semantically equivalent to: + // enc.WriteToken(String(f.name)) + // but specialized and simplified because: + // 1. The Encoder must be expecting an object name. + // 2. The object namespace is guaranteed to be disabled. + // 3. The object name is guaranteed to be valid and pre-escaped. + // 4. There is no need to flush the buffer (for unwrite purposes). + // 5. There is no possibility of an error occurring. + if optimizeCommon { + // Append any delimiters or optional whitespace. + b := xe.Buf + if xe.Tokens.Last.Length() > 0 { + b = append(b, ',') + if mo.Flags.Get(jsonflags.SpaceAfterComma) { + b = append(b, ' ') + } + } + if mo.Flags.Get(jsonflags.Multiline) { + b = xe.AppendIndent(b, xe.Tokens.NeedIndent('"')) + } + + // Append the token to the output and to the state machine. + n0 := len(b) // offset before calling AppendQuote + if !f.nameNeedEscape { + b = append(b, f.quotedName...) + } else { + b, _ = jsonwire.AppendQuote(b, f.name, &mo.Flags) + } + xe.Buf = b + xe.Names.ReplaceLastQuotedOffset(n0) + xe.Tokens.Last.Increment() + } else { + if err := enc.WriteToken(jsontext.String(f.name)); err != nil { + return err + } + } + + // Write the object member value. + flagsOriginal := mo.Flags + if f.string { + if !mo.Flags.Get(jsonflags.StringifyWithLegacySemantics) { + mo.Flags.Set(jsonflags.StringifyNumbers | 1) + } else if canLegacyStringify(f.typ) { + mo.Flags.Set(jsonflags.StringifyNumbers | jsonflags.StringifyBoolsAndStrings | 1) + } + } + if f.format != "" { + mo.FormatDepth = xe.Tokens.Depth() + mo.Format = f.format + } + err := marshal(enc, v, mo) + mo.Flags = flagsOriginal + mo.Format = "" + if err != nil { + return err + } + + // Try unwriting the member if empty (slow path for omitempty). + if f.omitempty && !mo.Flags.Get(jsonflags.OmitEmptyWithLegacySemantics) { + var prevName *string + if prevIdx >= 0 { + prevName = &fields.flattened[prevIdx].name + } + if xe.UnwriteEmptyObjectMember(prevName) { + continue + } + } + + // Remember the previous written object member. + // The set of seen fields only needs to be updated to detect + // duplicate names with those from the inlined fallback. + if !mo.Flags.Get(jsonflags.AllowDuplicateNames) && fields.inlinedFallback != nil { + seenIdxs.insert(uint(f.id)) + } + prevIdx = f.id + } + if fields.inlinedFallback != nil && !(mo.Flags.Get(jsonflags.DiscardUnknownMembers) && fields.inlinedFallback.unknown) { + var insertUnquotedName func([]byte) bool + if !mo.Flags.Get(jsonflags.AllowDuplicateNames) { + insertUnquotedName = func(name []byte) bool { + // Check that the name from inlined fallback does not match + // one of the previously marshaled names from known fields. + if foldedFields := fields.lookupByFoldedName(name); len(foldedFields) > 0 { + if f := fields.byActualName[string(name)]; f != nil { + return seenIdxs.insert(uint(f.id)) + } + for _, f := range foldedFields { + if f.matchFoldedName(name, &mo.Flags) { + return seenIdxs.insert(uint(f.id)) + } + } + } + + // Check that the name does not match any other name + // previously marshaled from the inlined fallback. + return xe.Namespaces.Last().InsertUnquoted(name) + } + } + if err := marshalInlinedFallbackAll(enc, va, mo, fields.inlinedFallback, insertUnquotedName); err != nil { + return err + } + } + if err := enc.WriteToken(jsontext.EndObject); err != nil { + return err + } + return nil + } + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + xd := export.Decoder(dec) + if uo.Format != "" && uo.FormatDepth == xd.Tokens.Depth() { + return newInvalidFormatError(dec, t) + } + tok, err := dec.ReadToken() + if err != nil { + return err + } + k := tok.Kind() + switch k { + case 'n': + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + va.SetZero() + } + return nil + case '{': + once.Do(init) + if errInit != nil && !uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return newUnmarshalErrorAfter(dec, errInit.GoType, errInit.Err) + } + var seenIdxs uintSet + xd.Tokens.Last.DisableNamespace() + var errUnmarshal error + for dec.PeekKind() != '}' { + // Process the object member name. + var flags jsonwire.ValueFlags + val, err := xd.ReadValue(&flags) + if err != nil { + return err + } + name := jsonwire.UnquoteMayCopy(val, flags.IsVerbatim()) + f := fields.byActualName[string(name)] + if f == nil { + for _, f2 := range fields.lookupByFoldedName(name) { + if f2.matchFoldedName(name, &uo.Flags) { + f = f2 + break + } + } + if f == nil { + if uo.Flags.Get(jsonflags.RejectUnknownMembers) && (fields.inlinedFallback == nil || fields.inlinedFallback.unknown) { + err := newUnmarshalErrorAfter(dec, t, ErrUnknownName) + if !uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return err + } + errUnmarshal = cmp.Or(errUnmarshal, err) + } + if !uo.Flags.Get(jsonflags.AllowDuplicateNames) && !xd.Namespaces.Last().InsertUnquoted(name) { + // TODO: Unread the object name. + return newDuplicateNameError(dec.StackPointer(), nil, dec.InputOffset()-len64(val)) + } + + if fields.inlinedFallback == nil { + // Skip unknown value since we have no place to store it. + if err := dec.SkipValue(); err != nil { + return err + } + } else { + // Marshal into value capable of storing arbitrary object members. + if err := unmarshalInlinedFallbackNext(dec, va, uo, fields.inlinedFallback, val, name); err != nil { + if isFatalError(err, uo.Flags) { + return err + } + errUnmarshal = cmp.Or(errUnmarshal, err) + } + } + continue + } + } + if !uo.Flags.Get(jsonflags.AllowDuplicateNames) && !seenIdxs.insert(uint(f.id)) { + // TODO: Unread the object name. + return newDuplicateNameError(dec.StackPointer(), nil, dec.InputOffset()-len64(val)) + } + + // Process the object member value. + unmarshal := f.fncs.unmarshal + if uo.Unmarshalers != nil { + unmarshal, _ = uo.Unmarshalers.(*Unmarshalers).lookup(unmarshal, f.typ) + } + flagsOriginal := uo.Flags + if f.string { + if !uo.Flags.Get(jsonflags.StringifyWithLegacySemantics) { + uo.Flags.Set(jsonflags.StringifyNumbers | 1) + } else if canLegacyStringify(f.typ) { + uo.Flags.Set(jsonflags.StringifyNumbers | jsonflags.StringifyBoolsAndStrings | 1) + } + } + if f.format != "" { + uo.FormatDepth = xd.Tokens.Depth() + uo.Format = f.format + } + v := addressableValue{va.Field(f.index0), va.forcedAddr} // addressable if struct value is addressable + if len(f.index) > 0 { + v = v.fieldByIndex(f.index, true) + if !v.IsValid() { + err := newUnmarshalErrorBefore(dec, t, errNilField) + if !uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return err + } + errUnmarshal = cmp.Or(errUnmarshal, err) + unmarshal = func(dec *jsontext.Decoder, _ addressableValue, _ *jsonopts.Struct) error { + return dec.SkipValue() + } + } + } + err = unmarshal(dec, v, uo) + uo.Flags = flagsOriginal + uo.Format = "" + if err != nil { + if isFatalError(err, uo.Flags) { + return err + } + errUnmarshal = cmp.Or(errUnmarshal, err) + } + } + if _, err := dec.ReadToken(); err != nil { + return err + } + return errUnmarshal + } + return newUnmarshalErrorAfterWithSkipping(dec, t, nil) + } + return &fncs +} + +func (va addressableValue) fieldByIndex(index []int, mayAlloc bool) addressableValue { + for _, i := range index { + va = va.indirect(mayAlloc) + if !va.IsValid() { + return va + } + va = addressableValue{va.Field(i), va.forcedAddr} // addressable if struct value is addressable + } + return va +} + +func (va addressableValue) indirect(mayAlloc bool) addressableValue { + if va.Kind() == reflect.Pointer { + if va.IsNil() { + if !mayAlloc || !va.CanSet() { + return addressableValue{} + } + va.Set(reflect.New(va.Type().Elem())) + } + va = addressableValue{va.Elem(), false} // dereferenced pointer is always addressable + } + return va +} + +// isLegacyEmpty reports whether a value is empty according to the v1 definition. +func isLegacyEmpty(v addressableValue) bool { + // Equivalent to encoding/json.isEmptyValue@v1.21.0. + switch v.Kind() { + case reflect.Bool: + return v.Bool() == false + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.String, reflect.Map, reflect.Slice, reflect.Array: + return v.Len() == 0 + case reflect.Pointer, reflect.Interface: + return v.IsNil() + } + return false +} + +// canLegacyStringify reports whether t can be stringified according to v1, +// where t is a bool, string, or number (or unnamed pointer to such). +// In v1, the `string` option does not apply recursively to nested types within +// a composite Go type (e.g., an array, slice, struct, map, or interface). +func canLegacyStringify(t reflect.Type) bool { + // Based on encoding/json.typeFields#L1126-L1143@v1.23.0 + if t.Name() == "" && t.Kind() == reflect.Ptr { + t = t.Elem() + } + switch t.Kind() { + case reflect.Bool, reflect.String, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, reflect.Float64: + return true + } + return false +} + +func makeSliceArshaler(t reflect.Type) *arshaler { + var fncs arshaler + var ( + once sync.Once + valFncs *arshaler + ) + init := func() { + valFncs = lookupArshaler(t.Elem()) + } + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + // Check for cycles. + xe := export.Encoder(enc) + if xe.Tokens.Depth() > startDetectingCyclesAfter { + if err := visitPointer(&xe.SeenPointers, va.Value); err != nil { + return newMarshalErrorBefore(enc, t, err) + } + defer leavePointer(&xe.SeenPointers, va.Value) + } + + emitNull := mo.Flags.Get(jsonflags.FormatNilSliceAsNull) + if mo.Format != "" && mo.FormatDepth == xe.Tokens.Depth() { + switch mo.Format { + case "emitnull": + emitNull = true + mo.Format = "" + case "emitempty": + emitNull = false + mo.Format = "" + default: + return newInvalidFormatError(enc, t) + } + } + + // Handle empty slices. + n := va.Len() + if n == 0 { + if emitNull && va.IsNil() { + return enc.WriteToken(jsontext.Null) + } + // Optimize for marshaling an empty slice without any preceding whitespace. + if optimizeCommon && !mo.Flags.Get(jsonflags.AnyWhitespace) && !xe.Tokens.Last.NeedObjectName() { + xe.Buf = append(xe.Tokens.MayAppendDelim(xe.Buf, '['), "[]"...) + xe.Tokens.Last.Increment() + if xe.NeedFlush() { + return xe.Flush() + } + return nil + } + } + + once.Do(init) + if err := enc.WriteToken(jsontext.BeginArray); err != nil { + return err + } + marshal := valFncs.marshal + if mo.Marshalers != nil { + marshal, _ = mo.Marshalers.(*Marshalers).lookup(marshal, t.Elem()) + } + for i := range n { + v := addressableValue{va.Index(i), false} // indexed slice element is always addressable + if err := marshal(enc, v, mo); err != nil { + return err + } + } + if err := enc.WriteToken(jsontext.EndArray); err != nil { + return err + } + return nil + } + emptySlice := reflect.MakeSlice(t, 0, 0) + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + xd := export.Decoder(dec) + if uo.Format != "" && uo.FormatDepth == xd.Tokens.Depth() { + switch uo.Format { + case "emitnull", "emitempty": + uo.Format = "" // only relevant for marshaling + default: + return newInvalidFormatError(dec, t) + } + } + + tok, err := dec.ReadToken() + if err != nil { + return err + } + k := tok.Kind() + switch k { + case 'n': + va.SetZero() + return nil + case '[': + once.Do(init) + unmarshal := valFncs.unmarshal + if uo.Unmarshalers != nil { + unmarshal, _ = uo.Unmarshalers.(*Unmarshalers).lookup(unmarshal, t.Elem()) + } + mustZero := true // we do not know the cleanliness of unused capacity + cap := va.Cap() + if cap > 0 { + va.SetLen(cap) + } + var i int + var errUnmarshal error + for dec.PeekKind() != ']' { + if i == cap { + va.Value.Grow(1) + cap = va.Cap() + va.SetLen(cap) + mustZero = false // reflect.Value.Grow ensures new capacity is zero-initialized + } + v := addressableValue{va.Index(i), false} // indexed slice element is always addressable + i++ + if mustZero && !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + v.SetZero() + } + if err := unmarshal(dec, v, uo); err != nil { + if isFatalError(err, uo.Flags) { + va.SetLen(i) + return err + } + errUnmarshal = cmp.Or(errUnmarshal, err) + } + } + if i == 0 { + va.Set(emptySlice) + } else { + va.SetLen(i) + } + if _, err := dec.ReadToken(); err != nil { + return err + } + return errUnmarshal + } + return newUnmarshalErrorAfterWithSkipping(dec, t, nil) + } + return &fncs +} + +var errArrayUnderflow = errors.New("too few array elements") +var errArrayOverflow = errors.New("too many array elements") + +func makeArrayArshaler(t reflect.Type) *arshaler { + var fncs arshaler + var ( + once sync.Once + valFncs *arshaler + ) + init := func() { + valFncs = lookupArshaler(t.Elem()) + } + n := t.Len() + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + xe := export.Encoder(enc) + if mo.Format != "" && mo.FormatDepth == xe.Tokens.Depth() { + return newInvalidFormatError(enc, t) + } + once.Do(init) + if err := enc.WriteToken(jsontext.BeginArray); err != nil { + return err + } + marshal := valFncs.marshal + if mo.Marshalers != nil { + marshal, _ = mo.Marshalers.(*Marshalers).lookup(marshal, t.Elem()) + } + for i := range n { + v := addressableValue{va.Index(i), va.forcedAddr} // indexed array element is addressable if array is addressable + if err := marshal(enc, v, mo); err != nil { + return err + } + } + if err := enc.WriteToken(jsontext.EndArray); err != nil { + return err + } + return nil + } + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + xd := export.Decoder(dec) + if uo.Format != "" && uo.FormatDepth == xd.Tokens.Depth() { + return newInvalidFormatError(dec, t) + } + tok, err := dec.ReadToken() + if err != nil { + return err + } + k := tok.Kind() + switch k { + case 'n': + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + va.SetZero() + } + return nil + case '[': + once.Do(init) + unmarshal := valFncs.unmarshal + if uo.Unmarshalers != nil { + unmarshal, _ = uo.Unmarshalers.(*Unmarshalers).lookup(unmarshal, t.Elem()) + } + var i int + var errUnmarshal error + for dec.PeekKind() != ']' { + if i >= n { + if err := dec.SkipValue(); err != nil { + return err + } + err = errArrayOverflow + continue + } + v := addressableValue{va.Index(i), va.forcedAddr} // indexed array element is addressable if array is addressable + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + v.SetZero() + } + if err := unmarshal(dec, v, uo); err != nil { + if isFatalError(err, uo.Flags) { + return err + } + errUnmarshal = cmp.Or(errUnmarshal, err) + } + i++ + } + for ; i < n; i++ { + va.Index(i).SetZero() + err = errArrayUnderflow + } + if _, err := dec.ReadToken(); err != nil { + return err + } + if err != nil && !uo.Flags.Get(jsonflags.UnmarshalArrayFromAnyLength) { + return newUnmarshalErrorAfter(dec, t, err) + } + return errUnmarshal + } + return newUnmarshalErrorAfterWithSkipping(dec, t, nil) + } + return &fncs +} + +func makePointerArshaler(t reflect.Type) *arshaler { + var fncs arshaler + var ( + once sync.Once + valFncs *arshaler + ) + init := func() { + valFncs = lookupArshaler(t.Elem()) + } + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + // Check for cycles. + xe := export.Encoder(enc) + if xe.Tokens.Depth() > startDetectingCyclesAfter { + if err := visitPointer(&xe.SeenPointers, va.Value); err != nil { + return newMarshalErrorBefore(enc, t, err) + } + defer leavePointer(&xe.SeenPointers, va.Value) + } + + // NOTE: Struct.Format is forwarded to underlying marshal. + if va.IsNil() { + return enc.WriteToken(jsontext.Null) + } + once.Do(init) + marshal := valFncs.marshal + if mo.Marshalers != nil { + marshal, _ = mo.Marshalers.(*Marshalers).lookup(marshal, t.Elem()) + } + v := addressableValue{va.Elem(), false} // dereferenced pointer is always addressable + return marshal(enc, v, mo) + } + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + // NOTE: Struct.Format is forwarded to underlying unmarshal. + if dec.PeekKind() == 'n' { + if _, err := dec.ReadToken(); err != nil { + return err + } + va.SetZero() + return nil + } + once.Do(init) + unmarshal := valFncs.unmarshal + if uo.Unmarshalers != nil { + unmarshal, _ = uo.Unmarshalers.(*Unmarshalers).lookup(unmarshal, t.Elem()) + } + if va.IsNil() { + va.Set(reflect.New(t.Elem())) + } + v := addressableValue{va.Elem(), false} // dereferenced pointer is always addressable + if err := unmarshal(dec, v, uo); err != nil { + return err + } + if uo.Flags.Get(jsonflags.StringifyWithLegacySemantics) && + uo.Flags.Get(jsonflags.StringifyNumbers|jsonflags.StringifyBoolsAndStrings) { + // A JSON null quoted within a JSON string should take effect + // within the pointer value, rather than the indirect value. + // + // TODO: This does not correctly handle escaped nulls + // (e.g., "\u006e\u0075\u006c\u006c"), but is good enough + // for such an esoteric use case of the `string` option. + if string(export.Decoder(dec).PreviousTokenOrValue()) == `"null"` { + va.SetZero() + } + } + return nil + } + return &fncs +} + +func makeInterfaceArshaler(t reflect.Type) *arshaler { + // NOTE: Values retrieved from an interface are not addressable, + // so we shallow copy the values to make them addressable and + // store them back into the interface afterwards. + + var fncs arshaler + var whichMarshaler reflect.Type + for _, iface := range allMarshalerTypes { + if t.Implements(iface) { + whichMarshaler = t + break + } + } + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + xe := export.Encoder(enc) + if mo.Format != "" && mo.FormatDepth == xe.Tokens.Depth() { + return newInvalidFormatError(enc, t) + } + if va.IsNil() { + return enc.WriteToken(jsontext.Null) + } else if mo.Flags.Get(jsonflags.CallMethodsWithLegacySemantics) && whichMarshaler != nil { + // The marshaler for a pointer never calls the method on a nil receiver. + // Wrap the nil pointer within a struct type so that marshal + // instead appears on a value receiver and may be called. + if va.Elem().Kind() == reflect.Pointer && va.Elem().IsNil() { + v2 := newAddressableValue(whichMarshaler) + switch whichMarshaler { + case jsonMarshalerToType: + v2.Set(reflect.ValueOf(struct{ MarshalerTo }{va.Elem().Interface().(MarshalerTo)})) + case jsonMarshalerType: + v2.Set(reflect.ValueOf(struct{ Marshaler }{va.Elem().Interface().(Marshaler)})) + case textAppenderType: + v2.Set(reflect.ValueOf(struct{ encoding.TextAppender }{va.Elem().Interface().(encoding.TextAppender)})) + case textMarshalerType: + v2.Set(reflect.ValueOf(struct{ encoding.TextMarshaler }{va.Elem().Interface().(encoding.TextMarshaler)})) + } + va = v2 + } + } + v := newAddressableValue(va.Elem().Type()) + v.Set(va.Elem()) + marshal := lookupArshaler(v.Type()).marshal + if mo.Marshalers != nil { + marshal, _ = mo.Marshalers.(*Marshalers).lookup(marshal, v.Type()) + } + // Optimize for the any type if there are no special options. + if optimizeCommon && + t == anyType && !mo.Flags.Get(jsonflags.StringifyNumbers|jsonflags.StringifyBoolsAndStrings) && mo.Format == "" && + (mo.Marshalers == nil || !mo.Marshalers.(*Marshalers).fromAny) { + return marshalValueAny(enc, va.Elem().Interface(), mo) + } + return marshal(enc, v, mo) + } + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + xd := export.Decoder(dec) + if uo.Format != "" && uo.FormatDepth == xd.Tokens.Depth() { + return newInvalidFormatError(dec, t) + } + if uo.Flags.Get(jsonflags.MergeWithLegacySemantics) && !va.IsNil() { + // Legacy merge behavior is difficult to explain. + // In general, it only merges for non-nil pointer kinds. + // As a special case, unmarshaling a JSON null into a pointer + // sets a concrete nil pointer of the underlying type + // (rather than setting the interface value itself to nil). + e := va.Elem() + if e.Kind() == reflect.Pointer && !e.IsNil() { + if dec.PeekKind() == 'n' && e.Elem().Kind() == reflect.Pointer { + if _, err := dec.ReadToken(); err != nil { + return err + } + va.Elem().Elem().SetZero() + return nil + } + } else { + va.SetZero() + } + } + if dec.PeekKind() == 'n' { + if _, err := dec.ReadToken(); err != nil { + return err + } + va.SetZero() + return nil + } + var v addressableValue + if va.IsNil() { + // Optimize for the any type if there are no special options. + // We do not care about stringified numbers since JSON strings + // are always unmarshaled into an any value as Go strings. + // Duplicate name check must be enforced since unmarshalValueAny + // does not implement merge semantics. + if optimizeCommon && + t == anyType && !uo.Flags.Get(jsonflags.AllowDuplicateNames) && uo.Format == "" && + (uo.Unmarshalers == nil || !uo.Unmarshalers.(*Unmarshalers).fromAny) { + v, err := unmarshalValueAny(dec, uo) + // We must check for nil interface values up front. + // See https://go.dev/issue/52310. + if v != nil { + va.Set(reflect.ValueOf(v)) + } + return err + } + + k := dec.PeekKind() + if !isAnyType(t) { + return newUnmarshalErrorBeforeWithSkipping(dec, t, internal.ErrNilInterface) + } + switch k { + case 'f', 't': + v = newAddressableValue(boolType) + case '"': + v = newAddressableValue(stringType) + case '0': + if uo.Flags.Get(jsonflags.UnmarshalAnyWithRawNumber) { + v = addressableValue{reflect.ValueOf(internal.NewRawNumber()).Elem(), true} + } else { + v = newAddressableValue(float64Type) + } + case '{': + v = newAddressableValue(mapStringAnyType) + case '[': + v = newAddressableValue(sliceAnyType) + default: + // If k is invalid (e.g., due to an I/O or syntax error), then + // that will be cached by PeekKind and returned by ReadValue. + // If k is '}' or ']', then ReadValue must error since + // those are invalid kinds at the start of a JSON value. + _, err := dec.ReadValue() + return err + } + } else { + // Shallow copy the existing value to keep it addressable. + // Any mutations at the top-level of the value will be observable + // since we always store this value back into the interface value. + v = newAddressableValue(va.Elem().Type()) + v.Set(va.Elem()) + } + unmarshal := lookupArshaler(v.Type()).unmarshal + if uo.Unmarshalers != nil { + unmarshal, _ = uo.Unmarshalers.(*Unmarshalers).lookup(unmarshal, v.Type()) + } + err := unmarshal(dec, v, uo) + va.Set(v.Value) + return err + } + return &fncs +} + +// isAnyType reports wether t is equivalent to the any interface type. +func isAnyType(t reflect.Type) bool { + // This is forward compatible if the Go language permits type sets within + // ordinary interfaces where an interface with zero methods does not + // necessarily mean it can hold every possible Go type. + // See https://go.dev/issue/45346. + return t == anyType || anyType.Implements(t) +} + +func makeInvalidArshaler(t reflect.Type) *arshaler { + var fncs arshaler + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + return newMarshalErrorBefore(enc, t, nil) + } + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + return newUnmarshalErrorBefore(dec, t, nil) + } + return &fncs +} + +func stringOrNumberKind(isString bool) jsontext.Kind { + if isString { + return '"' + } else { + return '0' + } +} + +type uintSet64 uint64 + +func (s uintSet64) has(i uint) bool { return s&(1< 0 } +func (s *uintSet64) set(i uint) { *s |= 1 << i } + +// uintSet is a set of unsigned integers. +// It is optimized for most integers being close to zero. +type uintSet struct { + lo uintSet64 + hi []uintSet64 +} + +// has reports whether i is in the set. +func (s *uintSet) has(i uint) bool { + if i < 64 { + return s.lo.has(i) + } else { + i -= 64 + iHi, iLo := int(i/64), i%64 + return iHi < len(s.hi) && s.hi[iHi].has(iLo) + } +} + +// insert inserts i into the set and reports whether it was the first insertion. +func (s *uintSet) insert(i uint) bool { + // TODO: Make this inlinable at least for the lower 64-bit case. + if i < 64 { + has := s.lo.has(i) + s.lo.set(i) + return !has + } else { + i -= 64 + iHi, iLo := int(i/64), i%64 + if iHi >= len(s.hi) { + s.hi = append(s.hi, make([]uintSet64, iHi+1-len(s.hi))...) + s.hi = s.hi[:cap(s.hi)] + } + has := s.hi[iHi].has(iLo) + s.hi[iHi].set(iLo) + return !has + } +} diff --git a/src/encoding/json/v2/arshal_funcs.go b/src/encoding/json/v2/arshal_funcs.go new file mode 100644 index 00000000000000..28916af948db6e --- /dev/null +++ b/src/encoding/json/v2/arshal_funcs.go @@ -0,0 +1,440 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "errors" + "fmt" + "io" + "reflect" + "sync" + + "encoding/json/internal" + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonopts" + "encoding/json/jsontext" +) + +// SkipFunc may be returned by [MarshalToFunc] and [UnmarshalFromFunc] functions. +// +// Any function that returns SkipFunc must not cause observable side effects +// on the provided [jsontext.Encoder] or [jsontext.Decoder]. +// For example, it is permissible to call [jsontext.Decoder.PeekKind], +// but not permissible to call [jsontext.Decoder.ReadToken] or +// [jsontext.Encoder.WriteToken] since such methods mutate the state. +var SkipFunc = errors.New("json: skip function") + +var errSkipMutation = errors.New("must not read or write any tokens when skipping") +var errNonSingularValue = errors.New("must read or write exactly one value") + +// Marshalers is a list of functions that may override the marshal behavior +// of specific types. Populate [WithMarshalers] to use it with +// [Marshal], [MarshalWrite], or [MarshalEncode]. +// A nil *Marshalers is equivalent to an empty list. +// There are no exported fields or methods on Marshalers. +type Marshalers = typedMarshalers + +// JoinMarshalers constructs a flattened list of marshal functions. +// If multiple functions in the list are applicable for a value of a given type, +// then those earlier in the list take precedence over those that come later. +// If a function returns [SkipFunc], then the next applicable function is called, +// otherwise the default marshaling behavior is used. +// +// For example: +// +// m1 := JoinMarshalers(f1, f2) +// m2 := JoinMarshalers(f0, m1, f3) // equivalent to m3 +// m3 := JoinMarshalers(f0, f1, f2, f3) // equivalent to m2 +func JoinMarshalers(ms ...*Marshalers) *Marshalers { + return newMarshalers(ms...) +} + +// Unmarshalers is a list of functions that may override the unmarshal behavior +// of specific types. Populate [WithUnmarshalers] to use it with +// [Unmarshal], [UnmarshalRead], or [UnmarshalDecode]. +// A nil *Unmarshalers is equivalent to an empty list. +// There are no exported fields or methods on Unmarshalers. +type Unmarshalers = typedUnmarshalers + +// JoinUnmarshalers constructs a flattened list of unmarshal functions. +// If multiple functions in the list are applicable for a value of a given type, +// then those earlier in the list take precedence over those that come later. +// If a function returns [SkipFunc], then the next applicable function is called, +// otherwise the default unmarshaling behavior is used. +// +// For example: +// +// u1 := JoinUnmarshalers(f1, f2) +// u2 := JoinUnmarshalers(f0, u1, f3) // equivalent to u3 +// u3 := JoinUnmarshalers(f0, f1, f2, f3) // equivalent to u2 +func JoinUnmarshalers(us ...*Unmarshalers) *Unmarshalers { + return newUnmarshalers(us...) +} + +type typedMarshalers = typedArshalers[jsontext.Encoder] +type typedUnmarshalers = typedArshalers[jsontext.Decoder] +type typedArshalers[Coder any] struct { + nonComparable + + fncVals []typedArshaler[Coder] + fncCache sync.Map // map[reflect.Type]arshaler + + // fromAny reports whether any of Go types used to represent arbitrary JSON + // (i.e., any, bool, string, float64, map[string]any, or []any) matches + // any of the provided type-specific arshalers. + // + // This bit of information is needed in arshal_default.go to determine + // whether to use the specialized logic in arshal_any.go to handle + // the any interface type. The logic in arshal_any.go does not support + // type-specific arshal functions, so we must avoid using that logic + // if this is true. + fromAny bool +} +type typedMarshaler = typedArshaler[jsontext.Encoder] +type typedUnmarshaler = typedArshaler[jsontext.Decoder] +type typedArshaler[Coder any] struct { + typ reflect.Type + fnc func(*Coder, addressableValue, *jsonopts.Struct) error + maySkip bool +} + +func newMarshalers(ms ...*Marshalers) *Marshalers { return newTypedArshalers(ms...) } +func newUnmarshalers(us ...*Unmarshalers) *Unmarshalers { return newTypedArshalers(us...) } +func newTypedArshalers[Coder any](as ...*typedArshalers[Coder]) *typedArshalers[Coder] { + var a typedArshalers[Coder] + for _, a2 := range as { + if a2 != nil { + a.fncVals = append(a.fncVals, a2.fncVals...) + a.fromAny = a.fromAny || a2.fromAny + } + } + if len(a.fncVals) == 0 { + return nil + } + return &a +} + +func (a *typedArshalers[Coder]) lookup(fnc func(*Coder, addressableValue, *jsonopts.Struct) error, t reflect.Type) (func(*Coder, addressableValue, *jsonopts.Struct) error, bool) { + if a == nil { + return fnc, false + } + if v, ok := a.fncCache.Load(t); ok { + if v == nil { + return fnc, false + } + return v.(func(*Coder, addressableValue, *jsonopts.Struct) error), true + } + + // Collect a list of arshalers that can be called for this type. + // This list may be longer than 1 since some arshalers can be skipped. + var fncs []func(*Coder, addressableValue, *jsonopts.Struct) error + for _, fncVal := range a.fncVals { + if !castableTo(t, fncVal.typ) { + continue + } + fncs = append(fncs, fncVal.fnc) + if !fncVal.maySkip { + break // subsequent arshalers will never be called + } + } + + if len(fncs) == 0 { + a.fncCache.Store(t, nil) // nil to indicate that no funcs found + return fnc, false + } + + // Construct an arshaler that may call every applicable arshaler. + fncDefault := fnc + fnc = func(c *Coder, v addressableValue, o *jsonopts.Struct) error { + for _, fnc := range fncs { + if err := fnc(c, v, o); err != SkipFunc { + return err // may be nil or non-nil + } + } + return fncDefault(c, v, o) + } + + // Use the first stored so duplicate work can be garbage collected. + v, _ := a.fncCache.LoadOrStore(t, fnc) + return v.(func(*Coder, addressableValue, *jsonopts.Struct) error), true +} + +// MarshalFunc constructs a type-specific marshaler that +// specifies how to marshal values of type T. +// T can be any type except a named pointer. +// The function is always provided with a non-nil pointer value +// if T is an interface or pointer type. +// +// The function must marshal exactly one JSON value. +// The value of T must not be retained outside the function call. +// It may not return [SkipFunc]. +func MarshalFunc[T any](fn func(T) ([]byte, error)) *Marshalers { + t := reflect.TypeFor[T]() + assertCastableTo(t, true) + typFnc := typedMarshaler{ + typ: t, + fnc: func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + v, _ := reflect.TypeAssert[T](va.castTo(t)) + val, err := fn(v) + if err != nil { + err = wrapSkipFunc(err, "marshal function of type func(T) ([]byte, error)") + if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalFunc") // unlike unmarshal, always wrapped + } + err = newMarshalErrorBefore(enc, t, err) + return collapseSemanticErrors(err) + } + if err := enc.WriteValue(val); err != nil { + if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalFunc") // unlike unmarshal, always wrapped + } + if isSyntacticError(err) { + err = newMarshalErrorBefore(enc, t, err) + } + return err + } + return nil + }, + } + return &Marshalers{fncVals: []typedMarshaler{typFnc}, fromAny: castableToFromAny(t)} +} + +// MarshalToFunc constructs a type-specific marshaler that +// specifies how to marshal values of type T. +// T can be any type except a named pointer. +// The function is always provided with a non-nil pointer value +// if T is an interface or pointer type. +// +// The function must marshal exactly one JSON value by calling write methods +// on the provided encoder. It may return [SkipFunc] such that marshaling can +// move on to the next marshal function. However, no mutable method calls may +// be called on the encoder if [SkipFunc] is returned. +// The pointer to [jsontext.Encoder] and the value of T +// must not be retained outside the function call. +func MarshalToFunc[T any](fn func(*jsontext.Encoder, T) error) *Marshalers { + t := reflect.TypeFor[T]() + assertCastableTo(t, true) + typFnc := typedMarshaler{ + typ: t, + fnc: func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + xe := export.Encoder(enc) + prevDepth, prevLength := xe.Tokens.DepthLength() + xe.Flags.Set(jsonflags.WithinArshalCall | 1) + v, _ := reflect.TypeAssert[T](va.castTo(t)) + err := fn(enc, v) + xe.Flags.Set(jsonflags.WithinArshalCall | 0) + currDepth, currLength := xe.Tokens.DepthLength() + if err == nil && (prevDepth != currDepth || prevLength+1 != currLength) { + err = errNonSingularValue + } + if err != nil { + if err == SkipFunc { + if prevDepth == currDepth && prevLength == currLength { + return SkipFunc + } + err = errSkipMutation + } + if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalToFunc") // unlike unmarshal, always wrapped + } + if !export.IsIOError(err) { + err = newSemanticErrorWithPosition(enc, t, prevDepth, prevLength, err) + } + return err + } + return nil + }, + maySkip: true, + } + return &Marshalers{fncVals: []typedMarshaler{typFnc}, fromAny: castableToFromAny(t)} +} + +// UnmarshalFunc constructs a type-specific unmarshaler that +// specifies how to unmarshal values of type T. +// T must be an unnamed pointer or an interface type. +// The function is always provided with a non-nil pointer value. +// +// The function must unmarshal exactly one JSON value. +// The input []byte must not be mutated. +// The input []byte and value T must not be retained outside the function call. +// It may not return [SkipFunc]. +func UnmarshalFunc[T any](fn func([]byte, T) error) *Unmarshalers { + t := reflect.TypeFor[T]() + assertCastableTo(t, false) + typFnc := typedUnmarshaler{ + typ: t, + fnc: func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + val, err := dec.ReadValue() + if err != nil { + return err // must be a syntactic or I/O error + } + v, _ := reflect.TypeAssert[T](va.castTo(t)) + err = fn(val, v) + if err != nil { + err = wrapSkipFunc(err, "unmarshal function of type func([]byte, T) error") + if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return err // unlike marshal, never wrapped + } + err = newUnmarshalErrorAfter(dec, t, err) + return collapseSemanticErrors(err) + } + return nil + }, + } + return &Unmarshalers{fncVals: []typedUnmarshaler{typFnc}, fromAny: castableToFromAny(t)} +} + +// UnmarshalFromFunc constructs a type-specific unmarshaler that +// specifies how to unmarshal values of type T. +// T must be an unnamed pointer or an interface type. +// The function is always provided with a non-nil pointer value. +// +// The function must unmarshal exactly one JSON value by calling read methods +// on the provided decoder. It may return [SkipFunc] such that unmarshaling can +// move on to the next unmarshal function. However, no mutable method calls may +// be called on the decoder if [SkipFunc] is returned. +// The pointer to [jsontext.Decoder] and the value of T +// must not be retained outside the function call. +func UnmarshalFromFunc[T any](fn func(*jsontext.Decoder, T) error) *Unmarshalers { + t := reflect.TypeFor[T]() + assertCastableTo(t, false) + typFnc := typedUnmarshaler{ + typ: t, + fnc: func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + xd := export.Decoder(dec) + prevDepth, prevLength := xd.Tokens.DepthLength() + if prevDepth == 1 && xd.AtEOF() { + return io.EOF // check EOF early to avoid fn reporting an EOF + } + xd.Flags.Set(jsonflags.WithinArshalCall | 1) + v, _ := reflect.TypeAssert[T](va.castTo(t)) + err := fn(dec, v) + xd.Flags.Set(jsonflags.WithinArshalCall | 0) + currDepth, currLength := xd.Tokens.DepthLength() + if err == nil && (prevDepth != currDepth || prevLength+1 != currLength) { + err = errNonSingularValue + } + if err != nil { + if err == SkipFunc { + if prevDepth == currDepth && prevLength == currLength { + return SkipFunc + } + err = errSkipMutation + } + if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + if err2 := xd.SkipUntil(prevDepth, prevLength+1); err2 != nil { + return err2 + } + return err // unlike marshal, never wrapped + } + if !isSyntacticError(err) && !export.IsIOError(err) { + err = newSemanticErrorWithPosition(dec, t, prevDepth, prevLength, err) + } + return err + } + return nil + }, + maySkip: true, + } + return &Unmarshalers{fncVals: []typedUnmarshaler{typFnc}, fromAny: castableToFromAny(t)} +} + +// assertCastableTo asserts that "to" is a valid type to be casted to. +// These are the Go types that type-specific arshalers may operate upon. +// +// Let AllTypes be the universal set of all possible Go types. +// This function generally asserts that: +// +// len([from for from in AllTypes if castableTo(from, to)]) > 0 +// +// otherwise it panics. +// +// As a special-case if marshal is false, then we forbid any non-pointer or +// non-interface type since it is almost always a bug trying to unmarshal +// into something where the end-user caller did not pass in an addressable value +// since they will not observe the mutations. +func assertCastableTo(to reflect.Type, marshal bool) { + switch to.Kind() { + case reflect.Interface: + return + case reflect.Pointer: + // Only allow unnamed pointers to be consistent with the fact that + // taking the address of a value produces an unnamed pointer type. + if to.Name() == "" { + return + } + default: + // Technically, non-pointer types are permissible for unmarshal. + // However, they are often a bug since the receiver would be immutable. + // Thus, only allow them for marshaling. + if marshal { + return + } + } + if marshal { + panic(fmt.Sprintf("input type %v must be an interface type, an unnamed pointer type, or a non-pointer type", to)) + } else { + panic(fmt.Sprintf("input type %v must be an interface type or an unnamed pointer type", to)) + } +} + +// castableTo checks whether values of type "from" can be casted to type "to". +// Nil pointer or interface "from" values are never considered castable. +// +// This function must be kept in sync with addressableValue.castTo. +func castableTo(from, to reflect.Type) bool { + switch to.Kind() { + case reflect.Interface: + // TODO: This breaks when ordinary interfaces can have type sets + // since interfaces now exist where only the value form of a type (T) + // implements the interface, but not the pointer variant (*T). + // See https://go.dev/issue/45346. + return reflect.PointerTo(from).Implements(to) + case reflect.Pointer: + // Common case for unmarshaling. + // From must be a concrete or interface type. + return reflect.PointerTo(from) == to + default: + // Common case for marshaling. + // From must be a concrete type. + return from == to + } +} + +// castTo casts va to the specified type. +// If the type is an interface, then the underlying type will always +// be a non-nil pointer to a concrete type. +// +// Requirement: castableTo(va.Type(), to) must hold. +func (va addressableValue) castTo(to reflect.Type) reflect.Value { + switch to.Kind() { + case reflect.Interface: + return va.Addr().Convert(to) + case reflect.Pointer: + return va.Addr() + default: + return va.Value + } +} + +// castableToFromAny reports whether "to" can be casted to from any +// of the dynamic types used to represent arbitrary JSON. +func castableToFromAny(to reflect.Type) bool { + for _, from := range []reflect.Type{anyType, boolType, stringType, float64Type, mapStringAnyType, sliceAnyType} { + if castableTo(from, to) { + return true + } + } + return false +} + +func wrapSkipFunc(err error, what string) error { + if err == SkipFunc { + return errors.New(what + " cannot be skipped") + } + return err +} diff --git a/src/encoding/json/v2/arshal_inlined.go b/src/encoding/json/v2/arshal_inlined.go new file mode 100644 index 00000000000000..03e563a0c09545 --- /dev/null +++ b/src/encoding/json/v2/arshal_inlined.go @@ -0,0 +1,230 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "bytes" + "errors" + "io" + "reflect" + "slices" + + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonopts" + "encoding/json/internal/jsonwire" + "encoding/json/jsontext" +) + +// This package supports "inlining" a Go struct field, where the contents +// of the serialized field (which must be a JSON object) are treated as if +// they are part of the parent Go struct (which represents a JSON object). +// +// Generally, inlined fields are of a Go struct type, where the fields of the +// nested struct are virtually hoisted up to the parent struct using rules +// similar to how Go embedding works (but operating within the JSON namespace). +// +// However, inlined fields may also be of a Go map type with a string key or +// a jsontext.Value. Such inlined fields are called "fallback" fields since they +// represent any arbitrary JSON object member. Explicitly named fields take +// precedence over the inlined fallback. Only one inlined fallback is allowed. + +var errRawInlinedNotObject = errors.New("inlined raw value must be a JSON object") + +var jsontextValueType = reflect.TypeFor[jsontext.Value]() + +// marshalInlinedFallbackAll marshals all the members in an inlined fallback. +func marshalInlinedFallbackAll(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct, f *structField, insertUnquotedName func([]byte) bool) error { + v := addressableValue{va.Field(f.index0), va.forcedAddr} // addressable if struct value is addressable + if len(f.index) > 0 { + v = v.fieldByIndex(f.index, false) + if !v.IsValid() { + return nil // implies a nil inlined field + } + } + v = v.indirect(false) + if !v.IsValid() { + return nil + } + + if v.Type() == jsontextValueType { + b, _ := reflect.TypeAssert[jsontext.Value](v.Value) + if len(b) == 0 { // TODO: Should this be nil? What if it were all whitespace? + return nil + } + + dec := export.GetBufferedDecoder(b) + defer export.PutBufferedDecoder(dec) + xd := export.Decoder(dec) + xd.Flags.Set(jsonflags.AllowDuplicateNames | jsonflags.AllowInvalidUTF8 | 1) + + tok, err := dec.ReadToken() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return newMarshalErrorBefore(enc, v.Type(), err) + } + if tok.Kind() != '{' { + return newMarshalErrorBefore(enc, v.Type(), errRawInlinedNotObject) + } + for dec.PeekKind() != '}' { + // Parse the JSON object name. + var flags jsonwire.ValueFlags + val, err := xd.ReadValue(&flags) + if err != nil { + return newMarshalErrorBefore(enc, v.Type(), err) + } + if insertUnquotedName != nil { + name := jsonwire.UnquoteMayCopy(val, flags.IsVerbatim()) + if !insertUnquotedName(name) { + return newDuplicateNameError(enc.StackPointer().Parent(), val, enc.OutputOffset()) + } + } + if err := enc.WriteValue(val); err != nil { + return err + } + + // Parse the JSON object value. + val, err = xd.ReadValue(&flags) + if err != nil { + return newMarshalErrorBefore(enc, v.Type(), err) + } + if err := enc.WriteValue(val); err != nil { + return err + } + } + if _, err := dec.ReadToken(); err != nil { + return newMarshalErrorBefore(enc, v.Type(), err) + } + if err := xd.CheckEOF(); err != nil { + return newMarshalErrorBefore(enc, v.Type(), err) + } + return nil + } else { + m := v // must be a map[~string]V + n := m.Len() + if n == 0 { + return nil + } + mk := newAddressableValue(m.Type().Key()) + mv := newAddressableValue(m.Type().Elem()) + marshalKey := func(mk addressableValue) error { + b, err := jsonwire.AppendQuote(enc.AvailableBuffer(), mk.String(), &mo.Flags) + if err != nil { + return newMarshalErrorBefore(enc, m.Type().Key(), err) + } + if insertUnquotedName != nil { + isVerbatim := bytes.IndexByte(b, '\\') < 0 + name := jsonwire.UnquoteMayCopy(b, isVerbatim) + if !insertUnquotedName(name) { + return newDuplicateNameError(enc.StackPointer().Parent(), b, enc.OutputOffset()) + } + } + return enc.WriteValue(b) + } + marshalVal := f.fncs.marshal + if mo.Marshalers != nil { + marshalVal, _ = mo.Marshalers.(*Marshalers).lookup(marshalVal, mv.Type()) + } + if !mo.Flags.Get(jsonflags.Deterministic) || n <= 1 { + for iter := m.MapRange(); iter.Next(); { + mk.SetIterKey(iter) + if err := marshalKey(mk); err != nil { + return err + } + mv.Set(iter.Value()) + if err := marshalVal(enc, mv, mo); err != nil { + return err + } + } + } else { + names := getStrings(n) + for i, iter := 0, m.Value.MapRange(); i < n && iter.Next(); i++ { + mk.SetIterKey(iter) + (*names)[i] = mk.String() + } + slices.Sort(*names) + for _, name := range *names { + mk.SetString(name) + if err := marshalKey(mk); err != nil { + return err + } + // TODO(https://go.dev/issue/57061): Use mv.SetMapIndexOf. + mv.Set(m.MapIndex(mk.Value)) + if err := marshalVal(enc, mv, mo); err != nil { + return err + } + } + putStrings(names) + } + return nil + } +} + +// unmarshalInlinedFallbackNext unmarshals only the next member in an inlined fallback. +func unmarshalInlinedFallbackNext(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct, f *structField, quotedName, unquotedName []byte) error { + v := addressableValue{va.Field(f.index0), va.forcedAddr} // addressable if struct value is addressable + if len(f.index) > 0 { + v = v.fieldByIndex(f.index, true) + } + v = v.indirect(true) + + if v.Type() == jsontextValueType { + b, _ := reflect.TypeAssert[*jsontext.Value](v.Addr()) + if len(*b) == 0 { // TODO: Should this be nil? What if it were all whitespace? + *b = append(*b, '{') + } else { + *b = jsonwire.TrimSuffixWhitespace(*b) + if jsonwire.HasSuffixByte(*b, '}') { + // TODO: When merging into an object for the first time, + // should we verify that it is valid? + *b = jsonwire.TrimSuffixByte(*b, '}') + *b = jsonwire.TrimSuffixWhitespace(*b) + if !jsonwire.HasSuffixByte(*b, ',') && !jsonwire.HasSuffixByte(*b, '{') { + *b = append(*b, ',') + } + } else { + return newUnmarshalErrorAfterWithSkipping(dec, v.Type(), errRawInlinedNotObject) + } + } + *b = append(*b, quotedName...) + *b = append(*b, ':') + val, err := dec.ReadValue() + if err != nil { + return err + } + *b = append(*b, val...) + *b = append(*b, '}') + return nil + } else { + name := string(unquotedName) // TODO: Intern this? + + m := v // must be a map[~string]V + if m.IsNil() { + m.Set(reflect.MakeMap(m.Type())) + } + mk := reflect.ValueOf(name) + if mkt := m.Type().Key(); mkt != stringType { + mk = mk.Convert(mkt) + } + mv := newAddressableValue(m.Type().Elem()) // TODO: Cache across calls? + if v2 := m.MapIndex(mk); v2.IsValid() { + mv.Set(v2) + } + + unmarshal := f.fncs.unmarshal + if uo.Unmarshalers != nil { + unmarshal, _ = uo.Unmarshalers.(*Unmarshalers).lookup(unmarshal, mv.Type()) + } + err := unmarshal(dec, mv, uo) + m.SetMapIndex(mk, mv.Value) + if err != nil { + return err + } + return nil + } +} diff --git a/src/encoding/json/v2/arshal_methods.go b/src/encoding/json/v2/arshal_methods.go new file mode 100644 index 00000000000000..1621eadc080763 --- /dev/null +++ b/src/encoding/json/v2/arshal_methods.go @@ -0,0 +1,361 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "encoding" + "errors" + "io" + "reflect" + + "encoding/json/internal" + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonopts" + "encoding/json/internal/jsonwire" + "encoding/json/jsontext" +) + +var errNonStringValue = errors.New("JSON value must be string type") + +// Interfaces for custom serialization. +var ( + jsonMarshalerType = reflect.TypeFor[Marshaler]() + jsonMarshalerToType = reflect.TypeFor[MarshalerTo]() + jsonUnmarshalerType = reflect.TypeFor[Unmarshaler]() + jsonUnmarshalerFromType = reflect.TypeFor[UnmarshalerFrom]() + textAppenderType = reflect.TypeFor[encoding.TextAppender]() + textMarshalerType = reflect.TypeFor[encoding.TextMarshaler]() + textUnmarshalerType = reflect.TypeFor[encoding.TextUnmarshaler]() + + allMarshalerTypes = []reflect.Type{jsonMarshalerToType, jsonMarshalerType, textAppenderType, textMarshalerType} + allUnmarshalerTypes = []reflect.Type{jsonUnmarshalerFromType, jsonUnmarshalerType, textUnmarshalerType} + allMethodTypes = append(allMarshalerTypes, allUnmarshalerTypes...) +) + +// Marshaler is implemented by types that can marshal themselves. +// It is recommended that types implement [MarshalerTo] unless the implementation +// is trying to avoid a hard dependency on the "jsontext" package. +// +// It is recommended that implementations return a buffer that is safe +// for the caller to retain and potentially mutate. +// +// If the returned error is a [SemanticError], then unpopulated fields +// of the error may be populated by [json] with additional context. +// Errors of other types are wrapped within a [SemanticError]. +type Marshaler interface { + MarshalJSON() ([]byte, error) +} + +// MarshalerTo is implemented by types that can marshal themselves. +// It is recommended that types implement MarshalerTo instead of [Marshaler] +// since this is both more performant and flexible. +// If a type implements both Marshaler and MarshalerTo, +// then MarshalerTo takes precedence. In such a case, both implementations +// should aim to have equivalent behavior for the default marshal options. +// +// The implementation must write only one JSON value to the Encoder and +// must not retain the pointer to [jsontext.Encoder]. +// +// If the returned error is a [SemanticError], then unpopulated fields +// of the error may be populated by [json] with additional context. +// Errors of other types are wrapped within a [SemanticError], +// unless it is an IO error. +type MarshalerTo interface { + MarshalJSONTo(*jsontext.Encoder) error + + // TODO: Should users call the MarshalEncode function or + // should/can they call this method directly? Does it matter? +} + +// Unmarshaler is implemented by types that can unmarshal themselves. +// It is recommended that types implement [UnmarshalerFrom] unless the implementation +// is trying to avoid a hard dependency on the "jsontext" package. +// +// The input can be assumed to be a valid encoding of a JSON value +// if called from unmarshal functionality in this package. +// UnmarshalJSON must copy the JSON data if it is retained after returning. +// It is recommended that UnmarshalJSON implement merge semantics when +// unmarshaling into a pre-populated value. +// +// Implementations must not retain or mutate the input []byte. +// +// If the returned error is a [SemanticError], then unpopulated fields +// of the error may be populated by [json] with additional context. +// Errors of other types are wrapped within a [SemanticError]. +type Unmarshaler interface { + UnmarshalJSON([]byte) error +} + +// UnmarshalerFrom is implemented by types that can unmarshal themselves. +// It is recommended that types implement UnmarshalerFrom instead of [Unmarshaler] +// since this is both more performant and flexible. +// If a type implements both Unmarshaler and UnmarshalerFrom, +// then UnmarshalerFrom takes precedence. In such a case, both implementations +// should aim to have equivalent behavior for the default unmarshal options. +// +// The implementation must read only one JSON value from the Decoder. +// It is recommended that UnmarshalJSONFrom implement merge semantics when +// unmarshaling into a pre-populated value. +// +// Implementations must not retain the pointer to [jsontext.Decoder]. +// +// If the returned error is a [SemanticError], then unpopulated fields +// of the error may be populated by [json] with additional context. +// Errors of other types are wrapped within a [SemanticError], +// unless it is a [jsontext.SyntacticError] or an IO error. +type UnmarshalerFrom interface { + UnmarshalJSONFrom(*jsontext.Decoder) error + + // TODO: Should users call the UnmarshalDecode function or + // should/can they call this method directly? Does it matter? +} + +func makeMethodArshaler(fncs *arshaler, t reflect.Type) *arshaler { + // Avoid injecting method arshaler on the pointer or interface version + // to avoid ever calling the method on a nil pointer or interface receiver. + // Let it be injected on the value receiver (which is always addressable). + if t.Kind() == reflect.Pointer || t.Kind() == reflect.Interface { + return fncs + } + + if needAddr, ok := implements(t, textMarshalerType); ok { + fncs.nonDefault = true + prevMarshal := fncs.marshal + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + if mo.Flags.Get(jsonflags.CallMethodsWithLegacySemantics) && + (needAddr && va.forcedAddr) { + return prevMarshal(enc, va, mo) + } + marshaler, _ := reflect.TypeAssert[encoding.TextMarshaler](va.Addr()) + if err := export.Encoder(enc).AppendRaw('"', false, func(b []byte) ([]byte, error) { + b2, err := marshaler.MarshalText() + return append(b, b2...), err + }); err != nil { + err = wrapSkipFunc(err, "marshal method") + if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalText") // unlike unmarshal, always wrapped + } + if !isSemanticError(err) && !export.IsIOError(err) { + err = newMarshalErrorBefore(enc, t, err) + } + return err + } + return nil + } + } + + if needAddr, ok := implements(t, textAppenderType); ok { + fncs.nonDefault = true + prevMarshal := fncs.marshal + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) (err error) { + if mo.Flags.Get(jsonflags.CallMethodsWithLegacySemantics) && + (needAddr && va.forcedAddr) { + return prevMarshal(enc, va, mo) + } + appender, _ := reflect.TypeAssert[encoding.TextAppender](va.Addr()) + if err := export.Encoder(enc).AppendRaw('"', false, appender.AppendText); err != nil { + err = wrapSkipFunc(err, "append method") + if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return internal.NewMarshalerError(va.Addr().Interface(), err, "AppendText") // unlike unmarshal, always wrapped + } + if !isSemanticError(err) && !export.IsIOError(err) { + err = newMarshalErrorBefore(enc, t, err) + } + return err + } + return nil + } + } + + if needAddr, ok := implements(t, jsonMarshalerType); ok { + fncs.nonDefault = true + prevMarshal := fncs.marshal + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + if mo.Flags.Get(jsonflags.CallMethodsWithLegacySemantics) && + ((needAddr && va.forcedAddr) || export.Encoder(enc).Tokens.Last.NeedObjectName()) { + return prevMarshal(enc, va, mo) + } + marshaler, _ := reflect.TypeAssert[Marshaler](va.Addr()) + val, err := marshaler.MarshalJSON() + if err != nil { + err = wrapSkipFunc(err, "marshal method") + if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalJSON") // unlike unmarshal, always wrapped + } + err = newMarshalErrorBefore(enc, t, err) + return collapseSemanticErrors(err) + } + if err := enc.WriteValue(val); err != nil { + if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalJSON") // unlike unmarshal, always wrapped + } + if isSyntacticError(err) { + err = newMarshalErrorBefore(enc, t, err) + } + return err + } + return nil + } + } + + if needAddr, ok := implements(t, jsonMarshalerToType); ok { + fncs.nonDefault = true + prevMarshal := fncs.marshal + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + if mo.Flags.Get(jsonflags.CallMethodsWithLegacySemantics) && + ((needAddr && va.forcedAddr) || export.Encoder(enc).Tokens.Last.NeedObjectName()) { + return prevMarshal(enc, va, mo) + } + xe := export.Encoder(enc) + prevDepth, prevLength := xe.Tokens.DepthLength() + xe.Flags.Set(jsonflags.WithinArshalCall | 1) + marshaler, _ := reflect.TypeAssert[MarshalerTo](va.Addr()) + err := marshaler.MarshalJSONTo(enc) + xe.Flags.Set(jsonflags.WithinArshalCall | 0) + currDepth, currLength := xe.Tokens.DepthLength() + if (prevDepth != currDepth || prevLength+1 != currLength) && err == nil { + err = errNonSingularValue + } + if err != nil { + err = wrapSkipFunc(err, "marshal method") + if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalJSONTo") // unlike unmarshal, always wrapped + } + if !export.IsIOError(err) { + err = newSemanticErrorWithPosition(enc, t, prevDepth, prevLength, err) + } + return err + } + return nil + } + } + + if _, ok := implements(t, textUnmarshalerType); ok { + fncs.nonDefault = true + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + xd := export.Decoder(dec) + var flags jsonwire.ValueFlags + val, err := xd.ReadValue(&flags) + if err != nil { + return err // must be a syntactic or I/O error + } + if val.Kind() == 'n' { + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + va.SetZero() + } + return nil + } + if val.Kind() != '"' { + return newUnmarshalErrorAfter(dec, t, errNonStringValue) + } + s := jsonwire.UnquoteMayCopy(val, flags.IsVerbatim()) + unmarshaler, _ := reflect.TypeAssert[encoding.TextUnmarshaler](va.Addr()) + if err := unmarshaler.UnmarshalText(s); err != nil { + err = wrapSkipFunc(err, "unmarshal method") + if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return err // unlike marshal, never wrapped + } + if !isSemanticError(err) && !isSyntacticError(err) && !export.IsIOError(err) { + err = newUnmarshalErrorAfter(dec, t, err) + } + return err + } + return nil + } + } + + if _, ok := implements(t, jsonUnmarshalerType); ok { + fncs.nonDefault = true + prevUnmarshal := fncs.unmarshal + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + if uo.Flags.Get(jsonflags.CallMethodsWithLegacySemantics) && + export.Decoder(dec).Tokens.Last.NeedObjectName() { + return prevUnmarshal(dec, va, uo) + } + val, err := dec.ReadValue() + if err != nil { + return err // must be a syntactic or I/O error + } + unmarshaler, _ := reflect.TypeAssert[Unmarshaler](va.Addr()) + if err := unmarshaler.UnmarshalJSON(val); err != nil { + err = wrapSkipFunc(err, "unmarshal method") + if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return err // unlike marshal, never wrapped + } + err = newUnmarshalErrorAfter(dec, t, err) + return collapseSemanticErrors(err) + } + return nil + } + } + + if _, ok := implements(t, jsonUnmarshalerFromType); ok { + fncs.nonDefault = true + prevUnmarshal := fncs.unmarshal + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + if uo.Flags.Get(jsonflags.CallMethodsWithLegacySemantics) && + export.Decoder(dec).Tokens.Last.NeedObjectName() { + return prevUnmarshal(dec, va, uo) + } + xd := export.Decoder(dec) + prevDepth, prevLength := xd.Tokens.DepthLength() + if prevDepth == 1 && xd.AtEOF() { + return io.EOF // check EOF early to avoid fn reporting an EOF + } + xd.Flags.Set(jsonflags.WithinArshalCall | 1) + unmarshaler, _ := reflect.TypeAssert[UnmarshalerFrom](va.Addr()) + err := unmarshaler.UnmarshalJSONFrom(dec) + xd.Flags.Set(jsonflags.WithinArshalCall | 0) + currDepth, currLength := xd.Tokens.DepthLength() + if (prevDepth != currDepth || prevLength+1 != currLength) && err == nil { + err = errNonSingularValue + } + if err != nil { + err = wrapSkipFunc(err, "unmarshal method") + if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + if err2 := xd.SkipUntil(prevDepth, prevLength+1); err2 != nil { + return err2 + } + return err // unlike marshal, never wrapped + } + if !isSyntacticError(err) && !export.IsIOError(err) { + err = newSemanticErrorWithPosition(dec, t, prevDepth, prevLength, err) + } + return err + } + return nil + } + } + + return fncs +} + +// implementsAny is like t.Implements(ifaceType) for a list of interfaces, +// but checks whether either t or reflect.PointerTo(t) implements the interface. +func implementsAny(t reflect.Type, ifaceTypes ...reflect.Type) bool { + for _, ifaceType := range ifaceTypes { + if _, ok := implements(t, ifaceType); ok { + return true + } + } + return false +} + +// implements is like t.Implements(ifaceType) but checks whether +// either t or reflect.PointerTo(t) implements the interface. +// It also reports whether the value needs to be addressed +// in order to satisfy the interface. +func implements(t, ifaceType reflect.Type) (needAddr, ok bool) { + switch { + case t.Implements(ifaceType): + return false, true + case reflect.PointerTo(t).Implements(ifaceType): + return true, true + default: + return false, false + } +} diff --git a/src/encoding/json/v2/arshal_test.go b/src/encoding/json/v2/arshal_test.go new file mode 100644 index 00000000000000..dc15c5a5f531b5 --- /dev/null +++ b/src/encoding/json/v2/arshal_test.go @@ -0,0 +1,9620 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "bytes" + "encoding" + "encoding/base32" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "io" + "math" + "net" + "net/netip" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "encoding/json/internal" + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonopts" + "encoding/json/internal/jsontest" + "encoding/json/internal/jsonwire" + "encoding/json/jsontext" +) + +func newNonStringNameError(offset int64, pointer jsontext.Pointer) error { + return &jsontext.SyntacticError{ByteOffset: offset, JSONPointer: pointer, Err: jsontext.ErrNonStringName} +} + +func newInvalidCharacterError(prefix, where string, offset int64, pointer jsontext.Pointer) error { + return &jsontext.SyntacticError{ByteOffset: offset, JSONPointer: pointer, Err: jsonwire.NewInvalidCharacterError(prefix, where)} +} + +func newInvalidUTF8Error(offset int64, pointer jsontext.Pointer) error { + return &jsontext.SyntacticError{ByteOffset: offset, JSONPointer: pointer, Err: jsonwire.ErrInvalidUTF8} +} + +func newParseTimeError(layout, value, layoutElem, valueElem, message string) error { + return &time.ParseError{Layout: layout, Value: value, LayoutElem: layoutElem, ValueElem: valueElem, Message: message} +} + +func EM(err error) *SemanticError { + return &SemanticError{action: "marshal", Err: err} +} + +func EU(err error) *SemanticError { + return &SemanticError{action: "unmarshal", Err: err} +} + +func (e *SemanticError) withVal(val string) *SemanticError { + e.JSONValue = jsontext.Value(val) + return e +} + +func (e *SemanticError) withPos(prefix string, pointer jsontext.Pointer) *SemanticError { + e.ByteOffset = int64(len(prefix)) + e.JSONPointer = pointer + return e +} + +func (e *SemanticError) withType(k jsontext.Kind, t reflect.Type) *SemanticError { + e.JSONKind = k + e.GoType = t + return e +} + +var ( + errInvalidFormatFlag = errors.New(`invalid format flag "invalid"`) + errSomeError = errors.New("some error") + errMustNotCall = errors.New("must not call") +) + +func T[T any]() reflect.Type { return reflect.TypeFor[T]() } + +type ( + jsonObject = map[string]any + jsonArray = []any + + namedAny any + namedBool bool + namedString string + NamedString string + namedBytes []byte + namedInt64 int64 + namedUint64 uint64 + namedFloat64 float64 + namedByte byte + netipAddr = netip.Addr + + recursiveMap map[string]recursiveMap + recursiveSlice []recursiveSlice + recursivePointer struct{ P *recursivePointer } + + structEmpty struct{} + structConflicting struct { + A string `json:"conflict"` + B string `json:"conflict"` + } + structNoneExported struct { + unexported string + } + structUnexportedIgnored struct { + ignored string `json:"-"` + } + structMalformedTag struct { + Malformed string `json:"\""` + } + structUnexportedTag struct { + unexported string `json:"name"` + } + structExportedEmbedded struct { + NamedString + } + structExportedEmbeddedTag struct { + NamedString `json:"name"` + } + structUnexportedEmbedded struct { + namedString + } + structUnexportedEmbeddedTag struct { + namedString `json:"name"` + } + structUnexportedEmbeddedMethodTag struct { + // netipAddr cannot be marshaled since the MarshalText method + // cannot be called on an unexported field. + netipAddr `json:"name"` + + // Bogus MarshalText and AppendText methods are declared on + // structUnexportedEmbeddedMethodTag to prevent it from + // implementing those method interfaces. + } + structUnexportedEmbeddedStruct struct { + structOmitZeroAll + FizzBuzz int + structNestedAddr + } + structUnexportedEmbeddedStructPointer struct { + *structOmitZeroAll + FizzBuzz int + *structNestedAddr + } + structNestedAddr struct { + Addr netip.Addr + } + structIgnoredUnexportedEmbedded struct { + namedString `json:"-"` + } + structWeirdNames struct { + Empty string `json:"''"` + Comma string `json:"','"` + Quote string `json:"'\"'"` + } + structNoCase struct { + Aaa string `json:",case:strict"` + AA_A string + AaA string `json:",case:ignore"` + AAa string `json:",case:ignore"` + AAA string + } + structScalars struct { + unexported bool + Ignored bool `json:"-"` + + Bool bool + String string + Bytes []byte + Int int64 + Uint uint64 + Float float64 + } + structSlices struct { + unexported bool + Ignored bool `json:"-"` + + SliceBool []bool + SliceString []string + SliceBytes [][]byte + SliceInt []int64 + SliceUint []uint64 + SliceFloat []float64 + } + structMaps struct { + unexported bool + Ignored bool `json:"-"` + + MapBool map[string]bool + MapString map[string]string + MapBytes map[string][]byte + MapInt map[string]int64 + MapUint map[string]uint64 + MapFloat map[string]float64 + } + structAll struct { + Bool bool + String string + Bytes []byte + Int int64 + Uint uint64 + Float float64 + Map map[string]string + StructScalars structScalars + StructMaps structMaps + StructSlices structSlices + Slice []string + Array [1]string + Pointer *structAll + Interface any + } + structStringifiedAll struct { + Bool bool `json:",string"` + String string `json:",string"` + Bytes []byte `json:",string"` + Int int64 `json:",string"` + Uint uint64 `json:",string"` + Float float64 `json:",string"` + Map map[string]string `json:",string"` + StructScalars structScalars `json:",string"` + StructMaps structMaps `json:",string"` + StructSlices structSlices `json:",string"` + Slice []string `json:",string"` + Array [1]string `json:",string"` + Pointer *structStringifiedAll `json:",string"` + Interface any `json:",string"` + } + structOmitZeroAll struct { + Bool bool `json:",omitzero"` + String string `json:",omitzero"` + Bytes []byte `json:",omitzero"` + Int int64 `json:",omitzero"` + Uint uint64 `json:",omitzero"` + Float float64 `json:",omitzero"` + Map map[string]string `json:",omitzero"` + StructScalars structScalars `json:",omitzero"` + StructMaps structMaps `json:",omitzero"` + StructSlices structSlices `json:",omitzero"` + Slice []string `json:",omitzero"` + Array [1]string `json:",omitzero"` + Pointer *structOmitZeroAll `json:",omitzero"` + Interface any `json:",omitzero"` + } + structOmitZeroMethodAll struct { + ValueAlwaysZero valueAlwaysZero `json:",omitzero"` + ValueNeverZero valueNeverZero `json:",omitzero"` + PointerAlwaysZero pointerAlwaysZero `json:",omitzero"` + PointerNeverZero pointerNeverZero `json:",omitzero"` + PointerValueAlwaysZero *valueAlwaysZero `json:",omitzero"` + PointerValueNeverZero *valueNeverZero `json:",omitzero"` + PointerPointerAlwaysZero *pointerAlwaysZero `json:",omitzero"` + PointerPointerNeverZero *pointerNeverZero `json:",omitzero"` + PointerPointerValueAlwaysZero **valueAlwaysZero `json:",omitzero"` + PointerPointerValueNeverZero **valueNeverZero `json:",omitzero"` + PointerPointerPointerAlwaysZero **pointerAlwaysZero `json:",omitzero"` + PointerPointerPointerNeverZero **pointerNeverZero `json:",omitzero"` + } + structOmitZeroMethodInterfaceAll struct { + ValueAlwaysZero isZeroer `json:",omitzero"` + ValueNeverZero isZeroer `json:",omitzero"` + PointerValueAlwaysZero isZeroer `json:",omitzero"` + PointerValueNeverZero isZeroer `json:",omitzero"` + PointerPointerAlwaysZero isZeroer `json:",omitzero"` + PointerPointerNeverZero isZeroer `json:",omitzero"` + } + structOmitEmptyAll struct { + Bool bool `json:",omitempty"` + PointerBool *bool `json:",omitempty"` + String string `json:",omitempty"` + StringEmpty stringMarshalEmpty `json:",omitempty"` + StringNonEmpty stringMarshalNonEmpty `json:",omitempty"` + PointerString *string `json:",omitempty"` + PointerStringEmpty *stringMarshalEmpty `json:",omitempty"` + PointerStringNonEmpty *stringMarshalNonEmpty `json:",omitempty"` + Bytes []byte `json:",omitempty"` + BytesEmpty bytesMarshalEmpty `json:",omitempty"` + BytesNonEmpty bytesMarshalNonEmpty `json:",omitempty"` + PointerBytes *[]byte `json:",omitempty"` + PointerBytesEmpty *bytesMarshalEmpty `json:",omitempty"` + PointerBytesNonEmpty *bytesMarshalNonEmpty `json:",omitempty"` + Float float64 `json:",omitempty"` + PointerFloat *float64 `json:",omitempty"` + Map map[string]string `json:",omitempty"` + MapEmpty mapMarshalEmpty `json:",omitempty"` + MapNonEmpty mapMarshalNonEmpty `json:",omitempty"` + PointerMap *map[string]string `json:",omitempty"` + PointerMapEmpty *mapMarshalEmpty `json:",omitempty"` + PointerMapNonEmpty *mapMarshalNonEmpty `json:",omitempty"` + Slice []string `json:",omitempty"` + SliceEmpty sliceMarshalEmpty `json:",omitempty"` + SliceNonEmpty sliceMarshalNonEmpty `json:",omitempty"` + PointerSlice *[]string `json:",omitempty"` + PointerSliceEmpty *sliceMarshalEmpty `json:",omitempty"` + PointerSliceNonEmpty *sliceMarshalNonEmpty `json:",omitempty"` + Pointer *structOmitZeroEmptyAll `json:",omitempty"` + Interface any `json:",omitempty"` + } + structOmitZeroEmptyAll struct { + Bool bool `json:",omitzero,omitempty"` + String string `json:",omitzero,omitempty"` + Bytes []byte `json:",omitzero,omitempty"` + Int int64 `json:",omitzero,omitempty"` + Uint uint64 `json:",omitzero,omitempty"` + Float float64 `json:",omitzero,omitempty"` + Map map[string]string `json:",omitzero,omitempty"` + Slice []string `json:",omitzero,omitempty"` + Array [1]string `json:",omitzero,omitempty"` + Pointer *structOmitZeroEmptyAll `json:",omitzero,omitempty"` + Interface any `json:",omitzero,omitempty"` + } + structFormatBytes struct { + Base16 []byte `json:",format:base16"` + Base32 []byte `json:",format:base32"` + Base32Hex []byte `json:",format:base32hex"` + Base64 []byte `json:",format:base64"` + Base64URL []byte `json:",format:base64url"` + Array []byte `json:",format:array"` + } + structFormatArrayBytes struct { + Base16 [4]byte `json:",format:base16"` + Base32 [4]byte `json:",format:base32"` + Base32Hex [4]byte `json:",format:base32hex"` + Base64 [4]byte `json:",format:base64"` + Base64URL [4]byte `json:",format:base64url"` + Array [4]byte `json:",format:array"` + Default [4]byte + } + structFormatFloats struct { + NonFinite float64 `json:",format:nonfinite"` + PointerNonFinite *float64 `json:",format:nonfinite"` + } + structFormatMaps struct { + EmitNull map[string]string `json:",format:emitnull"` + PointerEmitNull *map[string]string `json:",format:emitnull"` + EmitEmpty map[string]string `json:",format:emitempty"` + PointerEmitEmpty *map[string]string `json:",format:emitempty"` + EmitDefault map[string]string + PointerEmitDefault *map[string]string + } + structFormatSlices struct { + EmitNull []string `json:",format:emitnull"` + PointerEmitNull *[]string `json:",format:emitnull"` + EmitEmpty []string `json:",format:emitempty"` + PointerEmitEmpty *[]string `json:",format:emitempty"` + EmitDefault []string + PointerEmitDefault *[]string + } + structFormatInvalid struct { + Bool bool `json:",omitzero,format:invalid"` + String string `json:",omitzero,format:invalid"` + Bytes []byte `json:",omitzero,format:invalid"` + Int int64 `json:",omitzero,format:invalid"` + Uint uint64 `json:",omitzero,format:invalid"` + Float float64 `json:",omitzero,format:invalid"` + Map map[string]string `json:",omitzero,format:invalid"` + Struct structAll `json:",omitzero,format:invalid"` + Slice []string `json:",omitzero,format:invalid"` + Array [1]string `json:",omitzero,format:invalid"` + Interface any `json:",omitzero,format:invalid"` + } + structDurationFormat struct { + D1 time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + D2 time.Duration `json:",format:units"` + D3 time.Duration `json:",format:sec"` + D4 time.Duration `json:",string,format:sec"` + D5 time.Duration `json:",format:milli"` + D6 time.Duration `json:",string,format:milli"` + D7 time.Duration `json:",format:micro"` + D8 time.Duration `json:",string,format:micro"` + D9 time.Duration `json:",format:nano"` + D10 time.Duration `json:",string,format:nano"` + D11 time.Duration `json:",format:iso8601"` + } + structTimeFormat struct { + T1 time.Time + T2 time.Time `json:",format:ANSIC"` + T3 time.Time `json:",format:UnixDate"` + T4 time.Time `json:",format:RubyDate"` + T5 time.Time `json:",format:RFC822"` + T6 time.Time `json:",format:RFC822Z"` + T7 time.Time `json:",format:RFC850"` + T8 time.Time `json:",format:RFC1123"` + T9 time.Time `json:",format:RFC1123Z"` + T10 time.Time `json:",format:RFC3339"` + T11 time.Time `json:",format:RFC3339Nano"` + T12 time.Time `json:",format:Kitchen"` + T13 time.Time `json:",format:Stamp"` + T14 time.Time `json:",format:StampMilli"` + T15 time.Time `json:",format:StampMicro"` + T16 time.Time `json:",format:StampNano"` + T17 time.Time `json:",format:DateTime"` + T18 time.Time `json:",format:DateOnly"` + T19 time.Time `json:",format:TimeOnly"` + T20 time.Time `json:",format:'2006-01-02'"` + T21 time.Time `json:",format:'\"weird\"2006'"` + T22 time.Time `json:",format:unix"` + T23 time.Time `json:",string,format:unix"` + T24 time.Time `json:",format:unixmilli"` + T25 time.Time `json:",string,format:unixmilli"` + T26 time.Time `json:",format:unixmicro"` + T27 time.Time `json:",string,format:unixmicro"` + T28 time.Time `json:",format:unixnano"` + T29 time.Time `json:",string,format:unixnano"` + } + structInlined struct { + X structInlinedL1 `json:",inline"` + *StructEmbed2 // implicit inline + } + structInlinedL1 struct { + X *structInlinedL2 `json:",inline"` + StructEmbed1 `json:",inline"` + } + structInlinedL2 struct{ A, B, C string } + StructEmbed1 struct{ C, D, E string } + StructEmbed2 struct{ E, F, G string } + structUnknownTextValue struct { + A int `json:",omitzero"` + X jsontext.Value `json:",unknown"` + B int `json:",omitzero"` + } + structInlineTextValue struct { + A int `json:",omitzero"` + X jsontext.Value `json:",inline"` + B int `json:",omitzero"` + } + structInlinePointerTextValue struct { + A int `json:",omitzero"` + X *jsontext.Value `json:",inline"` + B int `json:",omitzero"` + } + structInlinePointerInlineTextValue struct { + X *struct { + A int + X jsontext.Value `json:",inline"` + } `json:",inline"` + } + structInlineInlinePointerTextValue struct { + X struct { + X *jsontext.Value `json:",inline"` + } `json:",inline"` + } + structInlineMapStringAny struct { + A int `json:",omitzero"` + X jsonObject `json:",inline"` + B int `json:",omitzero"` + } + structInlinePointerMapStringAny struct { + A int `json:",omitzero"` + X *jsonObject `json:",inline"` + B int `json:",omitzero"` + } + structInlinePointerInlineMapStringAny struct { + X *struct { + A int + X jsonObject `json:",inline"` + } `json:",inline"` + } + structInlineInlinePointerMapStringAny struct { + X struct { + X *jsonObject `json:",inline"` + } `json:",inline"` + } + structInlineMapStringInt struct { + X map[string]int `json:",inline"` + } + structInlineMapNamedStringInt struct { + X map[namedString]int `json:",inline"` + } + structInlineMapNamedStringAny struct { + A int `json:",omitzero"` + X map[namedString]any `json:",inline"` + B int `json:",omitzero"` + } + structNoCaseInlineTextValue struct { + AAA string `json:",omitempty,case:strict"` + AA_b string `json:",omitempty"` + AaA string `json:",omitempty,case:ignore"` + AAa string `json:",omitempty,case:ignore"` + Aaa string `json:",omitempty"` + X jsontext.Value `json:",inline"` + } + structNoCaseInlineMapStringAny struct { + AAA string `json:",omitempty"` + AaA string `json:",omitempty,case:ignore"` + AAa string `json:",omitempty,case:ignore"` + Aaa string `json:",omitempty"` + X jsonObject `json:",inline"` + } + + allMethods struct { + method string // the method that was called + value []byte // the raw value to provide or store + } + allMethodsExceptJSONv2 struct { + allMethods + MarshalJSONTo struct{} // cancel out MarshalJSONTo method with collision + UnmarshalJSONFrom struct{} // cancel out UnmarshalJSONFrom method with collision + } + allMethodsExceptJSONv1 struct { + allMethods + MarshalJSON struct{} // cancel out MarshalJSON method with collision + UnmarshalJSON struct{} // cancel out UnmarshalJSON method with collision + } + allMethodsExceptText struct { + allMethods + MarshalText struct{} // cancel out MarshalText method with collision + UnmarshalText struct{} // cancel out UnmarshalText method with collision + } + onlyMethodJSONv2 struct { + allMethods + MarshalJSON struct{} // cancel out MarshalJSON method with collision + UnmarshalJSON struct{} // cancel out UnmarshalJSON method with collision + MarshalText struct{} // cancel out MarshalText method with collision + UnmarshalText struct{} // cancel out UnmarshalText method with collision + } + onlyMethodJSONv1 struct { + allMethods + MarshalJSONTo struct{} // cancel out MarshalJSONTo method with collision + UnmarshalJSONFrom struct{} // cancel out UnmarshalJSONFrom method with collision + MarshalText struct{} // cancel out MarshalText method with collision + UnmarshalText struct{} // cancel out UnmarshalText method with collision + } + onlyMethodText struct { + allMethods + MarshalJSONTo struct{} // cancel out MarshalJSONTo method with collision + UnmarshalJSONFrom struct{} // cancel out UnmarshalJSONFrom method with collision + MarshalJSON struct{} // cancel out MarshalJSON method with collision + UnmarshalJSON struct{} // cancel out UnmarshalJSON method with collision + } + + structMethodJSONv2 struct{ value string } + structMethodJSONv1 struct{ value string } + structMethodText struct{ value string } + + marshalJSONv2Func func(*jsontext.Encoder) error + marshalJSONv1Func func() ([]byte, error) + appendTextFunc func([]byte) ([]byte, error) + marshalTextFunc func() ([]byte, error) + unmarshalJSONv2Func func(*jsontext.Decoder) error + unmarshalJSONv1Func func([]byte) error + unmarshalTextFunc func([]byte) error + + nocaseString string + + stringMarshalEmpty string + stringMarshalNonEmpty string + bytesMarshalEmpty []byte + bytesMarshalNonEmpty []byte + mapMarshalEmpty map[string]string + mapMarshalNonEmpty map[string]string + sliceMarshalEmpty []string + sliceMarshalNonEmpty []string + + valueAlwaysZero string + valueNeverZero string + pointerAlwaysZero string + pointerNeverZero string + + valueStringer struct{} + pointerStringer struct{} + + cyclicA struct { + B1 cyclicB `json:",inline"` + B2 cyclicB `json:",inline"` + } + cyclicB struct { + F int + A *cyclicA `json:",inline"` + } +) + +func (structUnexportedEmbeddedMethodTag) MarshalText() {} +func (structUnexportedEmbeddedMethodTag) AppendText() {} + +func (p *allMethods) MarshalJSONTo(enc *jsontext.Encoder) error { + if got, want := "MarshalJSONTo", p.method; got != want { + return fmt.Errorf("called wrong method: got %v, want %v", got, want) + } + return enc.WriteValue(p.value) +} +func (p *allMethods) MarshalJSON() ([]byte, error) { + if got, want := "MarshalJSON", p.method; got != want { + return nil, fmt.Errorf("called wrong method: got %v, want %v", got, want) + } + return p.value, nil +} +func (p *allMethods) MarshalText() ([]byte, error) { + if got, want := "MarshalText", p.method; got != want { + return nil, fmt.Errorf("called wrong method: got %v, want %v", got, want) + } + return p.value, nil +} + +func (p *allMethods) UnmarshalJSONFrom(dec *jsontext.Decoder) error { + p.method = "UnmarshalJSONFrom" + val, err := dec.ReadValue() + p.value = val + return err +} +func (p *allMethods) UnmarshalJSON(val []byte) error { + p.method = "UnmarshalJSON" + p.value = val + return nil +} +func (p *allMethods) UnmarshalText(val []byte) error { + p.method = "UnmarshalText" + p.value = val + return nil +} + +func (s structMethodJSONv2) MarshalJSONTo(enc *jsontext.Encoder) error { + return enc.WriteToken(jsontext.String(s.value)) +} +func (s *structMethodJSONv2) UnmarshalJSONFrom(dec *jsontext.Decoder) error { + tok, err := dec.ReadToken() + if err != nil { + return err + } + if k := tok.Kind(); k != '"' { + return EU(nil).withType(k, T[structMethodJSONv2]()) + } + s.value = tok.String() + return nil +} + +func (s structMethodJSONv1) MarshalJSON() ([]byte, error) { + return jsontext.AppendQuote(nil, s.value) +} +func (s *structMethodJSONv1) UnmarshalJSON(b []byte) error { + if k := jsontext.Value(b).Kind(); k != '"' { + return EU(nil).withType(k, T[structMethodJSONv1]()) + } + b, _ = jsontext.AppendUnquote(nil, b) + s.value = string(b) + return nil +} + +func (s structMethodText) MarshalText() ([]byte, error) { + return []byte(s.value), nil +} +func (s *structMethodText) UnmarshalText(b []byte) error { + s.value = string(b) + return nil +} + +func (f marshalJSONv2Func) MarshalJSONTo(enc *jsontext.Encoder) error { + return f(enc) +} +func (f marshalJSONv1Func) MarshalJSON() ([]byte, error) { + return f() +} +func (f appendTextFunc) AppendText(b []byte) ([]byte, error) { + return f(b) +} +func (f marshalTextFunc) MarshalText() ([]byte, error) { + return f() +} +func (f unmarshalJSONv2Func) UnmarshalJSONFrom(dec *jsontext.Decoder) error { + return f(dec) +} +func (f unmarshalJSONv1Func) UnmarshalJSON(b []byte) error { + return f(b) +} +func (f unmarshalTextFunc) UnmarshalText(b []byte) error { + return f(b) +} + +func (k nocaseString) MarshalText() ([]byte, error) { + return []byte(strings.ToLower(string(k))), nil +} +func (k *nocaseString) UnmarshalText(b []byte) error { + *k = nocaseString(strings.ToLower(string(b))) + return nil +} + +func (stringMarshalEmpty) MarshalJSON() ([]byte, error) { return []byte(`""`), nil } +func (stringMarshalNonEmpty) MarshalJSON() ([]byte, error) { return []byte(`"value"`), nil } +func (bytesMarshalEmpty) MarshalJSON() ([]byte, error) { return []byte(`[]`), nil } +func (bytesMarshalNonEmpty) MarshalJSON() ([]byte, error) { return []byte(`["value"]`), nil } +func (mapMarshalEmpty) MarshalJSON() ([]byte, error) { return []byte(`{}`), nil } +func (mapMarshalNonEmpty) MarshalJSON() ([]byte, error) { return []byte(`{"key":"value"}`), nil } +func (sliceMarshalEmpty) MarshalJSON() ([]byte, error) { return []byte(`[]`), nil } +func (sliceMarshalNonEmpty) MarshalJSON() ([]byte, error) { return []byte(`["value"]`), nil } + +func (valueAlwaysZero) IsZero() bool { return true } +func (valueNeverZero) IsZero() bool { return false } +func (*pointerAlwaysZero) IsZero() bool { return true } +func (*pointerNeverZero) IsZero() bool { return false } + +func (valueStringer) String() string { return "" } +func (*pointerStringer) String() string { return "" } + +func addr[T any](v T) *T { + return &v +} + +func mustParseTime(layout, value string) time.Time { + t, err := time.Parse(layout, value) + if err != nil { + panic(err) + } + return t +} + +var invalidFormatOption = &jsonopts.Struct{ + ArshalValues: jsonopts.ArshalValues{FormatDepth: 1000, Format: "invalid"}, +} + +func TestMarshal(t *testing.T) { + tests := []struct { + name jsontest.CaseName + opts []Options + in any + want string + wantErr error + + canonicalize bool // canonicalize the output before comparing? + useWriter bool // call MarshalWrite instead of Marshal + }{{ + name: jsontest.Name("Nil"), + in: nil, + want: `null`, + }, { + name: jsontest.Name("Bools"), + in: []bool{false, true}, + want: `[false,true]`, + }, { + name: jsontest.Name("Bools/Named"), + in: []namedBool{false, true}, + want: `[false,true]`, + }, { + name: jsontest.Name("Bools/NotStringified"), + opts: []Options{StringifyNumbers(true)}, + in: []bool{false, true}, + want: `[false,true]`, + }, { + name: jsontest.Name("Bools/StringifiedBool"), + opts: []Options{jsonflags.StringifyBoolsAndStrings | 1}, + in: []bool{false, true}, + want: `["false","true"]`, + }, { + name: jsontest.Name("Bools/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + in: true, + want: `true`, + }, { + name: jsontest.Name("Strings"), + in: []string{"", "hello", "世界"}, + want: `["","hello","世界"]`, + }, { + name: jsontest.Name("Strings/Named"), + in: []namedString{"", "hello", "世界"}, + want: `["","hello","世界"]`, + }, { + name: jsontest.Name("Strings/StringifiedBool"), + opts: []Options{jsonflags.StringifyBoolsAndStrings | 1}, + in: []string{"", "hello", "世界"}, + want: `["\"\"","\"hello\"","\"世界\""]`, + }, { + name: jsontest.Name("Strings/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + in: "string", + want: `"string"`, + }, { + name: jsontest.Name("Bytes"), + in: [][]byte{nil, {}, {1}, {1, 2}, {1, 2, 3}}, + want: `["","","AQ==","AQI=","AQID"]`, + }, { + name: jsontest.Name("Bytes/FormatNilSliceAsNull"), + opts: []Options{FormatNilSliceAsNull(true)}, + in: [][]byte{nil, {}}, + want: `[null,""]`, + }, { + name: jsontest.Name("Bytes/Large"), + in: []byte("the quick brown fox jumped over the lazy dog and ate the homework that I spent so much time on."), + want: `"dGhlIHF1aWNrIGJyb3duIGZveCBqdW1wZWQgb3ZlciB0aGUgbGF6eSBkb2cgYW5kIGF0ZSB0aGUgaG9tZXdvcmsgdGhhdCBJIHNwZW50IHNvIG11Y2ggdGltZSBvbi4="`, + }, { + name: jsontest.Name("Bytes/Named"), + in: []namedBytes{nil, {}, {1}, {1, 2}, {1, 2, 3}}, + want: `["","","AQ==","AQI=","AQID"]`, + }, { + name: jsontest.Name("Bytes/NotStringified"), + opts: []Options{StringifyNumbers(true)}, + in: [][]byte{nil, {}, {1}, {1, 2}, {1, 2, 3}}, + want: `["","","AQ==","AQI=","AQID"]`, + }, { + // NOTE: []namedByte is not assignable to []byte, + // so the following should be treated as a slice of uints. + name: jsontest.Name("Bytes/Invariant"), + in: [][]namedByte{nil, {}, {1}, {1, 2}, {1, 2, 3}}, + want: `[[],[],[1],[1,2],[1,2,3]]`, + }, { + // NOTE: This differs in behavior from v1, + // but keeps the representation of slices and arrays more consistent. + name: jsontest.Name("Bytes/ByteArray"), + in: [5]byte{'h', 'e', 'l', 'l', 'o'}, + want: `"aGVsbG8="`, + }, { + // NOTE: []namedByte is not assignable to []byte, + // so the following should be treated as an array of uints. + name: jsontest.Name("Bytes/NamedByteArray"), + in: [5]namedByte{'h', 'e', 'l', 'l', 'o'}, + want: `[104,101,108,108,111]`, + }, { + name: jsontest.Name("Bytes/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + in: []byte("hello"), + want: `"aGVsbG8="`, + }, { + name: jsontest.Name("Ints"), + in: []any{ + int(0), int8(math.MinInt8), int16(math.MinInt16), int32(math.MinInt32), int64(math.MinInt64), namedInt64(-6464), + }, + want: `[0,-128,-32768,-2147483648,-9223372036854775808,-6464]`, + }, { + name: jsontest.Name("Ints/Stringified"), + opts: []Options{StringifyNumbers(true)}, + in: []any{ + int(0), int8(math.MinInt8), int16(math.MinInt16), int32(math.MinInt32), int64(math.MinInt64), namedInt64(-6464), + }, + want: `["0","-128","-32768","-2147483648","-9223372036854775808","-6464"]`, + }, { + name: jsontest.Name("Ints/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + in: int(0), + want: `0`, + }, { + name: jsontest.Name("Uints"), + in: []any{ + uint(0), uint8(math.MaxUint8), uint16(math.MaxUint16), uint32(math.MaxUint32), uint64(math.MaxUint64), namedUint64(6464), uintptr(1234), + }, + want: `[0,255,65535,4294967295,18446744073709551615,6464,1234]`, + }, { + name: jsontest.Name("Uints/Stringified"), + opts: []Options{StringifyNumbers(true)}, + in: []any{ + uint(0), uint8(math.MaxUint8), uint16(math.MaxUint16), uint32(math.MaxUint32), uint64(math.MaxUint64), namedUint64(6464), + }, + want: `["0","255","65535","4294967295","18446744073709551615","6464"]`, + }, { + name: jsontest.Name("Uints/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + in: uint(0), + want: `0`, + }, { + name: jsontest.Name("Floats"), + in: []any{ + float32(math.MaxFloat32), float64(math.MaxFloat64), namedFloat64(64.64), + }, + want: `[3.4028235e+38,1.7976931348623157e+308,64.64]`, + }, { + name: jsontest.Name("Floats/Stringified"), + opts: []Options{StringifyNumbers(true)}, + in: []any{ + float32(math.MaxFloat32), float64(math.MaxFloat64), namedFloat64(64.64), + }, + want: `["3.4028235e+38","1.7976931348623157e+308","64.64"]`, + }, { + name: jsontest.Name("Floats/Invalid/NaN"), + opts: []Options{StringifyNumbers(true)}, + in: math.NaN(), + wantErr: EM(fmt.Errorf("unsupported value: %v", math.NaN())).withType(0, float64Type), + }, { + name: jsontest.Name("Floats/Invalid/PositiveInfinity"), + in: math.Inf(+1), + wantErr: EM(fmt.Errorf("unsupported value: %v", math.Inf(+1))).withType(0, float64Type), + }, { + name: jsontest.Name("Floats/Invalid/NegativeInfinity"), + in: math.Inf(-1), + wantErr: EM(fmt.Errorf("unsupported value: %v", math.Inf(-1))).withType(0, float64Type), + }, { + name: jsontest.Name("Floats/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + in: float64(0), + want: `0`, + }, { + name: jsontest.Name("Maps/InvalidKey/Bool"), + in: map[bool]string{false: "value"}, + want: `{`, + wantErr: EM(newNonStringNameError(len64(`{`), "")).withPos(`{`, "").withType(0, boolType), + }, { + name: jsontest.Name("Maps/InvalidKey/NamedBool"), + in: map[namedBool]string{false: "value"}, + want: `{`, + wantErr: EM(newNonStringNameError(len64(`{`), "")).withPos(`{`, "").withType(0, T[namedBool]()), + }, { + name: jsontest.Name("Maps/InvalidKey/Array"), + in: map[[1]string]string{{"key"}: "value"}, + want: `{`, + wantErr: EM(newNonStringNameError(len64(`{`), "")).withPos(`{`, "").withType(0, T[[1]string]()), + }, { + name: jsontest.Name("Maps/InvalidKey/Channel"), + in: map[chan string]string{make(chan string): "value"}, + want: `{`, + wantErr: EM(nil).withPos(`{`, "").withType(0, T[chan string]()), + }, { + name: jsontest.Name("Maps/ValidKey/Int"), + in: map[int64]string{math.MinInt64: "MinInt64", 0: "Zero", math.MaxInt64: "MaxInt64"}, + canonicalize: true, + want: `{"-9223372036854775808":"MinInt64","0":"Zero","9223372036854775807":"MaxInt64"}`, + }, { + name: jsontest.Name("Maps/ValidKey/PointerInt"), + in: map[*int64]string{addr(int64(math.MinInt64)): "MinInt64", addr(int64(0)): "Zero", addr(int64(math.MaxInt64)): "MaxInt64"}, + canonicalize: true, + want: `{"-9223372036854775808":"MinInt64","0":"Zero","9223372036854775807":"MaxInt64"}`, + }, { + name: jsontest.Name("Maps/DuplicateName/PointerInt"), + in: map[*int64]string{addr(int64(0)): "0", addr(int64(0)): "0"}, + canonicalize: true, + want: `{"0":"0"`, + wantErr: newDuplicateNameError("", []byte(`"0"`), len64(`{"0":"0",`)), + }, { + name: jsontest.Name("Maps/ValidKey/NamedInt"), + in: map[namedInt64]string{math.MinInt64: "MinInt64", 0: "Zero", math.MaxInt64: "MaxInt64"}, + canonicalize: true, + want: `{"-9223372036854775808":"MinInt64","0":"Zero","9223372036854775807":"MaxInt64"}`, + }, { + name: jsontest.Name("Maps/ValidKey/Uint"), + in: map[uint64]string{0: "Zero", math.MaxUint64: "MaxUint64"}, + canonicalize: true, + want: `{"0":"Zero","18446744073709551615":"MaxUint64"}`, + }, { + name: jsontest.Name("Maps/ValidKey/NamedUint"), + in: map[namedUint64]string{0: "Zero", math.MaxUint64: "MaxUint64"}, + canonicalize: true, + want: `{"0":"Zero","18446744073709551615":"MaxUint64"}`, + }, { + name: jsontest.Name("Maps/ValidKey/Float"), + in: map[float64]string{3.14159: "value"}, + want: `{"3.14159":"value"}`, + }, { + name: jsontest.Name("Maps/InvalidKey/Float/NaN"), + in: map[float64]string{math.NaN(): "NaN", math.NaN(): "NaN"}, + want: `{`, + wantErr: EM(errors.New("unsupported value: NaN")).withPos(`{`, "").withType(0, float64Type), + }, { + name: jsontest.Name("Maps/ValidKey/Interface"), + in: map[any]any{ + "key": "key", + namedInt64(-64): int32(-32), + namedUint64(+64): uint32(+32), + namedFloat64(64.64): float32(32.32), + }, + canonicalize: true, + want: `{"-64":-32,"64":32,"64.64":32.32,"key":"key"}`, + }, { + name: jsontest.Name("Maps/DuplicateName/String/AllowInvalidUTF8+AllowDuplicateNames"), + opts: []Options{jsontext.AllowInvalidUTF8(true), jsontext.AllowDuplicateNames(true)}, + in: map[string]string{"\x80": "", "\x81": ""}, + want: `{"�":"","�":""}`, + }, { + name: jsontest.Name("Maps/DuplicateName/String/AllowInvalidUTF8"), + opts: []Options{jsontext.AllowInvalidUTF8(true)}, + in: map[string]string{"\x80": "", "\x81": ""}, + want: `{"�":""`, + wantErr: newDuplicateNameError("", []byte(`"�"`), len64(`{"�":"",`)), + }, { + name: jsontest.Name("Maps/DuplicateName/NoCaseString/AllowDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(true)}, + in: map[nocaseString]string{"hello": "", "HELLO": ""}, + want: `{"hello":"","hello":""}`, + }, { + name: jsontest.Name("Maps/DuplicateName/NoCaseString"), + in: map[nocaseString]string{"hello": "", "HELLO": ""}, + want: `{"hello":""`, + wantErr: EM(newDuplicateNameError("", []byte(`"hello"`), len64(`{"hello":"",`))).withPos(`{"hello":"",`, "").withType(0, T[nocaseString]()), + }, { + name: jsontest.Name("Maps/DuplicateName/NaNs/Deterministic+AllowDuplicateNames"), + opts: []Options{ + WithMarshalers( + MarshalFunc(func(v float64) ([]byte, error) { return []byte(`"NaN"`), nil }), + ), + Deterministic(true), + jsontext.AllowDuplicateNames(true), + }, + in: map[float64]string{math.NaN(): "NaN", math.NaN(): "NaN"}, + want: `{"NaN":"NaN","NaN":"NaN"}`, + }, { + name: jsontest.Name("Maps/InvalidValue/Channel"), + in: map[string]chan string{ + "key": nil, + }, + want: `{"key"`, + wantErr: EM(nil).withPos(`{"key":`, "/key").withType(0, T[chan string]()), + }, { + name: jsontest.Name("Maps/String/Deterministic"), + opts: []Options{Deterministic(true)}, + in: map[string]int{"a": 0, "b": 1, "c": 2}, + want: `{"a":0,"b":1,"c":2}`, + }, { + name: jsontest.Name("Maps/String/Deterministic+AllowInvalidUTF8+RejectDuplicateNames"), + opts: []Options{ + Deterministic(true), + jsontext.AllowInvalidUTF8(true), + jsontext.AllowDuplicateNames(false), + }, + in: map[string]int{"\xff": 0, "\xfe": 1}, + want: `{"�":1`, + wantErr: newDuplicateNameError("", []byte(`"�"`), len64(`{"�":1,`)), + }, { + name: jsontest.Name("Maps/String/Deterministic+AllowInvalidUTF8+AllowDuplicateNames"), + opts: []Options{ + Deterministic(true), + jsontext.AllowInvalidUTF8(true), + jsontext.AllowDuplicateNames(true), + }, + in: map[string]int{"\xff": 0, "\xfe": 1}, + want: `{"�":1,"�":0}`, + }, { + name: jsontest.Name("Maps/String/Deterministic+MarshalFuncs"), + opts: []Options{ + Deterministic(true), + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v string) error { + if p := enc.StackPointer(); p != "/X" { + return fmt.Errorf("invalid stack pointer: got %s, want /X", p) + } + switch v { + case "a": + return enc.WriteToken(jsontext.String("b")) + case "b": + return enc.WriteToken(jsontext.String("a")) + default: + return fmt.Errorf("invalid value: %q", v) + } + })), + }, + in: map[namedString]map[string]int{"X": {"a": -1, "b": 1}}, + want: `{"X":{"a":1,"b":-1}}`, + }, { + name: jsontest.Name("Maps/String/Deterministic+MarshalFuncs+RejectDuplicateNames"), + opts: []Options{ + Deterministic(true), + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v string) error { + if p := enc.StackPointer(); p != "/X" { + return fmt.Errorf("invalid stack pointer: got %s, want /X", p) + } + switch v { + case "a", "b": + return enc.WriteToken(jsontext.String("x")) + default: + return fmt.Errorf("invalid value: %q", v) + } + })), + jsontext.AllowDuplicateNames(false), + }, + in: map[namedString]map[string]int{"X": {"a": 1, "b": 1}}, + want: `{"X":{"x":1`, + wantErr: newDuplicateNameError("/X/x", nil, len64(`{"X":{"x":1,`)), + }, { + name: jsontest.Name("Maps/String/Deterministic+MarshalFuncs+AllowDuplicateNames"), + opts: []Options{ + Deterministic(true), + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v string) error { + if p := enc.StackPointer(); p != "/X" { + return fmt.Errorf("invalid stack pointer: got %s, want /0", p) + } + switch v { + case "a", "b": + return enc.WriteToken(jsontext.String("x")) + default: + return fmt.Errorf("invalid value: %q", v) + } + })), + jsontext.AllowDuplicateNames(true), + }, + in: map[namedString]map[string]int{"X": {"a": 1, "b": 1}}, + // NOTE: Since the names are identical, the exact values may be + // non-deterministic since sort cannot distinguish between members. + want: `{"X":{"x":1,"x":1}}`, + }, { + name: jsontest.Name("Maps/RecursiveMap"), + in: recursiveMap{ + "fizz": { + "foo": {}, + "bar": nil, + }, + "buzz": nil, + }, + canonicalize: true, + want: `{"buzz":{},"fizz":{"bar":{},"foo":{}}}`, + }, { + name: jsontest.Name("Maps/CyclicMap"), + in: func() recursiveMap { + m := recursiveMap{"k": nil} + m["k"] = m + return m + }(), + want: strings.Repeat(`{"k":`, startDetectingCyclesAfter) + `{"k"`, + wantErr: EM(internal.ErrCycle).withPos(strings.Repeat(`{"k":`, startDetectingCyclesAfter+1), jsontext.Pointer(strings.Repeat("/k", startDetectingCyclesAfter+1))).withType(0, T[recursiveMap]()), + }, { + name: jsontest.Name("Maps/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + in: map[string]string{}, + want: `{}`, + }, { + name: jsontest.Name("Structs/Empty"), + in: structEmpty{}, + want: `{}`, + }, { + name: jsontest.Name("Structs/UnexportedIgnored"), + in: structUnexportedIgnored{ignored: "ignored"}, + want: `{}`, + }, { + name: jsontest.Name("Structs/IgnoredUnexportedEmbedded"), + in: structIgnoredUnexportedEmbedded{namedString: "ignored"}, + want: `{}`, + }, { + name: jsontest.Name("Structs/WeirdNames"), + in: structWeirdNames{Empty: "empty", Comma: "comma", Quote: "quote"}, + want: `{"":"empty",",":"comma","\"":"quote"}`, + }, { + name: jsontest.Name("Structs/EscapedNames"), + opts: []Options{jsontext.EscapeForHTML(true), jsontext.EscapeForJS(true)}, + in: struct { + S string "json:\"'abc<>&\u2028\u2029xyz'\"" + M any + I structInlineTextValue + }{ + S: "abc<>&\u2028\u2029xyz", + M: map[string]string{"abc<>&\u2028\u2029xyz": "abc<>&\u2028\u2029xyz"}, + I: structInlineTextValue{X: jsontext.Value(`{"abc<>&` + "\u2028\u2029" + `xyz":"abc<>&` + "\u2028\u2029" + `xyz"}`)}, + }, + want: `{"abc\u003c\u003e\u0026\u2028\u2029xyz":"abc\u003c\u003e\u0026\u2028\u2029xyz","M":{"abc\u003c\u003e\u0026\u2028\u2029xyz":"abc\u003c\u003e\u0026\u2028\u2029xyz"},"I":{"abc\u003c\u003e\u0026\u2028\u2029xyz":"abc\u003c\u003e\u0026\u2028\u2029xyz"}}`, + }, { + name: jsontest.Name("Structs/NoCase"), + in: structNoCase{AaA: "AaA", AAa: "AAa", Aaa: "Aaa", AAA: "AAA", AA_A: "AA_A"}, + want: `{"Aaa":"Aaa","AA_A":"AA_A","AaA":"AaA","AAa":"AAa","AAA":"AAA"}`, + }, { + name: jsontest.Name("Structs/NoCase/MatchCaseInsensitiveNames"), + opts: []Options{MatchCaseInsensitiveNames(true)}, + in: structNoCase{AaA: "AaA", AAa: "AAa", Aaa: "Aaa", AAA: "AAA", AA_A: "AA_A"}, + want: `{"Aaa":"Aaa","AA_A":"AA_A","AaA":"AaA","AAa":"AAa","AAA":"AAA"}`, + }, { + name: jsontest.Name("Structs/NoCase/MatchCaseInsensitiveNames+MatchCaseSensitiveDelimiter"), + opts: []Options{MatchCaseInsensitiveNames(true), jsonflags.MatchCaseSensitiveDelimiter | 1}, + in: structNoCase{AaA: "AaA", AAa: "AAa", Aaa: "Aaa", AAA: "AAA", AA_A: "AA_A"}, + want: `{"Aaa":"Aaa","AA_A":"AA_A","AaA":"AaA","AAa":"AAa","AAA":"AAA"}`, + }, { + name: jsontest.Name("Structs/Normal"), + opts: []Options{jsontext.Multiline(true)}, + in: structAll{ + Bool: true, + String: "hello", + Bytes: []byte{1, 2, 3}, + Int: -64, + Uint: +64, + Float: 3.14159, + Map: map[string]string{"key": "value"}, + StructScalars: structScalars{ + Bool: true, + String: "hello", + Bytes: []byte{1, 2, 3}, + Int: -64, + Uint: +64, + Float: 3.14159, + }, + StructMaps: structMaps{ + MapBool: map[string]bool{"": true}, + MapString: map[string]string{"": "hello"}, + MapBytes: map[string][]byte{"": {1, 2, 3}}, + MapInt: map[string]int64{"": -64}, + MapUint: map[string]uint64{"": +64}, + MapFloat: map[string]float64{"": 3.14159}, + }, + StructSlices: structSlices{ + SliceBool: []bool{true}, + SliceString: []string{"hello"}, + SliceBytes: [][]byte{{1, 2, 3}}, + SliceInt: []int64{-64}, + SliceUint: []uint64{+64}, + SliceFloat: []float64{3.14159}, + }, + Slice: []string{"fizz", "buzz"}, + Array: [1]string{"goodbye"}, + Pointer: new(structAll), + Interface: (*structAll)(nil), + }, + want: `{ + "Bool": true, + "String": "hello", + "Bytes": "AQID", + "Int": -64, + "Uint": 64, + "Float": 3.14159, + "Map": { + "key": "value" + }, + "StructScalars": { + "Bool": true, + "String": "hello", + "Bytes": "AQID", + "Int": -64, + "Uint": 64, + "Float": 3.14159 + }, + "StructMaps": { + "MapBool": { + "": true + }, + "MapString": { + "": "hello" + }, + "MapBytes": { + "": "AQID" + }, + "MapInt": { + "": -64 + }, + "MapUint": { + "": 64 + }, + "MapFloat": { + "": 3.14159 + } + }, + "StructSlices": { + "SliceBool": [ + true + ], + "SliceString": [ + "hello" + ], + "SliceBytes": [ + "AQID" + ], + "SliceInt": [ + -64 + ], + "SliceUint": [ + 64 + ], + "SliceFloat": [ + 3.14159 + ] + }, + "Slice": [ + "fizz", + "buzz" + ], + "Array": [ + "goodbye" + ], + "Pointer": { + "Bool": false, + "String": "", + "Bytes": "", + "Int": 0, + "Uint": 0, + "Float": 0, + "Map": {}, + "StructScalars": { + "Bool": false, + "String": "", + "Bytes": "", + "Int": 0, + "Uint": 0, + "Float": 0 + }, + "StructMaps": { + "MapBool": {}, + "MapString": {}, + "MapBytes": {}, + "MapInt": {}, + "MapUint": {}, + "MapFloat": {} + }, + "StructSlices": { + "SliceBool": [], + "SliceString": [], + "SliceBytes": [], + "SliceInt": [], + "SliceUint": [], + "SliceFloat": [] + }, + "Slice": [], + "Array": [ + "" + ], + "Pointer": null, + "Interface": null + }, + "Interface": null +}`, + }, { + name: jsontest.Name("Structs/SpaceAfterColonAndComma"), + opts: []Options{jsontext.SpaceAfterColon(true), jsontext.SpaceAfterComma(true)}, + in: structOmitZeroAll{Int: 1, Uint: 1}, + want: `{"Int": 1, "Uint": 1}`, + }, { + name: jsontest.Name("Structs/SpaceAfterColon"), + opts: []Options{jsontext.SpaceAfterColon(true)}, + in: structOmitZeroAll{Int: 1, Uint: 1}, + want: `{"Int": 1,"Uint": 1}`, + }, { + name: jsontest.Name("Structs/SpaceAfterComma"), + opts: []Options{jsontext.SpaceAfterComma(true)}, + in: structOmitZeroAll{Int: 1, Uint: 1, Slice: []string{"a", "b"}}, + want: `{"Int":1, "Uint":1, "Slice":["a", "b"]}`, + }, { + name: jsontest.Name("Structs/Stringified"), + opts: []Options{jsontext.Multiline(true)}, + in: structStringifiedAll{ + Bool: true, + String: "hello", + Bytes: []byte{1, 2, 3}, + Int: -64, // should be stringified + Uint: +64, // should be stringified + Float: 3.14159, // should be stringified + Map: map[string]string{"key": "value"}, + StructScalars: structScalars{ + Bool: true, + String: "hello", + Bytes: []byte{1, 2, 3}, + Int: -64, // should be stringified + Uint: +64, // should be stringified + Float: 3.14159, // should be stringified + }, + StructMaps: structMaps{ + MapBool: map[string]bool{"": true}, + MapString: map[string]string{"": "hello"}, + MapBytes: map[string][]byte{"": {1, 2, 3}}, + MapInt: map[string]int64{"": -64}, // should be stringified + MapUint: map[string]uint64{"": +64}, // should be stringified + MapFloat: map[string]float64{"": 3.14159}, // should be stringified + }, + StructSlices: structSlices{ + SliceBool: []bool{true}, + SliceString: []string{"hello"}, + SliceBytes: [][]byte{{1, 2, 3}}, + SliceInt: []int64{-64}, // should be stringified + SliceUint: []uint64{+64}, // should be stringified + SliceFloat: []float64{3.14159}, // should be stringified + }, + Slice: []string{"fizz", "buzz"}, + Array: [1]string{"goodbye"}, + Pointer: new(structStringifiedAll), // should be stringified + Interface: (*structStringifiedAll)(nil), + }, + want: `{ + "Bool": true, + "String": "hello", + "Bytes": "AQID", + "Int": "-64", + "Uint": "64", + "Float": "3.14159", + "Map": { + "key": "value" + }, + "StructScalars": { + "Bool": true, + "String": "hello", + "Bytes": "AQID", + "Int": "-64", + "Uint": "64", + "Float": "3.14159" + }, + "StructMaps": { + "MapBool": { + "": true + }, + "MapString": { + "": "hello" + }, + "MapBytes": { + "": "AQID" + }, + "MapInt": { + "": "-64" + }, + "MapUint": { + "": "64" + }, + "MapFloat": { + "": "3.14159" + } + }, + "StructSlices": { + "SliceBool": [ + true + ], + "SliceString": [ + "hello" + ], + "SliceBytes": [ + "AQID" + ], + "SliceInt": [ + "-64" + ], + "SliceUint": [ + "64" + ], + "SliceFloat": [ + "3.14159" + ] + }, + "Slice": [ + "fizz", + "buzz" + ], + "Array": [ + "goodbye" + ], + "Pointer": { + "Bool": false, + "String": "", + "Bytes": "", + "Int": "0", + "Uint": "0", + "Float": "0", + "Map": {}, + "StructScalars": { + "Bool": false, + "String": "", + "Bytes": "", + "Int": "0", + "Uint": "0", + "Float": "0" + }, + "StructMaps": { + "MapBool": {}, + "MapString": {}, + "MapBytes": {}, + "MapInt": {}, + "MapUint": {}, + "MapFloat": {} + }, + "StructSlices": { + "SliceBool": [], + "SliceString": [], + "SliceBytes": [], + "SliceInt": [], + "SliceUint": [], + "SliceFloat": [] + }, + "Slice": [], + "Array": [ + "" + ], + "Pointer": null, + "Interface": null + }, + "Interface": null +}`, + }, { + name: jsontest.Name("Structs/LegacyStringified"), + opts: []Options{jsontext.Multiline(true), jsonflags.StringifyWithLegacySemantics | 1}, + in: structStringifiedAll{ + Bool: true, // should be stringified + String: "hello", // should be stringified + Bytes: []byte{1, 2, 3}, + Int: -64, // should be stringified + Uint: +64, // should be stringified + Float: 3.14159, // should be stringified + Map: map[string]string{"key": "value"}, + StructScalars: structScalars{ + Bool: true, + String: "hello", + Bytes: []byte{1, 2, 3}, + Int: -64, + Uint: +64, + Float: 3.14159, + }, + StructMaps: structMaps{ + MapBool: map[string]bool{"": true}, + MapString: map[string]string{"": "hello"}, + MapBytes: map[string][]byte{"": {1, 2, 3}}, + MapInt: map[string]int64{"": -64}, + MapUint: map[string]uint64{"": +64}, + MapFloat: map[string]float64{"": 3.14159}, + }, + StructSlices: structSlices{ + SliceBool: []bool{true}, + SliceString: []string{"hello"}, + SliceBytes: [][]byte{{1, 2, 3}}, + SliceInt: []int64{-64}, + SliceUint: []uint64{+64}, + SliceFloat: []float64{3.14159}, + }, + Slice: []string{"fizz", "buzz"}, + Array: [1]string{"goodbye"}, + Pointer: new(structStringifiedAll), // should be stringified + Interface: (*structStringifiedAll)(nil), + }, + want: `{ + "Bool": "true", + "String": "\"hello\"", + "Bytes": "AQID", + "Int": "-64", + "Uint": "64", + "Float": "3.14159", + "Map": { + "key": "value" + }, + "StructScalars": { + "Bool": true, + "String": "hello", + "Bytes": "AQID", + "Int": -64, + "Uint": 64, + "Float": 3.14159 + }, + "StructMaps": { + "MapBool": { + "": true + }, + "MapString": { + "": "hello" + }, + "MapBytes": { + "": "AQID" + }, + "MapInt": { + "": -64 + }, + "MapUint": { + "": 64 + }, + "MapFloat": { + "": 3.14159 + } + }, + "StructSlices": { + "SliceBool": [ + true + ], + "SliceString": [ + "hello" + ], + "SliceBytes": [ + "AQID" + ], + "SliceInt": [ + -64 + ], + "SliceUint": [ + 64 + ], + "SliceFloat": [ + 3.14159 + ] + }, + "Slice": [ + "fizz", + "buzz" + ], + "Array": [ + "goodbye" + ], + "Pointer": { + "Bool": "false", + "String": "\"\"", + "Bytes": "", + "Int": "0", + "Uint": "0", + "Float": "0", + "Map": {}, + "StructScalars": { + "Bool": false, + "String": "", + "Bytes": "", + "Int": 0, + "Uint": 0, + "Float": 0 + }, + "StructMaps": { + "MapBool": {}, + "MapString": {}, + "MapBytes": {}, + "MapInt": {}, + "MapUint": {}, + "MapFloat": {} + }, + "StructSlices": { + "SliceBool": [], + "SliceString": [], + "SliceBytes": [], + "SliceInt": [], + "SliceUint": [], + "SliceFloat": [] + }, + "Slice": [], + "Array": [ + "" + ], + "Pointer": null, + "Interface": null + }, + "Interface": null +}`, + }, { + name: jsontest.Name("Structs/OmitZero/Zero"), + in: structOmitZeroAll{}, + want: `{}`, + }, { + name: jsontest.Name("Structs/OmitZeroOption/Zero"), + opts: []Options{OmitZeroStructFields(true)}, + in: structAll{}, + want: `{}`, + }, { + name: jsontest.Name("Structs/OmitZero/NonZero"), + opts: []Options{jsontext.Multiline(true)}, + in: structOmitZeroAll{ + Bool: true, // not omitted since true is non-zero + String: " ", // not omitted since non-empty string is non-zero + Bytes: []byte{}, // not omitted since allocated slice is non-zero + Int: 1, // not omitted since 1 is non-zero + Uint: 1, // not omitted since 1 is non-zero + Float: math.SmallestNonzeroFloat64, // not omitted since still slightly above zero + Map: map[string]string{}, // not omitted since allocated map is non-zero + StructScalars: structScalars{unexported: true}, // not omitted since unexported is non-zero + StructSlices: structSlices{Ignored: true}, // not omitted since Ignored is non-zero + StructMaps: structMaps{MapBool: map[string]bool{}}, // not omitted since MapBool is non-zero + Slice: []string{}, // not omitted since allocated slice is non-zero + Array: [1]string{" "}, // not omitted since single array element is non-zero + Pointer: new(structOmitZeroAll), // not omitted since pointer is non-zero (even if all fields of the struct value are zero) + Interface: (*structOmitZeroAll)(nil), // not omitted since interface value is non-zero (even if interface value is a nil pointer) + }, + want: `{ + "Bool": true, + "String": " ", + "Bytes": "", + "Int": 1, + "Uint": 1, + "Float": 5e-324, + "Map": {}, + "StructScalars": { + "Bool": false, + "String": "", + "Bytes": "", + "Int": 0, + "Uint": 0, + "Float": 0 + }, + "StructMaps": { + "MapBool": {}, + "MapString": {}, + "MapBytes": {}, + "MapInt": {}, + "MapUint": {}, + "MapFloat": {} + }, + "StructSlices": { + "SliceBool": [], + "SliceString": [], + "SliceBytes": [], + "SliceInt": [], + "SliceUint": [], + "SliceFloat": [] + }, + "Slice": [], + "Array": [ + " " + ], + "Pointer": {}, + "Interface": null +}`, + }, { + name: jsontest.Name("Structs/OmitZeroOption/NonZero"), + opts: []Options{OmitZeroStructFields(true), jsontext.Multiline(true)}, + in: structAll{ + Bool: true, + String: " ", + Bytes: []byte{}, + Int: 1, + Uint: 1, + Float: math.SmallestNonzeroFloat64, + Map: map[string]string{}, + StructScalars: structScalars{unexported: true}, + StructSlices: structSlices{Ignored: true}, + StructMaps: structMaps{MapBool: map[string]bool{}}, + Slice: []string{}, + Array: [1]string{" "}, + Pointer: new(structAll), + Interface: (*structAll)(nil), + }, + want: `{ + "Bool": true, + "String": " ", + "Bytes": "", + "Int": 1, + "Uint": 1, + "Float": 5e-324, + "Map": {}, + "StructScalars": {}, + "StructMaps": { + "MapBool": {} + }, + "StructSlices": {}, + "Slice": [], + "Array": [ + " " + ], + "Pointer": {}, + "Interface": null +}`, + }, { + name: jsontest.Name("Structs/OmitZeroMethod/Zero"), + in: structOmitZeroMethodAll{}, + want: `{"ValueNeverZero":"","PointerNeverZero":""}`, + }, { + name: jsontest.Name("Structs/OmitZeroMethod/NonZero"), + opts: []Options{jsontext.Multiline(true)}, + in: structOmitZeroMethodAll{ + ValueAlwaysZero: valueAlwaysZero("nonzero"), + ValueNeverZero: valueNeverZero("nonzero"), + PointerAlwaysZero: pointerAlwaysZero("nonzero"), + PointerNeverZero: pointerNeverZero("nonzero"), + PointerValueAlwaysZero: addr(valueAlwaysZero("nonzero")), + PointerValueNeverZero: addr(valueNeverZero("nonzero")), + PointerPointerAlwaysZero: addr(pointerAlwaysZero("nonzero")), + PointerPointerNeverZero: addr(pointerNeverZero("nonzero")), + PointerPointerValueAlwaysZero: addr(addr(valueAlwaysZero("nonzero"))), // marshaled since **valueAlwaysZero does not implement IsZero + PointerPointerValueNeverZero: addr(addr(valueNeverZero("nonzero"))), + PointerPointerPointerAlwaysZero: addr(addr(pointerAlwaysZero("nonzero"))), // marshaled since **pointerAlwaysZero does not implement IsZero + PointerPointerPointerNeverZero: addr(addr(pointerNeverZero("nonzero"))), + }, + want: `{ + "ValueNeverZero": "nonzero", + "PointerNeverZero": "nonzero", + "PointerValueNeverZero": "nonzero", + "PointerPointerNeverZero": "nonzero", + "PointerPointerValueAlwaysZero": "nonzero", + "PointerPointerValueNeverZero": "nonzero", + "PointerPointerPointerAlwaysZero": "nonzero", + "PointerPointerPointerNeverZero": "nonzero" +}`, + }, { + name: jsontest.Name("Structs/OmitZeroMethod/Interface/Zero"), + opts: []Options{jsontext.Multiline(true)}, + in: structOmitZeroMethodInterfaceAll{}, + want: `{}`, + }, { + name: jsontest.Name("Structs/OmitZeroMethod/Interface/PartialZero"), + opts: []Options{jsontext.Multiline(true)}, + in: structOmitZeroMethodInterfaceAll{ + ValueAlwaysZero: valueAlwaysZero(""), + ValueNeverZero: valueNeverZero(""), + PointerValueAlwaysZero: (*valueAlwaysZero)(nil), + PointerValueNeverZero: (*valueNeverZero)(nil), // nil pointer, so method not called + PointerPointerAlwaysZero: (*pointerAlwaysZero)(nil), + PointerPointerNeverZero: (*pointerNeverZero)(nil), // nil pointer, so method not called + }, + want: `{ + "ValueNeverZero": "" +}`, + }, { + name: jsontest.Name("Structs/OmitZeroMethod/Interface/NonZero"), + opts: []Options{jsontext.Multiline(true)}, + in: structOmitZeroMethodInterfaceAll{ + ValueAlwaysZero: valueAlwaysZero("nonzero"), + ValueNeverZero: valueNeverZero("nonzero"), + PointerValueAlwaysZero: addr(valueAlwaysZero("nonzero")), + PointerValueNeverZero: addr(valueNeverZero("nonzero")), + PointerPointerAlwaysZero: addr(pointerAlwaysZero("nonzero")), + PointerPointerNeverZero: addr(pointerNeverZero("nonzero")), + }, + want: `{ + "ValueNeverZero": "nonzero", + "PointerValueNeverZero": "nonzero", + "PointerPointerNeverZero": "nonzero" +}`, + }, { + name: jsontest.Name("Structs/OmitEmpty/Zero"), + opts: []Options{jsontext.Multiline(true)}, + in: structOmitEmptyAll{}, + want: `{ + "Bool": false, + "StringNonEmpty": "value", + "BytesNonEmpty": [ + "value" + ], + "Float": 0, + "MapNonEmpty": { + "key": "value" + }, + "SliceNonEmpty": [ + "value" + ] +}`, + }, { + name: jsontest.Name("Structs/OmitEmpty/EmptyNonZero"), + opts: []Options{jsontext.Multiline(true)}, + in: structOmitEmptyAll{ + String: string(""), + StringEmpty: stringMarshalEmpty(""), + StringNonEmpty: stringMarshalNonEmpty(""), + PointerString: addr(string("")), + PointerStringEmpty: addr(stringMarshalEmpty("")), + PointerStringNonEmpty: addr(stringMarshalNonEmpty("")), + Bytes: []byte(""), + BytesEmpty: bytesMarshalEmpty([]byte("")), + BytesNonEmpty: bytesMarshalNonEmpty([]byte("")), + PointerBytes: addr([]byte("")), + PointerBytesEmpty: addr(bytesMarshalEmpty([]byte(""))), + PointerBytesNonEmpty: addr(bytesMarshalNonEmpty([]byte(""))), + Map: map[string]string{}, + MapEmpty: mapMarshalEmpty{}, + MapNonEmpty: mapMarshalNonEmpty{}, + PointerMap: addr(map[string]string{}), + PointerMapEmpty: addr(mapMarshalEmpty{}), + PointerMapNonEmpty: addr(mapMarshalNonEmpty{}), + Slice: []string{}, + SliceEmpty: sliceMarshalEmpty{}, + SliceNonEmpty: sliceMarshalNonEmpty{}, + PointerSlice: addr([]string{}), + PointerSliceEmpty: addr(sliceMarshalEmpty{}), + PointerSliceNonEmpty: addr(sliceMarshalNonEmpty{}), + Pointer: &structOmitZeroEmptyAll{}, + Interface: []string{}, + }, + want: `{ + "Bool": false, + "StringNonEmpty": "value", + "PointerStringNonEmpty": "value", + "BytesNonEmpty": [ + "value" + ], + "PointerBytesNonEmpty": [ + "value" + ], + "Float": 0, + "MapNonEmpty": { + "key": "value" + }, + "PointerMapNonEmpty": { + "key": "value" + }, + "SliceNonEmpty": [ + "value" + ], + "PointerSliceNonEmpty": [ + "value" + ] +}`, + }, { + name: jsontest.Name("Structs/OmitEmpty/NonEmpty"), + opts: []Options{jsontext.Multiline(true)}, + in: structOmitEmptyAll{ + Bool: true, + PointerBool: addr(true), + String: string("value"), + StringEmpty: stringMarshalEmpty("value"), + StringNonEmpty: stringMarshalNonEmpty("value"), + PointerString: addr(string("value")), + PointerStringEmpty: addr(stringMarshalEmpty("value")), + PointerStringNonEmpty: addr(stringMarshalNonEmpty("value")), + Bytes: []byte("value"), + BytesEmpty: bytesMarshalEmpty([]byte("value")), + BytesNonEmpty: bytesMarshalNonEmpty([]byte("value")), + PointerBytes: addr([]byte("value")), + PointerBytesEmpty: addr(bytesMarshalEmpty([]byte("value"))), + PointerBytesNonEmpty: addr(bytesMarshalNonEmpty([]byte("value"))), + Float: math.Copysign(0, -1), + PointerFloat: addr(math.Copysign(0, -1)), + Map: map[string]string{"": ""}, + MapEmpty: mapMarshalEmpty{"key": "value"}, + MapNonEmpty: mapMarshalNonEmpty{"key": "value"}, + PointerMap: addr(map[string]string{"": ""}), + PointerMapEmpty: addr(mapMarshalEmpty{"key": "value"}), + PointerMapNonEmpty: addr(mapMarshalNonEmpty{"key": "value"}), + Slice: []string{""}, + SliceEmpty: sliceMarshalEmpty{"value"}, + SliceNonEmpty: sliceMarshalNonEmpty{"value"}, + PointerSlice: addr([]string{""}), + PointerSliceEmpty: addr(sliceMarshalEmpty{"value"}), + PointerSliceNonEmpty: addr(sliceMarshalNonEmpty{"value"}), + Pointer: &structOmitZeroEmptyAll{Float: math.SmallestNonzeroFloat64}, + Interface: []string{""}, + }, + want: `{ + "Bool": true, + "PointerBool": true, + "String": "value", + "StringNonEmpty": "value", + "PointerString": "value", + "PointerStringNonEmpty": "value", + "Bytes": "dmFsdWU=", + "BytesNonEmpty": [ + "value" + ], + "PointerBytes": "dmFsdWU=", + "PointerBytesNonEmpty": [ + "value" + ], + "Float": -0, + "PointerFloat": -0, + "Map": { + "": "" + }, + "MapNonEmpty": { + "key": "value" + }, + "PointerMap": { + "": "" + }, + "PointerMapNonEmpty": { + "key": "value" + }, + "Slice": [ + "" + ], + "SliceNonEmpty": [ + "value" + ], + "PointerSlice": [ + "" + ], + "PointerSliceNonEmpty": [ + "value" + ], + "Pointer": { + "Float": 5e-324 + }, + "Interface": [ + "" + ] +}`, + }, { + name: jsontest.Name("Structs/OmitEmpty/Legacy/Zero"), + opts: []Options{jsonflags.OmitEmptyWithLegacySemantics | 1}, + in: structOmitEmptyAll{}, + want: `{}`, + }, { + name: jsontest.Name("Structs/OmitEmpty/Legacy/NonEmpty"), + opts: []Options{jsontext.Multiline(true), jsonflags.OmitEmptyWithLegacySemantics | 1}, + in: structOmitEmptyAll{ + Bool: true, + PointerBool: addr(true), + String: string("value"), + StringEmpty: stringMarshalEmpty("value"), + StringNonEmpty: stringMarshalNonEmpty("value"), + PointerString: addr(string("value")), + PointerStringEmpty: addr(stringMarshalEmpty("value")), + PointerStringNonEmpty: addr(stringMarshalNonEmpty("value")), + Bytes: []byte("value"), + BytesEmpty: bytesMarshalEmpty([]byte("value")), + BytesNonEmpty: bytesMarshalNonEmpty([]byte("value")), + PointerBytes: addr([]byte("value")), + PointerBytesEmpty: addr(bytesMarshalEmpty([]byte("value"))), + PointerBytesNonEmpty: addr(bytesMarshalNonEmpty([]byte("value"))), + Float: math.Copysign(0, -1), + PointerFloat: addr(math.Copysign(0, -1)), + Map: map[string]string{"": ""}, + MapEmpty: mapMarshalEmpty{"key": "value"}, + MapNonEmpty: mapMarshalNonEmpty{"key": "value"}, + PointerMap: addr(map[string]string{"": ""}), + PointerMapEmpty: addr(mapMarshalEmpty{"key": "value"}), + PointerMapNonEmpty: addr(mapMarshalNonEmpty{"key": "value"}), + Slice: []string{""}, + SliceEmpty: sliceMarshalEmpty{"value"}, + SliceNonEmpty: sliceMarshalNonEmpty{"value"}, + PointerSlice: addr([]string{""}), + PointerSliceEmpty: addr(sliceMarshalEmpty{"value"}), + PointerSliceNonEmpty: addr(sliceMarshalNonEmpty{"value"}), + Pointer: &structOmitZeroEmptyAll{Float: math.Copysign(0, -1)}, + Interface: []string{""}, + }, + want: `{ + "Bool": true, + "PointerBool": true, + "String": "value", + "StringEmpty": "", + "StringNonEmpty": "value", + "PointerString": "value", + "PointerStringEmpty": "", + "PointerStringNonEmpty": "value", + "Bytes": "dmFsdWU=", + "BytesEmpty": [], + "BytesNonEmpty": [ + "value" + ], + "PointerBytes": "dmFsdWU=", + "PointerBytesEmpty": [], + "PointerBytesNonEmpty": [ + "value" + ], + "PointerFloat": -0, + "Map": { + "": "" + }, + "MapEmpty": {}, + "MapNonEmpty": { + "key": "value" + }, + "PointerMap": { + "": "" + }, + "PointerMapEmpty": {}, + "PointerMapNonEmpty": { + "key": "value" + }, + "Slice": [ + "" + ], + "SliceEmpty": [], + "SliceNonEmpty": [ + "value" + ], + "PointerSlice": [ + "" + ], + "PointerSliceEmpty": [], + "PointerSliceNonEmpty": [ + "value" + ], + "Pointer": {}, + "Interface": [ + "" + ] +}`, + }, { + name: jsontest.Name("Structs/OmitEmpty/NonEmptyString"), + in: struct { + X string `json:",omitempty"` + }{`"`}, + want: `{"X":"\""}`, + }, { + name: jsontest.Name("Structs/OmitZeroEmpty/Zero"), + in: structOmitZeroEmptyAll{}, + want: `{}`, + }, { + name: jsontest.Name("Structs/OmitZeroEmpty/Empty"), + in: structOmitZeroEmptyAll{ + Bytes: []byte{}, + Map: map[string]string{}, + Slice: []string{}, + Pointer: &structOmitZeroEmptyAll{}, + Interface: []string{}, + }, + want: `{}`, + }, { + name: jsontest.Name("Structs/OmitEmpty/PathologicalDepth"), + in: func() any { + type X struct { + X *X `json:",omitempty"` + } + var make func(int) *X + make = func(n int) *X { + if n == 0 { + return nil + } + return &X{make(n - 1)} + } + return make(100) + }(), + want: `{}`, + useWriter: true, + }, { + name: jsontest.Name("Structs/OmitEmpty/PathologicalBreadth"), + in: func() any { + var fields []reflect.StructField + for i := range 100 { + fields = append(fields, reflect.StructField{ + Name: fmt.Sprintf("X%d", i), + Type: T[stringMarshalEmpty](), + Tag: `json:",omitempty"`, + }) + } + return reflect.New(reflect.StructOf(fields)).Interface() + }(), + want: `{}`, + useWriter: true, + }, { + name: jsontest.Name("Structs/OmitEmpty/PathologicalTree"), + in: func() any { + type X struct { + XL, XR *X `json:",omitempty"` + } + var make func(int) *X + make = func(n int) *X { + if n == 0 { + return nil + } + return &X{make(n - 1), make(n - 1)} + } + return make(8) + }(), + want: `{}`, + useWriter: true, + }, { + name: jsontest.Name("Structs/OmitZeroEmpty/NonEmpty"), + in: structOmitZeroEmptyAll{ + Bytes: []byte("value"), + Map: map[string]string{"": ""}, + Slice: []string{""}, + Pointer: &structOmitZeroEmptyAll{Bool: true}, + Interface: []string{""}, + }, + want: `{"Bytes":"dmFsdWU=","Map":{"":""},"Slice":[""],"Pointer":{"Bool":true},"Interface":[""]}`, + }, { + name: jsontest.Name("Structs/Format/Bytes"), + opts: []Options{jsontext.Multiline(true)}, + in: structFormatBytes{ + Base16: []byte("\x01\x23\x45\x67\x89\xab\xcd\xef"), + Base32: []byte("\x00D2\x14\xc7BT\xb65τe:V\xd7\xc6u\xbew\xdf"), + Base32Hex: []byte("\x00D2\x14\xc7BT\xb65τe:V\xd7\xc6u\xbew\xdf"), + Base64: []byte("\x00\x10\x83\x10Q\x87 \x92\x8b0ӏA\x14\x93QU\x97a\x96\x9bqן\x82\x18\xa3\x92Y\xa7\xa2\x9a\xab\xb2ۯ\xc3\x1c\xb3\xd3]\xb7㞻\xf3߿"), + Base64URL: []byte("\x00\x10\x83\x10Q\x87 \x92\x8b0ӏA\x14\x93QU\x97a\x96\x9bqן\x82\x18\xa3\x92Y\xa7\xa2\x9a\xab\xb2ۯ\xc3\x1c\xb3\xd3]\xb7㞻\xf3߿"), + Array: []byte{1, 2, 3, 4}, + }, + want: `{ + "Base16": "0123456789abcdef", + "Base32": "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567", + "Base32Hex": "0123456789ABCDEFGHIJKLMNOPQRSTUV", + "Base64": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", + "Base64URL": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", + "Array": [ + 1, + 2, + 3, + 4 + ] +}`}, { + name: jsontest.Name("Structs/Format/ArrayBytes"), + opts: []Options{jsontext.Multiline(true)}, + in: structFormatArrayBytes{ + Base16: [4]byte{1, 2, 3, 4}, + Base32: [4]byte{1, 2, 3, 4}, + Base32Hex: [4]byte{1, 2, 3, 4}, + Base64: [4]byte{1, 2, 3, 4}, + Base64URL: [4]byte{1, 2, 3, 4}, + Array: [4]byte{1, 2, 3, 4}, + Default: [4]byte{1, 2, 3, 4}, + }, + want: `{ + "Base16": "01020304", + "Base32": "AEBAGBA=", + "Base32Hex": "0410610=", + "Base64": "AQIDBA==", + "Base64URL": "AQIDBA==", + "Array": [ + 1, + 2, + 3, + 4 + ], + "Default": "AQIDBA==" +}`}, { + name: jsontest.Name("Structs/Format/ArrayBytes/Legacy"), + opts: []Options{jsontext.Multiline(true), jsonflags.FormatByteArrayAsArray | jsonflags.FormatBytesWithLegacySemantics | 1}, + in: structFormatArrayBytes{ + Base16: [4]byte{1, 2, 3, 4}, + Base32: [4]byte{1, 2, 3, 4}, + Base32Hex: [4]byte{1, 2, 3, 4}, + Base64: [4]byte{1, 2, 3, 4}, + Base64URL: [4]byte{1, 2, 3, 4}, + Array: [4]byte{1, 2, 3, 4}, + Default: [4]byte{1, 2, 3, 4}, + }, + want: `{ + "Base16": "01020304", + "Base32": "AEBAGBA=", + "Base32Hex": "0410610=", + "Base64": "AQIDBA==", + "Base64URL": "AQIDBA==", + "Array": [ + 1, + 2, + 3, + 4 + ], + "Default": [ + 1, + 2, + 3, + 4 + ] +}`}, { + name: jsontest.Name("Structs/Format/Bytes/Array"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(in byte) ([]byte, error) { + if in > 3 { + return []byte("true"), nil + } else { + return []byte("false"), nil + } + })), + }, + in: struct { + Array []byte `json:",format:array"` + }{ + Array: []byte{1, 6, 2, 5, 3, 4}, + }, + want: `{"Array":[false,true,false,true,false,true]}`, + }, { + name: jsontest.Name("Structs/Format/Floats"), + opts: []Options{jsontext.Multiline(true)}, + in: []structFormatFloats{ + {NonFinite: math.Pi, PointerNonFinite: addr(math.Pi)}, + {NonFinite: math.NaN(), PointerNonFinite: addr(math.NaN())}, + {NonFinite: math.Inf(-1), PointerNonFinite: addr(math.Inf(-1))}, + {NonFinite: math.Inf(+1), PointerNonFinite: addr(math.Inf(+1))}, + }, + want: `[ + { + "NonFinite": 3.141592653589793, + "PointerNonFinite": 3.141592653589793 + }, + { + "NonFinite": "NaN", + "PointerNonFinite": "NaN" + }, + { + "NonFinite": "-Infinity", + "PointerNonFinite": "-Infinity" + }, + { + "NonFinite": "Infinity", + "PointerNonFinite": "Infinity" + } +]`, + }, { + name: jsontest.Name("Structs/Format/Maps"), + opts: []Options{jsontext.Multiline(true)}, + in: []structFormatMaps{{ + EmitNull: map[string]string(nil), PointerEmitNull: addr(map[string]string(nil)), + EmitEmpty: map[string]string(nil), PointerEmitEmpty: addr(map[string]string(nil)), + EmitDefault: map[string]string(nil), PointerEmitDefault: addr(map[string]string(nil)), + }, { + EmitNull: map[string]string{}, PointerEmitNull: addr(map[string]string{}), + EmitEmpty: map[string]string{}, PointerEmitEmpty: addr(map[string]string{}), + EmitDefault: map[string]string{}, PointerEmitDefault: addr(map[string]string{}), + }, { + EmitNull: map[string]string{"k": "v"}, PointerEmitNull: addr(map[string]string{"k": "v"}), + EmitEmpty: map[string]string{"k": "v"}, PointerEmitEmpty: addr(map[string]string{"k": "v"}), + EmitDefault: map[string]string{"k": "v"}, PointerEmitDefault: addr(map[string]string{"k": "v"}), + }}, + want: `[ + { + "EmitNull": null, + "PointerEmitNull": null, + "EmitEmpty": {}, + "PointerEmitEmpty": {}, + "EmitDefault": {}, + "PointerEmitDefault": {} + }, + { + "EmitNull": {}, + "PointerEmitNull": {}, + "EmitEmpty": {}, + "PointerEmitEmpty": {}, + "EmitDefault": {}, + "PointerEmitDefault": {} + }, + { + "EmitNull": { + "k": "v" + }, + "PointerEmitNull": { + "k": "v" + }, + "EmitEmpty": { + "k": "v" + }, + "PointerEmitEmpty": { + "k": "v" + }, + "EmitDefault": { + "k": "v" + }, + "PointerEmitDefault": { + "k": "v" + } + } +]`, + }, { + name: jsontest.Name("Structs/Format/Maps/FormatNilMapAsNull"), + opts: []Options{ + FormatNilMapAsNull(true), + jsontext.Multiline(true), + }, + in: []structFormatMaps{{ + EmitNull: map[string]string(nil), PointerEmitNull: addr(map[string]string(nil)), + EmitEmpty: map[string]string(nil), PointerEmitEmpty: addr(map[string]string(nil)), + EmitDefault: map[string]string(nil), PointerEmitDefault: addr(map[string]string(nil)), + }, { + EmitNull: map[string]string{}, PointerEmitNull: addr(map[string]string{}), + EmitEmpty: map[string]string{}, PointerEmitEmpty: addr(map[string]string{}), + EmitDefault: map[string]string{}, PointerEmitDefault: addr(map[string]string{}), + }, { + EmitNull: map[string]string{"k": "v"}, PointerEmitNull: addr(map[string]string{"k": "v"}), + EmitEmpty: map[string]string{"k": "v"}, PointerEmitEmpty: addr(map[string]string{"k": "v"}), + EmitDefault: map[string]string{"k": "v"}, PointerEmitDefault: addr(map[string]string{"k": "v"}), + }}, + want: `[ + { + "EmitNull": null, + "PointerEmitNull": null, + "EmitEmpty": {}, + "PointerEmitEmpty": {}, + "EmitDefault": null, + "PointerEmitDefault": null + }, + { + "EmitNull": {}, + "PointerEmitNull": {}, + "EmitEmpty": {}, + "PointerEmitEmpty": {}, + "EmitDefault": {}, + "PointerEmitDefault": {} + }, + { + "EmitNull": { + "k": "v" + }, + "PointerEmitNull": { + "k": "v" + }, + "EmitEmpty": { + "k": "v" + }, + "PointerEmitEmpty": { + "k": "v" + }, + "EmitDefault": { + "k": "v" + }, + "PointerEmitDefault": { + "k": "v" + } + } +]`, + }, { + name: jsontest.Name("Structs/Format/Slices"), + opts: []Options{jsontext.Multiline(true)}, + in: []structFormatSlices{{ + EmitNull: []string(nil), PointerEmitNull: addr([]string(nil)), + EmitEmpty: []string(nil), PointerEmitEmpty: addr([]string(nil)), + EmitDefault: []string(nil), PointerEmitDefault: addr([]string(nil)), + }, { + EmitNull: []string{}, PointerEmitNull: addr([]string{}), + EmitEmpty: []string{}, PointerEmitEmpty: addr([]string{}), + EmitDefault: []string{}, PointerEmitDefault: addr([]string{}), + }, { + EmitNull: []string{"v"}, PointerEmitNull: addr([]string{"v"}), + EmitEmpty: []string{"v"}, PointerEmitEmpty: addr([]string{"v"}), + EmitDefault: []string{"v"}, PointerEmitDefault: addr([]string{"v"}), + }}, + want: `[ + { + "EmitNull": null, + "PointerEmitNull": null, + "EmitEmpty": [], + "PointerEmitEmpty": [], + "EmitDefault": [], + "PointerEmitDefault": [] + }, + { + "EmitNull": [], + "PointerEmitNull": [], + "EmitEmpty": [], + "PointerEmitEmpty": [], + "EmitDefault": [], + "PointerEmitDefault": [] + }, + { + "EmitNull": [ + "v" + ], + "PointerEmitNull": [ + "v" + ], + "EmitEmpty": [ + "v" + ], + "PointerEmitEmpty": [ + "v" + ], + "EmitDefault": [ + "v" + ], + "PointerEmitDefault": [ + "v" + ] + } +]`, + }, { + name: jsontest.Name("Structs/Format/Invalid/Bool"), + in: structFormatInvalid{Bool: true}, + want: `{"Bool"`, + wantErr: EM(errInvalidFormatFlag).withPos(`{"Bool":`, "/Bool").withType(0, boolType), + }, { + name: jsontest.Name("Structs/Format/Invalid/String"), + in: structFormatInvalid{String: "string"}, + want: `{"String"`, + wantErr: EM(errInvalidFormatFlag).withPos(`{"String":`, "/String").withType(0, stringType), + }, { + name: jsontest.Name("Structs/Format/Invalid/Bytes"), + in: structFormatInvalid{Bytes: []byte("bytes")}, + want: `{"Bytes"`, + wantErr: EM(errInvalidFormatFlag).withPos(`{"Bytes":`, "/Bytes").withType(0, bytesType), + }, { + name: jsontest.Name("Structs/Format/Invalid/Int"), + in: structFormatInvalid{Int: 1}, + want: `{"Int"`, + wantErr: EM(errInvalidFormatFlag).withPos(`{"Int":`, "/Int").withType(0, T[int64]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Uint"), + in: structFormatInvalid{Uint: 1}, + want: `{"Uint"`, + wantErr: EM(errInvalidFormatFlag).withPos(`{"Uint":`, "/Uint").withType(0, T[uint64]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Float"), + in: structFormatInvalid{Float: 1}, + want: `{"Float"`, + wantErr: EM(errInvalidFormatFlag).withPos(`{"Float":`, "/Float").withType(0, T[float64]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Map"), + in: structFormatInvalid{Map: map[string]string{}}, + want: `{"Map"`, + wantErr: EM(errInvalidFormatFlag).withPos(`{"Map":`, "/Map").withType(0, T[map[string]string]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Struct"), + in: structFormatInvalid{Struct: structAll{Bool: true}}, + want: `{"Struct"`, + wantErr: EM(errInvalidFormatFlag).withPos(`{"Struct":`, "/Struct").withType(0, T[structAll]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Slice"), + in: structFormatInvalid{Slice: []string{}}, + want: `{"Slice"`, + wantErr: EM(errInvalidFormatFlag).withPos(`{"Slice":`, "/Slice").withType(0, T[[]string]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Array"), + in: structFormatInvalid{Array: [1]string{"string"}}, + want: `{"Array"`, + wantErr: EM(errInvalidFormatFlag).withPos(`{"Array":`, "/Array").withType(0, T[[1]string]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Interface"), + in: structFormatInvalid{Interface: "anything"}, + want: `{"Interface"`, + wantErr: EM(errInvalidFormatFlag).withPos(`{"Interface":`, "/Interface").withType(0, T[any]()), + }, { + name: jsontest.Name("Structs/Inline/Zero"), + in: structInlined{}, + want: `{"D":""}`, + }, { + name: jsontest.Name("Structs/Inline/Alloc"), + in: structInlined{ + X: structInlinedL1{ + X: &structInlinedL2{}, + StructEmbed1: StructEmbed1{}, + }, + StructEmbed2: &StructEmbed2{}, + }, + want: `{"A":"","B":"","D":"","E":"","F":"","G":""}`, + }, { + name: jsontest.Name("Structs/Inline/NonZero"), + in: structInlined{ + X: structInlinedL1{ + X: &structInlinedL2{A: "A1", B: "B1", C: "C1"}, + StructEmbed1: StructEmbed1{C: "C2", D: "D2", E: "E2"}, + }, + StructEmbed2: &StructEmbed2{E: "E3", F: "F3", G: "G3"}, + }, + want: `{"A":"A1","B":"B1","D":"D2","E":"E3","F":"F3","G":"G3"}`, + }, { + name: jsontest.Name("Structs/Inline/DualCycle"), + in: cyclicA{ + B1: cyclicB{F: 1}, // B1.F ignored since it conflicts with B2.F + B2: cyclicB{F: 2}, // B2.F ignored since it conflicts with B1.F + }, + want: `{}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/Nil"), + in: structInlineTextValue{X: jsontext.Value(nil)}, + want: `{}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/Empty"), + in: structInlineTextValue{X: jsontext.Value("")}, + want: `{}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/NonEmptyN1"), + in: structInlineTextValue{X: jsontext.Value(` { "fizz" : "buzz" } `)}, + want: `{"fizz":"buzz"}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/NonEmptyN2"), + in: structInlineTextValue{X: jsontext.Value(` { "fizz" : "buzz" , "foo" : "bar" } `)}, + want: `{"fizz":"buzz","foo":"bar"}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/NonEmptyWithOthers"), + in: structInlineTextValue{ + A: 1, + X: jsontext.Value(` { "fizz" : "buzz" , "foo" : "bar" } `), + B: 2, + }, + // NOTE: Inlined fallback fields are always serialized last. + want: `{"A":1,"B":2,"fizz":"buzz","foo":"bar"}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/RejectDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(false)}, + in: structInlineTextValue{X: jsontext.Value(` { "fizz" : "buzz" , "fizz" : "buzz" } `)}, + want: `{"fizz":"buzz"`, + wantErr: newDuplicateNameError("/fizz", nil, len64(`{"fizz":"buzz"`)), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/AllowDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(true)}, + in: structInlineTextValue{X: jsontext.Value(` { "fizz" : "buzz" , "fizz" : "buzz" } `)}, + want: `{"fizz":"buzz","fizz":"buzz"}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/RejectInvalidUTF8"), + opts: []Options{jsontext.AllowInvalidUTF8(false)}, + in: structInlineTextValue{X: jsontext.Value(`{"` + "\xde\xad\xbe\xef" + `":"value"}`)}, + want: `{`, + wantErr: newInvalidUTF8Error(len64(`{"`+"\xde\xad"), ""), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/AllowInvalidUTF8"), + opts: []Options{jsontext.AllowInvalidUTF8(true)}, + in: structInlineTextValue{X: jsontext.Value(`{"` + "\xde\xad\xbe\xef" + `":"value"}`)}, + want: `{"ޭ��":"value"}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/InvalidWhitespace"), + in: structInlineTextValue{X: jsontext.Value("\n\r\t ")}, + want: `{`, + wantErr: EM(io.ErrUnexpectedEOF).withPos(`{`, "").withType(0, T[jsontext.Value]()), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/InvalidObject"), + in: structInlineTextValue{X: jsontext.Value(` true `)}, + want: `{`, + wantErr: EM(errRawInlinedNotObject).withPos(`{`, "").withType(0, T[jsontext.Value]()), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/InvalidObjectName"), + in: structInlineTextValue{X: jsontext.Value(` { true : false } `)}, + want: `{`, + wantErr: EM(newNonStringNameError(len64(" { "), "")).withPos(`{`, "").withType(0, T[jsontext.Value]()), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/InvalidEndObject"), + in: structInlineTextValue{X: jsontext.Value(` { "name" : false , } `)}, + want: `{"name":false`, + wantErr: EM(newInvalidCharacterError(",", "at start of value", len64(` { "name" : false `), "")).withPos(`{"name":false,`, "").withType(0, T[jsontext.Value]()), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/InvalidDualObject"), + in: structInlineTextValue{X: jsontext.Value(`{}{}`)}, + want: `{`, + wantErr: EM(newInvalidCharacterError("{", "after top-level value", len64(`{}`), "")).withPos(`{`, "").withType(0, T[jsontext.Value]()), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/Nested/Nil"), + in: structInlinePointerInlineTextValue{}, + want: `{}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/PointerTextValue/Nil"), + in: structInlinePointerTextValue{}, + want: `{}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/PointerTextValue/NonEmpty"), + in: structInlinePointerTextValue{X: addr(jsontext.Value(` { "fizz" : "buzz" } `))}, + want: `{"fizz":"buzz"}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/PointerTextValue/Nested/Nil"), + in: structInlineInlinePointerTextValue{}, + want: `{}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/Nil"), + in: structInlineMapStringAny{X: nil}, + want: `{}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/Empty"), + in: structInlineMapStringAny{X: make(jsonObject)}, + want: `{}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/NonEmptyN1"), + in: structInlineMapStringAny{X: jsonObject{"fizz": nil}}, + want: `{"fizz":null}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/NonEmptyN2"), + in: structInlineMapStringAny{X: jsonObject{"fizz": time.Time{}, "buzz": math.Pi}}, + want: `{"buzz":3.141592653589793,"fizz":"0001-01-01T00:00:00Z"}`, + canonicalize: true, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/NonEmptyWithOthers"), + in: structInlineMapStringAny{ + A: 1, + X: jsonObject{"fizz": nil}, + B: 2, + }, + // NOTE: Inlined fallback fields are always serialized last. + want: `{"A":1,"B":2,"fizz":null}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/RejectInvalidUTF8"), + opts: []Options{jsontext.AllowInvalidUTF8(false)}, + in: structInlineMapStringAny{X: jsonObject{"\xde\xad\xbe\xef": nil}}, + want: `{`, + wantErr: EM(jsonwire.ErrInvalidUTF8).withPos(`{`, "").withType(0, stringType), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/AllowInvalidUTF8"), + opts: []Options{jsontext.AllowInvalidUTF8(true)}, + in: structInlineMapStringAny{X: jsonObject{"\xde\xad\xbe\xef": nil}}, + want: `{"ޭ��":null}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/InvalidValue"), + opts: []Options{jsontext.AllowInvalidUTF8(true)}, + in: structInlineMapStringAny{X: jsonObject{"name": make(chan string)}}, + want: `{"name"`, + wantErr: EM(nil).withPos(`{"name":`, "/name").withType(0, T[chan string]()), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/Nested/Nil"), + in: structInlinePointerInlineMapStringAny{}, + want: `{}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/MarshalFunc"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v float64) ([]byte, error) { + return []byte(fmt.Sprintf(`"%v"`, v)), nil + })), + }, + in: structInlineMapStringAny{X: jsonObject{"fizz": 3.14159}}, + want: `{"fizz":"3.14159"}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/PointerMapStringAny/Nil"), + in: structInlinePointerMapStringAny{X: nil}, + want: `{}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/PointerMapStringAny/NonEmpty"), + in: structInlinePointerMapStringAny{X: addr(jsonObject{"name": "value"})}, + want: `{"name":"value"}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/PointerMapStringAny/Nested/Nil"), + in: structInlineInlinePointerMapStringAny{}, + want: `{}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringInt"), + in: structInlineMapStringInt{ + X: map[string]int{"zero": 0, "one": 1, "two": 2}, + }, + want: `{"one":1,"two":2,"zero":0}`, + canonicalize: true, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringInt/Deterministic"), + opts: []Options{Deterministic(true)}, + in: structInlineMapStringInt{ + X: map[string]int{"zero": 0, "one": 1, "two": 2}, + }, + want: `{"one":1,"two":2,"zero":0}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringInt/Deterministic+AllowInvalidUTF8+RejectDuplicateNames"), + opts: []Options{Deterministic(true), jsontext.AllowInvalidUTF8(true), jsontext.AllowDuplicateNames(false)}, + in: structInlineMapStringInt{ + X: map[string]int{"\xff": 0, "\xfe": 1}, + }, + want: `{"�":1`, + wantErr: newDuplicateNameError("", []byte(`"�"`), len64(`{"�":1`)), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringInt/Deterministic+AllowInvalidUTF8+AllowDuplicateNames"), + opts: []Options{Deterministic(true), jsontext.AllowInvalidUTF8(true), jsontext.AllowDuplicateNames(true)}, + in: structInlineMapStringInt{ + X: map[string]int{"\xff": 0, "\xfe": 1}, + }, + want: `{"�":1,"�":0}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringInt/StringifiedNumbers"), + opts: []Options{StringifyNumbers(true)}, + in: structInlineMapStringInt{ + X: map[string]int{"zero": 0, "one": 1, "two": 2}, + }, + want: `{"one":"1","two":"2","zero":"0"}`, + canonicalize: true, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringInt/MarshalFunc"), + opts: []Options{ + WithMarshalers(JoinMarshalers( + // Marshalers do not affect the string key of inlined maps. + MarshalFunc(func(v string) ([]byte, error) { + return []byte(fmt.Sprintf(`"%q"`, strings.ToUpper(v))), nil + }), + MarshalFunc(func(v int) ([]byte, error) { + return []byte(fmt.Sprintf(`"%v"`, v)), nil + }), + )), + }, + in: structInlineMapStringInt{ + X: map[string]int{"zero": 0, "one": 1, "two": 2}, + }, + want: `{"one":"1","two":"2","zero":"0"}`, + canonicalize: true, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringInt"), + in: structInlineMapNamedStringInt{ + X: map[namedString]int{"zero": 0, "one": 1, "two": 2}, + }, + want: `{"one":1,"two":2,"zero":0}`, + canonicalize: true, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringInt/Deterministic"), + opts: []Options{Deterministic(true)}, + in: structInlineMapNamedStringInt{ + X: map[namedString]int{"zero": 0, "one": 1, "two": 2}, + }, + want: `{"one":1,"two":2,"zero":0}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/Nil"), + in: structInlineMapNamedStringAny{X: nil}, + want: `{}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/Empty"), + in: structInlineMapNamedStringAny{X: make(map[namedString]any)}, + want: `{}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/NonEmptyN1"), + in: structInlineMapNamedStringAny{X: map[namedString]any{"fizz": nil}}, + want: `{"fizz":null}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/NonEmptyN2"), + in: structInlineMapNamedStringAny{X: map[namedString]any{"fizz": time.Time{}, "buzz": math.Pi}}, + want: `{"buzz":3.141592653589793,"fizz":"0001-01-01T00:00:00Z"}`, + canonicalize: true, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/NonEmptyWithOthers"), + in: structInlineMapNamedStringAny{ + A: 1, + X: map[namedString]any{"fizz": nil}, + B: 2, + }, + // NOTE: Inlined fallback fields are always serialized last. + want: `{"A":1,"B":2,"fizz":null}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/RejectInvalidUTF8"), + opts: []Options{jsontext.AllowInvalidUTF8(false)}, + in: structInlineMapNamedStringAny{X: map[namedString]any{"\xde\xad\xbe\xef": nil}}, + want: `{`, + wantErr: EM(jsonwire.ErrInvalidUTF8).withPos(`{`, "").withType(0, T[namedString]()), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/AllowInvalidUTF8"), + opts: []Options{jsontext.AllowInvalidUTF8(true)}, + in: structInlineMapNamedStringAny{X: map[namedString]any{"\xde\xad\xbe\xef": nil}}, + want: `{"ޭ��":null}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/InvalidValue"), + opts: []Options{jsontext.AllowInvalidUTF8(true)}, + in: structInlineMapNamedStringAny{X: map[namedString]any{"name": make(chan string)}}, + want: `{"name"`, + wantErr: EM(nil).withPos(`{"name":`, "/name").withType(0, T[chan string]()), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/MarshalFunc"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v float64) ([]byte, error) { + return []byte(fmt.Sprintf(`"%v"`, v)), nil + })), + }, + in: structInlineMapNamedStringAny{X: map[namedString]any{"fizz": 3.14159}}, + want: `{"fizz":"3.14159"}`, + }, { + name: jsontest.Name("Structs/InlinedFallback/DiscardUnknownMembers"), + opts: []Options{DiscardUnknownMembers(true)}, + in: structInlineTextValue{ + A: 1, + X: jsontext.Value(` { "fizz" : "buzz" } `), + B: 2, + }, + // NOTE: DiscardUnknownMembers has no effect since this is "inline". + want: `{"A":1,"B":2,"fizz":"buzz"}`, + }, { + name: jsontest.Name("Structs/UnknownFallback/DiscardUnknownMembers"), + opts: []Options{DiscardUnknownMembers(true)}, + in: structUnknownTextValue{ + A: 1, + X: jsontext.Value(` { "fizz" : "buzz" } `), + B: 2, + }, + want: `{"A":1,"B":2}`, + }, { + name: jsontest.Name("Structs/UnknownFallback"), + in: structUnknownTextValue{ + A: 1, + X: jsontext.Value(` { "fizz" : "buzz" } `), + B: 2, + }, + want: `{"A":1,"B":2,"fizz":"buzz"}`, + }, { + name: jsontest.Name("Structs/DuplicateName/NoCaseInlineTextValue/Other"), + in: structNoCaseInlineTextValue{ + X: jsontext.Value(`{"dupe":"","dupe":""}`), + }, + want: `{"dupe":""`, + wantErr: newDuplicateNameError("", []byte(`"dupe"`), len64(`{"dupe":""`)), + }, { + name: jsontest.Name("Structs/DuplicateName/NoCaseInlineTextValue/Other/AllowDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(true)}, + in: structNoCaseInlineTextValue{ + X: jsontext.Value(`{"dupe": "", "dupe": ""}`), + }, + want: `{"dupe":"","dupe":""}`, + }, { + name: jsontest.Name("Structs/DuplicateName/NoCaseInlineTextValue/ExactDifferent"), + in: structNoCaseInlineTextValue{ + X: jsontext.Value(`{"Aaa": "", "AaA": "", "AAa": "", "AAA": ""}`), + }, + want: `{"Aaa":"","AaA":"","AAa":"","AAA":""}`, + }, { + name: jsontest.Name("Structs/DuplicateName/NoCaseInlineTextValue/ExactConflict"), + in: structNoCaseInlineTextValue{ + X: jsontext.Value(`{"Aaa": "", "Aaa": ""}`), + }, + want: `{"Aaa":""`, + wantErr: newDuplicateNameError("", []byte(`"Aaa"`), len64(`{"Aaa":""`)), + }, { + name: jsontest.Name("Structs/DuplicateName/NoCaseInlineTextValue/ExactConflict/AllowDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(true)}, + in: structNoCaseInlineTextValue{ + X: jsontext.Value(`{"Aaa": "", "Aaa": ""}`), + }, + want: `{"Aaa":"","Aaa":""}`, + }, { + name: jsontest.Name("Structs/DuplicateName/NoCaseInlineTextValue/NoCaseConflict"), + in: structNoCaseInlineTextValue{ + X: jsontext.Value(`{"Aaa": "", "AaA": "", "aaa": ""}`), + }, + want: `{"Aaa":"","AaA":""`, + wantErr: newDuplicateNameError("", []byte(`"aaa"`), len64(`{"Aaa":"","AaA":""`)), + }, { + name: jsontest.Name("Structs/DuplicateName/NoCaseInlineTextValue/NoCaseConflict/AllowDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(true)}, + in: structNoCaseInlineTextValue{ + X: jsontext.Value(`{"Aaa": "", "AaA": "", "aaa": ""}`), + }, + want: `{"Aaa":"","AaA":"","aaa":""}`, + }, { + name: jsontest.Name("Structs/DuplicateName/NoCaseInlineTextValue/ExactDifferentWithField"), + in: structNoCaseInlineTextValue{ + AAA: "x", + AaA: "x", + X: jsontext.Value(`{"Aaa": ""}`), + }, + want: `{"AAA":"x","AaA":"x","Aaa":""}`, + }, { + name: jsontest.Name("Structs/DuplicateName/NoCaseInlineTextValue/ExactConflictWithField"), + in: structNoCaseInlineTextValue{ + AAA: "x", + AaA: "x", + X: jsontext.Value(`{"AAA": ""}`), + }, + want: `{"AAA":"x","AaA":"x"`, + wantErr: newDuplicateNameError("", []byte(`"AAA"`), len64(`{"AAA":"x","AaA":"x"`)), + }, { + name: jsontest.Name("Structs/DuplicateName/NoCaseInlineTextValue/NoCaseConflictWithField"), + in: structNoCaseInlineTextValue{ + AAA: "x", + AaA: "x", + X: jsontext.Value(`{"aaa": ""}`), + }, + want: `{"AAA":"x","AaA":"x"`, + wantErr: newDuplicateNameError("", []byte(`"aaa"`), len64(`{"AAA":"x","AaA":"x"`)), + }, { + name: jsontest.Name("Structs/DuplicateName/MatchCaseInsensitiveDelimiter"), + in: structNoCaseInlineTextValue{ + AaA: "x", + X: jsontext.Value(`{"aa_a": ""}`), + }, + want: `{"AaA":"x"`, + wantErr: newDuplicateNameError("", []byte(`"aa_a"`), len64(`{"AaA":"x"`)), + }, { + name: jsontest.Name("Structs/DuplicateName/MatchCaseSensitiveDelimiter"), + opts: []Options{jsonflags.MatchCaseSensitiveDelimiter | 1}, + in: structNoCaseInlineTextValue{ + AaA: "x", + X: jsontext.Value(`{"aa_a": ""}`), + }, + want: `{"AaA":"x","aa_a":""}`, + }, { + name: jsontest.Name("Structs/DuplicateName/MatchCaseInsensitiveNames+MatchCaseSensitiveDelimiter"), + opts: []Options{MatchCaseInsensitiveNames(true), jsonflags.MatchCaseSensitiveDelimiter | 1}, + in: structNoCaseInlineTextValue{ + AaA: "x", + X: jsontext.Value(`{"aa_a": ""}`), + }, + want: `{"AaA":"x","aa_a":""}`, + }, { + name: jsontest.Name("Structs/DuplicateName/MatchCaseInsensitiveNames+MatchCaseSensitiveDelimiter"), + opts: []Options{MatchCaseInsensitiveNames(true), jsonflags.MatchCaseSensitiveDelimiter | 1}, + in: structNoCaseInlineTextValue{ + AA_b: "x", + X: jsontext.Value(`{"aa_b": ""}`), + }, + want: `{"AA_b":"x"`, + wantErr: newDuplicateNameError("", []byte(`"aa_b"`), len64(`{"AA_b":"x"`)), + }, { + name: jsontest.Name("Structs/DuplicateName/NoCaseInlineMapStringAny/ExactDifferent"), + in: structNoCaseInlineMapStringAny{ + X: jsonObject{"Aaa": "", "AaA": "", "AAa": "", "AAA": ""}, + }, + want: `{"AAA":"","AAa":"","AaA":"","Aaa":""}`, + canonicalize: true, + }, { + name: jsontest.Name("Structs/DuplicateName/NoCaseInlineMapStringAny/ExactDifferentWithField"), + in: structNoCaseInlineMapStringAny{ + AAA: "x", + AaA: "x", + X: jsonObject{"Aaa": ""}, + }, + want: `{"AAA":"x","AaA":"x","Aaa":""}`, + }, { + name: jsontest.Name("Structs/DuplicateName/NoCaseInlineMapStringAny/ExactConflictWithField"), + in: structNoCaseInlineMapStringAny{ + AAA: "x", + AaA: "x", + X: jsonObject{"AAA": ""}, + }, + want: `{"AAA":"x","AaA":"x"`, + wantErr: newDuplicateNameError("", []byte(`"AAA"`), len64(`{"AAA":"x","AaA":"x"`)), + }, { + name: jsontest.Name("Structs/DuplicateName/NoCaseInlineMapStringAny/NoCaseConflictWithField"), + in: structNoCaseInlineMapStringAny{ + AAA: "x", + AaA: "x", + X: jsonObject{"aaa": ""}, + }, + want: `{"AAA":"x","AaA":"x"`, + wantErr: newDuplicateNameError("", []byte(`"aaa"`), len64(`{"AAA":"x","AaA":"x"`)), + }, { + name: jsontest.Name("Structs/Invalid/Conflicting"), + in: structConflicting{}, + want: ``, + wantErr: EM(errors.New("Go struct fields A and B conflict over JSON object name \"conflict\"")).withType(0, T[structConflicting]()), + }, { + name: jsontest.Name("Structs/Invalid/NoneExported"), + in: structNoneExported{}, + want: ``, + wantErr: EM(errNoExportedFields).withType(0, T[structNoneExported]()), + }, { + name: jsontest.Name("Structs/Invalid/MalformedTag"), + in: structMalformedTag{}, + want: ``, + wantErr: EM(errors.New("Go struct field Malformed has malformed `json` tag: invalid character '\"' at start of option (expecting Unicode letter or single quote)")).withType(0, T[structMalformedTag]()), + }, { + name: jsontest.Name("Structs/Invalid/UnexportedTag"), + in: structUnexportedTag{}, + want: ``, + wantErr: EM(errors.New("unexported Go struct field unexported cannot have non-ignored `json:\"name\"` tag")).withType(0, T[structUnexportedTag]()), + }, { + name: jsontest.Name("Structs/Invalid/ExportedEmbedded"), + in: structExportedEmbedded{"hello"}, + want: ``, + wantErr: EM(errors.New("embedded Go struct field NamedString of non-struct type must be explicitly given a JSON name")).withType(0, T[structExportedEmbedded]()), + }, { + name: jsontest.Name("Structs/Valid/ExportedEmbedded"), + opts: []Options{jsonflags.ReportErrorsWithLegacySemantics | 1}, + in: structExportedEmbedded{"hello"}, + want: `{"NamedString":"hello"}`, + }, { + name: jsontest.Name("Structs/Valid/ExportedEmbeddedTag"), + in: structExportedEmbeddedTag{"hello"}, + want: `{"name":"hello"}`, + }, { + name: jsontest.Name("Structs/Invalid/UnexportedEmbedded"), + in: structUnexportedEmbedded{}, + want: ``, + wantErr: EM(errors.New("embedded Go struct field namedString of non-struct type must be explicitly given a JSON name")).withType(0, T[structUnexportedEmbedded]()), + }, { + name: jsontest.Name("Structs/Valid/UnexportedEmbedded"), + opts: []Options{jsonflags.ReportErrorsWithLegacySemantics | 1}, + in: structUnexportedEmbedded{}, + want: `{}`, + }, { + name: jsontest.Name("Structs/Invalid/UnexportedEmbeddedTag"), + in: structUnexportedEmbeddedTag{}, + wantErr: EM(errors.New("Go struct field namedString is not exported")).withType(0, T[structUnexportedEmbeddedTag]()), + }, { + name: jsontest.Name("Structs/Valid/UnexportedEmbeddedTag"), + opts: []Options{jsonflags.ReportErrorsWithLegacySemantics | 1}, + in: structUnexportedEmbeddedTag{}, + want: `{}`, + }, { + name: jsontest.Name("Structs/Invalid/UnexportedEmbeddedMethodTag"), + opts: []Options{jsonflags.ReportErrorsWithLegacySemantics | 1}, + in: structUnexportedEmbeddedMethodTag{}, + want: `{}`, + }, { + name: jsontest.Name("Structs/UnexportedEmbeddedStruct/Zero"), + in: structUnexportedEmbeddedStruct{}, + want: `{"FizzBuzz":0,"Addr":""}`, + }, { + name: jsontest.Name("Structs/UnexportedEmbeddedStruct/NonZero"), + in: structUnexportedEmbeddedStruct{structOmitZeroAll{Bool: true}, 5, structNestedAddr{netip.AddrFrom4([4]byte{192, 168, 0, 1})}}, + want: `{"Bool":true,"FizzBuzz":5,"Addr":"192.168.0.1"}`, + }, { + name: jsontest.Name("Structs/UnexportedEmbeddedStructPointer/Nil"), + in: structUnexportedEmbeddedStructPointer{}, + want: `{"FizzBuzz":0}`, + }, { + name: jsontest.Name("Structs/UnexportedEmbeddedStructPointer/Zero"), + in: structUnexportedEmbeddedStructPointer{&structOmitZeroAll{}, 0, &structNestedAddr{}}, + want: `{"FizzBuzz":0,"Addr":""}`, + }, { + name: jsontest.Name("Structs/UnexportedEmbeddedStructPointer/NonZero"), + in: structUnexportedEmbeddedStructPointer{&structOmitZeroAll{Bool: true}, 5, &structNestedAddr{netip.AddrFrom4([4]byte{192, 168, 0, 1})}}, + want: `{"Bool":true,"FizzBuzz":5,"Addr":"192.168.0.1"}`, + }, { + name: jsontest.Name("Structs/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + in: struct{}{}, + want: `{}`, + }, { + name: jsontest.Name("Slices/Interface"), + in: []any{ + false, true, + "hello", []byte("world"), + int32(-32), namedInt64(-64), + uint32(+32), namedUint64(+64), + float32(32.32), namedFloat64(64.64), + }, + want: `[false,true,"hello","d29ybGQ=",-32,-64,32,64,32.32,64.64]`, + }, { + name: jsontest.Name("Slices/Invalid/Channel"), + in: [](chan string){nil}, + want: `[`, + wantErr: EM(nil).withPos(`[`, "/0").withType(0, T[chan string]()), + }, { + name: jsontest.Name("Slices/RecursiveSlice"), + in: recursiveSlice{ + nil, + {}, + {nil}, + {nil, {}}, + }, + want: `[[],[],[[]],[[],[]]]`, + }, { + name: jsontest.Name("Slices/CyclicSlice"), + in: func() recursiveSlice { + s := recursiveSlice{{}} + s[0] = s + return s + }(), + want: strings.Repeat(`[`, startDetectingCyclesAfter) + `[`, + wantErr: EM(internal.ErrCycle).withPos(strings.Repeat("[", startDetectingCyclesAfter+1), jsontext.Pointer(strings.Repeat("/0", startDetectingCyclesAfter+1))).withType(0, T[recursiveSlice]()), + }, { + name: jsontest.Name("Slices/NonCyclicSlice"), + in: func() []any { + v := []any{nil, nil} + v[1] = v[:1] + for i := 1000; i > 0; i-- { + v = []any{v} + } + return v + }(), + want: strings.Repeat(`[`, startDetectingCyclesAfter) + `[null,[null]]` + strings.Repeat(`]`, startDetectingCyclesAfter), + }, { + name: jsontest.Name("Slices/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + in: []string{"hello", "goodbye"}, + want: `["hello","goodbye"]`, + }, { + name: jsontest.Name("Arrays/Empty"), + in: [0]struct{}{}, + want: `[]`, + }, { + name: jsontest.Name("Arrays/Bool"), + in: [2]bool{false, true}, + want: `[false,true]`, + }, { + name: jsontest.Name("Arrays/String"), + in: [2]string{"hello", "goodbye"}, + want: `["hello","goodbye"]`, + }, { + name: jsontest.Name("Arrays/Bytes"), + in: [2][]byte{[]byte("hello"), []byte("goodbye")}, + want: `["aGVsbG8=","Z29vZGJ5ZQ=="]`, + }, { + name: jsontest.Name("Arrays/Int"), + in: [2]int64{math.MinInt64, math.MaxInt64}, + want: `[-9223372036854775808,9223372036854775807]`, + }, { + name: jsontest.Name("Arrays/Uint"), + in: [2]uint64{0, math.MaxUint64}, + want: `[0,18446744073709551615]`, + }, { + name: jsontest.Name("Arrays/Float"), + in: [2]float64{-math.MaxFloat64, +math.MaxFloat64}, + want: `[-1.7976931348623157e+308,1.7976931348623157e+308]`, + }, { + name: jsontest.Name("Arrays/Invalid/Channel"), + in: new([1]chan string), + want: `[`, + wantErr: EM(nil).withPos(`[`, "/0").withType(0, T[chan string]()), + }, { + name: jsontest.Name("Arrays/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + in: [2]string{"hello", "goodbye"}, + want: `["hello","goodbye"]`, + }, { + name: jsontest.Name("Pointers/NilL0"), + in: (*int)(nil), + want: `null`, + }, { + name: jsontest.Name("Pointers/NilL1"), + in: new(*int), + want: `null`, + }, { + name: jsontest.Name("Pointers/Bool"), + in: addr(addr(bool(true))), + want: `true`, + }, { + name: jsontest.Name("Pointers/String"), + in: addr(addr(string("string"))), + want: `"string"`, + }, { + name: jsontest.Name("Pointers/Bytes"), + in: addr(addr([]byte("bytes"))), + want: `"Ynl0ZXM="`, + }, { + name: jsontest.Name("Pointers/Int"), + in: addr(addr(int(-100))), + want: `-100`, + }, { + name: jsontest.Name("Pointers/Uint"), + in: addr(addr(uint(100))), + want: `100`, + }, { + name: jsontest.Name("Pointers/Float"), + in: addr(addr(float64(3.14159))), + want: `3.14159`, + }, { + name: jsontest.Name("Pointers/CyclicPointer"), + in: func() *recursivePointer { + p := new(recursivePointer) + p.P = p + return p + }(), + want: strings.Repeat(`{"P":`, startDetectingCyclesAfter) + `{"P"`, + wantErr: EM(internal.ErrCycle).withPos(strings.Repeat(`{"P":`, startDetectingCyclesAfter+1), jsontext.Pointer(strings.Repeat("/P", startDetectingCyclesAfter+1))).withType(0, T[*recursivePointer]()), + }, { + name: jsontest.Name("Pointers/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + in: addr(addr(bool(true))), + want: `true`, + }, { + name: jsontest.Name("Interfaces/Nil/Empty"), + in: [1]any{nil}, + want: `[null]`, + }, { + name: jsontest.Name("Interfaces/Nil/NonEmpty"), + in: [1]io.Reader{nil}, + want: `[null]`, + }, { + name: jsontest.Name("Interfaces/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + in: [1]io.Reader{nil}, + want: `[null]`, + }, { + name: jsontest.Name("Interfaces/Any"), + in: struct{ X any }{[]any{nil, false, "", 0.0, map[string]any{}, []any{}, [8]byte{}}}, + want: `{"X":[null,false,"",0,{},[],"AAAAAAAAAAA="]}`, + }, { + name: jsontest.Name("Interfaces/Any/Named"), + in: struct{ X namedAny }{[]namedAny{nil, false, "", 0.0, map[string]namedAny{}, []namedAny{}, [8]byte{}}}, + want: `{"X":[null,false,"",0,{},[],"AAAAAAAAAAA="]}`, + }, { + name: jsontest.Name("Interfaces/Any/Stringified"), + opts: []Options{StringifyNumbers(true)}, + in: struct{ X any }{0.0}, + want: `{"X":"0"}`, + }, { + name: jsontest.Name("Interfaces/Any/MarshalFunc/Any"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v any) ([]byte, error) { + return []byte(`"called"`), nil + })), + }, + in: struct{ X any }{[]any{nil, false, "", 0.0, map[string]any{}, []any{}}}, + want: `"called"`, + }, { + name: jsontest.Name("Interfaces/Any/MarshalFunc/Bool"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v bool) ([]byte, error) { + return []byte(`"called"`), nil + })), + }, + in: struct{ X any }{[]any{nil, false, "", 0.0, map[string]any{}, []any{}}}, + want: `{"X":[null,"called","",0,{},[]]}`, + }, { + name: jsontest.Name("Interfaces/Any/MarshalFunc/String"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v string) ([]byte, error) { + return []byte(`"called"`), nil + })), + }, + in: struct{ X any }{[]any{nil, false, "", 0.0, map[string]any{}, []any{}}}, + want: `{"X":[null,false,"called",0,{},[]]}`, + }, { + name: jsontest.Name("Interfaces/Any/MarshalFunc/Float64"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v float64) ([]byte, error) { + return []byte(`"called"`), nil + })), + }, + in: struct{ X any }{[]any{nil, false, "", 0.0, map[string]any{}, []any{}}}, + want: `{"X":[null,false,"","called",{},[]]}`, + }, { + name: jsontest.Name("Interfaces/Any/MarshalFunc/MapStringAny"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v map[string]any) ([]byte, error) { + return []byte(`"called"`), nil + })), + }, + in: struct{ X any }{[]any{nil, false, "", 0.0, map[string]any{}, []any{}}}, + want: `{"X":[null,false,"",0,"called",[]]}`, + }, { + name: jsontest.Name("Interfaces/Any/MarshalFunc/SliceAny"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v []any) ([]byte, error) { + return []byte(`"called"`), nil + })), + }, + in: struct{ X any }{[]any{nil, false, "", 0.0, map[string]any{}, []any{}}}, + want: `{"X":"called"}`, + }, { + name: jsontest.Name("Interfaces/Any/MarshalFunc/Bytes"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v [8]byte) ([]byte, error) { + return []byte(`"called"`), nil + })), + }, + in: struct{ X any }{[8]byte{}}, + want: `{"X":"called"}`, + }, { + name: jsontest.Name("Interfaces/Any/Float/NaN"), + in: struct{ X any }{math.NaN()}, + want: `{"X"`, + wantErr: EM(fmt.Errorf("unsupported value: %v", math.NaN())).withType(0, reflect.TypeFor[float64]()).withPos(`{"X":`, "/X"), + }, { + name: jsontest.Name("Interfaces/Any/Maps/Nil"), + in: struct{ X any }{map[string]any(nil)}, + want: `{"X":{}}`, + }, { + name: jsontest.Name("Interfaces/Any/Maps/Nil/FormatNilMapAsNull"), + opts: []Options{FormatNilMapAsNull(true)}, + in: struct{ X any }{map[string]any(nil)}, + want: `{"X":null}`, + }, { + name: jsontest.Name("Interfaces/Any/Maps/Empty"), + in: struct{ X any }{map[string]any{}}, + want: `{"X":{}}`, + }, { + name: jsontest.Name("Interfaces/Any/Maps/Empty/Multiline"), + opts: []Options{jsontext.Multiline(true), jsontext.WithIndent("")}, + in: struct{ X any }{map[string]any{}}, + want: "{\n\"X\": {}\n}", + }, { + name: jsontest.Name("Interfaces/Any/Maps/NonEmpty"), + in: struct{ X any }{map[string]any{"fizz": "buzz"}}, + want: `{"X":{"fizz":"buzz"}}`, + }, { + name: jsontest.Name("Interfaces/Any/Maps/Deterministic"), + opts: []Options{Deterministic(true)}, + in: struct{ X any }{map[string]any{"alpha": "", "bravo": ""}}, + want: `{"X":{"alpha":"","bravo":""}}`, + }, { + name: jsontest.Name("Interfaces/Any/Maps/Deterministic+AllowInvalidUTF8+RejectDuplicateNames"), + opts: []Options{Deterministic(true), jsontext.AllowInvalidUTF8(true), jsontext.AllowDuplicateNames(false)}, + in: struct{ X any }{map[string]any{"\xff": "", "\xfe": ""}}, + want: `{"X":{"�":""`, + wantErr: newDuplicateNameError("/X", []byte(`"�"`), len64(`{"X":{"�":"",`)), + }, { + name: jsontest.Name("Interfaces/Any/Maps/Deterministic+AllowInvalidUTF8+AllowDuplicateNames"), + opts: []Options{Deterministic(true), jsontext.AllowInvalidUTF8(true), jsontext.AllowDuplicateNames(true)}, + in: struct{ X any }{map[string]any{"\xff": "alpha", "\xfe": "bravo"}}, + want: `{"X":{"�":"bravo","�":"alpha"}}`, + }, { + name: jsontest.Name("Interfaces/Any/Maps/RejectInvalidUTF8"), + in: struct{ X any }{map[string]any{"\xff": "", "\xfe": ""}}, + want: `{"X":{`, + wantErr: newInvalidUTF8Error(len64(`{"X":{`), "/X"), + }, { + name: jsontest.Name("Interfaces/Any/Maps/AllowInvalidUTF8+RejectDuplicateNames"), + opts: []Options{jsontext.AllowInvalidUTF8(true)}, + in: struct{ X any }{map[string]any{"\xff": "", "\xfe": ""}}, + want: `{"X":{"�":""`, + wantErr: newDuplicateNameError("/X", []byte(`"�"`), len64(`{"X":{"�":"",`)), + }, { + name: jsontest.Name("Interfaces/Any/Maps/AllowInvalidUTF8+AllowDuplicateNames"), + opts: []Options{jsontext.AllowInvalidUTF8(true), jsontext.AllowDuplicateNames(true)}, + in: struct{ X any }{map[string]any{"\xff": "", "\xfe": ""}}, + want: `{"X":{"�":"","�":""}}`, + }, { + name: jsontest.Name("Interfaces/Any/Maps/Cyclic"), + in: func() any { + m := map[string]any{} + m[""] = m + return struct{ X any }{m} + }(), + want: `{"X"` + strings.Repeat(`:{""`, startDetectingCyclesAfter), + wantErr: EM(internal.ErrCycle).withPos(`{"X":`+strings.Repeat(`{"":`, startDetectingCyclesAfter), "/X"+jsontext.Pointer(strings.Repeat("/", startDetectingCyclesAfter))).withType(0, T[map[string]any]()), + }, { + name: jsontest.Name("Interfaces/Any/Slices/Nil"), + in: struct{ X any }{[]any(nil)}, + want: `{"X":[]}`, + }, { + name: jsontest.Name("Interfaces/Any/Slices/Nil/FormatNilSliceAsNull"), + opts: []Options{FormatNilSliceAsNull(true)}, + in: struct{ X any }{[]any(nil)}, + want: `{"X":null}`, + }, { + name: jsontest.Name("Interfaces/Any/Slices/Empty"), + in: struct{ X any }{[]any{}}, + want: `{"X":[]}`, + }, { + name: jsontest.Name("Interfaces/Any/Slices/Empty/Multiline"), + opts: []Options{jsontext.Multiline(true), jsontext.WithIndent("")}, + in: struct{ X any }{[]any{}}, + want: "{\n\"X\": []\n}", + }, { + name: jsontest.Name("Interfaces/Any/Slices/NonEmpty"), + in: struct{ X any }{[]any{"fizz", "buzz"}}, + want: `{"X":["fizz","buzz"]}`, + }, { + name: jsontest.Name("Interfaces/Any/Slices/Cyclic"), + in: func() any { + s := make([]any, 1) + s[0] = s + return struct{ X any }{s} + }(), + want: `{"X":` + strings.Repeat(`[`, startDetectingCyclesAfter), + wantErr: EM(internal.ErrCycle).withPos(`{"X":`+strings.Repeat(`[`, startDetectingCyclesAfter), "/X"+jsontext.Pointer(strings.Repeat("/0", startDetectingCyclesAfter))).withType(0, T[[]any]()), + }, { + name: jsontest.Name("Methods/NilPointer"), + in: struct{ X *allMethods }{X: (*allMethods)(nil)}, // method should not be called + want: `{"X":null}`, + }, { + // NOTE: Fixes https://github.com/dominikh/go-tools/issues/975. + name: jsontest.Name("Methods/NilInterface"), + in: struct{ X MarshalerTo }{X: (*allMethods)(nil)}, // method should not be called + want: `{"X":null}`, + }, { + name: jsontest.Name("Methods/AllMethods"), + in: struct{ X *allMethods }{X: &allMethods{method: "MarshalJSONTo", value: []byte(`"hello"`)}}, + want: `{"X":"hello"}`, + }, { + name: jsontest.Name("Methods/AllMethodsExceptJSONv2"), + in: struct{ X *allMethodsExceptJSONv2 }{X: &allMethodsExceptJSONv2{allMethods: allMethods{method: "MarshalJSON", value: []byte(`"hello"`)}}}, + want: `{"X":"hello"}`, + }, { + name: jsontest.Name("Methods/AllMethodsExceptJSONv1"), + in: struct{ X *allMethodsExceptJSONv1 }{X: &allMethodsExceptJSONv1{allMethods: allMethods{method: "MarshalJSONTo", value: []byte(`"hello"`)}}}, + want: `{"X":"hello"}`, + }, { + name: jsontest.Name("Methods/AllMethodsExceptText"), + in: struct{ X *allMethodsExceptText }{X: &allMethodsExceptText{allMethods: allMethods{method: "MarshalJSONTo", value: []byte(`"hello"`)}}}, + want: `{"X":"hello"}`, + }, { + name: jsontest.Name("Methods/OnlyMethodJSONv2"), + in: struct{ X *onlyMethodJSONv2 }{X: &onlyMethodJSONv2{allMethods: allMethods{method: "MarshalJSONTo", value: []byte(`"hello"`)}}}, + want: `{"X":"hello"}`, + }, { + name: jsontest.Name("Methods/OnlyMethodJSONv1"), + in: struct{ X *onlyMethodJSONv1 }{X: &onlyMethodJSONv1{allMethods: allMethods{method: "MarshalJSON", value: []byte(`"hello"`)}}}, + want: `{"X":"hello"}`, + }, { + name: jsontest.Name("Methods/OnlyMethodText"), + in: struct{ X *onlyMethodText }{X: &onlyMethodText{allMethods: allMethods{method: "MarshalText", value: []byte(`hello`)}}}, + want: `{"X":"hello"}`, + }, { + name: jsontest.Name("Methods/IP"), + in: net.IPv4(192, 168, 0, 100), + want: `"192.168.0.100"`, + }, { + name: jsontest.Name("Methods/NetIP"), + in: struct { + Addr netip.Addr + AddrPort netip.AddrPort + Prefix netip.Prefix + }{ + Addr: netip.AddrFrom4([4]byte{1, 2, 3, 4}), + AddrPort: netip.AddrPortFrom(netip.AddrFrom4([4]byte{1, 2, 3, 4}), 1234), + Prefix: netip.PrefixFrom(netip.AddrFrom4([4]byte{1, 2, 3, 4}), 24), + }, + want: `{"Addr":"1.2.3.4","AddrPort":"1.2.3.4:1234","Prefix":"1.2.3.4/24"}`, + }, { + // NOTE: Fixes https://go.dev/issue/46516. + name: jsontest.Name("Methods/Anonymous"), + in: struct{ X struct{ allMethods } }{X: struct{ allMethods }{allMethods{method: "MarshalJSONTo", value: []byte(`"hello"`)}}}, + want: `{"X":"hello"}`, + }, { + // NOTE: Fixes https://go.dev/issue/22967. + name: jsontest.Name("Methods/Addressable"), + in: struct { + V allMethods + M map[string]allMethods + I any + }{ + V: allMethods{method: "MarshalJSONTo", value: []byte(`"hello"`)}, + M: map[string]allMethods{"K": {method: "MarshalJSONTo", value: []byte(`"hello"`)}}, + I: allMethods{method: "MarshalJSONTo", value: []byte(`"hello"`)}, + }, + want: `{"V":"hello","M":{"K":"hello"},"I":"hello"}`, + }, { + // NOTE: Fixes https://go.dev/issue/29732. + name: jsontest.Name("Methods/MapKey/JSONv2"), + in: map[structMethodJSONv2]string{{"k1"}: "v1", {"k2"}: "v2"}, + want: `{"k1":"v1","k2":"v2"}`, + canonicalize: true, + }, { + // NOTE: Fixes https://go.dev/issue/29732. + name: jsontest.Name("Methods/MapKey/JSONv1"), + in: map[structMethodJSONv1]string{{"k1"}: "v1", {"k2"}: "v2"}, + want: `{"k1":"v1","k2":"v2"}`, + canonicalize: true, + }, { + name: jsontest.Name("Methods/MapKey/Text"), + in: map[structMethodText]string{{"k1"}: "v1", {"k2"}: "v2"}, + want: `{"k1":"v1","k2":"v2"}`, + canonicalize: true, + }, { + name: jsontest.Name("Methods/Invalid/JSONv2/Error"), + in: marshalJSONv2Func(func(*jsontext.Encoder) error { + return errSomeError + }), + wantErr: EM(errSomeError).withType(0, T[marshalJSONv2Func]()), + }, { + name: jsontest.Name("Methods/Invalid/JSONv2/TooFew"), + in: marshalJSONv2Func(func(*jsontext.Encoder) error { + return nil // do nothing + }), + wantErr: EM(errNonSingularValue).withType(0, T[marshalJSONv2Func]()), + }, { + name: jsontest.Name("Methods/Invalid/JSONv2/TooMany"), + in: marshalJSONv2Func(func(enc *jsontext.Encoder) error { + enc.WriteToken(jsontext.Null) + enc.WriteToken(jsontext.Null) + return nil + }), + want: `nullnull`, + wantErr: EM(errNonSingularValue).withPos(`nullnull`, "").withType(0, T[marshalJSONv2Func]()), + }, { + name: jsontest.Name("Methods/Invalid/JSONv2/SkipFunc"), + in: marshalJSONv2Func(func(enc *jsontext.Encoder) error { + return SkipFunc + }), + wantErr: EM(errors.New("marshal method cannot be skipped")).withType(0, T[marshalJSONv2Func]()), + }, { + name: jsontest.Name("Methods/Invalid/JSONv1/Error"), + in: marshalJSONv1Func(func() ([]byte, error) { + return nil, errSomeError + }), + wantErr: EM(errSomeError).withType(0, T[marshalJSONv1Func]()), + }, { + name: jsontest.Name("Methods/Invalid/JSONv1/Syntax"), + in: marshalJSONv1Func(func() ([]byte, error) { + return []byte("invalid"), nil + }), + wantErr: EM(newInvalidCharacterError("i", "at start of value", 0, "")).withType(0, T[marshalJSONv1Func]()), + }, { + name: jsontest.Name("Methods/Invalid/JSONv1/SkipFunc"), + in: marshalJSONv1Func(func() ([]byte, error) { + return nil, SkipFunc + }), + wantErr: EM(errors.New("marshal method cannot be skipped")).withType(0, T[marshalJSONv1Func]()), + }, { + name: jsontest.Name("Methods/AppendText"), + in: appendTextFunc(func(b []byte) ([]byte, error) { return append(b, "hello"...), nil }), + want: `"hello"`, + }, { + name: jsontest.Name("Methods/AppendText/Error"), + in: appendTextFunc(func(b []byte) ([]byte, error) { return append(b, "hello"...), errSomeError }), + wantErr: EM(errSomeError).withType(0, T[appendTextFunc]()), + }, { + name: jsontest.Name("Methods/AppendText/NeedEscape"), + in: appendTextFunc(func(b []byte) ([]byte, error) { return append(b, `"`...), nil }), + want: `"\""`, + }, { + name: jsontest.Name("Methods/AppendText/RejectInvalidUTF8"), + in: appendTextFunc(func(b []byte) ([]byte, error) { return append(b, "\xde\xad\xbe\xef"...), nil }), + wantErr: EM(newInvalidUTF8Error(0, "")).withType(0, T[appendTextFunc]()), + }, { + name: jsontest.Name("Methods/AppendText/AllowInvalidUTF8"), + opts: []Options{jsontext.AllowInvalidUTF8(true)}, + in: appendTextFunc(func(b []byte) ([]byte, error) { return append(b, "\xde\xad\xbe\xef"...), nil }), + want: "\"\xde\xad\ufffd\ufffd\"", + }, { + name: jsontest.Name("Methods/Invalid/Text/Error"), + in: marshalTextFunc(func() ([]byte, error) { + return nil, errSomeError + }), + wantErr: EM(errSomeError).withType(0, T[marshalTextFunc]()), + }, { + name: jsontest.Name("Methods/Text/RejectInvalidUTF8"), + in: marshalTextFunc(func() ([]byte, error) { + return []byte("\xde\xad\xbe\xef"), nil + }), + wantErr: EM(newInvalidUTF8Error(0, "")).withType(0, T[marshalTextFunc]()), + }, { + name: jsontest.Name("Methods/Text/AllowInvalidUTF8"), + opts: []Options{jsontext.AllowInvalidUTF8(true)}, + in: marshalTextFunc(func() ([]byte, error) { + return []byte("\xde\xad\xbe\xef"), nil + }), + want: "\"\xde\xad\ufffd\ufffd\"", + }, { + name: jsontest.Name("Methods/Invalid/Text/SkipFunc"), + in: marshalTextFunc(func() ([]byte, error) { + return nil, SkipFunc + }), + wantErr: EM(wrapSkipFunc(SkipFunc, "marshal method")).withType(0, T[marshalTextFunc]()), + }, { + name: jsontest.Name("Methods/Invalid/MapKey/JSONv2/Syntax"), + in: map[any]string{ + addr(marshalJSONv2Func(func(enc *jsontext.Encoder) error { + return enc.WriteToken(jsontext.Null) + })): "invalid", + }, + want: `{`, + wantErr: EM(newNonStringNameError(len64(`{`), "")).withPos(`{`, "").withType(0, T[marshalJSONv2Func]()), + }, { + name: jsontest.Name("Methods/Invalid/MapKey/JSONv1/Syntax"), + in: map[any]string{ + addr(marshalJSONv1Func(func() ([]byte, error) { + return []byte(`null`), nil + })): "invalid", + }, + want: `{`, + wantErr: EM(newNonStringNameError(len64(`{`), "")).withPos(`{`, "").withType(0, T[marshalJSONv1Func]()), + }, { + name: jsontest.Name("Functions/Bool/V1"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(bool) ([]byte, error) { + return []byte(`"called"`), nil + })), + }, + in: true, + want: `"called"`, + }, { + name: jsontest.Name("Functions/Bool/Empty"), + opts: []Options{WithMarshalers(nil)}, + in: true, + want: `true`, + }, { + name: jsontest.Name("Functions/NamedBool/V1/NoMatch"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(namedBool) ([]byte, error) { + return nil, errMustNotCall + })), + }, + in: true, + want: `true`, + }, { + name: jsontest.Name("Functions/NamedBool/V1/Match"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(namedBool) ([]byte, error) { + return []byte(`"called"`), nil + })), + }, + in: namedBool(true), + want: `"called"`, + }, { + name: jsontest.Name("Functions/PointerBool/V1/Match"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v *bool) ([]byte, error) { + _ = *v // must be a non-nil pointer + return []byte(`"called"`), nil + })), + }, + in: true, + want: `"called"`, + }, { + name: jsontest.Name("Functions/Bool/V2"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v bool) error { + return enc.WriteToken(jsontext.String("called")) + })), + }, + in: true, + want: `"called"`, + }, { + name: jsontest.Name("Functions/NamedBool/V2/NoMatch"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v namedBool) error { + return errMustNotCall + })), + }, + in: true, + want: `true`, + }, { + name: jsontest.Name("Functions/NamedBool/V2/Match"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v namedBool) error { + return enc.WriteToken(jsontext.String("called")) + })), + }, + in: namedBool(true), + want: `"called"`, + }, { + name: jsontest.Name("Functions/PointerBool/V2/Match"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v *bool) error { + _ = *v // must be a non-nil pointer + return enc.WriteToken(jsontext.String("called")) + })), + }, + in: true, + want: `"called"`, + }, { + name: jsontest.Name("Functions/Bool/Empty1/NoMatch"), + opts: []Options{ + WithMarshalers(new(Marshalers)), + }, + in: true, + want: `true`, + }, { + name: jsontest.Name("Functions/Bool/Empty2/NoMatch"), + opts: []Options{ + WithMarshalers(JoinMarshalers()), + }, + in: true, + want: `true`, + }, { + name: jsontest.Name("Functions/Bool/V1/DirectError"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(bool) ([]byte, error) { + return nil, errSomeError + })), + }, + in: true, + wantErr: EM(errSomeError).withType(0, T[bool]()), + }, { + name: jsontest.Name("Functions/Bool/V1/SkipError"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(bool) ([]byte, error) { + return nil, SkipFunc + })), + }, + in: true, + wantErr: EM(wrapSkipFunc(SkipFunc, "marshal function of type func(T) ([]byte, error)")).withType(0, T[bool]()), + }, { + name: jsontest.Name("Functions/Bool/V1/InvalidValue"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(bool) ([]byte, error) { + return []byte("invalid"), nil + })), + }, + in: true, + wantErr: EM(newInvalidCharacterError("i", "at start of value", 0, "")).withType(0, T[bool]()), + }, { + name: jsontest.Name("Functions/Bool/V2/DirectError"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v bool) error { + return errSomeError + })), + }, + in: true, + wantErr: EM(errSomeError).withType(0, T[bool]()), + }, { + name: jsontest.Name("Functions/Bool/V2/TooFew"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v bool) error { + return nil + })), + }, + in: true, + wantErr: EM(errNonSingularValue).withType(0, T[bool]()), + }, { + name: jsontest.Name("Functions/Bool/V2/TooMany"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v bool) error { + enc.WriteValue([]byte(`"hello"`)) + enc.WriteValue([]byte(`"world"`)) + return nil + })), + }, + in: true, + want: `"hello""world"`, + wantErr: EM(errNonSingularValue).withPos(`"hello""world"`, "").withType(0, T[bool]()), + }, { + name: jsontest.Name("Functions/Bool/V2/Skipped"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v bool) error { + return SkipFunc + })), + }, + in: true, + want: `true`, + }, { + name: jsontest.Name("Functions/Bool/V2/ProcessBeforeSkip"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v bool) error { + enc.WriteValue([]byte(`"hello"`)) + return SkipFunc + })), + }, + in: true, + want: `"hello"`, + wantErr: EM(errSkipMutation).withPos(`"hello"`, "").withType(0, T[bool]()), + }, { + name: jsontest.Name("Functions/Bool/V2/WrappedSkipError"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v bool) error { + return fmt.Errorf("wrap: %w", SkipFunc) + })), + }, + in: true, + wantErr: EM(fmt.Errorf("wrap: %w", SkipFunc)).withType(0, T[bool]()), + }, { + name: jsontest.Name("Functions/Map/Key/NoCaseString/V1"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v nocaseString) ([]byte, error) { + return []byte(`"called"`), nil + })), + }, + in: map[nocaseString]string{"hello": "world"}, + want: `{"called":"world"}`, + }, { + name: jsontest.Name("Functions/Map/Key/PointerNoCaseString/V1"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v *nocaseString) ([]byte, error) { + _ = *v // must be a non-nil pointer + return []byte(`"called"`), nil + })), + }, + in: map[nocaseString]string{"hello": "world"}, + want: `{"called":"world"}`, + }, { + name: jsontest.Name("Functions/Map/Key/TextMarshaler/V1"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v encoding.TextMarshaler) ([]byte, error) { + _ = *v.(*nocaseString) // must be a non-nil *nocaseString + return []byte(`"called"`), nil + })), + }, + in: map[nocaseString]string{"hello": "world"}, + want: `{"called":"world"}`, + }, { + name: jsontest.Name("Functions/Map/Key/NoCaseString/V1/InvalidValue"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v nocaseString) ([]byte, error) { + return []byte(`null`), nil + })), + }, + in: map[nocaseString]string{"hello": "world"}, + want: `{`, + wantErr: EM(newNonStringNameError(len64(`{`), "")).withPos(`{`, "").withType(0, T[nocaseString]()), + }, { + name: jsontest.Name("Functions/Map/Key/NoCaseString/V2/InvalidKind"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v nocaseString) ([]byte, error) { + return []byte(`null`), nil + })), + }, + in: map[nocaseString]string{"hello": "world"}, + want: `{`, + wantErr: EM(newNonStringNameError(len64(`{`), "")).withPos(`{`, "").withType(0, T[nocaseString]()), + }, { + name: jsontest.Name("Functions/Map/Key/String/V1/DuplicateName"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v string) ([]byte, error) { + return []byte(`"name"`), nil + })), + }, + in: map[string]string{"name1": "value", "name2": "value"}, + want: `{"name":"name"`, + wantErr: EM(newDuplicateNameError("", []byte(`"name"`), len64(`{"name":"name",`))). + withPos(`{"name":"name",`, "").withType(0, T[string]()), + }, { + name: jsontest.Name("Functions/Map/Key/NoCaseString/V2"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v nocaseString) error { + return enc.WriteValue([]byte(`"called"`)) + })), + }, + in: map[nocaseString]string{"hello": "world"}, + want: `{"called":"world"}`, + }, { + name: jsontest.Name("Functions/Map/Key/PointerNoCaseString/V2"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v *nocaseString) error { + _ = *v // must be a non-nil pointer + return enc.WriteValue([]byte(`"called"`)) + })), + }, + in: map[nocaseString]string{"hello": "world"}, + want: `{"called":"world"}`, + }, { + name: jsontest.Name("Functions/Map/Key/TextMarshaler/V2"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v encoding.TextMarshaler) error { + _ = *v.(*nocaseString) // must be a non-nil *nocaseString + return enc.WriteValue([]byte(`"called"`)) + })), + }, + in: map[nocaseString]string{"hello": "world"}, + want: `{"called":"world"}`, + }, { + name: jsontest.Name("Functions/Map/Key/NoCaseString/V2/InvalidToken"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v nocaseString) error { + return enc.WriteToken(jsontext.Null) + })), + }, + in: map[nocaseString]string{"hello": "world"}, + want: `{`, + wantErr: EM(newNonStringNameError(len64(`{`), "")).withPos(`{`, "").withType(0, T[nocaseString]()), + }, { + name: jsontest.Name("Functions/Map/Key/NoCaseString/V2/InvalidValue"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v nocaseString) error { + return enc.WriteValue([]byte(`null`)) + })), + }, + in: map[nocaseString]string{"hello": "world"}, + want: `{`, + wantErr: EM(newNonStringNameError(len64(`{`), "")).withPos(`{`, "").withType(0, T[nocaseString]()), + }, { + name: jsontest.Name("Functions/Map/Value/NoCaseString/V1"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v nocaseString) ([]byte, error) { + return []byte(`"called"`), nil + })), + }, + in: map[string]nocaseString{"hello": "world"}, + want: `{"hello":"called"}`, + }, { + name: jsontest.Name("Functions/Map/Value/PointerNoCaseString/V1"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v *nocaseString) ([]byte, error) { + _ = *v // must be a non-nil pointer + return []byte(`"called"`), nil + })), + }, + in: map[string]nocaseString{"hello": "world"}, + want: `{"hello":"called"}`, + }, { + name: jsontest.Name("Functions/Map/Value/TextMarshaler/V1"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v encoding.TextMarshaler) ([]byte, error) { + _ = *v.(*nocaseString) // must be a non-nil *nocaseString + return []byte(`"called"`), nil + })), + }, + in: map[string]nocaseString{"hello": "world"}, + want: `{"hello":"called"}`, + }, { + name: jsontest.Name("Functions/Map/Value/NoCaseString/V2"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v nocaseString) error { + return enc.WriteValue([]byte(`"called"`)) + })), + }, + in: map[string]nocaseString{"hello": "world"}, + want: `{"hello":"called"}`, + }, { + name: jsontest.Name("Functions/Map/Value/PointerNoCaseString/V2"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v *nocaseString) error { + _ = *v // must be a non-nil pointer + return enc.WriteValue([]byte(`"called"`)) + })), + }, + in: map[string]nocaseString{"hello": "world"}, + want: `{"hello":"called"}`, + }, { + name: jsontest.Name("Functions/Map/Value/TextMarshaler/V2"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v encoding.TextMarshaler) error { + _ = *v.(*nocaseString) // must be a non-nil *nocaseString + return enc.WriteValue([]byte(`"called"`)) + })), + }, + in: map[string]nocaseString{"hello": "world"}, + want: `{"hello":"called"}`, + }, { + name: jsontest.Name("Funtions/Struct/Fields"), + opts: []Options{ + WithMarshalers(JoinMarshalers( + MarshalFunc(func(v bool) ([]byte, error) { + return []byte(`"called1"`), nil + }), + MarshalFunc(func(v *string) ([]byte, error) { + return []byte(`"called2"`), nil + }), + MarshalToFunc(func(enc *jsontext.Encoder, v []byte) error { + return enc.WriteValue([]byte(`"called3"`)) + }), + MarshalToFunc(func(enc *jsontext.Encoder, v *int64) error { + return enc.WriteValue([]byte(`"called4"`)) + }), + )), + }, + in: structScalars{}, + want: `{"Bool":"called1","String":"called2","Bytes":"called3","Int":"called4","Uint":0,"Float":0}`, + }, { + name: jsontest.Name("Functions/Struct/OmitEmpty"), + opts: []Options{ + WithMarshalers(JoinMarshalers( + MarshalFunc(func(v bool) ([]byte, error) { + return []byte(`null`), nil + }), + MarshalFunc(func(v string) ([]byte, error) { + return []byte(`"called1"`), nil + }), + MarshalFunc(func(v *stringMarshalNonEmpty) ([]byte, error) { + return []byte(`""`), nil + }), + MarshalToFunc(func(enc *jsontext.Encoder, v bytesMarshalNonEmpty) error { + return enc.WriteValue([]byte(`{}`)) + }), + MarshalToFunc(func(enc *jsontext.Encoder, v *float64) error { + return enc.WriteValue([]byte(`[]`)) + }), + MarshalFunc(func(v mapMarshalNonEmpty) ([]byte, error) { + return []byte(`"called2"`), nil + }), + MarshalFunc(func(v []string) ([]byte, error) { + return []byte(`"called3"`), nil + }), + MarshalToFunc(func(enc *jsontext.Encoder, v *sliceMarshalNonEmpty) error { + return enc.WriteValue([]byte(`"called4"`)) + }), + )), + }, + in: structOmitEmptyAll{}, + want: `{"String":"called1","MapNonEmpty":"called2","Slice":"called3","SliceNonEmpty":"called4"}`, + }, { + name: jsontest.Name("Functions/Struct/OmitZero"), + opts: []Options{ + WithMarshalers(JoinMarshalers( + MarshalFunc(func(v bool) ([]byte, error) { + panic("should not be called") + }), + MarshalFunc(func(v *string) ([]byte, error) { + panic("should not be called") + }), + MarshalToFunc(func(enc *jsontext.Encoder, v []byte) error { + panic("should not be called") + }), + MarshalToFunc(func(enc *jsontext.Encoder, v *int64) error { + panic("should not be called") + }), + )), + }, + in: structOmitZeroAll{}, + want: `{}`, + }, { + name: jsontest.Name("Functions/Struct/Inlined"), + opts: []Options{ + WithMarshalers(JoinMarshalers( + MarshalFunc(func(v structInlinedL1) ([]byte, error) { + panic("should not be called") + }), + MarshalToFunc(func(enc *jsontext.Encoder, v *StructEmbed2) error { + panic("should not be called") + }), + )), + }, + in: structInlined{}, + want: `{"D":""}`, + }, { + name: jsontest.Name("Functions/Slice/Elem"), + opts: []Options{ + WithMarshalers(MarshalFunc(func(v bool) ([]byte, error) { + return []byte(`"` + strconv.FormatBool(v) + `"`), nil + })), + }, + in: []bool{true, false}, + want: `["true","false"]`, + }, { + name: jsontest.Name("Functions/Array/Elem"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v *bool) error { + return enc.WriteValue([]byte(`"` + strconv.FormatBool(*v) + `"`)) + })), + }, + in: [2]bool{true, false}, + want: `["true","false"]`, + }, { + name: jsontest.Name("Functions/Pointer/Nil"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v *bool) error { + panic("should not be called") + })), + }, + in: struct{ X *bool }{nil}, + want: `{"X":null}`, + }, { + name: jsontest.Name("Functions/Pointer/NonNil"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v *bool) error { + return enc.WriteValue([]byte(`"called"`)) + })), + }, + in: struct{ X *bool }{addr(false)}, + want: `{"X":"called"}`, + }, { + name: jsontest.Name("Functions/Interface/Nil"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v fmt.Stringer) error { + panic("should not be called") + })), + }, + in: struct{ X fmt.Stringer }{nil}, + want: `{"X":null}`, + }, { + name: jsontest.Name("Functions/Interface/NonNil/MatchInterface"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v fmt.Stringer) error { + return enc.WriteValue([]byte(`"called"`)) + })), + }, + in: struct{ X fmt.Stringer }{valueStringer{}}, + want: `{"X":"called"}`, + }, { + name: jsontest.Name("Functions/Interface/NonNil/MatchConcrete"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v valueStringer) error { + return enc.WriteValue([]byte(`"called"`)) + })), + }, + in: struct{ X fmt.Stringer }{valueStringer{}}, + want: `{"X":"called"}`, + }, { + name: jsontest.Name("Functions/Interface/NonNil/MatchPointer"), + opts: []Options{ + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, v *valueStringer) error { + return enc.WriteValue([]byte(`"called"`)) + })), + }, + in: struct{ X fmt.Stringer }{valueStringer{}}, + want: `{"X":"called"}`, + }, { + name: jsontest.Name("Functions/Interface/Any"), + in: []any{ + nil, // nil + valueStringer{}, // T + (*valueStringer)(nil), // *T + addr(valueStringer{}), // *T + (**valueStringer)(nil), // **T + addr((*valueStringer)(nil)), // **T + addr(addr(valueStringer{})), // **T + pointerStringer{}, // T + (*pointerStringer)(nil), // *T + addr(pointerStringer{}), // *T + (**pointerStringer)(nil), // **T + addr((*pointerStringer)(nil)), // **T + addr(addr(pointerStringer{})), // **T + "LAST", + }, + want: `[null,{},null,{},null,null,{},{},null,{},null,null,{},"LAST"]`, + opts: []Options{ + WithMarshalers(func() *Marshalers { + type P struct { + D int + N int64 + } + type PV struct { + P P + V any + } + + var lastChecks []func() error + checkLast := func() error { + for _, fn := range lastChecks { + if err := fn(); err != nil { + return err + } + } + return SkipFunc + } + makeValueChecker := func(name string, want []PV) func(e *jsontext.Encoder, v any) error { + checkNext := func(e *jsontext.Encoder, v any) error { + xe := export.Encoder(e) + p := P{len(xe.Tokens.Stack), xe.Tokens.Last.Length()} + rv := reflect.ValueOf(v) + pv := PV{p, v} + switch { + case len(want) == 0: + return fmt.Errorf("%s: %v: got more values than expected", name, p) + case !rv.IsValid() || rv.Kind() != reflect.Pointer || rv.IsNil(): + return fmt.Errorf("%s: %v: got %#v, want non-nil pointer type", name, p, v) + case !reflect.DeepEqual(pv, want[0]): + return fmt.Errorf("%s:\n\tgot %#v\n\twant %#v", name, pv, want[0]) + default: + want = want[1:] + return SkipFunc + } + } + lastChecks = append(lastChecks, func() error { + if len(want) > 0 { + return fmt.Errorf("%s: did not get enough values, want %d more", name, len(want)) + } + return nil + }) + return checkNext + } + makePositionChecker := func(name string, want []P) func(e *jsontext.Encoder, v any) error { + checkNext := func(e *jsontext.Encoder, v any) error { + xe := export.Encoder(e) + p := P{len(xe.Tokens.Stack), xe.Tokens.Last.Length()} + switch { + case len(want) == 0: + return fmt.Errorf("%s: %v: got more values than wanted", name, p) + case p != want[0]: + return fmt.Errorf("%s: got %v, want %v", name, p, want[0]) + default: + want = want[1:] + return SkipFunc + } + } + lastChecks = append(lastChecks, func() error { + if len(want) > 0 { + return fmt.Errorf("%s: did not get enough values, want %d more", name, len(want)) + } + return nil + }) + return checkNext + } + + wantAny := []PV{ + {P{0, 0}, addr([]any{ + nil, + valueStringer{}, + (*valueStringer)(nil), + addr(valueStringer{}), + (**valueStringer)(nil), + addr((*valueStringer)(nil)), + addr(addr(valueStringer{})), + pointerStringer{}, + (*pointerStringer)(nil), + addr(pointerStringer{}), + (**pointerStringer)(nil), + addr((*pointerStringer)(nil)), + addr(addr(pointerStringer{})), + "LAST", + })}, + {P{1, 0}, addr(any(nil))}, + {P{1, 1}, addr(any(valueStringer{}))}, + {P{1, 1}, addr(valueStringer{})}, + {P{1, 2}, addr(any((*valueStringer)(nil)))}, + {P{1, 2}, addr((*valueStringer)(nil))}, + {P{1, 3}, addr(any(addr(valueStringer{})))}, + {P{1, 3}, addr(addr(valueStringer{}))}, + {P{1, 3}, addr(valueStringer{})}, + {P{1, 4}, addr(any((**valueStringer)(nil)))}, + {P{1, 4}, addr((**valueStringer)(nil))}, + {P{1, 5}, addr(any(addr((*valueStringer)(nil))))}, + {P{1, 5}, addr(addr((*valueStringer)(nil)))}, + {P{1, 5}, addr((*valueStringer)(nil))}, + {P{1, 6}, addr(any(addr(addr(valueStringer{}))))}, + {P{1, 6}, addr(addr(addr(valueStringer{})))}, + {P{1, 6}, addr(addr(valueStringer{}))}, + {P{1, 6}, addr(valueStringer{})}, + {P{1, 7}, addr(any(pointerStringer{}))}, + {P{1, 7}, addr(pointerStringer{})}, + {P{1, 8}, addr(any((*pointerStringer)(nil)))}, + {P{1, 8}, addr((*pointerStringer)(nil))}, + {P{1, 9}, addr(any(addr(pointerStringer{})))}, + {P{1, 9}, addr(addr(pointerStringer{}))}, + {P{1, 9}, addr(pointerStringer{})}, + {P{1, 10}, addr(any((**pointerStringer)(nil)))}, + {P{1, 10}, addr((**pointerStringer)(nil))}, + {P{1, 11}, addr(any(addr((*pointerStringer)(nil))))}, + {P{1, 11}, addr(addr((*pointerStringer)(nil)))}, + {P{1, 11}, addr((*pointerStringer)(nil))}, + {P{1, 12}, addr(any(addr(addr(pointerStringer{}))))}, + {P{1, 12}, addr(addr(addr(pointerStringer{})))}, + {P{1, 12}, addr(addr(pointerStringer{}))}, + {P{1, 12}, addr(pointerStringer{})}, + {P{1, 13}, addr(any("LAST"))}, + {P{1, 13}, addr("LAST")}, + } + checkAny := makeValueChecker("any", wantAny) + anyMarshaler := MarshalToFunc(func(enc *jsontext.Encoder, v any) error { + return checkAny(enc, v) + }) + + var wantPointerAny []PV + for _, v := range wantAny { + if _, ok := v.V.(*any); ok { + wantPointerAny = append(wantPointerAny, v) + } + } + checkPointerAny := makeValueChecker("*any", wantPointerAny) + pointerAnyMarshaler := MarshalToFunc(func(enc *jsontext.Encoder, v *any) error { + return checkPointerAny(enc, v) + }) + + checkNamedAny := makeValueChecker("namedAny", wantAny) + namedAnyMarshaler := MarshalToFunc(func(enc *jsontext.Encoder, v namedAny) error { + return checkNamedAny(enc, v) + }) + + checkPointerNamedAny := makeValueChecker("*namedAny", nil) + pointerNamedAnyMarshaler := MarshalToFunc(func(enc *jsontext.Encoder, v *namedAny) error { + return checkPointerNamedAny(enc, v) + }) + + type stringer = fmt.Stringer + var wantStringer []PV + for _, v := range wantAny { + if _, ok := v.V.(stringer); ok { + wantStringer = append(wantStringer, v) + } + } + checkStringer := makeValueChecker("stringer", wantStringer) + stringerMarshaler := MarshalToFunc(func(enc *jsontext.Encoder, v stringer) error { + return checkStringer(enc, v) + }) + + checkPointerStringer := makeValueChecker("*stringer", nil) + pointerStringerMarshaler := MarshalToFunc(func(enc *jsontext.Encoder, v *stringer) error { + return checkPointerStringer(enc, v) + }) + + wantValueStringer := []P{{1, 1}, {1, 3}, {1, 6}} + checkValueValueStringer := makePositionChecker("valueStringer", wantValueStringer) + valueValueStringerMarshaler := MarshalToFunc(func(enc *jsontext.Encoder, v valueStringer) error { + return checkValueValueStringer(enc, v) + }) + + checkPointerValueStringer := makePositionChecker("*valueStringer", wantValueStringer) + pointerValueStringerMarshaler := MarshalToFunc(func(enc *jsontext.Encoder, v *valueStringer) error { + return checkPointerValueStringer(enc, v) + }) + + wantPointerStringer := []P{{1, 7}, {1, 9}, {1, 12}} + checkValuePointerStringer := makePositionChecker("pointerStringer", wantPointerStringer) + valuePointerStringerMarshaler := MarshalToFunc(func(enc *jsontext.Encoder, v pointerStringer) error { + return checkValuePointerStringer(enc, v) + }) + + checkPointerPointerStringer := makePositionChecker("*pointerStringer", wantPointerStringer) + pointerPointerStringerMarshaler := MarshalToFunc(func(enc *jsontext.Encoder, v *pointerStringer) error { + return checkPointerPointerStringer(enc, v) + }) + + lastMarshaler := MarshalToFunc(func(enc *jsontext.Encoder, v string) error { + return checkLast() + }) + + return JoinMarshalers( + anyMarshaler, + pointerAnyMarshaler, + namedAnyMarshaler, + pointerNamedAnyMarshaler, // never called + stringerMarshaler, + pointerStringerMarshaler, // never called + valueValueStringerMarshaler, + pointerValueStringerMarshaler, + valuePointerStringerMarshaler, + pointerPointerStringerMarshaler, + lastMarshaler, + ) + }()), + }, + }, { + name: jsontest.Name("Functions/Precedence/V1First"), + opts: []Options{ + WithMarshalers(JoinMarshalers( + MarshalFunc(func(bool) ([]byte, error) { + return []byte(`"called"`), nil + }), + MarshalToFunc(func(enc *jsontext.Encoder, v bool) error { + panic("should not be called") + }), + )), + }, + in: true, + want: `"called"`, + }, { + name: jsontest.Name("Functions/Precedence/V2First"), + opts: []Options{ + WithMarshalers(JoinMarshalers( + MarshalToFunc(func(enc *jsontext.Encoder, v bool) error { + return enc.WriteToken(jsontext.String("called")) + }), + MarshalFunc(func(bool) ([]byte, error) { + panic("should not be called") + }), + )), + }, + in: true, + want: `"called"`, + }, { + name: jsontest.Name("Functions/Precedence/V2Skipped"), + opts: []Options{ + WithMarshalers(JoinMarshalers( + MarshalToFunc(func(enc *jsontext.Encoder, v bool) error { + return SkipFunc + }), + MarshalFunc(func(bool) ([]byte, error) { + return []byte(`"called"`), nil + }), + )), + }, + in: true, + want: `"called"`, + }, { + name: jsontest.Name("Functions/Precedence/NestedFirst"), + opts: []Options{ + WithMarshalers(JoinMarshalers( + JoinMarshalers( + MarshalFunc(func(bool) ([]byte, error) { + return []byte(`"called"`), nil + }), + ), + MarshalFunc(func(bool) ([]byte, error) { + panic("should not be called") + }), + )), + }, + in: true, + want: `"called"`, + }, { + name: jsontest.Name("Functions/Precedence/NestedLast"), + opts: []Options{ + WithMarshalers(JoinMarshalers( + MarshalFunc(func(bool) ([]byte, error) { + return []byte(`"called"`), nil + }), + JoinMarshalers( + MarshalFunc(func(bool) ([]byte, error) { + panic("should not be called") + }), + ), + )), + }, + in: true, + want: `"called"`, + }, { + name: jsontest.Name("Duration/Zero"), + in: struct { + D1 time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + D2 time.Duration `json:",format:nano"` + }{0, 0}, + want: `{"D1":"0s","D2":0}`, + }, { + name: jsontest.Name("Duration/Positive"), + in: struct { + D1 time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + D2 time.Duration `json:",format:nano"` + }{ + 123456789123456789, + 123456789123456789, + }, + want: `{"D1":"34293h33m9.123456789s","D2":123456789123456789}`, + }, { + name: jsontest.Name("Duration/Negative"), + in: struct { + D1 time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + D2 time.Duration `json:",format:nano"` + }{ + -123456789123456789, + -123456789123456789, + }, + want: `{"D1":"-34293h33m9.123456789s","D2":-123456789123456789}`, + }, { + name: jsontest.Name("Duration/Nanos/String"), + in: struct { + D1 time.Duration `json:",string,format:nano"` + D2 time.Duration `json:",string,format:nano"` + D3 time.Duration `json:",string,format:nano"` + }{ + math.MinInt64, + 0, + math.MaxInt64, + }, + want: `{"D1":"-9223372036854775808","D2":"0","D3":"9223372036854775807"}`, + }, { + name: jsontest.Name("Duration/Format/Invalid"), + in: struct { + D time.Duration `json:",format:invalid"` + }{}, + want: `{"D"`, + wantErr: EM(errInvalidFormatFlag).withPos(`{"D":`, "/D").withType(0, T[time.Duration]()), + }, { + /* TODO(https://go.dev/issue/71631): Re-enable this test case. + name: jsontest.Name("Duration/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + in: time.Duration(0), + want: `"0s"`, + }, { */ + name: jsontest.Name("Duration/Format"), + opts: []Options{jsontext.Multiline(true)}, + in: structDurationFormat{ + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + }, + want: `{ + "D1": "12h34m56.078090012s", + "D2": "12h34m56.078090012s", + "D3": 45296.078090012, + "D4": "45296.078090012", + "D5": 45296078.090012, + "D6": "45296078.090012", + "D7": 45296078090.012, + "D8": "45296078090.012", + "D9": 45296078090012, + "D10": "45296078090012", + "D11": "PT12H34M56.078090012S" +}`, + }, { + /* TODO(https://go.dev/issue/71631): Re-enable this test case. + name: jsontest.Name("Duration/Format/Legacy"), + opts: []Options{jsonflags.FormatDurationAsNano | 1}, + in: structDurationFormat{ + D1: 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + D2: 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + }, + want: `{"D1":45296078090012,"D2":"12h34m56.078090012s","D3":0,"D4":"0","D5":0,"D6":"0","D7":0,"D8":"0","D9":0,"D10":"0","D11":"PT0S"}`, + }, { */ + /* TODO(https://go.dev/issue/71631): Re-enable this test case. + name: jsontest.Name("Duration/MapKey"), + in: map[time.Duration]string{time.Second: ""}, + want: `{"1s":""}`, + }, { */ + name: jsontest.Name("Duration/MapKey/Legacy"), + opts: []Options{jsonflags.FormatDurationAsNano | 1}, + in: map[time.Duration]string{time.Second: ""}, + want: `{"1000000000":""}`, + }, { + name: jsontest.Name("Time/Zero"), + in: struct { + T1 time.Time + T2 time.Time `json:",format:RFC822"` + T3 time.Time `json:",format:'2006-01-02'"` + T4 time.Time `json:",omitzero"` + T5 time.Time `json:",omitempty"` + }{ + time.Time{}, + time.Time{}, + time.Time{}, + // This is zero according to time.Time.IsZero, + // but non-zero according to reflect.Value.IsZero. + time.Date(1, 1, 1, 0, 0, 0, 0, time.FixedZone("UTC", 0)), + time.Time{}, + }, + want: `{"T1":"0001-01-01T00:00:00Z","T2":"01 Jan 01 00:00 UTC","T3":"0001-01-01","T5":"0001-01-01T00:00:00Z"}`, + }, { + name: jsontest.Name("Time/Format"), + opts: []Options{jsontext.Multiline(true)}, + in: structTimeFormat{ + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + time.Date(1234, 1, 2, 3, 4, 5, 6, time.UTC), + }, + want: `{ + "T1": "1234-01-02T03:04:05.000000006Z", + "T2": "Mon Jan 2 03:04:05 1234", + "T3": "Mon Jan 2 03:04:05 UTC 1234", + "T4": "Mon Jan 02 03:04:05 +0000 1234", + "T5": "02 Jan 34 03:04 UTC", + "T6": "02 Jan 34 03:04 +0000", + "T7": "Monday, 02-Jan-34 03:04:05 UTC", + "T8": "Mon, 02 Jan 1234 03:04:05 UTC", + "T9": "Mon, 02 Jan 1234 03:04:05 +0000", + "T10": "1234-01-02T03:04:05Z", + "T11": "1234-01-02T03:04:05.000000006Z", + "T12": "3:04AM", + "T13": "Jan 2 03:04:05", + "T14": "Jan 2 03:04:05.000", + "T15": "Jan 2 03:04:05.000000", + "T16": "Jan 2 03:04:05.000000006", + "T17": "1234-01-02 03:04:05", + "T18": "1234-01-02", + "T19": "03:04:05", + "T20": "1234-01-02", + "T21": "\"weird\"1234", + "T22": -23225777754.999999994, + "T23": "-23225777754.999999994", + "T24": -23225777754999.999994, + "T25": "-23225777754999.999994", + "T26": -23225777754999999.994, + "T27": "-23225777754999999.994", + "T28": -23225777754999999994, + "T29": "-23225777754999999994" +}`, + }, { + name: jsontest.Name("Time/Format/Invalid"), + in: struct { + T time.Time `json:",format:UndefinedConstant"` + }{}, + want: `{"T"`, + wantErr: EM(errors.New(`invalid format flag "UndefinedConstant"`)).withPos(`{"T":`, "/T").withType(0, timeTimeType), + }, { + name: jsontest.Name("Time/Format/YearOverflow"), + in: struct { + T1 time.Time + T2 time.Time + }{ + time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Add(-time.Second), + time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC), + }, + want: `{"T1":"9999-12-31T23:59:59Z","T2"`, + wantErr: EM(errors.New(`year outside of range [0,9999]`)).withPos(`{"T1":"9999-12-31T23:59:59Z","T2":`, "/T2").withType(0, timeTimeType), + }, { + name: jsontest.Name("Time/Format/YearUnderflow"), + in: struct { + T1 time.Time + T2 time.Time + }{ + time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC), + time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC).Add(-time.Second), + }, + want: `{"T1":"0000-01-01T00:00:00Z","T2"`, + wantErr: EM(errors.New(`year outside of range [0,9999]`)).withPos(`{"T1":"0000-01-01T00:00:00Z","T2":`, "/T2").withType(0, timeTimeType), + }, { + name: jsontest.Name("Time/Format/YearUnderflow"), + in: struct{ T time.Time }{time.Date(-998, 1, 1, 0, 0, 0, 0, time.UTC).Add(-time.Second)}, + want: `{"T"`, + wantErr: EM(errors.New(`year outside of range [0,9999]`)).withPos(`{"T":`, "/T").withType(0, timeTimeType), + }, { + name: jsontest.Name("Time/Format/ZoneExact"), + in: struct{ T time.Time }{time.Date(2020, 1, 1, 0, 0, 0, 0, time.FixedZone("", 23*60*60+59*60))}, + want: `{"T":"2020-01-01T00:00:00+23:59"}`, + }, { + name: jsontest.Name("Time/Format/ZoneHourOverflow"), + in: struct{ T time.Time }{time.Date(2020, 1, 1, 0, 0, 0, 0, time.FixedZone("", 24*60*60))}, + want: `{"T"`, + wantErr: EM(errors.New(`timezone hour outside of range [0,23]`)).withPos(`{"T":`, "/T").withType(0, timeTimeType), + }, { + name: jsontest.Name("Time/Format/ZoneHourOverflow"), + in: struct{ T time.Time }{time.Date(2020, 1, 1, 0, 0, 0, 0, time.FixedZone("", 123*60*60))}, + want: `{"T"`, + wantErr: EM(errors.New(`timezone hour outside of range [0,23]`)).withPos(`{"T":`, "/T").withType(0, timeTimeType), + }, { + name: jsontest.Name("Time/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + in: time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), + want: `"2000-01-01T00:00:00Z"`, + }} + + for _, tt := range tests { + t.Run(tt.name.Name, func(t *testing.T) { + var got []byte + var gotErr error + if tt.useWriter { + bb := new(struct{ bytes.Buffer }) // avoid optimizations with bytes.Buffer + gotErr = MarshalWrite(bb, tt.in, tt.opts...) + got = bb.Bytes() + } else { + got, gotErr = Marshal(tt.in, tt.opts...) + } + if tt.canonicalize { + (*jsontext.Value)(&got).Canonicalize() + } + if string(got) != tt.want { + t.Errorf("%s: Marshal output mismatch:\ngot %s\nwant %s", tt.name.Where, got, tt.want) + } + if !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("%s: Marshal error mismatch:\ngot %v\nwant %v", tt.name.Where, gotErr, tt.wantErr) + } + }) + } +} + +func TestUnmarshal(t *testing.T) { + tests := []struct { + name jsontest.CaseName + opts []Options + inBuf string + inVal any + want any + wantErr error + }{{ + name: jsontest.Name("Nil"), + inBuf: `null`, + wantErr: EU(internal.ErrNonNilReference), + }, { + name: jsontest.Name("NilPointer"), + inBuf: `null`, + inVal: (*string)(nil), + want: (*string)(nil), + wantErr: EU(internal.ErrNonNilReference).withType(0, T[*string]()), + }, { + name: jsontest.Name("NonPointer"), + inBuf: `null`, + inVal: "unchanged", + want: "unchanged", + wantErr: EU(internal.ErrNonNilReference).withType(0, T[string]()), + }, { + name: jsontest.Name("Bools/TrailingJunk"), + inBuf: `falsetrue`, + inVal: addr(true), + want: addr(false), + wantErr: newInvalidCharacterError("t", "after top-level value", len64(`false`), ""), + }, { + name: jsontest.Name("Bools/Null"), + inBuf: `null`, + inVal: addr(true), + want: addr(false), + }, { + name: jsontest.Name("Bools"), + inBuf: `[null,false,true]`, + inVal: new([]bool), + want: addr([]bool{false, false, true}), + }, { + name: jsontest.Name("Bools/Named"), + inBuf: `[null,false,true]`, + inVal: new([]namedBool), + want: addr([]namedBool{false, false, true}), + }, { + name: jsontest.Name("Bools/Invalid/StringifiedFalse"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"false"`, + inVal: addr(true), + want: addr(true), + wantErr: EU(nil).withType('"', boolType), + }, { + name: jsontest.Name("Bools/Invalid/StringifiedTrue"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"true"`, + inVal: addr(true), + want: addr(true), + wantErr: EU(nil).withType('"', boolType), + }, { + name: jsontest.Name("Bools/StringifiedBool/True"), + opts: []Options{jsonflags.StringifyBoolsAndStrings | 1}, + inBuf: `"true"`, + inVal: addr(false), + want: addr(true), + }, { + name: jsontest.Name("Bools/StringifiedBool/False"), + opts: []Options{jsonflags.StringifyBoolsAndStrings | 1}, + inBuf: `"false"`, + inVal: addr(true), + want: addr(false), + }, { + name: jsontest.Name("Bools/StringifiedBool/InvalidWhitespace"), + opts: []Options{jsonflags.StringifyBoolsAndStrings | 1}, + inBuf: `"false "`, + inVal: addr(true), + want: addr(true), + wantErr: EU(strconv.ErrSyntax).withVal(`"false "`).withType('"', boolType), + }, { + name: jsontest.Name("Bools/StringifiedBool/InvalidBool"), + opts: []Options{jsonflags.StringifyBoolsAndStrings | 1}, + inBuf: `false`, + inVal: addr(true), + want: addr(true), + wantErr: EU(nil).withType('f', boolType), + }, { + name: jsontest.Name("Bools/Invalid/Number"), + inBuf: `0`, + inVal: addr(true), + want: addr(true), + wantErr: EU(nil).withType('0', boolType), + }, { + name: jsontest.Name("Bools/Invalid/String"), + inBuf: `""`, + inVal: addr(true), + want: addr(true), + wantErr: EU(nil).withType('"', boolType), + }, { + name: jsontest.Name("Bools/Invalid/Object"), + inBuf: `{}`, + inVal: addr(true), + want: addr(true), + wantErr: EU(nil).withType('{', boolType), + }, { + name: jsontest.Name("Bools/Invalid/Array"), + inBuf: `[]`, + inVal: addr(true), + want: addr(true), + wantErr: EU(nil).withType('[', boolType), + }, { + name: jsontest.Name("Bools/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + inBuf: `false`, + inVal: addr(true), + want: addr(false), + }, { + name: jsontest.Name("Strings/Null"), + inBuf: `null`, + inVal: addr("something"), + want: addr(""), + }, { + name: jsontest.Name("Strings"), + inBuf: `[null,"","hello","世界"]`, + inVal: new([]string), + want: addr([]string{"", "", "hello", "世界"}), + }, { + name: jsontest.Name("Strings/Escaped"), + inBuf: `[null,"","\u0068\u0065\u006c\u006c\u006f","\u4e16\u754c"]`, + inVal: new([]string), + want: addr([]string{"", "", "hello", "世界"}), + }, { + name: jsontest.Name("Strings/Named"), + inBuf: `[null,"","hello","世界"]`, + inVal: new([]namedString), + want: addr([]namedString{"", "", "hello", "世界"}), + }, { + name: jsontest.Name("Strings/Invalid/False"), + inBuf: `false`, + inVal: addr("nochange"), + want: addr("nochange"), + wantErr: EU(nil).withType('f', stringType), + }, { + name: jsontest.Name("Strings/Invalid/True"), + inBuf: `true`, + inVal: addr("nochange"), + want: addr("nochange"), + wantErr: EU(nil).withType('t', stringType), + }, { + name: jsontest.Name("Strings/Invalid/Object"), + inBuf: `{}`, + inVal: addr("nochange"), + want: addr("nochange"), + wantErr: EU(nil).withType('{', stringType), + }, { + name: jsontest.Name("Strings/Invalid/Array"), + inBuf: `[]`, + inVal: addr("nochange"), + want: addr("nochange"), + wantErr: EU(nil).withType('[', stringType), + }, { + name: jsontest.Name("Strings/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + inBuf: `"hello"`, + inVal: addr("goodbye"), + want: addr("hello"), + }, { + name: jsontest.Name("Strings/StringifiedString"), + opts: []Options{jsonflags.StringifyBoolsAndStrings | 1}, + inBuf: `"\"foo\""`, + inVal: new(string), + want: addr("foo"), + }, { + name: jsontest.Name("Strings/StringifiedString/InvalidWhitespace"), + opts: []Options{jsonflags.StringifyBoolsAndStrings | 1}, + inBuf: `"\"foo\" "`, + inVal: new(string), + want: new(string), + wantErr: EU(newInvalidCharacterError(" ", "after string value", 0, "")).withType('"', stringType), + }, { + name: jsontest.Name("Strings/StringifiedString/InvalidString"), + opts: []Options{jsonflags.StringifyBoolsAndStrings | 1}, + inBuf: `""`, + inVal: new(string), + want: new(string), + wantErr: EU(&jsontext.SyntacticError{Err: io.ErrUnexpectedEOF}).withType('"', stringType), + }, { + name: jsontest.Name("Bytes/Null"), + inBuf: `null`, + inVal: addr([]byte("something")), + want: addr([]byte(nil)), + }, { + name: jsontest.Name("Bytes"), + inBuf: `[null,"","AQ==","AQI=","AQID"]`, + inVal: new([][]byte), + want: addr([][]byte{nil, {}, {1}, {1, 2}, {1, 2, 3}}), + }, { + name: jsontest.Name("Bytes/Large"), + inBuf: `"dGhlIHF1aWNrIGJyb3duIGZveCBqdW1wZWQgb3ZlciB0aGUgbGF6eSBkb2cgYW5kIGF0ZSB0aGUgaG9tZXdvcmsgdGhhdCBJIHNwZW50IHNvIG11Y2ggdGltZSBvbi4="`, + inVal: new([]byte), + want: addr([]byte("the quick brown fox jumped over the lazy dog and ate the homework that I spent so much time on.")), + }, { + name: jsontest.Name("Bytes/Reuse"), + inBuf: `"AQID"`, + inVal: addr([]byte("changed")), + want: addr([]byte{1, 2, 3}), + }, { + name: jsontest.Name("Bytes/Escaped"), + inBuf: `[null,"","\u0041\u0051\u003d\u003d","\u0041\u0051\u0049\u003d","\u0041\u0051\u0049\u0044"]`, + inVal: new([][]byte), + want: addr([][]byte{nil, {}, {1}, {1, 2}, {1, 2, 3}}), + }, { + name: jsontest.Name("Bytes/Named"), + inBuf: `[null,"","AQ==","AQI=","AQID"]`, + inVal: new([]namedBytes), + want: addr([]namedBytes{nil, {}, {1}, {1, 2}, {1, 2, 3}}), + }, { + name: jsontest.Name("Bytes/NotStringified"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `[null,"","AQ==","AQI=","AQID"]`, + inVal: new([][]byte), + want: addr([][]byte{nil, {}, {1}, {1, 2}, {1, 2, 3}}), + }, { + // NOTE: []namedByte is not assignable to []byte, + // so the following should be treated as a slice of uints. + name: jsontest.Name("Bytes/Invariant"), + inBuf: `[null,[],[1],[1,2],[1,2,3]]`, + inVal: new([][]namedByte), + want: addr([][]namedByte{nil, {}, {1}, {1, 2}, {1, 2, 3}}), + }, { + // NOTE: This differs in behavior from v1, + // but keeps the representation of slices and arrays more consistent. + name: jsontest.Name("Bytes/ByteArray"), + inBuf: `"aGVsbG8="`, + inVal: new([5]byte), + want: addr([5]byte{'h', 'e', 'l', 'l', 'o'}), + }, { + name: jsontest.Name("Bytes/ByteArray0/Valid"), + inBuf: `""`, + inVal: new([0]byte), + want: addr([0]byte{}), + }, { + name: jsontest.Name("Bytes/ByteArray0/Invalid"), + inBuf: `"A"`, + inVal: new([0]byte), + want: addr([0]byte{}), + wantErr: EU(func() error { + _, err := base64.StdEncoding.Decode(make([]byte, 0), []byte("A")) + return err + }()).withType('"', T[[0]byte]()), + }, { + name: jsontest.Name("Bytes/ByteArray0/Overflow"), + inBuf: `"AA=="`, + inVal: new([0]byte), + want: addr([0]byte{}), + wantErr: EU(errors.New("decoded length of 1 mismatches array length of 0")).withType('"', T[[0]byte]()), + }, { + name: jsontest.Name("Bytes/ByteArray1/Valid"), + inBuf: `"AQ=="`, + inVal: new([1]byte), + want: addr([1]byte{1}), + }, { + name: jsontest.Name("Bytes/ByteArray1/Invalid"), + inBuf: `"$$=="`, + inVal: new([1]byte), + want: addr([1]byte{}), + wantErr: EU(func() error { + _, err := base64.StdEncoding.Decode(make([]byte, 1), []byte("$$==")) + return err + }()).withType('"', T[[1]byte]()), + }, { + name: jsontest.Name("Bytes/ByteArray1/Underflow"), + inBuf: `""`, + inVal: new([1]byte), + want: addr([1]byte{}), + wantErr: EU(errors.New("decoded length of 0 mismatches array length of 1")).withType('"', T[[1]byte]()), + }, { + name: jsontest.Name("Bytes/ByteArray1/Overflow"), + inBuf: `"AQI="`, + inVal: new([1]byte), + want: addr([1]byte{1}), + wantErr: EU(errors.New("decoded length of 2 mismatches array length of 1")).withType('"', T[[1]byte]()), + }, { + name: jsontest.Name("Bytes/ByteArray2/Valid"), + inBuf: `"AQI="`, + inVal: new([2]byte), + want: addr([2]byte{1, 2}), + }, { + name: jsontest.Name("Bytes/ByteArray2/Invalid"), + inBuf: `"$$$="`, + inVal: new([2]byte), + want: addr([2]byte{}), + wantErr: EU(func() error { + _, err := base64.StdEncoding.Decode(make([]byte, 2), []byte("$$$=")) + return err + }()).withType('"', T[[2]byte]()), + }, { + name: jsontest.Name("Bytes/ByteArray2/Underflow"), + inBuf: `"AQ=="`, + inVal: new([2]byte), + want: addr([2]byte{1, 0}), + wantErr: EU(errors.New("decoded length of 1 mismatches array length of 2")).withType('"', T[[2]byte]()), + }, { + name: jsontest.Name("Bytes/ByteArray2/Underflow/Allowed"), + opts: []Options{jsonflags.UnmarshalArrayFromAnyLength | 1}, + inBuf: `"AQ=="`, + inVal: new([2]byte), + want: addr([2]byte{1, 0}), + }, { + name: jsontest.Name("Bytes/ByteArray2/Overflow"), + inBuf: `"AQID"`, + inVal: new([2]byte), + want: addr([2]byte{1, 2}), + wantErr: EU(errors.New("decoded length of 3 mismatches array length of 2")).withType('"', T[[2]byte]()), + }, { + name: jsontest.Name("Bytes/ByteArray2/Overflow/Allowed"), + opts: []Options{jsonflags.UnmarshalArrayFromAnyLength | 1}, + inBuf: `"AQID"`, + inVal: new([2]byte), + want: addr([2]byte{1, 2}), + }, { + name: jsontest.Name("Bytes/ByteArray3/Valid"), + inBuf: `"AQID"`, + inVal: new([3]byte), + want: addr([3]byte{1, 2, 3}), + }, { + name: jsontest.Name("Bytes/ByteArray3/Invalid"), + inBuf: `"$$$$"`, + inVal: new([3]byte), + want: addr([3]byte{}), + wantErr: EU(func() error { + _, err := base64.StdEncoding.Decode(make([]byte, 3), []byte("$$$$")) + return err + }()).withType('"', T[[3]byte]()), + }, { + name: jsontest.Name("Bytes/ByteArray3/Underflow"), + inBuf: `"AQI="`, + inVal: addr([3]byte{0xff, 0xff, 0xff}), + want: addr([3]byte{1, 2, 0}), + wantErr: EU(errors.New("decoded length of 2 mismatches array length of 3")).withType('"', T[[3]byte]()), + }, { + name: jsontest.Name("Bytes/ByteArray3/Overflow"), + inBuf: `"AQIDAQ=="`, + inVal: new([3]byte), + want: addr([3]byte{1, 2, 3}), + wantErr: EU(errors.New("decoded length of 4 mismatches array length of 3")).withType('"', T[[3]byte]()), + }, { + name: jsontest.Name("Bytes/ByteArray4/Valid"), + inBuf: `"AQIDBA=="`, + inVal: new([4]byte), + want: addr([4]byte{1, 2, 3, 4}), + }, { + name: jsontest.Name("Bytes/ByteArray4/Invalid"), + inBuf: `"$$$$$$=="`, + inVal: new([4]byte), + want: addr([4]byte{}), + wantErr: EU(func() error { + _, err := base64.StdEncoding.Decode(make([]byte, 4), []byte("$$$$$$==")) + return err + }()).withType('"', T[[4]byte]()), + }, { + name: jsontest.Name("Bytes/ByteArray4/Underflow"), + inBuf: `"AQID"`, + inVal: new([4]byte), + want: addr([4]byte{1, 2, 3, 0}), + wantErr: EU(errors.New("decoded length of 3 mismatches array length of 4")).withType('"', T[[4]byte]()), + }, { + name: jsontest.Name("Bytes/ByteArray4/Overflow"), + inBuf: `"AQIDBAU="`, + inVal: new([4]byte), + want: addr([4]byte{1, 2, 3, 4}), + wantErr: EU(errors.New("decoded length of 5 mismatches array length of 4")).withType('"', T[[4]byte]()), + }, { + // NOTE: []namedByte is not assignable to []byte, + // so the following should be treated as a array of uints. + name: jsontest.Name("Bytes/NamedByteArray"), + inBuf: `[104,101,108,108,111]`, + inVal: new([5]namedByte), + want: addr([5]namedByte{'h', 'e', 'l', 'l', 'o'}), + }, { + name: jsontest.Name("Bytes/Valid/Denormalized"), + inBuf: `"AR=="`, + inVal: new([]byte), + want: addr([]byte{1}), + }, { + name: jsontest.Name("Bytes/Invalid/Unpadded1"), + inBuf: `"AQ="`, + inVal: addr([]byte("nochange")), + want: addr([]byte("nochange")), + wantErr: EU(func() error { + _, err := base64.StdEncoding.Decode(make([]byte, 0), []byte("AQ=")) + return err + }()).withType('"', bytesType), + }, { + name: jsontest.Name("Bytes/Invalid/Unpadded2"), + inBuf: `"AQ"`, + inVal: addr([]byte("nochange")), + want: addr([]byte("nochange")), + wantErr: EU(func() error { + _, err := base64.StdEncoding.Decode(make([]byte, 0), []byte("AQ")) + return err + }()).withType('"', bytesType), + }, { + name: jsontest.Name("Bytes/Invalid/Character"), + inBuf: `"@@@@"`, + inVal: addr([]byte("nochange")), + want: addr([]byte("nochange")), + wantErr: EU(func() error { + _, err := base64.StdEncoding.Decode(make([]byte, 3), []byte("@@@@")) + return err + }()).withType('"', bytesType), + }, { + name: jsontest.Name("Bytes/Invalid/Bool"), + inBuf: `true`, + inVal: addr([]byte("nochange")), + want: addr([]byte("nochange")), + wantErr: EU(nil).withType('t', bytesType), + }, { + name: jsontest.Name("Bytes/Invalid/Number"), + inBuf: `0`, + inVal: addr([]byte("nochange")), + want: addr([]byte("nochange")), + wantErr: EU(nil).withType('0', bytesType), + }, { + name: jsontest.Name("Bytes/Invalid/Object"), + inBuf: `{}`, + inVal: addr([]byte("nochange")), + want: addr([]byte("nochange")), + wantErr: EU(nil).withType('{', bytesType), + }, { + name: jsontest.Name("Bytes/Invalid/Array"), + inBuf: `[]`, + inVal: addr([]byte("nochange")), + want: addr([]byte("nochange")), + wantErr: EU(nil).withType('[', bytesType), + }, { + name: jsontest.Name("Bytes/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + inBuf: `"aGVsbG8="`, + inVal: new([]byte), + want: addr([]byte("hello")), + }, { + name: jsontest.Name("Ints/Null"), + inBuf: `null`, + inVal: addr(int(1)), + want: addr(int(0)), + }, { + name: jsontest.Name("Ints/Int"), + inBuf: `1`, + inVal: addr(int(0)), + want: addr(int(1)), + }, { + name: jsontest.Name("Ints/Int8/MinOverflow"), + inBuf: `-129`, + inVal: addr(int8(-1)), + want: addr(int8(-1)), + wantErr: EU(strconv.ErrRange).withVal(`-129`).withType('0', T[int8]()), + }, { + name: jsontest.Name("Ints/Int8/Min"), + inBuf: `-128`, + inVal: addr(int8(0)), + want: addr(int8(-128)), + }, { + name: jsontest.Name("Ints/Int8/Max"), + inBuf: `127`, + inVal: addr(int8(0)), + want: addr(int8(127)), + }, { + name: jsontest.Name("Ints/Int8/MaxOverflow"), + inBuf: `128`, + inVal: addr(int8(-1)), + want: addr(int8(-1)), + wantErr: EU(strconv.ErrRange).withVal(`128`).withType('0', T[int8]()), + }, { + name: jsontest.Name("Ints/Int16/MinOverflow"), + inBuf: `-32769`, + inVal: addr(int16(-1)), + want: addr(int16(-1)), + wantErr: EU(strconv.ErrRange).withVal(`-32769`).withType('0', T[int16]()), + }, { + name: jsontest.Name("Ints/Int16/Min"), + inBuf: `-32768`, + inVal: addr(int16(0)), + want: addr(int16(-32768)), + }, { + name: jsontest.Name("Ints/Int16/Max"), + inBuf: `32767`, + inVal: addr(int16(0)), + want: addr(int16(32767)), + }, { + name: jsontest.Name("Ints/Int16/MaxOverflow"), + inBuf: `32768`, + inVal: addr(int16(-1)), + want: addr(int16(-1)), + wantErr: EU(strconv.ErrRange).withVal(`32768`).withType('0', T[int16]()), + }, { + name: jsontest.Name("Ints/Int32/MinOverflow"), + inBuf: `-2147483649`, + inVal: addr(int32(-1)), + want: addr(int32(-1)), + wantErr: EU(strconv.ErrRange).withVal(`-2147483649`).withType('0', T[int32]()), + }, { + name: jsontest.Name("Ints/Int32/Min"), + inBuf: `-2147483648`, + inVal: addr(int32(0)), + want: addr(int32(-2147483648)), + }, { + name: jsontest.Name("Ints/Int32/Max"), + inBuf: `2147483647`, + inVal: addr(int32(0)), + want: addr(int32(2147483647)), + }, { + name: jsontest.Name("Ints/Int32/MaxOverflow"), + inBuf: `2147483648`, + inVal: addr(int32(-1)), + want: addr(int32(-1)), + wantErr: EU(strconv.ErrRange).withVal(`2147483648`).withType('0', T[int32]()), + }, { + name: jsontest.Name("Ints/Int64/MinOverflow"), + inBuf: `-9223372036854775809`, + inVal: addr(int64(-1)), + want: addr(int64(-1)), + wantErr: EU(strconv.ErrRange).withVal(`-9223372036854775809`).withType('0', T[int64]()), + }, { + name: jsontest.Name("Ints/Int64/Min"), + inBuf: `-9223372036854775808`, + inVal: addr(int64(0)), + want: addr(int64(-9223372036854775808)), + }, { + name: jsontest.Name("Ints/Int64/Max"), + inBuf: `9223372036854775807`, + inVal: addr(int64(0)), + want: addr(int64(9223372036854775807)), + }, { + name: jsontest.Name("Ints/Int64/MaxOverflow"), + inBuf: `9223372036854775808`, + inVal: addr(int64(-1)), + want: addr(int64(-1)), + wantErr: EU(strconv.ErrRange).withVal(`9223372036854775808`).withType('0', T[int64]()), + }, { + name: jsontest.Name("Ints/Named"), + inBuf: `-6464`, + inVal: addr(namedInt64(0)), + want: addr(namedInt64(-6464)), + }, { + name: jsontest.Name("Ints/Stringified"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"-6464"`, + inVal: new(int), + want: addr(int(-6464)), + }, { + name: jsontest.Name("Ints/Stringified/Invalid"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `-6464`, + inVal: new(int), + want: new(int), + wantErr: EU(nil).withType('0', T[int]()), + }, { + name: jsontest.Name("Ints/Stringified/LeadingZero"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"00"`, + inVal: addr(int(-1)), + want: addr(int(-1)), + wantErr: EU(strconv.ErrSyntax).withVal(`"00"`).withType('"', T[int]()), + }, { + name: jsontest.Name("Ints/Escaped"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"\u002d\u0036\u0034\u0036\u0034"`, + inVal: new(int), + want: addr(int(-6464)), + }, { + name: jsontest.Name("Ints/Valid/NegativeZero"), + inBuf: `-0`, + inVal: addr(int(1)), + want: addr(int(0)), + }, { + name: jsontest.Name("Ints/Invalid/Fraction"), + inBuf: `1.0`, + inVal: addr(int(-1)), + want: addr(int(-1)), + wantErr: EU(strconv.ErrSyntax).withVal(`1.0`).withType('0', T[int]()), + }, { + name: jsontest.Name("Ints/Invalid/Exponent"), + inBuf: `1e0`, + inVal: addr(int(-1)), + want: addr(int(-1)), + wantErr: EU(strconv.ErrSyntax).withVal(`1e0`).withType('0', T[int]()), + }, { + name: jsontest.Name("Ints/Invalid/StringifiedFraction"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"1.0"`, + inVal: addr(int(-1)), + want: addr(int(-1)), + wantErr: EU(strconv.ErrSyntax).withVal(`"1.0"`).withType('"', T[int]()), + }, { + name: jsontest.Name("Ints/Invalid/StringifiedExponent"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"1e0"`, + inVal: addr(int(-1)), + want: addr(int(-1)), + wantErr: EU(strconv.ErrSyntax).withVal(`"1e0"`).withType('"', T[int]()), + }, { + name: jsontest.Name("Ints/Invalid/Overflow"), + inBuf: `100000000000000000000000000000`, + inVal: addr(int(-1)), + want: addr(int(-1)), + wantErr: EU(strconv.ErrRange).withVal(`100000000000000000000000000000`).withType('0', T[int]()), + }, { + name: jsontest.Name("Ints/Invalid/OverflowSyntax"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"100000000000000000000000000000x"`, + inVal: addr(int(-1)), + want: addr(int(-1)), + wantErr: EU(strconv.ErrSyntax).withVal(`"100000000000000000000000000000x"`).withType('"', T[int]()), + }, { + name: jsontest.Name("Ints/Invalid/Whitespace"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"0 "`, + inVal: addr(int(-1)), + want: addr(int(-1)), + wantErr: EU(strconv.ErrSyntax).withVal(`"0 "`).withType('"', T[int]()), + }, { + name: jsontest.Name("Ints/Invalid/Bool"), + inBuf: `true`, + inVal: addr(int(-1)), + want: addr(int(-1)), + wantErr: EU(nil).withType('t', T[int]()), + }, { + name: jsontest.Name("Ints/Invalid/String"), + inBuf: `"0"`, + inVal: addr(int(-1)), + want: addr(int(-1)), + wantErr: EU(nil).withType('"', T[int]()), + }, { + name: jsontest.Name("Ints/Invalid/Object"), + inBuf: `{}`, + inVal: addr(int(-1)), + want: addr(int(-1)), + wantErr: EU(nil).withType('{', T[int]()), + }, { + name: jsontest.Name("Ints/Invalid/Array"), + inBuf: `[]`, + inVal: addr(int(-1)), + want: addr(int(-1)), + wantErr: EU(nil).withType('[', T[int]()), + }, { + name: jsontest.Name("Ints/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + inBuf: `1`, + inVal: addr(int(0)), + want: addr(int(1)), + }, { + name: jsontest.Name("Uints/Null"), + inBuf: `null`, + inVal: addr(uint(1)), + want: addr(uint(0)), + }, { + name: jsontest.Name("Uints/Uint"), + inBuf: `1`, + inVal: addr(uint(0)), + want: addr(uint(1)), + }, { + name: jsontest.Name("Uints/Uint8/Min"), + inBuf: `0`, + inVal: addr(uint8(1)), + want: addr(uint8(0)), + }, { + name: jsontest.Name("Uints/Uint8/Max"), + inBuf: `255`, + inVal: addr(uint8(0)), + want: addr(uint8(255)), + }, { + name: jsontest.Name("Uints/Uint8/MaxOverflow"), + inBuf: `256`, + inVal: addr(uint8(1)), + want: addr(uint8(1)), + wantErr: EU(strconv.ErrRange).withVal(`256`).withType('0', T[uint8]()), + }, { + name: jsontest.Name("Uints/Uint16/Min"), + inBuf: `0`, + inVal: addr(uint16(1)), + want: addr(uint16(0)), + }, { + name: jsontest.Name("Uints/Uint16/Max"), + inBuf: `65535`, + inVal: addr(uint16(0)), + want: addr(uint16(65535)), + }, { + name: jsontest.Name("Uints/Uint16/MaxOverflow"), + inBuf: `65536`, + inVal: addr(uint16(1)), + want: addr(uint16(1)), + wantErr: EU(strconv.ErrRange).withVal(`65536`).withType('0', T[uint16]()), + }, { + name: jsontest.Name("Uints/Uint32/Min"), + inBuf: `0`, + inVal: addr(uint32(1)), + want: addr(uint32(0)), + }, { + name: jsontest.Name("Uints/Uint32/Max"), + inBuf: `4294967295`, + inVal: addr(uint32(0)), + want: addr(uint32(4294967295)), + }, { + name: jsontest.Name("Uints/Uint32/MaxOverflow"), + inBuf: `4294967296`, + inVal: addr(uint32(1)), + want: addr(uint32(1)), + wantErr: EU(strconv.ErrRange).withVal(`4294967296`).withType('0', T[uint32]()), + }, { + name: jsontest.Name("Uints/Uint64/Min"), + inBuf: `0`, + inVal: addr(uint64(1)), + want: addr(uint64(0)), + }, { + name: jsontest.Name("Uints/Uint64/Max"), + inBuf: `18446744073709551615`, + inVal: addr(uint64(0)), + want: addr(uint64(18446744073709551615)), + }, { + name: jsontest.Name("Uints/Uint64/MaxOverflow"), + inBuf: `18446744073709551616`, + inVal: addr(uint64(1)), + want: addr(uint64(1)), + wantErr: EU(strconv.ErrRange).withVal(`18446744073709551616`).withType('0', T[uint64]()), + }, { + name: jsontest.Name("Uints/Uintptr"), + inBuf: `1`, + inVal: addr(uintptr(0)), + want: addr(uintptr(1)), + }, { + name: jsontest.Name("Uints/Named"), + inBuf: `6464`, + inVal: addr(namedUint64(0)), + want: addr(namedUint64(6464)), + }, { + name: jsontest.Name("Uints/Stringified"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"6464"`, + inVal: new(uint), + want: addr(uint(6464)), + }, { + name: jsontest.Name("Uints/Stringified/Invalid"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `6464`, + inVal: new(uint), + want: new(uint), + wantErr: EU(nil).withType('0', T[uint]()), + }, { + name: jsontest.Name("Uints/Stringified/LeadingZero"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"00"`, + inVal: addr(uint(1)), + want: addr(uint(1)), + wantErr: EU(strconv.ErrSyntax).withVal(`"00"`).withType('"', T[uint]()), + }, { + name: jsontest.Name("Uints/Escaped"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"\u0036\u0034\u0036\u0034"`, + inVal: new(uint), + want: addr(uint(6464)), + }, { + name: jsontest.Name("Uints/Invalid/NegativeOne"), + inBuf: `-1`, + inVal: addr(uint(1)), + want: addr(uint(1)), + wantErr: EU(strconv.ErrSyntax).withVal(`-1`).withType('0', T[uint]()), + }, { + name: jsontest.Name("Uints/Invalid/NegativeZero"), + inBuf: `-0`, + inVal: addr(uint(1)), + want: addr(uint(1)), + wantErr: EU(strconv.ErrSyntax).withVal(`-0`).withType('0', T[uint]()), + }, { + name: jsontest.Name("Uints/Invalid/Fraction"), + inBuf: `1.0`, + inVal: addr(uint(10)), + want: addr(uint(10)), + wantErr: EU(strconv.ErrSyntax).withVal(`1.0`).withType('0', T[uint]()), + }, { + name: jsontest.Name("Uints/Invalid/Exponent"), + inBuf: `1e0`, + inVal: addr(uint(10)), + want: addr(uint(10)), + wantErr: EU(strconv.ErrSyntax).withVal(`1e0`).withType('0', T[uint]()), + }, { + name: jsontest.Name("Uints/Invalid/StringifiedFraction"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"1.0"`, + inVal: addr(uint(10)), + want: addr(uint(10)), + wantErr: EU(strconv.ErrSyntax).withVal(`"1.0"`).withType('"', T[uint]()), + }, { + name: jsontest.Name("Uints/Invalid/StringifiedExponent"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"1e0"`, + inVal: addr(uint(10)), + want: addr(uint(10)), + wantErr: EU(strconv.ErrSyntax).withVal(`"1e0"`).withType('"', T[uint]()), + }, { + name: jsontest.Name("Uints/Invalid/Overflow"), + inBuf: `100000000000000000000000000000`, + inVal: addr(uint(1)), + want: addr(uint(1)), + wantErr: EU(strconv.ErrRange).withVal(`100000000000000000000000000000`).withType('0', T[uint]()), + }, { + name: jsontest.Name("Uints/Invalid/OverflowSyntax"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"100000000000000000000000000000x"`, + inVal: addr(uint(1)), + want: addr(uint(1)), + wantErr: EU(strconv.ErrSyntax).withVal(`"100000000000000000000000000000x"`).withType('"', T[uint]()), + }, { + name: jsontest.Name("Uints/Invalid/Whitespace"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"0 "`, + inVal: addr(uint(1)), + want: addr(uint(1)), + wantErr: EU(strconv.ErrSyntax).withVal(`"0 "`).withType('"', T[uint]()), + }, { + name: jsontest.Name("Uints/Invalid/Bool"), + inBuf: `true`, + inVal: addr(uint(1)), + want: addr(uint(1)), + wantErr: EU(nil).withType('t', T[uint]()), + }, { + name: jsontest.Name("Uints/Invalid/String"), + inBuf: `"0"`, + inVal: addr(uint(1)), + want: addr(uint(1)), + wantErr: EU(nil).withType('"', T[uint]()), + }, { + name: jsontest.Name("Uints/Invalid/Object"), + inBuf: `{}`, + inVal: addr(uint(1)), + want: addr(uint(1)), + wantErr: EU(nil).withType('{', T[uint]()), + }, { + name: jsontest.Name("Uints/Invalid/Array"), + inBuf: `[]`, + inVal: addr(uint(1)), + want: addr(uint(1)), + wantErr: EU(nil).withType('[', T[uint]()), + }, { + name: jsontest.Name("Uints/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + inBuf: `1`, + inVal: addr(uint(0)), + want: addr(uint(1)), + }, { + name: jsontest.Name("Floats/Null"), + inBuf: `null`, + inVal: addr(float64(64.64)), + want: addr(float64(0)), + }, { + name: jsontest.Name("Floats/Float32/Pi"), + inBuf: `3.14159265358979323846264338327950288419716939937510582097494459`, + inVal: addr(float32(32.32)), + want: addr(float32(math.Pi)), + }, { + name: jsontest.Name("Floats/Float32/Underflow"), + inBuf: `1e-1000`, + inVal: addr(float32(32.32)), + want: addr(float32(0)), + }, { + name: jsontest.Name("Floats/Float32/Overflow"), + inBuf: `-1e1000`, + inVal: addr(float32(32.32)), + want: addr(float32(-math.MaxFloat32)), + wantErr: EU(strconv.ErrRange).withVal(`-1e1000`).withType('0', T[float32]()), + }, { + name: jsontest.Name("Floats/Float64/Pi"), + inBuf: `3.14159265358979323846264338327950288419716939937510582097494459`, + inVal: addr(float64(64.64)), + want: addr(float64(math.Pi)), + }, { + name: jsontest.Name("Floats/Float64/Underflow"), + inBuf: `1e-1000`, + inVal: addr(float64(64.64)), + want: addr(float64(0)), + }, { + name: jsontest.Name("Floats/Float64/Overflow"), + inBuf: `-1e1000`, + inVal: addr(float64(64.64)), + want: addr(float64(-math.MaxFloat64)), + wantErr: EU(strconv.ErrRange).withVal(`-1e1000`).withType('0', T[float64]()), + }, { + name: jsontest.Name("Floats/Any/Overflow"), + inBuf: `1e1000`, + inVal: new(any), + want: addr(any(float64(math.MaxFloat64))), + wantErr: EU(strconv.ErrRange).withVal(`1e1000`).withType('0', T[float64]()), + }, { + name: jsontest.Name("Floats/Named"), + inBuf: `64.64`, + inVal: addr(namedFloat64(0)), + want: addr(namedFloat64(64.64)), + }, { + name: jsontest.Name("Floats/Stringified"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"64.64"`, + inVal: new(float64), + want: addr(float64(64.64)), + }, { + name: jsontest.Name("Floats/Stringified/Invalid"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `64.64`, + inVal: new(float64), + want: new(float64), + wantErr: EU(nil).withType('0', T[float64]()), + }, { + name: jsontest.Name("Floats/Escaped"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"\u0036\u0034\u002e\u0036\u0034"`, + inVal: new(float64), + want: addr(float64(64.64)), + }, { + name: jsontest.Name("Floats/Invalid/NaN"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"NaN"`, + inVal: addr(float64(64.64)), + want: addr(float64(64.64)), + wantErr: EU(strconv.ErrSyntax).withVal(`"NaN"`).withType('"', float64Type), + }, { + name: jsontest.Name("Floats/Invalid/Infinity"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"Infinity"`, + inVal: addr(float64(64.64)), + want: addr(float64(64.64)), + wantErr: EU(strconv.ErrSyntax).withVal(`"Infinity"`).withType('"', float64Type), + }, { + name: jsontest.Name("Floats/Invalid/Whitespace"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"1 "`, + inVal: addr(float64(64.64)), + want: addr(float64(64.64)), + wantErr: EU(strconv.ErrSyntax).withVal(`"1 "`).withType('"', float64Type), + }, { + name: jsontest.Name("Floats/Invalid/GoSyntax"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `"1p-2"`, + inVal: addr(float64(64.64)), + want: addr(float64(64.64)), + wantErr: EU(strconv.ErrSyntax).withVal(`"1p-2"`).withType('"', float64Type), + }, { + name: jsontest.Name("Floats/Invalid/Bool"), + inBuf: `true`, + inVal: addr(float64(64.64)), + want: addr(float64(64.64)), + wantErr: EU(nil).withType('t', float64Type), + }, { + name: jsontest.Name("Floats/Invalid/String"), + inBuf: `"0"`, + inVal: addr(float64(64.64)), + want: addr(float64(64.64)), + wantErr: EU(nil).withType('"', float64Type), + }, { + name: jsontest.Name("Floats/Invalid/Object"), + inBuf: `{}`, + inVal: addr(float64(64.64)), + want: addr(float64(64.64)), + wantErr: EU(nil).withType('{', float64Type), + }, { + name: jsontest.Name("Floats/Invalid/Array"), + inBuf: `[]`, + inVal: addr(float64(64.64)), + want: addr(float64(64.64)), + wantErr: EU(nil).withType('[', float64Type), + }, { + name: jsontest.Name("Floats/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + inBuf: `1`, + inVal: addr(float64(0)), + want: addr(float64(1)), + }, { + name: jsontest.Name("Maps/Null"), + inBuf: `null`, + inVal: addr(map[string]string{"key": "value"}), + want: new(map[string]string), + }, { + name: jsontest.Name("Maps/InvalidKey/Bool"), + inBuf: `{"true":"false"}`, + inVal: new(map[bool]bool), + want: addr(make(map[bool]bool)), + wantErr: EU(nil).withPos(`{`, "/true").withType('"', boolType), + }, { + name: jsontest.Name("Maps/InvalidKey/NamedBool"), + inBuf: `{"true":"false"}`, + inVal: new(map[namedBool]bool), + want: addr(make(map[namedBool]bool)), + wantErr: EU(nil).withPos(`{`, "/true").withType('"', T[namedBool]()), + }, { + name: jsontest.Name("Maps/InvalidKey/Array"), + inBuf: `{"key":"value"}`, + inVal: new(map[[1]string]string), + want: addr(make(map[[1]string]string)), + wantErr: EU(nil).withPos(`{`, "/key").withType('"', T[[1]string]()), + }, { + name: jsontest.Name("Maps/InvalidKey/Channel"), + inBuf: `{"key":"value"}`, + inVal: new(map[chan string]string), + want: addr(make(map[chan string]string)), + wantErr: EU(nil).withPos(`{`, "").withType(0, T[chan string]()), + }, { + name: jsontest.Name("Maps/ValidKey/Int"), + inBuf: `{"0":0,"-1":1,"2":2,"-3":3}`, + inVal: new(map[int]int), + want: addr(map[int]int{0: 0, -1: 1, 2: 2, -3: 3}), + }, { + name: jsontest.Name("Maps/ValidKey/NamedInt"), + inBuf: `{"0":0,"-1":1,"2":2,"-3":3}`, + inVal: new(map[namedInt64]int), + want: addr(map[namedInt64]int{0: 0, -1: 1, 2: 2, -3: 3}), + }, { + name: jsontest.Name("Maps/ValidKey/Uint"), + inBuf: `{"0":0,"1":1,"2":2,"3":3}`, + inVal: new(map[uint]uint), + want: addr(map[uint]uint{0: 0, 1: 1, 2: 2, 3: 3}), + }, { + name: jsontest.Name("Maps/ValidKey/NamedUint"), + inBuf: `{"0":0,"1":1,"2":2,"3":3}`, + inVal: new(map[namedUint64]uint), + want: addr(map[namedUint64]uint{0: 0, 1: 1, 2: 2, 3: 3}), + }, { + name: jsontest.Name("Maps/ValidKey/Float"), + inBuf: `{"1.234":1.234,"12.34":12.34,"123.4":123.4}`, + inVal: new(map[float64]float64), + want: addr(map[float64]float64{1.234: 1.234, 12.34: 12.34, 123.4: 123.4}), + }, { + name: jsontest.Name("Maps/DuplicateName/Int"), + inBuf: `{"0":1,"-0":-1}`, + inVal: new(map[int]int), + want: addr(map[int]int{0: 1}), + wantErr: newDuplicateNameError("", []byte(`"-0"`), len64(`{"0":1,`)), + }, { + name: jsontest.Name("Maps/DuplicateName/Int/MergeWithLegacySemantics"), + opts: []Options{jsonflags.MergeWithLegacySemantics | 1}, + inBuf: `{"0":1,"-0":-1}`, + inVal: new(map[int]int), + want: addr(map[int]int{0: 1}), + wantErr: newDuplicateNameError("", []byte(`"-0"`), len64(`{"0":1,`)), + }, { + name: jsontest.Name("Maps/DuplicateName/Int/AllowDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(true)}, + inBuf: `{"0":1,"-0":-1}`, + inVal: new(map[int]int), + want: addr(map[int]int{0: -1}), // latter takes precedence + }, { + name: jsontest.Name("Maps/DuplicateName/Int/OverwriteExisting"), + inBuf: `{"-0":-1}`, + inVal: addr(map[int]int{0: 1}), + want: addr(map[int]int{0: -1}), + }, { + name: jsontest.Name("Maps/DuplicateName/Float"), + inBuf: `{"1.0":"1.0","1":"1","1e0":"1e0"}`, + inVal: new(map[float64]string), + want: addr(map[float64]string{1: "1.0"}), + wantErr: newDuplicateNameError("", []byte(`"1"`), len64(`{"1.0":"1.0",`)), + }, { + name: jsontest.Name("Maps/DuplicateName/Float/AllowDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(true)}, + inBuf: `{"1.0":"1.0","1":"1","1e0":"1e0"}`, + inVal: new(map[float64]string), + want: addr(map[float64]string{1: "1e0"}), // latter takes precedence + }, { + name: jsontest.Name("Maps/DuplicateName/Float/OverwriteExisting"), + inBuf: `{"1.0":"1.0"}`, + inVal: addr(map[float64]string{1: "1"}), + want: addr(map[float64]string{1: "1.0"}), + }, { + name: jsontest.Name("Maps/DuplicateName/NoCaseString"), + inBuf: `{"hello":"hello","HELLO":"HELLO"}`, + inVal: new(map[nocaseString]string), + want: addr(map[nocaseString]string{"hello": "hello"}), + wantErr: newDuplicateNameError("", []byte(`"HELLO"`), len64(`{"hello":"hello",`)), + }, { + name: jsontest.Name("Maps/DuplicateName/NoCaseString/AllowDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(true)}, + inBuf: `{"hello":"hello","HELLO":"HELLO"}`, + inVal: new(map[nocaseString]string), + want: addr(map[nocaseString]string{"hello": "HELLO"}), // latter takes precedence + }, { + name: jsontest.Name("Maps/DuplicateName/NoCaseString/OverwriteExisting"), + opts: []Options{jsontext.AllowDuplicateNames(true)}, + inBuf: `{"HELLO":"HELLO"}`, + inVal: addr(map[nocaseString]string{"hello": "hello"}), + want: addr(map[nocaseString]string{"hello": "HELLO"}), + }, { + name: jsontest.Name("Maps/ValidKey/Interface"), + inBuf: `{"false":"false","true":"true","string":"string","0":"0","[]":"[]","{}":"{}"}`, + inVal: new(map[any]string), + want: addr(map[any]string{ + "false": "false", + "true": "true", + "string": "string", + "0": "0", + "[]": "[]", + "{}": "{}", + }), + }, { + name: jsontest.Name("Maps/InvalidValue/Channel"), + inBuf: `{"key":"value"}`, + inVal: new(map[string]chan string), + want: addr(map[string]chan string{ + "key": nil, + }), + wantErr: EU(nil).withPos(`{"key":`, "/key").withType(0, T[chan string]()), + }, { + name: jsontest.Name("Maps/RecursiveMap"), + inBuf: `{"buzz":{},"fizz":{"bar":{},"foo":{}}}`, + inVal: new(recursiveMap), + want: addr(recursiveMap{ + "fizz": { + "foo": {}, + "bar": {}, + }, + "buzz": {}, + }), + }, { + // NOTE: The semantics differs from v1, + // where existing map entries were not merged into. + // See https://go.dev/issue/31924. + name: jsontest.Name("Maps/Merge"), + opts: []Options{jsontext.AllowDuplicateNames(true)}, + inBuf: `{"k1":{"k2":"v2"},"k2":{"k1":"v1"},"k2":{"k2":"v2"}}`, + inVal: addr(map[string]map[string]string{ + "k1": {"k1": "v1"}, + }), + want: addr(map[string]map[string]string{ + "k1": {"k1": "v1", "k2": "v2"}, + "k2": {"k1": "v1", "k2": "v2"}, + }), + }, { + name: jsontest.Name("Maps/Invalid/Bool"), + inBuf: `true`, + inVal: addr(map[string]string{"key": "value"}), + want: addr(map[string]string{"key": "value"}), + wantErr: EU(nil).withType('t', T[map[string]string]()), + }, { + name: jsontest.Name("Maps/Invalid/String"), + inBuf: `""`, + inVal: addr(map[string]string{"key": "value"}), + want: addr(map[string]string{"key": "value"}), + wantErr: EU(nil).withType('"', T[map[string]string]()), + }, { + name: jsontest.Name("Maps/Invalid/Number"), + inBuf: `0`, + inVal: addr(map[string]string{"key": "value"}), + want: addr(map[string]string{"key": "value"}), + wantErr: EU(nil).withType('0', T[map[string]string]()), + }, { + name: jsontest.Name("Maps/Invalid/Array"), + inBuf: `[]`, + inVal: addr(map[string]string{"key": "value"}), + want: addr(map[string]string{"key": "value"}), + wantErr: EU(nil).withType('[', T[map[string]string]()), + }, { + name: jsontest.Name("Maps/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + inBuf: `{"hello":"goodbye"}`, + inVal: addr(map[string]string{}), + want: addr(map[string]string{"hello": "goodbye"}), + }, { + name: jsontest.Name("Structs/Null"), + inBuf: `null`, + inVal: addr(structAll{String: "something"}), + want: addr(structAll{}), + }, { + name: jsontest.Name("Structs/Empty"), + inBuf: `{}`, + inVal: addr(structAll{ + String: "hello", + Map: map[string]string{}, + Slice: []string{}, + }), + want: addr(structAll{ + String: "hello", + Map: map[string]string{}, + Slice: []string{}, + }), + }, { + name: jsontest.Name("Structs/Normal"), + inBuf: `{ + "Bool": true, + "String": "hello", + "Bytes": "AQID", + "Int": -64, + "Uint": 64, + "Float": 3.14159, + "Map": {"key": "value"}, + "StructScalars": { + "Bool": true, + "String": "hello", + "Bytes": "AQID", + "Int": -64, + "Uint": 64, + "Float": 3.14159 + }, + "StructMaps": { + "MapBool": {"": true}, + "MapString": {"": "hello"}, + "MapBytes": {"": "AQID"}, + "MapInt": {"": -64}, + "MapUint": {"": 64}, + "MapFloat": {"": 3.14159} + }, + "StructSlices": { + "SliceBool": [true], + "SliceString": ["hello"], + "SliceBytes": ["AQID"], + "SliceInt": [-64], + "SliceUint": [64], + "SliceFloat": [3.14159] + }, + "Slice": ["fizz","buzz"], + "Array": ["goodbye"], + "Pointer": {}, + "Interface": null +}`, + inVal: new(structAll), + want: addr(structAll{ + Bool: true, + String: "hello", + Bytes: []byte{1, 2, 3}, + Int: -64, + Uint: +64, + Float: 3.14159, + Map: map[string]string{"key": "value"}, + StructScalars: structScalars{ + Bool: true, + String: "hello", + Bytes: []byte{1, 2, 3}, + Int: -64, + Uint: +64, + Float: 3.14159, + }, + StructMaps: structMaps{ + MapBool: map[string]bool{"": true}, + MapString: map[string]string{"": "hello"}, + MapBytes: map[string][]byte{"": {1, 2, 3}}, + MapInt: map[string]int64{"": -64}, + MapUint: map[string]uint64{"": +64}, + MapFloat: map[string]float64{"": 3.14159}, + }, + StructSlices: structSlices{ + SliceBool: []bool{true}, + SliceString: []string{"hello"}, + SliceBytes: [][]byte{{1, 2, 3}}, + SliceInt: []int64{-64}, + SliceUint: []uint64{+64}, + SliceFloat: []float64{3.14159}, + }, + Slice: []string{"fizz", "buzz"}, + Array: [1]string{"goodbye"}, + Pointer: new(structAll), + }), + }, { + name: jsontest.Name("Structs/Merge"), + inBuf: `{ + "Bool": false, + "String": "goodbye", + "Int": -64, + "Float": 3.14159, + "Map": {"k2": "v2"}, + "StructScalars": { + "Bool": true, + "String": "hello", + "Bytes": "AQID", + "Int": -64 + }, + "StructMaps": { + "MapBool": {"": true}, + "MapString": {"": "hello"}, + "MapBytes": {"": "AQID"}, + "MapInt": {"": -64}, + "MapUint": {"": 64}, + "MapFloat": {"": 3.14159} + }, + "StructSlices": { + "SliceString": ["hello"], + "SliceBytes": ["AQID"], + "SliceInt": [-64], + "SliceUint": [64] + }, + "Slice": ["fizz","buzz"], + "Array": ["goodbye"], + "Pointer": {}, + "Interface": {"k2":"v2"} +}`, + inVal: addr(structAll{ + Bool: true, + String: "hello", + Bytes: []byte{1, 2, 3}, + Uint: +64, + Float: math.NaN(), + Map: map[string]string{"k1": "v1"}, + StructScalars: structScalars{ + String: "hello", + Bytes: make([]byte, 2, 4), + Uint: +64, + Float: 3.14159, + }, + StructMaps: structMaps{ + MapBool: map[string]bool{"": false}, + MapBytes: map[string][]byte{"": {}}, + MapInt: map[string]int64{"": 123}, + MapFloat: map[string]float64{"": math.Inf(+1)}, + }, + StructSlices: structSlices{ + SliceBool: []bool{true}, + SliceBytes: [][]byte{nil, nil}, + SliceInt: []int64{-123}, + SliceUint: []uint64{+123}, + SliceFloat: []float64{3.14159}, + }, + Slice: []string{"buzz", "fizz", "gizz"}, + Array: [1]string{"hello"}, + Pointer: new(structAll), + Interface: map[string]string{"k1": "v1"}, + }), + want: addr(structAll{ + Bool: false, + String: "goodbye", + Bytes: []byte{1, 2, 3}, + Int: -64, + Uint: +64, + Float: 3.14159, + Map: map[string]string{"k1": "v1", "k2": "v2"}, + StructScalars: structScalars{ + Bool: true, + String: "hello", + Bytes: []byte{1, 2, 3}, + Int: -64, + Uint: +64, + Float: 3.14159, + }, + StructMaps: structMaps{ + MapBool: map[string]bool{"": true}, + MapString: map[string]string{"": "hello"}, + MapBytes: map[string][]byte{"": {1, 2, 3}}, + MapInt: map[string]int64{"": -64}, + MapUint: map[string]uint64{"": +64}, + MapFloat: map[string]float64{"": 3.14159}, + }, + StructSlices: structSlices{ + SliceBool: []bool{true}, + SliceString: []string{"hello"}, + SliceBytes: [][]byte{{1, 2, 3}}, + SliceInt: []int64{-64}, + SliceUint: []uint64{+64}, + SliceFloat: []float64{3.14159}, + }, + Slice: []string{"fizz", "buzz"}, + Array: [1]string{"goodbye"}, + Pointer: new(structAll), + Interface: map[string]string{"k1": "v1", "k2": "v2"}, + }), + }, { + name: jsontest.Name("Structs/Stringified/Normal"), + inBuf: `{ + "Bool": true, + "String": "hello", + "Bytes": "AQID", + "Int": "-64", + "Uint": "64", + "Float": "3.14159", + "Map": {"key": "value"}, + "StructScalars": { + "Bool": true, + "String": "hello", + "Bytes": "AQID", + "Int": "-64", + "Uint": "64", + "Float": "3.14159" + }, + "StructMaps": { + "MapBool": {"": true}, + "MapString": {"": "hello"}, + "MapBytes": {"": "AQID"}, + "MapInt": {"": "-64"}, + "MapUint": {"": "64"}, + "MapFloat": {"": "3.14159"} + }, + "StructSlices": { + "SliceBool": [true], + "SliceString": ["hello"], + "SliceBytes": ["AQID"], + "SliceInt": ["-64"], + "SliceUint": ["64"], + "SliceFloat": ["3.14159"] + }, + "Slice": ["fizz","buzz"], + "Array": ["goodbye"], + "Pointer": {}, + "Interface": null +}`, + inVal: new(structStringifiedAll), + want: addr(structStringifiedAll{ + Bool: true, + String: "hello", + Bytes: []byte{1, 2, 3}, + Int: -64, // may be stringified + Uint: +64, // may be stringified + Float: 3.14159, // may be stringified + Map: map[string]string{"key": "value"}, + StructScalars: structScalars{ + Bool: true, + String: "hello", + Bytes: []byte{1, 2, 3}, + Int: -64, // may be stringified + Uint: +64, // may be stringified + Float: 3.14159, // may be stringified + }, + StructMaps: structMaps{ + MapBool: map[string]bool{"": true}, + MapString: map[string]string{"": "hello"}, + MapBytes: map[string][]byte{"": {1, 2, 3}}, + MapInt: map[string]int64{"": -64}, // may be stringified + MapUint: map[string]uint64{"": +64}, // may be stringified + MapFloat: map[string]float64{"": 3.14159}, // may be stringified + }, + StructSlices: structSlices{ + SliceBool: []bool{true}, + SliceString: []string{"hello"}, + SliceBytes: [][]byte{{1, 2, 3}}, + SliceInt: []int64{-64}, // may be stringified + SliceUint: []uint64{+64}, // may be stringified + SliceFloat: []float64{3.14159}, // may be stringified + }, + Slice: []string{"fizz", "buzz"}, + Array: [1]string{"goodbye"}, + Pointer: new(structStringifiedAll), // may be stringified + }), + }, { + name: jsontest.Name("Structs/Stringified/String"), + inBuf: `{ + "Bool": true, + "String": "hello", + "Bytes": "AQID", + "Int": "-64", + "Uint": "64", + "Float": "3.14159", + "Map": {"key": "value"}, + "StructScalars": { + "Bool": true, + "String": "hello", + "Bytes": "AQID", + "Int": "-64", + "Uint": "64", + "Float": "3.14159" + }, + "StructMaps": { + "MapBool": {"": true}, + "MapString": {"": "hello"}, + "MapBytes": {"": "AQID"}, + "MapInt": {"": "-64"}, + "MapUint": {"": "64"}, + "MapFloat": {"": "3.14159"} + }, + "StructSlices": { + "SliceBool": [true], + "SliceString": ["hello"], + "SliceBytes": ["AQID"], + "SliceInt": ["-64"], + "SliceUint": ["64"], + "SliceFloat": ["3.14159"] + }, + "Slice": ["fizz","buzz"], + "Array": ["goodbye"], + "Pointer": {}, + "Interface": null +}`, + inVal: new(structStringifiedAll), + want: addr(structStringifiedAll{ + Bool: true, + String: "hello", + Bytes: []byte{1, 2, 3}, + Int: -64, // may be stringified + Uint: +64, // may be stringified + Float: 3.14159, // may be stringified + Map: map[string]string{"key": "value"}, + StructScalars: structScalars{ + Bool: true, + String: "hello", + Bytes: []byte{1, 2, 3}, + Int: -64, // may be stringified + Uint: +64, // may be stringified + Float: 3.14159, // may be stringified + }, + StructMaps: structMaps{ + MapBool: map[string]bool{"": true}, + MapString: map[string]string{"": "hello"}, + MapBytes: map[string][]byte{"": {1, 2, 3}}, + MapInt: map[string]int64{"": -64}, // may be stringified + MapUint: map[string]uint64{"": +64}, // may be stringified + MapFloat: map[string]float64{"": 3.14159}, // may be stringified + }, + StructSlices: structSlices{ + SliceBool: []bool{true}, + SliceString: []string{"hello"}, + SliceBytes: [][]byte{{1, 2, 3}}, + SliceInt: []int64{-64}, // may be stringified + SliceUint: []uint64{+64}, // may be stringified + SliceFloat: []float64{3.14159}, // may be stringified + }, + Slice: []string{"fizz", "buzz"}, + Array: [1]string{"goodbye"}, + Pointer: new(structStringifiedAll), // may be stringified + }), + }, { + name: jsontest.Name("Structs/Stringified/InvalidEmpty"), + inBuf: `{"Int":""}`, + inVal: new(structStringifiedAll), + want: new(structStringifiedAll), + wantErr: EU(strconv.ErrSyntax).withVal(`""`).withPos(`{"Int":`, "/Int").withType('"', T[int64]()), + }, { + name: jsontest.Name("Structs/LegacyStringified"), + opts: []Options{jsonflags.StringifyWithLegacySemantics | 1}, + inBuf: `{ + "Bool": "true", + "String": "\"hello\"", + "Bytes": "AQID", + "Int": "-64", + "Uint": "64", + "Float": "3.14159", + "Map": {"key": "value"}, + "StructScalars": { + "Bool": true, + "String": "hello", + "Bytes": "AQID", + "Int": -64, + "Uint": 64, + "Float": 3.14159 + }, + "StructMaps": { + "MapBool": {"": true}, + "MapString": {"": "hello"}, + "MapBytes": {"": "AQID"}, + "MapInt": {"": -64}, + "MapUint": {"": 64}, + "MapFloat": {"": 3.14159} + }, + "StructSlices": { + "SliceBool": [true], + "SliceString": ["hello"], + "SliceBytes": ["AQID"], + "SliceInt": [-64], + "SliceUint": [64], + "SliceFloat": [3.14159] + }, + "Slice": ["fizz", "buzz"], + "Array": ["goodbye"] +}`, + inVal: new(structStringifiedAll), + want: addr(structStringifiedAll{ + Bool: true, + String: "hello", + Bytes: []byte{1, 2, 3}, + Int: -64, + Uint: +64, + Float: 3.14159, + Map: map[string]string{"key": "value"}, + StructScalars: structScalars{ + Bool: true, + String: "hello", + Bytes: []byte{1, 2, 3}, + Int: -64, + Uint: +64, + Float: 3.14159, + }, + StructMaps: structMaps{ + MapBool: map[string]bool{"": true}, + MapString: map[string]string{"": "hello"}, + MapBytes: map[string][]byte{"": {1, 2, 3}}, + MapInt: map[string]int64{"": -64}, + MapUint: map[string]uint64{"": +64}, + MapFloat: map[string]float64{"": 3.14159}, + }, + StructSlices: structSlices{ + SliceBool: []bool{true}, + SliceString: []string{"hello"}, + SliceBytes: [][]byte{{1, 2, 3}}, + SliceInt: []int64{-64}, + SliceUint: []uint64{+64}, + SliceFloat: []float64{3.14159}, + }, + Slice: []string{"fizz", "buzz"}, + Array: [1]string{"goodbye"}, + }), + }, { + name: jsontest.Name("Structs/LegacyStringified/InvalidBool"), + opts: []Options{jsonflags.StringifyWithLegacySemantics | 1}, + inBuf: `{"Bool": true}`, + inVal: new(structStringifiedAll), + wantErr: EU(nil).withPos(`{"Bool": `, "/Bool").withType('t', T[bool]()), + }, { + name: jsontest.Name("Structs/LegacyStringified/InvalidString"), + opts: []Options{jsonflags.StringifyWithLegacySemantics | 1}, + inBuf: `{"String": "string"}`, + inVal: new(structStringifiedAll), + wantErr: EU(newInvalidCharacterError("s", "at start of string (expecting '\"')", 0, "")). + withPos(`{"String": `, "/String").withType('"', T[string]()), + }, { + name: jsontest.Name("Structs/Format/Bytes"), + inBuf: `{ + "Base16": "0123456789abcdef", + "Base32": "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567", + "Base32Hex": "0123456789ABCDEFGHIJKLMNOPQRSTUV", + "Base64": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", + "Base64URL": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", + "Array": [1, 2, 3, 4] +}`, + inVal: new(structFormatBytes), + want: addr(structFormatBytes{ + Base16: []byte("\x01\x23\x45\x67\x89\xab\xcd\xef"), + Base32: []byte("\x00D2\x14\xc7BT\xb65τe:V\xd7\xc6u\xbew\xdf"), + Base32Hex: []byte("\x00D2\x14\xc7BT\xb65τe:V\xd7\xc6u\xbew\xdf"), + Base64: []byte("\x00\x10\x83\x10Q\x87 \x92\x8b0ӏA\x14\x93QU\x97a\x96\x9bqן\x82\x18\xa3\x92Y\xa7\xa2\x9a\xab\xb2ۯ\xc3\x1c\xb3\xd3]\xb7㞻\xf3߿"), + Base64URL: []byte("\x00\x10\x83\x10Q\x87 \x92\x8b0ӏA\x14\x93QU\x97a\x96\x9bqן\x82\x18\xa3\x92Y\xa7\xa2\x9a\xab\xb2ۯ\xc3\x1c\xb3\xd3]\xb7㞻\xf3߿"), + Array: []byte{1, 2, 3, 4}, + }), + }, { + name: jsontest.Name("Structs/Format/ArrayBytes"), + inBuf: `{ + "Base16": "01020304", + "Base32": "AEBAGBA=", + "Base32Hex": "0410610=", + "Base64": "AQIDBA==", + "Base64URL": "AQIDBA==", + "Array": [1, 2, 3, 4], + "Default": "AQIDBA==" +}`, + inVal: new(structFormatArrayBytes), + want: addr(structFormatArrayBytes{ + Base16: [4]byte{1, 2, 3, 4}, + Base32: [4]byte{1, 2, 3, 4}, + Base32Hex: [4]byte{1, 2, 3, 4}, + Base64: [4]byte{1, 2, 3, 4}, + Base64URL: [4]byte{1, 2, 3, 4}, + Array: [4]byte{1, 2, 3, 4}, + Default: [4]byte{1, 2, 3, 4}, + }), + }, { + name: jsontest.Name("Structs/Format/ArrayBytes/Legacy"), + opts: []Options{jsonflags.FormatBytesWithLegacySemantics | 1}, + inBuf: `{ + "Base16": "01020304", + "Base32": "AEBAGBA=", + "Base32Hex": "0410610=", + "Base64": "AQIDBA==", + "Base64URL": "AQIDBA==", + "Array": [1, 2, 3, 4], + "Default": [1, 2, 3, 4] +}`, + inVal: new(structFormatArrayBytes), + want: addr(structFormatArrayBytes{ + Base16: [4]byte{1, 2, 3, 4}, + Base32: [4]byte{1, 2, 3, 4}, + Base32Hex: [4]byte{1, 2, 3, 4}, + Base64: [4]byte{1, 2, 3, 4}, + Base64URL: [4]byte{1, 2, 3, 4}, + Array: [4]byte{1, 2, 3, 4}, + Default: [4]byte{1, 2, 3, 4}, + }), + }, { + name: jsontest.Name("Structs/Format/Bytes/Array"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *byte) error { + if string(b) == "true" { + *v = 1 + } else { + *v = 0 + } + return nil + })), + }, + inBuf: `{"Array":[false,true,false,true,false,true]}`, + inVal: new(struct { + Array []byte `json:",format:array"` + }), + want: addr(struct { + Array []byte `json:",format:array"` + }{ + Array: []byte{0, 1, 0, 1, 0, 1}, + }), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base16/WrongKind"), + inBuf: `{"Base16": [1,2,3,4]}`, + inVal: new(structFormatBytes), + wantErr: EU(nil).withPos(`{"Base16": `, "/Base16").withType('[', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base16/AllPadding"), + inBuf: `{"Base16": "===="}`, + inVal: new(structFormatBytes), + wantErr: EU(func() error { + _, err := hex.Decode(make([]byte, 2), []byte("=====")) + return err + }()).withPos(`{"Base16": `, "/Base16").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base16/EvenPadding"), + inBuf: `{"Base16": "0123456789abcdef="}`, + inVal: new(structFormatBytes), + wantErr: EU(func() error { + _, err := hex.Decode(make([]byte, 8), []byte("0123456789abcdef=")) + return err + }()).withPos(`{"Base16": `, "/Base16").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base16/OddPadding"), + inBuf: `{"Base16": "0123456789abcdef0="}`, + inVal: new(structFormatBytes), + wantErr: EU(func() error { + _, err := hex.Decode(make([]byte, 9), []byte("0123456789abcdef0=")) + return err + }()).withPos(`{"Base16": `, "/Base16").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base16/NonAlphabet/LineFeed"), + inBuf: `{"Base16": "aa\naa"}`, + inVal: new(structFormatBytes), + wantErr: EU(func() error { + _, err := hex.Decode(make([]byte, 9), []byte("aa\naa")) + return err + }()).withPos(`{"Base16": `, "/Base16").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base16/NonAlphabet/CarriageReturn"), + inBuf: `{"Base16": "aa\raa"}`, + inVal: new(structFormatBytes), + wantErr: EU(func() error { + _, err := hex.Decode(make([]byte, 9), []byte("aa\raa")) + return err + }()).withPos(`{"Base16": `, "/Base16").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base16/NonAlphabet/Space"), + inBuf: `{"Base16": "aa aa"}`, + inVal: new(structFormatBytes), + wantErr: EU(func() error { + _, err := hex.Decode(make([]byte, 9), []byte("aa aa")) + return err + }()).withPos(`{"Base16": `, "/Base16").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base32/Padding"), + inBuf: `[ + {"Base32": "NA======"}, + {"Base32": "NBSQ===="}, + {"Base32": "NBSWY==="}, + {"Base32": "NBSWY3A="}, + {"Base32": "NBSWY3DP"} + ]`, + inVal: new([]structFormatBytes), + want: addr([]structFormatBytes{ + {Base32: []byte("h")}, + {Base32: []byte("he")}, + {Base32: []byte("hel")}, + {Base32: []byte("hell")}, + {Base32: []byte("hello")}, + }), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base32/Invalid/NoPadding"), + inBuf: `[ + {"Base32": "NA"}, + {"Base32": "NBSQ"}, + {"Base32": "NBSWY"}, + {"Base32": "NBSWY3A"}, + {"Base32": "NBSWY3DP"} + ]`, + inVal: new([]structFormatBytes), + wantErr: EU(func() error { + _, err := base32.StdEncoding.Decode(make([]byte, 1), []byte("NA")) + return err + }()).withPos(`[`+"\n\t\t\t\t"+`{"Base32": `, "/0/Base32").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base32/WrongAlphabet"), + inBuf: `{"Base32": "0123456789ABCDEFGHIJKLMNOPQRSTUV"}`, + inVal: new(structFormatBytes), + wantErr: EU(func() error { + _, err := base32.StdEncoding.Decode(make([]byte, 20), []byte("0123456789ABCDEFGHIJKLMNOPQRSTUV")) + return err + }()).withPos(`{"Base32": `, "/Base32").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base32Hex/WrongAlphabet"), + inBuf: `{"Base32Hex": "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"}`, + inVal: new(structFormatBytes), + wantErr: EU(func() error { + _, err := base32.HexEncoding.Decode(make([]byte, 20), []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ234567")) + return err + }()).withPos(`{"Base32Hex": `, "/Base32Hex").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base32/NonAlphabet/LineFeed"), + inBuf: `{"Base32": "AAAA\nAAAA"}`, + inVal: new(structFormatBytes), + wantErr: EU(errors.New("illegal character '\\n' at offset 4")).withPos(`{"Base32": `, "/Base32").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base32/NonAlphabet/CarriageReturn"), + inBuf: `{"Base32": "AAAA\rAAAA"}`, + inVal: new(structFormatBytes), + wantErr: EU(errors.New("illegal character '\\r' at offset 4")).withPos(`{"Base32": `, "/Base32").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base32/NonAlphabet/Space"), + inBuf: `{"Base32": "AAAA AAAA"}`, + inVal: new(structFormatBytes), + wantErr: EU(base32.CorruptInputError(4)).withPos(`{"Base32": `, "/Base32").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base64/WrongAlphabet"), + inBuf: `{"Base64": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"}`, + inVal: new(structFormatBytes), + wantErr: EU(func() error { + _, err := base64.StdEncoding.Decode(make([]byte, 48), []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_")) + return err + }()).withPos(`{"Base64": `, "/Base64").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base64URL/WrongAlphabet"), + inBuf: `{"Base64URL": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"}`, + inVal: new(structFormatBytes), + wantErr: EU(func() error { + _, err := base64.URLEncoding.Decode(make([]byte, 48), []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/")) + return err + }()).withPos(`{"Base64URL": `, "/Base64URL").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base64/NonAlphabet/LineFeed"), + inBuf: `{"Base64": "aa=\n="}`, + inVal: new(structFormatBytes), + wantErr: EU(errors.New("illegal character '\\n' at offset 3")).withPos(`{"Base64": `, "/Base64").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base64/NonAlphabet/CarriageReturn"), + inBuf: `{"Base64": "aa=\r="}`, + inVal: new(structFormatBytes), + wantErr: EU(errors.New("illegal character '\\r' at offset 3")).withPos(`{"Base64": `, "/Base64").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Bytes/Base64/NonAlphabet/Ignored"), + opts: []Options{jsonflags.ParseBytesWithLooseRFC4648 | 1}, + inBuf: `{"Base64": "aa=\r\n="}`, + inVal: new(structFormatBytes), + want: &structFormatBytes{Base64: []byte{105}}, + }, { + name: jsontest.Name("Structs/Format/Bytes/Invalid/Base64/NonAlphabet/Space"), + inBuf: `{"Base64": "aa= ="}`, + inVal: new(structFormatBytes), + wantErr: EU(base64.CorruptInputError(2)).withPos(`{"Base64": `, "/Base64").withType('"', T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Floats"), + inBuf: `[ + {"NonFinite": 3.141592653589793, "PointerNonFinite": 3.141592653589793}, + {"NonFinite": "-Infinity", "PointerNonFinite": "-Infinity"}, + {"NonFinite": "Infinity", "PointerNonFinite": "Infinity"} +]`, + inVal: new([]structFormatFloats), + want: addr([]structFormatFloats{ + {NonFinite: math.Pi, PointerNonFinite: addr(math.Pi)}, + {NonFinite: math.Inf(-1), PointerNonFinite: addr(math.Inf(-1))}, + {NonFinite: math.Inf(+1), PointerNonFinite: addr(math.Inf(+1))}, + }), + }, { + name: jsontest.Name("Structs/Format/Floats/NaN"), + inBuf: `{"NonFinite": "NaN"}`, + inVal: new(structFormatFloats), + // Avoid checking want since reflect.DeepEqual fails for NaNs. + }, { + name: jsontest.Name("Structs/Format/Floats/Invalid/NaN"), + inBuf: `{"NonFinite": "nan"}`, + inVal: new(structFormatFloats), + wantErr: EU(nil).withPos(`{"NonFinite": `, "/NonFinite").withType('"', T[float64]()), + }, { + name: jsontest.Name("Structs/Format/Floats/Invalid/PositiveInfinity"), + inBuf: `{"NonFinite": "+Infinity"}`, + inVal: new(structFormatFloats), + wantErr: EU(nil).withPos(`{"NonFinite": `, "/NonFinite").withType('"', T[float64]()), + }, { + name: jsontest.Name("Structs/Format/Floats/Invalid/NegativeInfinitySpace"), + inBuf: `{"NonFinite": "-Infinity "}`, + inVal: new(structFormatFloats), + wantErr: EU(nil).withPos(`{"NonFinite": `, "/NonFinite").withType('"', T[float64]()), + }, { + name: jsontest.Name("Structs/Format/Maps"), + inBuf: `[ + {"EmitNull": null, "PointerEmitNull": null, "EmitEmpty": null, "PointerEmitEmpty": null, "EmitDefault": null, "PointerEmitDefault": null}, + {"EmitNull": {}, "PointerEmitNull": {}, "EmitEmpty": {}, "PointerEmitEmpty": {}, "EmitDefault": {}, "PointerEmitDefault": {}}, + {"EmitNull": {"k": "v"}, "PointerEmitNull": {"k": "v"}, "EmitEmpty": {"k": "v"}, "PointerEmitEmpty": {"k": "v"}, "EmitDefault": {"k": "v"}, "PointerEmitDefault": {"k": "v"}} +]`, + inVal: new([]structFormatMaps), + want: addr([]structFormatMaps{{ + EmitNull: map[string]string(nil), PointerEmitNull: (*map[string]string)(nil), + EmitEmpty: map[string]string(nil), PointerEmitEmpty: (*map[string]string)(nil), + EmitDefault: map[string]string(nil), PointerEmitDefault: (*map[string]string)(nil), + }, { + EmitNull: map[string]string{}, PointerEmitNull: addr(map[string]string{}), + EmitEmpty: map[string]string{}, PointerEmitEmpty: addr(map[string]string{}), + EmitDefault: map[string]string{}, PointerEmitDefault: addr(map[string]string{}), + }, { + EmitNull: map[string]string{"k": "v"}, PointerEmitNull: addr(map[string]string{"k": "v"}), + EmitEmpty: map[string]string{"k": "v"}, PointerEmitEmpty: addr(map[string]string{"k": "v"}), + EmitDefault: map[string]string{"k": "v"}, PointerEmitDefault: addr(map[string]string{"k": "v"}), + }}), + }, { + name: jsontest.Name("Structs/Format/Slices"), + inBuf: `[ + {"EmitNull": null, "PointerEmitNull": null, "EmitEmpty": null, "PointerEmitEmpty": null, "EmitDefault": null, "PointerEmitDefault": null}, + {"EmitNull": [], "PointerEmitNull": [], "EmitEmpty": [], "PointerEmitEmpty": [], "EmitDefault": [], "PointerEmitDefault": []}, + {"EmitNull": ["v"], "PointerEmitNull": ["v"], "EmitEmpty": ["v"], "PointerEmitEmpty": ["v"], "EmitDefault": ["v"], "PointerEmitDefault": ["v"]} +]`, + inVal: new([]structFormatSlices), + want: addr([]structFormatSlices{{ + EmitNull: []string(nil), PointerEmitNull: (*[]string)(nil), + EmitEmpty: []string(nil), PointerEmitEmpty: (*[]string)(nil), + EmitDefault: []string(nil), PointerEmitDefault: (*[]string)(nil), + }, { + EmitNull: []string{}, PointerEmitNull: addr([]string{}), + EmitEmpty: []string{}, PointerEmitEmpty: addr([]string{}), + EmitDefault: []string{}, PointerEmitDefault: addr([]string{}), + }, { + EmitNull: []string{"v"}, PointerEmitNull: addr([]string{"v"}), + EmitEmpty: []string{"v"}, PointerEmitEmpty: addr([]string{"v"}), + EmitDefault: []string{"v"}, PointerEmitDefault: addr([]string{"v"}), + }}), + }, { + name: jsontest.Name("Structs/Format/Invalid/Bool"), + inBuf: `{"Bool":true}`, + inVal: new(structFormatInvalid), + wantErr: EU(errInvalidFormatFlag).withPos(`{"Bool":`, "/Bool").withType(0, T[bool]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/String"), + inBuf: `{"String": "string"}`, + inVal: new(structFormatInvalid), + wantErr: EU(errInvalidFormatFlag).withPos(`{"String": `, "/String").withType(0, T[string]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Bytes"), + inBuf: `{"Bytes": "bytes"}`, + inVal: new(structFormatInvalid), + wantErr: EU(errInvalidFormatFlag).withPos(`{"Bytes": `, "/Bytes").withType(0, T[[]byte]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Int"), + inBuf: `{"Int": 1}`, + inVal: new(structFormatInvalid), + wantErr: EU(errInvalidFormatFlag).withPos(`{"Int": `, "/Int").withType(0, T[int64]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Uint"), + inBuf: `{"Uint": 1}`, + inVal: new(structFormatInvalid), + wantErr: EU(errInvalidFormatFlag).withPos(`{"Uint": `, "/Uint").withType(0, T[uint64]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Float"), + inBuf: `{"Float" : 1}`, + inVal: new(structFormatInvalid), + wantErr: EU(errInvalidFormatFlag).withPos(`{"Float" : `, "/Float").withType(0, T[float64]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Map"), + inBuf: `{"Map":{}}`, + inVal: new(structFormatInvalid), + wantErr: EU(errInvalidFormatFlag).withPos(`{"Map":`, "/Map").withType(0, T[map[string]string]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Struct"), + inBuf: `{"Struct": {}}`, + inVal: new(structFormatInvalid), + wantErr: EU(errInvalidFormatFlag).withPos(`{"Struct": `, "/Struct").withType(0, T[structAll]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Slice"), + inBuf: `{"Slice": {}}`, + inVal: new(structFormatInvalid), + wantErr: EU(errInvalidFormatFlag).withPos(`{"Slice": `, "/Slice").withType(0, T[[]string]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Array"), + inBuf: `{"Array": []}`, + inVal: new(structFormatInvalid), + wantErr: EU(errInvalidFormatFlag).withPos(`{"Array": `, "/Array").withType(0, T[[1]string]()), + }, { + name: jsontest.Name("Structs/Format/Invalid/Interface"), + inBuf: `{"Interface": "anything"}`, + inVal: new(structFormatInvalid), + wantErr: EU(errInvalidFormatFlag).withPos(`{"Interface": `, "/Interface").withType(0, T[any]()), + }, { + name: jsontest.Name("Structs/Inline/Zero"), + inBuf: `{"D":""}`, + inVal: new(structInlined), + want: new(structInlined), + }, { + name: jsontest.Name("Structs/Inline/Alloc"), + inBuf: `{"E":"","F":"","G":"","A":"","B":"","D":""}`, + inVal: new(structInlined), + want: addr(structInlined{ + X: structInlinedL1{ + X: &structInlinedL2{}, + StructEmbed1: StructEmbed1{}, + }, + StructEmbed2: &StructEmbed2{}, + }), + }, { + name: jsontest.Name("Structs/Inline/NonZero"), + inBuf: `{"E":"E3","F":"F3","G":"G3","A":"A1","B":"B1","D":"D2"}`, + inVal: new(structInlined), + want: addr(structInlined{ + X: structInlinedL1{ + X: &structInlinedL2{A: "A1", B: "B1" /* C: "C1" */}, + StructEmbed1: StructEmbed1{ /* C: "C2" */ D: "D2" /* E: "E2" */}, + }, + StructEmbed2: &StructEmbed2{E: "E3", F: "F3", G: "G3"}, + }), + }, { + name: jsontest.Name("Structs/Inline/Merge"), + inBuf: `{"E":"E3","F":"F3","G":"G3","A":"A1","B":"B1","D":"D2"}`, + inVal: addr(structInlined{ + X: structInlinedL1{ + X: &structInlinedL2{B: "##", C: "C1"}, + StructEmbed1: StructEmbed1{C: "C2", E: "E2"}, + }, + StructEmbed2: &StructEmbed2{E: "##", G: "G3"}, + }), + want: addr(structInlined{ + X: structInlinedL1{ + X: &structInlinedL2{A: "A1", B: "B1", C: "C1"}, + StructEmbed1: StructEmbed1{C: "C2", D: "D2", E: "E2"}, + }, + StructEmbed2: &StructEmbed2{E: "E3", F: "F3", G: "G3"}, + }), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/Noop"), + inBuf: `{"A":1,"B":2}`, + inVal: new(structInlineTextValue), + want: addr(structInlineTextValue{A: 1, X: jsontext.Value(nil), B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/MergeN1/Nil"), + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: new(structInlineTextValue), + want: addr(structInlineTextValue{A: 1, X: jsontext.Value(`{"fizz":"buzz"}`), B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/MergeN1/Empty"), + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: addr(structInlineTextValue{X: jsontext.Value{}}), + want: addr(structInlineTextValue{A: 1, X: jsontext.Value(`{"fizz":"buzz"}`), B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/MergeN1/Whitespace"), + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: addr(structInlineTextValue{X: jsontext.Value("\n\r\t ")}), + want: addr(structInlineTextValue{A: 1, X: jsontext.Value("")}), + wantErr: EU(errRawInlinedNotObject).withPos(`{"A":1,`, "/fizz").withType('"', T[jsontext.Value]()), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/MergeN1/Null"), + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: addr(structInlineTextValue{X: jsontext.Value("null")}), + want: addr(structInlineTextValue{A: 1, X: jsontext.Value("null")}), + wantErr: EU(errRawInlinedNotObject).withPos(`{"A":1,`, "/fizz").withType('"', T[jsontext.Value]()), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/MergeN1/ObjectN0"), + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: addr(structInlineTextValue{X: jsontext.Value(` { } `)}), + want: addr(structInlineTextValue{A: 1, X: jsontext.Value(` {"fizz":"buzz"}`), B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/MergeN2/ObjectN1"), + inBuf: `{"A":1,"fizz":"buzz","B":2,"foo": [ 1 , 2 , 3 ]}`, + inVal: addr(structInlineTextValue{X: jsontext.Value(` { "fizz" : "buzz" } `)}), + want: addr(structInlineTextValue{A: 1, X: jsontext.Value(` { "fizz" : "buzz","fizz":"buzz","foo":[ 1 , 2 , 3 ]}`), B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/Merge/EndObject"), + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: addr(structInlineTextValue{X: jsontext.Value(` } `)}), + // NOTE: This produces invalid output, + // but the value being merged into is already invalid. + want: addr(structInlineTextValue{A: 1, X: jsontext.Value(`,"fizz":"buzz"}`), B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/MergeInvalidValue"), + inBuf: `{"A":1,"fizz":nil,"B":2}`, + inVal: new(structInlineTextValue), + want: addr(structInlineTextValue{A: 1, X: jsontext.Value(`{"fizz":`)}), + wantErr: newInvalidCharacterError("i", "in literal null (expecting 'u')", len64(`{"A":1,"fizz":n`), "/fizz"), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/CaseSensitive"), + inBuf: `{"A":1,"fizz":"buzz","B":2,"a":3}`, + inVal: new(structInlineTextValue), + want: addr(structInlineTextValue{A: 1, X: jsontext.Value(`{"fizz":"buzz","a":3}`), B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/RejectDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(false)}, + inBuf: `{"A":1,"fizz":"buzz","B":2,"fizz":"buzz"}`, + inVal: new(structInlineTextValue), + want: addr(structInlineTextValue{A: 1, X: jsontext.Value(`{"fizz":"buzz"}`), B: 2}), + wantErr: newDuplicateNameError("", []byte(`"fizz"`), len64(`{"A":1,"fizz":"buzz","B":2,`)), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/AllowDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(true)}, + inBuf: `{"A":1,"fizz":"buzz","B":2,"fizz":"buzz"}`, + inVal: new(structInlineTextValue), + want: addr(structInlineTextValue{A: 1, X: jsontext.Value(`{"fizz":"buzz","fizz":"buzz"}`), B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/Nested/Noop"), + inBuf: `{}`, + inVal: new(structInlinePointerInlineTextValue), + want: new(structInlinePointerInlineTextValue), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/Nested/Alloc"), + inBuf: `{"A":1,"fizz":"buzz"}`, + inVal: new(structInlinePointerInlineTextValue), + want: addr(structInlinePointerInlineTextValue{ + X: &struct { + A int + X jsontext.Value `json:",inline"` + }{A: 1, X: jsontext.Value(`{"fizz":"buzz"}`)}, + }), + }, { + name: jsontest.Name("Structs/InlinedFallback/TextValue/Nested/Merge"), + inBuf: `{"fizz":"buzz"}`, + inVal: addr(structInlinePointerInlineTextValue{ + X: &struct { + A int + X jsontext.Value `json:",inline"` + }{A: 1}, + }), + want: addr(structInlinePointerInlineTextValue{ + X: &struct { + A int + X jsontext.Value `json:",inline"` + }{A: 1, X: jsontext.Value(`{"fizz":"buzz"}`)}, + }), + }, { + name: jsontest.Name("Structs/InlinedFallback/PointerTextValue/Noop"), + inBuf: `{"A":1,"B":2}`, + inVal: new(structInlinePointerTextValue), + want: addr(structInlinePointerTextValue{A: 1, X: nil, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/PointerTextValue/Alloc"), + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: new(structInlinePointerTextValue), + want: addr(structInlinePointerTextValue{A: 1, X: addr(jsontext.Value(`{"fizz":"buzz"}`)), B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/PointerTextValue/Merge"), + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: addr(structInlinePointerTextValue{X: addr(jsontext.Value(`{"fizz":"buzz"}`))}), + want: addr(structInlinePointerTextValue{A: 1, X: addr(jsontext.Value(`{"fizz":"buzz","fizz":"buzz"}`)), B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/PointerTextValue/Nested/Nil"), + inBuf: `{"fizz":"buzz"}`, + inVal: new(structInlineInlinePointerTextValue), + want: addr(structInlineInlinePointerTextValue{ + X: struct { + X *jsontext.Value `json:",inline"` + }{X: addr(jsontext.Value(`{"fizz":"buzz"}`))}, + }), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/Noop"), + inBuf: `{"A":1,"B":2}`, + inVal: new(structInlineMapStringAny), + want: addr(structInlineMapStringAny{A: 1, X: nil, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/MergeN1/Nil"), + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: new(structInlineMapStringAny), + want: addr(structInlineMapStringAny{A: 1, X: jsonObject{"fizz": "buzz"}, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/MergeN1/Empty"), + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: addr(structInlineMapStringAny{X: jsonObject{}}), + want: addr(structInlineMapStringAny{A: 1, X: jsonObject{"fizz": "buzz"}, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/MergeN1/ObjectN1"), + inBuf: `{"A":1,"fizz":{"charlie":"DELTA","echo":"foxtrot"},"B":2}`, + inVal: addr(structInlineMapStringAny{X: jsonObject{"fizz": jsonObject{ + "alpha": "bravo", + "charlie": "delta", + }}}), + want: addr(structInlineMapStringAny{A: 1, X: jsonObject{"fizz": jsonObject{ + "alpha": "bravo", + "charlie": "DELTA", + "echo": "foxtrot", + }}, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/MergeN2/ObjectN1"), + inBuf: `{"A":1,"fizz":"buzz","B":2,"foo": [ 1 , 2 , 3 ]}`, + inVal: addr(structInlineMapStringAny{X: jsonObject{"fizz": "wuzz"}}), + want: addr(structInlineMapStringAny{A: 1, X: jsonObject{"fizz": "buzz", "foo": jsonArray{1.0, 2.0, 3.0}}, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/MergeInvalidValue"), + inBuf: `{"A":1,"fizz":nil,"B":2}`, + inVal: new(structInlineMapStringAny), + want: addr(structInlineMapStringAny{A: 1, X: jsonObject{"fizz": nil}}), + wantErr: newInvalidCharacterError("i", "in literal null (expecting 'u')", len64(`{"A":1,"fizz":n`), "/fizz"), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/MergeInvalidValue/Existing"), + inBuf: `{"A":1,"fizz":nil,"B":2}`, + inVal: addr(structInlineMapStringAny{A: 1, X: jsonObject{"fizz": true}}), + want: addr(structInlineMapStringAny{A: 1, X: jsonObject{"fizz": true}}), + wantErr: newInvalidCharacterError("i", "in literal null (expecting 'u')", len64(`{"A":1,"fizz":n`), "/fizz"), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/CaseSensitive"), + inBuf: `{"A":1,"fizz":"buzz","B":2,"a":3}`, + inVal: new(structInlineMapStringAny), + want: addr(structInlineMapStringAny{A: 1, X: jsonObject{"fizz": "buzz", "a": 3.0}, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/RejectDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(false)}, + inBuf: `{"A":1,"fizz":"buzz","B":2,"fizz":"buzz"}`, + inVal: new(structInlineMapStringAny), + want: addr(structInlineMapStringAny{A: 1, X: jsonObject{"fizz": "buzz"}, B: 2}), + wantErr: newDuplicateNameError("", []byte(`"fizz"`), len64(`{"A":1,"fizz":"buzz","B":2,`)), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/AllowDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(true)}, + inBuf: `{"A":1,"fizz":{"one":1,"two":-2},"B":2,"fizz":{"two":2,"three":3}}`, + inVal: new(structInlineMapStringAny), + want: addr(structInlineMapStringAny{A: 1, X: jsonObject{"fizz": jsonObject{"one": 1.0, "two": 2.0, "three": 3.0}}, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/Nested/Noop"), + inBuf: `{}`, + inVal: new(structInlinePointerInlineMapStringAny), + want: new(structInlinePointerInlineMapStringAny), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/Nested/Alloc"), + inBuf: `{"A":1,"fizz":"buzz"}`, + inVal: new(structInlinePointerInlineMapStringAny), + want: addr(structInlinePointerInlineMapStringAny{ + X: &struct { + A int + X jsonObject `json:",inline"` + }{A: 1, X: jsonObject{"fizz": "buzz"}}, + }), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringAny/Nested/Merge"), + inBuf: `{"fizz":"buzz"}`, + inVal: addr(structInlinePointerInlineMapStringAny{ + X: &struct { + A int + X jsonObject `json:",inline"` + }{A: 1}, + }), + want: addr(structInlinePointerInlineMapStringAny{ + X: &struct { + A int + X jsonObject `json:",inline"` + }{A: 1, X: jsonObject{"fizz": "buzz"}}, + }), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringInt/UnmarshalFunc"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *any) error { + var err error + *v, err = strconv.ParseFloat(string(bytes.Trim(b, `"`)), 64) + return err + })), + }, + inBuf: `{"D":"1.1","E":"2.2","F":"3.3"}`, + inVal: new(structInlineMapStringAny), + want: addr(structInlineMapStringAny{X: jsonObject{"D": 1.1, "E": 2.2, "F": 3.3}}), + }, { + name: jsontest.Name("Structs/InlinedFallback/PointerMapStringAny/Noop"), + inBuf: `{"A":1,"B":2}`, + inVal: new(structInlinePointerMapStringAny), + want: addr(structInlinePointerMapStringAny{A: 1, X: nil, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/PointerMapStringAny/Alloc"), + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: new(structInlinePointerMapStringAny), + want: addr(structInlinePointerMapStringAny{A: 1, X: addr(jsonObject{"fizz": "buzz"}), B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/PointerMapStringAny/Merge"), + inBuf: `{"A":1,"fizz":"wuzz","B":2}`, + inVal: addr(structInlinePointerMapStringAny{X: addr(jsonObject{"fizz": "buzz"})}), + want: addr(structInlinePointerMapStringAny{A: 1, X: addr(jsonObject{"fizz": "wuzz"}), B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/PointerMapStringAny/Nested/Nil"), + inBuf: `{"fizz":"buzz"}`, + inVal: new(structInlineInlinePointerMapStringAny), + want: addr(structInlineInlinePointerMapStringAny{ + X: struct { + X *jsonObject `json:",inline"` + }{X: addr(jsonObject{"fizz": "buzz"})}, + }), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringInt"), + inBuf: `{"zero": 0, "one": 1, "two": 2}`, + inVal: new(structInlineMapStringInt), + want: addr(structInlineMapStringInt{ + X: map[string]int{"zero": 0, "one": 1, "two": 2}, + }), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringInt/Null"), + inBuf: `{"zero": 0, "one": null, "two": 2}`, + inVal: new(structInlineMapStringInt), + want: addr(structInlineMapStringInt{ + X: map[string]int{"zero": 0, "one": 0, "two": 2}, + }), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringInt/Invalid"), + inBuf: `{"zero": 0, "one": {}, "two": 2}`, + inVal: new(structInlineMapStringInt), + want: addr(structInlineMapStringInt{ + X: map[string]int{"zero": 0, "one": 0}, + }), + wantErr: EU(nil).withPos(`{"zero": 0, "one": `, "/one").withType('{', T[int]()), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringInt/StringifiedNumbers"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `{"zero": "0", "one": "1", "two": "2"}`, + inVal: new(structInlineMapStringInt), + want: addr(structInlineMapStringInt{ + X: map[string]int{"zero": 0, "one": 1, "two": 2}, + }), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapStringInt/UnmarshalFunc"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *int) error { + i, err := strconv.ParseInt(string(bytes.Trim(b, `"`)), 10, 64) + if err != nil { + return err + } + *v = int(i) + return nil + })), + }, + inBuf: `{"zero": "0", "one": "1", "two": "2"}`, + inVal: new(structInlineMapStringInt), + want: addr(structInlineMapStringInt{ + X: map[string]int{"zero": 0, "one": 1, "two": 2}, + }), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringInt"), + inBuf: `{"zero": 0, "one": 1, "two": 2}`, + inVal: new(structInlineMapNamedStringInt), + want: addr(structInlineMapNamedStringInt{ + X: map[namedString]int{"zero": 0, "one": 1, "two": 2}, + }), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringInt/Null"), + inBuf: `{"zero": 0, "one": null, "two": 2}`, + inVal: new(structInlineMapNamedStringInt), + want: addr(structInlineMapNamedStringInt{ + X: map[namedString]int{"zero": 0, "one": 0, "two": 2}, + }), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringInt/Invalid"), + inBuf: `{"zero": 0, "one": {}, "two": 2}`, + inVal: new(structInlineMapNamedStringInt), + want: addr(structInlineMapNamedStringInt{ + X: map[namedString]int{"zero": 0, "one": 0}, + }), + wantErr: EU(nil).withPos(`{"zero": 0, "one": `, "/one").withType('{', T[int]()), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringInt/StringifiedNumbers"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `{"zero": "0", "one": 1, "two": "2"}`, + inVal: new(structInlineMapNamedStringInt), + want: addr(structInlineMapNamedStringInt{ + X: map[namedString]int{"zero": 0, "one": 0}, + }), + wantErr: EU(nil).withPos(`{"zero": "0", "one": `, "/one").withType('0', T[int]()), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringInt/UnmarshalFunc"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *int) error { + i, err := strconv.ParseInt(string(bytes.Trim(b, `"`)), 10, 64) + if err != nil { + return err + } + *v = int(i) + return nil + })), + }, + inBuf: `{"zero": "0", "one": "1", "two": "2"}`, + inVal: new(structInlineMapNamedStringInt), + want: addr(structInlineMapNamedStringInt{ + X: map[namedString]int{"zero": 0, "one": 1, "two": 2}, + }), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/Noop"), + inBuf: `{"A":1,"B":2}`, + inVal: new(structInlineMapNamedStringAny), + want: addr(structInlineMapNamedStringAny{A: 1, X: nil, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/MergeN1/Nil"), + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: new(structInlineMapNamedStringAny), + want: addr(structInlineMapNamedStringAny{A: 1, X: map[namedString]any{"fizz": "buzz"}, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/MergeN1/Empty"), + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: addr(structInlineMapNamedStringAny{X: map[namedString]any{}}), + want: addr(structInlineMapNamedStringAny{A: 1, X: map[namedString]any{"fizz": "buzz"}, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/MergeN1/ObjectN1"), + inBuf: `{"A":1,"fizz":{"charlie":"DELTA","echo":"foxtrot"},"B":2}`, + inVal: addr(structInlineMapNamedStringAny{X: map[namedString]any{"fizz": jsonObject{ + "alpha": "bravo", + "charlie": "delta", + }}}), + want: addr(structInlineMapNamedStringAny{A: 1, X: map[namedString]any{"fizz": jsonObject{ + "alpha": "bravo", + "charlie": "DELTA", + "echo": "foxtrot", + }}, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/MergeN2/ObjectN1"), + inBuf: `{"A":1,"fizz":"buzz","B":2,"foo": [ 1 , 2 , 3 ]}`, + inVal: addr(structInlineMapNamedStringAny{X: map[namedString]any{"fizz": "wuzz"}}), + want: addr(structInlineMapNamedStringAny{A: 1, X: map[namedString]any{"fizz": "buzz", "foo": jsonArray{1.0, 2.0, 3.0}}, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/MergeInvalidValue"), + inBuf: `{"A":1,"fizz":nil,"B":2}`, + inVal: new(structInlineMapNamedStringAny), + want: addr(structInlineMapNamedStringAny{A: 1, X: map[namedString]any{"fizz": nil}}), + wantErr: newInvalidCharacterError("i", "in literal null (expecting 'u')", len64(`{"A":1,"fizz":n`), "/fizz"), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/MergeInvalidValue/Existing"), + inBuf: `{"A":1,"fizz":nil,"B":2}`, + inVal: addr(structInlineMapNamedStringAny{A: 1, X: map[namedString]any{"fizz": true}}), + want: addr(structInlineMapNamedStringAny{A: 1, X: map[namedString]any{"fizz": true}}), + wantErr: newInvalidCharacterError("i", "in literal null (expecting 'u')", len64(`{"A":1,"fizz":n`), "/fizz"), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/CaseSensitive"), + inBuf: `{"A":1,"fizz":"buzz","B":2,"a":3}`, + inVal: new(structInlineMapNamedStringAny), + want: addr(structInlineMapNamedStringAny{A: 1, X: map[namedString]any{"fizz": "buzz", "a": 3.0}, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/RejectDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(false)}, + inBuf: `{"A":1,"fizz":"buzz","B":2,"fizz":"buzz"}`, + inVal: new(structInlineMapNamedStringAny), + want: addr(structInlineMapNamedStringAny{A: 1, X: map[namedString]any{"fizz": "buzz"}, B: 2}), + wantErr: newDuplicateNameError("", []byte(`"fizz"`), len64(`{"A":1,"fizz":"buzz","B":2,`)), + }, { + name: jsontest.Name("Structs/InlinedFallback/MapNamedStringAny/AllowDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(true)}, + inBuf: `{"A":1,"fizz":{"one":1,"two":-2},"B":2,"fizz":{"two":2,"three":3}}`, + inVal: new(structInlineMapNamedStringAny), + want: addr(structInlineMapNamedStringAny{A: 1, X: map[namedString]any{"fizz": map[string]any{"one": 1.0, "two": 2.0, "three": 3.0}}, B: 2}), + }, { + name: jsontest.Name("Structs/InlinedFallback/RejectUnknownMembers"), + opts: []Options{RejectUnknownMembers(true)}, + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: new(structInlineTextValue), + // NOTE: DiscardUnknownMembers has no effect since this is "inline". + want: addr(structInlineTextValue{ + A: 1, + X: jsontext.Value(`{"fizz":"buzz"}`), + B: 2, + }), + }, { + name: jsontest.Name("Structs/UnknownFallback/RejectUnknownMembers"), + opts: []Options{RejectUnknownMembers(true)}, + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: new(structUnknownTextValue), + want: addr(structUnknownTextValue{A: 1}), + wantErr: EU(ErrUnknownName).withPos(`{"A":1,`, "/fizz").withType('"', T[structUnknownTextValue]()), + }, { + name: jsontest.Name("Structs/UnknownFallback"), + inBuf: `{"A":1,"fizz":"buzz","B":2}`, + inVal: new(structUnknownTextValue), + want: addr(structUnknownTextValue{ + A: 1, + X: jsontext.Value(`{"fizz":"buzz"}`), + B: 2, + }), + }, { + name: jsontest.Name("Structs/UnknownIgnored"), + opts: []Options{RejectUnknownMembers(false)}, + inBuf: `{"unknown":"fizzbuzz"}`, + inVal: new(structAll), + want: new(structAll), + }, { + name: jsontest.Name("Structs/RejectUnknownMembers"), + opts: []Options{RejectUnknownMembers(true)}, + inBuf: `{"unknown":"fizzbuzz"}`, + inVal: new(structAll), + want: new(structAll), + wantErr: EU(ErrUnknownName).withPos(`{`, "/unknown").withType('"', T[structAll]()), + }, { + name: jsontest.Name("Structs/UnexportedIgnored"), + inBuf: `{"ignored":"unused"}`, + inVal: new(structUnexportedIgnored), + want: new(structUnexportedIgnored), + }, { + name: jsontest.Name("Structs/IgnoredUnexportedEmbedded"), + inBuf: `{"namedString":"unused"}`, + inVal: new(structIgnoredUnexportedEmbedded), + want: new(structIgnoredUnexportedEmbedded), + }, { + name: jsontest.Name("Structs/WeirdNames"), + inBuf: `{"":"empty",",":"comma","\"":"quote"}`, + inVal: new(structWeirdNames), + want: addr(structWeirdNames{Empty: "empty", Comma: "comma", Quote: "quote"}), + }, { + name: jsontest.Name("Structs/NoCase/Exact"), + inBuf: `{"Aaa":"Aaa","AA_A":"AA_A","AaA":"AaA","AAa":"AAa","AAA":"AAA"}`, + inVal: new(structNoCase), + want: addr(structNoCase{AaA: "AaA", AAa: "AAa", Aaa: "Aaa", AAA: "AAA", AA_A: "AA_A"}), + }, { + name: jsontest.Name("Structs/NoCase/CaseInsensitiveDefault"), + inBuf: `{"aa_a":"aa_a"}`, + inVal: new(structNoCase), + want: addr(structNoCase{AaA: "aa_a"}), + }, { + name: jsontest.Name("Structs/NoCase/MatchCaseSensitiveDelimiter"), + opts: []Options{jsonflags.MatchCaseSensitiveDelimiter | 1}, + inBuf: `{"aa_a":"aa_a"}`, + inVal: new(structNoCase), + want: addr(structNoCase{}), + }, { + name: jsontest.Name("Structs/NoCase/MatchCaseInsensitiveNames+MatchCaseSensitiveDelimiter"), + opts: []Options{MatchCaseInsensitiveNames(true), jsonflags.MatchCaseSensitiveDelimiter | 1}, + inBuf: `{"aa_a":"aa_a"}`, + inVal: new(structNoCase), + want: addr(structNoCase{AA_A: "aa_a"}), + }, { + name: jsontest.Name("Structs/NoCase/Merge/AllowDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(true)}, + inBuf: `{"AaA":"AaA","aaa":"aaa","aAa":"aAa"}`, + inVal: new(structNoCase), + want: addr(structNoCase{AaA: "aAa"}), + }, { + name: jsontest.Name("Structs/NoCase/Merge/RejectDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(false)}, + inBuf: `{"AaA":"AaA","aaa":"aaa"}`, + inVal: new(structNoCase), + want: addr(structNoCase{AaA: "AaA"}), + wantErr: newDuplicateNameError("", []byte(`"aaa"`), len64(`{"AaA":"AaA",`)), + }, { + name: jsontest.Name("Structs/CaseSensitive"), + inBuf: `{"BOOL": true, "STRING": "hello", "BYTES": "AQID", "INT": -64, "UINT": 64, "FLOAT": 3.14159}`, + inVal: new(structScalars), + want: addr(structScalars{}), + }, { + name: jsontest.Name("Structs/DuplicateName/NoCase/ExactDifferent"), + inBuf: `{"AAA":"AAA","AaA":"AaA","AAa":"AAa","Aaa":"Aaa"}`, + inVal: addr(structNoCaseInlineTextValue{}), + want: addr(structNoCaseInlineTextValue{AAA: "AAA", AaA: "AaA", AAa: "AAa", Aaa: "Aaa"}), + }, { + name: jsontest.Name("Structs/DuplicateName/NoCase/ExactConflict"), + inBuf: `{"AAA":"AAA","AAA":"AAA"}`, + inVal: addr(structNoCaseInlineTextValue{}), + want: addr(structNoCaseInlineTextValue{AAA: "AAA"}), + wantErr: newDuplicateNameError("", []byte(`"AAA"`), len64(`{"AAA":"AAA",`)), + }, { + name: jsontest.Name("Structs/DuplicateName/NoCase/OverwriteExact"), + inBuf: `{"AAA":"after"}`, + inVal: addr(structNoCaseInlineTextValue{AAA: "before"}), + want: addr(structNoCaseInlineTextValue{AAA: "after"}), + }, { + name: jsontest.Name("Structs/DuplicateName/NoCase/NoCaseConflict"), + inBuf: `{"aaa":"aaa","aaA":"aaA"}`, + inVal: addr(structNoCaseInlineTextValue{}), + want: addr(structNoCaseInlineTextValue{AaA: "aaa"}), + wantErr: newDuplicateNameError("", []byte(`"aaA"`), len64(`{"aaa":"aaa",`)), + }, { + name: jsontest.Name("Structs/DuplicateName/NoCase/OverwriteNoCase"), + inBuf: `{"aaa":"aaa","aaA":"aaA"}`, + inVal: addr(structNoCaseInlineTextValue{}), + want: addr(structNoCaseInlineTextValue{AaA: "aaa"}), + wantErr: newDuplicateNameError("", []byte(`"aaA"`), len64(`{"aaa":"aaa",`)), + }, { + name: jsontest.Name("Structs/DuplicateName/Inline/Unknown"), + inBuf: `{"unknown":""}`, + inVal: addr(structNoCaseInlineTextValue{}), + want: addr(structNoCaseInlineTextValue{X: jsontext.Value(`{"unknown":""}`)}), + }, { + name: jsontest.Name("Structs/DuplicateName/Inline/UnknownMerge"), + inBuf: `{"unknown":""}`, + inVal: addr(structNoCaseInlineTextValue{X: jsontext.Value(`{"unknown":""}`)}), + want: addr(structNoCaseInlineTextValue{X: jsontext.Value(`{"unknown":"","unknown":""}`)}), + }, { + name: jsontest.Name("Structs/DuplicateName/Inline/NoCaseOkay"), + inBuf: `{"b":"","B":""}`, + inVal: addr(structNoCaseInlineTextValue{}), + want: addr(structNoCaseInlineTextValue{X: jsontext.Value(`{"b":"","B":""}`)}), + }, { + name: jsontest.Name("Structs/DuplicateName/Inline/ExactConflict"), + inBuf: `{"b":"","b":""}`, + inVal: addr(structNoCaseInlineTextValue{}), + want: addr(structNoCaseInlineTextValue{X: jsontext.Value(`{"b":""}`)}), + wantErr: newDuplicateNameError("", []byte(`"b"`), len64(`{"b":"",`)), + }, { + name: jsontest.Name("Structs/Invalid/ErrUnexpectedEOF"), + inBuf: ``, + inVal: addr(structAll{}), + want: addr(structAll{}), + wantErr: &jsontext.SyntacticError{Err: io.ErrUnexpectedEOF}, + }, { + name: jsontest.Name("Structs/Invalid/ErrUnexpectedEOF"), + inBuf: " \n\r\t", + inVal: addr(structAll{}), + want: addr(structAll{}), + wantErr: &jsontext.SyntacticError{Err: io.ErrUnexpectedEOF, ByteOffset: len64(" \n\r\t")}, + }, { + name: jsontest.Name("Structs/Invalid/NestedErrUnexpectedEOF"), + inBuf: `{"Pointer":`, + inVal: addr(structAll{}), + want: addr(structAll{Pointer: new(structAll)}), + wantErr: &jsontext.SyntacticError{ByteOffset: len64(`{"Pointer":`), JSONPointer: "/Pointer", Err: io.ErrUnexpectedEOF}, + }, { + name: jsontest.Name("Structs/Invalid/Conflicting"), + inBuf: `{}`, + inVal: addr(structConflicting{}), + want: addr(structConflicting{}), + wantErr: EU(errors.New(`Go struct fields A and B conflict over JSON object name "conflict"`)).withType('{', T[structConflicting]()), + }, { + name: jsontest.Name("Structs/Invalid/NoneExported"), + inBuf: ` {}`, + inVal: addr(structNoneExported{}), + want: addr(structNoneExported{}), + wantErr: EU(errNoExportedFields).withPos(` `, "").withType('{', T[structNoneExported]()), + }, { + name: jsontest.Name("Structs/Invalid/MalformedTag"), + inBuf: `{}`, + inVal: addr(structMalformedTag{}), + want: addr(structMalformedTag{}), + wantErr: EU(errors.New("Go struct field Malformed has malformed `json` tag: invalid character '\"' at start of option (expecting Unicode letter or single quote)")).withType('{', T[structMalformedTag]()), + }, { + name: jsontest.Name("Structs/Invalid/UnexportedTag"), + inBuf: `{}`, + inVal: addr(structUnexportedTag{}), + want: addr(structUnexportedTag{}), + wantErr: EU(errors.New("unexported Go struct field unexported cannot have non-ignored `json:\"name\"` tag")).withType('{', T[structUnexportedTag]()), + }, { + name: jsontest.Name("Structs/Invalid/ExportedEmbedded"), + inBuf: `{"NamedString":"hello"}`, + inVal: addr(structExportedEmbedded{}), + want: addr(structExportedEmbedded{}), + wantErr: EU(errors.New("embedded Go struct field NamedString of non-struct type must be explicitly given a JSON name")).withType('{', T[structExportedEmbedded]()), + }, { + name: jsontest.Name("Structs/Valid/ExportedEmbedded"), + opts: []Options{jsonflags.ReportErrorsWithLegacySemantics | 1}, + inBuf: `{"NamedString":"hello"}`, + inVal: addr(structExportedEmbedded{}), + want: addr(structExportedEmbedded{"hello"}), + }, { + name: jsontest.Name("Structs/Valid/ExportedEmbeddedTag"), + inBuf: `{"name":"hello"}`, + inVal: addr(structExportedEmbeddedTag{}), + want: addr(structExportedEmbeddedTag{"hello"}), + }, { + name: jsontest.Name("Structs/Invalid/UnexportedEmbedded"), + inBuf: `{}`, + inVal: addr(structUnexportedEmbedded{}), + want: addr(structUnexportedEmbedded{}), + wantErr: EU(errors.New("embedded Go struct field namedString of non-struct type must be explicitly given a JSON name")).withType('{', T[structUnexportedEmbedded]()), + }, { + name: jsontest.Name("Structs/UnexportedEmbeddedStruct"), + inBuf: `{"Bool":true,"FizzBuzz":5,"Addr":"192.168.0.1"}`, + inVal: addr(structUnexportedEmbeddedStruct{}), + want: addr(structUnexportedEmbeddedStruct{structOmitZeroAll{Bool: true}, 5, structNestedAddr{netip.AddrFrom4([4]byte{192, 168, 0, 1})}}), + }, { + name: jsontest.Name("Structs/UnexportedEmbeddedStructPointer/Nil"), + inBuf: `{"Bool":true,"FizzBuzz":5}`, + inVal: addr(structUnexportedEmbeddedStructPointer{}), + wantErr: EU(errNilField).withPos(`{"Bool":`, "/Bool").withType(0, T[structUnexportedEmbeddedStructPointer]()), + }, { + name: jsontest.Name("Structs/UnexportedEmbeddedStructPointer/Nil"), + inBuf: `{"FizzBuzz":5,"Addr":"192.168.0.1"}`, + inVal: addr(structUnexportedEmbeddedStructPointer{}), + wantErr: EU(errNilField).withPos(`{"FizzBuzz":5,"Addr":`, "/Addr").withType(0, T[structUnexportedEmbeddedStructPointer]()), + }, { + name: jsontest.Name("Structs/UnexportedEmbeddedStructPointer/Nil"), + inBuf: `{"Bool":true,"FizzBuzz":10,"Addr":"192.168.0.1"}`, + inVal: addr(structUnexportedEmbeddedStructPointer{&structOmitZeroAll{Int: 5}, 5, &structNestedAddr{netip.AddrFrom4([4]byte{127, 0, 0, 1})}}), + want: addr(structUnexportedEmbeddedStructPointer{&structOmitZeroAll{Bool: true, Int: 5}, 10, &structNestedAddr{netip.AddrFrom4([4]byte{192, 168, 0, 1})}}), + }, { + name: jsontest.Name("Structs/Unknown"), + inBuf: `{ + "object0": {}, + "object1": {"key1": "value"}, + "object2": {"key1": "value", "key2": "value"}, + "objects": {"":{"":{"":{}}}}, + "array0": [], + "array1": ["value1"], + "array2": ["value1", "value2"], + "array": [[[]]], + "scalars": [null, false, true, "string", 12.345] +}`, + inVal: addr(struct{}{}), + want: addr(struct{}{}), + }, { + name: jsontest.Name("Structs/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + inBuf: `{"Field":"Value"}`, + inVal: addr(struct{ Field string }{}), + want: addr(struct{ Field string }{"Value"}), + }, { + name: jsontest.Name("Slices/Null"), + inBuf: `null`, + inVal: addr([]string{"something"}), + want: addr([]string(nil)), + }, { + name: jsontest.Name("Slices/Bool"), + inBuf: `[true,false]`, + inVal: new([]bool), + want: addr([]bool{true, false}), + }, { + name: jsontest.Name("Slices/String"), + inBuf: `["hello","goodbye"]`, + inVal: new([]string), + want: addr([]string{"hello", "goodbye"}), + }, { + name: jsontest.Name("Slices/Bytes"), + inBuf: `["aGVsbG8=","Z29vZGJ5ZQ=="]`, + inVal: new([][]byte), + want: addr([][]byte{[]byte("hello"), []byte("goodbye")}), + }, { + name: jsontest.Name("Slices/Int"), + inBuf: `[-2,-1,0,1,2]`, + inVal: new([]int), + want: addr([]int{-2, -1, 0, 1, 2}), + }, { + name: jsontest.Name("Slices/Uint"), + inBuf: `[0,1,2,3,4]`, + inVal: new([]uint), + want: addr([]uint{0, 1, 2, 3, 4}), + }, { + name: jsontest.Name("Slices/Float"), + inBuf: `[3.14159,12.34]`, + inVal: new([]float64), + want: addr([]float64{3.14159, 12.34}), + }, { + // NOTE: The semantics differs from v1, where the slice length is reset + // and new elements are appended to the end. + // See https://go.dev/issue/21092. + name: jsontest.Name("Slices/Merge"), + inBuf: `[{"k3":"v3"},{"k4":"v4"}]`, + inVal: addr([]map[string]string{{"k1": "v1"}, {"k2": "v2"}}[:1]), + want: addr([]map[string]string{{"k3": "v3"}, {"k4": "v4"}}), + }, { + name: jsontest.Name("Slices/Invalid/Channel"), + inBuf: `["hello"]`, + inVal: new([]chan string), + want: addr([]chan string{nil}), + wantErr: EU(nil).withPos(`[`, "/0").withType(0, T[chan string]()), + }, { + name: jsontest.Name("Slices/RecursiveSlice"), + inBuf: `[[],[],[[]],[[],[]]]`, + inVal: new(recursiveSlice), + want: addr(recursiveSlice{ + {}, + {}, + {{}}, + {{}, {}}, + }), + }, { + name: jsontest.Name("Slices/Invalid/Bool"), + inBuf: `true`, + inVal: addr([]string{"nochange"}), + want: addr([]string{"nochange"}), + wantErr: EU(nil).withType('t', T[[]string]()), + }, { + name: jsontest.Name("Slices/Invalid/String"), + inBuf: `""`, + inVal: addr([]string{"nochange"}), + want: addr([]string{"nochange"}), + wantErr: EU(nil).withType('"', T[[]string]()), + }, { + name: jsontest.Name("Slices/Invalid/Number"), + inBuf: `0`, + inVal: addr([]string{"nochange"}), + want: addr([]string{"nochange"}), + wantErr: EU(nil).withType('0', T[[]string]()), + }, { + name: jsontest.Name("Slices/Invalid/Object"), + inBuf: `{}`, + inVal: addr([]string{"nochange"}), + want: addr([]string{"nochange"}), + wantErr: EU(nil).withType('{', T[[]string]()), + }, { + name: jsontest.Name("Slices/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + inBuf: `[false,true]`, + inVal: addr([]bool{true, false}), + want: addr([]bool{false, true}), + }, { + name: jsontest.Name("Arrays/Null"), + inBuf: `null`, + inVal: addr([1]string{"something"}), + want: addr([1]string{}), + }, { + name: jsontest.Name("Arrays/Bool"), + inBuf: `[true,false]`, + inVal: new([2]bool), + want: addr([2]bool{true, false}), + }, { + name: jsontest.Name("Arrays/String"), + inBuf: `["hello","goodbye"]`, + inVal: new([2]string), + want: addr([2]string{"hello", "goodbye"}), + }, { + name: jsontest.Name("Arrays/Bytes"), + inBuf: `["aGVsbG8=","Z29vZGJ5ZQ=="]`, + inVal: new([2][]byte), + want: addr([2][]byte{[]byte("hello"), []byte("goodbye")}), + }, { + name: jsontest.Name("Arrays/Int"), + inBuf: `[-2,-1,0,1,2]`, + inVal: new([5]int), + want: addr([5]int{-2, -1, 0, 1, 2}), + }, { + name: jsontest.Name("Arrays/Uint"), + inBuf: `[0,1,2,3,4]`, + inVal: new([5]uint), + want: addr([5]uint{0, 1, 2, 3, 4}), + }, { + name: jsontest.Name("Arrays/Float"), + inBuf: `[3.14159,12.34]`, + inVal: new([2]float64), + want: addr([2]float64{3.14159, 12.34}), + }, { + // NOTE: The semantics differs from v1, where elements are not merged. + // This is to maintain consistent merge semantics with slices. + name: jsontest.Name("Arrays/Merge"), + inBuf: `[{"k3":"v3"},{"k4":"v4"}]`, + inVal: addr([2]map[string]string{{"k1": "v1"}, {"k2": "v2"}}), + want: addr([2]map[string]string{{"k3": "v3"}, {"k4": "v4"}}), + }, { + name: jsontest.Name("Arrays/Invalid/Channel"), + inBuf: `["hello"]`, + inVal: new([1]chan string), + want: new([1]chan string), + wantErr: EU(nil).withPos(`[`, "/0").withType(0, T[chan string]()), + }, { + name: jsontest.Name("Arrays/Invalid/Underflow"), + inBuf: `{"F":[ ]}`, + inVal: new(struct{ F [1]string }), + want: addr(struct{ F [1]string }{}), + wantErr: EU(errArrayUnderflow).withPos(`{"F":[ `, "/F").withType(']', T[[1]string]()), + }, { + name: jsontest.Name("Arrays/Invalid/Underflow/UnmarshalArrayFromAnyLength"), + opts: []Options{jsonflags.UnmarshalArrayFromAnyLength | 1}, + inBuf: `[-1,-2]`, + inVal: addr([4]int{1, 2, 3, 4}), + want: addr([4]int{-1, -2, 0, 0}), + }, { + name: jsontest.Name("Arrays/Invalid/Overflow"), + inBuf: `["1","2"]`, + inVal: new([1]string), + want: addr([1]string{"1"}), + wantErr: EU(errArrayOverflow).withPos(`["1","2"`, "").withType(']', T[[1]string]()), + }, { + name: jsontest.Name("Arrays/Invalid/Overflow/UnmarshalArrayFromAnyLength"), + opts: []Options{jsonflags.UnmarshalArrayFromAnyLength | 1}, + inBuf: `[-1,-2,-3,-4,-5,-6]`, + inVal: addr([4]int{1, 2, 3, 4}), + want: addr([4]int{-1, -2, -3, -4}), + }, { + name: jsontest.Name("Arrays/Invalid/Bool"), + inBuf: `true`, + inVal: addr([1]string{"nochange"}), + want: addr([1]string{"nochange"}), + wantErr: EU(nil).withType('t', T[[1]string]()), + }, { + name: jsontest.Name("Arrays/Invalid/String"), + inBuf: `""`, + inVal: addr([1]string{"nochange"}), + want: addr([1]string{"nochange"}), + wantErr: EU(nil).withType('"', T[[1]string]()), + }, { + name: jsontest.Name("Arrays/Invalid/Number"), + inBuf: `0`, + inVal: addr([1]string{"nochange"}), + want: addr([1]string{"nochange"}), + wantErr: EU(nil).withType('0', T[[1]string]()), + }, { + name: jsontest.Name("Arrays/Invalid/Object"), + inBuf: `{}`, + inVal: addr([1]string{"nochange"}), + want: addr([1]string{"nochange"}), + wantErr: EU(nil).withType('{', T[[1]string]()), + }, { + name: jsontest.Name("Arrays/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + inBuf: `[false,true]`, + inVal: addr([2]bool{true, false}), + want: addr([2]bool{false, true}), + }, { + name: jsontest.Name("Pointers/NullL0"), + inBuf: `null`, + inVal: new(*string), + want: addr((*string)(nil)), + }, { + name: jsontest.Name("Pointers/NullL1"), + inBuf: `null`, + inVal: addr(new(*string)), + want: addr((**string)(nil)), + }, { + name: jsontest.Name("Pointers/Bool"), + inBuf: `true`, + inVal: addr(new(bool)), + want: addr(addr(true)), + }, { + name: jsontest.Name("Pointers/String"), + inBuf: `"hello"`, + inVal: addr(new(string)), + want: addr(addr("hello")), + }, { + name: jsontest.Name("Pointers/Bytes"), + inBuf: `"aGVsbG8="`, + inVal: addr(new([]byte)), + want: addr(addr([]byte("hello"))), + }, { + name: jsontest.Name("Pointers/Int"), + inBuf: `-123`, + inVal: addr(new(int)), + want: addr(addr(int(-123))), + }, { + name: jsontest.Name("Pointers/Uint"), + inBuf: `123`, + inVal: addr(new(int)), + want: addr(addr(int(123))), + }, { + name: jsontest.Name("Pointers/Float"), + inBuf: `123.456`, + inVal: addr(new(float64)), + want: addr(addr(float64(123.456))), + }, { + name: jsontest.Name("Pointers/Allocate"), + inBuf: `"hello"`, + inVal: addr((*string)(nil)), + want: addr(addr("hello")), + }, { + name: jsontest.Name("Points/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + inBuf: `true`, + inVal: addr(new(bool)), + want: addr(addr(true)), + }, { + name: jsontest.Name("Interfaces/Empty/Null"), + inBuf: `null`, + inVal: new(any), + want: new(any), + }, { + name: jsontest.Name("Interfaces/NonEmpty/Null"), + inBuf: `null`, + inVal: new(io.Reader), + want: new(io.Reader), + }, { + name: jsontest.Name("Interfaces/NonEmpty/Invalid"), + inBuf: `"hello"`, + inVal: new(io.Reader), + want: new(io.Reader), + wantErr: EU(internal.ErrNilInterface).withType(0, T[io.Reader]()), + }, { + name: jsontest.Name("Interfaces/Empty/False"), + inBuf: `false`, + inVal: new(any), + want: func() any { + var vi any = false + return &vi + }(), + }, { + name: jsontest.Name("Interfaces/Empty/True"), + inBuf: `true`, + inVal: new(any), + want: func() any { + var vi any = true + return &vi + }(), + }, { + name: jsontest.Name("Interfaces/Empty/String"), + inBuf: `"string"`, + inVal: new(any), + want: func() any { + var vi any = "string" + return &vi + }(), + }, { + name: jsontest.Name("Interfaces/Empty/Number"), + inBuf: `3.14159`, + inVal: new(any), + want: func() any { + var vi any = 3.14159 + return &vi + }(), + }, { + name: jsontest.Name("Interfaces/Empty/Object"), + inBuf: `{"k":"v"}`, + inVal: new(any), + want: func() any { + var vi any = map[string]any{"k": "v"} + return &vi + }(), + }, { + name: jsontest.Name("Interfaces/Empty/Array"), + inBuf: `["v"]`, + inVal: new(any), + want: func() any { + var vi any = []any{"v"} + return &vi + }(), + }, { + name: jsontest.Name("Interfaces/NamedAny/String"), + inBuf: `"string"`, + inVal: new(namedAny), + want: func() namedAny { + var vi namedAny = "string" + return &vi + }(), + }, { + name: jsontest.Name("Interfaces/Invalid"), + inBuf: `]`, + inVal: new(any), + want: new(any), + wantErr: newInvalidCharacterError("]", "at start of value", 0, ""), + }, { + // NOTE: The semantics differs from v1, + // where existing map entries were not merged into. + // See https://go.dev/issue/26946. + // See https://go.dev/issue/33993. + name: jsontest.Name("Interfaces/Merge/Map"), + inBuf: `{"k2":"v2"}`, + inVal: func() any { + var vi any = map[string]string{"k1": "v1"} + return &vi + }(), + want: func() any { + var vi any = map[string]string{"k1": "v1", "k2": "v2"} + return &vi + }(), + }, { + name: jsontest.Name("Interfaces/Merge/Struct"), + inBuf: `{"Array":["goodbye"]}`, + inVal: func() any { + var vi any = structAll{String: "hello"} + return &vi + }(), + want: func() any { + var vi any = structAll{String: "hello", Array: [1]string{"goodbye"}} + return &vi + }(), + }, { + name: jsontest.Name("Interfaces/Merge/NamedInt"), + inBuf: `64`, + inVal: func() any { + var vi any = namedInt64(-64) + return &vi + }(), + want: func() any { + var vi any = namedInt64(+64) + return &vi + }(), + }, { + name: jsontest.Name("Interfaces/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + inBuf: `true`, + inVal: new(any), + want: func() any { + var vi any = true + return &vi + }(), + }, { + name: jsontest.Name("Interfaces/Any"), + inBuf: `{"X":[null,false,true,"",0,{},[]]}`, + inVal: new(struct{ X any }), + want: addr(struct{ X any }{[]any{nil, false, true, "", 0.0, map[string]any{}, []any{}}}), + }, { + name: jsontest.Name("Interfaces/Any/Named"), + inBuf: `{"X":[null,false,true,"",0,{},[]]}`, + inVal: new(struct{ X namedAny }), + want: addr(struct{ X namedAny }{[]any{nil, false, true, "", 0.0, map[string]any{}, []any{}}}), + }, { + name: jsontest.Name("Interfaces/Any/Stringified"), + opts: []Options{StringifyNumbers(true)}, + inBuf: `{"X":"0"}`, + inVal: new(struct{ X any }), + want: addr(struct{ X any }{"0"}), + }, { + name: jsontest.Name("Interfaces/Any/UnmarshalFunc/Any"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *any) error { + *v = "called" + return nil + })), + }, + inBuf: `{"X":[null,false,true,"",0,{},[]]}`, + inVal: new(struct{ X any }), + want: addr(struct{ X any }{"called"}), + }, { + name: jsontest.Name("Interfaces/Any/UnmarshalFunc/Bool"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *bool) error { + *v = string(b) != "true" + return nil + })), + }, + inBuf: `{"X":[null,false,true,"",0,{},[]]}`, + inVal: new(struct{ X any }), + want: addr(struct{ X any }{[]any{nil, true, false, "", 0.0, map[string]any{}, []any{}}}), + }, { + name: jsontest.Name("Interfaces/Any/UnmarshalFunc/String"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *string) error { + *v = "called" + return nil + })), + }, + inBuf: `{"X":[null,false,true,"",0,{},[]]}`, + inVal: new(struct{ X any }), + want: addr(struct{ X any }{[]any{nil, false, true, "called", 0.0, map[string]any{}, []any{}}}), + }, { + name: jsontest.Name("Interfaces/Any/UnmarshalFunc/Float64"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *float64) error { + *v = 3.14159 + return nil + })), + }, + inBuf: `{"X":[null,false,true,"",0,{},[]]}`, + inVal: new(struct{ X any }), + want: addr(struct{ X any }{[]any{nil, false, true, "", 3.14159, map[string]any{}, []any{}}}), + }, { + name: jsontest.Name("Interfaces/Any/UnmarshalFunc/MapStringAny"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *map[string]any) error { + *v = map[string]any{"called": nil} + return nil + })), + }, + inBuf: `{"X":[null,false,true,"",0,{},[]]}`, + inVal: new(struct{ X any }), + want: addr(struct{ X any }{[]any{nil, false, true, "", 0.0, map[string]any{"called": nil}, []any{}}}), + }, { + name: jsontest.Name("Interfaces/Any/UnmarshalFunc/SliceAny"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *[]any) error { + *v = []any{"called"} + return nil + })), + }, + inBuf: `{"X":[null,false,true,"",0,{},[]]}`, + inVal: new(struct{ X any }), + want: addr(struct{ X any }{[]any{"called"}}), + }, { + name: jsontest.Name("Interfaces/Any/Maps/NonEmpty"), + inBuf: `{"X":{"fizz":"buzz"}}`, + inVal: new(struct{ X any }), + want: addr(struct{ X any }{map[string]any{"fizz": "buzz"}}), + }, { + name: jsontest.Name("Interfaces/Any/Maps/RejectDuplicateNames"), + inBuf: `{"X":{"fizz":"buzz","fizz":true}}`, + inVal: new(struct{ X any }), + want: addr(struct{ X any }{map[string]any{"fizz": "buzz"}}), + wantErr: newDuplicateNameError("/X", []byte(`"fizz"`), len64(`{"X":{"fizz":"buzz",`)), + }, { + name: jsontest.Name("Interfaces/Any/Maps/AllowDuplicateNames"), + opts: []Options{jsontext.AllowDuplicateNames(true)}, + inBuf: `{"X":{"fizz":"buzz","fizz":true}}`, + inVal: new(struct{ X any }), + want: addr(struct{ X any }{map[string]any{"fizz": "buzz"}}), + wantErr: EU(nil).withPos(`{"X":{"fizz":"buzz","fizz":`, "/X/fizz").withType('t', T[string]()), + }, { + name: jsontest.Name("Interfaces/Any/Slices/NonEmpty"), + inBuf: `{"X":["fizz","buzz"]}`, + inVal: new(struct{ X any }), + want: addr(struct{ X any }{[]any{"fizz", "buzz"}}), + }, { + name: jsontest.Name("Methods/NilPointer/Null"), + inBuf: `{"X":null}`, + inVal: addr(struct{ X *allMethods }{X: (*allMethods)(nil)}), + want: addr(struct{ X *allMethods }{X: (*allMethods)(nil)}), // method should not be called + }, { + name: jsontest.Name("Methods/NilPointer/Value"), + inBuf: `{"X":"value"}`, + inVal: addr(struct{ X *allMethods }{X: (*allMethods)(nil)}), + want: addr(struct{ X *allMethods }{X: &allMethods{method: "UnmarshalJSONFrom", value: []byte(`"value"`)}}), + }, { + name: jsontest.Name("Methods/NilInterface/Null"), + inBuf: `{"X":null}`, + inVal: addr(struct{ X MarshalerTo }{X: (*allMethods)(nil)}), + want: addr(struct{ X MarshalerTo }{X: nil}), // interface value itself is nil'd out + }, { + name: jsontest.Name("Methods/NilInterface/Value"), + inBuf: `{"X":"value"}`, + inVal: addr(struct{ X MarshalerTo }{X: (*allMethods)(nil)}), + want: addr(struct{ X MarshalerTo }{X: &allMethods{method: "UnmarshalJSONFrom", value: []byte(`"value"`)}}), + }, { + name: jsontest.Name("Methods/AllMethods"), + inBuf: `{"X":"hello"}`, + inVal: new(struct{ X *allMethods }), + want: addr(struct{ X *allMethods }{X: &allMethods{method: "UnmarshalJSONFrom", value: []byte(`"hello"`)}}), + }, { + name: jsontest.Name("Methods/AllMethodsExceptJSONv2"), + inBuf: `{"X":"hello"}`, + inVal: new(struct{ X *allMethodsExceptJSONv2 }), + want: addr(struct{ X *allMethodsExceptJSONv2 }{X: &allMethodsExceptJSONv2{allMethods: allMethods{method: "UnmarshalJSON", value: []byte(`"hello"`)}}}), + }, { + name: jsontest.Name("Methods/AllMethodsExceptJSONv1"), + inBuf: `{"X":"hello"}`, + inVal: new(struct{ X *allMethodsExceptJSONv1 }), + want: addr(struct{ X *allMethodsExceptJSONv1 }{X: &allMethodsExceptJSONv1{allMethods: allMethods{method: "UnmarshalJSONFrom", value: []byte(`"hello"`)}}}), + }, { + name: jsontest.Name("Methods/AllMethodsExceptText"), + inBuf: `{"X":"hello"}`, + inVal: new(struct{ X *allMethodsExceptText }), + want: addr(struct{ X *allMethodsExceptText }{X: &allMethodsExceptText{allMethods: allMethods{method: "UnmarshalJSONFrom", value: []byte(`"hello"`)}}}), + }, { + name: jsontest.Name("Methods/OnlyMethodJSONv2"), + inBuf: `{"X":"hello"}`, + inVal: new(struct{ X *onlyMethodJSONv2 }), + want: addr(struct{ X *onlyMethodJSONv2 }{X: &onlyMethodJSONv2{allMethods: allMethods{method: "UnmarshalJSONFrom", value: []byte(`"hello"`)}}}), + }, { + name: jsontest.Name("Methods/OnlyMethodJSONv1"), + inBuf: `{"X":"hello"}`, + inVal: new(struct{ X *onlyMethodJSONv1 }), + want: addr(struct{ X *onlyMethodJSONv1 }{X: &onlyMethodJSONv1{allMethods: allMethods{method: "UnmarshalJSON", value: []byte(`"hello"`)}}}), + }, { + name: jsontest.Name("Methods/OnlyMethodText"), + inBuf: `{"X":"hello"}`, + inVal: new(struct{ X *onlyMethodText }), + want: addr(struct{ X *onlyMethodText }{X: &onlyMethodText{allMethods: allMethods{method: "UnmarshalText", value: []byte(`hello`)}}}), + }, { + name: jsontest.Name("Methods/Text/Null"), + inBuf: `{"X":null}`, + inVal: addr(struct{ X unmarshalTextFunc }{unmarshalTextFunc(func(b []byte) error { + return errMustNotCall + })}), + want: addr(struct{ X unmarshalTextFunc }{nil}), + }, { + name: jsontest.Name("Methods/IP"), + inBuf: `"192.168.0.100"`, + inVal: new(net.IP), + want: addr(net.IPv4(192, 168, 0, 100)), + }, { + // NOTE: Fixes https://go.dev/issue/46516. + name: jsontest.Name("Methods/Anonymous"), + inBuf: `{"X":"hello"}`, + inVal: new(struct{ X struct{ allMethods } }), + want: addr(struct{ X struct{ allMethods } }{X: struct{ allMethods }{allMethods{method: "UnmarshalJSONFrom", value: []byte(`"hello"`)}}}), + }, { + // NOTE: Fixes https://go.dev/issue/22967. + name: jsontest.Name("Methods/Addressable"), + inBuf: `{"V":"hello","M":{"K":"hello"},"I":"hello"}`, + inVal: addr(struct { + V allMethods + M map[string]allMethods + I any + }{ + I: allMethods{}, // need to initialize with concrete value + }), + want: addr(struct { + V allMethods + M map[string]allMethods + I any + }{ + V: allMethods{method: "UnmarshalJSONFrom", value: []byte(`"hello"`)}, + M: map[string]allMethods{"K": {method: "UnmarshalJSONFrom", value: []byte(`"hello"`)}}, + I: allMethods{method: "UnmarshalJSONFrom", value: []byte(`"hello"`)}, + }), + }, { + // NOTE: Fixes https://go.dev/issue/29732. + name: jsontest.Name("Methods/MapKey/JSONv2"), + inBuf: `{"k1":"v1b","k2":"v2"}`, + inVal: addr(map[structMethodJSONv2]string{{"k1"}: "v1a", {"k3"}: "v3"}), + want: addr(map[structMethodJSONv2]string{{"k1"}: "v1b", {"k2"}: "v2", {"k3"}: "v3"}), + }, { + // NOTE: Fixes https://go.dev/issue/29732. + name: jsontest.Name("Methods/MapKey/JSONv1"), + inBuf: `{"k1":"v1b","k2":"v2"}`, + inVal: addr(map[structMethodJSONv1]string{{"k1"}: "v1a", {"k3"}: "v3"}), + want: addr(map[structMethodJSONv1]string{{"k1"}: "v1b", {"k2"}: "v2", {"k3"}: "v3"}), + }, { + name: jsontest.Name("Methods/MapKey/Text"), + inBuf: `{"k1":"v1b","k2":"v2"}`, + inVal: addr(map[structMethodText]string{{"k1"}: "v1a", {"k3"}: "v3"}), + want: addr(map[structMethodText]string{{"k1"}: "v1b", {"k2"}: "v2", {"k3"}: "v3"}), + }, { + name: jsontest.Name("Methods/Invalid/JSONv2/Error"), + inBuf: `{}`, + inVal: addr(unmarshalJSONv2Func(func(*jsontext.Decoder) error { + return errSomeError + })), + wantErr: EU(errSomeError).withType(0, T[unmarshalJSONv2Func]()), + }, { + name: jsontest.Name("Methods/Invalid/JSONv2/TooFew"), + inBuf: `{}`, + inVal: addr(unmarshalJSONv2Func(func(*jsontext.Decoder) error { + return nil // do nothing + })), + wantErr: EU(errNonSingularValue).withType(0, T[unmarshalJSONv2Func]()), + }, { + name: jsontest.Name("Methods/Invalid/JSONv2/TooMany"), + inBuf: `{}{}`, + inVal: addr(unmarshalJSONv2Func(func(dec *jsontext.Decoder) error { + dec.ReadValue() + dec.ReadValue() + return nil + })), + wantErr: EU(errNonSingularValue).withPos(`{}`, "").withType(0, T[unmarshalJSONv2Func]()), + }, { + name: jsontest.Name("Methods/Invalid/JSONv2/SkipFunc"), + inBuf: `{}`, + inVal: addr(unmarshalJSONv2Func(func(*jsontext.Decoder) error { + return SkipFunc + })), + wantErr: EU(wrapSkipFunc(SkipFunc, "unmarshal method")).withType(0, T[unmarshalJSONv2Func]()), + }, { + name: jsontest.Name("Methods/Invalid/JSONv1/Error"), + inBuf: `{}`, + inVal: addr(unmarshalJSONv1Func(func([]byte) error { + return errSomeError + })), + wantErr: EU(errSomeError).withType('{', T[unmarshalJSONv1Func]()), + }, { + name: jsontest.Name("Methods/Invalid/JSONv1/SkipFunc"), + inBuf: `{}`, + inVal: addr(unmarshalJSONv1Func(func([]byte) error { + return SkipFunc + })), + wantErr: EU(wrapSkipFunc(SkipFunc, "unmarshal method")).withType('{', T[unmarshalJSONv1Func]()), + }, { + name: jsontest.Name("Methods/Invalid/Text/Error"), + inBuf: `"value"`, + inVal: addr(unmarshalTextFunc(func([]byte) error { + return errSomeError + })), + wantErr: EU(errSomeError).withType('"', T[unmarshalTextFunc]()), + }, { + name: jsontest.Name("Methods/Invalid/Text/Syntax"), + inBuf: `{}`, + inVal: addr(unmarshalTextFunc(func([]byte) error { + panic("should not be called") + })), + wantErr: EU(errNonStringValue).withType('{', T[unmarshalTextFunc]()), + }, { + name: jsontest.Name("Methods/Invalid/Text/SkipFunc"), + inBuf: `"value"`, + inVal: addr(unmarshalTextFunc(func([]byte) error { + return SkipFunc + })), + wantErr: EU(wrapSkipFunc(SkipFunc, "unmarshal method")).withType('"', T[unmarshalTextFunc]()), + }, { + name: jsontest.Name("Functions/String/V1"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *string) error { + if string(b) != `""` { + return fmt.Errorf("got %s, want %s", b, `""`) + } + *v = "called" + return nil + })), + }, + inBuf: `""`, + inVal: addr(""), + want: addr("called"), + }, { + name: jsontest.Name("Functions/String/Empty"), + opts: []Options{WithUnmarshalers(nil)}, + inBuf: `"hello"`, + inVal: addr(""), + want: addr("hello"), + }, { + name: jsontest.Name("Functions/NamedString/V1/NoMatch"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *namedString) error { + panic("should not be called") + })), + }, + inBuf: `""`, + inVal: addr(""), + want: addr(""), + }, { + name: jsontest.Name("Functions/NamedString/V1/Match"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *namedString) error { + if string(b) != `""` { + return fmt.Errorf("got %s, want %s", b, `""`) + } + *v = "called" + return nil + })), + }, + inBuf: `""`, + inVal: addr(namedString("")), + want: addr(namedString("called")), + }, { + name: jsontest.Name("Functions/String/V2"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *string) error { + switch b, err := dec.ReadValue(); { + case err != nil: + return err + case string(b) != `""`: + return fmt.Errorf("got %s, want %s", b, `""`) + } + *v = "called" + return nil + })), + }, + inBuf: `""`, + inVal: addr(""), + want: addr("called"), + }, { + name: jsontest.Name("Functions/NamedString/V2/NoMatch"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *namedString) error { + panic("should not be called") + })), + }, + inBuf: `""`, + inVal: addr(""), + want: addr(""), + }, { + name: jsontest.Name("Functions/NamedString/V2/Match"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *namedString) error { + switch t, err := dec.ReadToken(); { + case err != nil: + return err + case t.String() != ``: + return fmt.Errorf("got %q, want %q", t, ``) + } + *v = "called" + return nil + })), + }, + inBuf: `""`, + inVal: addr(namedString("")), + want: addr(namedString("called")), + }, { + name: jsontest.Name("Functions/String/Empty1/NoMatch"), + opts: []Options{ + WithUnmarshalers(new(Unmarshalers)), + }, + inBuf: `""`, + inVal: addr(""), + want: addr(""), + }, { + name: jsontest.Name("Functions/String/Empty2/NoMatch"), + opts: []Options{ + WithUnmarshalers(JoinUnmarshalers()), + }, + inBuf: `""`, + inVal: addr(""), + want: addr(""), + }, { + name: jsontest.Name("Functions/String/V1/DirectError"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func([]byte, *string) error { + return errSomeError + })), + }, + inBuf: `""`, + inVal: addr(""), + want: addr(""), + wantErr: EU(errSomeError).withType('"', reflect.PointerTo(stringType)), + }, { + name: jsontest.Name("Functions/String/V1/SkipError"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func([]byte, *string) error { + return SkipFunc + })), + }, + inBuf: `""`, + inVal: addr(""), + want: addr(""), + wantErr: EU(wrapSkipFunc(SkipFunc, "unmarshal function of type func([]byte, T) error")).withType('"', reflect.PointerTo(stringType)), + }, { + name: jsontest.Name("Functions/String/V2/DirectError"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *string) error { + return errSomeError + })), + }, + inBuf: `""`, + inVal: addr(""), + want: addr(""), + wantErr: EU(errSomeError).withType(0, reflect.PointerTo(stringType)), + }, { + name: jsontest.Name("Functions/String/V2/TooFew"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *string) error { + return nil + })), + }, + inBuf: `""`, + inVal: addr(""), + want: addr(""), + wantErr: EU(errNonSingularValue).withType(0, reflect.PointerTo(stringType)), + }, { + name: jsontest.Name("Functions/String/V2/TooMany"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *string) error { + if _, err := dec.ReadValue(); err != nil { + return err + } + if _, err := dec.ReadValue(); err != nil { + return err + } + return nil + })), + }, + inBuf: `{"X":["",""]}`, + inVal: addr(struct{ X []string }{}), + want: addr(struct{ X []string }{[]string{""}}), + wantErr: EU(errNonSingularValue).withPos(`{"X":["",`, "/X").withType(0, reflect.PointerTo(stringType)), + }, { + name: jsontest.Name("Functions/String/V2/Skipped"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *string) error { + return SkipFunc + })), + }, + inBuf: `""`, + inVal: addr(""), + want: addr(""), + }, { + name: jsontest.Name("Functions/String/V2/ProcessBeforeSkip"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *string) error { + if _, err := dec.ReadValue(); err != nil { + return err + } + return SkipFunc + })), + }, + inBuf: `""`, + inVal: addr(""), + want: addr(""), + wantErr: EU(errSkipMutation).withType(0, reflect.PointerTo(stringType)), + }, { + name: jsontest.Name("Functions/String/V2/WrappedSkipError"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *string) error { + return fmt.Errorf("wrap: %w", SkipFunc) + })), + }, + inBuf: `""`, + inVal: addr(""), + want: addr(""), + wantErr: EU(fmt.Errorf("wrap: %w", SkipFunc)).withType(0, reflect.PointerTo(stringType)), + }, { + name: jsontest.Name("Functions/Map/Key/NoCaseString/V1"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *nocaseString) error { + if string(b) != `"hello"` { + return fmt.Errorf("got %s, want %s", b, `"hello"`) + } + *v = "called" + return nil + })), + }, + inBuf: `{"hello":"world"}`, + inVal: addr(map[nocaseString]string{}), + want: addr(map[nocaseString]string{"called": "world"}), + }, { + name: jsontest.Name("Functions/Map/Key/TextMarshaler/V1"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v encoding.TextMarshaler) error { + if string(b) != `"hello"` { + return fmt.Errorf("got %s, want %s", b, `"hello"`) + } + *v.(*nocaseString) = "called" + return nil + })), + }, + inBuf: `{"hello":"world"}`, + inVal: addr(map[nocaseString]string{}), + want: addr(map[nocaseString]string{"called": "world"}), + }, { + name: jsontest.Name("Functions/Map/Key/NoCaseString/V2"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *nocaseString) error { + switch t, err := dec.ReadToken(); { + case err != nil: + return err + case t.String() != "hello": + return fmt.Errorf("got %q, want %q", t, "hello") + } + *v = "called" + return nil + })), + }, + inBuf: `{"hello":"world"}`, + inVal: addr(map[nocaseString]string{}), + want: addr(map[nocaseString]string{"called": "world"}), + }, { + name: jsontest.Name("Functions/Map/Key/TextMarshaler/V2"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v encoding.TextMarshaler) error { + switch b, err := dec.ReadValue(); { + case err != nil: + return err + case string(b) != `"hello"`: + return fmt.Errorf("got %s, want %s", b, `"hello"`) + } + *v.(*nocaseString) = "called" + return nil + })), + }, + inBuf: `{"hello":"world"}`, + inVal: addr(map[nocaseString]string{}), + want: addr(map[nocaseString]string{"called": "world"}), + }, { + name: jsontest.Name("Functions/Map/Key/String/V1/DuplicateName"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *string) error { + if _, err := dec.ReadValue(); err != nil { + return err + } + xd := export.Decoder(dec) + *v = fmt.Sprintf("%d-%d", len(xd.Tokens.Stack), xd.Tokens.Last.Length()) + return nil + })), + }, + inBuf: `{"name":"value","name":"value"}`, + inVal: addr(map[string]string{}), + want: addr(map[string]string{"1-1": "1-2"}), + wantErr: newDuplicateNameError("", []byte(`"name"`), len64(`{"name":"value",`)), + }, { + name: jsontest.Name("Functions/Map/Value/NoCaseString/V1"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *nocaseString) error { + if string(b) != `"world"` { + return fmt.Errorf("got %s, want %s", b, `"world"`) + } + *v = "called" + return nil + })), + }, + inBuf: `{"hello":"world"}`, + inVal: addr(map[string]nocaseString{}), + want: addr(map[string]nocaseString{"hello": "called"}), + }, { + name: jsontest.Name("Functions/Map/Value/TextMarshaler/V1"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v encoding.TextMarshaler) error { + if string(b) != `"world"` { + return fmt.Errorf("got %s, want %s", b, `"world"`) + } + *v.(*nocaseString) = "called" + return nil + })), + }, + inBuf: `{"hello":"world"}`, + inVal: addr(map[string]nocaseString{}), + want: addr(map[string]nocaseString{"hello": "called"}), + }, { + name: jsontest.Name("Functions/Map/Value/NoCaseString/V2"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *nocaseString) error { + switch t, err := dec.ReadToken(); { + case err != nil: + return err + case t.String() != "world": + return fmt.Errorf("got %q, want %q", t, "world") + } + *v = "called" + return nil + })), + }, + inBuf: `{"hello":"world"}`, + inVal: addr(map[string]nocaseString{}), + want: addr(map[string]nocaseString{"hello": "called"}), + }, { + name: jsontest.Name("Functions/Map/Value/TextMarshaler/V2"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v encoding.TextMarshaler) error { + switch b, err := dec.ReadValue(); { + case err != nil: + return err + case string(b) != `"world"`: + return fmt.Errorf("got %s, want %s", b, `"world"`) + } + *v.(*nocaseString) = "called" + return nil + })), + }, + inBuf: `{"hello":"world"}`, + inVal: addr(map[string]nocaseString{}), + want: addr(map[string]nocaseString{"hello": "called"}), + }, { + name: jsontest.Name("Funtions/Struct/Fields"), + opts: []Options{ + WithUnmarshalers(JoinUnmarshalers( + UnmarshalFunc(func(b []byte, v *bool) error { + if string(b) != `"called1"` { + return fmt.Errorf("got %s, want %s", b, `"called1"`) + } + *v = true + return nil + }), + UnmarshalFunc(func(b []byte, v *string) error { + if string(b) != `"called2"` { + return fmt.Errorf("got %s, want %s", b, `"called2"`) + } + *v = "called2" + return nil + }), + UnmarshalFromFunc(func(dec *jsontext.Decoder, v *[]byte) error { + switch t, err := dec.ReadToken(); { + case err != nil: + return err + case t.String() != "called3": + return fmt.Errorf("got %q, want %q", t, "called3") + } + *v = []byte("called3") + return nil + }), + UnmarshalFromFunc(func(dec *jsontext.Decoder, v *int64) error { + switch b, err := dec.ReadValue(); { + case err != nil: + return err + case string(b) != `"called4"`: + return fmt.Errorf("got %s, want %s", b, `"called4"`) + } + *v = 123 + return nil + }), + )), + }, + inBuf: `{"Bool":"called1","String":"called2","Bytes":"called3","Int":"called4","Uint":456,"Float":789}`, + inVal: addr(structScalars{}), + want: addr(structScalars{Bool: true, String: "called2", Bytes: []byte("called3"), Int: 123, Uint: 456, Float: 789}), + }, { + name: jsontest.Name("Functions/Struct/Inlined"), + opts: []Options{ + WithUnmarshalers(JoinUnmarshalers( + UnmarshalFunc(func([]byte, *structInlinedL1) error { + panic("should not be called") + }), + UnmarshalFromFunc(func(dec *jsontext.Decoder, v *StructEmbed2) error { + panic("should not be called") + }), + )), + }, + inBuf: `{"E":"E3","F":"F3","G":"G3","A":"A1","B":"B1","D":"D2"}`, + inVal: new(structInlined), + want: addr(structInlined{ + X: structInlinedL1{ + X: &structInlinedL2{A: "A1", B: "B1" /* C: "C1" */}, + StructEmbed1: StructEmbed1{ /* C: "C2" */ D: "D2" /* E: "E2" */}, + }, + StructEmbed2: &StructEmbed2{E: "E3", F: "F3", G: "G3"}, + }), + }, { + name: jsontest.Name("Functions/Slice/Elem"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *string) error { + *v = strings.Trim(strings.ToUpper(string(b)), `"`) + return nil + })), + }, + inBuf: `["hello","World"]`, + inVal: addr([]string{}), + want: addr([]string{"HELLO", "WORLD"}), + }, { + name: jsontest.Name("Functions/Array/Elem"), + opts: []Options{ + WithUnmarshalers(UnmarshalFunc(func(b []byte, v *string) error { + *v = strings.Trim(strings.ToUpper(string(b)), `"`) + return nil + })), + }, + inBuf: `["hello","World"]`, + inVal: addr([2]string{}), + want: addr([2]string{"HELLO", "WORLD"}), + }, { + name: jsontest.Name("Functions/Pointer/Nil"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *string) error { + t, err := dec.ReadToken() + *v = strings.ToUpper(t.String()) + return err + })), + }, + inBuf: `{"X":"hello"}`, + inVal: addr(struct{ X *string }{nil}), + want: addr(struct{ X *string }{addr("HELLO")}), + }, { + name: jsontest.Name("Functions/Pointer/NonNil"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *string) error { + t, err := dec.ReadToken() + *v = strings.ToUpper(t.String()) + return err + })), + }, + inBuf: `{"X":"hello"}`, + inVal: addr(struct{ X *string }{addr("")}), + want: addr(struct{ X *string }{addr("HELLO")}), + }, { + name: jsontest.Name("Functions/Interface/Nil"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v fmt.Stringer) error { + panic("should not be called") + })), + }, + inBuf: `{"X":"hello"}`, + inVal: addr(struct{ X fmt.Stringer }{nil}), + want: addr(struct{ X fmt.Stringer }{nil}), + wantErr: EU(internal.ErrNilInterface).withPos(`{"X":`, "/X").withType(0, T[fmt.Stringer]()), + }, { + name: jsontest.Name("Functions/Interface/NetIP"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *fmt.Stringer) error { + *v = net.IP{} + return SkipFunc + })), + }, + inBuf: `{"X":"1.1.1.1"}`, + inVal: addr(struct{ X fmt.Stringer }{nil}), + want: addr(struct{ X fmt.Stringer }{net.IPv4(1, 1, 1, 1)}), + }, { + name: jsontest.Name("Functions/Interface/NewPointerNetIP"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *fmt.Stringer) error { + *v = new(net.IP) + return SkipFunc + })), + }, + inBuf: `{"X":"1.1.1.1"}`, + inVal: addr(struct{ X fmt.Stringer }{nil}), + want: addr(struct{ X fmt.Stringer }{addr(net.IPv4(1, 1, 1, 1))}), + }, { + name: jsontest.Name("Functions/Interface/NilPointerNetIP"), + opts: []Options{ + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, v *fmt.Stringer) error { + *v = (*net.IP)(nil) + return SkipFunc + })), + }, + inBuf: `{"X":"1.1.1.1"}`, + inVal: addr(struct{ X fmt.Stringer }{nil}), + want: addr(struct{ X fmt.Stringer }{addr(net.IPv4(1, 1, 1, 1))}), + }, { + name: jsontest.Name("Functions/Interface/NilPointerNetIP/Override"), + opts: []Options{ + WithUnmarshalers(JoinUnmarshalers( + UnmarshalFromFunc(func(dec *jsontext.Decoder, v *fmt.Stringer) error { + *v = (*net.IP)(nil) + return SkipFunc + }), + UnmarshalFunc(func(b []byte, v *net.IP) error { + b = bytes.ReplaceAll(b, []byte(`1`), []byte(`8`)) + return v.UnmarshalText(bytes.Trim(b, `"`)) + }), + )), + }, + inBuf: `{"X":"1.1.1.1"}`, + inVal: addr(struct{ X fmt.Stringer }{nil}), + want: addr(struct{ X fmt.Stringer }{addr(net.IPv4(8, 8, 8, 8))}), + }, { + name: jsontest.Name("Functions/Interface/Any"), + inBuf: `[null,{},{},{},{},{},{},{},{},{},{},{},{},"LAST"]`, + inVal: addr([...]any{ + nil, // nil + valueStringer{}, // T + (*valueStringer)(nil), // *T + addr(valueStringer{}), // *T + (**valueStringer)(nil), // **T + addr((*valueStringer)(nil)), // **T + addr(addr(valueStringer{})), // **T + pointerStringer{}, // T + (*pointerStringer)(nil), // *T + addr(pointerStringer{}), // *T + (**pointerStringer)(nil), // **T + addr((*pointerStringer)(nil)), // **T + addr(addr(pointerStringer{})), // **T + "LAST", + }), + opts: []Options{ + WithUnmarshalers(func() *Unmarshalers { + type P struct { + D int + N int64 + } + type PV struct { + P P + V any + } + + var lastChecks []func() error + checkLast := func() error { + for _, fn := range lastChecks { + if err := fn(); err != nil { + return err + } + } + return SkipFunc + } + makeValueChecker := func(name string, want []PV) func(d *jsontext.Decoder, v any) error { + checkNext := func(d *jsontext.Decoder, v any) error { + xd := export.Decoder(d) + p := P{len(xd.Tokens.Stack), xd.Tokens.Last.Length()} + rv := reflect.ValueOf(v) + pv := PV{p, v} + switch { + case len(want) == 0: + return fmt.Errorf("%s: %v: got more values than expected", name, p) + case !rv.IsValid() || rv.Kind() != reflect.Pointer || rv.IsNil(): + return fmt.Errorf("%s: %v: got %#v, want non-nil pointer type", name, p, v) + case !reflect.DeepEqual(pv, want[0]): + return fmt.Errorf("%s:\n\tgot %#v\n\twant %#v", name, pv, want[0]) + default: + want = want[1:] + return SkipFunc + } + } + lastChecks = append(lastChecks, func() error { + if len(want) > 0 { + return fmt.Errorf("%s: did not get enough values, want %d more", name, len(want)) + } + return nil + }) + return checkNext + } + makePositionChecker := func(name string, want []P) func(d *jsontext.Decoder, v any) error { + checkNext := func(d *jsontext.Decoder, v any) error { + xd := export.Decoder(d) + p := P{len(xd.Tokens.Stack), xd.Tokens.Last.Length()} + switch { + case len(want) == 0: + return fmt.Errorf("%s: %v: got more values than wanted", name, p) + case p != want[0]: + return fmt.Errorf("%s: got %v, want %v", name, p, want[0]) + default: + want = want[1:] + return SkipFunc + } + } + lastChecks = append(lastChecks, func() error { + if len(want) > 0 { + return fmt.Errorf("%s: did not get enough values, want %d more", name, len(want)) + } + return nil + }) + return checkNext + } + + // In contrast to marshal, unmarshal automatically allocates for + // nil pointers, which causes unmarshal to visit more values. + wantAny := []PV{ + {P{1, 0}, addr(any(nil))}, + {P{1, 1}, addr(any(valueStringer{}))}, + {P{1, 1}, addr(valueStringer{})}, + {P{1, 2}, addr(any((*valueStringer)(nil)))}, + {P{1, 2}, addr((*valueStringer)(nil))}, + {P{1, 2}, addr(valueStringer{})}, + {P{1, 3}, addr(any(addr(valueStringer{})))}, + {P{1, 3}, addr(addr(valueStringer{}))}, + {P{1, 3}, addr(valueStringer{})}, + {P{1, 4}, addr(any((**valueStringer)(nil)))}, + {P{1, 4}, addr((**valueStringer)(nil))}, + {P{1, 4}, addr((*valueStringer)(nil))}, + {P{1, 4}, addr(valueStringer{})}, + {P{1, 5}, addr(any(addr((*valueStringer)(nil))))}, + {P{1, 5}, addr(addr((*valueStringer)(nil)))}, + {P{1, 5}, addr((*valueStringer)(nil))}, + {P{1, 5}, addr(valueStringer{})}, + {P{1, 6}, addr(any(addr(addr(valueStringer{}))))}, + {P{1, 6}, addr(addr(addr(valueStringer{})))}, + {P{1, 6}, addr(addr(valueStringer{}))}, + {P{1, 6}, addr(valueStringer{})}, + {P{1, 7}, addr(any(pointerStringer{}))}, + {P{1, 7}, addr(pointerStringer{})}, + {P{1, 8}, addr(any((*pointerStringer)(nil)))}, + {P{1, 8}, addr((*pointerStringer)(nil))}, + {P{1, 8}, addr(pointerStringer{})}, + {P{1, 9}, addr(any(addr(pointerStringer{})))}, + {P{1, 9}, addr(addr(pointerStringer{}))}, + {P{1, 9}, addr(pointerStringer{})}, + {P{1, 10}, addr(any((**pointerStringer)(nil)))}, + {P{1, 10}, addr((**pointerStringer)(nil))}, + {P{1, 10}, addr((*pointerStringer)(nil))}, + {P{1, 10}, addr(pointerStringer{})}, + {P{1, 11}, addr(any(addr((*pointerStringer)(nil))))}, + {P{1, 11}, addr(addr((*pointerStringer)(nil)))}, + {P{1, 11}, addr((*pointerStringer)(nil))}, + {P{1, 11}, addr(pointerStringer{})}, + {P{1, 12}, addr(any(addr(addr(pointerStringer{}))))}, + {P{1, 12}, addr(addr(addr(pointerStringer{})))}, + {P{1, 12}, addr(addr(pointerStringer{}))}, + {P{1, 12}, addr(pointerStringer{})}, + {P{1, 13}, addr(any("LAST"))}, + {P{1, 13}, addr("LAST")}, + } + checkAny := makeValueChecker("any", wantAny) + anyUnmarshaler := UnmarshalFromFunc(func(dec *jsontext.Decoder, v any) error { + return checkAny(dec, v) + }) + + var wantPointerAny []PV + for _, v := range wantAny { + if _, ok := v.V.(*any); ok { + wantPointerAny = append(wantPointerAny, v) + } + } + checkPointerAny := makeValueChecker("*any", wantPointerAny) + pointerAnyUnmarshaler := UnmarshalFromFunc(func(dec *jsontext.Decoder, v *any) error { + return checkPointerAny(dec, v) + }) + + checkNamedAny := makeValueChecker("namedAny", wantAny) + namedAnyUnmarshaler := UnmarshalFromFunc(func(dec *jsontext.Decoder, v namedAny) error { + return checkNamedAny(dec, v) + }) + + checkPointerNamedAny := makeValueChecker("*namedAny", nil) + pointerNamedAnyUnmarshaler := UnmarshalFromFunc(func(dec *jsontext.Decoder, v *namedAny) error { + return checkPointerNamedAny(dec, v) + }) + + type stringer = fmt.Stringer + var wantStringer []PV + for _, v := range wantAny { + if _, ok := v.V.(stringer); ok { + wantStringer = append(wantStringer, v) + } + } + checkStringer := makeValueChecker("stringer", wantStringer) + stringerUnmarshaler := UnmarshalFromFunc(func(dec *jsontext.Decoder, v stringer) error { + return checkStringer(dec, v) + }) + + checkPointerStringer := makeValueChecker("*stringer", nil) + pointerStringerUnmarshaler := UnmarshalFromFunc(func(dec *jsontext.Decoder, v *stringer) error { + return checkPointerStringer(dec, v) + }) + + wantValueStringer := []P{{1, 1}, {1, 2}, {1, 3}, {1, 4}, {1, 5}, {1, 6}} + checkPointerValueStringer := makePositionChecker("*valueStringer", wantValueStringer) + pointerValueStringerUnmarshaler := UnmarshalFromFunc(func(dec *jsontext.Decoder, v *valueStringer) error { + return checkPointerValueStringer(dec, v) + }) + + wantPointerStringer := []P{{1, 7}, {1, 8}, {1, 9}, {1, 10}, {1, 11}, {1, 12}} + checkPointerPointerStringer := makePositionChecker("*pointerStringer", wantPointerStringer) + pointerPointerStringerUnmarshaler := UnmarshalFromFunc(func(dec *jsontext.Decoder, v *pointerStringer) error { + return checkPointerPointerStringer(dec, v) + }) + + lastUnmarshaler := UnmarshalFromFunc(func(dec *jsontext.Decoder, v *string) error { + return checkLast() + }) + + return JoinUnmarshalers( + // This is just like unmarshaling into a Go array, + // but avoids zeroing the element before calling unmarshal. + UnmarshalFromFunc(func(dec *jsontext.Decoder, v *[14]any) error { + if _, err := dec.ReadToken(); err != nil { + return err + } + for i := range len(*v) { + if err := UnmarshalDecode(dec, &(*v)[i]); err != nil { + return err + } + } + if _, err := dec.ReadToken(); err != nil { + return err + } + return nil + }), + + anyUnmarshaler, + pointerAnyUnmarshaler, + namedAnyUnmarshaler, + pointerNamedAnyUnmarshaler, // never called + stringerUnmarshaler, + pointerStringerUnmarshaler, // never called + pointerValueStringerUnmarshaler, + pointerPointerStringerUnmarshaler, + lastUnmarshaler, + ) + }()), + }, + }, { + name: jsontest.Name("Functions/Precedence/V1First"), + opts: []Options{ + WithUnmarshalers(JoinUnmarshalers( + UnmarshalFunc(func(b []byte, v *string) error { + if string(b) != `"called"` { + return fmt.Errorf("got %s, want %s", b, `"called"`) + } + *v = "called" + return nil + }), + UnmarshalFromFunc(func(dec *jsontext.Decoder, v *string) error { + panic("should not be called") + }), + )), + }, + inBuf: `"called"`, + inVal: addr(""), + want: addr("called"), + }, { + name: jsontest.Name("Functions/Precedence/V2First"), + opts: []Options{ + WithUnmarshalers(JoinUnmarshalers( + UnmarshalFromFunc(func(dec *jsontext.Decoder, v *string) error { + switch t, err := dec.ReadToken(); { + case err != nil: + return err + case t.String() != "called": + return fmt.Errorf("got %q, want %q", t, "called") + } + *v = "called" + return nil + }), + UnmarshalFunc(func([]byte, *string) error { + panic("should not be called") + }), + )), + }, + inBuf: `"called"`, + inVal: addr(""), + want: addr("called"), + }, { + name: jsontest.Name("Functions/Precedence/V2Skipped"), + opts: []Options{ + WithUnmarshalers(JoinUnmarshalers( + UnmarshalFromFunc(func(dec *jsontext.Decoder, v *string) error { + return SkipFunc + }), + UnmarshalFunc(func(b []byte, v *string) error { + if string(b) != `"called"` { + return fmt.Errorf("got %s, want %s", b, `"called"`) + } + *v = "called" + return nil + }), + )), + }, + inBuf: `"called"`, + inVal: addr(""), + want: addr("called"), + }, { + name: jsontest.Name("Functions/Precedence/NestedFirst"), + opts: []Options{ + WithUnmarshalers(JoinUnmarshalers( + JoinUnmarshalers( + UnmarshalFunc(func(b []byte, v *string) error { + if string(b) != `"called"` { + return fmt.Errorf("got %s, want %s", b, `"called"`) + } + *v = "called" + return nil + }), + ), + UnmarshalFunc(func([]byte, *string) error { + panic("should not be called") + }), + )), + }, + inBuf: `"called"`, + inVal: addr(""), + want: addr("called"), + }, { + name: jsontest.Name("Functions/Precedence/NestedLast"), + opts: []Options{ + WithUnmarshalers(JoinUnmarshalers( + UnmarshalFunc(func(b []byte, v *string) error { + if string(b) != `"called"` { + return fmt.Errorf("got %s, want %s", b, `"called"`) + } + *v = "called" + return nil + }), + JoinUnmarshalers( + UnmarshalFunc(func([]byte, *string) error { + panic("should not be called") + }), + ), + )), + }, + inBuf: `"called"`, + inVal: addr(""), + want: addr("called"), + }, { + name: jsontest.Name("Duration/Null"), + inBuf: `{"D1":null,"D2":null}`, + inVal: addr(struct { + D1 time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + D2 time.Duration `json:",format:nano"` + }{1, 1}), + want: addr(struct { + D1 time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + D2 time.Duration `json:",format:nano"` + }{0, 0}), + }, { + name: jsontest.Name("Duration/Zero"), + inBuf: `{"D1":"0s","D2":0}`, + inVal: addr(struct { + D1 time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + D2 time.Duration `json:",format:nano"` + }{1, 1}), + want: addr(struct { + D1 time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + D2 time.Duration `json:",format:nano"` + }{0, 0}), + }, { + name: jsontest.Name("Duration/Positive"), + inBuf: `{"D1":"34293h33m9.123456789s","D2":123456789123456789}`, + inVal: new(struct { + D1 time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + D2 time.Duration `json:",format:nano"` + }), + want: addr(struct { + D1 time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + D2 time.Duration `json:",format:nano"` + }{ + 123456789123456789, + 123456789123456789, + }), + }, { + name: jsontest.Name("Duration/Negative"), + inBuf: `{"D1":"-34293h33m9.123456789s","D2":-123456789123456789}`, + inVal: new(struct { + D1 time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + D2 time.Duration `json:",format:nano"` + }), + want: addr(struct { + D1 time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + D2 time.Duration `json:",format:nano"` + }{ + -123456789123456789, + -123456789123456789, + }), + }, { + name: jsontest.Name("Duration/Nanos/String"), + inBuf: `{"D":"12345"}`, + inVal: addr(struct { + D time.Duration `json:",string,format:nano"` + }{1}), + want: addr(struct { + D time.Duration `json:",string,format:nano"` + }{12345}), + }, { + name: jsontest.Name("Duration/Nanos/String/Invalid"), + inBuf: `{"D":"+12345"}`, + inVal: addr(struct { + D time.Duration `json:",string,format:nano"` + }{1}), + want: addr(struct { + D time.Duration `json:",string,format:nano"` + }{1}), + wantErr: EU(fmt.Errorf(`invalid duration "+12345": %w`, strconv.ErrSyntax)).withPos(`{"D":`, "/D").withType('"', timeDurationType), + }, { + name: jsontest.Name("Duration/Nanos/Mismatch"), + inBuf: `{"D":"34293h33m9.123456789s"}`, + inVal: addr(struct { + D time.Duration `json:",format:nano"` + }{1}), + want: addr(struct { + D time.Duration `json:",format:nano"` + }{1}), + wantErr: EU(nil).withPos(`{"D":`, "/D").withType('"', timeDurationType), + }, { + name: jsontest.Name("Duration/Nanos"), + inBuf: `{"D":1.324}`, + inVal: addr(struct { + D time.Duration `json:",format:nano"` + }{-1}), + want: addr(struct { + D time.Duration `json:",format:nano"` + }{1}), + }, { + name: jsontest.Name("Duration/String/Mismatch"), + inBuf: `{"D":-123456789123456789}`, + inVal: addr(struct { + D time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + }{1}), + want: addr(struct { + D time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + }{1}), + wantErr: EU(nil).withPos(`{"D":`, "/D").withType('0', timeDurationType), + }, { + name: jsontest.Name("Duration/String/Invalid"), + inBuf: `{"D":"5minkutes"}`, + inVal: addr(struct { + D time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + }{1}), + want: addr(struct { + D time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + }{1}), + wantErr: EU(func() error { + _, err := time.ParseDuration("5minkutes") + return err + }()).withPos(`{"D":`, "/D").withType('"', timeDurationType), + }, { + name: jsontest.Name("Duration/Syntax/Invalid"), + inBuf: `{"D":x}`, + inVal: addr(struct { + D time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + }{1}), + want: addr(struct { + D time.Duration `json:",format:units"` // TODO(https://go.dev/issue/71631): Remove the format flag. + }{1}), + wantErr: newInvalidCharacterError("x", "at start of value", len64(`{"D":`), "/D"), + }, { + name: jsontest.Name("Duration/Format"), + inBuf: `{ + "D1": "12h34m56.078090012s", + "D2": "12h34m56.078090012s", + "D3": 45296.078090012, + "D4": "45296.078090012", + "D5": 45296078.090012, + "D6": "45296078.090012", + "D7": 45296078090.012, + "D8": "45296078090.012", + "D9": 45296078090012, + "D10": "45296078090012", + "D11": "PT12H34M56.078090012S" + }`, + inVal: new(structDurationFormat), + want: addr(structDurationFormat{ + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + }), + }, { + name: jsontest.Name("Duration/Format/Invalid"), + inBuf: `{"D":"0s"}`, + inVal: addr(struct { + D time.Duration `json:",format:invalid"` + }{1}), + want: addr(struct { + D time.Duration `json:",format:invalid"` + }{1}), + wantErr: EU(errInvalidFormatFlag).withPos(`{"D":`, "/D").withType(0, timeDurationType), + }, { + /* TODO(https://go.dev/issue/71631): Re-enable this test case. + name: jsontest.Name("Duration/Format/Legacy"), + inBuf: `{"D1":45296078090012,"D2":"12h34m56.078090012s"}`, + opts: []Options{jsonflags.FormatDurationAsNano | 1}, + inVal: new(structDurationFormat), + want: addr(structDurationFormat{ + D1: 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + D2: 12*time.Hour + 34*time.Minute + 56*time.Second + 78*time.Millisecond + 90*time.Microsecond + 12*time.Nanosecond, + }), + }, { */ + /* TODO(https://go.dev/issue/71631): Re-enable this test case. + name: jsontest.Name("Duration/MapKey"), + inBuf: `{"1s":""}`, + inVal: new(map[time.Duration]string), + want: addr(map[time.Duration]string{time.Second: ""}), + }, { */ + name: jsontest.Name("Duration/MapKey/Legacy"), + opts: []Options{jsonflags.FormatDurationAsNano | 1}, + inBuf: `{"1000000000":""}`, + inVal: new(map[time.Duration]string), + want: addr(map[time.Duration]string{time.Second: ""}), + }, { + /* TODO(https://go.dev/issue/71631): Re-enable this test case. + name: jsontest.Name("Duration/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + inBuf: `"1s"`, + inVal: addr(time.Duration(0)), + want: addr(time.Second), + }, { */ + name: jsontest.Name("Time/Zero"), + inBuf: `{"T1":"0001-01-01T00:00:00Z","T2":"01 Jan 01 00:00 UTC","T3":"0001-01-01","T4":"0001-01-01T00:00:00Z","T5":"0001-01-01T00:00:00Z"}`, + inVal: new(struct { + T1 time.Time + T2 time.Time `json:",format:RFC822"` + T3 time.Time `json:",format:'2006-01-02'"` + T4 time.Time `json:",omitzero"` + T5 time.Time `json:",omitempty"` + }), + want: addr(struct { + T1 time.Time + T2 time.Time `json:",format:RFC822"` + T3 time.Time `json:",format:'2006-01-02'"` + T4 time.Time `json:",omitzero"` + T5 time.Time `json:",omitempty"` + }{ + mustParseTime(time.RFC3339Nano, "0001-01-01T00:00:00Z"), + mustParseTime(time.RFC822, "01 Jan 01 00:00 UTC"), + mustParseTime("2006-01-02", "0001-01-01"), + mustParseTime(time.RFC3339Nano, "0001-01-01T00:00:00Z"), + mustParseTime(time.RFC3339Nano, "0001-01-01T00:00:00Z"), + }), + }, { + name: jsontest.Name("Time/Format"), + inBuf: `{ + "T1": "1234-01-02T03:04:05.000000006Z", + "T2": "Mon Jan 2 03:04:05 1234", + "T3": "Mon Jan 2 03:04:05 UTC 1234", + "T4": "Mon Jan 02 03:04:05 +0000 1234", + "T5": "02 Jan 34 03:04 UTC", + "T6": "02 Jan 34 03:04 +0000", + "T7": "Monday, 02-Jan-34 03:04:05 UTC", + "T8": "Mon, 02 Jan 1234 03:04:05 UTC", + "T9": "Mon, 02 Jan 1234 03:04:05 +0000", + "T10": "1234-01-02T03:04:05Z", + "T11": "1234-01-02T03:04:05.000000006Z", + "T12": "3:04AM", + "T13": "Jan 2 03:04:05", + "T14": "Jan 2 03:04:05.000", + "T15": "Jan 2 03:04:05.000000", + "T16": "Jan 2 03:04:05.000000006", + "T17": "1234-01-02 03:04:05", + "T18": "1234-01-02", + "T19": "03:04:05", + "T20": "1234-01-02", + "T21": "\"weird\"1234", + "T22": -23225777754.999999994, + "T23": "-23225777754.999999994", + "T24": -23225777754999.999994, + "T25": "-23225777754999.999994", + "T26": -23225777754999999.994, + "T27": "-23225777754999999.994", + "T28": -23225777754999999994, + "T29": "-23225777754999999994" + }`, + inVal: new(structTimeFormat), + want: addr(structTimeFormat{ + mustParseTime(time.RFC3339Nano, "1234-01-02T03:04:05.000000006Z"), + mustParseTime(time.ANSIC, "Mon Jan 2 03:04:05 1234"), + mustParseTime(time.UnixDate, "Mon Jan 2 03:04:05 UTC 1234"), + mustParseTime(time.RubyDate, "Mon Jan 02 03:04:05 +0000 1234"), + mustParseTime(time.RFC822, "02 Jan 34 03:04 UTC"), + mustParseTime(time.RFC822Z, "02 Jan 34 03:04 +0000"), + mustParseTime(time.RFC850, "Monday, 02-Jan-34 03:04:05 UTC"), + mustParseTime(time.RFC1123, "Mon, 02 Jan 1234 03:04:05 UTC"), + mustParseTime(time.RFC1123Z, "Mon, 02 Jan 1234 03:04:05 +0000"), + mustParseTime(time.RFC3339, "1234-01-02T03:04:05Z"), + mustParseTime(time.RFC3339Nano, "1234-01-02T03:04:05.000000006Z"), + mustParseTime(time.Kitchen, "3:04AM"), + mustParseTime(time.Stamp, "Jan 2 03:04:05"), + mustParseTime(time.StampMilli, "Jan 2 03:04:05.000"), + mustParseTime(time.StampMicro, "Jan 2 03:04:05.000000"), + mustParseTime(time.StampNano, "Jan 2 03:04:05.000000006"), + mustParseTime(time.DateTime, "1234-01-02 03:04:05"), + mustParseTime(time.DateOnly, "1234-01-02"), + mustParseTime(time.TimeOnly, "03:04:05"), + mustParseTime("2006-01-02", "1234-01-02"), + mustParseTime(`\"weird\"2006`, `\"weird\"1234`), + time.Unix(-23225777755, 6).UTC(), + time.Unix(-23225777755, 6).UTC(), + time.Unix(-23225777755, 6).UTC(), + time.Unix(-23225777755, 6).UTC(), + time.Unix(-23225777755, 6).UTC(), + time.Unix(-23225777755, 6).UTC(), + time.Unix(-23225777755, 6).UTC(), + time.Unix(-23225777755, 6).UTC(), + }), + }, { + name: jsontest.Name("Time/Format/UnixString/InvalidNumber"), + inBuf: `{ + "T23": -23225777754.999999994, + "T25": -23225777754999.999994, + "T27": -23225777754999999.994, + "T29": -23225777754999999994 + }`, + inVal: new(structTimeFormat), + want: new(structTimeFormat), + wantErr: EU(nil).withPos(`{`+"\n\t\t\t"+`"T23": `, "/T23").withType('0', timeTimeType), + }, { + name: jsontest.Name("Time/Format/UnixString/InvalidString"), + inBuf: `{ + "T22": "-23225777754.999999994", + "T24": "-23225777754999.999994", + "T26": "-23225777754999999.994", + "T28": "-23225777754999999994" + }`, + inVal: new(structTimeFormat), + want: new(structTimeFormat), + wantErr: EU(nil).withPos(`{`+"\n\t\t\t"+`"T22": `, "/T22").withType('"', timeTimeType), + }, { + name: jsontest.Name("Time/Format/Null"), + inBuf: `{"T1":null,"T2":null,"T3":null,"T4":null,"T5":null,"T6":null,"T7":null,"T8":null,"T9":null,"T10":null,"T11":null,"T12":null,"T13":null,"T14":null,"T15":null,"T16":null,"T17":null,"T18":null,"T19":null,"T20":null,"T21":null,"T22":null,"T23":null,"T24":null,"T25":null,"T26":null,"T27":null,"T28":null,"T29":null}`, + inVal: addr(structTimeFormat{ + mustParseTime(time.RFC3339Nano, "1234-01-02T03:04:05.000000006Z"), + mustParseTime(time.ANSIC, "Mon Jan 2 03:04:05 1234"), + mustParseTime(time.UnixDate, "Mon Jan 2 03:04:05 UTC 1234"), + mustParseTime(time.RubyDate, "Mon Jan 02 03:04:05 +0000 1234"), + mustParseTime(time.RFC822, "02 Jan 34 03:04 UTC"), + mustParseTime(time.RFC822Z, "02 Jan 34 03:04 +0000"), + mustParseTime(time.RFC850, "Monday, 02-Jan-34 03:04:05 UTC"), + mustParseTime(time.RFC1123, "Mon, 02 Jan 1234 03:04:05 UTC"), + mustParseTime(time.RFC1123Z, "Mon, 02 Jan 1234 03:04:05 +0000"), + mustParseTime(time.RFC3339, "1234-01-02T03:04:05Z"), + mustParseTime(time.RFC3339Nano, "1234-01-02T03:04:05.000000006Z"), + mustParseTime(time.Kitchen, "3:04AM"), + mustParseTime(time.Stamp, "Jan 2 03:04:05"), + mustParseTime(time.StampMilli, "Jan 2 03:04:05.000"), + mustParseTime(time.StampMicro, "Jan 2 03:04:05.000000"), + mustParseTime(time.StampNano, "Jan 2 03:04:05.000000006"), + mustParseTime(time.DateTime, "1234-01-02 03:04:05"), + mustParseTime(time.DateOnly, "1234-01-02"), + mustParseTime(time.TimeOnly, "03:04:05"), + mustParseTime("2006-01-02", "1234-01-02"), + mustParseTime(`\"weird\"2006`, `\"weird\"1234`), + time.Unix(-23225777755, 6).UTC(), + time.Unix(-23225777755, 6).UTC(), + time.Unix(-23225777755, 6).UTC(), + time.Unix(-23225777755, 6).UTC(), + time.Unix(-23225777755, 6).UTC(), + time.Unix(-23225777755, 6).UTC(), + time.Unix(-23225777755, 6).UTC(), + time.Unix(-23225777755, 6).UTC(), + }), + want: new(structTimeFormat), + }, { + name: jsontest.Name("Time/RFC3339/Mismatch"), + inBuf: `{"T":1234}`, + inVal: new(struct { + T time.Time + }), + wantErr: EU(nil).withPos(`{"T":`, "/T").withType('0', timeTimeType), + }, { + name: jsontest.Name("Time/RFC3339/ParseError"), + inBuf: `{"T":"2021-09-29T12:44:52"}`, + inVal: new(struct { + T time.Time + }), + wantErr: EU(func() error { + _, err := time.Parse(time.RFC3339, "2021-09-29T12:44:52") + return err + }()).withPos(`{"T":`, "/T").withType('"', timeTimeType), + }, { + name: jsontest.Name("Time/Format/Invalid"), + inBuf: `{"T":""}`, + inVal: new(struct { + T time.Time `json:",format:UndefinedConstant"` + }), + wantErr: EU(errors.New(`invalid format flag "UndefinedConstant"`)).withPos(`{"T":`, "/T").withType(0, timeTimeType), + }, { + name: jsontest.Name("Time/Format/SingleDigitHour"), + inBuf: `{"T":"2000-01-01T1:12:34Z"}`, + inVal: new(struct{ T time.Time }), + wantErr: EU(newParseTimeError(time.RFC3339, "2000-01-01T1:12:34Z", "15", "1", "")).withPos(`{"T":`, "/T").withType('"', timeTimeType), + }, { + name: jsontest.Name("Time/Format/SubsecondComma"), + inBuf: `{"T":"2000-01-01T00:00:00,000Z"}`, + inVal: new(struct{ T time.Time }), + wantErr: EU(newParseTimeError(time.RFC3339, "2000-01-01T00:00:00,000Z", ".", ",", "")).withPos(`{"T":`, "/T").withType('"', timeTimeType), + }, { + name: jsontest.Name("Time/Format/TimezoneHourOverflow"), + inBuf: `{"T":"2000-01-01T00:00:00+24:00"}`, + inVal: new(struct{ T time.Time }), + wantErr: EU(newParseTimeError(time.RFC3339, "2000-01-01T00:00:00+24:00", "Z07:00", "+24:00", ": timezone hour out of range")).withPos(`{"T":`, "/T").withType('"', timeTimeType), + }, { + name: jsontest.Name("Time/Format/TimezoneMinuteOverflow"), + inBuf: `{"T":"2000-01-01T00:00:00+00:60"}`, + inVal: new(struct{ T time.Time }), + wantErr: EU(newParseTimeError(time.RFC3339, "2000-01-01T00:00:00+00:60", "Z07:00", "+00:60", ": timezone minute out of range")).withPos(`{"T":`, "/T").withType('"', timeTimeType), + }, { + name: jsontest.Name("Time/Syntax/Invalid"), + inBuf: `{"T":x}`, + inVal: new(struct { + T time.Time + }), + wantErr: newInvalidCharacterError("x", "at start of value", len64(`{"T":`), "/T"), + }, { + name: jsontest.Name("Time/IgnoreInvalidFormat"), + opts: []Options{invalidFormatOption}, + inBuf: `"2000-01-01T00:00:00Z"`, + inVal: addr(time.Time{}), + want: addr(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)), + }} + + for _, tt := range tests { + t.Run(tt.name.Name, func(t *testing.T) { + got := tt.inVal + gotErr := Unmarshal([]byte(tt.inBuf), got, tt.opts...) + if !reflect.DeepEqual(got, tt.want) && tt.want != nil { + t.Errorf("%s: Unmarshal output mismatch:\ngot %v\nwant %v", tt.name.Where, got, tt.want) + } + if !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("%s: Unmarshal error mismatch:\ngot %v\nwant %v", tt.name.Where, gotErr, tt.wantErr) + } + }) + } +} + +func TestMarshalInvalidNamespace(t *testing.T) { + tests := []struct { + name jsontest.CaseName + val any + }{ + {jsontest.Name("Map"), map[string]string{"X": "\xde\xad\xbe\xef"}}, + {jsontest.Name("Struct"), struct{ X string }{"\xde\xad\xbe\xef"}}, + } + for _, tt := range tests { + t.Run(tt.name.Name, func(t *testing.T) { + enc := jsontext.NewEncoder(new(bytes.Buffer)) + if err := MarshalEncode(enc, tt.val); err == nil { + t.Fatalf("%s: MarshalEncode error is nil, want non-nil", tt.name.Where) + } + for _, tok := range []jsontext.Token{ + jsontext.Null, jsontext.String(""), jsontext.Int(0), jsontext.BeginObject, jsontext.EndObject, jsontext.BeginArray, jsontext.EndArray, + } { + if err := enc.WriteToken(tok); err == nil { + t.Fatalf("%s: WriteToken error is nil, want non-nil", tt.name.Where) + } + } + for _, val := range []string{`null`, `""`, `0`, `{}`, `[]`} { + if err := enc.WriteValue([]byte(val)); err == nil { + t.Fatalf("%s: WriteToken error is nil, want non-nil", tt.name.Where) + } + } + }) + } +} + +func TestUnmarshalInvalidNamespace(t *testing.T) { + tests := []struct { + name jsontest.CaseName + val any + }{ + {jsontest.Name("Map"), addr(map[string]int{})}, + {jsontest.Name("Struct"), addr(struct{ X int }{})}, + } + for _, tt := range tests { + t.Run(tt.name.Name, func(t *testing.T) { + dec := jsontext.NewDecoder(strings.NewReader(`{"X":""}`)) + if err := UnmarshalDecode(dec, tt.val); err == nil { + t.Fatalf("%s: UnmarshalDecode error is nil, want non-nil", tt.name.Where) + } + if _, err := dec.ReadToken(); err == nil { + t.Fatalf("%s: ReadToken error is nil, want non-nil", tt.name.Where) + } + if _, err := dec.ReadValue(); err == nil { + t.Fatalf("%s: ReadValue error is nil, want non-nil", tt.name.Where) + } + }) + } +} + +func TestUnmarshalReuse(t *testing.T) { + t.Run("Bytes", func(t *testing.T) { + in := make([]byte, 3) + want := &in[0] + if err := Unmarshal([]byte(`"AQID"`), &in); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + got := &in[0] + if got != want { + t.Errorf("input buffer was not reused") + } + }) + t.Run("Slices", func(t *testing.T) { + in := make([]int, 3) + want := &in[0] + if err := Unmarshal([]byte(`[0,1,2]`), &in); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + got := &in[0] + if got != want { + t.Errorf("input slice was not reused") + } + }) + t.Run("Maps", func(t *testing.T) { + in := make(map[string]string) + want := reflect.ValueOf(in).Pointer() + if err := Unmarshal([]byte(`{"key":"value"}`), &in); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + got := reflect.ValueOf(in).Pointer() + if got != want { + t.Errorf("input map was not reused") + } + }) + t.Run("Pointers", func(t *testing.T) { + in := addr(addr(addr("hello"))) + want := **in + if err := Unmarshal([]byte(`"goodbye"`), &in); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + got := **in + if got != want { + t.Errorf("input pointer was not reused") + } + }) +} + +type unmarshalerEOF struct{} + +func (unmarshalerEOF) UnmarshalJSONFrom(dec *jsontext.Decoder) error { + return io.EOF // should be wrapped and converted by Unmarshal to io.ErrUnexpectedEOF +} + +// TestUnmarshalEOF verifies that io.EOF is only ever returned by +// UnmarshalDecode for a top-level value. +func TestUnmarshalEOF(t *testing.T) { + opts := WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, _ *struct{}) error { + return io.EOF // should be wrapped and converted by Unmarshal to io.ErrUnexpectedEOF + })) + + for _, in := range []string{"", "[", "[null", "[null]"} { + for _, newOut := range []func() any{ + func() any { return new(unmarshalerEOF) }, + func() any { return new([]unmarshalerEOF) }, + func() any { return new(struct{}) }, + func() any { return new([]struct{}) }, + } { + wantErr := io.ErrUnexpectedEOF + if gotErr := Unmarshal([]byte(in), newOut(), opts); !errors.Is(gotErr, wantErr) { + t.Errorf("Unmarshal = %v, want %v", gotErr, wantErr) + } + if gotErr := UnmarshalRead(strings.NewReader(in), newOut(), opts); !errors.Is(gotErr, wantErr) { + t.Errorf("Unmarshal = %v, want %v", gotErr, wantErr) + } + switch gotErr := UnmarshalDecode(jsontext.NewDecoder(strings.NewReader(in)), newOut(), opts); { + case in != "" && !errors.Is(gotErr, wantErr): + t.Errorf("Unmarshal = %v, want %v", gotErr, wantErr) + case in == "" && gotErr != io.EOF: + t.Errorf("Unmarshal = %v, want %v", gotErr, io.EOF) + } + } + } +} + +type ReaderFunc func([]byte) (int, error) + +func (f ReaderFunc) Read(b []byte) (int, error) { return f(b) } + +type WriterFunc func([]byte) (int, error) + +func (f WriterFunc) Write(b []byte) (int, error) { return f(b) } + +func TestCoderBufferGrowth(t *testing.T) { + // The growth rate of the internal buffer should be exponential, + // but should not grow unbounded. + checkGrowth := func(ns []int) { + t.Helper() + var sumBytes, sumRates, numGrows float64 + prev := ns[0] + for i := 1; i < len(ns)-1; i++ { + n := ns[i] + if n != prev { + sumRates += float64(n) / float64(prev) + numGrows++ + prev = n + } + if n > 1<<20 { + t.Fatalf("single Read/Write too large: %d", n) + } + sumBytes += float64(n) + } + if mean := sumBytes / float64(len(ns)); mean < 1<<10 { + t.Fatalf("average Read/Write too small: %0.1f", mean) + } + switch mean := sumRates / numGrows; { + case mean < 1.25: + t.Fatalf("average growth rate too slow: %0.3f", mean) + case mean > 2.00: + t.Fatalf("average growth rate too fast: %0.3f", mean) + } + } + + // bb is identical to bytes.Buffer, + // but a different type to avoid any optimizations for bytes.Buffer. + bb := struct{ *bytes.Buffer }{new(bytes.Buffer)} + + var writeSizes []int + if err := MarshalWrite(WriterFunc(func(b []byte) (int, error) { + n, err := bb.Write(b) + writeSizes = append(writeSizes, n) + return n, err + }), make([]struct{}, 1e6)); err != nil { + t.Fatalf("MarshalWrite error: %v", err) + } + checkGrowth(writeSizes) + + var readSizes []int + if err := UnmarshalRead(ReaderFunc(func(b []byte) (int, error) { + n, err := bb.Read(b) + readSizes = append(readSizes, n) + return n, err + }), new([]struct{})); err != nil { + t.Fatalf("UnmarshalRead error: %v", err) + } + checkGrowth(readSizes) +} + +func TestUintSet(t *testing.T) { + type operation any // has | insert + type has struct { + in uint + want bool + } + type insert struct { + in uint + want bool + } + + // Sequence of operations to perform (order matters). + ops := []operation{ + has{0, false}, + has{63, false}, + has{64, false}, + has{1234, false}, + insert{3, true}, + has{2, false}, + has{3, true}, + has{4, false}, + has{63, false}, + insert{3, false}, + insert{63, true}, + has{63, true}, + insert{64, true}, + insert{64, false}, + has{64, true}, + insert{3264, true}, + has{3264, true}, + insert{3, false}, + has{3, true}, + } + + var us uintSet + for i, op := range ops { + switch op := op.(type) { + case has: + if got := us.has(op.in); got != op.want { + t.Fatalf("%d: uintSet.has(%v) = %v, want %v", i, op.in, got, op.want) + } + case insert: + if got := us.insert(op.in); got != op.want { + t.Fatalf("%d: uintSet.insert(%v) = %v, want %v", i, op.in, got, op.want) + } + default: + panic(fmt.Sprintf("unknown operation: %T", op)) + } + } +} + +func TestUnmarshalDecodeOptions(t *testing.T) { + var calledFuncs int + var calledOptions Options + in := strings.NewReader(strings.Repeat("\"\xde\xad\xbe\xef\"\n", 5)) + dec := jsontext.NewDecoder(in, + jsontext.AllowInvalidUTF8(true), // decoder-specific option + WithUnmarshalers(UnmarshalFromFunc(func(dec *jsontext.Decoder, _ any) error { + opts := dec.Options() + if v, _ := GetOption(opts, jsontext.AllowInvalidUTF8); !v { + t.Errorf("nested Options.AllowInvalidUTF8 = false, want true") + } + calledFuncs++ + calledOptions = opts + return SkipFunc + })), // unmarshal-specific option; only relevant for UnmarshalDecode + ) + + if err := UnmarshalDecode(dec, new(string)); err != nil { + t.Fatalf("UnmarshalDecode: %v", err) + } + if calledFuncs != 1 { + t.Fatalf("calledFuncs = %d, want 1", calledFuncs) + } + if err := UnmarshalDecode(dec, new(string), calledOptions); err != nil { + t.Fatalf("UnmarshalDecode: %v", err) + } + if calledFuncs != 2 { + t.Fatalf("calledFuncs = %d, want 2", calledFuncs) + } + if err := UnmarshalDecode(dec, new(string), + jsontext.AllowInvalidUTF8(false), // should be ignored + WithUnmarshalers(nil), // should override + ); err != nil { + t.Fatalf("UnmarshalDecode: %v", err) + } + if calledFuncs != 2 { + t.Fatalf("calledFuncs = %d, want 2", calledFuncs) + } + if err := UnmarshalDecode(dec, new(string)); err != nil { + t.Fatalf("UnmarshalDecode: %v", err) + } + if calledFuncs != 3 { + t.Fatalf("calledFuncs = %d, want 3", calledFuncs) + } + if err := UnmarshalDecode(dec, new(string), JoinOptions( + jsontext.AllowInvalidUTF8(false), // should be ignored + WithUnmarshalers(UnmarshalFromFunc(func(_ *jsontext.Decoder, _ any) error { + opts := dec.Options() + if v, _ := GetOption(opts, jsontext.AllowInvalidUTF8); !v { + t.Errorf("nested Options.AllowInvalidUTF8 = false, want true") + } + calledFuncs = math.MaxInt + return SkipFunc + })), // should override + )); err != nil { + t.Fatalf("UnmarshalDecode: %v", err) + } + if calledFuncs != math.MaxInt { + t.Fatalf("calledFuncs = %d, want %d", calledFuncs, math.MaxInt) + } + + // Reset with the decoder options as part of the arguments should not + // observe mutations to the options until after Reset is done. + opts := dec.Options() // AllowInvalidUTF8 is currently true + dec.Reset(in, jsontext.AllowInvalidUTF8(false), opts) // earlier AllowInvalidUTF8(false) should be overridden by latter AllowInvalidUTF8(true) in opts + if v, _ := GetOption(dec.Options(), jsontext.AllowInvalidUTF8); v == false { + t.Errorf("Options.AllowInvalidUTF8 = false, want true") + } +} + +func TestUnmarshalDecodeStream(t *testing.T) { + tests := []struct { + in string + want []any + err error + }{ + {in: ``, err: io.EOF}, + {in: `{`, err: &jsontext.SyntacticError{ByteOffset: len64(`{`), Err: io.ErrUnexpectedEOF}}, + {in: `{"`, err: &jsontext.SyntacticError{ByteOffset: len64(`{"`), Err: io.ErrUnexpectedEOF}}, + {in: `{"k"`, err: &jsontext.SyntacticError{ByteOffset: len64(`{"k"`), JSONPointer: "/k", Err: io.ErrUnexpectedEOF}}, + {in: `{"k":`, err: &jsontext.SyntacticError{ByteOffset: len64(`{"k":`), JSONPointer: "/k", Err: io.ErrUnexpectedEOF}}, + {in: `{"k",`, err: &jsontext.SyntacticError{ByteOffset: len64(`{"k"`), JSONPointer: "/k", Err: jsonwire.NewInvalidCharacterError(",", "after object name (expecting ':')")}}, + {in: `{"k"}`, err: &jsontext.SyntacticError{ByteOffset: len64(`{"k"`), JSONPointer: "/k", Err: jsonwire.NewInvalidCharacterError("}", "after object name (expecting ':')")}}, + {in: `[`, err: &jsontext.SyntacticError{ByteOffset: len64(`[`), Err: io.ErrUnexpectedEOF}}, + {in: `[0`, err: &jsontext.SyntacticError{ByteOffset: len64(`[0`), Err: io.ErrUnexpectedEOF}}, + {in: ` [0`, err: &jsontext.SyntacticError{ByteOffset: len64(` [0`), Err: io.ErrUnexpectedEOF}}, + {in: `[0.`, err: &jsontext.SyntacticError{ByteOffset: len64(`[`), JSONPointer: "/0", Err: io.ErrUnexpectedEOF}}, + {in: `[0. `, err: &jsontext.SyntacticError{ByteOffset: len64(`[0.`), JSONPointer: "/0", Err: jsonwire.NewInvalidCharacterError(" ", "in number (expecting digit)")}}, + {in: `[0,`, err: &jsontext.SyntacticError{ByteOffset: len64(`[0,`), Err: io.ErrUnexpectedEOF}}, + {in: `[0:`, err: &jsontext.SyntacticError{ByteOffset: len64(`[0`), Err: jsonwire.NewInvalidCharacterError(":", "after array element (expecting ',' or ']')")}}, + {in: `n`, err: &jsontext.SyntacticError{ByteOffset: len64(`n`), Err: io.ErrUnexpectedEOF}}, + {in: `nul`, err: &jsontext.SyntacticError{ByteOffset: len64(`nul`), Err: io.ErrUnexpectedEOF}}, + {in: `fal `, err: &jsontext.SyntacticError{ByteOffset: len64(`fal`), Err: jsonwire.NewInvalidCharacterError(" ", "in literal false (expecting 's')")}}, + {in: `false`, want: []any{false}, err: io.EOF}, + {in: `false0.0[]null`, want: []any{false, 0.0, []any{}, nil}, err: io.EOF}, + } + for _, tt := range tests { + d := jsontext.NewDecoder(strings.NewReader(tt.in)) + var got []any + for { + var v any + if err := UnmarshalDecode(d, &v); err != nil { + if !reflect.DeepEqual(err, tt.err) { + t.Errorf("`%s`: UnmarshalDecode error = %v, want %v", tt.in, err, tt.err) + } + break + } + got = append(got, v) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("`%s`: UnmarshalDecode = %v, want %v", tt.in, got, tt.want) + } + } +} + +// BenchmarkUnmarshalDecodeOptions is a minimal decode operation to measure +// the overhead options setup before the unmarshal operation. +func BenchmarkUnmarshalDecodeOptions(b *testing.B) { + var i int + in := new(bytes.Buffer) + dec := jsontext.NewDecoder(in) + makeBench := func(opts ...Options) func(*testing.B) { + return func(b *testing.B) { + for range b.N { + in.WriteString("0 ") + } + dec.Reset(in) + b.ResetTimer() + for range b.N { + UnmarshalDecode(dec, &i, opts...) + } + } + } + b.Run("None", makeBench()) + b.Run("Same", makeBench(&export.Decoder(dec).Struct)) + b.Run("New", makeBench(DefaultOptionsV2())) +} + +func TestMarshalEncodeOptions(t *testing.T) { + var calledFuncs int + var calledOptions Options + out := new(bytes.Buffer) + enc := jsontext.NewEncoder( + out, + jsontext.AllowInvalidUTF8(true), // encoder-specific option + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, _ any) error { + opts := enc.Options() + if v, _ := GetOption(opts, jsontext.AllowInvalidUTF8); !v { + t.Errorf("nested Options.AllowInvalidUTF8 = false, want true") + } + calledFuncs++ + calledOptions = opts + return SkipFunc + })), // marshal-specific option; only relevant for MarshalEncode + ) + + if err := MarshalEncode(enc, "\xde\xad\xbe\xef"); err != nil { + t.Fatalf("MarshalEncode: %v", err) + } + if calledFuncs != 1 { + t.Fatalf("calledFuncs = %d, want 1", calledFuncs) + } + if err := MarshalEncode(enc, "\xde\xad\xbe\xef", calledOptions); err != nil { + t.Fatalf("MarshalEncode: %v", err) + } + if calledFuncs != 2 { + t.Fatalf("calledFuncs = %d, want 2", calledFuncs) + } + if err := MarshalEncode(enc, "\xde\xad\xbe\xef", + jsontext.AllowInvalidUTF8(false), // should be ignored + WithMarshalers(nil), // should override + ); err != nil { + t.Fatalf("MarshalEncode: %v", err) + } + if calledFuncs != 2 { + t.Fatalf("calledFuncs = %d, want 2", calledFuncs) + } + if err := MarshalEncode(enc, "\xde\xad\xbe\xef"); err != nil { + t.Fatalf("MarshalEncode: %v", err) + } + if calledFuncs != 3 { + t.Fatalf("calledFuncs = %d, want 3", calledFuncs) + } + if err := MarshalEncode(enc, "\xde\xad\xbe\xef", JoinOptions( + jsontext.AllowInvalidUTF8(false), // should be ignored + WithMarshalers(MarshalToFunc(func(enc *jsontext.Encoder, _ any) error { + opts := enc.Options() + if v, _ := GetOption(opts, jsontext.AllowInvalidUTF8); !v { + t.Errorf("nested Options.AllowInvalidUTF8 = false, want true") + } + calledFuncs = math.MaxInt + return SkipFunc + })), // should override + )); err != nil { + t.Fatalf("MarshalEncode: %v", err) + } + if calledFuncs != math.MaxInt { + t.Fatalf("calledFuncs = %d, want %d", calledFuncs, math.MaxInt) + } + if out.String() != strings.Repeat("\"\xde\xad\ufffd\ufffd\"\n", 5) { + t.Fatalf("output mismatch:\n\tgot: %s\n\twant: %s", out.String(), strings.Repeat("\"\xde\xad\xbe\xef\"\n", 5)) + } + + // Reset with the encoder options as part of the arguments should not + // observe mutations to the options until after Reset is done. + opts := enc.Options() // AllowInvalidUTF8 is currently true + enc.Reset(out, jsontext.AllowInvalidUTF8(false), opts) // earlier AllowInvalidUTF8(false) should be overridden by latter AllowInvalidUTF8(true) in opts + if v, _ := GetOption(enc.Options(), jsontext.AllowInvalidUTF8); v == false { + t.Errorf("Options.AllowInvalidUTF8 = false, want true") + } +} + +// BenchmarkMarshalEncodeOptions is a minimal encode operation to measure +// the overhead of options setup before the marshal operation. +func BenchmarkMarshalEncodeOptions(b *testing.B) { + var i int + out := new(bytes.Buffer) + enc := jsontext.NewEncoder(out) + makeBench := func(opts ...Options) func(*testing.B) { + return func(b *testing.B) { + out.Reset() + enc.Reset(out) + b.ResetTimer() + for range b.N { + MarshalEncode(enc, &i, opts...) + } + } + } + b.Run("None", makeBench()) + b.Run("Same", makeBench(&export.Encoder(enc).Struct)) + b.Run("New", makeBench(DefaultOptionsV2())) +} diff --git a/src/encoding/json/v2/arshal_time.go b/src/encoding/json/v2/arshal_time.go new file mode 100644 index 00000000000000..027503734e783e --- /dev/null +++ b/src/encoding/json/v2/arshal_time.go @@ -0,0 +1,775 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "bytes" + "cmp" + "errors" + "fmt" + "math" + "math/bits" + "reflect" + "strconv" + "strings" + "time" + + "encoding/json/internal" + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonopts" + "encoding/json/internal/jsonwire" + "encoding/json/jsontext" +) + +var ( + timeDurationType = reflect.TypeFor[time.Duration]() + timeTimeType = reflect.TypeFor[time.Time]() +) + +func makeTimeArshaler(fncs *arshaler, t reflect.Type) *arshaler { + // Ideally, time types would implement MarshalerTo and UnmarshalerFrom, + // but that would incur a dependency on package json from package time. + // Given how widely used time is, it is more acceptable that we incur a + // dependency on time from json. + // + // Injecting the arshaling functionality like this will not be identical + // to actually declaring methods on the time types since embedding of the + // time types will not be able to forward this functionality. + switch t { + case timeDurationType: + fncs.nonDefault = true + marshalNano := fncs.marshal + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) error { + xe := export.Encoder(enc) + var m durationArshaler + if mo.Format != "" && mo.FormatDepth == xe.Tokens.Depth() { + if !m.initFormat(mo.Format) { + return newInvalidFormatError(enc, t) + } + } else if mo.Flags.Get(jsonflags.FormatDurationAsNano) { + return marshalNano(enc, va, mo) + } else { + // TODO(https://go.dev/issue/71631): Decide on default duration representation. + return newMarshalErrorBefore(enc, t, errors.New("no default representation (see https://go.dev/issue/71631); specify an explicit format")) + } + + m.td, _ = reflect.TypeAssert[time.Duration](va.Value) + k := stringOrNumberKind(!m.isNumeric() || xe.Tokens.Last.NeedObjectName() || mo.Flags.Get(jsonflags.StringifyNumbers)) + if err := xe.AppendRaw(k, true, m.appendMarshal); err != nil { + if !isSyntacticError(err) && !export.IsIOError(err) { + err = newMarshalErrorBefore(enc, t, err) + } + return err + } + return nil + } + unmarshalNano := fncs.unmarshal + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) error { + xd := export.Decoder(dec) + var u durationArshaler + if uo.Format != "" && uo.FormatDepth == xd.Tokens.Depth() { + if !u.initFormat(uo.Format) { + return newInvalidFormatError(dec, t) + } + } else if uo.Flags.Get(jsonflags.FormatDurationAsNano) { + return unmarshalNano(dec, va, uo) + } else { + // TODO(https://go.dev/issue/71631): Decide on default duration representation. + return newUnmarshalErrorBeforeWithSkipping(dec, t, errors.New("no default representation (see https://go.dev/issue/71631); specify an explicit format")) + } + + stringify := !u.isNumeric() || xd.Tokens.Last.NeedObjectName() || uo.Flags.Get(jsonflags.StringifyNumbers) + var flags jsonwire.ValueFlags + td, _ := reflect.TypeAssert[*time.Duration](va.Addr()) + val, err := xd.ReadValue(&flags) + if err != nil { + return err + } + switch k := val.Kind(); k { + case 'n': + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + *td = time.Duration(0) + } + return nil + case '"': + if !stringify { + break + } + val = jsonwire.UnquoteMayCopy(val, flags.IsVerbatim()) + if err := u.unmarshal(val); err != nil { + return newUnmarshalErrorAfter(dec, t, err) + } + *td = u.td + return nil + case '0': + if stringify { + break + } + if err := u.unmarshal(val); err != nil { + return newUnmarshalErrorAfter(dec, t, err) + } + *td = u.td + return nil + } + return newUnmarshalErrorAfter(dec, t, nil) + } + case timeTimeType: + fncs.nonDefault = true + fncs.marshal = func(enc *jsontext.Encoder, va addressableValue, mo *jsonopts.Struct) (err error) { + xe := export.Encoder(enc) + var m timeArshaler + if mo.Format != "" && mo.FormatDepth == xe.Tokens.Depth() { + if !m.initFormat(mo.Format) { + return newInvalidFormatError(enc, t) + } + } + + m.tt, _ = reflect.TypeAssert[time.Time](va.Value) + k := stringOrNumberKind(!m.isNumeric() || xe.Tokens.Last.NeedObjectName() || mo.Flags.Get(jsonflags.StringifyNumbers)) + if err := xe.AppendRaw(k, !m.hasCustomFormat(), m.appendMarshal); err != nil { + if mo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return internal.NewMarshalerError(va.Addr().Interface(), err, "MarshalJSON") // unlike unmarshal, always wrapped + } + if !isSyntacticError(err) && !export.IsIOError(err) { + err = newMarshalErrorBefore(enc, t, err) + } + return err + } + return nil + } + fncs.unmarshal = func(dec *jsontext.Decoder, va addressableValue, uo *jsonopts.Struct) (err error) { + xd := export.Decoder(dec) + var u timeArshaler + if uo.Format != "" && uo.FormatDepth == xd.Tokens.Depth() { + if !u.initFormat(uo.Format) { + return newInvalidFormatError(dec, t) + } + } else if uo.Flags.Get(jsonflags.ParseTimeWithLooseRFC3339) { + u.looseRFC3339 = true + } + + stringify := !u.isNumeric() || xd.Tokens.Last.NeedObjectName() || uo.Flags.Get(jsonflags.StringifyNumbers) + var flags jsonwire.ValueFlags + tt, _ := reflect.TypeAssert[*time.Time](va.Addr()) + val, err := xd.ReadValue(&flags) + if err != nil { + return err + } + switch k := val.Kind(); k { + case 'n': + if !uo.Flags.Get(jsonflags.MergeWithLegacySemantics) { + *tt = time.Time{} + } + return nil + case '"': + if !stringify { + break + } + val = jsonwire.UnquoteMayCopy(val, flags.IsVerbatim()) + if err := u.unmarshal(val); err != nil { + if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return err // unlike marshal, never wrapped + } + return newUnmarshalErrorAfter(dec, t, err) + } + *tt = u.tt + return nil + case '0': + if stringify { + break + } + if err := u.unmarshal(val); err != nil { + if uo.Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + return err // unlike marshal, never wrapped + } + return newUnmarshalErrorAfter(dec, t, err) + } + *tt = u.tt + return nil + } + return newUnmarshalErrorAfter(dec, t, nil) + } + } + return fncs +} + +type durationArshaler struct { + td time.Duration + + // base records the representation where: + // - 0 uses time.Duration.String + // - 1e0, 1e3, 1e6, or 1e9 use a decimal encoding of the duration as + // nanoseconds, microseconds, milliseconds, or seconds. + // - 8601 uses ISO 8601 + base uint64 +} + +func (a *durationArshaler) initFormat(format string) (ok bool) { + switch format { + case "units": + a.base = 0 + case "sec": + a.base = 1e9 + case "milli": + a.base = 1e6 + case "micro": + a.base = 1e3 + case "nano": + a.base = 1e0 + case "iso8601": + a.base = 8601 + default: + return false + } + return true +} + +func (a *durationArshaler) isNumeric() bool { + return a.base != 0 && a.base != 8601 +} + +func (a *durationArshaler) appendMarshal(b []byte) ([]byte, error) { + switch a.base { + case 0: + return append(b, a.td.String()...), nil + case 8601: + return appendDurationISO8601(b, a.td), nil + default: + return appendDurationBase10(b, a.td, a.base), nil + } +} + +func (a *durationArshaler) unmarshal(b []byte) (err error) { + switch a.base { + case 0: + a.td, err = time.ParseDuration(string(b)) + case 8601: + a.td, err = parseDurationISO8601(b) + default: + a.td, err = parseDurationBase10(b, a.base) + } + return err +} + +type timeArshaler struct { + tt time.Time + + // base records the representation where: + // - 0 uses RFC 3339 encoding of the timestamp + // - 1e0, 1e3, 1e6, or 1e9 use a decimal encoding of the timestamp as + // seconds, milliseconds, microseconds, or nanoseconds since Unix epoch. + // - math.MaxUint uses time.Time.Format to encode the timestamp + base uint64 + format string // time format passed to time.Parse + + looseRFC3339 bool +} + +func (a *timeArshaler) initFormat(format string) bool { + // We assume that an exported constant in the time package will + // always start with an uppercase ASCII letter. + if len(format) == 0 { + return false + } + a.base = math.MaxUint // implies custom format + if c := format[0]; !('a' <= c && c <= 'z') && !('A' <= c && c <= 'Z') { + a.format = format + return true + } + switch format { + case "ANSIC": + a.format = time.ANSIC + case "UnixDate": + a.format = time.UnixDate + case "RubyDate": + a.format = time.RubyDate + case "RFC822": + a.format = time.RFC822 + case "RFC822Z": + a.format = time.RFC822Z + case "RFC850": + a.format = time.RFC850 + case "RFC1123": + a.format = time.RFC1123 + case "RFC1123Z": + a.format = time.RFC1123Z + case "RFC3339": + a.base = 0 + a.format = time.RFC3339 + case "RFC3339Nano": + a.base = 0 + a.format = time.RFC3339Nano + case "Kitchen": + a.format = time.Kitchen + case "Stamp": + a.format = time.Stamp + case "StampMilli": + a.format = time.StampMilli + case "StampMicro": + a.format = time.StampMicro + case "StampNano": + a.format = time.StampNano + case "DateTime": + a.format = time.DateTime + case "DateOnly": + a.format = time.DateOnly + case "TimeOnly": + a.format = time.TimeOnly + case "unix": + a.base = 1e0 + case "unixmilli": + a.base = 1e3 + case "unixmicro": + a.base = 1e6 + case "unixnano": + a.base = 1e9 + default: + // Reject any Go identifier in case new constants are supported. + if strings.TrimFunc(format, isLetterOrDigit) == "" { + return false + } + a.format = format + } + return true +} + +func (a *timeArshaler) isNumeric() bool { + return int(a.base) > 0 +} + +func (a *timeArshaler) hasCustomFormat() bool { + return a.base == math.MaxUint +} + +func (a *timeArshaler) appendMarshal(b []byte) ([]byte, error) { + switch a.base { + case 0: + format := cmp.Or(a.format, time.RFC3339Nano) + n0 := len(b) + b = a.tt.AppendFormat(b, format) + // Not all Go timestamps can be represented as valid RFC 3339. + // Explicitly check for these edge cases. + // See https://go.dev/issue/4556 and https://go.dev/issue/54580. + switch b := b[n0:]; { + case b[len("9999")] != '-': // year must be exactly 4 digits wide + return b, errors.New("year outside of range [0,9999]") + case b[len(b)-1] != 'Z': + c := b[len(b)-len("Z07:00")] + if ('0' <= c && c <= '9') || parseDec2(b[len(b)-len("07:00"):]) >= 24 { + return b, errors.New("timezone hour outside of range [0,23]") + } + } + return b, nil + case math.MaxUint: + return a.tt.AppendFormat(b, a.format), nil + default: + return appendTimeUnix(b, a.tt, a.base), nil + } +} + +func (a *timeArshaler) unmarshal(b []byte) (err error) { + switch a.base { + case 0: + // Use time.Time.UnmarshalText to avoid possible string allocation. + if err := a.tt.UnmarshalText(b); err != nil { + return err + } + // TODO(https://go.dev/issue/57912): + // RFC 3339 specifies the grammar for a valid timestamp. + // However, the parsing functionality in "time" is too loose and + // incorrectly accepts invalid timestamps as valid. + // Remove these manual checks when "time" checks it for us. + newParseError := func(layout, value, layoutElem, valueElem, message string) error { + return &time.ParseError{Layout: layout, Value: value, LayoutElem: layoutElem, ValueElem: valueElem, Message: message} + } + switch { + case a.looseRFC3339: + return nil + case b[len("2006-01-02T")+1] == ':': // hour must be two digits + return newParseError(time.RFC3339, string(b), "15", string(b[len("2006-01-02T"):][:1]), "") + case b[len("2006-01-02T15:04:05")] == ',': // sub-second separator must be a period + return newParseError(time.RFC3339, string(b), ".", ",", "") + case b[len(b)-1] != 'Z': + switch { + case parseDec2(b[len(b)-len("07:00"):]) >= 24: // timezone hour must be in range + return newParseError(time.RFC3339, string(b), "Z07:00", string(b[len(b)-len("Z07:00"):]), ": timezone hour out of range") + case parseDec2(b[len(b)-len("00"):]) >= 60: // timezone minute must be in range + return newParseError(time.RFC3339, string(b), "Z07:00", string(b[len(b)-len("Z07:00"):]), ": timezone minute out of range") + } + } + return nil + case math.MaxUint: + a.tt, err = time.Parse(a.format, string(b)) + return err + default: + a.tt, err = parseTimeUnix(b, a.base) + return err + } +} + +// appendDurationBase10 appends d formatted as a decimal fractional number, +// where pow10 is a power-of-10 used to scale down the number. +func appendDurationBase10(b []byte, d time.Duration, pow10 uint64) []byte { + b, n := mayAppendDurationSign(b, d) // append sign + whole, frac := bits.Div64(0, n, uint64(pow10)) // compute whole and frac fields + b = strconv.AppendUint(b, whole, 10) // append whole field + return appendFracBase10(b, frac, pow10) // append frac field +} + +// parseDurationBase10 parses d from a decimal fractional number, +// where pow10 is a power-of-10 used to scale up the number. +func parseDurationBase10(b []byte, pow10 uint64) (time.Duration, error) { + suffix, neg := consumeSign(b, false) // consume sign + wholeBytes, fracBytes := bytesCutByte(suffix, '.', true) // consume whole and frac fields + whole, okWhole := jsonwire.ParseUint(wholeBytes) // parse whole field; may overflow + frac, okFrac := parseFracBase10(fracBytes, pow10) // parse frac field + hi, lo := bits.Mul64(whole, uint64(pow10)) // overflow if hi > 0 + sum, co := bits.Add64(lo, uint64(frac), 0) // overflow if co > 0 + switch d := mayApplyDurationSign(sum, neg); { // overflow if neg != (d < 0) + case (!okWhole && whole != math.MaxUint64) || !okFrac: + return 0, fmt.Errorf("invalid duration %q: %w", b, strconv.ErrSyntax) + case !okWhole || hi > 0 || co > 0 || neg != (d < 0): + return 0, fmt.Errorf("invalid duration %q: %w", b, strconv.ErrRange) + default: + return d, nil + } +} + +// appendDurationISO8601 appends an ISO 8601 duration with a restricted grammar, +// where leading and trailing zeroes and zero-value designators are omitted. +// It only uses hour, minute, and second designators since ISO 8601 defines +// those as being "accurate", while year, month, week, and day are "nominal". +func appendDurationISO8601(b []byte, d time.Duration) []byte { + if d == 0 { + return append(b, "PT0S"...) + } + b, n := mayAppendDurationSign(b, d) + b = append(b, "PT"...) + n, nsec := bits.Div64(0, n, 1e9) // compute nsec field + n, sec := bits.Div64(0, n, 60) // compute sec field + hour, min := bits.Div64(0, n, 60) // compute hour and min fields + if hour > 0 { + b = append(strconv.AppendUint(b, hour, 10), 'H') + } + if min > 0 { + b = append(strconv.AppendUint(b, min, 10), 'M') + } + if sec > 0 || nsec > 0 { + b = append(appendFracBase10(strconv.AppendUint(b, sec, 10), nsec, 1e9), 'S') + } + return b +} + +// daysPerYear is the exact average number of days in a year according to +// the Gregorian calendar, which has an extra day each year that is +// a multiple of 4, unless it is evenly divisible by 100 but not by 400. +// This does not take into account leap seconds, which are not deterministic. +const daysPerYear = 365.2425 + +var errInaccurateDateUnits = errors.New("inaccurate year, month, week, or day units") + +// parseDurationISO8601 parses a duration according to ISO 8601-1:2019, +// section 5.5.2.2 and 5.5.2.3 with the following restrictions or extensions: +// +// - A leading minus sign is permitted for negative duration according +// to ISO 8601-2:2019, section 4.4.1.9. We do not permit negative values +// for each "time scale component", which is permitted by section 4.4.1.1, +// but rarely supported by parsers. +// +// - A leading plus sign is permitted (and ignored). +// This is not required by ISO 8601, but not forbidden either. +// There is some precedent for this as it is supported by the principle of +// duration arithmetic as specified in ISO 8601-2-2019, section 14.1. +// Of note, the JavaScript grammar for ISO 8601 permits a leading plus sign. +// +// - A fractional value is only permitted for accurate units +// (i.e., hour, minute, and seconds) in the last time component, +// which is permissible by ISO 8601-1:2019, section 5.5.2.3. +// +// - Both periods ('.') and commas (',') are supported as the separator +// between the integer part and fraction part of a number, +// as specified in ISO 8601-1:2019, section 3.2.6. +// While ISO 8601 recommends comma as the default separator, +// most formatters uses a period. +// +// - Leading zeros are ignored. This is not required by ISO 8601, +// but also not forbidden by the standard. Many parsers support this. +// +// - Lowercase designators are supported. This is not required by ISO 8601, +// but also not forbidden by the standard. Many parsers support this. +// +// If the nominal units of year, month, week, or day are present, +// this produces a best-effort value and also reports [errInaccurateDateUnits]. +// +// The accepted grammar is identical to JavaScript's Duration: +// +// https://tc39.es/proposal-temporal/#prod-Duration +// +// We follow JavaScript's grammar as JSON itself is derived from JavaScript. +// The Temporal.Duration.toJSON method is guaranteed to produce an output +// that can be parsed by this function so long as arithmetic in JavaScript +// do not use a largestUnit value higher than "hours" (which is the default). +// Even if it does, this will do a best-effort parsing with inaccurate units, +// but report [errInaccurateDateUnits]. +func parseDurationISO8601(b []byte) (time.Duration, error) { + var invalid, overflow, inaccurate, sawFrac bool + var sumNanos, n, co uint64 + + // cutBytes is like [bytes.Cut], but uses either c0 or c1 as the separator. + cutBytes := func(b []byte, c0, c1 byte) (prefix, suffix []byte, ok bool) { + for i, c := range b { + if c == c0 || c == c1 { + return b[:i], b[i+1:], true + } + } + return b, nil, false + } + + // mayParseUnit attempts to parse another date or time number + // identified by the desHi and desLo unit characters. + // If the part is absent for current unit, it returns b as is. + mayParseUnit := func(b []byte, desHi, desLo byte, unit time.Duration) []byte { + number, suffix, ok := cutBytes(b, desHi, desLo) + if !ok || sawFrac { + return b // designator is not present or already saw fraction, which can only be in the last component + } + + // Parse the number. + // A fraction allowed for the accurate units in the last part. + whole, frac, ok := cutBytes(number, '.', ',') + if ok { + sawFrac = true + invalid = invalid || len(frac) == len("") || unit > time.Hour + if unit == time.Second { + n, ok = parsePaddedBase10(frac, uint64(time.Second)) + invalid = invalid || !ok + } else { + f, err := strconv.ParseFloat("0."+string(frac), 64) + invalid = invalid || err != nil || len(bytes.Trim(frac[len("."):], "0123456789")) > 0 + n = uint64(math.Round(f * float64(unit))) // never overflows since f is within [0..1] + } + sumNanos, co = bits.Add64(sumNanos, n, 0) // overflow if co > 0 + overflow = overflow || co > 0 + } + for len(whole) > 1 && whole[0] == '0' { + whole = whole[len("0"):] // trim leading zeros + } + n, ok := jsonwire.ParseUint(whole) // overflow if !ok && MaxUint64 + hi, lo := bits.Mul64(n, uint64(unit)) // overflow if hi > 0 + sumNanos, co = bits.Add64(sumNanos, lo, 0) // overflow if co > 0 + invalid = invalid || (!ok && n != math.MaxUint64) + overflow = overflow || (!ok && n == math.MaxUint64) || hi > 0 || co > 0 + inaccurate = inaccurate || unit > time.Hour + return suffix + } + + suffix, neg := consumeSign(b, true) + prefix, suffix, okP := cutBytes(suffix, 'P', 'p') + durDate, durTime, okT := cutBytes(suffix, 'T', 't') + invalid = invalid || len(prefix) > 0 || !okP || (okT && len(durTime) == 0) || len(durDate)+len(durTime) == 0 + if len(durDate) > 0 { // nominal portion of the duration + durDate = mayParseUnit(durDate, 'Y', 'y', time.Duration(daysPerYear*24*60*60*1e9)) + durDate = mayParseUnit(durDate, 'M', 'm', time.Duration(daysPerYear/12*24*60*60*1e9)) + durDate = mayParseUnit(durDate, 'W', 'w', time.Duration(7*24*60*60*1e9)) + durDate = mayParseUnit(durDate, 'D', 'd', time.Duration(24*60*60*1e9)) + invalid = invalid || len(durDate) > 0 // unknown elements + } + if len(durTime) > 0 { // accurate portion of the duration + durTime = mayParseUnit(durTime, 'H', 'h', time.Duration(60*60*1e9)) + durTime = mayParseUnit(durTime, 'M', 'm', time.Duration(60*1e9)) + durTime = mayParseUnit(durTime, 'S', 's', time.Duration(1e9)) + invalid = invalid || len(durTime) > 0 // unknown elements + } + d := mayApplyDurationSign(sumNanos, neg) + overflow = overflow || (neg != (d < 0) && d != 0) // overflows signed duration + + switch { + case invalid: + return 0, fmt.Errorf("invalid ISO 8601 duration %q: %w", b, strconv.ErrSyntax) + case overflow: + return 0, fmt.Errorf("invalid ISO 8601 duration %q: %w", b, strconv.ErrRange) + case inaccurate: + return d, fmt.Errorf("invalid ISO 8601 duration %q: %w", b, errInaccurateDateUnits) + default: + return d, nil + } +} + +// mayAppendDurationSign appends a negative sign if n is negative. +func mayAppendDurationSign(b []byte, d time.Duration) ([]byte, uint64) { + if d < 0 { + b = append(b, '-') + d *= -1 + } + return b, uint64(d) +} + +// mayApplyDurationSign inverts n if neg is specified. +func mayApplyDurationSign(n uint64, neg bool) time.Duration { + if neg { + return -1 * time.Duration(n) + } else { + return +1 * time.Duration(n) + } +} + +// appendTimeUnix appends t formatted as a decimal fractional number, +// where pow10 is a power-of-10 used to scale up the number. +func appendTimeUnix(b []byte, t time.Time, pow10 uint64) []byte { + sec, nsec := t.Unix(), int64(t.Nanosecond()) + if sec < 0 { + b = append(b, '-') + sec, nsec = negateSecNano(sec, nsec) + } + switch { + case pow10 == 1e0: // fast case where units is in seconds + b = strconv.AppendUint(b, uint64(sec), 10) + return appendFracBase10(b, uint64(nsec), 1e9) + case uint64(sec) < 1e9: // intermediate case where units is not seconds, but no overflow + b = strconv.AppendUint(b, uint64(sec)*uint64(pow10)+uint64(uint64(nsec)/(1e9/pow10)), 10) + return appendFracBase10(b, (uint64(nsec)*pow10)%1e9, 1e9) + default: // slow case where units is not seconds and overflow would occur + b = strconv.AppendUint(b, uint64(sec), 10) + b = appendPaddedBase10(b, uint64(nsec)/(1e9/pow10), pow10) + return appendFracBase10(b, (uint64(nsec)*pow10)%1e9, 1e9) + } +} + +// parseTimeUnix parses t formatted as a decimal fractional number, +// where pow10 is a power-of-10 used to scale down the number. +func parseTimeUnix(b []byte, pow10 uint64) (time.Time, error) { + suffix, neg := consumeSign(b, false) // consume sign + wholeBytes, fracBytes := bytesCutByte(suffix, '.', true) // consume whole and frac fields + whole, okWhole := jsonwire.ParseUint(wholeBytes) // parse whole field; may overflow + frac, okFrac := parseFracBase10(fracBytes, 1e9/pow10) // parse frac field + var sec, nsec int64 + switch { + case pow10 == 1e0: // fast case where units is in seconds + sec = int64(whole) // check overflow later after negation + nsec = int64(frac) // cannot overflow + case okWhole: // intermediate case where units is not seconds, but no overflow + sec = int64(whole / pow10) // check overflow later after negation + nsec = int64((whole%pow10)*(1e9/pow10) + frac) // cannot overflow + case !okWhole && whole == math.MaxUint64: // slow case where units is not seconds and overflow occurred + width := int(math.Log10(float64(pow10))) // compute len(strconv.Itoa(pow10-1)) + whole, okWhole = jsonwire.ParseUint(wholeBytes[:len(wholeBytes)-width]) // parse the upper whole field + mid, _ := parsePaddedBase10(wholeBytes[len(wholeBytes)-width:], pow10) // parse the lower whole field + sec = int64(whole) // check overflow later after negation + nsec = int64(mid*(1e9/pow10) + frac) // cannot overflow + } + if neg { + sec, nsec = negateSecNano(sec, nsec) + } + switch t := time.Unix(sec, nsec).UTC(); { + case (!okWhole && whole != math.MaxUint64) || !okFrac: + return time.Time{}, fmt.Errorf("invalid time %q: %w", b, strconv.ErrSyntax) + case !okWhole || neg != (t.Unix() < 0): + return time.Time{}, fmt.Errorf("invalid time %q: %w", b, strconv.ErrRange) + default: + return t, nil + } +} + +// negateSecNano negates a Unix timestamp, where nsec must be within [0, 1e9). +func negateSecNano(sec, nsec int64) (int64, int64) { + sec = ^sec // twos-complement negation (i.e., -1*sec + 1) + nsec = -nsec + 1e9 // negate nsec and add 1e9 (which is the extra +1 from sec negation) + sec += int64(nsec / 1e9) // handle possible overflow of nsec if it started as zero + nsec %= 1e9 // ensure nsec stays within [0, 1e9) + return sec, nsec +} + +// appendFracBase10 appends the fraction of n/max10, +// where max10 is a power-of-10 that is larger than n. +func appendFracBase10(b []byte, n, max10 uint64) []byte { + if n == 0 { + return b + } + return bytes.TrimRight(appendPaddedBase10(append(b, '.'), n, max10), "0") +} + +// parseFracBase10 parses the fraction of n/max10, +// where max10 is a power-of-10 that is larger than n. +func parseFracBase10(b []byte, max10 uint64) (n uint64, ok bool) { + switch { + case len(b) == 0: + return 0, true + case len(b) < len(".0") || b[0] != '.': + return 0, false + } + return parsePaddedBase10(b[len("."):], max10) +} + +// appendPaddedBase10 appends a zero-padded encoding of n, +// where max10 is a power-of-10 that is larger than n. +func appendPaddedBase10(b []byte, n, max10 uint64) []byte { + if n < max10/10 { + // Formatting of n is shorter than log10(max10), + // so add max10/10 to ensure the length is equal to log10(max10). + i := len(b) + b = strconv.AppendUint(b, n+max10/10, 10) + b[i]-- // subtract the addition of max10/10 + return b + } + return strconv.AppendUint(b, n, 10) +} + +// parsePaddedBase10 parses b as the zero-padded encoding of n, +// where max10 is a power-of-10 that is larger than n. +// Truncated suffix is treated as implicit zeros. +// Extended suffix is ignored, but verified to contain only digits. +func parsePaddedBase10(b []byte, max10 uint64) (n uint64, ok bool) { + pow10 := uint64(1) + for pow10 < max10 { + n *= 10 + if len(b) > 0 { + if b[0] < '0' || '9' < b[0] { + return n, false + } + n += uint64(b[0] - '0') + b = b[1:] + } + pow10 *= 10 + } + if len(b) > 0 && len(bytes.TrimRight(b, "0123456789")) > 0 { + return n, false // trailing characters are not digits + } + return n, true +} + +// consumeSign consumes an optional leading negative or positive sign. +func consumeSign(b []byte, allowPlus bool) ([]byte, bool) { + if len(b) > 0 { + if b[0] == '-' { + return b[len("-"):], true + } else if b[0] == '+' && allowPlus { + return b[len("+"):], false + } + } + return b, false +} + +// bytesCutByte is similar to bytes.Cut(b, []byte{c}), +// except c may optionally be included as part of the suffix. +func bytesCutByte(b []byte, c byte, include bool) ([]byte, []byte) { + if i := bytes.IndexByte(b, c); i >= 0 { + if include { + return b[:i], b[i:] + } + return b[:i], b[i+1:] + } + return b, nil +} + +// parseDec2 parses b as an unsigned, base-10, 2-digit number. +// The result is undefined if digits are not base-10. +func parseDec2(b []byte) byte { + if len(b) < 2 { + return 0 + } + return 10*(b[0]-'0') + (b[1] - '0') +} diff --git a/src/encoding/json/v2/arshal_time_test.go b/src/encoding/json/v2/arshal_time_test.go new file mode 100644 index 00000000000000..6c08e12494860b --- /dev/null +++ b/src/encoding/json/v2/arshal_time_test.go @@ -0,0 +1,394 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "errors" + "fmt" + "math" + "strconv" + "testing" + "time" + + "encoding/json/internal/jsonwire" +) + +func baseLabel(base uint64) string { + if log10 := math.Log10(float64(base)); log10 == float64(int64(log10)) { + return fmt.Sprintf("1e%d", int(log10)) + } + return fmt.Sprint(base) +} + +var formatDurationTestdata = []struct { + td time.Duration + base10Sec string + base10Milli string + base10Micro string + base10Nano string + iso8601 string +}{ + {math.MaxInt64, "9223372036.854775807", "9223372036854.775807", "9223372036854775.807", "9223372036854775807", "PT2562047H47M16.854775807S"}, + {123*time.Hour + 4*time.Minute + 56*time.Second, "443096", "443096000", "443096000000", "443096000000000", "PT123H4M56S"}, + {time.Hour, "3600", "3600000", "3600000000", "3600000000000", "PT1H"}, + {time.Minute, "60", "60000", "60000000", "60000000000", "PT1M"}, + {1e12 + 1e12, "2000", "2000000", "2000000000", "2000000000000", "PT33M20S"}, + {1e12 + 1e11, "1100", "1100000", "1100000000", "1100000000000", "PT18M20S"}, + {1e12 + 1e10, "1010", "1010000", "1010000000", "1010000000000", "PT16M50S"}, + {1e12 + 1e9, "1001", "1001000", "1001000000", "1001000000000", "PT16M41S"}, + {1e12 + 1e8, "1000.1", "1000100", "1000100000", "1000100000000", "PT16M40.1S"}, + {1e12 + 1e7, "1000.01", "1000010", "1000010000", "1000010000000", "PT16M40.01S"}, + {1e12 + 1e6, "1000.001", "1000001", "1000001000", "1000001000000", "PT16M40.001S"}, + {1e12 + 1e5, "1000.0001", "1000000.1", "1000000100", "1000000100000", "PT16M40.0001S"}, + {1e12 + 1e4, "1000.00001", "1000000.01", "1000000010", "1000000010000", "PT16M40.00001S"}, + {1e12 + 1e3, "1000.000001", "1000000.001", "1000000001", "1000000001000", "PT16M40.000001S"}, + {1e12 + 1e2, "1000.0000001", "1000000.0001", "1000000000.1", "1000000000100", "PT16M40.0000001S"}, + {1e12 + 1e1, "1000.00000001", "1000000.00001", "1000000000.01", "1000000000010", "PT16M40.00000001S"}, + {1e12 + 1e0, "1000.000000001", "1000000.000001", "1000000000.001", "1000000000001", "PT16M40.000000001S"}, + {+(1e9 + 1), "1.000000001", "1000.000001", "1000000.001", "1000000001", "PT1.000000001S"}, + {+(1e9), "1", "1000", "1000000", "1000000000", "PT1S"}, + {+(1e9 - 1), "0.999999999", "999.999999", "999999.999", "999999999", "PT0.999999999S"}, + {+100000000, "0.1", "100", "100000", "100000000", "PT0.1S"}, + {+120000000, "0.12", "120", "120000", "120000000", "PT0.12S"}, + {+123000000, "0.123", "123", "123000", "123000000", "PT0.123S"}, + {+123400000, "0.1234", "123.4", "123400", "123400000", "PT0.1234S"}, + {+123450000, "0.12345", "123.45", "123450", "123450000", "PT0.12345S"}, + {+123456000, "0.123456", "123.456", "123456", "123456000", "PT0.123456S"}, + {+123456700, "0.1234567", "123.4567", "123456.7", "123456700", "PT0.1234567S"}, + {+123456780, "0.12345678", "123.45678", "123456.78", "123456780", "PT0.12345678S"}, + {+123456789, "0.123456789", "123.456789", "123456.789", "123456789", "PT0.123456789S"}, + {+12345678, "0.012345678", "12.345678", "12345.678", "12345678", "PT0.012345678S"}, + {+1234567, "0.001234567", "1.234567", "1234.567", "1234567", "PT0.001234567S"}, + {+123456, "0.000123456", "0.123456", "123.456", "123456", "PT0.000123456S"}, + {+12345, "0.000012345", "0.012345", "12.345", "12345", "PT0.000012345S"}, + {+1234, "0.000001234", "0.001234", "1.234", "1234", "PT0.000001234S"}, + {+123, "0.000000123", "0.000123", "0.123", "123", "PT0.000000123S"}, + {+12, "0.000000012", "0.000012", "0.012", "12", "PT0.000000012S"}, + {+1, "0.000000001", "0.000001", "0.001", "1", "PT0.000000001S"}, + {0, "0", "0", "0", "0", "PT0S"}, + {-1, "-0.000000001", "-0.000001", "-0.001", "-1", "-PT0.000000001S"}, + {-12, "-0.000000012", "-0.000012", "-0.012", "-12", "-PT0.000000012S"}, + {-123, "-0.000000123", "-0.000123", "-0.123", "-123", "-PT0.000000123S"}, + {-1234, "-0.000001234", "-0.001234", "-1.234", "-1234", "-PT0.000001234S"}, + {-12345, "-0.000012345", "-0.012345", "-12.345", "-12345", "-PT0.000012345S"}, + {-123456, "-0.000123456", "-0.123456", "-123.456", "-123456", "-PT0.000123456S"}, + {-1234567, "-0.001234567", "-1.234567", "-1234.567", "-1234567", "-PT0.001234567S"}, + {-12345678, "-0.012345678", "-12.345678", "-12345.678", "-12345678", "-PT0.012345678S"}, + {-123456789, "-0.123456789", "-123.456789", "-123456.789", "-123456789", "-PT0.123456789S"}, + {-123456780, "-0.12345678", "-123.45678", "-123456.78", "-123456780", "-PT0.12345678S"}, + {-123456700, "-0.1234567", "-123.4567", "-123456.7", "-123456700", "-PT0.1234567S"}, + {-123456000, "-0.123456", "-123.456", "-123456", "-123456000", "-PT0.123456S"}, + {-123450000, "-0.12345", "-123.45", "-123450", "-123450000", "-PT0.12345S"}, + {-123400000, "-0.1234", "-123.4", "-123400", "-123400000", "-PT0.1234S"}, + {-123000000, "-0.123", "-123", "-123000", "-123000000", "-PT0.123S"}, + {-120000000, "-0.12", "-120", "-120000", "-120000000", "-PT0.12S"}, + {-100000000, "-0.1", "-100", "-100000", "-100000000", "-PT0.1S"}, + {-(1e9 - 1), "-0.999999999", "-999.999999", "-999999.999", "-999999999", "-PT0.999999999S"}, + {-(1e9), "-1", "-1000", "-1000000", "-1000000000", "-PT1S"}, + {-(1e9 + 1), "-1.000000001", "-1000.000001", "-1000000.001", "-1000000001", "-PT1.000000001S"}, + {math.MinInt64, "-9223372036.854775808", "-9223372036854.775808", "-9223372036854775.808", "-9223372036854775808", "-PT2562047H47M16.854775808S"}, +} + +func TestFormatDuration(t *testing.T) { + var gotBuf []byte + check := func(td time.Duration, s string, base uint64) { + a := durationArshaler{td, base} + gotBuf, _ = a.appendMarshal(gotBuf[:0]) + if string(gotBuf) != s { + t.Errorf("formatDuration(%d, %s) = %q, want %q", td, baseLabel(base), string(gotBuf), s) + } + if err := a.unmarshal(gotBuf); err != nil { + t.Errorf("parseDuration(%q, %s) error: %v", gotBuf, baseLabel(base), err) + } + if a.td != td { + t.Errorf("parseDuration(%q, %s) = %d, want %d", gotBuf, baseLabel(base), a.td, td) + } + } + for _, tt := range formatDurationTestdata { + check(tt.td, tt.base10Sec, 1e9) + check(tt.td, tt.base10Milli, 1e6) + check(tt.td, tt.base10Micro, 1e3) + check(tt.td, tt.base10Nano, 1e0) + check(tt.td, tt.iso8601, 8601) + } +} + +var parseDurationTestdata = []struct { + in string + base uint64 + want time.Duration + wantErr error +}{ + {"0", 1e0, 0, nil}, + {"0.", 1e0, 0, strconv.ErrSyntax}, + {"0.0", 1e0, 0, nil}, + {"0.00", 1e0, 0, nil}, + {"00.0", 1e0, 0, strconv.ErrSyntax}, + {"+0", 1e0, 0, strconv.ErrSyntax}, + {"1e0", 1e0, 0, strconv.ErrSyntax}, + {"1.000000000x", 1e9, 0, strconv.ErrSyntax}, + {"1.000000x", 1e6, 0, strconv.ErrSyntax}, + {"1.000x", 1e3, 0, strconv.ErrSyntax}, + {"1.x", 1e0, 0, strconv.ErrSyntax}, + {"1.0000000009", 1e9, +time.Second, nil}, + {"1.0000009", 1e6, +time.Millisecond, nil}, + {"1.0009", 1e3, +time.Microsecond, nil}, + {"1.9", 1e0, +time.Nanosecond, nil}, + {"-9223372036854775809", 1e0, 0, strconv.ErrRange}, + {"9223372036854775.808", 1e3, 0, strconv.ErrRange}, + {"-9223372036854.775809", 1e6, 0, strconv.ErrRange}, + {"9223372036.854775808", 1e9, 0, strconv.ErrRange}, + {"-1.9", 1e0, -time.Nanosecond, nil}, + {"-1.0009", 1e3, -time.Microsecond, nil}, + {"-1.0000009", 1e6, -time.Millisecond, nil}, + {"-1.0000000009", 1e9, -time.Second, nil}, + {"", 8601, 0, strconv.ErrSyntax}, + {"P", 8601, 0, strconv.ErrSyntax}, + {"PT", 8601, 0, strconv.ErrSyntax}, + {"PT0", 8601, 0, strconv.ErrSyntax}, + {"DT0S", 8601, 0, strconv.ErrSyntax}, + {"PT0S", 8601, 0, nil}, + {" PT0S", 8601, 0, strconv.ErrSyntax}, + {"PT0S ", 8601, 0, strconv.ErrSyntax}, + {"+PT0S", 8601, 0, nil}, + {"PT0.M", 8601, 0, strconv.ErrSyntax}, + {"PT0.S", 8601, 0, strconv.ErrSyntax}, + {"PT0.0S", 8601, 0, nil}, + {"PT0.0_0H", 8601, 0, strconv.ErrSyntax}, + {"PT0.0_0M", 8601, 0, strconv.ErrSyntax}, + {"PT0.0_0S", 8601, 0, strconv.ErrSyntax}, + {"PT.0S", 8601, 0, strconv.ErrSyntax}, + {"PT00.0S", 8601, 0, nil}, + {"PT0S", 8601, 0, nil}, + {"PT1,5S", 8601, time.Second + 500*time.Millisecond, nil}, + {"PT1H", 8601, time.Hour, nil}, + {"PT1H0S", 8601, time.Hour, nil}, + {"PT0S", 8601, 0, nil}, + {"PT00S", 8601, 0, nil}, + {"PT000S", 8601, 0, nil}, + {"PTS", 8601, 0, strconv.ErrSyntax}, + {"PT1M", 8601, time.Minute, nil}, + {"PT01M", 8601, time.Minute, nil}, + {"PT001M", 8601, time.Minute, nil}, + {"PT1H59S", 8601, time.Hour + 59*time.Second, nil}, + {"PT123H4M56.789S", 8601, 123*time.Hour + 4*time.Minute + 56*time.Second + 789*time.Millisecond, nil}, + {"-PT123H4M56.789S", 8601, -123*time.Hour - 4*time.Minute - 56*time.Second - 789*time.Millisecond, nil}, + {"PT0H0S", 8601, 0, nil}, + {"PT0H", 8601, 0, nil}, + {"PT0M", 8601, 0, nil}, + {"-PT0S", 8601, 0, nil}, + {"PT1M0S", 8601, time.Minute, nil}, + {"PT0H1M0S", 8601, time.Minute, nil}, + {"PT01H02M03S", 8601, 1*time.Hour + 2*time.Minute + 3*time.Second, nil}, + {"PT0,123S", 8601, 123 * time.Millisecond, nil}, + {"PT1.S", 8601, 0, strconv.ErrSyntax}, + {"PT1.000S", 8601, time.Second, nil}, + {"PT0.025H", 8601, time.Minute + 30*time.Second, nil}, + {"PT0.025H0M", 8601, 0, strconv.ErrSyntax}, + {"PT1.5M", 8601, time.Minute + 30*time.Second, nil}, + {"PT1.5M0S", 8601, 0, strconv.ErrSyntax}, + {"PT60M", 8601, time.Hour, nil}, + {"PT3600S", 8601, time.Hour, nil}, + {"PT1H2M3.0S", 8601, 1*time.Hour + 2*time.Minute + 3*time.Second, nil}, + {"pt1h2m3,0s", 8601, 1*time.Hour + 2*time.Minute + 3*time.Second, nil}, + {"PT-1H-2M-3S", 8601, 0, strconv.ErrSyntax}, + {"P1Y", 8601, time.Duration(daysPerYear * 24 * 60 * 60 * 1e9), errInaccurateDateUnits}, + {"P1.0Y", 8601, 0, strconv.ErrSyntax}, + {"P1M", 8601, time.Duration(daysPerYear / 12 * 24 * 60 * 60 * 1e9), errInaccurateDateUnits}, + {"P1.0M", 8601, 0, strconv.ErrSyntax}, + {"P1W", 8601, 7 * 24 * time.Hour, errInaccurateDateUnits}, + {"P1.0W", 8601, 0, strconv.ErrSyntax}, + {"P1D", 8601, 24 * time.Hour, errInaccurateDateUnits}, + {"P1.0D", 8601, 0, strconv.ErrSyntax}, + {"P1W1S", 8601, 0, strconv.ErrSyntax}, + {"-P1Y2M3W4DT5H6M7.8S", 8601, -(time.Duration(14*daysPerYear/12*24*60*60*1e9) + time.Duration((3*7+4)*24*60*60*1e9) + 5*time.Hour + 6*time.Minute + 7*time.Second + 800*time.Millisecond), errInaccurateDateUnits}, + {"-p1y2m3w4dt5h6m7.8s", 8601, -(time.Duration(14*daysPerYear/12*24*60*60*1e9) + time.Duration((3*7+4)*24*60*60*1e9) + 5*time.Hour + 6*time.Minute + 7*time.Second + 800*time.Millisecond), errInaccurateDateUnits}, + {"P0Y0M0DT1H2M3S", 8601, 1*time.Hour + 2*time.Minute + 3*time.Second, errInaccurateDateUnits}, + {"PT0.0000000001S", 8601, 0, nil}, + {"PT0.0000000005S", 8601, 0, nil}, + {"PT0.000000000500000000S", 8601, 0, nil}, + {"PT0.000000000499999999S", 8601, 0, nil}, + {"PT2562047H47M16.854775808S", 8601, 0, strconv.ErrRange}, + {"-PT2562047H47M16.854775809S", 8601, 0, strconv.ErrRange}, + {"PT9223372036.854775807S", 8601, math.MaxInt64, nil}, + {"PT9223372036.854775808S", 8601, 0, strconv.ErrRange}, + {"-PT9223372036.854775808S", 8601, math.MinInt64, nil}, + {"-PT9223372036.854775809S", 8601, 0, strconv.ErrRange}, + {"PT18446744073709551616S", 8601, 0, strconv.ErrRange}, + {"PT5124096H", 8601, 0, strconv.ErrRange}, + {"PT2562047.7880152155019444H", 8601, math.MaxInt64, nil}, + {"PT2562047.7880152155022222H", 8601, 0, strconv.ErrRange}, + {"PT5124094H94M33.709551616S", 8601, 0, strconv.ErrRange}, +} + +func TestParseDuration(t *testing.T) { + for _, tt := range parseDurationTestdata { + a := durationArshaler{base: tt.base} + switch err := a.unmarshal([]byte(tt.in)); { + case a.td != tt.want: + t.Errorf("parseDuration(%q, %s) = %v, want %v", tt.in, baseLabel(tt.base), a.td, tt.want) + case !errors.Is(err, tt.wantErr): + t.Errorf("parseDuration(%q, %s) error = %v, want %v", tt.in, baseLabel(tt.base), err, tt.wantErr) + } + } +} + +func FuzzFormatDuration(f *testing.F) { + for _, tt := range formatDurationTestdata { + f.Add(int64(tt.td)) + } + f.Fuzz(func(t *testing.T, want int64) { + var buf []byte + for _, base := range [...]uint64{1e0, 1e3, 1e6, 1e9, 8601} { + a := durationArshaler{td: time.Duration(want), base: base} + buf, _ = a.appendMarshal(buf[:0]) + switch err := a.unmarshal(buf); { + case err != nil: + t.Fatalf("parseDuration(%q, %s) error: %v", buf, baseLabel(base), err) + case a.td != time.Duration(want): + t.Fatalf("parseDuration(%q, %s) = %v, want %v", buf, baseLabel(base), a.td, time.Duration(want)) + } + } + }) +} + +func FuzzParseDuration(f *testing.F) { + for _, tt := range parseDurationTestdata { + f.Add([]byte(tt.in)) + } + f.Fuzz(func(t *testing.T, in []byte) { + for _, base := range [...]uint64{1e0, 1e3, 1e6, 1e9, 8601} { + a := durationArshaler{base: base} + switch err := a.unmarshal(in); { + case err != nil: // nothing else to check + case base != 8601: + if n, err := jsonwire.ConsumeNumber(in); err != nil || n != len(in) { + t.Fatalf("parseDuration(%q) error is nil for invalid JSON number", in) + } + } + } + }) +} + +type formatTimeTestdataEntry struct { + ts time.Time + unixSec string + unixMilli string + unixMicro string + unixNano string +} + +var formatTimeTestdata = func() []formatTimeTestdataEntry { + out := []formatTimeTestdataEntry{ + {time.Unix(math.MaxInt64/int64(1e0), 1e9-1).UTC(), "9223372036854775807.999999999", "9223372036854775807999.999999", "9223372036854775807999999.999", "9223372036854775807999999999"}, + {time.Unix(math.MaxInt64/int64(1e1), 1e9-1).UTC(), "922337203685477580.999999999", "922337203685477580999.999999", "922337203685477580999999.999", "922337203685477580999999999"}, + {time.Unix(math.MaxInt64/int64(1e2), 1e9-1).UTC(), "92233720368547758.999999999", "92233720368547758999.999999", "92233720368547758999999.999", "92233720368547758999999999"}, + {time.Unix(math.MinInt64, 1).UTC(), "-9223372036854775807.999999999", "-9223372036854775807999.999999", "-9223372036854775807999999.999", "-9223372036854775807999999999"}, + {time.Unix(math.MinInt64, 0).UTC(), "-9223372036854775808", "-9223372036854775808000", "-9223372036854775808000000", "-9223372036854775808000000000"}, + } + for _, tt := range formatDurationTestdata { + out = append(out, formatTimeTestdataEntry{time.Unix(0, int64(tt.td)).UTC(), tt.base10Sec, tt.base10Milli, tt.base10Micro, tt.base10Nano}) + } + return out +}() + +func TestFormatTime(t *testing.T) { + var gotBuf []byte + check := func(ts time.Time, s string, pow10 uint64) { + gotBuf = appendTimeUnix(gotBuf[:0], ts, pow10) + if string(gotBuf) != s { + t.Errorf("formatTime(time.Unix(%d, %d), %s) = %q, want %q", ts.Unix(), ts.Nanosecond(), baseLabel(pow10), string(gotBuf), s) + } + gotTS, err := parseTimeUnix(gotBuf, pow10) + if err != nil { + t.Errorf("parseTime(%q, %s) error: %v", gotBuf, baseLabel(pow10), err) + } + if !gotTS.Equal(ts) { + t.Errorf("parseTime(%q, %s) = time.Unix(%d, %d), want time.Unix(%d, %d)", gotBuf, baseLabel(pow10), gotTS.Unix(), gotTS.Nanosecond(), ts.Unix(), ts.Nanosecond()) + } + } + for _, tt := range formatTimeTestdata { + check(tt.ts, tt.unixSec, 1e0) + check(tt.ts, tt.unixMilli, 1e3) + check(tt.ts, tt.unixMicro, 1e6) + check(tt.ts, tt.unixNano, 1e9) + } +} + +var parseTimeTestdata = []struct { + in string + base uint64 + want time.Time + wantErr error +}{ + {"0", 1e0, time.Unix(0, 0).UTC(), nil}, + {"0.", 1e0, time.Time{}, strconv.ErrSyntax}, + {"0.0", 1e0, time.Unix(0, 0).UTC(), nil}, + {"0.00", 1e0, time.Unix(0, 0).UTC(), nil}, + {"00.0", 1e0, time.Time{}, strconv.ErrSyntax}, + {"+0", 1e0, time.Time{}, strconv.ErrSyntax}, + {"1e0", 1e0, time.Time{}, strconv.ErrSyntax}, + {"1234567890123456789012345678901234567890", 1e0, time.Time{}, strconv.ErrRange}, + {"9223372036854775808000.000000", 1e3, time.Time{}, strconv.ErrRange}, + {"9223372036854775807999999.9999", 1e6, time.Unix(math.MaxInt64, 1e9-1).UTC(), nil}, + {"9223372036854775807999999999.9", 1e9, time.Unix(math.MaxInt64, 1e9-1).UTC(), nil}, + {"9223372036854775807.999999999x", 1e0, time.Time{}, strconv.ErrSyntax}, + {"9223372036854775807000000000", 1e9, time.Unix(math.MaxInt64, 0).UTC(), nil}, + {"-9223372036854775808", 1e0, time.Unix(math.MinInt64, 0).UTC(), nil}, + {"-9223372036854775808000.000001", 1e3, time.Time{}, strconv.ErrRange}, + {"-9223372036854775808000000.0001", 1e6, time.Unix(math.MinInt64, 0).UTC(), nil}, + {"-9223372036854775808000000000.x", 1e9, time.Time{}, strconv.ErrSyntax}, + {"-1234567890123456789012345678901234567890", 1e9, time.Time{}, strconv.ErrRange}, +} + +func TestParseTime(t *testing.T) { + for _, tt := range parseTimeTestdata { + a := timeArshaler{base: tt.base} + switch err := a.unmarshal([]byte(tt.in)); { + case a.tt != tt.want: + t.Errorf("parseTime(%q, %s) = time.Unix(%d, %d), want time.Unix(%d, %d)", tt.in, baseLabel(tt.base), a.tt.Unix(), a.tt.Nanosecond(), tt.want.Unix(), tt.want.Nanosecond()) + case !errors.Is(err, tt.wantErr): + t.Errorf("parseTime(%q, %s) error = %v, want %v", tt.in, baseLabel(tt.base), err, tt.wantErr) + } + } +} + +func FuzzFormatTime(f *testing.F) { + for _, tt := range formatTimeTestdata { + f.Add(tt.ts.Unix(), int64(tt.ts.Nanosecond())) + } + f.Fuzz(func(t *testing.T, wantSec, wantNano int64) { + want := time.Unix(wantSec, int64(uint64(wantNano)%1e9)).UTC() + var buf []byte + for _, base := range [...]uint64{1e0, 1e3, 1e6, 1e9} { + a := timeArshaler{tt: want, base: base} + buf, _ = a.appendMarshal(buf[:0]) + switch err := a.unmarshal(buf); { + case err != nil: + t.Fatalf("parseTime(%q, %s) error: %v", buf, baseLabel(base), err) + case a.tt != want: + t.Fatalf("parseTime(%q, %s) = time.Unix(%d, %d), want time.Unix(%d, %d)", buf, baseLabel(base), a.tt.Unix(), a.tt.Nanosecond(), want.Unix(), want.Nanosecond()) + } + } + }) +} + +func FuzzParseTime(f *testing.F) { + for _, tt := range parseTimeTestdata { + f.Add([]byte(tt.in)) + } + f.Fuzz(func(t *testing.T, in []byte) { + for _, base := range [...]uint64{1e0, 1e3, 1e6, 1e9} { + a := timeArshaler{base: base} + if err := a.unmarshal(in); err == nil { + if n, err := jsonwire.ConsumeNumber(in); err != nil || n != len(in) { + t.Fatalf("parseTime(%q) error is nil for invalid JSON number", in) + } + } + } + }) +} diff --git a/src/encoding/json/v2/bench_test.go b/src/encoding/json/v2/bench_test.go new file mode 100644 index 00000000000000..ae4a5b20a5cd56 --- /dev/null +++ b/src/encoding/json/v2/bench_test.go @@ -0,0 +1,648 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json_test + +import ( + "bytes" + "cmp" + "fmt" + "io" + "os" + "path" + "reflect" + "strings" + "testing" + "testing/iotest" + "time" + + jsonv1 "encoding/json" + + jsonv1in2 "encoding/json" + "encoding/json/internal/jsontest" + "encoding/json/jsontext" + jsonv2 "encoding/json/v2" +) + +// benchVersion is the version to benchmark (either "v1", "v1in2", or "v2"). +var benchVersion = cmp.Or(os.Getenv("BENCHMARK_VERSION"), "v2") + +var jsonFuncs = func() (funcs struct { + marshal func(any) ([]byte, error) + unmarshal func([]byte, any) error + encodeValue func(w io.Writer, b []byte) error + encodeTokens func(w io.Writer, toks []jsontext.Token) error + decodeValue func(r io.Reader) error + decodeTokens func(r io.Reader) error +}) { + ignoreEOF := func(err error) error { + if err == io.EOF { + err = nil + } + return err + } + + switch benchVersion { + case "v1": + funcs.marshal = jsonv1.Marshal + funcs.unmarshal = jsonv1.Unmarshal + funcs.encodeValue = func(w io.Writer, b []byte) error { + return jsonv1.NewEncoder(w).Encode(jsonv1.RawMessage(b)) + } + funcs.decodeValue = func(r io.Reader) error { + var v jsonv1.RawMessage + return jsonv1.NewDecoder(r).Decode(&v) + } + funcs.decodeTokens = func(r io.Reader) error { + d := jsonv1.NewDecoder(r) + for { + if _, err := d.Token(); err != nil { + return ignoreEOF(err) + } + } + } + case "v1in2": + funcs.marshal = jsonv1in2.Marshal + funcs.unmarshal = jsonv1in2.Unmarshal + funcs.encodeValue = func(w io.Writer, b []byte) error { + return jsonv1in2.NewEncoder(w).Encode(jsonv1in2.RawMessage(b)) + } + funcs.decodeValue = func(r io.Reader) error { + var v jsonv1in2.RawMessage + return jsonv1in2.NewDecoder(r).Decode(&v) + } + funcs.decodeTokens = func(r io.Reader) error { + d := jsonv1in2.NewDecoder(r) + for { + if _, err := d.Token(); err != nil { + return ignoreEOF(err) + } + } + } + case "v2": + funcs.marshal = func(v any) ([]byte, error) { return jsonv2.Marshal(v) } + funcs.unmarshal = func(b []byte, v any) error { return jsonv2.Unmarshal(b, v) } + funcs.encodeValue = func(w io.Writer, b []byte) error { + return jsontext.NewEncoder(w).WriteValue(b) + } + funcs.encodeTokens = func(w io.Writer, toks []jsontext.Token) error { + e := jsontext.NewEncoder(w) + for _, tok := range toks { + if err := e.WriteToken(tok); err != nil { + return err + } + } + return nil + } + funcs.decodeValue = func(r io.Reader) error { + _, err := jsontext.NewDecoder(r).ReadValue() + return err + } + funcs.decodeTokens = func(r io.Reader) error { + d := jsontext.NewDecoder(r) + for { + if _, err := d.ReadToken(); err != nil { + return ignoreEOF(err) + } + } + } + default: + panic("unknown version: " + benchVersion) + } + return +}() + +// bytesBuffer is identical to bytes.Buffer, +// but a different type to avoid any optimizations for bytes.Buffer. +type bytesBuffer struct{ *bytes.Buffer } + +func addr[T any](v T) *T { + return &v +} + +func len64[Bytes ~[]byte | ~string](in Bytes) int64 { + return int64(len(in)) +} + +var arshalTestdata = []struct { + name string + raw []byte + val any + new func() any + skipV1 bool +}{{ + name: "Bool", + raw: []byte("true"), + val: addr(true), + new: func() any { return new(bool) }, +}, { + name: "String", + raw: []byte(`"hello, world!"`), + val: addr("hello, world!"), + new: func() any { return new(string) }, +}, { + name: "Int", + raw: []byte("-1234"), + val: addr(int64(-1234)), + new: func() any { return new(int64) }, +}, { + name: "Uint", + raw: []byte("1234"), + val: addr(uint64(1234)), + new: func() any { return new(uint64) }, +}, { + name: "Float", + raw: []byte("12.34"), + val: addr(float64(12.34)), + new: func() any { return new(float64) }, +}, { + name: "Map/ManyEmpty", + raw: []byte(`[{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}]`), + val: addr(func() (out []map[string]string) { + for range 100 { + out = append(out, map[string]string{}) + } + return out + }()), + new: func() any { return new([]map[string]string) }, +}, { + name: "Map/OneLarge", + raw: []byte(`{"A":"A","B":"B","C":"C","D":"D","E":"E","F":"F","G":"G","H":"H","I":"I","J":"J","K":"K","L":"L","M":"M","N":"N","O":"O","P":"P","Q":"Q","R":"R","S":"S","T":"T","U":"U","V":"V","W":"W","X":"X","Y":"Y","Z":"Z"}`), + val: addr(map[string]string{"A": "A", "B": "B", "C": "C", "D": "D", "E": "E", "F": "F", "G": "G", "H": "H", "I": "I", "J": "J", "K": "K", "L": "L", "M": "M", "N": "N", "O": "O", "P": "P", "Q": "Q", "R": "R", "S": "S", "T": "T", "U": "U", "V": "V", "W": "W", "X": "X", "Y": "Y", "Z": "Z"}), + new: func() any { return new(map[string]string) }, +}, { + name: "Map/ManySmall", + raw: []byte(`{"A":{"K":"V"},"B":{"K":"V"},"C":{"K":"V"},"D":{"K":"V"},"E":{"K":"V"},"F":{"K":"V"},"G":{"K":"V"},"H":{"K":"V"},"I":{"K":"V"},"J":{"K":"V"},"K":{"K":"V"},"L":{"K":"V"},"M":{"K":"V"},"N":{"K":"V"},"O":{"K":"V"},"P":{"K":"V"},"Q":{"K":"V"},"R":{"K":"V"},"S":{"K":"V"},"T":{"K":"V"},"U":{"K":"V"},"V":{"K":"V"},"W":{"K":"V"},"X":{"K":"V"},"Y":{"K":"V"},"Z":{"K":"V"}}`), + val: addr(map[string]map[string]string{"A": {"K": "V"}, "B": {"K": "V"}, "C": {"K": "V"}, "D": {"K": "V"}, "E": {"K": "V"}, "F": {"K": "V"}, "G": {"K": "V"}, "H": {"K": "V"}, "I": {"K": "V"}, "J": {"K": "V"}, "K": {"K": "V"}, "L": {"K": "V"}, "M": {"K": "V"}, "N": {"K": "V"}, "O": {"K": "V"}, "P": {"K": "V"}, "Q": {"K": "V"}, "R": {"K": "V"}, "S": {"K": "V"}, "T": {"K": "V"}, "U": {"K": "V"}, "V": {"K": "V"}, "W": {"K": "V"}, "X": {"K": "V"}, "Y": {"K": "V"}, "Z": {"K": "V"}}), + new: func() any { return new(map[string]map[string]string) }, +}, { + name: "Struct/ManyEmpty", + raw: []byte(`[{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}]`), + val: addr(make([]struct{}, 100)), + new: func() any { + return new([]struct{}) + }, +}, { + name: "Struct/OneLarge", + raw: []byte(`{"A":"A","B":"B","C":"C","D":"D","E":"E","F":"F","G":"G","H":"H","I":"I","J":"J","K":"K","L":"L","M":"M","N":"N","O":"O","P":"P","Q":"Q","R":"R","S":"S","T":"T","U":"U","V":"V","W":"W","X":"X","Y":"Y","Z":"Z"}`), + val: addr(struct{ A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z string }{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"}), + new: func() any { + return new(struct{ A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z string }) + }, +}, { + name: "Struct/ManySmall", + raw: []byte(`{"A":{"K":"V"},"B":{"K":"V"},"C":{"K":"V"},"D":{"K":"V"},"E":{"K":"V"},"F":{"K":"V"},"G":{"K":"V"},"H":{"K":"V"},"I":{"K":"V"},"J":{"K":"V"},"K":{"K":"V"},"L":{"K":"V"},"M":{"K":"V"},"N":{"K":"V"},"O":{"K":"V"},"P":{"K":"V"},"Q":{"K":"V"},"R":{"K":"V"},"S":{"K":"V"},"T":{"K":"V"},"U":{"K":"V"},"V":{"K":"V"},"W":{"K":"V"},"X":{"K":"V"},"Y":{"K":"V"},"Z":{"K":"V"}}`), + val: func() any { + V := struct{ K string }{"V"} + return addr(struct{ A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z struct{ K string } }{ + V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, + }) + }(), + new: func() any { + return new(struct{ A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z struct{ K string } }) + }, +}, { + name: "Slice/ManyEmpty", + raw: []byte(`[[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]]`), + val: addr(func() (out [][]string) { + for range 100 { + out = append(out, []string{}) + } + return out + }()), + new: func() any { return new([][]string) }, +}, { + name: "Slice/OneLarge", + raw: []byte(`["A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z"]`), + val: addr([]string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"}), + new: func() any { return new([]string) }, +}, { + name: "Slice/ManySmall", + raw: []byte(`[["A"],["B"],["C"],["D"],["E"],["F"],["G"],["H"],["I"],["J"],["K"],["L"],["M"],["N"],["O"],["P"],["Q"],["R"],["S"],["T"],["U"],["V"],["W"],["X"],["Y"],["Z"]]`), + val: addr([][]string{{"A"}, {"B"}, {"C"}, {"D"}, {"E"}, {"F"}, {"G"}, {"H"}, {"I"}, {"J"}, {"K"}, {"L"}, {"M"}, {"N"}, {"O"}, {"P"}, {"Q"}, {"R"}, {"S"}, {"T"}, {"U"}, {"V"}, {"W"}, {"X"}, {"Y"}, {"Z"}}), + new: func() any { return new([][]string) }, +}, { + name: "Array/OneLarge", + raw: []byte(`["A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z"]`), + val: addr([26]string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"}), + new: func() any { return new([26]string) }, +}, { + name: "Array/ManySmall", + raw: []byte(`[["A"],["B"],["C"],["D"],["E"],["F"],["G"],["H"],["I"],["J"],["K"],["L"],["M"],["N"],["O"],["P"],["Q"],["R"],["S"],["T"],["U"],["V"],["W"],["X"],["Y"],["Z"]]`), + val: addr([26][1]string{{"A"}, {"B"}, {"C"}, {"D"}, {"E"}, {"F"}, {"G"}, {"H"}, {"I"}, {"J"}, {"K"}, {"L"}, {"M"}, {"N"}, {"O"}, {"P"}, {"Q"}, {"R"}, {"S"}, {"T"}, {"U"}, {"V"}, {"W"}, {"X"}, {"Y"}, {"Z"}}), + new: func() any { return new([26][1]string) }, +}, { + name: "Bytes/Slice", + raw: []byte(`"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="`), + val: addr([]byte{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}), + new: func() any { return new([]byte) }, +}, { + name: "Bytes/Array", + raw: []byte(`"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="`), + val: addr([32]byte{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}), + new: func() any { return new([32]byte) }, + skipV1: true, +}, { + name: "Pointer", + raw: []byte("true"), + val: addr(addr(addr(addr(addr(addr(addr(addr(addr(addr(addr(true))))))))))), + new: func() any { return new(**********bool) }, +}, { + name: "TextArshal", + raw: []byte(`"method"`), + val: new(textArshaler), + new: func() any { return new(textArshaler) }, +}, { + name: "JSONArshalV1", + raw: []byte(`"method"`), + val: new(jsonArshalerV1), + new: func() any { return new(jsonArshalerV1) }, +}, { + name: "JSONArshalV2", + raw: []byte(`"method"`), + val: new(jsonArshalerV2), + new: func() any { return new(jsonArshalerV2) }, + skipV1: true, +}, { + /* TODO(https://go.dev/issue/71631): Re-enable this test case. + name: "Duration", + raw: []byte(`"1h1m1s"`), + val: addr(time.Hour + time.Minute + time.Second), + new: func() any { return new(time.Duration) }, + skipV1: true, + }, { */ + name: "Time", + raw: []byte(`"2006-01-02T22:04:05Z"`), + val: addr(time.Unix(1136239445, 0).UTC()), + new: func() any { return new(time.Time) }, +}} + +type textArshaler struct{ _ [4]int } + +func (textArshaler) MarshalText() ([]byte, error) { + return []byte("method"), nil +} +func (*textArshaler) UnmarshalText(b []byte) error { + if string(b) != "method" { + return fmt.Errorf("UnmarshalText: got %q, want %q", b, "method") + } + return nil +} + +type jsonArshalerV1 struct{ _ [4]int } + +func (jsonArshalerV1) MarshalJSON() ([]byte, error) { + return []byte(`"method"`), nil +} +func (*jsonArshalerV1) UnmarshalJSON(b []byte) error { + if string(b) != `"method"` { + return fmt.Errorf("UnmarshalJSON: got %q, want %q", b, `"method"`) + } + return nil +} + +type jsonArshalerV2 struct{ _ [4]int } + +func (jsonArshalerV2) MarshalJSONTo(enc *jsontext.Encoder) error { + return enc.WriteToken(jsontext.String("method")) +} +func (*jsonArshalerV2) UnmarshalJSONFrom(dec *jsontext.Decoder) error { + b, err := dec.ReadValue() + if string(b) != `"method"` { + return fmt.Errorf("UnmarshalJSONFrom: got %q, want %q", b, `"method"`) + } + return err +} + +func TestBenchmarkUnmarshal(t *testing.T) { runUnmarshal(t) } +func BenchmarkUnmarshal(b *testing.B) { runUnmarshal(b) } + +func runUnmarshal(tb testing.TB) { + for _, tt := range arshalTestdata { + if tt.skipV1 && strings.HasPrefix(benchVersion, "v1") { + runTestOrBench(tb, tt.name, 0, func(tb testing.TB) { tb.Skip("not supported in v1") }) + return + } + + // Setup the unmarshal operation. + var val any + run := func(tb testing.TB) { + val = tt.new() + if err := jsonFuncs.unmarshal(tt.raw, val); err != nil { + tb.Fatalf("Unmarshal error: %v", err) + } + } + + // Verify the results. + if _, ok := tb.(*testing.T); ok { + run0 := run + run = func(tb testing.TB) { + run0(tb) + if !reflect.DeepEqual(val, tt.val) { + tb.Fatalf("Unmarshal output mismatch:\ngot %v\nwant %v", val, tt.val) + } + } + } + + runTestOrBench(tb, tt.name, len64(tt.raw), run) + } +} + +func TestBenchmarkMarshal(t *testing.T) { runMarshal(t) } +func BenchmarkMarshal(b *testing.B) { runMarshal(b) } + +func runMarshal(tb testing.TB) { + for _, tt := range arshalTestdata { + if tt.skipV1 && strings.HasPrefix(benchVersion, "v1") { + runTestOrBench(tb, tt.name, 0, func(tb testing.TB) { tb.Skip("not supported in v1") }) + return + } + + // Setup the marshal operation. + var raw []byte + run := func(tb testing.TB) { + var err error + raw, err = jsonFuncs.marshal(tt.val) + if err != nil { + tb.Fatalf("Marshal error: %v", err) + } + } + + // Verify the results. + if _, ok := tb.(*testing.T); ok { + run0 := run + run = func(tb testing.TB) { + run0(tb) + if !bytes.Equal(raw, tt.raw) { + // Map marshaling in v2 is non-deterministic. + byteHistogram := func(b []byte) (h [256]int) { + for _, c := range b { + h[c]++ + } + return h + } + if !(strings.HasPrefix(tt.name, "Map/") && byteHistogram(raw) == byteHistogram(tt.raw)) { + tb.Fatalf("Marshal output mismatch:\ngot %s\nwant %s", raw, tt.raw) + } + } + } + } + + runTestOrBench(tb, tt.name, len64(tt.raw), run) + } +} + +func TestBenchmarkTestdata(t *testing.T) { runAllTestdata(t) } +func BenchmarkTestdata(b *testing.B) { runAllTestdata(b) } + +func runAllTestdata(tb testing.TB) { + for _, td := range jsontest.Data { + for _, arshalName := range []string{"Marshal", "Unmarshal"} { + for _, typeName := range []string{"Concrete", "Interface"} { + newValue := func() any { return new(any) } + if typeName == "Concrete" { + if td.New == nil { + continue + } + newValue = td.New + } + value := mustUnmarshalValue(tb, td.Data(), newValue) + name := path.Join(td.Name, arshalName, typeName) + runTestOrBench(tb, name, int64(len(td.Data())), func(tb testing.TB) { + runArshal(tb, arshalName, newValue, td.Data(), value) + }) + } + } + + tokens := mustDecodeTokens(tb, td.Data()) + buffer := make([]byte, 0, 2*len(td.Data())) + for _, codeName := range []string{"Encode", "Decode"} { + for _, typeName := range []string{"Token", "Value"} { + for _, modeName := range []string{"Streaming", "Buffered"} { + name := path.Join(td.Name, codeName, typeName, modeName) + runTestOrBench(tb, name, int64(len(td.Data())), func(tb testing.TB) { + runCode(tb, codeName, typeName, modeName, buffer, td.Data(), tokens) + }) + } + } + } + } +} + +func mustUnmarshalValue(t testing.TB, data []byte, newValue func() any) (value any) { + value = newValue() + if err := jsonv2.Unmarshal(data, value); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + return value +} + +func runArshal(t testing.TB, arshalName string, newValue func() any, data []byte, value any) { + switch arshalName { + case "Marshal": + if _, err := jsonFuncs.marshal(value); err != nil { + t.Fatalf("Marshal error: %v", err) + } + case "Unmarshal": + if err := jsonFuncs.unmarshal(data, newValue()); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + } +} + +func mustDecodeTokens(t testing.TB, data []byte) []jsontext.Token { + var tokens []jsontext.Token + dec := jsontext.NewDecoder(bytes.NewReader(data)) + for { + tok, err := dec.ReadToken() + if err != nil { + if err == io.EOF { + break + } + t.Fatalf("Decoder.ReadToken error: %v", err) + } + + // Prefer exact representation for JSON strings and numbers + // since this more closely matches common use cases. + switch tok.Kind() { + case '"': + tokens = append(tokens, jsontext.String(tok.String())) + case '0': + tokens = append(tokens, jsontext.Float(tok.Float())) + default: + tokens = append(tokens, tok.Clone()) + } + } + return tokens +} + +func runCode(t testing.TB, codeName, typeName, modeName string, buffer, data []byte, tokens []jsontext.Token) { + switch codeName { + case "Encode": + runEncode(t, typeName, modeName, buffer, data, tokens) + case "Decode": + runDecode(t, typeName, modeName, buffer, data, tokens) + } +} + +func runEncode(t testing.TB, typeName, modeName string, buffer, data []byte, tokens []jsontext.Token) { + if strings.HasPrefix(benchVersion, "v1") { + switch { + case modeName == "Buffered": + t.Skip("no support for direct buffered output in v1; see https://go.dev/issue/7872") + case typeName == "Token": + t.Skip("no support for encoding tokens in v1; see https://go.dev/issue/40127") + } + } + + var w io.Writer + switch modeName { + case "Streaming": + w = bytesBuffer{bytes.NewBuffer(buffer[:0])} + case "Buffered": + w = bytes.NewBuffer(buffer[:0]) + } + switch typeName { + case "Token": + if err := jsonFuncs.encodeTokens(w, tokens); err != nil { + t.Fatalf("Encoder.WriteToken error: %v", err) + } + case "Value": + if err := jsonFuncs.encodeValue(w, data); err != nil { + t.Fatalf("Encoder.WriteValue error: %v", err) + } + } +} + +func runDecode(t testing.TB, typeName, modeName string, buffer, data []byte, tokens []jsontext.Token) { + if strings.HasPrefix(benchVersion, "v1") && modeName == "Buffered" { + t.Skip("no support for direct buffered input in v1; see https://go.dev/issue/11046") + } + + var r io.Reader + switch modeName { + case "Streaming": + r = bytesBuffer{bytes.NewBuffer(data)} + case "Buffered": + r = bytes.NewBuffer(data) + } + switch typeName { + case "Token": + if err := jsonFuncs.decodeTokens(r); err != nil { + t.Fatalf("Decoder.ReadToken error: %v", err) + } + case "Value": + if err := jsonFuncs.decodeValue(r); err != nil { + t.Fatalf("Decoder.ReadValue error: %v", err) + } + } +} + +var ws = strings.Repeat(" ", 4<<10) +var slowStreamingDecodeTestdata = []struct { + name string + data []byte +}{ + {"LargeString", []byte(`"` + strings.Repeat(" ", 4<<10) + `"`)}, + {"LargeNumber", []byte("0." + strings.Repeat("0", 4<<10))}, + {"LargeWhitespace/Null", []byte(ws + "null" + ws)}, + {"LargeWhitespace/Object", []byte(ws + "{" + ws + `"name1"` + ws + ":" + ws + `"value"` + ws + "," + ws + `"name2"` + ws + ":" + ws + `"value"` + ws + "}" + ws)}, + {"LargeWhitespace/Array", []byte(ws + "[" + ws + `"value"` + ws + "," + ws + `"value"` + ws + "]" + ws)}, +} + +func TestBenchmarkSlowStreamingDecode(t *testing.T) { runAllSlowStreamingDecode(t) } +func BenchmarkSlowStreamingDecode(b *testing.B) { runAllSlowStreamingDecode(b) } + +func runAllSlowStreamingDecode(tb testing.TB) { + for _, td := range slowStreamingDecodeTestdata { + for _, typeName := range []string{"Token", "Value"} { + name := path.Join(td.name, typeName) + runTestOrBench(tb, name, len64(td.data), func(tb testing.TB) { + runSlowStreamingDecode(tb, typeName, td.data) + }) + } + } +} + +// runSlowStreamingDecode tests a streaming Decoder operating on +// a slow io.Reader that only returns 1 byte at a time, +// which tends to exercise pathological behavior. +func runSlowStreamingDecode(t testing.TB, typeName string, data []byte) { + r := iotest.OneByteReader(bytes.NewReader(data)) + switch typeName { + case "Token": + if err := jsonFuncs.decodeTokens(r); err != nil { + t.Fatalf("Decoder.ReadToken error: %v", err) + } + case "Value": + if err := jsonFuncs.decodeValue(r); err != nil { + t.Fatalf("Decoder.ReadValue error: %v", err) + } + } +} + +func TestBenchmarkTextValue(t *testing.T) { runValue(t) } +func BenchmarkTextValue(b *testing.B) { runValue(b) } + +func runValue(tb testing.TB) { + if testing.Short() { + tb.Skip() // CitmCatalog is not loaded in short mode + } + var data []byte + for _, ts := range jsontest.Data { + if ts.Name == "CitmCatalog" { + data = ts.Data() + } + } + + runTestOrBench(tb, "IsValid", len64(data), func(tb testing.TB) { + jsontext.Value(data).IsValid() + }) + + methods := []struct { + name string + format func(*jsontext.Value, ...jsontext.Options) error + }{ + {"Compact", (*jsontext.Value).Compact}, + {"Indent", (*jsontext.Value).Indent}, + {"Canonicalize", (*jsontext.Value).Canonicalize}, + } + + var v jsontext.Value + for _, method := range methods { + runTestOrBench(tb, method.name, len64(data), func(tb testing.TB) { + v = append(v[:0], data...) // reset with original input + if err := method.format(&v); err != nil { + tb.Errorf("jsontext.Value.%v error: %v", method.name, err) + } + }) + v = append(v[:0], data...) + method.format(&v) + runTestOrBench(tb, method.name+"/Noop", len64(data), func(tb testing.TB) { + if err := method.format(&v); err != nil { + tb.Errorf("jsontext.Value.%v error: %v", method.name, err) + } + }) + } +} + +func runTestOrBench(tb testing.TB, name string, numBytes int64, run func(tb testing.TB)) { + switch tb := tb.(type) { + case *testing.T: + tb.Run(name, func(t *testing.T) { + run(t) + }) + case *testing.B: + tb.Run(name, func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + b.SetBytes(numBytes) + for range b.N { + run(b) + } + }) + } +} diff --git a/src/encoding/json/v2/doc.go b/src/encoding/json/v2/doc.go new file mode 100644 index 00000000000000..8179f8ab179504 --- /dev/null +++ b/src/encoding/json/v2/doc.go @@ -0,0 +1,269 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +// Package json implements semantic processing of JSON as specified in RFC 8259. +// JSON is a simple data interchange format that can represent +// primitive data types such as booleans, strings, and numbers, +// in addition to structured data types such as objects and arrays. +// +// This package (encoding/json/v2) is experimental, +// and not subject to the Go 1 compatibility promise. +// It only exists when building with the GOEXPERIMENT=jsonv2 environment variable set. +// Most users should use [encoding/json]. +// +// [Marshal] and [Unmarshal] encode and decode Go values +// to/from JSON text contained within a []byte. +// [MarshalWrite] and [UnmarshalRead] operate on JSON text +// by writing to or reading from an [io.Writer] or [io.Reader]. +// [MarshalEncode] and [UnmarshalDecode] operate on JSON text +// by encoding to or decoding from a [jsontext.Encoder] or [jsontext.Decoder]. +// [Options] may be passed to each of the marshal or unmarshal functions +// to configure the semantic behavior of marshaling and unmarshaling +// (i.e., alter how JSON data is understood as Go data and vice versa). +// [jsontext.Options] may also be passed to the marshal or unmarshal functions +// to configure the syntactic behavior of encoding or decoding. +// +// The data types of JSON are mapped to/from the data types of Go based on +// the closest logical equivalent between the two type systems. For example, +// a JSON boolean corresponds with a Go bool, +// a JSON string corresponds with a Go string, +// a JSON number corresponds with a Go int, uint or float, +// a JSON array corresponds with a Go slice or array, and +// a JSON object corresponds with a Go struct or map. +// See the documentation on [Marshal] and [Unmarshal] for a comprehensive list +// of how the JSON and Go type systems correspond. +// +// Arbitrary Go types can customize their JSON representation by implementing +// [Marshaler], [MarshalerTo], [Unmarshaler], or [UnmarshalerFrom]. +// This provides authors of Go types with control over how their types are +// serialized as JSON. Alternatively, users can implement functions that match +// [MarshalFunc], [MarshalToFunc], [UnmarshalFunc], or [UnmarshalFromFunc] +// to specify the JSON representation for arbitrary types. +// This provides callers of JSON functionality with control over +// how any arbitrary type is serialized as JSON. +// +// # JSON Representation of Go structs +// +// A Go struct is naturally represented as a JSON object, +// where each Go struct field corresponds with a JSON object member. +// When marshaling, all Go struct fields are recursively encoded in depth-first +// order as JSON object members except those that are ignored or omitted. +// When unmarshaling, JSON object members are recursively decoded +// into the corresponding Go struct fields. +// Object members that do not match any struct fields, +// also known as “unknown members”, are ignored by default or rejected +// if [RejectUnknownMembers] is specified. +// +// The representation of each struct field can be customized in the +// "json" struct field tag, where the tag is a comma separated list of options. +// As a special case, if the entire tag is `json:"-"`, +// then the field is ignored with regard to its JSON representation. +// Some options also have equivalent behavior controlled by a caller-specified [Options]. +// Field-specified options take precedence over caller-specified options. +// +// The first option is the JSON object name override for the Go struct field. +// If the name is not specified, then the Go struct field name +// is used as the JSON object name. JSON names containing commas or quotes, +// or names identical to "" or "-", can be specified using +// a single-quoted string literal, where the syntax is identical to +// the Go grammar for a double-quoted string literal, +// but instead uses single quotes as the delimiters. +// By default, unmarshaling uses case-sensitive matching to identify +// the Go struct field associated with a JSON object name. +// +// After the name, the following tag options are supported: +// +// - omitzero: When marshaling, the "omitzero" option specifies that +// the struct field should be omitted if the field value is zero +// as determined by the "IsZero() bool" method if present, +// otherwise based on whether the field is the zero Go value. +// This option has no effect when unmarshaling. +// +// - omitempty: When marshaling, the "omitempty" option specifies that +// the struct field should be omitted if the field value would have been +// encoded as a JSON null, empty string, empty object, or empty array. +// This option has no effect when unmarshaling. +// +// - string: The "string" option specifies that [StringifyNumbers] +// be set when marshaling or unmarshaling a struct field value. +// This causes numeric types to be encoded as a JSON number +// within a JSON string, and to be decoded from a JSON string +// containing the JSON number without any surrounding whitespace. +// This extra level of encoding is often necessary since +// many JSON parsers cannot precisely represent 64-bit integers. +// +// - case: When unmarshaling, the "case" option specifies how +// JSON object names are matched with the JSON name for Go struct fields. +// The option is a key-value pair specified as "case:value" where +// the value must either be 'ignore' or 'strict'. +// The 'ignore' value specifies that matching is case-insensitive +// where dashes and underscores are also ignored. If multiple fields match, +// the first declared field in breadth-first order takes precedence. +// The 'strict' value specifies that matching is case-sensitive. +// This takes precedence over the [MatchCaseInsensitiveNames] option. +// +// - inline: The "inline" option specifies that +// the JSON representable content of this field type is to be promoted +// as if they were specified in the parent struct. +// It is the JSON equivalent of Go struct embedding. +// A Go embedded field is implicitly inlined unless an explicit JSON name +// is specified. The inlined field must be a Go struct +// (that does not implement any JSON methods), [jsontext.Value], +// map[~string]T, or an unnamed pointer to such types. When marshaling, +// inlined fields from a pointer type are omitted if it is nil. +// Inlined fields of type [jsontext.Value] and map[~string]T are called +// “inlined fallbacks” as they can represent all possible +// JSON object members not directly handled by the parent struct. +// Only one inlined fallback field may be specified in a struct, +// while many non-fallback fields may be specified. This option +// must not be specified with any other option (including the JSON name). +// +// - unknown: The "unknown" option is a specialized variant +// of the inlined fallback to indicate that this Go struct field +// contains any number of unknown JSON object members. The field type must +// be a [jsontext.Value], map[~string]T, or an unnamed pointer to such types. +// If [DiscardUnknownMembers] is specified when marshaling, +// the contents of this field are ignored. +// If [RejectUnknownMembers] is specified when unmarshaling, +// any unknown object members are rejected regardless of whether +// an inlined fallback with the "unknown" option exists. This option +// must not be specified with any other option (including the JSON name). +// +// - format: The "format" option specifies a format flag +// used to specialize the formatting of the field value. +// The option is a key-value pair specified as "format:value" where +// the value must be either a literal consisting of letters and numbers +// (e.g., "format:RFC3339") or a single-quoted string literal +// (e.g., "format:'2006-01-02'"). The interpretation of the format flag +// is determined by the struct field type. +// +// The "omitzero" and "omitempty" options are mostly semantically identical. +// The former is defined in terms of the Go type system, +// while the latter in terms of the JSON type system. +// Consequently they behave differently in some circumstances. +// For example, only a nil slice or map is omitted under "omitzero", while +// an empty slice or map is omitted under "omitempty" regardless of nilness. +// The "omitzero" option is useful for types with a well-defined zero value +// (e.g., [net/netip.Addr]) or have an IsZero method (e.g., [time.Time.IsZero]). +// +// Every Go struct corresponds to a list of JSON representable fields +// which is constructed by performing a breadth-first search over +// all struct fields (excluding unexported or ignored fields), +// where the search recursively descends into inlined structs. +// The set of non-inlined fields in a struct must have unique JSON names. +// If multiple fields all have the same JSON name, then the one +// at shallowest depth takes precedence and the other fields at deeper depths +// are excluded from the list of JSON representable fields. +// If multiple fields at the shallowest depth have the same JSON name, +// but exactly one is explicitly tagged with a JSON name, +// then that field takes precedence and all others are excluded from the list. +// This is analogous to Go visibility rules for struct field selection +// with embedded struct types. +// +// Marshaling or unmarshaling a non-empty struct +// without any JSON representable fields results in a [SemanticError]. +// Unexported fields must not have any `json` tags except for `json:"-"`. +// +// # Security Considerations +// +// JSON is frequently used as a data interchange format to communicate +// between different systems, possibly implemented in different languages. +// For interoperability and security reasons, it is important that +// all implementations agree upon the semantic meaning of the data. +// +// [For example, suppose we have two micro-services.] +// The first service is responsible for authenticating a JSON request, +// while the second service is responsible for executing the request +// (having assumed that the prior service authenticated the request). +// If an attacker were able to maliciously craft a JSON request such that +// both services believe that the same request is from different users, +// it could bypass the authenticator with valid credentials for one user, +// but maliciously perform an action on behalf of a different user. +// +// According to RFC 8259, there unfortunately exist many JSON texts +// that are syntactically valid but semantically ambiguous. +// For example, the standard does not define how to interpret duplicate +// names within an object. +// +// The v1 [encoding/json] and [encoding/json/v2] packages +// interpret some inputs in different ways. In particular: +// +// - The standard specifies that JSON must be encoded using UTF-8. +// By default, v1 replaces invalid bytes of UTF-8 in JSON strings +// with the Unicode replacement character, +// while v2 rejects inputs with invalid UTF-8. +// To change the default, specify the [jsontext.AllowInvalidUTF8] option. +// The replacement of invalid UTF-8 is a form of data corruption +// that alters the precise meaning of strings. +// +// - The standard does not specify a particular behavior when +// duplicate names are encountered within a JSON object, +// which means that different implementations may behave differently. +// By default, v1 allows for the presence of duplicate names, +// while v2 rejects duplicate names. +// To change the default, specify the [jsontext.AllowDuplicateNames] option. +// If allowed, object members are processed in the order they are observed, +// meaning that later values will replace or be merged into prior values, +// depending on the Go value type. +// +// - The standard defines a JSON object as an unordered collection of name/value pairs. +// While ordering can be observed through the underlying [jsontext] API, +// both v1 and v2 generally avoid exposing the ordering. +// No application should semantically depend on the order of object members. +// Allowing duplicate names is a vector through which ordering of members +// can accidentally be observed and depended upon. +// +// - The standard suggests that JSON object names are typically compared +// based on equality of the sequence of Unicode code points, +// which implies that comparing names is often case-sensitive. +// When unmarshaling a JSON object into a Go struct, +// by default, v1 uses a (loose) case-insensitive match on the name, +// while v2 uses a (strict) case-sensitive match on the name. +// To change the default, specify the [MatchCaseInsensitiveNames] option. +// The use of case-insensitive matching provides another vector through +// which duplicate names can occur. Allowing case-insensitive matching +// means that v1 or v2 might interpret JSON objects differently from most +// other JSON implementations (which typically use a case-sensitive match). +// +// - The standard does not specify a particular behavior when +// an unknown name in a JSON object is encountered. +// When unmarshaling a JSON object into a Go struct, by default +// both v1 and v2 ignore unknown names and their corresponding values. +// To change the default, specify the [RejectUnknownMembers] option. +// +// - The standard suggests that implementations may use a float64 +// to represent a JSON number. Consequently, large JSON integers +// may lose precision when stored as a floating-point type. +// Both v1 and v2 correctly preserve precision when marshaling and +// unmarshaling a concrete integer type. However, even if v1 and v2 +// preserve precision for concrete types, other JSON implementations +// may not be able to preserve precision for outputs produced by v1 or v2. +// The `string` tag option can be used to specify that an integer type +// is to be quoted within a JSON string to avoid loss of precision. +// Furthermore, v1 and v2 may still lose precision when unmarshaling +// into an any interface value, where unmarshal uses a float64 +// by default to represent a JSON number. +// To change the default, specify the [WithUnmarshalers] option +// with a custom unmarshaler that pre-populates the interface value +// with a concrete Go type that can preserve precision. +// +// RFC 8785 specifies a canonical form for any JSON text, +// which explicitly defines specific behaviors that RFC 8259 leaves undefined. +// In theory, if a text can successfully [jsontext.Value.Canonicalize] +// without changing the semantic meaning of the data, then it provides a +// greater degree of confidence that the data is more secure and interoperable. +// +// The v2 API generally chooses more secure defaults than v1, +// but care should still be taken with large integers or unknown members. +// +// [For example, suppose we have two micro-services.]: https://www.youtube.com/watch?v=avilmOcHKHE&t=1057s +package json + +// requireKeyedLiterals can be embedded in a struct to require keyed literals. +type requireKeyedLiterals struct{} + +// nonComparable can be embedded in a struct to prevent comparability. +type nonComparable [0]func() diff --git a/src/encoding/json/v2/errors.go b/src/encoding/json/v2/errors.go new file mode 100644 index 00000000000000..4895386fe2ca37 --- /dev/null +++ b/src/encoding/json/v2/errors.go @@ -0,0 +1,445 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "cmp" + "errors" + "fmt" + "io" + "reflect" + "strconv" + "strings" + "sync" + + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonopts" + "encoding/json/internal/jsonwire" + "encoding/json/jsontext" +) + +// ErrUnknownName indicates that a JSON object member could not be +// unmarshaled because the name is not known to the target Go struct. +// This error is directly wrapped within a [SemanticError] when produced. +// +// The name of an unknown JSON object member can be extracted as: +// +// err := ... +// serr, ok := errors.AsType[*json.SemanticError](err) +// if ok && serr.Err == json.ErrUnknownName { +// ptr := serr.JSONPointer // JSON pointer to unknown name +// name := ptr.LastToken() // unknown name itself +// ... +// } +// +// This error is only returned if [RejectUnknownMembers] is true. +var ErrUnknownName = errors.New("unknown object member name") + +const errorPrefix = "json: " + +func isSemanticError(err error) bool { + _, ok := err.(*SemanticError) + return ok +} + +func isSyntacticError(err error) bool { + _, ok := err.(*jsontext.SyntacticError) + return ok +} + +// isFatalError reports whether this error must terminate asharling. +// All errors are considered fatal unless operating under +// [jsonflags.ReportErrorsWithLegacySemantics] in which case only +// syntactic errors and I/O errors are considered fatal. +func isFatalError(err error, flags jsonflags.Flags) bool { + return !flags.Get(jsonflags.ReportErrorsWithLegacySemantics) || + isSyntacticError(err) || export.IsIOError(err) +} + +// SemanticError describes an error determining the meaning +// of JSON data as Go data or vice-versa. +// +// If a [Marshaler], [MarshalerTo], [Unmarshaler], or [UnmarshalerFrom] method +// returns a SemanticError when called by the [json] package, +// then the ByteOffset, JSONPointer, and GoType fields are automatically +// populated by the calling context if they are the zero value. +// +// The contents of this error as produced by this package may change over time. +type SemanticError struct { + requireKeyedLiterals + nonComparable + + action string // either "marshal" or "unmarshal" + + // ByteOffset indicates that an error occurred after this byte offset. + ByteOffset int64 + // JSONPointer indicates that an error occurred within this JSON value + // as indicated using the JSON Pointer notation (see RFC 6901). + JSONPointer jsontext.Pointer + + // JSONKind is the JSON kind that could not be handled. + JSONKind jsontext.Kind // may be zero if unknown + // JSONValue is the JSON number or string that could not be unmarshaled. + // It is not populated during marshaling. + JSONValue jsontext.Value // may be nil if irrelevant or unknown + // GoType is the Go type that could not be handled. + GoType reflect.Type // may be nil if unknown + + // Err is the underlying error. + Err error // may be nil +} + +// coder is implemented by [jsontext.Encoder] or [jsontext.Decoder]. +type coder interface { + StackPointer() jsontext.Pointer + Options() Options +} + +// newInvalidFormatError wraps err in a SemanticError because +// the current type t cannot handle the provided options format. +// This error must be called before producing or consuming the next value. +// +// If [jsonflags.ReportErrorsWithLegacySemantics] is specified, +// then this automatically skips the next value when unmarshaling +// to ensure that the value is fully consumed. +func newInvalidFormatError(c coder, t reflect.Type) error { + err := fmt.Errorf("invalid format flag %q", c.Options().(*jsonopts.Struct).Format) + switch c := c.(type) { + case *jsontext.Encoder: + err = newMarshalErrorBefore(c, t, err) + case *jsontext.Decoder: + err = newUnmarshalErrorBeforeWithSkipping(c, t, err) + } + return err +} + +// newMarshalErrorBefore wraps err in a SemanticError assuming that e +// is positioned right before the next token or value, which causes an error. +func newMarshalErrorBefore(e *jsontext.Encoder, t reflect.Type, err error) error { + return &SemanticError{action: "marshal", GoType: t, Err: toUnexpectedEOF(err), + ByteOffset: e.OutputOffset() + int64(export.Encoder(e).CountNextDelimWhitespace()), + JSONPointer: jsontext.Pointer(export.Encoder(e).AppendStackPointer(nil, +1))} +} + +// newUnmarshalErrorBefore wraps err in a SemanticError assuming that d +// is positioned right before the next token or value, which causes an error. +// It does not record the next JSON kind as this error is used to indicate +// the receiving Go value is invalid to unmarshal into (and not a JSON error). +// However, if [jsonflags.ReportErrorsWithLegacySemantics] is specified, +// then it does record the next JSON kind for historical reporting reasons. +func newUnmarshalErrorBefore(d *jsontext.Decoder, t reflect.Type, err error) error { + var k jsontext.Kind + if export.Decoder(d).Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + k = d.PeekKind() + } + return &SemanticError{action: "unmarshal", GoType: t, Err: toUnexpectedEOF(err), + ByteOffset: d.InputOffset() + int64(export.Decoder(d).CountNextDelimWhitespace()), + JSONPointer: jsontext.Pointer(export.Decoder(d).AppendStackPointer(nil, +1)), + JSONKind: k} +} + +// newUnmarshalErrorBeforeWithSkipping is like [newUnmarshalErrorBefore], +// but automatically skips the next value if +// [jsonflags.ReportErrorsWithLegacySemantics] is specified. +func newUnmarshalErrorBeforeWithSkipping(d *jsontext.Decoder, t reflect.Type, err error) error { + err = newUnmarshalErrorBefore(d, t, err) + if export.Decoder(d).Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + if err2 := export.Decoder(d).SkipValue(); err2 != nil { + return err2 + } + } + return err +} + +// newUnmarshalErrorAfter wraps err in a SemanticError assuming that d +// is positioned right after the previous token or value, which caused an error. +func newUnmarshalErrorAfter(d *jsontext.Decoder, t reflect.Type, err error) error { + tokOrVal := export.Decoder(d).PreviousTokenOrValue() + return &SemanticError{action: "unmarshal", GoType: t, Err: toUnexpectedEOF(err), + ByteOffset: d.InputOffset() - int64(len(tokOrVal)), + JSONPointer: jsontext.Pointer(export.Decoder(d).AppendStackPointer(nil, -1)), + JSONKind: jsontext.Value(tokOrVal).Kind()} +} + +// newUnmarshalErrorAfter wraps err in a SemanticError assuming that d +// is positioned right after the previous token or value, which caused an error. +// It also stores a copy of the last JSON value if it is a string or number. +func newUnmarshalErrorAfterWithValue(d *jsontext.Decoder, t reflect.Type, err error) error { + serr := newUnmarshalErrorAfter(d, t, err).(*SemanticError) + if serr.JSONKind == '"' || serr.JSONKind == '0' { + serr.JSONValue = jsontext.Value(export.Decoder(d).PreviousTokenOrValue()).Clone() + } + return serr +} + +// newUnmarshalErrorAfterWithSkipping is like [newUnmarshalErrorAfter], +// but automatically skips the remainder of the current value if +// [jsonflags.ReportErrorsWithLegacySemantics] is specified. +func newUnmarshalErrorAfterWithSkipping(d *jsontext.Decoder, t reflect.Type, err error) error { + err = newUnmarshalErrorAfter(d, t, err) + if export.Decoder(d).Flags.Get(jsonflags.ReportErrorsWithLegacySemantics) { + if err2 := export.Decoder(d).SkipValueRemainder(); err2 != nil { + return err2 + } + } + return err +} + +// newSemanticErrorWithPosition wraps err in a SemanticError assuming that +// the error occurred at the provided depth, and length. +// If err is already a SemanticError, then position information is only +// injected if it is currently unpopulated. +// +// If the position is unpopulated, it is ambiguous where the error occurred +// in the user code, whether it was before or after the current position. +// For the byte offset, we assume that the error occurred before the last read +// token or value when decoding, or before the next value when encoding. +// For the JSON pointer, we point to the parent object or array unless +// we can be certain that it happened with an object member. +// +// This is used to annotate errors returned by user-provided +// v2 MarshalJSON or UnmarshalJSON methods or functions. +func newSemanticErrorWithPosition(c coder, t reflect.Type, prevDepth int, prevLength int64, err error) error { + serr, _ := err.(*SemanticError) + if serr == nil { + serr = &SemanticError{Err: err} + } + serr.Err = toUnexpectedEOF(serr.Err) + var currDepth int + var currLength int64 + var coderState interface{ AppendStackPointer([]byte, int) []byte } + var offset int64 + switch c := c.(type) { + case *jsontext.Encoder: + e := export.Encoder(c) + serr.action = cmp.Or(serr.action, "marshal") + currDepth, currLength = e.Tokens.DepthLength() + offset = c.OutputOffset() + int64(export.Encoder(c).CountNextDelimWhitespace()) + coderState = e + case *jsontext.Decoder: + d := export.Decoder(c) + serr.action = cmp.Or(serr.action, "unmarshal") + currDepth, currLength = d.Tokens.DepthLength() + tokOrVal := d.PreviousTokenOrValue() + offset = c.InputOffset() - int64(len(tokOrVal)) + if (prevDepth == currDepth && prevLength == currLength) || len(tokOrVal) == 0 { + // If no Read method was called in the user-defined method or + // if the Peek method was called, then use the offset of the next value. + offset = c.InputOffset() + int64(export.Decoder(c).CountNextDelimWhitespace()) + } + coderState = d + } + serr.ByteOffset = cmp.Or(serr.ByteOffset, offset) + if serr.JSONPointer == "" { + where := 0 // default to ambiguous positioning + switch { + case prevDepth == currDepth && prevLength+0 == currLength: + where = +1 + case prevDepth == currDepth && prevLength+1 == currLength: + where = -1 + } + serr.JSONPointer = jsontext.Pointer(coderState.AppendStackPointer(nil, where)) + } + serr.GoType = cmp.Or(serr.GoType, t) + return serr +} + +// collapseSemanticErrors collapses double SemanticErrors at the outer levels +// into a single SemanticError by preserving the inner error, +// but prepending the ByteOffset and JSONPointer with the outer error. +// +// For example: +// +// collapseSemanticErrors(&SemanticError{ +// ByteOffset: len64(`[0,{"alpha":[0,1,`), +// JSONPointer: "/1/alpha/2", +// GoType: reflect.TypeFor[outerType](), +// Err: &SemanticError{ +// ByteOffset: len64(`{"foo":"bar","fizz":[0,`), +// JSONPointer: "/fizz/1", +// GoType: reflect.TypeFor[innerType](), +// Err: ..., +// }, +// }) +// +// results in: +// +// &SemanticError{ +// ByteOffset: len64(`[0,{"alpha":[0,1,`) + len64(`{"foo":"bar","fizz":[0,`), +// JSONPointer: "/1/alpha/2" + "/fizz/1", +// GoType: reflect.TypeFor[innerType](), +// Err: ..., +// } +// +// This is used to annotate errors returned by user-provided +// v1 MarshalJSON or UnmarshalJSON methods with precise position information +// if they themselves happened to return a SemanticError. +// Since MarshalJSON and UnmarshalJSON are not operating on the root JSON value, +// their positioning must be relative to the nested JSON value +// returned by UnmarshalJSON or passed to MarshalJSON. +// Therefore, we can construct an absolute position by concatenating +// the outer with the inner positions. +// +// Note that we do not use collapseSemanticErrors with user-provided functions +// that take in an [jsontext.Encoder] or [jsontext.Decoder] since they contain +// methods to report position relative to the root JSON value. +// We assume user-constructed errors are correctly precise about position. +func collapseSemanticErrors(err error) error { + if serr1, ok := err.(*SemanticError); ok { + if serr2, ok := serr1.Err.(*SemanticError); ok { + serr2.ByteOffset = serr1.ByteOffset + serr2.ByteOffset + serr2.JSONPointer = serr1.JSONPointer + serr2.JSONPointer + *serr1 = *serr2 + } + } + return err +} + +// errorModalVerb is a modal verb like "cannot" or "unable to". +// +// Once per process, Hyrum-proof the error message by deliberately +// switching between equivalent renderings of the same error message. +// The randomization is tied to the Hyrum-proofing already applied +// on map iteration in Go. +var errorModalVerb = sync.OnceValue(func() string { + for phrase := range map[string]struct{}{"cannot": {}, "unable to": {}} { + return phrase // use whichever phrase we get in the first iteration + } + return "" +}) + +func (e *SemanticError) Error() string { + var sb strings.Builder + sb.WriteString(errorPrefix) + sb.WriteString(errorModalVerb()) + + // Format action. + var preposition string + switch e.action { + case "marshal": + sb.WriteString(" marshal") + preposition = " from" + case "unmarshal": + sb.WriteString(" unmarshal") + preposition = " into" + default: + sb.WriteString(" handle") + preposition = " with" + } + + // Format JSON kind. + switch e.JSONKind { + case 'n': + sb.WriteString(" JSON null") + case 'f', 't': + sb.WriteString(" JSON boolean") + case '"': + sb.WriteString(" JSON string") + case '0': + sb.WriteString(" JSON number") + case '{', '}': + sb.WriteString(" JSON object") + case '[', ']': + sb.WriteString(" JSON array") + default: + if e.action == "" { + preposition = "" + } + } + if len(e.JSONValue) > 0 && len(e.JSONValue) < 100 { + sb.WriteByte(' ') + sb.Write(e.JSONValue) + } + + // Format Go type. + if e.GoType != nil { + typeString := e.GoType.String() + if len(typeString) > 100 { + // An excessively long type string most likely occurs for + // an anonymous struct declaration with many fields. + // Reduce the noise by just printing the kind, + // and optionally prepending it with the package name + // if the struct happens to include an unexported field. + typeString = e.GoType.Kind().String() + if e.GoType.Kind() == reflect.Struct && e.GoType.Name() == "" { + for i := range e.GoType.NumField() { + if pkgPath := e.GoType.Field(i).PkgPath; pkgPath != "" { + typeString = pkgPath[strings.LastIndexByte(pkgPath, '/')+len("/"):] + ".struct" + break + } + } + } + } + sb.WriteString(preposition) + sb.WriteString(" Go ") + sb.WriteString(typeString) + } + + // Special handling for unknown names. + if e.Err == ErrUnknownName { + sb.WriteString(": ") + sb.WriteString(ErrUnknownName.Error()) + sb.WriteString(" ") + sb.WriteString(strconv.Quote(e.JSONPointer.LastToken())) + if parent := e.JSONPointer.Parent(); parent != "" { + sb.WriteString(" within ") + sb.WriteString(strconv.Quote(jsonwire.TruncatePointer(string(parent), 100))) + } + return sb.String() + } + + // Format where. + // Avoid printing if it overlaps with a wrapped SyntacticError. + switch serr, _ := e.Err.(*jsontext.SyntacticError); { + case e.JSONPointer != "": + if serr == nil || !e.JSONPointer.Contains(serr.JSONPointer) { + sb.WriteString(" within ") + sb.WriteString(strconv.Quote(jsonwire.TruncatePointer(string(e.JSONPointer), 100))) + } + case e.ByteOffset > 0: + if serr == nil || !(e.ByteOffset <= serr.ByteOffset) { + sb.WriteString(" after offset ") + sb.WriteString(strconv.FormatInt(e.ByteOffset, 10)) + } + } + + // Format underlying error. + if e.Err != nil { + errString := e.Err.Error() + if isSyntacticError(e.Err) { + errString = strings.TrimPrefix(errString, "jsontext: ") + } + sb.WriteString(": ") + sb.WriteString(errString) + } + + return sb.String() +} + +func (e *SemanticError) Unwrap() error { + return e.Err +} + +func newDuplicateNameError(ptr jsontext.Pointer, quotedName []byte, offset int64) error { + if quotedName != nil { + name, _ := jsonwire.AppendUnquote(nil, quotedName) + ptr = ptr.AppendToken(string(name)) + } + return &jsontext.SyntacticError{ + ByteOffset: offset, + JSONPointer: ptr, + Err: jsontext.ErrDuplicateName, + } +} + +// toUnexpectedEOF converts [io.EOF] to [io.ErrUnexpectedEOF]. +func toUnexpectedEOF(err error) error { + if err == io.EOF { + return io.ErrUnexpectedEOF + } + return err +} diff --git a/src/encoding/json/v2/errors_test.go b/src/encoding/json/v2/errors_test.go new file mode 100644 index 00000000000000..76a7f2ae31d4c8 --- /dev/null +++ b/src/encoding/json/v2/errors_test.go @@ -0,0 +1,115 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "archive/tar" + "bytes" + "errors" + "io" + "strings" + "testing" + + "encoding/json/internal/jsonwire" + "encoding/json/jsontext" +) + +func TestSemanticError(t *testing.T) { + tests := []struct { + err error + want string + }{{ + err: &SemanticError{}, + want: `json: cannot handle`, + }, { + err: &SemanticError{JSONKind: 'n'}, + want: `json: cannot handle JSON null`, + }, { + err: &SemanticError{action: "unmarshal", JSONKind: 't'}, + want: `json: cannot unmarshal JSON boolean`, + }, { + err: &SemanticError{action: "unmarshal", JSONKind: 'x'}, + want: `json: cannot unmarshal`, // invalid token kinds are ignored + }, { + err: &SemanticError{action: "marshal", JSONKind: '"'}, + want: `json: cannot marshal JSON string`, + }, { + err: &SemanticError{GoType: T[bool]()}, + want: `json: cannot handle Go bool`, + }, { + err: &SemanticError{action: "marshal", GoType: T[int]()}, + want: `json: cannot marshal from Go int`, + }, { + err: &SemanticError{action: "unmarshal", GoType: T[uint]()}, + want: `json: cannot unmarshal into Go uint`, + }, { + err: &SemanticError{GoType: T[struct{ Alpha, Bravo, Charlie, Delta, Echo, Foxtrot, Golf, Hotel string }]()}, + want: `json: cannot handle Go struct`, + }, { + err: &SemanticError{GoType: T[struct{ Alpha, Bravo, Charlie, Delta, Echo, Foxtrot, Golf, Hotel, x string }]()}, + want: `json: cannot handle Go v2.struct`, + }, { + err: &SemanticError{JSONKind: '0', GoType: T[tar.Header]()}, + want: `json: cannot handle JSON number with Go tar.Header`, + }, { + err: &SemanticError{action: "unmarshal", JSONKind: '0', JSONValue: jsontext.Value(`1e1000`), GoType: T[int]()}, + want: `json: cannot unmarshal JSON number 1e1000 into Go int`, + }, { + err: &SemanticError{action: "marshal", JSONKind: '{', GoType: T[bytes.Buffer]()}, + want: `json: cannot marshal JSON object from Go bytes.Buffer`, + }, { + err: &SemanticError{action: "unmarshal", JSONKind: ']', GoType: T[strings.Reader]()}, + want: `json: cannot unmarshal JSON array into Go strings.Reader`, + }, { + err: &SemanticError{action: "unmarshal", JSONKind: '{', GoType: T[float64](), ByteOffset: 123}, + want: `json: cannot unmarshal JSON object into Go float64 after offset 123`, + }, { + err: &SemanticError{action: "marshal", JSONKind: 'f', GoType: T[complex128](), ByteOffset: 123, JSONPointer: "/foo/2/bar/3"}, + want: `json: cannot marshal JSON boolean from Go complex128 within "/foo/2/bar/3"`, + }, { + err: &SemanticError{action: "unmarshal", JSONKind: '}', GoType: T[io.Reader](), ByteOffset: 123, JSONPointer: "/foo/2/bar/3", Err: errors.New("some underlying error")}, + want: `json: cannot unmarshal JSON object into Go io.Reader within "/foo/2/bar/3": some underlying error`, + }, { + err: &SemanticError{Err: errors.New("some underlying error")}, + want: `json: cannot handle: some underlying error`, + }, { + err: &SemanticError{ByteOffset: 123}, + want: `json: cannot handle after offset 123`, + }, { + err: &SemanticError{JSONPointer: "/foo/2/bar/3"}, + want: `json: cannot handle within "/foo/2/bar/3"`, + }, { + err: &SemanticError{action: "unmarshal", JSONPointer: "/3", GoType: T[struct{ Fizz, Buzz string }](), Err: ErrUnknownName}, + want: `json: cannot unmarshal into Go struct { Fizz string; Buzz string }: unknown object member name "3"`, + }, { + err: &SemanticError{action: "unmarshal", JSONPointer: "/foo/2/bar/3", GoType: T[struct{ Foo string }](), Err: ErrUnknownName}, + want: `json: cannot unmarshal into Go struct { Foo string }: unknown object member name "3" within "/foo/2/bar"`, + }, { + err: &SemanticError{JSONPointer: "/foo/bar", ByteOffset: 16, GoType: T[string](), Err: &jsontext.SyntacticError{JSONPointer: "/foo/bar/baz", ByteOffset: 53, Err: jsonwire.ErrInvalidUTF8}}, + want: `json: cannot handle Go string: invalid UTF-8 within "/foo/bar/baz" after offset 53`, + }, { + err: &SemanticError{JSONPointer: "/fizz/bar", ByteOffset: 16, GoType: T[string](), Err: &jsontext.SyntacticError{JSONPointer: "/foo/bar/baz", ByteOffset: 53, Err: jsonwire.ErrInvalidUTF8}}, + want: `json: cannot handle Go string within "/fizz/bar": invalid UTF-8 within "/foo/bar/baz" after offset 53`, + }, { + err: &SemanticError{ByteOffset: 16, GoType: T[string](), Err: &jsontext.SyntacticError{JSONPointer: "/foo/bar/baz", ByteOffset: 53, Err: jsonwire.ErrInvalidUTF8}}, + want: `json: cannot handle Go string: invalid UTF-8 within "/foo/bar/baz" after offset 53`, + }, { + err: &SemanticError{ByteOffset: 85, GoType: T[string](), Err: &jsontext.SyntacticError{JSONPointer: "/foo/bar/baz", ByteOffset: 53, Err: jsonwire.ErrInvalidUTF8}}, + want: `json: cannot handle Go string after offset 85: invalid UTF-8 within "/foo/bar/baz" after offset 53`, + }} + + for _, tt := range tests { + got := tt.err.Error() + // Cleanup the error of non-deterministic rendering effects. + if strings.HasPrefix(got, errorPrefix+"unable to ") { + got = errorPrefix + "cannot " + strings.TrimPrefix(got, errorPrefix+"unable to ") + } + if got != tt.want { + t.Errorf("%#v.Error mismatch:\ngot %v\nwant %v", tt.err, got, tt.want) + } + } +} diff --git a/src/encoding/json/v2/example_orderedobject_test.go b/src/encoding/json/v2/example_orderedobject_test.go new file mode 100644 index 00000000000000..fc231325040190 --- /dev/null +++ b/src/encoding/json/v2/example_orderedobject_test.go @@ -0,0 +1,115 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json_test + +import ( + "fmt" + "log" + "reflect" + + "encoding/json/jsontext" + "encoding/json/v2" +) + +// OrderedObject is an ordered sequence of name/value members in a JSON object. +// +// RFC 8259 defines an object as an "unordered collection". +// JSON implementations need not make "ordering of object members visible" +// to applications nor will they agree on the semantic meaning of an object if +// "the names within an object are not unique". For maximum compatibility, +// applications should avoid relying on ordering or duplicity of object names. +type OrderedObject[V any] []ObjectMember[V] + +// ObjectMember is a JSON object member. +type ObjectMember[V any] struct { + Name string + Value V +} + +// MarshalJSONTo encodes obj as a JSON object into enc. +func (obj *OrderedObject[V]) MarshalJSONTo(enc *jsontext.Encoder) error { + if err := enc.WriteToken(jsontext.BeginObject); err != nil { + return err + } + for i := range *obj { + member := &(*obj)[i] + if err := json.MarshalEncode(enc, &member.Name); err != nil { + return err + } + if err := json.MarshalEncode(enc, &member.Value); err != nil { + return err + } + } + if err := enc.WriteToken(jsontext.EndObject); err != nil { + return err + } + return nil +} + +// UnmarshalJSONFrom decodes a JSON object from dec into obj. +func (obj *OrderedObject[V]) UnmarshalJSONFrom(dec *jsontext.Decoder) error { + if k := dec.PeekKind(); k != '{' { + // The [json] package automatically populates relevant fields + // in a [json.SemanticError] to provide additional context. + return &json.SemanticError{JSONKind: k} + } + if _, err := dec.ReadToken(); err != nil { + return err + } + for dec.PeekKind() != '}' { + *obj = append(*obj, ObjectMember[V]{}) + member := &(*obj)[len(*obj)-1] + if err := json.UnmarshalDecode(dec, &member.Name); err != nil { + return err + } + if err := json.UnmarshalDecode(dec, &member.Value); err != nil { + return err + } + } + if _, err := dec.ReadToken(); err != nil { + return err + } + return nil +} + +// The exact order of JSON object can be preserved through the use of a +// specialized type that implements [MarshalerTo] and [UnmarshalerFrom]. +func Example_orderedObject() { + // Round-trip marshal and unmarshal an ordered object. + // We expect the order and duplicity of JSON object members to be preserved. + // Specify jsontext.AllowDuplicateNames since this object contains "fizz" twice. + want := OrderedObject[string]{ + {"fizz", "buzz"}, + {"hello", "world"}, + {"fizz", "wuzz"}, + } + b, err := json.Marshal(&want, jsontext.AllowDuplicateNames(true)) + if err != nil { + log.Fatal(err) + } + var got OrderedObject[string] + err = json.Unmarshal(b, &got, jsontext.AllowDuplicateNames(true)) + if err != nil { + log.Fatal(err) + } + + // Sanity check. + if !reflect.DeepEqual(got, want) { + log.Fatalf("roundtrip mismatch: got %v, want %v", got, want) + } + + // Print the serialized JSON object. + (*jsontext.Value)(&b).Indent() // indent for readability + fmt.Println(string(b)) + + // Output: + // { + // "fizz": "buzz", + // "hello": "world", + // "fizz": "wuzz" + // } +} diff --git a/src/encoding/json/v2/example_test.go b/src/encoding/json/v2/example_test.go new file mode 100644 index 00000000000000..dc1f06674ce24a --- /dev/null +++ b/src/encoding/json/v2/example_test.go @@ -0,0 +1,695 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json_test + +import ( + "bytes" + "errors" + "fmt" + "log" + "math" + "net/http" + "net/netip" + "os" + "reflect" + "strconv" + "strings" + "sync/atomic" + "time" + + "encoding/json/jsontext" + "encoding/json/v2" +) + +// If a type implements [encoding.TextMarshaler] and/or [encoding.TextUnmarshaler], +// then the MarshalText and UnmarshalText methods are used to encode/decode +// the value to/from a JSON string. +func Example_textMarshal() { + // Round-trip marshal and unmarshal a hostname map where the netip.Addr type + // implements both encoding.TextMarshaler and encoding.TextUnmarshaler. + want := map[netip.Addr]string{ + netip.MustParseAddr("192.168.0.100"): "carbonite", + netip.MustParseAddr("192.168.0.101"): "obsidian", + netip.MustParseAddr("192.168.0.102"): "diamond", + } + b, err := json.Marshal(&want, json.Deterministic(true)) + if err != nil { + log.Fatal(err) + } + var got map[netip.Addr]string + err = json.Unmarshal(b, &got) + if err != nil { + log.Fatal(err) + } + + // Sanity check. + if !reflect.DeepEqual(got, want) { + log.Fatalf("roundtrip mismatch: got %v, want %v", got, want) + } + + // Print the serialized JSON object. + (*jsontext.Value)(&b).Indent() // indent for readability + fmt.Println(string(b)) + + // Output: + // { + // "192.168.0.100": "carbonite", + // "192.168.0.101": "obsidian", + // "192.168.0.102": "diamond" + // } +} + +// By default, JSON object names for Go struct fields are derived from +// the Go field name, but may be specified in the `json` tag. +// Due to JSON's heritage in JavaScript, the most common naming convention +// used for JSON object names is camelCase. +func Example_fieldNames() { + var value struct { + // This field is explicitly ignored with the special "-" name. + Ignored any `json:"-"` + // No JSON name is not provided, so the Go field name is used. + GoName any + // A JSON name is provided without any special characters. + JSONName any `json:"jsonName"` + // No JSON name is not provided, so the Go field name is used. + Option any `json:",case:ignore"` + // An empty JSON name specified using an single-quoted string literal. + Empty any `json:"''"` + // A dash JSON name specified using an single-quoted string literal. + Dash any `json:"'-'"` + // A comma JSON name specified using an single-quoted string literal. + Comma any `json:"','"` + // JSON name with quotes specified using a single-quoted string literal. + Quote any `json:"'\"\\''"` + // An unexported field is always ignored. + unexported any + } + + b, err := json.Marshal(value) + if err != nil { + log.Fatal(err) + } + (*jsontext.Value)(&b).Indent() // indent for readability + fmt.Println(string(b)) + + // Output: + // { + // "GoName": null, + // "jsonName": null, + // "Option": null, + // "": null, + // "-": null, + // ",": null, + // "\"'": null + // } +} + +// Unmarshal matches JSON object names with Go struct fields using +// a case-sensitive match, but can be configured to use a case-insensitive +// match with the "case:ignore" option. This permits unmarshaling from inputs +// that use naming conventions such as camelCase, snake_case, or kebab-case. +func Example_caseSensitivity() { + // JSON input using various naming conventions. + const input = `[ + {"firstname": true}, + {"firstName": true}, + {"FirstName": true}, + {"FIRSTNAME": true}, + {"first_name": true}, + {"FIRST_NAME": true}, + {"first-name": true}, + {"FIRST-NAME": true}, + {"unknown": true} + ]` + + // Without "case:ignore", Unmarshal looks for an exact match. + var caseStrict []struct { + X bool `json:"firstName"` + } + if err := json.Unmarshal([]byte(input), &caseStrict); err != nil { + log.Fatal(err) + } + fmt.Println(caseStrict) // exactly 1 match found + + // With "case:ignore", Unmarshal looks first for an exact match, + // then for a case-insensitive match if none found. + var caseIgnore []struct { + X bool `json:"firstName,case:ignore"` + } + if err := json.Unmarshal([]byte(input), &caseIgnore); err != nil { + log.Fatal(err) + } + fmt.Println(caseIgnore) // 8 matches found + + // Output: + // [{false} {true} {false} {false} {false} {false} {false} {false} {false}] + // [{true} {true} {true} {true} {true} {true} {true} {true} {false}] +} + +// Go struct fields can be omitted from the output depending on either +// the input Go value or the output JSON encoding of the value. +// The "omitzero" option omits a field if it is the zero Go value or +// implements a "IsZero() bool" method that reports true. +// The "omitempty" option omits a field if it encodes as an empty JSON value, +// which we define as a JSON null or empty JSON string, object, or array. +// In many cases, the behavior of "omitzero" and "omitempty" are equivalent. +// If both provide the desired effect, then using "omitzero" is preferred. +func Example_omitFields() { + type MyStruct struct { + Foo string `json:",omitzero"` + Bar []int `json:",omitempty"` + // Both "omitzero" and "omitempty" can be specified together, + // in which case the field is omitted if either would take effect. + // This omits the Baz field either if it is a nil pointer or + // if it would have encoded as an empty JSON object. + Baz *MyStruct `json:",omitzero,omitempty"` + } + + // Demonstrate behavior of "omitzero". + b, err := json.Marshal(struct { + Bool bool `json:",omitzero"` + Int int `json:",omitzero"` + String string `json:",omitzero"` + Time time.Time `json:",omitzero"` + Addr netip.Addr `json:",omitzero"` + Struct MyStruct `json:",omitzero"` + SliceNil []int `json:",omitzero"` + Slice []int `json:",omitzero"` + MapNil map[int]int `json:",omitzero"` + Map map[int]int `json:",omitzero"` + PointerNil *string `json:",omitzero"` + Pointer *string `json:",omitzero"` + InterfaceNil any `json:",omitzero"` + Interface any `json:",omitzero"` + }{ + // Bool is omitted since false is the zero value for a Go bool. + Bool: false, + // Int is omitted since 0 is the zero value for a Go int. + Int: 0, + // String is omitted since "" is the zero value for a Go string. + String: "", + // Time is omitted since time.Time.IsZero reports true. + Time: time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), + // Addr is omitted since netip.Addr{} is the zero value for a Go struct. + Addr: netip.Addr{}, + // Struct is NOT omitted since it is not the zero value for a Go struct. + Struct: MyStruct{Bar: []int{}, Baz: new(MyStruct)}, + // SliceNil is omitted since nil is the zero value for a Go slice. + SliceNil: nil, + // Slice is NOT omitted since []int{} is not the zero value for a Go slice. + Slice: []int{}, + // MapNil is omitted since nil is the zero value for a Go map. + MapNil: nil, + // Map is NOT omitted since map[int]int{} is not the zero value for a Go map. + Map: map[int]int{}, + // PointerNil is omitted since nil is the zero value for a Go pointer. + PointerNil: nil, + // Pointer is NOT omitted since new(string) is not the zero value for a Go pointer. + Pointer: new(string), + // InterfaceNil is omitted since nil is the zero value for a Go interface. + InterfaceNil: nil, + // Interface is NOT omitted since (*string)(nil) is not the zero value for a Go interface. + Interface: (*string)(nil), + }) + if err != nil { + log.Fatal(err) + } + (*jsontext.Value)(&b).Indent() // indent for readability + fmt.Println("OmitZero:", string(b)) // outputs "Struct", "Slice", "Map", "Pointer", and "Interface" + + // Demonstrate behavior of "omitempty". + b, err = json.Marshal(struct { + Bool bool `json:",omitempty"` + Int int `json:",omitempty"` + String string `json:",omitempty"` + Time time.Time `json:",omitempty"` + Addr netip.Addr `json:",omitempty"` + Struct MyStruct `json:",omitempty"` + Slice []int `json:",omitempty"` + Map map[int]int `json:",omitempty"` + PointerNil *string `json:",omitempty"` + Pointer *string `json:",omitempty"` + InterfaceNil any `json:",omitempty"` + Interface any `json:",omitempty"` + }{ + // Bool is NOT omitted since false is not an empty JSON value. + Bool: false, + // Int is NOT omitted since 0 is not a empty JSON value. + Int: 0, + // String is omitted since "" is an empty JSON string. + String: "", + // Time is NOT omitted since this encodes as a non-empty JSON string. + Time: time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), + // Addr is omitted since this encodes as an empty JSON string. + Addr: netip.Addr{}, + // Struct is omitted since {} is an empty JSON object. + Struct: MyStruct{Bar: []int{}, Baz: new(MyStruct)}, + // Slice is omitted since [] is an empty JSON array. + Slice: []int{}, + // Map is omitted since {} is an empty JSON object. + Map: map[int]int{}, + // PointerNil is omitted since null is an empty JSON value. + PointerNil: nil, + // Pointer is omitted since "" is an empty JSON string. + Pointer: new(string), + // InterfaceNil is omitted since null is an empty JSON value. + InterfaceNil: nil, + // Interface is omitted since null is an empty JSON value. + Interface: (*string)(nil), + }) + if err != nil { + log.Fatal(err) + } + (*jsontext.Value)(&b).Indent() // indent for readability + fmt.Println("OmitEmpty:", string(b)) // outputs "Bool", "Int", and "Time" + + // Output: + // OmitZero: { + // "Struct": {}, + // "Slice": [], + // "Map": {}, + // "Pointer": "", + // "Interface": null + // } + // OmitEmpty: { + // "Bool": false, + // "Int": 0, + // "Time": "0001-01-01T00:00:00Z" + // } +} + +// JSON objects can be inlined within a parent object similar to +// how Go structs can be embedded within a parent struct. +// The inlining rules are similar to those of Go embedding, +// but operates upon the JSON namespace. +func Example_inlinedFields() { + // Base is embedded within Container. + type Base struct { + // ID is promoted into the JSON object for Container. + ID string + // Type is ignored due to presence of Container.Type. + Type string + // Time cancels out with Container.Inlined.Time. + Time time.Time + } + // Other is embedded within Container. + type Other struct{ Cost float64 } + // Container embeds Base and Other. + type Container struct { + // Base is an embedded struct and is implicitly JSON inlined. + Base + // Type takes precedence over Base.Type. + Type int + // Inlined is a named Go field, but is explicitly JSON inlined. + Inlined struct { + // User is promoted into the JSON object for Container. + User string + // Time cancels out with Base.Time. + Time string + } `json:",inline"` + // ID does not conflict with Base.ID since the JSON name is different. + ID string `json:"uuid"` + // Other is not JSON inlined since it has an explicit JSON name. + Other `json:"other"` + } + + // Format an empty Container to show what fields are JSON serializable. + var input Container + b, err := json.Marshal(&input) + if err != nil { + log.Fatal(err) + } + (*jsontext.Value)(&b).Indent() // indent for readability + fmt.Println(string(b)) + + // Output: + // { + // "ID": "", + // "Type": 0, + // "User": "", + // "uuid": "", + // "other": { + // "Cost": 0 + // } + // } +} + +// Due to version skew, the set of JSON object members known at compile-time +// may differ from the set of members encountered at execution-time. +// As such, it may be useful to have finer grain handling of unknown members. +// This package supports preserving, rejecting, or discarding such members. +func Example_unknownMembers() { + const input = `{ + "Name": "Teal", + "Value": "#008080", + "WebSafe": false + }` + type Color struct { + Name string + Value string + + // Unknown is a Go struct field that holds unknown JSON object members. + // It is marked as having this behavior with the "unknown" tag option. + // + // The type may be a jsontext.Value or map[string]T. + Unknown jsontext.Value `json:",unknown"` + } + + // By default, unknown members are stored in a Go field marked as "unknown" + // or ignored if no such field exists. + var color Color + err := json.Unmarshal([]byte(input), &color) + if err != nil { + log.Fatal(err) + } + fmt.Println("Unknown members:", string(color.Unknown)) + + // Specifying RejectUnknownMembers causes Unmarshal + // to reject the presence of any unknown members. + err = json.Unmarshal([]byte(input), new(Color), json.RejectUnknownMembers(true)) + serr, ok := errors.AsType[*json.SemanticError](err) + if ok && serr.Err == json.ErrUnknownName { + fmt.Println("Unmarshal error:", serr.Err, strconv.Quote(serr.JSONPointer.LastToken())) + } + + // By default, Marshal preserves unknown members stored in + // a Go struct field marked as "unknown". + b, err := json.Marshal(color) + if err != nil { + log.Fatal(err) + } + fmt.Println("Output with unknown members: ", string(b)) + + // Specifying DiscardUnknownMembers causes Marshal + // to discard any unknown members. + b, err = json.Marshal(color, json.DiscardUnknownMembers(true)) + if err != nil { + log.Fatal(err) + } + fmt.Println("Output without unknown members:", string(b)) + + // Output: + // Unknown members: {"WebSafe":false} + // Unmarshal error: unknown object member name "WebSafe" + // Output with unknown members: {"Name":"Teal","Value":"#008080","WebSafe":false} + // Output without unknown members: {"Name":"Teal","Value":"#008080"} +} + +// The "format" tag option can be used to alter the formatting of certain types. +func Example_formatFlags() { + value := struct { + BytesBase64 []byte `json:",format:base64"` + BytesHex [8]byte `json:",format:hex"` + BytesArray []byte `json:",format:array"` + FloatNonFinite float64 `json:",format:nonfinite"` + MapEmitNull map[string]any `json:",format:emitnull"` + SliceEmitNull []any `json:",format:emitnull"` + TimeDateOnly time.Time `json:",format:'2006-01-02'"` + TimeUnixSec time.Time `json:",format:unix"` + DurationSecs time.Duration `json:",format:sec"` + DurationNanos time.Duration `json:",format:nano"` + DurationISO8601 time.Duration `json:",format:iso8601"` + }{ + BytesBase64: []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + BytesHex: [8]byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + BytesArray: []byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + FloatNonFinite: math.NaN(), + MapEmitNull: nil, + SliceEmitNull: nil, + TimeDateOnly: time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), + TimeUnixSec: time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), + DurationSecs: 12*time.Hour + 34*time.Minute + 56*time.Second + 7*time.Millisecond + 8*time.Microsecond + 9*time.Nanosecond, + DurationNanos: 12*time.Hour + 34*time.Minute + 56*time.Second + 7*time.Millisecond + 8*time.Microsecond + 9*time.Nanosecond, + DurationISO8601: 12*time.Hour + 34*time.Minute + 56*time.Second + 7*time.Millisecond + 8*time.Microsecond + 9*time.Nanosecond, + } + + b, err := json.Marshal(&value) + if err != nil { + log.Fatal(err) + } + (*jsontext.Value)(&b).Indent() // indent for readability + fmt.Println(string(b)) + + // Output: + // { + // "BytesBase64": "ASNFZ4mrze8=", + // "BytesHex": "0123456789abcdef", + // "BytesArray": [ + // 1, + // 35, + // 69, + // 103, + // 137, + // 171, + // 205, + // 239 + // ], + // "FloatNonFinite": "NaN", + // "MapEmitNull": null, + // "SliceEmitNull": null, + // "TimeDateOnly": "2000-01-01", + // "TimeUnixSec": 946684800, + // "DurationSecs": 45296.007008009, + // "DurationNanos": 45296007008009, + // "DurationISO8601": "PT12H34M56.007008009S" + // } +} + +// When implementing HTTP endpoints, it is common to be operating with an +// [io.Reader] and an [io.Writer]. The [MarshalWrite] and [UnmarshalRead] functions +// assist in operating on such input/output types. +// [UnmarshalRead] reads the entirety of the [io.Reader] to ensure that [io.EOF] +// is encountered without any unexpected bytes after the top-level JSON value. +func Example_serveHTTP() { + // Some global state maintained by the server. + var n int64 + + // The "add" endpoint accepts a POST request with a JSON object + // containing a number to atomically add to the server's global counter. + // It returns the updated value of the counter. + http.HandleFunc("/api/add", func(w http.ResponseWriter, r *http.Request) { + // Unmarshal the request from the client. + var val struct{ N int64 } + if err := json.UnmarshalRead(r.Body, &val); err != nil { + // Inability to unmarshal the input suggests a client-side problem. + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Marshal a response from the server. + val.N = atomic.AddInt64(&n, val.N) + if err := json.MarshalWrite(w, &val); err != nil { + // Inability to marshal the output suggests a server-side problem. + // This error is not always observable by the client since + // json.MarshalWrite may have already written to the output. + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + }) +} + +// Some Go types have a custom JSON representation where the implementation +// is delegated to some external package. Consequently, the "json" package +// will not know how to use that external implementation. +// For example, the [google.golang.org/protobuf/encoding/protojson] package +// implements JSON for all [google.golang.org/protobuf/proto.Message] types. +// [WithMarshalers] and [WithUnmarshalers] can be used +// to configure "json" and "protojson" to cooperate together. +func Example_protoJSON() { + // Let protoMessage be "google.golang.org/protobuf/proto".Message. + type protoMessage interface{ ProtoReflect() } + // Let foopbMyMessage be a concrete implementation of proto.Message. + type foopbMyMessage struct{ protoMessage } + // Let protojson be an import of "google.golang.org/protobuf/encoding/protojson". + var protojson struct { + Marshal func(protoMessage) ([]byte, error) + Unmarshal func([]byte, protoMessage) error + } + + // This value mixes both non-proto.Message types and proto.Message types. + // It should use the "json" package to handle non-proto.Message types and + // should use the "protojson" package to handle proto.Message types. + var value struct { + // GoStruct does not implement proto.Message and + // should use the default behavior of the "json" package. + GoStruct struct { + Name string + Age int + } + + // ProtoMessage implements proto.Message and + // should be handled using protojson.Marshal. + ProtoMessage *foopbMyMessage + } + + // Marshal using protojson.Marshal for proto.Message types. + b, err := json.Marshal(&value, + // Use protojson.Marshal as a type-specific marshaler. + json.WithMarshalers(json.MarshalFunc(protojson.Marshal))) + if err != nil { + log.Fatal(err) + } + + // Unmarshal using protojson.Unmarshal for proto.Message types. + err = json.Unmarshal(b, &value, + // Use protojson.Unmarshal as a type-specific unmarshaler. + json.WithUnmarshalers(json.UnmarshalFunc(protojson.Unmarshal))) + if err != nil { + log.Fatal(err) + } +} + +// Many error types are not serializable since they tend to be Go structs +// without any exported fields (e.g., errors constructed with [errors.New]). +// Some applications, may desire to marshal an error as a JSON string +// even if these errors cannot be unmarshaled. +func ExampleWithMarshalers_errors() { + // Response to serialize with some Go errors encountered. + response := []struct { + Result string `json:",omitzero"` + Error error `json:",omitzero"` + }{ + {Result: "Oranges are a good source of Vitamin C."}, + {Error: &strconv.NumError{Func: "ParseUint", Num: "-1234", Err: strconv.ErrSyntax}}, + {Error: &os.PathError{Op: "ReadFile", Path: "/path/to/secret/file", Err: os.ErrPermission}}, + } + + b, err := json.Marshal(&response, + // Intercept every attempt to marshal an error type. + json.WithMarshalers(json.JoinMarshalers( + // Suppose we consider strconv.NumError to be a safe to serialize: + // this type-specific marshal function intercepts this type + // and encodes the error message as a JSON string. + json.MarshalToFunc(func(enc *jsontext.Encoder, err *strconv.NumError) error { + return enc.WriteToken(jsontext.String(err.Error())) + }), + // Error messages may contain sensitive information that may not + // be appropriate to serialize. For all errors not handled above, + // report some generic error message. + json.MarshalFunc(func(error) ([]byte, error) { + return []byte(`"internal server error"`), nil + }), + )), + jsontext.Multiline(true)) // expand for readability + if err != nil { + log.Fatal(err) + } + fmt.Println(string(b)) + + // Output: + // [ + // { + // "Result": "Oranges are a good source of Vitamin C." + // }, + // { + // "Error": "strconv.ParseUint: parsing \"-1234\": invalid syntax" + // }, + // { + // "Error": "internal server error" + // } + // ] +} + +// In some applications, the exact precision of JSON numbers needs to be +// preserved when unmarshaling. This can be accomplished using a type-specific +// unmarshal function that intercepts all any types and pre-populates the +// interface value with a [jsontext.Value], which can represent a JSON number exactly. +func ExampleWithUnmarshalers_rawNumber() { + // Input with JSON numbers beyond the representation of a float64. + const input = `[false, 1e-1000, 3.141592653589793238462643383279, 1e+1000, true]` + + var value any + err := json.Unmarshal([]byte(input), &value, + // Intercept every attempt to unmarshal into the any type. + json.WithUnmarshalers( + json.UnmarshalFromFunc(func(dec *jsontext.Decoder, val *any) error { + // If the next value to be decoded is a JSON number, + // then provide a concrete Go type to unmarshal into. + if dec.PeekKind() == '0' { + *val = jsontext.Value(nil) + } + // Return SkipFunc to fallback on default unmarshal behavior. + return json.SkipFunc + }), + )) + if err != nil { + log.Fatal(err) + } + fmt.Println(value) + + // Sanity check. + want := []any{false, jsontext.Value("1e-1000"), jsontext.Value("3.141592653589793238462643383279"), jsontext.Value("1e+1000"), true} + if !reflect.DeepEqual(value, want) { + log.Fatalf("value mismatch:\ngot %v\nwant %v", value, want) + } + + // Output: + // [false 1e-1000 3.141592653589793238462643383279 1e+1000 true] +} + +// When using JSON for parsing configuration files, +// the parsing logic often needs to report an error with a line and column +// indicating where in the input an error occurred. +func ExampleWithUnmarshalers_recordOffsets() { + // Hypothetical configuration file. + const input = `[ + {"Source": "192.168.0.100:1234", "Destination": "192.168.0.1:80"}, + {"Source": "192.168.0.251:4004"}, + {"Source": "192.168.0.165:8080", "Destination": "0.0.0.0:80"} + ]` + type Tunnel struct { + Source netip.AddrPort + Destination netip.AddrPort + + // ByteOffset is populated during unmarshal with the byte offset + // within the JSON input of the JSON object for this Go struct. + ByteOffset int64 `json:"-"` // metadata to be ignored for JSON serialization + } + + var tunnels []Tunnel + err := json.Unmarshal([]byte(input), &tunnels, + // Intercept every attempt to unmarshal into the Tunnel type. + json.WithUnmarshalers( + json.UnmarshalFromFunc(func(dec *jsontext.Decoder, tunnel *Tunnel) error { + // Decoder.InputOffset reports the offset after the last token, + // but we want to record the offset before the next token. + // + // Call Decoder.PeekKind to buffer enough to reach the next token. + // Add the number of leading whitespace, commas, and colons + // to locate the start of the next token. + dec.PeekKind() + unread := dec.UnreadBuffer() + n := len(unread) - len(bytes.TrimLeft(unread, " \n\r\t,:")) + tunnel.ByteOffset = dec.InputOffset() + int64(n) + + // Return SkipFunc to fallback on default unmarshal behavior. + return json.SkipFunc + }), + )) + if err != nil { + log.Fatal(err) + } + + // lineColumn converts a byte offset into a one-indexed line and column. + // The offset must be within the bounds of the input. + lineColumn := func(input string, offset int) (line, column int) { + line = 1 + strings.Count(input[:offset], "\n") + column = 1 + offset - (strings.LastIndex(input[:offset], "\n") + len("\n")) + return line, column + } + + // Verify that the configuration file is valid. + for _, tunnel := range tunnels { + if !tunnel.Source.IsValid() || !tunnel.Destination.IsValid() { + line, column := lineColumn(input, int(tunnel.ByteOffset)) + fmt.Printf("%d:%d: source and destination must both be specified", line, column) + } + } + + // Output: + // 3:3: source and destination must both be specified +} diff --git a/src/encoding/json/v2/fields.go b/src/encoding/json/v2/fields.go new file mode 100644 index 00000000000000..4a02be7327a04b --- /dev/null +++ b/src/encoding/json/v2/fields.go @@ -0,0 +1,654 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "cmp" + "errors" + "fmt" + "io" + "reflect" + "slices" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonwire" +) + +type isZeroer interface { + IsZero() bool +} + +var isZeroerType = reflect.TypeFor[isZeroer]() + +type structFields struct { + flattened []structField // listed in depth-first ordering + byActualName map[string]*structField + byFoldedName map[string][]*structField + inlinedFallback *structField +} + +// reindex recomputes index to avoid bounds check during runtime. +// +// During the construction of each [structField] in [makeStructFields], +// the index field is 0-indexed. However, before it returns, +// the 0th field is stored in index0 and index stores the remainder. +func (sf *structFields) reindex() { + reindex := func(f *structField) { + f.index0 = f.index[0] + f.index = f.index[1:] + if len(f.index) == 0 { + f.index = nil // avoid pinning the backing slice + } + } + for i := range sf.flattened { + reindex(&sf.flattened[i]) + } + if sf.inlinedFallback != nil { + reindex(sf.inlinedFallback) + } +} + +// lookupByFoldedName looks up name by a case-insensitive match +// that also ignores the presence of dashes and underscores. +func (fs *structFields) lookupByFoldedName(name []byte) []*structField { + return fs.byFoldedName[string(foldName(name))] +} + +type structField struct { + id int // unique numeric ID in breadth-first ordering + index0 int // 0th index into a struct according to [reflect.Type.FieldByIndex] + index []int // 1st index and remainder according to [reflect.Type.FieldByIndex] + typ reflect.Type + fncs *arshaler + isZero func(addressableValue) bool + isEmpty func(addressableValue) bool + fieldOptions +} + +var errNoExportedFields = errors.New("Go struct has no exported fields") + +func makeStructFields(root reflect.Type) (fs structFields, serr *SemanticError) { + orErrorf := func(serr *SemanticError, t reflect.Type, f string, a ...any) *SemanticError { + return cmp.Or(serr, &SemanticError{GoType: t, Err: fmt.Errorf(f, a...)}) + } + + // Setup a queue for a breath-first search. + var queueIndex int + type queueEntry struct { + typ reflect.Type + index []int + visitChildren bool // whether to recursively visit inlined field in this struct + } + queue := []queueEntry{{root, nil, true}} + seen := map[reflect.Type]bool{root: true} + + // Perform a breadth-first search over all reachable fields. + // This ensures that len(f.index) will be monotonically increasing. + var allFields, inlinedFallbacks []structField + for queueIndex < len(queue) { + qe := queue[queueIndex] + queueIndex++ + + t := qe.typ + inlinedFallbackIndex := -1 // index of last inlined fallback field in current struct + namesIndex := make(map[string]int) // index of each field with a given JSON object name in current struct + var hasAnyJSONTag bool // whether any Go struct field has a `json` tag + var hasAnyJSONField bool // whether any JSON serializable fields exist in current struct + for i := range t.NumField() { + sf := t.Field(i) + _, hasTag := sf.Tag.Lookup("json") + hasAnyJSONTag = hasAnyJSONTag || hasTag + options, ignored, err := parseFieldOptions(sf) + if err != nil { + serr = cmp.Or(serr, &SemanticError{GoType: t, Err: err}) + } + if ignored { + continue + } + hasAnyJSONField = true + f := structField{ + // Allocate a new slice (len=N+1) to hold both + // the parent index (len=N) and the current index (len=1). + // Do this to avoid clobbering the memory of the parent index. + index: append(append(make([]int, 0, len(qe.index)+1), qe.index...), i), + typ: sf.Type, + fieldOptions: options, + } + if sf.Anonymous && !f.hasName { + if indirectType(f.typ).Kind() != reflect.Struct { + serr = orErrorf(serr, t, "embedded Go struct field %s of non-struct type must be explicitly given a JSON name", sf.Name) + } else { + f.inline = true // implied by use of Go embedding without an explicit name + } + } + if f.inline || f.unknown { + // Handle an inlined field that serializes to/from + // zero or more JSON object members. + + switch f.fieldOptions { + case fieldOptions{name: f.name, quotedName: f.quotedName, inline: true}: + case fieldOptions{name: f.name, quotedName: f.quotedName, unknown: true}: + case fieldOptions{name: f.name, quotedName: f.quotedName, inline: true, unknown: true}: + serr = orErrorf(serr, t, "Go struct field %s cannot have both `inline` and `unknown` specified", sf.Name) + f.inline = false // let `unknown` take precedence + default: + serr = orErrorf(serr, t, "Go struct field %s cannot have any options other than `inline` or `unknown` specified", sf.Name) + if f.hasName { + continue // invalid inlined field; treat as ignored + } + f.fieldOptions = fieldOptions{name: f.name, quotedName: f.quotedName, inline: f.inline, unknown: f.unknown} + if f.inline && f.unknown { + f.inline = false // let `unknown` take precedence + } + } + + // Reject any types with custom serialization otherwise + // it becomes impossible to know what sub-fields to inline. + tf := indirectType(f.typ) + if implementsAny(tf, allMethodTypes...) && tf != jsontextValueType { + serr = orErrorf(serr, t, "inlined Go struct field %s of type %s must not implement marshal or unmarshal methods", sf.Name, tf) + } + + // Handle an inlined field that serializes to/from + // a finite number of JSON object members backed by a Go struct. + if tf.Kind() == reflect.Struct { + if f.unknown { + serr = orErrorf(serr, t, "inlined Go struct field %s of type %s with `unknown` tag must be a Go map of string key or a jsontext.Value", sf.Name, tf) + continue // invalid inlined field; treat as ignored + } + if qe.visitChildren { + queue = append(queue, queueEntry{tf, f.index, !seen[tf]}) + } + seen[tf] = true + continue + } else if !sf.IsExported() { + serr = orErrorf(serr, t, "inlined Go struct field %s is not exported", sf.Name) + continue // invalid inlined field; treat as ignored + } + + // Handle an inlined field that serializes to/from any number of + // JSON object members back by a Go map or jsontext.Value. + switch { + case tf == jsontextValueType: + f.fncs = nil // specially handled in arshal_inlined.go + case tf.Kind() == reflect.Map && tf.Key().Kind() == reflect.String: + if implementsAny(tf.Key(), allMethodTypes...) { + serr = orErrorf(serr, t, "inlined map field %s of type %s must have a string key that does not implement marshal or unmarshal methods", sf.Name, tf) + continue // invalid inlined field; treat as ignored + } + f.fncs = lookupArshaler(tf.Elem()) + default: + serr = orErrorf(serr, t, "inlined Go struct field %s of type %s must be a Go struct, Go map of string key, or jsontext.Value", sf.Name, tf) + continue // invalid inlined field; treat as ignored + } + + // Reject multiple inlined fallback fields within the same struct. + if inlinedFallbackIndex >= 0 { + serr = orErrorf(serr, t, "inlined Go struct fields %s and %s cannot both be a Go map or jsontext.Value", t.Field(inlinedFallbackIndex).Name, sf.Name) + // Still append f to inlinedFallbacks as there is still a + // check for a dominant inlined fallback before returning. + } + inlinedFallbackIndex = i + + inlinedFallbacks = append(inlinedFallbacks, f) + } else { + // Handle normal Go struct field that serializes to/from + // a single JSON object member. + + // Unexported fields cannot be serialized except for + // embedded fields of a struct type, + // which might promote exported fields of their own. + if !sf.IsExported() { + tf := indirectType(f.typ) + if !(sf.Anonymous && tf.Kind() == reflect.Struct) { + serr = orErrorf(serr, t, "Go struct field %s is not exported", sf.Name) + continue + } + // Unfortunately, methods on the unexported field + // still cannot be called. + if implementsAny(tf, allMethodTypes...) || + (f.omitzero && implementsAny(tf, isZeroerType)) { + serr = orErrorf(serr, t, "Go struct field %s is not exported for method calls", sf.Name) + continue + } + } + + // Provide a function that uses a type's IsZero method. + switch { + case sf.Type.Kind() == reflect.Interface && sf.Type.Implements(isZeroerType): + f.isZero = func(va addressableValue) bool { + // Avoid panics calling IsZero on a nil interface or + // non-nil interface with nil pointer. + return va.IsNil() || (va.Elem().Kind() == reflect.Pointer && va.Elem().IsNil()) || va.Interface().(isZeroer).IsZero() + } + case sf.Type.Kind() == reflect.Pointer && sf.Type.Implements(isZeroerType): + f.isZero = func(va addressableValue) bool { + // Avoid panics calling IsZero on nil pointer. + return va.IsNil() || va.Interface().(isZeroer).IsZero() + } + case sf.Type.Implements(isZeroerType): + f.isZero = func(va addressableValue) bool { return va.Interface().(isZeroer).IsZero() } + case reflect.PointerTo(sf.Type).Implements(isZeroerType): + f.isZero = func(va addressableValue) bool { return va.Addr().Interface().(isZeroer).IsZero() } + } + + // Provide a function that can determine whether the value would + // serialize as an empty JSON value. + switch sf.Type.Kind() { + case reflect.String, reflect.Map, reflect.Array, reflect.Slice: + f.isEmpty = func(va addressableValue) bool { return va.Len() == 0 } + case reflect.Pointer, reflect.Interface: + f.isEmpty = func(va addressableValue) bool { return va.IsNil() } + } + + // Reject multiple fields with same name within the same struct. + if j, ok := namesIndex[f.name]; ok { + serr = orErrorf(serr, t, "Go struct fields %s and %s conflict over JSON object name %q", t.Field(j).Name, sf.Name, f.name) + // Still append f to allFields as there is still a + // check for a dominant field before returning. + } + namesIndex[f.name] = i + + f.id = len(allFields) + f.fncs = lookupArshaler(sf.Type) + allFields = append(allFields, f) + } + } + + // NOTE: New users to the json package are occasionally surprised that + // unexported fields are ignored. This occurs by necessity due to our + // inability to directly introspect such fields with Go reflection + // without the use of unsafe. + // + // To reduce friction here, refuse to serialize any Go struct that + // has no JSON serializable fields, has at least one Go struct field, + // and does not have any `json` tags present. For example, + // errors returned by errors.New would fail to serialize. + isEmptyStruct := t.NumField() == 0 + if !isEmptyStruct && !hasAnyJSONTag && !hasAnyJSONField { + serr = cmp.Or(serr, &SemanticError{GoType: t, Err: errNoExportedFields}) + } + } + + // Sort the fields by exact name (breaking ties by depth and + // then by presence of an explicitly provided JSON name). + // Select the dominant field from each set of fields with the same name. + // If multiple fields have the same name, then the dominant field + // is the one that exists alone at the shallowest depth, + // or the one that is uniquely tagged with a JSON name. + // Otherwise, no dominant field exists for the set. + flattened := allFields[:0] + slices.SortStableFunc(allFields, func(x, y structField) int { + return cmp.Or( + strings.Compare(x.name, y.name), + cmp.Compare(len(x.index), len(y.index)), + boolsCompare(!x.hasName, !y.hasName)) + }) + for len(allFields) > 0 { + n := 1 // number of fields with the same exact name + for n < len(allFields) && allFields[n-1].name == allFields[n].name { + n++ + } + if n == 1 || len(allFields[0].index) != len(allFields[1].index) || allFields[0].hasName != allFields[1].hasName { + flattened = append(flattened, allFields[0]) // only keep field if there is a dominant field + } + allFields = allFields[n:] + } + + // Sort the fields according to a breadth-first ordering + // so that we can re-number IDs with the smallest possible values. + // This optimizes use of uintSet such that it fits in the 64-entry bit set. + slices.SortFunc(flattened, func(x, y structField) int { + return cmp.Compare(x.id, y.id) + }) + for i := range flattened { + flattened[i].id = i + } + + // Sort the fields according to a depth-first ordering + // as the typical order that fields are marshaled. + slices.SortFunc(flattened, func(x, y structField) int { + return slices.Compare(x.index, y.index) + }) + + // Compute the mapping of fields in the byActualName map. + // Pre-fold all names so that we can lookup folded names quickly. + fs = structFields{ + flattened: flattened, + byActualName: make(map[string]*structField, len(flattened)), + byFoldedName: make(map[string][]*structField, len(flattened)), + } + for i, f := range fs.flattened { + foldedName := string(foldName([]byte(f.name))) + fs.byActualName[f.name] = &fs.flattened[i] + fs.byFoldedName[foldedName] = append(fs.byFoldedName[foldedName], &fs.flattened[i]) + } + for foldedName, fields := range fs.byFoldedName { + if len(fields) > 1 { + // The precedence order for conflicting ignoreCase names + // is by breadth-first order, rather than depth-first order. + slices.SortFunc(fields, func(x, y *structField) int { + return cmp.Compare(x.id, y.id) + }) + fs.byFoldedName[foldedName] = fields + } + } + if n := len(inlinedFallbacks); n == 1 || (n > 1 && len(inlinedFallbacks[0].index) != len(inlinedFallbacks[1].index)) { + fs.inlinedFallback = &inlinedFallbacks[0] // dominant inlined fallback field + } + fs.reindex() + return fs, serr +} + +// indirectType unwraps one level of pointer indirection +// similar to how Go only allows embedding either T or *T, +// but not **T or P (which is a named pointer). +func indirectType(t reflect.Type) reflect.Type { + if t.Kind() == reflect.Pointer && t.Name() == "" { + t = t.Elem() + } + return t +} + +// matchFoldedName matches a case-insensitive name depending on the options. +// It assumes that foldName(f.name) == foldName(name). +// +// Case-insensitive matching is used if the `case:ignore` tag option is specified +// or the MatchCaseInsensitiveNames call option is specified +// (and the `case:strict` tag option is not specified). +// Functionally, the `case:ignore` and `case:strict` tag options take precedence. +// +// The v1 definition of case-insensitivity operated under strings.EqualFold +// and would strictly compare dashes and underscores, +// while the v2 definition would ignore the presence of dashes and underscores. +// Thus, if the MatchCaseSensitiveDelimiter call option is specified, +// the match is further restricted to using strings.EqualFold. +func (f *structField) matchFoldedName(name []byte, flags *jsonflags.Flags) bool { + if f.casing == caseIgnore || (flags.Get(jsonflags.MatchCaseInsensitiveNames) && f.casing != caseStrict) { + if !flags.Get(jsonflags.MatchCaseSensitiveDelimiter) || strings.EqualFold(string(name), f.name) { + return true + } + } + return false +} + +const ( + caseIgnore = 1 + caseStrict = 2 +) + +type fieldOptions struct { + name string + quotedName string // quoted name per RFC 8785, section 3.2.2.2. + hasName bool + nameNeedEscape bool + casing int8 // either 0, caseIgnore, or caseStrict + inline bool + unknown bool + omitzero bool + omitempty bool + string bool + format string +} + +// parseFieldOptions parses the `json` tag in a Go struct field as +// a structured set of options configuring parameters such as +// the JSON member name and other features. +func parseFieldOptions(sf reflect.StructField) (out fieldOptions, ignored bool, err error) { + tag, hasTag := sf.Tag.Lookup("json") + tagOrig := tag + + // Check whether this field is explicitly ignored. + if tag == "-" { + return fieldOptions{}, true, nil + } + + // Check whether this field is unexported and not embedded, + // which Go reflection cannot mutate for the sake of serialization. + // + // An embedded field of an unexported type is still capable of + // forwarding exported fields, which may be JSON serialized. + // This technically operates on the edge of what is permissible by + // the Go language, but the most recent decision is to permit this. + // + // See https://go.dev/issue/24153 and https://go.dev/issue/32772. + if !sf.IsExported() && !sf.Anonymous { + // Tag options specified on an unexported field suggests user error. + if hasTag { + err = cmp.Or(err, fmt.Errorf("unexported Go struct field %s cannot have non-ignored `json:%q` tag", sf.Name, tag)) + } + return fieldOptions{}, true, err + } + + // Determine the JSON member name for this Go field. A user-specified name + // may be provided as either an identifier or a single-quoted string. + // The single-quoted string allows arbitrary characters in the name. + // See https://go.dev/issue/2718 and https://go.dev/issue/3546. + out.name = sf.Name // always starts with an uppercase character + if len(tag) > 0 && !strings.HasPrefix(tag, ",") { + // For better compatibility with v1, accept almost any unescaped name. + n := len(tag) - len(strings.TrimLeftFunc(tag, func(r rune) bool { + return !strings.ContainsRune(",\\'\"`", r) // reserve comma, backslash, and quotes + })) + name := tag[:n] + + // If the next character is not a comma, then the name is either + // malformed (if n > 0) or a single-quoted name. + // In either case, call consumeTagOption to handle it further. + var err2 error + if !strings.HasPrefix(tag[n:], ",") && len(name) != len(tag) { + name, n, err2 = consumeTagOption(tag) + if err2 != nil { + err = cmp.Or(err, fmt.Errorf("Go struct field %s has malformed `json` tag: %v", sf.Name, err2)) + } + } + if !utf8.ValidString(name) { + err = cmp.Or(err, fmt.Errorf("Go struct field %s has JSON object name %q with invalid UTF-8", sf.Name, name)) + name = string([]rune(name)) // replace invalid UTF-8 with utf8.RuneError + } + if name == "-" && tag[0] == '-' { + defer func() { // defer to let other errors take precedence + err = cmp.Or(err, fmt.Errorf("Go struct field %s has JSON object name %q; either "+ + "use `json:\"-\"` to ignore the field or "+ + "use `json:\"'-'%s` to specify %q as the name", sf.Name, out.name, strings.TrimPrefix(strconv.Quote(tagOrig), `"-`), name)) + }() + } + if err2 == nil { + out.hasName = true + out.name = name + } + tag = tag[n:] + } + b, _ := jsonwire.AppendQuote(nil, out.name, &jsonflags.Flags{}) + out.quotedName = string(b) + out.nameNeedEscape = jsonwire.NeedEscape(out.name) + + // Handle any additional tag options (if any). + var wasFormat bool + seenOpts := make(map[string]bool) + for len(tag) > 0 { + // Consume comma delimiter. + if tag[0] != ',' { + err = cmp.Or(err, fmt.Errorf("Go struct field %s has malformed `json` tag: invalid character %q before next option (expecting ',')", sf.Name, tag[0])) + } else { + tag = tag[len(","):] + if len(tag) == 0 { + err = cmp.Or(err, fmt.Errorf("Go struct field %s has malformed `json` tag: invalid trailing ',' character", sf.Name)) + break + } + } + + // Consume and process the tag option. + opt, n, err2 := consumeTagOption(tag) + if err2 != nil { + err = cmp.Or(err, fmt.Errorf("Go struct field %s has malformed `json` tag: %v", sf.Name, err2)) + } + rawOpt := tag[:n] + tag = tag[n:] + switch { + case wasFormat: + err = cmp.Or(err, fmt.Errorf("Go struct field %s has `format` tag option that was not specified last", sf.Name)) + case strings.HasPrefix(rawOpt, "'") && strings.TrimFunc(opt, isLetterOrDigit) == "": + err = cmp.Or(err, fmt.Errorf("Go struct field %s has unnecessarily quoted appearance of `%s` tag option; specify `%s` instead", sf.Name, rawOpt, opt)) + } + switch opt { + case "case": + if !strings.HasPrefix(tag, ":") { + err = cmp.Or(err, fmt.Errorf("Go struct field %s is missing value for `case` tag option; specify `case:ignore` or `case:strict` instead", sf.Name)) + break + } + tag = tag[len(":"):] + opt, n, err2 := consumeTagOption(tag) + if err2 != nil { + err = cmp.Or(err, fmt.Errorf("Go struct field %s has malformed value for `case` tag option: %v", sf.Name, err2)) + break + } + rawOpt := tag[:n] + tag = tag[n:] + if strings.HasPrefix(rawOpt, "'") { + err = cmp.Or(err, fmt.Errorf("Go struct field %s has unnecessarily quoted appearance of `case:%s` tag option; specify `case:%s` instead", sf.Name, rawOpt, opt)) + } + switch opt { + case "ignore": + out.casing |= caseIgnore + case "strict": + out.casing |= caseStrict + default: + err = cmp.Or(err, fmt.Errorf("Go struct field %s has unknown `case:%s` tag value", sf.Name, rawOpt)) + } + case "inline": + out.inline = true + case "unknown": + out.unknown = true + case "omitzero": + out.omitzero = true + case "omitempty": + out.omitempty = true + case "string": + out.string = true + case "format": + if !strings.HasPrefix(tag, ":") { + err = cmp.Or(err, fmt.Errorf("Go struct field %s is missing value for `format` tag option", sf.Name)) + break + } + tag = tag[len(":"):] + opt, n, err2 := consumeTagOption(tag) + if err2 != nil { + err = cmp.Or(err, fmt.Errorf("Go struct field %s has malformed value for `format` tag option: %v", sf.Name, err2)) + break + } + tag = tag[n:] + out.format = opt + wasFormat = true + default: + // Reject keys that resemble one of the supported options. + // This catches invalid mutants such as "omitEmpty" or "omit_empty". + normOpt := strings.ReplaceAll(strings.ToLower(opt), "_", "") + switch normOpt { + case "case", "inline", "unknown", "omitzero", "omitempty", "string", "format": + err = cmp.Or(err, fmt.Errorf("Go struct field %s has invalid appearance of `%s` tag option; specify `%s` instead", sf.Name, opt, normOpt)) + } + + // NOTE: Everything else is ignored. This does not mean it is + // forward compatible to insert arbitrary tag options since + // a future version of this package may understand that tag. + } + + // Reject duplicates. + switch { + case out.casing == caseIgnore|caseStrict: + err = cmp.Or(err, fmt.Errorf("Go struct field %s cannot have both `case:ignore` and `case:strict` tag options", sf.Name)) + case seenOpts[opt]: + err = cmp.Or(err, fmt.Errorf("Go struct field %s has duplicate appearance of `%s` tag option", sf.Name, rawOpt)) + } + seenOpts[opt] = true + } + return out, false, err +} + +// consumeTagOption consumes the next option, +// which is either a Go identifier or a single-quoted string. +// If the next option is invalid, it returns all of in until the next comma, +// and reports an error. +func consumeTagOption(in string) (string, int, error) { + // For legacy compatibility with v1, assume options are comma-separated. + i := strings.IndexByte(in, ',') + if i < 0 { + i = len(in) + } + + switch r, _ := utf8.DecodeRuneInString(in); { + // Option as a Go identifier. + case r == '_' || unicode.IsLetter(r): + n := len(in) - len(strings.TrimLeftFunc(in, isLetterOrDigit)) + return in[:n], n, nil + // Option as a single-quoted string. + case r == '\'': + // The grammar is nearly identical to a double-quoted Go string literal, + // but uses single quotes as the terminators. The reason for a custom + // grammar is because both backtick and double quotes cannot be used + // verbatim in a struct tag. + // + // Convert a single-quoted string to a double-quote string and rely on + // strconv.Unquote to handle the rest. + var inEscape bool + b := []byte{'"'} + n := len(`'`) + for len(in) > n { + r, rn := utf8.DecodeRuneInString(in[n:]) + switch { + case inEscape: + if r == '\'' { + b = b[:len(b)-1] // remove escape character: `\'` => `'` + } + inEscape = false + case r == '\\': + inEscape = true + case r == '"': + b = append(b, '\\') // insert escape character: `"` => `\"` + case r == '\'': + b = append(b, '"') + n += len(`'`) + out, err := strconv.Unquote(string(b)) + if err != nil { + return in[:i], i, fmt.Errorf("invalid single-quoted string: %s", in[:n]) + } + return out, n, nil + } + b = append(b, in[n:][:rn]...) + n += rn + } + if n > 10 { + n = 10 // limit the amount of context printed in the error + } + return in[:i], i, fmt.Errorf("single-quoted string not terminated: %s...", in[:n]) + case len(in) == 0: + return in[:i], i, io.ErrUnexpectedEOF + default: + return in[:i], i, fmt.Errorf("invalid character %q at start of option (expecting Unicode letter or single quote)", r) + } +} + +func isLetterOrDigit(r rune) bool { + return r == '_' || unicode.IsLetter(r) || unicode.IsNumber(r) +} + +// boolsCompare compares x and y, ordering false before true. +func boolsCompare(x, y bool) int { + switch { + case !x && y: + return -1 + default: + return 0 + case x && !y: + return +1 + } +} diff --git a/src/encoding/json/v2/fields_test.go b/src/encoding/json/v2/fields_test.go new file mode 100644 index 00000000000000..ae58182f298ab4 --- /dev/null +++ b/src/encoding/json/v2/fields_test.go @@ -0,0 +1,834 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "encoding" + "errors" + "reflect" + "testing" + + "encoding/json/internal/jsontest" + "encoding/json/jsontext" +) + +type unexported struct{} + +func TestMakeStructFields(t *testing.T) { + type Embed struct { + Foo string + } + type Recursive struct { + A string + *Recursive `json:",inline"` + B string + } + type MapStringAny map[string]any + tests := []struct { + name jsontest.CaseName + in any + want structFields + wantErr error + }{{ + name: jsontest.Name("Names"), + in: struct { + F1 string + F2 string `json:"-"` + F3 string `json:"json_name"` + f3 string + F5 string `json:"json_name_nocase,case:ignore"` + }{}, + want: structFields{ + flattened: []structField{ + {id: 0, index: []int{0}, typ: stringType, fieldOptions: fieldOptions{name: "F1", quotedName: `"F1"`}}, + {id: 1, index: []int{2}, typ: stringType, fieldOptions: fieldOptions{name: "json_name", quotedName: `"json_name"`, hasName: true}}, + {id: 2, index: []int{4}, typ: stringType, fieldOptions: fieldOptions{name: "json_name_nocase", quotedName: `"json_name_nocase"`, hasName: true, casing: caseIgnore}}, + }, + }, + }, { + name: jsontest.Name("BreadthFirstSearch"), + in: struct { + L1A string + L1B struct { + L2A string + L2B struct { + L3A string + } `json:",inline"` + L2C string + } `json:",inline"` + L1C string + L1D struct { + L2D string + L2E struct { + L3B string + } `json:",inline"` + L2F string + } `json:",inline"` + L1E string + }{}, + want: structFields{ + flattened: []structField{ + {id: 0, index: []int{0}, typ: stringType, fieldOptions: fieldOptions{name: "L1A", quotedName: `"L1A"`}}, + {id: 3, index: []int{1, 0}, typ: stringType, fieldOptions: fieldOptions{name: "L2A", quotedName: `"L2A"`}}, + {id: 7, index: []int{1, 1, 0}, typ: stringType, fieldOptions: fieldOptions{name: "L3A", quotedName: `"L3A"`}}, + {id: 4, index: []int{1, 2}, typ: stringType, fieldOptions: fieldOptions{name: "L2C", quotedName: `"L2C"`}}, + {id: 1, index: []int{2}, typ: stringType, fieldOptions: fieldOptions{name: "L1C", quotedName: `"L1C"`}}, + {id: 5, index: []int{3, 0}, typ: stringType, fieldOptions: fieldOptions{name: "L2D", quotedName: `"L2D"`}}, + {id: 8, index: []int{3, 1, 0}, typ: stringType, fieldOptions: fieldOptions{name: "L3B", quotedName: `"L3B"`}}, + {id: 6, index: []int{3, 2}, typ: stringType, fieldOptions: fieldOptions{name: "L2F", quotedName: `"L2F"`}}, + {id: 2, index: []int{4}, typ: stringType, fieldOptions: fieldOptions{name: "L1E", quotedName: `"L1E"`}}, + }, + }, + }, { + name: jsontest.Name("NameResolution"), + in: struct { + X1 struct { + X struct { + A string // loses in precedence to A + B string // cancels out with X2.X.B + D string // loses in precedence to D + } `json:",inline"` + } `json:",inline"` + X2 struct { + X struct { + B string // cancels out with X1.X.B + C string + D string // loses in precedence to D + } `json:",inline"` + } `json:",inline"` + A string // takes precedence over X1.X.A + D string // takes precedence over X1.X.D and X2.X.D + }{}, + want: structFields{ + flattened: []structField{ + {id: 2, index: []int{1, 0, 1}, typ: stringType, fieldOptions: fieldOptions{name: "C", quotedName: `"C"`}}, + {id: 0, index: []int{2}, typ: stringType, fieldOptions: fieldOptions{name: "A", quotedName: `"A"`}}, + {id: 1, index: []int{3}, typ: stringType, fieldOptions: fieldOptions{name: "D", quotedName: `"D"`}}, + }, + }, + }, { + name: jsontest.Name("NameResolution/ExplicitNameUniquePrecedence"), + in: struct { + X1 struct { + A string // loses in precedence to X2.A + } `json:",inline"` + X2 struct { + A string `json:"A"` + } `json:",inline"` + X3 struct { + A string // loses in precedence to X2.A + } `json:",inline"` + }{}, + want: structFields{ + flattened: []structField{ + {id: 0, index: []int{1, 0}, typ: stringType, fieldOptions: fieldOptions{hasName: true, name: "A", quotedName: `"A"`}}, + }, + }, + }, { + name: jsontest.Name("NameResolution/ExplicitNameCancelsOut"), + in: struct { + X1 struct { + A string // loses in precedence to X2.A or X3.A + } `json:",inline"` + X2 struct { + A string `json:"A"` // cancels out with X3.A + } `json:",inline"` + X3 struct { + A string `json:"A"` // cancels out with X2.A + } `json:",inline"` + }{}, + want: structFields{flattened: []structField{}}, + }, { + name: jsontest.Name("Embed/Implicit"), + in: struct { + Embed + }{}, + want: structFields{ + flattened: []structField{ + {id: 0, index: []int{0, 0}, typ: stringType, fieldOptions: fieldOptions{name: "Foo", quotedName: `"Foo"`}}, + }, + }, + }, { + name: jsontest.Name("Embed/Explicit"), + in: struct { + Embed `json:",inline"` + }{}, + want: structFields{ + flattened: []structField{ + {id: 0, index: []int{0, 0}, typ: stringType, fieldOptions: fieldOptions{name: "Foo", quotedName: `"Foo"`}}, + }, + }, + }, { + name: jsontest.Name("Recursive"), + in: struct { + A string + Recursive `json:",inline"` + C string + }{}, + want: structFields{ + flattened: []structField{ + {id: 0, index: []int{0}, typ: stringType, fieldOptions: fieldOptions{name: "A", quotedName: `"A"`}}, + {id: 2, index: []int{1, 2}, typ: stringType, fieldOptions: fieldOptions{name: "B", quotedName: `"B"`}}, + {id: 1, index: []int{2}, typ: stringType, fieldOptions: fieldOptions{name: "C", quotedName: `"C"`}}, + }, + }, + }, { + name: jsontest.Name("InlinedFallback/Cancelation"), + in: struct { + X1 struct { + X jsontext.Value `json:",inline"` + } `json:",inline"` + X2 struct { + X map[string]any `json:",unknown"` + } `json:",inline"` + }{}, + want: structFields{}, + }, { + name: jsontest.Name("InlinedFallback/Precedence"), + in: struct { + X1 struct { + X jsontext.Value `json:",inline"` + } `json:",inline"` + X2 struct { + X map[string]any `json:",unknown"` + } `json:",inline"` + X map[string]jsontext.Value `json:",unknown"` + }{}, + want: structFields{ + inlinedFallback: &structField{id: 0, index: []int{2}, typ: T[map[string]jsontext.Value](), fieldOptions: fieldOptions{name: "X", quotedName: `"X"`, unknown: true}}, + }, + }, { + name: jsontest.Name("InlinedFallback/InvalidImplicit"), + in: struct { + MapStringAny + }{}, + want: structFields{ + flattened: []structField{ + {id: 0, index: []int{0}, typ: reflect.TypeOf(MapStringAny(nil)), fieldOptions: fieldOptions{name: "MapStringAny", quotedName: `"MapStringAny"`}}, + }, + }, + wantErr: errors.New("embedded Go struct field MapStringAny of non-struct type must be explicitly given a JSON name"), + }, { + name: jsontest.Name("InvalidUTF8"), + in: struct { + Name string `json:"'\\xde\\xad\\xbe\\xef'"` + }{}, + want: structFields{ + flattened: []structField{ + {id: 0, index: []int{0}, typ: stringType, fieldOptions: fieldOptions{hasName: true, name: "\u07ad\ufffd\ufffd", quotedName: "\"\u07ad\ufffd\ufffd\"", nameNeedEscape: true}}, + }, + }, + wantErr: errors.New(`Go struct field Name has JSON object name "ޭ\xbe\xef" with invalid UTF-8`), + }, { + name: jsontest.Name("DuplicateName"), + in: struct { + A string `json:"same"` + B string `json:"same"` + }{}, + want: structFields{flattened: []structField{}}, + wantErr: errors.New(`Go struct fields A and B conflict over JSON object name "same"`), + }, { + name: jsontest.Name("BothInlineAndUnknown"), + in: struct { + A struct{} `json:",inline,unknown"` + }{}, + wantErr: errors.New("Go struct field A cannot have both `inline` and `unknown` specified"), + }, { + name: jsontest.Name("InlineWithOptions"), + in: struct { + A struct{} `json:",inline,omitempty"` + }{}, + wantErr: errors.New("Go struct field A cannot have any options other than `inline` or `unknown` specified"), + }, { + name: jsontest.Name("UnknownWithOptions"), + in: struct { + A map[string]any `json:",inline,omitempty"` + }{}, + want: structFields{inlinedFallback: &structField{ + index: []int{0}, + typ: reflect.TypeFor[map[string]any](), + fieldOptions: fieldOptions{ + name: "A", + quotedName: `"A"`, + inline: true, + }, + }}, + wantErr: errors.New("Go struct field A cannot have any options other than `inline` or `unknown` specified"), + }, { + name: jsontest.Name("InlineTextMarshaler"), + in: struct { + A struct{ encoding.TextMarshaler } `json:",inline"` + }{}, + want: structFields{flattened: []structField{{ + index: []int{0, 0}, + typ: reflect.TypeFor[encoding.TextMarshaler](), + fieldOptions: fieldOptions{ + name: "TextMarshaler", + quotedName: `"TextMarshaler"`, + }, + }}}, + wantErr: errors.New(`inlined Go struct field A of type struct { encoding.TextMarshaler } must not implement marshal or unmarshal methods`), + }, { + name: jsontest.Name("InlineTextAppender"), + in: struct { + A struct{ encoding.TextAppender } `json:",inline"` + }{}, + want: structFields{flattened: []structField{{ + index: []int{0, 0}, + typ: reflect.TypeFor[encoding.TextAppender](), + fieldOptions: fieldOptions{ + name: "TextAppender", + quotedName: `"TextAppender"`, + }, + }}}, + wantErr: errors.New(`inlined Go struct field A of type struct { encoding.TextAppender } must not implement marshal or unmarshal methods`), + }, { + name: jsontest.Name("UnknownJSONMarshaler"), + in: struct { + A struct{ Marshaler } `json:",unknown"` + }{}, + wantErr: errors.New(`inlined Go struct field A of type struct { json.Marshaler } must not implement marshal or unmarshal methods`), + }, { + name: jsontest.Name("InlineJSONMarshalerTo"), + in: struct { + A struct{ MarshalerTo } `json:",inline"` + }{}, + want: structFields{flattened: []structField{{ + index: []int{0, 0}, + typ: reflect.TypeFor[MarshalerTo](), + fieldOptions: fieldOptions{ + name: "MarshalerTo", + quotedName: `"MarshalerTo"`, + }, + }}}, + wantErr: errors.New(`inlined Go struct field A of type struct { json.MarshalerTo } must not implement marshal or unmarshal methods`), + }, { + name: jsontest.Name("UnknownTextUnmarshaler"), + in: struct { + A *struct{ encoding.TextUnmarshaler } `json:",unknown"` + }{}, + wantErr: errors.New(`inlined Go struct field A of type struct { encoding.TextUnmarshaler } must not implement marshal or unmarshal methods`), + }, { + name: jsontest.Name("InlineJSONUnmarshaler"), + in: struct { + A *struct{ Unmarshaler } `json:",inline"` + }{}, + want: structFields{flattened: []structField{{ + index: []int{0, 0}, + typ: reflect.TypeFor[Unmarshaler](), + fieldOptions: fieldOptions{ + name: "Unmarshaler", + quotedName: `"Unmarshaler"`, + }, + }}}, + wantErr: errors.New(`inlined Go struct field A of type struct { json.Unmarshaler } must not implement marshal or unmarshal methods`), + }, { + name: jsontest.Name("UnknownJSONUnmarshalerFrom"), + in: struct { + A struct{ UnmarshalerFrom } `json:",unknown"` + }{}, + wantErr: errors.New(`inlined Go struct field A of type struct { json.UnmarshalerFrom } must not implement marshal or unmarshal methods`), + }, { + name: jsontest.Name("UnknownStruct"), + in: struct { + A struct { + X, Y, Z string + } `json:",unknown"` + }{}, + wantErr: errors.New("inlined Go struct field A of type struct { X string; Y string; Z string } with `unknown` tag must be a Go map of string key or a jsontext.Value"), + }, { + name: jsontest.Name("InlineUnsupported/MapIntKey"), + in: struct { + A map[int]any `json:",unknown"` + }{}, + wantErr: errors.New(`inlined Go struct field A of type map[int]interface {} must be a Go struct, Go map of string key, or jsontext.Value`), + }, { + name: jsontest.Name("InlineUnsupported/MapTextMarshalerStringKey"), + in: struct { + A map[nocaseString]any `json:",inline"` + }{}, + wantErr: errors.New(`inlined map field A of type map[json.nocaseString]interface {} must have a string key that does not implement marshal or unmarshal methods`), + }, { + name: jsontest.Name("InlineUnsupported/MapMarshalerStringKey"), + in: struct { + A map[stringMarshalEmpty]any `json:",inline"` + }{}, + wantErr: errors.New(`inlined map field A of type map[json.stringMarshalEmpty]interface {} must have a string key that does not implement marshal or unmarshal methods`), + }, { + name: jsontest.Name("InlineUnsupported/DoublePointer"), + in: struct { + A **struct{} `json:",inline"` + }{}, + wantErr: errors.New(`inlined Go struct field A of type *struct {} must be a Go struct, Go map of string key, or jsontext.Value`), + }, { + name: jsontest.Name("DuplicateInline"), + in: struct { + A map[string]any `json:",inline"` + B jsontext.Value `json:",inline"` + }{}, + wantErr: errors.New(`inlined Go struct fields A and B cannot both be a Go map or jsontext.Value`), + }, { + name: jsontest.Name("DuplicateEmbedInline"), + in: struct { + A MapStringAny `json:",inline"` + B jsontext.Value `json:",inline"` + }{}, + wantErr: errors.New(`inlined Go struct fields A and B cannot both be a Go map or jsontext.Value`), + }} + + for _, tt := range tests { + t.Run(tt.name.Name, func(t *testing.T) { + got, err := makeStructFields(reflect.TypeOf(tt.in)) + + // Sanity check that pointers are consistent. + pointers := make(map[*structField]bool) + for i := range got.flattened { + pointers[&got.flattened[i]] = true + } + for _, f := range got.byActualName { + if !pointers[f] { + t.Errorf("%s: byActualName pointer not in flattened", tt.name.Where) + } + } + for _, fs := range got.byFoldedName { + for _, f := range fs { + if !pointers[f] { + t.Errorf("%s: byFoldedName pointer not in flattened", tt.name.Where) + } + } + } + + // Zero out fields that are incomparable. + for i := range got.flattened { + got.flattened[i].fncs = nil + got.flattened[i].isEmpty = nil + } + if got.inlinedFallback != nil { + got.inlinedFallback.fncs = nil + got.inlinedFallback.isEmpty = nil + } + + // Reproduce maps in want. + tt.want.byActualName = make(map[string]*structField) + for i := range tt.want.flattened { + f := &tt.want.flattened[i] + tt.want.byActualName[f.name] = f + } + tt.want.byFoldedName = make(map[string][]*structField) + for i, f := range tt.want.flattened { + foldedName := string(foldName([]byte(f.name))) + tt.want.byFoldedName[foldedName] = append(tt.want.byFoldedName[foldedName], &tt.want.flattened[i]) + } + + // Only compare underlying error to simplify test logic. + var gotErr error + if err != nil { + gotErr = err.Err + } + + tt.want.reindex() + if !reflect.DeepEqual(got, tt.want) || !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("%s: makeStructFields(%T):\n\tgot (%v, %v)\n\twant (%v, %v)", tt.name.Where, tt.in, got, gotErr, tt.want, tt.wantErr) + } + }) + } +} + +func TestParseTagOptions(t *testing.T) { + tests := []struct { + name jsontest.CaseName + in any // must be a struct with a single field + wantOpts fieldOptions + wantIgnored bool + wantErr error + }{{ + name: jsontest.Name("GoName"), + in: struct { + FieldName int + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`}, + }, { + name: jsontest.Name("GoNameWithOptions"), + in: struct { + FieldName int `json:",inline"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, inline: true}, + }, { + name: jsontest.Name("Empty"), + in: struct { + V int `json:""` + }{}, + wantOpts: fieldOptions{name: "V", quotedName: `"V"`}, + }, { + name: jsontest.Name("Unexported"), + in: struct { + v int `json:"Hello"` + }{}, + wantIgnored: true, + wantErr: errors.New("unexported Go struct field v cannot have non-ignored `json:\"Hello\"` tag"), + }, { + name: jsontest.Name("UnexportedEmpty"), + in: struct { + v int `json:""` + }{}, + wantIgnored: true, + wantErr: errors.New("unexported Go struct field v cannot have non-ignored `json:\"\"` tag"), + }, { + name: jsontest.Name("EmbedUnexported"), + in: struct { + unexported + }{}, + wantOpts: fieldOptions{name: "unexported", quotedName: `"unexported"`}, + }, { + name: jsontest.Name("Ignored"), + in: struct { + V int `json:"-"` + }{}, + wantIgnored: true, + }, { + name: jsontest.Name("IgnoredEmbedUnexported"), + in: struct { + unexported `json:"-"` + }{}, + wantIgnored: true, + }, { + name: jsontest.Name("DashComma"), + in: struct { + V int `json:"-,"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "-", quotedName: `"-"`}, + wantErr: errors.New("Go struct field V has malformed `json` tag: invalid trailing ',' character"), + }, { + name: jsontest.Name("DashCommaOmitEmpty"), + in: struct { + V int `json:"-,omitempty"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "-", quotedName: `"-"`, omitempty: true}, + wantErr: errors.New("Go struct field V has JSON object name \"-\"; either use `json:\"-\"` to ignore the field or use `json:\"'-',omitempty\"` to specify \"-\" as the name"), + }, { + name: jsontest.Name("QuotedDashCommaOmitEmpty"), + in: struct { + V int `json:"'-',omitempty"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "-", quotedName: `"-"`, omitempty: true}, + }, { + name: jsontest.Name("QuotedDashName"), + in: struct { + V int `json:"'-'"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "-", quotedName: `"-"`}, + }, { + name: jsontest.Name("LatinPunctuationName"), + in: struct { + V int `json:"$%-/"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "$%-/", quotedName: `"$%-/"`}, + }, { + name: jsontest.Name("QuotedLatinPunctuationName"), + in: struct { + V int `json:"'$%-/'"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "$%-/", quotedName: `"$%-/"`}, + }, { + name: jsontest.Name("LatinDigitsName"), + in: struct { + V int `json:"0123456789"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "0123456789", quotedName: `"0123456789"`}, + }, { + name: jsontest.Name("QuotedLatinDigitsName"), + in: struct { + V int `json:"'0123456789'"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "0123456789", quotedName: `"0123456789"`}, + }, { + name: jsontest.Name("LatinUppercaseName"), + in: struct { + V int `json:"ABCDEFGHIJKLMOPQRSTUVWXYZ"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "ABCDEFGHIJKLMOPQRSTUVWXYZ", quotedName: `"ABCDEFGHIJKLMOPQRSTUVWXYZ"`}, + }, { + name: jsontest.Name("LatinLowercaseName"), + in: struct { + V int `json:"abcdefghijklmnopqrstuvwxyz_"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "abcdefghijklmnopqrstuvwxyz_", quotedName: `"abcdefghijklmnopqrstuvwxyz_"`}, + }, { + name: jsontest.Name("GreekName"), + in: struct { + V string `json:"Ελλάδα"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "Ελλάδα", quotedName: `"Ελλάδα"`}, + }, { + name: jsontest.Name("QuotedGreekName"), + in: struct { + V string `json:"'Ελλάδα'"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "Ελλάδα", quotedName: `"Ελλάδα"`}, + }, { + name: jsontest.Name("ChineseName"), + in: struct { + V string `json:"世界"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "世界", quotedName: `"世界"`}, + }, { + name: jsontest.Name("QuotedChineseName"), + in: struct { + V string `json:"'世界'"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "世界", quotedName: `"世界"`}, + }, { + name: jsontest.Name("PercentSlashName"), + in: struct { + V int `json:"text/html%"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "text/html%", quotedName: `"text/html%"`}, + }, { + name: jsontest.Name("QuotedPercentSlashName"), + in: struct { + V int `json:"'text/html%'"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "text/html%", quotedName: `"text/html%"`}, + }, { + name: jsontest.Name("PunctuationName"), + in: struct { + V string `json:"!#$%&()*+-./:;<=>?@[]^_{|}~ "` + }{}, + wantOpts: fieldOptions{hasName: true, name: "!#$%&()*+-./:;<=>?@[]^_{|}~ ", quotedName: `"!#$%&()*+-./:;<=>?@[]^_{|}~ "`, nameNeedEscape: true}, + }, { + name: jsontest.Name("QuotedPunctuationName"), + in: struct { + V string `json:"'!#$%&()*+-./:;<=>?@[]^_{|}~ '"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "!#$%&()*+-./:;<=>?@[]^_{|}~ ", quotedName: `"!#$%&()*+-./:;<=>?@[]^_{|}~ "`, nameNeedEscape: true}, + }, { + name: jsontest.Name("EmptyName"), + in: struct { + V int `json:"''"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "", quotedName: `""`}, + }, { + name: jsontest.Name("SpaceName"), + in: struct { + V int `json:"' '"` + }{}, + wantOpts: fieldOptions{hasName: true, name: " ", quotedName: `" "`}, + }, { + name: jsontest.Name("CommaQuotes"), + in: struct { + V int `json:"',\\'\"\\\"'"` + }{}, + wantOpts: fieldOptions{hasName: true, name: `,'""`, quotedName: `",'\"\""`, nameNeedEscape: true}, + }, { + name: jsontest.Name("SingleComma"), + in: struct { + V int `json:","` + }{}, + wantOpts: fieldOptions{name: "V", quotedName: `"V"`}, + wantErr: errors.New("Go struct field V has malformed `json` tag: invalid trailing ',' character"), + }, { + name: jsontest.Name("SuperfluousCommas"), + in: struct { + V int `json:",,,,\"\",,inline,unknown,,,,"` + }{}, + wantOpts: fieldOptions{name: "V", quotedName: `"V"`, inline: true, unknown: true}, + wantErr: errors.New("Go struct field V has malformed `json` tag: invalid character ',' at start of option (expecting Unicode letter or single quote)"), + }, { + name: jsontest.Name("CaseAloneOption"), + in: struct { + FieldName int `json:",case"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`}, + wantErr: errors.New("Go struct field FieldName is missing value for `case` tag option; specify `case:ignore` or `case:strict` instead"), + }, { + name: jsontest.Name("CaseIgnoreOption"), + in: struct { + FieldName int `json:",case:ignore"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, casing: caseIgnore}, + }, { + name: jsontest.Name("CaseStrictOption"), + in: struct { + FieldName int `json:",case:strict"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, casing: caseStrict}, + }, { + name: jsontest.Name("CaseUnknownOption"), + in: struct { + FieldName int `json:",case:unknown"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`}, + wantErr: errors.New("Go struct field FieldName has unknown `case:unknown` tag value"), + }, { + name: jsontest.Name("CaseQuotedOption"), + in: struct { + FieldName int `json:",case:'ignore'"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, casing: caseIgnore}, + wantErr: errors.New("Go struct field FieldName has unnecessarily quoted appearance of `case:'ignore'` tag option; specify `case:ignore` instead"), + }, { + name: jsontest.Name("BothCaseOptions"), + in: struct { + FieldName int `json:",case:ignore,case:strict"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, casing: caseIgnore | caseStrict}, + wantErr: errors.New("Go struct field FieldName cannot have both `case:ignore` and `case:strict` tag options"), + }, { + name: jsontest.Name("InlineOption"), + in: struct { + FieldName int `json:",inline"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, inline: true}, + }, { + name: jsontest.Name("UnknownOption"), + in: struct { + FieldName int `json:",unknown"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, unknown: true}, + }, { + name: jsontest.Name("OmitZeroOption"), + in: struct { + FieldName int `json:",omitzero"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, omitzero: true}, + }, { + name: jsontest.Name("OmitEmptyOption"), + in: struct { + FieldName int `json:",omitempty"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, omitempty: true}, + }, { + name: jsontest.Name("StringOption"), + in: struct { + FieldName int `json:",string"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, string: true}, + }, { + name: jsontest.Name("FormatOptionEqual"), + in: struct { + FieldName int `json:",format=fizzbuzz"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`}, + wantErr: errors.New("Go struct field FieldName is missing value for `format` tag option"), + }, { + name: jsontest.Name("FormatOptionColon"), + in: struct { + FieldName int `json:",format:fizzbuzz"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, format: "fizzbuzz"}, + }, { + name: jsontest.Name("FormatOptionQuoted"), + in: struct { + FieldName int `json:",format:'2006-01-02'"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, format: "2006-01-02"}, + }, { + name: jsontest.Name("FormatOptionInvalid"), + in: struct { + FieldName int `json:",format:'2006-01-02"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`}, + wantErr: errors.New("Go struct field FieldName has malformed value for `format` tag option: single-quoted string not terminated: '2006-01-0..."), + }, { + name: jsontest.Name("FormatOptionNotLast"), + in: struct { + FieldName int `json:",format:alpha,ordered"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, format: "alpha"}, + wantErr: errors.New("Go struct field FieldName has `format` tag option that was not specified last"), + }, { + name: jsontest.Name("AllOptions"), + in: struct { + FieldName int `json:",case:ignore,inline,unknown,omitzero,omitempty,string,format:format"` + }{}, + wantOpts: fieldOptions{ + name: "FieldName", + quotedName: `"FieldName"`, + casing: caseIgnore, + inline: true, + unknown: true, + omitzero: true, + omitempty: true, + string: true, + format: "format", + }, + }, { + name: jsontest.Name("AllOptionsQuoted"), + in: struct { + FieldName int `json:",'case':'ignore','inline','unknown','omitzero','omitempty','string','format':'format'"` + }{}, + wantOpts: fieldOptions{ + name: "FieldName", + quotedName: `"FieldName"`, + casing: caseIgnore, + inline: true, + unknown: true, + omitzero: true, + omitempty: true, + string: true, + format: "format", + }, + wantErr: errors.New("Go struct field FieldName has unnecessarily quoted appearance of `'case'` tag option; specify `case` instead"), + }, { + name: jsontest.Name("AllOptionsCaseSensitive"), + in: struct { + FieldName int `json:",CASE:IGNORE,INLINE,UNKNOWN,OMITZERO,OMITEMPTY,STRING,FORMAT:FORMAT"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`}, + wantErr: errors.New("Go struct field FieldName has invalid appearance of `CASE` tag option; specify `case` instead"), + }, { + name: jsontest.Name("AllOptionsSpaceSensitive"), + in: struct { + FieldName int `json:", case:ignore , inline , unknown , omitzero , omitempty , string , format:format "` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`}, + wantErr: errors.New("Go struct field FieldName has malformed `json` tag: invalid character ' ' at start of option (expecting Unicode letter or single quote)"), + }, { + name: jsontest.Name("UnknownTagOption"), + in: struct { + FieldName int `json:",inline,whoknows,string"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, inline: true, string: true}, + }, { + name: jsontest.Name("MalformedQuotedString/MissingQuote"), + in: struct { + FieldName int `json:"'hello,string"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, string: true}, + wantErr: errors.New("Go struct field FieldName has malformed `json` tag: single-quoted string not terminated: 'hello,str..."), + }, { + name: jsontest.Name("MalformedQuotedString/MissingComma"), + in: struct { + FieldName int `json:"'hello'inline,string"` + }{}, + wantOpts: fieldOptions{hasName: true, name: "hello", quotedName: `"hello"`, inline: true, string: true}, + wantErr: errors.New("Go struct field FieldName has malformed `json` tag: invalid character 'i' before next option (expecting ',')"), + }, { + name: jsontest.Name("MalformedQuotedString/InvalidEscape"), + in: struct { + FieldName int `json:"'hello\\u####',inline,string"` + }{}, + wantOpts: fieldOptions{name: "FieldName", quotedName: `"FieldName"`, inline: true, string: true}, + wantErr: errors.New("Go struct field FieldName has malformed `json` tag: invalid single-quoted string: 'hello\\u####'"), + }, { + name: jsontest.Name("MisnamedTag"), + in: struct { + V int `jsom:"Misnamed"` + }{}, + wantOpts: fieldOptions{name: "V", quotedName: `"V"`}, + }} + + for _, tt := range tests { + t.Run(tt.name.Name, func(t *testing.T) { + fs := reflect.TypeOf(tt.in).Field(0) + gotOpts, gotIgnored, gotErr := parseFieldOptions(fs) + if !reflect.DeepEqual(gotOpts, tt.wantOpts) || gotIgnored != tt.wantIgnored || !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("%s: parseFieldOptions(%T) = (\n\t%v,\n\t%v,\n\t%v\n), want (\n\t%v,\n\t%v,\n\t%v\n)", tt.name.Where, tt.in, gotOpts, gotIgnored, gotErr, tt.wantOpts, tt.wantIgnored, tt.wantErr) + } + }) + } +} diff --git a/src/encoding/json/v2/fold.go b/src/encoding/json/v2/fold.go new file mode 100644 index 00000000000000..ca33efe85fd9aa --- /dev/null +++ b/src/encoding/json/v2/fold.go @@ -0,0 +1,58 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "unicode" + "unicode/utf8" +) + +// foldName returns a folded string such that foldName(x) == foldName(y) +// is similar to strings.EqualFold(x, y), but ignores underscore and dashes. +// This allows foldName to match common naming conventions. +func foldName(in []byte) []byte { + // This is inlinable to take advantage of "function outlining". + // See https://blog.filippo.io/efficient-go-apis-with-the-inliner/ + var arr [32]byte // large enough for most JSON names + return appendFoldedName(arr[:0], in) +} +func appendFoldedName(out, in []byte) []byte { + for i := 0; i < len(in); { + // Handle single-byte ASCII. + if c := in[i]; c < utf8.RuneSelf { + if c != '_' && c != '-' { + if 'a' <= c && c <= 'z' { + c -= 'a' - 'A' + } + out = append(out, c) + } + i++ + continue + } + // Handle multi-byte Unicode. + r, n := utf8.DecodeRune(in[i:]) + out = utf8.AppendRune(out, foldRune(r)) + i += n + } + return out +} + +// foldRune is a variation on unicode.SimpleFold that returns the same rune +// for all runes in the same fold set. +// +// Invariant: +// +// foldRune(x) == foldRune(y) ⇔ strings.EqualFold(string(x), string(y)) +func foldRune(r rune) rune { + for { + r2 := unicode.SimpleFold(r) + if r2 <= r { + return r2 // smallest character in the fold set + } + r = r2 + } +} diff --git a/src/encoding/json/v2/fold_test.go b/src/encoding/json/v2/fold_test.go new file mode 100644 index 00000000000000..a1c89723802983 --- /dev/null +++ b/src/encoding/json/v2/fold_test.go @@ -0,0 +1,127 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "fmt" + "reflect" + "testing" + "unicode" +) + +var equalFoldTestdata = []struct { + in1, in2 string + want bool +}{ + {"", "", true}, + {"abc", "abc", true}, + {"ABcd", "ABcd", true}, + {"123abc", "123ABC", true}, + {"_1_2_-_3__--a-_-b-c-", "123ABC", true}, + {"αβδ", "ΑΒΔ", true}, + {"abc", "xyz", false}, + {"abc", "XYZ", false}, + {"abcdefghijk", "abcdefghijX", false}, + {"abcdefghijk", "abcdefghij\u212A", true}, + {"abcdefghijK", "abcdefghij\u212A", true}, + {"abcdefghijkz", "abcdefghij\u212Ay", false}, + {"abcdefghijKz", "abcdefghij\u212Ay", false}, + {"1", "2", false}, + {"utf-8", "US-ASCII", false}, + {"hello, world!", "hello, world!", true}, + {"hello, world!", "Hello, World!", true}, + {"hello, world!", "HELLO, WORLD!", true}, + {"hello, world!", "jello, world!", false}, + {"γειά, κόσμε!", "γειά, κόσμε!", true}, + {"γειά, κόσμε!", "Γειά, Κόσμε!", true}, + {"γειά, κόσμε!", "ΓΕΙΆ, ΚΌΣΜΕ!", true}, + {"γειά, κόσμε!", "ΛΕΙΆ, ΚΌΣΜΕ!", false}, + {"AESKey", "aesKey", true}, + {"γειά, κόσμε!", "Γ\xce_\xb5ιά, Κόσμε!", false}, + {"aeskey", "AESKEY", true}, + {"AESKEY", "aes_key", true}, + {"aes_key", "AES_KEY", true}, + {"AES_KEY", "aes-key", true}, + {"aes-key", "AES-KEY", true}, + {"AES-KEY", "aesKey", true}, + {"aesKey", "AesKey", true}, + {"AesKey", "AESKey", true}, + {"AESKey", "aeskey", true}, + {"DESKey", "aeskey", false}, + {"AES Key", "aeskey", false}, + {"aes﹏key", "aeskey", false}, // Unicode underscore not handled + {"aes〰key", "aeskey", false}, // Unicode dash not handled +} + +func TestEqualFold(t *testing.T) { + for _, tt := range equalFoldTestdata { + got := equalFold([]byte(tt.in1), []byte(tt.in2)) + if got != tt.want { + t.Errorf("equalFold(%q, %q) = %v, want %v", tt.in1, tt.in2, got, tt.want) + } + } +} + +func equalFold(x, y []byte) bool { + return string(foldName(x)) == string(foldName(y)) +} + +func TestFoldRune(t *testing.T) { + if testing.Short() { + t.Skip() + } + + var foldSet []rune + for r := range rune(unicode.MaxRune + 1) { + // Derive all runes that are all part of the same fold set. + foldSet = foldSet[:0] + for r0 := r; r != r0 || len(foldSet) == 0; r = unicode.SimpleFold(r) { + foldSet = append(foldSet, r) + } + + // Normalized form of each rune in a foldset must be the same and + // also be within the set itself. + var withinSet bool + rr0 := foldRune(foldSet[0]) + for _, r := range foldSet { + withinSet = withinSet || rr0 == r + rr := foldRune(r) + if rr0 != rr { + t.Errorf("foldRune(%q) = %q, want %q", r, rr, rr0) + } + } + if !withinSet { + t.Errorf("foldRune(%q) = %q not in fold set %q", foldSet[0], rr0, string(foldSet)) + } + } +} + +// TestBenchmarkUnmarshalUnknown unmarshals an unknown field into a struct with +// varying number of fields. Since the unknown field does not directly match +// any known field by name, it must fall back on case-insensitive matching. +func TestBenchmarkUnmarshalUnknown(t *testing.T) { + in := []byte(`{"NameUnknown":null}`) + for _, n := range []int{1, 2, 5, 10, 20, 50, 100} { + unmarshal := Unmarshal + + var fields []reflect.StructField + for i := range n { + fields = append(fields, reflect.StructField{ + Name: fmt.Sprintf("Name%d", i), + Type: T[int](), + Tag: `json:",case:ignore"`, + }) + } + out := reflect.New(reflect.StructOf(fields)).Interface() + + t.Run(fmt.Sprintf("N%d", n), func(t *testing.T) { + if err := unmarshal(in, out); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + }) + } +} diff --git a/src/encoding/json/v2/fuzz_test.go b/src/encoding/json/v2/fuzz_test.go new file mode 100644 index 00000000000000..491a08311eb0a3 --- /dev/null +++ b/src/encoding/json/v2/fuzz_test.go @@ -0,0 +1,39 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "bytes" + "testing" +) + +func FuzzEqualFold(f *testing.F) { + for _, tt := range equalFoldTestdata { + f.Add([]byte(tt.in1), []byte(tt.in2)) + } + + equalFoldSimple := func(x, y []byte) bool { + strip := func(b []byte) []byte { + return bytes.Map(func(r rune) rune { + if r == '_' || r == '-' { + return -1 // ignore underscores and dashes + } + return r + }, b) + } + return bytes.EqualFold(strip(x), strip(y)) + } + + f.Fuzz(func(t *testing.T, s1, s2 []byte) { + // Compare the optimized and simplified implementations. + got := equalFold(s1, s2) + want := equalFoldSimple(s1, s2) + if got != want { + t.Errorf("equalFold(%q, %q) = %v, want %v", s1, s2, got, want) + } + }) +} diff --git a/src/encoding/json/v2/inline_test.go b/src/encoding/json/v2/inline_test.go new file mode 100644 index 00000000000000..b68fefb0642579 --- /dev/null +++ b/src/encoding/json/v2/inline_test.go @@ -0,0 +1,109 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "os" + "os/exec" + "strings" + "testing" +) + +// Whether a function is inlinable is dependent on the Go compiler version +// and also relies on the presence of the Go toolchain itself being installed. +// This test is disabled by default and explicitly enabled with an +// environment variable that is specified in our integration tests, +// which have fine control over exactly which Go version is being tested. +var testInline = os.Getenv("TEST_INLINE") != "" + +func TestInline(t *testing.T) { + if !testInline { + t.SkipNow() + } + + pkgs := map[string]map[string]bool{ + ".": { + "hash64": true, + "foldName": true, // thin wrapper over appendFoldedName + }, + "./internal/jsonwire": { + "ConsumeWhitespace": true, + "ConsumeNull": true, + "ConsumeFalse": true, + "ConsumeTrue": true, + "ConsumeSimpleString": true, + "ConsumeString": true, // thin wrapper over consumeStringResumable + "ConsumeSimpleNumber": true, + "ConsumeNumber": true, // thin wrapper over consumeNumberResumable + "UnquoteMayCopy": true, // thin wrapper over unescapeString + "HasSuffixByte": true, + "TrimSuffixByte": true, + "TrimSuffixString": true, + "TrimSuffixWhitespace": true, + }, + "./jsontext": { + "encoderState.NeedFlush": true, + "Decoder.ReadToken": true, // thin wrapper over decoderState.ReadToken + "Decoder.ReadValue": true, // thin wrapper over decoderState.ReadValue + "Encoder.WriteToken": true, // thin wrapper over encoderState.WriteToken + "Encoder.WriteValue": true, // thin wrapper over encoderState.WriteValue + "decodeBuffer.needMore": true, + "stateMachine.appendLiteral": true, + "stateMachine.appendNumber": true, + "stateMachine.appendString": true, + "stateMachine.Depth": true, + "stateMachine.reset": true, + "stateMachine.MayAppendDelim": true, + "stateMachine.needDelim": true, + "stateMachine.popArray": true, + "stateMachine.popObject": true, + "stateMachine.pushArray": true, + "stateMachine.pushObject": true, + "stateEntry.Increment": true, + "stateEntry.decrement": true, + "stateEntry.isArray": true, + "stateEntry.isObject": true, + "stateEntry.Length": true, + "stateEntry.needImplicitColon": true, + "stateEntry.needImplicitComma": true, + "stateEntry.NeedObjectName": true, + "stateEntry.needObjectValue": true, + "objectNameStack.reset": true, + "objectNameStack.length": true, + "objectNameStack.getUnquoted": true, + "objectNameStack.push": true, + "objectNameStack.ReplaceLastQuotedOffset": true, + "objectNameStack.replaceLastUnquotedName": true, + "objectNameStack.pop": true, + "objectNameStack.ensureCopiedBuffer": true, + "objectNamespace.insertQuoted": true, // thin wrapper over objectNamespace.insert + "objectNamespace.InsertUnquoted": true, // thin wrapper over objectNamespace.insert + "Token.String": true, // thin wrapper over Token.string + }, + } + + for pkg, fncs := range pkgs { + cmd := exec.Command("go", "build", "-gcflags=-m", pkg) + b, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("exec.Command error: %v\n\n%s", err, b) + } + for _, line := range strings.Split(string(b), "\n") { + const phrase = ": can inline " + if i := strings.Index(line, phrase); i >= 0 { + fnc := line[i+len(phrase):] + fnc = strings.ReplaceAll(fnc, "(", "") + fnc = strings.ReplaceAll(fnc, "*", "") + fnc = strings.ReplaceAll(fnc, ")", "") + delete(fncs, fnc) + } + } + for fnc := range fncs { + t.Errorf("%v is not inlinable, expected it to be", fnc) + } + } +} diff --git a/src/encoding/json/v2/intern.go b/src/encoding/json/v2/intern.go new file mode 100644 index 00000000000000..3c75e034f05537 --- /dev/null +++ b/src/encoding/json/v2/intern.go @@ -0,0 +1,88 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "encoding/binary" + "math/bits" +) + +// stringCache is a cache for strings converted from a []byte. +type stringCache = [256]string // 256*unsafe.Sizeof(string("")) => 4KiB + +// makeString returns the string form of b. +// It returns a pre-allocated string from c if present, otherwise +// it allocates a new string, inserts it into the cache, and returns it. +func makeString(c *stringCache, b []byte) string { + const ( + minCachedLen = 2 // single byte strings are already interned by the runtime + maxCachedLen = 256 // large enough for UUIDs, IPv6 addresses, SHA-256 checksums, etc. + ) + if c == nil || len(b) < minCachedLen || len(b) > maxCachedLen { + return string(b) + } + + // Compute a hash from the fixed-width prefix and suffix of the string. + // This ensures hashing a string is a constant time operation. + var h uint32 + switch { + case len(b) >= 8: + lo := binary.LittleEndian.Uint64(b[:8]) + hi := binary.LittleEndian.Uint64(b[len(b)-8:]) + h = hash64(uint32(lo), uint32(lo>>32)) ^ hash64(uint32(hi), uint32(hi>>32)) + case len(b) >= 4: + lo := binary.LittleEndian.Uint32(b[:4]) + hi := binary.LittleEndian.Uint32(b[len(b)-4:]) + h = hash64(lo, hi) + case len(b) >= 2: + lo := binary.LittleEndian.Uint16(b[:2]) + hi := binary.LittleEndian.Uint16(b[len(b)-2:]) + h = hash64(uint32(lo), uint32(hi)) + } + + // Check the cache for the string. + i := h % uint32(len(*c)) + if s := (*c)[i]; s == string(b) { + return s + } + s := string(b) + (*c)[i] = s + return s +} + +// hash64 returns the hash of two uint32s as a single uint32. +func hash64(lo, hi uint32) uint32 { + // If avalanche=true, this is identical to XXH32 hash on a 8B string: + // var b [8]byte + // binary.LittleEndian.PutUint32(b[:4], lo) + // binary.LittleEndian.PutUint32(b[4:], hi) + // return xxhash.Sum32(b[:]) + const ( + prime1 = 0x9e3779b1 + prime2 = 0x85ebca77 + prime3 = 0xc2b2ae3d + prime4 = 0x27d4eb2f + prime5 = 0x165667b1 + ) + h := prime5 + uint32(8) + h += lo * prime3 + h = bits.RotateLeft32(h, 17) * prime4 + h += hi * prime3 + h = bits.RotateLeft32(h, 17) * prime4 + // Skip final mix (avalanche) step of XXH32 for performance reasons. + // Empirical testing shows that the improvements in unbiased distribution + // does not outweigh the extra cost in computational complexity. + const avalanche = false + if avalanche { + h ^= h >> 15 + h *= prime2 + h ^= h >> 13 + h *= prime3 + h ^= h >> 16 + } + return h +} diff --git a/src/encoding/json/v2/intern_test.go b/src/encoding/json/v2/intern_test.go new file mode 100644 index 00000000000000..9163f41006a4ef --- /dev/null +++ b/src/encoding/json/v2/intern_test.go @@ -0,0 +1,146 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "bytes" + "fmt" + "io" + "testing" + + "encoding/json/internal/jsontest" + "encoding/json/jsontext" +) + +func TestIntern(t *testing.T) { + var sc stringCache + const alphabet = "abcdefghijklmnopqrstuvwxyz" + for i := range len(alphabet) + 1 { + want := alphabet[i:] + if got := makeString(&sc, []byte(want)); got != want { + t.Fatalf("make = %v, want %v", got, want) + } + } + for i := range 1000 { + want := fmt.Sprintf("test%b", i) + if got := makeString(&sc, []byte(want)); got != want { + t.Fatalf("make = %v, want %v", got, want) + } + } +} + +var sink string + +func BenchmarkIntern(b *testing.B) { + datasetStrings := func(name string) (out [][]byte) { + var data []byte + for _, ts := range jsontest.Data { + if ts.Name == name { + data = ts.Data() + } + } + dec := jsontext.NewDecoder(bytes.NewReader(data)) + for { + k, n := dec.StackIndex(dec.StackDepth()) + isObjectName := k == '{' && n%2 == 0 + tok, err := dec.ReadToken() + if err != nil { + if err == io.EOF { + break + } + b.Fatalf("ReadToken error: %v", err) + } + if tok.Kind() == '"' && !isObjectName { + out = append(out, []byte(tok.String())) + } + } + return out + } + + tests := []struct { + label string + data [][]byte + }{ + // Best is the best case scenario where every string is the same. + {"Best", func() (out [][]byte) { + for range 1000 { + out = append(out, []byte("hello, world!")) + } + return out + }()}, + + // Repeat is a sequence of the same set of names repeated. + // This commonly occurs when unmarshaling a JSON array of JSON objects, + // where the set of all names is usually small. + {"Repeat", func() (out [][]byte) { + for range 100 { + for _, s := range []string{"first_name", "last_name", "age", "address", "street_address", "city", "state", "postal_code", "phone_numbers", "gender"} { + out = append(out, []byte(s)) + } + } + return out + }()}, + + // Synthea is all string values encountered in the Synthea FHIR dataset. + {"Synthea", datasetStrings("SyntheaFhir")}, + + // Twitter is all string values encountered in the Twitter dataset. + {"Twitter", datasetStrings("TwitterStatus")}, + + // Worst is the worst case scenario where every string is different + // resulting in wasted time looking up a string that will never match. + {"Worst", func() (out [][]byte) { + for i := range 1000 { + out = append(out, []byte(fmt.Sprintf("%016x", i))) + } + return out + }()}, + } + + for _, tt := range tests { + b.Run(tt.label, func(b *testing.B) { + // Alloc simply heap allocates each string. + // This provides an upper bound on the number of allocations. + b.Run("Alloc", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + for _, b := range tt.data { + sink = string(b) + } + } + }) + // Cache interns strings using stringCache. + // We want to optimize for having a faster runtime than Alloc, + // and also keeping the number of allocations closer to GoMap. + b.Run("Cache", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + var sc stringCache + for _, b := range tt.data { + sink = makeString(&sc, b) + } + } + }) + // GoMap interns all strings in a simple Go map. + // This provides a lower bound on the number of allocations. + b.Run("GoMap", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + m := make(map[string]string) + for _, b := range tt.data { + s, ok := m[string(b)] + if !ok { + s = string(b) + m[s] = s + } + sink = s + } + } + }) + }) + } +} diff --git a/src/encoding/json/v2/options.go b/src/encoding/json/v2/options.go new file mode 100644 index 00000000000000..9685f20f9f805f --- /dev/null +++ b/src/encoding/json/v2/options.go @@ -0,0 +1,288 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "fmt" + + "encoding/json/internal" + "encoding/json/internal/jsonflags" + "encoding/json/internal/jsonopts" +) + +// Options configure [Marshal], [MarshalWrite], [MarshalEncode], +// [Unmarshal], [UnmarshalRead], and [UnmarshalDecode] with specific features. +// Each function takes in a variadic list of options, where properties +// set in later options override the value of previously set properties. +// +// The Options type is identical to [encoding/json.Options] and +// [encoding/json/jsontext.Options]. Options from the other packages can +// be used interchangeably with functionality in this package. +// +// Options represent either a singular option or a set of options. +// It can be functionally thought of as a Go map of option properties +// (even though the underlying implementation avoids Go maps for performance). +// +// The constructors (e.g., [Deterministic]) return a singular option value: +// +// opt := Deterministic(true) +// +// which is analogous to creating a single entry map: +// +// opt := Options{"Deterministic": true} +// +// [JoinOptions] composes multiple options values to together: +// +// out := JoinOptions(opts...) +// +// which is analogous to making a new map and copying the options over: +// +// out := make(Options) +// for _, m := range opts { +// for k, v := range m { +// out[k] = v +// } +// } +// +// [GetOption] looks up the value of options parameter: +// +// v, ok := GetOption(opts, Deterministic) +// +// which is analogous to a Go map lookup: +// +// v, ok := Options["Deterministic"] +// +// There is a single Options type, which is used with both marshal and unmarshal. +// Some options affect both operations, while others only affect one operation: +// +// - [StringifyNumbers] affects marshaling and unmarshaling +// - [Deterministic] affects marshaling only +// - [FormatNilSliceAsNull] affects marshaling only +// - [FormatNilMapAsNull] affects marshaling only +// - [OmitZeroStructFields] affects marshaling only +// - [MatchCaseInsensitiveNames] affects marshaling and unmarshaling +// - [DiscardUnknownMembers] affects marshaling only +// - [RejectUnknownMembers] affects unmarshaling only +// - [WithMarshalers] affects marshaling only +// - [WithUnmarshalers] affects unmarshaling only +// +// Options that do not affect a particular operation are ignored. +type Options = jsonopts.Options + +// JoinOptions coalesces the provided list of options into a single Options. +// Properties set in later options override the value of previously set properties. +func JoinOptions(srcs ...Options) Options { + var dst jsonopts.Struct + dst.Join(srcs...) + return &dst +} + +// GetOption returns the value stored in opts with the provided setter, +// reporting whether the value is present. +// +// Example usage: +// +// v, ok := json.GetOption(opts, json.Deterministic) +// +// Options are most commonly introspected to alter the JSON representation of +// [MarshalerTo.MarshalJSONTo] and [UnmarshalerFrom.UnmarshalJSONFrom] methods, and +// [MarshalToFunc] and [UnmarshalFromFunc] functions. +// In such cases, the presence bit should generally be ignored. +func GetOption[T any](opts Options, setter func(T) Options) (T, bool) { + return jsonopts.GetOption(opts, setter) +} + +// DefaultOptionsV2 is the full set of all options that define v2 semantics. +// It is equivalent to the set of options in [encoding/json.DefaultOptionsV1] +// all being set to false. All other options are not present. +func DefaultOptionsV2() Options { + return &jsonopts.DefaultOptionsV2 +} + +// StringifyNumbers specifies that numeric Go types should be marshaled +// as a JSON string containing the equivalent JSON number value. +// When unmarshaling, numeric Go types are parsed from a JSON string +// containing the JSON number without any surrounding whitespace. +// +// According to RFC 8259, section 6, a JSON implementation may choose to +// limit the representation of a JSON number to an IEEE 754 binary64 value. +// This may cause decoders to lose precision for int64 and uint64 types. +// Quoting JSON numbers as a JSON string preserves the exact precision. +// +// This affects either marshaling or unmarshaling. +func StringifyNumbers(v bool) Options { + if v { + return jsonflags.StringifyNumbers | 1 + } else { + return jsonflags.StringifyNumbers | 0 + } +} + +// Deterministic specifies that the same input value will be serialized +// as the exact same output bytes. Different processes of +// the same program will serialize equal values to the same bytes, +// but different versions of the same program are not guaranteed +// to produce the exact same sequence of bytes. +// +// This only affects marshaling and is ignored when unmarshaling. +func Deterministic(v bool) Options { + if v { + return jsonflags.Deterministic | 1 + } else { + return jsonflags.Deterministic | 0 + } +} + +// FormatNilSliceAsNull specifies that a nil Go slice should marshal as a +// JSON null instead of the default representation as an empty JSON array +// (or an empty JSON string in the case of ~[]byte). +// Slice fields explicitly marked with `format:emitempty` still marshal +// as an empty JSON array. +// +// This only affects marshaling and is ignored when unmarshaling. +func FormatNilSliceAsNull(v bool) Options { + if v { + return jsonflags.FormatNilSliceAsNull | 1 + } else { + return jsonflags.FormatNilSliceAsNull | 0 + } +} + +// FormatNilMapAsNull specifies that a nil Go map should marshal as a +// JSON null instead of the default representation as an empty JSON object. +// Map fields explicitly marked with `format:emitempty` still marshal +// as an empty JSON object. +// +// This only affects marshaling and is ignored when unmarshaling. +func FormatNilMapAsNull(v bool) Options { + if v { + return jsonflags.FormatNilMapAsNull | 1 + } else { + return jsonflags.FormatNilMapAsNull | 0 + } +} + +// OmitZeroStructFields specifies that a Go struct should marshal in such a way +// that all struct fields that are zero are omitted from the marshaled output +// if the value is zero as determined by the "IsZero() bool" method if present, +// otherwise based on whether the field is the zero Go value. +// This is semantically equivalent to specifying the `omitzero` tag option +// on every field in a Go struct. +// +// This only affects marshaling and is ignored when unmarshaling. +func OmitZeroStructFields(v bool) Options { + if v { + return jsonflags.OmitZeroStructFields | 1 + } else { + return jsonflags.OmitZeroStructFields | 0 + } +} + +// MatchCaseInsensitiveNames specifies that JSON object members are matched +// against Go struct fields using a case-insensitive match of the name. +// Go struct fields explicitly marked with `case:strict` or `case:ignore` +// always use case-sensitive (or case-insensitive) name matching, +// regardless of the value of this option. +// +// This affects either marshaling or unmarshaling. +// For marshaling, this option may alter the detection of duplicate names +// (assuming [jsontext.AllowDuplicateNames] is false) from inlined fields +// if it matches one of the declared fields in the Go struct. +func MatchCaseInsensitiveNames(v bool) Options { + if v { + return jsonflags.MatchCaseInsensitiveNames | 1 + } else { + return jsonflags.MatchCaseInsensitiveNames | 0 + } +} + +// DiscardUnknownMembers specifies that marshaling should ignore any +// JSON object members stored in Go struct fields dedicated to storing +// unknown JSON object members. +// +// This only affects marshaling and is ignored when unmarshaling. +func DiscardUnknownMembers(v bool) Options { + if v { + return jsonflags.DiscardUnknownMembers | 1 + } else { + return jsonflags.DiscardUnknownMembers | 0 + } +} + +// RejectUnknownMembers specifies that unknown members should be rejected +// when unmarshaling a JSON object, regardless of whether there is a field +// to store unknown members. +// +// This only affects unmarshaling and is ignored when marshaling. +func RejectUnknownMembers(v bool) Options { + if v { + return jsonflags.RejectUnknownMembers | 1 + } else { + return jsonflags.RejectUnknownMembers | 0 + } +} + +// WithMarshalers specifies a list of type-specific marshalers to use, +// which can be used to override the default marshal behavior for values +// of particular types. +// +// This only affects marshaling and is ignored when unmarshaling. +func WithMarshalers(v *Marshalers) Options { + return (*marshalersOption)(v) +} + +// WithUnmarshalers specifies a list of type-specific unmarshalers to use, +// which can be used to override the default unmarshal behavior for values +// of particular types. +// +// This only affects unmarshaling and is ignored when marshaling. +func WithUnmarshalers(v *Unmarshalers) Options { + return (*unmarshalersOption)(v) +} + +// These option types are declared here instead of "jsonopts" +// to avoid a dependency on "reflect" from "jsonopts". +type ( + marshalersOption Marshalers + unmarshalersOption Unmarshalers +) + +func (*marshalersOption) JSONOptions(internal.NotForPublicUse) {} +func (*unmarshalersOption) JSONOptions(internal.NotForPublicUse) {} + +// Inject support into "jsonopts" to handle these types. +func init() { + jsonopts.GetUnknownOption = func(src jsonopts.Struct, zero jsonopts.Options) (any, bool) { + switch zero.(type) { + case *marshalersOption: + if !src.Flags.Has(jsonflags.Marshalers) { + return (*Marshalers)(nil), false + } + return src.Marshalers.(*Marshalers), true + case *unmarshalersOption: + if !src.Flags.Has(jsonflags.Unmarshalers) { + return (*Unmarshalers)(nil), false + } + return src.Unmarshalers.(*Unmarshalers), true + default: + panic(fmt.Sprintf("unknown option %T", zero)) + } + } + jsonopts.JoinUnknownOption = func(dst jsonopts.Struct, src jsonopts.Options) jsonopts.Struct { + switch src := src.(type) { + case *marshalersOption: + dst.Flags.Set(jsonflags.Marshalers | 1) + dst.Marshalers = (*Marshalers)(src) + case *unmarshalersOption: + dst.Flags.Set(jsonflags.Unmarshalers | 1) + dst.Unmarshalers = (*Unmarshalers)(src) + default: + panic(fmt.Sprintf("unknown option %T", src)) + } + return dst + } +} diff --git a/src/encoding/json/v2_bench_test.go b/src/encoding/json/v2_bench_test.go new file mode 100644 index 00000000000000..b9ed7b62207be1 --- /dev/null +++ b/src/encoding/json/v2_bench_test.go @@ -0,0 +1,483 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +// Large data benchmark. +// The JSON data is a summary of agl's changes in the +// go, webkit, and chromium open source projects. +// We benchmark converting between the JSON form +// and in-memory data structures. + +package json + +import ( + "bytes" + "io" + "strings" + "testing" + + "encoding/json/internal/jsontest" +) + +type codeResponse struct { + Tree *codeNode `json:"tree"` + Username string `json:"username"` +} + +type codeNode struct { + Name string `json:"name"` + Kids []*codeNode `json:"kids"` + CLWeight float64 `json:"cl_weight"` + Touches int `json:"touches"` + MinT int64 `json:"min_t"` + MaxT int64 `json:"max_t"` + MeanT int64 `json:"mean_t"` +} + +var codeJSON []byte +var codeStruct codeResponse + +func codeInit() { + var data []byte + for _, entry := range jsontest.Data { + if entry.Name == "GolangSource" { + data = entry.Data() + } + } + codeJSON = data + + if err := Unmarshal(codeJSON, &codeStruct); err != nil { + panic("unmarshal code.json: " + err.Error()) + } + + var err error + if data, err = Marshal(&codeStruct); err != nil { + panic("marshal code.json: " + err.Error()) + } + + if !bytes.Equal(data, codeJSON) { + println("different lengths", len(data), len(codeJSON)) + for i := 0; i < len(data) && i < len(codeJSON); i++ { + if data[i] != codeJSON[i] { + println("re-marshal: changed at byte", i) + println("orig: ", string(codeJSON[i-10:i+10])) + println("new: ", string(data[i-10:i+10])) + break + } + } + panic("re-marshal code.json: different result") + } +} + +func BenchmarkCodeEncoder(b *testing.B) { + b.ReportAllocs() + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + b.RunParallel(func(pb *testing.PB) { + enc := NewEncoder(io.Discard) + for pb.Next() { + if err := enc.Encode(&codeStruct); err != nil { + b.Fatalf("Encode error: %v", err) + } + } + }) + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkCodeEncoderError(b *testing.B) { + b.ReportAllocs() + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + + // Trigger an error in Marshal with cyclic data. + type Dummy struct { + Name string + Next *Dummy + } + dummy := Dummy{Name: "Dummy"} + dummy.Next = &dummy + + b.RunParallel(func(pb *testing.PB) { + enc := NewEncoder(io.Discard) + for pb.Next() { + if err := enc.Encode(&codeStruct); err != nil { + b.Fatalf("Encode error: %v", err) + } + if _, err := Marshal(dummy); err == nil { + b.Fatal("Marshal error: got nil, want non-nil") + } + } + }) + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkCodeMarshal(b *testing.B) { + b.ReportAllocs() + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := Marshal(&codeStruct); err != nil { + b.Fatalf("Marshal error: %v", err) + } + } + }) + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkCodeMarshalError(b *testing.B) { + b.ReportAllocs() + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + + // Trigger an error in Marshal with cyclic data. + type Dummy struct { + Name string + Next *Dummy + } + dummy := Dummy{Name: "Dummy"} + dummy.Next = &dummy + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := Marshal(&codeStruct); err != nil { + b.Fatalf("Marshal error: %v", err) + } + if _, err := Marshal(dummy); err == nil { + b.Fatal("Marshal error: got nil, want non-nil") + } + } + }) + b.SetBytes(int64(len(codeJSON))) +} + +func benchMarshalBytes(n int) func(*testing.B) { + sample := []byte("hello world") + // Use a struct pointer, to avoid an allocation when passing it as an + // interface parameter to Marshal. + v := &struct { + Bytes []byte + }{ + bytes.Repeat(sample, (n/len(sample))+1)[:n], + } + return func(b *testing.B) { + for i := 0; i < b.N; i++ { + if _, err := Marshal(v); err != nil { + b.Fatalf("Marshal error: %v", err) + } + } + } +} + +func benchMarshalBytesError(n int) func(*testing.B) { + sample := []byte("hello world") + // Use a struct pointer, to avoid an allocation when passing it as an + // interface parameter to Marshal. + v := &struct { + Bytes []byte + }{ + bytes.Repeat(sample, (n/len(sample))+1)[:n], + } + + // Trigger an error in Marshal with cyclic data. + type Dummy struct { + Name string + Next *Dummy + } + dummy := Dummy{Name: "Dummy"} + dummy.Next = &dummy + + return func(b *testing.B) { + for i := 0; i < b.N; i++ { + if _, err := Marshal(v); err != nil { + b.Fatalf("Marshal error: %v", err) + } + if _, err := Marshal(dummy); err == nil { + b.Fatal("Marshal error: got nil, want non-nil") + } + } + } +} + +func BenchmarkMarshalBytes(b *testing.B) { + b.ReportAllocs() + // 32 fits within encodeState.scratch. + b.Run("32", benchMarshalBytes(32)) + // 256 doesn't fit in encodeState.scratch, but is small enough to + // allocate and avoid the slower base64.NewEncoder. + b.Run("256", benchMarshalBytes(256)) + // 4096 is large enough that we want to avoid allocating for it. + b.Run("4096", benchMarshalBytes(4096)) +} + +func BenchmarkMarshalBytesError(b *testing.B) { + b.ReportAllocs() + // 32 fits within encodeState.scratch. + b.Run("32", benchMarshalBytesError(32)) + // 256 doesn't fit in encodeState.scratch, but is small enough to + // allocate and avoid the slower base64.NewEncoder. + b.Run("256", benchMarshalBytesError(256)) + // 4096 is large enough that we want to avoid allocating for it. + b.Run("4096", benchMarshalBytesError(4096)) +} + +func BenchmarkMarshalMap(b *testing.B) { + b.ReportAllocs() + m := map[string]int{ + "key3": 3, + "key2": 2, + "key1": 1, + } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := Marshal(m); err != nil { + b.Fatal("Marshal:", err) + } + } + }) +} + +func BenchmarkCodeDecoder(b *testing.B) { + b.ReportAllocs() + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + b.RunParallel(func(pb *testing.PB) { + var buf bytes.Buffer + dec := NewDecoder(&buf) + var r codeResponse + for pb.Next() { + buf.Write(codeJSON) + // hide EOF + buf.WriteByte('\n') + buf.WriteByte('\n') + buf.WriteByte('\n') + if err := dec.Decode(&r); err != nil { + b.Fatalf("Decode error: %v", err) + } + } + }) + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkUnicodeDecoder(b *testing.B) { + b.ReportAllocs() + j := []byte(`"\uD83D\uDE01"`) + b.SetBytes(int64(len(j))) + r := bytes.NewReader(j) + dec := NewDecoder(r) + var out string + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := dec.Decode(&out); err != nil { + b.Fatalf("Decode error: %v", err) + } + r.Seek(0, 0) + } +} + +func BenchmarkDecoderStream(b *testing.B) { + b.ReportAllocs() + b.StopTimer() + var buf bytes.Buffer + dec := NewDecoder(&buf) + buf.WriteString(`"` + strings.Repeat("x", 1000000) + `"` + "\n\n\n") + var x any + if err := dec.Decode(&x); err != nil { + b.Fatalf("Decode error: %v", err) + } + ones := strings.Repeat(" 1\n", 300000) + "\n\n\n" + b.StartTimer() + for i := 0; i < b.N; i++ { + if i%300000 == 0 { + buf.WriteString(ones) + } + x = nil + switch err := dec.Decode(&x); { + case err != nil: + b.Fatalf("Decode error: %v", err) + case x != 1.0: + b.Fatalf("Decode: got %v want 1.0", i) + } + } +} + +func BenchmarkCodeUnmarshal(b *testing.B) { + b.ReportAllocs() + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + var r codeResponse + if err := Unmarshal(codeJSON, &r); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkCodeUnmarshalReuse(b *testing.B) { + b.ReportAllocs() + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + b.RunParallel(func(pb *testing.PB) { + var r codeResponse + for pb.Next() { + if err := Unmarshal(codeJSON, &r); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkUnmarshalString(b *testing.B) { + b.ReportAllocs() + data := []byte(`"hello, world"`) + b.RunParallel(func(pb *testing.PB) { + var s string + for pb.Next() { + if err := Unmarshal(data, &s); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) +} + +func BenchmarkUnmarshalFloat64(b *testing.B) { + b.ReportAllocs() + data := []byte(`3.14`) + b.RunParallel(func(pb *testing.PB) { + var f float64 + for pb.Next() { + if err := Unmarshal(data, &f); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) +} + +func BenchmarkUnmarshalInt64(b *testing.B) { + b.ReportAllocs() + data := []byte(`3`) + b.RunParallel(func(pb *testing.PB) { + var x int64 + for pb.Next() { + if err := Unmarshal(data, &x); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) +} + +func BenchmarkUnmarshalMap(b *testing.B) { + b.ReportAllocs() + data := []byte(`{"key1":"value1","key2":"value2","key3":"value3"}`) + b.RunParallel(func(pb *testing.PB) { + x := make(map[string]string, 3) + for pb.Next() { + if err := Unmarshal(data, &x); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) +} + +func BenchmarkIssue10335(b *testing.B) { + b.ReportAllocs() + j := []byte(`{"a":{ }}`) + b.RunParallel(func(pb *testing.PB) { + var s struct{} + for pb.Next() { + if err := Unmarshal(j, &s); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) +} + +func BenchmarkIssue34127(b *testing.B) { + b.ReportAllocs() + j := struct { + Bar string `json:"bar,string"` + }{ + Bar: `foobar`, + } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := Marshal(&j); err != nil { + b.Fatalf("Marshal error: %v", err) + } + } + }) +} + +func BenchmarkUnmapped(b *testing.B) { + b.ReportAllocs() + j := []byte(`{"s": "hello", "y": 2, "o": {"x": 0}, "a": [1, 99, {"x": 1}]}`) + b.RunParallel(func(pb *testing.PB) { + var s struct{} + for pb.Next() { + if err := Unmarshal(j, &s); err != nil { + b.Fatalf("Unmarshal error: %v", err) + } + } + }) +} + +func BenchmarkEncodeMarshaler(b *testing.B) { + b.ReportAllocs() + + m := struct { + A int + B RawMessage + }{} + + b.RunParallel(func(pb *testing.PB) { + enc := NewEncoder(io.Discard) + + for pb.Next() { + if err := enc.Encode(&m); err != nil { + b.Fatalf("Encode error: %v", err) + } + } + }) +} + +func BenchmarkEncoderEncode(b *testing.B) { + b.ReportAllocs() + type T struct { + X, Y string + } + v := &T{"foo", "bar"} + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := NewEncoder(io.Discard).Encode(v); err != nil { + b.Fatalf("Encode error: %v", err) + } + } + }) +} diff --git a/src/encoding/json/v2_decode.go b/src/encoding/json/v2_decode.go new file mode 100644 index 00000000000000..f17d7ebccada0a --- /dev/null +++ b/src/encoding/json/v2_decode.go @@ -0,0 +1,259 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +// Represents JSON data structure using native Go types: booleans, floats, +// strings, arrays, and maps. + +package json + +import ( + "cmp" + "fmt" + "reflect" + "strconv" + "strings" + + "encoding/json/internal/jsonwire" + "encoding/json/jsontext" + jsonv2 "encoding/json/v2" +) + +// Unmarshal parses the JSON-encoded data and stores the result +// in the value pointed to by v. If v is nil or not a pointer, +// Unmarshal returns an [InvalidUnmarshalError]. +// +// Unmarshal uses the inverse of the encodings that +// [Marshal] uses, allocating maps, slices, and pointers as necessary, +// with the following additional rules: +// +// To unmarshal JSON into a pointer, Unmarshal first handles the case of +// the JSON being the JSON literal null. In that case, Unmarshal sets +// the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into +// the value pointed at by the pointer. If the pointer is nil, Unmarshal +// allocates a new value for it to point to. +// +// To unmarshal JSON into a value implementing [Unmarshaler], +// Unmarshal calls that value's [Unmarshaler.UnmarshalJSON] method, including +// when the input is a JSON null. +// Otherwise, if the value implements [encoding.TextUnmarshaler] +// and the input is a JSON quoted string, Unmarshal calls +// [encoding.TextUnmarshaler.UnmarshalText] with the unquoted form of the string. +// +// To unmarshal JSON into a struct, Unmarshal matches incoming object +// keys to the keys used by [Marshal] (either the struct field name or its tag), +// preferring an exact match but also accepting a case-insensitive match. By +// default, object keys which don't have a corresponding struct field are +// ignored (see [Decoder.DisallowUnknownFields] for an alternative). +// +// To unmarshal JSON into an interface value, +// Unmarshal stores one of these in the interface value: +// +// - bool, for JSON booleans +// - float64, for JSON numbers +// - string, for JSON strings +// - []any, for JSON arrays +// - map[string]any, for JSON objects +// - nil for JSON null +// +// To unmarshal a JSON array into a slice, Unmarshal resets the slice length +// to zero and then appends each element to the slice. +// As a special case, to unmarshal an empty JSON array into a slice, +// Unmarshal replaces the slice with a new empty slice. +// +// To unmarshal a JSON array into a Go array, Unmarshal decodes +// JSON array elements into corresponding Go array elements. +// If the Go array is smaller than the JSON array, +// the additional JSON array elements are discarded. +// If the JSON array is smaller than the Go array, +// the additional Go array elements are set to zero values. +// +// To unmarshal a JSON object into a map, Unmarshal first establishes a map to +// use. If the map is nil, Unmarshal allocates a new map. Otherwise Unmarshal +// reuses the existing map, keeping existing entries. Unmarshal then stores +// key-value pairs from the JSON object into the map. The map's key type must +// either be any string type, an integer, or implement [encoding.TextUnmarshaler]. +// +// If the JSON-encoded data contain a syntax error, Unmarshal returns a [SyntaxError]. +// +// If a JSON value is not appropriate for a given target type, +// or if a JSON number overflows the target type, Unmarshal +// skips that field and completes the unmarshaling as best it can. +// If no more serious errors are encountered, Unmarshal returns +// an [UnmarshalTypeError] describing the earliest such error. In any +// case, it's not guaranteed that all the remaining fields following +// the problematic one will be unmarshaled into the target object. +// +// The JSON null value unmarshals into an interface, map, pointer, or slice +// by setting that Go value to nil. Because null is often used in JSON to mean +// “not present,” unmarshaling a JSON null into any other Go type has no effect +// on the value and produces no error. +// +// When unmarshaling quoted strings, invalid UTF-8 or +// invalid UTF-16 surrogate pairs are not treated as an error. +// Instead, they are replaced by the Unicode replacement +// character U+FFFD. +func Unmarshal(data []byte, v any) error { + return jsonv2.Unmarshal(data, v, DefaultOptionsV1()) +} + +// Unmarshaler is the interface implemented by types +// that can unmarshal a JSON description of themselves. +// The input can be assumed to be a valid encoding of +// a JSON value. UnmarshalJSON must copy the JSON data +// if it wishes to retain the data after returning. +type Unmarshaler = jsonv2.Unmarshaler + +// An UnmarshalTypeError describes a JSON value that was +// not appropriate for a value of a specific Go type. +type UnmarshalTypeError struct { + Value string // description of JSON value - "bool", "array", "number -5" + Type reflect.Type // type of Go value it could not be assigned to + Offset int64 // error occurred after reading Offset bytes + Struct string // name of the root type containing the field + Field string // the full path from root node to the value + Err error // may be nil +} + +func (e *UnmarshalTypeError) Error() string { + var s string + if e.Struct != "" || e.Field != "" { + // The design of UnmarshalTypeError overly assumes a struct-based + // Go representation for the JSON value. + // The logic in jsontext represents paths using a JSON Pointer, + // which is agnostic to the Go type system. + // Trying to convert a JSON Pointer into a UnmarshalTypeError.Field + // is difficult. As a heuristic, if the last path token looks like + // an index into a JSON array (e.g., ".foo.bar.0"), + // avoid the phrase "Go struct field ". + intoWhat := "Go struct field " + i := strings.LastIndexByte(e.Field, '.') + len(".") + if len(e.Field[i:]) > 0 && strings.TrimRight(e.Field[i:], "0123456789") == "" { + intoWhat = "" // likely a Go slice or array + } + s = "json: cannot unmarshal " + e.Value + " into " + intoWhat + e.Struct + "." + e.Field + " of type " + e.Type.String() + } else { + s = "json: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() + } + if e.Err != nil { + s += ": " + e.Err.Error() + } + return s +} + +func (e *UnmarshalTypeError) Unwrap() error { + return e.Err +} + +// An UnmarshalFieldError describes a JSON object key that +// led to an unexported (and therefore unwritable) struct field. +// +// Deprecated: No longer used; kept for compatibility. +type UnmarshalFieldError struct { + Key string + Type reflect.Type + Field reflect.StructField +} + +func (e *UnmarshalFieldError) Error() string { + return "json: cannot unmarshal object key " + strconv.Quote(e.Key) + " into unexported field " + e.Field.Name + " of type " + e.Type.String() +} + +// An InvalidUnmarshalError describes an invalid argument passed to [Unmarshal]. +// (The argument to [Unmarshal] must be a non-nil pointer.) +type InvalidUnmarshalError struct { + Type reflect.Type +} + +func (e *InvalidUnmarshalError) Error() string { + if e.Type == nil { + return "json: Unmarshal(nil)" + } + + if e.Type.Kind() != reflect.Pointer { + return "json: Unmarshal(non-pointer " + e.Type.String() + ")" + } + return "json: Unmarshal(nil " + e.Type.String() + ")" +} + +// A Number represents a JSON number literal. +type Number string + +// String returns the literal text of the number. +func (n Number) String() string { return string(n) } + +// Float64 returns the number as a float64. +func (n Number) Float64() (float64, error) { + return strconv.ParseFloat(string(n), 64) +} + +// Int64 returns the number as an int64. +func (n Number) Int64() (int64, error) { + return strconv.ParseInt(string(n), 10, 64) +} + +var numberType = reflect.TypeFor[Number]() + +// MarshalJSONTo implements [jsonv2.MarshalerTo]. +func (n Number) MarshalJSONTo(enc *jsontext.Encoder) error { + opts := enc.Options() + stringify, _ := jsonv2.GetOption(opts, jsonv2.StringifyNumbers) + if k, n := enc.StackIndex(enc.StackDepth()); k == '{' && n%2 == 0 { + stringify = true // expecting a JSON object name + } + n = cmp.Or(n, "0") + var num []byte + val := enc.AvailableBuffer() + if stringify { + val = append(val, '"') + val = append(val, n...) + val = append(val, '"') + num = val[len(`"`) : len(val)-len(`"`)] + } else { + val = append(val, n...) + num = val + } + if n, err := jsonwire.ConsumeNumber(num); n != len(num) || err != nil { + return fmt.Errorf("cannot parse %q as JSON number: %w", val, strconv.ErrSyntax) + } + return enc.WriteValue(val) +} + +// UnmarshalJSONFrom implements [jsonv2.UnmarshalerFrom]. +func (n *Number) UnmarshalJSONFrom(dec *jsontext.Decoder) error { + opts := dec.Options() + stringify, _ := jsonv2.GetOption(opts, jsonv2.StringifyNumbers) + if k, n := dec.StackIndex(dec.StackDepth()); k == '{' && n%2 == 0 { + stringify = true // expecting a JSON object name + } + val, err := dec.ReadValue() + if err != nil { + return err + } + val0 := val + k := val.Kind() + switch k { + case 'n': + if legacy, _ := jsonv2.GetOption(opts, MergeWithLegacySemantics); !legacy { + *n = "" + } + return nil + case '"': + verbatim := jsonwire.ConsumeSimpleString(val) == len(val) + val = jsonwire.UnquoteMayCopy(val, verbatim) + if n, err := jsonwire.ConsumeNumber(val); n != len(val) || err != nil { + return &jsonv2.SemanticError{JSONKind: val0.Kind(), JSONValue: val0.Clone(), GoType: numberType, Err: strconv.ErrSyntax} + } + *n = Number(val) + return nil + case '0': + if stringify { + break + } + *n = Number(val) + return nil + } + return &jsonv2.SemanticError{JSONKind: k, GoType: numberType} +} diff --git a/src/encoding/json/v2_decode_test.go b/src/encoding/json/v2_decode_test.go new file mode 100644 index 00000000000000..26b4448721e4ec --- /dev/null +++ b/src/encoding/json/v2_decode_test.go @@ -0,0 +1,2945 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json + +import ( + "bytes" + "encoding" + "errors" + "fmt" + "image" + "io" + "maps" + "math" + "math/big" + "net" + "reflect" + "slices" + "strconv" + "strings" + "testing" + "time" +) + +func len64(s string) int64 { + return int64(len(s)) +} + +type T struct { + X string + Y int + Z int `json:"-"` +} + +type U struct { + Alphabet string `json:"alpha"` +} + +type V struct { + F1 any + F2 int32 + F3 Number + F4 *VOuter +} + +type VOuter struct { + V V +} + +type W struct { + S SS +} + +type P struct { + PP PP +} + +type PP struct { + T T + Ts []T +} + +type SS string + +func (*SS) UnmarshalJSON(data []byte) error { + return &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[SS]()} +} + +type TAlias T + +func (tt *TAlias) UnmarshalJSON(data []byte) error { + t := T{} + if err := Unmarshal(data, &t); err != nil { + return err + } + *tt = TAlias(t) + return nil +} + +type TOuter struct { + T TAlias +} + +// ifaceNumAsFloat64/ifaceNumAsNumber are used to test unmarshaling with and +// without UseNumber +var ifaceNumAsFloat64 = map[string]any{ + "k1": float64(1), + "k2": "s", + "k3": []any{float64(1), float64(2.0), float64(3e-3)}, + "k4": map[string]any{"kk1": "s", "kk2": float64(2)}, +} + +var ifaceNumAsNumber = map[string]any{ + "k1": Number("1"), + "k2": "s", + "k3": []any{Number("1"), Number("2.0"), Number("3e-3")}, + "k4": map[string]any{"kk1": "s", "kk2": Number("2")}, +} + +type tx struct { + x int +} + +type u8 uint8 + +// A type that can unmarshal itself. + +type unmarshaler struct { + T bool +} + +func (u *unmarshaler) UnmarshalJSON(b []byte) error { + *u = unmarshaler{true} // All we need to see that UnmarshalJSON is called. + return nil +} + +type ustruct struct { + M unmarshaler +} + +type unmarshalerText struct { + A, B string +} + +// needed for re-marshaling tests +func (u unmarshalerText) MarshalText() ([]byte, error) { + return []byte(u.A + ":" + u.B), nil +} + +func (u *unmarshalerText) UnmarshalText(b []byte) error { + pos := bytes.IndexByte(b, ':') + if pos == -1 { + return errors.New("missing separator") + } + u.A, u.B = string(b[:pos]), string(b[pos+1:]) + return nil +} + +var _ encoding.TextUnmarshaler = (*unmarshalerText)(nil) + +type ustructText struct { + M unmarshalerText +} + +// u8marshal is an integer type that can marshal/unmarshal itself. +type u8marshal uint8 + +func (u8 u8marshal) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf("u%d", u8)), nil +} + +var errMissingU8Prefix = errors.New("missing 'u' prefix") + +func (u8 *u8marshal) UnmarshalText(b []byte) error { + if !bytes.HasPrefix(b, []byte{'u'}) { + return errMissingU8Prefix + } + n, err := strconv.Atoi(string(b[1:])) + if err != nil { + return err + } + *u8 = u8marshal(n) + return nil +} + +var _ encoding.TextUnmarshaler = (*u8marshal)(nil) + +var ( + umtrue = unmarshaler{true} + umslice = []unmarshaler{{true}} + umstruct = ustruct{unmarshaler{true}} + + umtrueXY = unmarshalerText{"x", "y"} + umsliceXY = []unmarshalerText{{"x", "y"}} + umstructXY = ustructText{unmarshalerText{"x", "y"}} + + ummapXY = map[unmarshalerText]bool{{"x", "y"}: true} +) + +// Test data structures for anonymous fields. + +type Point struct { + Z int +} + +type Top struct { + Level0 int + Embed0 + *Embed0a + *Embed0b `json:"e,omitempty"` // treated as named + Embed0c `json:"-"` // ignored + Loop + Embed0p // has Point with X, Y, used + Embed0q // has Point with Z, used + embed // contains exported field +} + +type Embed0 struct { + Level1a int // overridden by Embed0a's Level1a with json tag + Level1b int // used because Embed0a's Level1b is renamed + Level1c int // used because Embed0a's Level1c is ignored + Level1d int // annihilated by Embed0a's Level1d + Level1e int `json:"x"` // annihilated by Embed0a.Level1e +} + +type Embed0a struct { + Level1a int `json:"Level1a,omitempty"` + Level1b int `json:"LEVEL1B,omitempty"` + Level1c int `json:"-"` + Level1d int // annihilated by Embed0's Level1d + Level1f int `json:"x"` // annihilated by Embed0's Level1e +} + +type Embed0b Embed0 + +type Embed0c Embed0 + +type Embed0p struct { + image.Point +} + +type Embed0q struct { + Point +} + +type embed struct { + Q int +} + +type Loop struct { + Loop1 int `json:",omitempty"` + Loop2 int `json:",omitempty"` + *Loop +} + +// From reflect test: +// The X in S6 and S7 annihilate, but they also block the X in S8.S9. +type S5 struct { + S6 + S7 + S8 +} + +type S6 struct { + X int +} + +type S7 S6 + +type S8 struct { + S9 +} + +type S9 struct { + X int + Y int +} + +// From reflect test: +// The X in S11.S6 and S12.S6 annihilate, but they also block the X in S13.S8.S9. +type S10 struct { + S11 + S12 + S13 +} + +type S11 struct { + S6 +} + +type S12 struct { + S6 +} + +type S13 struct { + S8 +} + +type Ambig struct { + // Given "hello", the first match should win. + First int `json:"HELLO"` + Second int `json:"Hello"` +} + +type XYZ struct { + X any + Y any + Z any +} + +type unexportedWithMethods struct{} + +func (unexportedWithMethods) F() {} + +type byteWithMarshalJSON byte + +func (b byteWithMarshalJSON) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"Z%.2x"`, byte(b))), nil +} + +func (b *byteWithMarshalJSON) UnmarshalJSON(data []byte) error { + if len(data) != 5 || data[0] != '"' || data[1] != 'Z' || data[4] != '"' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[2:4]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = byteWithMarshalJSON(i) + return nil +} + +type byteWithPtrMarshalJSON byte + +func (b *byteWithPtrMarshalJSON) MarshalJSON() ([]byte, error) { + return byteWithMarshalJSON(*b).MarshalJSON() +} + +func (b *byteWithPtrMarshalJSON) UnmarshalJSON(data []byte) error { + return (*byteWithMarshalJSON)(b).UnmarshalJSON(data) +} + +type byteWithMarshalText byte + +func (b byteWithMarshalText) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf(`Z%.2x`, byte(b))), nil +} + +func (b *byteWithMarshalText) UnmarshalText(data []byte) error { + if len(data) != 3 || data[0] != 'Z' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[1:3]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = byteWithMarshalText(i) + return nil +} + +type byteWithPtrMarshalText byte + +func (b *byteWithPtrMarshalText) MarshalText() ([]byte, error) { + return byteWithMarshalText(*b).MarshalText() +} + +func (b *byteWithPtrMarshalText) UnmarshalText(data []byte) error { + return (*byteWithMarshalText)(b).UnmarshalText(data) +} + +type intWithMarshalJSON int + +func (b intWithMarshalJSON) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"Z%.2x"`, int(b))), nil +} + +func (b *intWithMarshalJSON) UnmarshalJSON(data []byte) error { + if len(data) != 5 || data[0] != '"' || data[1] != 'Z' || data[4] != '"' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[2:4]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = intWithMarshalJSON(i) + return nil +} + +type intWithPtrMarshalJSON int + +func (b *intWithPtrMarshalJSON) MarshalJSON() ([]byte, error) { + return intWithMarshalJSON(*b).MarshalJSON() +} + +func (b *intWithPtrMarshalJSON) UnmarshalJSON(data []byte) error { + return (*intWithMarshalJSON)(b).UnmarshalJSON(data) +} + +type intWithMarshalText int + +func (b intWithMarshalText) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf(`Z%.2x`, int(b))), nil +} + +func (b *intWithMarshalText) UnmarshalText(data []byte) error { + if len(data) != 3 || data[0] != 'Z' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[1:3]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = intWithMarshalText(i) + return nil +} + +type intWithPtrMarshalText int + +func (b *intWithPtrMarshalText) MarshalText() ([]byte, error) { + return intWithMarshalText(*b).MarshalText() +} + +func (b *intWithPtrMarshalText) UnmarshalText(data []byte) error { + return (*intWithMarshalText)(b).UnmarshalText(data) +} + +type mapStringToStringData struct { + Data map[string]string `json:"data"` +} + +type B struct { + B bool `json:",string"` +} + +type DoublePtr struct { + I **int + J **int +} + +type NestedUnamed struct{ F struct{ V int } } + +var unmarshalTests = []struct { + CaseName + in string + ptr any // new(type) + out any + err error + useNumber bool + golden bool + disallowUnknownFields bool +}{ + // basic types + {CaseName: Name(""), in: `true`, ptr: new(bool), out: true}, + {CaseName: Name(""), in: `1`, ptr: new(int), out: 1}, + {CaseName: Name(""), in: `1.2`, ptr: new(float64), out: 1.2}, + {CaseName: Name(""), in: `-5`, ptr: new(int16), out: int16(-5)}, + {CaseName: Name(""), in: `2`, ptr: new(Number), out: Number("2"), useNumber: true}, + {CaseName: Name(""), in: `2`, ptr: new(Number), out: Number("2")}, + {CaseName: Name(""), in: `2`, ptr: new(any), out: float64(2.0)}, + {CaseName: Name(""), in: `2`, ptr: new(any), out: Number("2"), useNumber: true}, + {CaseName: Name(""), in: `"a\u1234"`, ptr: new(string), out: "a\u1234"}, + {CaseName: Name(""), in: `"http:\/\/"`, ptr: new(string), out: "http://"}, + {CaseName: Name(""), in: `"g-clef: \uD834\uDD1E"`, ptr: new(string), out: "g-clef: \U0001D11E"}, + {CaseName: Name(""), in: `"invalid: \uD834x\uDD1E"`, ptr: new(string), out: "invalid: \uFFFDx\uFFFD"}, + {CaseName: Name(""), in: "null", ptr: new(any), out: nil}, + {CaseName: Name(""), in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &UnmarshalTypeError{"array", reflect.TypeFor[string](), len64(`{"X": `), "T", "X", nil}}, + {CaseName: Name(""), in: `{"X": 23}`, ptr: new(T), out: T{}, err: &UnmarshalTypeError{"number", reflect.TypeFor[string](), len64(`{"X": `), "T", "X", nil}}, + {CaseName: Name(""), in: `{"x": 1}`, ptr: new(tx), out: tx{}}, + {CaseName: Name(""), in: `{"x": 1}`, ptr: new(tx), out: tx{}}, + {CaseName: Name(""), in: `{"x": 1}`, ptr: new(tx), err: fmt.Errorf("json: unknown field \"x\""), disallowUnknownFields: true}, + {CaseName: Name(""), in: `{"S": 23}`, ptr: new(W), out: W{}, err: &UnmarshalTypeError{"number", reflect.TypeFor[SS](), 0, "", "", nil}}, + {CaseName: Name(""), in: `{"T": {"X": 23}}`, ptr: new(TOuter), out: TOuter{}, err: &UnmarshalTypeError{"number", reflect.TypeFor[string](), len64(`{"X": `), "T", "X", nil}}, + {CaseName: Name(""), in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: Number("3")}}, + {CaseName: Name(""), in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: Number("1"), F2: int32(2), F3: Number("3")}, useNumber: true}, + {CaseName: Name(""), in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(any), out: ifaceNumAsFloat64}, + {CaseName: Name(""), in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(any), out: ifaceNumAsNumber, useNumber: true}, + + // raw values with whitespace + {CaseName: Name(""), in: "\n true ", ptr: new(bool), out: true}, + {CaseName: Name(""), in: "\t 1 ", ptr: new(int), out: 1}, + {CaseName: Name(""), in: "\r 1.2 ", ptr: new(float64), out: 1.2}, + {CaseName: Name(""), in: "\t -5 \n", ptr: new(int16), out: int16(-5)}, + {CaseName: Name(""), in: "\t \"a\\u1234\" \n", ptr: new(string), out: "a\u1234"}, + + // Z has a "-" tag. + {CaseName: Name(""), in: `{"Y": 1, "Z": 2}`, ptr: new(T), out: T{Y: 1}}, + {CaseName: Name(""), in: `{"Y": 1, "Z": 2}`, ptr: new(T), out: T{Y: 1}, err: fmt.Errorf("json: unknown field \"Z\""), disallowUnknownFields: true}, + + {CaseName: Name(""), in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), out: U{Alphabet: "abc"}}, + {CaseName: Name(""), in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), out: U{Alphabet: "abc"}, err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, + {CaseName: Name(""), in: `{"alpha": "abc"}`, ptr: new(U), out: U{Alphabet: "abc"}}, + {CaseName: Name(""), in: `{"alphabet": "xyz"}`, ptr: new(U), out: U{}}, + {CaseName: Name(""), in: `{"alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, + + // syntax errors + {CaseName: Name(""), in: ``, ptr: new(any), err: &SyntaxError{errUnexpectedEnd.Error(), 0}}, + {CaseName: Name(""), in: " \n\r\t", ptr: new(any), err: &SyntaxError{errUnexpectedEnd.Error(), len64(" \n\r\t")}}, + {CaseName: Name(""), in: `[2, 3`, ptr: new(any), err: &SyntaxError{errUnexpectedEnd.Error(), len64(`[2, 3`)}}, + {CaseName: Name(""), in: `{"X": "foo", "Y"}`, err: &SyntaxError{"invalid character '}' after object key", len64(`{"X": "foo", "Y"`)}}, + {CaseName: Name(""), in: `[1, 2, 3+]`, err: &SyntaxError{"invalid character '+' after array element", len64(`[1, 2, 3`)}}, + {CaseName: Name(""), in: `{"X":12x}`, err: &SyntaxError{"invalid character 'x' after object key:value pair", len64(`{"X":12`)}, useNumber: true}, + {CaseName: Name(""), in: `{"F3": -}`, ptr: new(V), err: &SyntaxError{"invalid character '}' in numeric literal", len64(`{"F3": -`)}}, + + // raw value errors + {CaseName: Name(""), in: "\x01 42", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", len64(``)}}, + {CaseName: Name(""), in: " 42 \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", len64(` 42 `)}}, + {CaseName: Name(""), in: "\x01 true", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", len64(``)}}, + {CaseName: Name(""), in: " false \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", len64(` false `)}}, + {CaseName: Name(""), in: "\x01 1.2", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", len64(``)}}, + {CaseName: Name(""), in: " 3.4 \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", len64(` 3.4 `)}}, + {CaseName: Name(""), in: "\x01 \"string\"", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", len64(``)}}, + {CaseName: Name(""), in: " \"string\" \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", len64(` "string" `)}}, + + // array tests + {CaseName: Name(""), in: `[1, 2, 3]`, ptr: new([3]int), out: [3]int{1, 2, 3}}, + {CaseName: Name(""), in: `[1, 2, 3]`, ptr: new([1]int), out: [1]int{1}}, + {CaseName: Name(""), in: `[1, 2, 3]`, ptr: new([5]int), out: [5]int{1, 2, 3, 0, 0}}, + {CaseName: Name(""), in: `[1, 2, 3]`, ptr: new(MustNotUnmarshalJSON), err: errors.New("MustNotUnmarshalJSON was used")}, + + // empty array to interface test + {CaseName: Name(""), in: `[]`, ptr: new([]any), out: []any{}}, + {CaseName: Name(""), in: `null`, ptr: new([]any), out: []any(nil)}, + {CaseName: Name(""), in: `{"T":[]}`, ptr: new(map[string]any), out: map[string]any{"T": []any{}}}, + {CaseName: Name(""), in: `{"T":null}`, ptr: new(map[string]any), out: map[string]any{"T": any(nil)}}, + + // composite tests + {CaseName: Name(""), in: allValueIndent, ptr: new(All), out: allValue}, + {CaseName: Name(""), in: allValueCompact, ptr: new(All), out: allValue}, + {CaseName: Name(""), in: allValueIndent, ptr: new(*All), out: &allValue}, + {CaseName: Name(""), in: allValueCompact, ptr: new(*All), out: &allValue}, + {CaseName: Name(""), in: pallValueIndent, ptr: new(All), out: pallValue}, + {CaseName: Name(""), in: pallValueCompact, ptr: new(All), out: pallValue}, + {CaseName: Name(""), in: pallValueIndent, ptr: new(*All), out: &pallValue}, + {CaseName: Name(""), in: pallValueCompact, ptr: new(*All), out: &pallValue}, + + // unmarshal interface test + {CaseName: Name(""), in: `{"T":false}`, ptr: new(unmarshaler), out: umtrue}, // use "false" so test will fail if custom unmarshaler is not called + {CaseName: Name(""), in: `{"T":false}`, ptr: new(*unmarshaler), out: &umtrue}, + {CaseName: Name(""), in: `[{"T":false}]`, ptr: new([]unmarshaler), out: umslice}, + {CaseName: Name(""), in: `[{"T":false}]`, ptr: new(*[]unmarshaler), out: &umslice}, + {CaseName: Name(""), in: `{"M":{"T":"x:y"}}`, ptr: new(ustruct), out: umstruct}, + + // UnmarshalText interface test + {CaseName: Name(""), in: `"x:y"`, ptr: new(unmarshalerText), out: umtrueXY}, + {CaseName: Name(""), in: `"x:y"`, ptr: new(*unmarshalerText), out: &umtrueXY}, + {CaseName: Name(""), in: `["x:y"]`, ptr: new([]unmarshalerText), out: umsliceXY}, + {CaseName: Name(""), in: `["x:y"]`, ptr: new(*[]unmarshalerText), out: &umsliceXY}, + {CaseName: Name(""), in: `{"M":"x:y"}`, ptr: new(ustructText), out: umstructXY}, + + // integer-keyed map test + { + CaseName: Name(""), + in: `{"-1":"a","0":"b","1":"c"}`, + ptr: new(map[int]string), + out: map[int]string{-1: "a", 0: "b", 1: "c"}, + }, + { + CaseName: Name(""), + in: `{"0":"a","10":"c","9":"b"}`, + ptr: new(map[u8]string), + out: map[u8]string{0: "a", 9: "b", 10: "c"}, + }, + { + CaseName: Name(""), + in: `{"-9223372036854775808":"min","9223372036854775807":"max"}`, + ptr: new(map[int64]string), + out: map[int64]string{math.MinInt64: "min", math.MaxInt64: "max"}, + }, + { + CaseName: Name(""), + in: `{"18446744073709551615":"max"}`, + ptr: new(map[uint64]string), + out: map[uint64]string{math.MaxUint64: "max"}, + }, + { + CaseName: Name(""), + in: `{"0":false,"10":true}`, + ptr: new(map[uintptr]bool), + out: map[uintptr]bool{0: false, 10: true}, + }, + + // Check that MarshalText and UnmarshalText take precedence + // over default integer handling in map keys. + { + CaseName: Name(""), + in: `{"u2":4}`, + ptr: new(map[u8marshal]int), + out: map[u8marshal]int{2: 4}, + }, + { + CaseName: Name(""), + in: `{"2":4}`, + ptr: new(map[u8marshal]int), + out: map[u8marshal]int{}, + err: errMissingU8Prefix, + }, + + // integer-keyed map errors + { + CaseName: Name(""), + in: `{"abc":"abc"}`, + ptr: new(map[int]string), + out: map[int]string{}, + err: &UnmarshalTypeError{Value: "number abc", Type: reflect.TypeFor[int](), Field: "abc", Offset: len64(`{`)}, + }, + { + CaseName: Name(""), + in: `{"256":"abc"}`, + ptr: new(map[uint8]string), + out: map[uint8]string{}, + err: &UnmarshalTypeError{Value: "number 256", Type: reflect.TypeFor[uint8](), Field: "256", Offset: len64(`{`)}, + }, + { + CaseName: Name(""), + in: `{"128":"abc"}`, + ptr: new(map[int8]string), + out: map[int8]string{}, + err: &UnmarshalTypeError{Value: "number 128", Type: reflect.TypeFor[int8](), Field: "128", Offset: len64(`{`)}, + }, + { + CaseName: Name(""), + in: `{"-1":"abc"}`, + ptr: new(map[uint8]string), + out: map[uint8]string{}, + err: &UnmarshalTypeError{Value: "number -1", Type: reflect.TypeFor[uint8](), Field: "-1", Offset: len64(`{`)}, + }, + { + CaseName: Name(""), + in: `{"F":{"a":2,"3":4}}`, + ptr: new(map[string]map[int]int), + out: map[string]map[int]int{"F": {3: 4}}, + err: &UnmarshalTypeError{Value: "number a", Type: reflect.TypeFor[int](), Field: "F.a", Offset: len64(`{"F":{`)}, + }, + { + CaseName: Name(""), + in: `{"F":{"a":2,"3":4}}`, + ptr: new(map[string]map[uint]int), + out: map[string]map[uint]int{"F": {3: 4}}, + err: &UnmarshalTypeError{Value: "number a", Type: reflect.TypeFor[uint](), Field: "F.a", Offset: len64(`{"F":{`)}, + }, + + // Map keys can be encoding.TextUnmarshalers. + {CaseName: Name(""), in: `{"x:y":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY}, + // If multiple values for the same key exists, only the most recent value is used. + {CaseName: Name(""), in: `{"x:y":false,"x:y":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY}, + + { + CaseName: Name(""), + in: `{ + "Level0": 1, + "Level1b": 2, + "Level1c": 3, + "x": 4, + "Level1a": 5, + "LEVEL1B": 6, + "e": { + "Level1a": 8, + "Level1b": 9, + "Level1c": 10, + "Level1d": 11, + "x": 12 + }, + "Loop1": 13, + "Loop2": 14, + "X": 15, + "Y": 16, + "Z": 17, + "Q": 18 + }`, + ptr: new(Top), + out: Top{ + Level0: 1, + Embed0: Embed0{ + Level1b: 2, + Level1c: 3, + }, + Embed0a: &Embed0a{ + Level1a: 5, + Level1b: 6, + }, + Embed0b: &Embed0b{ + Level1a: 8, + Level1b: 9, + Level1c: 10, + Level1d: 11, + Level1e: 12, + }, + Loop: Loop{ + Loop1: 13, + Loop2: 14, + }, + Embed0p: Embed0p{ + Point: image.Point{X: 15, Y: 16}, + }, + Embed0q: Embed0q{ + Point: Point{Z: 17}, + }, + embed: embed{ + Q: 18, + }, + }, + }, + { + CaseName: Name(""), + in: `{"hello": 1}`, + ptr: new(Ambig), + out: Ambig{First: 1}, + }, + + { + CaseName: Name(""), + in: `{"X": 1,"Y":2}`, + ptr: new(S5), + out: S5{S8: S8{S9: S9{Y: 2}}}, + }, + { + CaseName: Name(""), + in: `{"X": 1,"Y":2}`, + ptr: new(S5), + out: S5{S8: S8{S9{Y: 2}}}, + err: fmt.Errorf("json: unknown field \"X\""), + disallowUnknownFields: true, + }, + { + CaseName: Name(""), + in: `{"X": 1,"Y":2}`, + ptr: new(S10), + out: S10{S13: S13{S8: S8{S9: S9{Y: 2}}}}, + }, + { + CaseName: Name(""), + in: `{"X": 1,"Y":2}`, + ptr: new(S10), + out: S10{S13: S13{S8{S9{Y: 2}}}}, + err: fmt.Errorf("json: unknown field \"X\""), + disallowUnknownFields: true, + }, + { + CaseName: Name(""), + in: `{"I": 0, "I": null, "J": null}`, + ptr: new(DoublePtr), + out: DoublePtr{I: nil, J: nil}, + }, + + // invalid UTF-8 is coerced to valid UTF-8. + { + CaseName: Name(""), + in: "\"hello\xffworld\"", + ptr: new(string), + out: "hello\ufffdworld", + }, + { + CaseName: Name(""), + in: "\"hello\xc2\xc2world\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + CaseName: Name(""), + in: "\"hello\xc2\xffworld\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + CaseName: Name(""), + in: "\"hello\\ud800world\"", + ptr: new(string), + out: "hello\ufffdworld", + }, + { + CaseName: Name(""), + in: "\"hello\\ud800\\ud800world\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + CaseName: Name(""), + in: "\"hello\\ud800\\ud800world\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + CaseName: Name(""), + in: "\"hello\xed\xa0\x80\xed\xb0\x80world\"", + ptr: new(string), + out: "hello\ufffd\ufffd\ufffd\ufffd\ufffd\ufffdworld", + }, + + // Used to be issue 8305, but time.Time implements encoding.TextUnmarshaler so this works now. + { + CaseName: Name(""), + in: `{"2009-11-10T23:00:00Z": "hello world"}`, + ptr: new(map[time.Time]string), + out: map[time.Time]string{time.Date(2009, 11, 10, 23, 0, 0, 0, time.UTC): "hello world"}, + }, + + // issue 8305 + { + CaseName: Name(""), + in: `{"2009-11-10T23:00:00Z": "hello world"}`, + ptr: new(map[Point]string), + out: map[Point]string{}, + err: &UnmarshalTypeError{Value: "string", Type: reflect.TypeFor[Point](), Field: `2009-11-10T23:00:00Z`, Offset: len64(`{`)}, + }, + { + CaseName: Name(""), + in: `{"asdf": "hello world"}`, + ptr: new(map[unmarshaler]string), + out: map[unmarshaler]string{}, + err: &UnmarshalTypeError{Value: "string", Type: reflect.TypeFor[unmarshaler](), Field: "asdf", Offset: len64(`{`)}, + }, + + // related to issue 13783. + // Go 1.7 changed marshaling a slice of typed byte to use the methods on the byte type, + // similar to marshaling a slice of typed int. + // These tests check that, assuming the byte type also has valid decoding methods, + // either the old base64 string encoding or the new per-element encoding can be + // successfully unmarshaled. The custom unmarshalers were accessible in earlier + // versions of Go, even though the custom marshaler was not. + { + CaseName: Name(""), + in: `"AQID"`, + ptr: new([]byteWithMarshalJSON), + out: []byteWithMarshalJSON{1, 2, 3}, + }, + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithMarshalJSON), + out: []byteWithMarshalJSON{1, 2, 3}, + golden: true, + }, + { + CaseName: Name(""), + in: `"AQID"`, + ptr: new([]byteWithMarshalText), + out: []byteWithMarshalText{1, 2, 3}, + }, + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithMarshalText), + out: []byteWithMarshalText{1, 2, 3}, + golden: true, + }, + { + CaseName: Name(""), + in: `"AQID"`, + ptr: new([]byteWithPtrMarshalJSON), + out: []byteWithPtrMarshalJSON{1, 2, 3}, + }, + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithPtrMarshalJSON), + out: []byteWithPtrMarshalJSON{1, 2, 3}, + golden: true, + }, + { + CaseName: Name(""), + in: `"AQID"`, + ptr: new([]byteWithPtrMarshalText), + out: []byteWithPtrMarshalText{1, 2, 3}, + }, + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithPtrMarshalText), + out: []byteWithPtrMarshalText{1, 2, 3}, + golden: true, + }, + + // ints work with the marshaler but not the base64 []byte case + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithMarshalJSON), + out: []intWithMarshalJSON{1, 2, 3}, + golden: true, + }, + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithMarshalText), + out: []intWithMarshalText{1, 2, 3}, + golden: true, + }, + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithPtrMarshalJSON), + out: []intWithPtrMarshalJSON{1, 2, 3}, + golden: true, + }, + { + CaseName: Name(""), + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithPtrMarshalText), + out: []intWithPtrMarshalText{1, 2, 3}, + golden: true, + }, + + {CaseName: Name(""), in: `0.000001`, ptr: new(float64), out: 0.000001, golden: true}, + {CaseName: Name(""), in: `1e-7`, ptr: new(float64), out: 1e-7, golden: true}, + {CaseName: Name(""), in: `100000000000000000000`, ptr: new(float64), out: 100000000000000000000.0, golden: true}, + {CaseName: Name(""), in: `1e+21`, ptr: new(float64), out: 1e21, golden: true}, + {CaseName: Name(""), in: `-0.000001`, ptr: new(float64), out: -0.000001, golden: true}, + {CaseName: Name(""), in: `-1e-7`, ptr: new(float64), out: -1e-7, golden: true}, + {CaseName: Name(""), in: `-100000000000000000000`, ptr: new(float64), out: -100000000000000000000.0, golden: true}, + {CaseName: Name(""), in: `-1e+21`, ptr: new(float64), out: -1e21, golden: true}, + {CaseName: Name(""), in: `999999999999999900000`, ptr: new(float64), out: 999999999999999900000.0, golden: true}, + {CaseName: Name(""), in: `9007199254740992`, ptr: new(float64), out: 9007199254740992.0, golden: true}, + {CaseName: Name(""), in: `9007199254740993`, ptr: new(float64), out: 9007199254740992.0, golden: false}, + + { + CaseName: Name(""), + in: `{"V": {"F2": "hello"}}`, + ptr: new(VOuter), + err: &UnmarshalTypeError{ + Value: "string", + Struct: "VOuter", + Field: "V.F2", + Type: reflect.TypeFor[int32](), + Offset: len64(`{"V": {"F2": `), + }, + }, + { + CaseName: Name(""), + in: `{"V": {"F4": {}, "F2": "hello"}}`, + ptr: new(VOuter), + out: VOuter{V: V{F4: &VOuter{}}}, + err: &UnmarshalTypeError{ + Value: "string", + Struct: "VOuter", + Field: "V.F2", + Type: reflect.TypeFor[int32](), + Offset: len64(`{"V": {"F4": {}, "F2": `), + }, + }, + + { + CaseName: Name(""), + in: `{"Level1a": "hello"}`, + ptr: new(Top), + out: Top{Embed0a: &Embed0a{}}, + err: &UnmarshalTypeError{ + Value: "string", + Struct: "Top", + Field: "Level1a", + Type: reflect.TypeFor[int](), + Offset: len64(`{"Level1a": `), + }, + }, + + // issue 15146. + // invalid inputs in wrongStringTests below. + {CaseName: Name(""), in: `{"B":"true"}`, ptr: new(B), out: B{true}, golden: true}, + {CaseName: Name(""), in: `{"B":"false"}`, ptr: new(B), out: B{false}, golden: true}, + {CaseName: Name(""), in: `{"B": "maybe"}`, ptr: new(B), err: &UnmarshalTypeError{Value: `string "maybe"`, Type: reflect.TypeFor[bool](), Struct: "B", Field: "B", Offset: len64(`{"B": `), Err: strconv.ErrSyntax}}, + {CaseName: Name(""), in: `{"B": "tru"}`, ptr: new(B), err: &UnmarshalTypeError{Value: `string "tru"`, Type: reflect.TypeFor[bool](), Struct: "B", Field: "B", Offset: len64(`{"B": `), Err: strconv.ErrSyntax}}, + {CaseName: Name(""), in: `{"B": "False"}`, ptr: new(B), err: &UnmarshalTypeError{Value: `string "False"`, Type: reflect.TypeFor[bool](), Struct: "B", Field: "B", Offset: len64(`{"B": `), Err: strconv.ErrSyntax}}, + {CaseName: Name(""), in: `{"B": "null"}`, ptr: new(B), out: B{false}}, + {CaseName: Name(""), in: `{"B": "nul"}`, ptr: new(B), err: &UnmarshalTypeError{Value: `string "nul"`, Type: reflect.TypeFor[bool](), Struct: "B", Field: "B", Offset: len64(`{"B": `), Err: strconv.ErrSyntax}}, + {CaseName: Name(""), in: `{"B": [2, 3]}`, ptr: new(B), err: &UnmarshalTypeError{Value: "array", Type: reflect.TypeFor[bool](), Struct: "B", Field: "B", Offset: len64(`{"B": `)}}, + + // additional tests for disallowUnknownFields + { + CaseName: Name(""), + in: `{ + "Level0": 1, + "Level1b": 2, + "Level1c": 3, + "x": 4, + "Level1a": 5, + "LEVEL1B": 6, + "e": { + "Level1a": 8, + "Level1b": 9, + "Level1c": 10, + "Level1d": 11, + "x": 12 + }, + "Loop1": 13, + "Loop2": 14, + "X": 15, + "Y": 16, + "Z": 17, + "Q": 18, + "extra": true + }`, + ptr: new(Top), + out: Top{ + Level0: 1, + Embed0: Embed0{ + Level1b: 2, + Level1c: 3, + }, + Embed0a: &Embed0a{Level1a: 5, Level1b: 6}, + Embed0b: &Embed0b{Level1a: 8, Level1b: 9, Level1c: 10, Level1d: 11, Level1e: 12}, + Loop: Loop{ + Loop1: 13, + Loop2: 14, + Loop: nil, + }, + Embed0p: Embed0p{ + Point: image.Point{ + X: 15, + Y: 16, + }, + }, + Embed0q: Embed0q{Point: Point{Z: 17}}, + embed: embed{Q: 18}, + }, + err: fmt.Errorf("json: unknown field \"extra\""), + disallowUnknownFields: true, + }, + { + CaseName: Name(""), + in: `{ + "Level0": 1, + "Level1b": 2, + "Level1c": 3, + "x": 4, + "Level1a": 5, + "LEVEL1B": 6, + "e": { + "Level1a": 8, + "Level1b": 9, + "Level1c": 10, + "Level1d": 11, + "x": 12, + "extra": null + }, + "Loop1": 13, + "Loop2": 14, + "X": 15, + "Y": 16, + "Z": 17, + "Q": 18 + }`, + ptr: new(Top), + out: Top{ + Level0: 1, + Embed0: Embed0{ + Level1b: 2, + Level1c: 3, + }, + Embed0a: &Embed0a{Level1a: 5, Level1b: 6}, + Embed0b: &Embed0b{Level1a: 8, Level1b: 9, Level1c: 10, Level1d: 11, Level1e: 12}, + Loop: Loop{ + Loop1: 13, + Loop2: 14, + Loop: nil, + }, + Embed0p: Embed0p{ + Point: image.Point{ + X: 15, + Y: 16, + }, + }, + Embed0q: Embed0q{Point: Point{Z: 17}}, + embed: embed{Q: 18}, + }, + err: fmt.Errorf("json: unknown field \"extra\""), + disallowUnknownFields: true, + }, + // issue 26444 + // UnmarshalTypeError without field & struct values + { + CaseName: Name(""), + in: `{"data":{"test1": "bob", "test2": 123}}`, + ptr: new(mapStringToStringData), + out: mapStringToStringData{map[string]string{"test1": "bob", "test2": ""}}, + err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[string](), Offset: len64(`{"data":{"test1": "bob", "test2": `), Struct: "mapStringToStringData", Field: "data.test2"}, + }, + { + CaseName: Name(""), + in: `{"data":{"test1": 123, "test2": "bob"}}`, + ptr: new(mapStringToStringData), + out: mapStringToStringData{Data: map[string]string{"test1": "", "test2": "bob"}}, + err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[string](), Offset: len64(`{"data":{"test1": `), Struct: "mapStringToStringData", Field: "data.test1"}, + }, + + // trying to decode JSON arrays or objects via TextUnmarshaler + { + CaseName: Name(""), + in: `[1, 2, 3]`, + ptr: new(MustNotUnmarshalText), + err: &UnmarshalTypeError{Value: "array", Type: reflect.TypeFor[MustNotUnmarshalText](), Err: errors.New("JSON value must be string type")}, + }, + { + CaseName: Name(""), + in: `{"foo": "bar"}`, + ptr: new(MustNotUnmarshalText), + err: &UnmarshalTypeError{Value: "object", Type: reflect.TypeFor[MustNotUnmarshalText](), Err: errors.New("JSON value must be string type")}, + }, + // #22369 + { + CaseName: Name(""), + in: `{"PP": {"T": {"Y": "bad-type"}}}`, + ptr: new(P), + err: &UnmarshalTypeError{ + Value: "string", + Struct: "P", + Field: "PP.T.Y", + Type: reflect.TypeFor[int](), + Offset: len64(`{"PP": {"T": {"Y": `), + }, + }, + { + CaseName: Name(""), + in: `{"Ts": [{"Y": 1}, {"Y": 2}, {"Y": "bad-type"}]}`, + ptr: new(PP), + out: PP{Ts: []T{{Y: 1}, {Y: 2}, {Y: 0}}}, + err: &UnmarshalTypeError{ + Value: "string", + Struct: "PP", + Field: "Ts.2.Y", + Type: reflect.TypeFor[int](), + Offset: len64(`{"Ts": [{"Y": 1}, {"Y": 2}, {"Y": `), + }, + }, + // #14702 + { + CaseName: Name(""), + in: `invalid`, + ptr: new(Number), + err: &SyntaxError{ + msg: "invalid character 'i' looking for beginning of value", + Offset: len64(``), + }, + }, + { + CaseName: Name(""), + in: `"invalid"`, + ptr: new(Number), + err: &UnmarshalTypeError{Value: `string "invalid"`, Type: reflect.TypeFor[Number](), Err: strconv.ErrSyntax}, + }, + { + CaseName: Name(""), + in: `{"A":"invalid"}`, + ptr: new(struct{ A Number }), + err: &UnmarshalTypeError{Value: `string "invalid"`, Type: reflect.TypeFor[Number](), Err: strconv.ErrSyntax}, + }, + { + CaseName: Name(""), + in: `{"A":"invalid"}`, + ptr: new(struct { + A Number `json:",string"` + }), + err: &UnmarshalTypeError{Value: `string "invalid"`, Type: reflect.TypeFor[Number](), Err: strconv.ErrSyntax}, + }, + { + CaseName: Name(""), + in: `{"A":"invalid"}`, + ptr: new(map[string]Number), + out: map[string]Number{"A": ""}, + err: &UnmarshalTypeError{Value: `string "invalid"`, Type: reflect.TypeFor[Number](), Err: strconv.ErrSyntax}, + }, + + { + CaseName: Name(""), + in: `5`, + ptr: new(Number), + out: Number("5"), + }, + { + CaseName: Name(""), + in: `"5"`, + ptr: new(Number), + out: Number("5"), + }, + { + CaseName: Name(""), + in: `{"N":5}`, + ptr: new(struct{ N Number }), + out: struct{ N Number }{"5"}, + }, + { + CaseName: Name(""), + in: `{"N":"5"}`, + ptr: new(struct{ N Number }), + out: struct{ N Number }{"5"}, + }, + { + CaseName: Name(""), + in: `{"N":5}`, + ptr: new(struct { + N Number `json:",string"` + }), + err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[Number]()}, + }, + { + CaseName: Name(""), + in: `{"N":"5"}`, + ptr: new(struct { + N Number `json:",string"` + }), + out: struct { + N Number `json:",string"` + }{"5"}, + }, + + // Verify that syntactic errors are immediately fatal, + // while semantic errors are lazily reported + // (i.e., allow processing to continue). + { + CaseName: Name(""), + in: `[1,2,true,4,5}`, + ptr: new([]int), + err: &SyntaxError{msg: "invalid character '}' after array element", Offset: len64(`[1,2,true,4,5`)}, + }, + { + CaseName: Name(""), + in: `[1,2,true,4,5]`, + ptr: new([]int), + out: []int{1, 2, 0, 4, 5}, + err: &UnmarshalTypeError{Value: "bool", Type: reflect.TypeFor[int](), Field: "2", Offset: len64(`[1,2,`)}, + }, + + { + CaseName: Name("DashComma"), + in: `{"-":"hello"}`, + ptr: new(struct { + F string `json:"-,"` + }), + out: struct { + F string `json:"-,"` + }{"hello"}, + }, + { + CaseName: Name("DashCommaOmitEmpty"), + in: `{"-":"hello"}`, + ptr: new(struct { + F string `json:"-,omitempty"` + }), + out: struct { + F string `json:"-,omitempty"` + }{"hello"}, + }, + + { + CaseName: Name("ErrorForNestedUnamed"), + in: `{"F":{"V":"s"}}`, + ptr: new(NestedUnamed), + out: NestedUnamed{}, + err: &UnmarshalTypeError{Value: "string", Type: reflect.TypeFor[int](), Offset: 10, Struct: "NestedUnamed", Field: "F.V"}, + }, + { + CaseName: Name("ErrorInterface"), + in: `1`, + ptr: new(error), + out: error(nil), + err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[error]()}, + }, + { + CaseName: Name("ErrorChan"), + in: `1`, + ptr: new(chan int), + out: (chan int)(nil), + err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[chan int]()}, + }, + + // #75619 + { + CaseName: Name("QuotedInt/GoSyntax"), + in: `{"X": "-0000123"}`, + ptr: new(struct { + X int64 `json:",string"` + }), + out: struct { + X int64 `json:",string"` + }{-123}, + }, + { + CaseName: Name("QuotedInt/Invalid"), + in: `{"X": "123 "}`, + ptr: new(struct { + X int64 `json:",string"` + }), + err: &UnmarshalTypeError{Value: "number 123 ", Type: reflect.TypeFor[int64](), Field: "X", Offset: int64(len(`{"X": `))}, + }, + { + CaseName: Name("QuotedUint/GoSyntax"), + in: `{"X": "0000123"}`, + ptr: new(struct { + X uint64 `json:",string"` + }), + out: struct { + X uint64 `json:",string"` + }{123}, + }, + { + CaseName: Name("QuotedUint/Invalid"), + in: `{"X": "0x123"}`, + ptr: new(struct { + X uint64 `json:",string"` + }), + err: &UnmarshalTypeError{Value: "number 0x123", Type: reflect.TypeFor[uint64](), Field: "X", Offset: int64(len(`{"X": `))}, + }, + { + CaseName: Name("QuotedFloat/GoSyntax"), + in: `{"X": "0x1_4p-2"}`, + ptr: new(struct { + X float64 `json:",string"` + }), + out: struct { + X float64 `json:",string"` + }{0x1_4p-2}, + }, + { + CaseName: Name("QuotedFloat/Invalid"), + in: `{"X": "1.5e1_"}`, + ptr: new(struct { + X float64 `json:",string"` + }), + err: &UnmarshalTypeError{Value: "number 1.5e1_", Type: reflect.TypeFor[float64](), Field: "X", Offset: int64(len(`{"X": `))}, + }, +} + +func TestMarshal(t *testing.T) { + b, err := Marshal(allValue) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + if string(b) != allValueCompact { + t.Errorf("Marshal:") + diff(t, b, []byte(allValueCompact)) + return + } + + b, err = Marshal(pallValue) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + if string(b) != pallValueCompact { + t.Errorf("Marshal:") + diff(t, b, []byte(pallValueCompact)) + return + } +} + +func TestMarshalInvalidUTF8(t *testing.T) { + tests := []struct { + CaseName + in string + want string + }{ + {Name(""), "hello\xffworld", "\"hello\ufffdworld\""}, + {Name(""), "", `""`}, + {Name(""), "\xff", "\"\ufffd\""}, + {Name(""), "\xff\xff", "\"\ufffd\ufffd\""}, + {Name(""), "a\xffb", "\"a\ufffdb\""}, + {Name(""), "\xe6\x97\xa5\xe6\x9c\xac\xff\xaa\x9e", "\"日本\ufffd\ufffd\ufffd\""}, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + got, err := Marshal(tt.in) + if string(got) != tt.want || err != nil { + t.Errorf("%s: Marshal(%q):\n\tgot: (%q, %v)\n\twant: (%q, nil)", tt.Where, tt.in, got, err, tt.want) + } + }) + } +} + +func TestMarshalNumberZeroVal(t *testing.T) { + var n Number + out, err := Marshal(n) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + got := string(out) + if got != "0" { + t.Fatalf("Marshal: got %s, want 0", got) + } +} + +func TestMarshalEmbeds(t *testing.T) { + top := &Top{ + Level0: 1, + Embed0: Embed0{ + Level1b: 2, + Level1c: 3, + }, + Embed0a: &Embed0a{ + Level1a: 5, + Level1b: 6, + }, + Embed0b: &Embed0b{ + Level1a: 8, + Level1b: 9, + Level1c: 10, + Level1d: 11, + Level1e: 12, + }, + Loop: Loop{ + Loop1: 13, + Loop2: 14, + }, + Embed0p: Embed0p{ + Point: image.Point{X: 15, Y: 16}, + }, + Embed0q: Embed0q{ + Point: Point{Z: 17}, + }, + embed: embed{ + Q: 18, + }, + } + got, err := Marshal(top) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + want := "{\"Level0\":1,\"Level1b\":2,\"Level1c\":3,\"Level1a\":5,\"LEVEL1B\":6,\"e\":{\"Level1a\":8,\"Level1b\":9,\"Level1c\":10,\"Level1d\":11,\"x\":12},\"Loop1\":13,\"Loop2\":14,\"X\":15,\"Y\":16,\"Z\":17,\"Q\":18}" + if string(got) != want { + t.Errorf("Marshal:\n\tgot: %s\n\twant: %s", got, want) + } +} + +func equalError(a, b error) bool { + isJSONError := func(err error) bool { + switch err.(type) { + case + *InvalidUTF8Error, + *InvalidUnmarshalError, + *MarshalerError, + *SyntaxError, + *UnmarshalFieldError, + *UnmarshalTypeError, + *UnsupportedTypeError, + *UnsupportedValueError: + return true + } + return false + } + + if a == nil || b == nil { + return a == nil && b == nil + } + if isJSONError(a) || isJSONError(b) { + return reflect.DeepEqual(a, b) // safe for locally defined error types + } + return a.Error() == b.Error() +} + +func TestUnmarshal(t *testing.T) { + for _, tt := range unmarshalTests { + t.Run(tt.Name, func(t *testing.T) { + in := []byte(tt.in) + if err := checkValid(in); err != nil { + if !equalError(err, tt.err) { + t.Fatalf("%s: checkValid error:\n\tgot %#v\n\twant %#v", tt.Where, err, tt.err) + } + } + if tt.ptr == nil { + return + } + + typ := reflect.TypeOf(tt.ptr) + if typ.Kind() != reflect.Pointer { + t.Fatalf("%s: unmarshalTest.ptr %T is not a pointer type", tt.Where, tt.ptr) + } + typ = typ.Elem() + + // v = new(right-type) + v := reflect.New(typ) + + if !reflect.DeepEqual(tt.ptr, v.Interface()) { + // There's no reason for ptr to point to non-zero data, + // as we decode into new(right-type), so the data is + // discarded. + // This can easily mean tests that silently don't test + // what they should. To test decoding into existing + // data, see TestPrefilled. + t.Fatalf("%s: unmarshalTest.ptr %#v is not a pointer to a zero value", tt.Where, tt.ptr) + } + + dec := NewDecoder(bytes.NewReader(in)) + if tt.useNumber { + dec.UseNumber() + } + if tt.disallowUnknownFields { + dec.DisallowUnknownFields() + } + if tt.err != nil && strings.Contains(tt.err.Error(), errUnexpectedEnd.Error()) { + // In streaming mode, we expect EOF or ErrUnexpectedEOF instead. + if strings.TrimSpace(tt.in) == "" { + tt.err = io.EOF + } else { + tt.err = io.ErrUnexpectedEOF + } + } + if err := dec.Decode(v.Interface()); !equalError(err, tt.err) { + t.Fatalf("%s: Decode error:\n\tgot: %v\n\twant: %v\n\n\tgot: %#v\n\twant: %#v", tt.Where, err, tt.err, err, tt.err) + } else if err != nil && tt.out == nil { + // Initialize tt.out during an error where there are no mutations, + // so the output is just the zero value of the input type. + tt.out = reflect.Zero(v.Elem().Type()).Interface() + } + if got := v.Elem().Interface(); !reflect.DeepEqual(got, tt.out) { + gotJSON, _ := Marshal(got) + wantJSON, _ := Marshal(tt.out) + t.Fatalf("%s: Decode:\n\tgot: %#+v\n\twant: %#+v\n\n\tgotJSON: %s\n\twantJSON: %s", tt.Where, got, tt.out, gotJSON, wantJSON) + } + + // Check round trip also decodes correctly. + if tt.err == nil { + enc, err := Marshal(v.Interface()) + if err != nil { + t.Fatalf("%s: Marshal error after roundtrip: %v", tt.Where, err) + } + if tt.golden && !bytes.Equal(enc, in) { + t.Errorf("%s: Marshal:\n\tgot: %s\n\twant: %s", tt.Where, enc, in) + } + vv := reflect.New(reflect.TypeOf(tt.ptr).Elem()) + dec = NewDecoder(bytes.NewReader(enc)) + if tt.useNumber { + dec.UseNumber() + } + if err := dec.Decode(vv.Interface()); err != nil { + t.Fatalf("%s: Decode(%#q) error after roundtrip: %v", tt.Where, enc, err) + } + if !reflect.DeepEqual(v.Elem().Interface(), vv.Elem().Interface()) { + t.Fatalf("%s: Decode:\n\tgot: %#+v\n\twant: %#+v\n\n\tgotJSON: %s\n\twantJSON: %s", + tt.Where, v.Elem().Interface(), vv.Elem().Interface(), + stripWhitespace(string(enc)), stripWhitespace(string(in))) + } + } + }) + } +} + +func TestUnmarshalMarshal(t *testing.T) { + initBig() + var v any + if err := Unmarshal(jsonBig, &v); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + b, err := Marshal(v) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + if !bytes.Equal(jsonBig, b) { + t.Errorf("Marshal:") + diff(t, b, jsonBig) + return + } +} + +// Independent of Decode, basic coverage of the accessors in Number +func TestNumberAccessors(t *testing.T) { + tests := []struct { + CaseName + in string + i int64 + intErr string + f float64 + floatErr string + }{ + {CaseName: Name(""), in: "-1.23e1", intErr: "strconv.ParseInt: parsing \"-1.23e1\": invalid syntax", f: -1.23e1}, + {CaseName: Name(""), in: "-12", i: -12, f: -12.0}, + {CaseName: Name(""), in: "1e1000", intErr: "strconv.ParseInt: parsing \"1e1000\": invalid syntax", floatErr: "strconv.ParseFloat: parsing \"1e1000\": value out of range"}, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + n := Number(tt.in) + if got := n.String(); got != tt.in { + t.Errorf("%s: Number(%q).String() = %s, want %s", tt.Where, tt.in, got, tt.in) + } + if i, err := n.Int64(); err == nil && tt.intErr == "" && i != tt.i { + t.Errorf("%s: Number(%q).Int64() = %d, want %d", tt.Where, tt.in, i, tt.i) + } else if (err == nil && tt.intErr != "") || (err != nil && err.Error() != tt.intErr) { + t.Errorf("%s: Number(%q).Int64() error:\n\tgot: %v\n\twant: %v", tt.Where, tt.in, err, tt.intErr) + } + if f, err := n.Float64(); err == nil && tt.floatErr == "" && f != tt.f { + t.Errorf("%s: Number(%q).Float64() = %g, want %g", tt.Where, tt.in, f, tt.f) + } else if (err == nil && tt.floatErr != "") || (err != nil && err.Error() != tt.floatErr) { + t.Errorf("%s: Number(%q).Float64() error:\n\tgot %v\n\twant: %v", tt.Where, tt.in, err, tt.floatErr) + } + }) + } +} + +func TestLargeByteSlice(t *testing.T) { + s0 := make([]byte, 2000) + for i := range s0 { + s0[i] = byte(i) + } + b, err := Marshal(s0) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + var s1 []byte + if err := Unmarshal(b, &s1); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if !bytes.Equal(s0, s1) { + t.Errorf("Marshal:") + diff(t, s0, s1) + } +} + +type Xint struct { + X int +} + +func TestUnmarshalInterface(t *testing.T) { + var xint Xint + var i any = &xint + if err := Unmarshal([]byte(`{"X":1}`), &i); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if xint.X != 1 { + t.Fatalf("xint.X = %d, want 1", xint.X) + } +} + +func TestUnmarshalPtrPtr(t *testing.T) { + var xint Xint + pxint := &xint + if err := Unmarshal([]byte(`{"X":1}`), &pxint); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if xint.X != 1 { + t.Fatalf("xint.X = %d, want 1", xint.X) + } +} + +func TestEscape(t *testing.T) { + const input = `"foobar"` + " [\u2028 \u2029]" + const want = `"\"foobar\"\u003chtml\u003e [\u2028 \u2029]"` + got, err := Marshal(input) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + if string(got) != want { + t.Errorf("Marshal(%#q):\n\tgot: %s\n\twant: %s", input, got, want) + } +} + +// If people misuse the ,string modifier, the error message should be +// helpful, telling the user that they're doing it wrong. +func TestErrorMessageFromMisusedString(t *testing.T) { + // WrongString is a struct that's misusing the ,string modifier. + type WrongString struct { + Message string `json:"result,string"` + } + tests := []struct { + CaseName + in, err string + }{ + {Name(""), `{"result":"x"}`, `json: cannot unmarshal string into Go struct field WrongString.result of type string: invalid character 'x' looking for beginning of object key string`}, + {Name(""), `{"result":"foo"}`, `json: cannot unmarshal string into Go struct field WrongString.result of type string: invalid character 'f' looking for beginning of object key string`}, + {Name(""), `{"result":"123"}`, `json: cannot unmarshal string into Go struct field WrongString.result of type string: invalid character '1' looking for beginning of object key string`}, + {Name(""), `{"result":123}`, `json: cannot unmarshal number into Go struct field WrongString.result of type string`}, + {Name(""), `{"result":"\""}`, `json: cannot unmarshal string into Go struct field WrongString.result of type string: unexpected end of JSON input`}, + {Name(""), `{"result":"\"foo"}`, `json: cannot unmarshal string into Go struct field WrongString.result of type string: unexpected end of JSON input`}, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + r := strings.NewReader(tt.in) + var s WrongString + err := NewDecoder(r).Decode(&s) + got := fmt.Sprintf("%v", err) + if got != tt.err { + t.Errorf("%s: Decode error:\n\tgot: %s\n\twant: %s", tt.Where, got, tt.err) + } + }) + } +} + +type All struct { + Bool bool + Int int + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + Uintptr uintptr + Float32 float32 + Float64 float64 + + Foo string `json:"bar"` + Foo2 string `json:"bar2,dummyopt"` + + IntStr int64 `json:",string"` + UintptrStr uintptr `json:",string"` + + PBool *bool + PInt *int + PInt8 *int8 + PInt16 *int16 + PInt32 *int32 + PInt64 *int64 + PUint *uint + PUint8 *uint8 + PUint16 *uint16 + PUint32 *uint32 + PUint64 *uint64 + PUintptr *uintptr + PFloat32 *float32 + PFloat64 *float64 + + String string + PString *string + + Map map[string]Small + MapP map[string]*Small + PMap *map[string]Small + PMapP *map[string]*Small + + EmptyMap map[string]Small + NilMap map[string]Small + + Slice []Small + SliceP []*Small + PSlice *[]Small + PSliceP *[]*Small + + EmptySlice []Small + NilSlice []Small + + StringSlice []string + ByteSlice []byte + + Small Small + PSmall *Small + PPSmall **Small + + Interface any + PInterface *any + + unexported int +} + +type Small struct { + Tag string +} + +var allValue = All{ + Bool: true, + Int: 2, + Int8: 3, + Int16: 4, + Int32: 5, + Int64: 6, + Uint: 7, + Uint8: 8, + Uint16: 9, + Uint32: 10, + Uint64: 11, + Uintptr: 12, + Float32: 14.1, + Float64: 15.1, + Foo: "foo", + Foo2: "foo2", + IntStr: 42, + UintptrStr: 44, + String: "16", + Map: map[string]Small{ + "17": {Tag: "tag17"}, + "18": {Tag: "tag18"}, + }, + MapP: map[string]*Small{ + "19": {Tag: "tag19"}, + "20": nil, + }, + EmptyMap: map[string]Small{}, + Slice: []Small{{Tag: "tag20"}, {Tag: "tag21"}}, + SliceP: []*Small{{Tag: "tag22"}, nil, {Tag: "tag23"}}, + EmptySlice: []Small{}, + StringSlice: []string{"str24", "str25", "str26"}, + ByteSlice: []byte{27, 28, 29}, + Small: Small{Tag: "tag30"}, + PSmall: &Small{Tag: "tag31"}, + Interface: 5.2, +} + +var pallValue = All{ + PBool: &allValue.Bool, + PInt: &allValue.Int, + PInt8: &allValue.Int8, + PInt16: &allValue.Int16, + PInt32: &allValue.Int32, + PInt64: &allValue.Int64, + PUint: &allValue.Uint, + PUint8: &allValue.Uint8, + PUint16: &allValue.Uint16, + PUint32: &allValue.Uint32, + PUint64: &allValue.Uint64, + PUintptr: &allValue.Uintptr, + PFloat32: &allValue.Float32, + PFloat64: &allValue.Float64, + PString: &allValue.String, + PMap: &allValue.Map, + PMapP: &allValue.MapP, + PSlice: &allValue.Slice, + PSliceP: &allValue.SliceP, + PPSmall: &allValue.PSmall, + PInterface: &allValue.Interface, +} + +var allValueIndent = `{ + "Bool": true, + "Int": 2, + "Int8": 3, + "Int16": 4, + "Int32": 5, + "Int64": 6, + "Uint": 7, + "Uint8": 8, + "Uint16": 9, + "Uint32": 10, + "Uint64": 11, + "Uintptr": 12, + "Float32": 14.1, + "Float64": 15.1, + "bar": "foo", + "bar2": "foo2", + "IntStr": "42", + "UintptrStr": "44", + "PBool": null, + "PInt": null, + "PInt8": null, + "PInt16": null, + "PInt32": null, + "PInt64": null, + "PUint": null, + "PUint8": null, + "PUint16": null, + "PUint32": null, + "PUint64": null, + "PUintptr": null, + "PFloat32": null, + "PFloat64": null, + "String": "16", + "PString": null, + "Map": { + "17": { + "Tag": "tag17" + }, + "18": { + "Tag": "tag18" + } + }, + "MapP": { + "19": { + "Tag": "tag19" + }, + "20": null + }, + "PMap": null, + "PMapP": null, + "EmptyMap": {}, + "NilMap": null, + "Slice": [ + { + "Tag": "tag20" + }, + { + "Tag": "tag21" + } + ], + "SliceP": [ + { + "Tag": "tag22" + }, + null, + { + "Tag": "tag23" + } + ], + "PSlice": null, + "PSliceP": null, + "EmptySlice": [], + "NilSlice": null, + "StringSlice": [ + "str24", + "str25", + "str26" + ], + "ByteSlice": "Gxwd", + "Small": { + "Tag": "tag30" + }, + "PSmall": { + "Tag": "tag31" + }, + "PPSmall": null, + "Interface": 5.2, + "PInterface": null +}` + +var allValueCompact = stripWhitespace(allValueIndent) + +var pallValueIndent = `{ + "Bool": false, + "Int": 0, + "Int8": 0, + "Int16": 0, + "Int32": 0, + "Int64": 0, + "Uint": 0, + "Uint8": 0, + "Uint16": 0, + "Uint32": 0, + "Uint64": 0, + "Uintptr": 0, + "Float32": 0, + "Float64": 0, + "bar": "", + "bar2": "", + "IntStr": "0", + "UintptrStr": "0", + "PBool": true, + "PInt": 2, + "PInt8": 3, + "PInt16": 4, + "PInt32": 5, + "PInt64": 6, + "PUint": 7, + "PUint8": 8, + "PUint16": 9, + "PUint32": 10, + "PUint64": 11, + "PUintptr": 12, + "PFloat32": 14.1, + "PFloat64": 15.1, + "String": "", + "PString": "16", + "Map": null, + "MapP": null, + "PMap": { + "17": { + "Tag": "tag17" + }, + "18": { + "Tag": "tag18" + } + }, + "PMapP": { + "19": { + "Tag": "tag19" + }, + "20": null + }, + "EmptyMap": null, + "NilMap": null, + "Slice": null, + "SliceP": null, + "PSlice": [ + { + "Tag": "tag20" + }, + { + "Tag": "tag21" + } + ], + "PSliceP": [ + { + "Tag": "tag22" + }, + null, + { + "Tag": "tag23" + } + ], + "EmptySlice": null, + "NilSlice": null, + "StringSlice": null, + "ByteSlice": null, + "Small": { + "Tag": "" + }, + "PSmall": null, + "PPSmall": { + "Tag": "tag31" + }, + "Interface": null, + "PInterface": 5.2 +}` + +var pallValueCompact = stripWhitespace(pallValueIndent) + +func TestRefUnmarshal(t *testing.T) { + type S struct { + // Ref is defined in encode_test.go. + R0 Ref + R1 *Ref + R2 RefText + R3 *RefText + } + want := S{ + R0: 12, + R1: new(Ref), + R2: 13, + R3: new(RefText), + } + *want.R1 = 12 + *want.R3 = 13 + + var got S + if err := Unmarshal([]byte(`{"R0":"ref","R1":"ref","R2":"ref","R3":"ref"}`), &got); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Unmarsha:\n\tgot: %+v\n\twant: %+v", got, want) + } +} + +// Test that the empty string doesn't panic decoding when ,string is specified +// Issue 3450 +func TestEmptyString(t *testing.T) { + type T2 struct { + Number1 int `json:",string"` + Number2 int `json:",string"` + } + data := `{"Number1":"1", "Number2":""}` + dec := NewDecoder(strings.NewReader(data)) + var got T2 + switch err := dec.Decode(&got); { + case err == nil: + t.Fatalf("Decode error: got nil, want non-nil") + case got.Number1 != 1: + t.Fatalf("Decode: got.Number1 = %d, want 1", got.Number1) + } +} + +// Test that a null for ,string is not replaced with the previous quoted string (issue 7046). +// It should also not be an error (issue 2540, issue 8587). +func TestNullString(t *testing.T) { + type T struct { + A int `json:",string"` + B int `json:",string"` + C *int `json:",string"` + } + data := []byte(`{"A": "1", "B": null, "C": null}`) + var s T + s.B = 1 + s.C = new(int) + *s.C = 2 + switch err := Unmarshal(data, &s); { + case err != nil: + t.Fatalf("Unmarshal error: %v", err) + case s.B != 1: + t.Fatalf("Unmarshal: s.B = %d, want 1", s.B) + case s.C != nil: + t.Fatalf("Unmarshal: s.C = %d, want non-nil", s.C) + } +} + +func addr[T any](v T) *T { + return &v +} + +func TestInterfaceSet(t *testing.T) { + errUnmarshal := &UnmarshalTypeError{Value: "object", Offset: 5, Type: reflect.TypeFor[int](), Field: "X"} + tests := []struct { + CaseName + pre any + json string + post any + }{ + {Name(""), "foo", `"bar"`, "bar"}, + {Name(""), "foo", `2`, 2.0}, + {Name(""), "foo", `true`, true}, + {Name(""), "foo", `null`, nil}, + {Name(""), map[string]any{}, `true`, true}, + {Name(""), []string{}, `true`, true}, + + {Name(""), any(nil), `null`, any(nil)}, + {Name(""), (*int)(nil), `null`, any(nil)}, + {Name(""), (*int)(addr(0)), `null`, any(nil)}, + {Name(""), (*int)(addr(1)), `null`, any(nil)}, + {Name(""), (**int)(nil), `null`, any(nil)}, + {Name(""), (**int)(addr[*int](nil)), `null`, (**int)(addr[*int](nil))}, + {Name(""), (**int)(addr(addr(1))), `null`, (**int)(addr[*int](nil))}, + {Name(""), (***int)(nil), `null`, any(nil)}, + {Name(""), (***int)(addr[**int](nil)), `null`, (***int)(addr[**int](nil))}, + {Name(""), (***int)(addr(addr[*int](nil))), `null`, (***int)(addr[**int](nil))}, + {Name(""), (***int)(addr(addr(addr(1)))), `null`, (***int)(addr[**int](nil))}, + + {Name(""), any(nil), `2`, float64(2)}, + {Name(""), (int)(1), `2`, float64(2)}, + {Name(""), (*int)(nil), `2`, float64(2)}, + {Name(""), (*int)(addr(0)), `2`, (*int)(addr(2))}, + {Name(""), (*int)(addr(1)), `2`, (*int)(addr(2))}, + {Name(""), (**int)(nil), `2`, float64(2)}, + {Name(""), (**int)(addr[*int](nil)), `2`, (**int)(addr(addr(2)))}, + {Name(""), (**int)(addr(addr(1))), `2`, (**int)(addr(addr(2)))}, + {Name(""), (***int)(nil), `2`, float64(2)}, + {Name(""), (***int)(addr[**int](nil)), `2`, (***int)(addr(addr(addr(2))))}, + {Name(""), (***int)(addr(addr[*int](nil))), `2`, (***int)(addr(addr(addr(2))))}, + {Name(""), (***int)(addr(addr(addr(1)))), `2`, (***int)(addr(addr(addr(2))))}, + + {Name(""), any(nil), `{}`, map[string]any{}}, + {Name(""), (int)(1), `{}`, map[string]any{}}, + {Name(""), (*int)(nil), `{}`, map[string]any{}}, + {Name(""), (*int)(addr(0)), `{}`, errUnmarshal}, + {Name(""), (*int)(addr(1)), `{}`, errUnmarshal}, + {Name(""), (**int)(nil), `{}`, map[string]any{}}, + {Name(""), (**int)(addr[*int](nil)), `{}`, errUnmarshal}, + {Name(""), (**int)(addr(addr(1))), `{}`, errUnmarshal}, + {Name(""), (***int)(nil), `{}`, map[string]any{}}, + {Name(""), (***int)(addr[**int](nil)), `{}`, errUnmarshal}, + {Name(""), (***int)(addr(addr[*int](nil))), `{}`, errUnmarshal}, + {Name(""), (***int)(addr(addr(addr(1)))), `{}`, errUnmarshal}, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + b := struct{ X any }{tt.pre} + blob := `{"X":` + tt.json + `}` + if err := Unmarshal([]byte(blob), &b); err != nil { + if wantErr, _ := tt.post.(error); equalError(err, wantErr) { + return + } + t.Fatalf("%s: Unmarshal(%#q) error: %v", tt.Where, blob, err) + } + if !reflect.DeepEqual(b.X, tt.post) { + t.Errorf("%s: Unmarshal(%#q):\n\tpre.X: %#v\n\tgot.X: %#v\n\twant.X: %#v", tt.Where, blob, tt.pre, b.X, tt.post) + } + }) + } +} + +type NullTest struct { + Bool bool + Int int + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + Float32 float32 + Float64 float64 + String string + PBool *bool + Map map[string]string + Slice []string + Interface any + + PRaw *RawMessage + PTime *time.Time + PBigInt *big.Int + PText *MustNotUnmarshalText + PBuffer *bytes.Buffer // has methods, just not relevant ones + PStruct *struct{} + + Raw RawMessage + Time time.Time + BigInt big.Int + Text MustNotUnmarshalText + Buffer bytes.Buffer + Struct struct{} +} + +// JSON null values should be ignored for primitives and string values instead of resulting in an error. +// Issue 2540 +func TestUnmarshalNulls(t *testing.T) { + // Unmarshal docs: + // The JSON null value unmarshals into an interface, map, pointer, or slice + // by setting that Go value to nil. Because null is often used in JSON to mean + // ``not present,'' unmarshaling a JSON null into any other Go type has no effect + // on the value and produces no error. + + jsonData := []byte(`{ + "Bool" : null, + "Int" : null, + "Int8" : null, + "Int16" : null, + "Int32" : null, + "Int64" : null, + "Uint" : null, + "Uint8" : null, + "Uint16" : null, + "Uint32" : null, + "Uint64" : null, + "Float32" : null, + "Float64" : null, + "String" : null, + "PBool": null, + "Map": null, + "Slice": null, + "Interface": null, + "PRaw": null, + "PTime": null, + "PBigInt": null, + "PText": null, + "PBuffer": null, + "PStruct": null, + "Raw": null, + "Time": null, + "BigInt": null, + "Text": null, + "Buffer": null, + "Struct": null + }`) + nulls := NullTest{ + Bool: true, + Int: 2, + Int8: 3, + Int16: 4, + Int32: 5, + Int64: 6, + Uint: 7, + Uint8: 8, + Uint16: 9, + Uint32: 10, + Uint64: 11, + Float32: 12.1, + Float64: 13.1, + String: "14", + PBool: new(bool), + Map: map[string]string{}, + Slice: []string{}, + Interface: new(MustNotUnmarshalJSON), + PRaw: new(RawMessage), + PTime: new(time.Time), + PBigInt: new(big.Int), + PText: new(MustNotUnmarshalText), + PStruct: new(struct{}), + PBuffer: new(bytes.Buffer), + Raw: RawMessage("123"), + Time: time.Unix(123456789, 0), + BigInt: *big.NewInt(123), + } + + before := nulls.Time.String() + + err := Unmarshal(jsonData, &nulls) + if err != nil { + t.Errorf("Unmarshal of null values failed: %v", err) + } + if !nulls.Bool || nulls.Int != 2 || nulls.Int8 != 3 || nulls.Int16 != 4 || nulls.Int32 != 5 || nulls.Int64 != 6 || + nulls.Uint != 7 || nulls.Uint8 != 8 || nulls.Uint16 != 9 || nulls.Uint32 != 10 || nulls.Uint64 != 11 || + nulls.Float32 != 12.1 || nulls.Float64 != 13.1 || nulls.String != "14" { + t.Errorf("Unmarshal of null values affected primitives") + } + + if nulls.PBool != nil { + t.Errorf("Unmarshal of null did not clear nulls.PBool") + } + if nulls.Map != nil { + t.Errorf("Unmarshal of null did not clear nulls.Map") + } + if nulls.Slice != nil { + t.Errorf("Unmarshal of null did not clear nulls.Slice") + } + if nulls.Interface != nil { + t.Errorf("Unmarshal of null did not clear nulls.Interface") + } + if nulls.PRaw != nil { + t.Errorf("Unmarshal of null did not clear nulls.PRaw") + } + if nulls.PTime != nil { + t.Errorf("Unmarshal of null did not clear nulls.PTime") + } + if nulls.PBigInt != nil { + t.Errorf("Unmarshal of null did not clear nulls.PBigInt") + } + if nulls.PText != nil { + t.Errorf("Unmarshal of null did not clear nulls.PText") + } + if nulls.PBuffer != nil { + t.Errorf("Unmarshal of null did not clear nulls.PBuffer") + } + if nulls.PStruct != nil { + t.Errorf("Unmarshal of null did not clear nulls.PStruct") + } + + if string(nulls.Raw) != "null" { + t.Errorf("Unmarshal of RawMessage null did not record null: %v", string(nulls.Raw)) + } + if nulls.Time.String() != before { + t.Errorf("Unmarshal of time.Time null set time to %v", nulls.Time.String()) + } + if nulls.BigInt.String() != "123" { + t.Errorf("Unmarshal of big.Int null set int to %v", nulls.BigInt.String()) + } +} + +type MustNotUnmarshalJSON struct{} + +func (x MustNotUnmarshalJSON) UnmarshalJSON(data []byte) error { + return errors.New("MustNotUnmarshalJSON was used") +} + +type MustNotUnmarshalText struct{} + +func (x MustNotUnmarshalText) UnmarshalText(text []byte) error { + return errors.New("MustNotUnmarshalText was used") +} + +func TestStringKind(t *testing.T) { + type stringKind string + want := map[stringKind]int{"foo": 42} + data, err := Marshal(want) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + var got map[stringKind]int + err = Unmarshal(data, &got) + if err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if !maps.Equal(got, want) { + t.Fatalf("Marshal/Unmarshal mismatch:\n\tgot: %v\n\twant: %v", got, want) + } +} + +// Custom types with []byte as underlying type could not be marshaled +// and then unmarshaled. +// Issue 8962. +func TestByteKind(t *testing.T) { + type byteKind []byte + want := byteKind("hello") + data, err := Marshal(want) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + var got byteKind + err = Unmarshal(data, &got) + if err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if !slices.Equal(got, want) { + t.Fatalf("Marshal/Unmarshal mismatch:\n\tgot: %v\n\twant: %v", got, want) + } +} + +// The fix for issue 8962 introduced a regression. +// Issue 12921. +func TestSliceOfCustomByte(t *testing.T) { + type Uint8 uint8 + want := []Uint8("hello") + data, err := Marshal(want) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + var got []Uint8 + err = Unmarshal(data, &got) + if err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if !slices.Equal(got, want) { + t.Fatalf("Marshal/Unmarshal mismatch:\n\tgot: %v\n\twant: %v", got, want) + } +} + +func TestUnmarshalTypeError(t *testing.T) { + tests := []struct { + CaseName + dest any + in string + }{ + {Name(""), new(string), `{"user": "name"}`}, // issue 4628. + {Name(""), new(error), `{}`}, // issue 4222 + {Name(""), new(error), `[]`}, + {Name(""), new(error), `""`}, + {Name(""), new(error), `123`}, + {Name(""), new(error), `true`}, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + err := Unmarshal([]byte(tt.in), tt.dest) + if _, ok := err.(*UnmarshalTypeError); !ok { + t.Errorf("%s: Unmarshal(%#q, %T):\n\tgot: %T\n\twant: %T", + tt.Where, tt.in, tt.dest, err, new(UnmarshalTypeError)) + } + }) + } +} + +func TestUnmarshalTypeErrorMessage(t *testing.T) { + err := &UnmarshalTypeError{ + Value: "number 5", + Type: reflect.TypeFor[int](), + Offset: 1234, + Struct: "Root", + } + + for _, tt := range []struct { + field string + want string + }{ + {"", "json: cannot unmarshal number 5 into Go struct field Root. of type int"}, + {"1", "json: cannot unmarshal number 5 into Root.1 of type int"}, + {"foo", "json: cannot unmarshal number 5 into Go struct field Root.foo of type int"}, + {"foo.1", "json: cannot unmarshal number 5 into Root.foo.1 of type int"}, + {"foo.bar", "json: cannot unmarshal number 5 into Go struct field Root.foo.bar of type int"}, + {"foo.bar.1", "json: cannot unmarshal number 5 into Root.foo.bar.1 of type int"}, + {"foo.bar.baz", "json: cannot unmarshal number 5 into Go struct field Root.foo.bar.baz of type int"}, + } { + err.Field = tt.field + got := err.Error() + if got != tt.want { + t.Errorf("Error:\n\tgot: %v\n\twant: %v", got, tt.want) + } + } +} + +func TestUnmarshalSyntax(t *testing.T) { + var x any + tests := []struct { + CaseName + in string + }{ + {Name(""), "tru"}, + {Name(""), "fals"}, + {Name(""), "nul"}, + {Name(""), "123e"}, + {Name(""), `"hello`}, + {Name(""), `[1,2,3`}, + {Name(""), `{"key":1`}, + {Name(""), `{"key":1,`}, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + err := Unmarshal([]byte(tt.in), &x) + if _, ok := err.(*SyntaxError); !ok { + t.Errorf("%s: Unmarshal(%#q, any):\n\tgot: %T\n\twant: %T", + tt.Where, tt.in, err, new(SyntaxError)) + } + }) + } +} + +// Test handling of unexported fields that should be ignored. +// Issue 4660 +type unexportedFields struct { + Name string + m map[string]any `json:"-"` + m2 map[string]any `json:"abcd"` + + s []int `json:"-"` +} + +func TestUnmarshalUnexported(t *testing.T) { + input := `{"Name": "Bob", "m": {"x": 123}, "m2": {"y": 456}, "abcd": {"z": 789}, "s": [2, 3]}` + want := &unexportedFields{Name: "Bob"} + + out := &unexportedFields{} + err := Unmarshal([]byte(input), out) + if err != nil { + t.Errorf("Unmarshal error: %v", err) + } + if !reflect.DeepEqual(out, want) { + t.Errorf("Unmarshal:\n\tgot: %+v\n\twant: %+v", out, want) + } +} + +// Time3339 is a time.Time which encodes to and from JSON +// as an RFC 3339 time in UTC. +type Time3339 time.Time + +func (t *Time3339) UnmarshalJSON(b []byte) error { + if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' { + return fmt.Errorf("types: failed to unmarshal non-string value %q as an RFC 3339 time", b) + } + tm, err := time.Parse(time.RFC3339, string(b[1:len(b)-1])) + if err != nil { + return err + } + *t = Time3339(tm) + return nil +} + +func TestUnmarshalJSONLiteralError(t *testing.T) { + var t3 Time3339 + switch err := Unmarshal([]byte(`"0000-00-00T00:00:00Z"`), &t3); { + case err == nil: + t.Fatalf("Unmarshal error: got nil, want non-nil") + case !strings.Contains(err.Error(), "range"): + t.Errorf("Unmarshal error:\n\tgot: %v\n\twant: out of range", err) + } +} + +// Test that extra object elements in an array do not result in a +// "data changing underfoot" error. +// Issue 3717 +func TestSkipArrayObjects(t *testing.T) { + json := `[{}]` + var dest [0]any + + err := Unmarshal([]byte(json), &dest) + if err != nil { + t.Errorf("Unmarshal error: %v", err) + } +} + +// Test semantics of pre-filled data, such as struct fields, map elements, +// slices, and arrays. +// Issues 4900 and 8837, among others. +func TestPrefilled(t *testing.T) { + // Values here change, cannot reuse table across runs. + tests := []struct { + CaseName + in string + ptr any + out any + }{{ + CaseName: Name(""), + in: `{"X": 1, "Y": 2}`, + ptr: &XYZ{X: float32(3), Y: int16(4), Z: 1.5}, + out: &XYZ{X: float64(1), Y: float64(2), Z: 1.5}, + }, { + CaseName: Name(""), + in: `{"X": 1, "Y": 2}`, + ptr: &map[string]any{"X": float32(3), "Y": int16(4), "Z": 1.5}, + out: &map[string]any{"X": float64(1), "Y": float64(2), "Z": 1.5}, + }, { + CaseName: Name(""), + in: `[2]`, + ptr: &[]int{1}, + out: &[]int{2}, + }, { + CaseName: Name(""), + in: `[2, 3]`, + ptr: &[]int{1}, + out: &[]int{2, 3}, + }, { + CaseName: Name(""), + in: `[2, 3]`, + ptr: &[...]int{1}, + out: &[...]int{2}, + }, { + CaseName: Name(""), + in: `[3]`, + ptr: &[...]int{1, 2}, + out: &[...]int{3, 0}, + }} + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + ptrstr := fmt.Sprintf("%v", tt.ptr) + err := Unmarshal([]byte(tt.in), tt.ptr) // tt.ptr edited here + if err != nil { + t.Errorf("%s: Unmarshal error: %v", tt.Where, err) + } + if !reflect.DeepEqual(tt.ptr, tt.out) { + t.Errorf("%s: Unmarshal(%#q, %T):\n\tgot: %v\n\twant: %v", tt.Where, tt.in, ptrstr, tt.ptr, tt.out) + } + }) + } +} + +func TestInvalidUnmarshal(t *testing.T) { + tests := []struct { + CaseName + in string + v any + wantErr error + }{ + {Name(""), `{"a":"1"}`, nil, &InvalidUnmarshalError{}}, + {Name(""), `{"a":"1"}`, struct{}{}, &InvalidUnmarshalError{reflect.TypeFor[struct{}]()}}, + {Name(""), `{"a":"1"}`, (*int)(nil), &InvalidUnmarshalError{reflect.TypeFor[*int]()}}, + {Name(""), `123`, nil, &InvalidUnmarshalError{}}, + {Name(""), `123`, struct{}{}, &InvalidUnmarshalError{reflect.TypeFor[struct{}]()}}, + {Name(""), `123`, (*int)(nil), &InvalidUnmarshalError{reflect.TypeFor[*int]()}}, + {Name(""), `123`, new(net.IP), &UnmarshalTypeError{Value: "number", Type: reflect.TypeFor[net.IP](), Offset: len64(``), Err: errors.New("JSON value must be string type")}}, + } + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + switch gotErr := Unmarshal([]byte(tt.in), tt.v); { + case gotErr == nil: + t.Fatalf("%s: Unmarshal error: got nil, want non-nil", tt.Where) + case !reflect.DeepEqual(gotErr, tt.wantErr): + t.Errorf("%s: Unmarshal error:\n\tgot: %#v\n\twant: %#v", tt.Where, gotErr, tt.wantErr) + } + }) + } +} + +// Test that string option is ignored for invalid types. +// Issue 9812. +func TestInvalidStringOption(t *testing.T) { + num := 0 + item := struct { + T time.Time `json:",string"` + M map[string]string `json:",string"` + S []string `json:",string"` + A [1]string `json:",string"` + I any `json:",string"` + P *int `json:",string"` + }{M: make(map[string]string), S: make([]string, 0), I: num, P: &num} + + data, err := Marshal(item) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + + err = Unmarshal(data, &item) + if err != nil { + t.Fatalf("Unmarshal error: %v", err) + } +} + +// Test unmarshal behavior with regards to embedded unexported structs. +// +// (Issue 21357) If the embedded struct is a pointer and is unallocated, +// this returns an error because unmarshal cannot set the field. +// +// (Issue 24152) If the embedded struct is given an explicit name, +// ensure that the normal unmarshal logic does not panic in reflect. +// +// (Issue 28145) If the embedded struct is given an explicit name and has +// exported methods, don't cause a panic trying to get its value. +func TestUnmarshalEmbeddedUnexported(t *testing.T) { + type ( + embed1 struct{ Q int } + embed2 struct{ Q int } + embed3 struct { + Q int64 `json:",string"` + } + S1 struct { + *embed1 + R int + } + S2 struct { + *embed1 + Q int + } + S3 struct { + embed1 + R int + } + S4 struct { + *embed1 + embed2 + } + S5 struct { + *embed3 + R int + } + S6 struct { + embed1 `json:"embed1"` + } + S7 struct { + embed1 `json:"embed1"` + embed2 + } + S8 struct { + embed1 `json:"embed1"` + embed2 `json:"embed2"` + Q int + } + S9 struct { + unexportedWithMethods `json:"embed"` + } + ) + + tests := []struct { + CaseName + in string + ptr any + out any + err error + }{{ + // Error since we cannot set S1.embed1, but still able to set S1.R. + CaseName: Name(""), + in: `{"R":2,"Q":1}`, + ptr: new(S1), + out: &S1{R: 2}, + err: &UnmarshalTypeError{ + Value: "number", + Type: reflect.TypeFor[S1](), + Offset: len64(`{"R":2,"Q":`), + Struct: "S1", + Field: "Q", + Err: errors.New("cannot set embedded pointer to unexported struct type"), + }, + }, { + // The top level Q field takes precedence. + CaseName: Name(""), + in: `{"Q":1}`, + ptr: new(S2), + out: &S2{Q: 1}, + }, { + // No issue with non-pointer variant. + CaseName: Name(""), + in: `{"R":2,"Q":1}`, + ptr: new(S3), + out: &S3{embed1: embed1{Q: 1}, R: 2}, + }, { + // No error since both embedded structs have field R, which annihilate each other. + // Thus, no attempt is made at setting S4.embed1. + CaseName: Name(""), + in: `{"R":2}`, + ptr: new(S4), + out: new(S4), + }, { + // Error since we cannot set S5.embed1, but still able to set S5.R. + CaseName: Name(""), + in: `{"R":2,"Q":1}`, + ptr: new(S5), + out: &S5{R: 2}, + err: &UnmarshalTypeError{ + Value: "number", + Type: reflect.TypeFor[S5](), + Offset: len64(`{"R":2,"Q":`), + Struct: "S5", + Field: "Q", + Err: errors.New("cannot set embedded pointer to unexported struct type"), + }, + }, { + // Issue 24152, ensure decodeState.indirect does not panic. + CaseName: Name(""), + in: `{"embed1": {"Q": 1}}`, + ptr: new(S6), + out: &S6{embed1{1}}, + }, { + // Issue 24153, check that we can still set forwarded fields even in + // the presence of a name conflict. + // + // This relies on obscure behavior of reflect where it is possible + // to set a forwarded exported field on an unexported embedded struct + // even though there is a name conflict, even when it would have been + // impossible to do so according to Go visibility rules. + // Go forbids this because it is ambiguous whether S7.Q refers to + // S7.embed1.Q or S7.embed2.Q. Since embed1 and embed2 are unexported, + // it should be impossible for an external package to set either Q. + // + // It is probably okay for a future reflect change to break this. + CaseName: Name(""), + in: `{"embed1": {"Q": 1}, "Q": 2}`, + ptr: new(S7), + out: &S7{embed1{1}, embed2{2}}, + }, { + // Issue 24153, similar to the S7 case. + CaseName: Name(""), + in: `{"embed1": {"Q": 1}, "embed2": {"Q": 2}, "Q": 3}`, + ptr: new(S8), + out: &S8{embed1{1}, embed2{2}, 3}, + }, { + // Issue 228145, similar to the cases above. + CaseName: Name(""), + in: `{"embed": {}}`, + ptr: new(S9), + out: &S9{}, + }} + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + err := Unmarshal([]byte(tt.in), tt.ptr) + if !equalError(err, tt.err) { + t.Errorf("%s: Unmarshal error:\n\tgot: %v\n\twant: %v", tt.Where, err, tt.err) + } + if !reflect.DeepEqual(tt.ptr, tt.out) { + t.Errorf("%s: Unmarshal:\n\tgot: %#+v\n\twant: %#+v", tt.Where, tt.ptr, tt.out) + } + }) + } +} + +func TestUnmarshalErrorAfterMultipleJSON(t *testing.T) { + tests := []struct { + CaseName + in string + err error + }{{ + CaseName: Name(""), + in: `1 false null :`, + err: &SyntaxError{"invalid character ':' looking for beginning of value", len64(`1 false null `)}, + }, { + CaseName: Name(""), + in: `1 [] [,]`, + err: &SyntaxError{"invalid character ',' looking for beginning of value", len64(`1 [] [`)}, + }, { + CaseName: Name(""), + in: `1 [] [true:]`, + err: &SyntaxError{"invalid character ':' after array element", len64(`1 [] [true`)}, + }, { + CaseName: Name(""), + in: `1 {} {"x"=}`, + err: &SyntaxError{"invalid character '=' after object key", len64(`1 {} {"x"`)}, + }, { + CaseName: Name(""), + in: `falsetruenul#`, + err: &SyntaxError{"invalid character '#' in literal null (expecting 'l')", len64(`falsetruenul`)}, + }} + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + dec := NewDecoder(strings.NewReader(tt.in)) + var err error + for err == nil { + var v any + err = dec.Decode(&v) + } + if !reflect.DeepEqual(err, tt.err) { + t.Errorf("%s: Decode error:\n\tgot: %v\n\twant: %v", tt.Where, err, tt.err) + } + }) + } +} + +type unmarshalPanic struct{} + +func (unmarshalPanic) UnmarshalJSON([]byte) error { panic(0xdead) } + +func TestUnmarshalPanic(t *testing.T) { + defer func() { + if got := recover(); !reflect.DeepEqual(got, 0xdead) { + t.Errorf("panic() = (%T)(%v), want 0xdead", got, got) + } + }() + Unmarshal([]byte("{}"), &unmarshalPanic{}) + t.Fatalf("Unmarshal should have panicked") +} + +type textUnmarshalerString string + +func (m *textUnmarshalerString) UnmarshalText(text []byte) error { + *m = textUnmarshalerString(strings.ToLower(string(text))) + return nil +} + +// Test unmarshal to a map, where the map key is a user defined type. +// See golang.org/issues/34437. +func TestUnmarshalMapWithTextUnmarshalerStringKey(t *testing.T) { + var p map[textUnmarshalerString]string + if err := Unmarshal([]byte(`{"FOO": "1"}`), &p); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + + if _, ok := p["foo"]; !ok { + t.Errorf(`key "foo" missing in map: %v`, p) + } +} + +func TestUnmarshalRescanLiteralMangledUnquote(t *testing.T) { + // See golang.org/issues/38105. + var p map[textUnmarshalerString]string + if err := Unmarshal([]byte(`{"开源":"12345开源"}`), &p); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if _, ok := p["开源"]; !ok { + t.Errorf(`key "开源" missing in map: %v`, p) + } + + // See golang.org/issues/38126. + type T struct { + F1 string `json:"F1,string"` + } + wantT := T{"aaa\tbbb"} + + b, err := Marshal(wantT) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + var gotT T + if err := Unmarshal(b, &gotT); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if gotT != wantT { + t.Errorf("Marshal/Unmarshal roundtrip:\n\tgot: %q\n\twant: %q", gotT, wantT) + } + + // See golang.org/issues/39555. + input := map[textUnmarshalerString]string{"FOO": "", `"`: ""} + + encoded, err := Marshal(input) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + var got map[textUnmarshalerString]string + if err := Unmarshal(encoded, &got); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + want := map[textUnmarshalerString]string{"foo": "", `"`: ""} + if !maps.Equal(got, want) { + t.Errorf("Marshal/Unmarshal roundtrip:\n\tgot: %q\n\twant: %q", gotT, wantT) + } +} + +func TestUnmarshalMaxDepth(t *testing.T) { + tests := []struct { + CaseName + data string + errMaxDepth bool + }{{ + CaseName: Name("ArrayUnderMaxNestingDepth"), + data: `{"a":` + strings.Repeat(`[`, 10000-1) + strings.Repeat(`]`, 10000-1) + `}`, + errMaxDepth: false, + }, { + CaseName: Name("ArrayOverMaxNestingDepth"), + data: `{"a":` + strings.Repeat(`[`, 10000) + strings.Repeat(`]`, 10000) + `}`, + errMaxDepth: true, + }, { + CaseName: Name("ArrayOverStackDepth"), + data: `{"a":` + strings.Repeat(`[`, 3000000) + strings.Repeat(`]`, 3000000) + `}`, + errMaxDepth: true, + }, { + CaseName: Name("ObjectUnderMaxNestingDepth"), + data: `{"a":` + strings.Repeat(`{"a":`, 10000-1) + `0` + strings.Repeat(`}`, 10000-1) + `}`, + errMaxDepth: false, + }, { + CaseName: Name("ObjectOverMaxNestingDepth"), + data: `{"a":` + strings.Repeat(`{"a":`, 10000) + `0` + strings.Repeat(`}`, 10000) + `}`, + errMaxDepth: true, + }, { + CaseName: Name("ObjectOverStackDepth"), + data: `{"a":` + strings.Repeat(`{"a":`, 3000000) + `0` + strings.Repeat(`}`, 3000000) + `}`, + errMaxDepth: true, + }} + + targets := []struct { + CaseName + newValue func() any + }{{ + CaseName: Name("unstructured"), + newValue: func() any { + var v any + return &v + }, + }, { + CaseName: Name("typed named field"), + newValue: func() any { + v := struct { + A any `json:"a"` + }{} + return &v + }, + }, { + CaseName: Name("typed missing field"), + newValue: func() any { + v := struct { + B any `json:"b"` + }{} + return &v + }, + }, { + CaseName: Name("custom unmarshaler"), + newValue: func() any { + v := unmarshaler{} + return &v + }, + }} + + for _, tt := range tests { + for _, target := range targets { + t.Run(target.Name+"-"+tt.Name, func(t *testing.T) { + err := Unmarshal([]byte(tt.data), target.newValue()) + if !tt.errMaxDepth { + if err != nil { + t.Errorf("%s: %s: Unmarshal error: %v", tt.Where, target.Where, err) + } + } else { + if err == nil || !strings.Contains(err.Error(), "exceeded max depth") { + t.Errorf("%s: %s: Unmarshal error:\n\tgot: %v\n\twant: exceeded max depth", tt.Where, target.Where, err) + } + } + }) + } + } +} diff --git a/src/encoding/json/v2_diff_test.go b/src/encoding/json/v2_diff_test.go new file mode 100644 index 00000000000000..9d0798ed1da462 --- /dev/null +++ b/src/encoding/json/v2_diff_test.go @@ -0,0 +1,1130 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +package json_test + +import ( + "errors" + "path" + "reflect" + "strings" + "testing" + "time" + + jsonv1 "encoding/json" + "encoding/json/jsontext" + jsonv2 "encoding/json/v2" +) + +// NOTE: This file serves as a list of semantic differences between v1 and v2. +// Each test explains how v1 behaves, how v2 behaves, and +// a rationale for why the behavior was changed. + +var jsonPackages = []struct { + Version string + Marshal func(any) ([]byte, error) + Unmarshal func([]byte, any) error +}{ + {"v1", jsonv1.Marshal, jsonv1.Unmarshal}, + {"v2", + func(in any) ([]byte, error) { return jsonv2.Marshal(in) }, + func(in []byte, out any) error { return jsonv2.Unmarshal(in, out) }}, +} + +// In v1, unmarshal matches struct fields using a case-insensitive match. +// In v2, unmarshal matches struct fields using a case-sensitive match. +// +// Case-insensitive matching is a surprising default and +// incurs significant performance cost when unmarshaling unknown fields. +// In v2, we can opt into v1-like behavior with the `case:ignore` tag option. +// The case-insensitive matching performed by v2 is looser than that of v1 +// where it also ignores dashes and underscores. +// This allows v2 to match fields regardless of whether the name is in +// snake_case, camelCase, or kebab-case. +// +// Related issue: +// +// https://go.dev/issue/14750 +func TestCaseSensitivity(t *testing.T) { + type Fields struct { + FieldA bool + FieldB bool `json:"fooBar"` + FieldC bool `json:"fizzBuzz,case:ignore"` // `case:ignore` is used by v2 to explicitly enable case-insensitive matching + } + + for _, json := range jsonPackages { + t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) { + // This is a mapping from Go field names to JSON member names to + // whether the JSON member name would match the Go field name. + type goName = string + type jsonName = string + onlyV1 := json.Version == "v1" + onlyV2 := json.Version == "v2" + allMatches := map[goName]map[jsonName]bool{ + "FieldA": { + "FieldA": true, // exact match + "fielda": onlyV1, // v1 is case-insensitive by default + "fieldA": onlyV1, // v1 is case-insensitive by default + "FIELDA": onlyV1, // v1 is case-insensitive by default + "FieldB": false, + "FieldC": false, + }, + "FieldB": { + "fooBar": true, // exact match for explicitly specified JSON name + "FooBar": onlyV1, // v1 is case-insensitive even if an explicit JSON name is provided + "foobar": onlyV1, // v1 is case-insensitive even if an explicit JSON name is provided + "FOOBAR": onlyV1, // v1 is case-insensitive even if an explicit JSON name is provided + "fizzBuzz": false, + "FieldA": false, + "FieldB": false, // explicit JSON name means that the Go field name is not used for matching + "FieldC": false, + }, + "FieldC": { + "fizzBuzz": true, // exact match for explicitly specified JSON name + "fizzbuzz": true, // v2 is case-insensitive due to `case:ignore` tag + "FIZZBUZZ": true, // v2 is case-insensitive due to `case:ignore` tag + "fizz_buzz": onlyV2, // case-insensitivity in v2 ignores dashes and underscores + "fizz-buzz": onlyV2, // case-insensitivity in v2 ignores dashes and underscores + "fooBar": false, + "FieldA": false, + "FieldC": false, // explicit JSON name means that the Go field name is not used for matching + "FieldB": false, + }, + } + + for goFieldName, matches := range allMatches { + for jsonMemberName, wantMatch := range matches { + in := `{"` + jsonMemberName + `":true}` + var s Fields + if err := json.Unmarshal([]byte(in), &s); err != nil { + t.Fatalf("json.Unmarshal error: %v", err) + } + gotMatch := reflect.ValueOf(s).FieldByName(goFieldName).Bool() + if gotMatch != wantMatch { + t.Fatalf("%T.%s = %v, want %v", s, goFieldName, gotMatch, wantMatch) + } + } + } + }) + } +} + +// In v1, the "omitempty" option specifies that a struct field is omitted +// when marshaling if it is an empty Go value, which is defined as +// false, 0, a nil pointer, a nil interface value, and +// any empty array, slice, map, or string. +// +// In v2, the "omitempty" option specifies that a struct field is omitted +// when marshaling if it is an empty JSON value, which is defined as +// a JSON null or empty JSON string, object, or array. +// +// In v2, we also provide the "omitzero" option which specifies that a field +// is omitted if it is the zero Go value or if it implements an "IsZero() bool" +// method that reports true. Together, "omitzero" and "omitempty" can cover +// all the prior use cases of the v1 definition of "omitempty". +// Note that "omitempty" is defined in terms of the Go type system in v1, +// but now defined in terms of the JSON type system in v2. +// +// Related issues: +// +// https://go.dev/issue/11939 +// https://go.dev/issue/22480 +// https://go.dev/issue/29310 +// https://go.dev/issue/32675 +// https://go.dev/issue/45669 +// https://go.dev/issue/45787 +// https://go.dev/issue/50480 +// https://go.dev/issue/52803 +func TestOmitEmptyOption(t *testing.T) { + type Struct struct { + Foo string `json:",omitempty"` + Bar []int `json:",omitempty"` + Baz *Struct `json:",omitempty"` + } + type Types struct { + Bool bool `json:",omitempty"` + StringA string `json:",omitempty"` + StringB string `json:",omitempty"` + BytesA []byte `json:",omitempty"` + BytesB []byte `json:",omitempty"` + BytesC []byte `json:",omitempty"` + Int int `json:",omitempty"` + MapA map[string]string `json:",omitempty"` + MapB map[string]string `json:",omitempty"` + MapC map[string]string `json:",omitempty"` + StructA Struct `json:",omitempty"` + StructB Struct `json:",omitempty"` + StructC Struct `json:",omitempty"` + SliceA []string `json:",omitempty"` + SliceB []string `json:",omitempty"` + SliceC []string `json:",omitempty"` + Array [1]string `json:",omitempty"` + PointerA *string `json:",omitempty"` + PointerB *string `json:",omitempty"` + PointerC *string `json:",omitempty"` + InterfaceA any `json:",omitempty"` + InterfaceB any `json:",omitempty"` + InterfaceC any `json:",omitempty"` + InterfaceD any `json:",omitempty"` + } + + something := "something" + for _, json := range jsonPackages { + t.Run(path.Join("Marshal", json.Version), func(t *testing.T) { + in := Types{ + Bool: false, + StringA: "", + StringB: something, + BytesA: nil, + BytesB: []byte{}, + BytesC: []byte(something), + Int: 0, + MapA: nil, + MapB: map[string]string{}, + MapC: map[string]string{something: something}, + StructA: Struct{}, + StructB: Struct{Bar: []int{}, Baz: new(Struct)}, + StructC: Struct{Foo: something}, + SliceA: nil, + SliceB: []string{}, + SliceC: []string{something}, + Array: [1]string{something}, + PointerA: nil, + PointerB: new(string), + PointerC: &something, + InterfaceA: nil, + InterfaceB: (*string)(nil), + InterfaceC: new(string), + InterfaceD: &something, + } + b, err := json.Marshal(in) + if err != nil { + t.Fatalf("json.Marshal error: %v", err) + } + var out map[string]any + if err := json.Unmarshal(b, &out); err != nil { + t.Fatalf("json.Unmarshal error: %v", err) + } + + onlyV1 := json.Version == "v1" + onlyV2 := json.Version == "v2" + wantPresent := map[string]bool{ + "Bool": onlyV2, // false is an empty Go bool, but is NOT an empty JSON value + "StringA": false, + "StringB": true, + "BytesA": false, + "BytesB": false, + "BytesC": true, + "Int": onlyV2, // 0 is an empty Go integer, but NOT an empty JSON value + "MapA": false, + "MapB": false, + "MapC": true, + "StructA": onlyV1, // Struct{} is NOT an empty Go value, but {} is an empty JSON value + "StructB": onlyV1, // Struct{...} is NOT an empty Go value, but {} is an empty JSON value + "StructC": true, + "SliceA": false, + "SliceB": false, + "SliceC": true, + "Array": true, + "PointerA": false, + "PointerB": onlyV1, // new(string) is NOT a nil Go pointer, but "" is an empty JSON value + "PointerC": true, + "InterfaceA": false, + "InterfaceB": onlyV1, // (*string)(nil) is NOT a nil Go interface, but null is an empty JSON value + "InterfaceC": onlyV1, // new(string) is NOT a nil Go interface, but "" is an empty JSON value + "InterfaceD": true, + } + for field, want := range wantPresent { + _, got := out[field] + if got != want { + t.Fatalf("%T.%s = %v, want %v", in, field, got, want) + } + } + }) + } +} + +func addr[T any](v T) *T { + return &v +} + +// In v1, the "string" option specifies that Go strings, bools, and numeric +// values are encoded within a JSON string when marshaling and +// are unmarshaled from its native representation escaped within a JSON string. +// The "string" option is not applied recursively, and so does not affect +// strings, bools, and numeric values within a Go slice or map, but +// does have special handling to affect the underlying value within a pointer. +// When unmarshaling, the "string" option permits decoding from a JSON null +// escaped within a JSON string in some inconsistent cases. +// +// In v2, the "string" option specifies that only numeric values are encoded as +// a JSON number within a JSON string when marshaling and are unmarshaled +// from either a JSON number or a JSON string containing a JSON number. +// The "string" option is applied recursively to all numeric sub-values, +// and thus affects numeric values within a Go slice or map. +// There is no support for escaped JSON nulls within a JSON string. +// +// The main utility for stringifying JSON numbers is because JSON parsers +// often represents numbers as IEEE 754 floating-point numbers. +// This results in a loss of precision representing 64-bit integer values. +// Consequently, many JSON-based APIs actually requires that such values +// be encoded within a JSON string. Since the main utility of stringification +// is for numeric values, v2 limits the effect of the "string" option +// to just numeric Go types. According to all code known by the Go module proxy, +// there are close to zero usages of the "string" option on a Go string or bool. +// +// Regarding the recursive application of the "string" option, +// there have been a number of issues filed about users being surprised that +// the "string" option does not recursively affect numeric values +// within a composite type like a Go map, slice, or interface value. +// In v1, specifying the "string" option on composite type has no effect +// and so this would be a largely backwards compatible change. +// +// The ability to decode from a JSON null wrapped within a JSON string +// is removed in v2 because this behavior was surprising and inconsistent in v1. +// +// Related issues: +// +// https://go.dev/issue/15624 +// https://go.dev/issue/20651 +// https://go.dev/issue/22177 +// https://go.dev/issue/32055 +// https://go.dev/issue/32117 +// https://go.dev/issue/50997 +func TestStringOption(t *testing.T) { + type Types struct { + String string `json:",string"` + Bool bool `json:",string"` + Int int `json:",string"` + Float float64 `json:",string"` + Map map[string]int `json:",string"` + Struct struct{ Field int } `json:",string"` + Slice []int `json:",string"` + Array [1]int `json:",string"` + PointerA *int `json:",string"` + PointerB *int `json:",string"` + PointerC **int `json:",string"` + InterfaceA any `json:",string"` + InterfaceB any `json:",string"` + } + + for _, json := range jsonPackages { + t.Run(path.Join("Marshal", json.Version), func(t *testing.T) { + in := Types{ + String: "string", + Bool: true, + Int: 1, + Float: 1, + Map: map[string]int{"Name": 1}, + Struct: struct{ Field int }{1}, + Slice: []int{1}, + Array: [1]int{1}, + PointerA: nil, + PointerB: addr(1), + PointerC: addr(addr(1)), + InterfaceA: nil, + InterfaceB: 1, + } + quote := func(s string) string { + b, _ := jsontext.AppendQuote(nil, s) + return string(b) + } + quoteOnlyV1 := func(s string) string { + if json.Version == "v1" { + s = quote(s) + } + return s + } + quoteOnlyV2 := func(s string) string { + if json.Version == "v2" { + s = quote(s) + } + return s + } + want := strings.Join([]string{ + `{`, + `"String":` + quoteOnlyV1(`"string"`) + `,`, // in v1, Go strings are also stringified + `"Bool":` + quoteOnlyV1("true") + `,`, // in v1, Go bools are also stringified + `"Int":` + quote("1") + `,`, + `"Float":` + quote("1") + `,`, + `"Map":{"Name":` + quoteOnlyV2("1") + `},`, // in v2, numbers are recursively stringified + `"Struct":{"Field":` + quoteOnlyV2("1") + `},`, // in v2, numbers are recursively stringified + `"Slice":[` + quoteOnlyV2("1") + `],`, // in v2, numbers are recursively stringified + `"Array":[` + quoteOnlyV2("1") + `],`, // in v2, numbers are recursively stringified + `"PointerA":null,`, + `"PointerB":` + quote("1") + `,`, // in v1, numbers are stringified after a single pointer indirection + `"PointerC":` + quoteOnlyV2("1") + `,`, // in v2, numbers are recursively stringified + `"InterfaceA":null,`, + `"InterfaceB":` + quoteOnlyV2("1") + ``, // in v2, numbers are recursively stringified + `}`}, "") + got, err := json.Marshal(in) + if err != nil { + t.Fatalf("json.Marshal error: %v", err) + } + if string(got) != want { + t.Fatalf("json.Marshal = %s, want %s", got, want) + } + }) + } + + for _, json := range jsonPackages { + t.Run(path.Join("Unmarshal/Null", json.Version), func(t *testing.T) { + var got Types + err := json.Unmarshal([]byte(`{ + "Bool": "null", + "Int": "null", + "PointerA": "null" + }`), &got) + switch { + case !reflect.DeepEqual(got, Types{}): + t.Fatalf("json.Unmarshal = %v, want %v", got, Types{}) + case json.Version == "v1" && err != nil: + t.Fatalf("json.Unmarshal error: %v", err) + case json.Version == "v2" && err == nil: + t.Fatal("json.Unmarshal error is nil, want non-nil") + } + }) + + t.Run(path.Join("Unmarshal/Bool", json.Version), func(t *testing.T) { + var got Types + want := map[string]Types{ + "v1": {Bool: true}, + "v2": {Bool: false}, + }[json.Version] + err := json.Unmarshal([]byte(`{"Bool": "true"}`), &got) + switch { + case !reflect.DeepEqual(got, want): + t.Fatalf("json.Unmarshal = %v, want %v", got, want) + case json.Version == "v1" && err != nil: + t.Fatalf("json.Unmarshal error: %v", err) + case json.Version == "v2" && err == nil: + t.Fatal("json.Unmarshal error is nil, want non-nil") + } + }) + + t.Run(path.Join("Unmarshal/Shallow", json.Version), func(t *testing.T) { + var got Types + want := Types{Int: 1, PointerB: addr(1)} + err := json.Unmarshal([]byte(`{ + "Int": "1", + "PointerB": "1" + }`), &got) + switch { + case !reflect.DeepEqual(got, want): + t.Fatalf("json.Unmarshal = %v, want %v", got, want) + case err != nil: + t.Fatalf("json.Unmarshal error: %v", err) + } + }) + + t.Run(path.Join("Unmarshal/Deep", json.Version), func(t *testing.T) { + var got Types + want := map[string]Types{ + "v1": { + Map: map[string]int{"Name": 0}, + Slice: []int{0}, + PointerC: addr(addr(0)), + }, + "v2": { + Map: map[string]int{"Name": 1}, + Struct: struct{ Field int }{1}, + Slice: []int{1}, + Array: [1]int{1}, + PointerC: addr(addr(1)), + }, + }[json.Version] + err := json.Unmarshal([]byte(`{ + "Map": {"Name":"1"}, + "Struct": {"Field":"1"}, + "Slice": ["1"], + "Array": ["1"], + "PointerC": "1" + }`), &got) + switch { + case !reflect.DeepEqual(got, want): + t.Fatalf("json.Unmarshal =\n%v, want\n%v", got, want) + case json.Version == "v1" && err == nil: + t.Fatal("json.Unmarshal error is nil, want non-nil") + case json.Version == "v2" && err != nil: + t.Fatalf("json.Unmarshal error: %v", err) + } + }) + } +} + +// In v1, nil slices and maps are marshaled as a JSON null. +// In v2, nil slices and maps are marshaled as an empty JSON object or array. +// +// Users of v2 can opt into the v1 behavior by setting +// the "format:emitnull" option in the `json` struct field tag: +// +// struct { +// S []string `json:",format:emitnull"` +// M map[string]string `json:",format:emitnull"` +// } +// +// JSON is a language-agnostic data interchange format. +// The fact that maps and slices are nil-able in Go is a semantic detail of the +// Go language. We should avoid leaking such details to the JSON representation. +// When JSON implementations leak language-specific details, +// it complicates transition to/from languages with different type systems. +// +// Furthermore, consider two related Go types: string and []byte. +// It's an asymmetric oddity of v1 that zero values of string and []byte marshal +// as an empty JSON string for the former, while the latter as a JSON null. +// The non-zero values of those types always marshal as JSON strings. +// +// Related issues: +// +// https://go.dev/issue/27589 +// https://go.dev/issue/37711 +func TestNilSlicesAndMaps(t *testing.T) { + type Composites struct { + B []byte // always encoded in v2 as a JSON string + S []string // always encoded in v2 as a JSON array + M map[string]string // always encoded in v2 as a JSON object + } + + for _, json := range jsonPackages { + t.Run(path.Join("Marshal", json.Version), func(t *testing.T) { + in := []Composites{ + {B: []byte(nil), S: []string(nil), M: map[string]string(nil)}, + {B: []byte{}, S: []string{}, M: map[string]string{}}, + } + want := map[string]string{ + "v1": `[{"B":null,"S":null,"M":null},{"B":"","S":[],"M":{}}]`, + "v2": `[{"B":"","S":[],"M":{}},{"B":"","S":[],"M":{}}]`, // v2 emits nil slices and maps as empty JSON objects and arrays + }[json.Version] + got, err := json.Marshal(in) + if err != nil { + t.Fatalf("json.Marshal error: %v", err) + } + if string(got) != want { + t.Fatalf("json.Marshal = %s, want %s", got, want) + } + }) + } +} + +// In v1, unmarshaling into a Go array permits JSON arrays with any length. +// In v2, unmarshaling into a Go array requires that the JSON array +// have the exact same number of elements as the Go array. +// +// Go arrays are often used because the exact length has significant meaning. +// Ignoring this detail seems like a mistake. Also, the v1 behavior leads to +// silent data loss when excess JSON array elements are discarded. +func TestArrays(t *testing.T) { + for _, json := range jsonPackages { + t.Run(path.Join("Unmarshal/TooFew", json.Version), func(t *testing.T) { + var got [2]int + err := json.Unmarshal([]byte(`[1]`), &got) + switch { + case got != [2]int{1, 0}: + t.Fatalf(`json.Unmarshal = %v, want [1 0]`, got) + case json.Version == "v1" && err != nil: + t.Fatalf("json.Unmarshal error: %v", err) + case json.Version == "v2" && err == nil: + t.Fatal("json.Unmarshal error is nil, want non-nil") + } + }) + } + + for _, json := range jsonPackages { + t.Run(path.Join("Unmarshal/TooMany", json.Version), func(t *testing.T) { + var got [2]int + err := json.Unmarshal([]byte(`[1,2,3]`), &got) + switch { + case got != [2]int{1, 2}: + t.Fatalf(`json.Unmarshal = %v, want [1 2]`, got) + case json.Version == "v1" && err != nil: + t.Fatalf("json.Unmarshal error: %v", err) + case json.Version == "v2" && err == nil: + t.Fatal("json.Unmarshal error is nil, want non-nil") + } + }) + } +} + +// In v1, byte arrays are treated as arrays of unsigned integers. +// In v2, byte arrays are treated as binary values (similar to []byte). +// This is to make the behavior of [N]byte and []byte more consistent. +// +// Users of v2 can opt into the v1 behavior by setting +// the "format:array" option in the `json` struct field tag: +// +// struct { +// B [32]byte `json:",format:array"` +// } +func TestByteArrays(t *testing.T) { + for _, json := range jsonPackages { + t.Run(path.Join("Marshal", json.Version), func(t *testing.T) { + in := [4]byte{1, 2, 3, 4} + got, err := json.Marshal(in) + if err != nil { + t.Fatalf("json.Marshal error: %v", err) + } + want := map[string]string{ + "v1": `[1,2,3,4]`, + "v2": `"AQIDBA=="`, + }[json.Version] + if string(got) != want { + t.Fatalf("json.Marshal = %s, want %s", got, want) + } + }) + } + + for _, json := range jsonPackages { + t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) { + in := map[string]string{ + "v1": `[1,2,3,4]`, + "v2": `"AQIDBA=="`, + }[json.Version] + var got [4]byte + err := json.Unmarshal([]byte(in), &got) + switch { + case err != nil: + t.Fatalf("json.Unmarshal error: %v", err) + case got != [4]byte{1, 2, 3, 4}: + t.Fatalf("json.Unmarshal = %v, want [1 2 3 4]", got) + } + }) + } +} + +// CallCheck implements json.{Marshaler,Unmarshaler} on a pointer receiver. +type CallCheck string + +// MarshalJSON always returns a JSON string with the literal "CALLED". +func (*CallCheck) MarshalJSON() ([]byte, error) { + return []byte(`"CALLED"`), nil +} + +// UnmarshalJSON always stores a string with the literal "CALLED". +func (v *CallCheck) UnmarshalJSON([]byte) error { + *v = `CALLED` + return nil +} + +// In v1, the implementation is inconsistent about whether it calls +// MarshalJSON and UnmarshalJSON methods declared on pointer receivers +// when it has an unaddressable value (per reflect.Value.CanAddr) on hand. +// When marshaling, it never boxes the value on the heap to make it addressable, +// while it sometimes boxes values (e.g., for map entries) when unmarshaling. +// +// In v2, the implementation always calls MarshalJSON and UnmarshalJSON methods +// by boxing the value on the heap if necessary. +// +// The v1 behavior is surprising at best and buggy at worst. +// Unfortunately, it cannot be changed without breaking existing usages. +// +// Related issues: +// +// https://go.dev/issue/27722 +// https://go.dev/issue/33993 +// https://go.dev/issue/42508 +func TestPointerReceiver(t *testing.T) { + type Values struct { + S []CallCheck + A [1]CallCheck + M map[string]CallCheck + V CallCheck + I any + } + + for _, json := range jsonPackages { + t.Run(path.Join("Marshal", json.Version), func(t *testing.T) { + var cc CallCheck + in := Values{ + S: []CallCheck{cc}, + A: [1]CallCheck{cc}, // MarshalJSON not called on v1 + M: map[string]CallCheck{"": cc}, // MarshalJSON not called on v1 + V: cc, // MarshalJSON not called on v1 + I: cc, // MarshalJSON not called on v1 + } + want := map[string]string{ + "v1": `{"S":["CALLED"],"A":[""],"M":{"":""},"V":"","I":""}`, + "v2": `{"S":["CALLED"],"A":["CALLED"],"M":{"":"CALLED"},"V":"CALLED","I":"CALLED"}`, + }[json.Version] + got, err := json.Marshal(in) + if err != nil { + t.Fatalf("json.Marshal error: %v", err) + } + if string(got) != want { + t.Fatalf("json.Marshal = %s, want %s", got, want) + } + }) + } + + for _, json := range jsonPackages { + t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) { + in := `{"S":[""],"A":[""],"M":{"":""},"V":"","I":""}` + called := CallCheck("CALLED") // resulting state if UnmarshalJSON is called + want := map[string]Values{ + "v1": { + S: []CallCheck{called}, + A: [1]CallCheck{called}, + M: map[string]CallCheck{"": called}, + V: called, + I: "", // UnmarshalJSON not called on v1; replaced with Go string + }, + "v2": { + S: []CallCheck{called}, + A: [1]CallCheck{called}, + M: map[string]CallCheck{"": called}, + V: called, + I: called, + }, + }[json.Version] + got := Values{ + A: [1]CallCheck{CallCheck("")}, + S: []CallCheck{CallCheck("")}, + M: map[string]CallCheck{"": CallCheck("")}, + V: CallCheck(""), + I: CallCheck(""), + } + if err := json.Unmarshal([]byte(in), &got); err != nil { + t.Fatalf("json.Unmarshal error: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("json.Unmarshal = %v, want %v", got, want) + } + }) + } +} + +// In v1, maps are marshaled in a deterministic order. +// In v2, maps are marshaled in a non-deterministic order. +// +// The reason for the change is that v2 prioritizes performance and +// the guarantee that marshaling operates primarily in a streaming manner. +// +// The v2 API provides jsontext.Value.Canonicalize if stability is needed: +// +// (*jsontext.Value)(&b).Canonicalize() +// +// Related issue: +// +// https://go.dev/issue/7872 +// https://go.dev/issue/33714 +func TestMapDeterminism(t *testing.T) { + const iterations = 10 + in := map[int]int{0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7, 8: 8, 9: 9} + + for _, json := range jsonPackages { + t.Run(path.Join("Marshal", json.Version), func(t *testing.T) { + outs := make(map[string]bool) + for range iterations { + b, err := json.Marshal(in) + if err != nil { + t.Fatalf("json.Marshal error: %v", err) + } + outs[string(b)] = true + } + switch { + case json.Version == "v1" && len(outs) != 1: + t.Fatalf("json.Marshal encoded to %d unique forms, expected 1", len(outs)) + case json.Version == "v2" && len(outs) == 1: + t.Logf("json.Marshal encoded to 1 unique form by chance; are you feeling lucky?") + } + }) + } +} + +// In v1, JSON string encoding escapes special characters related to HTML. +// In v2, JSON string encoding uses a normalized representation (per RFC 8785). +// +// Users of v2 can opt into the v1 behavior by setting EscapeForHTML and EscapeForJS. +// +// Escaping HTML-specific characters in a JSON library is a layering violation. +// It presumes that JSON is always used with HTML and ignores other +// similar classes of injection attacks (e.g., SQL injection). +// Users of JSON with HTML should either manually ensure that embedded JSON is +// properly escaped or be relying on a module like "github.com/google/safehtml" +// to handle safe interoperability of JSON and HTML. +func TestEscapeHTML(t *testing.T) { + for _, json := range jsonPackages { + t.Run(path.Join("Marshal", json.Version), func(t *testing.T) { + const in = `` + got, err := json.Marshal(in) + if err != nil { + t.Fatalf("json.Marshal error: %v", err) + } + want := map[string]string{ + "v1": `"\u003cscript\u003e console.log(\"Hello, world!\"); \u003c/script\u003e"`, + "v2": `""`, + }[json.Version] + if string(got) != want { + t.Fatalf("json.Marshal = %s, want %s", got, want) + } + }) + } +} + +// In v1, JSON serialization silently ignored invalid UTF-8 by +// replacing such bytes with the Unicode replacement character. +// In v2, JSON serialization reports an error if invalid UTF-8 is encountered. +// +// Users of v2 can opt into the v1 behavior by setting [AllowInvalidUTF8]. +// +// Silently allowing invalid UTF-8 causes data corruption that can be difficult +// to detect until it is too late. Once it has been discovered, strict UTF-8 +// behavior sometimes cannot be enabled since other logic may be depending +// on the current behavior due to Hyrum's Law. +// +// Tim Bray, the author of RFC 8259 recommends that implementations should +// go beyond RFC 8259 and instead target compliance with RFC 7493, +// which makes strict decisions about behavior left undefined in RFC 8259. +// In particular, RFC 7493 rejects the presence of invalid UTF-8. +// See https://www.tbray.org/ongoing/When/201x/2017/12/14/RFC-8259-STD-90 +func TestInvalidUTF8(t *testing.T) { + for _, json := range jsonPackages { + t.Run(path.Join("Marshal", json.Version), func(t *testing.T) { + got, err := json.Marshal("\xff") + switch { + case json.Version == "v1" && err != nil: + t.Fatalf("json.Marshal error: %v", err) + case json.Version == "v1" && string(got) != "\"\ufffd\"": + t.Fatalf(`json.Marshal = %s, want %q`, got, "\ufffd") + case json.Version == "v2" && err == nil: + t.Fatal("json.Marshal error is nil, want non-nil") + } + }) + } + + for _, json := range jsonPackages { + t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) { + const in = "\"\xff\"" + var got string + err := json.Unmarshal([]byte(in), &got) + switch { + case json.Version == "v1" && err != nil: + t.Fatalf("json.Unmarshal error: %v", err) + case json.Version == "v1" && got != "\ufffd": + t.Fatalf(`json.Unmarshal = %q, want "\ufffd"`, got) + case json.Version == "v2" && err == nil: + t.Fatal("json.Unmarshal error is nil, want non-nil") + } + }) + } +} + +// In v1, duplicate JSON object names are permitted by default where +// they follow the inconsistent and difficult-to-explain merge semantics of v1. +// In v2, duplicate JSON object names are rejected by default where +// they follow the merge semantics of v2 based on RFC 7396. +// +// Users of v2 can opt into the v1 behavior by setting [AllowDuplicateNames]. +// +// Per RFC 8259, the handling of duplicate names is left as undefined behavior. +// Rejecting such inputs is within the realm of valid behavior. +// Tim Bray, the author of RFC 8259 recommends that implementations should +// go beyond RFC 8259 and instead target compliance with RFC 7493, +// which makes strict decisions about behavior left undefined in RFC 8259. +// In particular, RFC 7493 rejects the presence of duplicate object names. +// See https://www.tbray.org/ongoing/When/201x/2017/12/14/RFC-8259-STD-90 +// +// The lack of duplicate name rejection has correctness implications where +// roundtrip unmarshal/marshal do not result in semantically equivalent JSON. +// This is surprising behavior for users when they accidentally +// send JSON objects with duplicate names. +// +// The lack of duplicate name rejection may have security implications since it +// becomes difficult for a security tool to validate the semantic meaning of a +// JSON object since meaning is undefined in the presence of duplicate names. +// See https://labs.bishopfox.com/tech-blog/an-exploration-of-json-interoperability-vulnerabilities +// +// Related issue: +// +// https://go.dev/issue/48298 +func TestDuplicateNames(t *testing.T) { + for _, json := range jsonPackages { + t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) { + const in = `{"Name":1,"Name":2}` + var got struct{ Name int } + err := json.Unmarshal([]byte(in), &got) + switch { + case json.Version == "v1" && err != nil: + t.Fatalf("json.Unmarshal error: %v", err) + case json.Version == "v1" && got != struct{ Name int }{2}: + t.Fatalf(`json.Unmarshal = %v, want {2}`, got) + case json.Version == "v2" && err == nil: + t.Fatal("json.Unmarshal error is nil, want non-nil") + } + }) + } +} + +// In v1, unmarshaling a JSON null into a non-empty value was inconsistent +// in that sometimes it would be ignored and other times clear the value. +// In v2, unmarshaling a JSON null into a non-empty value would consistently +// always clear the value regardless of the value's type. +// +// The purpose of this change is to have consistent behavior with how JSON nulls +// are handled during Unmarshal. This semantic detail has no effect +// when Unmarshaling into a empty value. +// +// Related issues: +// +// https://go.dev/issue/22177 +// https://go.dev/issue/33835 +func TestMergeNull(t *testing.T) { + type Types struct { + Bool bool + String string + Bytes []byte + Int int + Map map[string]string + Struct struct{ Field string } + Slice []string + Array [1]string + Pointer *string + Interface any + } + + for _, json := range jsonPackages { + t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) { + // Start with a non-empty value where all fields are populated. + in := Types{ + Bool: true, + String: "old", + Bytes: []byte("old"), + Int: 1234, + Map: map[string]string{"old": "old"}, + Struct: struct{ Field string }{"old"}, + Slice: []string{"old"}, + Array: [1]string{"old"}, + Pointer: new(string), + Interface: "old", + } + + // Unmarshal a JSON null into every field. + if err := json.Unmarshal([]byte(`{ + "Bool": null, + "String": null, + "Bytes": null, + "Int": null, + "Map": null, + "Struct": null, + "Slice": null, + "Array": null, + "Pointer": null, + "Interface": null + }`), &in); err != nil { + t.Fatalf("json.Unmarshal error: %v", err) + } + + want := map[string]Types{ + "v1": { + Bool: true, + String: "old", + Int: 1234, + Struct: struct{ Field string }{"old"}, + Array: [1]string{"old"}, + }, + "v2": {}, // all fields are zeroed + }[json.Version] + if !reflect.DeepEqual(in, want) { + t.Fatalf("json.Unmarshal = %+v, want %+v", in, want) + } + }) + } +} + +// In v1, merge semantics are inconsistent and difficult to explain. +// In v2, merge semantics replaces the destination value for anything +// other than a JSON object, and recursively merges JSON objects. +// +// Merge semantics in v1 are inconsistent and difficult to explain +// largely because the behavior came about organically, rather than +// having a principled approach to how the semantics should operate. +// In v2, merging follows behavior based on RFC 7396. +// +// Related issues: +// +// https://go.dev/issue/21092 +// https://go.dev/issue/26946 +// https://go.dev/issue/27172 +// https://go.dev/issue/30701 +// https://go.dev/issue/31924 +// https://go.dev/issue/43664 +func TestMergeComposite(t *testing.T) { + type Tuple struct{ Old, New bool } + type Composites struct { + Slice []Tuple + Array [1]Tuple + Map map[string]Tuple + MapPointer map[string]*Tuple + Struct struct{ Tuple Tuple } + StructPointer *struct{ Tuple Tuple } + Interface any + InterfacePointer any + } + + for _, json := range jsonPackages { + t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) { + // Start with a non-empty value where all fields are populated. + in := Composites{ + Slice: []Tuple{{Old: true}, {Old: true}}[:1], + Array: [1]Tuple{{Old: true}}, + Map: map[string]Tuple{"Tuple": {Old: true}}, + MapPointer: map[string]*Tuple{"Tuple": {Old: true}}, + Struct: struct{ Tuple Tuple }{Tuple{Old: true}}, + StructPointer: &struct{ Tuple Tuple }{Tuple{Old: true}}, + Interface: Tuple{Old: true}, + InterfacePointer: &Tuple{Old: true}, + } + + // Unmarshal into every pre-populated field. + if err := json.Unmarshal([]byte(`{ + "Slice": [{"New":true}, {"New":true}], + "Array": [{"New":true}], + "Map": {"Tuple": {"New":true}}, + "MapPointer": {"Tuple": {"New":true}}, + "Struct": {"Tuple": {"New":true}}, + "StructPointer": {"Tuple": {"New":true}}, + "Interface": {"New":true}, + "InterfacePointer": {"New":true} + }`), &in); err != nil { + t.Fatalf("json.Unmarshal error: %v", err) + } + + merged := Tuple{Old: true, New: true} + replaced := Tuple{Old: false, New: true} + want := map[string]Composites{ + "v1": { + Slice: []Tuple{merged, merged}, // merged + Array: [1]Tuple{merged}, // merged + Map: map[string]Tuple{"Tuple": replaced}, // replaced + MapPointer: map[string]*Tuple{"Tuple": &replaced}, // replaced + Struct: struct{ Tuple Tuple }{merged}, // merged (same as v2) + StructPointer: &struct{ Tuple Tuple }{merged}, // merged (same as v2) + Interface: map[string]any{"New": true}, // replaced + InterfacePointer: &merged, // merged (same as v2) + }, + "v2": { + Slice: []Tuple{replaced, replaced}, // replaced + Array: [1]Tuple{replaced}, // replaced + Map: map[string]Tuple{"Tuple": merged}, // merged + MapPointer: map[string]*Tuple{"Tuple": &merged}, // merged + Struct: struct{ Tuple Tuple }{merged}, // merged (same as v1) + StructPointer: &struct{ Tuple Tuple }{merged}, // merged (same as v1) + Interface: merged, // merged + InterfacePointer: &merged, // merged (same as v1) + }, + }[json.Version] + if !reflect.DeepEqual(in, want) { + t.Fatalf("json.Unmarshal = %+v, want %+v", in, want) + } + }) + } +} + +// In v1, there was no special support for time.Duration, +// which resulted in that type simply being treated as a signed integer. +// In v2, there is now first-class support for time.Duration, where the type is +// formatted and parsed using time.Duration.String and time.ParseDuration. +// +// Users of v2 can opt into the v1 behavior by setting +// the "format:nano" option in the `json` struct field tag: +// +// struct { +// Duration time.Duration `json:",format:nano"` +// } +// +// Related issue: +// +// https://go.dev/issue/10275 +func TestTimeDurations(t *testing.T) { + t.SkipNow() // TODO(https://go.dev/issue/71631): The default representation of time.Duration is still undecided. + for _, json := range jsonPackages { + t.Run(path.Join("Marshal", json.Version), func(t *testing.T) { + got, err := json.Marshal(time.Minute) + switch { + case err != nil: + t.Fatalf("json.Marshal error: %v", err) + case json.Version == "v1" && string(got) != "60000000000": + t.Fatalf("json.Marshal = %s, want 60000000000", got) + case json.Version == "v2" && string(got) != `"1m0s"`: + t.Fatalf(`json.Marshal = %s, want "1m0s"`, got) + } + }) + } + + for _, json := range jsonPackages { + t.Run(path.Join("Unmarshal", json.Version), func(t *testing.T) { + in := map[string]string{ + "v1": "60000000000", + "v2": `"1m0s"`, + }[json.Version] + var got time.Duration + err := json.Unmarshal([]byte(in), &got) + switch { + case err != nil: + t.Fatalf("json.Unmarshal error: %v", err) + case got != time.Minute: + t.Fatalf("json.Unmarshal = %v, want 1m0s", got) + } + }) + } +} + +// In v1, non-empty structs without any JSON serializable fields are permitted. +// In v2, non-empty structs without any JSON serializable fields are rejected. +// +// The purpose of this change is to avoid a common pitfall for new users +// where they expect JSON serialization to handle unexported fields. +// However, this does not work since Go reflection does not +// provide the package the ability to mutate such fields. +// Rejecting unserializable structs in v2 is intended to be a clear signal +// that the type is not supposed to be serialized. +func TestEmptyStructs(t *testing.T) { + never := func(string) bool { return false } + onlyV2 := func(v string) bool { return v == "v2" } + values := []struct { + in any + wantError func(string) bool + }{ + // It is okay to marshal a truly empty struct in v1 and v2. + {in: addr(struct{}{}), wantError: never}, + // In v1, a non-empty struct without exported fields + // is equivalent to an empty struct, but is rejected in v2. + // Note that errors.errorString type has only unexported fields. + {in: errors.New("error"), wantError: onlyV2}, + // A mix of exported and unexported fields is permitted. + {in: addr(struct{ Exported, unexported int }{}), wantError: never}, + } + + for _, json := range jsonPackages { + t.Run("Marshal", func(t *testing.T) { + for _, value := range values { + wantError := value.wantError(json.Version) + _, err := json.Marshal(value.in) + switch { + case (err == nil) && wantError: + t.Fatalf("json.Marshal error is nil, want non-nil") + case (err != nil) && !wantError: + t.Fatalf("json.Marshal error: %v", err) + } + } + }) + } + + for _, json := range jsonPackages { + t.Run("Unmarshal", func(t *testing.T) { + for _, value := range values { + wantError := value.wantError(json.Version) + out := reflect.New(reflect.TypeOf(value.in).Elem()).Interface() + err := json.Unmarshal([]byte("{}"), out) + switch { + case (err == nil) && wantError: + t.Fatalf("json.Unmarshal error is nil, want non-nil") + case (err != nil) && !wantError: + t.Fatalf("json.Unmarshal error: %v", err) + } + } + }) + } +} diff --git a/src/encoding/json/v2_encode.go b/src/encoding/json/v2_encode.go new file mode 100644 index 00000000000000..c2d620bcbb2d32 --- /dev/null +++ b/src/encoding/json/v2_encode.go @@ -0,0 +1,251 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.jsonv2 + +// Package json implements encoding and decoding of JSON as defined in +// RFC 7159. The mapping between JSON and Go values is described +// in the documentation for the Marshal and Unmarshal functions. +// +// See "JSON and Go" for an introduction to this package: +// https://golang.org/doc/articles/json_and_go.html +// +// # Security Considerations +// +// See the "Security Considerations" section in [encoding/json/v2]. +// +// For historical reasons, the default behavior of v1 [encoding/json] +// unfortunately operates with less secure defaults. +// New usages of JSON in Go are encouraged to use [encoding/json/v2] instead. +package json + +import ( + "reflect" + "strconv" + + jsonv2 "encoding/json/v2" +) + +// Marshal returns the JSON encoding of v. +// +// Marshal traverses the value v recursively. +// If an encountered value implements [Marshaler] +// and is not a nil pointer, Marshal calls [Marshaler.MarshalJSON] +// to produce JSON. If no [Marshaler.MarshalJSON] method is present but the +// value implements [encoding.TextMarshaler] instead, Marshal calls +// [encoding.TextMarshaler.MarshalText] and encodes the result as a JSON string. +// The nil pointer exception is not strictly necessary +// but mimics a similar, necessary exception in the behavior of +// [Unmarshaler.UnmarshalJSON]. +// +// Otherwise, Marshal uses the following type-dependent default encodings: +// +// Boolean values encode as JSON booleans. +// +// Floating point, integer, and [Number] values encode as JSON numbers. +// NaN and +/-Inf values will return an [UnsupportedValueError]. +// +// String values encode as JSON strings coerced to valid UTF-8, +// replacing invalid bytes with the Unicode replacement rune. +// So that the JSON will be safe to embed inside HTML = min(r0.Max.X, s0.Max.X) || likewiseForY { etc } if r.Empty() { - return ZR + return Rectangle{} } return r } diff --git a/src/image/gif/writer.go b/src/image/gif/writer.go index 0d2a1321c01b07..129d0ab2820759 100644 --- a/src/image/gif/writer.go +++ b/src/image/gif/writer.go @@ -146,8 +146,8 @@ func (e *encoder) writeHeader() { } // Logical screen width and height. - byteorder.LePutUint16(e.buf[0:2], uint16(e.g.Config.Width)) - byteorder.LePutUint16(e.buf[2:4], uint16(e.g.Config.Height)) + byteorder.LEPutUint16(e.buf[0:2], uint16(e.g.Config.Width)) + byteorder.LEPutUint16(e.buf[2:4], uint16(e.g.Config.Height)) e.write(e.buf[:4]) if p, ok := e.g.Config.ColorModel.(color.Palette); ok && len(p) > 0 { @@ -185,7 +185,7 @@ func (e *encoder) writeHeader() { } e.buf[0] = 0x03 // Block Size. e.buf[1] = 0x01 // Sub-block Index. - byteorder.LePutUint16(e.buf[2:4], uint16(e.g.LoopCount)) + byteorder.LEPutUint16(e.buf[2:4], uint16(e.g.LoopCount)) e.buf[4] = 0x00 // Block Terminator. e.write(e.buf[:5]) } @@ -271,7 +271,7 @@ func (e *encoder) writeImageBlock(pm *image.Paletted, delay int, disposal byte) } else { e.buf[3] = 0x00 | disposal<<2 } - byteorder.LePutUint16(e.buf[4:6], uint16(delay)) // Delay Time (1/100ths of a second) + byteorder.LEPutUint16(e.buf[4:6], uint16(delay)) // Delay Time (1/100ths of a second) // Transparent color index. if transparentIndex != -1 { @@ -283,10 +283,10 @@ func (e *encoder) writeImageBlock(pm *image.Paletted, delay int, disposal byte) e.write(e.buf[:8]) } e.buf[0] = sImageDescriptor - byteorder.LePutUint16(e.buf[1:3], uint16(b.Min.X)) - byteorder.LePutUint16(e.buf[3:5], uint16(b.Min.Y)) - byteorder.LePutUint16(e.buf[5:7], uint16(b.Dx())) - byteorder.LePutUint16(e.buf[7:9], uint16(b.Dy())) + byteorder.LEPutUint16(e.buf[1:3], uint16(b.Min.X)) + byteorder.LEPutUint16(e.buf[3:5], uint16(b.Min.Y)) + byteorder.LEPutUint16(e.buf[5:7], uint16(b.Dx())) + byteorder.LEPutUint16(e.buf[7:9], uint16(b.Dy())) e.write(e.buf[:9]) // To determine whether or not this frame's palette is the same as the diff --git a/src/image/image.go b/src/image/image.go index f08182ba06fa2c..2cc94075b29ed3 100644 --- a/src/image/image.go +++ b/src/image/image.go @@ -34,7 +34,9 @@ // regardless of whether the image is itself malformed or not. A call to // [DecodeConfig] which returns a header which does not match the image returned // by [Decode] may be considered a security issue, and should be reported per the -// [Go Security Policy](https://go.dev/security/policy). +// [Go Security Policy]. +// +// [Go Security Policy]: https://go.dev/security/policy package image import ( diff --git a/src/image/jpeg/dct.go b/src/image/jpeg/dct.go new file mode 100644 index 00000000000000..b2bf27d8260ac1 --- /dev/null +++ b/src/image/jpeg/dct.go @@ -0,0 +1,521 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package jpeg + +// Discrete Cosine Transformation (DCT) implementations using the algorithm from +// Christoph Loeffler, Adriaan Lightenberg, and George S. Mostchytz, +// “Practical Fast 1-D DCT Algorithms with 11 Multiplications,” ICASSP 1989. +// https://ieeexplore.ieee.org/document/266596 +// +// Since the paper is paywalled, the rest of this comment gives a summary. +// +// A 1-dimensional forward DCT (1D FDCT) takes as input 8 values x0..x7 +// and transforms them in place into the result values. +// +// The mathematical definition of the N-point 1D FDCT is: +// +// X[k] = α_k Σ_n x[n] * cos (2n+1)*k*π/2N +// +// where α₀ = √2 and α_k = 1 for k > 0. +// +// For our purposes, N=8, so the angles end up being multiples of π/16. +// The most direct implementation of this definition would require 64 multiplications. +// +// Loeffler's paper presents a more efficient computation that requires only +// 11 multiplications and works in terms of three basic operations: +// +// - A “butterfly” x0, x1 = x0+x1, x0-x1. +// The inverse is x0, x1 = (x0+x1)/2, (x0-x1)/2. +// +// - A scaling of x0 by k: x0 *= k. The inverse is scaling by 1/k. +// +// - A rotation of x0, x1 by θ, defined as: +// x0, x1 = x0 cos θ + x1 sin θ, -x0 sin θ + x1 cos θ. +// The inverse is rotation by -θ. +// +// The algorithm proceeds in four stages: +// +// Stage 1: +// - butterfly x0, x7; x1, x6; x2, x5; x3, x4. +// +// Stage 2: +// - butterfly x0, x3; x1, x2 +// - rotate x4, x7 by 3π/16 +// - rotate x5, x6 by π/16. +// +// Stage 3: +// - butterfly x0, x1; x4, x6; x7, x5 +// - rotate x2, x3 by 6π/16 and scale by √2. +// +// Stage 4: +// - butterfly x7, x4 +// - scale x5, x6 by √2. +// +// Finally, the values are permuted. The permutation can be read as either: +// - x0, x4, x2, x6, x7, x3, x5, x1 = x0, x1, x2, x3, x4, x5, x6, x7 (paper's form) +// - x0, x1, x2, x3, x4, x5, x6, x7 = x0, x7, x2, x5, x1, x6, x3, x4 (sorted by LHS) +// The code below uses the second form to make it easier to merge adjacent stores. +// (Note that unlike in recursive FFT implementations, the permutation here is +// not always mapping indexes to their bit reversals.) +// +// As written above, the rotation requires four multiplications, but it can be +// reduced to three by refactoring (see [dctBox] below), and the scaling in +// stage 3 can be merged into the rotation constants, so the overall cost +// of a 1D FDCT is 11 multiplies. +// +// The 1D inverse DCT (IDCT) is the 1D FDCT run backward +// with all the basic operations inverted. + +// dctBox implements a 3-multiply, 3-add rotation+scaling. +// Given x0, x1, k*cos θ, and k*sin θ, dctBox returns the +// rotated and scaled coordinates. +// (It is called dctBox because the rotate+scale operation +// is drawn as a box in Figures 1 and 2 in the paper.) +func dctBox(x0, x1, kcos, ksin int32) (y0, y1 int32) { + // y0 = x0*kcos + x1*ksin + // y1 = -x0*ksin + x1*kcos + ksum := kcos * (x0 + x1) + y0 = ksum + (ksin-kcos)*x1 + y1 = ksum - (kcos+ksin)*x0 + return y0, y1 +} + +// A block is an 8x8 input to a 2D DCT (either the FDCT or IDCT). +// The input is actually only 8x8 uint8 values, and the outputs are 8x8 int16, +// but it is convenient to use int32s for intermediate storage, +// so we define only a single block type of [8*8]int32. +// +// A 2D DCT is implemented as 1D DCTs over the rows and columns. +// +// dct_test.go defines a String method for nice printing in tests. +type block [blockSize]int32 + +const blockSize = 8 * 8 + +// Note on Numerical Precision +// +// The inputs to both the FDCT and IDCT are uint8 values stored in a block, +// and the outputs are int16s in the same block, but the overall operation +// uses int32 values as fixed-point intermediate values. +// In the code comments below, the notation “QN.M” refers to a +// signed value of 1+N+M significant bits, one of which is the sign bit, +// and M of which hold fractional (sub-integer) precision. +// For example, 255 as a Q8.0 value is stored as int32(255), +// while 255 as a Q8.1 value is stored as int32(510), +// and 255.5 as a Q8.1 value is int32(511). +// The notation UQN.M refers to an unsigned value of N+M significant bits. +// See https://en.wikipedia.org/wiki/Q_(number_format) for more. +// +// In general we only need to keep about 16 significant bits, but it is more +// efficient and somewhat more precise to let unnecessary fractional bits +// accumulate and shift them away in bulk rather than after every operation. +// As such, it is important to keep track of the number of fractional bits +// in each variable at different points in the code, to avoid mistakes like +// adding numbers with different fractional precisions, as well as to keep +// track of the total number of bits, to avoid overflow. A comment like: +// +// // x[123] now Q8.2. +// +// means that x1, x2, and x3 are all Q8.2 (11-bit) values. +// Keeping extra precision bits also reduces the size of the errors introduced +// by using right shift to approximate rounded division. + +// Constants needed for the implementation. +// These are all 60-bit precision fixed-point constants. +// The function c(val, b) rounds the constant to b bits. +// c is simple enough that calls to it with constant args +// are inlined and constant-propagated down to an inline constant. +// Each constant is commented with its Ivy definition (see robpike.io/ivy), +// using this scaling helper function: +// +// op fix x = floor 0.5 + x * 2**60 +const ( + cos1 = 1130768441178740757 // fix cos 1*pi/16 + sin1 = 224923827593068887 // fix sin 1*pi/16 + cos3 = 958619196450722178 // fix cos 3*pi/16 + sin3 = 640528868967736374 // fix sin 3*pi/16 + sqrt2 = 1630477228166597777 // fix sqrt 2 + sqrt2_cos6 = 623956622067911264 // fix (sqrt 2)*cos 6*pi/16 + sqrt2_sin6 = 1506364539328854985 // fix (sqrt 2)*sin 6*pi/16 + sqrt2inv = 815238614083298888 // fix 1/sqrt 2 + sqrt2inv_cos6 = 311978311033955632 // fix (1/sqrt 2)*cos 6*pi/16 + sqrt2inv_sin6 = 753182269664427492 // fix (1/sqrt 2)*sin 6*pi/16 +) + +func c(x uint64, bits int) int32 { + return int32((x + (1 << (59 - bits))) >> (60 - bits)) +} + +// fdct implements the forward DCT. +// Inputs are UQ8.0; outputs are Q13.0. +func fdct(b *block) { + fdctCols(b) + fdctRows(b) +} + +// fdctCols applies the 1D DCT to the columns of b. +// Inputs are UQ8.0 in [0,255] but interpreted as [-128,127]. +// Outputs are Q10.18. +func fdctCols(b *block) { + for i := range 8 { + x0 := b[0*8+i] + x1 := b[1*8+i] + x2 := b[2*8+i] + x3 := b[3*8+i] + x4 := b[4*8+i] + x5 := b[5*8+i] + x6 := b[6*8+i] + x7 := b[7*8+i] + + // x[01234567] are UQ8.0 in [0,255]. + + // Stage 1: four butterflies. + // In general a butterfly of QN.M inputs produces Q(N+1).M outputs. + // A butterfly of UQN.M inputs produces a UQ(N+1).M sum and a QN.M difference. + + x0, x7 = x0+x7, x0-x7 + x1, x6 = x1+x6, x1-x6 + x2, x5 = x2+x5, x2-x5 + x3, x4 = x3+x4, x3-x4 + // x[0123] now UQ9.0 in [0, 510]. + // x[4567] now Q8.0 in [-255,255]. + + // Stage 2: two boxes and two butterflies. + // A box on QN.M inputs with B-bit constants + // produces Q(N+1).(M+B) outputs. + // (The +1 is from the addition.) + + x4, x7 = dctBox(x4, x7, c(cos3, 18), c(sin3, 18)) + x5, x6 = dctBox(x5, x6, c(cos1, 18), c(sin1, 18)) + // x[47] now Q9.18 in [-354, 354]. + // x[56] now Q9.18 in [-300, 300]. + + x0, x3 = x0+x3, x0-x3 + x1, x2 = x1+x2, x1-x2 + // x[01] now UQ10.0 in [0, 1020]. + // x[23] now Q9.0 in [-510, 510]. + + // Stage 3: one box and three butterflies. + + x2, x3 = dctBox(x2, x3, c(sqrt2_cos6, 18), c(sqrt2_sin6, 18)) + // x[23] now Q10.18 in [-943, 943]. + + x0, x1 = x0+x1, x0-x1 + // x0 now UQ11.0 in [0, 2040]. + // x1 now Q10.0 in [-1020, 1020]. + + // Store x0, x1, x2, x3 to their permuted targets. + // The original +128 in every input value + // has cancelled out except in the “DC signal” x0. + // Subtracting 128*8 here is equivalent to subtracting 128 + // from every input before we started, but cheaper. + // It also converts x0 from UQ11.18 to Q10.18. + b[0*8+i] = (x0 - 128*8) << 18 + b[4*8+i] = x1 << 18 + b[2*8+i] = x2 + b[6*8+i] = x3 + + x4, x6 = x4+x6, x4-x6 + x7, x5 = x7+x5, x7-x5 + // x[4567] now Q10.18 in [-654, 654]. + + // Stage 4: two √2 scalings and one butterfly. + + x5 = (x5 >> 12) * c(sqrt2, 12) + x6 = (x6 >> 12) * c(sqrt2, 12) + // x[56] still Q10.18 in [-925, 925] (= 654√2). + x7, x4 = x7+x4, x7-x4 + // x[47] still Q10.18 in [-925, 925] (not Q11.18!). + // This is not obvious at all! See “Note on 925” below. + + // Store x4 x5 x6 x7 to their permuted targets. + b[1*8+i] = x7 + b[3*8+i] = x5 + b[5*8+i] = x6 + b[7*8+i] = x4 + } +} + +// fdctRows applies the 1D DCT to the rows of b. +// Inputs are Q10.18; outputs are Q13.0. +func fdctRows(b *block) { + for i := range 8 { + x := b[8*i : 8*i+8 : 8*i+8] + x0 := x[0] + x1 := x[1] + x2 := x[2] + x3 := x[3] + x4 := x[4] + x5 := x[5] + x6 := x[6] + x7 := x[7] + + // x[01234567] are Q10.18 [-1020, 1020]. + + // Stage 1: four butterflies. + + x0, x7 = x0+x7, x0-x7 + x1, x6 = x1+x6, x1-x6 + x2, x5 = x2+x5, x2-x5 + x3, x4 = x3+x4, x3-x4 + // x[01234567] now Q11.18 in [-2040, 2040]. + + // Stage 2: two boxes and two butterflies. + + x4, x7 = dctBox(x4>>14, x7>>14, c(cos3, 14), c(sin3, 14)) + x5, x6 = dctBox(x5>>14, x6>>14, c(cos1, 14), c(sin1, 14)) + // x[47] now Q12.18 in [-2830, 2830]. + // x[56] now Q12.18 in [-2400, 2400]. + x0, x3 = x0+x3, x0-x3 + x1, x2 = x1+x2, x1-x2 + // x[01234567] now Q12.18 in [-4080, 4080]. + + // Stage 3: one box and three butterflies. + + x2, x3 = dctBox(x2>>14, x3>>14, c(sqrt2_cos6, 14), c(sqrt2_sin6, 14)) + // x[23] now Q13.18 in [-7539, 7539]. + x0, x1 = x0+x1, x0-x1 + // x[01] now Q13.18 in [-8160, 8160]. + x4, x6 = x4+x6, x4-x6 + x7, x5 = x7+x5, x7-x5 + // x[4567] now Q13.18 in [-5230, 5230]. + + // Stage 4: two √2 scalings and one butterfly. + + x5 = (x5 >> 14) * c(sqrt2, 14) + x6 = (x6 >> 14) * c(sqrt2, 14) + // x[56] still Q13.18 in [-7397, 7397] (= 5230√2). + x7, x4 = x7+x4, x7-x4 + // x[47] still Q13.18 in [-7395, 7395] (= 2040*3.6246). + // See “Note on 925” below. + + // Cut from Q13.18 to Q13.0. + x0 = (x0 + 1<<17) >> 18 + x1 = (x1 + 1<<17) >> 18 + x2 = (x2 + 1<<17) >> 18 + x3 = (x3 + 1<<17) >> 18 + x4 = (x4 + 1<<17) >> 18 + x5 = (x5 + 1<<17) >> 18 + x6 = (x6 + 1<<17) >> 18 + x7 = (x7 + 1<<17) >> 18 + + // Note: Unlike in fdctCols, saved all stores for the end + // because they are adjacent memory locations and some systems + // can use multiword stores. + x[0] = x0 + x[1] = x7 + x[2] = x2 + x[3] = x5 + x[4] = x1 + x[5] = x6 + x[6] = x3 + x[7] = x4 + } +} + +// “Note on 925”, deferred from above to avoid interrupting code. +// +// In fdctCols, heading into stage 2, the values x4, x5, x6, x7 are in [-255, 255]. +// Let's call those specific values b4, b5, b6, b7, and trace how x[4567] evolve: +// +// Stage 2: +// x4 = b4*cos3 + b7*sin3 +// x7 = -b4*sin3 + b7*cos3 +// x5 = b5*cos1 + b6*sin1 +// x6 = -b5*sin1 + b6*cos1 +// +// Stage 3: +// +// x4 = x4+x6 = b4*cos3 + b7*sin3 - b5*sin1 + b6*cos1 +// x6 = x4-x6 = b4*cos3 + b7*sin3 + b5*sin1 - b6*cos1 +// x7 = x7+x5 = -b4*sin3 + b7*cos3 + b5*cos1 + b6*sin1 +// x5 = x7-x5 = -b4*sin3 + b7*cos3 - b5*cos1 - b6*sin1 +// +// Stage 4: +// +// x7 = x7+x4 = -b4*sin3 + b7*cos3 + b5*cos1 + b6*sin1 + b4*cos3 + b7*sin3 - b5*sin1 + b6*cos1 +// = b4*(cos3-sin3) + b5*(cos1-sin1) + b6*(cos1+sin1) + b7*(cos3+sin3) +// < 255*(0.2759 + 0.7857 + 1.1759 + 1.3871) = 255*3.6246 < 925. +// +// x4 = x7-x4 = -b4*sin3 + b7*cos3 + b5*cos1 + b6*sin1 - b4*cos3 - b7*sin3 + b5*sin1 - b6*cos1 +// = -b4*(cos3+sin3) + b5*(cos1+sin1) + b6*(sin1-cos1) + b7*(cos3-sin3) +// < same 925. +// +// The fact that x5, x6 are also at most 925 is not a coincidence: we are computing +// the same kinds of numbers for all four, just with different paths to them. +// +// In fdctRows, the same analysis applies, but the initial values are +// in [-2040, 2040] instead of [-255, 255], so the bound is 2040*3.6246 < 7395. + +// idct implements the inverse DCT. +// Inputs are UQ8.0; outputs are Q10.3. +func idct(b *block) { + // A 2D IDCT is a 1D IDCT on rows followed by columns. + idctRows(b) + idctCols(b) +} + +// idctRows applies the 1D IDCT to the rows of b. +// Inputs are UQ8.0; outputs are Q9.20. +func idctRows(b *block) { + for i := range 8 { + x := b[8*i : 8*i+8 : 8*i+8] + x0 := x[0] + x7 := x[1] + x2 := x[2] + x5 := x[3] + x1 := x[4] + x6 := x[5] + x3 := x[6] + x4 := x[7] + + // Run FDCT backward. + // Independent operations have been reordered somewhat + // to make precision tracking easier. + // + // Note that “x0, x1 = x0+x1, x0-x1” is now a reverse butterfly + // and carries with it an implicit divide by two: the extra bit + // is added to the precision, not the value size. + + // x[01234567] are UQ8.0 in [0, 255]. + + // Stages 4, 3, 2: x0, x1, x2, x3. + + x0 <<= 17 + x1 <<= 17 + // x0, x1 now UQ8.17. + x0, x1 = x0+x1, x0-x1 + // x0 now UQ8.18 in [0, 255]. + // x1 now Q7.18 in [-127½, 127½]. + + // Note: (1/sqrt 2)*((cos 6*pi/16)+(sin 6*pi/16)) < 0.924, so no new high bit. + x2, x3 = dctBox(x2, x3, c(sqrt2inv_cos6, 18), -c(sqrt2inv_sin6, 18)) + // x[23] now Q8.18 in [-236, 236]. + x1, x2 = x1+x2, x1-x2 + x0, x3 = x0+x3, x0-x3 + // x[0123] now Q8.19 in [-246, 246]. + + // Stages 4, 3, 2: x4, x5, x6, x7. + + x4 <<= 7 + x7 <<= 7 + // x[47] now UQ8.7 + x7, x4 = x7+x4, x7-x4 + // x7 now UQ8.8 in [0, 255]. + // x4 now Q7.8 in [-127½, 127½]. + + x6 = x6 * c(sqrt2inv, 8) + x5 = x5 * c(sqrt2inv, 8) + // x[56] now UQ8.8 in [0, 181]. + // Note that 1/√2 has five 0s in its binary representation after + // the 8th bit, so this multipliy is actually producing 12 bits of precision. + + x7, x5 = x7+x5, x7-x5 + x4, x6 = x4+x6, x4-x6 + // x[4567] now Q8.9 in [-218, 218]. + + x4, x7 = dctBox(x4>>2, x7>>2, c(cos3, 12), -c(sin3, 12)) + x5, x6 = dctBox(x5>>2, x6>>2, c(cos1, 12), -c(sin1, 12)) + // x[4567] now Q9.19 in [-303, 303]. + + // Stage 1. + + x0, x7 = x0+x7, x0-x7 + x1, x6 = x1+x6, x1-x6 + x2, x5 = x2+x5, x2-x5 + x3, x4 = x3+x4, x3-x4 + // x[01234567] now Q9.20 in [-275, 275]. + + // Note: we don't need all 20 bits of “precision”, + // but it is faster to let idctCols shift it away as part + // of other operations rather than downshift here. + + x[0] = x0 + x[1] = x1 + x[2] = x2 + x[3] = x3 + x[4] = x4 + x[5] = x5 + x[6] = x6 + x[7] = x7 + } +} + +// idctCols applies the 1D IDCT to the columns of b. +// Inputs are Q9.20. +// Outputs are Q10.3. That is, the result is the IDCT*8. +func idctCols(b *block) { + for i := range 8 { + x0 := b[0*8+i] + x7 := b[1*8+i] + x2 := b[2*8+i] + x5 := b[3*8+i] + x1 := b[4*8+i] + x6 := b[5*8+i] + x3 := b[6*8+i] + x4 := b[7*8+i] + + // x[012345678] are Q9.20. + + // Start by adding 0.5 to x0 (the incoming DC signal). + // The butterflies will add it to all the other values, + // and then the final shifts will round properly. + x0 += 1 << 19 + + // Stages 4, 3, 2: x0, x1, x2, x3. + + x0, x1 = (x0+x1)>>2, (x0-x1)>>2 + // x[01] now Q9.19. + // Note: (1/sqrt 2)*((cos 6*pi/16)+(sin 6*pi/16)) < 1, so no new high bit. + x2, x3 = dctBox(x2>>13, x3>>13, c(sqrt2inv_cos6, 12), -c(sqrt2inv_sin6, 12)) + // x[0123] now Q9.19. + + x1, x2 = x1+x2, x1-x2 + x0, x3 = x0+x3, x0-x3 + // x[0123] now Q9.20. + + // Stages 4, 3, 2: x4, x5, x6, x7. + + x7, x4 = x7+x4, x7-x4 + // x[47] now Q9.21. + + x5 = (x5 >> 13) * c(sqrt2inv, 14) + x6 = (x6 >> 13) * c(sqrt2inv, 14) + // x[56] now Q9.21. + + x7, x5 = x7+x5, x7-x5 + x4, x6 = x4+x6, x4-x6 + // x[4567] now Q9.22. + + x4, x7 = dctBox(x4>>14, x7>>14, c(cos3, 12), -c(sin3, 12)) + x5, x6 = dctBox(x5>>14, x6>>14, c(cos1, 12), -c(sin1, 12)) + // x[4567] now Q10.20. + + x0, x7 = x0+x7, x0-x7 + x1, x6 = x1+x6, x1-x6 + x2, x5 = x2+x5, x2-x5 + x3, x4 = x3+x4, x3-x4 + // x[01234567] now Q10.21. + + x0 >>= 18 + x1 >>= 18 + x2 >>= 18 + x3 >>= 18 + x4 >>= 18 + x5 >>= 18 + x6 >>= 18 + x7 >>= 18 + // x[01234567] now Q10.3. + + b[0*8+i] = x0 + b[1*8+i] = x1 + b[2*8+i] = x2 + b[3*8+i] = x3 + b[4*8+i] = x4 + b[5*8+i] = x5 + b[6*8+i] = x6 + b[7*8+i] = x7 + } +} diff --git a/src/image/jpeg/dct_test.go b/src/image/jpeg/dct_test.go index ed5b73d562cbdb..0375a7113dee9f 100644 --- a/src/image/jpeg/dct_test.go +++ b/src/image/jpeg/dct_test.go @@ -7,20 +7,18 @@ package jpeg import ( "fmt" "math" + "math/big" "math/rand" "strings" "testing" ) func benchmarkDCT(b *testing.B, f func(*block)) { - b.StopTimer() - blocks := make([]block, 0, b.N*len(testBlocks)) - for i := 0; i < b.N; i++ { - blocks = append(blocks, testBlocks[:]...) - } - b.StartTimer() - for i := range blocks { - f(&blocks[i]) + var blk block // avoid potential allocation in loop + for b.Loop() { + for _, blk = range testBlocks { + f(&blk) + } } } @@ -32,11 +30,37 @@ func BenchmarkIDCT(b *testing.B) { benchmarkDCT(b, idct) } +const testSlowVsBig = true + func TestDCT(t *testing.T) { blocks := make([]block, len(testBlocks)) copy(blocks, testBlocks[:]) - // Append some randomly generated blocks of varying sparseness. + // All zeros + blocks = append(blocks, block{}) + + // Every possible unit impulse. + for i := range blockSize { + var b block + b[i] = 255 + blocks = append(blocks, b) + } + + // All ones. + var ones block + for i := range ones { + ones[i] = 255 + } + blocks = append(blocks, ones) + + // Every possible inverted unit impulse. + for i := range blockSize { + ones[i] = 0 + blocks = append(blocks, ones) + ones[i] = 255 + } + + // Some randomly generated blocks of varying sparseness. r := rand.New(rand.NewSource(123)) for i := 0; i < 100; i++ { b := block{} @@ -47,61 +71,84 @@ func TestDCT(t *testing.T) { blocks = append(blocks, b) } - // Check that the FDCT and IDCT functions are inverses, after a scale and - // level shift. Scaling reduces the rounding errors in the conversion from - // floats to ints. - for i, b := range blocks { - got, want := b, b - for j := range got { - got[j] = (got[j] - 128) * 8 - } - slowFDCT(&got) - slowIDCT(&got) - for j := range got { - got[j] = got[j]/8 + 128 - } - if differ(&got, &want) { - t.Errorf("i=%d: IDCT(FDCT)\nsrc\n%s\ngot\n%s\nwant\n%s\n", i, &b, &got, &want) + // Check that the slow FDCT and IDCT functions are inverses, + // after a scale and level shift. + // Scaling reduces the rounding errors in the conversion. + // The “fast” ones are not inverses because the fast IDCT + // is optimized for 8-bit inputs, not full 16-bit ones. + slowRoundTrip := func(b *block) { + slowFDCT(b) + slowIDCT(b) + for j := range b { + b[j] = b[j]/8 + 128 } } + nop := func(*block) {} + testDCT(t, "IDCT(FDCT)", blocks, slowRoundTrip, nop, 1, 8) - // Check that the optimized and slow FDCT implementations agree. - // The fdct function already does a scale and level shift. - for i, b := range blocks { - got, want := b, b - fdct(&got) - for j := range want { - want[j] = (want[j] - 128) * 8 - } - slowFDCT(&want) - if differ(&got, &want) { - t.Errorf("i=%d: FDCT\nsrc\n%s\ngot\n%s\nwant\n%s\n", i, &b, &got, &want) - } + if testSlowVsBig { + testDCT(t, "slowFDCT", blocks, slowFDCT, slowerFDCT, 0, 64) + testDCT(t, "slowIDCT", blocks, slowIDCT, slowerIDCT, 0, 64) } - // Check that the optimized and slow IDCT implementations agree. - for i, b := range blocks { - got, want := b, b - idct(&got) - slowIDCT(&want) - if differ(&got, &want) { - t.Errorf("i=%d: IDCT\nsrc\n%s\ngot\n%s\nwant\n%s\n", i, &b, &got, &want) + // Check that the optimized and slow FDCT implementations agree. + testDCT(t, "FDCT", blocks, fdct, slowFDCT, 1, 8) + testDCT(t, "IDCT", blocks, idct, slowIDCT, 1, 8) +} + +func testDCT(t *testing.T, name string, blocks []block, fhave, fwant func(*block), tolerance int32, maxCloseCalls int) { + t.Run(name, func(t *testing.T) { + totalClose := 0 + for i, b := range blocks { + have, want := b, b + fhave(&have) + fwant(&want) + d, n := differ(&have, &want, tolerance) + if d >= 0 || n > maxCloseCalls { + fail := "" + if d >= 0 { + fail = fmt.Sprintf("diff at %d,%d", d/8, d%8) + } + if n > maxCloseCalls { + if fail != "" { + fail += "; " + } + fail += fmt.Sprintf("%d close calls", n) + } + t.Errorf("i=%d: %s (%s)\nsrc\n%s\nhave\n%s\nwant\n%s\n", + i, name, fail, &b, &have, &want) + } + totalClose += n } - } + if tolerance > 0 { + t.Logf("%d/%d total close calls", totalClose, len(blocks)*blockSize) + } + }) } -// differ reports whether any pair-wise elements in b0 and b1 differ by 2 or -// more. That tolerance is because there isn't a single definitive decoding of -// a given JPEG image, even before the YCbCr to RGB conversion; implementations +// differ returns the index of the first pair-wise elements in b0 and b1 +// that differ by more than 'ok', along with the total number of elements +// that differ by at least ok ("close calls"). +// +// There isn't a single definitive decoding of a given JPEG image, +// even before the YCbCr to RGB conversion; implementations // can have different IDCT rounding errors. -func differ(b0, b1 *block) bool { +// +// If there are no differences, differ returns -1, 0. +func differ(b0, b1 *block, ok int32) (index, closeCalls int) { + index = -1 for i := range b0 { delta := b0[i] - b1[i] - if delta < -2 || +2 < delta { - return true + if delta < -ok || ok < delta { + if index < 0 { + index = i + } + } + if delta <= -ok || ok <= delta { + closeCalls++ } } - return false + return } // alpha returns 1 if i is 0 and returns √2 otherwise. @@ -112,12 +159,101 @@ func alpha(i int) float64 { return math.Sqrt2 } -var cosines [32]float64 // cosines[k] = cos(π/2 * k/8) +// bigAlpha returns 1 if i is 0 and returns √2 otherwise. +func bigAlpha(i int) *big.Float { + if i == 0 { + return bigFloat1 + } + return bigFloatSqrt2 +} + +var cosines = [32]float64{ + +1.0000000000000000000000000000000000000000000000000000000000000000, // cos(π/16 * 0) + +0.9807852804032304491261822361342390369739337308933360950029160885, // cos(π/16 * 1) + +0.9238795325112867561281831893967882868224166258636424861150977312, // cos(π/16 * 2) + +0.8314696123025452370787883776179057567385608119872499634461245902, // cos(π/16 * 3) + +0.7071067811865475244008443621048490392848359376884740365883398689, // cos(π/16 * 4) + +0.5555702330196022247428308139485328743749371907548040459241535282, // cos(π/16 * 5) + +0.3826834323650897717284599840303988667613445624856270414338006356, // cos(π/16 * 6) + +0.1950903220161282678482848684770222409276916177519548077545020894, // cos(π/16 * 7) -func init() { - for k := range cosines { - cosines[k] = math.Cos(math.Pi * float64(k) / 16) + -0.0000000000000000000000000000000000000000000000000000000000000000, // cos(π/16 * 8) + -0.1950903220161282678482848684770222409276916177519548077545020894, // cos(π/16 * 9) + -0.3826834323650897717284599840303988667613445624856270414338006356, // cos(π/16 * 10) + -0.5555702330196022247428308139485328743749371907548040459241535282, // cos(π/16 * 11) + -0.7071067811865475244008443621048490392848359376884740365883398689, // cos(π/16 * 12) + -0.8314696123025452370787883776179057567385608119872499634461245902, // cos(π/16 * 13) + -0.9238795325112867561281831893967882868224166258636424861150977312, // cos(π/16 * 14) + -0.9807852804032304491261822361342390369739337308933360950029160885, // cos(π/16 * 15) + + -1.0000000000000000000000000000000000000000000000000000000000000000, // cos(π/16 * 16) + -0.9807852804032304491261822361342390369739337308933360950029160885, // cos(π/16 * 17) + -0.9238795325112867561281831893967882868224166258636424861150977312, // cos(π/16 * 18) + -0.8314696123025452370787883776179057567385608119872499634461245902, // cos(π/16 * 19) + -0.7071067811865475244008443621048490392848359376884740365883398689, // cos(π/16 * 20) + -0.5555702330196022247428308139485328743749371907548040459241535282, // cos(π/16 * 21) + -0.3826834323650897717284599840303988667613445624856270414338006356, // cos(π/16 * 22) + -0.1950903220161282678482848684770222409276916177519548077545020894, // cos(π/16 * 23) + + +0.0000000000000000000000000000000000000000000000000000000000000000, // cos(π/16 * 24) + +0.1950903220161282678482848684770222409276916177519548077545020894, // cos(π/16 * 25) + +0.3826834323650897717284599840303988667613445624856270414338006356, // cos(π/16 * 26) + +0.5555702330196022247428308139485328743749371907548040459241535282, // cos(π/16 * 27) + +0.7071067811865475244008443621048490392848359376884740365883398689, // cos(π/16 * 28) + +0.8314696123025452370787883776179057567385608119872499634461245902, // cos(π/16 * 29) + +0.9238795325112867561281831893967882868224166258636424861150977312, // cos(π/16 * 30) + +0.9807852804032304491261822361342390369739337308933360950029160885, // cos(π/16 * 31) +} + +func bigFloat(s string) *big.Float { + f, ok := new(big.Float).SetString(s) + if !ok { + panic("bad float") } + return f +} + +var ( + bigFloat1 = big.NewFloat(1) + bigFloatSqrt2 = bigFloat("1.41421356237309504880168872420969807856967187537694807317667974") +) + +var bigCosines = [32]*big.Float{ + bigFloat("+1.0000000000000000000000000000000000000000000000000000000000000000"), // cos(π/16 * 0) + bigFloat("+0.9807852804032304491261822361342390369739337308933360950029160885"), // cos(π/16 * 1) + bigFloat("+0.9238795325112867561281831893967882868224166258636424861150977312"), // cos(π/16 * 2) + bigFloat("+0.8314696123025452370787883776179057567385608119872499634461245902"), // cos(π/16 * 3) + bigFloat("+0.7071067811865475244008443621048490392848359376884740365883398689"), // cos(π/16 * 4) + bigFloat("+0.5555702330196022247428308139485328743749371907548040459241535282"), // cos(π/16 * 5) + bigFloat("+0.3826834323650897717284599840303988667613445624856270414338006356"), // cos(π/16 * 6) + bigFloat("+0.1950903220161282678482848684770222409276916177519548077545020894"), // cos(π/16 * 7) + + bigFloat("-0.0000000000000000000000000000000000000000000000000000000000000000"), // cos(π/16 * 8) + bigFloat("-0.1950903220161282678482848684770222409276916177519548077545020894"), // cos(π/16 * 9) + bigFloat("-0.3826834323650897717284599840303988667613445624856270414338006356"), // cos(π/16 * 10) + bigFloat("-0.5555702330196022247428308139485328743749371907548040459241535282"), // cos(π/16 * 11) + bigFloat("-0.7071067811865475244008443621048490392848359376884740365883398689"), // cos(π/16 * 12) + bigFloat("-0.8314696123025452370787883776179057567385608119872499634461245902"), // cos(π/16 * 13) + bigFloat("-0.9238795325112867561281831893967882868224166258636424861150977312"), // cos(π/16 * 14) + bigFloat("-0.9807852804032304491261822361342390369739337308933360950029160885"), // cos(π/16 * 15) + + bigFloat("-1.0000000000000000000000000000000000000000000000000000000000000000"), // cos(π/16 * 16) + bigFloat("-0.9807852804032304491261822361342390369739337308933360950029160885"), // cos(π/16 * 17) + bigFloat("-0.9238795325112867561281831893967882868224166258636424861150977312"), // cos(π/16 * 18) + bigFloat("-0.8314696123025452370787883776179057567385608119872499634461245902"), // cos(π/16 * 19) + bigFloat("-0.7071067811865475244008443621048490392848359376884740365883398689"), // cos(π/16 * 20) + bigFloat("-0.5555702330196022247428308139485328743749371907548040459241535282"), // cos(π/16 * 21) + bigFloat("-0.3826834323650897717284599840303988667613445624856270414338006356"), // cos(π/16 * 22) + bigFloat("-0.1950903220161282678482848684770222409276916177519548077545020894"), // cos(π/16 * 23) + + bigFloat("+0.0000000000000000000000000000000000000000000000000000000000000000"), // cos(π/16 * 24) + bigFloat("+0.1950903220161282678482848684770222409276916177519548077545020894"), // cos(π/16 * 25) + bigFloat("+0.3826834323650897717284599840303988667613445624856270414338006356"), // cos(π/16 * 26) + bigFloat("+0.5555702330196022247428308139485328743749371907548040459241535282"), // cos(π/16 * 27) + bigFloat("+0.7071067811865475244008443621048490392848359376884740365883398689"), // cos(π/16 * 28) + bigFloat("+0.8314696123025452370787883776179057567385608119872499634461245902"), // cos(π/16 * 29) + bigFloat("+0.9238795325112867561281831893967882868224166258636424861150977312"), // cos(π/16 * 30) + bigFloat("+0.9807852804032304491261822361342390369739337308933360950029160885"), // cos(π/16 * 31) } // slowFDCT performs the 8*8 2-dimensional forward discrete cosine transform: @@ -130,24 +266,51 @@ func init() { // // b acts as both dst and src. func slowFDCT(b *block) { - var dst [blockSize]float64 + var dst block for v := 0; v < 8; v++ { for u := 0; u < 8; u++ { sum := 0.0 for y := 0; y < 8; y++ { for x := 0; x < 8; x++ { - sum += alpha(u) * alpha(v) * float64(b[8*y+x]) * + sum += alpha(u) * alpha(v) * float64(b[8*y+x]-128) * cosines[((2*x+1)*u)%32] * cosines[((2*y+1)*v)%32] } } - dst[8*v+u] = sum / 8 + dst[8*v+u] = int32(math.Round(sum)) } } - // Convert from float64 to int32. - for i := range dst { - b[i] = int32(dst[i] + 0.5) + *b = dst +} + +// slowerFDCT is slowFDCT but using big.Floats to validate slowFDCT. +func slowerFDCT(b *block) { + var dst block + for v := 0; v < 8; v++ { + for u := 0; u < 8; u++ { + sum := big.NewFloat(0) + for y := 0; y < 8; y++ { + for x := 0; x < 8; x++ { + f := big.NewFloat(float64(b[8*y+x] - 128)) + f = new(big.Float).Mul(f, bigAlpha(u)) + f = new(big.Float).Mul(f, bigAlpha(v)) + f = new(big.Float).Mul(f, bigCosines[((2*x+1)*u)%32]) + f = new(big.Float).Mul(f, bigCosines[((2*y+1)*v)%32]) + sum = new(big.Float).Add(sum, f) + } + } + // Int64 truncates toward zero, so add ±0.5 + // as needed to round + if sum.Sign() > 0 { + sum = new(big.Float).Add(sum, big.NewFloat(+0.5)) + } else { + sum = new(big.Float).Add(sum, big.NewFloat(-0.5)) + } + i, _ := sum.Int64() + dst[8*v+u] = int32(i) + } } + *b = dst } // slowIDCT performs the 8*8 2-dimensional inverse discrete cosine transform: @@ -160,7 +323,7 @@ func slowFDCT(b *block) { // // b acts as both dst and src. func slowIDCT(b *block) { - var dst [blockSize]float64 + var dst block for y := 0; y < 8; y++ { for x := 0; x < 8; x++ { sum := 0.0 @@ -171,13 +334,41 @@ func slowIDCT(b *block) { cosines[((2*y+1)*v)%32] } } - dst[8*y+x] = sum / 8 + dst[8*y+x] = int32(math.Round(sum / 8)) } } - // Convert from float64 to int32. - for i := range dst { - b[i] = int32(dst[i] + 0.5) + *b = dst +} + +// slowerIDCT is slowIDCT but using big.Floats to validate slowIDCT. +func slowerIDCT(b *block) { + var dst block + for y := 0; y < 8; y++ { + for x := 0; x < 8; x++ { + sum := big.NewFloat(0) + for v := 0; v < 8; v++ { + for u := 0; u < 8; u++ { + f := big.NewFloat(float64(b[8*v+u])) + f = new(big.Float).Mul(f, bigAlpha(u)) + f = new(big.Float).Mul(f, bigAlpha(v)) + f = new(big.Float).Mul(f, bigCosines[((2*x+1)*u)%32]) + f = new(big.Float).Mul(f, bigCosines[((2*y+1)*v)%32]) + f = new(big.Float).Quo(f, big.NewFloat(8)) + sum = new(big.Float).Add(sum, f) + } + } + // Int64 truncates toward zero, so add ±0.5 + // as needed to round + if sum.Sign() > 0 { + sum = new(big.Float).Add(sum, big.NewFloat(+0.5)) + } else { + sum = new(big.Float).Add(sum, big.NewFloat(-0.5)) + } + i, _ := sum.Int64() + dst[8*y+x] = int32(i) + } } + *b = dst } func (b *block) String() string { diff --git a/src/image/jpeg/fdct.go b/src/image/jpeg/fdct.go deleted file mode 100644 index c7a973ec3c001d..00000000000000 --- a/src/image/jpeg/fdct.go +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package jpeg - -// This file implements a Forward Discrete Cosine Transformation. - -/* -It is based on the code in jfdctint.c from the Independent JPEG Group, -found at http://www.ijg.org/files/jpegsrc.v8c.tar.gz. - -The "LEGAL ISSUES" section of the README in that archive says: - -In plain English: - -1. We don't promise that this software works. (But if you find any bugs, - please let us know!) -2. You can use this software for whatever you want. You don't have to pay us. -3. You may not pretend that you wrote this software. If you use it in a - program, you must acknowledge somewhere in your documentation that - you've used the IJG code. - -In legalese: - -The authors make NO WARRANTY or representation, either express or implied, -with respect to this software, its quality, accuracy, merchantability, or -fitness for a particular purpose. This software is provided "AS IS", and you, -its user, assume the entire risk as to its quality and accuracy. - -This software is copyright (C) 1991-2011, Thomas G. Lane, Guido Vollbeding. -All Rights Reserved except as specified below. - -Permission is hereby granted to use, copy, modify, and distribute this -software (or portions thereof) for any purpose, without fee, subject to these -conditions: -(1) If any part of the source code for this software is distributed, then this -README file must be included, with this copyright and no-warranty notice -unaltered; and any additions, deletions, or changes to the original files -must be clearly indicated in accompanying documentation. -(2) If only executable code is distributed, then the accompanying -documentation must state that "this software is based in part on the work of -the Independent JPEG Group". -(3) Permission for use of this software is granted only if the user accepts -full responsibility for any undesirable consequences; the authors accept -NO LIABILITY for damages of any kind. - -These conditions apply to any software derived from or based on the IJG code, -not just to the unmodified library. If you use our work, you ought to -acknowledge us. - -Permission is NOT granted for the use of any IJG author's name or company name -in advertising or publicity relating to this software or products derived from -it. This software may be referred to only as "the Independent JPEG Group's -software". - -We specifically permit and encourage the use of this software as the basis of -commercial products, provided that all warranty or liability claims are -assumed by the product vendor. -*/ - -// Trigonometric constants in 13-bit fixed point format. -const ( - fix_0_298631336 = 2446 - fix_0_390180644 = 3196 - fix_0_541196100 = 4433 - fix_0_765366865 = 6270 - fix_0_899976223 = 7373 - fix_1_175875602 = 9633 - fix_1_501321110 = 12299 - fix_1_847759065 = 15137 - fix_1_961570560 = 16069 - fix_2_053119869 = 16819 - fix_2_562915447 = 20995 - fix_3_072711026 = 25172 -) - -const ( - constBits = 13 - pass1Bits = 2 - centerJSample = 128 -) - -// fdct performs a forward DCT on an 8x8 block of coefficients, including a -// level shift. -func fdct(b *block) { - // Pass 1: process rows. - for y := 0; y < 8; y++ { - y8 := y * 8 - s := b[y8 : y8+8 : y8+8] // Small cap improves performance, see https://golang.org/issue/27857 - x0 := s[0] - x1 := s[1] - x2 := s[2] - x3 := s[3] - x4 := s[4] - x5 := s[5] - x6 := s[6] - x7 := s[7] - - tmp0 := x0 + x7 - tmp1 := x1 + x6 - tmp2 := x2 + x5 - tmp3 := x3 + x4 - - tmp10 := tmp0 + tmp3 - tmp12 := tmp0 - tmp3 - tmp11 := tmp1 + tmp2 - tmp13 := tmp1 - tmp2 - - tmp0 = x0 - x7 - tmp1 = x1 - x6 - tmp2 = x2 - x5 - tmp3 = x3 - x4 - - s[0] = (tmp10 + tmp11 - 8*centerJSample) << pass1Bits - s[4] = (tmp10 - tmp11) << pass1Bits - z1 := (tmp12 + tmp13) * fix_0_541196100 - z1 += 1 << (constBits - pass1Bits - 1) - s[2] = (z1 + tmp12*fix_0_765366865) >> (constBits - pass1Bits) - s[6] = (z1 - tmp13*fix_1_847759065) >> (constBits - pass1Bits) - - tmp10 = tmp0 + tmp3 - tmp11 = tmp1 + tmp2 - tmp12 = tmp0 + tmp2 - tmp13 = tmp1 + tmp3 - z1 = (tmp12 + tmp13) * fix_1_175875602 - z1 += 1 << (constBits - pass1Bits - 1) - tmp0 *= fix_1_501321110 - tmp1 *= fix_3_072711026 - tmp2 *= fix_2_053119869 - tmp3 *= fix_0_298631336 - tmp10 *= -fix_0_899976223 - tmp11 *= -fix_2_562915447 - tmp12 *= -fix_0_390180644 - tmp13 *= -fix_1_961570560 - - tmp12 += z1 - tmp13 += z1 - s[1] = (tmp0 + tmp10 + tmp12) >> (constBits - pass1Bits) - s[3] = (tmp1 + tmp11 + tmp13) >> (constBits - pass1Bits) - s[5] = (tmp2 + tmp11 + tmp12) >> (constBits - pass1Bits) - s[7] = (tmp3 + tmp10 + tmp13) >> (constBits - pass1Bits) - } - // Pass 2: process columns. - // We remove pass1Bits scaling, but leave results scaled up by an overall factor of 8. - for x := 0; x < 8; x++ { - tmp0 := b[0*8+x] + b[7*8+x] - tmp1 := b[1*8+x] + b[6*8+x] - tmp2 := b[2*8+x] + b[5*8+x] - tmp3 := b[3*8+x] + b[4*8+x] - - tmp10 := tmp0 + tmp3 + 1<<(pass1Bits-1) - tmp12 := tmp0 - tmp3 - tmp11 := tmp1 + tmp2 - tmp13 := tmp1 - tmp2 - - tmp0 = b[0*8+x] - b[7*8+x] - tmp1 = b[1*8+x] - b[6*8+x] - tmp2 = b[2*8+x] - b[5*8+x] - tmp3 = b[3*8+x] - b[4*8+x] - - b[0*8+x] = (tmp10 + tmp11) >> pass1Bits - b[4*8+x] = (tmp10 - tmp11) >> pass1Bits - - z1 := (tmp12 + tmp13) * fix_0_541196100 - z1 += 1 << (constBits + pass1Bits - 1) - b[2*8+x] = (z1 + tmp12*fix_0_765366865) >> (constBits + pass1Bits) - b[6*8+x] = (z1 - tmp13*fix_1_847759065) >> (constBits + pass1Bits) - - tmp10 = tmp0 + tmp3 - tmp11 = tmp1 + tmp2 - tmp12 = tmp0 + tmp2 - tmp13 = tmp1 + tmp3 - z1 = (tmp12 + tmp13) * fix_1_175875602 - z1 += 1 << (constBits + pass1Bits - 1) - tmp0 *= fix_1_501321110 - tmp1 *= fix_3_072711026 - tmp2 *= fix_2_053119869 - tmp3 *= fix_0_298631336 - tmp10 *= -fix_0_899976223 - tmp11 *= -fix_2_562915447 - tmp12 *= -fix_0_390180644 - tmp13 *= -fix_1_961570560 - - tmp12 += z1 - tmp13 += z1 - b[1*8+x] = (tmp0 + tmp10 + tmp12) >> (constBits + pass1Bits) - b[3*8+x] = (tmp1 + tmp11 + tmp13) >> (constBits + pass1Bits) - b[5*8+x] = (tmp2 + tmp11 + tmp12) >> (constBits + pass1Bits) - b[7*8+x] = (tmp3 + tmp10 + tmp13) >> (constBits + pass1Bits) - } -} diff --git a/src/image/jpeg/idct.go b/src/image/jpeg/idct.go deleted file mode 100644 index a3957c8ada59ac..00000000000000 --- a/src/image/jpeg/idct.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package jpeg - -// This is a Go translation of idct.c from -// -// http://standards.iso.org/ittf/PubliclyAvailableStandards/ISO_IEC_13818-4_2004_Conformance_Testing/Video/verifier/mpeg2decode_960109.tar.gz -// -// which carries the following notice: - -/* Copyright (C) 1996, MPEG Software Simulation Group. All Rights Reserved. */ - -/* - * Disclaimer of Warranty - * - * These software programs are available to the user without any license fee or - * royalty on an "as is" basis. The MPEG Software Simulation Group disclaims - * any and all warranties, whether express, implied, or statuary, including any - * implied warranties or merchantability or of fitness for a particular - * purpose. In no event shall the copyright-holder be liable for any - * incidental, punitive, or consequential damages of any kind whatsoever - * arising from the use of these programs. - * - * This disclaimer of warranty extends to the user of these programs and user's - * customers, employees, agents, transferees, successors, and assigns. - * - * The MPEG Software Simulation Group does not represent or warrant that the - * programs furnished hereunder are free of infringement of any third-party - * patents. - * - * Commercial implementations of MPEG-1 and MPEG-2 video, including shareware, - * are subject to royalty fees to patent holders. Many of these patents are - * general enough such that they are unavoidable regardless of implementation - * design. - * - */ - -const blockSize = 64 // A DCT block is 8x8. - -type block [blockSize]int32 - -const ( - w1 = 2841 // 2048*sqrt(2)*cos(1*pi/16) - w2 = 2676 // 2048*sqrt(2)*cos(2*pi/16) - w3 = 2408 // 2048*sqrt(2)*cos(3*pi/16) - w5 = 1609 // 2048*sqrt(2)*cos(5*pi/16) - w6 = 1108 // 2048*sqrt(2)*cos(6*pi/16) - w7 = 565 // 2048*sqrt(2)*cos(7*pi/16) - - w1pw7 = w1 + w7 - w1mw7 = w1 - w7 - w2pw6 = w2 + w6 - w2mw6 = w2 - w6 - w3pw5 = w3 + w5 - w3mw5 = w3 - w5 - - r2 = 181 // 256/sqrt(2) -) - -// idct performs a 2-D Inverse Discrete Cosine Transformation. -// -// The input coefficients should already have been multiplied by the -// appropriate quantization table. We use fixed-point computation, with the -// number of bits for the fractional component varying over the intermediate -// stages. -// -// For more on the actual algorithm, see Z. Wang, "Fast algorithms for the -// discrete W transform and for the discrete Fourier transform", IEEE Trans. on -// ASSP, Vol. ASSP- 32, pp. 803-816, Aug. 1984. -func idct(src *block) { - // Horizontal 1-D IDCT. - for y := 0; y < 8; y++ { - y8 := y * 8 - s := src[y8 : y8+8 : y8+8] // Small cap improves performance, see https://golang.org/issue/27857 - // If all the AC components are zero, then the IDCT is trivial. - if s[1] == 0 && s[2] == 0 && s[3] == 0 && - s[4] == 0 && s[5] == 0 && s[6] == 0 && s[7] == 0 { - dc := s[0] << 3 - s[0] = dc - s[1] = dc - s[2] = dc - s[3] = dc - s[4] = dc - s[5] = dc - s[6] = dc - s[7] = dc - continue - } - - // Prescale. - x0 := (s[0] << 11) + 128 - x1 := s[4] << 11 - x2 := s[6] - x3 := s[2] - x4 := s[1] - x5 := s[7] - x6 := s[5] - x7 := s[3] - - // Stage 1. - x8 := w7 * (x4 + x5) - x4 = x8 + w1mw7*x4 - x5 = x8 - w1pw7*x5 - x8 = w3 * (x6 + x7) - x6 = x8 - w3mw5*x6 - x7 = x8 - w3pw5*x7 - - // Stage 2. - x8 = x0 + x1 - x0 -= x1 - x1 = w6 * (x3 + x2) - x2 = x1 - w2pw6*x2 - x3 = x1 + w2mw6*x3 - x1 = x4 + x6 - x4 -= x6 - x6 = x5 + x7 - x5 -= x7 - - // Stage 3. - x7 = x8 + x3 - x8 -= x3 - x3 = x0 + x2 - x0 -= x2 - x2 = (r2*(x4+x5) + 128) >> 8 - x4 = (r2*(x4-x5) + 128) >> 8 - - // Stage 4. - s[0] = (x7 + x1) >> 8 - s[1] = (x3 + x2) >> 8 - s[2] = (x0 + x4) >> 8 - s[3] = (x8 + x6) >> 8 - s[4] = (x8 - x6) >> 8 - s[5] = (x0 - x4) >> 8 - s[6] = (x3 - x2) >> 8 - s[7] = (x7 - x1) >> 8 - } - - // Vertical 1-D IDCT. - for x := 0; x < 8; x++ { - // Similar to the horizontal 1-D IDCT case, if all the AC components are zero, then the IDCT is trivial. - // However, after performing the horizontal 1-D IDCT, there are typically non-zero AC components, so - // we do not bother to check for the all-zero case. - s := src[x : x+57 : x+57] // Small cap improves performance, see https://golang.org/issue/27857 - - // Prescale. - y0 := (s[8*0] << 8) + 8192 - y1 := s[8*4] << 8 - y2 := s[8*6] - y3 := s[8*2] - y4 := s[8*1] - y5 := s[8*7] - y6 := s[8*5] - y7 := s[8*3] - - // Stage 1. - y8 := w7*(y4+y5) + 4 - y4 = (y8 + w1mw7*y4) >> 3 - y5 = (y8 - w1pw7*y5) >> 3 - y8 = w3*(y6+y7) + 4 - y6 = (y8 - w3mw5*y6) >> 3 - y7 = (y8 - w3pw5*y7) >> 3 - - // Stage 2. - y8 = y0 + y1 - y0 -= y1 - y1 = w6*(y3+y2) + 4 - y2 = (y1 - w2pw6*y2) >> 3 - y3 = (y1 + w2mw6*y3) >> 3 - y1 = y4 + y6 - y4 -= y6 - y6 = y5 + y7 - y5 -= y7 - - // Stage 3. - y7 = y8 + y3 - y8 -= y3 - y3 = y0 + y2 - y0 -= y2 - y2 = (r2*(y4+y5) + 128) >> 8 - y4 = (r2*(y4-y5) + 128) >> 8 - - // Stage 4. - s[8*0] = (y7 + y1) >> 14 - s[8*1] = (y3 + y2) >> 14 - s[8*2] = (y0 + y4) >> 14 - s[8*3] = (y8 + y6) >> 14 - s[8*4] = (y8 - y6) >> 14 - s[8*5] = (y0 - y4) >> 14 - s[8*6] = (y3 - y2) >> 14 - s[8*7] = (y7 - y1) >> 14 - } -} diff --git a/src/image/jpeg/writer.go b/src/image/jpeg/writer.go index 87c109ab772978..0bda55c6b66f76 100644 --- a/src/image/jpeg/writer.go +++ b/src/image/jpeg/writer.go @@ -50,8 +50,8 @@ const ( // unscaledQuant are the unscaled quantization tables in zig-zag order. Each // encoder copies and scales the tables according to its quality parameter. -// The values are derived from section K.1 after converting from natural to -// zig-zag order. +// The values are derived from section K.1 of the spec, after converting from +// natural to zig-zag order. var unscaledQuant = [nQuantIndex][blockSize]byte{ // Luminance. { @@ -89,14 +89,22 @@ const ( // huffmanSpec specifies a Huffman encoding. type huffmanSpec struct { - // count[i] is the number of codes of length i bits. + // count[i] is the number of codes of length i+1 bits. count [16]byte // value[i] is the decoded value of the i'th codeword. value []byte } // theHuffmanSpec is the Huffman encoding specifications. -// This encoder uses the same Huffman encoding for all images. +// +// This encoder uses the same Huffman encoding for all images. It is also the +// same Huffman encoding used by section K.3 of the spec. +// +// The DC tables have 12 decoded values, called categories. +// +// The AC tables have 162 decoded values: bytes that pack a 4-bit Run and a +// 4-bit Size. There are 16 valid Runs and 10 valid Sizes, plus two special R|S +// cases: 0|0 (meaning EOB) and F|0 (meaning ZRL). var theHuffmanSpec = [nHuffIndex]huffmanSpec{ // Luminance DC. { diff --git a/src/image/jpeg/writer_test.go b/src/image/jpeg/writer_test.go index 341477044a3a00..197b49ec136c7f 100644 --- a/src/image/jpeg/writer_test.go +++ b/src/image/jpeg/writer_test.go @@ -154,8 +154,8 @@ func TestWriter(t *testing.T) { continue } // Compare the average delta to the tolerance level. - if averageDelta(m0, m1) > tc.tolerance { - t.Errorf("%s, quality=%d: average delta is too high", tc.filename, tc.quality) + if d := averageDelta(m0, m1); d > tc.tolerance { + t.Errorf("%s, quality=%d: average delta is too high (%d > %d)", tc.filename, tc.quality, d, tc.tolerance) continue } } diff --git a/src/internal/abi/abi.go b/src/internal/abi/abi.go index e1c8adccc71c02..6026701659a201 100644 --- a/src/internal/abi/abi.go +++ b/src/internal/abi/abi.go @@ -98,5 +98,7 @@ func (b *IntArgRegBitmap) Set(i int) { // //go:nosplit func (b *IntArgRegBitmap) Get(i int) bool { - return b[i/8]&(uint8(1)<<(i%8)) != 0 + // Compute p=&b[i/8], but without a bounds check. We don't have the stack for it. + p := (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(b)) + uintptr(i/8))) + return *p&(uint8(1)<<(i%8)) != 0 } diff --git a/src/internal/abi/bounds.go b/src/internal/abi/bounds.go new file mode 100644 index 00000000000000..d6859802d2e075 --- /dev/null +++ b/src/internal/abi/bounds.go @@ -0,0 +1,113 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package abi + +// This type and constants are for encoding different +// kinds of bounds check failures. +type BoundsErrorCode uint8 + +const ( + BoundsIndex BoundsErrorCode = iota // s[x], 0 <= x < len(s) failed + BoundsSliceAlen // s[?:x], 0 <= x <= len(s) failed + BoundsSliceAcap // s[?:x], 0 <= x <= cap(s) failed + BoundsSliceB // s[x:y], 0 <= x <= y failed (but boundsSliceA didn't happen) + BoundsSlice3Alen // s[?:?:x], 0 <= x <= len(s) failed + BoundsSlice3Acap // s[?:?:x], 0 <= x <= cap(s) failed + BoundsSlice3B // s[?:x:y], 0 <= x <= y failed (but boundsSlice3A didn't happen) + BoundsSlice3C // s[x:y:?], 0 <= x <= y failed (but boundsSlice3A/B didn't happen) + BoundsConvert // (*[x]T)(s), 0 <= x <= len(s) failed + numBoundsCodes +) + +const ( + BoundsMaxReg = 15 + BoundsMaxConst = 31 +) + +// Here's how we encode PCDATA_PanicBounds entries: + +// We allow 16 registers (0-15) and 32 constants (0-31). +// Encode the following constant c: +// bits use +// ----------------------------- +// 0 x is in a register +// 1 y is in a register +// +// if x is in a register +// 2 x is signed +// [3:6] x's register number +// else +// [2:6] x's constant value +// +// if y is in a register +// [7:10] y's register number +// else +// [7:11] y's constant value +// +// The final integer is c * numBoundsCode + code + +// TODO: 32-bit + +// Encode bounds failure information into an integer for PCDATA_PanicBounds. +// Register numbers must be in 0-15. Constants must be in 0-31. +func BoundsEncode(code BoundsErrorCode, signed, xIsReg, yIsReg bool, xVal, yVal int) int { + c := int(0) + if xIsReg { + c |= 1 << 0 + if signed { + c |= 1 << 2 + } + if xVal < 0 || xVal > BoundsMaxReg { + panic("bad xReg") + } + c |= xVal << 3 + } else { + if xVal < 0 || xVal > BoundsMaxConst { + panic("bad xConst") + } + c |= xVal << 2 + } + if yIsReg { + c |= 1 << 1 + if yVal < 0 || yVal > BoundsMaxReg { + panic("bad yReg") + } + c |= yVal << 7 + } else { + if yVal < 0 || yVal > BoundsMaxConst { + panic("bad yConst") + } + c |= yVal << 7 + } + return c*int(numBoundsCodes) + int(code) +} +func BoundsDecode(v int) (code BoundsErrorCode, signed, xIsReg, yIsReg bool, xVal, yVal int) { + code = BoundsErrorCode(v % int(numBoundsCodes)) + c := v / int(numBoundsCodes) + xIsReg = c&1 != 0 + c >>= 1 + yIsReg = c&1 != 0 + c >>= 1 + if xIsReg { + signed = c&1 != 0 + c >>= 1 + xVal = c & 0xf + c >>= 4 + } else { + xVal = c & 0x1f + c >>= 5 + } + if yIsReg { + yVal = c & 0xf + c >>= 4 + } else { + yVal = c & 0x1f + c >>= 5 + } + if c != 0 { + panic("BoundsDecode decoding error") + } + return +} diff --git a/src/internal/abi/escape.go b/src/internal/abi/escape.go index 8cdae1438e8578..d37be0177eccd1 100644 --- a/src/internal/abi/escape.go +++ b/src/internal/abi/escape.go @@ -31,3 +31,35 @@ func Escape[T any](x T) T { } return x } + +// EscapeNonString forces v to be on the heap, if v contains a +// non-string pointer. +// +// This is used in hash/maphash.Comparable. We cannot hash pointers +// to local variables on stack, as their addresses might change on +// stack growth. Strings are okay as the hash depends on only the +// content, not the pointer. +// +// This is essentially +// +// if hasNonStringPointers(T) { Escape(v) } +// +// Implemented as a compiler intrinsic. +func EscapeNonString[T any](v T) { panic("intrinsic") } + +// EscapeToResultNonString models a data flow edge from v to the result, +// if v contains a non-string pointer. If v contains only string pointers, +// it returns a copy of v, but is not modeled as a data flow edge +// from the escape analysis's perspective. +// +// This is used in unique.clone, to model the data flow edge on the +// value with strings excluded, because strings are cloned (by +// content). +// +// TODO: probably we should define this as a intrinsic and EscapeNonString +// could just be "heap = EscapeToResultNonString(v)". This way we can model +// an edge to the result but not necessarily heap. +func EscapeToResultNonString[T any](v T) T { + EscapeNonString(v) + return *(*T)(NoEscape(unsafe.Pointer(&v))) +} diff --git a/src/internal/abi/iface.go b/src/internal/abi/iface.go index 676a27d20439af..f53d7e1f4f8864 100644 --- a/src/internal/abi/iface.go +++ b/src/internal/abi/iface.go @@ -25,3 +25,17 @@ type EmptyInterface struct { Type *Type Data unsafe.Pointer } + +// NonEmptyInterface describes the layout of an interface that contains any methods. +type NonEmptyInterface struct { + ITab *ITab + Data unsafe.Pointer +} + +// CommonInterface describes the layout of both [EmptyInterface] and [NonEmptyInterface]. +type CommonInterface struct { + // Either an *ITab or a *Type, unexported to avoid accidental use. + _ unsafe.Pointer + + Data unsafe.Pointer +} diff --git a/src/internal/abi/map.go b/src/internal/abi/map.go index 12ad1b891a78a0..4476dda5cab5b5 100644 --- a/src/internal/abi/map.go +++ b/src/internal/abi/map.go @@ -4,16 +4,61 @@ package abi +import ( + "unsafe" +) + // Map constants common to several packages // runtime/runtime-gdb.py:MapTypePrinter contains its own copy const ( - // Maximum number of key/elem pairs a bucket can hold. - MapBucketCountBits = 3 // log2 of number of elements in a bucket. - MapBucketCount = 1 << MapBucketCountBits + // Number of bits in the group.slot count. + MapGroupSlotsBits = 3 + + // Number of slots in a group. + MapGroupSlots = 1 << MapGroupSlotsBits // 8 // Maximum key or elem size to keep inline (instead of mallocing per element). // Must fit in a uint8. - // Note: fast map functions cannot handle big elems (bigger than MapMaxElemBytes). MapMaxKeyBytes = 128 - MapMaxElemBytes = 128 // Must fit in a uint8. + MapMaxElemBytes = 128 + + ctrlEmpty = 0b10000000 + bitsetLSB = 0x0101010101010101 + + // Value of control word with all empty slots. + MapCtrlEmpty = bitsetLSB * uint64(ctrlEmpty) ) + +type MapType struct { + Type + Key *Type + Elem *Type + Group *Type // internal type representing a slot group + // function for hashing keys (ptr to key, seed) -> hash + Hasher func(unsafe.Pointer, uintptr) uintptr + GroupSize uintptr // == Group.Size_ + SlotSize uintptr // size of key/elem slot + ElemOff uintptr // offset of elem in key/elem slot + Flags uint32 +} + +// Flag values +const ( + MapNeedKeyUpdate = 1 << iota + MapHashMightPanic + MapIndirectKey + MapIndirectElem +) + +func (mt *MapType) NeedKeyUpdate() bool { // true if we need to update key on an overwrite + return mt.Flags&MapNeedKeyUpdate != 0 +} +func (mt *MapType) HashMightPanic() bool { // true if hash function might panic + return mt.Flags&MapHashMightPanic != 0 +} +func (mt *MapType) IndirectKey() bool { // store ptr to key instead of key itself + return mt.Flags&MapIndirectKey != 0 +} +func (mt *MapType) IndirectElem() bool { // store ptr to elem instead of elem itself + return mt.Flags&MapIndirectElem != 0 +} diff --git a/src/internal/abi/switch.go b/src/internal/abi/switch.go index 9669fe51d5aa75..a30fdd078b8ee3 100644 --- a/src/internal/abi/switch.go +++ b/src/internal/abi/switch.go @@ -4,6 +4,8 @@ package abi +import "internal/goarch" + type InterfaceSwitch struct { Cache *InterfaceSwitchCache NCases int @@ -27,16 +29,11 @@ type InterfaceSwitchCacheEntry struct { Itab uintptr } -const go122InterfaceSwitchCache = true - -func UseInterfaceSwitchCache(goarch string) bool { - if !go122InterfaceSwitchCache { - return false - } +func UseInterfaceSwitchCache(arch goarch.ArchFamilyType) bool { // We need an atomic load instruction to make the cache multithreaded-safe. // (AtomicLoadPtr needs to be implemented in cmd/compile/internal/ssa/_gen/ARCH.rules.) - switch goarch { - case "amd64", "arm64", "loong64", "mips", "mipsle", "mips64", "mips64le", "ppc64", "ppc64le", "riscv64", "s390x": + switch arch { + case goarch.AMD64, goarch.ARM64, goarch.LOONG64, goarch.MIPS, goarch.MIPS64, goarch.PPC64, goarch.RISCV64, goarch.S390X: return true default: return false diff --git a/src/internal/abi/symtab.go b/src/internal/abi/symtab.go index a3c9be7aa1ee54..86d67003886fed 100644 --- a/src/internal/abi/symtab.go +++ b/src/internal/abi/symtab.go @@ -56,8 +56,9 @@ const ( FuncID_mstart FuncID_panicwrap FuncID_rt0_go - FuncID_runfinq FuncID_runtime_main + FuncID_runFinalizers + FuncID_runCleanups FuncID_sigpanic FuncID_systemstack FuncID_systemstack_switch @@ -78,6 +79,7 @@ const ( PCDATA_StackMapIndex = 1 PCDATA_InlTreeIndex = 2 PCDATA_ArgLiveIndex = 3 + PCDATA_PanicBounds = 4 FUNCDATA_ArgsPointerMaps = 0 FUNCDATA_LocalsPointerMaps = 1 diff --git a/src/internal/abi/type.go b/src/internal/abi/type.go index 786bafff723c96..1920a8a37fb255 100644 --- a/src/internal/abi/type.go +++ b/src/internal/abi/type.go @@ -24,13 +24,21 @@ type Type struct { TFlag TFlag // extra type information flags Align_ uint8 // alignment of variable with this type FieldAlign_ uint8 // alignment of struct field with this type - Kind_ Kind // enumeration for C + Kind_ Kind // what kind of type this is (string, int, ...) // function for comparing objects of this type // (ptr to object A, ptr to object B) -> ==? Equal func(unsafe.Pointer, unsafe.Pointer) bool // GCData stores the GC type data for the garbage collector. - // If the KindGCProg bit is set in kind, GCData is a GC program. - // Otherwise it is a ptrmask bitmap. See mbitmap.go for details. + // Normally, GCData points to a bitmask that describes the + // ptr/nonptr fields of the type. The bitmask will have at + // least PtrBytes/ptrSize bits. + // If the TFlagGCMaskOnDemand bit is set, GCData is instead a + // **byte and the pointer to the bitmask is one dereference away. + // The runtime will build the bitmask if needed. + // (See runtime/type.go:getGCMask.) + // Note: multiple types may have the same value of GCData, + // including when TFlagGCMaskOnDemand is set. The types will, of course, + // have the same pointer layout (but not necessarily the same size). GCData *byte Str NameOff // string form PtrToThis TypeOff // type for pointer to this type, may be zero @@ -70,13 +78,6 @@ const ( UnsafePointer ) -const ( - // TODO (khr, drchase) why aren't these in TFlag? Investigate, fix if possible. - KindDirectIface Kind = 1 << 5 - KindGCProg Kind = 1 << 6 // Type.gc points to GC program - KindMask Kind = (1 << 5) - 1 -) - // TFlag is used by a Type to signal what extra type information is // available in the memory directly following the Type value. type TFlag uint8 @@ -112,11 +113,21 @@ const ( // this type as a single region of t.size bytes. TFlagRegularMemory TFlag = 1 << 3 - // TFlagUnrolledBitmap marks special types that are unrolled-bitmap - // versions of types with GC programs. - // These types need to be deallocated when the underlying object - // is freed. - TFlagUnrolledBitmap TFlag = 1 << 4 + // TFlagGCMaskOnDemand means that the GC pointer bitmask will be + // computed on demand at runtime instead of being precomputed at + // compile time. If this flag is set, the GCData field effectively + // has type **byte instead of *byte. The runtime will store a + // pointer to the GC pointer bitmask in *GCData. + TFlagGCMaskOnDemand TFlag = 1 << 4 + + // TFlagDirectIface means that a value of this type is stored directly + // in the data field of an interface, instead of indirectly. Normally + // this means the type is pointer-ish. + TFlagDirectIface TFlag = 1 << 5 + + // Leaving this breadcrumb behind for dlv. It should not be used, and no + // Kind should be big enough to set this bit. + KindDirectIface Kind = 1 << 5 ) // NameOff is the offset to a name from moduledata.types. See resolveNameOff in runtime. @@ -177,7 +188,12 @@ func TypeOf(a any) *Type { return (*Type)(NoEscape(unsafe.Pointer(eface.Type))) } -func (t *Type) Kind() Kind { return t.Kind_ & KindMask } +// TypeFor returns the abi.Type for a type parameter. +func TypeFor[T any]() *Type { + return (*PtrType)(unsafe.Pointer(TypeOf((*T)(nil)))).Elem +} + +func (t *Type) Kind() Kind { return t.Kind_ } func (t *Type) HasName() bool { return t.TFlag&TFlagNamed != 0 @@ -186,17 +202,15 @@ func (t *Type) HasName() bool { // Pointers reports whether t contains pointers. func (t *Type) Pointers() bool { return t.PtrBytes != 0 } -// IfaceIndir reports whether t is stored indirectly in an interface value. -func (t *Type) IfaceIndir() bool { - return t.Kind_&KindDirectIface == 0 -} - -// isDirectIface reports whether t is stored directly in an interface value. +// IsDirectIface reports whether t is stored directly in an interface value. func (t *Type) IsDirectIface() bool { - return t.Kind_&KindDirectIface != 0 + return t.TFlag&TFlagDirectIface != 0 } func (t *Type) GcSlice(begin, end uintptr) []byte { + if t.TFlag&TFlagGCMaskOnDemand != 0 { + panic("GcSlice can't handle on-demand gcdata types") + } return unsafe.Slice(t.GCData, int(end))[begin:] } @@ -455,37 +469,6 @@ func (t *Type) NumMethod() int { // NumMethod returns the number of interface methods in the type's method set. func (t *InterfaceType) NumMethod() int { return len(t.Methods) } -type MapType struct { - Type - Key *Type - Elem *Type - Bucket *Type // internal type representing a hash bucket - // function for hashing keys (ptr to key, seed) -> hash - Hasher func(unsafe.Pointer, uintptr) uintptr - KeySize uint8 // size of key slot - ValueSize uint8 // size of elem slot - BucketSize uint16 // size of bucket - Flags uint32 -} - -// Note: flag values must match those used in the TMAP case -// in ../cmd/compile/internal/reflectdata/reflect.go:writeType. -func (mt *MapType) IndirectKey() bool { // store ptr to key instead of key itself - return mt.Flags&1 != 0 -} -func (mt *MapType) IndirectElem() bool { // store ptr to elem instead of elem itself - return mt.Flags&2 != 0 -} -func (mt *MapType) ReflexiveKey() bool { // true if k==k for all keys - return mt.Flags&4 != 0 -} -func (mt *MapType) NeedKeyUpdate() bool { // true if we need to update key on an overwrite - return mt.Flags&8 != 0 -} -func (mt *MapType) HashMightPanic() bool { // true if hash function might panic - return mt.Flags&16 != 0 -} - func (t *Type) Key() *Type { if t.Kind() == Map { return (*MapType)(unsafe.Pointer(t)).Key @@ -498,7 +481,7 @@ type SliceType struct { Elem *Type // slice element type } -// funcType represents a function type. +// FuncType represents a function type. // // A *Type for each in and out parameter is stored in an array that // directly follows the funcType (and possibly its uncommonType). So diff --git a/src/internal/buildcfg/cfg.go b/src/internal/buildcfg/cfg.go index a16e76b3055845..a75960b8e6c034 100644 --- a/src/internal/buildcfg/cfg.go +++ b/src/internal/buildcfg/cfg.go @@ -23,7 +23,7 @@ var ( GOROOT = os.Getenv("GOROOT") // cached for efficiency GOARCH = envOr("GOARCH", defaultGOARCH) GOOS = envOr("GOOS", defaultGOOS) - GO386 = envOr("GO386", defaultGO386) + GO386 = envOr("GO386", DefaultGO386) GOAMD64 = goamd64() GOARM = goarm() GOARM64 = goarm64() @@ -34,6 +34,7 @@ var ( GOWASM = gowasm() ToolTags = toolTags() GO_LDSO = defaultGO_LDSO + GOFIPS140 = gofips140() Version = version ) @@ -56,7 +57,7 @@ func envOr(key, value string) string { } func goamd64() int { - switch v := envOr("GOAMD64", defaultGOAMD64); v { + switch v := envOr("GOAMD64", DefaultGOAMD64); v { case "v1": return 1 case "v2": @@ -67,15 +68,57 @@ func goamd64() int { return 4 } Error = fmt.Errorf("invalid GOAMD64: must be v1, v2, v3, v4") - return int(defaultGOAMD64[len("v")] - '0') + return int(DefaultGOAMD64[len("v")] - '0') } -type goarmFeatures struct { +func gofips140() string { + v := envOr("GOFIPS140", DefaultGOFIPS140) + switch v { + case "off", "latest", "inprocess", "certified": + return v + } + if isFIPSVersion(v) { + return v + } + Error = fmt.Errorf("invalid GOFIPS140: must be off, latest, inprocess, certified, or vX.Y.Z") + return DefaultGOFIPS140 +} + +// isFIPSVersion reports whether v is a valid FIPS version, +// of the form vX.Y.Z or vX.Y.Z-hash. +func isFIPSVersion(v string) bool { + if !strings.HasPrefix(v, "v") { + return false + } + v, ok := skipNum(v[len("v"):]) + if !ok || !strings.HasPrefix(v, ".") { + return false + } + v, ok = skipNum(v[len("."):]) + if !ok || !strings.HasPrefix(v, ".") { + return false + } + v, ok = skipNum(v[len("."):]) + hasHash := strings.HasPrefix(v, "-") && len(v) == len("-")+8 + return ok && (v == "" || hasHash) +} + +// skipNum skips the leading text matching [0-9]+ +// in s, returning the rest and whether such text was found. +func skipNum(s string) (rest string, ok bool) { + i := 0 + for i < len(s) && '0' <= s[i] && s[i] <= '9' { + i++ + } + return s[i:], i > 0 +} + +type GoarmFeatures struct { Version int SoftFloat bool } -func (g goarmFeatures) String() string { +func (g GoarmFeatures) String() string { armStr := strconv.Itoa(g.Version) if g.SoftFloat { armStr += ",softfloat" @@ -85,12 +128,12 @@ func (g goarmFeatures) String() string { return armStr } -func goarm() (g goarmFeatures) { +func goarm() (g GoarmFeatures) { const ( softFloatOpt = ",softfloat" hardFloatOpt = ",hardfloat" ) - def := defaultGOARM + def := DefaultGOARM if GOOS == "android" && GOARCH == "arm" { // Android arm devices always support GOARM=7. def = "7" @@ -186,14 +229,14 @@ func ParseGoarm64(v string) (g Goarm64Features, e error) { default: e = fmt.Errorf("invalid GOARM64: must start with v8.{0-9} or v9.{0-5} and may optionally end in %q and/or %q", lseOpt, cryptoOpt) - g.Version = defaultGOARM64 + g.Version = DefaultGOARM64 } return } func goarm64() (g Goarm64Features) { - g, Error = ParseGoarm64(envOr("GOARM64", defaultGOARM64)) + g, Error = ParseGoarm64(envOr("GOARM64", DefaultGOARM64)) return } @@ -229,25 +272,25 @@ func (g Goarm64Features) Supports(s string) bool { } func gomips() string { - switch v := envOr("GOMIPS", defaultGOMIPS); v { + switch v := envOr("GOMIPS", DefaultGOMIPS); v { case "hardfloat", "softfloat": return v } Error = fmt.Errorf("invalid GOMIPS: must be hardfloat, softfloat") - return defaultGOMIPS + return DefaultGOMIPS } func gomips64() string { - switch v := envOr("GOMIPS64", defaultGOMIPS64); v { + switch v := envOr("GOMIPS64", DefaultGOMIPS64); v { case "hardfloat", "softfloat": return v } Error = fmt.Errorf("invalid GOMIPS64: must be hardfloat, softfloat") - return defaultGOMIPS64 + return DefaultGOMIPS64 } func goppc64() int { - switch v := envOr("GOPPC64", defaultGOPPC64); v { + switch v := envOr("GOPPC64", DefaultGOPPC64); v { case "power8": return 8 case "power9": @@ -256,18 +299,20 @@ func goppc64() int { return 10 } Error = fmt.Errorf("invalid GOPPC64: must be power8, power9, power10") - return int(defaultGOPPC64[len("power")] - '0') + return int(DefaultGOPPC64[len("power")] - '0') } func goriscv64() int { - switch v := envOr("GORISCV64", defaultGORISCV64); v { + switch v := envOr("GORISCV64", DefaultGORISCV64); v { case "rva20u64": return 20 case "rva22u64": return 22 + case "rva23u64": + return 23 } - Error = fmt.Errorf("invalid GORISCV64: must be rva20u64, rva22u64") - v := defaultGORISCV64[len("rva"):] + Error = fmt.Errorf("invalid GORISCV64: must be rva20u64, rva22u64, rva23u64") + v := DefaultGORISCV64[len("rva"):] i := strings.IndexFunc(v, func(r rune) bool { return r < '0' || r > '9' }) @@ -276,28 +321,23 @@ func goriscv64() int { } type gowasmFeatures struct { - SatConv bool - SignExt bool + // Legacy features, now always enabled + //SatConv bool + //SignExt bool } func (f gowasmFeatures) String() string { var flags []string - if f.SatConv { - flags = append(flags, "satconv") - } - if f.SignExt { - flags = append(flags, "signext") - } return strings.Join(flags, ",") } func gowasm() (f gowasmFeatures) { - for _, opt := range strings.Split(envOr("GOWASM", ""), ",") { + for opt := range strings.SplitSeq(envOr("GOWASM", ""), ",") { switch opt { case "satconv": - f.SatConv = true + // ignore, always enabled case "signext": - f.SignExt = true + // ignore, always enabled case "": // ignore default: @@ -348,6 +388,8 @@ func GOGOARCH() (name, value string) { return "GOMIPS64", GOMIPS64 case "ppc64", "ppc64le": return "GOPPC64", fmt.Sprintf("power%d", GOPPC64) + case "riscv64": + return "GORISCV64", fmt.Sprintf("rva%du64", GORISCV64) case "wasm": return "GOWASM", GOWASM.String() } @@ -399,15 +441,16 @@ func gogoarchTags() []string { if GORISCV64 >= 22 { list = append(list, GOARCH+"."+"rva22u64") } + if GORISCV64 >= 23 { + list = append(list, GOARCH+"."+"rva23u64") + } return list case "wasm": var list []string - if GOWASM.SatConv { - list = append(list, GOARCH+".satconv") - } - if GOWASM.SignExt { - list = append(list, GOARCH+".signext") - } + // SatConv is always enabled + list = append(list, GOARCH+".satconv") + // SignExt is always enabled + list = append(list, GOARCH+".signext") return list } return nil diff --git a/src/internal/buildcfg/cfg_test.go b/src/internal/buildcfg/cfg_test.go index d01cdd010961b4..2bbd478280241e 100644 --- a/src/internal/buildcfg/cfg_test.go +++ b/src/internal/buildcfg/cfg_test.go @@ -32,6 +32,10 @@ func TestConfigFlags(t *testing.T) { if goriscv64() != 22 { t.Errorf("Wrong parsing of RISCV64=rva22u64") } + os.Setenv("GORISCV64", "rva23u64") + if goriscv64() != 23 { + t.Errorf("Wrong parsing of RISCV64=rva23u64") + } Error = nil os.Setenv("GORISCV64", "rva22") if _ = goriscv64(); Error == nil { @@ -123,3 +127,39 @@ func TestGogoarchTags(t *testing.T) { GOARCH = old_goarch GOARM64 = old_goarm64 } + +var goodFIPS = []string{ + "v1.0.0", + "v1.0.1", + "v1.2.0", + "v1.2.3", +} + +var badFIPS = []string{ + "v1.0.0-fips", + "v1.0.0+fips", + "1.0.0", + "x1.0.0", +} + +func TestIsFIPSVersion(t *testing.T) { + // good + for _, s := range goodFIPS { + if !isFIPSVersion(s) { + t.Errorf("isFIPSVersion(%q) = false, want true", s) + } + } + // truncated + const v = "v1.2.3" + for i := 0; i < len(v); i++ { + if isFIPSVersion(v[:i]) { + t.Errorf("isFIPSVersion(%q) = true, want false", v[:i]) + } + } + // bad + for _, s := range badFIPS { + if isFIPSVersion(s) { + t.Errorf("isFIPSVersion(%q) = true, want false", s) + } + } +} diff --git a/src/internal/buildcfg/exp.go b/src/internal/buildcfg/exp.go index 7c7cefba7b2a6f..31195f94c08c86 100644 --- a/src/internal/buildcfg/exp.go +++ b/src/internal/buildcfg/exp.go @@ -25,7 +25,7 @@ type ExperimentFlags struct { // (This is not necessarily the set of experiments the compiler itself // was built with.) // -// experimentBaseline specifies the experiment flags that are enabled by +// Experiment.baseline specifies the experiment flags that are enabled by // default in the current toolchain. This is, in effect, the "control" // configuration and any variation from this is an experiment. var Experiment ExperimentFlags = func() ExperimentFlags { @@ -54,7 +54,7 @@ var FramePointerEnabled = GOARCH == "amd64" || GOARCH == "arm64" // configuration tuple and returns the enabled and baseline experiment // flag sets. // -// TODO(mdempsky): Move to internal/goexperiment. +// TODO(mdempsky): Move to [internal/goexperiment]. func ParseGOEXPERIMENT(goos, goarch, goexp string) (*ExperimentFlags, error) { // regabiSupported is set to true on platforms where register ABI is // supported and enabled by default. @@ -67,10 +67,25 @@ func ParseGOEXPERIMENT(goos, goarch, goexp string) (*ExperimentFlags, error) { regabiSupported = true } + // Older versions (anything before V16) of dsymutil don't handle + // the .debug_rnglists section in DWARF5. See + // https://github.com/golang/go/issues/26379#issuecomment-2677068742 + // for more context. This disables all DWARF5 on mac, which is not + // ideal (would be better to disable just for cases where we know + // the build will use external linking). In the GOOS=aix case, the + // XCOFF format (as far as can be determined) doesn't seem to + // support the necessary section subtypes for DWARF-specific + // things like .debug_addr (needed for DWARF 5). + dwarf5Supported := (goos != "darwin" && goos != "ios" && goos != "aix") + baseline := goexperiment.Flags{ - RegabiWrappers: regabiSupported, - RegabiArgs: regabiSupported, - CoverageRedesign: true, + RegabiWrappers: regabiSupported, + RegabiArgs: regabiSupported, + Dwarf5: dwarf5Supported, + RandomizedHeapBase64: true, + RuntimeFree: true, + SizeSpecializedMalloc: true, + GreenTeaGC: true, } // Start with the statically enabled set of experiments. @@ -102,7 +117,7 @@ func ParseGOEXPERIMENT(goos, goarch, goexp string) (*ExperimentFlags, error) { } // Parse names. - for _, f := range strings.Split(goexp, ",") { + for f := range strings.SplitSeq(goexp, ",") { if f == "" { continue } diff --git a/src/internal/bytealg/bytealg.go b/src/internal/bytealg/bytealg.go index 6b79a2e1fabb90..319ea54ba3c77f 100644 --- a/src/internal/bytealg/bytealg.go +++ b/src/internal/bytealg/bytealg.go @@ -11,13 +11,18 @@ import ( // Offsets into internal/cpu records for use in assembly. const ( - offsetX86HasSSE42 = unsafe.Offsetof(cpu.X86.HasSSE42) - offsetX86HasAVX2 = unsafe.Offsetof(cpu.X86.HasAVX2) - offsetX86HasPOPCNT = unsafe.Offsetof(cpu.X86.HasPOPCNT) + offsetPPC64HasPOWER9 = unsafe.Offsetof(cpu.PPC64.IsPOWER9) + + offsetRISCV64HasV = unsafe.Offsetof(cpu.RISCV64.HasV) + + offsetLOONG64HasLSX = unsafe.Offsetof(cpu.Loong64.HasLSX) + offsetLOONG64HasLASX = unsafe.Offsetof(cpu.Loong64.HasLASX) offsetS390xHasVX = unsafe.Offsetof(cpu.S390X.HasVX) - offsetPPC64HasPOWER9 = unsafe.Offsetof(cpu.PPC64.IsPOWER9) + offsetX86HasSSE42 = unsafe.Offsetof(cpu.X86.HasSSE42) + offsetX86HasAVX2 = unsafe.Offsetof(cpu.X86.HasAVX2) + offsetX86HasPOPCNT = unsafe.Offsetof(cpu.X86.HasPOPCNT) ) // MaxLen is the maximum length of the string to be searched for (argument b) in Index. diff --git a/src/internal/bytealg/compare_loong64.s b/src/internal/bytealg/compare_loong64.s index df72a1122bc714..933053196449d5 100644 --- a/src/internal/bytealg/compare_loong64.s +++ b/src/internal/bytealg/compare_loong64.s @@ -23,66 +23,341 @@ TEXT runtime·cmpstring(SB),NOSPLIT,$0-40 // R7 = b_len JMP cmpbody<>(SB) -// On entry: -// R5 length of a -// R7 length of b -// R4 points to the start of a -// R6 points to the start of b -// R13 points to the return value (-1/0/1) +// input: +// R4: points to the start of a +// R5: length of a +// R6: points to the start of b +// R7: length of b +// for regabi the return value (-1/0/1) in R4 TEXT cmpbody<>(SB),NOSPLIT|NOFRAME,$0 - BEQ R4, R6, samebytes // same start of a and b + BEQ R4, R6, cmp_len // same start of a and b, then compare lengths SGTU R5, R7, R9 - BNE R0, R9, r2_lt_r1 + BNE R9, b_lt_a MOVV R5, R14 JMP entry -r2_lt_r1: - MOVV R7, R14 // R14 is min(R4, R5) + +b_lt_a: + MOVV R7, R14 + entry: - ADDV R4, R14, R12 // R6 start of a, R14 end of a - BEQ R4, R12, samebytes // length is 0 - - SRLV $4, R14 // R14 is number of chunks - BEQ R0, R14, byte_loop - - // make sure both a and b are aligned. - OR R4, R6, R15 - AND $7, R15 - BNE R0, R15, byte_loop - - PCALIGN $16 -chunk16_loop: - BEQ R0, R14, byte_loop - MOVV (R4), R8 - MOVV (R6), R9 - BNE R8, R9, byte_loop - MOVV 8(R4), R16 - MOVV 8(R6), R17 - ADDV $16, R4 - ADDV $16, R6 - SUBVU $1, R14 - BEQ R16, R17, chunk16_loop - SUBV $8, R4 - SUBV $8, R6 - -byte_loop: - BEQ R4, R12, samebytes - MOVBU (R4), R8 - ADDVU $1, R4 - MOVBU (R6), R9 - ADDVU $1, R6 - BEQ R8, R9, byte_loop - -byte_cmp: - SGTU R8, R9, R4 // R12 = 1 if (R8 > R9) + BEQ R14, cmp_len // minlength is 0 + + MOVV $32, R15 + BGE R14, R15, lasx +tail: + MOVV $8, R15 + BLT R14, R15, lt_8 +generic8_loop: + MOVV (R4), R10 + MOVV (R6), R11 + BEQ R10, R11, generic8_equal + +cmp8: + AND $0xff, R10, R16 + AND $0xff, R11, R17 + BNE R16, R17, cmp_byte + + BSTRPICKV $15, R10, $8, R16 + BSTRPICKV $15, R11, $8, R17 + BNE R16, R17, cmp_byte + + BSTRPICKV $23, R10, $16, R16 + BSTRPICKV $23, R11, $16, R17 + BNE R16, R17, cmp_byte + + BSTRPICKV $31, R10, $24, R16 + BSTRPICKV $31, R11, $24, R17 + BNE R16, R17, cmp_byte + + BSTRPICKV $39, R10, $32, R16 + BSTRPICKV $39, R11, $32, R17 + BNE R16, R17, cmp_byte + + BSTRPICKV $47, R10, $40, R16 + BSTRPICKV $47, R11, $40, R17 + BNE R16, R17, cmp_byte + + BSTRPICKV $55, R10, $48, R16 + BSTRPICKV $55, R11, $48, R17 + BNE R16, R17, cmp_byte + + BSTRPICKV $63, R10, $56, R16 + BSTRPICKV $63, R11, $56, R17 + BNE R16, R17, cmp_byte + +generic8_equal: + ADDV $-8, R14 + BEQ R14, cmp_len + ADDV $8, R4 + ADDV $8, R6 + BGE R14, R15, generic8_loop + +lt_8: + MOVV $4, R15 + BLT R14, R15, lt_4 + + MOVWU (R4), R10 + MOVWU (R6), R11 + BEQ R10, R11, lt_8_equal + + AND $0xff, R10, R16 + AND $0xff, R11, R17 + BNE R16, R17, cmp_byte + + BSTRPICKV $15, R10, $8, R16 + BSTRPICKV $15, R11, $8, R17 + BNE R16, R17, cmp_byte + + BSTRPICKV $23, R10, $16, R16 + BSTRPICKV $23, R11, $16, R17 + BNE R16, R17, cmp_byte + + BSTRPICKV $31, R10, $24, R16 + BSTRPICKV $31, R11, $24, R17 + BNE R16, R17, cmp_byte + +lt_8_equal: + ADDV $-4, R14 + BEQ R14, cmp_len + ADDV $4, R4 + ADDV $4, R6 + +lt_4: + MOVV $2, R15 + BLT R14, R15, lt_2 + + MOVHU (R4), R10 + MOVHU (R6), R11 + BEQ R10, R11, lt_4_equal + + AND $0xff, R10, R16 + AND $0xff, R11, R17 + BNE R16, R17, cmp_byte + + BSTRPICKV $15, R10, $8, R16 + BSTRPICKV $15, R11, $8, R17 + BNE R16, R17, cmp_byte + +lt_4_equal: + ADDV $-2, R14 + BEQ R14, cmp_len + ADDV $2, R4 + ADDV $2, R6 + +lt_2: + MOVBU (R4), R16 + MOVBU (R6), R17 + BNE R16, R17, cmp_byte + JMP cmp_len + + // Compare 1 byte taken from R16/R17 that are known to differ. +cmp_byte: + SGTU R16, R17, R4 // R4 = 1 if (R16 > R17) BNE R0, R4, ret MOVV $-1, R4 - JMP ret + RET -samebytes: +cmp_len: SGTU R5, R7, R8 SGTU R7, R5, R9 SUBV R9, R8, R4 ret: RET + +lasx: + MOVV $64, R20 + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R9 + BEQ R9, lsx + + MOVV $128, R15 + BLT R14, R15, lasx32_loop +lasx128_loop: + XVMOVQ (R4), X0 + XVMOVQ (R6), X1 + XVSEQB X0, X1, X0 + XVSETANYEQB X0, FCC0 + BFPT lasx_found_0 + + XVMOVQ 32(R4), X0 + XVMOVQ 32(R6), X1 + XVSEQB X0, X1, X0 + XVSETANYEQB X0, FCC0 + BFPT lasx_found_32 + + XVMOVQ 64(R4), X0 + XVMOVQ 64(R6), X1 + XVSEQB X0, X1, X0 + XVSETANYEQB X0, FCC0 + BFPT lasx_found_64 + + XVMOVQ 96(R4), X0 + XVMOVQ 96(R6), X1 + XVSEQB X0, X1, X0 + XVSETANYEQB X0, FCC0 + BFPT lasx_found_96 + + ADDV $-128, R14 + BEQ R14, cmp_len + ADDV $128, R4 + ADDV $128, R6 + BGE R14, R15, lasx128_loop + + MOVV $32, R15 + BLT R14, R15, tail +lasx32_loop: + XVMOVQ (R4), X0 + XVMOVQ (R6), X1 + XVSEQB X0, X1, X0 + XVSETANYEQB X0, FCC0 + BFPT lasx_found_0 + + ADDV $-32, R14 + BEQ R14, cmp_len + ADDV $32, R4 + ADDV $32, R6 + BGE R14, R15, lasx32_loop + JMP tail + +lasx_found_0: + MOVV R0, R11 + JMP lasx_find_byte + +lasx_found_32: + MOVV $32, R11 + JMP lasx_find_byte + +lasx_found_64: + MOVV $64, R11 + JMP lasx_find_byte + +lasx_found_96: + MOVV $96, R11 + +lasx_find_byte: + XVMOVQ X0.V[0], R10 + CTOV R10, R10 + BNE R10, R20, find_byte + ADDV $8, R11 + + XVMOVQ X0.V[1], R10 + CTOV R10, R10 + BNE R10, R20, find_byte + ADDV $8, R11 + + XVMOVQ X0.V[2], R10 + CTOV R10, R10 + BNE R10, R20, find_byte + ADDV $8, R11 + + XVMOVQ X0.V[3], R10 + CTOV R10, R10 + JMP find_byte + +lsx: + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R9 + BEQ R9, generic32_loop + + MOVV $64, R15 + BLT R14, R15, lsx16_loop +lsx64_loop: + VMOVQ (R4), V0 + VMOVQ (R6), V1 + VSEQB V0, V1, V0 + VSETANYEQB V0, FCC0 + BFPT lsx_found_0 + + VMOVQ 16(R4), V0 + VMOVQ 16(R6), V1 + VSEQB V0, V1, V0 + VSETANYEQB V0, FCC0 + BFPT lsx_found_16 + + VMOVQ 32(R4), V0 + VMOVQ 32(R6), V1 + VSEQB V0, V1, V0 + VSETANYEQB V0, FCC0 + BFPT lsx_found_32 + + VMOVQ 48(R4), V0 + VMOVQ 48(R6), V1 + VSEQB V0, V1, V0 + VSETANYEQB V0, FCC0 + BFPT lsx_found_48 + + ADDV $-64, R14 + BEQ R14, cmp_len + ADDV $64, R4 + ADDV $64, R6 + BGE R14, R15, lsx64_loop + + MOVV $16, R15 + BLT R14, R15, tail +lsx16_loop: + VMOVQ (R4), V0 + VMOVQ (R6), V1 + VSEQB V0, V1, V0 + VSETANYEQB V0, FCC0 + BFPT lsx_found_0 + + ADDV $-16, R14 + BEQ R14, cmp_len + ADDV $16, R4 + ADDV $16, R6 + BGE R14, R15, lsx16_loop + JMP tail + +lsx_found_0: + MOVV R0, R11 + JMP lsx_find_byte + +lsx_found_16: + MOVV $16, R11 + JMP lsx_find_byte + +lsx_found_32: + MOVV $32, R11 + JMP lsx_find_byte + +lsx_found_48: + MOVV $48, R11 + +lsx_find_byte: + VMOVQ V0.V[0], R10 + CTOV R10, R10 + BNE R10, R20, find_byte + ADDV $8, R11 + + VMOVQ V0.V[1], R10 + CTOV R10, R10 + +find_byte: + SRLV $3, R10 + ADDV R10, R11 + ADDV R11, R4 + ADDV R11, R6 + MOVB (R4), R16 + MOVB (R6), R17 + JMP cmp_byte + +generic32_loop: + MOVV (R4), R10 + MOVV (R6), R11 + BNE R10, R11, cmp8 + MOVV 8(R4), R10 + MOVV 8(R6), R11 + BNE R10, R11, cmp8 + MOVV 16(R4), R10 + MOVV 16(R6), R11 + BNE R10, R11, cmp8 + MOVV 24(R4), R10 + MOVV 24(R6), R11 + BNE R10, R11, cmp8 + ADDV $-32, R14 + BEQ R14, cmp_len + ADDV $32, R4 + ADDV $32, R6 + MOVV $32, R15 + BGE R14, R15, generic32_loop + JMP tail diff --git a/src/internal/bytealg/compare_ppc64x.s b/src/internal/bytealg/compare_ppc64x.s index 2629251e430a8e..a3d56cfd6999fa 100644 --- a/src/internal/bytealg/compare_ppc64x.s +++ b/src/internal/bytealg/compare_ppc64x.s @@ -61,7 +61,7 @@ TEXT ·Compare(SB),NOSPLIT|NOFRAME,$0-56 CMP R3,R6,CR7 ISEL CR0LT,R4,R7,R9 SETB_CR0(R3) - BC $12,30,LR // beqlr cr7 + BEQ CR7,LR BR cmpbody<>(SB) TEXT runtime·cmpstring(SB),NOSPLIT|NOFRAME,$0-40 @@ -83,7 +83,7 @@ TEXT runtime·cmpstring(SB),NOSPLIT|NOFRAME,$0-40 MOVD R5,R6 MOVD R3,R5 SETB_CR0(R3) - BC $12,30,LR // beqlr cr7 + BEQ CR7,LR BR cmpbody<>(SB) #ifdef GOARCH_ppc64le @@ -143,7 +143,7 @@ cmp64_loop: ADD $64,R5,R5 // increment to next 64 bytes of A ADD $64,R6,R6 // increment to next 64 bytes of B BDNZ cmp64_loop - BC $12,2,LR // beqlr + BEQ CR0,LR // beqlr // Finish out tail with minimal overlapped checking. // Note, 0 tail is handled by beqlr above. @@ -215,7 +215,7 @@ cmp32: // 32 - 63B VCMPEQUDCC V3,V4,V1 BGE CR6,different - BC $12,2,LR // beqlr + BEQ CR0,LR ADD R9,R10,R10 LXVD2X (R9)(R5),V3 @@ -236,7 +236,7 @@ cmp16: // 16 - 31B LXVD2X (R0)(R6),V4 VCMPEQUDCC V3,V4,V1 BGE CR6,different - BC $12,2,LR // beqlr + BEQ CR0,LR LXVD2X (R9)(R5),V3 LXVD2X (R9)(R6),V4 diff --git a/src/internal/bytealg/compare_riscv64.s b/src/internal/bytealg/compare_riscv64.s index b1e1f7bcc76c5b..3b1523dfbf7f3b 100644 --- a/src/internal/bytealg/compare_riscv64.s +++ b/src/internal/bytealg/compare_riscv64.s @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include "asm_riscv64.h" #include "go_asm.h" #include "textflag.h" @@ -28,17 +29,53 @@ TEXT runtime·cmpstring(SB),NOSPLIT|NOFRAME,$0-40 // X11 length of a // X12 points to start of b // X13 length of b -// for non-regabi X14 points to the address to store the return value (-1/0/1) -// for regabi the return value in X10 +// return value in X10 (-1/0/1) TEXT compare<>(SB),NOSPLIT|NOFRAME,$0 BEQ X10, X12, cmp_len - MOV X11, X5 - BGE X13, X5, use_a_len // X5 = min(len(a), len(b)) - MOV X13, X5 -use_a_len: + MIN X11, X13, X5 BEQZ X5, cmp_len + MOV $16, X6 + BLT X5, X6, check8_unaligned + +#ifndef hasV + MOVB internal∕cpu·RISCV64+const_offsetRISCV64HasV(SB), X6 + BEQZ X6, compare_scalar +#endif + + // Use vector if not 8 byte aligned. + OR X10, X12, X6 + AND $7, X6 + BNEZ X6, vector_loop + + // Use scalar if 8 byte aligned and <= 128 bytes. + SUB $128, X5, X6 + BLEZ X6, compare_scalar_aligned + + PCALIGN $16 +vector_loop: + VSETVLI X5, E8, M8, TA, MA, X6 + VLE8V (X10), V8 + VLE8V (X12), V16 + VMSNEVV V8, V16, V0 + VFIRSTM V0, X7 + BGEZ X7, vector_not_eq + ADD X6, X10 + ADD X6, X12 + SUB X6, X5 + BNEZ X5, vector_loop + JMP cmp_len + +vector_not_eq: + // Load first differing bytes in X8/X9. + ADD X7, X10 + ADD X7, X12 + MOVBU (X10), X8 + MOVBU (X12), X9 + JMP cmp + +compare_scalar: MOV $32, X6 BLT X5, X6, check8_unaligned @@ -61,9 +98,9 @@ align: ADD $1, X12 BNEZ X7, align -check32: - // X6 contains $32 - BLT X5, X6, compare16 +compare_scalar_aligned: + MOV $32, X6 + BLT X5, X6, check16 compare32: MOV 0(X10), X15 MOV 0(X12), X16 diff --git a/src/internal/bytealg/count_arm64.s b/src/internal/bytealg/count_arm64.s index e616627b1a210d..1e39cd5f3dab2f 100644 --- a/src/internal/bytealg/count_arm64.s +++ b/src/internal/bytealg/count_arm64.s @@ -5,33 +5,45 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·Count(SB),NOSPLIT,$0-40 - MOVD b_base+0(FP), R0 - MOVD b_len+8(FP), R2 - MOVBU c+24(FP), R1 - MOVD $ret+32(FP), R8 - B countbytebody<>(SB) - -TEXT ·CountString(SB),NOSPLIT,$0-32 - MOVD s_base+0(FP), R0 - MOVD s_len+8(FP), R2 - MOVBU c+16(FP), R1 - MOVD $ret+24(FP), R8 - B countbytebody<>(SB) +// func Count(b []byte, c byte) int +// input: +// R0: b ptr +// R1: b len +// R2: b cap +// R3: c byte to search +// return: +// R0: result +TEXT ·Count(SB),NOSPLIT,$0-40 + MOVD R3, R2 + B ·CountString(SB) +// func CountString(s string, c byte) int // input: -// R0: data -// R2: data len -// R1: byte to find -// R8: address to put result -TEXT countbytebody<>(SB),NOSPLIT,$0 +// R0: s ptr +// R1: s len +// R2: c byte to search (due to ABIInternal upper bits can contain junk) +// return: +// R0: result +TEXT ·CountString(SB),NOSPLIT,$0-32 // R11 = count of byte to search MOVD $0, R11 // short path to handle 0-byte case - CBZ R2, done - CMP $0x20, R2 - // jump directly to tail if length < 32 - BLO tail + CBZ R1, done + CMP $0x20, R1 + // jump directly to head if length >= 32 + BHS head +tail: + // Work with tail shorter than 32 bytes + MOVBU.P 1(R0), R5 + SUB $1, R1, R1 + CMP R2.UXTB, R5 + CINC EQ, R11, R11 + CBNZ R1, tail +done: + MOVD R11, R0 + RET + PCALIGN $16 +head: ANDS $0x1f, R0, R9 BEQ chunk // Work with not 32-byte aligned head @@ -40,24 +52,23 @@ TEXT countbytebody<>(SB),NOSPLIT,$0 PCALIGN $16 head_loop: MOVBU.P 1(R0), R5 - CMP R5, R1 + CMP R2.UXTB, R5 CINC EQ, R11, R11 - SUB $1, R2, R2 + SUB $1, R1, R1 CMP R0, R3 BNE head_loop - // Work with 32-byte aligned chunks chunk: - BIC $0x1f, R2, R9 + BIC $0x1f, R1, R9 // The first chunk can also be the last CBZ R9, tail // R3 = end of 32-byte chunks ADD R0, R9, R3 MOVD $1, R5 VMOV R5, V5.B16 - // R2 = length of tail - SUB R9, R2, R2 - // Duplicate R1 (byte to search) to 16 1-byte elements of V0 - VMOV R1, V0.B16 + // R1 = length of tail + SUB R9, R1, R1 + // Duplicate R2 (byte to search) to 16 1-byte elements of V0 + VMOV R2, V0.B16 // Clear the low 64-bit element of V7 and V8 VEOR V7.B8, V7.B8, V7.B8 VEOR V8.B8, V8.B8, V8.B8 @@ -79,14 +90,5 @@ chunk_loop: BNE chunk_loop VMOV V8.D[0], R6 ADD R6, R11, R11 - CBZ R2, done -tail: - // Work with tail shorter than 32 bytes - MOVBU.P 1(R0), R5 - SUB $1, R2, R2 - CMP R5, R1 - CINC EQ, R11, R11 - CBNZ R2, tail -done: - MOVD R11, (R8) - RET + CBZ R1, done + B tail diff --git a/src/internal/bytealg/count_generic.go b/src/internal/bytealg/count_generic.go index 932a7c584c1ae6..e269a21dbd81c9 100644 --- a/src/internal/bytealg/count_generic.go +++ b/src/internal/bytealg/count_generic.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !amd64 && !arm && !arm64 && !ppc64le && !ppc64 && !riscv64 && !s390x +//go:build !amd64 && !arm && !arm64 && !loong64 && !mips64le && !mips64 && !ppc64le && !ppc64 && !riscv64 && !s390x package bytealg diff --git a/src/internal/bytealg/count_loong64.s b/src/internal/bytealg/count_loong64.s new file mode 100644 index 00000000000000..74c4c2472ae46d --- /dev/null +++ b/src/internal/bytealg/count_loong64.s @@ -0,0 +1,238 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "textflag.h" + +TEXT ·Count(SB),NOSPLIT,$0-40 + // R4 = b_base + // R5 = b_len + // R6 = b_cap (unused) + // R7 = byte to count + AND $0xff, R7, R6 + JMP countbody<>(SB) + +TEXT ·CountString(SB),NOSPLIT,$0-32 + // R4 = s_base + // R5 = s_len + // R6 = byte to count + AND $0xff, R6 + JMP countbody<>(SB) + +// input: +// R4 = s_base +// R5 = s_len +// R6 = byte to count +TEXT countbody<>(SB),NOSPLIT,$0 + MOVV R0, R7 // count + + // short path to handle 0-byte case + BEQ R5, done + + // jump directly to tail length < 8 + MOVV $8, R8 + BLT R5, R8, tail + + // Implemented using 256-bit SMID instructions +lasxCountBody: + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R8 + BEQ R8, lsxCountBody + XVMOVQ R6, X0.B32 + + // jump directly to lasx32 if length < 128 + MOVV $128, R8 + BLT R5, R8, lasx32 +lasx128: +lasx128Loop: + XVMOVQ 0(R4), X1 + XVMOVQ 32(R4), X2 + XVMOVQ 64(R4), X3 + XVMOVQ 96(R4), X4 + + XVSEQB X0, X1, X5 + XVSEQB X0, X2, X6 + XVSEQB X0, X3, X7 + XVSEQB X0, X4, X8 + + XVANDB $1, X5, X5 + XVANDB $1, X6, X6 + XVANDB $1, X7, X7 + XVANDB $1, X8, X8 + + XVPCNTV X5, X1 + XVPCNTV X6, X2 + XVPCNTV X7, X3 + XVPCNTV X8, X4 + + XVADDV X2, X1 + XVADDV X4, X3 + XVADDV X3, X1 + + XVMOVQ X1.V[0], R9 + XVMOVQ X1.V[1], R10 + XVMOVQ X1.V[2], R11 + XVMOVQ X1.V[3], R12 + + ADDV R9, R10 + ADDV R11, R12 + ADDV R10, R7 + ADDV R12, R7 + + ADDV $-128, R5 + ADDV $128, R4 + BGE R5, R8, lasx128Loop + +lasx32: + // jump directly to lasx8 if length < 32 + MOVV $32, R8 + BLT R5, R8, lasx8 +lasx32Loop: + XVMOVQ 0(R4), X1 + XVSEQB X0, X1, X2 + XVANDB $1, X2, X2 + XVPCNTV X2, X1 + XVMOVQ X1.V[0], R9 + XVMOVQ X1.V[1], R10 + XVMOVQ X1.V[2], R11 + XVMOVQ X1.V[3], R12 + ADDV R9, R10 + ADDV R11, R12 + ADDV R10, R7 + ADDV R12, R7 + ADDV $-32, R5 + ADDV $32, R4 + BGE R5, R8, lasx32Loop +lasx8: + // jump directly to tail if length < 8 + MOVV $8, R8 + BLT R5, R8, tail +lasx8Loop: + MOVV 0(R4), R9 + VMOVQ R9, V1.V[0] + VSEQB V0, V1, V2 + VANDB $1, V2, V2 + VPCNTV V2, V1 + + VMOVQ V1.V[0], R9 + ADDV R9, R7 + ADDV $-8, R5 + ADDV $8, R4 + BGE R5, R8, lasx8Loop + JMP tail + + // Implemented using 128-bit SMID instructions +lsxCountBody: + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R8 + BEQ R8, genericCountBody + VMOVQ R6, V0.B16 + + // jump directly to lsx16 if length < 64 + MOVV $64, R8 + BLT R5, R8, lsx16 +lsx64: +lsx64Loop: + VMOVQ 0(R4), V1 + VMOVQ 16(R4), V2 + VMOVQ 32(R4), V3 + VMOVQ 48(R4), V4 + + VSEQB V0, V1, V5 + VSEQB V0, V2, V6 + VSEQB V0, V3, V7 + VSEQB V0, V4, V8 + + VANDB $1, V5, V5 + VANDB $1, V6, V6 + VANDB $1, V7, V7 + VANDB $1, V8, V8 + + VPCNTV V5, V1 + VPCNTV V6, V2 + VPCNTV V7, V3 + VPCNTV V8, V4 + + VADDV V2, V1 + VADDV V4, V3 + VADDV V3, V1 + + VMOVQ V1.V[0], R9 + VMOVQ V1.V[1], R10 + ADDV R9, R7 + ADDV R10, R7 + + ADDV $-64, R5 + ADDV $64, R4 + BGE R5, R8, lsx64Loop + +lsx16: + // jump directly to lsx8 if length < 16 + MOVV $16, R8 + BLT R5, R8, lsx8 +lsx16Loop: + VMOVQ 0(R4), V1 + VSEQB V0, V1, V2 + VANDB $1, V2, V2 + VPCNTV V2, V1 + VMOVQ V1.V[0], R9 + VMOVQ V1.V[1], R10 + ADDV R9, R7 + ADDV R10, R7 + ADDV $-16, R5 + ADDV $16, R4 + BGE R5, R8, lsx16Loop +lsx8: + // jump directly to tail if length < 8 + MOVV $8, R8 + BLT R5, R8, tail +lsx8Loop: + MOVV 0(R4), R9 + VMOVQ R9, V1.V[0] + VSEQB V0, V1, V2 + VANDB $1, V2, V2 + VPCNTV V2, V1 + + VMOVQ V1.V[0], R9 + ADDV R9, R7 + ADDV $-8, R5 + ADDV $8, R4 + BGE R5, R8, lsx8Loop + JMP tail + + // Implemented using general instructions +genericCountBody: + MOVV $4, R8 + MOVV $1, R9 +genericLoop: + BLT R5, R8, tail + ADDV $-4, R5 + MOVWU (R4)(R5), R10 + BSTRPICKW $7, R10, $0, R11 + BSTRPICKW $15, R10, $8, R12 + XOR R6, R11 + XOR R6, R12 + MASKNEZ R11, R9, R13 + MASKNEZ R12, R9, R14 + ADDV R13, R7 + ADDV R14, R7 + BSTRPICKW $23, R10, $16, R11 + BSTRPICKW $31, R10, $24, R12 + XOR R6, R11 + XOR R6, R12 + MASKNEZ R11, R9, R13 + MASKNEZ R12, R9, R14 + ADDV R13, R7 + ADDV R14, R7 + JMP genericLoop + + // Work with tail shorter than 8 bytes +tail: + BEQ R5, done + ADDV $-1, R5 + MOVBU (R4)(R5), R8 + BNE R6, R8, tail + ADDV $1, R7 + JMP tail +done: + MOVV R7, R4 + RET diff --git a/src/internal/bytealg/count_mips64x.s b/src/internal/bytealg/count_mips64x.s new file mode 100644 index 00000000000000..247d0b35f4cc86 --- /dev/null +++ b/src/internal/bytealg/count_mips64x.s @@ -0,0 +1,52 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build mips64 || mips64le + +#include "go_asm.h" +#include "textflag.h" + +TEXT ·Count(SB),NOSPLIT,$0-40 + // R1 = b_base + // R2 = b_len + // R3 = byte to count + MOVV b_base+0(FP), R1 + MOVV b_len+8(FP), R2 + MOVBU c+24(FP), R3 + MOVV R0, R5 // count + ADDV R1, R2 // end + +loop: + BEQ R1, R2, done + MOVBU (R1), R6 + ADDV $1, R1 + BNE R3, R6, loop + ADDV $1, R5 + JMP loop + +done: + MOVV R5, ret+32(FP) + RET + +TEXT ·CountString(SB),NOSPLIT,$0-32 + // R1 = s_base + // R2 = s_len + // R3 = byte to count + MOVV s_base+0(FP), R1 + MOVV s_len+8(FP), R2 + MOVBU c+16(FP), R3 + MOVV R0, R5 // count + ADDV R1, R2 // end + +loop: + BEQ R1, R2, done + MOVBU (R1), R6 + ADDV $1, R1 + BNE R3, R6, loop + ADDV $1, R5 + JMP loop + +done: + MOVV R5, ret+24(FP) + RET diff --git a/src/internal/bytealg/count_native.go b/src/internal/bytealg/count_native.go index 90189c9fe051bd..ba48e242d7deb3 100644 --- a/src/internal/bytealg/count_native.go +++ b/src/internal/bytealg/count_native.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build amd64 || arm || arm64 || ppc64le || ppc64 || riscv64 || s390x +//go:build amd64 || arm || arm64 || loong64 || mips64le || mips64 || ppc64le || ppc64 || riscv64 || s390x package bytealg diff --git a/src/internal/bytealg/count_riscv64.s b/src/internal/bytealg/count_riscv64.s index 3f255cd2630c64..6cc49d13882b1b 100644 --- a/src/internal/bytealg/count_riscv64.s +++ b/src/internal/bytealg/count_riscv64.s @@ -5,6 +5,13 @@ #include "go_asm.h" #include "textflag.h" +TEXT ·CountString(SB),NOSPLIT,$0-32 + // X10 = s_base + // X11 = s_len + // X12 = byte to count + MOV X12, X13 + JMP ·Count(SB) + TEXT ·Count(SB),NOSPLIT,$0-40 // X10 = b_base // X11 = b_len @@ -26,24 +33,3 @@ loop: done: MOV X14, X10 RET - -TEXT ·CountString(SB),NOSPLIT,$0-32 - // X10 = s_base - // X11 = s_len - // X12 = byte to count - AND $0xff, X12 - MOV ZERO, X14 // count - ADD X10, X11 // end - - PCALIGN $16 -loop: - BEQ X10, X11, done - MOVBU (X10), X15 - ADD $1, X10 - BNE X12, X15, loop - ADD $1, X14 - JMP loop - -done: - MOV X14, X10 - RET diff --git a/src/internal/bytealg/equal_arm64.s b/src/internal/bytealg/equal_arm64.s index 4db951547443d5..408ab374e62994 100644 --- a/src/internal/bytealg/equal_arm64.s +++ b/src/internal/bytealg/equal_arm64.s @@ -5,25 +5,11 @@ #include "go_asm.h" #include "textflag.h" -// memequal(a, b unsafe.Pointer, size uintptr) bool -TEXT runtime·memequal(SB),NOSPLIT|NOFRAME,$0-25 - // short path to handle 0-byte case - CBZ R2, equal - // short path to handle equal pointers - CMP R0, R1 - BEQ equal - B memeqbody<>(SB) -equal: - MOVD $1, R0 - RET - // memequal_varlen(a, b unsafe.Pointer) bool TEXT runtime·memequal_varlen(SB),NOSPLIT,$0-17 - CMP R0, R1 - BEQ eq MOVD 8(R26), R2 // compiler stores size at offset 8 in the closure CBZ R2, eq - B memeqbody<>(SB) + B runtime·memequal(SB) eq: MOVD $1, R0 RET @@ -33,7 +19,13 @@ eq: // R1: pointer b // R2: data len // at return: result in R0 -TEXT memeqbody<>(SB),NOSPLIT,$0 +// memequal(a, b unsafe.Pointer, size uintptr) bool +TEXT runtime·memequal(SB),NOSPLIT|NOFRAME,$0-25 + // short path to handle 0-byte case + CBZ R2, equal + // short path to handle equal pointers + CMP R0, R1 + BEQ equal CMP $1, R2 // handle 1-byte special case for better performance BEQ one @@ -91,6 +83,7 @@ tail: EOR R4, R5 CBNZ R5, not_equal B equal + PCALIGN $16 lt_8: TBZ $2, R2, lt_4 MOVWU (R0), R4 @@ -103,6 +96,7 @@ lt_8: EOR R4, R5 CBNZ R5, not_equal B equal + PCALIGN $16 lt_4: TBZ $1, R2, lt_2 MOVHU.P 2(R0), R4 diff --git a/src/internal/bytealg/equal_loong64.s b/src/internal/bytealg/equal_loong64.s index 830b09bd2cf3d5..8f570e8eaedb8d 100644 --- a/src/internal/bytealg/equal_loong64.s +++ b/src/internal/bytealg/equal_loong64.s @@ -8,37 +8,266 @@ #define REGCTXT R29 // memequal(a, b unsafe.Pointer, size uintptr) bool -TEXT runtime·memequal(SB),NOSPLIT|NOFRAME,$0-25 - BEQ R4, R5, eq - ADDV R4, R6, R7 - PCALIGN $16 -loop: - BNE R4, R7, test - MOVV $1, R4 - RET -test: - MOVBU (R4), R9 - ADDV $1, R4 - MOVBU (R5), R10 - ADDV $1, R5 - BEQ R9, R10, loop - - MOVB R0, R4 - RET -eq: - MOVV $1, R4 - RET +TEXT runtime·memequal(SB),NOSPLIT|NOFRAME,$0 + // R4 = a_base + // R5 = b_base + // R6 = size + JMP equalbody<>(SB) // memequal_varlen(a, b unsafe.Pointer) bool -TEXT runtime·memequal_varlen(SB),NOSPLIT,$40-17 - BEQ R4, R5, eq +TEXT runtime·memequal_varlen(SB),NOSPLIT,$0 + // R4 = a_base + // R5 = b_base MOVV 8(REGCTXT), R6 // compiler stores size at offset 8 in the closure - MOVV R4, 8(R3) - MOVV R5, 16(R3) - MOVV R6, 24(R3) - JAL runtime·memequal(SB) - MOVBU 32(R3), R4 - RET -eq: + JMP equalbody<>(SB) + +// input: +// R4 = a_base +// R5 = b_base +// R6 = size +TEXT equalbody<>(SB),NOSPLIT|NOFRAME,$0 + // a_base == b_base + BEQ R4, R5, equal + // 0 bytes + BEQ R6, equal + + MOVV $64, R7 + BGE R6, R7, lasx + + // size < 64 bytes +tail: + MOVV $16, R7 + BLT R6, R7, lt_16 +generic16_loop: + ADDV $-16, R6 + MOVV 0(R4), R8 + MOVV 8(R4), R9 + MOVV 0(R5), R10 + MOVV 8(R5), R11 + BNE R8, R10, not_equal + BNE R9, R11, not_equal + BEQ R6, equal + ADDV $16, R4 + ADDV $16, R5 + BGE R6, R7, generic16_loop + + // size < 16 bytes +lt_16: + MOVV $8, R7 + BLT R6, R7, lt_8 + ADDV $-8, R6 + MOVV 0(R4), R8 + MOVV 0(R5), R9 + BNE R8, R9, not_equal + BEQ R6, equal + ADDV $8, R4 + ADDV $8, R5 + + // size < 8 bytes +lt_8: + MOVV $4, R7 + BLT R6, R7, lt_4 + ADDV $-4, R6 + MOVW 0(R4), R8 + MOVW 0(R5), R9 + BNE R8, R9, not_equal + BEQ R6, equal + ADDV $4, R4 + ADDV $4, R5 + + // size < 4 bytes +lt_4: + MOVV $2, R7 + BLT R6, R7, lt_2 + ADDV $-2, R6 + MOVH 0(R4), R8 + MOVH 0(R5), R9 + BNE R8, R9, not_equal + BEQ R6, equal + ADDV $2, R4 + ADDV $2, R5 + + // size < 2 bytes +lt_2: + MOVB 0(R4), R8 + MOVB 0(R5), R9 + BNE R8, R9, not_equal + +equal: MOVV $1, R4 RET + +not_equal: + MOVV R0, R4 + RET + + // Implemented using 256-bit SIMD instructions +lasx: + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R7 + BEQ R7, lsx + +lasx256: + MOVV $256, R7 + BLT R6, R7, lasx64 +lasx256_loop: + ADDV $-256, R6 + XVMOVQ 0(R4), X0 + XVMOVQ 32(R4), X1 + XVMOVQ 64(R4), X2 + XVMOVQ 96(R4), X3 + XVMOVQ 128(R4), X4 + XVMOVQ 160(R4), X5 + XVMOVQ 192(R4), X6 + XVMOVQ 224(R4), X7 + XVMOVQ 0(R5), X8 + XVMOVQ 32(R5), X9 + XVMOVQ 64(R5), X10 + XVMOVQ 96(R5), X11 + XVMOVQ 128(R5), X12 + XVMOVQ 160(R5), X13 + XVMOVQ 192(R5), X14 + XVMOVQ 224(R5), X15 + XVSEQV X0, X8, X0 + XVSEQV X1, X9, X1 + XVSEQV X2, X10, X2 + XVSEQV X3, X11, X3 + XVSEQV X4, X12, X4 + XVSEQV X5, X13, X5 + XVSEQV X6, X14, X6 + XVSEQV X7, X15, X7 + XVANDV X0, X1, X0 + XVANDV X2, X3, X2 + XVANDV X4, X5, X4 + XVANDV X6, X7, X6 + XVANDV X0, X2, X0 + XVANDV X4, X6, X4 + XVANDV X0, X4, X0 + XVSETALLNEV X0, FCC0 + BFPF not_equal + BEQ R6, equal + ADDV $256, R4 + ADDV $256, R5 + BGE R6, R7, lasx256_loop + +lasx64: + MOVV $64, R7 + BLT R6, R7, tail +lasx64_loop: + ADDV $-64, R6 + XVMOVQ 0(R4), X0 + XVMOVQ 32(R4), X1 + XVMOVQ 0(R5), X2 + XVMOVQ 32(R5), X3 + XVSEQV X0, X2, X0 + XVSEQV X1, X3, X1 + XVANDV X0, X1, X0 + XVSETALLNEV X0, FCC0 + BFPF not_equal + BEQ R6, equal + ADDV $64, R4 + ADDV $64, R5 + BGE R6, R7, lasx64_loop + JMP tail + + // Implemented using 128-bit SIMD instructions +lsx: + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R7 + BEQ R7, generic64_loop + +lsx128: + MOVV $128, R7 + BLT R6, R7, lsx32 +lsx128_loop: + ADDV $-128, R6 + VMOVQ 0(R4), V0 + VMOVQ 16(R4), V1 + VMOVQ 32(R4), V2 + VMOVQ 48(R4), V3 + VMOVQ 64(R4), V4 + VMOVQ 80(R4), V5 + VMOVQ 96(R4), V6 + VMOVQ 112(R4), V7 + VMOVQ 0(R5), V8 + VMOVQ 16(R5), V9 + VMOVQ 32(R5), V10 + VMOVQ 48(R5), V11 + VMOVQ 64(R5), V12 + VMOVQ 80(R5), V13 + VMOVQ 96(R5), V14 + VMOVQ 112(R5), V15 + VSEQV V0, V8, V0 + VSEQV V1, V9, V1 + VSEQV V2, V10, V2 + VSEQV V3, V11, V3 + VSEQV V4, V12, V4 + VSEQV V5, V13, V5 + VSEQV V6, V14, V6 + VSEQV V7, V15, V7 + VANDV V0, V1, V0 + VANDV V2, V3, V2 + VANDV V4, V5, V4 + VANDV V6, V7, V6 + VANDV V0, V2, V0 + VANDV V4, V6, V4 + VANDV V0, V4, V0 + VSETALLNEV V0, FCC0 + BFPF not_equal + BEQ R6, equal + + ADDV $128, R4 + ADDV $128, R5 + BGE R6, R7, lsx128_loop + +lsx32: + MOVV $32, R7 + BLT R6, R7, tail +lsx32_loop: + ADDV $-32, R6 + VMOVQ 0(R4), V0 + VMOVQ 16(R4), V1 + VMOVQ 0(R5), V2 + VMOVQ 16(R5), V3 + VSEQV V0, V2, V0 + VSEQV V1, V3, V1 + VANDV V0, V1, V0 + VSETALLNEV V0, FCC0 + BFPF not_equal + BEQ R6, equal + ADDV $32, R4 + ADDV $32, R5 + BGE R6, R7, lsx32_loop + JMP tail + + // Implemented using general instructions +generic64_loop: + ADDV $-64, R6 + MOVV 0(R4), R7 + MOVV 8(R4), R8 + MOVV 16(R4), R9 + MOVV 24(R4), R10 + MOVV 0(R5), R15 + MOVV 8(R5), R16 + MOVV 16(R5), R17 + MOVV 24(R5), R18 + BNE R7, R15, not_equal + BNE R8, R16, not_equal + BNE R9, R17, not_equal + BNE R10, R18, not_equal + MOVV 32(R4), R11 + MOVV 40(R4), R12 + MOVV 48(R4), R13 + MOVV 56(R4), R14 + MOVV 32(R5), R19 + MOVV 40(R5), R20 + MOVV 48(R5), R21 + MOVV 56(R5), R23 + BNE R11, R19, not_equal + BNE R12, R20, not_equal + BNE R13, R21, not_equal + BNE R14, R23, not_equal + BEQ R6, equal + ADDV $64, R4 + ADDV $64, R5 + MOVV $64, R7 + BGE R6, R7, generic64_loop + JMP tail diff --git a/src/internal/bytealg/equal_riscv64.s b/src/internal/bytealg/equal_riscv64.s index 7f470ce0a01d25..58e033f8479b69 100644 --- a/src/internal/bytealg/equal_riscv64.s +++ b/src/internal/bytealg/equal_riscv64.s @@ -2,34 +2,62 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include "asm_riscv64.h" #include "go_asm.h" #include "textflag.h" #define CTXT S10 -// func memequal(a, b unsafe.Pointer, size uintptr) bool -TEXT runtime·memequal(SB),NOSPLIT|NOFRAME,$0-25 - // X10 = a_base - // X11 = b_base - // X12 = size - JMP memequal<>(SB) - // func memequal_varlen(a, b unsafe.Pointer) bool TEXT runtime·memequal_varlen(SB),NOSPLIT|NOFRAME,$0-17 + // X10 = a_base + // X11 = b_base MOV 8(CTXT), X12 // compiler stores size at offset 8 in the closure + JMP runtime·memequal(SB) + +// func memequal(a, b unsafe.Pointer, size uintptr) bool +TEXT runtime·memequal(SB),NOSPLIT|NOFRAME,$0-25 // X10 = a_base // X11 = b_base - JMP memequal<>(SB) + // X12 = size + BNE X10, X11, length_check + MOV $0, X12 -// On entry X10 and X11 contain pointers, X12 contains length. -// For non-regabi X13 contains address for return value. -// For regabi return value in X10. -TEXT memequal<>(SB),NOSPLIT|NOFRAME,$0 - BEQ X10, X11, eq +length_check: + BEQZ X12, done MOV $32, X23 BLT X12, X23, loop4_check +#ifndef hasV + MOVB internal∕cpu·RISCV64+const_offsetRISCV64HasV(SB), X5 + BEQZ X5, equal_scalar +#endif + + // Use vector if not 8 byte aligned. + OR X10, X11, X5 + AND $7, X5 + BNEZ X5, vector_loop + + // Use scalar if 8 byte aligned and <= 64 bytes. + SUB $64, X12, X6 + BLEZ X6, loop32_check + + PCALIGN $16 +vector_loop: + VSETVLI X12, E8, M8, TA, MA, X5 + VLE8V (X10), V8 + VLE8V (X11), V16 + VMSNEVV V8, V16, V0 + VFIRSTM V0, X6 + BGEZ X6, done + ADD X5, X10 + ADD X5, X11 + SUB X5, X12 + BNEZ X12, vector_loop + JMP done + +equal_scalar: // Check alignment - if alignment differs we have to do one byte at a time. AND $7, X10, X9 AND $7, X11, X19 @@ -44,7 +72,7 @@ align: SUB $1, X9 MOVBU 0(X10), X19 MOVBU 0(X11), X20 - BNE X19, X20, not_eq + BNE X19, X20, done ADD $1, X10 ADD $1, X11 BNEZ X9, align @@ -57,19 +85,19 @@ loop32: MOV 0(X11), X20 MOV 8(X10), X21 MOV 8(X11), X22 - BNE X19, X20, not_eq - BNE X21, X22, not_eq + BNE X19, X20, done + BNE X21, X22, done MOV 16(X10), X14 MOV 16(X11), X15 MOV 24(X10), X16 MOV 24(X11), X17 - BNE X14, X15, not_eq - BNE X16, X17, not_eq + BNE X14, X15, done + BNE X16, X17, done ADD $32, X10 ADD $32, X11 SUB $32, X12 BGE X12, X9, loop32 - BEQZ X12, eq + BEQZ X12, done loop16_check: MOV $16, X23 @@ -79,13 +107,13 @@ loop16: MOV 0(X11), X20 MOV 8(X10), X21 MOV 8(X11), X22 - BNE X19, X20, not_eq - BNE X21, X22, not_eq + BNE X19, X20, done + BNE X21, X22, done ADD $16, X10 ADD $16, X11 SUB $16, X12 BGE X12, X23, loop16 - BEQZ X12, eq + BEQZ X12, done loop4_check: MOV $4, X23 @@ -95,32 +123,30 @@ loop4: MOVBU 0(X11), X20 MOVBU 1(X10), X21 MOVBU 1(X11), X22 - BNE X19, X20, not_eq - BNE X21, X22, not_eq + BNE X19, X20, done + BNE X21, X22, done MOVBU 2(X10), X14 MOVBU 2(X11), X15 MOVBU 3(X10), X16 MOVBU 3(X11), X17 - BNE X14, X15, not_eq - BNE X16, X17, not_eq + BNE X14, X15, done + BNE X16, X17, done ADD $4, X10 ADD $4, X11 SUB $4, X12 BGE X12, X23, loop4 loop1: - BEQZ X12, eq + BEQZ X12, done MOVBU 0(X10), X19 MOVBU 0(X11), X20 - BNE X19, X20, not_eq + BNE X19, X20, done ADD $1, X10 ADD $1, X11 SUB $1, X12 JMP loop1 -not_eq: - MOVB ZERO, X10 - RET -eq: - MOV $1, X10 +done: + // If X12 is zero then memory is equivalent. + SEQZ X12, X10 RET diff --git a/src/internal/bytealg/index_arm64.s b/src/internal/bytealg/index_arm64.s index 3a551a72da1393..38e0b14e75781a 100644 --- a/src/internal/bytealg/index_arm64.s +++ b/src/internal/bytealg/index_arm64.s @@ -5,29 +5,30 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·Index(SB),NOSPLIT,$0-56 - MOVD a_base+0(FP), R0 - MOVD a_len+8(FP), R1 - MOVD b_base+24(FP), R2 - MOVD b_len+32(FP), R3 - MOVD $ret+48(FP), R9 - B indexbody<>(SB) - -TEXT ·IndexString(SB),NOSPLIT,$0-40 - MOVD a_base+0(FP), R0 - MOVD a_len+8(FP), R1 - MOVD b_base+16(FP), R2 - MOVD b_len+24(FP), R3 - MOVD $ret+32(FP), R9 - B indexbody<>(SB) +// func Index(a, b []byte) int +// input: +// R0: a ptr (haystack) +// R1: a len (haystack) +// R2: a cap (haystack) (unused) +// R3: b ptr (needle) +// R4: b len (needle) (2 <= len <= 32) +// R5: b cap (needle) (unused) +// return: +// R0: result +TEXT ·Index(SB),NOSPLIT,$0-56 + MOVD R3, R2 + MOVD R4, R3 + B ·IndexString(SB) +// func IndexString(a, b string) int // input: -// R0: haystack -// R1: length of haystack -// R2: needle -// R3: length of needle (2 <= len <= 32) -// R9: address to put result -TEXT indexbody<>(SB),NOSPLIT,$0-56 +// R0: a ptr (haystack) +// R1: a len (haystack) +// R2: b ptr (needle) +// R3: b len (needle) (2 <= len <= 32) +// return: +// R0: result +TEXT ·IndexString(SB),NOSPLIT,$0-40 // main idea is to load 'sep' into separate register(s) // to avoid repeatedly re-load it again and again // for sebsequent substring comparisons @@ -136,11 +137,9 @@ loop_2: BNE loop_2 found: SUB R8, R0, R0 - MOVD R0, (R9) RET not_found: MOVD $-1, R0 - MOVD R0, (R9) RET greater_8: SUB $9, R3, R11 // len(sep) - 9, offset of R0 for last 8 bytes diff --git a/src/internal/bytealg/index_generic.go b/src/internal/bytealg/index_generic.go index a59e32938e76ec..643bb59ab1edbb 100644 --- a/src/internal/bytealg/index_generic.go +++ b/src/internal/bytealg/index_generic.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !amd64 && !arm64 && !s390x && !ppc64le && !ppc64 +//go:build !amd64 && !arm64 && !loong64 && !s390x && !ppc64le && !ppc64 package bytealg diff --git a/src/internal/bytealg/index_loong64.go b/src/internal/bytealg/index_loong64.go new file mode 100644 index 00000000000000..ad574d66faee18 --- /dev/null +++ b/src/internal/bytealg/index_loong64.go @@ -0,0 +1,30 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytealg + +import "internal/cpu" + +// Empirical data shows that using Index can get better +// performance when len(s) <= 16. +const MaxBruteForce = 16 + +func init() { + // If SIMD is supported, optimize the cases where the substring length is less than 64 bytes, + // otherwise, cases the length less than 32 bytes is optimized. + if cpu.Loong64.HasLASX || cpu.Loong64.HasLSX { + MaxLen = 64 + } else { + MaxLen = 32 + } +} + +// Cutover reports the number of failures of IndexByte we should tolerate +// before switching over to Index. +// n is the number of bytes processed so far. +// See the bytes.Index implementation for details. +func Cutover(n int) int { + // 1 error per 8 characters, plus a few slop to start. + return (n + 16) / 8 +} diff --git a/src/internal/bytealg/index_loong64.s b/src/internal/bytealg/index_loong64.s new file mode 100644 index 00000000000000..1016db738dee1b --- /dev/null +++ b/src/internal/bytealg/index_loong64.s @@ -0,0 +1,303 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "textflag.h" + +TEXT ·Index(SB),NOSPLIT,$0-56 + MOVV R7, R6 // R6 = separator pointer + MOVV R8, R7 // R7 = separator length + JMP indexbody<>(SB) + +TEXT ·IndexString(SB),NOSPLIT,$0-40 + JMP indexbody<>(SB) + +// input: +// R4 = string +// R5 = length +// R6 = separator pointer +// R7 = separator length (2 <= len <= 64) +TEXT indexbody<>(SB),NOSPLIT,$0 + // main idea is to load 'sep' into separate register(s) + // to avoid repeatedly re-load it again and again + // for sebsequent substring comparisons + SUBV R7, R5, R8 + ADDV R4, R8 // R8 contains the start of last substring for comparison + ADDV $1, R4, R9 // store base for later + + MOVV $8, R5 + BGE R7, R5, len_gt_or_eq_8 +len_2_7: + AND $0x4, R7, R5 + BNE R5, len_4_7 + +len_2_3: + AND $0x1, R7, R5 + BNE R5, len_3 + +len_2: + MOVHU (R6), R10 +loop_2: + BLT R8, R4, not_found + MOVHU (R4), R11 + ADDV $1, R4 + BNE R10, R11, loop_2 + JMP found + +len_3: + MOVHU (R6), R10 + MOVBU 2(R6), R11 +loop_3: + BLT R8, R4, not_found + MOVHU (R4), R12 + ADDV $1, R4 + BNE R10, R12, loop_3 + MOVBU 1(R4), R13 + BNE R11, R13, loop_3 + JMP found + +len_4_7: + AND $0x2, R7, R5 + BNE R5, len_6_7 + AND $0x1, R7, R5 + BNE R5, len_5 +len_4: + MOVWU (R6), R10 +loop_4: + BLT R8, R4, not_found + MOVWU (R4), R11 + ADDV $1, R4 + BNE R10, R11, loop_4 + JMP found + +len_5: + MOVWU (R6), R10 + MOVBU 4(R6), R11 +loop_5: + BLT R8, R4, not_found + MOVWU (R4), R12 + ADDV $1, R4 + BNE R10, R12, loop_5 + MOVBU 3(R4), R13 + BNE R11, R13, loop_5 + JMP found + +len_6_7: + AND $0x1, R7, R5 + BNE R5, len_7 +len_6: + MOVWU (R6), R10 + MOVHU 4(R6), R11 +loop_6: + BLT R8, R4, not_found + MOVWU (R4), R12 + ADDV $1, R4 + BNE R10, R12, loop_6 + MOVHU 3(R4), R13 + BNE R11, R13, loop_6 + JMP found + +len_7: + MOVWU (R6), R10 + MOVWU 3(R6), R11 +loop_7: + BLT R8, R4, not_found + MOVWU (R4), R12 + ADDV $1, R4 + BNE R10, R12, loop_7 + MOVWU 2(R4), R13 + BNE R11, R13, loop_7 + JMP found + +len_gt_or_eq_8: + BEQ R5, R7, len_8 + MOVV $17, R5 + BGE R7, R5, len_gt_or_eq_17 + JMP len_9_16 +len_8: + MOVV (R6), R10 +loop_8: + BLT R8, R4, not_found + MOVV (R4), R11 + ADDV $1, R4 + BNE R10, R11, loop_8 + JMP found + +len_9_16: + MOVV (R6), R10 + SUBV $8, R7 + MOVV (R6)(R7), R11 + SUBV $1, R7 +loop_9_16: + BLT R8, R4, not_found + MOVV (R4), R12 + ADDV $1, R4 + BNE R10, R12, loop_9_16 + MOVV (R4)(R7), R13 + BNE R11, R13, loop_9_16 + JMP found + +len_gt_or_eq_17: + MOVV $25, R5 + BGE R7, R5, len_gt_or_eq_25 +len_17_24: + MOVV 0(R6), R10 + MOVV 8(R6), R11 + SUBV $8, R7 + MOVV (R6)(R7), R12 + SUBV $1, R7 +loop_17_24: + BLT R8, R4, not_found + MOVV (R4), R13 + ADDV $1, R4 + BNE R10, R13, loop_17_24 + MOVV 7(R4), R14 + BNE R11, R14, loop_17_24 + MOVV (R4)(R7), R15 + BNE R12, R15, loop_17_24 + JMP found + +len_gt_or_eq_25: + MOVV $33, R5 + BGE R7, R5, len_gt_or_eq_33 + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R10 + BNE R10, lsx_len_25_32 +len_25_32: + MOVV 0(R6), R10 + MOVV 8(R6), R11 + MOVV 16(R6), R12 + SUBV $8, R7 + MOVV (R6)(R7), R13 + SUBV $1, R7 +loop_25_32: + BLT R8, R4, not_found + MOVV (R4), R14 + ADDV $1, R4 + BNE R10, R14, loop_25_32 + MOVV 7(R4), R15 + BNE R11, R15, loop_25_32 + MOVV 15(R4), R16 + BNE R12, R16, loop_25_32 + MOVV (R4)(R7), R17 + BNE R13, R17, loop_25_32 + JMP found + + // On loong64, LSX is included if LASX is supported. +lasx_len_25_32: +lsx_len_25_32: + VMOVQ 0(R6), V0 + SUBV $16, R7 + VMOVQ (R6)(R7), V1 + SUBV $1, R7 +lsx_loop_25_32: + BLT R8, R4, not_found + VMOVQ (R4), V2 + ADDV $1, R4 + VSEQV V0, V2, V2 + VSETANYEQV V2, FCC0 + BFPT FCC0, lsx_loop_25_32 + + VMOVQ (R4)(R7), V3 + VSEQV V1, V3, V3 + VSETANYEQV V3, FCC1 + BFPT FCC1, lsx_loop_25_32 + JMP found + +len_gt_or_eq_33: + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R10 + MOVV $49, R5 + BGE R7, R5, len_gt_or_eq_49 +len_33_48: + BNE R10, lasx_len_33_48 + JMP lsx_len_33_48 + +len_gt_or_eq_49: +len_49_64: + BNE R10, lasx_len_49_64 + JMP lsx_len_49_64 + +lsx_len_33_48: + VMOVQ 0(R6), V0 + VMOVQ 16(R6), V1 + SUBV $16, R7 + VMOVQ (R6)(R7), V2 + SUBV $1, R7 +lsx_loop_33_48: + BLT R8, R4, not_found + VMOVQ 0(R4), V3 + ADDV $1, R4 + VSEQV V0, V3, V3 + VSETANYEQV V3, FCC0 + BFPT FCC0, lsx_loop_33_48 + + VMOVQ 15(R4), V4 + VSEQV V1, V4, V4 + VSETANYEQV V4, FCC1 + BFPT FCC1, lsx_loop_33_48 + + VMOVQ (R4)(R7), V5 + VSEQV V2, V5, V5 + VSETANYEQV V5, FCC2 + BFPT FCC2, lsx_loop_33_48 + JMP found + +lsx_len_49_64: + VMOVQ 0(R6), V0 + VMOVQ 16(R6), V1 + VMOVQ 32(R6), V2 + SUBV $16, R7 + VMOVQ (R6)(R7), V3 + SUBV $1, R7 +lsx_loop_49_64: + BLT R8, R4, not_found + VMOVQ 0(R4), V4 + ADDV $1, R4 + VSEQV V0, V4, V4 + VSETANYEQV V4, FCC0 + BFPT FCC0, lsx_loop_49_64 + + VMOVQ 15(R4), V5 + VSEQV V1, V5, V5 + VSETANYEQV V5, FCC1 + BFPT FCC1, lsx_loop_49_64 + + VMOVQ 31(R4), V6 + VSEQV V2, V6, V6 + VSETANYEQV V6, FCC2 + BFPT FCC2, lsx_loop_49_64 + + VMOVQ (R4)(R7), V7 + VSEQV V3, V7, V7 + VSETANYEQV V7, FCC3 + BFPT FCC3, lsx_loop_49_64 + JMP found + +lasx_len_33_48: +lasx_len_49_64: +lasx_len_33_64: + XVMOVQ (R6), X0 + SUBV $32, R7 + XVMOVQ (R6)(R7), X1 + SUBV $1, R7 +lasx_loop_33_64: + BLT R8, R4, not_found + XVMOVQ (R4), X2 + ADDV $1, R4 + XVSEQV X0, X2, X3 + XVSETANYEQV X3, FCC0 + BFPT FCC0, lasx_loop_33_64 + + XVMOVQ (R4)(R7), X4 + XVSEQV X1, X4, X5 + XVSETANYEQV X5, FCC1 + BFPT FCC1, lasx_loop_33_64 + JMP found + +found: + SUBV R9, R4 + RET + +not_found: + MOVV $-1, R4 + RET diff --git a/src/internal/bytealg/index_native.go b/src/internal/bytealg/index_native.go index 59c93f9d126b90..f917c7a92adbf1 100644 --- a/src/internal/bytealg/index_native.go +++ b/src/internal/bytealg/index_native.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build amd64 || arm64 || s390x || ppc64le || ppc64 +//go:build amd64 || arm64 || loong64 || s390x || ppc64le || ppc64 package bytealg diff --git a/src/internal/bytealg/indexbyte_arm64.s b/src/internal/bytealg/indexbyte_arm64.s index 40843fbc5b443a..92a61a43029a86 100644 --- a/src/internal/bytealg/indexbyte_arm64.s +++ b/src/internal/bytealg/indexbyte_arm64.s @@ -4,26 +4,26 @@ #include "textflag.h" -TEXT ·IndexByte(SB),NOSPLIT,$0-40 - MOVD b_base+0(FP), R0 - MOVD b_len+8(FP), R2 - MOVBU c+24(FP), R1 - MOVD $ret+32(FP), R8 - B indexbytebody<>(SB) - -TEXT ·IndexByteString(SB),NOSPLIT,$0-32 - MOVD s_base+0(FP), R0 - MOVD s_len+8(FP), R2 - MOVBU c+16(FP), R1 - MOVD $ret+24(FP), R8 - B indexbytebody<>(SB) +// func IndexByte(b []byte, c byte) int +// input: +// R0: b ptr +// R1: b len +// R2: b cap (unused) +// R3: c byte to search +// return +// R0: result +TEXT ·IndexByte(SB),NOSPLIT,$0-40 + MOVD R3, R2 + B ·IndexByteString(SB) +// func IndexByteString(s string, c byte) int // input: -// R0: data -// R1: byte to search -// R2: data len -// R8: address to put result -TEXT indexbytebody<>(SB),NOSPLIT,$0 +// R0: s ptr +// R1: s len +// R2: c byte to search +// return +// R0: result +TEXT ·IndexByteString(SB),NOSPLIT,$0-32 // Core algorithm: // For each 32-byte chunk we calculate a 64-bit syndrome value, // with two bits per byte. For each tuple, bit 0 is set if the @@ -33,19 +33,19 @@ TEXT indexbytebody<>(SB),NOSPLIT,$0 // in the original string, counting trailing zeros allows to // identify exactly which byte has matched. - CBZ R2, fail + CBZ R1, fail MOVD R0, R11 // Magic constant 0x40100401 allows us to identify // which lane matches the requested byte. // 0x40100401 = ((1<<0) + (4<<8) + (16<<16) + (64<<24)) // Different bytes have different bit masks (i.e: 1, 4, 16, 64) MOVD $0x40100401, R5 - VMOV R1, V0.B16 + VMOV R2, V0.B16 // Work with aligned 32-byte chunks BIC $0x1f, R0, R3 VMOV R5, V5.S4 ANDS $0x1f, R0, R9 - AND $0x1f, R2, R10 + AND $0x1f, R1, R10 BEQ loop // Input string is not 32-byte aligned. We calculate the @@ -53,7 +53,7 @@ TEXT indexbytebody<>(SB),NOSPLIT,$0 // the first bytes and mask off the irrelevant part. VLD1.P (R3), [V1.B16, V2.B16] SUB $0x20, R9, R4 - ADDS R4, R2, R2 + ADDS R4, R1, R1 VCMEQ V0.B16, V1.B16, V3.B16 VCMEQ V0.B16, V2.B16, V4.B16 VAND V5.B16, V3.B16, V3.B16 @@ -72,7 +72,7 @@ TEXT indexbytebody<>(SB),NOSPLIT,$0 loop: VLD1.P (R3), [V1.B16, V2.B16] - SUBS $0x20, R2, R2 + SUBS $0x20, R1, R1 VCMEQ V0.B16, V1.B16, V3.B16 VCMEQ V0.B16, V2.B16, V4.B16 // If we're out of data we finish regardless of the result @@ -117,10 +117,8 @@ tail: ADD R6>>1, R3, R0 // Compute the offset result SUB R11, R0, R0 - MOVD R0, (R8) RET fail: MOVD $-1, R0 - MOVD R0, (R8) RET diff --git a/src/internal/bytealg/indexbyte_loong64.s b/src/internal/bytealg/indexbyte_loong64.s index c9591b3cdaa569..3618501063b312 100644 --- a/src/internal/bytealg/indexbyte_loong64.s +++ b/src/internal/bytealg/indexbyte_loong64.s @@ -5,48 +5,288 @@ #include "go_asm.h" #include "textflag.h" +// input: +// R4 = b_base +// R5 = b_len +// R6 = b_cap (unused) +// R7 = byte to find TEXT ·IndexByte(SB),NOSPLIT,$0-40 - // R4 = b_base - // R5 = b_len - // R6 = b_cap (unused) - // R7 = byte to find AND $0xff, R7 + JMP indexbytebody<>(SB) + +// input: +// R4 = s_base +// R5 = s_len +// R6 = byte to find +TEXT ·IndexByteString(SB),NOSPLIT,$0-32 + AND $0xff, R6, R7 // byte to find + JMP indexbytebody<>(SB) + +// input: +// R4: b_base +// R5: len +// R7: byte to find +TEXT indexbytebody<>(SB),NOSPLIT,$0 + BEQ R5, notfound // len == 0 + MOVV R4, R6 // store base for later - ADDV R4, R5 // end - ADDV $-1, R4 + ADDV R4, R5, R8 // end + + MOVV $32, R9 + BGE R5, R9, lasx +tail: + MOVV $8, R9 + BLT R5, R9, lt_8 +generic8_loop: + MOVV (R4), R10 + + AND $0xff, R10, R11 + BEQ R7, R11, found + + BSTRPICKV $15, R10, $8, R11 + BEQ R7, R11, byte_1th + + BSTRPICKV $23, R10, $16, R11 + BEQ R7, R11, byte_2th + + BSTRPICKV $31, R10, $24, R11 + BEQ R7, R11, byte_3th - PCALIGN $16 -loop: + BSTRPICKV $39, R10, $32, R11 + BEQ R7, R11, byte_4th + + BSTRPICKV $47, R10, $40, R11 + BEQ R7, R11, byte_5th + + BSTRPICKV $55, R10, $48, R11 + BEQ R7, R11, byte_6th + + BSTRPICKV $63, R10, $56, R11 + BEQ R7, R11, byte_7th + + ADDV $8, R4 + ADDV $-8, R5 + BGE R5, R9, generic8_loop + +lt_8: + BEQ R4, R8, notfound + MOVBU (R4), R10 + BEQ R7, R10, found ADDV $1, R4 - BEQ R4, R5, notfound - MOVBU (R4), R8 - BNE R7, R8, loop + JMP lt_8 - SUBV R6, R4 // remove base +byte_1th: + ADDV $1, R4 + SUBV R6, R4 RET -notfound: - MOVV $-1, R4 +byte_2th: + ADDV $2, R4 + SUBV R6, R4 RET -TEXT ·IndexByteString(SB),NOSPLIT,$0-32 - // R4 = s_base - // R5 = s_len - // R6 = byte to find - MOVV R4, R7 // store base for later - ADDV R4, R5 // end - ADDV $-1, R4 - - PCALIGN $16 -loop: - ADDV $1, R4 - BEQ R4, R5, notfound - MOVBU (R4), R8 - BNE R6, R8, loop +byte_3th: + ADDV $3, R4 + SUBV R6, R4 + RET + +byte_4th: + ADDV $4, R4 + SUBV R6, R4 + RET + +byte_5th: + ADDV $5, R4 + SUBV R6, R4 + RET + +byte_6th: + ADDV $6, R4 + SUBV R6, R4 + RET + +byte_7th: + ADDV $7, R4 - SUBV R7, R4 // remove base +found: + SUBV R6, R4 RET notfound: MOVV $-1, R4 RET + +lasx: + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R9 + BEQ R9, lsx + XVMOVQ R7, X0.B32 + + MOVV $128, R9 + BLT R5, R9, lasx32_loop +lasx128_loop: + XVMOVQ 0(R4), X1 + XVMOVQ 32(R4), X2 + XVMOVQ 64(R4), X3 + XVMOVQ 96(R4), X4 + + XVSEQB X1, X0, X1 + XVSETNEV X1, FCC0 + BFPT lasx_found_add_0 + + XVSEQB X2, X0, X1 + XVSETNEV X1, FCC0 + BFPT lasx_found_add_32 + + XVSEQB X3, X0, X1 + XVSETNEV X1, FCC0 + BFPT lasx_found_add_64 + + XVSEQB X4, X0, X1 + XVSETNEV X1, FCC0 + BFPT lasx_found_add_96 + + ADDV $128, R4 + ADDV $-128, R5 + BGE R5, R9, lasx128_loop + + BEQ R5, notfound + + MOVV $32, R9 + BLT R5, R9, tail +lasx32_loop: + XVMOVQ 0(R4), X1 + + XVSEQB X1, X0, X1 + XVSETNEV X1, FCC0 + BFPT lasx_found_add_0 + + ADDV $32, R4 + ADDV $-32, R5 + BGE R5, R9, lasx32_loop + + BEQ R5, notfound + + JMP tail + +lasx_found_add_0: + MOVV R0, R11 + JMP lasx_index_cal + +lasx_found_add_32: + MOVV $32, R11 + JMP lasx_index_cal + +lasx_found_add_64: + MOVV $64, R11 + JMP lasx_index_cal + +lasx_found_add_96: + MOVV $96, R11 + JMP lasx_index_cal + +lasx_index_cal: + MOVV $64, R9 + XVMOVQ X1.V[0], R10 + CTZV R10, R10 + BNE R10, R9, index_cal + ADDV $8, R11 + + XVMOVQ X1.V[1], R10 + CTZV R10, R10 + BNE R10, R9, index_cal + ADDV $8, R11 + + XVMOVQ X1.V[2], R10 + CTZV R10, R10 + BNE R10, R9, index_cal + ADDV $8, R11 + + XVMOVQ X1.V[3], R10 + CTZV R10, R10 + JMP index_cal + +lsx: + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R9 + BEQ R9, tail + VMOVQ R7, V0.B16 + + MOVV $64, R9 + BLT R5, R9, lsx16_loop +lsx64_loop: + VMOVQ 0(R4), V1 + VMOVQ 16(R4), V2 + VMOVQ 32(R4), V3 + VMOVQ 48(R4), V4 + + VSEQB V1, V0, V1 + VSETNEV V1, FCC0 + BFPT lsx_found_add_0 + + VSEQB V2, V0, V1 + VSETNEV V1, FCC0 + BFPT lsx_found_add_16 + + VSEQB V3, V0, V1 + VSETNEV V1, FCC0 + BFPT lsx_found_add_32 + + VSEQB V4, V0, V1 + VSETNEV V1, FCC0 + BFPT lsx_found_add_48 + + ADDV $64, R4 + ADDV $-64, R5 + BGE R5, R9, lsx64_loop + + BEQ R5, notfound + + MOVV $16, R9 + BLT R5, R9, tail +lsx16_loop: + VMOVQ 0(R4), V1 + + VSEQB V1, V0, V1 + VSETNEV V1, FCC0 + BFPT lsx_found_add_0 + + ADDV $16, R4 + ADDV $-16, R5 + BGE R5, R9, lsx16_loop + + BEQ R5, notfound + + JMP tail + +lsx_found_add_0: + MOVV R0, R11 + JMP lsx_index_cal + +lsx_found_add_16: + MOVV $16, R11 + JMP lsx_index_cal + +lsx_found_add_32: + MOVV $32, R11 + JMP lsx_index_cal + +lsx_found_add_48: + MOVV $48, R11 + JMP lsx_index_cal + +lsx_index_cal: + MOVV $64, R9 + + VMOVQ V1.V[0], R10 + CTZV R10, R10 + BNE R10, R9, index_cal + ADDV $8, R11 + + VMOVQ V1.V[1], R10 + CTZV R10, R10 + JMP index_cal + +index_cal: + SRLV $3, R10 + ADDV R11, R10 + ADDV R10, R4 + JMP found diff --git a/src/internal/bytealg/indexbyte_riscv64.s b/src/internal/bytealg/indexbyte_riscv64.s index de00983c7b9bec..527ae6d35ed55b 100644 --- a/src/internal/bytealg/indexbyte_riscv64.s +++ b/src/internal/bytealg/indexbyte_riscv64.s @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include "asm_riscv64.h" #include "go_asm.h" #include "textflag.h" @@ -10,18 +11,23 @@ TEXT ·IndexByte(SB),NOSPLIT,$0-40 // X11 = b_len // X12 = b_cap (unused) // X13 = byte to find - AND $0xff, X13 - MOV X10, X12 // store base for later + AND $0xff, X13, X12 // x12 byte to look for + + SLTI $24, X11, X14 + BNEZ X14, small + JMP indexByteBig<>(SB) + +small: + MOV X10, X13 // store base for later ADD X10, X11 // end SUB $1, X10 - loop: ADD $1, X10 BEQ X10, X11, notfound MOVBU (X10), X14 - BNE X13, X14, loop + BNE X12, X14, loop - SUB X12, X10 // remove base + SUB X13, X10 // remove base RET notfound: @@ -32,10 +38,113 @@ TEXT ·IndexByteString(SB),NOSPLIT,$0-32 // X10 = b_base // X11 = b_len // X12 = byte to find - AND $0xff, X12 + AND $0xff, X12 // x12 byte to look for + + SLTI $24, X11, X14 + BNEZ X14, small + JMP indexByteBig<>(SB) + +small: MOV X10, X13 // store base for later ADD X10, X11 // end SUB $1, X10 +loop: + ADD $1, X10 + BEQ X10, X11, notfound + MOVBU (X10), X14 + BNE X12, X14, loop + + SUB X13, X10 // remove base + RET + +notfound: + MOV $-1, X10 + RET + +TEXT indexByteBig<>(SB),NOSPLIT|NOFRAME,$0 + // On entry: + // X10 = b_base + // X11 = b_len (at least 16 bytes) + // X12 = byte to find + // On exit: + // X10 = index of first instance of sought byte, if found, or -1 otherwise + + MOV X10, X13 // store base for later + +#ifndef hasV + MOVB internal∕cpu·RISCV64+const_offsetRISCV64HasV(SB), X5 + BEQZ X5, indexbyte_scalar +#endif + + PCALIGN $16 +vector_loop: + VSETVLI X11, E8, M8, TA, MA, X5 + VLE8V (X10), V8 + VMSEQVX X12, V8, V0 + VFIRSTM V0, X6 + BGEZ X6, vector_found + ADD X5, X10 + SUB X5, X11 + BNEZ X11, vector_loop + JMP notfound + +vector_found: + SUB X13, X10 + ADD X6, X10 + RET + +indexbyte_scalar: + ADD X10, X11 // end + + // Process the first few bytes until we get to an 8 byte boundary + // No need to check for end here as we have at least 16 bytes in + // the buffer. + +unalignedloop: + AND $7, X10, X14 + BEQZ X14, aligned + MOVBU (X10), X14 + BEQ X12, X14, found + ADD $1, X10 + JMP unalignedloop + +aligned: + AND $~7, X11, X15 // X15 = end of aligned data + + // We have at least 9 bytes left + + // Use 'Determine if a word has a byte equal to n' bit hack from + // https://graphics.stanford.edu/~seander/bithacks.html to determine + // whether the byte is present somewhere in the next 8 bytes of the + // array. + + MOV $0x0101010101010101, X16 + SLLI $7, X16, X17 // X17 = 0x8080808080808080 + + MUL X12, X16, X18 // broadcast X12 to every byte in X18 + +alignedloop: + MOV (X10), X14 + XOR X14, X18, X19 + + // If the LSB in X12 is present somewhere in the 8 bytes we've just + // loaded into X14 then at least one of the bytes in X19 will be 0 + // after the XOR. If any of the bytes in X19 are zero then + // + // ((X19 - X16) & (~X19) & X17) + // + // will be non-zero. The expression will evaluate to zero if none of + // the bytes in X19 are zero, i.e., X12 is not present in X14. + + SUB X16, X19, X20 + ANDN X19, X17, X21 + AND X20, X21 + BNEZ X21, tailloop // If X21 != 0 X12 is present in X14 + ADD $8, X10 + BNE X10, X15, alignedloop + +tailloop: + SUB $1, X10 loop: ADD $1, X10 @@ -43,6 +152,7 @@ loop: MOVBU (X10), X14 BNE X12, X14, loop +found: SUB X13, X10 // remove base RET diff --git a/src/internal/byteorder/byteorder.go b/src/internal/byteorder/byteorder.go index ba37856ccd97fc..01500a87175c2d 100644 --- a/src/internal/byteorder/byteorder.go +++ b/src/internal/byteorder/byteorder.go @@ -6,30 +6,30 @@ // little and big endian integer types from/to byte slices. package byteorder -func LeUint16(b []byte) uint16 { +func LEUint16(b []byte) uint16 { _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 return uint16(b[0]) | uint16(b[1])<<8 } -func LePutUint16(b []byte, v uint16) { +func LEPutUint16(b []byte, v uint16) { _ = b[1] // early bounds check to guarantee safety of writes below b[0] = byte(v) b[1] = byte(v >> 8) } -func LeAppendUint16(b []byte, v uint16) []byte { +func LEAppendUint16(b []byte, v uint16) []byte { return append(b, byte(v), byte(v>>8), ) } -func LeUint32(b []byte) uint32 { +func LEUint32(b []byte) uint32 { _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 } -func LePutUint32(b []byte, v uint32) { +func LEPutUint32(b []byte, v uint32) { _ = b[3] // early bounds check to guarantee safety of writes below b[0] = byte(v) b[1] = byte(v >> 8) @@ -37,7 +37,7 @@ func LePutUint32(b []byte, v uint32) { b[3] = byte(v >> 24) } -func LeAppendUint32(b []byte, v uint32) []byte { +func LEAppendUint32(b []byte, v uint32) []byte { return append(b, byte(v), byte(v>>8), @@ -46,13 +46,13 @@ func LeAppendUint32(b []byte, v uint32) []byte { ) } -func LeUint64(b []byte) uint64 { +func LEUint64(b []byte) uint64 { _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 } -func LePutUint64(b []byte, v uint64) { +func LEPutUint64(b []byte, v uint64) { _ = b[7] // early bounds check to guarantee safety of writes below b[0] = byte(v) b[1] = byte(v >> 8) @@ -64,7 +64,7 @@ func LePutUint64(b []byte, v uint64) { b[7] = byte(v >> 56) } -func LeAppendUint64(b []byte, v uint64) []byte { +func LEAppendUint64(b []byte, v uint64) []byte { return append(b, byte(v), byte(v>>8), @@ -77,30 +77,30 @@ func LeAppendUint64(b []byte, v uint64) []byte { ) } -func BeUint16(b []byte) uint16 { +func BEUint16(b []byte) uint16 { _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 return uint16(b[1]) | uint16(b[0])<<8 } -func BePutUint16(b []byte, v uint16) { +func BEPutUint16(b []byte, v uint16) { _ = b[1] // early bounds check to guarantee safety of writes below b[0] = byte(v >> 8) b[1] = byte(v) } -func BeAppendUint16(b []byte, v uint16) []byte { +func BEAppendUint16(b []byte, v uint16) []byte { return append(b, byte(v>>8), byte(v), ) } -func BeUint32(b []byte) uint32 { +func BEUint32(b []byte) uint32 { _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 } -func BePutUint32(b []byte, v uint32) { +func BEPutUint32(b []byte, v uint32) { _ = b[3] // early bounds check to guarantee safety of writes below b[0] = byte(v >> 24) b[1] = byte(v >> 16) @@ -108,7 +108,7 @@ func BePutUint32(b []byte, v uint32) { b[3] = byte(v) } -func BeAppendUint32(b []byte, v uint32) []byte { +func BEAppendUint32(b []byte, v uint32) []byte { return append(b, byte(v>>24), byte(v>>16), @@ -117,13 +117,13 @@ func BeAppendUint32(b []byte, v uint32) []byte { ) } -func BeUint64(b []byte) uint64 { +func BEUint64(b []byte) uint64 { _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 } -func BePutUint64(b []byte, v uint64) { +func BEPutUint64(b []byte, v uint64) { _ = b[7] // early bounds check to guarantee safety of writes below b[0] = byte(v >> 56) b[1] = byte(v >> 48) @@ -135,7 +135,7 @@ func BePutUint64(b []byte, v uint64) { b[7] = byte(v) } -func BeAppendUint64(b []byte, v uint64) []byte { +func BEAppendUint64(b []byte, v uint64) []byte { return append(b, byte(v>>56), byte(v>>48), diff --git a/src/internal/cfg/cfg.go b/src/internal/cfg/cfg.go index 08d210b797385b..9329769721b7de 100644 --- a/src/internal/cfg/cfg.go +++ b/src/internal/cfg/cfg.go @@ -37,12 +37,14 @@ const KnownEnv = ` GOARCH GOARM GOARM64 + GOAUTH GOBIN GOCACHE GOCACHEPROG GOENV GOEXE GOEXPERIMENT + GOFIPS140 GOFLAGS GOGCCFLAGS GOHOSTARCH diff --git a/src/internal/cgrouptest/cgrouptest_linux.go b/src/internal/cgrouptest/cgrouptest_linux.go new file mode 100644 index 00000000000000..8437f992f741f4 --- /dev/null +++ b/src/internal/cgrouptest/cgrouptest_linux.go @@ -0,0 +1,206 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cgrouptest provides best-effort helpers for running tests inside a +// cgroup. +package cgrouptest + +import ( + "fmt" + "internal/runtime/cgroup" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "syscall" + "testing" +) + +type CgroupV2 struct { + orig string + path string +} + +func (c *CgroupV2) Path() string { + return c.path +} + +// Path to cpu.max. +func (c *CgroupV2) CPUMaxPath() string { + return filepath.Join(c.path, "cpu.max") +} + +// Set cpu.max. Pass -1 for quota to disable the limit. +func (c *CgroupV2) SetCPUMax(quota, period int64) error { + q := "max" + if quota >= 0 { + q = strconv.FormatInt(quota, 10) + } + buf := fmt.Sprintf("%s %d", q, period) + return os.WriteFile(c.CPUMaxPath(), []byte(buf), 0) +} + +// InCgroupV2 creates a new v2 cgroup, migrates the current process into it, +// and then calls fn. When fn returns, the current process is migrated back to +// the original cgroup and the new cgroup is destroyed. +// +// If a new cgroup cannot be created, the test is skipped. +// +// This must not be used in parallel tests, as it affects the entire process. +func InCgroupV2(t *testing.T, fn func(*CgroupV2)) { + mount, rel := findCurrent(t) + parent := findOwnedParent(t, mount, rel) + orig := filepath.Join(mount, rel) + + // Make sure the parent allows children to control cpu. + b, err := os.ReadFile(filepath.Join(parent, "cgroup.subtree_control")) + if err != nil { + t.Skipf("unable to read cgroup.subtree_control: %v", err) + } + if !slices.Contains(strings.Fields(string(b)), "cpu") { + // N.B. We should have permission to add cpu to + // subtree_control, but it seems like a bad idea to change this + // on a high-level cgroup that probably has lots of existing + // children. + t.Skipf("Parent cgroup %s does not allow children to control cpu, only %q", parent, string(b)) + } + + path, err := os.MkdirTemp(parent, "go-cgrouptest") + if err != nil { + t.Skipf("unable to create cgroup directory: %v", err) + } + // Important: defer cleanups so they run even in the event of panic. + // + // TODO(prattmic): Consider running everything in a subprocess just so + // we can clean up if it throws or otherwise doesn't run the defers. + defer func() { + if err := os.Remove(path); err != nil { + // Not much we can do, but at least inform of the + // problem. + t.Errorf("Error removing cgroup directory: %v", err) + } + }() + + migrateTo(t, path) + defer migrateTo(t, orig) + + c := &CgroupV2{ + orig: orig, + path: path, + } + fn(c) +} + +// Returns the mount and relative directory of the current cgroup the process +// is in. +func findCurrent(t *testing.T) (string, string) { + // Find the path to our current CPU cgroup. Currently this package is + // only used for CPU cgroup testing, so the distinction of different + // controllers doesn't matter. + var scratch [cgroup.ParseSize]byte + buf := make([]byte, cgroup.PathSize) + n, err := cgroup.FindCPUMountPoint(buf, scratch[:]) + if err != nil { + t.Skipf("cgroup: unable to find current cgroup mount: %v", err) + } + mount := string(buf[:n]) + + n, ver, err := cgroup.FindCPURelativePath(buf, scratch[:]) + if err != nil { + t.Skipf("cgroup: unable to find current cgroup path: %v", err) + } + if ver != cgroup.V2 { + t.Skipf("cgroup: running on cgroup v%d want v2", ver) + } + rel := string(buf[1:n]) // The returned path always starts with /, skip it. + rel = filepath.Join(".", rel) // Make sure this isn't empty string at root. + return mount, rel +} + +// Returns a parent directory in which we can create our own cgroup subdirectory. +func findOwnedParent(t *testing.T, mount, rel string) string { + // There are many ways cgroups may be set up on a system. We don't try + // to cover all of them, just common ones. + // + // To start with, systemd: + // + // Our test process is likely running inside a user session, in which + // case we are likely inside a cgroup that looks something like: + // + // /sys/fs/cgroup/user.slice/user-1234.slice/user@1234.service/vte-spawn-1.scope/ + // + // Possibly with additional slice layers between user@1234.service and + // the leaf scope. + // + // On new enough kernel and systemd versions (exact versions unknown), + // full unprivileged control of the user's cgroups is permitted + // directly via the cgroup filesystem. Specifically, the + // user@1234.service directory is owned by the user, as are all + // subdirectories. + + // We want to create our own subdirectory that we can migrate into and + // then manipulate at will. It is tempting to create a new subdirectory + // inside the current cgroup we are already in, however that will likey + // not work. cgroup v2 only allows processes to be in leaf cgroups. Our + // current cgroup likely contains multiple processes (at least this one + // and the cmd/go test runner). If we make a subdirectory and try to + // move our process into that cgroup, then the subdirectory and parent + // would both contain processes. Linux won't allow us to do that [1]. + // + // Instead, we will simply walk up to the highest directory that our + // user owns and create our new subdirectory. Since that directory + // already has a bunch of subdirectories, it must not directly contain + // and processes. + // + // (This would fall apart if we already in the highest directory we + // own, such as if there was simply a single cgroup for the entire + // user. Luckily systemd at least does not do this.) + // + // [1] Minor technicality: By default a new subdirectory has no cgroup + // controller (they must be explicitly enabled in the parent's + // cgroup.subtree_control). Linux will allow moving processes into a + // subdirectory that has no controllers while there are still processes + // in the parent, but it won't allow adding controller until the parent + // is empty. As far as I tell, the only purpose of this is to allow + // reorganizing processes into a new set of subdirectories and then + // adding controllers once done. + root, err := os.OpenRoot(mount) + if err != nil { + t.Fatalf("error opening cgroup mount root: %v", err) + } + + uid := os.Getuid() + var prev string + for rel != "." { + fi, err := root.Stat(rel) + if err != nil { + t.Fatalf("error stating cgroup path: %v", err) + } + + st := fi.Sys().(*syscall.Stat_t) + if int(st.Uid) != uid { + // Stop at first directory we don't own. + break + } + + prev = rel + rel = filepath.Join(rel, "..") + } + + if prev == "" { + t.Skipf("No parent cgroup owned by UID %d", uid) + } + + // We actually want the last directory where we were the owner. + return filepath.Join(mount, prev) +} + +// Migrate the current process to the cgroup directory dst. +func migrateTo(t *testing.T, dst string) { + pid := []byte(strconv.FormatInt(int64(os.Getpid()), 10)) + if err := os.WriteFile(filepath.Join(dst, "cgroup.procs"), pid, 0); err != nil { + t.Skipf("Unable to migrate into %s: %v", dst, err) + } +} diff --git a/src/internal/cgrouptest/cgrouptest_linux_test.go b/src/internal/cgrouptest/cgrouptest_linux_test.go new file mode 100644 index 00000000000000..f3a4a133e81825 --- /dev/null +++ b/src/internal/cgrouptest/cgrouptest_linux_test.go @@ -0,0 +1,19 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgrouptest + +import ( + "fmt" + "testing" +) + +func TestInCgroupV2(t *testing.T) { + InCgroupV2(t, func(c *CgroupV2) { + fmt.Println("Created", c.Path()) + if err := c.SetCPUMax(500000, 100000); err != nil { + t.Errorf("Erroring setting cpu.max: %v", err) + } + }) +} diff --git a/src/internal/chacha8rand/chacha8.go b/src/internal/chacha8rand/chacha8.go index 8f1b4e5315c99e..720fc5701a8615 100644 --- a/src/internal/chacha8rand/chacha8.go +++ b/src/internal/chacha8rand/chacha8.go @@ -7,7 +7,17 @@ // and must have minimal dependencies. package chacha8rand -import "internal/byteorder" +import ( + "internal/byteorder" + "internal/cpu" + "unsafe" +) + +// Offsets into internal/cpu records for use in assembly. +const ( + offsetLOONG64HasLSX = unsafe.Offsetof(cpu.Loong64.HasLSX) + offsetRISCV64HasV = unsafe.Offsetof(cpu.RISCV64.HasV) +) const ( ctrInc = 4 // increment counter by 4 between block calls @@ -53,10 +63,10 @@ func (s *State) Next() (uint64, bool) { // Init seeds the State with the given seed value. func (s *State) Init(seed [32]byte) { s.Init64([4]uint64{ - byteorder.LeUint64(seed[0*8:]), - byteorder.LeUint64(seed[1*8:]), - byteorder.LeUint64(seed[2*8:]), - byteorder.LeUint64(seed[3*8:]), + byteorder.LEUint64(seed[0*8:]), + byteorder.LEUint64(seed[1*8:]), + byteorder.LEUint64(seed[2*8:]), + byteorder.LEUint64(seed[3*8:]), }) } @@ -124,9 +134,9 @@ func Marshal(s *State) []byte { data := make([]byte, 6*8) copy(data, "chacha8:") used := (s.c/ctrInc)*chunk + s.i - byteorder.BePutUint64(data[1*8:], uint64(used)) + byteorder.BEPutUint64(data[1*8:], uint64(used)) for i, seed := range s.seed { - byteorder.LePutUint64(data[(2+i)*8:], seed) + byteorder.LEPutUint64(data[(2+i)*8:], seed) } return data } @@ -142,12 +152,12 @@ func Unmarshal(s *State, data []byte) error { if len(data) != 6*8 || string(data[:8]) != "chacha8:" { return new(errUnmarshalChaCha8) } - used := byteorder.BeUint64(data[1*8:]) + used := byteorder.BEUint64(data[1*8:]) if used > (ctrMax/ctrInc)*chunk-reseed { return new(errUnmarshalChaCha8) } for i := range s.seed { - s.seed[i] = byteorder.LeUint64(data[(2+i)*8:]) + s.seed[i] = byteorder.LEUint64(data[(2+i)*8:]) } s.c = ctrInc * (uint32(used) / chunk) block(&s.seed, &s.buf, s.c) diff --git a/src/internal/chacha8rand/chacha8_loong64.s b/src/internal/chacha8rand/chacha8_loong64.s new file mode 100644 index 00000000000000..73a1e5bf05f659 --- /dev/null +++ b/src/internal/chacha8rand/chacha8_loong64.s @@ -0,0 +1,133 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "textflag.h" + +DATA ·chachaConst+0x00(SB)/4, $0x61707865 +DATA ·chachaConst+0x04(SB)/4, $0x3320646e +DATA ·chachaConst+0x08(SB)/4, $0x79622d32 +DATA ·chachaConst+0x0c(SB)/4, $0x6b206574 +GLOBL ·chachaConst(SB), NOPTR|RODATA, $32 + +DATA ·chachaIncRot+0x00(SB)/4, $0x00000000 +DATA ·chachaIncRot+0x04(SB)/4, $0x00000001 +DATA ·chachaIncRot+0x08(SB)/4, $0x00000002 +DATA ·chachaIncRot+0x0c(SB)/4, $0x00000003 +GLOBL ·chachaIncRot(SB), NOPTR|RODATA, $32 + +// QR is the ChaCha8 quarter-round on a, b, c, and d. +#define QR(a, b, c, d) \ + VADDW a, b, a; \ + VXORV d, a, d; \ + VROTRW $16, d; \ + VADDW c, d, c; \ + VXORV b, c, b; \ + VROTRW $20, b; \ + VADDW a, b, a; \ + VXORV d, a, d; \ + VROTRW $24, d; \ + VADDW c, d, c; \ + VXORV b, c, b; \ + VROTRW $25, b + + +// func block(seed *[8]uint32, blocks *[4][16]uint32, counter uint32) +TEXT ·block(SB), NOSPLIT, $0 + // seed in R4 + // blocks in R5 + // counter in R6 + + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R7 + BNE R7, lsx_chacha8 + JMP ·block_generic(SB) + RET + +lsx_chacha8: + MOVV $·chachaConst(SB), R10 + MOVV $·chachaIncRot(SB), R11 + + // load contants + VMOVQ (R10), V0.W4 + VMOVQ 4(R10), V1.W4 + VMOVQ 8(R10), V2.W4 + VMOVQ 12(R10), V3.W4 + + // load 4-32bit data from incRotMatrix added to counter + VMOVQ (R11), V30 + + // load seed + VMOVQ (R4), V4.W4 + VMOVQ 4(R4), V5.W4 + VMOVQ 8(R4), V6.W4 + VMOVQ 12(R4), V7.W4 + VMOVQ 16(R4), V8.W4 + VMOVQ 20(R4), V9.W4 + VMOVQ 24(R4), V10.W4 + VMOVQ 28(R4), V11.W4 + + // load counter and update counter + VMOVQ R6, V12.W4 + VADDW V12, V30, V12 + + // zeros for remaining three matrix entries + VXORV V13, V13, V13 + VXORV V14, V14, V14 + VXORV V15, V15, V15 + + // save seed state for adding back later + VORV V4, V13, V20 + VORV V5, V13, V21 + VORV V6, V13, V22 + VORV V7, V13, V23 + VORV V8, V13, V24 + VORV V9, V13, V25 + VORV V10, V13, V26 + VORV V11, V13, V27 + + // 4 iterations. Each iteration is 8 quarter-rounds. + MOVV $4, R7 +loop: + QR(V0, V4, V8, V12) + QR(V1, V5, V9, V13) + QR(V2, V6, V10, V14) + QR(V3, V7, V11, V15) + + QR(V0, V5, V10, V15) + QR(V1, V6, V11, V12) + QR(V2, V7, V8, V13) + QR(V3, V4, V9, V14) + + SUBV $1, R7 + BNE R7, R0, loop + + // add seed back + VADDW V4, V20, V4 + VADDW V5, V21, V5 + VADDW V6, V22, V6 + VADDW V7, V23, V7 + VADDW V8, V24, V8 + VADDW V9, V25, V9 + VADDW V10, V26, V10 + VADDW V11, V27, V11 + + // store blocks back to output buffer + VMOVQ V0, (R5) + VMOVQ V1, 16(R5) + VMOVQ V2, 32(R5) + VMOVQ V3, 48(R5) + VMOVQ V4, 64(R5) + VMOVQ V5, 80(R5) + VMOVQ V6, 96(R5) + VMOVQ V7, 112(R5) + VMOVQ V8, 128(R5) + VMOVQ V9, 144(R5) + VMOVQ V10, 160(R5) + VMOVQ V11, 176(R5) + VMOVQ V12, 192(R5) + VMOVQ V13, 208(R5) + VMOVQ V14, 224(R5) + VMOVQ V15, 240(R5) + + RET diff --git a/src/internal/chacha8rand/chacha8_riscv64.s b/src/internal/chacha8rand/chacha8_riscv64.s new file mode 100644 index 00000000000000..5514bacc6cba13 --- /dev/null +++ b/src/internal/chacha8rand/chacha8_riscv64.s @@ -0,0 +1,113 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "asm_riscv64.h" +#include "go_asm.h" +#include "textflag.h" + +// TODO(mzh): use Zvkb if possible + +#define QR(A, B, C, D) \ + VADDVV A, B, A \ + VXORVV D, A, D \ + VSLLVI $16, D, V28 \ + VSRLVI $16, D, D \ + VXORVV V28, D, D \ + VADDVV D, C, C \ + VXORVV C, B, B \ + VSLLVI $12, B, V29 \ + VSRLVI $20, B, B \ + VXORVV V29, B, B \ + VADDVV B, A, A \ + VXORVV A, D, D \ + VSLLVI $8, D, V30 \ + VSRLVI $24, D, D \ + VXORVV V30, D, D \ + VADDVV D, C, C \ + VXORVV C, B, B \ + VSLLVI $7, B, V31 \ + VSRLVI $25, B, B \ + VXORVV V31, B, B + +// block runs four ChaCha8 block transformations using four elements in each V register. +// func block(seed *[8]uint32, blocks *[16][4]uint32, counter uint32) +TEXT ·block(SB), NOSPLIT, $0 + // seed in X10 + // blocks in X11 + // counter in X12 + +#ifndef hasV + MOVB internal∕cpu·RISCV64+const_offsetRISCV64HasV(SB), X13 + BNEZ X13, vector_chacha8 + JMP ·block_generic(SB) +#endif + +vector_chacha8: + // At least VLEN >= 128 + VSETIVLI $4, E32, M1, TA, MA, X0 + // Load initial constants into top row. + MOV $·chachaConst(SB), X14 + VLSSEG4E32V (X14), X0, V0 // V0, V1, V2, V3 = const row + VLSSEG8E32V (X10), X0, V4 // V4 ... V11, seed + VIDV V12 + VADDVX X12, V12, V12 // counter + + // Clear all nonces. + VXORVV V13, V13, V13 + VXORVV V14, V14, V14 + VXORVV V15, V15, V15 + + // Copy initial state. + VMV4RV V4, V20 + VMV4RV V8, V24 + + MOV $4, X15 + PCALIGN $16 +loop: + QR(V0, V4, V8, V12) + QR(V1, V5, V9, V13) + QR(V2, V6, V10, V14) + QR(V3, V7, V11, V15) + + QR(V0, V5, V10, V15) + QR(V1, V6, V11, V12) + QR(V2, V7, V8, V13) + QR(V3, V4, V9, V14) + + SUB $1, X15 + BNEZ X15, loop + + VADDVV V20, V4, V4 + VADDVV V21, V5, V5 + VADDVV V22, V6, V6 + VADDVV V23, V7, V7 + VADDVV V24, V8, V8 + VADDVV V25, V9, V9 + VADDVV V26, V10, V10 + VADDVV V27, V11, V11 + + VSE32V V0, (X11); ADD $16, X11; + VSE32V V1, (X11); ADD $16, X11; + VSE32V V2, (X11); ADD $16, X11; + VSE32V V3, (X11); ADD $16, X11; + VSE32V V4, (X11); ADD $16, X11; + VSE32V V5, (X11); ADD $16, X11; + VSE32V V6, (X11); ADD $16, X11; + VSE32V V7, (X11); ADD $16, X11; + VSE32V V8, (X11); ADD $16, X11; + VSE32V V9, (X11); ADD $16, X11; + VSE32V V10, (X11); ADD $16, X11; + VSE32V V11, (X11); ADD $16, X11; + VSE32V V12, (X11); ADD $16, X11; + VSE32V V13, (X11); ADD $16, X11; + VSE32V V14, (X11); ADD $16, X11; + VSE32V V15, (X11); ADD $16, X11; + + RET + +GLOBL ·chachaConst(SB), NOPTR|RODATA, $32 +DATA ·chachaConst+0x00(SB)/4, $0x61707865 +DATA ·chachaConst+0x04(SB)/4, $0x3320646e +DATA ·chachaConst+0x08(SB)/4, $0x79622d32 +DATA ·chachaConst+0x0c(SB)/4, $0x6b206574 diff --git a/src/internal/chacha8rand/chacha8_stub.s b/src/internal/chacha8rand/chacha8_stub.s index 09be558fcb5d64..64245e28af956b 100644 --- a/src/internal/chacha8rand/chacha8_stub.s +++ b/src/internal/chacha8rand/chacha8_stub.s @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !amd64 && !arm64 +//go:build !amd64 && !arm64 && !loong64 && !riscv64 #include "textflag.h" diff --git a/src/internal/concurrent/hashtriemap.go b/src/internal/concurrent/hashtriemap.go deleted file mode 100644 index 4f7e730d4fca2c..00000000000000 --- a/src/internal/concurrent/hashtriemap.go +++ /dev/null @@ -1,408 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package concurrent - -import ( - "internal/abi" - "internal/goarch" - "math/rand/v2" - "sync" - "sync/atomic" - "unsafe" -) - -// HashTrieMap is an implementation of a concurrent hash-trie. The implementation -// is designed around frequent loads, but offers decent performance for stores -// and deletes as well, especially if the map is larger. It's primary use-case is -// the unique package, but can be used elsewhere as well. -type HashTrieMap[K, V comparable] struct { - root *indirect[K, V] - keyHash hashFunc - keyEqual equalFunc - valEqual equalFunc - seed uintptr -} - -// NewHashTrieMap creates a new HashTrieMap for the provided key and value. -func NewHashTrieMap[K, V comparable]() *HashTrieMap[K, V] { - var m map[K]V - mapType := abi.TypeOf(m).MapType() - ht := &HashTrieMap[K, V]{ - root: newIndirectNode[K, V](nil), - keyHash: mapType.Hasher, - keyEqual: mapType.Key.Equal, - valEqual: mapType.Elem.Equal, - seed: uintptr(rand.Uint64()), - } - return ht -} - -type hashFunc func(unsafe.Pointer, uintptr) uintptr -type equalFunc func(unsafe.Pointer, unsafe.Pointer) bool - -// Load returns the value stored in the map for a key, or nil if no -// value is present. -// The ok result indicates whether value was found in the map. -func (ht *HashTrieMap[K, V]) Load(key K) (value V, ok bool) { - hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) - - i := ht.root - hashShift := 8 * goarch.PtrSize - for hashShift != 0 { - hashShift -= nChildrenLog2 - - n := i.children[(hash>>hashShift)&nChildrenMask].Load() - if n == nil { - return *new(V), false - } - if n.isEntry { - return n.entry().lookup(key, ht.keyEqual) - } - i = n.indirect() - } - panic("internal/concurrent.HashMapTrie: ran out of hash bits while iterating") -} - -// LoadOrStore returns the existing value for the key if present. -// Otherwise, it stores and returns the given value. -// The loaded result is true if the value was loaded, false if stored. -func (ht *HashTrieMap[K, V]) LoadOrStore(key K, value V) (result V, loaded bool) { - hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) - var i *indirect[K, V] - var hashShift uint - var slot *atomic.Pointer[node[K, V]] - var n *node[K, V] - for { - // Find the key or a candidate location for insertion. - i = ht.root - hashShift = 8 * goarch.PtrSize - haveInsertPoint := false - for hashShift != 0 { - hashShift -= nChildrenLog2 - - slot = &i.children[(hash>>hashShift)&nChildrenMask] - n = slot.Load() - if n == nil { - // We found a nil slot which is a candidate for insertion. - haveInsertPoint = true - break - } - if n.isEntry { - // We found an existing entry, which is as far as we can go. - // If it stays this way, we'll have to replace it with an - // indirect node. - if v, ok := n.entry().lookup(key, ht.keyEqual); ok { - return v, true - } - haveInsertPoint = true - break - } - i = n.indirect() - } - if !haveInsertPoint { - panic("internal/concurrent.HashMapTrie: ran out of hash bits while iterating") - } - - // Grab the lock and double-check what we saw. - i.mu.Lock() - n = slot.Load() - if (n == nil || n.isEntry) && !i.dead.Load() { - // What we saw is still true, so we can continue with the insert. - break - } - // We have to start over. - i.mu.Unlock() - } - // N.B. This lock is held from when we broke out of the outer loop above. - // We specifically break this out so that we can use defer here safely. - // One option is to break this out into a new function instead, but - // there's so much local iteration state used below that this turns out - // to be cleaner. - defer i.mu.Unlock() - - var oldEntry *entry[K, V] - if n != nil { - oldEntry = n.entry() - if v, ok := oldEntry.lookup(key, ht.keyEqual); ok { - // Easy case: by loading again, it turns out exactly what we wanted is here! - return v, true - } - } - newEntry := newEntryNode(key, value) - if oldEntry == nil { - // Easy case: create a new entry and store it. - slot.Store(&newEntry.node) - } else { - // We possibly need to expand the entry already there into one or more new nodes. - // - // Publish the node last, which will make both oldEntry and newEntry visible. We - // don't want readers to be able to observe that oldEntry isn't in the tree. - slot.Store(ht.expand(oldEntry, newEntry, hash, hashShift, i)) - } - return value, false -} - -// expand takes oldEntry and newEntry whose hashes conflict from bit 64 down to hashShift and -// produces a subtree of indirect nodes to hold the two new entries. -func (ht *HashTrieMap[K, V]) expand(oldEntry, newEntry *entry[K, V], newHash uintptr, hashShift uint, parent *indirect[K, V]) *node[K, V] { - // Check for a hash collision. - oldHash := ht.keyHash(unsafe.Pointer(&oldEntry.key), ht.seed) - if oldHash == newHash { - // Store the old entry in the new entry's overflow list, then store - // the new entry. - newEntry.overflow.Store(oldEntry) - return &newEntry.node - } - // We have to add an indirect node. Worse still, we may need to add more than one. - newIndirect := newIndirectNode(parent) - top := newIndirect - for { - if hashShift == 0 { - panic("internal/concurrent.HashMapTrie: ran out of hash bits while inserting") - } - hashShift -= nChildrenLog2 // hashShift is for the level parent is at. We need to go deeper. - oi := (oldHash >> hashShift) & nChildrenMask - ni := (newHash >> hashShift) & nChildrenMask - if oi != ni { - newIndirect.children[oi].Store(&oldEntry.node) - newIndirect.children[ni].Store(&newEntry.node) - break - } - nextIndirect := newIndirectNode(newIndirect) - newIndirect.children[oi].Store(&nextIndirect.node) - newIndirect = nextIndirect - } - return &top.node -} - -// CompareAndDelete deletes the entry for key if its value is equal to old. -// -// If there is no current value for key in the map, CompareAndDelete returns false -// (even if the old value is the nil interface value). -func (ht *HashTrieMap[K, V]) CompareAndDelete(key K, old V) (deleted bool) { - hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) - var i *indirect[K, V] - var hashShift uint - var slot *atomic.Pointer[node[K, V]] - var n *node[K, V] - for { - // Find the key or return when there's nothing to delete. - i = ht.root - hashShift = 8 * goarch.PtrSize - found := false - for hashShift != 0 { - hashShift -= nChildrenLog2 - - slot = &i.children[(hash>>hashShift)&nChildrenMask] - n = slot.Load() - if n == nil { - // Nothing to delete. Give up. - return - } - if n.isEntry { - // We found an entry. Check if it matches. - if _, ok := n.entry().lookup(key, ht.keyEqual); !ok { - // No match, nothing to delete. - return - } - // We've got something to delete. - found = true - break - } - i = n.indirect() - } - if !found { - panic("internal/concurrent.HashMapTrie: ran out of hash bits while iterating") - } - - // Grab the lock and double-check what we saw. - i.mu.Lock() - n = slot.Load() - if !i.dead.Load() { - if n == nil { - // Valid node that doesn't contain what we need. Nothing to delete. - i.mu.Unlock() - return - } - if n.isEntry { - // What we saw is still true, so we can continue with the delete. - break - } - } - // We have to start over. - i.mu.Unlock() - } - // Try to delete the entry. - e, deleted := n.entry().compareAndDelete(key, old, ht.keyEqual, ht.valEqual) - if !deleted { - // Nothing was actually deleted, which means the node is no longer there. - i.mu.Unlock() - return false - } - if e != nil { - // We didn't actually delete the whole entry, just one entry in the chain. - // Nothing else to do, since the parent is definitely not empty. - slot.Store(&e.node) - i.mu.Unlock() - return true - } - // Delete the entry. - slot.Store(nil) - - // Check if the node is now empty (and isn't the root), and delete it if able. - for i.parent != nil && i.empty() { - if hashShift == 8*goarch.PtrSize { - panic("internal/concurrent.HashMapTrie: ran out of hash bits while iterating") - } - hashShift += nChildrenLog2 - - // Delete the current node in the parent. - parent := i.parent - parent.mu.Lock() - i.dead.Store(true) - parent.children[(hash>>hashShift)&nChildrenMask].Store(nil) - i.mu.Unlock() - i = parent - } - i.mu.Unlock() - return true -} - -// All returns an iter.Seq2 that produces all key-value pairs in the map. -// The enumeration does not represent any consistent snapshot of the map, -// but is guaranteed to visit each unique key-value pair only once. It is -// safe to operate on the tree during iteration. No particular enumeration -// order is guaranteed. -func (ht *HashTrieMap[K, V]) All() func(yield func(K, V) bool) { - return func(yield func(key K, value V) bool) { - ht.iter(ht.root, yield) - } -} - -func (ht *HashTrieMap[K, V]) iter(i *indirect[K, V], yield func(key K, value V) bool) bool { - for j := range i.children { - n := i.children[j].Load() - if n == nil { - continue - } - if !n.isEntry { - if !ht.iter(n.indirect(), yield) { - return false - } - continue - } - e := n.entry() - for e != nil { - if !yield(e.key, e.value) { - return false - } - e = e.overflow.Load() - } - } - return true -} - -const ( - // 16 children. This seems to be the sweet spot for - // load performance: any smaller and we lose out on - // 50% or more in CPU performance. Any larger and the - // returns are minuscule (~1% improvement for 32 children). - nChildrenLog2 = 4 - nChildren = 1 << nChildrenLog2 - nChildrenMask = nChildren - 1 -) - -// indirect is an internal node in the hash-trie. -type indirect[K, V comparable] struct { - node[K, V] - dead atomic.Bool - mu sync.Mutex // Protects mutation to children and any children that are entry nodes. - parent *indirect[K, V] - children [nChildren]atomic.Pointer[node[K, V]] -} - -func newIndirectNode[K, V comparable](parent *indirect[K, V]) *indirect[K, V] { - return &indirect[K, V]{node: node[K, V]{isEntry: false}, parent: parent} -} - -func (i *indirect[K, V]) empty() bool { - nc := 0 - for j := range i.children { - if i.children[j].Load() != nil { - nc++ - } - } - return nc == 0 -} - -// entry is a leaf node in the hash-trie. -type entry[K, V comparable] struct { - node[K, V] - overflow atomic.Pointer[entry[K, V]] // Overflow for hash collisions. - key K - value V -} - -func newEntryNode[K, V comparable](key K, value V) *entry[K, V] { - return &entry[K, V]{ - node: node[K, V]{isEntry: true}, - key: key, - value: value, - } -} - -func (e *entry[K, V]) lookup(key K, equal equalFunc) (V, bool) { - for e != nil { - if equal(unsafe.Pointer(&e.key), abi.NoEscape(unsafe.Pointer(&key))) { - return e.value, true - } - e = e.overflow.Load() - } - return *new(V), false -} - -// compareAndDelete deletes an entry in the overflow chain if both the key and value compare -// equal. Returns the new entry chain and whether or not anything was deleted. -// -// compareAndDelete must be called under the mutex of the indirect node which e is a child of. -func (head *entry[K, V]) compareAndDelete(key K, value V, keyEqual, valEqual equalFunc) (*entry[K, V], bool) { - if keyEqual(unsafe.Pointer(&head.key), abi.NoEscape(unsafe.Pointer(&key))) && - valEqual(unsafe.Pointer(&head.value), abi.NoEscape(unsafe.Pointer(&value))) { - // Drop the head of the list. - return head.overflow.Load(), true - } - i := &head.overflow - e := i.Load() - for e != nil { - if keyEqual(unsafe.Pointer(&e.key), abi.NoEscape(unsafe.Pointer(&key))) && - valEqual(unsafe.Pointer(&e.value), abi.NoEscape(unsafe.Pointer(&value))) { - i.Store(e.overflow.Load()) - return head, true - } - i = &e.overflow - e = e.overflow.Load() - } - return head, false -} - -// node is the header for a node. It's polymorphic and -// is actually either an entry or an indirect. -type node[K, V comparable] struct { - isEntry bool -} - -func (n *node[K, V]) entry() *entry[K, V] { - if !n.isEntry { - panic("called entry on non-entry node") - } - return (*entry[K, V])(unsafe.Pointer(n)) -} - -func (n *node[K, V]) indirect() *indirect[K, V] { - if n.isEntry { - panic("called indirect on entry node") - } - return (*indirect[K, V])(unsafe.Pointer(n)) -} diff --git a/src/internal/concurrent/hashtriemap_test.go b/src/internal/concurrent/hashtriemap_test.go deleted file mode 100644 index e233824c0f08fe..00000000000000 --- a/src/internal/concurrent/hashtriemap_test.go +++ /dev/null @@ -1,371 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package concurrent - -import ( - "fmt" - "math" - "runtime" - "strconv" - "strings" - "sync" - "testing" - "unsafe" -) - -func TestHashTrieMap(t *testing.T) { - testHashTrieMap(t, func() *HashTrieMap[string, int] { - return NewHashTrieMap[string, int]() - }) -} - -func TestHashTrieMapBadHash(t *testing.T) { - testHashTrieMap(t, func() *HashTrieMap[string, int] { - // Stub out the good hash function with a terrible one. - // Everything should still work as expected. - m := NewHashTrieMap[string, int]() - m.keyHash = func(_ unsafe.Pointer, _ uintptr) uintptr { - return 0 - } - return m - }) -} - -func testHashTrieMap(t *testing.T, newMap func() *HashTrieMap[string, int]) { - t.Run("LoadEmpty", func(t *testing.T) { - m := newMap() - - for _, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - } - }) - t.Run("LoadOrStore", func(t *testing.T) { - m := newMap() - - for i, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - expectStored(t, s, i)(m.LoadOrStore(s, i)) - expectPresent(t, s, i)(m.Load(s)) - expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) - } - for i, s := range testData { - expectPresent(t, s, i)(m.Load(s)) - expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) - } - }) - t.Run("CompareAndDeleteAll", func(t *testing.T) { - m := newMap() - - for range 3 { - for i, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - expectStored(t, s, i)(m.LoadOrStore(s, i)) - expectPresent(t, s, i)(m.Load(s)) - expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) - } - for i, s := range testData { - expectPresent(t, s, i)(m.Load(s)) - expectNotDeleted(t, s, math.MaxInt)(m.CompareAndDelete(s, math.MaxInt)) - expectDeleted(t, s, i)(m.CompareAndDelete(s, i)) - expectNotDeleted(t, s, i)(m.CompareAndDelete(s, i)) - expectMissing(t, s, 0)(m.Load(s)) - } - for _, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - } - } - }) - t.Run("CompareAndDeleteOne", func(t *testing.T) { - m := newMap() - - for i, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - expectStored(t, s, i)(m.LoadOrStore(s, i)) - expectPresent(t, s, i)(m.Load(s)) - expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) - } - expectNotDeleted(t, testData[15], math.MaxInt)(m.CompareAndDelete(testData[15], math.MaxInt)) - expectDeleted(t, testData[15], 15)(m.CompareAndDelete(testData[15], 15)) - expectNotDeleted(t, testData[15], 15)(m.CompareAndDelete(testData[15], 15)) - for i, s := range testData { - if i == 15 { - expectMissing(t, s, 0)(m.Load(s)) - } else { - expectPresent(t, s, i)(m.Load(s)) - } - } - }) - t.Run("DeleteMultiple", func(t *testing.T) { - m := newMap() - - for i, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - expectStored(t, s, i)(m.LoadOrStore(s, i)) - expectPresent(t, s, i)(m.Load(s)) - expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) - } - for _, i := range []int{1, 105, 6, 85} { - expectNotDeleted(t, testData[i], math.MaxInt)(m.CompareAndDelete(testData[i], math.MaxInt)) - expectDeleted(t, testData[i], i)(m.CompareAndDelete(testData[i], i)) - expectNotDeleted(t, testData[i], i)(m.CompareAndDelete(testData[i], i)) - } - for i, s := range testData { - if i == 1 || i == 105 || i == 6 || i == 85 { - expectMissing(t, s, 0)(m.Load(s)) - } else { - expectPresent(t, s, i)(m.Load(s)) - } - } - }) - t.Run("All", func(t *testing.T) { - m := newMap() - - testAll(t, m, testDataMap(testData[:]), func(_ string, _ int) bool { - return true - }) - }) - t.Run("AllDelete", func(t *testing.T) { - m := newMap() - - testAll(t, m, testDataMap(testData[:]), func(s string, i int) bool { - expectDeleted(t, s, i)(m.CompareAndDelete(s, i)) - return true - }) - for _, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - } - }) - t.Run("ConcurrentLifecycleUnsharedKeys", func(t *testing.T) { - m := newMap() - - gmp := runtime.GOMAXPROCS(-1) - var wg sync.WaitGroup - for i := range gmp { - wg.Add(1) - go func(id int) { - defer wg.Done() - - makeKey := func(s string) string { - return s + "-" + strconv.Itoa(id) - } - for _, s := range testData { - key := makeKey(s) - expectMissing(t, key, 0)(m.Load(key)) - expectStored(t, key, id)(m.LoadOrStore(key, id)) - expectPresent(t, key, id)(m.Load(key)) - expectLoaded(t, key, id)(m.LoadOrStore(key, 0)) - } - for _, s := range testData { - key := makeKey(s) - expectPresent(t, key, id)(m.Load(key)) - expectDeleted(t, key, id)(m.CompareAndDelete(key, id)) - expectMissing(t, key, 0)(m.Load(key)) - } - for _, s := range testData { - key := makeKey(s) - expectMissing(t, key, 0)(m.Load(key)) - } - }(i) - } - wg.Wait() - }) - t.Run("ConcurrentDeleteSharedKeys", func(t *testing.T) { - m := newMap() - - // Load up the map. - for i, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - expectStored(t, s, i)(m.LoadOrStore(s, i)) - } - gmp := runtime.GOMAXPROCS(-1) - var wg sync.WaitGroup - for i := range gmp { - wg.Add(1) - go func(id int) { - defer wg.Done() - - for i, s := range testData { - expectNotDeleted(t, s, math.MaxInt)(m.CompareAndDelete(s, math.MaxInt)) - m.CompareAndDelete(s, i) - expectMissing(t, s, 0)(m.Load(s)) - } - for _, s := range testData { - expectMissing(t, s, 0)(m.Load(s)) - } - }(i) - } - wg.Wait() - }) -} - -func testAll[K, V comparable](t *testing.T, m *HashTrieMap[K, V], testData map[K]V, yield func(K, V) bool) { - for k, v := range testData { - expectStored(t, k, v)(m.LoadOrStore(k, v)) - } - visited := make(map[K]int) - m.All()(func(key K, got V) bool { - want, ok := testData[key] - if !ok { - t.Errorf("unexpected key %v in map", key) - return false - } - if got != want { - t.Errorf("expected key %v to have value %v, got %v", key, want, got) - return false - } - visited[key]++ - return yield(key, got) - }) - for key, n := range visited { - if n > 1 { - t.Errorf("visited key %v more than once", key) - } - } -} - -func expectPresent[K, V comparable](t *testing.T, key K, want V) func(got V, ok bool) { - t.Helper() - return func(got V, ok bool) { - t.Helper() - - if !ok { - t.Errorf("expected key %v to be present in map", key) - } - if ok && got != want { - t.Errorf("expected key %v to have value %v, got %v", key, want, got) - } - } -} - -func expectMissing[K, V comparable](t *testing.T, key K, want V) func(got V, ok bool) { - t.Helper() - if want != *new(V) { - // This is awkward, but the want argument is necessary to smooth over type inference. - // Just make sure the want argument always looks the same. - panic("expectMissing must always have a zero value variable") - } - return func(got V, ok bool) { - t.Helper() - - if ok { - t.Errorf("expected key %v to be missing from map, got value %v", key, got) - } - if !ok && got != want { - t.Errorf("expected missing key %v to be paired with the zero value; got %v", key, got) - } - } -} - -func expectLoaded[K, V comparable](t *testing.T, key K, want V) func(got V, loaded bool) { - t.Helper() - return func(got V, loaded bool) { - t.Helper() - - if !loaded { - t.Errorf("expected key %v to have been loaded, not stored", key) - } - if got != want { - t.Errorf("expected key %v to have value %v, got %v", key, want, got) - } - } -} - -func expectStored[K, V comparable](t *testing.T, key K, want V) func(got V, loaded bool) { - t.Helper() - return func(got V, loaded bool) { - t.Helper() - - if loaded { - t.Errorf("expected inserted key %v to have been stored, not loaded", key) - } - if got != want { - t.Errorf("expected inserted key %v to have value %v, got %v", key, want, got) - } - } -} - -func expectDeleted[K, V comparable](t *testing.T, key K, old V) func(deleted bool) { - t.Helper() - return func(deleted bool) { - t.Helper() - - if !deleted { - t.Errorf("expected key %v with value %v to be in map and deleted", key, old) - } - } -} - -func expectNotDeleted[K, V comparable](t *testing.T, key K, old V) func(deleted bool) { - t.Helper() - return func(deleted bool) { - t.Helper() - - if deleted { - t.Errorf("expected key %v with value %v to not be in map and thus not deleted", key, old) - } - } -} - -func testDataMap(data []string) map[string]int { - m := make(map[string]int) - for i, s := range data { - m[s] = i - } - return m -} - -var ( - testDataSmall [8]string - testData [128]string - testDataLarge [128 << 10]string -) - -func init() { - for i := range testDataSmall { - testDataSmall[i] = fmt.Sprintf("%b", i) - } - for i := range testData { - testData[i] = fmt.Sprintf("%b", i) - } - for i := range testDataLarge { - testDataLarge[i] = fmt.Sprintf("%b", i) - } -} - -func dumpMap[K, V comparable](ht *HashTrieMap[K, V]) { - dumpNode(ht, &ht.root.node, 0) -} - -func dumpNode[K, V comparable](ht *HashTrieMap[K, V], n *node[K, V], depth int) { - var sb strings.Builder - for range depth { - fmt.Fprintf(&sb, "\t") - } - prefix := sb.String() - if n.isEntry { - e := n.entry() - for e != nil { - fmt.Printf("%s%p [Entry Key=%v Value=%v Overflow=%p, Hash=%016x]\n", prefix, e, e.key, e.value, e.overflow.Load(), ht.keyHash(unsafe.Pointer(&e.key), ht.seed)) - e = e.overflow.Load() - } - return - } - i := n.indirect() - fmt.Printf("%s%p [Indirect Parent=%p Dead=%t Children=[", prefix, i, i.parent, i.dead.Load()) - for j := range i.children { - c := i.children[j].Load() - fmt.Printf("%p", c) - if j != len(i.children)-1 { - fmt.Printf(", ") - } - } - fmt.Printf("]]\n") - for j := range i.children { - c := i.children[j].Load() - if c != nil { - dumpNode(ht, c, depth+1) - } - } -} diff --git a/src/internal/copyright/copyright_test.go b/src/internal/copyright/copyright_test.go new file mode 100644 index 00000000000000..11b6a9b22940b6 --- /dev/null +++ b/src/internal/copyright/copyright_test.go @@ -0,0 +1,72 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package copyright + +import ( + "bytes" + "internal/testenv" + "io" + "io/fs" + "os" + "path/filepath" + "testing" +) + +var copyright = []byte("Copyright") + +var permitted = [][]byte{ + []byte("// Code generated by "), + []byte("// Code generated from "), + []byte("// Created by cgo -cdefs"), + []byte("// DO NOT EDIT\n// generated by:"), + []byte("// Empty assembly file"), + []byte("// Generated using cgo"), + []byte("// Original source:\n//\thttp://www.zorinaq.com/papers/md5-amd64.html"), // public domain crypto/md5 + []byte("// created by cgo -cdefs"), + []byte("// go run mkasm.go"), + []byte("// mkerrors"), + []byte("// mksys"), + []byte("// run\n// Code generated by"), // cmd/compile/internal/test/constFold_test.go + []byte("//go:build !nethttpomithttp2"), // hack to pacify h2_bundle, which drops copyright header +} + +func TestCopyright(t *testing.T) { + buf := make([]byte, 2048) + filepath.WalkDir(filepath.Join(testenv.GOROOT(t), "src"), func(path string, d fs.DirEntry, err error) error { + if d.IsDir() && (d.Name() == "testdata" || d.Name() == "vendor") { + return filepath.SkipDir + } + switch filepath.Ext(d.Name()) { + default: + return nil + case ".s", ".go": + // check + } + + f, err := os.Open(path) + if err != nil { + t.Error(err) + return nil + } + defer f.Close() + n, err := f.Read(buf) + if err != nil && err != io.EOF { + t.Error(err) + return nil + } + b := buf[:n] + if bytes.Contains(b, copyright) { + return nil + } + for _, ok := range permitted { + if bytes.HasPrefix(b, ok) { + return nil + } + } + t.Logf("%.100s %s", b, path) + t.Errorf("%s: missing copyright notice", path) + return nil + }) +} diff --git a/src/internal/coverage/cfile/emit.go b/src/internal/coverage/cfile/emit.go index 3993e9cb4247e9..47de7778b0b24f 100644 --- a/src/internal/coverage/cfile/emit.go +++ b/src/internal/coverage/cfile/emit.go @@ -9,8 +9,8 @@ package cfile import ( - "crypto/md5" "fmt" + "hash/fnv" "internal/coverage" "internal/coverage/encodecounter" "internal/coverage/encodemeta" @@ -206,7 +206,7 @@ func prepareForMetaEmit() ([]rtcov.CovMetaBlob, error) { } } - h := md5.New() + h := fnv.New128a() tlen := uint64(unsafe.Sizeof(coverage.MetaFileHeader{})) for _, entry := range ml { if _, err := h.Write(entry.Hash[:]); err != nil { diff --git a/src/internal/coverage/cfile/emitdata_test.go b/src/internal/coverage/cfile/emitdata_test.go index c522048173e4c7..d127c6b2a976ba 100644 --- a/src/internal/coverage/cfile/emitdata_test.go +++ b/src/internal/coverage/cfile/emitdata_test.go @@ -7,7 +7,6 @@ package cfile import ( "fmt" "internal/coverage" - "internal/goexperiment" "internal/platform" "internal/testenv" "os" @@ -25,9 +24,6 @@ func TestCoverageApis(t *testing.T) { if testing.Short() { t.Skipf("skipping test: too long for short mode") } - if !goexperiment.CoverageRedesign { - t.Skipf("skipping new coverage tests (experiment not enabled)") - } testenv.MustHaveGoBuild(t) dir := t.TempDir() if fixedTestDir { @@ -465,9 +461,6 @@ func TestIssue56006EmitDataRaceCoverRunningGoroutine(t *testing.T) { if testing.Short() { t.Skipf("skipping test: too long for short mode") } - if !goexperiment.CoverageRedesign { - t.Skipf("skipping new coverage tests (experiment not enabled)") - } // This test requires "go test -race -cover", meaning that we need // go build, go run, and "-race" support. diff --git a/src/internal/coverage/cfile/testsupport.go b/src/internal/coverage/cfile/testsupport.go index 3594b32aee1a8d..adab47fd212d1e 100644 --- a/src/internal/coverage/cfile/testsupport.go +++ b/src/internal/coverage/cfile/testsupport.go @@ -109,7 +109,7 @@ func ProcessCoverTestDir(dir string, cfile string, cm string, cpkg string, w io. // Emit text output. if tf != nil { - if err := ts.cf.EmitTextual(tf); err != nil { + if err := ts.cf.EmitTextual(selpkgs, tf); err != nil { return err } tfClosed = true diff --git a/src/internal/coverage/cfile/ts_test.go b/src/internal/coverage/cfile/ts_test.go index d3441821a436d3..d106d3b132ce0a 100644 --- a/src/internal/coverage/cfile/ts_test.go +++ b/src/internal/coverage/cfile/ts_test.go @@ -8,7 +8,6 @@ import ( "encoding/json" "flag" "internal/coverage" - "internal/goexperiment" "internal/testenv" "os" "os/exec" @@ -32,9 +31,6 @@ func testGoCoverDir(t *testing.T) string { // relying on other test paths will provide a better signal when // running "go test -cover" for this package). func TestTestSupport(t *testing.T) { - if !goexperiment.CoverageRedesign { - return - } if testing.CoverMode() == "" { return } @@ -128,9 +124,6 @@ func genAuxMeta(t *testing.T, dstdir string) (string, string) { } func TestAuxMetaDataFiles(t *testing.T) { - if !goexperiment.CoverageRedesign { - return - } if testing.CoverMode() == "" { return } diff --git a/src/internal/coverage/cformat/fmt_test.go b/src/internal/coverage/cformat/fmt_test.go index d296939d5cd20c..a26de964c43538 100644 --- a/src/internal/coverage/cformat/fmt_test.go +++ b/src/internal/coverage/cformat/fmt_test.go @@ -47,8 +47,8 @@ func TestBasics(t *testing.T) { fm.AddUnit("lit.go", "f3", true, u, 0) } - var b1, b2, b3, b4 strings.Builder - if err := fm.EmitTextual(&b1); err != nil { + var b1, b2, b3, b4, b5 strings.Builder + if err := fm.EmitTextual(nil, &b1); err != nil { t.Fatalf("EmitTextual returned %v", err) } wantText := strings.TrimSpace(` @@ -64,6 +64,18 @@ lit.go:99.0,100.0 1 0`) t.Errorf("emit text: got:\n%s\nwant:\n%s\n", gotText, wantText) } + selected := []string{"my/pack2"} + if err := fm.EmitTextual(selected, &b5); err != nil { + t.Fatalf("EmitTextual returned %v", err) + } + wantText = strings.TrimSpace(` +mode: atomic +lit.go:99.0,100.0 1 0`) + gotText = strings.TrimSpace(b5.String()) + if wantText != gotText { + t.Errorf("emit text: got:\n%s\nwant:\n%s\n", gotText, wantText) + } + // Percent output with no aggregation. noCoverPkg := "" if err := fm.EmitPercent(&b2, nil, noCoverPkg, false, false); err != nil { diff --git a/src/internal/coverage/cformat/format.go b/src/internal/coverage/cformat/format.go index 0f9e981df8b12d..01d3109e31fd9d 100644 --- a/src/internal/coverage/cformat/format.go +++ b/src/internal/coverage/cformat/format.go @@ -24,7 +24,7 @@ package cformat // } // } // myformatter.EmitPercent(os.Stdout, nil, "", true, true) -// myformatter.EmitTextual(somefile) +// myformatter.EmitTextual(nil, somefile) // // These apis are linked into tests that are built with "-cover", and // called at the end of test execution to produce text output or @@ -36,7 +36,9 @@ import ( "internal/coverage" "internal/coverage/cmerge" "io" + "maps" "slices" + "sort" "strings" "text/tabwriter" ) @@ -162,25 +164,31 @@ func (p *pstate) sortUnits(units []extcu) { }) } -// EmitTextual writes the accumulated coverage data in the legacy -// cmd/cover text format to the writer 'w'. We sort the data items by +// EmitTextual writes the accumulated coverage data for 'pkgs' in the legacy +// cmd/cover text format to the writer 'w'; if pkgs is empty, text output +// is emitted for all packages recorded. We sort the data items by // importpath, source file, and line number before emitting (this sorting // is not explicitly mandated by the format, but seems like a good idea // for repeatable/deterministic dumps). -func (fm *Formatter) EmitTextual(w io.Writer) error { +func (fm *Formatter) EmitTextual(pkgs []string, w io.Writer) error { if fm.cm == coverage.CtrModeInvalid { panic("internal error, counter mode unset") } + if len(pkgs) == 0 { + pkgs = make([]string, 0, len(fm.pm)) + for importpath := range fm.pm { + pkgs = append(pkgs, importpath) + } + } if _, err := fmt.Fprintf(w, "mode: %s\n", fm.cm.String()); err != nil { return err } - pkgs := make([]string, 0, len(fm.pm)) - for importpath := range fm.pm { - pkgs = append(pkgs, importpath) - } - slices.Sort(pkgs) + sort.Strings(pkgs) for _, importpath := range pkgs { p := fm.pm[importpath] + if p == nil { + continue + } units := make([]extcu, 0, len(p.unitTable)) for u := range p.unitTable { units = append(units, u) @@ -281,14 +289,8 @@ func (fm *Formatter) EmitFuncs(w io.Writer) error { allStmts := uint64(0) covStmts := uint64(0) - pkgs := make([]string, 0, len(fm.pm)) - for importpath := range fm.pm { - pkgs = append(pkgs, importpath) - } - slices.Sort(pkgs) - // Emit functions for each package, sorted by import path. - for _, importpath := range pkgs { + for _, importpath := range slices.Sorted(maps.Keys(fm.pm)) { p := fm.pm[importpath] if len(p.unitTable) == 0 { continue diff --git a/src/internal/coverage/cmerge/merge.go b/src/internal/coverage/cmerge/merge.go index 1339803d0865c6..9fc548eced5091 100644 --- a/src/internal/coverage/cmerge/merge.go +++ b/src/internal/coverage/cmerge/merge.go @@ -57,7 +57,7 @@ func (m *Merger) MergeCounters(dst, src []uint32) (error, bool) { return nil, ovf } -// Saturating add does a saturating addition of 'dst' and 'src', +// SaturatingAdd does a saturating addition of 'dst' and 'src', // returning added value or math.MaxUint32 if there is an overflow. // Overflows are recorded in case the client needs to track them. func (m *Merger) SaturatingAdd(dst, src uint32) uint32 { @@ -68,7 +68,7 @@ func (m *Merger) SaturatingAdd(dst, src uint32) uint32 { return result } -// Saturating add does a saturating addition of 'dst' and 'src', +// SaturatingAdd does a saturating addition of 'dst' and 'src', // returning added value or math.MaxUint32 plus an overflow flag. func SaturatingAdd(dst, src uint32) (uint32, bool) { d, s := uint64(dst), uint64(src) diff --git a/src/internal/coverage/decodemeta/decodefile.go b/src/internal/coverage/decodemeta/decodefile.go index 96e076596f8bf5..6f4dd1a3ec0c74 100644 --- a/src/internal/coverage/decodemeta/decodefile.go +++ b/src/internal/coverage/decodemeta/decodefile.go @@ -12,9 +12,9 @@ package decodemeta import ( "bufio" - "crypto/md5" "encoding/binary" "fmt" + "hash/fnv" "internal/coverage" "internal/coverage/slicereader" "internal/coverage/stringtab" @@ -171,8 +171,10 @@ func (r *CoverageMetaFileReader) FileHash() [16]byte { func (r *CoverageMetaFileReader) GetPackageDecoder(pkIdx uint32, payloadbuf []byte) (*CoverageMetaDataDecoder, []byte, error) { pp, err := r.GetPackagePayload(pkIdx, payloadbuf) if r.debug { + h := fnv.New128a() + h.Write(pp) fmt.Fprintf(os.Stderr, "=-= pkidx=%d payload length is %d hash=%s\n", - pkIdx, len(pp), fmt.Sprintf("%x", md5.Sum(pp))) + pkIdx, len(pp), fmt.Sprintf("%x", h.Sum(nil))) } if err != nil { return nil, nil, err diff --git a/src/internal/coverage/encodecounter/encode.go b/src/internal/coverage/encodecounter/encode.go index d50827014b2065..82fd328faf8125 100644 --- a/src/internal/coverage/encodecounter/encode.go +++ b/src/internal/coverage/encodecounter/encode.go @@ -13,6 +13,7 @@ import ( "internal/coverage/stringtab" "internal/coverage/uleb128" "io" + "maps" "os" "slices" ) @@ -122,11 +123,7 @@ func (cfw *CoverageDataWriter) writeSegmentPreamble(args map[string]string, ws * } cfw.csh.StrTabLen = uint32(len(ws.BytesWritten())) - hdrsz - akeys := make([]string, 0, len(args)) - for k := range args { - akeys = append(akeys, k) - } - slices.Sort(akeys) + akeys := slices.Sorted(maps.Keys(args)) wrULEB128 := func(v uint) error { cfw.tmp = cfw.tmp[:0] diff --git a/src/internal/coverage/encodemeta/encode.go b/src/internal/coverage/encodemeta/encode.go index 549b3f55a8e304..e8f70fe575ad1e 100644 --- a/src/internal/coverage/encodemeta/encode.go +++ b/src/internal/coverage/encodemeta/encode.go @@ -10,10 +10,10 @@ package encodemeta import ( "bytes" - "crypto/md5" "encoding/binary" "fmt" "hash" + "hash/fnv" "internal/coverage" "internal/coverage/stringtab" "internal/coverage/uleb128" @@ -39,7 +39,7 @@ func NewCoverageMetaDataBuilder(pkgpath string, pkgname string, modulepath strin } x := &CoverageMetaDataBuilder{ tmp: make([]byte, 0, 256), - h: md5.New(), + h: fnv.New128a(), } x.stab.InitWriter() x.stab.Lookup("") @@ -188,7 +188,7 @@ func (b *CoverageMetaDataBuilder) Emit(w io.WriteSeeker) ([16]byte, error) { // HashFuncDesc computes an md5 sum of a coverage.FuncDesc and returns // a digest for it. func HashFuncDesc(f *coverage.FuncDesc) [16]byte { - h := md5.New() + h := fnv.New128a() tmp := make([]byte, 0, 32) hashFuncDesc(h, f, tmp) var r [16]byte diff --git a/src/internal/coverage/encodemeta/encodefile.go b/src/internal/coverage/encodemeta/encodefile.go index 38ae46e4f5617f..ae7c23ad976216 100644 --- a/src/internal/coverage/encodemeta/encodefile.go +++ b/src/internal/coverage/encodemeta/encodefile.go @@ -6,9 +6,9 @@ package encodemeta import ( "bufio" - "crypto/md5" "encoding/binary" "fmt" + "hash/fnv" "internal/coverage" "internal/coverage/stringtab" "io" @@ -112,7 +112,9 @@ func (m *CoverageMetaFileWriter) Write(finalHash [16]byte, blobs [][]byte, mode // Now emit blobs themselves. for k, blob := range blobs { if m.debug { - fmt.Fprintf(os.Stderr, "=+= writing blob %d len %d at off=%d hash %s\n", k, len(blob), off2, fmt.Sprintf("%x", md5.Sum(blob))) + h := fnv.New128a() + h.Write(blob) + fmt.Fprintf(os.Stderr, "=+= writing blob %d len %d at off=%d hash %s\n", k, len(blob), off2, fmt.Sprintf("%x", h.Sum(nil))) } if _, err = m.w.Write(blob); err != nil { return fmt.Errorf("error writing %s: %v", m.mfname, err) diff --git a/src/internal/coverage/pkid.go b/src/internal/coverage/pkid.go index 46de9fd0ced2e0..09501e6bd2a622 100644 --- a/src/internal/coverage/pkid.go +++ b/src/internal/coverage/pkid.go @@ -31,7 +31,7 @@ package coverage // slot: 6 path='internal/runtime/math' hard-coded id: 6 // slot: 7 path='internal/bytealg' hard-coded id: 7 // slot: 8 path='internal/goexperiment' -// slot: 9 path='internal/runtime/syscall' hard-coded id: 8 +// slot: 9 path='internal/runtime/syscall/linux' hard-coded id: 8 // slot: 10 path='runtime' hard-coded id: 9 // fatal error: runtime.addCovMeta // @@ -45,17 +45,31 @@ package coverage // as opposed to a fixed list. var rtPkgs = [...]string{ + "internal/asan", + "internal/byteorder", + "internal/coverage/rtcov", "internal/cpu", + "internal/bytealg", "internal/goarch", - "internal/runtime/atomic", - "internal/goos", - "internal/chacha8rand", - "internal/runtime/sys", "internal/abi", - "internal/runtime/math", - "internal/bytealg", + "internal/chacha8rand", + "internal/godebugs", "internal/goexperiment", - "internal/runtime/syscall", + "internal/goos", + "internal/msan", + "internal/profilerecord", + "internal/race", + "internal/runtime/atomic", + "internal/runtime/exithook", + "internal/runtime/gc", + "internal/runtime/math", + "internal/runtime/strconv", + "internal/runtime/sys", + "internal/runtime/maps", + "internal/runtime/syscall/linux", + "internal/runtime/syscall/windows", + "internal/runtime/cgroup", + "internal/stringslite", "runtime", } diff --git a/src/internal/coverage/pods/pods_test.go b/src/internal/coverage/pods/pods_test.go index 69c16e00eb01d4..eed01698e27a55 100644 --- a/src/internal/coverage/pods/pods_test.go +++ b/src/internal/coverage/pods/pods_test.go @@ -5,8 +5,8 @@ package pods_test import ( - "crypto/md5" "fmt" + "hash/fnv" "internal/coverage" "internal/coverage/pods" "os" @@ -35,13 +35,17 @@ func TestPodCollection(t *testing.T) { } mkmeta := func(dir string, tag string) string { - hash := md5.Sum([]byte(tag)) + h := fnv.New128a() + h.Write([]byte(tag)) + hash := h.Sum(nil) fn := fmt.Sprintf("%s.%x", coverage.MetaFilePref, hash) return mkfile(dir, fn) } mkcounter := func(dir string, tag string, nt int, pid int) string { - hash := md5.Sum([]byte(tag)) + h := fnv.New128a() + h.Write([]byte(tag)) + hash := h.Sum(nil) fn := fmt.Sprintf(coverage.CounterFileTempl, coverage.CounterFilePref, hash, pid, nt) return mkfile(dir, fn) } @@ -112,16 +116,16 @@ func TestPodCollection(t *testing.T) { } expected := []string{ - `o1/covmeta.ae7be26cdaa742ca148068d5ac90eaca [ -o1/covcounters.ae7be26cdaa742ca148068d5ac90eaca.40.2 o:0 -o1/covcounters.ae7be26cdaa742ca148068d5ac90eaca.41.2 o:0 -o1/covcounters.ae7be26cdaa742ca148068d5ac90eaca.42.1 o:0 -o2/covcounters.ae7be26cdaa742ca148068d5ac90eaca.35.11 o:1 + `o1/covmeta.0880952782ab1be95aa0733055a4d06b [ +o1/covcounters.0880952782ab1be95aa0733055a4d06b.40.2 o:0 +o1/covcounters.0880952782ab1be95aa0733055a4d06b.41.2 o:0 +o1/covcounters.0880952782ab1be95aa0733055a4d06b.42.1 o:0 +o2/covcounters.0880952782ab1be95aa0733055a4d06b.35.11 o:1 ]`, - `o2/covmeta.aaf2f89992379705dac844c0a2a1d45f [ -o2/covcounters.aaf2f89992379705dac844c0a2a1d45f.36.3 o:1 -o2/covcounters.aaf2f89992379705dac844c0a2a1d45f.37.2 o:1 -o2/covcounters.aaf2f89992379705dac844c0a2a1d45f.38.1 o:1 + `o2/covmeta.0880952783ab1be95aa0733055a4d1a6 [ +o2/covcounters.0880952783ab1be95aa0733055a4d1a6.36.3 o:1 +o2/covcounters.0880952783ab1be95aa0733055a4d1a6.37.2 o:1 +o2/covcounters.0880952783ab1be95aa0733055a4d1a6.38.1 o:1 ]`, } for k, exp := range expected { diff --git a/src/internal/coverage/test/counter_test.go b/src/internal/coverage/test/counter_test.go index e29baeddc0eaf1..f7f74dc9edd666 100644 --- a/src/internal/coverage/test/counter_test.go +++ b/src/internal/coverage/test/counter_test.go @@ -141,7 +141,7 @@ func TestCounterDataAppendSegment(t *testing.T) { const numSegments = 2 - // Write a counter with with multiple segments. + // Write a counter with multiple segments. args := map[string]string{"argc": "1", "argv0": "prog.exe"} allfuncs := [][]decodecounter.FuncPayload{} ctrs := []uint32{} diff --git a/src/internal/cpu/cpu.go b/src/internal/cpu/cpu.go index 7174076c5e43c4..9470b532df46e8 100644 --- a/src/internal/cpu/cpu.go +++ b/src/internal/cpu/cpu.go @@ -8,11 +8,6 @@ package cpu import _ "unsafe" // for linkname -// DebugOptions is set to true by the runtime if the OS supports reading -// GODEBUG early in runtime startup. -// This should not be changed after it is initialized. -var DebugOptions bool - // CacheLinePad is used to pad structs to avoid false sharing. type CacheLinePad struct{ _ [CacheLinePadSize]byte } @@ -26,29 +21,37 @@ var CacheLineSize uintptr = CacheLinePadSize // in addition to the cpuid feature bit being set. // The struct is padded to avoid false sharing. var X86 struct { - _ CacheLinePad - HasAES bool - HasADX bool - HasAVX bool - HasAVX2 bool - HasAVX512F bool - HasAVX512BW bool - HasAVX512VL bool - HasBMI1 bool - HasBMI2 bool - HasERMS bool - HasFSRM bool - HasFMA bool - HasOSXSAVE bool - HasPCLMULQDQ bool - HasPOPCNT bool - HasRDTSCP bool - HasSHA bool - HasSSE3 bool - HasSSSE3 bool - HasSSE41 bool - HasSSE42 bool - _ CacheLinePad + _ CacheLinePad + HasAES bool + HasADX bool + HasAVX bool + HasAVX2 bool + HasAVX512 bool // Virtual feature: F+CD+BW+DQ+VL + HasAVX512F bool + HasAVX512CD bool + HasAVX512BITALG bool + HasAVX512BW bool + HasAVX512DQ bool + HasAVX512VL bool + HasAVX512VPCLMULQDQ bool + HasAVX512VBMI bool + HasAVX512VBMI2 bool + HasBMI1 bool + HasBMI2 bool + HasERMS bool + HasFSRM bool + HasFMA bool + HasGFNI bool + HasOSXSAVE bool + HasPCLMULQDQ bool + HasPOPCNT bool + HasRDTSCP bool + HasSHA bool + HasSSE3 bool + HasSSSE3 bool + HasSSE41 bool + HasSSE42 bool + _ CacheLinePad } // The booleans in ARM contain the correspondingly named cpu feature bit. @@ -70,6 +73,7 @@ var ARM64 struct { HasSHA1 bool HasSHA2 bool HasSHA512 bool + HasSHA3 bool HasCRC32 bool HasATOMICS bool HasCPUID bool @@ -78,6 +82,18 @@ var ARM64 struct { _ CacheLinePad } +// The booleans in Loong64 contain the correspondingly named cpu feature bit. +// The struct is padded to avoid false sharing. +var Loong64 struct { + _ CacheLinePad + HasLSX bool // support 128-bit vector extension + HasLASX bool // support 256-bit vector extension + HasCRC32 bool // support CRC instruction + HasLAMCAS bool // support AMCAS[_DB].{B/H/W/D} + HasLAM_BH bool // support AM{SWAP/ADD}[_DB].{B/H} instruction + _ CacheLinePad +} + var MIPS64X struct { _ CacheLinePad HasMSA bool // MIPS SIMD architecture @@ -125,13 +141,31 @@ var S390X struct { _ CacheLinePad } +// RISCV64 contains the supported CPU features and performance characteristics for riscv64 +// platforms. The booleans in RISCV64, with the exception of HasFastMisaligned, indicate +// the presence of RISC-V extensions. +// The struct is padded to avoid false sharing. +var RISCV64 struct { + _ CacheLinePad + HasFastMisaligned bool // Fast misaligned accesses + HasV bool // Vector extension compatible with RVV 1.0 + HasZbb bool // Basic bit-manipulation extension + _ CacheLinePad +} + // CPU feature variables are accessed by assembly code in various packages. //go:linkname X86 //go:linkname ARM //go:linkname ARM64 +//go:linkname Loong64 //go:linkname MIPS64X //go:linkname PPC64 //go:linkname S390X +//go:linkname RISCV64 + +// doDerived, if non-nil, is called after processing GODEBUG to set "derived" +// feature flags. +var doDerived func() // Initialize examines the processor and sets the relevant variables above. // This is called by the runtime package early in program initialization, @@ -140,6 +174,9 @@ var S390X struct { func Initialize(env string) { doinit() processOptions(env) + if doDerived != nil { + doDerived() + } } // options contains the cpu debug options that can be used in GODEBUG. diff --git a/src/internal/cpu/cpu_arm64.go b/src/internal/cpu/cpu_arm64.go index 1365991e783492..7709966d5706bb 100644 --- a/src/internal/cpu/cpu_arm64.go +++ b/src/internal/cpu/cpu_arm64.go @@ -16,6 +16,7 @@ func doinit() { {Name: "sha1", Feature: &ARM64.HasSHA1}, {Name: "sha2", Feature: &ARM64.HasSHA2}, {Name: "sha512", Feature: &ARM64.HasSHA512}, + {Name: "sha3", Feature: &ARM64.HasSHA3}, {Name: "crc32", Feature: &ARM64.HasCRC32}, {Name: "atomics", Feature: &ARM64.HasATOMICS}, {Name: "cpuid", Feature: &ARM64.HasCPUID}, @@ -38,6 +39,7 @@ func extractBits(data uint64, start, end uint) uint { func parseARM64SystemRegisters(isar0, pfr0 uint64) { // ID_AA64ISAR0_EL1 + // https://developer.arm.com/documentation/ddi0601/2025-03/AArch64-Registers/ID-AA64ISAR0-EL1--AArch64-Instruction-Set-Attribute-Register-0 switch extractBits(isar0, 4, 7) { case 1: ARM64.HasAES = true @@ -69,6 +71,11 @@ func parseARM64SystemRegisters(isar0, pfr0 uint64) { ARM64.HasATOMICS = true } + switch extractBits(isar0, 32, 35) { + case 1: + ARM64.HasSHA3 = true + } + switch extractBits(pfr0, 48, 51) { case 1: ARM64.HasDIT = true diff --git a/src/internal/cpu/cpu_arm64_darwin.go b/src/internal/cpu/cpu_arm64_darwin.go index 57cf631e891b0a..28b47d60e8aecf 100644 --- a/src/internal/cpu/cpu_arm64_darwin.go +++ b/src/internal/cpu/cpu_arm64_darwin.go @@ -9,15 +9,21 @@ package cpu import _ "unsafe" // for linkname func osInit() { + // macOS 12 moved these to the hw.optional.arm tree, but as of Go 1.24 we + // still support macOS 11. See [Determine Encryption Capabilities]. + // + // [Determine Encryption Capabilities]: https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics#3918855 ARM64.HasATOMICS = sysctlEnabled([]byte("hw.optional.armv8_1_atomics\x00")) ARM64.HasCRC32 = sysctlEnabled([]byte("hw.optional.armv8_crc32\x00")) ARM64.HasSHA512 = sysctlEnabled([]byte("hw.optional.armv8_2_sha512\x00")) + ARM64.HasSHA3 = sysctlEnabled([]byte("hw.optional.armv8_2_sha3\x00")) + ARM64.HasDIT = sysctlEnabled([]byte("hw.optional.arm.FEAT_DIT\x00")) - // There are no hw.optional sysctl values for the below features on Mac OS 11.0 - // to detect their supported state dynamically. Assume the CPU features that - // Apple Silicon M1 supports to be available as a minimal set of features - // to all Go programs running on darwin/arm64. + // There are no hw.optional sysctl values for the below features on macOS 11 + // to detect their supported state dynamically (although they are available + // in the hw.optional.arm tree on macOS 12). Assume the CPU features that + // Apple Silicon M1 supports to be available on all future iterations. ARM64.HasAES = true ARM64.HasPMULL = true ARM64.HasSHA1 = true diff --git a/src/internal/cpu/cpu_arm64_hwcap.go b/src/internal/cpu/cpu_arm64_hwcap.go index cdc1d89c9e42be..b6cc1f4d988877 100644 --- a/src/internal/cpu/cpu_arm64_hwcap.go +++ b/src/internal/cpu/cpu_arm64_hwcap.go @@ -22,6 +22,7 @@ import _ "unsafe" // for linkname var HWCap uint // HWCAP bits. These are exposed by Linux. +// See arch/arm64/include/uapi/asm/hwcap.h. const ( hwcap_AES = 1 << 3 hwcap_PMULL = 1 << 4 @@ -30,18 +31,21 @@ const ( hwcap_CRC32 = 1 << 7 hwcap_ATOMICS = 1 << 8 hwcap_CPUID = 1 << 11 + hwcap_SHA3 = 1 << 17 hwcap_SHA512 = 1 << 21 hwcap_DIT = 1 << 24 ) func hwcapInit(os string) { // HWCap was populated by the runtime from the auxiliary vector. + // See https://docs.kernel.org/arch/arm64/elf_hwcaps.html. // Use HWCap information since reading aarch64 system registers // is not supported in user space on older linux kernels. ARM64.HasAES = isSet(HWCap, hwcap_AES) ARM64.HasPMULL = isSet(HWCap, hwcap_PMULL) ARM64.HasSHA1 = isSet(HWCap, hwcap_SHA1) ARM64.HasSHA2 = isSet(HWCap, hwcap_SHA2) + ARM64.HasSHA3 = isSet(HWCap, hwcap_SHA3) ARM64.HasCRC32 = isSet(HWCap, hwcap_CRC32) ARM64.HasCPUID = isSet(HWCap, hwcap_CPUID) ARM64.HasSHA512 = isSet(HWCap, hwcap_SHA512) @@ -67,9 +71,14 @@ func hwcapInit(os string) { // d40 - NeoverseV1 // d49 - NeoverseN2 // d4f - NeoverseV2 - if implementer == 'A' && (part_num == 0xd0c || part_num == 0xd40 || - part_num == 0xd49 || part_num == 0xd4f) { - ARM64.IsNeoverse = true + // d8e - NeoverseN3 + // d84 - NeoverseV3 + // d83 - NeoverseV3ae + if implementer == 'A' { + switch part_num { + case 0xd0c, 0xd40, 0xd49, 0xd4f, 0xd8e, 0xd84, 0xd83: + ARM64.IsNeoverse = true + } } } } diff --git a/src/internal/cpu/cpu_loong64.go b/src/internal/cpu/cpu_loong64.go index 1c90c24fe318d9..de7eaf0c6c99bb 100644 --- a/src/internal/cpu/cpu_loong64.go +++ b/src/internal/cpu/cpu_loong64.go @@ -10,4 +10,46 @@ package cpu // We choose 64 because Loongson 3A5000 the L1 Dcache is 4-way 256-line 64-byte-per-line. const CacheLinePadSize = 64 -func doinit() {} +// Bit fields for CPUCFG registers, Related reference documents: +// https://loongson.github.io/LoongArch-Documentation/LoongArch-Vol1-EN.html#_cpucfg +const ( + // CPUCFG1 bits + cpucfg1_CRC32 = 1 << 25 + + // CPUCFG2 bits + cpucfg2_LAM_BH = 1 << 27 + cpucfg2_LAMCAS = 1 << 28 +) + +// get_cpucfg is implemented in cpu_loong64.s. +func get_cpucfg(reg uint32) uint32 + +func doinit() { + options = []option{ + {Name: "lsx", Feature: &Loong64.HasLSX}, + {Name: "lasx", Feature: &Loong64.HasLASX}, + {Name: "crc32", Feature: &Loong64.HasCRC32}, + {Name: "lamcas", Feature: &Loong64.HasLAMCAS}, + {Name: "lam_bh", Feature: &Loong64.HasLAM_BH}, + } + + // The CPUCFG data on Loong64 only reflects the hardware capabilities, + // not the kernel support status, so features such as LSX and LASX that + // require kernel support cannot be obtained from the CPUCFG data. + // + // These features only require hardware capability support and do not + // require kernel specific support, so they can be obtained directly + // through CPUCFG + cfg1 := get_cpucfg(1) + cfg2 := get_cpucfg(2) + + Loong64.HasCRC32 = cfgIsSet(cfg1, cpucfg1_CRC32) + Loong64.HasLAMCAS = cfgIsSet(cfg2, cpucfg2_LAMCAS) + Loong64.HasLAM_BH = cfgIsSet(cfg2, cpucfg2_LAM_BH) + + osInit() +} + +func cfgIsSet(cfg uint32, val uint32) bool { + return cfg&val != 0 +} diff --git a/src/internal/cpu/cpu_loong64.s b/src/internal/cpu/cpu_loong64.s new file mode 100644 index 00000000000000..f02a27803d64de --- /dev/null +++ b/src/internal/cpu/cpu_loong64.s @@ -0,0 +1,12 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// func get_cpucfg(reg uint32) uint32 +TEXT ·get_cpucfg(SB), NOSPLIT|NOFRAME, $0-12 + MOVW reg+0(FP), R5 + CPUCFG R5, R4 + MOVW R4, ret+8(FP) + RET diff --git a/src/internal/cpu/cpu_loong64_hwcap.go b/src/internal/cpu/cpu_loong64_hwcap.go new file mode 100644 index 00000000000000..2b25cc6b4a6c2d --- /dev/null +++ b/src/internal/cpu/cpu_loong64_hwcap.go @@ -0,0 +1,28 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build loong64 && linux + +package cpu + +// This is initialized by archauxv and should not be changed after it is +// initialized. +var HWCap uint + +// HWCAP bits. These are exposed by the Linux kernel. +const ( + hwcap_LOONGARCH_LSX = 1 << 4 + hwcap_LOONGARCH_LASX = 1 << 5 +) + +func hwcapInit() { + // TODO: Features that require kernel support like LSX and LASX can + // be detected here once needed in std library or by the compiler. + Loong64.HasLSX = hwcIsSet(HWCap, hwcap_LOONGARCH_LSX) + Loong64.HasLASX = hwcIsSet(HWCap, hwcap_LOONGARCH_LASX) +} + +func hwcIsSet(hwc uint, val uint) bool { + return hwc&val != 0 +} diff --git a/src/internal/cpu/cpu_loong64_linux.go b/src/internal/cpu/cpu_loong64_linux.go new file mode 100644 index 00000000000000..73bc384a54e843 --- /dev/null +++ b/src/internal/cpu/cpu_loong64_linux.go @@ -0,0 +1,11 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build loong64 && linux + +package cpu + +func osInit() { + hwcapInit() +} diff --git a/src/internal/cpu/cpu_riscv64.go b/src/internal/cpu/cpu_riscv64.go index 2173fe88860904..0fe1704855fa0c 100644 --- a/src/internal/cpu/cpu_riscv64.go +++ b/src/internal/cpu/cpu_riscv64.go @@ -6,5 +6,17 @@ package cpu const CacheLinePadSize = 64 +// RISC-V doesn't have a 'cpuid' equivalent. On Linux we rely on the riscv_hwprobe syscall. + func doinit() { + options = []option{ + {Name: "fastmisaligned", Feature: &RISCV64.HasFastMisaligned}, + {Name: "v", Feature: &RISCV64.HasV}, + {Name: "zbb", Feature: &RISCV64.HasZbb}, + } + osInit() +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 } diff --git a/src/internal/cpu/cpu_riscv64_linux.go b/src/internal/cpu/cpu_riscv64_linux.go new file mode 100644 index 00000000000000..b67bdf58766f48 --- /dev/null +++ b/src/internal/cpu/cpu_riscv64_linux.go @@ -0,0 +1,93 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build riscv64 && linux + +package cpu + +import _ "unsafe" + +// RISC-V extension discovery code for Linux. +// +// A note on detection of the Vector extension using HWCAP. +// +// Support for the Vector extension version 1.0 was added to the Linux kernel in release 6.5. +// Support for the riscv_hwprobe syscall was added in 6.4. It follows that if the riscv_hwprobe +// syscall is not available then neither is the Vector extension (which needs kernel support). +// The riscv_hwprobe syscall should then be all we need to detect the Vector extension. +// However, some RISC-V board manufacturers ship boards with an older kernel on top of which +// they have back-ported various versions of the Vector extension patches but not the riscv_hwprobe +// patches. These kernels advertise support for the Vector extension using HWCAP. Falling +// back to HWCAP to detect the Vector extension, if riscv_hwprobe is not available, or simply not +// bothering with riscv_hwprobe at all and just using HWCAP may then seem like an attractive option. +// +// Unfortunately, simply checking the 'V' bit in AT_HWCAP will not work as this bit is used by +// RISC-V board and cloud instance providers to mean different things. The Lichee Pi 4A board +// and the Scaleway RV1 cloud instances use the 'V' bit to advertise their support for the unratified +// 0.7.1 version of the Vector Specification. The Banana Pi BPI-F3 and the CanMV-K230 board use +// it to advertise support for 1.0 of the Vector extension. Versions 0.7.1 and 1.0 of the Vector +// extension are binary incompatible. HWCAP can then not be used in isolation to populate the +// HasV field as this field indicates that the underlying CPU is compatible with RVV 1.0. +// Go will only support the ratified versions >= 1.0 and so any vector code it might generate +// would crash on a Scaleway RV1 instance or a Lichee Pi 4a, if allowed to run. +// +// There is a way at runtime to distinguish between versions 0.7.1 and 1.0 of the Vector +// specification by issuing a RVV 1.0 vsetvli instruction and checking the vill bit of the vtype +// register. This check would allow us to safely detect version 1.0 of the Vector extension +// with HWCAP, if riscv_hwprobe were not available. However, the check cannot +// be added until the assembler supports the Vector instructions. +// +// Note the riscv_hwprobe syscall does not suffer from these ambiguities by design as all of the +// extensions it advertises support for are explicitly versioned. It's also worth noting that +// the riscv_hwprobe syscall is the only way to detect multi-letter RISC-V extensions, e.g., Zvbb. +// These cannot be detected using HWCAP and so riscv_hwprobe must be used to detect the majority +// of RISC-V extensions. +// +// Please see https://docs.kernel.org/arch/riscv/hwprobe.html for more information. + +const ( + // Copied from golang.org/x/sys/unix/ztypes_linux_riscv64.go. + riscv_HWPROBE_KEY_IMA_EXT_0 = 0x4 + riscv_HWPROBE_IMA_V = 0x4 + riscv_HWPROBE_EXT_ZBB = 0x10 + riscv_HWPROBE_KEY_CPUPERF_0 = 0x5 + riscv_HWPROBE_MISALIGNED_FAST = 0x3 + riscv_HWPROBE_MISALIGNED_MASK = 0x7 +) + +// riscvHWProbePairs is copied from golang.org/x/sys/unix/ztypes_linux_riscv64.go. +type riscvHWProbePairs struct { + key int64 + value uint64 +} + +//go:linkname riscvHWProbe +func riscvHWProbe(pairs []riscvHWProbePairs, flags uint) bool + +func osInit() { + // A slice of key/value pair structures is passed to the RISCVHWProbe syscall. The key + // field should be initialised with one of the key constants defined above, e.g., + // RISCV_HWPROBE_KEY_IMA_EXT_0. The syscall will set the value field to the appropriate value. + // If the kernel does not recognise a key it will set the key field to -1 and the value field to 0. + + pairs := []riscvHWProbePairs{ + {riscv_HWPROBE_KEY_IMA_EXT_0, 0}, + {riscv_HWPROBE_KEY_CPUPERF_0, 0}, + } + + // This call only indicates that extensions are supported if they are implemented on all cores. + if !riscvHWProbe(pairs, 0) { + return + } + + if pairs[0].key != -1 { + v := uint(pairs[0].value) + RISCV64.HasV = isSet(v, riscv_HWPROBE_IMA_V) + RISCV64.HasZbb = isSet(v, riscv_HWPROBE_EXT_ZBB) + } + if pairs[1].key != -1 { + v := pairs[1].value & riscv_HWPROBE_MISALIGNED_MASK + RISCV64.HasFastMisaligned = v == riscv_HWPROBE_MISALIGNED_FAST + } +} diff --git a/src/internal/cpu/cpu_riscv64_other.go b/src/internal/cpu/cpu_riscv64_other.go new file mode 100644 index 00000000000000..1307d822b3257f --- /dev/null +++ b/src/internal/cpu/cpu_riscv64_other.go @@ -0,0 +1,11 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build riscv64 && !linux + +package cpu + +func osInit() { + // Other operating systems do not support the riscv_hwprobe syscall. +} diff --git a/src/internal/cpu/cpu_test.go b/src/internal/cpu/cpu_test.go index a6fe7f77f34585..62e250d1e82006 100644 --- a/src/internal/cpu/cpu_test.go +++ b/src/internal/cpu/cpu_test.go @@ -8,13 +8,15 @@ import ( . "internal/cpu" "internal/godebug" "internal/testenv" - "os" "os/exec" + "runtime" "testing" ) func MustHaveDebugOptionsSupport(t *testing.T) { - if !DebugOptions { + switch runtime.GOOS { + case "aix", "darwin", "ios", "dragonfly", "freebsd", "netbsd", "openbsd", "illumos", "solaris", "linux": + default: t.Skipf("skipping test: cpu feature options not supported by OS") } } @@ -26,11 +28,9 @@ func MustSupportFeatureDetection(t *testing.T) { func runDebugOptionsTest(t *testing.T, test string, options string) { MustHaveDebugOptionsSupport(t) - testenv.MustHaveExec(t) - env := "GODEBUG=" + options - cmd := exec.Command(os.Args[0], "-test.run=^"+test+"$") + cmd := exec.Command(testenv.Executable(t), "-test.run=^"+test+"$") cmd.Env = append(cmd.Env, env) output, err := cmd.CombinedOutput() diff --git a/src/internal/cpu/cpu_x86.go b/src/internal/cpu/cpu_x86.go index ee812076e96c49..6fa30b776310b3 100644 --- a/src/internal/cpu/cpu_x86.go +++ b/src/internal/cpu/cpu_x86.go @@ -18,7 +18,7 @@ func xgetbv() (eax, edx uint32) func getGOAMD64level() int32 const ( - // ecx bits + // Bits returned in ECX for CPUID EAX=0x1 ECX=0x0 cpuid_SSE3 = 1 << 0 cpuid_PCLMULQDQ = 1 << 1 cpuid_SSSE3 = 1 << 9 @@ -30,16 +30,26 @@ const ( cpuid_OSXSAVE = 1 << 27 cpuid_AVX = 1 << 28 - // ebx bits + // "Extended Feature Flag" bits returned in EBX for CPUID EAX=0x7 ECX=0x0 cpuid_BMI1 = 1 << 3 cpuid_AVX2 = 1 << 5 cpuid_BMI2 = 1 << 8 cpuid_ERMS = 1 << 9 cpuid_AVX512F = 1 << 16 + cpuid_AVX512DQ = 1 << 17 cpuid_ADX = 1 << 19 + cpuid_AVX512CD = 1 << 28 cpuid_SHA = 1 << 29 cpuid_AVX512BW = 1 << 30 cpuid_AVX512VL = 1 << 31 + + // "Extended Feature Flag" bits returned in ECX for CPUID EAX=0x7 ECX=0x0 + cpuid_AVX512_VBMI = 1 << 1 + cpuid_AVX512_VBMI2 = 1 << 6 + cpuid_GFNI = 1 << 8 + cpuid_AVX512VPCLMULQDQ = 1 << 10 + cpuid_AVX512_BITALG = 1 << 12 + // edx bits cpuid_FSRM = 1 << 4 // edx bits for CPUID 0x80000001 @@ -57,6 +67,7 @@ func doinit() { {Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ}, {Name: "rdtscp", Feature: &X86.HasRDTSCP}, {Name: "sha", Feature: &X86.HasSHA}, + {Name: "vpclmulqdq", Feature: &X86.HasAVX512VPCLMULQDQ}, } level := getGOAMD64level() if level < 2 { @@ -84,7 +95,9 @@ func doinit() { // they can be turned off. options = append(options, option{Name: "avx512f", Feature: &X86.HasAVX512F}, + option{Name: "avx512cd", Feature: &X86.HasAVX512CD}, option{Name: "avx512bw", Feature: &X86.HasAVX512BW}, + option{Name: "avx512dq", Feature: &X86.HasAVX512DQ}, option{Name: "avx512vl", Feature: &X86.HasAVX512VL}, ) } @@ -139,7 +152,7 @@ func doinit() { return } - _, ebx7, _, edx7 := cpuid(7, 0) + _, ebx7, ecx7, edx7 := cpuid(7, 0) X86.HasBMI1 = isSet(ebx7, cpuid_BMI1) X86.HasAVX2 = isSet(ebx7, cpuid_AVX2) && osSupportsAVX X86.HasBMI2 = isSet(ebx7, cpuid_BMI2) @@ -149,8 +162,15 @@ func doinit() { X86.HasAVX512F = isSet(ebx7, cpuid_AVX512F) && osSupportsAVX512 if X86.HasAVX512F { + X86.HasAVX512CD = isSet(ebx7, cpuid_AVX512CD) X86.HasAVX512BW = isSet(ebx7, cpuid_AVX512BW) + X86.HasAVX512DQ = isSet(ebx7, cpuid_AVX512DQ) X86.HasAVX512VL = isSet(ebx7, cpuid_AVX512VL) + X86.HasAVX512VPCLMULQDQ = isSet(ecx7, cpuid_AVX512VPCLMULQDQ) + X86.HasAVX512VBMI = isSet(ecx7, cpuid_AVX512_VBMI) + X86.HasAVX512VBMI2 = isSet(ecx7, cpuid_AVX512_VBMI2) + X86.HasGFNI = isSet(ecx7, cpuid_GFNI) + X86.HasAVX512BITALG = isSet(ecx7, cpuid_AVX512_BITALG) } X86.HasFSRM = isSet(edx7, cpuid_FSRM) @@ -164,6 +184,17 @@ func doinit() { _, _, _, edxExt1 := cpuid(0x80000001, 0) X86.HasRDTSCP = isSet(edxExt1, cpuid_RDTSCP) + + doDerived = func() { + // Rather than carefully gating on fundamental AVX-512 features, we have + // a virtual "AVX512" feature that captures F+CD+BW+DQ+VL. BW, DQ, and + // VL have a huge effect on which AVX-512 instructions are available, + // and these have all been supported on everything except the earliest + // Phi chips with AVX-512. No CPU has had CD without F, so we include + // it. GOAMD64=v4 also implies exactly this set, and these are all + // included in AVX10.1. + X86.HasAVX512 = X86.HasAVX512F && X86.HasAVX512CD && X86.HasAVX512BW && X86.HasAVX512DQ && X86.HasAVX512VL + } } func isSet(hwc uint32, value uint32) bool { diff --git a/src/internal/cpu/datacache_unsupported.go b/src/internal/cpu/datacache_unsupported.go new file mode 100644 index 00000000000000..44544aa8c9e5be --- /dev/null +++ b/src/internal/cpu/datacache_unsupported.go @@ -0,0 +1,11 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !386 && !amd64 + +package cpu + +func DataCacheSizes() []uintptr { + return nil +} diff --git a/src/internal/cpu/datacache_x86.go b/src/internal/cpu/datacache_x86.go new file mode 100644 index 00000000000000..eb7b93b0a26392 --- /dev/null +++ b/src/internal/cpu/datacache_x86.go @@ -0,0 +1,121 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 || amd64 + +package cpu + +// DataCacheSizes returns the size of each data cache from lowest +// level in the hierarchy to highest. +// +// Unlike other parts of this package's public API, it is not safe +// to reference early in runtime initialization because it allocates. +// It's intended for testing only. +func DataCacheSizes() []uintptr { + maxFunctionInformation, ebx0, ecx0, edx0 := cpuid(0, 0) + if maxFunctionInformation < 1 { + return nil + } + + switch { + // Check for "GenuineIntel" + case ebx0 == 0x756E6547 && ecx0 == 0x6C65746E && edx0 == 0x49656E69: + return getDataCacheSizesIntel(maxFunctionInformation) + // Check for "AuthenticAMD" + case ebx0 == 0x68747541 && ecx0 == 0x444D4163 && edx0 == 0x69746E65: + return getDataCacheSizesAMD() + } + return nil +} + +func extractBits(arg uint32, l int, r int) uint32 { + if l > r { + panic("bad bit range") + } + return (arg >> l) & ((1 << (r - l + 1)) - 1) +} + +func getDataCacheSizesIntel(maxID uint32) []uintptr { + // Constants for cache types + const ( + noCache = 0 + dataCache = 1 + instructionCache = 2 + unifiedCache = 3 + ) + if maxID < 4 { + return nil + } + + // Iterate through CPUID leaf 4 (deterministic cache parameters) + var caches []uintptr + for i := uint32(0); i < 0xFFFF; i++ { + eax, ebx, ecx, _ := cpuid(4, i) + + cacheType := eax & 0xF // EAX bits 4-0: Cache Type + if cacheType == 0 { + break + } + + // Report only data caches. + if !(cacheType == dataCache || cacheType == unifiedCache) { + continue + } + + // Guaranteed to always start counting from 1. + level := (eax >> 5) & 0x7 + + lineSize := extractBits(ebx, 0, 11) + 1 // Bits 11-0: Line size in bytes - 1 + partitions := extractBits(ebx, 12, 21) + 1 // Bits 21-12: Physical line partitions - 1 + ways := extractBits(ebx, 22, 31) + 1 // Bits 31-22: Ways of associativity - 1 + sets := uint64(ecx) + 1 // Number of sets - 1 + size := uint64(ways*partitions*lineSize) * sets // Calculate cache size in bytes + + caches = append(caches, uintptr(size)) + + // If we see more than one cache described per level, or they appear + // out of order, crash. + // + // Going by the SDM, it's not clear whether this is actually possible, + // so this code is purely defensive. + if level != uint32(len(caches)) { + panic("expected levels to be in order and for there to be one data/unified cache per level") + } + } + return caches +} + +func getDataCacheSizesAMD() []uintptr { + maxExtendedFunctionInformation, _, _, _ := cpuid(0x80000000, 0) + if maxExtendedFunctionInformation < 0x80000006 { + return nil + } + + var caches []uintptr + + _, _, ecx5, _ := cpuid(0x80000005, 0) + _, _, ecx6, edx6 := cpuid(0x80000006, 0) + + // The size is return in kb, turning into bytes. + l1dSize := uintptr(extractBits(ecx5, 24, 31) << 10) + caches = append(caches, l1dSize) + + // Check that L2 cache is present. + if l2Assoc := extractBits(ecx6, 12, 15); l2Assoc == 0 { + return caches + } + l2Size := uintptr(extractBits(ecx6, 16, 31) << 10) + caches = append(caches, l2Size) + + // Check that L3 cache is present. + if l3Assoc := extractBits(edx6, 12, 15); l3Assoc == 0 { + return caches + } + // Specifies the L3 cache size is within the following range: + // (L3Size[31:18] * 512KB) <= L3 cache size < ((L3Size[31:18]+1) * 512KB). + l3Size := uintptr(extractBits(edx6, 18, 31) * (512 << 10)) + caches = append(caches, l3Size) + + return caches +} diff --git a/src/internal/cpu/datacache_x86_test.go b/src/internal/cpu/datacache_x86_test.go new file mode 100644 index 00000000000000..425c525be099f0 --- /dev/null +++ b/src/internal/cpu/datacache_x86_test.go @@ -0,0 +1,26 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 || amd64 + +package cpu_test + +import ( + "internal/cpu" + "testing" +) + +// Tests fetching data cache sizes. This test only checks that DataCacheSizes +// won't explode. Otherwise it's just informational, and dumps the current +// data cache sizes. +func TestDataCacheSizes(t *testing.T) { + // N.B. Don't try to check these values because we don't know what + // kind of environment we're running in. We don't want this test to + // fail on some random x86 chip that happens to not support the right + // CPUID bits for some reason. + caches := cpu.DataCacheSizes() + for i, size := range caches { + t.Logf("L%d: %d", i+1, size) + } +} diff --git a/src/internal/exportdata/exportdata.go b/src/internal/exportdata/exportdata.go new file mode 100644 index 00000000000000..861a47f49f94f9 --- /dev/null +++ b/src/internal/exportdata/exportdata.go @@ -0,0 +1,355 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package exportdata implements common utilities for finding +// and reading gc-generated object files. +package exportdata + +// This file should be kept in sync with src/cmd/compile/internal/gc/obj.go . + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "go/build" + "internal/saferio" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" +) + +// ReadUnified reads the contents of the unified export data from a reader r +// that contains the contents of a GC-created archive file. +// +// On success, the reader will be positioned after the end-of-section marker "\n$$\n". +// +// Supported GC-created archive files have 4 layers of nesting: +// - An archive file containing a package definition file. +// - The package definition file contains headers followed by a data section. +// Headers are lines (≤ 4kb) that do not start with "$$". +// - The data section starts with "$$B\n" followed by export data followed +// by an end of section marker "\n$$\n". (The section start "$$\n" is no +// longer supported.) +// - The export data starts with a format byte ('u') followed by the in +// the given format. (See ReadExportDataHeader for older formats.) +// +// Putting this together, the bytes in a GC-created archive files are expected +// to look like the following. +// See cmd/internal/archive for more details on ar file headers. +// +// | \n | ar file signature +// | __.PKGDEF...size...\n | ar header for __.PKGDEF including size. +// | go object <...>\n | objabi header +// | \n | other headers such as build id +// | $$B\n | binary format marker +// | u\n | unified export +// | $$\n | end-of-section marker +// | [optional padding] | padding byte (0x0A) if size is odd +// | [ar file header] | other ar files +// | [ar file data] | +func ReadUnified(r *bufio.Reader) (data []byte, err error) { + // We historically guaranteed headers at the default buffer size (4096) work. + // This ensures we can use ReadSlice throughout. + const minBufferSize = 4096 + r = bufio.NewReaderSize(r, minBufferSize) + + size, err := FindPackageDefinition(r) + if err != nil { + return + } + n := size + + objapi, headers, err := ReadObjectHeaders(r) + if err != nil { + return + } + n -= len(objapi) + for _, h := range headers { + n -= len(h) + } + + hdrlen, err := ReadExportDataHeader(r) + if err != nil { + return + } + n -= hdrlen + + // size also includes the end of section marker. Remove that many bytes from the end. + const marker = "\n$$\n" + n -= len(marker) + + if n < 0 { + err = fmt.Errorf("invalid size (%d) in the archive file: %d bytes remain without section headers (recompile package)", size, n) + return + } + + // Read n bytes from buf. + data, err = saferio.ReadData(r, uint64(n)) + if err != nil { + return + } + + // Check for marker at the end. + var suffix [len(marker)]byte + _, err = io.ReadFull(r, suffix[:]) + if err != nil { + return + } + if s := string(suffix[:]); s != marker { + err = fmt.Errorf("read %q instead of end-of-section marker (%q)", s, marker) + return + } + + return +} + +// FindPackageDefinition positions the reader r at the beginning of a package +// definition file ("__.PKGDEF") within a GC-created archive by reading +// from it, and returns the size of the package definition file in the archive. +// +// The reader must be positioned at the start of the archive file before calling +// this function, and "__.PKGDEF" is assumed to be the first file in the archive. +// +// See cmd/internal/archive for details on the archive format. +func FindPackageDefinition(r *bufio.Reader) (size int, err error) { + // Uses ReadSlice to limit risk of malformed inputs. + + // Read first line to make sure this is an object file. + line, err := r.ReadSlice('\n') + if err != nil { + err = fmt.Errorf("can't find export data (%v)", err) + return + } + + // Is the first line an archive file signature? + if string(line) != "!\n" { + err = fmt.Errorf("not the start of an archive file (%q)", line) + return + } + + // package export block should be first + size = readArchiveHeader(r, "__.PKGDEF") + if size <= 0 { + err = fmt.Errorf("not a package file") + return + } + + return +} + +// ReadObjectHeaders reads object headers from the reader. Object headers are +// lines that do not start with an end-of-section marker "$$". The first header +// is the objabi header. On success, the reader will be positioned at the beginning +// of the end-of-section marker. +// +// It returns an error if any header does not fit in r.Size() bytes. +func ReadObjectHeaders(r *bufio.Reader) (objapi string, headers []string, err error) { + // line is a temporary buffer for headers. + // Use bounded reads (ReadSlice, Peek) to limit risk of malformed inputs. + var line []byte + + // objapi header should be the first line + if line, err = r.ReadSlice('\n'); err != nil { + err = fmt.Errorf("can't find export data (%v)", err) + return + } + objapi = string(line) + + // objapi header begins with "go object ". + if !strings.HasPrefix(objapi, "go object ") { + err = fmt.Errorf("not a go object file: %s", objapi) + return + } + + // process remaining object header lines + for { + // check for an end of section marker "$$" + line, err = r.Peek(2) + if err != nil { + return + } + if string(line) == "$$" { + return // stop + } + + // read next header + line, err = r.ReadSlice('\n') + if err != nil { + return + } + headers = append(headers, string(line)) + } +} + +// ReadExportDataHeader reads the export data header and format from r. +// It returns the number of bytes read, or an error if the format is no longer +// supported or it failed to read. +// +// The only currently supported format is binary export data in the +// unified export format. +func ReadExportDataHeader(r *bufio.Reader) (n int, err error) { + // Read export data header. + line, err := r.ReadSlice('\n') + if err != nil { + return + } + + hdr := string(line) + switch hdr { + case "$$\n": + err = fmt.Errorf("old textual export format no longer supported (recompile package)") + return + + case "$$B\n": + var format byte + format, err = r.ReadByte() + if err != nil { + return + } + // The unified export format starts with a 'u'. + switch format { + case 'u': + default: + // Older no longer supported export formats include: + // indexed export format which started with an 'i'; and + // the older binary export format which started with a 'c', + // 'd', or 'v' (from "version"). + err = fmt.Errorf("binary export format %q is no longer supported (recompile package)", format) + return + } + + default: + err = fmt.Errorf("unknown export data header: %q", hdr) + return + } + + n = len(hdr) + 1 // + 1 is for 'u' + return +} + +// FindPkg returns the filename and unique package id for an import +// path based on package information provided by build.Import (using +// the build.Default build.Context). A relative srcDir is interpreted +// relative to the current working directory. +func FindPkg(path, srcDir string) (filename, id string, err error) { + if path == "" { + return "", "", errors.New("path is empty") + } + + var noext string + switch { + default: + // "x" -> "$GOPATH/pkg/$GOOS_$GOARCH/x.ext", "x" + // Don't require the source files to be present. + if abs, err := filepath.Abs(srcDir); err == nil { // see issue 14282 + srcDir = abs + } + var bp *build.Package + bp, err = build.Import(path, srcDir, build.FindOnly|build.AllowBinary) + if bp.PkgObj == "" { + if bp.Goroot && bp.Dir != "" { + filename, err = lookupGorootExport(bp.Dir) + if err == nil { + _, err = os.Stat(filename) + } + if err == nil { + return filename, bp.ImportPath, nil + } + } + goto notfound + } else { + noext = strings.TrimSuffix(bp.PkgObj, ".a") + } + id = bp.ImportPath + + case build.IsLocalImport(path): + // "./x" -> "/this/directory/x.ext", "/this/directory/x" + noext = filepath.Join(srcDir, path) + id = noext + + case filepath.IsAbs(path): + // for completeness only - go/build.Import + // does not support absolute imports + // "/x" -> "/x.ext", "/x" + noext = path + id = path + } + + if false { // for debugging + if path != id { + fmt.Printf("%s -> %s\n", path, id) + } + } + + // try extensions + for _, ext := range pkgExts { + filename = noext + ext + f, statErr := os.Stat(filename) + if statErr == nil && !f.IsDir() { + return filename, id, nil + } + if err == nil { + err = statErr + } + } + +notfound: + if err == nil { + return "", path, fmt.Errorf("can't find import: %q", path) + } + return "", path, fmt.Errorf("can't find import: %q: %w", path, err) +} + +var pkgExts = [...]string{".a", ".o"} // a file from the build cache will have no extension + +var exportMap sync.Map // package dir → func() (string, error) + +// lookupGorootExport returns the location of the export data +// (normally found in the build cache, but located in GOROOT/pkg +// in prior Go releases) for the package located in pkgDir. +// +// (We use the package's directory instead of its import path +// mainly to simplify handling of the packages in src/vendor +// and cmd/vendor.) +func lookupGorootExport(pkgDir string) (string, error) { + f, ok := exportMap.Load(pkgDir) + if !ok { + var ( + listOnce sync.Once + exportPath string + err error + ) + f, _ = exportMap.LoadOrStore(pkgDir, func() (string, error) { + listOnce.Do(func() { + cmd := exec.Command(filepath.Join(build.Default.GOROOT, "bin", "go"), "list", "-export", "-f", "{{.Export}}", pkgDir) + cmd.Dir = build.Default.GOROOT + cmd.Env = append(os.Environ(), "PWD="+cmd.Dir, "GOROOT="+build.Default.GOROOT) + var output []byte + output, err = cmd.Output() + if err != nil { + if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 { + err = errors.New(string(ee.Stderr)) + } + return + } + + exports := strings.Split(string(bytes.TrimSpace(output)), "\n") + if len(exports) != 1 { + err = fmt.Errorf("go list reported %d exports; expected 1", len(exports)) + return + } + + exportPath = exports[0] + }) + + return exportPath, err + }) + } + + return f.(func() (string, error))() +} diff --git a/src/internal/exportdata/support.go b/src/internal/exportdata/support.go new file mode 100644 index 00000000000000..a06401db391e01 --- /dev/null +++ b/src/internal/exportdata/support.go @@ -0,0 +1,32 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains support functions for exportdata. + +package exportdata + +import ( + "bufio" + "io" + "strconv" + "strings" +) + +// Copy of cmd/internal/archive.ReadHeader. +func readArchiveHeader(b *bufio.Reader, name string) int { + // architecture-independent object file output + const HeaderSize = 60 + + var buf [HeaderSize]byte + if _, err := io.ReadFull(b, buf[:]); err != nil { + return -1 + } + aname := strings.Trim(string(buf[0:16]), " ") + if !strings.HasPrefix(aname, name) { + return -1 + } + asize := strings.Trim(string(buf[48:58]), " ") + i, _ := strconv.Atoi(asize) + return i +} diff --git a/src/internal/filepathlite/path.go b/src/internal/filepathlite/path.go index e3daa447d97632..4a3729832fa8c8 100644 --- a/src/internal/filepathlite/path.go +++ b/src/internal/filepathlite/path.go @@ -180,7 +180,7 @@ func ToSlash(path string) string { return replaceStringByte(path, Separator, '/') } -// FromSlash is filepath.ToSlash. +// FromSlash is filepath.FromSlash. func FromSlash(path string) string { if Separator == '/' { return path diff --git a/src/internal/filepathlite/path_windows.go b/src/internal/filepathlite/path_windows.go index 8f34838a98c5e5..011baa96f02432 100644 --- a/src/internal/filepathlite/path_windows.go +++ b/src/internal/filepathlite/path_windows.go @@ -7,6 +7,7 @@ package filepathlite import ( "internal/bytealg" "internal/stringslite" + "internal/syscall/windows" "syscall" ) @@ -114,13 +115,14 @@ func isReservedName(name string) bool { return true } // The path element is a reserved name with an extension. - // Some Windows versions consider this a reserved name, - // while others do not. Use FullPath to see if the name is - // reserved. - if p, _ := syscall.FullPath(name); len(p) >= 4 && p[:4] == `\\.\` { - return true + // Since Windows 11, reserved names with extensions are no + // longer reserved. For example, "CON.txt" is a valid file + // name. Use RtlIsDosDeviceName_U to see if the name is reserved. + p, err := syscall.UTF16PtrFromString(name) + if err != nil { + return false } - return false + return windows.RtlIsDosDeviceName_U(p) > 0 } func isReservedBaseName(name string) bool { @@ -297,11 +299,6 @@ func cutPath(path string) (before, after string, found bool) { return path, "", false } -// isUNC reports whether path is a UNC path. -func isUNC(path string) bool { - return len(path) > 1 && IsPathSeparator(path[0]) && IsPathSeparator(path[1]) -} - // postClean adjusts the results of Clean to avoid turning a relative path // into an absolute or rooted one. func postClean(out *lazybuf) { diff --git a/src/internal/fuzz/counters_supported.go b/src/internal/fuzz/counters_supported.go index 79e27d27e1af28..a877239186b348 100644 --- a/src/internal/fuzz/counters_supported.go +++ b/src/internal/fuzz/counters_supported.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (darwin || linux || windows || freebsd) && (amd64 || arm64) +//go:build (darwin || linux || windows || freebsd || openbsd) && (amd64 || arm64 || loong64) package fuzz diff --git a/src/internal/fuzz/counters_unsupported.go b/src/internal/fuzz/counters_unsupported.go index 287bb4bd3ceb3a..af2b56cdd857e8 100644 --- a/src/internal/fuzz/counters_unsupported.go +++ b/src/internal/fuzz/counters_unsupported.go @@ -8,7 +8,7 @@ // // If you update this constraint, also update internal/platform.FuzzInstrumented. // -//go:build !((darwin || linux || windows || freebsd) && (amd64 || arm64)) +//go:build !((darwin || linux || windows || freebsd || openbsd) && (amd64 || arm64 || loong64)) package fuzz diff --git a/src/internal/fuzz/coverage.go b/src/internal/fuzz/coverage.go index e214a7bf3e1846..8b39949b5dd622 100644 --- a/src/internal/fuzz/coverage.go +++ b/src/internal/fuzz/coverage.go @@ -23,11 +23,7 @@ func ResetCoverage() { func SnapshotCoverage() { cov := coverage() for i, b := range cov { - b |= b >> 1 - b |= b >> 2 - b |= b >> 4 - b -= b >> 1 - coverageSnapshot[i] = b + coverageSnapshot[i] = pow2Table[b] } } @@ -102,4 +98,18 @@ var ( // the 8-bit coverage counters reside in memory. They're known to cmd/link, // which specially assigns their addresses for this purpose. _counters, _ecounters [0]byte + + // lookup table for faster power of two rounding + pow2Table [256]byte ) + +func init() { + for i := range pow2Table { + b := byte(i) + b |= b >> 1 + b |= b >> 2 + b |= b >> 4 + b -= b >> 1 + pow2Table[i] = b + } +} diff --git a/src/internal/fuzz/encoding_test.go b/src/internal/fuzz/encoding_test.go index a46a3474038099..5f2af4476b8652 100644 --- a/src/internal/fuzz/encoding_test.go +++ b/src/internal/fuzz/encoding_test.go @@ -260,7 +260,6 @@ func BenchmarkMarshalCorpusFile(b *testing.B) { } for sz := 1; sz <= len(buf); sz <<= 1 { - sz := sz b.Run(strconv.Itoa(sz), func(b *testing.B) { for i := 0; i < b.N; i++ { b.SetBytes(int64(sz)) @@ -280,7 +279,6 @@ func BenchmarkUnmarshalCorpusFile(b *testing.B) { } for sz := 1; sz <= len(buf); sz <<= 1 { - sz := sz data := marshalCorpusFile(buf[:sz]) b.Run(strconv.Itoa(sz), func(b *testing.B) { for i := 0; i < b.N; i++ { diff --git a/src/internal/fuzz/minimize_test.go b/src/internal/fuzz/minimize_test.go index 2db26338967e27..79d986374f08a1 100644 --- a/src/internal/fuzz/minimize_test.go +++ b/src/internal/fuzz/minimize_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build darwin || freebsd || linux || windows +//go:build darwin || freebsd || linux || openbsd || windows package fuzz @@ -132,7 +132,6 @@ func TestMinimizeInput(t *testing.T) { } for _, tc := range cases { - tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() ws := &workerServer{ diff --git a/src/internal/fuzz/mutators_byteslice_test.go b/src/internal/fuzz/mutators_byteslice_test.go index 78869678818724..b12ef6cbcdd0bd 100644 --- a/src/internal/fuzz/mutators_byteslice_test.go +++ b/src/internal/fuzz/mutators_byteslice_test.go @@ -6,6 +6,7 @@ package fuzz import ( "bytes" + "fmt" "testing" ) @@ -33,12 +34,6 @@ func (mr *mockRand) uint32n(n uint32) uint32 { return uint32(c) % n } -func (mr *mockRand) exp2() int { - c := mr.values[mr.counter] - mr.counter++ - return c -} - func (mr *mockRand) bool() bool { b := mr.b mr.b = !mr.b @@ -184,3 +179,43 @@ func TestByteSliceMutators(t *testing.T) { }) } } + +func BenchmarkByteSliceMutators(b *testing.B) { + tests := [...]struct { + name string + mutator func(*mutator, []byte) []byte + }{ + {"RemoveBytes", byteSliceRemoveBytes}, + {"InsertRandomBytes", byteSliceInsertRandomBytes}, + {"DuplicateBytes", byteSliceDuplicateBytes}, + {"OverwriteBytes", byteSliceOverwriteBytes}, + {"BitFlip", byteSliceBitFlip}, + {"XORByte", byteSliceXORByte}, + {"SwapByte", byteSliceSwapByte}, + {"ArithmeticUint8", byteSliceArithmeticUint8}, + {"ArithmeticUint16", byteSliceArithmeticUint16}, + {"ArithmeticUint32", byteSliceArithmeticUint32}, + {"ArithmeticUint64", byteSliceArithmeticUint64}, + {"OverwriteInterestingUint8", byteSliceOverwriteInterestingUint8}, + {"OverwriteInterestingUint16", byteSliceOverwriteInterestingUint16}, + {"OverwriteInterestingUint32", byteSliceOverwriteInterestingUint32}, + {"InsertConstantBytes", byteSliceInsertConstantBytes}, + {"OverwriteConstantBytes", byteSliceOverwriteConstantBytes}, + {"ShuffleBytes", byteSliceShuffleBytes}, + {"SwapBytes", byteSliceSwapBytes}, + } + + for _, tc := range tests { + b.Run(tc.name, func(b *testing.B) { + for size := 64; size <= 1024; size *= 2 { + b.Run(fmt.Sprintf("%d", size), func(b *testing.B) { + m := &mutator{r: newPcgRand()} + input := make([]byte, size) + for i := 0; i < b.N; i++ { + tc.mutator(m, input) + } + }) + } + }) + } +} diff --git a/src/internal/fuzz/pcg.go b/src/internal/fuzz/pcg.go index dc07b9f5bdc437..b8251043f1c129 100644 --- a/src/internal/fuzz/pcg.go +++ b/src/internal/fuzz/pcg.go @@ -17,7 +17,6 @@ type mutatorRand interface { uint32() uint32 intn(int) int uint32n(uint32) uint32 - exp2() int bool() bool save(randState, randInc *uint64) @@ -123,11 +122,6 @@ func (r *pcgRand) uint32n(n uint32) uint32 { return uint32(prod >> 32) } -// exp2 generates n with probability 1/2^(n+1). -func (r *pcgRand) exp2() int { - return bits.TrailingZeros32(r.uint32()) -} - // bool generates a random bool. func (r *pcgRand) bool() bool { return r.uint32()&1 == 0 diff --git a/src/internal/fuzz/sys_posix.go b/src/internal/fuzz/sys_posix.go index fec6054f671cc8..40d3771c2ac0e6 100644 --- a/src/internal/fuzz/sys_posix.go +++ b/src/internal/fuzz/sys_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build darwin || freebsd || linux +//go:build darwin || freebsd || linux || openbsd package fuzz diff --git a/src/internal/fuzz/sys_unimplemented.go b/src/internal/fuzz/sys_unimplemented.go index 8687c1f963e1ba..30766ba525f2c8 100644 --- a/src/internal/fuzz/sys_unimplemented.go +++ b/src/internal/fuzz/sys_unimplemented.go @@ -4,7 +4,7 @@ // If you update this constraint, also update internal/platform.FuzzSupported. // -//go:build !darwin && !freebsd && !linux && !windows +//go:build !darwin && !freebsd && !linux && !openbsd && !windows package fuzz diff --git a/src/internal/fuzz/worker.go b/src/internal/fuzz/worker.go index e8a74217789189..9ee2f27296dc3a 100644 --- a/src/internal/fuzz/worker.go +++ b/src/internal/fuzz/worker.go @@ -682,7 +682,7 @@ func (ws *workerServer) serve(ctx context.Context) error { } // chainedMutations is how many mutations are applied before the worker -// resets the input to it's original state. +// resets the input to its original state. // NOTE: this number was picked without much thought. It is low enough that // it seems to create a significant diversity in mutated inputs. We may want // to consider looking into this more closely once we have a proper performance diff --git a/src/internal/fuzz/worker_test.go b/src/internal/fuzz/worker_test.go index d0b21da7838643..9420248d2c0d60 100644 --- a/src/internal/fuzz/worker_test.go +++ b/src/internal/fuzz/worker_test.go @@ -182,7 +182,6 @@ func BenchmarkWorkerMinimize(b *testing.B) { bytes := make([]byte, 1024) ctx := context.Background() for sz := 1; sz <= len(bytes); sz <<= 1 { - sz := sz input := []any{bytes[:sz]} encodedVals := marshalCorpusFile(input...) mem = <-ws.memMu diff --git a/src/internal/goarch/goarch.go b/src/internal/goarch/goarch.go index 3dda62fadc21f3..efcf298d3b7b35 100644 --- a/src/internal/goarch/goarch.go +++ b/src/internal/goarch/goarch.go @@ -12,6 +12,8 @@ package goarch // //go:generate go run gengoarch.go +// ArchFamilyType represents a family of one or more related architectures. +// For example, ppc64 and ppc64le are both members of the PPC64 family. type ArchFamilyType int const ( @@ -32,6 +34,9 @@ const ( // It is also the size of the machine's native word size (that is, 4 on 32-bit systems, 8 on 64-bit). const PtrSize = 4 << (^uintptr(0) >> 63) +// PtrBits is bit width of a pointer. +const PtrBits = PtrSize * 8 + // ArchFamily is the architecture family (AMD64, ARM, ...) const ArchFamily ArchFamilyType = _ArchFamily diff --git a/src/internal/godebug/godebug.go b/src/internal/godebug/godebug.go index 0756d313e6b9e0..8c66a8a19af4ba 100644 --- a/src/internal/godebug/godebug.go +++ b/src/internal/godebug/godebug.go @@ -237,8 +237,14 @@ func update(def, env string) { // Update all the cached values, creating new ones as needed. // We parse the environment variable first, so that any settings it has // are already locked in place (did[name] = true) before we consider - // the defaults. + // the defaults. Existing immutable settings are always locked. did := make(map[string]bool) + cache.Range(func(name, s any) bool { + if info := s.(*setting).info; info != nil && info.Immutable { + did[name.(string)] = true + } + return true + }) parse(did, env) parse(did, def) diff --git a/src/internal/godebug/godebug_test.go b/src/internal/godebug/godebug_test.go index 69296303560505..47f4cc27610216 100644 --- a/src/internal/godebug/godebug_test.go +++ b/src/internal/godebug/godebug_test.go @@ -78,7 +78,7 @@ func TestPanicNilRace(t *testing.T) { t.Skip("Skipping test intended for use with -race.") } if os.Getenv("GODEBUG") != "panicnil=1" { - cmd := testenv.CleanCmdEnv(testenv.Command(t, os.Args[0], "-test.run=^TestPanicNilRace$", "-test.v", "-test.parallel=2", "-test.count=1")) + cmd := testenv.CleanCmdEnv(testenv.Command(t, testenv.Executable(t), "-test.run=^TestPanicNilRace$", "-test.v", "-test.parallel=2", "-test.count=1")) cmd.Env = append(cmd.Env, "GODEBUG=panicnil=1") out, err := cmd.CombinedOutput() t.Logf("output:\n%s", out) @@ -101,14 +101,17 @@ func TestPanicNilRace(t *testing.T) { } func TestCmdBisect(t *testing.T) { - testenv.MustHaveGoBuild(t) - out, err := exec.Command("go", "run", "cmd/vendor/golang.org/x/tools/cmd/bisect", "GODEBUG=buggy=1#PATTERN", os.Args[0], "-test.run=^TestBisectTestCase$").CombinedOutput() + testenv.MustHaveGoRun(t) + out, err := exec.Command(testenv.GoToolPath(t), "run", "cmd/vendor/golang.org/x/tools/cmd/bisect", "GODEBUG=buggy=1#PATTERN", os.Args[0], "-test.run=^TestBisectTestCase$").CombinedOutput() if err != nil { t.Fatalf("exec bisect: %v\n%s", err, out) } var want []string src, err := os.ReadFile("godebug_test.go") + if err != nil { + t.Fatal(err) + } for i, line := range strings.Split(string(src), "\n") { if strings.Contains(line, "BISECT"+" "+"BUG") { want = append(want, fmt.Sprintf("godebug_test.go:%d", i+1)) @@ -159,3 +162,32 @@ func TestBisectTestCase(t *testing.T) { } } } + +func TestImmutable(t *testing.T) { + defer func(godebug string) { + os.Setenv("GODEBUG", godebug) + }(os.Getenv("GODEBUG")) + + setting := New("fips140") + value := setting.Value() + + os.Setenv("GODEBUG", "fips140=off") + if setting.Value() != value { + t.Errorf("Value() changed after setting GODEBUG=fips140=off") + } + + os.Setenv("GODEBUG", "fips140=on") + if setting.Value() != value { + t.Errorf("Value() changed after setting GODEBUG=fips140=on") + } + + os.Setenv("GODEBUG", "fips140=") + if setting.Value() != value { + t.Errorf("Value() changed after setting GODEBUG=fips140=") + } + + os.Setenv("GODEBUG", "") + if setting.Value() != value { + t.Errorf("Value() changed after setting GODEBUG=") + } +} diff --git a/src/internal/godebugs/godebugs_test.go b/src/internal/godebugs/godebugs_test.go index 046193b5c6b13b..168acc134aa753 100644 --- a/src/internal/godebugs/godebugs_test.go +++ b/src/internal/godebugs/godebugs_test.go @@ -46,7 +46,8 @@ func TestAll(t *testing.T) { if info.Old != "" && info.Changed == 0 { t.Errorf("Name=%s has Old, missing Changed", info.Name) } - if !strings.Contains(doc, "`"+info.Name+"`") { + if !strings.Contains(doc, "`"+info.Name+"`") && + !strings.Contains(doc, "`"+info.Name+"=") { t.Errorf("Name=%s not documented in doc/godebug.md", info.Name) } if !info.Opaque && !incs[info.Name] { diff --git a/src/internal/godebugs/table.go b/src/internal/godebugs/table.go index a802ac9c3708d1..852305e8553aab 100644 --- a/src/internal/godebugs/table.go +++ b/src/internal/godebugs/table.go @@ -9,11 +9,12 @@ package godebugs // An Info describes a single known GODEBUG setting. type Info struct { - Name string // name of the setting ("panicnil") - Package string // package that uses the setting ("runtime") - Changed int // minor version when default changed, if any; 21 means Go 1.21 - Old string // value that restores behavior prior to Changed - Opaque bool // setting does not export information to runtime/metrics using [internal/godebug.Setting.IncNonDefault] + Name string // name of the setting ("panicnil") + Package string // package that uses the setting ("runtime") + Changed int // minor version when default changed, if any; 21 means Go 1.21 + Old string // value that restores behavior prior to Changed + Opaque bool // setting does not export information to runtime/metrics using [internal/godebug.Setting.IncNonDefault] + Immutable bool // setting cannot be changed after program start } // All is the table of known settings, sorted by Name. @@ -25,15 +26,23 @@ type Info struct { // Note: After adding entries to this table, update the list in doc/godebug.md as well. // (Otherwise the test in this package will fail.) var All = []Info{ + {Name: "allowmultiplevcs", Package: "cmd/go"}, {Name: "asynctimerchan", Package: "time", Changed: 23, Old: "1"}, + {Name: "containermaxprocs", Package: "runtime", Changed: 25, Old: "0"}, + {Name: "dataindependenttiming", Package: "crypto/subtle", Opaque: true}, + {Name: "decoratemappings", Package: "runtime", Opaque: true, Changed: 25, Old: "0"}, + {Name: "embedfollowsymlinks", Package: "cmd/go"}, {Name: "execerrdot", Package: "os/exec"}, + {Name: "fips140", Package: "crypto/fips140", Opaque: true, Immutable: true}, {Name: "gocachehash", Package: "cmd/go"}, {Name: "gocachetest", Package: "cmd/go"}, {Name: "gocacheverify", Package: "cmd/go"}, + {Name: "gotestjsonbuildtext", Package: "cmd/go", Changed: 24, Old: "1"}, {Name: "gotypesalias", Package: "go/types", Changed: 23, Old: "0"}, {Name: "http2client", Package: "net/http"}, {Name: "http2debug", Package: "net/http", Opaque: true}, {Name: "http2server", Package: "net/http"}, + {Name: "httpcookiemaxnum", Package: "net/http", Changed: 24, Old: "0"}, {Name: "httplaxcontentlength", Package: "net/http", Changed: 22, Old: "1"}, {Name: "httpmuxgo121", Package: "net/http", Changed: 22, Old: "1"}, {Name: "httpservecontentkeepheaders", Package: "net/http", Changed: 23, Old: "1"}, @@ -42,25 +51,30 @@ var All = []Info{ //{Name: "multipartfiles", Package: "mime/multipart"}, {Name: "multipartmaxheaders", Package: "mime/multipart"}, {Name: "multipartmaxparts", Package: "mime/multipart"}, - {Name: "multipathtcp", Package: "net"}, + {Name: "multipathtcp", Package: "net", Changed: 24, Old: "0"}, {Name: "netdns", Package: "net", Opaque: true}, {Name: "netedns0", Package: "net", Changed: 19, Old: "0"}, {Name: "panicnil", Package: "runtime", Changed: 21, Old: "1"}, {Name: "randautoseed", Package: "math/rand"}, + {Name: "randseednop", Package: "math/rand", Changed: 24, Old: "0"}, + {Name: "rsa1024min", Package: "crypto/rsa", Changed: 24, Old: "0"}, {Name: "tarinsecurepath", Package: "archive/tar"}, {Name: "tls10server", Package: "crypto/tls", Changed: 22, Old: "1"}, {Name: "tls3des", Package: "crypto/tls", Changed: 23, Old: "1"}, - {Name: "tlskyber", Package: "crypto/tls", Changed: 23, Old: "0", Opaque: true}, {Name: "tlsmaxrsasize", Package: "crypto/tls"}, + {Name: "tlsmlkem", Package: "crypto/tls", Changed: 24, Old: "0", Opaque: true}, {Name: "tlsrsakex", Package: "crypto/tls", Changed: 22, Old: "1"}, + {Name: "tlssha1", Package: "crypto/tls", Changed: 25, Old: "1"}, {Name: "tlsunsafeekm", Package: "crypto/tls", Changed: 22, Old: "1"}, - {Name: "winreadlinkvolume", Package: "os", Changed: 22, Old: "0"}, - {Name: "winsymlink", Package: "os", Changed: 22, Old: "0"}, + {Name: "updatemaxprocs", Package: "runtime", Changed: 25, Old: "0"}, + {Name: "winreadlinkvolume", Package: "os", Changed: 23, Old: "0"}, + {Name: "winsymlink", Package: "os", Changed: 23, Old: "0"}, {Name: "x509keypairleaf", Package: "crypto/tls", Changed: 23, Old: "0"}, {Name: "x509negativeserial", Package: "crypto/x509", Changed: 23, Old: "1"}, - {Name: "x509sha1", Package: "crypto/x509"}, + {Name: "x509rsacrt", Package: "crypto/x509", Changed: 24, Old: "0"}, + {Name: "x509sha256skid", Package: "crypto/x509", Changed: 25, Old: "0"}, {Name: "x509usefallbackroots", Package: "crypto/x509"}, - {Name: "x509usepolicies", Package: "crypto/x509"}, + {Name: "x509usepolicies", Package: "crypto/x509", Changed: 24, Old: "0"}, {Name: "zipinsecurepath", Package: "archive/zip"}, } diff --git a/src/internal/goexperiment/exp_aliastypeparams_off.go b/src/internal/goexperiment/exp_aliastypeparams_off.go deleted file mode 100644 index 620d34ec795a0f..00000000000000 --- a/src/internal/goexperiment/exp_aliastypeparams_off.go +++ /dev/null @@ -1,8 +0,0 @@ -// Code generated by mkconsts.go. DO NOT EDIT. - -//go:build !goexperiment.aliastypeparams - -package goexperiment - -const AliasTypeParams = false -const AliasTypeParamsInt = 0 diff --git a/src/internal/goexperiment/exp_aliastypeparams_on.go b/src/internal/goexperiment/exp_aliastypeparams_on.go deleted file mode 100644 index 8f6872cdcd361b..00000000000000 --- a/src/internal/goexperiment/exp_aliastypeparams_on.go +++ /dev/null @@ -1,8 +0,0 @@ -// Code generated by mkconsts.go. DO NOT EDIT. - -//go:build goexperiment.aliastypeparams - -package goexperiment - -const AliasTypeParams = true -const AliasTypeParamsInt = 1 diff --git a/src/internal/goexperiment/exp_cacheprog_off.go b/src/internal/goexperiment/exp_cacheprog_off.go deleted file mode 100644 index 276855c7d6f587..00000000000000 --- a/src/internal/goexperiment/exp_cacheprog_off.go +++ /dev/null @@ -1,8 +0,0 @@ -// Code generated by mkconsts.go. DO NOT EDIT. - -//go:build !goexperiment.cacheprog - -package goexperiment - -const CacheProg = false -const CacheProgInt = 0 diff --git a/src/internal/goexperiment/exp_cacheprog_on.go b/src/internal/goexperiment/exp_cacheprog_on.go deleted file mode 100644 index b959dd68b9f22e..00000000000000 --- a/src/internal/goexperiment/exp_cacheprog_on.go +++ /dev/null @@ -1,8 +0,0 @@ -// Code generated by mkconsts.go. DO NOT EDIT. - -//go:build goexperiment.cacheprog - -package goexperiment - -const CacheProg = true -const CacheProgInt = 1 diff --git a/src/internal/goexperiment/exp_coverageredesign_off.go b/src/internal/goexperiment/exp_coverageredesign_off.go deleted file mode 100644 index 2c33177322d1f4..00000000000000 --- a/src/internal/goexperiment/exp_coverageredesign_off.go +++ /dev/null @@ -1,8 +0,0 @@ -// Code generated by mkconsts.go. DO NOT EDIT. - -//go:build !goexperiment.coverageredesign - -package goexperiment - -const CoverageRedesign = false -const CoverageRedesignInt = 0 diff --git a/src/internal/goexperiment/exp_coverageredesign_on.go b/src/internal/goexperiment/exp_coverageredesign_on.go deleted file mode 100644 index 3fc6c2f70a9ab5..00000000000000 --- a/src/internal/goexperiment/exp_coverageredesign_on.go +++ /dev/null @@ -1,8 +0,0 @@ -// Code generated by mkconsts.go. DO NOT EDIT. - -//go:build goexperiment.coverageredesign - -package goexperiment - -const CoverageRedesign = true -const CoverageRedesignInt = 1 diff --git a/src/internal/goexperiment/exp_dwarf5_off.go b/src/internal/goexperiment/exp_dwarf5_off.go new file mode 100644 index 00000000000000..b89dd9f2d98d75 --- /dev/null +++ b/src/internal/goexperiment/exp_dwarf5_off.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build !goexperiment.dwarf5 + +package goexperiment + +const Dwarf5 = false +const Dwarf5Int = 0 diff --git a/src/internal/goexperiment/exp_dwarf5_on.go b/src/internal/goexperiment/exp_dwarf5_on.go new file mode 100644 index 00000000000000..02b28d74e249d4 --- /dev/null +++ b/src/internal/goexperiment/exp_dwarf5_on.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build goexperiment.dwarf5 + +package goexperiment + +const Dwarf5 = true +const Dwarf5Int = 1 diff --git a/src/internal/goexperiment/exp_goroutineleakprofile_off.go b/src/internal/goexperiment/exp_goroutineleakprofile_off.go new file mode 100644 index 00000000000000..63eafe9e6c74db --- /dev/null +++ b/src/internal/goexperiment/exp_goroutineleakprofile_off.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build !goexperiment.goroutineleakprofile + +package goexperiment + +const GoroutineLeakProfile = false +const GoroutineLeakProfileInt = 0 diff --git a/src/internal/goexperiment/exp_goroutineleakprofile_on.go b/src/internal/goexperiment/exp_goroutineleakprofile_on.go new file mode 100644 index 00000000000000..28a662eceb46f8 --- /dev/null +++ b/src/internal/goexperiment/exp_goroutineleakprofile_on.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build goexperiment.goroutineleakprofile + +package goexperiment + +const GoroutineLeakProfile = true +const GoroutineLeakProfileInt = 1 diff --git a/src/internal/goexperiment/exp_greenteagc_off.go b/src/internal/goexperiment/exp_greenteagc_off.go new file mode 100644 index 00000000000000..dce9d8c997434c --- /dev/null +++ b/src/internal/goexperiment/exp_greenteagc_off.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build !goexperiment.greenteagc + +package goexperiment + +const GreenTeaGC = false +const GreenTeaGCInt = 0 diff --git a/src/internal/goexperiment/exp_greenteagc_on.go b/src/internal/goexperiment/exp_greenteagc_on.go new file mode 100644 index 00000000000000..10a007d757c148 --- /dev/null +++ b/src/internal/goexperiment/exp_greenteagc_on.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build goexperiment.greenteagc + +package goexperiment + +const GreenTeaGC = true +const GreenTeaGCInt = 1 diff --git a/src/internal/goexperiment/exp_jsonv2_off.go b/src/internal/goexperiment/exp_jsonv2_off.go new file mode 100644 index 00000000000000..a973a35ec2fe62 --- /dev/null +++ b/src/internal/goexperiment/exp_jsonv2_off.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build !goexperiment.jsonv2 + +package goexperiment + +const JSONv2 = false +const JSONv2Int = 0 diff --git a/src/internal/goexperiment/exp_jsonv2_on.go b/src/internal/goexperiment/exp_jsonv2_on.go new file mode 100644 index 00000000000000..119d086874f834 --- /dev/null +++ b/src/internal/goexperiment/exp_jsonv2_on.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build goexperiment.jsonv2 + +package goexperiment + +const JSONv2 = true +const JSONv2Int = 1 diff --git a/src/internal/goexperiment/exp_randomizedheapbase64_off.go b/src/internal/goexperiment/exp_randomizedheapbase64_off.go new file mode 100644 index 00000000000000..0a578535a44442 --- /dev/null +++ b/src/internal/goexperiment/exp_randomizedheapbase64_off.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build !goexperiment.randomizedheapbase64 + +package goexperiment + +const RandomizedHeapBase64 = false +const RandomizedHeapBase64Int = 0 diff --git a/src/internal/goexperiment/exp_randomizedheapbase64_on.go b/src/internal/goexperiment/exp_randomizedheapbase64_on.go new file mode 100644 index 00000000000000..10d59c702884a5 --- /dev/null +++ b/src/internal/goexperiment/exp_randomizedheapbase64_on.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build goexperiment.randomizedheapbase64 + +package goexperiment + +const RandomizedHeapBase64 = true +const RandomizedHeapBase64Int = 1 diff --git a/src/internal/goexperiment/exp_rangefunc_off.go b/src/internal/goexperiment/exp_rangefunc_off.go deleted file mode 100644 index fc028205db9719..00000000000000 --- a/src/internal/goexperiment/exp_rangefunc_off.go +++ /dev/null @@ -1,8 +0,0 @@ -// Code generated by mkconsts.go. DO NOT EDIT. - -//go:build !goexperiment.rangefunc - -package goexperiment - -const RangeFunc = false -const RangeFuncInt = 0 diff --git a/src/internal/goexperiment/exp_rangefunc_on.go b/src/internal/goexperiment/exp_rangefunc_on.go deleted file mode 100644 index 25e7bd361b75f5..00000000000000 --- a/src/internal/goexperiment/exp_rangefunc_on.go +++ /dev/null @@ -1,8 +0,0 @@ -// Code generated by mkconsts.go. DO NOT EDIT. - -//go:build goexperiment.rangefunc - -package goexperiment - -const RangeFunc = true -const RangeFuncInt = 1 diff --git a/src/internal/goexperiment/exp_runtimefree_off.go b/src/internal/goexperiment/exp_runtimefree_off.go new file mode 100644 index 00000000000000..3affe434f2fbc2 --- /dev/null +++ b/src/internal/goexperiment/exp_runtimefree_off.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build !goexperiment.runtimefree + +package goexperiment + +const RuntimeFree = false +const RuntimeFreeInt = 0 diff --git a/src/internal/goexperiment/exp_runtimefree_on.go b/src/internal/goexperiment/exp_runtimefree_on.go new file mode 100644 index 00000000000000..176278b5425534 --- /dev/null +++ b/src/internal/goexperiment/exp_runtimefree_on.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build goexperiment.runtimefree + +package goexperiment + +const RuntimeFree = true +const RuntimeFreeInt = 1 diff --git a/src/internal/goexperiment/exp_sizespecializedmalloc_off.go b/src/internal/goexperiment/exp_sizespecializedmalloc_off.go new file mode 100644 index 00000000000000..2642a12d798607 --- /dev/null +++ b/src/internal/goexperiment/exp_sizespecializedmalloc_off.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build !goexperiment.sizespecializedmalloc + +package goexperiment + +const SizeSpecializedMalloc = false +const SizeSpecializedMallocInt = 0 diff --git a/src/internal/goexperiment/exp_sizespecializedmalloc_on.go b/src/internal/goexperiment/exp_sizespecializedmalloc_on.go new file mode 100644 index 00000000000000..a135cc889e84a7 --- /dev/null +++ b/src/internal/goexperiment/exp_sizespecializedmalloc_on.go @@ -0,0 +1,8 @@ +// Code generated by mkconsts.go. DO NOT EDIT. + +//go:build goexperiment.sizespecializedmalloc + +package goexperiment + +const SizeSpecializedMalloc = true +const SizeSpecializedMallocInt = 1 diff --git a/src/internal/goexperiment/flags.go b/src/internal/goexperiment/flags.go index 3f9c5af68e36eb..b3a3c8a4974975 100644 --- a/src/internal/goexperiment/flags.go +++ b/src/internal/goexperiment/flags.go @@ -14,32 +14,32 @@ // // Experiments are exposed to the build in the following ways: // -// - Build tag goexperiment.x is set if experiment x (lower case) is -// enabled. +// - Build tag goexperiment.x is set if experiment x (lower case) is +// enabled. // -// - For each experiment x (in camel case), this package contains a -// boolean constant x and an integer constant xInt. +// - For each experiment x (in camel case), this package contains a +// boolean constant x and an integer constant xInt. // -// - In runtime assembly, the macro GOEXPERIMENT_x is defined if -// experiment x (lower case) is enabled. +// - In runtime assembly, the macro GOEXPERIMENT_x is defined if +// experiment x (lower case) is enabled. // // In the toolchain, the set of experiments enabled for the current // build should be accessed via objabi.Experiment. // -// The set of experiments is included in the output of runtime.Version() +// The set of experiments is included in the output of [runtime.Version]() // and "go version " if it differs from the default experiments. // // For the set of experiments supported by the current toolchain, see // "go doc goexperiment.Flags". // -// Note that this package defines the set of experiments (in Flags) +// Note that this package defines the set of experiments (in [Flags]) // and records the experiments that were enabled when the package // was compiled (as boolean and integer constants). // // Note especially that this package does not itself change behavior // at run time based on the GOEXPERIMENT variable. // The code used in builds to interpret the GOEXPERIMENT variable -// is in the separate package internal/buildcfg. +// is in the separate package [internal/buildcfg]. package goexperiment //go:generate go run mkconsts.go @@ -51,7 +51,7 @@ package goexperiment // tags, experiments use the strings.ToLower of their field name. // // For the baseline experimental configuration, see -// objabi.experimentBaseline. +// [internal/buildcfg.Experiment]. // // If you change this struct definition, run "go generate". type Flags struct { @@ -83,10 +83,6 @@ type Flags struct { // by default. HeapMinimum512KiB bool - // CoverageRedesign enables the new compiler-based code coverage - // tooling. - CoverageRedesign bool - // Arenas causes the "arena" standard library package to be visible // to the outside world. Arenas bool @@ -100,19 +96,29 @@ type Flags struct { // copy of the iteration variable. LoopVar bool - // CacheProg adds support to cmd/go to use a child process to implement - // the build cache; see https://github.com/golang/go/issues/59719. - CacheProg bool - // NewInliner enables a new+improved version of the function // inlining phase within the Go compiler. NewInliner bool - // RangeFunc enables range over func. - RangeFunc bool + // Dwarf5 enables DWARF version 5 debug info generation. + Dwarf5 bool + + // JSONv2 enables the json/v2 package. + JSONv2 bool + + // GreenTeaGC enables the Green Tea GC implementation. + GreenTeaGC bool + + // RandomizedHeapBase enables heap base address randomization on 64-bit + // platforms. + RandomizedHeapBase64 bool + + // RuntimeFree enables the runtime to free and reuse memory more eagerly in some circumstances with compiler help. + RuntimeFree bool + + // SizeSpecializedMalloc enables malloc implementations that are specialized per size class. + SizeSpecializedMalloc bool - // AliasTypeParams enables type parameters for alias types. - // Requires that gotypesalias=1 is set with GODEBUG. - // This flag will be removed with Go 1.24. - AliasTypeParams bool + // GoroutineLeakProfile enables the collection of goroutine leak profiles. + GoroutineLeakProfile bool } diff --git a/src/internal/goos/gengoos.go b/src/internal/goos/gengoos.go index aba0d3c3356f4f..e0d4d38e898e89 100644 --- a/src/internal/goos/gengoos.go +++ b/src/internal/goos/gengoos.go @@ -17,7 +17,7 @@ import ( var gooses []string func main() { - data, err := os.ReadFile("../../internal/syslist/syslist..go") + data, err := os.ReadFile("../../internal/syslist/syslist.go") if err != nil { log.Fatal(err) } diff --git a/src/internal/goroot/gc.go b/src/internal/goroot/gc.go index 6b37dfa4c75efd..534ad57e709fa6 100644 --- a/src/internal/goroot/gc.go +++ b/src/internal/goroot/gc.go @@ -37,7 +37,7 @@ func IsStandardPackage(goroot, compiler, path string) bool { } } -// gccgoSearch holds the gccgo search directories. +// gccgoDirs holds the gccgo search directories. type gccgoDirs struct { once sync.Once dirs []string @@ -77,8 +77,8 @@ func (gd *gccgoDirs) init() { const prefix = "libraries: =" var dirs []string for _, dirEntry := range dirsEntries { - if strings.HasPrefix(dirEntry, prefix) { - dirs = filepath.SplitList(strings.TrimPrefix(dirEntry, prefix)) + if after, ok := strings.CutPrefix(dirEntry, prefix); ok { + dirs = filepath.SplitList(after) break } } diff --git a/src/internal/goversion/goversion.go b/src/internal/goversion/goversion.go index de2bcf4c82eb33..d8bfe7e085b62e 100644 --- a/src/internal/goversion/goversion.go +++ b/src/internal/goversion/goversion.go @@ -9,4 +9,4 @@ package goversion // // It should be updated at the start of each development cycle to be // the version of the next Go 1.x release. See go.dev/issue/40705. -const Version = 24 +const Version = 26 diff --git a/src/internal/pkgbits/decoder.go b/src/internal/pkgbits/decoder.go index 4fe024d4f19103..f9e37b9e1e4f25 100644 --- a/src/internal/pkgbits/decoder.go +++ b/src/internal/pkgbits/decoder.go @@ -21,7 +21,7 @@ import ( // export data. type PkgDecoder struct { // version is the file format version. - version uint32 + version Version // sync indicates whether the file uses sync markers. sync bool @@ -54,7 +54,7 @@ type PkgDecoder struct { // (or 0, if K==0) and end at elemEndsEnds[K]. elemEndsEnds [numRelocs]uint32 - scratchRelocEnt []RelocEnt + scratchRelocEnt []RefTableEntry } // PkgPath returns the package path for the package @@ -68,8 +68,6 @@ func (pr *PkgDecoder) SyncMarkers() bool { return pr.sync } // NewPkgDecoder returns a PkgDecoder initialized to read the Unified // IR export data from input. pkgPath is the package path for the // compilation unit that produced the export data. -// -// TODO(mdempsky): Remove pkgPath parameter; unneeded since CL 391014. func NewPkgDecoder(pkgPath, input string) PkgDecoder { pr := PkgDecoder{ pkgPath: pkgPath, @@ -80,14 +78,15 @@ func NewPkgDecoder(pkgPath, input string) PkgDecoder { r := strings.NewReader(input) - assert(binary.Read(r, binary.LittleEndian, &pr.version) == nil) + var ver uint32 + assert(binary.Read(r, binary.LittleEndian, &ver) == nil) + pr.version = Version(ver) - switch pr.version { - default: - panic(fmt.Errorf("unsupported version: %v", pr.version)) - case 0: - // no flags - case 1: + if pr.version >= numVersions { + panic(fmt.Errorf("cannot decode %q, export data version %d is greater than maximum supported version %d", pkgPath, pr.version, numVersions-1)) + } + + if pr.version.Has(Flags) { var flags uint32 assert(binary.Read(r, binary.LittleEndian, &flags) == nil) pr.sync = flags&flagSyncMarkers != 0 @@ -102,13 +101,15 @@ func NewPkgDecoder(pkgPath, input string) PkgDecoder { assert(err == nil) pr.elemData = input[pos:] - assert(len(pr.elemData)-8 == int(pr.elemEnds[len(pr.elemEnds)-1])) + + const fingerprintSize = 8 + assert(len(pr.elemData)-fingerprintSize == int(pr.elemEnds[len(pr.elemEnds)-1])) return pr } // NumElems returns the number of elements in section k. -func (pr *PkgDecoder) NumElems(k RelocKind) int { +func (pr *PkgDecoder) NumElems(k SectionKind) int { count := int(pr.elemEndsEnds[k]) if k > 0 { count -= int(pr.elemEndsEnds[k-1]) @@ -130,20 +131,20 @@ func (pr *PkgDecoder) Fingerprint() [8]byte { // AbsIdx returns the absolute index for the given (section, index) // pair. -func (pr *PkgDecoder) AbsIdx(k RelocKind, idx Index) int { +func (pr *PkgDecoder) AbsIdx(k SectionKind, idx RelElemIdx) int { absIdx := int(idx) if k > 0 { absIdx += int(pr.elemEndsEnds[k-1]) } if absIdx >= int(pr.elemEndsEnds[k]) { - errorf("%v:%v is out of bounds; %v", k, idx, pr.elemEndsEnds) + panicf("%v:%v is out of bounds; %v", k, idx, pr.elemEndsEnds) } return absIdx } // DataIdx returns the raw element bitstream for the given (section, // index) pair. -func (pr *PkgDecoder) DataIdx(k RelocKind, idx Index) string { +func (pr *PkgDecoder) DataIdx(k SectionKind, idx RelElemIdx) string { absIdx := pr.AbsIdx(k, idx) var start uint32 @@ -156,13 +157,13 @@ func (pr *PkgDecoder) DataIdx(k RelocKind, idx Index) string { } // StringIdx returns the string value for the given string index. -func (pr *PkgDecoder) StringIdx(idx Index) string { - return pr.DataIdx(RelocString, idx) +func (pr *PkgDecoder) StringIdx(idx RelElemIdx) string { + return pr.DataIdx(SectionString, idx) } // NewDecoder returns a Decoder for the given (section, index) pair, // and decodes the given SyncMarker from the element bitstream. -func (pr *PkgDecoder) NewDecoder(k RelocKind, idx Index, marker SyncMarker) Decoder { +func (pr *PkgDecoder) NewDecoder(k SectionKind, idx RelElemIdx, marker SyncMarker) Decoder { r := pr.NewDecoderRaw(k, idx) r.Sync(marker) return r @@ -172,7 +173,7 @@ func (pr *PkgDecoder) NewDecoder(k RelocKind, idx Index, marker SyncMarker) Deco // and decodes the given SyncMarker from the element bitstream. // If possible the Decoder should be RetireDecoder'd when it is no longer // needed, this will avoid heap allocations. -func (pr *PkgDecoder) TempDecoder(k RelocKind, idx Index, marker SyncMarker) Decoder { +func (pr *PkgDecoder) TempDecoder(k SectionKind, idx RelElemIdx, marker SyncMarker) Decoder { r := pr.TempDecoderRaw(k, idx) r.Sync(marker) return r @@ -186,7 +187,7 @@ func (pr *PkgDecoder) RetireDecoder(d *Decoder) { // NewDecoderRaw returns a Decoder for the given (section, index) pair. // // Most callers should use NewDecoder instead. -func (pr *PkgDecoder) NewDecoderRaw(k RelocKind, idx Index) Decoder { +func (pr *PkgDecoder) NewDecoderRaw(k SectionKind, idx RelElemIdx) Decoder { r := Decoder{ common: pr, k: k, @@ -195,16 +196,16 @@ func (pr *PkgDecoder) NewDecoderRaw(k RelocKind, idx Index) Decoder { r.Data.Reset(pr.DataIdx(k, idx)) r.Sync(SyncRelocs) - r.Relocs = make([]RelocEnt, r.Len()) + r.Relocs = make([]RefTableEntry, r.Len()) for i := range r.Relocs { r.Sync(SyncReloc) - r.Relocs[i] = RelocEnt{RelocKind(r.Len()), Index(r.Len())} + r.Relocs[i] = RefTableEntry{SectionKind(r.Len()), RelElemIdx(r.Len())} } return r } -func (pr *PkgDecoder) TempDecoderRaw(k RelocKind, idx Index) Decoder { +func (pr *PkgDecoder) TempDecoderRaw(k SectionKind, idx RelElemIdx) Decoder { r := Decoder{ common: pr, k: k, @@ -218,11 +219,11 @@ func (pr *PkgDecoder) TempDecoderRaw(k RelocKind, idx Index) Decoder { r.Relocs = pr.scratchRelocEnt[:l] pr.scratchRelocEnt = nil } else { - r.Relocs = make([]RelocEnt, l) + r.Relocs = make([]RefTableEntry, l) } for i := range r.Relocs { r.Sync(SyncReloc) - r.Relocs[i] = RelocEnt{RelocKind(r.Len()), Index(r.Len())} + r.Relocs[i] = RefTableEntry{SectionKind(r.Len()), RelElemIdx(r.Len())} } return r @@ -233,16 +234,16 @@ func (pr *PkgDecoder) TempDecoderRaw(k RelocKind, idx Index) Decoder { type Decoder struct { common *PkgDecoder - Relocs []RelocEnt + Relocs []RefTableEntry Data strings.Reader - k RelocKind - Idx Index + k SectionKind + Idx RelElemIdx } func (r *Decoder) checkErr(err error) { if err != nil { - errorf("unexpected decoding error: %w", err) + panicf("unexpected decoding error: %w", err) } } @@ -291,7 +292,7 @@ func (r *Decoder) rawVarint() int64 { return x } -func (r *Decoder) rawReloc(k RelocKind, idx int) Index { +func (r *Decoder) rawReloc(k SectionKind, idx int) RelElemIdx { e := r.Relocs[idx] assert(e.Kind == k) return e.Idx @@ -340,7 +341,7 @@ func (r *Decoder) Sync(mWant SyncMarker) { fmt.Printf("\t[stack trace unavailable; recompile package %q with -d=syncframes]\n", r.common.pkgPath) } for _, pc := range writerPCs { - fmt.Printf("\t%s\n", r.common.StringIdx(r.rawReloc(RelocString, pc))) + fmt.Printf("\t%s\n", r.common.StringIdx(r.rawReloc(SectionString, pc))) } fmt.Printf("\nexpected %v, reading at:\n", mWant) @@ -371,7 +372,7 @@ func (r *Decoder) Int64() int64 { return r.rawVarint() } -// Int64 decodes and returns a uint64 value from the element bitstream. +// Uint64 decodes and returns a uint64 value from the element bitstream. func (r *Decoder) Uint64() uint64 { r.Sync(SyncUint64) return r.rawUvarint() @@ -400,7 +401,7 @@ func (r *Decoder) Code(mark SyncMarker) int { // Reloc decodes a relocation of expected section k from the element // bitstream and returns an index to the referenced element. -func (r *Decoder) Reloc(k RelocKind) Index { +func (r *Decoder) Reloc(k SectionKind) RelElemIdx { r.Sync(SyncUseReloc) return r.rawReloc(k, r.Len()) } @@ -409,7 +410,7 @@ func (r *Decoder) Reloc(k RelocKind) Index { // bitstream. func (r *Decoder) String() string { r.Sync(SyncString) - return r.common.StringIdx(r.Reloc(RelocString)) + return r.common.StringIdx(r.Reloc(SectionString)) } // Strings decodes and returns a variable-length slice of strings from @@ -477,10 +478,10 @@ func (r *Decoder) bigFloat() *big.Float { // PeekPkgPath returns the package path for the specified package // index. -func (pr *PkgDecoder) PeekPkgPath(idx Index) string { +func (pr *PkgDecoder) PeekPkgPath(idx RelElemIdx) string { var path string { - r := pr.TempDecoder(RelocPkg, idx, SyncPkgDef) + r := pr.TempDecoder(SectionPkg, idx, SyncPkgDef) path = r.String() pr.RetireDecoder(&r) } @@ -492,15 +493,15 @@ func (pr *PkgDecoder) PeekPkgPath(idx Index) string { // PeekObj returns the package path, object name, and CodeObj for the // specified object index. -func (pr *PkgDecoder) PeekObj(idx Index) (string, string, CodeObj) { - var ridx Index +func (pr *PkgDecoder) PeekObj(idx RelElemIdx) (string, string, CodeObj) { + var ridx RelElemIdx var name string var rcode int { - r := pr.TempDecoder(RelocName, idx, SyncObject1) + r := pr.TempDecoder(SectionName, idx, SyncObject1) r.Sync(SyncSym) r.Sync(SyncPkg) - ridx = r.Reloc(RelocPkg) + ridx = r.Reloc(SectionPkg) name = r.String() rcode = r.Code(SyncCodeObj) pr.RetireDecoder(&r) @@ -513,3 +514,6 @@ func (pr *PkgDecoder) PeekObj(idx Index) (string, string, CodeObj) { return path, name, tag } + +// Version reports the version of the bitstream. +func (w *Decoder) Version() Version { return w.common.version } diff --git a/src/internal/pkgbits/doc.go b/src/internal/pkgbits/doc.go index 4862e390493532..5c2a937afadb5a 100644 --- a/src/internal/pkgbits/doc.go +++ b/src/internal/pkgbits/doc.go @@ -2,29 +2,106 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package pkgbits implements low-level coding abstractions for -// Unified IR's export data format. -// -// At a low-level, a package is a collection of bitstream elements. -// Each element has a "kind" and a dense, non-negative index. -// Elements can be randomly accessed given their kind and index. +/* +The Unified IR (UIR) format for primitive types is implicitly defined by the +package pkgbits. + +The most basic primitives are laid out as below. + + Bool = [ Sync ] byte . + Int64 = [ Sync ] zvarint . + Uint64 = [ Sync ] uvarint . + + zvarint = (* a zig-zag encoded signed variable-width integer *) . + uvarint = (* an unsigned variable-width integer *) . + +# References +References specify the location of a value. While the representation here is +fixed, the interpretation of a reference is left to other packages. + + Ref[T] = [ Sync ] Uint64 . // points to a value of type T + +# Markers +Markers provide a mechanism for asserting that encoders and decoders +are synchronized. If an unexpected marker is found, decoding panics. + + Sync = uvarint // indicates what should follow if synchronized + WriterPCs + . + +A marker also records a configurable number of program counters (PCs) during +encoding to assist with debugging. + + WriterPCs = uvarint // the number of PCs that follow + { uvarint } // the PCs + . + +Note that markers are always defined using terminals — they never contain a +marker themselves. + +# Strings +A string is a series of bytes. + + // TODO(markfreeman): Does this need a marker? + String = { byte } . + +Strings are typically not encoded directly. Rather, they are deduplicated +during encoding and referenced where needed; this process is called interning. + + StringRef = [ Sync ] Ref[String] . + +Note that StringRef is *not* equivalent to Ref[String] due to the extra marker. + +# Slices +Slices are a convenience for encoding a series of values of the same type. + + // TODO(markfreeman): Does this need a marker? + Slice[T] = Uint64 // the number of values in the slice + { T } // the values + . + +# Constants +Constants appear as defined via the package constant. + + Constant = [ Sync ] + Bool // whether the constant is a complex number + Scalar // the real part + [ Scalar ] // if complex, the imaginary part + . + +A scalar represents a value using one of several potential formats. The exact +format and interpretation is distinguished by a code preceding the value. + + Scalar = [ Sync ] + Uint64 // the code indicating the type of Val + Val + . + + Val = Bool + | Int64 + | StringRef + | Term // big integer + | Term Term // big ratio, numerator / denominator + | BigBytes // big float, precision 512 + . + + Term = BigBytes + Bool // whether the term is negative + . + + BigBytes = StringRef . // bytes of a big value +*/ + +// Package pkgbits implements low-level coding abstractions for Unified IR's +// (UIR) binary export data format. // -// Individual elements are sequences of variable-length values (e.g., -// integers, booleans, strings, go/constant values, cross-references -// to other elements). Package pkgbits provides APIs for encoding and -// decoding these low-level values, but the details of mapping -// higher-level Go constructs into elements is left to higher-level -// abstractions. +// At a low-level, the exported objects of a package are encoded as a byte +// array. This array contains byte representations of primitive, potentially +// variable-length values, such as integers, booleans, strings, and constants. // -// Elements may cross-reference each other with "relocations." For -// example, an element representing a pointer type has a relocation -// referring to the element type. +// Additionally, the array may contain values which denote indices in the byte +// array itself. These are termed "relocations" and allow for references. // -// Go constructs may be composed as a constellation of multiple -// elements. For example, a declared function may have one element to -// describe the object (e.g., its name, type, position), and a -// separate element to describe its function body. This allows readers -// some flexibility in efficiently seeking or re-reading data (e.g., -// inlining requires re-reading the function body for each inlined -// call, without needing to re-read the object-level details). +// The details of mapping high-level Go constructs to primitives are left to +// other packages. package pkgbits diff --git a/src/internal/pkgbits/encoder.go b/src/internal/pkgbits/encoder.go index 70a2cbae5106c8..6e3716570fcd31 100644 --- a/src/internal/pkgbits/encoder.go +++ b/src/internal/pkgbits/encoder.go @@ -6,7 +6,7 @@ package pkgbits import ( "bytes" - "crypto/md5" + "crypto/sha256" "encoding/binary" "go/constant" "io" @@ -15,27 +15,19 @@ import ( "strings" ) -// currentVersion is the current version number. -// -// - v0: initial prototype -// -// - v1: adds the flags uint32 word -// -// TODO(mdempsky): For the next version bump: -// - remove the legacy "has init" bool from the public root -// - remove obj's "derived func instance" bool -const currentVersion uint32 = 1 - // A PkgEncoder provides methods for encoding a package's Unified IR // export data. type PkgEncoder struct { + // version of the bitstream. + version Version + // elems holds the bitstream for previously encoded elements. elems [numRelocs][]string // stringsIdx maps previously encoded strings to their index within // the RelocString section, to allow deduplication. That is, // elems[RelocString][stringsIdx[s]] == s (if present). - stringsIdx map[string]Index + stringsIdx map[string]RelElemIdx // syncFrames is the number of frames to write at each sync // marker. A negative value means sync markers are omitted. @@ -52,9 +44,10 @@ func (pw *PkgEncoder) SyncMarkers() bool { return pw.syncFrames >= 0 } // export data files, but can help diagnosing desync errors in // higher-level Unified IR reader/writer code. If syncFrames is // negative, then sync markers are omitted entirely. -func NewPkgEncoder(syncFrames int) PkgEncoder { +func NewPkgEncoder(version Version, syncFrames int) PkgEncoder { return PkgEncoder{ - stringsIdx: make(map[string]Index), + version: version, + stringsIdx: make(map[string]RelElemIdx), syncFrames: syncFrames, } } @@ -62,29 +55,34 @@ func NewPkgEncoder(syncFrames int) PkgEncoder { // DumpTo writes the package's encoded data to out0 and returns the // package fingerprint. func (pw *PkgEncoder) DumpTo(out0 io.Writer) (fingerprint [8]byte) { - h := md5.New() + h := sha256.New() out := io.MultiWriter(out0, h) writeUint32 := func(x uint32) { assert(binary.Write(out, binary.LittleEndian, x) == nil) } - writeUint32(currentVersion) + writeUint32(uint32(pw.version)) - var flags uint32 - if pw.SyncMarkers() { - flags |= flagSyncMarkers + if pw.version.Has(Flags) { + var flags uint32 + if pw.SyncMarkers() { + flags |= flagSyncMarkers + } + writeUint32(flags) } - writeUint32(flags) - // Write elemEndsEnds. + // TODO(markfreeman): Also can use delta encoding to write section ends, + // but not as impactful. var sum uint32 for _, elems := range &pw.elems { sum += uint32(len(elems)) writeUint32(sum) } - // Write elemEnds. + // TODO(markfreeman): Use delta encoding to store element ends and inflate + // back to this representation during decoding; the numbers will be much + // smaller. sum = 0 for _, elems := range &pw.elems { for _, elem := range elems { @@ -111,14 +109,14 @@ func (pw *PkgEncoder) DumpTo(out0 io.Writer) (fingerprint [8]byte) { // StringIdx adds a string value to the strings section, if not // already present, and returns its index. -func (pw *PkgEncoder) StringIdx(s string) Index { +func (pw *PkgEncoder) StringIdx(s string) RelElemIdx { if idx, ok := pw.stringsIdx[s]; ok { - assert(pw.elems[RelocString][idx] == s) + assert(pw.elems[SectionString][idx] == s) return idx } - idx := Index(len(pw.elems[RelocString])) - pw.elems[RelocString] = append(pw.elems[RelocString], s) + idx := RelElemIdx(len(pw.elems[SectionString])) + pw.elems[SectionString] = append(pw.elems[SectionString], s) pw.stringsIdx[s] = idx return idx } @@ -126,7 +124,7 @@ func (pw *PkgEncoder) StringIdx(s string) Index { // NewEncoder returns an Encoder for a new element within the given // section, and encodes the given SyncMarker as the start of the // element bitstream. -func (pw *PkgEncoder) NewEncoder(k RelocKind, marker SyncMarker) Encoder { +func (pw *PkgEncoder) NewEncoder(k SectionKind, marker SyncMarker) *Encoder { e := pw.NewEncoderRaw(k) e.Sync(marker) return e @@ -136,11 +134,11 @@ func (pw *PkgEncoder) NewEncoder(k RelocKind, marker SyncMarker) Encoder { // section. // // Most callers should use NewEncoder instead. -func (pw *PkgEncoder) NewEncoderRaw(k RelocKind) Encoder { - idx := Index(len(pw.elems[k])) +func (pw *PkgEncoder) NewEncoderRaw(k SectionKind) *Encoder { + idx := RelElemIdx(len(pw.elems[k])) pw.elems[k] = append(pw.elems[k], "") // placeholder - return Encoder{ + return &Encoder{ p: pw, k: k, Idx: idx, @@ -152,18 +150,18 @@ func (pw *PkgEncoder) NewEncoderRaw(k RelocKind) Encoder { type Encoder struct { p *PkgEncoder - Relocs []RelocEnt - RelocMap map[RelocEnt]uint32 + Relocs []RefTableEntry + RelocMap map[RefTableEntry]uint32 Data bytes.Buffer // accumulated element bitstream data encodingRelocHeader bool - k RelocKind - Idx Index // index within relocation section + k SectionKind + Idx RelElemIdx // index within relocation section } -// Flush finalizes the element's bitstream and returns its Index. -func (w *Encoder) Flush() Index { +// Flush finalizes the element's bitstream and returns its [RelElemIdx]. +func (w *Encoder) Flush() RelElemIdx { var sb strings.Builder // Backup the data so we write the relocations at the front. @@ -194,7 +192,7 @@ func (w *Encoder) Flush() Index { func (w *Encoder) checkErr(err error) { if err != nil { - errorf("unexpected encoding error: %v", err) + panicf("unexpected encoding error: %v", err) } } @@ -215,14 +213,14 @@ func (w *Encoder) rawVarint(x int64) { w.rawUvarint(ux) } -func (w *Encoder) rawReloc(r RelocKind, idx Index) int { - e := RelocEnt{r, idx} +func (w *Encoder) rawReloc(k SectionKind, idx RelElemIdx) int { + e := RefTableEntry{k, idx} if w.RelocMap != nil { if i, ok := w.RelocMap[e]; ok { return int(i) } } else { - w.RelocMap = make(map[RelocEnt]uint32) + w.RelocMap = make(map[RefTableEntry]uint32) } i := len(w.Relocs) @@ -252,7 +250,7 @@ func (w *Encoder) Sync(m SyncMarker) { w.rawUvarint(uint64(m)) w.rawUvarint(uint64(len(frames))) for _, frame := range frames { - w.rawUvarint(uint64(w.rawReloc(RelocString, w.p.StringIdx(frame)))) + w.rawUvarint(uint64(w.rawReloc(SectionString, w.p.StringIdx(frame)))) } } @@ -298,7 +296,7 @@ func (w *Encoder) Len(x int) { assert(x >= 0); w.Uint64(uint64(x)) } // Int encodes and writes an int value into the element bitstream. func (w *Encoder) Int(x int) { w.Int64(int64(x)) } -// Len encodes and writes a uint value into the element bitstream. +// Uint encodes and writes a uint value into the element bitstream. func (w *Encoder) Uint(x uint) { w.Uint64(uint64(x)) } // Reloc encodes and writes a relocation for the given (section, @@ -307,9 +305,9 @@ func (w *Encoder) Uint(x uint) { w.Uint64(uint64(x)) } // Note: Only the index is formally written into the element // bitstream, so bitstream decoders must know from context which // section an encoded relocation refers to. -func (w *Encoder) Reloc(r RelocKind, idx Index) { +func (w *Encoder) Reloc(k SectionKind, idx RelElemIdx) { w.Sync(SyncUseReloc) - w.Len(w.rawReloc(r, idx)) + w.Len(w.rawReloc(k, idx)) } // Code encodes and writes a Code value into the element bitstream. @@ -330,9 +328,9 @@ func (w *Encoder) String(s string) { // StringRef writes a reference to the given index, which must be a // previously encoded string value. -func (w *Encoder) StringRef(idx Index) { +func (w *Encoder) StringRef(idx RelElemIdx) { w.Sync(SyncString) - w.Reloc(RelocString, idx) + w.Reloc(SectionString, idx) } // Strings encodes and writes a variable-length slice of strings into @@ -359,7 +357,7 @@ func (w *Encoder) Value(val constant.Value) { func (w *Encoder) scalar(val constant.Value) { switch v := constant.Val(val).(type) { default: - errorf("unhandled %v (%v)", val, val.Kind()) + panicf("unhandled %v (%v)", val, val.Kind()) case bool: w.Code(ValBool) w.Bool(v) @@ -392,3 +390,6 @@ func (w *Encoder) bigFloat(v *big.Float) { b := v.Append(nil, 'p', -1) w.String(string(b)) // TODO: More efficient encoding. } + +// Version reports the version of the bitstream. +func (w *Encoder) Version() Version { return w.p.version } diff --git a/src/internal/pkgbits/pkgbits_test.go b/src/internal/pkgbits/pkgbits_test.go new file mode 100644 index 00000000000000..a4755bd35a4e5e --- /dev/null +++ b/src/internal/pkgbits/pkgbits_test.go @@ -0,0 +1,76 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pkgbits_test + +import ( + "internal/pkgbits" + "strings" + "testing" +) + +func TestRoundTrip(t *testing.T) { + for _, version := range []pkgbits.Version{ + pkgbits.V0, + pkgbits.V1, + pkgbits.V2, + } { + pw := pkgbits.NewPkgEncoder(version, -1) + w := pw.NewEncoder(pkgbits.SectionMeta, pkgbits.SyncPublic) + w.Flush() + + var b strings.Builder + _ = pw.DumpTo(&b) + input := b.String() + + pr := pkgbits.NewPkgDecoder("package_id", input) + r := pr.NewDecoder(pkgbits.SectionMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic) + + if r.Version() != w.Version() { + t.Errorf("Expected reader version %q to be the writer version %q", r.Version(), w.Version()) + } + } +} + +// Type checker to enforce that know V* have the constant values they must have. +var _ [0]bool = [pkgbits.V0]bool{} +var _ [1]bool = [pkgbits.V1]bool{} + +func TestVersions(t *testing.T) { + type vfpair struct { + v pkgbits.Version + f pkgbits.Field + } + + // has field tests + for _, c := range []vfpair{ + {pkgbits.V1, pkgbits.Flags}, + {pkgbits.V2, pkgbits.Flags}, + {pkgbits.V0, pkgbits.HasInit}, + {pkgbits.V1, pkgbits.HasInit}, + {pkgbits.V0, pkgbits.DerivedFuncInstance}, + {pkgbits.V1, pkgbits.DerivedFuncInstance}, + {pkgbits.V0, pkgbits.DerivedInfoNeeded}, + {pkgbits.V1, pkgbits.DerivedInfoNeeded}, + {pkgbits.V2, pkgbits.AliasTypeParamNames}, + } { + if !c.v.Has(c.f) { + t.Errorf("Expected version %v to have field %v", c.v, c.f) + } + } + + // does not have field tests + for _, c := range []vfpair{ + {pkgbits.V0, pkgbits.Flags}, + {pkgbits.V2, pkgbits.HasInit}, + {pkgbits.V2, pkgbits.DerivedFuncInstance}, + {pkgbits.V2, pkgbits.DerivedInfoNeeded}, + {pkgbits.V0, pkgbits.AliasTypeParamNames}, + {pkgbits.V1, pkgbits.AliasTypeParamNames}, + } { + if c.v.Has(c.f) { + t.Errorf("Expected version %v to not have field %v", c.v, c.f) + } + } +} diff --git a/src/internal/pkgbits/reloc.go b/src/internal/pkgbits/reloc.go index fcdfb97ca99261..6074296d9ee3f9 100644 --- a/src/internal/pkgbits/reloc.go +++ b/src/internal/pkgbits/reloc.go @@ -4,39 +4,111 @@ package pkgbits -// A RelocKind indicates a particular section within a unified IR export. -type RelocKind int32 +// A SectionKind indicates a section, as well as the ordering of sections within +// unified export data. Any object given a dedicated section can be referred to +// via a section / index pair (and thus dereferenced) in other sections. +type SectionKind int32 // TODO(markfreeman): Replace with uint8. -// An Index represents a bitstream element index within a particular -// section. +const ( + SectionString SectionKind = iota + SectionMeta + SectionPosBase + SectionPkg + SectionName + SectionType + SectionObj + SectionObjExt + SectionObjDict + SectionBody + + numRelocs = iota +) + +// An Index represents a bitstream element index *within* (i.e., relative to) a +// particular section. type Index int32 -// A relocEnt (relocation entry) is an entry in an element's local -// reference table. -// -// TODO(mdempsky): Rename this too. -type RelocEnt struct { - Kind RelocKind - Idx Index -} +// An AbsElemIdx, or absolute element index, is an index into the elements +// that is not relative to some other index. +type AbsElemIdx = uint32 -// Reserved indices within the meta relocation section. -const ( - PublicRootIdx Index = 0 - PrivateRootIdx Index = 1 -) +// TODO(markfreeman): Make this its own type. +// A RelElemIdx, or relative element index, is an index into the elements +// relative to some other index, such as the start of a section. +type RelElemIdx = Index -const ( - RelocString RelocKind = iota - RelocMeta - RelocPosBase - RelocPkg - RelocName - RelocType - RelocObj - RelocObjExt - RelocObjDict - RelocBody +/* +All elements are preceded by a reference table. Reference tables provide an +additional indirection layer for element references. That is, for element A to +reference element B, A encodes the reference table index pointing to B, rather +than the table entry itself. - numRelocs = iota +# Functional Considerations +Reference table layout is important primarily to the UIR linker. After noding, +the UIR linker sees a UIR file for each package with imported objects +represented as stubs. In a simple sense, the job of the UIR linker is to merge +these "stubbed" UIR files into a single "linked" UIR file for the target package +with stubs replaced by object definitions. + +To do this, the UIR linker walks each stubbed UIR file and pulls in elements in +dependency order; that is, if A references B, then B must be placed into the +linked UIR file first. This depth-first traversal is done by recursing through +each element's reference table. + +When placing A in the linked UIR file, the reference table entry for B must be +updated, since B is unlikely to be at the same relative element index as it was +in the stubbed UIR file. + +Without reference tables, the UIR linker would need to read in the element to +discover its references. Note that the UIR linker cannot jump directly to the +reference locations after discovering merely the type of the element; +variable-width primitives prevent this. + +After updating the reference table, the rest of the element may be copied +directly into the linked UIR file. Note that the UIR linker may decide to read +in the element anyway (for unrelated reasons). + +In short, reference tables provide an efficient mechanism for traversing, +discovering, and updating element references during UIR linking. + +# Storage Considerations +Reference tables also have compactness benefits: + - If A refers to B multiple times, the entry is deduplicated and referred to + more compactly by the index. + - Relative (to a section) element indices are typically smaller than absolute + element indices, and thus fit into smaller varints. + - Most elements do not reference many elements; thus table size indicators and + table indices are typically a byte each. + +Thus, the storage performance is as follows: ++-----------------------------+-----------+--------------+ +| Scenario | Best Case | Typical Case | ++-----------------------------+-----------+--------------+ +| First reference from A to B | 3 Bytes | 4 Bytes | +| Other reference from A to B | 1 Byte | 1 Byte | ++-----------------------------+-----------+--------------+ + +The typical case for the first scenario changes because many sections have more +than 127 (range of a 1-byte uvarint) elements and thus the relative index is +typically 2 bytes, though this depends on the distribution of referenced indices +within the section. + +The second does not because most elements do not reference more than 127 +elements and the table index can thus keep to 1 byte. + +Typically, A will only reference B once, so most references are 4 bytes. +*/ + +// A RefTableEntry is an entry in an element's reference table. All +// elements are preceded by a reference table which provides locations +// for referenced elements. +type RefTableEntry struct { + Kind SectionKind + Idx RelElemIdx +} + +// Reserved indices within the [SectionMeta] section. +const ( + PublicRootIdx RelElemIdx = 0 + PrivateRootIdx RelElemIdx = 1 ) diff --git a/src/internal/pkgbits/support.go b/src/internal/pkgbits/support.go index f7579dfdc4ed96..50534a29553638 100644 --- a/src/internal/pkgbits/support.go +++ b/src/internal/pkgbits/support.go @@ -12,6 +12,6 @@ func assert(b bool) { } } -func errorf(format string, args ...any) { +func panicf(format string, args ...any) { panic(fmt.Errorf(format, args...)) } diff --git a/src/internal/pkgbits/version.go b/src/internal/pkgbits/version.go new file mode 100644 index 00000000000000..ba664f45554aab --- /dev/null +++ b/src/internal/pkgbits/version.go @@ -0,0 +1,85 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pkgbits + +// Version indicates a version of a unified IR bitstream. +// Each Version indicates the addition, removal, or change of +// new data in the bitstream. +// +// These are serialized to disk and the interpretation remains fixed. +type Version uint32 + +const ( + // V0: initial prototype. + // + // All data that is not assigned a Field is in version V0 + // and has not been deprecated. + V0 Version = iota + + // V1: adds the Flags uint32 word + V1 + + // V2: removes unused legacy fields and supports type parameters for aliases. + // - remove the legacy "has init" bool from the public root + // - remove obj's "derived func instance" bool + // - add a TypeParamNames field to ObjAlias + // - remove derived info "needed" bool + V2 + + numVersions = iota +) + +// Field denotes a unit of data in the serialized unified IR bitstream. +// It is conceptually a like field in a structure. +// +// We only really need Fields when the data may or may not be present +// in a stream based on the Version of the bitstream. +// +// Unlike much of pkgbits, Fields are not serialized and +// can change values as needed. +type Field int + +const ( + // Flags in a uint32 in the header of a bitstream + // that is used to indicate whether optional features are enabled. + Flags Field = iota + + // Deprecated: HasInit was a bool indicating whether a package + // has any init functions. + HasInit + + // Deprecated: DerivedFuncInstance was a bool indicating + // whether an object was a function instance. + DerivedFuncInstance + + // ObjAlias has a list of TypeParamNames. + AliasTypeParamNames + + // Deprecated: DerivedInfoNeeded was a bool indicating + // whether a type was a derived type. + DerivedInfoNeeded + + numFields = iota +) + +// introduced is the version a field was added. +var introduced = [numFields]Version{ + Flags: V1, + AliasTypeParamNames: V2, +} + +// removed is the version a field was removed in or 0 for fields +// that have not yet been deprecated. +// (So removed[f]-1 is the last version it is included in.) +var removed = [numFields]Version{ + HasInit: V2, + DerivedFuncInstance: V2, + DerivedInfoNeeded: V2, +} + +// Has reports whether field f is present in a bitstream at version v. +func (v Version) Has(f Field) bool { + return introduced[f] <= v && (v < removed[f] || removed[f] == V0) +} diff --git a/src/internal/platform/supported.go b/src/internal/platform/supported.go index a774247e6bbc62..f9706a6988e94b 100644 --- a/src/internal/platform/supported.go +++ b/src/internal/platform/supported.go @@ -23,7 +23,7 @@ func (p OSArch) String() string { func RaceDetectorSupported(goos, goarch string) bool { switch goos { case "linux": - return goarch == "amd64" || goarch == "ppc64le" || goarch == "arm64" || goarch == "s390x" + return goarch == "amd64" || goarch == "arm64" || goarch == "loong64" || goarch == "ppc64le" || goarch == "riscv64" || goarch == "s390x" case "darwin": return goarch == "amd64" || goarch == "arm64" case "freebsd", "netbsd", "windows": @@ -61,7 +61,7 @@ func ASanSupported(goos, goarch string) bool { // ('go test -fuzz=.'). func FuzzSupported(goos, goarch string) bool { switch goos { - case "darwin", "freebsd", "linux", "windows": + case "darwin", "freebsd", "linux", "openbsd", "windows": return true default: return false @@ -72,7 +72,7 @@ func FuzzSupported(goos, goarch string) bool { // instrumentation. (FuzzInstrumented implies FuzzSupported.) func FuzzInstrumented(goos, goarch string) bool { switch goarch { - case "amd64", "arm64": + case "amd64", "arm64", "loong64": // TODO(#14565): support more architectures. return FuzzSupported(goos, goarch) default: @@ -85,7 +85,7 @@ func FuzzInstrumented(goos, goarch string) bool { func MustLinkExternal(goos, goarch string, withCgo bool) bool { if withCgo { switch goarch { - case "loong64", "mips", "mipsle", "mips64", "mips64le": + case "mips", "mipsle", "mips64", "mips64le": // Internally linking cgo is incomplete on some architectures. // https://go.dev/issue/14449 return true @@ -173,7 +173,8 @@ func BuildModeSupported(compiler, buildmode, goos, goarch string) bool { "android/amd64", "android/arm", "android/arm64", "android/386", "freebsd/amd64", "darwin/amd64", "darwin/arm64", - "windows/amd64", "windows/386", "windows/arm64": + "windows/amd64", "windows/386", "windows/arm64", + "wasip1/wasm": return true } return false @@ -193,7 +194,7 @@ func BuildModeSupported(compiler, buildmode, goos, goarch string) bool { "ios/amd64", "ios/arm64", "aix/ppc64", "openbsd/arm64", - "windows/386", "windows/amd64", "windows/arm", "windows/arm64": + "windows/386", "windows/amd64", "windows/arm64": return true } return false @@ -207,7 +208,7 @@ func BuildModeSupported(compiler, buildmode, goos, goarch string) bool { case "plugin": switch platform { - case "linux/amd64", "linux/arm", "linux/arm64", "linux/386", "linux/loong64", "linux/s390x", "linux/ppc64le", + case "linux/amd64", "linux/arm", "linux/arm64", "linux/386", "linux/loong64", "linux/riscv64", "linux/s390x", "linux/ppc64le", "android/amd64", "android/386", "darwin/amd64", "darwin/arm64", "freebsd/amd64": @@ -224,8 +225,8 @@ func InternalLinkPIESupported(goos, goarch string) bool { switch goos + "/" + goarch { case "android/arm64", "darwin/amd64", "darwin/arm64", - "linux/amd64", "linux/arm64", "linux/ppc64le", - "windows/386", "windows/amd64", "windows/arm", "windows/arm64": + "linux/amd64", "linux/arm64", "linux/loong64", "linux/ppc64le", + "windows/386", "windows/amd64", "windows/arm64": return true } return false @@ -280,7 +281,7 @@ func FirstClass(goos, goarch string) bool { return distInfo[OSArch{goos, goarch}].FirstClass } -// Broken reportsr whether goos/goarch is considered a broken port. +// Broken reports whether goos/goarch is considered a broken port. // (See https://go.dev/wiki/PortingPolicy#broken-ports.) func Broken(goos, goarch string) bool { return distInfo[OSArch{goos, goarch}].Broken diff --git a/src/internal/platform/zosarch.go b/src/internal/platform/zosarch.go index 1df348518cc9b9..a2f5b22ea9a656 100644 --- a/src/internal/platform/zosarch.go +++ b/src/internal/platform/zosarch.go @@ -57,7 +57,6 @@ var List = []OSArch{ {"wasip1", "wasm"}, {"windows", "386"}, {"windows", "amd64"}, - {"windows", "arm"}, {"windows", "arm64"}, } @@ -74,7 +73,7 @@ var distInfo = map[OSArch]osArchInfo{ {"freebsd", "amd64"}: {CgoSupported: true}, {"freebsd", "arm"}: {CgoSupported: true}, {"freebsd", "arm64"}: {CgoSupported: true}, - {"freebsd", "riscv64"}: {CgoSupported: true}, + {"freebsd", "riscv64"}: {CgoSupported: true, Broken: true}, {"illumos", "amd64"}: {CgoSupported: true}, {"ios", "amd64"}: {CgoSupported: true}, {"ios", "arm64"}: {CgoSupported: true}, @@ -111,6 +110,5 @@ var distInfo = map[OSArch]osArchInfo{ {"wasip1", "wasm"}: {}, {"windows", "386"}: {CgoSupported: true, FirstClass: true}, {"windows", "amd64"}: {CgoSupported: true, FirstClass: true}, - {"windows", "arm"}: {}, {"windows", "arm64"}: {CgoSupported: true}, } diff --git a/src/internal/poll/copy_file_range_freebsd.go b/src/internal/poll/copy_file_range_freebsd.go new file mode 100644 index 00000000000000..63fa013e46cc3d --- /dev/null +++ b/src/internal/poll/copy_file_range_freebsd.go @@ -0,0 +1,53 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package poll + +import ( + "internal/syscall/unix" + "syscall" +) + +func supportCopyFileRange() bool { + return unix.SupportCopyFileRange() +} + +// For best performance, call copy_file_range() with the largest len value +// possible. It is interruptible on most file systems, so there is no penalty +// for using very large len values, even SSIZE_MAX. +const maxCopyFileRangeRound = 1<<31 - 1 + +func handleCopyFileRangeErr(err error, copied, written int64) (bool, error) { + switch err { + case syscall.ENOSYS: + // The copy_file_range(2) function first appeared in FreeBSD 13.0. + // Go supports FreeBSD >= 12, so the system call + // may not be present. We've detected the FreeBSD version with + // unix.SupportCopyFileRange() at the beginning of this function, + // but we still want to check for ENOSYS here to prevent some rare + // case like https://go.dev/issue/58592 + // + // If we see ENOSYS, we have certainly not transferred + // any data, so we can tell the caller that we + // couldn't handle the transfer and let them fall + // back to more generic code. + return false, nil + case syscall.EFBIG, syscall.EINVAL, syscall.EIO: + // For EFBIG, the copy has exceeds the process's file size limit + // or the maximum file size for the filesystem dst resides on, in + // this case, we leave it to generic copy. + // + // For EINVAL, there could be a few reasons: + // 1. Either dst or src refers to a file object that + // is not a regular file, for instance, a pipe. + // 2. src and dst refer to the same file and byte ranges + // overlap. + // 3. The flags argument is not 0. + // Neither of these cases should be considered handled by + // copy_file_range(2) because there is no data transfer, so + // just fall back to generic copy. + return false, nil + } + return true, err +} diff --git a/src/internal/poll/copy_file_range_linux.go b/src/internal/poll/copy_file_range_linux.go index 3d51333d73e9cf..d7cbd984651678 100644 --- a/src/internal/poll/copy_file_range_linux.go +++ b/src/internal/poll/copy_file_range_linux.go @@ -10,112 +10,70 @@ import ( "syscall" ) -var isKernelVersionGE53 = sync.OnceValue(func() bool { - major, minor := unix.KernelVersion() +var supportCopyFileRange = sync.OnceValue(func() bool { // copy_file_range(2) is broken in various ways on kernels older than 5.3, // see https://go.dev/issue/42400 and // https://man7.org/linux/man-pages/man2/copy_file_range.2.html#VERSIONS - return major > 5 || (major == 5 && minor >= 3) + return unix.KernelVersionGE(5, 3) }) -const maxCopyFileRangeRound = 1 << 30 +// For best performance, call copy_file_range() with the largest len value +// possible. Linux sets up a limitation of data transfer for most of its I/O +// system calls, as MAX_RW_COUNT (INT_MAX & PAGE_MASK). This value equals to +// the maximum integer value minus a page size that is typically 2^12=4096 bytes. +// That is to say, it's the maximum integer value with the lowest 12 bits unset, +// which is 0x7ffff000. +const maxCopyFileRangeRound = 0x7ffff000 -// CopyFileRange copies at most remain bytes of data from src to dst, using -// the copy_file_range system call. dst and src must refer to regular files. -func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err error) { - if !isKernelVersionGE53() { - return 0, false, nil - } - - for remain > 0 { - max := remain - if max > maxCopyFileRangeRound { - max = maxCopyFileRangeRound - } - n, err := copyFileRange(dst, src, int(max)) - switch err { - case syscall.ENOSYS: - // copy_file_range(2) was introduced in Linux 4.5. - // Go supports Linux >= 2.6.33, so the system call - // may not be present. - // - // If we see ENOSYS, we have certainly not transferred - // any data, so we can tell the caller that we - // couldn't handle the transfer and let them fall - // back to more generic code. - return 0, false, nil - case syscall.EXDEV, syscall.EINVAL, syscall.EIO, syscall.EOPNOTSUPP, syscall.EPERM: - // Prior to Linux 5.3, it was not possible to - // copy_file_range across file systems. Similarly to - // the ENOSYS case above, if we see EXDEV, we have - // not transferred any data, and we can let the caller - // fall back to generic code. - // - // As for EINVAL, that is what we see if, for example, - // dst or src refer to a pipe rather than a regular - // file. This is another case where no data has been - // transferred, so we consider it unhandled. - // - // If src and dst are on CIFS, we can see EIO. - // See issue #42334. - // - // If the file is on NFS, we can see EOPNOTSUPP. - // See issue #40731. - // - // If the process is running inside a Docker container, - // we might see EPERM instead of ENOSYS. See issue - // #40893. Since EPERM might also be a legitimate error, - // don't mark copy_file_range(2) as unsupported. - return 0, false, nil - case nil: - if n == 0 { - // If we did not read any bytes at all, - // then this file may be in a file system - // where copy_file_range silently fails. - // https://lore.kernel.org/linux-fsdevel/20210126233840.GG4626@dread.disaster.area/T/#m05753578c7f7882f6e9ffe01f981bc223edef2b0 - if written == 0 { - return 0, false, nil - } - // Otherwise src is at EOF, which means - // we are done. - return written, true, nil +func handleCopyFileRangeErr(err error, copied, written int64) (bool, error) { + switch err { + case syscall.ENOSYS: + // copy_file_range(2) was introduced in Linux 4.5. + // Go supports Linux >= 3.2, so the system call + // may not be present. + // + // If we see ENOSYS, we have certainly not transferred + // any data, so we can tell the caller that we + // couldn't handle the transfer and let them fall + // back to more generic code. + return false, nil + case syscall.EXDEV, syscall.EINVAL, syscall.EIO, syscall.EOPNOTSUPP, syscall.EPERM: + // Prior to Linux 5.3, it was not possible to + // copy_file_range across file systems. Similarly to + // the ENOSYS case above, if we see EXDEV, we have + // not transferred any data, and we can let the caller + // fall back to generic code. + // + // As for EINVAL, that is what we see if, for example, + // dst or src refer to a pipe rather than a regular + // file. This is another case where no data has been + // transferred, so we consider it unhandled. + // + // If src and dst are on CIFS, we can see EIO. + // See issue #42334. + // + // If the file is on NFS, we can see EOPNOTSUPP. + // See issue #40731. + // + // If the process is running inside a Docker container, + // we might see EPERM instead of ENOSYS. See issue + // #40893. Since EPERM might also be a legitimate error, + // don't mark copy_file_range(2) as unsupported. + return false, nil + case nil: + if copied == 0 { + // Prior to Linux 5.19 + // (https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=868f9f2f8e004bfe0d3935b1976f625b2924893b), + // copy_file_range can silently fail by reporting + // success and 0 bytes written. Assume such cases are + // failure and fallback to a different copy mechanism. + if written == 0 { + return false, nil } - remain -= n - written += n - default: - return written, true, err - } - } - return written, true, nil -} -// copyFileRange performs one round of copy_file_range(2). -func copyFileRange(dst, src *FD, max int) (written int64, err error) { - // The signature of copy_file_range(2) is: - // - // ssize_t copy_file_range(int fd_in, loff_t *off_in, - // int fd_out, loff_t *off_out, - // size_t len, unsigned int flags); - // - // Note that in the call to unix.CopyFileRange below, we use nil - // values for off_in and off_out. For the system call, this means - // "use and update the file offsets". That is why we must acquire - // locks for both file descriptors (and why this whole machinery is - // in the internal/poll package to begin with). - if err := dst.writeLock(); err != nil { - return 0, err - } - defer dst.writeUnlock() - if err := src.readLock(); err != nil { - return 0, err - } - defer src.readUnlock() - var n int - for { - n, err = unix.CopyFileRange(src.Sysfd, nil, dst.Sysfd, nil, max, 0) - if err != syscall.EINTR { - break + // Otherwise src is at EOF, which means + // we are done. } } - return int64(n), err + return true, err } diff --git a/src/internal/poll/copy_file_range_unix.go b/src/internal/poll/copy_file_range_unix.go new file mode 100644 index 00000000000000..d39c5a339d5895 --- /dev/null +++ b/src/internal/poll/copy_file_range_unix.go @@ -0,0 +1,67 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build freebsd || linux + +package poll + +import "internal/syscall/unix" + +// CopyFileRange copies at most remain bytes of data from src to dst, using +// the copy_file_range system call. dst and src must refer to regular files. +func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err error) { + if !supportCopyFileRange() { + return 0, false, nil + } + + for remain > 0 { + max := min(remain, maxCopyFileRangeRound) + n, e := copyFileRange(dst, src, int(max)) + if n > 0 { + remain -= n + written += n + } + handled, err = handleCopyFileRangeErr(e, n, written) + if n == 0 || !handled || err != nil { + return + } + } + + return written, true, nil +} + +// copyFileRange performs one round of copy_file_range(2). +func copyFileRange(dst, src *FD, max int) (written int64, err error) { + // For Linux, the signature of copy_file_range(2) is: + // + // ssize_t copy_file_range(int fd_in, loff_t *off_in, + // int fd_out, loff_t *off_out, + // size_t len, unsigned int flags); + // + // For FreeBSD, the signature of copy_file_range(2) is: + // + // ssize_t + // copy_file_range(int infd, off_t *inoffp, int outfd, off_t *outoffp, + // size_t len, unsigned int flags); + // + // Note that in the call to unix.CopyFileRange below, we use nil + // values for off_in/off_out and inoffp/outoffp, which means "the file + // offset for infd(fd_in) or outfd(fd_out) respectively will be used and + // updated by the number of bytes copied". + // + // That is why we must acquire locks for both file descriptors (and why + // this whole machinery is in the internal/poll package to begin with). + if err := dst.writeLock(); err != nil { + return 0, err + } + defer dst.writeUnlock() + if err := src.readLock(); err != nil { + return 0, err + } + defer src.readUnlock() + return ignoringEINTR2(func() (int64, error) { + n, err := unix.CopyFileRange(src.Sysfd, nil, dst.Sysfd, nil, max, 0) + return int64(n), err + }) +} diff --git a/src/internal/poll/export_windows_test.go b/src/internal/poll/export_windows_test.go deleted file mode 100644 index 88ed71ad84aeeb..00000000000000 --- a/src/internal/poll/export_windows_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Export guts for testing on windows. -// Since testing imports os and os imports internal/poll, -// the internal/poll tests can not be in package poll. - -package poll - -var ( - LogInitFD = &logInitFD -) - -func (fd *FD) IsPartOfNetpoll() bool { - return fd.pd.runtimeCtx != 0 -} diff --git a/src/internal/poll/fd_mutex.go b/src/internal/poll/fd_mutex.go index 0a8ee6f0d49d5c..c1c92976870658 100644 --- a/src/internal/poll/fd_mutex.go +++ b/src/internal/poll/fd_mutex.go @@ -250,3 +250,25 @@ func (fd *FD) writeUnlock() { fd.destroy() } } + +// readWriteLock adds a reference to fd and locks fd for reading and writing. +// It returns an error when fd cannot be used for reading and writing. +func (fd *FD) readWriteLock() error { + if !fd.fdmu.rwlock(true) || !fd.fdmu.rwlock(false) { + return errClosing(fd.isFile) + } + return nil +} + +// readWriteUnlock removes a reference from fd and unlocks fd for reading and writing. +// It also closes fd when the state of fd is set to closed and there +// is no remaining reference. +func (fd *FD) readWriteUnlock() { + fd.fdmu.rwunlock(true) + fd.fdmu.rwunlock(false) +} + +// closing returns true if fd is closing. +func (fd *FD) closing() bool { + return atomic.LoadUint64(&fd.fdmu.state)&mutexClosed != 0 +} diff --git a/src/internal/poll/fd_poll_runtime.go b/src/internal/poll/fd_poll_runtime.go index b78d1564766917..2aef11243ad34e 100644 --- a/src/internal/poll/fd_poll_runtime.go +++ b/src/internal/poll/fd_poll_runtime.go @@ -155,6 +155,7 @@ func setDeadlineImpl(fd *FD, t time.Time, mode int) error { return err } defer fd.decref() + if fd.pd.runtimeCtx == 0 { return ErrNoDeadline } diff --git a/src/internal/poll/fd_posix.go b/src/internal/poll/fd_posix.go index 5bd333b4daaeb5..12f138644b38af 100644 --- a/src/internal/poll/fd_posix.go +++ b/src/internal/poll/fd_posix.go @@ -77,3 +77,13 @@ func ignoringEINTR(fn func() error) error { } } } + +// ignoringEINTR2 is ignoringEINTR, but returning an additional value. +func ignoringEINTR2[T any](fn func() (T, error)) (T, error) { + for { + v, err := fn() + if err != syscall.EINTR { + return v, err + } + } +} diff --git a/src/internal/poll/fd_unix.go b/src/internal/poll/fd_unix.go index 2535a3ae4dd43b..31e6e21120fde3 100644 --- a/src/internal/poll/fd_unix.go +++ b/src/internal/poll/fd_unix.go @@ -183,16 +183,9 @@ func (fd *FD) Pread(p []byte, off int64) (int, error) { if fd.IsStream && len(p) > maxRW { p = p[:maxRW] } - var ( - n int - err error - ) - for { - n, err = syscall.Pread(fd.Sysfd, p, off) - if err != syscall.EINTR { - break - } - } + n, err := ignoringEINTR2(func() (int, error) { + return syscall.Pread(fd.Sysfd, p, off) + }) if err != nil { n = 0 } diff --git a/src/internal/poll/fd_wasip1.go b/src/internal/poll/fd_wasip1.go index 195aaa9517d117..4f83d287370c12 100644 --- a/src/internal/poll/fd_wasip1.go +++ b/src/internal/poll/fd_wasip1.go @@ -225,11 +225,11 @@ func readIntLE(b []byte, size uintptr) uint64 { case 1: return uint64(b[0]) case 2: - return uint64(byteorder.LeUint16(b)) + return uint64(byteorder.LEUint16(b)) case 4: - return uint64(byteorder.LeUint32(b)) + return uint64(byteorder.LEUint32(b)) case 8: - return uint64(byteorder.LeUint64(b)) + return uint64(byteorder.LEUint64(b)) default: panic("internal/poll: readInt with unsupported size") } diff --git a/src/internal/poll/fd_windows.go b/src/internal/poll/fd_windows.go index 5eefeb90f193ee..6443f6eb30b8a5 100644 --- a/src/internal/poll/fd_windows.go +++ b/src/internal/poll/fd_windows.go @@ -9,7 +9,9 @@ import ( "internal/race" "internal/syscall/windows" "io" + "runtime" "sync" + "sync/atomic" "syscall" "unicode/utf16" "unicode/utf8" @@ -27,7 +29,7 @@ var ( // SetFileCompletionNotificationModes crashes on some systems (see // https://support.microsoft.com/kb/2568167 for details). -var useSetFileCompletionNotificationModes bool // determines is SetFileCompletionNotificationModes is present and safe to use +var socketCanUseSetFileCompletionNotificationModes bool // determines is SetFileCompletionNotificationModes is present and sockets can safely use it // checkSetFileCompletionNotificationModes verifies that // SetFileCompletionNotificationModes Windows API is present @@ -50,7 +52,7 @@ func checkSetFileCompletionNotificationModes() { return } } - useSetFileCompletionNotificationModes = true + socketCanUseSetFileCompletionNotificationModes = true } // InitWSA initiates the use of the Winsock DLL by the current process. @@ -68,153 +70,266 @@ var InitWSA = sync.OnceFunc(func() { // operation contains superset of data necessary to perform all async IO. type operation struct { // Used by IOCP interface, it must be first field - // of the struct, as our code rely on it. + // of the struct, as our code relies on it. o syscall.Overlapped // fields used by runtime.netpoll runtimeCtx uintptr mode int32 - - // fields used only by net package - fd *FD - buf syscall.WSABuf - msg windows.WSAMsg - sa syscall.Sockaddr - rsa *syscall.RawSockaddrAny - rsan int32 - handle syscall.Handle - flags uint32 - qty uint32 - bufs []syscall.WSABuf } -func (o *operation) InitBuf(buf []byte) { - o.buf.Len = uint32(len(buf)) - o.buf.Buf = nil - if len(buf) != 0 { - o.buf.Buf = &buf[0] +func (fd *FD) overlapped(o *operation) *syscall.Overlapped { + if fd.isBlocking { + // Don't return the overlapped object if the file handle + // doesn't use overlapped I/O. It could be used, but + // that would then use the file pointer stored in the + // overlapped object rather than the real file pointer. + return nil } + return &o.o } -func (o *operation) InitBufs(buf *[][]byte) { - if o.bufs == nil { - o.bufs = make([]syscall.WSABuf, 0, len(*buf)) - } else { - o.bufs = o.bufs[:0] - } +func newWsaBuf(b []byte) *syscall.WSABuf { + return &syscall.WSABuf{Buf: unsafe.SliceData(b), Len: uint32(len(b))} +} + +var wsaBufsPool = sync.Pool{ + New: func() any { + buf := make([]syscall.WSABuf, 0, 16) + return &buf + }, +} + +func newWSABufs(buf *[][]byte) *[]syscall.WSABuf { + bufsPtr := wsaBufsPool.Get().(*[]syscall.WSABuf) + *bufsPtr = (*bufsPtr)[:0] for _, b := range *buf { if len(b) == 0 { - o.bufs = append(o.bufs, syscall.WSABuf{}) + *bufsPtr = append(*bufsPtr, syscall.WSABuf{}) continue } for len(b) > maxRW { - o.bufs = append(o.bufs, syscall.WSABuf{Len: maxRW, Buf: &b[0]}) + *bufsPtr = append(*bufsPtr, syscall.WSABuf{Len: maxRW, Buf: &b[0]}) b = b[maxRW:] } if len(b) > 0 { - o.bufs = append(o.bufs, syscall.WSABuf{Len: uint32(len(b)), Buf: &b[0]}) + *bufsPtr = append(*bufsPtr, syscall.WSABuf{Len: uint32(len(b)), Buf: &b[0]}) } } + return bufsPtr } -// ClearBufs clears all pointers to Buffers parameter captured -// by InitBufs, so it can be released by garbage collector. -func (o *operation) ClearBufs() { - for i := range o.bufs { - o.bufs[i].Buf = nil - } - o.bufs = o.bufs[:0] +func freeWSABufs(bufsPtr *[]syscall.WSABuf) { + // Clear pointers to buffers so they can be released by garbage collector. + bufs := *bufsPtr + for i := range bufs { + bufs[i].Buf = nil + } + // Proper usage of a sync.Pool requires each entry to have approximately + // the same memory cost. To obtain this property when the stored type + // contains a variably-sized buffer, we add a hard limit on the maximum buffer + // to place back in the pool. + // + // See https://go.dev/issue/23199 + if cap(*bufsPtr) > 128 { + *bufsPtr = nil + } + wsaBufsPool.Put(bufsPtr) } -func (o *operation) InitMsg(p []byte, oob []byte) { - o.InitBuf(p) - o.msg.Buffers = &o.buf - o.msg.BufferCount = 1 +// wsaMsgPool is a pool of WSAMsg structures that can only hold a single WSABuf. +var wsaMsgPool = sync.Pool{ + New: func() any { + return &windows.WSAMsg{ + Buffers: &syscall.WSABuf{}, + BufferCount: 1, + } + }, +} - o.msg.Name = nil - o.msg.Namelen = 0 +// newWSAMsg creates a new WSAMsg with the provided parameters. +// Use [freeWSAMsg] to free it. +func newWSAMsg(p []byte, oob []byte, flags int, unconnected bool) *windows.WSAMsg { + // The returned object can't be allocated in the stack because it is accessed asynchronously + // by Windows in between several system calls. If the stack frame is moved while that happens, + // then Windows may access invalid memory. + // TODO(qmuntal): investigate using runtime.Pinner keeping this path allocation-free. + + // Use a pool to reuse allocations. + msg := wsaMsgPool.Get().(*windows.WSAMsg) + msg.Buffers.Len = uint32(len(p)) + msg.Buffers.Buf = unsafe.SliceData(p) + msg.Control = syscall.WSABuf{ + Len: uint32(len(oob)), + Buf: unsafe.SliceData(oob), + } + msg.Flags = uint32(flags) + if unconnected { + msg.Name = wsaRsaPool.Get().(*syscall.RawSockaddrAny) + msg.Namelen = int32(unsafe.Sizeof(syscall.RawSockaddrAny{})) + } + return msg +} - o.msg.Flags = 0 - o.msg.Control.Len = uint32(len(oob)) - o.msg.Control.Buf = nil - if len(oob) != 0 { - o.msg.Control.Buf = &oob[0] - } +func freeWSAMsg(msg *windows.WSAMsg) { + // Clear pointers to buffers so they can be released by garbage collector. + msg.Buffers.Len = 0 + msg.Buffers.Buf = nil + msg.Control.Len = 0 + msg.Control.Buf = nil + if msg.Name != nil { + *msg.Name = syscall.RawSockaddrAny{} + wsaRsaPool.Put(msg.Name) + msg.Name = nil + msg.Namelen = 0 + } + wsaMsgPool.Put(msg) } -// execIO executes a single IO operation o. It submits and cancels -// IO in the current thread for systems where Windows CancelIoEx API -// is available. Alternatively, it passes the request onto -// runtime netpoll and waits for completion or cancels request. -func execIO(o *operation, submit func(o *operation) error) (int, error) { - if o.fd.pd.runtimeCtx == 0 { - return 0, errors.New("internal error: polling on unsupported descriptor type") - } +var wsaRsaPool = sync.Pool{ + New: func() any { + return new(syscall.RawSockaddrAny) + }, +} - fd := o.fd - // Notify runtime netpoll about starting IO. - err := fd.pd.prepare(int(o.mode), fd.isFile) - if err != nil { - return 0, err +var operationPool = sync.Pool{ + New: func() any { + return new(operation) + }, +} + +// waitIO waits for the IO operation o to complete. +func (fd *FD) waitIO(o *operation) error { + if fd.isBlocking { + panic("can't wait on blocking operations") } - // Start IO. - err = submit(o) - switch err { - case nil: - // IO completed immediately - if o.fd.skipSyncNotif { - // No completion message will follow, so return immediately. - return int(o.qty), nil - } - // Need to get our completion message anyway. - case syscall.ERROR_IO_PENDING: - // IO started, and we have to wait for its completion. - err = nil - default: - return 0, err + if !fd.pollable() { + // The overlapped handle is not added to the runtime poller, + // the only way to wait for the IO to complete is block until + // the overlapped event is signaled. + _, err := syscall.WaitForSingleObject(o.o.HEvent, syscall.INFINITE) + return err } // Wait for our request to complete. - err = fd.pd.wait(int(o.mode), fd.isFile) - if err == nil { - err = windows.WSAGetOverlappedResult(fd.Sysfd, &o.o, &o.qty, false, &o.flags) - // All is good. Extract our IO results and return. - if err != nil { - // More data available. Return back the size of received data. - if err == syscall.ERROR_MORE_DATA || err == windows.WSAEMSGSIZE { - return int(o.qty), err - } - return 0, err - } - return int(o.qty), nil - } - // IO is interrupted by "close" or "timeout" - netpollErr := err - switch netpollErr { - case ErrNetClosing, ErrFileClosing, ErrDeadlineExceeded: - // will deal with those. + err := fd.pd.wait(int(o.mode), fd.isFile) + switch err { + case nil, ErrNetClosing, ErrFileClosing, ErrDeadlineExceeded: + // No other error is expected. default: - panic("unexpected runtime.netpoll error: " + netpollErr.Error()) + panic("unexpected runtime.netpoll error: " + err.Error()) + } + return err +} + +// cancelIO cancels the IO operation o and waits for it to complete. +func (fd *FD) cancelIO(o *operation) { + if !fd.pollable() { + return } // Cancel our request. - err = syscall.CancelIoEx(fd.Sysfd, &o.o) + err := syscall.CancelIoEx(fd.Sysfd, &o.o) // Assuming ERROR_NOT_FOUND is returned, if IO is completed. if err != nil && err != syscall.ERROR_NOT_FOUND { // TODO(brainman): maybe do something else, but panic. panic(err) } - // Wait for cancellation to complete. fd.pd.waitCanceled(int(o.mode)) - err = windows.WSAGetOverlappedResult(fd.Sysfd, &o.o, &o.qty, false, &o.flags) +} + +// pin pins ptr for the duration of the IO operation. +// If fd is in blocking mode, pin does nothing. +func (fd *FD) pin(mode int, ptr any) { + if fd.isBlocking { + return + } + if mode == 'r' { + fd.readPinner.Pin(ptr) + } else { + fd.writePinner.Pin(ptr) + } +} + +// execIO executes a single IO operation o. +// It supports both synchronous and asynchronous IO. +func (fd *FD) execIO(mode int, submit func(o *operation) (uint32, error)) (int, error) { + if mode == 'r' { + defer fd.readPinner.Unpin() + } else { + defer fd.writePinner.Unpin() + } + // Notify runtime netpoll about starting IO. + err := fd.pd.prepare(mode, fd.isFile) if err != nil { - if err == syscall.ERROR_OPERATION_ABORTED { // IO Canceled - err = netpollErr - } return 0, err } - // We issued a cancellation request. But, it seems, IO operation succeeded - // before the cancellation request run. We need to treat the IO operation as - // succeeded (the bytes are actually sent/recv from network). - return int(o.qty), nil + o := operationPool.Get().(*operation) + defer operationPool.Put(o) + *o = operation{ + o: syscall.Overlapped{ + OffsetHigh: uint32(fd.offset >> 32), + Offset: uint32(fd.offset), + }, + runtimeCtx: fd.pd.runtimeCtx, + mode: int32(mode), + } + // Start IO. + if !fd.isBlocking && !fd.pollable() { + // If the handle is opened for overlapped IO but we can't + // use the runtime poller, then we need to use an + // event to wait for the IO to complete. + h, err := windows.CreateEvent(nil, 0, 0, nil) + if err != nil { + // This shouldn't happen when all CreateEvent arguments are zero. + panic(err) + } + // Set the low bit so that the external IOCP doesn't receive the completion packet. + o.o.HEvent = h | 1 + defer syscall.CloseHandle(h) + } + fd.pin(mode, o) + qty, err := submit(o) + var waitErr error + // Blocking operations shouldn't return ERROR_IO_PENDING. + // Continue without waiting if that happens. + if !fd.isBlocking && (err == syscall.ERROR_IO_PENDING || (err == nil && !fd.skipSyncNotif)) { + // IO started asynchronously or completed synchronously but + // a sync notification is required. Wait for it to complete. + waitErr = fd.waitIO(o) + if waitErr != nil { + // IO interrupted by "close" or "timeout". + fd.cancelIO(o) + // We issued a cancellation request, but the IO operation may still succeeded + // before the cancellation request runs. + } + if fd.isFile { + err = windows.GetOverlappedResult(fd.Sysfd, &o.o, &qty, false) + } else { + var flags uint32 + err = windows.WSAGetOverlappedResult(fd.Sysfd, &o.o, &qty, false, &flags) + } + } + switch err { + case syscall.ERROR_OPERATION_ABORTED: + // ERROR_OPERATION_ABORTED may have been caused by us. In that case, + // map it to our own error. Don't do more than that, each submitted + // function may have its own meaning for each error. + if waitErr != nil { + // IO canceled by the poller while waiting for completion. + err = waitErr + } else if fd.kind == kindPipe && fd.closing() { + // Close uses CancelIoEx to interrupt concurrent I/O for pipes. + // If the fd is a pipe and the Write was interrupted by CancelIoEx, + // we assume it is interrupted by Close. + err = errClosing(fd.isFile) + } + case windows.ERROR_IO_INCOMPLETE: + // waitIO couldn't wait for the IO to complete. + if waitErr != nil { + // The wait error will be more informative. + err = waitErr + } + } + return int(qty), err } // FD is a file descriptor. The net and os packages embed this type in @@ -226,16 +341,13 @@ type FD struct { // System file descriptor. Immutable until Close. Sysfd syscall.Handle - // Read operation. - rop operation - // Write operation. - wop operation - // I/O poller. pd pollDesc - // Used to implement pread/pwrite. - l sync.Mutex + // The file offset for the next read or write. + // Overlapped IO operations don't use the real file pointer, + // so we need to keep track of the offset ourselves. + offset int64 // For console I/O. lastbits []byte // first few bytes of the last incomplete rune in last write @@ -256,11 +368,49 @@ type FD struct { // message based socket connection. ZeroReadIsEOF bool - // Whether this is a file rather than a network socket. + // Whether the handle is owned by os.File. isFile bool // The kind of this file. kind fileKind + + // Whether FILE_FLAG_OVERLAPPED was not set when opening the file. + isBlocking bool + + disassociated atomic.Bool + + // readPinner and writePinner are automatically unpinned + // before execIO returns. + readPinner runtime.Pinner + writePinner runtime.Pinner +} + +// setOffset sets the offset fields of the overlapped object +// to the given offset. The fd read/write lock must be held. +// +// Overlapped IO operations don't update the offset fields +// of the overlapped object nor the file pointer automatically, +// so we do that manually here. +// Note that this is a best effort that only works if the file +// pointer is completely owned by this operation. We could +// call seek to allow other processes or other operations on the +// same file to see the updated offset. That would be inefficient +// and won't work for concurrent operations anyway. If concurrent +// operations are needed, then the caller should serialize them +// using an external mechanism. +func (fd *FD) setOffset(off int64) { + fd.offset = off +} + +// addOffset adds the given offset to the current offset. +func (fd *FD) addOffset(off int) { + fd.setOffset(fd.offset + int64(off)) +} + +// pollable should be used instead of fd.pd.pollable(), +// as it is aware of the disassociated state. +func (fd *FD) pollable() bool { + return fd.pd.pollable() && !fd.disassociated.Load() } // fileKind describes the kind of file. @@ -271,91 +421,75 @@ const ( kindFile kindConsole kindPipe + kindFileNet ) -// logInitFD is set by tests to enable file descriptor initialization logging. -var logInitFD func(net string, fd *FD, err error) - // Init initializes the FD. The Sysfd field should already be set. // This can be called multiple times on a single FD. // The net argument is a network name from the net package (e.g., "tcp"), // or "file" or "console" or "dir". // Set pollable to true if fd should be managed by runtime netpoll. -func (fd *FD) Init(net string, pollable bool) (string, error) { +// Pollable must be set to true for overlapped fds. +func (fd *FD) Init(net string, pollable bool) error { if initErr != nil { - return "", initErr + return initErr } switch net { - case "file", "dir": + case "file": fd.kind = kindFile case "console": fd.kind = kindConsole case "pipe": fd.kind = kindPipe - case "tcp", "tcp4", "tcp6", - "udp", "udp4", "udp6", - "ip", "ip4", "ip6", - "unix", "unixgram", "unixpacket": - fd.kind = kindNet + case "file+net": + fd.kind = kindFileNet default: - return "", errors.New("internal error: unknown network type " + net) + // We don't actually care about the various network types. + fd.kind = kindNet } fd.isFile = fd.kind != kindNet + fd.isBlocking = !pollable - var err error - if pollable { - // Only call init for a network socket. - // This means that we don't add files to the runtime poller. - // Adding files to the runtime poller can confuse matters - // if the user is doing their own overlapped I/O. - // See issue #21172. - // - // In general the code below avoids calling the execIO - // function for non-network sockets. If some method does - // somehow call execIO, then execIO, and therefore the - // calling method, will return an error, because - // fd.pd.runtimeCtx will be 0. - err = fd.pd.init(fd) - } - if logInitFD != nil { - logInitFD(net, fd, err) - } + // It is safe to add overlapped handles that also perform I/O + // outside of the runtime poller. The runtime poller will ignore + // I/O completion notifications not initiated by us. + err := fd.pd.init(fd) if err != nil { - return "", err - } - if pollable && useSetFileCompletionNotificationModes { - // We do not use events, so we can skip them always. - flags := uint8(syscall.FILE_SKIP_SET_EVENT_ON_HANDLE) - switch net { - case "tcp", "tcp4", "tcp6", - "udp", "udp4", "udp6": - flags |= syscall.FILE_SKIP_COMPLETION_PORT_ON_SUCCESS - } - err := syscall.SetFileCompletionNotificationModes(fd.Sysfd, flags) - if err == nil && flags&syscall.FILE_SKIP_COMPLETION_PORT_ON_SUCCESS != 0 { - fd.skipSyncNotif = true - } + return err } - // Disable SIO_UDP_CONNRESET behavior. - // http://support.microsoft.com/kb/263823 - switch net { - case "udp", "udp4", "udp6": - ret := uint32(0) - flag := uint32(0) - size := uint32(unsafe.Sizeof(flag)) - err := syscall.WSAIoctl(fd.Sysfd, syscall.SIO_UDP_CONNRESET, (*byte)(unsafe.Pointer(&flag)), size, nil, 0, &ret, nil, 0) - if err != nil { - return "wsaioctl", err - } + if fd.kind != kindNet || socketCanUseSetFileCompletionNotificationModes { + // Non-socket handles can use SetFileCompletionNotificationModes without problems. + err := syscall.SetFileCompletionNotificationModes(fd.Sysfd, + syscall.FILE_SKIP_SET_EVENT_ON_HANDLE|syscall.FILE_SKIP_COMPLETION_PORT_ON_SUCCESS, + ) + fd.skipSyncNotif = err == nil } - fd.rop.mode = 'r' - fd.wop.mode = 'w' - fd.rop.fd = fd - fd.wop.fd = fd - fd.rop.runtimeCtx = fd.pd.runtimeCtx - fd.wop.runtimeCtx = fd.pd.runtimeCtx - return "", nil + return nil +} + +// DisassociateIOCP disassociates the file handle from the IOCP. +// The disassociate operation will not succeed if there is any +// in-progress IO operation on the file handle. +func (fd *FD) DisassociateIOCP() error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + + if fd.isBlocking || !fd.pollable() { + // Nothing to disassociate. + return nil + } + + info := windows.FILE_COMPLETION_INFORMATION{} + if err := windows.NtSetInformationFile(fd.Sysfd, &windows.IO_STATUS_BLOCK{}, unsafe.Pointer(&info), uint32(unsafe.Sizeof(info)), windows.FileReplaceCompletionInformation); err != nil { + return err + } + fd.disassociated.Store(true) + // Don't call fd.pd.close(), it would be too racy. + // There is no harm on leaving fd.pd open until Close is called. + return nil } func (fd *FD) destroy() error { @@ -367,7 +501,7 @@ func (fd *FD) destroy() error { fd.pd.close() var err error switch fd.kind { - case kindNet: + case kindNet, kindFileNet: // The net package uses the CloseFunc variable for testing. err = CloseFunc(fd.Sysfd) default: @@ -384,6 +518,7 @@ func (fd *FD) Close() error { if !fd.fdmu.increfAndClose() { return errClosing(fd.isFile) } + if fd.kind == kindPipe { syscall.CancelIoEx(fd.Sysfd, nil) } @@ -403,10 +538,21 @@ const maxRW = 1 << 30 // 1GB is large enough and keeps subsequent reads aligned // Read implements io.Reader. func (fd *FD) Read(buf []byte) (int, error) { - if err := fd.readLock(); err != nil { - return 0, err + if fd.kind == kindFile { + if err := fd.readWriteLock(); err != nil { + return 0, err + } + defer fd.readWriteUnlock() + } else { + if err := fd.readLock(); err != nil { + return 0, err + } + defer fd.readUnlock() + } + + if len(buf) > 0 { + fd.pin('r', &buf[0]) } - defer fd.readUnlock() if len(buf) > maxRW { buf = buf[:maxRW] @@ -414,29 +560,29 @@ func (fd *FD) Read(buf []byte) (int, error) { var n int var err error - if fd.isFile { - fd.l.Lock() - defer fd.l.Unlock() - switch fd.kind { - case kindConsole: - n, err = fd.readConsole(buf) - default: - n, err = syscall.Read(fd.Sysfd, buf) - if fd.kind == kindPipe && err == syscall.ERROR_OPERATION_ABORTED { - // Close uses CancelIoEx to interrupt concurrent I/O for pipes. - // If the fd is a pipe and the Read was interrupted by CancelIoEx, - // we assume it is interrupted by Close. - err = ErrFileClosing + switch fd.kind { + case kindConsole: + n, err = fd.readConsole(buf) + case kindFile, kindPipe: + n, err = fd.execIO('r', func(o *operation) (qty uint32, err error) { + err = syscall.ReadFile(fd.Sysfd, buf, &qty, fd.overlapped(o)) + return qty, err + }) + fd.addOffset(n) + switch err { + case syscall.ERROR_HANDLE_EOF: + err = io.EOF + case syscall.ERROR_BROKEN_PIPE: + // ReadFile only documents ERROR_BROKEN_PIPE for pipes. + if fd.kind == kindPipe { + err = io.EOF } } - if err != nil { - n = 0 - } - } else { - o := &fd.rop - o.InitBuf(buf) - n, err = execIO(o, func(o *operation) error { - return syscall.WSARecv(o.fd.Sysfd, &o.buf, 1, &o.qty, &o.flags, &o.o, nil) + case kindNet: + n, err = fd.execIO('r', func(o *operation) (qty uint32, err error) { + var flags uint32 + err = syscall.WSARecv(fd.Sysfd, newWsaBuf(buf), 1, &qty, &flags, &o.o, nil) + return qty, err }) if race.Enabled { race.Acquire(unsafe.Pointer(&ioSync)) @@ -523,45 +669,53 @@ func (fd *FD) readConsole(b []byte) (int, error) { } // Pread emulates the Unix pread system call. -func (fd *FD) Pread(b []byte, off int64) (int, error) { +func (fd *FD) Pread(buf []byte, off int64) (int, error) { if fd.kind == kindPipe { // Pread does not work with pipes return 0, syscall.ESPIPE } - // Call incref, not readLock, because since pread specifies the - // offset it is independent from other reads. - if err := fd.incref(); err != nil { + + if err := fd.readWriteLock(); err != nil { return 0, err } - defer fd.decref() + defer fd.readWriteUnlock() - if len(b) > maxRW { - b = b[:maxRW] + if len(buf) > 0 { + fd.pin('r', &buf[0]) } - fd.l.Lock() - defer fd.l.Unlock() - curoffset, e := syscall.Seek(fd.Sysfd, 0, io.SeekCurrent) - if e != nil { - return 0, e - } - defer syscall.Seek(fd.Sysfd, curoffset, io.SeekStart) - o := syscall.Overlapped{ - OffsetHigh: uint32(off >> 32), - Offset: uint32(off), + if len(buf) > maxRW { + buf = buf[:maxRW] } - var done uint32 - e = syscall.ReadFile(fd.Sysfd, b, &done, &o) - if e != nil { - done = 0 - if e == syscall.ERROR_HANDLE_EOF { - e = io.EOF + + if fd.isBlocking { + curoffset, err := syscall.Seek(fd.Sysfd, 0, io.SeekCurrent) + if err != nil { + return 0, err } + defer syscall.Seek(fd.Sysfd, curoffset, io.SeekStart) + defer fd.setOffset(curoffset) + } else { + // Overlapped handles don't have the file pointer updated + // when performing I/O operations, so there is no need to + // call Seek to reset the file pointer. + // Also, some overlapped file handles don't support seeking. + // See https://go.dev/issues/74951. + curoffset := fd.offset + defer fd.setOffset(curoffset) + } + fd.setOffset(off) + n, err := fd.execIO('r', func(o *operation) (qty uint32, err error) { + err = syscall.ReadFile(fd.Sysfd, buf, &qty, &o.o) + return qty, err + }) + if err == syscall.ERROR_HANDLE_EOF { + err = io.EOF } - if len(b) != 0 { - e = fd.eofError(int(done), e) + if len(buf) != 0 { + err = fd.eofError(n, err) } - return int(done), e + return n, err } // ReadFrom wraps the recvfrom network call. @@ -576,20 +730,22 @@ func (fd *FD) ReadFrom(buf []byte) (int, syscall.Sockaddr, error) { return 0, nil, err } defer fd.readUnlock() - o := &fd.rop - o.InitBuf(buf) - n, err := execIO(o, func(o *operation) error { - if o.rsa == nil { - o.rsa = new(syscall.RawSockaddrAny) - } - o.rsan = int32(unsafe.Sizeof(*o.rsa)) - return syscall.WSARecvFrom(o.fd.Sysfd, &o.buf, 1, &o.qty, &o.flags, o.rsa, &o.rsan, &o.o, nil) + + fd.pin('r', &buf[0]) + + rsa := wsaRsaPool.Get().(*syscall.RawSockaddrAny) + defer wsaRsaPool.Put(rsa) + n, err := fd.execIO('r', func(o *operation) (qty uint32, err error) { + rsan := int32(unsafe.Sizeof(*rsa)) + var flags uint32 + err = syscall.WSARecvFrom(fd.Sysfd, newWsaBuf(buf), 1, &qty, &flags, rsa, &rsan, &o.o, nil) + return qty, err }) err = fd.eofError(n, err) if err != nil { return n, nil, err } - sa, _ := o.rsa.Sockaddr() + sa, _ := rsa.Sockaddr() return n, sa, nil } @@ -605,20 +761,22 @@ func (fd *FD) ReadFromInet4(buf []byte, sa4 *syscall.SockaddrInet4) (int, error) return 0, err } defer fd.readUnlock() - o := &fd.rop - o.InitBuf(buf) - n, err := execIO(o, func(o *operation) error { - if o.rsa == nil { - o.rsa = new(syscall.RawSockaddrAny) - } - o.rsan = int32(unsafe.Sizeof(*o.rsa)) - return syscall.WSARecvFrom(o.fd.Sysfd, &o.buf, 1, &o.qty, &o.flags, o.rsa, &o.rsan, &o.o, nil) + + fd.pin('r', &buf[0]) + + rsa := wsaRsaPool.Get().(*syscall.RawSockaddrAny) + defer wsaRsaPool.Put(rsa) + n, err := fd.execIO('r', func(o *operation) (qty uint32, err error) { + rsan := int32(unsafe.Sizeof(*rsa)) + var flags uint32 + err = syscall.WSARecvFrom(fd.Sysfd, newWsaBuf(buf), 1, &qty, &flags, rsa, &rsan, &o.o, nil) + return qty, err }) err = fd.eofError(n, err) if err != nil { return n, err } - rawToSockaddrInet4(o.rsa, sa4) + rawToSockaddrInet4(rsa, sa4) return n, err } @@ -634,75 +792,77 @@ func (fd *FD) ReadFromInet6(buf []byte, sa6 *syscall.SockaddrInet6) (int, error) return 0, err } defer fd.readUnlock() - o := &fd.rop - o.InitBuf(buf) - n, err := execIO(o, func(o *operation) error { - if o.rsa == nil { - o.rsa = new(syscall.RawSockaddrAny) - } - o.rsan = int32(unsafe.Sizeof(*o.rsa)) - return syscall.WSARecvFrom(o.fd.Sysfd, &o.buf, 1, &o.qty, &o.flags, o.rsa, &o.rsan, &o.o, nil) + + fd.pin('r', &buf[0]) + + rsa := wsaRsaPool.Get().(*syscall.RawSockaddrAny) + defer wsaRsaPool.Put(rsa) + n, err := fd.execIO('r', func(o *operation) (qty uint32, err error) { + rsan := int32(unsafe.Sizeof(*rsa)) + var flags uint32 + err = syscall.WSARecvFrom(fd.Sysfd, newWsaBuf(buf), 1, &qty, &flags, rsa, &rsan, &o.o, nil) + return qty, err }) err = fd.eofError(n, err) if err != nil { return n, err } - rawToSockaddrInet6(o.rsa, sa6) + rawToSockaddrInet6(rsa, sa6) return n, err } // Write implements io.Writer. func (fd *FD) Write(buf []byte) (int, error) { - if err := fd.writeLock(); err != nil { - return 0, err - } - defer fd.writeUnlock() - if fd.isFile { - fd.l.Lock() - defer fd.l.Unlock() + if fd.kind == kindFile { + if err := fd.readWriteLock(); err != nil { + return 0, err + } + defer fd.readWriteUnlock() + } else { + if err := fd.writeLock(); err != nil { + return 0, err + } + defer fd.writeUnlock() } - ntotal := 0 - for len(buf) > 0 { - b := buf - if len(b) > maxRW { - b = b[:maxRW] + if len(buf) > 0 { + fd.pin('w', &buf[0]) + } + var ntotal int + for { + max := len(buf) + if max-ntotal > maxRW { + max = ntotal + maxRW } + b := buf[ntotal:max] var n int var err error - if fd.isFile { - switch fd.kind { - case kindConsole: - n, err = fd.writeConsole(b) - default: - n, err = syscall.Write(fd.Sysfd, b) - if fd.kind == kindPipe && err == syscall.ERROR_OPERATION_ABORTED { - // Close uses CancelIoEx to interrupt concurrent I/O for pipes. - // If the fd is a pipe and the Write was interrupted by CancelIoEx, - // we assume it is interrupted by Close. - err = ErrFileClosing - } - } - if err != nil { - n = 0 - } - } else { + switch fd.kind { + case kindConsole: + n, err = fd.writeConsole(b) + case kindPipe, kindFile: + n, err = fd.execIO('w', func(o *operation) (qty uint32, err error) { + err = syscall.WriteFile(fd.Sysfd, b, &qty, fd.overlapped(o)) + return qty, err + }) + fd.addOffset(n) + case kindNet: if race.Enabled { race.ReleaseMerge(unsafe.Pointer(&ioSync)) } - o := &fd.wop - o.InitBuf(b) - n, err = execIO(o, func(o *operation) error { - return syscall.WSASend(o.fd.Sysfd, &o.buf, 1, &o.qty, 0, &o.o, nil) + n, err = fd.execIO('w', func(o *operation) (qty uint32, err error) { + err = syscall.WSASend(fd.Sysfd, newWsaBuf(b), 1, &qty, 0, &o.o, nil) + return qty, err }) } ntotal += n - if err != nil { + if ntotal == len(buf) || err != nil { return ntotal, err } - buf = buf[n:] + if n == 0 { + return ntotal, io.ErrUnexpectedEOF + } } - return ntotal, nil } // writeConsole writes len(b) bytes to the console File. @@ -754,41 +914,54 @@ func (fd *FD) Pwrite(buf []byte, off int64) (int, error) { // Pwrite does not work with pipes return 0, syscall.ESPIPE } - // Call incref, not writeLock, because since pwrite specifies the - // offset it is independent from other writes. - if err := fd.incref(); err != nil { + + if err := fd.readWriteLock(); err != nil { return 0, err } - defer fd.decref() + defer fd.readWriteUnlock() - fd.l.Lock() - defer fd.l.Unlock() - curoffset, e := syscall.Seek(fd.Sysfd, 0, io.SeekCurrent) - if e != nil { - return 0, e + if len(buf) > 0 { + fd.pin('w', &buf[0]) } - defer syscall.Seek(fd.Sysfd, curoffset, io.SeekStart) - ntotal := 0 - for len(buf) > 0 { - b := buf - if len(b) > maxRW { - b = b[:maxRW] + if fd.isBlocking { + curoffset, err := syscall.Seek(fd.Sysfd, 0, io.SeekCurrent) + if err != nil { + return 0, err } - var n uint32 - o := syscall.Overlapped{ - OffsetHigh: uint32(off >> 32), - Offset: uint32(off), + defer syscall.Seek(fd.Sysfd, curoffset, io.SeekStart) + defer fd.setOffset(curoffset) + } else { + // Overlapped handles don't have the file pointer updated + // when performing I/O operations, so there is no need to + // call Seek to reset the file pointer. + // Also, some overlapped file handles don't support seeking. + // See https://go.dev/issues/74951. + curoffset := fd.offset + defer fd.setOffset(curoffset) + } + + var ntotal int + for { + max := len(buf) + if max-ntotal > maxRW { + max = ntotal + maxRW } - e = syscall.WriteFile(fd.Sysfd, b, &n, &o) - ntotal += int(n) - if e != nil { - return ntotal, e + fd.setOffset(off + int64(ntotal)) + n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { + err = syscall.WriteFile(fd.Sysfd, buf[ntotal:max], &qty, &o.o) + return qty, err + }) + if n > 0 { + ntotal += n + } + if ntotal == len(buf) || err != nil { + return ntotal, err + } + if n == 0 { + return ntotal, io.ErrUnexpectedEOF } - buf = buf[n:] - off += int64(n) } - return ntotal, nil } // Writev emulates the Unix writev system call. @@ -803,12 +976,12 @@ func (fd *FD) Writev(buf *[][]byte) (int64, error) { if race.Enabled { race.ReleaseMerge(unsafe.Pointer(&ioSync)) } - o := &fd.wop - o.InitBufs(buf) - n, err := execIO(o, func(o *operation) error { - return syscall.WSASend(o.fd.Sysfd, &o.bufs[0], uint32(len(o.bufs)), &o.qty, 0, &o.o, nil) + bufs := newWSABufs(buf) + defer freeWSABufs(bufs) + n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { + err = syscall.WSASend(fd.Sysfd, &(*bufs)[0], uint32(len(*bufs)), &qty, 0, &o.o, nil) + return qty, err }) - o.ClearBufs() TestHookDidWritev(n) consume(buf, int64(n)) return int64(n), err @@ -823,26 +996,24 @@ func (fd *FD) WriteTo(buf []byte, sa syscall.Sockaddr) (int, error) { if len(buf) == 0 { // handle zero-byte payload - o := &fd.wop - o.InitBuf(buf) - o.sa = sa - n, err := execIO(o, func(o *operation) error { - return syscall.WSASendto(o.fd.Sysfd, &o.buf, 1, &o.qty, 0, o.sa, &o.o, nil) + n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { + err = syscall.WSASendto(fd.Sysfd, &syscall.WSABuf{}, 1, &qty, 0, sa, &o.o, nil) + return qty, err }) return n, err } + fd.pin('w', &buf[0]) + ntotal := 0 for len(buf) > 0 { b := buf if len(b) > maxRW { b = b[:maxRW] } - o := &fd.wop - o.InitBuf(b) - o.sa = sa - n, err := execIO(o, func(o *operation) error { - return syscall.WSASendto(o.fd.Sysfd, &o.buf, 1, &o.qty, 0, o.sa, &o.o, nil) + n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { + err = syscall.WSASendto(fd.Sysfd, newWsaBuf(b), 1, &qty, 0, sa, &o.o, nil) + return qty, err }) ntotal += int(n) if err != nil { @@ -862,24 +1033,24 @@ func (fd *FD) WriteToInet4(buf []byte, sa4 *syscall.SockaddrInet4) (int, error) if len(buf) == 0 { // handle zero-byte payload - o := &fd.wop - o.InitBuf(buf) - n, err := execIO(o, func(o *operation) error { - return windows.WSASendtoInet4(o.fd.Sysfd, &o.buf, 1, &o.qty, 0, sa4, &o.o, nil) + n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { + err = windows.WSASendtoInet4(fd.Sysfd, &syscall.WSABuf{}, 1, &qty, 0, sa4, &o.o, nil) + return qty, err }) return n, err } + fd.pin('w', &buf[0]) + ntotal := 0 for len(buf) > 0 { b := buf if len(b) > maxRW { b = b[:maxRW] } - o := &fd.wop - o.InitBuf(b) - n, err := execIO(o, func(o *operation) error { - return windows.WSASendtoInet4(o.fd.Sysfd, &o.buf, 1, &o.qty, 0, sa4, &o.o, nil) + n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { + err = windows.WSASendtoInet4(fd.Sysfd, newWsaBuf(b), 1, &qty, 0, sa4, &o.o, nil) + return qty, err }) ntotal += int(n) if err != nil { @@ -899,24 +1070,24 @@ func (fd *FD) WriteToInet6(buf []byte, sa6 *syscall.SockaddrInet6) (int, error) if len(buf) == 0 { // handle zero-byte payload - o := &fd.wop - o.InitBuf(buf) - n, err := execIO(o, func(o *operation) error { - return windows.WSASendtoInet6(o.fd.Sysfd, &o.buf, 1, &o.qty, 0, sa6, &o.o, nil) + n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { + err = windows.WSASendtoInet6(fd.Sysfd, &syscall.WSABuf{}, 1, &qty, 0, sa6, &o.o, nil) + return qty, err }) return n, err } + fd.pin('w', &buf[0]) + ntotal := 0 for len(buf) > 0 { b := buf if len(b) > maxRW { b = b[:maxRW] } - o := &fd.wop - o.InitBuf(b) - n, err := execIO(o, func(o *operation) error { - return windows.WSASendtoInet6(o.fd.Sysfd, &o.buf, 1, &o.qty, 0, sa6, &o.o, nil) + n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { + err = windows.WSASendtoInet6(fd.Sysfd, newWsaBuf(b), 1, &qty, 0, sa6, &o.o, nil) + return qty, err }) ntotal += int(n) if err != nil { @@ -931,20 +1102,19 @@ func (fd *FD) WriteToInet6(buf []byte, sa6 *syscall.SockaddrInet6) (int, error) // called when the descriptor is first created. This is here rather // than in the net package so that it can use fd.wop. func (fd *FD) ConnectEx(ra syscall.Sockaddr) error { - o := &fd.wop - o.sa = ra - _, err := execIO(o, func(o *operation) error { - return ConnectExFunc(o.fd.Sysfd, o.sa, nil, 0, nil, &o.o) + _, err := fd.execIO('w', func(o *operation) (uint32, error) { + return 0, ConnectExFunc(fd.Sysfd, ra, nil, 0, nil, &o.o) }) return err } -func (fd *FD) acceptOne(s syscall.Handle, rawsa []syscall.RawSockaddrAny, o *operation) (string, error) { +func (fd *FD) acceptOne(s syscall.Handle, rawsa []syscall.RawSockaddrAny) (string, error) { // Submit accept request. - o.handle = s - o.rsan = int32(unsafe.Sizeof(rawsa[0])) - _, err := execIO(o, func(o *operation) error { - return AcceptFunc(o.fd.Sysfd, o.handle, (*byte)(unsafe.Pointer(&rawsa[0])), 0, uint32(o.rsan), uint32(o.rsan), &o.qty, &o.o) + rsan := uint32(unsafe.Sizeof(rawsa[0])) + _, err := fd.execIO('r', func(o *operation) (qty uint32, err error) { + err = AcceptFunc(fd.Sysfd, s, (*byte)(unsafe.Pointer(&rawsa[0])), 0, rsan, rsan, &qty, &o.o) + return qty, err + }) if err != nil { CloseFunc(s) @@ -969,7 +1139,6 @@ func (fd *FD) Accept(sysSocket func() (syscall.Handle, error)) (syscall.Handle, } defer fd.readUnlock() - o := &fd.rop var rawsa [2]syscall.RawSockaddrAny for { s, err := sysSocket() @@ -977,9 +1146,9 @@ func (fd *FD) Accept(sysSocket func() (syscall.Handle, error)) (syscall.Handle, return syscall.InvalidHandle, nil, 0, "", err } - errcall, err := fd.acceptOne(s, rawsa[:], o) + errcall, err := fd.acceptOne(s, rawsa[:]) if err == nil { - return s, rawsa[:], uint32(o.rsan), "", nil + return s, rawsa[:], uint32(unsafe.Sizeof(rawsa[0])), "", nil } // Sometimes we see WSAECONNRESET and ERROR_NETNAME_DELETED is @@ -1005,15 +1174,38 @@ func (fd *FD) Seek(offset int64, whence int) (int64, error) { if fd.kind == kindPipe { return 0, syscall.ESPIPE } - if err := fd.incref(); err != nil { + if err := fd.readWriteLock(); err != nil { return 0, err } - defer fd.decref() - - fd.l.Lock() - defer fd.l.Unlock() - - return syscall.Seek(fd.Sysfd, offset, whence) + defer fd.readWriteUnlock() + + if !fd.isBlocking { + // Windows doesn't use the file pointer for overlapped file handles, + // there is no point on calling syscall.Seek. + var newOffset int64 + switch whence { + case io.SeekStart: + newOffset = offset + case io.SeekCurrent: + newOffset = fd.offset + offset + case io.SeekEnd: + var size int64 + if err := windows.GetFileSizeEx(fd.Sysfd, &size); err != nil { + return 0, err + } + newOffset = size + offset + default: + return 0, windows.ERROR_INVALID_PARAMETER + } + if newOffset < 0 { + return 0, windows.ERROR_NEGATIVE_SEEK + } + fd.setOffset(newOffset) + return newOffset, nil + } + n, err := syscall.Seek(fd.Sysfd, offset, whence) + fd.setOffset(n) + return n, err } // Fchmod updates syscall.ByHandleFileInformation.Fileattributes when needed. @@ -1082,13 +1274,13 @@ func (fd *FD) RawRead(f func(uintptr) bool) error { // Use a zero-byte read as a way to get notified when this // socket is readable. h/t https://stackoverflow.com/a/42019668/332798 - o := &fd.rop - o.InitBuf(nil) - if !fd.IsStream { - o.flags |= windows.MSG_PEEK - } - _, err := execIO(o, func(o *operation) error { - return syscall.WSARecv(o.fd.Sysfd, &o.buf, 1, &o.qty, &o.flags, &o.o, nil) + _, err := fd.execIO('r', func(o *operation) (qty uint32, err error) { + var flags uint32 + if !fd.IsStream { + flags |= windows.MSG_PEEK + } + err = syscall.WSARecv(fd.Sysfd, &syscall.WSABuf{}, 1, &qty, &flags, &o.o, nil) + return qty, err }) if err == windows.WSAEMSGSIZE { // expected with a 0-byte peek, ignore. @@ -1175,23 +1367,18 @@ func (fd *FD) ReadMsg(p []byte, oob []byte, flags int) (int, int, int, syscall.S p = p[:maxRW] } - o := &fd.rop - o.InitMsg(p, oob) - if o.rsa == nil { - o.rsa = new(syscall.RawSockaddrAny) - } - o.msg.Name = (syscall.Pointer)(unsafe.Pointer(o.rsa)) - o.msg.Namelen = int32(unsafe.Sizeof(*o.rsa)) - o.msg.Flags = uint32(flags) - n, err := execIO(o, func(o *operation) error { - return windows.WSARecvMsg(o.fd.Sysfd, &o.msg, &o.qty, &o.o, nil) + msg := newWSAMsg(p, oob, flags, true) + defer freeWSAMsg(msg) + n, err := fd.execIO('r', func(o *operation) (qty uint32, err error) { + err = windows.WSARecvMsg(fd.Sysfd, msg, &qty, &o.o, nil) + return qty, err }) err = fd.eofError(n, err) var sa syscall.Sockaddr if err == nil { - sa, err = o.rsa.Sockaddr() + sa, err = msg.Name.Sockaddr() } - return n, int(o.msg.Control.Len), int(o.msg.Flags), sa, err + return n, int(msg.Control.Len), int(msg.Flags), sa, err } // ReadMsgInet4 is ReadMsg, but specialized to return a syscall.SockaddrInet4. @@ -1205,22 +1392,17 @@ func (fd *FD) ReadMsgInet4(p []byte, oob []byte, flags int, sa4 *syscall.Sockadd p = p[:maxRW] } - o := &fd.rop - o.InitMsg(p, oob) - if o.rsa == nil { - o.rsa = new(syscall.RawSockaddrAny) - } - o.msg.Name = (syscall.Pointer)(unsafe.Pointer(o.rsa)) - o.msg.Namelen = int32(unsafe.Sizeof(*o.rsa)) - o.msg.Flags = uint32(flags) - n, err := execIO(o, func(o *operation) error { - return windows.WSARecvMsg(o.fd.Sysfd, &o.msg, &o.qty, &o.o, nil) + msg := newWSAMsg(p, oob, flags, true) + defer freeWSAMsg(msg) + n, err := fd.execIO('r', func(o *operation) (qty uint32, err error) { + err = windows.WSARecvMsg(fd.Sysfd, msg, &qty, &o.o, nil) + return qty, err }) err = fd.eofError(n, err) if err == nil { - rawToSockaddrInet4(o.rsa, sa4) + rawToSockaddrInet4(msg.Name, sa4) } - return n, int(o.msg.Control.Len), int(o.msg.Flags), err + return n, int(msg.Control.Len), int(msg.Flags), err } // ReadMsgInet6 is ReadMsg, but specialized to return a syscall.SockaddrInet6. @@ -1234,22 +1416,17 @@ func (fd *FD) ReadMsgInet6(p []byte, oob []byte, flags int, sa6 *syscall.Sockadd p = p[:maxRW] } - o := &fd.rop - o.InitMsg(p, oob) - if o.rsa == nil { - o.rsa = new(syscall.RawSockaddrAny) - } - o.msg.Name = (syscall.Pointer)(unsafe.Pointer(o.rsa)) - o.msg.Namelen = int32(unsafe.Sizeof(*o.rsa)) - o.msg.Flags = uint32(flags) - n, err := execIO(o, func(o *operation) error { - return windows.WSARecvMsg(o.fd.Sysfd, &o.msg, &o.qty, &o.o, nil) + msg := newWSAMsg(p, oob, flags, true) + defer freeWSAMsg(msg) + n, err := fd.execIO('r', func(o *operation) (qty uint32, err error) { + err = windows.WSARecvMsg(fd.Sysfd, msg, &qty, &o.o, nil) + return qty, err }) err = fd.eofError(n, err) if err == nil { - rawToSockaddrInet6(o.rsa, sa6) + rawToSockaddrInet6(msg.Name, sa6) } - return n, int(o.msg.Control.Len), int(o.msg.Flags), err + return n, int(msg.Control.Len), int(msg.Flags), err } // WriteMsg wraps the WSASendMsg network call. @@ -1263,23 +1440,20 @@ func (fd *FD) WriteMsg(p []byte, oob []byte, sa syscall.Sockaddr) (int, int, err } defer fd.writeUnlock() - o := &fd.wop - o.InitMsg(p, oob) + msg := newWSAMsg(p, oob, 0, sa != nil) + defer freeWSAMsg(msg) if sa != nil { - if o.rsa == nil { - o.rsa = new(syscall.RawSockaddrAny) - } - len, err := sockaddrToRaw(o.rsa, sa) + var err error + msg.Namelen, err = sockaddrToRaw(msg.Name, sa) if err != nil { return 0, 0, err } - o.msg.Name = (syscall.Pointer)(unsafe.Pointer(o.rsa)) - o.msg.Namelen = len } - n, err := execIO(o, func(o *operation) error { - return windows.WSASendMsg(o.fd.Sysfd, &o.msg, 0, &o.qty, &o.o, nil) + n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { + err = windows.WSASendMsg(fd.Sysfd, msg, 0, nil, &o.o, nil) + return qty, err }) - return n, int(o.msg.Control.Len), err + return n, int(msg.Control.Len), err } // WriteMsgInet4 is WriteMsg specialized for syscall.SockaddrInet4. @@ -1293,18 +1467,16 @@ func (fd *FD) WriteMsgInet4(p []byte, oob []byte, sa *syscall.SockaddrInet4) (in } defer fd.writeUnlock() - o := &fd.wop - o.InitMsg(p, oob) - if o.rsa == nil { - o.rsa = new(syscall.RawSockaddrAny) + msg := newWSAMsg(p, oob, 0, sa != nil) + defer freeWSAMsg(msg) + if sa != nil { + msg.Namelen = sockaddrInet4ToRaw(msg.Name, sa) } - len := sockaddrInet4ToRaw(o.rsa, sa) - o.msg.Name = (syscall.Pointer)(unsafe.Pointer(o.rsa)) - o.msg.Namelen = len - n, err := execIO(o, func(o *operation) error { - return windows.WSASendMsg(o.fd.Sysfd, &o.msg, 0, &o.qty, &o.o, nil) + n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { + err = windows.WSASendMsg(fd.Sysfd, msg, 0, nil, &o.o, nil) + return qty, err }) - return n, int(o.msg.Control.Len), err + return n, int(msg.Control.Len), err } // WriteMsgInet6 is WriteMsg specialized for syscall.SockaddrInet6. @@ -1318,18 +1490,16 @@ func (fd *FD) WriteMsgInet6(p []byte, oob []byte, sa *syscall.SockaddrInet6) (in } defer fd.writeUnlock() - o := &fd.wop - o.InitMsg(p, oob) - if o.rsa == nil { - o.rsa = new(syscall.RawSockaddrAny) + msg := newWSAMsg(p, oob, 0, sa != nil) + defer freeWSAMsg(msg) + if sa != nil { + msg.Namelen = sockaddrInet6ToRaw(msg.Name, sa) } - len := sockaddrInet6ToRaw(o.rsa, sa) - o.msg.Name = (syscall.Pointer)(unsafe.Pointer(o.rsa)) - o.msg.Namelen = len - n, err := execIO(o, func(o *operation) error { - return windows.WSASendMsg(o.fd.Sysfd, &o.msg, 0, &o.qty, &o.o, nil) + n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { + err = windows.WSASendMsg(fd.Sysfd, msg, 0, nil, &o.o, nil) + return qty, err }) - return n, int(o.msg.Control.Len), err + return n, int(msg.Control.Len), err } func DupCloseOnExec(fd int) (int, string, error) { diff --git a/src/internal/poll/fd_windows_test.go b/src/internal/poll/fd_windows_test.go index 8bf92be7c32b91..6c7604fd74d305 100644 --- a/src/internal/poll/fd_windows_test.go +++ b/src/internal/poll/fd_windows_test.go @@ -6,123 +6,28 @@ package poll_test import ( "errors" - "fmt" "internal/poll" "internal/syscall/windows" + "io" "os" - "sync" + "path/filepath" "syscall" "testing" "unsafe" ) -type loggedFD struct { - Net string - FD *poll.FD - Err error -} - -var ( - logMu sync.Mutex - loggedFDs map[syscall.Handle]*loggedFD -) - -func logFD(net string, fd *poll.FD, err error) { - logMu.Lock() - defer logMu.Unlock() - - loggedFDs[fd.Sysfd] = &loggedFD{ - Net: net, - FD: fd, - Err: err, - } -} - func init() { - loggedFDs = make(map[syscall.Handle]*loggedFD) - *poll.LogInitFD = logFD - poll.InitWSA() } -func findLoggedFD(h syscall.Handle) (lfd *loggedFD, found bool) { - logMu.Lock() - defer logMu.Unlock() - - lfd, found = loggedFDs[h] - return lfd, found -} - -// checkFileIsNotPartOfNetpoll verifies that f is not managed by netpoll. -// It returns error, if check fails. -func checkFileIsNotPartOfNetpoll(f *os.File) error { - lfd, found := findLoggedFD(syscall.Handle(f.Fd())) - if !found { - return fmt.Errorf("%v fd=%v: is not found in the log", f.Name(), f.Fd()) - } - if lfd.FD.IsPartOfNetpoll() { - return fmt.Errorf("%v fd=%v: is part of netpoll, but should not be (logged: net=%v err=%v)", f.Name(), f.Fd(), lfd.Net, lfd.Err) - } - return nil -} - -func TestFileFdsAreInitialised(t *testing.T) { - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - f, err := os.Open(exe) - if err != nil { - t.Fatal(err) - } - defer f.Close() - - err = checkFileIsNotPartOfNetpoll(f) - if err != nil { - t.Fatal(err) - } -} - -func TestSerialFdsAreInitialised(t *testing.T) { - for _, name := range []string{"COM1", "COM2", "COM3", "COM4"} { - t.Run(name, func(t *testing.T) { - h, err := syscall.CreateFile(syscall.StringToUTF16Ptr(name), - syscall.GENERIC_READ|syscall.GENERIC_WRITE, - 0, - nil, - syscall.OPEN_EXISTING, - syscall.FILE_ATTRIBUTE_NORMAL|syscall.FILE_FLAG_OVERLAPPED, - 0) - if err != nil { - if errno, ok := err.(syscall.Errno); ok { - switch errno { - case syscall.ERROR_FILE_NOT_FOUND, - syscall.ERROR_ACCESS_DENIED: - t.Log("Skipping: ", err) - return - } - } - t.Fatal(err) - } - f := os.NewFile(uintptr(h), name) - defer f.Close() - - err = checkFileIsNotPartOfNetpoll(f) - if err != nil { - t.Fatal(err) - } - }) - } -} - func TestWSASocketConflict(t *testing.T) { + t.Parallel() s, err := windows.WSASocket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP, nil, 0, windows.WSA_FLAG_OVERLAPPED) if err != nil { t.Fatal(err) } fd := poll.FD{Sysfd: s, IsStream: true, ZeroReadIsEOF: true} - _, err = fd.Init("tcp", true) - if err != nil { + if err = fd.Init("tcp", true); err != nil { syscall.CloseHandle(s) t.Fatal(err) } @@ -185,3 +90,77 @@ type _TCP_INFO_v0 struct { TimeoutEpisodes uint32 SynRetrans uint8 } + +func newFD(t testing.TB, h syscall.Handle, kind string, overlapped bool) *poll.FD { + fd := poll.FD{ + Sysfd: h, + IsStream: true, + ZeroReadIsEOF: true, + } + err := fd.Init(kind, overlapped) + if overlapped && err != nil { + // Overlapped file handles should not error. + fd.Close() + t.Fatal(err) + } + t.Cleanup(func() { + fd.Close() + }) + return &fd +} + +func newFile(t testing.TB, name string, overlapped bool) *poll.FD { + namep, err := syscall.UTF16PtrFromString(name) + if err != nil { + t.Fatal(err) + } + flags := syscall.FILE_ATTRIBUTE_NORMAL + if overlapped { + flags |= syscall.FILE_FLAG_OVERLAPPED + } + h, err := syscall.CreateFile(namep, + syscall.GENERIC_READ|syscall.GENERIC_WRITE, + syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_READ, + nil, syscall.OPEN_ALWAYS, uint32(flags), 0) + if err != nil { + t.Fatal(err) + } + typ, err := syscall.GetFileType(h) + if err != nil { + syscall.CloseHandle(h) + t.Fatal(err) + } + kind := "file" + if typ == syscall.FILE_TYPE_PIPE { + kind = "pipe" + } + return newFD(t, h, kind, overlapped) +} + +func BenchmarkReadOverlapped(b *testing.B) { + benchmarkRead(b, true) +} + +func BenchmarkReadSync(b *testing.B) { + benchmarkRead(b, false) +} + +func benchmarkRead(b *testing.B, overlapped bool) { + name := filepath.Join(b.TempDir(), "foo") + const content = "hello world" + err := os.WriteFile(name, []byte(content), 0644) + if err != nil { + b.Fatal(err) + } + file := newFile(b, name, overlapped) + var buf [len(content)]byte + for b.Loop() { + _, err := io.ReadFull(file, buf[:]) + if err != nil { + b.Fatal(err) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + b.Fatal(err) + } + } +} diff --git a/src/internal/poll/sendfile.go b/src/internal/poll/sendfile.go index 41b0481c1aa38a..696d93353ebea2 100644 --- a/src/internal/poll/sendfile.go +++ b/src/internal/poll/sendfile.go @@ -4,4 +4,4 @@ package poll -var TestHookDidSendFile = func(dstFD *FD, src int, written int64, err error, handled bool) {} +var TestHookDidSendFile = func(dstFD *FD, src uintptr, written int64, err error, handled bool) {} diff --git a/src/internal/poll/sendfile_bsd.go b/src/internal/poll/sendfile_bsd.go deleted file mode 100644 index 669df94cc12e0d..00000000000000 --- a/src/internal/poll/sendfile_bsd.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build darwin || dragonfly || freebsd - -package poll - -import "syscall" - -// maxSendfileSize is the largest chunk size we ask the kernel to copy -// at a time. -const maxSendfileSize int = 4 << 20 - -// SendFile wraps the sendfile system call. -func SendFile(dstFD *FD, src int, pos, remain int64) (written int64, err error, handled bool) { - defer func() { - TestHookDidSendFile(dstFD, src, written, err, handled) - }() - if err := dstFD.writeLock(); err != nil { - return 0, err, false - } - defer dstFD.writeUnlock() - - if err := dstFD.pd.prepareWrite(dstFD.isFile); err != nil { - return 0, err, false - } - - dst := dstFD.Sysfd - for remain > 0 { - n := maxSendfileSize - if int64(n) > remain { - n = int(remain) - } - pos1 := pos - n, err = syscall.Sendfile(dst, src, &pos1, n) - if n > 0 { - pos += int64(n) - written += int64(n) - remain -= int64(n) - } - if err == syscall.EINTR { - continue - } - // This includes syscall.ENOSYS (no kernel - // support) and syscall.EINVAL (fd types which - // don't implement sendfile), and other errors. - // We should end the loop when there is no error - // returned from sendfile(2) or it is not a retryable error. - if err != syscall.EAGAIN { - break - } - if err = dstFD.pd.waitWrite(dstFD.isFile); err != nil { - break - } - } - handled = written != 0 || (err != syscall.ENOSYS && err != syscall.EINVAL) - return -} diff --git a/src/internal/poll/sendfile_linux.go b/src/internal/poll/sendfile_linux.go deleted file mode 100644 index d1c4d5c0d3d34d..00000000000000 --- a/src/internal/poll/sendfile_linux.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package poll - -import "syscall" - -// maxSendfileSize is the largest chunk size we ask the kernel to copy -// at a time. -const maxSendfileSize int = 4 << 20 - -// SendFile wraps the sendfile system call. -func SendFile(dstFD *FD, src int, remain int64) (written int64, err error, handled bool) { - defer func() { - TestHookDidSendFile(dstFD, src, written, err, handled) - }() - if err := dstFD.writeLock(); err != nil { - return 0, err, false - } - defer dstFD.writeUnlock() - - if err := dstFD.pd.prepareWrite(dstFD.isFile); err != nil { - return 0, err, false - } - - dst := dstFD.Sysfd - for remain > 0 { - n := maxSendfileSize - if int64(n) > remain { - n = int(remain) - } - n, err = syscall.Sendfile(dst, src, nil, n) - if n > 0 { - written += int64(n) - remain -= int64(n) - continue - } else if err != syscall.EAGAIN && err != syscall.EINTR { - // This includes syscall.ENOSYS (no kernel - // support) and syscall.EINVAL (fd types which - // don't implement sendfile), and other errors. - // We should end the loop when there is no error - // returned from sendfile(2) or it is not a retryable error. - break - } - if err == syscall.EINTR { - continue - } - if err = dstFD.pd.waitWrite(dstFD.isFile); err != nil { - break - } - } - handled = written != 0 || (err != syscall.ENOSYS && err != syscall.EINVAL) - return -} diff --git a/src/internal/poll/sendfile_solaris.go b/src/internal/poll/sendfile_solaris.go index ec675833a225dc..605323b976686d 100644 --- a/src/internal/poll/sendfile_solaris.go +++ b/src/internal/poll/sendfile_solaris.go @@ -4,63 +4,9 @@ package poll -import "syscall" +//go:cgo_ldflag "-lsendfile" // Not strictly needed, but very helpful for debugging, see issue #10221. // //go:cgo_import_dynamic _ _ "libsendfile.so" //go:cgo_import_dynamic _ _ "libsocket.so" - -// maxSendfileSize is the largest chunk size we ask the kernel to copy -// at a time. -const maxSendfileSize int = 4 << 20 - -// SendFile wraps the sendfile system call. -func SendFile(dstFD *FD, src int, pos, remain int64) (written int64, err error, handled bool) { - defer func() { - TestHookDidSendFile(dstFD, src, written, err, handled) - }() - if err := dstFD.writeLock(); err != nil { - return 0, err, false - } - defer dstFD.writeUnlock() - - if err := dstFD.pd.prepareWrite(dstFD.isFile); err != nil { - return 0, err, false - } - - dst := dstFD.Sysfd - for remain > 0 { - n := maxSendfileSize - if int64(n) > remain { - n = int(remain) - } - pos1 := pos - n, err = syscall.Sendfile(dst, src, &pos1, n) - if err == syscall.EAGAIN || err == syscall.EINTR { - // partial write may have occurred - n = int(pos1 - pos) - } - if n > 0 { - pos += int64(n) - written += int64(n) - remain -= int64(n) - continue - } else if err != syscall.EAGAIN && err != syscall.EINTR { - // This includes syscall.ENOSYS (no kernel - // support) and syscall.EINVAL (fd types which - // don't implement sendfile), and other errors. - // We should end the loop when there is no error - // returned from sendfile(2) or it is not a retryable error. - break - } - if err == syscall.EINTR { - continue - } - if err = dstFD.pd.waitWrite(dstFD.isFile); err != nil { - break - } - } - handled = written != 0 || (err != syscall.ENOSYS && err != syscall.EINVAL) - return -} diff --git a/src/internal/poll/sendfile_unix.go b/src/internal/poll/sendfile_unix.go new file mode 100644 index 00000000000000..4b7e9fea9e8dc9 --- /dev/null +++ b/src/internal/poll/sendfile_unix.go @@ -0,0 +1,170 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin || dragonfly || freebsd || linux || solaris + +package poll + +import ( + "io" + "runtime" + "syscall" +) + +// SendFile wraps the sendfile system call. +// +// It copies data from src (a file descriptor) to dstFD, +// starting at the current position of src. +// It updates the current position of src to after the +// copied data. +// +// If size is zero, it copies the rest of src. +// Otherwise, it copies up to size bytes. +// +// The handled return parameter indicates whether SendFile +// was able to handle some or all of the operation. +// If handled is false, sendfile was unable to perform the copy, +// has not modified the source or destination, +// and the caller should perform the copy using a fallback implementation. +func SendFile(dstFD *FD, src uintptr, size int64) (n int64, err error, handled bool) { + if goos := runtime.GOOS; goos == "linux" || goos == "android" { + // Linux's sendfile doesn't require any setup: + // It sends from the current position of the source file and + // updates the position of the source after sending. + return sendFile(dstFD, int(src), nil, size) + } + + // Non-Linux sendfile implementations don't use the current position of the source file, + // so we need to look up the position, pass it explicitly, and adjust it after + // sendfile returns. + start, err := ignoringEINTR2(func() (int64, error) { + return syscall.Seek(int(src), 0, io.SeekCurrent) + }) + if err != nil { + return 0, err, false + } + + pos := start + n, err, handled = sendFile(dstFD, int(src), &pos, size) + if n > 0 { + ignoringEINTR2(func() (int64, error) { + return syscall.Seek(int(src), start+n, io.SeekStart) + }) + } + return n, err, handled +} + +// sendFile wraps the sendfile system call. +func sendFile(dstFD *FD, src int, offset *int64, size int64) (written int64, err error, handled bool) { + defer func() { + TestHookDidSendFile(dstFD, uintptr(src), written, err, handled) + }() + if err := dstFD.writeLock(); err != nil { + return 0, err, false + } + defer dstFD.writeUnlock() + + if err := dstFD.pd.prepareWrite(dstFD.isFile); err != nil { + return 0, err, false + } + + dst := dstFD.Sysfd + for { + // Some platforms support passing 0 to read to the end of the source, + // but all platforms support just writing a large value. + // + // Limit the maximum size to fit in an int32, to avoid any possible overflow. + chunk := 1<<31 - 1 + if size > 0 { + chunk = int(min(size-written, int64(chunk))) + } + var n int + n, err = sendFileChunk(dst, src, offset, chunk, written) + if n > 0 { + written += int64(n) + } + switch err { + case nil: + // We're done if sendfile copied no bytes + // (we're at the end of the source) + // or if we have a size limit and have reached it. + // + // If sendfile copied some bytes and we don't have a size limit, + // try again to see if there is more data to copy. + if n == 0 || (size > 0 && written >= size) { + return written, nil, true + } + case syscall.EAGAIN: + // *BSD and Darwin can return EAGAIN with n > 0, + // so check to see if the write has completed. + // So far as we know all other platforms only + // return EAGAIN when n == 0, but checking is harmless. + if size > 0 && written >= size { + return written, nil, true + } + if err = dstFD.pd.waitWrite(dstFD.isFile); err != nil { + return written, err, true + } + case syscall.EINTR: + // Retry. + case syscall.ENOSYS, syscall.EOPNOTSUPP, syscall.EINVAL: + // ENOSYS indicates no kernel support for sendfile. + // EINVAL indicates a FD type that does not support sendfile. + // + // On Linux, copy_file_range can return EOPNOTSUPP when copying + // to a NFS file (issue #40731); check for it here just in case. + return written, err, written > 0 + default: + // We want to handle ENOTSUP like EOPNOTSUPP. + // It's a pain to put it as a switch case + // because on Linux systems ENOTSUP == EOPNOTSUPP, + // so the compiler complains about a duplicate case. + if err == syscall.ENOTSUP { + return written, err, written > 0 + } + + // Not a retryable error. + return written, err, true + } + } +} + +func sendFileChunk(dst, src int, offset *int64, size int, written int64) (n int, err error) { + switch runtime.GOOS { + case "linux", "android": + // The offset is always nil on Linux. + n, err = syscall.Sendfile(dst, src, offset, size) + case "solaris", "illumos": + // Trust the offset, not the return value from sendfile. + start := *offset + n, err = syscall.Sendfile(dst, src, offset, size) + n = int(*offset - start) + // A quirk on Solaris/illumos: sendfile claims to support out_fd + // as a regular file but returns EINVAL when the out_fd + // is not a socket of SOCK_STREAM, while it actually sends + // out data anyway and updates the file offset. + // + // Another quirk: sendfile transfers data and returns EINVAL when being + // asked to transfer bytes more than the actual file size. For instance, + // the source file is wrapped in an io.LimitedReader with larger size + // than the actual file size. + // + // To handle these cases we ignore EINVAL if any call to sendfile was + // able to send data. + if err == syscall.EINVAL && (n > 0 || written > 0) { + err = nil + } + default: + start := *offset + n, err = syscall.Sendfile(dst, src, offset, size) + if n > 0 { + // The BSD implementations of syscall.Sendfile don't + // update the offset parameter (despite it being a *int64). + // + // Trust the return value from sendfile, not the offset. + *offset = start + int64(n) + } + } + return +} diff --git a/src/internal/poll/sendfile_windows.go b/src/internal/poll/sendfile_windows.go index 2ae8a8d1d791e7..2bdfecf0134b62 100644 --- a/src/internal/poll/sendfile_windows.go +++ b/src/internal/poll/sendfile_windows.go @@ -10,78 +10,79 @@ import ( ) // SendFile wraps the TransmitFile call. -func SendFile(fd *FD, src syscall.Handle, n int64) (written int64, err error) { +func SendFile(fd *FD, src uintptr, size int64) (written int64, err error, handled bool) { defer func() { TestHookDidSendFile(fd, 0, written, err, written > 0) }() if fd.kind == kindPipe { // TransmitFile does not work with pipes - return 0, syscall.ESPIPE + return 0, syscall.ESPIPE, false } - if ft, _ := syscall.GetFileType(src); ft == syscall.FILE_TYPE_PIPE { - return 0, syscall.ESPIPE + hsrc := syscall.Handle(src) + if ft, _ := syscall.GetFileType(hsrc); ft == syscall.FILE_TYPE_PIPE { + return 0, syscall.ESPIPE, false } if err := fd.writeLock(); err != nil { - return 0, err + return 0, err, false } defer fd.writeUnlock() - o := &fd.wop - o.handle = src - - // TODO(brainman): skip calling syscall.Seek if OS allows it - curpos, err := syscall.Seek(o.handle, 0, io.SeekCurrent) + // Get the file size so we don't read past the end of the file. + var fi syscall.ByHandleFileInformation + if err := syscall.GetFileInformationByHandle(hsrc, &fi); err != nil { + return 0, err, false + } + fileSize := int64(fi.FileSizeHigh)<<32 + int64(fi.FileSizeLow) + startpos, err := syscall.Seek(hsrc, 0, io.SeekCurrent) if err != nil { - return 0, err + return 0, err, false + } + maxSize := fileSize - startpos + if size <= 0 { + size = maxSize + } else { + size = min(size, maxSize) } - if n <= 0 { // We don't know the size of the file so infer it. - // Find the number of bytes offset from curpos until the end of the file. - n, err = syscall.Seek(o.handle, -curpos, io.SeekEnd) - if err != nil { - return - } - // Now seek back to the original position. - if _, err = syscall.Seek(o.handle, curpos, io.SeekStart); err != nil { - return + defer func() { + if written > 0 { + // Some versions of Windows (Windows 10 1803) do not set + // file position after TransmitFile completes. + // So just use Seek to set file position. + _, serr := syscall.Seek(hsrc, startpos+written, io.SeekStart) + if err != nil { + err = serr + } } - } + }() // TransmitFile can be invoked in one call with at most // 2,147,483,646 bytes: the maximum value for a 32-bit integer minus 1. // See https://docs.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-transmitfile const maxChunkSizePerCall = int64(0x7fffffff - 1) - for n > 0 { + for size > 0 { chunkSize := maxChunkSizePerCall - if chunkSize > n { - chunkSize = n + if chunkSize > size { + chunkSize = size } - o.qty = uint32(chunkSize) - o.o.Offset = uint32(curpos) - o.o.OffsetHigh = uint32(curpos >> 32) - - nw, err := execIO(o, func(o *operation) error { - return syscall.TransmitFile(o.fd.Sysfd, o.handle, o.qty, 0, &o.o, nil, syscall.TF_WRITE_BEHIND) + fd.setOffset(startpos + written) + n, err := fd.execIO('w', func(o *operation) (uint32, error) { + err := syscall.TransmitFile(fd.Sysfd, hsrc, uint32(chunkSize), 0, &o.o, nil, syscall.TF_WRITE_BEHIND) + if err != nil { + return 0, err + } + return uint32(chunkSize), nil }) if err != nil { - return written, err - } - - curpos += int64(nw) - - // Some versions of Windows (Windows 10 1803) do not set - // file position after TransmitFile completes. - // So just use Seek to set file position. - if _, err = syscall.Seek(o.handle, curpos, io.SeekStart); err != nil { - return written, err + return written, err, written > 0 } - n -= int64(nw) - written += int64(nw) + size -= int64(n) + written += int64(n) } - return + return written, nil, written > 0 } diff --git a/src/internal/poll/sock_cloexec.go b/src/internal/poll/sock_cloexec.go index cbf7021804fa82..466ee3139ccdac 100644 --- a/src/internal/poll/sock_cloexec.go +++ b/src/internal/poll/sock_cloexec.go @@ -5,7 +5,7 @@ // This file implements accept for platforms that provide a fast path for // setting SetNonblock and CloseOnExec. -//go:build dragonfly || freebsd || (linux && !arm) || netbsd || openbsd +//go:build dragonfly || freebsd || linux || netbsd || openbsd package poll diff --git a/src/internal/poll/sock_cloexec_accept.go b/src/internal/poll/sock_cloexec_accept.go deleted file mode 100644 index 4b86de59e09299..00000000000000 --- a/src/internal/poll/sock_cloexec_accept.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file implements accept for platforms that provide a fast path for -// setting SetNonblock and CloseOnExec, but don't necessarily have accept4. -// This is the code we used for accept in Go 1.17 and earlier. -// On Linux the accept4 system call was introduced in 2.6.28 kernel, -// and our minimum requirement is 2.6.32, so we simplified the function. -// Unfortunately, on ARM accept4 wasn't added until 2.6.36, so for ARM -// only we continue using the older code. - -//go:build linux && arm - -package poll - -import "syscall" - -// Wrapper around the accept system call that marks the returned file -// descriptor as nonblocking and close-on-exec. -func accept(s int) (int, syscall.Sockaddr, string, error) { - ns, sa, err := Accept4Func(s, syscall.SOCK_NONBLOCK|syscall.SOCK_CLOEXEC) - switch err { - case nil: - return ns, sa, "", nil - default: // errors other than the ones listed - return -1, sa, "accept4", err - case syscall.ENOSYS: // syscall missing - case syscall.EINVAL: // some Linux use this instead of ENOSYS - case syscall.EACCES: // some Linux use this instead of ENOSYS - case syscall.EFAULT: // some Linux use this instead of ENOSYS - } - - // See ../syscall/exec_unix.go for description of ForkLock. - // It is probably okay to hold the lock across syscall.Accept - // because we have put fd.sysfd into non-blocking mode. - // However, a call to the File method will put it back into - // blocking mode. We can't take that risk, so no use of ForkLock here. - ns, sa, err = AcceptFunc(s) - if err == nil { - syscall.CloseOnExec(ns) - } - if err != nil { - return -1, nil, "accept", err - } - if err = syscall.SetNonblock(ns, true); err != nil { - CloseFunc(ns) - return -1, nil, "setnonblock", err - } - return ns, sa, "", nil -} diff --git a/src/internal/poll/splice_linux.go b/src/internal/poll/splice_linux.go index 193a56215c727c..4409d2f33600a4 100644 --- a/src/internal/poll/splice_linux.go +++ b/src/internal/poll/splice_linux.go @@ -9,7 +9,6 @@ import ( "runtime" "sync" "syscall" - "unsafe" ) const ( @@ -179,10 +178,7 @@ type splicePipeFields struct { type splicePipe struct { splicePipeFields - - // We want to use a finalizer, so ensure that the size is - // large enough to not use the tiny allocator. - _ [24 - unsafe.Sizeof(splicePipeFields{})%24]byte + cleanup runtime.Cleanup } // splicePipePool caches pipes to avoid high-frequency construction and destruction of pipe buffers. @@ -197,7 +193,10 @@ func newPoolPipe() any { if p == nil { return nil } - runtime.SetFinalizer(p, destroyPipe) + + p.cleanup = runtime.AddCleanup(p, func(spf splicePipeFields) { + destroyPipe(&splicePipe{splicePipeFields: spf}) + }, p.splicePipeFields) return p } @@ -214,7 +213,7 @@ func putPipe(p *splicePipe) { // If there is still data left in the pipe, // then close and discard it instead of putting it back into the pool. if p.data != 0 { - runtime.SetFinalizer(p, nil) + p.cleanup.Stop() destroyPipe(p) return } diff --git a/src/internal/profile/graph.go b/src/internal/profile/graph.go index 0e8e33c1ac4b9b..e3c755a65d8523 100644 --- a/src/internal/profile/graph.go +++ b/src/internal/profile/graph.go @@ -253,12 +253,8 @@ func NewGraph(prof *Profile, o *Options) *Graph { if dw == 0 && w == 0 { continue } - for k := range seenNode { - delete(seenNode, k) - } - for k := range seenEdge { - delete(seenEdge, k) - } + clear(seenNode) + clear(seenEdge) var parent *Node // A residual edge goes over one or more nodes that were not kept. residual := false diff --git a/src/internal/race/norace.go b/src/internal/race/norace.go index da650489fc4eb8..3fb00573a070ef 100644 --- a/src/internal/race/norace.go +++ b/src/internal/race/norace.go @@ -7,6 +7,7 @@ package race import ( + "internal/abi" "unsafe" ) @@ -30,9 +31,21 @@ func Enable() { func Read(addr unsafe.Pointer) { } +func ReadPC(addr unsafe.Pointer, callerpc, pc uintptr) { +} + +func ReadObjectPC(t *abi.Type, addr unsafe.Pointer, callerpc, pc uintptr) { +} + func Write(addr unsafe.Pointer) { } +func WritePC(addr unsafe.Pointer, callerpc, pc uintptr) { +} + +func WriteObjectPC(t *abi.Type, addr unsafe.Pointer, callerpc, pc uintptr) { +} + func ReadRange(addr unsafe.Pointer, len int) { } diff --git a/src/internal/race/race.go b/src/internal/race/race.go index d2c7e53e418260..bfcb24a2694e85 100644 --- a/src/internal/race/race.go +++ b/src/internal/race/race.go @@ -7,48 +7,52 @@ package race import ( - "runtime" + "internal/abi" "unsafe" ) const Enabled = true -func Acquire(addr unsafe.Pointer) { - runtime.RaceAcquire(addr) -} +// Functions below pushed from runtime. -func Release(addr unsafe.Pointer) { - runtime.RaceRelease(addr) -} +//go:linkname Acquire +func Acquire(addr unsafe.Pointer) -func ReleaseMerge(addr unsafe.Pointer) { - runtime.RaceReleaseMerge(addr) -} +//go:linkname Release +func Release(addr unsafe.Pointer) -func Disable() { - runtime.RaceDisable() -} +//go:linkname ReleaseMerge +func ReleaseMerge(addr unsafe.Pointer) -func Enable() { - runtime.RaceEnable() -} +//go:linkname Disable +func Disable() -func Read(addr unsafe.Pointer) { - runtime.RaceRead(addr) -} +//go:linkname Enable +func Enable() -func Write(addr unsafe.Pointer) { - runtime.RaceWrite(addr) -} +//go:linkname Read +func Read(addr unsafe.Pointer) -func ReadRange(addr unsafe.Pointer, len int) { - runtime.RaceReadRange(addr, len) -} +//go:linkname ReadPC +func ReadPC(addr unsafe.Pointer, callerpc, pc uintptr) -func WriteRange(addr unsafe.Pointer, len int) { - runtime.RaceWriteRange(addr, len) -} +//go:linkname ReadObjectPC +func ReadObjectPC(t *abi.Type, addr unsafe.Pointer, callerpc, pc uintptr) -func Errors() int { - return runtime.RaceErrors() -} +//go:linkname Write +func Write(addr unsafe.Pointer) + +//go:linkname WritePC +func WritePC(addr unsafe.Pointer, callerpc, pc uintptr) + +//go:linkname WriteObjectPC +func WriteObjectPC(t *abi.Type, addr unsafe.Pointer, callerpc, pc uintptr) + +//go:linkname ReadRange +func ReadRange(addr unsafe.Pointer, len int) + +//go:linkname WriteRange +func WriteRange(addr unsafe.Pointer, len int) + +//go:linkname Errors +func Errors() int diff --git a/src/internal/reflectlite/export_test.go b/src/internal/reflectlite/export_test.go index 0ad3d97c15f995..93762a83874106 100644 --- a/src/internal/reflectlite/export_test.go +++ b/src/internal/reflectlite/export_test.go @@ -70,7 +70,7 @@ func Zero(typ Type) Value { } t := typ.common() fl := flag(t.Kind()) - if t.IfaceIndir() { + if !t.IsDirectIface() { return Value{t, unsafe_New(t), fl | flagIndir} } return Value{t, nil, fl} diff --git a/src/internal/reflectlite/reflect_mirror_test.go b/src/internal/reflectlite/reflect_mirror_test.go index c87573903454ed..c5642d092d9982 100644 --- a/src/internal/reflectlite/reflect_mirror_test.go +++ b/src/internal/reflectlite/reflect_mirror_test.go @@ -13,6 +13,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "strings" "sync" "testing" @@ -40,12 +41,7 @@ func newVisitor() visitor { return v } func (v visitor) filter(name string) bool { - for _, typeName := range typeNames { - if typeName == name { - return true - } - } - return false + return slices.Contains(typeNames, name) } func (v visitor) Visit(n ast.Node) ast.Visitor { @@ -105,7 +101,6 @@ func TestMirrorWithReflect(t *testing.T) { {".", "reflectlite", rl}, {reflectDir, "reflect", r}, } { - tc := tc wg.Add(1) go func() { defer wg.Done() diff --git a/src/internal/reflectlite/value.go b/src/internal/reflectlite/value.go index c38b498ea7e24c..a92df613f55d1e 100644 --- a/src/internal/reflectlite/value.go +++ b/src/internal/reflectlite/value.go @@ -43,17 +43,19 @@ type Value struct { ptr unsafe.Pointer // flag holds metadata about the value. - // The lowest bits are flag bits: + // + // The lowest five bits give the Kind of the value, mirroring typ.Kind(). + // + // The next set of bits are flag bits: // - flagStickyRO: obtained via unexported not embedded field, so read-only // - flagEmbedRO: obtained via unexported embedded field, so read-only // - flagIndir: val holds a pointer to the data - // - flagAddr: v.CanAddr is true (implies flagIndir) - // Value cannot represent method values. - // The next five bits give the Kind of the value. - // This repeats typ.Kind() except for method values. - // The remaining 23+ bits give a method number for method values. + // - flagAddr: v.CanAddr is true (implies flagIndir and ptr is non-nil) + // - flagMethod: v is a method value. + // If !typ.IsDirectIface(), code can assume that flagIndir is set. + // + // The remaining 22+ bits give a method number for method values. // If flag.kind() != Func, code can assume that flagMethod is unset. - // If ifaceIndir(typ), code can assume that flagIndir is set. flag // A method value represents a curried method invocation @@ -116,7 +118,7 @@ func packEface(v Value) any { e := (*abi.EmptyInterface)(unsafe.Pointer(&i)) // First, fill in the data portion of the interface. switch { - case t.IfaceIndir(): + case !t.IsDirectIface(): if v.flag&flagIndir == 0 { panic("bad indir") } @@ -153,7 +155,7 @@ func unpackEface(i any) Value { return Value{} } f := flag(t.Kind()) - if t.IfaceIndir() { + if !t.IsDirectIface() { f |= flagIndir } return Value{t, e.Data, f} diff --git a/src/internal/routebsd/address.go b/src/internal/routebsd/address.go new file mode 100644 index 00000000000000..aa1bc21d3f00ae --- /dev/null +++ b/src/internal/routebsd/address.go @@ -0,0 +1,300 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin || dragonfly || freebsd || netbsd || openbsd + +package routebsd + +import ( + "net/netip" + "runtime" + "syscall" +) + +// An Addr represents an address associated with packet routing. +type Addr interface { + // Family returns an address family. + Family() int +} + +// A LinkAddr represents a link-layer address. +type LinkAddr struct { + Index int // interface index when attached + Name string // interface name when attached + Addr []byte // link-layer address when attached +} + +// Family implements the Family method of Addr interface. +func (a *LinkAddr) Family() int { return syscall.AF_LINK } + +func parseLinkAddr(b []byte) (Addr, error) { + if len(b) < 8 { + return nil, errInvalidAddr + } + _, a, err := parseKernelLinkAddr(syscall.AF_LINK, b[4:]) + if err != nil { + return nil, err + } + a.(*LinkAddr).Index = int(nativeEndian.Uint16(b[2:4])) + return a, nil +} + +// parseKernelLinkAddr parses b as a link-layer address in +// conventional BSD kernel form. +func parseKernelLinkAddr(_ int, b []byte) (int, Addr, error) { + // The encoding looks like the following: + // +----------------------------+ + // | Type (1 octet) | + // +----------------------------+ + // | Name length (1 octet) | + // +----------------------------+ + // | Address length (1 octet) | + // +----------------------------+ + // | Selector length (1 octet) | + // +----------------------------+ + // | Data (variable) | + // +----------------------------+ + // + // On some platforms, all-bit-one of length field means "don't + // care". + nlen, alen, slen := int(b[1]), int(b[2]), int(b[3]) + if nlen == 0xff { + nlen = 0 + } + if alen == 0xff { + alen = 0 + } + if slen == 0xff { + slen = 0 + } + l := 4 + nlen + alen + slen + if len(b) < l { + return 0, nil, errInvalidAddr + } + data := b[4:] + var name string + var addr []byte + if nlen > 0 { + name = string(data[:nlen]) + data = data[nlen:] + } + if alen > 0 { + addr = data[:alen] + data = data[alen:] + } + return l, &LinkAddr{Name: name, Addr: addr}, nil +} + +// An InetAddr represent an internet address using IPv4 or IPv6. +type InetAddr struct { + IP netip.Addr +} + +func (a *InetAddr) Family() int { + if a.IP.Is4() { + return syscall.AF_INET + } else { + return syscall.AF_INET6 + } +} + +// parseInetAddr parses b as an internet address for IPv4 or IPv6. +func parseInetAddr(af int, b []byte) (Addr, error) { + const ( + off4 = 4 // offset of in_addr + off6 = 8 // offset of in6_addr + ipv4Len = 4 // length of IPv4 address in bytes + ipv6Len = 16 // length of IPv6 address in bytes + ) + switch af { + case syscall.AF_INET: + if len(b) < (off4+1) || len(b) < int(b[0]) { + return nil, errInvalidAddr + } + sockAddrLen := int(b[0]) + var ip [ipv4Len]byte + if sockAddrLen != 0 { + // Calculate how many bytes of the address to copy: + // either full IPv4 length or the available length. + n := off4 + ipv4Len + if sockAddrLen < n { + n = sockAddrLen + } + copy(ip[:], b[off4:n]) + } + a := &InetAddr{ + IP: netip.AddrFrom4(ip), + } + return a, nil + case syscall.AF_INET6: + if len(b) < (off6+1) || len(b) < int(b[0]) { + return nil, errInvalidAddr + } + var ip [ipv6Len]byte + sockAddrLen := int(b[0]) + if sockAddrLen != 0 { + n := off6 + ipv6Len + if sockAddrLen < n { + n = sockAddrLen + } + copy(ip[:], b[off6:n]) + if ip[0] == 0xfe && ip[1]&0xc0 == 0x80 || ip[0] == 0xff && (ip[1]&0x0f == 0x01 || ip[1]&0x0f == 0x02) { + // KAME based IPv6 protocol stack usually + // embeds the interface index in the + // interface-local or link-local address as + // the kernel-internal form. + id := int(bigEndian.Uint16(ip[2:4])) + if id != 0 { + ip[2], ip[3] = 0, 0 + } + } + } + // The kernel can provide an integer zone ID. + // We ignore it. + a := &InetAddr{ + IP: netip.AddrFrom16(ip), + } + return a, nil + default: + return nil, errInvalidAddr + } +} + +// parseKernelInetAddr parses b as an internet address in conventional +// BSD kernel form. +func parseKernelInetAddr(af int, b []byte) (int, Addr, error) { + // The encoding looks similar to the NLRI encoding. + // +----------------------------+ + // | Length (1 octet) | + // +----------------------------+ + // | Address prefix (variable) | + // +----------------------------+ + // + // The differences between the kernel form and the NLRI + // encoding are: + // + // - The length field of the kernel form indicates the prefix + // length in bytes, not in bits + // + // - In the kernel form, zero value of the length field + // doesn't mean 0.0.0.0/0 or ::/0 + // + // - The kernel form appends leading bytes to the prefix field + // to make the tuple to be conformed with + // the routing message boundary + l := int(b[0]) + if runtime.GOOS == "darwin" || runtime.GOOS == "ios" { + // On Darwin, an address in the kernel form is also + // used as a message filler. + if l == 0 || len(b) > roundup(l) { + l = roundup(l) + } + } else { + l = roundup(l) + } + if len(b) < l { + return 0, nil, errInvalidAddr + } + // Don't reorder case expressions. + // The case expressions for IPv6 must come first. + const ( + off4 = 4 // offset of in_addr + off6 = 8 // offset of in6_addr + ) + switch { + case b[0] == syscall.SizeofSockaddrInet6: + a := &InetAddr{ + IP: netip.AddrFrom16([16]byte(b[off6 : off6+16])), + } + return int(b[0]), a, nil + case af == syscall.AF_INET6: + var ab [16]byte + if l-1 < off6 { + copy(ab[:], b[1:l]) + } else { + copy(ab[:], b[l-off6:l]) + } + a := &InetAddr{ + IP: netip.AddrFrom16(ab), + } + return int(b[0]), a, nil + case b[0] == syscall.SizeofSockaddrInet4: + a := &InetAddr{ + IP: netip.AddrFrom4([4]byte(b[off4 : off4+4])), + } + return int(b[0]), a, nil + default: // an old fashion, AF_UNSPEC or unknown means AF_INET + var ab [4]byte + if l-1 < off4 { + copy(ab[:], b[1:l]) + } else { + copy(ab[:], b[l-off4:l]) + } + a := &InetAddr{ + IP: netip.AddrFrom4(ab), + } + return int(b[0]), a, nil + } +} + +func parseAddrs(attrs uint, b []byte) ([]Addr, error) { + var as [syscall.RTAX_MAX]Addr + af := int(syscall.AF_UNSPEC) + for i := uint(0); i < syscall.RTAX_MAX && len(b) >= roundup(0); i++ { + if attrs&(1< 4 { + nmsgs++ + l := int(nativeEndian.Uint16(b[:2])) + if l == 0 { + return nil, errInvalidMessage + } + if len(b) < l { + return nil, errMessageTooShort + } + if b[2] != rtmVersion { + b = b[l:] + continue + } + if w, ok := wireFormats[int(b[3])]; !ok { + nskips++ + } else { + m, err := w.parse(b[:l]) + if err != nil { + return nil, err + } + if m == nil { + nskips++ + } else { + msgs = append(msgs, m) + } + } + b = b[l:] + } + + // We failed to parse any of the messages - version mismatch? + if nmsgs != len(msgs)+nskips { + return nil, errMessageMismatch + } + return msgs, nil +} diff --git a/src/internal/routebsd/message_darwin_test.go b/src/internal/routebsd/message_darwin_test.go new file mode 100644 index 00000000000000..f224f0a73d4547 --- /dev/null +++ b/src/internal/routebsd/message_darwin_test.go @@ -0,0 +1,28 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package routebsd + +import ( + "syscall" + "testing" +) + +func TestFetchRIBMessagesOnDarwin(t *testing.T) { + for _, typ := range []int{syscall.NET_RT_FLAGS, syscall.NET_RT_DUMP2, syscall.NET_RT_IFLIST2} { + ms, err := FetchRIBMessages(typ, 0) + if err != nil { + t.Error(typ, err) + continue + } + ss, err := msgs(ms).validate() + if err != nil { + t.Error(typ, err) + continue + } + for _, s := range ss { + t.Log(s) + } + } +} diff --git a/src/internal/routebsd/message_freebsd_test.go b/src/internal/routebsd/message_freebsd_test.go new file mode 100644 index 00000000000000..f0065198c8f2ef --- /dev/null +++ b/src/internal/routebsd/message_freebsd_test.go @@ -0,0 +1,28 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package routebsd + +import ( + "syscall" + "testing" +) + +func TestFetchRIBMessagesOnFreeBSD(t *testing.T) { + for _, typ := range []int{syscall.NET_RT_IFMALIST} { + ms, err := FetchRIBMessages(typ, 0) + if err != nil { + t.Error(typ, err) + continue + } + ss, err := msgs(ms).validate() + if err != nil { + t.Error(typ, err) + continue + } + for _, s := range ss { + t.Log(s) + } + } +} diff --git a/src/internal/routebsd/message_test.go b/src/internal/routebsd/message_test.go new file mode 100644 index 00000000000000..958f2010628a06 --- /dev/null +++ b/src/internal/routebsd/message_test.go @@ -0,0 +1,51 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin || dragonfly || freebsd || netbsd || openbsd + +package routebsd + +import ( + "syscall" + "testing" +) + +func TestFetchRIBMessages(t *testing.T) { + for _, typ := range []int{syscall.NET_RT_DUMP, syscall.NET_RT_IFLIST} { + ms, err := FetchRIBMessages(typ, 0) + if err != nil { + t.Error(typ, err) + continue + } + ss, err := msgs(ms).validate() + if err != nil { + t.Error(typ, err) + continue + } + for _, s := range ss { + t.Log(typ, s) + } + } +} + +func TestParseRIBWithFuzz(t *testing.T) { + for _, fuzz := range []string{ + "0\x00\x05\x050000000000000000" + + "00000000000000000000" + + "00000000000000000000" + + "00000000000000000000" + + "0000000000000\x02000000" + + "00000000", + "\x02\x00\x05\f0000000000000000" + + "0\x0200000000000000", + "\x02\x00\x05\x100000000000000\x1200" + + "0\x00\xff\x00", + "\x02\x00\x05\f0000000000000000" + + "0\x12000\x00\x02\x0000", + "\x00\x00\x00\x01\x00", + "00000", + } { + parseRIB([]byte(fuzz)) + } +} diff --git a/src/internal/routebsd/route.go b/src/internal/routebsd/route.go new file mode 100644 index 00000000000000..5b6062e125869d --- /dev/null +++ b/src/internal/routebsd/route.go @@ -0,0 +1,59 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin || dragonfly || freebsd || netbsd || openbsd + +// Package routebsd supports reading interface addresses on BSD systems. +// This is a very stripped down version of x/net/route, +// for use by the net package in the standard library. +package routebsd + +import ( + "errors" + "syscall" +) + +var ( + errMessageMismatch = errors.New("message mismatch") + errMessageTooShort = errors.New("message too short") + errInvalidMessage = errors.New("invalid message") + errInvalidAddr = errors.New("invalid address") +) + +// fetchRIB fetches a routing information base from the operating +// system. +// +// The arg is an interface index or 0 for all. +func fetchRIB(typ, arg int) ([]byte, error) { + try := 0 + for { + try++ + b, err := syscall.RouteRIB(typ, arg) + + // If the sysctl failed because the data got larger + // between the two sysctl calls, try a few times + // before failing (issue #45736). + const maxTries = 3 + if err == syscall.ENOMEM && try < maxTries { + continue + } + + return b, err + } +} + +// FetchRIBMessages fetches a list of addressing messages for an interface. +// The typ argument is something like syscall.NET_RT_IFLIST. +// The argument is an interface index or 0 for all. +func FetchRIBMessages(typ, arg int) ([]Message, error) { + b, err := fetchRIB(typ, arg) + if err != nil { + return nil, err + } + ms, err := parseRIB(b) + if err != nil { + return nil, err + } + return ms, nil +} diff --git a/src/internal/routebsd/route_test.go b/src/internal/routebsd/route_test.go new file mode 100644 index 00000000000000..613175e388eceb --- /dev/null +++ b/src/internal/routebsd/route_test.go @@ -0,0 +1,239 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin || dragonfly || freebsd || netbsd || openbsd + +package routebsd + +import ( + "fmt" + "runtime" + "syscall" +) + +func (m *InterfaceMessage) String() string { + var attrs addrAttrs + if runtime.GOOS == "openbsd" { + attrs = addrAttrs(nativeEndian.Uint32(m.raw[12:16])) + } else { + attrs = addrAttrs(nativeEndian.Uint32(m.raw[4:8])) + } + return fmt.Sprintf("%s", attrs) +} + +func (m *InterfaceAddrMessage) String() string { + var attrs addrAttrs + if runtime.GOOS == "openbsd" { + attrs = addrAttrs(nativeEndian.Uint32(m.raw[12:16])) + } else { + attrs = addrAttrs(nativeEndian.Uint32(m.raw[4:8])) + } + return fmt.Sprintf("%s", attrs) +} + +func (m *InterfaceMulticastAddrMessage) String() string { + return fmt.Sprintf("%s", addrAttrs(nativeEndian.Uint32(m.raw[4:8]))) +} + +type addrAttrs uint + +var addrAttrNames = [...]string{ + "dst", + "gateway", + "netmask", + "genmask", + "ifp", + "ifa", + "author", + "brd", + "df:mpls1-n:tag-o:src", // mpls1 for dragonfly, tag for netbsd, src for openbsd + "df:mpls2-o:srcmask", // mpls2 for dragonfly, srcmask for openbsd + "df:mpls3-o:label", // mpls3 for dragonfly, label for openbsd + "o:bfd", // bfd for openbsd + "o:dns", // dns for openbsd + "o:static", // static for openbsd + "o:search", // search for openbsd +} + +func (attrs addrAttrs) String() string { + var s string + for i, name := range addrAttrNames { + if attrs&(1<" + } + return s +} + +type msgs []Message + +func (ms msgs) validate() ([]string, error) { + var ss []string + for _, m := range ms { + switch m := m.(type) { + case *InterfaceMessage: + var attrs addrAttrs + if runtime.GOOS == "openbsd" { + attrs = addrAttrs(nativeEndian.Uint32(m.raw[12:16])) + } else { + attrs = addrAttrs(nativeEndian.Uint32(m.raw[4:8])) + } + if err := addrs(m.Addrs).match(attrs); err != nil { + return nil, err + } + ss = append(ss, m.String()+" "+addrs(m.Addrs).String()) + case *InterfaceAddrMessage: + var attrs addrAttrs + if runtime.GOOS == "openbsd" { + attrs = addrAttrs(nativeEndian.Uint32(m.raw[12:16])) + } else { + attrs = addrAttrs(nativeEndian.Uint32(m.raw[4:8])) + } + if err := addrs(m.Addrs).match(attrs); err != nil { + return nil, err + } + ss = append(ss, m.String()+" "+addrs(m.Addrs).String()) + case *InterfaceMulticastAddrMessage: + if err := addrs(m.Addrs).match(addrAttrs(nativeEndian.Uint32(m.raw[4:8]))); err != nil { + return nil, err + } + ss = append(ss, m.String()+" "+addrs(m.Addrs).String()) + default: + ss = append(ss, fmt.Sprintf("%+v", m)) + } + } + return ss, nil +} + +type addrFamily int + +func (af addrFamily) String() string { + switch af { + case syscall.AF_UNSPEC: + return "unspec" + case syscall.AF_LINK: + return "link" + case syscall.AF_INET: + return "inet4" + case syscall.AF_INET6: + return "inet6" + default: + return fmt.Sprintf("%d", af) + } +} + +const hexDigit = "0123456789abcdef" + +type llAddr []byte + +func (a llAddr) String() string { + if len(a) == 0 { + return "" + } + buf := make([]byte, 0, len(a)*3-1) + for i, b := range a { + if i > 0 { + buf = append(buf, ':') + } + buf = append(buf, hexDigit[b>>4]) + buf = append(buf, hexDigit[b&0xF]) + } + return string(buf) +} + +type ipAddr []byte + +func (a ipAddr) String() string { + if len(a) == 0 { + return "" + } + if len(a) == 4 { + return fmt.Sprintf("%d.%d.%d.%d", a[0], a[1], a[2], a[3]) + } + if len(a) == 16 { + return fmt.Sprintf("%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x", a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11], a[12], a[13], a[14], a[15]) + } + s := make([]byte, len(a)*2) + for i, tn := range a { + s[i*2], s[i*2+1] = hexDigit[tn>>4], hexDigit[tn&0xf] + } + return string(s) +} + +func (a *LinkAddr) String() string { + name := a.Name + if name == "" { + name = "" + } + lla := llAddr(a.Addr).String() + if lla == "" { + lla = "" + } + return fmt.Sprintf("(%v %d %s %s)", addrFamily(a.Family()), a.Index, name, lla) +} + +func (a *InetAddr) String() string { + return fmt.Sprintf("(%v %v)", addrFamily(a.Family()), a.IP) +} + +type addrs []Addr + +func (as addrs) String() string { + var s string + for _, a := range as { + if a == nil { + continue + } + if len(s) > 0 { + s += " " + } + switch a := a.(type) { + case *LinkAddr: + s += a.String() + case *InetAddr: + s += a.String() + } + } + if s == "" { + return "" + } + return s +} + +func (as addrs) match(attrs addrAttrs) error { + var ts addrAttrs + af := syscall.AF_UNSPEC + for i := range as { + if as[i] != nil { + ts |= 1 << uint(i) + } + switch addr := as[i].(type) { + case *InetAddr: + got := 0 + if addr.IP.Is4() { + got = syscall.AF_INET + } else if addr.IP.Is6() { + got = syscall.AF_INET6 + } + if af == syscall.AF_UNSPEC { + if got != 0 { + af = got + } + } + if got != 0 && af != got { + return fmt.Errorf("got %v; want %v", addrs(as), addrFamily(af)) + } + } + } + if ts != attrs && ts > attrs { + return fmt.Errorf("%v not included in %v", ts, attrs) + } + return nil +} diff --git a/src/vendor/golang.org/x/net/route/sys.go b/src/internal/routebsd/sys.go similarity index 93% rename from src/vendor/golang.org/x/net/route/sys.go rename to src/internal/routebsd/sys.go index fcebee58ec6da2..8805bca97e7cf2 100644 --- a/src/vendor/golang.org/x/net/route/sys.go +++ b/src/internal/routebsd/sys.go @@ -4,7 +4,7 @@ //go:build darwin || dragonfly || freebsd || netbsd || openbsd -package route +package routebsd import ( "syscall" @@ -41,5 +41,5 @@ func roundup(l int) int { type wireFormat struct { extOff int // offset of header extension bodyOff int // offset of message body - parse func(RIBType, []byte) (Message, error) + parse func([]byte) (Message, error) } diff --git a/src/internal/routebsd/sys_darwin.go b/src/internal/routebsd/sys_darwin.go new file mode 100644 index 00000000000000..14864e2e327df2 --- /dev/null +++ b/src/internal/routebsd/sys_darwin.go @@ -0,0 +1,38 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package routebsd + +import "syscall" + +// MTU returns the interface MTU. +func (m *InterfaceMessage) MTU() int { + return int(nativeEndian.Uint32(m.raw[m.extOff+8 : m.extOff+12])) +} + +// sizeofIfMsghdr2 is copied from x/sys/unix. +const sizeofIfMsghdr2 = 0xa0 + +func probeRoutingStack() (int, map[int]*wireFormat) { + ifm := &wireFormat{extOff: 16, bodyOff: syscall.SizeofIfMsghdr} + ifm.parse = ifm.parseInterfaceMessage + ifm2 := &wireFormat{extOff: 32, bodyOff: sizeofIfMsghdr2} + ifm2.parse = ifm2.parseInterfaceMessage + ifam := &wireFormat{extOff: syscall.SizeofIfaMsghdr, bodyOff: syscall.SizeofIfaMsghdr} + ifam.parse = ifam.parseInterfaceAddrMessage + ifmam := &wireFormat{extOff: syscall.SizeofIfmaMsghdr, bodyOff: syscall.SizeofIfmaMsghdr} + ifmam.parse = ifmam.parseInterfaceMulticastAddrMessage + ifmam2 := &wireFormat{extOff: syscall.SizeofIfmaMsghdr2, bodyOff: syscall.SizeofIfmaMsghdr2} + ifmam2.parse = ifmam2.parseInterfaceMulticastAddrMessage + // Darwin kernels require 32-bit aligned access to routing facilities. + return 4, map[int]*wireFormat{ + syscall.RTM_NEWADDR: ifam, + syscall.RTM_DELADDR: ifam, + syscall.RTM_IFINFO: ifm, + syscall.RTM_NEWMADDR: ifmam, + syscall.RTM_DELMADDR: ifmam, + syscall.RTM_IFINFO2: ifm2, + syscall.RTM_NEWMADDR2: ifmam2, + } +} diff --git a/src/internal/routebsd/sys_dragonfly.go b/src/internal/routebsd/sys_dragonfly.go new file mode 100644 index 00000000000000..ce0e6fd4039245 --- /dev/null +++ b/src/internal/routebsd/sys_dragonfly.go @@ -0,0 +1,41 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package routebsd + +import ( + "syscall" + "unsafe" +) + +// MTU returns the interface MTU. +func (m *InterfaceMessage) MTU() int { + return int(nativeEndian.Uint32(m.raw[m.extOff+8 : m.extOff+12])) +} + +func probeRoutingStack() (int, map[int]*wireFormat) { + var p uintptr + ifm := &wireFormat{extOff: 16, bodyOff: syscall.SizeofIfMsghdr} + ifm.parse = ifm.parseInterfaceMessage + ifam := &wireFormat{extOff: syscall.SizeofIfaMsghdr, bodyOff: syscall.SizeofIfaMsghdr} + ifam.parse = ifam.parseInterfaceAddrMessage + ifmam := &wireFormat{extOff: syscall.SizeofIfmaMsghdr, bodyOff: syscall.SizeofIfmaMsghdr} + ifmam.parse = ifmam.parseInterfaceMulticastAddrMessage + + rel, _ := syscall.SysctlUint32("kern.osreldate") + if rel >= 500705 { + // https://github.com/DragonFlyBSD/DragonFlyBSD/commit/43a373152df2d405c9940983e584e6a25e76632d + // but only the size of struct ifa_msghdr actually changed + rtmVersion = 7 + ifam.bodyOff = 0x18 + } + + return int(unsafe.Sizeof(p)), map[int]*wireFormat{ + syscall.RTM_NEWADDR: ifam, + syscall.RTM_DELADDR: ifam, + syscall.RTM_IFINFO: ifm, + syscall.RTM_NEWMADDR: ifmam, + syscall.RTM_DELMADDR: ifmam, + } +} diff --git a/src/internal/routebsd/sys_freebsd.go b/src/internal/routebsd/sys_freebsd.go new file mode 100644 index 00000000000000..5d5f49e42e01ea --- /dev/null +++ b/src/internal/routebsd/sys_freebsd.go @@ -0,0 +1,62 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package routebsd + +import ( + "syscall" + "unsafe" +) + +// MTU returns the interface MTU. +func (m *InterfaceMessage) MTU() int { + return int(nativeEndian.Uint32(m.raw[m.extOff+8 : m.extOff+12])) +} + +// sizeofIfMsghdr is the size used on FreeBSD 11 for all platforms. +const sizeofIfMsghdr = 0xa8 + +func probeRoutingStack() (int, map[int]*wireFormat) { + var p uintptr + wordSize := int(unsafe.Sizeof(p)) + align := wordSize + // In the case of kern.supported_archs="amd64 i386", we need + // to know the underlying kernel's architecture because the + // alignment for routing facilities are set at the build time + // of the kernel. + conf, _ := syscall.Sysctl("kern.conftxt") + for i, j := 0, 0; j < len(conf); j++ { + if conf[j] != '\n' { + continue + } + s := conf[i:j] + i = j + 1 + if len(s) > len("machine") && s[:len("machine")] == "machine" { + s = s[len("machine"):] + for k := 0; k < len(s); k++ { + if s[k] == ' ' || s[k] == '\t' { + s = s[1:] + } + break + } + if s == "amd64" { + align = 8 + } + break + } + } + ifm := &wireFormat{extOff: 16, bodyOff: sizeofIfMsghdr} + ifm.parse = ifm.parseInterfaceMessage + ifam := &wireFormat{extOff: syscall.SizeofIfaMsghdr, bodyOff: syscall.SizeofIfaMsghdr} + ifam.parse = ifam.parseInterfaceAddrMessage + ifmam := &wireFormat{extOff: syscall.SizeofIfmaMsghdr, bodyOff: syscall.SizeofIfmaMsghdr} + ifmam.parse = ifmam.parseInterfaceMulticastAddrMessage + return align, map[int]*wireFormat{ + syscall.RTM_NEWADDR: ifam, + syscall.RTM_DELADDR: ifam, + syscall.RTM_IFINFO: ifm, + syscall.RTM_NEWMADDR: ifmam, + syscall.RTM_DELMADDR: ifmam, + } +} diff --git a/src/internal/routebsd/sys_netbsd.go b/src/internal/routebsd/sys_netbsd.go new file mode 100644 index 00000000000000..b181f727c25ca5 --- /dev/null +++ b/src/internal/routebsd/sys_netbsd.go @@ -0,0 +1,26 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package routebsd + +import "syscall" + +// MTU returns the interface MTU. +func (m *InterfaceMessage) MTU() int { + return int(nativeEndian.Uint32(m.raw[m.extOff+8 : m.extOff+12])) +} + +func probeRoutingStack() (int, map[int]*wireFormat) { + ifm := &wireFormat{extOff: 16, bodyOff: syscall.SizeofIfMsghdr} + ifm.parse = ifm.parseInterfaceMessage + ifam := &wireFormat{extOff: syscall.SizeofIfaMsghdr, bodyOff: syscall.SizeofIfaMsghdr} + ifam.parse = ifam.parseInterfaceAddrMessage + // NetBSD 6 and above kernels require 64-bit aligned access to + // routing facilities. + return 8, map[int]*wireFormat{ + syscall.RTM_NEWADDR: ifam, + syscall.RTM_DELADDR: ifam, + syscall.RTM_IFINFO: ifm, + } +} diff --git a/src/internal/routebsd/sys_openbsd.go b/src/internal/routebsd/sys_openbsd.go new file mode 100644 index 00000000000000..83256c8fabfa59 --- /dev/null +++ b/src/internal/routebsd/sys_openbsd.go @@ -0,0 +1,28 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package routebsd + +import ( + "syscall" + "unsafe" +) + +// MTU returns the interface MTU. +func (m *InterfaceMessage) MTU() int { + return int(nativeEndian.Uint32(m.raw[28:32])) +} + +func probeRoutingStack() (int, map[int]*wireFormat) { + var p uintptr + ifm := &wireFormat{extOff: -1, bodyOff: -1} + ifm.parse = ifm.parseInterfaceMessage + ifam := &wireFormat{extOff: -1, bodyOff: -1} + ifam.parse = ifam.parseInterfaceAddrMessage + return int(unsafe.Sizeof(p)), map[int]*wireFormat{ + syscall.RTM_NEWADDR: ifam, + syscall.RTM_DELADDR: ifam, + syscall.RTM_IFINFO: ifm, + } +} diff --git a/src/internal/runtime/atomic/atomic_386.go b/src/internal/runtime/atomic/atomic_386.go index a023baddb764fe..b6cdea61abbe9b 100644 --- a/src/internal/runtime/atomic/atomic_386.go +++ b/src/internal/runtime/atomic/atomic_386.go @@ -53,6 +53,9 @@ func Xchg64(ptr *uint64, new uint64) uint64 //go:noescape func Xchg(ptr *uint32, new uint32) uint32 +//go:noescape +func Xchg8(ptr *uint8, new uint8) uint8 + //go:noescape func Xchguintptr(ptr *uintptr, new uintptr) uintptr diff --git a/src/internal/runtime/atomic/atomic_386.s b/src/internal/runtime/atomic/atomic_386.s index 08812c37ecc613..e8745a1bb5441d 100644 --- a/src/internal/runtime/atomic/atomic_386.s +++ b/src/internal/runtime/atomic/atomic_386.s @@ -5,13 +5,14 @@ #include "textflag.h" #include "funcdata.h" -// bool Cas(int32 *val, int32 old, int32 new) +// func Cas(ptr *int32, old, new int32) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// }else -// return 0; +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } TEXT ·Cas(SB), NOSPLIT, $0-13 MOVL ptr+0(FP), BX MOVL old+4(FP), AX @@ -63,13 +64,13 @@ TEXT ·Xaddint32(SB), NOSPLIT, $0-12 TEXT ·Xaddint64(SB), NOSPLIT, $0-20 JMP ·Xadd64(SB) -// bool ·Cas64(uint64 *val, uint64 old, uint64 new) +// func Cas64(ptr *uint64, old, new uint64) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; +// if *ptr == old { +// *ptr = new +// return true // } else { -// return 0; +// return false // } TEXT ·Cas64(SB), NOSPLIT, $0-21 NO_LOCAL_POINTERS @@ -86,13 +87,14 @@ TEXT ·Cas64(SB), NOSPLIT, $0-21 SETEQ ret+20(FP) RET -// bool Casp1(void **p, void *old, void *new) +// func Casp1(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool // Atomically: -// if(*p == old){ -// *p = new; -// return 1; -// }else -// return 0; +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } TEXT ·Casp1(SB), NOSPLIT, $0-13 MOVL ptr+0(FP), BX MOVL old+4(FP), AX @@ -153,6 +155,14 @@ addloop: MOVL CX, ret_hi+16(FP) RET +// uint8 Xchg8(uint8 *ptr, uint8 new) +TEXT ·Xchg8(SB), NOSPLIT, $0-9 + MOVL ptr+0(FP), BX + MOVB new+4(FP), AX + XCHGB AX, 0(BX) + MOVB AX, ret+8(FP) + RET + TEXT ·Xchg(SB), NOSPLIT, $0-12 MOVL ptr+0(FP), BX MOVL new+4(FP), AX diff --git a/src/internal/runtime/atomic/atomic_amd64.go b/src/internal/runtime/atomic/atomic_amd64.go index b439954093cd28..2a2d07e5119139 100644 --- a/src/internal/runtime/atomic/atomic_amd64.go +++ b/src/internal/runtime/atomic/atomic_amd64.go @@ -57,6 +57,9 @@ func Xadd64(ptr *uint64, delta int64) uint64 //go:noescape func Xadduintptr(ptr *uintptr, delta uintptr) uintptr +//go:noescape +func Xchg8(ptr *uint8, new uint8) uint8 + //go:noescape func Xchg(ptr *uint32, new uint32) uint32 diff --git a/src/internal/runtime/atomic/atomic_amd64.s b/src/internal/runtime/atomic/atomic_amd64.s index ec75bf9332f79c..301725e6a9dcc9 100644 --- a/src/internal/runtime/atomic/atomic_amd64.s +++ b/src/internal/runtime/atomic/atomic_amd64.s @@ -19,13 +19,14 @@ TEXT ·Loadint32(SB), NOSPLIT, $0-12 TEXT ·Loadint64(SB), NOSPLIT, $0-16 JMP ·Load64(SB) -// bool Cas(int32 *val, int32 old, int32 new) +// func Cas(ptr *int32, old, new int32) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// } else -// return 0; +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } TEXT ·Cas(SB),NOSPLIT,$0-17 MOVQ ptr+0(FP), BX MOVL old+8(FP), AX @@ -35,13 +36,13 @@ TEXT ·Cas(SB),NOSPLIT,$0-17 SETEQ ret+16(FP) RET -// bool ·Cas64(uint64 *val, uint64 old, uint64 new) +// func Cas64(ptr *uint64, old, new uint64) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; +// if *ptr == old { +// *ptr = new +// return true // } else { -// return 0; +// return false // } TEXT ·Cas64(SB), NOSPLIT, $0-25 MOVQ ptr+0(FP), BX @@ -52,13 +53,14 @@ TEXT ·Cas64(SB), NOSPLIT, $0-25 SETEQ ret+24(FP) RET -// bool Casp1(void **val, void *old, void *new) +// func Casp1(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// } else -// return 0; +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } TEXT ·Casp1(SB), NOSPLIT, $0-25 MOVQ ptr+0(FP), BX MOVQ old+8(FP), AX @@ -117,6 +119,18 @@ TEXT ·Xaddint64(SB), NOSPLIT, $0-24 TEXT ·Xadduintptr(SB), NOSPLIT, $0-24 JMP ·Xadd64(SB) +// uint8 Xchg(ptr *uint8, new uint8) +// Atomically: +// old := *ptr; +// *ptr = new; +// return old; +TEXT ·Xchg8(SB), NOSPLIT, $0-17 + MOVQ ptr+0(FP), BX + MOVB new+8(FP), AX + XCHGB AX, 0(BX) + MOVB AX, ret+16(FP) + RET + // uint32 Xchg(ptr *uint32, new uint32) // Atomically: // old := *ptr; diff --git a/src/internal/runtime/atomic/atomic_arm.go b/src/internal/runtime/atomic/atomic_arm.go index b58f643ca34c1c..e858fb7cd1a99a 100644 --- a/src/internal/runtime/atomic/atomic_arm.go +++ b/src/internal/runtime/atomic/atomic_arm.go @@ -74,6 +74,9 @@ func Xchg(addr *uint32, v uint32) uint32 { } } +//go:noescape +func Xchg8(addr *uint8, v uint8) uint8 + //go:nosplit func Xchguintptr(addr *uintptr, v uintptr) uintptr { return uintptr(Xchg((*uint32)(unsafe.Pointer(addr)), uint32(v))) @@ -159,12 +162,14 @@ func goStore64(addr *uint64, v uint64) { addrLock(addr).unlock() } +//go:noescape +func Or8(addr *uint8, v uint8) + //go:nosplit -func Or8(addr *uint8, v uint8) { +func goOr8(addr *uint8, v uint8) { // Align down to 4 bytes and use 32-bit CAS. - uaddr := uintptr(unsafe.Pointer(addr)) - addr32 := (*uint32)(unsafe.Pointer(uaddr &^ 3)) - word := uint32(v) << ((uaddr & 3) * 8) // little endian + addr32 := (*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(addr)) &^ 3)) + word := uint32(v) << ((uintptr(unsafe.Pointer(addr)) & 3) * 8) // little endian for { old := *addr32 if Cas(addr32, old, old|word) { @@ -173,13 +178,15 @@ func Or8(addr *uint8, v uint8) { } } +//go:noescape +func And8(addr *uint8, v uint8) + //go:nosplit -func And8(addr *uint8, v uint8) { +func goAnd8(addr *uint8, v uint8) { // Align down to 4 bytes and use 32-bit CAS. - uaddr := uintptr(unsafe.Pointer(addr)) - addr32 := (*uint32)(unsafe.Pointer(uaddr &^ 3)) - word := uint32(v) << ((uaddr & 3) * 8) // little endian - mask := uint32(0xFF) << ((uaddr & 3) * 8) // little endian + addr32 := (*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(addr)) &^ 3)) + word := uint32(v) << ((uintptr(unsafe.Pointer(addr)) & 3) * 8) // little endian + mask := uint32(0xFF) << ((uintptr(unsafe.Pointer(addr)) & 3) * 8) // little endian word |= ^mask for { old := *addr32 diff --git a/src/internal/runtime/atomic/atomic_arm.s b/src/internal/runtime/atomic/atomic_arm.s index 1cf7d8f6ef3560..103f718a041c53 100644 --- a/src/internal/runtime/atomic/atomic_arm.s +++ b/src/internal/runtime/atomic/atomic_arm.s @@ -6,13 +6,14 @@ #include "textflag.h" #include "funcdata.h" -// bool armcas(int32 *val, int32 old, int32 new) +// func armcas(ptr *int32, old, new int32) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// }else -// return 0; +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } // // To implement ·cas in sys_$GOOS_arm.s // using the native instructions, use: @@ -228,6 +229,59 @@ store64loop: DMB MB_ISH RET +TEXT armAnd8<>(SB),NOSPLIT,$0-5 + // addr is already in R1 + MOVB v+4(FP), R2 + +and8loop: + LDREXB (R1), R6 + + DMB MB_ISHST + + AND R2, R6 + STREXB R6, (R1), R0 + CMP $0, R0 + BNE and8loop + + DMB MB_ISH + + RET + +TEXT armOr8<>(SB),NOSPLIT,$0-5 + // addr is already in R1 + MOVB v+4(FP), R2 + +or8loop: + LDREXB (R1), R6 + + DMB MB_ISHST + + ORR R2, R6 + STREXB R6, (R1), R0 + CMP $0, R0 + BNE or8loop + + DMB MB_ISH + + RET + +TEXT armXchg8<>(SB),NOSPLIT,$0-9 + // addr is already in R1 + MOVB v+4(FP), R2 +xchg8loop: + LDREXB (R1), R6 + + DMB MB_ISHST + + STREXB R2, (R1), R0 + CMP $0, R0 + BNE xchg8loop + + DMB MB_ISH + + MOVB R6, ret+8(FP) + RET + // The following functions all panic if their address argument isn't // 8-byte aligned. Since we're calling back into Go code to do this, // we have to cooperate with stack unwinding. In the normal case, the @@ -310,3 +364,45 @@ TEXT ·Store64(SB),NOSPLIT,$-4-12 JMP ·goStore64(SB) #endif JMP armStore64<>(SB) + +TEXT ·And8(SB),NOSPLIT,$-4-5 + NO_LOCAL_POINTERS + MOVW addr+0(FP), R1 + +// Uses STREXB/LDREXB that is armv6k or later. +// For simplicity we only enable this on armv7. +#ifndef GOARM_7 + MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11 + CMP $1, R11 + BEQ 2(PC) + JMP ·goAnd8(SB) +#endif + JMP armAnd8<>(SB) + +TEXT ·Or8(SB),NOSPLIT,$-4-5 + NO_LOCAL_POINTERS + MOVW addr+0(FP), R1 + +// Uses STREXB/LDREXB that is armv6k or later. +// For simplicity we only enable this on armv7. +#ifndef GOARM_7 + MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11 + CMP $1, R11 + BEQ 2(PC) + JMP ·goOr8(SB) +#endif + JMP armOr8<>(SB) + +TEXT ·Xchg8(SB),NOSPLIT,$-4-9 + NO_LOCAL_POINTERS + MOVW addr+0(FP), R1 + + // Uses STREXB/LDREXB that is armv6k or later. + // For simplicity we only enable this on armv7. +#ifndef GOARM_7 + MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11 + CMP $1, R11 + BEQ 2(PC) + JMP ·goXchg8(SB) +#endif + JMP armXchg8<>(SB) diff --git a/src/internal/runtime/atomic/atomic_arm64.go b/src/internal/runtime/atomic/atomic_arm64.go index c4c56ae8951c6b..f4aef19388d562 100644 --- a/src/internal/runtime/atomic/atomic_arm64.go +++ b/src/internal/runtime/atomic/atomic_arm64.go @@ -24,6 +24,9 @@ func Xadd64(ptr *uint64, delta int64) uint64 //go:noescape func Xadduintptr(ptr *uintptr, delta uintptr) uintptr +//go:noescape +func Xchg8(ptr *uint8, new uint8) uint8 + //go:noescape func Xchg(ptr *uint32, new uint32) uint32 diff --git a/src/internal/runtime/atomic/atomic_arm64.s b/src/internal/runtime/atomic/atomic_arm64.s index ede56538b880cb..360f7a28163b04 100644 --- a/src/internal/runtime/atomic/atomic_arm64.s +++ b/src/internal/runtime/atomic/atomic_arm64.s @@ -120,6 +120,30 @@ TEXT ·Store64(SB), NOSPLIT, $0-16 STLR R1, (R0) RET +// uint8 Xchg(ptr *uint8, new uint8) +// Atomically: +// old := *ptr; +// *ptr = new; +// return old; +TEXT ·Xchg8(SB), NOSPLIT, $0-17 + MOVD ptr+0(FP), R0 + MOVB new+8(FP), R1 +#ifndef GOARM64_LSE + MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4 + CBZ R4, load_store_loop +#endif + SWPALB R1, (R0), R2 + MOVB R2, ret+16(FP) + RET +#ifndef GOARM64_LSE +load_store_loop: + LDAXRB (R0), R2 + STLXRB R1, (R0), R3 + CBNZ R3, load_store_loop + MOVB R2, ret+16(FP) + RET +#endif + // uint32 Xchg(ptr *uint32, new uint32) // Atomically: // old := *ptr; @@ -168,13 +192,14 @@ load_store_loop: RET #endif -// bool Cas(uint32 *ptr, uint32 old, uint32 new) +// func Cas(ptr *uint32, old, new uint32) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// } else -// return 0; +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } TEXT ·Cas(SB), NOSPLIT, $0-17 MOVD ptr+0(FP), R0 MOVW old+8(FP), R1 @@ -202,14 +227,14 @@ ok: RET #endif -// bool ·Cas64(uint64 *ptr, uint64 old, uint64 new) +// func Cas64(ptr *uint64, old, new uint64) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// } else { -// return 0; -// } +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } TEXT ·Cas64(SB), NOSPLIT, $0-25 MOVD ptr+0(FP), R0 MOVD old+8(FP), R1 diff --git a/src/internal/runtime/atomic/atomic_loong64.go b/src/internal/runtime/atomic/atomic_loong64.go index de6d4b4ba67d7c..1fa1a9fa5ab4a1 100644 --- a/src/internal/runtime/atomic/atomic_loong64.go +++ b/src/internal/runtime/atomic/atomic_loong64.go @@ -6,7 +6,15 @@ package atomic -import "unsafe" +import ( + "internal/cpu" + "unsafe" +) + +const ( + offsetLOONG64HasLAMCAS = unsafe.Offsetof(cpu.Loong64.HasLAMCAS) + offsetLoong64HasLAM_BH = unsafe.Offsetof(cpu.Loong64.HasLAM_BH) +) //go:noescape func Xadd(ptr *uint32, delta int32) uint32 @@ -17,6 +25,9 @@ func Xadd64(ptr *uint64, delta int64) uint64 //go:noescape func Xadduintptr(ptr *uintptr, delta uintptr) uintptr +//go:noescape +func Xchg8(ptr *uint8, new uint8) uint8 + //go:noescape func Xchg(ptr *uint32, new uint32) uint32 diff --git a/src/internal/runtime/atomic/atomic_loong64.s b/src/internal/runtime/atomic/atomic_loong64.s index 1812cb95fd4edd..4215af24febaa2 100644 --- a/src/internal/runtime/atomic/atomic_loong64.s +++ b/src/internal/runtime/atomic/atomic_loong64.s @@ -2,61 +2,99 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include "go_asm.h" #include "textflag.h" -// bool cas(uint32 *ptr, uint32 old, uint32 new) +// func Cas(ptr *int32, old, new int32) bool // Atomically: -// if(*ptr == old){ -// *ptr = new; -// return 1; -// } else -// return 0; +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } TEXT ·Cas(SB), NOSPLIT, $0-17 MOVV ptr+0(FP), R4 MOVW old+8(FP), R5 MOVW new+12(FP), R6 - DBAR + + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8 + BEQ R8, ll_sc + MOVV R5, R7 // backup old value + AMCASDBW R6, (R4), R5 + BNE R7, R5, cas_fail0 + MOVV $1, R4 + MOVB R4, ret+16(FP) + RET +cas_fail0: + MOVB R0, ret+16(FP) + RET + +ll_sc: + // Implemented using the ll-sc instruction pair + DBAR $0x14 // LoadAcquire barrier cas_again: MOVV R6, R7 LL (R4), R8 - BNE R5, R8, cas_fail + BNE R5, R8, cas_fail1 SC R7, (R4) BEQ R7, cas_again MOVV $1, R4 MOVB R4, ret+16(FP) - DBAR + DBAR $0x12 // StoreRelease barrier RET -cas_fail: +cas_fail1: MOVV $0, R4 JMP -4(PC) -// bool cas64(uint64 *ptr, uint64 old, uint64 new) +// func Cas64(ptr *uint64, old, new uint64) bool // Atomically: -// if(*ptr == old){ -// *ptr = new; -// return 1; +// if *ptr == old { +// *ptr = new +// return true // } else { -// return 0; +// return false // } TEXT ·Cas64(SB), NOSPLIT, $0-25 MOVV ptr+0(FP), R4 MOVV old+8(FP), R5 MOVV new+16(FP), R6 - DBAR + + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8 + BEQ R8, ll_sc_64 + MOVV R5, R7 // backup old value + AMCASDBV R6, (R4), R5 + BNE R7, R5, cas64_fail0 + MOVV $1, R4 + MOVB R4, ret+24(FP) + RET +cas64_fail0: + MOVB R0, ret+24(FP) + RET + +ll_sc_64: + // Implemented using the ll-sc instruction pair + DBAR $0x14 cas64_again: MOVV R6, R7 LLV (R4), R8 - BNE R5, R8, cas64_fail + BNE R5, R8, cas64_fail1 SCV R7, (R4) BEQ R7, cas64_again MOVV $1, R4 MOVB R4, ret+24(FP) - DBAR + DBAR $0x12 RET -cas64_fail: +cas64_fail1: MOVV $0, R4 JMP -4(PC) +TEXT ·Casint32(SB),NOSPLIT,$0-17 + JMP ·Cas(SB) + +TEXT ·Casint64(SB),NOSPLIT,$0-25 + JMP ·Cas64(SB) + TEXT ·Casuintptr(SB), NOSPLIT, $0-25 JMP ·Cas64(SB) @@ -78,78 +116,109 @@ TEXT ·Xadduintptr(SB), NOSPLIT, $0-24 TEXT ·Loadint64(SB), NOSPLIT, $0-16 JMP ·Load64(SB) +TEXT ·Xaddint32(SB),NOSPLIT,$0-20 + JMP ·Xadd(SB) + TEXT ·Xaddint64(SB), NOSPLIT, $0-24 JMP ·Xadd64(SB) -// bool casp(void **val, void *old, void *new) +// func Casp(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// } else -// return 0; +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } TEXT ·Casp1(SB), NOSPLIT, $0-25 JMP ·Cas64(SB) -// uint32 xadd(uint32 volatile *ptr, int32 delta) +// uint32 Xadd(uint32 volatile *ptr, int32 delta) // Atomically: // *val += delta; // return *val; TEXT ·Xadd(SB), NOSPLIT, $0-20 MOVV ptr+0(FP), R4 MOVW delta+8(FP), R5 - DBAR - LL (R4), R6 - ADDU R6, R5, R7 - MOVV R7, R6 - SC R7, (R4) - BEQ R7, -4(PC) - MOVW R6, ret+16(FP) - DBAR + AMADDDBW R5, (R4), R6 + ADDV R6, R5, R4 + MOVW R4, ret+16(FP) RET +// func Xadd64(ptr *uint64, delta int64) uint64 TEXT ·Xadd64(SB), NOSPLIT, $0-24 MOVV ptr+0(FP), R4 MOVV delta+8(FP), R5 - DBAR - LLV (R4), R6 - ADDVU R6, R5, R7 - MOVV R7, R6 - SCV R7, (R4) - BEQ R7, -4(PC) - MOVV R6, ret+16(FP) - DBAR + AMADDDBV R5, (R4), R6 + ADDV R6, R5, R4 + MOVV R4, ret+16(FP) RET +// uint8 Xchg8(ptr *uint8, new uint8) +// Atomically: +// old := *ptr; +// *ptr = new; +// return old; +TEXT ·Xchg8(SB), NOSPLIT, $0-17 + MOVV ptr+0(FP), R4 + MOVBU new+8(FP), R5 + + // R6 = ((ptr & 3) * 8) + AND $3, R4, R6 + SLLV $3, R6 + + // R7 = ((0xFF) << R6) ^ (-1) + MOVV $0xFF, R8 + SLLV R6, R8, R7 + XOR $-1, R7 + + // R4 = ptr & (~3) + MOVV $~3, R8 + AND R8, R4 + + // R5 = ((val) << R6) + SLLV R6, R5 + + DBAR $0x14 // LoadAcquire barrier +_xchg8_again: + LL (R4), R8 + MOVV R8, R9 // backup old val + AND R7, R8 + OR R5, R8 + SC R8, (R4) + BEQ R8, _xchg8_again + DBAR $0x12 // StoreRelease barrier + SRLV R6, R9, R9 + MOVBU R9, ret+16(FP) + RET + +// func Xchg(ptr *uint32, new uint32) uint32 TEXT ·Xchg(SB), NOSPLIT, $0-20 MOVV ptr+0(FP), R4 MOVW new+8(FP), R5 - - DBAR - MOVV R5, R6 - LL (R4), R7 - SC R6, (R4) - BEQ R6, -3(PC) - MOVW R7, ret+16(FP) - DBAR + AMSWAPDBW R5, (R4), R6 + MOVW R6, ret+16(FP) RET +// func Xchg64(ptr *uint64, new uint64) uint64 TEXT ·Xchg64(SB), NOSPLIT, $0-24 MOVV ptr+0(FP), R4 MOVV new+8(FP), R5 - - DBAR - MOVV R5, R6 - LLV (R4), R7 - SCV R6, (R4) - BEQ R6, -3(PC) - MOVV R7, ret+16(FP) - DBAR + AMSWAPDBV R5, (R4), R6 + MOVV R6, ret+16(FP) RET TEXT ·Xchguintptr(SB), NOSPLIT, $0-24 JMP ·Xchg64(SB) +// func Xchgint32(ptr *int32, new int32) int32 +TEXT ·Xchgint32(SB), NOSPLIT, $0-20 + JMP ·Xchg(SB) + +// func Xchgint64(ptr *int64, new int64) int64 +TEXT ·Xchgint64(SB), NOSPLIT, $0-24 + JMP ·Xchg64(SB) + TEXT ·StorepNoWB(SB), NOSPLIT, $0-16 JMP ·Store64(SB) @@ -165,147 +234,105 @@ TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16 TEXT ·Store(SB), NOSPLIT, $0-12 MOVV ptr+0(FP), R4 MOVW val+8(FP), R5 - DBAR - MOVW R5, 0(R4) - DBAR + AMSWAPDBW R5, (R4), R0 RET TEXT ·Store8(SB), NOSPLIT, $0-9 MOVV ptr+0(FP), R4 MOVB val+8(FP), R5 - DBAR + MOVBU internal∕cpu·Loong64+const_offsetLoong64HasLAM_BH(SB), R6 + BEQ R6, _legacy_store8_ + AMSWAPDBB R5, (R4), R0 + RET +_legacy_store8_: + // StoreRelease barrier + DBAR $0x12 MOVB R5, 0(R4) - DBAR + DBAR $0x18 RET TEXT ·Store64(SB), NOSPLIT, $0-16 MOVV ptr+0(FP), R4 MOVV val+8(FP), R5 - DBAR - MOVV R5, 0(R4) - DBAR + AMSWAPDBV R5, (R4), R0 RET // void Or8(byte volatile*, byte); TEXT ·Or8(SB), NOSPLIT, $0-9 MOVV ptr+0(FP), R4 MOVBU val+8(FP), R5 - // Align ptr down to 4 bytes so we can use 32-bit load/store. + // R6 = ptr & (~3) MOVV $~3, R6 AND R4, R6 // R7 = ((ptr & 3) * 8) AND $3, R4, R7 SLLV $3, R7 - // Shift val for aligned ptr. R5 = val << R4 + // R5 = val << R7 SLLV R7, R5 - - DBAR - LL (R6), R7 - OR R5, R7 - SC R7, (R6) - BEQ R7, -4(PC) - DBAR + AMORDBW R5, (R6), R0 RET // void And8(byte volatile*, byte); TEXT ·And8(SB), NOSPLIT, $0-9 MOVV ptr+0(FP), R4 MOVBU val+8(FP), R5 - // Align ptr down to 4 bytes so we can use 32-bit load/store. + // R6 = ptr & (~3) MOVV $~3, R6 AND R4, R6 // R7 = ((ptr & 3) * 8) AND $3, R4, R7 SLLV $3, R7 - // Shift val for aligned ptr. R5 = val << R7 | ^(0xFF << R7) - MOVV $0xFF, R8 - SLLV R7, R5 - SLLV R7, R8 - NOR R0, R8 - OR R8, R5 - - DBAR - LL (R6), R7 - AND R5, R7 - SC R7, (R6) - BEQ R7, -4(PC) - DBAR + // R5 = ((val ^ 0xFF) << R7) ^ (-1) + XOR $255, R5 + SLLV R7, R5 + XOR $-1, R5 + AMANDDBW R5, (R6), R0 RET // func Or(addr *uint32, v uint32) TEXT ·Or(SB), NOSPLIT, $0-12 MOVV ptr+0(FP), R4 MOVW val+8(FP), R5 - DBAR - LL (R4), R6 - OR R5, R6 - SC R6, (R4) - BEQ R6, -4(PC) - DBAR + AMORDBW R5, (R4), R0 RET // func And(addr *uint32, v uint32) TEXT ·And(SB), NOSPLIT, $0-12 MOVV ptr+0(FP), R4 MOVW val+8(FP), R5 - DBAR - LL (R4), R6 - AND R5, R6 - SC R6, (R4) - BEQ R6, -4(PC) - DBAR + AMANDDBW R5, (R4), R0 RET // func Or32(addr *uint32, v uint32) old uint32 TEXT ·Or32(SB), NOSPLIT, $0-20 MOVV ptr+0(FP), R4 MOVW val+8(FP), R5 - DBAR - LL (R4), R6 - OR R5, R6, R7 - SC R7, (R4) - BEQ R7, -4(PC) - DBAR - MOVW R6, ret+16(FP) + AMORDBW R5, (R4), R6 + MOVW R6, ret+16(FP) RET // func And32(addr *uint32, v uint32) old uint32 TEXT ·And32(SB), NOSPLIT, $0-20 MOVV ptr+0(FP), R4 MOVW val+8(FP), R5 - DBAR - LL (R4), R6 - AND R5, R6, R7 - SC R7, (R4) - BEQ R7, -4(PC) - DBAR - MOVW R6, ret+16(FP) + AMANDDBW R5, (R4), R6 + MOVW R6, ret+16(FP) RET // func Or64(addr *uint64, v uint64) old uint64 TEXT ·Or64(SB), NOSPLIT, $0-24 MOVV ptr+0(FP), R4 MOVV val+8(FP), R5 - DBAR - LLV (R4), R6 - OR R5, R6, R7 - SCV R7, (R4) - BEQ R7, -4(PC) - DBAR - MOVV R6, ret+16(FP) + AMORDBV R5, (R4), R6 + MOVV R6, ret+16(FP) RET // func And64(addr *uint64, v uint64) old uint64 TEXT ·And64(SB), NOSPLIT, $0-24 MOVV ptr+0(FP), R4 MOVV val+8(FP), R5 - DBAR - LLV (R4), R6 - AND R5, R6, R7 - SCV R7, (R4) - BEQ R7, -4(PC) - DBAR - MOVV R6, ret+16(FP) + AMANDDBV R5, (R4), R6 + MOVV R6, ret+16(FP) RET // func Anduintptr(addr *uintptr, v uintptr) old uintptr @@ -319,38 +346,30 @@ TEXT ·Oruintptr(SB), NOSPLIT, $0-24 // uint32 internal∕runtime∕atomic·Load(uint32 volatile* ptr) TEXT ·Load(SB),NOSPLIT|NOFRAME,$0-12 MOVV ptr+0(FP), R19 - DBAR MOVWU 0(R19), R19 - DBAR + DBAR $0x14 // LoadAcquire barrier MOVW R19, ret+8(FP) RET // uint8 internal∕runtime∕atomic·Load8(uint8 volatile* ptr) TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-9 MOVV ptr+0(FP), R19 - DBAR MOVBU 0(R19), R19 - DBAR + DBAR $0x14 MOVB R19, ret+8(FP) RET // uint64 internal∕runtime∕atomic·Load64(uint64 volatile* ptr) TEXT ·Load64(SB),NOSPLIT|NOFRAME,$0-16 MOVV ptr+0(FP), R19 - DBAR MOVV 0(R19), R19 - DBAR + DBAR $0x14 MOVV R19, ret+8(FP) RET // void *internal∕runtime∕atomic·Loadp(void *volatile *ptr) TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-16 - MOVV ptr+0(FP), R19 - DBAR - MOVV 0(R19), R19 - DBAR - MOVV R19, ret+8(FP) - RET + JMP ·Load64(SB) // uint32 internal∕runtime∕atomic·LoadAcq(uint32 volatile* ptr) TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12 diff --git a/src/internal/runtime/atomic/atomic_mips64x.go b/src/internal/runtime/atomic/atomic_mips64x.go index f434c939e3c815..4c0148f8a75f35 100644 --- a/src/internal/runtime/atomic/atomic_mips64x.go +++ b/src/internal/runtime/atomic/atomic_mips64x.go @@ -20,6 +20,9 @@ func Xadduintptr(ptr *uintptr, delta uintptr) uintptr //go:noescape func Xchg(ptr *uint32, new uint32) uint32 +//go:noescape +func Xchg8(ptr *uint8, new uint8) uint8 + //go:noescape func Xchg64(ptr *uint64, new uint64) uint64 diff --git a/src/internal/runtime/atomic/atomic_mips64x.s b/src/internal/runtime/atomic/atomic_mips64x.s index 7b0e080238d0b1..ec56cb7c68a5a8 100644 --- a/src/internal/runtime/atomic/atomic_mips64x.s +++ b/src/internal/runtime/atomic/atomic_mips64x.s @@ -8,13 +8,14 @@ #define SYNC WORD $0xf -// bool cas(uint32 *ptr, uint32 old, uint32 new) +// func cas(ptr *uint32, old, new uint32) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// } else -// return 0; +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } TEXT ·Cas(SB), NOSPLIT, $0-17 MOVV ptr+0(FP), R1 MOVW old+8(FP), R2 @@ -34,13 +35,13 @@ cas_fail: MOVV $0, R1 JMP -4(PC) -// bool cas64(uint64 *ptr, uint64 old, uint64 new) +// func Cas64(ptr *uint64, old, new uint64) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; +// if *ptr == old { +// *ptr = new +// return true // } else { -// return 0; +// return false // } TEXT ·Cas64(SB), NOSPLIT, $0-25 MOVV ptr+0(FP), R1 @@ -103,13 +104,14 @@ TEXT ·Xaddint32(SB), NOSPLIT, $0-20 TEXT ·Xaddint64(SB), NOSPLIT, $0-24 JMP ·Xadd64(SB) -// bool casp(void **val, void *old, void *new) +// func Casp1(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// } else -// return 0; +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } TEXT ·Casp1(SB), NOSPLIT, $0-25 JMP ·Cas64(SB) @@ -147,6 +149,39 @@ TEXT ·Xadd64(SB), NOSPLIT, $0-24 SYNC RET +// uint8 Xchg(ptr *uint8, new uint8) +// Atomically: +// old := *ptr; +// *ptr = new; +// return old; +TEXT ·Xchg8(SB), NOSPLIT, $0-17 + MOVV ptr+0(FP), R2 + MOVBU new+8(FP), R5 +#ifdef GOARCH_mips64 + // Big endian. ptr = ptr ^ 3 + XOR $3, R2 +#endif + // R4 = ((ptr & 3) * 8) + AND $3, R2, R4 + SLLV $3, R4 + // Shift val for aligned ptr. R7 = (0xFF << R4) ^ (-1) + MOVV $0xFF, R7 + SLLV R4, R7 + XOR $-1, R7 + AND $~3, R2 + SLLV R4, R5 + + SYNC + LL (R2), R9 + AND R7, R9, R8 + OR R5, R8 + SC R8, (R2) + BEQ R8, -5(PC) + SYNC + SRLV R4, R9 + MOVBU R9, ret+16(FP) + RET + // uint32 Xchg(ptr *uint32, new uint32) // Atomically: // old := *ptr; diff --git a/src/internal/runtime/atomic/atomic_mipsx.go b/src/internal/runtime/atomic/atomic_mipsx.go index aba4143ea6613d..bf3578734a1069 100644 --- a/src/internal/runtime/atomic/atomic_mipsx.go +++ b/src/internal/runtime/atomic/atomic_mipsx.go @@ -135,6 +135,9 @@ func Xadduintptr(ptr *uintptr, delta uintptr) uintptr //go:noescape func Xchg(ptr *uint32, new uint32) uint32 +//go:noescape +func Xchg8(ptr *uint8, new uint8) uint8 + //go:noescape func Xchguintptr(ptr *uintptr, new uintptr) uintptr diff --git a/src/internal/runtime/atomic/atomic_mipsx.s b/src/internal/runtime/atomic/atomic_mipsx.s index 4ccc0a363b7f08..e4f80c913eb590 100644 --- a/src/internal/runtime/atomic/atomic_mipsx.s +++ b/src/internal/runtime/atomic/atomic_mipsx.s @@ -6,13 +6,14 @@ #include "textflag.h" -// bool Cas(int32 *val, int32 old, int32 new) +// func Cas(ptr *int32, old, new int32) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// } else -// return 0; +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } TEXT ·Cas(SB),NOSPLIT,$0-13 MOVW ptr+0(FP), R1 MOVW old+4(FP), R2 @@ -100,6 +101,39 @@ try_xchg: MOVW R1, ret+8(FP) RET +// uint8 Xchg(ptr *uint8, new uint8) +// Atomically: +// old := *ptr; +// *ptr = new; +// return old; +TEXT ·Xchg8(SB), NOSPLIT, $0-9 + MOVW ptr+0(FP), R2 + MOVBU new+4(FP), R5 +#ifdef GOARCH_mips + // Big endian. ptr = ptr ^ 3 + XOR $3, R2 +#endif + // R4 = ((ptr & 3) * 8) + AND $3, R2, R4 + SLL $3, R4 + // Shift val for aligned ptr. R7 = (0xFF << R4) ^ (-1) + MOVW $0xFF, R7 + SLL R4, R7 + XOR $-1, R7 + AND $~3, R2 + SLL R4, R5 + + SYNC + LL (R2), R9 + AND R7, R9, R8 + OR R5, R8 + SC R8, (R2) + BEQ R8, -5(PC) + SYNC + SRL R4, R9 + MOVBU R9, ret+8(FP) + RET + TEXT ·Casint32(SB),NOSPLIT,$0-13 JMP ·Cas(SB) diff --git a/src/internal/runtime/atomic/atomic_ppc64x.go b/src/internal/runtime/atomic/atomic_ppc64x.go index 33a92b53f47ff0..590ba03ecf5c9f 100644 --- a/src/internal/runtime/atomic/atomic_ppc64x.go +++ b/src/internal/runtime/atomic/atomic_ppc64x.go @@ -17,6 +17,9 @@ func Xadd64(ptr *uint64, delta int64) uint64 //go:noescape func Xadduintptr(ptr *uintptr, delta uintptr) uintptr +//go:noescape +func Xchg8(ptr *uint8, new uint8) uint8 + //go:noescape func Xchg(ptr *uint32, new uint32) uint32 diff --git a/src/internal/runtime/atomic/atomic_ppc64x.s b/src/internal/runtime/atomic/atomic_ppc64x.s index 75635b933d7423..bff7d1902a0e15 100644 --- a/src/internal/runtime/atomic/atomic_ppc64x.s +++ b/src/internal/runtime/atomic/atomic_ppc64x.s @@ -78,13 +78,14 @@ TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$-8-16 MOVD R3, ret+8(FP) RET -// bool cas(uint32 *ptr, uint32 old, uint32 new) +// func Cas(ptr *int32, old, new int32) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// } else -// return 0; +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } TEXT ·Cas(SB), NOSPLIT, $0-17 MOVD ptr+0(FP), R3 MOVWZ old+8(FP), R4 @@ -105,13 +106,13 @@ cas_fail: MOVB R0, ret+16(FP) RET -// bool ·Cas64(uint64 *ptr, uint64 old, uint64 new) +// func Cas64(ptr *uint64, old, new uint64) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; +// if *ptr == old { +// *ptr = new +// return true // } else { -// return 0; +// return false // } TEXT ·Cas64(SB), NOSPLIT, $0-25 MOVD ptr+0(FP), R3 @@ -196,13 +197,14 @@ TEXT ·Xaddint32(SB), NOSPLIT, $0-20 TEXT ·Xaddint64(SB), NOSPLIT, $0-24 BR ·Xadd64(SB) -// bool casp(void **val, void *old, void *new) +// func Casp1(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// } else -// return 0; +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } TEXT ·Casp1(SB), NOSPLIT, $0-25 BR ·Cas64(SB) @@ -236,6 +238,22 @@ TEXT ·Xadd64(SB), NOSPLIT, $0-24 MOVD R3, ret+16(FP) RET +// uint8 Xchg(ptr *uint8, new uint8) +// Atomically: +// old := *ptr; +// *ptr = new; +// return old; +TEXT ·Xchg8(SB), NOSPLIT, $0-17 + MOVD ptr+0(FP), R4 + MOVB new+8(FP), R5 + LWSYNC + LBAR (R4), R3 + STBCCC R5, (R4) + BNE -2(PC) + ISYNC + MOVB R3, ret+16(FP) + RET + // uint32 Xchg(ptr *uint32, new uint32) // Atomically: // old := *ptr; diff --git a/src/internal/runtime/atomic/atomic_riscv64.go b/src/internal/runtime/atomic/atomic_riscv64.go index 9fc38376aeee44..d09919d0fa4eac 100644 --- a/src/internal/runtime/atomic/atomic_riscv64.go +++ b/src/internal/runtime/atomic/atomic_riscv64.go @@ -18,6 +18,9 @@ func Xadduintptr(ptr *uintptr, delta uintptr) uintptr //go:noescape func Xchg(ptr *uint32, new uint32) uint32 +//go:noescape +func Xchg8(ptr *uint8, new uint8) uint8 + //go:noescape func Xchg64(ptr *uint64, new uint64) uint64 diff --git a/src/internal/runtime/atomic/atomic_riscv64.s b/src/internal/runtime/atomic/atomic_riscv64.s index bf6bd35ed73710..f68a607a8cf594 100644 --- a/src/internal/runtime/atomic/atomic_riscv64.s +++ b/src/internal/runtime/atomic/atomic_riscv64.s @@ -32,12 +32,12 @@ // func Cas(ptr *uint64, old, new uint64) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// } else { -// return 0; -// } +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } TEXT ·Cas(SB), NOSPLIT, $0-17 MOV ptr+0(FP), A0 MOVW old+8(FP), A1 @@ -199,6 +199,27 @@ TEXT ·Xchg(SB), NOSPLIT, $0-20 MOVW A1, ret+16(FP) RET +// func Xchg8(ptr *uint8, new uint8) uint8 +TEXT ·Xchg8(SB), NOSPLIT, $0-17 + MOV ptr+0(FP), A0 + MOVBU new+8(FP), A1 + AND $3, A0, A2 + SLL $3, A2 + MOV $255, A4 + SLL A2, A4 + NOT A4 + AND $~3, A0 + SLL A2, A1 +xchg8_again: + LRW (A0), A5 + AND A4, A5, A3 + OR A1, A3 + SCW A3, (A0), A6 + BNEZ A6, xchg8_again + SRL A2, A5 + MOVB A5, ret+16(FP) + RET + // func Xchg64(ptr *uint64, new uint64) uint64 TEXT ·Xchg64(SB), NOSPLIT, $0-24 MOV ptr+0(FP), A0 diff --git a/src/internal/runtime/atomic/atomic_s390x.go b/src/internal/runtime/atomic/atomic_s390x.go index 68b4e160f9a865..bd5d867bec4e62 100644 --- a/src/internal/runtime/atomic/atomic_s390x.go +++ b/src/internal/runtime/atomic/atomic_s390x.go @@ -128,6 +128,11 @@ func Xadduintptr(ptr *uintptr, delta uintptr) uintptr //go:noescape func Xchg(ptr *uint32, new uint32) uint32 +//go:nosplit +func Xchg8(addr *uint8, v uint8) uint8 { + return goXchg8(addr, v) +} + //go:noescape func Xchg64(ptr *uint64, new uint64) uint64 diff --git a/src/internal/runtime/atomic/atomic_wasm.go b/src/internal/runtime/atomic/atomic_wasm.go index d1dcfec7adde9f..3f14da138c79cc 100644 --- a/src/internal/runtime/atomic/atomic_wasm.go +++ b/src/internal/runtime/atomic/atomic_wasm.go @@ -114,6 +114,11 @@ func Xchg(ptr *uint32, new uint32) uint32 { return old } +//go:nosplit +func Xchg8(addr *uint8, v uint8) uint8 { + return goXchg8(addr, v) +} + //go:nosplit //go:noinline func Xchg64(ptr *uint64, new uint64) uint64 { diff --git a/src/internal/runtime/atomic/bench_test.go b/src/internal/runtime/atomic/bench_test.go index 798431cf7258c2..b5837c975961f9 100644 --- a/src/internal/runtime/atomic/bench_test.go +++ b/src/internal/runtime/atomic/bench_test.go @@ -43,6 +43,22 @@ func BenchmarkAtomicStore(b *testing.B) { } } +func BenchmarkAtomicLoad8(b *testing.B) { + var x uint8 + sink = &x + for i := 0; i < b.N; i++ { + atomic.Load8(&x) + } +} + +func BenchmarkAtomicStore8(b *testing.B) { + var x uint8 + sink = &x + for i := 0; i < b.N; i++ { + atomic.Store8(&x, 0) + } +} + func BenchmarkAnd8(b *testing.B) { var x [512]uint8 // give byte its own cache line sink = &x diff --git a/src/internal/runtime/atomic/sys_nonlinux_arm.s b/src/internal/runtime/atomic/sys_nonlinux_arm.s index b55bf908a26056..43a41d4edcf304 100644 --- a/src/internal/runtime/atomic/sys_nonlinux_arm.s +++ b/src/internal/runtime/atomic/sys_nonlinux_arm.s @@ -7,13 +7,14 @@ #include "textflag.h" // TODO(minux): this is only valid for ARMv6+ -// bool armcas(int32 *val, int32 old, int32 new) +// func armcas(ptr *int32, old int32, new int32) bool // Atomically: -// if(*val == old){ -// *val = new; -// return 1; -// }else -// return 0; +// if *ptr == old { +// *ptr = new +// return true +// } else { +// return false +// } TEXT ·Cas(SB),NOSPLIT,$0 JMP ·armcas(SB) diff --git a/src/internal/runtime/atomic/xchg8.go b/src/internal/runtime/atomic/xchg8.go new file mode 100644 index 00000000000000..4fdea9a3d6c81c --- /dev/null +++ b/src/internal/runtime/atomic/xchg8.go @@ -0,0 +1,32 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomic + +import ( + "internal/goarch" + "unsafe" +) + +//go:nosplit +func goXchg8(addr *uint8, v uint8) uint8 { + // Align down to 4 bytes and use 32-bit CAS. + addr32 := (*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(addr)) &^ 3)) + shift := (uintptr(unsafe.Pointer(addr)) & 3) + if goarch.BigEndian { + shift = shift ^ 3 + } + shift = shift * 8 + word := uint32(v) << shift + mask := uint32(0xFF) << shift + + for { + old := *addr32 // Read the old 32-bit value + // Clear the old 8 bits then insert the new value + if Cas(addr32, old, (old&^mask)|word) { + // Return the old 8-bit value + return uint8((old & mask) >> shift) + } + } +} diff --git a/src/internal/runtime/atomic/xchg8_test.go b/src/internal/runtime/atomic/xchg8_test.go new file mode 100644 index 00000000000000..1214c9cedcefbb --- /dev/null +++ b/src/internal/runtime/atomic/xchg8_test.go @@ -0,0 +1,57 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomic_test + +import ( + "internal/runtime/atomic" + "testing" +) + +func TestXchg8(t *testing.T) { + var a [16]uint8 + for i := range a { + next := uint8(i + 50) + a[i] = next + } + b := a + + // Compare behavior against non-atomic implementation. Expect the operation + // to work at any byte offset and to not clobber neighboring values. + for i := range a { + next := uint8(i + 100) + pa := atomic.Xchg8(&a[i], next) + pb := b[i] + b[i] = next + if pa != pb { + t.Errorf("atomic.Xchg8(a[%d]); %d != %d", i, pa, pb) + } + if a != b { + t.Errorf("after atomic.Xchg8(a[%d]); %d != %d", i, a, b) + } + if t.Failed() { + break + } + } +} + +func BenchmarkXchg8(b *testing.B) { + var x [512]uint8 // give byte its own cache line + sink = &x + for i := 0; i < b.N; i++ { + atomic.Xchg8(&x[255], uint8(i)) + } +} + +func BenchmarkXchg8Parallel(b *testing.B) { + var x [512]uint8 // give byte its own cache line + sink = &x + b.RunParallel(func(pb *testing.PB) { + i := uint8(0) + for pb.Next() { + atomic.Xchg8(&x[255], i) + i++ + } + }) +} diff --git a/src/internal/runtime/cgobench/bench_test.go b/src/internal/runtime/cgobench/bench_test.go new file mode 100644 index 00000000000000..b4d8efec5efcf6 --- /dev/null +++ b/src/internal/runtime/cgobench/bench_test.go @@ -0,0 +1,26 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build cgo + +package cgobench_test + +import ( + "internal/runtime/cgobench" + "testing" +) + +func BenchmarkCgoCall(b *testing.B) { + for b.Loop() { + cgobench.Empty() + } +} + +func BenchmarkCgoCallParallel(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + cgobench.Empty() + } + }) +} diff --git a/src/internal/runtime/cgobench/funcs.go b/src/internal/runtime/cgobench/funcs.go new file mode 100644 index 00000000000000..db685180a1b594 --- /dev/null +++ b/src/internal/runtime/cgobench/funcs.go @@ -0,0 +1,17 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build cgo + +package cgobench + +/* +static void empty() { +} +*/ +import "C" + +func Empty() { + C.empty() +} diff --git a/src/internal/runtime/cgroup/cgroup_linux.go b/src/internal/runtime/cgroup/cgroup_linux.go new file mode 100644 index 00000000000000..91815b4a1d0df8 --- /dev/null +++ b/src/internal/runtime/cgroup/cgroup_linux.go @@ -0,0 +1,710 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgroup + +import ( + "internal/bytealg" + "internal/runtime/strconv" + "internal/runtime/syscall/linux" +) + +var ( + ErrNoCgroup error = stringError("not in a cgroup") + + errMalformedFile error = stringError("malformed file") +) + +const _PATH_MAX = 4096 + +const ( + // Required amount of scratch space for CPULimit. + // + // TODO(prattmic): This is shockingly large (~70KiB) due to the (very + // unlikely) combination of extremely long paths consisting mostly + // escaped characters. The scratch buffer ends up in .bss in package + // runtime, so it doesn't contribute to binary size and generally won't + // be faulted in, but it would still be nice to shrink this. A more + // complex parser that did not need to keep entire lines in memory + // could get away with much less. Alternatively, we could do a one-off + // mmap allocation for this buffer, which is only mapped larger if we + // actually need the extra space. + ScratchSize = PathSize + ParseSize + + // Required space to store a path of the cgroup in the filesystem. + PathSize = _PATH_MAX + + // /proc/self/mountinfo path escape sequences are 4 characters long, so + // a path consisting entirely of escaped characters could be 4 times + // larger. + escapedPathMax = 4 * _PATH_MAX + + // Required space to parse /proc/self/mountinfo and /proc/self/cgroup. + // See findCPUMount and findCPURelativePath. + ParseSize = 4 * escapedPathMax +) + +// Include explicit NUL to be sure we include it in the slice. +const ( + v2MaxFile = "/cpu.max\x00" + v1QuotaFile = "/cpu.cfs_quota_us\x00" + v1PeriodFile = "/cpu.cfs_period_us\x00" +) + +// Version indicates the cgroup version. +type Version int + +const ( + VersionUnknown Version = iota + V1 + V2 +) + +// CPU owns the FDs required to read the CPU limit from a cgroup. +type CPU struct { + version Version + + // For cgroup v1, this is cpu.cfs_quota_us. + // For cgroup v2, this is cpu.max. + quotaFD int + + // For cgroup v1, this is cpu.cfs_period_us. + // For cgroup v2, this is unused. + periodFD int +} + +func (c CPU) Close() { + switch c.version { + case V1: + linux.Close(c.quotaFD) + linux.Close(c.periodFD) + case V2: + linux.Close(c.quotaFD) + default: + throw("impossible cgroup version") + } +} + +func checkBufferSize(s []byte, size int) { + if len(s) != size { + println("runtime: cgroup buffer length", len(s), "want", size) + throw("runtime: cgroup invalid buffer length") + } +} + +// OpenCPU returns a CPU for the CPU cgroup containing the current process, or +// ErrNoCgroup if the process is not in a CPU cgroup. +// +// scratch must have length ScratchSize. +func OpenCPU(scratch []byte) (CPU, error) { + checkBufferSize(scratch, ScratchSize) + + base := scratch[:PathSize] + scratch2 := scratch[PathSize:] + + n, version, err := FindCPU(base, scratch2) + if err != nil { + return CPU{}, err + } + + switch version { + case 1: + n2 := copy(base[n:], v1QuotaFile) + path := base[:n+n2] + quotaFD, errno := linux.Open(&path[0], linux.O_RDONLY|linux.O_CLOEXEC, 0) + if errno != 0 { + // This may fail if this process was migrated out of + // the cgroup found by FindCPU and that cgroup has been + // deleted. + return CPU{}, errSyscallFailed + } + + n2 = copy(base[n:], v1PeriodFile) + path = base[:n+n2] + periodFD, errno := linux.Open(&path[0], linux.O_RDONLY|linux.O_CLOEXEC, 0) + if errno != 0 { + // This may fail if this process was migrated out of + // the cgroup found by FindCPU and that cgroup has been + // deleted. + return CPU{}, errSyscallFailed + } + + c := CPU{ + version: 1, + quotaFD: quotaFD, + periodFD: periodFD, + } + return c, nil + case 2: + n2 := copy(base[n:], v2MaxFile) + path := base[:n+n2] + maxFD, errno := linux.Open(&path[0], linux.O_RDONLY|linux.O_CLOEXEC, 0) + if errno != 0 { + // This may fail if this process was migrated out of + // the cgroup found by FindCPU and that cgroup has been + // deleted. + return CPU{}, errSyscallFailed + } + + c := CPU{ + version: 2, + quotaFD: maxFD, + periodFD: -1, + } + return c, nil + default: + throw("impossible cgroup version") + panic("unreachable") + } +} + +// Returns average CPU throughput limit from the cgroup, or ok false if there +// is no limit. +func ReadCPULimit(c CPU) (float64, bool, error) { + switch c.version { + case 1: + quota, err := readV1Number(c.quotaFD) + if err != nil { + return 0, false, errMalformedFile + } + + if quota < 0 { + // No limit. + return 0, false, nil + } + + period, err := readV1Number(c.periodFD) + if err != nil { + return 0, false, errMalformedFile + } + + return float64(quota) / float64(period), true, nil + case 2: + // quotaFD is the cpu.max FD. + return readV2Limit(c.quotaFD) + default: + throw("impossible cgroup version") + panic("unreachable") + } +} + +// Returns the value from the quota/period file. +func readV1Number(fd int) (int64, error) { + // The format of the file is "\n" where the value is in + // int64 microseconds and, if quota, may be -1 to indicate no limit. + // + // MaxInt64 requires 19 bytes to display in base 10, thus the + // conservative max size of this file is 19 + 1 (newline) = 20 bytes. + // We'll provide a bit more for good measure. + // + // Always read from the beginning of the file to get a fresh value. + var b [64]byte + n, errno := linux.Pread(fd, b[:], 0) + if errno != 0 { + return 0, errSyscallFailed + } + if n == len(b) { + return 0, errMalformedFile + } + + buf := b[:n] + return parseV1Number(buf) +} + +func parseV1Number(buf []byte) (int64, error) { + // Ignore trailing newline. + i := bytealg.IndexByte(buf, '\n') + if i < 0 { + return 0, errMalformedFile + } + buf = buf[:i] + + val, ok := strconv.Atoi64(string(buf)) + if !ok { + return 0, errMalformedFile + } + + return val, nil +} + +// Returns CPU throughput limit, or ok false if there is no limit. +func readV2Limit(fd int) (float64, bool, error) { + // The format of the file is " \n" where quota and + // period are microseconds and quota may be "max" to indicate no limit. + // + // Note that the kernel is inconsistent about whether the values are + // uint64 or int64: values are parsed as uint64 but printed as int64. + // See kernel/sched/core.c:cpu_max_{show,write}. + // + // In practice, the kernel limits the period to 1s (1000000us) (see + // max_cfs_quota_period), and the quota to (1<<44)us (see + // max_cfs_runtime), so these values can't get large enough for the + // distinction to matter. + // + // MaxInt64 requires 19 bytes to display in base 10, thus the + // conservative max size of this file is 19 + 19 + 1 (space) + 1 + // (newline) = 40 bytes. We'll provide a bit more for good measure. + // + // Always read from the beginning of the file to get a fresh value. + var b [64]byte + n, errno := linux.Pread(fd, b[:], 0) + if errno != 0 { + return 0, false, errSyscallFailed + } + if n == len(b) { + return 0, false, errMalformedFile + } + + buf := b[:n] + return parseV2Limit(buf) +} + +func parseV2Limit(buf []byte) (float64, bool, error) { + i := bytealg.IndexByte(buf, ' ') + if i < 0 { + return 0, false, errMalformedFile + } + + quotaStr := buf[:i] + if bytealg.Compare(quotaStr, []byte("max")) == 0 { + // No limit. + return 0, false, nil + } + + periodStr := buf[i+1:] + // Ignore trailing newline, if any. + i = bytealg.IndexByte(periodStr, '\n') + if i < 0 { + return 0, false, errMalformedFile + } + periodStr = periodStr[:i] + + quota, ok := strconv.Atoi64(string(quotaStr)) + if !ok { + return 0, false, errMalformedFile + } + + period, ok := strconv.Atoi64(string(periodStr)) + if !ok { + return 0, false, errMalformedFile + } + + return float64(quota) / float64(period), true, nil +} + +// FindCPU finds the path to the CPU cgroup that this process is a member of +// and places it in out. scratch is a scratch buffer for internal use. +// +// out must have length PathSize. scratch must have length ParseSize. +// +// Returns the number of bytes written to out and the cgroup version (1 or 2). +// +// Returns ErrNoCgroup if the process is not in a CPU cgroup. +func FindCPU(out []byte, scratch []byte) (int, Version, error) { + checkBufferSize(out, PathSize) + checkBufferSize(scratch, ParseSize) + + // The cgroup path is + . + // + // This is racy if our cgroup is changed while this runs. For example, + // initially there is only a cgroup v2 mount and we are not in a + // cgroup. After, there a cgroup v1 mount with a CPU controller and we + // are placed in a cgroup in this hierarchy. In that case, findCPUMount + // could pick the v2 mount, and findCPURelativePath could find the v2 + // relative path. + // + // In this case we'll later fail to read the cgroup files and fall back + // to assuming no cgroup. + + n, err := FindCPUMountPoint(out, scratch) + if err != nil { + return 0, 0, err + } + + // The relative path always starts with /, so we can directly append it + // to the mount point. + n2, version, err := FindCPURelativePath(out[n:], scratch) + if err != nil { + return 0, 0, err + } + n += n2 + + return n, version, nil +} + +// FindCPURelativePath finds the path to the CPU cgroup that this process is a member of +// relative to the root of the cgroup mount and places it in out. scratch is a +// scratch buffer for internal use. +// +// out must have length PathSize minus the size of the cgroup mount root (if +// known). scratch must have length ParseSize. +// +// Returns the number of bytes written to out and the cgroup version (1 or 2). +// +// Returns ErrNoCgroup if the process is not in a CPU cgroup. +func FindCPURelativePath(out []byte, scratch []byte) (int, Version, error) { + path := []byte("/proc/self/cgroup\x00") + fd, errno := linux.Open(&path[0], linux.O_RDONLY|linux.O_CLOEXEC, 0) + if errno == linux.ENOENT { + return 0, 0, ErrNoCgroup + } else if errno != 0 { + return 0, 0, errSyscallFailed + } + + // The relative path always starts with /, so we can directly append it + // to the mount point. + n, version, err := parseCPURelativePath(fd, linux.Read, out[:], scratch) + if err != nil { + linux.Close(fd) + return 0, 0, err + } + + linux.Close(fd) + return n, version, nil +} + +// Finds the path of the current process's CPU cgroup relative to the cgroup +// mount and writes it to out. +// +// Returns the number of bytes written and the cgroup version (1 or 2). +func parseCPURelativePath(fd int, read func(fd int, b []byte) (int, uintptr), out []byte, scratch []byte) (int, Version, error) { + // The format of each line is + // + // hierarchy-ID:controller-list:cgroup-path + // + // controller-list is comma-separated. + // See man 5 cgroup for more details. + // + // cgroup v2 has hierarchy-ID 0. If a v1 hierarchy contains "cpu", that + // is the CPU controller. Otherwise the v2 hierarchy (if any) is the + // CPU controller. + // + // hierarchy-ID and controller-list have relatively small maximum + // sizes, and the path can be up to _PATH_MAX, so we need a bit more + // than 1 _PATH_MAX of scratch space. + + l := newLineReader(fd, scratch, read) + + // Bytes written to out. + n := 0 + + for { + err := l.next() + if err == errIncompleteLine { + // Don't allow incomplete lines. While in theory the + // incomplete line may be for a controller we don't + // care about, in practice all lines should be of + // similar length, so we should just have a buffer big + // enough for any. + return 0, 0, err + } else if err == errEOF { + break + } else if err != nil { + return 0, 0, err + } + + line := l.line() + + // The format of each line is + // + // hierarchy-ID:controller-list:cgroup-path + // + // controller-list is comma-separated. + // See man 5 cgroup for more details. + i := bytealg.IndexByte(line, ':') + if i < 0 { + return 0, 0, errMalformedFile + } + + hierarchy := line[:i] + line = line[i+1:] + + i = bytealg.IndexByte(line, ':') + if i < 0 { + return 0, 0, errMalformedFile + } + + controllers := line[:i] + line = line[i+1:] + + path := line + + if string(hierarchy) == "0" { + // v2 hierarchy. + n = copy(out, path) + // Keep searching, we might find a v1 hierarchy with a + // CPU controller, which takes precedence. + } else { + // v1 hierarchy + if containsCPU(controllers) { + // Found a v1 CPU controller. This must be the + // only one, so we're done. + return copy(out, path), V1, nil + } + } + } + + if n == 0 { + // Found nothing. + return 0, 0, ErrNoCgroup + } + + // Must be v2, v1 returns above. + return n, V2, nil +} + +// Returns true if comma-separated list b contains "cpu". +func containsCPU(b []byte) bool { + for len(b) > 0 { + i := bytealg.IndexByte(b, ',') + if i < 0 { + // Neither cmd/compile nor gccgo allocates for these string conversions. + return string(b) == "cpu" + } + + curr := b[:i] + rest := b[i+1:] + + if string(curr) == "cpu" { + return true + } + + b = rest + } + + return false +} + +// FindCPUMountPoint finds the root of the CPU cgroup mount places it in out. +// scratch is a scratch buffer for internal use. +// +// out must have length PathSize. scratch must have length ParseSize. +// +// Returns the number of bytes written to out. +// +// Returns ErrNoCgroup if the process is not in a CPU cgroup. +func FindCPUMountPoint(out []byte, scratch []byte) (int, error) { + checkBufferSize(out, PathSize) + checkBufferSize(scratch, ParseSize) + + path := []byte("/proc/self/mountinfo\x00") + fd, errno := linux.Open(&path[0], linux.O_RDONLY|linux.O_CLOEXEC, 0) + if errno == linux.ENOENT { + return 0, ErrNoCgroup + } else if errno != 0 { + return 0, errSyscallFailed + } + + n, err := parseCPUMount(fd, linux.Read, out, scratch) + if err != nil { + linux.Close(fd) + return 0, err + } + linux.Close(fd) + + return n, nil +} + +// Returns the mount point for the cpu cgroup controller (v1 or v2) from +// /proc/self/mountinfo. +func parseCPUMount(fd int, read func(fd int, b []byte) (int, uintptr), out []byte, scratch []byte) (int, error) { + // The format of each line is: + // + // 36 35 98:0 /mnt1 /mnt2 rw,noatime master:1 - ext3 /dev/root rw,errors=continue + // (1)(2)(3) (4) (5) (6) (7) (8) (9) (10) (11) + // + // (1) mount ID: unique identifier of the mount (may be reused after umount) + // (2) parent ID: ID of parent (or of self for the top of the mount tree) + // (3) major:minor: value of st_dev for files on filesystem + // (4) root: root of the mount within the filesystem + // (5) mount point: mount point relative to the process's root + // (6) mount options: per mount options + // (7) optional fields: zero or more fields of the form "tag[:value]" + // (8) separator: marks the end of the optional fields + // (9) filesystem type: name of filesystem of the form "type[.subtype]" + // (10) mount source: filesystem specific information or "none" + // (11) super options: per super block options + // + // See man 5 proc_pid_mountinfo for more details. + // + // Note that emitted paths will not contain space, tab, newline, or + // carriage return. Those are escaped. See Linux show_mountinfo -> + // show_path. We must unescape before returning. + // + // We return the mount point (5) if the filesystem type (9) is cgroup2, + // or cgroup with "cpu" in the super options (11). + // + // (4), (5), and (10) are up to _PATH_MAX. The remaining fields have a + // small fixed maximum size, so 4*_PATH_MAX is plenty of scratch space. + // Note that non-cgroup mounts may have arbitrarily long (11), but we + // can skip those when parsing. + + l := newLineReader(fd, scratch, read) + + // Bytes written to out. + n := 0 + + for { + //incomplete := false + err := l.next() + if err == errIncompleteLine { + // An incomplete line is fine as long as it doesn't + // impede parsing the fields we need. It shouldn't be + // possible for any mount to use more than 3*PATH_MAX + // before (9) because there are two paths and all other + // earlier fields have bounded options. Only (11) has + // unbounded options. + } else if err == errEOF { + break + } else if err != nil { + return 0, err + } + + line := l.line() + + // Skip first four fields. + for range 4 { + i := bytealg.IndexByte(line, ' ') + if i < 0 { + return 0, errMalformedFile + } + line = line[i+1:] + } + + // (5) mount point: mount point relative to the process's root + i := bytealg.IndexByte(line, ' ') + if i < 0 { + return 0, errMalformedFile + } + mnt := line[:i] + line = line[i+1:] + + // Skip ahead past optional fields, delimited by " - ". + for { + i = bytealg.IndexByte(line, ' ') + if i < 0 { + return 0, errMalformedFile + } + if i+3 >= len(line) { + return 0, errMalformedFile + } + delim := line[i : i+3] + if string(delim) == " - " { + line = line[i+3:] + break + } + line = line[i+1:] + } + + // (9) filesystem type: name of filesystem of the form "type[.subtype]" + i = bytealg.IndexByte(line, ' ') + if i < 0 { + return 0, errMalformedFile + } + ftype := line[:i] + line = line[i+1:] + + if string(ftype) != "cgroup" && string(ftype) != "cgroup2" { + continue + } + + // As in findCPUPath, cgroup v1 with a CPU controller takes + // precendence over cgroup v2. + if string(ftype) == "cgroup2" { + // v2 hierarchy. + n, err = unescapePath(out, mnt) + if err != nil { + // Don't keep searching on error. The kernel + // should never produce broken escaping. + return n, err + } + // Keep searching, we might find a v1 hierarchy with a + // CPU controller, which takes precedence. + continue + } + + // (10) mount source: filesystem specific information or "none" + i = bytealg.IndexByte(line, ' ') + if i < 0 { + return 0, errMalformedFile + } + // Don't care about mount source. + line = line[i+1:] + + // (11) super options: per super block options + superOpt := line + + // v1 hierarchy + if containsCPU(superOpt) { + // Found a v1 CPU controller. This must be the + // only one, so we're done. + return unescapePath(out, mnt) + } + } + + if n == 0 { + // Found nothing. + return 0, ErrNoCgroup + } + + return n, nil +} + +var errInvalidEscape error = stringError("invalid path escape sequence") + +// unescapePath copies in to out, unescaping escape sequences generated by +// Linux's show_path. +// +// That is, '\', ' ', '\t', and '\n' are converted to octal escape sequences, +// like '\040' for space. +// +// out must be at least as large as in. +// +// Returns the number of bytes written to out. +// +// Also see escapePath in cgroup_linux_test.go. +func unescapePath(out []byte, in []byte) (int, error) { + // Not strictly necessary, but simplifies the implementation and will + // always hold in users. + if len(out) < len(in) { + throw("output too small") + } + + var outi, ini int + for ini < len(in) { + c := in[ini] + if c != '\\' { + out[outi] = c + outi++ + ini++ + continue + } + + // Start of escape sequence. + + // Escape sequence is always 4 characters: one slash and three + // digits. + if ini+3 >= len(in) { + return outi, errInvalidEscape + } + + var outc byte + for i := range 3 { + c := in[ini+1+i] + if c < '0' || c > '9' { + return outi, errInvalidEscape + } + + outc *= 8 + outc += c - '0' + } + + out[outi] = outc + outi++ + + ini += 4 + } + + return outi, nil +} diff --git a/src/internal/runtime/cgroup/cgroup_linux_test.go b/src/internal/runtime/cgroup/cgroup_linux_test.go new file mode 100644 index 00000000000000..d47fe4206749dc --- /dev/null +++ b/src/internal/runtime/cgroup/cgroup_linux_test.go @@ -0,0 +1,476 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgroup_test + +import ( + "fmt" + "internal/runtime/cgroup" + "io" + "strconv" + "strings" + "testing" +) + +const _PATH_MAX = 4096 + +func TestParseV1Number(t *testing.T) { + tests := []struct { + name string + contents string + want int64 + wantErr bool + }{ + { + name: "disabled", + contents: "-1\n", + want: -1, + }, + { + name: "500000", + contents: "500000\n", + want: 500000, + }, + { + name: "MaxInt64", + contents: "9223372036854775807\n", + want: 9223372036854775807, + }, + { + name: "missing-newline", + contents: "500000", + wantErr: true, + }, + { + name: "not-a-number", + contents: "123max\n", + wantErr: true, + }, + { + name: "v2", + contents: "1000 5000\n", + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := cgroup.ParseV1Number([]byte(tc.contents)) + if tc.wantErr { + if err == nil { + t.Fatalf("parseV1Number got err nil want non-nil") + } + return + } + if err != nil { + t.Fatalf("parseV1Number got err %v want nil", err) + } + + if got != tc.want { + t.Errorf("parseV1Number got %d want %d", got, tc.want) + } + }) + } +} + +func TestParseV2Limit(t *testing.T) { + tests := []struct { + name string + contents string + want float64 + wantOK bool + wantErr bool + }{ + { + name: "disabled", + contents: "max 100000\n", + wantOK: false, + }, + { + name: "5", + contents: "500000 100000\n", + want: 5, + wantOK: true, + }, + { + name: "0.5", + contents: "50000 100000\n", + want: 0.5, + wantOK: true, + }, + { + name: "2.5", + contents: "250000 100000\n", + want: 2.5, + wantOK: true, + }, + { + name: "MaxInt64", + contents: "9223372036854775807 9223372036854775807\n", + want: 1, + wantOK: true, + }, + { + name: "missing-newline", + contents: "500000 100000", + wantErr: true, + }, + { + name: "v1", + contents: "500000\n", + wantErr: true, + }, + { + name: "quota-not-a-number", + contents: "500000us 100000\n", + wantErr: true, + }, + { + name: "period-not-a-number", + contents: "500000 100000us\n", + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, gotOK, err := cgroup.ParseV2Limit([]byte(tc.contents)) + if tc.wantErr { + if err == nil { + t.Fatalf("parseV1Limit got err nil want non-nil") + } + return + } + if err != nil { + t.Fatalf("parseV2Limit got err %v want nil", err) + } + + if gotOK != tc.wantOK { + t.Errorf("parseV2Limit got ok %v want %v", gotOK, tc.wantOK) + } + + if tc.wantOK && got != tc.want { + t.Errorf("parseV2Limit got %f want %f", got, tc.want) + } + }) + } +} + +func TestParseCPURelativePath(t *testing.T) { + tests := []struct { + name string + contents string + want string + wantVer cgroup.Version + wantErr bool + }{ + { + name: "empty", + contents: "", + wantErr: true, + }, + { + name: "v1", + contents: `2:cpu,cpuacct:/a/b/cpu +1:blkio:/a/b/blkio +`, + want: "/a/b/cpu", + wantVer: cgroup.V1, + }, + { + name: "v2", + contents: "0::/a/b/c\n", + want: "/a/b/c", + wantVer: cgroup.V2, + }, + { + name: "mixed", + contents: `2:cpu,cpuacct:/a/b/cpu +1:blkio:/a/b/blkio +0::/a/b/v2 +`, + want: "/a/b/cpu", + wantVer: cgroup.V1, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + r := strings.NewReader(tc.contents) + read := func(fd int, b []byte) (int, uintptr) { + n, err := r.Read(b) + if err != nil && err != io.EOF { + const dummyErrno = 42 + return n, dummyErrno + } + return n, 0 + } + + var got [cgroup.PathSize]byte + var scratch [cgroup.ParseSize]byte + n, gotVer, err := cgroup.ParseCPURelativePath(0, read, got[:], scratch[:]) + if (err != nil) != tc.wantErr { + t.Fatalf("parseCPURelativePath got err %v want %v", err, tc.wantErr) + } + + if gotVer != tc.wantVer { + t.Errorf("parseCPURelativePath got cgroup version %d want %d", gotVer, tc.wantVer) + } + + if string(got[:n]) != tc.want { + t.Errorf("parseCPURelativePath got %q want %q", string(got[:n]), tc.want) + } + }) + } +} + +func TestContainsCPU(t *testing.T) { + tests := []struct { + in string + want bool + }{ + { + in: "", + want: false, + }, + { + in: ",", + want: false, + }, + { + in: "cpu", + want: true, + }, + { + in: "memory,cpu", + want: true, + }, + { + in: "cpu,memory", + want: true, + }, + { + in: "memory,cpu,block", + want: true, + }, + { + in: "memory,cpuacct,block", + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.in, func(t *testing.T) { + got := cgroup.ContainsCPU([]byte(tc.in)) + if got != tc.want { + t.Errorf("containsCPU(%q) got %v want %v", tc.in, got, tc.want) + } + }) + } +} + +func TestParseCPUMount(t *testing.T) { + // Used for v2-longline. We want an overlayfs mount to have an option + // so long that the entire line can't possibly fit in the scratch + // buffer. + const lowerPath = "/so/many/overlay/layers" + overlayLongLowerDir := lowerPath + for i := 0; len(overlayLongLowerDir) < cgroup.ScratchSize; i++ { + overlayLongLowerDir += fmt.Sprintf(":%s%d", lowerPath, i) + } + + tests := []struct { + name string + contents string + want string + wantErr bool + }{ + { + name: "empty", + contents: "", + wantErr: true, + }, + { + name: "v1", + contents: `22 1 8:1 / / rw,relatime - ext4 /dev/root rw +20 22 0:19 / /proc rw,nosuid,nodev,noexec - proc proc rw +21 22 0:20 / /sys rw,nosuid,nodev,noexec - sysfs sysfs rw +49 22 0:37 / /sys/fs/cgroup/memory rw - cgroup cgroup rw,memory +54 22 0:38 / /sys/fs/cgroup/io rw - cgroup cgroup rw,io +56 22 0:40 / /sys/fs/cgroup/cpu rw - cgroup cgroup rw,cpu,cpuacct +58 22 0:42 / /sys/fs/cgroup/net rw - cgroup cgroup rw,net +59 22 0:43 / /sys/fs/cgroup/cpuset rw - cgroup cgroup rw,cpuset +`, + want: "/sys/fs/cgroup/cpu", + }, + { + name: "v2", + contents: `22 1 8:1 / / rw,relatime - ext4 /dev/root rw +20 22 0:19 / /proc rw,nosuid,nodev,noexec - proc proc rw +21 22 0:20 / /sys rw,nosuid,nodev,noexec - sysfs sysfs rw +25 21 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec - cgroup2 cgroup2 rw +`, + want: "/sys/fs/cgroup", + }, + { + name: "mixed", + contents: `22 1 8:1 / / rw,relatime - ext4 /dev/root rw +20 22 0:19 / /proc rw,nosuid,nodev,noexec - proc proc rw +21 22 0:20 / /sys rw,nosuid,nodev,noexec - sysfs sysfs rw +25 21 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec - cgroup2 cgroup2 rw +49 22 0:37 / /sys/fs/cgroup/memory rw - cgroup cgroup rw,memory +54 22 0:38 / /sys/fs/cgroup/io rw - cgroup cgroup rw,io +56 22 0:40 / /sys/fs/cgroup/cpu rw - cgroup cgroup rw,cpu,cpuacct +58 22 0:42 / /sys/fs/cgroup/net rw - cgroup cgroup rw,net +59 22 0:43 / /sys/fs/cgroup/cpuset rw - cgroup cgroup rw,cpuset +`, + want: "/sys/fs/cgroup/cpu", + }, + { + name: "v2-escaped", + contents: `22 1 8:1 / / rw,relatime - ext4 /dev/root rw +20 22 0:19 / /proc rw,nosuid,nodev,noexec - proc proc rw +21 22 0:20 / /sys rw,nosuid,nodev,noexec - sysfs sysfs rw +25 21 0:22 / /sys/fs/cgroup/tab\011tab rw,nosuid,nodev,noexec - cgroup2 cgroup2 rw +`, + want: `/sys/fs/cgroup/tab tab`, + }, + { + // Overly long line on a different mount doesn't matter. + name: "v2-longline", + contents: `22 1 8:1 / / rw,relatime - ext4 /dev/root rw +20 22 0:19 / /proc rw,nosuid,nodev,noexec - proc proc rw +21 22 0:20 / /sys rw,nosuid,nodev,noexec - sysfs sysfs rw +262 31 0:72 / /tmp/overlay2/0143e063b02f4801de9c847ad1c5ddc21fd2ead00653064d0c72ea967b248870/merged rw,relatime shared:729 - overlay overlay rw,lowerdir=` + overlayLongLowerDir + `,upperdir=/tmp/diff,workdir=/tmp/work +25 21 0:22 / /sys/fs/cgroup rw,nosuid,nodev,noexec - cgroup2 cgroup2 rw +`, + want: "/sys/fs/cgroup", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + r := strings.NewReader(tc.contents) + read := func(fd int, b []byte) (int, uintptr) { + n, err := r.Read(b) + if err != nil && err != io.EOF { + const dummyErrno = 42 + return n, dummyErrno + } + return n, 0 + } + + var got [cgroup.PathSize]byte + var scratch [cgroup.ParseSize]byte + n, err := cgroup.ParseCPUMount(0, read, got[:], scratch[:]) + if (err != nil) != tc.wantErr { + t.Fatalf("parseCPUMount got err %v want %v", err, tc.wantErr) + } + + if string(got[:n]) != tc.want { + t.Errorf("parseCPUMount got %q want %q", string(got[:n]), tc.want) + } + }) + } +} + +// escapePath performs escaping equivalent to Linux's show_path. +// +// That is, '\', ' ', '\t', and '\n' are converted to octal escape sequences, +// like '\040' for space. +func escapePath(s string) string { + out := make([]rune, 0, len(s)) + for _, c := range s { + switch c { + case '\\', ' ', '\t', '\n': + out = append(out, '\\') + cs := strconv.FormatInt(int64(c), 8) + if len(cs) <= 2 { + out = append(out, '0') + } + if len(cs) <= 1 { + out = append(out, '0') + } + for _, csc := range cs { + out = append(out, csc) + } + default: + out = append(out, c) + } + } + return string(out) +} + +func TestEscapePath(t *testing.T) { + tests := []struct { + name string + unescaped string + escaped string + }{ + { + name: "boring", + unescaped: `/a/b/c`, + escaped: `/a/b/c`, + }, + { + name: "space", + unescaped: `/a/b b/c`, + escaped: `/a/b\040b/c`, + }, + { + name: "tab", + unescaped: `/a/b b/c`, + escaped: `/a/b\011b/c`, + }, + { + name: "newline", + unescaped: `/a/b +b/c`, + escaped: `/a/b\012b/c`, + }, + { + name: "slash", + unescaped: `/a/b\b/c`, + escaped: `/a/b\134b/c`, + }, + { + name: "beginning", + unescaped: `\b/c`, + escaped: `\134b/c`, + }, + { + name: "ending", + unescaped: `/a/\`, + escaped: `/a/\134`, + }, + } + + t.Run("escapePath", func(t *testing.T) { + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := escapePath(tc.unescaped) + if got != tc.escaped { + t.Errorf("escapePath got %q want %q", got, tc.escaped) + } + }) + } + }) + + t.Run("unescapePath", func(t *testing.T) { + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + in := []byte(tc.escaped) + out := make([]byte, len(in)) + n, err := cgroup.UnescapePath(out, in) + if err != nil { + t.Errorf("unescapePath got err %v want nil", err) + } + got := string(out[:n]) + if got != tc.unescaped { + t.Errorf("unescapePath got %q want %q", got, tc.escaped) + } + }) + } + }) +} diff --git a/src/internal/runtime/cgroup/export_linux_test.go b/src/internal/runtime/cgroup/export_linux_test.go new file mode 100644 index 00000000000000..653fcd1b2fdc36 --- /dev/null +++ b/src/internal/runtime/cgroup/export_linux_test.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgroup + +var ContainsCPU = containsCPU + +var ParseV1Number = parseV1Number +var ParseV2Limit = parseV2Limit + +var ParseCPURelativePath = parseCPURelativePath +var ParseCPUMount = parseCPUMount + +var UnescapePath = unescapePath diff --git a/src/internal/runtime/cgroup/export_test.go b/src/internal/runtime/cgroup/export_test.go new file mode 100644 index 00000000000000..200e5aee121094 --- /dev/null +++ b/src/internal/runtime/cgroup/export_test.go @@ -0,0 +1,24 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgroup + +type LineReader = lineReader + +func (l *LineReader) Next() error { + return l.next() +} + +func (l *LineReader) Line() []byte { + return l.line() +} + +func NewLineReader(fd int, scratch []byte, read func(fd int, b []byte) (int, uintptr)) *LineReader { + return newLineReader(fd, scratch, read) +} + +var ( + ErrEOF = errEOF + ErrIncompleteLine = errIncompleteLine +) diff --git a/src/internal/runtime/cgroup/line_reader.go b/src/internal/runtime/cgroup/line_reader.go new file mode 100644 index 00000000000000..9a7213327c04ec --- /dev/null +++ b/src/internal/runtime/cgroup/line_reader.go @@ -0,0 +1,179 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgroup + +import ( + "internal/bytealg" +) + +// stringError is a trival implementation of error, equivalent to errors.New, +// which cannot be imported from a runtime package. +type stringError string + +func (e stringError) Error() string { + return string(e) +} + +// All errors are explicit converted to type error in global initialization to +// ensure that the linker allocates a static interface value. This is necessary +// because these errors may be used before the allocator is available. + +var ( + // The entire line did not fit into the scratch buffer. + errIncompleteLine error = stringError("incomplete line") + + // A system call failed. + errSyscallFailed error = stringError("syscall failed") + + // Reached EOF. + errEOF error = stringError("end of file") +) + +// lineReader reads line-by-line using only a single fixed scratch buffer. +// +// When a single line is too long for the scratch buffer, the remainder of the +// line will be skipped. +type lineReader struct { + read func(fd int, b []byte) (int, uintptr) + fd int + scratch []byte + + n int // bytes of scratch in use. + newline int // index of the first newline in scratch. + + eof bool // read reached EOF. +} + +// newLineReader returns a lineReader which reads lines from fd. +// +// fd is the file descriptor to read from. +// +// scratch is the scratch buffer to read into. Note that len(scratch) is the +// longest line that can be read. Lines longer than len(scratch) will have the +// remainder of the line skipped. See next for more details. +// +// read is the function used to read more bytes from fd. This is usually +// internal/runtime/syscall/linux.Read. Note that this follows syscall semantics (not +// io.Reader), so EOF is indicated with n=0, errno=0. +func newLineReader(fd int, scratch []byte, read func(fd int, b []byte) (n int, errno uintptr)) *lineReader { + return &lineReader{ + read: read, + fd: fd, + scratch: scratch, + n: 0, + newline: -1, + } +} + +// next advances to the next line. +// +// May return errIncompleteLine if the scratch buffer is too small to hold the +// entire line, in which case [r.line] will return the beginning of the line. A +// subsequent call to next will skip the remainder of the incomplete line. +// +// N.B. this behavior is important for /proc/self/mountinfo. Some lines +// (mounts), such as overlayfs, may be extremely long due to long super-block +// options, but we don't care about those. The mount type will appear early in +// the line. +// +// Returns errEOF when there are no more lines. +func (r *lineReader) next() error { + // Three cases: + // + // 1. First call, no data read. + // 2. Previous call had a complete line. Drop it and look for the end + // of the next line. + // 3. Previous call had an incomplete line. Find the end of that line + // (start of the next line), and the end of the next line. + + prevComplete := r.newline >= 0 + firstCall := r.n == 0 + + for { + if prevComplete { + // Drop the previous line. + copy(r.scratch, r.scratch[r.newline+1:r.n]) + r.n -= r.newline + 1 + + r.newline = bytealg.IndexByte(r.scratch[:r.n], '\n') + if r.newline >= 0 { + // We have another line already in scratch. Done. + return nil + } + } + + // No newline available. + + if !prevComplete { + // If the previous line was incomplete, we are + // searching for the end of that line and have no need + // for any buffered data. + r.n = 0 + } + + n, errno := r.read(r.fd, r.scratch[r.n:len(r.scratch)]) + if errno != 0 { + return errSyscallFailed + } + r.n += n + + if r.n == 0 { + // Nothing left. + // + // N.B. we can't immediately return EOF when read + // returns 0 as we may still need to return an + // incomplete line. + return errEOF + } + + r.newline = bytealg.IndexByte(r.scratch[:r.n], '\n') + if prevComplete || firstCall { + // Already have the start of the line, just need to find the end. + + if r.newline < 0 { + // We filled the entire buffer or hit EOF, but + // still no newline. + return errIncompleteLine + } + + // Found the end of the line. Done. + return nil + } else { + // Don't have the start of the line. We are currently + // looking for the end of the previous line. + + if r.newline < 0 { + // Not there yet. + if n == 0 { + // No more to read. + return errEOF + } + continue + } + + // Found the end of the previous line. The next + // iteration will drop the remainder of the previous + // line and look for the next line. + prevComplete = true + } + } +} + +// line returns a view of the current line, excluding the trailing newline. +// +// If [r.next] returned errIncompleteLine, then this returns only the beginning +// of the line. +// +// Preconditions: [r.next] is called prior to the first call to line. +// +// Postconditions: The caller must not keep a reference to the returned slice. +func (r *lineReader) line() []byte { + if r.newline < 0 { + // Incomplete line + return r.scratch[:r.n] + } + // Complete line. + return r.scratch[:r.newline] +} diff --git a/src/internal/runtime/cgroup/line_reader_test.go b/src/internal/runtime/cgroup/line_reader_test.go new file mode 100644 index 00000000000000..ceef1b5b4cae46 --- /dev/null +++ b/src/internal/runtime/cgroup/line_reader_test.go @@ -0,0 +1,170 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgroup_test + +import ( + "internal/runtime/cgroup" + "io" + "strings" + "testing" +) + +func TestLineReader(t *testing.T) { + type nextLine struct { + line string + incomplete bool // next call before this line should return incomplete + } + complete := func(s string) nextLine { + return nextLine{line: s} + } + incomplete := func(s string) nextLine { + return nextLine{line: s, incomplete: true} + } + + const scratchSize = 8 + + tests := []struct { + name string + contents string + want []nextLine + }{ + { + name: "empty", + contents: "", + }, + { + name: "single", + contents: "1234\n", + want: []nextLine{ + complete("1234"), + }, + }, + { + name: "single-incomplete", + contents: "1234", + want: []nextLine{ + incomplete("1234"), + }, + }, + { + name: "single-exact", + contents: "1234567\n", + want: []nextLine{ + complete("1234567"), + }, + }, + { + name: "single-exact-incomplete", + contents: "12345678", + want: []nextLine{ + incomplete("12345678"), + }, + }, + { + name: "multi", + contents: `1234 +5678 +`, + want: []nextLine{ + complete("1234"), + complete("5678"), + }, + }, + { + name: "multi-short", + contents: `12 +34 +56 +78 +`, + want: []nextLine{ + complete("12"), + complete("34"), + complete("56"), + complete("78"), + }, + }, + { + name: "multi-notrailingnewline", + contents: `1234 +5678`, + want: []nextLine{ + complete("1234"), + incomplete("5678"), + }, + }, + { + name: "middle-too-long", + contents: `1234 +1234567890 +5678 +`, + want: []nextLine{ + complete("1234"), + incomplete("12345678"), + complete("5678"), + }, + }, + { + // Multiple reads required to find newline. + name: "middle-way-too-long", + contents: `1234 +12345678900000000000000000000000000000000000000000000000000 +5678 +`, + want: []nextLine{ + complete("1234"), + incomplete("12345678"), + complete("5678"), + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + r := strings.NewReader(tc.contents) + read := func(fd int, b []byte) (int, uintptr) { + n, err := r.Read(b) + if err != nil && err != io.EOF { + const dummyErrno = 42 + return n, dummyErrno + } + return n, 0 + } + + var scratch [scratchSize]byte + l := cgroup.NewLineReader(0, scratch[:], read) + + var got []nextLine + for { + err := l.Next() + if err == cgroup.ErrEOF { + break + } else if err == cgroup.ErrIncompleteLine { + got = append(got, incomplete(string(l.Line()))) + } else if err != nil { + t.Fatalf("next got err %v", err) + } else { + got = append(got, complete(string(l.Line()))) + } + } + + if len(got) != len(tc.want) { + t.Logf("got lines %+v", got) + t.Logf("want lines %+v", tc.want) + t.Fatalf("lineReader got %d lines, want %d", len(got), len(tc.want)) + } + + for i := range got { + if got[i].line != tc.want[i].line { + t.Errorf("line %d got %q want %q", i, got[i].line, tc.want[i].line) + } + if got[i].incomplete != tc.want[i].incomplete { + t.Errorf("line %d got incomplete %v want %v", i, got[i].incomplete, tc.want[i].incomplete) + } + } + }) + } +} diff --git a/src/internal/runtime/cgroup/runtime.go b/src/internal/runtime/cgroup/runtime.go new file mode 100644 index 00000000000000..39c9295b076a1f --- /dev/null +++ b/src/internal/runtime/cgroup/runtime.go @@ -0,0 +1,14 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgroup + +import ( + _ "unsafe" // for linkname +) + +// Functions below pushed from runtime. + +//go:linkname throw +func throw(s string) diff --git a/src/internal/runtime/exithook/hooks.go b/src/internal/runtime/exithook/hooks.go index eb8aa1ce0a5c01..8dcfb9ed3c06c5 100644 --- a/src/internal/runtime/exithook/hooks.go +++ b/src/internal/runtime/exithook/hooks.go @@ -8,7 +8,7 @@ // from a safe context (e.g. not an error/panic path or signal // handler, preemption enabled, allocation allowed, write barriers // allowed, etc), and that the exit function F will be invoked under -// similar circumstances. That is the say, we are expecting that F +// similar circumstances. That is to say, we are expecting that F // uses normal / high-level Go code as opposed to one of the more // restricted dialects used for the trickier parts of the runtime. package exithook diff --git a/src/internal/runtime/gc/internal/gen/gen.go b/src/internal/runtime/gc/internal/gen/gen.go new file mode 100644 index 00000000000000..0758f9b242b3a5 --- /dev/null +++ b/src/internal/runtime/gc/internal/gen/gen.go @@ -0,0 +1,537 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gen + +import ( + "container/heap" + "encoding/binary" + "fmt" + "hash/maphash" + "io" + "log" + "os" + "reflect" + "strings" +) + +const logCompile = true + +func fatalf(f string, args ...any) { + panic(fmt.Sprintf(f, args...)) +} + +type File struct { + w io.Writer + funcs []*Func + consts []fileConst +} + +func NewFile(w io.Writer) *File { + return &File{w: w} +} + +func (f *File) AddFunc(fn *Func) { + f.funcs = append(f.funcs, fn) +} + +type fileConst struct { + name string + data any +} + +func (f *File) AddConst(name string, data any) { + // TODO: It would be nice if this were unified with "const" ops, but the + // reason I added this was for []*Func consts, which would take an overhaul + // to represent in "const" ops. + f.consts = append(f.consts, fileConst{name, data}) +} + +type Func struct { + name string + nArgs int + idGen int + ops []*op +} + +func NewFunc(name string) *Func { + fn := &Func{name: name} + return fn +} + +// attach adds x to fn's op list. If x has any unattached arguments, this adds +// those first (recursively). +func (fn *Func) attach(x *op) { + // Make sure the arguments are attached to the function. + for _, arg := range x.args { + argFn := arg.fn + if argFn == nil { + fn.attach(arg) + } else if argFn != fn { + panic("ops from different functions") + } + } + + x.fn = fn + x.id = fn.idGen + fn.idGen++ + fn.ops = append(fn.ops, x) +} + +func Arg[W wrap[T], T Word](fn *Func) T { + loc := locReg{cls: regClassGP, reg: fn.nArgs} + fn.nArgs++ + var x W + o := &op{op: "arg", kind: x.kind(), c: loc} + fn.attach(o) + return x.wrap(o) +} + +func Return(results ...Value) { + args := make([]*op, len(results)) + for i, res := range results { + args[i] = res.getOp() + } + var x void + x.initOp(&op{op: "return", kind: voidKind, args: args}) +} + +type op struct { + op string + kind *kind + args []*op + + id int + fn *Func + + // c depends on "op". + // + // arg locReg - The register containing the argument value + // const any - The constant value + // deref int - Byte offset from args[0] + c any + name string +} + +func (o *op) String() string { + return fmt.Sprintf("v%02d", o.id) +} + +func imm(val any) *op { + return &op{op: "imm", c: val} +} + +func (o *op) equalNoName(o2 *op) bool { + if o.op != o2.op || o.c != o2.c || len(o.args) != len(o2.args) { + return false + } + for i, arg := range o.args { + if o2.args[i] != arg { + return false + } + } + return true +} + +func (o *op) write(w io.Writer) { + fmt.Fprintf(w, "v%02d = %s", o.id, o.op) + for _, arg := range o.args { + fmt.Fprintf(w, " v%02d", arg.id) + } + if o.c != nil { + fmt.Fprintf(w, " %v", o.c) + } + if o.name != "" { + fmt.Fprintf(w, " %q", o.name) + } + if o.kind != nil { + fmt.Fprintf(w, " [%s]", o.kind.typ) + } + fmt.Fprintf(w, "\n") +} + +func (fn *Func) write(w io.Writer) { + fmt.Fprintf(w, "FUNC %s\n", fn.name) + for _, op := range fn.ops { + op.write(w) + } +} + +func (f *File) Compile() { + // TODO: CSE constants across the whole file + + fmt.Fprintf(f.w, `#include "go_asm.h" +#include "textflag.h" + +`) + + for _, c := range f.consts { + f.emitConst(c.name, c.data) + } + + trace := func(fn *Func, step string) { + if !logCompile { + return + } + log.Printf("## Compiling %s: %s", fn.name, step) + fn.write(os.Stderr) + } + + for _, fn := range f.funcs { + trace(fn, "initial") + + for { + if fn.cse() { + trace(fn, "post cse") + continue + } + if fn.deadcode() { + trace(fn, "post deadcode") + continue + } + break + } + fn.addLoads() + trace(fn, "post addLoads") + + // Assigning locations requires ops to be in dependency order. + fn.schedule() + trace(fn, "post schedule") + + locs := fn.assignLocs() + + fn.emit(f, locs) + } +} + +// cse performs common subexpression elimination. +func (fn *Func) cse() bool { + // Compute structural hashes + hashes := make(map[*op]uint64) + var h maphash.Hash + var bbuf [8]byte + for _, op := range fn.ops { + // We ignore the name for canonicalization. + h.Reset() + h.WriteString(op.op) + // TODO: Ideally we would hash o1.c, but we don't have a good way to do that. + for _, arg := range op.args { + if _, ok := hashes[arg]; !ok { + panic("ops not in dependency order") + } + binary.NativeEndian.PutUint64(bbuf[:], hashes[arg]) + h.Write(bbuf[:]) + } + hashes[op] = h.Sum64() + } + + canon := make(map[uint64][]*op) + lookup := func(o *op) *op { + hash := hashes[o] + for _, o2 := range canon[hash] { + if o.equalNoName(o2) { + return o2 + } + } + canon[hash] = append(canon[hash], o) + return o + } + + // Canonicalize ops. + dirty := false + for _, op := range fn.ops { + for i, arg := range op.args { + newArg := lookup(arg) + if arg != newArg { + dirty = true + op.args[i] = newArg + } + } + } + return dirty +} + +// deadcode eliminates unused ops. +func (fn *Func) deadcode() bool { + marks := make(map[*op]bool) + var mark func(o *op) + mark = func(o *op) { + if marks[o] { + return + } + marks[o] = true + for _, arg := range o.args { + mark(arg) + } + } + // Mark operations that have a side-effect. + for _, op := range fn.ops { + switch op.op { + case "return": + mark(op) + } + } + // Discard unmarked operations + if len(marks) == len(fn.ops) { + return false + } + newOps := make([]*op, 0, len(marks)) + for _, op := range fn.ops { + if marks[op] { + newOps = append(newOps, op) + } + } + fn.ops = newOps + return true +} + +// canMem is a map from operation to a bitmap of which arguments can use a +// direct memory reference. +var canMem = map[string]uint64{ + "VPERMB": 1 << 0, + "VPERMI2B": 1 << 0, + "VPERMT2B": 1 << 0, + "VGF2P8AFFINEQB": 1 << 0, + "VPORQ": 1 << 0, + "VPSUBQ": 1 << 0, + "VPSHUFBITQMB": 1 << 0, +} + +// addLoads inserts load ops for ops that can't take memory inputs directly. +func (fn *Func) addLoads() { + // A lot of operations can directly take memory locations. If there's only a + // single reference to a deref operation, and the operation can do the deref + // itself, eliminate the deref. If there's more than one reference, then we + // leave the load so we can share the value in the register. + nRefs := fn.opRefs() + loads := make(map[*op]*op) // deref -> load + for _, o := range fn.ops { + canMask := canMem[o.op] + for i, arg := range o.args { + // TODO: Many AVX-512 operations that support memory operands also + // support a ".BCST" suffix that performs a broadcasting memory + // load. If the const can be broadcast and all uses support + // broadcast load, it would be nice to use .BCST. I'm not sure if + // that belongs in this pass or a different one. + if arg.op == "deref" || arg.op == "const" { + // These produce memory locations. + if canMask&(1< 1 { + // This argument needs to be loaded into a register. + load, ok := loads[arg] + if !ok { + load = makeLoad(arg) + fn.attach(load) + loads[arg] = load + } + o.args[i] = load + } + } + } + } +} + +func (fn *Func) opRefs() map[*op]int { + refs := make(map[*op]int) + for _, o1 := range fn.ops { + for _, arg := range o1.args { + refs[arg]++ + } + } + return refs +} + +func makeLoad(deref *op) *op { + var inst string + switch deref.kind.reg { + default: + fatalf("don't know how to load %v", deref.kind.reg) + case regClassGP: + inst = "MOVQ" + case regClassZ: + inst = "VMOVDQU64" + } + // The load references deref rather than deref.args[0] because when we + // assign locations, the deref op gets the memory location to load from, + // while its argument has some other location (like a register). Also, the + // offset to deref is attached to the deref op. + return &op{op: inst, kind: deref.kind, args: []*op{deref}} +} + +type opHeap []*op + +func (h opHeap) Len() int { return len(h) } +func (h opHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h opHeap) Less(i, j int) bool { + priority := func(o *op) int { + if o.op == "deref" || o.op == "const" { + // Input to memory load + return 1 + } + if len(o.args) > 0 && (o.args[0].op == "deref" || o.args[0].op == "const") { + // Memory load + return 2 + } + return 100 + } + if p1, p2 := priority(h[i]), priority(h[j]); p1 != p2 { + return p1 < p2 + } + return h[i].id < h[j].id +} + +func (h *opHeap) Push(x any) { + *h = append(*h, x.(*op)) +} + +func (h *opHeap) Pop() any { + old := *h + n := len(old) + x := old[n-1] + *h = old[0 : n-1] + return x +} + +// schedule ensures fn's ops are in dependency order. +func (fn *Func) schedule() { + // TODO: This tends to generate a huge amount of register pressure, mostly + // because it floats loads as early as possible and partly because it has no + // concept of rematerialization and CSE can make rematerializable values + // live for a very long time. It some sense it doesn't matter because we + // don't run out of registers for anything we need. + + missing := make(map[*op]int) + uses := make(map[*op][]*op) + var h opHeap + for _, op := range fn.ops { + if len(op.args) == 0 { + h = append(h, op) + } else { + missing[op] = len(op.args) + } + for _, arg := range op.args { + uses[arg] = append(uses[arg], op) + } + } + heap.Init(&h) + + newOps := make([]*op, 0, len(fn.ops)) + for len(h) > 0 { + if false { + log.Printf("schedule: %s", h) + } + top := h[0] + newOps = append(newOps, top) + heap.Pop(&h) + for _, o := range uses[top] { + missing[o]-- + if missing[o] == 0 { + heap.Push(&h, o) + } + } + } + if len(newOps) != len(fn.ops) { + log.Print("schedule didn't schedule all ops") + log.Print("before:") + fn.write(os.Stderr) + fn.ops = newOps + log.Print("after:") + fn.write(os.Stderr) + log.Fatal("bad schedule") + } + + fn.ops = newOps +} + +func (fn *Func) emit(f *File, locs map[*op]loc) { + w := f.w + + // Emit constants first + for _, o := range fn.ops { + if o.op == "const" { + name := locs[o].(locMem).name + f.emitConst(name, o.c) + } + } + + fmt.Fprintf(w, "TEXT %s(SB), NOSPLIT, $0-0\n", fn.name) + + // Emit body + for _, o := range fn.ops { + switch o.op { + case "const", "arg", "return", "deref", "imm": + // Does not produce code + continue + } + switch o.op { + case "addConst": + fatalf("addConst not lowered") + } + + opName := o.op + // A ".mask" suffix is used to distinguish AVX-512 ops that use the same + // mnemonic for regular and masked mode. + opName = strings.TrimSuffix(opName, ".mask") + + fmt.Fprintf(w, "\t%s", opName) + if o.op == "VGF2P8AFFINEQB" { + // Hidden immediate, but always 0 + // + // TODO: Replace this with an imm input. + fmt.Fprintf(w, " $0,") + } + for i, arg := range o.args { + if i == 0 { + fmt.Fprintf(w, " ") + } else { + fmt.Fprintf(w, ", ") + } + if arg.op == "imm" { + fmt.Fprintf(w, "$0x%x", arg.c) + } else { + fmt.Fprint(w, locs[arg].LocString()) + } + } + if _, ok := opRMW[o.op]; ok { + // Read-modify-write instructions, so the output is already in the + // arguments above. + } else { + fmt.Fprintf(w, ", %s", locs[o].LocString()) + } + fmt.Fprintf(w, "\n") + } + fmt.Fprintf(w, "\tRET\n") + fmt.Fprintf(w, "\n") +} + +func (f *File) emitConst(name string, data any) { + switch data := data.(type) { + case []*Func: + fmt.Fprintf(f.w, "GLOBL %s(SB), RODATA, $%#x\n", name, len(data)*8) + for i, fn := range data { + fmt.Fprintf(f.w, "DATA %s+%#02x(SB)/8, ", name, 8*i) + if fn == nil { + fmt.Fprintf(f.w, "$0\n") + } else { + fmt.Fprintf(f.w, "$%s(SB)\n", fn.name) + } + } + fmt.Fprintf(f.w, "\n") + return + } + + // Assume it's a numeric slice or array + rv := reflect.ValueOf(data) + sz := int(rv.Type().Elem().Size()) + fmt.Fprintf(f.w, "GLOBL %s(SB), RODATA, $%#x\n", name, rv.Len()*sz) + for wi := 0; wi < sz*rv.Len()/8; wi++ { // Iterate over words + var word uint64 + for j := 0; j < 8/sz; j++ { // Iterate over elements in this word + d := rv.Index(wi*8/sz + j).Uint() + word |= d << (j * sz * 8) + } + fmt.Fprintf(f.w, "DATA %s+%#02x(SB)/8, $%#016x\n", name, 8*wi, word) + } + + fmt.Fprintf(f.w, "\n") +} diff --git a/src/internal/runtime/gc/internal/gen/gp.go b/src/internal/runtime/gc/internal/gen/gp.go new file mode 100644 index 00000000000000..390d6e50eda178 --- /dev/null +++ b/src/internal/runtime/gc/internal/gen/gp.go @@ -0,0 +1,26 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gen + +type Uint64 struct { + valGP +} + +var kindUint64 = &kind{typ: "Uint64", reg: regClassGP} + +func ConstUint64(c uint64, name string) (y Uint64) { + y.initOp(&op{op: "const", kind: y.kind(), c: c, name: name}) + return y +} + +func (Uint64) kind() *kind { + return kindUint64 +} + +func (Uint64) wrap(x *op) Uint64 { + var y Uint64 + y.initOp(x) + return y +} diff --git a/src/internal/runtime/gc/internal/gen/regalloc.go b/src/internal/runtime/gc/internal/gen/regalloc.go new file mode 100644 index 00000000000000..424a295afb8062 --- /dev/null +++ b/src/internal/runtime/gc/internal/gen/regalloc.go @@ -0,0 +1,338 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gen + +import ( + "fmt" + "log" + "math/bits" + "strings" +) + +const traceRegAlloc = true + +type regClass uint8 + +const ( + regClassFixed regClass = iota + regClassGP + regClassZ + regClassK + + numRegClasses + + regClassNone = ^regClass(0) +) + +type locReg struct { + cls regClass + reg int +} + +func (l locReg) LocString() string { + switch l.cls { + case regClassFixed: + return fixedRegs[l.reg] + case regClassGP: + return gpRegs[l.reg] + case regClassZ: + return fmt.Sprintf("Z%d", l.reg) + case regClassK: + return fmt.Sprintf("K%d", l.reg) + } + panic("bad register class") +} + +func (l locReg) Deref(off int) (loc, error) { + return locMem{l, off, ""}, nil +} + +func (l locReg) Reg() (locReg, bool) { + return l, true +} + +type locMem struct { + base locReg + off int + name string +} + +func (l locMem) LocString() string { + if l.base.cls == regClassFixed && l.base.reg == regSB && l.off == 0 { + return l.name + "(SB)" + } + if l.name != "" { + return fmt.Sprintf("%s+%d(%s)", l.name, l.off, l.base.LocString()) + } + if l.off != 0 { + return fmt.Sprintf("%d(%s)", l.off, l.base.LocString()) + } + return "(" + l.base.LocString() + ")" +} + +func (l locMem) Deref(off int) (loc, error) { + return nil, fmt.Errorf("cannot dereference already memory address %s", l.LocString()) +} + +func (l locMem) Reg() (locReg, bool) { + if l.base.cls == regClassFixed { + return locReg{}, false + } + return l.base, true +} + +type loc interface { + LocString() string // Return the assembly syntax for this location + Deref(off int) (loc, error) // Treat this location as an address and return a location with the contents of memory at that address + Reg() (locReg, bool) // Register used by this location +} + +var opRMW = map[string]int{ + "VPERMI2B": 2, // Overwrites third argument + "VPERMI2B.Z": 3, // Overwrites fourth argument + "VPERMI2B.mask": 3, // Overwrites fourth argument + "VPERMT2B": 1, // Overwrites second argument TODO: Check this. Unused for now. + "VPBROADCASTQ.mask": 2, // Overwrites last argument +} + +// TODO: Should we have a general rule that all ".mask" instructions overwrite +// their last argument? + +const ( + regSB = iota + regFP +) + +var fixedRegs = []string{regSB: "SB", regFP: "FP"} +var gpRegs = []string{"AX", "BX", "CX", "DI", "SI", "R8", "R9", "R10", "R11"} // ABI argument order + +type regSet struct { + inUse [numRegClasses]uint32 +} + +func (s *regSet) used(o *op, l loc) { + if l == nil { + return + } + reg, ok := l.Reg() + if !ok { + return + } + if traceRegAlloc { + log.Printf(" alloc %s @ v%02d", reg.LocString(), o.id) + } + if s.inUse[reg.cls]&(1<") + + // Create map from op -> fn.ops index + opIndexes := make(map[*op]int, len(fn.ops)) + for i, o := range fn.ops { + opIndexes[o] = i + } + + // Read-modify-write operations share a location with one of their inputs. + // Likewise, deref ops extend the lifetime of their input (but in a shared + // way, unlike RMW ops). + // + // Compute a map from each op to the earliest "canonical" op whose live + // range we'll use. + canon := make(map[*op]*op) + overwritten := make(map[*op]bool) + for _, o := range fn.ops { + // Check that this op doesn't use any overwritten inputs. + for _, arg := range o.args { + if overwritten[arg] { + // TODO: The solution to this is to insert copy ops. + fatalf("op %+v uses overwritten input %+v", o, arg) + } + } + + // Record canonical op. + rmw, ok := opRMW[o.op] + if ok { + canon[o] = canon[o.args[rmw]] + // Record that the input is dead now and must not be referenced. + overwritten[o.args[rmw]] = true + } else if o.op == "deref" { + canon[o] = canon[o.args[0]] + } else { + canon[o] = o + } + } + + // Compute live ranges of each canonical op. + // + // First, find the last use of each op. + lastUses := make(map[*op]*op) // Canonical creation op -> last use op + for _, op := range fn.ops { + for _, arg := range op.args { + lastUses[canon[arg]] = op + } + } + // Invert the last uses map to get a map from op to the (canonical) values + // that die at that op. + lastUseMap := make(map[*op][]*op) // op of last use -> (canonical) creation ops + for def, lastUse := range lastUses { + lastUseMap[lastUse] = append(lastUseMap[lastUse], def) + } + + // Prepare for assignments + regUsed := make([]regSet, len(fn.ops)) // In-use registers at each op + for i := range regUsed { + // X15/Y15/Z15 is reserved by the Go ABI + regUsed[i].inUse[regClassZ] |= 1 << 15 + // K0 is contextual (if used as an opmask, it means no mask). Too + // complicated, so just ignore it. + regUsed[i].inUse[regClassK] |= 1 << 0 + } + locs := make(map[*op]loc) + assign := func(o *op, l loc) { + if have, ok := locs[o]; ok { + fatalf("op %+v already assigned location %v (new %v)", o, have, l) + return + } + if o == canon[o] { + // Mark this location used over o's live range + for i := opIndexes[o]; i < opIndexes[lastUses[o]]; i++ { + regUsed[i].used(fn.ops[i], l) + } + } + locs[o] = l + } + + // Assign fixed locations + id := 0 + for _, o := range fn.ops { + switch o.op { + case "arg": + if traceRegAlloc { + log.Printf("fixed op %+v", o) + } + assign(o, o.c.(locReg)) + case "const": + if traceRegAlloc { + log.Printf("fixed op %+v", o) + } + name := o.name + if name == "" { + name = fmt.Sprintf("%s_%d<>", nameBase, id) + id++ + } else if name[0] == '*' { + name = nameBase + name[1:] + } + assign(o, locMem{locReg{cls: regClassFixed, reg: regSB}, 0, name}) + case "return": + if traceRegAlloc { + log.Printf("fixed op %+v", o) + } + assign(o, nil) // no location + // TODO: argZ should start at 0. + argGP, argZ := 0, 1 + for _, arg := range o.args { + switch arg.kind.reg { + default: + fatalf("bad register class for return value") + case regClassGP: + assign(canon[arg], locReg{regClassGP, argGP}) + argGP++ + case regClassZ: + assign(canon[arg], locReg{regClassZ, argZ}) + argZ++ + } + } + case "imm": + assign(o, nil) // no location + } + } + + // Assign locations. + for _, o := range fn.ops { + if traceRegAlloc { + log.Printf("assign %+v", o) + } + + if _, ok := locs[o]; ok { + // Already assigned a fixed location above. + continue + } + + if o.op == "deref" { + loc, err := locs[o.args[0]].Deref(o.c.(int)) + if err != nil { + fatalf("%v", err) + } + // We don't "assign" here because we've already processed the + // canonical op, which marked loc's register as in-use for the whole + // live range. + locs[o] = loc + continue + } + + if canon[o] != o { + // Copy the canonical op's location. + locs[o] = locs[canon[o]] + continue + } + // Below here we know that o is already a canonical op. + + if _, ok := opRMW[o.op]; ok { + fatalf("read-modify-write op not canonicalized") + } + + // Find a free register of the right class. + cls := o.kind.reg + var used uint32 + for i := opIndexes[o]; i < opIndexes[lastUses[o]]; i++ { + used |= regUsed[i].inUse[cls] + } + + // Assign result location. + num := bits.TrailingZeros32(^used) + switch cls { + default: + fatalf("unknown reg class %v", cls) + case regClassGP: + if num >= len(gpRegs) { + panic("out of GP regs") + } + case regClassZ: + if num >= 32 { + panic("out of Z regs") + } + case regClassK: + if num >= 8 { + panic("out of K regs") + } + } + loc := locReg{cls, num} + assign(o, loc) + } + + return locs +} diff --git a/src/internal/runtime/gc/internal/gen/simd.go b/src/internal/runtime/gc/internal/gen/simd.go new file mode 100644 index 00000000000000..0360aa4b06560f --- /dev/null +++ b/src/internal/runtime/gc/internal/gen/simd.go @@ -0,0 +1,246 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gen + +type Uint8x64 struct { + valAny +} + +var kindUint8x64 = &kind{typ: "Uint8x64", reg: regClassZ} + +func ConstUint8x64(c [64]uint8, name string) (y Uint8x64) { + y.initOp(&op{op: "const", kind: y.kind(), c: c, name: name}) + return y +} + +func (Uint8x64) kind() *kind { + return kindUint8x64 +} + +func (Uint8x64) wrap(x *op) Uint8x64 { + var y Uint8x64 + y.initOp(x) + return y +} + +func (x Uint8x64) ToUint64x8() (z Uint64x8) { + z.op = x.op + return z +} + +func (x Uint8x64) Shuffle(shuf Uint8x64) (y Uint8x64) { + if shuf.op.op == "const" { + // TODO: There are often patterns we can take advantage of here. Sometimes + // we can do a broadcast. Sometimes we can at least do a quadword + // permutation instead of a full byte permutation. + + // Range check the shuffle + for i, inp := range shuf.op.c.([64]uint8) { + // 0xff is a special "don't care" value + if !(inp == 0xff || inp < 64) { + fatalf("shuffle[%d] = %d out of range [0, %d) or 0xff", i, inp, 64) + } + } + } + + args := []*op{x.op, shuf.op} + y.initOp(&op{op: "VPERMB", kind: y.kind(), args: args}) + return y +} + +func (x Uint8x64) ShuffleZeroed(shuf Uint8x64, mask Mask64) (y Uint8x64) { + args := []*op{x.op, shuf.op, mask.op} + y.initOp(&op{op: "VPERMB.Z", kind: y.kind(), args: args}) + return y +} + +func (x Uint8x64) ShuffleMasked(shuf Uint8x64, mask Mask64) (y Uint8x64) { + args := []*op{x.op, shuf.op, mask.op} + y.initOp(&op{op: "VPERMB.mask", kind: y.kind(), args: args}) + return y +} + +// TODO: The two-argument shuffle is a little weird. You almost want the +// receiver to be the shuffle and the two arguments to be the two inputs, but +// that's almost certainly *not* what you want for the single input shuffle. + +func (x Uint8x64) Shuffle2(y Uint8x64, shuf Uint8x64) (z Uint8x64) { + // Confusingly, the inputs are in the opposite order from what you'd expect. + args := []*op{y.op, x.op, shuf.op} + z.initOp(&op{op: "VPERMI2B", kind: z.kind(), args: args}) + return z +} + +func (x Uint8x64) Shuffle2Zeroed(y Uint8x64, shuf Uint8x64, mask Mask64) (z Uint8x64) { + // Confusingly, the inputs are in the opposite order from what you'd expect. + args := []*op{y.op, x.op, mask.op, shuf.op} + z.initOp(&op{op: "VPERMI2B.Z", kind: z.kind(), args: args}) + return z +} + +func (x Uint8x64) Shuffle2Masked(y Uint8x64, shuf Uint8x64, mask Mask64) (z Uint8x64) { + // Confusingly, the inputs are in the opposite order from what you'd expect. + args := []*op{y.op, x.op, mask.op, shuf.op} + z.initOp(&op{op: "VPERMI2B.mask", kind: z.kind(), args: args}) + return z +} + +type Uint64x8 struct { + valAny +} + +var kindUint64x8 = &kind{typ: "Uint64x8", reg: regClassZ} + +func ConstUint64x8(c [8]uint64, name string) (y Uint64x8) { + // TODO: Sometimes these can be optimized into broadcast loads. + y.initOp(&op{op: "const", kind: y.kind(), c: c, name: name}) + return y +} + +func BroadcastUint64x8Zeroed(src Uint64, mask Mask8) (z Uint64x8) { + z.initOp(&op{op: "VPBROADCASTQ.Z", kind: z.kind(), args: []*op{src.op, mask.op}}) + return z +} + +func (x Uint64x8) BroadcastMasked(src Uint64, mask Mask8) (z Uint64x8) { + z.initOp(&op{op: "VPBROADCASTQ.mask", kind: z.kind(), args: []*op{src.op, mask.op, x.op}}) + return z +} + +func (Uint64x8) kind() *kind { + return kindUint64x8 +} + +func (Uint64x8) wrap(x *op) Uint64x8 { + var y Uint64x8 + y.initOp(x) + return y +} + +func (x Uint64x8) Or(y Uint64x8) (z Uint64x8) { + z.initOp(&op{op: "VPORQ", kind: z.kind(), args: []*op{y.op, x.op}}) + return z +} + +func (x Uint64x8) Sub(y Uint64x8) (z Uint64x8) { + // Arguments are backwards + z.initOp(&op{op: "VPSUBQ", kind: z.kind(), args: []*op{y.op, x.op}}) + return z +} + +func (x Uint64x8) ToUint8x64() (z Uint8x64) { + z.op = x.op + return z +} + +func (x Uint64x8) GF2P8Affine(y Uint8x64) (z Uint8x64) { + // matrix, vector + z.initOp(&op{op: "VGF2P8AFFINEQB", kind: z.kind(), args: []*op{x.op, y.op}}) + return z +} + +func (x Uint64x8) ShuffleBits(y Uint8x64) (z Mask64) { + z.initOp(&op{op: "VPSHUFBITQMB", kind: z.kind(), args: []*op{y.op, x.op}}) + return z +} + +func (x Uint64x8) ShuffleBitsMasked(y Uint8x64, mask Mask64) (z Mask64) { + // This is always zeroing if the mask is provided. + z.initOp(&op{op: "VPSHUFBITQMB", kind: z.kind(), args: []*op{y.op, x.op, mask.op}}) + return z +} + +type Mask8 struct { + valAny +} + +var kindMask8 = &kind{typ: "Mask8", reg: regClassK} + +func ConstMask8(c uint8) (y Mask8) { + var tmp Uint64 + tmp.initOp(&op{op: "MOVQ", kind: tmp.kind(), args: []*op{imm(c)}}) + y.initOp(&op{op: "KMOVB", kind: y.kind(), args: []*op{tmp.op}}) + return y +} + +func (Mask8) kind() *kind { + return kindMask8 +} + +func (Mask8) wrap(x *op) Mask8 { + var y Mask8 + y.initOp(x) + return y +} + +func (x Mask8) ToUint8() (z Uint64) { + z.initOp(&op{op: "KMOVB", kind: z.kind(), args: []*op{x.op}}) + return z +} + +func (x Mask8) Or(y Mask8) (z Mask8) { + z.initOp(&op{op: "KORQ", kind: z.kind(), args: []*op{y.op, x.op}}) + return z +} + +func (x Mask8) ShiftLeft(c uint8) (z Mask8) { + if c == 0 { + z = x + } else { + z.initOp(&op{op: "KSHIFTLB", kind: z.kind(), args: []*op{imm(c), x.op}}) + } + return z +} + +type Mask64 struct { + valAny +} + +var kindMask64 = &kind{typ: "Mask64", reg: regClassK} + +func ConstMask64(c uint64) (y Mask64) { + var tmp Uint64 + tmp.initOp(&op{op: "MOVQ", kind: tmp.kind(), args: []*op{imm(c)}}) + y.initOp(&op{op: "KMOVQ", kind: y.kind(), args: []*op{tmp.op}}) + return y +} + +func (Mask64) kind() *kind { + return kindMask64 +} + +func (Mask64) wrap(x *op) Mask64 { + var y Mask64 + y.initOp(x) + return y +} + +func (x Mask64) ToUint64() (z Uint64) { + z.initOp(&op{op: "KMOVQ", kind: z.kind(), args: []*op{x.op}}) + return z +} + +func (x Mask64) Or(y Mask64) (z Mask64) { + z.initOp(&op{op: "KORQ", kind: z.kind(), args: []*op{y.op, x.op}}) + return z +} + +func (x Mask64) ShiftLeft(c uint8) (z Mask64) { + if c == 0 { + z = x + } else { + z.initOp(&op{op: "KSHIFTLQ", kind: z.kind(), args: []*op{imm(c), x.op}}) + } + return z +} + +func (x Mask64) ShiftRight(c uint8) (z Mask64) { + if c == 0 { + z = x + } else { + z.initOp(&op{op: "KSHIFTRQ", kind: z.kind(), args: []*op{imm(c), x.op}}) + } + return z +} diff --git a/src/internal/runtime/gc/internal/gen/val.go b/src/internal/runtime/gc/internal/gen/val.go new file mode 100644 index 00000000000000..24a843a62c4257 --- /dev/null +++ b/src/internal/runtime/gc/internal/gen/val.go @@ -0,0 +1,137 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gen + +import "sync" + +type Value interface { + kind() *kind + getOp() *op +} + +type Word interface { + Value + isWord() +} + +// wrap is an unfortunate necessity so that we can pass Value types around as +// values (not pointers), but still have generic functions that can construct a +// new Value. Ideally we would just have a method on Value to initialize its op, +// but that needs to have a non-pointer receiver to satisfy the interface and +// then it can't mutate the Value. +type wrap[T Value] interface { + Value + wrap(x *op) T +} + +type kind struct { + typ string + reg regClass +} + +type void struct { + valAny +} + +var voidKind = &kind{typ: "void", reg: regClassNone} + +func (void) kind() *kind { return voidKind } + +type Ptr[T Value] struct { + valGP +} + +// Ptr is a Word +var _ Word = Ptr[void]{} + +var ptrKinds = sync.Map{} // *kind -> *kind + +func (Ptr[T]) kind() *kind { + var x T + xk := x.kind() + pk, ok := ptrKinds.Load(xk) + if !ok { + k := &kind{typ: "Ptr[" + x.kind().typ + "]", reg: regClassGP} + pk, _ = ptrKinds.LoadOrStore(xk, k) + } + return pk.(*kind) +} + +func (Ptr[T]) wrap(x *op) Ptr[T] { + var y Ptr[T] + y.initOp(x) + return y +} + +func (x Ptr[T]) AddConst(off int) (y Ptr[T]) { + base := x.op + for base.op == "addConst" { + off += base.args[1].c.(int) + base = base.args[0] + } + y.initOp(&op{op: "addConst", kind: y.kind(), args: []*op{base, imm(off)}}) + return y +} + +func Deref[W wrap[T], T Value](ptr Ptr[W]) T { + var off int + base := ptr.op + for base.op == "addConst" { + off += base.args[1].c.(int) + base = base.args[0] + } + + var y W + return y.wrap(&op{op: "deref", kind: y.kind(), args: []*op{base}, c: off}) +} + +type Array[T Value] struct { + valAny +} + +func ConstArray[T Value](vals []T, name string) (y Array[T]) { + // TODO: This probably doesn't actually work because emitConst won't + // understand vals. + y.initOp(&op{op: "const", kind: y.kind(), c: vals, name: name}) + return y +} + +func (Array[T]) kind() *kind { + // TODO: Cache this like Ptr.kind. + var x T + return &kind{typ: "Array[" + x.kind().typ + "]", reg: regClassNone} +} + +type valGP struct { + valAny +} + +func (valGP) isWord() {} + +type valAny struct { + *op +} + +func (v *valAny) initOp(x *op) { + if v.op != nil { + panic("double init of val") + } + if x.kind == nil { + panic("val missing kind") + } + v.op = x + + // Figure out this value's function. + for _, arg := range x.args { + if fn := arg.fn; fn != nil { + fn.attach(x) + break + } + } +} + +func (v valAny) getOp() *op { + return v.op +} diff --git a/src/internal/runtime/gc/malloc.go b/src/internal/runtime/gc/malloc.go new file mode 100644 index 00000000000000..7c36a6bfbe942a --- /dev/null +++ b/src/internal/runtime/gc/malloc.go @@ -0,0 +1,51 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gc + +import "internal/goarch" + +const ( + // PageWords is the number of pointer-words per page. + PageWords = PageSize / goarch.PtrSize + + // A malloc header is functionally a single type pointer, but + // we need to use 8 here to ensure 8-byte alignment of allocations + // on 32-bit platforms. It's wasteful, but a lot of code relies on + // 8-byte alignment for 8-byte atomics. + MallocHeaderSize = 8 + + // The minimum object size that has a malloc header, exclusive. + // + // The size of this value controls overheads from the malloc header. + // The minimum size is bound by writeHeapBitsSmall, which assumes that the + // pointer bitmap for objects of a size smaller than this doesn't cross + // more than one pointer-word boundary. This sets an upper-bound on this + // value at the number of bits in a uintptr, multiplied by the pointer + // size in bytes. + // + // We choose a value here that has a natural cutover point in terms of memory + // overheads. This value just happens to be the maximum possible value this + // can be. + // + // A span with heap bits in it will have 128 bytes of heap bits on 64-bit + // platforms, and 256 bytes of heap bits on 32-bit platforms. The first size + // class where malloc headers match this overhead for 64-bit platforms is + // 512 bytes (8 KiB / 512 bytes * 8 bytes-per-header = 128 bytes of overhead). + // On 32-bit platforms, this same point is the 256 byte size class + // (8 KiB / 256 bytes * 8 bytes-per-header = 256 bytes of overhead). + // + // Guaranteed to be exactly at a size class boundary. The reason this value is + // an exclusive minimum is subtle. Suppose we're allocating a 504-byte object + // and its rounded up to 512 bytes for the size class. If minSizeForMallocHeader + // is 512 and an inclusive minimum, then a comparison against minSizeForMallocHeader + // by the two values would produce different results. In other words, the comparison + // would not be invariant to size-class rounding. Eschewing this property means a + // more complex check or possibly storing additional state to determine whether a + // span has malloc headers. + MinSizeForMallocHeader = goarch.PtrSize * goarch.PtrBits + + // PageSize is the increment in which spans are managed. + PageSize = 1 << PageShift +) diff --git a/src/internal/runtime/gc/scan.go b/src/internal/runtime/gc/scan.go new file mode 100644 index 00000000000000..863a4bf0f30c3b --- /dev/null +++ b/src/internal/runtime/gc/scan.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gc + +import "internal/goarch" + +// ObjMask is a bitmap where each bit corresponds to an object in a span. +// +// It is sized to accommodate all size classes. +type ObjMask [MaxObjsPerSpan / (goarch.PtrSize * 8)]uintptr + +// PtrMask is a bitmap where each bit represents a pointer-word in a single runtime page. +type PtrMask [PageSize / goarch.PtrSize / (goarch.PtrSize * 8)]uintptr diff --git a/src/internal/runtime/gc/scan/expand_amd64.go b/src/internal/runtime/gc/scan/expand_amd64.go new file mode 100644 index 00000000000000..9bea471abec6c5 --- /dev/null +++ b/src/internal/runtime/gc/scan/expand_amd64.go @@ -0,0 +1,22 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scan + +import "internal/runtime/gc" + +// ExpandAVX512 expands each bit in packed into f consecutive bits in unpacked, +// where f is the word size of objects in sizeClass. +// +// This is a testing entrypoint to the expanders used by scanSpanPacked*. +// +//go:noescape +func ExpandAVX512(sizeClass int, packed *gc.ObjMask, unpacked *gc.PtrMask) + +// gcExpandersAVX512 is the PCs of expander functions. These cannot be called directly +// as they don't follow the Go ABI, but you can use this to check if a given +// expander PC is 0. +// +// It is defined in assembly. +var gcExpandersAVX512 [len(gc.SizeClassToSize)]uintptr diff --git a/src/internal/runtime/gc/scan/expand_amd64.s b/src/internal/runtime/gc/scan/expand_amd64.s new file mode 100644 index 00000000000000..6b0be44cc10450 --- /dev/null +++ b/src/internal/runtime/gc/scan/expand_amd64.s @@ -0,0 +1,2631 @@ +// Code generated by mkasm.go. DO NOT EDIT. + +#include "go_asm.h" +#include "textflag.h" + +GLOBL ·gcExpandersAVX512(SB), RODATA, $0x220 +DATA ·gcExpandersAVX512+0x00(SB)/8, $0 +DATA ·gcExpandersAVX512+0x08(SB)/8, $expandAVX512_1<>(SB) +DATA ·gcExpandersAVX512+0x10(SB)/8, $expandAVX512_2<>(SB) +DATA ·gcExpandersAVX512+0x18(SB)/8, $expandAVX512_3<>(SB) +DATA ·gcExpandersAVX512+0x20(SB)/8, $expandAVX512_4<>(SB) +DATA ·gcExpandersAVX512+0x28(SB)/8, $expandAVX512_6<>(SB) +DATA ·gcExpandersAVX512+0x30(SB)/8, $expandAVX512_8<>(SB) +DATA ·gcExpandersAVX512+0x38(SB)/8, $expandAVX512_10<>(SB) +DATA ·gcExpandersAVX512+0x40(SB)/8, $expandAVX512_12<>(SB) +DATA ·gcExpandersAVX512+0x48(SB)/8, $expandAVX512_14<>(SB) +DATA ·gcExpandersAVX512+0x50(SB)/8, $expandAVX512_16<>(SB) +DATA ·gcExpandersAVX512+0x58(SB)/8, $expandAVX512_18<>(SB) +DATA ·gcExpandersAVX512+0x60(SB)/8, $expandAVX512_20<>(SB) +DATA ·gcExpandersAVX512+0x68(SB)/8, $expandAVX512_22<>(SB) +DATA ·gcExpandersAVX512+0x70(SB)/8, $expandAVX512_24<>(SB) +DATA ·gcExpandersAVX512+0x78(SB)/8, $expandAVX512_26<>(SB) +DATA ·gcExpandersAVX512+0x80(SB)/8, $expandAVX512_28<>(SB) +DATA ·gcExpandersAVX512+0x88(SB)/8, $expandAVX512_30<>(SB) +DATA ·gcExpandersAVX512+0x90(SB)/8, $expandAVX512_32<>(SB) +DATA ·gcExpandersAVX512+0x98(SB)/8, $expandAVX512_36<>(SB) +DATA ·gcExpandersAVX512+0xa0(SB)/8, $expandAVX512_40<>(SB) +DATA ·gcExpandersAVX512+0xa8(SB)/8, $expandAVX512_44<>(SB) +DATA ·gcExpandersAVX512+0xb0(SB)/8, $expandAVX512_48<>(SB) +DATA ·gcExpandersAVX512+0xb8(SB)/8, $expandAVX512_52<>(SB) +DATA ·gcExpandersAVX512+0xc0(SB)/8, $expandAVX512_56<>(SB) +DATA ·gcExpandersAVX512+0xc8(SB)/8, $expandAVX512_60<>(SB) +DATA ·gcExpandersAVX512+0xd0(SB)/8, $expandAVX512_64<>(SB) +DATA ·gcExpandersAVX512+0xd8(SB)/8, $0 +DATA ·gcExpandersAVX512+0xe0(SB)/8, $0 +DATA ·gcExpandersAVX512+0xe8(SB)/8, $0 +DATA ·gcExpandersAVX512+0xf0(SB)/8, $0 +DATA ·gcExpandersAVX512+0xf8(SB)/8, $0 +DATA ·gcExpandersAVX512+0x100(SB)/8, $0 +DATA ·gcExpandersAVX512+0x108(SB)/8, $0 +DATA ·gcExpandersAVX512+0x110(SB)/8, $0 +DATA ·gcExpandersAVX512+0x118(SB)/8, $0 +DATA ·gcExpandersAVX512+0x120(SB)/8, $0 +DATA ·gcExpandersAVX512+0x128(SB)/8, $0 +DATA ·gcExpandersAVX512+0x130(SB)/8, $0 +DATA ·gcExpandersAVX512+0x138(SB)/8, $0 +DATA ·gcExpandersAVX512+0x140(SB)/8, $0 +DATA ·gcExpandersAVX512+0x148(SB)/8, $0 +DATA ·gcExpandersAVX512+0x150(SB)/8, $0 +DATA ·gcExpandersAVX512+0x158(SB)/8, $0 +DATA ·gcExpandersAVX512+0x160(SB)/8, $0 +DATA ·gcExpandersAVX512+0x168(SB)/8, $0 +DATA ·gcExpandersAVX512+0x170(SB)/8, $0 +DATA ·gcExpandersAVX512+0x178(SB)/8, $0 +DATA ·gcExpandersAVX512+0x180(SB)/8, $0 +DATA ·gcExpandersAVX512+0x188(SB)/8, $0 +DATA ·gcExpandersAVX512+0x190(SB)/8, $0 +DATA ·gcExpandersAVX512+0x198(SB)/8, $0 +DATA ·gcExpandersAVX512+0x1a0(SB)/8, $0 +DATA ·gcExpandersAVX512+0x1a8(SB)/8, $0 +DATA ·gcExpandersAVX512+0x1b0(SB)/8, $0 +DATA ·gcExpandersAVX512+0x1b8(SB)/8, $0 +DATA ·gcExpandersAVX512+0x1c0(SB)/8, $0 +DATA ·gcExpandersAVX512+0x1c8(SB)/8, $0 +DATA ·gcExpandersAVX512+0x1d0(SB)/8, $0 +DATA ·gcExpandersAVX512+0x1d8(SB)/8, $0 +DATA ·gcExpandersAVX512+0x1e0(SB)/8, $0 +DATA ·gcExpandersAVX512+0x1e8(SB)/8, $0 +DATA ·gcExpandersAVX512+0x1f0(SB)/8, $0 +DATA ·gcExpandersAVX512+0x1f8(SB)/8, $0 +DATA ·gcExpandersAVX512+0x200(SB)/8, $0 +DATA ·gcExpandersAVX512+0x208(SB)/8, $0 +DATA ·gcExpandersAVX512+0x210(SB)/8, $0 +DATA ·gcExpandersAVX512+0x218(SB)/8, $0 + +TEXT expandAVX512_1<>(SB), NOSPLIT, $0-0 + VMOVDQU64 (AX), Z1 + VMOVDQU64 64(AX), Z2 + RET + +GLOBL expandAVX512_2_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_2_inShuf0<>+0x00(SB)/8, $0x0706050403020100 +DATA expandAVX512_2_inShuf0<>+0x08(SB)/8, $0x0706050403020100 +DATA expandAVX512_2_inShuf0<>+0x10(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_2_inShuf0<>+0x18(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_2_inShuf0<>+0x20(SB)/8, $0x1716151413121110 +DATA expandAVX512_2_inShuf0<>+0x28(SB)/8, $0x1716151413121110 +DATA expandAVX512_2_inShuf0<>+0x30(SB)/8, $0x1f1e1d1c1b1a1918 +DATA expandAVX512_2_inShuf0<>+0x38(SB)/8, $0x1f1e1d1c1b1a1918 + +GLOBL expandAVX512_2_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_2_mat0<>+0x00(SB)/8, $0x0101020204040808 +DATA expandAVX512_2_mat0<>+0x08(SB)/8, $0x1010202040408080 +DATA expandAVX512_2_mat0<>+0x10(SB)/8, $0x0101020204040808 +DATA expandAVX512_2_mat0<>+0x18(SB)/8, $0x1010202040408080 +DATA expandAVX512_2_mat0<>+0x20(SB)/8, $0x0101020204040808 +DATA expandAVX512_2_mat0<>+0x28(SB)/8, $0x1010202040408080 +DATA expandAVX512_2_mat0<>+0x30(SB)/8, $0x0101020204040808 +DATA expandAVX512_2_mat0<>+0x38(SB)/8, $0x1010202040408080 + +GLOBL expandAVX512_2_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_2_inShuf1<>+0x00(SB)/8, $0x2726252423222120 +DATA expandAVX512_2_inShuf1<>+0x08(SB)/8, $0x2726252423222120 +DATA expandAVX512_2_inShuf1<>+0x10(SB)/8, $0x2f2e2d2c2b2a2928 +DATA expandAVX512_2_inShuf1<>+0x18(SB)/8, $0x2f2e2d2c2b2a2928 +DATA expandAVX512_2_inShuf1<>+0x20(SB)/8, $0x3736353433323130 +DATA expandAVX512_2_inShuf1<>+0x28(SB)/8, $0x3736353433323130 +DATA expandAVX512_2_inShuf1<>+0x30(SB)/8, $0x3f3e3d3c3b3a3938 +DATA expandAVX512_2_inShuf1<>+0x38(SB)/8, $0x3f3e3d3c3b3a3938 + +GLOBL expandAVX512_2_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_2_outShufLo+0x00(SB)/8, $0x0b030a0209010800 +DATA expandAVX512_2_outShufLo+0x08(SB)/8, $0x0f070e060d050c04 +DATA expandAVX512_2_outShufLo+0x10(SB)/8, $0x1b131a1219111810 +DATA expandAVX512_2_outShufLo+0x18(SB)/8, $0x1f171e161d151c14 +DATA expandAVX512_2_outShufLo+0x20(SB)/8, $0x2b232a2229212820 +DATA expandAVX512_2_outShufLo+0x28(SB)/8, $0x2f272e262d252c24 +DATA expandAVX512_2_outShufLo+0x30(SB)/8, $0x3b333a3239313830 +DATA expandAVX512_2_outShufLo+0x38(SB)/8, $0x3f373e363d353c34 + +TEXT expandAVX512_2<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_2_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_2_mat0<>(SB), Z1 + VMOVDQU64 expandAVX512_2_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_2_outShufLo(SB), Z3 + VMOVDQU64 (AX), Z4 + VPERMB Z4, Z0, Z0 + VGF2P8AFFINEQB $0, Z1, Z0, Z0 + VPERMB Z4, Z2, Z2 + VGF2P8AFFINEQB $0, Z1, Z2, Z2 + VPERMB Z0, Z3, Z1 + VPERMB Z2, Z3, Z2 + RET + +GLOBL expandAVX512_3_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_3_inShuf0<>+0x00(SB)/8, $0x0706050403020100 +DATA expandAVX512_3_inShuf0<>+0x08(SB)/8, $0x0706050403020100 +DATA expandAVX512_3_inShuf0<>+0x10(SB)/8, $0x0706050403020100 +DATA expandAVX512_3_inShuf0<>+0x18(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_3_inShuf0<>+0x20(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_3_inShuf0<>+0x28(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_3_inShuf0<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_3_inShuf0<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_3_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_3_mat0<>+0x00(SB)/8, $0x0101010202020404 +DATA expandAVX512_3_mat0<>+0x08(SB)/8, $0x0408080810101020 +DATA expandAVX512_3_mat0<>+0x10(SB)/8, $0x2020404040808080 +DATA expandAVX512_3_mat0<>+0x18(SB)/8, $0x0101010202020404 +DATA expandAVX512_3_mat0<>+0x20(SB)/8, $0x0408080810101020 +DATA expandAVX512_3_mat0<>+0x28(SB)/8, $0x2020404040808080 +DATA expandAVX512_3_mat0<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_3_mat0<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_3_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_3_inShuf1<>+0x00(SB)/8, $0x1716151413121110 +DATA expandAVX512_3_inShuf1<>+0x08(SB)/8, $0x1716151413121110 +DATA expandAVX512_3_inShuf1<>+0x10(SB)/8, $0x1716151413121110 +DATA expandAVX512_3_inShuf1<>+0x18(SB)/8, $0x1f1e1d1c1b1a1918 +DATA expandAVX512_3_inShuf1<>+0x20(SB)/8, $0x1f1e1d1c1b1a1918 +DATA expandAVX512_3_inShuf1<>+0x28(SB)/8, $0x1f1e1d1c1b1a1918 +DATA expandAVX512_3_inShuf1<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_3_inShuf1<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_3_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_3_inShuf2<>+0x00(SB)/8, $0x2726252423222120 +DATA expandAVX512_3_inShuf2<>+0x08(SB)/8, $0x2726252423222120 +DATA expandAVX512_3_inShuf2<>+0x10(SB)/8, $0x2726252423222120 +DATA expandAVX512_3_inShuf2<>+0x18(SB)/8, $0xffffffffff2a2928 +DATA expandAVX512_3_inShuf2<>+0x20(SB)/8, $0xffffffffff2a2928 +DATA expandAVX512_3_inShuf2<>+0x28(SB)/8, $0xffffffffffff2928 +DATA expandAVX512_3_inShuf2<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_3_inShuf2<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_3_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_3_outShufLo+0x00(SB)/8, $0x0a02110901100800 +DATA expandAVX512_3_outShufLo+0x08(SB)/8, $0x05140c04130b0312 +DATA expandAVX512_3_outShufLo+0x10(SB)/8, $0x170f07160e06150d +DATA expandAVX512_3_outShufLo+0x18(SB)/8, $0x221a292119282018 +DATA expandAVX512_3_outShufLo+0x20(SB)/8, $0x1d2c241c2b231b2a +DATA expandAVX512_3_outShufLo+0x28(SB)/8, $0x2f271f2e261e2d25 +DATA expandAVX512_3_outShufLo+0x30(SB)/8, $0x4a42514941504840 +DATA expandAVX512_3_outShufLo+0x38(SB)/8, $0x45544c44534b4352 + +GLOBL expandAVX512_3_outShufHi(SB), RODATA, $0x40 +DATA expandAVX512_3_outShufHi+0x00(SB)/8, $0x170f07160e06150d +DATA expandAVX512_3_outShufHi+0x08(SB)/8, $0x221a292119282018 +DATA expandAVX512_3_outShufHi+0x10(SB)/8, $0x1d2c241c2b231b2a +DATA expandAVX512_3_outShufHi+0x18(SB)/8, $0x2f271f2e261e2d25 +DATA expandAVX512_3_outShufHi+0x20(SB)/8, $0x4a42514941504840 +DATA expandAVX512_3_outShufHi+0x28(SB)/8, $0x45544c44534b4352 +DATA expandAVX512_3_outShufHi+0x30(SB)/8, $0x574f47564e46554d +DATA expandAVX512_3_outShufHi+0x38(SB)/8, $0x625a696159686058 + +TEXT expandAVX512_3<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_3_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_3_mat0<>(SB), Z3 + VMOVDQU64 expandAVX512_3_inShuf1<>(SB), Z4 + VMOVDQU64 expandAVX512_3_inShuf2<>(SB), Z5 + VMOVDQU64 expandAVX512_3_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_3_outShufHi(SB), Z2 + VMOVDQU64 (AX), Z6 + VPERMB Z6, Z0, Z0 + VGF2P8AFFINEQB $0, Z3, Z0, Z0 + VPERMB Z6, Z4, Z4 + VGF2P8AFFINEQB $0, Z3, Z4, Z4 + VPERMB Z6, Z5, Z5 + VGF2P8AFFINEQB $0, Z3, Z5, Z3 + VPERMI2B Z4, Z0, Z1 + VPERMI2B Z3, Z4, Z2 + RET + +GLOBL expandAVX512_4_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_4_inShuf0<>+0x00(SB)/8, $0x0706050403020100 +DATA expandAVX512_4_inShuf0<>+0x08(SB)/8, $0x0706050403020100 +DATA expandAVX512_4_inShuf0<>+0x10(SB)/8, $0x0706050403020100 +DATA expandAVX512_4_inShuf0<>+0x18(SB)/8, $0x0706050403020100 +DATA expandAVX512_4_inShuf0<>+0x20(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_4_inShuf0<>+0x28(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_4_inShuf0<>+0x30(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_4_inShuf0<>+0x38(SB)/8, $0x0f0e0d0c0b0a0908 + +GLOBL expandAVX512_4_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_4_mat0<>+0x00(SB)/8, $0x0101010102020202 +DATA expandAVX512_4_mat0<>+0x08(SB)/8, $0x0404040408080808 +DATA expandAVX512_4_mat0<>+0x10(SB)/8, $0x1010101020202020 +DATA expandAVX512_4_mat0<>+0x18(SB)/8, $0x4040404080808080 +DATA expandAVX512_4_mat0<>+0x20(SB)/8, $0x0101010102020202 +DATA expandAVX512_4_mat0<>+0x28(SB)/8, $0x0404040408080808 +DATA expandAVX512_4_mat0<>+0x30(SB)/8, $0x1010101020202020 +DATA expandAVX512_4_mat0<>+0x38(SB)/8, $0x4040404080808080 + +GLOBL expandAVX512_4_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_4_inShuf1<>+0x00(SB)/8, $0x1716151413121110 +DATA expandAVX512_4_inShuf1<>+0x08(SB)/8, $0x1716151413121110 +DATA expandAVX512_4_inShuf1<>+0x10(SB)/8, $0x1716151413121110 +DATA expandAVX512_4_inShuf1<>+0x18(SB)/8, $0x1716151413121110 +DATA expandAVX512_4_inShuf1<>+0x20(SB)/8, $0x1f1e1d1c1b1a1918 +DATA expandAVX512_4_inShuf1<>+0x28(SB)/8, $0x1f1e1d1c1b1a1918 +DATA expandAVX512_4_inShuf1<>+0x30(SB)/8, $0x1f1e1d1c1b1a1918 +DATA expandAVX512_4_inShuf1<>+0x38(SB)/8, $0x1f1e1d1c1b1a1918 + +GLOBL expandAVX512_4_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_4_outShufLo+0x00(SB)/8, $0x1911090118100800 +DATA expandAVX512_4_outShufLo+0x08(SB)/8, $0x1b130b031a120a02 +DATA expandAVX512_4_outShufLo+0x10(SB)/8, $0x1d150d051c140c04 +DATA expandAVX512_4_outShufLo+0x18(SB)/8, $0x1f170f071e160e06 +DATA expandAVX512_4_outShufLo+0x20(SB)/8, $0x3931292138302820 +DATA expandAVX512_4_outShufLo+0x28(SB)/8, $0x3b332b233a322a22 +DATA expandAVX512_4_outShufLo+0x30(SB)/8, $0x3d352d253c342c24 +DATA expandAVX512_4_outShufLo+0x38(SB)/8, $0x3f372f273e362e26 + +TEXT expandAVX512_4<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_4_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_4_mat0<>(SB), Z1 + VMOVDQU64 expandAVX512_4_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_4_outShufLo(SB), Z3 + VMOVDQU64 (AX), Z4 + VPERMB Z4, Z0, Z0 + VGF2P8AFFINEQB $0, Z1, Z0, Z0 + VPERMB Z4, Z2, Z2 + VGF2P8AFFINEQB $0, Z1, Z2, Z2 + VPERMB Z0, Z3, Z1 + VPERMB Z2, Z3, Z2 + RET + +GLOBL expandAVX512_6_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_6_inShuf0<>+0x00(SB)/8, $0x0706050403020100 +DATA expandAVX512_6_inShuf0<>+0x08(SB)/8, $0x0706050403020100 +DATA expandAVX512_6_inShuf0<>+0x10(SB)/8, $0x0706050403020100 +DATA expandAVX512_6_inShuf0<>+0x18(SB)/8, $0x0706050403020100 +DATA expandAVX512_6_inShuf0<>+0x20(SB)/8, $0x0706050403020100 +DATA expandAVX512_6_inShuf0<>+0x28(SB)/8, $0x0706050403020100 +DATA expandAVX512_6_inShuf0<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_6_inShuf0<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_6_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_6_mat0<>+0x00(SB)/8, $0x0101010101010202 +DATA expandAVX512_6_mat0<>+0x08(SB)/8, $0x0202020204040404 +DATA expandAVX512_6_mat0<>+0x10(SB)/8, $0x0404080808080808 +DATA expandAVX512_6_mat0<>+0x18(SB)/8, $0x1010101010102020 +DATA expandAVX512_6_mat0<>+0x20(SB)/8, $0x2020202040404040 +DATA expandAVX512_6_mat0<>+0x28(SB)/8, $0x4040808080808080 +DATA expandAVX512_6_mat0<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_6_mat0<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_6_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_6_inShuf1<>+0x00(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_6_inShuf1<>+0x08(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_6_inShuf1<>+0x10(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_6_inShuf1<>+0x18(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_6_inShuf1<>+0x20(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_6_inShuf1<>+0x28(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_6_inShuf1<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_6_inShuf1<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_6_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_6_inShuf2<>+0x00(SB)/8, $0xffff151413121110 +DATA expandAVX512_6_inShuf2<>+0x08(SB)/8, $0xffff151413121110 +DATA expandAVX512_6_inShuf2<>+0x10(SB)/8, $0xffffff1413121110 +DATA expandAVX512_6_inShuf2<>+0x18(SB)/8, $0xffffff1413121110 +DATA expandAVX512_6_inShuf2<>+0x20(SB)/8, $0xffffff1413121110 +DATA expandAVX512_6_inShuf2<>+0x28(SB)/8, $0xffffff1413121110 +DATA expandAVX512_6_inShuf2<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_6_inShuf2<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_6_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_6_outShufLo+0x00(SB)/8, $0x0901282018100800 +DATA expandAVX512_6_outShufLo+0x08(SB)/8, $0x1a120a0229211911 +DATA expandAVX512_6_outShufLo+0x10(SB)/8, $0x2b231b130b032a22 +DATA expandAVX512_6_outShufLo+0x18(SB)/8, $0x0d052c241c140c04 +DATA expandAVX512_6_outShufLo+0x20(SB)/8, $0x1e160e062d251d15 +DATA expandAVX512_6_outShufLo+0x28(SB)/8, $0x2f271f170f072e26 +DATA expandAVX512_6_outShufLo+0x30(SB)/8, $0x4941686058504840 +DATA expandAVX512_6_outShufLo+0x38(SB)/8, $0x5a524a4269615951 + +GLOBL expandAVX512_6_outShufHi(SB), RODATA, $0x40 +DATA expandAVX512_6_outShufHi+0x00(SB)/8, $0x2b231b130b032a22 +DATA expandAVX512_6_outShufHi+0x08(SB)/8, $0x0d052c241c140c04 +DATA expandAVX512_6_outShufHi+0x10(SB)/8, $0x1e160e062d251d15 +DATA expandAVX512_6_outShufHi+0x18(SB)/8, $0x2f271f170f072e26 +DATA expandAVX512_6_outShufHi+0x20(SB)/8, $0x4941686058504840 +DATA expandAVX512_6_outShufHi+0x28(SB)/8, $0x5a524a4269615951 +DATA expandAVX512_6_outShufHi+0x30(SB)/8, $0x6b635b534b436a62 +DATA expandAVX512_6_outShufHi+0x38(SB)/8, $0x4d456c645c544c44 + +TEXT expandAVX512_6<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_6_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_6_mat0<>(SB), Z3 + VMOVDQU64 expandAVX512_6_inShuf1<>(SB), Z4 + VMOVDQU64 expandAVX512_6_inShuf2<>(SB), Z5 + VMOVDQU64 expandAVX512_6_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_6_outShufHi(SB), Z2 + VMOVDQU64 (AX), Z6 + VPERMB Z6, Z0, Z0 + VGF2P8AFFINEQB $0, Z3, Z0, Z0 + VPERMB Z6, Z4, Z4 + VGF2P8AFFINEQB $0, Z3, Z4, Z4 + VPERMB Z6, Z5, Z5 + VGF2P8AFFINEQB $0, Z3, Z5, Z3 + VPERMI2B Z4, Z0, Z1 + VPERMI2B Z3, Z4, Z2 + RET + +GLOBL expandAVX512_8_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_8_inShuf0<>+0x00(SB)/8, $0x0706050403020100 +DATA expandAVX512_8_inShuf0<>+0x08(SB)/8, $0x0706050403020100 +DATA expandAVX512_8_inShuf0<>+0x10(SB)/8, $0x0706050403020100 +DATA expandAVX512_8_inShuf0<>+0x18(SB)/8, $0x0706050403020100 +DATA expandAVX512_8_inShuf0<>+0x20(SB)/8, $0x0706050403020100 +DATA expandAVX512_8_inShuf0<>+0x28(SB)/8, $0x0706050403020100 +DATA expandAVX512_8_inShuf0<>+0x30(SB)/8, $0x0706050403020100 +DATA expandAVX512_8_inShuf0<>+0x38(SB)/8, $0x0706050403020100 + +GLOBL expandAVX512_8_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_8_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_8_mat0<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_8_mat0<>+0x10(SB)/8, $0x0404040404040404 +DATA expandAVX512_8_mat0<>+0x18(SB)/8, $0x0808080808080808 +DATA expandAVX512_8_mat0<>+0x20(SB)/8, $0x1010101010101010 +DATA expandAVX512_8_mat0<>+0x28(SB)/8, $0x2020202020202020 +DATA expandAVX512_8_mat0<>+0x30(SB)/8, $0x4040404040404040 +DATA expandAVX512_8_mat0<>+0x38(SB)/8, $0x8080808080808080 + +GLOBL expandAVX512_8_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_8_inShuf1<>+0x00(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_8_inShuf1<>+0x08(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_8_inShuf1<>+0x10(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_8_inShuf1<>+0x18(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_8_inShuf1<>+0x20(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_8_inShuf1<>+0x28(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_8_inShuf1<>+0x30(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_8_inShuf1<>+0x38(SB)/8, $0x0f0e0d0c0b0a0908 + +GLOBL expandAVX512_8_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_8_outShufLo+0x00(SB)/8, $0x3830282018100800 +DATA expandAVX512_8_outShufLo+0x08(SB)/8, $0x3931292119110901 +DATA expandAVX512_8_outShufLo+0x10(SB)/8, $0x3a322a221a120a02 +DATA expandAVX512_8_outShufLo+0x18(SB)/8, $0x3b332b231b130b03 +DATA expandAVX512_8_outShufLo+0x20(SB)/8, $0x3c342c241c140c04 +DATA expandAVX512_8_outShufLo+0x28(SB)/8, $0x3d352d251d150d05 +DATA expandAVX512_8_outShufLo+0x30(SB)/8, $0x3e362e261e160e06 +DATA expandAVX512_8_outShufLo+0x38(SB)/8, $0x3f372f271f170f07 + +TEXT expandAVX512_8<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_8_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_8_mat0<>(SB), Z1 + VMOVDQU64 expandAVX512_8_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_8_outShufLo(SB), Z3 + VMOVDQU64 (AX), Z4 + VPERMB Z4, Z0, Z0 + VGF2P8AFFINEQB $0, Z1, Z0, Z0 + VPERMB Z4, Z2, Z2 + VGF2P8AFFINEQB $0, Z1, Z2, Z2 + VPERMB Z0, Z3, Z1 + VPERMB Z2, Z3, Z2 + RET + +GLOBL expandAVX512_10_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_10_inShuf0<>+0x00(SB)/8, $0xff06050403020100 +DATA expandAVX512_10_inShuf0<>+0x08(SB)/8, $0xff06050403020100 +DATA expandAVX512_10_inShuf0<>+0x10(SB)/8, $0xff06050403020100 +DATA expandAVX512_10_inShuf0<>+0x18(SB)/8, $0xff06050403020100 +DATA expandAVX512_10_inShuf0<>+0x20(SB)/8, $0xffff050403020100 +DATA expandAVX512_10_inShuf0<>+0x28(SB)/8, $0xffff050403020100 +DATA expandAVX512_10_inShuf0<>+0x30(SB)/8, $0xffff050403020100 +DATA expandAVX512_10_inShuf0<>+0x38(SB)/8, $0xffff050403020100 + +GLOBL expandAVX512_10_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_10_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_10_mat0<>+0x08(SB)/8, $0x0101020202020202 +DATA expandAVX512_10_mat0<>+0x10(SB)/8, $0x0202020204040404 +DATA expandAVX512_10_mat0<>+0x18(SB)/8, $0x0404040404040808 +DATA expandAVX512_10_mat0<>+0x20(SB)/8, $0x0808080808080808 +DATA expandAVX512_10_mat0<>+0x28(SB)/8, $0x1010101010101010 +DATA expandAVX512_10_mat0<>+0x30(SB)/8, $0x1010202020202020 +DATA expandAVX512_10_mat0<>+0x38(SB)/8, $0x2020202040404040 + +GLOBL expandAVX512_10_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_10_inShuf1<>+0x00(SB)/8, $0xffff050403020100 +DATA expandAVX512_10_inShuf1<>+0x08(SB)/8, $0xffff050403020100 +DATA expandAVX512_10_inShuf1<>+0x10(SB)/8, $0xff0c0b0a09080706 +DATA expandAVX512_10_inShuf1<>+0x18(SB)/8, $0xff0c0b0a09080706 +DATA expandAVX512_10_inShuf1<>+0x20(SB)/8, $0xff0c0b0a09080706 +DATA expandAVX512_10_inShuf1<>+0x28(SB)/8, $0xff0c0b0a09080706 +DATA expandAVX512_10_inShuf1<>+0x30(SB)/8, $0xffff0b0a09080706 +DATA expandAVX512_10_inShuf1<>+0x38(SB)/8, $0xffff0b0a09080706 + +GLOBL expandAVX512_10_mat1<>(SB), RODATA, $0x40 +DATA expandAVX512_10_mat1<>+0x00(SB)/8, $0x4040404040408080 +DATA expandAVX512_10_mat1<>+0x08(SB)/8, $0x8080808080808080 +DATA expandAVX512_10_mat1<>+0x10(SB)/8, $0x0808080808080808 +DATA expandAVX512_10_mat1<>+0x18(SB)/8, $0x1010101010101010 +DATA expandAVX512_10_mat1<>+0x20(SB)/8, $0x1010202020202020 +DATA expandAVX512_10_mat1<>+0x28(SB)/8, $0x2020202040404040 +DATA expandAVX512_10_mat1<>+0x30(SB)/8, $0x4040404040408080 +DATA expandAVX512_10_mat1<>+0x38(SB)/8, $0x8080808080808080 + +GLOBL expandAVX512_10_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_10_inShuf2<>+0x00(SB)/8, $0xffff0c0b0a090807 +DATA expandAVX512_10_inShuf2<>+0x08(SB)/8, $0xffff0c0b0a090807 +DATA expandAVX512_10_inShuf2<>+0x10(SB)/8, $0xffff0c0b0a090807 +DATA expandAVX512_10_inShuf2<>+0x18(SB)/8, $0xffff0c0b0a090807 +DATA expandAVX512_10_inShuf2<>+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_10_inShuf2<>+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_10_inShuf2<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_10_inShuf2<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_10_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_10_mat2<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_10_mat2<>+0x08(SB)/8, $0x0101020202020202 +DATA expandAVX512_10_mat2<>+0x10(SB)/8, $0x0202020204040404 +DATA expandAVX512_10_mat2<>+0x18(SB)/8, $0x0404040404040808 +DATA expandAVX512_10_mat2<>+0x20(SB)/8, $0x0000000000000000 +DATA expandAVX512_10_mat2<>+0x28(SB)/8, $0x0000000000000000 +DATA expandAVX512_10_mat2<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_10_mat2<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_10_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_10_outShufLo+0x00(SB)/8, $0x3830282018100800 +DATA expandAVX512_10_outShufLo+0x08(SB)/8, $0x2921191109014840 +DATA expandAVX512_10_outShufLo+0x10(SB)/8, $0x1a120a0249413931 +DATA expandAVX512_10_outShufLo+0x18(SB)/8, $0x0b034a423a322a22 +DATA expandAVX512_10_outShufLo+0x20(SB)/8, $0x4b433b332b231b13 +DATA expandAVX512_10_outShufLo+0x28(SB)/8, $0x3c342c241c140c04 +DATA expandAVX512_10_outShufLo+0x30(SB)/8, $0x2d251d150d054c44 +DATA expandAVX512_10_outShufLo+0x38(SB)/8, $0x1e160e064d453d35 + +GLOBL expandAVX512_10_outShufHi(SB), RODATA, $0x40 +DATA expandAVX512_10_outShufHi+0x00(SB)/8, $0x4840383028201810 +DATA expandAVX512_10_outShufHi+0x08(SB)/8, $0x3931292119115850 +DATA expandAVX512_10_outShufHi+0x10(SB)/8, $0x2a221a1259514941 +DATA expandAVX512_10_outShufHi+0x18(SB)/8, $0x1b135a524a423a32 +DATA expandAVX512_10_outShufHi+0x20(SB)/8, $0x5b534b433b332b23 +DATA expandAVX512_10_outShufHi+0x28(SB)/8, $0x4c443c342c241c14 +DATA expandAVX512_10_outShufHi+0x30(SB)/8, $0x3d352d251d155c54 +DATA expandAVX512_10_outShufHi+0x38(SB)/8, $0x2e261e165d554d45 + +TEXT expandAVX512_10<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_10_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_10_inShuf1<>(SB), Z3 + VMOVDQU64 expandAVX512_10_inShuf2<>(SB), Z4 + VMOVDQU64 expandAVX512_10_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_10_outShufHi(SB), Z2 + VMOVDQU64 (AX), Z5 + VPERMB Z5, Z0, Z0 + VGF2P8AFFINEQB $0, expandAVX512_10_mat0<>(SB), Z0, Z0 + VPERMB Z5, Z3, Z3 + VGF2P8AFFINEQB $0, expandAVX512_10_mat1<>(SB), Z3, Z3 + VPERMB Z5, Z4, Z4 + VGF2P8AFFINEQB $0, expandAVX512_10_mat2<>(SB), Z4, Z4 + VPERMI2B Z3, Z0, Z1 + VPERMI2B Z4, Z3, Z2 + RET + +GLOBL expandAVX512_12_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_12_inShuf0<>+0x00(SB)/8, $0xffff050403020100 +DATA expandAVX512_12_inShuf0<>+0x08(SB)/8, $0xffff050403020100 +DATA expandAVX512_12_inShuf0<>+0x10(SB)/8, $0xffff050403020100 +DATA expandAVX512_12_inShuf0<>+0x18(SB)/8, $0xffff050403020100 +DATA expandAVX512_12_inShuf0<>+0x20(SB)/8, $0xffffff0403020100 +DATA expandAVX512_12_inShuf0<>+0x28(SB)/8, $0xffffff0403020100 +DATA expandAVX512_12_inShuf0<>+0x30(SB)/8, $0xffffff0403020100 +DATA expandAVX512_12_inShuf0<>+0x38(SB)/8, $0xffffff0403020100 + +GLOBL expandAVX512_12_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_12_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_12_mat0<>+0x08(SB)/8, $0x0101010102020202 +DATA expandAVX512_12_mat0<>+0x10(SB)/8, $0x0202020202020202 +DATA expandAVX512_12_mat0<>+0x18(SB)/8, $0x0404040404040404 +DATA expandAVX512_12_mat0<>+0x20(SB)/8, $0x0404040408080808 +DATA expandAVX512_12_mat0<>+0x28(SB)/8, $0x0808080808080808 +DATA expandAVX512_12_mat0<>+0x30(SB)/8, $0x1010101010101010 +DATA expandAVX512_12_mat0<>+0x38(SB)/8, $0x1010101020202020 + +GLOBL expandAVX512_12_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_12_inShuf1<>+0x00(SB)/8, $0xffffff0403020100 +DATA expandAVX512_12_inShuf1<>+0x08(SB)/8, $0xffffff0403020100 +DATA expandAVX512_12_inShuf1<>+0x10(SB)/8, $0xffffff0403020100 +DATA expandAVX512_12_inShuf1<>+0x18(SB)/8, $0xffffff0403020100 +DATA expandAVX512_12_inShuf1<>+0x20(SB)/8, $0xffff0a0908070605 +DATA expandAVX512_12_inShuf1<>+0x28(SB)/8, $0xffff0a0908070605 +DATA expandAVX512_12_inShuf1<>+0x30(SB)/8, $0xffff0a0908070605 +DATA expandAVX512_12_inShuf1<>+0x38(SB)/8, $0xffff0a0908070605 + +GLOBL expandAVX512_12_mat1<>(SB), RODATA, $0x40 +DATA expandAVX512_12_mat1<>+0x00(SB)/8, $0x2020202020202020 +DATA expandAVX512_12_mat1<>+0x08(SB)/8, $0x4040404040404040 +DATA expandAVX512_12_mat1<>+0x10(SB)/8, $0x4040404080808080 +DATA expandAVX512_12_mat1<>+0x18(SB)/8, $0x8080808080808080 +DATA expandAVX512_12_mat1<>+0x20(SB)/8, $0x0404040408080808 +DATA expandAVX512_12_mat1<>+0x28(SB)/8, $0x0808080808080808 +DATA expandAVX512_12_mat1<>+0x30(SB)/8, $0x1010101010101010 +DATA expandAVX512_12_mat1<>+0x38(SB)/8, $0x1010101020202020 + +GLOBL expandAVX512_12_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_12_inShuf2<>+0x00(SB)/8, $0xffffff0908070605 +DATA expandAVX512_12_inShuf2<>+0x08(SB)/8, $0xffffff0908070605 +DATA expandAVX512_12_inShuf2<>+0x10(SB)/8, $0xffffff0908070605 +DATA expandAVX512_12_inShuf2<>+0x18(SB)/8, $0xffffff0908070605 +DATA expandAVX512_12_inShuf2<>+0x20(SB)/8, $0xffffff0a09080706 +DATA expandAVX512_12_inShuf2<>+0x28(SB)/8, $0xffffff0a09080706 +DATA expandAVX512_12_inShuf2<>+0x30(SB)/8, $0xffffff0a09080706 +DATA expandAVX512_12_inShuf2<>+0x38(SB)/8, $0xffffff0a09080706 + +GLOBL expandAVX512_12_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_12_mat2<>+0x00(SB)/8, $0x2020202020202020 +DATA expandAVX512_12_mat2<>+0x08(SB)/8, $0x4040404040404040 +DATA expandAVX512_12_mat2<>+0x10(SB)/8, $0x4040404080808080 +DATA expandAVX512_12_mat2<>+0x18(SB)/8, $0x8080808080808080 +DATA expandAVX512_12_mat2<>+0x20(SB)/8, $0x0101010101010101 +DATA expandAVX512_12_mat2<>+0x28(SB)/8, $0x0101010102020202 +DATA expandAVX512_12_mat2<>+0x30(SB)/8, $0x0202020202020202 +DATA expandAVX512_12_mat2<>+0x38(SB)/8, $0x0404040404040404 + +GLOBL expandAVX512_12_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_12_outShufLo+0x00(SB)/8, $0x3830282018100800 +DATA expandAVX512_12_outShufLo+0x08(SB)/8, $0x1911090158504840 +DATA expandAVX512_12_outShufLo+0x10(SB)/8, $0x5951494139312921 +DATA expandAVX512_12_outShufLo+0x18(SB)/8, $0x3a322a221a120a02 +DATA expandAVX512_12_outShufLo+0x20(SB)/8, $0x1b130b035a524a42 +DATA expandAVX512_12_outShufLo+0x28(SB)/8, $0x5b534b433b332b23 +DATA expandAVX512_12_outShufLo+0x30(SB)/8, $0x3c342c241c140c04 +DATA expandAVX512_12_outShufLo+0x38(SB)/8, $0x1d150d055c544c44 + +GLOBL expandAVX512_12_outShufHi(SB), RODATA, $0x40 +DATA expandAVX512_12_outShufHi+0x00(SB)/8, $0x5850484038302820 +DATA expandAVX512_12_outShufHi+0x08(SB)/8, $0x3931292178706860 +DATA expandAVX512_12_outShufHi+0x10(SB)/8, $0x7971696159514941 +DATA expandAVX512_12_outShufHi+0x18(SB)/8, $0x5a524a423a322a22 +DATA expandAVX512_12_outShufHi+0x20(SB)/8, $0x3b332b237a726a62 +DATA expandAVX512_12_outShufHi+0x28(SB)/8, $0x7b736b635b534b43 +DATA expandAVX512_12_outShufHi+0x30(SB)/8, $0x5c544c443c342c24 +DATA expandAVX512_12_outShufHi+0x38(SB)/8, $0x3d352d257c746c64 + +TEXT expandAVX512_12<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_12_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_12_inShuf1<>(SB), Z3 + VMOVDQU64 expandAVX512_12_inShuf2<>(SB), Z4 + VMOVDQU64 expandAVX512_12_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_12_outShufHi(SB), Z2 + VMOVDQU64 (AX), Z5 + VPERMB Z5, Z0, Z0 + VGF2P8AFFINEQB $0, expandAVX512_12_mat0<>(SB), Z0, Z0 + VPERMB Z5, Z3, Z3 + VGF2P8AFFINEQB $0, expandAVX512_12_mat1<>(SB), Z3, Z3 + VPERMB Z5, Z4, Z4 + VGF2P8AFFINEQB $0, expandAVX512_12_mat2<>(SB), Z4, Z4 + VPERMI2B Z3, Z0, Z1 + VPERMI2B Z4, Z3, Z2 + RET + +GLOBL expandAVX512_14_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_14_inShuf0<>+0x00(SB)/8, $0xffffff0403020100 +DATA expandAVX512_14_inShuf0<>+0x08(SB)/8, $0xffffff0403020100 +DATA expandAVX512_14_inShuf0<>+0x10(SB)/8, $0xffffff0403020100 +DATA expandAVX512_14_inShuf0<>+0x18(SB)/8, $0xffffff0403020100 +DATA expandAVX512_14_inShuf0<>+0x20(SB)/8, $0xffffff0403020100 +DATA expandAVX512_14_inShuf0<>+0x28(SB)/8, $0xffffff0403020100 +DATA expandAVX512_14_inShuf0<>+0x30(SB)/8, $0xffffff0403020100 +DATA expandAVX512_14_inShuf0<>+0x38(SB)/8, $0xffffff0403020100 + +GLOBL expandAVX512_14_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_14_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_14_mat0<>+0x08(SB)/8, $0x0101010101010202 +DATA expandAVX512_14_mat0<>+0x10(SB)/8, $0x0202020202020202 +DATA expandAVX512_14_mat0<>+0x18(SB)/8, $0x0202020204040404 +DATA expandAVX512_14_mat0<>+0x20(SB)/8, $0x0404040404040404 +DATA expandAVX512_14_mat0<>+0x28(SB)/8, $0x0404080808080808 +DATA expandAVX512_14_mat0<>+0x30(SB)/8, $0x0808080808080808 +DATA expandAVX512_14_mat0<>+0x38(SB)/8, $0x1010101010101010 + +GLOBL expandAVX512_14_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_14_inShuf1<>+0x00(SB)/8, $0xffffffff03020100 +DATA expandAVX512_14_inShuf1<>+0x08(SB)/8, $0xffffffff03020100 +DATA expandAVX512_14_inShuf1<>+0x10(SB)/8, $0xffffffff03020100 +DATA expandAVX512_14_inShuf1<>+0x18(SB)/8, $0xffffffff03020100 +DATA expandAVX512_14_inShuf1<>+0x20(SB)/8, $0xffffffff03020100 +DATA expandAVX512_14_inShuf1<>+0x28(SB)/8, $0xffffffff03020100 +DATA expandAVX512_14_inShuf1<>+0x30(SB)/8, $0xffffff0807060504 +DATA expandAVX512_14_inShuf1<>+0x38(SB)/8, $0xffffff0807060504 + +GLOBL expandAVX512_14_mat1<>(SB), RODATA, $0x40 +DATA expandAVX512_14_mat1<>+0x00(SB)/8, $0x1010101010102020 +DATA expandAVX512_14_mat1<>+0x08(SB)/8, $0x2020202020202020 +DATA expandAVX512_14_mat1<>+0x10(SB)/8, $0x2020202040404040 +DATA expandAVX512_14_mat1<>+0x18(SB)/8, $0x4040404040404040 +DATA expandAVX512_14_mat1<>+0x20(SB)/8, $0x4040808080808080 +DATA expandAVX512_14_mat1<>+0x28(SB)/8, $0x8080808080808080 +DATA expandAVX512_14_mat1<>+0x30(SB)/8, $0x1010101010102020 +DATA expandAVX512_14_mat1<>+0x38(SB)/8, $0x2020202020202020 + +GLOBL expandAVX512_14_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_14_inShuf2<>+0x00(SB)/8, $0xffffff0807060504 +DATA expandAVX512_14_inShuf2<>+0x08(SB)/8, $0xffffff0807060504 +DATA expandAVX512_14_inShuf2<>+0x10(SB)/8, $0xffffff0807060504 +DATA expandAVX512_14_inShuf2<>+0x18(SB)/8, $0xffffff0807060504 +DATA expandAVX512_14_inShuf2<>+0x20(SB)/8, $0xffffff0908070605 +DATA expandAVX512_14_inShuf2<>+0x28(SB)/8, $0xffffff0908070605 +DATA expandAVX512_14_inShuf2<>+0x30(SB)/8, $0xffffffff08070605 +DATA expandAVX512_14_inShuf2<>+0x38(SB)/8, $0xffffffff08070605 + +GLOBL expandAVX512_14_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_14_mat2<>+0x00(SB)/8, $0x2020202040404040 +DATA expandAVX512_14_mat2<>+0x08(SB)/8, $0x4040404040404040 +DATA expandAVX512_14_mat2<>+0x10(SB)/8, $0x4040808080808080 +DATA expandAVX512_14_mat2<>+0x18(SB)/8, $0x8080808080808080 +DATA expandAVX512_14_mat2<>+0x20(SB)/8, $0x0101010101010101 +DATA expandAVX512_14_mat2<>+0x28(SB)/8, $0x0101010101010202 +DATA expandAVX512_14_mat2<>+0x30(SB)/8, $0x0202020202020202 +DATA expandAVX512_14_mat2<>+0x38(SB)/8, $0x0202020204040404 + +GLOBL expandAVX512_14_inShuf3<>(SB), RODATA, $0x40 +DATA expandAVX512_14_inShuf3<>+0x00(SB)/8, $0xffffffff08070605 +DATA expandAVX512_14_inShuf3<>+0x08(SB)/8, $0xffffffff08070605 +DATA expandAVX512_14_inShuf3<>+0x10(SB)/8, $0xffffffff08070605 +DATA expandAVX512_14_inShuf3<>+0x18(SB)/8, $0xffffffff08070605 +DATA expandAVX512_14_inShuf3<>+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_14_inShuf3<>+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_14_inShuf3<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_14_inShuf3<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_14_mat3<>(SB), RODATA, $0x40 +DATA expandAVX512_14_mat3<>+0x00(SB)/8, $0x0404040404040404 +DATA expandAVX512_14_mat3<>+0x08(SB)/8, $0x0404080808080808 +DATA expandAVX512_14_mat3<>+0x10(SB)/8, $0x0808080808080808 +DATA expandAVX512_14_mat3<>+0x18(SB)/8, $0x1010101010101010 +DATA expandAVX512_14_mat3<>+0x20(SB)/8, $0x0000000000000000 +DATA expandAVX512_14_mat3<>+0x28(SB)/8, $0x0000000000000000 +DATA expandAVX512_14_mat3<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_14_mat3<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_14_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_14_outShufLo+0x00(SB)/8, $0x3830282018100800 +DATA expandAVX512_14_outShufLo+0x08(SB)/8, $0x0901686058504840 +DATA expandAVX512_14_outShufLo+0x10(SB)/8, $0x4941393129211911 +DATA expandAVX512_14_outShufLo+0x18(SB)/8, $0x1a120a0269615951 +DATA expandAVX512_14_outShufLo+0x20(SB)/8, $0x5a524a423a322a22 +DATA expandAVX512_14_outShufLo+0x28(SB)/8, $0x2b231b130b036a62 +DATA expandAVX512_14_outShufLo+0x30(SB)/8, $0x6b635b534b433b33 +DATA expandAVX512_14_outShufLo+0x38(SB)/8, $0x3c342c241c140c04 + +GLOBL expandAVX512_14_outShufHi0(SB), RODATA, $0x40 +DATA expandAVX512_14_outShufHi0+0x00(SB)/8, $0x6860585048403830 +DATA expandAVX512_14_outShufHi0+0x08(SB)/8, $0x3931ffffffff7870 +DATA expandAVX512_14_outShufHi0+0x10(SB)/8, $0x7971696159514941 +DATA expandAVX512_14_outShufHi0+0x18(SB)/8, $0x4a423a32ffffffff +DATA expandAVX512_14_outShufHi0+0x20(SB)/8, $0xffff7a726a625a52 +DATA expandAVX512_14_outShufHi0+0x28(SB)/8, $0x5b534b433b33ffff +DATA expandAVX512_14_outShufHi0+0x30(SB)/8, $0xffffffff7b736b63 +DATA expandAVX512_14_outShufHi0+0x38(SB)/8, $0x6c645c544c443c34 + +GLOBL expandAVX512_14_outShufHi1(SB), RODATA, $0x40 +DATA expandAVX512_14_outShufHi1+0x00(SB)/8, $0xffffffffffffffff +DATA expandAVX512_14_outShufHi1+0x08(SB)/8, $0xffff18100800ffff +DATA expandAVX512_14_outShufHi1+0x10(SB)/8, $0xffffffffffffffff +DATA expandAVX512_14_outShufHi1+0x18(SB)/8, $0xffffffff19110901 +DATA expandAVX512_14_outShufHi1+0x20(SB)/8, $0x0a02ffffffffffff +DATA expandAVX512_14_outShufHi1+0x28(SB)/8, $0xffffffffffff1a12 +DATA expandAVX512_14_outShufHi1+0x30(SB)/8, $0x1b130b03ffffffff +DATA expandAVX512_14_outShufHi1+0x38(SB)/8, $0xffffffffffffffff + +TEXT expandAVX512_14<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_14_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_14_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_14_inShuf2<>(SB), Z3 + VMOVDQU64 expandAVX512_14_inShuf3<>(SB), Z4 + VMOVDQU64 expandAVX512_14_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_14_outShufHi0(SB), Z5 + VMOVDQU64 expandAVX512_14_outShufHi1(SB), Z6 + VMOVDQU64 (AX), Z7 + VPERMB Z7, Z0, Z0 + VGF2P8AFFINEQB $0, expandAVX512_14_mat0<>(SB), Z0, Z0 + VPERMB Z7, Z2, Z2 + VGF2P8AFFINEQB $0, expandAVX512_14_mat1<>(SB), Z2, Z2 + VPERMB Z7, Z3, Z3 + VGF2P8AFFINEQB $0, expandAVX512_14_mat2<>(SB), Z3, Z3 + VPERMB Z7, Z4, Z4 + VGF2P8AFFINEQB $0, expandAVX512_14_mat3<>(SB), Z4, Z4 + VPERMI2B Z2, Z0, Z1 + MOVQ $0xff0ffc3ff0ffc3ff, AX + KMOVQ AX, K1 + VPERMI2B.Z Z3, Z2, K1, Z5 + MOVQ $0xf003c00f003c00, AX + KMOVQ AX, K1 + VPERMB.Z Z4, Z6, K1, Z0 + VPORQ Z0, Z5, Z2 + RET + +GLOBL expandAVX512_16_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_16_inShuf0<>+0x00(SB)/8, $0x0303020201010000 +DATA expandAVX512_16_inShuf0<>+0x08(SB)/8, $0x0303020201010000 +DATA expandAVX512_16_inShuf0<>+0x10(SB)/8, $0x0303020201010000 +DATA expandAVX512_16_inShuf0<>+0x18(SB)/8, $0x0303020201010000 +DATA expandAVX512_16_inShuf0<>+0x20(SB)/8, $0x0303020201010000 +DATA expandAVX512_16_inShuf0<>+0x28(SB)/8, $0x0303020201010000 +DATA expandAVX512_16_inShuf0<>+0x30(SB)/8, $0x0303020201010000 +DATA expandAVX512_16_inShuf0<>+0x38(SB)/8, $0x0303020201010000 + +GLOBL expandAVX512_16_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_16_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_16_mat0<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_16_mat0<>+0x10(SB)/8, $0x0404040404040404 +DATA expandAVX512_16_mat0<>+0x18(SB)/8, $0x0808080808080808 +DATA expandAVX512_16_mat0<>+0x20(SB)/8, $0x1010101010101010 +DATA expandAVX512_16_mat0<>+0x28(SB)/8, $0x2020202020202020 +DATA expandAVX512_16_mat0<>+0x30(SB)/8, $0x4040404040404040 +DATA expandAVX512_16_mat0<>+0x38(SB)/8, $0x8080808080808080 + +GLOBL expandAVX512_16_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_16_inShuf1<>+0x00(SB)/8, $0x0707060605050404 +DATA expandAVX512_16_inShuf1<>+0x08(SB)/8, $0x0707060605050404 +DATA expandAVX512_16_inShuf1<>+0x10(SB)/8, $0x0707060605050404 +DATA expandAVX512_16_inShuf1<>+0x18(SB)/8, $0x0707060605050404 +DATA expandAVX512_16_inShuf1<>+0x20(SB)/8, $0x0707060605050404 +DATA expandAVX512_16_inShuf1<>+0x28(SB)/8, $0x0707060605050404 +DATA expandAVX512_16_inShuf1<>+0x30(SB)/8, $0x0707060605050404 +DATA expandAVX512_16_inShuf1<>+0x38(SB)/8, $0x0707060605050404 + +GLOBL expandAVX512_16_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_16_outShufLo+0x00(SB)/8, $0x1918111009080100 +DATA expandAVX512_16_outShufLo+0x08(SB)/8, $0x3938313029282120 +DATA expandAVX512_16_outShufLo+0x10(SB)/8, $0x1b1a13120b0a0302 +DATA expandAVX512_16_outShufLo+0x18(SB)/8, $0x3b3a33322b2a2322 +DATA expandAVX512_16_outShufLo+0x20(SB)/8, $0x1d1c15140d0c0504 +DATA expandAVX512_16_outShufLo+0x28(SB)/8, $0x3d3c35342d2c2524 +DATA expandAVX512_16_outShufLo+0x30(SB)/8, $0x1f1e17160f0e0706 +DATA expandAVX512_16_outShufLo+0x38(SB)/8, $0x3f3e37362f2e2726 + +TEXT expandAVX512_16<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_16_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_16_mat0<>(SB), Z1 + VMOVDQU64 expandAVX512_16_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_16_outShufLo(SB), Z3 + VMOVDQU64 (AX), Z4 + VPERMB Z4, Z0, Z0 + VGF2P8AFFINEQB $0, Z1, Z0, Z0 + VPERMB Z4, Z2, Z2 + VGF2P8AFFINEQB $0, Z1, Z2, Z2 + VPERMB Z0, Z3, Z1 + VPERMB Z2, Z3, Z2 + RET + +GLOBL expandAVX512_18_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_18_inShuf0<>+0x00(SB)/8, $0x0303020201010000 +DATA expandAVX512_18_inShuf0<>+0x08(SB)/8, $0xffffffff03020100 +DATA expandAVX512_18_inShuf0<>+0x10(SB)/8, $0xffffffff03020100 +DATA expandAVX512_18_inShuf0<>+0x18(SB)/8, $0xffffffff03020100 +DATA expandAVX512_18_inShuf0<>+0x20(SB)/8, $0xffffffff03020100 +DATA expandAVX512_18_inShuf0<>+0x28(SB)/8, $0xffffffff03020100 +DATA expandAVX512_18_inShuf0<>+0x30(SB)/8, $0x0303020201010000 +DATA expandAVX512_18_inShuf0<>+0x38(SB)/8, $0xff03020201010000 + +GLOBL expandAVX512_18_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_18_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_18_mat0<>+0x08(SB)/8, $0x0101020202020202 +DATA expandAVX512_18_mat0<>+0x10(SB)/8, $0x0202020202020202 +DATA expandAVX512_18_mat0<>+0x18(SB)/8, $0x0202020204040404 +DATA expandAVX512_18_mat0<>+0x20(SB)/8, $0x0404040404040404 +DATA expandAVX512_18_mat0<>+0x28(SB)/8, $0x0404040404040808 +DATA expandAVX512_18_mat0<>+0x30(SB)/8, $0x0808080808080808 +DATA expandAVX512_18_mat0<>+0x38(SB)/8, $0x1010101010101010 + +GLOBL expandAVX512_18_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_18_inShuf1<>+0x00(SB)/8, $0xffffffffff020100 +DATA expandAVX512_18_inShuf1<>+0x08(SB)/8, $0xffffffffff020100 +DATA expandAVX512_18_inShuf1<>+0x10(SB)/8, $0xffffffffff020100 +DATA expandAVX512_18_inShuf1<>+0x18(SB)/8, $0xffffffffff020100 +DATA expandAVX512_18_inShuf1<>+0x20(SB)/8, $0xffffffffff020100 +DATA expandAVX512_18_inShuf1<>+0x28(SB)/8, $0xffff020201010000 +DATA expandAVX512_18_inShuf1<>+0x30(SB)/8, $0xff06060505040403 +DATA expandAVX512_18_inShuf1<>+0x38(SB)/8, $0xffffffff06050403 + +GLOBL expandAVX512_18_mat1<>(SB), RODATA, $0x40 +DATA expandAVX512_18_mat1<>+0x00(SB)/8, $0x1010202020202020 +DATA expandAVX512_18_mat1<>+0x08(SB)/8, $0x2020202020202020 +DATA expandAVX512_18_mat1<>+0x10(SB)/8, $0x2020202040404040 +DATA expandAVX512_18_mat1<>+0x18(SB)/8, $0x4040404040404040 +DATA expandAVX512_18_mat1<>+0x20(SB)/8, $0x4040404040408080 +DATA expandAVX512_18_mat1<>+0x28(SB)/8, $0x8080808080808080 +DATA expandAVX512_18_mat1<>+0x30(SB)/8, $0x1010101010101010 +DATA expandAVX512_18_mat1<>+0x38(SB)/8, $0x1010202020202020 + +GLOBL expandAVX512_18_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_18_inShuf2<>+0x00(SB)/8, $0xffffffff06050403 +DATA expandAVX512_18_inShuf2<>+0x08(SB)/8, $0xffffffff06050403 +DATA expandAVX512_18_inShuf2<>+0x10(SB)/8, $0xffffffff06050403 +DATA expandAVX512_18_inShuf2<>+0x18(SB)/8, $0xffffffff06050403 +DATA expandAVX512_18_inShuf2<>+0x20(SB)/8, $0x0606050504040303 +DATA expandAVX512_18_inShuf2<>+0x28(SB)/8, $0x0707060605050404 +DATA expandAVX512_18_inShuf2<>+0x30(SB)/8, $0xffffffffff060504 +DATA expandAVX512_18_inShuf2<>+0x38(SB)/8, $0xffffffffff060504 + +GLOBL expandAVX512_18_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_18_mat2<>+0x00(SB)/8, $0x2020202020202020 +DATA expandAVX512_18_mat2<>+0x08(SB)/8, $0x2020202040404040 +DATA expandAVX512_18_mat2<>+0x10(SB)/8, $0x4040404040404040 +DATA expandAVX512_18_mat2<>+0x18(SB)/8, $0x4040404040408080 +DATA expandAVX512_18_mat2<>+0x20(SB)/8, $0x8080808080808080 +DATA expandAVX512_18_mat2<>+0x28(SB)/8, $0x0101010101010101 +DATA expandAVX512_18_mat2<>+0x30(SB)/8, $0x0101020202020202 +DATA expandAVX512_18_mat2<>+0x38(SB)/8, $0x0202020202020202 + +GLOBL expandAVX512_18_inShuf3<>(SB), RODATA, $0x40 +DATA expandAVX512_18_inShuf3<>+0x00(SB)/8, $0xffffffffff060504 +DATA expandAVX512_18_inShuf3<>+0x08(SB)/8, $0xffffffffff060504 +DATA expandAVX512_18_inShuf3<>+0x10(SB)/8, $0xffffffffff060504 +DATA expandAVX512_18_inShuf3<>+0x18(SB)/8, $0xffff060605050404 +DATA expandAVX512_18_inShuf3<>+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_18_inShuf3<>+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_18_inShuf3<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_18_inShuf3<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_18_mat3<>(SB), RODATA, $0x40 +DATA expandAVX512_18_mat3<>+0x00(SB)/8, $0x0202020204040404 +DATA expandAVX512_18_mat3<>+0x08(SB)/8, $0x0404040404040404 +DATA expandAVX512_18_mat3<>+0x10(SB)/8, $0x0404040404040808 +DATA expandAVX512_18_mat3<>+0x18(SB)/8, $0x0808080808080808 +DATA expandAVX512_18_mat3<>+0x20(SB)/8, $0x0000000000000000 +DATA expandAVX512_18_mat3<>+0x28(SB)/8, $0x0000000000000000 +DATA expandAVX512_18_mat3<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_18_mat3<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_18_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_18_outShufLo+0x00(SB)/8, $0x3028201810080100 +DATA expandAVX512_18_outShufLo+0x08(SB)/8, $0x6058504840393831 +DATA expandAVX512_18_outShufLo+0x10(SB)/8, $0x2119110903026968 +DATA expandAVX512_18_outShufLo+0x18(SB)/8, $0x5149413b3a333229 +DATA expandAVX512_18_outShufLo+0x20(SB)/8, $0x120a05046b6a6159 +DATA expandAVX512_18_outShufLo+0x28(SB)/8, $0x423d3c35342a221a +DATA expandAVX512_18_outShufLo+0x30(SB)/8, $0x07066d6c625a524a +DATA expandAVX512_18_outShufLo+0x38(SB)/8, $0x3e37362b231b130b + +GLOBL expandAVX512_18_outShufHi0(SB), RODATA, $0x40 +DATA expandAVX512_18_outShufHi0+0x00(SB)/8, $0x6160585048403830 +DATA expandAVX512_18_outShufHi0+0x08(SB)/8, $0xffffffff78706968 +DATA expandAVX512_18_outShufHi0+0x10(SB)/8, $0x59514941393231ff +DATA expandAVX512_18_outShufHi0+0x18(SB)/8, $0xffff79716b6a6362 +DATA expandAVX512_18_outShufHi0+0x20(SB)/8, $0x4a423a3433ffffff +DATA expandAVX512_18_outShufHi0+0x28(SB)/8, $0x7a726d6c65645a52 +DATA expandAVX512_18_outShufHi0+0x30(SB)/8, $0x3b3635ffffffffff +DATA expandAVX512_18_outShufHi0+0x38(SB)/8, $0x6f6e67665b534b43 + +GLOBL expandAVX512_18_outShufHi1(SB), RODATA, $0x40 +DATA expandAVX512_18_outShufHi1+0x00(SB)/8, $0xffffffffffffffff +DATA expandAVX512_18_outShufHi1+0x08(SB)/8, $0x18100800ffffffff +DATA expandAVX512_18_outShufHi1+0x10(SB)/8, $0xffffffffffffff19 +DATA expandAVX512_18_outShufHi1+0x18(SB)/8, $0x0901ffffffffffff +DATA expandAVX512_18_outShufHi1+0x20(SB)/8, $0xffffffffff1b1a11 +DATA expandAVX512_18_outShufHi1+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_18_outShufHi1+0x30(SB)/8, $0xffffff1d1c120a02 +DATA expandAVX512_18_outShufHi1+0x38(SB)/8, $0xffffffffffffffff + +TEXT expandAVX512_18<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_18_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_18_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_18_inShuf2<>(SB), Z3 + VMOVDQU64 expandAVX512_18_inShuf3<>(SB), Z4 + VMOVDQU64 expandAVX512_18_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_18_outShufHi0(SB), Z5 + VMOVDQU64 expandAVX512_18_outShufHi1(SB), Z6 + VMOVDQU64 (AX), Z7 + VPERMB Z7, Z0, Z0 + VGF2P8AFFINEQB $0, expandAVX512_18_mat0<>(SB), Z0, Z0 + VPERMB Z7, Z2, Z2 + VGF2P8AFFINEQB $0, expandAVX512_18_mat1<>(SB), Z2, Z2 + VPERMB Z7, Z3, Z3 + VGF2P8AFFINEQB $0, expandAVX512_18_mat2<>(SB), Z3, Z3 + VPERMB Z7, Z4, Z4 + VGF2P8AFFINEQB $0, expandAVX512_18_mat3<>(SB), Z4, Z4 + VPERMI2B Z2, Z0, Z1 + MOVQ $0xffe0fff83ffe0fff, AX + KMOVQ AX, K1 + VPERMI2B.Z Z3, Z2, K1, Z5 + MOVQ $0x1f0007c001f000, AX + KMOVQ AX, K1 + VPERMB.Z Z4, Z6, K1, Z0 + VPORQ Z0, Z5, Z2 + RET + +GLOBL expandAVX512_20_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_20_inShuf0<>+0x00(SB)/8, $0x0303020201010000 +DATA expandAVX512_20_inShuf0<>+0x08(SB)/8, $0xffffffff03020100 +DATA expandAVX512_20_inShuf0<>+0x10(SB)/8, $0xff03020201010000 +DATA expandAVX512_20_inShuf0<>+0x18(SB)/8, $0xffff020201010000 +DATA expandAVX512_20_inShuf0<>+0x20(SB)/8, $0xffffffffff020100 +DATA expandAVX512_20_inShuf0<>+0x28(SB)/8, $0xffff020201010000 +DATA expandAVX512_20_inShuf0<>+0x30(SB)/8, $0xffff020201010000 +DATA expandAVX512_20_inShuf0<>+0x38(SB)/8, $0xffffffffff020100 + +GLOBL expandAVX512_20_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_20_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_20_mat0<>+0x08(SB)/8, $0x0101010102020202 +DATA expandAVX512_20_mat0<>+0x10(SB)/8, $0x0202020202020202 +DATA expandAVX512_20_mat0<>+0x18(SB)/8, $0x0404040404040404 +DATA expandAVX512_20_mat0<>+0x20(SB)/8, $0x0404040408080808 +DATA expandAVX512_20_mat0<>+0x28(SB)/8, $0x0808080808080808 +DATA expandAVX512_20_mat0<>+0x30(SB)/8, $0x1010101010101010 +DATA expandAVX512_20_mat0<>+0x38(SB)/8, $0x1010101020202020 + +GLOBL expandAVX512_20_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_20_inShuf1<>+0x00(SB)/8, $0xffff020201010000 +DATA expandAVX512_20_inShuf1<>+0x08(SB)/8, $0xffff020201010000 +DATA expandAVX512_20_inShuf1<>+0x10(SB)/8, $0xffffffffff020100 +DATA expandAVX512_20_inShuf1<>+0x18(SB)/8, $0xffff020201010000 +DATA expandAVX512_20_inShuf1<>+0x20(SB)/8, $0xff06060505040403 +DATA expandAVX512_20_inShuf1<>+0x28(SB)/8, $0x0606050504040303 +DATA expandAVX512_20_inShuf1<>+0x30(SB)/8, $0xffffffff06050403 +DATA expandAVX512_20_inShuf1<>+0x38(SB)/8, $0xffff050504040303 + +GLOBL expandAVX512_20_mat1<>(SB), RODATA, $0x40 +DATA expandAVX512_20_mat1<>+0x00(SB)/8, $0x2020202020202020 +DATA expandAVX512_20_mat1<>+0x08(SB)/8, $0x4040404040404040 +DATA expandAVX512_20_mat1<>+0x10(SB)/8, $0x4040404080808080 +DATA expandAVX512_20_mat1<>+0x18(SB)/8, $0x8080808080808080 +DATA expandAVX512_20_mat1<>+0x20(SB)/8, $0x0202020202020202 +DATA expandAVX512_20_mat1<>+0x28(SB)/8, $0x0404040404040404 +DATA expandAVX512_20_mat1<>+0x30(SB)/8, $0x0404040408080808 +DATA expandAVX512_20_mat1<>+0x38(SB)/8, $0x0808080808080808 + +GLOBL expandAVX512_20_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_20_inShuf2<>+0x00(SB)/8, $0xffff050504040303 +DATA expandAVX512_20_inShuf2<>+0x08(SB)/8, $0xffffffffff050403 +DATA expandAVX512_20_inShuf2<>+0x10(SB)/8, $0xffff050504040303 +DATA expandAVX512_20_inShuf2<>+0x18(SB)/8, $0xffff050504040303 +DATA expandAVX512_20_inShuf2<>+0x20(SB)/8, $0xffffffffff050403 +DATA expandAVX512_20_inShuf2<>+0x28(SB)/8, $0xffff050504040303 +DATA expandAVX512_20_inShuf2<>+0x30(SB)/8, $0xffff060605050404 +DATA expandAVX512_20_inShuf2<>+0x38(SB)/8, $0xffffffffff060504 + +GLOBL expandAVX512_20_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_20_mat2<>+0x00(SB)/8, $0x1010101010101010 +DATA expandAVX512_20_mat2<>+0x08(SB)/8, $0x1010101020202020 +DATA expandAVX512_20_mat2<>+0x10(SB)/8, $0x2020202020202020 +DATA expandAVX512_20_mat2<>+0x18(SB)/8, $0x4040404040404040 +DATA expandAVX512_20_mat2<>+0x20(SB)/8, $0x4040404080808080 +DATA expandAVX512_20_mat2<>+0x28(SB)/8, $0x8080808080808080 +DATA expandAVX512_20_mat2<>+0x30(SB)/8, $0x0101010101010101 +DATA expandAVX512_20_mat2<>+0x38(SB)/8, $0x0101010102020202 + +GLOBL expandAVX512_20_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_20_outShufLo+0x00(SB)/8, $0x2019181110080100 +DATA expandAVX512_20_outShufLo+0x08(SB)/8, $0x4841403831302928 +DATA expandAVX512_20_outShufLo+0x10(SB)/8, $0x1209030259585049 +DATA expandAVX512_20_outShufLo+0x18(SB)/8, $0x33322b2a211b1a13 +DATA expandAVX512_20_outShufLo+0x20(SB)/8, $0x5b5a514b4a434239 +DATA expandAVX512_20_outShufLo+0x28(SB)/8, $0x221d1c15140a0504 +DATA expandAVX512_20_outShufLo+0x30(SB)/8, $0x4c45443a35342d2c +DATA expandAVX512_20_outShufLo+0x38(SB)/8, $0x160b07065d5c524d + +GLOBL expandAVX512_20_outShufHi(SB), RODATA, $0x40 +DATA expandAVX512_20_outShufHi+0x00(SB)/8, $0x4140393830292820 +DATA expandAVX512_20_outShufHi+0x08(SB)/8, $0x6968605958515048 +DATA expandAVX512_20_outShufHi+0x10(SB)/8, $0x312b2a2221787170 +DATA expandAVX512_20_outShufHi+0x18(SB)/8, $0x5a53524943423b3a +DATA expandAVX512_20_outShufHi+0x20(SB)/8, $0x237973726b6a615b +DATA expandAVX512_20_outShufHi+0x28(SB)/8, $0x45443d3c322d2c24 +DATA expandAVX512_20_outShufHi+0x30(SB)/8, $0x6d6c625d5c55544a +DATA expandAVX512_20_outShufHi+0x38(SB)/8, $0x332f2e26257a7574 + +TEXT expandAVX512_20<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_20_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_20_inShuf1<>(SB), Z3 + VMOVDQU64 expandAVX512_20_inShuf2<>(SB), Z4 + VMOVDQU64 expandAVX512_20_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_20_outShufHi(SB), Z2 + VMOVDQU64 (AX), Z5 + VPERMB Z5, Z0, Z0 + VGF2P8AFFINEQB $0, expandAVX512_20_mat0<>(SB), Z0, Z0 + VPERMB Z5, Z3, Z3 + VGF2P8AFFINEQB $0, expandAVX512_20_mat1<>(SB), Z3, Z3 + VPERMB Z5, Z4, Z4 + VGF2P8AFFINEQB $0, expandAVX512_20_mat2<>(SB), Z4, Z4 + VPERMI2B Z3, Z0, Z1 + VPERMI2B Z4, Z3, Z2 + RET + +GLOBL expandAVX512_22_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_22_inShuf0<>+0x00(SB)/8, $0xffff020201010000 +DATA expandAVX512_22_inShuf0<>+0x08(SB)/8, $0xffffffffff020100 +DATA expandAVX512_22_inShuf0<>+0x10(SB)/8, $0xffff020201010000 +DATA expandAVX512_22_inShuf0<>+0x18(SB)/8, $0xffffffffff020100 +DATA expandAVX512_22_inShuf0<>+0x20(SB)/8, $0xffff020201010000 +DATA expandAVX512_22_inShuf0<>+0x28(SB)/8, $0xffffffffff020100 +DATA expandAVX512_22_inShuf0<>+0x30(SB)/8, $0xffff020201010000 +DATA expandAVX512_22_inShuf0<>+0x38(SB)/8, $0xffff020201010000 + +GLOBL expandAVX512_22_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_22_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_22_mat0<>+0x08(SB)/8, $0x0101010101010202 +DATA expandAVX512_22_mat0<>+0x10(SB)/8, $0x0202020202020202 +DATA expandAVX512_22_mat0<>+0x18(SB)/8, $0x0202020204040404 +DATA expandAVX512_22_mat0<>+0x20(SB)/8, $0x0404040404040404 +DATA expandAVX512_22_mat0<>+0x28(SB)/8, $0x0404080808080808 +DATA expandAVX512_22_mat0<>+0x30(SB)/8, $0x0808080808080808 +DATA expandAVX512_22_mat0<>+0x38(SB)/8, $0x1010101010101010 + +GLOBL expandAVX512_22_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_22_inShuf1<>+0x00(SB)/8, $0xffffffffff020100 +DATA expandAVX512_22_inShuf1<>+0x08(SB)/8, $0xffff020201010000 +DATA expandAVX512_22_inShuf1<>+0x10(SB)/8, $0xffffffffff020100 +DATA expandAVX512_22_inShuf1<>+0x18(SB)/8, $0xffff020201010000 +DATA expandAVX512_22_inShuf1<>+0x20(SB)/8, $0xffffffffff020100 +DATA expandAVX512_22_inShuf1<>+0x28(SB)/8, $0xffffffff01010000 +DATA expandAVX512_22_inShuf1<>+0x30(SB)/8, $0xffff040403030202 +DATA expandAVX512_22_inShuf1<>+0x38(SB)/8, $0xffff050504040303 + +GLOBL expandAVX512_22_mat1<>(SB), RODATA, $0x40 +DATA expandAVX512_22_mat1<>+0x00(SB)/8, $0x1010101010102020 +DATA expandAVX512_22_mat1<>+0x08(SB)/8, $0x2020202020202020 +DATA expandAVX512_22_mat1<>+0x10(SB)/8, $0x2020202040404040 +DATA expandAVX512_22_mat1<>+0x18(SB)/8, $0x4040404040404040 +DATA expandAVX512_22_mat1<>+0x20(SB)/8, $0x4040808080808080 +DATA expandAVX512_22_mat1<>+0x28(SB)/8, $0x8080808080808080 +DATA expandAVX512_22_mat1<>+0x30(SB)/8, $0x8080808080808080 +DATA expandAVX512_22_mat1<>+0x38(SB)/8, $0x0101010101010101 + +GLOBL expandAVX512_22_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_22_inShuf2<>+0x00(SB)/8, $0xffffffffff050403 +DATA expandAVX512_22_inShuf2<>+0x08(SB)/8, $0xffff050504040303 +DATA expandAVX512_22_inShuf2<>+0x10(SB)/8, $0xffffffffff050403 +DATA expandAVX512_22_inShuf2<>+0x18(SB)/8, $0xffff050504040303 +DATA expandAVX512_22_inShuf2<>+0x20(SB)/8, $0xffffffffff050403 +DATA expandAVX512_22_inShuf2<>+0x28(SB)/8, $0xffff050504040303 +DATA expandAVX512_22_inShuf2<>+0x30(SB)/8, $0xffff050504040303 +DATA expandAVX512_22_inShuf2<>+0x38(SB)/8, $0xffffffffff050403 + +GLOBL expandAVX512_22_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_22_mat2<>+0x00(SB)/8, $0x0101010101010202 +DATA expandAVX512_22_mat2<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_22_mat2<>+0x10(SB)/8, $0x0202020204040404 +DATA expandAVX512_22_mat2<>+0x18(SB)/8, $0x0404040404040404 +DATA expandAVX512_22_mat2<>+0x20(SB)/8, $0x0404080808080808 +DATA expandAVX512_22_mat2<>+0x28(SB)/8, $0x0808080808080808 +DATA expandAVX512_22_mat2<>+0x30(SB)/8, $0x1010101010101010 +DATA expandAVX512_22_mat2<>+0x38(SB)/8, $0x1010101010102020 + +GLOBL expandAVX512_22_inShuf3<>(SB), RODATA, $0x40 +DATA expandAVX512_22_inShuf3<>+0x00(SB)/8, $0xffff050504040303 +DATA expandAVX512_22_inShuf3<>+0x08(SB)/8, $0xffffffffff050403 +DATA expandAVX512_22_inShuf3<>+0x10(SB)/8, $0xffffff0504040303 +DATA expandAVX512_22_inShuf3<>+0x18(SB)/8, $0xffffffffffff0403 +DATA expandAVX512_22_inShuf3<>+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_22_inShuf3<>+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_22_inShuf3<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_22_inShuf3<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_22_mat3<>(SB), RODATA, $0x40 +DATA expandAVX512_22_mat3<>+0x00(SB)/8, $0x2020202020202020 +DATA expandAVX512_22_mat3<>+0x08(SB)/8, $0x2020202040404040 +DATA expandAVX512_22_mat3<>+0x10(SB)/8, $0x4040404040404040 +DATA expandAVX512_22_mat3<>+0x18(SB)/8, $0x4040808080808080 +DATA expandAVX512_22_mat3<>+0x20(SB)/8, $0x0000000000000000 +DATA expandAVX512_22_mat3<>+0x28(SB)/8, $0x0000000000000000 +DATA expandAVX512_22_mat3<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_22_mat3<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_22_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_22_outShufLo+0x00(SB)/8, $0x2120181110080100 +DATA expandAVX512_22_outShufLo+0x08(SB)/8, $0x4948403938313028 +DATA expandAVX512_22_outShufLo+0x10(SB)/8, $0x0302696860595850 +DATA expandAVX512_22_outShufLo+0x18(SB)/8, $0x3229232219131209 +DATA expandAVX512_22_outShufLo+0x20(SB)/8, $0x5a514b4a413b3a33 +DATA expandAVX512_22_outShufLo+0x28(SB)/8, $0x140a05046b6a615b +DATA expandAVX512_22_outShufLo+0x30(SB)/8, $0x3c35342a25241a15 +DATA expandAVX512_22_outShufLo+0x38(SB)/8, $0x625d5c524d4c423d + +GLOBL expandAVX512_22_outShufHi0(SB), RODATA, $0x40 +DATA expandAVX512_22_outShufHi0+0x00(SB)/8, $0x5049484039383130 +DATA expandAVX512_22_outShufHi0+0x08(SB)/8, $0x7871706968605958 +DATA expandAVX512_22_outShufHi0+0x10(SB)/8, $0x3332ffffffffffff +DATA expandAVX512_22_outShufHi0+0x18(SB)/8, $0x5b5a514b4a413b3a +DATA expandAVX512_22_outShufHi0+0x20(SB)/8, $0xffff7973726b6a61 +DATA expandAVX512_22_outShufHi0+0x28(SB)/8, $0x3d3c3534ffffffff +DATA expandAVX512_22_outShufHi0+0x30(SB)/8, $0x6c625d5c524d4c42 +DATA expandAVX512_22_outShufHi0+0x38(SB)/8, $0xffffffff7a75746d + +GLOBL expandAVX512_22_outShufHi1(SB), RODATA, $0x40 +DATA expandAVX512_22_outShufHi1+0x00(SB)/8, $0xffffffffffffffff +DATA expandAVX512_22_outShufHi1+0x08(SB)/8, $0xffffffffffffffff +DATA expandAVX512_22_outShufHi1+0x10(SB)/8, $0xffff181110080100 +DATA expandAVX512_22_outShufHi1+0x18(SB)/8, $0xffffffffffffffff +DATA expandAVX512_22_outShufHi1+0x20(SB)/8, $0x0302ffffffffffff +DATA expandAVX512_22_outShufHi1+0x28(SB)/8, $0xffffffff19131209 +DATA expandAVX512_22_outShufHi1+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_22_outShufHi1+0x38(SB)/8, $0x140a0504ffffffff + +TEXT expandAVX512_22<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_22_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_22_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_22_inShuf2<>(SB), Z3 + VMOVDQU64 expandAVX512_22_inShuf3<>(SB), Z4 + VMOVDQU64 expandAVX512_22_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_22_outShufHi0(SB), Z5 + VMOVDQU64 expandAVX512_22_outShufHi1(SB), Z6 + VMOVDQU64 (AX), Z7 + VPERMB Z7, Z0, Z0 + VGF2P8AFFINEQB $0, expandAVX512_22_mat0<>(SB), Z0, Z0 + VPERMB Z7, Z2, Z2 + VGF2P8AFFINEQB $0, expandAVX512_22_mat1<>(SB), Z2, Z2 + VPERMB Z7, Z3, Z3 + VGF2P8AFFINEQB $0, expandAVX512_22_mat2<>(SB), Z3, Z3 + VPERMB Z7, Z4, Z4 + VGF2P8AFFINEQB $0, expandAVX512_22_mat3<>(SB), Z4, Z4 + VPERMI2B Z2, Z0, Z1 + MOVQ $0xffff03fffc0ffff, AX + KMOVQ AX, K1 + VPERMI2B.Z Z3, Z2, K1, Z5 + MOVQ $0xf0000fc0003f0000, AX + KMOVQ AX, K1 + VPERMB.Z Z4, Z6, K1, Z0 + VPORQ Z0, Z5, Z2 + RET + +GLOBL expandAVX512_24_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_24_inShuf0<>+0x00(SB)/8, $0x0202010101000000 +DATA expandAVX512_24_inShuf0<>+0x08(SB)/8, $0x0202010101000000 +DATA expandAVX512_24_inShuf0<>+0x10(SB)/8, $0x0202010101000000 +DATA expandAVX512_24_inShuf0<>+0x18(SB)/8, $0x0202010101000000 +DATA expandAVX512_24_inShuf0<>+0x20(SB)/8, $0x0202010101000000 +DATA expandAVX512_24_inShuf0<>+0x28(SB)/8, $0xff02010101000000 +DATA expandAVX512_24_inShuf0<>+0x30(SB)/8, $0xffff010101000000 +DATA expandAVX512_24_inShuf0<>+0x38(SB)/8, $0xffff010101000000 + +GLOBL expandAVX512_24_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_24_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_24_mat0<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_24_mat0<>+0x10(SB)/8, $0x0404040404040404 +DATA expandAVX512_24_mat0<>+0x18(SB)/8, $0x0808080808080808 +DATA expandAVX512_24_mat0<>+0x20(SB)/8, $0x1010101010101010 +DATA expandAVX512_24_mat0<>+0x28(SB)/8, $0x2020202020202020 +DATA expandAVX512_24_mat0<>+0x30(SB)/8, $0x4040404040404040 +DATA expandAVX512_24_mat0<>+0x38(SB)/8, $0x8080808080808080 + +GLOBL expandAVX512_24_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_24_inShuf1<>+0x00(SB)/8, $0xffffffffffffff02 +DATA expandAVX512_24_inShuf1<>+0x08(SB)/8, $0xffffffffffffff02 +DATA expandAVX512_24_inShuf1<>+0x10(SB)/8, $0xffffffffffffff02 +DATA expandAVX512_24_inShuf1<>+0x18(SB)/8, $0xffffffffffffff02 +DATA expandAVX512_24_inShuf1<>+0x20(SB)/8, $0xffffffffffffff02 +DATA expandAVX512_24_inShuf1<>+0x28(SB)/8, $0x0404040303030202 +DATA expandAVX512_24_inShuf1<>+0x30(SB)/8, $0x0404030303020202 +DATA expandAVX512_24_inShuf1<>+0x38(SB)/8, $0x0404030303020202 + +GLOBL expandAVX512_24_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_24_inShuf2<>+0x00(SB)/8, $0x0505040404030303 +DATA expandAVX512_24_inShuf2<>+0x08(SB)/8, $0x0505040404030303 +DATA expandAVX512_24_inShuf2<>+0x10(SB)/8, $0x0505040404030303 +DATA expandAVX512_24_inShuf2<>+0x18(SB)/8, $0xffff040404030303 +DATA expandAVX512_24_inShuf2<>+0x20(SB)/8, $0xffff040404030303 +DATA expandAVX512_24_inShuf2<>+0x28(SB)/8, $0xffffffffffffff04 +DATA expandAVX512_24_inShuf2<>+0x30(SB)/8, $0xffffffffffffff04 +DATA expandAVX512_24_inShuf2<>+0x38(SB)/8, $0xffffffffffffff05 + +GLOBL expandAVX512_24_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_24_mat2<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_24_mat2<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_24_mat2<>+0x10(SB)/8, $0x0404040404040404 +DATA expandAVX512_24_mat2<>+0x18(SB)/8, $0x0808080808080808 +DATA expandAVX512_24_mat2<>+0x20(SB)/8, $0x1010101010101010 +DATA expandAVX512_24_mat2<>+0x28(SB)/8, $0x4040404040404040 +DATA expandAVX512_24_mat2<>+0x30(SB)/8, $0x8080808080808080 +DATA expandAVX512_24_mat2<>+0x38(SB)/8, $0x0101010101010101 + +GLOBL expandAVX512_24_inShuf3<>(SB), RODATA, $0x40 +DATA expandAVX512_24_inShuf3<>+0x00(SB)/8, $0xffffffffffffff05 +DATA expandAVX512_24_inShuf3<>+0x08(SB)/8, $0xffffffffffffffff +DATA expandAVX512_24_inShuf3<>+0x10(SB)/8, $0xffffffffffffffff +DATA expandAVX512_24_inShuf3<>+0x18(SB)/8, $0xffffffffffffffff +DATA expandAVX512_24_inShuf3<>+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_24_inShuf3<>+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_24_inShuf3<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_24_inShuf3<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_24_mat3<>(SB), RODATA, $0x40 +DATA expandAVX512_24_mat3<>+0x00(SB)/8, $0x0202020202020202 +DATA expandAVX512_24_mat3<>+0x08(SB)/8, $0x0000000000000000 +DATA expandAVX512_24_mat3<>+0x10(SB)/8, $0x0000000000000000 +DATA expandAVX512_24_mat3<>+0x18(SB)/8, $0x0000000000000000 +DATA expandAVX512_24_mat3<>+0x20(SB)/8, $0x0000000000000000 +DATA expandAVX512_24_mat3<>+0x28(SB)/8, $0x0000000000000000 +DATA expandAVX512_24_mat3<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_24_mat3<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_24_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_24_outShufLo+0x00(SB)/8, $0x11100a0908020100 +DATA expandAVX512_24_outShufLo+0x08(SB)/8, $0x282221201a191812 +DATA expandAVX512_24_outShufLo+0x10(SB)/8, $0x3a39383231302a29 +DATA expandAVX512_24_outShufLo+0x18(SB)/8, $0x14130d0c0b050403 +DATA expandAVX512_24_outShufLo+0x20(SB)/8, $0x2b2524231d1c1b15 +DATA expandAVX512_24_outShufLo+0x28(SB)/8, $0x3d3c3b3534332d2c +DATA expandAVX512_24_outShufLo+0x30(SB)/8, $0x1716480f0e400706 +DATA expandAVX512_24_outShufLo+0x38(SB)/8, $0x2e602726581f1e50 + +GLOBL expandAVX512_24_outShufHi0(SB), RODATA, $0x40 +DATA expandAVX512_24_outShufHi0+0x00(SB)/8, $0x3a39383231302928 +DATA expandAVX512_24_outShufHi0+0x08(SB)/8, $0x51504a4948424140 +DATA expandAVX512_24_outShufHi0+0x10(SB)/8, $0x2a6261605a595852 +DATA expandAVX512_24_outShufHi0+0x18(SB)/8, $0x3d3c3b3534332c2b +DATA expandAVX512_24_outShufHi0+0x20(SB)/8, $0x54534d4c4b454443 +DATA expandAVX512_24_outShufHi0+0x28(SB)/8, $0x2d6564635d5c5b55 +DATA expandAVX512_24_outShufHi0+0x30(SB)/8, $0x703f3e6837362f2e +DATA expandAVX512_24_outShufHi0+0x38(SB)/8, $0x5756ff4f4e784746 + +GLOBL expandAVX512_24_outShufHi1(SB), RODATA, $0x40 +DATA expandAVX512_24_outShufHi1+0x00(SB)/8, $0xffffffffffffffff +DATA expandAVX512_24_outShufHi1+0x08(SB)/8, $0xffffffffffffffff +DATA expandAVX512_24_outShufHi1+0x10(SB)/8, $0xffffffffffffffff +DATA expandAVX512_24_outShufHi1+0x18(SB)/8, $0xffffffffffffffff +DATA expandAVX512_24_outShufHi1+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_24_outShufHi1+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_24_outShufHi1+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_24_outShufHi1+0x38(SB)/8, $0xffff00ffffffffff + +TEXT expandAVX512_24<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_24_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_24_mat0<>(SB), Z2 + VMOVDQU64 expandAVX512_24_inShuf1<>(SB), Z3 + VMOVDQU64 expandAVX512_24_inShuf2<>(SB), Z4 + VMOVDQU64 expandAVX512_24_inShuf3<>(SB), Z5 + VMOVDQU64 expandAVX512_24_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_24_outShufHi0(SB), Z6 + VMOVDQU64 expandAVX512_24_outShufHi1(SB), Z7 + VMOVDQU64 (AX), Z8 + VPERMB Z8, Z0, Z0 + VGF2P8AFFINEQB $0, Z2, Z0, Z0 + VPERMB Z8, Z3, Z3 + VGF2P8AFFINEQB $0, Z2, Z3, Z2 + VPERMB Z8, Z4, Z3 + VGF2P8AFFINEQB $0, expandAVX512_24_mat2<>(SB), Z3, Z3 + VPERMB Z8, Z5, Z4 + VGF2P8AFFINEQB $0, expandAVX512_24_mat3<>(SB), Z4, Z4 + VPERMI2B Z2, Z0, Z1 + MOVQ $0xdfffffffffffffff, AX + KMOVQ AX, K1 + VPERMI2B.Z Z3, Z2, K1, Z6 + MOVQ $0x2000000000000000, AX + KMOVQ AX, K1 + VPERMB.Z Z4, Z7, K1, Z0 + VPORQ Z0, Z6, Z2 + RET + +GLOBL expandAVX512_26_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_26_inShuf0<>+0x00(SB)/8, $0x0202010101000000 +DATA expandAVX512_26_inShuf0<>+0x08(SB)/8, $0xffffffffff020100 +DATA expandAVX512_26_inShuf0<>+0x10(SB)/8, $0xffff020201010000 +DATA expandAVX512_26_inShuf0<>+0x18(SB)/8, $0xffffffffff020100 +DATA expandAVX512_26_inShuf0<>+0x20(SB)/8, $0xffff020201010000 +DATA expandAVX512_26_inShuf0<>+0x28(SB)/8, $0xffffffffff020100 +DATA expandAVX512_26_inShuf0<>+0x30(SB)/8, $0x0202010101000000 +DATA expandAVX512_26_inShuf0<>+0x38(SB)/8, $0xffff010101000000 + +GLOBL expandAVX512_26_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_26_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_26_mat0<>+0x08(SB)/8, $0x0101020202020202 +DATA expandAVX512_26_mat0<>+0x10(SB)/8, $0x0202020202020202 +DATA expandAVX512_26_mat0<>+0x18(SB)/8, $0x0202020204040404 +DATA expandAVX512_26_mat0<>+0x20(SB)/8, $0x0404040404040404 +DATA expandAVX512_26_mat0<>+0x28(SB)/8, $0x0404040404040808 +DATA expandAVX512_26_mat0<>+0x30(SB)/8, $0x0808080808080808 +DATA expandAVX512_26_mat0<>+0x38(SB)/8, $0x1010101010101010 + +GLOBL expandAVX512_26_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_26_inShuf1<>+0x00(SB)/8, $0xffffffffffff0100 +DATA expandAVX512_26_inShuf1<>+0x08(SB)/8, $0xffffffff01010000 +DATA expandAVX512_26_inShuf1<>+0x10(SB)/8, $0xffffffffffff0100 +DATA expandAVX512_26_inShuf1<>+0x18(SB)/8, $0xffffffff01010000 +DATA expandAVX512_26_inShuf1<>+0x20(SB)/8, $0xffffffffffff0100 +DATA expandAVX512_26_inShuf1<>+0x28(SB)/8, $0xffff010101000000 +DATA expandAVX512_26_inShuf1<>+0x30(SB)/8, $0xffffffffffffff02 +DATA expandAVX512_26_inShuf1<>+0x38(SB)/8, $0xff04040403030302 + +GLOBL expandAVX512_26_mat1<>(SB), RODATA, $0x40 +DATA expandAVX512_26_mat1<>+0x00(SB)/8, $0x1010202020202020 +DATA expandAVX512_26_mat1<>+0x08(SB)/8, $0x2020202020202020 +DATA expandAVX512_26_mat1<>+0x10(SB)/8, $0x2020202040404040 +DATA expandAVX512_26_mat1<>+0x18(SB)/8, $0x4040404040404040 +DATA expandAVX512_26_mat1<>+0x20(SB)/8, $0x4040404040408080 +DATA expandAVX512_26_mat1<>+0x28(SB)/8, $0x8080808080808080 +DATA expandAVX512_26_mat1<>+0x30(SB)/8, $0x0101010101010101 +DATA expandAVX512_26_mat1<>+0x38(SB)/8, $0x0808080808080808 + +GLOBL expandAVX512_26_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_26_inShuf2<>+0x00(SB)/8, $0x0404030303020202 +DATA expandAVX512_26_inShuf2<>+0x08(SB)/8, $0xffffffffff040302 +DATA expandAVX512_26_inShuf2<>+0x10(SB)/8, $0xffff040403030202 +DATA expandAVX512_26_inShuf2<>+0x18(SB)/8, $0xffffffffff040302 +DATA expandAVX512_26_inShuf2<>+0x20(SB)/8, $0xffff040403030202 +DATA expandAVX512_26_inShuf2<>+0x28(SB)/8, $0xffffffffff040302 +DATA expandAVX512_26_inShuf2<>+0x30(SB)/8, $0xff04030303020202 +DATA expandAVX512_26_inShuf2<>+0x38(SB)/8, $0xffff040404030303 + +GLOBL expandAVX512_26_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_26_mat2<>+0x00(SB)/8, $0x1010101010101010 +DATA expandAVX512_26_mat2<>+0x08(SB)/8, $0x1010202020202020 +DATA expandAVX512_26_mat2<>+0x10(SB)/8, $0x2020202020202020 +DATA expandAVX512_26_mat2<>+0x18(SB)/8, $0x2020202040404040 +DATA expandAVX512_26_mat2<>+0x20(SB)/8, $0x4040404040404040 +DATA expandAVX512_26_mat2<>+0x28(SB)/8, $0x4040404040408080 +DATA expandAVX512_26_mat2<>+0x30(SB)/8, $0x8080808080808080 +DATA expandAVX512_26_mat2<>+0x38(SB)/8, $0x0101010101010101 + +GLOBL expandAVX512_26_inShuf3<>(SB), RODATA, $0x40 +DATA expandAVX512_26_inShuf3<>+0x00(SB)/8, $0xffffffffffff0403 +DATA expandAVX512_26_inShuf3<>+0x08(SB)/8, $0xffffffff04040303 +DATA expandAVX512_26_inShuf3<>+0x10(SB)/8, $0xffffffffffff0403 +DATA expandAVX512_26_inShuf3<>+0x18(SB)/8, $0xffffffff04040303 +DATA expandAVX512_26_inShuf3<>+0x20(SB)/8, $0xffffffffffff0403 +DATA expandAVX512_26_inShuf3<>+0x28(SB)/8, $0xffffffffffffff04 +DATA expandAVX512_26_inShuf3<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_26_inShuf3<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_26_mat3<>(SB), RODATA, $0x40 +DATA expandAVX512_26_mat3<>+0x00(SB)/8, $0x0101020202020202 +DATA expandAVX512_26_mat3<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_26_mat3<>+0x10(SB)/8, $0x0202020204040404 +DATA expandAVX512_26_mat3<>+0x18(SB)/8, $0x0404040404040404 +DATA expandAVX512_26_mat3<>+0x20(SB)/8, $0x0404040404040808 +DATA expandAVX512_26_mat3<>+0x28(SB)/8, $0x1010101010101010 +DATA expandAVX512_26_mat3<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_26_mat3<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_26_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_26_outShufLo+0x00(SB)/8, $0x2018111008020100 +DATA expandAVX512_26_outShufLo+0x08(SB)/8, $0x3a39383231302821 +DATA expandAVX512_26_outShufLo+0x10(SB)/8, $0x6860595850494840 +DATA expandAVX512_26_outShufLo+0x18(SB)/8, $0x1312090504036a69 +DATA expandAVX512_26_outShufLo+0x20(SB)/8, $0x3b35343329232219 +DATA expandAVX512_26_outShufLo+0x28(SB)/8, $0x5b5a514b4a413d3c +DATA expandAVX512_26_outShufLo+0x30(SB)/8, $0x0a7007066d6c6b61 +DATA expandAVX512_26_outShufLo+0x38(SB)/8, $0x37362a25241a1514 + +GLOBL expandAVX512_26_outShufHi0(SB), RODATA, $0x40 +DATA expandAVX512_26_outShufHi0+0x00(SB)/8, $0x5851504842414038 +DATA expandAVX512_26_outShufHi0+0x08(SB)/8, $0x7978727170686160 +DATA expandAVX512_26_outShufHi0+0x10(SB)/8, $0xffffffffffffff7a +DATA expandAVX512_26_outShufHi0+0x18(SB)/8, $0x52494544433b3a39 +DATA expandAVX512_26_outShufHi0+0x20(SB)/8, $0x7574736963625953 +DATA expandAVX512_26_outShufHi0+0x28(SB)/8, $0xffffffffff7d7c7b +DATA expandAVX512_26_outShufHi0+0x30(SB)/8, $0xff47463e3d3cffff +DATA expandAVX512_26_outShufHi0+0x38(SB)/8, $0x766a65645a55544a + +GLOBL expandAVX512_26_outShufHi1(SB), RODATA, $0x40 +DATA expandAVX512_26_outShufHi1+0x00(SB)/8, $0xffffffffffffffff +DATA expandAVX512_26_outShufHi1+0x08(SB)/8, $0xffffffffffffffff +DATA expandAVX512_26_outShufHi1+0x10(SB)/8, $0x20191810090800ff +DATA expandAVX512_26_outShufHi1+0x18(SB)/8, $0xffffffffffffffff +DATA expandAVX512_26_outShufHi1+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_26_outShufHi1+0x28(SB)/8, $0x1a110b0a01ffffff +DATA expandAVX512_26_outShufHi1+0x30(SB)/8, $0x28ffffffffff211b +DATA expandAVX512_26_outShufHi1+0x38(SB)/8, $0xffffffffffffffff + +TEXT expandAVX512_26<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_26_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_26_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_26_inShuf2<>(SB), Z3 + VMOVDQU64 expandAVX512_26_inShuf3<>(SB), Z4 + VMOVDQU64 expandAVX512_26_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_26_outShufHi0(SB), Z5 + VMOVDQU64 expandAVX512_26_outShufHi1(SB), Z6 + VMOVDQU64 (AX), Z7 + VPERMB Z7, Z0, Z0 + VGF2P8AFFINEQB $0, expandAVX512_26_mat0<>(SB), Z0, Z0 + VPERMB Z7, Z2, Z2 + VGF2P8AFFINEQB $0, expandAVX512_26_mat1<>(SB), Z2, Z2 + VPERMB Z7, Z3, Z3 + VGF2P8AFFINEQB $0, expandAVX512_26_mat2<>(SB), Z3, Z3 + VPERMB Z7, Z4, Z4 + VGF2P8AFFINEQB $0, expandAVX512_26_mat3<>(SB), Z4, Z4 + VPERMI2B Z2, Z0, Z1 + MOVQ $0xff7c07ffff01ffff, AX + KMOVQ AX, K1 + VPERMI2B.Z Z3, Z2, K1, Z5 + MOVQ $0x83f80000fe0000, AX + KMOVQ AX, K1 + VPERMB.Z Z4, Z6, K1, Z0 + VPORQ Z0, Z5, Z2 + RET + +GLOBL expandAVX512_28_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_28_inShuf0<>+0x00(SB)/8, $0x0202010101000000 +DATA expandAVX512_28_inShuf0<>+0x08(SB)/8, $0xffffffffff020100 +DATA expandAVX512_28_inShuf0<>+0x10(SB)/8, $0x0202010101000000 +DATA expandAVX512_28_inShuf0<>+0x18(SB)/8, $0xff02010101000000 +DATA expandAVX512_28_inShuf0<>+0x20(SB)/8, $0xffffffffffff0100 +DATA expandAVX512_28_inShuf0<>+0x28(SB)/8, $0xffff010101000000 +DATA expandAVX512_28_inShuf0<>+0x30(SB)/8, $0xffff010101000000 +DATA expandAVX512_28_inShuf0<>+0x38(SB)/8, $0xffffffffffff0100 + +GLOBL expandAVX512_28_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_28_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_28_mat0<>+0x08(SB)/8, $0x0101010102020202 +DATA expandAVX512_28_mat0<>+0x10(SB)/8, $0x0202020202020202 +DATA expandAVX512_28_mat0<>+0x18(SB)/8, $0x0404040404040404 +DATA expandAVX512_28_mat0<>+0x20(SB)/8, $0x0404040408080808 +DATA expandAVX512_28_mat0<>+0x28(SB)/8, $0x0808080808080808 +DATA expandAVX512_28_mat0<>+0x30(SB)/8, $0x1010101010101010 +DATA expandAVX512_28_mat0<>+0x38(SB)/8, $0x1010101020202020 + +GLOBL expandAVX512_28_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_28_inShuf1<>+0x00(SB)/8, $0xffff010101000000 +DATA expandAVX512_28_inShuf1<>+0x08(SB)/8, $0xffff010101000000 +DATA expandAVX512_28_inShuf1<>+0x10(SB)/8, $0xffffffffffff0100 +DATA expandAVX512_28_inShuf1<>+0x18(SB)/8, $0xffff010101000000 +DATA expandAVX512_28_inShuf1<>+0x20(SB)/8, $0xffffffffffffff02 +DATA expandAVX512_28_inShuf1<>+0x28(SB)/8, $0xffffffffffffff02 +DATA expandAVX512_28_inShuf1<>+0x30(SB)/8, $0x0404040303030202 +DATA expandAVX512_28_inShuf1<>+0x38(SB)/8, $0xffffffffff040302 + +GLOBL expandAVX512_28_mat1<>(SB), RODATA, $0x40 +DATA expandAVX512_28_mat1<>+0x00(SB)/8, $0x2020202020202020 +DATA expandAVX512_28_mat1<>+0x08(SB)/8, $0x4040404040404040 +DATA expandAVX512_28_mat1<>+0x10(SB)/8, $0x4040404080808080 +DATA expandAVX512_28_mat1<>+0x18(SB)/8, $0x8080808080808080 +DATA expandAVX512_28_mat1<>+0x20(SB)/8, $0x0101010101010101 +DATA expandAVX512_28_mat1<>+0x28(SB)/8, $0x0202020202020202 +DATA expandAVX512_28_mat1<>+0x30(SB)/8, $0x0404040404040404 +DATA expandAVX512_28_mat1<>+0x38(SB)/8, $0x0404040408080808 + +GLOBL expandAVX512_28_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_28_inShuf2<>+0x00(SB)/8, $0x0404030303020202 +DATA expandAVX512_28_inShuf2<>+0x08(SB)/8, $0x0404030303020202 +DATA expandAVX512_28_inShuf2<>+0x10(SB)/8, $0xffffffffffff0302 +DATA expandAVX512_28_inShuf2<>+0x18(SB)/8, $0xffff030303020202 +DATA expandAVX512_28_inShuf2<>+0x20(SB)/8, $0xffff030303020202 +DATA expandAVX512_28_inShuf2<>+0x28(SB)/8, $0xffffffffffff0302 +DATA expandAVX512_28_inShuf2<>+0x30(SB)/8, $0xffff030303020202 +DATA expandAVX512_28_inShuf2<>+0x38(SB)/8, $0xffff040404030303 + +GLOBL expandAVX512_28_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_28_mat2<>+0x00(SB)/8, $0x0808080808080808 +DATA expandAVX512_28_mat2<>+0x08(SB)/8, $0x1010101010101010 +DATA expandAVX512_28_mat2<>+0x10(SB)/8, $0x1010101020202020 +DATA expandAVX512_28_mat2<>+0x18(SB)/8, $0x2020202020202020 +DATA expandAVX512_28_mat2<>+0x20(SB)/8, $0x4040404040404040 +DATA expandAVX512_28_mat2<>+0x28(SB)/8, $0x4040404080808080 +DATA expandAVX512_28_mat2<>+0x30(SB)/8, $0x8080808080808080 +DATA expandAVX512_28_mat2<>+0x38(SB)/8, $0x0101010101010101 + +GLOBL expandAVX512_28_inShuf3<>(SB), RODATA, $0x40 +DATA expandAVX512_28_inShuf3<>+0x00(SB)/8, $0xffffffffffff0403 +DATA expandAVX512_28_inShuf3<>+0x08(SB)/8, $0xffff040404030303 +DATA expandAVX512_28_inShuf3<>+0x10(SB)/8, $0xffffffffffffff04 +DATA expandAVX512_28_inShuf3<>+0x18(SB)/8, $0xffffffffffffffff +DATA expandAVX512_28_inShuf3<>+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_28_inShuf3<>+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_28_inShuf3<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_28_inShuf3<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_28_mat3<>(SB), RODATA, $0x40 +DATA expandAVX512_28_mat3<>+0x00(SB)/8, $0x0101010102020202 +DATA expandAVX512_28_mat3<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_28_mat3<>+0x10(SB)/8, $0x0808080808080808 +DATA expandAVX512_28_mat3<>+0x18(SB)/8, $0x0000000000000000 +DATA expandAVX512_28_mat3<>+0x20(SB)/8, $0x0000000000000000 +DATA expandAVX512_28_mat3<>+0x28(SB)/8, $0x0000000000000000 +DATA expandAVX512_28_mat3<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_28_mat3<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_28_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_28_outShufLo+0x00(SB)/8, $0x1812111008020100 +DATA expandAVX512_28_outShufLo+0x08(SB)/8, $0x31302a2928201a19 +DATA expandAVX512_28_outShufLo+0x10(SB)/8, $0x4a49484241403832 +DATA expandAVX512_28_outShufLo+0x18(SB)/8, $0x090504035a595850 +DATA expandAVX512_28_outShufLo+0x20(SB)/8, $0x2b211d1c1b151413 +DATA expandAVX512_28_outShufLo+0x28(SB)/8, $0x4443393534332d2c +DATA expandAVX512_28_outShufLo+0x30(SB)/8, $0x5d5c5b514d4c4b45 +DATA expandAVX512_28_outShufLo+0x38(SB)/8, $0x1e6817160a600706 + +GLOBL expandAVX512_28_outShufHi0(SB), RODATA, $0x40 +DATA expandAVX512_28_outShufHi0+0x00(SB)/8, $0x4948424140383130 +DATA expandAVX512_28_outShufHi0+0x08(SB)/8, $0x6261605a5958504a +DATA expandAVX512_28_outShufHi0+0x10(SB)/8, $0xff7a797872717068 +DATA expandAVX512_28_outShufHi0+0x18(SB)/8, $0x4339343332ffffff +DATA expandAVX512_28_outShufHi0+0x20(SB)/8, $0x5c5b514d4c4b4544 +DATA expandAVX512_28_outShufHi0+0x28(SB)/8, $0x757473696564635d +DATA expandAVX512_28_outShufHi0+0x30(SB)/8, $0x35ffffffff7d7c7b +DATA expandAVX512_28_outShufHi0+0x38(SB)/8, $0x4f4eff47463a3736 + +GLOBL expandAVX512_28_outShufHi1(SB), RODATA, $0x40 +DATA expandAVX512_28_outShufHi1+0x00(SB)/8, $0xffffffffffffffff +DATA expandAVX512_28_outShufHi1+0x08(SB)/8, $0xffffffffffffffff +DATA expandAVX512_28_outShufHi1+0x10(SB)/8, $0x00ffffffffffffff +DATA expandAVX512_28_outShufHi1+0x18(SB)/8, $0xffffffffff0a0908 +DATA expandAVX512_28_outShufHi1+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_28_outShufHi1+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_28_outShufHi1+0x30(SB)/8, $0xff0d0c0b01ffffff +DATA expandAVX512_28_outShufHi1+0x38(SB)/8, $0xffff10ffffffffff + +TEXT expandAVX512_28<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_28_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_28_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_28_inShuf2<>(SB), Z3 + VMOVDQU64 expandAVX512_28_inShuf3<>(SB), Z4 + VMOVDQU64 expandAVX512_28_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_28_outShufHi0(SB), Z5 + VMOVDQU64 expandAVX512_28_outShufHi1(SB), Z6 + VMOVDQU64 (AX), Z7 + VPERMB Z7, Z0, Z0 + VGF2P8AFFINEQB $0, expandAVX512_28_mat0<>(SB), Z0, Z0 + VPERMB Z7, Z2, Z2 + VGF2P8AFFINEQB $0, expandAVX512_28_mat1<>(SB), Z2, Z2 + VPERMB Z7, Z3, Z3 + VGF2P8AFFINEQB $0, expandAVX512_28_mat2<>(SB), Z3, Z3 + VPERMB Z7, Z4, Z4 + VGF2P8AFFINEQB $0, expandAVX512_28_mat3<>(SB), Z4, Z4 + VPERMI2B Z2, Z0, Z1 + MOVQ $0xdf87fffff87fffff, AX + KMOVQ AX, K1 + VPERMI2B.Z Z3, Z2, K1, Z5 + MOVQ $0x2078000007800000, AX + KMOVQ AX, K1 + VPERMB.Z Z4, Z6, K1, Z0 + VPORQ Z0, Z5, Z2 + RET + +GLOBL expandAVX512_30_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_30_inShuf0<>+0x00(SB)/8, $0x0202010101000000 +DATA expandAVX512_30_inShuf0<>+0x08(SB)/8, $0xffffffffff020100 +DATA expandAVX512_30_inShuf0<>+0x10(SB)/8, $0xffff010101000000 +DATA expandAVX512_30_inShuf0<>+0x18(SB)/8, $0xffffffffffff0100 +DATA expandAVX512_30_inShuf0<>+0x20(SB)/8, $0xffff010101000000 +DATA expandAVX512_30_inShuf0<>+0x28(SB)/8, $0xffffffffffff0100 +DATA expandAVX512_30_inShuf0<>+0x30(SB)/8, $0xffff010101000000 +DATA expandAVX512_30_inShuf0<>+0x38(SB)/8, $0xffff010101000000 + +GLOBL expandAVX512_30_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_30_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_30_mat0<>+0x08(SB)/8, $0x0101010101010202 +DATA expandAVX512_30_mat0<>+0x10(SB)/8, $0x0202020202020202 +DATA expandAVX512_30_mat0<>+0x18(SB)/8, $0x0202020204040404 +DATA expandAVX512_30_mat0<>+0x20(SB)/8, $0x0404040404040404 +DATA expandAVX512_30_mat0<>+0x28(SB)/8, $0x0404080808080808 +DATA expandAVX512_30_mat0<>+0x30(SB)/8, $0x0808080808080808 +DATA expandAVX512_30_mat0<>+0x38(SB)/8, $0x1010101010101010 + +GLOBL expandAVX512_30_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_30_inShuf1<>+0x00(SB)/8, $0xffffffffffff0100 +DATA expandAVX512_30_inShuf1<>+0x08(SB)/8, $0xffff010101000000 +DATA expandAVX512_30_inShuf1<>+0x10(SB)/8, $0xffffffffffff0100 +DATA expandAVX512_30_inShuf1<>+0x18(SB)/8, $0xffff010101000000 +DATA expandAVX512_30_inShuf1<>+0x20(SB)/8, $0xffffffffffff0100 +DATA expandAVX512_30_inShuf1<>+0x28(SB)/8, $0xffff010101000000 +DATA expandAVX512_30_inShuf1<>+0x30(SB)/8, $0xffffffffffffff02 +DATA expandAVX512_30_inShuf1<>+0x38(SB)/8, $0x0404030303020202 + +GLOBL expandAVX512_30_mat1<>(SB), RODATA, $0x40 +DATA expandAVX512_30_mat1<>+0x00(SB)/8, $0x1010101010102020 +DATA expandAVX512_30_mat1<>+0x08(SB)/8, $0x2020202020202020 +DATA expandAVX512_30_mat1<>+0x10(SB)/8, $0x2020202040404040 +DATA expandAVX512_30_mat1<>+0x18(SB)/8, $0x4040404040404040 +DATA expandAVX512_30_mat1<>+0x20(SB)/8, $0x4040808080808080 +DATA expandAVX512_30_mat1<>+0x28(SB)/8, $0x8080808080808080 +DATA expandAVX512_30_mat1<>+0x30(SB)/8, $0x0101010101010101 +DATA expandAVX512_30_mat1<>+0x38(SB)/8, $0x0202020202020202 + +GLOBL expandAVX512_30_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_30_inShuf2<>+0x00(SB)/8, $0xffffffffff040302 +DATA expandAVX512_30_inShuf2<>+0x08(SB)/8, $0xffff030303020202 +DATA expandAVX512_30_inShuf2<>+0x10(SB)/8, $0xffffffffffff0302 +DATA expandAVX512_30_inShuf2<>+0x18(SB)/8, $0xffff030303020202 +DATA expandAVX512_30_inShuf2<>+0x20(SB)/8, $0xffff030303020202 +DATA expandAVX512_30_inShuf2<>+0x28(SB)/8, $0xffffffffffff0302 +DATA expandAVX512_30_inShuf2<>+0x30(SB)/8, $0xffff030303020202 +DATA expandAVX512_30_inShuf2<>+0x38(SB)/8, $0xffffffffffff0302 + +GLOBL expandAVX512_30_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_30_mat2<>+0x00(SB)/8, $0x0202020204040404 +DATA expandAVX512_30_mat2<>+0x08(SB)/8, $0x0404040404040404 +DATA expandAVX512_30_mat2<>+0x10(SB)/8, $0x0404080808080808 +DATA expandAVX512_30_mat2<>+0x18(SB)/8, $0x0808080808080808 +DATA expandAVX512_30_mat2<>+0x20(SB)/8, $0x1010101010101010 +DATA expandAVX512_30_mat2<>+0x28(SB)/8, $0x1010101010102020 +DATA expandAVX512_30_mat2<>+0x30(SB)/8, $0x2020202020202020 +DATA expandAVX512_30_mat2<>+0x38(SB)/8, $0x2020202040404040 + +GLOBL expandAVX512_30_inShuf3<>(SB), RODATA, $0x40 +DATA expandAVX512_30_inShuf3<>+0x00(SB)/8, $0xffff030303020202 +DATA expandAVX512_30_inShuf3<>+0x08(SB)/8, $0xffffffffffff0302 +DATA expandAVX512_30_inShuf3<>+0x10(SB)/8, $0xffff030303020202 +DATA expandAVX512_30_inShuf3<>+0x18(SB)/8, $0xffff040404030303 +DATA expandAVX512_30_inShuf3<>+0x20(SB)/8, $0xffffffffffff0403 +DATA expandAVX512_30_inShuf3<>+0x28(SB)/8, $0xffffffffffffff04 +DATA expandAVX512_30_inShuf3<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_30_inShuf3<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_30_mat3<>(SB), RODATA, $0x40 +DATA expandAVX512_30_mat3<>+0x00(SB)/8, $0x4040404040404040 +DATA expandAVX512_30_mat3<>+0x08(SB)/8, $0x4040808080808080 +DATA expandAVX512_30_mat3<>+0x10(SB)/8, $0x8080808080808080 +DATA expandAVX512_30_mat3<>+0x18(SB)/8, $0x0101010101010101 +DATA expandAVX512_30_mat3<>+0x20(SB)/8, $0x0101010101010202 +DATA expandAVX512_30_mat3<>+0x28(SB)/8, $0x0202020202020202 +DATA expandAVX512_30_mat3<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_30_mat3<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_30_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_30_outShufLo+0x00(SB)/8, $0x1812111008020100 +DATA expandAVX512_30_outShufLo+0x08(SB)/8, $0x3832313028222120 +DATA expandAVX512_30_outShufLo+0x10(SB)/8, $0x58504a4948403a39 +DATA expandAVX512_30_outShufLo+0x18(SB)/8, $0x04036a6968605a59 +DATA expandAVX512_30_outShufLo+0x20(SB)/8, $0x2423191514130905 +DATA expandAVX512_30_outShufLo+0x28(SB)/8, $0x3d3c3b3534332925 +DATA expandAVX512_30_outShufLo+0x30(SB)/8, $0x5d5c5b514d4c4b41 +DATA expandAVX512_30_outShufLo+0x38(SB)/8, $0x0a7007066d6c6b61 + +GLOBL expandAVX512_30_outShufHi0(SB), RODATA, $0x40 +DATA expandAVX512_30_outShufHi0+0x00(SB)/8, $0x504a4948403a3938 +DATA expandAVX512_30_outShufHi0+0x08(SB)/8, $0x70686261605a5958 +DATA expandAVX512_30_outShufHi0+0x10(SB)/8, $0xffffffffff787271 +DATA expandAVX512_30_outShufHi0+0x18(SB)/8, $0x3c3bffffffffffff +DATA expandAVX512_30_outShufHi0+0x20(SB)/8, $0x5c5b514d4c4b413d +DATA expandAVX512_30_outShufHi0+0x28(SB)/8, $0x757473696564635d +DATA expandAVX512_30_outShufHi0+0x30(SB)/8, $0xffffffffffffff79 +DATA expandAVX512_30_outShufHi0+0x38(SB)/8, $0x42ff3f3effffffff + +GLOBL expandAVX512_30_outShufHi1(SB), RODATA, $0x40 +DATA expandAVX512_30_outShufHi1+0x00(SB)/8, $0xffffffffffffffff +DATA expandAVX512_30_outShufHi1+0x08(SB)/8, $0xffffffffffffffff +DATA expandAVX512_30_outShufHi1+0x10(SB)/8, $0x1008020100ffffff +DATA expandAVX512_30_outShufHi1+0x18(SB)/8, $0xffff201a19181211 +DATA expandAVX512_30_outShufHi1+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_30_outShufHi1+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_30_outShufHi1+0x30(SB)/8, $0x15141309050403ff +DATA expandAVX512_30_outShufHi1+0x38(SB)/8, $0xff28ffff211d1c1b + +TEXT expandAVX512_30<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_30_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_30_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_30_inShuf2<>(SB), Z3 + VMOVDQU64 expandAVX512_30_inShuf3<>(SB), Z4 + VMOVDQU64 expandAVX512_30_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_30_outShufHi0(SB), Z5 + VMOVDQU64 expandAVX512_30_outShufHi1(SB), Z6 + VMOVDQU64 (AX), Z7 + VPERMB Z7, Z0, Z0 + VGF2P8AFFINEQB $0, expandAVX512_30_mat0<>(SB), Z0, Z0 + VPERMB Z7, Z2, Z2 + VGF2P8AFFINEQB $0, expandAVX512_30_mat1<>(SB), Z2, Z2 + VPERMB Z7, Z3, Z3 + VGF2P8AFFINEQB $0, expandAVX512_30_mat2<>(SB), Z3, Z3 + VPERMB Z7, Z4, Z4 + VGF2P8AFFINEQB $0, expandAVX512_30_mat3<>(SB), Z4, Z4 + VPERMI2B Z2, Z0, Z1 + MOVQ $0xb001ffffc007ffff, AX + KMOVQ AX, K1 + VPERMI2B.Z Z3, Z2, K1, Z5 + MOVQ $0x4ffe00003ff80000, AX + KMOVQ AX, K1 + VPERMB.Z Z4, Z6, K1, Z0 + VPORQ Z0, Z5, Z2 + RET + +GLOBL expandAVX512_32_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_32_inShuf0<>+0x00(SB)/8, $0x0101010100000000 +DATA expandAVX512_32_inShuf0<>+0x08(SB)/8, $0x0101010100000000 +DATA expandAVX512_32_inShuf0<>+0x10(SB)/8, $0x0101010100000000 +DATA expandAVX512_32_inShuf0<>+0x18(SB)/8, $0x0101010100000000 +DATA expandAVX512_32_inShuf0<>+0x20(SB)/8, $0x0101010100000000 +DATA expandAVX512_32_inShuf0<>+0x28(SB)/8, $0x0101010100000000 +DATA expandAVX512_32_inShuf0<>+0x30(SB)/8, $0x0101010100000000 +DATA expandAVX512_32_inShuf0<>+0x38(SB)/8, $0x0101010100000000 + +GLOBL expandAVX512_32_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_32_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_32_mat0<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_32_mat0<>+0x10(SB)/8, $0x0404040404040404 +DATA expandAVX512_32_mat0<>+0x18(SB)/8, $0x0808080808080808 +DATA expandAVX512_32_mat0<>+0x20(SB)/8, $0x1010101010101010 +DATA expandAVX512_32_mat0<>+0x28(SB)/8, $0x2020202020202020 +DATA expandAVX512_32_mat0<>+0x30(SB)/8, $0x4040404040404040 +DATA expandAVX512_32_mat0<>+0x38(SB)/8, $0x8080808080808080 + +GLOBL expandAVX512_32_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_32_inShuf1<>+0x00(SB)/8, $0x0303030302020202 +DATA expandAVX512_32_inShuf1<>+0x08(SB)/8, $0x0303030302020202 +DATA expandAVX512_32_inShuf1<>+0x10(SB)/8, $0x0303030302020202 +DATA expandAVX512_32_inShuf1<>+0x18(SB)/8, $0x0303030302020202 +DATA expandAVX512_32_inShuf1<>+0x20(SB)/8, $0x0303030302020202 +DATA expandAVX512_32_inShuf1<>+0x28(SB)/8, $0x0303030302020202 +DATA expandAVX512_32_inShuf1<>+0x30(SB)/8, $0x0303030302020202 +DATA expandAVX512_32_inShuf1<>+0x38(SB)/8, $0x0303030302020202 + +GLOBL expandAVX512_32_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_32_outShufLo+0x00(SB)/8, $0x0b0a090803020100 +DATA expandAVX512_32_outShufLo+0x08(SB)/8, $0x1b1a191813121110 +DATA expandAVX512_32_outShufLo+0x10(SB)/8, $0x2b2a292823222120 +DATA expandAVX512_32_outShufLo+0x18(SB)/8, $0x3b3a393833323130 +DATA expandAVX512_32_outShufLo+0x20(SB)/8, $0x0f0e0d0c07060504 +DATA expandAVX512_32_outShufLo+0x28(SB)/8, $0x1f1e1d1c17161514 +DATA expandAVX512_32_outShufLo+0x30(SB)/8, $0x2f2e2d2c27262524 +DATA expandAVX512_32_outShufLo+0x38(SB)/8, $0x3f3e3d3c37363534 + +TEXT expandAVX512_32<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_32_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_32_mat0<>(SB), Z1 + VMOVDQU64 expandAVX512_32_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_32_outShufLo(SB), Z3 + VMOVDQU64 (AX), Z4 + VPERMB Z4, Z0, Z0 + VGF2P8AFFINEQB $0, Z1, Z0, Z0 + VPERMB Z4, Z2, Z2 + VGF2P8AFFINEQB $0, Z1, Z2, Z2 + VPERMB Z0, Z3, Z1 + VPERMB Z2, Z3, Z2 + RET + +GLOBL expandAVX512_36_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_36_inShuf0<>+0x00(SB)/8, $0x0101010100000000 +DATA expandAVX512_36_inShuf0<>+0x08(SB)/8, $0xffffffffffff0100 +DATA expandAVX512_36_inShuf0<>+0x10(SB)/8, $0x0101010100000000 +DATA expandAVX512_36_inShuf0<>+0x18(SB)/8, $0x0101010100000000 +DATA expandAVX512_36_inShuf0<>+0x20(SB)/8, $0xffffffffffff0100 +DATA expandAVX512_36_inShuf0<>+0x28(SB)/8, $0x0101010100000000 +DATA expandAVX512_36_inShuf0<>+0x30(SB)/8, $0x0101010100000000 +DATA expandAVX512_36_inShuf0<>+0x38(SB)/8, $0xffffffffffff0100 + +GLOBL expandAVX512_36_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_36_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_36_mat0<>+0x08(SB)/8, $0x0101010102020202 +DATA expandAVX512_36_mat0<>+0x10(SB)/8, $0x0202020202020202 +DATA expandAVX512_36_mat0<>+0x18(SB)/8, $0x0404040404040404 +DATA expandAVX512_36_mat0<>+0x20(SB)/8, $0x0404040408080808 +DATA expandAVX512_36_mat0<>+0x28(SB)/8, $0x0808080808080808 +DATA expandAVX512_36_mat0<>+0x30(SB)/8, $0x1010101010101010 +DATA expandAVX512_36_mat0<>+0x38(SB)/8, $0x1010101020202020 + +GLOBL expandAVX512_36_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_36_inShuf1<>+0x00(SB)/8, $0x0101010100000000 +DATA expandAVX512_36_inShuf1<>+0x08(SB)/8, $0xffffff0100000000 +DATA expandAVX512_36_inShuf1<>+0x10(SB)/8, $0xffffffffffffff00 +DATA expandAVX512_36_inShuf1<>+0x18(SB)/8, $0xffffffff00000000 +DATA expandAVX512_36_inShuf1<>+0x20(SB)/8, $0xff02020202010101 +DATA expandAVX512_36_inShuf1<>+0x28(SB)/8, $0xffffffffffff0201 +DATA expandAVX512_36_inShuf1<>+0x30(SB)/8, $0x0202020201010101 +DATA expandAVX512_36_inShuf1<>+0x38(SB)/8, $0x0303030302020202 + +GLOBL expandAVX512_36_mat1<>(SB), RODATA, $0x40 +DATA expandAVX512_36_mat1<>+0x00(SB)/8, $0x2020202020202020 +DATA expandAVX512_36_mat1<>+0x08(SB)/8, $0x4040404040404040 +DATA expandAVX512_36_mat1<>+0x10(SB)/8, $0x4040404080808080 +DATA expandAVX512_36_mat1<>+0x18(SB)/8, $0x8080808080808080 +DATA expandAVX512_36_mat1<>+0x20(SB)/8, $0x4040404040404040 +DATA expandAVX512_36_mat1<>+0x28(SB)/8, $0x4040404080808080 +DATA expandAVX512_36_mat1<>+0x30(SB)/8, $0x8080808080808080 +DATA expandAVX512_36_mat1<>+0x38(SB)/8, $0x0101010101010101 + +GLOBL expandAVX512_36_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_36_inShuf2<>+0x00(SB)/8, $0xffffffffffff0302 +DATA expandAVX512_36_inShuf2<>+0x08(SB)/8, $0x0303030302020202 +DATA expandAVX512_36_inShuf2<>+0x10(SB)/8, $0x0303030302020202 +DATA expandAVX512_36_inShuf2<>+0x18(SB)/8, $0xffffffffffff0302 +DATA expandAVX512_36_inShuf2<>+0x20(SB)/8, $0x0303030302020202 +DATA expandAVX512_36_inShuf2<>+0x28(SB)/8, $0xffff030302020202 +DATA expandAVX512_36_inShuf2<>+0x30(SB)/8, $0xffffffffffffff02 +DATA expandAVX512_36_inShuf2<>+0x38(SB)/8, $0xffffffff02020202 + +GLOBL expandAVX512_36_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_36_mat2<>+0x00(SB)/8, $0x0101010102020202 +DATA expandAVX512_36_mat2<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_36_mat2<>+0x10(SB)/8, $0x0404040404040404 +DATA expandAVX512_36_mat2<>+0x18(SB)/8, $0x0404040408080808 +DATA expandAVX512_36_mat2<>+0x20(SB)/8, $0x0808080808080808 +DATA expandAVX512_36_mat2<>+0x28(SB)/8, $0x1010101010101010 +DATA expandAVX512_36_mat2<>+0x30(SB)/8, $0x1010101020202020 +DATA expandAVX512_36_mat2<>+0x38(SB)/8, $0x2020202020202020 + +GLOBL expandAVX512_36_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_36_outShufLo+0x00(SB)/8, $0x1211100803020100 +DATA expandAVX512_36_outShufLo+0x08(SB)/8, $0x2928201b1a191813 +DATA expandAVX512_36_outShufLo+0x10(SB)/8, $0x4038333231302b2a +DATA expandAVX512_36_outShufLo+0x18(SB)/8, $0x504b4a4948434241 +DATA expandAVX512_36_outShufLo+0x20(SB)/8, $0x070605045b5a5958 +DATA expandAVX512_36_outShufLo+0x28(SB)/8, $0x1e1d1c1716151409 +DATA expandAVX512_36_outShufLo+0x30(SB)/8, $0x35342f2e2d2c211f +DATA expandAVX512_36_outShufLo+0x38(SB)/8, $0x4c47464544393736 + +GLOBL expandAVX512_36_outShufHi(SB), RODATA, $0x40 +DATA expandAVX512_36_outShufHi+0x00(SB)/8, $0x3332313028222120 +DATA expandAVX512_36_outShufHi+0x08(SB)/8, $0x4a4948403b3a3938 +DATA expandAVX512_36_outShufHi+0x10(SB)/8, $0x616058535251504b +DATA expandAVX512_36_outShufHi+0x18(SB)/8, $0x78706b6a69686362 +DATA expandAVX512_36_outShufHi+0x20(SB)/8, $0x29262524237b7a79 +DATA expandAVX512_36_outShufHi+0x28(SB)/8, $0x3f3e3d3c37363534 +DATA expandAVX512_36_outShufHi+0x30(SB)/8, $0x5655544f4e4d4c41 +DATA expandAVX512_36_outShufHi+0x38(SB)/8, $0x6d6c676665645957 + +TEXT expandAVX512_36<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_36_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_36_inShuf1<>(SB), Z3 + VMOVDQU64 expandAVX512_36_inShuf2<>(SB), Z4 + VMOVDQU64 expandAVX512_36_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_36_outShufHi(SB), Z2 + VMOVDQU64 (AX), Z5 + VPERMB Z5, Z0, Z0 + VGF2P8AFFINEQB $0, expandAVX512_36_mat0<>(SB), Z0, Z0 + VPERMB Z5, Z3, Z3 + VGF2P8AFFINEQB $0, expandAVX512_36_mat1<>(SB), Z3, Z3 + VPERMB Z5, Z4, Z4 + VGF2P8AFFINEQB $0, expandAVX512_36_mat2<>(SB), Z4, Z4 + VPERMI2B Z3, Z0, Z1 + VPERMI2B Z4, Z3, Z2 + RET + +GLOBL expandAVX512_40_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_40_inShuf0<>+0x00(SB)/8, $0x0101010000000000 +DATA expandAVX512_40_inShuf0<>+0x08(SB)/8, $0x0101010000000000 +DATA expandAVX512_40_inShuf0<>+0x10(SB)/8, $0x0101010000000000 +DATA expandAVX512_40_inShuf0<>+0x18(SB)/8, $0x0101010000000000 +DATA expandAVX512_40_inShuf0<>+0x20(SB)/8, $0x0101010000000000 +DATA expandAVX512_40_inShuf0<>+0x28(SB)/8, $0xffffff0000000000 +DATA expandAVX512_40_inShuf0<>+0x30(SB)/8, $0xffffff0000000000 +DATA expandAVX512_40_inShuf0<>+0x38(SB)/8, $0xffffff0000000000 + +GLOBL expandAVX512_40_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_40_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_40_mat0<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_40_mat0<>+0x10(SB)/8, $0x0404040404040404 +DATA expandAVX512_40_mat0<>+0x18(SB)/8, $0x0808080808080808 +DATA expandAVX512_40_mat0<>+0x20(SB)/8, $0x1010101010101010 +DATA expandAVX512_40_mat0<>+0x28(SB)/8, $0x2020202020202020 +DATA expandAVX512_40_mat0<>+0x30(SB)/8, $0x4040404040404040 +DATA expandAVX512_40_mat0<>+0x38(SB)/8, $0x8080808080808080 + +GLOBL expandAVX512_40_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_40_inShuf1<>+0x00(SB)/8, $0xffffffffffff0101 +DATA expandAVX512_40_inShuf1<>+0x08(SB)/8, $0xffffffffffff0101 +DATA expandAVX512_40_inShuf1<>+0x10(SB)/8, $0xffffffffffff0101 +DATA expandAVX512_40_inShuf1<>+0x18(SB)/8, $0xffffffffffff0101 +DATA expandAVX512_40_inShuf1<>+0x20(SB)/8, $0xffffffffffffff01 +DATA expandAVX512_40_inShuf1<>+0x28(SB)/8, $0xffff020202020201 +DATA expandAVX512_40_inShuf1<>+0x30(SB)/8, $0x0202020101010101 +DATA expandAVX512_40_inShuf1<>+0x38(SB)/8, $0x0202020101010101 + +GLOBL expandAVX512_40_mat1<>(SB), RODATA, $0x40 +DATA expandAVX512_40_mat1<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_40_mat1<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_40_mat1<>+0x10(SB)/8, $0x0404040404040404 +DATA expandAVX512_40_mat1<>+0x18(SB)/8, $0x0808080808080808 +DATA expandAVX512_40_mat1<>+0x20(SB)/8, $0x1010101010101010 +DATA expandAVX512_40_mat1<>+0x28(SB)/8, $0x1010101010101010 +DATA expandAVX512_40_mat1<>+0x30(SB)/8, $0x2020202020202020 +DATA expandAVX512_40_mat1<>+0x38(SB)/8, $0x4040404040404040 + +GLOBL expandAVX512_40_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_40_inShuf2<>+0x00(SB)/8, $0x0202020101010101 +DATA expandAVX512_40_inShuf2<>+0x08(SB)/8, $0x0303030202020202 +DATA expandAVX512_40_inShuf2<>+0x10(SB)/8, $0x0303030202020202 +DATA expandAVX512_40_inShuf2<>+0x18(SB)/8, $0xffffff0202020202 +DATA expandAVX512_40_inShuf2<>+0x20(SB)/8, $0xffffff0202020202 +DATA expandAVX512_40_inShuf2<>+0x28(SB)/8, $0xffffffffffff0202 +DATA expandAVX512_40_inShuf2<>+0x30(SB)/8, $0xffffffffffff0202 +DATA expandAVX512_40_inShuf2<>+0x38(SB)/8, $0xffffffffffff0202 + +GLOBL expandAVX512_40_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_40_mat2<>+0x00(SB)/8, $0x8080808080808080 +DATA expandAVX512_40_mat2<>+0x08(SB)/8, $0x0101010101010101 +DATA expandAVX512_40_mat2<>+0x10(SB)/8, $0x0202020202020202 +DATA expandAVX512_40_mat2<>+0x18(SB)/8, $0x0404040404040404 +DATA expandAVX512_40_mat2<>+0x20(SB)/8, $0x0808080808080808 +DATA expandAVX512_40_mat2<>+0x28(SB)/8, $0x2020202020202020 +DATA expandAVX512_40_mat2<>+0x30(SB)/8, $0x4040404040404040 +DATA expandAVX512_40_mat2<>+0x38(SB)/8, $0x8080808080808080 + +GLOBL expandAVX512_40_inShuf3<>(SB), RODATA, $0x40 +DATA expandAVX512_40_inShuf3<>+0x00(SB)/8, $0xffffffffffff0303 +DATA expandAVX512_40_inShuf3<>+0x08(SB)/8, $0xffffffffffffffff +DATA expandAVX512_40_inShuf3<>+0x10(SB)/8, $0xffffffffffffffff +DATA expandAVX512_40_inShuf3<>+0x18(SB)/8, $0xffffffffffffffff +DATA expandAVX512_40_inShuf3<>+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_40_inShuf3<>+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_40_inShuf3<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_40_inShuf3<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_40_mat3<>(SB), RODATA, $0x40 +DATA expandAVX512_40_mat3<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_40_mat3<>+0x08(SB)/8, $0x0000000000000000 +DATA expandAVX512_40_mat3<>+0x10(SB)/8, $0x0000000000000000 +DATA expandAVX512_40_mat3<>+0x18(SB)/8, $0x0000000000000000 +DATA expandAVX512_40_mat3<>+0x20(SB)/8, $0x0000000000000000 +DATA expandAVX512_40_mat3<>+0x28(SB)/8, $0x0000000000000000 +DATA expandAVX512_40_mat3<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_40_mat3<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_40_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_40_outShufLo+0x00(SB)/8, $0x0a09080403020100 +DATA expandAVX512_40_outShufLo+0x08(SB)/8, $0x1814131211100c0b +DATA expandAVX512_40_outShufLo+0x10(SB)/8, $0x232221201c1b1a19 +DATA expandAVX512_40_outShufLo+0x18(SB)/8, $0x31302c2b2a292824 +DATA expandAVX512_40_outShufLo+0x20(SB)/8, $0x3c3b3a3938343332 +DATA expandAVX512_40_outShufLo+0x28(SB)/8, $0x0f0e0d4140070605 +DATA expandAVX512_40_outShufLo+0x30(SB)/8, $0x1d51501716154948 +DATA expandAVX512_40_outShufLo+0x38(SB)/8, $0x6027262559581f1e + +GLOBL expandAVX512_40_outShufHi0(SB), RODATA, $0x40 +DATA expandAVX512_40_outShufHi0+0x00(SB)/8, $0x3938343332313028 +DATA expandAVX512_40_outShufHi0+0x08(SB)/8, $0x44434241403c3b3a +DATA expandAVX512_40_outShufHi0+0x10(SB)/8, $0x5251504c4b4a4948 +DATA expandAVX512_40_outShufHi0+0x18(SB)/8, $0x605c5b5a59585453 +DATA expandAVX512_40_outShufHi0+0x20(SB)/8, $0x2c2b2a2964636261 +DATA expandAVX512_40_outShufHi0+0x28(SB)/8, $0x3e3d69683736352d +DATA expandAVX512_40_outShufHi0+0x30(SB)/8, $0x797847464571703f +DATA expandAVX512_40_outShufHi0+0x38(SB)/8, $0x575655ffff4f4e4d + +GLOBL expandAVX512_40_outShufHi1(SB), RODATA, $0x40 +DATA expandAVX512_40_outShufHi1+0x00(SB)/8, $0xffffffffffffffff +DATA expandAVX512_40_outShufHi1+0x08(SB)/8, $0xffffffffffffffff +DATA expandAVX512_40_outShufHi1+0x10(SB)/8, $0xffffffffffffffff +DATA expandAVX512_40_outShufHi1+0x18(SB)/8, $0xffffffffffffffff +DATA expandAVX512_40_outShufHi1+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_40_outShufHi1+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_40_outShufHi1+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_40_outShufHi1+0x38(SB)/8, $0xffffff0100ffffff + +TEXT expandAVX512_40<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_40_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_40_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_40_inShuf2<>(SB), Z3 + VMOVDQU64 expandAVX512_40_inShuf3<>(SB), Z4 + VMOVDQU64 expandAVX512_40_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_40_outShufHi0(SB), Z5 + VMOVDQU64 expandAVX512_40_outShufHi1(SB), Z6 + VMOVDQU64 (AX), Z7 + VPERMB Z7, Z0, Z0 + VGF2P8AFFINEQB $0, expandAVX512_40_mat0<>(SB), Z0, Z0 + VPERMB Z7, Z2, Z2 + VGF2P8AFFINEQB $0, expandAVX512_40_mat1<>(SB), Z2, Z2 + VPERMB Z7, Z3, Z3 + VGF2P8AFFINEQB $0, expandAVX512_40_mat2<>(SB), Z3, Z3 + VPERMB Z7, Z4, Z4 + VGF2P8AFFINEQB $0, expandAVX512_40_mat3<>(SB), Z4, Z4 + VPERMI2B Z2, Z0, Z1 + MOVQ $0xe7ffffffffffffff, AX + KMOVQ AX, K1 + VPERMI2B.Z Z3, Z2, K1, Z5 + MOVQ $0x1800000000000000, AX + KMOVQ AX, K1 + VPERMB.Z Z4, Z6, K1, Z0 + VPORQ Z0, Z5, Z2 + RET + +GLOBL expandAVX512_44_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_44_inShuf0<>+0x00(SB)/8, $0x0101010000000000 +DATA expandAVX512_44_inShuf0<>+0x08(SB)/8, $0xffffffffffff0100 +DATA expandAVX512_44_inShuf0<>+0x10(SB)/8, $0x0101010000000000 +DATA expandAVX512_44_inShuf0<>+0x18(SB)/8, $0x0101010000000000 +DATA expandAVX512_44_inShuf0<>+0x20(SB)/8, $0xffffffffffff0100 +DATA expandAVX512_44_inShuf0<>+0x28(SB)/8, $0x0101010000000000 +DATA expandAVX512_44_inShuf0<>+0x30(SB)/8, $0xffffff0000000000 +DATA expandAVX512_44_inShuf0<>+0x38(SB)/8, $0xffffffffffffff00 + +GLOBL expandAVX512_44_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_44_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_44_mat0<>+0x08(SB)/8, $0x0101010102020202 +DATA expandAVX512_44_mat0<>+0x10(SB)/8, $0x0202020202020202 +DATA expandAVX512_44_mat0<>+0x18(SB)/8, $0x0404040404040404 +DATA expandAVX512_44_mat0<>+0x20(SB)/8, $0x0404040408080808 +DATA expandAVX512_44_mat0<>+0x28(SB)/8, $0x0808080808080808 +DATA expandAVX512_44_mat0<>+0x30(SB)/8, $0x1010101010101010 +DATA expandAVX512_44_mat0<>+0x38(SB)/8, $0x1010101020202020 + +GLOBL expandAVX512_44_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_44_inShuf1<>+0x00(SB)/8, $0xffffff0000000000 +DATA expandAVX512_44_inShuf1<>+0x08(SB)/8, $0xffffff0000000000 +DATA expandAVX512_44_inShuf1<>+0x10(SB)/8, $0xffffffffffffff00 +DATA expandAVX512_44_inShuf1<>+0x18(SB)/8, $0xffffff0000000000 +DATA expandAVX512_44_inShuf1<>+0x20(SB)/8, $0xffffffffffff0101 +DATA expandAVX512_44_inShuf1<>+0x28(SB)/8, $0xffffffffffff0101 +DATA expandAVX512_44_inShuf1<>+0x30(SB)/8, $0xffffffffffff0101 +DATA expandAVX512_44_inShuf1<>+0x38(SB)/8, $0xff02020202020101 + +GLOBL expandAVX512_44_mat1<>(SB), RODATA, $0x40 +DATA expandAVX512_44_mat1<>+0x00(SB)/8, $0x2020202020202020 +DATA expandAVX512_44_mat1<>+0x08(SB)/8, $0x4040404040404040 +DATA expandAVX512_44_mat1<>+0x10(SB)/8, $0x4040404080808080 +DATA expandAVX512_44_mat1<>+0x18(SB)/8, $0x8080808080808080 +DATA expandAVX512_44_mat1<>+0x20(SB)/8, $0x0101010101010101 +DATA expandAVX512_44_mat1<>+0x28(SB)/8, $0x0202020202020202 +DATA expandAVX512_44_mat1<>+0x30(SB)/8, $0x0404040404040404 +DATA expandAVX512_44_mat1<>+0x38(SB)/8, $0x0808080808080808 + +GLOBL expandAVX512_44_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_44_inShuf2<>+0x00(SB)/8, $0x0202020101010101 +DATA expandAVX512_44_inShuf2<>+0x08(SB)/8, $0xffffffffffff0201 +DATA expandAVX512_44_inShuf2<>+0x10(SB)/8, $0x0202020101010101 +DATA expandAVX512_44_inShuf2<>+0x18(SB)/8, $0x0202020101010101 +DATA expandAVX512_44_inShuf2<>+0x20(SB)/8, $0xffffffffffff0201 +DATA expandAVX512_44_inShuf2<>+0x28(SB)/8, $0xffff020101010101 +DATA expandAVX512_44_inShuf2<>+0x30(SB)/8, $0xffffff0202020202 +DATA expandAVX512_44_inShuf2<>+0x38(SB)/8, $0xffffffffffffff02 + +GLOBL expandAVX512_44_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_44_mat2<>+0x00(SB)/8, $0x1010101010101010 +DATA expandAVX512_44_mat2<>+0x08(SB)/8, $0x1010101020202020 +DATA expandAVX512_44_mat2<>+0x10(SB)/8, $0x2020202020202020 +DATA expandAVX512_44_mat2<>+0x18(SB)/8, $0x4040404040404040 +DATA expandAVX512_44_mat2<>+0x20(SB)/8, $0x4040404080808080 +DATA expandAVX512_44_mat2<>+0x28(SB)/8, $0x8080808080808080 +DATA expandAVX512_44_mat2<>+0x30(SB)/8, $0x0101010101010101 +DATA expandAVX512_44_mat2<>+0x38(SB)/8, $0x0101010102020202 + +GLOBL expandAVX512_44_inShuf3<>(SB), RODATA, $0x40 +DATA expandAVX512_44_inShuf3<>+0x00(SB)/8, $0xffffff0202020202 +DATA expandAVX512_44_inShuf3<>+0x08(SB)/8, $0xffffff0202020202 +DATA expandAVX512_44_inShuf3<>+0x10(SB)/8, $0xffffffffffffff02 +DATA expandAVX512_44_inShuf3<>+0x18(SB)/8, $0xffffffffffff0202 +DATA expandAVX512_44_inShuf3<>+0x20(SB)/8, $0xffffffffffff0202 +DATA expandAVX512_44_inShuf3<>+0x28(SB)/8, $0xffffffffffff0202 +DATA expandAVX512_44_inShuf3<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_44_inShuf3<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_44_mat3<>(SB), RODATA, $0x40 +DATA expandAVX512_44_mat3<>+0x00(SB)/8, $0x0202020202020202 +DATA expandAVX512_44_mat3<>+0x08(SB)/8, $0x0404040404040404 +DATA expandAVX512_44_mat3<>+0x10(SB)/8, $0x0404040408080808 +DATA expandAVX512_44_mat3<>+0x18(SB)/8, $0x1010101010101010 +DATA expandAVX512_44_mat3<>+0x20(SB)/8, $0x2020202020202020 +DATA expandAVX512_44_mat3<>+0x28(SB)/8, $0x4040404040404040 +DATA expandAVX512_44_mat3<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_44_mat3<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_44_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_44_outShufLo+0x00(SB)/8, $0x1110080403020100 +DATA expandAVX512_44_outShufLo+0x08(SB)/8, $0x1c1b1a1918141312 +DATA expandAVX512_44_outShufLo+0x10(SB)/8, $0x31302c2b2a292820 +DATA expandAVX512_44_outShufLo+0x18(SB)/8, $0x4342414038343332 +DATA expandAVX512_44_outShufLo+0x20(SB)/8, $0x58504c4b4a494844 +DATA expandAVX512_44_outShufLo+0x28(SB)/8, $0x600706055c5b5a59 +DATA expandAVX512_44_outShufLo+0x30(SB)/8, $0x1d69681716150961 +DATA expandAVX512_44_outShufLo+0x38(SB)/8, $0x2f2e2d2171701f1e + +GLOBL expandAVX512_44_outShufHi0(SB), RODATA, $0x40 +DATA expandAVX512_44_outShufHi0+0x00(SB)/8, $0x4844434241403938 +DATA expandAVX512_44_outShufHi0+0x08(SB)/8, $0x5a59585453525150 +DATA expandAVX512_44_outShufHi0+0x10(SB)/8, $0x6c6b6a6968605c5b +DATA expandAVX512_44_outShufHi0+0x18(SB)/8, $0xffff787473727170 +DATA expandAVX512_44_outShufHi0+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_44_outShufHi0+0x28(SB)/8, $0x46453e3d3c3b3aff +DATA expandAVX512_44_outShufHi0+0x30(SB)/8, $0xff57565549ffff47 +DATA expandAVX512_44_outShufHi0+0x38(SB)/8, $0x6d61ffff5f5e5dff + +GLOBL expandAVX512_44_outShufHi1(SB), RODATA, $0x40 +DATA expandAVX512_44_outShufHi1+0x00(SB)/8, $0xffffffffffffffff +DATA expandAVX512_44_outShufHi1+0x08(SB)/8, $0xffffffffffffffff +DATA expandAVX512_44_outShufHi1+0x10(SB)/8, $0xffffffffffffffff +DATA expandAVX512_44_outShufHi1+0x18(SB)/8, $0x0100ffffffffffff +DATA expandAVX512_44_outShufHi1+0x20(SB)/8, $0x0c0b0a0908040302 +DATA expandAVX512_44_outShufHi1+0x28(SB)/8, $0xffffffffffffff10 +DATA expandAVX512_44_outShufHi1+0x30(SB)/8, $0x20ffffffff1918ff +DATA expandAVX512_44_outShufHi1+0x38(SB)/8, $0xffff2928ffffff21 + +TEXT expandAVX512_44<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_44_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_44_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_44_inShuf2<>(SB), Z3 + VMOVDQU64 expandAVX512_44_inShuf3<>(SB), Z4 + VMOVDQU64 expandAVX512_44_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_44_outShufHi0(SB), Z5 + VMOVDQU64 expandAVX512_44_outShufHi1(SB), Z6 + VMOVDQU64 (AX), Z7 + VPERMB Z7, Z0, Z0 + VGF2P8AFFINEQB $0, expandAVX512_44_mat0<>(SB), Z0, Z0 + VPERMB Z7, Z2, Z2 + VGF2P8AFFINEQB $0, expandAVX512_44_mat1<>(SB), Z2, Z2 + VPERMB Z7, Z3, Z3 + VGF2P8AFFINEQB $0, expandAVX512_44_mat2<>(SB), Z3, Z3 + VPERMB Z7, Z4, Z4 + VGF2P8AFFINEQB $0, expandAVX512_44_mat3<>(SB), Z4, Z4 + VPERMI2B Z2, Z0, Z1 + MOVQ $0xce79fe003fffffff, AX + KMOVQ AX, K1 + VPERMI2B.Z Z3, Z2, K1, Z5 + MOVQ $0x318601ffc0000000, AX + KMOVQ AX, K1 + VPERMB.Z Z4, Z6, K1, Z0 + VPORQ Z0, Z5, Z2 + RET + +GLOBL expandAVX512_48_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_48_inShuf0<>+0x00(SB)/8, $0x0101000000000000 +DATA expandAVX512_48_inShuf0<>+0x08(SB)/8, $0x0101000000000000 +DATA expandAVX512_48_inShuf0<>+0x10(SB)/8, $0x0101000000000000 +DATA expandAVX512_48_inShuf0<>+0x18(SB)/8, $0xffff000000000000 +DATA expandAVX512_48_inShuf0<>+0x20(SB)/8, $0xffff000000000000 +DATA expandAVX512_48_inShuf0<>+0x28(SB)/8, $0xffff000000000000 +DATA expandAVX512_48_inShuf0<>+0x30(SB)/8, $0xffff000000000000 +DATA expandAVX512_48_inShuf0<>+0x38(SB)/8, $0xffff000000000000 + +GLOBL expandAVX512_48_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_48_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_48_mat0<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_48_mat0<>+0x10(SB)/8, $0x0404040404040404 +DATA expandAVX512_48_mat0<>+0x18(SB)/8, $0x0808080808080808 +DATA expandAVX512_48_mat0<>+0x20(SB)/8, $0x1010101010101010 +DATA expandAVX512_48_mat0<>+0x28(SB)/8, $0x2020202020202020 +DATA expandAVX512_48_mat0<>+0x30(SB)/8, $0x4040404040404040 +DATA expandAVX512_48_mat0<>+0x38(SB)/8, $0x8080808080808080 + +GLOBL expandAVX512_48_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_48_inShuf1<>+0x00(SB)/8, $0xffffffff01010101 +DATA expandAVX512_48_inShuf1<>+0x08(SB)/8, $0xffffffff01010101 +DATA expandAVX512_48_inShuf1<>+0x10(SB)/8, $0xffffffffffff0101 +DATA expandAVX512_48_inShuf1<>+0x18(SB)/8, $0x0202020202020101 +DATA expandAVX512_48_inShuf1<>+0x20(SB)/8, $0x0202010101010101 +DATA expandAVX512_48_inShuf1<>+0x28(SB)/8, $0x0202010101010101 +DATA expandAVX512_48_inShuf1<>+0x30(SB)/8, $0x0202010101010101 +DATA expandAVX512_48_inShuf1<>+0x38(SB)/8, $0xffff010101010101 + +GLOBL expandAVX512_48_mat1<>(SB), RODATA, $0x40 +DATA expandAVX512_48_mat1<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_48_mat1<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_48_mat1<>+0x10(SB)/8, $0x0404040404040404 +DATA expandAVX512_48_mat1<>+0x18(SB)/8, $0x0404040404040404 +DATA expandAVX512_48_mat1<>+0x20(SB)/8, $0x0808080808080808 +DATA expandAVX512_48_mat1<>+0x28(SB)/8, $0x1010101010101010 +DATA expandAVX512_48_mat1<>+0x30(SB)/8, $0x2020202020202020 +DATA expandAVX512_48_mat1<>+0x38(SB)/8, $0x4040404040404040 + +GLOBL expandAVX512_48_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_48_inShuf2<>+0x00(SB)/8, $0xffff010101010101 +DATA expandAVX512_48_inShuf2<>+0x08(SB)/8, $0xffff020202020202 +DATA expandAVX512_48_inShuf2<>+0x10(SB)/8, $0xffff020202020202 +DATA expandAVX512_48_inShuf2<>+0x18(SB)/8, $0xffffffff02020202 +DATA expandAVX512_48_inShuf2<>+0x20(SB)/8, $0xffffffff02020202 +DATA expandAVX512_48_inShuf2<>+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_48_inShuf2<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_48_inShuf2<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_48_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_48_mat2<>+0x00(SB)/8, $0x8080808080808080 +DATA expandAVX512_48_mat2<>+0x08(SB)/8, $0x0101010101010101 +DATA expandAVX512_48_mat2<>+0x10(SB)/8, $0x0202020202020202 +DATA expandAVX512_48_mat2<>+0x18(SB)/8, $0x0808080808080808 +DATA expandAVX512_48_mat2<>+0x20(SB)/8, $0x1010101010101010 +DATA expandAVX512_48_mat2<>+0x28(SB)/8, $0x0000000000000000 +DATA expandAVX512_48_mat2<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_48_mat2<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_48_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_48_outShufLo+0x00(SB)/8, $0x0908050403020100 +DATA expandAVX512_48_outShufLo+0x08(SB)/8, $0x131211100d0c0b0a +DATA expandAVX512_48_outShufLo+0x10(SB)/8, $0x1d1c1b1a19181514 +DATA expandAVX512_48_outShufLo+0x18(SB)/8, $0x2928252423222120 +DATA expandAVX512_48_outShufLo+0x20(SB)/8, $0x333231302d2c2b2a +DATA expandAVX512_48_outShufLo+0x28(SB)/8, $0x3d3c3b3a39383534 +DATA expandAVX512_48_outShufLo+0x30(SB)/8, $0x0f0e434241400706 +DATA expandAVX512_48_outShufLo+0x38(SB)/8, $0x515017164b4a4948 + +GLOBL expandAVX512_48_outShufHi(SB), RODATA, $0x40 +DATA expandAVX512_48_outShufHi+0x00(SB)/8, $0x2524232221201918 +DATA expandAVX512_48_outShufHi+0x08(SB)/8, $0x31302d2c2b2a2928 +DATA expandAVX512_48_outShufHi+0x10(SB)/8, $0x3b3a393835343332 +DATA expandAVX512_48_outShufHi+0x18(SB)/8, $0x4544434241403d3c +DATA expandAVX512_48_outShufHi+0x20(SB)/8, $0x51504d4c4b4a4948 +DATA expandAVX512_48_outShufHi+0x28(SB)/8, $0x1d1c1b1a55545352 +DATA expandAVX512_48_outShufHi+0x30(SB)/8, $0x5b5a595827261f1e +DATA expandAVX512_48_outShufHi+0x38(SB)/8, $0x3736636261602f2e + +TEXT expandAVX512_48<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_48_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_48_inShuf1<>(SB), Z3 + VMOVDQU64 expandAVX512_48_inShuf2<>(SB), Z4 + VMOVDQU64 expandAVX512_48_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_48_outShufHi(SB), Z2 + VMOVDQU64 (AX), Z5 + VPERMB Z5, Z0, Z0 + VGF2P8AFFINEQB $0, expandAVX512_48_mat0<>(SB), Z0, Z0 + VPERMB Z5, Z3, Z3 + VGF2P8AFFINEQB $0, expandAVX512_48_mat1<>(SB), Z3, Z3 + VPERMB Z5, Z4, Z4 + VGF2P8AFFINEQB $0, expandAVX512_48_mat2<>(SB), Z4, Z4 + VPERMI2B Z3, Z0, Z1 + VPERMI2B Z4, Z3, Z2 + RET + +GLOBL expandAVX512_52_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_52_inShuf0<>+0x00(SB)/8, $0x0101000000000000 +DATA expandAVX512_52_inShuf0<>+0x08(SB)/8, $0xffffffffffff0100 +DATA expandAVX512_52_inShuf0<>+0x10(SB)/8, $0x0101000000000000 +DATA expandAVX512_52_inShuf0<>+0x18(SB)/8, $0xffff000000000000 +DATA expandAVX512_52_inShuf0<>+0x20(SB)/8, $0xffffffffffffff00 +DATA expandAVX512_52_inShuf0<>+0x28(SB)/8, $0xffff000000000000 +DATA expandAVX512_52_inShuf0<>+0x30(SB)/8, $0xffff000000000000 +DATA expandAVX512_52_inShuf0<>+0x38(SB)/8, $0xffffffffffffff00 + +GLOBL expandAVX512_52_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_52_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_52_mat0<>+0x08(SB)/8, $0x0101010102020202 +DATA expandAVX512_52_mat0<>+0x10(SB)/8, $0x0202020202020202 +DATA expandAVX512_52_mat0<>+0x18(SB)/8, $0x0404040404040404 +DATA expandAVX512_52_mat0<>+0x20(SB)/8, $0x0404040408080808 +DATA expandAVX512_52_mat0<>+0x28(SB)/8, $0x0808080808080808 +DATA expandAVX512_52_mat0<>+0x30(SB)/8, $0x1010101010101010 +DATA expandAVX512_52_mat0<>+0x38(SB)/8, $0x1010101020202020 + +GLOBL expandAVX512_52_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_52_inShuf1<>+0x00(SB)/8, $0xffff000000000000 +DATA expandAVX512_52_inShuf1<>+0x08(SB)/8, $0xffff000000000000 +DATA expandAVX512_52_inShuf1<>+0x10(SB)/8, $0xffffffffffffff00 +DATA expandAVX512_52_inShuf1<>+0x18(SB)/8, $0xffff000000000000 +DATA expandAVX512_52_inShuf1<>+0x20(SB)/8, $0xffffffff01010101 +DATA expandAVX512_52_inShuf1<>+0x28(SB)/8, $0xffffffffff010101 +DATA expandAVX512_52_inShuf1<>+0x30(SB)/8, $0xff02020202020201 +DATA expandAVX512_52_inShuf1<>+0x38(SB)/8, $0x0202010101010101 + +GLOBL expandAVX512_52_mat1<>(SB), RODATA, $0x40 +DATA expandAVX512_52_mat1<>+0x00(SB)/8, $0x2020202020202020 +DATA expandAVX512_52_mat1<>+0x08(SB)/8, $0x4040404040404040 +DATA expandAVX512_52_mat1<>+0x10(SB)/8, $0x4040404080808080 +DATA expandAVX512_52_mat1<>+0x18(SB)/8, $0x8080808080808080 +DATA expandAVX512_52_mat1<>+0x20(SB)/8, $0x0101010101010101 +DATA expandAVX512_52_mat1<>+0x28(SB)/8, $0x0202020202020202 +DATA expandAVX512_52_mat1<>+0x30(SB)/8, $0x0202020202020202 +DATA expandAVX512_52_mat1<>+0x38(SB)/8, $0x0404040404040404 + +GLOBL expandAVX512_52_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_52_inShuf2<>+0x00(SB)/8, $0xffffffffffff0201 +DATA expandAVX512_52_inShuf2<>+0x08(SB)/8, $0x0202010101010101 +DATA expandAVX512_52_inShuf2<>+0x10(SB)/8, $0xffff010101010101 +DATA expandAVX512_52_inShuf2<>+0x18(SB)/8, $0xffffffffffffff01 +DATA expandAVX512_52_inShuf2<>+0x20(SB)/8, $0xffff010101010101 +DATA expandAVX512_52_inShuf2<>+0x28(SB)/8, $0xffff010101010101 +DATA expandAVX512_52_inShuf2<>+0x30(SB)/8, $0xffffffffffffff01 +DATA expandAVX512_52_inShuf2<>+0x38(SB)/8, $0xffff010101010101 + +GLOBL expandAVX512_52_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_52_mat2<>+0x00(SB)/8, $0x0404040408080808 +DATA expandAVX512_52_mat2<>+0x08(SB)/8, $0x0808080808080808 +DATA expandAVX512_52_mat2<>+0x10(SB)/8, $0x1010101010101010 +DATA expandAVX512_52_mat2<>+0x18(SB)/8, $0x1010101020202020 +DATA expandAVX512_52_mat2<>+0x20(SB)/8, $0x2020202020202020 +DATA expandAVX512_52_mat2<>+0x28(SB)/8, $0x4040404040404040 +DATA expandAVX512_52_mat2<>+0x30(SB)/8, $0x4040404080808080 +DATA expandAVX512_52_mat2<>+0x38(SB)/8, $0x8080808080808080 + +GLOBL expandAVX512_52_inShuf3<>(SB), RODATA, $0x40 +DATA expandAVX512_52_inShuf3<>+0x00(SB)/8, $0xffff020202020202 +DATA expandAVX512_52_inShuf3<>+0x08(SB)/8, $0xffffffffffffff02 +DATA expandAVX512_52_inShuf3<>+0x10(SB)/8, $0xffffffff02020202 +DATA expandAVX512_52_inShuf3<>+0x18(SB)/8, $0xffffffffffff0202 +DATA expandAVX512_52_inShuf3<>+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_52_inShuf3<>+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_52_inShuf3<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_52_inShuf3<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_52_mat3<>(SB), RODATA, $0x40 +DATA expandAVX512_52_mat3<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_52_mat3<>+0x08(SB)/8, $0x0101010102020202 +DATA expandAVX512_52_mat3<>+0x10(SB)/8, $0x0404040404040404 +DATA expandAVX512_52_mat3<>+0x18(SB)/8, $0x0808080808080808 +DATA expandAVX512_52_mat3<>+0x20(SB)/8, $0x0000000000000000 +DATA expandAVX512_52_mat3<>+0x28(SB)/8, $0x0000000000000000 +DATA expandAVX512_52_mat3<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_52_mat3<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_52_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_52_outShufLo+0x00(SB)/8, $0x1008050403020100 +DATA expandAVX512_52_outShufLo+0x08(SB)/8, $0x1a19181514131211 +DATA expandAVX512_52_outShufLo+0x10(SB)/8, $0x2b2a2928201d1c1b +DATA expandAVX512_52_outShufLo+0x18(SB)/8, $0x3534333231302d2c +DATA expandAVX512_52_outShufLo+0x20(SB)/8, $0x4845444342414038 +DATA expandAVX512_52_outShufLo+0x28(SB)/8, $0x5958504d4c4b4a49 +DATA expandAVX512_52_outShufLo+0x30(SB)/8, $0x616007065d5c5b5a +DATA expandAVX512_52_outShufLo+0x38(SB)/8, $0x6a69681716096362 + +GLOBL expandAVX512_52_outShufHi0(SB), RODATA, $0x40 +DATA expandAVX512_52_outShufHi0+0x00(SB)/8, $0x403d3c3b3a393830 +DATA expandAVX512_52_outShufHi0+0x08(SB)/8, $0x51504d4c4b4a4948 +DATA expandAVX512_52_outShufHi0+0x10(SB)/8, $0x6261605855545352 +DATA expandAVX512_52_outShufHi0+0x18(SB)/8, $0x6c6b6a6968656463 +DATA expandAVX512_52_outShufHi0+0x20(SB)/8, $0x7d7c7b7a7978706d +DATA expandAVX512_52_outShufHi0+0x28(SB)/8, $0x31ffffffffffffff +DATA expandAVX512_52_outShufHi0+0x30(SB)/8, $0xff3f3e3635343332 +DATA expandAVX512_52_outShufHi0+0x38(SB)/8, $0xffff4f4e41ffffff + +GLOBL expandAVX512_52_outShufHi1(SB), RODATA, $0x40 +DATA expandAVX512_52_outShufHi1+0x00(SB)/8, $0xffffffffffffffff +DATA expandAVX512_52_outShufHi1+0x08(SB)/8, $0xffffffffffffffff +DATA expandAVX512_52_outShufHi1+0x10(SB)/8, $0xffffffffffffffff +DATA expandAVX512_52_outShufHi1+0x18(SB)/8, $0xffffffffffffffff +DATA expandAVX512_52_outShufHi1+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_52_outShufHi1+0x28(SB)/8, $0xff08050403020100 +DATA expandAVX512_52_outShufHi1+0x30(SB)/8, $0x10ffffffffffffff +DATA expandAVX512_52_outShufHi1+0x38(SB)/8, $0x1918ffffff131211 + +TEXT expandAVX512_52<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_52_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_52_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_52_inShuf2<>(SB), Z3 + VMOVDQU64 expandAVX512_52_inShuf3<>(SB), Z4 + VMOVDQU64 expandAVX512_52_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_52_outShufHi0(SB), Z5 + VMOVDQU64 expandAVX512_52_outShufHi1(SB), Z6 + VMOVDQU64 (AX), Z7 + VPERMB Z7, Z0, Z0 + VGF2P8AFFINEQB $0, expandAVX512_52_mat0<>(SB), Z0, Z0 + VPERMB Z7, Z2, Z2 + VGF2P8AFFINEQB $0, expandAVX512_52_mat1<>(SB), Z2, Z2 + VPERMB Z7, Z3, Z3 + VGF2P8AFFINEQB $0, expandAVX512_52_mat2<>(SB), Z3, Z3 + VPERMB Z7, Z4, Z4 + VGF2P8AFFINEQB $0, expandAVX512_52_mat3<>(SB), Z4, Z4 + VPERMI2B Z2, Z0, Z1 + MOVQ $0x387f80ffffffffff, AX + KMOVQ AX, K1 + VPERMI2B.Z Z3, Z2, K1, Z5 + MOVQ $0xc7807f0000000000, AX + KMOVQ AX, K1 + VPERMB.Z Z4, Z6, K1, Z0 + VPORQ Z0, Z5, Z2 + RET + +GLOBL expandAVX512_56_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_56_inShuf0<>+0x00(SB)/8, $0x0100000000000000 +DATA expandAVX512_56_inShuf0<>+0x08(SB)/8, $0x0100000000000000 +DATA expandAVX512_56_inShuf0<>+0x10(SB)/8, $0xff00000000000000 +DATA expandAVX512_56_inShuf0<>+0x18(SB)/8, $0xff00000000000000 +DATA expandAVX512_56_inShuf0<>+0x20(SB)/8, $0xff00000000000000 +DATA expandAVX512_56_inShuf0<>+0x28(SB)/8, $0xff00000000000000 +DATA expandAVX512_56_inShuf0<>+0x30(SB)/8, $0xff00000000000000 +DATA expandAVX512_56_inShuf0<>+0x38(SB)/8, $0xff00000000000000 + +GLOBL expandAVX512_56_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_56_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_56_mat0<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_56_mat0<>+0x10(SB)/8, $0x0404040404040404 +DATA expandAVX512_56_mat0<>+0x18(SB)/8, $0x0808080808080808 +DATA expandAVX512_56_mat0<>+0x20(SB)/8, $0x1010101010101010 +DATA expandAVX512_56_mat0<>+0x28(SB)/8, $0x2020202020202020 +DATA expandAVX512_56_mat0<>+0x30(SB)/8, $0x4040404040404040 +DATA expandAVX512_56_mat0<>+0x38(SB)/8, $0x8080808080808080 + +GLOBL expandAVX512_56_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_56_inShuf1<>+0x00(SB)/8, $0xffff010101010101 +DATA expandAVX512_56_inShuf1<>+0x08(SB)/8, $0x0202010101010101 +DATA expandAVX512_56_inShuf1<>+0x10(SB)/8, $0x0201010101010101 +DATA expandAVX512_56_inShuf1<>+0x18(SB)/8, $0xff01010101010101 +DATA expandAVX512_56_inShuf1<>+0x20(SB)/8, $0xff01010101010101 +DATA expandAVX512_56_inShuf1<>+0x28(SB)/8, $0xff01010101010101 +DATA expandAVX512_56_inShuf1<>+0x30(SB)/8, $0xff01010101010101 +DATA expandAVX512_56_inShuf1<>+0x38(SB)/8, $0xff01010101010101 + +GLOBL expandAVX512_56_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_56_inShuf2<>+0x00(SB)/8, $0xff02020202020202 +DATA expandAVX512_56_inShuf2<>+0x08(SB)/8, $0xffffff0202020202 +DATA expandAVX512_56_inShuf2<>+0x10(SB)/8, $0xffffffffffffff02 +DATA expandAVX512_56_inShuf2<>+0x18(SB)/8, $0xffffffffffffffff +DATA expandAVX512_56_inShuf2<>+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_56_inShuf2<>+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_56_inShuf2<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_56_inShuf2<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_56_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_56_mat2<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_56_mat2<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_56_mat2<>+0x10(SB)/8, $0x0404040404040404 +DATA expandAVX512_56_mat2<>+0x18(SB)/8, $0x0000000000000000 +DATA expandAVX512_56_mat2<>+0x20(SB)/8, $0x0000000000000000 +DATA expandAVX512_56_mat2<>+0x28(SB)/8, $0x0000000000000000 +DATA expandAVX512_56_mat2<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_56_mat2<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_56_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_56_outShufLo+0x00(SB)/8, $0x0806050403020100 +DATA expandAVX512_56_outShufLo+0x08(SB)/8, $0x11100e0d0c0b0a09 +DATA expandAVX512_56_outShufLo+0x10(SB)/8, $0x1a19181615141312 +DATA expandAVX512_56_outShufLo+0x18(SB)/8, $0x232221201e1d1c1b +DATA expandAVX512_56_outShufLo+0x20(SB)/8, $0x2c2b2a2928262524 +DATA expandAVX512_56_outShufLo+0x28(SB)/8, $0x3534333231302e2d +DATA expandAVX512_56_outShufLo+0x30(SB)/8, $0x3e3d3c3b3a393836 +DATA expandAVX512_56_outShufLo+0x38(SB)/8, $0x0f45444342414007 + +GLOBL expandAVX512_56_outShufHi(SB), RODATA, $0x40 +DATA expandAVX512_56_outShufHi+0x00(SB)/8, $0x11100d0c0b0a0908 +DATA expandAVX512_56_outShufHi+0x08(SB)/8, $0x1a19181615141312 +DATA expandAVX512_56_outShufHi+0x10(SB)/8, $0x232221201e1d1c1b +DATA expandAVX512_56_outShufHi+0x18(SB)/8, $0x2c2b2a2928262524 +DATA expandAVX512_56_outShufHi+0x20(SB)/8, $0x3534333231302e2d +DATA expandAVX512_56_outShufHi+0x28(SB)/8, $0x3e3d3c3b3a393836 +DATA expandAVX512_56_outShufHi+0x30(SB)/8, $0x0e46454443424140 +DATA expandAVX512_56_outShufHi+0x38(SB)/8, $0x50174c4b4a49480f + +TEXT expandAVX512_56<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_56_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_56_mat0<>(SB), Z3 + VMOVDQU64 expandAVX512_56_inShuf1<>(SB), Z4 + VMOVDQU64 expandAVX512_56_inShuf2<>(SB), Z5 + VMOVDQU64 expandAVX512_56_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_56_outShufHi(SB), Z2 + VMOVDQU64 (AX), Z6 + VPERMB Z6, Z0, Z0 + VGF2P8AFFINEQB $0, Z3, Z0, Z0 + VPERMB Z6, Z4, Z4 + VGF2P8AFFINEQB $0, Z3, Z4, Z3 + VPERMB Z6, Z5, Z4 + VGF2P8AFFINEQB $0, expandAVX512_56_mat2<>(SB), Z4, Z4 + VPERMI2B Z3, Z0, Z1 + VPERMI2B Z4, Z3, Z2 + RET + +GLOBL expandAVX512_60_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_60_inShuf0<>+0x00(SB)/8, $0x0100000000000000 +DATA expandAVX512_60_inShuf0<>+0x08(SB)/8, $0xffffffffffffff00 +DATA expandAVX512_60_inShuf0<>+0x10(SB)/8, $0xff00000000000000 +DATA expandAVX512_60_inShuf0<>+0x18(SB)/8, $0xff00000000000000 +DATA expandAVX512_60_inShuf0<>+0x20(SB)/8, $0xffffffffffffff00 +DATA expandAVX512_60_inShuf0<>+0x28(SB)/8, $0xff00000000000000 +DATA expandAVX512_60_inShuf0<>+0x30(SB)/8, $0xff00000000000000 +DATA expandAVX512_60_inShuf0<>+0x38(SB)/8, $0xffffffffffffff00 + +GLOBL expandAVX512_60_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_60_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_60_mat0<>+0x08(SB)/8, $0x0101010102020202 +DATA expandAVX512_60_mat0<>+0x10(SB)/8, $0x0202020202020202 +DATA expandAVX512_60_mat0<>+0x18(SB)/8, $0x0404040404040404 +DATA expandAVX512_60_mat0<>+0x20(SB)/8, $0x0404040408080808 +DATA expandAVX512_60_mat0<>+0x28(SB)/8, $0x0808080808080808 +DATA expandAVX512_60_mat0<>+0x30(SB)/8, $0x1010101010101010 +DATA expandAVX512_60_mat0<>+0x38(SB)/8, $0x1010101020202020 + +GLOBL expandAVX512_60_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_60_inShuf1<>+0x00(SB)/8, $0xff00000000000000 +DATA expandAVX512_60_inShuf1<>+0x08(SB)/8, $0xff00000000000000 +DATA expandAVX512_60_inShuf1<>+0x10(SB)/8, $0xffffffffffffff00 +DATA expandAVX512_60_inShuf1<>+0x18(SB)/8, $0xff00000000000000 +DATA expandAVX512_60_inShuf1<>+0x20(SB)/8, $0xffffffffff010101 +DATA expandAVX512_60_inShuf1<>+0x28(SB)/8, $0x0202020202010101 +DATA expandAVX512_60_inShuf1<>+0x30(SB)/8, $0xffffffffffff0201 +DATA expandAVX512_60_inShuf1<>+0x38(SB)/8, $0xff01010101010101 + +GLOBL expandAVX512_60_mat1<>(SB), RODATA, $0x40 +DATA expandAVX512_60_mat1<>+0x00(SB)/8, $0x2020202020202020 +DATA expandAVX512_60_mat1<>+0x08(SB)/8, $0x4040404040404040 +DATA expandAVX512_60_mat1<>+0x10(SB)/8, $0x4040404080808080 +DATA expandAVX512_60_mat1<>+0x18(SB)/8, $0x8080808080808080 +DATA expandAVX512_60_mat1<>+0x20(SB)/8, $0x0101010101010101 +DATA expandAVX512_60_mat1<>+0x28(SB)/8, $0x0101010101010101 +DATA expandAVX512_60_mat1<>+0x30(SB)/8, $0x0101010102020202 +DATA expandAVX512_60_mat1<>+0x38(SB)/8, $0x0202020202020202 + +GLOBL expandAVX512_60_inShuf2<>(SB), RODATA, $0x40 +DATA expandAVX512_60_inShuf2<>+0x00(SB)/8, $0xff01010101010101 +DATA expandAVX512_60_inShuf2<>+0x08(SB)/8, $0xffffffffffffff01 +DATA expandAVX512_60_inShuf2<>+0x10(SB)/8, $0xff01010101010101 +DATA expandAVX512_60_inShuf2<>+0x18(SB)/8, $0xff01010101010101 +DATA expandAVX512_60_inShuf2<>+0x20(SB)/8, $0xffffffffffffff01 +DATA expandAVX512_60_inShuf2<>+0x28(SB)/8, $0xff01010101010101 +DATA expandAVX512_60_inShuf2<>+0x30(SB)/8, $0xff01010101010101 +DATA expandAVX512_60_inShuf2<>+0x38(SB)/8, $0xffffffffffffff01 + +GLOBL expandAVX512_60_mat2<>(SB), RODATA, $0x40 +DATA expandAVX512_60_mat2<>+0x00(SB)/8, $0x0404040404040404 +DATA expandAVX512_60_mat2<>+0x08(SB)/8, $0x0404040408080808 +DATA expandAVX512_60_mat2<>+0x10(SB)/8, $0x0808080808080808 +DATA expandAVX512_60_mat2<>+0x18(SB)/8, $0x1010101010101010 +DATA expandAVX512_60_mat2<>+0x20(SB)/8, $0x1010101020202020 +DATA expandAVX512_60_mat2<>+0x28(SB)/8, $0x2020202020202020 +DATA expandAVX512_60_mat2<>+0x30(SB)/8, $0x4040404040404040 +DATA expandAVX512_60_mat2<>+0x38(SB)/8, $0x4040404080808080 + +GLOBL expandAVX512_60_inShuf3<>(SB), RODATA, $0x40 +DATA expandAVX512_60_inShuf3<>+0x00(SB)/8, $0xff01010101010101 +DATA expandAVX512_60_inShuf3<>+0x08(SB)/8, $0xffffffffffff0202 +DATA expandAVX512_60_inShuf3<>+0x10(SB)/8, $0xffffffffffffffff +DATA expandAVX512_60_inShuf3<>+0x18(SB)/8, $0xffffffffffffffff +DATA expandAVX512_60_inShuf3<>+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_60_inShuf3<>+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_60_inShuf3<>+0x30(SB)/8, $0xffffffffffffffff +DATA expandAVX512_60_inShuf3<>+0x38(SB)/8, $0xffffffffffffffff + +GLOBL expandAVX512_60_mat3<>(SB), RODATA, $0x40 +DATA expandAVX512_60_mat3<>+0x00(SB)/8, $0x8080808080808080 +DATA expandAVX512_60_mat3<>+0x08(SB)/8, $0x0101010101010101 +DATA expandAVX512_60_mat3<>+0x10(SB)/8, $0x0000000000000000 +DATA expandAVX512_60_mat3<>+0x18(SB)/8, $0x0000000000000000 +DATA expandAVX512_60_mat3<>+0x20(SB)/8, $0x0000000000000000 +DATA expandAVX512_60_mat3<>+0x28(SB)/8, $0x0000000000000000 +DATA expandAVX512_60_mat3<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_60_mat3<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_60_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_60_outShufLo+0x00(SB)/8, $0x0806050403020100 +DATA expandAVX512_60_outShufLo+0x08(SB)/8, $0x1816151413121110 +DATA expandAVX512_60_outShufLo+0x10(SB)/8, $0x28201e1d1c1b1a19 +DATA expandAVX512_60_outShufLo+0x18(SB)/8, $0x31302e2d2c2b2a29 +DATA expandAVX512_60_outShufLo+0x20(SB)/8, $0x4140383635343332 +DATA expandAVX512_60_outShufLo+0x28(SB)/8, $0x4a49484645444342 +DATA expandAVX512_60_outShufLo+0x30(SB)/8, $0x5a5958504e4d4c4b +DATA expandAVX512_60_outShufLo+0x38(SB)/8, $0x626160075e5d5c5b + +GLOBL expandAVX512_60_outShufHi0(SB), RODATA, $0x40 +DATA expandAVX512_60_outShufHi0+0x00(SB)/8, $0x3b3a3938302a2928 +DATA expandAVX512_60_outShufHi0+0x08(SB)/8, $0x44434241403e3d3c +DATA expandAVX512_60_outShufHi0+0x10(SB)/8, $0x5453525150484645 +DATA expandAVX512_60_outShufHi0+0x18(SB)/8, $0x5d5c5b5a59585655 +DATA expandAVX512_60_outShufHi0+0x20(SB)/8, $0x6d6c6b6a6968605e +DATA expandAVX512_60_outShufHi0+0x28(SB)/8, $0x767574737271706e +DATA expandAVX512_60_outShufHi0+0x30(SB)/8, $0xffffffffffffff78 +DATA expandAVX512_60_outShufHi0+0x38(SB)/8, $0x31ffff2f2e2d2c2b + +GLOBL expandAVX512_60_outShufHi1(SB), RODATA, $0x40 +DATA expandAVX512_60_outShufHi1+0x00(SB)/8, $0xffffffffffffffff +DATA expandAVX512_60_outShufHi1+0x08(SB)/8, $0xffffffffffffffff +DATA expandAVX512_60_outShufHi1+0x10(SB)/8, $0xffffffffffffffff +DATA expandAVX512_60_outShufHi1+0x18(SB)/8, $0xffffffffffffffff +DATA expandAVX512_60_outShufHi1+0x20(SB)/8, $0xffffffffffffffff +DATA expandAVX512_60_outShufHi1+0x28(SB)/8, $0xffffffffffffffff +DATA expandAVX512_60_outShufHi1+0x30(SB)/8, $0x06050403020100ff +DATA expandAVX512_60_outShufHi1+0x38(SB)/8, $0xff0908ffffffffff + +TEXT expandAVX512_60<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_60_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_60_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_60_inShuf2<>(SB), Z3 + VMOVDQU64 expandAVX512_60_inShuf3<>(SB), Z4 + VMOVDQU64 expandAVX512_60_outShufLo(SB), Z1 + VMOVDQU64 expandAVX512_60_outShufHi0(SB), Z5 + VMOVDQU64 expandAVX512_60_outShufHi1(SB), Z6 + VMOVDQU64 (AX), Z7 + VPERMB Z7, Z0, Z0 + VGF2P8AFFINEQB $0, expandAVX512_60_mat0<>(SB), Z0, Z0 + VPERMB Z7, Z2, Z2 + VGF2P8AFFINEQB $0, expandAVX512_60_mat1<>(SB), Z2, Z2 + VPERMB Z7, Z3, Z3 + VGF2P8AFFINEQB $0, expandAVX512_60_mat2<>(SB), Z3, Z3 + VPERMB Z7, Z4, Z4 + VGF2P8AFFINEQB $0, expandAVX512_60_mat3<>(SB), Z4, Z4 + VPERMI2B Z2, Z0, Z1 + MOVQ $0x9f01ffffffffffff, AX + KMOVQ AX, K1 + VPERMI2B.Z Z3, Z2, K1, Z5 + MOVQ $0x60fe000000000000, AX + KMOVQ AX, K1 + VPERMB.Z Z4, Z6, K1, Z0 + VPORQ Z0, Z5, Z2 + RET + +GLOBL expandAVX512_64_inShuf0<>(SB), RODATA, $0x40 +DATA expandAVX512_64_inShuf0<>+0x00(SB)/8, $0x0000000000000000 +DATA expandAVX512_64_inShuf0<>+0x08(SB)/8, $0x0000000000000000 +DATA expandAVX512_64_inShuf0<>+0x10(SB)/8, $0x0000000000000000 +DATA expandAVX512_64_inShuf0<>+0x18(SB)/8, $0x0000000000000000 +DATA expandAVX512_64_inShuf0<>+0x20(SB)/8, $0x0000000000000000 +DATA expandAVX512_64_inShuf0<>+0x28(SB)/8, $0x0000000000000000 +DATA expandAVX512_64_inShuf0<>+0x30(SB)/8, $0x0000000000000000 +DATA expandAVX512_64_inShuf0<>+0x38(SB)/8, $0x0000000000000000 + +GLOBL expandAVX512_64_mat0<>(SB), RODATA, $0x40 +DATA expandAVX512_64_mat0<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_64_mat0<>+0x08(SB)/8, $0x0202020202020202 +DATA expandAVX512_64_mat0<>+0x10(SB)/8, $0x0404040404040404 +DATA expandAVX512_64_mat0<>+0x18(SB)/8, $0x0808080808080808 +DATA expandAVX512_64_mat0<>+0x20(SB)/8, $0x1010101010101010 +DATA expandAVX512_64_mat0<>+0x28(SB)/8, $0x2020202020202020 +DATA expandAVX512_64_mat0<>+0x30(SB)/8, $0x4040404040404040 +DATA expandAVX512_64_mat0<>+0x38(SB)/8, $0x8080808080808080 + +GLOBL expandAVX512_64_inShuf1<>(SB), RODATA, $0x40 +DATA expandAVX512_64_inShuf1<>+0x00(SB)/8, $0x0101010101010101 +DATA expandAVX512_64_inShuf1<>+0x08(SB)/8, $0x0101010101010101 +DATA expandAVX512_64_inShuf1<>+0x10(SB)/8, $0x0101010101010101 +DATA expandAVX512_64_inShuf1<>+0x18(SB)/8, $0x0101010101010101 +DATA expandAVX512_64_inShuf1<>+0x20(SB)/8, $0x0101010101010101 +DATA expandAVX512_64_inShuf1<>+0x28(SB)/8, $0x0101010101010101 +DATA expandAVX512_64_inShuf1<>+0x30(SB)/8, $0x0101010101010101 +DATA expandAVX512_64_inShuf1<>+0x38(SB)/8, $0x0101010101010101 + +GLOBL expandAVX512_64_outShufLo(SB), RODATA, $0x40 +DATA expandAVX512_64_outShufLo+0x00(SB)/8, $0x0706050403020100 +DATA expandAVX512_64_outShufLo+0x08(SB)/8, $0x0f0e0d0c0b0a0908 +DATA expandAVX512_64_outShufLo+0x10(SB)/8, $0x1716151413121110 +DATA expandAVX512_64_outShufLo+0x18(SB)/8, $0x1f1e1d1c1b1a1918 +DATA expandAVX512_64_outShufLo+0x20(SB)/8, $0x2726252423222120 +DATA expandAVX512_64_outShufLo+0x28(SB)/8, $0x2f2e2d2c2b2a2928 +DATA expandAVX512_64_outShufLo+0x30(SB)/8, $0x3736353433323130 +DATA expandAVX512_64_outShufLo+0x38(SB)/8, $0x3f3e3d3c3b3a3938 + +TEXT expandAVX512_64<>(SB), NOSPLIT, $0-0 + VMOVDQU64 expandAVX512_64_inShuf0<>(SB), Z0 + VMOVDQU64 expandAVX512_64_mat0<>(SB), Z1 + VMOVDQU64 expandAVX512_64_inShuf1<>(SB), Z2 + VMOVDQU64 expandAVX512_64_outShufLo(SB), Z3 + VMOVDQU64 (AX), Z4 + VPERMB Z4, Z0, Z0 + VGF2P8AFFINEQB $0, Z1, Z0, Z0 + VPERMB Z4, Z2, Z2 + VGF2P8AFFINEQB $0, Z1, Z2, Z2 + VPERMB Z0, Z3, Z1 + VPERMB Z2, Z3, Z2 + RET + diff --git a/src/internal/runtime/gc/scan/expand_amd64_test.go b/src/internal/runtime/gc/scan/expand_amd64_test.go new file mode 100644 index 00000000000000..a8f5b88c5cb3a6 --- /dev/null +++ b/src/internal/runtime/gc/scan/expand_amd64_test.go @@ -0,0 +1,19 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build amd64 + +package scan_test + +import ( + "internal/runtime/gc/scan" + "testing" +) + +func TestExpandAVX512(t *testing.T) { + if !scan.CanAVX512() { + t.Skip("no AVX512") + } + testExpand(t, scan.ExpandAVX512) +} diff --git a/src/internal/runtime/gc/scan/expand_reference.go b/src/internal/runtime/gc/scan/expand_reference.go new file mode 100644 index 00000000000000..dd23f4fc103048 --- /dev/null +++ b/src/internal/runtime/gc/scan/expand_reference.go @@ -0,0 +1,39 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scan + +import ( + "internal/goarch" + "internal/runtime/gc" +) + +// ExpandReference is a reference implementation of an expander function +// that translates object mark bits into a bitmap of one bit per word of +// marked object, assuming the object is of the provided size class. +func ExpandReference(sizeClass int, packed *gc.ObjMask, unpacked *gc.PtrMask) { + // Look up the size and derive the number of objects in a span. + // We're only concerned with small objects in single-page spans, + // and gc.PtrMask enforces this by being statically sized to + // accommodate only such spans. + size := uintptr(gc.SizeClassToSize[sizeClass]) + nObj := uintptr(gc.SizeClassToNPages[sizeClass]) * gc.PageSize / size + + // f is the expansion factor. For example, if our objects are of size 48, + // then each mark bit will translate into 6 (48/8 = 6) set bits in the + // pointer bitmap. + f := size / goarch.PtrSize + for i := range nObj { + // Check if the object is marked. + if packed[i/goarch.PtrBits]&(uintptr(1)<<(i%goarch.PtrBits)) == 0 { + continue + } + // Propagate that mark into the destination into one bit per the + // expansion factor f, offset to the object's offset within the span. + for j := range f { + b := i*f + j // i*f is the start bit for the object, j indexes into each corresponding word after. + unpacked[b/goarch.PtrBits] |= uintptr(1) << (b % goarch.PtrBits) + } + } +} diff --git a/src/internal/runtime/gc/scan/expand_test.go b/src/internal/runtime/gc/scan/expand_test.go new file mode 100644 index 00000000000000..692817d8b2bd96 --- /dev/null +++ b/src/internal/runtime/gc/scan/expand_test.go @@ -0,0 +1,37 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scan_test + +import ( + "internal/goarch" + "internal/runtime/gc" + "internal/runtime/gc/scan" + "testing" +) + +type expandFunc func(sizeClass int, packed *gc.ObjMask, unpacked *gc.PtrMask) + +func testExpand(t *testing.T, expF expandFunc) { + expR := scan.ExpandReference + + testObjs(t, func(t *testing.T, sizeClass int, objs *gc.ObjMask) { + var want, got gc.PtrMask + expR(sizeClass, objs, &want) + expF(sizeClass, objs, &got) + + for i := range want { + if got[i] != want[i] { + t.Errorf("expansion differs from reference at bit %d", i*goarch.PtrSize) + if goarch.PtrSize == 4 { + t.Logf("got: %032b", got[i]) + t.Logf("want: %032b", want[i]) + } else { + t.Logf("got: %064b", got[i]) + t.Logf("want: %064b", want[i]) + } + } + } + }) +} diff --git a/src/internal/runtime/gc/scan/filter.go b/src/internal/runtime/gc/scan/filter.go new file mode 100644 index 00000000000000..63cee9abf066a9 --- /dev/null +++ b/src/internal/runtime/gc/scan/filter.go @@ -0,0 +1,35 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scan + +import "unsafe" + +// FilterNil packs non-nil (non-zero) values in bufp together +// at the beginning of bufp, returning the length of the +// packed buffer. It treats bufp as an array of size n. +// +// TODO(mknyszek): Add a faster SIMD-based implementation. +func FilterNil(bufp *uintptr, n int32) int32 { + buf := unsafe.Slice(bufp, int(n)) + lo := 0 + hi := len(buf) - 1 + for lo < hi { + for lo < hi && buf[hi] == 0 { + hi-- + } + for lo < hi && buf[lo] != 0 { + lo++ + } + if lo >= hi { + break + } + buf[lo] = buf[hi] + hi-- + } + if hi >= 0 && buf[hi] == 0 { + hi-- + } + return int32(hi) + 1 +} diff --git a/src/internal/runtime/gc/scan/filter_test.go b/src/internal/runtime/gc/scan/filter_test.go new file mode 100644 index 00000000000000..115fbfb8bcf348 --- /dev/null +++ b/src/internal/runtime/gc/scan/filter_test.go @@ -0,0 +1,94 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scan_test + +import ( + "internal/runtime/gc/scan" + "testing" +) + +func TestFilterNil(t *testing.T) { + t.Run("empty", func(t *testing.T) { + testFilterNil(t, []uintptr{}, []uintptr{}) + }) + t.Run("one", func(t *testing.T) { + testFilterNil(t, []uintptr{4}, []uintptr{4}) + }) + t.Run("elimOne", func(t *testing.T) { + testFilterNil(t, []uintptr{0}, []uintptr{}) + }) + t.Run("oneElimBegin", func(t *testing.T) { + testFilterNil(t, []uintptr{0, 4}, []uintptr{4}) + }) + t.Run("oneElimEnd", func(t *testing.T) { + testFilterNil(t, []uintptr{4, 0}, []uintptr{4}) + }) + t.Run("oneElimMultiBegin", func(t *testing.T) { + testFilterNil(t, []uintptr{0, 0, 0, 4}, []uintptr{4}) + }) + t.Run("oneElimMultiEnd", func(t *testing.T) { + testFilterNil(t, []uintptr{4, 0, 0, 0}, []uintptr{4}) + }) + t.Run("oneElimMulti", func(t *testing.T) { + testFilterNil(t, []uintptr{0, 0, 0, 4, 0}, []uintptr{4}) + }) + t.Run("two", func(t *testing.T) { + testFilterNil(t, []uintptr{5, 12}, []uintptr{5, 12}) + }) + t.Run("twoElimBegin", func(t *testing.T) { + testFilterNil(t, []uintptr{0, 5, 12}, []uintptr{5, 12}) + }) + t.Run("twoElimMid", func(t *testing.T) { + testFilterNil(t, []uintptr{5, 0, 12}, []uintptr{5, 12}) + }) + t.Run("twoElimEnd", func(t *testing.T) { + testFilterNil(t, []uintptr{5, 12, 0}, []uintptr{5, 12}) + }) + t.Run("twoElimMulti", func(t *testing.T) { + testFilterNil(t, []uintptr{0, 5, 0, 12, 0}, []uintptr{5, 12}) + }) + t.Run("Multi", func(t *testing.T) { + testFilterNil(t, []uintptr{1, 5, 5, 0, 0, 0, 12, 0, 121, 5, 0}, []uintptr{1, 5, 5, 12, 121, 5}) + }) +} + +func testFilterNil(t *testing.T, buf, want []uintptr) { + var bufp *uintptr + if len(buf) != 0 { + bufp = &buf[0] + } + n := scan.FilterNil(bufp, int32(len(buf))) + if n > int32(len(buf)) { + t.Errorf("bogus new length returned: %d > %d", n, len(buf)) + return + } + buf = buf[:n] + if len(buf) != len(want) { + t.Errorf("lengths differ: got %d, want %d", len(buf), len(want)) + } + + wantMap := make(map[uintptr]int) + gotMap := make(map[uintptr]int) + for _, p := range want { + wantMap[p]++ + } + for _, p := range buf { + gotMap[p]++ + } + for p, nWant := range wantMap { + if nGot, ok := gotMap[p]; !ok { + t.Errorf("want %d, but missing from output", p) + } else if nGot != nWant { + t.Errorf("want %d copies of %d, but got %d", nWant, p, nGot) + } + } + for p := range gotMap { + if _, ok := wantMap[p]; !ok { + t.Errorf("got %d, but didn't want it", p) + } + } + t.Logf("got: %v", buf) + t.Logf("want: %v", want) +} diff --git a/src/internal/runtime/gc/scan/mem_nounix_test.go b/src/internal/runtime/gc/scan/mem_nounix_test.go new file mode 100644 index 00000000000000..f4d21d8a8504ea --- /dev/null +++ b/src/internal/runtime/gc/scan/mem_nounix_test.go @@ -0,0 +1,16 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !unix + +package scan_test + +import ( + "testing" +) + +func makeMem(t testing.TB, nPages int) ([]uintptr, func()) { + t.Skip("mmap unsupported") + return nil, nil +} diff --git a/src/internal/runtime/gc/scan/mem_unix_test.go b/src/internal/runtime/gc/scan/mem_unix_test.go new file mode 100644 index 00000000000000..03f0bd5dd08090 --- /dev/null +++ b/src/internal/runtime/gc/scan/mem_unix_test.go @@ -0,0 +1,25 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix + +package scan_test + +import ( + "internal/runtime/gc" + "syscall" + "testing" + "unsafe" +) + +func makeMem(t testing.TB, nPages int) ([]uintptr, func()) { + mem, err := syscall.Mmap(-1, 0, int(gc.PageSize*nPages), syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_PRIVATE|syscall.MAP_ANON) + if err != nil { + t.Fatalf("mmap failed: %s", err) + } + free := func() { + syscall.Munmap(mem) + } + return unsafe.Slice((*uintptr)(unsafe.Pointer(unsafe.SliceData(mem))), len(mem)/8), free +} diff --git a/src/internal/runtime/gc/scan/mkasm.go b/src/internal/runtime/gc/scan/mkasm.go new file mode 100644 index 00000000000000..e36defb2e18056 --- /dev/null +++ b/src/internal/runtime/gc/scan/mkasm.go @@ -0,0 +1,412 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ignore + +package main + +import ( + "bytes" + "fmt" + "io" + "log" + "os" + "slices" + "strconv" + + "internal/runtime/gc" + "internal/runtime/gc/internal/gen" +) + +const header = "// Code generated by mkasm.go. DO NOT EDIT.\n\n" + +func main() { + generate("expand_amd64.s", genExpanders) +} + +func generate(fileName string, genFunc func(*gen.File)) { + var buf bytes.Buffer + tee := io.MultiWriter(&buf, os.Stdout) + + file := gen.NewFile(tee) + + genFunc(file) + + fmt.Fprintf(tee, header) + file.Compile() + + f, err := os.Create(fileName) + if err != nil { + log.Fatal(err) + } + defer f.Close() + _, err = f.Write(buf.Bytes()) + if err != nil { + log.Fatal(err) + } +} + +func genExpanders(file *gen.File) { + gcExpandersAVX512 := make([]*gen.Func, len(gc.SizeClassToSize)) + for sc, ob := range gc.SizeClassToSize { + if gc.SizeClassToNPages[sc] != 1 { + // These functions all produce a bitmap that covers exactly one + // page. + continue + } + if ob > gc.MinSizeForMallocHeader { + // This size class is too big to have a packed pointer/scalar bitmap. + break + } + + xf := int(ob) / 8 + log.Printf("size class %d bytes, expansion %dx", ob, xf) + + fn := gen.NewFunc(fmt.Sprintf("expandAVX512_%d<>", xf)) + ptrObjBits := gen.Arg[gen.Ptr[gen.Uint8x64]](fn) + + if xf == 1 { + expandIdentity(ptrObjBits) + } else { + ok := gfExpander(xf, ptrObjBits) + if !ok { + log.Printf("failed to generate expander for size class %d", sc) + } + } + file.AddFunc(fn) + gcExpandersAVX512[sc] = fn + } + + // Generate table mapping size class to expander PC + file.AddConst("·gcExpandersAVX512", gcExpandersAVX512) +} + +// mat8x8 is an 8x8 bit matrix. +type mat8x8 struct { + mat [8]uint8 +} + +func matGroupToVec(mats *[8]mat8x8) [8]uint64 { + var out [8]uint64 + for i, mat := range mats { + for j, row := range mat.mat { + // For some reason, Intel flips the rows. + out[i] |= uint64(row) << ((7 - j) * 8) + } + } + return out +} + +// expandIdentity implements 1x expansion (that is, no expansion). +func expandIdentity(ptrObjBits gen.Ptr[gen.Uint8x64]) { + objBitsLo := gen.Deref(ptrObjBits) + objBitsHi := gen.Deref(ptrObjBits.AddConst(64)) + gen.Return(objBitsLo, objBitsHi) +} + +// gfExpander produces a function that expands each bit in an input bitmap into +// f consecutive bits in an output bitmap. +// +// The input is +// +// AX *[8]uint64 = A pointer to floor(1024/f) bits (f >= 2, so at most 512 bits) +// +// The output is +// +// Z1 [64]uint8 = The bottom 512 bits of the expanded bitmap +// Z2 [64]uint8 = The top 512 bits of the expanded bitmap +// +// TODO(austin): This should Z0/Z1. +func gfExpander(f int, ptrObjBits gen.Ptr[gen.Uint8x64]) bool { + // TODO(austin): For powers of 2 >= 8, we can use mask expansion ops to make this much simpler. + + // TODO(austin): For f >= 8, I suspect there are better ways to do this. + // + // For example, we could use a mask expansion to get a full byte for each + // input bit, and separately create the bytes that blend adjacent bits, then + // shuffle those bytes together. Certainly for f >= 16 this makes sense + // because each of those bytes will be used, possibly more than once. + + objBits := gen.Deref(ptrObjBits) + + type term struct { + iByte, oByte int + mat mat8x8 + } + var terms []term + + // Iterate over all output bytes and construct the 8x8 GF2 matrix to compute + // the output byte from the appropriate input byte. Gather all of these into + // "terms". + for oByte := 0; oByte < 1024/8; oByte++ { + var byteMat mat8x8 + iByte := -1 + for oBit := oByte * 8; oBit < oByte*8+8; oBit++ { + iBit := oBit / f + if iByte == -1 { + iByte = iBit / 8 + } else if iByte != iBit/8 { + log.Printf("output byte %d straddles input bytes %d and %d", oByte, iByte, iBit/8) + return false + } + // One way to view this is that the i'th row of the matrix will be + // ANDed with the input byte, and the parity of the result will set + // the i'th bit in the output. We use a simple 1 bit mask, so the + // parity is irrelevant beyond selecting out that one bit. + byteMat.mat[oBit%8] = 1 << (iBit % 8) + } + terms = append(terms, term{iByte, oByte, byteMat}) + } + + if false { + // Print input byte -> output byte as a matrix + maxIByte, maxOByte := 0, 0 + for _, term := range terms { + maxIByte = max(maxIByte, term.iByte) + maxOByte = max(maxOByte, term.oByte) + } + iToO := make([][]rune, maxIByte+1) + for i := range iToO { + iToO[i] = make([]rune, maxOByte+1) + } + matMap := make(map[mat8x8]int) + for _, term := range terms { + i, ok := matMap[term.mat] + if !ok { + i = len(matMap) + matMap[term.mat] = i + } + iToO[term.iByte][term.oByte] = 'A' + rune(i) + } + for o := range maxOByte + 1 { + fmt.Printf("%d", o) + for i := range maxIByte + 1 { + fmt.Printf(",") + if mat := iToO[i][o]; mat != 0 { + fmt.Printf("%c", mat) + } + } + fmt.Println() + } + } + + // In hardware, each (8 byte) matrix applies to 8 bytes of data in parallel, + // and we get to operate on up to 8 matrixes in parallel (or 64 values). That is: + // + // abcdefgh ijklmnop qrstuvwx yzABCDEF GHIJKLMN OPQRSTUV WXYZ0123 456789_+ + // mat0 mat1 mat2 mat3 mat4 mat5 mat6 mat7 + + // Group the terms by matrix, but limit each group to 8 terms. + const termsPerGroup = 8 // Number of terms we can multiply by the same matrix. + const groupsPerSuperGroup = 8 // Number of matrixes we can fit in a vector. + + matMap := make(map[mat8x8]int) + allMats := make(map[mat8x8]bool) + var termGroups [][]term + for _, term := range terms { + allMats[term.mat] = true + + i, ok := matMap[term.mat] + if ok && f > groupsPerSuperGroup { + // The output is ultimately produced in two [64]uint8 registers. + // Getting every byte in the right place of each of these requires a + // final permutation that often requires more than one source. + // + // Up to 8x expansion, we can get a really nice grouping so we can use + // the same 8 matrix vector several times, without producing + // permutations that require more than two sources. + // + // Above 8x, however, we can't get nice matrixes anyway, so we + // instead prefer reducing the complexity of the permutations we + // need to produce the final outputs. To do this, avoid grouping + // together terms that are split across the two registers. + outRegister := termGroups[i][0].oByte / 64 + if term.oByte/64 != outRegister { + ok = false + } + } + if !ok { + // Start a new term group. + i = len(termGroups) + matMap[term.mat] = i + termGroups = append(termGroups, nil) + } + + termGroups[i] = append(termGroups[i], term) + + if len(termGroups[i]) == termsPerGroup { + // This term group is full. + delete(matMap, term.mat) + } + } + + for i, termGroup := range termGroups { + log.Printf("term group %d:", i) + for _, term := range termGroup { + log.Printf(" %+v", term) + } + } + + // We can do 8 matrix multiplies in parallel, which is 8 term groups. Pack + // as many term groups as we can into each super-group to minimize the + // number of matrix multiplies. + // + // Ideally, we use the same matrix in each super-group, which might mean + // doing fewer than 8 multiplies at a time. That's fine because it never + // increases the total number of matrix multiplies. + // + // TODO: Packing the matrixes less densely may let us use more broadcast + // loads instead of general permutations, though. That replaces a load of + // the permutation with a load of the matrix, but is probably still slightly + // better. + var sgSize, nSuperGroups int + oneMatVec := f <= groupsPerSuperGroup + if oneMatVec { + // We can use the same matrix in each multiply by doing sgSize + // multiplies at a time. + sgSize = groupsPerSuperGroup / len(allMats) * len(allMats) + nSuperGroups = (len(termGroups) + sgSize - 1) / sgSize + } else { + // We can't use the same matrix for each multiply. Just do as many at a + // time as we can. + // + // TODO: This is going to produce several distinct matrixes, when we + // probably only need two. Be smarter about how we create super-groups + // in this case. Maybe we build up an array of super-groups and then the + // loop below just turns them into ops? + sgSize = 8 + nSuperGroups = (len(termGroups) + groupsPerSuperGroup - 1) / groupsPerSuperGroup + } + + // Construct each super-group. + var matGroup [8]mat8x8 + var matMuls []gen.Uint8x64 + var perm [128]int + for sgi := range nSuperGroups { + var iperm [64]uint8 + for i := range iperm { + iperm[i] = 0xff // "Don't care" + } + // Pick off sgSize term groups. + superGroup := termGroups[:min(len(termGroups), sgSize)] + termGroups = termGroups[len(superGroup):] + // Build the matrix and permutations for this super-group. + var thisMatGroup [8]mat8x8 + for i, termGroup := range superGroup { + // All terms in this group have the same matrix. Pick one. + thisMatGroup[i] = termGroup[0].mat + for j, term := range termGroup { + // Build the input permutation. + iperm[i*termsPerGroup+j] = uint8(term.iByte) + // Build the output permutation. + perm[term.oByte] = sgi*groupsPerSuperGroup*termsPerGroup + i*termsPerGroup + j + } + } + log.Printf("input permutation %d: %v", sgi, iperm) + + // Check that we're not making more distinct matrixes than expected. + if oneMatVec { + if sgi == 0 { + matGroup = thisMatGroup + } else if matGroup != thisMatGroup { + log.Printf("super-groups have different matrixes:\n%+v\n%+v", matGroup, thisMatGroup) + return false + } + } + + // Emit matrix op. + matConst := gen.ConstUint64x8(matGroupToVec(&thisMatGroup), fmt.Sprintf("*_mat%d<>", sgi)) + inOp := objBits.Shuffle(gen.ConstUint8x64(iperm, fmt.Sprintf("*_inShuf%d<>", sgi))) + matMul := matConst.GF2P8Affine(inOp) + matMuls = append(matMuls, matMul) + } + + log.Printf("output permutation: %v", perm) + + outLo, ok := genShuffle("*_outShufLo", (*[64]int)(perm[:64]), matMuls...) + if !ok { + log.Printf("bad number of inputs to final shuffle: %d != 1, 2, or 4", len(matMuls)) + return false + } + outHi, ok := genShuffle("*_outShufHi", (*[64]int)(perm[64:]), matMuls...) + if !ok { + log.Printf("bad number of inputs to final shuffle: %d != 1, 2, or 4", len(matMuls)) + return false + } + gen.Return(outLo, outHi) + + return true +} + +func genShuffle(name string, perm *[64]int, args ...gen.Uint8x64) (gen.Uint8x64, bool) { + // Construct flattened permutation. + var vperm [64]byte + + // Get the inputs used by this permutation. + var inputs []int + for i, src := range perm { + inputIdx := slices.Index(inputs, src/64) + if inputIdx == -1 { + inputIdx = len(inputs) + inputs = append(inputs, src/64) + } + vperm[i] = byte(src%64 | (inputIdx << 6)) + } + + // Emit instructions for easy cases. + switch len(inputs) { + case 1: + constOp := gen.ConstUint8x64(vperm, name) + return args[inputs[0]].Shuffle(constOp), true + case 2: + constOp := gen.ConstUint8x64(vperm, name) + return args[inputs[0]].Shuffle2(args[inputs[1]], constOp), true + } + + // Harder case, we need to shuffle in from up to 2 more tables. + // + // Perform two shuffles. One shuffle will get its data from the first + // two inputs, the other shuffle will get its data from the other one + // or two inputs. All values they don't care each don't care about will + // be zeroed. + var vperms [2][64]byte + var masks [2]uint64 + for j, idx := range vperm { + for i := range vperms { + vperms[i][j] = 0xff // "Don't care" + } + if idx == 0xff { + continue + } + vperms[idx/128][j] = idx % 128 + masks[idx/128] |= uint64(1) << j + } + + // Validate that the masks are fully disjoint. + if masks[0]^masks[1] != ^uint64(0) { + panic("bad shuffle!") + } + + // Generate constants. + constOps := make([]gen.Uint8x64, len(vperms)) + for i, v := range vperms { + constOps[i] = gen.ConstUint8x64(v, name+strconv.Itoa(i)) + } + + // Generate shuffles. + switch len(inputs) { + case 3: + r0 := args[inputs[0]].Shuffle2Zeroed(args[inputs[1]], constOps[0], gen.ConstMask64(masks[0])) + r1 := args[inputs[2]].ShuffleZeroed(constOps[1], gen.ConstMask64(masks[1])) + return r0.ToUint64x8().Or(r1.ToUint64x8()).ToUint8x64(), true + case 4: + r0 := args[inputs[0]].Shuffle2Zeroed(args[inputs[1]], constOps[0], gen.ConstMask64(masks[0])) + r1 := args[inputs[2]].Shuffle2Zeroed(args[inputs[3]], constOps[1], gen.ConstMask64(masks[1])) + return r0.ToUint64x8().Or(r1.ToUint64x8()).ToUint8x64(), true + } + + // Too many inputs. To support more, we'd need to separate tables much earlier. + // Right now all the indices fit in a byte, but with >4 inputs they might not (>256 bytes). + return args[0], false +} diff --git a/src/internal/runtime/gc/scan/scan_amd64.go b/src/internal/runtime/gc/scan/scan_amd64.go new file mode 100644 index 00000000000000..2ac181f97e5b66 --- /dev/null +++ b/src/internal/runtime/gc/scan/scan_amd64.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scan + +import ( + "internal/cpu" + "internal/runtime/gc" + "unsafe" +) + +func ScanSpanPacked(mem unsafe.Pointer, bufp *uintptr, objMarks *gc.ObjMask, sizeClass uintptr, ptrMask *gc.PtrMask) (count int32) { + if CanAVX512() { + return ScanSpanPackedAVX512(mem, bufp, objMarks, sizeClass, ptrMask) + } + panic("not implemented") +} + +func HasFastScanSpanPacked() bool { + return avx512ScanPackedReqsMet +} + +// -- AVX512 -- + +func CanAVX512() bool { + return avx512ScanPackedReqsMet +} + +func ScanSpanPackedAVX512(mem unsafe.Pointer, bufp *uintptr, objMarks *gc.ObjMask, sizeClass uintptr, ptrMask *gc.PtrMask) (count int32) { + return FilterNil(bufp, scanSpanPackedAVX512(mem, bufp, objMarks, sizeClass, ptrMask)) +} + +//go:noescape +func scanSpanPackedAVX512(mem unsafe.Pointer, bufp *uintptr, objMarks *gc.ObjMask, sizeClass uintptr, ptrMask *gc.PtrMask) (count int32) + +var avx512ScanPackedReqsMet = cpu.X86.HasAVX512VL && + cpu.X86.HasAVX512BW && + cpu.X86.HasGFNI && + cpu.X86.HasAVX512BITALG && + cpu.X86.HasAVX512VBMI diff --git a/src/internal/runtime/gc/scan/scan_amd64.s b/src/internal/runtime/gc/scan/scan_amd64.s new file mode 100644 index 00000000000000..055995fa38c987 --- /dev/null +++ b/src/internal/runtime/gc/scan/scan_amd64.s @@ -0,0 +1,103 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "textflag.h" + +// Test-only. +TEXT ·ExpandAVX512(SB), NOSPLIT, $0-24 + MOVQ sizeClass+0(FP), CX + MOVQ packed+8(FP), AX + + // Call the expander for this size class + LEAQ ·gcExpandersAVX512(SB), BX + CALL (BX)(CX*8) + + MOVQ unpacked+16(FP), DI // Expanded output bitmap pointer + VMOVDQU64 Z1, 0(DI) + VMOVDQU64 Z2, 64(DI) + VZEROUPPER + RET + +TEXT ·scanSpanPackedAVX512(SB), NOSPLIT, $256-44 + // Z1+Z2 = Expand the grey object mask into a grey word mask + MOVQ objMarks+16(FP), AX + MOVQ sizeClass+24(FP), CX + LEAQ ·gcExpandersAVX512(SB), BX + CALL (BX)(CX*8) + + // Z3+Z4 = Load the pointer mask + MOVQ ptrMask+32(FP), AX + VMOVDQU64 0(AX), Z3 + VMOVDQU64 64(AX), Z4 + + // Z1+Z2 = Combine the grey word mask with the pointer mask to get the scan mask + VPANDQ Z1, Z3, Z1 + VPANDQ Z2, Z4, Z2 + + // Now each bit of Z1+Z2 represents one word of the span. + // Thus, each byte covers 64 bytes of memory, which is also how + // much we can fix in a Z register. + // + // We do a load/compress for each 64 byte frame. + // + // Z3+Z4 [128]uint8 = Number of memory words to scan in each 64 byte frame + VPOPCNTB Z1, Z3 // Requires BITALG + VPOPCNTB Z2, Z4 + + // Store the scan mask and word counts at 0(SP) and 128(SP). + // + // TODO: Is it better to read directly from the registers? + VMOVDQU64 Z1, 0(SP) + VMOVDQU64 Z2, 64(SP) + VMOVDQU64 Z3, 128(SP) + VMOVDQU64 Z4, 192(SP) + + // SI = Current address in span + MOVQ mem+0(FP), SI + // DI = Scan buffer base + MOVQ bufp+8(FP), DI + // DX = Index in scan buffer, (DI)(DX*8) = Current position in scan buffer + MOVQ $0, DX + + // AX = address in scan mask, 128(AX) = address in popcount + LEAQ 0(SP), AX + + // Loop over the 64 byte frames in this span. + // BX = 1 past the end of the scan mask + LEAQ 128(SP), BX + + // Align loop to a cache line so that performance is less sensitive + // to how this function ends up laid out in memory. This is a hot + // function in the GC, and this is a tight loop. We don't want + // performance to waver wildly due to unrelated changes. + PCALIGN $64 +loop: + // CX = Fetch the mask of words to load from this frame. + MOVBQZX 0(AX), CX + // Skip empty frames. + TESTQ CX, CX + JZ skip + + // Load the 64 byte frame. + KMOVB CX, K1 + VMOVDQA64 0(SI), Z1 + + // Collect just the pointers from the greyed objects into the scan buffer, + // i.e., copy the word indices in the mask from Z1 into contiguous memory. + VPCOMPRESSQ Z1, K1, (DI)(DX*8) + // Advance the scan buffer position by the number of pointers. + MOVBQZX 128(AX), CX + ADDQ CX, DX + +skip: + ADDQ $64, SI + ADDQ $1, AX + CMPQ AX, BX + JB loop + +end: + MOVL DX, count+40(FP) + VZEROUPPER + RET diff --git a/src/internal/runtime/gc/scan/scan_amd64_test.go b/src/internal/runtime/gc/scan/scan_amd64_test.go new file mode 100644 index 00000000000000..a914b4f4d7a64f --- /dev/null +++ b/src/internal/runtime/gc/scan/scan_amd64_test.go @@ -0,0 +1,19 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build amd64 + +package scan_test + +import ( + "internal/runtime/gc/scan" + "testing" +) + +func TestScanSpanPackedAVX512(t *testing.T) { + if !scan.CanAVX512() { + t.Skip("no AVX512") + } + testScanSpanPacked(t, scan.ScanSpanPackedAVX512) +} diff --git a/src/internal/runtime/gc/scan/scan_generic.go b/src/internal/runtime/gc/scan/scan_generic.go new file mode 100644 index 00000000000000..a4d51827cc6a20 --- /dev/null +++ b/src/internal/runtime/gc/scan/scan_generic.go @@ -0,0 +1,23 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !amd64 + +package scan + +import ( + "internal/runtime/gc" + "unsafe" +) + +func HasFastScanSpanPacked() bool { + // N.B. ScanSpanPackedGeneric isn't actually fast enough to serve as a general-purpose implementation. + // The runtime's alternative of jumping between each object is still substantially better, even at + // relatively high object densities. + return false +} + +func ScanSpanPacked(mem unsafe.Pointer, bufp *uintptr, objMarks *gc.ObjMask, sizeClass uintptr, ptrMask *gc.PtrMask) (count int32) { + return ScanSpanPackedGo(mem, bufp, objMarks, sizeClass, ptrMask) +} diff --git a/src/internal/runtime/gc/scan/scan_generic_test.go b/src/internal/runtime/gc/scan/scan_generic_test.go new file mode 100644 index 00000000000000..250135eca482dc --- /dev/null +++ b/src/internal/runtime/gc/scan/scan_generic_test.go @@ -0,0 +1,14 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scan_test + +import ( + "internal/runtime/gc/scan" + "testing" +) + +func TestScanSpanPackedGo(t *testing.T) { + testScanSpanPacked(t, scan.ScanSpanPackedGo) +} diff --git a/src/internal/runtime/gc/scan/scan_go.go b/src/internal/runtime/gc/scan/scan_go.go new file mode 100644 index 00000000000000..9a2985a3ccd220 --- /dev/null +++ b/src/internal/runtime/gc/scan/scan_go.go @@ -0,0 +1,104 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scan + +import ( + "internal/goarch" + "internal/runtime/gc" + "internal/runtime/sys" + "unsafe" +) + +// ScanSpanPackedGo is an optimized pure Go implementation of ScanSpanPacked. +func ScanSpanPackedGo(mem unsafe.Pointer, bufp *uintptr, objMarks *gc.ObjMask, sizeClass uintptr, ptrMask *gc.PtrMask) (count int32) { + buf := newUnsafeBuf(bufp) + objBytes := uintptr(gc.SizeClassToSize[sizeClass]) + // TODO(austin): Trim objMarks to the number of objects in this size class? + for markI, markWord := range objMarks { + for range sys.OnesCount64(uint64(markWord)) { + bitI := sys.TrailingZeros64(uint64(markWord)) + markWord &^= 1 << bitI + + objIndex := markI*goarch.PtrBits + bitI + + // objStartInSpan is the index of the word from mem where the + // object stats. objEndInSpan points to the next object, i.e. + // it's an exclusive upper bound. + objStartInSpan := objBytes * uintptr(objIndex) / goarch.PtrSize + objEndInSpan := objStartInSpan + objBytes/goarch.PtrSize + + // TODO: Another way to do this would be to extract the pointer mask + // for this object (it's at most 64 bits) and do a bit iteration + // over that. + + for wordI := objStartInSpan; wordI < objEndInSpan; wordI++ { + val := *(*uintptr)(unsafe.Add(mem, wordI*goarch.PtrSize)) + // Check if we should enqueue this word. + // + // We load the word before the check because, even though this + // can lead to loading much more than necessary, it's faster. + // Most likely this is because it warms up the hardware + // prefetcher much better, and gives us more time before we need + // the value. + // + // We discard values that can't possibly be useful pointers + // here, too, because this filters out a lot of words and does + // so with as little processing as possible. + // + // TODO: This is close to, but not entirely branchless. + isPtr := bool2int(ptrMask[wordI/goarch.PtrBits]&(1<<(wordI%goarch.PtrBits)) != 0) + isNonNil := bool2int(val >= 4096) + pred := isPtr&isNonNil != 0 + buf.addIf(val, pred) + } + } + } + // We don't know the true size of bufp, but we can at least catch obvious errors + // in this function by making sure we didn't write more than gc.PageWords pointers + // into the buffer. + buf.check(gc.PageWords) + return int32(buf.n) +} + +// unsafeBuf allows for appending to a buffer without bounds-checks or branches. +type unsafeBuf[T any] struct { + base *T + n int +} + +func newUnsafeBuf[T any](base *T) unsafeBuf[T] { + return unsafeBuf[T]{base, 0} +} + +// addIf appends a value to the buffer if the predicate is true. +// +// addIf speculatively writes to the next index of the buffer, so the caller +// must be certain that such a write will still be in-bounds with respect +// to the buffer's true capacity. +func (b *unsafeBuf[T]) addIf(val T, pred bool) { + *(*T)(unsafe.Add(unsafe.Pointer(b.base), b.n*int(unsafe.Sizeof(val)))) = val + b.n += bool2int(pred) +} + +// check performs a bounds check on speculative writes into the buffer. +// Calling this shortly after a series of addIf calls is important to +// catch any misuse as fast as possible. Separating the bounds check from +// the append is more efficient, but one check to cover several appends is +// still efficient and much more memory safe. +func (b unsafeBuf[T]) check(cap int) { + // We fail even if b.n == cap because addIf speculatively writes one past b.n. + if b.n >= cap { + panic("unsafeBuf overflow") + } +} + +func bool2int(x bool) int { + // This particular pattern gets optimized by the compiler. + var b int + if x { + b = 1 + } + return b +} diff --git a/src/internal/runtime/gc/scan/scan_reference.go b/src/internal/runtime/gc/scan/scan_reference.go new file mode 100644 index 00000000000000..05eca98df7abaf --- /dev/null +++ b/src/internal/runtime/gc/scan/scan_reference.go @@ -0,0 +1,40 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scan + +import ( + "internal/goarch" + "internal/runtime/gc" + "unsafe" +) + +// ScanSpanPackedReference is the reference implementation of ScanScanPacked. It prioritizes clarity over performance. +// +// Concretely, ScanScanPacked functions read pointers from mem, assumed to be gc.PageSize-aligned and gc.PageSize in size, +// and writes them to bufp, which is large enough to guarantee that even if pointer-word of mem is a pointer, it will fit. +// Therefore bufp, is always at least gc.PageSize in size. +// +// ScanSpanPacked is supposed to identify pointers by first filtering words by objMarks, where each bit of the mask +// represents gc.SizeClassToSize[sizeClass] bytes of memory, and then filtering again by the bits in ptrMask. +func ScanSpanPackedReference(mem unsafe.Pointer, bufp *uintptr, objMarks *gc.ObjMask, sizeClass uintptr, ptrMask *gc.PtrMask) (count int32) { + buf := unsafe.Slice(bufp, gc.PageWords) + expandBy := uintptr(gc.SizeClassToSize[sizeClass]) / goarch.PtrSize + for word := range gc.PageWords { + objI := uintptr(word) / expandBy + if objMarks[objI/goarch.PtrBits]&(1<<(objI%goarch.PtrBits)) == 0 { + continue + } + if ptrMask[word/goarch.PtrBits]&(1<<(word%goarch.PtrBits)) == 0 { + continue + } + ptr := *(*uintptr)(unsafe.Add(mem, word*goarch.PtrSize)) + if ptr == 0 { + continue + } + buf[count] = ptr + count++ + } + return count +} diff --git a/src/internal/runtime/gc/scan/scan_test.go b/src/internal/runtime/gc/scan/scan_test.go new file mode 100644 index 00000000000000..14a0f6f7f48a47 --- /dev/null +++ b/src/internal/runtime/gc/scan/scan_test.go @@ -0,0 +1,257 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scan_test + +import ( + "fmt" + "internal/cpu" + "internal/goarch" + "internal/runtime/gc" + "internal/runtime/gc/scan" + "math/bits" + "math/rand/v2" + "slices" + "sync" + "testing" + "unsafe" +) + +type scanFunc func(mem unsafe.Pointer, bufp *uintptr, objMarks *gc.ObjMask, sizeClass uintptr, ptrMask *gc.PtrMask) (count int32) + +func testScanSpanPacked(t *testing.T, scanF scanFunc) { + scanR := scan.ScanSpanPackedReference + + // Construct a fake memory + mem, free := makeMem(t, 1) + defer free() + for i := range mem { + // Use values > heap.PageSize because a scan function can discard + // pointers smaller than this. + mem[i] = uintptr(int(gc.PageSize) + i + 1) + } + + // Construct a random pointer mask + rnd := rand.New(rand.NewPCG(42, 42)) + var ptrs gc.PtrMask + for i := range ptrs { + ptrs[i] = uintptr(rnd.Uint64()) + } + + bufF := make([]uintptr, gc.PageWords) + bufR := make([]uintptr, gc.PageWords) + testObjs(t, func(t *testing.T, sizeClass int, objs *gc.ObjMask) { + nF := scanF(unsafe.Pointer(&mem[0]), &bufF[0], objs, uintptr(sizeClass), &ptrs) + nR := scanR(unsafe.Pointer(&mem[0]), &bufR[0], objs, uintptr(sizeClass), &ptrs) + + if nR != nF { + t.Errorf("want %d count, got %d", nR, nF) + } else if !slices.Equal(bufF[:nF], bufR[:nR]) { + t.Errorf("want scanned pointers %d, got %d", bufR[:nR], bufF[:nF]) + } + }) +} + +func testObjs(t *testing.T, f func(t *testing.T, sizeClass int, objMask *gc.ObjMask)) { + for sizeClass := range gc.NumSizeClasses { + if sizeClass == 0 { + continue + } + size := uintptr(gc.SizeClassToSize[sizeClass]) + if size > gc.MinSizeForMallocHeader { + break // Pointer/scalar metadata is not packed for larger sizes. + } + t.Run(fmt.Sprintf("size=%d", size), func(t *testing.T) { + // Scan a few objects near i to test boundary conditions. + const objMask = 0x101 + nObj := uintptr(gc.SizeClassToNPages[sizeClass]) * gc.PageSize / size + for i := range nObj - uintptr(bits.Len(objMask)-1) { + t.Run(fmt.Sprintf("objs=0x%x<<%d", objMask, i), func(t *testing.T) { + var objs gc.ObjMask + objs[i/goarch.PtrBits] = objMask << (i % goarch.PtrBits) + f(t, sizeClass, &objs) + }) + } + }) + } +} + +var dataCacheSizes = sync.OnceValue(func() []uintptr { + cs := cpu.DataCacheSizes() + for i, c := range cs { + fmt.Printf("# L%d cache: %d (%d Go pages)\n", i+1, c, c/gc.PageSize) + } + return cs +}) + +func BenchmarkScanSpanPacked(b *testing.B) { + benchmarkCacheSizes(b, benchmarkScanSpanPackedAllSizeClasses) +} + +func benchmarkCacheSizes(b *testing.B, fn func(b *testing.B, heapPages int)) { + cacheSizes := dataCacheSizes() + b.Run("cache=tiny/pages=1", func(b *testing.B) { + fn(b, 1) + }) + for i, cacheBytes := range cacheSizes { + pages := int(cacheBytes*3/4) / gc.PageSize + b.Run(fmt.Sprintf("cache=L%d/pages=%d", i+1, pages), func(b *testing.B) { + fn(b, pages) + }) + } + if len(cacheSizes) == 0 { + return + } + ramPages := int(cacheSizes[len(cacheSizes)-1]*3/2) / gc.PageSize + b.Run(fmt.Sprintf("cache=ram/pages=%d", ramPages), func(b *testing.B) { + fn(b, ramPages) + }) +} + +func benchmarkScanSpanPackedAllSizeClasses(b *testing.B, nPages int) { + for sc := range gc.NumSizeClasses { + if sc == 0 { + continue + } + if sc >= gc.MinSizeForMallocHeader { + break + } + b.Run(fmt.Sprintf("sizeclass=%d", sc), func(b *testing.B) { + benchmarkScanSpanPacked(b, nPages, sc) + }) + } +} + +func benchmarkScanSpanPacked(b *testing.B, nPages int, sizeClass int) { + rnd := rand.New(rand.NewPCG(42, 42)) + + // Construct a fake memory + mem, free := makeMem(b, nPages) + defer free() + for i := range mem { + // Use values > heap.PageSize because a scan function can discard + // pointers smaller than this. + mem[i] = uintptr(int(gc.PageSize) + i + 1) + } + + // Construct a random pointer mask + ptrs := make([]gc.PtrMask, nPages) + for i := range ptrs { + for j := range ptrs[i] { + ptrs[i][j] = uintptr(rnd.Uint64()) + } + } + + // Visit the pages in a random order + pageOrder := rnd.Perm(nPages) + + // Create the scan buffer. + buf := make([]uintptr, gc.PageWords) + + // Sweep from 0 marks to all marks. We'll use the same marks for each page + // because I don't think that predictability matters. + objBytes := uintptr(gc.SizeClassToSize[sizeClass]) + nObj := gc.PageSize / objBytes + markOrder := rnd.Perm(int(nObj)) + const steps = 11 + for i := 0; i < steps; i++ { + frac := float64(i) / float64(steps-1) + // Set frac marks. + nMarks := int(float64(len(markOrder))*frac + 0.5) + var objMarks gc.ObjMask + for _, mark := range markOrder[:nMarks] { + objMarks[mark/goarch.PtrBits] |= 1 << (mark % goarch.PtrBits) + } + greyClusters := 0 + for page := range ptrs { + greyClusters += countGreyClusters(sizeClass, &objMarks, &ptrs[page]) + } + + // Report MB/s of how much memory they're actually hitting. This assumes + // 64 byte cache lines (TODO: Should it assume 128 byte cache lines?) + // and expands each access to the whole cache line. This is useful for + // comparing against memory bandwidth. + // + // TODO: Add a benchmark that just measures single core memory bandwidth + // for comparison. (See runtime memcpy benchmarks.) + // + // TODO: Should there be a separate measure where we don't expand to + // cache lines? + avgBytes := int64(greyClusters) * int64(cpu.CacheLineSize) / int64(len(ptrs)) + + b.Run(fmt.Sprintf("pct=%d", int(100*frac)), func(b *testing.B) { + b.Run("impl=Reference", func(b *testing.B) { + b.SetBytes(avgBytes) + for i := range b.N { + page := pageOrder[i%len(pageOrder)] + scan.ScanSpanPackedReference(unsafe.Pointer(&mem[gc.PageWords*page]), &buf[0], &objMarks, uintptr(sizeClass), &ptrs[page]) + } + }) + b.Run("impl=Go", func(b *testing.B) { + b.SetBytes(avgBytes) + for i := range b.N { + page := pageOrder[i%len(pageOrder)] + scan.ScanSpanPackedGo(unsafe.Pointer(&mem[gc.PageWords*page]), &buf[0], &objMarks, uintptr(sizeClass), &ptrs[page]) + } + }) + if scan.HasFastScanSpanPacked() { + b.Run("impl=Platform", func(b *testing.B) { + b.SetBytes(avgBytes) + for i := range b.N { + page := pageOrder[i%len(pageOrder)] + scan.ScanSpanPacked(unsafe.Pointer(&mem[gc.PageWords*page]), &buf[0], &objMarks, uintptr(sizeClass), &ptrs[page]) + } + }) + } + }) + } +} + +func countGreyClusters(sizeClass int, objMarks *gc.ObjMask, ptrMask *gc.PtrMask) int { + clusters := 0 + lastCluster := -1 + + expandBy := uintptr(gc.SizeClassToSize[sizeClass]) / goarch.PtrSize + for word := range gc.PageWords { + objI := uintptr(word) / expandBy + if objMarks[objI/goarch.PtrBits]&(1<<(objI%goarch.PtrBits)) == 0 { + continue + } + if ptrMask[word/goarch.PtrBits]&(1<<(word%goarch.PtrBits)) == 0 { + continue + } + c := word * 8 / goarch.PtrBits + if c != lastCluster { + lastCluster = c + clusters++ + } + } + return clusters +} + +func BenchmarkScanMaxBandwidth(b *testing.B) { + // Measure the theoretical "maximum" bandwidth of scanning by reproducing + // the memory access pattern of a full page scan, but using memcpy as the + // kernel instead of scanning. + benchmarkCacheSizes(b, func(b *testing.B, heapPages int) { + mem, free := makeMem(b, heapPages) + defer free() + for i := range mem { + mem[i] = uintptr(int(gc.PageSize) + i + 1) + } + buf := make([]uintptr, gc.PageWords) + + // Visit the pages in a random order + rnd := rand.New(rand.NewPCG(42, 42)) + pageOrder := rnd.Perm(heapPages) + + b.SetBytes(int64(gc.PageSize)) + + b.ResetTimer() + for i := range b.N { + page := pageOrder[i%len(pageOrder)] + copy(buf, mem[gc.PageWords*page:]) + } + }) +} diff --git a/src/internal/runtime/gc/sizeclasses.go b/src/internal/runtime/gc/sizeclasses.go new file mode 100644 index 00000000000000..befba425cc3bd5 --- /dev/null +++ b/src/internal/runtime/gc/sizeclasses.go @@ -0,0 +1,102 @@ +// Code generated by mksizeclasses.go; DO NOT EDIT. +//go:generate go -C ../../../runtime/_mkmalloc run mksizeclasses.go + +package gc + +// class bytes/obj bytes/span objects tail waste max waste min align +// 1 8 8192 1024 0 87.50% 8 +// 2 16 8192 512 0 43.75% 16 +// 3 24 8192 341 8 29.24% 8 +// 4 32 8192 256 0 21.88% 32 +// 5 48 8192 170 32 31.52% 16 +// 6 64 8192 128 0 23.44% 64 +// 7 80 8192 102 32 19.07% 16 +// 8 96 8192 85 32 15.95% 32 +// 9 112 8192 73 16 13.56% 16 +// 10 128 8192 64 0 11.72% 128 +// 11 144 8192 56 128 11.82% 16 +// 12 160 8192 51 32 9.73% 32 +// 13 176 8192 46 96 9.59% 16 +// 14 192 8192 42 128 9.25% 64 +// 15 208 8192 39 80 8.12% 16 +// 16 224 8192 36 128 8.15% 32 +// 17 240 8192 34 32 6.62% 16 +// 18 256 8192 32 0 5.86% 256 +// 19 288 8192 28 128 12.16% 32 +// 20 320 8192 25 192 11.80% 64 +// 21 352 8192 23 96 9.88% 32 +// 22 384 8192 21 128 9.51% 128 +// 23 416 8192 19 288 10.71% 32 +// 24 448 8192 18 128 8.37% 64 +// 25 480 8192 17 32 6.82% 32 +// 26 512 8192 16 0 6.05% 512 +// 27 576 8192 14 128 12.33% 64 +// 28 640 8192 12 512 15.48% 128 +// 29 704 8192 11 448 13.93% 64 +// 30 768 8192 10 512 13.94% 256 +// 31 896 8192 9 128 15.52% 128 +// 32 1024 8192 8 0 12.40% 1024 +// 33 1152 8192 7 128 12.41% 128 +// 34 1280 8192 6 512 15.55% 256 +// 35 1408 16384 11 896 14.00% 128 +// 36 1536 8192 5 512 14.00% 512 +// 37 1792 16384 9 256 15.57% 256 +// 38 2048 8192 4 0 12.45% 2048 +// 39 2304 16384 7 256 12.46% 256 +// 40 2688 8192 3 128 15.59% 128 +// 41 3072 24576 8 0 12.47% 1024 +// 42 3200 16384 5 384 6.22% 128 +// 43 3456 24576 7 384 8.83% 128 +// 44 4096 8192 2 0 15.60% 4096 +// 45 4864 24576 5 256 16.65% 256 +// 46 5376 16384 3 256 10.92% 256 +// 47 6144 24576 4 0 12.48% 2048 +// 48 6528 32768 5 128 6.23% 128 +// 49 6784 40960 6 256 4.36% 128 +// 50 6912 49152 7 768 3.37% 256 +// 51 8192 8192 1 0 15.61% 8192 +// 52 9472 57344 6 512 14.28% 256 +// 53 9728 49152 5 512 3.64% 512 +// 54 10240 40960 4 0 4.99% 2048 +// 55 10880 32768 3 128 6.24% 128 +// 56 12288 24576 2 0 11.45% 4096 +// 57 13568 40960 3 256 9.99% 256 +// 58 14336 57344 4 0 5.35% 2048 +// 59 16384 16384 1 0 12.49% 8192 +// 60 18432 73728 4 0 11.11% 2048 +// 61 19072 57344 3 128 3.57% 128 +// 62 20480 40960 2 0 6.87% 4096 +// 63 21760 65536 3 256 6.25% 256 +// 64 24576 24576 1 0 11.45% 8192 +// 65 27264 81920 3 128 10.00% 128 +// 66 28672 57344 2 0 4.91% 4096 +// 67 32768 32768 1 0 12.50% 8192 + +// alignment bits min obj size +// 8 3 8 +// 16 4 32 +// 32 5 256 +// 64 6 512 +// 128 7 768 +// 4096 12 28672 +// 8192 13 32768 + +const ( + MinHeapAlign = 8 + MaxSmallSize = 32768 + SmallSizeDiv = 8 + SmallSizeMax = 1024 + LargeSizeDiv = 128 + NumSizeClasses = 68 + PageShift = 13 + MaxObjsPerSpan = 1024 + MaxSizeClassNPages = 10 + TinySize = 16 + TinySizeClass = 2 +) + +var SizeClassToSize = [NumSizeClasses]uint16{0, 8, 16, 24, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 896, 1024, 1152, 1280, 1408, 1536, 1792, 2048, 2304, 2688, 3072, 3200, 3456, 4096, 4864, 5376, 6144, 6528, 6784, 6912, 8192, 9472, 9728, 10240, 10880, 12288, 13568, 14336, 16384, 18432, 19072, 20480, 21760, 24576, 27264, 28672, 32768} +var SizeClassToNPages = [NumSizeClasses]uint8{0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 3, 2, 3, 1, 3, 2, 3, 4, 5, 6, 1, 7, 6, 5, 4, 3, 5, 7, 2, 9, 7, 5, 8, 3, 10, 7, 4} +var SizeClassToDivMagic = [NumSizeClasses]uint32{0, ^uint32(0)/8 + 1, ^uint32(0)/16 + 1, ^uint32(0)/24 + 1, ^uint32(0)/32 + 1, ^uint32(0)/48 + 1, ^uint32(0)/64 + 1, ^uint32(0)/80 + 1, ^uint32(0)/96 + 1, ^uint32(0)/112 + 1, ^uint32(0)/128 + 1, ^uint32(0)/144 + 1, ^uint32(0)/160 + 1, ^uint32(0)/176 + 1, ^uint32(0)/192 + 1, ^uint32(0)/208 + 1, ^uint32(0)/224 + 1, ^uint32(0)/240 + 1, ^uint32(0)/256 + 1, ^uint32(0)/288 + 1, ^uint32(0)/320 + 1, ^uint32(0)/352 + 1, ^uint32(0)/384 + 1, ^uint32(0)/416 + 1, ^uint32(0)/448 + 1, ^uint32(0)/480 + 1, ^uint32(0)/512 + 1, ^uint32(0)/576 + 1, ^uint32(0)/640 + 1, ^uint32(0)/704 + 1, ^uint32(0)/768 + 1, ^uint32(0)/896 + 1, ^uint32(0)/1024 + 1, ^uint32(0)/1152 + 1, ^uint32(0)/1280 + 1, ^uint32(0)/1408 + 1, ^uint32(0)/1536 + 1, ^uint32(0)/1792 + 1, ^uint32(0)/2048 + 1, ^uint32(0)/2304 + 1, ^uint32(0)/2688 + 1, ^uint32(0)/3072 + 1, ^uint32(0)/3200 + 1, ^uint32(0)/3456 + 1, ^uint32(0)/4096 + 1, ^uint32(0)/4864 + 1, ^uint32(0)/5376 + 1, ^uint32(0)/6144 + 1, ^uint32(0)/6528 + 1, ^uint32(0)/6784 + 1, ^uint32(0)/6912 + 1, ^uint32(0)/8192 + 1, ^uint32(0)/9472 + 1, ^uint32(0)/9728 + 1, ^uint32(0)/10240 + 1, ^uint32(0)/10880 + 1, ^uint32(0)/12288 + 1, ^uint32(0)/13568 + 1, ^uint32(0)/14336 + 1, ^uint32(0)/16384 + 1, ^uint32(0)/18432 + 1, ^uint32(0)/19072 + 1, ^uint32(0)/20480 + 1, ^uint32(0)/21760 + 1, ^uint32(0)/24576 + 1, ^uint32(0)/27264 + 1, ^uint32(0)/28672 + 1, ^uint32(0)/32768 + 1} +var SizeToSizeClass8 = [SmallSizeMax/SmallSizeDiv + 1]uint8{0, 1, 2, 3, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 23, 23, 23, 23, 24, 24, 24, 24, 25, 25, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27, 27, 27, 28, 28, 28, 28, 28, 28, 28, 28, 29, 29, 29, 29, 29, 29, 29, 29, 30, 30, 30, 30, 30, 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32} +var SizeToSizeClass128 = [(MaxSmallSize-SmallSizeMax)/LargeSizeDiv + 1]uint8{32, 33, 34, 35, 36, 37, 37, 38, 38, 39, 39, 40, 40, 40, 41, 41, 41, 42, 43, 43, 44, 44, 44, 44, 44, 45, 45, 45, 45, 45, 45, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 48, 48, 48, 49, 49, 50, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 53, 53, 54, 54, 54, 54, 55, 55, 55, 55, 55, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 58, 58, 58, 58, 58, 58, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 61, 61, 61, 61, 61, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67} diff --git a/src/internal/runtime/maps/export_test.go b/src/internal/runtime/maps/export_test.go new file mode 100644 index 00000000000000..adce44ba935085 --- /dev/null +++ b/src/internal/runtime/maps/export_test.go @@ -0,0 +1,131 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "internal/abi" + "unsafe" +) + +type CtrlGroup = ctrlGroup + +const DebugLog = debugLog + +var AlignUpPow2 = alignUpPow2 + +const MaxTableCapacity = maxTableCapacity +const MaxAvgGroupLoad = maxAvgGroupLoad + +// This isn't equivalent to runtime.maxAlloc. It is fine for basic testing but +// we can't properly test hint alloc overflows with this. +const maxAllocTest = 1 << 30 + +func newTestMapType[K comparable, V any]() *abi.MapType { + var m map[K]V + mTyp := abi.TypeOf(m) + mt := (*abi.MapType)(unsafe.Pointer(mTyp)) + return mt +} + +func NewTestMap[K comparable, V any](hint uintptr) (*Map, *abi.MapType) { + mt := newTestMapType[K, V]() + return NewMap(mt, hint, nil, maxAllocTest), mt +} + +func (m *Map) TableCount() int { + if m.dirLen <= 0 { + return 0 + } + return m.dirLen +} + +// Total group count, summed across all tables. +func (m *Map) GroupCount() uint64 { + if m.dirLen <= 0 { + if m.dirPtr == nil { + return 0 + } + return 1 + } + + var n uint64 + var lastTab *table + for i := range m.dirLen { + t := m.directoryAt(uintptr(i)) + if t == lastTab { + continue + } + lastTab = t + n += t.groups.lengthMask + 1 + } + return n +} + +// Return a key from a group containing no empty slots. +// +// Returns nil if there are no full groups. +// Returns nil if a group is full but contains entirely deleted slots. +// Returns nil if the map is small. +func (m *Map) KeyFromFullGroup(typ *abi.MapType) unsafe.Pointer { + if m.dirLen <= 0 { + return nil + } + + var lastTab *table + for i := range m.dirLen { + t := m.directoryAt(uintptr(i)) + if t == lastTab { + continue + } + lastTab = t + + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + match := g.ctrls().matchEmpty() + if match != 0 { + continue + } + + // All full or deleted slots. + for j := uintptr(0); j < abi.MapGroupSlots; j++ { + if g.ctrls().get(j) == ctrlDeleted { + continue + } + slotKey := g.key(typ, j) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + return slotKey + } + } + } + + return nil +} + +// Returns nil if the map is small. +func (m *Map) TableFor(typ *abi.MapType, key unsafe.Pointer) *table { + if m.dirLen <= 0 { + return nil + } + + hash := typ.Hasher(key, m.seed) + idx := m.directoryIndex(hash) + return m.directoryAt(idx) +} + +func (t *table) GrowthLeft() uint64 { + return uint64(t.growthLeft) +} + +// Returns the start address of the groups array. +func (t *table) GroupsStart() unsafe.Pointer { + return t.groups.data +} + +// Returns the length of the groups array. +func (t *table) GroupsLength() uintptr { + return uintptr(t.groups.lengthMask + 1) +} diff --git a/src/internal/runtime/maps/fuzz_test.go b/src/internal/runtime/maps/fuzz_test.go new file mode 100644 index 00000000000000..07a7d0380a6b08 --- /dev/null +++ b/src/internal/runtime/maps/fuzz_test.go @@ -0,0 +1,211 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps_test + +import ( + "bytes" + "encoding/binary" + "fmt" + "internal/runtime/maps" + "reflect" + "testing" + "unsafe" +) + +// The input to FuzzTable is a binary-encoded array of fuzzCommand structs. +// +// Each fuzz call begins with an empty Map[uint16, uint32]. +// +// Each command is then executed on the map in sequence. Operations with +// output (e.g., Get) are verified against a reference map. +type fuzzCommand struct { + Op fuzzOp + + // Used for Get, Put, Delete. + Key uint16 + + // Used for Put. + Elem uint32 +} + +// Encoded size of fuzzCommand. +var fuzzCommandSize = binary.Size(fuzzCommand{}) + +type fuzzOp uint8 + +const ( + fuzzOpGet fuzzOp = iota + fuzzOpPut + fuzzOpDelete +) + +func encode(fc []fuzzCommand) []byte { + var buf bytes.Buffer + if err := binary.Write(&buf, binary.LittleEndian, fc); err != nil { + panic(fmt.Sprintf("error writing %v: %v", fc, err)) + } + return buf.Bytes() +} + +func decode(b []byte) []fuzzCommand { + // Round b down to a multiple of fuzzCommand size. i.e., ignore extra + // bytes of input. + entries := len(b) / fuzzCommandSize + usefulSize := entries * fuzzCommandSize + b = b[:usefulSize] + + fc := make([]fuzzCommand, entries) + buf := bytes.NewReader(b) + if err := binary.Read(buf, binary.LittleEndian, &fc); err != nil { + panic(fmt.Sprintf("error reading %v: %v", b, err)) + } + + return fc +} + +func TestEncodeDecode(t *testing.T) { + fc := []fuzzCommand{ + { + Op: fuzzOpPut, + Key: 123, + Elem: 456, + }, + { + Op: fuzzOpGet, + Key: 123, + }, + } + + b := encode(fc) + got := decode(b) + if !reflect.DeepEqual(fc, got) { + t.Errorf("encode-decode roundtrip got %+v want %+v", got, fc) + } + + // Extra trailing bytes ignored. + b = append(b, 42) + got = decode(b) + if !reflect.DeepEqual(fc, got) { + t.Errorf("encode-decode (extra byte) roundtrip got %+v want %+v", got, fc) + } +} + +func FuzzTable(f *testing.F) { + // All of the ops. + f.Add(encode([]fuzzCommand{ + { + Op: fuzzOpPut, + Key: 123, + Elem: 456, + }, + { + Op: fuzzOpDelete, + Key: 123, + }, + { + Op: fuzzOpGet, + Key: 123, + }, + })) + + // Add enough times to trigger grow. + f.Add(encode([]fuzzCommand{ + { + Op: fuzzOpPut, + Key: 1, + Elem: 101, + }, + { + Op: fuzzOpPut, + Key: 2, + Elem: 102, + }, + { + Op: fuzzOpPut, + Key: 3, + Elem: 103, + }, + { + Op: fuzzOpPut, + Key: 4, + Elem: 104, + }, + { + Op: fuzzOpPut, + Key: 5, + Elem: 105, + }, + { + Op: fuzzOpPut, + Key: 6, + Elem: 106, + }, + { + Op: fuzzOpPut, + Key: 7, + Elem: 107, + }, + { + Op: fuzzOpPut, + Key: 8, + Elem: 108, + }, + { + Op: fuzzOpGet, + Key: 1, + }, + { + Op: fuzzOpDelete, + Key: 2, + }, + { + Op: fuzzOpPut, + Key: 2, + Elem: 42, + }, + { + Op: fuzzOpGet, + Key: 2, + }, + })) + + f.Fuzz(func(t *testing.T, in []byte) { + fc := decode(in) + if len(fc) == 0 { + return + } + + m, typ := maps.NewTestMap[uint16, uint32](8) + ref := make(map[uint16]uint32) + for _, c := range fc { + switch c.Op { + case fuzzOpGet: + elemPtr, ok := m.Get(typ, unsafe.Pointer(&c.Key)) + refElem, refOK := ref[c.Key] + + if ok != refOK { + t.Errorf("Get(%d) got ok %v want ok %v", c.Key, ok, refOK) + } + if !ok { + continue + } + gotElem := *(*uint32)(elemPtr) + if gotElem != refElem { + t.Errorf("Get(%d) got %d want %d", c.Key, gotElem, refElem) + } + case fuzzOpPut: + m.Put(typ, unsafe.Pointer(&c.Key), unsafe.Pointer(&c.Elem)) + ref[c.Key] = c.Elem + case fuzzOpDelete: + m.Delete(typ, unsafe.Pointer(&c.Key)) + delete(ref, c.Key) + default: + // Just skip this command to keep the fuzzer + // less constrained. + continue + } + } + }) +} diff --git a/src/internal/runtime/maps/group.go b/src/internal/runtime/maps/group.go new file mode 100644 index 00000000000000..a56d32aca0aef5 --- /dev/null +++ b/src/internal/runtime/maps/group.go @@ -0,0 +1,364 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "internal/abi" + "internal/goarch" + "internal/runtime/sys" + "unsafe" +) + +const ( + // Maximum load factor prior to growing. + // + // 7/8 is the same load factor used by Abseil, but Abseil defaults to + // 16 slots per group, so they get two empty slots vs our one empty + // slot. We may want to reevaluate if this is best for us. + maxAvgGroupLoad = 7 + + ctrlEmpty ctrl = 0b10000000 + ctrlDeleted ctrl = 0b11111110 + + bitsetLSB = 0x0101010101010101 + bitsetMSB = 0x8080808080808080 + bitsetEmpty = bitsetLSB * uint64(ctrlEmpty) +) + +// bitset represents a set of slots within a group. +// +// The underlying representation depends on GOARCH. +// +// On AMD64, bitset uses one bit per slot, where the bit is set if the slot is +// part of the set. All of the ctrlGroup.match* methods are replaced with +// intrinsics that return this packed representation. +// +// On other architectures, bitset uses one byte per slot, where each byte is +// either 0x80 if the slot is part of the set or 0x00 otherwise. This makes it +// convenient to calculate for an entire group at once using standard +// arithmetic instructions. +type bitset uint64 + +// first returns the relative index of the first control byte in the group that +// is in the set. +// +// Preconditions: b is not 0 (empty). +func (b bitset) first() uintptr { + return bitsetFirst(b) +} + +// Portable implementation of first. +// +// On AMD64, this is replaced with an intrisic that simply does +// TrailingZeros64. There is no need to shift as the bitset is packed. +func bitsetFirst(b bitset) uintptr { + return uintptr(sys.TrailingZeros64(uint64(b))) >> 3 +} + +// removeFirst clears the first set bit (that is, resets the least significant +// set bit to 0). +func (b bitset) removeFirst() bitset { + return b & (b - 1) +} + +// removeBelow clears all set bits below slot i (non-inclusive). +func (b bitset) removeBelow(i uintptr) bitset { + return bitsetRemoveBelow(b, i) +} + +// Portable implementation of removeBelow. +// +// On AMD64, this is replaced with an intrisic that clears the lower i bits. +func bitsetRemoveBelow(b bitset, i uintptr) bitset { + // Clear all bits below slot i's byte. + mask := (uint64(1) << (8 * uint64(i))) - 1 + return b &^ bitset(mask) +} + +// lowestSet returns true if the bit is set for the lowest index in the bitset. +// +// This is intended for use with shiftOutLowest to loop over all entries in the +// bitset regardless of whether they are set. +func (b bitset) lowestSet() bool { + return bitsetLowestSet(b) +} + +// Portable implementation of lowestSet. +// +// On AMD64, this is replaced with an intrisic that checks the lowest bit. +func bitsetLowestSet(b bitset) bool { + return b&(1<<7) != 0 +} + +// shiftOutLowest shifts the lowest entry out of the bitset. Afterwards, the +// lowest entry in the bitset corresponds to the next slot. +func (b bitset) shiftOutLowest() bitset { + return bitsetShiftOutLowest(b) +} + +// Portable implementation of shiftOutLowest. +// +// On AMD64, this is replaced with an intrisic that shifts a single bit. +func bitsetShiftOutLowest(b bitset) bitset { + return b >> 8 +} + +// count returns the number of bits set in b. +func (b bitset) count() int { + // Note: works for both bitset representations (AMD64 and generic) + return sys.OnesCount64(uint64(b)) +} + +// Each slot in the hash table has a control byte which can have one of three +// states: empty, deleted, and full. They have the following bit patterns: +// +// empty: 1 0 0 0 0 0 0 0 +// deleted: 1 1 1 1 1 1 1 0 +// full: 0 h h h h h h h // h represents the H2 hash bits +// +// TODO(prattmic): Consider inverting the top bit so that the zero value is empty. +type ctrl uint8 + +// ctrlGroup is a fixed size array of abi.MapGroupSlots control bytes +// stored in a uint64. +type ctrlGroup uint64 + +// get returns the i-th control byte. +func (g *ctrlGroup) get(i uintptr) ctrl { + if goarch.BigEndian { + return *(*ctrl)(unsafe.Add(unsafe.Pointer(g), 7-i)) + } + return *(*ctrl)(unsafe.Add(unsafe.Pointer(g), i)) +} + +// set sets the i-th control byte. +func (g *ctrlGroup) set(i uintptr, c ctrl) { + if goarch.BigEndian { + *(*ctrl)(unsafe.Add(unsafe.Pointer(g), 7-i)) = c + return + } + *(*ctrl)(unsafe.Add(unsafe.Pointer(g), i)) = c +} + +// setEmpty sets all the control bytes to empty. +func (g *ctrlGroup) setEmpty() { + *g = ctrlGroup(bitsetEmpty) +} + +// matchH2 returns the set of slots which are full and for which the 7-bit hash +// matches the given value. May return false positives. +func (g ctrlGroup) matchH2(h uintptr) bitset { + return ctrlGroupMatchH2(g, h) +} + +// Portable implementation of matchH2. +// +// Note: On AMD64, this is an intrinsic implemented with SIMD instructions. See +// note on bitset about the packed intrinsified return value. +func ctrlGroupMatchH2(g ctrlGroup, h uintptr) bitset { + // NB: This generic matching routine produces false positive matches when + // h is 2^N and the control bytes have a seq of 2^N followed by 2^N+1. For + // example: if ctrls==0x0302 and h=02, we'll compute v as 0x0100. When we + // subtract off 0x0101 the first 2 bytes we'll become 0xffff and both be + // considered matches of h. The false positive matches are not a problem, + // just a rare inefficiency. Note that they only occur if there is a real + // match and never occur on ctrlEmpty, or ctrlDeleted. The subsequent key + // comparisons ensure that there is no correctness issue. + v := uint64(g) ^ (bitsetLSB * uint64(h)) + return bitset(((v - bitsetLSB) &^ v) & bitsetMSB) +} + +// matchEmpty returns the set of slots in the group that are empty. +func (g ctrlGroup) matchEmpty() bitset { + return ctrlGroupMatchEmpty(g) +} + +// Portable implementation of matchEmpty. +// +// Note: On AMD64, this is an intrinsic implemented with SIMD instructions. See +// note on bitset about the packed intrinsified return value. +func ctrlGroupMatchEmpty(g ctrlGroup) bitset { + // An empty slot is 1000 0000 + // A deleted slot is 1111 1110 + // A full slot is 0??? ???? + // + // A slot is empty iff bit 7 is set and bit 1 is not. We could select any + // of the other bits here (e.g. v << 1 would also work). + v := uint64(g) + return bitset((v &^ (v << 6)) & bitsetMSB) +} + +// matchEmptyOrDeleted returns the set of slots in the group that are empty or +// deleted. +func (g ctrlGroup) matchEmptyOrDeleted() bitset { + return ctrlGroupMatchEmptyOrDeleted(g) +} + +// Portable implementation of matchEmptyOrDeleted. +// +// Note: On AMD64, this is an intrinsic implemented with SIMD instructions. See +// note on bitset about the packed intrinsified return value. +func ctrlGroupMatchEmptyOrDeleted(g ctrlGroup) bitset { + // An empty slot is 1000 0000 + // A deleted slot is 1111 1110 + // A full slot is 0??? ???? + // + // A slot is empty or deleted iff bit 7 is set. + v := uint64(g) + return bitset(v & bitsetMSB) +} + +// matchFull returns the set of slots in the group that are full. +func (g ctrlGroup) matchFull() bitset { + return ctrlGroupMatchFull(g) +} + +// anyFull reports whether any slots in the group are full. +func (g ctrlGroup) anyFull() bool { + // A slot is full iff bit 7 is unset. Test whether any slot has bit 7 unset. + return (^g)&bitsetMSB != 0 +} + +// Portable implementation of matchFull. +// +// Note: On AMD64, this is an intrinsic implemented with SIMD instructions. See +// note on bitset about the packed intrinsified return value. +func ctrlGroupMatchFull(g ctrlGroup) bitset { + // An empty slot is 1000 0000 + // A deleted slot is 1111 1110 + // A full slot is 0??? ???? + // + // A slot is full iff bit 7 is unset. + v := uint64(g) + return bitset(^v & bitsetMSB) +} + +// groupReference is a wrapper type representing a single slot group stored at +// data. +// +// A group holds abi.MapGroupSlots slots (key/elem pairs) plus their +// control word. +type groupReference struct { + // data points to the group, which is described by typ.Group and has + // layout: + // + // type group struct { + // ctrls ctrlGroup + // slots [abi.MapGroupSlots]slot + // } + // + // type slot struct { + // key typ.Key + // elem typ.Elem + // } + data unsafe.Pointer // data *typ.Group +} + +const ( + ctrlGroupsSize = unsafe.Sizeof(ctrlGroup(0)) + groupSlotsOffset = ctrlGroupsSize +) + +// alignUp rounds n up to a multiple of a. a must be a power of 2. +func alignUp(n, a uintptr) uintptr { + return (n + a - 1) &^ (a - 1) +} + +// alignUpPow2 rounds n up to the next power of 2. +// +// Returns true if round up causes overflow. +func alignUpPow2(n uint64) (uint64, bool) { + if n == 0 { + return 0, false + } + v := (uint64(1) << sys.Len64(n-1)) + if v == 0 { + return 0, true + } + return v, false +} + +// ctrls returns the group control word. +func (g *groupReference) ctrls() *ctrlGroup { + return (*ctrlGroup)(g.data) +} + +// key returns a pointer to the key at index i. +func (g *groupReference) key(typ *abi.MapType, i uintptr) unsafe.Pointer { + offset := groupSlotsOffset + i*typ.SlotSize + + return unsafe.Pointer(uintptr(g.data) + offset) +} + +// elem returns a pointer to the element at index i. +func (g *groupReference) elem(typ *abi.MapType, i uintptr) unsafe.Pointer { + offset := groupSlotsOffset + i*typ.SlotSize + typ.ElemOff + + return unsafe.Pointer(uintptr(g.data) + offset) +} + +// groupsReference is a wrapper type describing an array of groups stored at +// data. +type groupsReference struct { + // data points to an array of groups. See groupReference above for the + // definition of group. + data unsafe.Pointer // data *[length]typ.Group + + // lengthMask is the number of groups in data minus one (note that + // length must be a power of two). This allows computing i%length + // quickly using bitwise AND. + lengthMask uint64 +} + +// newGroups allocates a new array of length groups. +// +// Length must be a power of two. +func newGroups(typ *abi.MapType, length uint64) groupsReference { + return groupsReference{ + // TODO: make the length type the same throughout. + data: newarray(typ.Group, int(length)), + lengthMask: length - 1, + } +} + +// group returns the group at index i. +func (g *groupsReference) group(typ *abi.MapType, i uint64) groupReference { + // TODO(prattmic): Do something here about truncation on cast to + // uintptr on 32-bit systems? + offset := uintptr(i) * typ.GroupSize + + return groupReference{ + data: unsafe.Pointer(uintptr(g.data) + offset), + } +} + +func cloneGroup(typ *abi.MapType, newGroup, oldGroup groupReference) { + typedmemmove(typ.Group, newGroup.data, oldGroup.data) + if typ.IndirectKey() { + // Deep copy keys if indirect. + for i := uintptr(0); i < abi.MapGroupSlots; i++ { + oldKey := *(*unsafe.Pointer)(oldGroup.key(typ, i)) + if oldKey == nil { + continue + } + newKey := newobject(typ.Key) + typedmemmove(typ.Key, newKey, oldKey) + *(*unsafe.Pointer)(newGroup.key(typ, i)) = newKey + } + } + if typ.IndirectElem() { + // Deep copy elems if indirect. + for i := uintptr(0); i < abi.MapGroupSlots; i++ { + oldElem := *(*unsafe.Pointer)(oldGroup.elem(typ, i)) + if oldElem == nil { + continue + } + newElem := newobject(typ.Elem) + typedmemmove(typ.Elem, newElem, oldElem) + *(*unsafe.Pointer)(newGroup.elem(typ, i)) = newElem + } + } + +} diff --git a/src/internal/runtime/maps/map.go b/src/internal/runtime/maps/map.go new file mode 100644 index 00000000000000..515558a94f635a --- /dev/null +++ b/src/internal/runtime/maps/map.go @@ -0,0 +1,898 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package maps implements Go's builtin map type. +package maps + +import ( + "internal/abi" + "internal/goarch" + "internal/runtime/math" + "internal/runtime/sys" + "unsafe" +) + +// This package contains the implementation of Go's builtin map type. +// +// The map design is based on Abseil's "Swiss Table" map design +// (https://abseil.io/about/design/swisstables), with additional modifications +// to cover Go's additional requirements, discussed below. +// +// Terminology: +// - Slot: A storage location of a single key/element pair. +// - Group: A group of abi.MapGroupSlots (8) slots, plus a control word. +// - Control word: An 8-byte word which denotes whether each slot is empty, +// deleted, or used. If a slot is used, its control byte also contains the +// lower 7 bits of the hash (H2). +// - H1: Upper 57 bits of a hash. +// - H2: Lower 7 bits of a hash. +// - Table: A complete "Swiss Table" hash table. A table consists of one or +// more groups for storage plus metadata to handle operation and determining +// when to grow. +// - Map: The top-level Map type consists of zero or more tables for storage. +// The upper bits of the hash select which table a key belongs to. +// - Directory: Array of the tables used by the map. +// +// At its core, the table design is similar to a traditional open-addressed +// hash table. Storage consists of an array of groups, which effectively means +// an array of key/elem slots with some control words interspersed. Lookup uses +// the hash to determine an initial group to check. If, due to collisions, this +// group contains no match, the probe sequence selects the next group to check +// (see below for more detail about the probe sequence). +// +// The key difference occurs within a group. In a standard open-addressed +// linear probed hash table, we would check each slot one at a time to find a +// match. A swiss table utilizes the extra control word to check all 8 slots in +// parallel. +// +// Each byte in the control word corresponds to one of the slots in the group. +// In each byte, 1 bit is used to indicate whether the slot is in use, or if it +// is empty/deleted. The other 7 bits contain the lower 7 bits of the hash for +// the key in that slot. See [ctrl] for the exact encoding. +// +// During lookup, we can use some clever bitwise manipulation to compare all 8 +// 7-bit hashes against the input hash in parallel (see [ctrlGroup.matchH2]). +// That is, we effectively perform 8 steps of probing in a single operation. +// With SIMD instructions, this could be extended to 16 slots with a 16-byte +// control word. +// +// Since we only use 7 bits of the 64 bit hash, there is a 1 in 128 (~0.7%) +// probability of false positive on each slot, but that's fine: we always need +// double check each match with a standard key comparison regardless. +// +// Probing +// +// Probing is done using the upper 57 bits (H1) of the hash as an index into +// the groups array. Probing walks through the groups using quadratic probing +// until it finds a group with a match or a group with an empty slot. See +// [probeSeq] for specifics about the probe sequence. Note the probe +// invariants: the number of groups must be a power of two, and the end of a +// probe sequence must be a group with an empty slot (the table can never be +// 100% full). +// +// Deletion +// +// Probing stops when it finds a group with an empty slot. This affects +// deletion: when deleting from a completely full group, we must not mark the +// slot as empty, as there could be more slots used later in a probe sequence +// and this deletion would cause probing to stop too early. Instead, we mark +// such slots as "deleted" with a tombstone. If the group still has an empty +// slot, we don't need a tombstone and directly mark the slot empty. Insert +// prioritizes reuse of tombstones over filling an empty slots. Otherwise, +// tombstones are only completely cleared during grow, as an in-place cleanup +// complicates iteration. +// +// Growth +// +// The probe sequence depends on the number of groups. Thus, when growing the +// group count all slots must be reordered to match the new probe sequence. In +// other words, an entire table must be grown at once. +// +// In order to support incremental growth, the map splits its contents across +// multiple tables. Each table is still a full hash table, but an individual +// table may only service a subset of the hash space. Growth occurs on +// individual tables, so while an entire table must grow at once, each of these +// grows is only a small portion of a map. The maximum size of a single grow is +// limited by limiting the maximum size of a table before it is split into +// multiple tables. +// +// A map starts with a single table. Up to [maxTableCapacity], growth simply +// replaces this table with a replacement with double capacity. Beyond this +// limit, growth splits the table into two. +// +// The map uses "extendible hashing" to select which table to use. In +// extendible hashing, we use the upper bits of the hash as an index into an +// array of tables (called the "directory"). The number of bits uses increases +// as the number of tables increases. For example, when there is only 1 table, +// we use 0 bits (no selection necessary). When there are 2 tables, we use 1 +// bit to select either the 0th or 1st table. [Map.globalDepth] is the number +// of bits currently used for table selection, and by extension (1 << +// globalDepth), the size of the directory. +// +// Note that each table has its own load factor and grows independently. If the +// 1st bucket grows, it will split. We'll need 2 bits to select tables, though +// we'll have 3 tables total rather than 4. We support this by allowing +// multiple indices to point to the same table. This example: +// +// directory (globalDepth=2) +// +----+ +// | 00 | --\ +// +----+ +--> table (localDepth=1) +// | 01 | --/ +// +----+ +// | 10 | ------> table (localDepth=2) +// +----+ +// | 11 | ------> table (localDepth=2) +// +----+ +// +// Tables track the depth they were created at (localDepth). It is necessary to +// grow the directory when splitting a table where globalDepth == localDepth. +// +// Iteration +// +// Iteration is the most complex part of the map due to Go's generous iteration +// semantics. A summary of semantics from the spec: +// 1. Adding and/or deleting entries during iteration MUST NOT cause iteration +// to return the same entry more than once. +// 2. Entries added during iteration MAY be returned by iteration. +// 3. Entries modified during iteration MUST return their latest value. +// 4. Entries deleted during iteration MUST NOT be returned by iteration. +// 5. Iteration order is unspecified. In the implementation, it is explicitly +// randomized. +// +// If the map never grows, these semantics are straightforward: just iterate +// over every table in the directory and every group and slot in each table. +// These semantics all land as expected. +// +// If the map grows during iteration, things complicate significantly. First +// and foremost, we need to track which entries we already returned to satisfy +// (1). There are three types of grow: +// a. A table replaced by a single larger table. +// b. A table split into two replacement tables. +// c. Growing the directory (occurs as part of (b) if necessary). +// +// For all of these cases, the replacement table(s) will have a different probe +// sequence, so simply tracking the current group and slot indices is not +// sufficient. +// +// For (a) and (b), note that grows of tables other than the one we are +// currently iterating over are irrelevant. +// +// We handle (a) and (b) by having the iterator keep a reference to the table +// it is currently iterating over, even after the table is replaced. We keep +// iterating over the original table to maintain the iteration order and avoid +// violating (1). Any new entries added only to the replacement table(s) will +// be skipped (allowed by (2)). To avoid violating (3) or (4), while we use the +// original table to select the keys, we must look them up again in the new +// table(s) to determine if they have been modified or deleted. There is yet +// another layer of complexity if the key does not compare equal itself. See +// [Iter.Next] for the gory details. +// +// Note that for (b) once we finish iterating over the old table we'll need to +// skip the next entry in the directory, as that contains the second split of +// the old table. We can use the old table's localDepth to determine the next +// logical index to use. +// +// For (b), we must adjust the current directory index when the directory +// grows. This is more straightforward, as the directory orders remains the +// same after grow, so we just double the index if the directory size doubles. + +// Extracts the H1 portion of a hash: the 57 upper bits. +// TODO(prattmic): what about 32-bit systems? +func h1(h uintptr) uintptr { + return h >> 7 +} + +// Extracts the H2 portion of a hash: the 7 bits not used for h1. +// +// These are used as an occupied control byte. +func h2(h uintptr) uintptr { + return h & 0x7f +} + +// Note: changes here must be reflected in cmd/compile/internal/reflectdata/map.go:MapType. +type Map struct { + // The number of filled slots (i.e. the number of elements in all + // tables). Excludes deleted slots. + // Must be first (known by the compiler, for len() builtin). + used uint64 + + // seed is the hash seed, computed as a unique random number per map. + seed uintptr + + // The directory of tables. + // + // Normally dirPtr points to an array of table pointers + // + // dirPtr *[dirLen]*table + // + // The length (dirLen) of this array is `1 << globalDepth`. Multiple + // entries may point to the same table. See top-level comment for more + // details. + // + // Small map optimization: if the map always contained + // abi.MapGroupSlots or fewer entries, it fits entirely in a + // single group. In that case dirPtr points directly to a single group. + // + // dirPtr *group + // + // In this case, dirLen is 0. used counts the number of used slots in + // the group. Note that small maps never have deleted slots (as there + // is no probe sequence to maintain). + dirPtr unsafe.Pointer + dirLen int + + // The number of bits to use in table directory lookups. + globalDepth uint8 + + // The number of bits to shift out of the hash for directory lookups. + // On 64-bit systems, this is 64 - globalDepth. + globalShift uint8 + + // writing is a flag that is toggled (XOR 1) while the map is being + // written. Normally it is set to 1 when writing, but if there are + // multiple concurrent writers, then toggling increases the probability + // that both sides will detect the race. + writing uint8 + + // tombstonePossible is false if we know that no table in this map + // contains a tombstone. + tombstonePossible bool + + // clearSeq is a sequence counter of calls to Clear. It is used to + // detect map clears during iteration. + clearSeq uint64 +} + +// Use 64-bit hash on 64-bit systems, except on Wasm, where we use +// 32-bit hash (see runtime/hash32.go). +const Use64BitHash = goarch.PtrSize == 8 && goarch.IsWasm == 0 + +func depthToShift(depth uint8) uint8 { + if !Use64BitHash { + return 32 - depth + } + return 64 - depth +} + +// If m is non-nil, it should be used rather than allocating. +// +// maxAlloc should be runtime.maxAlloc. +// +// TODO(prattmic): Put maxAlloc somewhere accessible. +func NewMap(mt *abi.MapType, hint uintptr, m *Map, maxAlloc uintptr) *Map { + if m == nil { + m = new(Map) + } + + m.seed = uintptr(rand()) + + if hint <= abi.MapGroupSlots { + // A small map can fill all 8 slots, so no need to increase + // target capacity. + // + // In fact, since an 8 slot group is what the first assignment + // to an empty map would allocate anyway, it doesn't matter if + // we allocate here or on the first assignment. + // + // Thus we just return without allocating. (We'll save the + // allocation completely if no assignment comes.) + + // Note that the compiler may have initialized m.dirPtr with a + // pointer to a stack-allocated group, in which case we already + // have a group. The control word is already initialized. + + return m + } + + // Full size map. + + // Set initial capacity to hold hint entries without growing in the + // average case. + targetCapacity := (hint * abi.MapGroupSlots) / maxAvgGroupLoad + if targetCapacity < hint { // overflow + return m // return an empty map. + } + + dirSize := (uint64(targetCapacity) + maxTableCapacity - 1) / maxTableCapacity + dirSize, overflow := alignUpPow2(dirSize) + if overflow || dirSize > uint64(math.MaxUintptr) { + return m // return an empty map. + } + + // Reject hints that are obviously too large. + groups, overflow := math.MulUintptr(uintptr(dirSize), maxTableCapacity) + if overflow { + return m // return an empty map. + } else { + mem, overflow := math.MulUintptr(groups, mt.GroupSize) + if overflow || mem > maxAlloc { + return m // return an empty map. + } + } + + m.globalDepth = uint8(sys.TrailingZeros64(dirSize)) + m.globalShift = depthToShift(m.globalDepth) + + directory := make([]*table, dirSize) + + for i := range directory { + // TODO: Think more about initial table capacity. + directory[i] = newTable(mt, uint64(targetCapacity)/dirSize, i, m.globalDepth) + } + + m.dirPtr = unsafe.Pointer(&directory[0]) + m.dirLen = len(directory) + + return m +} + +func NewEmptyMap() *Map { + m := new(Map) + m.seed = uintptr(rand()) + // See comment in NewMap. No need to eager allocate a group. + return m +} + +func (m *Map) directoryIndex(hash uintptr) uintptr { + if m.dirLen == 1 { + return 0 + } + return hash >> (m.globalShift & 63) +} + +func (m *Map) directoryAt(i uintptr) *table { + return *(**table)(unsafe.Pointer(uintptr(m.dirPtr) + goarch.PtrSize*i)) +} + +func (m *Map) directorySet(i uintptr, nt *table) { + *(**table)(unsafe.Pointer(uintptr(m.dirPtr) + goarch.PtrSize*i)) = nt +} + +func (m *Map) replaceTable(nt *table) { + // The number of entries that reference the same table doubles for each + // time the globalDepth grows without the table splitting. + entries := 1 << (m.globalDepth - nt.localDepth) + for i := 0; i < entries; i++ { + //m.directory[nt.index+i] = nt + m.directorySet(uintptr(nt.index+i), nt) + } +} + +func (m *Map) installTableSplit(old, left, right *table) { + if old.localDepth == m.globalDepth { + // No room for another level in the directory. Grow the + // directory. + newDir := make([]*table, m.dirLen*2) + for i := range m.dirLen { + t := m.directoryAt(uintptr(i)) + newDir[2*i] = t + newDir[2*i+1] = t + // t may already exist in multiple indices. We should + // only update t.index once. Since the index must + // increase, seeing the original index means this must + // be the first time we've encountered this table. + if t.index == i { + t.index = 2 * i + } + } + m.globalDepth++ + m.globalShift-- + //m.directory = newDir + m.dirPtr = unsafe.Pointer(&newDir[0]) + m.dirLen = len(newDir) + } + + // N.B. left and right may still consume multiple indices if the + // directory has grown multiple times since old was last split. + left.index = old.index + m.replaceTable(left) + + entries := 1 << (m.globalDepth - left.localDepth) + right.index = left.index + entries + m.replaceTable(right) +} + +func (m *Map) Used() uint64 { + return m.used +} + +// Get performs a lookup of the key that key points to. It returns a pointer to +// the element, or false if the key doesn't exist. +func (m *Map) Get(typ *abi.MapType, key unsafe.Pointer) (unsafe.Pointer, bool) { + return m.getWithoutKey(typ, key) +} + +func (m *Map) getWithKey(typ *abi.MapType, key unsafe.Pointer) (unsafe.Pointer, unsafe.Pointer, bool) { + if m.Used() == 0 { + return nil, nil, false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + } + + hash := typ.Hasher(key, m.seed) + + if m.dirLen == 0 { + return m.getWithKeySmall(typ, hash, key) + } + + idx := m.directoryIndex(hash) + return m.directoryAt(idx).getWithKey(typ, hash, key) +} + +func (m *Map) getWithoutKey(typ *abi.MapType, key unsafe.Pointer) (unsafe.Pointer, bool) { + if m.Used() == 0 { + return nil, false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + } + + hash := typ.Hasher(key, m.seed) + + if m.dirLen == 0 { + _, elem, ok := m.getWithKeySmall(typ, hash, key) + return elem, ok + } + + idx := m.directoryIndex(hash) + return m.directoryAt(idx).getWithoutKey(typ, hash, key) +} + +func (m *Map) getWithKeySmall(typ *abi.MapType, hash uintptr, key unsafe.Pointer) (unsafe.Pointer, unsafe.Pointer, bool) { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + + if typ.Key.Equal(key, slotKey) { + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + return slotKey, slotElem, true + } + + match = match.removeFirst() + } + + // No match here means key is not in the map. + // (A single group means no need to probe or check for empty). + return nil, nil, false +} + +func (m *Map) Put(typ *abi.MapType, key, elem unsafe.Pointer) { + slotElem := m.PutSlot(typ, key) + typedmemmove(typ.Elem, slotElem, elem) +} + +// PutSlot returns a pointer to the element slot where an inserted element +// should be written. +// +// PutSlot never returns nil. +func (m *Map) PutSlot(typ *abi.MapType, key unsafe.Pointer) unsafe.Pointer { + if m.writing != 0 { + fatal("concurrent map writes") + } + + hash := typ.Hasher(key, m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.MapGroupSlots { + elem := m.putSlotSmall(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + // + // TODO(prattmic): If this is an update to an existing key then + // we actually don't need to grow. + m.growToTable(typ) + } + + for { + idx := m.directoryIndex(hash) + elem, ok := m.directoryAt(idx).PutSlot(typ, m, hash, key) + if !ok { + continue + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } +} + +func (m *Map) putSlotSmall(typ *abi.MapType, hash uintptr, key unsafe.Pointer) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + if typ.NeedKeyUpdate() { + typedmemmove(typ.Key, slotKey, key) + } + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + + return slotElem + } + match = match.removeFirst() + } + + // There can't be deleted slots, small maps can't have them + // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit + // more efficient than matchEmpty. + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + fatal("small map with no empty slot (concurrent map writes?)") + return nil + } + + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + kmem := newobject(typ.Key) + *(*unsafe.Pointer)(slotKey) = kmem + slotKey = kmem + } + typedmemmove(typ.Key, slotKey, key) + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + emem := newobject(typ.Elem) + *(*unsafe.Pointer)(slotElem) = emem + slotElem = emem + } + + g.ctrls().set(i, ctrl(h2(hash))) + m.used++ + + return slotElem +} + +func (m *Map) growToSmall(typ *abi.MapType) { + grp := newGroups(typ, 1) + m.dirPtr = grp.data + + g := groupReference{ + data: m.dirPtr, + } + g.ctrls().setEmpty() +} + +func (m *Map) growToTable(typ *abi.MapType) { + tab := newTable(typ, 2*abi.MapGroupSlots, 0, 0) + + g := groupReference{ + data: m.dirPtr, + } + + for i := uintptr(0); i < abi.MapGroupSlots; i++ { + if (g.ctrls().get(i) & ctrlEmpty) == ctrlEmpty { + // Empty + continue + } + + key := g.key(typ, i) + if typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + elem := g.elem(typ, i) + if typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + + hash := typ.Hasher(key, m.seed) + + tab.uncheckedPutSlot(typ, hash, key, elem) + } + + directory := make([]*table, 1) + + directory[0] = tab + + m.dirPtr = unsafe.Pointer(&directory[0]) + m.dirLen = len(directory) + + m.globalDepth = 0 + m.globalShift = depthToShift(m.globalDepth) +} + +func (m *Map) Delete(typ *abi.MapType, key unsafe.Pointer) { + if m == nil || m.Used() == 0 { + if err := mapKeyError(typ, key); err != nil { + panic(err) // see issue 23734 + } + return + } + + if m.writing != 0 { + fatal("concurrent map writes") + } + + hash := typ.Hasher(key, m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirLen == 0 { + m.deleteSmall(typ, hash, key) + } else { + idx := m.directoryIndex(hash) + if m.directoryAt(idx).Delete(typ, m, hash, key) { + m.tombstonePossible = true + } + } + + if m.used == 0 { + // Reset the hash seed to make it more difficult for attackers + // to repeatedly trigger hash collisions. See + // https://go.dev/issue/25237. + m.seed = uintptr(rand()) + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 +} + +func (m *Map) deleteSmall(typ *abi.MapType, hash uintptr, key unsafe.Pointer) { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + for match != 0 { + i := match.first() + slotKey := g.key(typ, i) + origSlotKey := slotKey + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + m.used-- + + if typ.IndirectKey() { + // Clearing the pointer is sufficient. + *(*unsafe.Pointer)(origSlotKey) = nil + } else if typ.Key.Pointers() { + // Only bother clearing if there are pointers. + typedmemclr(typ.Key, slotKey) + } + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + // Clearing the pointer is sufficient. + *(*unsafe.Pointer)(slotElem) = nil + } else { + // Unlike keys, always clear the elem (even if + // it contains no pointers), as compound + // assignment operations depend on cleared + // deleted values. See + // https://go.dev/issue/25936. + typedmemclr(typ.Elem, slotElem) + } + + // We only have 1 group, so it is OK to immediately + // reuse deleted slots. + g.ctrls().set(i, ctrlEmpty) + return + } + match = match.removeFirst() + } +} + +// Clear deletes all entries from the map resulting in an empty map. +func (m *Map) Clear(typ *abi.MapType) { + if m == nil || m.Used() == 0 && !m.tombstonePossible { + return + } + + if m.writing != 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 // toggle, see comment on writing + + if m.dirLen == 0 { + m.clearSmall(typ) + } else { + var lastTab *table + for i := range m.dirLen { + t := m.directoryAt(uintptr(i)) + if t == lastTab { + continue + } + t.Clear(typ) + lastTab = t + } + m.used = 0 + m.tombstonePossible = false + // TODO: shrink directory? + } + m.clearSeq++ + + // Reset the hash seed to make it more difficult for attackers to + // repeatedly trigger hash collisions. See https://go.dev/issue/25237. + m.seed = uintptr(rand()) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 +} + +func (m *Map) clearSmall(typ *abi.MapType) { + g := groupReference{ + data: m.dirPtr, + } + + typedmemclr(typ.Group, g.data) + g.ctrls().setEmpty() + + m.used = 0 +} + +func (m *Map) Clone(typ *abi.MapType) *Map { + // Note: this should never be called with a nil map. + if m.writing != 0 { + fatal("concurrent map clone and map write") + } + + // Shallow copy the Map structure. + m2 := new(Map) + *m2 = *m + m = m2 + + // We need to just deep copy the dirPtr field. + if m.dirPtr == nil { + // delayed group allocation, nothing to do. + } else if m.dirLen == 0 { + // Clone one group. + oldGroup := groupReference{data: m.dirPtr} + newGroup := groupReference{data: newGroups(typ, 1).data} + cloneGroup(typ, newGroup, oldGroup) + m.dirPtr = newGroup.data + } else { + // Clone each (different) table. + oldDir := unsafe.Slice((**table)(m.dirPtr), m.dirLen) + newDir := make([]*table, m.dirLen) + for i, t := range oldDir { + if i > 0 && t == oldDir[i-1] { + newDir[i] = newDir[i-1] + continue + } + newDir[i] = t.clone(typ) + } + m.dirPtr = unsafe.Pointer(&newDir[0]) + } + + return m +} + +func mapKeyError(t *abi.MapType, p unsafe.Pointer) error { + if !t.HashMightPanic() { + return nil + } + return mapKeyError2(t.Key, p) +} + +func mapKeyError2(t *abi.Type, p unsafe.Pointer) error { + if t.TFlag&abi.TFlagRegularMemory != 0 { + return nil + } + switch t.Kind() { + case abi.Float32, abi.Float64, abi.Complex64, abi.Complex128, abi.String: + return nil + case abi.Interface: + i := (*abi.InterfaceType)(unsafe.Pointer(t)) + var t *abi.Type + var pdata *unsafe.Pointer + if len(i.Methods) == 0 { + a := (*abi.EmptyInterface)(p) + t = a.Type + if t == nil { + return nil + } + pdata = &a.Data + } else { + a := (*abi.NonEmptyInterface)(p) + if a.ITab == nil { + return nil + } + t = a.ITab.Type + pdata = &a.Data + } + + if t.Equal == nil { + return unhashableTypeError{t} + } + + if t.IsDirectIface() { + return mapKeyError2(t, unsafe.Pointer(pdata)) + } else { + return mapKeyError2(t, *pdata) + } + case abi.Array: + a := (*abi.ArrayType)(unsafe.Pointer(t)) + for i := uintptr(0); i < a.Len; i++ { + if err := mapKeyError2(a.Elem, unsafe.Pointer(uintptr(p)+i*a.Elem.Size_)); err != nil { + return err + } + } + return nil + case abi.Struct: + s := (*abi.StructType)(unsafe.Pointer(t)) + for _, f := range s.Fields { + if f.Name.IsBlank() { + continue + } + if err := mapKeyError2(f.Typ, unsafe.Pointer(uintptr(p)+f.Offset)); err != nil { + return err + } + } + return nil + default: + // Should never happen, keep this case for robustness. + return unhashableTypeError{t} + } +} + +type unhashableTypeError struct{ typ *abi.Type } + +func (unhashableTypeError) RuntimeError() {} + +func (e unhashableTypeError) Error() string { return "hash of unhashable type: " + typeString(e.typ) } + +// Pushed from runtime +// +//go:linkname typeString +func typeString(typ *abi.Type) string diff --git a/src/internal/runtime/maps/map_test.go b/src/internal/runtime/maps/map_test.go new file mode 100644 index 00000000000000..c8ef25af9a946c --- /dev/null +++ b/src/internal/runtime/maps/map_test.go @@ -0,0 +1,950 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps_test + +import ( + "fmt" + "internal/abi" + "internal/runtime/maps" + "math" + "testing" + "unsafe" +) + +func TestCtrlSize(t *testing.T) { + cs := unsafe.Sizeof(maps.CtrlGroup(0)) + if cs != abi.MapGroupSlots { + t.Errorf("ctrlGroup size got %d want abi.MapGroupSlots %d", cs, abi.MapGroupSlots) + } +} + +func TestMapPut(t *testing.T) { + m, typ := maps.NewTestMap[uint32, uint64](8) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + if m.Used() != 31 { + t.Errorf("Used() used got %d want 31", m.Used()) + } + + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + got, ok := m.Get(typ, unsafe.Pointer(&key)) + if !ok { + t.Errorf("Get(%d) got ok false want true", key) + } + gotElem := *(*uint64)(got) + if gotElem != elem { + t.Errorf("Get(%d) got elem %d want %d", key, gotElem, elem) + } + } +} + +// Grow enough to cause a table split. +func TestMapSplit(t *testing.T) { + m, typ := maps.NewTestMap[uint32, uint64](0) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 2*maps.MaxTableCapacity; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + if m.Used() != 2*maps.MaxTableCapacity { + t.Errorf("Used() used got %d want 31", m.Used()) + } + + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 2*maps.MaxTableCapacity; i++ { + key += 1 + elem += 1 + got, ok := m.Get(typ, unsafe.Pointer(&key)) + if !ok { + t.Errorf("Get(%d) got ok false want true", key) + } + gotElem := *(*uint64)(got) + if gotElem != elem { + t.Errorf("Get(%d) got elem %d want %d", key, gotElem, elem) + } + } +} + +func TestMapDelete(t *testing.T) { + m, typ := maps.NewTestMap[uint32, uint64](32) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + m.Delete(typ, unsafe.Pointer(&key)) + } + + if m.Used() != 0 { + t.Errorf("Used() used got %d want 0", m.Used()) + } + + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + _, ok := m.Get(typ, unsafe.Pointer(&key)) + if ok { + t.Errorf("Get(%d) got ok true want false", key) + } + } +} + +func TestTableClear(t *testing.T) { + m, typ := maps.NewTestMap[uint32, uint64](32) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + m.Clear(typ) + + if m.Used() != 0 { + t.Errorf("Clear() used got %d want 0", m.Used()) + } + + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + _, ok := m.Get(typ, unsafe.Pointer(&key)) + if ok { + t.Errorf("Get(%d) got ok true want false", key) + } + } +} + +// +0.0 and -0.0 compare equal, but we must still must update the key slot when +// overwriting. +func TestTableKeyUpdate(t *testing.T) { + m, typ := maps.NewTestMap[float64, uint64](8) + + zero := float64(0.0) + negZero := math.Copysign(zero, -1.0) + elem := uint64(0) + + m.Put(typ, unsafe.Pointer(&zero), unsafe.Pointer(&elem)) + if maps.DebugLog { + fmt.Printf("After put %f: %v\n", zero, m) + } + + elem = 1 + m.Put(typ, unsafe.Pointer(&negZero), unsafe.Pointer(&elem)) + if maps.DebugLog { + fmt.Printf("After put %f: %v\n", negZero, m) + } + + if m.Used() != 1 { + t.Errorf("Used() used got %d want 1", m.Used()) + } + + it := new(maps.Iter) + it.Init(typ, m) + it.Next() + keyPtr, elemPtr := it.Key(), it.Elem() + if keyPtr == nil { + t.Fatal("it.Key() got nil want key") + } + + key := *(*float64)(keyPtr) + elem = *(*uint64)(elemPtr) + if math.Copysign(1.0, key) > 0 { + t.Errorf("map key %f has positive sign", key) + } + if elem != 1 { + t.Errorf("map elem got %d want 1", elem) + } +} + +// Put should reuse a deleted slot rather than consuming an empty slot. +func TestTablePutDelete(t *testing.T) { + // Put will reuse the first deleted slot it encounters. + // + // This is awkward to test because Delete will only install ctrlDeleted + // if the group is full, otherwise it goes straight to empty. + // + // So first we must add to the table continuously until we happen to + // fill a group. + + // Avoid small maps, they have no tables. + m, typ := maps.NewTestMap[uint32, uint32](16) + + key := uint32(0) + elem := uint32(256 + 0) + + for { + key += 1 + elem += 1 + + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + // Normally a Put that fills a group would fill it with the + // inserted key, so why search the whole map for a potentially + // different key in a full group? + // + // Put may grow/split a table. Initial construction of the new + // table(s) could result in a full group consisting of + // arbitrary keys. + fullKeyPtr := m.KeyFromFullGroup(typ) + if fullKeyPtr != nil { + // Found a full group. + key = *(*uint32)(fullKeyPtr) + elem = 256 + key + break + } + } + + // Key is in a full group. Deleting it will result in a ctrlDeleted + // slot. + m.Delete(typ, unsafe.Pointer(&key)) + + // Re-insert key. This should reuse the deleted slot rather than + // consuming space. + tabWant := m.TableFor(typ, unsafe.Pointer(&key)) + growthLeftWant := tabWant.GrowthLeft() + + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + tabGot := m.TableFor(typ, unsafe.Pointer(&key)) + growthLeftGot := tabGot.GrowthLeft() + + if tabGot != tabWant { + // There shouldn't be a grow, as replacing a deleted slot + // doesn't require more space. + t.Errorf("Put(%d) grew table got %v want %v map %v", key, tabGot, tabWant, m) + } + + if growthLeftGot != growthLeftWant { + t.Errorf("GrowthLeft got %d want %d: map %v tab %v", growthLeftGot, growthLeftWant, m, tabGot) + } +} + +func TestTableIteration(t *testing.T) { + m, typ := maps.NewTestMap[uint32, uint64](8) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + got := make(map[uint32]uint64) + + it := new(maps.Iter) + it.Init(typ, m) + for { + it.Next() + keyPtr, elemPtr := it.Key(), it.Elem() + if keyPtr == nil { + break + } + + key := *(*uint32)(keyPtr) + elem := *(*uint64)(elemPtr) + got[key] = elem + } + + if len(got) != 31 { + t.Errorf("Iteration got %d entries, want 31: %+v", len(got), got) + } + + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + gotElem, ok := got[key] + if !ok { + t.Errorf("Iteration missing key %d", key) + continue + } + if gotElem != elem { + t.Errorf("Iteration key %d got elem %d want %d", key, gotElem, elem) + } + } +} + +// Deleted keys shouldn't be visible in iteration. +func TestTableIterationDelete(t *testing.T) { + m, typ := maps.NewTestMap[uint32, uint64](8) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + got := make(map[uint32]uint64) + first := true + deletedKey := uint32(1) + it := new(maps.Iter) + it.Init(typ, m) + for { + it.Next() + keyPtr, elemPtr := it.Key(), it.Elem() + if keyPtr == nil { + break + } + + key := *(*uint32)(keyPtr) + elem := *(*uint64)(elemPtr) + got[key] = elem + + if first { + first = false + + // If the key we intended to delete was the one we just + // saw, pick another to delete. + if key == deletedKey { + deletedKey++ + } + m.Delete(typ, unsafe.Pointer(&deletedKey)) + } + } + + if len(got) != 30 { + t.Errorf("Iteration got %d entries, want 30: %+v", len(got), got) + } + + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + + wantOK := true + if key == deletedKey { + wantOK = false + } + + gotElem, gotOK := got[key] + if gotOK != wantOK { + t.Errorf("Iteration key %d got ok %v want ok %v", key, gotOK, wantOK) + continue + } + if wantOK && gotElem != elem { + t.Errorf("Iteration key %d got elem %d want %d", key, gotElem, elem) + } + } +} + +// Deleted keys shouldn't be visible in iteration even after a grow. +func TestTableIterationGrowDelete(t *testing.T) { + m, typ := maps.NewTestMap[uint32, uint64](8) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + got := make(map[uint32]uint64) + first := true + deletedKey := uint32(1) + it := new(maps.Iter) + it.Init(typ, m) + for { + it.Next() + keyPtr, elemPtr := it.Key(), it.Elem() + if keyPtr == nil { + break + } + + key := *(*uint32)(keyPtr) + elem := *(*uint64)(elemPtr) + got[key] = elem + + if first { + first = false + + // If the key we intended to delete was the one we just + // saw, pick another to delete. + if key == deletedKey { + deletedKey++ + } + + // Double the number of elements to force a grow. + key := uint32(32) + elem := uint64(256 + 32) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + // Then delete from the grown map. + m.Delete(typ, unsafe.Pointer(&deletedKey)) + } + } + + // Don't check length: the number of new elements we'll see is + // unspecified. + + // Check values only of the original pre-iteration entries. + key = uint32(0) + elem = uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + + wantOK := true + if key == deletedKey { + wantOK = false + } + + gotElem, gotOK := got[key] + if gotOK != wantOK { + t.Errorf("Iteration key %d got ok %v want ok %v", key, gotOK, wantOK) + continue + } + if wantOK && gotElem != elem { + t.Errorf("Iteration key %d got elem %d want %d", key, gotElem, elem) + } + } +} + +func testTableIterationGrowDuplicate(t *testing.T, grow int) { + m, typ := maps.NewTestMap[uint32, uint64](8) + + key := uint32(0) + elem := uint64(256 + 0) + + for i := 0; i < 31; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + + got := make(map[uint32]uint64) + it := new(maps.Iter) + it.Init(typ, m) + for i := 0; ; i++ { + it.Next() + keyPtr, elemPtr := it.Key(), it.Elem() + if keyPtr == nil { + break + } + + key := *(*uint32)(keyPtr) + elem := *(*uint64)(elemPtr) + if elem != 256+uint64(key) { + t.Errorf("iteration got key %d elem %d want elem %d", key, elem, 256+uint64(key)) + } + if _, ok := got[key]; ok { + t.Errorf("iteration got key %d more than once", key) + } + got[key] = elem + + // Grow halfway through iteration. + if i == 16 { + key := uint32(32) + elem := uint64(256 + 32) + + for i := 0; i < grow; i++ { + key += 1 + elem += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + } + } + } + + // Don't check length: the number of new elements we'll see is + // unspecified. +} + +// Grow should not allow duplicate keys to appear. +func TestTableIterationGrowDuplicate(t *testing.T) { + // Small grow, only enough to cause table grow. + t.Run("grow", func(t *testing.T) { testTableIterationGrowDuplicate(t, 32) }) + + // Large grow, to cause table split. + t.Run("split", func(t *testing.T) { testTableIterationGrowDuplicate(t, 2*maps.MaxTableCapacity) }) +} + +func TestAlignUpPow2(t *testing.T) { + tests := []struct { + in uint64 + want uint64 + overflow bool + }{ + { + in: 0, + want: 0, + }, + { + in: 3, + want: 4, + }, + { + in: 4, + want: 4, + }, + { + in: 1 << 63, + want: 1 << 63, + }, + { + in: (1 << 63) - 1, + want: 1 << 63, + }, + { + in: (1 << 63) + 1, + overflow: true, + }, + } + + for _, tc := range tests { + got, overflow := maps.AlignUpPow2(tc.in) + if got != tc.want { + t.Errorf("alignUpPow2(%d) got %d, want %d", tc.in, got, tc.want) + } + if overflow != tc.overflow { + t.Errorf("alignUpPow2(%d) got overflow %v, want %v", tc.in, overflow, tc.overflow) + } + } +} + +// Verify that a map with zero-size slot is safe to use. +func TestMapZeroSizeSlot(t *testing.T) { + m, typ := maps.NewTestMap[struct{}, struct{}](16) + + key := struct{}{} + elem := struct{}{} + + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + + got, ok := m.Get(typ, unsafe.Pointer(&key)) + if !ok { + t.Errorf("Get(%d) got ok false want true", key) + } + gotElem := *(*struct{})(got) + if gotElem != elem { + t.Errorf("Get(%d) got elem %d want %d", key, gotElem, elem) + } + + tab := m.TableFor(typ, unsafe.Pointer(&key)) + start := tab.GroupsStart() + length := tab.GroupsLength() + end := unsafe.Pointer(uintptr(start) + length*typ.GroupSize - 1) // inclusive to ensure we have a valid pointer + if uintptr(got) < uintptr(start) || uintptr(got) > uintptr(end) { + t.Errorf("elem address outside groups allocation; got %p want [%p, %p]", got, start, end) + } +} + +func TestMapIndirect(t *testing.T) { + type big [abi.MapMaxKeyBytes + abi.MapMaxElemBytes]byte + + m, typ := maps.NewTestMap[big, big](8) + + key := big{} + elem := big{} + elem[0] = 128 + + for i := 0; i < 31; i++ { + key[0] += 1 + elem[0] += 1 + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %v: %v\n", key, m) + } + } + + if m.Used() != 31 { + t.Errorf("Used() used got %d want 31", m.Used()) + } + + key = big{} + elem = big{} + elem[0] = 128 + + for i := 0; i < 31; i++ { + key[0] += 1 + elem[0] += 1 + got, ok := m.Get(typ, unsafe.Pointer(&key)) + if !ok { + t.Errorf("Get(%v) got ok false want true", key) + } + gotElem := *(*big)(got) + if gotElem != elem { + t.Errorf("Get(%v) got elem %v want %v", key, gotElem, elem) + } + } +} + +// Delete should clear element. See https://go.dev/issue/25936. +func TestMapDeleteClear(t *testing.T) { + m, typ := maps.NewTestMap[int64, int64](8) + + key := int64(0) + elem := int64(128) + + m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem)) + + if maps.DebugLog { + fmt.Printf("After put %d: %v\n", key, m) + } + + got, ok := m.Get(typ, unsafe.Pointer(&key)) + if !ok { + t.Errorf("Get(%d) got ok false want true", key) + } + gotElem := *(*int64)(got) + if gotElem != elem { + t.Errorf("Get(%d) got elem %d want %d", key, gotElem, elem) + } + + m.Delete(typ, unsafe.Pointer(&key)) + + gotElem = *(*int64)(got) + if gotElem != 0 { + t.Errorf("Delete(%d) failed to clear element. got %d want 0", key, gotElem) + } +} + +var alwaysFalse bool +var escapeSink any + +func escape[T any](x T) T { + if alwaysFalse { + escapeSink = x + } + return x +} + +const ( + belowMax = abi.MapGroupSlots * 3 / 2 // 1.5 * group max = 2 groups @ 75% + atMax = (2 * abi.MapGroupSlots * maps.MaxAvgGroupLoad) / abi.MapGroupSlots // 2 groups at 7/8 full. +) + +func TestTableGroupCount(t *testing.T) { + // Test that maps of different sizes have the right number of + // tables/groups. + + type mapCount struct { + tables int + groups uint64 + } + + type mapCase struct { + initialLit mapCount + initialHint mapCount + after mapCount + } + + var testCases = []struct { + n int // n is the number of map elements + escape mapCase // expected values for escaping map + }{ + { + n: -(1 << 30), + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{0, 0}, + after: mapCount{0, 0}, + }, + }, + { + n: -1, + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{0, 0}, + after: mapCount{0, 0}, + }, + }, + { + n: 0, + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{0, 0}, + after: mapCount{0, 0}, + }, + }, + { + n: 1, + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{0, 0}, + after: mapCount{0, 1}, + }, + }, + { + n: abi.MapGroupSlots, + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{0, 0}, + after: mapCount{0, 1}, + }, + }, + { + n: abi.MapGroupSlots + 1, + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{1, 2}, + after: mapCount{1, 2}, + }, + }, + { + n: belowMax, // 1.5 group max = 2 groups @ 75% + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{1, 2}, + after: mapCount{1, 2}, + }, + }, + { + n: atMax, // 2 groups at max + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{1, 2}, + after: mapCount{1, 2}, + }, + }, + { + n: atMax + 1, // 2 groups at max + 1 -> grow to 4 groups + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{1, 4}, + after: mapCount{1, 4}, + }, + }, + { + n: 2 * belowMax, // 3 * group max = 4 groups @75% + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{1, 4}, + after: mapCount{1, 4}, + }, + }, + { + n: 2*atMax + 1, // 4 groups at max + 1 -> grow to 8 groups + escape: mapCase{ + initialLit: mapCount{0, 0}, + initialHint: mapCount{1, 8}, + after: mapCount{1, 8}, + }, + }, + } + + testMap := func(t *testing.T, m map[int]int, n int, initial, after mapCount) { + mm := *(**maps.Map)(unsafe.Pointer(&m)) + + gotTab := mm.TableCount() + if gotTab != initial.tables { + t.Errorf("initial TableCount got %d want %d", gotTab, initial.tables) + } + + gotGroup := mm.GroupCount() + if gotGroup != initial.groups { + t.Errorf("initial GroupCount got %d want %d", gotGroup, initial.groups) + } + + for i := 0; i < n; i++ { + m[i] = i + } + + gotTab = mm.TableCount() + if gotTab != after.tables { + t.Errorf("after TableCount got %d want %d", gotTab, after.tables) + } + + gotGroup = mm.GroupCount() + if gotGroup != after.groups { + t.Errorf("after GroupCount got %d want %d", gotGroup, after.groups) + } + } + + t.Run("mapliteral", func(t *testing.T) { + for _, tc := range testCases { + t.Run(fmt.Sprintf("n=%d", tc.n), func(t *testing.T) { + t.Run("escape", func(t *testing.T) { + m := escape(map[int]int{}) + testMap(t, m, tc.n, tc.escape.initialLit, tc.escape.after) + }) + }) + } + }) + t.Run("nohint", func(t *testing.T) { + for _, tc := range testCases { + t.Run(fmt.Sprintf("n=%d", tc.n), func(t *testing.T) { + t.Run("escape", func(t *testing.T) { + m := escape(make(map[int]int)) + testMap(t, m, tc.n, tc.escape.initialLit, tc.escape.after) + }) + }) + } + }) + t.Run("makemap", func(t *testing.T) { + for _, tc := range testCases { + t.Run(fmt.Sprintf("n=%d", tc.n), func(t *testing.T) { + t.Run("escape", func(t *testing.T) { + m := escape(make(map[int]int, tc.n)) + testMap(t, m, tc.n, tc.escape.initialHint, tc.escape.after) + }) + }) + } + }) + t.Run("makemap64", func(t *testing.T) { + for _, tc := range testCases { + t.Run(fmt.Sprintf("n=%d", tc.n), func(t *testing.T) { + t.Run("escape", func(t *testing.T) { + m := escape(make(map[int]int, int64(tc.n))) + testMap(t, m, tc.n, tc.escape.initialHint, tc.escape.after) + }) + }) + } + }) +} + +func TestTombstoneGrow(t *testing.T) { + tableSizes := []int{16, 32, 64, 128, 256} + for _, tableSize := range tableSizes { + for _, load := range []string{"low", "mid", "high"} { + capacity := tableSize * 7 / 8 + var initialElems int + switch load { + case "low": + initialElems = capacity / 8 + case "mid": + initialElems = capacity / 2 + case "high": + initialElems = capacity + } + t.Run(fmt.Sprintf("tableSize=%d/elems=%d/load=%0.3f", tableSize, initialElems, float64(initialElems)/float64(tableSize)), func(t *testing.T) { + allocs := testing.AllocsPerRun(1, func() { + // Fill the map with elements. + m := make(map[int]int, capacity) + for i := range initialElems { + m[i] = i + } + + // This is the heart of our test. + // Loop over the map repeatedly, deleting a key then adding a not-yet-seen key + // while keeping the map at a ~constant number of elements (+/-1). + nextKey := initialElems + for range 100000 { + for k := range m { + delete(m, k) + break + } + m[nextKey] = nextKey + nextKey++ + if len(m) != initialElems { + t.Fatal("len(m) should remain constant") + } + } + }) + + // The make has 4 allocs (map, directory, table, groups). + // Each growth has 2 allocs (table, groups). + // We allow two growths if we start full, 1 otherwise. + // Fail (somewhat arbitrarily) if there are more than that. + allowed := float64(4 + 1*2) + if initialElems == capacity { + allowed += 2 + } + if allocs > allowed { + t.Fatalf("got %v allocations, allowed %v", allocs, allowed) + } + }) + } + } +} diff --git a/src/internal/runtime/maps/runtime.go b/src/internal/runtime/maps/runtime.go new file mode 100644 index 00000000000000..8bba23f07003bd --- /dev/null +++ b/src/internal/runtime/maps/runtime.go @@ -0,0 +1,371 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "internal/abi" + "internal/asan" + "internal/msan" + "internal/race" + "internal/runtime/sys" + "unsafe" +) + +// Functions below pushed from runtime. + +//go:linkname fatal +func fatal(s string) + +//go:linkname rand +func rand() uint64 + +//go:linkname typedmemmove +func typedmemmove(typ *abi.Type, dst, src unsafe.Pointer) + +//go:linkname typedmemclr +func typedmemclr(typ *abi.Type, ptr unsafe.Pointer) + +//go:linkname newarray +func newarray(typ *abi.Type, n int) unsafe.Pointer + +//go:linkname newobject +func newobject(typ *abi.Type) unsafe.Pointer + +// Pushed from runtime in order to use runtime.plainError +// +//go:linkname errNilAssign +var errNilAssign error + +// Pull from runtime. It is important that is this the exact same copy as the +// runtime because runtime.mapaccess1_fat compares the returned pointer with +// &runtime.zeroVal[0]. +// TODO: move zeroVal to internal/abi? +// +//go:linkname zeroVal runtime.zeroVal +var zeroVal [abi.ZeroValSize]byte + +// mapaccess1 returns a pointer to h[key]. Never returns nil, instead +// it will return a reference to the zero object for the elem type if +// the key is not in the map. +// NOTE: The returned pointer may keep the whole map live, so don't +// hold onto it for very long. +// +//go:linkname runtime_mapaccess1 runtime.mapaccess1 +func runtime_mapaccess1(typ *abi.MapType, m *Map, key unsafe.Pointer) unsafe.Pointer { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess1) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + race.ReadObjectPC(typ.Key, key, callerpc, pc) + } + if msan.Enabled && m != nil { + msan.Read(key, typ.Key.Size_) + } + if asan.Enabled && m != nil { + asan.Read(key, typ.Key.Size_) + } + + if m == nil || m.Used() == 0 { + if err := mapKeyError(typ, key); err != nil { + panic(err) // see issue 23734 + } + return unsafe.Pointer(&zeroVal[0]) + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + } + + hash := typ.Hasher(key, m.seed) + + if m.dirLen <= 0 { + _, elem, ok := m.getWithKeySmall(typ, hash, key) + if !ok { + return unsafe.Pointer(&zeroVal[0]) + } + return elem + } + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + slotKeyOrig := slotKey + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKeyOrig) + typ.ElemOff) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + return slotElem + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]) + } + } +} + +//go:linkname runtime_mapaccess2 runtime.mapaccess2 +func runtime_mapaccess2(typ *abi.MapType, m *Map, key unsafe.Pointer) (unsafe.Pointer, bool) { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess1) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + race.ReadObjectPC(typ.Key, key, callerpc, pc) + } + if msan.Enabled && m != nil { + msan.Read(key, typ.Key.Size_) + } + if asan.Enabled && m != nil { + asan.Read(key, typ.Key.Size_) + } + + if m == nil || m.Used() == 0 { + if err := mapKeyError(typ, key); err != nil { + panic(err) // see issue 23734 + } + return unsafe.Pointer(&zeroVal[0]), false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + } + + hash := typ.Hasher(key, m.seed) + + if m.dirLen == 0 { + _, elem, ok := m.getWithKeySmall(typ, hash, key) + if !ok { + return unsafe.Pointer(&zeroVal[0]), false + } + return elem, true + } + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + slotKeyOrig := slotKey + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKeyOrig) + typ.ElemOff) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + return slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]), false + } + } +} + +//go:linkname runtime_mapassign runtime.mapassign +func runtime_mapassign(typ *abi.MapType, m *Map, key unsafe.Pointer) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapassign) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + race.ReadObjectPC(typ.Key, key, callerpc, pc) + } + if msan.Enabled { + msan.Read(key, typ.Key.Size_) + } + if asan.Enabled { + asan.Read(key, typ.Key.Size_) + } + if m.writing != 0 { + fatal("concurrent map writes") + } + + hash := typ.Hasher(key, m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.MapGroupSlots { + elem := m.putSlotSmall(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot + // we find, which we'll use to insert the new entry if + // necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + slotKeyOrig := slotKey + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + if typ.NeedKeyUpdate() { + typedmemmove(typ.Key, slotKey, key) + } + + slotElem = unsafe.Pointer(uintptr(slotKeyOrig) + typ.ElemOff) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + + var i uintptr + + // If we found a deleted slot along the way, we + // can replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } else { + // Otherwise, use the empty slot. + i = match.first() + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + slotKeyOrig := slotKey + if typ.IndirectKey() { + kmem := newobject(typ.Key) + *(*unsafe.Pointer)(slotKey) = kmem + slotKey = kmem + } + typedmemmove(typ.Key, slotKey, key) + + slotElem = unsafe.Pointer(uintptr(slotKeyOrig) + typ.ElemOff) + if typ.IndirectElem() { + emem := newobject(typ.Elem) + *(*unsafe.Pointer)(slotElem) = emem + slotElem = emem + } + + g.ctrls().set(i, ctrl(h2Hash)) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + + // No empty slots in this group. Check for a deleted + // slot, which we'll use if we don't find a match later + // in the probe sequence. + // + // We only need to remember a single deleted slot. + if firstDeletedGroup.data == nil { + // Since we already checked for empty slots + // above, matches here must be deleted slots. + match = g.ctrls().matchEmptyOrDeleted() + if match != 0 { + firstDeletedGroup = g + firstDeletedSlot = match.first() + } + } + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} diff --git a/src/internal/runtime/maps/runtime_fast32.go b/src/internal/runtime/maps/runtime_fast32.go new file mode 100644 index 00000000000000..d5be04afd450c0 --- /dev/null +++ b/src/internal/runtime/maps/runtime_fast32.go @@ -0,0 +1,477 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "internal/abi" + "internal/race" + "internal/runtime/sys" + "unsafe" +) + +//go:linkname runtime_mapaccess1_fast32 runtime.mapaccess1_fast32 +func runtime_mapaccess1_fast32(typ *abi.MapType, m *Map, key uint32) unsafe.Pointer { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess1_fast32) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]) + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil + } + + if m.dirLen == 0 { + g := groupReference{ + data: m.dirPtr, + } + full := g.ctrls().matchFull() + slotKey := g.key(typ, 0) + slotSize := typ.SlotSize + for full != 0 { + if key == *(*uint32)(slotKey) && full.lowestSet() { + slotElem := unsafe.Pointer(uintptr(slotKey) + typ.ElemOff) + return slotElem + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + full = full.shiftOutLowest() + } + return unsafe.Pointer(&zeroVal[0]) + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint32)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + typ.ElemOff) + return slotElem + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]) + } + } +} + +//go:linkname runtime_mapaccess2_fast32 runtime.mapaccess2_fast32 +func runtime_mapaccess2_fast32(typ *abi.MapType, m *Map, key uint32) (unsafe.Pointer, bool) { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess2_fast32) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]), false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil, false + } + + if m.dirLen == 0 { + g := groupReference{ + data: m.dirPtr, + } + full := g.ctrls().matchFull() + slotKey := g.key(typ, 0) + slotSize := typ.SlotSize + for full != 0 { + if key == *(*uint32)(slotKey) && full.lowestSet() { + slotElem := unsafe.Pointer(uintptr(slotKey) + typ.ElemOff) + return slotElem, true + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + full = full.shiftOutLowest() + } + return unsafe.Pointer(&zeroVal[0]), false + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint32)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + typ.ElemOff) + return slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]), false + } + } +} + +func (m *Map) putSlotSmallFast32(typ *abi.MapType, hash uintptr, key uint32) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint32)(slotKey) { + slotElem := g.elem(typ, i) + return slotElem + } + match = match.removeFirst() + } + + // There can't be deleted slots, small maps can't have them + // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit + // more efficient than matchEmpty. + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + fatal("small map with no empty slot (concurrent map writes?)") + } + + i := match.first() + + slotKey := g.key(typ, i) + *(*uint32)(slotKey) = key + + slotElem := g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + m.used++ + + return slotElem +} + +//go:linkname runtime_mapassign_fast32 runtime.mapassign_fast32 +func runtime_mapassign_fast32(typ *abi.MapType, m *Map, key uint32) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapassign_fast32) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + if m.writing != 0 { + fatal("concurrent map writes") + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.MapGroupSlots { + elem := m.putSlotSmallFast32(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot + // we find, which we'll use to insert the new entry if + // necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint32)(slotKey) { + slotElem = g.elem(typ, i) + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If we have no space left, first try to remove some tombstones. + if t.growthLeft == 0 { + t.pruneTombstones(typ, m) + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + *(*uint32)(slotKey) = key + + slotElem = g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2Hash)) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} + +// Key is a 32-bit pointer (only called on 32-bit GOARCH). This source is identical to fast64ptr. +// +// TODO(prattmic): With some compiler refactoring we could avoid duplication of this function. +// +//go:linkname runtime_mapassign_fast32ptr runtime.mapassign_fast32ptr +func runtime_mapassign_fast32ptr(typ *abi.MapType, m *Map, key unsafe.Pointer) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapassign_fast32ptr) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + if m.writing != 0 { + fatal("concurrent map writes") + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.MapGroupSlots { + elem := m.putSlotSmallFastPtr(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot we + // find, which we'll use to insert the new entry if necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*unsafe.Pointer)(slotKey) { + slotElem = g.elem(typ, i) + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + *(*unsafe.Pointer)(slotKey) = key + + slotElem = g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2Hash)) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} + +//go:linkname runtime_mapdelete_fast32 runtime.mapdelete_fast32 +func runtime_mapdelete_fast32(typ *abi.MapType, m *Map, key uint32) { + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapdelete_fast32) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return + } + + m.Delete(typ, abi.NoEscape(unsafe.Pointer(&key))) +} diff --git a/src/internal/runtime/maps/runtime_fast64.go b/src/internal/runtime/maps/runtime_fast64.go new file mode 100644 index 00000000000000..2bee2d4be019b3 --- /dev/null +++ b/src/internal/runtime/maps/runtime_fast64.go @@ -0,0 +1,517 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "internal/abi" + "internal/race" + "internal/runtime/sys" + "unsafe" +) + +//go:linkname runtime_mapaccess1_fast64 runtime.mapaccess1_fast64 +func runtime_mapaccess1_fast64(typ *abi.MapType, m *Map, key uint64) unsafe.Pointer { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess1_fast64) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]) + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil + } + + if m.dirLen == 0 { + g := groupReference{ + data: m.dirPtr, + } + full := g.ctrls().matchFull() + slotKey := g.key(typ, 0) + slotSize := typ.SlotSize + for full != 0 { + if key == *(*uint64)(slotKey) && full.lowestSet() { + slotElem := unsafe.Pointer(uintptr(slotKey) + 8) + return slotElem + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + full = full.shiftOutLowest() + } + return unsafe.Pointer(&zeroVal[0]) + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint64)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + 8) + return slotElem + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]) + } + } +} + +//go:linkname runtime_mapaccess2_fast64 runtime.mapaccess2_fast64 +func runtime_mapaccess2_fast64(typ *abi.MapType, m *Map, key uint64) (unsafe.Pointer, bool) { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess2_fast64) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]), false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil, false + } + + if m.dirLen == 0 { + g := groupReference{ + data: m.dirPtr, + } + full := g.ctrls().matchFull() + slotKey := g.key(typ, 0) + slotSize := typ.SlotSize + for full != 0 { + if key == *(*uint64)(slotKey) && full.lowestSet() { + slotElem := unsafe.Pointer(uintptr(slotKey) + 8) + return slotElem, true + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + full = full.shiftOutLowest() + } + return unsafe.Pointer(&zeroVal[0]), false + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint64)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + 8) + return slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]), false + } + } +} + +func (m *Map) putSlotSmallFast64(typ *abi.MapType, hash uintptr, key uint64) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint64)(slotKey) { + slotElem := g.elem(typ, i) + return slotElem + } + match = match.removeFirst() + } + + // There can't be deleted slots, small maps can't have them + // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit + // more efficient than matchEmpty. + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + fatal("small map with no empty slot (concurrent map writes?)") + } + + i := match.first() + + slotKey := g.key(typ, i) + *(*uint64)(slotKey) = key + + slotElem := g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + m.used++ + + return slotElem +} + +//go:linkname runtime_mapassign_fast64 runtime.mapassign_fast64 +func runtime_mapassign_fast64(typ *abi.MapType, m *Map, key uint64) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapassign_fast64) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + if m.writing != 0 { + fatal("concurrent map writes") + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.MapGroupSlots { + elem := m.putSlotSmallFast64(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot + // we find, which we'll use to insert the new entry if + // necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*uint64)(slotKey) { + slotElem = g.elem(typ, i) + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If we have no space left, first try to remove some tombstones. + if t.growthLeft == 0 { + t.pruneTombstones(typ, m) + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + *(*uint64)(slotKey) = key + + slotElem = g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2Hash)) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} + +func (m *Map) putSlotSmallFastPtr(typ *abi.MapType, hash uintptr, key unsafe.Pointer) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*unsafe.Pointer)(slotKey) { + slotElem := g.elem(typ, i) + return slotElem + } + match = match.removeFirst() + } + + // There can't be deleted slots, small maps can't have them + // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit + // more efficient than matchEmpty. + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + fatal("small map with no empty slot (concurrent map writes?)") + } + + i := match.first() + + slotKey := g.key(typ, i) + *(*unsafe.Pointer)(slotKey) = key + + slotElem := g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + m.used++ + + return slotElem +} + +// Key is a 64-bit pointer (only called on 64-bit GOARCH). +// +//go:linkname runtime_mapassign_fast64ptr runtime.mapassign_fast64ptr +func runtime_mapassign_fast64ptr(typ *abi.MapType, m *Map, key unsafe.Pointer) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapassign_fast64ptr) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + if m.writing != 0 { + fatal("concurrent map writes") + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.MapGroupSlots { + elem := m.putSlotSmallFastPtr(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot + // we find, which we'll use to insert the new entry if + // necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*unsafe.Pointer)(slotKey) { + slotElem = g.elem(typ, i) + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + *(*unsafe.Pointer)(slotKey) = key + + slotElem = g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2Hash)) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} + +//go:linkname runtime_mapdelete_fast64 runtime.mapdelete_fast64 +func runtime_mapdelete_fast64(typ *abi.MapType, m *Map, key uint64) { + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapdelete_fast64) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return + } + + m.Delete(typ, abi.NoEscape(unsafe.Pointer(&key))) +} diff --git a/src/internal/runtime/maps/runtime_faststr.go b/src/internal/runtime/maps/runtime_faststr.go new file mode 100644 index 00000000000000..374468b66438a6 --- /dev/null +++ b/src/internal/runtime/maps/runtime_faststr.go @@ -0,0 +1,414 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "internal/abi" + "internal/goarch" + "internal/race" + "internal/runtime/sys" + "unsafe" +) + +func (m *Map) getWithoutKeySmallFastStr(typ *abi.MapType, key string) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + ctrls := *g.ctrls() + slotKey := g.key(typ, 0) + slotSize := typ.SlotSize + + // The 64 threshold was chosen based on performance of BenchmarkMapStringKeysEight, + // where there are 8 keys to check, all of which don't quick-match the lookup key. + // In that case, we can save hashing the lookup key. That savings is worth this extra code + // for strings that are long enough that hashing is expensive. + if len(key) > 64 { + // String hashing and equality might be expensive. Do a quick check first. + j := abi.MapGroupSlots + for i := range abi.MapGroupSlots { + if ctrls&(1<<7) == 0 && longStringQuickEqualityTest(key, *(*string)(slotKey)) { + if j < abi.MapGroupSlots { + // 2 strings both passed the quick equality test. + // Break out of this loop and do it the slow way. + goto dohash + } + j = i + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + ctrls >>= 8 + } + if j == abi.MapGroupSlots { + // No slot passed the quick test. + return nil + } + // There's exactly one slot that passed the quick test. Do the single expensive comparison. + slotKey = g.key(typ, uintptr(j)) + if key == *(*string)(slotKey) { + return unsafe.Pointer(uintptr(slotKey) + 2*goarch.PtrSize) + } + return nil + } + +dohash: + // This path will cost 1 hash and 1+ε comparisons. + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&key)), m.seed) + h2 := uint8(h2(hash)) + ctrls = *g.ctrls() + slotKey = g.key(typ, 0) + + for range abi.MapGroupSlots { + if uint8(ctrls) == h2 && key == *(*string)(slotKey) { + return unsafe.Pointer(uintptr(slotKey) + 2*goarch.PtrSize) + } + slotKey = unsafe.Pointer(uintptr(slotKey) + slotSize) + ctrls >>= 8 + } + return nil +} + +// Returns true if a and b might be equal. +// Returns false if a and b are definitely not equal. +// Requires len(a)>=8. +func longStringQuickEqualityTest(a, b string) bool { + if len(a) != len(b) { + return false + } + x, y := stringPtr(a), stringPtr(b) + // Check first 8 bytes. + if *(*[8]byte)(x) != *(*[8]byte)(y) { + return false + } + // Check last 8 bytes. + x = unsafe.Pointer(uintptr(x) + uintptr(len(a)) - 8) + y = unsafe.Pointer(uintptr(y) + uintptr(len(a)) - 8) + if *(*[8]byte)(x) != *(*[8]byte)(y) { + return false + } + return true +} +func stringPtr(s string) unsafe.Pointer { + type stringStruct struct { + ptr unsafe.Pointer + len int + } + return (*stringStruct)(unsafe.Pointer(&s)).ptr +} + +//go:linkname runtime_mapaccess1_faststr runtime.mapaccess1_faststr +func runtime_mapaccess1_faststr(typ *abi.MapType, m *Map, key string) unsafe.Pointer { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess1_faststr) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]) + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil + } + + if m.dirLen <= 0 { + elem := m.getWithoutKeySmallFastStr(typ, key) + if elem == nil { + return unsafe.Pointer(&zeroVal[0]) + } + return elem + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*string)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + 2*goarch.PtrSize) + return slotElem + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]) + } + } +} + +//go:linkname runtime_mapaccess2_faststr runtime.mapaccess2_faststr +func runtime_mapaccess2_faststr(typ *abi.MapType, m *Map, key string) (unsafe.Pointer, bool) { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapaccess2_faststr) + race.ReadPC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return unsafe.Pointer(&zeroVal[0]), false + } + + if m.writing != 0 { + fatal("concurrent map read and map write") + return nil, false + } + + if m.dirLen <= 0 { + elem := m.getWithoutKeySmallFastStr(typ, key) + if elem == nil { + return unsafe.Pointer(&zeroVal[0]), false + } + return elem, true + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + // Probe table. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*string)(slotKey) { + slotElem := unsafe.Pointer(uintptr(slotKey) + 2*goarch.PtrSize) + return slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return unsafe.Pointer(&zeroVal[0]), false + } + } +} + +func (m *Map) putSlotSmallFastStr(typ *abi.MapType, hash uintptr, key string) unsafe.Pointer { + g := groupReference{ + data: m.dirPtr, + } + + match := g.ctrls().matchH2(h2(hash)) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*string)(slotKey) { + // Key needs update, as the backing storage may differ. + *(*string)(slotKey) = key + slotElem := g.elem(typ, i) + return slotElem + } + match = match.removeFirst() + } + + // There can't be deleted slots, small maps can't have them + // (see deleteSmall). Use matchEmptyOrDeleted as it is a bit + // more efficient than matchEmpty. + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + fatal("small map with no empty slot (concurrent map writes?)") + } + + i := match.first() + + slotKey := g.key(typ, i) + *(*string)(slotKey) = key + + slotElem := g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2(hash))) + m.used++ + + return slotElem +} + +//go:linkname runtime_mapassign_faststr runtime.mapassign_faststr +func runtime_mapassign_faststr(typ *abi.MapType, m *Map, key string) unsafe.Pointer { + if m == nil { + panic(errNilAssign) + } + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapassign_faststr) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + if m.writing != 0 { + fatal("concurrent map writes") + } + + k := key + hash := typ.Hasher(abi.NoEscape(unsafe.Pointer(&k)), m.seed) + + // Set writing after calling Hasher, since Hasher may panic, in which + // case we have not actually done a write. + m.writing ^= 1 // toggle, see comment on writing + + if m.dirPtr == nil { + m.growToSmall(typ) + } + + if m.dirLen == 0 { + if m.used < abi.MapGroupSlots { + elem := m.putSlotSmallFastStr(typ, hash, key) + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return elem + } + + // Can't fit another entry, grow to full size map. + m.growToTable(typ) + } + + var slotElem unsafe.Pointer +outer: + for { + // Select table. + idx := m.directoryIndex(hash) + t := m.directoryAt(idx) + + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot + // we find, which we'll use to insert the new entry if + // necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if key == *(*string)(slotKey) { + // Key needs update, as the backing + // storage may differ. + *(*string)(slotKey) = key + slotElem = g.elem(typ, i) + + t.checkInvariants(typ, m) + break outer + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If we have no space left, first try to remove some tombstones. + if t.growthLeft == 0 { + t.pruneTombstones(typ, m) + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + *(*string)(slotKey) = key + + slotElem = g.elem(typ, i) + + g.ctrls().set(i, ctrl(h2Hash)) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + break outer + } + + t.rehash(typ, m) + continue outer + } + } + + if m.writing == 0 { + fatal("concurrent map writes") + } + m.writing ^= 1 + + return slotElem +} + +//go:linkname runtime_mapdelete_faststr runtime.mapdelete_faststr +func runtime_mapdelete_faststr(typ *abi.MapType, m *Map, key string) { + if race.Enabled { + callerpc := sys.GetCallerPC() + pc := abi.FuncPCABIInternal(runtime_mapdelete_faststr) + race.WritePC(unsafe.Pointer(m), callerpc, pc) + } + + if m == nil || m.Used() == 0 { + return + } + + m.Delete(typ, abi.NoEscape(unsafe.Pointer(&key))) +} diff --git a/src/internal/runtime/maps/table.go b/src/internal/runtime/maps/table.go new file mode 100644 index 00000000000000..fbce099655e79e --- /dev/null +++ b/src/internal/runtime/maps/table.go @@ -0,0 +1,1309 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "internal/abi" + "internal/runtime/math" + "unsafe" +) + +// Maximum size of a table before it is split at the directory level. +// +// TODO: Completely made up value. This should be tuned for performance vs grow +// latency. +// TODO: This should likely be based on byte size, as copying costs will +// dominate grow latency for large objects. +const maxTableCapacity = 1024 + +// Ensure the max capacity fits in uint16, used for capacity and growthLeft +// below. +var _ = uint16(maxTableCapacity) + +// table is a Swiss table hash table structure. +// +// Each table is a complete hash table implementation. +// +// Map uses one or more tables to store entries. Extendible hashing (hash +// prefix) is used to select the table to use for a specific key. Using +// multiple tables enables incremental growth by growing only one table at a +// time. +type table struct { + // The number of filled slots (i.e. the number of elements in the table). + used uint16 + + // The total number of slots (always 2^N). Equal to + // `(groups.lengthMask+1)*abi.MapGroupSlots`. + capacity uint16 + + // The number of slots we can still fill without needing to rehash. + // + // We rehash when used + tombstones > loadFactor*capacity, including + // tombstones so the table doesn't overfill with tombstones. This field + // counts down remaining empty slots before the next rehash. + growthLeft uint16 + + // The number of bits used by directory lookups above this table. Note + // that this may be less then globalDepth, if the directory has grown + // but this table has not yet been split. + localDepth uint8 + + // Index of this table in the Map directory. This is the index of the + // _first_ location in the directory. The table may occur in multiple + // sequential indices. + // + // index is -1 if the table is stale (no longer installed in the + // directory). + index int + + // groups is an array of slot groups. Each group holds abi.MapGroupSlots + // key/elem slots and their control bytes. A table has a fixed size + // groups array. The table is replaced (in rehash) when more space is + // required. + // + // TODO(prattmic): keys and elements are interleaved to maximize + // locality, but it comes at the expense of wasted space for some types + // (consider uint8 key, uint64 element). Consider placing all keys + // together in these cases to save space. + groups groupsReference +} + +func newTable(typ *abi.MapType, capacity uint64, index int, localDepth uint8) *table { + if capacity < abi.MapGroupSlots { + capacity = abi.MapGroupSlots + } + + t := &table{ + index: index, + localDepth: localDepth, + } + + if capacity > maxTableCapacity { + panic("initial table capacity too large") + } + + // N.B. group count must be a power of two for probeSeq to visit every + // group. + capacity, overflow := alignUpPow2(capacity) + if overflow { + panic("rounded-up capacity overflows uint64") + } + + t.reset(typ, uint16(capacity)) + + return t +} + +// reset resets the table with new, empty groups with the specified new total +// capacity. +func (t *table) reset(typ *abi.MapType, capacity uint16) { + groupCount := uint64(capacity) / abi.MapGroupSlots + t.groups = newGroups(typ, groupCount) + t.capacity = capacity + t.growthLeft = t.maxGrowthLeft() + + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + g.ctrls().setEmpty() + } +} + +// maxGrowthLeft is the number of inserts we can do before +// resizing, starting from an empty table. +func (t *table) maxGrowthLeft() uint16 { + if t.capacity == 0 { + // No real reason to support zero capacity table, since an + // empty Map simply won't have a table. + panic("table must have positive capacity") + } else if t.capacity <= abi.MapGroupSlots { + // If the map fits in a single group then we're able to fill all of + // the slots except 1 (an empty slot is needed to terminate find + // operations). + // + // TODO(go.dev/issue/54766): With a special case in probing for + // single-group tables, we could fill all slots. + return t.capacity - 1 + } else { + if t.capacity > math.MaxUint16/maxAvgGroupLoad { + panic("overflow") + } + return (t.capacity * maxAvgGroupLoad) / abi.MapGroupSlots + } + +} + +func (t *table) Used() uint64 { + return uint64(t.used) +} + +// Get performs a lookup of the key that key points to. It returns a pointer to +// the element, or false if the key doesn't exist. +func (t *table) Get(typ *abi.MapType, m *Map, key unsafe.Pointer) (unsafe.Pointer, bool) { + // TODO(prattmic): We could avoid hashing in a variety of special + // cases. + // + // - One entry maps could just directly compare the single entry + // without hashing. + // - String keys could do quick checks of a few bytes before hashing. + hash := typ.Hasher(key, m.seed) + _, elem, ok := t.getWithKey(typ, hash, key) + return elem, ok +} + +// getWithKey performs a lookup of key, returning a pointer to the version of +// the key in the map in addition to the element. +// +// This is relevant when multiple different key values compare equal (e.g., +// +0.0 and -0.0). When a grow occurs during iteration, iteration perform a +// lookup of keys from the old group in the new group in order to correctly +// expose updated elements. For NeedsKeyUpdate keys, iteration also must return +// the new key value, not the old key value. +// hash must be the hash of the key. +func (t *table) getWithKey(typ *abi.MapType, hash uintptr, key unsafe.Pointer) (unsafe.Pointer, unsafe.Pointer, bool) { + // To find the location of a key in the table, we compute hash(key). From + // h1(hash(key)) and the capacity, we construct a probeSeq that visits + // every group of slots in some interesting order. See [probeSeq]. + // + // We walk through these indices. At each index, we select the entire + // group starting with that index and extract potential candidates: + // occupied slots with a control byte equal to h2(hash(key)). The key + // at candidate slot i is compared with key; if key == g.slot(i).key + // we are done and return the slot; if there is an empty slot in the + // group, we stop and return an error; otherwise we continue to the + // next probe index. Tombstones (ctrlDeleted) effectively behave like + // full slots that never match the value we're looking for. + // + // The h2 bits ensure when we compare a key we are likely to have + // actually found the object. That is, the chance is low that keys + // compare false. Thus, when we search for an object, we are unlikely + // to call Equal many times. This likelihood can be analyzed as follows + // (assuming that h2 is a random enough hash function). + // + // Let's assume that there are k "wrong" objects that must be examined + // in a probe sequence. For example, when doing a find on an object + // that is in the table, k is the number of objects between the start + // of the probe sequence and the final found object (not including the + // final found object). The expected number of objects with an h2 match + // is then k/128. Measurements and analysis indicate that even at high + // load factors, k is less than 32, meaning that the number of false + // positive comparisons we must perform is less than 1/8 per find. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + return slotKey, slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return nil, nil, false + } + } +} + +func (t *table) getWithoutKey(typ *abi.MapType, hash uintptr, key unsafe.Pointer) (unsafe.Pointer, bool) { + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + return slotElem, true + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return nil, false + } + } +} + +// PutSlot returns a pointer to the element slot where an inserted element +// should be written, and ok if it returned a valid slot. +// +// PutSlot returns ok false if the table was split and the Map needs to find +// the new table. +// +// hash must be the hash of key. +func (t *table) PutSlot(typ *abi.MapType, m *Map, hash uintptr, key unsafe.Pointer) (unsafe.Pointer, bool) { + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + + // As we look for a match, keep track of the first deleted slot we + // find, which we'll use to insert the new entry if necessary. + var firstDeletedGroup groupReference + var firstDeletedSlot uintptr + + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + // Look for an existing slot containing this key. + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + if typ.Key.Equal(key, slotKey) { + if typ.NeedKeyUpdate() { + typedmemmove(typ.Key, slotKey, key) + } + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + slotElem = *((*unsafe.Pointer)(slotElem)) + } + + t.checkInvariants(typ, m) + return slotElem, true + } + match = match.removeFirst() + } + + // No existing slot for this key in this group. Is this the end + // of the probe sequence? + match = g.ctrls().matchEmptyOrDeleted() + if match == 0 { + continue // nothing but filled slots. Keep probing. + } + i := match.first() + if g.ctrls().get(i) == ctrlDeleted { + // There are some deleted slots. Remember + // the first one, and keep probing. + if firstDeletedGroup.data == nil { + firstDeletedGroup = g + firstDeletedSlot = i + } + continue + } + // We've found an empty slot, which means we've reached the end of + // the probe sequence. + + // If we found a deleted slot along the way, we can + // replace it without consuming growthLeft. + if firstDeletedGroup.data != nil { + g = firstDeletedGroup + i = firstDeletedSlot + t.growthLeft++ // will be decremented below to become a no-op. + } + + // If we have no space left, first try to remove some tombstones. + if t.growthLeft == 0 { + t.pruneTombstones(typ, m) + } + + // If there is room left to grow, just insert the new entry. + if t.growthLeft > 0 { + slotKey := g.key(typ, i) + if typ.IndirectKey() { + kmem := newobject(typ.Key) + *(*unsafe.Pointer)(slotKey) = kmem + slotKey = kmem + } + typedmemmove(typ.Key, slotKey, key) + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + emem := newobject(typ.Elem) + *(*unsafe.Pointer)(slotElem) = emem + slotElem = emem + } + + g.ctrls().set(i, ctrl(h2Hash)) + t.growthLeft-- + t.used++ + m.used++ + + t.checkInvariants(typ, m) + return slotElem, true + } + + t.rehash(typ, m) + return nil, false + } +} + +// uncheckedPutSlot inserts an entry known not to be in the table. +// This is used for grow/split where we are making a new table from +// entries in an existing table. +// +// Decrements growthLeft and increments used. +// +// Requires that the entry does not exist in the table, and that the table has +// room for another element without rehashing. +// +// Requires that there are no deleted entries in the table. +// +// For indirect keys and/or elements, the key and elem pointers can be +// put directly into the map, they do not need to be copied. This +// requires the caller to ensure that the referenced memory never +// changes (by sourcing those pointers from another indirect key/elem +// map). +func (t *table) uncheckedPutSlot(typ *abi.MapType, hash uintptr, key, elem unsafe.Pointer) { + if t.growthLeft == 0 { + panic("invariant failed: growthLeft is unexpectedly 0") + } + + // Given key and its hash hash(key), to insert it, we construct a + // probeSeq, and use it to find the first group with an unoccupied (empty + // or deleted) slot. We place the key/value into the first such slot in + // the group and mark it as full with key's H2. + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + + match := g.ctrls().matchEmptyOrDeleted() + if match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + if typ.IndirectKey() { + *(*unsafe.Pointer)(slotKey) = key + } else { + typedmemmove(typ.Key, slotKey, key) + } + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + *(*unsafe.Pointer)(slotElem) = elem + } else { + typedmemmove(typ.Elem, slotElem, elem) + } + + t.growthLeft-- + t.used++ + g.ctrls().set(i, ctrl(h2(hash))) + return + } + } +} + +// Delete returns true if it put a tombstone in t. +func (t *table) Delete(typ *abi.MapType, m *Map, hash uintptr, key unsafe.Pointer) bool { + seq := makeProbeSeq(h1(hash), t.groups.lengthMask) + h2Hash := h2(hash) + for ; ; seq = seq.next() { + g := t.groups.group(typ, seq.offset) + match := g.ctrls().matchH2(h2Hash) + + for match != 0 { + i := match.first() + + slotKey := g.key(typ, i) + origSlotKey := slotKey + if typ.IndirectKey() { + slotKey = *((*unsafe.Pointer)(slotKey)) + } + + if typ.Key.Equal(key, slotKey) { + t.used-- + m.used-- + + if typ.IndirectKey() { + // Clearing the pointer is sufficient. + *(*unsafe.Pointer)(origSlotKey) = nil + } else if typ.Key.Pointers() { + // Only bothing clear the key if there + // are pointers in it. + typedmemclr(typ.Key, slotKey) + } + + slotElem := g.elem(typ, i) + if typ.IndirectElem() { + // Clearing the pointer is sufficient. + *(*unsafe.Pointer)(slotElem) = nil + } else { + // Unlike keys, always clear the elem (even if + // it contains no pointers), as compound + // assignment operations depend on cleared + // deleted values. See + // https://go.dev/issue/25936. + typedmemclr(typ.Elem, slotElem) + } + + // Only a full group can appear in the middle + // of a probe sequence (a group with at least + // one empty slot terminates probing). Once a + // group becomes full, it stays full until + // rehashing/resizing. So if the group isn't + // full now, we can simply remove the element. + // Otherwise, we create a tombstone to mark the + // slot as deleted. + var tombstone bool + if g.ctrls().matchEmpty() != 0 { + g.ctrls().set(i, ctrlEmpty) + t.growthLeft++ + } else { + g.ctrls().set(i, ctrlDeleted) + tombstone = true + } + + t.checkInvariants(typ, m) + return tombstone + } + match = match.removeFirst() + } + + match = g.ctrls().matchEmpty() + if match != 0 { + // Finding an empty slot means we've reached the end of + // the probe sequence. + return false + } + } +} + +// pruneTombstones goes through the table and tries to remove +// tombstones that are no longer needed. Best effort. +// Note that it only removes tombstones, it does not move elements. +// Moving elements would do a better job but is infeasbile due to +// iterator semantics. +// +// Pruning should only succeed if it can remove O(n) tombstones. +// It would be bad if we did O(n) work to find 1 tombstone to remove. +// Then the next insert would spend another O(n) work to find 1 more +// tombstone to remove, etc. +// +// We really need to remove O(n) tombstones so we can pay for the cost +// of finding them. If we can't, then we need to grow (which is also O(n), +// but guarantees O(n) subsequent inserts can happen in constant time). +func (t *table) pruneTombstones(typ *abi.MapType, m *Map) { + if t.tombstones()*10 < t.capacity { // 10% of capacity + // Not enough tombstones to be worth the effort. + return + } + + // Bit set marking all the groups whose tombstones are needed. + var needed [(maxTableCapacity/abi.MapGroupSlots + 31) / 32]uint32 + + // Trace the probe sequence of every full entry. + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + match := g.ctrls().matchFull() + for match != 0 { + j := match.first() + match = match.removeFirst() + key := g.key(typ, j) + if typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + if !typ.Key.Equal(key, key) { + // Key not equal to itself. We never have to find these + // keys on lookup (only on iteration), so we can break + // their probe sequences at will. + continue + } + // Walk probe sequence for this key. + // Each tombstone group we need to walk past is marked required. + hash := typ.Hasher(key, m.seed) + for seq := makeProbeSeq(h1(hash), t.groups.lengthMask); ; seq = seq.next() { + if seq.offset == i { + break // reached group of element in probe sequence + } + g := t.groups.group(typ, seq.offset) + m := g.ctrls().matchEmptyOrDeleted() + if m != 0 { // must be deleted, not empty, as we haven't found our key yet + // Mark this group's tombstone as required. + needed[seq.offset/32] |= 1 << (seq.offset % 32) + } + } + } + if g.ctrls().matchEmpty() != 0 { + // Also mark non-tombstone-containing groups, so we don't try + // to remove tombstones from them below. + needed[i/32] |= 1 << (i % 32) + } + } + + // First, see if we can remove enough tombstones to restore capacity. + // This function is O(n), so only remove tombstones if we can remove + // enough of them to justify the O(n) cost. + cnt := 0 + for i := uint64(0); i <= t.groups.lengthMask; i++ { + if needed[i/32]>>(i%32)&1 != 0 { + continue + } + g := t.groups.group(typ, i) + m := g.ctrls().matchEmptyOrDeleted() // must be deleted + cnt += m.count() + } + if cnt*10 < int(t.capacity) { // Can we restore 10% of capacity? + return // don't bother removing tombstones. Caller will grow instead. + } + + // Prune unneeded tombstones. + for i := uint64(0); i <= t.groups.lengthMask; i++ { + if needed[i/32]>>(i%32)&1 != 0 { + continue + } + g := t.groups.group(typ, i) + m := g.ctrls().matchEmptyOrDeleted() // must be deleted + for m != 0 { + k := m.first() + m = m.removeFirst() + g.ctrls().set(k, ctrlEmpty) + t.growthLeft++ + } + // TODO: maybe we could convert all slots at once + // using some bitvector trickery. + } +} + +// tombstones returns the number of deleted (tombstone) entries in the table. A +// tombstone is a slot that has been deleted but is still considered occupied +// so as not to violate the probing invariant. +func (t *table) tombstones() uint16 { + return (t.capacity*maxAvgGroupLoad)/abi.MapGroupSlots - t.used - t.growthLeft +} + +// Clear deletes all entries from the map resulting in an empty map. +func (t *table) Clear(typ *abi.MapType) { + mgl := t.maxGrowthLeft() + if t.used == 0 && t.growthLeft == mgl { // no current entries and no tombstones + return + } + // We only want to do the work of clearing slots + // if they are full. But we also don't want to do too + // much work to figure out whether a slot is full or not, + // especially if clearing a slot is cheap. + // 1) We decide group-by-group instead of slot-by-slot. + // If any slot in a group is full, we zero the whole group. + // 2) If groups are unlikely to be empty, don't bother + // testing for it. + // 3) If groups are 50%/50% likely to be empty, also don't + // bother testing, as it confuses the branch predictor. See #75097. + // 4) But if a group is really large, do the test anyway, as + // clearing is expensive. + fullTest := uint64(t.used)*4 <= t.groups.lengthMask // less than ~0.25 entries per group -> >3/4 empty groups + if typ.SlotSize > 32 { + // For large slots, it is always worth doing the test first. + fullTest = true + } + if fullTest { + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + if g.ctrls().anyFull() { + typedmemclr(typ.Group, g.data) + } + g.ctrls().setEmpty() + } + } else { + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + typedmemclr(typ.Group, g.data) + g.ctrls().setEmpty() + } + } + t.used = 0 + t.growthLeft = mgl +} + +type Iter struct { + key unsafe.Pointer // Must be in first position. Write nil to indicate iteration end (see cmd/compile/internal/walk/range.go). + elem unsafe.Pointer // Must be in second position (see cmd/compile/internal/walk/range.go). + typ *abi.MapType + m *Map + + // Randomize iteration order by starting iteration at a random slot + // offset. The offset into the directory uses a separate offset, as it + // must adjust when the directory grows. + entryOffset uint64 + dirOffset uint64 + + // Snapshot of Map.clearSeq at iteration initialization time. Used to + // detect clear during iteration. + clearSeq uint64 + + // Value of Map.globalDepth during the last call to Next. Used to + // detect directory grow during iteration. + globalDepth uint8 + + // dirIdx is the current directory index, prior to adjustment by + // dirOffset. + dirIdx int + + // tab is the table at dirIdx during the previous call to Next. + tab *table + + // group is the group at entryIdx during the previous call to Next. + group groupReference + + // entryIdx is the current entry index, prior to adjustment by entryOffset. + // The lower 3 bits of the index are the slot index, and the upper bits + // are the group index. + entryIdx uint64 +} + +// Init initializes Iter for iteration. +func (it *Iter) Init(typ *abi.MapType, m *Map) { + it.typ = typ + + if m == nil || m.used == 0 { + return + } + + dirIdx := 0 + var groupSmall groupReference + if m.dirLen <= 0 { + // Use dirIdx == -1 as sentinel for small maps. + dirIdx = -1 + groupSmall.data = m.dirPtr + } + + it.m = m + it.entryOffset = rand() + it.dirOffset = rand() + it.globalDepth = m.globalDepth + it.dirIdx = dirIdx + it.group = groupSmall + it.clearSeq = m.clearSeq +} + +func (it *Iter) Initialized() bool { + return it.typ != nil +} + +// Map returns the map this iterator is iterating over. +func (it *Iter) Map() *Map { + return it.m +} + +// Key returns a pointer to the current key. nil indicates end of iteration. +// +// Must not be called prior to Next. +func (it *Iter) Key() unsafe.Pointer { + return it.key +} + +// Elem returns a pointer to the current element. nil indicates end of +// iteration. +// +// Must not be called prior to Next. +func (it *Iter) Elem() unsafe.Pointer { + return it.elem +} + +func (it *Iter) nextDirIdx() { + // Skip other entries in the directory that refer to the same + // logical table. There are two cases of this: + // + // Consider this directory: + // + // - 0: *t1 + // - 1: *t1 + // - 2: *t2a + // - 3: *t2b + // + // At some point, the directory grew to accommodate a split of + // t2. t1 did not split, so entries 0 and 1 both point to t1. + // t2 did split, so the two halves were installed in entries 2 + // and 3. + // + // If dirIdx is 0 and it.tab is t1, then we should skip past + // entry 1 to avoid repeating t1. + // + // If dirIdx is 2 and it.tab is t2 (pre-split), then we should + // skip past entry 3 because our pre-split t2 already covers + // all keys from t2a and t2b (except for new insertions, which + // iteration need not return). + // + // We can achieve both of these by using to difference between + // the directory and table depth to compute how many entries + // the table covers. + entries := 1 << (it.m.globalDepth - it.tab.localDepth) + it.dirIdx += entries + it.tab = nil + it.group = groupReference{} + it.entryIdx = 0 +} + +// Return the appropriate key/elem for key at slotIdx index within it.group, if +// any. +func (it *Iter) grownKeyElem(key unsafe.Pointer, slotIdx uintptr) (unsafe.Pointer, unsafe.Pointer, bool) { + newKey, newElem, ok := it.m.getWithKey(it.typ, key) + if !ok { + // Key has likely been deleted, and + // should be skipped. + // + // One exception is keys that don't + // compare equal to themselves (e.g., + // NaN). These keys cannot be looked + // up, so getWithKey will fail even if + // the key exists. + // + // However, we are in luck because such + // keys cannot be updated and they + // cannot be deleted except with clear. + // Thus if no clear has occurred, the + // key/elem must still exist exactly as + // in the old groups, so we can return + // them from there. + // + // TODO(prattmic): Consider checking + // clearSeq early. If a clear occurred, + // Next could always return + // immediately, as iteration doesn't + // need to return anything added after + // clear. + if it.clearSeq == it.m.clearSeq && !it.typ.Key.Equal(key, key) { + elem := it.group.elem(it.typ, slotIdx) + if it.typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + return key, elem, true + } + + // This entry doesn't exist anymore. + return nil, nil, false + } + + return newKey, newElem, true +} + +// Next proceeds to the next element in iteration, which can be accessed via +// the Key and Elem methods. +// +// The table can be mutated during iteration, though there is no guarantee that +// the mutations will be visible to the iteration. +// +// Init must be called prior to Next. +func (it *Iter) Next() { + if it.m == nil { + // Map was empty at Iter.Init. + it.key = nil + it.elem = nil + return + } + + if it.m.writing != 0 { + fatal("concurrent map iteration and map write") + return + } + + if it.dirIdx < 0 { + // Map was small at Init. + for ; it.entryIdx < abi.MapGroupSlots; it.entryIdx++ { + k := uintptr(it.entryIdx+it.entryOffset) % abi.MapGroupSlots + + if (it.group.ctrls().get(k) & ctrlEmpty) == ctrlEmpty { + // Empty or deleted. + continue + } + + key := it.group.key(it.typ, k) + if it.typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + // As below, if we have grown to a full map since Init, + // we continue to use the old group to decide the keys + // to return, but must look them up again in the new + // tables. + grown := it.m.dirLen > 0 + var elem unsafe.Pointer + if grown { + var ok bool + newKey, newElem, ok := it.m.getWithKey(it.typ, key) + if !ok { + // See comment below. + if it.clearSeq == it.m.clearSeq && !it.typ.Key.Equal(key, key) { + elem = it.group.elem(it.typ, k) + if it.typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + } else { + continue + } + } else { + key = newKey + elem = newElem + } + } else { + elem = it.group.elem(it.typ, k) + if it.typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + } + + it.entryIdx++ + it.key = key + it.elem = elem + return + } + it.key = nil + it.elem = nil + return + } + + if it.globalDepth != it.m.globalDepth { + // Directory has grown since the last call to Next. Adjust our + // directory index. + // + // Consider: + // + // Before: + // - 0: *t1 + // - 1: *t2 <- dirIdx + // + // After: + // - 0: *t1a (split) + // - 1: *t1b (split) + // - 2: *t2 <- dirIdx + // - 3: *t2 + // + // That is, we want to double the current index when the + // directory size doubles (or quadruple when the directory size + // quadruples, etc). + // + // The actual (randomized) dirIdx is computed below as: + // + // dirIdx := (it.dirIdx + it.dirOffset) % it.m.dirLen + // + // Multiplication is associative across modulo operations, + // A * (B % C) = (A * B) % (A * C), + // provided that A is positive. + // + // Thus we can achieve this by adjusting it.dirIdx, + // it.dirOffset, and it.m.dirLen individually. + orders := it.m.globalDepth - it.globalDepth + it.dirIdx <<= orders + it.dirOffset <<= orders + // it.m.dirLen was already adjusted when the directory grew. + + it.globalDepth = it.m.globalDepth + } + + // Continue iteration until we find a full slot. + for ; it.dirIdx < it.m.dirLen; it.nextDirIdx() { + // Resolve the table. + if it.tab == nil { + dirIdx := int((uint64(it.dirIdx) + it.dirOffset) & uint64(it.m.dirLen-1)) + newTab := it.m.directoryAt(uintptr(dirIdx)) + if newTab.index != dirIdx { + // Normally we skip past all duplicates of the + // same entry in the table (see updates to + // it.dirIdx at the end of the loop below), so + // this case wouldn't occur. + // + // But on the very first call, we have a + // completely randomized dirIdx that may refer + // to a middle of a run of tables in the + // directory. Do a one-time adjustment of the + // offset to ensure we start at first index for + // newTable. + diff := dirIdx - newTab.index + it.dirOffset -= uint64(diff) + dirIdx = newTab.index + } + it.tab = newTab + } + + // N.B. Use it.tab, not newTab. It is important to use the old + // table for key selection if the table has grown. See comment + // on grown below. + + entryMask := uint64(it.tab.capacity) - 1 + if it.entryIdx > entryMask { + // Continue to next table. + continue + } + + // Fast path: skip matching and directly check if entryIdx is a + // full slot. + // + // In the slow path below, we perform an 8-slot match check to + // look for full slots within the group. + // + // However, with a max load factor of 7/8, each slot in a + // mostly full map has a high probability of being full. Thus + // it is cheaper to check a single slot than do a full control + // match. + + entryIdx := (it.entryIdx + it.entryOffset) & entryMask + slotIdx := uintptr(entryIdx & (abi.MapGroupSlots - 1)) + if slotIdx == 0 || it.group.data == nil { + // Only compute the group (a) when we switch + // groups (slotIdx rolls over) and (b) on the + // first iteration in this table (slotIdx may + // not be zero due to entryOffset). + groupIdx := entryIdx >> abi.MapGroupSlotsBits + it.group = it.tab.groups.group(it.typ, groupIdx) + } + + if (it.group.ctrls().get(slotIdx) & ctrlEmpty) == 0 { + // Slot full. + + key := it.group.key(it.typ, slotIdx) + if it.typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + grown := it.tab.index == -1 + var elem unsafe.Pointer + if grown { + newKey, newElem, ok := it.grownKeyElem(key, slotIdx) + if !ok { + // This entry doesn't exist + // anymore. Continue to the + // next one. + goto next + } else { + key = newKey + elem = newElem + } + } else { + elem = it.group.elem(it.typ, slotIdx) + if it.typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + } + + it.entryIdx++ + it.key = key + it.elem = elem + return + } + + next: + it.entryIdx++ + + // Slow path: use a match on the control word to jump ahead to + // the next full slot. + // + // This is highly effective for maps with particularly low load + // (e.g., map allocated with large hint but few insertions). + // + // For maps with medium load (e.g., 3-4 empty slots per group) + // it also tends to work pretty well. Since slots within a + // group are filled in order, then if there have been no + // deletions, a match will allow skipping past all empty slots + // at once. + // + // Note: it is tempting to cache the group match result in the + // iterator to use across Next calls. However because entries + // may be deleted between calls later calls would still need to + // double-check the control value. + + var groupMatch bitset + for it.entryIdx <= entryMask { + entryIdx := (it.entryIdx + it.entryOffset) & entryMask + slotIdx := uintptr(entryIdx & (abi.MapGroupSlots - 1)) + + if slotIdx == 0 || it.group.data == nil { + // Only compute the group (a) when we switch + // groups (slotIdx rolls over) and (b) on the + // first iteration in this table (slotIdx may + // not be zero due to entryOffset). + groupIdx := entryIdx >> abi.MapGroupSlotsBits + it.group = it.tab.groups.group(it.typ, groupIdx) + } + + if groupMatch == 0 { + groupMatch = it.group.ctrls().matchFull() + + if slotIdx != 0 { + // Starting in the middle of the group. + // Ignore earlier groups. + groupMatch = groupMatch.removeBelow(slotIdx) + } + + // Skip over groups that are composed of only empty or + // deleted slots. + if groupMatch == 0 { + // Jump past remaining slots in this + // group. + it.entryIdx += abi.MapGroupSlots - uint64(slotIdx) + continue + } + + i := groupMatch.first() + it.entryIdx += uint64(i - slotIdx) + if it.entryIdx > entryMask { + // Past the end of this table's iteration. + continue + } + entryIdx += uint64(i - slotIdx) + slotIdx = i + } + + key := it.group.key(it.typ, slotIdx) + if it.typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + // If the table has changed since the last + // call, then it has grown or split. In this + // case, further mutations (changes to + // key->elem or deletions) will not be visible + // in our snapshot table. Instead we must + // consult the new table by doing a full + // lookup. + // + // We still use our old table to decide which + // keys to lookup in order to avoid returning + // the same key twice. + grown := it.tab.index == -1 + var elem unsafe.Pointer + if grown { + newKey, newElem, ok := it.grownKeyElem(key, slotIdx) + if !ok { + // This entry doesn't exist anymore. + // Continue to the next one. + groupMatch = groupMatch.removeFirst() + if groupMatch == 0 { + // No more entries in this + // group. Continue to next + // group. + it.entryIdx += abi.MapGroupSlots - uint64(slotIdx) + continue + } + + // Next full slot. + i := groupMatch.first() + it.entryIdx += uint64(i - slotIdx) + continue + } else { + key = newKey + elem = newElem + } + } else { + elem = it.group.elem(it.typ, slotIdx) + if it.typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + } + + // Jump ahead to the next full slot or next group. + groupMatch = groupMatch.removeFirst() + if groupMatch == 0 { + // No more entries in + // this group. Continue + // to next group. + it.entryIdx += abi.MapGroupSlots - uint64(slotIdx) + } else { + // Next full slot. + i := groupMatch.first() + it.entryIdx += uint64(i - slotIdx) + } + + it.key = key + it.elem = elem + return + } + + // Continue to next table. + } + + it.key = nil + it.elem = nil + return +} + +// Replaces the table with one larger table or two split tables to fit more +// entries. Since the table is replaced, t is now stale and should not be +// modified. +func (t *table) rehash(typ *abi.MapType, m *Map) { + // TODO(prattmic): SwissTables typically perform a "rehash in place" + // operation which recovers capacity consumed by tombstones without growing + // the table by reordering slots as necessary to maintain the probe + // invariant while eliminating all tombstones. + // + // However, it is unclear how to make rehash in place work with + // iteration. Since iteration simply walks through all slots in order + // (with random start offset), reordering the slots would break + // iteration. + // + // As an alternative, we could do a "resize" to new groups allocation + // of the same size. This would eliminate the tombstones, but using a + // new allocation, so the existing grow support in iteration would + // continue to work. + + newCapacity := 2 * t.capacity + if newCapacity <= maxTableCapacity { + t.grow(typ, m, newCapacity) + return + } + + t.split(typ, m) +} + +// Bitmask for the last selection bit at this depth. +func localDepthMask(localDepth uint8) uintptr { + if !Use64BitHash { + return uintptr(1) << (32 - localDepth) + } + return uintptr(1) << (64 - localDepth) +} + +// split the table into two, installing the new tables in the map directory. +func (t *table) split(typ *abi.MapType, m *Map) { + localDepth := t.localDepth + localDepth++ + + // TODO: is this the best capacity? + left := newTable(typ, maxTableCapacity, -1, localDepth) + right := newTable(typ, maxTableCapacity, -1, localDepth) + + // Split in half at the localDepth bit from the top. + mask := localDepthMask(localDepth) + + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + for j := uintptr(0); j < abi.MapGroupSlots; j++ { + if (g.ctrls().get(j) & ctrlEmpty) == ctrlEmpty { + // Empty or deleted + continue + } + + key := g.key(typ, j) + if typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + elem := g.elem(typ, j) + if typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + + hash := typ.Hasher(key, m.seed) + var newTable *table + if hash&mask == 0 { + newTable = left + } else { + newTable = right + } + newTable.uncheckedPutSlot(typ, hash, key, elem) + } + } + + m.installTableSplit(t, left, right) + t.index = -1 +} + +// grow the capacity of the table by allocating a new table with a bigger array +// and uncheckedPutting each element of the table into the new table (we know +// that no insertion here will Put an already-present value), and discard the +// old table. +func (t *table) grow(typ *abi.MapType, m *Map, newCapacity uint16) { + newTable := newTable(typ, uint64(newCapacity), t.index, t.localDepth) + + if t.capacity > 0 { + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + for j := uintptr(0); j < abi.MapGroupSlots; j++ { + if (g.ctrls().get(j) & ctrlEmpty) == ctrlEmpty { + // Empty or deleted + continue + } + + key := g.key(typ, j) + if typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + elem := g.elem(typ, j) + if typ.IndirectElem() { + elem = *((*unsafe.Pointer)(elem)) + } + + hash := typ.Hasher(key, m.seed) + + newTable.uncheckedPutSlot(typ, hash, key, elem) + } + } + } + + newTable.checkInvariants(typ, m) + m.replaceTable(newTable) + t.index = -1 +} + +// probeSeq maintains the state for a probe sequence that iterates through the +// groups in a table. The sequence is a triangular progression of the form +// +// p(i) := (i^2 + i)/2 + hash (mod mask+1) +// +// The sequence effectively outputs the indexes of *groups*. The group +// machinery allows us to check an entire group with minimal branching. +// +// It turns out that this probe sequence visits every group exactly once if +// the number of groups is a power of two, since (i^2+i)/2 is a bijection in +// Z/(2^m). See https://en.wikipedia.org/wiki/Quadratic_probing +type probeSeq struct { + mask uint64 + offset uint64 + index uint64 +} + +func makeProbeSeq(hash uintptr, mask uint64) probeSeq { + return probeSeq{ + mask: mask, + offset: uint64(hash) & mask, + index: 0, + } +} + +func (s probeSeq) next() probeSeq { + s.index++ + s.offset = (s.offset + s.index) & s.mask + return s +} + +func (t *table) clone(typ *abi.MapType) *table { + // Shallow copy the table structure. + t2 := new(table) + *t2 = *t + t = t2 + + // We need to just deep copy the groups.data field. + oldGroups := t.groups + newGroups := newGroups(typ, oldGroups.lengthMask+1) + for i := uint64(0); i <= oldGroups.lengthMask; i++ { + oldGroup := oldGroups.group(typ, i) + newGroup := newGroups.group(typ, i) + cloneGroup(typ, newGroup, oldGroup) + } + t.groups = newGroups + + return t +} diff --git a/src/internal/runtime/maps/table_debug.go b/src/internal/runtime/maps/table_debug.go new file mode 100644 index 00000000000000..d680a4b2222607 --- /dev/null +++ b/src/internal/runtime/maps/table_debug.go @@ -0,0 +1,130 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package maps + +import ( + "internal/abi" + "unsafe" +) + +const debugLog = false + +func (t *table) checkInvariants(typ *abi.MapType, m *Map) { + if !debugLog { + return + } + + // For every non-empty slot, verify we can retrieve the key using Get. + // Count the number of used and deleted slots. + var used uint16 + var deleted uint16 + var empty uint16 + for i := uint64(0); i <= t.groups.lengthMask; i++ { + g := t.groups.group(typ, i) + for j := uintptr(0); j < abi.MapGroupSlots; j++ { + c := g.ctrls().get(j) + switch { + case c == ctrlDeleted: + deleted++ + case c == ctrlEmpty: + empty++ + default: + used++ + + key := g.key(typ, j) + if typ.IndirectKey() { + key = *((*unsafe.Pointer)(key)) + } + + // Can't lookup keys that don't compare equal + // to themselves (e.g., NaN). + if !typ.Key.Equal(key, key) { + continue + } + + if _, ok := t.Get(typ, m, key); !ok { + hash := typ.Hasher(key, m.seed) + print("invariant failed: slot(", i, "/", j, "): key ") + dump(key, typ.Key.Size_) + print(" not found [hash=", hash, ", h2=", h2(hash), " h1=", h1(hash), "]\n") + t.Print(typ, m) + panic("invariant failed: slot: key not found") + } + } + } + } + + if used != t.used { + print("invariant failed: found ", used, " used slots, but used count is ", t.used, "\n") + t.Print(typ, m) + panic("invariant failed: found mismatched used slot count") + } + + growthLeft := (t.capacity*maxAvgGroupLoad)/abi.MapGroupSlots - t.used - deleted + if growthLeft != t.growthLeft { + print("invariant failed: found ", t.growthLeft, " growthLeft, but expected ", growthLeft, "\n") + t.Print(typ, m) + panic("invariant failed: found mismatched growthLeft") + } + if deleted != t.tombstones() { + print("invariant failed: found ", deleted, " tombstones, but expected ", t.tombstones(), "\n") + t.Print(typ, m) + panic("invariant failed: found mismatched tombstones") + } + + if empty == 0 { + print("invariant failed: found no empty slots (violates probe invariant)\n") + t.Print(typ, m) + panic("invariant failed: found no empty slots (violates probe invariant)") + } +} +func (t *table) Print(typ *abi.MapType, m *Map) { + print(`table{ + index: `, t.index, ` + localDepth: `, t.localDepth, ` + capacity: `, t.capacity, ` + used: `, t.used, ` + growthLeft: `, t.growthLeft, ` + groups: +`) + + for i := uint64(0); i <= t.groups.lengthMask; i++ { + print("\t\tgroup ", i, "\n") + + g := t.groups.group(typ, i) + ctrls := g.ctrls() + for j := uintptr(0); j < abi.MapGroupSlots; j++ { + print("\t\t\tslot ", j, "\n") + + c := ctrls.get(j) + print("\t\t\t\tctrl ", c) + switch c { + case ctrlEmpty: + print(" (empty)\n") + case ctrlDeleted: + print(" (deleted)\n") + default: + print("\n") + } + + print("\t\t\t\tkey ") + dump(g.key(typ, j), typ.Key.Size_) + println("") + print("\t\t\t\telem ") + dump(g.elem(typ, j), typ.Elem.Size_) + println("") + } + } +} + +// TODO(prattmic): not in hex because print doesn't have a way to print in hex +// outside the runtime. +func dump(ptr unsafe.Pointer, size uintptr) { + for size > 0 { + print(*(*byte)(ptr), " ") + ptr = unsafe.Pointer(uintptr(ptr) + 1) + size-- + } +} diff --git a/src/internal/runtime/math/math.go b/src/internal/runtime/math/math.go index b2e55086511d3f..0af5aa3f7610d8 100644 --- a/src/internal/runtime/math/math.go +++ b/src/internal/runtime/math/math.go @@ -6,7 +6,14 @@ package math import "internal/goarch" -const MaxUintptr = ^uintptr(0) +const ( + MaxUint16 = ^uint16(0) + MaxUint32 = ^uint32(0) + MaxUint64 = ^uint64(0) + MaxUintptr = ^uintptr(0) + + MaxInt64 = int64(MaxUint64 >> 1) +) // MulUintptr returns a * b and whether the multiplication overflowed. // On supported platforms this is an intrinsic lowered by the compiler. diff --git a/src/runtime/internal/startlinetest/func_amd64.go b/src/internal/runtime/startlinetest/func_amd64.go similarity index 100% rename from src/runtime/internal/startlinetest/func_amd64.go rename to src/internal/runtime/startlinetest/func_amd64.go diff --git a/src/runtime/internal/startlinetest/func_amd64.s b/src/internal/runtime/startlinetest/func_amd64.s similarity index 100% rename from src/runtime/internal/startlinetest/func_amd64.s rename to src/internal/runtime/startlinetest/func_amd64.s diff --git a/src/internal/runtime/strconv/atoi.go b/src/internal/runtime/strconv/atoi.go new file mode 100644 index 00000000000000..0308757c6f7b31 --- /dev/null +++ b/src/internal/runtime/strconv/atoi.go @@ -0,0 +1,75 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package strconv + +import ( + "internal/runtime/math" +) + +// Atoi64 parses an int64 from a string s. +// The bool result reports whether s is a number +// representable by a value of type int64. +func Atoi64(s string) (int64, bool) { + if s == "" { + return 0, false + } + + neg := false + if s[0] == '-' { + neg = true + s = s[1:] + } + + un := uint64(0) + for i := 0; i < len(s); i++ { + c := s[i] + if c < '0' || c > '9' { + return 0, false + } + if un > math.MaxUint64/10 { + // overflow + return 0, false + } + un *= 10 + un1 := un + uint64(c) - '0' + if un1 < un { + // overflow + return 0, false + } + un = un1 + } + + if !neg && un > uint64(math.MaxInt64) { + return 0, false + } + if neg && un > uint64(math.MaxInt64)+1 { + return 0, false + } + + n := int64(un) + if neg { + n = -n + } + + return n, true +} + +// Atoi is like Atoi64 but for integers +// that fit into an int. +func Atoi(s string) (int, bool) { + if n, ok := Atoi64(s); n == int64(int(n)) { + return int(n), ok + } + return 0, false +} + +// Atoi32 is like Atoi but for integers +// that fit into an int32. +func Atoi32(s string) (int32, bool) { + if n, ok := Atoi64(s); n == int64(int32(n)) { + return int32(n), ok + } + return 0, false +} diff --git a/src/internal/runtime/strconv/atoi_test.go b/src/internal/runtime/strconv/atoi_test.go new file mode 100644 index 00000000000000..71a8030b1d03fc --- /dev/null +++ b/src/internal/runtime/strconv/atoi_test.go @@ -0,0 +1,104 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package strconv_test + +import ( + "internal/runtime/strconv" + "testing" +) + +const intSize = 32 << (^uint(0) >> 63) + +type atoi64Test struct { + in string + out int64 + ok bool +} + +var atoi64tests = []atoi64Test{ + {"", 0, false}, + {"0", 0, true}, + {"-0", 0, true}, + {"1", 1, true}, + {"-1", -1, true}, + {"12345", 12345, true}, + {"-12345", -12345, true}, + {"012345", 12345, true}, + {"-012345", -12345, true}, + {"12345x", 0, false}, + {"-12345x", 0, false}, + {"98765432100", 98765432100, true}, + {"-98765432100", -98765432100, true}, + {"20496382327982653440", 0, false}, + {"-20496382327982653440", 0, false}, + {"9223372036854775807", 1<<63 - 1, true}, + {"-9223372036854775807", -(1<<63 - 1), true}, + {"9223372036854775808", 0, false}, + {"-9223372036854775808", -1 << 63, true}, + {"9223372036854775809", 0, false}, + {"-9223372036854775809", 0, false}, +} + +func TestAtoi(t *testing.T) { + switch intSize { + case 32: + for i := range atoi32tests { + test := &atoi32tests[i] + out, ok := strconv.Atoi(test.in) + if test.out != int32(out) || test.ok != ok { + t.Errorf("Atoi(%q) = (%v, %v) want (%v, %v)", + test.in, out, ok, test.out, test.ok) + } + } + case 64: + for i := range atoi64tests { + test := &atoi64tests[i] + out, ok := strconv.Atoi(test.in) + if test.out != int64(out) || test.ok != ok { + t.Errorf("Atoi(%q) = (%v, %v) want (%v, %v)", + test.in, out, ok, test.out, test.ok) + } + } + } +} + +type atoi32Test struct { + in string + out int32 + ok bool +} + +var atoi32tests = []atoi32Test{ + {"", 0, false}, + {"0", 0, true}, + {"-0", 0, true}, + {"1", 1, true}, + {"-1", -1, true}, + {"12345", 12345, true}, + {"-12345", -12345, true}, + {"012345", 12345, true}, + {"-012345", -12345, true}, + {"12345x", 0, false}, + {"-12345x", 0, false}, + {"987654321", 987654321, true}, + {"-987654321", -987654321, true}, + {"2147483647", 1<<31 - 1, true}, + {"-2147483647", -(1<<31 - 1), true}, + {"2147483648", 0, false}, + {"-2147483648", -1 << 31, true}, + {"2147483649", 0, false}, + {"-2147483649", 0, false}, +} + +func TestAtoi32(t *testing.T) { + for i := range atoi32tests { + test := &atoi32tests[i] + out, ok := strconv.Atoi32(test.in) + if test.out != out || test.ok != ok { + t.Errorf("Atoi32(%q) = (%v, %v) want (%v, %v)", + test.in, out, ok, test.out, test.ok) + } + } +} diff --git a/src/internal/runtime/sys/consts.go b/src/internal/runtime/sys/consts.go index 98c0f09ef163d1..96e630ed1c3e77 100644 --- a/src/internal/runtime/sys/consts.go +++ b/src/internal/runtime/sys/consts.go @@ -9,10 +9,10 @@ import ( "internal/goos" ) -// AIX requires a larger stack for syscalls. +// AIX and OpenBSD require a larger stack for syscalls. // The race build also needs more stack. See issue 54291. // This arithmetic must match that in cmd/internal/objabi/stack.go:stackGuardMultiplier. -const StackGuardMultiplier = 1 + goos.IsAix + isRace +const StackGuardMultiplier = 1 + goos.IsAix + goos.IsOpenbsd + isRace // DefaultPhysPageSize is the default physical page size. const DefaultPhysPageSize = goarch.DefaultPhysPageSize diff --git a/src/internal/runtime/sys/dit_arm64.go b/src/internal/runtime/sys/dit_arm64.go new file mode 100644 index 00000000000000..643fd770d5f30f --- /dev/null +++ b/src/internal/runtime/sys/dit_arm64.go @@ -0,0 +1,17 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build arm64 + +package sys + +import ( + "internal/cpu" +) + +var DITSupported = cpu.ARM64.HasDIT + +func EnableDIT() bool +func DITEnabled() bool +func DisableDIT() diff --git a/src/internal/runtime/sys/dit_arm64.s b/src/internal/runtime/sys/dit_arm64.s new file mode 100644 index 00000000000000..c27dfc9af3c1ec --- /dev/null +++ b/src/internal/runtime/sys/dit_arm64.s @@ -0,0 +1,22 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +TEXT ·EnableDIT(SB),$0-1 + MRS DIT, R0 + UBFX $24, R0, $1, R1 + MOVB R1, ret+0(FP) + MSR $1, DIT + RET + +TEXT ·DITEnabled(SB),$0-1 + MRS DIT, R0 + UBFX $24, R0, $1, R1 + MOVB R1, ret+0(FP) + RET + +TEXT ·DisableDIT(SB),$0 + MSR $0, DIT + RET diff --git a/src/internal/runtime/sys/empty.s b/src/internal/runtime/sys/empty.s new file mode 100644 index 00000000000000..3e62b7dac3e93f --- /dev/null +++ b/src/internal/runtime/sys/empty.s @@ -0,0 +1 @@ +// Empty assembly file to allow empty function bodies for intrinsics. diff --git a/src/internal/runtime/sys/intrinsics.go b/src/internal/runtime/sys/intrinsics.go index e6a3758447f95e..147d5581f254f3 100644 --- a/src/internal/runtime/sys/intrinsics.go +++ b/src/internal/runtime/sys/intrinsics.go @@ -206,3 +206,51 @@ func Prefetch(addr uintptr) {} // // ARM64: Produce PRFM instruction with PLDL1STRM option func PrefetchStreamed(addr uintptr) {} + +// GetCallerPC returns the program counter (PC) of its caller's caller. +// GetCallerSP returns the stack pointer (SP) of its caller's caller. +// Both are implemented as intrinsics on every platform. +// +// For example: +// +// func f(arg1, arg2, arg3 int) { +// pc := GetCallerPC() +// sp := GetCallerSP() +// } +// +// These two lines find the PC and SP immediately following +// the call to f (where f will return). +// +// The call to GetCallerPC and GetCallerSP must be done in the +// frame being asked about. +// +// The result of GetCallerSP is correct at the time of the return, +// but it may be invalidated by any subsequent call to a function +// that might relocate the stack in order to grow or shrink it. +// A general rule is that the result of GetCallerSP should be used +// immediately and can only be passed to nosplit functions. + +func GetCallerPC() uintptr + +func GetCallerSP() uintptr + +// GetClosurePtr returns the pointer to the current closure. +// GetClosurePtr can only be used in an assignment statement +// at the entry of a function. Moreover, go:nosplit directive +// must be specified at the declaration of caller function, +// so that the function prolog does not clobber the closure register. +// for example: +// +// //go:nosplit +// func f(arg1, arg2, arg3 int) { +// dx := GetClosurePtr() +// } +// +// The compiler rewrites calls to this function into instructions that fetch the +// pointer from a well-known register (DX on x86 architecture, etc.) directly. +// +// WARNING: PGO-based devirtualization cannot detect that caller of +// GetClosurePtr requires closure context, and thus must maintain a list of +// these functions, which is in +// cmd/compile/internal/devirtualize/pgo.maybeDevirtualizeFunctionCall. +func GetClosurePtr() uintptr diff --git a/src/internal/runtime/sys/nih.go b/src/internal/runtime/sys/nih.go index 17eab67345b06e..a9cbc48e1434e0 100644 --- a/src/internal/runtime/sys/nih.go +++ b/src/internal/runtime/sys/nih.go @@ -14,7 +14,7 @@ type nih struct{} // Other types can embed NotInHeap to make it not-in-heap. Specifically, pointers // to these types must always fail the `runtime.inheap` check. The type may be used // for global variables, or for objects in unmanaged memory (e.g., allocated with -// `sysAlloc`, `persistentalloc`, r`fixalloc`, or from a manually-managed span). +// `sysAlloc`, `persistentalloc`, `fixalloc`, or from a manually-managed span). // // Specifically: // diff --git a/src/internal/runtime/sys/no_dit.go b/src/internal/runtime/sys/no_dit.go new file mode 100644 index 00000000000000..0589d0ca14e643 --- /dev/null +++ b/src/internal/runtime/sys/no_dit.go @@ -0,0 +1,13 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !arm64 + +package sys + +var DITSupported = false + +func EnableDIT() bool { return false } +func DITEnabled() bool { return false } +func DisableDIT() {} diff --git a/src/internal/runtime/syscall/defs_linux.go b/src/internal/runtime/syscall/defs_linux.go deleted file mode 100644 index b2e36a244f8201..00000000000000 --- a/src/internal/runtime/syscall/defs_linux.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package syscall - -const ( - EPOLLIN = 0x1 - EPOLLOUT = 0x4 - EPOLLERR = 0x8 - EPOLLHUP = 0x10 - EPOLLRDHUP = 0x2000 - EPOLLET = 0x80000000 - EPOLL_CLOEXEC = 0x80000 - EPOLL_CTL_ADD = 0x1 - EPOLL_CTL_DEL = 0x2 - EPOLL_CTL_MOD = 0x3 - EFD_CLOEXEC = 0x80000 -) diff --git a/src/internal/runtime/syscall/defs_linux_386.go b/src/internal/runtime/syscall/defs_linux_386.go deleted file mode 100644 index 68e687fb14b71b..00000000000000 --- a/src/internal/runtime/syscall/defs_linux_386.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package syscall - -const ( - SYS_FCNTL = 55 - SYS_MPROTECT = 125 - SYS_EPOLL_CTL = 255 - SYS_EPOLL_PWAIT = 319 - SYS_EPOLL_CREATE1 = 329 - SYS_EPOLL_PWAIT2 = 441 - SYS_EVENTFD2 = 328 - - EFD_NONBLOCK = 0x800 -) - -type EpollEvent struct { - Events uint32 - Data [8]byte // to match amd64 -} diff --git a/src/internal/runtime/syscall/defs_linux_amd64.go b/src/internal/runtime/syscall/defs_linux_amd64.go deleted file mode 100644 index ec480f5817e031..00000000000000 --- a/src/internal/runtime/syscall/defs_linux_amd64.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package syscall - -const ( - SYS_MPROTECT = 10 - SYS_FCNTL = 72 - SYS_EPOLL_CTL = 233 - SYS_EPOLL_PWAIT = 281 - SYS_EPOLL_CREATE1 = 291 - SYS_EPOLL_PWAIT2 = 441 - SYS_EVENTFD2 = 290 - - EFD_NONBLOCK = 0x800 -) - -type EpollEvent struct { - Events uint32 - Data [8]byte // unaligned uintptr -} diff --git a/src/internal/runtime/syscall/defs_linux_arm.go b/src/internal/runtime/syscall/defs_linux_arm.go deleted file mode 100644 index c5d1503012304c..00000000000000 --- a/src/internal/runtime/syscall/defs_linux_arm.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package syscall - -const ( - SYS_FCNTL = 55 - SYS_MPROTECT = 125 - SYS_EPOLL_CTL = 251 - SYS_EPOLL_PWAIT = 346 - SYS_EPOLL_CREATE1 = 357 - SYS_EPOLL_PWAIT2 = 441 - SYS_EVENTFD2 = 356 - - EFD_NONBLOCK = 0x800 -) - -type EpollEvent struct { - Events uint32 - _pad uint32 - Data [8]byte // to match amd64 -} diff --git a/src/internal/runtime/syscall/defs_linux_arm64.go b/src/internal/runtime/syscall/defs_linux_arm64.go deleted file mode 100644 index f743fe31a58d1a..00000000000000 --- a/src/internal/runtime/syscall/defs_linux_arm64.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package syscall - -const ( - SYS_EPOLL_CREATE1 = 20 - SYS_EPOLL_CTL = 21 - SYS_EPOLL_PWAIT = 22 - SYS_FCNTL = 25 - SYS_MPROTECT = 226 - SYS_EPOLL_PWAIT2 = 441 - SYS_EVENTFD2 = 19 - - EFD_NONBLOCK = 0x800 -) - -type EpollEvent struct { - Events uint32 - _pad uint32 - Data [8]byte // to match amd64 -} diff --git a/src/internal/runtime/syscall/defs_linux_mipsx.go b/src/internal/runtime/syscall/defs_linux_mipsx.go deleted file mode 100644 index b87a355093e489..00000000000000 --- a/src/internal/runtime/syscall/defs_linux_mipsx.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build linux && (mips || mipsle) - -package syscall - -const ( - SYS_FCNTL = 4055 - SYS_MPROTECT = 4125 - SYS_EPOLL_CTL = 4249 - SYS_EPOLL_PWAIT = 4313 - SYS_EPOLL_CREATE1 = 4326 - SYS_EPOLL_PWAIT2 = 4441 - SYS_EVENTFD2 = 4325 - - EFD_NONBLOCK = 0x80 -) - -type EpollEvent struct { - Events uint32 - pad_cgo_0 [4]byte - Data uint64 -} diff --git a/src/internal/runtime/syscall/asm_linux_386.s b/src/internal/runtime/syscall/linux/asm_linux_386.s similarity index 100% rename from src/internal/runtime/syscall/asm_linux_386.s rename to src/internal/runtime/syscall/linux/asm_linux_386.s diff --git a/src/internal/runtime/syscall/asm_linux_amd64.s b/src/internal/runtime/syscall/linux/asm_linux_amd64.s similarity index 100% rename from src/internal/runtime/syscall/asm_linux_amd64.s rename to src/internal/runtime/syscall/linux/asm_linux_amd64.s diff --git a/src/internal/runtime/syscall/asm_linux_arm.s b/src/internal/runtime/syscall/linux/asm_linux_arm.s similarity index 100% rename from src/internal/runtime/syscall/asm_linux_arm.s rename to src/internal/runtime/syscall/linux/asm_linux_arm.s diff --git a/src/internal/runtime/syscall/asm_linux_arm64.s b/src/internal/runtime/syscall/linux/asm_linux_arm64.s similarity index 100% rename from src/internal/runtime/syscall/asm_linux_arm64.s rename to src/internal/runtime/syscall/linux/asm_linux_arm64.s diff --git a/src/internal/runtime/syscall/asm_linux_loong64.s b/src/internal/runtime/syscall/linux/asm_linux_loong64.s similarity index 100% rename from src/internal/runtime/syscall/asm_linux_loong64.s rename to src/internal/runtime/syscall/linux/asm_linux_loong64.s diff --git a/src/internal/runtime/syscall/asm_linux_mips64x.s b/src/internal/runtime/syscall/linux/asm_linux_mips64x.s similarity index 100% rename from src/internal/runtime/syscall/asm_linux_mips64x.s rename to src/internal/runtime/syscall/linux/asm_linux_mips64x.s diff --git a/src/internal/runtime/syscall/asm_linux_mipsx.s b/src/internal/runtime/syscall/linux/asm_linux_mipsx.s similarity index 100% rename from src/internal/runtime/syscall/asm_linux_mipsx.s rename to src/internal/runtime/syscall/linux/asm_linux_mipsx.s diff --git a/src/internal/runtime/syscall/asm_linux_ppc64x.s b/src/internal/runtime/syscall/linux/asm_linux_ppc64x.s similarity index 100% rename from src/internal/runtime/syscall/asm_linux_ppc64x.s rename to src/internal/runtime/syscall/linux/asm_linux_ppc64x.s diff --git a/src/internal/runtime/syscall/asm_linux_riscv64.s b/src/internal/runtime/syscall/linux/asm_linux_riscv64.s similarity index 100% rename from src/internal/runtime/syscall/asm_linux_riscv64.s rename to src/internal/runtime/syscall/linux/asm_linux_riscv64.s diff --git a/src/internal/runtime/syscall/asm_linux_s390x.s b/src/internal/runtime/syscall/linux/asm_linux_s390x.s similarity index 100% rename from src/internal/runtime/syscall/asm_linux_s390x.s rename to src/internal/runtime/syscall/linux/asm_linux_s390x.s diff --git a/src/internal/runtime/syscall/linux/defs_linux.go b/src/internal/runtime/syscall/linux/defs_linux.go new file mode 100644 index 00000000000000..1ca3c352386b46 --- /dev/null +++ b/src/internal/runtime/syscall/linux/defs_linux.go @@ -0,0 +1,29 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package linux + +const ( + AT_FDCWD = -0x64 + + ENOENT = 0x2 + + EPOLLIN = 0x1 + EPOLLOUT = 0x4 + EPOLLERR = 0x8 + EPOLLHUP = 0x10 + EPOLLRDHUP = 0x2000 + EPOLLET = 0x80000000 + EPOLL_CLOEXEC = 0x80000 + EPOLL_CTL_ADD = 0x1 + EPOLL_CTL_DEL = 0x2 + EPOLL_CTL_MOD = 0x3 + EFD_CLOEXEC = 0x80000 + + O_RDONLY = 0x0 + O_CLOEXEC = 0x80000 + + PR_SET_VMA = 0x53564d41 + PR_SET_VMA_ANON_NAME = 0 +) diff --git a/src/internal/runtime/syscall/linux/defs_linux_386.go b/src/internal/runtime/syscall/linux/defs_linux_386.go new file mode 100644 index 00000000000000..7fdf5d3f8062fa --- /dev/null +++ b/src/internal/runtime/syscall/linux/defs_linux_386.go @@ -0,0 +1,29 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package linux + +const ( + SYS_CLOSE = 6 + SYS_FCNTL = 55 + SYS_MPROTECT = 125 + SYS_PRCTL = 172 + SYS_EPOLL_CTL = 255 + SYS_EPOLL_PWAIT = 319 + SYS_EPOLL_CREATE1 = 329 + SYS_EPOLL_PWAIT2 = 441 + SYS_EVENTFD2 = 328 + SYS_OPENAT = 295 + SYS_PREAD64 = 180 + SYS_READ = 3 + + EFD_NONBLOCK = 0x800 + + O_LARGEFILE = 0x8000 +) + +type EpollEvent struct { + Events uint32 + Data [8]byte // to match amd64 +} diff --git a/src/internal/runtime/syscall/linux/defs_linux_amd64.go b/src/internal/runtime/syscall/linux/defs_linux_amd64.go new file mode 100644 index 00000000000000..2c8676e6e9b4d9 --- /dev/null +++ b/src/internal/runtime/syscall/linux/defs_linux_amd64.go @@ -0,0 +1,29 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package linux + +const ( + SYS_CLOSE = 3 + SYS_MPROTECT = 10 + SYS_FCNTL = 72 + SYS_PRCTL = 157 + SYS_EPOLL_CTL = 233 + SYS_EPOLL_PWAIT = 281 + SYS_EPOLL_CREATE1 = 291 + SYS_EPOLL_PWAIT2 = 441 + SYS_EVENTFD2 = 290 + SYS_OPENAT = 257 + SYS_PREAD64 = 17 + SYS_READ = 0 + + EFD_NONBLOCK = 0x800 + + O_LARGEFILE = 0x0 +) + +type EpollEvent struct { + Events uint32 + Data [8]byte // unaligned uintptr +} diff --git a/src/internal/runtime/syscall/linux/defs_linux_arm.go b/src/internal/runtime/syscall/linux/defs_linux_arm.go new file mode 100644 index 00000000000000..a0b395d6762734 --- /dev/null +++ b/src/internal/runtime/syscall/linux/defs_linux_arm.go @@ -0,0 +1,30 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package linux + +const ( + SYS_CLOSE = 6 + SYS_FCNTL = 55 + SYS_MPROTECT = 125 + SYS_PRCTL = 172 + SYS_EPOLL_CTL = 251 + SYS_EPOLL_PWAIT = 346 + SYS_EPOLL_CREATE1 = 357 + SYS_EPOLL_PWAIT2 = 441 + SYS_EVENTFD2 = 356 + SYS_OPENAT = 322 + SYS_PREAD64 = 180 + SYS_READ = 3 + + EFD_NONBLOCK = 0x800 + + O_LARGEFILE = 0x20000 +) + +type EpollEvent struct { + Events uint32 + _pad uint32 + Data [8]byte // to match amd64 +} diff --git a/src/internal/runtime/syscall/linux/defs_linux_arm64.go b/src/internal/runtime/syscall/linux/defs_linux_arm64.go new file mode 100644 index 00000000000000..223dce0c5b4281 --- /dev/null +++ b/src/internal/runtime/syscall/linux/defs_linux_arm64.go @@ -0,0 +1,30 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package linux + +const ( + SYS_CLOSE = 57 + SYS_EPOLL_CREATE1 = 20 + SYS_EPOLL_CTL = 21 + SYS_EPOLL_PWAIT = 22 + SYS_FCNTL = 25 + SYS_PRCTL = 167 + SYS_MPROTECT = 226 + SYS_EPOLL_PWAIT2 = 441 + SYS_EVENTFD2 = 19 + SYS_OPENAT = 56 + SYS_PREAD64 = 67 + SYS_READ = 63 + + EFD_NONBLOCK = 0x800 + + O_LARGEFILE = 0x0 +) + +type EpollEvent struct { + Events uint32 + _pad uint32 + Data [8]byte // to match amd64 +} diff --git a/src/internal/runtime/syscall/defs_linux_loong64.go b/src/internal/runtime/syscall/linux/defs_linux_loong64.go similarity index 75% rename from src/internal/runtime/syscall/defs_linux_loong64.go rename to src/internal/runtime/syscall/linux/defs_linux_loong64.go index 82218d15099aa5..8aa61c391dcdcb 100644 --- a/src/internal/runtime/syscall/defs_linux_loong64.go +++ b/src/internal/runtime/syscall/linux/defs_linux_loong64.go @@ -2,18 +2,25 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package syscall +package linux const ( + SYS_CLOSE = 57 SYS_EPOLL_CREATE1 = 20 SYS_EPOLL_CTL = 21 SYS_EPOLL_PWAIT = 22 SYS_FCNTL = 25 + SYS_PRCTL = 167 SYS_MPROTECT = 226 SYS_EPOLL_PWAIT2 = 441 SYS_EVENTFD2 = 19 + SYS_OPENAT = 56 + SYS_PREAD64 = 67 + SYS_READ = 63 EFD_NONBLOCK = 0x800 + + O_LARGEFILE = 0x0 ) type EpollEvent struct { diff --git a/src/internal/runtime/syscall/defs_linux_mips64x.go b/src/internal/runtime/syscall/linux/defs_linux_mips64x.go similarity index 76% rename from src/internal/runtime/syscall/defs_linux_mips64x.go rename to src/internal/runtime/syscall/linux/defs_linux_mips64x.go index 4e0fd1f5d1b10a..84b760dc1b5545 100644 --- a/src/internal/runtime/syscall/defs_linux_mips64x.go +++ b/src/internal/runtime/syscall/linux/defs_linux_mips64x.go @@ -4,18 +4,25 @@ //go:build linux && (mips64 || mips64le) -package syscall +package linux const ( + SYS_CLOSE = 5003 SYS_MPROTECT = 5010 SYS_FCNTL = 5070 + SYS_PRCTL = 5153 SYS_EPOLL_CTL = 5208 SYS_EPOLL_PWAIT = 5272 SYS_EPOLL_CREATE1 = 5285 SYS_EPOLL_PWAIT2 = 5441 SYS_EVENTFD2 = 5284 + SYS_OPENAT = 5247 + SYS_PREAD64 = 5016 + SYS_READ = 5000 EFD_NONBLOCK = 0x80 + + O_LARGEFILE = 0x0 ) type EpollEvent struct { diff --git a/src/internal/runtime/syscall/linux/defs_linux_mipsx.go b/src/internal/runtime/syscall/linux/defs_linux_mipsx.go new file mode 100644 index 00000000000000..a9be21414c26f9 --- /dev/null +++ b/src/internal/runtime/syscall/linux/defs_linux_mipsx.go @@ -0,0 +1,32 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (mips || mipsle) + +package linux + +const ( + SYS_CLOSE = 4006 + SYS_FCNTL = 4055 + SYS_MPROTECT = 4125 + SYS_PRCTL = 4192 + SYS_EPOLL_CTL = 4249 + SYS_EPOLL_PWAIT = 4313 + SYS_EPOLL_CREATE1 = 4326 + SYS_EPOLL_PWAIT2 = 4441 + SYS_EVENTFD2 = 4325 + SYS_OPENAT = 4288 + SYS_PREAD64 = 4200 + SYS_READ = 4003 + + EFD_NONBLOCK = 0x80 + + O_LARGEFILE = 0x2000 +) + +type EpollEvent struct { + Events uint32 + pad_cgo_0 [4]byte + Data uint64 +} diff --git a/src/internal/runtime/syscall/defs_linux_ppc64x.go b/src/internal/runtime/syscall/linux/defs_linux_ppc64x.go similarity index 76% rename from src/internal/runtime/syscall/defs_linux_ppc64x.go rename to src/internal/runtime/syscall/linux/defs_linux_ppc64x.go index 8235edd795fa85..63f4e5d7864de4 100644 --- a/src/internal/runtime/syscall/defs_linux_ppc64x.go +++ b/src/internal/runtime/syscall/linux/defs_linux_ppc64x.go @@ -4,18 +4,25 @@ //go:build linux && (ppc64 || ppc64le) -package syscall +package linux const ( + SYS_CLOSE = 6 SYS_FCNTL = 55 SYS_MPROTECT = 125 + SYS_PRCTL = 171 SYS_EPOLL_CTL = 237 SYS_EPOLL_PWAIT = 303 SYS_EPOLL_CREATE1 = 315 SYS_EPOLL_PWAIT2 = 441 SYS_EVENTFD2 = 314 + SYS_OPENAT = 286 + SYS_PREAD64 = 179 + SYS_READ = 3 EFD_NONBLOCK = 0x800 + + O_LARGEFILE = 0x0 ) type EpollEvent struct { diff --git a/src/internal/runtime/syscall/defs_linux_riscv64.go b/src/internal/runtime/syscall/linux/defs_linux_riscv64.go similarity index 75% rename from src/internal/runtime/syscall/defs_linux_riscv64.go rename to src/internal/runtime/syscall/linux/defs_linux_riscv64.go index 82218d15099aa5..8aa61c391dcdcb 100644 --- a/src/internal/runtime/syscall/defs_linux_riscv64.go +++ b/src/internal/runtime/syscall/linux/defs_linux_riscv64.go @@ -2,18 +2,25 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package syscall +package linux const ( + SYS_CLOSE = 57 SYS_EPOLL_CREATE1 = 20 SYS_EPOLL_CTL = 21 SYS_EPOLL_PWAIT = 22 SYS_FCNTL = 25 + SYS_PRCTL = 167 SYS_MPROTECT = 226 SYS_EPOLL_PWAIT2 = 441 SYS_EVENTFD2 = 19 + SYS_OPENAT = 56 + SYS_PREAD64 = 67 + SYS_READ = 63 EFD_NONBLOCK = 0x800 + + O_LARGEFILE = 0x0 ) type EpollEvent struct { diff --git a/src/internal/runtime/syscall/defs_linux_s390x.go b/src/internal/runtime/syscall/linux/defs_linux_s390x.go similarity index 75% rename from src/internal/runtime/syscall/defs_linux_s390x.go rename to src/internal/runtime/syscall/linux/defs_linux_s390x.go index 08073c01f091b4..52945db0e5b72f 100644 --- a/src/internal/runtime/syscall/defs_linux_s390x.go +++ b/src/internal/runtime/syscall/linux/defs_linux_s390x.go @@ -2,18 +2,25 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package syscall +package linux const ( + SYS_CLOSE = 6 SYS_FCNTL = 55 SYS_MPROTECT = 125 + SYS_PRCTL = 172 SYS_EPOLL_CTL = 250 SYS_EPOLL_PWAIT = 312 SYS_EPOLL_CREATE1 = 327 SYS_EPOLL_PWAIT2 = 441 SYS_EVENTFD2 = 323 + SYS_OPENAT = 288 + SYS_PREAD64 = 180 + SYS_READ = 3 EFD_NONBLOCK = 0x800 + + O_LARGEFILE = 0x0 ) type EpollEvent struct { diff --git a/src/internal/runtime/syscall/linux/syscall_linux.go b/src/internal/runtime/syscall/linux/syscall_linux.go new file mode 100644 index 00000000000000..8201e7d1907444 --- /dev/null +++ b/src/internal/runtime/syscall/linux/syscall_linux.go @@ -0,0 +1,88 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package linux provides the syscall primitives required for the runtime. +package linux + +import ( + "internal/goarch" + "unsafe" +) + +// TODO(https://go.dev/issue/51087): Move remaining syscalls to this package. + +// Syscall6 calls system call number 'num' with arguments a1-6. +func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) + +func EpollCreate1(flags int32) (fd int32, errno uintptr) { + r1, _, e := Syscall6(SYS_EPOLL_CREATE1, uintptr(flags), 0, 0, 0, 0, 0) + return int32(r1), e +} + +var _zero uintptr + +func EpollWait(epfd int32, events []EpollEvent, maxev, waitms int32) (n int32, errno uintptr) { + var ev unsafe.Pointer + if len(events) > 0 { + ev = unsafe.Pointer(&events[0]) + } else { + ev = unsafe.Pointer(&_zero) + } + r1, _, e := Syscall6(SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(ev), uintptr(maxev), uintptr(waitms), 0, 0) + return int32(r1), e +} + +func EpollCtl(epfd, op, fd int32, event *EpollEvent) (errno uintptr) { + _, _, e := Syscall6(SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0) + return e +} + +func Eventfd(initval, flags int32) (fd int32, errno uintptr) { + r1, _, e := Syscall6(SYS_EVENTFD2, uintptr(initval), uintptr(flags), 0, 0, 0, 0) + return int32(r1), e +} + +func Open(path *byte, mode int, perm uint32) (fd int, errno uintptr) { + // Use SYS_OPENAT to match the syscall package. + dfd := AT_FDCWD + r1, _, e := Syscall6(SYS_OPENAT, uintptr(dfd), uintptr(unsafe.Pointer(path)), uintptr(mode|O_LARGEFILE), uintptr(perm), 0, 0) + return int(r1), e +} + +func Close(fd int) (errno uintptr) { + _, _, e := Syscall6(SYS_CLOSE, uintptr(fd), 0, 0, 0, 0, 0) + return e +} + +func Read(fd int, p []byte) (n int, errno uintptr) { + var p0 unsafe.Pointer + if len(p) > 0 { + p0 = unsafe.Pointer(&p[0]) + } else { + p0 = unsafe.Pointer(&_zero) + } + r1, _, e := Syscall6(SYS_READ, uintptr(fd), uintptr(p0), uintptr(len(p)), 0, 0, 0) + return int(r1), e +} + +func Pread(fd int, p []byte, offset int64) (n int, errno uintptr) { + var p0 unsafe.Pointer + if len(p) > 0 { + p0 = unsafe.Pointer(&p[0]) + } else { + p0 = unsafe.Pointer(&_zero) + } + var r1, e uintptr + switch goarch.GOARCH { + case "386": + r1, _, e = Syscall6(SYS_PREAD64, uintptr(fd), uintptr(p0), uintptr(len(p)), uintptr(offset), uintptr(offset>>32), 0) + case "arm", "mipsle": + r1, _, e = Syscall6(SYS_PREAD64, uintptr(fd), uintptr(p0), uintptr(len(p)), 0, uintptr(offset), uintptr(offset>>32)) + case "mips": + r1, _, e = Syscall6(SYS_PREAD64, uintptr(fd), uintptr(p0), uintptr(len(p)), 0, uintptr(offset>>32), uintptr(offset)) + default: + r1, _, e = Syscall6(SYS_PREAD64, uintptr(fd), uintptr(p0), uintptr(len(p)), uintptr(offset), 0, 0) + } + return int(r1), e +} diff --git a/src/internal/runtime/syscall/linux/syscall_linux_test.go b/src/internal/runtime/syscall/linux/syscall_linux_test.go new file mode 100644 index 00000000000000..10eb2d56bfe66e --- /dev/null +++ b/src/internal/runtime/syscall/linux/syscall_linux_test.go @@ -0,0 +1,19 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package linux_test + +import ( + "internal/runtime/syscall/linux" + "testing" +) + +func TestEpollctlErrorSign(t *testing.T) { + v := linux.EpollCtl(-1, 1, -1, &linux.EpollEvent{}) + + const EBADF = 0x09 + if v != EBADF { + t.Errorf("epollctl = %v, want %v", v, EBADF) + } +} diff --git a/src/internal/runtime/syscall/syscall_linux.go b/src/internal/runtime/syscall/syscall_linux.go deleted file mode 100644 index 83df825169b907..00000000000000 --- a/src/internal/runtime/syscall/syscall_linux.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package syscall provides the syscall primitives required for the runtime. -package syscall - -import ( - "unsafe" -) - -// TODO(https://go.dev/issue/51087): This package is incomplete and currently -// only contains very minimal support for Linux. - -// Syscall6 calls system call number 'num' with arguments a1-6. -func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) - -func EpollCreate1(flags int32) (fd int32, errno uintptr) { - r1, _, e := Syscall6(SYS_EPOLL_CREATE1, uintptr(flags), 0, 0, 0, 0, 0) - return int32(r1), e -} - -var _zero uintptr - -func EpollWait(epfd int32, events []EpollEvent, maxev, waitms int32) (n int32, errno uintptr) { - var ev unsafe.Pointer - if len(events) > 0 { - ev = unsafe.Pointer(&events[0]) - } else { - ev = unsafe.Pointer(&_zero) - } - r1, _, e := Syscall6(SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(ev), uintptr(maxev), uintptr(waitms), 0, 0) - return int32(r1), e -} - -func EpollCtl(epfd, op, fd int32, event *EpollEvent) (errno uintptr) { - _, _, e := Syscall6(SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0) - return e -} - -func Eventfd(initval, flags int32) (fd int32, errno uintptr) { - r1, _, e := Syscall6(SYS_EVENTFD2, uintptr(initval), uintptr(flags), 0, 0, 0, 0) - return int32(r1), e -} diff --git a/src/internal/runtime/syscall/syscall_linux_test.go b/src/internal/runtime/syscall/syscall_linux_test.go deleted file mode 100644 index 14bb31c4b8c9b8..00000000000000 --- a/src/internal/runtime/syscall/syscall_linux_test.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package syscall_test - -import ( - "internal/runtime/syscall" - "testing" -) - -func TestEpollctlErrorSign(t *testing.T) { - v := syscall.EpollCtl(-1, 1, -1, &syscall.EpollEvent{}) - - const EBADF = 0x09 - if v != EBADF { - t.Errorf("epollctl = %v, want %v", v, EBADF) - } -} diff --git a/src/internal/runtime/syscall/windows/asm_windows_386.s b/src/internal/runtime/syscall/windows/asm_windows_386.s new file mode 100644 index 00000000000000..29cce00309edc6 --- /dev/null +++ b/src/internal/runtime/syscall/windows/asm_windows_386.s @@ -0,0 +1,48 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "textflag.h" + +TEXT ·StdCall(SB),NOSPLIT,$0 + JMP ·asmstdcall(SB) + +TEXT ·asmstdcall(SB),NOSPLIT,$0 + MOVL fn+0(FP), BX + MOVL SP, BP // save stack pointer + + // SetLastError(0). + MOVL $0, 0x34(FS) + + MOVL StdCallInfo_N(BX), CX + + // Fast version, do not store args on the stack. + CMPL CX, $0 + JE docall + + // Copy args to the stack. + MOVL CX, AX + SALL $2, AX + SUBL AX, SP // room for args + MOVL SP, DI + MOVL StdCallInfo_Args(BX), SI + CLD + REP; MOVSL + +docall: + // Call stdcall or cdecl function. + // DI SI BP BX are preserved, SP is not + CALL StdCallInfo_Fn(BX) + MOVL BP, SP + + // Return result. + MOVL fn+0(FP), BX + MOVL AX, StdCallInfo_R1(BX) + MOVL DX, StdCallInfo_R2(BX) + + // GetLastError(). + MOVL 0x34(FS), AX + MOVL AX, StdCallInfo_Err(BX) + + RET diff --git a/src/internal/runtime/syscall/windows/asm_windows_amd64.s b/src/internal/runtime/syscall/windows/asm_windows_amd64.s new file mode 100644 index 00000000000000..c31cbcdd14b218 --- /dev/null +++ b/src/internal/runtime/syscall/windows/asm_windows_amd64.s @@ -0,0 +1,84 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "textflag.h" + +TEXT ·StdCall(SB),NOSPLIT,$0 + MOVQ AX, CX + JMP ·asmstdcall(SB) + +TEXT ·asmstdcall(SB),NOSPLIT,$16 + MOVQ SP, AX + ANDQ $~15, SP // alignment as per Windows requirement + MOVQ AX, 8(SP) + MOVQ CX, 0(SP) // asmcgocall will put first argument into CX. + + MOVQ StdCallInfo_Fn(CX), AX + MOVQ StdCallInfo_Args(CX), SI + MOVQ StdCallInfo_N(CX), CX + + // SetLastError(0). + MOVQ 0x30(GS), DI + MOVL $0, 0x68(DI) + + SUBQ $(const_MaxArgs*8), SP // room for args + + // Fast version, do not store args on the stack. + CMPL CX, $0; JE _0args + CMPL CX, $1; JE _1args + CMPL CX, $2; JE _2args + CMPL CX, $3; JE _3args + CMPL CX, $4; JE _4args + + // Check we have enough room for args. + CMPL CX, $const_MaxArgs + JLE 2(PC) + INT $3 // not enough room -> crash + + // Copy args to the stack. + MOVQ SP, DI + CLD + REP; MOVSQ + MOVQ SP, SI + + // Load first 4 args into correspondent registers. + // Floating point arguments are passed in the XMM + // registers. Set them here in case any of the arguments + // are floating point values. For details see + // https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention?view=msvc-170 +_4args: + MOVQ 24(SI), R9 + MOVQ R9, X3 +_3args: + MOVQ 16(SI), R8 + MOVQ R8, X2 +_2args: + MOVQ 8(SI), DX + MOVQ DX, X1 +_1args: + MOVQ 0(SI), CX + MOVQ CX, X0 +_0args: + + // Call stdcall function. + CALL AX + + ADDQ $(const_MaxArgs*8), SP + + // Return result. + MOVQ 0(SP), CX + MOVQ 8(SP), SP + MOVQ AX, StdCallInfo_R1(CX) + // Floating point return values are returned in XMM0. Setting r2 to this + // value in case this call returned a floating point value. For details, + // see https://docs.microsoft.com/en-us/cpp/build/x64-calling-convention + MOVQ X0, StdCallInfo_R2(CX) + + // GetLastError(). + MOVQ 0x30(GS), DI + MOVL 0x68(DI), AX + MOVQ AX, StdCallInfo_Err(CX) + + RET diff --git a/src/internal/runtime/syscall/windows/asm_windows_arm64.s b/src/internal/runtime/syscall/windows/asm_windows_arm64.s new file mode 100644 index 00000000000000..fb4cda0f830c77 --- /dev/null +++ b/src/internal/runtime/syscall/windows/asm_windows_arm64.s @@ -0,0 +1,90 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "textflag.h" + +// Offsets into Thread Environment Block (pointer in R18) +#define TEB_error 0x68 + +TEXT ·StdCall(SB),NOSPLIT,$0 + B ·asmstdcall(SB) + +TEXT ·asmstdcall(SB),NOSPLIT,$16 + STP (R19, R20), 16(RSP) // save old R19, R20 + MOVD R0, R19 // save fn pointer + MOVD RSP, R20 // save stack pointer + + // SetLastError(0) + MOVD $0, TEB_error(R18_PLATFORM) + MOVD StdCallInfo_Args(R19), R12 + + // Do we have more than 8 arguments? + MOVD StdCallInfo_N(R19), R0 + CMP $0, R0; BEQ _0args + CMP $1, R0; BEQ _1args + CMP $2, R0; BEQ _2args + CMP $3, R0; BEQ _3args + CMP $4, R0; BEQ _4args + CMP $5, R0; BEQ _5args + CMP $6, R0; BEQ _6args + CMP $7, R0; BEQ _7args + CMP $8, R0; BEQ _8args + + // Reserve stack space for remaining args + SUB $8, R0, R2 + ADD $1, R2, R3 // make even number of words for stack alignment + AND $~1, R3 + LSL $3, R3 + SUB R3, RSP + + // R4: size of stack arguments (n-8)*8 + // R5: &args[8] + // R6: loop counter, from 0 to (n-8)*8 + // R7: scratch + // R8: copy of RSP - (R2)(RSP) assembles as (R2)(ZR) + SUB $8, R0, R4 + LSL $3, R4 + ADD $(8*8), R12, R5 + MOVD $0, R6 + MOVD RSP, R8 +stackargs: + MOVD (R6)(R5), R7 + MOVD R7, (R6)(R8) + ADD $8, R6 + CMP R6, R4 + BNE stackargs + +_8args: + MOVD (7*8)(R12), R7 +_7args: + MOVD (6*8)(R12), R6 +_6args: + MOVD (5*8)(R12), R5 +_5args: + MOVD (4*8)(R12), R4 +_4args: + MOVD (3*8)(R12), R3 +_3args: + MOVD (2*8)(R12), R2 +_2args: + MOVD (1*8)(R12), R1 +_1args: + MOVD (0*8)(R12), R0 +_0args: + + MOVD StdCallInfo_Fn(R19), R12 + BL (R12) + + MOVD R20, RSP // free stack space + MOVD R0, StdCallInfo_R1(R19) // save return value + // TODO(rsc) floating point like amd64 in StdCallInfo_R2? + + // GetLastError + MOVD TEB_error(R18_PLATFORM), R0 + MOVD R0, StdCallInfo_Err(R19) + + // Restore callee-saved registers. + LDP 16(RSP), (R19, R20) + RET diff --git a/src/internal/runtime/syscall/windows/defs_windows.go b/src/internal/runtime/syscall/windows/defs_windows.go new file mode 100644 index 00000000000000..6b1098098c933a --- /dev/null +++ b/src/internal/runtime/syscall/windows/defs_windows.go @@ -0,0 +1,159 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Architecture-independent definitions. + +package windows + +// Pseudo handles. +const ( + CurrentProcess = ^uintptr(0) // -1 = current process + CurrentThread = ^uintptr(1) // -2 = current thread +) + +const INVALID_HANDLE_VALUE = ^uintptr(0) + +const DWORD_MAX = 0xffffffff + +const ( + PROT_NONE = 0 + PROT_READ = 1 + PROT_WRITE = 2 + PROT_EXEC = 4 +) + +const ( + MAP_ANON = 1 + MAP_PRIVATE = 2 +) + +const DUPLICATE_SAME_ACCESS = 0x2 + +const THREAD_PRIORITY_HIGHEST = 0x2 + +const ( + SIGINT = 0x2 + SIGTERM = 0xF +) + +const ( + CTRL_C_EVENT = 0x0 + CTRL_BREAK_EVENT = 0x1 + CTRL_CLOSE_EVENT = 0x2 + CTRL_LOGOFF_EVENT = 0x5 + CTRL_SHUTDOWN_EVENT = 0x6 +) + +const ( + EXCEPTION_ACCESS_VIOLATION = 0xc0000005 + EXCEPTION_IN_PAGE_ERROR = 0xc0000006 + EXCEPTION_BREAKPOINT = 0x80000003 + EXCEPTION_ILLEGAL_INSTRUCTION = 0xc000001d + EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d + EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e + EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f + EXCEPTION_FLT_OVERFLOW = 0xc0000091 + EXCEPTION_FLT_UNDERFLOW = 0xc0000093 + EXCEPTION_INT_DIVIDE_BY_ZERO = 0xc0000094 + EXCEPTION_INT_OVERFLOW = 0xc0000095 +) + +const ( + SEM_FAILCRITICALERRORS = 0x0001 + SEM_NOGPFAULTERRORBOX = 0x0002 + SEM_NOOPENFILEERRORBOX = 0x8000 +) + +const WER_FAULT_REPORTING_NO_UI = 0x0020 + +const INFINITE = 0xffffffff + +const WAIT_TIMEOUT = 258 + +const FAIL_FAST_GENERATE_EXCEPTION_ADDRESS = 0x1 + +const ( + EXCEPTION_CONTINUE_EXECUTION = -0x1 + EXCEPTION_CONTINUE_SEARCH = 0x0 + EXCEPTION_CONTINUE_SEARCH_SEH = 0x1 +) + +const CREATE_WAITABLE_TIMER_HIGH_RESOLUTION = 0x00000002 + +const ( + SYNCHRONIZE = 0x00100000 + TIMER_QUERY_STATE = 0x0001 + TIMER_MODIFY_STATE = 0x0002 +) + +// https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-erref/596a1078-e883-4972-9bbc-49e60bebca55 +const ( + STATUS_SUCCESS = 0x00000000 + STATUS_PENDING = 0x00000103 + STATUS_CANCELLED = 0xC0000120 +) + +// https://learn.microsoft.com/en-us/windows/win32/api/sysinfoapi/ns-sysinfoapi-system_info +type SystemInfo struct { + ProcessorArchitecture uint16 + Reserved uint16 + PageSize uint32 + MinimumApplicationAddress *byte + MaximumApplicationAddress *byte + ActiveProcessorMask uintptr + NumberOfProcessors uint32 + ProcessorType uint32 + AllocationGranularity uint32 + ProcessorLevel uint16 + ProcessorRevision uint16 +} + +// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_pointers +type ExceptionPointers struct { + Record *ExceptionRecord + Context *Context +} + +// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record +type ExceptionRecord struct { + ExceptionCode uint32 + ExceptionFlags uint32 + ExceptionRecord *ExceptionRecord + ExceptionAddress uintptr + NumberParameters uint32 + ExceptionInformation [15]uintptr +} + +type Handle uintptr + +// https://learn.microsoft.com/en-us/windows/win32/api/minwinbase/ns-minwinbase-overlapped +type Overlapped struct { + Internal uintptr + InternalHigh uintptr + Offset uint32 + OffsetHigh uint32 + HEvent Handle +} + +// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-memory_basic_information +type MemoryBasicInformation struct { + BaseAddress uintptr + AllocationBase uintptr + AllocationProtect uint32 + PartitionId uint16 + RegionSize uintptr + State uint32 + Protect uint32 + Type uint32 +} + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_osversioninfow +type OSVERSIONINFOW struct { + OSVersionInfoSize uint32 + MajorVersion uint32 + MinorVersion uint32 + BuildNumber uint32 + PlatformID uint32 + CSDVersion [128]uint16 +} diff --git a/src/internal/runtime/syscall/windows/defs_windows_386.go b/src/internal/runtime/syscall/windows/defs_windows_386.go new file mode 100644 index 00000000000000..6c2271efb0f544 --- /dev/null +++ b/src/internal/runtime/syscall/windows/defs_windows_386.go @@ -0,0 +1,79 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package windows + +import ( + "internal/goarch" + "unsafe" +) + +const CONTEXT_CONTROL = 0x10001 + +type FloatingSaveArea struct { + ControlWord uint32 + StatusWord uint32 + TagWord uint32 + ErrorOffset uint32 + ErrorSelector uint32 + DataOffset uint32 + DataSelector uint32 + RegisterArea [80]uint8 + Cr0NpxState uint32 +} + +type Context struct { + ContextFlags uint32 + Dr0 uint32 + Dr1 uint32 + Dr2 uint32 + Dr3 uint32 + Dr6 uint32 + Dr7 uint32 + FloatingSaveArea FloatingSaveArea + SegGs uint32 + SegFs uint32 + SegEs uint32 + SegDs uint32 + Edi uint32 + Esi uint32 + Ebx uint32 + Edx uint32 + Ecx uint32 + Eax uint32 + Ebp uint32 + Eip uint32 + SegCs uint32 + EFlags uint32 + Esp uint32 + SegSs uint32 + ExtendedRegisters [512]uint8 +} + +func (c *Context) PC() uintptr { return uintptr(c.Eip) } +func (c *Context) SP() uintptr { return uintptr(c.Esp) } + +// 386 does not have link register, so this returns 0. +func (c *Context) LR() uintptr { return 0 } +func (c *Context) SetLR(x uintptr) {} + +func (c *Context) SetPC(x uintptr) { c.Eip = uint32(x) } +func (c *Context) SetSP(x uintptr) { c.Esp = uint32(x) } + +// 386 does not have frame pointer register. +func (c *Context) SetFP(x uintptr) {} + +func (c *Context) PushCall(targetPC, resumePC uintptr) { + sp := c.SP() - goarch.StackAlign + *(*uintptr)(unsafe.Pointer(sp)) = resumePC + c.SetSP(sp) + c.SetPC(targetPC) +} + +// DISPATCHER_CONTEXT is not defined on 386. +type DISPATCHER_CONTEXT struct{} + +func (c *DISPATCHER_CONTEXT) Ctx() *Context { + return nil +} diff --git a/src/internal/runtime/syscall/windows/defs_windows_amd64.go b/src/internal/runtime/syscall/windows/defs_windows_amd64.go new file mode 100644 index 00000000000000..018124488b7a95 --- /dev/null +++ b/src/internal/runtime/syscall/windows/defs_windows_amd64.go @@ -0,0 +1,99 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package windows + +import ( + "internal/goarch" + "unsafe" +) + +const CONTEXT_CONTROL = 0x100001 + +type M128 struct { + Low uint64 + High int64 +} + +type Context struct { + P1Home uint64 + P2Home uint64 + P3Home uint64 + P4Home uint64 + P5Home uint64 + P6Home uint64 + ContextFlags uint32 + MxCsr uint32 + SegCs uint16 + SegDs uint16 + SegEs uint16 + SegFs uint16 + SegGs uint16 + SegSs uint16 + EFlags uint32 + DR0 uint64 + DR1 uint64 + DR2 uint64 + DR3 uint64 + DR6 uint64 + DR7 uint64 + Rax uint64 + Rcx uint64 + Rdx uint64 + Rbx uint64 + Rsp uint64 + Rbp uint64 + Rsi uint64 + Rdi uint64 + R8 uint64 + R9 uint64 + R10 uint64 + R11 uint64 + R12 uint64 + R13 uint64 + R14 uint64 + R15 uint64 + Rip uint64 + _ [512]byte + VectorRegister [26]M128 + VectorControl uint64 + DebugControl uint64 + LastBranchToRip uint64 + LastBranchFromRip uint64 + LastExceptionToRip uint64 + LastExceptionFromRip uint64 +} + +func (c *Context) PC() uintptr { return uintptr(c.Rip) } +func (c *Context) SP() uintptr { return uintptr(c.Rsp) } + +// AMD64 does not have link register, so this returns 0. +func (c *Context) LR() uintptr { return 0 } +func (c *Context) SetLR(x uintptr) {} + +func (c *Context) SetPC(x uintptr) { c.Rip = uint64(x) } +func (c *Context) SetSP(x uintptr) { c.Rsp = uint64(x) } +func (c *Context) SetFP(x uintptr) { c.Rbp = uint64(x) } + +func (c *Context) PushCall(targetPC, resumePC uintptr) { + sp := c.SP() - goarch.StackAlign + *(*uintptr)(unsafe.Pointer(sp)) = resumePC + c.SetSP(sp) + c.SetPC(targetPC) +} + +type DISPATCHER_CONTEXT struct { + ControlPc uint64 + ImageBase uint64 + FunctionEntry uintptr + EstablisherFrame uint64 + TargetIp uint64 + Context *Context + LanguageHandler uintptr + HandlerData uintptr +} + +func (c *DISPATCHER_CONTEXT) Ctx() *Context { + return c.Context +} diff --git a/src/internal/runtime/syscall/windows/defs_windows_arm64.go b/src/internal/runtime/syscall/windows/defs_windows_arm64.go new file mode 100644 index 00000000000000..c05573a53113bf --- /dev/null +++ b/src/internal/runtime/syscall/windows/defs_windows_arm64.go @@ -0,0 +1,75 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package windows + +import ( + "internal/goarch" + "unsafe" +) + +// NOTE(rsc): CONTEXT_CONTROL is actually 0x400001 and should include PC, SP, and LR. +// However, empirically, LR doesn't come along on Windows 10 +// unless you also set CONTEXT_INTEGER (0x400002). +// Without LR, we skip over the next-to-bottom function in profiles +// when the bottom function is frameless. +// So we set both here, to make a working CONTEXT_CONTROL. +const CONTEXT_CONTROL = 0x400003 + +type Neon128 struct { + Low uint64 + High int64 +} + +// See https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-arm64_nt_context +type Context struct { + ContextFlags uint32 + Cpsr uint32 + X [31]uint64 // fp is x[29], lr is x[30] + XSp uint64 + Pc uint64 + V [32]Neon128 + Fpcr uint32 + Fpsr uint32 + Bcr [8]uint32 + Bvr [8]uint64 + Wcr [2]uint32 + Wvr [2]uint64 +} + +func (c *Context) PC() uintptr { return uintptr(c.Pc) } +func (c *Context) SP() uintptr { return uintptr(c.XSp) } +func (c *Context) LR() uintptr { return uintptr(c.X[30]) } + +func (c *Context) SetPC(x uintptr) { c.Pc = uint64(x) } +func (c *Context) SetSP(x uintptr) { c.XSp = uint64(x) } +func (c *Context) SetLR(x uintptr) { c.X[30] = uint64(x) } +func (c *Context) SetFP(x uintptr) { c.X[29] = uint64(x) } + +func (c *Context) PushCall(targetPC, resumePC uintptr) { + // Push LR. The injected call is responsible + // for restoring LR. gentraceback is aware of + // this extra slot. See sigctxt.pushCall in + // signal_arm64.go. + sp := c.SP() - goarch.StackAlign + c.SetSP(sp) + *(*uint64)(unsafe.Pointer(sp)) = uint64(c.LR()) + c.SetLR(resumePC) + c.SetPC(targetPC) +} + +type DISPATCHER_CONTEXT struct { + ControlPc uint64 + ImageBase uint64 + FunctionEntry uintptr + EstablisherFrame uint64 + TargetIp uint64 + Context *Context + LanguageHandler uintptr + HandlerData uintptr +} + +func (c *DISPATCHER_CONTEXT) Ctx() *Context { + return c.Context +} diff --git a/src/internal/runtime/syscall/windows/syscall_windows.go b/src/internal/runtime/syscall/windows/syscall_windows.go new file mode 100644 index 00000000000000..0d350f0d7f139b --- /dev/null +++ b/src/internal/runtime/syscall/windows/syscall_windows.go @@ -0,0 +1,44 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package windows provides the syscall primitives required for the runtime. + +package windows + +import ( + "internal/abi" +) + +// MaxArgs should be divisible by 2, as Windows stack +// must be kept 16-byte aligned on syscall entry. +// +// Although it only permits maximum 42 parameters, it +// is arguably large enough. +const MaxArgs = 42 + +// StdCallInfo is a structure used to pass parameters to the system call. +type StdCallInfo struct { + Fn uintptr + N uintptr // number of parameters + Args uintptr // parameters + R1 uintptr // return values + R2 uintptr + Err uintptr // error number +} + +// StdCall calls a function using Windows' stdcall convention. +// +//go:noescape +func StdCall(fn *StdCallInfo) + +// asmstdcall is the function pointer for [AsmStdCallAddr]. +func asmstdcall(fn *StdCallInfo) + +// AsmStdCallAddr is the address of a function that accepts a pointer +// to [StdCallInfo] stored on the stack following the C calling convention, +// and calls the function using Windows' stdcall calling convention. +// Shouldn't be called directly from Go. +func AsmStdCallAddr() uintptr { + return abi.FuncPCABI0(asmstdcall) +} diff --git a/src/runtime/internal/wasitest/host_test.go b/src/internal/runtime/wasitest/host_test.go similarity index 100% rename from src/runtime/internal/wasitest/host_test.go rename to src/internal/runtime/wasitest/host_test.go diff --git a/src/runtime/internal/wasitest/nonblock_test.go b/src/internal/runtime/wasitest/nonblock_test.go similarity index 95% rename from src/runtime/internal/wasitest/nonblock_test.go rename to src/internal/runtime/wasitest/nonblock_test.go index 3072b96ed81343..f5f6bb84d0ef11 100644 --- a/src/runtime/internal/wasitest/nonblock_test.go +++ b/src/internal/runtime/wasitest/nonblock_test.go @@ -10,6 +10,7 @@ package wasi_test import ( "bufio" "fmt" + "internal/testenv" "io" "math/rand" "os" @@ -41,6 +42,8 @@ func TestNonblock(t *testing.T) { t.Skip("wasmer does not support non-blocking I/O") } + testenv.MustHaveGoRun(t) + for _, mode := range []string{"os.OpenFile", "os.NewFile"} { t.Run(mode, func(t *testing.T) { args := []string{"run", "./testdata/nonblock.go", mode} @@ -62,7 +65,7 @@ func TestNonblock(t *testing.T) { fifos[len(fifos)-i-1] = &fifo{file, path} } - subProcess := exec.Command("go", args...) + subProcess := exec.Command(testenv.GoToolPath(t), args...) subProcess.Env = append(os.Environ(), "GOOS=wasip1", "GOARCH=wasm") diff --git a/src/runtime/internal/wasitest/tcpecho_test.go b/src/internal/runtime/wasitest/tcpecho_test.go similarity index 94% rename from src/runtime/internal/wasitest/tcpecho_test.go rename to src/internal/runtime/wasitest/tcpecho_test.go index bbcea903106914..aa57d9e2e340a3 100644 --- a/src/runtime/internal/wasitest/tcpecho_test.go +++ b/src/internal/runtime/wasitest/tcpecho_test.go @@ -7,6 +7,7 @@ package wasi_test import ( "bytes" "fmt" + "internal/testenv" "math/rand" "net" "os" @@ -20,6 +21,8 @@ func TestTCPEcho(t *testing.T) { t.Skip() } + testenv.MustHaveGoRun(t) + // We're unable to use port 0 here (let the OS choose a spare port). // Although the WASM runtime accepts port 0, and the WASM module listens // successfully, there's no way for this test to query the selected port @@ -44,7 +47,7 @@ func TestTCPEcho(t *testing.T) { port++ } - subProcess := exec.Command("go", "run", "./testdata/tcpecho.go") + subProcess := exec.Command(testenv.GoToolPath(t), "run", "./testdata/tcpecho.go") subProcess.Env = append(os.Environ(), "GOOS=wasip1", "GOARCH=wasm") diff --git a/src/runtime/internal/wasitest/testdata/nonblock.go b/src/internal/runtime/wasitest/testdata/nonblock.go similarity index 100% rename from src/runtime/internal/wasitest/testdata/nonblock.go rename to src/internal/runtime/wasitest/testdata/nonblock.go diff --git a/src/runtime/internal/wasitest/testdata/tcpecho.go b/src/internal/runtime/wasitest/testdata/tcpecho.go similarity index 95% rename from src/runtime/internal/wasitest/testdata/tcpecho.go rename to src/internal/runtime/wasitest/testdata/tcpecho.go index 819e3526885642..6da56acba10a1c 100644 --- a/src/runtime/internal/wasitest/testdata/tcpecho.go +++ b/src/internal/runtime/wasitest/testdata/tcpecho.go @@ -62,8 +62,7 @@ func findListener() (net.Listener, error) { l, err := net.FileListener(f) f.Close() - var se syscall.Errno - switch errors.As(err, &se); se { + switch se, _ := errors.AsType[syscall.Errno](err); se { case syscall.ENOTSOCK: continue case syscall.EBADF: diff --git a/src/internal/stringslite/strings.go b/src/internal/stringslite/strings.go index 4114b861300066..3a09e08cf4f7bd 100644 --- a/src/internal/stringslite/strings.go +++ b/src/internal/stringslite/strings.go @@ -14,7 +14,7 @@ import ( ) func HasPrefix(s, prefix string) bool { - return len(s) >= len(prefix) && s[0:len(prefix)] == prefix + return len(s) >= len(prefix) && s[:len(prefix)] == prefix } func HasSuffix(s, suffix string) bool { diff --git a/src/internal/sync/export_test.go b/src/internal/sync/export_test.go new file mode 100644 index 00000000000000..c6449bf3e0aa8b --- /dev/null +++ b/src/internal/sync/export_test.go @@ -0,0 +1,38 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync + +import ( + "internal/abi" + "unsafe" +) + +// NewBadHashTrieMap creates a new HashTrieMap for the provided key and value +// but with an intentionally bad hash function. +func NewBadHashTrieMap[K, V comparable]() *HashTrieMap[K, V] { + // Stub out the good hash function with a terrible one. + // Everything should still work as expected. + var m HashTrieMap[K, V] + m.init() + m.keyHash = func(_ unsafe.Pointer, _ uintptr) uintptr { + return 0 + } + return &m +} + +// NewTruncHashTrieMap creates a new HashTrieMap for the provided key and value +// but with an intentionally bad hash function. +func NewTruncHashTrieMap[K, V comparable]() *HashTrieMap[K, V] { + // Stub out the good hash function with a terrible one. + // Everything should still work as expected. + var m HashTrieMap[K, V] + var mx map[string]int + mapType := abi.TypeOf(mx).MapType() + hasher := mapType.Hasher + m.keyHash = func(p unsafe.Pointer, n uintptr) uintptr { + return hasher(p, n) & ((uintptr(1) << 4) - 1) + } + return &m +} diff --git a/src/internal/sync/hashtriemap.go b/src/internal/sync/hashtriemap.go new file mode 100644 index 00000000000000..db832974278f16 --- /dev/null +++ b/src/internal/sync/hashtriemap.go @@ -0,0 +1,724 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync + +import ( + "internal/abi" + "internal/goarch" + "sync/atomic" + "unsafe" +) + +// HashTrieMap is an implementation of a concurrent hash-trie. The implementation +// is designed around frequent loads, but offers decent performance for stores +// and deletes as well, especially if the map is larger. Its primary use-case is +// the unique package, but can be used elsewhere as well. +// +// The zero HashTrieMap is empty and ready to use. +// It must not be copied after first use. +type HashTrieMap[K comparable, V any] struct { + inited atomic.Uint32 + initMu Mutex + root atomic.Pointer[indirect[K, V]] + keyHash hashFunc + valEqual equalFunc + seed uintptr +} + +func (ht *HashTrieMap[K, V]) init() { + if ht.inited.Load() == 0 { + ht.initSlow() + } +} + +//go:noinline +func (ht *HashTrieMap[K, V]) initSlow() { + ht.initMu.Lock() + defer ht.initMu.Unlock() + + if ht.inited.Load() != 0 { + // Someone got to it while we were waiting. + return + } + + // Set up root node, derive the hash function for the key, and the + // equal function for the value, if any. + var m map[K]V + mapType := abi.TypeOf(m).MapType() + ht.root.Store(newIndirectNode[K, V](nil)) + ht.keyHash = mapType.Hasher + ht.valEqual = mapType.Elem.Equal + ht.seed = uintptr(runtime_rand()) + + ht.inited.Store(1) +} + +type hashFunc func(unsafe.Pointer, uintptr) uintptr +type equalFunc func(unsafe.Pointer, unsafe.Pointer) bool + +// Load returns the value stored in the map for a key, or nil if no +// value is present. +// The ok result indicates whether value was found in the map. +func (ht *HashTrieMap[K, V]) Load(key K) (value V, ok bool) { + ht.init() + hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) + + i := ht.root.Load() + hashShift := 8 * goarch.PtrSize + for hashShift != 0 { + hashShift -= nChildrenLog2 + + n := i.children[(hash>>hashShift)&nChildrenMask].Load() + if n == nil { + return *new(V), false + } + if n.isEntry { + return n.entry().lookup(key) + } + i = n.indirect() + } + panic("internal/sync.HashTrieMap: ran out of hash bits while iterating") +} + +// LoadOrStore returns the existing value for the key if present. +// Otherwise, it stores and returns the given value. +// The loaded result is true if the value was loaded, false if stored. +func (ht *HashTrieMap[K, V]) LoadOrStore(key K, value V) (result V, loaded bool) { + ht.init() + hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) + var i *indirect[K, V] + var hashShift uint + var slot *atomic.Pointer[node[K, V]] + var n *node[K, V] + for { + // Find the key or a candidate location for insertion. + i = ht.root.Load() + hashShift = 8 * goarch.PtrSize + haveInsertPoint := false + for hashShift != 0 { + hashShift -= nChildrenLog2 + + slot = &i.children[(hash>>hashShift)&nChildrenMask] + n = slot.Load() + if n == nil { + // We found a nil slot which is a candidate for insertion. + haveInsertPoint = true + break + } + if n.isEntry { + // We found an existing entry, which is as far as we can go. + // If it stays this way, we'll have to replace it with an + // indirect node. + if v, ok := n.entry().lookup(key); ok { + return v, true + } + haveInsertPoint = true + break + } + i = n.indirect() + } + if !haveInsertPoint { + panic("internal/sync.HashTrieMap: ran out of hash bits while iterating") + } + + // Grab the lock and double-check what we saw. + i.mu.Lock() + n = slot.Load() + if (n == nil || n.isEntry) && !i.dead.Load() { + // What we saw is still true, so we can continue with the insert. + break + } + // We have to start over. + i.mu.Unlock() + } + // N.B. This lock is held from when we broke out of the outer loop above. + // We specifically break this out so that we can use defer here safely. + // One option is to break this out into a new function instead, but + // there's so much local iteration state used below that this turns out + // to be cleaner. + defer i.mu.Unlock() + + var oldEntry *entry[K, V] + if n != nil { + oldEntry = n.entry() + if v, ok := oldEntry.lookup(key); ok { + // Easy case: by loading again, it turns out exactly what we wanted is here! + return v, true + } + } + newEntry := newEntryNode(key, value) + if oldEntry == nil { + // Easy case: create a new entry and store it. + slot.Store(&newEntry.node) + } else { + // We possibly need to expand the entry already there into one or more new nodes. + // + // Publish the node last, which will make both oldEntry and newEntry visible. We + // don't want readers to be able to observe that oldEntry isn't in the tree. + slot.Store(ht.expand(oldEntry, newEntry, hash, hashShift, i)) + } + return value, false +} + +// expand takes oldEntry and newEntry whose hashes conflict from bit 64 down to hashShift and +// produces a subtree of indirect nodes to hold the two new entries. +func (ht *HashTrieMap[K, V]) expand(oldEntry, newEntry *entry[K, V], newHash uintptr, hashShift uint, parent *indirect[K, V]) *node[K, V] { + // Check for a hash collision. + oldHash := ht.keyHash(unsafe.Pointer(&oldEntry.key), ht.seed) + if oldHash == newHash { + // Store the old entry in the new entry's overflow list, then store + // the new entry. + newEntry.overflow.Store(oldEntry) + return &newEntry.node + } + // We have to add an indirect node. Worse still, we may need to add more than one. + newIndirect := newIndirectNode(parent) + top := newIndirect + for { + if hashShift == 0 { + panic("internal/sync.HashTrieMap: ran out of hash bits while inserting (incorrect use of unsafe or cgo, or data race?)") + } + hashShift -= nChildrenLog2 // hashShift is for the level parent is at. We need to go deeper. + oi := (oldHash >> hashShift) & nChildrenMask + ni := (newHash >> hashShift) & nChildrenMask + if oi != ni { + newIndirect.children[oi].Store(&oldEntry.node) + newIndirect.children[ni].Store(&newEntry.node) + break + } + nextIndirect := newIndirectNode(newIndirect) + newIndirect.children[oi].Store(&nextIndirect.node) + newIndirect = nextIndirect + } + return &top.node +} + +// Store sets the value for a key. +func (ht *HashTrieMap[K, V]) Store(key K, new V) { + _, _ = ht.Swap(key, new) +} + +// Swap swaps the value for a key and returns the previous value if any. +// The loaded result reports whether the key was present. +func (ht *HashTrieMap[K, V]) Swap(key K, new V) (previous V, loaded bool) { + ht.init() + hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) + var i *indirect[K, V] + var hashShift uint + var slot *atomic.Pointer[node[K, V]] + var n *node[K, V] + for { + // Find the key or a candidate location for insertion. + i = ht.root.Load() + hashShift = 8 * goarch.PtrSize + haveInsertPoint := false + for hashShift != 0 { + hashShift -= nChildrenLog2 + + slot = &i.children[(hash>>hashShift)&nChildrenMask] + n = slot.Load() + if n == nil || n.isEntry { + // We found a nil slot which is a candidate for insertion, + // or an existing entry that we'll replace. + haveInsertPoint = true + break + } + i = n.indirect() + } + if !haveInsertPoint { + panic("internal/sync.HashTrieMap: ran out of hash bits while iterating") + } + + // Grab the lock and double-check what we saw. + i.mu.Lock() + n = slot.Load() + if (n == nil || n.isEntry) && !i.dead.Load() { + // What we saw is still true, so we can continue with the insert. + break + } + // We have to start over. + i.mu.Unlock() + } + // N.B. This lock is held from when we broke out of the outer loop above. + // We specifically break this out so that we can use defer here safely. + // One option is to break this out into a new function instead, but + // there's so much local iteration state used below that this turns out + // to be cleaner. + defer i.mu.Unlock() + + var zero V + var oldEntry *entry[K, V] + if n != nil { + // Swap if the keys compare. + oldEntry = n.entry() + newEntry, old, swapped := oldEntry.swap(key, new) + if swapped { + slot.Store(&newEntry.node) + return old, true + } + } + // The keys didn't compare, so we're doing an insertion. + newEntry := newEntryNode(key, new) + if oldEntry == nil { + // Easy case: create a new entry and store it. + slot.Store(&newEntry.node) + } else { + // We possibly need to expand the entry already there into one or more new nodes. + // + // Publish the node last, which will make both oldEntry and newEntry visible. We + // don't want readers to be able to observe that oldEntry isn't in the tree. + slot.Store(ht.expand(oldEntry, newEntry, hash, hashShift, i)) + } + return zero, false +} + +// CompareAndSwap swaps the old and new values for key +// if the value stored in the map is equal to old. +// The value type must be of a comparable type, otherwise CompareAndSwap will panic. +func (ht *HashTrieMap[K, V]) CompareAndSwap(key K, old, new V) (swapped bool) { + ht.init() + if ht.valEqual == nil { + panic("called CompareAndSwap when value is not of comparable type") + } + hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) + + // Find a node with the key and compare with it. n != nil if we found the node. + i, _, slot, n := ht.find(key, hash, ht.valEqual, old) + if i != nil { + defer i.mu.Unlock() + } + if n == nil { + return false + } + + // Try to swap the entry. + e, swapped := n.entry().compareAndSwap(key, old, new, ht.valEqual) + if !swapped { + // Nothing was actually swapped, which means the node is no longer there. + return false + } + // Store the entry back because it changed. + slot.Store(&e.node) + return true +} + +// LoadAndDelete deletes the value for a key, returning the previous value if any. +// The loaded result reports whether the key was present. +func (ht *HashTrieMap[K, V]) LoadAndDelete(key K) (value V, loaded bool) { + ht.init() + hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) + + // Find a node with the key and compare with it. n != nil if we found the node. + i, hashShift, slot, n := ht.find(key, hash, nil, *new(V)) + if n == nil { + if i != nil { + i.mu.Unlock() + } + return *new(V), false + } + + // Try to delete the entry. + v, e, loaded := n.entry().loadAndDelete(key) + if !loaded { + // Nothing was actually deleted, which means the node is no longer there. + i.mu.Unlock() + return *new(V), false + } + if e != nil { + // We didn't actually delete the whole entry, just one entry in the chain. + // Nothing else to do, since the parent is definitely not empty. + slot.Store(&e.node) + i.mu.Unlock() + return v, true + } + // Delete the entry. + slot.Store(nil) + + // Check if the node is now empty (and isn't the root), and delete it if able. + for i.parent != nil && i.empty() { + if hashShift == 8*goarch.PtrSize { + panic("internal/sync.HashTrieMap: ran out of hash bits while iterating") + } + hashShift += nChildrenLog2 + + // Delete the current node in the parent. + parent := i.parent + parent.mu.Lock() + i.dead.Store(true) + parent.children[(hash>>hashShift)&nChildrenMask].Store(nil) + i.mu.Unlock() + i = parent + } + i.mu.Unlock() + return v, true +} + +// Delete deletes the value for a key. +func (ht *HashTrieMap[K, V]) Delete(key K) { + _, _ = ht.LoadAndDelete(key) +} + +// CompareAndDelete deletes the entry for key if its value is equal to old. +// The value type must be comparable, otherwise this CompareAndDelete will panic. +// +// If there is no current value for key in the map, CompareAndDelete returns false +// (even if the old value is the nil interface value). +func (ht *HashTrieMap[K, V]) CompareAndDelete(key K, old V) (deleted bool) { + ht.init() + if ht.valEqual == nil { + panic("called CompareAndDelete when value is not of comparable type") + } + hash := ht.keyHash(abi.NoEscape(unsafe.Pointer(&key)), ht.seed) + + // Find a node with the key. n != nil if we found the node. + i, hashShift, slot, n := ht.find(key, hash, nil, *new(V)) + if n == nil { + if i != nil { + i.mu.Unlock() + } + return false + } + + // Try to delete the entry. + e, deleted := n.entry().compareAndDelete(key, old, ht.valEqual) + if !deleted { + // Nothing was actually deleted, which means the node is no longer there. + i.mu.Unlock() + return false + } + if e != nil { + // We didn't actually delete the whole entry, just one entry in the chain. + // Nothing else to do, since the parent is definitely not empty. + slot.Store(&e.node) + i.mu.Unlock() + return true + } + // Delete the entry. + slot.Store(nil) + + // Check if the node is now empty (and isn't the root), and delete it if able. + for i.parent != nil && i.empty() { + if hashShift == 8*goarch.PtrSize { + panic("internal/sync.HashTrieMap: ran out of hash bits while iterating") + } + hashShift += nChildrenLog2 + + // Delete the current node in the parent. + parent := i.parent + parent.mu.Lock() + i.dead.Store(true) + parent.children[(hash>>hashShift)&nChildrenMask].Store(nil) + i.mu.Unlock() + i = parent + } + i.mu.Unlock() + return true +} + +// find searches the tree for a node that contains key (hash must be the hash of key). +// If valEqual != nil, then it will also enforce that the values are equal as well. +// +// Returns a non-nil node, which will always be an entry, if found. +// +// If i != nil then i.mu is locked, and it is the caller's responsibility to unlock it. +func (ht *HashTrieMap[K, V]) find(key K, hash uintptr, valEqual equalFunc, value V) (i *indirect[K, V], hashShift uint, slot *atomic.Pointer[node[K, V]], n *node[K, V]) { + for { + // Find the key or return if it's not there. + i = ht.root.Load() + hashShift = 8 * goarch.PtrSize + found := false + for hashShift != 0 { + hashShift -= nChildrenLog2 + + slot = &i.children[(hash>>hashShift)&nChildrenMask] + n = slot.Load() + if n == nil { + // Nothing to compare with. Give up. + i = nil + return + } + if n.isEntry { + // We found an entry. Check if it matches. + if _, ok := n.entry().lookupWithValue(key, value, valEqual); !ok { + // No match, comparison failed. + i = nil + n = nil + return + } + // We've got a match. Prepare to perform an operation on the key. + found = true + break + } + i = n.indirect() + } + if !found { + panic("internal/sync.HashTrieMap: ran out of hash bits while iterating") + } + + // Grab the lock and double-check what we saw. + i.mu.Lock() + n = slot.Load() + if !i.dead.Load() && (n == nil || n.isEntry) { + // Either we've got a valid node or the node is now nil under the lock. + // In either case, we're done here. + return + } + // We have to start over. + i.mu.Unlock() + } +} + +// All returns an iterator over each key and value present in the map. +// +// The iterator does not necessarily correspond to any consistent snapshot of the +// HashTrieMap's contents: no key will be visited more than once, but if the value +// for any key is stored or deleted concurrently (including by yield), the iterator +// may reflect any mapping for that key from any point during iteration. The iterator +// does not block other methods on the receiver; even yield itself may call any +// method on the HashTrieMap. +func (ht *HashTrieMap[K, V]) All() func(yield func(K, V) bool) { + ht.init() + return func(yield func(key K, value V) bool) { + ht.iter(ht.root.Load(), yield) + } +} + +// Range calls f sequentially for each key and value present in the map. +// If f returns false, range stops the iteration. +// +// This exists for compatibility with sync.Map; All should be preferred. +// It provides the same guarantees as sync.Map, and All. +func (ht *HashTrieMap[K, V]) Range(yield func(K, V) bool) { + ht.init() + ht.iter(ht.root.Load(), yield) +} + +func (ht *HashTrieMap[K, V]) iter(i *indirect[K, V], yield func(key K, value V) bool) bool { + for j := range i.children { + n := i.children[j].Load() + if n == nil { + continue + } + if !n.isEntry { + if !ht.iter(n.indirect(), yield) { + return false + } + continue + } + e := n.entry() + for e != nil { + if !yield(e.key, e.value) { + return false + } + e = e.overflow.Load() + } + } + return true +} + +// Clear deletes all the entries, resulting in an empty HashTrieMap. +func (ht *HashTrieMap[K, V]) Clear() { + ht.init() + + // It's sufficient to just drop the root on the floor, but the root + // must always be non-nil. + ht.root.Store(newIndirectNode[K, V](nil)) +} + +const ( + // 16 children. This seems to be the sweet spot for + // load performance: any smaller and we lose out on + // 50% or more in CPU performance. Any larger and the + // returns are minuscule (~1% improvement for 32 children). + nChildrenLog2 = 4 + nChildren = 1 << nChildrenLog2 + nChildrenMask = nChildren - 1 +) + +// indirect is an internal node in the hash-trie. +type indirect[K comparable, V any] struct { + node[K, V] + dead atomic.Bool + mu Mutex // Protects mutation to children and any children that are entry nodes. + parent *indirect[K, V] + children [nChildren]atomic.Pointer[node[K, V]] +} + +func newIndirectNode[K comparable, V any](parent *indirect[K, V]) *indirect[K, V] { + return &indirect[K, V]{node: node[K, V]{isEntry: false}, parent: parent} +} + +func (i *indirect[K, V]) empty() bool { + nc := 0 + for j := range i.children { + if i.children[j].Load() != nil { + nc++ + } + } + return nc == 0 +} + +// entry is a leaf node in the hash-trie. +type entry[K comparable, V any] struct { + node[K, V] + overflow atomic.Pointer[entry[K, V]] // Overflow for hash collisions. + key K + value V +} + +func newEntryNode[K comparable, V any](key K, value V) *entry[K, V] { + return &entry[K, V]{ + node: node[K, V]{isEntry: true}, + key: key, + value: value, + } +} + +func (e *entry[K, V]) lookup(key K) (V, bool) { + for e != nil { + if e.key == key { + return e.value, true + } + e = e.overflow.Load() + } + return *new(V), false +} + +func (e *entry[K, V]) lookupWithValue(key K, value V, valEqual equalFunc) (V, bool) { + for e != nil { + if e.key == key && (valEqual == nil || valEqual(unsafe.Pointer(&e.value), abi.NoEscape(unsafe.Pointer(&value)))) { + return e.value, true + } + e = e.overflow.Load() + } + return *new(V), false +} + +// swap replaces an entry in the overflow chain if keys compare equal. Returns the new entry chain, +// the old value, and whether or not anything was swapped. +// +// swap must be called under the mutex of the indirect node which e is a child of. +func (head *entry[K, V]) swap(key K, new V) (*entry[K, V], V, bool) { + if head.key == key { + // Return the new head of the list. + e := newEntryNode(key, new) + if chain := head.overflow.Load(); chain != nil { + e.overflow.Store(chain) + } + return e, head.value, true + } + i := &head.overflow + e := i.Load() + for e != nil { + if e.key == key { + eNew := newEntryNode(key, new) + eNew.overflow.Store(e.overflow.Load()) + i.Store(eNew) + return head, e.value, true + } + i = &e.overflow + e = e.overflow.Load() + } + var zero V + return head, zero, false +} + +// compareAndSwap replaces an entry in the overflow chain if both the key and value compare +// equal. Returns the new entry chain and whether or not anything was swapped. +// +// compareAndSwap must be called under the mutex of the indirect node which e is a child of. +func (head *entry[K, V]) compareAndSwap(key K, old, new V, valEqual equalFunc) (*entry[K, V], bool) { + if head.key == key && valEqual(unsafe.Pointer(&head.value), abi.NoEscape(unsafe.Pointer(&old))) { + // Return the new head of the list. + e := newEntryNode(key, new) + if chain := head.overflow.Load(); chain != nil { + e.overflow.Store(chain) + } + return e, true + } + i := &head.overflow + e := i.Load() + for e != nil { + if e.key == key && valEqual(unsafe.Pointer(&e.value), abi.NoEscape(unsafe.Pointer(&old))) { + eNew := newEntryNode(key, new) + eNew.overflow.Store(e.overflow.Load()) + i.Store(eNew) + return head, true + } + i = &e.overflow + e = e.overflow.Load() + } + return head, false +} + +// loadAndDelete deletes an entry in the overflow chain by key. Returns the value for the key, the new +// entry chain and whether or not anything was loaded (and deleted). +// +// loadAndDelete must be called under the mutex of the indirect node which e is a child of. +func (head *entry[K, V]) loadAndDelete(key K) (V, *entry[K, V], bool) { + if head.key == key { + // Drop the head of the list. + return head.value, head.overflow.Load(), true + } + i := &head.overflow + e := i.Load() + for e != nil { + if e.key == key { + i.Store(e.overflow.Load()) + return e.value, head, true + } + i = &e.overflow + e = e.overflow.Load() + } + return *new(V), head, false +} + +// compareAndDelete deletes an entry in the overflow chain if both the key and value compare +// equal. Returns the new entry chain and whether or not anything was deleted. +// +// compareAndDelete must be called under the mutex of the indirect node which e is a child of. +func (head *entry[K, V]) compareAndDelete(key K, value V, valEqual equalFunc) (*entry[K, V], bool) { + if head.key == key && valEqual(unsafe.Pointer(&head.value), abi.NoEscape(unsafe.Pointer(&value))) { + // Drop the head of the list. + return head.overflow.Load(), true + } + i := &head.overflow + e := i.Load() + for e != nil { + if e.key == key && valEqual(unsafe.Pointer(&e.value), abi.NoEscape(unsafe.Pointer(&value))) { + i.Store(e.overflow.Load()) + return head, true + } + i = &e.overflow + e = e.overflow.Load() + } + return head, false +} + +// node is the header for a node. It's polymorphic and +// is actually either an entry or an indirect. +type node[K comparable, V any] struct { + isEntry bool +} + +func (n *node[K, V]) entry() *entry[K, V] { + if !n.isEntry { + panic("called entry on non-entry node") + } + return (*entry[K, V])(unsafe.Pointer(n)) +} + +func (n *node[K, V]) indirect() *indirect[K, V] { + if n.isEntry { + panic("called indirect on entry node") + } + return (*indirect[K, V])(unsafe.Pointer(n)) +} + +// Pull in runtime.rand so that we don't need to take a dependency +// on math/rand/v2. +// +//go:linkname runtime_rand runtime.rand +func runtime_rand() uint64 diff --git a/src/internal/concurrent/hashtriemap_bench_test.go b/src/internal/sync/hashtriemap_bench_test.go similarity index 89% rename from src/internal/concurrent/hashtriemap_bench_test.go rename to src/internal/sync/hashtriemap_bench_test.go index 32a263d540ca36..fc10d822028345 100644 --- a/src/internal/concurrent/hashtriemap_bench_test.go +++ b/src/internal/sync/hashtriemap_bench_test.go @@ -2,9 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package concurrent +package sync_test -import "testing" +import ( + isync "internal/sync" + "testing" +) func BenchmarkHashTrieMapLoadSmall(b *testing.B) { benchmarkHashTrieMapLoad(b, testDataSmall[:]) @@ -20,7 +23,7 @@ func BenchmarkHashTrieMapLoadLarge(b *testing.B) { func benchmarkHashTrieMapLoad(b *testing.B, data []string) { b.ReportAllocs() - m := NewHashTrieMap[string, int]() + var m isync.HashTrieMap[string, int] for i := range data { m.LoadOrStore(data[i], i) } @@ -47,7 +50,7 @@ func BenchmarkHashTrieMapLoadOrStoreLarge(b *testing.B) { func benchmarkHashTrieMapLoadOrStore(b *testing.B, data []string) { b.ReportAllocs() - m := NewHashTrieMap[string, int]() + var m isync.HashTrieMap[string, int] b.RunParallel(func(pb *testing.PB) { i := 0 diff --git a/src/internal/sync/hashtriemap_test.go b/src/internal/sync/hashtriemap_test.go new file mode 100644 index 00000000000000..d9219f841a8bdf --- /dev/null +++ b/src/internal/sync/hashtriemap_test.go @@ -0,0 +1,982 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync_test + +import ( + "fmt" + isync "internal/sync" + "math" + "runtime" + "strconv" + "sync" + "testing" + "weak" +) + +func TestHashTrieMap(t *testing.T) { + testHashTrieMap(t, func() *isync.HashTrieMap[string, int] { + var m isync.HashTrieMap[string, int] + return &m + }) +} + +func TestHashTrieMapBadHash(t *testing.T) { + testHashTrieMap(t, func() *isync.HashTrieMap[string, int] { + return isync.NewBadHashTrieMap[string, int]() + }) +} + +func TestHashTrieMapTruncHash(t *testing.T) { + testHashTrieMap(t, func() *isync.HashTrieMap[string, int] { + // Stub out the good hash function with a different terrible one + // (truncated hash). Everything should still work as expected. + // This is useful to test independently to catch issues with + // near collisions, where only the last few bits of the hash differ. + return isync.NewTruncHashTrieMap[string, int]() + }) +} + +func testHashTrieMap(t *testing.T, newMap func() *isync.HashTrieMap[string, int]) { + t.Run("LoadEmpty", func(t *testing.T) { + m := newMap() + + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }) + t.Run("LoadOrStore", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for i, s := range testData { + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + }) + t.Run("All", func(t *testing.T) { + m := newMap() + + testAll(t, m, testDataMap(testData[:]), func(_ string, _ int) bool { + return true + }) + }) + t.Run("Clear", func(t *testing.T) { + t.Run("Simple", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + m.Clear() + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }) + t.Run("Concurrent", func(t *testing.T) { + m := newMap() + + // Load up the map. + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + } + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + for _, s := range testData { + // Try a couple things to interfere with the clear. + expectNotDeleted(t, s, math.MaxInt)(m.CompareAndDelete(s, math.MaxInt)) + m.CompareAndSwap(s, i, i+1) // May succeed or fail; we don't care. + } + }(i) + } + + // Concurrently clear the map. + runtime.Gosched() + m.Clear() + + // Wait for workers to finish. + wg.Wait() + + // It should all be empty now. + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }) + }) + t.Run("CompareAndDelete", func(t *testing.T) { + t.Run("All", func(t *testing.T) { + m := newMap() + + for range 3 { + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for i, s := range testData { + expectPresent(t, s, i)(m.Load(s)) + expectNotDeleted(t, s, math.MaxInt)(m.CompareAndDelete(s, math.MaxInt)) + expectDeleted(t, s, i)(m.CompareAndDelete(s, i)) + expectNotDeleted(t, s, i)(m.CompareAndDelete(s, i)) + expectMissing(t, s, 0)(m.Load(s)) + } + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + } + }) + t.Run("One", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + expectNotDeleted(t, testData[15], math.MaxInt)(m.CompareAndDelete(testData[15], math.MaxInt)) + expectDeleted(t, testData[15], 15)(m.CompareAndDelete(testData[15], 15)) + expectNotDeleted(t, testData[15], 15)(m.CompareAndDelete(testData[15], 15)) + for i, s := range testData { + if i == 15 { + expectMissing(t, s, 0)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Multiple", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for _, i := range []int{1, 105, 6, 85} { + expectNotDeleted(t, testData[i], math.MaxInt)(m.CompareAndDelete(testData[i], math.MaxInt)) + expectDeleted(t, testData[i], i)(m.CompareAndDelete(testData[i], i)) + expectNotDeleted(t, testData[i], i)(m.CompareAndDelete(testData[i], i)) + } + for i, s := range testData { + if i == 1 || i == 105 || i == 6 || i == 85 { + expectMissing(t, s, 0)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Iterate", func(t *testing.T) { + m := newMap() + + testAll(t, m, testDataMap(testData[:]), func(s string, i int) bool { + expectDeleted(t, s, i)(m.CompareAndDelete(s, i)) + return true + }) + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }) + t.Run("ConcurrentUnsharedKeys", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectStored(t, key, id)(m.LoadOrStore(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoaded(t, key, id)(m.LoadOrStore(key, 0)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectDeleted(t, key, id)(m.CompareAndDelete(key, id)) + expectMissing(t, key, 0)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentSharedKeys", func(t *testing.T) { + m := newMap() + + // Load up the map. + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + } + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + for i, s := range testData { + expectNotDeleted(t, s, math.MaxInt)(m.CompareAndDelete(s, math.MaxInt)) + m.CompareAndDelete(s, i) + expectMissing(t, s, 0)(m.Load(s)) + } + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }(i) + } + wg.Wait() + }) + }) + t.Run("CompareAndSwap", func(t *testing.T) { + t.Run("All", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for j := range 3 { + for i, s := range testData { + expectPresent(t, s, i+j)(m.Load(s)) + expectNotSwapped(t, s, math.MaxInt, i+j+1)(m.CompareAndSwap(s, math.MaxInt, i+j+1)) + expectSwapped(t, s, i, i+j+1)(m.CompareAndSwap(s, i+j, i+j+1)) + expectNotSwapped(t, s, i+j, i+j+1)(m.CompareAndSwap(s, i+j, i+j+1)) + expectPresent(t, s, i+j+1)(m.Load(s)) + } + } + for i, s := range testData { + expectPresent(t, s, i+3)(m.Load(s)) + } + }) + t.Run("One", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + expectNotSwapped(t, testData[15], math.MaxInt, 16)(m.CompareAndSwap(testData[15], math.MaxInt, 16)) + expectSwapped(t, testData[15], 15, 16)(m.CompareAndSwap(testData[15], 15, 16)) + expectNotSwapped(t, testData[15], 15, 16)(m.CompareAndSwap(testData[15], 15, 16)) + for i, s := range testData { + if i == 15 { + expectPresent(t, s, 16)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Multiple", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for _, i := range []int{1, 105, 6, 85} { + expectNotSwapped(t, testData[i], math.MaxInt, i+1)(m.CompareAndSwap(testData[i], math.MaxInt, i+1)) + expectSwapped(t, testData[i], i, i+1)(m.CompareAndSwap(testData[i], i, i+1)) + expectNotSwapped(t, testData[i], i, i+1)(m.CompareAndSwap(testData[i], i, i+1)) + } + for i, s := range testData { + if i == 1 || i == 105 || i == 6 || i == 85 { + expectPresent(t, s, i+1)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + + t.Run("ConcurrentUnsharedKeys", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectStored(t, key, id)(m.LoadOrStore(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoaded(t, key, id)(m.LoadOrStore(key, 0)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectSwapped(t, key, id, id+1)(m.CompareAndSwap(key, id, id+1)) + expectPresent(t, key, id+1)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id+1)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentUnsharedKeysWithDelete", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectStored(t, key, id)(m.LoadOrStore(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoaded(t, key, id)(m.LoadOrStore(key, 0)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectSwapped(t, key, id, id+1)(m.CompareAndSwap(key, id, id+1)) + expectPresent(t, key, id+1)(m.Load(key)) + expectDeleted(t, key, id+1)(m.CompareAndDelete(key, id+1)) + expectNotSwapped(t, key, id+1, id+2)(m.CompareAndSwap(key, id+1, id+2)) + expectNotDeleted(t, key, id+1)(m.CompareAndDelete(key, id+1)) + expectMissing(t, key, 0)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentSharedKeys", func(t *testing.T) { + m := newMap() + + // Load up the map. + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + } + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + for i, s := range testData { + expectNotSwapped(t, s, math.MaxInt, i+1)(m.CompareAndSwap(s, math.MaxInt, i+1)) + m.CompareAndSwap(s, i, i+1) + expectPresent(t, s, i+1)(m.Load(s)) + } + for i, s := range testData { + expectPresent(t, s, i+1)(m.Load(s)) + } + }(i) + } + wg.Wait() + }) + }) + t.Run("Swap", func(t *testing.T) { + t.Run("All", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectNotLoadedFromSwap(t, s, i)(m.Swap(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoadedFromSwap(t, s, i, i)(m.Swap(s, i)) + } + for j := range 3 { + for i, s := range testData { + expectPresent(t, s, i+j)(m.Load(s)) + expectLoadedFromSwap(t, s, i+j, i+j+1)(m.Swap(s, i+j+1)) + expectPresent(t, s, i+j+1)(m.Load(s)) + } + } + for i, s := range testData { + expectLoadedFromSwap(t, s, i+3, i+3)(m.Swap(s, i+3)) + } + }) + t.Run("One", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectNotLoadedFromSwap(t, s, i)(m.Swap(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoadedFromSwap(t, s, i, i)(m.Swap(s, i)) + } + expectLoadedFromSwap(t, testData[15], 15, 16)(m.Swap(testData[15], 16)) + for i, s := range testData { + if i == 15 { + expectPresent(t, s, 16)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Multiple", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectNotLoadedFromSwap(t, s, i)(m.Swap(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoadedFromSwap(t, s, i, i)(m.Swap(s, i)) + } + for _, i := range []int{1, 105, 6, 85} { + expectLoadedFromSwap(t, testData[i], i, i+1)(m.Swap(testData[i], i+1)) + } + for i, s := range testData { + if i == 1 || i == 105 || i == 6 || i == 85 { + expectPresent(t, s, i+1)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("ConcurrentUnsharedKeys", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectNotLoadedFromSwap(t, key, id)(m.Swap(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoadedFromSwap(t, key, id, id)(m.Swap(key, id)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectLoadedFromSwap(t, key, id, id+1)(m.Swap(key, id+1)) + expectPresent(t, key, id+1)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id+1)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentUnsharedKeysWithDelete", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectNotLoadedFromSwap(t, key, id)(m.Swap(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoadedFromSwap(t, key, id, id)(m.Swap(key, id)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectLoadedFromSwap(t, key, id, id+1)(m.Swap(key, id+1)) + expectPresent(t, key, id+1)(m.Load(key)) + expectDeleted(t, key, id+1)(m.CompareAndDelete(key, id+1)) + expectNotLoadedFromSwap(t, key, id+2)(m.Swap(key, id+2)) + expectPresent(t, key, id+2)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id+2)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentSharedKeys", func(t *testing.T) { + m := newMap() + + // Load up the map. + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + } + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + for i, s := range testData { + m.Swap(s, i+1) + expectPresent(t, s, i+1)(m.Load(s)) + } + for i, s := range testData { + expectPresent(t, s, i+1)(m.Load(s)) + } + }(i) + } + wg.Wait() + }) + }) + t.Run("LoadAndDelete", func(t *testing.T) { + t.Run("All", func(t *testing.T) { + m := newMap() + + for range 3 { + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for i, s := range testData { + expectPresent(t, s, i)(m.Load(s)) + expectLoadedFromDelete(t, s, i)(m.LoadAndDelete(s)) + expectMissing(t, s, 0)(m.Load(s)) + expectNotLoadedFromDelete(t, s, 0)(m.LoadAndDelete(s)) + } + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + } + }) + t.Run("One", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + expectPresent(t, testData[15], 15)(m.Load(testData[15])) + expectLoadedFromDelete(t, testData[15], 15)(m.LoadAndDelete(testData[15])) + expectMissing(t, testData[15], 0)(m.Load(testData[15])) + expectNotLoadedFromDelete(t, testData[15], 0)(m.LoadAndDelete(testData[15])) + for i, s := range testData { + if i == 15 { + expectMissing(t, s, 0)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Multiple", func(t *testing.T) { + m := newMap() + + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + expectPresent(t, s, i)(m.Load(s)) + expectLoaded(t, s, i)(m.LoadOrStore(s, 0)) + } + for _, i := range []int{1, 105, 6, 85} { + expectPresent(t, testData[i], i)(m.Load(testData[i])) + expectLoadedFromDelete(t, testData[i], i)(m.LoadAndDelete(testData[i])) + expectMissing(t, testData[i], 0)(m.Load(testData[i])) + expectNotLoadedFromDelete(t, testData[i], 0)(m.LoadAndDelete(testData[i])) + } + for i, s := range testData { + if i == 1 || i == 105 || i == 6 || i == 85 { + expectMissing(t, s, 0)(m.Load(s)) + } else { + expectPresent(t, s, i)(m.Load(s)) + } + } + }) + t.Run("Iterate", func(t *testing.T) { + m := newMap() + + testAll(t, m, testDataMap(testData[:]), func(s string, i int) bool { + expectLoadedFromDelete(t, s, i)(m.LoadAndDelete(s)) + return true + }) + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }) + t.Run("ConcurrentUnsharedKeys", func(t *testing.T) { + m := newMap() + + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + expectStored(t, key, id)(m.LoadOrStore(key, id)) + expectPresent(t, key, id)(m.Load(key)) + expectLoaded(t, key, id)(m.LoadOrStore(key, 0)) + } + for _, s := range testData { + key := makeKey(s) + expectPresent(t, key, id)(m.Load(key)) + expectLoadedFromDelete(t, key, id)(m.LoadAndDelete(key)) + expectMissing(t, key, 0)(m.Load(key)) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key, 0)(m.Load(key)) + } + }(i) + } + wg.Wait() + }) + t.Run("ConcurrentSharedKeys", func(t *testing.T) { + m := newMap() + + // Load up the map. + for i, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + expectStored(t, s, i)(m.LoadOrStore(s, i)) + } + gmp := runtime.GOMAXPROCS(-1) + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + for _, s := range testData { + m.LoadAndDelete(s) + expectMissing(t, s, 0)(m.Load(s)) + } + for _, s := range testData { + expectMissing(t, s, 0)(m.Load(s)) + } + }(i) + } + wg.Wait() + }) + }) +} + +func testAll[K, V comparable](t *testing.T, m *isync.HashTrieMap[K, V], testData map[K]V, yield func(K, V) bool) { + for k, v := range testData { + expectStored(t, k, v)(m.LoadOrStore(k, v)) + } + visited := make(map[K]int) + m.All()(func(key K, got V) bool { + want, ok := testData[key] + if !ok { + t.Errorf("unexpected key %v in map", key) + return false + } + if got != want { + t.Errorf("expected key %v to have value %v, got %v", key, want, got) + return false + } + visited[key]++ + return yield(key, got) + }) + for key, n := range visited { + if n > 1 { + t.Errorf("visited key %v more than once", key) + } + } +} + +func expectPresent[K, V comparable](t *testing.T, key K, want V) func(got V, ok bool) { + t.Helper() + return func(got V, ok bool) { + t.Helper() + + if !ok { + t.Errorf("expected key %v to be present in map", key) + } + if ok && got != want { + t.Errorf("expected key %v to have value %v, got %v", key, want, got) + } + } +} + +func expectMissing[K, V comparable](t *testing.T, key K, want V) func(got V, ok bool) { + t.Helper() + if want != *new(V) { + // This is awkward, but the want argument is necessary to smooth over type inference. + // Just make sure the want argument always looks the same. + panic("expectMissing must always have a zero value variable") + } + return func(got V, ok bool) { + t.Helper() + + if ok { + t.Errorf("expected key %v to be missing from map, got value %v", key, got) + } + if !ok && got != want { + t.Errorf("expected missing key %v to be paired with the zero value; got %v", key, got) + } + } +} + +func expectLoaded[K, V comparable](t *testing.T, key K, want V) func(got V, loaded bool) { + t.Helper() + return func(got V, loaded bool) { + t.Helper() + + if !loaded { + t.Errorf("expected key %v to have been loaded, not stored", key) + } + if got != want { + t.Errorf("expected key %v to have value %v, got %v", key, want, got) + } + } +} + +func expectStored[K, V comparable](t *testing.T, key K, want V) func(got V, loaded bool) { + t.Helper() + return func(got V, loaded bool) { + t.Helper() + + if loaded { + t.Errorf("expected inserted key %v to have been stored, not loaded", key) + } + if got != want { + t.Errorf("expected inserted key %v to have value %v, got %v", key, want, got) + } + } +} + +func expectDeleted[K, V comparable](t *testing.T, key K, old V) func(deleted bool) { + t.Helper() + return func(deleted bool) { + t.Helper() + + if !deleted { + t.Errorf("expected key %v with value %v to be in map and deleted", key, old) + } + } +} + +func expectNotDeleted[K, V comparable](t *testing.T, key K, old V) func(deleted bool) { + t.Helper() + return func(deleted bool) { + t.Helper() + + if deleted { + t.Errorf("expected key %v with value %v to not be in map and thus not deleted", key, old) + } + } +} + +func expectSwapped[K, V comparable](t *testing.T, key K, old, new V) func(swapped bool) { + t.Helper() + return func(swapped bool) { + t.Helper() + + if !swapped { + t.Errorf("expected key %v with value %v to be in map and swapped for %v", key, old, new) + } + } +} + +func expectNotSwapped[K, V comparable](t *testing.T, key K, old, new V) func(swapped bool) { + t.Helper() + return func(swapped bool) { + t.Helper() + + if swapped { + t.Errorf("expected key %v with value %v to not be in map or not swapped for %v", key, old, new) + } + } +} + +func expectLoadedFromSwap[K, V comparable](t *testing.T, key K, want, new V) func(got V, loaded bool) { + t.Helper() + return func(got V, loaded bool) { + t.Helper() + + if !loaded { + t.Errorf("expected key %v to be in map and for %v to have been swapped for %v", key, want, new) + } else if want != got { + t.Errorf("key %v had its value %v swapped for %v, but expected it to have value %v", key, got, new, want) + } + } +} + +func expectNotLoadedFromSwap[K, V comparable](t *testing.T, key K, new V) func(old V, loaded bool) { + t.Helper() + return func(old V, loaded bool) { + t.Helper() + + if loaded { + t.Errorf("expected key %v to not be in map, but found value %v for it", key, old) + } + } +} + +func expectLoadedFromDelete[K, V comparable](t *testing.T, key K, want V) func(got V, loaded bool) { + t.Helper() + return func(got V, loaded bool) { + t.Helper() + + if !loaded { + t.Errorf("expected key %v to be in map to be deleted", key) + } else if want != got { + t.Errorf("key %v was deleted with value %v, but expected it to have value %v", key, got, want) + } + } +} + +func expectNotLoadedFromDelete[K, V comparable](t *testing.T, key K, _ V) func(old V, loaded bool) { + t.Helper() + return func(old V, loaded bool) { + t.Helper() + + if loaded { + t.Errorf("expected key %v to not be in map, but found value %v for it", key, old) + } + } +} + +func testDataMap(data []string) map[string]int { + m := make(map[string]int) + for i, s := range data { + m[s] = i + } + return m +} + +var ( + testDataSmall [8]string + testData [128]string + testDataLarge [128 << 10]string +) + +func init() { + for i := range testDataSmall { + testDataSmall[i] = fmt.Sprintf("%b", i) + } + for i := range testData { + testData[i] = fmt.Sprintf("%b", i) + } + for i := range testDataLarge { + testDataLarge[i] = fmt.Sprintf("%b", i) + } +} + +// TestConcurrentCache tests HashTrieMap in a scenario where it is used as +// the basis of a memory-efficient concurrent cache. We're specifically +// looking to make sure that CompareAndSwap and CompareAndDelete are +// atomic with respect to one another. When competing for the same +// key-value pair, they must not both succeed. +// +// This test is a regression test for issue #70970. +func TestConcurrentCache(t *testing.T) { + type dummy [32]byte + + var m isync.HashTrieMap[int, weak.Pointer[dummy]] + + type cleanupArg struct { + key int + value weak.Pointer[dummy] + } + cleanup := func(arg cleanupArg) { + m.CompareAndDelete(arg.key, arg.value) + } + get := func(m *isync.HashTrieMap[int, weak.Pointer[dummy]], key int) *dummy { + nv := new(dummy) + nw := weak.Make(nv) + for { + w, loaded := m.LoadOrStore(key, nw) + if !loaded { + runtime.AddCleanup(nv, cleanup, cleanupArg{key, nw}) + return nv + } + if v := w.Value(); v != nil { + return v + } + + // Weak pointer was reclaimed, try to replace it with nw. + if m.CompareAndSwap(key, w, nw) { + runtime.AddCleanup(nv, cleanup, cleanupArg{key, nw}) + return nv + } + } + } + + const N = 100_000 + const P = 5_000 + + var wg sync.WaitGroup + wg.Add(N) + for i := range N { + go func() { + defer wg.Done() + a := get(&m, i%P) + b := get(&m, i%P) + if a != b { + t.Errorf("consecutive cache reads returned different values: a != b (%p vs %p)\n", a, b) + } + }() + } + wg.Wait() +} diff --git a/src/internal/sync/mutex.go b/src/internal/sync/mutex.go new file mode 100644 index 00000000000000..c0c526a77cbb77 --- /dev/null +++ b/src/internal/sync/mutex.go @@ -0,0 +1,234 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package sync provides basic synchronization primitives such as mutual +// exclusion locks to internal packages (including ones that depend on sync). +// +// Tests are defined in package [sync]. +package sync + +import ( + "internal/race" + "sync/atomic" + "unsafe" +) + +// A Mutex is a mutual exclusion lock. +// +// See package [sync.Mutex] documentation. +type Mutex struct { + state int32 + sema uint32 +} + +const ( + mutexLocked = 1 << iota // mutex is locked + mutexWoken + mutexStarving + mutexWaiterShift = iota + + // Mutex fairness. + // + // Mutex can be in 2 modes of operations: normal and starvation. + // In normal mode waiters are queued in FIFO order, but a woken up waiter + // does not own the mutex and competes with new arriving goroutines over + // the ownership. New arriving goroutines have an advantage -- they are + // already running on CPU and there can be lots of them, so a woken up + // waiter has good chances of losing. In such case it is queued at front + // of the wait queue. If a waiter fails to acquire the mutex for more than 1ms, + // it switches mutex to the starvation mode. + // + // In starvation mode ownership of the mutex is directly handed off from + // the unlocking goroutine to the waiter at the front of the queue. + // New arriving goroutines don't try to acquire the mutex even if it appears + // to be unlocked, and don't try to spin. Instead they queue themselves at + // the tail of the wait queue. + // + // If a waiter receives ownership of the mutex and sees that either + // (1) it is the last waiter in the queue, or (2) it waited for less than 1 ms, + // it switches mutex back to normal operation mode. + // + // Normal mode has considerably better performance as a goroutine can acquire + // a mutex several times in a row even if there are blocked waiters. + // Starvation mode is important to prevent pathological cases of tail latency. + starvationThresholdNs = 1e6 +) + +// Lock locks m. +// +// See package [sync.Mutex] documentation. +func (m *Mutex) Lock() { + // Fast path: grab unlocked mutex. + if atomic.CompareAndSwapInt32(&m.state, 0, mutexLocked) { + if race.Enabled { + race.Acquire(unsafe.Pointer(m)) + } + return + } + // Slow path (outlined so that the fast path can be inlined) + m.lockSlow() +} + +// TryLock tries to lock m and reports whether it succeeded. +// +// See package [sync.Mutex] documentation. +func (m *Mutex) TryLock() bool { + old := m.state + if old&(mutexLocked|mutexStarving) != 0 { + return false + } + + // There may be a goroutine waiting for the mutex, but we are + // running now and can try to grab the mutex before that + // goroutine wakes up. + if !atomic.CompareAndSwapInt32(&m.state, old, old|mutexLocked) { + return false + } + + if race.Enabled { + race.Acquire(unsafe.Pointer(m)) + } + return true +} + +func (m *Mutex) lockSlow() { + var waitStartTime int64 + starving := false + awoke := false + iter := 0 + old := m.state + for { + // Don't spin in starvation mode, ownership is handed off to waiters + // so we won't be able to acquire the mutex anyway. + if old&(mutexLocked|mutexStarving) == mutexLocked && runtime_canSpin(iter) { + // Active spinning makes sense. + // Try to set mutexWoken flag to inform Unlock + // to not wake other blocked goroutines. + if !awoke && old&mutexWoken == 0 && old>>mutexWaiterShift != 0 && + atomic.CompareAndSwapInt32(&m.state, old, old|mutexWoken) { + awoke = true + } + runtime_doSpin() + iter++ + old = m.state + continue + } + new := old + // Don't try to acquire starving mutex, new arriving goroutines must queue. + if old&mutexStarving == 0 { + new |= mutexLocked + } + if old&(mutexLocked|mutexStarving) != 0 { + new += 1 << mutexWaiterShift + } + // The current goroutine switches mutex to starvation mode. + // But if the mutex is currently unlocked, don't do the switch. + // Unlock expects that starving mutex has waiters, which will not + // be true in this case. + if starving && old&mutexLocked != 0 { + new |= mutexStarving + } + if awoke { + // The goroutine has been woken from sleep, + // so we need to reset the flag in either case. + if new&mutexWoken == 0 { + throw("sync: inconsistent mutex state") + } + new &^= mutexWoken + } + if atomic.CompareAndSwapInt32(&m.state, old, new) { + if old&(mutexLocked|mutexStarving) == 0 { + break // locked the mutex with CAS + } + // If we were already waiting before, queue at the front of the queue. + queueLifo := waitStartTime != 0 + if waitStartTime == 0 { + waitStartTime = runtime_nanotime() + } + runtime_SemacquireMutex(&m.sema, queueLifo, 2) + starving = starving || runtime_nanotime()-waitStartTime > starvationThresholdNs + old = m.state + if old&mutexStarving != 0 { + // If this goroutine was woken and mutex is in starvation mode, + // ownership was handed off to us but mutex is in somewhat + // inconsistent state: mutexLocked is not set and we are still + // accounted as waiter. Fix that. + if old&(mutexLocked|mutexWoken) != 0 || old>>mutexWaiterShift == 0 { + throw("sync: inconsistent mutex state") + } + delta := int32(mutexLocked - 1<>mutexWaiterShift == 1 { + // Exit starvation mode. + // Critical to do it here and consider wait time. + // Starvation mode is so inefficient, that two goroutines + // can go lock-step infinitely once they switch mutex + // to starvation mode. + delta -= mutexStarving + } + atomic.AddInt32(&m.state, delta) + break + } + awoke = true + iter = 0 + } else { + old = m.state + } + } + + if race.Enabled { + race.Acquire(unsafe.Pointer(m)) + } +} + +// Unlock unlocks m. +// +// See package [sync.Mutex] documentation. +func (m *Mutex) Unlock() { + if race.Enabled { + _ = m.state + race.Release(unsafe.Pointer(m)) + } + + // Fast path: drop lock bit. + new := atomic.AddInt32(&m.state, -mutexLocked) + if new != 0 { + // Outlined slow path to allow inlining the fast path. + // To hide unlockSlow during tracing we skip one extra frame when tracing GoUnblock. + m.unlockSlow(new) + } +} + +func (m *Mutex) unlockSlow(new int32) { + if (new+mutexLocked)&mutexLocked == 0 { + fatal("sync: unlock of unlocked mutex") + } + if new&mutexStarving == 0 { + old := new + for { + // If there are no waiters or a goroutine has already + // been woken or grabbed the lock, no need to wake anyone. + // In starvation mode ownership is directly handed off from unlocking + // goroutine to the next waiter. We are not part of this chain, + // since we did not observe mutexStarving when we unlocked the mutex above. + // So get off the way. + if old>>mutexWaiterShift == 0 || old&(mutexLocked|mutexWoken|mutexStarving) != 0 { + return + } + // Grab the right to wake someone. + new = (old - 1<= 3 { + break + } + } + }() + want := []time.Time{ + time.Now(), + time.Now().Add(1 * time.Second), + time.Now().Add(2 * time.Second), + } + time.Sleep(5 * time.Second) + synctest.Wait() + if !slices.Equal(got, want) { + t.Errorf("got: %v; want: %v", got, want) + } + }) +} + +func TestIteratorPull(t *testing.T) { + synctest.Run(func() { + seq := func(yield func(time.Time) bool) { + for yield(time.Now()) { + time.Sleep(1 * time.Second) + } + } + var got []time.Time + go func() { + next, stop := iter.Pull(seq) + defer stop() + for len(got) < 3 { + now, _ := next() + got = append(got, now) + } + }() + want := []time.Time{ + time.Now(), + time.Now().Add(1 * time.Second), + time.Now().Add(2 * time.Second), + } + time.Sleep(5 * time.Second) + synctest.Wait() + if !slices.Equal(got, want) { + t.Errorf("got: %v; want: %v", got, want) + } + }) +} + +func TestReflectFuncOf(t *testing.T) { + mkfunc := func(name string, i int) { + reflect.FuncOf([]reflect.Type{ + reflect.StructOf([]reflect.StructField{{ + Name: name + strconv.Itoa(i), + Type: reflect.TypeOf(0), + }}), + }, nil, false) + } + go func() { + for i := 0; i < 100000; i++ { + mkfunc("A", i) + } + }() + synctest.Run(func() { + for i := 0; i < 100000; i++ { + mkfunc("A", i) + } + }) +} + +func TestWaitGroupInBubble(t *testing.T) { + synctest.Run(func() { + var wg sync.WaitGroup + wg.Add(1) + const delay = 1 * time.Second + go func() { + time.Sleep(delay) + wg.Done() + }() + start := time.Now() + wg.Wait() + if got := time.Since(start); got != delay { + t.Fatalf("WaitGroup.Wait() took %v, want %v", got, delay) + } + }) +} + +// https://go.dev/issue/74386 +func TestWaitGroupRacingAdds(t *testing.T) { + synctest.Run(func() { + var wg sync.WaitGroup + for range 100 { + wg.Go(func() {}) + } + wg.Wait() + }) +} + +func TestWaitGroupOutOfBubble(t *testing.T) { + var wg sync.WaitGroup + wg.Add(1) + donec := make(chan struct{}) + go synctest.Run(func() { + // Since wg.Add was called outside the bubble, Wait is not durably blocking + // and this waits until wg.Done is called below. + wg.Wait() + close(donec) + }) + select { + case <-donec: + t.Fatalf("synctest.Run finished before WaitGroup.Done called") + case <-time.After(1 * time.Millisecond): + } + wg.Done() + <-donec +} + +func TestWaitGroupMovedIntoBubble(t *testing.T) { + wantFatal(t, "fatal error: sync: WaitGroup.Add called from inside and outside synctest bubble", func() { + var wg sync.WaitGroup + wg.Add(1) + synctest.Run(func() { + wg.Add(1) + }) + }) +} + +func TestWaitGroupMovedOutOfBubble(t *testing.T) { + wantFatal(t, "fatal error: sync: WaitGroup.Add called from inside and outside synctest bubble", func() { + var wg sync.WaitGroup + synctest.Run(func() { + wg.Add(1) + }) + wg.Add(1) + }) +} + +func TestWaitGroupMovedBetweenBubblesWithNonZeroCount(t *testing.T) { + wantFatal(t, "fatal error: sync: WaitGroup.Add called from multiple synctest bubbles", func() { + var wg sync.WaitGroup + synctest.Run(func() { + wg.Add(1) + }) + synctest.Run(func() { + wg.Add(1) + }) + }) +} + +func TestWaitGroupDisassociateInWait(t *testing.T) { + var wg sync.WaitGroup + synctest.Run(func() { + wg.Add(1) + wg.Done() + // Count and waiters are 0, so Wait disassociates the WaitGroup. + wg.Wait() + }) + synctest.Run(func() { + // Reusing the WaitGroup is safe, because it is no longer bubbled. + wg.Add(1) + wg.Done() + }) +} + +func TestWaitGroupDisassociateInAdd(t *testing.T) { + var wg sync.WaitGroup + synctest.Run(func() { + wg.Add(1) + go wg.Wait() + synctest.Wait() // wait for Wait to block + // Count is 0 and waiters != 0, so Done wakes the waiters and + // disassociates the WaitGroup. + wg.Done() + }) + synctest.Run(func() { + // Reusing the WaitGroup is safe, because it is no longer bubbled. + wg.Add(1) + wg.Done() + }) +} + +var testWaitGroupLinkerAllocatedWG sync.WaitGroup + +func TestWaitGroupLinkerAllocated(t *testing.T) { + synctest.Run(func() { + // This WaitGroup is probably linker-allocated and has no span, + // so we won't be able to add a special to it associating it with + // this bubble. + // + // Operations on it may not be durably blocking, + // but they shouldn't fail. + testWaitGroupLinkerAllocatedWG.Go(func() {}) + testWaitGroupLinkerAllocatedWG.Wait() + }) +} + +var testWaitGroupHeapAllocatedWG = new(sync.WaitGroup) + +func TestWaitGroupHeapAllocated(t *testing.T) { + synctest.Run(func() { + // This package-scoped WaitGroup var should have been heap-allocated, + // so we can associate it with a bubble. + testWaitGroupHeapAllocatedWG.Add(1) + go testWaitGroupHeapAllocatedWG.Wait() + synctest.Wait() + testWaitGroupHeapAllocatedWG.Done() + }) +} + +// Issue #75134: Many racing bubble associations. +func TestWaitGroupManyBubbles(t *testing.T) { + var wg sync.WaitGroup + for range 100 { + wg.Go(func() { + synctest.Run(func() { + cancelc := make(chan struct{}) + var wg2 sync.WaitGroup + for range 100 { + wg2.Go(func() { + <-cancelc + }) + } + synctest.Wait() + close(cancelc) + wg2.Wait() + }) + }) + } + wg.Wait() +} + +func TestHappensBefore(t *testing.T) { + // Use two parallel goroutines accessing different vars to ensure that + // we correctly account for multiple goroutines in the bubble. + var v1 int + var v2 int + synctest.Run(func() { + v1++ // 1 + v2++ // 1 + + // Wait returns after these goroutines exit. + go func() { + v1++ // 2 + }() + go func() { + v2++ // 2 + }() + synctest.Wait() + + v1++ // 3 + v2++ // 3 + + // Wait returns after these goroutines block. + ch1 := make(chan struct{}) + go func() { + v1++ // 4 + <-ch1 + }() + go func() { + v2++ // 4 + <-ch1 + }() + synctest.Wait() + + v1++ // 5 + v2++ // 5 + close(ch1) + + // Wait returns after these timers run. + time.AfterFunc(0, func() { + v1++ // 6 + }) + time.AfterFunc(0, func() { + v2++ // 6 + }) + synctest.Wait() + + v1++ // 7 + v2++ // 7 + + // Wait returns after these timer goroutines block. + ch2 := make(chan struct{}) + time.AfterFunc(0, func() { + v1++ // 8 + <-ch2 + }) + time.AfterFunc(0, func() { + v2++ // 8 + <-ch2 + }) + synctest.Wait() + + v1++ // 9 + v2++ // 9 + close(ch2) + }) + // This Run happens after the previous Run returns. + synctest.Run(func() { + go func() { + go func() { + v1++ // 10 + }() + }() + go func() { + go func() { + v2++ // 10 + }() + }() + }) + // These tests happen after Run returns. + if got, want := v1, 10; got != want { + t.Errorf("v1 = %v, want %v", got, want) + } + if got, want := v2, 10; got != want { + t.Errorf("v2 = %v, want %v", got, want) + } +} + +// https://go.dev/issue/73817 +func TestWeak(t *testing.T) { + synctest.Run(func() { + for range 5 { + runtime.GC() + b := make([]byte, 1024) + weak.Make(&b) + } + }) +} + +func wantPanic(t *testing.T, want string) { + if e := recover(); e != nil { + if got := fmt.Sprint(e); got != want { + t.Errorf("got panic message %q, want %q", got, want) + } + } else { + t.Errorf("got no panic, want one") + } +} + +func wantFatal(t *testing.T, want string, f func()) { + t.Helper() + + if os.Getenv("GO_WANT_HELPER_PROCESS") == "1" { + f() + return + } + + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^"+t.Name()+"$") + cmd = testenv.CleanCmdEnv(cmd) + cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=1") + out, err := cmd.CombinedOutput() + if err == nil { + t.Errorf("expected test function to panic, but test returned successfully") + } + if !strings.Contains(string(out), want) { + t.Errorf("wanted test output contaiing %q; got %q", want, string(out)) + } +} diff --git a/src/internal/syscall/unix/arandom_netbsd.go b/src/internal/syscall/unix/arandom_netbsd.go new file mode 100644 index 00000000000000..23ca8739e81b23 --- /dev/null +++ b/src/internal/syscall/unix/arandom_netbsd.go @@ -0,0 +1,36 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix + +import ( + "syscall" + "unsafe" +) + +const ( + _CTL_KERN = 1 + + _KERN_ARND = 81 +) + +func Arandom(p []byte) error { + mib := [2]uint32{_CTL_KERN, _KERN_ARND} + n := uintptr(len(p)) + _, _, errno := syscall.Syscall6( + syscall.SYS___SYSCTL, + uintptr(unsafe.Pointer(&mib[0])), + uintptr(len(mib)), + uintptr(unsafe.Pointer(&p[0])), // olddata + uintptr(unsafe.Pointer(&n)), // &oldlen + uintptr(unsafe.Pointer(nil)), // newdata + 0) // newlen + if errno != 0 { + return syscall.Errno(errno) + } + if n != uintptr(len(p)) { + return syscall.EINVAL + } + return nil +} diff --git a/src/internal/syscall/unix/arc4random_openbsd.go b/src/internal/syscall/unix/arc4random_openbsd.go new file mode 100644 index 00000000000000..652e0cb19d600f --- /dev/null +++ b/src/internal/syscall/unix/arc4random_openbsd.go @@ -0,0 +1,23 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix + +import ( + "internal/abi" + "syscall" + "unsafe" +) + +//go:linkname syscall_syscall syscall.syscall +func syscall_syscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno) + +//go:cgo_import_dynamic libc_arc4random_buf arc4random_buf "libc.so" + +func libc_arc4random_buf_trampoline() + +func ARC4Random(p []byte) { + syscall_syscall(abi.FuncPCABI0(libc_arc4random_buf_trampoline), + uintptr(unsafe.Pointer(unsafe.SliceData(p))), uintptr(len(p)), 0) +} diff --git a/src/internal/syscall/unix/asm_darwin.s b/src/internal/syscall/unix/asm_darwin.s index 99f28765fe0ccd..9803c7260f08fe 100644 --- a/src/internal/syscall/unix/asm_darwin.s +++ b/src/internal/syscall/unix/asm_darwin.s @@ -23,3 +23,10 @@ TEXT ·libc_getgrnam_r_trampoline(SB),NOSPLIT,$0-0; JMP libc_getgrnam_r(SB) TEXT ·libc_getgrgid_r_trampoline(SB),NOSPLIT,$0-0; JMP libc_getgrgid_r(SB) TEXT ·libc_sysconf_trampoline(SB),NOSPLIT,$0-0; JMP libc_sysconf(SB) TEXT ·libc_faccessat_trampoline(SB),NOSPLIT,$0-0; JMP libc_faccessat(SB) +TEXT ·libc_readlinkat_trampoline(SB),NOSPLIT,$0-0; JMP libc_readlinkat(SB) +TEXT ·libc_mkdirat_trampoline(SB),NOSPLIT,$0-0; JMP libc_mkdirat(SB) +TEXT ·libc_fchmodat_trampoline(SB),NOSPLIT,$0-0; JMP libc_fchmodat(SB) +TEXT ·libc_fchownat_trampoline(SB),NOSPLIT,$0-0; JMP libc_fchownat(SB) +TEXT ·libc_renameat_trampoline(SB),NOSPLIT,$0-0; JMP libc_renameat(SB) +TEXT ·libc_linkat_trampoline(SB),NOSPLIT,$0-0; JMP libc_linkat(SB) +TEXT ·libc_symlinkat_trampoline(SB),NOSPLIT,$0-0; JMP libc_symlinkat(SB) diff --git a/src/internal/syscall/unix/asm_openbsd.s b/src/internal/syscall/unix/asm_openbsd.s index cc54a14ca5e36a..d7c230555c3bbc 100644 --- a/src/internal/syscall/unix/asm_openbsd.s +++ b/src/internal/syscall/unix/asm_openbsd.s @@ -8,3 +8,19 @@ TEXT ·libc_faccessat_trampoline(SB),NOSPLIT,$0-0 JMP libc_faccessat(SB) +TEXT ·libc_arc4random_buf_trampoline(SB),NOSPLIT,$0-0 + JMP libc_arc4random_buf(SB) +TEXT ·libc_readlinkat_trampoline(SB),NOSPLIT,$0-0 + JMP libc_readlinkat(SB) +TEXT ·libc_mkdirat_trampoline(SB),NOSPLIT,$0-0 + JMP libc_mkdirat(SB) +TEXT ·libc_fchmodat_trampoline(SB),NOSPLIT,$0-0 + JMP libc_fchmodat(SB) +TEXT ·libc_fchownat_trampoline(SB),NOSPLIT,$0-0 + JMP libc_fchownat(SB) +TEXT ·libc_renameat_trampoline(SB),NOSPLIT,$0-0 + JMP libc_renameat(SB) +TEXT ·libc_linkat_trampoline(SB),NOSPLIT,$0-0 + JMP libc_linkat(SB) +TEXT ·libc_symlinkat_trampoline(SB),NOSPLIT,$0-0 + JMP libc_symlinkat(SB) diff --git a/src/internal/syscall/unix/at.go b/src/internal/syscall/unix/at.go index cfb6e410b122a8..96272afc7b3db3 100644 --- a/src/internal/syscall/unix/at.go +++ b/src/internal/syscall/unix/at.go @@ -38,3 +38,142 @@ func Openat(dirfd int, path string, flags int, perm uint32) (int, error) { return int(fd), nil } + +func Readlinkat(dirfd int, path string, buf []byte) (int, error) { + p0, err := syscall.BytePtrFromString(path) + if err != nil { + return 0, err + } + var p1 unsafe.Pointer + if len(buf) > 0 { + p1 = unsafe.Pointer(&buf[0]) + } else { + p1 = unsafe.Pointer(&_zero) + } + n, _, errno := syscall.Syscall6(readlinkatTrap, + uintptr(dirfd), + uintptr(unsafe.Pointer(p0)), + uintptr(p1), + uintptr(len(buf)), + 0, 0) + if errno != 0 { + return 0, errno + } + + return int(n), nil +} + +func Mkdirat(dirfd int, path string, mode uint32) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + + _, _, errno := syscall.Syscall6(mkdiratTrap, + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(mode), + 0, 0, 0) + if errno != 0 { + return errno + } + return nil +} + +func Fchmodat(dirfd int, path string, mode uint32, flags int) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + _, _, errno := syscall.Syscall6(fchmodatTrap, + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(mode), + uintptr(flags), + 0, 0) + if errno != 0 { + return errno + } + return nil +} + +func Fchownat(dirfd int, path string, uid, gid int, flags int) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + _, _, errno := syscall.Syscall6(fchownatTrap, + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(uid), + uintptr(gid), + uintptr(flags), + 0) + if errno != 0 { + return errno + } + return nil +} + +func Renameat(olddirfd int, oldpath string, newdirfd int, newpath string) error { + oldp, err := syscall.BytePtrFromString(oldpath) + if err != nil { + return err + } + newp, err := syscall.BytePtrFromString(newpath) + if err != nil { + return err + } + _, _, errno := syscall.Syscall6(renameatTrap, + uintptr(olddirfd), + uintptr(unsafe.Pointer(oldp)), + uintptr(newdirfd), + uintptr(unsafe.Pointer(newp)), + 0, + 0) + if errno != 0 { + return errno + } + return nil +} + +func Linkat(olddirfd int, oldpath string, newdirfd int, newpath string, flag int) error { + oldp, err := syscall.BytePtrFromString(oldpath) + if err != nil { + return err + } + newp, err := syscall.BytePtrFromString(newpath) + if err != nil { + return err + } + _, _, errno := syscall.Syscall6(linkatTrap, + uintptr(olddirfd), + uintptr(unsafe.Pointer(oldp)), + uintptr(newdirfd), + uintptr(unsafe.Pointer(newp)), + uintptr(flag), + 0) + if errno != 0 { + return errno + } + return nil +} + +func Symlinkat(oldpath string, newdirfd int, newpath string) error { + oldp, err := syscall.BytePtrFromString(oldpath) + if err != nil { + return err + } + newp, err := syscall.BytePtrFromString(newpath) + if err != nil { + return err + } + _, _, errno := syscall.Syscall(symlinkatTrap, + uintptr(unsafe.Pointer(oldp)), + uintptr(newdirfd), + uintptr(unsafe.Pointer(newp))) + if errno != 0 { + return errno + } + return nil +} diff --git a/src/internal/syscall/unix/at_aix.go b/src/internal/syscall/unix/at_aix.go index 3fe3285ce21257..8bf7b4dd81bb8e 100644 --- a/src/internal/syscall/unix/at_aix.go +++ b/src/internal/syscall/unix/at_aix.go @@ -4,11 +4,20 @@ package unix +//go:cgo_import_dynamic libc_fchmodat fchmodat "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_fchownat fchownat "libc.a/shr_64.o" //go:cgo_import_dynamic libc_fstatat fstatat "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_linkat linkat "libc.a/shr_64.o" //go:cgo_import_dynamic libc_openat openat "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_renameat renameat "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_symlinkat symlinkat "libc.a/shr_64.o" //go:cgo_import_dynamic libc_unlinkat unlinkat "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_readlinkat readlinkat "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_mkdirat mkdirat "libc.a/shr_64.o" const ( + AT_EACCESS = 0x1 + AT_FDCWD = -0x02 AT_REMOVEDIR = 0x1 AT_SYMLINK_NOFOLLOW = 0x1 UTIME_OMIT = -0x3 diff --git a/src/internal/syscall/unix/at_darwin.go b/src/internal/syscall/unix/at_darwin.go new file mode 100644 index 00000000000000..c74c827626a575 --- /dev/null +++ b/src/internal/syscall/unix/at_darwin.go @@ -0,0 +1,182 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin + +package unix + +import ( + "internal/abi" + "syscall" + "unsafe" +) + +func libc_readlinkat_trampoline() + +//go:cgo_import_dynamic libc_readlinkat readlinkat "/usr/lib/libSystem.B.dylib" + +func Readlinkat(dirfd int, path string, buf []byte) (int, error) { + p0, err := syscall.BytePtrFromString(path) + if err != nil { + return 0, err + } + var p1 unsafe.Pointer + if len(buf) > 0 { + p1 = unsafe.Pointer(&buf[0]) + } else { + p1 = unsafe.Pointer(&_zero) + } + n, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_readlinkat_trampoline), + uintptr(dirfd), + uintptr(unsafe.Pointer(p0)), + uintptr(p1), + uintptr(len(buf)), + 0, + 0) + if errno != 0 { + return 0, errno + } + return int(n), nil +} + +func libc_mkdirat_trampoline() + +//go:cgo_import_dynamic libc_mkdirat mkdirat "/usr/lib/libSystem.B.dylib" + +func Mkdirat(dirfd int, path string, mode uint32) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + _, _, errno := syscall_syscall(abi.FuncPCABI0(libc_mkdirat_trampoline), + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(mode)) + if errno != 0 { + return errno + } + return nil +} + +func libc_fchmodat_trampoline() + +//go:cgo_import_dynamic libc_fchmodat fchmodat "/usr/lib/libSystem.B.dylib" + +func Fchmodat(dirfd int, path string, mode uint32, flags int) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_fchmodat_trampoline), + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(mode), + uintptr(flags), + 0, + 0) + if errno != 0 { + return errno + } + return nil +} + +func libc_fchownat_trampoline() + +//go:cgo_import_dynamic libc_fchownat fchownat "/usr/lib/libSystem.B.dylib" + +func Fchownat(dirfd int, path string, uid, gid int, flags int) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_fchownat_trampoline), + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(uid), + uintptr(gid), + uintptr(flags), + 0) + if errno != 0 { + return errno + } + return nil +} + +func libc_renameat_trampoline() + +//go:cgo_import_dynamic libc_renameat renameat "/usr/lib/libSystem.B.dylib" + +func Renameat(olddirfd int, oldpath string, newdirfd int, newpath string) error { + oldp, err := syscall.BytePtrFromString(oldpath) + if err != nil { + return err + } + newp, err := syscall.BytePtrFromString(newpath) + if err != nil { + return err + } + _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_renameat_trampoline), + uintptr(olddirfd), + uintptr(unsafe.Pointer(oldp)), + uintptr(newdirfd), + uintptr(unsafe.Pointer(newp)), + 0, + 0) + if errno != 0 { + return errno + } + return nil +} + +func libc_linkat_trampoline() + +//go:cgo_import_dynamic libc_linkat linkat "/usr/lib/libSystem.B.dylib" + +func Linkat(olddirfd int, oldpath string, newdirfd int, newpath string, flag int) error { + oldp, err := syscall.BytePtrFromString(oldpath) + if err != nil { + return err + } + newp, err := syscall.BytePtrFromString(newpath) + if err != nil { + return err + } + _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_linkat_trampoline), + uintptr(olddirfd), + uintptr(unsafe.Pointer(oldp)), + uintptr(newdirfd), + uintptr(unsafe.Pointer(newp)), + uintptr(flag), + 0) + if errno != 0 { + return errno + } + return nil +} + +func libc_symlinkat_trampoline() + +//go:cgo_import_dynamic libc_symlinkat symlinkat "/usr/lib/libSystem.B.dylib" + +func Symlinkat(oldpath string, newdirfd int, newpath string) error { + oldp, err := syscall.BytePtrFromString(oldpath) + if err != nil { + return err + } + newp, err := syscall.BytePtrFromString(newpath) + if err != nil { + return err + } + _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_symlinkat_trampoline), + uintptr(unsafe.Pointer(oldp)), + uintptr(newdirfd), + uintptr(unsafe.Pointer(newp)), + 0, + 0, + 0) + if errno != 0 { + return errno + } + return nil +} diff --git a/src/internal/syscall/unix/at_fstatat.go b/src/internal/syscall/unix/at_fstatat.go index 25de336a8041c4..18cd62be203556 100644 --- a/src/internal/syscall/unix/at_fstatat.go +++ b/src/internal/syscall/unix/at_fstatat.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build dragonfly || (linux && !loong64) || netbsd || (openbsd && mips64) +//go:build dragonfly || (linux && !(loong64 || mips64 || mips64le)) || netbsd || (openbsd && mips64) package unix diff --git a/src/internal/syscall/unix/at_fstatat2.go b/src/internal/syscall/unix/at_fstatat2.go index 8d20e1a885bd13..b09aecbcdda4dd 100644 --- a/src/internal/syscall/unix/at_fstatat2.go +++ b/src/internal/syscall/unix/at_fstatat2.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build freebsd || (linux && loong64) +//go:build freebsd || (linux && (loong64 || mips64 || mips64le)) package unix diff --git a/src/internal/syscall/unix/at_libc.go b/src/internal/syscall/unix/at_libc.go index f48d3791e370c8..5c64b34d48f5d7 100644 --- a/src/internal/syscall/unix/at_libc.go +++ b/src/internal/syscall/unix/at_libc.go @@ -14,11 +14,25 @@ import ( //go:linkname procFstatat libc_fstatat //go:linkname procOpenat libc_openat //go:linkname procUnlinkat libc_unlinkat +//go:linkname procReadlinkat libc_readlinkat +//go:linkname procMkdirat libc_mkdirat +//go:linkname procFchmodat libc_fchmodat +//go:linkname procFchownat libc_fchownat +//go:linkname procRenameat libc_renameat +//go:linkname procLinkat libc_linkat +//go:linkname procSymlinkat libc_symlinkat var ( procFstatat, procOpenat, - procUnlinkat uintptr + procUnlinkat, + procReadlinkat, + procMkdirat, + procFchmodat, + procFchownat, + procRenameat, + procLinkat, + procSymlinkat uintptr ) func Unlinkat(dirfd int, path string, flags int) error { @@ -27,7 +41,11 @@ func Unlinkat(dirfd int, path string, flags int) error { return err } - _, _, errno := syscall6(uintptr(unsafe.Pointer(&procUnlinkat)), 3, uintptr(dirfd), uintptr(unsafe.Pointer(p)), uintptr(flags), 0, 0, 0) + _, _, errno := syscall6(uintptr(unsafe.Pointer(&procUnlinkat)), 3, + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(flags), + 0, 0, 0) if errno != 0 { return errno } @@ -41,7 +59,12 @@ func Openat(dirfd int, path string, flags int, perm uint32) (int, error) { return 0, err } - fd, _, errno := syscall6(uintptr(unsafe.Pointer(&procOpenat)), 4, uintptr(dirfd), uintptr(unsafe.Pointer(p)), uintptr(flags), uintptr(perm), 0, 0) + fd, _, errno := syscall6(uintptr(unsafe.Pointer(&procOpenat)), 4, + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(flags), + uintptr(perm), + 0, 0) if errno != 0 { return 0, errno } @@ -55,10 +78,155 @@ func Fstatat(dirfd int, path string, stat *syscall.Stat_t, flags int) error { return err } - _, _, errno := syscall6(uintptr(unsafe.Pointer(&procFstatat)), 4, uintptr(dirfd), uintptr(unsafe.Pointer(p)), uintptr(unsafe.Pointer(stat)), uintptr(flags), 0, 0) + _, _, errno := syscall6(uintptr(unsafe.Pointer(&procFstatat)), 4, + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(unsafe.Pointer(stat)), + uintptr(flags), + 0, 0) if errno != 0 { return errno } return nil } + +func Readlinkat(dirfd int, path string, buf []byte) (int, error) { + p0, err := syscall.BytePtrFromString(path) + if err != nil { + return 0, err + } + var p1 unsafe.Pointer + if len(buf) > 0 { + p1 = unsafe.Pointer(&buf[0]) + } else { + p1 = unsafe.Pointer(&_zero) + } + n, _, errno := syscall6(uintptr(unsafe.Pointer(&procReadlinkat)), 4, + uintptr(dirfd), + uintptr(unsafe.Pointer(p0)), + uintptr(p1), + uintptr(len(buf)), + 0, 0) + if errno != 0 { + return 0, errno + } + + return int(n), nil +} + +func Mkdirat(dirfd int, path string, mode uint32) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + + _, _, errno := syscall6(uintptr(unsafe.Pointer(&procMkdirat)), 3, + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(mode), + 0, 0, 0) + if errno != 0 { + return errno + } + return nil +} + +func Fchmodat(dirfd int, path string, mode uint32, flags int) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + _, _, errno := syscall6(uintptr(unsafe.Pointer(&procFchmodat)), 4, + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(mode), + uintptr(flags), + 0, 0) + if errno != 0 { + return errno + } + return nil +} + +func Fchownat(dirfd int, path string, uid, gid int, flags int) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + _, _, errno := syscall6(uintptr(unsafe.Pointer(&procFchownat)), 5, + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(uid), + uintptr(gid), + uintptr(flags), + 0) + if errno != 0 { + return errno + } + return nil +} + +func Renameat(olddirfd int, oldpath string, newdirfd int, newpath string) error { + oldp, err := syscall.BytePtrFromString(oldpath) + if err != nil { + return err + } + newp, err := syscall.BytePtrFromString(newpath) + if err != nil { + return err + } + _, _, errno := syscall6(uintptr(unsafe.Pointer(&procRenameat)), 4, + uintptr(olddirfd), + uintptr(unsafe.Pointer(oldp)), + uintptr(newdirfd), + uintptr(unsafe.Pointer(newp)), + 0, + 0) + if errno != 0 { + return errno + } + return nil +} + +func Linkat(olddirfd int, oldpath string, newdirfd int, newpath string, flag int) error { + oldp, err := syscall.BytePtrFromString(oldpath) + if err != nil { + return err + } + newp, err := syscall.BytePtrFromString(newpath) + if err != nil { + return err + } + _, _, errno := syscall6(uintptr(unsafe.Pointer(&procLinkat)), 5, + uintptr(olddirfd), + uintptr(unsafe.Pointer(oldp)), + uintptr(newdirfd), + uintptr(unsafe.Pointer(newp)), + uintptr(flag), + 0) + if errno != 0 { + return errno + } + return nil +} + +func Symlinkat(oldpath string, newdirfd int, newpath string) error { + oldp, err := syscall.BytePtrFromString(oldpath) + if err != nil { + return err + } + newp, err := syscall.BytePtrFromString(newpath) + if err != nil { + return err + } + _, _, errno := syscall6(uintptr(unsafe.Pointer(&procSymlinkat)), 3, + uintptr(unsafe.Pointer(oldp)), + uintptr(newdirfd), + uintptr(unsafe.Pointer(newp)), + 0, 0, 0) + if errno != 0 { + return errno + } + return nil +} diff --git a/src/internal/syscall/unix/at_openbsd.go b/src/internal/syscall/unix/at_openbsd.go new file mode 100644 index 00000000000000..96e77eb408454b --- /dev/null +++ b/src/internal/syscall/unix/at_openbsd.go @@ -0,0 +1,173 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build openbsd && !mips64 + +package unix + +import ( + "internal/abi" + "syscall" + "unsafe" +) + +//go:cgo_import_dynamic libc_readlinkat readlinkat "libc.so" + +func libc_readlinkat_trampoline() + +func Readlinkat(dirfd int, path string, buf []byte) (int, error) { + p0, err := syscall.BytePtrFromString(path) + if err != nil { + return 0, err + } + var p1 unsafe.Pointer + if len(buf) > 0 { + p1 = unsafe.Pointer(&buf[0]) + } else { + p1 = unsafe.Pointer(&_zero) + } + n, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_readlinkat_trampoline), uintptr(dirfd), uintptr(unsafe.Pointer(p0)), uintptr(p1), uintptr(len(buf)), 0, 0) + if errno != 0 { + return 0, errno + } + return int(n), nil +} + +//go:cgo_import_dynamic libc_mkdirat mkdirat "libc.so" + +func libc_mkdirat_trampoline() + +func Mkdirat(dirfd int, path string, mode uint32) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_mkdirat_trampoline), uintptr(dirfd), uintptr(unsafe.Pointer(p)), uintptr(mode), 0, 0, 0) + if errno != 0 { + return errno + } + return nil +} + +//go:cgo_import_dynamic libc_fchmodat fchmodat "libc.so" + +func libc_fchmodat_trampoline() + +func Fchmodat(dirfd int, path string, mode uint32, flags int) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_fchmodat_trampoline), + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(mode), + uintptr(flags), + 0, + 0) + if errno != 0 { + return errno + } + return nil +} + +//go:cgo_import_dynamic libc_fchownat fchownat "libc.so" + +func libc_fchownat_trampoline() + +func Fchownat(dirfd int, path string, uid, gid int, flags int) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_fchownat_trampoline), + uintptr(dirfd), + uintptr(unsafe.Pointer(p)), + uintptr(uid), + uintptr(gid), + uintptr(flags), + 0) + if errno != 0 { + return errno + } + return nil +} + +//go:cgo_import_dynamic libc_renameat renameat "libc.so" + +func libc_renameat_trampoline() + +func Renameat(olddirfd int, oldpath string, newdirfd int, newpath string) error { + oldp, err := syscall.BytePtrFromString(oldpath) + if err != nil { + return err + } + newp, err := syscall.BytePtrFromString(newpath) + if err != nil { + return err + } + _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_renameat_trampoline), + uintptr(olddirfd), + uintptr(unsafe.Pointer(oldp)), + uintptr(newdirfd), + uintptr(unsafe.Pointer(newp)), + 0, + 0) + if errno != 0 { + return errno + } + return nil +} + +func libc_linkat_trampoline() + +//go:cgo_import_dynamic libc_linkat linkat "libc.so" + +func Linkat(olddirfd int, oldpath string, newdirfd int, newpath string, flag int) error { + oldp, err := syscall.BytePtrFromString(oldpath) + if err != nil { + return err + } + newp, err := syscall.BytePtrFromString(newpath) + if err != nil { + return err + } + _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_linkat_trampoline), + uintptr(olddirfd), + uintptr(unsafe.Pointer(oldp)), + uintptr(newdirfd), + uintptr(unsafe.Pointer(newp)), + uintptr(flag), + 0) + if errno != 0 { + return errno + } + return nil +} + +func libc_symlinkat_trampoline() + +//go:cgo_import_dynamic libc_symlinkat symlinkat "libc.so" + +func Symlinkat(oldpath string, newdirfd int, newpath string) error { + oldp, err := syscall.BytePtrFromString(oldpath) + if err != nil { + return err + } + newp, err := syscall.BytePtrFromString(newpath) + if err != nil { + return err + } + _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_symlinkat_trampoline), + uintptr(unsafe.Pointer(oldp)), + uintptr(newdirfd), + uintptr(unsafe.Pointer(newp)), + 0, + 0, + 0) + if errno != 0 { + return errno + } + return nil +} diff --git a/src/internal/syscall/unix/at_solaris.go b/src/internal/syscall/unix/at_solaris.go index ae1c1d64ca7bc2..5d69ae5bee7f89 100644 --- a/src/internal/syscall/unix/at_solaris.go +++ b/src/internal/syscall/unix/at_solaris.go @@ -12,12 +12,22 @@ func syscall6(trap, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err // Implemented as rawsysvicall6 in runtime/syscall_solaris.go. func rawSyscall6(trap, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) +//go:cgo_import_dynamic libc_faccessat faccessat "libc.so" +//go:cgo_import_dynamic libc_fchmodat fchmodat "libc.so" +//go:cgo_import_dynamic libc_fchownat fchownat "libc.so" //go:cgo_import_dynamic libc_fstatat fstatat "libc.so" +//go:cgo_import_dynamic libc_linkat linkat "libc.so" //go:cgo_import_dynamic libc_openat openat "libc.so" +//go:cgo_import_dynamic libc_renameat renameat "libc.so" +//go:cgo_import_dynamic libc_symlinkat symlinkat "libc.so" //go:cgo_import_dynamic libc_unlinkat unlinkat "libc.so" +//go:cgo_import_dynamic libc_readlinkat readlinkat "libc.so" +//go:cgo_import_dynamic libc_mkdirat mkdirat "libc.so" //go:cgo_import_dynamic libc_uname uname "libc.so" const ( + AT_EACCESS = 0x4 + AT_FDCWD = 0xffd19553 AT_REMOVEDIR = 0x1 AT_SYMLINK_NOFOLLOW = 0x1000 diff --git a/src/internal/syscall/unix/at_sysnum_dragonfly.go b/src/internal/syscall/unix/at_sysnum_dragonfly.go index a8164dcc8ec7be..9728b969c4e8d9 100644 --- a/src/internal/syscall/unix/at_sysnum_dragonfly.go +++ b/src/internal/syscall/unix/at_sysnum_dragonfly.go @@ -7,9 +7,16 @@ package unix import "syscall" const ( - unlinkatTrap uintptr = syscall.SYS_UNLINKAT - openatTrap uintptr = syscall.SYS_OPENAT - fstatatTrap uintptr = syscall.SYS_FSTATAT + unlinkatTrap uintptr = syscall.SYS_UNLINKAT + openatTrap uintptr = syscall.SYS_OPENAT + fstatatTrap uintptr = syscall.SYS_FSTATAT + readlinkatTrap uintptr = syscall.SYS_READLINKAT + mkdiratTrap uintptr = syscall.SYS_MKDIRAT + fchmodatTrap uintptr = syscall.SYS_FCHMODAT + fchownatTrap uintptr = syscall.SYS_FCHOWNAT + renameatTrap uintptr = syscall.SYS_RENAMEAT + linkatTrap uintptr = syscall.SYS_LINKAT + symlinkatTrap uintptr = syscall.SYS_SYMLINKAT AT_EACCESS = 0x4 AT_FDCWD = 0xfffafdcd diff --git a/src/internal/syscall/unix/at_sysnum_freebsd.go b/src/internal/syscall/unix/at_sysnum_freebsd.go index f74961d508623d..c1fdcabf4136aa 100644 --- a/src/internal/syscall/unix/at_sysnum_freebsd.go +++ b/src/internal/syscall/unix/at_sysnum_freebsd.go @@ -17,4 +17,11 @@ const ( unlinkatTrap uintptr = syscall.SYS_UNLINKAT openatTrap uintptr = syscall.SYS_OPENAT posixFallocateTrap uintptr = syscall.SYS_POSIX_FALLOCATE + readlinkatTrap uintptr = syscall.SYS_READLINKAT + mkdiratTrap uintptr = syscall.SYS_MKDIRAT + fchmodatTrap uintptr = syscall.SYS_FCHMODAT + fchownatTrap uintptr = syscall.SYS_FCHOWNAT + renameatTrap uintptr = syscall.SYS_RENAMEAT + linkatTrap uintptr = syscall.SYS_LINKAT + symlinkatTrap uintptr = syscall.SYS_SYMLINKAT ) diff --git a/src/internal/syscall/unix/at_sysnum_linux.go b/src/internal/syscall/unix/at_sysnum_linux.go index 7c3b15c30302c8..bb7f244fe2e19c 100644 --- a/src/internal/syscall/unix/at_sysnum_linux.go +++ b/src/internal/syscall/unix/at_sysnum_linux.go @@ -6,8 +6,16 @@ package unix import "syscall" -const unlinkatTrap uintptr = syscall.SYS_UNLINKAT -const openatTrap uintptr = syscall.SYS_OPENAT +const ( + unlinkatTrap uintptr = syscall.SYS_UNLINKAT + openatTrap uintptr = syscall.SYS_OPENAT + readlinkatTrap uintptr = syscall.SYS_READLINKAT + mkdiratTrap uintptr = syscall.SYS_MKDIRAT + fchmodatTrap uintptr = syscall.SYS_FCHMODAT + fchownatTrap uintptr = syscall.SYS_FCHOWNAT + linkatTrap uintptr = syscall.SYS_LINKAT + symlinkatTrap uintptr = syscall.SYS_SYMLINKAT +) const ( AT_EACCESS = 0x200 diff --git a/src/internal/syscall/unix/at_sysnum_netbsd.go b/src/internal/syscall/unix/at_sysnum_netbsd.go index ffb1d2eaf8b1f4..db17852b748e32 100644 --- a/src/internal/syscall/unix/at_sysnum_netbsd.go +++ b/src/internal/syscall/unix/at_sysnum_netbsd.go @@ -6,9 +6,19 @@ package unix import "syscall" -const unlinkatTrap uintptr = syscall.SYS_UNLINKAT -const openatTrap uintptr = syscall.SYS_OPENAT -const fstatatTrap uintptr = syscall.SYS_FSTATAT +const ( + unlinkatTrap uintptr = syscall.SYS_UNLINKAT + openatTrap uintptr = syscall.SYS_OPENAT + fstatatTrap uintptr = syscall.SYS_FSTATAT + readlinkatTrap uintptr = syscall.SYS_READLINKAT + mkdiratTrap uintptr = syscall.SYS_MKDIRAT + fchmodatTrap uintptr = syscall.SYS_FCHMODAT + fchownatTrap uintptr = syscall.SYS_FCHOWNAT + renameatTrap uintptr = syscall.SYS_RENAMEAT + linkatTrap uintptr = syscall.SYS_LINKAT + symlinkatTrap uintptr = syscall.SYS_SYMLINKAT + posixFallocateTrap uintptr = 479 +) const ( AT_EACCESS = 0x100 diff --git a/src/internal/syscall/unix/at_sysnum_openbsd.go b/src/internal/syscall/unix/at_sysnum_openbsd.go index 3b0c0dbd19cc46..7672414cf7690e 100644 --- a/src/internal/syscall/unix/at_sysnum_openbsd.go +++ b/src/internal/syscall/unix/at_sysnum_openbsd.go @@ -6,9 +6,13 @@ package unix import "syscall" -const unlinkatTrap uintptr = syscall.SYS_UNLINKAT -const openatTrap uintptr = syscall.SYS_OPENAT -const fstatatTrap uintptr = syscall.SYS_FSTATAT +const ( + unlinkatTrap uintptr = syscall.SYS_UNLINKAT + openatTrap uintptr = syscall.SYS_OPENAT + fstatatTrap uintptr = syscall.SYS_FSTATAT + readlinkatTrap uintptr = syscall.SYS_READLINKAT + mkdiratTrap uintptr = syscall.SYS_MKDIRAT +) const ( AT_EACCESS = 0x1 diff --git a/src/internal/syscall/unix/at_wasip1.go b/src/internal/syscall/unix/at_wasip1.go index 3d47d7ebe0b641..dfbb365f2a6fed 100644 --- a/src/internal/syscall/unix/at_wasip1.go +++ b/src/internal/syscall/unix/at_wasip1.go @@ -2,12 +2,176 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build wasip1 + package unix +import ( + "syscall" + "unsafe" +) + +// The values of these constants are not part of the WASI API. const ( // UTIME_OMIT is the sentinel value to indicate that a time value should not // be changed. It is useful for example to indicate for example with UtimesNano // to avoid changing AccessTime or ModifiedTime. // Its value must match syscall/fs_wasip1.go UTIME_OMIT = -0x2 + + AT_REMOVEDIR = 0x200 + AT_SYMLINK_NOFOLLOW = 0x100 +) + +func Unlinkat(dirfd int, path string, flags int) error { + if flags&AT_REMOVEDIR == 0 { + return errnoErr(path_unlink_file( + int32(dirfd), + unsafe.StringData(path), + size(len(path)), + )) + } else { + return errnoErr(path_remove_directory( + int32(dirfd), + unsafe.StringData(path), + size(len(path)), + )) + } +} + +//go:wasmimport wasi_snapshot_preview1 path_unlink_file +//go:noescape +func path_unlink_file(fd int32, path *byte, pathLen size) syscall.Errno + +//go:wasmimport wasi_snapshot_preview1 path_remove_directory +//go:noescape +func path_remove_directory(fd int32, path *byte, pathLen size) syscall.Errno + +func Openat(dirfd int, path string, flags int, perm uint32) (int, error) { + return syscall.Openat(dirfd, path, flags, perm) +} + +func Fstatat(dirfd int, path string, stat *syscall.Stat_t, flags int) error { + var filestatFlags uint32 + if flags&AT_SYMLINK_NOFOLLOW == 0 { + filestatFlags |= syscall.LOOKUP_SYMLINK_FOLLOW + } + return errnoErr(path_filestat_get( + int32(dirfd), + uint32(filestatFlags), + unsafe.StringData(path), + size(len(path)), + unsafe.Pointer(stat), + )) +} + +//go:wasmimport wasi_snapshot_preview1 path_filestat_get +//go:noescape +func path_filestat_get(fd int32, flags uint32, path *byte, pathLen size, buf unsafe.Pointer) syscall.Errno + +func Readlinkat(dirfd int, path string, buf []byte) (int, error) { + var nwritten size + errno := path_readlink( + int32(dirfd), + unsafe.StringData(path), + size(len(path)), + &buf[0], + size(len(buf)), + &nwritten) + return int(nwritten), errnoErr(errno) + +} + +type ( + size = uint32 ) + +//go:wasmimport wasi_snapshot_preview1 path_readlink +//go:noescape +func path_readlink(fd int32, path *byte, pathLen size, buf *byte, bufLen size, nwritten *size) syscall.Errno + +func Mkdirat(dirfd int, path string, mode uint32) error { + if path == "" { + return syscall.EINVAL + } + return errnoErr(path_create_directory( + int32(dirfd), + unsafe.StringData(path), + size(len(path)), + )) +} + +//go:wasmimport wasi_snapshot_preview1 path_create_directory +//go:noescape +func path_create_directory(fd int32, path *byte, pathLen size) syscall.Errno + +func Fchmodat(dirfd int, path string, mode uint32, flags int) error { + // WASI preview 1 doesn't support changing file modes. + return syscall.ENOSYS +} + +func Fchownat(dirfd int, path string, uid, gid int, flags int) error { + // WASI preview 1 doesn't support changing file ownership. + return syscall.ENOSYS +} + +func Renameat(olddirfd int, oldpath string, newdirfd int, newpath string) error { + if oldpath == "" || newpath == "" { + return syscall.EINVAL + } + return errnoErr(path_rename( + int32(olddirfd), + unsafe.StringData(oldpath), + size(len(oldpath)), + int32(newdirfd), + unsafe.StringData(newpath), + size(len(newpath)), + )) +} + +//go:wasmimport wasi_snapshot_preview1 path_rename +//go:noescape +func path_rename(oldFd int32, oldPath *byte, oldPathLen size, newFd int32, newPath *byte, newPathLen size) syscall.Errno + +func Linkat(olddirfd int, oldpath string, newdirfd int, newpath string, flag int) error { + if oldpath == "" || newpath == "" { + return syscall.EINVAL + } + return errnoErr(path_link( + int32(olddirfd), + 0, + unsafe.StringData(oldpath), + size(len(oldpath)), + int32(newdirfd), + unsafe.StringData(newpath), + size(len(newpath)), + )) +} + +//go:wasmimport wasi_snapshot_preview1 path_link +//go:noescape +func path_link(oldFd int32, oldFlags uint32, oldPath *byte, oldPathLen size, newFd int32, newPath *byte, newPathLen size) syscall.Errno + +func Symlinkat(oldpath string, newdirfd int, newpath string) error { + if oldpath == "" || newpath == "" { + return syscall.EINVAL + } + return errnoErr(path_symlink( + unsafe.StringData(oldpath), + size(len(oldpath)), + int32(newdirfd), + unsafe.StringData(newpath), + size(len(newpath)), + )) +} + +//go:wasmimport wasi_snapshot_preview1 path_symlink +//go:noescape +func path_symlink(oldPath *byte, oldPathLen size, fd int32, newPath *byte, newPathLen size) syscall.Errno + +func errnoErr(errno syscall.Errno) error { + if errno == 0 { + return nil + } + return errno +} diff --git a/src/internal/syscall/unix/constants.go b/src/internal/syscall/unix/constants.go index 28092c2ddfde99..6a78dda7956586 100644 --- a/src/internal/syscall/unix/constants.go +++ b/src/internal/syscall/unix/constants.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build unix +//go:build unix || wasip1 package unix diff --git a/src/internal/syscall/unix/copy_file_range_linux.go b/src/internal/syscall/unix/copy_file_range_linux.go deleted file mode 100644 index cf0a279a7a910e..00000000000000 --- a/src/internal/syscall/unix/copy_file_range_linux.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package unix - -import ( - "syscall" - "unsafe" -) - -func CopyFileRange(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int, err error) { - r1, _, errno := syscall.Syscall6(copyFileRangeTrap, - uintptr(rfd), - uintptr(unsafe.Pointer(roff)), - uintptr(wfd), - uintptr(unsafe.Pointer(woff)), - uintptr(len), - uintptr(flags), - ) - n = int(r1) - if errno != 0 { - err = errno - } - return -} diff --git a/src/internal/syscall/unix/copy_file_range_unix.go b/src/internal/syscall/unix/copy_file_range_unix.go new file mode 100644 index 00000000000000..16a434219e1987 --- /dev/null +++ b/src/internal/syscall/unix/copy_file_range_unix.go @@ -0,0 +1,28 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build freebsd || linux + +package unix + +import ( + "syscall" + "unsafe" +) + +func CopyFileRange(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int, err error) { + r1, _, errno := syscall.Syscall6(copyFileRangeTrap, + uintptr(rfd), + uintptr(unsafe.Pointer(roff)), + uintptr(wfd), + uintptr(unsafe.Pointer(woff)), + uintptr(len), + uintptr(flags), + ) + n = int(r1) + if errno != 0 { + err = errno + } + return +} diff --git a/src/internal/syscall/unix/eaccess.go b/src/internal/syscall/unix/eaccess.go new file mode 100644 index 00000000000000..3c12314a9c7d44 --- /dev/null +++ b/src/internal/syscall/unix/eaccess.go @@ -0,0 +1,24 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix + +package unix + +import ( + "runtime" + "syscall" +) + +func Eaccess(path string, mode uint32) error { + if runtime.GOOS == "android" { + // syscall.Faccessat for Android implements AT_EACCESS check in + // userspace. Since Android doesn't have setuid programs and + // never runs code with euid!=uid, AT_EACCESS check is not + // really required. Return ENOSYS so the callers can fall back + // to permission bits check. + return syscall.ENOSYS + } + return faccessat(AT_FDCWD, path, mode, AT_EACCESS) +} diff --git a/src/internal/syscall/unix/eaccess_bsd.go b/src/internal/syscall/unix/eaccess_bsd.go deleted file mode 100644 index 7077af17b672e5..00000000000000 --- a/src/internal/syscall/unix/eaccess_bsd.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build dragonfly || freebsd || netbsd || (openbsd && mips64) - -package unix - -import ( - "syscall" - "unsafe" -) - -func faccessat(dirfd int, path string, mode uint32, flags int) error { - p, err := syscall.BytePtrFromString(path) - if err != nil { - return err - } - _, _, errno := syscall.Syscall6(syscall.SYS_FACCESSAT, uintptr(dirfd), uintptr(unsafe.Pointer(p)), uintptr(mode), uintptr(flags), 0, 0) - if errno != 0 { - return errno - } - return err -} - -func Eaccess(path string, mode uint32) error { - return faccessat(AT_FDCWD, path, mode, AT_EACCESS) -} diff --git a/src/internal/syscall/unix/eaccess_darwin.go b/src/internal/syscall/unix/eaccess_darwin.go deleted file mode 100644 index 0fa8d17afeaefd..00000000000000 --- a/src/internal/syscall/unix/eaccess_darwin.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package unix - -import ( - "internal/abi" - "syscall" - "unsafe" -) - -func libc_faccessat_trampoline() - -//go:cgo_import_dynamic libc_faccessat faccessat "/usr/lib/libSystem.B.dylib" - -func faccessat(dirfd int, path string, mode uint32, flags int) error { - p, err := syscall.BytePtrFromString(path) - if err != nil { - return err - } - _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_faccessat_trampoline), uintptr(dirfd), uintptr(unsafe.Pointer(p)), uintptr(mode), uintptr(flags), 0, 0) - if errno != 0 { - return errno - } - return nil -} - -func Eaccess(path string, mode uint32) error { - return faccessat(AT_FDCWD, path, mode, AT_EACCESS) -} diff --git a/src/internal/syscall/unix/eaccess_linux.go b/src/internal/syscall/unix/eaccess_linux.go deleted file mode 100644 index 5695a5e4ce7cdf..00000000000000 --- a/src/internal/syscall/unix/eaccess_linux.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package unix - -import "syscall" - -func Eaccess(path string, mode uint32) error { - return syscall.Faccessat(AT_FDCWD, path, mode, AT_EACCESS) -} diff --git a/src/internal/syscall/unix/eaccess_openbsd.go b/src/internal/syscall/unix/eaccess_openbsd.go deleted file mode 100644 index 5e91f11f66542a..00000000000000 --- a/src/internal/syscall/unix/eaccess_openbsd.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build openbsd && !mips64 - -package unix - -import ( - "internal/abi" - "syscall" - "unsafe" -) - -//go:linkname syscall_syscall6 syscall.syscall6 -func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) - -func libc_faccessat_trampoline() - -//go:cgo_import_dynamic libc_faccessat faccessat "libc.so" - -func faccessat(dirfd int, path string, mode uint32, flags int) error { - p, err := syscall.BytePtrFromString(path) - if err != nil { - return err - } - _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_faccessat_trampoline), uintptr(dirfd), uintptr(unsafe.Pointer(p)), uintptr(mode), uintptr(flags), 0, 0) - if errno != 0 { - return errno - } - return err -} - -func Eaccess(path string, mode uint32) error { - return faccessat(AT_FDCWD, path, mode, AT_EACCESS) -} diff --git a/src/internal/syscall/unix/eaccess_other.go b/src/internal/syscall/unix/eaccess_other.go deleted file mode 100644 index 3da3a64f0e6a5e..00000000000000 --- a/src/internal/syscall/unix/eaccess_other.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build unix && !darwin && !dragonfly && !freebsd && !linux && !openbsd && !netbsd - -package unix - -import "syscall" - -func Eaccess(path string, mode uint32) error { - return syscall.ENOSYS -} diff --git a/src/internal/syscall/unix/faccessat_bsd.go b/src/internal/syscall/unix/faccessat_bsd.go new file mode 100644 index 00000000000000..1db54c35b26ac5 --- /dev/null +++ b/src/internal/syscall/unix/faccessat_bsd.go @@ -0,0 +1,24 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build dragonfly || freebsd || netbsd || (openbsd && mips64) + +package unix + +import ( + "syscall" + "unsafe" +) + +func faccessat(dirfd int, path string, mode uint32, flags int) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + _, _, errno := syscall.Syscall6(syscall.SYS_FACCESSAT, uintptr(dirfd), uintptr(unsafe.Pointer(p)), uintptr(mode), uintptr(flags), 0, 0) + if errno != 0 { + return errno + } + return nil +} diff --git a/src/internal/syscall/unix/faccessat_darwin.go b/src/internal/syscall/unix/faccessat_darwin.go new file mode 100644 index 00000000000000..ef790aa949a9a4 --- /dev/null +++ b/src/internal/syscall/unix/faccessat_darwin.go @@ -0,0 +1,27 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix + +import ( + "internal/abi" + "syscall" + "unsafe" +) + +func libc_faccessat_trampoline() + +//go:cgo_import_dynamic libc_faccessat faccessat "/usr/lib/libSystem.B.dylib" + +func faccessat(dirfd int, path string, mode uint32, flags int) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_faccessat_trampoline), uintptr(dirfd), uintptr(unsafe.Pointer(p)), uintptr(mode), uintptr(flags), 0, 0) + if errno != 0 { + return errno + } + return nil +} diff --git a/src/internal/syscall/unix/faccessat_openbsd.go b/src/internal/syscall/unix/faccessat_openbsd.go new file mode 100644 index 00000000000000..3519532154a75d --- /dev/null +++ b/src/internal/syscall/unix/faccessat_openbsd.go @@ -0,0 +1,32 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build openbsd && !mips64 + +package unix + +import ( + "internal/abi" + "syscall" + "unsafe" +) + +//go:linkname syscall_syscall6 syscall.syscall6 +func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) + +func libc_faccessat_trampoline() + +//go:cgo_import_dynamic libc_faccessat faccessat "libc.so" + +func faccessat(dirfd int, path string, mode uint32, flags int) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + _, _, errno := syscall_syscall6(abi.FuncPCABI0(libc_faccessat_trampoline), uintptr(dirfd), uintptr(unsafe.Pointer(p)), uintptr(mode), uintptr(flags), 0, 0) + if errno != 0 { + return errno + } + return nil +} diff --git a/src/internal/syscall/unix/faccessat_solaris.go b/src/internal/syscall/unix/faccessat_solaris.go new file mode 100644 index 00000000000000..47e05fb2c09b6f --- /dev/null +++ b/src/internal/syscall/unix/faccessat_solaris.go @@ -0,0 +1,28 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix + +import ( + "syscall" + "unsafe" +) + +//go:linkname procFaccessat libc_faccessat + +var procFaccessat uintptr + +func faccessat(dirfd int, path string, mode uint32, flags int) error { + p, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + + _, _, errno := syscall6(uintptr(unsafe.Pointer(&procFaccessat)), 4, uintptr(dirfd), uintptr(unsafe.Pointer(p)), uintptr(mode), uintptr(flags), 0, 0) + if errno != 0 { + return errno + } + + return nil +} diff --git a/src/internal/syscall/unix/faccessat_syscall.go b/src/internal/syscall/unix/faccessat_syscall.go new file mode 100644 index 00000000000000..865e40b2c6f3f4 --- /dev/null +++ b/src/internal/syscall/unix/faccessat_syscall.go @@ -0,0 +1,11 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || linux + +package unix + +import "syscall" + +var faccessat = syscall.Faccessat diff --git a/src/internal/syscall/unix/fallocate_bsd_386.go b/src/internal/syscall/unix/fallocate_bsd_386.go new file mode 100644 index 00000000000000..1dcdff4a5391d0 --- /dev/null +++ b/src/internal/syscall/unix/fallocate_bsd_386.go @@ -0,0 +1,20 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (freebsd || netbsd) && 386 + +package unix + +import "syscall" + +func PosixFallocate(fd int, off int64, size int64) error { + // If successful, posix_fallocate() returns zero. It returns an error on failure, without + // setting errno. See https://man.freebsd.org/cgi/man.cgi?query=posix_fallocate&sektion=2&n=1 + // and https://man.netbsd.org/posix_fallocate.2#RETURN%20VALUES + r1, _, _ := syscall.Syscall6(posixFallocateTrap, uintptr(fd), uintptr(off), uintptr(off>>32), uintptr(size), uintptr(size>>32), 0) + if r1 != 0 { + return syscall.Errno(r1) + } + return nil +} diff --git a/src/internal/syscall/unix/fallocate_bsd_64bit.go b/src/internal/syscall/unix/fallocate_bsd_64bit.go new file mode 100644 index 00000000000000..177bb48382d54c --- /dev/null +++ b/src/internal/syscall/unix/fallocate_bsd_64bit.go @@ -0,0 +1,20 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (freebsd || netbsd) && (amd64 || arm64 || riscv64) + +package unix + +import "syscall" + +func PosixFallocate(fd int, off int64, size int64) error { + // If successful, posix_fallocate() returns zero. It returns an error on failure, without + // setting errno. See https://man.freebsd.org/cgi/man.cgi?query=posix_fallocate&sektion=2&n=1 + // and https://man.netbsd.org/posix_fallocate.2#RETURN%20VALUES + r1, _, _ := syscall.Syscall(posixFallocateTrap, uintptr(fd), uintptr(off), uintptr(size)) + if r1 != 0 { + return syscall.Errno(r1) + } + return nil +} diff --git a/src/internal/syscall/unix/fallocate_bsd_arm.go b/src/internal/syscall/unix/fallocate_bsd_arm.go new file mode 100644 index 00000000000000..15e99d02b1c790 --- /dev/null +++ b/src/internal/syscall/unix/fallocate_bsd_arm.go @@ -0,0 +1,25 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (freebsd || netbsd) && arm + +package unix + +import "syscall" + +func PosixFallocate(fd int, off int64, size int64) error { + // If successful, posix_fallocate() returns zero. It returns an error on failure, without + // setting errno. See https://man.freebsd.org/cgi/man.cgi?query=posix_fallocate&sektion=2&n=1 + // and https://man.netbsd.org/posix_fallocate.2#RETURN%20VALUES + // + // The padding 0 argument is needed because the ARM calling convention requires that if an + // argument (off in this case) needs double-word alignment (8-byte), the NCRN (next core + // register number) is rounded up to the next even register number. + // See https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aapcs32/aapcs32.rst#parameter-passing + r1, _, _ := syscall.Syscall6(posixFallocateTrap, uintptr(fd), 0, uintptr(off), uintptr(off>>32), uintptr(size), uintptr(size>>32)) + if r1 != 0 { + return syscall.Errno(r1) + } + return nil +} diff --git a/src/internal/syscall/unix/fallocate_freebsd_386.go b/src/internal/syscall/unix/fallocate_freebsd_386.go deleted file mode 100644 index 535b23dbc5b7eb..00000000000000 --- a/src/internal/syscall/unix/fallocate_freebsd_386.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package unix - -import "syscall" - -func PosixFallocate(fd int, off int64, size int64) error { - // If successful, posix_fallocate() returns zero. It returns an error on failure, without - // setting errno. See https://man.freebsd.org/cgi/man.cgi?query=posix_fallocate&sektion=2&n=1 - r1, _, _ := syscall.Syscall6(posixFallocateTrap, uintptr(fd), uintptr(off), uintptr(off>>32), uintptr(size), uintptr(size>>32), 0) - if r1 != 0 { - return syscall.Errno(r1) - } - return nil -} diff --git a/src/internal/syscall/unix/fallocate_freebsd_64bit.go b/src/internal/syscall/unix/fallocate_freebsd_64bit.go deleted file mode 100644 index a9d52283f06a9b..00000000000000 --- a/src/internal/syscall/unix/fallocate_freebsd_64bit.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build freebsd && (amd64 || arm64 || riscv64) - -package unix - -import "syscall" - -func PosixFallocate(fd int, off int64, size int64) error { - // If successful, posix_fallocate() returns zero. It returns an error on failure, without - // setting errno. See https://man.freebsd.org/cgi/man.cgi?query=posix_fallocate&sektion=2&n=1 - r1, _, _ := syscall.Syscall(posixFallocateTrap, uintptr(fd), uintptr(off), uintptr(size)) - if r1 != 0 { - return syscall.Errno(r1) - } - return nil -} diff --git a/src/internal/syscall/unix/fallocate_freebsd_arm.go b/src/internal/syscall/unix/fallocate_freebsd_arm.go deleted file mode 100644 index 1ded50f3b9a168..00000000000000 --- a/src/internal/syscall/unix/fallocate_freebsd_arm.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package unix - -import "syscall" - -func PosixFallocate(fd int, off int64, size int64) error { - // If successful, posix_fallocate() returns zero. It returns an error on failure, without - // setting errno. See https://man.freebsd.org/cgi/man.cgi?query=posix_fallocate&sektion=2&n=1 - // - // The padding 0 argument is needed because the ARM calling convention requires that if an - // argument (off in this case) needs double-word alignment (8-byte), the NCRN (next core - // register number) is rounded up to the next even register number. - // See https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aapcs32/aapcs32.rst#parameter-passing - r1, _, _ := syscall.Syscall6(posixFallocateTrap, uintptr(fd), 0, uintptr(off), uintptr(off>>32), uintptr(size), uintptr(size>>32)) - if r1 != 0 { - return syscall.Errno(r1) - } - return nil -} diff --git a/src/internal/syscall/unix/getentropy_netbsd.go b/src/internal/syscall/unix/getentropy_netbsd.go deleted file mode 100644 index 02bac1be00ccfe..00000000000000 --- a/src/internal/syscall/unix/getentropy_netbsd.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build netbsd - -package unix - -import ( - "syscall" - "unsafe" -) - -const ( - _CTL_KERN = 1 - - _KERN_ARND = 81 -) - -func GetEntropy(p []byte) error { - mib := [2]uint32{_CTL_KERN, _KERN_ARND} - n := uintptr(len(p)) - _, _, errno := syscall.Syscall6( - syscall.SYS___SYSCTL, - uintptr(unsafe.Pointer(&mib[0])), - uintptr(len(mib)), - uintptr(unsafe.Pointer(&p[0])), // olddata - uintptr(unsafe.Pointer(&n)), // &oldlen - uintptr(unsafe.Pointer(nil)), // newdata - 0) // newlen - if errno != 0 { - return syscall.Errno(errno) - } - if n != uintptr(len(p)) { - return syscall.EINVAL - } - return nil -} diff --git a/src/internal/syscall/unix/getentropy_openbsd.go b/src/internal/syscall/unix/getentropy_openbsd.go deleted file mode 100644 index ad0914da903b98..00000000000000 --- a/src/internal/syscall/unix/getentropy_openbsd.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build openbsd && !mips64 - -package unix - -import _ "unsafe" // for linkname - -// GetEntropy calls the OpenBSD getentropy system call. -func GetEntropy(p []byte) error { - return getentropy(p) -} - -//go:linkname getentropy syscall.getentropy -func getentropy(p []byte) error diff --git a/src/internal/syscall/unix/getentropy_openbsd_mips64.go b/src/internal/syscall/unix/getentropy_openbsd_mips64.go deleted file mode 100644 index d5caa8095a68bf..00000000000000 --- a/src/internal/syscall/unix/getentropy_openbsd_mips64.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package unix - -import ( - "syscall" - "unsafe" -) - -// getentropy(2)'s syscall number, from /usr/src/sys/kern/syscalls.master -const entropyTrap uintptr = 7 - -// GetEntropy calls the OpenBSD getentropy system call. -func GetEntropy(p []byte) error { - _, _, errno := syscall.Syscall(entropyTrap, - uintptr(unsafe.Pointer(&p[0])), - uintptr(len(p)), - 0) - if errno != 0 { - return errno - } - return nil -} diff --git a/src/internal/syscall/unix/getrandom.go b/src/internal/syscall/unix/getrandom.go index e83f0cd6f95733..db3e7ac0f0d3be 100644 --- a/src/internal/syscall/unix/getrandom.go +++ b/src/internal/syscall/unix/getrandom.go @@ -12,6 +12,10 @@ import ( "unsafe" ) +//go:linkname vgetrandom runtime.vgetrandom +//go:noescape +func vgetrandom(p []byte, flags uint32) (ret int, supported bool) + var getrandomUnsupported atomic.Bool // GetRandomFlag is a flag supported by the getrandom system call. @@ -19,14 +23,18 @@ type GetRandomFlag uintptr // GetRandom calls the getrandom system call. func GetRandom(p []byte, flags GetRandomFlag) (n int, err error) { - if len(p) == 0 { - return 0, nil + ret, supported := vgetrandom(p, uint32(flags)) + if supported { + if ret < 0 { + return 0, syscall.Errno(-ret) + } + return ret, nil } if getrandomUnsupported.Load() { return 0, syscall.ENOSYS } r1, _, errno := syscall.Syscall(getrandomTrap, - uintptr(unsafe.Pointer(&p[0])), + uintptr(unsafe.Pointer(unsafe.SliceData(p))), uintptr(len(p)), uintptr(flags)) if errno != 0 { diff --git a/src/internal/syscall/unix/getrandom_linux_test.go b/src/internal/syscall/unix/getrandom_linux_test.go new file mode 100644 index 00000000000000..e1778c19e82493 --- /dev/null +++ b/src/internal/syscall/unix/getrandom_linux_test.go @@ -0,0 +1,22 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix_test + +import ( + "internal/syscall/unix" + "testing" +) + +func BenchmarkParallelGetRandom(b *testing.B) { + b.SetBytes(4) + b.RunParallel(func(pb *testing.PB) { + var buf [4]byte + for pb.Next() { + if _, err := unix.GetRandom(buf[:], 0); err != nil { + b.Fatal(err) + } + } + }) +} diff --git a/src/internal/syscall/unix/kernel_version_freebsd.go b/src/internal/syscall/unix/kernel_version_freebsd.go new file mode 100644 index 00000000000000..db3519ddfb70f6 --- /dev/null +++ b/src/internal/syscall/unix/kernel_version_freebsd.go @@ -0,0 +1,50 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix + +import ( + "sync" + "syscall" +) + +// KernelVersion returns major and minor kernel version numbers +// parsed from the syscall.Sysctl("kern.osrelease")'s value, +// or (0, 0) if the version can't be obtained or parsed. +func KernelVersion() (major, minor int) { + release, err := syscall.Sysctl("kern.osrelease") + if err != nil { + return 0, 0 + } + + parseNext := func() (n int) { + for i, c := range release { + if c == '.' { + release = release[i+1:] + return + } + if '0' <= c && c <= '9' { + n = n*10 + int(c-'0') + } + } + release = "" + return + } + + major = parseNext() + minor = parseNext() + + return +} + +// SupportCopyFileRange reports whether the kernel supports the copy_file_range(2). +// This function will examine both the kernel version and the availability of the system call. +var SupportCopyFileRange = sync.OnceValue(func() bool { + // The copy_file_range() function first appeared in FreeBSD 13.0. + if !KernelVersionGE(13, 0) { + return false + } + _, err := CopyFileRange(0, nil, 0, nil, 0, 0) + return err != syscall.ENOSYS +}) diff --git a/src/internal/syscall/unix/kernel_version_freebsd_test.go b/src/internal/syscall/unix/kernel_version_freebsd_test.go new file mode 100644 index 00000000000000..694440e3254d82 --- /dev/null +++ b/src/internal/syscall/unix/kernel_version_freebsd_test.go @@ -0,0 +1,23 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix_test + +import ( + "internal/syscall/unix" + "syscall" + "testing" +) + +func TestSupportCopyFileRange(t *testing.T) { + major, minor := unix.KernelVersion() + t.Logf("Running on FreeBSD %d.%d\n", major, minor) + + _, err := unix.CopyFileRange(0, nil, 0, nil, 0, 0) + want := err != syscall.ENOSYS + got := unix.SupportCopyFileRange() + if want != got { + t.Fatalf("SupportCopyFileRange, got %t; want %t", got, want) + } +} diff --git a/src/internal/syscall/unix/kernel_version_ge.go b/src/internal/syscall/unix/kernel_version_ge.go new file mode 100644 index 00000000000000..a1424041993b85 --- /dev/null +++ b/src/internal/syscall/unix/kernel_version_ge.go @@ -0,0 +1,13 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix + +// KernelVersionGE checks if the running kernel version +// is greater than or equal to the provided version. +func KernelVersionGE(x, y int) bool { + xx, yy := KernelVersion() + + return xx > x || (xx == x && yy >= y) +} diff --git a/src/internal/syscall/unix/kernel_version_ge_test.go b/src/internal/syscall/unix/kernel_version_ge_test.go new file mode 100644 index 00000000000000..f4b1b23cd2e839 --- /dev/null +++ b/src/internal/syscall/unix/kernel_version_ge_test.go @@ -0,0 +1,67 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix_test + +import ( + "internal/syscall/unix" + "testing" +) + +func TestKernelVersionGE(t *testing.T) { + major, minor := unix.KernelVersion() + t.Logf("Running on kernel %d.%d", major, minor) + + tests := []struct { + name string + x, y int + want bool + }{ + { + name: "current version equals itself", + x: major, + y: minor, + want: true, + }, + { + name: "older major version", + x: major - 1, + y: 0, + want: true, + }, + { + name: "same major, older minor version", + x: major, + y: minor - 1, + want: true, + }, + { + name: "newer major version", + x: major + 1, + y: 0, + want: false, + }, + { + name: "same major, newer minor version", + x: major, + y: minor + 1, + want: false, + }, + { + name: "min version (0.0)", + x: 0, + y: 0, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := unix.KernelVersionGE(tt.x, tt.y) + if got != tt.want { + t.Errorf("KernelVersionGE(%d, %d): got %v, want %v", tt.x, tt.y, got, tt.want) + } + }) + } +} diff --git a/src/internal/syscall/unix/kernel_version_linux.go b/src/internal/syscall/unix/kernel_version_linux.go index 71e8aa4c57164c..f3656288ef33fe 100644 --- a/src/internal/syscall/unix/kernel_version_linux.go +++ b/src/internal/syscall/unix/kernel_version_linux.go @@ -8,11 +8,9 @@ import ( "syscall" ) -// KernelVersion returns major and minor kernel version numbers, parsed from -// the syscall.Uname's Release field, or 0, 0 if the version can't be obtained -// or parsed. -// -// Currently only implemented for Linux. +// KernelVersion returns major and minor kernel version numbers +// parsed from the syscall.Uname's Release field, or (0, 0) if +// the version can't be obtained or parsed. func KernelVersion() (major, minor int) { var uname syscall.Utsname if err := syscall.Uname(&uname); err != nil { diff --git a/src/internal/syscall/unix/kernel_version_other.go b/src/internal/syscall/unix/kernel_version_other.go index fc65c1c823c7b8..91c14b31d999bc 100644 --- a/src/internal/syscall/unix/kernel_version_other.go +++ b/src/internal/syscall/unix/kernel_version_other.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !linux && !solaris +//go:build !freebsd && !linux && !solaris package unix diff --git a/src/internal/syscall/unix/kernel_version_solaris.go b/src/internal/syscall/unix/kernel_version_solaris.go index 3f399411d76bde..7904e1f4295707 100644 --- a/src/internal/syscall/unix/kernel_version_solaris.go +++ b/src/internal/syscall/unix/kernel_version_solaris.go @@ -77,11 +77,10 @@ var SupportSockNonblockCloexec = sync.OnceValue(func() bool { } if err != syscall.EPROTONOSUPPORT && err != syscall.EINVAL { // Something wrong with socket(), fall back to checking the kernel version. - major, minor := KernelVersion() if runtime.GOOS == "illumos" { - return major > 5 || (major == 5 && minor >= 11) // minimal requirement is SunOS 5.11 + return KernelVersionGE(5, 11) // Minimal requirement is SunOS 5.11. } - return major > 11 || (major == 11 && minor >= 4) + return KernelVersionGE(11, 4) } return false }) @@ -101,6 +100,5 @@ var SupportAccept4 = sync.OnceValue(func() bool { // SupportTCPKeepAliveIdleIntvlCNT determines whether the TCP_KEEPIDLE, TCP_KEEPINTVL and TCP_KEEPCNT // are available by checking the kernel version for Solaris 11.4. var SupportTCPKeepAliveIdleIntvlCNT = sync.OnceValue(func() bool { - major, minor := KernelVersion() - return major > 11 || (major == 11 && minor >= 4) + return KernelVersionGE(11, 4) }) diff --git a/src/internal/syscall/unix/net_darwin.go b/src/internal/syscall/unix/net_darwin.go index bbaa94b0d21d49..7e8f2ac12c6ba4 100644 --- a/src/internal/syscall/unix/net_darwin.go +++ b/src/internal/syscall/unix/net_darwin.go @@ -16,12 +16,13 @@ const ( AI_V4MAPPED = 0x800 AI_MASK = 0x1407 - EAI_AGAIN = 2 - EAI_NODATA = 7 - EAI_NONAME = 8 - EAI_SERVICE = 9 - EAI_SYSTEM = 11 - EAI_OVERFLOW = 14 + EAI_ADDRFAMILY = 1 + EAI_AGAIN = 2 + EAI_NODATA = 7 + EAI_NONAME = 8 + EAI_SERVICE = 9 + EAI_SYSTEM = 11 + EAI_OVERFLOW = 14 NI_NAMEREQD = 4 ) diff --git a/src/internal/syscall/unix/nofollow_posix.go b/src/internal/syscall/unix/nofollow_posix.go index de2ea14fc80bb5..3a5e0af05d33fd 100644 --- a/src/internal/syscall/unix/nofollow_posix.go +++ b/src/internal/syscall/unix/nofollow_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build unix && !dragonfly && !freebsd && !netbsd +//go:build (unix && !dragonfly && !freebsd && !netbsd) || wasip1 package unix diff --git a/src/internal/syscall/unix/renameat2_sysnum_linux.go b/src/internal/syscall/unix/renameat2_sysnum_linux.go new file mode 100644 index 00000000000000..e41a65db79abc8 --- /dev/null +++ b/src/internal/syscall/unix/renameat2_sysnum_linux.go @@ -0,0 +1,16 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (loong64 || riscv64) + +package unix + +import "syscall" + +const ( + // loong64 and riscv64 only have renameat2. + // renameat2 has an extra flags parameter. + // When called with a 0 flags it is identical to renameat. + renameatTrap uintptr = syscall.SYS_RENAMEAT2 +) diff --git a/src/internal/syscall/unix/renameat_sysnum_linux.go b/src/internal/syscall/unix/renameat_sysnum_linux.go new file mode 100644 index 00000000000000..d3663ad1dc4832 --- /dev/null +++ b/src/internal/syscall/unix/renameat_sysnum_linux.go @@ -0,0 +1,13 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && !(loong64 || riscv64) + +package unix + +import "syscall" + +const ( + renameatTrap uintptr = syscall.SYS_RENAMEAT +) diff --git a/src/internal/syscall/unix/syscall.go b/src/internal/syscall/unix/syscall.go new file mode 100644 index 00000000000000..99805d986be973 --- /dev/null +++ b/src/internal/syscall/unix/syscall.go @@ -0,0 +1,8 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix + +// Single-word zero for use when we need a valid pointer to 0 bytes. +var _zero uintptr diff --git a/src/internal/syscall/unix/sysnum_freebsd.go b/src/internal/syscall/unix/sysnum_freebsd.go new file mode 100644 index 00000000000000..2c81110409e6f0 --- /dev/null +++ b/src/internal/syscall/unix/sysnum_freebsd.go @@ -0,0 +1,7 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix + +const copyFileRangeTrap uintptr = 569 diff --git a/src/internal/syscall/unix/sysnum_linux_386.go b/src/internal/syscall/unix/sysnum_linux_386.go index be048bcf734b4b..c83beef74906bf 100644 --- a/src/internal/syscall/unix/sysnum_linux_386.go +++ b/src/internal/syscall/unix/sysnum_linux_386.go @@ -9,4 +9,5 @@ const ( copyFileRangeTrap uintptr = 377 pidfdSendSignalTrap uintptr = 424 pidfdOpenTrap uintptr = 434 + openat2Trap uintptr = 437 ) diff --git a/src/internal/syscall/unix/sysnum_linux_amd64.go b/src/internal/syscall/unix/sysnum_linux_amd64.go index 525de9cbd8c1a2..098e3320a0c7ed 100644 --- a/src/internal/syscall/unix/sysnum_linux_amd64.go +++ b/src/internal/syscall/unix/sysnum_linux_amd64.go @@ -9,4 +9,5 @@ const ( copyFileRangeTrap uintptr = 326 pidfdSendSignalTrap uintptr = 424 pidfdOpenTrap uintptr = 434 + openat2Trap uintptr = 437 ) diff --git a/src/internal/syscall/unix/sysnum_linux_arm.go b/src/internal/syscall/unix/sysnum_linux_arm.go index b8038922786828..f0cd45f9b00489 100644 --- a/src/internal/syscall/unix/sysnum_linux_arm.go +++ b/src/internal/syscall/unix/sysnum_linux_arm.go @@ -9,4 +9,5 @@ const ( copyFileRangeTrap uintptr = 391 pidfdSendSignalTrap uintptr = 424 pidfdOpenTrap uintptr = 434 + openat2Trap uintptr = 437 ) diff --git a/src/internal/syscall/unix/sysnum_linux_generic.go b/src/internal/syscall/unix/sysnum_linux_generic.go index b06bf6927312dd..ec622cff0d6ef6 100644 --- a/src/internal/syscall/unix/sysnum_linux_generic.go +++ b/src/internal/syscall/unix/sysnum_linux_generic.go @@ -15,4 +15,5 @@ const ( copyFileRangeTrap uintptr = 285 pidfdSendSignalTrap uintptr = 424 pidfdOpenTrap uintptr = 434 + openat2Trap uintptr = 437 ) diff --git a/src/internal/syscall/unix/sysnum_linux_mips64x.go b/src/internal/syscall/unix/sysnum_linux_mips64x.go index 8764f5dc8fcf7d..3875105d7de02e 100644 --- a/src/internal/syscall/unix/sysnum_linux_mips64x.go +++ b/src/internal/syscall/unix/sysnum_linux_mips64x.go @@ -11,4 +11,5 @@ const ( copyFileRangeTrap uintptr = 5320 pidfdSendSignalTrap uintptr = 5424 pidfdOpenTrap uintptr = 5434 + openat2Trap uintptr = 5437 ) diff --git a/src/internal/syscall/unix/sysnum_linux_mipsx.go b/src/internal/syscall/unix/sysnum_linux_mipsx.go index 9b2e587ba55fbd..bdd2fef2c30881 100644 --- a/src/internal/syscall/unix/sysnum_linux_mipsx.go +++ b/src/internal/syscall/unix/sysnum_linux_mipsx.go @@ -11,4 +11,5 @@ const ( copyFileRangeTrap uintptr = 4360 pidfdSendSignalTrap uintptr = 4424 pidfdOpenTrap uintptr = 4434 + openat2Trap uintptr = 4437 ) diff --git a/src/internal/syscall/unix/sysnum_linux_ppc64x.go b/src/internal/syscall/unix/sysnum_linux_ppc64x.go index 03e9c197433e1b..9291d58f5de9e0 100644 --- a/src/internal/syscall/unix/sysnum_linux_ppc64x.go +++ b/src/internal/syscall/unix/sysnum_linux_ppc64x.go @@ -11,4 +11,5 @@ const ( copyFileRangeTrap uintptr = 379 pidfdSendSignalTrap uintptr = 424 pidfdOpenTrap uintptr = 434 + openat2Trap uintptr = 437 ) diff --git a/src/internal/syscall/unix/sysnum_linux_s390x.go b/src/internal/syscall/unix/sysnum_linux_s390x.go index c6e3e02e46e36c..65a82d1c3d6368 100644 --- a/src/internal/syscall/unix/sysnum_linux_s390x.go +++ b/src/internal/syscall/unix/sysnum_linux_s390x.go @@ -9,4 +9,5 @@ const ( copyFileRangeTrap uintptr = 375 pidfdSendSignalTrap uintptr = 424 pidfdOpenTrap uintptr = 434 + openat2Trap uintptr = 437 ) diff --git a/src/internal/syscall/unix/utimes.go b/src/internal/syscall/unix/utimes.go new file mode 100644 index 00000000000000..332a4b69bb2f33 --- /dev/null +++ b/src/internal/syscall/unix/utimes.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix && !wasip1 + +package unix + +import ( + "syscall" + _ "unsafe" // for //go:linkname +) + +//go:linkname Utimensat syscall.utimensat +func Utimensat(dirfd int, path string, times *[2]syscall.Timespec, flag int) error diff --git a/src/internal/syscall/unix/utimes_wasip1.go b/src/internal/syscall/unix/utimes_wasip1.go new file mode 100644 index 00000000000000..a0711b0c042910 --- /dev/null +++ b/src/internal/syscall/unix/utimes_wasip1.go @@ -0,0 +1,42 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build wasip1 + +package unix + +import ( + "syscall" + "unsafe" +) + +//go:wasmimport wasi_snapshot_preview1 path_filestat_set_times +//go:noescape +func path_filestat_set_times(fd int32, flags uint32, path *byte, pathLen size, atim uint64, mtim uint64, fstflags uint32) syscall.Errno + +func Utimensat(dirfd int, path string, times *[2]syscall.Timespec, flag int) error { + if path == "" { + return syscall.EINVAL + } + atime := syscall.TimespecToNsec(times[0]) + mtime := syscall.TimespecToNsec(times[1]) + + var fflag uint32 + if times[0].Nsec != UTIME_OMIT { + fflag |= syscall.FILESTAT_SET_ATIM + } + if times[1].Nsec != UTIME_OMIT { + fflag |= syscall.FILESTAT_SET_MTIM + } + errno := path_filestat_set_times( + int32(dirfd), + syscall.LOOKUP_SYMLINK_FOLLOW, + unsafe.StringData(path), + size(len(path)), + uint64(atime), + uint64(mtime), + fflag, + ) + return errnoErr(errno) +} diff --git a/src/internal/syscall/unix/waitid_linux.go b/src/internal/syscall/unix/waitid_linux.go new file mode 100644 index 00000000000000..240a7f744ce21f --- /dev/null +++ b/src/internal/syscall/unix/waitid_linux.go @@ -0,0 +1,23 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unix + +import ( + "syscall" + "unsafe" +) + +const ( + P_PID = 1 + P_PIDFD = 3 +) + +func Waitid(idType int, id int, info *SiginfoChild, options int, rusage *syscall.Rusage) error { + _, _, errno := syscall.Syscall6(syscall.SYS_WAITID, uintptr(idType), uintptr(id), uintptr(unsafe.Pointer(info)), uintptr(options), uintptr(unsafe.Pointer(rusage)), 0) + if errno != 0 { + return errno + } + return nil +} diff --git a/src/internal/syscall/windows/at_windows.go b/src/internal/syscall/windows/at_windows.go new file mode 100644 index 00000000000000..2890e1fdcfc511 --- /dev/null +++ b/src/internal/syscall/windows/at_windows.go @@ -0,0 +1,603 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package windows + +import ( + "runtime" + "structs" + "syscall" + "unsafe" +) + +// Openat flags supported by syscall.Open. +const ( + O_DIRECTORY = 0x04000 // target must be a directory +) + +// Openat flags not supported by syscall.Open. +// +// These are invented values, use values in the 33-63 bit range +// to avoid overlap with flags and attributes supported by [syscall.Open]. +// +// When adding a new flag here, add an unexported version to +// the set of invented O_ values in syscall/types_windows.go +// to avoid overlap. +const ( + O_NOFOLLOW_ANY = 0x200000000 // disallow symlinks anywhere in the path + O_OPEN_REPARSE = 0x400000000 // FILE_OPEN_REPARSE_POINT, used by Lstat + O_WRITE_ATTRS = 0x800000000 // FILE_WRITE_ATTRIBUTES, used by Chmod +) + +func Openat(dirfd syscall.Handle, name string, flag uint64, perm uint32) (_ syscall.Handle, e1 error) { + if len(name) == 0 { + return syscall.InvalidHandle, syscall.ERROR_FILE_NOT_FOUND + } + + var access, options uint32 + switch flag & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) { + case syscall.O_RDONLY: + // FILE_GENERIC_READ includes FILE_LIST_DIRECTORY. + access = FILE_GENERIC_READ + case syscall.O_WRONLY: + access = FILE_GENERIC_WRITE + options |= FILE_NON_DIRECTORY_FILE + case syscall.O_RDWR: + access = FILE_GENERIC_READ | FILE_GENERIC_WRITE + options |= FILE_NON_DIRECTORY_FILE + default: + // Stat opens files without requesting read or write permissions, + // but we still need to request SYNCHRONIZE. + access = SYNCHRONIZE + } + if flag&syscall.O_CREAT != 0 { + access |= FILE_GENERIC_WRITE + } + if flag&syscall.O_APPEND != 0 { + access |= FILE_APPEND_DATA + // Remove FILE_WRITE_DATA access unless O_TRUNC is set, + // in which case we need it to truncate the file. + if flag&syscall.O_TRUNC == 0 { + access &^= FILE_WRITE_DATA + } + } + if flag&O_DIRECTORY != 0 { + options |= FILE_DIRECTORY_FILE + access |= FILE_LIST_DIRECTORY + } + if flag&syscall.O_SYNC != 0 { + options |= FILE_WRITE_THROUGH + } + if flag&O_WRITE_ATTRS != 0 { + access |= FILE_WRITE_ATTRIBUTES + } + // Allow File.Stat. + access |= STANDARD_RIGHTS_READ | FILE_READ_ATTRIBUTES | FILE_READ_EA + + objAttrs := &OBJECT_ATTRIBUTES{} + if flag&O_NOFOLLOW_ANY != 0 { + objAttrs.Attributes |= OBJ_DONT_REPARSE + } + if flag&syscall.O_CLOEXEC == 0 { + objAttrs.Attributes |= OBJ_INHERIT + } + if err := objAttrs.init(dirfd, name); err != nil { + return syscall.InvalidHandle, err + } + + if flag&O_OPEN_REPARSE != 0 { + options |= FILE_OPEN_REPARSE_POINT + } + + // We don't use FILE_OVERWRITE/FILE_OVERWRITE_IF, because when opening + // a file with FILE_ATTRIBUTE_READONLY these will replace an existing + // file with a new, read-only one. + // + // Instead, we ftruncate the file after opening when O_TRUNC is set. + var disposition uint32 + switch { + case flag&(syscall.O_CREAT|syscall.O_EXCL) == (syscall.O_CREAT | syscall.O_EXCL): + disposition = FILE_CREATE + options |= FILE_OPEN_REPARSE_POINT // don't follow symlinks + case flag&syscall.O_CREAT == syscall.O_CREAT: + disposition = FILE_OPEN_IF + default: + disposition = FILE_OPEN + } + + fileAttrs := uint32(FILE_ATTRIBUTE_NORMAL) + if perm&syscall.S_IWRITE == 0 { + fileAttrs = FILE_ATTRIBUTE_READONLY + } + + var h syscall.Handle + err := NtCreateFile( + &h, + SYNCHRONIZE|access, + objAttrs, + &IO_STATUS_BLOCK{}, + nil, + fileAttrs, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + disposition, + FILE_SYNCHRONOUS_IO_NONALERT|FILE_OPEN_FOR_BACKUP_INTENT|options, + nil, + 0, + ) + if err != nil { + return h, ntCreateFileError(err, flag) + } + + if flag&syscall.O_TRUNC != 0 { + err = syscall.Ftruncate(h, 0) + if err != nil { + syscall.CloseHandle(h) + return syscall.InvalidHandle, err + } + } + + return h, nil +} + +// ntCreateFileError maps error returns from NTCreateFile to user-visible errors. +func ntCreateFileError(err error, flag uint64) error { + s, ok := err.(NTStatus) + if !ok { + // Shouldn't really be possible, NtCreateFile always returns NTStatus. + return err + } + switch s { + case STATUS_REPARSE_POINT_ENCOUNTERED: + return syscall.ELOOP + case STATUS_NOT_A_DIRECTORY: + // ENOTDIR is the errno returned by open when O_DIRECTORY is specified + // and the target is not a directory. + // + // NtCreateFile can return STATUS_NOT_A_DIRECTORY under other circumstances, + // such as when opening "file/" where "file" is not a directory. + // (This might be Windows version dependent.) + // + // Only map STATUS_NOT_A_DIRECTORY to ENOTDIR when O_DIRECTORY is specified. + if flag&O_DIRECTORY != 0 { + return syscall.ENOTDIR + } + case STATUS_FILE_IS_A_DIRECTORY: + return syscall.EISDIR + case STATUS_OBJECT_NAME_COLLISION: + return syscall.EEXIST + } + return s.Errno() +} + +func Mkdirat(dirfd syscall.Handle, name string, mode uint32) error { + objAttrs := &OBJECT_ATTRIBUTES{} + if err := objAttrs.init(dirfd, name); err != nil { + return err + } + var h syscall.Handle + err := NtCreateFile( + &h, + FILE_GENERIC_READ, + objAttrs, + &IO_STATUS_BLOCK{}, + nil, + syscall.FILE_ATTRIBUTE_NORMAL, + syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, + FILE_CREATE, + FILE_DIRECTORY_FILE, + nil, + 0, + ) + if err != nil { + return ntCreateFileError(err, 0) + } + syscall.CloseHandle(h) + return nil +} + +func Deleteat(dirfd syscall.Handle, name string, options uint32) error { + if name == "." { + // NtOpenFile's documentation isn't explicit about what happens when deleting ".". + // Make this an error consistent with that of POSIX. + return syscall.EINVAL + } + objAttrs := &OBJECT_ATTRIBUTES{} + if err := objAttrs.init(dirfd, name); err != nil { + return err + } + var h syscall.Handle + err := NtOpenFile( + &h, + SYNCHRONIZE|FILE_READ_ATTRIBUTES|DELETE, + objAttrs, + &IO_STATUS_BLOCK{}, + FILE_SHARE_DELETE|FILE_SHARE_READ|FILE_SHARE_WRITE, + FILE_OPEN_REPARSE_POINT|FILE_OPEN_FOR_BACKUP_INTENT|FILE_SYNCHRONOUS_IO_NONALERT|options, + ) + if err != nil { + return ntCreateFileError(err, 0) + } + defer syscall.CloseHandle(h) + + if TestDeleteatFallback { + return deleteatFallback(h) + } + + const FileDispositionInformationEx = 64 + + // First, attempt to delete the file using POSIX semantics + // (which permit a file to be deleted while it is still open). + // This matches the behavior of DeleteFileW. + // + // The following call uses features available on different Windows versions: + // - FILE_DISPOSITION_INFORMATION_EX: Windows 10, version 1607 (aka RS1) + // - FILE_DISPOSITION_POSIX_SEMANTICS: Windows 10, version 1607 (aka RS1) + // - FILE_DISPOSITION_IGNORE_READONLY_ATTRIBUTE: Windows 10, version 1809 (aka RS5) + // + // Also, some file systems, like FAT32, don't support POSIX semantics. + err = NtSetInformationFile( + h, + &IO_STATUS_BLOCK{}, + unsafe.Pointer(&FILE_DISPOSITION_INFORMATION_EX{ + Flags: FILE_DISPOSITION_DELETE | + FILE_DISPOSITION_FORCE_IMAGE_SECTION_CHECK | + FILE_DISPOSITION_POSIX_SEMANTICS | + // This differs from DeleteFileW, but matches os.Remove's + // behavior on Unix platforms of permitting deletion of + // read-only files. + FILE_DISPOSITION_IGNORE_READONLY_ATTRIBUTE, + }), + uint32(unsafe.Sizeof(FILE_DISPOSITION_INFORMATION_EX{})), + FileDispositionInformationEx, + ) + switch err { + case nil: + return nil + case STATUS_INVALID_INFO_CLASS, // the operating system doesn't support FileDispositionInformationEx + STATUS_INVALID_PARAMETER, // the operating system doesn't support one of the flags + STATUS_NOT_SUPPORTED: // the file system doesn't support FILE_DISPOSITION_INFORMATION_EX or one of the flags + return deleteatFallback(h) + default: + return err.(NTStatus).Errno() + } +} + +// TestDeleteatFallback should only be used for testing purposes. +// When set, [Deleteat] uses the fallback path unconditionally. +var TestDeleteatFallback bool + +// deleteatFallback is a deleteat implementation that strives +// for compatibility with older Windows versions and file systems +// over performance. +func deleteatFallback(h syscall.Handle) error { + var data syscall.ByHandleFileInformation + if err := syscall.GetFileInformationByHandle(h, &data); err == nil && data.FileAttributes&syscall.FILE_ATTRIBUTE_READONLY != 0 { + // Remove read-only attribute. Reopen the file, as it was previously open without FILE_WRITE_ATTRIBUTES access + // in order to maximize compatibility in the happy path. + wh, err := ReOpenFile(h, + FILE_WRITE_ATTRIBUTES, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + syscall.FILE_FLAG_OPEN_REPARSE_POINT|syscall.FILE_FLAG_BACKUP_SEMANTICS, + ) + if err != nil { + return err + } + err = SetFileInformationByHandle( + wh, + FileBasicInfo, + unsafe.Pointer(&FILE_BASIC_INFO{ + FileAttributes: data.FileAttributes &^ FILE_ATTRIBUTE_READONLY, + }), + uint32(unsafe.Sizeof(FILE_BASIC_INFO{})), + ) + syscall.CloseHandle(wh) + if err != nil { + return err + } + } + + return SetFileInformationByHandle( + h, + FileDispositionInfo, + unsafe.Pointer(&FILE_DISPOSITION_INFO{ + DeleteFile: true, + }), + uint32(unsafe.Sizeof(FILE_DISPOSITION_INFO{})), + ) +} + +func Renameat(olddirfd syscall.Handle, oldpath string, newdirfd syscall.Handle, newpath string) error { + objAttrs := &OBJECT_ATTRIBUTES{} + if err := objAttrs.init(olddirfd, oldpath); err != nil { + return err + } + var h syscall.Handle + err := NtOpenFile( + &h, + SYNCHRONIZE|DELETE, + objAttrs, + &IO_STATUS_BLOCK{}, + FILE_SHARE_DELETE|FILE_SHARE_READ|FILE_SHARE_WRITE, + FILE_OPEN_REPARSE_POINT|FILE_OPEN_FOR_BACKUP_INTENT|FILE_SYNCHRONOUS_IO_NONALERT, + ) + if err != nil { + return ntCreateFileError(err, 0) + } + defer syscall.CloseHandle(h) + + renameInfoEx := FILE_RENAME_INFORMATION_EX{ + Flags: FILE_RENAME_REPLACE_IF_EXISTS | + FILE_RENAME_POSIX_SEMANTICS, + RootDirectory: newdirfd, + } + p16, err := syscall.UTF16FromString(newpath) + if err != nil { + return err + } + if len(p16) > len(renameInfoEx.FileName) { + return syscall.EINVAL + } + copy(renameInfoEx.FileName[:], p16) + renameInfoEx.FileNameLength = uint32((len(p16) - 1) * 2) + + const ( + FileRenameInformation = 10 + FileRenameInformationEx = 65 + ) + err = NtSetInformationFile( + h, + &IO_STATUS_BLOCK{}, + unsafe.Pointer(&renameInfoEx), + uint32(unsafe.Sizeof(FILE_RENAME_INFORMATION_EX{})), + FileRenameInformationEx, + ) + if err == nil { + return nil + } + + // If the prior rename failed, the filesystem might not support + // POSIX semantics (for example, FAT), or might not have implemented + // FILE_RENAME_INFORMATION_EX. + // + // Try again. + renameInfo := FILE_RENAME_INFORMATION{ + ReplaceIfExists: true, + RootDirectory: newdirfd, + } + copy(renameInfo.FileName[:], p16) + renameInfo.FileNameLength = renameInfoEx.FileNameLength + + err = NtSetInformationFile( + h, + &IO_STATUS_BLOCK{}, + unsafe.Pointer(&renameInfo), + uint32(unsafe.Sizeof(FILE_RENAME_INFORMATION{})), + FileRenameInformation, + ) + if st, ok := err.(NTStatus); ok { + return st.Errno() + } + return err +} + +func Linkat(olddirfd syscall.Handle, oldpath string, newdirfd syscall.Handle, newpath string) error { + objAttrs := &OBJECT_ATTRIBUTES{} + if err := objAttrs.init(olddirfd, oldpath); err != nil { + return err + } + var h syscall.Handle + err := NtOpenFile( + &h, + SYNCHRONIZE|FILE_WRITE_ATTRIBUTES, + objAttrs, + &IO_STATUS_BLOCK{}, + FILE_SHARE_DELETE|FILE_SHARE_READ|FILE_SHARE_WRITE, + FILE_OPEN_REPARSE_POINT|FILE_OPEN_FOR_BACKUP_INTENT|FILE_SYNCHRONOUS_IO_NONALERT, + ) + if err != nil { + return ntCreateFileError(err, 0) + } + defer syscall.CloseHandle(h) + + linkInfo := FILE_LINK_INFORMATION{ + RootDirectory: newdirfd, + } + p16, err := syscall.UTF16FromString(newpath) + if err != nil { + return err + } + if len(p16) > len(linkInfo.FileName) { + return syscall.EINVAL + } + copy(linkInfo.FileName[:], p16) + linkInfo.FileNameLength = uint32((len(p16) - 1) * 2) + + const ( + FileLinkInformation = 11 + ) + err = NtSetInformationFile( + h, + &IO_STATUS_BLOCK{}, + unsafe.Pointer(&linkInfo), + uint32(unsafe.Sizeof(FILE_LINK_INFORMATION{})), + FileLinkInformation, + ) + if st, ok := err.(NTStatus); ok { + return st.Errno() + } + return err +} + +// SymlinkatFlags configure Symlinkat. +// +// Symbolic links have two properties: They may be directory or file links, +// and they may be absolute or relative. +// +// The Windows API defines flags describing these properties +// (SYMBOLIC_LINK_FLAG_DIRECTORY and SYMLINK_FLAG_RELATIVE), +// but the flags are passed to different system calls and +// do not have distinct values, so we define our own enumeration +// that permits expressing both. +type SymlinkatFlags uint + +const ( + SYMLINKAT_DIRECTORY = SymlinkatFlags(1 << iota) + SYMLINKAT_RELATIVE +) + +func Symlinkat(oldname string, newdirfd syscall.Handle, newname string, flags SymlinkatFlags) error { + // Temporarily acquire symlink-creating privileges if possible. + // This is the behavior of CreateSymbolicLinkW. + // + // (When passed the SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE flag, + // CreateSymbolicLinkW ignores errors in acquiring privileges, as we do here.) + return withPrivilege("SeCreateSymbolicLinkPrivilege", func() error { + return symlinkat(oldname, newdirfd, newname, flags) + }) +} + +func symlinkat(oldname string, newdirfd syscall.Handle, newname string, flags SymlinkatFlags) error { + oldnameu16, err := syscall.UTF16FromString(oldname) + if err != nil { + return err + } + oldnameu16 = oldnameu16[:len(oldnameu16)-1] // trim off terminal NUL + + var options uint32 + if flags&SYMLINKAT_DIRECTORY != 0 { + options |= FILE_DIRECTORY_FILE + } else { + options |= FILE_NON_DIRECTORY_FILE + } + + objAttrs := &OBJECT_ATTRIBUTES{} + if err := objAttrs.init(newdirfd, newname); err != nil { + return err + } + var h syscall.Handle + err = NtCreateFile( + &h, + SYNCHRONIZE|FILE_WRITE_ATTRIBUTES|DELETE, + objAttrs, + &IO_STATUS_BLOCK{}, + nil, + syscall.FILE_ATTRIBUTE_NORMAL, + 0, + FILE_CREATE, + FILE_OPEN_REPARSE_POINT|FILE_OPEN_FOR_BACKUP_INTENT|FILE_SYNCHRONOUS_IO_NONALERT|options, + nil, + 0, + ) + if err != nil { + return ntCreateFileError(err, 0) + } + defer syscall.CloseHandle(h) + + // https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/ns-ntifs-_reparse_data_buffer + type reparseDataBufferT struct { + _ structs.HostLayout + + ReparseTag uint32 + ReparseDataLength uint16 + Reserved uint16 + + SubstituteNameOffset uint16 + SubstituteNameLength uint16 + PrintNameOffset uint16 + PrintNameLength uint16 + Flags uint32 + } + + const ( + headerSize = uint16(unsafe.Offsetof(reparseDataBufferT{}.SubstituteNameOffset)) + bufferSize = uint16(unsafe.Sizeof(reparseDataBufferT{})) + ) + + // Data buffer containing a SymbolicLinkReparseBuffer followed by the link target. + rdbbuf := make([]byte, bufferSize+uint16(2*len(oldnameu16))) + + rdb := (*reparseDataBufferT)(unsafe.Pointer(&rdbbuf[0])) + rdb.ReparseTag = syscall.IO_REPARSE_TAG_SYMLINK + rdb.ReparseDataLength = uint16(len(rdbbuf)) - uint16(headerSize) + rdb.SubstituteNameOffset = 0 + rdb.SubstituteNameLength = uint16(2 * len(oldnameu16)) + rdb.PrintNameOffset = 0 + rdb.PrintNameLength = rdb.SubstituteNameLength + if flags&SYMLINKAT_RELATIVE != 0 { + rdb.Flags = SYMLINK_FLAG_RELATIVE + } + + namebuf := rdbbuf[bufferSize:] + copy(namebuf, unsafe.String((*byte)(unsafe.Pointer(&oldnameu16[0])), 2*len(oldnameu16))) + + err = syscall.DeviceIoControl( + h, + FSCTL_SET_REPARSE_POINT, + &rdbbuf[0], + uint32(len(rdbbuf)), + nil, + 0, + nil, + nil) + if err != nil { + // Creating the symlink has failed, so try to remove the file. + const FileDispositionInformation = 13 + NtSetInformationFile( + h, + &IO_STATUS_BLOCK{}, + unsafe.Pointer(&FILE_DISPOSITION_INFORMATION{ + DeleteFile: true, + }), + uint32(unsafe.Sizeof(FILE_DISPOSITION_INFORMATION{})), + FileDispositionInformation, + ) + return err + } + + return nil +} + +// withPrivilege temporariliy acquires the named privilege and runs f. +// If the privilege cannot be acquired it runs f anyway, +// which should fail with an appropriate error. +func withPrivilege(privilege string, f func() error) error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + err := ImpersonateSelf(SecurityImpersonation) + if err != nil { + return f() + } + defer RevertToSelf() + + curThread, err := GetCurrentThread() + if err != nil { + return f() + } + var token syscall.Token + err = OpenThreadToken(curThread, syscall.TOKEN_QUERY|TOKEN_ADJUST_PRIVILEGES, false, &token) + if err != nil { + return f() + } + defer syscall.CloseHandle(syscall.Handle(token)) + + privStr, err := syscall.UTF16PtrFromString(privilege) + if err != nil { + return f() + } + var tokenPriv TOKEN_PRIVILEGES + err = LookupPrivilegeValue(nil, privStr, &tokenPriv.Privileges[0].Luid) + if err != nil { + return f() + } + + tokenPriv.PrivilegeCount = 1 + tokenPriv.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED + err = AdjustTokenPrivileges(token, false, &tokenPriv, 0, nil, nil) + if err != nil { + return f() + } + + return f() +} diff --git a/src/internal/syscall/windows/at_windows_test.go b/src/internal/syscall/windows/at_windows_test.go new file mode 100644 index 00000000000000..daeb4fcde37d70 --- /dev/null +++ b/src/internal/syscall/windows/at_windows_test.go @@ -0,0 +1,58 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package windows_test + +import ( + "internal/syscall/windows" + "os" + "path/filepath" + "syscall" + "testing" +) + +func TestOpen(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + file := filepath.Join(dir, "a") + f, err := os.Create(file) + if err != nil { + t.Fatal(err) + } + f.Close() + + tests := []struct { + path string + flag int + err error + }{ + {dir, syscall.O_RDONLY, nil}, + {dir, syscall.O_CREAT, nil}, + {dir, syscall.O_RDONLY | syscall.O_CREAT, nil}, + {file, syscall.O_APPEND | syscall.O_WRONLY | os.O_CREATE, nil}, + {file, syscall.O_APPEND | syscall.O_WRONLY | os.O_CREATE | os.O_TRUNC, nil}, + {dir, syscall.O_RDONLY | syscall.O_TRUNC, syscall.ERROR_ACCESS_DENIED}, + {dir, syscall.O_WRONLY | syscall.O_RDWR, nil}, // TODO: syscall.Open returns EISDIR here, we should reconcile this + {dir, syscall.O_WRONLY, syscall.EISDIR}, + {dir, syscall.O_RDWR, syscall.EISDIR}, + } + for i, tt := range tests { + dir := filepath.Dir(tt.path) + dirfd, err := syscall.Open(dir, syscall.O_RDONLY, 0) + if err != nil { + t.Error(err) + continue + } + base := filepath.Base(tt.path) + h, err := windows.Openat(dirfd, base, uint64(tt.flag), 0o660) + syscall.CloseHandle(dirfd) + if err == nil { + syscall.CloseHandle(h) + } + if err != tt.err { + t.Errorf("%d: Open got %q, want %q", i, err, tt.err) + } + } +} diff --git a/src/internal/syscall/windows/exec_windows_test.go b/src/internal/syscall/windows/exec_windows_test.go index 72550b5a84eac8..d10d8c269c5871 100644 --- a/src/internal/syscall/windows/exec_windows_test.go +++ b/src/internal/syscall/windows/exec_windows_test.go @@ -9,6 +9,7 @@ package windows_test import ( "fmt" "internal/syscall/windows" + "internal/testenv" "os" "os/exec" "syscall" @@ -29,7 +30,7 @@ func TestRunAtLowIntegrity(t *testing.T) { return } - cmd := exec.Command(os.Args[0], "-test.run=^TestRunAtLowIntegrity$", "--") + cmd := exec.Command(testenv.Executable(t), "-test.run=^TestRunAtLowIntegrity$", "--") cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"} token, err := getIntegrityLevelToken(sidWilLow) @@ -129,7 +130,7 @@ func getIntegrityLevelToken(wns string) (syscall.Token, error) { err = windows.SetTokenInformation(token, syscall.TokenIntegrityLevel, - uintptr(unsafe.Pointer(tml)), + unsafe.Pointer(tml), tml.Size()) if err != nil { token.Close() diff --git a/src/internal/syscall/windows/memory_windows.go b/src/internal/syscall/windows/memory_windows.go deleted file mode 100644 index 8fb34cf34924bc..00000000000000 --- a/src/internal/syscall/windows/memory_windows.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package windows - -type MemoryBasicInformation struct { - // A pointer to the base address of the region of pages. - BaseAddress uintptr - // A pointer to the base address of a range of pages allocated by the VirtualAlloc function. - // The page pointed to by the BaseAddress member is contained within this allocation range. - AllocationBase uintptr - // The memory protection option when the region was initially allocated - AllocationProtect uint32 - PartitionId uint16 - // The size of the region beginning at the base address in which all pages have identical attributes, in bytes. - RegionSize uintptr - // The state of the pages in the region. - State uint32 - // The access protection of the pages in the region. - Protect uint32 - // The type of pages in the region. - Type uint32 -} diff --git a/src/internal/syscall/windows/net_windows.go b/src/internal/syscall/windows/net_windows.go index 9fa5ecf8408321..023ddaaa8c1af6 100644 --- a/src/internal/syscall/windows/net_windows.go +++ b/src/internal/syscall/windows/net_windows.go @@ -18,6 +18,7 @@ func WSASendtoInet4(s syscall.Handle, bufs *syscall.WSABuf, bufcnt uint32, sent func WSASendtoInet6(s syscall.Handle, bufs *syscall.WSABuf, bufcnt uint32, sent *uint32, flags uint32, to *syscall.SockaddrInet6, overlapped *syscall.Overlapped, croutine *byte) (err error) const ( + SO_TYPE = 0x1008 SIO_TCP_INITIAL_RTO = syscall.IOC_IN | syscall.IOC_VENDOR | 17 TCP_INITIAL_RTO_UNSPECIFIED_RTT = ^uint16(0) TCP_INITIAL_RTO_NO_SYN_RETRANSMISSIONS = ^uint8(1) diff --git a/src/internal/syscall/windows/nonblocking_windows.go b/src/internal/syscall/windows/nonblocking_windows.go new file mode 100644 index 00000000000000..1565e87e2658f7 --- /dev/null +++ b/src/internal/syscall/windows/nonblocking_windows.go @@ -0,0 +1,21 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package windows + +import ( + "syscall" + "unsafe" +) + +// IsNonblock returns whether the file descriptor fd is opened +// in non-blocking mode, that is, the [windows.O_FILE_FLAG_OVERLAPPED] flag +// was set when the file was opened. +func IsNonblock(fd syscall.Handle) (nonblocking bool, err error) { + var info FILE_MODE_INFORMATION + if err := NtQueryInformationFile(syscall.Handle(fd), &IO_STATUS_BLOCK{}, unsafe.Pointer(&info), uint32(unsafe.Sizeof(info)), FileModeInformation); err != nil { + return false, err + } + return info.Mode&(FILE_SYNCHRONOUS_IO_ALERT|FILE_SYNCHRONOUS_IO_NONALERT) == 0, nil +} diff --git a/src/internal/syscall/windows/registry/registry_test.go b/src/internal/syscall/windows/registry/registry_test.go index 12eae54a8f3814..1e41d7d9f9f1dc 100644 --- a/src/internal/syscall/windows/registry/registry_test.go +++ b/src/internal/syscall/windows/registry/registry_test.go @@ -10,6 +10,7 @@ import ( "bytes" "crypto/rand" "os" + "slices" "syscall" "testing" "unsafe" @@ -100,21 +101,6 @@ func TestCreateOpenDeleteKey(t *testing.T) { } } -func equalStringSlice(a, b []string) bool { - if len(a) != len(b) { - return false - } - if a == nil { - return true - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} - type ValueTest struct { Type uint32 Name string @@ -304,7 +290,7 @@ func testGetStringsValue(t *testing.T, k registry.Key, test ValueTest) { t.Errorf("GetStringsValue(%s) failed: %v", test.Name, err) return } - if !equalStringSlice(got, test.Value.([]string)) { + if !slices.Equal(got, test.Value.([]string)) { t.Errorf("want %s value %#v, got %#v", test.Name, test.Value, got) return } diff --git a/src/internal/syscall/windows/registry/zsyscall_windows.go b/src/internal/syscall/windows/registry/zsyscall_windows.go index cab13193749c1b..07371f137ddb75 100644 --- a/src/internal/syscall/windows/registry/zsyscall_windows.go +++ b/src/internal/syscall/windows/registry/zsyscall_windows.go @@ -50,7 +50,7 @@ var ( ) func regCreateKeyEx(key syscall.Handle, subkey *uint16, reserved uint32, class *uint16, options uint32, desired uint32, sa *syscall.SecurityAttributes, result *syscall.Handle, disposition *uint32) (regerrno error) { - r0, _, _ := syscall.Syscall9(procRegCreateKeyExW.Addr(), 9, uintptr(key), uintptr(unsafe.Pointer(subkey)), uintptr(reserved), uintptr(unsafe.Pointer(class)), uintptr(options), uintptr(desired), uintptr(unsafe.Pointer(sa)), uintptr(unsafe.Pointer(result)), uintptr(unsafe.Pointer(disposition))) + r0, _, _ := syscall.SyscallN(procRegCreateKeyExW.Addr(), uintptr(key), uintptr(unsafe.Pointer(subkey)), uintptr(reserved), uintptr(unsafe.Pointer(class)), uintptr(options), uintptr(desired), uintptr(unsafe.Pointer(sa)), uintptr(unsafe.Pointer(result)), uintptr(unsafe.Pointer(disposition))) if r0 != 0 { regerrno = syscall.Errno(r0) } @@ -58,7 +58,7 @@ func regCreateKeyEx(key syscall.Handle, subkey *uint16, reserved uint32, class * } func regDeleteKey(key syscall.Handle, subkey *uint16) (regerrno error) { - r0, _, _ := syscall.Syscall(procRegDeleteKeyW.Addr(), 2, uintptr(key), uintptr(unsafe.Pointer(subkey)), 0) + r0, _, _ := syscall.SyscallN(procRegDeleteKeyW.Addr(), uintptr(key), uintptr(unsafe.Pointer(subkey))) if r0 != 0 { regerrno = syscall.Errno(r0) } @@ -66,7 +66,7 @@ func regDeleteKey(key syscall.Handle, subkey *uint16) (regerrno error) { } func regDeleteValue(key syscall.Handle, name *uint16) (regerrno error) { - r0, _, _ := syscall.Syscall(procRegDeleteValueW.Addr(), 2, uintptr(key), uintptr(unsafe.Pointer(name)), 0) + r0, _, _ := syscall.SyscallN(procRegDeleteValueW.Addr(), uintptr(key), uintptr(unsafe.Pointer(name))) if r0 != 0 { regerrno = syscall.Errno(r0) } @@ -74,7 +74,7 @@ func regDeleteValue(key syscall.Handle, name *uint16) (regerrno error) { } func regEnumValue(key syscall.Handle, index uint32, name *uint16, nameLen *uint32, reserved *uint32, valtype *uint32, buf *byte, buflen *uint32) (regerrno error) { - r0, _, _ := syscall.Syscall9(procRegEnumValueW.Addr(), 8, uintptr(key), uintptr(index), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(nameLen)), uintptr(unsafe.Pointer(reserved)), uintptr(unsafe.Pointer(valtype)), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(buflen)), 0) + r0, _, _ := syscall.SyscallN(procRegEnumValueW.Addr(), uintptr(key), uintptr(index), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(nameLen)), uintptr(unsafe.Pointer(reserved)), uintptr(unsafe.Pointer(valtype)), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(buflen))) if r0 != 0 { regerrno = syscall.Errno(r0) } @@ -82,7 +82,7 @@ func regEnumValue(key syscall.Handle, index uint32, name *uint16, nameLen *uint3 } func regLoadMUIString(key syscall.Handle, name *uint16, buf *uint16, buflen uint32, buflenCopied *uint32, flags uint32, dir *uint16) (regerrno error) { - r0, _, _ := syscall.Syscall9(procRegLoadMUIStringW.Addr(), 7, uintptr(key), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(buf)), uintptr(buflen), uintptr(unsafe.Pointer(buflenCopied)), uintptr(flags), uintptr(unsafe.Pointer(dir)), 0, 0) + r0, _, _ := syscall.SyscallN(procRegLoadMUIStringW.Addr(), uintptr(key), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(buf)), uintptr(buflen), uintptr(unsafe.Pointer(buflenCopied)), uintptr(flags), uintptr(unsafe.Pointer(dir))) if r0 != 0 { regerrno = syscall.Errno(r0) } @@ -90,7 +90,7 @@ func regLoadMUIString(key syscall.Handle, name *uint16, buf *uint16, buflen uint } func regSetValueEx(key syscall.Handle, valueName *uint16, reserved uint32, vtype uint32, buf *byte, bufsize uint32) (regerrno error) { - r0, _, _ := syscall.Syscall6(procRegSetValueExW.Addr(), 6, uintptr(key), uintptr(unsafe.Pointer(valueName)), uintptr(reserved), uintptr(vtype), uintptr(unsafe.Pointer(buf)), uintptr(bufsize)) + r0, _, _ := syscall.SyscallN(procRegSetValueExW.Addr(), uintptr(key), uintptr(unsafe.Pointer(valueName)), uintptr(reserved), uintptr(vtype), uintptr(unsafe.Pointer(buf)), uintptr(bufsize)) if r0 != 0 { regerrno = syscall.Errno(r0) } @@ -98,7 +98,7 @@ func regSetValueEx(key syscall.Handle, valueName *uint16, reserved uint32, vtype } func expandEnvironmentStrings(src *uint16, dst *uint16, size uint32) (n uint32, err error) { - r0, _, e1 := syscall.Syscall(procExpandEnvironmentStringsW.Addr(), 3, uintptr(unsafe.Pointer(src)), uintptr(unsafe.Pointer(dst)), uintptr(size)) + r0, _, e1 := syscall.SyscallN(procExpandEnvironmentStringsW.Addr(), uintptr(unsafe.Pointer(src)), uintptr(unsafe.Pointer(dst)), uintptr(size)) n = uint32(r0) if n == 0 { err = errnoErr(e1) diff --git a/src/internal/syscall/windows/security_windows.go b/src/internal/syscall/windows/security_windows.go index c8c5cbed747360..f0ab52ac819baf 100644 --- a/src/internal/syscall/windows/security_windows.go +++ b/src/internal/syscall/windows/security_windows.go @@ -5,6 +5,7 @@ package windows import ( + "runtime" "syscall" "unsafe" ) @@ -18,6 +19,8 @@ const ( //sys ImpersonateSelf(impersonationlevel uint32) (err error) = advapi32.ImpersonateSelf //sys RevertToSelf() (err error) = advapi32.RevertToSelf +//sys ImpersonateLoggedOnUser(token syscall.Token) (err error) = advapi32.ImpersonateLoggedOnUser +//sys LogonUser(username *uint16, domain *uint16, password *uint16, logonType uint32, logonProvider uint32, token *syscall.Token) (err error) = advapi32.LogonUserW const ( TOKEN_ADJUST_PRIVILEGES = 0x0020 @@ -58,7 +61,7 @@ func AdjustTokenPrivileges(token syscall.Token, disableAllPrivileges bool, newst } //sys DuplicateTokenEx(hExistingToken syscall.Token, dwDesiredAccess uint32, lpTokenAttributes *syscall.SecurityAttributes, impersonationLevel uint32, tokenType TokenType, phNewToken *syscall.Token) (err error) = advapi32.DuplicateTokenEx -//sys SetTokenInformation(tokenHandle syscall.Token, tokenInformationClass uint32, tokenInformation uintptr, tokenInformationLength uint32) (err error) = advapi32.SetTokenInformation +//sys SetTokenInformation(tokenHandle syscall.Token, tokenInformationClass uint32, tokenInformation unsafe.Pointer, tokenInformationLength uint32) (err error) = advapi32.SetTokenInformation type SID_AND_ATTRIBUTES struct { Sid *syscall.SID @@ -93,6 +96,26 @@ type LocalGroupUserInfo0 struct { Name *uint16 } +const ( + NERR_UserNotFound syscall.Errno = 2221 + NERR_UserExists syscall.Errno = 2224 +) + +const ( + USER_PRIV_USER = 1 +) + +type UserInfo1 struct { + Name *uint16 + Password *uint16 + PasswordAge uint32 + Priv uint32 + HomeDir *uint16 + Comment *uint16 + Flags uint32 + ScriptPath *uint16 +} + type UserInfo4 struct { Name *uint16 Password *uint16 @@ -125,6 +148,8 @@ type UserInfo4 struct { PasswordExpired uint32 } +//sys NetUserAdd(serverName *uint16, level uint32, buf *byte, parmErr *uint32) (neterr error) = netapi32.NetUserAdd +//sys NetUserDel(serverName *uint16, userName *uint16) (neterr error) = netapi32.NetUserDel //sys NetUserGetLocalGroups(serverName *uint16, userName *uint16, level uint32, flags uint32, buf **byte, prefMaxLen uint32, entriesRead *uint32, totalEntries *uint32) (neterr error) = netapi32.NetUserGetLocalGroups // GetSystemDirectory retrieves the path to current location of the system @@ -132,3 +157,108 @@ type UserInfo4 struct { // //go:linkname GetSystemDirectory func GetSystemDirectory() string // Implemented in runtime package. + +// GetUserName retrieves the user name of the current thread +// in the specified format. +func GetUserName(format uint32) (string, error) { + n := uint32(50) + for { + b := make([]uint16, n) + e := syscall.GetUserNameEx(format, &b[0], &n) + if e == nil { + return syscall.UTF16ToString(b[:n]), nil + } + if e != syscall.ERROR_MORE_DATA { + return "", e + } + if n <= uint32(len(b)) { + return "", e + } + } +} + +// getTokenInfo retrieves a specified type of information about an access token. +func getTokenInfo(t syscall.Token, class uint32, initSize int) (unsafe.Pointer, error) { + n := uint32(initSize) + for { + b := make([]byte, n) + e := syscall.GetTokenInformation(t, class, &b[0], uint32(len(b)), &n) + if e == nil { + return unsafe.Pointer(&b[0]), nil + } + if e != syscall.ERROR_INSUFFICIENT_BUFFER { + return nil, e + } + if n <= uint32(len(b)) { + return nil, e + } + } +} + +type TOKEN_GROUPS struct { + GroupCount uint32 + Groups [1]SID_AND_ATTRIBUTES +} + +func (g *TOKEN_GROUPS) AllGroups() []SID_AND_ATTRIBUTES { + return (*[(1 << 28) - 1]SID_AND_ATTRIBUTES)(unsafe.Pointer(&g.Groups[0]))[:g.GroupCount:g.GroupCount] +} + +func GetTokenGroups(t syscall.Token) (*TOKEN_GROUPS, error) { + i, e := getTokenInfo(t, syscall.TokenGroups, 50) + if e != nil { + return nil, e + } + return (*TOKEN_GROUPS)(i), nil +} + +// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-sid_identifier_authority +type SID_IDENTIFIER_AUTHORITY struct { + Value [6]byte +} + +const ( + SID_REVISION = 1 + // https://learn.microsoft.com/en-us/windows/win32/services/localsystem-account + SECURITY_LOCAL_SYSTEM_RID = 18 + // https://learn.microsoft.com/en-us/windows/win32/services/localservice-account + SECURITY_LOCAL_SERVICE_RID = 19 + // https://learn.microsoft.com/en-us/windows/win32/services/networkservice-account + SECURITY_NETWORK_SERVICE_RID = 20 +) + +var SECURITY_NT_AUTHORITY = SID_IDENTIFIER_AUTHORITY{ + Value: [6]byte{0, 0, 0, 0, 0, 5}, +} + +//sys IsValidSid(sid *syscall.SID) (valid bool) = advapi32.IsValidSid +//sys getSidIdentifierAuthority(sid *syscall.SID) (idauth uintptr) = advapi32.GetSidIdentifierAuthority +//sys getSidSubAuthority(sid *syscall.SID, subAuthorityIdx uint32) (subAuth uintptr) = advapi32.GetSidSubAuthority +//sys getSidSubAuthorityCount(sid *syscall.SID) (count uintptr) = advapi32.GetSidSubAuthorityCount + +// The following GetSid* functions are marked as //go:nocheckptr because checkptr +// instrumentation can't see that the pointer returned by the syscall is pointing +// into the sid's memory, which is normally allocated on the Go heap. Therefore, +// the checkptr instrumentation would incorrectly flag the pointer dereference +// as pointing to an invalid allocation. +// Also, use runtime.KeepAlive to ensure that the sid is not garbage collected +// before the GetSid* functions return, as the Go GC is not aware that the +// pointers returned by the syscall are pointing into the sid's memory. + +//go:nocheckptr +func GetSidIdentifierAuthority(sid *syscall.SID) SID_IDENTIFIER_AUTHORITY { + defer runtime.KeepAlive(sid) + return *(*SID_IDENTIFIER_AUTHORITY)(unsafe.Pointer(getSidIdentifierAuthority(sid))) +} + +//go:nocheckptr +func GetSidSubAuthority(sid *syscall.SID, subAuthorityIdx uint32) uint32 { + defer runtime.KeepAlive(sid) + return *(*uint32)(unsafe.Pointer(getSidSubAuthority(sid, subAuthorityIdx))) +} + +//go:nocheckptr +func GetSidSubAuthorityCount(sid *syscall.SID) uint8 { + defer runtime.KeepAlive(sid) + return *(*uint8)(unsafe.Pointer(getSidSubAuthorityCount(sid))) +} diff --git a/src/internal/syscall/windows/string_windows.go b/src/internal/syscall/windows/string_windows.go new file mode 100644 index 00000000000000..eb6893d792b4c7 --- /dev/null +++ b/src/internal/syscall/windows/string_windows.go @@ -0,0 +1,32 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package windows + +import "syscall" + +// NTUnicodeString is a UTF-16 string for NT native APIs, corresponding to UNICODE_STRING. +type NTUnicodeString struct { + Length uint16 + MaximumLength uint16 + Buffer *uint16 +} + +// NewNTUnicodeString returns a new NTUnicodeString structure for use with native +// NT APIs that work over the NTUnicodeString type. Note that most Windows APIs +// do not use NTUnicodeString, and instead UTF16PtrFromString should be used for +// the more common *uint16 string type. +func NewNTUnicodeString(s string) (*NTUnicodeString, error) { + s16, err := syscall.UTF16FromString(s) + if err != nil { + return nil, err + } + n := uint16(len(s16) * 2) + // https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdmsec/nf-wdmsec-wdmlibrtlinitunicodestringex + return &NTUnicodeString{ + Length: n - 2, // subtract 2 bytes for the NUL terminator + MaximumLength: n, + Buffer: &s16[0], + }, nil +} diff --git a/src/internal/syscall/windows/symlink_windows.go b/src/internal/syscall/windows/symlink_windows.go index b91246037b5efe..b8249b3848ea02 100644 --- a/src/internal/syscall/windows/symlink_windows.go +++ b/src/internal/syscall/windows/symlink_windows.go @@ -19,6 +19,7 @@ const ( FileBasicInfo = 0 // FILE_BASIC_INFO FileStandardInfo = 1 // FILE_STANDARD_INFO FileNameInfo = 2 // FILE_NAME_INFO + FileDispositionInfo = 4 // FILE_DISPOSITION_INFO FileStreamInfo = 7 // FILE_STREAM_INFO FileCompressionInfo = 8 // FILE_COMPRESSION_INFO FileAttributeTagInfo = 9 // FILE_ATTRIBUTE_TAG_INFO diff --git a/src/internal/syscall/windows/syscall_windows.go b/src/internal/syscall/windows/syscall_windows.go index cc26a50bb0acf2..deea3f35ca8d25 100644 --- a/src/internal/syscall/windows/syscall_windows.go +++ b/src/internal/syscall/windows/syscall_windows.go @@ -32,14 +32,19 @@ func UTF16PtrToString(p *uint16) string { } const ( + ERROR_INVALID_HANDLE syscall.Errno = 6 ERROR_BAD_LENGTH syscall.Errno = 24 ERROR_SHARING_VIOLATION syscall.Errno = 32 ERROR_LOCK_VIOLATION syscall.Errno = 33 ERROR_NOT_SUPPORTED syscall.Errno = 50 ERROR_CALL_NOT_IMPLEMENTED syscall.Errno = 120 ERROR_INVALID_NAME syscall.Errno = 123 + ERROR_NEGATIVE_SEEK syscall.Errno = 131 ERROR_LOCK_FAILED syscall.Errno = 167 + ERROR_IO_INCOMPLETE syscall.Errno = 996 + ERROR_NO_TOKEN syscall.Errno = 1008 ERROR_NO_UNICODE_TRANSLATION syscall.Errno = 1113 + ERROR_CANT_ACCESS_FILE syscall.Errno = 1920 ) const ( @@ -184,13 +189,14 @@ const ( IfOperStatusLowerLayerDown = 7 ) -//sys GetAdaptersAddresses(family uint32, flags uint32, reserved uintptr, adapterAddresses *IpAdapterAddresses, sizePointer *uint32) (errcode error) = iphlpapi.GetAdaptersAddresses +//sys GetAdaptersAddresses(family uint32, flags uint32, reserved unsafe.Pointer, adapterAddresses *IpAdapterAddresses, sizePointer *uint32) (errcode error) = iphlpapi.GetAdaptersAddresses //sys GetComputerNameEx(nameformat uint32, buf *uint16, n *uint32) (err error) = GetComputerNameExW //sys MoveFileEx(from *uint16, to *uint16, flags uint32) (err error) = MoveFileExW //sys GetModuleFileName(module syscall.Handle, fn *uint16, len uint32) (n uint32, err error) = kernel32.GetModuleFileNameW //sys SetFileInformationByHandle(handle syscall.Handle, fileInformationClass uint32, buf unsafe.Pointer, bufsize uint32) (err error) = kernel32.SetFileInformationByHandle //sys VirtualQuery(address uintptr, buffer *MemoryBasicInformation, length uintptr) (err error) = kernel32.VirtualQuery //sys GetTempPath2(buflen uint32, buf *uint16) (n uint32, err error) = GetTempPath2W +//sys GetFileSizeEx(handle syscall.Handle, size *int64) (err error) = kernel32.GetFileSizeEx const ( // flags for CreateToolhelp32Snapshot @@ -255,7 +261,7 @@ var sendRecvMsgFunc struct { } type WSAMsg struct { - Name syscall.Pointer + Name *syscall.RawSockaddrAny Namelen int32 Buffers *syscall.WSABuf BufferCount uint32 @@ -264,6 +270,7 @@ type WSAMsg struct { } //sys WSASocket(af int32, typ int32, protocol int32, protinfo *syscall.WSAProtocolInfo, group uint32, flags uint32) (handle syscall.Handle, err error) [failretval==syscall.InvalidHandle] = ws2_32.WSASocketW +//sys WSADuplicateSocket(s syscall.Handle, processID uint32, info *syscall.WSAProtocolInfo) (err error) [failretval!=0] = ws2_32.WSADuplicateSocketW //sys WSAGetOverlappedResult(h syscall.Handle, o *syscall.Overlapped, bytes *uint32, wait bool, flags *uint32) (err error) = ws2_32.WSAGetOverlappedResult func loadWSASendRecvMsg() error { @@ -452,8 +459,14 @@ type FILE_FULL_DIR_INFO struct { //sys GetVolumeInformationByHandle(file syscall.Handle, volumeNameBuffer *uint16, volumeNameSize uint32, volumeNameSerialNumber *uint32, maximumComponentLength *uint32, fileSystemFlags *uint32, fileSystemNameBuffer *uint16, fileSystemNameSize uint32) (err error) = GetVolumeInformationByHandleW //sys GetVolumeNameForVolumeMountPoint(volumeMountPoint *uint16, volumeName *uint16, bufferlength uint32) (err error) = GetVolumeNameForVolumeMountPointW -//sys RtlLookupFunctionEntry(pc uintptr, baseAddress *uintptr, table *byte) (ret uintptr) = kernel32.RtlLookupFunctionEntry -//sys RtlVirtualUnwind(handlerType uint32, baseAddress uintptr, pc uintptr, entry uintptr, ctxt uintptr, data *uintptr, frame *uintptr, ctxptrs *byte) (ret uintptr) = kernel32.RtlVirtualUnwind +type RUNTIME_FUNCTION struct { + BeginAddress uint32 + EndAddress uint32 + UnwindData uint32 +} + +//sys RtlLookupFunctionEntry(pc uintptr, baseAddress *uintptr, table unsafe.Pointer) (ret *RUNTIME_FUNCTION) = kernel32.RtlLookupFunctionEntry +//sys RtlVirtualUnwind(handlerType uint32, baseAddress uintptr, pc uintptr, entry *RUNTIME_FUNCTION, ctxt unsafe.Pointer, data unsafe.Pointer, frame *uintptr, ctxptrs unsafe.Pointer) (ret uintptr) = kernel32.RtlVirtualUnwind type SERVICE_STATUS struct { ServiceType uint32 @@ -499,3 +512,70 @@ func QueryPerformanceCounter() int64 // Implemented in runtime package. // //go:linkname QueryPerformanceFrequency func QueryPerformanceFrequency() int64 // Implemented in runtime package. + +//sys GetModuleHandle(modulename *uint16) (handle syscall.Handle, err error) = kernel32.GetModuleHandleW + +const ( + PIPE_ACCESS_INBOUND = 0x00000001 + PIPE_ACCESS_OUTBOUND = 0x00000002 + PIPE_ACCESS_DUPLEX = 0x00000003 + + PIPE_TYPE_BYTE = 0x00000000 + PIPE_TYPE_MESSAGE = 0x00000004 + + PIPE_READMODE_BYTE = 0x00000000 + PIPE_READMODE_MESSAGE = 0x00000002 +) + +//sys CreateIoCompletionPort(filehandle syscall.Handle, cphandle syscall.Handle, key uintptr, threadcnt uint32) (handle syscall.Handle, err error) +//sys GetOverlappedResult(handle syscall.Handle, overlapped *syscall.Overlapped, done *uint32, wait bool) (err error) +//sys CreateNamedPipe(name *uint16, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) [failretval==syscall.InvalidHandle] = CreateNamedPipeW + +//sys ReOpenFile(filehandle syscall.Handle, desiredAccess uint32, shareMode uint32, flagAndAttributes uint32) (handle syscall.Handle, err error) + +// NTStatus corresponds with NTSTATUS, error values returned by ntdll.dll and +// other native functions. +type NTStatus uint32 + +func (s NTStatus) Errno() syscall.Errno { + return rtlNtStatusToDosErrorNoTeb(s) +} + +func langID(pri, sub uint16) uint32 { return uint32(sub)<<10 | uint32(pri) } + +func (s NTStatus) Error() string { + return s.Errno().Error() +} + +// x/sys/windows/mkerrors.bash can generate a complete list of NTStatus codes. +// +// At the moment, we only need a couple, so just put them here manually. +// If this list starts getting long, we should consider generating the full set. +const ( + STATUS_OBJECT_NAME_COLLISION NTStatus = 0xC0000035 + STATUS_FILE_IS_A_DIRECTORY NTStatus = 0xC00000BA + STATUS_DIRECTORY_NOT_EMPTY NTStatus = 0xC0000101 + STATUS_NOT_A_DIRECTORY NTStatus = 0xC0000103 + STATUS_CANNOT_DELETE NTStatus = 0xC0000121 + STATUS_REPARSE_POINT_ENCOUNTERED NTStatus = 0xC000050B + STATUS_NOT_SUPPORTED NTStatus = 0xC00000BB + STATUS_INVALID_PARAMETER NTStatus = 0xC000000D + STATUS_INVALID_INFO_CLASS NTStatus = 0xC0000003 +) + +const ( + FileModeInformation = 16 +) + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/ns-ntifs-_file_mode_information +type FILE_MODE_INFORMATION struct { + Mode uint32 +} + +// NT Native APIs +//sys NtCreateFile(handle *syscall.Handle, access uint32, oa *OBJECT_ATTRIBUTES, iosb *IO_STATUS_BLOCK, allocationSize *int64, attributes uint32, share uint32, disposition uint32, options uint32, eabuffer unsafe.Pointer, ealength uint32) (ntstatus error) = ntdll.NtCreateFile +//sys NtOpenFile(handle *syscall.Handle, access uint32, oa *OBJECT_ATTRIBUTES, iosb *IO_STATUS_BLOCK, share uint32, options uint32) (ntstatus error) = ntdll.NtOpenFile +//sys rtlNtStatusToDosErrorNoTeb(ntstatus NTStatus) (ret syscall.Errno) = ntdll.RtlNtStatusToDosErrorNoTeb +//sys NtSetInformationFile(handle syscall.Handle, iosb *IO_STATUS_BLOCK, inBuffer unsafe.Pointer, inBufferLen uint32, class uint32) (ntstatus error) = ntdll.NtSetInformationFile +//sys RtlIsDosDeviceName_U(name *uint16) (ret uint32) = ntdll.RtlIsDosDeviceName_U +//sys NtQueryInformationFile(handle syscall.Handle, iosb *IO_STATUS_BLOCK, inBuffer unsafe.Pointer, inBufferLen uint32, class uint32) (ntstatus error) = ntdll.NtQueryInformationFile diff --git a/src/internal/syscall/windows/types_windows.go b/src/internal/syscall/windows/types_windows.go index 126e07b8834cab..49daf9b31b1ff7 100644 --- a/src/internal/syscall/windows/types_windows.go +++ b/src/internal/syscall/windows/types_windows.go @@ -4,9 +4,285 @@ package windows +import ( + "internal/runtime/syscall/windows" + "syscall" + "unsafe" +) + // Socket related. const ( TCP_KEEPIDLE = 0x03 TCP_KEEPCNT = 0x10 TCP_KEEPINTVL = 0x11 + + SIO_UDP_NETRESET = syscall.IOC_IN | syscall.IOC_VENDOR | 15 +) + +const ( + FILE_READ_DATA = 0x00000001 + FILE_READ_ATTRIBUTES = 0x00000080 + FILE_READ_EA = 0x00000008 + FILE_WRITE_DATA = 0x00000002 + FILE_WRITE_ATTRIBUTES = 0x00000100 + FILE_WRITE_EA = 0x00000010 + FILE_APPEND_DATA = 0x00000004 + FILE_EXECUTE = 0x00000020 + + FILE_GENERIC_READ = STANDARD_RIGHTS_READ | FILE_READ_DATA | FILE_READ_ATTRIBUTES | FILE_READ_EA | SYNCHRONIZE + FILE_GENERIC_WRITE = STANDARD_RIGHTS_WRITE | FILE_WRITE_DATA | FILE_WRITE_ATTRIBUTES | FILE_WRITE_EA | FILE_APPEND_DATA | SYNCHRONIZE + FILE_GENERIC_EXECUTE = STANDARD_RIGHTS_EXECUTE | FILE_READ_ATTRIBUTES | FILE_EXECUTE | SYNCHRONIZE + + FILE_LIST_DIRECTORY = 0x00000001 + FILE_TRAVERSE = 0x00000020 + + FILE_SHARE_READ = 0x00000001 + FILE_SHARE_WRITE = 0x00000002 + FILE_SHARE_DELETE = 0x00000004 + FILE_ATTRIBUTE_READONLY = 0x00000001 + FILE_ATTRIBUTE_HIDDEN = 0x00000002 + FILE_ATTRIBUTE_SYSTEM = 0x00000004 + FILE_ATTRIBUTE_DIRECTORY = 0x00000010 + FILE_ATTRIBUTE_ARCHIVE = 0x00000020 + FILE_ATTRIBUTE_DEVICE = 0x00000040 + FILE_ATTRIBUTE_NORMAL = 0x00000080 + FILE_ATTRIBUTE_TEMPORARY = 0x00000100 + FILE_ATTRIBUTE_SPARSE_FILE = 0x00000200 + FILE_ATTRIBUTE_REPARSE_POINT = 0x00000400 + FILE_ATTRIBUTE_COMPRESSED = 0x00000800 + FILE_ATTRIBUTE_OFFLINE = 0x00001000 + FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 0x00002000 + FILE_ATTRIBUTE_ENCRYPTED = 0x00004000 + FILE_ATTRIBUTE_INTEGRITY_STREAM = 0x00008000 + FILE_ATTRIBUTE_VIRTUAL = 0x00010000 + FILE_ATTRIBUTE_NO_SCRUB_DATA = 0x00020000 + FILE_ATTRIBUTE_RECALL_ON_OPEN = 0x00040000 + FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 0x00400000 + + INVALID_FILE_ATTRIBUTES = 0xffffffff +) + +// https://learn.microsoft.com/en-us/windows/win32/secauthz/access-mask +type ACCESS_MASK uint32 + +// Constants for type ACCESS_MASK +const ( + DELETE = 0x00010000 + READ_CONTROL = 0x00020000 + WRITE_DAC = 0x00040000 + WRITE_OWNER = 0x00080000 + SYNCHRONIZE = 0x00100000 + STANDARD_RIGHTS_REQUIRED = 0x000F0000 + STANDARD_RIGHTS_READ = READ_CONTROL + STANDARD_RIGHTS_WRITE = READ_CONTROL + STANDARD_RIGHTS_EXECUTE = READ_CONTROL + STANDARD_RIGHTS_ALL = 0x001F0000 + SPECIFIC_RIGHTS_ALL = 0x0000FFFF + ACCESS_SYSTEM_SECURITY = 0x01000000 + MAXIMUM_ALLOWED = 0x02000000 + GENERIC_READ = 0x80000000 + GENERIC_WRITE = 0x40000000 + GENERIC_EXECUTE = 0x20000000 + GENERIC_ALL = 0x10000000 +) + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_acl +type ACL struct { + AclRevision byte + Sbz1 byte + AclSize uint16 + AceCount uint16 + Sbz2 uint16 +} + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_io_status_block +type IO_STATUS_BLOCK struct { + Status NTStatus + Information uintptr +} + +// https://learn.microsoft.com/en-us/windows/win32/api/ntdef/ns-ntdef-_object_attributes +type OBJECT_ATTRIBUTES struct { + Length uint32 + RootDirectory syscall.Handle + ObjectName *NTUnicodeString + Attributes uint32 + SecurityDescriptor *SECURITY_DESCRIPTOR + SecurityQoS *SECURITY_QUALITY_OF_SERVICE +} + +// init sets o's RootDirectory, ObjectName, and Length. +func (o *OBJECT_ATTRIBUTES) init(root syscall.Handle, name string) error { + if name == "." { + name = "" + } + objectName, err := NewNTUnicodeString(name) + if err != nil { + return err + } + o.ObjectName = objectName + if root != syscall.InvalidHandle { + o.RootDirectory = root + } + o.Length = uint32(unsafe.Sizeof(*o)) + return nil +} + +// Values for the Attributes member of OBJECT_ATTRIBUTES. +const ( + OBJ_INHERIT = 0x00000002 + OBJ_PERMANENT = 0x00000010 + OBJ_EXCLUSIVE = 0x00000020 + OBJ_CASE_INSENSITIVE = 0x00000040 + OBJ_OPENIF = 0x00000080 + OBJ_OPENLINK = 0x00000100 + OBJ_KERNEL_HANDLE = 0x00000200 + OBJ_FORCE_ACCESS_CHECK = 0x00000400 + OBJ_IGNORE_IMPERSONATED_DEVICEMAP = 0x00000800 + OBJ_DONT_REPARSE = 0x00001000 + OBJ_VALID_ATTRIBUTES = 0x00001FF2 +) + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/ns-ntifs-_security_descriptor +type SECURITY_DESCRIPTOR struct { + revision byte + sbz1 byte + control SECURITY_DESCRIPTOR_CONTROL + owner *syscall.SID + group *syscall.SID + sacl *ACL + dacl *ACL +} + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ifs/security-descriptor-control +type SECURITY_DESCRIPTOR_CONTROL uint16 + +// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-security_quality_of_service +type SECURITY_QUALITY_OF_SERVICE struct { + Length uint32 + ImpersonationLevel uint32 // type SECURITY_IMPERSONATION_LEVEL + ContextTrackingMode byte // type SECURITY_CONTEXT_TRACKING_MODE + EffectiveOnly byte +} + +// File flags for [os.OpenFile]. The O_ prefix is used to indicate +// that these flags are specific to the OpenFile function. +const ( + O_FILE_FLAG_OPEN_NO_RECALL = 0x00100000 + O_FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000 + O_FILE_FLAG_SESSION_AWARE = 0x00800000 + O_FILE_FLAG_POSIX_SEMANTICS = 0x01000000 + O_FILE_FLAG_BACKUP_SEMANTICS = 0x02000000 + O_FILE_FLAG_DELETE_ON_CLOSE = 0x04000000 + O_FILE_FLAG_SEQUENTIAL_SCAN = 0x08000000 + O_FILE_FLAG_RANDOM_ACCESS = 0x10000000 + O_FILE_FLAG_NO_BUFFERING = 0x20000000 + O_FILE_FLAG_OVERLAPPED = 0x40000000 + O_FILE_FLAG_WRITE_THROUGH = 0x80000000 +) + +const ( + // CreateDisposition flags for NtCreateFile and NtCreateNamedPipeFile. + FILE_SUPERSEDE = 0x00000000 + FILE_OPEN = 0x00000001 + FILE_CREATE = 0x00000002 + FILE_OPEN_IF = 0x00000003 + FILE_OVERWRITE = 0x00000004 + FILE_OVERWRITE_IF = 0x00000005 + FILE_MAXIMUM_DISPOSITION = 0x00000005 + + // CreateOptions flags for NtCreateFile and NtCreateNamedPipeFile. + FILE_DIRECTORY_FILE = 0x00000001 + FILE_WRITE_THROUGH = 0x00000002 + FILE_SEQUENTIAL_ONLY = 0x00000004 + FILE_NO_INTERMEDIATE_BUFFERING = 0x00000008 + FILE_SYNCHRONOUS_IO_ALERT = 0x00000010 + FILE_SYNCHRONOUS_IO_NONALERT = 0x00000020 + FILE_NON_DIRECTORY_FILE = 0x00000040 + FILE_CREATE_TREE_CONNECTION = 0x00000080 + FILE_COMPLETE_IF_OPLOCKED = 0x00000100 + FILE_NO_EA_KNOWLEDGE = 0x00000200 + FILE_OPEN_REMOTE_INSTANCE = 0x00000400 + FILE_RANDOM_ACCESS = 0x00000800 + FILE_DELETE_ON_CLOSE = 0x00001000 + FILE_OPEN_BY_FILE_ID = 0x00002000 + FILE_OPEN_FOR_BACKUP_INTENT = 0x00004000 + FILE_NO_COMPRESSION = 0x00008000 + FILE_OPEN_REQUIRING_OPLOCK = 0x00010000 + FILE_DISALLOW_EXCLUSIVE = 0x00020000 + FILE_RESERVE_OPFILTER = 0x00100000 + FILE_OPEN_REPARSE_POINT = 0x00200000 + FILE_OPEN_NO_RECALL = 0x00400000 + FILE_OPEN_FOR_FREE_SPACE_QUERY = 0x00800000 +) + +// https://learn.microsoft.com/en-us/windows/win32/api/winbase/ns-winbase-file_disposition_info +type FILE_DISPOSITION_INFO struct { + DeleteFile bool +} + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntddk/ns-ntddk-_file_disposition_information +type FILE_DISPOSITION_INFORMATION struct { + DeleteFile bool +} + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntddk/ns-ntddk-_file_disposition_information_ex +type FILE_DISPOSITION_INFORMATION_EX struct { + Flags uint32 +} + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntddk/ns-ntddk-_file_disposition_information_ex +const ( + FILE_DISPOSITION_DO_NOT_DELETE = 0x00000000 + FILE_DISPOSITION_DELETE = 0x00000001 + FILE_DISPOSITION_POSIX_SEMANTICS = 0x00000002 + FILE_DISPOSITION_FORCE_IMAGE_SECTION_CHECK = 0x00000004 + FILE_DISPOSITION_ON_CLOSE = 0x00000008 + FILE_DISPOSITION_IGNORE_READONLY_ATTRIBUTE = 0x00000010 +) + +// Flags for FILE_RENAME_INFORMATION_EX. +const ( + FILE_RENAME_REPLACE_IF_EXISTS = 0x00000001 + FILE_RENAME_POSIX_SEMANTICS = 0x00000002 ) + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/ns-ntifs-_file_rename_information +type FILE_RENAME_INFORMATION struct { + ReplaceIfExists bool + RootDirectory syscall.Handle + FileNameLength uint32 + FileName [syscall.MAX_PATH]uint16 +} + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/ns-ntifs-_file_rename_information +type FILE_RENAME_INFORMATION_EX struct { + Flags uint32 + RootDirectory syscall.Handle + FileNameLength uint32 + FileName [syscall.MAX_PATH]uint16 +} + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/ns-ntifs-_file_link_information +type FILE_LINK_INFORMATION struct { + ReplaceIfExists bool + RootDirectory syscall.Handle + FileNameLength uint32 + FileName [syscall.MAX_PATH]uint16 +} + +const FileReplaceCompletionInformation = 61 + +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/ns-ntifs-_file_completion_information +type FILE_COMPLETION_INFORMATION struct { + Port syscall.Handle + Key uintptr +} + +// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-osversioninfoexa +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_osversioninfoexw +const VER_NT_WORKSTATION = 0x0000001 + +type MemoryBasicInformation = windows.MemoryBasicInformation + +type Context = windows.Context diff --git a/src/internal/syscall/windows/version_windows.go b/src/internal/syscall/windows/version_windows.go index ff21fc59e5bf53..5edf7a01e270d4 100644 --- a/src/internal/syscall/windows/version_windows.go +++ b/src/internal/syscall/windows/version_windows.go @@ -11,28 +11,53 @@ import ( "unsafe" ) -// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_osversioninfow -type _OSVERSIONINFOW struct { +// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_osversioninfoexw +type _OSVERSIONINFOEXW struct { osVersionInfoSize uint32 majorVersion uint32 minorVersion uint32 buildNumber uint32 platformId uint32 csdVersion [128]uint16 + servicePackMajor uint16 + servicePackMinor uint16 + suiteMask uint16 + productType byte + reserved byte } // According to documentation, RtlGetVersion function always succeeds. -//sys rtlGetVersion(info *_OSVERSIONINFOW) = ntdll.RtlGetVersion +//sys rtlGetVersion(info *_OSVERSIONINFOEXW) = ntdll.RtlGetVersion -// version retrieves the major, minor, and build version numbers -// of the current Windows OS from the RtlGetVersion API. -func version() (major, minor, build uint32) { - info := _OSVERSIONINFOW{} +// Retrieves version information of the current Windows OS +// from the RtlGetVersion API. +func getVersionInfo() *_OSVERSIONINFOEXW { + info := _OSVERSIONINFOEXW{} info.osVersionInfoSize = uint32(unsafe.Sizeof(info)) rtlGetVersion(&info) + return &info +} + +// Version retrieves the major, minor, and build version numbers +// of the current Windows OS from the RtlGetVersion API. +func Version() (major, minor, build uint32) { + info := getVersionInfo() return info.majorVersion, info.minorVersion, info.buildNumber } +// SupportUnlimitedTransmitFile indicates whether the current +// Windows version's TransmitFile function imposes any +// concurrent operation limits. +// Workstation and client versions of Windows limit the number +// of concurrent TransmitFile operations allowed on the system +// to a maximum of two. Please see: +// https://learn.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-transmitfile +// https://golang.org/issue/73746 +var SupportUnlimitedTransmitFile = sync.OnceValue(func() bool { + info := getVersionInfo() + return info.productType != VER_NT_WORKSTATION +}) + var ( supportTCPKeepAliveIdle bool supportTCPKeepAliveInterval bool @@ -43,7 +68,7 @@ var initTCPKeepAlive = sync.OnceFunc(func() { s, err := WSASocket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP, nil, 0, WSA_FLAG_NO_HANDLE_INHERIT) if err != nil { // Fallback to checking the Windows version. - major, _, build := version() + major, _, build := Version() supportTCPKeepAliveIdle = major >= 10 && build >= 16299 supportTCPKeepAliveInterval = major >= 10 && build >= 16299 supportTCPKeepAliveCount = major >= 10 && build >= 15063 @@ -59,7 +84,7 @@ var initTCPKeepAlive = sync.OnceFunc(func() { supportTCPKeepAliveCount = optSupported(TCP_KEEPCNT) }) -// SupportTCPKeepAliveInterval indicates whether TCP_KEEPIDLE is supported. +// SupportTCPKeepAliveIdle indicates whether TCP_KEEPIDLE is supported. // The minimal requirement is Windows 10.0.16299. func SupportTCPKeepAliveIdle() bool { initTCPKeepAlive() @@ -85,7 +110,7 @@ func SupportTCPKeepAliveCount() bool { // Windows version supports the TCP_INITIAL_RTO_NO_SYN_RETRANSMISSIONS. // The minimal requirement is Windows 10.0.16299. var SupportTCPInitialRTONoSYNRetransmissions = sync.OnceValue(func() bool { - major, _, build := version() + major, _, build := Version() return major >= 10 && build >= 16299 }) diff --git a/src/internal/syscall/windows/zsyscall_windows.go b/src/internal/syscall/windows/zsyscall_windows.go index 414ad2647d1abd..64bb5041073946 100644 --- a/src/internal/syscall/windows/zsyscall_windows.go +++ b/src/internal/syscall/windows/zsyscall_windows.go @@ -49,7 +49,13 @@ var ( procAdjustTokenPrivileges = modadvapi32.NewProc("AdjustTokenPrivileges") procDuplicateTokenEx = modadvapi32.NewProc("DuplicateTokenEx") + procGetSidIdentifierAuthority = modadvapi32.NewProc("GetSidIdentifierAuthority") + procGetSidSubAuthority = modadvapi32.NewProc("GetSidSubAuthority") + procGetSidSubAuthorityCount = modadvapi32.NewProc("GetSidSubAuthorityCount") + procImpersonateLoggedOnUser = modadvapi32.NewProc("ImpersonateLoggedOnUser") procImpersonateSelf = modadvapi32.NewProc("ImpersonateSelf") + procIsValidSid = modadvapi32.NewProc("IsValidSid") + procLogonUserW = modadvapi32.NewProc("LogonUserW") procLookupPrivilegeValueW = modadvapi32.NewProc("LookupPrivilegeValueW") procOpenSCManagerW = modadvapi32.NewProc("OpenSCManagerW") procOpenServiceW = modadvapi32.NewProc("OpenServiceW") @@ -60,13 +66,18 @@ var ( procProcessPrng = modbcryptprimitives.NewProc("ProcessPrng") procGetAdaptersAddresses = modiphlpapi.NewProc("GetAdaptersAddresses") procCreateEventW = modkernel32.NewProc("CreateEventW") + procCreateIoCompletionPort = modkernel32.NewProc("CreateIoCompletionPort") + procCreateNamedPipeW = modkernel32.NewProc("CreateNamedPipeW") procGetACP = modkernel32.NewProc("GetACP") procGetComputerNameExW = modkernel32.NewProc("GetComputerNameExW") procGetConsoleCP = modkernel32.NewProc("GetConsoleCP") procGetCurrentThread = modkernel32.NewProc("GetCurrentThread") procGetFileInformationByHandleEx = modkernel32.NewProc("GetFileInformationByHandleEx") + procGetFileSizeEx = modkernel32.NewProc("GetFileSizeEx") procGetFinalPathNameByHandleW = modkernel32.NewProc("GetFinalPathNameByHandleW") procGetModuleFileNameW = modkernel32.NewProc("GetModuleFileNameW") + procGetModuleHandleW = modkernel32.NewProc("GetModuleHandleW") + procGetOverlappedResult = modkernel32.NewProc("GetOverlappedResult") procGetTempPath2W = modkernel32.NewProc("GetTempPath2W") procGetVolumeInformationByHandleW = modkernel32.NewProc("GetVolumeInformationByHandleW") procGetVolumeNameForVolumeMountPointW = modkernel32.NewProc("GetVolumeNameForVolumeMountPointW") @@ -75,6 +86,7 @@ var ( procModule32NextW = modkernel32.NewProc("Module32NextW") procMoveFileExW = modkernel32.NewProc("MoveFileExW") procMultiByteToWideChar = modkernel32.NewProc("MultiByteToWideChar") + procReOpenFile = modkernel32.NewProc("ReOpenFile") procRtlLookupFunctionEntry = modkernel32.NewProc("RtlLookupFunctionEntry") procRtlVirtualUnwind = modkernel32.NewProc("RtlVirtualUnwind") procSetFileInformationByHandle = modkernel32.NewProc("SetFileInformationByHandle") @@ -82,12 +94,21 @@ var ( procVirtualQuery = modkernel32.NewProc("VirtualQuery") procNetShareAdd = modnetapi32.NewProc("NetShareAdd") procNetShareDel = modnetapi32.NewProc("NetShareDel") + procNetUserAdd = modnetapi32.NewProc("NetUserAdd") + procNetUserDel = modnetapi32.NewProc("NetUserDel") procNetUserGetLocalGroups = modnetapi32.NewProc("NetUserGetLocalGroups") + procNtCreateFile = modntdll.NewProc("NtCreateFile") + procNtOpenFile = modntdll.NewProc("NtOpenFile") + procNtQueryInformationFile = modntdll.NewProc("NtQueryInformationFile") + procNtSetInformationFile = modntdll.NewProc("NtSetInformationFile") procRtlGetVersion = modntdll.NewProc("RtlGetVersion") + procRtlIsDosDeviceName_U = modntdll.NewProc("RtlIsDosDeviceName_U") + procRtlNtStatusToDosErrorNoTeb = modntdll.NewProc("RtlNtStatusToDosErrorNoTeb") procGetProcessMemoryInfo = modpsapi.NewProc("GetProcessMemoryInfo") procCreateEnvironmentBlock = moduserenv.NewProc("CreateEnvironmentBlock") procDestroyEnvironmentBlock = moduserenv.NewProc("DestroyEnvironmentBlock") procGetProfilesDirectoryW = moduserenv.NewProc("GetProfilesDirectoryW") + procWSADuplicateSocketW = modws2_32.NewProc("WSADuplicateSocketW") procWSAGetOverlappedResult = modws2_32.NewProc("WSAGetOverlappedResult") procWSASocketW = modws2_32.NewProc("WSASocketW") ) @@ -97,7 +118,7 @@ func adjustTokenPrivileges(token syscall.Token, disableAllPrivileges bool, newst if disableAllPrivileges { _p0 = 1 } - r0, _, e1 := syscall.Syscall6(procAdjustTokenPrivileges.Addr(), 6, uintptr(token), uintptr(_p0), uintptr(unsafe.Pointer(newstate)), uintptr(buflen), uintptr(unsafe.Pointer(prevstate)), uintptr(unsafe.Pointer(returnlen))) + r0, _, e1 := syscall.SyscallN(procAdjustTokenPrivileges.Addr(), uintptr(token), uintptr(_p0), uintptr(unsafe.Pointer(newstate)), uintptr(buflen), uintptr(unsafe.Pointer(prevstate)), uintptr(unsafe.Pointer(returnlen))) ret = uint32(r0) if true { err = errnoErr(e1) @@ -106,7 +127,33 @@ func adjustTokenPrivileges(token syscall.Token, disableAllPrivileges bool, newst } func DuplicateTokenEx(hExistingToken syscall.Token, dwDesiredAccess uint32, lpTokenAttributes *syscall.SecurityAttributes, impersonationLevel uint32, tokenType TokenType, phNewToken *syscall.Token) (err error) { - r1, _, e1 := syscall.Syscall6(procDuplicateTokenEx.Addr(), 6, uintptr(hExistingToken), uintptr(dwDesiredAccess), uintptr(unsafe.Pointer(lpTokenAttributes)), uintptr(impersonationLevel), uintptr(tokenType), uintptr(unsafe.Pointer(phNewToken))) + r1, _, e1 := syscall.SyscallN(procDuplicateTokenEx.Addr(), uintptr(hExistingToken), uintptr(dwDesiredAccess), uintptr(unsafe.Pointer(lpTokenAttributes)), uintptr(impersonationLevel), uintptr(tokenType), uintptr(unsafe.Pointer(phNewToken))) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func getSidIdentifierAuthority(sid *syscall.SID) (idauth uintptr) { + r0, _, _ := syscall.SyscallN(procGetSidIdentifierAuthority.Addr(), uintptr(unsafe.Pointer(sid))) + idauth = uintptr(r0) + return +} + +func getSidSubAuthority(sid *syscall.SID, subAuthorityIdx uint32) (subAuth uintptr) { + r0, _, _ := syscall.SyscallN(procGetSidSubAuthority.Addr(), uintptr(unsafe.Pointer(sid)), uintptr(subAuthorityIdx)) + subAuth = uintptr(r0) + return +} + +func getSidSubAuthorityCount(sid *syscall.SID) (count uintptr) { + r0, _, _ := syscall.SyscallN(procGetSidSubAuthorityCount.Addr(), uintptr(unsafe.Pointer(sid))) + count = uintptr(r0) + return +} + +func ImpersonateLoggedOnUser(token syscall.Token) (err error) { + r1, _, e1 := syscall.SyscallN(procImpersonateLoggedOnUser.Addr(), uintptr(token)) if r1 == 0 { err = errnoErr(e1) } @@ -114,7 +161,21 @@ func DuplicateTokenEx(hExistingToken syscall.Token, dwDesiredAccess uint32, lpTo } func ImpersonateSelf(impersonationlevel uint32) (err error) { - r1, _, e1 := syscall.Syscall(procImpersonateSelf.Addr(), 1, uintptr(impersonationlevel), 0, 0) + r1, _, e1 := syscall.SyscallN(procImpersonateSelf.Addr(), uintptr(impersonationlevel)) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func IsValidSid(sid *syscall.SID) (valid bool) { + r0, _, _ := syscall.SyscallN(procIsValidSid.Addr(), uintptr(unsafe.Pointer(sid))) + valid = r0 != 0 + return +} + +func LogonUser(username *uint16, domain *uint16, password *uint16, logonType uint32, logonProvider uint32, token *syscall.Token) (err error) { + r1, _, e1 := syscall.SyscallN(procLogonUserW.Addr(), uintptr(unsafe.Pointer(username)), uintptr(unsafe.Pointer(domain)), uintptr(unsafe.Pointer(password)), uintptr(logonType), uintptr(logonProvider), uintptr(unsafe.Pointer(token))) if r1 == 0 { err = errnoErr(e1) } @@ -122,7 +183,7 @@ func ImpersonateSelf(impersonationlevel uint32) (err error) { } func LookupPrivilegeValue(systemname *uint16, name *uint16, luid *LUID) (err error) { - r1, _, e1 := syscall.Syscall(procLookupPrivilegeValueW.Addr(), 3, uintptr(unsafe.Pointer(systemname)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(luid))) + r1, _, e1 := syscall.SyscallN(procLookupPrivilegeValueW.Addr(), uintptr(unsafe.Pointer(systemname)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(luid))) if r1 == 0 { err = errnoErr(e1) } @@ -130,7 +191,7 @@ func LookupPrivilegeValue(systemname *uint16, name *uint16, luid *LUID) (err err } func OpenSCManager(machineName *uint16, databaseName *uint16, access uint32) (handle syscall.Handle, err error) { - r0, _, e1 := syscall.Syscall(procOpenSCManagerW.Addr(), 3, uintptr(unsafe.Pointer(machineName)), uintptr(unsafe.Pointer(databaseName)), uintptr(access)) + r0, _, e1 := syscall.SyscallN(procOpenSCManagerW.Addr(), uintptr(unsafe.Pointer(machineName)), uintptr(unsafe.Pointer(databaseName)), uintptr(access)) handle = syscall.Handle(r0) if handle == 0 { err = errnoErr(e1) @@ -139,7 +200,7 @@ func OpenSCManager(machineName *uint16, databaseName *uint16, access uint32) (ha } func OpenService(mgr syscall.Handle, serviceName *uint16, access uint32) (handle syscall.Handle, err error) { - r0, _, e1 := syscall.Syscall(procOpenServiceW.Addr(), 3, uintptr(mgr), uintptr(unsafe.Pointer(serviceName)), uintptr(access)) + r0, _, e1 := syscall.SyscallN(procOpenServiceW.Addr(), uintptr(mgr), uintptr(unsafe.Pointer(serviceName)), uintptr(access)) handle = syscall.Handle(r0) if handle == 0 { err = errnoErr(e1) @@ -152,7 +213,7 @@ func OpenThreadToken(h syscall.Handle, access uint32, openasself bool, token *sy if openasself { _p0 = 1 } - r1, _, e1 := syscall.Syscall6(procOpenThreadToken.Addr(), 4, uintptr(h), uintptr(access), uintptr(_p0), uintptr(unsafe.Pointer(token)), 0, 0) + r1, _, e1 := syscall.SyscallN(procOpenThreadToken.Addr(), uintptr(h), uintptr(access), uintptr(_p0), uintptr(unsafe.Pointer(token))) if r1 == 0 { err = errnoErr(e1) } @@ -160,7 +221,7 @@ func OpenThreadToken(h syscall.Handle, access uint32, openasself bool, token *sy } func QueryServiceStatus(hService syscall.Handle, lpServiceStatus *SERVICE_STATUS) (err error) { - r1, _, e1 := syscall.Syscall(procQueryServiceStatus.Addr(), 2, uintptr(hService), uintptr(unsafe.Pointer(lpServiceStatus)), 0) + r1, _, e1 := syscall.SyscallN(procQueryServiceStatus.Addr(), uintptr(hService), uintptr(unsafe.Pointer(lpServiceStatus))) if r1 == 0 { err = errnoErr(e1) } @@ -168,15 +229,15 @@ func QueryServiceStatus(hService syscall.Handle, lpServiceStatus *SERVICE_STATUS } func RevertToSelf() (err error) { - r1, _, e1 := syscall.Syscall(procRevertToSelf.Addr(), 0, 0, 0, 0) + r1, _, e1 := syscall.SyscallN(procRevertToSelf.Addr()) if r1 == 0 { err = errnoErr(e1) } return } -func SetTokenInformation(tokenHandle syscall.Token, tokenInformationClass uint32, tokenInformation uintptr, tokenInformationLength uint32) (err error) { - r1, _, e1 := syscall.Syscall6(procSetTokenInformation.Addr(), 4, uintptr(tokenHandle), uintptr(tokenInformationClass), uintptr(tokenInformation), uintptr(tokenInformationLength), 0, 0) +func SetTokenInformation(tokenHandle syscall.Token, tokenInformationClass uint32, tokenInformation unsafe.Pointer, tokenInformationLength uint32) (err error) { + r1, _, e1 := syscall.SyscallN(procSetTokenInformation.Addr(), uintptr(tokenHandle), uintptr(tokenInformationClass), uintptr(tokenInformation), uintptr(tokenInformationLength)) if r1 == 0 { err = errnoErr(e1) } @@ -188,15 +249,15 @@ func ProcessPrng(buf []byte) (err error) { if len(buf) > 0 { _p0 = &buf[0] } - r1, _, e1 := syscall.Syscall(procProcessPrng.Addr(), 2, uintptr(unsafe.Pointer(_p0)), uintptr(len(buf)), 0) + r1, _, e1 := syscall.SyscallN(procProcessPrng.Addr(), uintptr(unsafe.Pointer(_p0)), uintptr(len(buf))) if r1 == 0 { err = errnoErr(e1) } return } -func GetAdaptersAddresses(family uint32, flags uint32, reserved uintptr, adapterAddresses *IpAdapterAddresses, sizePointer *uint32) (errcode error) { - r0, _, _ := syscall.Syscall6(procGetAdaptersAddresses.Addr(), 5, uintptr(family), uintptr(flags), uintptr(reserved), uintptr(unsafe.Pointer(adapterAddresses)), uintptr(unsafe.Pointer(sizePointer)), 0) +func GetAdaptersAddresses(family uint32, flags uint32, reserved unsafe.Pointer, adapterAddresses *IpAdapterAddresses, sizePointer *uint32) (errcode error) { + r0, _, _ := syscall.SyscallN(procGetAdaptersAddresses.Addr(), uintptr(family), uintptr(flags), uintptr(reserved), uintptr(unsafe.Pointer(adapterAddresses)), uintptr(unsafe.Pointer(sizePointer))) if r0 != 0 { errcode = syscall.Errno(r0) } @@ -204,7 +265,7 @@ func GetAdaptersAddresses(family uint32, flags uint32, reserved uintptr, adapter } func CreateEvent(eventAttrs *SecurityAttributes, manualReset uint32, initialState uint32, name *uint16) (handle syscall.Handle, err error) { - r0, _, e1 := syscall.Syscall6(procCreateEventW.Addr(), 4, uintptr(unsafe.Pointer(eventAttrs)), uintptr(manualReset), uintptr(initialState), uintptr(unsafe.Pointer(name)), 0, 0) + r0, _, e1 := syscall.SyscallN(procCreateEventW.Addr(), uintptr(unsafe.Pointer(eventAttrs)), uintptr(manualReset), uintptr(initialState), uintptr(unsafe.Pointer(name))) handle = syscall.Handle(r0) if handle == 0 { err = errnoErr(e1) @@ -212,14 +273,32 @@ func CreateEvent(eventAttrs *SecurityAttributes, manualReset uint32, initialStat return } +func CreateIoCompletionPort(filehandle syscall.Handle, cphandle syscall.Handle, key uintptr, threadcnt uint32) (handle syscall.Handle, err error) { + r0, _, e1 := syscall.SyscallN(procCreateIoCompletionPort.Addr(), uintptr(filehandle), uintptr(cphandle), uintptr(key), uintptr(threadcnt)) + handle = syscall.Handle(r0) + if handle == 0 { + err = errnoErr(e1) + } + return +} + +func CreateNamedPipe(name *uint16, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) { + r0, _, e1 := syscall.SyscallN(procCreateNamedPipeW.Addr(), uintptr(unsafe.Pointer(name)), uintptr(flags), uintptr(pipeMode), uintptr(maxInstances), uintptr(outSize), uintptr(inSize), uintptr(defaultTimeout), uintptr(unsafe.Pointer(sa))) + handle = syscall.Handle(r0) + if handle == syscall.InvalidHandle { + err = errnoErr(e1) + } + return +} + func GetACP() (acp uint32) { - r0, _, _ := syscall.Syscall(procGetACP.Addr(), 0, 0, 0, 0) + r0, _, _ := syscall.SyscallN(procGetACP.Addr()) acp = uint32(r0) return } func GetComputerNameEx(nameformat uint32, buf *uint16, n *uint32) (err error) { - r1, _, e1 := syscall.Syscall(procGetComputerNameExW.Addr(), 3, uintptr(nameformat), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(n))) + r1, _, e1 := syscall.SyscallN(procGetComputerNameExW.Addr(), uintptr(nameformat), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(n))) if r1 == 0 { err = errnoErr(e1) } @@ -227,13 +306,13 @@ func GetComputerNameEx(nameformat uint32, buf *uint16, n *uint32) (err error) { } func GetConsoleCP() (ccp uint32) { - r0, _, _ := syscall.Syscall(procGetConsoleCP.Addr(), 0, 0, 0, 0) + r0, _, _ := syscall.SyscallN(procGetConsoleCP.Addr()) ccp = uint32(r0) return } func GetCurrentThread() (pseudoHandle syscall.Handle, err error) { - r0, _, e1 := syscall.Syscall(procGetCurrentThread.Addr(), 0, 0, 0, 0) + r0, _, e1 := syscall.SyscallN(procGetCurrentThread.Addr()) pseudoHandle = syscall.Handle(r0) if pseudoHandle == 0 { err = errnoErr(e1) @@ -242,7 +321,15 @@ func GetCurrentThread() (pseudoHandle syscall.Handle, err error) { } func GetFileInformationByHandleEx(handle syscall.Handle, class uint32, info *byte, bufsize uint32) (err error) { - r1, _, e1 := syscall.Syscall6(procGetFileInformationByHandleEx.Addr(), 4, uintptr(handle), uintptr(class), uintptr(unsafe.Pointer(info)), uintptr(bufsize), 0, 0) + r1, _, e1 := syscall.SyscallN(procGetFileInformationByHandleEx.Addr(), uintptr(handle), uintptr(class), uintptr(unsafe.Pointer(info)), uintptr(bufsize)) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func GetFileSizeEx(handle syscall.Handle, size *int64) (err error) { + r1, _, e1 := syscall.SyscallN(procGetFileSizeEx.Addr(), uintptr(handle), uintptr(unsafe.Pointer(size))) if r1 == 0 { err = errnoErr(e1) } @@ -250,7 +337,7 @@ func GetFileInformationByHandleEx(handle syscall.Handle, class uint32, info *byt } func GetFinalPathNameByHandle(file syscall.Handle, filePath *uint16, filePathSize uint32, flags uint32) (n uint32, err error) { - r0, _, e1 := syscall.Syscall6(procGetFinalPathNameByHandleW.Addr(), 4, uintptr(file), uintptr(unsafe.Pointer(filePath)), uintptr(filePathSize), uintptr(flags), 0, 0) + r0, _, e1 := syscall.SyscallN(procGetFinalPathNameByHandleW.Addr(), uintptr(file), uintptr(unsafe.Pointer(filePath)), uintptr(filePathSize), uintptr(flags)) n = uint32(r0) if n == 0 { err = errnoErr(e1) @@ -259,7 +346,7 @@ func GetFinalPathNameByHandle(file syscall.Handle, filePath *uint16, filePathSiz } func GetModuleFileName(module syscall.Handle, fn *uint16, len uint32) (n uint32, err error) { - r0, _, e1 := syscall.Syscall(procGetModuleFileNameW.Addr(), 3, uintptr(module), uintptr(unsafe.Pointer(fn)), uintptr(len)) + r0, _, e1 := syscall.SyscallN(procGetModuleFileNameW.Addr(), uintptr(module), uintptr(unsafe.Pointer(fn)), uintptr(len)) n = uint32(r0) if n == 0 { err = errnoErr(e1) @@ -267,8 +354,29 @@ func GetModuleFileName(module syscall.Handle, fn *uint16, len uint32) (n uint32, return } +func GetModuleHandle(modulename *uint16) (handle syscall.Handle, err error) { + r0, _, e1 := syscall.SyscallN(procGetModuleHandleW.Addr(), uintptr(unsafe.Pointer(modulename))) + handle = syscall.Handle(r0) + if handle == 0 { + err = errnoErr(e1) + } + return +} + +func GetOverlappedResult(handle syscall.Handle, overlapped *syscall.Overlapped, done *uint32, wait bool) (err error) { + var _p0 uint32 + if wait { + _p0 = 1 + } + r1, _, e1 := syscall.SyscallN(procGetOverlappedResult.Addr(), uintptr(handle), uintptr(unsafe.Pointer(overlapped)), uintptr(unsafe.Pointer(done)), uintptr(_p0)) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + func GetTempPath2(buflen uint32, buf *uint16) (n uint32, err error) { - r0, _, e1 := syscall.Syscall(procGetTempPath2W.Addr(), 2, uintptr(buflen), uintptr(unsafe.Pointer(buf)), 0) + r0, _, e1 := syscall.SyscallN(procGetTempPath2W.Addr(), uintptr(buflen), uintptr(unsafe.Pointer(buf))) n = uint32(r0) if n == 0 { err = errnoErr(e1) @@ -277,7 +385,7 @@ func GetTempPath2(buflen uint32, buf *uint16) (n uint32, err error) { } func GetVolumeInformationByHandle(file syscall.Handle, volumeNameBuffer *uint16, volumeNameSize uint32, volumeNameSerialNumber *uint32, maximumComponentLength *uint32, fileSystemFlags *uint32, fileSystemNameBuffer *uint16, fileSystemNameSize uint32) (err error) { - r1, _, e1 := syscall.Syscall9(procGetVolumeInformationByHandleW.Addr(), 8, uintptr(file), uintptr(unsafe.Pointer(volumeNameBuffer)), uintptr(volumeNameSize), uintptr(unsafe.Pointer(volumeNameSerialNumber)), uintptr(unsafe.Pointer(maximumComponentLength)), uintptr(unsafe.Pointer(fileSystemFlags)), uintptr(unsafe.Pointer(fileSystemNameBuffer)), uintptr(fileSystemNameSize), 0) + r1, _, e1 := syscall.SyscallN(procGetVolumeInformationByHandleW.Addr(), uintptr(file), uintptr(unsafe.Pointer(volumeNameBuffer)), uintptr(volumeNameSize), uintptr(unsafe.Pointer(volumeNameSerialNumber)), uintptr(unsafe.Pointer(maximumComponentLength)), uintptr(unsafe.Pointer(fileSystemFlags)), uintptr(unsafe.Pointer(fileSystemNameBuffer)), uintptr(fileSystemNameSize)) if r1 == 0 { err = errnoErr(e1) } @@ -285,7 +393,7 @@ func GetVolumeInformationByHandle(file syscall.Handle, volumeNameBuffer *uint16, } func GetVolumeNameForVolumeMountPoint(volumeMountPoint *uint16, volumeName *uint16, bufferlength uint32) (err error) { - r1, _, e1 := syscall.Syscall(procGetVolumeNameForVolumeMountPointW.Addr(), 3, uintptr(unsafe.Pointer(volumeMountPoint)), uintptr(unsafe.Pointer(volumeName)), uintptr(bufferlength)) + r1, _, e1 := syscall.SyscallN(procGetVolumeNameForVolumeMountPointW.Addr(), uintptr(unsafe.Pointer(volumeMountPoint)), uintptr(unsafe.Pointer(volumeName)), uintptr(bufferlength)) if r1 == 0 { err = errnoErr(e1) } @@ -293,7 +401,7 @@ func GetVolumeNameForVolumeMountPoint(volumeMountPoint *uint16, volumeName *uint } func LockFileEx(file syscall.Handle, flags uint32, reserved uint32, bytesLow uint32, bytesHigh uint32, overlapped *syscall.Overlapped) (err error) { - r1, _, e1 := syscall.Syscall6(procLockFileEx.Addr(), 6, uintptr(file), uintptr(flags), uintptr(reserved), uintptr(bytesLow), uintptr(bytesHigh), uintptr(unsafe.Pointer(overlapped))) + r1, _, e1 := syscall.SyscallN(procLockFileEx.Addr(), uintptr(file), uintptr(flags), uintptr(reserved), uintptr(bytesLow), uintptr(bytesHigh), uintptr(unsafe.Pointer(overlapped))) if r1 == 0 { err = errnoErr(e1) } @@ -301,7 +409,7 @@ func LockFileEx(file syscall.Handle, flags uint32, reserved uint32, bytesLow uin } func Module32First(snapshot syscall.Handle, moduleEntry *ModuleEntry32) (err error) { - r1, _, e1 := syscall.Syscall(procModule32FirstW.Addr(), 2, uintptr(snapshot), uintptr(unsafe.Pointer(moduleEntry)), 0) + r1, _, e1 := syscall.SyscallN(procModule32FirstW.Addr(), uintptr(snapshot), uintptr(unsafe.Pointer(moduleEntry))) if r1 == 0 { err = errnoErr(e1) } @@ -309,7 +417,7 @@ func Module32First(snapshot syscall.Handle, moduleEntry *ModuleEntry32) (err err } func Module32Next(snapshot syscall.Handle, moduleEntry *ModuleEntry32) (err error) { - r1, _, e1 := syscall.Syscall(procModule32NextW.Addr(), 2, uintptr(snapshot), uintptr(unsafe.Pointer(moduleEntry)), 0) + r1, _, e1 := syscall.SyscallN(procModule32NextW.Addr(), uintptr(snapshot), uintptr(unsafe.Pointer(moduleEntry))) if r1 == 0 { err = errnoErr(e1) } @@ -317,7 +425,7 @@ func Module32Next(snapshot syscall.Handle, moduleEntry *ModuleEntry32) (err erro } func MoveFileEx(from *uint16, to *uint16, flags uint32) (err error) { - r1, _, e1 := syscall.Syscall(procMoveFileExW.Addr(), 3, uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(to)), uintptr(flags)) + r1, _, e1 := syscall.SyscallN(procMoveFileExW.Addr(), uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(to)), uintptr(flags)) if r1 == 0 { err = errnoErr(e1) } @@ -325,7 +433,7 @@ func MoveFileEx(from *uint16, to *uint16, flags uint32) (err error) { } func MultiByteToWideChar(codePage uint32, dwFlags uint32, str *byte, nstr int32, wchar *uint16, nwchar int32) (nwrite int32, err error) { - r0, _, e1 := syscall.Syscall6(procMultiByteToWideChar.Addr(), 6, uintptr(codePage), uintptr(dwFlags), uintptr(unsafe.Pointer(str)), uintptr(nstr), uintptr(unsafe.Pointer(wchar)), uintptr(nwchar)) + r0, _, e1 := syscall.SyscallN(procMultiByteToWideChar.Addr(), uintptr(codePage), uintptr(dwFlags), uintptr(unsafe.Pointer(str)), uintptr(nstr), uintptr(unsafe.Pointer(wchar)), uintptr(nwchar)) nwrite = int32(r0) if nwrite == 0 { err = errnoErr(e1) @@ -333,20 +441,29 @@ func MultiByteToWideChar(codePage uint32, dwFlags uint32, str *byte, nstr int32, return } -func RtlLookupFunctionEntry(pc uintptr, baseAddress *uintptr, table *byte) (ret uintptr) { - r0, _, _ := syscall.Syscall(procRtlLookupFunctionEntry.Addr(), 3, uintptr(pc), uintptr(unsafe.Pointer(baseAddress)), uintptr(unsafe.Pointer(table))) - ret = uintptr(r0) +func ReOpenFile(filehandle syscall.Handle, desiredAccess uint32, shareMode uint32, flagAndAttributes uint32) (handle syscall.Handle, err error) { + r0, _, e1 := syscall.SyscallN(procReOpenFile.Addr(), uintptr(filehandle), uintptr(desiredAccess), uintptr(shareMode), uintptr(flagAndAttributes)) + handle = syscall.Handle(r0) + if handle == 0 { + err = errnoErr(e1) + } return } -func RtlVirtualUnwind(handlerType uint32, baseAddress uintptr, pc uintptr, entry uintptr, ctxt uintptr, data *uintptr, frame *uintptr, ctxptrs *byte) (ret uintptr) { - r0, _, _ := syscall.Syscall9(procRtlVirtualUnwind.Addr(), 8, uintptr(handlerType), uintptr(baseAddress), uintptr(pc), uintptr(entry), uintptr(ctxt), uintptr(unsafe.Pointer(data)), uintptr(unsafe.Pointer(frame)), uintptr(unsafe.Pointer(ctxptrs)), 0) +func RtlLookupFunctionEntry(pc uintptr, baseAddress *uintptr, table unsafe.Pointer) (ret *RUNTIME_FUNCTION) { + r0, _, _ := syscall.SyscallN(procRtlLookupFunctionEntry.Addr(), uintptr(pc), uintptr(unsafe.Pointer(baseAddress)), uintptr(table)) + ret = (*RUNTIME_FUNCTION)(unsafe.Pointer(r0)) + return +} + +func RtlVirtualUnwind(handlerType uint32, baseAddress uintptr, pc uintptr, entry *RUNTIME_FUNCTION, ctxt unsafe.Pointer, data unsafe.Pointer, frame *uintptr, ctxptrs unsafe.Pointer) (ret uintptr) { + r0, _, _ := syscall.SyscallN(procRtlVirtualUnwind.Addr(), uintptr(handlerType), uintptr(baseAddress), uintptr(pc), uintptr(unsafe.Pointer(entry)), uintptr(ctxt), uintptr(data), uintptr(unsafe.Pointer(frame)), uintptr(ctxptrs)) ret = uintptr(r0) return } func SetFileInformationByHandle(handle syscall.Handle, fileInformationClass uint32, buf unsafe.Pointer, bufsize uint32) (err error) { - r1, _, e1 := syscall.Syscall6(procSetFileInformationByHandle.Addr(), 4, uintptr(handle), uintptr(fileInformationClass), uintptr(buf), uintptr(bufsize), 0, 0) + r1, _, e1 := syscall.SyscallN(procSetFileInformationByHandle.Addr(), uintptr(handle), uintptr(fileInformationClass), uintptr(buf), uintptr(bufsize)) if r1 == 0 { err = errnoErr(e1) } @@ -354,7 +471,7 @@ func SetFileInformationByHandle(handle syscall.Handle, fileInformationClass uint } func UnlockFileEx(file syscall.Handle, reserved uint32, bytesLow uint32, bytesHigh uint32, overlapped *syscall.Overlapped) (err error) { - r1, _, e1 := syscall.Syscall6(procUnlockFileEx.Addr(), 5, uintptr(file), uintptr(reserved), uintptr(bytesLow), uintptr(bytesHigh), uintptr(unsafe.Pointer(overlapped)), 0) + r1, _, e1 := syscall.SyscallN(procUnlockFileEx.Addr(), uintptr(file), uintptr(reserved), uintptr(bytesLow), uintptr(bytesHigh), uintptr(unsafe.Pointer(overlapped))) if r1 == 0 { err = errnoErr(e1) } @@ -362,7 +479,7 @@ func UnlockFileEx(file syscall.Handle, reserved uint32, bytesLow uint32, bytesHi } func VirtualQuery(address uintptr, buffer *MemoryBasicInformation, length uintptr) (err error) { - r1, _, e1 := syscall.Syscall(procVirtualQuery.Addr(), 3, uintptr(address), uintptr(unsafe.Pointer(buffer)), uintptr(length)) + r1, _, e1 := syscall.SyscallN(procVirtualQuery.Addr(), uintptr(address), uintptr(unsafe.Pointer(buffer)), uintptr(length)) if r1 == 0 { err = errnoErr(e1) } @@ -370,7 +487,7 @@ func VirtualQuery(address uintptr, buffer *MemoryBasicInformation, length uintpt } func NetShareAdd(serverName *uint16, level uint32, buf *byte, parmErr *uint16) (neterr error) { - r0, _, _ := syscall.Syscall6(procNetShareAdd.Addr(), 4, uintptr(unsafe.Pointer(serverName)), uintptr(level), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(parmErr)), 0, 0) + r0, _, _ := syscall.SyscallN(procNetShareAdd.Addr(), uintptr(unsafe.Pointer(serverName)), uintptr(level), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(parmErr))) if r0 != 0 { neterr = syscall.Errno(r0) } @@ -378,7 +495,23 @@ func NetShareAdd(serverName *uint16, level uint32, buf *byte, parmErr *uint16) ( } func NetShareDel(serverName *uint16, netName *uint16, reserved uint32) (neterr error) { - r0, _, _ := syscall.Syscall(procNetShareDel.Addr(), 3, uintptr(unsafe.Pointer(serverName)), uintptr(unsafe.Pointer(netName)), uintptr(reserved)) + r0, _, _ := syscall.SyscallN(procNetShareDel.Addr(), uintptr(unsafe.Pointer(serverName)), uintptr(unsafe.Pointer(netName)), uintptr(reserved)) + if r0 != 0 { + neterr = syscall.Errno(r0) + } + return +} + +func NetUserAdd(serverName *uint16, level uint32, buf *byte, parmErr *uint32) (neterr error) { + r0, _, _ := syscall.SyscallN(procNetUserAdd.Addr(), uintptr(unsafe.Pointer(serverName)), uintptr(level), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(parmErr))) + if r0 != 0 { + neterr = syscall.Errno(r0) + } + return +} + +func NetUserDel(serverName *uint16, userName *uint16) (neterr error) { + r0, _, _ := syscall.SyscallN(procNetUserDel.Addr(), uintptr(unsafe.Pointer(serverName)), uintptr(unsafe.Pointer(userName))) if r0 != 0 { neterr = syscall.Errno(r0) } @@ -386,20 +519,64 @@ func NetShareDel(serverName *uint16, netName *uint16, reserved uint32) (neterr e } func NetUserGetLocalGroups(serverName *uint16, userName *uint16, level uint32, flags uint32, buf **byte, prefMaxLen uint32, entriesRead *uint32, totalEntries *uint32) (neterr error) { - r0, _, _ := syscall.Syscall9(procNetUserGetLocalGroups.Addr(), 8, uintptr(unsafe.Pointer(serverName)), uintptr(unsafe.Pointer(userName)), uintptr(level), uintptr(flags), uintptr(unsafe.Pointer(buf)), uintptr(prefMaxLen), uintptr(unsafe.Pointer(entriesRead)), uintptr(unsafe.Pointer(totalEntries)), 0) + r0, _, _ := syscall.SyscallN(procNetUserGetLocalGroups.Addr(), uintptr(unsafe.Pointer(serverName)), uintptr(unsafe.Pointer(userName)), uintptr(level), uintptr(flags), uintptr(unsafe.Pointer(buf)), uintptr(prefMaxLen), uintptr(unsafe.Pointer(entriesRead)), uintptr(unsafe.Pointer(totalEntries))) if r0 != 0 { neterr = syscall.Errno(r0) } return } -func rtlGetVersion(info *_OSVERSIONINFOW) { - syscall.Syscall(procRtlGetVersion.Addr(), 1, uintptr(unsafe.Pointer(info)), 0, 0) +func NtCreateFile(handle *syscall.Handle, access uint32, oa *OBJECT_ATTRIBUTES, iosb *IO_STATUS_BLOCK, allocationSize *int64, attributes uint32, share uint32, disposition uint32, options uint32, eabuffer unsafe.Pointer, ealength uint32) (ntstatus error) { + r0, _, _ := syscall.SyscallN(procNtCreateFile.Addr(), uintptr(unsafe.Pointer(handle)), uintptr(access), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(unsafe.Pointer(allocationSize)), uintptr(attributes), uintptr(share), uintptr(disposition), uintptr(options), uintptr(eabuffer), uintptr(ealength)) + if r0 != 0 { + ntstatus = NTStatus(r0) + } + return +} + +func NtOpenFile(handle *syscall.Handle, access uint32, oa *OBJECT_ATTRIBUTES, iosb *IO_STATUS_BLOCK, share uint32, options uint32) (ntstatus error) { + r0, _, _ := syscall.SyscallN(procNtOpenFile.Addr(), uintptr(unsafe.Pointer(handle)), uintptr(access), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(share), uintptr(options)) + if r0 != 0 { + ntstatus = NTStatus(r0) + } + return +} + +func NtQueryInformationFile(handle syscall.Handle, iosb *IO_STATUS_BLOCK, inBuffer unsafe.Pointer, inBufferLen uint32, class uint32) (ntstatus error) { + r0, _, _ := syscall.SyscallN(procNtQueryInformationFile.Addr(), uintptr(handle), uintptr(unsafe.Pointer(iosb)), uintptr(inBuffer), uintptr(inBufferLen), uintptr(class)) + if r0 != 0 { + ntstatus = NTStatus(r0) + } + return +} + +func NtSetInformationFile(handle syscall.Handle, iosb *IO_STATUS_BLOCK, inBuffer unsafe.Pointer, inBufferLen uint32, class uint32) (ntstatus error) { + r0, _, _ := syscall.SyscallN(procNtSetInformationFile.Addr(), uintptr(handle), uintptr(unsafe.Pointer(iosb)), uintptr(inBuffer), uintptr(inBufferLen), uintptr(class)) + if r0 != 0 { + ntstatus = NTStatus(r0) + } + return +} + +func rtlGetVersion(info *_OSVERSIONINFOEXW) { + syscall.SyscallN(procRtlGetVersion.Addr(), uintptr(unsafe.Pointer(info))) + return +} + +func RtlIsDosDeviceName_U(name *uint16) (ret uint32) { + r0, _, _ := syscall.SyscallN(procRtlIsDosDeviceName_U.Addr(), uintptr(unsafe.Pointer(name))) + ret = uint32(r0) + return +} + +func rtlNtStatusToDosErrorNoTeb(ntstatus NTStatus) (ret syscall.Errno) { + r0, _, _ := syscall.SyscallN(procRtlNtStatusToDosErrorNoTeb.Addr(), uintptr(ntstatus)) + ret = syscall.Errno(r0) return } func GetProcessMemoryInfo(handle syscall.Handle, memCounters *PROCESS_MEMORY_COUNTERS, cb uint32) (err error) { - r1, _, e1 := syscall.Syscall(procGetProcessMemoryInfo.Addr(), 3, uintptr(handle), uintptr(unsafe.Pointer(memCounters)), uintptr(cb)) + r1, _, e1 := syscall.SyscallN(procGetProcessMemoryInfo.Addr(), uintptr(handle), uintptr(unsafe.Pointer(memCounters)), uintptr(cb)) if r1 == 0 { err = errnoErr(e1) } @@ -411,7 +588,7 @@ func CreateEnvironmentBlock(block **uint16, token syscall.Token, inheritExisting if inheritExisting { _p0 = 1 } - r1, _, e1 := syscall.Syscall(procCreateEnvironmentBlock.Addr(), 3, uintptr(unsafe.Pointer(block)), uintptr(token), uintptr(_p0)) + r1, _, e1 := syscall.SyscallN(procCreateEnvironmentBlock.Addr(), uintptr(unsafe.Pointer(block)), uintptr(token), uintptr(_p0)) if r1 == 0 { err = errnoErr(e1) } @@ -419,7 +596,7 @@ func CreateEnvironmentBlock(block **uint16, token syscall.Token, inheritExisting } func DestroyEnvironmentBlock(block *uint16) (err error) { - r1, _, e1 := syscall.Syscall(procDestroyEnvironmentBlock.Addr(), 1, uintptr(unsafe.Pointer(block)), 0, 0) + r1, _, e1 := syscall.SyscallN(procDestroyEnvironmentBlock.Addr(), uintptr(unsafe.Pointer(block))) if r1 == 0 { err = errnoErr(e1) } @@ -427,19 +604,27 @@ func DestroyEnvironmentBlock(block *uint16) (err error) { } func GetProfilesDirectory(dir *uint16, dirLen *uint32) (err error) { - r1, _, e1 := syscall.Syscall(procGetProfilesDirectoryW.Addr(), 2, uintptr(unsafe.Pointer(dir)), uintptr(unsafe.Pointer(dirLen)), 0) + r1, _, e1 := syscall.SyscallN(procGetProfilesDirectoryW.Addr(), uintptr(unsafe.Pointer(dir)), uintptr(unsafe.Pointer(dirLen))) if r1 == 0 { err = errnoErr(e1) } return } +func WSADuplicateSocket(s syscall.Handle, processID uint32, info *syscall.WSAProtocolInfo) (err error) { + r1, _, e1 := syscall.SyscallN(procWSADuplicateSocketW.Addr(), uintptr(s), uintptr(processID), uintptr(unsafe.Pointer(info))) + if r1 != 0 { + err = errnoErr(e1) + } + return +} + func WSAGetOverlappedResult(h syscall.Handle, o *syscall.Overlapped, bytes *uint32, wait bool, flags *uint32) (err error) { var _p0 uint32 if wait { _p0 = 1 } - r1, _, e1 := syscall.Syscall6(procWSAGetOverlappedResult.Addr(), 5, uintptr(h), uintptr(unsafe.Pointer(o)), uintptr(unsafe.Pointer(bytes)), uintptr(_p0), uintptr(unsafe.Pointer(flags)), 0) + r1, _, e1 := syscall.SyscallN(procWSAGetOverlappedResult.Addr(), uintptr(h), uintptr(unsafe.Pointer(o)), uintptr(unsafe.Pointer(bytes)), uintptr(_p0), uintptr(unsafe.Pointer(flags))) if r1 == 0 { err = errnoErr(e1) } @@ -447,7 +632,7 @@ func WSAGetOverlappedResult(h syscall.Handle, o *syscall.Overlapped, bytes *uint } func WSASocket(af int32, typ int32, protocol int32, protinfo *syscall.WSAProtocolInfo, group uint32, flags uint32) (handle syscall.Handle, err error) { - r0, _, e1 := syscall.Syscall6(procWSASocketW.Addr(), 6, uintptr(af), uintptr(typ), uintptr(protocol), uintptr(unsafe.Pointer(protinfo)), uintptr(group), uintptr(flags)) + r0, _, e1 := syscall.SyscallN(procWSASocketW.Addr(), uintptr(af), uintptr(typ), uintptr(protocol), uintptr(unsafe.Pointer(protinfo)), uintptr(group), uintptr(flags)) handle = syscall.Handle(r0) if handle == syscall.InvalidHandle { err = errnoErr(e1) diff --git a/src/internal/syslist/syslist.go b/src/internal/syslist/syslist.go index 7d1a2b3c3d6e49..2349b6ea64a6d3 100644 --- a/src/internal/syslist/syslist.go +++ b/src/internal/syslist/syslist.go @@ -37,8 +37,7 @@ var KnownOS = map[string]bool{ // UnixOS is the set of GOOS values matched by the "unix" build tag. // This is not used for filename matching. -// This list also appears in cmd/dist/build.go and -// cmd/go/internal/imports/build.go. +// This list also appears in cmd/dist/build.go. var UnixOS = map[string]bool{ "aix": true, "android": true, diff --git a/src/internal/testenv/exec.go b/src/internal/testenv/exec.go index 7f6ad5cac4e39b..7b251b602237f3 100644 --- a/src/internal/testenv/exec.go +++ b/src/internal/testenv/exec.go @@ -31,20 +31,17 @@ import ( // If exec is not supported, testenv.SyscallIsNotSupported will return true // for the resulting error. func MustHaveExec(t testing.TB) { - tryExecOnce.Do(func() { - tryExecErr = tryExec() - }) - if tryExecErr != nil { - t.Skipf("skipping test: cannot exec subprocess on %s/%s: %v", runtime.GOOS, runtime.GOARCH, tryExecErr) + if err := tryExec(); err != nil { + msg := fmt.Sprintf("cannot exec subprocess on %s/%s: %v", runtime.GOOS, runtime.GOARCH, err) + if t == nil { + panic(msg) + } + t.Helper() + t.Skip("skipping test:", msg) } } -var ( - tryExecOnce sync.Once - tryExecErr error -) - -func tryExec() error { +var tryExec = sync.OnceValue(func() error { switch runtime.GOOS { case "wasip1", "js", "ios": default: @@ -70,15 +67,37 @@ func tryExec() error { // We know that this is a test executable. We should be able to run it with a // no-op flag to check for overall exec support. - exe, err := os.Executable() + exe, err := exePath() if err != nil { return fmt.Errorf("can't probe for exec support: %w", err) } cmd := exec.Command(exe, "-test.list=^$") cmd.Env = origEnv return cmd.Run() +}) + +// Executable is a wrapper around [MustHaveExec] and [os.Executable]. +// It returns the path name for the executable that started the current process, +// or skips the test if the current system can't start new processes, +// or fails the test if the path can not be obtained. +func Executable(t testing.TB) string { + MustHaveExec(t) + + exe, err := exePath() + if err != nil { + msg := fmt.Sprintf("os.Executable error: %v", err) + if t == nil { + panic(msg) + } + t.Fatal(msg) + } + return exe } +var exePath = sync.OnceValues(func() (string, error) { + return os.Executable() +}) + var execPaths sync.Map // path -> error // MustHaveExecPath checks that the current system can start the named executable @@ -93,6 +112,7 @@ func MustHaveExecPath(t testing.TB, path string) { err, _ = execPaths.LoadOrStore(path, err) } if err != nil { + t.Helper() t.Skipf("skipping test: %s: %s", path, err) } } diff --git a/src/internal/testenv/testenv.go b/src/internal/testenv/testenv.go index 9fb92406e8d85e..96eacc60a3a07d 100644 --- a/src/internal/testenv/testenv.go +++ b/src/internal/testenv/testenv.go @@ -34,8 +34,13 @@ import ( // environment might cause environment checks to behave erratically. var origEnv = os.Environ() -// Builder reports the name of the builder running this test -// (for example, "linux-amd64" or "windows-386-gce"). +// Builder reports the name of the builder running this test. For example, +// "gotip-linux-amd64_avx512-test_only" or "go1.24-windows-arm64" on LUCI, +// or "linux-amd64" on our old infrastructure. Prefer using runtime.GOOS, +// runtime.GOARCH, race.Enabled, reading the OS version, checking CPU +// feature flags with internal/cpu, etc. over parsing builder names when +// possible. When matching builder names, prefer a fuzzy match instead +// of a strict comparison. // If the test is not running on the build infrastructure, // Builder returns the empty string. func Builder() string { @@ -53,63 +58,59 @@ func HasGoBuild() bool { return false } - goBuildOnce.Do(func() { - // To run 'go build', we need to be able to exec a 'go' command. - // We somewhat arbitrarily choose to exec 'go tool -n compile' because that - // also confirms that cmd/go can find the compiler. (Before CL 472096, - // we sometimes ended up with cmd/go installed in the test environment - // without a cmd/compile it could use to actually build things.) - cmd := exec.Command("go", "tool", "-n", "compile") - cmd.Env = origEnv - out, err := cmd.Output() - if err != nil { - goBuildErr = fmt.Errorf("%v: %w", cmd, err) - return - } - out = bytes.TrimSpace(out) - if len(out) == 0 { - goBuildErr = fmt.Errorf("%v: no tool reported", cmd) - return - } - if _, err := exec.LookPath(string(out)); err != nil { - goBuildErr = err - return - } + return tryGoBuild() == nil +} + +var tryGoBuild = sync.OnceValue(func() error { + // To run 'go build', we need to be able to exec a 'go' command. + // We somewhat arbitrarily choose to exec 'go tool -n compile' because that + // also confirms that cmd/go can find the compiler. (Before CL 472096, + // we sometimes ended up with cmd/go installed in the test environment + // without a cmd/compile it could use to actually build things.) + goTool, err := goTool() + if err != nil { + return err + } + cmd := exec.Command(goTool, "tool", "-n", "compile") + cmd.Env = origEnv + out, err := cmd.Output() + if err != nil { + return fmt.Errorf("%v: %w", cmd, err) + } + out = bytes.TrimSpace(out) + if len(out) == 0 { + return fmt.Errorf("%v: no tool reported", cmd) + } + if _, err := exec.LookPath(string(out)); err != nil { + return err + } - if platform.MustLinkExternal(runtime.GOOS, runtime.GOARCH, false) { - // We can assume that we always have a complete Go toolchain available. - // However, this platform requires a C linker to build even pure Go - // programs, including tests. Do we have one in the test environment? - // (On Android, for example, the device running the test might not have a - // C toolchain installed.) - // - // If CC is set explicitly, assume that we do. Otherwise, use 'go env CC' - // to determine which toolchain it would use by default. - if os.Getenv("CC") == "" { - cmd := exec.Command("go", "env", "CC") - cmd.Env = origEnv - out, err := cmd.Output() - if err != nil { - goBuildErr = fmt.Errorf("%v: %w", cmd, err) - return - } - out = bytes.TrimSpace(out) - if len(out) == 0 { - goBuildErr = fmt.Errorf("%v: no CC reported", cmd) - return - } - _, goBuildErr = exec.LookPath(string(out)) + if platform.MustLinkExternal(runtime.GOOS, runtime.GOARCH, false) { + // We can assume that we always have a complete Go toolchain available. + // However, this platform requires a C linker to build even pure Go + // programs, including tests. Do we have one in the test environment? + // (On Android, for example, the device running the test might not have a + // C toolchain installed.) + // + // If CC is set explicitly, assume that we do. Otherwise, use 'go env CC' + // to determine which toolchain it would use by default. + if os.Getenv("CC") == "" { + cmd := exec.Command(goTool, "env", "CC") + cmd.Env = origEnv + out, err := cmd.Output() + if err != nil { + return fmt.Errorf("%v: %w", cmd, err) + } + out = bytes.TrimSpace(out) + if len(out) == 0 { + return fmt.Errorf("%v: no CC reported", cmd) } + _, err = exec.LookPath(string(out)) + return err } - }) - - return goBuildErr == nil -} - -var ( - goBuildOnce sync.Once - goBuildErr error -) + } + return nil +}) // MustHaveGoBuild checks that the current system can build programs with “go build” // and then run them with os.StartProcess or exec.Command. @@ -121,7 +122,7 @@ func MustHaveGoBuild(t testing.TB) { } if !HasGoBuild() { t.Helper() - t.Skipf("skipping test: 'go build' unavailable: %v", goBuildErr) + t.Skipf("skipping test: 'go build' unavailable: %v", tryGoBuild()) } } @@ -135,6 +136,7 @@ func HasGoRun() bool { // If not, MustHaveGoRun calls t.Skip with an explanation. func MustHaveGoRun(t testing.TB) { if !HasGoRun() { + t.Helper() t.Skipf("skipping test: 'go run' not available on %s/%s", runtime.GOOS, runtime.GOARCH) } } @@ -154,6 +156,7 @@ func HasParallelism() bool { // threads in parallel. If not, MustHaveParallelism calls t.Skip with an explanation. func MustHaveParallelism(t testing.TB) { if !HasParallelism() { + t.Helper() t.Skipf("skipping test: no parallelism available on %s/%s", runtime.GOOS, runtime.GOARCH) } } @@ -177,82 +180,67 @@ func GoToolPath(t testing.TB) string { return path } -var ( - gorootOnce sync.Once - gorootPath string - gorootErr error -) - -func findGOROOT() (string, error) { - gorootOnce.Do(func() { - gorootPath = runtime.GOROOT() - if gorootPath != "" { - // If runtime.GOROOT() is non-empty, assume that it is valid. - // - // (It might not be: for example, the user may have explicitly set GOROOT - // to the wrong directory. But this case is - // rare, and if that happens the user can fix what they broke.) - return - } - - // runtime.GOROOT doesn't know where GOROOT is (perhaps because the test - // binary was built with -trimpath). - // - // Since this is internal/testenv, we can cheat and assume that the caller - // is a test of some package in a subdirectory of GOROOT/src. ('go test' - // runs the test in the directory containing the packaged under test.) That - // means that if we start walking up the tree, we should eventually find - // GOROOT/src/go.mod, and we can report the parent directory of that. +var findGOROOT = sync.OnceValues(func() (path string, err error) { + if path := runtime.GOROOT(); path != "" { + // If runtime.GOROOT() is non-empty, assume that it is valid. // - // Notably, this works even if we can't run 'go env GOROOT' as a - // subprocess. + // (It might not be: for example, the user may have explicitly set GOROOT + // to the wrong directory. But this case is + // rare, and if that happens the user can fix what they broke.) + return path, nil + } - cwd, err := os.Getwd() - if err != nil { - gorootErr = fmt.Errorf("finding GOROOT: %w", err) - return + // runtime.GOROOT doesn't know where GOROOT is (perhaps because the test + // binary was built with -trimpath). + // + // Since this is internal/testenv, we can cheat and assume that the caller + // is a test of some package in a subdirectory of GOROOT/src. ('go test' + // runs the test in the directory containing the packaged under test.) That + // means that if we start walking up the tree, we should eventually find + // GOROOT/src/go.mod, and we can report the parent directory of that. + // + // Notably, this works even if we can't run 'go env GOROOT' as a + // subprocess. + + cwd, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("finding GOROOT: %w", err) + } + + dir := cwd + for { + parent := filepath.Dir(dir) + if parent == dir { + // dir is either "." or only a volume name. + return "", fmt.Errorf("failed to locate GOROOT/src in any parent directory") } - dir := cwd - for { - parent := filepath.Dir(dir) - if parent == dir { - // dir is either "." or only a volume name. - gorootErr = fmt.Errorf("failed to locate GOROOT/src in any parent directory") - return - } + if base := filepath.Base(dir); base != "src" { + dir = parent + continue // dir cannot be GOROOT/src if it doesn't end in "src". + } - if base := filepath.Base(dir); base != "src" { + b, err := os.ReadFile(filepath.Join(dir, "go.mod")) + if err != nil { + if os.IsNotExist(err) { dir = parent - continue // dir cannot be GOROOT/src if it doesn't end in "src". - } - - b, err := os.ReadFile(filepath.Join(dir, "go.mod")) - if err != nil { - if os.IsNotExist(err) { - dir = parent - continue - } - gorootErr = fmt.Errorf("finding GOROOT: %w", err) - return + continue } - goMod := string(b) - - for goMod != "" { - var line string - line, goMod, _ = strings.Cut(goMod, "\n") - fields := strings.Fields(line) - if len(fields) >= 2 && fields[0] == "module" && fields[1] == "std" { - // Found "module std", which is the module declaration in GOROOT/src! - gorootPath = parent - return - } + return "", fmt.Errorf("finding GOROOT: %w", err) + } + goMod := string(b) + + for goMod != "" { + var line string + line, goMod, _ = strings.Cut(goMod, "\n") + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == "module" && fields[1] == "std" { + // Found "module std", which is the module declaration in GOROOT/src! + return parent, nil } } - }) - - return gorootPath, gorootErr -} + } +}) // GOROOT reports the path to the directory containing the root of the Go // project source tree. This is normally equivalent to runtime.GOROOT, but @@ -278,25 +266,21 @@ func GoTool() (string, error) { if !HasGoBuild() { return "", errors.New("platform cannot run go tool") } - goToolOnce.Do(func() { - goToolPath, goToolErr = exec.LookPath("go") - }) - return goToolPath, goToolErr + return goTool() } -var ( - goToolOnce sync.Once - goToolPath string - goToolErr error -) +var goTool = sync.OnceValues(func() (string, error) { + return exec.LookPath("go") +}) -// HasSrc reports whether the entire source tree is available under GOROOT. -func HasSrc() bool { +// MustHaveSource checks that the entire source tree is available under GOROOT. +// If not, it calls t.Skip with an explanation. +func MustHaveSource(t testing.TB) { switch runtime.GOOS { case "ios": - return false + t.Helper() + t.Skip("skipping test: no source tree on " + runtime.GOOS) } - return true } // HasExternalNetwork reports whether the current system can use @@ -321,33 +305,31 @@ func MustHaveExternalNetwork(t testing.TB) { // HasCGO reports whether the current system can use cgo. func HasCGO() bool { - hasCgoOnce.Do(func() { - goTool, err := GoTool() - if err != nil { - return - } - cmd := exec.Command(goTool, "env", "CGO_ENABLED") - cmd.Env = origEnv - out, err := cmd.Output() - if err != nil { - panic(fmt.Sprintf("%v: %v", cmd, out)) - } - hasCgo, err = strconv.ParseBool(string(bytes.TrimSpace(out))) - if err != nil { - panic(fmt.Sprintf("%v: non-boolean output %q", cmd, out)) - } - }) - return hasCgo + return hasCgo() } -var ( - hasCgoOnce sync.Once - hasCgo bool -) +var hasCgo = sync.OnceValue(func() bool { + goTool, err := goTool() + if err != nil { + return false + } + cmd := exec.Command(goTool, "env", "CGO_ENABLED") + cmd.Env = origEnv + out, err := cmd.Output() + if err != nil { + panic(fmt.Sprintf("%v: %v", cmd, out)) + } + ok, err := strconv.ParseBool(string(bytes.TrimSpace(out))) + if err != nil { + panic(fmt.Sprintf("%v: non-boolean output %q", cmd, out)) + } + return ok +}) // MustHaveCGO calls t.Skip if cgo is not available. func MustHaveCGO(t testing.TB) { if !HasCGO() { + t.Helper() t.Skipf("skipping test: no cgo") } } @@ -358,12 +340,27 @@ func CanInternalLink(withCgo bool) bool { return !platform.MustLinkExternal(runtime.GOOS, runtime.GOARCH, withCgo) } +// SpecialBuildTypes are interesting build types that may affect linking. +type SpecialBuildTypes struct { + Cgo bool + Asan bool + Msan bool + Race bool +} + +// NoSpecialBuildTypes indicates a standard, no cgo go build. +var NoSpecialBuildTypes SpecialBuildTypes + // MustInternalLink checks that the current system can link programs with internal // linking. // If not, MustInternalLink calls t.Skip with an explanation. -func MustInternalLink(t testing.TB, withCgo bool) { - if !CanInternalLink(withCgo) { - if withCgo && CanInternalLink(false) { +func MustInternalLink(t testing.TB, with SpecialBuildTypes) { + if with.Asan || with.Msan || with.Race { + t.Skipf("skipping test: internal linking with sanitizers is not supported") + } + if !CanInternalLink(with.Cgo) { + t.Helper() + if with.Cgo && CanInternalLink(false) { t.Skipf("skipping test: internal linking on %s/%s is not supported with cgo", runtime.GOOS, runtime.GOARCH) } t.Skipf("skipping test: internal linking on %s/%s is not supported", runtime.GOOS, runtime.GOARCH) @@ -375,6 +372,7 @@ func MustInternalLink(t testing.TB, withCgo bool) { // If not, MustInternalLinkPIE calls t.Skip with an explanation. func MustInternalLinkPIE(t testing.TB) { if !platform.InternalLinkPIESupported(runtime.GOOS, runtime.GOARCH) { + t.Helper() t.Skipf("skipping test: internal linking for buildmode=pie on %s/%s is not supported", runtime.GOOS, runtime.GOARCH) } } @@ -384,6 +382,7 @@ func MustInternalLinkPIE(t testing.TB) { // If not, MustHaveBuildMode calls t.Skip with an explanation. func MustHaveBuildMode(t testing.TB, buildmode string) { if !platform.BuildModeSupported(runtime.Compiler, buildmode, runtime.GOOS, runtime.GOARCH) { + t.Helper() t.Skipf("skipping test: build mode %s on %s/%s is not supported by the %s compiler", buildmode, runtime.GOOS, runtime.GOARCH, runtime.Compiler) } } @@ -399,6 +398,7 @@ func HasSymlink() bool { func MustHaveSymlink(t testing.TB) { ok, reason := hasSymlink() if !ok { + t.Helper() t.Skipf("skipping test: cannot make symlinks on %s/%s: %s", runtime.GOOS, runtime.GOARCH, reason) } } @@ -415,6 +415,7 @@ func HasLink() bool { // If not, MustHaveLink calls t.Skip with an explanation. func MustHaveLink(t testing.TB) { if !HasLink() { + t.Helper() t.Skipf("skipping test: hardlinks are not supported on %s/%s", runtime.GOOS, runtime.GOARCH) } } @@ -422,15 +423,15 @@ func MustHaveLink(t testing.TB) { var flaky = flag.Bool("flaky", false, "run known-flaky tests too") func SkipFlaky(t testing.TB, issue int) { - t.Helper() if !*flaky { + t.Helper() t.Skipf("skipping known flaky test without the -flaky flag; see golang.org/issue/%d", issue) } } func SkipFlakyNet(t testing.TB) { - t.Helper() if v, _ := strconv.ParseBool(os.Getenv("GO_BUILDER_FLAKY_NET")); v { + t.Helper() t.Skip("skipping test on builder known to have frequent network failures") } } @@ -488,7 +489,7 @@ func WriteImportcfg(t testing.TB, dstPath string, packageFiles map[string]string t.Fatalf("%v: %v\n%s", cmd, err, cmd.Stderr) } - for _, line := range strings.Split(string(out), "\n") { + for line := range strings.SplitSeq(string(out), "\n") { if line == "" { continue } @@ -522,3 +523,26 @@ func ParallelOn64Bit(t *testing.T) { } t.Parallel() } + +// CPUProfilingBroken returns true if CPU profiling has known issues on this +// platform. +func CPUProfilingBroken() bool { + switch runtime.GOOS { + case "plan9": + // Profiling unimplemented. + return true + case "aix": + // See https://golang.org/issue/45170. + return true + case "ios", "dragonfly", "netbsd", "illumos", "solaris": + // See https://golang.org/issue/13841. + return true + case "openbsd": + if runtime.GOARCH == "arm" || runtime.GOARCH == "arm64" { + // See https://golang.org/issue/13841. + return true + } + } + + return false +} diff --git a/src/internal/testenv/testenv_notwin.go b/src/internal/testenv/testenv_notwin.go index 30e159a6ecd4a2..9dddea94d05798 100644 --- a/src/internal/testenv/testenv_notwin.go +++ b/src/internal/testenv/testenv_notwin.go @@ -11,9 +11,10 @@ import ( "os" "path/filepath" "runtime" + "sync" ) -func hasSymlink() (ok bool, reason string) { +var hasSymlink = sync.OnceValues(func() (ok bool, reason string) { switch runtime.GOOS { case "plan9": return false, "" @@ -43,4 +44,4 @@ func hasSymlink() (ok bool, reason string) { } return true, "" -} +}) diff --git a/src/internal/testenv/testenv_unix.go b/src/internal/testenv/testenv_unix.go index a629078842eadc..22eeca220da017 100644 --- a/src/internal/testenv/testenv_unix.go +++ b/src/internal/testenv/testenv_unix.go @@ -21,8 +21,7 @@ func syscallIsNotSupported(err error) bool { return false } - var errno syscall.Errno - if errors.As(err, &errno) { + if errno, ok := errors.AsType[syscall.Errno](err); ok { switch errno { case syscall.EPERM, syscall.EROFS: // User lacks permission: either the call requires root permission and the diff --git a/src/internal/testenv/testenv_windows.go b/src/internal/testenv/testenv_windows.go index 4802b139518e3e..eed53cdfb2b906 100644 --- a/src/internal/testenv/testenv_windows.go +++ b/src/internal/testenv/testenv_windows.go @@ -5,16 +5,14 @@ package testenv import ( + "errors" "os" "path/filepath" "sync" "syscall" ) -var symlinkOnce sync.Once -var winSymlinkErr error - -func initWinHasSymlink() { +var hasSymlink = sync.OnceValues(func() (bool, string) { tmpdir, err := os.MkdirTemp("", "symtest") if err != nil { panic("failed to create temp directory: " + err.Error()) @@ -22,26 +20,13 @@ func initWinHasSymlink() { defer os.RemoveAll(tmpdir) err = os.Symlink("target", filepath.Join(tmpdir, "symlink")) - if err != nil { - err = err.(*os.LinkError).Err - switch err { - case syscall.EWINDOWS, syscall.ERROR_PRIVILEGE_NOT_HELD: - winSymlinkErr = err - } - } -} - -func hasSymlink() (ok bool, reason string) { - symlinkOnce.Do(initWinHasSymlink) - - switch winSymlinkErr { - case nil: + switch { + case err == nil: return true, "" - case syscall.EWINDOWS: + case errors.Is(err, syscall.EWINDOWS): return false, ": symlinks are not supported on your version of Windows" - case syscall.ERROR_PRIVILEGE_NOT_HELD: + case errors.Is(err, syscall.ERROR_PRIVILEGE_NOT_HELD): return false, ": you don't have enough privileges to create symlinks" } - return false, "" -} +}) diff --git a/src/internal/testhash/hash.go b/src/internal/testhash/hash.go new file mode 100644 index 00000000000000..3413d5c20de29b --- /dev/null +++ b/src/internal/testhash/hash.go @@ -0,0 +1,231 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testhash + +import ( + "bytes" + "hash" + "io" + "math/rand" + "testing" + "time" +) + +type MakeHash func() hash.Hash + +// TestHash performs a set of tests on hash.Hash implementations, checking the +// documented requirements of Write, Sum, Reset, Size, and BlockSize. +func TestHash(t *testing.T, mh MakeHash) { + TestHashWithoutClone(t, mh) + + // Test whether the results after cloning are consistent. + t.Run("Clone", func(t *testing.T) { + h, ok := mh().(hash.Cloner) + if !ok { + t.Fatalf("%T does not implement hash.Cloner", mh) + } + h3, err := h.Clone() + if err != nil { + t.Fatalf("Clone failed: %v", err) + } + prefix := []byte("tmp") + writeToHash(t, h, prefix) + h2, err := h.Clone() + if err != nil { + t.Fatalf("Clone failed: %v", err) + } + prefixSum := h.Sum(nil) + if !bytes.Equal(prefixSum, h2.Sum(nil)) { + t.Fatalf("%T Clone results are inconsistent", h) + } + suffix := []byte("tmp2") + writeToHash(t, h, suffix) + writeToHash(t, h3, append(prefix, suffix...)) + compositeSum := h3.Sum(nil) + if !bytes.Equal(h.Sum(nil), compositeSum) { + t.Fatalf("%T Clone results are inconsistent", h) + } + if !bytes.Equal(h2.Sum(nil), prefixSum) { + t.Fatalf("%T Clone results are inconsistent", h) + } + writeToHash(t, h2, suffix) + if !bytes.Equal(h.Sum(nil), compositeSum) { + t.Fatalf("%T Clone results are inconsistent", h) + } + if !bytes.Equal(h2.Sum(nil), compositeSum) { + t.Fatalf("%T Clone results are inconsistent", h) + } + }) +} + +func TestHashWithoutClone(t *testing.T, mh MakeHash) { + // Test that Sum returns an appended digest matching output of Size + t.Run("SumAppend", func(t *testing.T) { + h := mh() + rng := newRandReader(t) + + emptyBuff := []byte("") + shortBuff := []byte("a") + longBuff := make([]byte, h.BlockSize()+1) + rng.Read(longBuff) + + // Set of example strings to append digest to + prefixes := [][]byte{nil, emptyBuff, shortBuff, longBuff} + + // Go to each string and check digest gets appended to and is correct size. + for _, prefix := range prefixes { + h.Reset() + + sum := getSum(t, h, prefix) // Append new digest to prefix + + // Check that Sum didn't alter the prefix + if !bytes.Equal(sum[:len(prefix)], prefix) { + t.Errorf("Sum alters passed buffer instead of appending; got %x, want %x", sum[:len(prefix)], prefix) + } + + // Check that the appended sum wasn't affected by the prefix + if expectedSum := getSum(t, h, nil); !bytes.Equal(sum[len(prefix):], expectedSum) { + t.Errorf("Sum behavior affected by data in the input buffer; got %x, want %x", sum[len(prefix):], expectedSum) + } + + // Check size of append + if got, want := len(sum)-len(prefix), h.Size(); got != want { + t.Errorf("Sum appends number of bytes != Size; got %v , want %v", got, want) + } + } + }) + + // Test that Hash.Write never returns error. + t.Run("WriteWithoutError", func(t *testing.T) { + h := mh() + rng := newRandReader(t) + + emptySlice := []byte("") + shortSlice := []byte("a") + longSlice := make([]byte, h.BlockSize()+1) + rng.Read(longSlice) + + // Set of example strings to append digest to + slices := [][]byte{emptySlice, shortSlice, longSlice} + + for _, slice := range slices { + writeToHash(t, h, slice) // Writes and checks Write doesn't error + } + }) + + t.Run("ResetState", func(t *testing.T) { + h := mh() + rng := newRandReader(t) + + emptySum := getSum(t, h, nil) + + // Write to hash and then Reset it and see if Sum is same as emptySum + writeEx := make([]byte, h.BlockSize()) + rng.Read(writeEx) + writeToHash(t, h, writeEx) + h.Reset() + resetSum := getSum(t, h, nil) + + if !bytes.Equal(emptySum, resetSum) { + t.Errorf("Reset hash yields different Sum than new hash; got %x, want %x", emptySum, resetSum) + } + }) + + // Check that Write isn't reading from beyond input slice's bounds + t.Run("OutOfBoundsRead", func(t *testing.T) { + h := mh() + blockSize := h.BlockSize() + rng := newRandReader(t) + + msg := make([]byte, blockSize) + rng.Read(msg) + writeToHash(t, h, msg) + expectedDigest := getSum(t, h, nil) // Record control digest + + h.Reset() + + // Make a buffer with msg in the middle and data on either end + buff := make([]byte, blockSize*3) + endOfPrefix, startOfSuffix := blockSize, blockSize*2 + + copy(buff[endOfPrefix:startOfSuffix], msg) + rng.Read(buff[:endOfPrefix]) + rng.Read(buff[startOfSuffix:]) + + writeToHash(t, h, buff[endOfPrefix:startOfSuffix]) + testDigest := getSum(t, h, nil) + + if !bytes.Equal(testDigest, expectedDigest) { + t.Errorf("Write affected by data outside of input slice bounds; got %x, want %x", testDigest, expectedDigest) + } + }) + + // Test that multiple calls to Write is stateful + t.Run("StatefulWrite", func(t *testing.T) { + h := mh() + rng := newRandReader(t) + + prefix, suffix := make([]byte, h.BlockSize()), make([]byte, h.BlockSize()) + rng.Read(prefix) + rng.Read(suffix) + + // Write prefix then suffix sequentially and record resulting hash + writeToHash(t, h, prefix) + writeToHash(t, h, suffix) + serialSum := getSum(t, h, nil) + + h.Reset() + + // Write prefix and suffix at the same time and record resulting hash + writeToHash(t, h, append(prefix, suffix...)) + compositeSum := getSum(t, h, nil) + + // Check that sequential writing results in the same as writing all at once + if !bytes.Equal(compositeSum, serialSum) { + t.Errorf("two successive Write calls resulted in a different Sum than a single one; got %x, want %x", compositeSum, serialSum) + } + }) +} + +// Helper function for writing. Verifies that Write does not error. +func writeToHash(t *testing.T, h hash.Hash, p []byte) { + t.Helper() + + before := make([]byte, len(p)) + copy(before, p) + + n, err := h.Write(p) + if err != nil || n != len(p) { + t.Errorf("Write returned error; got (%v, %v), want (nil, %v)", err, n, len(p)) + } + + if !bytes.Equal(p, before) { + t.Errorf("Write modified input slice; got %x, want %x", p, before) + } +} + +// Helper function for getting Sum. Checks that Sum doesn't change hash state. +func getSum(t *testing.T, h hash.Hash, buff []byte) []byte { + t.Helper() + + testBuff := make([]byte, len(buff)) + copy(testBuff, buff) + + sum := h.Sum(buff) + testSum := h.Sum(testBuff) + + // Check that Sum doesn't change underlying hash state + if !bytes.Equal(sum, testSum) { + t.Errorf("successive calls to Sum yield different results; got %x, want %x", sum, testSum) + } + + return sum +} + +func newRandReader(t *testing.T) io.Reader { + seed := time.Now().UnixNano() + t.Logf("Deterministic RNG seed: 0x%x", seed) + return rand.New(rand.NewSource(seed)) +} diff --git a/src/internal/testlog/log.go b/src/internal/testlog/log.go index 3c5f780ac4d62b..d8b9dcfafe3d88 100644 --- a/src/internal/testlog/log.go +++ b/src/internal/testlog/log.go @@ -21,20 +21,19 @@ type Interface interface { } // logger is the current logger Interface. -// We use an atomic.Value in case test startup +// We use an atomic.Pointer in case test startup // is racing with goroutines started during init. // That must not cause a race detector failure, // although it will still result in limited visibility // into exactly what those goroutines do. -var logger atomic.Value +var logger atomic.Pointer[Interface] // SetLogger sets the test logger implementation for the current process. // It must be called only once, at process startup. func SetLogger(impl Interface) { - if logger.Load() != nil { + if !logger.CompareAndSwap(nil, &impl) { panic("testlog: SetLogger must be called only once") } - logger.Store(&impl) } // Logger returns the current test logger implementation. @@ -44,7 +43,7 @@ func Logger() Interface { if impl == nil { return nil } - return *impl.(*Interface) + return *impl } // Getenv calls Logger().Getenv, if a logger has been set. diff --git a/src/internal/trace/base.go b/src/internal/trace/base.go index 4f4ce486305434..1f17daa5f5349c 100644 --- a/src/internal/trace/base.go +++ b/src/internal/trace/base.go @@ -12,23 +12,18 @@ import ( "math" "strings" - "internal/trace/event" - "internal/trace/event/go122" + "internal/trace/tracev2" "internal/trace/version" ) -// maxArgs is the maximum number of arguments for "plain" events, -// i.e. anything that could reasonably be represented as a baseEvent. -const maxArgs = 5 - // timedEventArgs is an array that is able to hold the arguments for any // timed event. -type timedEventArgs [maxArgs - 1]uint64 +type timedEventArgs [tracev2.MaxTimedEventArgs - 1]uint64 // baseEvent is the basic unprocessed event. This serves as a common // fundamental data structure across. type baseEvent struct { - typ event.Type + typ tracev2.EventType time Time args timedEventArgs } @@ -38,7 +33,7 @@ type baseEvent struct { func (e *baseEvent) extra(v version.Version) []uint64 { switch v { case version.Go122: - return e.args[len(go122.Specs()[e.typ].Args)-1:] + return e.args[len(tracev2.Specs()[e.typ].Args)-1:] } panic(fmt.Sprintf("unsupported version: go 1.%d", v)) } @@ -46,7 +41,7 @@ func (e *baseEvent) extra(v version.Version) []uint64 { // evTable contains the per-generation data necessary to // interpret an individual event. type evTable struct { - freq frequency + sync strings dataTable[stringID, string] stacks dataTable[stackID, stack] pcs map[uint64]frame @@ -58,9 +53,8 @@ type evTable struct { extraStringIDs map[string]extraStringID nextExtra extraStringID - // expData contains extra unparsed data that is accessible - // only to ExperimentEvent via an EventExperimental event. - expData map[event.Experiment]*ExperimentalData + // expBatches contains extra unparsed data relevant to a specific experiment. + expBatches map[tracev2.Experiment][]ExperimentalBatch } // addExtraString adds an extra string to the evTable and returns @@ -240,7 +234,7 @@ func (s cpuSample) asEvent(table *evTable) Event { table: table, ctx: s.schedCtx, base: baseEvent{ - typ: go122.EvCPUSample, + typ: tracev2.EvCPUSample, time: s.time, }, } diff --git a/src/internal/trace/batch.go b/src/internal/trace/batch.go index 2b47c9b72cb8d9..1f50350273d3e7 100644 --- a/src/internal/trace/batch.go +++ b/src/internal/trace/batch.go @@ -10,8 +10,8 @@ import ( "fmt" "io" - "internal/trace/event" - "internal/trace/event/go122" + "internal/trace/tracev2" + "internal/trace/version" ) // timestamp is an unprocessed timestamp. @@ -23,23 +23,29 @@ type batch struct { m ThreadID time timestamp data []byte - exp event.Experiment + exp tracev2.Experiment } func (b *batch) isStringsBatch() bool { - return b.exp == event.NoExperiment && len(b.data) > 0 && event.Type(b.data[0]) == go122.EvStrings + return b.exp == tracev2.NoExperiment && len(b.data) > 0 && tracev2.EventType(b.data[0]) == tracev2.EvStrings } func (b *batch) isStacksBatch() bool { - return b.exp == event.NoExperiment && len(b.data) > 0 && event.Type(b.data[0]) == go122.EvStacks + return b.exp == tracev2.NoExperiment && len(b.data) > 0 && tracev2.EventType(b.data[0]) == tracev2.EvStacks } func (b *batch) isCPUSamplesBatch() bool { - return b.exp == event.NoExperiment && len(b.data) > 0 && event.Type(b.data[0]) == go122.EvCPUSamples + return b.exp == tracev2.NoExperiment && len(b.data) > 0 && tracev2.EventType(b.data[0]) == tracev2.EvCPUSamples } -func (b *batch) isFreqBatch() bool { - return b.exp == event.NoExperiment && len(b.data) > 0 && event.Type(b.data[0]) == go122.EvFrequency +func (b *batch) isSyncBatch(ver version.Version) bool { + return (b.exp == tracev2.NoExperiment && len(b.data) > 0) && + ((tracev2.EventType(b.data[0]) == tracev2.EvFrequency && ver < version.Go125) || + (tracev2.EventType(b.data[0]) == tracev2.EvSync && ver >= version.Go125)) +} + +func (b *batch) isEndOfGeneration() bool { + return b.exp == tracev2.NoExperiment && len(b.data) > 0 && tracev2.EventType(b.data[0]) == tracev2.EvEndOfGeneration } // readBatch reads the next full batch from r. @@ -52,18 +58,21 @@ func readBatch(r interface { if err != nil { return batch{}, 0, err } - if typ := event.Type(b); typ != go122.EvEventBatch && typ != go122.EvExperimentalBatch { - return batch{}, 0, fmt.Errorf("expected batch event, got %s", go122.EventString(typ)) + if typ := tracev2.EventType(b); typ == tracev2.EvEndOfGeneration { + return batch{m: NoThread, exp: tracev2.NoExperiment, data: []byte{b}}, 0, nil + } + if typ := tracev2.EventType(b); typ != tracev2.EvEventBatch && typ != tracev2.EvExperimentalBatch { + return batch{}, 0, fmt.Errorf("expected batch event, got event %d", typ) } // Read the experiment of we have one. - exp := event.NoExperiment - if event.Type(b) == go122.EvExperimentalBatch { + exp := tracev2.NoExperiment + if tracev2.EventType(b) == tracev2.EvExperimentalBatch { e, err := r.ReadByte() if err != nil { return batch{}, 0, err } - exp = event.Experiment(e) + exp = tracev2.Experiment(e) } // Read the batch header: gen (generation), thread (M) ID, base timestamp @@ -86,8 +95,8 @@ func readBatch(r interface { if err != nil { return batch{}, gen, fmt.Errorf("error reading batch size: %w", err) } - if size > go122.MaxBatchSize { - return batch{}, gen, fmt.Errorf("invalid batch size %d, maximum is %d", size, go122.MaxBatchSize) + if size > tracev2.MaxBatchSize { + return batch{}, gen, fmt.Errorf("invalid batch size %d, maximum is %d", size, tracev2.MaxBatchSize) } // Copy out the batch for later processing. diff --git a/src/internal/trace/batchcursor.go b/src/internal/trace/batchcursor.go index 66d297ee33a486..8582f30bb06e88 100644 --- a/src/internal/trace/batchcursor.go +++ b/src/internal/trace/batchcursor.go @@ -9,8 +9,7 @@ import ( "encoding/binary" "fmt" - "internal/trace/event" - "internal/trace/event/go122" + "internal/trace/tracev2" ) type batchCursor struct { @@ -66,8 +65,8 @@ func (b *batchCursor) compare(a *batchCursor) int { // be the case for every event in a plain EventBatch. func readTimedBaseEvent(b []byte, e *baseEvent) (int, timestamp, error) { // Get the event type. - typ := event.Type(b[0]) - specs := go122.Specs() + typ := tracev2.EventType(b[0]) + specs := tracev2.Specs() if int(typ) >= len(specs) { return 0, 0, fmt.Errorf("found invalid event type: %v", typ) } diff --git a/src/internal/trace/event.go b/src/internal/trace/event.go index 4c80a7e5ec4fd7..b78e52329469d3 100644 --- a/src/internal/trace/event.go +++ b/src/internal/trace/event.go @@ -6,12 +6,13 @@ package trace import ( "fmt" + "iter" "math" + "strconv" "strings" "time" - "internal/trace/event" - "internal/trace/event/go122" + "internal/trace/tracev2" "internal/trace/version" ) @@ -196,7 +197,7 @@ type Range struct { Scope ResourceID } -// RangeAttributes provides attributes about a completed Range. +// RangeAttribute provides attributes about a completed Range. type RangeAttribute struct { // Name is the human-readable name for the range. Name string @@ -265,24 +266,25 @@ type Stack struct { } // Frames is an iterator over the frames in a Stack. -func (s Stack) Frames(yield func(f StackFrame) bool) bool { - if s.id == 0 { - return true - } - stk := s.table.stacks.mustGet(s.id) - for _, pc := range stk.pcs { - f := s.table.pcs[pc] - sf := StackFrame{ - PC: f.pc, - Func: s.table.strings.mustGet(f.funcID), - File: s.table.strings.mustGet(f.fileID), - Line: f.line, +func (s Stack) Frames() iter.Seq[StackFrame] { + return func(yield func(StackFrame) bool) { + if s.id == 0 { + return } - if !yield(sf) { - return false + stk := s.table.stacks.mustGet(s.id) + for _, pc := range stk.pcs { + f := s.table.pcs[pc] + sf := StackFrame{ + PC: f.pc, + Func: s.table.strings.mustGet(f.funcID), + File: s.table.strings.mustGet(f.fileID), + Line: f.line, + } + if !yield(sf) { + return + } } } - return true } // NoStack is a sentinel value that can be compared against any Stack value, indicating @@ -306,31 +308,33 @@ type StackFrame struct { Line uint64 } -// ExperimentalEvent presents a raw view of an experimental event's arguments and thier names. +// ExperimentalEvent presents a raw view of an experimental event's arguments and their names. type ExperimentalEvent struct { // Name is the name of the event. Name string - // ArgNames is the names of the event's arguments in order. - // This may refer to a globally shared slice. Copy before mutating. - ArgNames []string + // Experiment is the name of the experiment this event is a part of. + Experiment string - // Args contains the event's arguments. - Args []uint64 + // Args lists the names of the event's arguments in order. + Args []string - // Data is additional unparsed data that is associated with the experimental event. - // Data is likely to be shared across many ExperimentalEvents, so callers that parse - // Data are encouraged to cache the parse result and look it up by the value of Data. - Data *ExperimentalData + // argValues contains the raw integer arguments which are interpreted + // by ArgValue using table. + table *evTable + argValues []uint64 } -// ExperimentalData represents some raw and unparsed sidecar data present in the trace that is -// associated with certain kinds of experimental events. For example, this data may contain -// tables needed to interpret ExperimentalEvent arguments, or the ExperimentEvent could just be -// a placeholder for a differently encoded event that's actually present in the experimental data. -type ExperimentalData struct { - // Batches contain the actual experimental data, along with metadata about each batch. - Batches []ExperimentalBatch +// ArgValue returns a typed Value for the i'th argument in the experimental event. +func (e ExperimentalEvent) ArgValue(i int) Value { + if i < 0 || i >= len(e.Args) { + panic(fmt.Sprintf("experimental event argument index %d out of bounds [0, %d)", i, len(e.Args))) + } + if strings.HasSuffix(e.Args[i], "string") { + s := e.table.strings.mustGet(stringID(e.argValues[i])) + return stringValue(s) + } + return uint64Value(e.argValues[i]) } // ExperimentalBatch represents a packet of unparsed data along with metadata about that packet. @@ -351,7 +355,7 @@ type Event struct { // Kind returns the kind of event that this is. func (e Event) Kind() EventKind { - return go122Type2Kind[e.base.typ] + return tracev2Type2Kind[e.base.typ] } // Time returns the timestamp of the event. @@ -403,10 +407,10 @@ func (e Event) Stack() Stack { if e.base.typ == evSync { return NoStack } - if e.base.typ == go122.EvCPUSample { + if e.base.typ == tracev2.EvCPUSample { return Stack{table: e.table, id: stackID(e.base.args[0])} } - spec := go122.Specs()[e.base.typ] + spec := tracev2.Specs()[e.base.typ] if len(spec.StackIDs) == 0 { return NoStack } @@ -429,17 +433,17 @@ func (e Event) Metric() Metric { } var m Metric switch e.base.typ { - case go122.EvProcsChange: + case tracev2.EvProcsChange: m.Name = "/sched/gomaxprocs:threads" - m.Value = Value{kind: ValueUint64, scalar: e.base.args[0]} - case go122.EvHeapAlloc: + m.Value = uint64Value(e.base.args[0]) + case tracev2.EvHeapAlloc: m.Name = "/memory/classes/heap/objects:bytes" - m.Value = Value{kind: ValueUint64, scalar: e.base.args[0]} - case go122.EvHeapGoal: + m.Value = uint64Value(e.base.args[0]) + case tracev2.EvHeapGoal: m.Name = "/gc/heap/goal:bytes" - m.Value = Value{kind: ValueUint64, scalar: e.base.args[0]} + m.Value = uint64Value(e.base.args[0]) default: - panic(fmt.Sprintf("internal error: unexpected event type for Metric kind: %s", go122.EventString(e.base.typ))) + panic(fmt.Sprintf("internal error: unexpected wire-format event type for Metric kind: %d", e.base.typ)) } return m } @@ -451,8 +455,8 @@ func (e Event) Label() Label { if e.Kind() != EventLabel { panic("Label called on non-Label event") } - if e.base.typ != go122.EvGoLabel { - panic(fmt.Sprintf("internal error: unexpected event type for Label kind: %s", go122.EventString(e.base.typ))) + if e.base.typ != tracev2.EvGoLabel { + panic(fmt.Sprintf("internal error: unexpected wire-format event type for Label kind: %d", e.base.typ)) } return Label{ Label: e.table.strings.mustGet(stringID(e.base.args[0])), @@ -469,33 +473,32 @@ func (e Event) Range() Range { } var r Range switch e.base.typ { - case go122.EvSTWBegin, go122.EvSTWEnd: + case tracev2.EvSTWBegin, tracev2.EvSTWEnd: // N.B. ordering.advance smuggles in the STW reason as e.base.args[0] - // for go122.EvSTWEnd (it's already there for Begin). + // for tracev2.EvSTWEnd (it's already there for Begin). r.Name = "stop-the-world (" + e.table.strings.mustGet(stringID(e.base.args[0])) + ")" r.Scope = ResourceID{Kind: ResourceGoroutine, id: int64(e.Goroutine())} - case go122.EvGCBegin, go122.EvGCActive, go122.EvGCEnd: + case tracev2.EvGCBegin, tracev2.EvGCActive, tracev2.EvGCEnd: r.Name = "GC concurrent mark phase" r.Scope = ResourceID{Kind: ResourceNone} - case go122.EvGCSweepBegin, go122.EvGCSweepActive, go122.EvGCSweepEnd: + case tracev2.EvGCSweepBegin, tracev2.EvGCSweepActive, tracev2.EvGCSweepEnd: r.Name = "GC incremental sweep" r.Scope = ResourceID{Kind: ResourceProc} - if e.base.typ == go122.EvGCSweepActive { + if e.base.typ == tracev2.EvGCSweepActive { r.Scope.id = int64(e.base.args[0]) } else { r.Scope.id = int64(e.Proc()) } - r.Scope.id = int64(e.Proc()) - case go122.EvGCMarkAssistBegin, go122.EvGCMarkAssistActive, go122.EvGCMarkAssistEnd: + case tracev2.EvGCMarkAssistBegin, tracev2.EvGCMarkAssistActive, tracev2.EvGCMarkAssistEnd: r.Name = "GC mark assist" r.Scope = ResourceID{Kind: ResourceGoroutine} - if e.base.typ == go122.EvGCMarkAssistActive { + if e.base.typ == tracev2.EvGCMarkAssistActive { r.Scope.id = int64(e.base.args[0]) } else { r.Scope.id = int64(e.Goroutine()) } default: - panic(fmt.Sprintf("internal error: unexpected event type for Range kind: %s", go122.EventString(e.base.typ))) + panic(fmt.Sprintf("internal error: unexpected wire-event type for Range kind: %d", e.base.typ)) } return r } @@ -507,17 +510,17 @@ func (e Event) RangeAttributes() []RangeAttribute { if e.Kind() != EventRangeEnd { panic("Range called on non-Range event") } - if e.base.typ != go122.EvGCSweepEnd { + if e.base.typ != tracev2.EvGCSweepEnd { return nil } return []RangeAttribute{ { Name: "bytes swept", - Value: Value{kind: ValueUint64, scalar: e.base.args[0]}, + Value: uint64Value(e.base.args[0]), }, { Name: "bytes reclaimed", - Value: Value{kind: ValueUint64, scalar: e.base.args[1]}, + Value: uint64Value(e.base.args[1]), }, } } @@ -532,14 +535,14 @@ func (e Event) Task() Task { parentID := NoTask var typ string switch e.base.typ { - case go122.EvUserTaskBegin: + case tracev2.EvUserTaskBegin: parentID = TaskID(e.base.args[1]) typ = e.table.strings.mustGet(stringID(e.base.args[2])) - case go122.EvUserTaskEnd: + case tracev2.EvUserTaskEnd: parentID = TaskID(e.base.extra(version.Go122)[0]) typ = e.table.getExtraString(extraStringID(e.base.extra(version.Go122)[1])) default: - panic(fmt.Sprintf("internal error: unexpected event type for Task kind: %s", go122.EventString(e.base.typ))) + panic(fmt.Sprintf("internal error: unexpected wire-format event type for Task kind: %d", e.base.typ)) } return Task{ ID: TaskID(e.base.args[0]), @@ -555,8 +558,8 @@ func (e Event) Region() Region { if kind := e.Kind(); kind != EventRegionBegin && kind != EventRegionEnd { panic("Region called on non-Region event") } - if e.base.typ != go122.EvUserRegionBegin && e.base.typ != go122.EvUserRegionEnd { - panic(fmt.Sprintf("internal error: unexpected event type for Region kind: %s", go122.EventString(e.base.typ))) + if e.base.typ != tracev2.EvUserRegionBegin && e.base.typ != tracev2.EvUserRegionEnd { + panic(fmt.Sprintf("internal error: unexpected wire-format event type for Region kind: %d", e.base.typ)) } return Region{ Task: TaskID(e.base.args[0]), @@ -571,8 +574,8 @@ func (e Event) Log() Log { if e.Kind() != EventLog { panic("Log called on non-Log event") } - if e.base.typ != go122.EvUserLog { - panic(fmt.Sprintf("internal error: unexpected event type for Log kind: %s", go122.EventString(e.base.typ))) + if e.base.typ != tracev2.EvUserLog { + panic(fmt.Sprintf("internal error: unexpected wire-format event type for Log kind: %d", e.base.typ)) } return Log{ Task: TaskID(e.base.args[0]), @@ -590,14 +593,14 @@ func (e Event) StateTransition() StateTransition { } var s StateTransition switch e.base.typ { - case go122.EvProcStart: + case tracev2.EvProcStart: s = procStateTransition(ProcID(e.base.args[0]), ProcIdle, ProcRunning) - case go122.EvProcStop: + case tracev2.EvProcStop: s = procStateTransition(e.ctx.P, ProcRunning, ProcIdle) - case go122.EvProcSteal: + case tracev2.EvProcSteal: // N.B. ordering.advance populates e.base.extra. beforeState := ProcRunning - if go122.ProcStatus(e.base.extra(version.Go122)[0]) == go122.ProcSyscallAbandoned { + if tracev2.ProcStatus(e.base.extra(version.Go122)[0]) == tracev2.ProcSyscallAbandoned { // We've lost information because this ProcSteal advanced on a // SyscallAbandoned state. Treat the P as idle because ProcStatus // treats SyscallAbandoned as Idle. Otherwise we'll have an invalid @@ -605,57 +608,110 @@ func (e Event) StateTransition() StateTransition { beforeState = ProcIdle } s = procStateTransition(ProcID(e.base.args[0]), beforeState, ProcIdle) - case go122.EvProcStatus: + case tracev2.EvProcStatus: // N.B. ordering.advance populates e.base.extra. - s = procStateTransition(ProcID(e.base.args[0]), ProcState(e.base.extra(version.Go122)[0]), go122ProcStatus2ProcState[e.base.args[1]]) - case go122.EvGoCreate, go122.EvGoCreateBlocked: + s = procStateTransition(ProcID(e.base.args[0]), ProcState(e.base.extra(version.Go122)[0]), tracev2ProcStatus2ProcState[e.base.args[1]]) + case tracev2.EvGoCreate, tracev2.EvGoCreateBlocked: status := GoRunnable - if e.base.typ == go122.EvGoCreateBlocked { + if e.base.typ == tracev2.EvGoCreateBlocked { status = GoWaiting } s = goStateTransition(GoID(e.base.args[0]), GoNotExist, status) s.Stack = Stack{table: e.table, id: stackID(e.base.args[1])} - case go122.EvGoCreateSyscall: + case tracev2.EvGoCreateSyscall: s = goStateTransition(GoID(e.base.args[0]), GoNotExist, GoSyscall) - case go122.EvGoStart: + case tracev2.EvGoStart: s = goStateTransition(GoID(e.base.args[0]), GoRunnable, GoRunning) - case go122.EvGoDestroy: + case tracev2.EvGoDestroy: s = goStateTransition(e.ctx.G, GoRunning, GoNotExist) - s.Stack = e.Stack() // This event references the resource the event happened on. - case go122.EvGoDestroySyscall: + case tracev2.EvGoDestroySyscall: s = goStateTransition(e.ctx.G, GoSyscall, GoNotExist) - case go122.EvGoStop: + case tracev2.EvGoStop: s = goStateTransition(e.ctx.G, GoRunning, GoRunnable) s.Reason = e.table.strings.mustGet(stringID(e.base.args[0])) s.Stack = e.Stack() // This event references the resource the event happened on. - case go122.EvGoBlock: + case tracev2.EvGoBlock: s = goStateTransition(e.ctx.G, GoRunning, GoWaiting) s.Reason = e.table.strings.mustGet(stringID(e.base.args[0])) s.Stack = e.Stack() // This event references the resource the event happened on. - case go122.EvGoUnblock, go122.EvGoSwitch, go122.EvGoSwitchDestroy: + case tracev2.EvGoUnblock, tracev2.EvGoSwitch, tracev2.EvGoSwitchDestroy: // N.B. GoSwitch and GoSwitchDestroy both emit additional events, but // the first thing they both do is unblock the goroutine they name, // identically to an unblock event (even their arguments match). s = goStateTransition(GoID(e.base.args[0]), GoWaiting, GoRunnable) - case go122.EvGoSyscallBegin: + case tracev2.EvGoSyscallBegin: s = goStateTransition(e.ctx.G, GoRunning, GoSyscall) s.Stack = e.Stack() // This event references the resource the event happened on. - case go122.EvGoSyscallEnd: + case tracev2.EvGoSyscallEnd: s = goStateTransition(e.ctx.G, GoSyscall, GoRunning) - s.Stack = e.Stack() // This event references the resource the event happened on. - case go122.EvGoSyscallEndBlocked: + case tracev2.EvGoSyscallEndBlocked: s = goStateTransition(e.ctx.G, GoSyscall, GoRunnable) - s.Stack = e.Stack() // This event references the resource the event happened on. - case go122.EvGoStatus, go122.EvGoStatusStack: + case tracev2.EvGoStatus, tracev2.EvGoStatusStack: packedStatus := e.base.args[2] from, to := packedStatus>>32, packedStatus&((1<<32)-1) - s = goStateTransition(GoID(e.base.args[0]), GoState(from), go122GoStatus2GoState[to]) + s = goStateTransition(GoID(e.base.args[0]), GoState(from), tracev2GoStatus2GoState[to]) default: - panic(fmt.Sprintf("internal error: unexpected event type for StateTransition kind: %s", go122.EventString(e.base.typ))) + panic(fmt.Sprintf("internal error: unexpected wire-format event type for StateTransition kind: %d", e.base.typ)) + } + return s +} + +// Sync returns details that are relevant for the following events, up to but excluding the +// next EventSync event. +func (e Event) Sync() Sync { + if e.Kind() != EventSync { + panic("Sync called on non-Sync event") + } + s := Sync{N: int(e.base.args[0])} + if e.table != nil { + expBatches := make(map[string][]ExperimentalBatch) + for exp, batches := range e.table.expBatches { + expBatches[tracev2.Experiments()[exp]] = batches + } + s.ExperimentalBatches = expBatches + if e.table.hasClockSnapshot { + s.ClockSnapshot = &ClockSnapshot{ + Trace: e.table.freq.mul(e.table.snapTime), + Wall: e.table.snapWall, + Mono: e.table.snapMono, + } + } } return s } +// Sync contains details potentially relevant to all the following events, up to but excluding +// the next EventSync event. +type Sync struct { + // N indicates that this is the Nth sync event in the trace. + N int + + // ClockSnapshot represents a near-simultaneous clock reading of several + // different system clocks. The snapshot can be used as a reference to + // convert timestamps to different clocks, which is helpful for correlating + // timestamps with data captured by other tools. The value is nil for traces + // before go1.25. + ClockSnapshot *ClockSnapshot + + // ExperimentalBatches contain all the unparsed batches of data for a given experiment. + ExperimentalBatches map[string][]ExperimentalBatch +} + +// ClockSnapshot represents a near-simultaneous clock reading of several +// different system clocks. The snapshot can be used as a reference to convert +// timestamps to different clocks, which is helpful for correlating timestamps +// with data captured by other tools. +type ClockSnapshot struct { + // Trace is a snapshot of the trace clock. + Trace Time + + // Wall is a snapshot of the system's wall clock. + Wall time.Time + + // Mono is a snapshot of the system's monotonic clock. + Mono uint64 +} + // Experimental returns a view of the raw event for an experimental event. // // Panics if Kind != EventExperimental. @@ -663,84 +719,85 @@ func (e Event) Experimental() ExperimentalEvent { if e.Kind() != EventExperimental { panic("Experimental called on non-Experimental event") } - spec := go122.Specs()[e.base.typ] + spec := tracev2.Specs()[e.base.typ] argNames := spec.Args[1:] // Skip timestamp; already handled. return ExperimentalEvent{ - Name: spec.Name, - ArgNames: argNames, - Args: e.base.args[:len(argNames)], - Data: e.table.expData[spec.Experiment], - } -} - -const evSync = ^event.Type(0) - -var go122Type2Kind = [...]EventKind{ - go122.EvCPUSample: EventStackSample, - go122.EvProcsChange: EventMetric, - go122.EvProcStart: EventStateTransition, - go122.EvProcStop: EventStateTransition, - go122.EvProcSteal: EventStateTransition, - go122.EvProcStatus: EventStateTransition, - go122.EvGoCreate: EventStateTransition, - go122.EvGoCreateSyscall: EventStateTransition, - go122.EvGoStart: EventStateTransition, - go122.EvGoDestroy: EventStateTransition, - go122.EvGoDestroySyscall: EventStateTransition, - go122.EvGoStop: EventStateTransition, - go122.EvGoBlock: EventStateTransition, - go122.EvGoUnblock: EventStateTransition, - go122.EvGoSyscallBegin: EventStateTransition, - go122.EvGoSyscallEnd: EventStateTransition, - go122.EvGoSyscallEndBlocked: EventStateTransition, - go122.EvGoStatus: EventStateTransition, - go122.EvSTWBegin: EventRangeBegin, - go122.EvSTWEnd: EventRangeEnd, - go122.EvGCActive: EventRangeActive, - go122.EvGCBegin: EventRangeBegin, - go122.EvGCEnd: EventRangeEnd, - go122.EvGCSweepActive: EventRangeActive, - go122.EvGCSweepBegin: EventRangeBegin, - go122.EvGCSweepEnd: EventRangeEnd, - go122.EvGCMarkAssistActive: EventRangeActive, - go122.EvGCMarkAssistBegin: EventRangeBegin, - go122.EvGCMarkAssistEnd: EventRangeEnd, - go122.EvHeapAlloc: EventMetric, - go122.EvHeapGoal: EventMetric, - go122.EvGoLabel: EventLabel, - go122.EvUserTaskBegin: EventTaskBegin, - go122.EvUserTaskEnd: EventTaskEnd, - go122.EvUserRegionBegin: EventRegionBegin, - go122.EvUserRegionEnd: EventRegionEnd, - go122.EvUserLog: EventLog, - go122.EvGoSwitch: EventStateTransition, - go122.EvGoSwitchDestroy: EventStateTransition, - go122.EvGoCreateBlocked: EventStateTransition, - go122.EvGoStatusStack: EventStateTransition, - go122.EvSpan: EventExperimental, - go122.EvSpanAlloc: EventExperimental, - go122.EvSpanFree: EventExperimental, - go122.EvHeapObject: EventExperimental, - go122.EvHeapObjectAlloc: EventExperimental, - go122.EvHeapObjectFree: EventExperimental, - go122.EvGoroutineStack: EventExperimental, - go122.EvGoroutineStackAlloc: EventExperimental, - go122.EvGoroutineStackFree: EventExperimental, - evSync: EventSync, -} - -var go122GoStatus2GoState = [...]GoState{ - go122.GoRunnable: GoRunnable, - go122.GoRunning: GoRunning, - go122.GoWaiting: GoWaiting, - go122.GoSyscall: GoSyscall, -} - -var go122ProcStatus2ProcState = [...]ProcState{ - go122.ProcRunning: ProcRunning, - go122.ProcIdle: ProcIdle, - go122.ProcSyscall: ProcRunning, - go122.ProcSyscallAbandoned: ProcIdle, + Name: spec.Name, + Experiment: tracev2.Experiments()[spec.Experiment], + Args: argNames, + table: e.table, + argValues: e.base.args[:len(argNames)], + } +} + +const evSync = ^tracev2.EventType(0) + +var tracev2Type2Kind = [...]EventKind{ + tracev2.EvCPUSample: EventStackSample, + tracev2.EvProcsChange: EventMetric, + tracev2.EvProcStart: EventStateTransition, + tracev2.EvProcStop: EventStateTransition, + tracev2.EvProcSteal: EventStateTransition, + tracev2.EvProcStatus: EventStateTransition, + tracev2.EvGoCreate: EventStateTransition, + tracev2.EvGoCreateSyscall: EventStateTransition, + tracev2.EvGoStart: EventStateTransition, + tracev2.EvGoDestroy: EventStateTransition, + tracev2.EvGoDestroySyscall: EventStateTransition, + tracev2.EvGoStop: EventStateTransition, + tracev2.EvGoBlock: EventStateTransition, + tracev2.EvGoUnblock: EventStateTransition, + tracev2.EvGoSyscallBegin: EventStateTransition, + tracev2.EvGoSyscallEnd: EventStateTransition, + tracev2.EvGoSyscallEndBlocked: EventStateTransition, + tracev2.EvGoStatus: EventStateTransition, + tracev2.EvSTWBegin: EventRangeBegin, + tracev2.EvSTWEnd: EventRangeEnd, + tracev2.EvGCActive: EventRangeActive, + tracev2.EvGCBegin: EventRangeBegin, + tracev2.EvGCEnd: EventRangeEnd, + tracev2.EvGCSweepActive: EventRangeActive, + tracev2.EvGCSweepBegin: EventRangeBegin, + tracev2.EvGCSweepEnd: EventRangeEnd, + tracev2.EvGCMarkAssistActive: EventRangeActive, + tracev2.EvGCMarkAssistBegin: EventRangeBegin, + tracev2.EvGCMarkAssistEnd: EventRangeEnd, + tracev2.EvHeapAlloc: EventMetric, + tracev2.EvHeapGoal: EventMetric, + tracev2.EvGoLabel: EventLabel, + tracev2.EvUserTaskBegin: EventTaskBegin, + tracev2.EvUserTaskEnd: EventTaskEnd, + tracev2.EvUserRegionBegin: EventRegionBegin, + tracev2.EvUserRegionEnd: EventRegionEnd, + tracev2.EvUserLog: EventLog, + tracev2.EvGoSwitch: EventStateTransition, + tracev2.EvGoSwitchDestroy: EventStateTransition, + tracev2.EvGoCreateBlocked: EventStateTransition, + tracev2.EvGoStatusStack: EventStateTransition, + tracev2.EvSpan: EventExperimental, + tracev2.EvSpanAlloc: EventExperimental, + tracev2.EvSpanFree: EventExperimental, + tracev2.EvHeapObject: EventExperimental, + tracev2.EvHeapObjectAlloc: EventExperimental, + tracev2.EvHeapObjectFree: EventExperimental, + tracev2.EvGoroutineStack: EventExperimental, + tracev2.EvGoroutineStackAlloc: EventExperimental, + tracev2.EvGoroutineStackFree: EventExperimental, + evSync: EventSync, +} + +var tracev2GoStatus2GoState = [...]GoState{ + tracev2.GoRunnable: GoRunnable, + tracev2.GoRunning: GoRunning, + tracev2.GoWaiting: GoWaiting, + tracev2.GoSyscall: GoSyscall, +} + +var tracev2ProcStatus2ProcState = [...]ProcState{ + tracev2.ProcRunning: ProcRunning, + tracev2.ProcIdle: ProcIdle, + tracev2.ProcSyscall: ProcRunning, + tracev2.ProcSyscallAbandoned: ProcIdle, } // String returns the event as a human-readable string. @@ -754,7 +811,11 @@ func (e Event) String() string { switch kind := e.Kind(); kind { case EventMetric: m := e.Metric() - fmt.Fprintf(&sb, " Name=%q Value=%s", m.Name, valueAsString(m.Value)) + v := m.Value.String() + if m.Value.Kind() == ValueString { + v = strconv.Quote(v) + } + fmt.Fprintf(&sb, " Name=%q Value=%s", m.Name, m.Value) case EventLabel: l := e.Label() fmt.Fprintf(&sb, " Label=%q Resource=%s", l.Label, l.Resource) @@ -767,7 +828,7 @@ func (e Event) String() string { if i != 0 { fmt.Fprintf(&sb, " ") } - fmt.Fprintf(&sb, "%q=%s", attr.Name, valueAsString(attr.Value)) + fmt.Fprintf(&sb, "%q=%s", attr.Name, attr.Value) } fmt.Fprintf(&sb, "]") } @@ -782,7 +843,6 @@ func (e Event) String() string { fmt.Fprintf(&sb, " Task=%d Category=%q Message=%q", l.Task, l.Category, l.Message) case EventStateTransition: s := e.StateTransition() - fmt.Fprintf(&sb, " Resource=%s Reason=%q", s.Resource, s.Reason) switch s.Resource.Kind { case ResourceGoroutine: id := s.Resource.Goroutine() @@ -793,27 +853,43 @@ func (e Event) String() string { old, new := s.Proc() fmt.Fprintf(&sb, " ProcID=%d %s->%s", id, old, new) } + fmt.Fprintf(&sb, " Reason=%q", s.Reason) if s.Stack != NoStack { fmt.Fprintln(&sb) fmt.Fprintln(&sb, "TransitionStack=") - s.Stack.Frames(func(f StackFrame) bool { + for f := range s.Stack.Frames() { fmt.Fprintf(&sb, "\t%s @ 0x%x\n", f.Func, f.PC) fmt.Fprintf(&sb, "\t\t%s:%d\n", f.File, f.Line) - return true - }) + } } case EventExperimental: r := e.Experimental() - fmt.Fprintf(&sb, " Name=%s ArgNames=%v Args=%v", r.Name, r.ArgNames, r.Args) + fmt.Fprintf(&sb, " Name=%s Args=[", r.Name) + for i, arg := range r.Args { + if i != 0 { + fmt.Fprintf(&sb, ", ") + } + fmt.Fprintf(&sb, "%s=%s", arg, r.ArgValue(i).String()) + } + fmt.Fprintf(&sb, "]") + case EventSync: + s := e.Sync() + fmt.Fprintf(&sb, " N=%d", s.N) + if s.ClockSnapshot != nil { + fmt.Fprintf(&sb, " Trace=%d Mono=%d Wall=%s", + s.ClockSnapshot.Trace, + s.ClockSnapshot.Mono, + s.ClockSnapshot.Wall.Format(time.RFC3339Nano), + ) + } } if stk := e.Stack(); stk != NoStack { fmt.Fprintln(&sb) fmt.Fprintln(&sb, "Stack=") - stk.Frames(func(f StackFrame) bool { + for f := range stk.Frames() { fmt.Fprintf(&sb, "\t%s @ 0x%x\n", f.Func, f.PC) fmt.Fprintf(&sb, "\t\t%s:%d\n", f.File, f.Line) - return true - }) + } } return sb.String() } @@ -824,7 +900,7 @@ func (e Event) validateTableIDs() error { if e.base.typ == evSync { return nil } - spec := go122.Specs()[e.base.typ] + spec := tracev2.Specs()[e.base.typ] // Check stacks. for _, i := range spec.StackIDs { @@ -848,8 +924,8 @@ func (e Event) validateTableIDs() error { return nil } -func syncEvent(table *evTable, ts Time) Event { - return Event{ +func syncEvent(table *evTable, ts Time, n int) Event { + ev := Event{ table: table, ctx: schedCtx{ G: NoGoroutine, @@ -861,4 +937,6 @@ func syncEvent(table *evTable, ts Time) Event { time: ts, }, } + ev.base.args[0] = uint64(n) + return ev } diff --git a/src/internal/trace/event/event.go b/src/internal/trace/event/event.go deleted file mode 100644 index 9a9bf74c6afe59..00000000000000 --- a/src/internal/trace/event/event.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package event - -// Type indicates an event's type from which its arguments and semantics can be -// derived. Its representation matches the wire format's representation of the event -// types that precede all event data. -type Type uint8 - -// Spec is a specification for a trace event. It contains sufficient information -// to perform basic parsing of any trace event for any version of Go. -type Spec struct { - // Name is the human-readable name of the trace event. - Name string - - // Args contains the names of each trace event's argument. - // Its length determines the number of arguments an event has. - // - // Argument names follow a certain structure and this structure - // is relied on by the testing framework to type-check arguments. - // The structure is is: - // - // (?P[A-Za-z]+_)?(?P[A-Za-z]+) - // - // In sum, it's an optional name followed by a type. If the name - // is present, it is separated from the type with an underscore. - // The valid argument types and the Go types they map to are listed - // in the ArgTypes variable. - Args []string - - // StringIDs indicates which of the arguments are string IDs. - StringIDs []int - - // StackIDs indicates which of the arguments are stack IDs. - // - // The list is not sorted. The first index always refers to - // the main stack for the current execution context of the event. - StackIDs []int - - // StartEv indicates the event type of the corresponding "start" - // event, if this event is an "end," for a pair of events that - // represent a time range. - StartEv Type - - // IsTimedEvent indicates whether this is an event that both - // appears in the main event stream and is surfaced to the - // trace reader. - // - // Events that are not "timed" are considered "structural" - // since they either need significant reinterpretation or - // otherwise aren't actually surfaced by the trace reader. - IsTimedEvent bool - - // HasData is true if the event has trailer consisting of a - // varint length followed by unencoded bytes of some data. - // - // An event may not be both a timed event and have data. - HasData bool - - // IsStack indicates that the event represents a complete - // stack trace. Specifically, it means that after the arguments - // there's a varint length, followed by 4*length varints. Each - // group of 4 represents the PC, file ID, func ID, and line number - // in that order. - IsStack bool - - // Experiment indicates the ID of an experiment this event is associated - // with. If Experiment is not NoExperiment, then the event is experimental - // and will be exposed as an EventExperiment. - Experiment Experiment -} - -// ArgTypes is a list of valid argument types for use in Args. -// -// See the documentation of Args for more details. -var ArgTypes = [...]string{ - "seq", // sequence number - "pstatus", // P status - "gstatus", // G status - "g", // trace.GoID - "m", // trace.ThreadID - "p", // trace.ProcID - "string", // string ID - "stack", // stack ID - "value", // uint64 - "task", // trace.TaskID -} - -// Names is a helper that produces a mapping of event names to event types. -func Names(specs []Spec) map[string]Type { - nameToType := make(map[string]Type) - for i, spec := range specs { - nameToType[spec.Name] = Type(byte(i)) - } - return nameToType -} - -// Experiment is an experiment ID that events may be associated with. -type Experiment uint - -// NoExperiment is the reserved ID 0 indicating no experiment. -const NoExperiment Experiment = 0 diff --git a/src/internal/trace/event/go122/event.go b/src/internal/trace/event/go122/event.go deleted file mode 100644 index f6075e3ed51413..00000000000000 --- a/src/internal/trace/event/go122/event.go +++ /dev/null @@ -1,511 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package go122 - -import ( - "fmt" - "internal/trace/event" -) - -const ( - EvNone event.Type = iota // unused - - // Structural events. - EvEventBatch // start of per-M batch of events [generation, M ID, timestamp, batch length] - EvStacks // start of a section of the stack table [...EvStack] - EvStack // stack table entry [ID, ...{PC, func string ID, file string ID, line #}] - EvStrings // start of a section of the string dictionary [...EvString] - EvString // string dictionary entry [ID, length, string] - EvCPUSamples // start of a section of CPU samples [...EvCPUSample] - EvCPUSample // CPU profiling sample [timestamp, M ID, P ID, goroutine ID, stack ID] - EvFrequency // timestamp units per sec [freq] - - // Procs. - EvProcsChange // current value of GOMAXPROCS [timestamp, GOMAXPROCS, stack ID] - EvProcStart // start of P [timestamp, P ID, P seq] - EvProcStop // stop of P [timestamp] - EvProcSteal // P was stolen [timestamp, P ID, P seq, M ID] - EvProcStatus // P status at the start of a generation [timestamp, P ID, status] - - // Goroutines. - EvGoCreate // goroutine creation [timestamp, new goroutine ID, new stack ID, stack ID] - EvGoCreateSyscall // goroutine appears in syscall (cgo callback) [timestamp, new goroutine ID] - EvGoStart // goroutine starts running [timestamp, goroutine ID, goroutine seq] - EvGoDestroy // goroutine ends [timestamp] - EvGoDestroySyscall // goroutine ends in syscall (cgo callback) [timestamp] - EvGoStop // goroutine yields its time, but is runnable [timestamp, reason, stack ID] - EvGoBlock // goroutine blocks [timestamp, reason, stack ID] - EvGoUnblock // goroutine is unblocked [timestamp, goroutine ID, goroutine seq, stack ID] - EvGoSyscallBegin // syscall enter [timestamp, P seq, stack ID] - EvGoSyscallEnd // syscall exit [timestamp] - EvGoSyscallEndBlocked // syscall exit and it blocked at some point [timestamp] - EvGoStatus // goroutine status at the start of a generation [timestamp, goroutine ID, thread ID, status] - - // STW. - EvSTWBegin // STW start [timestamp, kind] - EvSTWEnd // STW done [timestamp] - - // GC events. - EvGCActive // GC active [timestamp, seq] - EvGCBegin // GC start [timestamp, seq, stack ID] - EvGCEnd // GC done [timestamp, seq] - EvGCSweepActive // GC sweep active [timestamp, P ID] - EvGCSweepBegin // GC sweep start [timestamp, stack ID] - EvGCSweepEnd // GC sweep done [timestamp, swept bytes, reclaimed bytes] - EvGCMarkAssistActive // GC mark assist active [timestamp, goroutine ID] - EvGCMarkAssistBegin // GC mark assist start [timestamp, stack ID] - EvGCMarkAssistEnd // GC mark assist done [timestamp] - EvHeapAlloc // gcController.heapLive change [timestamp, heap alloc in bytes] - EvHeapGoal // gcController.heapGoal() change [timestamp, heap goal in bytes] - - // Annotations. - EvGoLabel // apply string label to current running goroutine [timestamp, label string ID] - EvUserTaskBegin // trace.NewTask [timestamp, internal task ID, internal parent task ID, name string ID, stack ID] - EvUserTaskEnd // end of a task [timestamp, internal task ID, stack ID] - EvUserRegionBegin // trace.{Start,With}Region [timestamp, internal task ID, name string ID, stack ID] - EvUserRegionEnd // trace.{End,With}Region [timestamp, internal task ID, name string ID, stack ID] - EvUserLog // trace.Log [timestamp, internal task ID, key string ID, value string ID, stack] - - // Coroutines. Added in Go 1.23. - EvGoSwitch // goroutine switch (coroswitch) [timestamp, goroutine ID, goroutine seq] - EvGoSwitchDestroy // goroutine switch and destroy [timestamp, goroutine ID, goroutine seq] - EvGoCreateBlocked // goroutine creation (starts blocked) [timestamp, new goroutine ID, new stack ID, stack ID] - - // GoStatus with stack. Added in Go 1.23. - EvGoStatusStack // goroutine status at the start of a generation, with a stack [timestamp, goroutine ID, M ID, status, stack ID] - - // Batch event for an experimental batch with a custom format. Added in Go 1.23. - EvExperimentalBatch // start of extra data [experiment ID, generation, M ID, timestamp, batch length, batch data...] -) - -// Experiments. -const ( - // AllocFree is the alloc-free events experiment. - AllocFree event.Experiment = 1 + iota -) - -// Experimental events. -const ( - _ event.Type = 127 + iota - - // Experimental events for AllocFree. - - // Experimental heap span events. Added in Go 1.23. - EvSpan // heap span exists [timestamp, id, npages, type/class] - EvSpanAlloc // heap span alloc [timestamp, id, npages, type/class] - EvSpanFree // heap span free [timestamp, id] - - // Experimental heap object events. Added in Go 1.23. - EvHeapObject // heap object exists [timestamp, id, type] - EvHeapObjectAlloc // heap object alloc [timestamp, id, type] - EvHeapObjectFree // heap object free [timestamp, id] - - // Experimental goroutine stack events. Added in Go 1.23. - EvGoroutineStack // stack exists [timestamp, id, order] - EvGoroutineStackAlloc // stack alloc [timestamp, id, order] - EvGoroutineStackFree // stack free [timestamp, id] -) - -// EventString returns the name of a Go 1.22 event. -func EventString(typ event.Type) string { - if int(typ) < len(specs) { - return specs[typ].Name - } - return fmt.Sprintf("Invalid(%d)", typ) -} - -func Specs() []event.Spec { - return specs[:] -} - -var specs = [...]event.Spec{ - // "Structural" Events. - EvEventBatch: event.Spec{ - Name: "EventBatch", - Args: []string{"gen", "m", "time", "size"}, - }, - EvStacks: event.Spec{ - Name: "Stacks", - }, - EvStack: event.Spec{ - Name: "Stack", - Args: []string{"id", "nframes"}, - IsStack: true, - }, - EvStrings: event.Spec{ - Name: "Strings", - }, - EvString: event.Spec{ - Name: "String", - Args: []string{"id"}, - HasData: true, - }, - EvCPUSamples: event.Spec{ - Name: "CPUSamples", - }, - EvCPUSample: event.Spec{ - Name: "CPUSample", - Args: []string{"time", "m", "p", "g", "stack"}, - // N.B. There's clearly a timestamp here, but these Events - // are special in that they don't appear in the regular - // M streams. - }, - EvFrequency: event.Spec{ - Name: "Frequency", - Args: []string{"freq"}, - }, - EvExperimentalBatch: event.Spec{ - Name: "ExperimentalBatch", - Args: []string{"exp", "gen", "m", "time"}, - HasData: true, // Easier to represent for raw readers. - }, - - // "Timed" Events. - EvProcsChange: event.Spec{ - Name: "ProcsChange", - Args: []string{"dt", "procs_value", "stack"}, - IsTimedEvent: true, - StackIDs: []int{2}, - }, - EvProcStart: event.Spec{ - Name: "ProcStart", - Args: []string{"dt", "p", "p_seq"}, - IsTimedEvent: true, - }, - EvProcStop: event.Spec{ - Name: "ProcStop", - Args: []string{"dt"}, - IsTimedEvent: true, - }, - EvProcSteal: event.Spec{ - Name: "ProcSteal", - Args: []string{"dt", "p", "p_seq", "m"}, - IsTimedEvent: true, - }, - EvProcStatus: event.Spec{ - Name: "ProcStatus", - Args: []string{"dt", "p", "pstatus"}, - IsTimedEvent: true, - }, - EvGoCreate: event.Spec{ - Name: "GoCreate", - Args: []string{"dt", "new_g", "new_stack", "stack"}, - IsTimedEvent: true, - StackIDs: []int{3, 2}, - }, - EvGoCreateSyscall: event.Spec{ - Name: "GoCreateSyscall", - Args: []string{"dt", "new_g"}, - IsTimedEvent: true, - }, - EvGoStart: event.Spec{ - Name: "GoStart", - Args: []string{"dt", "g", "g_seq"}, - IsTimedEvent: true, - }, - EvGoDestroy: event.Spec{ - Name: "GoDestroy", - Args: []string{"dt"}, - IsTimedEvent: true, - }, - EvGoDestroySyscall: event.Spec{ - Name: "GoDestroySyscall", - Args: []string{"dt"}, - IsTimedEvent: true, - }, - EvGoStop: event.Spec{ - Name: "GoStop", - Args: []string{"dt", "reason_string", "stack"}, - IsTimedEvent: true, - StackIDs: []int{2}, - StringIDs: []int{1}, - }, - EvGoBlock: event.Spec{ - Name: "GoBlock", - Args: []string{"dt", "reason_string", "stack"}, - IsTimedEvent: true, - StackIDs: []int{2}, - StringIDs: []int{1}, - }, - EvGoUnblock: event.Spec{ - Name: "GoUnblock", - Args: []string{"dt", "g", "g_seq", "stack"}, - IsTimedEvent: true, - StackIDs: []int{3}, - }, - EvGoSyscallBegin: event.Spec{ - Name: "GoSyscallBegin", - Args: []string{"dt", "p_seq", "stack"}, - IsTimedEvent: true, - StackIDs: []int{2}, - }, - EvGoSyscallEnd: event.Spec{ - Name: "GoSyscallEnd", - Args: []string{"dt"}, - StartEv: EvGoSyscallBegin, - IsTimedEvent: true, - }, - EvGoSyscallEndBlocked: event.Spec{ - Name: "GoSyscallEndBlocked", - Args: []string{"dt"}, - StartEv: EvGoSyscallBegin, - IsTimedEvent: true, - }, - EvGoStatus: event.Spec{ - Name: "GoStatus", - Args: []string{"dt", "g", "m", "gstatus"}, - IsTimedEvent: true, - }, - EvSTWBegin: event.Spec{ - Name: "STWBegin", - Args: []string{"dt", "kind_string", "stack"}, - IsTimedEvent: true, - StackIDs: []int{2}, - StringIDs: []int{1}, - }, - EvSTWEnd: event.Spec{ - Name: "STWEnd", - Args: []string{"dt"}, - StartEv: EvSTWBegin, - IsTimedEvent: true, - }, - EvGCActive: event.Spec{ - Name: "GCActive", - Args: []string{"dt", "gc_seq"}, - IsTimedEvent: true, - StartEv: EvGCBegin, - }, - EvGCBegin: event.Spec{ - Name: "GCBegin", - Args: []string{"dt", "gc_seq", "stack"}, - IsTimedEvent: true, - StackIDs: []int{2}, - }, - EvGCEnd: event.Spec{ - Name: "GCEnd", - Args: []string{"dt", "gc_seq"}, - StartEv: EvGCBegin, - IsTimedEvent: true, - }, - EvGCSweepActive: event.Spec{ - Name: "GCSweepActive", - Args: []string{"dt", "p"}, - StartEv: EvGCSweepBegin, - IsTimedEvent: true, - }, - EvGCSweepBegin: event.Spec{ - Name: "GCSweepBegin", - Args: []string{"dt", "stack"}, - IsTimedEvent: true, - StackIDs: []int{1}, - }, - EvGCSweepEnd: event.Spec{ - Name: "GCSweepEnd", - Args: []string{"dt", "swept_value", "reclaimed_value"}, - StartEv: EvGCSweepBegin, - IsTimedEvent: true, - }, - EvGCMarkAssistActive: event.Spec{ - Name: "GCMarkAssistActive", - Args: []string{"dt", "g"}, - StartEv: EvGCMarkAssistBegin, - IsTimedEvent: true, - }, - EvGCMarkAssistBegin: event.Spec{ - Name: "GCMarkAssistBegin", - Args: []string{"dt", "stack"}, - IsTimedEvent: true, - StackIDs: []int{1}, - }, - EvGCMarkAssistEnd: event.Spec{ - Name: "GCMarkAssistEnd", - Args: []string{"dt"}, - StartEv: EvGCMarkAssistBegin, - IsTimedEvent: true, - }, - EvHeapAlloc: event.Spec{ - Name: "HeapAlloc", - Args: []string{"dt", "heapalloc_value"}, - IsTimedEvent: true, - }, - EvHeapGoal: event.Spec{ - Name: "HeapGoal", - Args: []string{"dt", "heapgoal_value"}, - IsTimedEvent: true, - }, - EvGoLabel: event.Spec{ - Name: "GoLabel", - Args: []string{"dt", "label_string"}, - IsTimedEvent: true, - StringIDs: []int{1}, - }, - EvUserTaskBegin: event.Spec{ - Name: "UserTaskBegin", - Args: []string{"dt", "task", "parent_task", "name_string", "stack"}, - IsTimedEvent: true, - StackIDs: []int{4}, - StringIDs: []int{3}, - }, - EvUserTaskEnd: event.Spec{ - Name: "UserTaskEnd", - Args: []string{"dt", "task", "stack"}, - IsTimedEvent: true, - StackIDs: []int{2}, - }, - EvUserRegionBegin: event.Spec{ - Name: "UserRegionBegin", - Args: []string{"dt", "task", "name_string", "stack"}, - IsTimedEvent: true, - StackIDs: []int{3}, - StringIDs: []int{2}, - }, - EvUserRegionEnd: event.Spec{ - Name: "UserRegionEnd", - Args: []string{"dt", "task", "name_string", "stack"}, - StartEv: EvUserRegionBegin, - IsTimedEvent: true, - StackIDs: []int{3}, - StringIDs: []int{2}, - }, - EvUserLog: event.Spec{ - Name: "UserLog", - Args: []string{"dt", "task", "key_string", "value_string", "stack"}, - IsTimedEvent: true, - StackIDs: []int{4}, - StringIDs: []int{2, 3}, - }, - EvGoSwitch: event.Spec{ - Name: "GoSwitch", - Args: []string{"dt", "g", "g_seq"}, - IsTimedEvent: true, - }, - EvGoSwitchDestroy: event.Spec{ - Name: "GoSwitchDestroy", - Args: []string{"dt", "g", "g_seq"}, - IsTimedEvent: true, - }, - EvGoCreateBlocked: event.Spec{ - Name: "GoCreateBlocked", - Args: []string{"dt", "new_g", "new_stack", "stack"}, - IsTimedEvent: true, - StackIDs: []int{3, 2}, - }, - EvGoStatusStack: event.Spec{ - Name: "GoStatusStack", - Args: []string{"dt", "g", "m", "gstatus", "stack"}, - IsTimedEvent: true, - StackIDs: []int{4}, - }, - - // Experimental events. - - EvSpan: event.Spec{ - Name: "Span", - Args: []string{"dt", "id", "npages_value", "kindclass"}, - IsTimedEvent: true, - Experiment: AllocFree, - }, - EvSpanAlloc: event.Spec{ - Name: "SpanAlloc", - Args: []string{"dt", "id", "npages_value", "kindclass"}, - IsTimedEvent: true, - Experiment: AllocFree, - }, - EvSpanFree: event.Spec{ - Name: "SpanFree", - Args: []string{"dt", "id"}, - IsTimedEvent: true, - Experiment: AllocFree, - }, - EvHeapObject: event.Spec{ - Name: "HeapObject", - Args: []string{"dt", "id", "type"}, - IsTimedEvent: true, - Experiment: AllocFree, - }, - EvHeapObjectAlloc: event.Spec{ - Name: "HeapObjectAlloc", - Args: []string{"dt", "id", "type"}, - IsTimedEvent: true, - Experiment: AllocFree, - }, - EvHeapObjectFree: event.Spec{ - Name: "HeapObjectFree", - Args: []string{"dt", "id"}, - IsTimedEvent: true, - Experiment: AllocFree, - }, - EvGoroutineStack: event.Spec{ - Name: "GoroutineStack", - Args: []string{"dt", "id", "order"}, - IsTimedEvent: true, - Experiment: AllocFree, - }, - EvGoroutineStackAlloc: event.Spec{ - Name: "GoroutineStackAlloc", - Args: []string{"dt", "id", "order"}, - IsTimedEvent: true, - Experiment: AllocFree, - }, - EvGoroutineStackFree: event.Spec{ - Name: "GoroutineStackFree", - Args: []string{"dt", "id"}, - IsTimedEvent: true, - Experiment: AllocFree, - }, -} - -type GoStatus uint8 - -const ( - GoBad GoStatus = iota - GoRunnable - GoRunning - GoSyscall - GoWaiting -) - -func (s GoStatus) String() string { - switch s { - case GoRunnable: - return "Runnable" - case GoRunning: - return "Running" - case GoSyscall: - return "Syscall" - case GoWaiting: - return "Waiting" - } - return "Bad" -} - -type ProcStatus uint8 - -const ( - ProcBad ProcStatus = iota - ProcRunning - ProcIdle - ProcSyscall - ProcSyscallAbandoned -) - -func (s ProcStatus) String() string { - switch s { - case ProcRunning: - return "Running" - case ProcIdle: - return "Idle" - case ProcSyscall: - return "Syscall" - } - return "Bad" -} - -const ( - // Various format-specific constants. - MaxBatchSize = 64 << 10 - MaxFramesPerStack = 128 - MaxStringSize = 1 << 10 -) diff --git a/src/internal/trace/event/requirements.go b/src/internal/trace/event/requirements.go deleted file mode 100644 index c5adf2e0c239d1..00000000000000 --- a/src/internal/trace/event/requirements.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package event - -// SchedReqs is a set of constraints on what the scheduling -// context must look like. -type SchedReqs struct { - Thread Constraint - Proc Constraint - Goroutine Constraint -} - -// Constraint represents a various presence requirements. -type Constraint uint8 - -const ( - MustNotHave Constraint = iota - MayHave - MustHave -) - -// UserGoReqs is a common requirement among events that are running -// or are close to running user code. -var UserGoReqs = SchedReqs{Thread: MustHave, Proc: MustHave, Goroutine: MustHave} diff --git a/src/internal/trace/event_test.go b/src/internal/trace/event_test.go index c81a45185dc475..d39d6b75bd7613 100644 --- a/src/internal/trace/event_test.go +++ b/src/internal/trace/event_test.go @@ -8,7 +8,7 @@ import "testing" func TestPanicEvent(t *testing.T) { // Use a sync event for this because it doesn't have any extra metadata. - ev := syncEvent(nil, 0) + ev := syncEvent(nil, 0, 0) mustPanic(t, func() { _ = ev.Range() diff --git a/src/internal/trace/export_reader_test.go b/src/internal/trace/export_reader_test.go new file mode 100644 index 00000000000000..042c70864cf8a8 --- /dev/null +++ b/src/internal/trace/export_reader_test.go @@ -0,0 +1,12 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package trace + +import "internal/trace/version" + +// GoVersion is the version set in the trace header. +func (r *Reader) GoVersion() version.Version { + return r.version +} diff --git a/src/internal/trace/gc.go b/src/internal/trace/gc.go index bf271ed73baf20..46890e784df24c 100644 --- a/src/internal/trace/gc.go +++ b/src/internal/trace/gc.go @@ -75,7 +75,7 @@ func MutatorUtilizationV2(events []Event, flags UtilFlags) [][]MutatorUtil { states := make(map[GoID]GoState) bgMark := make(map[GoID]bool) procs := []procsCount{} - seenSync := false + nSync := 0 // Helpers. handleSTW := func(r Range) bool { @@ -97,7 +97,7 @@ func MutatorUtilizationV2(events []Event, flags UtilFlags) [][]MutatorUtil { // Process the event. switch ev.Kind() { case EventSync: - seenSync = true + nSync = ev.Sync().N case EventMetric: m := ev.Metric() if m.Name != "/sched/gomaxprocs:threads" { @@ -135,9 +135,9 @@ func MutatorUtilizationV2(events []Event, flags UtilFlags) [][]MutatorUtil { switch ev.Kind() { case EventRangeActive: - if seenSync { - // If we've seen a sync, then we can be sure we're not finding out about - // something late; we have complete information after that point, and these + if nSync > 1 { + // If we've seen a full generation, then we can be sure we're not finding out + // about something late; we have complete information after that point, and these // active events will just be redundant. break } diff --git a/src/internal/trace/gc_test.go b/src/internal/trace/gc_test.go index 39f28cc0f6977e..9c6d0fcb0e56d8 100644 --- a/src/internal/trace/gc_test.go +++ b/src/internal/trace/gc_test.go @@ -118,7 +118,7 @@ func TestMMUTrace(t *testing.T) { } t.Run("V2", func(t *testing.T) { testPath := "testdata/tests/go122-gc-stress.test" - r, _, err := testtrace.ParseFile(testPath) + r, _, _, err := testtrace.ParseFile(testPath) if err != nil { t.Fatalf("malformed test %s: bad trace file: %v", testPath, err) } diff --git a/src/internal/trace/generation.go b/src/internal/trace/generation.go index 98bbf4398569f9..90ceef5f748896 100644 --- a/src/internal/trace/generation.go +++ b/src/internal/trace/generation.go @@ -9,13 +9,15 @@ import ( "bytes" "cmp" "encoding/binary" + "errors" "fmt" "io" "slices" "strings" + "time" - "internal/trace/event" - "internal/trace/event/go122" + "internal/trace/tracev2" + "internal/trace/version" ) // generation contains all the trace data for a single @@ -27,25 +29,106 @@ type generation struct { batches map[ThreadID][]batch batchMs []ThreadID cpuSamples []cpuSample + minTs timestamp *evTable } +// readGeneration buffers and decodes the structural elements of a trace generation +// out of r. +func readGeneration(r *bufio.Reader, ver version.Version) (*generation, error) { + if ver < version.Go126 { + return nil, errors.New("internal error: readGeneration called for <1.26 trace") + } + g := &generation{ + evTable: &evTable{ + pcs: make(map[uint64]frame), + }, + batches: make(map[ThreadID][]batch), + } + + // Read batches one at a time until we either hit the next generation. + for { + b, gen, err := readBatch(r) + if err == io.EOF { + if len(g.batches) != 0 { + return nil, errors.New("incomplete generation found; trace likely truncated") + } + return nil, nil // All done. + } + if err != nil { + return nil, err + } + if g.gen == 0 { + // Initialize gen. + g.gen = gen + } + if b.isEndOfGeneration() { + break + } + if gen == 0 { + // 0 is a sentinel used by the runtime, so we'll never see it. + return nil, fmt.Errorf("invalid generation number %d", gen) + } + if gen != g.gen { + return nil, fmt.Errorf("broken trace: missing end-of-generation event, or generations are interleaved") + } + if g.minTs == 0 || b.time < g.minTs { + g.minTs = b.time + } + if err := processBatch(g, b, ver); err != nil { + return nil, err + } + } + + // Check some invariants. + if g.freq == 0 { + return nil, fmt.Errorf("no frequency event found") + } + if !g.hasClockSnapshot { + return nil, fmt.Errorf("no clock snapshot event found") + } + + // N.B. Trust that the batch order is correct. We can't validate the batch order + // by timestamp because the timestamps could just be plain wrong. The source of + // truth is the order things appear in the trace and the partial order sequence + // numbers on certain events. If it turns out the batch order is actually incorrect + // we'll very likely fail to advance a partial order from the frontier. + + // Compactify stacks and strings for better lookup performance later. + g.stacks.compactify() + g.strings.compactify() + + // Validate stacks. + if err := validateStackStrings(&g.stacks, &g.strings, g.pcs); err != nil { + return nil, err + } + + // Now that we have the frequency, fix up CPU samples. + fixUpCPUSamples(g.cpuSamples, g.freq) + return g, nil +} + // spilledBatch represents a batch that was read out for the next generation, // while reading the previous one. It's passed on when parsing the next // generation. +// +// Used only for trace versions < Go126. type spilledBatch struct { gen uint64 *batch } -// readGeneration buffers and decodes the structural elements of a trace generation +// readGenerationWithSpill buffers and decodes the structural elements of a trace generation // out of r. spill is the first batch of the new generation (already buffered and // parsed from reading the last generation). Returns the generation and the first // batch read of the next generation, if any. // // If gen is non-nil, it is valid and must be processed before handling the returned // error. -func readGeneration(r *bufio.Reader, spill *spilledBatch) (*generation, *spilledBatch, error) { +func readGenerationWithSpill(r *bufio.Reader, spill *spilledBatch, ver version.Version) (*generation, *spilledBatch, error) { + if ver >= version.Go126 { + return nil, nil, errors.New("internal error: readGenerationWithSpill called for Go 1.26+ trace") + } g := &generation{ evTable: &evTable{ pcs: make(map[uint64]frame), @@ -54,14 +137,15 @@ func readGeneration(r *bufio.Reader, spill *spilledBatch) (*generation, *spilled } // Process the spilled batch. if spill != nil { + // Process the spilled batch, which contains real data. g.gen = spill.gen - if err := processBatch(g, *spill.batch); err != nil { + g.minTs = spill.batch.time + if err := processBatch(g, *spill.batch, ver); err != nil { return nil, nil, err } spill = nil } - // Read batches one at a time until we either hit EOF or - // the next generation. + // Read batches one at a time until we either hit the next generation. var spillErr error for { b, gen, err := readBatch(r) @@ -70,7 +154,7 @@ func readGeneration(r *bufio.Reader, spill *spilledBatch) (*generation, *spilled } if err != nil { if g.gen != 0 { - // This is an error reading the first batch of the next generation. + // This may be an error reading the first batch of the next generation. // This is fine. Let's forge ahead assuming that what we've got so // far is fine. spillErr = err @@ -86,7 +170,8 @@ func readGeneration(r *bufio.Reader, spill *spilledBatch) (*generation, *spilled // Initialize gen. g.gen = gen } - if gen == g.gen+1 { // TODO: advance this the same way the runtime does. + if gen == g.gen+1 { + // TODO: Increment the generation with wraparound the same way the runtime does. spill = &spilledBatch{gen: gen, batch: &b} break } @@ -100,7 +185,10 @@ func readGeneration(r *bufio.Reader, spill *spilledBatch) (*generation, *spilled // problem as soon as we see it. return nil, nil, fmt.Errorf("generations out of order") } - if err := processBatch(g, b); err != nil { + if g.minTs == 0 || b.time < g.minTs { + g.minTs = b.time + } + if err := processBatch(g, b, ver); err != nil { return nil, nil, err } } @@ -109,6 +197,10 @@ func readGeneration(r *bufio.Reader, spill *spilledBatch) (*generation, *spilled if g.freq == 0 { return nil, nil, fmt.Errorf("no frequency event found") } + if ver >= version.Go125 && !g.hasClockSnapshot { + return nil, nil, fmt.Errorf("no clock snapshot event found") + } + // N.B. Trust that the batch order is correct. We can't validate the batch order // by timestamp because the timestamps could just be plain wrong. The source of // truth is the order things appear in the trace and the partial order sequence @@ -124,20 +216,13 @@ func readGeneration(r *bufio.Reader, spill *spilledBatch) (*generation, *spilled return nil, nil, err } - // Fix up the CPU sample timestamps, now that we have freq. - for i := range g.cpuSamples { - s := &g.cpuSamples[i] - s.time = g.freq.mul(timestamp(s.time)) - } - // Sort the CPU samples. - slices.SortFunc(g.cpuSamples, func(a, b cpuSample) int { - return cmp.Compare(a.time, b.time) - }) + // Now that we have the frequency, fix up CPU samples. + fixUpCPUSamples(g.cpuSamples, g.freq) return g, spill, spillErr } // processBatch adds the batch to the generation. -func processBatch(g *generation, b batch) error { +func processBatch(g *generation, b batch, ver version.Version) error { switch { case b.isStringsBatch(): if err := addStrings(&g.strings, b); err != nil { @@ -153,22 +238,19 @@ func processBatch(g *generation, b batch) error { return err } g.cpuSamples = samples - case b.isFreqBatch(): - freq, err := parseFreq(b) - if err != nil { + case b.isSyncBatch(ver): + if err := setSyncBatch(&g.sync, b, ver); err != nil { return err } - if g.freq != 0 { - return fmt.Errorf("found multiple frequency events") - } - g.freq = freq - case b.exp != event.NoExperiment: - if g.expData == nil { - g.expData = make(map[event.Experiment]*ExperimentalData) + case b.exp != tracev2.NoExperiment: + if g.expBatches == nil { + g.expBatches = make(map[tracev2.Experiment][]ExperimentalBatch) } - if err := addExperimentalData(g.expData, b); err != nil { + if err := addExperimentalBatch(g.expBatches, b); err != nil { return err } + case b.isEndOfGeneration(): + return errors.New("internal error: unexpectedly processing EndOfGeneration; broken trace?") default: if _, ok := g.batches[b.m]; !ok { g.batchMs = append(g.batchMs, b.m) @@ -218,7 +300,7 @@ func addStrings(stringTable *dataTable[stringID, string], b batch) error { } r := bytes.NewReader(b.data) hdr, err := r.ReadByte() // Consume the EvStrings byte. - if err != nil || event.Type(hdr) != go122.EvStrings { + if err != nil || tracev2.EventType(hdr) != tracev2.EvStrings { return fmt.Errorf("missing strings batch header") } @@ -229,7 +311,7 @@ func addStrings(stringTable *dataTable[stringID, string], b batch) error { if err != nil { return err } - if event.Type(ev) != go122.EvString { + if tracev2.EventType(ev) != tracev2.EvString { return fmt.Errorf("expected string event, got %d", ev) } @@ -244,8 +326,8 @@ func addStrings(stringTable *dataTable[stringID, string], b batch) error { if err != nil { return err } - if len > go122.MaxStringSize { - return fmt.Errorf("invalid string size %d, maximum is %d", len, go122.MaxStringSize) + if len > tracev2.MaxEventTrailerDataSize { + return fmt.Errorf("invalid string size %d, maximum is %d", len, tracev2.MaxEventTrailerDataSize) } // Copy out the string. @@ -276,7 +358,7 @@ func addStacks(stackTable *dataTable[stackID, stack], pcs map[uint64]frame, b ba } r := bytes.NewReader(b.data) hdr, err := r.ReadByte() // Consume the EvStacks byte. - if err != nil || event.Type(hdr) != go122.EvStacks { + if err != nil || tracev2.EventType(hdr) != tracev2.EvStacks { return fmt.Errorf("missing stacks batch header") } @@ -286,7 +368,7 @@ func addStacks(stackTable *dataTable[stackID, stack], pcs map[uint64]frame, b ba if err != nil { return err } - if event.Type(ev) != go122.EvStack { + if tracev2.EventType(ev) != tracev2.EvStack { return fmt.Errorf("expected stack event, got %d", ev) } @@ -301,8 +383,8 @@ func addStacks(stackTable *dataTable[stackID, stack], pcs map[uint64]frame, b ba if err != nil { return err } - if nFrames > go122.MaxFramesPerStack { - return fmt.Errorf("invalid stack size %d, maximum is %d", nFrames, go122.MaxFramesPerStack) + if nFrames > tracev2.MaxFramesPerStack { + return fmt.Errorf("invalid stack size %d, maximum is %d", nFrames, tracev2.MaxFramesPerStack) } // Each frame consists of 4 fields: pc, funcID (string), fileID (string), line. @@ -354,7 +436,7 @@ func addCPUSamples(samples []cpuSample, b batch) ([]cpuSample, error) { } r := bytes.NewReader(b.data) hdr, err := r.ReadByte() // Consume the EvCPUSamples byte. - if err != nil || event.Type(hdr) != go122.EvCPUSamples { + if err != nil || tracev2.EventType(hdr) != tracev2.EvCPUSamples { return nil, fmt.Errorf("missing CPU samples batch header") } @@ -364,7 +446,7 @@ func addCPUSamples(samples []cpuSample, b batch) ([]cpuSample, error) { if err != nil { return nil, err } - if event.Type(ev) != go122.EvCPUSample { + if tracev2.EventType(ev) != tracev2.EvCPUSample { return nil, fmt.Errorf("expected CPU sample event, got %d", ev) } @@ -418,37 +500,104 @@ func addCPUSamples(samples []cpuSample, b batch) ([]cpuSample, error) { return samples, nil } -// parseFreq parses out a lone EvFrequency from a batch. -func parseFreq(b batch) (frequency, error) { - if !b.isFreqBatch() { - return 0, fmt.Errorf("internal error: parseFreq called on non-frequency batch") +// sync holds the per-generation sync data. +type sync struct { + freq frequency + hasClockSnapshot bool + snapTime timestamp + snapMono uint64 + snapWall time.Time +} + +func setSyncBatch(s *sync, b batch, ver version.Version) error { + if !b.isSyncBatch(ver) { + return fmt.Errorf("internal error: setSyncBatch called on non-sync batch") } r := bytes.NewReader(b.data) - r.ReadByte() // Consume the EvFrequency byte. + if ver >= version.Go125 { + hdr, err := r.ReadByte() // Consume the EvSync byte. + if err != nil || tracev2.EventType(hdr) != tracev2.EvSync { + return fmt.Errorf("missing sync batch header") + } + } - // Read the frequency. It'll come out as timestamp units per second. - f, err := binary.ReadUvarint(r) - if err != nil { - return 0, err + lastTs := b.time + for r.Len() != 0 { + // Read the header + ev, err := r.ReadByte() + if err != nil { + return err + } + et := tracev2.EventType(ev) + switch { + case et == tracev2.EvFrequency: + if s.freq != 0 { + return fmt.Errorf("found multiple frequency events") + } + // Read the frequency. It'll come out as timestamp units per second. + f, err := binary.ReadUvarint(r) + if err != nil { + return err + } + // Convert to nanoseconds per timestamp unit. + s.freq = frequency(1.0 / (float64(f) / 1e9)) + case et == tracev2.EvClockSnapshot && ver >= version.Go125: + if s.hasClockSnapshot { + return fmt.Errorf("found multiple clock snapshot events") + } + s.hasClockSnapshot = true + // Read the EvClockSnapshot arguments. + tdiff, err := binary.ReadUvarint(r) + if err != nil { + return err + } + lastTs += timestamp(tdiff) + s.snapTime = lastTs + mono, err := binary.ReadUvarint(r) + if err != nil { + return err + } + s.snapMono = mono + sec, err := binary.ReadUvarint(r) + if err != nil { + return err + } + nsec, err := binary.ReadUvarint(r) + if err != nil { + return err + } + // TODO(felixge): In theory we could inject s.snapMono into the time + // value below to make it comparable. But there is no API for this + // in the time package right now. + s.snapWall = time.Unix(int64(sec), int64(nsec)) + default: + return fmt.Errorf("expected frequency or clock snapshot event, got %d", ev) + } } - // Convert to nanoseconds per timestamp unit. - return frequency(1.0 / (float64(f) / 1e9)), nil + return nil } -// addExperimentalData takes an experimental batch and adds it to the ExperimentalData -// for the experiment its a part of. -func addExperimentalData(expData map[event.Experiment]*ExperimentalData, b batch) error { - if b.exp == event.NoExperiment { - return fmt.Errorf("internal error: addExperimentalData called on non-experimental batch") - } - ed, ok := expData[b.exp] - if !ok { - ed = new(ExperimentalData) - expData[b.exp] = ed +// addExperimentalBatch takes an experimental batch and adds it to the list of experimental +// batches for the experiment its a part of. +func addExperimentalBatch(expBatches map[tracev2.Experiment][]ExperimentalBatch, b batch) error { + if b.exp == tracev2.NoExperiment { + return fmt.Errorf("internal error: addExperimentalBatch called on non-experimental batch") } - ed.Batches = append(ed.Batches, ExperimentalBatch{ + expBatches[b.exp] = append(expBatches[b.exp], ExperimentalBatch{ Thread: b.m, Data: b.data, }) return nil } + +func fixUpCPUSamples(samples []cpuSample, freq frequency) { + // Fix up the CPU sample timestamps. + for i := range samples { + s := &samples[i] + s.time = freq.mul(timestamp(s.time)) + } + // Sort the CPU samples. + slices.SortFunc(samples, func(a, b cpuSample) int { + return cmp.Compare(a.time, b.time) + }) +} diff --git a/src/internal/trace/internal/oldtrace/parser.go b/src/internal/trace/internal/oldtrace/parser.go deleted file mode 100644 index afbf0ed50ed776..00000000000000 --- a/src/internal/trace/internal/oldtrace/parser.go +++ /dev/null @@ -1,1544 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package oldtrace implements a parser for Go execution traces from versions -// 1.11–1.21. -// -// The package started as a copy of Go 1.19's internal/trace, but has been -// optimized to be faster while using less memory and fewer allocations. It has -// been further modified for the specific purpose of converting traces to the -// new 1.22+ format. -package oldtrace - -import ( - "bytes" - "cmp" - "encoding/binary" - "errors" - "fmt" - "internal/trace/event" - "internal/trace/version" - "io" - "math" - "slices" - "sort" -) - -// Timestamp represents a count of nanoseconds since the beginning of the trace. -// They can only be meaningfully compared with other timestamps from the same -// trace. -type Timestamp int64 - -// Event describes one event in the trace. -type Event struct { - // The Event type is carefully laid out to optimize its size and to avoid - // pointers, the latter so that the garbage collector won't have to scan any - // memory of our millions of events. - - Ts Timestamp // timestamp in nanoseconds - G uint64 // G on which the event happened - Args [4]uint64 // event-type-specific arguments - StkID uint32 // unique stack ID - P int32 // P on which the event happened (can be a real P or one of TimerP, NetpollP, SyscallP) - Type event.Type // one of Ev* -} - -// Frame is a frame in stack traces. -type Frame struct { - PC uint64 - // string ID of the function name - Fn uint64 - // string ID of the file name - File uint64 - Line int -} - -const ( - // Special P identifiers: - FakeP = 1000000 + iota - TimerP // contains timer unblocks - NetpollP // contains network unblocks - SyscallP // contains returns from syscalls - GCP // contains GC state - ProfileP // contains recording of CPU profile samples -) - -// Trace is the result of Parse. -type Trace struct { - Version version.Version - - // Events is the sorted list of Events in the trace. - Events Events - // Stacks is the stack traces (stored as slices of PCs), keyed by stack IDs - // from the trace. - Stacks map[uint32][]uint64 - PCs map[uint64]Frame - Strings map[uint64]string - InlineStrings []string -} - -// batchOffset records the byte offset of, and number of events in, a batch. A -// batch is a sequence of events emitted by a P. Events within a single batch -// are sorted by time. -type batchOffset struct { - offset int - numEvents int -} - -type parser struct { - ver version.Version - data []byte - off int - - strings map[uint64]string - inlineStrings []string - inlineStringsMapping map[string]int - // map from Ps to their batch offsets - batchOffsets map[int32][]batchOffset - stacks map[uint32][]uint64 - stacksData []uint64 - ticksPerSec int64 - pcs map[uint64]Frame - cpuSamples []Event - timerGoids map[uint64]bool - - // state for readRawEvent - args []uint64 - - // state for parseEvent - lastTs Timestamp - lastG uint64 - // map from Ps to the last Gs that ran on them - lastGs map[int32]uint64 - lastP int32 -} - -func (p *parser) discard(n uint64) bool { - if n > math.MaxInt { - return false - } - if noff := p.off + int(n); noff < p.off || noff > len(p.data) { - return false - } else { - p.off = noff - } - return true -} - -func newParser(r io.Reader, ver version.Version) (*parser, error) { - var buf []byte - if seeker, ok := r.(io.Seeker); ok { - // Determine the size of the reader so that we can allocate a buffer - // without having to grow it later. - cur, err := seeker.Seek(0, io.SeekCurrent) - if err != nil { - return nil, err - } - end, err := seeker.Seek(0, io.SeekEnd) - if err != nil { - return nil, err - } - _, err = seeker.Seek(cur, io.SeekStart) - if err != nil { - return nil, err - } - - buf = make([]byte, end-cur) - _, err = io.ReadFull(r, buf) - if err != nil { - return nil, err - } - } else { - var err error - buf, err = io.ReadAll(r) - if err != nil { - return nil, err - } - } - return &parser{data: buf, ver: ver, timerGoids: make(map[uint64]bool)}, nil -} - -// Parse parses Go execution traces from versions 1.11–1.21. The provided reader -// will be read to completion and the entire trace will be materialized in -// memory. That is, this function does not allow incremental parsing. -// -// The reader has to be positioned just after the trace header and vers needs to -// be the version of the trace. This can be achieved by using -// version.ReadHeader. -func Parse(r io.Reader, vers version.Version) (Trace, error) { - // We accept the version as an argument because internal/trace will have - // already read the version to determine which parser to use. - p, err := newParser(r, vers) - if err != nil { - return Trace{}, err - } - return p.parse() -} - -// parse parses, post-processes and verifies the trace. -func (p *parser) parse() (Trace, error) { - defer func() { - p.data = nil - }() - - // We parse a trace by running the following steps in order: - // - // 1. In the initial pass we collect information about batches (their - // locations and sizes.) We also parse CPU profiling samples in this - // step, simply to reduce the number of full passes that we need. - // - // 2. In the second pass we parse batches and merge them into a globally - // ordered event stream. This uses the batch information from the first - // pass to quickly find batches. - // - // 3. After all events have been parsed we convert their timestamps from CPU - // ticks to wall time. Furthermore we move timers and syscalls to - // dedicated, fake Ps. - // - // 4. Finally, we validate the trace. - - p.strings = make(map[uint64]string) - p.batchOffsets = make(map[int32][]batchOffset) - p.lastGs = make(map[int32]uint64) - p.stacks = make(map[uint32][]uint64) - p.pcs = make(map[uint64]Frame) - p.inlineStringsMapping = make(map[string]int) - - if err := p.collectBatchesAndCPUSamples(); err != nil { - return Trace{}, err - } - - events, err := p.parseEventBatches() - if err != nil { - return Trace{}, err - } - - if p.ticksPerSec == 0 { - return Trace{}, errors.New("no EvFrequency event") - } - - if events.Len() > 0 { - // Translate cpu ticks to real time. - minTs := events.Ptr(0).Ts - // Use floating point to avoid integer overflows. - freq := 1e9 / float64(p.ticksPerSec) - for i := 0; i < events.Len(); i++ { - ev := events.Ptr(i) - ev.Ts = Timestamp(float64(ev.Ts-minTs) * freq) - // Move timers and syscalls to separate fake Ps. - if p.timerGoids[ev.G] && ev.Type == EvGoUnblock { - ev.P = TimerP - } - if ev.Type == EvGoSysExit { - ev.P = SyscallP - } - } - } - - if err := p.postProcessTrace(events); err != nil { - return Trace{}, err - } - - res := Trace{ - Version: p.ver, - Events: events, - Stacks: p.stacks, - Strings: p.strings, - InlineStrings: p.inlineStrings, - PCs: p.pcs, - } - return res, nil -} - -// rawEvent is a helper type used during parsing. -type rawEvent struct { - typ event.Type - args []uint64 - sargs []string - - // if typ == EvBatch, these fields describe the batch. - batchPid int32 - batchOffset int -} - -type proc struct { - pid int32 - // the remaining events in the current batch - events []Event - // buffer for reading batches into, aliased by proc.events - buf []Event - - // there are no more batches left - done bool -} - -const eventsBucketSize = 524288 // 32 MiB of events - -type Events struct { - // Events is a slice of slices that grows one slice of size eventsBucketSize - // at a time. This avoids the O(n) cost of slice growth in append, and - // additionally allows consumers to drop references to parts of the data, - // freeing memory piecewise. - n int - buckets []*[eventsBucketSize]Event - off int -} - -// grow grows the slice by one and returns a pointer to the new element, without -// overwriting it. -func (l *Events) grow() *Event { - a, b := l.index(l.n) - if a >= len(l.buckets) { - l.buckets = append(l.buckets, new([eventsBucketSize]Event)) - } - ptr := &l.buckets[a][b] - l.n++ - return ptr -} - -// append appends v to the slice and returns a pointer to the new element. -func (l *Events) append(v Event) *Event { - ptr := l.grow() - *ptr = v - return ptr -} - -func (l *Events) Ptr(i int) *Event { - a, b := l.index(i + l.off) - return &l.buckets[a][b] -} - -func (l *Events) index(i int) (int, int) { - // Doing the division on uint instead of int compiles this function to a - // shift and an AND (for power of 2 bucket sizes), versus a whole bunch of - // instructions for int. - return int(uint(i) / eventsBucketSize), int(uint(i) % eventsBucketSize) -} - -func (l *Events) Len() int { - return l.n - l.off -} - -func (l *Events) Less(i, j int) bool { - return l.Ptr(i).Ts < l.Ptr(j).Ts -} - -func (l *Events) Swap(i, j int) { - *l.Ptr(i), *l.Ptr(j) = *l.Ptr(j), *l.Ptr(i) -} - -func (l *Events) Pop() (*Event, bool) { - if l.off == l.n { - return nil, false - } - a, b := l.index(l.off) - ptr := &l.buckets[a][b] - l.off++ - if b == eventsBucketSize-1 || l.off == l.n { - // We've consumed the last event from the bucket, so drop the bucket and - // allow GC to collect it. - l.buckets[a] = nil - } - return ptr, true -} - -func (l *Events) All() func(yield func(ev *Event) bool) { - return func(yield func(ev *Event) bool) { - for i := 0; i < l.Len(); i++ { - a, b := l.index(i + l.off) - ptr := &l.buckets[a][b] - if !yield(ptr) { - return - } - } - } -} - -// parseEventBatches reads per-P event batches and merges them into a single, consistent -// stream. The high level idea is as follows. Events within an individual batch -// are in correct order, because they are emitted by a single P. So we need to -// produce a correct interleaving of the batches. To do this we take first -// unmerged event from each batch (frontier). Then choose subset that is "ready" -// to be merged, that is, events for which all dependencies are already merged. -// Then we choose event with the lowest timestamp from the subset, merge it and -// repeat. This approach ensures that we form a consistent stream even if -// timestamps are incorrect (condition observed on some machines). -func (p *parser) parseEventBatches() (Events, error) { - // The ordering of CPU profile sample events in the data stream is based on - // when each run of the signal handler was able to acquire the spinlock, - // with original timestamps corresponding to when ReadTrace pulled the data - // off of the profBuf queue. Re-sort them by the timestamp we captured - // inside the signal handler. - slices.SortFunc(p.cpuSamples, func(a, b Event) int { - return cmp.Compare(a.Ts, b.Ts) - }) - - allProcs := make([]proc, 0, len(p.batchOffsets)) - for pid := range p.batchOffsets { - allProcs = append(allProcs, proc{pid: pid}) - } - allProcs = append(allProcs, proc{pid: ProfileP, events: p.cpuSamples}) - - events := Events{} - - // Merge events as long as at least one P has more events - gs := make(map[uint64]gState) - // Note: technically we don't need a priority queue here. We're only ever - // interested in the earliest elligible event, which means we just have to - // track the smallest element. However, in practice, the priority queue - // performs better, because for each event we only have to compute its state - // transition once, not on each iteration. If it was elligible before, it'll - // already be in the queue. Furthermore, on average, we only have one P to - // look at in each iteration, because all other Ps are already in the queue. - var frontier orderEventList - - availableProcs := make([]*proc, len(allProcs)) - for i := range allProcs { - availableProcs[i] = &allProcs[i] - } - for { - pidLoop: - for i := 0; i < len(availableProcs); i++ { - proc := availableProcs[i] - - for len(proc.events) == 0 { - // Call loadBatch in a loop because sometimes batches are empty - evs, err := p.loadBatch(proc.pid, proc.buf[:0]) - proc.buf = evs[:0] - if err == io.EOF { - // This P has no more events - proc.done = true - availableProcs[i], availableProcs[len(availableProcs)-1] = availableProcs[len(availableProcs)-1], availableProcs[i] - availableProcs = availableProcs[:len(availableProcs)-1] - // We swapped the element at i with another proc, so look at - // the index again - i-- - continue pidLoop - } else if err != nil { - return Events{}, err - } else { - proc.events = evs - } - } - - ev := &proc.events[0] - g, init, _ := stateTransition(ev) - - // TODO(dh): This implementation matches the behavior of the - // upstream 'go tool trace', and works in practice, but has run into - // the following inconsistency during fuzzing: what happens if - // multiple Ps have events for the same G? While building the - // frontier we will check all of the events against the current - // state of the G. However, when we process the frontier, the state - // of the G changes, and a transition that was valid while building - // the frontier may no longer be valid when processing the frontier. - // Is this something that can happen for real, valid traces, or is - // this only possible with corrupt data? - if !transitionReady(g, gs[g], init) { - continue - } - proc.events = proc.events[1:] - availableProcs[i], availableProcs[len(availableProcs)-1] = availableProcs[len(availableProcs)-1], availableProcs[i] - availableProcs = availableProcs[:len(availableProcs)-1] - frontier.Push(orderEvent{*ev, proc}) - - // We swapped the element at i with another proc, so look at the - // index again - i-- - } - - if len(frontier) == 0 { - for i := range allProcs { - if !allProcs[i].done { - return Events{}, fmt.Errorf("no consistent ordering of events possible") - } - } - break - } - f := frontier.Pop() - - // We're computing the state transition twice, once when computing the - // frontier, and now to apply the transition. This is fine because - // stateTransition is a pure function. Computing it again is cheaper - // than storing large items in the frontier. - g, init, next := stateTransition(&f.ev) - - // Get rid of "Local" events, they are intended merely for ordering. - switch f.ev.Type { - case EvGoStartLocal: - f.ev.Type = EvGoStart - case EvGoUnblockLocal: - f.ev.Type = EvGoUnblock - case EvGoSysExitLocal: - f.ev.Type = EvGoSysExit - } - events.append(f.ev) - - if err := transition(gs, g, init, next); err != nil { - return Events{}, err - } - availableProcs = append(availableProcs, f.proc) - } - - // At this point we have a consistent stream of events. Make sure time - // stamps respect the ordering. The tests will skip (not fail) the test case - // if they see this error. - if !sort.IsSorted(&events) { - return Events{}, ErrTimeOrder - } - - // The last part is giving correct timestamps to EvGoSysExit events. The - // problem with EvGoSysExit is that actual syscall exit timestamp - // (ev.Args[2]) is potentially acquired long before event emission. So far - // we've used timestamp of event emission (ev.Ts). We could not set ev.Ts = - // ev.Args[2] earlier, because it would produce seemingly broken timestamps - // (misplaced event). We also can't simply update the timestamp and resort - // events, because if timestamps are broken we will misplace the event and - // later report logically broken trace (instead of reporting broken - // timestamps). - lastSysBlock := make(map[uint64]Timestamp) - for i := 0; i < events.Len(); i++ { - ev := events.Ptr(i) - switch ev.Type { - case EvGoSysBlock, EvGoInSyscall: - lastSysBlock[ev.G] = ev.Ts - case EvGoSysExit: - ts := Timestamp(ev.Args[2]) - if ts == 0 { - continue - } - block := lastSysBlock[ev.G] - if block == 0 { - return Events{}, fmt.Errorf("stray syscall exit") - } - if ts < block { - return Events{}, ErrTimeOrder - } - ev.Ts = ts - } - } - sort.Stable(&events) - - return events, nil -} - -// collectBatchesAndCPUSamples records the offsets of batches and parses CPU samples. -func (p *parser) collectBatchesAndCPUSamples() error { - // Read events. - var raw rawEvent - var curP int32 - for n := uint64(0); ; n++ { - err := p.readRawEvent(skipArgs|skipStrings, &raw) - if err == io.EOF { - break - } - if err != nil { - return err - } - if raw.typ == EvNone { - continue - } - - if raw.typ == EvBatch { - bo := batchOffset{offset: raw.batchOffset} - p.batchOffsets[raw.batchPid] = append(p.batchOffsets[raw.batchPid], bo) - curP = raw.batchPid - } - - batches := p.batchOffsets[curP] - if len(batches) == 0 { - return fmt.Errorf("read event %d with current P of %d, but P has no batches yet", - raw.typ, curP) - } - batches[len(batches)-1].numEvents++ - - if raw.typ == EvCPUSample { - e := Event{Type: raw.typ} - - argOffset := 1 - narg := raw.argNum() - if len(raw.args) != narg { - return fmt.Errorf("CPU sample has wrong number of arguments: want %d, got %d", narg, len(raw.args)) - } - for i := argOffset; i < narg; i++ { - if i == narg-1 { - e.StkID = uint32(raw.args[i]) - } else { - e.Args[i-argOffset] = raw.args[i] - } - } - - e.Ts = Timestamp(e.Args[0]) - e.P = int32(e.Args[1]) - e.G = e.Args[2] - e.Args[0] = 0 - - // Most events are written out by the active P at the exact moment - // they describe. CPU profile samples are different because they're - // written to the tracing log after some delay, by a separate worker - // goroutine, into a separate buffer. - // - // We keep these in their own batch until all of the batches are - // merged in timestamp order. We also (right before the merge) - // re-sort these events by the timestamp captured in the profiling - // signal handler. - // - // Note that we're not concerned about the memory usage of storing - // all CPU samples during the indexing phase. There are orders of - // magnitude fewer CPU samples than runtime events. - p.cpuSamples = append(p.cpuSamples, e) - } - } - - return nil -} - -const ( - skipArgs = 1 << iota - skipStrings -) - -func (p *parser) readByte() (byte, bool) { - if p.off < len(p.data) && p.off >= 0 { - b := p.data[p.off] - p.off++ - return b, true - } else { - return 0, false - } -} - -func (p *parser) readFull(n int) ([]byte, error) { - if p.off >= len(p.data) || p.off < 0 || p.off+n > len(p.data) { - // p.off < 0 is impossible but makes BCE happy. - // - // We do fail outright if there's not enough data, we don't care about - // partial results. - return nil, io.ErrUnexpectedEOF - } - buf := p.data[p.off : p.off+n] - p.off += n - return buf, nil -} - -// readRawEvent reads a raw event into ev. The slices in ev are only valid until -// the next call to readRawEvent, even when storing to a different location. -func (p *parser) readRawEvent(flags uint, ev *rawEvent) error { - // The number of arguments is encoded using two bits and can thus only - // represent the values 0–3. The value 3 (on the wire) indicates that - // arguments are prefixed by their byte length, to encode >=3 arguments. - const inlineArgs = 3 - - // Read event type and number of arguments (1 byte). - b, ok := p.readByte() - if !ok { - return io.EOF - } - typ := event.Type(b << 2 >> 2) - // Most events have a timestamp before the actual arguments, so we add 1 and - // parse it like it's the first argument. EvString has a special format and - // the number of arguments doesn't matter. EvBatch writes '1' as the number - // of arguments, but actually has two: a pid and a timestamp, but here the - // timestamp is the second argument, not the first; adding 1 happens to come - // up with the correct number, but it doesn't matter, because EvBatch has - // custom logic for parsing. - // - // Note that because we're adding 1, inlineArgs == 3 describes the largest - // number of logical arguments that isn't length-prefixed, even though the - // value 3 on the wire indicates length-prefixing. For us, that becomes narg - // == 4. - narg := b>>6 + 1 - if typ == EvNone || typ >= EvCount || EventDescriptions[typ].minVersion > p.ver { - return fmt.Errorf("unknown event type %d", typ) - } - - switch typ { - case EvString: - if flags&skipStrings != 0 { - // String dictionary entry [ID, length, string]. - if _, err := p.readVal(); err != nil { - return errMalformedVarint - } - ln, err := p.readVal() - if err != nil { - return err - } - if !p.discard(ln) { - return fmt.Errorf("failed to read trace: %w", io.EOF) - } - } else { - // String dictionary entry [ID, length, string]. - id, err := p.readVal() - if err != nil { - return err - } - if id == 0 { - return errors.New("string has invalid id 0") - } - if p.strings[id] != "" { - return fmt.Errorf("string has duplicate id %d", id) - } - var ln uint64 - ln, err = p.readVal() - if err != nil { - return err - } - if ln == 0 { - return errors.New("string has invalid length 0") - } - if ln > 1e6 { - return fmt.Errorf("string has too large length %d", ln) - } - buf, err := p.readFull(int(ln)) - if err != nil { - return fmt.Errorf("failed to read trace: %w", err) - } - p.strings[id] = string(buf) - } - - ev.typ = EvNone - return nil - case EvBatch: - if want := byte(2); narg != want { - return fmt.Errorf("EvBatch has wrong number of arguments: got %d, want %d", narg, want) - } - - // -1 because we've already read the first byte of the batch - off := p.off - 1 - - pid, err := p.readVal() - if err != nil { - return err - } - if pid != math.MaxUint64 && pid > math.MaxInt32 { - return fmt.Errorf("processor ID %d is larger than maximum of %d", pid, uint64(math.MaxUint)) - } - - var pid32 int32 - if pid == math.MaxUint64 { - pid32 = -1 - } else { - pid32 = int32(pid) - } - - v, err := p.readVal() - if err != nil { - return err - } - - *ev = rawEvent{ - typ: EvBatch, - args: p.args[:0], - batchPid: pid32, - batchOffset: off, - } - ev.args = append(ev.args, pid, v) - return nil - default: - *ev = rawEvent{typ: typ, args: p.args[:0]} - if narg <= inlineArgs { - if flags&skipArgs == 0 { - for i := 0; i < int(narg); i++ { - v, err := p.readVal() - if err != nil { - return fmt.Errorf("failed to read event %d argument: %w", typ, err) - } - ev.args = append(ev.args, v) - } - } else { - for i := 0; i < int(narg); i++ { - if _, err := p.readVal(); err != nil { - return fmt.Errorf("failed to read event %d argument: %w", typ, errMalformedVarint) - } - } - } - } else { - // More than inlineArgs args, the first value is length of the event - // in bytes. - v, err := p.readVal() - if err != nil { - return fmt.Errorf("failed to read event %d argument: %w", typ, err) - } - - if limit := uint64(2048); v > limit { - // At the time of Go 1.19, v seems to be at most 128. Set 2048 - // as a generous upper limit and guard against malformed traces. - return fmt.Errorf("failed to read event %d argument: length-prefixed argument too big: %d bytes, limit is %d", typ, v, limit) - } - - if flags&skipArgs == 0 || typ == EvCPUSample { - buf, err := p.readFull(int(v)) - if err != nil { - return fmt.Errorf("failed to read trace: %w", err) - } - for len(buf) > 0 { - var v uint64 - v, buf, err = readValFrom(buf) - if err != nil { - return err - } - ev.args = append(ev.args, v) - } - } else { - // Skip over arguments - if !p.discard(v) { - return fmt.Errorf("failed to read trace: %w", io.EOF) - } - } - if typ == EvUserLog { - // EvUserLog records are followed by a value string - if flags&skipArgs == 0 { - // Read string - s, err := p.readStr() - if err != nil { - return err - } - ev.sargs = append(ev.sargs, s) - } else { - // Skip string - v, err := p.readVal() - if err != nil { - return err - } - if !p.discard(v) { - return io.EOF - } - } - } - } - - p.args = ev.args[:0] - return nil - } -} - -// loadBatch loads the next batch for pid and appends its contents to to events. -func (p *parser) loadBatch(pid int32, events []Event) ([]Event, error) { - offsets := p.batchOffsets[pid] - if len(offsets) == 0 { - return nil, io.EOF - } - n := offsets[0].numEvents - offset := offsets[0].offset - offsets = offsets[1:] - p.batchOffsets[pid] = offsets - - p.off = offset - - if cap(events) < n { - events = make([]Event, 0, n) - } - - gotHeader := false - var raw rawEvent - var ev Event - for { - err := p.readRawEvent(0, &raw) - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - if raw.typ == EvNone || raw.typ == EvCPUSample { - continue - } - if raw.typ == EvBatch { - if gotHeader { - break - } else { - gotHeader = true - } - } - - err = p.parseEvent(&raw, &ev) - if err != nil { - return nil, err - } - if ev.Type != EvNone { - events = append(events, ev) - } - } - - return events, nil -} - -func (p *parser) readStr() (s string, err error) { - sz, err := p.readVal() - if err != nil { - return "", err - } - if sz == 0 { - return "", nil - } - if sz > 1e6 { - return "", fmt.Errorf("string is too large (len=%d)", sz) - } - buf, err := p.readFull(int(sz)) - if err != nil { - return "", fmt.Errorf("failed to read trace: %w", err) - } - return string(buf), nil -} - -// parseEvent transforms raw events into events. -// It does analyze and verify per-event-type arguments. -func (p *parser) parseEvent(raw *rawEvent, ev *Event) error { - desc := &EventDescriptions[raw.typ] - if desc.Name == "" { - return fmt.Errorf("missing description for event type %d", raw.typ) - } - narg := raw.argNum() - if len(raw.args) != narg { - return fmt.Errorf("%s has wrong number of arguments: want %d, got %d", desc.Name, narg, len(raw.args)) - } - switch raw.typ { - case EvBatch: - p.lastGs[p.lastP] = p.lastG - if raw.args[0] != math.MaxUint64 && raw.args[0] > math.MaxInt32 { - return fmt.Errorf("processor ID %d is larger than maximum of %d", raw.args[0], uint64(math.MaxInt32)) - } - if raw.args[0] == math.MaxUint64 { - p.lastP = -1 - } else { - p.lastP = int32(raw.args[0]) - } - p.lastG = p.lastGs[p.lastP] - p.lastTs = Timestamp(raw.args[1]) - case EvFrequency: - p.ticksPerSec = int64(raw.args[0]) - if p.ticksPerSec <= 0 { - // The most likely cause for this is tick skew on different CPUs. - // For example, solaris/amd64 seems to have wildly different - // ticks on different CPUs. - return ErrTimeOrder - } - case EvTimerGoroutine: - p.timerGoids[raw.args[0]] = true - case EvStack: - if len(raw.args) < 2 { - return fmt.Errorf("EvStack has wrong number of arguments: want at least 2, got %d", len(raw.args)) - } - size := raw.args[1] - if size > 1000 { - return fmt.Errorf("EvStack has bad number of frames: %d", size) - } - want := 2 + 4*size - if uint64(len(raw.args)) != want { - return fmt.Errorf("EvStack has wrong number of arguments: want %d, got %d", want, len(raw.args)) - } - id := uint32(raw.args[0]) - if id != 0 && size > 0 { - stk := p.allocateStack(size) - for i := 0; i < int(size); i++ { - pc := raw.args[2+i*4+0] - fn := raw.args[2+i*4+1] - file := raw.args[2+i*4+2] - line := raw.args[2+i*4+3] - stk[i] = pc - - if _, ok := p.pcs[pc]; !ok { - p.pcs[pc] = Frame{PC: pc, Fn: fn, File: file, Line: int(line)} - } - } - p.stacks[id] = stk - } - case EvCPUSample: - // These events get parsed during the indexing step and don't strictly - // belong to the batch. - default: - *ev = Event{Type: raw.typ, P: p.lastP, G: p.lastG} - var argOffset int - ev.Ts = p.lastTs + Timestamp(raw.args[0]) - argOffset = 1 - p.lastTs = ev.Ts - for i := argOffset; i < narg; i++ { - if i == narg-1 && desc.Stack { - ev.StkID = uint32(raw.args[i]) - } else { - ev.Args[i-argOffset] = raw.args[i] - } - } - switch raw.typ { - case EvGoStart, EvGoStartLocal, EvGoStartLabel: - p.lastG = ev.Args[0] - ev.G = p.lastG - case EvGoEnd, EvGoStop, EvGoSched, EvGoPreempt, - EvGoSleep, EvGoBlock, EvGoBlockSend, EvGoBlockRecv, - EvGoBlockSelect, EvGoBlockSync, EvGoBlockCond, EvGoBlockNet, - EvGoSysBlock, EvGoBlockGC: - p.lastG = 0 - case EvGoSysExit, EvGoWaiting, EvGoInSyscall: - ev.G = ev.Args[0] - case EvUserTaskCreate: - // e.Args 0: taskID, 1:parentID, 2:nameID - case EvUserRegion: - // e.Args 0: taskID, 1: mode, 2:nameID - case EvUserLog: - // e.Args 0: taskID, 1:keyID, 2: stackID, 3: messageID - // raw.sargs 0: message - - if id, ok := p.inlineStringsMapping[raw.sargs[0]]; ok { - ev.Args[3] = uint64(id) - } else { - id := len(p.inlineStrings) - p.inlineStringsMapping[raw.sargs[0]] = id - p.inlineStrings = append(p.inlineStrings, raw.sargs[0]) - ev.Args[3] = uint64(id) - } - } - - return nil - } - - ev.Type = EvNone - return nil -} - -// ErrTimeOrder is returned by Parse when the trace contains -// time stamps that do not respect actual event ordering. -var ErrTimeOrder = errors.New("time stamps out of order") - -// postProcessTrace does inter-event verification and information restoration. -// The resulting trace is guaranteed to be consistent -// (for example, a P does not run two Gs at the same time, or a G is indeed -// blocked before an unblock event). -func (p *parser) postProcessTrace(events Events) error { - const ( - gDead = iota - gRunnable - gRunning - gWaiting - ) - type gdesc struct { - state int - ev *Event - evStart *Event - evCreate *Event - evMarkAssist *Event - } - type pdesc struct { - running bool - g uint64 - evSweep *Event - } - - gs := make(map[uint64]gdesc) - ps := make(map[int32]pdesc) - tasks := make(map[uint64]*Event) // task id to task creation events - activeRegions := make(map[uint64][]*Event) // goroutine id to stack of regions - gs[0] = gdesc{state: gRunning} - var evGC, evSTW *Event - - checkRunning := func(p pdesc, g gdesc, ev *Event, allowG0 bool) error { - name := EventDescriptions[ev.Type].Name - if g.state != gRunning { - return fmt.Errorf("g %d is not running while %s (time %d)", ev.G, name, ev.Ts) - } - if p.g != ev.G { - return fmt.Errorf("p %d is not running g %d while %s (time %d)", ev.P, ev.G, name, ev.Ts) - } - if !allowG0 && ev.G == 0 { - return fmt.Errorf("g 0 did %s (time %d)", name, ev.Ts) - } - return nil - } - - for evIdx := 0; evIdx < events.Len(); evIdx++ { - ev := events.Ptr(evIdx) - - switch ev.Type { - case EvProcStart: - p := ps[ev.P] - if p.running { - return fmt.Errorf("p %d is running before start (time %d)", ev.P, ev.Ts) - } - p.running = true - - ps[ev.P] = p - case EvProcStop: - p := ps[ev.P] - if !p.running { - return fmt.Errorf("p %d is not running before stop (time %d)", ev.P, ev.Ts) - } - if p.g != 0 { - return fmt.Errorf("p %d is running a goroutine %d during stop (time %d)", ev.P, p.g, ev.Ts) - } - p.running = false - - ps[ev.P] = p - case EvGCStart: - if evGC != nil { - return fmt.Errorf("previous GC is not ended before a new one (time %d)", ev.Ts) - } - evGC = ev - // Attribute this to the global GC state. - ev.P = GCP - case EvGCDone: - if evGC == nil { - return fmt.Errorf("bogus GC end (time %d)", ev.Ts) - } - evGC = nil - case EvSTWStart: - evp := &evSTW - if *evp != nil { - return fmt.Errorf("previous STW is not ended before a new one (time %d)", ev.Ts) - } - *evp = ev - case EvSTWDone: - evp := &evSTW - if *evp == nil { - return fmt.Errorf("bogus STW end (time %d)", ev.Ts) - } - *evp = nil - case EvGCSweepStart: - p := ps[ev.P] - if p.evSweep != nil { - return fmt.Errorf("previous sweeping is not ended before a new one (time %d)", ev.Ts) - } - p.evSweep = ev - - ps[ev.P] = p - case EvGCMarkAssistStart: - g := gs[ev.G] - if g.evMarkAssist != nil { - return fmt.Errorf("previous mark assist is not ended before a new one (time %d)", ev.Ts) - } - g.evMarkAssist = ev - - gs[ev.G] = g - case EvGCMarkAssistDone: - // Unlike most events, mark assists can be in progress when a - // goroutine starts tracing, so we can't report an error here. - g := gs[ev.G] - if g.evMarkAssist != nil { - g.evMarkAssist = nil - } - - gs[ev.G] = g - case EvGCSweepDone: - p := ps[ev.P] - if p.evSweep == nil { - return fmt.Errorf("bogus sweeping end (time %d)", ev.Ts) - } - p.evSweep = nil - - ps[ev.P] = p - case EvGoWaiting: - g := gs[ev.G] - if g.state != gRunnable { - return fmt.Errorf("g %d is not runnable before EvGoWaiting (time %d)", ev.G, ev.Ts) - } - g.state = gWaiting - g.ev = ev - - gs[ev.G] = g - case EvGoInSyscall: - g := gs[ev.G] - if g.state != gRunnable { - return fmt.Errorf("g %d is not runnable before EvGoInSyscall (time %d)", ev.G, ev.Ts) - } - g.state = gWaiting - g.ev = ev - - gs[ev.G] = g - case EvGoCreate: - g := gs[ev.G] - p := ps[ev.P] - if err := checkRunning(p, g, ev, true); err != nil { - return err - } - if _, ok := gs[ev.Args[0]]; ok { - return fmt.Errorf("g %d already exists (time %d)", ev.Args[0], ev.Ts) - } - gs[ev.Args[0]] = gdesc{state: gRunnable, ev: ev, evCreate: ev} - - case EvGoStart, EvGoStartLabel: - g := gs[ev.G] - p := ps[ev.P] - if g.state != gRunnable { - return fmt.Errorf("g %d is not runnable before start (time %d)", ev.G, ev.Ts) - } - if p.g != 0 { - return fmt.Errorf("p %d is already running g %d while start g %d (time %d)", ev.P, p.g, ev.G, ev.Ts) - } - g.state = gRunning - g.evStart = ev - p.g = ev.G - if g.evCreate != nil { - ev.StkID = uint32(g.evCreate.Args[1]) - g.evCreate = nil - } - - if g.ev != nil { - g.ev = nil - } - - gs[ev.G] = g - ps[ev.P] = p - case EvGoEnd, EvGoStop: - g := gs[ev.G] - p := ps[ev.P] - if err := checkRunning(p, g, ev, false); err != nil { - return err - } - g.evStart = nil - g.state = gDead - p.g = 0 - - if ev.Type == EvGoEnd { // flush all active regions - delete(activeRegions, ev.G) - } - - gs[ev.G] = g - ps[ev.P] = p - case EvGoSched, EvGoPreempt: - g := gs[ev.G] - p := ps[ev.P] - if err := checkRunning(p, g, ev, false); err != nil { - return err - } - g.state = gRunnable - g.evStart = nil - p.g = 0 - g.ev = ev - - gs[ev.G] = g - ps[ev.P] = p - case EvGoUnblock: - g := gs[ev.G] - p := ps[ev.P] - if g.state != gRunning { - return fmt.Errorf("g %d is not running while unpark (time %d)", ev.G, ev.Ts) - } - if ev.P != TimerP && p.g != ev.G { - return fmt.Errorf("p %d is not running g %d while unpark (time %d)", ev.P, ev.G, ev.Ts) - } - g1 := gs[ev.Args[0]] - if g1.state != gWaiting { - return fmt.Errorf("g %d is not waiting before unpark (time %d)", ev.Args[0], ev.Ts) - } - if g1.ev != nil && g1.ev.Type == EvGoBlockNet { - ev.P = NetpollP - } - g1.state = gRunnable - g1.ev = ev - gs[ev.Args[0]] = g1 - - case EvGoSysCall: - g := gs[ev.G] - p := ps[ev.P] - if err := checkRunning(p, g, ev, false); err != nil { - return err - } - g.ev = ev - - gs[ev.G] = g - case EvGoSysBlock: - g := gs[ev.G] - p := ps[ev.P] - if err := checkRunning(p, g, ev, false); err != nil { - return err - } - g.state = gWaiting - g.evStart = nil - p.g = 0 - - gs[ev.G] = g - ps[ev.P] = p - case EvGoSysExit: - g := gs[ev.G] - if g.state != gWaiting { - return fmt.Errorf("g %d is not waiting during syscall exit (time %d)", ev.G, ev.Ts) - } - g.state = gRunnable - g.ev = ev - - gs[ev.G] = g - case EvGoSleep, EvGoBlock, EvGoBlockSend, EvGoBlockRecv, - EvGoBlockSelect, EvGoBlockSync, EvGoBlockCond, EvGoBlockNet, EvGoBlockGC: - g := gs[ev.G] - p := ps[ev.P] - if err := checkRunning(p, g, ev, false); err != nil { - return err - } - g.state = gWaiting - g.ev = ev - g.evStart = nil - p.g = 0 - - gs[ev.G] = g - ps[ev.P] = p - case EvUserTaskCreate: - taskid := ev.Args[0] - if prevEv, ok := tasks[taskid]; ok { - return fmt.Errorf("task id conflicts (id:%d), %q vs %q", taskid, ev, prevEv) - } - tasks[ev.Args[0]] = ev - - case EvUserTaskEnd: - taskid := ev.Args[0] - delete(tasks, taskid) - - case EvUserRegion: - mode := ev.Args[1] - regions := activeRegions[ev.G] - if mode == 0 { // region start - activeRegions[ev.G] = append(regions, ev) // push - } else if mode == 1 { // region end - n := len(regions) - if n > 0 { // matching region start event is in the trace. - s := regions[n-1] - if s.Args[0] != ev.Args[0] || s.Args[2] != ev.Args[2] { // task id, region name mismatch - return fmt.Errorf("misuse of region in goroutine %d: span end %q when the inner-most active span start event is %q", ev.G, ev, s) - } - - if n > 1 { - activeRegions[ev.G] = regions[:n-1] - } else { - delete(activeRegions, ev.G) - } - } - } else { - return fmt.Errorf("invalid user region mode: %q", ev) - } - } - - if ev.StkID != 0 && len(p.stacks[ev.StkID]) == 0 { - // Make sure events don't refer to stacks that don't exist or to - // stacks with zero frames. Neither of these should be possible, but - // better be safe than sorry. - - ev.StkID = 0 - } - - } - - // TODO(mknyszek): restore stacks for EvGoStart events. - return nil -} - -var errMalformedVarint = errors.New("malformatted base-128 varint") - -// readVal reads unsigned base-128 value from r. -func (p *parser) readVal() (uint64, error) { - v, n := binary.Uvarint(p.data[p.off:]) - if n <= 0 { - return 0, errMalformedVarint - } - p.off += n - return v, nil -} - -func readValFrom(buf []byte) (v uint64, rem []byte, err error) { - v, n := binary.Uvarint(buf) - if n <= 0 { - return 0, nil, errMalformedVarint - } - return v, buf[n:], nil -} - -func (ev *Event) String() string { - desc := &EventDescriptions[ev.Type] - w := new(bytes.Buffer) - fmt.Fprintf(w, "%d %s p=%d g=%d stk=%d", ev.Ts, desc.Name, ev.P, ev.G, ev.StkID) - for i, a := range desc.Args { - fmt.Fprintf(w, " %s=%d", a, ev.Args[i]) - } - return w.String() -} - -// argNum returns total number of args for the event accounting for timestamps, -// sequence numbers and differences between trace format versions. -func (raw *rawEvent) argNum() int { - desc := &EventDescriptions[raw.typ] - if raw.typ == EvStack { - return len(raw.args) - } - narg := len(desc.Args) - if desc.Stack { - narg++ - } - switch raw.typ { - case EvBatch, EvFrequency, EvTimerGoroutine: - return narg - } - narg++ // timestamp - return narg -} - -// Event types in the trace. -// Verbatim copy from src/runtime/trace.go with the "trace" prefix removed. -const ( - EvNone event.Type = 0 // unused - EvBatch event.Type = 1 // start of per-P batch of events [pid, timestamp] - EvFrequency event.Type = 2 // contains tracer timer frequency [frequency (ticks per second)] - EvStack event.Type = 3 // stack [stack id, number of PCs, array of {PC, func string ID, file string ID, line}] - EvGomaxprocs event.Type = 4 // current value of GOMAXPROCS [timestamp, GOMAXPROCS, stack id] - EvProcStart event.Type = 5 // start of P [timestamp, thread id] - EvProcStop event.Type = 6 // stop of P [timestamp] - EvGCStart event.Type = 7 // GC start [timestamp, seq, stack id] - EvGCDone event.Type = 8 // GC done [timestamp] - EvSTWStart event.Type = 9 // GC mark termination start [timestamp, kind] - EvSTWDone event.Type = 10 // GC mark termination done [timestamp] - EvGCSweepStart event.Type = 11 // GC sweep start [timestamp, stack id] - EvGCSweepDone event.Type = 12 // GC sweep done [timestamp, swept, reclaimed] - EvGoCreate event.Type = 13 // goroutine creation [timestamp, new goroutine id, new stack id, stack id] - EvGoStart event.Type = 14 // goroutine starts running [timestamp, goroutine id, seq] - EvGoEnd event.Type = 15 // goroutine ends [timestamp] - EvGoStop event.Type = 16 // goroutine stops (like in select{}) [timestamp, stack] - EvGoSched event.Type = 17 // goroutine calls Gosched [timestamp, stack] - EvGoPreempt event.Type = 18 // goroutine is preempted [timestamp, stack] - EvGoSleep event.Type = 19 // goroutine calls Sleep [timestamp, stack] - EvGoBlock event.Type = 20 // goroutine blocks [timestamp, stack] - EvGoUnblock event.Type = 21 // goroutine is unblocked [timestamp, goroutine id, seq, stack] - EvGoBlockSend event.Type = 22 // goroutine blocks on chan send [timestamp, stack] - EvGoBlockRecv event.Type = 23 // goroutine blocks on chan recv [timestamp, stack] - EvGoBlockSelect event.Type = 24 // goroutine blocks on select [timestamp, stack] - EvGoBlockSync event.Type = 25 // goroutine blocks on Mutex/RWMutex [timestamp, stack] - EvGoBlockCond event.Type = 26 // goroutine blocks on Cond [timestamp, stack] - EvGoBlockNet event.Type = 27 // goroutine blocks on network [timestamp, stack] - EvGoSysCall event.Type = 28 // syscall enter [timestamp, stack] - EvGoSysExit event.Type = 29 // syscall exit [timestamp, goroutine id, seq, real timestamp] - EvGoSysBlock event.Type = 30 // syscall blocks [timestamp] - EvGoWaiting event.Type = 31 // denotes that goroutine is blocked when tracing starts [timestamp, goroutine id] - EvGoInSyscall event.Type = 32 // denotes that goroutine is in syscall when tracing starts [timestamp, goroutine id] - EvHeapAlloc event.Type = 33 // gcController.heapLive change [timestamp, heap live bytes] - EvHeapGoal event.Type = 34 // gcController.heapGoal change [timestamp, heap goal bytes] - EvTimerGoroutine event.Type = 35 // denotes timer goroutine [timer goroutine id] - EvFutileWakeup event.Type = 36 // denotes that the previous wakeup of this goroutine was futile [timestamp] - EvString event.Type = 37 // string dictionary entry [ID, length, string] - EvGoStartLocal event.Type = 38 // goroutine starts running on the same P as the last event [timestamp, goroutine id] - EvGoUnblockLocal event.Type = 39 // goroutine is unblocked on the same P as the last event [timestamp, goroutine id, stack] - EvGoSysExitLocal event.Type = 40 // syscall exit on the same P as the last event [timestamp, goroutine id, real timestamp] - EvGoStartLabel event.Type = 41 // goroutine starts running with label [timestamp, goroutine id, seq, label string id] - EvGoBlockGC event.Type = 42 // goroutine blocks on GC assist [timestamp, stack] - EvGCMarkAssistStart event.Type = 43 // GC mark assist start [timestamp, stack] - EvGCMarkAssistDone event.Type = 44 // GC mark assist done [timestamp] - EvUserTaskCreate event.Type = 45 // trace.NewTask [timestamp, internal task id, internal parent id, stack, name string] - EvUserTaskEnd event.Type = 46 // end of task [timestamp, internal task id, stack] - EvUserRegion event.Type = 47 // trace.WithRegion [timestamp, internal task id, mode(0:start, 1:end), name string] - EvUserLog event.Type = 48 // trace.Log [timestamp, internal id, key string id, stack, value string] - EvCPUSample event.Type = 49 // CPU profiling sample [timestamp, stack, real timestamp, real P id (-1 when absent), goroutine id] - EvCount event.Type = 50 -) - -var EventDescriptions = [256]struct { - Name string - minVersion version.Version - Stack bool - Args []string - SArgs []string // string arguments -}{ - EvNone: {"None", 5, false, []string{}, nil}, - EvBatch: {"Batch", 5, false, []string{"p", "ticks"}, nil}, // in 1.5 format it was {"p", "seq", "ticks"} - EvFrequency: {"Frequency", 5, false, []string{"freq"}, nil}, // in 1.5 format it was {"freq", "unused"} - EvStack: {"Stack", 5, false, []string{"id", "siz"}, nil}, - EvGomaxprocs: {"Gomaxprocs", 5, true, []string{"procs"}, nil}, - EvProcStart: {"ProcStart", 5, false, []string{"thread"}, nil}, - EvProcStop: {"ProcStop", 5, false, []string{}, nil}, - EvGCStart: {"GCStart", 5, true, []string{"seq"}, nil}, // in 1.5 format it was {} - EvGCDone: {"GCDone", 5, false, []string{}, nil}, - EvSTWStart: {"GCSTWStart", 5, false, []string{"kindid"}, []string{"kind"}}, // <= 1.9, args was {} (implicitly {0}) - EvSTWDone: {"GCSTWDone", 5, false, []string{}, nil}, - EvGCSweepStart: {"GCSweepStart", 5, true, []string{}, nil}, - EvGCSweepDone: {"GCSweepDone", 5, false, []string{"swept", "reclaimed"}, nil}, // before 1.9, format was {} - EvGoCreate: {"GoCreate", 5, true, []string{"g", "stack"}, nil}, - EvGoStart: {"GoStart", 5, false, []string{"g", "seq"}, nil}, // in 1.5 format it was {"g"} - EvGoEnd: {"GoEnd", 5, false, []string{}, nil}, - EvGoStop: {"GoStop", 5, true, []string{}, nil}, - EvGoSched: {"GoSched", 5, true, []string{}, nil}, - EvGoPreempt: {"GoPreempt", 5, true, []string{}, nil}, - EvGoSleep: {"GoSleep", 5, true, []string{}, nil}, - EvGoBlock: {"GoBlock", 5, true, []string{}, nil}, - EvGoUnblock: {"GoUnblock", 5, true, []string{"g", "seq"}, nil}, // in 1.5 format it was {"g"} - EvGoBlockSend: {"GoBlockSend", 5, true, []string{}, nil}, - EvGoBlockRecv: {"GoBlockRecv", 5, true, []string{}, nil}, - EvGoBlockSelect: {"GoBlockSelect", 5, true, []string{}, nil}, - EvGoBlockSync: {"GoBlockSync", 5, true, []string{}, nil}, - EvGoBlockCond: {"GoBlockCond", 5, true, []string{}, nil}, - EvGoBlockNet: {"GoBlockNet", 5, true, []string{}, nil}, - EvGoSysCall: {"GoSysCall", 5, true, []string{}, nil}, - EvGoSysExit: {"GoSysExit", 5, false, []string{"g", "seq", "ts"}, nil}, - EvGoSysBlock: {"GoSysBlock", 5, false, []string{}, nil}, - EvGoWaiting: {"GoWaiting", 5, false, []string{"g"}, nil}, - EvGoInSyscall: {"GoInSyscall", 5, false, []string{"g"}, nil}, - EvHeapAlloc: {"HeapAlloc", 5, false, []string{"mem"}, nil}, - EvHeapGoal: {"HeapGoal", 5, false, []string{"mem"}, nil}, - EvTimerGoroutine: {"TimerGoroutine", 5, false, []string{"g"}, nil}, // in 1.5 format it was {"g", "unused"} - EvFutileWakeup: {"FutileWakeup", 5, false, []string{}, nil}, - EvString: {"String", 7, false, []string{}, nil}, - EvGoStartLocal: {"GoStartLocal", 7, false, []string{"g"}, nil}, - EvGoUnblockLocal: {"GoUnblockLocal", 7, true, []string{"g"}, nil}, - EvGoSysExitLocal: {"GoSysExitLocal", 7, false, []string{"g", "ts"}, nil}, - EvGoStartLabel: {"GoStartLabel", 8, false, []string{"g", "seq", "labelid"}, []string{"label"}}, - EvGoBlockGC: {"GoBlockGC", 8, true, []string{}, nil}, - EvGCMarkAssistStart: {"GCMarkAssistStart", 9, true, []string{}, nil}, - EvGCMarkAssistDone: {"GCMarkAssistDone", 9, false, []string{}, nil}, - EvUserTaskCreate: {"UserTaskCreate", 11, true, []string{"taskid", "pid", "typeid"}, []string{"name"}}, - EvUserTaskEnd: {"UserTaskEnd", 11, true, []string{"taskid"}, nil}, - EvUserRegion: {"UserRegion", 11, true, []string{"taskid", "mode", "typeid"}, []string{"name"}}, - EvUserLog: {"UserLog", 11, true, []string{"id", "keyid"}, []string{"category", "message"}}, - EvCPUSample: {"CPUSample", 19, true, []string{"ts", "p", "g"}, nil}, -} - -//gcassert:inline -func (p *parser) allocateStack(size uint64) []uint64 { - if size == 0 { - return nil - } - - // Stacks are plentiful but small. For our "Staticcheck on std" trace with - // 11e6 events, we have roughly 500,000 stacks, using 200 MiB of memory. To - // avoid making 500,000 small allocations we allocate backing arrays 1 MiB - // at a time. - out := p.stacksData - if uint64(len(out)) < size { - out = make([]uint64, 1024*128) - } - p.stacksData = out[size:] - return out[:size:size] -} - -func (tr *Trace) STWReason(kindID uint64) STWReason { - if tr.Version < 21 { - if kindID == 0 || kindID == 1 { - return STWReason(kindID + 1) - } else { - return STWUnknown - } - } else if tr.Version == 21 { - if kindID < NumSTWReasons { - return STWReason(kindID) - } else { - return STWUnknown - } - } else { - return STWUnknown - } -} - -type STWReason int - -const ( - STWUnknown STWReason = 0 - STWGCMarkTermination STWReason = 1 - STWGCSweepTermination STWReason = 2 - STWWriteHeapDump STWReason = 3 - STWGoroutineProfile STWReason = 4 - STWGoroutineProfileCleanup STWReason = 5 - STWAllGoroutinesStackTrace STWReason = 6 - STWReadMemStats STWReason = 7 - STWAllThreadsSyscall STWReason = 8 - STWGOMAXPROCS STWReason = 9 - STWStartTrace STWReason = 10 - STWStopTrace STWReason = 11 - STWCountPagesInUse STWReason = 12 - STWReadMetricsSlow STWReason = 13 - STWReadMemStatsSlow STWReason = 14 - STWPageCachePagesLeaked STWReason = 15 - STWResetDebugLog STWReason = 16 - - NumSTWReasons = 17 -) diff --git a/src/internal/trace/internal/testgen/go122/trace.go b/src/internal/trace/internal/testgen/go122/trace.go deleted file mode 100644 index 5fd995ae9aeee3..00000000000000 --- a/src/internal/trace/internal/testgen/go122/trace.go +++ /dev/null @@ -1,401 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package testkit - -import ( - "bytes" - "encoding/binary" - "fmt" - "os" - "regexp" - "strings" - - "internal/trace" - "internal/trace/event" - "internal/trace/event/go122" - "internal/trace/raw" - "internal/trace/version" - "internal/txtar" -) - -func Main(f func(*Trace)) { - // Create an output file. - out, err := os.Create(os.Args[1]) - if err != nil { - panic(err.Error()) - } - defer out.Close() - - // Create a new trace. - trace := NewTrace() - - // Call the generator. - f(trace) - - // Write out the generator's state. - if _, err := out.Write(trace.Generate()); err != nil { - panic(err.Error()) - } -} - -// Trace represents an execution trace for testing. -// -// It does a little bit of work to ensure that the produced trace is valid, -// just for convenience. It mainly tracks batches and batch sizes (so they're -// trivially correct), tracks strings and stacks, and makes sure emitted string -// and stack batches are valid. That last part can be controlled by a few options. -// -// Otherwise, it performs no validation on the trace at all. -type Trace struct { - // Trace data state. - ver version.Version - names map[string]event.Type - specs []event.Spec - events []raw.Event - gens []*Generation - validTimestamps bool - - // Expectation state. - bad bool - badMatch *regexp.Regexp -} - -// NewTrace creates a new trace. -func NewTrace() *Trace { - ver := version.Go122 - return &Trace{ - names: event.Names(ver.Specs()), - specs: ver.Specs(), - validTimestamps: true, - } -} - -// ExpectFailure writes down that the trace should be broken. The caller -// must provide a pattern matching the expected error produced by the parser. -func (t *Trace) ExpectFailure(pattern string) { - t.bad = true - t.badMatch = regexp.MustCompile(pattern) -} - -// ExpectSuccess writes down that the trace should successfully parse. -func (t *Trace) ExpectSuccess() { - t.bad = false -} - -// RawEvent emits an event into the trace. name must correspond to one -// of the names in Specs() result for the version that was passed to -// this trace. -func (t *Trace) RawEvent(typ event.Type, data []byte, args ...uint64) { - t.events = append(t.events, t.createEvent(typ, data, args...)) -} - -// DisableTimestamps makes the timestamps for all events generated after -// this call zero. Raw events are exempted from this because the caller -// has to pass their own timestamp into those events anyway. -func (t *Trace) DisableTimestamps() { - t.validTimestamps = false -} - -// Generation creates a new trace generation. -// -// This provides more structure than Event to allow for more easily -// creating complex traces that are mostly or completely correct. -func (t *Trace) Generation(gen uint64) *Generation { - g := &Generation{ - trace: t, - gen: gen, - strings: make(map[string]uint64), - stacks: make(map[stack]uint64), - } - t.gens = append(t.gens, g) - return g -} - -// Generate creates a test file for the trace. -func (t *Trace) Generate() []byte { - // Trace file contents. - var buf bytes.Buffer - tw, err := raw.NewTextWriter(&buf, version.Go122) - if err != nil { - panic(err.Error()) - } - - // Write raw top-level events. - for _, e := range t.events { - tw.WriteEvent(e) - } - - // Write generations. - for _, g := range t.gens { - g.writeEventsTo(tw) - } - - // Expectation file contents. - expect := []byte("SUCCESS\n") - if t.bad { - expect = []byte(fmt.Sprintf("FAILURE %q\n", t.badMatch)) - } - - // Create the test file's contents. - return txtar.Format(&txtar.Archive{ - Files: []txtar.File{ - {Name: "expect", Data: expect}, - {Name: "trace", Data: buf.Bytes()}, - }, - }) -} - -func (t *Trace) createEvent(ev event.Type, data []byte, args ...uint64) raw.Event { - spec := t.specs[ev] - if ev != go122.EvStack { - if arity := len(spec.Args); len(args) != arity { - panic(fmt.Sprintf("expected %d args for %s, got %d", arity, spec.Name, len(args))) - } - } - return raw.Event{ - Version: version.Go122, - Ev: ev, - Args: args, - Data: data, - } -} - -type stack struct { - stk [32]trace.StackFrame - len int -} - -var ( - NoString = "" - NoStack = []trace.StackFrame{} -) - -// Generation represents a single generation in the trace. -type Generation struct { - trace *Trace - gen uint64 - batches []*Batch - strings map[string]uint64 - stacks map[stack]uint64 - - // Options applied when Trace.Generate is called. - ignoreStringBatchSizeLimit bool - ignoreStackBatchSizeLimit bool -} - -// Batch starts a new event batch in the trace data. -// -// This is convenience function for generating correct batches. -func (g *Generation) Batch(thread trace.ThreadID, time Time) *Batch { - if !g.trace.validTimestamps { - time = 0 - } - b := &Batch{ - gen: g, - thread: thread, - timestamp: time, - } - g.batches = append(g.batches, b) - return b -} - -// String registers a string with the trace. -// -// This is a convenience function for easily adding correct -// strings to traces. -func (g *Generation) String(s string) uint64 { - if len(s) == 0 { - return 0 - } - if id, ok := g.strings[s]; ok { - return id - } - id := uint64(len(g.strings) + 1) - g.strings[s] = id - return id -} - -// Stack registers a stack with the trace. -// -// This is a convenience function for easily adding correct -// stacks to traces. -func (g *Generation) Stack(stk []trace.StackFrame) uint64 { - if len(stk) == 0 { - return 0 - } - if len(stk) > 32 { - panic("stack too big for test") - } - var stkc stack - copy(stkc.stk[:], stk) - stkc.len = len(stk) - if id, ok := g.stacks[stkc]; ok { - return id - } - id := uint64(len(g.stacks) + 1) - g.stacks[stkc] = id - return id -} - -// writeEventsTo emits event batches in the generation to tw. -func (g *Generation) writeEventsTo(tw *raw.TextWriter) { - // Write event batches for the generation. - for _, b := range g.batches { - b.writeEventsTo(tw) - } - - // Write frequency. - b := g.newStructuralBatch() - b.RawEvent(go122.EvFrequency, nil, 15625000) - b.writeEventsTo(tw) - - // Write stacks. - b = g.newStructuralBatch() - b.RawEvent(go122.EvStacks, nil) - for stk, id := range g.stacks { - stk := stk.stk[:stk.len] - args := []uint64{id} - for _, f := range stk { - args = append(args, f.PC, g.String(f.Func), g.String(f.File), f.Line) - } - b.RawEvent(go122.EvStack, nil, args...) - - // Flush the batch if necessary. - if !g.ignoreStackBatchSizeLimit && b.size > go122.MaxBatchSize/2 { - b.writeEventsTo(tw) - b = g.newStructuralBatch() - } - } - b.writeEventsTo(tw) - - // Write strings. - b = g.newStructuralBatch() - b.RawEvent(go122.EvStrings, nil) - for s, id := range g.strings { - b.RawEvent(go122.EvString, []byte(s), id) - - // Flush the batch if necessary. - if !g.ignoreStringBatchSizeLimit && b.size > go122.MaxBatchSize/2 { - b.writeEventsTo(tw) - b = g.newStructuralBatch() - } - } - b.writeEventsTo(tw) -} - -func (g *Generation) newStructuralBatch() *Batch { - return &Batch{gen: g, thread: trace.NoThread} -} - -// Batch represents an event batch. -type Batch struct { - gen *Generation - thread trace.ThreadID - timestamp Time - size uint64 - events []raw.Event -} - -// Event emits an event into a batch. name must correspond to one -// of the names in Specs() result for the version that was passed to -// this trace. Callers must omit the timestamp delta. -func (b *Batch) Event(name string, args ...any) { - ev, ok := b.gen.trace.names[name] - if !ok { - panic(fmt.Sprintf("invalid or unknown event %s", name)) - } - var uintArgs []uint64 - argOff := 0 - if b.gen.trace.specs[ev].IsTimedEvent { - if b.gen.trace.validTimestamps { - uintArgs = []uint64{1} - } else { - uintArgs = []uint64{0} - } - argOff = 1 - } - spec := b.gen.trace.specs[ev] - if arity := len(spec.Args) - argOff; len(args) != arity { - panic(fmt.Sprintf("expected %d args for %s, got %d", arity, spec.Name, len(args))) - } - for i, arg := range args { - uintArgs = append(uintArgs, b.uintArgFor(arg, spec.Args[i+argOff])) - } - b.RawEvent(ev, nil, uintArgs...) -} - -func (b *Batch) uintArgFor(arg any, argSpec string) uint64 { - components := strings.SplitN(argSpec, "_", 2) - typStr := components[0] - if len(components) == 2 { - typStr = components[1] - } - var u uint64 - switch typStr { - case "value": - u = arg.(uint64) - case "stack": - u = b.gen.Stack(arg.([]trace.StackFrame)) - case "seq": - u = uint64(arg.(Seq)) - case "pstatus": - u = uint64(arg.(go122.ProcStatus)) - case "gstatus": - u = uint64(arg.(go122.GoStatus)) - case "g": - u = uint64(arg.(trace.GoID)) - case "m": - u = uint64(arg.(trace.ThreadID)) - case "p": - u = uint64(arg.(trace.ProcID)) - case "string": - u = b.gen.String(arg.(string)) - case "task": - u = uint64(arg.(trace.TaskID)) - default: - panic(fmt.Sprintf("unsupported arg type %q for spec %q", typStr, argSpec)) - } - return u -} - -// RawEvent emits an event into a batch. name must correspond to one -// of the names in Specs() result for the version that was passed to -// this trace. -func (b *Batch) RawEvent(typ event.Type, data []byte, args ...uint64) { - ev := b.gen.trace.createEvent(typ, data, args...) - - // Compute the size of the event and add it to the batch. - b.size += 1 // One byte for the event header. - var buf [binary.MaxVarintLen64]byte - for _, arg := range args { - b.size += uint64(binary.PutUvarint(buf[:], arg)) - } - if len(data) != 0 { - b.size += uint64(binary.PutUvarint(buf[:], uint64(len(data)))) - b.size += uint64(len(data)) - } - - // Add the event. - b.events = append(b.events, ev) -} - -// writeEventsTo emits events in the batch, including the batch header, to tw. -func (b *Batch) writeEventsTo(tw *raw.TextWriter) { - tw.WriteEvent(raw.Event{ - Version: version.Go122, - Ev: go122.EvEventBatch, - Args: []uint64{b.gen.gen, uint64(b.thread), uint64(b.timestamp), b.size}, - }) - for _, e := range b.events { - tw.WriteEvent(e) - } -} - -// Seq represents a sequence counter. -type Seq uint64 - -// Time represents a low-level trace timestamp (which does not necessarily -// correspond to nanoseconds, like trace.Time does). -type Time uint64 diff --git a/src/internal/trace/internal/testgen/trace.go b/src/internal/trace/internal/testgen/trace.go new file mode 100644 index 00000000000000..2eade48de70e00 --- /dev/null +++ b/src/internal/trace/internal/testgen/trace.go @@ -0,0 +1,459 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testgen + +import ( + "bytes" + "encoding/binary" + "fmt" + "os" + "regexp" + "strings" + "time" + + "internal/trace" + "internal/trace/raw" + "internal/trace/tracev2" + "internal/trace/version" + "internal/txtar" +) + +func Main(ver version.Version, f func(*Trace)) { + // Create an output file. + out, err := os.Create(os.Args[1]) + if err != nil { + panic(err.Error()) + } + defer out.Close() + + // Create a new trace. + trace := NewTrace(ver) + + // Call the generator. + f(trace) + + // Write out the generator's state. + if _, err := out.Write(trace.Generate()); err != nil { + panic(err.Error()) + } +} + +// Trace represents an execution trace for testing. +// +// It does a little bit of work to ensure that the produced trace is valid, +// just for convenience. It mainly tracks batches and batch sizes (so they're +// trivially correct), tracks strings and stacks, and makes sure emitted string +// and stack batches are valid. That last part can be controlled by a few options. +// +// Otherwise, it performs no validation on the trace at all. +type Trace struct { + // Trace data state. + ver version.Version + names map[string]tracev2.EventType + specs []tracev2.EventSpec + events []raw.Event + gens []*Generation + validTimestamps bool + lastTs Time + + // Expectation state. + bad bool + badMatch *regexp.Regexp +} + +// NewTrace creates a new trace. +func NewTrace(ver version.Version) *Trace { + return &Trace{ + names: tracev2.EventNames(ver.Specs()), + specs: ver.Specs(), + ver: ver, + validTimestamps: true, + } +} + +// ExpectFailure writes down that the trace should be broken. The caller +// must provide a pattern matching the expected error produced by the parser. +func (t *Trace) ExpectFailure(pattern string) { + t.bad = true + t.badMatch = regexp.MustCompile(pattern) +} + +// ExpectSuccess writes down that the trace should successfully parse. +func (t *Trace) ExpectSuccess() { + t.bad = false +} + +// RawEvent emits an event into the trace. name must correspond to one +// of the names in Specs() result for the version that was passed to +// this trace. +func (t *Trace) RawEvent(typ tracev2.EventType, data []byte, args ...uint64) { + t.events = append(t.events, t.createEvent(typ, data, args...)) +} + +// DisableTimestamps makes the timestamps for all events generated after +// this call zero. Raw events are exempted from this because the caller +// has to pass their own timestamp into those events anyway. +func (t *Trace) DisableTimestamps() { + t.validTimestamps = false +} + +// Generation creates a new trace generation. +// +// This provides more structure than Event to allow for more easily +// creating complex traces that are mostly or completely correct. +func (t *Trace) Generation(gen uint64) *Generation { + g := &Generation{ + trace: t, + gen: gen, + strings: make(map[string]uint64), + stacks: make(map[stack]uint64), + sync: sync{freq: 15625000}, + } + t.gens = append(t.gens, g) + return g +} + +// Generate creates a test file for the trace. +func (t *Trace) Generate() []byte { + // Trace file contents. + var buf bytes.Buffer + tw, err := raw.NewTextWriter(&buf, t.ver) + if err != nil { + panic(err.Error()) + } + + // Write raw top-level events. + for _, e := range t.events { + tw.WriteEvent(e) + } + + // Write generations. + for _, g := range t.gens { + g.writeEventsTo(tw) + } + + // Expectation file contents. + expect := []byte("SUCCESS\n") + if t.bad { + expect = []byte(fmt.Sprintf("FAILURE %q\n", t.badMatch)) + } + + // Create the test file's contents. + return txtar.Format(&txtar.Archive{ + Files: []txtar.File{ + {Name: "expect", Data: expect}, + {Name: "trace", Data: buf.Bytes()}, + }, + }) +} + +func (t *Trace) createEvent(ev tracev2.EventType, data []byte, args ...uint64) raw.Event { + spec := t.specs[ev] + if ev != tracev2.EvStack { + if arity := len(spec.Args); len(args) != arity { + panic(fmt.Sprintf("expected %d args for %s, got %d", arity, spec.Name, len(args))) + } + } + return raw.Event{ + Version: t.ver, + Ev: ev, + Args: args, + Data: data, + } +} + +type stack struct { + stk [32]trace.StackFrame + len int +} + +var ( + NoString = "" + NoStack = []trace.StackFrame{} +) + +// Generation represents a single generation in the trace. +type Generation struct { + trace *Trace + gen uint64 + batches []*Batch + strings map[string]uint64 + stacks map[stack]uint64 + sync sync + + // Options applied when Trace.Generate is called. + ignoreStringBatchSizeLimit bool + ignoreStackBatchSizeLimit bool +} + +// Batch starts a new event batch in the trace data. +// +// This is convenience function for generating correct batches. +func (g *Generation) Batch(thread trace.ThreadID, time Time) *Batch { + b := &Batch{ + gen: g, + thread: thread, + } + b.setTimestamp(time) + g.batches = append(g.batches, b) + return b +} + +// String registers a string with the trace. +// +// This is a convenience function for easily adding correct +// strings to traces. +func (g *Generation) String(s string) uint64 { + if len(s) == 0 { + return 0 + } + if id, ok := g.strings[s]; ok { + return id + } + id := uint64(len(g.strings) + 1) + g.strings[s] = id + return id +} + +// Stack registers a stack with the trace. +// +// This is a convenience function for easily adding correct +// stacks to traces. +func (g *Generation) Stack(stk []trace.StackFrame) uint64 { + if len(stk) == 0 { + return 0 + } + if len(stk) > 32 { + panic("stack too big for test") + } + var stkc stack + copy(stkc.stk[:], stk) + stkc.len = len(stk) + if id, ok := g.stacks[stkc]; ok { + return id + } + id := uint64(len(g.stacks) + 1) + g.stacks[stkc] = id + return id +} + +// Sync configures the sync batch for the generation. For go1.25 and later, +// the time value is the timestamp of the EvClockSnapshot event. For earlier +// version, the time value is the timestamp of the batch containing a lone +// EvFrequency event. +func (g *Generation) Sync(freq uint64, time Time, mono uint64, wall time.Time) { + if g.trace.ver < version.Go125 && (mono != 0 || !wall.IsZero()) { + panic(fmt.Sprintf("mono and wall args are not supported in go1.%d traces", g.trace.ver)) + } + g.sync = sync{ + freq: freq, + time: time, + mono: mono, + walltime: wall, + } +} + +type sync struct { + freq uint64 + time Time + mono uint64 + walltime time.Time +} + +// writeEventsTo emits event batches in the generation to tw. +func (g *Generation) writeEventsTo(tw *raw.TextWriter) { + // go1.25+ sync batches are emitted at the start of the generation. + if g.trace.ver >= version.Go125 { + b := g.newStructuralBatch() + // Arrange for EvClockSnapshot's ts to be exactly g.sync.time. + b.setTimestamp(g.sync.time - 1) + b.RawEvent(tracev2.EvSync, nil) + b.RawEvent(tracev2.EvFrequency, nil, g.sync.freq) + sec := uint64(g.sync.walltime.Unix()) + nsec := uint64(g.sync.walltime.Nanosecond()) + b.Event("ClockSnapshot", g.sync.mono, sec, nsec) + b.writeEventsTo(tw) + } + + // Write event batches for the generation. + for _, b := range g.batches { + b.writeEventsTo(tw) + } + + // Write lone EvFrequency sync batch for older traces. + if g.trace.ver < version.Go125 { + b := g.newStructuralBatch() + b.setTimestamp(g.sync.time) + b.RawEvent(tracev2.EvFrequency, nil, g.sync.freq) + b.writeEventsTo(tw) + } + + // Write stacks. + b := g.newStructuralBatch() + b.RawEvent(tracev2.EvStacks, nil) + for stk, id := range g.stacks { + stk := stk.stk[:stk.len] + args := []uint64{id, uint64(len(stk))} + for _, f := range stk { + args = append(args, f.PC, g.String(f.Func), g.String(f.File), f.Line) + } + b.RawEvent(tracev2.EvStack, nil, args...) + + // Flush the batch if necessary. + if !g.ignoreStackBatchSizeLimit && b.size > tracev2.MaxBatchSize/2 { + b.writeEventsTo(tw) + b = g.newStructuralBatch() + } + } + b.writeEventsTo(tw) + + // Write strings. + b = g.newStructuralBatch() + b.RawEvent(tracev2.EvStrings, nil) + for s, id := range g.strings { + b.RawEvent(tracev2.EvString, []byte(s), id) + + // Flush the batch if necessary. + if !g.ignoreStringBatchSizeLimit && b.size > tracev2.MaxBatchSize/2 { + b.writeEventsTo(tw) + b = g.newStructuralBatch() + } + } + b.writeEventsTo(tw) + + // Write end-of-generation event if necessary. + if g.trace.ver >= version.Go126 { + tw.WriteEvent(raw.Event{ + Version: g.trace.ver, + Ev: tracev2.EvEndOfGeneration, + }) + } +} + +func (g *Generation) newStructuralBatch() *Batch { + b := &Batch{gen: g, thread: trace.NoThread} + b.setTimestamp(g.trace.lastTs + 1) + return b +} + +// Batch represents an event batch. +type Batch struct { + gen *Generation + thread trace.ThreadID + timestamp Time + size uint64 + events []raw.Event +} + +// Event emits an event into a batch. name must correspond to one +// of the names in Specs() result for the version that was passed to +// this trace. Callers must omit the timestamp delta. +func (b *Batch) Event(name string, args ...any) { + ev, ok := b.gen.trace.names[name] + if !ok { + panic(fmt.Sprintf("invalid or unknown event %s", name)) + } + var uintArgs []uint64 + argOff := 0 + if b.gen.trace.specs[ev].IsTimedEvent { + if b.gen.trace.validTimestamps { + uintArgs = []uint64{1} + b.gen.trace.lastTs += 1 + } else { + uintArgs = []uint64{0} + } + argOff = 1 + } + spec := b.gen.trace.specs[ev] + if arity := len(spec.Args) - argOff; len(args) != arity { + panic(fmt.Sprintf("expected %d args for %s, got %d", arity, spec.Name, len(args))) + } + for i, arg := range args { + uintArgs = append(uintArgs, b.uintArgFor(arg, spec.Args[i+argOff])) + } + b.RawEvent(ev, nil, uintArgs...) +} + +func (b *Batch) uintArgFor(arg any, argSpec string) uint64 { + components := strings.SplitN(argSpec, "_", 2) + typStr := components[0] + if len(components) == 2 { + typStr = components[1] + } + var u uint64 + switch typStr { + case "value", "mono", "sec", "nsec": + u = arg.(uint64) + case "stack": + u = b.gen.Stack(arg.([]trace.StackFrame)) + case "seq": + u = uint64(arg.(Seq)) + case "pstatus": + u = uint64(arg.(tracev2.ProcStatus)) + case "gstatus": + u = uint64(arg.(tracev2.GoStatus)) + case "g": + u = uint64(arg.(trace.GoID)) + case "m": + u = uint64(arg.(trace.ThreadID)) + case "p": + u = uint64(arg.(trace.ProcID)) + case "string": + u = b.gen.String(arg.(string)) + case "task": + u = uint64(arg.(trace.TaskID)) + default: + panic(fmt.Sprintf("unsupported arg type %q for spec %q", typStr, argSpec)) + } + return u +} + +// RawEvent emits an event into a batch. name must correspond to one +// of the names in Specs() result for the version that was passed to +// this trace. +func (b *Batch) RawEvent(typ tracev2.EventType, data []byte, args ...uint64) { + ev := b.gen.trace.createEvent(typ, data, args...) + + // Compute the size of the event and add it to the batch. + b.size += 1 // One byte for the event header. + var buf [binary.MaxVarintLen64]byte + for _, arg := range args { + b.size += uint64(binary.PutUvarint(buf[:], arg)) + } + if len(data) != 0 { + b.size += uint64(binary.PutUvarint(buf[:], uint64(len(data)))) + b.size += uint64(len(data)) + } + + // Add the event. + b.events = append(b.events, ev) +} + +// writeEventsTo emits events in the batch, including the batch header, to tw. +func (b *Batch) writeEventsTo(tw *raw.TextWriter) { + tw.WriteEvent(raw.Event{ + Version: b.gen.trace.ver, + Ev: tracev2.EvEventBatch, + Args: []uint64{b.gen.gen, uint64(b.thread), uint64(b.timestamp), b.size}, + }) + for _, e := range b.events { + tw.WriteEvent(e) + } +} + +// setTimestamp sets the timestamp for the batch. +func (b *Batch) setTimestamp(t Time) { + if b.gen.trace.validTimestamps { + b.timestamp = t + b.gen.trace.lastTs = t + } +} + +// Seq represents a sequence counter. +type Seq uint64 + +// Time represents a low-level trace timestamp (which does not necessarily +// correspond to nanoseconds, like trace.Time does). +type Time uint64 diff --git a/src/internal/trace/internal/oldtrace/order.go b/src/internal/trace/internal/tracev1/order.go similarity index 99% rename from src/internal/trace/internal/oldtrace/order.go rename to src/internal/trace/internal/tracev1/order.go index b9d63e2f0c22f0..683d7f03b47f2b 100644 --- a/src/internal/trace/internal/oldtrace/order.go +++ b/src/internal/trace/internal/tracev1/order.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package oldtrace +package tracev1 import "errors" diff --git a/src/internal/trace/internal/tracev1/parser.go b/src/internal/trace/internal/tracev1/parser.go new file mode 100644 index 00000000000000..d47de9088a992b --- /dev/null +++ b/src/internal/trace/internal/tracev1/parser.go @@ -0,0 +1,1553 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package tracev1 implements a parser for Go execution traces from versions +// 1.11–1.21. +// +// The package started as a copy of Go 1.19's internal/trace, but has been +// optimized to be faster while using less memory and fewer allocations. It has +// been further modified for the specific purpose of converting traces to the +// new 1.22+ format. +package tracev1 + +import ( + "bytes" + "cmp" + "encoding/binary" + "errors" + "fmt" + "internal/trace/version" + "io" + "math" + "slices" + "sort" +) + +// Timestamp represents a count of nanoseconds since the beginning of the trace. +// They can only be meaningfully compared with other timestamps from the same +// trace. +type Timestamp int64 + +// Event describes one event in the trace. +type Event struct { + // The Event type is carefully laid out to optimize its size and to avoid + // pointers, the latter so that the garbage collector won't have to scan any + // memory of our millions of events. + + Ts Timestamp // timestamp in nanoseconds + G uint64 // G on which the event happened + Args [4]uint64 // event-type-specific arguments + StkID uint32 // unique stack ID + P int32 // P on which the event happened (can be a real P or one of TimerP, NetpollP, SyscallP) + Type EventType // one of Ev* +} + +// Frame is a frame in stack traces. +type Frame struct { + PC uint64 + // string ID of the function name + Fn uint64 + // string ID of the file name + File uint64 + Line int +} + +const ( + // Special P identifiers: + FakeP = 1000000 + iota + TimerP // contains timer unblocks + NetpollP // contains network unblocks + SyscallP // contains returns from syscalls + GCP // contains GC state + ProfileP // contains recording of CPU profile samples +) + +// Trace is the result of Parse. +type Trace struct { + Version version.Version + + // Events is the sorted list of Events in the trace. + Events Events + // Stacks is the stack traces (stored as slices of PCs), keyed by stack IDs + // from the trace. + Stacks map[uint32][]uint64 + PCs map[uint64]Frame + Strings map[uint64]string + InlineStrings []string +} + +// batchOffset records the byte offset of, and number of events in, a batch. A +// batch is a sequence of events emitted by a P. Events within a single batch +// are sorted by time. +type batchOffset struct { + offset int + numEvents int +} + +type parser struct { + ver version.Version + data []byte + off int + + strings map[uint64]string + inlineStrings []string + inlineStringsMapping map[string]int + // map from Ps to their batch offsets + batchOffsets map[int32][]batchOffset + stacks map[uint32][]uint64 + stacksData []uint64 + ticksPerSec int64 + pcs map[uint64]Frame + cpuSamples []Event + timerGoids map[uint64]bool + + // state for readRawEvent + args []uint64 + + // state for parseEvent + lastTs Timestamp + lastG uint64 + // map from Ps to the last Gs that ran on them + lastGs map[int32]uint64 + lastP int32 +} + +func (p *parser) discard(n uint64) bool { + if n > math.MaxInt { + return false + } + if noff := p.off + int(n); noff < p.off || noff > len(p.data) { + return false + } else { + p.off = noff + } + return true +} + +func newParser(r io.Reader, ver version.Version) (*parser, error) { + var buf []byte + if seeker, ok := r.(io.Seeker); ok { + // Determine the size of the reader so that we can allocate a buffer + // without having to grow it later. + cur, err := seeker.Seek(0, io.SeekCurrent) + if err != nil { + return nil, err + } + end, err := seeker.Seek(0, io.SeekEnd) + if err != nil { + return nil, err + } + _, err = seeker.Seek(cur, io.SeekStart) + if err != nil { + return nil, err + } + + buf = make([]byte, end-cur) + _, err = io.ReadFull(r, buf) + if err != nil { + return nil, err + } + } else { + var err error + buf, err = io.ReadAll(r) + if err != nil { + return nil, err + } + } + return &parser{data: buf, ver: ver, timerGoids: make(map[uint64]bool)}, nil +} + +// Parse parses Go execution traces from versions 1.11–1.21. The provided reader +// will be read to completion and the entire trace will be materialized in +// memory. That is, this function does not allow incremental parsing. +// +// The reader has to be positioned just after the trace header and vers needs to +// be the version of the trace. This can be achieved by using +// version.ReadHeader. +func Parse(r io.Reader, vers version.Version) (Trace, error) { + // We accept the version as an argument because internal/trace will have + // already read the version to determine which parser to use. + p, err := newParser(r, vers) + if err != nil { + return Trace{}, err + } + return p.parse() +} + +// parse parses, post-processes and verifies the trace. +func (p *parser) parse() (Trace, error) { + defer func() { + p.data = nil + }() + + // We parse a trace by running the following steps in order: + // + // 1. In the initial pass we collect information about batches (their + // locations and sizes.) We also parse CPU profiling samples in this + // step, simply to reduce the number of full passes that we need. + // + // 2. In the second pass we parse batches and merge them into a globally + // ordered event stream. This uses the batch information from the first + // pass to quickly find batches. + // + // 3. After all events have been parsed we convert their timestamps from CPU + // ticks to wall time. Furthermore we move timers and syscalls to + // dedicated, fake Ps. + // + // 4. Finally, we validate the trace. + + p.strings = make(map[uint64]string) + p.batchOffsets = make(map[int32][]batchOffset) + p.lastGs = make(map[int32]uint64) + p.stacks = make(map[uint32][]uint64) + p.pcs = make(map[uint64]Frame) + p.inlineStringsMapping = make(map[string]int) + + if err := p.collectBatchesAndCPUSamples(); err != nil { + return Trace{}, err + } + + events, err := p.parseEventBatches() + if err != nil { + return Trace{}, err + } + + if p.ticksPerSec == 0 { + return Trace{}, errors.New("no EvFrequency event") + } + + if events.Len() > 0 { + // Translate cpu ticks to real time. + minTs := events.Ptr(0).Ts + // Use floating point to avoid integer overflows. + freq := 1e9 / float64(p.ticksPerSec) + for i := 0; i < events.Len(); i++ { + ev := events.Ptr(i) + ev.Ts = Timestamp(float64(ev.Ts-minTs) * freq) + // Move timers and syscalls to separate fake Ps. + if p.timerGoids[ev.G] && ev.Type == EvGoUnblock { + ev.P = TimerP + } + if ev.Type == EvGoSysExit { + ev.P = SyscallP + } + } + } + + if err := p.postProcessTrace(events); err != nil { + return Trace{}, err + } + + res := Trace{ + Version: p.ver, + Events: events, + Stacks: p.stacks, + Strings: p.strings, + InlineStrings: p.inlineStrings, + PCs: p.pcs, + } + return res, nil +} + +// rawEvent is a helper type used during parsing. +type rawEvent struct { + typ EventType + args []uint64 + sargs []string + + // if typ == EvBatch, these fields describe the batch. + batchPid int32 + batchOffset int +} + +type proc struct { + pid int32 + // the remaining events in the current batch + events []Event + // buffer for reading batches into, aliased by proc.events + buf []Event + + // there are no more batches left + done bool +} + +const eventsBucketSize = 524288 // 32 MiB of events + +type Events struct { + // Events is a slice of slices that grows one slice of size eventsBucketSize + // at a time. This avoids the O(n) cost of slice growth in append, and + // additionally allows consumers to drop references to parts of the data, + // freeing memory piecewise. + n int + buckets []*[eventsBucketSize]Event + off int +} + +// grow grows the slice by one and returns a pointer to the new element, without +// overwriting it. +func (l *Events) grow() *Event { + a, b := l.index(l.n) + if a >= len(l.buckets) { + l.buckets = append(l.buckets, new([eventsBucketSize]Event)) + } + ptr := &l.buckets[a][b] + l.n++ + return ptr +} + +// append appends v to the slice and returns a pointer to the new element. +func (l *Events) append(v Event) *Event { + ptr := l.grow() + *ptr = v + return ptr +} + +func (l *Events) Ptr(i int) *Event { + a, b := l.index(i + l.off) + return &l.buckets[a][b] +} + +func (l *Events) index(i int) (int, int) { + // Doing the division on uint instead of int compiles this function to a + // shift and an AND (for power of 2 bucket sizes), versus a whole bunch of + // instructions for int. + return int(uint(i) / eventsBucketSize), int(uint(i) % eventsBucketSize) +} + +func (l *Events) Len() int { + return l.n - l.off +} + +func (l *Events) Less(i, j int) bool { + return l.Ptr(i).Ts < l.Ptr(j).Ts +} + +func (l *Events) Swap(i, j int) { + *l.Ptr(i), *l.Ptr(j) = *l.Ptr(j), *l.Ptr(i) +} + +func (l *Events) Pop() (*Event, bool) { + if l.off == l.n { + return nil, false + } + a, b := l.index(l.off) + ptr := &l.buckets[a][b] + l.off++ + if b == eventsBucketSize-1 || l.off == l.n { + // We've consumed the last event from the bucket, so drop the bucket and + // allow GC to collect it. + l.buckets[a] = nil + } + return ptr, true +} + +func (l *Events) Peek() (*Event, bool) { + if l.off == l.n { + return nil, false + } + a, b := l.index(l.off) + return &l.buckets[a][b], true +} + +func (l *Events) All() func(yield func(ev *Event) bool) { + return func(yield func(ev *Event) bool) { + for i := 0; i < l.Len(); i++ { + a, b := l.index(i + l.off) + ptr := &l.buckets[a][b] + if !yield(ptr) { + return + } + } + } +} + +// parseEventBatches reads per-P event batches and merges them into a single, consistent +// stream. The high level idea is as follows. Events within an individual batch +// are in correct order, because they are emitted by a single P. So we need to +// produce a correct interleaving of the batches. To do this we take first +// unmerged event from each batch (frontier). Then choose subset that is "ready" +// to be merged, that is, events for which all dependencies are already merged. +// Then we choose event with the lowest timestamp from the subset, merge it and +// repeat. This approach ensures that we form a consistent stream even if +// timestamps are incorrect (condition observed on some machines). +func (p *parser) parseEventBatches() (Events, error) { + // The ordering of CPU profile sample events in the data stream is based on + // when each run of the signal handler was able to acquire the spinlock, + // with original timestamps corresponding to when ReadTrace pulled the data + // off of the profBuf queue. Re-sort them by the timestamp we captured + // inside the signal handler. + slices.SortFunc(p.cpuSamples, func(a, b Event) int { + return cmp.Compare(a.Ts, b.Ts) + }) + + allProcs := make([]proc, 0, len(p.batchOffsets)) + for pid := range p.batchOffsets { + allProcs = append(allProcs, proc{pid: pid}) + } + allProcs = append(allProcs, proc{pid: ProfileP, events: p.cpuSamples}) + + events := Events{} + + // Merge events as long as at least one P has more events + gs := make(map[uint64]gState) + // Note: technically we don't need a priority queue here. We're only ever + // interested in the earliest eligible event, which means we just have to + // track the smallest element. However, in practice, the priority queue + // performs better, because for each event we only have to compute its state + // transition once, not on each iteration. If it was eligible before, it'll + // already be in the queue. Furthermore, on average, we only have one P to + // look at in each iteration, because all other Ps are already in the queue. + var frontier orderEventList + + availableProcs := make([]*proc, len(allProcs)) + for i := range allProcs { + availableProcs[i] = &allProcs[i] + } + for { + pidLoop: + for i := 0; i < len(availableProcs); i++ { + proc := availableProcs[i] + + for len(proc.events) == 0 { + // Call loadBatch in a loop because sometimes batches are empty + evs, err := p.loadBatch(proc.pid, proc.buf[:0]) + proc.buf = evs[:0] + if err == io.EOF { + // This P has no more events + proc.done = true + availableProcs[i], availableProcs[len(availableProcs)-1] = availableProcs[len(availableProcs)-1], availableProcs[i] + availableProcs = availableProcs[:len(availableProcs)-1] + // We swapped the element at i with another proc, so look at + // the index again + i-- + continue pidLoop + } else if err != nil { + return Events{}, err + } else { + proc.events = evs + } + } + + ev := &proc.events[0] + g, init, _ := stateTransition(ev) + + // TODO(dh): This implementation matches the behavior of the + // upstream 'go tool trace', and works in practice, but has run into + // the following inconsistency during fuzzing: what happens if + // multiple Ps have events for the same G? While building the + // frontier we will check all of the events against the current + // state of the G. However, when we process the frontier, the state + // of the G changes, and a transition that was valid while building + // the frontier may no longer be valid when processing the frontier. + // Is this something that can happen for real, valid traces, or is + // this only possible with corrupt data? + if !transitionReady(g, gs[g], init) { + continue + } + proc.events = proc.events[1:] + availableProcs[i], availableProcs[len(availableProcs)-1] = availableProcs[len(availableProcs)-1], availableProcs[i] + availableProcs = availableProcs[:len(availableProcs)-1] + frontier.Push(orderEvent{*ev, proc}) + + // We swapped the element at i with another proc, so look at the + // index again + i-- + } + + if len(frontier) == 0 { + for i := range allProcs { + if !allProcs[i].done { + return Events{}, fmt.Errorf("no consistent ordering of events possible") + } + } + break + } + f := frontier.Pop() + + // We're computing the state transition twice, once when computing the + // frontier, and now to apply the transition. This is fine because + // stateTransition is a pure function. Computing it again is cheaper + // than storing large items in the frontier. + g, init, next := stateTransition(&f.ev) + + // Get rid of "Local" events, they are intended merely for ordering. + switch f.ev.Type { + case EvGoStartLocal: + f.ev.Type = EvGoStart + case EvGoUnblockLocal: + f.ev.Type = EvGoUnblock + case EvGoSysExitLocal: + f.ev.Type = EvGoSysExit + } + events.append(f.ev) + + if err := transition(gs, g, init, next); err != nil { + return Events{}, err + } + availableProcs = append(availableProcs, f.proc) + } + + // At this point we have a consistent stream of events. Make sure time + // stamps respect the ordering. The tests will skip (not fail) the test case + // if they see this error. + if !sort.IsSorted(&events) { + return Events{}, ErrTimeOrder + } + + // The last part is giving correct timestamps to EvGoSysExit events. The + // problem with EvGoSysExit is that actual syscall exit timestamp + // (ev.Args[2]) is potentially acquired long before event emission. So far + // we've used timestamp of event emission (ev.Ts). We could not set ev.Ts = + // ev.Args[2] earlier, because it would produce seemingly broken timestamps + // (misplaced event). We also can't simply update the timestamp and resort + // events, because if timestamps are broken we will misplace the event and + // later report logically broken trace (instead of reporting broken + // timestamps). + lastSysBlock := make(map[uint64]Timestamp) + for i := 0; i < events.Len(); i++ { + ev := events.Ptr(i) + switch ev.Type { + case EvGoSysBlock, EvGoInSyscall: + lastSysBlock[ev.G] = ev.Ts + case EvGoSysExit: + ts := Timestamp(ev.Args[2]) + if ts == 0 { + continue + } + block := lastSysBlock[ev.G] + if block == 0 { + return Events{}, fmt.Errorf("stray syscall exit") + } + if ts < block { + return Events{}, ErrTimeOrder + } + ev.Ts = ts + } + } + sort.Stable(&events) + + return events, nil +} + +// collectBatchesAndCPUSamples records the offsets of batches and parses CPU samples. +func (p *parser) collectBatchesAndCPUSamples() error { + // Read events. + var raw rawEvent + var curP int32 + for n := uint64(0); ; n++ { + err := p.readRawEvent(skipArgs|skipStrings, &raw) + if err == io.EOF { + break + } + if err != nil { + return err + } + if raw.typ == EvNone { + continue + } + + if raw.typ == EvBatch { + bo := batchOffset{offset: raw.batchOffset} + p.batchOffsets[raw.batchPid] = append(p.batchOffsets[raw.batchPid], bo) + curP = raw.batchPid + } + + batches := p.batchOffsets[curP] + if len(batches) == 0 { + return fmt.Errorf("read event %d with current P of %d, but P has no batches yet", + raw.typ, curP) + } + batches[len(batches)-1].numEvents++ + + if raw.typ == EvCPUSample { + e := Event{Type: raw.typ} + + argOffset := 1 + narg := raw.argNum() + if len(raw.args) != narg { + return fmt.Errorf("CPU sample has wrong number of arguments: want %d, got %d", narg, len(raw.args)) + } + for i := argOffset; i < narg; i++ { + if i == narg-1 { + e.StkID = uint32(raw.args[i]) + } else { + e.Args[i-argOffset] = raw.args[i] + } + } + + e.Ts = Timestamp(e.Args[0]) + e.P = int32(e.Args[1]) + e.G = e.Args[2] + e.Args[0] = 0 + + // Most events are written out by the active P at the exact moment + // they describe. CPU profile samples are different because they're + // written to the tracing log after some delay, by a separate worker + // goroutine, into a separate buffer. + // + // We keep these in their own batch until all of the batches are + // merged in timestamp order. We also (right before the merge) + // re-sort these events by the timestamp captured in the profiling + // signal handler. + // + // Note that we're not concerned about the memory usage of storing + // all CPU samples during the indexing phase. There are orders of + // magnitude fewer CPU samples than runtime events. + p.cpuSamples = append(p.cpuSamples, e) + } + } + + return nil +} + +const ( + skipArgs = 1 << iota + skipStrings +) + +func (p *parser) readByte() (byte, bool) { + if p.off < len(p.data) && p.off >= 0 { + b := p.data[p.off] + p.off++ + return b, true + } else { + return 0, false + } +} + +func (p *parser) readFull(n int) ([]byte, error) { + if p.off >= len(p.data) || p.off < 0 || p.off+n > len(p.data) { + // p.off < 0 is impossible but makes BCE happy. + // + // We do fail outright if there's not enough data, we don't care about + // partial results. + return nil, io.ErrUnexpectedEOF + } + buf := p.data[p.off : p.off+n] + p.off += n + return buf, nil +} + +// readRawEvent reads a raw event into ev. The slices in ev are only valid until +// the next call to readRawEvent, even when storing to a different location. +func (p *parser) readRawEvent(flags uint, ev *rawEvent) error { + // The number of arguments is encoded using two bits and can thus only + // represent the values 0–3. The value 3 (on the wire) indicates that + // arguments are prefixed by their byte length, to encode >=3 arguments. + const inlineArgs = 3 + + // Read event type and number of arguments (1 byte). + b, ok := p.readByte() + if !ok { + return io.EOF + } + typ := EventType(b << 2 >> 2) + // Most events have a timestamp before the actual arguments, so we add 1 and + // parse it like it's the first argument. EvString has a special format and + // the number of arguments doesn't matter. EvBatch writes '1' as the number + // of arguments, but actually has two: a pid and a timestamp, but here the + // timestamp is the second argument, not the first; adding 1 happens to come + // up with the correct number, but it doesn't matter, because EvBatch has + // custom logic for parsing. + // + // Note that because we're adding 1, inlineArgs == 3 describes the largest + // number of logical arguments that isn't length-prefixed, even though the + // value 3 on the wire indicates length-prefixing. For us, that becomes narg + // == 4. + narg := b>>6 + 1 + if typ == EvNone || typ >= EvCount || EventDescriptions[typ].minVersion > p.ver { + return fmt.Errorf("unknown event type %d", typ) + } + + switch typ { + case EvString: + if flags&skipStrings != 0 { + // String dictionary entry [ID, length, string]. + if _, err := p.readVal(); err != nil { + return errMalformedVarint + } + ln, err := p.readVal() + if err != nil { + return err + } + if !p.discard(ln) { + return fmt.Errorf("failed to read trace: %w", io.EOF) + } + } else { + // String dictionary entry [ID, length, string]. + id, err := p.readVal() + if err != nil { + return err + } + if id == 0 { + return errors.New("string has invalid id 0") + } + if p.strings[id] != "" { + return fmt.Errorf("string has duplicate id %d", id) + } + var ln uint64 + ln, err = p.readVal() + if err != nil { + return err + } + if ln == 0 { + return errors.New("string has invalid length 0") + } + if ln > 1e6 { + return fmt.Errorf("string has too large length %d", ln) + } + buf, err := p.readFull(int(ln)) + if err != nil { + return fmt.Errorf("failed to read trace: %w", err) + } + p.strings[id] = string(buf) + } + + ev.typ = EvNone + return nil + case EvBatch: + if want := byte(2); narg != want { + return fmt.Errorf("EvBatch has wrong number of arguments: got %d, want %d", narg, want) + } + + // -1 because we've already read the first byte of the batch + off := p.off - 1 + + pid, err := p.readVal() + if err != nil { + return err + } + if pid != math.MaxUint64 && pid > math.MaxInt32 { + return fmt.Errorf("processor ID %d is larger than maximum of %d", pid, uint64(math.MaxUint)) + } + + var pid32 int32 + if pid == math.MaxUint64 { + pid32 = -1 + } else { + pid32 = int32(pid) + } + + v, err := p.readVal() + if err != nil { + return err + } + + *ev = rawEvent{ + typ: EvBatch, + args: p.args[:0], + batchPid: pid32, + batchOffset: off, + } + ev.args = append(ev.args, pid, v) + return nil + default: + *ev = rawEvent{typ: typ, args: p.args[:0]} + if narg <= inlineArgs { + if flags&skipArgs == 0 { + for i := 0; i < int(narg); i++ { + v, err := p.readVal() + if err != nil { + return fmt.Errorf("failed to read event %d argument: %w", typ, err) + } + ev.args = append(ev.args, v) + } + } else { + for i := 0; i < int(narg); i++ { + if _, err := p.readVal(); err != nil { + return fmt.Errorf("failed to read event %d argument: %w", typ, errMalformedVarint) + } + } + } + } else { + // More than inlineArgs args, the first value is length of the event + // in bytes. + v, err := p.readVal() + if err != nil { + return fmt.Errorf("failed to read event %d argument: %w", typ, err) + } + + if limit := uint64(2048); v > limit { + // At the time of Go 1.19, v seems to be at most 128. Set 2048 + // as a generous upper limit and guard against malformed traces. + return fmt.Errorf("failed to read event %d argument: length-prefixed argument too big: %d bytes, limit is %d", typ, v, limit) + } + + if flags&skipArgs == 0 || typ == EvCPUSample { + buf, err := p.readFull(int(v)) + if err != nil { + return fmt.Errorf("failed to read trace: %w", err) + } + for len(buf) > 0 { + var v uint64 + v, buf, err = readValFrom(buf) + if err != nil { + return err + } + ev.args = append(ev.args, v) + } + } else { + // Skip over arguments + if !p.discard(v) { + return fmt.Errorf("failed to read trace: %w", io.EOF) + } + } + if typ == EvUserLog { + // EvUserLog records are followed by a value string + if flags&skipArgs == 0 { + // Read string + s, err := p.readStr() + if err != nil { + return err + } + ev.sargs = append(ev.sargs, s) + } else { + // Skip string + v, err := p.readVal() + if err != nil { + return err + } + if !p.discard(v) { + return io.EOF + } + } + } + } + + p.args = ev.args[:0] + return nil + } +} + +// loadBatch loads the next batch for pid and appends its contents to events. +func (p *parser) loadBatch(pid int32, events []Event) ([]Event, error) { + offsets := p.batchOffsets[pid] + if len(offsets) == 0 { + return nil, io.EOF + } + n := offsets[0].numEvents + offset := offsets[0].offset + offsets = offsets[1:] + p.batchOffsets[pid] = offsets + + p.off = offset + + if cap(events) < n { + events = make([]Event, 0, n) + } + + gotHeader := false + var raw rawEvent + var ev Event + for { + err := p.readRawEvent(0, &raw) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if raw.typ == EvNone || raw.typ == EvCPUSample { + continue + } + if raw.typ == EvBatch { + if gotHeader { + break + } else { + gotHeader = true + } + } + + err = p.parseEvent(&raw, &ev) + if err != nil { + return nil, err + } + if ev.Type != EvNone { + events = append(events, ev) + } + } + + return events, nil +} + +func (p *parser) readStr() (s string, err error) { + sz, err := p.readVal() + if err != nil { + return "", err + } + if sz == 0 { + return "", nil + } + if sz > 1e6 { + return "", fmt.Errorf("string is too large (len=%d)", sz) + } + buf, err := p.readFull(int(sz)) + if err != nil { + return "", fmt.Errorf("failed to read trace: %w", err) + } + return string(buf), nil +} + +// parseEvent transforms raw events into events. +// It does analyze and verify per-event-type arguments. +func (p *parser) parseEvent(raw *rawEvent, ev *Event) error { + desc := &EventDescriptions[raw.typ] + if desc.Name == "" { + return fmt.Errorf("missing description for event type %d", raw.typ) + } + narg := raw.argNum() + if len(raw.args) != narg { + return fmt.Errorf("%s has wrong number of arguments: want %d, got %d", desc.Name, narg, len(raw.args)) + } + switch raw.typ { + case EvBatch: + p.lastGs[p.lastP] = p.lastG + if raw.args[0] != math.MaxUint64 && raw.args[0] > math.MaxInt32 { + return fmt.Errorf("processor ID %d is larger than maximum of %d", raw.args[0], uint64(math.MaxInt32)) + } + if raw.args[0] == math.MaxUint64 { + p.lastP = -1 + } else { + p.lastP = int32(raw.args[0]) + } + p.lastG = p.lastGs[p.lastP] + p.lastTs = Timestamp(raw.args[1]) + case EvFrequency: + p.ticksPerSec = int64(raw.args[0]) + if p.ticksPerSec <= 0 { + // The most likely cause for this is tick skew on different CPUs. + // For example, solaris/amd64 seems to have wildly different + // ticks on different CPUs. + return ErrTimeOrder + } + case EvTimerGoroutine: + p.timerGoids[raw.args[0]] = true + case EvStack: + if len(raw.args) < 2 { + return fmt.Errorf("EvStack has wrong number of arguments: want at least 2, got %d", len(raw.args)) + } + size := raw.args[1] + if size > 1000 { + return fmt.Errorf("EvStack has bad number of frames: %d", size) + } + want := 2 + 4*size + if uint64(len(raw.args)) != want { + return fmt.Errorf("EvStack has wrong number of arguments: want %d, got %d", want, len(raw.args)) + } + id := uint32(raw.args[0]) + if id != 0 && size > 0 { + stk := p.allocateStack(size) + for i := 0; i < int(size); i++ { + pc := raw.args[2+i*4+0] + fn := raw.args[2+i*4+1] + file := raw.args[2+i*4+2] + line := raw.args[2+i*4+3] + stk[i] = pc + + if _, ok := p.pcs[pc]; !ok { + p.pcs[pc] = Frame{PC: pc, Fn: fn, File: file, Line: int(line)} + } + } + p.stacks[id] = stk + } + case EvCPUSample: + // These events get parsed during the indexing step and don't strictly + // belong to the batch. + default: + *ev = Event{Type: raw.typ, P: p.lastP, G: p.lastG} + var argOffset int + ev.Ts = p.lastTs + Timestamp(raw.args[0]) + argOffset = 1 + p.lastTs = ev.Ts + for i := argOffset; i < narg; i++ { + if i == narg-1 && desc.Stack { + ev.StkID = uint32(raw.args[i]) + } else { + ev.Args[i-argOffset] = raw.args[i] + } + } + switch raw.typ { + case EvGoStart, EvGoStartLocal, EvGoStartLabel: + p.lastG = ev.Args[0] + ev.G = p.lastG + case EvGoEnd, EvGoStop, EvGoSched, EvGoPreempt, + EvGoSleep, EvGoBlock, EvGoBlockSend, EvGoBlockRecv, + EvGoBlockSelect, EvGoBlockSync, EvGoBlockCond, EvGoBlockNet, + EvGoSysBlock, EvGoBlockGC: + p.lastG = 0 + case EvGoSysExit, EvGoWaiting, EvGoInSyscall: + ev.G = ev.Args[0] + case EvUserTaskCreate: + // e.Args 0: taskID, 1:parentID, 2:nameID + case EvUserRegion: + // e.Args 0: taskID, 1: mode, 2:nameID + case EvUserLog: + // e.Args 0: taskID, 1:keyID, 2: stackID, 3: messageID + // raw.sargs 0: message + + if id, ok := p.inlineStringsMapping[raw.sargs[0]]; ok { + ev.Args[3] = uint64(id) + } else { + id := len(p.inlineStrings) + p.inlineStringsMapping[raw.sargs[0]] = id + p.inlineStrings = append(p.inlineStrings, raw.sargs[0]) + ev.Args[3] = uint64(id) + } + } + + return nil + } + + ev.Type = EvNone + return nil +} + +// ErrTimeOrder is returned by Parse when the trace contains +// time stamps that do not respect actual event ordering. +var ErrTimeOrder = errors.New("time stamps out of order") + +// postProcessTrace does inter-event verification and information restoration. +// The resulting trace is guaranteed to be consistent +// (for example, a P does not run two Gs at the same time, or a G is indeed +// blocked before an unblock event). +func (p *parser) postProcessTrace(events Events) error { + const ( + gDead = iota + gRunnable + gRunning + gWaiting + ) + type gdesc struct { + state int + ev *Event + evStart *Event + evCreate *Event + evMarkAssist *Event + } + type pdesc struct { + running bool + g uint64 + evSweep *Event + } + + gs := make(map[uint64]gdesc) + ps := make(map[int32]pdesc) + tasks := make(map[uint64]*Event) // task id to task creation events + activeRegions := make(map[uint64][]*Event) // goroutine id to stack of regions + gs[0] = gdesc{state: gRunning} + var evGC, evSTW *Event + + checkRunning := func(p pdesc, g gdesc, ev *Event, allowG0 bool) error { + name := EventDescriptions[ev.Type].Name + if g.state != gRunning { + return fmt.Errorf("g %d is not running while %s (time %d)", ev.G, name, ev.Ts) + } + if p.g != ev.G { + return fmt.Errorf("p %d is not running g %d while %s (time %d)", ev.P, ev.G, name, ev.Ts) + } + if !allowG0 && ev.G == 0 { + return fmt.Errorf("g 0 did %s (time %d)", name, ev.Ts) + } + return nil + } + + for evIdx := 0; evIdx < events.Len(); evIdx++ { + ev := events.Ptr(evIdx) + + switch ev.Type { + case EvProcStart: + p := ps[ev.P] + if p.running { + return fmt.Errorf("p %d is running before start (time %d)", ev.P, ev.Ts) + } + p.running = true + + ps[ev.P] = p + case EvProcStop: + p := ps[ev.P] + if !p.running { + return fmt.Errorf("p %d is not running before stop (time %d)", ev.P, ev.Ts) + } + if p.g != 0 { + return fmt.Errorf("p %d is running a goroutine %d during stop (time %d)", ev.P, p.g, ev.Ts) + } + p.running = false + + ps[ev.P] = p + case EvGCStart: + if evGC != nil { + return fmt.Errorf("previous GC is not ended before a new one (time %d)", ev.Ts) + } + evGC = ev + // Attribute this to the global GC state. + ev.P = GCP + case EvGCDone: + if evGC == nil { + return fmt.Errorf("bogus GC end (time %d)", ev.Ts) + } + evGC = nil + case EvSTWStart: + evp := &evSTW + if *evp != nil { + return fmt.Errorf("previous STW is not ended before a new one (time %d)", ev.Ts) + } + *evp = ev + case EvSTWDone: + evp := &evSTW + if *evp == nil { + return fmt.Errorf("bogus STW end (time %d)", ev.Ts) + } + *evp = nil + case EvGCSweepStart: + p := ps[ev.P] + if p.evSweep != nil { + return fmt.Errorf("previous sweeping is not ended before a new one (time %d)", ev.Ts) + } + p.evSweep = ev + + ps[ev.P] = p + case EvGCMarkAssistStart: + g := gs[ev.G] + if g.evMarkAssist != nil { + return fmt.Errorf("previous mark assist is not ended before a new one (time %d)", ev.Ts) + } + g.evMarkAssist = ev + + gs[ev.G] = g + case EvGCMarkAssistDone: + // Unlike most events, mark assists can be in progress when a + // goroutine starts tracing, so we can't report an error here. + g := gs[ev.G] + if g.evMarkAssist != nil { + g.evMarkAssist = nil + } + + gs[ev.G] = g + case EvGCSweepDone: + p := ps[ev.P] + if p.evSweep == nil { + return fmt.Errorf("bogus sweeping end (time %d)", ev.Ts) + } + p.evSweep = nil + + ps[ev.P] = p + case EvGoWaiting: + g := gs[ev.G] + if g.state != gRunnable { + return fmt.Errorf("g %d is not runnable before EvGoWaiting (time %d)", ev.G, ev.Ts) + } + g.state = gWaiting + g.ev = ev + + gs[ev.G] = g + case EvGoInSyscall: + g := gs[ev.G] + if g.state != gRunnable { + return fmt.Errorf("g %d is not runnable before EvGoInSyscall (time %d)", ev.G, ev.Ts) + } + g.state = gWaiting + g.ev = ev + + gs[ev.G] = g + case EvGoCreate: + g := gs[ev.G] + p := ps[ev.P] + if err := checkRunning(p, g, ev, true); err != nil { + return err + } + if _, ok := gs[ev.Args[0]]; ok { + return fmt.Errorf("g %d already exists (time %d)", ev.Args[0], ev.Ts) + } + gs[ev.Args[0]] = gdesc{state: gRunnable, ev: ev, evCreate: ev} + + case EvGoStart, EvGoStartLabel: + g := gs[ev.G] + p := ps[ev.P] + if g.state != gRunnable { + return fmt.Errorf("g %d is not runnable before start (time %d)", ev.G, ev.Ts) + } + if p.g != 0 { + return fmt.Errorf("p %d is already running g %d while start g %d (time %d)", ev.P, p.g, ev.G, ev.Ts) + } + g.state = gRunning + g.evStart = ev + p.g = ev.G + if g.evCreate != nil { + ev.StkID = uint32(g.evCreate.Args[1]) + g.evCreate = nil + } + + if g.ev != nil { + g.ev = nil + } + + gs[ev.G] = g + ps[ev.P] = p + case EvGoEnd, EvGoStop: + g := gs[ev.G] + p := ps[ev.P] + if err := checkRunning(p, g, ev, false); err != nil { + return err + } + g.evStart = nil + g.state = gDead + p.g = 0 + + if ev.Type == EvGoEnd { // flush all active regions + delete(activeRegions, ev.G) + } + + gs[ev.G] = g + ps[ev.P] = p + case EvGoSched, EvGoPreempt: + g := gs[ev.G] + p := ps[ev.P] + if err := checkRunning(p, g, ev, false); err != nil { + return err + } + g.state = gRunnable + g.evStart = nil + p.g = 0 + g.ev = ev + + gs[ev.G] = g + ps[ev.P] = p + case EvGoUnblock: + g := gs[ev.G] + p := ps[ev.P] + if g.state != gRunning { + return fmt.Errorf("g %d is not running while unpark (time %d)", ev.G, ev.Ts) + } + if ev.P != TimerP && p.g != ev.G { + return fmt.Errorf("p %d is not running g %d while unpark (time %d)", ev.P, ev.G, ev.Ts) + } + g1 := gs[ev.Args[0]] + if g1.state != gWaiting { + return fmt.Errorf("g %d is not waiting before unpark (time %d)", ev.Args[0], ev.Ts) + } + if g1.ev != nil && g1.ev.Type == EvGoBlockNet { + ev.P = NetpollP + } + g1.state = gRunnable + g1.ev = ev + gs[ev.Args[0]] = g1 + + case EvGoSysCall: + g := gs[ev.G] + p := ps[ev.P] + if err := checkRunning(p, g, ev, false); err != nil { + return err + } + g.ev = ev + + gs[ev.G] = g + case EvGoSysBlock: + g := gs[ev.G] + p := ps[ev.P] + if err := checkRunning(p, g, ev, false); err != nil { + return err + } + g.state = gWaiting + g.evStart = nil + p.g = 0 + + gs[ev.G] = g + ps[ev.P] = p + case EvGoSysExit: + g := gs[ev.G] + if g.state != gWaiting { + return fmt.Errorf("g %d is not waiting during syscall exit (time %d)", ev.G, ev.Ts) + } + g.state = gRunnable + g.ev = ev + + gs[ev.G] = g + case EvGoSleep, EvGoBlock, EvGoBlockSend, EvGoBlockRecv, + EvGoBlockSelect, EvGoBlockSync, EvGoBlockCond, EvGoBlockNet, EvGoBlockGC: + g := gs[ev.G] + p := ps[ev.P] + if err := checkRunning(p, g, ev, false); err != nil { + return err + } + g.state = gWaiting + g.ev = ev + g.evStart = nil + p.g = 0 + + gs[ev.G] = g + ps[ev.P] = p + case EvUserTaskCreate: + taskid := ev.Args[0] + if prevEv, ok := tasks[taskid]; ok { + return fmt.Errorf("task id conflicts (id:%d), %q vs %q", taskid, ev, prevEv) + } + tasks[ev.Args[0]] = ev + + case EvUserTaskEnd: + taskid := ev.Args[0] + delete(tasks, taskid) + + case EvUserRegion: + mode := ev.Args[1] + regions := activeRegions[ev.G] + if mode == 0 { // region start + activeRegions[ev.G] = append(regions, ev) // push + } else if mode == 1 { // region end + n := len(regions) + if n > 0 { // matching region start event is in the trace. + s := regions[n-1] + if s.Args[0] != ev.Args[0] || s.Args[2] != ev.Args[2] { // task id, region name mismatch + return fmt.Errorf("misuse of region in goroutine %d: span end %q when the inner-most active span start event is %q", ev.G, ev, s) + } + + if n > 1 { + activeRegions[ev.G] = regions[:n-1] + } else { + delete(activeRegions, ev.G) + } + } + } else { + return fmt.Errorf("invalid user region mode: %q", ev) + } + } + + if ev.StkID != 0 && len(p.stacks[ev.StkID]) == 0 { + // Make sure events don't refer to stacks that don't exist or to + // stacks with zero frames. Neither of these should be possible, but + // better be safe than sorry. + + ev.StkID = 0 + } + + } + + // TODO(mknyszek): restore stacks for EvGoStart events. + return nil +} + +var errMalformedVarint = errors.New("malformatted base-128 varint") + +// readVal reads unsigned base-128 value from r. +func (p *parser) readVal() (uint64, error) { + v, n := binary.Uvarint(p.data[p.off:]) + if n <= 0 { + return 0, errMalformedVarint + } + p.off += n + return v, nil +} + +func readValFrom(buf []byte) (v uint64, rem []byte, err error) { + v, n := binary.Uvarint(buf) + if n <= 0 { + return 0, nil, errMalformedVarint + } + return v, buf[n:], nil +} + +func (ev *Event) String() string { + desc := &EventDescriptions[ev.Type] + w := new(bytes.Buffer) + fmt.Fprintf(w, "%d %s p=%d g=%d stk=%d", ev.Ts, desc.Name, ev.P, ev.G, ev.StkID) + for i, a := range desc.Args { + fmt.Fprintf(w, " %s=%d", a, ev.Args[i]) + } + return w.String() +} + +// argNum returns total number of args for the event accounting for timestamps, +// sequence numbers and differences between trace format versions. +func (raw *rawEvent) argNum() int { + desc := &EventDescriptions[raw.typ] + if raw.typ == EvStack { + return len(raw.args) + } + narg := len(desc.Args) + if desc.Stack { + narg++ + } + switch raw.typ { + case EvBatch, EvFrequency, EvTimerGoroutine: + return narg + } + narg++ // timestamp + return narg +} + +type EventType uint8 + +// Event types in the trace. +// Verbatim copy from src/runtime/trace.go with the "trace" prefix removed. +const ( + EvNone EventType = 0 // unused + EvBatch EventType = 1 // start of per-P batch of events [pid, timestamp] + EvFrequency EventType = 2 // contains tracer timer frequency [frequency (ticks per second)] + EvStack EventType = 3 // stack [stack id, number of PCs, array of {PC, func string ID, file string ID, line}] + EvGomaxprocs EventType = 4 // current value of GOMAXPROCS [timestamp, GOMAXPROCS, stack id] + EvProcStart EventType = 5 // start of P [timestamp, thread id] + EvProcStop EventType = 6 // stop of P [timestamp] + EvGCStart EventType = 7 // GC start [timestamp, seq, stack id] + EvGCDone EventType = 8 // GC done [timestamp] + EvSTWStart EventType = 9 // GC mark termination start [timestamp, kind] + EvSTWDone EventType = 10 // GC mark termination done [timestamp] + EvGCSweepStart EventType = 11 // GC sweep start [timestamp, stack id] + EvGCSweepDone EventType = 12 // GC sweep done [timestamp, swept, reclaimed] + EvGoCreate EventType = 13 // goroutine creation [timestamp, new goroutine id, new stack id, stack id] + EvGoStart EventType = 14 // goroutine starts running [timestamp, goroutine id, seq] + EvGoEnd EventType = 15 // goroutine ends [timestamp] + EvGoStop EventType = 16 // goroutine stops (like in select{}) [timestamp, stack] + EvGoSched EventType = 17 // goroutine calls Gosched [timestamp, stack] + EvGoPreempt EventType = 18 // goroutine is preempted [timestamp, stack] + EvGoSleep EventType = 19 // goroutine calls Sleep [timestamp, stack] + EvGoBlock EventType = 20 // goroutine blocks [timestamp, stack] + EvGoUnblock EventType = 21 // goroutine is unblocked [timestamp, goroutine id, seq, stack] + EvGoBlockSend EventType = 22 // goroutine blocks on chan send [timestamp, stack] + EvGoBlockRecv EventType = 23 // goroutine blocks on chan recv [timestamp, stack] + EvGoBlockSelect EventType = 24 // goroutine blocks on select [timestamp, stack] + EvGoBlockSync EventType = 25 // goroutine blocks on Mutex/RWMutex [timestamp, stack] + EvGoBlockCond EventType = 26 // goroutine blocks on Cond [timestamp, stack] + EvGoBlockNet EventType = 27 // goroutine blocks on network [timestamp, stack] + EvGoSysCall EventType = 28 // syscall enter [timestamp, stack] + EvGoSysExit EventType = 29 // syscall exit [timestamp, goroutine id, seq, real timestamp] + EvGoSysBlock EventType = 30 // syscall blocks [timestamp] + EvGoWaiting EventType = 31 // denotes that goroutine is blocked when tracing starts [timestamp, goroutine id] + EvGoInSyscall EventType = 32 // denotes that goroutine is in syscall when tracing starts [timestamp, goroutine id] + EvHeapAlloc EventType = 33 // gcController.heapLive change [timestamp, heap live bytes] + EvHeapGoal EventType = 34 // gcController.heapGoal change [timestamp, heap goal bytes] + EvTimerGoroutine EventType = 35 // denotes timer goroutine [timer goroutine id] + EvFutileWakeup EventType = 36 // denotes that the previous wakeup of this goroutine was futile [timestamp] + EvString EventType = 37 // string dictionary entry [ID, length, string] + EvGoStartLocal EventType = 38 // goroutine starts running on the same P as the last event [timestamp, goroutine id] + EvGoUnblockLocal EventType = 39 // goroutine is unblocked on the same P as the last event [timestamp, goroutine id, stack] + EvGoSysExitLocal EventType = 40 // syscall exit on the same P as the last event [timestamp, goroutine id, real timestamp] + EvGoStartLabel EventType = 41 // goroutine starts running with label [timestamp, goroutine id, seq, label string id] + EvGoBlockGC EventType = 42 // goroutine blocks on GC assist [timestamp, stack] + EvGCMarkAssistStart EventType = 43 // GC mark assist start [timestamp, stack] + EvGCMarkAssistDone EventType = 44 // GC mark assist done [timestamp] + EvUserTaskCreate EventType = 45 // trace.NewTask [timestamp, internal task id, internal parent id, stack, name string] + EvUserTaskEnd EventType = 46 // end of task [timestamp, internal task id, stack] + EvUserRegion EventType = 47 // trace.WithRegion [timestamp, internal task id, mode(0:start, 1:end), name string] + EvUserLog EventType = 48 // trace.Log [timestamp, internal id, key string id, stack, value string] + EvCPUSample EventType = 49 // CPU profiling sample [timestamp, stack, real timestamp, real P id (-1 when absent), goroutine id] + EvCount EventType = 50 +) + +var EventDescriptions = [256]struct { + Name string + minVersion version.Version + Stack bool + Args []string + SArgs []string // string arguments +}{ + EvNone: {"None", 5, false, []string{}, nil}, + EvBatch: {"Batch", 5, false, []string{"p", "ticks"}, nil}, // in 1.5 format it was {"p", "seq", "ticks"} + EvFrequency: {"Frequency", 5, false, []string{"freq"}, nil}, // in 1.5 format it was {"freq", "unused"} + EvStack: {"Stack", 5, false, []string{"id", "siz"}, nil}, + EvGomaxprocs: {"Gomaxprocs", 5, true, []string{"procs"}, nil}, + EvProcStart: {"ProcStart", 5, false, []string{"thread"}, nil}, + EvProcStop: {"ProcStop", 5, false, []string{}, nil}, + EvGCStart: {"GCStart", 5, true, []string{"seq"}, nil}, // in 1.5 format it was {} + EvGCDone: {"GCDone", 5, false, []string{}, nil}, + EvSTWStart: {"GCSTWStart", 5, false, []string{"kindid"}, []string{"kind"}}, // <= 1.9, args was {} (implicitly {0}) + EvSTWDone: {"GCSTWDone", 5, false, []string{}, nil}, + EvGCSweepStart: {"GCSweepStart", 5, true, []string{}, nil}, + EvGCSweepDone: {"GCSweepDone", 5, false, []string{"swept", "reclaimed"}, nil}, // before 1.9, format was {} + EvGoCreate: {"GoCreate", 5, true, []string{"g", "stack"}, nil}, + EvGoStart: {"GoStart", 5, false, []string{"g", "seq"}, nil}, // in 1.5 format it was {"g"} + EvGoEnd: {"GoEnd", 5, false, []string{}, nil}, + EvGoStop: {"GoStop", 5, true, []string{}, nil}, + EvGoSched: {"GoSched", 5, true, []string{}, nil}, + EvGoPreempt: {"GoPreempt", 5, true, []string{}, nil}, + EvGoSleep: {"GoSleep", 5, true, []string{}, nil}, + EvGoBlock: {"GoBlock", 5, true, []string{}, nil}, + EvGoUnblock: {"GoUnblock", 5, true, []string{"g", "seq"}, nil}, // in 1.5 format it was {"g"} + EvGoBlockSend: {"GoBlockSend", 5, true, []string{}, nil}, + EvGoBlockRecv: {"GoBlockRecv", 5, true, []string{}, nil}, + EvGoBlockSelect: {"GoBlockSelect", 5, true, []string{}, nil}, + EvGoBlockSync: {"GoBlockSync", 5, true, []string{}, nil}, + EvGoBlockCond: {"GoBlockCond", 5, true, []string{}, nil}, + EvGoBlockNet: {"GoBlockNet", 5, true, []string{}, nil}, + EvGoSysCall: {"GoSysCall", 5, true, []string{}, nil}, + EvGoSysExit: {"GoSysExit", 5, false, []string{"g", "seq", "ts"}, nil}, + EvGoSysBlock: {"GoSysBlock", 5, false, []string{}, nil}, + EvGoWaiting: {"GoWaiting", 5, false, []string{"g"}, nil}, + EvGoInSyscall: {"GoInSyscall", 5, false, []string{"g"}, nil}, + EvHeapAlloc: {"HeapAlloc", 5, false, []string{"mem"}, nil}, + EvHeapGoal: {"HeapGoal", 5, false, []string{"mem"}, nil}, + EvTimerGoroutine: {"TimerGoroutine", 5, false, []string{"g"}, nil}, // in 1.5 format it was {"g", "unused"} + EvFutileWakeup: {"FutileWakeup", 5, false, []string{}, nil}, + EvString: {"String", 7, false, []string{}, nil}, + EvGoStartLocal: {"GoStartLocal", 7, false, []string{"g"}, nil}, + EvGoUnblockLocal: {"GoUnblockLocal", 7, true, []string{"g"}, nil}, + EvGoSysExitLocal: {"GoSysExitLocal", 7, false, []string{"g", "ts"}, nil}, + EvGoStartLabel: {"GoStartLabel", 8, false, []string{"g", "seq", "labelid"}, []string{"label"}}, + EvGoBlockGC: {"GoBlockGC", 8, true, []string{}, nil}, + EvGCMarkAssistStart: {"GCMarkAssistStart", 9, true, []string{}, nil}, + EvGCMarkAssistDone: {"GCMarkAssistDone", 9, false, []string{}, nil}, + EvUserTaskCreate: {"UserTaskCreate", 11, true, []string{"taskid", "pid", "typeid"}, []string{"name"}}, + EvUserTaskEnd: {"UserTaskEnd", 11, true, []string{"taskid"}, nil}, + EvUserRegion: {"UserRegion", 11, true, []string{"taskid", "mode", "typeid"}, []string{"name"}}, + EvUserLog: {"UserLog", 11, true, []string{"id", "keyid"}, []string{"category", "message"}}, + EvCPUSample: {"CPUSample", 19, true, []string{"ts", "p", "g"}, nil}, +} + +//gcassert:inline +func (p *parser) allocateStack(size uint64) []uint64 { + if size == 0 { + return nil + } + + // Stacks are plentiful but small. For our "Staticcheck on std" trace with + // 11e6 events, we have roughly 500,000 stacks, using 200 MiB of memory. To + // avoid making 500,000 small allocations we allocate backing arrays 1 MiB + // at a time. + out := p.stacksData + if uint64(len(out)) < size { + out = make([]uint64, 1024*128) + } + p.stacksData = out[size:] + return out[:size:size] +} + +func (tr *Trace) STWReason(kindID uint64) STWReason { + if tr.Version < 21 { + if kindID == 0 || kindID == 1 { + return STWReason(kindID + 1) + } else { + return STWUnknown + } + } else if tr.Version == 21 { + if kindID < NumSTWReasons { + return STWReason(kindID) + } else { + return STWUnknown + } + } else { + return STWUnknown + } +} + +type STWReason int + +const ( + STWUnknown STWReason = 0 + STWGCMarkTermination STWReason = 1 + STWGCSweepTermination STWReason = 2 + STWWriteHeapDump STWReason = 3 + STWGoroutineProfile STWReason = 4 + STWGoroutineProfileCleanup STWReason = 5 + STWAllGoroutinesStackTrace STWReason = 6 + STWReadMemStats STWReason = 7 + STWAllThreadsSyscall STWReason = 8 + STWGOMAXPROCS STWReason = 9 + STWStartTrace STWReason = 10 + STWStopTrace STWReason = 11 + STWCountPagesInUse STWReason = 12 + STWReadMetricsSlow STWReason = 13 + STWReadMemStatsSlow STWReason = 14 + STWPageCachePagesLeaked STWReason = 15 + STWResetDebugLog STWReason = 16 + + NumSTWReasons = 17 +) diff --git a/src/internal/trace/internal/oldtrace/parser_test.go b/src/internal/trace/internal/tracev1/parser_test.go similarity index 99% rename from src/internal/trace/internal/oldtrace/parser_test.go rename to src/internal/trace/internal/tracev1/parser_test.go index 6fe31e2e7e1fe1..af6d8db2340eab 100644 --- a/src/internal/trace/internal/oldtrace/parser_test.go +++ b/src/internal/trace/internal/tracev1/parser_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package oldtrace +package tracev1 import ( "bytes" diff --git a/src/internal/trace/internal/oldtrace/testdata/fmt_1_21_pprof_good b/src/internal/trace/internal/tracev1/testdata/fmt_1_21_pprof_good similarity index 100% rename from src/internal/trace/internal/oldtrace/testdata/fmt_1_21_pprof_good rename to src/internal/trace/internal/tracev1/testdata/fmt_1_21_pprof_good diff --git a/src/internal/trace/internal/oldtrace/testdata/http_1_19_good b/src/internal/trace/internal/tracev1/testdata/http_1_19_good similarity index 100% rename from src/internal/trace/internal/oldtrace/testdata/http_1_19_good rename to src/internal/trace/internal/tracev1/testdata/http_1_19_good diff --git a/src/internal/trace/internal/oldtrace/testdata/http_1_21_good b/src/internal/trace/internal/tracev1/testdata/http_1_21_good similarity index 100% rename from src/internal/trace/internal/oldtrace/testdata/http_1_21_good rename to src/internal/trace/internal/tracev1/testdata/http_1_21_good diff --git a/src/internal/trace/internal/oldtrace/testdata/stress_1_11_good b/src/internal/trace/internal/tracev1/testdata/stress_1_11_good similarity index 100% rename from src/internal/trace/internal/oldtrace/testdata/stress_1_11_good rename to src/internal/trace/internal/tracev1/testdata/stress_1_11_good diff --git a/src/internal/trace/internal/oldtrace/testdata/stress_1_19_good b/src/internal/trace/internal/tracev1/testdata/stress_1_19_good similarity index 100% rename from src/internal/trace/internal/oldtrace/testdata/stress_1_19_good rename to src/internal/trace/internal/tracev1/testdata/stress_1_19_good diff --git a/src/internal/trace/internal/oldtrace/testdata/stress_1_21_good b/src/internal/trace/internal/tracev1/testdata/stress_1_21_good similarity index 100% rename from src/internal/trace/internal/oldtrace/testdata/stress_1_21_good rename to src/internal/trace/internal/tracev1/testdata/stress_1_21_good diff --git a/src/internal/trace/internal/oldtrace/testdata/stress_start_stop_1_11_good b/src/internal/trace/internal/tracev1/testdata/stress_start_stop_1_11_good similarity index 100% rename from src/internal/trace/internal/oldtrace/testdata/stress_start_stop_1_11_good rename to src/internal/trace/internal/tracev1/testdata/stress_start_stop_1_11_good diff --git a/src/internal/trace/internal/oldtrace/testdata/stress_start_stop_1_19_good b/src/internal/trace/internal/tracev1/testdata/stress_start_stop_1_19_good similarity index 100% rename from src/internal/trace/internal/oldtrace/testdata/stress_start_stop_1_19_good rename to src/internal/trace/internal/tracev1/testdata/stress_start_stop_1_19_good diff --git a/src/internal/trace/internal/oldtrace/testdata/stress_start_stop_1_21_good b/src/internal/trace/internal/tracev1/testdata/stress_start_stop_1_21_good similarity index 100% rename from src/internal/trace/internal/oldtrace/testdata/stress_start_stop_1_21_good rename to src/internal/trace/internal/tracev1/testdata/stress_start_stop_1_21_good diff --git a/src/internal/trace/internal/oldtrace/testdata/user_task_region_1_11_good b/src/internal/trace/internal/tracev1/testdata/user_task_region_1_11_good similarity index 100% rename from src/internal/trace/internal/oldtrace/testdata/user_task_region_1_11_good rename to src/internal/trace/internal/tracev1/testdata/user_task_region_1_11_good diff --git a/src/internal/trace/internal/oldtrace/testdata/user_task_region_1_19_good b/src/internal/trace/internal/tracev1/testdata/user_task_region_1_19_good similarity index 100% rename from src/internal/trace/internal/oldtrace/testdata/user_task_region_1_19_good rename to src/internal/trace/internal/tracev1/testdata/user_task_region_1_19_good diff --git a/src/internal/trace/internal/oldtrace/testdata/user_task_region_1_21_good b/src/internal/trace/internal/tracev1/testdata/user_task_region_1_21_good similarity index 100% rename from src/internal/trace/internal/oldtrace/testdata/user_task_region_1_21_good rename to src/internal/trace/internal/tracev1/testdata/user_task_region_1_21_good diff --git a/src/internal/trace/oldtrace.go b/src/internal/trace/oldtrace.go deleted file mode 100644 index c49f8c7474d49a..00000000000000 --- a/src/internal/trace/oldtrace.go +++ /dev/null @@ -1,568 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file implements conversion from old (Go 1.11–Go 1.21) traces to the Go -// 1.22 format. -// -// Most events have direct equivalents in 1.22, at worst requiring arguments to -// be reordered. Some events, such as GoWaiting need to look ahead for follow-up -// events to determine the correct translation. GoSyscall, which is an -// instantaneous event, gets turned into a 1 ns long pair of -// GoSyscallStart+GoSyscallEnd, unless we observe a GoSysBlock, in which case we -// emit a GoSyscallStart+GoSyscallEndBlocked pair with the correct duration -// (i.e. starting at the original GoSyscall). -// -// The resulting trace treats the old trace as a single, large generation, -// sharing a single evTable for all events. -// -// We use a new (compared to what was used for 'go tool trace' in earlier -// versions of Go) parser for old traces that is optimized for speed, low memory -// usage, and minimal GC pressure. It allocates events in batches so that even -// though we have to load the entire trace into memory, the conversion process -// shouldn't result in a doubling of memory usage, even if all converted events -// are kept alive, as we free batches once we're done with them. -// -// The conversion process is lossless. - -package trace - -import ( - "errors" - "fmt" - "internal/trace/event" - "internal/trace/event/go122" - "internal/trace/internal/oldtrace" - "io" -) - -type oldTraceConverter struct { - trace oldtrace.Trace - evt *evTable - preInit bool - createdPreInit map[GoID]struct{} - events oldtrace.Events - extra []Event - extraArr [3]Event - tasks map[TaskID]taskState - seenProcs map[ProcID]struct{} - lastTs Time - procMs map[ProcID]ThreadID - lastStwReason uint64 - - inlineToStringID []uint64 - builtinToStringID []uint64 -} - -const ( - // Block reasons - sForever = iota - sPreempted - sGosched - sSleep - sChanSend - sChanRecv - sNetwork - sSync - sSyncCond - sSelect - sEmpty - sMarkAssistWait - - // STW kinds - sSTWUnknown - sSTWGCMarkTermination - sSTWGCSweepTermination - sSTWWriteHeapDump - sSTWGoroutineProfile - sSTWGoroutineProfileCleanup - sSTWAllGoroutinesStackTrace - sSTWReadMemStats - sSTWAllThreadsSyscall - sSTWGOMAXPROCS - sSTWStartTrace - sSTWStopTrace - sSTWCountPagesInUse - sSTWReadMetricsSlow - sSTWReadMemStatsSlow - sSTWPageCachePagesLeaked - sSTWResetDebugLog - - sLast -) - -func (it *oldTraceConverter) init(pr oldtrace.Trace) error { - it.trace = pr - it.preInit = true - it.createdPreInit = make(map[GoID]struct{}) - it.evt = &evTable{pcs: make(map[uint64]frame)} - it.events = pr.Events - it.extra = it.extraArr[:0] - it.tasks = make(map[TaskID]taskState) - it.seenProcs = make(map[ProcID]struct{}) - it.procMs = make(map[ProcID]ThreadID) - it.lastTs = -1 - - evt := it.evt - - // Convert from oldtracer's Strings map to our dataTable. - var max uint64 - for id, s := range pr.Strings { - evt.strings.insert(stringID(id), s) - if id > max { - max = id - } - } - pr.Strings = nil - - // Add all strings used for UserLog. In the old trace format, these were - // stored inline and didn't have IDs. We generate IDs for them. - if max+uint64(len(pr.InlineStrings)) < max { - return errors.New("trace contains too many strings") - } - var addErr error - add := func(id stringID, s string) { - if err := evt.strings.insert(id, s); err != nil && addErr == nil { - addErr = err - } - } - for id, s := range pr.InlineStrings { - nid := max + 1 + uint64(id) - it.inlineToStringID = append(it.inlineToStringID, nid) - add(stringID(nid), s) - } - max += uint64(len(pr.InlineStrings)) - pr.InlineStrings = nil - - // Add strings that the converter emits explicitly. - if max+uint64(sLast) < max { - return errors.New("trace contains too many strings") - } - it.builtinToStringID = make([]uint64, sLast) - addBuiltin := func(c int, s string) { - nid := max + 1 + uint64(c) - it.builtinToStringID[c] = nid - add(stringID(nid), s) - } - addBuiltin(sForever, "forever") - addBuiltin(sPreempted, "preempted") - addBuiltin(sGosched, "runtime.Gosched") - addBuiltin(sSleep, "sleep") - addBuiltin(sChanSend, "chan send") - addBuiltin(sChanRecv, "chan receive") - addBuiltin(sNetwork, "network") - addBuiltin(sSync, "sync") - addBuiltin(sSyncCond, "sync.(*Cond).Wait") - addBuiltin(sSelect, "select") - addBuiltin(sEmpty, "") - addBuiltin(sMarkAssistWait, "GC mark assist wait for work") - addBuiltin(sSTWUnknown, "") - addBuiltin(sSTWGCMarkTermination, "GC mark termination") - addBuiltin(sSTWGCSweepTermination, "GC sweep termination") - addBuiltin(sSTWWriteHeapDump, "write heap dump") - addBuiltin(sSTWGoroutineProfile, "goroutine profile") - addBuiltin(sSTWGoroutineProfileCleanup, "goroutine profile cleanup") - addBuiltin(sSTWAllGoroutinesStackTrace, "all goroutine stack trace") - addBuiltin(sSTWReadMemStats, "read mem stats") - addBuiltin(sSTWAllThreadsSyscall, "AllThreadsSyscall") - addBuiltin(sSTWGOMAXPROCS, "GOMAXPROCS") - addBuiltin(sSTWStartTrace, "start trace") - addBuiltin(sSTWStopTrace, "stop trace") - addBuiltin(sSTWCountPagesInUse, "CountPagesInUse (test)") - addBuiltin(sSTWReadMetricsSlow, "ReadMetricsSlow (test)") - addBuiltin(sSTWReadMemStatsSlow, "ReadMemStatsSlow (test)") - addBuiltin(sSTWPageCachePagesLeaked, "PageCachePagesLeaked (test)") - addBuiltin(sSTWResetDebugLog, "ResetDebugLog (test)") - - if addErr != nil { - // This should be impossible but let's be safe. - return fmt.Errorf("couldn't add strings: %w", addErr) - } - - it.evt.strings.compactify() - - // Convert stacks. - for id, stk := range pr.Stacks { - evt.stacks.insert(stackID(id), stack{pcs: stk}) - } - - // OPT(dh): if we could share the frame type between this package and - // oldtrace we wouldn't have to copy the map. - for pc, f := range pr.PCs { - evt.pcs[pc] = frame{ - pc: pc, - funcID: stringID(f.Fn), - fileID: stringID(f.File), - line: uint64(f.Line), - } - } - pr.Stacks = nil - pr.PCs = nil - evt.stacks.compactify() - return nil -} - -// next returns the next event, io.EOF if there are no more events, or a -// descriptive error for invalid events. -func (it *oldTraceConverter) next() (Event, error) { - if len(it.extra) > 0 { - ev := it.extra[0] - it.extra = it.extra[1:] - - if len(it.extra) == 0 { - it.extra = it.extraArr[:0] - } - // Two events aren't allowed to fall on the same timestamp in the new API, - // but this may happen when we produce EvGoStatus events - if ev.base.time <= it.lastTs { - ev.base.time = it.lastTs + 1 - } - it.lastTs = ev.base.time - return ev, nil - } - - oev, ok := it.events.Pop() - if !ok { - return Event{}, io.EOF - } - - ev, err := it.convertEvent(oev) - - if err == errSkip { - return it.next() - } else if err != nil { - return Event{}, err - } - - // Two events aren't allowed to fall on the same timestamp in the new API, - // but this may happen when we produce EvGoStatus events - if ev.base.time <= it.lastTs { - ev.base.time = it.lastTs + 1 - } - it.lastTs = ev.base.time - return ev, nil -} - -var errSkip = errors.New("skip event") - -// convertEvent converts an event from the old trace format to zero or more -// events in the new format. Most events translate 1 to 1. Some events don't -// result in an event right away, in which case convertEvent returns errSkip. -// Some events result in more than one new event; in this case, convertEvent -// returns the first event and stores additional events in it.extra. When -// encountering events that oldtrace shouldn't be able to emit, ocnvertEvent -// returns a descriptive error. -func (it *oldTraceConverter) convertEvent(ev *oldtrace.Event) (OUT Event, ERR error) { - var mappedType event.Type - var mappedArgs timedEventArgs - copy(mappedArgs[:], ev.Args[:]) - - switch ev.Type { - case oldtrace.EvGomaxprocs: - mappedType = go122.EvProcsChange - if it.preInit { - // The first EvGomaxprocs signals the end of trace initialization. At this point we've seen - // all goroutines that already existed at trace begin. - it.preInit = false - for gid := range it.createdPreInit { - // These are goroutines that already existed when tracing started but for which we - // received neither GoWaiting, GoInSyscall, or GoStart. These are goroutines that are in - // the states _Gidle or _Grunnable. - it.extra = append(it.extra, Event{ - ctx: schedCtx{ - // G: GoID(gid), - G: NoGoroutine, - P: NoProc, - M: NoThread, - }, - table: it.evt, - base: baseEvent{ - typ: go122.EvGoStatus, - time: Time(ev.Ts), - args: timedEventArgs{uint64(gid), ^uint64(0), uint64(go122.GoRunnable)}, - }, - }) - } - it.createdPreInit = nil - return Event{}, errSkip - } - case oldtrace.EvProcStart: - it.procMs[ProcID(ev.P)] = ThreadID(ev.Args[0]) - if _, ok := it.seenProcs[ProcID(ev.P)]; ok { - mappedType = go122.EvProcStart - mappedArgs = timedEventArgs{uint64(ev.P)} - } else { - it.seenProcs[ProcID(ev.P)] = struct{}{} - mappedType = go122.EvProcStatus - mappedArgs = timedEventArgs{uint64(ev.P), uint64(go122.ProcRunning)} - } - case oldtrace.EvProcStop: - if _, ok := it.seenProcs[ProcID(ev.P)]; ok { - mappedType = go122.EvProcStop - mappedArgs = timedEventArgs{uint64(ev.P)} - } else { - it.seenProcs[ProcID(ev.P)] = struct{}{} - mappedType = go122.EvProcStatus - mappedArgs = timedEventArgs{uint64(ev.P), uint64(go122.ProcIdle)} - } - case oldtrace.EvGCStart: - mappedType = go122.EvGCBegin - case oldtrace.EvGCDone: - mappedType = go122.EvGCEnd - case oldtrace.EvSTWStart: - sid := it.builtinToStringID[sSTWUnknown+it.trace.STWReason(ev.Args[0])] - it.lastStwReason = sid - mappedType = go122.EvSTWBegin - mappedArgs = timedEventArgs{uint64(sid)} - case oldtrace.EvSTWDone: - mappedType = go122.EvSTWEnd - mappedArgs = timedEventArgs{it.lastStwReason} - case oldtrace.EvGCSweepStart: - mappedType = go122.EvGCSweepBegin - case oldtrace.EvGCSweepDone: - mappedType = go122.EvGCSweepEnd - case oldtrace.EvGoCreate: - if it.preInit { - it.createdPreInit[GoID(ev.Args[0])] = struct{}{} - return Event{}, errSkip - } - mappedType = go122.EvGoCreate - case oldtrace.EvGoStart: - if it.preInit { - mappedType = go122.EvGoStatus - mappedArgs = timedEventArgs{ev.Args[0], ^uint64(0), uint64(go122.GoRunning)} - delete(it.createdPreInit, GoID(ev.Args[0])) - } else { - mappedType = go122.EvGoStart - } - case oldtrace.EvGoStartLabel: - it.extra = []Event{{ - ctx: schedCtx{ - G: GoID(ev.G), - P: ProcID(ev.P), - M: it.procMs[ProcID(ev.P)], - }, - table: it.evt, - base: baseEvent{ - typ: go122.EvGoLabel, - time: Time(ev.Ts), - args: timedEventArgs{ev.Args[2]}, - }, - }} - return Event{ - ctx: schedCtx{ - G: GoID(ev.G), - P: ProcID(ev.P), - M: it.procMs[ProcID(ev.P)], - }, - table: it.evt, - base: baseEvent{ - typ: go122.EvGoStart, - time: Time(ev.Ts), - args: mappedArgs, - }, - }, nil - case oldtrace.EvGoEnd: - mappedType = go122.EvGoDestroy - case oldtrace.EvGoStop: - mappedType = go122.EvGoBlock - mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sForever]), uint64(ev.StkID)} - case oldtrace.EvGoSched: - mappedType = go122.EvGoStop - mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sGosched]), uint64(ev.StkID)} - case oldtrace.EvGoPreempt: - mappedType = go122.EvGoStop - mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sPreempted]), uint64(ev.StkID)} - case oldtrace.EvGoSleep: - mappedType = go122.EvGoBlock - mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sSleep]), uint64(ev.StkID)} - case oldtrace.EvGoBlock: - mappedType = go122.EvGoBlock - mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sEmpty]), uint64(ev.StkID)} - case oldtrace.EvGoUnblock: - mappedType = go122.EvGoUnblock - case oldtrace.EvGoBlockSend: - mappedType = go122.EvGoBlock - mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sChanSend]), uint64(ev.StkID)} - case oldtrace.EvGoBlockRecv: - mappedType = go122.EvGoBlock - mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sChanRecv]), uint64(ev.StkID)} - case oldtrace.EvGoBlockSelect: - mappedType = go122.EvGoBlock - mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sSelect]), uint64(ev.StkID)} - case oldtrace.EvGoBlockSync: - mappedType = go122.EvGoBlock - mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sSync]), uint64(ev.StkID)} - case oldtrace.EvGoBlockCond: - mappedType = go122.EvGoBlock - mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sSyncCond]), uint64(ev.StkID)} - case oldtrace.EvGoBlockNet: - mappedType = go122.EvGoBlock - mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sNetwork]), uint64(ev.StkID)} - case oldtrace.EvGoBlockGC: - mappedType = go122.EvGoBlock - mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sMarkAssistWait]), uint64(ev.StkID)} - case oldtrace.EvGoSysCall: - // Look for the next event for the same G to determine if the syscall - // blocked. - blocked := false - it.events.All()(func(nev *oldtrace.Event) bool { - if nev.G != ev.G { - return true - } - // After an EvGoSysCall, the next event on the same G will either be - // EvGoSysBlock to denote a blocking syscall, or some other event - // (or the end of the trace) if the syscall didn't block. - if nev.Type == oldtrace.EvGoSysBlock { - blocked = true - } - return false - }) - if blocked { - mappedType = go122.EvGoSyscallBegin - mappedArgs = timedEventArgs{1: uint64(ev.StkID)} - } else { - // Convert the old instantaneous syscall event to a pair of syscall - // begin and syscall end and give it the shortest possible duration, - // 1ns. - out1 := Event{ - ctx: schedCtx{ - G: GoID(ev.G), - P: ProcID(ev.P), - M: it.procMs[ProcID(ev.P)], - }, - table: it.evt, - base: baseEvent{ - typ: go122.EvGoSyscallBegin, - time: Time(ev.Ts), - args: timedEventArgs{1: uint64(ev.StkID)}, - }, - } - - out2 := Event{ - ctx: out1.ctx, - table: it.evt, - base: baseEvent{ - typ: go122.EvGoSyscallEnd, - time: Time(ev.Ts + 1), - args: timedEventArgs{}, - }, - } - - it.extra = append(it.extra, out2) - return out1, nil - } - - case oldtrace.EvGoSysExit: - mappedType = go122.EvGoSyscallEndBlocked - case oldtrace.EvGoSysBlock: - return Event{}, errSkip - case oldtrace.EvGoWaiting: - mappedType = go122.EvGoStatus - mappedArgs = timedEventArgs{ev.Args[0], ^uint64(0), uint64(go122.GoWaiting)} - delete(it.createdPreInit, GoID(ev.Args[0])) - case oldtrace.EvGoInSyscall: - mappedType = go122.EvGoStatus - // In the new tracer, GoStatus with GoSyscall knows what thread the - // syscall is on. In the old tracer, EvGoInSyscall doesn't contain that - // information and all we can do here is specify NoThread. - mappedArgs = timedEventArgs{ev.Args[0], ^uint64(0), uint64(go122.GoSyscall)} - delete(it.createdPreInit, GoID(ev.Args[0])) - case oldtrace.EvHeapAlloc: - mappedType = go122.EvHeapAlloc - case oldtrace.EvHeapGoal: - mappedType = go122.EvHeapGoal - case oldtrace.EvGCMarkAssistStart: - mappedType = go122.EvGCMarkAssistBegin - case oldtrace.EvGCMarkAssistDone: - mappedType = go122.EvGCMarkAssistEnd - case oldtrace.EvUserTaskCreate: - mappedType = go122.EvUserTaskBegin - parent := ev.Args[1] - if parent == 0 { - parent = uint64(NoTask) - } - mappedArgs = timedEventArgs{ev.Args[0], parent, ev.Args[2], uint64(ev.StkID)} - name, _ := it.evt.strings.get(stringID(ev.Args[2])) - it.tasks[TaskID(ev.Args[0])] = taskState{name: name, parentID: TaskID(ev.Args[1])} - case oldtrace.EvUserTaskEnd: - mappedType = go122.EvUserTaskEnd - // Event.Task expects the parent and name to be smuggled in extra args - // and as extra strings. - ts, ok := it.tasks[TaskID(ev.Args[0])] - if ok { - delete(it.tasks, TaskID(ev.Args[0])) - mappedArgs = timedEventArgs{ - ev.Args[0], - ev.Args[1], - uint64(ts.parentID), - uint64(it.evt.addExtraString(ts.name)), - } - } else { - mappedArgs = timedEventArgs{ev.Args[0], ev.Args[1], uint64(NoTask), uint64(it.evt.addExtraString(""))} - } - case oldtrace.EvUserRegion: - switch ev.Args[1] { - case 0: // start - mappedType = go122.EvUserRegionBegin - case 1: // end - mappedType = go122.EvUserRegionEnd - } - mappedArgs = timedEventArgs{ev.Args[0], ev.Args[2], uint64(ev.StkID)} - case oldtrace.EvUserLog: - mappedType = go122.EvUserLog - mappedArgs = timedEventArgs{ev.Args[0], ev.Args[1], it.inlineToStringID[ev.Args[3]], uint64(ev.StkID)} - case oldtrace.EvCPUSample: - mappedType = go122.EvCPUSample - // When emitted by the Go 1.22 tracer, CPU samples have 5 arguments: - // timestamp, M, P, G, stack. However, after they get turned into Event, - // they have the arguments stack, M, P, G. - // - // In Go 1.21, CPU samples did not have Ms. - mappedArgs = timedEventArgs{uint64(ev.StkID), ^uint64(0), uint64(ev.P), ev.G} - default: - return Event{}, fmt.Errorf("unexpected event type %v", ev.Type) - } - - if oldtrace.EventDescriptions[ev.Type].Stack { - if stackIDs := go122.Specs()[mappedType].StackIDs; len(stackIDs) > 0 { - mappedArgs[stackIDs[0]-1] = uint64(ev.StkID) - } - } - - m := NoThread - if ev.P != -1 && ev.Type != oldtrace.EvCPUSample { - if t, ok := it.procMs[ProcID(ev.P)]; ok { - m = ThreadID(t) - } - } - if ev.Type == oldtrace.EvProcStop { - delete(it.procMs, ProcID(ev.P)) - } - g := GoID(ev.G) - if g == 0 { - g = NoGoroutine - } - out := Event{ - ctx: schedCtx{ - G: GoID(g), - P: ProcID(ev.P), - M: m, - }, - table: it.evt, - base: baseEvent{ - typ: mappedType, - time: Time(ev.Ts), - args: mappedArgs, - }, - } - return out, nil -} - -// convertOldFormat takes a fully loaded trace in the old trace format and -// returns an iterator over events in the new format. -func convertOldFormat(pr oldtrace.Trace) *oldTraceConverter { - it := &oldTraceConverter{} - it.init(pr) - return it -} diff --git a/src/internal/trace/oldtrace_test.go b/src/internal/trace/oldtrace_test.go deleted file mode 100644 index f812d5ef840700..00000000000000 --- a/src/internal/trace/oldtrace_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package trace_test - -import ( - "internal/trace" - "internal/trace/testtrace" - "io" - "os" - "path/filepath" - "testing" -) - -func TestOldtrace(t *testing.T) { - traces, err := filepath.Glob("./internal/oldtrace/testdata/*_good") - if err != nil { - t.Fatalf("failed to glob for tests: %s", err) - } - var testedUserRegions bool - for _, p := range traces { - p := p - testName, err := filepath.Rel("./internal/oldtrace/testdata", p) - if err != nil { - t.Fatalf("failed to relativize testdata path: %s", err) - } - t.Run(testName, func(t *testing.T) { - f, err := os.Open(p) - if err != nil { - t.Fatalf("failed to open test %q: %s", p, err) - } - defer f.Close() - - tr, err := trace.NewReader(f) - if err != nil { - t.Fatalf("failed to create reader: %s", err) - } - - v := testtrace.NewValidator() - v.Go121 = true - for { - ev, err := tr.ReadEvent() - if err != nil { - if err == io.EOF { - break - } - t.Fatalf("couldn't read converted event: %s", err) - } - if err := v.Event(ev); err != nil { - t.Fatalf("converted event did not validate; event: \n%s\nerror: %s", ev, err) - } - - if testName == "user_task_region_1_21_good" { - testedUserRegions = true - validRegions := map[string]struct{}{ - "post-existing region": struct{}{}, - "region0": struct{}{}, - "region1": struct{}{}, - } - // Check that we correctly convert user regions. These - // strings were generated by - // runtime/trace.TestUserTaskRegion, which is the basis for - // the user_task_region_* test cases. We only check for the - // Go 1.21 traces because earlier traces used different - // strings. - switch ev.Kind() { - case trace.EventRegionBegin, trace.EventRegionEnd: - if _, ok := validRegions[ev.Region().Type]; !ok { - t.Fatalf("converted event has unexpected region type:\n%s", ev) - } - case trace.EventTaskBegin, trace.EventTaskEnd: - if ev.Task().Type != "task0" { - t.Fatalf("converted event has unexpected task type name:\n%s", ev) - } - case trace.EventLog: - l := ev.Log() - if l.Task != 1 || l.Category != "key0" || l.Message != "0123456789abcdef" { - t.Fatalf("converted event has unexpected user log:\n%s", ev) - } - } - } - } - }) - } - if !testedUserRegions { - t.Fatal("didn't see expected test case user_task_region_1_21_good") - } -} diff --git a/src/internal/trace/order.go b/src/internal/trace/order.go index d0818a500c03c8..7b6075d5637732 100644 --- a/src/internal/trace/order.go +++ b/src/internal/trace/order.go @@ -6,10 +6,10 @@ package trace import ( "fmt" + "slices" "strings" - "internal/trace/event" - "internal/trace/event/go122" + "internal/trace/tracev2" "internal/trace/version" ) @@ -21,6 +21,7 @@ import ( // add completed events to the ordering. Next is used to pick // off events in the ordering. type ordering struct { + traceVer version.Version gStates map[GoID]*gState pStates map[ProcID]*pState // TODO: The keys are dense, so this can be a slice. mStates map[ThreadID]*mState @@ -87,101 +88,105 @@ func (o *ordering) Advance(ev *baseEvent, evt *evTable, m ThreadID, gen uint64) return ok, err } +func (o *ordering) evName(typ tracev2.EventType) string { + return o.traceVer.EventName(typ) +} + type orderingHandleFunc func(o *ordering, ev *baseEvent, evt *evTable, m ThreadID, gen uint64, curCtx schedCtx) (schedCtx, bool, error) var orderingDispatch = [256]orderingHandleFunc{ // Procs. - go122.EvProcsChange: (*ordering).advanceAnnotation, - go122.EvProcStart: (*ordering).advanceProcStart, - go122.EvProcStop: (*ordering).advanceProcStop, - go122.EvProcSteal: (*ordering).advanceProcSteal, - go122.EvProcStatus: (*ordering).advanceProcStatus, + tracev2.EvProcsChange: (*ordering).advanceAnnotation, + tracev2.EvProcStart: (*ordering).advanceProcStart, + tracev2.EvProcStop: (*ordering).advanceProcStop, + tracev2.EvProcSteal: (*ordering).advanceProcSteal, + tracev2.EvProcStatus: (*ordering).advanceProcStatus, // Goroutines. - go122.EvGoCreate: (*ordering).advanceGoCreate, - go122.EvGoCreateSyscall: (*ordering).advanceGoCreateSyscall, - go122.EvGoStart: (*ordering).advanceGoStart, - go122.EvGoDestroy: (*ordering).advanceGoStopExec, - go122.EvGoDestroySyscall: (*ordering).advanceGoDestroySyscall, - go122.EvGoStop: (*ordering).advanceGoStopExec, - go122.EvGoBlock: (*ordering).advanceGoStopExec, - go122.EvGoUnblock: (*ordering).advanceGoUnblock, - go122.EvGoSyscallBegin: (*ordering).advanceGoSyscallBegin, - go122.EvGoSyscallEnd: (*ordering).advanceGoSyscallEnd, - go122.EvGoSyscallEndBlocked: (*ordering).advanceGoSyscallEndBlocked, - go122.EvGoStatus: (*ordering).advanceGoStatus, + tracev2.EvGoCreate: (*ordering).advanceGoCreate, + tracev2.EvGoCreateSyscall: (*ordering).advanceGoCreateSyscall, + tracev2.EvGoStart: (*ordering).advanceGoStart, + tracev2.EvGoDestroy: (*ordering).advanceGoStopExec, + tracev2.EvGoDestroySyscall: (*ordering).advanceGoDestroySyscall, + tracev2.EvGoStop: (*ordering).advanceGoStopExec, + tracev2.EvGoBlock: (*ordering).advanceGoStopExec, + tracev2.EvGoUnblock: (*ordering).advanceGoUnblock, + tracev2.EvGoSyscallBegin: (*ordering).advanceGoSyscallBegin, + tracev2.EvGoSyscallEnd: (*ordering).advanceGoSyscallEnd, + tracev2.EvGoSyscallEndBlocked: (*ordering).advanceGoSyscallEndBlocked, + tracev2.EvGoStatus: (*ordering).advanceGoStatus, // STW. - go122.EvSTWBegin: (*ordering).advanceGoRangeBegin, - go122.EvSTWEnd: (*ordering).advanceGoRangeEnd, + tracev2.EvSTWBegin: (*ordering).advanceGoRangeBegin, + tracev2.EvSTWEnd: (*ordering).advanceGoRangeEnd, // GC events. - go122.EvGCActive: (*ordering).advanceGCActive, - go122.EvGCBegin: (*ordering).advanceGCBegin, - go122.EvGCEnd: (*ordering).advanceGCEnd, - go122.EvGCSweepActive: (*ordering).advanceGCSweepActive, - go122.EvGCSweepBegin: (*ordering).advanceGCSweepBegin, - go122.EvGCSweepEnd: (*ordering).advanceGCSweepEnd, - go122.EvGCMarkAssistActive: (*ordering).advanceGoRangeActive, - go122.EvGCMarkAssistBegin: (*ordering).advanceGoRangeBegin, - go122.EvGCMarkAssistEnd: (*ordering).advanceGoRangeEnd, - go122.EvHeapAlloc: (*ordering).advanceHeapMetric, - go122.EvHeapGoal: (*ordering).advanceHeapMetric, + tracev2.EvGCActive: (*ordering).advanceGCActive, + tracev2.EvGCBegin: (*ordering).advanceGCBegin, + tracev2.EvGCEnd: (*ordering).advanceGCEnd, + tracev2.EvGCSweepActive: (*ordering).advanceGCSweepActive, + tracev2.EvGCSweepBegin: (*ordering).advanceGCSweepBegin, + tracev2.EvGCSweepEnd: (*ordering).advanceGCSweepEnd, + tracev2.EvGCMarkAssistActive: (*ordering).advanceGoRangeActive, + tracev2.EvGCMarkAssistBegin: (*ordering).advanceGoRangeBegin, + tracev2.EvGCMarkAssistEnd: (*ordering).advanceGoRangeEnd, + tracev2.EvHeapAlloc: (*ordering).advanceHeapMetric, + tracev2.EvHeapGoal: (*ordering).advanceHeapMetric, // Annotations. - go122.EvGoLabel: (*ordering).advanceAnnotation, - go122.EvUserTaskBegin: (*ordering).advanceUserTaskBegin, - go122.EvUserTaskEnd: (*ordering).advanceUserTaskEnd, - go122.EvUserRegionBegin: (*ordering).advanceUserRegionBegin, - go122.EvUserRegionEnd: (*ordering).advanceUserRegionEnd, - go122.EvUserLog: (*ordering).advanceAnnotation, + tracev2.EvGoLabel: (*ordering).advanceAnnotation, + tracev2.EvUserTaskBegin: (*ordering).advanceUserTaskBegin, + tracev2.EvUserTaskEnd: (*ordering).advanceUserTaskEnd, + tracev2.EvUserRegionBegin: (*ordering).advanceUserRegionBegin, + tracev2.EvUserRegionEnd: (*ordering).advanceUserRegionEnd, + tracev2.EvUserLog: (*ordering).advanceAnnotation, // Coroutines. Added in Go 1.23. - go122.EvGoSwitch: (*ordering).advanceGoSwitch, - go122.EvGoSwitchDestroy: (*ordering).advanceGoSwitch, - go122.EvGoCreateBlocked: (*ordering).advanceGoCreate, + tracev2.EvGoSwitch: (*ordering).advanceGoSwitch, + tracev2.EvGoSwitchDestroy: (*ordering).advanceGoSwitch, + tracev2.EvGoCreateBlocked: (*ordering).advanceGoCreate, // GoStatus event with a stack. Added in Go 1.23. - go122.EvGoStatusStack: (*ordering).advanceGoStatus, + tracev2.EvGoStatusStack: (*ordering).advanceGoStatus, // Experimental events. // Experimental heap span events. Added in Go 1.23. - go122.EvSpan: (*ordering).advanceAllocFree, - go122.EvSpanAlloc: (*ordering).advanceAllocFree, - go122.EvSpanFree: (*ordering).advanceAllocFree, + tracev2.EvSpan: (*ordering).advanceAllocFree, + tracev2.EvSpanAlloc: (*ordering).advanceAllocFree, + tracev2.EvSpanFree: (*ordering).advanceAllocFree, // Experimental heap object events. Added in Go 1.23. - go122.EvHeapObject: (*ordering).advanceAllocFree, - go122.EvHeapObjectAlloc: (*ordering).advanceAllocFree, - go122.EvHeapObjectFree: (*ordering).advanceAllocFree, + tracev2.EvHeapObject: (*ordering).advanceAllocFree, + tracev2.EvHeapObjectAlloc: (*ordering).advanceAllocFree, + tracev2.EvHeapObjectFree: (*ordering).advanceAllocFree, // Experimental goroutine stack events. Added in Go 1.23. - go122.EvGoroutineStack: (*ordering).advanceAllocFree, - go122.EvGoroutineStackAlloc: (*ordering).advanceAllocFree, - go122.EvGoroutineStackFree: (*ordering).advanceAllocFree, + tracev2.EvGoroutineStack: (*ordering).advanceAllocFree, + tracev2.EvGoroutineStackAlloc: (*ordering).advanceAllocFree, + tracev2.EvGoroutineStackFree: (*ordering).advanceAllocFree, } func (o *ordering) advanceProcStatus(ev *baseEvent, evt *evTable, m ThreadID, gen uint64, curCtx schedCtx) (schedCtx, bool, error) { pid := ProcID(ev.args[0]) - status := go122.ProcStatus(ev.args[1]) - if int(status) >= len(go122ProcStatus2ProcState) { + status := tracev2.ProcStatus(ev.args[1]) + if int(status) >= len(tracev2ProcStatus2ProcState) { return curCtx, false, fmt.Errorf("invalid status for proc %d: %d", pid, status) } - oldState := go122ProcStatus2ProcState[status] + oldState := tracev2ProcStatus2ProcState[status] if s, ok := o.pStates[pid]; ok { - if status == go122.ProcSyscallAbandoned && s.status == go122.ProcSyscall { + if status == tracev2.ProcSyscallAbandoned && s.status == tracev2.ProcSyscall { // ProcSyscallAbandoned is a special case of ProcSyscall. It indicates a // potential loss of information, but if we're already in ProcSyscall, // we haven't lost the relevant information. Promote the status and advance. oldState = ProcRunning - ev.args[1] = uint64(go122.ProcSyscall) - } else if status == go122.ProcSyscallAbandoned && s.status == go122.ProcSyscallAbandoned { + ev.args[1] = uint64(tracev2.ProcSyscall) + } else if status == tracev2.ProcSyscallAbandoned && s.status == tracev2.ProcSyscallAbandoned { // If we're passing through ProcSyscallAbandoned, then there's no promotion // to do. We've lost the M that this P is associated with. However it got there, // it's going to appear as idle in the API, so pass through as idle. oldState = ProcIdle - ev.args[1] = uint64(go122.ProcSyscallAbandoned) + ev.args[1] = uint64(tracev2.ProcSyscallAbandoned) } else if s.status != status { return curCtx, false, fmt.Errorf("inconsistent status for proc %d: old %v vs. new %v", pid, s.status, status) } @@ -198,7 +203,7 @@ func (o *ordering) advanceProcStatus(ev *baseEvent, evt *evTable, m ThreadID, ge // Bind the proc to the new context, if it's running. newCtx := curCtx - if status == go122.ProcRunning || status == go122.ProcSyscall { + if status == tracev2.ProcRunning || status == tracev2.ProcSyscall { newCtx.P = pid } // If we're advancing through ProcSyscallAbandoned *but* oldState is running then we've @@ -207,7 +212,7 @@ func (o *ordering) advanceProcStatus(ev *baseEvent, evt *evTable, m ThreadID, ge // thread it was bound to. Since this status is Running -> Running and Running is binding, // we need to make sure we emit it in the right context: the context to which it is bound. // Find it, and set our current context to it. - if status == go122.ProcSyscallAbandoned && oldState == ProcRunning { + if status == tracev2.ProcSyscallAbandoned && oldState == ProcRunning { // N.B. This is slow but it should be fairly rare. found := false for mid, ms := range o.mStates { @@ -234,7 +239,7 @@ func (o *ordering) advanceProcStart(ev *baseEvent, evt *evTable, m ThreadID, gen // had a status emitted, or because we already have a P and we're in a syscall, // and we haven't observed that it was stolen from us yet. state, ok := o.pStates[pid] - if !ok || state.status != go122.ProcIdle || !seq.succeeds(state.seq) || curCtx.P != NoProc { + if !ok || state.status != tracev2.ProcIdle || !seq.succeeds(state.seq) || curCtx.P != NoProc { // We can't make an inference as to whether this is bad. We could just be seeing // a ProcStart on a different M before the proc's state was emitted, or before we // got to the right point in the trace. @@ -245,11 +250,11 @@ func (o *ordering) advanceProcStart(ev *baseEvent, evt *evTable, m ThreadID, gen // We can advance this P. Check some invariants. // // We might have a goroutine if a goroutine is exiting a syscall. - reqs := event.SchedReqs{Thread: event.MustHave, Proc: event.MustNotHave, Goroutine: event.MayHave} + reqs := schedReqs{M: mustHave, P: mustNotHave, G: mayHave} if err := validateCtx(curCtx, reqs); err != nil { return curCtx, false, err } - state.status = go122.ProcRunning + state.status = tracev2.ProcRunning state.seq = seq newCtx := curCtx newCtx.P = pid @@ -269,16 +274,16 @@ func (o *ordering) advanceProcStop(ev *baseEvent, evt *evTable, m ThreadID, gen // ProcStop doesn't need a sequence number. state, ok := o.pStates[curCtx.P] if !ok { - return curCtx, false, fmt.Errorf("event %s for proc (%v) that doesn't exist", go122.EventString(ev.typ), curCtx.P) + return curCtx, false, fmt.Errorf("event %s for proc (%v) that doesn't exist", o.evName(ev.typ), curCtx.P) } - if state.status != go122.ProcRunning && state.status != go122.ProcSyscall { - return curCtx, false, fmt.Errorf("%s event for proc that's not %s or %s", go122.EventString(ev.typ), go122.ProcRunning, go122.ProcSyscall) + if state.status != tracev2.ProcRunning && state.status != tracev2.ProcSyscall { + return curCtx, false, fmt.Errorf("%s event for proc that's not %s or %s", o.evName(ev.typ), tracev2.ProcRunning, tracev2.ProcSyscall) } - reqs := event.SchedReqs{Thread: event.MustHave, Proc: event.MustHave, Goroutine: event.MayHave} + reqs := schedReqs{M: mustHave, P: mustHave, G: mayHave} if err := validateCtx(curCtx, reqs); err != nil { return curCtx, false, err } - state.status = go122.ProcIdle + state.status = tracev2.ProcIdle newCtx := curCtx newCtx.P = NoProc o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) @@ -289,14 +294,14 @@ func (o *ordering) advanceProcSteal(ev *baseEvent, evt *evTable, m ThreadID, gen pid := ProcID(ev.args[0]) seq := makeSeq(gen, ev.args[1]) state, ok := o.pStates[pid] - if !ok || (state.status != go122.ProcSyscall && state.status != go122.ProcSyscallAbandoned) || !seq.succeeds(state.seq) { + if !ok || (state.status != tracev2.ProcSyscall && state.status != tracev2.ProcSyscallAbandoned) || !seq.succeeds(state.seq) { // We can't make an inference as to whether this is bad. We could just be seeing // a ProcStart on a different M before the proc's state was emitted, or before we // got to the right point in the trace. return curCtx, false, nil } // We can advance this P. Check some invariants. - reqs := event.SchedReqs{Thread: event.MustHave, Proc: event.MayHave, Goroutine: event.MayHave} + reqs := schedReqs{M: mustHave, P: mayHave, G: mayHave} if err := validateCtx(curCtx, reqs); err != nil { return curCtx, false, err } @@ -310,12 +315,12 @@ func (o *ordering) advanceProcSteal(ev *baseEvent, evt *evTable, m ThreadID, gen ev.extra(version.Go122)[0] = uint64(oldStatus) // Update the P's status and sequence number. - state.status = go122.ProcIdle + state.status = tracev2.ProcIdle state.seq = seq // If we've lost information then don't try to do anything with the M. // It may have moved on and we can't be sure. - if oldStatus == go122.ProcSyscallAbandoned { + if oldStatus == tracev2.ProcSyscallAbandoned { o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) return curCtx, true, nil } @@ -359,12 +364,12 @@ func (o *ordering) advanceProcSteal(ev *baseEvent, evt *evTable, m ThreadID, gen func (o *ordering) advanceGoStatus(ev *baseEvent, evt *evTable, m ThreadID, gen uint64, curCtx schedCtx) (schedCtx, bool, error) { gid := GoID(ev.args[0]) mid := ThreadID(ev.args[1]) - status := go122.GoStatus(ev.args[2]) + status := tracev2.GoStatus(ev.args[2]) - if int(status) >= len(go122GoStatus2GoState) { + if int(status) >= len(tracev2GoStatus2GoState) { return curCtx, false, fmt.Errorf("invalid status for goroutine %d: %d", gid, status) } - oldState := go122GoStatus2GoState[status] + oldState := tracev2GoStatus2GoState[status] if s, ok := o.gStates[gid]; ok { if s.status != status { return curCtx, false, fmt.Errorf("inconsistent status for goroutine %d: old %v vs. new %v", gid, s.status, status) @@ -381,10 +386,10 @@ func (o *ordering) advanceGoStatus(ev *baseEvent, evt *evTable, m ThreadID, gen newCtx := curCtx switch status { - case go122.GoRunning: + case tracev2.GoRunning: // Bind the goroutine to the new context, since it's running. newCtx.G = gid - case go122.GoSyscall: + case tracev2.GoSyscall: if mid == NoThread { return curCtx, false, fmt.Errorf("found goroutine %d in syscall without a thread", gid) } @@ -436,22 +441,22 @@ func (o *ordering) advanceGoStatus(ev *baseEvent, evt *evTable, m ThreadID, gen func (o *ordering) advanceGoCreate(ev *baseEvent, evt *evTable, m ThreadID, gen uint64, curCtx schedCtx) (schedCtx, bool, error) { // Goroutines must be created on a running P, but may or may not be created // by a running goroutine. - reqs := event.SchedReqs{Thread: event.MustHave, Proc: event.MustHave, Goroutine: event.MayHave} + reqs := schedReqs{M: mustHave, P: mustHave, G: mayHave} if err := validateCtx(curCtx, reqs); err != nil { return curCtx, false, err } // If we have a goroutine, it must be running. - if state, ok := o.gStates[curCtx.G]; ok && state.status != go122.GoRunning { - return curCtx, false, fmt.Errorf("%s event for goroutine that's not %s", go122.EventString(ev.typ), GoRunning) + if state, ok := o.gStates[curCtx.G]; ok && state.status != tracev2.GoRunning { + return curCtx, false, fmt.Errorf("%s event for goroutine that's not %s", o.evName(ev.typ), GoRunning) } // This goroutine created another. Add a state for it. newgid := GoID(ev.args[0]) if _, ok := o.gStates[newgid]; ok { return curCtx, false, fmt.Errorf("tried to create goroutine (%v) that already exists", newgid) } - status := go122.GoRunnable - if ev.typ == go122.EvGoCreateBlocked { - status = go122.GoWaiting + status := tracev2.GoRunnable + if ev.typ == tracev2.EvGoCreateBlocked { + status = tracev2.GoWaiting } o.gStates[newgid] = &gState{id: newgid, status: status, seq: makeSeq(gen, 0)} o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) @@ -462,31 +467,31 @@ func (o *ordering) advanceGoStopExec(ev *baseEvent, evt *evTable, m ThreadID, ge // These are goroutine events that all require an active running // goroutine on some thread. They must *always* be advance-able, // since running goroutines are bound to their M. - if err := validateCtx(curCtx, event.UserGoReqs); err != nil { + if err := validateCtx(curCtx, userGoReqs); err != nil { return curCtx, false, err } state, ok := o.gStates[curCtx.G] if !ok { - return curCtx, false, fmt.Errorf("event %s for goroutine (%v) that doesn't exist", go122.EventString(ev.typ), curCtx.G) + return curCtx, false, fmt.Errorf("event %s for goroutine (%v) that doesn't exist", o.evName(ev.typ), curCtx.G) } - if state.status != go122.GoRunning { - return curCtx, false, fmt.Errorf("%s event for goroutine that's not %s", go122.EventString(ev.typ), GoRunning) + if state.status != tracev2.GoRunning { + return curCtx, false, fmt.Errorf("%s event for goroutine that's not %s", o.evName(ev.typ), GoRunning) } // Handle each case slightly differently; we just group them together // because they have shared preconditions. newCtx := curCtx switch ev.typ { - case go122.EvGoDestroy: + case tracev2.EvGoDestroy: // This goroutine is exiting itself. delete(o.gStates, curCtx.G) newCtx.G = NoGoroutine - case go122.EvGoStop: + case tracev2.EvGoStop: // Goroutine stopped (yielded). It's runnable but not running on this M. - state.status = go122.GoRunnable + state.status = tracev2.GoRunnable newCtx.G = NoGoroutine - case go122.EvGoBlock: + case tracev2.EvGoBlock: // Goroutine blocked. It's waiting now and not running on this M. - state.status = go122.GoWaiting + state.status = tracev2.GoWaiting newCtx.G = NoGoroutine } o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) @@ -497,18 +502,18 @@ func (o *ordering) advanceGoStart(ev *baseEvent, evt *evTable, m ThreadID, gen u gid := GoID(ev.args[0]) seq := makeSeq(gen, ev.args[1]) state, ok := o.gStates[gid] - if !ok || state.status != go122.GoRunnable || !seq.succeeds(state.seq) { + if !ok || state.status != tracev2.GoRunnable || !seq.succeeds(state.seq) { // We can't make an inference as to whether this is bad. We could just be seeing // a GoStart on a different M before the goroutine was created, before it had its // state emitted, or before we got to the right point in the trace yet. return curCtx, false, nil } // We can advance this goroutine. Check some invariants. - reqs := event.SchedReqs{Thread: event.MustHave, Proc: event.MustHave, Goroutine: event.MustNotHave} + reqs := schedReqs{M: mustHave, P: mustHave, G: mustNotHave} if err := validateCtx(curCtx, reqs); err != nil { return curCtx, false, err } - state.status = go122.GoRunning + state.status = tracev2.GoRunning state.seq = seq newCtx := curCtx newCtx.G = gid @@ -521,13 +526,13 @@ func (o *ordering) advanceGoUnblock(ev *baseEvent, evt *evTable, m ThreadID, gen gid := GoID(ev.args[0]) seq := makeSeq(gen, ev.args[1]) state, ok := o.gStates[gid] - if !ok || state.status != go122.GoWaiting || !seq.succeeds(state.seq) { + if !ok || state.status != tracev2.GoWaiting || !seq.succeeds(state.seq) { // We can't make an inference as to whether this is bad. We could just be seeing // a GoUnblock on a different M before the goroutine was created and blocked itself, // before it had its state emitted, or before we got to the right point in the trace yet. return curCtx, false, nil } - state.status = go122.GoRunnable + state.status = tracev2.GoRunnable state.seq = seq // N.B. No context to validate. Basically anything can unblock // a goroutine (e.g. sysmon). @@ -545,20 +550,20 @@ func (o *ordering) advanceGoSwitch(ev *baseEvent, evt *evTable, m ThreadID, gen // only advance it if the sequence numbers line up. // // The current goroutine on the thread must be actively running. - if err := validateCtx(curCtx, event.UserGoReqs); err != nil { + if err := validateCtx(curCtx, userGoReqs); err != nil { return curCtx, false, err } curGState, ok := o.gStates[curCtx.G] if !ok { - return curCtx, false, fmt.Errorf("event %s for goroutine (%v) that doesn't exist", go122.EventString(ev.typ), curCtx.G) + return curCtx, false, fmt.Errorf("event %s for goroutine (%v) that doesn't exist", o.evName(ev.typ), curCtx.G) } - if curGState.status != go122.GoRunning { - return curCtx, false, fmt.Errorf("%s event for goroutine that's not %s", go122.EventString(ev.typ), GoRunning) + if curGState.status != tracev2.GoRunning { + return curCtx, false, fmt.Errorf("%s event for goroutine that's not %s", o.evName(ev.typ), GoRunning) } nextg := GoID(ev.args[0]) seq := makeSeq(gen, ev.args[1]) // seq is for nextg, not curCtx.G. nextGState, ok := o.gStates[nextg] - if !ok || nextGState.status != go122.GoWaiting || !seq.succeeds(nextGState.seq) { + if !ok || nextGState.status != tracev2.GoWaiting || !seq.succeeds(nextGState.seq) { // We can't make an inference as to whether this is bad. We could just be seeing // a GoSwitch on a different M before the goroutine was created, before it had its // state emitted, or before we got to the right point in the trace yet. @@ -570,22 +575,22 @@ func (o *ordering) advanceGoSwitch(ev *baseEvent, evt *evTable, m ThreadID, gen // (GoSwitch and GoSwitchDestroy will be interpreted as GoUnblock events // for nextg). switch ev.typ { - case go122.EvGoSwitch: + case tracev2.EvGoSwitch: // Goroutine blocked. It's waiting now and not running on this M. - curGState.status = go122.GoWaiting + curGState.status = tracev2.GoWaiting // Emit a GoBlock event. // TODO(mknyszek): Emit a reason. - o.queue.push(makeEvent(evt, curCtx, go122.EvGoBlock, ev.time, 0 /* no reason */, 0 /* no stack */)) - case go122.EvGoSwitchDestroy: + o.queue.push(makeEvent(evt, curCtx, tracev2.EvGoBlock, ev.time, 0 /* no reason */, 0 /* no stack */)) + case tracev2.EvGoSwitchDestroy: // This goroutine is exiting itself. delete(o.gStates, curCtx.G) // Emit a GoDestroy event. - o.queue.push(makeEvent(evt, curCtx, go122.EvGoDestroy, ev.time)) + o.queue.push(makeEvent(evt, curCtx, tracev2.EvGoDestroy, ev.time)) } // Update the state of the next goroutine. - nextGState.status = go122.GoRunning + nextGState.status = tracev2.GoRunning nextGState.seq = seq newCtx := curCtx newCtx.G = nextg @@ -593,30 +598,30 @@ func (o *ordering) advanceGoSwitch(ev *baseEvent, evt *evTable, m ThreadID, gen // Queue an event for the next goroutine starting to run. startCtx := curCtx startCtx.G = NoGoroutine - o.queue.push(makeEvent(evt, startCtx, go122.EvGoStart, ev.time, uint64(nextg), ev.args[1])) + o.queue.push(makeEvent(evt, startCtx, tracev2.EvGoStart, ev.time, uint64(nextg), ev.args[1])) return newCtx, true, nil } func (o *ordering) advanceGoSyscallBegin(ev *baseEvent, evt *evTable, m ThreadID, gen uint64, curCtx schedCtx) (schedCtx, bool, error) { // Entering a syscall requires an active running goroutine with a // proc on some thread. It is always advancable. - if err := validateCtx(curCtx, event.UserGoReqs); err != nil { + if err := validateCtx(curCtx, userGoReqs); err != nil { return curCtx, false, err } state, ok := o.gStates[curCtx.G] if !ok { - return curCtx, false, fmt.Errorf("event %s for goroutine (%v) that doesn't exist", go122.EventString(ev.typ), curCtx.G) + return curCtx, false, fmt.Errorf("event %s for goroutine (%v) that doesn't exist", o.evName(ev.typ), curCtx.G) } - if state.status != go122.GoRunning { - return curCtx, false, fmt.Errorf("%s event for goroutine that's not %s", go122.EventString(ev.typ), GoRunning) + if state.status != tracev2.GoRunning { + return curCtx, false, fmt.Errorf("%s event for goroutine that's not %s", o.evName(ev.typ), GoRunning) } // Goroutine entered a syscall. It's still running on this P and M. - state.status = go122.GoSyscall + state.status = tracev2.GoSyscall pState, ok := o.pStates[curCtx.P] if !ok { - return curCtx, false, fmt.Errorf("uninitialized proc %d found during %s", curCtx.P, go122.EventString(ev.typ)) + return curCtx, false, fmt.Errorf("uninitialized proc %d found during %s", curCtx.P, o.evName(ev.typ)) } - pState.status = go122.ProcSyscall + pState.status = tracev2.ProcSyscall // Validate the P sequence number on the event and advance it. // // We have a P sequence number for what is supposed to be a goroutine event @@ -630,7 +635,7 @@ func (o *ordering) advanceGoSyscallBegin(ev *baseEvent, evt *evTable, m ThreadID // to back off and see if any other events will advance. This is a running P. pSeq := makeSeq(gen, ev.args[0]) if !pSeq.succeeds(pState.seq) { - return curCtx, false, fmt.Errorf("failed to advance %s: can't make sequence: %s -> %s", go122.EventString(ev.typ), pState.seq, pSeq) + return curCtx, false, fmt.Errorf("failed to advance %s: can't make sequence: %s -> %s", o.evName(ev.typ), pState.seq, pSeq) } pState.seq = pSeq o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) @@ -641,27 +646,27 @@ func (o *ordering) advanceGoSyscallEnd(ev *baseEvent, evt *evTable, m ThreadID, // This event is always advance-able because it happens on the same // thread that EvGoSyscallStart happened, and the goroutine can't leave // that thread until its done. - if err := validateCtx(curCtx, event.UserGoReqs); err != nil { + if err := validateCtx(curCtx, userGoReqs); err != nil { return curCtx, false, err } state, ok := o.gStates[curCtx.G] if !ok { - return curCtx, false, fmt.Errorf("event %s for goroutine (%v) that doesn't exist", go122.EventString(ev.typ), curCtx.G) + return curCtx, false, fmt.Errorf("event %s for goroutine (%v) that doesn't exist", o.evName(ev.typ), curCtx.G) } - if state.status != go122.GoSyscall { - return curCtx, false, fmt.Errorf("%s event for goroutine that's not %s", go122.EventString(ev.typ), GoRunning) + if state.status != tracev2.GoSyscall { + return curCtx, false, fmt.Errorf("%s event for goroutine that's not %s", o.evName(ev.typ), GoRunning) } - state.status = go122.GoRunning + state.status = tracev2.GoRunning // Transfer the P back to running from syscall. pState, ok := o.pStates[curCtx.P] if !ok { - return curCtx, false, fmt.Errorf("uninitialized proc %d found during %s", curCtx.P, go122.EventString(ev.typ)) + return curCtx, false, fmt.Errorf("uninitialized proc %d found during %s", curCtx.P, o.evName(ev.typ)) } - if pState.status != go122.ProcSyscall { - return curCtx, false, fmt.Errorf("expected proc %d in state %v, but got %v instead", curCtx.P, go122.ProcSyscall, pState.status) + if pState.status != tracev2.ProcSyscall { + return curCtx, false, fmt.Errorf("expected proc %d in state %v, but got %v instead", curCtx.P, tracev2.ProcSyscall, pState.status) } - pState.status = go122.ProcRunning + pState.status = tracev2.ProcRunning o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) return curCtx, true, nil } @@ -680,27 +685,27 @@ func (o *ordering) advanceGoSyscallEndBlocked(ev *baseEvent, evt *evTable, m Thr if curCtx.P != NoProc { pState, ok := o.pStates[curCtx.P] if !ok { - return curCtx, false, fmt.Errorf("uninitialized proc %d found during %s", curCtx.P, go122.EventString(ev.typ)) + return curCtx, false, fmt.Errorf("uninitialized proc %d found during %s", curCtx.P, o.evName(ev.typ)) } - if pState.status == go122.ProcSyscall { + if pState.status == tracev2.ProcSyscall { return curCtx, false, nil } } // As mentioned above, we may have a P here if we ProcStart // before this event. - if err := validateCtx(curCtx, event.SchedReqs{Thread: event.MustHave, Proc: event.MayHave, Goroutine: event.MustHave}); err != nil { + if err := validateCtx(curCtx, schedReqs{M: mustHave, P: mayHave, G: mustHave}); err != nil { return curCtx, false, err } state, ok := o.gStates[curCtx.G] if !ok { - return curCtx, false, fmt.Errorf("event %s for goroutine (%v) that doesn't exist", go122.EventString(ev.typ), curCtx.G) + return curCtx, false, fmt.Errorf("event %s for goroutine (%v) that doesn't exist", o.evName(ev.typ), curCtx.G) } - if state.status != go122.GoSyscall { - return curCtx, false, fmt.Errorf("%s event for goroutine that's not %s", go122.EventString(ev.typ), GoRunning) + if state.status != tracev2.GoSyscall { + return curCtx, false, fmt.Errorf("%s event for goroutine that's not %s", o.evName(ev.typ), GoRunning) } newCtx := curCtx newCtx.G = NoGoroutine - state.status = go122.GoRunnable + state.status = tracev2.GoRunnable o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) return newCtx, true, nil } @@ -709,7 +714,7 @@ func (o *ordering) advanceGoCreateSyscall(ev *baseEvent, evt *evTable, m ThreadI // This event indicates that a goroutine is effectively // being created out of a cgo callback. Such a goroutine // is 'created' in the syscall state. - if err := validateCtx(curCtx, event.SchedReqs{Thread: event.MustHave, Proc: event.MayHave, Goroutine: event.MustNotHave}); err != nil { + if err := validateCtx(curCtx, schedReqs{M: mustHave, P: mayHave, G: mustNotHave}); err != nil { return curCtx, false, err } // This goroutine is effectively being created. Add a state for it. @@ -717,7 +722,7 @@ func (o *ordering) advanceGoCreateSyscall(ev *baseEvent, evt *evTable, m ThreadI if _, ok := o.gStates[newgid]; ok { return curCtx, false, fmt.Errorf("tried to create goroutine (%v) in syscall that already exists", newgid) } - o.gStates[newgid] = &gState{id: newgid, status: go122.GoSyscall, seq: makeSeq(gen, 0)} + o.gStates[newgid] = &gState{id: newgid, status: tracev2.GoSyscall, seq: makeSeq(gen, 0)} // Goroutine is executing. Bind it to the context. newCtx := curCtx newCtx.G = newgid @@ -742,16 +747,16 @@ func (o *ordering) advanceGoDestroySyscall(ev *baseEvent, evt *evTable, m Thread // Note: we might have a P here. The P might not be released // eagerly by the runtime, and it might get stolen back later // (or never again, if the program is going to exit). - if err := validateCtx(curCtx, event.SchedReqs{Thread: event.MustHave, Proc: event.MayHave, Goroutine: event.MustHave}); err != nil { + if err := validateCtx(curCtx, schedReqs{M: mustHave, P: mayHave, G: mustHave}); err != nil { return curCtx, false, err } // Check to make sure the goroutine exists in the right state. state, ok := o.gStates[curCtx.G] if !ok { - return curCtx, false, fmt.Errorf("event %s for goroutine (%v) that doesn't exist", go122.EventString(ev.typ), curCtx.G) + return curCtx, false, fmt.Errorf("event %s for goroutine (%v) that doesn't exist", o.evName(ev.typ), curCtx.G) } - if state.status != go122.GoSyscall { - return curCtx, false, fmt.Errorf("%s event for goroutine that's not %v", go122.EventString(ev.typ), GoSyscall) + if state.status != tracev2.GoSyscall { + return curCtx, false, fmt.Errorf("%s event for goroutine that's not %v", o.evName(ev.typ), GoSyscall) } // This goroutine is exiting itself. delete(o.gStates, curCtx.G) @@ -762,18 +767,18 @@ func (o *ordering) advanceGoDestroySyscall(ev *baseEvent, evt *evTable, m Thread if curCtx.P != NoProc { pState, ok := o.pStates[curCtx.P] if !ok { - return curCtx, false, fmt.Errorf("found invalid proc %d during %s", curCtx.P, go122.EventString(ev.typ)) + return curCtx, false, fmt.Errorf("found invalid proc %d during %s", curCtx.P, o.evName(ev.typ)) } - if pState.status != go122.ProcSyscall { - return curCtx, false, fmt.Errorf("proc %d in unexpected state %s during %s", curCtx.P, pState.status, go122.EventString(ev.typ)) + if pState.status != tracev2.ProcSyscall { + return curCtx, false, fmt.Errorf("proc %d in unexpected state %s during %s", curCtx.P, pState.status, o.evName(ev.typ)) } // See the go122-create-syscall-reuse-thread-id test case for more details. - pState.status = go122.ProcSyscallAbandoned + pState.status = tracev2.ProcSyscallAbandoned newCtx.P = NoProc // Queue an extra self-ProcSteal event. - extra := makeEvent(evt, curCtx, go122.EvProcSteal, ev.time, uint64(curCtx.P)) - extra.base.extra(version.Go122)[0] = uint64(go122.ProcSyscall) + extra := makeEvent(evt, curCtx, tracev2.EvProcSteal, ev.time, uint64(curCtx.P)) + extra.base.extra(version.Go122)[0] = uint64(tracev2.ProcSyscall) o.queue.push(extra) } o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) @@ -811,7 +816,7 @@ func (o *ordering) advanceUserTaskBegin(ev *baseEvent, evt *evTable, m ThreadID, return curCtx, false, fmt.Errorf("invalid string ID %v for %v event", nameID, ev.typ) } o.activeTasks[id] = taskState{name: name, parentID: parentID} - if err := validateCtx(curCtx, event.UserGoReqs); err != nil { + if err := validateCtx(curCtx, userGoReqs); err != nil { return curCtx, false, err } o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) @@ -832,7 +837,7 @@ func (o *ordering) advanceUserTaskEnd(ev *baseEvent, evt *evTable, m ThreadID, g ev.extra(version.Go122)[0] = uint64(NoTask) ev.extra(version.Go122)[1] = uint64(evt.addExtraString("")) } - if err := validateCtx(curCtx, event.UserGoReqs); err != nil { + if err := validateCtx(curCtx, userGoReqs); err != nil { return curCtx, false, err } o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) @@ -840,7 +845,7 @@ func (o *ordering) advanceUserTaskEnd(ev *baseEvent, evt *evTable, m ThreadID, g } func (o *ordering) advanceUserRegionBegin(ev *baseEvent, evt *evTable, m ThreadID, gen uint64, curCtx schedCtx) (schedCtx, bool, error) { - if err := validateCtx(curCtx, event.UserGoReqs); err != nil { + if err := validateCtx(curCtx, userGoReqs); err != nil { return curCtx, false, err } tid := TaskID(ev.args[0]) @@ -861,7 +866,7 @@ func (o *ordering) advanceUserRegionBegin(ev *baseEvent, evt *evTable, m ThreadI } func (o *ordering) advanceUserRegionEnd(ev *baseEvent, evt *evTable, m ThreadID, gen uint64, curCtx schedCtx) (schedCtx, bool, error) { - if err := validateCtx(curCtx, event.UserGoReqs); err != nil { + if err := validateCtx(curCtx, userGoReqs); err != nil { return curCtx, false, err } tid := TaskID(ev.args[0]) @@ -907,7 +912,7 @@ func (o *ordering) advanceGCActive(ev *baseEvent, evt *evTable, m ThreadID, gen return curCtx, false, fmt.Errorf("encountered GCActive while GC was not in progress") } o.gcSeq = seq - if err := validateCtx(curCtx, event.UserGoReqs); err != nil { + if err := validateCtx(curCtx, userGoReqs); err != nil { return curCtx, false, err } o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) @@ -931,7 +936,7 @@ func (o *ordering) advanceGCBegin(ev *baseEvent, evt *evTable, m ThreadID, gen u } o.gcSeq = seq o.gcState = gcRunning - if err := validateCtx(curCtx, event.UserGoReqs); err != nil { + if err := validateCtx(curCtx, userGoReqs); err != nil { return curCtx, false, err } o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) @@ -952,7 +957,7 @@ func (o *ordering) advanceGCEnd(ev *baseEvent, evt *evTable, m ThreadID, gen uin } o.gcSeq = seq o.gcState = gcNotRunning - if err := validateCtx(curCtx, event.UserGoReqs); err != nil { + if err := validateCtx(curCtx, userGoReqs); err != nil { return curCtx, false, err } o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) @@ -961,7 +966,7 @@ func (o *ordering) advanceGCEnd(ev *baseEvent, evt *evTable, m ThreadID, gen uin func (o *ordering) advanceAnnotation(ev *baseEvent, evt *evTable, m ThreadID, gen uint64, curCtx schedCtx) (schedCtx, bool, error) { // Handle simple instantaneous events that require a G. - if err := validateCtx(curCtx, event.UserGoReqs); err != nil { + if err := validateCtx(curCtx, userGoReqs); err != nil { return curCtx, false, err } o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) @@ -970,7 +975,7 @@ func (o *ordering) advanceAnnotation(ev *baseEvent, evt *evTable, m ThreadID, ge func (o *ordering) advanceHeapMetric(ev *baseEvent, evt *evTable, m ThreadID, gen uint64, curCtx schedCtx) (schedCtx, bool, error) { // Handle allocation metrics, which don't require a G. - if err := validateCtx(curCtx, event.SchedReqs{Thread: event.MustHave, Proc: event.MustHave, Goroutine: event.MayHave}); err != nil { + if err := validateCtx(curCtx, schedReqs{M: mustHave, P: mustHave, G: mayHave}); err != nil { return curCtx, false, err } o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) @@ -979,7 +984,7 @@ func (o *ordering) advanceHeapMetric(ev *baseEvent, evt *evTable, m ThreadID, ge func (o *ordering) advanceGCSweepBegin(ev *baseEvent, evt *evTable, m ThreadID, gen uint64, curCtx schedCtx) (schedCtx, bool, error) { // Handle sweep, which is bound to a P and doesn't require a G. - if err := validateCtx(curCtx, event.SchedReqs{Thread: event.MustHave, Proc: event.MustHave, Goroutine: event.MayHave}); err != nil { + if err := validateCtx(curCtx, schedReqs{M: mustHave, P: mustHave, G: mayHave}); err != nil { return curCtx, false, err } if err := o.pStates[curCtx.P].beginRange(makeRangeType(ev.typ, 0)); err != nil { @@ -1007,7 +1012,7 @@ func (o *ordering) advanceGCSweepActive(ev *baseEvent, evt *evTable, m ThreadID, } func (o *ordering) advanceGCSweepEnd(ev *baseEvent, evt *evTable, m ThreadID, gen uint64, curCtx schedCtx) (schedCtx, bool, error) { - if err := validateCtx(curCtx, event.SchedReqs{Thread: event.MustHave, Proc: event.MustHave, Goroutine: event.MayHave}); err != nil { + if err := validateCtx(curCtx, schedReqs{M: mustHave, P: mustHave, G: mayHave}); err != nil { return curCtx, false, err } _, err := o.pStates[curCtx.P].endRange(ev.typ) @@ -1020,11 +1025,11 @@ func (o *ordering) advanceGCSweepEnd(ev *baseEvent, evt *evTable, m ThreadID, ge func (o *ordering) advanceGoRangeBegin(ev *baseEvent, evt *evTable, m ThreadID, gen uint64, curCtx schedCtx) (schedCtx, bool, error) { // Handle special goroutine-bound event ranges. - if err := validateCtx(curCtx, event.UserGoReqs); err != nil { + if err := validateCtx(curCtx, userGoReqs); err != nil { return curCtx, false, err } desc := stringID(0) - if ev.typ == go122.EvSTWBegin { + if ev.typ == tracev2.EvSTWBegin { desc = stringID(ev.args[0]) } gState, ok := o.gStates[curCtx.G] @@ -1045,7 +1050,7 @@ func (o *ordering) advanceGoRangeActive(ev *baseEvent, evt *evTable, m ThreadID, // current scheduler context. gState, ok := o.gStates[gid] if !ok { - return curCtx, false, fmt.Errorf("uninitialized goroutine %d found during %s", gid, go122.EventString(ev.typ)) + return curCtx, false, fmt.Errorf("uninitialized goroutine %d found during %s", gid, o.evName(ev.typ)) } if err := gState.activeRange(makeRangeType(ev.typ, 0), gen == o.initialGen); err != nil { return curCtx, false, err @@ -1055,7 +1060,7 @@ func (o *ordering) advanceGoRangeActive(ev *baseEvent, evt *evTable, m ThreadID, } func (o *ordering) advanceGoRangeEnd(ev *baseEvent, evt *evTable, m ThreadID, gen uint64, curCtx schedCtx) (schedCtx, bool, error) { - if err := validateCtx(curCtx, event.UserGoReqs); err != nil { + if err := validateCtx(curCtx, userGoReqs); err != nil { return curCtx, false, err } gState, ok := o.gStates[curCtx.G] @@ -1066,7 +1071,7 @@ func (o *ordering) advanceGoRangeEnd(ev *baseEvent, evt *evTable, m ThreadID, ge if err != nil { return curCtx, false, err } - if ev.typ == go122.EvSTWEnd { + if ev.typ == tracev2.EvSTWEnd { // Smuggle the kind into the event. // Don't use ev.extra here so we have symmetry with STWBegin. ev.args[0] = uint64(desc) @@ -1077,7 +1082,7 @@ func (o *ordering) advanceGoRangeEnd(ev *baseEvent, evt *evTable, m ThreadID, ge func (o *ordering) advanceAllocFree(ev *baseEvent, evt *evTable, m ThreadID, gen uint64, curCtx schedCtx) (schedCtx, bool, error) { // Handle simple instantaneous events that may or may not have a P. - if err := validateCtx(curCtx, event.SchedReqs{Thread: event.MustHave, Proc: event.MayHave, Goroutine: event.MayHave}); err != nil { + if err := validateCtx(curCtx, schedReqs{M: mustHave, P: mayHave, G: mayHave}); err != nil { return curCtx, false, err } o.queue.push(Event{table: evt, ctx: curCtx, base: *ev}) @@ -1098,25 +1103,25 @@ type schedCtx struct { // validateCtx ensures that ctx conforms to some reqs, returning an error if // it doesn't. -func validateCtx(ctx schedCtx, reqs event.SchedReqs) error { +func validateCtx(ctx schedCtx, reqs schedReqs) error { // Check thread requirements. - if reqs.Thread == event.MustHave && ctx.M == NoThread { + if reqs.M == mustHave && ctx.M == NoThread { return fmt.Errorf("expected a thread but didn't have one") - } else if reqs.Thread == event.MustNotHave && ctx.M != NoThread { + } else if reqs.M == mustNotHave && ctx.M != NoThread { return fmt.Errorf("expected no thread but had one") } // Check proc requirements. - if reqs.Proc == event.MustHave && ctx.P == NoProc { + if reqs.P == mustHave && ctx.P == NoProc { return fmt.Errorf("expected a proc but didn't have one") - } else if reqs.Proc == event.MustNotHave && ctx.P != NoProc { + } else if reqs.P == mustNotHave && ctx.P != NoProc { return fmt.Errorf("expected no proc but had one") } // Check goroutine requirements. - if reqs.Goroutine == event.MustHave && ctx.G == NoGoroutine { + if reqs.G == mustHave && ctx.G == NoGoroutine { return fmt.Errorf("expected a goroutine but didn't have one") - } else if reqs.Goroutine == event.MustNotHave && ctx.G != NoGoroutine { + } else if reqs.G == mustNotHave && ctx.G != NoGoroutine { return fmt.Errorf("expected no goroutine but had one") } return nil @@ -1161,13 +1166,13 @@ type userRegion struct { // they may have an optional subtype that describes the range // in more detail. type rangeType struct { - typ event.Type // "Begin" event. - desc stringID // Optional subtype. + typ tracev2.EventType // "Begin" event. + desc stringID // Optional subtype. } // makeRangeType constructs a new rangeType. -func makeRangeType(typ event.Type, desc stringID) rangeType { - if styp := go122.Specs()[typ].StartEv; styp != go122.EvNone { +func makeRangeType(typ tracev2.EventType, desc stringID) rangeType { + if styp := tracev2.Specs()[typ].StartEv; styp != tracev2.EvNone { typ = styp } return rangeType{typ, desc} @@ -1176,7 +1181,7 @@ func makeRangeType(typ event.Type, desc stringID) rangeType { // gState is the state of a goroutine at a point in the trace. type gState struct { id GoID - status go122.GoStatus + status tracev2.GoStatus seq seqCounter // regions are the active user regions for this goroutine. @@ -1208,7 +1213,7 @@ func (s *gState) endRegion(r userRegion) error { // pState is the state of a proc at a point in the trace. type pState struct { id ProcID - status go122.ProcStatus + status tracev2.ProcStatus seq seqCounter // rangeState is the state of special time ranges bound to this proc. @@ -1232,7 +1237,7 @@ type rangeState struct { // Returns an error if the range is already in progress. func (s *rangeState) beginRange(typ rangeType) error { if s.hasRange(typ) { - return fmt.Errorf("discovered event already in-flight for when starting event %v", go122.Specs()[typ.typ].Name) + return fmt.Errorf("discovered event already in-flight for when starting event %v", tracev2.Specs()[typ.typ].Name) } s.inFlight = append(s.inFlight, typ) return nil @@ -1247,26 +1252,21 @@ func (s *rangeState) activeRange(typ rangeType, isInitialGen bool) error { } s.inFlight = append(s.inFlight, typ) } else if !s.hasRange(typ) { - return fmt.Errorf("resource is missing active range: %v %v", go122.Specs()[typ.typ].Name, s.inFlight) + return fmt.Errorf("resource is missing active range: %v %v", tracev2.Specs()[typ.typ].Name, s.inFlight) } return nil } // hasRange returns true if a special time range on the goroutine as in progress. func (s *rangeState) hasRange(typ rangeType) bool { - for _, ftyp := range s.inFlight { - if ftyp == typ { - return true - } - } - return false + return slices.Contains(s.inFlight, typ) } // endRange ends a special range in time on the goroutine. // // This must line up with the start event type of the range the goroutine is currently in. -func (s *rangeState) endRange(typ event.Type) (stringID, error) { - st := go122.Specs()[typ].StartEv +func (s *rangeState) endRange(typ tracev2.EventType) (stringID, error) { + st := tracev2.Specs()[typ].StartEv idx := -1 for i, r := range s.inFlight { if r.typ == st { @@ -1275,7 +1275,7 @@ func (s *rangeState) endRange(typ event.Type) (stringID, error) { } } if idx < 0 { - return 0, fmt.Errorf("tried to end event %v, but not in-flight", go122.Specs()[st].Name) + return 0, fmt.Errorf("tried to end event %v, but not in-flight", tracev2.Specs()[st].Name) } // Swap remove. desc := s.inFlight[idx].desc @@ -1385,7 +1385,7 @@ func (q *queue[T]) pop() (T, bool) { // It's just a convenience function; it's always OK to construct // an Event manually if this isn't quite the right way to express // the contents of the event. -func makeEvent(table *evTable, ctx schedCtx, typ event.Type, time Time, args ...uint64) Event { +func makeEvent(table *evTable, ctx schedCtx, typ tracev2.EventType, time Time, args ...uint64) Event { ev := Event{ table: table, ctx: ctx, @@ -1397,3 +1397,24 @@ func makeEvent(table *evTable, ctx schedCtx, typ event.Type, time Time, args ... copy(ev.base.args[:], args) return ev } + +// schedReqs is a set of constraints on what the scheduling +// context must look like. +type schedReqs struct { + M constraint + P constraint + G constraint +} + +// constraint represents a various presence requirements. +type constraint uint8 + +const ( + mustNotHave constraint = iota + mayHave + mustHave +) + +// userGoReqs is a common requirement among events that are running +// or are close to running user code. +var userGoReqs = schedReqs{M: mustHave, P: mustHave, G: mustHave} diff --git a/src/internal/trace/parser.go b/src/internal/trace/parser.go deleted file mode 100644 index d6fff84d55b63e..00000000000000 --- a/src/internal/trace/parser.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package trace - -// Frame is a frame in stack traces. -type Frame struct { - PC uint64 - Fn string - File string - Line int -} - -const ( - // Special P identifiers: - FakeP = 1000000 + iota - TimerP // depicts timer unblocks - NetpollP // depicts network unblocks - SyscallP // depicts returns from syscalls - GCP // depicts GC state - ProfileP // depicts recording of CPU profile samples -) - -// Event types in the trace. -// Verbatim copy from src/runtime/trace.go with the "trace" prefix removed. -const ( - EvNone = 0 // unused - EvBatch = 1 // start of per-P batch of events [pid, timestamp] - EvFrequency = 2 // contains tracer timer frequency [frequency (ticks per second)] - EvStack = 3 // stack [stack id, number of PCs, array of {PC, func string ID, file string ID, line}] - EvGomaxprocs = 4 // current value of GOMAXPROCS [timestamp, GOMAXPROCS, stack id] - EvProcStart = 5 // start of P [timestamp, thread id] - EvProcStop = 6 // stop of P [timestamp] - EvGCStart = 7 // GC start [timestamp, seq, stack id] - EvGCDone = 8 // GC done [timestamp] - EvSTWStart = 9 // GC mark termination start [timestamp, kind] - EvSTWDone = 10 // GC mark termination done [timestamp] - EvGCSweepStart = 11 // GC sweep start [timestamp, stack id] - EvGCSweepDone = 12 // GC sweep done [timestamp, swept, reclaimed] - EvGoCreate = 13 // goroutine creation [timestamp, new goroutine id, new stack id, stack id] - EvGoStart = 14 // goroutine starts running [timestamp, goroutine id, seq] - EvGoEnd = 15 // goroutine ends [timestamp] - EvGoStop = 16 // goroutine stops (like in select{}) [timestamp, stack] - EvGoSched = 17 // goroutine calls Gosched [timestamp, stack] - EvGoPreempt = 18 // goroutine is preempted [timestamp, stack] - EvGoSleep = 19 // goroutine calls Sleep [timestamp, stack] - EvGoBlock = 20 // goroutine blocks [timestamp, stack] - EvGoUnblock = 21 // goroutine is unblocked [timestamp, goroutine id, seq, stack] - EvGoBlockSend = 22 // goroutine blocks on chan send [timestamp, stack] - EvGoBlockRecv = 23 // goroutine blocks on chan recv [timestamp, stack] - EvGoBlockSelect = 24 // goroutine blocks on select [timestamp, stack] - EvGoBlockSync = 25 // goroutine blocks on Mutex/RWMutex [timestamp, stack] - EvGoBlockCond = 26 // goroutine blocks on Cond [timestamp, stack] - EvGoBlockNet = 27 // goroutine blocks on network [timestamp, stack] - EvGoSysCall = 28 // syscall enter [timestamp, stack] - EvGoSysExit = 29 // syscall exit [timestamp, goroutine id, seq, real timestamp] - EvGoSysBlock = 30 // syscall blocks [timestamp] - EvGoWaiting = 31 // denotes that goroutine is blocked when tracing starts [timestamp, goroutine id] - EvGoInSyscall = 32 // denotes that goroutine is in syscall when tracing starts [timestamp, goroutine id] - EvHeapAlloc = 33 // gcController.heapLive change [timestamp, heap live bytes] - EvHeapGoal = 34 // gcController.heapGoal change [timestamp, heap goal bytes] - EvTimerGoroutine = 35 // denotes timer goroutine [timer goroutine id] - EvFutileWakeup = 36 // denotes that the previous wakeup of this goroutine was futile [timestamp] - EvString = 37 // string dictionary entry [ID, length, string] - EvGoStartLocal = 38 // goroutine starts running on the same P as the last event [timestamp, goroutine id] - EvGoUnblockLocal = 39 // goroutine is unblocked on the same P as the last event [timestamp, goroutine id, stack] - EvGoSysExitLocal = 40 // syscall exit on the same P as the last event [timestamp, goroutine id, real timestamp] - EvGoStartLabel = 41 // goroutine starts running with label [timestamp, goroutine id, seq, label string id] - EvGoBlockGC = 42 // goroutine blocks on GC assist [timestamp, stack] - EvGCMarkAssistStart = 43 // GC mark assist start [timestamp, stack] - EvGCMarkAssistDone = 44 // GC mark assist done [timestamp] - EvUserTaskCreate = 45 // trace.NewTask [timestamp, internal task id, internal parent id, name string, stack] - EvUserTaskEnd = 46 // end of task [timestamp, internal task id, stack] - EvUserRegion = 47 // trace.WithRegion [timestamp, internal task id, mode(0:start, 1:end), name string, stack] - EvUserLog = 48 // trace.Log [timestamp, internal id, key string id, stack, value string] - EvCPUSample = 49 // CPU profiling sample [timestamp, real timestamp, real P id (-1 when absent), goroutine id, stack] - EvCount = 50 -) diff --git a/src/internal/trace/raw/event.go b/src/internal/trace/raw/event.go index e163a2c6ef820b..9042d3f2151eb4 100644 --- a/src/internal/trace/raw/event.go +++ b/src/internal/trace/raw/event.go @@ -9,7 +9,7 @@ import ( "strconv" "strings" - "internal/trace/event" + "internal/trace/tracev2" "internal/trace/version" ) @@ -20,7 +20,7 @@ import ( // trace format's framing. (But not interpreted.) type Event struct { Version version.Version - Ev event.Type + Ev tracev2.EventType Args []uint64 Data []byte } diff --git a/src/internal/trace/raw/reader.go b/src/internal/trace/raw/reader.go index 37f36a1a24b712..af5dfac0e77a72 100644 --- a/src/internal/trace/raw/reader.go +++ b/src/internal/trace/raw/reader.go @@ -10,7 +10,7 @@ import ( "fmt" "io" - "internal/trace/event" + "internal/trace/tracev2" "internal/trace/version" ) @@ -19,7 +19,7 @@ import ( type Reader struct { r *bufio.Reader v version.Version - specs []event.Spec + specs []tracev2.EventSpec } // NewReader creates a new reader for the trace wire format. @@ -49,7 +49,7 @@ func (r *Reader) ReadEvent() (Event, error) { if int(evb) >= len(r.specs) || evb == 0 { return Event{}, fmt.Errorf("invalid event type: %d", evb) } - ev := event.Type(evb) + ev := tracev2.EventType(evb) spec := r.specs[ev] args, err := r.readArgs(len(spec.Args)) if err != nil { diff --git a/src/internal/trace/raw/textreader.go b/src/internal/trace/raw/textreader.go index 37bfcfefdf62f7..f1544a7052c56f 100644 --- a/src/internal/trace/raw/textreader.go +++ b/src/internal/trace/raw/textreader.go @@ -12,7 +12,7 @@ import ( "strings" "unicode" - "internal/trace/event" + "internal/trace/tracev2" "internal/trace/version" ) @@ -20,8 +20,8 @@ import ( // into an event stream. type TextReader struct { v version.Version - specs []event.Spec - names map[string]event.Type + specs []tracev2.EventSpec + names map[string]tracev2.EventType s *bufio.Scanner } @@ -50,7 +50,7 @@ func NewTextReader(r io.Reader) (*TextReader, error) { } tr.v = v tr.specs = v.Specs() - tr.names = event.Names(tr.specs) + tr.names = tracev2.EventNames(tr.specs) for _, r := range line { if !unicode.IsSpace(r) { return nil, fmt.Errorf("encountered unexpected non-space at the end of the header: %q", line) @@ -165,14 +165,13 @@ func readArg(s string) (arg string, value uint64, rest string, err error) { if len(tok) == 0 { return "", 0, s, fmt.Errorf("no argument") } - parts := strings.SplitN(tok, "=", 2) - if len(parts) < 2 { + arg, val, found := strings.Cut(tok, "=") + if !found { return "", 0, s, fmt.Errorf("malformed argument: %q", tok) } - arg = parts[0] - value, err = strconv.ParseUint(parts[1], 10, 64) + value, err = strconv.ParseUint(val, 10, 64) if err != nil { - return arg, value, s, fmt.Errorf("failed to parse argument value %q for arg %q", parts[1], parts[0]) + return arg, value, s, fmt.Errorf("failed to parse argument value %q for arg %q", val, arg) } return } @@ -205,11 +204,11 @@ func readToken(s string) (token, rest string) { } func readData(line string) ([]byte, error) { - parts := strings.SplitN(line, "=", 2) - if len(parts) < 2 || strings.TrimSpace(parts[0]) != "data" { + dk, dv, found := strings.Cut(line, "=") + if !found || strings.TrimSpace(dk) != "data" { return nil, fmt.Errorf("malformed data: %q", line) } - data, err := strconv.Unquote(strings.TrimSpace(parts[1])) + data, err := strconv.Unquote(strings.TrimSpace(dv)) if err != nil { return nil, fmt.Errorf("failed to parse data: %q: %v", line, err) } diff --git a/src/internal/trace/raw/writer.go b/src/internal/trace/raw/writer.go index 9b87995aa77738..6b7042cf2af101 100644 --- a/src/internal/trace/raw/writer.go +++ b/src/internal/trace/raw/writer.go @@ -9,7 +9,7 @@ import ( "fmt" "io" - "internal/trace/event" + "internal/trace/tracev2" "internal/trace/version" ) @@ -23,7 +23,7 @@ type Writer struct { w io.Writer buf []byte v version.Version - specs []event.Spec + specs []tracev2.EventSpec } // NewWriter creates a new byte format writer. diff --git a/src/internal/trace/reader.go b/src/internal/trace/reader.go index 81157292fb3f08..bb9cc280f5c981 100644 --- a/src/internal/trace/reader.go +++ b/src/internal/trace/reader.go @@ -6,29 +6,48 @@ package trace import ( "bufio" + "errors" "fmt" "io" "slices" "strings" - "internal/trace/event/go122" - "internal/trace/internal/oldtrace" + "internal/trace/internal/tracev1" + "internal/trace/tracev2" "internal/trace/version" ) // Reader reads a byte stream, validates it, and produces trace events. +// +// Provided the trace is non-empty the Reader always produces a Sync +// event as the first event, and a Sync event as the last event. +// (There may also be any number of Sync events in the middle, too.) type Reader struct { - r *bufio.Reader - lastTs Time - gen *generation - spill *spilledBatch - spillErr error // error from reading spill - frontier []*batchCursor - cpuSamples []cpuSample - order ordering - emittedSync bool - - go121Events *oldTraceConverter + version version.Version + r *bufio.Reader + lastTs Time + gen *generation + frontier []*batchCursor + cpuSamples []cpuSample + order ordering + syncs int + readGenErr error + done bool + + // Spill state. + // + // Traces before Go 1.26 had no explicit end-of-generation signal, and + // so the first batch of the next generation needed to be parsed to identify + // a new generation. This batch is the "spilled" so we don't lose track + // of it when parsing the next generation. + // + // This is unnecessary after Go 1.26 because of an explicit end-of-generation + // signal. + spill *spilledBatch + spillErr error // error from reading spill + spillErrSync bool // whether we emitted a Sync before reporting spillErr + + v1Events *traceV1Converter } // NewReader creates a new trace reader. @@ -40,24 +59,24 @@ func NewReader(r io.Reader) (*Reader, error) { } switch v { case version.Go111, version.Go119, version.Go121: - tr, err := oldtrace.Parse(br, v) + tr, err := tracev1.Parse(br, v) if err != nil { return nil, err } return &Reader{ - go121Events: convertOldFormat(tr), + v1Events: convertV1Trace(tr), }, nil - case version.Go122, version.Go123: + case version.Go122, version.Go123, version.Go125, version.Go126: return &Reader{ - r: br, + version: v, + r: br, order: ordering{ + traceVer: v, mStates: make(map[ThreadID]*mState), pStates: make(map[ProcID]*pState), gStates: make(map[GoID]*gState), activeTasks: make(map[TaskID]taskState), }, - // Don't emit a sync event when we first go to emit events. - emittedSync: true, }, nil default: return nil, fmt.Errorf("unknown or unsupported version go 1.%d", v) @@ -66,19 +85,36 @@ func NewReader(r io.Reader) (*Reader, error) { // ReadEvent reads a single event from the stream. // -// If the stream has been exhausted, it returns an invalid -// event and io.EOF. +// If the stream has been exhausted, it returns an invalid event and io.EOF. func (r *Reader) ReadEvent() (e Event, err error) { - if r.go121Events != nil { - ev, err := r.go121Events.next() - if err != nil { - // XXX do we have to emit an EventSync when the trace is done? + // Return only io.EOF if we're done. + if r.done { + return Event{}, io.EOF + } + + // Handle v1 execution traces. + if r.v1Events != nil { + if r.syncs == 0 { + // Always emit a sync event first, if we have any events at all. + ev, ok := r.v1Events.events.Peek() + if ok { + r.syncs++ + return syncEvent(r.v1Events.evt, Time(ev.Ts-1), r.syncs), nil + } + } + ev, err := r.v1Events.next() + if err == io.EOF { + // Always emit a sync event at the end. + r.done = true + r.syncs++ + return syncEvent(nil, r.v1Events.lastTs+1, r.syncs), nil + } else if err != nil { return Event{}, err } return ev, nil } - // Go 1.22+ trace parsing algorithm. + // Trace v2 parsing algorithm. // // (1) Read in all the batches for the next generation from the stream. // (a) Use the size field in the header to quickly find all batches. @@ -115,49 +151,23 @@ func (r *Reader) ReadEvent() (e Event, err error) { // Check if we need to refresh the generation. if len(r.frontier) == 0 && len(r.cpuSamples) == 0 { - if !r.emittedSync { - r.emittedSync = true - return syncEvent(r.gen.evTable, r.lastTs), nil + if r.version < version.Go126 { + return r.nextGenWithSpill() } - if r.spillErr != nil { - return Event{}, r.spillErr - } - if r.gen != nil && r.spill == nil { - // If we have a generation from the last read, - // and there's nothing left in the frontier, and - // there's no spilled batch, indicating that there's - // no further generation, it means we're done. - // Return io.EOF. - return Event{}, io.EOF - } - // Read the next generation. - var err error - r.gen, r.spill, err = readGeneration(r.r, r.spill) - if r.gen == nil { - return Event{}, err + if r.readGenErr != nil { + return Event{}, r.readGenErr } - r.spillErr = err - - // Reset CPU samples cursor. - r.cpuSamples = r.gen.cpuSamples - - // Reset frontier. - for _, m := range r.gen.batchMs { - batches := r.gen.batches[m] - bc := &batchCursor{m: m} - ok, err := bc.nextEvent(batches, r.gen.freq) - if err != nil { - return Event{}, err - } - if !ok { - // Turns out there aren't actually any events in these batches. - continue - } - r.frontier = heapInsert(r.frontier, bc) + gen, err := readGeneration(r.r, r.version) + if err != nil { + // Before returning an error, emit the sync event + // for the current generation and queue up the error + // for the next call. + r.readGenErr = err + r.gen = nil + r.syncs++ + return syncEvent(nil, r.lastTs, r.syncs), nil } - - // Reset emittedSync. - r.emittedSync = false + return r.installGen(gen) } tryAdvance := func(i int) (bool, error) { bc := r.frontier[i] @@ -224,10 +234,82 @@ func (r *Reader) ReadEvent() (e Event, err error) { return ev, nil } +// nextGenWithSpill reads the generation and calls nextGen while +// also handling any spilled batches. +func (r *Reader) nextGenWithSpill() (Event, error) { + if r.version >= version.Go126 { + return Event{}, errors.New("internal error: nextGenWithSpill called for Go 1.26+ trace") + } + if r.spillErr != nil { + if r.spillErrSync { + return Event{}, r.spillErr + } + r.spillErrSync = true + r.syncs++ + return syncEvent(nil, r.lastTs, r.syncs), nil + } + if r.gen != nil && r.spill == nil { + // If we have a generation from the last read, + // and there's nothing left in the frontier, and + // there's no spilled batch, indicating that there's + // no further generation, it means we're done. + // Emit the final sync event. + r.done = true + r.syncs++ + return syncEvent(nil, r.lastTs, r.syncs), nil + } + + // Read the next generation. + var gen *generation + gen, r.spill, r.spillErr = readGenerationWithSpill(r.r, r.spill, r.version) + if gen == nil { + r.gen = nil + r.spillErrSync = true + r.syncs++ + return syncEvent(nil, r.lastTs, r.syncs), nil + } + return r.installGen(gen) +} + +// installGen installs the new generation into the Reader and returns +// a Sync event for the new generation. +func (r *Reader) installGen(gen *generation) (Event, error) { + if gen == nil { + // Emit the final sync event. + r.gen = nil + r.done = true + r.syncs++ + return syncEvent(nil, r.lastTs, r.syncs), nil + } + r.gen = gen + + // Reset CPU samples cursor. + r.cpuSamples = r.gen.cpuSamples + + // Reset frontier. + for _, m := range r.gen.batchMs { + batches := r.gen.batches[m] + bc := &batchCursor{m: m} + ok, err := bc.nextEvent(batches, r.gen.freq) + if err != nil { + return Event{}, err + } + if !ok { + // Turns out there aren't actually any events in these batches. + continue + } + r.frontier = heapInsert(r.frontier, bc) + } + r.syncs++ + + // Always emit a sync event at the beginning of the generation. + return syncEvent(r.gen.evTable, r.gen.freq.mul(r.gen.minTs), r.syncs), nil +} + func dumpFrontier(frontier []*batchCursor) string { var sb strings.Builder for _, bc := range frontier { - spec := go122.Specs()[bc.ev.typ] + spec := tracev2.Specs()[bc.ev.typ] fmt.Fprintf(&sb, "M %d [%s time=%d", bc.m, spec.Name, bc.ev.time) for i, arg := range spec.Args[1:] { fmt.Fprintf(&sb, " %s=%d", arg, bc.ev.args[i]) diff --git a/src/internal/trace/reader_test.go b/src/internal/trace/reader_test.go index 37c21da667ef34..c03d0676a07c7a 100644 --- a/src/internal/trace/reader_test.go +++ b/src/internal/trace/reader_test.go @@ -7,17 +7,14 @@ package trace_test import ( "bytes" "flag" - "fmt" "io" - "os" "path/filepath" - "strings" + "runtime" "testing" + "time" "internal/trace" - "internal/trace/raw" "internal/trace/testtrace" - "internal/trace/version" ) var ( @@ -31,17 +28,18 @@ func TestReaderGolden(t *testing.T) { t.Fatalf("failed to glob for tests: %v", err) } for _, testPath := range matches { - testPath := testPath testName, err := filepath.Rel("./testdata", testPath) if err != nil { t.Fatalf("failed to relativize testdata path: %v", err) } t.Run(testName, func(t *testing.T) { - tr, exp, err := testtrace.ParseFile(testPath) + tr, ver, exp, err := testtrace.ParseFile(testPath) if err != nil { t.Fatalf("failed to parse test file at %s: %v", testPath, err) } - testReader(t, tr, exp) + v := testtrace.NewValidator() + v.GoVersion = ver + testReader(t, tr, v, exp) }) } } @@ -93,7 +91,7 @@ func FuzzReader(f *testing.F) { }) } -func testReader(t *testing.T, tr io.Reader, exp *testtrace.Expectation) { +func testReader(t *testing.T, tr io.Reader, v *testtrace.Validator, exp *testtrace.Expectation) { r, err := trace.NewReader(tr) if err != nil { if err := exp.Check(err); err != nil { @@ -101,12 +99,15 @@ func testReader(t *testing.T, tr io.Reader, exp *testtrace.Expectation) { } return } - v := testtrace.NewValidator() for { ev, err := r.ReadEvent() if err == io.EOF { break } + v.GoVersion = r.GoVersion() + if runtime.GOOS == "windows" || runtime.GOARCH == "wasm" { + v.SkipClockSnapshotChecks() + } if err != nil { if err := exp.Check(err); err != nil { t.Error(err) @@ -125,48 +126,75 @@ func testReader(t *testing.T, tr io.Reader, exp *testtrace.Expectation) { } } -func dumpTraceToText(t *testing.T, b []byte) string { - t.Helper() - - br, err := raw.NewReader(bytes.NewReader(b)) - if err != nil { - t.Fatalf("dumping trace: %v", err) - } - var sb strings.Builder - tw, err := raw.NewTextWriter(&sb, version.Current) - if err != nil { - t.Fatalf("dumping trace: %v", err) +func TestTraceGenSync(t *testing.T) { + type sync struct { + Time trace.Time + ClockSnapshot *trace.ClockSnapshot } - for { - ev, err := br.ReadEvent() - if err == io.EOF { - break - } - if err != nil { - t.Fatalf("dumping trace: %v", err) - } - if err := tw.WriteEvent(ev); err != nil { - t.Fatalf("dumping trace: %v", err) - } - } - return sb.String() -} + runTest := func(testName string, wantSyncs []sync) { + t.Run(testName, func(t *testing.T) { + testPath := "testdata/tests/" + testName + r, _, _, err := testtrace.ParseFile(testPath) + if err != nil { + t.Fatalf("malformed test %s: bad trace file: %v", testPath, err) + } + tr, err := trace.NewReader(r) + if err != nil { + t.Fatalf("malformed test %s: bad trace file: %v", testPath, err) + } + var syncEvents []trace.Event + for { + ev, err := tr.ReadEvent() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("malformed test %s: bad trace file: %v", testPath, err) + } + if ev.Kind() == trace.EventSync { + syncEvents = append(syncEvents, ev) + } + } -func dumpTraceToFile(t *testing.T, testName string, stress bool, b []byte) string { - t.Helper() + if got, want := len(syncEvents), len(wantSyncs); got != want { + t.Errorf("got %d sync events, want %d", got, want) + } - desc := "default" - if stress { - desc = "stress" - } - name := fmt.Sprintf("%s.%s.trace.", testName, desc) - f, err := os.CreateTemp("", name) - if err != nil { - t.Fatalf("creating temp file: %v", err) - } - defer f.Close() - if _, err := io.Copy(f, bytes.NewReader(b)); err != nil { - t.Fatalf("writing trace dump to %q: %v", f.Name(), err) + for i, want := range wantSyncs { + got := syncEvents[i] + gotSync := syncEvents[i].Sync() + if got.Time() != want.Time { + t.Errorf("sync=%d got time %d, want %d", i+1, got.Time(), want.Time) + } + if gotSync.ClockSnapshot == nil && want.ClockSnapshot == nil { + continue + } + if gotSync.ClockSnapshot.Trace != want.ClockSnapshot.Trace { + t.Errorf("sync=%d got trace time %d, want %d", i+1, gotSync.ClockSnapshot.Trace, want.ClockSnapshot.Trace) + } + if !gotSync.ClockSnapshot.Wall.Equal(want.ClockSnapshot.Wall) { + t.Errorf("sync=%d got wall time %s, want %s", i+1, gotSync.ClockSnapshot.Wall, want.ClockSnapshot.Wall) + } + if gotSync.ClockSnapshot.Mono != want.ClockSnapshot.Mono { + t.Errorf("sync=%d got mono time %d, want %d", i+1, gotSync.ClockSnapshot.Mono, want.ClockSnapshot.Mono) + } + } + }) } - return f.Name() + + runTest("go123-sync.test", []sync{ + {10, nil}, + {40, nil}, + // The EvFrequency batch for generation 3 is emitted at trace.Time(80), + // but 60 is the minTs of the generation, see b30 in the go generator. + {60, nil}, + {63, nil}, + }) + + runTest("go125-sync.test", []sync{ + {9, &trace.ClockSnapshot{Trace: 10, Mono: 99, Wall: time.Date(2025, 2, 28, 15, 4, 9, 123, time.UTC)}}, + {38, &trace.ClockSnapshot{Trace: 40, Mono: 199, Wall: time.Date(2025, 2, 28, 15, 4, 10, 123, time.UTC)}}, + {58, &trace.ClockSnapshot{Trace: 60, Mono: 299, Wall: time.Date(2025, 2, 28, 15, 4, 11, 123, time.UTC)}}, + {83, nil}, + }) } diff --git a/src/internal/trace/resources.go b/src/internal/trace/resources.go index f49696f91c56bd..24db2f8d77a66a 100644 --- a/src/internal/trace/resources.go +++ b/src/internal/trace/resources.go @@ -224,7 +224,6 @@ type StateTransition struct { // The actual transition data. Stored in a neutral form so that // we don't need fields for every kind of resource. - id int64 oldState uint8 newState uint8 } diff --git a/src/internal/trace/summary.go b/src/internal/trace/summary.go index fa3e3359c7c59d..f31439feb8ff0b 100644 --- a/src/internal/trace/summary.go +++ b/src/internal/trace/summary.go @@ -390,24 +390,14 @@ func (s *Summarizer) Event(ev *Event) { // This root frame will be identical for all transitions on this // goroutine, because it represents its immutable start point. if g.Name == "" { - stk := st.Stack - if stk != NoStack { - var frame StackFrame - var ok bool - stk.Frames(func(f StackFrame) bool { - frame = f - ok = true - return true - }) - if ok { - // NB: this PC won't actually be consistent for - // goroutines which existed at the start of the - // trace. The UI doesn't use it directly; this - // mainly serves as an indication that we - // actually saw a call stack for the goroutine - g.PC = frame.PC - g.Name = frame.Func - } + for frame := range st.Stack.Frames() { + // NB: this PC won't actually be consistent for + // goroutines which existed at the start of the + // trace. The UI doesn't use it directly; this + // mainly serves as an indication that we + // actually saw a call stack for the goroutine + g.PC = frame.PC + g.Name = frame.Func } } diff --git a/src/internal/trace/summary_test.go b/src/internal/trace/summary_test.go index 73865652ebac44..c5f1910262cb2a 100644 --- a/src/internal/trace/summary_test.go +++ b/src/internal/trace/summary_test.go @@ -264,7 +264,7 @@ func basicGoroutineSummaryChecks(t *testing.T, summary *trace.GoroutineSummary) } func summarizeTraceTest(t *testing.T, testPath string) *trace.Summary { - trc, _, err := testtrace.ParseFile(testPath) + trc, _, _, err := testtrace.ParseFile(testPath) if err != nil { t.Fatalf("malformed test %s: bad trace file: %v", testPath, err) } @@ -388,7 +388,7 @@ func basicGoroutineExecStatsChecks(t *testing.T, stats *trace.GoroutineExecStats func TestRelatedGoroutinesV2Trace(t *testing.T) { testPath := "testdata/tests/go122-gc-stress.test" - trc, _, err := testtrace.ParseFile(testPath) + trc, _, _, err := testtrace.ParseFile(testPath) if err != nil { t.Fatalf("malformed test %s: bad trace file: %v", testPath, err) } @@ -416,10 +416,10 @@ func TestRelatedGoroutinesV2Trace(t *testing.T) { targetg := trace.GoID(86) got := trace.RelatedGoroutinesV2(events, targetg) want := map[trace.GoID]struct{}{ - trace.GoID(86): struct{}{}, // N.B. Result includes target. - trace.GoID(71): struct{}{}, - trace.GoID(25): struct{}{}, - trace.GoID(122): struct{}{}, + trace.GoID(86): {}, // N.B. Result includes target. + trace.GoID(71): {}, + trace.GoID(25): {}, + trace.GoID(122): {}, } for goid := range got { if _, ok := want[goid]; ok { diff --git a/src/internal/trace/testdata/generators/go122-confuse-seq-across-generations.go b/src/internal/trace/testdata/generators/go122-confuse-seq-across-generations.go index 9f27ae046ff49c..9b98723c4de8b1 100644 --- a/src/internal/trace/testdata/generators/go122-confuse-seq-across-generations.go +++ b/src/internal/trace/testdata/generators/go122-confuse-seq-across-generations.go @@ -23,12 +23,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -36,13 +37,13 @@ func gen(t *testgen.Trace) { // A running goroutine blocks. b10 := g1.Batch(trace.ThreadID(0), 0) - b10.Event("ProcStatus", trace.ProcID(0), go122.ProcRunning) - b10.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), go122.GoRunning) + b10.Event("ProcStatus", trace.ProcID(0), tracev2.ProcRunning) + b10.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), tracev2.GoRunning) b10.Event("GoStop", "whatever", testgen.NoStack) // The running goroutine gets unblocked. b11 := g1.Batch(trace.ThreadID(1), 0) - b11.Event("ProcStatus", trace.ProcID(1), go122.ProcRunning) + b11.Event("ProcStatus", trace.ProcID(1), tracev2.ProcRunning) b11.Event("GoStart", trace.GoID(1), testgen.Seq(1)) b11.Event("GoStop", "whatever", testgen.NoStack) @@ -50,13 +51,13 @@ func gen(t *testgen.Trace) { // Start running the goroutine, but later. b21 := g2.Batch(trace.ThreadID(1), 3) - b21.Event("ProcStatus", trace.ProcID(1), go122.ProcRunning) + b21.Event("ProcStatus", trace.ProcID(1), tracev2.ProcRunning) b21.Event("GoStart", trace.GoID(1), testgen.Seq(2)) // The goroutine starts running, then stops, then starts again. b20 := g2.Batch(trace.ThreadID(0), 5) - b20.Event("ProcStatus", trace.ProcID(0), go122.ProcRunning) - b20.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), go122.GoRunnable) + b20.Event("ProcStatus", trace.ProcID(0), tracev2.ProcRunning) + b20.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), tracev2.GoRunnable) b20.Event("GoStart", trace.GoID(1), testgen.Seq(1)) b20.Event("GoStop", "whatever", testgen.NoStack) } diff --git a/src/internal/trace/testdata/generators/go122-create-syscall-reuse-thread-id.go b/src/internal/trace/testdata/generators/go122-create-syscall-reuse-thread-id.go index a14d376779ced6..dc5c4a52572caf 100644 --- a/src/internal/trace/testdata/generators/go122-create-syscall-reuse-thread-id.go +++ b/src/internal/trace/testdata/generators/go122-create-syscall-reuse-thread-id.go @@ -4,7 +4,7 @@ // Tests a G being created from within a syscall. // -// Specifically, it tests a scenerio wherein a C +// Specifically, it tests a scenario wherein a C // thread is calling into Go, creating a goroutine in // a syscall (in the tracer's model). The system is free // to reuse thread IDs, so first a thread ID is used to @@ -25,12 +25,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -41,9 +42,9 @@ func gen(t *testgen.Trace) { b0 := g.Batch(trace.ThreadID(0), 0) b0.Event("GoCreateSyscall", trace.GoID(4)) b0.Event("GoSyscallEndBlocked") - b0.Event("ProcStatus", trace.ProcID(0), go122.ProcIdle) + b0.Event("ProcStatus", trace.ProcID(0), tracev2.ProcIdle) b0.Event("ProcStart", trace.ProcID(0), testgen.Seq(1)) - b0.Event("GoStatus", trace.GoID(4), trace.NoThread, go122.GoRunnable) + b0.Event("GoStatus", trace.GoID(4), trace.NoThread, tracev2.GoRunnable) b0.Event("GoStart", trace.GoID(4), testgen.Seq(1)) b0.Event("GoSyscallBegin", testgen.Seq(2), testgen.NoStack) b0.Event("GoDestroySyscall") @@ -55,7 +56,7 @@ func gen(t *testgen.Trace) { // have a self-steal here potentially that doesn't make // sense. b1 := g.Batch(trace.ThreadID(0), 0) - b1.Event("ProcStatus", trace.ProcID(1), go122.ProcIdle) + b1.Event("ProcStatus", trace.ProcID(1), tracev2.ProcIdle) b1.Event("ProcStart", trace.ProcID(1), testgen.Seq(1)) b1.Event("ProcSteal", trace.ProcID(0), testgen.Seq(3), trace.ThreadID(0)) } diff --git a/src/internal/trace/testdata/generators/go122-create-syscall-with-p.go b/src/internal/trace/testdata/generators/go122-create-syscall-with-p.go index 1eb18026ecb3df..90729d7c528881 100644 --- a/src/internal/trace/testdata/generators/go122-create-syscall-with-p.go +++ b/src/internal/trace/testdata/generators/go122-create-syscall-with-p.go @@ -4,7 +4,7 @@ // Tests a G being created from within a syscall. // -// Specifically, it tests a scenerio wherein a C +// Specifically, it tests a scenario wherein a C // thread is calling into Go, creating a goroutine in // a syscall (in the tracer's model). Because the actual // m can be reused, it's possible for that m to have never @@ -16,12 +16,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -39,7 +40,7 @@ func gen(t *testgen.Trace) { // possible on other platforms, however. b0 := g.Batch(trace.ThreadID(0), 0) b0.Event("GoCreateSyscall", trace.GoID(4)) - b0.Event("ProcStatus", trace.ProcID(0), go122.ProcIdle) + b0.Event("ProcStatus", trace.ProcID(0), tracev2.ProcIdle) b0.Event("ProcStart", trace.ProcID(0), testgen.Seq(1)) b0.Event("GoSyscallEndBlocked") b0.Event("GoStart", trace.GoID(4), testgen.Seq(1)) diff --git a/src/internal/trace/testdata/generators/go122-fail-first-gen-first.go b/src/internal/trace/testdata/generators/go122-fail-first-gen-first.go index f0e02be15560d2..c8ead6772c0dcb 100644 --- a/src/internal/trace/testdata/generators/go122-fail-first-gen-first.go +++ b/src/internal/trace/testdata/generators/go122-fail-first-gen-first.go @@ -19,25 +19,26 @@ package main import ( - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { // A running goroutine emits a task begin. - t.RawEvent(go122.EvEventBatch, nil, 1 /*gen*/, 0 /*thread ID*/, 0 /*timestamp*/, 5 /*batch length*/) - t.RawEvent(go122.EvFrequency, nil, 15625000) + t.RawEvent(tracev2.EvEventBatch, nil, 1 /*gen*/, 0 /*thread ID*/, 0 /*timestamp*/, 5 /*batch length*/) + t.RawEvent(tracev2.EvFrequency, nil, 15625000) // A running goroutine emits a task begin. - t.RawEvent(go122.EvEventBatch, nil, 1 /*gen*/, 0 /*thread ID*/, 0 /*timestamp*/, 5 /*batch length*/) - t.RawEvent(go122.EvGoCreate, nil, 0 /*timestamp delta*/, 1 /*go ID*/, 0, 0) + t.RawEvent(tracev2.EvEventBatch, nil, 1 /*gen*/, 0 /*thread ID*/, 0 /*timestamp*/, 5 /*batch length*/) + t.RawEvent(tracev2.EvGoCreate, nil, 0 /*timestamp delta*/, 1 /*go ID*/, 0, 0) // Write an invalid batch event for the next generation. - t.RawEvent(go122.EvEventBatch, nil, 2 /*gen*/, 0 /*thread ID*/, 0 /*timestamp*/, 50 /*batch length (invalid)*/) + t.RawEvent(tracev2.EvEventBatch, nil, 2 /*gen*/, 0 /*thread ID*/, 0 /*timestamp*/, 50 /*batch length (invalid)*/) // We should fail at the first issue, not the second one. t.ExpectFailure("expected a proc but didn't have one") diff --git a/src/internal/trace/testdata/generators/go122-go-create-without-running-g.go b/src/internal/trace/testdata/generators/go122-go-create-without-running-g.go index 217089975861fd..2e9b571d46ecca 100644 --- a/src/internal/trace/testdata/generators/go122-go-create-without-running-g.go +++ b/src/internal/trace/testdata/generators/go122-go-create-without-running-g.go @@ -13,12 +13,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -26,7 +27,7 @@ func gen(t *testgen.Trace) { // A goroutine gets created on a running P, then starts running. b0 := g1.Batch(trace.ThreadID(0), 0) - b0.Event("ProcStatus", trace.ProcID(0), go122.ProcRunning) + b0.Event("ProcStatus", trace.ProcID(0), tracev2.ProcRunning) b0.Event("GoCreate", trace.GoID(5), testgen.NoStack, testgen.NoStack) b0.Event("GoStart", trace.GoID(5), testgen.Seq(1)) b0.Event("GoStop", "whatever", testgen.NoStack) diff --git a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-ambiguous.go b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-ambiguous.go index 1f01cc9e78b21e..28d187c37e4782 100644 --- a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-ambiguous.go +++ b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-ambiguous.go @@ -4,7 +4,7 @@ // Tests syscall P stealing. // -// Specifically, it tests a scenerio wherein, without a +// Specifically, it tests a scenario wherein, without a // P sequence number of GoSyscallBegin, the syscall that // a ProcSteal applies to is ambiguous. This only happens in // practice when the events aren't already properly ordered @@ -15,12 +15,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -31,8 +32,8 @@ func gen(t *testgen.Trace) { // One goroutine does a syscall without blocking, then another one where // it's P gets stolen. b0 := g.Batch(trace.ThreadID(0), 0) - b0.Event("ProcStatus", trace.ProcID(0), go122.ProcRunning) - b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), go122.GoRunning) + b0.Event("ProcStatus", trace.ProcID(0), tracev2.ProcRunning) + b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), tracev2.GoRunning) b0.Event("GoSyscallBegin", testgen.Seq(1), testgen.NoStack) b0.Event("GoSyscallEnd") b0.Event("GoSyscallBegin", testgen.Seq(2), testgen.NoStack) @@ -40,7 +41,7 @@ func gen(t *testgen.Trace) { // A running goroutine steals proc 0. b1 := g.Batch(trace.ThreadID(1), 0) - b1.Event("ProcStatus", trace.ProcID(2), go122.ProcRunning) - b1.Event("GoStatus", trace.GoID(2), trace.ThreadID(1), go122.GoRunning) + b1.Event("ProcStatus", trace.ProcID(2), tracev2.ProcRunning) + b1.Event("GoStatus", trace.GoID(2), trace.ThreadID(1), tracev2.GoRunning) b1.Event("ProcSteal", trace.ProcID(0), testgen.Seq(3), trace.ThreadID(0)) } diff --git a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary-bare-m.go b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary-bare-m.go index 1d7fe9c57c55d8..5350b197404fe1 100644 --- a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary-bare-m.go +++ b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary-bare-m.go @@ -8,12 +8,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -22,12 +23,12 @@ func gen(t *testgen.Trace) { // One goroutine is exiting with a syscall. It already // acquired a new P. b0 := g.Batch(trace.ThreadID(0), 0) - b0.Event("ProcStatus", trace.ProcID(1), go122.ProcRunning) - b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), go122.GoSyscall) + b0.Event("ProcStatus", trace.ProcID(1), tracev2.ProcRunning) + b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), tracev2.GoSyscall) b0.Event("GoSyscallEndBlocked") // A bare M stole the goroutine's P at the generation boundary. b1 := g.Batch(trace.ThreadID(1), 0) - b1.Event("ProcStatus", trace.ProcID(0), go122.ProcSyscallAbandoned) + b1.Event("ProcStatus", trace.ProcID(0), tracev2.ProcSyscallAbandoned) b1.Event("ProcSteal", trace.ProcID(0), testgen.Seq(1), trace.ThreadID(0)) } diff --git a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary-reacquire-new-proc-bare-m.go b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary-reacquire-new-proc-bare-m.go index a94b8f058d5922..f7611c5c0824c5 100644 --- a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary-reacquire-new-proc-bare-m.go +++ b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary-reacquire-new-proc-bare-m.go @@ -8,12 +8,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -22,13 +23,13 @@ func gen(t *testgen.Trace) { // One goroutine is exiting with a syscall. It already // acquired a new P. b0 := g.Batch(trace.ThreadID(0), 0) - b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), go122.GoSyscall) - b0.Event("ProcStatus", trace.ProcID(1), go122.ProcIdle) + b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), tracev2.GoSyscall) + b0.Event("ProcStatus", trace.ProcID(1), tracev2.ProcIdle) b0.Event("ProcStart", trace.ProcID(1), testgen.Seq(1)) b0.Event("GoSyscallEndBlocked") // A bare M stole the goroutine's P at the generation boundary. b1 := g.Batch(trace.ThreadID(1), 0) - b1.Event("ProcStatus", trace.ProcID(0), go122.ProcSyscallAbandoned) + b1.Event("ProcStatus", trace.ProcID(0), tracev2.ProcSyscallAbandoned) b1.Event("ProcSteal", trace.ProcID(0), testgen.Seq(1), trace.ThreadID(0)) } diff --git a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary-reacquire-new-proc.go b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary-reacquire-new-proc.go index 04aef0644cdffb..521363b0942b99 100644 --- a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary-reacquire-new-proc.go +++ b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary-reacquire-new-proc.go @@ -8,12 +8,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -22,15 +23,15 @@ func gen(t *testgen.Trace) { // One goroutine is exiting with a syscall. It already // acquired a new P. b0 := g.Batch(trace.ThreadID(0), 0) - b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), go122.GoSyscall) - b0.Event("ProcStatus", trace.ProcID(1), go122.ProcIdle) + b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), tracev2.GoSyscall) + b0.Event("ProcStatus", trace.ProcID(1), tracev2.ProcIdle) b0.Event("ProcStart", trace.ProcID(1), testgen.Seq(1)) b0.Event("GoSyscallEndBlocked") // A running goroutine stole P0 at the generation boundary. b1 := g.Batch(trace.ThreadID(1), 0) - b1.Event("ProcStatus", trace.ProcID(2), go122.ProcRunning) - b1.Event("GoStatus", trace.GoID(2), trace.ThreadID(1), go122.GoRunning) - b1.Event("ProcStatus", trace.ProcID(0), go122.ProcSyscallAbandoned) + b1.Event("ProcStatus", trace.ProcID(2), tracev2.ProcRunning) + b1.Event("GoStatus", trace.GoID(2), trace.ThreadID(1), tracev2.GoRunning) + b1.Event("ProcStatus", trace.ProcID(0), tracev2.ProcSyscallAbandoned) b1.Event("ProcSteal", trace.ProcID(0), testgen.Seq(1), trace.ThreadID(0)) } diff --git a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary.go b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary.go index 769203ab4ae9dc..6c171c9cd1f89e 100644 --- a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary.go +++ b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-gen-boundary.go @@ -8,12 +8,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -22,14 +23,14 @@ func gen(t *testgen.Trace) { // One goroutine is exiting with a syscall. It already // acquired a new P. b0 := g.Batch(trace.ThreadID(0), 0) - b0.Event("ProcStatus", trace.ProcID(1), go122.ProcRunning) - b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), go122.GoSyscall) + b0.Event("ProcStatus", trace.ProcID(1), tracev2.ProcRunning) + b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), tracev2.GoSyscall) b0.Event("GoSyscallEndBlocked") // A running goroutine stole P0 at the generation boundary. b1 := g.Batch(trace.ThreadID(1), 0) - b1.Event("ProcStatus", trace.ProcID(2), go122.ProcRunning) - b1.Event("GoStatus", trace.GoID(2), trace.ThreadID(1), go122.GoRunning) - b1.Event("ProcStatus", trace.ProcID(0), go122.ProcSyscallAbandoned) + b1.Event("ProcStatus", trace.ProcID(2), tracev2.ProcRunning) + b1.Event("GoStatus", trace.GoID(2), trace.ThreadID(1), tracev2.GoRunning) + b1.Event("ProcStatus", trace.ProcID(0), tracev2.ProcSyscallAbandoned) b1.Event("ProcSteal", trace.ProcID(0), testgen.Seq(1), trace.ThreadID(0)) } diff --git a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-reacquire-new-proc-bare-m.go b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-reacquire-new-proc-bare-m.go index c1c39569f8e7f7..18493dd5c383aa 100644 --- a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-reacquire-new-proc-bare-m.go +++ b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-reacquire-new-proc-bare-m.go @@ -8,12 +8,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -21,9 +22,9 @@ func gen(t *testgen.Trace) { // One goroutine enters a syscall, grabs a P, and starts running. b0 := g.Batch(trace.ThreadID(0), 0) - b0.Event("ProcStatus", trace.ProcID(1), go122.ProcIdle) - b0.Event("ProcStatus", trace.ProcID(0), go122.ProcRunning) - b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), go122.GoRunning) + b0.Event("ProcStatus", trace.ProcID(1), tracev2.ProcIdle) + b0.Event("ProcStatus", trace.ProcID(0), tracev2.ProcRunning) + b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), tracev2.GoRunning) b0.Event("GoSyscallBegin", testgen.Seq(1), testgen.NoStack) b0.Event("ProcStart", trace.ProcID(1), testgen.Seq(1)) b0.Event("GoSyscallEndBlocked") diff --git a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-reacquire-new-proc.go b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-reacquire-new-proc.go index 217e4e6f962045..d4e6ed3e2a2ed4 100644 --- a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-reacquire-new-proc.go +++ b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-reacquire-new-proc.go @@ -8,12 +8,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -21,16 +22,16 @@ func gen(t *testgen.Trace) { // One goroutine enters a syscall, grabs a P, and starts running. b0 := g.Batch(trace.ThreadID(0), 0) - b0.Event("ProcStatus", trace.ProcID(1), go122.ProcIdle) - b0.Event("ProcStatus", trace.ProcID(0), go122.ProcRunning) - b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), go122.GoRunning) + b0.Event("ProcStatus", trace.ProcID(1), tracev2.ProcIdle) + b0.Event("ProcStatus", trace.ProcID(0), tracev2.ProcRunning) + b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), tracev2.GoRunning) b0.Event("GoSyscallBegin", testgen.Seq(1), testgen.NoStack) b0.Event("ProcStart", trace.ProcID(1), testgen.Seq(1)) b0.Event("GoSyscallEndBlocked") // A running goroutine steals proc 0. b1 := g.Batch(trace.ThreadID(1), 0) - b1.Event("ProcStatus", trace.ProcID(2), go122.ProcRunning) - b1.Event("GoStatus", trace.GoID(2), trace.ThreadID(1), go122.GoRunning) + b1.Event("ProcStatus", trace.ProcID(2), tracev2.ProcRunning) + b1.Event("GoStatus", trace.GoID(2), trace.ThreadID(1), tracev2.GoRunning) b1.Event("ProcSteal", trace.ProcID(0), testgen.Seq(2), trace.ThreadID(0)) } diff --git a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-self.go b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-self.go index a12f47177fb988..6dfb465b0a1819 100644 --- a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-self.go +++ b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-self.go @@ -12,12 +12,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -28,8 +29,8 @@ func gen(t *testgen.Trace) { // A goroutine execute a syscall and steals its own P, then starts running // on that P. b0 := g.Batch(trace.ThreadID(0), 0) - b0.Event("ProcStatus", trace.ProcID(0), go122.ProcRunning) - b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), go122.GoRunning) + b0.Event("ProcStatus", trace.ProcID(0), tracev2.ProcRunning) + b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), tracev2.GoRunning) b0.Event("GoSyscallBegin", testgen.Seq(1), testgen.NoStack) b0.Event("ProcSteal", trace.ProcID(0), testgen.Seq(2), trace.ThreadID(0)) b0.Event("ProcStart", trace.ProcID(0), testgen.Seq(3)) diff --git a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-simple-bare-m.go b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-simple-bare-m.go index 34c7415cae613f..ac314a66478049 100644 --- a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-simple-bare-m.go +++ b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-simple-bare-m.go @@ -8,12 +8,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -21,8 +22,8 @@ func gen(t *testgen.Trace) { // One goroutine enters a syscall. b0 := g.Batch(trace.ThreadID(0), 0) - b0.Event("ProcStatus", trace.ProcID(0), go122.ProcRunning) - b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), go122.GoRunning) + b0.Event("ProcStatus", trace.ProcID(0), tracev2.ProcRunning) + b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), tracev2.GoRunning) b0.Event("GoSyscallBegin", testgen.Seq(1), testgen.NoStack) b0.Event("GoSyscallEndBlocked") diff --git a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-simple.go b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-simple.go index 6304105af04cff..010272e5523c17 100644 --- a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-simple.go +++ b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-simple.go @@ -8,12 +8,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -21,14 +22,14 @@ func gen(t *testgen.Trace) { // One goroutine enters a syscall. b0 := g.Batch(trace.ThreadID(0), 0) - b0.Event("ProcStatus", trace.ProcID(0), go122.ProcRunning) - b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), go122.GoRunning) + b0.Event("ProcStatus", trace.ProcID(0), tracev2.ProcRunning) + b0.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), tracev2.GoRunning) b0.Event("GoSyscallBegin", testgen.Seq(1), testgen.NoStack) b0.Event("GoSyscallEndBlocked") // A running goroutine steals proc 0. b1 := g.Batch(trace.ThreadID(1), 0) - b1.Event("ProcStatus", trace.ProcID(2), go122.ProcRunning) - b1.Event("GoStatus", trace.GoID(2), trace.ThreadID(1), go122.GoRunning) + b1.Event("ProcStatus", trace.ProcID(2), tracev2.ProcRunning) + b1.Event("GoStatus", trace.GoID(2), trace.ThreadID(1), tracev2.GoRunning) b1.Event("ProcSteal", trace.ProcID(0), testgen.Seq(2), trace.ThreadID(0)) } diff --git a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-sitting-in-syscall.go b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-sitting-in-syscall.go index ac84261f020593..410f9b7a089be8 100644 --- a/src/internal/trace/testdata/generators/go122-syscall-steal-proc-sitting-in-syscall.go +++ b/src/internal/trace/testdata/generators/go122-syscall-steal-proc-sitting-in-syscall.go @@ -9,12 +9,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -23,10 +24,10 @@ func gen(t *testgen.Trace) { // Steal proc from a goroutine that's been blocked // in a syscall the entire generation. b0 := g.Batch(trace.ThreadID(0), 0) - b0.Event("ProcStatus", trace.ProcID(0), go122.ProcSyscallAbandoned) + b0.Event("ProcStatus", trace.ProcID(0), tracev2.ProcSyscallAbandoned) b0.Event("ProcSteal", trace.ProcID(0), testgen.Seq(1), trace.ThreadID(1)) // Status event for a goroutine blocked in a syscall for the entire generation. bz := g.Batch(trace.NoThread, 0) - bz.Event("GoStatus", trace.GoID(1), trace.ThreadID(1), go122.GoSyscall) + bz.Event("GoStatus", trace.GoID(1), trace.ThreadID(1), tracev2.GoSyscall) } diff --git a/src/internal/trace/testdata/generators/go122-task-across-generations.go b/src/internal/trace/testdata/generators/go122-task-across-generations.go index 83b1bcdb5ef491..e8def318b403c5 100644 --- a/src/internal/trace/testdata/generators/go122-task-across-generations.go +++ b/src/internal/trace/testdata/generators/go122-task-across-generations.go @@ -14,12 +14,13 @@ package main import ( "internal/trace" - "internal/trace/event/go122" - testgen "internal/trace/internal/testgen/go122" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" ) func main() { - testgen.Main(gen) + testgen.Main(version.Go122, gen) } func gen(t *testgen.Trace) { @@ -27,15 +28,15 @@ func gen(t *testgen.Trace) { // A running goroutine emits a task begin. b1 := g1.Batch(trace.ThreadID(0), 0) - b1.Event("ProcStatus", trace.ProcID(0), go122.ProcRunning) - b1.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), go122.GoRunning) + b1.Event("ProcStatus", trace.ProcID(0), tracev2.ProcRunning) + b1.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), tracev2.GoRunning) b1.Event("UserTaskBegin", trace.TaskID(2), trace.TaskID(0) /* 0 means no parent, not background */, "my task", testgen.NoStack) g2 := t.Generation(2) // That same goroutine emits a task end in the following generation. b2 := g2.Batch(trace.ThreadID(0), 5) - b2.Event("ProcStatus", trace.ProcID(0), go122.ProcRunning) - b2.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), go122.GoRunning) + b2.Event("ProcStatus", trace.ProcID(0), tracev2.ProcRunning) + b2.Event("GoStatus", trace.GoID(1), trace.ThreadID(0), tracev2.GoRunning) b2.Event("UserTaskEnd", trace.TaskID(2), testgen.NoStack) } diff --git a/src/internal/trace/testdata/generators/go123-sync.go b/src/internal/trace/testdata/generators/go123-sync.go new file mode 100644 index 00000000000000..257581c7ea034c --- /dev/null +++ b/src/internal/trace/testdata/generators/go123-sync.go @@ -0,0 +1,30 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "internal/trace" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" + "time" +) + +func main() { + testgen.Main(version.Go123, gen) +} + +func gen(t *testgen.Trace) { + g1 := t.Generation(1) + g1.Sync(1000000000, 10, 0, time.Time{}) + b10 := g1.Batch(1, 15) + b10.Event("ProcStatus", trace.ProcID(0), tracev2.ProcRunning) + g2 := t.Generation(2) + g2.Sync(500000000, 20, 0, time.Time{}) + g3 := t.Generation(3) + b30 := g3.Batch(1, 30) + b30.Event("ProcStatus", trace.ProcID(0), tracev2.ProcRunning) + g3.Sync(500000000, 40, 0, time.Time{}) +} diff --git a/src/internal/trace/testdata/generators/go125-sync.go b/src/internal/trace/testdata/generators/go125-sync.go new file mode 100644 index 00000000000000..30ebf6717a7629 --- /dev/null +++ b/src/internal/trace/testdata/generators/go125-sync.go @@ -0,0 +1,31 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "internal/trace" + "internal/trace/internal/testgen" + "internal/trace/tracev2" + "internal/trace/version" + "time" +) + +func main() { + testgen.Main(version.Go125, gen) +} + +func gen(t *testgen.Trace) { + start := time.Date(2025, 2, 28, 15, 4, 9, 123, time.UTC) + g1 := t.Generation(1) + g1.Sync(1000000000, 10, 99, start) + b10 := g1.Batch(1, 15) + b10.Event("ProcStatus", trace.ProcID(0), tracev2.ProcRunning) + g2 := t.Generation(2) + g2.Sync(500000000, 20, 199, start.Add(1*time.Second)) + g3 := t.Generation(3) + g3.Sync(500000000, 30, 299, start.Add(2*time.Second)) + b30 := g3.Batch(1, 40) + b30.Event("ProcStatus", trace.ProcID(0), tracev2.ProcRunning) +} diff --git a/src/internal/trace/testdata/testprog/gc-stress.go b/src/internal/trace/testdata/testprog/gc-stress.go index 7979234c40a571..74b63606d5e619 100644 --- a/src/internal/trace/testdata/testprog/gc-stress.go +++ b/src/internal/trace/testdata/testprog/gc-stress.go @@ -13,6 +13,7 @@ import ( "log" "os" "runtime" + "runtime/debug" "runtime/trace" "time" ) @@ -36,11 +37,25 @@ func makeTree(depth int) *node { } } +func initTree(n *node) { + if n == nil { + return + } + for i := range n.data { + n.data[i] = 'a' + } + for i := range n.children { + initTree(n.children[i]) + } +} + var trees [16]*node var ballast *[16]*[1024]*node -var sink [][]byte +var sink []*node func main() { + debug.SetMemoryLimit(50 << 20) + for i := range trees { trees[i] = makeTree(6) } @@ -55,13 +70,17 @@ func main() { } procs := runtime.GOMAXPROCS(-1) - sink = make([][]byte, procs) + sink = make([]*node, procs) for i := 0; i < procs; i++ { i := i go func() { for { - sink[i] = make([]byte, 4<<10) + sink[i] = makeTree(3) + for range 5 { + initTree(sink[i]) + runtime.Gosched() + } } }() } diff --git a/src/internal/trace/testdata/testprog/stacks.go b/src/internal/trace/testdata/testprog/stacks.go index e64bc86844c4c1..478daa0d941b2b 100644 --- a/src/internal/trace/testdata/testprog/stacks.go +++ b/src/internal/trace/testdata/testprog/stacks.go @@ -97,6 +97,11 @@ func main() { rp.Read(data[:]) pipeReadDone <- true }() + go func() { // func12 + for { + syncPreemptPoint() + } + }() time.Sleep(100 * time.Millisecond) runtime.GC() @@ -127,3 +132,12 @@ func main() { runtime.GOMAXPROCS(oldGoMaxProcs) } + +//go:noinline +func syncPreemptPoint() { + if never { + syncPreemptPoint() + } +} + +var never bool diff --git a/src/internal/trace/testdata/tests/go123-sync.test b/src/internal/trace/testdata/tests/go123-sync.test new file mode 100644 index 00000000000000..44e98e6c959a11 --- /dev/null +++ b/src/internal/trace/testdata/tests/go123-sync.test @@ -0,0 +1,26 @@ +-- expect -- +SUCCESS +-- trace -- +Trace Go1.23 +EventBatch gen=1 m=1 time=15 size=4 +ProcStatus dt=1 p=0 pstatus=1 +EventBatch gen=1 m=18446744073709551615 time=10 size=6 +Frequency freq=1000000000 +EventBatch gen=1 m=18446744073709551615 time=11 size=1 +Stacks +EventBatch gen=1 m=18446744073709551615 time=12 size=1 +Strings +EventBatch gen=2 m=18446744073709551615 time=20 size=6 +Frequency freq=500000000 +EventBatch gen=2 m=18446744073709551615 time=21 size=1 +Stacks +EventBatch gen=2 m=18446744073709551615 time=22 size=1 +Strings +EventBatch gen=3 m=1 time=30 size=4 +ProcStatus dt=1 p=0 pstatus=1 +EventBatch gen=3 m=18446744073709551615 time=40 size=6 +Frequency freq=500000000 +EventBatch gen=3 m=18446744073709551615 time=41 size=1 +Stacks +EventBatch gen=3 m=18446744073709551615 time=42 size=1 +Strings diff --git a/src/internal/trace/testdata/tests/go125-sync.test b/src/internal/trace/testdata/tests/go125-sync.test new file mode 100644 index 00000000000000..dac3723b62f5a1 --- /dev/null +++ b/src/internal/trace/testdata/tests/go125-sync.test @@ -0,0 +1,32 @@ +-- expect -- +SUCCESS +-- trace -- +Trace Go1.25 +EventBatch gen=1 m=18446744073709551615 time=9 size=16 +Sync +Frequency freq=1000000000 +ClockSnapshot dt=1 mono=99 sec=1740755049 nsec=123 +EventBatch gen=1 m=1 time=15 size=4 +ProcStatus dt=1 p=0 pstatus=1 +EventBatch gen=1 m=18446744073709551615 time=11 size=1 +Stacks +EventBatch gen=1 m=18446744073709551615 time=12 size=1 +Strings +EventBatch gen=2 m=18446744073709551615 time=19 size=17 +Sync +Frequency freq=500000000 +ClockSnapshot dt=1 mono=199 sec=1740755050 nsec=123 +EventBatch gen=2 m=18446744073709551615 time=21 size=1 +Stacks +EventBatch gen=2 m=18446744073709551615 time=22 size=1 +Strings +EventBatch gen=3 m=18446744073709551615 time=29 size=17 +Sync +Frequency freq=500000000 +ClockSnapshot dt=1 mono=299 sec=1740755051 nsec=123 +EventBatch gen=3 m=1 time=40 size=4 +ProcStatus dt=1 p=0 pstatus=1 +EventBatch gen=3 m=18446744073709551615 time=31 size=1 +Stacks +EventBatch gen=3 m=18446744073709551615 time=32 size=1 +Strings diff --git a/src/internal/trace/testtrace/format.go b/src/internal/trace/testtrace/format.go index 82b69340ae28e0..5c3c64787d2582 100644 --- a/src/internal/trace/testtrace/format.go +++ b/src/internal/trace/testtrace/format.go @@ -8,33 +8,34 @@ import ( "bytes" "fmt" "internal/trace/raw" + "internal/trace/version" "internal/txtar" "io" ) // ParseFile parses a test file generated by the testgen package. -func ParseFile(testPath string) (io.Reader, *Expectation, error) { +func ParseFile(testPath string) (io.Reader, version.Version, *Expectation, error) { ar, err := txtar.ParseFile(testPath) if err != nil { - return nil, nil, fmt.Errorf("failed to read test file for %s: %v", testPath, err) + return nil, 0, nil, fmt.Errorf("failed to read test file for %s: %v", testPath, err) } if len(ar.Files) != 2 { - return nil, nil, fmt.Errorf("malformed test %s: wrong number of files", testPath) + return nil, 0, nil, fmt.Errorf("malformed test %s: wrong number of files", testPath) } if ar.Files[0].Name != "expect" { - return nil, nil, fmt.Errorf("malformed test %s: bad filename %s", testPath, ar.Files[0].Name) + return nil, 0, nil, fmt.Errorf("malformed test %s: bad filename %s", testPath, ar.Files[0].Name) } if ar.Files[1].Name != "trace" { - return nil, nil, fmt.Errorf("malformed test %s: bad filename %s", testPath, ar.Files[1].Name) + return nil, 0, nil, fmt.Errorf("malformed test %s: bad filename %s", testPath, ar.Files[1].Name) } tr, err := raw.NewTextReader(bytes.NewReader(ar.Files[1].Data)) if err != nil { - return nil, nil, fmt.Errorf("malformed test %s: bad trace file: %v", testPath, err) + return nil, 0, nil, fmt.Errorf("malformed test %s: bad trace file: %v", testPath, err) } var buf bytes.Buffer tw, err := raw.NewWriter(&buf, tr.Version()) if err != nil { - return nil, nil, fmt.Errorf("failed to create trace byte writer: %v", err) + return nil, 0, nil, fmt.Errorf("failed to create trace byte writer: %v", err) } for { ev, err := tr.ReadEvent() @@ -42,15 +43,15 @@ func ParseFile(testPath string) (io.Reader, *Expectation, error) { break } if err != nil { - return nil, nil, fmt.Errorf("malformed test %s: bad trace file: %v", testPath, err) + return nil, 0, nil, fmt.Errorf("malformed test %s: bad trace file: %v", testPath, err) } if err := tw.WriteEvent(ev); err != nil { - return nil, nil, fmt.Errorf("internal error during %s: failed to write trace bytes: %v", testPath, err) + return nil, 0, nil, fmt.Errorf("internal error during %s: failed to write trace bytes: %v", testPath, err) } } exp, err := ParseExpectation(ar.Files[0].Data) if err != nil { - return nil, nil, fmt.Errorf("internal error during %s: failed to parse expectation %q: %v", testPath, string(ar.Files[0].Data), err) + return nil, 0, nil, fmt.Errorf("internal error during %s: failed to parse expectation %q: %v", testPath, string(ar.Files[0].Data), err) } - return &buf, exp, nil + return &buf, tr.Version(), exp, nil } diff --git a/src/internal/trace/testtrace/helpers.go b/src/internal/trace/testtrace/helpers.go new file mode 100644 index 00000000000000..50f6825bab5df9 --- /dev/null +++ b/src/internal/trace/testtrace/helpers.go @@ -0,0 +1,87 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testtrace + +import ( + "bytes" + "fmt" + "internal/testenv" + "internal/trace/raw" + "internal/trace/version" + "io" + "os" + "strings" + "testing" +) + +// MustHaveSyscallEvents skips the current test if the current +// platform does not support true system call events. +func Dump(t *testing.T, testName string, traceBytes []byte, forceToFile bool) { + onBuilder := testenv.Builder() != "" + onOldBuilder := !strings.Contains(testenv.Builder(), "gotip") && !strings.Contains(testenv.Builder(), "go1") + + if onBuilder && !forceToFile { + // Dump directly to the test log on the builder, since this + // data is critical for debugging and this is the only way + // we can currently make sure it's retained. + s := dumpTraceToText(t, traceBytes) + if onOldBuilder && len(s) > 1<<20+512<<10 { + // The old build infrastructure truncates logs at ~2 MiB. + // Let's assume we're the only failure and give ourselves + // up to 1.5 MiB to dump the trace. + // + // TODO(mknyszek): Remove this when we've migrated off of + // the old infrastructure. + t.Logf("text trace too large to dump (%d bytes)", len(s)) + } else { + t.Log(s) + } + } else { + // We asked to dump the trace or failed. Write the trace to a file. + t.Logf("wrote trace to file: %s", dumpTraceToFile(t, testName, traceBytes)) + } +} + +func dumpTraceToText(t *testing.T, b []byte) string { + t.Helper() + + br, err := raw.NewReader(bytes.NewReader(b)) + if err != nil { + t.Fatalf("dumping trace: %v", err) + } + var sb strings.Builder + tw, err := raw.NewTextWriter(&sb, version.Current) + if err != nil { + t.Fatalf("dumping trace: %v", err) + } + for { + ev, err := br.ReadEvent() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("dumping trace: %v", err) + } + if err := tw.WriteEvent(ev); err != nil { + t.Fatalf("dumping trace: %v", err) + } + } + return sb.String() +} + +func dumpTraceToFile(t *testing.T, testName string, b []byte) string { + t.Helper() + + name := fmt.Sprintf("%s.trace.", testName) + f, err := os.CreateTemp(t.ArtifactDir(), name) + if err != nil { + t.Fatalf("creating temp file: %v", err) + } + defer f.Close() + if _, err := io.Copy(f, bytes.NewReader(b)); err != nil { + t.Fatalf("writing trace dump to %q: %v", f.Name(), err) + } + return f.Name() +} diff --git a/src/internal/trace/testtrace/platforms.go b/src/internal/trace/testtrace/platforms.go new file mode 100644 index 00000000000000..937a2dcc841903 --- /dev/null +++ b/src/internal/trace/testtrace/platforms.go @@ -0,0 +1,32 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testtrace + +import ( + "runtime" + "testing" +) + +// MustHaveSyscallEvents skips the current test if the current +// platform does not support true system call events. +func MustHaveSyscallEvents(t *testing.T) { + if HasSyscallEvents() { + return + } + t.Skip("current platform has no true syscall events") +} + +// HasSyscallEvents returns true if the current platform +// has real syscall events available. +func HasSyscallEvents() bool { + switch runtime.GOOS { + case "js", "wasip1": + // js and wasip1 emulate system calls by blocking on channels + // while yielding back to the environment. They never actually + // call entersyscall/exitsyscall. + return false + } + return true +} diff --git a/src/internal/trace/testtrace/validation.go b/src/internal/trace/testtrace/validation.go index ec492110e2a29d..5edcf3a5b2dc0a 100644 --- a/src/internal/trace/testtrace/validation.go +++ b/src/internal/trace/testtrace/validation.go @@ -8,20 +8,24 @@ import ( "errors" "fmt" "internal/trace" + "internal/trace/version" "slices" "strings" ) // Validator is a type used for validating a stream of trace.Events. type Validator struct { - lastTs trace.Time - gs map[trace.GoID]*goState - ps map[trace.ProcID]*procState - ms map[trace.ThreadID]*schedContext - ranges map[trace.ResourceID][]string - tasks map[trace.TaskID]string - seenSync bool - Go121 bool + lastTs trace.Time + gs map[trace.GoID]*goState + ps map[trace.ProcID]*procState + ms map[trace.ThreadID]*schedContext + ranges map[trace.ResourceID][]string + tasks map[trace.TaskID]string + lastSync trace.Sync + GoVersion version.Version + + // Flags to modify validation behavior. + skipClockSnapshotChecks bool // Some platforms can't guarantee a monotonically increasing clock reading. } type schedContext struct { @@ -43,14 +47,23 @@ type procState struct { // NewValidator creates a new Validator. func NewValidator() *Validator { return &Validator{ - gs: make(map[trace.GoID]*goState), - ps: make(map[trace.ProcID]*procState), - ms: make(map[trace.ThreadID]*schedContext), - ranges: make(map[trace.ResourceID][]string), - tasks: make(map[trace.TaskID]string), + gs: make(map[trace.GoID]*goState), + ps: make(map[trace.ProcID]*procState), + ms: make(map[trace.ThreadID]*schedContext), + ranges: make(map[trace.ResourceID][]string), + tasks: make(map[trace.TaskID]string), + GoVersion: version.Current, } } +// SkipClockSnapshotChecks causes the validator to skip checks on the clock snapshots. +// +// Some platforms like Windows, with a small enough trace period, are unable to produce +// monotonically increasing timestamps due to very coarse clock granularity. +func (v *Validator) SkipClockSnapshotChecks() { + v.skipClockSnapshotChecks = true +} + // Event validates ev as the next event in a stream of trace.Events. // // Returns an error if validation fails. @@ -60,7 +73,7 @@ func (v *Validator) Event(ev trace.Event) error { // Validate timestamp order. if v.lastTs != 0 { if ev.Time() <= v.lastTs { - e.Errorf("timestamp out-of-order for %+v", ev) + e.Errorf("timestamp out-of-order (want > %v) for %+v", v.lastTs, ev) } else { v.lastTs = ev.Time() } @@ -73,8 +86,42 @@ func (v *Validator) Event(ev trace.Event) error { switch ev.Kind() { case trace.EventSync: - // Just record that we've seen a Sync at some point. - v.seenSync = true + s := ev.Sync() + if s.N != v.lastSync.N+1 { + e.Errorf("sync count is not sequential: expected %d, got %d", v.lastSync.N+1, s.N) + } + // The trace reader currently emits synthetic sync events at the end of + // a trace. Those don't contain clock snapshots data, so we don't try + // to validate them. + // + // TODO(felixge): Drop the synthetic syncs as discussed in CL 653576. + if v.GoVersion >= version.Go125 && !(s.N > 1 && s.ClockSnapshot == nil) { + if s.ClockSnapshot == nil { + e.Errorf("sync %d has no clock snapshot", s.N) + } else { + if s.ClockSnapshot.Wall.IsZero() { + e.Errorf("sync %d has zero wall time", s.N) + } + if s.ClockSnapshot.Mono == 0 { + e.Errorf("sync %d has zero mono time", s.N) + } + if s.ClockSnapshot.Trace == 0 { + e.Errorf("sync %d has zero trace time", s.N) + } + if !v.skipClockSnapshotChecks { + if s.N >= 2 && !s.ClockSnapshot.Wall.After(v.lastSync.ClockSnapshot.Wall) { + e.Errorf("sync %d has non-increasing wall time: %v vs %v", s.N, s.ClockSnapshot.Wall, v.lastSync.ClockSnapshot.Wall) + } + if s.N >= 2 && !(s.ClockSnapshot.Mono > v.lastSync.ClockSnapshot.Mono) { + e.Errorf("sync %d has non-increasing mono time: %v vs %v", s.N, s.ClockSnapshot.Mono, v.lastSync.ClockSnapshot.Mono) + } + if s.N >= 2 && !(s.ClockSnapshot.Trace > v.lastSync.ClockSnapshot.Trace) { + e.Errorf("sync %d has non-increasing trace time: %v vs %v", s.N, s.ClockSnapshot.Trace, v.lastSync.ClockSnapshot.Trace) + } + } + } + } + v.lastSync = s case trace.EventMetric: m := ev.Metric() if !strings.Contains(m.Name, ":") { @@ -140,7 +187,7 @@ func (v *Validator) Event(ev trace.Event) error { if new == trace.GoUndetermined { e.Errorf("transition to undetermined state for goroutine %d", id) } - if v.seenSync && old == trace.GoUndetermined { + if v.lastSync.N > 1 && old == trace.GoUndetermined { e.Errorf("undetermined goroutine %d after first global sync", id) } if new == trace.GoNotExist && v.hasAnyRange(trace.MakeResourceID(id)) { @@ -193,7 +240,7 @@ func (v *Validator) Event(ev trace.Event) error { if new == trace.ProcUndetermined { e.Errorf("transition to undetermined state for proc %d", id) } - if v.seenSync && old == trace.ProcUndetermined { + if v.lastSync.N > 1 && old == trace.ProcUndetermined { e.Errorf("undetermined proc %d after first global sync", id) } if new == trace.ProcNotExist && v.hasAnyRange(trace.MakeResourceID(id)) { @@ -322,7 +369,7 @@ func (v *Validator) getOrCreateThread(e *errAccumulator, ev trace.Event, m trace // Be lenient about GoUndetermined -> GoSyscall transitions if they // originate from an old trace. These transitions lack thread // information in trace formats older than 1.22. - if !v.Go121 { + if v.GoVersion >= version.Go122 { return false } if ev.Kind() != trace.EventStateTransition { @@ -350,20 +397,17 @@ func (v *Validator) getOrCreateThread(e *errAccumulator, ev trace.Event, m trace func checkStack(e *errAccumulator, stk trace.Stack) { // Check for non-empty values, but we also check for crashes due to incorrect validation. - i := 0 - stk.Frames(func(f trace.StackFrame) bool { + for i, f := range slices.Collect(stk.Frames()) { if i == 0 { // Allow for one fully zero stack. // // TODO(mknyszek): Investigate why that happens. - return true + continue } if f.Func == "" || f.File == "" || f.PC == 0 || f.Line == 0 { e.Errorf("invalid stack frame %#v: missing information", f) } - i++ - return true - }) + } } type errAccumulator struct { diff --git a/src/internal/trace/trace_test.go b/src/internal/trace/trace_test.go index 1929069cc5da97..479411548f22b2 100644 --- a/src/internal/trace/trace_test.go +++ b/src/internal/trace/trace_test.go @@ -12,16 +12,18 @@ import ( "internal/testenv" "internal/trace" "internal/trace/testtrace" + "internal/trace/version" "io" "os" "path/filepath" "runtime" + "slices" "strings" "testing" ) func TestTraceAnnotations(t *testing.T) { - testTraceProg(t, "annotations.go", func(t *testing.T, tb, _ []byte, _ bool) { + testTraceProg(t, "annotations.go", func(t *testing.T, tb, _ []byte, _ string) { type evDesc struct { kind trace.EventKind task trace.TaskID @@ -96,7 +98,7 @@ func TestTraceCgoCallback(t *testing.T) { } func TestTraceCPUProfile(t *testing.T) { - testTraceProg(t, "cpu-profile.go", func(t *testing.T, tb, stderr []byte, _ bool) { + testTraceProg(t, "cpu-profile.go", func(t *testing.T, tb, stderr []byte, _ string) { // Parse stderr which has a CPU profile summary, if everything went well. // (If it didn't, we shouldn't even make it here.) scanner := bufio.NewScanner(bytes.NewReader(stderr)) @@ -148,12 +150,11 @@ func TestTraceCPUProfile(t *testing.T) { if hogRegion != nil && ev.Goroutine() == hogRegion.Goroutine() { traceSamples++ var fns []string - ev.Stack().Frames(func(frame trace.StackFrame) bool { + for frame := range ev.Stack().Frames() { if frame.Func != "runtime.goexit" { fns = append(fns, fmt.Sprintf("%s:%d", frame.Func, frame.Line)) } - return true - }) + } stack := strings.Join(fns, "|") traceStacks[stack]++ } @@ -210,7 +211,7 @@ func TestTraceCPUProfile(t *testing.T) { } func TestTraceFutileWakeup(t *testing.T) { - testTraceProg(t, "futile-wakeup.go", func(t *testing.T, tb, _ []byte, _ bool) { + testTraceProg(t, "futile-wakeup.go", func(t *testing.T, tb, _ []byte, _ string) { // Check to make sure that no goroutine in the "special" trace region // ends up blocking, unblocking, then immediately blocking again. // @@ -311,7 +312,7 @@ func TestTraceGOMAXPROCS(t *testing.T) { } func TestTraceStacks(t *testing.T) { - testTraceProg(t, "stacks.go", func(t *testing.T, tb, _ []byte, stress bool) { + testTraceProg(t, "stacks.go", func(t *testing.T, tb, _ []byte, godebug string) { type frame struct { fn string line int @@ -325,7 +326,8 @@ func TestTraceStacks(t *testing.T) { const mainLine = 21 want := []evDesc{ {trace.EventStateTransition, "Goroutine Running->Runnable", []frame{ - {"main.main", mainLine + 82}, + {"runtime.Gosched", 0}, + {"main.main", mainLine + 87}, }}, {trace.EventStateTransition, "Goroutine NotExist->Runnable", []frame{ {"main.main", mainLine + 11}, @@ -348,7 +350,7 @@ func TestTraceStacks(t *testing.T) { }}, {trace.EventStateTransition, "Goroutine Waiting->Runnable", []frame{ {"runtime.chansend1", 0}, - {"main.main", mainLine + 84}, + {"main.main", mainLine + 89}, }}, {trace.EventStateTransition, "Goroutine Running->Waiting", []frame{ {"runtime.chansend1", 0}, @@ -356,7 +358,7 @@ func TestTraceStacks(t *testing.T) { }}, {trace.EventStateTransition, "Goroutine Waiting->Runnable", []frame{ {"runtime.chanrecv1", 0}, - {"main.main", mainLine + 85}, + {"main.main", mainLine + 90}, }}, {trace.EventStateTransition, "Goroutine Running->Waiting", []frame{ {"runtime.selectgo", 0}, @@ -364,7 +366,7 @@ func TestTraceStacks(t *testing.T) { }}, {trace.EventStateTransition, "Goroutine Waiting->Runnable", []frame{ {"runtime.selectgo", 0}, - {"main.main", mainLine + 86}, + {"main.main", mainLine + 91}, }}, {trace.EventStateTransition, "Goroutine Running->Waiting", []frame{ {"sync.(*Mutex).Lock", 0}, @@ -381,7 +383,7 @@ func TestTraceStacks(t *testing.T) { {trace.EventStateTransition, "Goroutine Waiting->Runnable", []frame{ {"sync.(*WaitGroup).Add", 0}, {"sync.(*WaitGroup).Done", 0}, - {"main.main", mainLine + 91}, + {"main.main", mainLine + 96}, }}, {trace.EventStateTransition, "Goroutine Running->Waiting", []frame{ {"sync.(*Cond).Wait", 0}, @@ -402,6 +404,18 @@ func TestTraceStacks(t *testing.T) { {"main.main", 0}, }}, } + asyncPreemptOff := strings.Contains(godebug, "asyncpreemptoff=1") + if asyncPreemptOff { + // Only check for syncPreemptPoint if asynchronous preemption is disabled. + // Otherwise, the syncPreemptPoint goroutine might be exclusively async + // preempted, causing this test to flake. + syncPreemptEv := evDesc{trace.EventStateTransition, "Goroutine Running->Runnable", []frame{ + {"main.syncPreemptPoint", 0}, + {"main.main.func12", 0}, + }} + want = append(want, syncPreemptEv) + } + stress := strings.Contains(godebug, "traceadvanceperiod=0") if !stress { // Only check for this stack if !stress because traceAdvance alone could // allocate enough memory to trigger a GC if called frequently enough. @@ -434,23 +448,20 @@ func TestTraceStacks(t *testing.T) { {"main.main.func11", 0}, }}, }...) + if runtime.GOOS == "darwin" { + want[len(want)-1].frames = append([]frame{{"syscall.syscall", 0}}, want[len(want)-1].frames...) + } } stackMatches := func(stk trace.Stack, frames []frame) bool { - i := 0 - match := true - stk.Frames(func(f trace.StackFrame) bool { + for i, f := range slices.Collect(stk.Frames()) { if f.Func != frames[i].fn { - match = false return false } if line := uint64(frames[i].line); line != 0 && line != f.Line { - match = false return false } - i++ - return true - }) - return match + } + return true } r, err := trace.NewReader(bytes.NewReader(tb)) if err != nil { @@ -535,7 +546,7 @@ func TestTraceIterPull(t *testing.T) { testTraceProg(t, "iter-pull.go", nil) } -func checkReaderDeterminism(t *testing.T, tb, _ []byte, _ bool) { +func checkReaderDeterminism(t *testing.T, tb, _ []byte, _ string) { events := func() []trace.Event { var evs []trace.Event @@ -572,23 +583,44 @@ func checkReaderDeterminism(t *testing.T, tb, _ []byte, _ bool) { } } -func testTraceProg(t *testing.T, progName string, extra func(t *testing.T, trace, stderr []byte, stress bool)) { +func testTraceProg(t *testing.T, progName string, extra func(t *testing.T, trace, stderr []byte, godebug string)) { testenv.MustHaveGoRun(t) // Check if we're on a builder. onBuilder := testenv.Builder() != "" onOldBuilder := !strings.Contains(testenv.Builder(), "gotip") && !strings.Contains(testenv.Builder(), "go1") + if progName == "cgo-callback.go" && onBuilder && !onOldBuilder && + runtime.GOOS == "freebsd" && runtime.GOARCH == "amd64" && race.Enabled { + t.Skip("test fails on freebsd-amd64-race in LUCI; see go.dev/issue/71556") + } + testPath := filepath.Join("./testdata/testprog", progName) testName := progName runTest := func(t *testing.T, stress bool, extraGODEBUG string) { - // Run the program and capture the trace, which is always written to stdout. - cmd := testenv.Command(t, testenv.GoToolPath(t), "run") + // Build the program. + binFile, err := os.CreateTemp("", progName) + if err != nil { + t.Fatalf("failed to create temporary output file: %v", err) + } + bin := binFile.Name() + binFile.Close() + t.Cleanup(func() { + os.Remove(bin) + }) + buildCmd := testenv.CommandContext(t, t.Context(), testenv.GoToolPath(t), "build", "-o", bin) if race.Enabled { - cmd.Args = append(cmd.Args, "-race") + buildCmd.Args = append(buildCmd.Args, "-race") } - cmd.Args = append(cmd.Args, testPath) - cmd.Env = append(os.Environ(), "GOEXPERIMENT=rangefunc") + buildCmd.Args = append(buildCmd.Args, testPath) + buildOutput, err := buildCmd.CombinedOutput() + if err != nil { + t.Fatalf("failed to build %s: %v: output:\n%s", testPath, err, buildOutput) + } + + // Run the program and capture the trace, which is always written to stdout. + cmd := testenv.CommandContext(t, t.Context(), bin) + // Add a stack ownership check. This is cheap enough for testing. godebug := "tracecheckstackownership=1" if stress { @@ -600,6 +632,10 @@ func testTraceProg(t *testing.T, progName string, extra func(t *testing.T, trace godebug += "," + extraGODEBUG } cmd.Env = append(cmd.Env, "GODEBUG="+godebug) + if _, ok := os.LookupEnv("GOTRACEBACK"); !ok { + // Unless overriden, set GOTRACEBACK=crash. + cmd.Env = append(cmd.Env, "GOTRACEBACK=crash") + } // Capture stdout and stderr. // @@ -618,39 +654,42 @@ func testTraceProg(t *testing.T, progName string, extra func(t *testing.T, trace tb := traceBuf.Bytes() // Test the trace and the parser. - testReader(t, bytes.NewReader(tb), testtrace.ExpectSuccess()) + v := testtrace.NewValidator() + v.GoVersion = version.Current + if runtime.GOOS == "windows" && stress { + // Under stress mode we're constantly advancing trace generations. + // Windows' clock granularity is too coarse to guarantee monotonic + // timestamps for monotonic and wall clock time in this case, so + // skip the checks. + v.SkipClockSnapshotChecks() + } + testReader(t, bytes.NewReader(tb), v, testtrace.ExpectSuccess()) // Run some extra validation. if !t.Failed() && extra != nil { - extra(t, tb, errBuf.Bytes(), stress) + extra(t, tb, errBuf.Bytes(), godebug) } // Dump some more information on failure. - if t.Failed() && onBuilder { - // Dump directly to the test log on the builder, since this - // data is critical for debugging and this is the only way - // we can currently make sure it's retained. - t.Log("found bad trace; dumping to test log...") - s := dumpTraceToText(t, tb) - if onOldBuilder && len(s) > 1<<20+512<<10 { - // The old build infrastructure truncates logs at ~2 MiB. - // Let's assume we're the only failure and give ourselves - // up to 1.5 MiB to dump the trace. - // - // TODO(mknyszek): Remove this when we've migrated off of - // the old infrastructure. - t.Logf("text trace too large to dump (%d bytes)", len(s)) - } else { - t.Log(s) - } - } else if t.Failed() || *dumpTraces { - // We asked to dump the trace or failed. Write the trace to a file. - t.Logf("wrote trace to file: %s", dumpTraceToFile(t, testName, stress, tb)) + if t.Failed() || *dumpTraces { + suffix := func(stress bool) string { + if stress { + return "stress" + } + return "default" + } + testtrace.Dump(t, fmt.Sprintf("%s.%s", testName, suffix(stress)), tb, *dumpTraces) } } t.Run("Default", func(t *testing.T) { runTest(t, false, "") }) + t.Run("AsyncPreemptOff", func(t *testing.T) { + if testing.Short() && runtime.NumCPU() < 2 { + t.Skip("skipping trace async preempt off tests in short mode") + } + runTest(t, false, "asyncpreemptoff=1") + }) t.Run("Stress", func(t *testing.T) { if testing.Short() { t.Skip("skipping trace stress tests in short mode") diff --git a/src/internal/trace/tracev1.go b/src/internal/trace/tracev1.go new file mode 100644 index 00000000000000..667d7be1cd1b4b --- /dev/null +++ b/src/internal/trace/tracev1.go @@ -0,0 +1,567 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements conversion from v1 (Go 1.11–Go 1.21) traces to the v2 +// format (Go 1.22+). +// +// Most events have direct equivalents in v2, at worst requiring arguments to +// be reordered. Some events, such as GoWaiting need to look ahead for follow-up +// events to determine the correct translation. GoSyscall, which is an +// instantaneous event, gets turned into a 1 ns long pair of +// GoSyscallStart+GoSyscallEnd, unless we observe a GoSysBlock, in which case we +// emit a GoSyscallStart+GoSyscallEndBlocked pair with the correct duration +// (i.e. starting at the original GoSyscall). +// +// The resulting trace treats the trace v1 as a single, large generation, +// sharing a single evTable for all events. +// +// We use a new (compared to what was used for 'go tool trace' in earlier +// versions of Go) parser for v1 traces that is optimized for speed, low memory +// usage, and minimal GC pressure. It allocates events in batches so that even +// though we have to load the entire trace into memory, the conversion process +// shouldn't result in a doubling of memory usage, even if all converted events +// are kept alive, as we free batches once we're done with them. +// +// The conversion process is lossless. + +package trace + +import ( + "errors" + "fmt" + "internal/trace/internal/tracev1" + "internal/trace/tracev2" + "io" +) + +type traceV1Converter struct { + trace tracev1.Trace + evt *evTable + preInit bool + createdPreInit map[GoID]struct{} + events tracev1.Events + extra []Event + extraArr [3]Event + tasks map[TaskID]taskState + seenProcs map[ProcID]struct{} + lastTs Time + procMs map[ProcID]ThreadID + lastStwReason uint64 + + inlineToStringID []uint64 + builtinToStringID []uint64 +} + +const ( + // Block reasons + sForever = iota + sPreempted + sGosched + sSleep + sChanSend + sChanRecv + sNetwork + sSync + sSyncCond + sSelect + sEmpty + sMarkAssistWait + + // STW kinds + sSTWUnknown + sSTWGCMarkTermination + sSTWGCSweepTermination + sSTWWriteHeapDump + sSTWGoroutineProfile + sSTWGoroutineProfileCleanup + sSTWAllGoroutinesStackTrace + sSTWReadMemStats + sSTWAllThreadsSyscall + sSTWGOMAXPROCS + sSTWStartTrace + sSTWStopTrace + sSTWCountPagesInUse + sSTWReadMetricsSlow + sSTWReadMemStatsSlow + sSTWPageCachePagesLeaked + sSTWResetDebugLog + + sLast +) + +func (it *traceV1Converter) init(pr tracev1.Trace) error { + it.trace = pr + it.preInit = true + it.createdPreInit = make(map[GoID]struct{}) + it.evt = &evTable{pcs: make(map[uint64]frame)} + it.events = pr.Events + it.extra = it.extraArr[:0] + it.tasks = make(map[TaskID]taskState) + it.seenProcs = make(map[ProcID]struct{}) + it.procMs = make(map[ProcID]ThreadID) + it.lastTs = -1 + + evt := it.evt + + // Convert from trace v1's Strings map to our dataTable. + var max uint64 + for id, s := range pr.Strings { + evt.strings.insert(stringID(id), s) + if id > max { + max = id + } + } + pr.Strings = nil + + // Add all strings used for UserLog. In the trace v1 format, these were + // stored inline and didn't have IDs. We generate IDs for them. + if max+uint64(len(pr.InlineStrings)) < max { + return errors.New("trace contains too many strings") + } + var addErr error + add := func(id stringID, s string) { + if err := evt.strings.insert(id, s); err != nil && addErr == nil { + addErr = err + } + } + for id, s := range pr.InlineStrings { + nid := max + 1 + uint64(id) + it.inlineToStringID = append(it.inlineToStringID, nid) + add(stringID(nid), s) + } + max += uint64(len(pr.InlineStrings)) + pr.InlineStrings = nil + + // Add strings that the converter emits explicitly. + if max+uint64(sLast) < max { + return errors.New("trace contains too many strings") + } + it.builtinToStringID = make([]uint64, sLast) + addBuiltin := func(c int, s string) { + nid := max + 1 + uint64(c) + it.builtinToStringID[c] = nid + add(stringID(nid), s) + } + addBuiltin(sForever, "forever") + addBuiltin(sPreempted, "preempted") + addBuiltin(sGosched, "runtime.Gosched") + addBuiltin(sSleep, "sleep") + addBuiltin(sChanSend, "chan send") + addBuiltin(sChanRecv, "chan receive") + addBuiltin(sNetwork, "network") + addBuiltin(sSync, "sync") + addBuiltin(sSyncCond, "sync.(*Cond).Wait") + addBuiltin(sSelect, "select") + addBuiltin(sEmpty, "") + addBuiltin(sMarkAssistWait, "GC mark assist wait for work") + addBuiltin(sSTWUnknown, "") + addBuiltin(sSTWGCMarkTermination, "GC mark termination") + addBuiltin(sSTWGCSweepTermination, "GC sweep termination") + addBuiltin(sSTWWriteHeapDump, "write heap dump") + addBuiltin(sSTWGoroutineProfile, "goroutine profile") + addBuiltin(sSTWGoroutineProfileCleanup, "goroutine profile cleanup") + addBuiltin(sSTWAllGoroutinesStackTrace, "all goroutine stack trace") + addBuiltin(sSTWReadMemStats, "read mem stats") + addBuiltin(sSTWAllThreadsSyscall, "AllThreadsSyscall") + addBuiltin(sSTWGOMAXPROCS, "GOMAXPROCS") + addBuiltin(sSTWStartTrace, "start trace") + addBuiltin(sSTWStopTrace, "stop trace") + addBuiltin(sSTWCountPagesInUse, "CountPagesInUse (test)") + addBuiltin(sSTWReadMetricsSlow, "ReadMetricsSlow (test)") + addBuiltin(sSTWReadMemStatsSlow, "ReadMemStatsSlow (test)") + addBuiltin(sSTWPageCachePagesLeaked, "PageCachePagesLeaked (test)") + addBuiltin(sSTWResetDebugLog, "ResetDebugLog (test)") + + if addErr != nil { + // This should be impossible but let's be safe. + return fmt.Errorf("couldn't add strings: %w", addErr) + } + + it.evt.strings.compactify() + + // Convert stacks. + for id, stk := range pr.Stacks { + evt.stacks.insert(stackID(id), stack{pcs: stk}) + } + + // OPT(dh): if we could share the frame type between this package and + // tracev1 we wouldn't have to copy the map. + for pc, f := range pr.PCs { + evt.pcs[pc] = frame{ + pc: pc, + funcID: stringID(f.Fn), + fileID: stringID(f.File), + line: uint64(f.Line), + } + } + pr.Stacks = nil + pr.PCs = nil + evt.stacks.compactify() + return nil +} + +// next returns the next event, io.EOF if there are no more events, or a +// descriptive error for invalid events. +func (it *traceV1Converter) next() (Event, error) { + if len(it.extra) > 0 { + ev := it.extra[0] + it.extra = it.extra[1:] + + if len(it.extra) == 0 { + it.extra = it.extraArr[:0] + } + // Two events aren't allowed to fall on the same timestamp in the new API, + // but this may happen when we produce EvGoStatus events + if ev.base.time <= it.lastTs { + ev.base.time = it.lastTs + 1 + } + it.lastTs = ev.base.time + return ev, nil + } + + oev, ok := it.events.Pop() + if !ok { + return Event{}, io.EOF + } + + ev, err := it.convertEvent(oev) + + if err == errSkip { + return it.next() + } else if err != nil { + return Event{}, err + } + + // Two events aren't allowed to fall on the same timestamp in the new API, + // but this may happen when we produce EvGoStatus events + if ev.base.time <= it.lastTs { + ev.base.time = it.lastTs + 1 + } + it.lastTs = ev.base.time + return ev, nil +} + +var errSkip = errors.New("skip event") + +// convertEvent converts an event from the trace v1 format to zero or more +// events in the new format. Most events translate 1 to 1. Some events don't +// result in an event right away, in which case convertEvent returns errSkip. +// Some events result in more than one new event; in this case, convertEvent +// returns the first event and stores additional events in it.extra. When +// encountering events that tracev1 shouldn't be able to emit, ocnvertEvent +// returns a descriptive error. +func (it *traceV1Converter) convertEvent(ev *tracev1.Event) (OUT Event, ERR error) { + var mappedType tracev2.EventType + var mappedArgs timedEventArgs + copy(mappedArgs[:], ev.Args[:]) + + switch ev.Type { + case tracev1.EvGomaxprocs: + mappedType = tracev2.EvProcsChange + if it.preInit { + // The first EvGomaxprocs signals the end of trace initialization. At this point we've seen + // all goroutines that already existed at trace begin. + it.preInit = false + for gid := range it.createdPreInit { + // These are goroutines that already existed when tracing started but for which we + // received neither GoWaiting, GoInSyscall, or GoStart. These are goroutines that are in + // the states _Gidle or _Grunnable. + it.extra = append(it.extra, Event{ + ctx: schedCtx{ + // G: GoID(gid), + G: NoGoroutine, + P: NoProc, + M: NoThread, + }, + table: it.evt, + base: baseEvent{ + typ: tracev2.EvGoStatus, + time: Time(ev.Ts), + args: timedEventArgs{uint64(gid), ^uint64(0), uint64(tracev2.GoRunnable)}, + }, + }) + } + it.createdPreInit = nil + return Event{}, errSkip + } + case tracev1.EvProcStart: + it.procMs[ProcID(ev.P)] = ThreadID(ev.Args[0]) + if _, ok := it.seenProcs[ProcID(ev.P)]; ok { + mappedType = tracev2.EvProcStart + mappedArgs = timedEventArgs{uint64(ev.P)} + } else { + it.seenProcs[ProcID(ev.P)] = struct{}{} + mappedType = tracev2.EvProcStatus + mappedArgs = timedEventArgs{uint64(ev.P), uint64(tracev2.ProcRunning)} + } + case tracev1.EvProcStop: + if _, ok := it.seenProcs[ProcID(ev.P)]; ok { + mappedType = tracev2.EvProcStop + mappedArgs = timedEventArgs{uint64(ev.P)} + } else { + it.seenProcs[ProcID(ev.P)] = struct{}{} + mappedType = tracev2.EvProcStatus + mappedArgs = timedEventArgs{uint64(ev.P), uint64(tracev2.ProcIdle)} + } + case tracev1.EvGCStart: + mappedType = tracev2.EvGCBegin + case tracev1.EvGCDone: + mappedType = tracev2.EvGCEnd + case tracev1.EvSTWStart: + sid := it.builtinToStringID[sSTWUnknown+it.trace.STWReason(ev.Args[0])] + it.lastStwReason = sid + mappedType = tracev2.EvSTWBegin + mappedArgs = timedEventArgs{uint64(sid)} + case tracev1.EvSTWDone: + mappedType = tracev2.EvSTWEnd + mappedArgs = timedEventArgs{it.lastStwReason} + case tracev1.EvGCSweepStart: + mappedType = tracev2.EvGCSweepBegin + case tracev1.EvGCSweepDone: + mappedType = tracev2.EvGCSweepEnd + case tracev1.EvGoCreate: + if it.preInit { + it.createdPreInit[GoID(ev.Args[0])] = struct{}{} + return Event{}, errSkip + } + mappedType = tracev2.EvGoCreate + case tracev1.EvGoStart: + if it.preInit { + mappedType = tracev2.EvGoStatus + mappedArgs = timedEventArgs{ev.Args[0], ^uint64(0), uint64(tracev2.GoRunning)} + delete(it.createdPreInit, GoID(ev.Args[0])) + } else { + mappedType = tracev2.EvGoStart + } + case tracev1.EvGoStartLabel: + it.extra = []Event{{ + ctx: schedCtx{ + G: GoID(ev.G), + P: ProcID(ev.P), + M: it.procMs[ProcID(ev.P)], + }, + table: it.evt, + base: baseEvent{ + typ: tracev2.EvGoLabel, + time: Time(ev.Ts), + args: timedEventArgs{ev.Args[2]}, + }, + }} + return Event{ + ctx: schedCtx{ + G: GoID(ev.G), + P: ProcID(ev.P), + M: it.procMs[ProcID(ev.P)], + }, + table: it.evt, + base: baseEvent{ + typ: tracev2.EvGoStart, + time: Time(ev.Ts), + args: mappedArgs, + }, + }, nil + case tracev1.EvGoEnd: + mappedType = tracev2.EvGoDestroy + case tracev1.EvGoStop: + mappedType = tracev2.EvGoBlock + mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sForever]), uint64(ev.StkID)} + case tracev1.EvGoSched: + mappedType = tracev2.EvGoStop + mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sGosched]), uint64(ev.StkID)} + case tracev1.EvGoPreempt: + mappedType = tracev2.EvGoStop + mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sPreempted]), uint64(ev.StkID)} + case tracev1.EvGoSleep: + mappedType = tracev2.EvGoBlock + mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sSleep]), uint64(ev.StkID)} + case tracev1.EvGoBlock: + mappedType = tracev2.EvGoBlock + mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sEmpty]), uint64(ev.StkID)} + case tracev1.EvGoUnblock: + mappedType = tracev2.EvGoUnblock + case tracev1.EvGoBlockSend: + mappedType = tracev2.EvGoBlock + mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sChanSend]), uint64(ev.StkID)} + case tracev1.EvGoBlockRecv: + mappedType = tracev2.EvGoBlock + mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sChanRecv]), uint64(ev.StkID)} + case tracev1.EvGoBlockSelect: + mappedType = tracev2.EvGoBlock + mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sSelect]), uint64(ev.StkID)} + case tracev1.EvGoBlockSync: + mappedType = tracev2.EvGoBlock + mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sSync]), uint64(ev.StkID)} + case tracev1.EvGoBlockCond: + mappedType = tracev2.EvGoBlock + mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sSyncCond]), uint64(ev.StkID)} + case tracev1.EvGoBlockNet: + mappedType = tracev2.EvGoBlock + mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sNetwork]), uint64(ev.StkID)} + case tracev1.EvGoBlockGC: + mappedType = tracev2.EvGoBlock + mappedArgs = timedEventArgs{uint64(it.builtinToStringID[sMarkAssistWait]), uint64(ev.StkID)} + case tracev1.EvGoSysCall: + // Look for the next event for the same G to determine if the syscall + // blocked. + blocked := false + it.events.All()(func(nev *tracev1.Event) bool { + if nev.G != ev.G { + return true + } + // After an EvGoSysCall, the next event on the same G will either be + // EvGoSysBlock to denote a blocking syscall, or some other event + // (or the end of the trace) if the syscall didn't block. + if nev.Type == tracev1.EvGoSysBlock { + blocked = true + } + return false + }) + if blocked { + mappedType = tracev2.EvGoSyscallBegin + mappedArgs = timedEventArgs{1: uint64(ev.StkID)} + } else { + // Convert the old instantaneous syscall event to a pair of syscall + // begin and syscall end and give it the shortest possible duration, + // 1ns. + out1 := Event{ + ctx: schedCtx{ + G: GoID(ev.G), + P: ProcID(ev.P), + M: it.procMs[ProcID(ev.P)], + }, + table: it.evt, + base: baseEvent{ + typ: tracev2.EvGoSyscallBegin, + time: Time(ev.Ts), + args: timedEventArgs{1: uint64(ev.StkID)}, + }, + } + + out2 := Event{ + ctx: out1.ctx, + table: it.evt, + base: baseEvent{ + typ: tracev2.EvGoSyscallEnd, + time: Time(ev.Ts + 1), + args: timedEventArgs{}, + }, + } + + it.extra = append(it.extra, out2) + return out1, nil + } + + case tracev1.EvGoSysExit: + mappedType = tracev2.EvGoSyscallEndBlocked + case tracev1.EvGoSysBlock: + return Event{}, errSkip + case tracev1.EvGoWaiting: + mappedType = tracev2.EvGoStatus + mappedArgs = timedEventArgs{ev.Args[0], ^uint64(0), uint64(tracev2.GoWaiting)} + delete(it.createdPreInit, GoID(ev.Args[0])) + case tracev1.EvGoInSyscall: + mappedType = tracev2.EvGoStatus + // In the new tracer, GoStatus with GoSyscall knows what thread the + // syscall is on. In trace v1, EvGoInSyscall doesn't contain that + // information and all we can do here is specify NoThread. + mappedArgs = timedEventArgs{ev.Args[0], ^uint64(0), uint64(tracev2.GoSyscall)} + delete(it.createdPreInit, GoID(ev.Args[0])) + case tracev1.EvHeapAlloc: + mappedType = tracev2.EvHeapAlloc + case tracev1.EvHeapGoal: + mappedType = tracev2.EvHeapGoal + case tracev1.EvGCMarkAssistStart: + mappedType = tracev2.EvGCMarkAssistBegin + case tracev1.EvGCMarkAssistDone: + mappedType = tracev2.EvGCMarkAssistEnd + case tracev1.EvUserTaskCreate: + mappedType = tracev2.EvUserTaskBegin + parent := ev.Args[1] + if parent == 0 { + parent = uint64(NoTask) + } + mappedArgs = timedEventArgs{ev.Args[0], parent, ev.Args[2], uint64(ev.StkID)} + name, _ := it.evt.strings.get(stringID(ev.Args[2])) + it.tasks[TaskID(ev.Args[0])] = taskState{name: name, parentID: TaskID(ev.Args[1])} + case tracev1.EvUserTaskEnd: + mappedType = tracev2.EvUserTaskEnd + // Event.Task expects the parent and name to be smuggled in extra args + // and as extra strings. + ts, ok := it.tasks[TaskID(ev.Args[0])] + if ok { + delete(it.tasks, TaskID(ev.Args[0])) + mappedArgs = timedEventArgs{ + ev.Args[0], + ev.Args[1], + uint64(ts.parentID), + uint64(it.evt.addExtraString(ts.name)), + } + } else { + mappedArgs = timedEventArgs{ev.Args[0], ev.Args[1], uint64(NoTask), uint64(it.evt.addExtraString(""))} + } + case tracev1.EvUserRegion: + switch ev.Args[1] { + case 0: // start + mappedType = tracev2.EvUserRegionBegin + case 1: // end + mappedType = tracev2.EvUserRegionEnd + } + mappedArgs = timedEventArgs{ev.Args[0], ev.Args[2], uint64(ev.StkID)} + case tracev1.EvUserLog: + mappedType = tracev2.EvUserLog + mappedArgs = timedEventArgs{ev.Args[0], ev.Args[1], it.inlineToStringID[ev.Args[3]], uint64(ev.StkID)} + case tracev1.EvCPUSample: + mappedType = tracev2.EvCPUSample + // When emitted by the Go 1.22 tracer, CPU samples have 5 arguments: + // timestamp, M, P, G, stack. However, after they get turned into Event, + // they have the arguments stack, M, P, G. + // + // In Go 1.21, CPU samples did not have Ms. + mappedArgs = timedEventArgs{uint64(ev.StkID), ^uint64(0), uint64(ev.P), ev.G} + default: + return Event{}, fmt.Errorf("unexpected event type %v", ev.Type) + } + + if tracev1.EventDescriptions[ev.Type].Stack { + if stackIDs := tracev2.Specs()[mappedType].StackIDs; len(stackIDs) > 0 { + mappedArgs[stackIDs[0]-1] = uint64(ev.StkID) + } + } + + m := NoThread + if ev.P != -1 && ev.Type != tracev1.EvCPUSample { + if t, ok := it.procMs[ProcID(ev.P)]; ok { + m = ThreadID(t) + } + } + if ev.Type == tracev1.EvProcStop { + delete(it.procMs, ProcID(ev.P)) + } + g := GoID(ev.G) + if g == 0 { + g = NoGoroutine + } + out := Event{ + ctx: schedCtx{ + G: GoID(g), + P: ProcID(ev.P), + M: m, + }, + table: it.evt, + base: baseEvent{ + typ: mappedType, + time: Time(ev.Ts), + args: mappedArgs, + }, + } + return out, nil +} + +// convertV1Trace takes a fully loaded trace in the v1 trace format and +// returns an iterator over events in the new format. +func convertV1Trace(pr tracev1.Trace) *traceV1Converter { + it := &traceV1Converter{} + it.init(pr) + return it +} diff --git a/src/internal/trace/tracev1_test.go b/src/internal/trace/tracev1_test.go new file mode 100644 index 00000000000000..355a6ff529f2ae --- /dev/null +++ b/src/internal/trace/tracev1_test.go @@ -0,0 +1,89 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package trace_test + +import ( + "internal/trace" + "internal/trace/testtrace" + "internal/trace/version" + "io" + "os" + "path/filepath" + "testing" +) + +func TestTraceV1(t *testing.T) { + traces, err := filepath.Glob("./internal/tracev1/testdata/*_good") + if err != nil { + t.Fatalf("failed to glob for tests: %s", err) + } + var testedUserRegions bool + for _, p := range traces { + testName, err := filepath.Rel("./internal/tracev1/testdata", p) + if err != nil { + t.Fatalf("failed to relativize testdata path: %s", err) + } + t.Run(testName, func(t *testing.T) { + f, err := os.Open(p) + if err != nil { + t.Fatalf("failed to open test %q: %s", p, err) + } + defer f.Close() + + tr, err := trace.NewReader(f) + if err != nil { + t.Fatalf("failed to create reader: %s", err) + } + + v := testtrace.NewValidator() + v.GoVersion = version.Go121 + for { + ev, err := tr.ReadEvent() + if err != nil { + if err == io.EOF { + break + } + t.Fatalf("couldn't read converted event: %s", err) + } + if err := v.Event(ev); err != nil { + t.Fatalf("converted event did not validate; event: \n%s\nerror: %s", ev, err) + } + + if testName == "user_task_region_1_21_good" { + testedUserRegions = true + validRegions := map[string]struct{}{ + "post-existing region": {}, + "region0": {}, + "region1": {}, + } + // Check that we correctly convert user regions. These + // strings were generated by + // runtime/trace.TestUserTaskRegion, which is the basis for + // the user_task_region_* test cases. We only check for the + // Go 1.21 traces because earlier traces used different + // strings. + switch ev.Kind() { + case trace.EventRegionBegin, trace.EventRegionEnd: + if _, ok := validRegions[ev.Region().Type]; !ok { + t.Fatalf("converted event has unexpected region type:\n%s", ev) + } + case trace.EventTaskBegin, trace.EventTaskEnd: + if ev.Task().Type != "task0" { + t.Fatalf("converted event has unexpected task type name:\n%s", ev) + } + case trace.EventLog: + l := ev.Log() + if l.Task != 1 || l.Category != "key0" || l.Message != "0123456789abcdef" { + t.Fatalf("converted event has unexpected user log:\n%s", ev) + } + } + } + } + }) + } + if !testedUserRegions { + t.Fatal("didn't see expected test case user_task_region_1_21_good") + } +} diff --git a/src/internal/trace/tracev2/EXPERIMENTS.md b/src/internal/trace/tracev2/EXPERIMENTS.md new file mode 100644 index 00000000000000..fedf7bdbdc8db7 --- /dev/null +++ b/src/internal/trace/tracev2/EXPERIMENTS.md @@ -0,0 +1,101 @@ +# Trace experiments + +Execution traces allow for trialing new events on an experimental basis via +trace experiments. +This document is a guide that explains how you can define your own trace +experiments. + +Note that if you're just trying to do some debugging or perform some light +instrumentation, then a trace experiment is way overkill. +Use `runtime/trace.Log` instead. +Even if you're just trying to create a proof-of-concept for a low-frequency +event, `runtime/trace.Log` will probably be easier overall if you can make +it work. + +Consider a trace experiment if: +- The volume of new trace events will be relatively high, and so the events + would benefit from a more compact representation (creating new tables to + deduplicate data, taking advantage of the varint representation, etc.). +- It's not safe to call `runtime/trace.Log` (or its runtime equivalent) in + the contexts you want to generate an event (for example, for events about + timers). + +## Defining a new experiment + +To define a new experiment, modify `internal/trace/tracev2` to define a +new `Experiment` enum value. + +An experiment consists of two parts: timed events and experimental batches. +Timed events are events like any other and follow the same format. +They are easier to order and require less work to make use of. +Experimental batches are essentially bags of bytes that correspond to +an entire trace generation. +What they contain and how they're interpreted is totally up to you, but +they're most often useful for tables that your other events can refer into. +For example, the AllocFree experiment uses them to store type information +that allocation events can refer to. + +### Defining new events + +1. Define your new experiment event types (by convention, experimental events + types start at ID 127, so look for the `const` block defining events + starting there). +2. Describe your new events in `specs`. + Use the documentation for `Spec` to write your new specs, and check your + work by running the tests in the `internal/trace/tracev2` package. + If you wish for your event argument to be interpreted in a particular + way, follow the naming convention in + `src/internal/trace/tracev2/spec.go`. + For example, if you intend to emit a string argument, make sure the + argument name has the suffix `string`. +3. Add ordering and validation logic for your new events to + `src/internal/trace/order.go` by listing handlers for those events in + the `orderingDispatch` table. + If your events are always emitted in a regular user goroutine context, + then the handler should be trivial and just validate the scheduling + context to match userGoReqs. + If it's more complicated, see `(*ordering).advanceAllocFree` for a + slightly more complicated example that handles events from a larger + variety of execution environments. + If you need to encode a partial ordering, look toward the scheduler + events (names beginning with `Go`) or just ask someone for help. +4. Add your new events to the `tracev2Type2Kind` table in + `src/internal/trace/event.go`. + +## Emitting data + +### Emitting your new events + +1. Define helper methods on `runtime.traceEventWriter` for emitting your + events. +2. Instrument the runtime with calls to these helper methods. + Make sure to call `traceAcquire` and `traceRelease` around the operation + your event represents, otherwise it will not be emitted atomically with + that operation completing, resulting in a potentially misleading trace. + +### Emitting experimental batches + +To emit experimental batches, use the `runtime.unsafeTraceExpWriter` to +write experimental batches associated with your experiment. +Heed the warnings and make sure that while you write them, the trace +generation cannot advance. +Note that each experiment can only have one distinguishable set of +batches. + +## Recovering experimental data + +### Recovering experimental events from the trace + +Experimental events will appear in the event stream as an event with the +`EventExperimental` `Kind`. +Use the `Experimental` method to collect the raw data inserted into the +trace. +It's essentially up to you to interpret the event from here. +I recommend writing a thin wrapper API to present a cleaner interface if you +so desire. + +### Recovering experimental batches + +Parse out all the experimental batches from `Sync` events as they come. +These experimental batches are all for the same generation as all the +experimental events up until the next `Sync` event. diff --git a/src/internal/trace/tracev2/doc.go b/src/internal/trace/tracev2/doc.go new file mode 100644 index 00000000000000..725fc0821d0fd7 --- /dev/null +++ b/src/internal/trace/tracev2/doc.go @@ -0,0 +1,11 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package tracev2 contains definitions for the v2 execution trace wire format. + +These definitions are shared between the trace parser and the runtime, so it +must not depend on any package that depends on the runtime (most packages). +*/ +package tracev2 diff --git a/src/internal/trace/tracev2/events.go b/src/internal/trace/tracev2/events.go new file mode 100644 index 00000000000000..c5e3c94136946b --- /dev/null +++ b/src/internal/trace/tracev2/events.go @@ -0,0 +1,589 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tracev2 + +// Event types in the trace, args are given in square brackets. +// +// Naming scheme: +// - Time range event pairs have suffixes "Begin" and "End". +// - "Start", "Stop", "Create", "Destroy", "Block", "Unblock" +// are suffixes reserved for scheduling resources. +// +// NOTE: If you add an event type, make sure you also update all +// tables in this file! +const ( + EvNone EventType = iota // unused + + // Structural events. + EvEventBatch // start of per-M batch of events [generation, M ID, timestamp, batch length] + EvStacks // start of a section of the stack table [...EvStack] + EvStack // stack table entry [ID, ...{PC, func string ID, file string ID, line #}] + EvStrings // start of a section of the string dictionary [...EvString] + EvString // string dictionary entry [ID, length, string] + EvCPUSamples // start of a section of CPU samples [...EvCPUSample] + EvCPUSample // CPU profiling sample [timestamp, M ID, P ID, goroutine ID, stack ID] + EvFrequency // timestamp units per sec [freq] + + // Procs. + EvProcsChange // current value of GOMAXPROCS [timestamp, GOMAXPROCS, stack ID] + EvProcStart // start of P [timestamp, P ID, P seq] + EvProcStop // stop of P [timestamp] + EvProcSteal // P was stolen [timestamp, P ID, P seq, M ID] + EvProcStatus // P status at the start of a generation [timestamp, P ID, status] + + // Goroutines. + EvGoCreate // goroutine creation [timestamp, new goroutine ID, new stack ID, stack ID] + EvGoCreateSyscall // goroutine appears in syscall (cgo callback) [timestamp, new goroutine ID] + EvGoStart // goroutine starts running [timestamp, goroutine ID, goroutine seq] + EvGoDestroy // goroutine ends [timestamp] + EvGoDestroySyscall // goroutine ends in syscall (cgo callback) [timestamp] + EvGoStop // goroutine yields its time, but is runnable [timestamp, reason, stack ID] + EvGoBlock // goroutine blocks [timestamp, reason, stack ID] + EvGoUnblock // goroutine is unblocked [timestamp, goroutine ID, goroutine seq, stack ID] + EvGoSyscallBegin // syscall enter [timestamp, P seq, stack ID] + EvGoSyscallEnd // syscall exit [timestamp] + EvGoSyscallEndBlocked // syscall exit and it blocked at some point [timestamp] + EvGoStatus // goroutine status at the start of a generation [timestamp, goroutine ID, M ID, status] + + // STW. + EvSTWBegin // STW start [timestamp, kind, stack ID] + EvSTWEnd // STW done [timestamp] + + // GC events. + EvGCActive // GC active [timestamp, seq] + EvGCBegin // GC start [timestamp, seq, stack ID] + EvGCEnd // GC done [timestamp, seq] + EvGCSweepActive // GC sweep active [timestamp, P ID] + EvGCSweepBegin // GC sweep start [timestamp, stack ID] + EvGCSweepEnd // GC sweep done [timestamp, swept bytes, reclaimed bytes] + EvGCMarkAssistActive // GC mark assist active [timestamp, goroutine ID] + EvGCMarkAssistBegin // GC mark assist start [timestamp, stack ID] + EvGCMarkAssistEnd // GC mark assist done [timestamp] + EvHeapAlloc // gcController.heapLive change [timestamp, heap alloc in bytes] + EvHeapGoal // gcController.heapGoal() change [timestamp, heap goal in bytes] + + // Annotations. + EvGoLabel // apply string label to current running goroutine [timestamp, label string ID] + EvUserTaskBegin // trace.NewTask [timestamp, internal task ID, internal parent task ID, name string ID, stack ID] + EvUserTaskEnd // end of a task [timestamp, internal task ID, stack ID] + EvUserRegionBegin // trace.{Start,With}Region [timestamp, internal task ID, name string ID, stack ID] + EvUserRegionEnd // trace.{End,With}Region [timestamp, internal task ID, name string ID, stack ID] + EvUserLog // trace.Log [timestamp, internal task ID, key string ID, value string ID, stack] + + // Coroutines. Added in Go 1.23. + EvGoSwitch // goroutine switch (coroswitch) [timestamp, goroutine ID, goroutine seq] + EvGoSwitchDestroy // goroutine switch and destroy [timestamp, goroutine ID, goroutine seq] + EvGoCreateBlocked // goroutine creation (starts blocked) [timestamp, new goroutine ID, new stack ID, stack ID] + + // GoStatus with stack. Added in Go 1.23. + EvGoStatusStack // goroutine status at the start of a generation, with a stack [timestamp, goroutine ID, M ID, status, stack ID] + + // Batch event for an experimental batch with a custom format. Added in Go 1.23. + EvExperimentalBatch // start of extra data [experiment ID, generation, M ID, timestamp, batch length, batch data...] + + // Sync batch. Added in Go 1.25. Previously a lone EvFrequency event. + EvSync // start of a sync batch [...EvFrequency|EvClockSnapshot] + EvClockSnapshot // snapshot of trace, mono and wall clocks [timestamp, mono, sec, nsec] + + // In-band end-of-generation signal. Added in Go 1.26. + // Used in Go 1.25 only internally. + EvEndOfGeneration + + NumEvents +) + +func (ev EventType) Experimental() bool { + return ev > MaxEvent && ev < MaxExperimentalEvent +} + +// Experiments. +const ( + // AllocFree is the alloc-free events experiment. + AllocFree Experiment = 1 + iota + + NumExperiments +) + +func Experiments() []string { + return experiments[:] +} + +var experiments = [...]string{ + NoExperiment: "None", + AllocFree: "AllocFree", +} + +// Experimental events. +const ( + MaxEvent EventType = 127 + iota + + // Experimental events for AllocFree. + + // Experimental heap span events. Added in Go 1.23. + EvSpan // heap span exists [timestamp, id, npages, type/class] + EvSpanAlloc // heap span alloc [timestamp, id, npages, type/class] + EvSpanFree // heap span free [timestamp, id] + + // Experimental heap object events. Added in Go 1.23. + EvHeapObject // heap object exists [timestamp, id, type] + EvHeapObjectAlloc // heap object alloc [timestamp, id, type] + EvHeapObjectFree // heap object free [timestamp, id] + + // Experimental goroutine stack events. Added in Go 1.23. + EvGoroutineStack // stack exists [timestamp, id, order] + EvGoroutineStackAlloc // stack alloc [timestamp, id, order] + EvGoroutineStackFree // stack free [timestamp, id] + + MaxExperimentalEvent +) + +const NumExperimentalEvents = MaxExperimentalEvent - MaxEvent + +// MaxTimedEventArgs is the maximum number of arguments for timed events. +const MaxTimedEventArgs = 5 + +func Specs() []EventSpec { + return specs[:] +} + +var specs = [...]EventSpec{ + // "Structural" Events. + EvEventBatch: { + Name: "EventBatch", + Args: []string{"gen", "m", "time", "size"}, + }, + EvStacks: { + Name: "Stacks", + }, + EvStack: { + Name: "Stack", + Args: []string{"id", "nframes"}, + IsStack: true, + }, + EvStrings: { + Name: "Strings", + }, + EvString: { + Name: "String", + Args: []string{"id"}, + HasData: true, + }, + EvCPUSamples: { + Name: "CPUSamples", + }, + EvCPUSample: { + Name: "CPUSample", + Args: []string{"time", "m", "p", "g", "stack"}, + // N.B. There's clearly a timestamp here, but these Events + // are special in that they don't appear in the regular + // M streams. + StackIDs: []int{4}, + }, + EvFrequency: { + Name: "Frequency", + Args: []string{"freq"}, + }, + EvExperimentalBatch: { + Name: "ExperimentalBatch", + Args: []string{"exp", "gen", "m", "time"}, + HasData: true, // Easier to represent for raw readers. + }, + EvSync: { + Name: "Sync", + }, + EvEndOfGeneration: { + Name: "EndOfGeneration", + }, + + // "Timed" Events. + EvProcsChange: { + Name: "ProcsChange", + Args: []string{"dt", "procs_value", "stack"}, + IsTimedEvent: true, + StackIDs: []int{2}, + }, + EvProcStart: { + Name: "ProcStart", + Args: []string{"dt", "p", "p_seq"}, + IsTimedEvent: true, + }, + EvProcStop: { + Name: "ProcStop", + Args: []string{"dt"}, + IsTimedEvent: true, + }, + EvProcSteal: { + Name: "ProcSteal", + Args: []string{"dt", "p", "p_seq", "m"}, + IsTimedEvent: true, + }, + EvProcStatus: { + Name: "ProcStatus", + Args: []string{"dt", "p", "pstatus"}, + IsTimedEvent: true, + }, + EvGoCreate: { + Name: "GoCreate", + Args: []string{"dt", "new_g", "new_stack", "stack"}, + IsTimedEvent: true, + StackIDs: []int{3, 2}, + }, + EvGoCreateSyscall: { + Name: "GoCreateSyscall", + Args: []string{"dt", "new_g"}, + IsTimedEvent: true, + }, + EvGoStart: { + Name: "GoStart", + Args: []string{"dt", "g", "g_seq"}, + IsTimedEvent: true, + }, + EvGoDestroy: { + Name: "GoDestroy", + Args: []string{"dt"}, + IsTimedEvent: true, + }, + EvGoDestroySyscall: { + Name: "GoDestroySyscall", + Args: []string{"dt"}, + IsTimedEvent: true, + }, + EvGoStop: { + Name: "GoStop", + Args: []string{"dt", "reason_string", "stack"}, + IsTimedEvent: true, + StackIDs: []int{2}, + StringIDs: []int{1}, + }, + EvGoBlock: { + Name: "GoBlock", + Args: []string{"dt", "reason_string", "stack"}, + IsTimedEvent: true, + StackIDs: []int{2}, + StringIDs: []int{1}, + }, + EvGoUnblock: { + Name: "GoUnblock", + Args: []string{"dt", "g", "g_seq", "stack"}, + IsTimedEvent: true, + StackIDs: []int{3}, + }, + EvGoSyscallBegin: { + Name: "GoSyscallBegin", + Args: []string{"dt", "p_seq", "stack"}, + IsTimedEvent: true, + StackIDs: []int{2}, + }, + EvGoSyscallEnd: { + Name: "GoSyscallEnd", + Args: []string{"dt"}, + StartEv: EvGoSyscallBegin, + IsTimedEvent: true, + }, + EvGoSyscallEndBlocked: { + Name: "GoSyscallEndBlocked", + Args: []string{"dt"}, + StartEv: EvGoSyscallBegin, + IsTimedEvent: true, + }, + EvGoStatus: { + Name: "GoStatus", + Args: []string{"dt", "g", "m", "gstatus"}, + IsTimedEvent: true, + }, + EvSTWBegin: { + Name: "STWBegin", + Args: []string{"dt", "kind_string", "stack"}, + IsTimedEvent: true, + StackIDs: []int{2}, + StringIDs: []int{1}, + }, + EvSTWEnd: { + Name: "STWEnd", + Args: []string{"dt"}, + StartEv: EvSTWBegin, + IsTimedEvent: true, + }, + EvGCActive: { + Name: "GCActive", + Args: []string{"dt", "gc_seq"}, + IsTimedEvent: true, + StartEv: EvGCBegin, + }, + EvGCBegin: { + Name: "GCBegin", + Args: []string{"dt", "gc_seq", "stack"}, + IsTimedEvent: true, + StackIDs: []int{2}, + }, + EvGCEnd: { + Name: "GCEnd", + Args: []string{"dt", "gc_seq"}, + StartEv: EvGCBegin, + IsTimedEvent: true, + }, + EvGCSweepActive: { + Name: "GCSweepActive", + Args: []string{"dt", "p"}, + StartEv: EvGCSweepBegin, + IsTimedEvent: true, + }, + EvGCSweepBegin: { + Name: "GCSweepBegin", + Args: []string{"dt", "stack"}, + IsTimedEvent: true, + StackIDs: []int{1}, + }, + EvGCSweepEnd: { + Name: "GCSweepEnd", + Args: []string{"dt", "swept_value", "reclaimed_value"}, + StartEv: EvGCSweepBegin, + IsTimedEvent: true, + }, + EvGCMarkAssistActive: { + Name: "GCMarkAssistActive", + Args: []string{"dt", "g"}, + StartEv: EvGCMarkAssistBegin, + IsTimedEvent: true, + }, + EvGCMarkAssistBegin: { + Name: "GCMarkAssistBegin", + Args: []string{"dt", "stack"}, + IsTimedEvent: true, + StackIDs: []int{1}, + }, + EvGCMarkAssistEnd: { + Name: "GCMarkAssistEnd", + Args: []string{"dt"}, + StartEv: EvGCMarkAssistBegin, + IsTimedEvent: true, + }, + EvHeapAlloc: { + Name: "HeapAlloc", + Args: []string{"dt", "heapalloc_value"}, + IsTimedEvent: true, + }, + EvHeapGoal: { + Name: "HeapGoal", + Args: []string{"dt", "heapgoal_value"}, + IsTimedEvent: true, + }, + EvGoLabel: { + Name: "GoLabel", + Args: []string{"dt", "label_string"}, + IsTimedEvent: true, + StringIDs: []int{1}, + }, + EvUserTaskBegin: { + Name: "UserTaskBegin", + Args: []string{"dt", "task", "parent_task", "name_string", "stack"}, + IsTimedEvent: true, + StackIDs: []int{4}, + StringIDs: []int{3}, + }, + EvUserTaskEnd: { + Name: "UserTaskEnd", + Args: []string{"dt", "task", "stack"}, + IsTimedEvent: true, + StackIDs: []int{2}, + }, + EvUserRegionBegin: { + Name: "UserRegionBegin", + Args: []string{"dt", "task", "name_string", "stack"}, + IsTimedEvent: true, + StackIDs: []int{3}, + StringIDs: []int{2}, + }, + EvUserRegionEnd: { + Name: "UserRegionEnd", + Args: []string{"dt", "task", "name_string", "stack"}, + StartEv: EvUserRegionBegin, + IsTimedEvent: true, + StackIDs: []int{3}, + StringIDs: []int{2}, + }, + EvUserLog: { + Name: "UserLog", + Args: []string{"dt", "task", "key_string", "value_string", "stack"}, + IsTimedEvent: true, + StackIDs: []int{4}, + StringIDs: []int{2, 3}, + }, + EvGoSwitch: { + Name: "GoSwitch", + Args: []string{"dt", "g", "g_seq"}, + IsTimedEvent: true, + }, + EvGoSwitchDestroy: { + Name: "GoSwitchDestroy", + Args: []string{"dt", "g", "g_seq"}, + IsTimedEvent: true, + }, + EvGoCreateBlocked: { + Name: "GoCreateBlocked", + Args: []string{"dt", "new_g", "new_stack", "stack"}, + IsTimedEvent: true, + StackIDs: []int{3, 2}, + }, + EvGoStatusStack: { + Name: "GoStatusStack", + Args: []string{"dt", "g", "m", "gstatus", "stack"}, + IsTimedEvent: true, + StackIDs: []int{4}, + }, + EvClockSnapshot: { + Name: "ClockSnapshot", + Args: []string{"dt", "mono", "sec", "nsec"}, + IsTimedEvent: true, + }, + + // Experimental events. + + EvSpan: { + Name: "Span", + Args: []string{"dt", "id", "npages_value", "kindclass"}, + IsTimedEvent: true, + Experiment: AllocFree, + }, + EvSpanAlloc: { + Name: "SpanAlloc", + Args: []string{"dt", "id", "npages_value", "kindclass"}, + IsTimedEvent: true, + Experiment: AllocFree, + }, + EvSpanFree: { + Name: "SpanFree", + Args: []string{"dt", "id"}, + IsTimedEvent: true, + Experiment: AllocFree, + }, + EvHeapObject: { + Name: "HeapObject", + Args: []string{"dt", "id", "type"}, + IsTimedEvent: true, + Experiment: AllocFree, + }, + EvHeapObjectAlloc: { + Name: "HeapObjectAlloc", + Args: []string{"dt", "id", "type"}, + IsTimedEvent: true, + Experiment: AllocFree, + }, + EvHeapObjectFree: { + Name: "HeapObjectFree", + Args: []string{"dt", "id"}, + IsTimedEvent: true, + Experiment: AllocFree, + }, + EvGoroutineStack: { + Name: "GoroutineStack", + Args: []string{"dt", "id", "order"}, + IsTimedEvent: true, + Experiment: AllocFree, + }, + EvGoroutineStackAlloc: { + Name: "GoroutineStackAlloc", + Args: []string{"dt", "id", "order"}, + IsTimedEvent: true, + Experiment: AllocFree, + }, + EvGoroutineStackFree: { + Name: "GoroutineStackFree", + Args: []string{"dt", "id"}, + IsTimedEvent: true, + Experiment: AllocFree, + }, +} + +// GoStatus is the status of a goroutine. +// +// They correspond directly to the various goroutine states. +type GoStatus uint8 + +const ( + GoBad GoStatus = iota + GoRunnable + GoRunning + GoSyscall + GoWaiting +) + +func (s GoStatus) String() string { + switch s { + case GoRunnable: + return "Runnable" + case GoRunning: + return "Running" + case GoSyscall: + return "Syscall" + case GoWaiting: + return "Waiting" + } + return "Bad" +} + +// ProcStatus is the status of a P. +// +// They mostly correspond to the various P states. +type ProcStatus uint8 + +const ( + ProcBad ProcStatus = iota + ProcRunning + ProcIdle + ProcSyscall + + // ProcSyscallAbandoned is a special case of + // ProcSyscall. It's used in the very specific case + // where the first a P is mentioned in a generation is + // part of a ProcSteal event. If that's the first time + // it's mentioned, then there's no GoSyscallBegin to + // connect the P stealing back to at that point. This + // special state indicates this to the parser, so it + // doesn't try to find a GoSyscallEndBlocked that + // corresponds with the ProcSteal. + ProcSyscallAbandoned +) + +func (s ProcStatus) String() string { + switch s { + case ProcRunning: + return "Running" + case ProcIdle: + return "Idle" + case ProcSyscall: + return "Syscall" + } + return "Bad" +} + +const ( + // MaxBatchSize sets the maximum size that a batch can be. + // + // Directly controls the trace batch size in the runtime. + // + // NOTE: If this number decreases, the trace format version must change. + MaxBatchSize = 64 << 10 + + // Maximum number of PCs in a single stack trace. + // + // Since events contain only stack ID rather than whole stack trace, + // we can allow quite large values here. + // + // Directly controls the maximum number of frames per stack + // in the runtime. + // + // NOTE: If this number decreases, the trace format version must change. + MaxFramesPerStack = 128 + + // MaxEventTrailerDataSize controls the amount of trailer data that + // an event can have in bytes. Must be smaller than MaxBatchSize. + // Controls the maximum string size in the trace. + // + // Directly controls the maximum such value in the runtime. + // + // NOTE: If this number decreases, the trace format version must change. + MaxEventTrailerDataSize = 1 << 10 +) diff --git a/src/internal/trace/tracev2/events_test.go b/src/internal/trace/tracev2/events_test.go new file mode 100644 index 00000000000000..60c4c08c34a127 --- /dev/null +++ b/src/internal/trace/tracev2/events_test.go @@ -0,0 +1,101 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tracev2_test + +import ( + "internal/trace/tracev2" + "iter" + "regexp" + "slices" + "strings" + "testing" +) + +var argNameRegexp = regexp.MustCompile(`((?P[A-Za-z]+)_)?(?P[A-Za-z]+)`) + +func TestSpecs(t *testing.T) { + if tracev2.NumEvents <= 0 { + t.Fatalf("no trace events?") + } + if tracev2.MaxExperimentalEvent < tracev2.MaxEvent { + t.Fatalf("max experimental event (%d) is < max event (%d)", tracev2.MaxExperimentalEvent, tracev2.MaxEvent) + } + specs := tracev2.Specs() + for ev := range allEvents() { + spec := &specs[ev] + if spec.Name == "" { + t.Errorf("expected event %d to be defined in specs", ev) + continue + } + if spec.IsTimedEvent && spec.Args[0] != "dt" { + t.Errorf("%s is a timed event, but its first argument is not 'dt'", spec.Name) + } + if spec.HasData && spec.Name != "String" && spec.Name != "ExperimentalBatch" { + t.Errorf("%s has data, but is not a special kind of event (unsupported, but could be)", spec.Name) + } + if spec.IsStack && spec.Name != "Stack" { + t.Errorf("%s listed as being a stack, but is not the Stack event (unsupported)", spec.Name) + } + if spec.IsTimedEvent && len(spec.Args) > tracev2.MaxTimedEventArgs { + t.Errorf("%s has too many timed event args: have %d, want %d at most", spec.Name, len(spec.Args), tracev2.MaxTimedEventArgs) + } + if ev.Experimental() && spec.Experiment == tracev2.NoExperiment { + t.Errorf("experimental event %s must have an experiment", spec.Name) + } + + // Check arg types. + for _, arg := range spec.Args { + matches := argNameRegexp.FindStringSubmatch(arg) + if len(matches) == 0 { + t.Errorf("malformed argument %s for event %s", arg, spec.Name) + } + } + + // Check stacks. + for _, i := range spec.StackIDs { + if !strings.HasSuffix(spec.Args[i], "stack") { + t.Errorf("stack argument listed at %d in %s, but argument name %s does not imply stack type", i, spec.Name, spec.Args[i]) + } + } + for i, arg := range spec.Args { + if !strings.HasSuffix(spec.Args[i], "stack") { + continue + } + if !slices.Contains(spec.StackIDs, i) { + t.Errorf("found stack argument %s in %s at index %d not listed in StackIDs", arg, spec.Name, i) + } + } + + // Check strings. + for _, i := range spec.StringIDs { + if !strings.HasSuffix(spec.Args[i], "string") { + t.Errorf("string argument listed at %d in %s, but argument name %s does not imply string type", i, spec.Name, spec.Args[i]) + } + } + for i, arg := range spec.Args { + if !strings.HasSuffix(spec.Args[i], "string") { + continue + } + if !slices.Contains(spec.StringIDs, i) { + t.Errorf("found string argument %s in %s at index %d not listed in StringIDs", arg, spec.Name, i) + } + } + } +} + +func allEvents() iter.Seq[tracev2.EventType] { + return func(yield func(tracev2.EventType) bool) { + for ev := tracev2.EvNone + 1; ev < tracev2.NumEvents; ev++ { + if !yield(ev) { + return + } + } + for ev := tracev2.MaxEvent + 1; ev < tracev2.NumExperimentalEvents; ev++ { + if !yield(ev) { + return + } + } + } +} diff --git a/src/internal/trace/tracev2/spec.go b/src/internal/trace/tracev2/spec.go new file mode 100644 index 00000000000000..6e54c399f49ddf --- /dev/null +++ b/src/internal/trace/tracev2/spec.go @@ -0,0 +1,107 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tracev2 + +// EventType indicates an event's type from which its arguments and semantics can be +// derived. Its representation matches the wire format's representation of the event +// types that precede all event data. +type EventType uint8 + +// EventSpec is a specification for a trace event. It contains sufficient information +// to perform basic parsing of any trace event for any version of Go. +type EventSpec struct { + // Name is the human-readable name of the trace event. + Name string + + // Args contains the names of each trace event's argument. + // Its length determines the number of arguments an event has. + // + // Argument names follow a certain structure and this structure + // is relied on by the testing framework to type-check arguments + // and to produce Values for experimental events. + // + // The structure is: + // + // (?P[A-Za-z]+)(_(?P[A-Za-z]+))? + // + // In sum, it's a name followed by an optional type. + // If the type is present, it is preceded with an underscore. + // Arguments without types will be interpreted as just raw uint64s. + // The valid argument types and the Go types they map to are listed + // in the ArgTypes variable. + Args []string + + // StringIDs indicates which of the arguments are string IDs. + StringIDs []int + + // StackIDs indicates which of the arguments are stack IDs. + // + // The list is not sorted. The first index always refers to + // the main stack for the current execution context of the event. + StackIDs []int + + // StartEv indicates the event type of the corresponding "start" + // event, if this event is an "end," for a pair of events that + // represent a time range. + StartEv EventType + + // IsTimedEvent indicates whether this is an event that both + // appears in the main event stream and is surfaced to the + // trace reader. + // + // Events that are not "timed" are considered "structural" + // since they either need significant reinterpretation or + // otherwise aren't actually surfaced by the trace reader. + IsTimedEvent bool + + // HasData is true if the event has trailer consisting of a + // varint length followed by unencoded bytes of some data. + // + // An event may not be both a timed event and have data. + HasData bool + + // IsStack indicates that the event represents a complete + // stack trace. Specifically, it means that after the arguments + // there's a varint length, followed by 4*length varints. Each + // group of 4 represents the PC, file ID, func ID, and line number + // in that order. + IsStack bool + + // Experiment indicates the ID of an experiment this event is associated + // with. If Experiment is not NoExperiment, then the event is experimental + // and will be exposed as an EventExperiment. + Experiment Experiment +} + +// EventArgTypes is a list of valid argument types for use in Args. +// +// See the documentation of Args for more details. +var EventArgTypes = [...]string{ + "seq", // sequence number + "pstatus", // P status + "gstatus", // G status + "g", // trace.GoID + "m", // trace.ThreadID + "p", // trace.ProcID + "string", // string ID + "stack", // stack ID + "value", // uint64 + "task", // trace.TaskID +} + +// EventNames is a helper that produces a mapping of event names to event types. +func EventNames(specs []EventSpec) map[string]EventType { + nameToType := make(map[string]EventType) + for i, spec := range specs { + nameToType[spec.Name] = EventType(byte(i)) + } + return nameToType +} + +// Experiment is an experiment ID that events may be associated with. +type Experiment uint + +// NoExperiment is the reserved ID 0 indicating no experiment. +const NoExperiment Experiment = 0 diff --git a/src/internal/trace/traceviewer/emitter.go b/src/internal/trace/traceviewer/emitter.go index c91c743a7bfb6c..9167ff81b45f83 100644 --- a/src/internal/trace/traceviewer/emitter.go +++ b/src/internal/trace/traceviewer/emitter.go @@ -614,7 +614,7 @@ func (e *Emitter) tsWithinRange(ts time.Duration) bool { return e.rangeStart <= ts && ts <= e.rangeEnd } -// OptionalEvent emits ev if it's within the time range of of the consumer, i.e. +// OptionalEvent emits ev if it's within the time range of the consumer, i.e. // the selected trace split range. func (e *Emitter) OptionalEvent(ev *format.Event) { e.c.ConsumeViewerEvent(ev, false) @@ -632,10 +632,10 @@ func (e *Emitter) Flush() { e.processMeta(format.ProcsSection, e.resourceType, 2) - e.threadMeta(format.ProcsSection, trace.GCP, "GC", -6) - e.threadMeta(format.ProcsSection, trace.NetpollP, "Network", -5) - e.threadMeta(format.ProcsSection, trace.TimerP, "Timers", -4) - e.threadMeta(format.ProcsSection, trace.SyscallP, "Syscalls", -3) + e.threadMeta(format.ProcsSection, GCP, "GC", -6) + e.threadMeta(format.ProcsSection, NetpollP, "Network", -5) + e.threadMeta(format.ProcsSection, TimerP, "Timers", -4) + e.threadMeta(format.ProcsSection, SyscallP, "Syscalls", -3) for id, name := range e.resources { priority := int(id) @@ -683,12 +683,12 @@ func (e *Emitter) processMeta(sectionID uint64, name string, priority int) { // Stack emits the given frames and returns a unique id for the stack. No // pointers to the given data are being retained beyond the call to Stack. -func (e *Emitter) Stack(stk []*trace.Frame) int { +func (e *Emitter) Stack(stk []trace.StackFrame) int { return e.buildBranch(e.frameTree, stk) } // buildBranch builds one branch in the prefix tree rooted at ctx.frameTree. -func (e *Emitter) buildBranch(parent frameNode, stk []*trace.Frame) int { +func (e *Emitter) buildBranch(parent frameNode, stk []trace.StackFrame) int { if len(stk) == 0 { return parent.id } @@ -702,7 +702,7 @@ func (e *Emitter) buildBranch(parent frameNode, stk []*trace.Frame) int { node.id = e.frameSeq node.children = make(map[uint64]frameNode) parent.children[frame.PC] = node - e.c.ConsumeViewerFrame(strconv.Itoa(node.id), format.Frame{Name: fmt.Sprintf("%v:%v", frame.Fn, frame.Line), Parent: parent.id}) + e.c.ConsumeViewerFrame(strconv.Itoa(node.id), format.Frame{Name: fmt.Sprintf("%v:%v", frame.Func, frame.Line), Parent: parent.id}) } return e.buildBranch(node, stk) } diff --git a/src/internal/trace/traceviewer/fakep.go b/src/internal/trace/traceviewer/fakep.go new file mode 100644 index 00000000000000..655938b213bc36 --- /dev/null +++ b/src/internal/trace/traceviewer/fakep.go @@ -0,0 +1,15 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package traceviewer + +const ( + // Special P identifiers: + FakeP = 1000000 + iota + TimerP // depicts timer unblocks + NetpollP // depicts network unblocks + SyscallP // depicts returns from syscalls + GCP // depicts GC state + ProfileP // depicts recording of CPU profile samples +) diff --git a/src/internal/trace/traceviewer/mmu.go b/src/internal/trace/traceviewer/mmu.go index 0bc1233b44dee9..190ce5afcad6e1 100644 --- a/src/internal/trace/traceviewer/mmu.go +++ b/src/internal/trace/traceviewer/mmu.go @@ -69,7 +69,7 @@ var utilFlagNames = map[string]trace.UtilFlags{ func requestUtilFlags(r *http.Request) trace.UtilFlags { var flags trace.UtilFlags - for _, flagStr := range strings.Split(r.FormValue("flags"), "|") { + for flagStr := range strings.SplitSeq(r.FormValue("flags"), "|") { flags |= utilFlagNames[flagStr] } return flags @@ -119,7 +119,7 @@ func (m *mmu) HandlePlot(w http.ResponseWriter, r *http.Request) { } var quantiles []float64 - for _, flagStr := range strings.Split(r.FormValue("flags"), "|") { + for flagStr := range strings.SplitSeq(r.FormValue("flags"), "|") { if flagStr == "mut" { quantiles = []float64{0, 1 - .999, 1 - .99, 1 - .95} break diff --git a/src/internal/trace/traceviewer/pprof.go b/src/internal/trace/traceviewer/pprof.go index 1377b3c614c81c..141b2687b781ac 100644 --- a/src/internal/trace/traceviewer/pprof.go +++ b/src/internal/trace/traceviewer/pprof.go @@ -82,7 +82,7 @@ func SVGProfileHandlerFunc(f ProfileFunc) http.HandlerFunc { } type ProfileRecord struct { - Stack []*trace.Frame + Stack []trace.StackFrame Count uint64 Time time.Duration } @@ -103,16 +103,16 @@ func BuildProfile(prof []ProfileRecord) *profile.Profile { for _, frame := range rec.Stack { loc := locs[frame.PC] if loc == nil { - fn := funcs[frame.File+frame.Fn] + fn := funcs[frame.File+frame.Func] if fn == nil { fn = &profile.Function{ ID: uint64(len(p.Function) + 1), - Name: frame.Fn, - SystemName: frame.Fn, + Name: frame.Func, + SystemName: frame.Func, Filename: frame.File, } p.Function = append(p.Function, fn) - funcs[frame.File+frame.Fn] = fn + funcs[frame.File+frame.Func] = fn } loc = &profile.Location{ ID: uint64(len(p.Location) + 1), diff --git a/src/internal/trace/value.go b/src/internal/trace/value.go index bd2cba7878355d..fc2808e59753a8 100644 --- a/src/internal/trace/value.go +++ b/src/internal/trace/value.go @@ -4,12 +4,16 @@ package trace -import "fmt" +import ( + "fmt" + "unsafe" +) // Value is a dynamically-typed value obtained from a trace. type Value struct { - kind ValueKind - scalar uint64 + kind ValueKind + pointer unsafe.Pointer + scalar uint64 } // ValueKind is the type of a dynamically-typed value from a trace. @@ -18,6 +22,7 @@ type ValueKind uint8 const ( ValueBad ValueKind = iota ValueUint64 + ValueString ) // Kind returns the ValueKind of the value. @@ -30,9 +35,9 @@ func (v Value) Kind() ValueKind { return v.kind } -// Uint64 returns the uint64 value for a MetricSampleUint64. +// Uint64 returns the uint64 value for a ValueUint64. // -// Panics if this metric sample's Kind is not MetricSampleUint64. +// Panics if this Value's Kind is not ValueUint64. func (v Value) Uint64() uint64 { if v.kind != ValueUint64 { panic("Uint64 called on Value of a different Kind") @@ -40,14 +45,23 @@ func (v Value) Uint64() uint64 { return v.scalar } -// valueAsString produces a debug string value. -// -// This isn't just Value.String because we may want to use that to store -// string values in the future. -func valueAsString(v Value) string { - switch v.Kind() { +// String returns the string value for a ValueString, and otherwise +// a string representation of the value for other kinds of values. +func (v Value) String() string { + if v.kind == ValueString { + return unsafe.String((*byte)(v.pointer), int(v.scalar)) + } + switch v.kind { case ValueUint64: - return fmt.Sprintf("Uint64(%d)", v.scalar) + return fmt.Sprintf("Value{Uint64(%d)}", v.Uint64()) } - return "Bad" + return "Value{Bad}" +} + +func uint64Value(x uint64) Value { + return Value{kind: ValueUint64, scalar: x} +} + +func stringValue(s string) Value { + return Value{kind: ValueString, scalar: uint64(len(s)), pointer: unsafe.Pointer(unsafe.StringData(s))} } diff --git a/src/internal/trace/version/version.go b/src/internal/trace/version/version.go index 4951bd97d76645..328a261a93081f 100644 --- a/src/internal/trace/version/version.go +++ b/src/internal/trace/version/version.go @@ -8,41 +8,57 @@ import ( "fmt" "io" - "internal/trace/event" - "internal/trace/event/go122" + "internal/trace/tracev2" ) // Version represents the version of a trace file. type Version uint32 const ( - Go111 Version = 11 - Go119 Version = 19 - Go121 Version = 21 - Go122 Version = 22 - Go123 Version = 23 - Current = Go123 + Go111 Version = 11 // v1 + Go119 Version = 19 // v1 + Go121 Version = 21 // v1 + Go122 Version = 22 // v2 + Go123 Version = 23 // v2 + Go125 Version = 25 // v2 + Go126 Version = 26 // v2 + Current = Go126 ) -var versions = map[Version][]event.Spec{ +var versions = map[Version][]tracev2.EventSpec{ // Go 1.11–1.21 use a different parser and are only set here for the sake of // Version.Valid. Go111: nil, Go119: nil, Go121: nil, - Go122: go122.Specs(), - // Go 1.23 adds backwards-incompatible events, but - // traces produced by Go 1.22 are also always valid - // Go 1.23 traces. - Go123: go122.Specs(), + Go122: tracev2.Specs()[:tracev2.EvUserLog+1], // All events after are Go 1.23+. + Go123: tracev2.Specs()[:tracev2.EvExperimentalBatch+1], // All events after are Go 1.25+. + Go125: tracev2.Specs()[:tracev2.EvClockSnapshot+1], // All events after are Go 1.26+. + Go126: tracev2.Specs(), } // Specs returns the set of event.Specs for this version. -func (v Version) Specs() []event.Spec { +func (v Version) Specs() []tracev2.EventSpec { return versions[v] } +// EventName returns a string name of a wire format event +// for a particular trace version. +func (v Version) EventName(typ tracev2.EventType) string { + if !v.Valid() { + return "" + } + s := v.Specs() + if len(s) == 0 { + return "" + } + if int(typ) < len(s) && s[typ].Name != "" { + return s[typ].Name + } + return fmt.Sprintf("Invalid(%d)", typ) +} + func (v Version) Valid() bool { _, ok := versions[v] return ok diff --git a/src/internal/types/errors/code_string.go b/src/internal/types/errors/code_string.go index 9ae675ef849d87..26d7b48ee78a63 100644 --- a/src/internal/types/errors/code_string.go +++ b/src/internal/types/errors/code_string.go @@ -86,7 +86,6 @@ func _() { _ = x[MissingFieldOrMethod-76] _ = x[BadDotDotDotSyntax-77] _ = x[NonVariadicDotDotDot-78] - _ = x[MisplacedDotDotDot-79] _ = x[InvalidDotDotDot-81] _ = x[UncalledBuiltin-82] _ = x[InvalidAppend-83] @@ -161,7 +160,7 @@ func _() { const ( _Code_name_0 = "InvalidSyntaxTree" _Code_name_1 = "TestBlankPkgNameMismatchedPkgNameInvalidPkgUseBadImportPathBrokenImportImportCRenamedUnusedImportInvalidInitCycleDuplicateDeclInvalidDeclCycleInvalidTypeCycleInvalidConstInitInvalidConstValInvalidConstTypeUntypedNilUseWrongAssignCountUnassignableOperandNoNewVarMultiValAssignOpInvalidIfaceAssignInvalidChanAssignIncompatibleAssignUnaddressableFieldAssignNotATypeInvalidArrayLenBlankIfaceMethodIncomparableMapKey" - _Code_name_2 = "InvalidPtrEmbedBadRecvInvalidRecvDuplicateFieldAndMethodDuplicateMethodInvalidBlankInvalidIotaMissingInitBodyInvalidInitSigInvalidInitDeclInvalidMainDeclTooManyValuesNotAnExprTruncatedFloatNumericOverflowUndefinedOpMismatchedTypesDivByZeroNonNumericIncDecUnaddressableOperandInvalidIndirectionNonIndexableOperandInvalidIndexSwappedSliceIndicesNonSliceableOperandInvalidSliceExprInvalidShiftCountInvalidShiftOperandInvalidReceiveInvalidSendDuplicateLitKeyMissingLitKeyInvalidLitIndexOversizeArrayLitMixedStructLitInvalidStructLitMissingLitFieldDuplicateLitFieldUnexportedLitFieldInvalidLitFieldUntypedLitInvalidLitAmbiguousSelectorUndeclaredImportedNameUnexportedNameUndeclaredNameMissingFieldOrMethodBadDotDotDotSyntaxNonVariadicDotDotDotMisplacedDotDotDot" + _Code_name_2 = "InvalidPtrEmbedBadRecvInvalidRecvDuplicateFieldAndMethodDuplicateMethodInvalidBlankInvalidIotaMissingInitBodyInvalidInitSigInvalidInitDeclInvalidMainDeclTooManyValuesNotAnExprTruncatedFloatNumericOverflowUndefinedOpMismatchedTypesDivByZeroNonNumericIncDecUnaddressableOperandInvalidIndirectionNonIndexableOperandInvalidIndexSwappedSliceIndicesNonSliceableOperandInvalidSliceExprInvalidShiftCountInvalidShiftOperandInvalidReceiveInvalidSendDuplicateLitKeyMissingLitKeyInvalidLitIndexOversizeArrayLitMixedStructLitInvalidStructLitMissingLitFieldDuplicateLitFieldUnexportedLitFieldInvalidLitFieldUntypedLitInvalidLitAmbiguousSelectorUndeclaredImportedNameUnexportedNameUndeclaredNameMissingFieldOrMethodBadDotDotDotSyntaxNonVariadicDotDotDot" _Code_name_3 = "InvalidDotDotDotUncalledBuiltinInvalidAppendInvalidCapInvalidCloseInvalidCopyInvalidComplexInvalidDeleteInvalidImagInvalidLenSwappedMakeArgsInvalidMakeInvalidRealInvalidAssertImpossibleAssertInvalidConversionInvalidUntypedConversionBadOffsetofSyntaxInvalidOffsetofUnusedExprUnusedVarMissingReturnWrongResultCountOutOfScopeResultInvalidCondInvalidPostDecl" _Code_name_4 = "InvalidIterVarInvalidRangeExprMisplacedBreakMisplacedContinueMisplacedFallthroughDuplicateCaseDuplicateDefaultBadTypeKeywordInvalidTypeSwitchInvalidExprSwitchInvalidSelectCaseUndeclaredLabelDuplicateLabelMisplacedLabelUnusedLabelJumpOverDeclJumpIntoBlockInvalidMethodExprWrongArgCountInvalidCallUnusedResultsInvalidDeferInvalidGoBadDeclRepeatedDeclInvalidUnsafeAddInvalidUnsafeSliceUnsupportedFeatureNotAGenericTypeWrongTypeArgCountCannotInferTypeArgsInvalidTypeArgInvalidInstanceCycleInvalidUnionMisplacedConstraintIfaceInvalidMethodTypeParamsMisplacedTypeParamInvalidUnsafeSliceDataInvalidUnsafeString" _Code_name_5 = "InvalidClearTypeTooLargeInvalidMinMaxOperandTooNew" @@ -169,7 +168,7 @@ const ( var ( _Code_index_1 = [...]uint16{0, 4, 16, 33, 46, 59, 71, 85, 97, 113, 126, 142, 158, 174, 189, 205, 218, 234, 253, 261, 277, 295, 312, 330, 354, 362, 377, 393, 411} - _Code_index_2 = [...]uint16{0, 15, 22, 33, 56, 71, 83, 94, 109, 123, 138, 153, 166, 175, 189, 204, 215, 230, 239, 255, 275, 293, 312, 324, 343, 362, 378, 395, 414, 428, 439, 454, 467, 482, 498, 512, 528, 543, 560, 578, 593, 603, 613, 630, 652, 666, 680, 700, 718, 738, 756} + _Code_index_2 = [...]uint16{0, 15, 22, 33, 56, 71, 83, 94, 109, 123, 138, 153, 166, 175, 189, 204, 215, 230, 239, 255, 275, 293, 312, 324, 343, 362, 378, 395, 414, 428, 439, 454, 467, 482, 498, 512, 528, 543, 560, 578, 593, 603, 613, 630, 652, 666, 680, 700, 718, 738} _Code_index_3 = [...]uint16{0, 16, 31, 44, 54, 66, 77, 91, 104, 115, 125, 140, 151, 162, 175, 191, 208, 232, 249, 264, 274, 283, 296, 312, 328, 339, 354} _Code_index_4 = [...]uint16{0, 14, 30, 44, 61, 81, 94, 110, 124, 141, 158, 175, 190, 204, 218, 229, 241, 254, 271, 284, 295, 308, 320, 329, 336, 348, 364, 382, 400, 415, 432, 451, 465, 485, 497, 521, 544, 562, 584, 603} _Code_index_5 = [...]uint8{0, 12, 24, 44, 50} @@ -182,7 +181,7 @@ func (i Code) String() string { case 1 <= i && i <= 28: i -= 1 return _Code_name_1[_Code_index_1[i]:_Code_index_1[i+1]] - case 30 <= i && i <= 79: + case 30 <= i && i <= 78: i -= 30 return _Code_name_2[_Code_index_2[i]:_Code_index_2[i+1]] case 81 <= i && i <= 106: diff --git a/src/internal/types/errors/codes.go b/src/internal/types/errors/codes.go index c0e6aa6c2daf52..b0f7d2d4466429 100644 --- a/src/internal/types/errors/codes.go +++ b/src/internal/types/errors/codes.go @@ -719,10 +719,7 @@ const ( // MisplacedDotDotDot occurs when a "..." is used somewhere other than the // final argument in a function declaration. - // - // Example: - // func f(...int, int) - MisplacedDotDotDot + _ // not used anymore (error reported by parser) _ // InvalidDotDotDotOperand was removed. @@ -884,7 +881,9 @@ const ( // context in which it is used. // // Example: - // var _ = 1 + []int{} + // func f[T ~int8 | ~int16 | ~int32 | ~int64](x T) T { + // return x + 1024 + // } InvalidUntypedConversion // BadOffsetofSyntax occurs when unsafe.Offsetof is called with an argument diff --git a/src/internal/types/errors/generrordocs.go b/src/internal/types/errors/generrordocs.go index 46343be3ef08bc..613c77426928de 100644 --- a/src/internal/types/errors/generrordocs.go +++ b/src/internal/types/errors/generrordocs.go @@ -1,9 +1,9 @@ -//go:build ignore - // Copyright 2023 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build ignore + // generrordocs creates a Markdown file for each (compiler) error code // and its associated documentation. // Note: this program must be run in this directory. diff --git a/src/internal/types/testdata/check/builtins0.go b/src/internal/types/testdata/check/builtins0.go index 1c0e69200eba84..9b99a890acf996 100644 --- a/src/internal/types/testdata/check/builtins0.go +++ b/src/internal/types/testdata/check/builtins0.go @@ -260,9 +260,9 @@ func complex2() { func copy1() { copy() // ERROR "not enough arguments" copy("foo") // ERROR "not enough arguments" - copy([ /* ERROR "copy expects slice arguments" */ ...]int{}, []int{}) - copy([ /* ERROR "copy expects slice arguments" */ ]int{}, [...]int{}) - copy([ /* ERROR "different element types" */ ]int8{}, "foo") + copy([ /* ERROR "invalid copy: argument must be a slice; have [...]int{} (value of type [0]int)" */ ...]int{}, []int{}) + copy([]int{}, [ /* ERROR "invalid copy: argument must be a slice; have [...]int{} (value of type [0]int)" */ ...]int{}) + copy([ /* ERROR "invalid copy: arguments []int8{} (value of type []int8) and \"foo\" (untyped string constant) have different element types int8 and byte" */ ]int8{}, "foo") // spec examples var a = [...]int{0, 1, 2, 3, 4, 5, 6, 7} @@ -275,9 +275,9 @@ func copy1() { var t [][]int copy(t, t) - copy(t /* ERROR "copy expects slice arguments" */ , nil) - copy(nil /* ERROR "copy expects slice arguments" */ , t) - copy(nil /* ERROR "copy expects slice arguments" */ , nil) + copy(t, nil /* ERROR "invalid copy: argument must be a slice; have untyped nil" */ ) + copy(nil /* ERROR "invalid copy: argument must be a slice; have untyped nil" */ , t) + copy(nil /* ERROR "invalid copy: argument must be a slice; have untyped nil" */ , nil) copy(t... /* ERROR "invalid use of ..." */ ) } @@ -515,7 +515,7 @@ func max1() { _ = max(s) _ = max(x, x) _ = max(x, x, x, x, x) - var _ int = max /* ERROR "cannot use max(m) (value of type myint) as int value" */ (m) + var _ int = max /* ERROR "cannot use max(m) (value of int type myint) as int value" */ (m) _ = max(x, m /* ERROR "invalid argument: mismatched types int (previous argument) and myint (type of m)" */ , x) _ = max(1, x) @@ -569,7 +569,7 @@ func min1() { _ = min(s) _ = min(x, x) _ = min(x, x, x, x, x) - var _ int = min /* ERROR "cannot use min(m) (value of type myint) as int value" */ (m) + var _ int = min /* ERROR "cannot use min(m) (value of int type myint) as int value" */ (m) _ = min(x, m /* ERROR "invalid argument: mismatched types int (previous argument) and myint (type of m)" */ , x) _ = min(1, x) @@ -609,24 +609,58 @@ func min2() { ) } + func new1() { - _ = new() // ERROR "not enough arguments" + _ = new() // ERROR "not enough arguments" _ = new(1, 2) // ERROR "too many arguments" - _ = new("foo" /* ERROR "not a type" */) - p := new(float64) + _ = new(unsafe /* ERROR "use of package unsafe not in selector" */ ) + _ = new(struct{ x, y int }) + p := new(float64) q := new(*float64) _ = *p == **q - new /* ERROR "not used" */ (int) - _ = &new /* ERROR "cannot take address" */ (int) - _ = new(int... /* ERROR "invalid use of ..." */ ) + type G[P any] struct{} + _ = new(G[int]) + _ = new(G /* ERROR "cannot use generic type G without instantiation" */ ) + + new /* ERROR "not used" */ (int) + _ = &new /* ERROR "cannot take address" */ (int) + _ = new(int... /* ERROR "invalid use of ..." */) + _ = new(f0 /* ERROR "f0() (no value) used as value or type" */ ()) + _ = new(len /* ERROR "len (built-in) must be called" */) + _ = new(1 /* ERROR "argument to new (overflows)" */ << 70) } func new2() { - f1 := func() (x []int) { return } - _ = new(f0 /* ERROR "not a type" */ ()) - _ = new(f1 /* ERROR "not a type" */ ()) + // new(expr), added in go1.26 + f1 := func() []int { panic(0) } + f2 := func() (int, int) { panic(0) } + var ( + _ *[]int = new(f1()) + _ *func() []int = new(f1) + _ *bool = new(false) + _ *bool = new(1 < 2) + _ *int = new(123) + _ *float64 = new(1.0) + _ *uint = new(uint(3)) + _ *rune = new('a') + _ *string = new("A") + _ *struct{} = new(struct{}{}) + _ *any = new(any) + + _ = new(f2 /* ERRORx "multiple-value.*in single-value context" */ ()) + _ = new(1 << /* ERROR "constant shift overflow" */ 1000) + _ = new(1e10000 /* ERROR "cannot use 1e10000 (untyped float constant 1e+10000) as float64 value in argument to new (overflows)" */ ) + _ = new(nil /* ERROR "use of untyped nil in argument to new" */ ) + _ = new(comparable /* ERROR "cannot use type comparable outside a type constraint" */ ) + _ = new(new /* ERROR "new (built-in) must be called" */ ) + _ = new(panic /* ERROR "panic(0) (no value) used as value or type" */ (0)) + + // from issue 43125 + _ = new(-1) + _ = new(1 + 1) + ) } func panic1() { diff --git a/src/internal/types/testdata/check/builtins1.go b/src/internal/types/testdata/check/builtins1.go index f7ac72d4b9ae60..341723ccb07e17 100644 --- a/src/internal/types/testdata/check/builtins1.go +++ b/src/internal/types/testdata/check/builtins1.go @@ -62,13 +62,13 @@ func _[T C5[X], X any](ch T) { // copy func _[T any](x, y T) { - copy(x /* ERROR "copy expects slice arguments" */ , y) + copy(x /* ERROR "invalid copy: argument must be a slice; have x (variable of type T constrained by any)" */ , y) } func _[T ~[]byte](x, y T) { copy(x, y) copy(x, "foo") - copy("foo" /* ERROR "expects slice arguments" */ , y) + copy("foo" /* ERROR "argument must be a slice; have \"foo\" (untyped string constant)" */ , y) var x2 []byte copy(x2, y) // element types are identical @@ -82,16 +82,17 @@ func _[T ~[]byte](x, y T) { func _[T ~[]E, E any](x T, y []E) { copy(x, y) - copy(x /* ERROR "different element types" */ , "foo") + copy(x /* ERROR "arguments x (variable of type T constrained by ~[]E) and \"foo\" (untyped string constant) have different element types E and byte" */ , "foo") } func _[T ~string](x []byte, y T) { copy(x, y) - copy(y /* ERROR "expects slice arguments" */ , x) + copy([ /* ERROR "arguments []int{} (value of type []int) and y (variable of type T constrained by ~string) have different element types int and byte" */ ]int{}, y) + copy(y /* ERROR "argument must be a slice; have y (variable of type T constrained by ~string)" */ , x) } func _[T ~[]byte|~string](x T, y []byte) { - copy(x /* ERROR "expects slice arguments" */ , y) + copy(x /* ERROR "argument must be a slice; have x (variable of type T constrained by ~[]byte | ~string)" */ , y) copy(y, x) } @@ -144,6 +145,9 @@ func _[T M4[K, V], K comparable, V any](m T) { type myChan chan int func _[ + A1 ~[10]byte, + A2 ~[]byte | ~[10]byte, + S1 ~[]int, S2 ~[]int | ~chan int, @@ -152,8 +156,15 @@ func _[ C1 ~chan int, C2 ~chan int | ~chan string, - C3 chan int | myChan, // single underlying type + C3 chan int | myChan, // single underlying type + C4 chan int | chan<- int, // channels may have different (non-conflicting) directions + C5 <-chan int | chan<- int, ]() { + type A0 [10]byte + _ = make([ /* ERROR "cannot make [10]byte: type must be slice, map, or channel" */ 10]byte) + _ = make(A1 /* ERROR "cannot make A1: type must be slice, map, or channel" */ ) + _ = make(A2 /* ERROR "cannot make A2: type must be slice, map, or channel" */ ) + type S0 []int _ = make([]int, 10) _ = make(S0, 10) @@ -162,7 +173,7 @@ func _[ _ = make /* ERROR "expects 2 or 3 arguments" */ (S1) _ = make(S1, 10, 20) _ = make /* ERROR "expects 2 or 3 arguments" */ (S1, 10, 20, 30) - _ = make(S2 /* ERROR "cannot make S2: no core type" */ , 10) + _ = make(S2 /* ERROR "cannot make S2: []int and chan int have different underlying types" */ , 10) type M0 map[string]int _ = make(map[string]int) @@ -170,7 +181,7 @@ func _[ _ = make(M1) _ = make(M1, 10) _ = make/* ERROR "expects 1 or 2 arguments" */(M1, 10, 20) - _ = make(M2 /* ERROR "cannot make M2: no core type" */ ) + _ = make(M2 /* ERROR "cannot make M2: map[string]int and chan int have different underlying types" */ ) type C0 chan int _ = make(chan int) @@ -178,8 +189,10 @@ func _[ _ = make(C1) _ = make(C1, 10) _ = make/* ERROR "expects 1 or 2 arguments" */(C1, 10, 20) - _ = make(C2 /* ERROR "cannot make C2: no core type" */ ) + _ = make(C2 /* ERROR "cannot make C2: channels chan int and chan string have different element types" */ ) _ = make(C3) + _ = make(C4) + _ = make(C5 /* ERROR "cannot make C5: channels <-chan int and chan<- int have conflicting directions" */ ) } // max @@ -198,7 +211,7 @@ func _[ var x2 P2 _ = max(x2) _ = max(x2, x2) - _ = max(1, 2 /* ERROR "cannot convert 2 (untyped int constant) to type P2" */, x2) // error at 2 because max is 2 + _ = max(1, 2, x2 /* ERROR "mismatched types untyped int (previous argument) and P2 (type of x2)" */ ) _ = max(x1, x2 /* ERROR "mismatched types P1 (previous argument) and P2 (type of x2)" */ ) } @@ -219,7 +232,7 @@ func _[ var x2 P2 _ = min(x2) _ = min(x2, x2) - _ = min(1 /* ERROR "cannot convert 1 (untyped int constant) to type P2" */ , 2, x2) // error at 1 because min is 1 + _ = min(1, 2, x2 /* ERROR "mismatched types untyped int (previous argument) and P2 (type of x2)" */ ) _ = min(x1, x2 /* ERROR "mismatched types P1 (previous argument) and P2 (type of x2)" */ ) } diff --git a/src/internal/types/testdata/check/decls0.go b/src/internal/types/testdata/check/decls0.go index f9b0849dad7744..f5345135db4193 100644 --- a/src/internal/types/testdata/check/decls0.go +++ b/src/internal/types/testdata/check/decls0.go @@ -43,7 +43,7 @@ type init /* ERROR "cannot declare init" */ struct{} var _, init /* ERROR "cannot declare init" */ int func init() {} -func init /* ERROR "missing function body" */ () +func init /* ERROR "func init must have a body" */ () func _() { const init = 0 } func _() { type init int } diff --git a/src/internal/types/testdata/check/decls2/decls2a.go b/src/internal/types/testdata/check/decls2/decls2a.go index 2362bb96ff190b..58fdbfe1321f76 100644 --- a/src/internal/types/testdata/check/decls2/decls2a.go +++ b/src/internal/types/testdata/check/decls2/decls2a.go @@ -83,7 +83,7 @@ func (T5 /* ERROR "invalid receiver" */ ) m2() {} // Methods associated with a named pointer type. type ptr *int func (ptr /* ERROR "invalid receiver" */ ) _() {} -func (* /* ERROR "invalid receiver" */ ptr) _() {} +func (*ptr /* ERROR "invalid receiver" */ ) _() {} // Methods with zero or multiple receivers. func ( /* ERROR "method has no receiver" */ ) _() {} @@ -96,13 +96,13 @@ func (a, b, c /* ERROR "method has multiple receivers" */ T3) _() {} func (int /* ERROR "cannot define new methods on non-local type int" */ ) m() {} func ([ /* ERROR "invalid receiver" */ ]int) m() {} func (time /* ERROR "cannot define new methods on non-local type time.Time" */ .Time) m() {} -func (* /* ERROR "cannot define new methods on non-local type time.Time" */ time.Time) m() {} -func (x /* ERROR "invalid receiver" */ interface{}) m() {} +func (*time /* ERROR "cannot define new methods on non-local type time.Time" */ .Time) m() {} +func (x any /* ERROR "invalid receiver" */ ) m() {} // Unsafe.Pointer is treated like a pointer when used as receiver type. type UP unsafe.Pointer func (UP /* ERROR "invalid" */ ) m1() {} -func (* /* ERROR "invalid" */ UP) m2() {} +func (*UP /* ERROR "invalid" */ ) m2() {} // Double declarations across package files const c_double = 0 diff --git a/src/internal/types/testdata/check/doubled_labels.go b/src/internal/types/testdata/check/doubled_labels.go new file mode 100644 index 00000000000000..f3de27020ba35b --- /dev/null +++ b/src/internal/types/testdata/check/doubled_labels.go @@ -0,0 +1,26 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func _() { +outer: +inner: + for { + continue inner + break inner + } + goto outer +} + +func _() { +outer: +inner: + for { + continue inner + continue outer /* ERROR "invalid continue label outer" */ + break outer /* ERROR "invalid break label outer" */ + } + goto outer +} diff --git a/src/internal/types/testdata/check/errors.go b/src/internal/types/testdata/check/errors.go index 10b6a22eb1b880..615cf862d1b89d 100644 --- a/src/internal/types/testdata/check/errors.go +++ b/src/internal/types/testdata/check/errors.go @@ -58,7 +58,7 @@ func _() { // Use unqualified names for package-local objects. type T struct{} -var _ int = T /* ERROR "value of type T" */ {} // use T in error message rather than errors.T +var _ int = T /* ERROR "value of struct type T" */ {} // use T in error message rather than errors.T // Don't report errors containing "invalid type" (issue #24182). func _(x *missing /* ERROR "undefined: missing" */ ) { diff --git a/src/internal/types/testdata/check/expr2.go b/src/internal/types/testdata/check/expr2.go index ebb85eb2339d55..603f5ae190734f 100644 --- a/src/internal/types/testdata/check/expr2.go +++ b/src/internal/types/testdata/check/expr2.go @@ -201,7 +201,7 @@ func interfaces() { var s11 S11 var s2 S2 - _ = i == 0 /* ERROR "cannot convert" */ + _ = i == 0 /* ERROR "invalid operation: i == 0 (mismatched types interface{m() int} and untyped int)" */ _ = i == s1 /* ERROR "mismatched types" */ _ = i == &s1 _ = i == &s11 diff --git a/src/internal/types/testdata/check/go1_20_19.go b/src/internal/types/testdata/check/go1_20_19.go index 08365a7cfb564d..892179c72fc02f 100644 --- a/src/internal/types/testdata/check/go1_20_19.go +++ b/src/internal/types/testdata/check/go1_20_19.go @@ -14,4 +14,4 @@ type Slice []byte type Array [8]byte var s Slice -var p = (Array)(s /* ok because Go 1.20 ignored the //go:build go1.19 */) +var p = (Array)(s /* ok because file versions below go1.21 set the language version to go1.21 */) diff --git a/src/internal/types/testdata/check/go1_21_19.go b/src/internal/types/testdata/check/go1_21_19.go index 2acd25865d4b69..febf653cb1b49c 100644 --- a/src/internal/types/testdata/check/go1_21_19.go +++ b/src/internal/types/testdata/check/go1_21_19.go @@ -14,4 +14,4 @@ type Slice []byte type Array [8]byte var s Slice -var p = (Array)(s /* ERROR "requires go1.20 or later" */) +var p = (Array)(s /* ok because file versions below go1.21 set the language version to go1.21 */) diff --git a/src/internal/types/testdata/check/go1_21_22.go b/src/internal/types/testdata/check/go1_21_22.go new file mode 100644 index 00000000000000..3939b7b1d868c0 --- /dev/null +++ b/src/internal/types/testdata/check/go1_21_22.go @@ -0,0 +1,16 @@ +// -lang=go1.21 + +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Check Go language version-specific errors. + +//go:build go1.22 + +package p + +func f() { + for _ = range /* ok because of upgrade to 1.22 */ 10 { + } +} diff --git a/src/internal/types/testdata/check/go1_22_21.go b/src/internal/types/testdata/check/go1_22_21.go new file mode 100644 index 00000000000000..f910ecb59cbc78 --- /dev/null +++ b/src/internal/types/testdata/check/go1_22_21.go @@ -0,0 +1,16 @@ +// -lang=go1.22 + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Check Go language version-specific errors. + +//go:build go1.21 + +package p + +func f() { + for _ = range 10 /* ERROR "requires go1.22 or later" */ { + } +} diff --git a/src/internal/types/testdata/check/go1_25.go b/src/internal/types/testdata/check/go1_25.go new file mode 100644 index 00000000000000..3799bc02b46466 --- /dev/null +++ b/src/internal/types/testdata/check/go1_25.go @@ -0,0 +1,18 @@ +// -lang=go1.25 + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Check Go language version-specific errors. + +//go:build go1.25 + +package p + +func f(x int) { + _ = new /* ERROR "new(123) requires go1.26 or later" */ (123) + _ = new /* ERROR "new(x) requires go1.26 or later" */ (x) + _ = new /* ERROR "new(f) requires go1.26 or later" */ (f) + _ = new /* ERROR "new(1 < 2) requires go1.26 or later" */ (1 < 2) +} diff --git a/src/internal/types/testdata/check/issue70974.go b/src/internal/types/testdata/check/issue70974.go new file mode 100644 index 00000000000000..59b11653cee18f --- /dev/null +++ b/src/internal/types/testdata/check/issue70974.go @@ -0,0 +1,27 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func _() { +outer: + for { + break outer + } + + for { + break outer /* ERROR "invalid break label outer" */ + } +} + +func _() { +outer: + for { + continue outer + } + + for { + continue outer /* ERROR "invalid continue label outer" */ + } +} diff --git a/src/internal/types/testdata/check/issues0.go b/src/internal/types/testdata/check/issues0.go index d78b65705a9597..2b59a9c9b5c88a 100644 --- a/src/internal/types/testdata/check/issues0.go +++ b/src/internal/types/testdata/check/issues0.go @@ -326,9 +326,9 @@ func issue28281b(a, b int, c ...int) func issue28281c(a, b, c ... /* ERROR "can only use ... with final parameter" */ int) func issue28281d(... /* ERROR "can only use ... with final parameter" */ int, int) func issue28281e(a, b, c ... /* ERROR "can only use ... with final parameter" */ int, d int) -func issue28281f(... /* ERROR "can only use ... with final parameter" */ int, ... /* ERROR "can only use ... with final parameter" */ int, int) -func (... /* ERROR "invalid use of '...'" */ TT) f() -func issue28281g() (... /* ERROR "can only use ... with final parameter" */ TT) +func issue28281f(... /* ERROR "can only use ... with final parameter" */ int, ... int, int) +func (... /* ERROR "invalid use of ..." */ TT) f() +func issue28281g() (... /* ERROR "invalid use of ..." */ TT) // Issue #26234: Make various field/method lookup errors easier to read by matching cmd/compile's output func issue26234a(f *syn.Prog) { @@ -363,7 +363,7 @@ func issue35895() { // Because both t1 and t2 have the same global package name (template), // qualify packages with full path name in this case. - var _ t1.Template = t2 /* ERRORx `cannot use .* \(value of type .html/template.\.Template\) as .text/template.\.Template` */ .Template{} + var _ t1.Template = t2 /* ERRORx `cannot use .* \(value of struct type .html/template.\.Template\) as .text/template.\.Template` */ .Template{} } func issue42989(s uint) { diff --git a/src/internal/types/testdata/check/lookup1.go b/src/internal/types/testdata/check/lookup1.go index 048288db773f64..669767b27876dc 100644 --- a/src/internal/types/testdata/check/lookup1.go +++ b/src/internal/types/testdata/check/lookup1.go @@ -11,7 +11,7 @@ func _() { x, aBc int } _ = s.x - _ = s /* ERROR "invalid operation: cannot call non-function s.x (variable of type int)" */ .x() + _ = s /* ERROR "invalid operation: cannot call s.x (variable of type int): int is not a function" */ .x() _ = s.X // ERROR "s.X undefined (type struct{x int; aBc int} has no field or method X, but does have field x)" _ = s.X /* ERROR "s.X undefined (type struct{x int; aBc int} has no field or method X, but does have field x)" */ () @@ -26,7 +26,7 @@ func _() { } var s S _ = s.x - _ = s /* ERROR "invalid operation: cannot call non-function s.x (variable of type int)" */ .x() + _ = s /* ERROR "invalid operation: cannot call s.x (variable of type int): int is not a function" */ .x() _ = s.X // ERROR "s.X undefined (type S has no field or method X, but does have field x)" _ = s.X /* ERROR "s.X undefined (type S has no field or method X, but does have field x)" */ () } @@ -71,3 +71,15 @@ func _() { _ = x.Form // ERROR "x.Form undefined (type big.Float has no field or method Form, but does have unexported field form)" _ = x.FOrm // ERROR "x.FOrm undefined (type big.Float has no field or method FOrm)" } + +func _[P any](x P) { + x /* ERROR "cannot call x (variable of type P constrained by any): no specific type" */ () +} + +func _[P int](x P) { + x /* ERROR "cannot call x (variable of type P constrained by int): int is not a function" */ () +} + +func _[P int | string](x P) { + x /* ERROR "cannot call x (variable of type P constrained by int | string): int is not a function" */ () +} diff --git a/src/internal/types/testdata/check/typeparams.go b/src/internal/types/testdata/check/typeparams.go index b002377df79e02..b73f1fee6d51da 100644 --- a/src/internal/types/testdata/check/typeparams.go +++ b/src/internal/types/testdata/check/typeparams.go @@ -58,10 +58,10 @@ func min[T interface{ ~int }](x, y T) T { } func _[T interface{~int | ~float32}](x, y T) bool { return x < y } -func _[T any](x, y T) bool { return x /* ERROR "type parameter T is not comparable" */ < y } -func _[T interface{~int | ~float32 | ~bool}](x, y T) bool { return x /* ERROR "type parameter T is not comparable" */ < y } +func _[T any](x, y T) bool { return x /* ERROR "type parameter T cannot use operator <" */ < y } +func _[T interface{~int | ~float32 | ~bool}](x, y T) bool { return x /* ERROR "type parameter T cannot use operator <" */ < y } -func _[T C1[T]](x, y T) bool { return x /* ERROR "type parameter T is not comparable" */ < y } +func _[T C1[T]](x, y T) bool { return x /* ERROR "type parameter T cannot use operator <" */ < y } func _[T C2[T]](x, y T) bool { return x < y } type C1[T any] interface{} @@ -134,11 +134,11 @@ func _[T interface{ ~string }] (x T, i, j, k int) { var _ T = x[i:j:k /* ERROR " type myByte1 []byte type myByte2 []byte func _[T interface{ []byte | myByte1 | myByte2 }] (x T, i, j, k int) { var _ T = x[i:j:k] } -func _[T interface{ []byte | myByte1 | []int }] (x T, i, j, k int) { var _ T = x /* ERROR "no core type" */ [i:j:k] } +func _[T interface{ []byte | myByte1 | []int }] (x T, i, j, k int) { var _ T = x /* ERROR "[]byte and []int have different underlying types" */ [i:j:k] } func _[T interface{ []byte | myByte1 | myByte2 | string }] (x T, i, j, k int) { var _ T = x[i:j] } func _[T interface{ []byte | myByte1 | myByte2 | string }] (x T, i, j, k int) { var _ T = x[i:j:k /* ERROR "3-index slice of string" */ ] } -func _[T interface{ []byte | myByte1 | []int | string }] (x T, i, j, k int) { var _ T = x /* ERROR "no core type" */ [i:j] } +func _[T interface{ []byte | myByte1 | []int | string }] (x T, i, j, k int) { var _ T = x /* ERROR "[]byte and []int have different underlying types" */ [i:j] } // len/cap built-ins @@ -230,7 +230,7 @@ func _[ for _, _ = range s1 {} var s2 S2 - for range s2 /* ERRORx `cannot range over s2.*no core type` */ {} + for range s2 /* ERRORx `cannot range over s2.*\[\]int and \[10\]int have different underlying types` */ {} var a0 []int for range a0 {} @@ -243,7 +243,7 @@ func _[ for _, _ = range a1 {} var a2 A2 - for range a2 /* ERRORx `cannot range over a2.*no core type` */ {} + for range a2 /* ERRORx `cannot range over a2.*\[10\]int and \[\]int have different underlying types` */ {} var p0 *[10]int for range p0 {} @@ -256,7 +256,7 @@ func _[ for _, _ = range p1 {} var p2 P2 - for range p2 /* ERRORx `cannot range over p2.*no core type` */ {} + for range p2 /* ERRORx `cannot range over p2.*\*\[10\]int and \*\[\]int have different underlying types` */ {} var m0 map[string]int for range m0 {} @@ -269,7 +269,7 @@ func _[ for _, _ = range m1 {} var m2 M2 - for range m2 /* ERRORx `cannot range over m2.*no core type` */ {} + for range m2 /* ERRORx `cannot range over m2.*map\[string\]int and map\[string\]string` */ {} } // type inference checks diff --git a/src/internal/types/testdata/examples/types.go b/src/internal/types/testdata/examples/types.go index 67f1534be39479..d6da2c5f6f9f9f 100644 --- a/src/internal/types/testdata/examples/types.go +++ b/src/internal/types/testdata/examples/types.go @@ -114,7 +114,7 @@ type I1[T any] interface{ } // There is no such thing as a variadic generic type. -type _[T ... /* ERROR "invalid use of '...'" */ any] struct{} +type _[T ... /* ERROR "invalid use of ..." */ any] struct{} // Generic interfaces may be embedded as one would expect. type I2 interface { diff --git a/src/internal/types/testdata/fixedbugs/issue43125.go b/src/internal/types/testdata/fixedbugs/issue43125.go deleted file mode 100644 index d0d6feb2a8516c..00000000000000 --- a/src/internal/types/testdata/fixedbugs/issue43125.go +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package p - -var _ = new(- /* ERROR "not a type" */ 1) -var _ = new(1 /* ERROR "not a type" */ + 1) diff --git a/src/internal/types/testdata/fixedbugs/issue43671.go b/src/internal/types/testdata/fixedbugs/issue43671.go index be4c9ee5dd1f51..5b44682a7accd7 100644 --- a/src/internal/types/testdata/fixedbugs/issue43671.go +++ b/src/internal/types/testdata/fixedbugs/issue43671.go @@ -12,11 +12,11 @@ type C4 interface{ chan int | chan<- int } type C5[T any] interface{ ~chan T | <-chan T } func _[T any](ch T) { - <-ch // ERRORx `cannot receive from ch .* \(no core type\)` + <-ch // ERRORx `cannot receive from ch .*: no specific channel type` } func _[T C0](ch T) { - <-ch // ERROR "cannot receive from non-channel ch" + <-ch // ERRORx `cannot receive from ch .*: non-channel int` } func _[T C1](ch T) { @@ -28,11 +28,11 @@ func _[T C2](ch T) { } func _[T C3](ch T) { - <-ch // ERRORx `cannot receive from ch .* \(no core type\)` + <-ch // ERRORx `cannot receive from ch .*: channels chan int and chan float32 have different element types` } func _[T C4](ch T) { - <-ch // ERROR "cannot receive from send-only channel" + <-ch // ERRORx `cannot receive from ch .*: send-only channel chan<- int` } func _[T C5[X], X any](ch T, x X) { diff --git a/src/internal/types/testdata/fixedbugs/issue45550.go b/src/internal/types/testdata/fixedbugs/issue45550.go index 2ea4ffe3079540..32fdde6740c7a5 100644 --- a/src/internal/types/testdata/fixedbugs/issue45550.go +++ b/src/internal/types/testdata/fixedbugs/issue45550.go @@ -4,7 +4,7 @@ package p -type Builder /* ERROR "invalid recursive type" */ [T interface{ struct{ Builder[T] } }] struct{} +type Builder[T ~struct{ Builder[T] }] struct{} type myBuilder struct { Builder[myBuilder] } diff --git a/src/internal/types/testdata/fixedbugs/issue46461.go b/src/internal/types/testdata/fixedbugs/issue46461.go index e823013f995167..454f7e836537c6 100644 --- a/src/internal/types/testdata/fixedbugs/issue46461.go +++ b/src/internal/types/testdata/fixedbugs/issue46461.go @@ -7,16 +7,16 @@ package p // test case 1 -type T /* ERROR "invalid recursive type" */ [U interface{ M() T[U] }] int +type T[U interface{ M() T[U] }] int type X int func (X) M() T[X] { return 0 } // test case 2 -type A /* ERROR "invalid recursive type" */ [T interface{ A[T] }] interface{} +type A[T interface{ A[T] }] interface{} // test case 3 -type A2 /* ERROR "invalid recursive type" */ [U interface{ A2[U] }] interface{ M() A2[U] } +type A2[U interface{ A2[U] }] interface{ M() A2[U] } type I interface{ A2[I]; M() A2[I] } diff --git a/src/internal/types/testdata/fixedbugs/issue46461a.go b/src/internal/types/testdata/fixedbugs/issue46461a.go index e4b8e1a240a973..74ed6c4882719c 100644 --- a/src/internal/types/testdata/fixedbugs/issue46461a.go +++ b/src/internal/types/testdata/fixedbugs/issue46461a.go @@ -7,17 +7,16 @@ package p // test case 1 -type T /* ERROR "invalid recursive type" */ [U interface{ M() T[U] }] int +type T[U interface{ M() T[U] }] int type X int func (X) M() T[X] { return 0 } // test case 2 -type A /* ERROR "invalid recursive type" */ [T interface{ A[T] }] interface{} +type A[T interface{ A[T] }] interface{} // test case 3 -// TODO(gri) should report error only once -type A2 /* ERROR "invalid recursive type" */ /* ERROR "invalid recursive type" */ [U interface{ A2[U] }] interface{ M() A2[U] } +type A2[U interface{ A2[U] }] interface{ M() A2[U] } type I interface{ A2[I]; M() A2[I] } diff --git a/src/internal/types/testdata/fixedbugs/issue47115.go b/src/internal/types/testdata/fixedbugs/issue47115.go index 2d2be34104216e..58b668ce4f1cdb 100644 --- a/src/internal/types/testdata/fixedbugs/issue47115.go +++ b/src/internal/types/testdata/fixedbugs/issue47115.go @@ -12,11 +12,11 @@ type C4 interface{ chan int | chan<- int } type C5[T any] interface{ ~chan T | chan<- T } func _[T any](ch T) { - ch <- /* ERRORx `cannot send to ch .* no core type` */ 0 + ch <- /* ERRORx `cannot send to ch .*: no specific channel type` */ 0 } func _[T C0](ch T) { - ch <- /* ERROR "cannot send to non-channel" */ 0 + ch <- /* ERRORx `cannot send to ch .*: non-channel int` */ 0 } func _[T C1](ch T) { @@ -24,11 +24,11 @@ func _[T C1](ch T) { } func _[T C2](ch T) { - ch <-/* ERROR "cannot send to receive-only channel" */ 0 + ch <- /* ERRORx `cannot send to ch .*: receive-only channel <-chan int` */ 0 } func _[T C3](ch T) { - ch <- /* ERRORx `cannot send to ch .* no core type` */ 0 + ch <- /* ERRORx `cannot send to ch .*: channels chan int and chan float32 have different element types` */ 0 } func _[T C4](ch T) { diff --git a/src/internal/types/testdata/fixedbugs/issue47796.go b/src/internal/types/testdata/fixedbugs/issue47796.go index 7f719ff6745eaa..b07cdddababf67 100644 --- a/src/internal/types/testdata/fixedbugs/issue47796.go +++ b/src/internal/types/testdata/fixedbugs/issue47796.go @@ -6,16 +6,16 @@ package p // parameterized types with self-recursive constraints type ( - T1 /* ERROR "invalid recursive type" */ [P T1[P]] interface{} - T2 /* ERROR "invalid recursive type" */ [P, Q T2[P, Q]] interface{} + T1[P T1[P]] interface{} + T2[P, Q T2[P, Q]] interface{} T3[P T2[P, Q], Q interface{ ~string }] interface{} - T4a /* ERROR "invalid recursive type" */ [P T4a[P]] interface{ ~int } - T4b /* ERROR "invalid recursive type" */ [P T4b[int]] interface{ ~int } - T4c /* ERROR "invalid recursive type" */ [P T4c[string]] interface{ ~int } + T4a[P T4a[P]] interface{ ~int } + T4b[P T4b[int]] interface{ ~int } + T4c[P T4c[string /* ERROR "string does not satisfy T4c[string]" */]] interface{ ~int } // mutually recursive constraints - T5 /* ERROR "invalid recursive type" */ [P T6[P]] interface{ int } + T5[P T6[P]] interface{ int } T6[P T5[P]] interface{ int } ) @@ -28,6 +28,6 @@ var ( // test case from issue -type Eq /* ERROR "invalid recursive type" */ [a Eq[a]] interface { +type Eq[a Eq[a]] interface { Equal(that a) bool } diff --git a/src/internal/types/testdata/fixedbugs/issue47968.go b/src/internal/types/testdata/fixedbugs/issue47968.go index 83a1786133f297..e260c63a76da0d 100644 --- a/src/internal/types/testdata/fixedbugs/issue47968.go +++ b/src/internal/types/testdata/fixedbugs/issue47968.go @@ -1,3 +1,5 @@ +// -gotypesalias=1 + // Copyright 2021 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -10,12 +12,12 @@ func (T[P]) m1() type A1 = T // ERROR "cannot use generic type" -func (A1[P]) m2() {} +func (A1[P]) m2() {} // don't report a follow-on error on A1 type A2 = T[int] -func (A2 /* ERRORx `cannot define new methods on instantiated type (T\[int\]|A2)` */) m3() {} -func (_ /* ERRORx `cannot define new methods on instantiated type (T\[int\]|A2)` */ A2) m4() {} +func (A2 /* ERROR "cannot define new methods on instantiated type T[int]" */) m3() {} +func (_ A2 /* ERROR "cannot define new methods on instantiated type T[int]" */) m4() {} -func (T[int]) m5() {} // int is the type parameter name, not an instantiation +func (T[int]) m5() {} // int is the type parameter name, not an instantiation func (T[* /* ERROR "must be an identifier" */ int]) m6() {} // syntax error diff --git a/src/internal/types/testdata/fixedbugs/issue48529.go b/src/internal/types/testdata/fixedbugs/issue48529.go index bcc5e3536d3457..eca1da89232b08 100644 --- a/src/internal/types/testdata/fixedbugs/issue48529.go +++ b/src/internal/types/testdata/fixedbugs/issue48529.go @@ -4,7 +4,7 @@ package p -type T /* ERROR "invalid recursive type" */ [U interface{ M() T[U, int] }] int +type T[U interface{ M() T /* ERROR "too many type arguments for type T" */ [U, int] }] int type X int diff --git a/src/internal/types/testdata/fixedbugs/issue48712.go b/src/internal/types/testdata/fixedbugs/issue48712.go index 76ad16cd8feea9..028660fb1e30bc 100644 --- a/src/internal/types/testdata/fixedbugs/issue48712.go +++ b/src/internal/types/testdata/fixedbugs/issue48712.go @@ -10,7 +10,7 @@ func _[P comparable](x, y P) { _ = y == x _ = y == y - _ = x /* ERROR "type parameter P is not comparable with <" */ < y + _ = x /* ERROR "type parameter P cannot use operator <" */ < y } func _[P comparable](x P, y any) { @@ -19,7 +19,7 @@ func _[P comparable](x P, y any) { _ = y == x _ = y == y - _ = x /* ERROR "type parameter P is not comparable with <" */ < y + _ = x /* ERROR "type parameter P cannot use operator <" */ < y } func _[P any](x, y P) { @@ -28,7 +28,7 @@ func _[P any](x, y P) { _ = y /* ERROR "incomparable types in type set" */ == x _ = y /* ERROR "incomparable types in type set" */ == y - _ = x /* ERROR "type parameter P is not comparable with <" */ < y + _ = x /* ERROR "type parameter P cannot use operator <" */ < y } func _[P any](x P, y any) { @@ -37,5 +37,5 @@ func _[P any](x P, y any) { _ = y == x // ERROR "incomparable types in type set" _ = y == y - _ = x /* ERROR "type parameter P is not comparable with <" */ < y + _ = x /* ERROR "type parameter P cannot use operator <" */ < y } diff --git a/src/internal/types/testdata/fixedbugs/issue49005.go b/src/internal/types/testdata/fixedbugs/issue49005.go index d91c2078730baa..6ec926ec6163dc 100644 --- a/src/internal/types/testdata/fixedbugs/issue49005.go +++ b/src/internal/types/testdata/fixedbugs/issue49005.go @@ -26,6 +26,6 @@ type X2 struct{} func _() { switch F2().(type) { - case * /* ERROR "impossible type switch case: *X2\n\tF2() (value of type T2) cannot have dynamic type *X2 (missing method M)" */ X2: + case * /* ERROR "impossible type switch case: *X2\n\tF2() (value of interface type T2) cannot have dynamic type *X2 (missing method M)" */ X2: } } diff --git a/src/internal/types/testdata/fixedbugs/issue49439.go b/src/internal/types/testdata/fixedbugs/issue49439.go index 3852f160948913..63bedf61911914 100644 --- a/src/internal/types/testdata/fixedbugs/issue49439.go +++ b/src/internal/types/testdata/fixedbugs/issue49439.go @@ -6,21 +6,21 @@ package p import "unsafe" -type T0 /* ERROR "invalid recursive type" */ [P T0[P]] struct{} +type T0[P T0[P]] struct{} -type T1 /* ERROR "invalid recursive type" */ [P T2[P]] struct{} -type T2[P T1[P]] struct{} +type T1[P T2[P /* ERROR "P does not satisfy T1[P]" */]] struct{} +type T2[P T1[P /* ERROR "P does not satisfy T2[P]" */]] struct{} -type T3 /* ERROR "invalid recursive type" */ [P interface{ ~struct{ f T3[int] } }] struct{} +type T3[P interface{ ~struct{ f T3[int /* ERROR "int does not satisfy" */ ] } }] struct{} // valid cycle in M type N[P M[P]] struct{} -type M[Q any] struct { F *M[Q] } +type M[Q any] struct{ F *M[Q] } // "crazy" case type TC[P [unsafe.Sizeof(func() { - type T [P [unsafe.Sizeof(func(){})]byte] struct{} + type T[P [unsafe.Sizeof(func() {})]byte] struct{} })]byte] struct{} // test case from issue -type X /* ERROR "invalid recursive type" */ [T any, PT X[T]] interface{} +type X[T any, PT X /* ERROR "not enough type arguments for type X" */ [T]] interface{} diff --git a/src/internal/types/testdata/fixedbugs/issue49482.go b/src/internal/types/testdata/fixedbugs/issue49482.go index 7139baebc0cb66..bc6b60099cf5ae 100644 --- a/src/internal/types/testdata/fixedbugs/issue49482.go +++ b/src/internal/types/testdata/fixedbugs/issue49482.go @@ -13,7 +13,7 @@ const P = 2 // declare P to avoid noisy 'undefined' errors below. // The following parse as invalid array types due to parsing ambiguitiues. type _ [P *int /* ERROR "int (type) is not an expression" */ ]int -type _ [P /* ERROR "non-function P" */ (*int)]int +type _ [P /* ERROR "cannot call P (untyped int constant 2): untyped int is not a function" */ (*int)]int // Adding a trailing comma or an enclosing interface resolves the ambiguity. type _[P *int,] int diff --git a/src/internal/types/testdata/fixedbugs/issue49735.go b/src/internal/types/testdata/fixedbugs/issue49735.go index 0fcc778a066529..b719e1353f0e92 100644 --- a/src/internal/types/testdata/fixedbugs/issue49735.go +++ b/src/internal/types/testdata/fixedbugs/issue49735.go @@ -4,8 +4,9 @@ package p -func _[P1 any, P2 ~byte](s1 P1, s2 P2) { - _ = append(nil /* ERROR "first argument to append must be a slice; have untyped nil" */ , 0) - _ = append(s1 /* ERRORx `s1 .* has no core type` */ , 0) - _ = append(s2 /* ERRORx `s2 .* has core type byte` */ , 0) +func _[P1 any, P2 ~byte, P3 []int | []byte](s1 P1, s2 P2, s3 P3) { + _ = append(nil /* ERROR "invalid append: argument must be a slice; have untyped nil" */, 0) + _ = append(s1 /* ERROR "invalid append: argument must be a slice; have s1 (variable of type P1 constrained by any)" */, 0) + _ = append(s2 /* ERROR "invalid append: argument must be a slice; have s2 (variable of type P2 constrained by ~byte)" */, 0) + _ = append(s3 /* ERROR "invalid append: mismatched slice element types int and byte in s3 (variable of type P3 constrained by []int | []byte)" */, 0) } diff --git a/src/internal/types/testdata/fixedbugs/issue51335.go b/src/internal/types/testdata/fixedbugs/issue51335.go index 04dc04e1d7a36b..5eb52138430ab5 100644 --- a/src/internal/types/testdata/fixedbugs/issue51335.go +++ b/src/internal/types/testdata/fixedbugs/issue51335.go @@ -8,9 +8,9 @@ type S1 struct{} type S2 struct{} func _[P *S1|*S2]() { - _= []P{{ /* ERROR "invalid composite literal element type P (no core type)" */ }} + _= []P{{ /* ERROR "invalid composite literal element type P (no common underlying type)" */ }} } func _[P *S1|S1]() { - _= []P{{ /* ERROR "invalid composite literal element type P (no core type)" */ }} + _= []P{{ /* ERROR "invalid composite literal element type P (no common underlying type)" */ }} } diff --git a/src/internal/types/testdata/fixedbugs/issue51360.go b/src/internal/types/testdata/fixedbugs/issue51360.go index 1b9c45a9342f08..1798a4ab2fd3f8 100644 --- a/src/internal/types/testdata/fixedbugs/issue51360.go +++ b/src/internal/types/testdata/fixedbugs/issue51360.go @@ -5,9 +5,9 @@ package p func _() { - len.Println /* ERROR "cannot select on len" */ - len.Println /* ERROR "cannot select on len" */ () - _ = len.Println /* ERROR "cannot select on len" */ + len.Println /* ERROR "invalid use of len (built-in) in selector expression" */ + len.Println /* ERROR "invalid use of len (built-in) in selector expression" */ () + _ = len.Println /* ERROR "invalid use of len (built-in) in selector expression" */ _ = len /* ERROR "cannot index len" */ [0] _ = *len /* ERROR "cannot indirect len" */ } diff --git a/src/internal/types/testdata/fixedbugs/issue53535.go b/src/internal/types/testdata/fixedbugs/issue53535.go new file mode 100644 index 00000000000000..127b8a8b452d35 --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue53535.go @@ -0,0 +1,35 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +import "io" + +// test using struct with invalid embedded field +var _ io.Writer = W{} // no error expected here because W has invalid embedded field + +type W struct { + *bufio /* ERROR "undefined: bufio" */ .Writer +} + +// test using an invalid type +var _ interface{ m() } = &M{} // no error expected here because M is invalid + +type M undefined // ERROR "undefined: undefined" + +// test using struct with invalid embedded field and containing a self-reference (cycle) +var _ interface{ m() } = &S{} // no error expected here because S is invalid + +type S struct { + *S + undefined // ERROR "undefined: undefined" +} + +// test using a generic struct with invalid embedded field and containing a self-reference (cycle) +var _ interface{ m() } = &G[int]{} // no error expected here because S is invalid + +type G[P any] struct { + *G[P] + undefined // ERROR "undefined: undefined" +} diff --git a/src/internal/types/testdata/fixedbugs/issue60377.go b/src/internal/types/testdata/fixedbugs/issue60377.go index b754f89df7321c..17a9deb6d17010 100644 --- a/src/internal/types/testdata/fixedbugs/issue60377.go +++ b/src/internal/types/testdata/fixedbugs/issue60377.go @@ -57,7 +57,7 @@ func _() { var x S[int] g4(x) // we can infer int for P g4[int](x) // int is the correct type argument - g4[string](x /* ERROR "cannot use x (variable of type S[int]) as S[string] value in argument to g4[string]" */) + g4[string](x /* ERROR "cannot use x (variable of struct type S[int]) as S[string] value in argument to g4[string]" */) } // This is similar to the first example but here T1 is a component diff --git a/src/internal/types/testdata/fixedbugs/issue60434.go b/src/internal/types/testdata/fixedbugs/issue60434.go index e1d76527f3db63..68aa7c2fdfd11b 100644 --- a/src/internal/types/testdata/fixedbugs/issue60434.go +++ b/src/internal/types/testdata/fixedbugs/issue60434.go @@ -13,5 +13,5 @@ var s struct{ x int } func _() { f(s.y /* ERROR "s.y undefined" */) - f(1 /* ERROR "cannot convert 1" */ / s) + f(1 /* ERROR "invalid operation: 1 / s (mismatched types untyped int and struct{x int})" */ / s) } diff --git a/src/internal/types/testdata/fixedbugs/issue62157.go b/src/internal/types/testdata/fixedbugs/issue62157.go index c44f921f44bf45..67a110df317604 100644 --- a/src/internal/types/testdata/fixedbugs/issue62157.go +++ b/src/internal/types/testdata/fixedbugs/issue62157.go @@ -90,7 +90,7 @@ func _() { B = f(B, b, a) // verify type error - A = f /* ERROR "cannot use f(B, b, a) (value of type namedB) as namedA value in assignment" */ (B, b, a) + A = f /* ERROR "cannot use f(B, b, a) (value of chan type namedB) as namedA value in assignment" */ (B, b, a) } // Test case 4: some more combinations diff --git a/src/internal/types/testdata/fixedbugs/issue66285.go b/src/internal/types/testdata/fixedbugs/issue66285.go index 9811fec3f35549..4af76f05da8e41 100644 --- a/src/internal/types/testdata/fixedbugs/issue66285.go +++ b/src/internal/types/testdata/fixedbugs/issue66285.go @@ -1,14 +1,9 @@ -// -lang=go1.21 +// -lang=go1.13 // Copyright 2024 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Note: Downgrading to go1.13 requires at least go1.21, -// hence the need for -lang=go1.21 at the top. - -//go:build go1.13 - package p import "io" diff --git a/src/internal/types/testdata/fixedbugs/issue66751.go b/src/internal/types/testdata/fixedbugs/issue66751.go new file mode 100644 index 00000000000000..5a64b4dcc17222 --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue66751.go @@ -0,0 +1,62 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type S struct{} + +func (*S) m(int) {} + +func f[A interface { + ~*B + m(C) +}, B, C any]() { +} + +var _ = f[*S] // must be able to infer all remaining type arguments + +// original test case from issue + +type ptrTo[A any] interface{ ~*A } +type hasFoo[A any] interface{ foo(A) } +type both[A, B any] interface { + ptrTo[A] + hasFoo[B] +} + +type fooer[A any] struct{} + +func (f *fooer[A]) foo(A) {} + +func withPtr[A ptrTo[B], B any]() {} +func withFoo[A hasFoo[B], B any]() {} +func withBoth[A both[B, C], B, C any]() {} + +func _() { + withPtr[*fooer[int]]() // ok + withFoo[*fooer[int]]() // ok + withBoth[*fooer[int]]() // should be able to infer C +} + +// related test case reported in issue + +type X struct{} + +func (x X) M() int { return 42 } + +func CallM1[T interface{ M() R }, R any](t T) R { + return t.M() +} + +func CallM2[T interface { + X + M() R +}, R any](t T) R { + return t.M() +} + +func _() { + CallM1(X{}) // ok + CallM2(X{}) // should be able to infer R +} diff --git a/src/internal/types/testdata/fixedbugs/issue67683.go b/src/internal/types/testdata/fixedbugs/issue67683.go index f7c9bcdd0114c6..c9ad5f6788bec4 100644 --- a/src/internal/types/testdata/fixedbugs/issue67683.go +++ b/src/internal/types/testdata/fixedbugs/issue67683.go @@ -1,4 +1,4 @@ -// -goexperiment=aliastypeparams -gotypesalias=1 +// -gotypesalias=1 // Copyright 2024 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/src/internal/types/testdata/fixedbugs/issue68162.go b/src/internal/types/testdata/fixedbugs/issue68162.go new file mode 100644 index 00000000000000..8efd8a66dff4be --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue68162.go @@ -0,0 +1,24 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +type N[B N[B]] interface { + Add(B) B +} + +func Add[P N[P]](x, y P) P { + return x.Add(y) +} + +type MyInt int + +func (x MyInt) Add(y MyInt) MyInt { + return x + y +} + +func main() { + var x, y MyInt = 2, 3 + println(Add(x, y)) +} diff --git a/src/internal/types/testdata/fixedbugs/issue68903.go b/src/internal/types/testdata/fixedbugs/issue68903.go new file mode 100644 index 00000000000000..b1369aa0f6faa7 --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue68903.go @@ -0,0 +1,24 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type A = [4]int +type B = map[string]interface{} + +func _[T ~A](x T) { + _ = len(x) +} + +func _[U ~A](x U) { + _ = cap(x) +} + +func _[V ~A]() { + _ = V{} +} + +func _[W ~B](a interface{}) { + _ = a.(W)["key"] +} diff --git a/src/internal/types/testdata/fixedbugs/issue68935.go b/src/internal/types/testdata/fixedbugs/issue68935.go new file mode 100644 index 00000000000000..2e72468f05eb0c --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue68935.go @@ -0,0 +1,26 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type A = struct { + F string + G int +} + +func Make[T ~A]() T { + return T{ + F: "blah", + G: 1234, + } +} + +type N struct { + F string + G int +} + +func _() { + _ = Make[N]() +} diff --git a/src/internal/types/testdata/fixedbugs/issue69576.go b/src/internal/types/testdata/fixedbugs/issue69576.go new file mode 100644 index 00000000000000..fc436bbfd38424 --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue69576.go @@ -0,0 +1,11 @@ +// -gotypesalias=1 + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type A[P int] = struct{} + +var _ A[string /* ERROR "string does not satisfy int (string missing in int)" */] diff --git a/src/internal/types/testdata/fixedbugs/issue69955.go b/src/internal/types/testdata/fixedbugs/issue69955.go new file mode 100644 index 00000000000000..68ddf4108ce8ce --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue69955.go @@ -0,0 +1,42 @@ +// -gotypesalias=1 + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +import "math/big" + +type ( + S struct{} + N int + + A = S + B = int + C = N +) + +var ( + i int + s S + n N + a A + b B + c C + w big.Word +) + +const ( + _ = i // ERROR "i (variable of type int) is not constant" + _ = s // ERROR "s (variable of struct type S) is not constant" + _ = struct /* ERROR "struct{}{} (value of type struct{}) is not constant" */ {}{} + _ = n // ERROR "n (variable of int type N) is not constant" + + _ = a // ERROR "a (variable of struct type A) is not constant" + _ = b // ERROR "b (variable of int type B) is not constant" + _ = c // ERROR "c (variable of int type C) is not constant" + _ = w // ERROR "w (variable of uint type big.Word) is not constant" +) + +var _ int = w /* ERROR "cannot use w + 1 (value of uint type big.Word) as int value in variable declaration" */ + 1 diff --git a/src/internal/types/testdata/fixedbugs/issue70150.go b/src/internal/types/testdata/fixedbugs/issue70150.go new file mode 100644 index 00000000000000..5baf4a66306b05 --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue70150.go @@ -0,0 +1,15 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func _() { + var values []int + vf(values /* ERROR "(variable of type []int) as string value" */) + vf(values...) /* ERROR "have ([]int...)\n\twant (string, ...int)" */ + vf("ab", "cd", values /* ERROR "have (string, string, []int...)\n\twant (string, ...int)" */ ...) +} + +func vf(method string, values ...int) { +} diff --git a/src/internal/types/testdata/fixedbugs/issue70417.go b/src/internal/types/testdata/fixedbugs/issue70417.go new file mode 100644 index 00000000000000..74bdea3b8a4156 --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue70417.go @@ -0,0 +1,58 @@ +// -gotypesalias=1 + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type T[P any] struct{} + +// A0 +type A0 = T[int] +type B0 = *T[int] + +func (A0 /* ERROR "cannot define new methods on instantiated type T[int]" */) m() {} +func (*A0 /* ERROR "cannot define new methods on instantiated type T[int]" */) m() {} +func (B0 /* ERROR "cannot define new methods on instantiated type T[int]" */) m() {} + +// A1 +type A1[P any] = T[P] +type B1[P any] = *T[P] + +func (A1 /* ERROR "cannot define new methods on generic alias type A1[P any]" */ [P]) m() {} +func (*A1 /* ERROR "cannot define new methods on generic alias type A1[P any]" */ [P]) m() {} +func (B1 /* ERROR "cannot define new methods on generic alias type B1[P any]" */ [P]) m() {} + +// A2 +type A2[P any] = T[int] +type B2[P any] = *T[int] + +func (A2 /* ERROR "cannot define new methods on generic alias type A2[P any]" */ [P]) m() {} +func (*A2 /* ERROR "cannot define new methods on generic alias type A2[P any]" */ [P]) m() {} +func (B2 /* ERROR "cannot define new methods on generic alias type B2[P any]" */ [P]) m() {} + +// A3 +type A3 = T[int] +type B3 = *T[int] + +func (A3 /* ERROR "cannot define new methods on instantiated type T[int]" */) m() {} +func (*A3 /* ERROR "cannot define new methods on instantiated type T[int]" */) m() {} +func (B3 /* ERROR "cannot define new methods on instantiated type T[int]" */) m() {} + +// A4 +type A4 = T // ERROR "cannot use generic type T[P any] without instantiation" +type B4 = *T // ERROR "cannot use generic type T[P any] without instantiation" + +func (A4[P]) m1() {} // don't report a follow-on error on A4 +func (*A4[P]) m2() {} // don't report a follow-on error on A4 +func (B4[P]) m3() {} // don't report a follow-on error on B4 + +// instantiation in the middle of an alias chain +type S struct{} +type C0 = S +type C1[P any] = C0 +type C2 = *C1[int] + +func (C2 /* ERROR "cannot define new methods on instantiated type C1[int]" */) m() {} +func (*C2 /* ERROR "cannot define new methods on instantiated type C1[int]" */) m() {} diff --git a/src/internal/types/testdata/fixedbugs/issue70526.go b/src/internal/types/testdata/fixedbugs/issue70526.go new file mode 100644 index 00000000000000..56b20bfc3cbe6b --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue70526.go @@ -0,0 +1,13 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func f(...any) + +func _(x int, s []int) { + f(0, x /* ERROR "have (number, int...)\n\twant (...any)" */ ...) + f(0, s /* ERROR "have (number, []int...)\n\twant (...any)" */ ...) + f(0, 0 /* ERROR "have (number, number...)\n\twant (...any)" */ ...) +} diff --git a/src/internal/types/testdata/fixedbugs/issue70549.go b/src/internal/types/testdata/fixedbugs/issue70549.go new file mode 100644 index 00000000000000..fa54b458eed360 --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue70549.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +import "math" + +var ( + _ = math.Sqrt + _ = math.SQrt /* ERROR "undefined: math.SQrt (but have Sqrt)" */ + _ = math.sqrt /* ERROR "name sqrt not exported by package math" */ + _ = math.Foo /* ERROR "undefined: math.Foo" */ + _ = math.foo /* ERROR "undefined: math.foo" */ +) diff --git a/src/internal/types/testdata/fixedbugs/issue71131.go b/src/internal/types/testdata/fixedbugs/issue71131.go new file mode 100644 index 00000000000000..8e7575b02828de --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue71131.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func _() { + type Bool bool + for range func /* ERROR "yield func returns user-defined boolean, not bool" */ (func() Bool) {} { + } + for range func /* ERROR "yield func returns user-defined boolean, not bool" */ (func(int) Bool) {} { + } + for range func /* ERROR "yield func returns user-defined boolean, not bool" */ (func(int, string) Bool) {} { + } +} diff --git a/src/internal/types/testdata/fixedbugs/issue71198.go b/src/internal/types/testdata/fixedbugs/issue71198.go new file mode 100644 index 00000000000000..479f8e2b0c4e64 --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue71198.go @@ -0,0 +1,16 @@ +// -gotypesalias=1 + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type A[_ any] = any + +// This must not panic; also the error message must match the style for non-alias types, below. +func _[_ A /* ERROR "too many type arguments for type A: have 2, want 1" */ [int, string]]() {} + +type T[_ any] any + +func _[_ T /* ERROR "too many type arguments for type T: have 2, want 1" */ [int, string]]() {} diff --git a/src/internal/types/testdata/fixedbugs/issue71284.go b/src/internal/types/testdata/fixedbugs/issue71284.go new file mode 100644 index 00000000000000..4b73087a78a191 --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue71284.go @@ -0,0 +1,10 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package A + +type ( + _ = A + A /* ERROR "invalid recursive type: A refers to itself" */ = A +) diff --git a/src/internal/types/testdata/fixedbugs/issue72936.go b/src/internal/types/testdata/fixedbugs/issue72936.go new file mode 100644 index 00000000000000..eb4942c0d94fa0 --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue72936.go @@ -0,0 +1,23 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func _[C chan<- int | chan int](c C) { c <- 0 } +func _[C chan int | chan<- int](c C) { c <- 0 } +func _[C <-chan int | chan<- int](c C) { c <- /* ERROR "receive-only channel <-chan int" */ 0 } + +func _[C <-chan int | chan int](c C) { <-c } +func _[C chan int | <-chan int](c C) { <-c } +func _[C chan<- int | <-chan int](c C) { <-c /* ERROR "send-only channel chan<- int" */ } + +// from issue report + +func send[C interface{ ~chan<- V | ~chan V }, V any](c C, v V) { + c <- v +} + +func receive[C interface{ ~<-chan V | ~chan V }, V any](c C) V { + return <-c +} diff --git a/src/internal/types/testdata/fixedbugs/issue73428.go b/src/internal/types/testdata/fixedbugs/issue73428.go new file mode 100644 index 00000000000000..b452b90fe350ad --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue73428.go @@ -0,0 +1,13 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func f() {} + +const c = 0 + +var v int +var _ = f < c // ERROR "invalid operation: f < c (mismatched types func() and untyped int)" +var _ = f < v // ERROR "invalid operation: f < v (mismatched types func() and int)" diff --git a/src/internal/types/testdata/fixedbugs/issue75194.go b/src/internal/types/testdata/fixedbugs/issue75194.go new file mode 100644 index 00000000000000..ec2f9249ec9247 --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue75194.go @@ -0,0 +1,14 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type A /* ERROR "invalid recursive type: A refers to itself" */ struct { + a A +} + +type B /* ERROR "invalid recursive type: B refers to itself" */ struct { + a A + b B +} diff --git a/src/internal/types/testdata/fixedbugs/issue75986.go b/src/internal/types/testdata/fixedbugs/issue75986.go new file mode 100644 index 00000000000000..b2b1509e03babe --- /dev/null +++ b/src/internal/types/testdata/fixedbugs/issue75986.go @@ -0,0 +1,28 @@ +// -lang=go1.25 + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +import "strings" + +type T int +type G[P any] struct{} + +var x T + +// Verify that we don't get a version error when there's another error present in new(expr). + +func f() { + _ = new(U /* ERROR "undefined: U" */) + _ = new(strings.BUILDER /* ERROR "undefined: strings.BUILDER (but have Builder)" */) + _ = new(T) // ok + _ = new(G[int]) // ok + _ = new(G /* ERROR "cannot use generic type G without instantiation" */) + _ = new(nil /* ERROR "use of untyped nil in argument to new" */) + _ = new(comparable /* ERROR "cannot use type comparable outside a type constraint" */) + _ = new(new /* ERROR "new (built-in) must be called" */) + _ = new(panic /* ERROR "panic(0) (no value) used as value or type" */ (0)) +} diff --git a/src/internal/types/testdata/spec/comparisons.go b/src/internal/types/testdata/spec/comparisons.go index 492890e49e9734..9f2b247b8089c3 100644 --- a/src/internal/types/testdata/spec/comparisons.go +++ b/src/internal/types/testdata/spec/comparisons.go @@ -31,7 +31,7 @@ var ( func _() { _ = nil == nil // ERROR "operator == not defined on untyped nil" _ = b == b - _ = a /* ERROR "[10]func() cannot be compared" */ == a + _ = a /* ERROR "A cannot be compared" */ == a _ = l /* ERROR "slice can only be compared to nil" */ == l _ = s /* ERROR "struct containing []byte cannot be compared" */ == s _ = p == p @@ -108,13 +108,13 @@ func _[ _ = c == nil _ = b < b - _ = a /* ERROR "type parameter A is not comparable with <" */ < a - _ = l /* ERROR "type parameter L is not comparable with <" */ < l - _ = s /* ERROR "type parameter S is not comparable with <" */ < s - _ = p /* ERROR "type parameter P is not comparable with <" */ < p - _ = f /* ERROR "type parameter F is not comparable with <" */ < f - _ = i /* ERROR "type parameter I is not comparable with <" */ < i - _ = j /* ERROR "type parameter J is not comparable with <" */ < j - _ = m /* ERROR "type parameter M is not comparable with <" */ < m - _ = c /* ERROR "type parameter C is not comparable with <" */ < c + _ = a /* ERROR "type parameter A cannot use operator <" */ < a + _ = l /* ERROR "type parameter L cannot use operator <" */ < l + _ = s /* ERROR "type parameter S cannot use operator <" */ < s + _ = p /* ERROR "type parameter P cannot use operator <" */ < p + _ = f /* ERROR "type parameter F cannot use operator <" */ < f + _ = i /* ERROR "type parameter I cannot use operator <" */ < i + _ = j /* ERROR "type parameter J cannot use operator <" */ < j + _ = m /* ERROR "type parameter M cannot use operator <" */ < m + _ = c /* ERROR "type parameter C cannot use operator <" */ < c } diff --git a/src/internal/types/testdata/spec/range.go b/src/internal/types/testdata/spec/range.go index 9e32256fb74888..d77511ece05754 100644 --- a/src/internal/types/testdata/spec/range.go +++ b/src/internal/types/testdata/spec/range.go @@ -5,7 +5,7 @@ package p type MyInt int32 -type MyBool bool +type MyBool = bool // TODO(gri) remove alias declaration - see go.dev/issues/71131, go.dev/issues/71164 type MyString string type MyFunc1 func(func(int) bool) type MyFunc2 func(int) bool @@ -102,11 +102,11 @@ func test() { for mi, ms := range f8 { _, _ = mi, ms } - for i /* ERROR "cannot use i (value of type MyInt) as int value in assignment" */, s /* ERROR "cannot use s (value of type MyString) as string value in assignment" */ = range f8 { + for i /* ERROR "cannot use i (value of int32 type MyInt) as int value in assignment" */, s /* ERROR "cannot use s (value of string type MyString) as string value in assignment" */ = range f8 { _, _ = mi, ms } for mi, ms := range f8 { - i, s = mi /* ERROR "cannot use mi (variable of type MyInt) as int value in assignment" */, ms /* ERROR "cannot use ms (variable of type MyString) as string value in assignment" */ + i, s = mi /* ERROR "cannot use mi (variable of int32 type MyInt) as int value in assignment" */, ms /* ERROR "cannot use ms (variable of string type MyString) as string value in assignment" */ } for mi, ms = range f8 { _, _ = mi, ms @@ -129,13 +129,23 @@ func test() { } } +func _[T any](x T) { + for range x /* ERROR "cannot range over x (variable of type T constrained by any): no specific type" */ { + } +} + +func _[T interface{int; string}](x T) { + for range x /* ERROR "cannot range over x (variable of type T constrained by interface{int; string} with empty type set): no specific type" */ { + } +} + func _[T int | string](x T) { - for range x /* ERROR "cannot range over x (variable of type T constrained by int | string): no core type" */ { + for range x /* ERROR "cannot range over x (variable of type T constrained by int | string): int and string have different underlying types" */ { } } func _[T int | int64](x T) { - for range x /* ERROR "cannot range over x (variable of type T constrained by int | int64): no core type" */ { + for range x /* ERROR "cannot range over x (variable of type T constrained by int | int64): int and int64 have different underlying types" */ { } } @@ -154,6 +164,11 @@ func _[T ~func(func(int) bool)](x T) { } } +func _[T func() bool | func(int) bool]() { + for range func /* ERROR "func must be func(yield func(...) bool): in yield type, func() bool and func(int) bool have different underlying types" */ (T) {} { + } +} + // go.dev/issue/65236 func seq0(func() bool) {} diff --git a/src/internal/types/testdata/spec/range_int.go b/src/internal/types/testdata/spec/range_int.go index 766736cc1556ef..81b8ed62290d82 100644 --- a/src/internal/types/testdata/spec/range_int.go +++ b/src/internal/types/testdata/spec/range_int.go @@ -44,7 +44,7 @@ func _() { for i, j /* ERROR "range over 10 (untyped int constant) permits only one iteration variable" */ := range 10 { _, _ = i, j } - for i = range MyInt /* ERROR "cannot use MyInt(10) (constant 10 of type MyInt) as int value in range clause" */ (10) { + for i = range MyInt /* ERROR "cannot use MyInt(10) (constant 10 of int32 type MyInt) as int value in range clause" */ (10) { _ = i } for mi := range MyInt(10) { @@ -56,12 +56,12 @@ func _() { } func _[T int | string](x T) { - for range x /* ERROR "cannot range over x (variable of type T constrained by int | string): no core type" */ { + for range x /* ERROR "cannot range over x (variable of type T constrained by int | string): int and string have different underlying types" */ { } } func _[T int | int64](x T) { - for range x /* ERROR "cannot range over x (variable of type T constrained by int | int64): no core type" */ { + for range x /* ERROR "cannot range over x (variable of type T constrained by int | int64): int and int64 have different underlying types" */ { } } diff --git a/src/internal/types/testdata/spec/receivers.go b/src/internal/types/testdata/spec/receivers.go new file mode 100644 index 00000000000000..010c5511c1d0dc --- /dev/null +++ b/src/internal/types/testdata/spec/receivers.go @@ -0,0 +1,14 @@ +// -gotypesalias=1 + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package receivers + +// TODO(gri) add more tests checking the various restrictions on receivers + +type G[P any] struct{} +type A[P any] = G[P] + +func (a A /* ERROR "cannot define new methods on generic alias type A[P any]" */ [P]) m() {} diff --git a/src/internal/types/testdata/spec/typeAliases1.23b.go b/src/internal/types/testdata/spec/typeAliases1.23b.go index c92c3d3a7e0ea9..8a09899066fe18 100644 --- a/src/internal/types/testdata/spec/typeAliases1.23b.go +++ b/src/internal/types/testdata/spec/typeAliases1.23b.go @@ -1,4 +1,4 @@ -// -lang=go1.23 -gotypesalias=1 -goexperiment=aliastypeparams +// -lang=go1.23 -gotypesalias=1 // Copyright 2024 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/src/internal/weak/pointer.go b/src/internal/weak/pointer.go deleted file mode 100644 index 8e05af2d23f076..00000000000000 --- a/src/internal/weak/pointer.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* -The weak package is a package for managing weak pointers. - -Weak pointers are pointers that explicitly do not keep a value live and -must be queried for a regular Go pointer. -The result of such a query may be observed as nil at any point after a -weakly-pointed-to object becomes eligible for reclamation by the garbage -collector. -More specifically, weak pointers become nil as soon as the garbage collector -identifies that the object is unreachable, before it is made reachable -again by a finalizer. -In terms of the C# language, these semantics are roughly equivalent to the -the semantics of "short" weak references. -In terms of the Java language, these semantics are roughly equivalent to the -semantics of the WeakReference type. - -Using go:linkname to access this package and the functions it references -is explicitly forbidden by the toolchain because the semantics of this -package have not gone through the proposal process. By exposing this -functionality, we risk locking in the existing semantics due to Hyrum's Law. - -If you believe you have a good use-case for weak references not already -covered by the standard library, file a proposal issue at -https://github.com/golang/go/issues instead of relying on this package. -*/ -package weak - -import ( - "internal/abi" - "runtime" - "unsafe" -) - -// Pointer is a weak pointer to a value of type T. -// -// This value is comparable is guaranteed to compare equal if the pointers -// that they were created from compare equal. This property is retained even -// after the object referenced by the pointer used to create a weak reference -// is reclaimed. -// -// If multiple weak pointers are made to different offsets within same object -// (for example, pointers to different fields of the same struct), those pointers -// will not compare equal. -// If a weak pointer is created from an object that becomes reachable again due -// to a finalizer, that weak pointer will not compare equal with weak pointers -// created before it became unreachable. -type Pointer[T any] struct { - u unsafe.Pointer -} - -// Make creates a weak pointer from a strong pointer to some value of type T. -func Make[T any](ptr *T) Pointer[T] { - // Explicitly force ptr to escape to the heap. - ptr = abi.Escape(ptr) - - var u unsafe.Pointer - if ptr != nil { - u = runtime_registerWeakPointer(unsafe.Pointer(ptr)) - } - runtime.KeepAlive(ptr) - return Pointer[T]{u} -} - -// Strong creates a strong pointer from the weak pointer. -// Returns nil if the original value for the weak pointer was reclaimed by -// the garbage collector. -// If a weak pointer points to an object with a finalizer, then Strong will -// return nil as soon as the object's finalizer is queued for execution. -func (p Pointer[T]) Strong() *T { - return (*T)(runtime_makeStrongFromWeak(p.u)) -} - -// Implemented in runtime. - -//go:linkname runtime_registerWeakPointer -func runtime_registerWeakPointer(unsafe.Pointer) unsafe.Pointer - -//go:linkname runtime_makeStrongFromWeak -func runtime_makeStrongFromWeak(unsafe.Pointer) unsafe.Pointer diff --git a/src/internal/weak/pointer_test.go b/src/internal/weak/pointer_test.go deleted file mode 100644 index e143749230f0a5..00000000000000 --- a/src/internal/weak/pointer_test.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package weak_test - -import ( - "internal/weak" - "runtime" - "testing" -) - -type T struct { - // N.B. This must contain a pointer, otherwise the weak handle might get placed - // in a tiny block making the tests in this package flaky. - t *T - a int -} - -func TestPointer(t *testing.T) { - bt := new(T) - wt := weak.Make(bt) - if st := wt.Strong(); st != bt { - t.Fatalf("weak pointer is not the same as strong pointer: %p vs. %p", st, bt) - } - // bt is still referenced. - runtime.GC() - - if st := wt.Strong(); st != bt { - t.Fatalf("weak pointer is not the same as strong pointer after GC: %p vs. %p", st, bt) - } - // bt is no longer referenced. - runtime.GC() - - if st := wt.Strong(); st != nil { - t.Fatalf("expected weak pointer to be nil, got %p", st) - } -} - -func TestPointerEquality(t *testing.T) { - bt := make([]*T, 10) - wt := make([]weak.Pointer[T], 10) - for i := range bt { - bt[i] = new(T) - wt[i] = weak.Make(bt[i]) - } - for i := range bt { - st := wt[i].Strong() - if st != bt[i] { - t.Fatalf("weak pointer is not the same as strong pointer: %p vs. %p", st, bt[i]) - } - if wp := weak.Make(st); wp != wt[i] { - t.Fatalf("new weak pointer not equal to existing weak pointer: %v vs. %v", wp, wt[i]) - } - if i == 0 { - continue - } - if wt[i] == wt[i-1] { - t.Fatalf("expected weak pointers to not be equal to each other, but got %v", wt[i]) - } - } - // bt is still referenced. - runtime.GC() - for i := range bt { - st := wt[i].Strong() - if st != bt[i] { - t.Fatalf("weak pointer is not the same as strong pointer: %p vs. %p", st, bt[i]) - } - if wp := weak.Make(st); wp != wt[i] { - t.Fatalf("new weak pointer not equal to existing weak pointer: %v vs. %v", wp, wt[i]) - } - if i == 0 { - continue - } - if wt[i] == wt[i-1] { - t.Fatalf("expected weak pointers to not be equal to each other, but got %v", wt[i]) - } - } - bt = nil - // bt is no longer referenced. - runtime.GC() - for i := range bt { - st := wt[i].Strong() - if st != nil { - t.Fatalf("expected weak pointer to be nil, got %p", st) - } - if i == 0 { - continue - } - if wt[i] == wt[i-1] { - t.Fatalf("expected weak pointers to not be equal to each other, but got %v", wt[i]) - } - } -} - -func TestPointerFinalizer(t *testing.T) { - bt := new(T) - wt := weak.Make(bt) - done := make(chan struct{}, 1) - runtime.SetFinalizer(bt, func(bt *T) { - if wt.Strong() != nil { - t.Errorf("weak pointer did not go nil before finalizer ran") - } - done <- struct{}{} - }) - - // Make sure the weak pointer stays around while bt is live. - runtime.GC() - if wt.Strong() == nil { - t.Errorf("weak pointer went nil too soon") - } - runtime.KeepAlive(bt) - - // bt is no longer referenced. - // - // Run one cycle to queue the finalizer. - runtime.GC() - if wt.Strong() != nil { - t.Errorf("weak pointer did not go nil when finalizer was enqueued") - } - - // Wait for the finalizer to run. - <-done - - // The weak pointer should still be nil after the finalizer runs. - runtime.GC() - if wt.Strong() != nil { - t.Errorf("weak pointer is non-nil even after finalization: %v", wt) - } -} diff --git a/src/internal/zstd/fse_test.go b/src/internal/zstd/fse_test.go index 6f106b65b77814..20365745a5402e 100644 --- a/src/internal/zstd/fse_test.go +++ b/src/internal/zstd/fse_test.go @@ -68,7 +68,6 @@ func TestPredefinedTables(t *testing.T) { }, } for _, test := range tests { - test := test t.Run(test.name, func(t *testing.T) { var r Reader table := make([]fseEntry, 1<= chunk { - if len(skip) == 0 { - skip = make([]byte, chunk) - } - if _, err := io.ReadFull(r.r, skip); err != nil { - return r.wrapNonEOFError(relativeOffset, err) - } - relativeOffset += chunk - size -= chunk - } - if size > 0 { - if len(skip) == 0 { - skip = make([]byte, size) - } - if _, err := io.ReadFull(r.r, skip); err != nil { - return r.wrapNonEOFError(relativeOffset, err) - } - relativeOffset += int(size) + n, err := io.CopyN(io.Discard, r.r, int64(size)) + relativeOffset += int(n) + if err != nil { + return r.wrapNonEOFError(relativeOffset, err) } - r.blockOffset += int64(relativeOffset) - return nil } diff --git a/src/internal/zstd/zstd_test.go b/src/internal/zstd/zstd_test.go index f2a2e1b5851314..eb06af8a14b324 100644 --- a/src/internal/zstd/zstd_test.go +++ b/src/internal/zstd/zstd_test.go @@ -112,7 +112,6 @@ var tests = []struct { func TestSamples(t *testing.T) { for _, test := range tests { - test := test t.Run(test.name, func(t *testing.T) { r := NewReader(strings.NewReader(test.compressed)) got, err := io.ReadAll(r) @@ -131,7 +130,6 @@ func TestReset(t *testing.T) { input := strings.NewReader("") r := NewReader(input) for _, test := range tests { - test := test t.Run(test.name, func(t *testing.T) { input.Reset(test.compressed) r.Reset(input) diff --git a/src/io/fs/example_test.go b/src/io/fs/example_test.go index c9027034c4a5b5..e9ad2a3d83f629 100644 --- a/src/io/fs/example_test.go +++ b/src/io/fs/example_test.go @@ -9,8 +9,89 @@ import ( "io/fs" "log" "os" + "testing/fstest" ) +func ExampleGlob() { + fsys := fstest.MapFS{ + "file.txt": {}, + "file.go": {}, + "dir/file.txt": {}, + "dir/file.go": {}, + "dir/subdir/x.go": {}, + } + + patterns := []string{ + "*.txt", + "*.go", + "dir/*.go", + "dir/*/x.go", + } + + for _, pattern := range patterns { + matches, err := fs.Glob(fsys, pattern) + if err != nil { + log.Fatal(err) + } + fmt.Printf("%q matches: %v\n", pattern, matches) + } + + // Output: + // "*.txt" matches: [file.txt] + // "*.go" matches: [file.go] + // "dir/*.go" matches: [dir/file.go] + // "dir/*/x.go" matches: [dir/subdir/x.go] +} + +func ExampleReadFile() { + fsys := fstest.MapFS{ + "hello.txt": { + Data: []byte("Hello, World!\n"), + }, + } + + data, err := fs.ReadFile(fsys, "hello.txt") + if err != nil { + log.Fatal(err) + } + + fmt.Print(string(data)) + + // Output: + // Hello, World! +} + +func ExampleValidPath() { + paths := []string{ + ".", + "x", + "x/y/z", + "", + "..", + "/x", + "x/", + "x//y", + "x/./y", + "x/../y", + } + + for _, path := range paths { + fmt.Printf("ValidPath(%q) = %t\n", path, fs.ValidPath(path)) + } + + // Output: + // ValidPath(".") = true + // ValidPath("x") = true + // ValidPath("x/y/z") = true + // ValidPath("") = false + // ValidPath("..") = false + // ValidPath("/x") = false + // ValidPath("x/") = false + // ValidPath("x//y") = false + // ValidPath("x/./y") = false + // ValidPath("x/../y") = false +} + func ExampleWalkDir() { root := "/usr/local/go/bin" fileSystem := os.DirFS(root) diff --git a/src/io/fs/fs.go b/src/io/fs/fs.go index 6891d75a0e838a..fca07b818c9a3b 100644 --- a/src/io/fs/fs.go +++ b/src/io/fs/fs.go @@ -6,6 +6,19 @@ // A file system can be provided by the host operating system // but also by other packages. // +// # Path Names +// +// The interfaces in this package all operate on the same +// path name syntax, regardless of the host operating system. +// +// Path names are UTF-8-encoded, +// unrooted, slash-separated sequences of path elements, like “x/y/z”. +// Path names must not contain an element that is “.” or “..” or the empty string, +// except for the special case that the name "." may be used for the root directory. +// Paths must not start or end with a slash: “/x” and “x/” are invalid. +// +// # Testing +// // See the [testing/fstest] package for support with testing // implementations of file systems. package fs @@ -26,6 +39,7 @@ import ( // correctness. type FS interface { // Open opens the named file. + // [File.Close] must be called to release any associated resources. // // When Open returns an error, it should be of type *PathError // with the Op field set to "open", the Path field set to name, @@ -40,16 +54,13 @@ type FS interface { // ValidPath reports whether the given path name // is valid for use in a call to Open. // -// Path names passed to open are UTF-8-encoded, -// unrooted, slash-separated sequences of path elements, like “x/y/z”. -// Path names must not contain an element that is “.” or “..” or the empty string, -// except for the special case that the root directory is named “.”. -// Paths must not start or end with a slash: “/x” and “x/” are invalid. -// // Note that paths are slash-separated on all systems, even Windows. // Paths containing other characters such as backslash and colon // are accepted as valid, but those characters must never be // interpreted by an [FS] implementation as path element separators. +// See the [Path Names] section for more details. +// +// [Path Names]: https://pkg.go.dev/io/fs#hdr-Path_Names func ValidPath(name string) bool { if !utf8.ValidString(name) { return false @@ -128,7 +139,7 @@ type ReadDirFile interface { // At the end of a directory, the error is io.EOF. // (ReadDir must return io.EOF itself, not an error wrapping io.EOF.) // - // If n <= 0, ReadDir returns all the DirEntry values from the directory + // If n <= 0, ReadDir returns all remaining DirEntry values from the directory // in a single slice. In this case, if ReadDir succeeds (reads all the way // to the end of the directory), it returns the slice and a nil error. // If it encounters an error before the end of the directory, diff --git a/src/io/fs/readdir_test.go b/src/io/fs/readdir_test.go index 4c409ae7a010e2..b729bf27ac40ac 100644 --- a/src/io/fs/readdir_test.go +++ b/src/io/fs/readdir_test.go @@ -72,7 +72,6 @@ func TestFileInfoToDirEntry(t *testing.T) { } for _, test := range tests { - test := test t.Run(test.path, func(t *testing.T) { fi, err := Stat(testFs, test.path) if err != nil { @@ -94,8 +93,8 @@ func TestFileInfoToDirEntry(t *testing.T) { } func errorPath(err error) string { - var perr *PathError - if !errors.As(err, &perr) { + perr, ok := errors.AsType[*PathError](err) + if !ok { return "" } return perr.Path diff --git a/src/io/fs/readlink.go b/src/io/fs/readlink.go new file mode 100644 index 00000000000000..64340b9fb4c57b --- /dev/null +++ b/src/io/fs/readlink.go @@ -0,0 +1,45 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs + +// ReadLinkFS is the interface implemented by a file system +// that supports reading symbolic links. +type ReadLinkFS interface { + FS + + // ReadLink returns the destination of the named symbolic link. + // If there is an error, it should be of type [*PathError]. + ReadLink(name string) (string, error) + + // Lstat returns a [FileInfo] describing the named file. + // If the file is a symbolic link, the returned [FileInfo] describes the symbolic link. + // Lstat makes no attempt to follow the link. + // If there is an error, it should be of type [*PathError]. + Lstat(name string) (FileInfo, error) +} + +// ReadLink returns the destination of the named symbolic link. +// +// If fsys does not implement [ReadLinkFS], then ReadLink returns an error. +func ReadLink(fsys FS, name string) (string, error) { + sym, ok := fsys.(ReadLinkFS) + if !ok { + return "", &PathError{Op: "readlink", Path: name, Err: ErrInvalid} + } + return sym.ReadLink(name) +} + +// Lstat returns a [FileInfo] describing the named file. +// If the file is a symbolic link, the returned [FileInfo] describes the symbolic link. +// Lstat makes no attempt to follow the link. +// +// If fsys does not implement [ReadLinkFS], then Lstat is identical to [Stat]. +func Lstat(fsys FS, name string) (FileInfo, error) { + sym, ok := fsys.(ReadLinkFS) + if !ok { + return Stat(fsys, name) + } + return sym.Lstat(name) +} diff --git a/src/io/fs/readlink_test.go b/src/io/fs/readlink_test.go new file mode 100644 index 00000000000000..3932c7b7785daf --- /dev/null +++ b/src/io/fs/readlink_test.go @@ -0,0 +1,106 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs_test + +import ( + . "io/fs" + "testing" + "testing/fstest" +) + +func TestReadLink(t *testing.T) { + testFS := fstest.MapFS{ + "foo": { + Data: []byte("bar"), + Mode: ModeSymlink | 0o777, + }, + "bar": { + Data: []byte("Hello, World!\n"), + Mode: 0o644, + }, + + "dir/parentlink": { + Data: []byte("../bar"), + Mode: ModeSymlink | 0o777, + }, + "dir/link": { + Data: []byte("file"), + Mode: ModeSymlink | 0o777, + }, + "dir/file": { + Data: []byte("Hello, World!\n"), + Mode: 0o644, + }, + } + + check := func(fsys FS, name string, want string) { + t.Helper() + got, err := ReadLink(fsys, name) + if got != want || err != nil { + t.Errorf("ReadLink(%q) = %q, %v; want %q, ", name, got, err, want) + } + } + + check(testFS, "foo", "bar") + check(testFS, "dir/parentlink", "../bar") + check(testFS, "dir/link", "file") + + // Test that ReadLink on Sub works. + sub, err := Sub(testFS, "dir") + if err != nil { + t.Fatal(err) + } + + check(sub, "link", "file") + check(sub, "parentlink", "../bar") +} + +func TestLstat(t *testing.T) { + testFS := fstest.MapFS{ + "foo": { + Data: []byte("bar"), + Mode: ModeSymlink | 0o777, + }, + "bar": { + Data: []byte("Hello, World!\n"), + Mode: 0o644, + }, + + "dir/parentlink": { + Data: []byte("../bar"), + Mode: ModeSymlink | 0o777, + }, + "dir/link": { + Data: []byte("file"), + Mode: ModeSymlink | 0o777, + }, + "dir/file": { + Data: []byte("Hello, World!\n"), + Mode: 0o644, + }, + } + + check := func(fsys FS, name string, want FileMode) { + t.Helper() + info, err := Lstat(fsys, name) + var got FileMode + if err == nil { + got = info.Mode() + } + if got != want || err != nil { + t.Errorf("Lstat(%q) = %v, %v; want %v, ", name, got, err, want) + } + } + + check(testFS, "foo", ModeSymlink|0o777) + check(testFS, "bar", 0o644) + + // Test that Lstat on Sub works. + sub, err := Sub(testFS, "dir") + if err != nil { + t.Fatal(err) + } + check(sub, "link", ModeSymlink|0o777) +} diff --git a/src/io/fs/sub.go b/src/io/fs/sub.go index 70ac62307778ed..376d561bad8447 100644 --- a/src/io/fs/sub.go +++ b/src/io/fs/sub.go @@ -23,7 +23,8 @@ type SubFS interface { // Otherwise, if fs implements [SubFS], Sub returns fsys.Sub(dir). // Otherwise, Sub returns a new [FS] implementation sub that, // in effect, implements sub.Open(name) as fsys.Open(path.Join(dir, name)). -// The implementation also translates calls to ReadDir, ReadFile, and Glob appropriately. +// The implementation also translates calls to ReadDir, ReadFile, +// ReadLink, Lstat, and Glob appropriately. // // Note that Sub(os.DirFS("/"), "prefix") is equivalent to os.DirFS("/prefix") // and that neither of them guarantees to avoid operating system @@ -44,6 +45,12 @@ func Sub(fsys FS, dir string) (FS, error) { return &subFS{fsys, dir}, nil } +var _ FS = (*subFS)(nil) +var _ ReadDirFS = (*subFS)(nil) +var _ ReadFileFS = (*subFS)(nil) +var _ ReadLinkFS = (*subFS)(nil) +var _ GlobFS = (*subFS)(nil) + type subFS struct { fsys FS dir string @@ -105,6 +112,30 @@ func (f *subFS) ReadFile(name string) ([]byte, error) { return data, f.fixErr(err) } +func (f *subFS) ReadLink(name string) (string, error) { + full, err := f.fullName("readlink", name) + if err != nil { + return "", err + } + target, err := ReadLink(f.fsys, full) + if err != nil { + return "", f.fixErr(err) + } + return target, nil +} + +func (f *subFS) Lstat(name string) (FileInfo, error) { + full, err := f.fullName("lstat", name) + if err != nil { + return nil, err + } + info, err := Lstat(f.fsys, full) + if err != nil { + return nil, f.fixErr(err) + } + return info, nil +} + func (f *subFS) Glob(pattern string) ([]string, error) { // Check pattern is well-formed. if _, err := path.Match(pattern, ""); err != nil { diff --git a/src/io/fs/walk_test.go b/src/io/fs/walk_test.go index 4934df164b58c9..91f195f7be3df2 100644 --- a/src/io/fs/walk_test.go +++ b/src/io/fs/walk_test.go @@ -86,16 +86,7 @@ func mark(entry DirEntry, err error, errors *[]error, clear bool) error { } func TestWalkDir(t *testing.T) { - tmpDir := t.TempDir() - - origDir, err := os.Getwd() - if err != nil { - t.Fatal("finding working dir:", err) - } - if err = os.Chdir(tmpDir); err != nil { - t.Fatal("entering temp dir:", err) - } - defer os.Chdir(origDir) + t.Chdir(t.TempDir()) fsys := makeTree() errors := make([]error, 0, 10) @@ -104,7 +95,7 @@ func TestWalkDir(t *testing.T) { return mark(entry, err, &errors, clear) } // Expect no errors. - err = WalkDir(fsys, ".", markFn) + err := WalkDir(fsys, ".", markFn) if err != nil { t.Fatalf("no error expected, found: %s", err) } @@ -119,6 +110,47 @@ func TestWalkDir(t *testing.T) { }) } +func TestWalkDirSymlink(t *testing.T) { + fsys := fstest.MapFS{ + "link": {Data: []byte("dir"), Mode: ModeSymlink}, + "dir/a": {}, + "dir/b/c": {}, + "dir/d": {Data: []byte("b"), Mode: ModeSymlink}, + } + + wantTypes := map[string]FileMode{ + "link": ModeDir, + "link/a": 0, + "link/b": ModeDir, + "link/b/c": 0, + "link/d": ModeSymlink, + } + marks := make(map[string]int) + walkFn := func(path string, entry DirEntry, err error) error { + marks[path]++ + if want, ok := wantTypes[path]; !ok { + t.Errorf("Unexpected path %q in walk", path) + } else if got := entry.Type(); got != want { + t.Errorf("%s entry type = %v; want %v", path, got, want) + } + if err != nil { + t.Errorf("Walking %s: %v", path, err) + } + return nil + } + + // Expect no errors. + err := WalkDir(fsys, "link", walkFn) + if err != nil { + t.Fatalf("no error expected, found: %s", err) + } + for path := range wantTypes { + if got := marks[path]; got != 1 { + t.Errorf("%s visited %d times; expected 1", path, got) + } + } +} + func TestIssue51617(t *testing.T) { dir := t.TempDir() for _, sub := range []string{"a", filepath.Join("a", "bad"), filepath.Join("a", "next")} { diff --git a/src/io/io_test.go b/src/io/io_test.go index 9491ffae614c61..38bec8243e7baa 100644 --- a/src/io/io_test.go +++ b/src/io/io_test.go @@ -505,7 +505,7 @@ func TestNopCloserWriterToForwarding(t *testing.T) { func TestOffsetWriter_Seek(t *testing.T) { tmpfilename := "TestOffsetWriter_Seek" tmpfile, err := os.CreateTemp(t.TempDir(), tmpfilename) - if err != nil || tmpfile == nil { + if err != nil { t.Fatalf("CreateTemp(%s) failed: %v", tmpfilename, err) } defer tmpfile.Close() @@ -564,15 +564,12 @@ func TestOffsetWriter_Seek(t *testing.T) { func TestOffsetWriter_WriteAt(t *testing.T) { const content = "0123456789ABCDEF" contentSize := int64(len(content)) - tmpdir, err := os.MkdirTemp(t.TempDir(), "TestOffsetWriter_WriteAt") - if err != nil { - t.Fatal(err) - } + tmpdir := t.TempDir() work := func(off, at int64) { position := fmt.Sprintf("off_%d_at_%d", off, at) tmpfile, err := os.CreateTemp(tmpdir, position) - if err != nil || tmpfile == nil { + if err != nil { t.Fatalf("CreateTemp(%s) failed: %v", position, err) } defer tmpfile.Close() @@ -642,7 +639,7 @@ func TestOffsetWriter_Write(t *testing.T) { makeOffsetWriter := func(name string) (*OffsetWriter, *os.File) { tmpfilename := "TestOffsetWriter_Write_" + name tmpfile, err := os.CreateTemp(tmpdir, tmpfilename) - if err != nil || tmpfile == nil { + if err != nil { t.Fatalf("CreateTemp(%s) failed: %v", tmpfilename, err) } return NewOffsetWriter(tmpfile, 0), tmpfile diff --git a/src/io/ioutil/ioutil.go b/src/io/ioutil/ioutil.go index af8ebe38501d4c..0ab4b5a0c333d9 100644 --- a/src/io/ioutil/ioutil.go +++ b/src/io/ioutil/ioutil.go @@ -24,6 +24,8 @@ import ( // as an error to be reported. // // Deprecated: As of Go 1.16, this function simply calls [io.ReadAll]. +// +//go:fix inline func ReadAll(r io.Reader) ([]byte, error) { return io.ReadAll(r) } @@ -34,6 +36,8 @@ func ReadAll(r io.Reader) ([]byte, error) { // to be reported. // // Deprecated: As of Go 1.16, this function simply calls [os.ReadFile]. +// +//go:fix inline func ReadFile(filename string) ([]byte, error) { return os.ReadFile(filename) } @@ -43,6 +47,8 @@ func ReadFile(filename string) ([]byte, error) { // (before umask); otherwise WriteFile truncates it before writing, without changing permissions. // // Deprecated: As of Go 1.16, this function simply calls [os.WriteFile]. +// +//go:fix inline func WriteFile(filename string, data []byte, perm fs.FileMode) error { return os.WriteFile(filename, data, perm) } @@ -87,6 +93,8 @@ func ReadDir(dirname string) ([]fs.FileInfo, error) { // the provided Reader r. // // Deprecated: As of Go 1.16, this function simply calls [io.NopCloser]. +// +//go:fix inline func NopCloser(r io.Reader) io.ReadCloser { return io.NopCloser(r) } diff --git a/src/io/ioutil/tempfile.go b/src/io/ioutil/tempfile.go index 47b2e4012f7622..ef2ce404d9ae8f 100644 --- a/src/io/ioutil/tempfile.go +++ b/src/io/ioutil/tempfile.go @@ -21,6 +21,8 @@ import ( // to remove the file when no longer needed. // // Deprecated: As of Go 1.17, this function simply calls [os.CreateTemp]. +// +//go:fix inline func TempFile(dir, pattern string) (f *os.File, err error) { return os.CreateTemp(dir, pattern) } @@ -36,6 +38,8 @@ func TempFile(dir, pattern string) (f *os.File, err error) { // to remove the directory when no longer needed. // // Deprecated: As of Go 1.17, this function simply calls [os.MkdirTemp]. +// +//go:fix inline func TempDir(dir, pattern string) (name string, err error) { return os.MkdirTemp(dir, pattern) } diff --git a/src/io/multi_test.go b/src/io/multi_test.go index 7a24a8afc5a419..934a6ec785fda4 100644 --- a/src/io/multi_test.go +++ b/src/io/multi_test.go @@ -332,9 +332,7 @@ func TestMultiReaderFreesExhaustedReaders(t *testing.T) { buf1 := bytes.NewReader([]byte("foo")) buf2 := bytes.NewReader([]byte("bar")) mr = MultiReader(buf1, buf2) - runtime.SetFinalizer(buf1, func(*bytes.Reader) { - close(closed) - }) + runtime.AddCleanup(buf1, func(ch chan struct{}) { close(ch) }, closed) }() buf := make([]byte, 4) diff --git a/src/iter/iter.go b/src/iter/iter.go index 14fd8f8115f3f6..3d0caebf1ed5c5 100644 --- a/src/iter/iter.go +++ b/src/iter/iter.go @@ -28,7 +28,24 @@ or index-value pairs. Yield returns true if the iterator should continue with the next element in the sequence, false if it should stop. -Iterator functions are most often called by a range loop, as in: +Yield panics if called after it returns false. + +For instance, [maps.Keys] returns an iterator that produces the sequence +of keys of the map m, implemented as follows: + + func Keys[Map ~map[K]V, K comparable, V any](m Map) iter.Seq[K] { + return func(yield func(K) bool) { + for k := range m { + if !yield(k) { + return + } + } + } + } + +Further examples can be found in [The Go Blog: Range Over Function Types]. + +Iterator functions are most often called by a [range loop], as in: func PrintAll[V any](seq iter.Seq[V]) { for v := range seq { @@ -165,7 +182,7 @@ with the extra operations and then provide an iterator over positions. For example, a tree implementation might provide: // Positions returns an iterator over positions in the sequence. - func (t *Tree[V]) Positions() iter.Seq[*Pos] + func (t *Tree[V]) Positions() iter.Seq[*Pos[V]] // A Pos represents a position in the sequence. // It is only valid during the yield call it is passed to. @@ -187,6 +204,9 @@ And then a client could delete boring values from the tree using: p.Delete() } } + +[The Go Blog: Range Over Function Types]: https://go.dev/blog/range-functions +[range loop]: https://go.dev/ref/spec#For_range */ package iter @@ -239,91 +259,91 @@ func coroswitch(*coro) // If the iterator panics during a call to next (or stop), // then next (or stop) itself panics with the same value. func Pull[V any](seq Seq[V]) (next func() (V, bool), stop func()) { - var ( + var pull struct { v V ok bool done bool yieldNext bool + seqDone bool // to detect Goexit racer int panicValue any - seqDone bool // to detect Goexit - ) + } c := newcoro(func(c *coro) { - race.Acquire(unsafe.Pointer(&racer)) - if done { - race.Release(unsafe.Pointer(&racer)) + race.Acquire(unsafe.Pointer(&pull.racer)) + if pull.done { + race.Release(unsafe.Pointer(&pull.racer)) return } yield := func(v1 V) bool { - if done { + if pull.done { return false } - if !yieldNext { + if !pull.yieldNext { panic("iter.Pull: yield called again before next") } - yieldNext = false - v, ok = v1, true - race.Release(unsafe.Pointer(&racer)) + pull.yieldNext = false + pull.v, pull.ok = v1, true + race.Release(unsafe.Pointer(&pull.racer)) coroswitch(c) - race.Acquire(unsafe.Pointer(&racer)) - return !done + race.Acquire(unsafe.Pointer(&pull.racer)) + return !pull.done } // Recover and propagate panics from seq. defer func() { if p := recover(); p != nil { - panicValue = p - } else if !seqDone { - panicValue = goexitPanicValue + pull.panicValue = p + } else if !pull.seqDone { + pull.panicValue = goexitPanicValue } - done = true // Invalidate iterator - race.Release(unsafe.Pointer(&racer)) + pull.done = true // Invalidate iterator + race.Release(unsafe.Pointer(&pull.racer)) }() seq(yield) var v0 V - v, ok = v0, false - seqDone = true + pull.v, pull.ok = v0, false + pull.seqDone = true }) next = func() (v1 V, ok1 bool) { - race.Write(unsafe.Pointer(&racer)) // detect races + race.Write(unsafe.Pointer(&pull.racer)) // detect races - if done { + if pull.done { return } - if yieldNext { + if pull.yieldNext { panic("iter.Pull: next called again before yield") } - yieldNext = true - race.Release(unsafe.Pointer(&racer)) + pull.yieldNext = true + race.Release(unsafe.Pointer(&pull.racer)) coroswitch(c) - race.Acquire(unsafe.Pointer(&racer)) + race.Acquire(unsafe.Pointer(&pull.racer)) // Propagate panics and goexits from seq. - if panicValue != nil { - if panicValue == goexitPanicValue { + if pull.panicValue != nil { + if pull.panicValue == goexitPanicValue { // Propagate runtime.Goexit from seq. runtime.Goexit() } else { - panic(panicValue) + panic(pull.panicValue) } } - return v, ok + return pull.v, pull.ok } stop = func() { - race.Write(unsafe.Pointer(&racer)) // detect races + race.Write(unsafe.Pointer(&pull.racer)) // detect races - if !done { - done = true - race.Release(unsafe.Pointer(&racer)) + if !pull.done { + pull.done = true + race.Release(unsafe.Pointer(&pull.racer)) coroswitch(c) - race.Acquire(unsafe.Pointer(&racer)) + race.Acquire(unsafe.Pointer(&pull.racer)) // Propagate panics and goexits from seq. - if panicValue != nil { - if panicValue == goexitPanicValue { + if pull.panicValue != nil { + if pull.panicValue == goexitPanicValue { // Propagate runtime.Goexit from seq. runtime.Goexit() } else { - panic(panicValue) + panic(pull.panicValue) } } } @@ -354,93 +374,93 @@ func Pull[V any](seq Seq[V]) (next func() (V, bool), stop func()) { // If the iterator panics during a call to next (or stop), // then next (or stop) itself panics with the same value. func Pull2[K, V any](seq Seq2[K, V]) (next func() (K, V, bool), stop func()) { - var ( + var pull struct { k K v V ok bool done bool yieldNext bool + seqDone bool racer int panicValue any - seqDone bool - ) + } c := newcoro(func(c *coro) { - race.Acquire(unsafe.Pointer(&racer)) - if done { - race.Release(unsafe.Pointer(&racer)) + race.Acquire(unsafe.Pointer(&pull.racer)) + if pull.done { + race.Release(unsafe.Pointer(&pull.racer)) return } yield := func(k1 K, v1 V) bool { - if done { + if pull.done { return false } - if !yieldNext { + if !pull.yieldNext { panic("iter.Pull2: yield called again before next") } - yieldNext = false - k, v, ok = k1, v1, true - race.Release(unsafe.Pointer(&racer)) + pull.yieldNext = false + pull.k, pull.v, pull.ok = k1, v1, true + race.Release(unsafe.Pointer(&pull.racer)) coroswitch(c) - race.Acquire(unsafe.Pointer(&racer)) - return !done + race.Acquire(unsafe.Pointer(&pull.racer)) + return !pull.done } // Recover and propagate panics from seq. defer func() { if p := recover(); p != nil { - panicValue = p - } else if !seqDone { - panicValue = goexitPanicValue + pull.panicValue = p + } else if !pull.seqDone { + pull.panicValue = goexitPanicValue } - done = true // Invalidate iterator. - race.Release(unsafe.Pointer(&racer)) + pull.done = true // Invalidate iterator. + race.Release(unsafe.Pointer(&pull.racer)) }() seq(yield) var k0 K var v0 V - k, v, ok = k0, v0, false - seqDone = true + pull.k, pull.v, pull.ok = k0, v0, false + pull.seqDone = true }) next = func() (k1 K, v1 V, ok1 bool) { - race.Write(unsafe.Pointer(&racer)) // detect races + race.Write(unsafe.Pointer(&pull.racer)) // detect races - if done { + if pull.done { return } - if yieldNext { + if pull.yieldNext { panic("iter.Pull2: next called again before yield") } - yieldNext = true - race.Release(unsafe.Pointer(&racer)) + pull.yieldNext = true + race.Release(unsafe.Pointer(&pull.racer)) coroswitch(c) - race.Acquire(unsafe.Pointer(&racer)) + race.Acquire(unsafe.Pointer(&pull.racer)) // Propagate panics and goexits from seq. - if panicValue != nil { - if panicValue == goexitPanicValue { + if pull.panicValue != nil { + if pull.panicValue == goexitPanicValue { // Propagate runtime.Goexit from seq. runtime.Goexit() } else { - panic(panicValue) + panic(pull.panicValue) } } - return k, v, ok + return pull.k, pull.v, pull.ok } stop = func() { - race.Write(unsafe.Pointer(&racer)) // detect races + race.Write(unsafe.Pointer(&pull.racer)) // detect races - if !done { - done = true - race.Release(unsafe.Pointer(&racer)) + if !pull.done { + pull.done = true + race.Release(unsafe.Pointer(&pull.racer)) coroswitch(c) - race.Acquire(unsafe.Pointer(&racer)) + race.Acquire(unsafe.Pointer(&pull.racer)) // Propagate panics and goexits from seq. - if panicValue != nil { - if panicValue == goexitPanicValue { + if pull.panicValue != nil { + if pull.panicValue == goexitPanicValue { // Propagate runtime.Goexit from seq. runtime.Goexit() } else { - panic(panicValue) + panic(pull.panicValue) } } } diff --git a/src/iter/pull_test.go b/src/iter/pull_test.go index 449edee031dcd7..c66e20897bd8af 100644 --- a/src/iter/pull_test.go +++ b/src/iter/pull_test.go @@ -193,7 +193,11 @@ func doDoubleNext2() Seq2[int, int] { } func TestPullDoubleYield(t *testing.T) { - _, stop := Pull(storeYield()) + next, stop := Pull(storeYield()) + next() + if yieldSlot == nil { + t.Fatal("yield failed") + } defer func() { if recover() != nil { yieldSlot = nil @@ -218,7 +222,11 @@ func storeYield() Seq[int] { var yieldSlot func(int) bool func TestPullDoubleYield2(t *testing.T) { - _, stop := Pull2(storeYield2()) + next, stop := Pull2(storeYield2()) + next() + if yieldSlot2 == nil { + t.Fatal("yield failed") + } defer func() { if recover() != nil { yieldSlot2 = nil @@ -483,3 +491,19 @@ func TestPull2ImmediateStop(t *testing.T) { t.Fatal("next returned true after iterator was stopped") } } + +func BenchmarkPull(b *testing.B) { + seq := count(1) + for range b.N { + _, stop := Pull(seq) + stop() + } +} + +func BenchmarkPull2(b *testing.B) { + seq := squares(1) + for range b.N { + _, stop := Pull2(seq) + stop() + } +} diff --git a/src/log/log.go b/src/log/log.go index d4c9c1378fe15f..c79b3a9b7428db 100644 --- a/src/log/log.go +++ b/src/log/log.go @@ -191,8 +191,7 @@ func putBuffer(p *[]byte) { // provided for generality, although at the moment on all pre-defined // paths it will be 2. func (l *Logger) Output(calldepth int, s string) error { - calldepth++ // +1 for this frame. - return l.output(0, calldepth, func(b []byte) []byte { + return l.output(0, calldepth+1, func(b []byte) []byte { // +1 for this frame. return append(b, s...) }) } @@ -280,40 +279,52 @@ func (l *Logger) Println(v ...any) { // Fatal is equivalent to l.Print() followed by a call to [os.Exit](1). func (l *Logger) Fatal(v ...any) { - l.Output(2, fmt.Sprint(v...)) + l.output(0, 2, func(b []byte) []byte { + return fmt.Append(b, v...) + }) os.Exit(1) } // Fatalf is equivalent to l.Printf() followed by a call to [os.Exit](1). func (l *Logger) Fatalf(format string, v ...any) { - l.Output(2, fmt.Sprintf(format, v...)) + l.output(0, 2, func(b []byte) []byte { + return fmt.Appendf(b, format, v...) + }) os.Exit(1) } // Fatalln is equivalent to l.Println() followed by a call to [os.Exit](1). func (l *Logger) Fatalln(v ...any) { - l.Output(2, fmt.Sprintln(v...)) + l.output(0, 2, func(b []byte) []byte { + return fmt.Appendln(b, v...) + }) os.Exit(1) } // Panic is equivalent to l.Print() followed by a call to panic(). func (l *Logger) Panic(v ...any) { s := fmt.Sprint(v...) - l.Output(2, s) + l.output(0, 2, func(b []byte) []byte { + return append(b, s...) + }) panic(s) } // Panicf is equivalent to l.Printf() followed by a call to panic(). func (l *Logger) Panicf(format string, v ...any) { s := fmt.Sprintf(format, v...) - l.Output(2, s) + l.output(0, 2, func(b []byte) []byte { + return append(b, s...) + }) panic(s) } // Panicln is equivalent to l.Println() followed by a call to panic(). func (l *Logger) Panicln(v ...any) { s := fmt.Sprintln(v...) - l.Output(2, s) + l.output(0, 2, func(b []byte) []byte { + return append(b, s...) + }) panic(s) } @@ -409,40 +420,52 @@ func Println(v ...any) { // Fatal is equivalent to [Print] followed by a call to [os.Exit](1). func Fatal(v ...any) { - std.Output(2, fmt.Sprint(v...)) + std.output(0, 2, func(b []byte) []byte { + return fmt.Append(b, v...) + }) os.Exit(1) } // Fatalf is equivalent to [Printf] followed by a call to [os.Exit](1). func Fatalf(format string, v ...any) { - std.Output(2, fmt.Sprintf(format, v...)) + std.output(0, 2, func(b []byte) []byte { + return fmt.Appendf(b, format, v...) + }) os.Exit(1) } // Fatalln is equivalent to [Println] followed by a call to [os.Exit](1). func Fatalln(v ...any) { - std.Output(2, fmt.Sprintln(v...)) + std.output(0, 2, func(b []byte) []byte { + return fmt.Appendln(b, v...) + }) os.Exit(1) } // Panic is equivalent to [Print] followed by a call to panic(). func Panic(v ...any) { s := fmt.Sprint(v...) - std.Output(2, s) + std.output(0, 2, func(b []byte) []byte { + return append(b, s...) + }) panic(s) } // Panicf is equivalent to [Printf] followed by a call to panic(). func Panicf(format string, v ...any) { s := fmt.Sprintf(format, v...) - std.Output(2, s) + std.output(0, 2, func(b []byte) []byte { + return append(b, s...) + }) panic(s) } // Panicln is equivalent to [Println] followed by a call to panic(). func Panicln(v ...any) { s := fmt.Sprintln(v...) - std.Output(2, s) + std.output(0, 2, func(b []byte) []byte { + return append(b, s...) + }) panic(s) } @@ -454,5 +477,7 @@ func Panicln(v ...any) { // if [Llongfile] or [Lshortfile] is set; a value of 1 will print the details // for the caller of Output. func Output(calldepth int, s string) error { - return std.Output(calldepth+1, s) // +1 for this frame. + return std.output(0, calldepth+1, func(b []byte) []byte { // +1 for this frame. + return append(b, s...) + }) } diff --git a/src/log/log_test.go b/src/log/log_test.go index c7fa78f5ad95be..5d5d38cc10dbae 100644 --- a/src/log/log_test.go +++ b/src/log/log_test.go @@ -7,10 +7,14 @@ package log // These tests are too simple. import ( + "bufio" "bytes" + "errors" "fmt" + "internal/testenv" "io" "os" + "os/exec" "regexp" "runtime" "strings" @@ -23,7 +27,7 @@ const ( Rdate = `[0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9]` Rtime = `[0-9][0-9]:[0-9][0-9]:[0-9][0-9]` Rmicroseconds = `\.[0-9][0-9][0-9][0-9][0-9][0-9]` - Rline = `(63|65):` // must update if the calls to l.Printf / l.Print below move + Rline = `(67|69):` // must update if the calls to l.Printf / l.Print below move Rlongfile = `.*/[A-Za-z0-9_\-]+\.go:` + Rline Rshortfile = `[A-Za-z0-9_\-]+\.go:` + Rline ) @@ -212,6 +216,86 @@ func TestDiscard(t *testing.T) { } } +func TestCallDepth(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } + + testenv.MustHaveExec(t) + ep, err := os.Executable() + if err != nil { + t.Fatalf("Executable failed: %v", err) + } + + tests := []struct { + name string + log func() + }{ + {"Fatal", func() { Fatal("Fatal") }}, + {"Fatalf", func() { Fatalf("Fatalf") }}, + {"Fatalln", func() { Fatalln("Fatalln") }}, + {"Output", func() { Output(1, "Output") }}, + {"Panic", func() { Panic("Panic") }}, + {"Panicf", func() { Panicf("Panicf") }}, + {"Panicln", func() { Panicf("Panicln") }}, + {"Default.Fatal", func() { Default().Fatal("Default.Fatal") }}, + {"Default.Fatalf", func() { Default().Fatalf("Default.Fatalf") }}, + {"Default.Fatalln", func() { Default().Fatalln("Default.Fatalln") }}, + {"Default.Output", func() { Default().Output(1, "Default.Output") }}, + {"Default.Panic", func() { Default().Panic("Default.Panic") }}, + {"Default.Panicf", func() { Default().Panicf("Default.Panicf") }}, + {"Default.Panicln", func() { Default().Panicf("Default.Panicln") }}, + } + + // calculate the line offset until the first test case + _, _, line, ok := runtime.Caller(0) + if !ok { + t.Fatalf("runtime.Caller failed") + } + line -= len(tests) + 3 + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // some of these calls uses os.Exit() to spawn a command and capture output + const envVar = "LOGTEST_CALL_DEPTH" + if os.Getenv(envVar) == "1" { + SetFlags(Lshortfile) + tt.log() + os.Exit(1) + } + + // spawn test executable + cmd := testenv.Command(t, ep, + "-test.run=^"+regexp.QuoteMeta(t.Name())+"$", + "-test.count=1", + ) + cmd.Env = append(cmd.Environ(), envVar+"=1") + + out, err := cmd.CombinedOutput() + if _, ok := errors.AsType[*exec.ExitError](err); !ok { + t.Fatalf("expected exec.ExitError: %v", err) + } + + _, firstLine, err := bufio.ScanLines(out, true) + if err != nil { + t.Fatalf("failed to split line: %v", err) + } + got := string(firstLine) + + want := fmt.Sprintf( + "log_test.go:%d: %s", + line+i, tt.name, + ) + if got != want { + t.Errorf( + "output from %s() mismatch:\n\t got: %s\n\twant: %s", + tt.name, got, want, + ) + } + }) + } +} + func BenchmarkItoa(b *testing.B) { dst := make([]byte, 0, 64) for i := 0; i < b.N; i++ { diff --git a/src/log/slog/attr.go b/src/log/slog/attr.go index 067c537cc973fc..c592e54eaf9aba 100644 --- a/src/log/slog/attr.go +++ b/src/log/slog/attr.go @@ -67,6 +67,15 @@ func Group(key string, args ...any) Attr { return Attr{key, GroupValue(argsToAttrSlice(args)...)} } +// GroupAttrs returns an Attr for a Group [Value] +// consisting of the given Attrs. +// +// GroupAttrs is a more efficient version of [Group] +// that accepts only [Attr] values. +func GroupAttrs(key string, attrs ...Attr) Attr { + return Attr{key, GroupValue(attrs...)} +} + func argsToAttrSlice(args []any) []Attr { var ( attr Attr diff --git a/src/log/slog/attr_test.go b/src/log/slog/attr_test.go index e01447cfedd19e..1f2ce2621b5b58 100644 --- a/src/log/slog/attr_test.go +++ b/src/log/slog/attr_test.go @@ -5,12 +5,16 @@ package slog import ( + "internal/asan" "internal/testenv" "testing" "time" ) func TestAttrNoAlloc(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates with -asan") + } testenv.SkipIfOptimizationOff(t) // Assign values just to make sure the compiler doesn't optimize away the statements. var ( diff --git a/src/log/slog/example_discard_test.go b/src/log/slog/example_discard_test.go new file mode 100644 index 00000000000000..7b829df181ec58 --- /dev/null +++ b/src/log/slog/example_discard_test.go @@ -0,0 +1,32 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package slog_test + +import ( + "log/slog" + "os" +) + +func Example_discardHandler() { + removeTime := func(groups []string, a slog.Attr) slog.Attr { + if a.Key == slog.TimeKey && len(groups) == 0 { + return slog.Attr{} + } + return a + } + // A slog.TextHandler can output log messages. + logger1 := slog.New(slog.NewTextHandler( + os.Stdout, + &slog.HandlerOptions{ReplaceAttr: removeTime}, + )) + logger1.Info("message 1") + + // A slog.DiscardHandler will discard all messages. + logger2 := slog.New(slog.DiscardHandler) + logger2.Info("message 2") + + // Output: + // level=INFO msg="message 1" +} diff --git a/src/log/slog/example_level_handler_test.go b/src/log/slog/example_level_handler_test.go index 1ff91d47635faa..97b2f3abda4318 100644 --- a/src/log/slog/example_level_handler_test.go +++ b/src/log/slog/example_level_handler_test.go @@ -7,7 +7,6 @@ package slog_test import ( "context" "log/slog" - "log/slog/internal/slogtest" "os" ) @@ -63,7 +62,13 @@ func (h *LevelHandler) Handler() slog.Handler { // Another typical use would be to decrease the log level (to LevelDebug, say) // during a part of the program that was suspected of containing a bug. func ExampleHandler_levelHandler() { - th := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ReplaceAttr: slogtest.RemoveTime}) + removeTime := func(groups []string, a slog.Attr) slog.Attr { + if a.Key == slog.TimeKey && len(groups) == 0 { + return slog.Attr{} + } + return a + } + th := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ReplaceAttr: removeTime}) logger := slog.New(NewLevelHandler(slog.LevelWarn, th)) logger.Info("not printed") logger.Warn("printed") diff --git a/src/log/slog/example_log_level_test.go b/src/log/slog/example_log_level_test.go index ca8db416e5cdfb..258851b762972f 100644 --- a/src/log/slog/example_log_level_test.go +++ b/src/log/slog/example_log_level_test.go @@ -7,7 +7,6 @@ package slog_test import ( "log" "log/slog" - "log/slog/internal/slogtest" "os" ) @@ -49,7 +48,13 @@ func ExampleSetLogLoggerLevel_slog() { defer slog.SetLogLoggerLevel(currentLogLevel) // revert changes after the example defer slog.SetDefault(slog.Default()) // revert changes after the example - slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ReplaceAttr: slogtest.RemoveTime}))) + removeTime := func(groups []string, a slog.Attr) slog.Attr { + if a.Key == slog.TimeKey && len(groups) == 0 { + return slog.Attr{} + } + return a + } + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ReplaceAttr: removeTime}))) log.Print("error") // level=ERROR msg=error diff --git a/src/log/slog/example_logvaluer_secret_test.go b/src/log/slog/example_logvaluer_secret_test.go index 51d002079383a5..da4c123617eca3 100644 --- a/src/log/slog/example_logvaluer_secret_test.go +++ b/src/log/slog/example_logvaluer_secret_test.go @@ -6,7 +6,6 @@ package slog_test import ( "log/slog" - "log/slog/internal/slogtest" "os" ) @@ -23,7 +22,13 @@ func (Token) LogValue() slog.Value { // with an alternative representation to avoid revealing secrets. func ExampleLogValuer_secret() { t := Token("shhhh!") - logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ReplaceAttr: slogtest.RemoveTime})) + removeTime := func(groups []string, a slog.Attr) slog.Attr { + if a.Key == slog.TimeKey && len(groups) == 0 { + return slog.Attr{} + } + return a + } + logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ReplaceAttr: removeTime})) logger.Info("permission granted", "user", "Perry", "token", t) // Output: diff --git a/src/log/slog/example_multi_handler_test.go b/src/log/slog/example_multi_handler_test.go new file mode 100644 index 00000000000000..daba82c47d72df --- /dev/null +++ b/src/log/slog/example_multi_handler_test.go @@ -0,0 +1,39 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package slog_test + +import ( + "bytes" + "log/slog" + "os" +) + +func ExampleMultiHandler() { + removeTime := func(groups []string, a slog.Attr) slog.Attr { + if a.Key == slog.TimeKey && len(groups) == 0 { + return slog.Attr{} + } + return a + } + + var textBuf, jsonBuf bytes.Buffer + textHandler := slog.NewTextHandler(&textBuf, &slog.HandlerOptions{ReplaceAttr: removeTime}) + jsonHandler := slog.NewJSONHandler(&jsonBuf, &slog.HandlerOptions{ReplaceAttr: removeTime}) + + multiHandler := slog.NewMultiHandler(textHandler, jsonHandler) + logger := slog.New(multiHandler) + + logger.Info("login", + slog.String("name", "whoami"), + slog.Int("id", 42), + ) + + os.Stdout.WriteString(textBuf.String()) + os.Stdout.WriteString(jsonBuf.String()) + + // Output: + // level=INFO msg=login name=whoami id=42 + // {"level":"INFO","msg":"login","name":"whoami","id":42} +} diff --git a/src/log/slog/example_test.go b/src/log/slog/example_test.go index b03cc01066ccca..c8a05a7bd520ad 100644 --- a/src/log/slog/example_test.go +++ b/src/log/slog/example_test.go @@ -5,6 +5,7 @@ package slog_test import ( + "context" "log/slog" "net/http" "os" @@ -35,3 +36,46 @@ func ExampleGroup() { // Output: // level=INFO msg=finished req.method=GET req.url=localhost status=200 duration=1s } + +func ExampleGroupAttrs() { + r, _ := http.NewRequest("POST", "localhost", http.NoBody) + // ... + + logger := slog.New( + slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelDebug, + ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { + if a.Key == slog.TimeKey && len(groups) == 0 { + return slog.Attr{} + } + return a + }, + }), + ) + + // Use []slog.Attr to accumulate attributes. + attrs := []slog.Attr{slog.String("method", r.Method)} + attrs = append(attrs, slog.String("url", r.URL.String())) + + if r.Method == "POST" { + attrs = append(attrs, slog.Int("content-length", int(r.ContentLength))) + } + + // Group the attributes under a key. + logger.LogAttrs(context.Background(), slog.LevelInfo, + "finished", + slog.Int("status", http.StatusOK), + slog.GroupAttrs("req", attrs...), + ) + + // Groups with empty keys are inlined. + logger.LogAttrs(context.Background(), slog.LevelInfo, + "finished", + slog.Int("status", http.StatusOK), + slog.GroupAttrs("", attrs...), + ) + + // Output: + // level=INFO msg=finished status=200 req.method=POST req.url=localhost req.content-length=0 + // level=INFO msg=finished status=200 method=POST url=localhost content-length=0 +} diff --git a/src/log/slog/handler.go b/src/log/slog/handler.go index 2ff85b582ef32e..26eb4b82fc8b57 100644 --- a/src/log/slog/handler.go +++ b/src/log/slog/handler.go @@ -28,6 +28,8 @@ import ( // // Users of the slog package should not invoke Handler methods directly. // They should use the methods of [Logger] instead. +// +// Before implementing your own handler, consult https://go.dev/s/slog-handler-guide. type Handler interface { // Enabled reports whether the handler handles records at the given level. // The handler ignores records whose level is lower. @@ -57,6 +59,9 @@ type Handler interface { // - If a group's key is empty, inline the group's Attrs. // - If a group has no Attrs (even if it has a non-empty key), // ignore it. + // + // [Logger] discards any errors from Handle. Wrap the Handle method to + // process any errors from Handlers. Handle(context.Context, Record) error // WithAttrs returns a new Handler whose attributes consist of @@ -294,7 +299,11 @@ func (h *commonHandler) handle(r Record) error { } // source if h.opts.AddSource { - state.appendAttr(Any(SourceKey, r.source())) + src := r.Source() + if src == nil { + src = &Source{} + } + state.appendAttr(Any(SourceKey, src)) } key = MessageKey msg := r.Message @@ -483,6 +492,9 @@ func (s *handleState) appendAttr(a Attr) bool { // Special case: Source. if v := a.Value; v.Kind() == KindAny { if src, ok := v.Any().(*Source); ok { + if src.isEmpty() { + return false + } if s.h.json { a.Value = src.group() } else { @@ -525,8 +537,7 @@ func (s *handleState) appendError(err error) { func (s *handleState) appendKey(key string) { s.buf.WriteString(s.sep) if s.prefix != nil && len(*s.prefix) > 0 { - // TODO: optimize by avoiding allocation. - s.appendString(string(*s.prefix) + key) + s.appendTwoStrings(string(*s.prefix), key) } else { s.appendString(key) } @@ -538,6 +549,24 @@ func (s *handleState) appendKey(key string) { s.sep = s.h.attrSep() } +// appendTwoStrings implements appendString(prefix + key), but faster. +func (s *handleState) appendTwoStrings(x, y string) { + buf := *s.buf + switch { + case s.h.json: + buf.WriteByte('"') + buf = appendEscapedJSONString(buf, x) + buf = appendEscapedJSONString(buf, y) + buf.WriteByte('"') + case !needsQuoting(x) && !needsQuoting(y): + buf.WriteString(x) + buf.WriteString(y) + default: + buf = strconv.AppendQuote(buf, x+y) + } + *s.buf = buf +} + func (s *handleState) appendString(str string) { if s.h.json { s.buf.WriteByte('"') @@ -602,3 +631,14 @@ func appendRFC3339Millis(b []byte, t time.Time) []byte { b = append(b[:n+prefixLen], b[n+prefixLen+1:]...) // drop the 4th digit return b } + +// DiscardHandler discards all log output. +// DiscardHandler.Enabled returns false for all Levels. +var DiscardHandler Handler = discardHandler{} + +type discardHandler struct{} + +func (dh discardHandler) Enabled(context.Context, Level) bool { return false } +func (dh discardHandler) Handle(context.Context, Record) error { return nil } +func (dh discardHandler) WithAttrs(attrs []Attr) Handler { return dh } +func (dh discardHandler) WithGroup(name string) Handler { return dh } diff --git a/src/log/slog/handler_test.go b/src/log/slog/handler_test.go index 8ce34526d0ba44..3c4c36912c2f96 100644 --- a/src/log/slog/handler_test.go +++ b/src/log/slog/handler_test.go @@ -10,7 +10,10 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" + "log/slog/internal/buffer" + "os" "path/filepath" "slices" "strconv" @@ -528,9 +531,27 @@ func TestJSONAndTextHandlers(t *testing.T) { wantText: "name.first=Perry name.last=Platypus", wantJSON: `{"name":{"first":"Perry","last":"Platypus"}}`, }, + { + name: "group and key (or both) needs quoting", + replace: removeKeys(TimeKey, LevelKey), + attrs: []Attr{ + Group("prefix", + String(" needs quoting ", "v"), String("NotNeedsQuoting", "v"), + ), + Group("prefix needs quoting", + String(" needs quoting ", "v"), String("NotNeedsQuoting", "v"), + ), + }, + wantText: `msg=message "prefix. needs quoting "=v prefix.NotNeedsQuoting=v "prefix needs quoting. needs quoting "=v "prefix needs quoting.NotNeedsQuoting"=v`, + wantJSON: `{"msg":"message","prefix":{" needs quoting ":"v","NotNeedsQuoting":"v"},"prefix needs quoting":{" needs quoting ":"v","NotNeedsQuoting":"v"}}`, + }, } { r := NewRecord(testTime, LevelInfo, "message", callerPC(2)) - line := strconv.Itoa(r.source().Line) + source := r.Source() + if source == nil { + t.Fatal("source is nil") + } + line := strconv.Itoa(source.Line) r.AddAttrs(test.attrs...) var buf bytes.Buffer opts := HandlerOptions{ReplaceAttr: test.replace, AddSource: test.addSource} @@ -617,6 +638,40 @@ func TestHandlerEnabled(t *testing.T) { } } +func TestJSONAndTextHandlersWithUnavailableSource(t *testing.T) { + // Verify that a nil source does not cause a panic. + // and that the source is empty. + var buf bytes.Buffer + opts := &HandlerOptions{ + ReplaceAttr: removeKeys(LevelKey), + AddSource: true, + } + + for _, test := range []struct { + name string + h Handler + want string + }{ + {"text", NewTextHandler(&buf, opts), "msg=message"}, + {"json", NewJSONHandler(&buf, opts), `{"msg":"message"}`}, + } { + t.Run(test.name, func(t *testing.T) { + buf.Reset() + r := NewRecord(time.Time{}, LevelInfo, "message", 0) + err := test.h.Handle(t.Context(), r) + if err != nil { + t.Fatal(err) + } + + want := strings.TrimSpace(test.want) + got := strings.TrimSpace(buf.String()) + if got != want { + t.Errorf("\ngot %s\nwant %s", got, want) + } + }) + } +} + func TestSecondWith(t *testing.T) { // Verify that a second call to Logger.With does not corrupt // the original. @@ -711,3 +766,51 @@ func BenchmarkWriteTime(b *testing.B) { buf = appendRFC3339Millis(buf[:0], tm) } } + +func TestDiscardHandler(t *testing.T) { + ctx := context.Background() + stdout, stderr := os.Stdout, os.Stderr + os.Stdout, os.Stderr = nil, nil // panic on write + t.Cleanup(func() { + os.Stdout, os.Stderr = stdout, stderr + }) + + // Just ensure nothing panics during normal usage + l := New(DiscardHandler) + l.Info("msg", "a", 1, "b", 2) + l.Debug("bg", Int("a", 1), "b", 2) + l.Warn("w", Duration("dur", 3*time.Second)) + l.Error("bad", "a", 1) + l.Log(ctx, LevelWarn+1, "w", Int("a", 1), String("b", "two")) + l.LogAttrs(ctx, LevelInfo+1, "a b c", Int("a", 1), String("b", "two")) + l.Info("info", "a", []Attr{Int("i", 1)}) + l.Info("info", "a", GroupValue(Int("i", 1))) +} + +func BenchmarkAppendKey(b *testing.B) { + for _, size := range []int{5, 10, 30, 50, 100} { + for _, quoting := range []string{"no_quoting", "pre_quoting", "key_quoting", "both_quoting"} { + b.Run(fmt.Sprintf("%s_prefix_size_%d", quoting, size), func(b *testing.B) { + var ( + hs = NewJSONHandler(io.Discard, nil).newHandleState(buffer.New(), false, "") + prefix = bytes.Repeat([]byte("x"), size) + key = "key" + ) + + if quoting == "pre_quoting" || quoting == "both_quoting" { + prefix[0] = '"' + } + if quoting == "key_quoting" || quoting == "both_quoting" { + key = "ke\"" + } + + hs.prefix = (*buffer.Buffer)(&prefix) + + for b.Loop() { + hs.appendKey(key) + hs.buf.Reset() + } + }) + } + } +} diff --git a/src/log/slog/internal/benchmarks/benchmarks.go b/src/log/slog/internal/benchmarks/benchmarks.go index 3a28523beb44ae..a94ecfa7435663 100644 --- a/src/log/slog/internal/benchmarks/benchmarks.go +++ b/src/log/slog/internal/benchmarks/benchmarks.go @@ -31,12 +31,19 @@ import ( const testMessage = "Test logging, but use a somewhat realistic message length." +type event struct { + ID string + Index int + Flag bool +} + var ( testTime = time.Date(2022, time.May, 1, 0, 0, 0, 0, time.UTC) testString = "7e3b3b2aaeff56a7108fe11e154200dd/7819479873059528190" testInt = 32768 testDuration = 23 * time.Second testError = errors.New("fail") + testEvent = event{"abcdefgh", 65536, true} ) var testAttrs = []slog.Attr{ diff --git a/src/log/slog/internal/benchmarks/benchmarks_test.go b/src/log/slog/internal/benchmarks/benchmarks_test.go index 18643b73e6c8a0..3fec2612797f2e 100644 --- a/src/log/slog/internal/benchmarks/benchmarks_test.go +++ b/src/log/slog/internal/benchmarks/benchmarks_test.go @@ -80,12 +80,12 @@ func BenchmarkAttrs(b *testing.B) { slog.Int("status", testInt), slog.Duration("duration", testDuration), slog.Time("time", testTime), - slog.Any("error", testError), + slog.Any("event", testEvent), slog.String("string", testString), slog.Int("status", testInt), slog.Duration("duration", testDuration), slog.Time("time", testTime), - slog.Any("error", testError), + slog.Any("event", testEvent), ) }, }, @@ -103,37 +103,37 @@ func BenchmarkAttrs(b *testing.B) { slog.Int("status", testInt), slog.Duration("duration", testDuration), slog.Time("time", testTime), - slog.Any("error", testError), + slog.Any("event", testEvent), slog.String("string", testString), slog.Int("status", testInt), slog.Duration("duration", testDuration), slog.Time("time", testTime), - slog.Any("error", testError), + slog.Any("event", testEvent), slog.String("string", testString), slog.Int("status", testInt), slog.Duration("duration", testDuration), slog.Time("time", testTime), - slog.Any("error", testError), + slog.Any("event", testEvent), slog.String("string", testString), slog.Int("status", testInt), slog.Duration("duration", testDuration), slog.Time("time", testTime), - slog.Any("error", testError), + slog.Any("event", testEvent), slog.String("string", testString), slog.Int("status", testInt), slog.Duration("duration", testDuration), slog.Time("time", testTime), - slog.Any("error", testError), + slog.Any("event", testEvent), slog.String("string", testString), slog.Int("status", testInt), slog.Duration("duration", testDuration), slog.Time("time", testTime), - slog.Any("error", testError), + slog.Any("event", testEvent), slog.String("string", testString), slog.Int("status", testInt), slog.Duration("duration", testDuration), slog.Time("time", testTime), - slog.Any("error", testError), + slog.Any("event", testEvent), ) }, }, diff --git a/src/log/slog/internal/slogtest/slogtest.go b/src/log/slog/internal/slogtest/slogtest.go deleted file mode 100644 index d58766284430ce..00000000000000 --- a/src/log/slog/internal/slogtest/slogtest.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package slogtest contains support functions for testing slog. -package slogtest - -import "log/slog" - -// RemoveTime removes the top-level time attribute. -// It is intended to be used as a ReplaceAttr function, -// to make example output deterministic. -func RemoveTime(groups []string, a slog.Attr) slog.Attr { - if a.Key == slog.TimeKey && len(groups) == 0 { - return slog.Attr{} - } - return a -} diff --git a/src/log/slog/json_handler.go b/src/log/slog/json_handler.go index da3eae1a8ec0d3..9c7045cda91769 100644 --- a/src/log/slog/json_handler.go +++ b/src/log/slog/json_handler.go @@ -63,9 +63,7 @@ func (h *JSONHandler) WithGroup(name string) Handler { // Otherwise, the key is "time" // and the value is output as with json.Marshal. // -// If the Record's level is zero, the level is omitted. -// Otherwise, the key is "level" -// and the value of [Level.String] is output. +// The level's key is "level" and its value is the result of calling [Level.String]. // // If the AddSource option is set and source information is available, // the key is "source", and the value is a record of type [Source]. @@ -139,15 +137,40 @@ func appendJSONValue(s *handleState, v Value) error { return nil } -func appendJSONMarshal(buf *buffer.Buffer, v any) error { +type jsonEncoder struct { + buf *bytes.Buffer // Use a json.Encoder to avoid escaping HTML. - var bb bytes.Buffer - enc := json.NewEncoder(&bb) - enc.SetEscapeHTML(false) - if err := enc.Encode(v); err != nil { + json *json.Encoder +} + +var jsonEncoderPool = &sync.Pool{ + New: func() any { + enc := &jsonEncoder{ + buf: new(bytes.Buffer), + } + enc.json = json.NewEncoder(enc.buf) + enc.json.SetEscapeHTML(false) + return enc + }, +} + +func appendJSONMarshal(buf *buffer.Buffer, v any) error { + j := jsonEncoderPool.Get().(*jsonEncoder) + defer func() { + // To reduce peak allocation, return only smaller buffers to the pool. + const maxBufferSize = 16 << 10 + if j.buf.Cap() > maxBufferSize { + return + } + j.buf.Reset() + jsonEncoderPool.Put(j) + }() + + if err := j.json.Encode(v); err != nil { return err } - bs := bb.Bytes() + + bs := j.buf.Bytes() buf.Write(bs[:len(bs)-1]) // remove final newline return nil } diff --git a/src/log/slog/json_handler_test.go b/src/log/slog/json_handler_test.go index 65130f2426e70d..fd6b2e39d4bd02 100644 --- a/src/log/slog/json_handler_test.go +++ b/src/log/slog/json_handler_test.go @@ -142,6 +142,39 @@ func jsonValueString(v Value) string { return string(buf) } +func TestJSONAllocs(t *testing.T) { + ctx := t.Context() + l := New(NewJSONHandler(io.Discard, &HandlerOptions{})) + testErr := errors.New("an error occurred") + testEvent := struct { + ID int + Scope string + Enabled bool + }{ + 123456, "abcdefgh", true, + } + + t.Run("message", func(t *testing.T) { + wantAllocs(t, 0, func() { + l.LogAttrs(ctx, LevelInfo, + "hello world", + ) + }) + }) + t.Run("attrs", func(t *testing.T) { + wantAllocs(t, 1, func() { + l.LogAttrs(ctx, LevelInfo, + "hello world", + String("component", "subtest"), + Int("id", 67890), + Bool("flag", true), + Any("error", testErr), + Any("event", testEvent), + ) + }) + }) +} + func BenchmarkJSONHandler(b *testing.B) { for _, bench := range []struct { name string diff --git a/src/log/slog/level.go b/src/log/slog/level.go index 7cddf4cfba5ca2..9e812eb165b40a 100644 --- a/src/log/slog/level.go +++ b/src/log/slog/level.go @@ -61,7 +61,11 @@ func (l Level) String() string { if val == 0 { return base } - return fmt.Sprintf("%s%+d", base, val) + sval := strconv.Itoa(int(val)) + if val > 0 { + sval = "+" + sval + } + return base + sval } switch { @@ -98,10 +102,16 @@ func (l *Level) UnmarshalJSON(data []byte) error { return l.parse(s) } -// MarshalText implements [encoding.TextMarshaler] +// AppendText implements [encoding.TextAppender] // by calling [Level.String]. +func (l Level) AppendText(b []byte) ([]byte, error) { + return append(b, l.String()...), nil +} + +// MarshalText implements [encoding.TextMarshaler] +// by calling [Level.AppendText]. func (l Level) MarshalText() ([]byte, error) { - return []byte(l.String()), nil + return l.AppendText(nil) } // UnmarshalText implements [encoding.TextUnmarshaler]. @@ -172,10 +182,16 @@ func (v *LevelVar) String() string { return fmt.Sprintf("LevelVar(%s)", v.Level()) } +// AppendText implements [encoding.TextAppender] +// by calling [Level.AppendText]. +func (v *LevelVar) AppendText(b []byte) ([]byte, error) { + return v.Level().AppendText(b) +} + // MarshalText implements [encoding.TextMarshaler] -// by calling [Level.MarshalText]. +// by calling [LevelVar.AppendText]. func (v *LevelVar) MarshalText() ([]byte, error) { - return v.Level().MarshalText() + return v.AppendText(nil) } // UnmarshalText implements [encoding.TextUnmarshaler] diff --git a/src/log/slog/level_test.go b/src/log/slog/level_test.go index 217a0d7204915a..c9a1c16c036255 100644 --- a/src/log/slog/level_test.go +++ b/src/log/slog/level_test.go @@ -89,6 +89,19 @@ func TestLevelMarshalText(t *testing.T) { } } +func TestLevelAppendText(t *testing.T) { + buf := make([]byte, 4, 16) + want := LevelWarn - 3 + wantData := []byte("\x00\x00\x00\x00INFO+1") + data, err := want.AppendText(buf) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, wantData) { + t.Errorf("got %s, want %s", string(data), string(wantData)) + } +} + func TestLevelParse(t *testing.T) { for _, test := range []struct { in string @@ -162,6 +175,23 @@ func TestLevelVarMarshalText(t *testing.T) { } } +func TestLevelVarAppendText(t *testing.T) { + var v LevelVar + v.Set(LevelWarn) + buf := make([]byte, 4, 16) + data, err := v.AppendText(buf) + if err != nil { + t.Fatal(err) + } + var v2 LevelVar + if err := v2.UnmarshalText(data[4:]); err != nil { + t.Fatal(err) + } + if g, w := v2.Level(), LevelWarn; g != w { + t.Errorf("got %s, want %s", g, w) + } +} + func TestLevelVarFlag(t *testing.T) { fs := flag.NewFlagSet("test", flag.ContinueOnError) v := &LevelVar{} @@ -185,3 +215,25 @@ func TestLevelVarString(t *testing.T) { t.Errorf("got %q, want %q", got, want) } } + +func BenchmarkLevelString(b *testing.B) { + levels := []Level{ + 0, + LevelError, + LevelError + 2, + LevelError - 2, + LevelWarn, + LevelWarn - 1, + LevelInfo, + LevelInfo + 1, + LevelInfo - 3, + LevelDebug, + LevelDebug - 2, + } + b.ResetTimer() + for b.Loop() { + for _, level := range levels { + _ = level.String() + } + } +} diff --git a/src/log/slog/logger.go b/src/log/slog/logger.go index 10aa6a2b314feb..69e1cf9f152860 100644 --- a/src/log/slog/logger.go +++ b/src/log/slog/logger.go @@ -5,6 +5,7 @@ package slog import ( + "bytes" "context" "log" loginternal "log/internal" @@ -96,9 +97,7 @@ func (w *handlerWriter) Write(buf []byte) (int, error) { // Remove final newline. origLen := len(buf) // Report that the entire buf was written. - if len(buf) > 0 && buf[len(buf)-1] == '\n' { - buf = buf[:len(buf)-1] - } + buf = bytes.TrimSuffix(buf, []byte{'\n'}) r := NewRecord(time.Now(), level, string(buf), pc) return origLen, w.h.Handle(context.Background(), r) } @@ -239,6 +238,9 @@ func (l *Logger) ErrorContext(ctx context.Context, msg string, args ...any) { // It must always be called directly by an exported logging method // or function, because it uses a fixed call depth to obtain the pc. func (l *Logger) log(ctx context.Context, level Level, msg string, args ...any) { + if ctx == nil { + ctx = context.Background() + } if !l.Enabled(ctx, level) { return } @@ -251,14 +253,14 @@ func (l *Logger) log(ctx context.Context, level Level, msg string, args ...any) } r := NewRecord(time.Now(), level, msg, pc) r.Add(args...) - if ctx == nil { - ctx = context.Background() - } _ = l.Handler().Handle(ctx, r) } // logAttrs is like [Logger.log], but for methods that take ...Attr. func (l *Logger) logAttrs(ctx context.Context, level Level, msg string, attrs ...Attr) { + if ctx == nil { + ctx = context.Background() + } if !l.Enabled(ctx, level) { return } @@ -271,9 +273,6 @@ func (l *Logger) logAttrs(ctx context.Context, level Level, msg string, attrs .. } r := NewRecord(time.Now(), level, msg, pc) r.AddAttrs(attrs...) - if ctx == nil { - ctx = context.Background() - } _ = l.Handler().Handle(ctx, r) } diff --git a/src/log/slog/logger_test.go b/src/log/slog/logger_test.go index bb1c8a16ea46ac..edacef13a4d518 100644 --- a/src/log/slog/logger_test.go +++ b/src/log/slog/logger_test.go @@ -5,13 +5,20 @@ package slog import ( + "bufio" "bytes" "context" + "errors" + "fmt" + "internal/asan" + "internal/msan" "internal/race" "internal/testenv" "io" "log" loginternal "log/internal" + "os" + "os/exec" "path/filepath" "regexp" "runtime" @@ -183,7 +190,10 @@ func TestCallDepth(t *testing.T) { const wantFunc = "log/slog.TestCallDepth" const wantFile = "logger_test.go" wantLine := startLine + count*2 - got := h.r.source() + got := h.r.Source() + if got == nil { + t.Fatal("got nil source") + } gotFile := filepath.Base(got.File) if got.Function != wantFunc || gotFile != wantFile || got.Line != wantLine { t.Errorf("got (%s, %s, %d), want (%s, %s, %d)", @@ -227,9 +237,99 @@ func TestCallDepth(t *testing.T) { check(11) } +func TestCallDepthConnection(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } + + testenv.MustHaveExec(t) + ep, err := os.Executable() + if err != nil { + t.Fatalf("Executable failed: %v", err) + } + + tests := []struct { + name string + log func() + }{ + {"log.Fatal", func() { log.Fatal("log.Fatal") }}, + {"log.Fatalf", func() { log.Fatalf("log.Fatalf") }}, + {"log.Fatalln", func() { log.Fatalln("log.Fatalln") }}, + {"log.Output", func() { log.Output(1, "log.Output") }}, + {"log.Panic", func() { log.Panic("log.Panic") }}, + {"log.Panicf", func() { log.Panicf("log.Panicf") }}, + {"log.Panicln", func() { log.Panicf("log.Panicln") }}, + {"log.Default.Fatal", func() { log.Default().Fatal("log.Default.Fatal") }}, + {"log.Default.Fatalf", func() { log.Default().Fatalf("log.Default.Fatalf") }}, + {"log.Default.Fatalln", func() { log.Default().Fatalln("log.Default.Fatalln") }}, + {"log.Default.Output", func() { log.Default().Output(1, "log.Default.Output") }}, + {"log.Default.Panic", func() { log.Default().Panic("log.Default.Panic") }}, + {"log.Default.Panicf", func() { log.Default().Panicf("log.Default.Panicf") }}, + {"log.Default.Panicln", func() { log.Default().Panicf("log.Default.Panicln") }}, + } + + // calculate the line offset until the first test case + _, _, line, ok := runtime.Caller(0) + if !ok { + t.Fatalf("runtime.Caller failed") + } + line -= len(tests) + 3 + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // inside spawned test executable + const envVar = "SLOGTEST_CALL_DEPTH_CONNECTION" + if os.Getenv(envVar) == "1" { + h := NewTextHandler(os.Stderr, &HandlerOptions{ + AddSource: true, + ReplaceAttr: func(groups []string, a Attr) Attr { + if (a.Key == MessageKey || a.Key == SourceKey) && len(groups) == 0 { + return a + } + return Attr{} + }, + }) + SetDefault(New(h)) + log.SetFlags(log.Lshortfile) + tt.log() + os.Exit(1) + } + + // spawn test executable + cmd := testenv.Command(t, ep, + "-test.run=^"+regexp.QuoteMeta(t.Name())+"$", + "-test.count=1", + ) + cmd.Env = append(cmd.Environ(), envVar+"=1") + + out, err := cmd.CombinedOutput() + if _, ok := errors.AsType[*exec.ExitError](err); !ok { + t.Fatalf("expected exec.ExitError: %v", err) + } + + _, firstLine, err := bufio.ScanLines(out, true) + if err != nil { + t.Fatalf("failed to split line: %v", err) + } + got := string(firstLine) + + want := fmt.Sprintf( + `msg="logger_test.go:%d: %s"`, + line+i, tt.name, + ) + if got != want { + t.Errorf( + "output from %s() mismatch:\n\t got: %s\n\twant: %s", + tt.name, got, want, + ) + } + }) + } +} + func TestAlloc(t *testing.T) { ctx := context.Background() - dl := New(discardHandler{}) + dl := New(discardTestHandler{}) defer SetDefault(Default()) // restore SetDefault(dl) @@ -248,7 +348,7 @@ func TestAlloc(t *testing.T) { t.Run("2 pairs", func(t *testing.T) { s := "abc" i := 2000 - wantAllocs(t, 2, func() { + wantAllocs(t, 0, func() { dl.Info("hello", "n", i, "s", s, @@ -256,10 +356,10 @@ func TestAlloc(t *testing.T) { }) }) t.Run("2 pairs disabled inline", func(t *testing.T) { - l := New(discardHandler{disabled: true}) + l := New(DiscardHandler) s := "abc" i := 2000 - wantAllocs(t, 2, func() { + wantAllocs(t, 0, func() { l.Log(ctx, LevelInfo, "hello", "n", i, "s", s, @@ -267,7 +367,7 @@ func TestAlloc(t *testing.T) { }) }) t.Run("2 pairs disabled", func(t *testing.T) { - l := New(discardHandler{disabled: true}) + l := New(DiscardHandler) s := "abc" i := 2000 wantAllocs(t, 0, func() { @@ -283,7 +383,7 @@ func TestAlloc(t *testing.T) { s := "abc" i := 2000 d := time.Second - wantAllocs(t, 10, func() { + wantAllocs(t, 1, func() { dl.Info("hello", "n", i, "s", s, "d", d, "n", i, "s", s, "d", d, @@ -303,7 +403,7 @@ func TestAlloc(t *testing.T) { }) }) t.Run("attrs3 disabled", func(t *testing.T) { - logger := New(discardHandler{disabled: true}) + logger := New(DiscardHandler) wantAllocs(t, 0, func() { logger.LogAttrs(ctx, LevelInfo, "hello", Int("a", 1), String("b", "two"), Duration("c", time.Second)) }) @@ -566,18 +666,17 @@ func (c *captureHandler) clear() { c.r = Record{} } -type discardHandler struct { - disabled bool - attrs []Attr +type discardTestHandler struct { + attrs []Attr } -func (d discardHandler) Enabled(context.Context, Level) bool { return !d.disabled } -func (discardHandler) Handle(context.Context, Record) error { return nil } -func (d discardHandler) WithAttrs(as []Attr) Handler { +func (d discardTestHandler) Enabled(context.Context, Level) bool { return true } +func (discardTestHandler) Handle(context.Context, Record) error { return nil } +func (d discardTestHandler) WithAttrs(as []Attr) Handler { d.attrs = concat(d.attrs, as) return d } -func (h discardHandler) WithGroup(name string) Handler { +func (h discardTestHandler) WithGroup(name string) Handler { return h } @@ -644,8 +743,8 @@ func callerPC(depth int) uintptr { } func wantAllocs(t *testing.T, want int, f func()) { - if race.Enabled { - t.Skip("skipping test in race mode") + if race.Enabled || asan.Enabled || msan.Enabled { + t.Skip("skipping test in race, asan, and msan modes") } testenv.SkipIfOptimizationOff(t) t.Helper() diff --git a/src/log/slog/multi_handler.go b/src/log/slog/multi_handler.go new file mode 100644 index 00000000000000..4cc802b29ba653 --- /dev/null +++ b/src/log/slog/multi_handler.go @@ -0,0 +1,61 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package slog + +import ( + "context" + "errors" +) + +// NewMultiHandler creates a [MultiHandler] with the given Handlers. +func NewMultiHandler(handlers ...Handler) *MultiHandler { + h := make([]Handler, len(handlers)) + copy(h, handlers) + return &MultiHandler{multi: h} +} + +// MultiHandler is a [Handler] that invokes all the given Handlers. +// Its Enable method reports whether any of the handlers' Enabled methods return true. +// Its Handle, WithAttr and WithGroup methods call the corresponding method on each of the enabled handlers. +type MultiHandler struct { + multi []Handler +} + +func (h *MultiHandler) Enabled(ctx context.Context, l Level) bool { + for i := range h.multi { + if h.multi[i].Enabled(ctx, l) { + return true + } + } + return false +} + +func (h *MultiHandler) Handle(ctx context.Context, r Record) error { + var errs []error + for i := range h.multi { + if h.multi[i].Enabled(ctx, r.Level) { + if err := h.multi[i].Handle(ctx, r.Clone()); err != nil { + errs = append(errs, err) + } + } + } + return errors.Join(errs...) +} + +func (h *MultiHandler) WithAttrs(attrs []Attr) Handler { + handlers := make([]Handler, 0, len(h.multi)) + for i := range h.multi { + handlers = append(handlers, h.multi[i].WithAttrs(attrs)) + } + return &MultiHandler{multi: handlers} +} + +func (h *MultiHandler) WithGroup(name string) Handler { + handlers := make([]Handler, 0, len(h.multi)) + for i := range h.multi { + handlers = append(handlers, h.multi[i].WithGroup(name)) + } + return &MultiHandler{multi: handlers} +} diff --git a/src/log/slog/multi_handler_test.go b/src/log/slog/multi_handler_test.go new file mode 100644 index 00000000000000..86844a661b8bfd --- /dev/null +++ b/src/log/slog/multi_handler_test.go @@ -0,0 +1,139 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package slog + +import ( + "bytes" + "context" + "errors" + "testing" + "time" +) + +// mockFailingHandler is a handler that always returns an error +// from its Handle method. +type mockFailingHandler struct { + Handler + err error +} + +func (h *mockFailingHandler) Handle(ctx context.Context, r Record) error { + _ = h.Handler.Handle(ctx, r) + return h.err +} + +func TestMultiHandler(t *testing.T) { + t.Run("Handle sends log to all handlers", func(t *testing.T) { + var buf1, buf2 bytes.Buffer + h1 := NewTextHandler(&buf1, nil) + h2 := NewJSONHandler(&buf2, nil) + + multi := NewMultiHandler(h1, h2) + logger := New(multi) + + logger.Info("hello world", "user", "test") + + checkLogOutput(t, buf1.String(), "time="+textTimeRE+` level=INFO msg="hello world" user=test`) + checkLogOutput(t, buf2.String(), `{"time":"`+jsonTimeRE+`","level":"INFO","msg":"hello world","user":"test"}`) + }) + + t.Run("Enabled returns true if any handler is enabled", func(t *testing.T) { + h1 := NewTextHandler(&bytes.Buffer{}, &HandlerOptions{Level: LevelError}) + h2 := NewTextHandler(&bytes.Buffer{}, &HandlerOptions{Level: LevelInfo}) + + multi := NewMultiHandler(h1, h2) + + if !multi.Enabled(context.Background(), LevelInfo) { + t.Error("Enabled should be true for INFO level, but got false") + } + if !multi.Enabled(context.Background(), LevelError) { + t.Error("Enabled should be true for ERROR level, but got false") + } + }) + + t.Run("Enabled returns false if no handlers are enabled", func(t *testing.T) { + h1 := NewTextHandler(&bytes.Buffer{}, &HandlerOptions{Level: LevelError}) + h2 := NewTextHandler(&bytes.Buffer{}, &HandlerOptions{Level: LevelInfo}) + + multi := NewMultiHandler(h1, h2) + + if multi.Enabled(context.Background(), LevelDebug) { + t.Error("Enabled should be false for DEBUG level, but got true") + } + }) + + t.Run("WithAttrs propagates attributes to all handlers", func(t *testing.T) { + var buf1, buf2 bytes.Buffer + h1 := NewTextHandler(&buf1, nil) + h2 := NewJSONHandler(&buf2, nil) + + multi := NewMultiHandler(h1, h2).WithAttrs([]Attr{String("request_id", "123")}) + logger := New(multi) + + logger.Info("request processed") + + checkLogOutput(t, buf1.String(), "time="+textTimeRE+` level=INFO msg="request processed" request_id=123`) + checkLogOutput(t, buf2.String(), `{"time":"`+jsonTimeRE+`","level":"INFO","msg":"request processed","request_id":"123"}`) + }) + + t.Run("WithGroup propagates group to all handlers", func(t *testing.T) { + var buf1, buf2 bytes.Buffer + h1 := NewTextHandler(&buf1, &HandlerOptions{AddSource: false}) + h2 := NewJSONHandler(&buf2, &HandlerOptions{AddSource: false}) + + multi := NewMultiHandler(h1, h2).WithGroup("req") + logger := New(multi) + + logger.Info("user login", "user_id", 42) + + checkLogOutput(t, buf1.String(), "time="+textTimeRE+` level=INFO msg="user login" req.user_id=42`) + checkLogOutput(t, buf2.String(), `{"time":"`+jsonTimeRE+`","level":"INFO","msg":"user login","req":{"user_id":42}}`) + }) + + t.Run("Handle propagates errors from handlers", func(t *testing.T) { + errFail := errors.New("mock failing") + + var buf1, buf2 bytes.Buffer + h1 := NewTextHandler(&buf1, nil) + h2 := &mockFailingHandler{Handler: NewJSONHandler(&buf2, nil), err: errFail} + + multi := NewMultiHandler(h2, h1) + + err := multi.Handle(context.Background(), NewRecord(time.Now(), LevelInfo, "test message", 0)) + if !errors.Is(err, errFail) { + t.Errorf("Expected error: %v, but got: %v", errFail, err) + } + + checkLogOutput(t, buf1.String(), "time="+textTimeRE+` level=INFO msg="test message"`) + checkLogOutput(t, buf2.String(), `{"time":"`+jsonTimeRE+`","level":"INFO","msg":"test message"}`) + }) + + t.Run("Handle with no handlers", func(t *testing.T) { + multi := NewMultiHandler() + logger := New(multi) + + logger.Info("nothing") + + err := multi.Handle(context.Background(), NewRecord(time.Now(), LevelInfo, "test", 0)) + if err != nil { + t.Errorf("Handle with no sub-handlers should return nil, but got: %v", err) + } + }) +} + +// Test that NewMultiHandler copies the input slice and is insulated from future modification. +func TestNewMultiHandlerCopy(t *testing.T) { + var buf1 bytes.Buffer + h1 := NewTextHandler(&buf1, nil) + slice := []Handler{h1} + multi := NewMultiHandler(slice...) + slice[0] = nil + + err := multi.Handle(context.Background(), NewRecord(time.Now(), LevelInfo, "test message", 0)) + if err != nil { + t.Errorf("Expected nil error, but got: %v", err) + } + checkLogOutput(t, buf1.String(), "time="+textTimeRE+` level=INFO msg="test message"`) +} diff --git a/src/log/slog/record.go b/src/log/slog/record.go index 97c87019a6aff5..3b4e68ce766f5f 100644 --- a/src/log/slog/record.go +++ b/src/log/slog/record.go @@ -211,11 +211,17 @@ func (s *Source) group() Value { return GroupValue(as...) } -// source returns a Source for the log event. -// If the Record was created without the necessary information, -// or if the location is unavailable, it returns a non-nil *Source -// with zero fields. -func (r Record) source() *Source { +// isEmpty returns whether the Source struct is nil or only contains zero fields. +func (s *Source) isEmpty() bool { return s == nil || *s == Source{} } + +// Source returns a new Source for the log event using r's PC. +// If the PC field is zero, meaning the Record was created without the necessary information +// or the location is unavailable, then nil is returned. +func (r Record) Source() *Source { + if r.PC == 0 { + return nil + } + fs := runtime.CallersFrames([]uintptr{r.PC}) f, _ := fs.Next() return &Source{ diff --git a/src/log/slog/record_test.go b/src/log/slog/record_test.go index 931ab660418db0..939dc34ac8557d 100644 --- a/src/log/slog/record_test.go +++ b/src/log/slog/record_test.go @@ -39,24 +39,35 @@ func TestRecordAttrs(t *testing.T) { } func TestRecordSource(t *testing.T) { - // Zero call depth => empty *Source. + // Zero call depth => nil *Source. for _, test := range []struct { depth int wantFunction string wantFile string wantLinePositive bool + wantNil bool }{ - {0, "", "", false}, - {-16, "", "", false}, - {1, "log/slog.TestRecordSource", "record_test.go", true}, // 1: caller of NewRecord - {2, "testing.tRunner", "testing.go", true}, + {0, "", "", false, true}, + {-16, "", "", false, true}, + {1, "log/slog.TestRecordSource", "record_test.go", true, false}, // 1: caller of NewRecord + {2, "testing.tRunner", "testing.go", true, false}, } { var pc uintptr if test.depth > 0 { pc = callerPC(test.depth + 1) } r := NewRecord(time.Time{}, 0, "", pc) - got := r.source() + got := r.Source() + if test.wantNil { + if got != nil { + t.Errorf("depth %d: got non-nil Source, want nil", test.depth) + } + continue + } + if got == nil { + t.Errorf("depth %d: got nil Source, want non-nil", test.depth) + continue + } if i := strings.LastIndexByte(got.File, '/'); i >= 0 { got.File = got.File[i+1:] } diff --git a/src/log/slog/slogtest_test.go b/src/log/slog/slogtest_test.go index 4887cc8c77d1e9..23b27de8bbce13 100644 --- a/src/log/slog/slogtest_test.go +++ b/src/log/slog/slogtest_test.go @@ -43,7 +43,7 @@ func TestSlogtest(t *testing.T) { func parseLines(src []byte, parse func([]byte) (map[string]any, error)) ([]map[string]any, error) { var records []map[string]any - for _, line := range bytes.Split(src, []byte{'\n'}) { + for line := range bytes.SplitSeq(src, []byte{'\n'}) { if len(line) == 0 { continue } diff --git a/src/log/slog/text_handler.go b/src/log/slog/text_handler.go index 6819e633bb713f..5a0d0a4a7ed42b 100644 --- a/src/log/slog/text_handler.go +++ b/src/log/slog/text_handler.go @@ -62,9 +62,7 @@ func (h *TextHandler) WithGroup(name string) Handler { // Otherwise, the key is "time" // and the value is output in RFC3339 format with millisecond precision. // -// If the Record's level is zero, the level is omitted. -// Otherwise, the key is "level" -// and the value of [Level.String] is output. +// The level's key is "level" and its value is the result of calling [Level.String]. // // If the AddSource option is set and source information is available, // the key is "source" and the value is output as FILE:LINE. diff --git a/src/log/slog/value_test.go b/src/log/slog/value_test.go index 3e191589c5f6ae..4f405938ced42a 100644 --- a/src/log/slog/value_test.go +++ b/src/log/slog/value_test.go @@ -6,6 +6,7 @@ package slog import ( "fmt" + "internal/asan" "reflect" "strings" "testing" @@ -86,6 +87,10 @@ func TestValueString(t *testing.T) { } func TestValueNoAlloc(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } + // Assign values just to make sure the compiler doesn't optimize away the statements. var ( i int64 diff --git a/src/log/syslog/syslog.go b/src/log/syslog/syslog.go index 362dd950ba29b0..a7fa674db91e73 100644 --- a/src/log/syslog/syslog.go +++ b/src/log/syslog/syslog.go @@ -140,7 +140,7 @@ func Dial(network, raddr string, priority Priority, tag string) (*Writer, error) if err != nil { return nil, err } - return w, err + return w, nil } // connect makes a connection to the syslog server. diff --git a/src/log/syslog/syslog_test.go b/src/log/syslog/syslog_test.go index cec225f75131f2..0d94bcbc4cb41a 100644 --- a/src/log/syslog/syslog_test.go +++ b/src/log/syslog/syslog_test.go @@ -145,7 +145,6 @@ func TestWithSimulated(t *testing.T) { continue } - tr := tr t.Run(tr, func(t *testing.T) { t.Parallel() diff --git a/src/make.bash b/src/make.bash index 10e263b3d09d8b..d4e927dfda7c43 100755 --- a/src/make.bash +++ b/src/make.bash @@ -64,14 +64,14 @@ # timing information to this file. Useful for profiling where the # time goes when these scripts run. # -# GOROOT_BOOTSTRAP: A working Go tree >= Go 1.20.6 for bootstrap. +# GOROOT_BOOTSTRAP: A working Go tree >= Go 1.24.6 for bootstrap. # If $GOROOT_BOOTSTRAP/bin/go is missing, $(go env GOROOT) is -# tried for all "go" in $PATH. By default, one of $HOME/go1.20.6, -# $HOME/sdk/go1.20.6, or $HOME/go1.4, whichever exists, in that order. +# tried for all "go" in $PATH. By default, one of $HOME/go1.24.6, +# $HOME/sdk/go1.24.6, or $HOME/go1.4, whichever exists, in that order. # We still check $HOME/go1.4 to allow for build scripts that still hard-code # that name even though they put newer Go toolchains there. -bootgo=1.20.6 +bootgo=1.24.6 set -e diff --git a/src/make.bat b/src/make.bat index 53122cbaefd0c3..29105cd8a54d98 100644 --- a/src/make.bat +++ b/src/make.bat @@ -34,25 +34,12 @@ @echo off -:: Keep environment variables within this script -:: unless invoked with --no-local. -if x%1==x-no-local goto nolocal -if x%2==x-no-local goto nolocal -if x%3==x-no-local goto nolocal -if x%4==x-no-local goto nolocal -if x%1==x--no-local goto nolocal -if x%2==x--no-local goto nolocal -if x%3==x--no-local goto nolocal -if x%4==x--no-local goto nolocal setlocal -:nolocal -set GOBUILDFAIL=0 - -if exist make.bat goto ok -echo Must run make.bat from Go src directory. -goto fail -:ok +if not exist make.bat ( + echo Must run make.bat from Go src directory. + exit /b 1 +) :: Clean old generated file that will cause problems in the build. del /F ".\pkg\runtime\runtime_defs.go" 2>NUL @@ -73,25 +60,28 @@ if not exist ..\bin\tool mkdir ..\bin\tool :: Calculating GOROOT_BOOTSTRAP if not "x%GOROOT_BOOTSTRAP%"=="x" goto bootstrapset for /f "tokens=*" %%g in ('where go 2^>nul') do ( - if "x%GOROOT_BOOTSTRAP%"=="x" ( - setlocal - call :nogoenv - for /f "tokens=*" %%i in ('"%%g" env GOROOT 2^>nul') do ( - endlocal - if /I not "%%i"=="%GOROOT_TEMP%" ( - set GOROOT_BOOTSTRAP=%%i - ) + setlocal + call :nogoenv + for /f "tokens=*" %%i in ('"%%g" env GOROOT 2^>nul') do ( + endlocal + if /I not "%%i"=="%GOROOT_TEMP%" ( + set GOROOT_BOOTSTRAP=%%i + goto bootstrapset ) ) ) -set bootgo=1.20.6 +set bootgo=1.24.6 if "x%GOROOT_BOOTSTRAP%"=="x" if exist "%HOMEDRIVE%%HOMEPATH%\go%bootgo%" set GOROOT_BOOTSTRAP=%HOMEDRIVE%%HOMEPATH%\go%bootgo% if "x%GOROOT_BOOTSTRAP%"=="x" if exist "%HOMEDRIVE%%HOMEPATH%\sdk\go%bootgo%" set GOROOT_BOOTSTRAP=%HOMEDRIVE%%HOMEPATH%\sdk\go%bootgo% if "x%GOROOT_BOOTSTRAP%"=="x" set GOROOT_BOOTSTRAP=%HOMEDRIVE%%HOMEPATH%\Go1.4 :bootstrapset -if not exist "%GOROOT_BOOTSTRAP%\bin\go.exe" goto bootstrapfail +if not exist "%GOROOT_BOOTSTRAP%\bin\go.exe" ( + echo ERROR: Cannot find %GOROOT_BOOTSTRAP%\bin\go.exe + echo Set GOROOT_BOOTSTRAP to a working Go tree ^>= Go %bootgo%. + exit /b 1 +) set GOROOT=%GOROOT_TEMP% set GOROOT_TEMP= @@ -103,55 +93,26 @@ echo Building Go cmd/dist using %GOROOT_BOOTSTRAP%. (%GOROOT_BOOTSTRAP_VERSION%) if x%vflag==x-v echo cmd/dist set GOROOT=%GOROOT_BOOTSTRAP% set GOBIN= -"%GOROOT_BOOTSTRAP%\bin\go.exe" build -o cmd\dist\dist.exe .\cmd\dist +"%GOROOT_BOOTSTRAP%\bin\go.exe" build -o cmd\dist\dist.exe .\cmd\dist || exit /b 1 endlocal -if errorlevel 1 goto fail -.\cmd\dist\dist.exe env -w -p >env.bat -if errorlevel 1 goto fail +.\cmd\dist\dist.exe env -w -p >env.bat || exit /b 1 call .\env.bat del env.bat if x%vflag==x-v echo. -if x%1==x-dist-tool goto copydist -if x%2==x-dist-tool goto copydist -if x%3==x-dist-tool goto copydist -if x%4==x-dist-tool goto copydist -if x%1==x--dist-tool goto copydist -if x%2==x--dist-tool goto copydist -if x%3==x--dist-tool goto copydist -if x%4==x--dist-tool goto copydist - -set bootstrapflags= -if x%1==x-no-clean set bootstrapflags=-no-clean -if x%2==x-no-clean set bootstrapflags=-no-clean -if x%3==x-no-clean set bootstrapflags=-no-clean -if x%4==x-no-clean set bootstrapflags=-no-clean -if x%1==x--no-clean set bootstrapflags=-no-clean -if x%2==x--no-clean set bootstrapflags=-no-clean -if x%3==x--no-clean set bootstrapflags=-no-clean -if x%4==x--no-clean set bootstrapflags=-no-clean -if x%1==x-no-banner set bootstrapflags=%bootstrapflags% -no-banner -if x%2==x-no-banner set bootstrapflags=%bootstrapflags% -no-banner -if x%3==x-no-banner set bootstrapflags=%bootstrapflags% -no-banner -if x%4==x-no-banner set bootstrapflags=%bootstrapflags% -no-banner -if x%1==x--no-banner set bootstrapflags=%bootstrapflags% -no-banner -if x%2==x--no-banner set bootstrapflags=%bootstrapflags% -no-banner -if x%3==x--no-banner set bootstrapflags=%bootstrapflags% -no-banner -if x%4==x--no-banner set bootstrapflags=%bootstrapflags% -no-banner -if x%1==x-distpack set bootstrapflags=%bootstrapflags% -distpack -if x%2==x-distpack set bootstrapflags=%bootstrapflags% -distpack -if x%3==x-distpack set bootstrapflags=%bootstrapflags% -distpack -if x%4==x-distpack set bootstrapflags=%bootstrapflags% -distpack -if x%1==x--distpack set bootstrapflags=%bootstrapflags% -distpack -if x%2==x--distpack set bootstrapflags=%bootstrapflags% -distpack -if x%3==x--distpack set bootstrapflags=%bootstrapflags% -distpack -if x%4==x--distpack set bootstrapflags=%bootstrapflags% -distpack +if x%1==x--dist-tool ( + mkdir "%GOTOOLDIR%" 2>NUL + if not x%2==x ( + copy cmd\dist\dist.exe "%2" + ) + move cmd\dist\dist.exe "%GOTOOLDIR%\dist.exe" + goto :eof +) :: Run dist bootstrap to complete make.bash. :: Bootstrap installs a proper cmd/dist, built with the new toolchain. :: Throw ours, built with the bootstrap toolchain, away after bootstrap. -.\cmd\dist\dist.exe bootstrap -a %vflag% %bootstrapflags% -if errorlevel 1 goto fail +.\cmd\dist\dist.exe bootstrap -a %* || exit /b 1 del .\cmd\dist\dist.exe goto :eof @@ -161,11 +122,6 @@ goto :eof :: to avoid needing three copies in three different shell languages :: (make.bash, make.bat, make.rc). -:copydist -mkdir "%GOTOOLDIR%" 2>NUL -copy cmd\dist\dist.exe "%GOTOOLDIR%\" -goto :eof - :nogoenv set GO111MODULE=off set GOENV=off @@ -173,12 +129,3 @@ set GOOS= set GOARCH= set GOEXPERIMENT= set GOFLAGS= -goto :eof - -:bootstrapfail -echo ERROR: Cannot find %GOROOT_BOOTSTRAP%\bin\go.exe -echo Set GOROOT_BOOTSTRAP to a working Go tree ^>= Go %bootgo%. - -:fail -set GOBUILDFAIL=1 -if x%GOBUILDEXIT%==x1 exit %GOBUILDFAIL% diff --git a/src/make.rc b/src/make.rc index 54250011dcedf7..9ba2b7d76d7018 100755 --- a/src/make.rc +++ b/src/make.rc @@ -48,10 +48,10 @@ fn bootstrapenv { GOROOT=$GOROOT_BOOTSTRAP GO111MODULE=off GOENV=off GOOS=() GOARCH=() GOEXPERIMENT=() GOFLAGS=() $* } -bootgo = 1.20.6 +bootgo = 1.24.6 GOROOT = `{cd .. && pwd} goroot_bootstrap_set = 'true' -if(! ~ $#GOROOT_BOOTSTRAP 1){ +if(~ $"GOROOT_BOOTSTRAP ''){ goroot_bootstrap_set = 'false' GOROOT_BOOTSTRAP = $home/go1.4 for(d in sdk/go$bootgo go$bootgo) diff --git a/src/maps/example_test.go b/src/maps/example_test.go index 3d6b7d1ba09b6d..0795941d9064bf 100644 --- a/src/maps/example_test.go +++ b/src/maps/example_test.go @@ -7,6 +7,7 @@ package maps_test import ( "fmt" "maps" + "slices" "strings" ) @@ -133,3 +134,60 @@ func ExampleEqualFunc() { // Output: // true } + +func ExampleAll() { + m1 := map[string]int{ + "one": 1, + "two": 2, + } + m2 := map[string]int{ + "one": 10, + } + maps.Insert(m2, maps.All(m1)) + fmt.Println("m2 is:", m2) + // Output: + // m2 is: map[one:1 two:2] +} + +func ExampleKeys() { + m1 := map[int]string{ + 1: "one", + 10: "Ten", + 1000: "THOUSAND", + } + keys := slices.Sorted(maps.Keys(m1)) + fmt.Println(keys) + // Output: + // [1 10 1000] +} + +func ExampleValues() { + m1 := map[int]string{ + 1: "one", + 10: "Ten", + 1000: "THOUSAND", + } + values := slices.Sorted(maps.Values(m1)) + fmt.Println(values) + // Output: + // [THOUSAND Ten one] +} + +func ExampleInsert() { + m1 := map[int]string{ + 1000: "THOUSAND", + } + s1 := []string{"zero", "one", "two", "three"} + maps.Insert(m1, slices.All(s1)) + fmt.Println("m1 is:", m1) + // Output: + // m1 is: map[0:zero 1:one 2:two 3:three 1000:THOUSAND] +} + +func ExampleCollect() { + s1 := []string{"zero", "one", "two", "three"} + m1 := maps.Collect(slices.All(s1)) + fmt.Println("m1 is:", m1) + // Output: + // m1 is: map[0:zero 1:one 2:two 3:three] +} diff --git a/src/maps/maps_test.go b/src/maps/maps_test.go index fa30fe8c2bcb23..c8ee3cd0311f5f 100644 --- a/src/maps/maps_test.go +++ b/src/maps/maps_test.go @@ -48,7 +48,7 @@ func equalNaN[T comparable](v1, v2 T) bool { return v1 == v2 || (isNaN(v1) && isNaN(v2)) } -// equalStr compares ints and strings. +// equalIntStr compares ints and strings. func equalIntStr(v1 int, v2 string) bool { return strconv.Itoa(v1) == v2 } diff --git a/src/math/acos_s390x.s b/src/math/acos_s390x.s index d2288b8cd8e6b1..4431b99e1cbd7e 100644 --- a/src/math/acos_s390x.s +++ b/src/math/acos_s390x.s @@ -121,7 +121,7 @@ L4: L12: FMOVD 24(R9), F0 FMADD F10, F10, F0 - WORD $0xB3130080 //lcdbr %f8,%f0 + LCDBR F0, F8 WORD $0xED009008 //cdb %f0,.L37-.L13(%r9) BYTE $0x00 BYTE $0x19 @@ -129,7 +129,7 @@ L12: L5: MOVW R12, R4 CMPBLE R4, $0, L7 - WORD $0xB31300AA //lcdbr %f10,%f10 + LCDBR F10, F10 FMOVD $0, F1 BR L3 L9: diff --git a/src/math/all_test.go b/src/math/all_test.go index af3c38c2a690dd..4e5f4517629dd8 100644 --- a/src/math/all_test.go +++ b/src/math/all_test.go @@ -899,8 +899,17 @@ var vfceilSC = []float64{ 0, Inf(1), NaN(), + 1<<52 - 1, + 1<<52 - 0.5, // largest fractional float64 + 1 << 52, + -1 << 52, + -1<<52 + 0.5, // smallest fractional float64 + -1<<52 + 1, + 1 << 53, + -1 << 53, } -var ceilSC = []float64{ + +var ceilBaseSC = []float64{ Inf(-1), Copysign(0, -1), 0, @@ -908,6 +917,39 @@ var ceilSC = []float64{ NaN(), } +var ceilSC = append(ceilBaseSC, + 1<<52-1, + 1<<52, + 1<<52, + -1<<52, + -1<<52+1, + -1<<52+1, + 1<<53, + -1<<53, +) + +var floorSC = append(ceilBaseSC, + 1<<52-1, + 1<<52-1, + 1<<52, + -1<<52, + -1<<52, + -1<<52+1, + 1<<53, + -1<<53, +) + +var truncSC = append(ceilBaseSC, + 1<<52-1, + 1<<52-1, + 1<<52, + -1<<52, + -1<<52+1, + -1<<52+1, + 1<<53, + -1<<53, +) + var vfcopysignSC = []float64{ Inf(-1), Inf(1), @@ -2084,6 +2126,11 @@ var fmaC = []struct{ x, y, z, want float64 }{ // Issue #61130 {-1, 1, 1, 0}, {1, 1, -1, 0}, + + // Issue #73757 + {0x1p-1022, -0x1p-1022, 0, Copysign(0, -1)}, + {Copysign(0, -1), 1, 0, 0}, + {1, Copysign(0, -1), 0, 0}, } var sqrt32 = []float32{ @@ -2489,8 +2536,8 @@ func TestFloor(t *testing.T) { } } for i := 0; i < len(vfceilSC); i++ { - if f := Floor(vfceilSC[i]); !alike(ceilSC[i], f) { - t.Errorf("Floor(%g) = %g, want %g", vfceilSC[i], f, ceilSC[i]) + if f := Floor(vfceilSC[i]); !alike(floorSC[i], f) { + t.Errorf("Floor(%g) = %g, want %g", vfceilSC[i], f, floorSC[i]) } } } @@ -3034,8 +3081,8 @@ func TestTrunc(t *testing.T) { } } for i := 0; i < len(vfceilSC); i++ { - if f := Trunc(vfceilSC[i]); !alike(ceilSC[i], f) { - t.Errorf("Trunc(%g) = %g, want %g", vfceilSC[i], f, ceilSC[i]) + if f := Trunc(vfceilSC[i]); !alike(truncSC[i], f) { + t.Errorf("Trunc(%g) = %g, want %g", vfceilSC[i], f, truncSC[i]) } } } diff --git a/src/math/asin_s390x.s b/src/math/asin_s390x.s index dc54d053f1cab9..e5887c9bea3336 100644 --- a/src/math/asin_s390x.s +++ b/src/math/asin_s390x.s @@ -139,7 +139,7 @@ L4: L14: FMOVD 200(R9), F0 FMADD F8, F8, F0 - WORD $0xB31300A0 //lcdbr %f10,%f0 + LCDBR F0, F10 WORD $0xED009020 //cdb %f0,.L39-.L15(%r9) BYTE $0x00 BYTE $0x19 @@ -147,7 +147,7 @@ L14: L6: MOVW R7, R6 CMPBLE R6, $0, L8 - WORD $0xB3130088 //lcdbr %f8,%f8 + LCDBR F8, F8 FMOVD 24(R9), F1 BR L4 L10: diff --git a/src/math/asinh_s390x.s b/src/math/asinh_s390x.s index 1bcf2954c44d65..94081d3b15ce05 100644 --- a/src/math/asinh_s390x.s +++ b/src/math/asinh_s390x.s @@ -174,7 +174,7 @@ L6: MOVD R12, R6 CMPBGT R6, $0, L1 - WORD $0xB3130000 //lcdbr %f0,%f0 + LCDBR F0, F0 FMOVD F0, ret+8(FP) RET L16: @@ -206,8 +206,8 @@ L16: FMOVD F0, ret+8(FP) RET L17: - WORD $0xB31300A0 //lcdbr %f10,%f0 + LCDBR F0, F10 BR L4 L15: - WORD $0xB31300A0 //lcdbr %f10,%f0 + LCDBR F0, F10 BR L9 diff --git a/src/math/atan2_s390x.s b/src/math/atan2_s390x.s index 587b89e9b5bec0..13ff31977b5376 100644 --- a/src/math/atan2_s390x.s +++ b/src/math/atan2_s390x.s @@ -244,16 +244,16 @@ L17: WFDDB V0, V2, V4 BYTE $0x18 //lr %r2,%r5 BYTE $0x25 - WORD $0xB3130034 //lcdbr %f3,%f4 + LCDBR F4, F3 BR L3 L23: - WORD $0xB3130062 //lcdbr %f6,%f2 + LCDBR F2, F6 BR L4 L22: VLR V20, V0 BR L9 L21: - WORD $0xB3130022 //lcdbr %f2,%f2 + LCDBR F2, F2 BR L8 L24: VLR V20, V4 diff --git a/src/math/big/alias_test.go b/src/math/big/alias_test.go index 36c37fb0656e1d..8bfb632544721a 100644 --- a/src/math/big/alias_test.go +++ b/src/math/big/alias_test.go @@ -309,4 +309,28 @@ func TestAliasing(t *testing.T) { } }) } + + t.Run("Issue72043a", func(t *testing.T) { + v := new(big.Int).SetBytes([]byte{0x4b, 0xd2, 0x7a, 0x6c, 0xb3, 0x16, 0x5a, 0xf3, 0xa0, 0x45, 0x2b, 0xbe, 0x38, 0xba, 0x18, 0xb8, 0x70, 0xbb, 0xde, 0xd2, 0x20, 0xe9, 0xb9, 0x9d, 0x1d, 0xc6, 0x3e, 0xa, 0xa5, 0x60, 0x11, 0xcc, 0xb8, 0xff, 0x8e, 0x62}) + x := new(big.Int).SetBytes([]byte{0xb, 0x40, 0x70, 0x8b, 0x64, 0x78, 0x29, 0x9e, 0xd0}) + y := new(big.Int).SetBytes([]byte{0x1}) + z := new(big.Int).SetBytes([]byte{0xe, 0x91, 0x4d, 0xaa, 0x69, 0x60, 0x51, 0x42, 0x9, 0xa1, 0xf6, 0xfa, 0x5, 0x97, 0x1c, 0x5e, 0x18, 0xaa, 0x91, 0x87, 0x1f, 0xe4, 0x70, 0x84, 0x54, 0x97, 0x3a, 0xc9, 0xe1, 0x92, 0xd4, 0xf3, 0xdf, 0x79, 0x90, 0x18, 0xdd, 0xdf, 0x19, 0x44, 0xf3, 0xf, 0x9d, 0xa6, 0xf, 0xd9}) + v.Exp(x, y, z) + }) + + t.Run("Issue72043b", func(t *testing.T) { + v := new(big.Int).SetBytes([]byte{0x1, 0x6b, 0x8a, 0x26, 0x2, 0x1d, 0x3b, 0x4, 0x8d, 0x81, 0x85, 0xdb, 0x78, 0x4e, 0x81, 0xb7, 0xa7, 0xc0, 0x6e, 0xe7, 0xa0, 0xa, 0x54, 0x39, 0x3e, 0xa8, 0xb1, 0xe0, 0x0, 0xdb, 0x5a, 0x45, 0xde, 0xbf, 0xe2, 0xc8, 0x4d, 0x2b, 0x80, 0x65, 0x73, 0x5d, 0x6d, 0x65, 0x40, 0x50, 0xd4, 0x8}) + x := new(big.Int).SetBytes([]byte{0x7, 0xd6, 0x79, 0xc5, 0x30, 0xc2, 0x1f, 0x39, 0x5e, 0x3d, 0xd2, 0x52, 0xc6, 0xfb, 0xa7, 0xe2, 0x8a, 0x37, 0xdd}) + y := new(big.Int).SetBytes([]byte{0x1}) + z := new(big.Int).SetBytes([]byte{0x1c, 0x44, 0xba, 0x75, 0xfa, 0xe5, 0x3a, 0x56, 0xc0, 0x94, 0xa, 0x1e, 0x8f, 0xbd, 0xad, 0xdb, 0x17, 0xb8, 0x2b, 0x46, 0xf8, 0x69, 0xc1, 0x54, 0x25, 0x68, 0x53, 0xb5, 0xd5, 0x89, 0x6c, 0x16, 0xef, 0xf2, 0x83, 0x3e, 0x55, 0xa7, 0x54, 0x32, 0xbd, 0xa7, 0x70, 0x13, 0xa4, 0xfa}) + v.Exp(x, y, z) + }) + + t.Run("Issue72043c", func(t *testing.T) { + v := new(big.Int).SetBytes([]byte{0x2, 0x2a, 0xfc, 0x65, 0xc2, 0x66, 0xb1, 0x63, 0x56, 0xaf, 0x5c, 0x6, 0x4a, 0x19, 0x82, 0x6b, 0xaa, 0x90, 0x57, 0xd9, 0xd0, 0x3e, 0x83, 0xc8, 0x2a, 0x85, 0xe0}) + x := new(big.Int).SetBytes([]byte{0x8, 0x6f, 0xc5, 0x74, 0x83, 0x56, 0xa5, 0x6e, 0xfe, 0x7b, 0x32, 0xfb, 0x69, 0x4, 0x3, 0x51, 0xf5, 0x2c, 0x41, 0x7, 0xf7, 0xbb, 0x89, 0x8e}) + y := new(big.Int).SetBytes([]byte{0x1}) + z := new(big.Int).SetBytes([]byte{0x34, 0xe1, 0x5f, 0xd0, 0xa7, 0xf8, 0x34, 0x7c, 0x3c, 0x63, 0x82, 0x1e, 0xc3, 0x84, 0x9f, 0xa, 0x70, 0x22, 0xd3, 0xc8, 0x24, 0x64, 0x2b, 0xd7, 0x59, 0x20, 0xf2, 0xdd, 0xb5, 0xb2, 0xfa, 0x11, 0x2e, 0x98, 0x95}) + v.Exp(x, y, z) + }) } diff --git a/src/math/big/arith.go b/src/math/big/arith.go index 06e63e2574f31d..bc27ca6a562e44 100644 --- a/src/math/big/arith.go +++ b/src/math/big/arith.go @@ -10,7 +10,10 @@ package big -import "math/bits" +import ( + "math/bits" + _ "unsafe" // for go:linkname +) // A Word represents a single digit of a multi-precision unsigned integer. type Word uint @@ -23,17 +26,13 @@ const ( _M = _B - 1 // digit mask ) -// Many of the loops in this file are of the form -// for i := 0; i < len(z) && i < len(x) && i < len(y); i++ -// i < len(z) is the real condition. -// However, checking i < len(x) && i < len(y) as well is faster than -// having the compiler do a bounds check in the body of the loop; -// remarkably it is even faster than hoisting the bounds check -// out of the loop, by doing something like -// _, _ = x[len(z)-1], y[len(z)-1] -// There are other ways to hoist the bounds check out of the loop, -// but the compiler's BCE isn't powerful enough for them (yet?). -// See the discussion in CL 164966. +// In these routines, it is the caller's responsibility to arrange for +// x, y, and z to all have the same length. We check this and panic. +// The assembly versions of these routines do not include that check. +// +// The check+panic also has the effect of teaching the compiler that +// “i in range for z” implies “i in range for x and y”, eliminating all +// bounds checks in loops from 0 to len(z) and vice versa. // ---------------------------------------------------------------------------- // Elementary operations on words @@ -62,8 +61,11 @@ func nlz(x Word) uint { // The resulting carry c is either 0 or 1. func addVV_g(z, x, y []Word) (c Word) { - // The comment near the top of this file discusses this for loop condition. - for i := 0; i < len(z) && i < len(x) && i < len(y); i++ { + if len(x) != len(z) || len(y) != len(z) { + panic("addVV len") + } + + for i := range z { zi, cc := bits.Add(uint(x[i]), uint(y[i]), uint(c)) z[i] = Word(zi) c = Word(cc) @@ -73,8 +75,11 @@ func addVV_g(z, x, y []Word) (c Word) { // The resulting carry c is either 0 or 1. func subVV_g(z, x, y []Word) (c Word) { - // The comment near the top of this file discusses this for loop condition. - for i := 0; i < len(z) && i < len(x) && i < len(y); i++ { + if len(x) != len(z) || len(y) != len(z) { + panic("subVV len") + } + + for i := range z { zi, cc := bits.Sub(uint(x[i]), uint(y[i]), uint(c)) z[i] = Word(zi) c = Word(cc) @@ -82,33 +87,53 @@ func subVV_g(z, x, y []Word) (c Word) { return } -// The resulting carry c is either 0 or 1. -func addVW_g(z, x []Word, y Word) (c Word) { - c = y - // The comment near the top of this file discusses this for loop condition. - for i := 0; i < len(z) && i < len(x); i++ { - zi, cc := bits.Add(uint(x[i]), uint(c), 0) - z[i] = Word(zi) - c = Word(cc) +// addVW sets z = x + y, returning the final carry c. +// The behavior is undefined if len(x) != len(z). +// If len(z) == 0, c = y; otherwise, c is 0 or 1. +// +// addVW should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/remyoudompheng/bigfft +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname addVW +func addVW(z, x []Word, y Word) (c Word) { + if len(x) != len(z) { + panic("addVW len") } - return + + if len(z) == 0 { + return y + } + zi, cc := bits.Add(uint(x[0]), uint(y), 0) + z[0] = Word(zi) + if cc == 0 { + if &z[0] != &x[0] { + copy(z[1:], x[1:]) + } + return 0 + } + for i := 1; i < len(z); i++ { + xi := x[i] + if xi != ^Word(0) { + z[i] = xi + 1 + if &z[0] != &x[0] { + copy(z[i+1:], x[i+1:]) + } + return 0 + } + z[i] = 0 + } + return 1 } -// addVWlarge is addVW, but intended for large z. -// The only difference is that we check on every iteration -// whether we are done with carries, -// and if so, switch to a much faster copy instead. -// This is only a good idea for large z, -// because the overhead of the check and the function call -// outweigh the benefits when z is small. -func addVWlarge(z, x []Word, y Word) (c Word) { +// addVW_ref is the reference implementation for addVW, used only for testing. +func addVW_ref(z, x []Word, y Word) (c Word) { c = y - // The comment near the top of this file discusses this for loop condition. - for i := 0; i < len(z) && i < len(x); i++ { - if c == 0 { - copy(z[i:], x[i:]) - return - } + for i := range z { zi, cc := bits.Add(uint(x[i]), uint(c), 0) z[i] = Word(zi) c = Word(cc) @@ -116,34 +141,65 @@ func addVWlarge(z, x []Word, y Word) (c Word) { return } -func subVW_g(z, x []Word, y Word) (c Word) { - c = y - // The comment near the top of this file discusses this for loop condition. - for i := 0; i < len(z) && i < len(x); i++ { - zi, cc := bits.Sub(uint(x[i]), uint(c), 0) - z[i] = Word(zi) - c = Word(cc) +// subVW sets z = x - y, returning the final carry c. +// The behavior is undefined if len(x) != len(z). +// If len(z) == 0, c = y; otherwise, c is 0 or 1. +// +// subVW should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/remyoudompheng/bigfft +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname subVW +func subVW(z, x []Word, y Word) (c Word) { + if len(x) != len(z) { + panic("subVW len") } - return + + if len(z) == 0 { + return y + } + zi, cc := bits.Sub(uint(x[0]), uint(y), 0) + z[0] = Word(zi) + if cc == 0 { + if &z[0] != &x[0] { + copy(z[1:], x[1:]) + } + return 0 + } + for i := 1; i < len(z); i++ { + xi := x[i] + if xi != 0 { + z[i] = xi - 1 + if &z[0] != &x[0] { + copy(z[i+1:], x[i+1:]) + } + return 0 + } + z[i] = ^Word(0) + } + return 1 } -// subVWlarge is to subVW as addVWlarge is to addVW. -func subVWlarge(z, x []Word, y Word) (c Word) { +// subVW_ref is the reference implementation for subVW, used only for testing. +func subVW_ref(z, x []Word, y Word) (c Word) { c = y - // The comment near the top of this file discusses this for loop condition. - for i := 0; i < len(z) && i < len(x); i++ { - if c == 0 { - copy(z[i:], x[i:]) - return - } + for i := range z { zi, cc := bits.Sub(uint(x[i]), uint(c), 0) z[i] = Word(zi) c = Word(cc) } - return + return c } -func shlVU_g(z, x []Word, s uint) (c Word) { +func lshVU_g(z, x []Word, s uint) (c Word) { + if len(x) != len(z) { + panic("lshVU len") + } + if s == 0 { copy(z, x) return @@ -162,7 +218,11 @@ func shlVU_g(z, x []Word, s uint) (c Word) { return } -func shrVU_g(z, x []Word, s uint) (c Word) { +func rshVU_g(z, x []Word, s uint) (c Word) { + if len(x) != len(z) { + panic("rshVU len") + } + if s == 0 { copy(z, x) return @@ -170,10 +230,6 @@ func shrVU_g(z, x []Word, s uint) (c Word) { if len(z) == 0 { return } - if len(x) != len(z) { - // This is an invariant guaranteed by the caller. - panic("len(x) != len(z)") - } s &= _W - 1 // hint to the compiler that shifts by s don't need guard code ŝ := _W - s ŝ &= _W - 1 // ditto @@ -186,18 +242,24 @@ func shrVU_g(z, x []Word, s uint) (c Word) { } func mulAddVWW_g(z, x []Word, y, r Word) (c Word) { + if len(x) != len(z) { + panic("mulAddVWW len") + } c = r - // The comment near the top of this file discusses this for loop condition. - for i := 0; i < len(z) && i < len(x); i++ { + for i := range z { c, z[i] = mulAddWWW_g(x[i], y, c) } return } -func addMulVVW_g(z, x []Word, y Word) (c Word) { - // The comment near the top of this file discusses this for loop condition. - for i := 0; i < len(z) && i < len(x); i++ { - z1, z0 := mulAddWWW_g(x[i], y, z[i]) +func addMulVVWW_g(z, x, y []Word, m, a Word) (c Word) { + if len(x) != len(z) || len(y) != len(z) { + panic("rshVU len") + } + + c = a + for i := range z { + z1, z0 := mulAddWWW_g(y[i], m, x[i]) lo, cc := bits.Add(uint(z0), uint(c), 0) c, z[i] = Word(cc), Word(lo) c += z1 diff --git a/src/math/big/arith_386.s b/src/math/big/arith_386.s index 90f6a8c70e1d53..242c869af35fd2 100644 --- a/src/math/big/arith_386.s +++ b/src/math/big/arith_386.s @@ -1,235 +1,240 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2025 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Code generated by 'go generate' (with ./internal/asmgen). DO NOT EDIT. + //go:build !math_big_pure_go #include "textflag.h" -// This file provides fast assembly versions for the elementary -// arithmetic operations on vectors implemented in arith.go. - // func addVV(z, x, y []Word) (c Word) -TEXT ·addVV(SB),NOSPLIT,$0 - MOVL z+0(FP), DI - MOVL x+12(FP), SI - MOVL y+24(FP), CX - MOVL z_len+4(FP), BP - MOVL $0, BX // i = 0 - MOVL $0, DX // c = 0 - JMP E1 - -L1: MOVL (SI)(BX*4), AX - ADDL DX, DX // restore CF - ADCL (CX)(BX*4), AX - SBBL DX, DX // save CF - MOVL AX, (DI)(BX*4) - ADDL $1, BX // i++ - -E1: CMPL BX, BP // i < n - JL L1 - - NEGL DX +TEXT ·addVV(SB), NOSPLIT, $0 + MOVL z_len+4(FP), BX + MOVL x_base+12(FP), SI + MOVL y_base+24(FP), DI + MOVL z_base+0(FP), BP + // compute unrolled loop lengths + MOVL BX, CX + ANDL $3, CX + SHRL $2, BX + MOVL $0, DX // clear saved carry +loop1: + TESTL CX, CX; JZ loop1done +loop1cont: + // unroll 1X in batches of 1 + ADDL DX, DX // restore carry + MOVL 0(SI), DX + ADCL 0(DI), DX + MOVL DX, 0(BP) + SBBL DX, DX // save carry + LEAL 4(SI), SI // ADD $4, SI + LEAL 4(DI), DI // ADD $4, DI + LEAL 4(BP), BP // ADD $4, BP + SUBL $1, CX; JNZ loop1cont +loop1done: +loop4: + TESTL BX, BX; JZ loop4done +loop4cont: + // unroll 4X in batches of 1 + ADDL DX, DX // restore carry + MOVL 0(SI), CX + ADCL 0(DI), CX + MOVL CX, 0(BP) + MOVL 4(SI), CX + ADCL 4(DI), CX + MOVL CX, 4(BP) + MOVL 8(SI), CX + ADCL 8(DI), CX + MOVL CX, 8(BP) + MOVL 12(SI), CX + ADCL 12(DI), CX + MOVL CX, 12(BP) + SBBL DX, DX // save carry + LEAL 16(SI), SI // ADD $16, SI + LEAL 16(DI), DI // ADD $16, DI + LEAL 16(BP), BP // ADD $16, BP + SUBL $1, BX; JNZ loop4cont +loop4done: + NEGL DX // convert add carry MOVL DX, c+36(FP) RET - // func subVV(z, x, y []Word) (c Word) -// (same as addVV except for SBBL instead of ADCL and label names) -TEXT ·subVV(SB),NOSPLIT,$0 - MOVL z+0(FP), DI - MOVL x+12(FP), SI - MOVL y+24(FP), CX - MOVL z_len+4(FP), BP - MOVL $0, BX // i = 0 - MOVL $0, DX // c = 0 - JMP E2 - -L2: MOVL (SI)(BX*4), AX - ADDL DX, DX // restore CF - SBBL (CX)(BX*4), AX - SBBL DX, DX // save CF - MOVL AX, (DI)(BX*4) - ADDL $1, BX // i++ - -E2: CMPL BX, BP // i < n - JL L2 - - NEGL DX +TEXT ·subVV(SB), NOSPLIT, $0 + MOVL z_len+4(FP), BX + MOVL x_base+12(FP), SI + MOVL y_base+24(FP), DI + MOVL z_base+0(FP), BP + // compute unrolled loop lengths + MOVL BX, CX + ANDL $3, CX + SHRL $2, BX + MOVL $0, DX // clear saved carry +loop1: + TESTL CX, CX; JZ loop1done +loop1cont: + // unroll 1X in batches of 1 + ADDL DX, DX // restore carry + MOVL 0(SI), DX + SBBL 0(DI), DX + MOVL DX, 0(BP) + SBBL DX, DX // save carry + LEAL 4(SI), SI // ADD $4, SI + LEAL 4(DI), DI // ADD $4, DI + LEAL 4(BP), BP // ADD $4, BP + SUBL $1, CX; JNZ loop1cont +loop1done: +loop4: + TESTL BX, BX; JZ loop4done +loop4cont: + // unroll 4X in batches of 1 + ADDL DX, DX // restore carry + MOVL 0(SI), CX + SBBL 0(DI), CX + MOVL CX, 0(BP) + MOVL 4(SI), CX + SBBL 4(DI), CX + MOVL CX, 4(BP) + MOVL 8(SI), CX + SBBL 8(DI), CX + MOVL CX, 8(BP) + MOVL 12(SI), CX + SBBL 12(DI), CX + MOVL CX, 12(BP) + SBBL DX, DX // save carry + LEAL 16(SI), SI // ADD $16, SI + LEAL 16(DI), DI // ADD $16, DI + LEAL 16(BP), BP // ADD $16, BP + SUBL $1, BX; JNZ loop4cont +loop4done: + NEGL DX // convert sub carry MOVL DX, c+36(FP) RET - -// func addVW(z, x []Word, y Word) (c Word) -TEXT ·addVW(SB),NOSPLIT,$0 - MOVL z+0(FP), DI - MOVL x+12(FP), SI - MOVL y+24(FP), AX // c = y - MOVL z_len+4(FP), BP - MOVL $0, BX // i = 0 - JMP E3 - -L3: ADDL (SI)(BX*4), AX - MOVL AX, (DI)(BX*4) - SBBL AX, AX // save CF - NEGL AX - ADDL $1, BX // i++ - -E3: CMPL BX, BP // i < n - JL L3 - - MOVL AX, c+28(FP) - RET - - -// func subVW(z, x []Word, y Word) (c Word) -TEXT ·subVW(SB),NOSPLIT,$0 - MOVL z+0(FP), DI - MOVL x+12(FP), SI - MOVL y+24(FP), AX // c = y - MOVL z_len+4(FP), BP - MOVL $0, BX // i = 0 - JMP E4 - -L4: MOVL (SI)(BX*4), DX - SUBL AX, DX - MOVL DX, (DI)(BX*4) - SBBL AX, AX // save CF - NEGL AX - ADDL $1, BX // i++ - -E4: CMPL BX, BP // i < n - JL L4 - - MOVL AX, c+28(FP) - RET - - -// func shlVU(z, x []Word, s uint) (c Word) -TEXT ·shlVU(SB),NOSPLIT,$0 - MOVL z_len+4(FP), BX // i = z - SUBL $1, BX // i-- - JL X8b // i < 0 (n <= 0) - - // n > 0 - MOVL z+0(FP), DI - MOVL x+12(FP), SI +// func lshVU(z, x []Word, s uint) (c Word) +TEXT ·lshVU(SB), NOSPLIT, $0 + MOVL z_len+4(FP), BX + TESTL BX, BX; JZ ret0 MOVL s+24(FP), CX - MOVL (SI)(BX*4), AX // w1 = x[n-1] + MOVL x_base+12(FP), SI + MOVL z_base+0(FP), DI + // run loop backward, using counter as positive index + // shift first word into carry + MOVL -4(SI)(BX*4), BP MOVL $0, DX - SHLL CX, AX, DX // w1>>ŝ + SHLL CX, BP, DX MOVL DX, c+28(FP) - - CMPL BX, $0 - JLE X8a // i <= 0 - - // i > 0 -L8: MOVL AX, DX // w = w1 - MOVL -4(SI)(BX*4), AX // w1 = x[i-1] - SHLL CX, AX, DX // w<>ŝ - MOVL DX, (DI)(BX*4) // z[i] = w<>ŝ - SUBL $1, BX // i-- - JG L8 // i > 0 - - // i <= 0 -X8a: SHLL CX, AX // w1< 0 - MOVL z+0(FP), DI - MOVL x+12(FP), SI +// func rshVU(z, x []Word, s uint) (c Word) +TEXT ·rshVU(SB), NOSPLIT, $0 + MOVL z_len+4(FP), BX + TESTL BX, BX; JZ ret0 MOVL s+24(FP), CX - MOVL (SI), AX // w1 = x[0] + MOVL x_base+12(FP), SI + MOVL z_base+0(FP), DI + // use counter as negative index + LEAL (SI)(BX*4), SI + LEAL (DI)(BX*4), DI + NEGL BX + // shift first word into carry + MOVL 0(SI)(BX*4), BP MOVL $0, DX - SHRL CX, AX, DX // w1<<ŝ + SHRL CX, BP, DX MOVL DX, c+28(FP) - - MOVL $0, BX // i = 0 - JMP E9 - - // i < n-1 -L9: MOVL AX, DX // w = w1 - MOVL 4(SI)(BX*4), AX // w1 = x[i+1] - SHRL CX, AX, DX // w>>s | w1<<ŝ - MOVL DX, (DI)(BX*4) // z[i] = w>>s | w1<<ŝ - ADDL $1, BX // i++ - -E9: CMPL BX, BP - JL L9 // i < n-1 - - // i >= n-1 -X9a: SHRL CX, AX // w1>>s - MOVL AX, (DI)(BP*4) // z[n-1] = w1>>s + // shift remaining words + ADDL $1, BX +loop1: + TESTL BX, BX; JZ loop1done +loop1cont: + // unroll 1X in batches of 1 + MOVL 0(SI)(BX*4), DX + SHRL CX, DX, BP + MOVL BP, -4(DI)(BX*4) + MOVL DX, BP + ADDL $1, BX; JNZ loop1cont +loop1done: + // store final shifted bits + SHRL CX, BP + MOVL BP, -4(DI)(BX*4) RET - -X9b: MOVL $0, c+28(FP) +ret0: + MOVL $0, c+28(FP) RET - -// func mulAddVWW(z, x []Word, y, r Word) (c Word) -TEXT ·mulAddVWW(SB),NOSPLIT,$0 - MOVL z+0(FP), DI - MOVL x+12(FP), SI - MOVL y+24(FP), BP - MOVL r+28(FP), CX // c = r - MOVL z_len+4(FP), BX - LEAL (DI)(BX*4), DI - LEAL (SI)(BX*4), SI - NEGL BX // i = -n - JMP E5 - -L5: MOVL (SI)(BX*4), AX - MULL BP - ADDL CX, AX - ADCL $0, DX - MOVL AX, (DI)(BX*4) - MOVL DX, CX - ADDL $1, BX // i++ - -E5: CMPL BX, $0 // i < 0 - JL L5 - - MOVL CX, c+32(FP) +// func mulAddVWW(z, x []Word, m, a Word) (c Word) +TEXT ·mulAddVWW(SB), NOSPLIT, $0 + MOVL m+24(FP), BX + MOVL a+28(FP), SI + MOVL z_len+4(FP), DI + MOVL x_base+12(FP), BP + MOVL z_base+0(FP), CX + // use counter as negative index + LEAL (BP)(DI*4), BP + LEAL (CX)(DI*4), CX + NEGL DI +loop1: + TESTL DI, DI; JZ loop1done +loop1cont: + // unroll 1X in batches of 1 + MOVL 0(BP)(DI*4), AX + // multiply + MULL BX + ADDL SI, AX + MOVL DX, SI + ADCL $0, SI + MOVL AX, 0(CX)(DI*4) + ADDL $1, DI; JNZ loop1cont +loop1done: + MOVL SI, c+32(FP) RET - -// func addMulVVW(z, x []Word, y Word) (c Word) -TEXT ·addMulVVW(SB),NOSPLIT,$0 - MOVL z+0(FP), DI - MOVL x+12(FP), SI - MOVL y+24(FP), BP - MOVL z_len+4(FP), BX - LEAL (DI)(BX*4), DI - LEAL (SI)(BX*4), SI - NEGL BX // i = -n - MOVL $0, CX // c = 0 - JMP E6 - -L6: MOVL (SI)(BX*4), AX - MULL BP - ADDL CX, AX - ADCL $0, DX - ADDL AX, (DI)(BX*4) - ADCL $0, DX - MOVL DX, CX - ADDL $1, BX // i++ - -E6: CMPL BX, $0 // i < 0 - JL L6 - - MOVL CX, c+28(FP) +// func addMulVVWW(z, x, y []Word, m, a Word) (c Word) +TEXT ·addMulVVWW(SB), NOSPLIT, $0 + MOVL a+40(FP), BX + MOVL z_len+4(FP), SI + MOVL x_base+12(FP), DI + MOVL y_base+24(FP), BP + MOVL z_base+0(FP), CX + // use counter as negative index + LEAL (DI)(SI*4), DI + LEAL (BP)(SI*4), BP + LEAL (CX)(SI*4), CX + NEGL SI +loop1: + TESTL SI, SI; JZ loop1done +loop1cont: + // unroll 1X in batches of 1 + MOVL 0(BP)(SI*4), AX + // multiply + MULL m+36(FP) + ADDL BX, AX + MOVL DX, BX + ADCL $0, BX + // add + ADDL 0(DI)(SI*4), AX + ADCL $0, BX + MOVL AX, 0(CX)(SI*4) + ADDL $1, SI; JNZ loop1cont +loop1done: + MOVL BX, c+44(FP) RET - - - diff --git a/src/math/big/arith_amd64.go b/src/math/big/arith_amd64.go index 3db72582bbc0dd..3e3c916c1fd38c 100644 --- a/src/math/big/arith_amd64.go +++ b/src/math/big/arith_amd64.go @@ -8,4 +8,4 @@ package big import "internal/cpu" -var support_adx = cpu.X86.HasADX && cpu.X86.HasBMI2 +var hasADX = cpu.X86.HasADX && cpu.X86.HasBMI2 diff --git a/src/math/big/arith_amd64.s b/src/math/big/arith_amd64.s index a5b65b1d3c75f6..9b79ca96f3a222 100644 --- a/src/math/big/arith_amd64.s +++ b/src/math/big/arith_amd64.s @@ -1,515 +1,462 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2025 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Code generated by 'go generate' (with ./internal/asmgen). DO NOT EDIT. + //go:build !math_big_pure_go #include "textflag.h" -// This file provides fast assembly versions for the elementary -// arithmetic operations on vectors implemented in arith.go. - -// The carry bit is saved with SBBQ Rx, Rx: if the carry was set, Rx is -1, otherwise it is 0. -// It is restored with ADDQ Rx, Rx: if Rx was -1 the carry is set, otherwise it is cleared. -// This is faster than using rotate instructions. - // func addVV(z, x, y []Word) (c Word) -TEXT ·addVV(SB),NOSPLIT,$0 - MOVQ z_len+8(FP), DI - MOVQ x+24(FP), R8 - MOVQ y+48(FP), R9 - MOVQ z+0(FP), R10 - - MOVQ $0, CX // c = 0 - MOVQ $0, SI // i = 0 - - // s/JL/JMP/ below to disable the unrolled loop - SUBQ $4, DI // n -= 4 - JL V1 // if n < 0 goto V1 - -U1: // n >= 0 - // regular loop body unrolled 4x - ADDQ CX, CX // restore CF - MOVQ 0(R8)(SI*8), R11 - MOVQ 8(R8)(SI*8), R12 - MOVQ 16(R8)(SI*8), R13 - MOVQ 24(R8)(SI*8), R14 - ADCQ 0(R9)(SI*8), R11 - ADCQ 8(R9)(SI*8), R12 - ADCQ 16(R9)(SI*8), R13 - ADCQ 24(R9)(SI*8), R14 - MOVQ R11, 0(R10)(SI*8) - MOVQ R12, 8(R10)(SI*8) - MOVQ R13, 16(R10)(SI*8) - MOVQ R14, 24(R10)(SI*8) - SBBQ CX, CX // save CF - - ADDQ $4, SI // i += 4 - SUBQ $4, DI // n -= 4 - JGE U1 // if n >= 0 goto U1 - -V1: ADDQ $4, DI // n += 4 - JLE E1 // if n <= 0 goto E1 - -L1: // n > 0 - ADDQ CX, CX // restore CF - MOVQ 0(R8)(SI*8), R11 - ADCQ 0(R9)(SI*8), R11 - MOVQ R11, 0(R10)(SI*8) - SBBQ CX, CX // save CF - - ADDQ $1, SI // i++ - SUBQ $1, DI // n-- - JG L1 // if n > 0 goto L1 - -E1: NEGQ CX - MOVQ CX, c+72(FP) // return c +TEXT ·addVV(SB), NOSPLIT, $0 + MOVQ z_len+8(FP), BX + MOVQ x_base+24(FP), SI + MOVQ y_base+48(FP), DI + MOVQ z_base+0(FP), R8 + // compute unrolled loop lengths + MOVQ BX, R9 + ANDQ $3, R9 + SHRQ $2, BX + MOVQ $0, R10 // clear saved carry +loop1: + TESTQ R9, R9; JZ loop1done +loop1cont: + // unroll 1X + ADDQ R10, R10 // restore carry + MOVQ 0(SI), R10 + ADCQ 0(DI), R10 + MOVQ R10, 0(R8) + SBBQ R10, R10 // save carry + LEAQ 8(SI), SI // ADD $8, SI + LEAQ 8(DI), DI // ADD $8, DI + LEAQ 8(R8), R8 // ADD $8, R8 + SUBQ $1, R9; JNZ loop1cont +loop1done: +loop4: + TESTQ BX, BX; JZ loop4done +loop4cont: + // unroll 4X + ADDQ R10, R10 // restore carry + MOVQ 0(SI), R9 + MOVQ 8(SI), R10 + MOVQ 16(SI), R11 + MOVQ 24(SI), R12 + ADCQ 0(DI), R9 + ADCQ 8(DI), R10 + ADCQ 16(DI), R11 + ADCQ 24(DI), R12 + MOVQ R9, 0(R8) + MOVQ R10, 8(R8) + MOVQ R11, 16(R8) + MOVQ R12, 24(R8) + SBBQ R10, R10 // save carry + LEAQ 32(SI), SI // ADD $32, SI + LEAQ 32(DI), DI // ADD $32, DI + LEAQ 32(R8), R8 // ADD $32, R8 + SUBQ $1, BX; JNZ loop4cont +loop4done: + NEGQ R10 // convert add carry + MOVQ R10, c+72(FP) RET - // func subVV(z, x, y []Word) (c Word) -// (same as addVV except for SBBQ instead of ADCQ and label names) -TEXT ·subVV(SB),NOSPLIT,$0 - MOVQ z_len+8(FP), DI - MOVQ x+24(FP), R8 - MOVQ y+48(FP), R9 - MOVQ z+0(FP), R10 - - MOVQ $0, CX // c = 0 - MOVQ $0, SI // i = 0 - - // s/JL/JMP/ below to disable the unrolled loop - SUBQ $4, DI // n -= 4 - JL V2 // if n < 0 goto V2 - -U2: // n >= 0 - // regular loop body unrolled 4x - ADDQ CX, CX // restore CF - MOVQ 0(R8)(SI*8), R11 - MOVQ 8(R8)(SI*8), R12 - MOVQ 16(R8)(SI*8), R13 - MOVQ 24(R8)(SI*8), R14 - SBBQ 0(R9)(SI*8), R11 - SBBQ 8(R9)(SI*8), R12 - SBBQ 16(R9)(SI*8), R13 - SBBQ 24(R9)(SI*8), R14 - MOVQ R11, 0(R10)(SI*8) - MOVQ R12, 8(R10)(SI*8) - MOVQ R13, 16(R10)(SI*8) - MOVQ R14, 24(R10)(SI*8) - SBBQ CX, CX // save CF - - ADDQ $4, SI // i += 4 - SUBQ $4, DI // n -= 4 - JGE U2 // if n >= 0 goto U2 - -V2: ADDQ $4, DI // n += 4 - JLE E2 // if n <= 0 goto E2 - -L2: // n > 0 - ADDQ CX, CX // restore CF - MOVQ 0(R8)(SI*8), R11 - SBBQ 0(R9)(SI*8), R11 - MOVQ R11, 0(R10)(SI*8) - SBBQ CX, CX // save CF - - ADDQ $1, SI // i++ - SUBQ $1, DI // n-- - JG L2 // if n > 0 goto L2 - -E2: NEGQ CX - MOVQ CX, c+72(FP) // return c - RET - - -// func addVW(z, x []Word, y Word) (c Word) -TEXT ·addVW(SB),NOSPLIT,$0 - MOVQ z_len+8(FP), DI - CMPQ DI, $32 - JG large - MOVQ x+24(FP), R8 - MOVQ y+48(FP), CX // c = y - MOVQ z+0(FP), R10 - - MOVQ $0, SI // i = 0 - - // s/JL/JMP/ below to disable the unrolled loop - SUBQ $4, DI // n -= 4 - JL V3 // if n < 4 goto V3 - -U3: // n >= 0 - // regular loop body unrolled 4x - MOVQ 0(R8)(SI*8), R11 - MOVQ 8(R8)(SI*8), R12 - MOVQ 16(R8)(SI*8), R13 - MOVQ 24(R8)(SI*8), R14 - ADDQ CX, R11 - ADCQ $0, R12 - ADCQ $0, R13 - ADCQ $0, R14 - SBBQ CX, CX // save CF - NEGQ CX - MOVQ R11, 0(R10)(SI*8) - MOVQ R12, 8(R10)(SI*8) - MOVQ R13, 16(R10)(SI*8) - MOVQ R14, 24(R10)(SI*8) - - ADDQ $4, SI // i += 4 - SUBQ $4, DI // n -= 4 - JGE U3 // if n >= 0 goto U3 - -V3: ADDQ $4, DI // n += 4 - JLE E3 // if n <= 0 goto E3 - -L3: // n > 0 - ADDQ 0(R8)(SI*8), CX - MOVQ CX, 0(R10)(SI*8) - SBBQ CX, CX // save CF - NEGQ CX - - ADDQ $1, SI // i++ - SUBQ $1, DI // n-- - JG L3 // if n > 0 goto L3 - -E3: MOVQ CX, c+56(FP) // return c - RET -large: - JMP ·addVWlarge(SB) - - -// func subVW(z, x []Word, y Word) (c Word) -// (same as addVW except for SUBQ/SBBQ instead of ADDQ/ADCQ and label names) -TEXT ·subVW(SB),NOSPLIT,$0 - MOVQ z_len+8(FP), DI - CMPQ DI, $32 - JG large - MOVQ x+24(FP), R8 - MOVQ y+48(FP), CX // c = y - MOVQ z+0(FP), R10 - - MOVQ $0, SI // i = 0 - - // s/JL/JMP/ below to disable the unrolled loop - SUBQ $4, DI // n -= 4 - JL V4 // if n < 4 goto V4 - -U4: // n >= 0 - // regular loop body unrolled 4x - MOVQ 0(R8)(SI*8), R11 - MOVQ 8(R8)(SI*8), R12 - MOVQ 16(R8)(SI*8), R13 - MOVQ 24(R8)(SI*8), R14 - SUBQ CX, R11 - SBBQ $0, R12 - SBBQ $0, R13 - SBBQ $0, R14 - SBBQ CX, CX // save CF - NEGQ CX - MOVQ R11, 0(R10)(SI*8) - MOVQ R12, 8(R10)(SI*8) - MOVQ R13, 16(R10)(SI*8) - MOVQ R14, 24(R10)(SI*8) - - ADDQ $4, SI // i += 4 - SUBQ $4, DI // n -= 4 - JGE U4 // if n >= 0 goto U4 - -V4: ADDQ $4, DI // n += 4 - JLE E4 // if n <= 0 goto E4 - -L4: // n > 0 - MOVQ 0(R8)(SI*8), R11 - SUBQ CX, R11 - MOVQ R11, 0(R10)(SI*8) - SBBQ CX, CX // save CF - NEGQ CX - - ADDQ $1, SI // i++ - SUBQ $1, DI // n-- - JG L4 // if n > 0 goto L4 - -E4: MOVQ CX, c+56(FP) // return c +TEXT ·subVV(SB), NOSPLIT, $0 + MOVQ z_len+8(FP), BX + MOVQ x_base+24(FP), SI + MOVQ y_base+48(FP), DI + MOVQ z_base+0(FP), R8 + // compute unrolled loop lengths + MOVQ BX, R9 + ANDQ $3, R9 + SHRQ $2, BX + MOVQ $0, R10 // clear saved carry +loop1: + TESTQ R9, R9; JZ loop1done +loop1cont: + // unroll 1X + ADDQ R10, R10 // restore carry + MOVQ 0(SI), R10 + SBBQ 0(DI), R10 + MOVQ R10, 0(R8) + SBBQ R10, R10 // save carry + LEAQ 8(SI), SI // ADD $8, SI + LEAQ 8(DI), DI // ADD $8, DI + LEAQ 8(R8), R8 // ADD $8, R8 + SUBQ $1, R9; JNZ loop1cont +loop1done: +loop4: + TESTQ BX, BX; JZ loop4done +loop4cont: + // unroll 4X + ADDQ R10, R10 // restore carry + MOVQ 0(SI), R9 + MOVQ 8(SI), R10 + MOVQ 16(SI), R11 + MOVQ 24(SI), R12 + SBBQ 0(DI), R9 + SBBQ 8(DI), R10 + SBBQ 16(DI), R11 + SBBQ 24(DI), R12 + MOVQ R9, 0(R8) + MOVQ R10, 8(R8) + MOVQ R11, 16(R8) + MOVQ R12, 24(R8) + SBBQ R10, R10 // save carry + LEAQ 32(SI), SI // ADD $32, SI + LEAQ 32(DI), DI // ADD $32, DI + LEAQ 32(R8), R8 // ADD $32, R8 + SUBQ $1, BX; JNZ loop4cont +loop4done: + NEGQ R10 // convert sub carry + MOVQ R10, c+72(FP) RET -large: - JMP ·subVWlarge(SB) - - -// func shlVU(z, x []Word, s uint) (c Word) -TEXT ·shlVU(SB),NOSPLIT,$0 - MOVQ z_len+8(FP), BX // i = z - SUBQ $1, BX // i-- - JL X8b // i < 0 (n <= 0) - // n > 0 - MOVQ z+0(FP), R10 - MOVQ x+24(FP), R8 +// func lshVU(z, x []Word, s uint) (c Word) +TEXT ·lshVU(SB), NOSPLIT, $0 + MOVQ z_len+8(FP), BX + TESTQ BX, BX; JZ ret0 MOVQ s+48(FP), CX - MOVQ (R8)(BX*8), AX // w1 = x[n-1] - MOVQ $0, DX - SHLQ CX, AX, DX // w1>>ŝ - MOVQ DX, c+56(FP) - - CMPQ BX, $0 - JLE X8a // i <= 0 - - // i > 0 -L8: MOVQ AX, DX // w = w1 - MOVQ -8(R8)(BX*8), AX // w1 = x[i-1] - SHLQ CX, AX, DX // w<>ŝ - MOVQ DX, (R10)(BX*8) // z[i] = w<>ŝ - SUBQ $1, BX // i-- - JG L8 // i > 0 - - // i <= 0 -X8a: SHLQ CX, AX // w1< 0 - MOVQ z+0(FP), R10 - MOVQ x+24(FP), R8 +// func rshVU(z, x []Word, s uint) (c Word) +TEXT ·rshVU(SB), NOSPLIT, $0 + MOVQ z_len+8(FP), BX + TESTQ BX, BX; JZ ret0 MOVQ s+48(FP), CX - MOVQ (R8), AX // w1 = x[0] - MOVQ $0, DX - SHRQ CX, AX, DX // w1<<ŝ - MOVQ DX, c+56(FP) - - MOVQ $0, BX // i = 0 - JMP E9 - - // i < n-1 -L9: MOVQ AX, DX // w = w1 - MOVQ 8(R8)(BX*8), AX // w1 = x[i+1] - SHRQ CX, AX, DX // w>>s | w1<<ŝ - MOVQ DX, (R10)(BX*8) // z[i] = w>>s | w1<<ŝ - ADDQ $1, BX // i++ - -E9: CMPQ BX, R11 - JL L9 // i < n-1 - - // i >= n-1 -X9a: SHRQ CX, AX // w1>>s - MOVQ AX, (R10)(R11*8) // z[n-1] = w1>>s - RET - -X9b: MOVQ $0, c+56(FP) + MOVQ x_base+24(FP), SI + MOVQ z_base+0(FP), DI + // shift first word into carry + MOVQ 0(SI), R8 + MOVQ $0, R9 + SHRQ CX, R8, R9 + MOVQ R9, c+56(FP) + // shift remaining words + SUBQ $1, BX + // compute unrolled loop lengths + MOVQ BX, R9 + ANDQ $3, R9 + SHRQ $2, BX +loop1: + TESTQ R9, R9; JZ loop1done +loop1cont: + // unroll 1X + MOVQ 8(SI), R10 + SHRQ CX, R10, R8 + MOVQ R8, 0(DI) + MOVQ R10, R8 + LEAQ 8(SI), SI // ADD $8, SI + LEAQ 8(DI), DI // ADD $8, DI + SUBQ $1, R9; JNZ loop1cont +loop1done: +loop4: + TESTQ BX, BX; JZ loop4done +loop4cont: + // unroll 4X + MOVQ 8(SI), R9 + MOVQ 16(SI), R10 + MOVQ 24(SI), R11 + MOVQ 32(SI), R12 + SHRQ CX, R9, R8 + SHRQ CX, R10, R9 + SHRQ CX, R11, R10 + SHRQ CX, R12, R11 + MOVQ R8, 0(DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, R8 + LEAQ 32(SI), SI // ADD $32, SI + LEAQ 32(DI), DI // ADD $32, DI + SUBQ $1, BX; JNZ loop4cont +loop4done: + // store final shifted bits + SHRQ CX, R8 + MOVQ R8, 0(DI) RET - - -// func mulAddVWW(z, x []Word, y, r Word) (c Word) -TEXT ·mulAddVWW(SB),NOSPLIT,$0 - MOVQ z+0(FP), R10 - MOVQ x+24(FP), R8 - MOVQ y+48(FP), R9 - MOVQ r+56(FP), CX // c = r - MOVQ z_len+8(FP), R11 - MOVQ $0, BX // i = 0 - - CMPQ R11, $4 - JL E5 - -U5: // i+4 <= n - // regular loop body unrolled 4x - MOVQ (0*8)(R8)(BX*8), AX - MULQ R9 - ADDQ CX, AX - ADCQ $0, DX - MOVQ AX, (0*8)(R10)(BX*8) - MOVQ DX, CX - MOVQ (1*8)(R8)(BX*8), AX - MULQ R9 - ADDQ CX, AX - ADCQ $0, DX - MOVQ AX, (1*8)(R10)(BX*8) - MOVQ DX, CX - MOVQ (2*8)(R8)(BX*8), AX - MULQ R9 - ADDQ CX, AX - ADCQ $0, DX - MOVQ AX, (2*8)(R10)(BX*8) - MOVQ DX, CX - MOVQ (3*8)(R8)(BX*8), AX - MULQ R9 - ADDQ CX, AX - ADCQ $0, DX - MOVQ AX, (3*8)(R10)(BX*8) - MOVQ DX, CX - ADDQ $4, BX // i += 4 - - LEAQ 4(BX), DX - CMPQ DX, R11 - JLE U5 - JMP E5 - -L5: MOVQ (R8)(BX*8), AX - MULQ R9 - ADDQ CX, AX - ADCQ $0, DX - MOVQ AX, (R10)(BX*8) - MOVQ DX, CX - ADDQ $1, BX // i++ - -E5: CMPQ BX, R11 // i < n - JL L5 - - MOVQ CX, c+64(FP) +ret0: + MOVQ $0, c+56(FP) RET - -// func addMulVVW(z, x []Word, y Word) (c Word) -TEXT ·addMulVVW(SB),NOSPLIT,$0 - CMPB ·support_adx(SB), $1 - JEQ adx - MOVQ z+0(FP), R10 - MOVQ x+24(FP), R8 - MOVQ y+48(FP), R9 - MOVQ z_len+8(FP), R11 - MOVQ $0, BX // i = 0 - MOVQ $0, CX // c = 0 - MOVQ R11, R12 - ANDQ $-2, R12 - CMPQ R11, $2 - JAE A6 - JMP E6 - -A6: - MOVQ (R8)(BX*8), AX - MULQ R9 - ADDQ (R10)(BX*8), AX - ADCQ $0, DX - ADDQ CX, AX - ADCQ $0, DX - MOVQ DX, CX - MOVQ AX, (R10)(BX*8) - - MOVQ (8)(R8)(BX*8), AX - MULQ R9 - ADDQ (8)(R10)(BX*8), AX - ADCQ $0, DX - ADDQ CX, AX - ADCQ $0, DX - MOVQ DX, CX - MOVQ AX, (8)(R10)(BX*8) - - ADDQ $2, BX - CMPQ BX, R12 - JL A6 - JMP E6 - -L6: MOVQ (R8)(BX*8), AX - MULQ R9 - ADDQ CX, AX - ADCQ $0, DX - ADDQ AX, (R10)(BX*8) - ADCQ $0, DX - MOVQ DX, CX - ADDQ $1, BX // i++ - -E6: CMPQ BX, R11 // i < n - JL L6 - - MOVQ CX, c+56(FP) - RET - -adx: - MOVQ z_len+8(FP), R11 - MOVQ z+0(FP), R10 - MOVQ x+24(FP), R8 - MOVQ y+48(FP), DX - MOVQ $0, BX // i = 0 - MOVQ $0, CX // carry - CMPQ R11, $8 - JAE adx_loop_header - CMPQ BX, R11 - JL adx_short - MOVQ CX, c+56(FP) +// func mulAddVWW(z, x []Word, m, a Word) (c Word) +TEXT ·mulAddVWW(SB), NOSPLIT, $0 + MOVQ m+48(FP), BX + MOVQ a+56(FP), SI + MOVQ z_len+8(FP), DI + MOVQ x_base+24(FP), R8 + MOVQ z_base+0(FP), R9 + // compute unrolled loop lengths + MOVQ DI, R10 + ANDQ $3, R10 + SHRQ $2, DI +loop1: + TESTQ R10, R10; JZ loop1done +loop1cont: + // unroll 1X in batches of 1 + MOVQ 0(R8), AX + // multiply + MULQ BX + ADDQ SI, AX + MOVQ DX, SI + ADCQ $0, SI + MOVQ AX, 0(R9) + LEAQ 8(R8), R8 // ADD $8, R8 + LEAQ 8(R9), R9 // ADD $8, R9 + SUBQ $1, R10; JNZ loop1cont +loop1done: +loop4: + TESTQ DI, DI; JZ loop4done +loop4cont: + // unroll 4X in batches of 1 + MOVQ 0(R8), AX + // multiply + MULQ BX + ADDQ SI, AX + MOVQ DX, SI + ADCQ $0, SI + MOVQ AX, 0(R9) + MOVQ 8(R8), AX + // multiply + MULQ BX + ADDQ SI, AX + MOVQ DX, SI + ADCQ $0, SI + MOVQ AX, 8(R9) + MOVQ 16(R8), AX + // multiply + MULQ BX + ADDQ SI, AX + MOVQ DX, SI + ADCQ $0, SI + MOVQ AX, 16(R9) + MOVQ 24(R8), AX + // multiply + MULQ BX + ADDQ SI, AX + MOVQ DX, SI + ADCQ $0, SI + MOVQ AX, 24(R9) + LEAQ 32(R8), R8 // ADD $32, R8 + LEAQ 32(R9), R9 // ADD $32, R9 + SUBQ $1, DI; JNZ loop4cont +loop4done: + MOVQ SI, c+64(FP) RET -adx_loop_header: - MOVQ R11, R13 - ANDQ $-8, R13 -adx_loop: - XORQ R9, R9 // unset flags - MULXQ (R8), SI, DI - ADCXQ CX,SI - ADOXQ (R10), SI - MOVQ SI,(R10) - - MULXQ 8(R8), AX, CX - ADCXQ DI, AX - ADOXQ 8(R10), AX - MOVQ AX, 8(R10) - - MULXQ 16(R8), SI, DI - ADCXQ CX, SI - ADOXQ 16(R10), SI - MOVQ SI, 16(R10) - - MULXQ 24(R8), AX, CX - ADCXQ DI, AX - ADOXQ 24(R10), AX - MOVQ AX, 24(R10) - - MULXQ 32(R8), SI, DI - ADCXQ CX, SI - ADOXQ 32(R10), SI - MOVQ SI, 32(R10) - - MULXQ 40(R8), AX, CX - ADCXQ DI, AX - ADOXQ 40(R10), AX - MOVQ AX, 40(R10) - - MULXQ 48(R8), SI, DI - ADCXQ CX, SI - ADOXQ 48(R10), SI - MOVQ SI, 48(R10) - - MULXQ 56(R8), AX, CX - ADCXQ DI, AX - ADOXQ 56(R10), AX - MOVQ AX, 56(R10) - - ADCXQ R9, CX - ADOXQ R9, CX - - ADDQ $64, R8 - ADDQ $64, R10 - ADDQ $8, BX - - CMPQ BX, R13 - JL adx_loop - MOVQ z+0(FP), R10 - MOVQ x+24(FP), R8 - CMPQ BX, R11 - JL adx_short - MOVQ CX, c+56(FP) +// func addMulVVWW(z, x, y []Word, m, a Word) (c Word) +TEXT ·addMulVVWW(SB), NOSPLIT, $0 + CMPB ·hasADX(SB), $0; JNZ altcarry + MOVQ m+72(FP), BX + MOVQ a+80(FP), SI + MOVQ z_len+8(FP), DI + MOVQ x_base+24(FP), R8 + MOVQ y_base+48(FP), R9 + MOVQ z_base+0(FP), R10 + // compute unrolled loop lengths + MOVQ DI, R11 + ANDQ $3, R11 + SHRQ $2, DI +loop1: + TESTQ R11, R11; JZ loop1done +loop1cont: + // unroll 1X in batches of 1 + MOVQ 0(R9), AX + // multiply + MULQ BX + ADDQ SI, AX + MOVQ DX, SI + ADCQ $0, SI + // add + ADDQ 0(R8), AX + ADCQ $0, SI + MOVQ AX, 0(R10) + LEAQ 8(R8), R8 // ADD $8, R8 + LEAQ 8(R9), R9 // ADD $8, R9 + LEAQ 8(R10), R10 // ADD $8, R10 + SUBQ $1, R11; JNZ loop1cont +loop1done: +loop4: + TESTQ DI, DI; JZ loop4done +loop4cont: + // unroll 4X in batches of 1 + MOVQ 0(R9), AX + // multiply + MULQ BX + ADDQ SI, AX + MOVQ DX, SI + ADCQ $0, SI + // add + ADDQ 0(R8), AX + ADCQ $0, SI + MOVQ AX, 0(R10) + MOVQ 8(R9), AX + // multiply + MULQ BX + ADDQ SI, AX + MOVQ DX, SI + ADCQ $0, SI + // add + ADDQ 8(R8), AX + ADCQ $0, SI + MOVQ AX, 8(R10) + MOVQ 16(R9), AX + // multiply + MULQ BX + ADDQ SI, AX + MOVQ DX, SI + ADCQ $0, SI + // add + ADDQ 16(R8), AX + ADCQ $0, SI + MOVQ AX, 16(R10) + MOVQ 24(R9), AX + // multiply + MULQ BX + ADDQ SI, AX + MOVQ DX, SI + ADCQ $0, SI + // add + ADDQ 24(R8), AX + ADCQ $0, SI + MOVQ AX, 24(R10) + LEAQ 32(R8), R8 // ADD $32, R8 + LEAQ 32(R9), R9 // ADD $32, R9 + LEAQ 32(R10), R10 // ADD $32, R10 + SUBQ $1, DI; JNZ loop4cont +loop4done: + MOVQ SI, c+88(FP) RET - -adx_short: - MULXQ (R8)(BX*8), SI, DI - ADDQ CX, SI - ADCQ $0, DI - ADDQ SI, (R10)(BX*8) - ADCQ $0, DI - MOVQ DI, CX - ADDQ $1, BX // i++ - - CMPQ BX, R11 - JL adx_short - - MOVQ CX, c+56(FP) +altcarry: + MOVQ m+72(FP), DX + MOVQ a+80(FP), BX + MOVQ z_len+8(FP), SI + MOVQ $0, DI + MOVQ x_base+24(FP), R8 + MOVQ y_base+48(FP), R9 + MOVQ z_base+0(FP), R10 + // compute unrolled loop lengths + MOVQ SI, R11 + ANDQ $7, R11 + SHRQ $3, SI +alt1: + TESTQ R11, R11; JZ alt1done +alt1cont: + // unroll 1X + // multiply and add + TESTQ AX, AX // clear carry + TESTQ AX, AX // clear carry + MULXQ 0(R9), R13, R12 + ADCXQ BX, R13 + ADOXQ 0(R8), R13 + MOVQ R13, 0(R10) + MOVQ R12, BX + ADCXQ DI, BX + ADOXQ DI, BX + LEAQ 8(R8), R8 // ADD $8, R8 + LEAQ 8(R9), R9 // ADD $8, R9 + LEAQ 8(R10), R10 // ADD $8, R10 + SUBQ $1, R11; JNZ alt1cont +alt1done: +alt8: + TESTQ SI, SI; JZ alt8done +alt8cont: + // unroll 8X in batches of 2 + // multiply and add + TESTQ AX, AX // clear carry + TESTQ AX, AX // clear carry + MULXQ 0(R9), R13, R11 + ADCXQ BX, R13 + ADOXQ 0(R8), R13 + MULXQ 8(R9), R14, BX + ADCXQ R11, R14 + ADOXQ 8(R8), R14 + MOVQ R13, 0(R10) + MOVQ R14, 8(R10) + MULXQ 16(R9), R13, R11 + ADCXQ BX, R13 + ADOXQ 16(R8), R13 + MULXQ 24(R9), R14, BX + ADCXQ R11, R14 + ADOXQ 24(R8), R14 + MOVQ R13, 16(R10) + MOVQ R14, 24(R10) + MULXQ 32(R9), R13, R11 + ADCXQ BX, R13 + ADOXQ 32(R8), R13 + MULXQ 40(R9), R14, BX + ADCXQ R11, R14 + ADOXQ 40(R8), R14 + MOVQ R13, 32(R10) + MOVQ R14, 40(R10) + MULXQ 48(R9), R13, R11 + ADCXQ BX, R13 + ADOXQ 48(R8), R13 + MULXQ 56(R9), R14, BX + ADCXQ R11, R14 + ADOXQ 56(R8), R14 + MOVQ R13, 48(R10) + MOVQ R14, 56(R10) + ADCXQ DI, BX + ADOXQ DI, BX + LEAQ 64(R8), R8 // ADD $64, R8 + LEAQ 64(R9), R9 // ADD $64, R9 + LEAQ 64(R10), R10 // ADD $64, R10 + SUBQ $1, SI; JNZ alt8cont +alt8done: + MOVQ BX, c+88(FP) RET - - - diff --git a/src/math/big/arith_amd64_test.go b/src/math/big/arith_amd64_test.go new file mode 100644 index 00000000000000..62a67151190b70 --- /dev/null +++ b/src/math/big/arith_amd64_test.go @@ -0,0 +1,14 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !math_big_pure_go + +package big + +import "testing" + +func TestAddMulVVWWNoADX(t *testing.T) { + setDuringTest(t, &hasADX, false) + TestAddMulVVWW(t) +} diff --git a/src/math/big/arith_arm.s b/src/math/big/arith_arm.s index ece3a96f51246b..638b03ed40e0ce 100644 --- a/src/math/big/arith_arm.s +++ b/src/math/big/arith_arm.s @@ -1,272 +1,355 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2025 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Code generated by 'go generate' (with ./internal/asmgen). DO NOT EDIT. + //go:build !math_big_pure_go #include "textflag.h" -// This file provides fast assembly versions for the elementary -// arithmetic operations on vectors implemented in arith.go. - // func addVV(z, x, y []Word) (c Word) -TEXT ·addVV(SB),NOSPLIT,$0 - ADD.S $0, R0 // clear carry flag - MOVW z+0(FP), R1 - MOVW z_len+4(FP), R4 - MOVW x+12(FP), R2 - MOVW y+24(FP), R3 - ADD R4<<2, R1, R4 - B E1 -L1: - MOVW.P 4(R2), R5 - MOVW.P 4(R3), R6 - ADC.S R6, R5 - MOVW.P R5, 4(R1) -E1: - TEQ R1, R4 - BNE L1 - - MOVW $0, R0 - MOVW.CS $1, R0 - MOVW R0, c+36(FP) +TEXT ·addVV(SB), NOSPLIT, $0 + MOVW z_len+4(FP), R0 + MOVW x_base+12(FP), R1 + MOVW y_base+24(FP), R2 + MOVW z_base+0(FP), R3 + // compute unrolled loop lengths + AND $3, R0, R4 + MOVW R0>>2, R0 + ADD.S $0, R0 // clear carry +loop1: + TEQ $0, R4; BEQ loop1done +loop1cont: + // unroll 1X + MOVW.P 4(R1), R5 + MOVW.P 4(R2), R6 + ADC.S R6, R5 + MOVW.P R5, 4(R3) + SUB $1, R4 + TEQ $0, R4; BNE loop1cont +loop1done: +loop4: + TEQ $0, R0; BEQ loop4done +loop4cont: + // unroll 4X + MOVW.P 4(R1), R4 + MOVW.P 4(R1), R5 + MOVW.P 4(R1), R6 + MOVW.P 4(R1), R7 + MOVW.P 4(R2), R8 + MOVW.P 4(R2), R9 + MOVW.P 4(R2), R11 + MOVW.P 4(R2), R12 + ADC.S R8, R4 + ADC.S R9, R5 + ADC.S R11, R6 + ADC.S R12, R7 + MOVW.P R4, 4(R3) + MOVW.P R5, 4(R3) + MOVW.P R6, 4(R3) + MOVW.P R7, 4(R3) + SUB $1, R0 + TEQ $0, R0; BNE loop4cont +loop4done: + SBC R1, R1 // save carry + ADD $1, R1 // convert add carry + MOVW R1, c+36(FP) RET - // func subVV(z, x, y []Word) (c Word) -// (same as addVV except for SBC instead of ADC and label names) -TEXT ·subVV(SB),NOSPLIT,$0 - SUB.S $0, R0 // clear borrow flag - MOVW z+0(FP), R1 - MOVW z_len+4(FP), R4 - MOVW x+12(FP), R2 - MOVW y+24(FP), R3 - ADD R4<<2, R1, R4 - B E2 -L2: - MOVW.P 4(R2), R5 - MOVW.P 4(R3), R6 - SBC.S R6, R5 - MOVW.P R5, 4(R1) -E2: - TEQ R1, R4 - BNE L2 - - MOVW $0, R0 - MOVW.CC $1, R0 - MOVW R0, c+36(FP) +TEXT ·subVV(SB), NOSPLIT, $0 + MOVW z_len+4(FP), R0 + MOVW x_base+12(FP), R1 + MOVW y_base+24(FP), R2 + MOVW z_base+0(FP), R3 + // compute unrolled loop lengths + AND $3, R0, R4 + MOVW R0>>2, R0 + SUB.S $0, R0 // clear carry +loop1: + TEQ $0, R4; BEQ loop1done +loop1cont: + // unroll 1X + MOVW.P 4(R1), R5 + MOVW.P 4(R2), R6 + SBC.S R6, R5 + MOVW.P R5, 4(R3) + SUB $1, R4 + TEQ $0, R4; BNE loop1cont +loop1done: +loop4: + TEQ $0, R0; BEQ loop4done +loop4cont: + // unroll 4X + MOVW.P 4(R1), R4 + MOVW.P 4(R1), R5 + MOVW.P 4(R1), R6 + MOVW.P 4(R1), R7 + MOVW.P 4(R2), R8 + MOVW.P 4(R2), R9 + MOVW.P 4(R2), R11 + MOVW.P 4(R2), R12 + SBC.S R8, R4 + SBC.S R9, R5 + SBC.S R11, R6 + SBC.S R12, R7 + MOVW.P R4, 4(R3) + MOVW.P R5, 4(R3) + MOVW.P R6, 4(R3) + MOVW.P R7, 4(R3) + SUB $1, R0 + TEQ $0, R0; BNE loop4cont +loop4done: + SBC R1, R1 // save carry + RSB $0, R1, R1 // convert sub carry + MOVW R1, c+36(FP) RET - -// func addVW(z, x []Word, y Word) (c Word) -TEXT ·addVW(SB),NOSPLIT,$0 - MOVW z+0(FP), R1 - MOVW z_len+4(FP), R4 - MOVW x+12(FP), R2 - MOVW y+24(FP), R3 - ADD R4<<2, R1, R4 - TEQ R1, R4 - BNE L3a - MOVW R3, c+28(FP) - RET -L3a: - MOVW.P 4(R2), R5 - ADD.S R3, R5 - MOVW.P R5, 4(R1) - B E3 -L3: - MOVW.P 4(R2), R5 - ADC.S $0, R5 - MOVW.P R5, 4(R1) -E3: - TEQ R1, R4 - BNE L3 - - MOVW $0, R0 - MOVW.CS $1, R0 - MOVW R0, c+28(FP) +// func lshVU(z, x []Word, s uint) (c Word) +TEXT ·lshVU(SB), NOSPLIT, $0 + MOVW z_len+4(FP), R0 + TEQ $0, R0; BEQ ret0 + MOVW s+24(FP), R1 + MOVW x_base+12(FP), R2 + MOVW z_base+0(FP), R3 + // run loop backward + ADD R0<<2, R2, R2 + ADD R0<<2, R3, R3 + // shift first word into carry + MOVW.W -4(R2), R4 + MOVW $32, R5 + SUB R1, R5 + MOVW R4>>R5, R6 + MOVW R4<>2, R0 +loop1: + TEQ $0, R6; BEQ loop1done +loop1cont: + // unroll 1X + MOVW.W -4(R2), R7 + ORR R7>>R5, R4 + MOVW.W R4, -4(R3) + MOVW R7<>R5, R4 + MOVW.W R4, -4(R3) + MOVW R6<>R5, R4 + MOVW.W R4, -4(R3) + MOVW R7<>R5, R4 + MOVW.W R4, -4(R3) + MOVW R8<>R5, R4 + MOVW.W R4, -4(R3) + MOVW R9<>R1, R4 + MOVW R6, c+28(FP) + // shift remaining words + SUB $1, R0 + // compute unrolled loop lengths + AND $3, R0, R6 + MOVW R0>>2, R0 +loop1: + TEQ $0, R6; BEQ loop1done +loop1cont: + // unroll 1X + MOVW.P 4(R2), R7 + ORR R7<>R1, R4 + SUB $1, R6 + TEQ $0, R6; BNE loop1cont +loop1done: +loop4: + TEQ $0, R0; BEQ loop4done +loop4cont: + // unroll 4X + MOVW.P 4(R2), R6 + MOVW.P 4(R2), R7 + MOVW.P 4(R2), R8 + MOVW.P 4(R2), R9 + ORR R6<>R1, R4 + ORR R7<>R1, R4 + ORR R8<>R1, R4 + ORR R9<>R1, R4 + SUB $1, R0 + TEQ $0, R0; BNE loop4cont +loop4done: + // store final shifted bits + MOVW.P R4, 4(R3) RET - - -// func shlVU(z, x []Word, s uint) (c Word) -TEXT ·shlVU(SB),NOSPLIT,$0 - MOVW z_len+4(FP), R5 - TEQ $0, R5 - BEQ X7 - - MOVW z+0(FP), R1 - MOVW x+12(FP), R2 - ADD R5<<2, R2, R2 - ADD R5<<2, R1, R5 - MOVW s+24(FP), R3 - TEQ $0, R3 // shift 0 is special - BEQ Y7 - ADD $4, R1 // stop one word early - MOVW $32, R4 - SUB R3, R4 - MOVW $0, R7 - - MOVW.W -4(R2), R6 - MOVW R6<>R4, R6 - MOVW R6, c+28(FP) - B E7 - -L7: - MOVW.W -4(R2), R6 - ORR R6>>R4, R7 - MOVW.W R7, -4(R5) - MOVW R6<>2, R2 +loop1: + TEQ $0, R5; BEQ loop1done +loop1cont: + // unroll 1X + MOVW.P 4(R3), R6 + // multiply + MULLU R0, R6, (R7, R6) + ADD.S R1, R6 + ADC $0, R7, R1 + MOVW.P R6, 4(R4) + SUB $1, R5 + TEQ $0, R5; BNE loop1cont +loop1done: +loop4: + TEQ $0, R2; BEQ loop4done +loop4cont: + // unroll 4X in batches of 2 + MOVW.P 4(R3), R5 + MOVW.P 4(R3), R6 + // multiply + MULLU R0, R5, (R7, R5) + ADD.S R1, R5 + MULLU R0, R6, (R8, R6) + ADC.S R7, R6 + ADC $0, R8, R1 + MOVW.P R5, 4(R4) + MOVW.P R6, 4(R4) + MOVW.P 4(R3), R5 + MOVW.P 4(R3), R6 + // multiply + MULLU R0, R5, (R7, R5) + ADD.S R1, R5 + MULLU R0, R6, (R8, R6) + ADC.S R7, R6 + ADC $0, R8, R1 + MOVW.P R5, 4(R4) + MOVW.P R6, 4(R4) + SUB $1, R2 + TEQ $0, R2; BNE loop4cont +loop4done: + MOVW R1, c+32(FP) RET - -// func shrVU(z, x []Word, s uint) (c Word) -TEXT ·shrVU(SB),NOSPLIT,$0 - MOVW z_len+4(FP), R5 - TEQ $0, R5 - BEQ X6 - - MOVW z+0(FP), R1 - MOVW x+12(FP), R2 - ADD R5<<2, R1, R5 - MOVW s+24(FP), R3 - TEQ $0, R3 // shift 0 is special - BEQ Y6 - SUB $4, R5 // stop one word early - MOVW $32, R4 - SUB R3, R4 - MOVW $0, R7 - - // first word - MOVW.P 4(R2), R6 - MOVW R6>>R3, R7 - MOVW R6<>R3, R7 -E6: - TEQ R1, R5 - BNE L6 - - MOVW R7, 0(R1) - RET - -Y6: // copy loop, because shift 0 == shift 32 - MOVW.P 4(R2), R6 - MOVW.P R6, 4(R1) - TEQ R1, R5 - BNE Y6 - -X6: - MOVW $0, R1 - MOVW R1, c+28(FP) - RET - - -// func mulAddVWW(z, x []Word, y, r Word) (c Word) -TEXT ·mulAddVWW(SB),NOSPLIT,$0 - MOVW $0, R0 - MOVW z+0(FP), R1 - MOVW z_len+4(FP), R5 - MOVW x+12(FP), R2 - MOVW y+24(FP), R3 - MOVW r+28(FP), R4 - ADD R5<<2, R1, R5 - B E8 - - // word loop -L8: - MOVW.P 4(R2), R6 - MULLU R6, R3, (R7, R6) - ADD.S R4, R6 - ADC R0, R7 - MOVW.P R6, 4(R1) - MOVW R7, R4 -E8: - TEQ R1, R5 - BNE L8 - - MOVW R4, c+32(FP) - RET - - -// func addMulVVW(z, x []Word, y Word) (c Word) -TEXT ·addMulVVW(SB),NOSPLIT,$0 - MOVW $0, R0 - MOVW z+0(FP), R1 - MOVW z_len+4(FP), R5 - MOVW x+12(FP), R2 - MOVW y+24(FP), R3 - ADD R5<<2, R1, R5 - MOVW $0, R4 - B E9 - - // word loop -L9: - MOVW.P 4(R2), R6 - MULLU R6, R3, (R7, R6) - ADD.S R4, R6 - ADC R0, R7 - MOVW 0(R1), R4 - ADD.S R4, R6 - ADC R0, R7 - MOVW.P R6, 4(R1) - MOVW R7, R4 -E9: - TEQ R1, R5 - BNE L9 - - MOVW R4, c+28(FP) +// func addMulVVWW(z, x, y []Word, m, a Word) (c Word) +TEXT ·addMulVVWW(SB), NOSPLIT, $0 + MOVW m+36(FP), R0 + MOVW a+40(FP), R1 + MOVW z_len+4(FP), R2 + MOVW x_base+12(FP), R3 + MOVW y_base+24(FP), R4 + MOVW z_base+0(FP), R5 + // compute unrolled loop lengths + AND $3, R2, R6 + MOVW R2>>2, R2 +loop1: + TEQ $0, R6; BEQ loop1done +loop1cont: + // unroll 1X + MOVW.P 4(R3), R7 + MOVW.P 4(R4), R8 + // multiply + MULLU R0, R8, (R9, R8) + ADD.S R1, R8 + ADC $0, R9, R1 + // add + ADD.S R7, R8 + ADC $0, R1 + MOVW.P R8, 4(R5) + SUB $1, R6 + TEQ $0, R6; BNE loop1cont +loop1done: +loop4: + TEQ $0, R2; BEQ loop4done +loop4cont: + // unroll 4X in batches of 2 + MOVW.P 4(R3), R6 + MOVW.P 4(R3), R7 + MOVW.P 4(R4), R8 + MOVW.P 4(R4), R9 + // multiply + MULLU R0, R8, (R11, R8) + ADD.S R1, R8 + MULLU R0, R9, (R12, R9) + ADC.S R11, R9 + ADC $0, R12, R1 + // add + ADD.S R6, R8 + ADC.S R7, R9 + ADC $0, R1 + MOVW.P R8, 4(R5) + MOVW.P R9, 4(R5) + MOVW.P 4(R3), R6 + MOVW.P 4(R3), R7 + MOVW.P 4(R4), R8 + MOVW.P 4(R4), R9 + // multiply + MULLU R0, R8, (R11, R8) + ADD.S R1, R8 + MULLU R0, R9, (R12, R9) + ADC.S R11, R9 + ADC $0, R12, R1 + // add + ADD.S R6, R8 + ADC.S R7, R9 + ADC $0, R1 + MOVW.P R8, 4(R5) + MOVW.P R9, 4(R5) + SUB $1, R2 + TEQ $0, R2; BNE loop4cont +loop4done: + MOVW R1, c+44(FP) RET diff --git a/src/math/big/arith_arm64.s b/src/math/big/arith_arm64.s index 204006e01dcfe3..874930352b9a56 100644 --- a/src/math/big/arith_arm64.s +++ b/src/math/big/arith_arm64.s @@ -1,572 +1,374 @@ -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2025 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Code generated by 'go generate' (with ./internal/asmgen). DO NOT EDIT. + //go:build !math_big_pure_go #include "textflag.h" -// This file provides fast assembly versions for the elementary -// arithmetic operations on vectors implemented in arith.go. - -// TODO: Consider re-implementing using Advanced SIMD -// once the assembler supports those instructions. - // func addVV(z, x, y []Word) (c Word) -TEXT ·addVV(SB),NOSPLIT,$0 - MOVD z_len+8(FP), R0 - MOVD x+24(FP), R8 - MOVD y+48(FP), R9 - MOVD z+0(FP), R10 - ADDS $0, R0 // clear carry flag - TBZ $0, R0, two - MOVD.P 8(R8), R11 - MOVD.P 8(R9), R15 - ADCS R15, R11 - MOVD.P R11, 8(R10) - SUB $1, R0 -two: - TBZ $1, R0, loop - LDP.P 16(R8), (R11, R12) - LDP.P 16(R9), (R15, R16) - ADCS R15, R11 - ADCS R16, R12 - STP.P (R11, R12), 16(R10) - SUB $2, R0 -loop: - CBZ R0, done // careful not to touch the carry flag - LDP.P 32(R8), (R11, R12) - LDP -16(R8), (R13, R14) - LDP.P 32(R9), (R15, R16) - LDP -16(R9), (R17, R19) - ADCS R15, R11 - ADCS R16, R12 - ADCS R17, R13 - ADCS R19, R14 - STP.P (R11, R12), 32(R10) - STP (R13, R14), -16(R10) - SUB $4, R0 - B loop -done: - CSET HS, R0 // extract carry flag - MOVD R0, c+72(FP) +TEXT ·addVV(SB), NOSPLIT, $0 + MOVD z_len+8(FP), R0 + MOVD x_base+24(FP), R1 + MOVD y_base+48(FP), R2 + MOVD z_base+0(FP), R3 + // compute unrolled loop lengths + AND $3, R0, R4 + LSR $2, R0 + ADDS ZR, R0 // clear carry +loop1: + CBZ R4, loop1done +loop1cont: + // unroll 1X + MOVD.P 8(R1), R5 + MOVD.P 8(R2), R6 + ADCS R6, R5 + MOVD.P R5, 8(R3) + SUB $1, R4 + CBNZ R4, loop1cont +loop1done: +loop4: + CBZ R0, loop4done +loop4cont: + // unroll 4X + LDP.P 32(R1), (R4, R5) + LDP -16(R1), (R6, R7) + LDP.P 32(R2), (R8, R9) + LDP -16(R2), (R10, R11) + ADCS R8, R4 + ADCS R9, R5 + ADCS R10, R6 + ADCS R11, R7 + STP.P (R4, R5), 32(R3) + STP (R6, R7), -16(R3) + SUB $1, R0 + CBNZ R0, loop4cont +loop4done: + ADC ZR, ZR, R1 // save & convert add carry + MOVD R1, c+72(FP) RET - // func subVV(z, x, y []Word) (c Word) -TEXT ·subVV(SB),NOSPLIT,$0 - MOVD z_len+8(FP), R0 - MOVD x+24(FP), R8 - MOVD y+48(FP), R9 - MOVD z+0(FP), R10 - CMP R0, R0 // set carry flag - TBZ $0, R0, two - MOVD.P 8(R8), R11 - MOVD.P 8(R9), R15 - SBCS R15, R11 - MOVD.P R11, 8(R10) - SUB $1, R0 -two: - TBZ $1, R0, loop - LDP.P 16(R8), (R11, R12) - LDP.P 16(R9), (R15, R16) - SBCS R15, R11 - SBCS R16, R12 - STP.P (R11, R12), 16(R10) - SUB $2, R0 -loop: - CBZ R0, done // careful not to touch the carry flag - LDP.P 32(R8), (R11, R12) - LDP -16(R8), (R13, R14) - LDP.P 32(R9), (R15, R16) - LDP -16(R9), (R17, R19) - SBCS R15, R11 - SBCS R16, R12 - SBCS R17, R13 - SBCS R19, R14 - STP.P (R11, R12), 32(R10) - STP (R13, R14), -16(R10) - SUB $4, R0 - B loop -done: - CSET LO, R0 // extract carry flag - MOVD R0, c+72(FP) - RET - -#define vwOneOp(instr, op1) \ - MOVD.P 8(R1), R4; \ - instr op1, R4; \ - MOVD.P R4, 8(R3); - -// handle the first 1~4 elements before starting iteration in addVW/subVW -#define vwPreIter(instr1, instr2, counter, target) \ - vwOneOp(instr1, R2); \ - SUB $1, counter; \ - CBZ counter, target; \ - vwOneOp(instr2, $0); \ - SUB $1, counter; \ - CBZ counter, target; \ - vwOneOp(instr2, $0); \ - SUB $1, counter; \ - CBZ counter, target; \ - vwOneOp(instr2, $0); - -// do one iteration of add or sub in addVW/subVW -#define vwOneIter(instr, counter, exit) \ - CBZ counter, exit; \ // careful not to touch the carry flag - LDP.P 32(R1), (R4, R5); \ - LDP -16(R1), (R6, R7); \ - instr $0, R4, R8; \ - instr $0, R5, R9; \ - instr $0, R6, R10; \ - instr $0, R7, R11; \ - STP.P (R8, R9), 32(R3); \ - STP (R10, R11), -16(R3); \ - SUB $4, counter; - -// do one iteration of copy in addVW/subVW -#define vwOneIterCopy(counter, exit) \ - CBZ counter, exit; \ - LDP.P 32(R1), (R4, R5); \ - LDP -16(R1), (R6, R7); \ - STP.P (R4, R5), 32(R3); \ - STP (R6, R7), -16(R3); \ - SUB $4, counter; - -// func addVW(z, x []Word, y Word) (c Word) -// The 'large' branch handles large 'z'. It checks the carry flag on every iteration -// and switches to copy if we are done with carries. The copying is skipped as well -// if 'x' and 'z' happen to share the same underlying storage. -// The overhead of the checking and branching is visible when 'z' are small (~5%), -// so set a threshold of 32, and remain the small-sized part entirely untouched. -TEXT ·addVW(SB),NOSPLIT,$0 - MOVD z+0(FP), R3 - MOVD z_len+8(FP), R0 - MOVD x+24(FP), R1 - MOVD y+48(FP), R2 - CMP $32, R0 - BGE large // large-sized 'z' and 'x' - CBZ R0, len0 // the length of z is 0 - MOVD.P 8(R1), R4 - ADDS R2, R4 // z[0] = x[0] + y, set carry - MOVD.P R4, 8(R3) - SUB $1, R0 - CBZ R0, len1 // the length of z is 1 - TBZ $0, R0, two - MOVD.P 8(R1), R4 // do it once - ADCS $0, R4 - MOVD.P R4, 8(R3) - SUB $1, R0 -two: // do it twice - TBZ $1, R0, loop - LDP.P 16(R1), (R4, R5) - ADCS $0, R4, R8 // c, z[i] = x[i] + c - ADCS $0, R5, R9 - STP.P (R8, R9), 16(R3) - SUB $2, R0 -loop: // do four times per round - vwOneIter(ADCS, R0, len1) - B loop -len1: - CSET HS, R2 // extract carry flag -len0: - MOVD R2, c+56(FP) -done: +TEXT ·subVV(SB), NOSPLIT, $0 + MOVD z_len+8(FP), R0 + MOVD x_base+24(FP), R1 + MOVD y_base+48(FP), R2 + MOVD z_base+0(FP), R3 + // compute unrolled loop lengths + AND $3, R0, R4 + LSR $2, R0 + SUBS ZR, R0 // clear carry +loop1: + CBZ R4, loop1done +loop1cont: + // unroll 1X + MOVD.P 8(R1), R5 + MOVD.P 8(R2), R6 + SBCS R6, R5 + MOVD.P R5, 8(R3) + SUB $1, R4 + CBNZ R4, loop1cont +loop1done: +loop4: + CBZ R0, loop4done +loop4cont: + // unroll 4X + LDP.P 32(R1), (R4, R5) + LDP -16(R1), (R6, R7) + LDP.P 32(R2), (R8, R9) + LDP -16(R2), (R10, R11) + SBCS R8, R4 + SBCS R9, R5 + SBCS R10, R6 + SBCS R11, R7 + STP.P (R4, R5), 32(R3) + STP (R6, R7), -16(R3) + SUB $1, R0 + CBNZ R0, loop4cont +loop4done: + SBC R1, R1 // save carry + SUB R1, ZR, R1 // convert sub carry + MOVD R1, c+72(FP) RET -large: - AND $0x3, R0, R10 - AND $~0x3, R0 - // unrolling for the first 1~4 elements to avoid saving the carry - // flag in each step, adjust $R0 if we unrolled 4 elements - vwPreIter(ADDS, ADCS, R10, add4) - SUB $4, R0 -add4: - BCC copy - vwOneIter(ADCS, R0, len1) - B add4 -copy: - MOVD ZR, c+56(FP) - CMP R1, R3 - BEQ done -copy_4: // no carry flag, copy the rest - vwOneIterCopy(R0, done) - B copy_4 -// func subVW(z, x []Word, y Word) (c Word) -// The 'large' branch handles large 'z'. It checks the carry flag on every iteration -// and switches to copy if we are done with carries. The copying is skipped as well -// if 'x' and 'z' happen to share the same underlying storage. -// The overhead of the checking and branching is visible when 'z' are small (~5%), -// so set a threshold of 32, and remain the small-sized part entirely untouched. -TEXT ·subVW(SB),NOSPLIT,$0 - MOVD z+0(FP), R3 - MOVD z_len+8(FP), R0 - MOVD x+24(FP), R1 - MOVD y+48(FP), R2 - CMP $32, R0 - BGE large // large-sized 'z' and 'x' - CBZ R0, len0 // the length of z is 0 - MOVD.P 8(R1), R4 - SUBS R2, R4 // z[0] = x[0] - y, set carry - MOVD.P R4, 8(R3) - SUB $1, R0 - CBZ R0, len1 // the length of z is 1 - TBZ $0, R0, two // do it once - MOVD.P 8(R1), R4 - SBCS $0, R4 - MOVD.P R4, 8(R3) - SUB $1, R0 -two: // do it twice - TBZ $1, R0, loop - LDP.P 16(R1), (R4, R5) - SBCS $0, R4, R8 // c, z[i] = x[i] + c - SBCS $0, R5, R9 - STP.P (R8, R9), 16(R3) - SUB $2, R0 -loop: // do four times per round - vwOneIter(SBCS, R0, len1) - B loop -len1: - CSET LO, R2 // extract carry flag -len0: - MOVD R2, c+56(FP) -done: +// func lshVU(z, x []Word, s uint) (c Word) +TEXT ·lshVU(SB), NOSPLIT, $0 + MOVD z_len+8(FP), R0 + CBZ R0, ret0 + MOVD s+48(FP), R1 + MOVD x_base+24(FP), R2 + MOVD z_base+0(FP), R3 + // run loop backward + ADD R0<<3, R2, R2 + ADD R0<<3, R3, R3 + // shift first word into carry + MOVD.W -8(R2), R4 + MOVD $64, R5 + SUB R1, R5 + LSR R5, R4, R6 + LSL R1, R4 + MOVD R6, c+56(FP) + // shift remaining words + SUB $1, R0 + // compute unrolled loop lengths + AND $3, R0, R6 + LSR $2, R0 +loop1: + CBZ R6, loop1done +loop1cont: + // unroll 1X + MOVD.W -8(R2), R7 + LSR R5, R7, R8 + ORR R4, R8 + LSL R1, R7, R4 + MOVD.W R8, -8(R3) + SUB $1, R6 + CBNZ R6, loop1cont +loop1done: +loop4: + CBZ R0, loop4done +loop4cont: + // unroll 4X + LDP.W -32(R2), (R9, R8) + LDP 16(R2), (R7, R6) + LSR R5, R6, R10 + ORR R4, R10 + LSL R1, R6, R4 + LSR R5, R7, R6 + ORR R4, R6 + LSL R1, R7, R4 + LSR R5, R8, R7 + ORR R4, R7 + LSL R1, R8, R4 + LSR R5, R9, R8 + ORR R4, R8 + LSL R1, R9, R4 + STP.W (R8, R7), -32(R3) + STP (R6, R10), 16(R3) + SUB $1, R0 + CBNZ R0, loop4cont +loop4done: + // store final shifted bits + MOVD.W R4, -8(R3) RET -large: - AND $0x3, R0, R10 - AND $~0x3, R0 - // unrolling for the first 1~4 elements to avoid saving the carry - // flag in each step, adjust $R0 if we unrolled 4 elements - vwPreIter(SUBS, SBCS, R10, sub4) - SUB $4, R0 -sub4: - BCS copy - vwOneIter(SBCS, R0, len1) - B sub4 -copy: - MOVD ZR, c+56(FP) - CMP R1, R3 - BEQ done -copy_4: // no carry flag, copy the rest - vwOneIterCopy(R0, done) - B copy_4 - -// func shlVU(z, x []Word, s uint) (c Word) -// This implementation handles the shift operation from the high word to the low word, -// which may be an error for the case where the low word of x overlaps with the high -// word of z. When calling this function directly, you need to pay attention to this -// situation. -TEXT ·shlVU(SB),NOSPLIT,$0 - LDP z+0(FP), (R0, R1) // R0 = z.ptr, R1 = len(z) - MOVD x+24(FP), R2 - MOVD s+48(FP), R3 - ADD R1<<3, R0 // R0 = &z[n] - ADD R1<<3, R2 // R2 = &x[n] - CBZ R1, len0 - CBZ R3, copy // if the number of shift is 0, just copy x to z - MOVD $64, R4 - SUB R3, R4 - // handling the most significant element x[n-1] - MOVD.W -8(R2), R6 - LSR R4, R6, R5 // return value - LSL R3, R6, R8 // x[i] << s - SUB $1, R1 -one: TBZ $0, R1, two - MOVD.W -8(R2), R6 - LSR R4, R6, R7 - ORR R8, R7 - LSL R3, R6, R8 - SUB $1, R1 - MOVD.W R7, -8(R0) -two: - TBZ $1, R1, loop - LDP.W -16(R2), (R6, R7) - LSR R4, R7, R10 - ORR R8, R10 - LSL R3, R7 - LSR R4, R6, R9 - ORR R7, R9 - LSL R3, R6, R8 - SUB $2, R1 - STP.W (R9, R10), -16(R0) -loop: - CBZ R1, done - LDP.W -32(R2), (R10, R11) - LDP 16(R2), (R12, R13) - LSR R4, R13, R23 - ORR R8, R23 // z[i] = (x[i] << s) | (x[i-1] >> (64 - s)) - LSL R3, R13 - LSR R4, R12, R22 - ORR R13, R22 - LSL R3, R12 - LSR R4, R11, R21 - ORR R12, R21 - LSL R3, R11 - LSR R4, R10, R20 - ORR R11, R20 - LSL R3, R10, R8 - STP.W (R20, R21), -32(R0) - STP (R22, R23), 16(R0) - SUB $4, R1 - B loop -done: - MOVD.W R8, -8(R0) // the first element x[0] - MOVD R5, c+56(FP) // the part moved out from x[n-1] - RET -copy: - CMP R0, R2 - BEQ len0 - TBZ $0, R1, ctwo - MOVD.W -8(R2), R4 - MOVD.W R4, -8(R0) - SUB $1, R1 -ctwo: - TBZ $1, R1, cloop - LDP.W -16(R2), (R4, R5) - STP.W (R4, R5), -16(R0) - SUB $2, R1 -cloop: - CBZ R1, len0 - LDP.W -32(R2), (R4, R5) - LDP 16(R2), (R6, R7) - STP.W (R4, R5), -32(R0) - STP (R6, R7), 16(R0) - SUB $4, R1 - B cloop -len0: - MOVD $0, c+56(FP) +ret0: + MOVD ZR, c+56(FP) RET -// func shrVU(z, x []Word, s uint) (c Word) -// This implementation handles the shift operation from the low word to the high word, -// which may be an error for the case where the high word of x overlaps with the low -// word of z. When calling this function directly, you need to pay attention to this -// situation. -TEXT ·shrVU(SB),NOSPLIT,$0 - MOVD z+0(FP), R0 - MOVD z_len+8(FP), R1 - MOVD x+24(FP), R2 - MOVD s+48(FP), R3 - MOVD $0, R8 - MOVD $64, R4 - SUB R3, R4 - CBZ R1, len0 - CBZ R3, copy // if the number of shift is 0, just copy x to z - - MOVD.P 8(R2), R20 - LSR R3, R20, R8 - LSL R4, R20 - MOVD R20, c+56(FP) // deal with the first element - SUB $1, R1 - - TBZ $0, R1, two - MOVD.P 8(R2), R6 - LSL R4, R6, R20 - ORR R8, R20 - LSR R3, R6, R8 - MOVD.P R20, 8(R0) - SUB $1, R1 -two: - TBZ $1, R1, loop - LDP.P 16(R2), (R6, R7) - LSL R4, R6, R20 - LSR R3, R6 - ORR R8, R20 - LSL R4, R7, R21 - LSR R3, R7, R8 - ORR R6, R21 - STP.P (R20, R21), 16(R0) - SUB $2, R1 -loop: - CBZ R1, done - LDP.P 32(R2), (R10, R11) - LDP -16(R2), (R12, R13) - LSL R4, R10, R20 - LSR R3, R10 - ORR R8, R20 // z[i] = (x[i] >> s) | (x[i+1] << (64 - s)) - LSL R4, R11, R21 - LSR R3, R11 - ORR R10, R21 - LSL R4, R12, R22 - LSR R3, R12 - ORR R11, R22 - LSL R4, R13, R23 - LSR R3, R13, R8 - ORR R12, R23 - STP.P (R20, R21), 32(R0) - STP (R22, R23), -16(R0) - SUB $4, R1 - B loop -done: - MOVD R8, (R0) // deal with the last element +// func rshVU(z, x []Word, s uint) (c Word) +TEXT ·rshVU(SB), NOSPLIT, $0 + MOVD z_len+8(FP), R0 + CBZ R0, ret0 + MOVD s+48(FP), R1 + MOVD x_base+24(FP), R2 + MOVD z_base+0(FP), R3 + // shift first word into carry + MOVD.P 8(R2), R4 + MOVD $64, R5 + SUB R1, R5 + LSL R5, R4, R6 + LSR R1, R4 + MOVD R6, c+56(FP) + // shift remaining words + SUB $1, R0 + // compute unrolled loop lengths + AND $3, R0, R6 + LSR $2, R0 +loop1: + CBZ R6, loop1done +loop1cont: + // unroll 1X + MOVD.P 8(R2), R7 + LSL R5, R7, R8 + ORR R4, R8 + LSR R1, R7, R4 + MOVD.P R8, 8(R3) + SUB $1, R6 + CBNZ R6, loop1cont +loop1done: +loop4: + CBZ R0, loop4done +loop4cont: + // unroll 4X + LDP.P 32(R2), (R6, R7) + LDP -16(R2), (R8, R9) + LSL R5, R6, R10 + ORR R4, R10 + LSR R1, R6, R4 + LSL R5, R7, R6 + ORR R4, R6 + LSR R1, R7, R4 + LSL R5, R8, R7 + ORR R4, R7 + LSR R1, R8, R4 + LSL R5, R9, R8 + ORR R4, R8 + LSR R1, R9, R4 + STP.P (R10, R6), 32(R3) + STP (R7, R8), -16(R3) + SUB $1, R0 + CBNZ R0, loop4cont +loop4done: + // store final shifted bits + MOVD.P R4, 8(R3) RET -copy: - CMP R0, R2 - BEQ len0 - TBZ $0, R1, ctwo - MOVD.P 8(R2), R3 - MOVD.P R3, 8(R0) - SUB $1, R1 -ctwo: - TBZ $1, R1, cloop - LDP.P 16(R2), (R4, R5) - STP.P (R4, R5), 16(R0) - SUB $2, R1 -cloop: - CBZ R1, len0 - LDP.P 32(R2), (R4, R5) - LDP -16(R2), (R6, R7) - STP.P (R4, R5), 32(R0) - STP (R6, R7), -16(R0) - SUB $4, R1 - B cloop -len0: - MOVD $0, c+56(FP) +ret0: + MOVD ZR, c+56(FP) RET - -// func mulAddVWW(z, x []Word, y, r Word) (c Word) -TEXT ·mulAddVWW(SB),NOSPLIT,$0 - MOVD z+0(FP), R1 - MOVD z_len+8(FP), R0 - MOVD x+24(FP), R2 - MOVD y+48(FP), R3 - MOVD r+56(FP), R4 - // c, z = x * y + r - TBZ $0, R0, two - MOVD.P 8(R2), R5 - MUL R3, R5, R7 - UMULH R3, R5, R8 - ADDS R4, R7 - ADC $0, R8, R4 // c, z[i] = x[i] * y + r - MOVD.P R7, 8(R1) - SUB $1, R0 -two: - TBZ $1, R0, loop - LDP.P 16(R2), (R5, R6) - MUL R3, R5, R10 - UMULH R3, R5, R11 - ADDS R4, R10 - MUL R3, R6, R12 - UMULH R3, R6, R13 - ADCS R12, R11 - ADC $0, R13, R4 - - STP.P (R10, R11), 16(R1) - SUB $2, R0 -loop: - CBZ R0, done - LDP.P 32(R2), (R5, R6) - LDP -16(R2), (R7, R8) - - MUL R3, R5, R10 - UMULH R3, R5, R11 - ADDS R4, R10 - MUL R3, R6, R12 - UMULH R3, R6, R13 - ADCS R11, R12 - - MUL R3, R7, R14 - UMULH R3, R7, R15 - ADCS R13, R14 - MUL R3, R8, R16 - UMULH R3, R8, R17 - ADCS R15, R16 - ADC $0, R17, R4 - - STP.P (R10, R12), 32(R1) - STP (R14, R16), -16(R1) - SUB $4, R0 - B loop -done: - MOVD R4, c+64(FP) +// func mulAddVWW(z, x []Word, m, a Word) (c Word) +TEXT ·mulAddVWW(SB), NOSPLIT, $0 + MOVD m+48(FP), R0 + MOVD a+56(FP), R1 + MOVD z_len+8(FP), R2 + MOVD x_base+24(FP), R3 + MOVD z_base+0(FP), R4 + // compute unrolled loop lengths + AND $7, R2, R5 + LSR $3, R2 +loop1: + CBZ R5, loop1done +loop1cont: + // unroll 1X + MOVD.P 8(R3), R6 + // multiply + UMULH R0, R6, R7 + MUL R0, R6 + ADDS R1, R6 + ADC ZR, R7, R1 + MOVD.P R6, 8(R4) + SUB $1, R5 + CBNZ R5, loop1cont +loop1done: +loop8: + CBZ R2, loop8done +loop8cont: + // unroll 8X + LDP.P 64(R3), (R5, R6) + LDP -48(R3), (R7, R8) + LDP -32(R3), (R9, R10) + LDP -16(R3), (R11, R12) + // multiply + UMULH R0, R5, R13 + MUL R0, R5 + ADDS R1, R5 + UMULH R0, R6, R14 + MUL R0, R6 + ADCS R13, R6 + UMULH R0, R7, R13 + MUL R0, R7 + ADCS R14, R7 + UMULH R0, R8, R14 + MUL R0, R8 + ADCS R13, R8 + UMULH R0, R9, R13 + MUL R0, R9 + ADCS R14, R9 + UMULH R0, R10, R14 + MUL R0, R10 + ADCS R13, R10 + UMULH R0, R11, R13 + MUL R0, R11 + ADCS R14, R11 + UMULH R0, R12, R14 + MUL R0, R12 + ADCS R13, R12 + ADC ZR, R14, R1 + STP.P (R5, R6), 64(R4) + STP (R7, R8), -48(R4) + STP (R9, R10), -32(R4) + STP (R11, R12), -16(R4) + SUB $1, R2 + CBNZ R2, loop8cont +loop8done: + MOVD R1, c+64(FP) RET - -// func addMulVVW(z, x []Word, y Word) (c Word) -TEXT ·addMulVVW(SB),NOSPLIT,$0 - MOVD z+0(FP), R1 - MOVD z_len+8(FP), R0 - MOVD x+24(FP), R2 - MOVD y+48(FP), R3 - MOVD $0, R4 - - TBZ $0, R0, two - - MOVD.P 8(R2), R5 - MOVD (R1), R6 - - MUL R5, R3, R7 - UMULH R5, R3, R8 - - ADDS R7, R6 - ADC $0, R8, R4 - - MOVD.P R6, 8(R1) - SUB $1, R0 - -two: - TBZ $1, R0, loop - - LDP.P 16(R2), (R5, R10) - LDP (R1), (R6, R11) - - MUL R10, R3, R13 - UMULH R10, R3, R12 - - MUL R5, R3, R7 - UMULH R5, R3, R8 - - ADDS R4, R6 - ADCS R13, R11 - ADC $0, R12 - - ADDS R7, R6 - ADCS R8, R11 - ADC $0, R12, R4 - - STP.P (R6, R11), 16(R1) - SUB $2, R0 - -// The main loop of this code operates on a block of 4 words every iteration -// performing [R4:R12:R11:R10:R9] = R4 + R3 * [R8:R7:R6:R5] + [R12:R11:R10:R9] -// where R4 is carried from the previous iteration, R8:R7:R6:R5 hold the next -// 4 words of x, R3 is y and R12:R11:R10:R9 are part of the result z. -loop: - CBZ R0, done - - LDP.P 16(R2), (R5, R6) - LDP.P 16(R2), (R7, R8) - - LDP (R1), (R9, R10) - ADDS R4, R9 - MUL R6, R3, R14 - ADCS R14, R10 - MUL R7, R3, R15 - LDP 16(R1), (R11, R12) - ADCS R15, R11 - MUL R8, R3, R16 - ADCS R16, R12 - UMULH R8, R3, R20 - ADC $0, R20 - - MUL R5, R3, R13 - ADDS R13, R9 - UMULH R5, R3, R17 - ADCS R17, R10 - UMULH R6, R3, R21 - STP.P (R9, R10), 16(R1) - ADCS R21, R11 - UMULH R7, R3, R19 - ADCS R19, R12 - STP.P (R11, R12), 16(R1) - ADC $0, R20, R4 - - SUB $4, R0 - B loop - -done: - MOVD R4, c+56(FP) +// func addMulVVWW(z, x, y []Word, m, a Word) (c Word) +TEXT ·addMulVVWW(SB), NOSPLIT, $0 + MOVD m+72(FP), R0 + MOVD a+80(FP), R1 + MOVD z_len+8(FP), R2 + MOVD x_base+24(FP), R3 + MOVD y_base+48(FP), R4 + MOVD z_base+0(FP), R5 + // compute unrolled loop lengths + AND $7, R2, R6 + LSR $3, R2 +loop1: + CBZ R6, loop1done +loop1cont: + // unroll 1X + MOVD.P 8(R3), R7 + MOVD.P 8(R4), R8 + // multiply + UMULH R0, R8, R9 + MUL R0, R8 + ADDS R1, R8 + ADC ZR, R9, R1 + // add + ADDS R7, R8 + ADC ZR, R1 + MOVD.P R8, 8(R5) + SUB $1, R6 + CBNZ R6, loop1cont +loop1done: +loop8: + CBZ R2, loop8done +loop8cont: + // unroll 8X + LDP.P 64(R3), (R6, R7) + LDP -48(R3), (R8, R9) + LDP -32(R3), (R10, R11) + LDP -16(R3), (R12, R13) + LDP.P 64(R4), (R14, R15) + LDP -48(R4), (R16, R17) + LDP -32(R4), (R19, R20) + LDP -16(R4), (R21, R22) + // multiply + UMULH R0, R14, R23 + MUL R0, R14 + ADDS R1, R14 + UMULH R0, R15, R24 + MUL R0, R15 + ADCS R23, R15 + UMULH R0, R16, R23 + MUL R0, R16 + ADCS R24, R16 + UMULH R0, R17, R24 + MUL R0, R17 + ADCS R23, R17 + UMULH R0, R19, R23 + MUL R0, R19 + ADCS R24, R19 + UMULH R0, R20, R24 + MUL R0, R20 + ADCS R23, R20 + UMULH R0, R21, R23 + MUL R0, R21 + ADCS R24, R21 + UMULH R0, R22, R24 + MUL R0, R22 + ADCS R23, R22 + ADC ZR, R24, R1 + // add + ADDS R6, R14 + ADCS R7, R15 + ADCS R8, R16 + ADCS R9, R17 + ADCS R10, R19 + ADCS R11, R20 + ADCS R12, R21 + ADCS R13, R22 + ADC ZR, R1 + STP.P (R14, R15), 64(R5) + STP (R16, R17), -48(R5) + STP (R19, R20), -32(R5) + STP (R21, R22), -16(R5) + SUB $1, R2 + CBNZ R2, loop8cont +loop8done: + MOVD R1, c+88(FP) RET - - diff --git a/src/math/big/arith_decl.go b/src/math/big/arith_decl.go index 3230a781a9de1f..7dc94a5090a81a 100644 --- a/src/math/big/arith_decl.go +++ b/src/math/big/arith_decl.go @@ -4,6 +4,8 @@ //go:build !math_big_pure_go +//go:generate go test ./internal/asmgen -generate + package big import _ "unsafe" // for linkname @@ -34,7 +36,7 @@ func addVV(z, x, y []Word) (c Word) //go:noescape func subVV(z, x, y []Word) (c Word) -// addVW should be an internal detail, +// shlVU should be an internal detail (and a stale one at that), // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/remyoudompheng/bigfft @@ -42,36 +44,24 @@ func subVV(z, x, y []Word) (c Word) // Do not remove or change the type signature. // See go.dev/issue/67401. // -//go:linkname addVW -//go:noescape -func addVW(z, x []Word, y Word) (c Word) +//go:linkname shlVU +func shlVU(z, x []Word, s uint) (c Word) { + if s == 0 { + copy(z, x) + return 0 + } + return lshVU(z, x, s) +} -// subVW should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/remyoudompheng/bigfft +// lshVU sets z = x<>s, returning the low bits c. 1 ≤ s ≤ _B-1. // -//go:linkname shlVU -//go:noescape -func shlVU(z, x []Word, s uint) (c Word) - //go:noescape -func shrVU(z, x []Word, s uint) (c Word) +func rshVU(z, x []Word, s uint) (c Word) // mulAddVWW should be an internal detail, // but widely used packages access it using linkname. @@ -83,9 +73,9 @@ func shrVU(z, x []Word, s uint) (c Word) // //go:linkname mulAddVWW //go:noescape -func mulAddVWW(z, x []Word, y, r Word) (c Word) +func mulAddVWW(z, x []Word, m, a Word) (c Word) -// addMulVVW should be an internal detail, +// addMulVVW should be an internal detail (and a stale one at that), // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/remyoudompheng/bigfft @@ -94,5 +84,11 @@ func mulAddVWW(z, x []Word, y, r Word) (c Word) // See go.dev/issue/67401. // //go:linkname addMulVVW +func addMulVVW(z, x []Word, y Word) (c Word) { + return addMulVVWW(z, z, x, y, 0) +} + +// addMulVVWW sets z = x+y*m+a. +// //go:noescape -func addMulVVW(z, x []Word, y Word) (c Word) +func addMulVVWW(z, x, y []Word, m, a Word) (c Word) diff --git a/src/math/big/arith_decl_pure.go b/src/math/big/arith_decl_pure.go index 4d7bbc87716331..3b051356fb24b7 100644 --- a/src/math/big/arith_decl_pure.go +++ b/src/math/big/arith_decl_pure.go @@ -14,36 +14,18 @@ func subVV(z, x, y []Word) (c Word) { return subVV_g(z, x, y) } -func addVW(z, x []Word, y Word) (c Word) { - // TODO: remove indirect function call when golang.org/issue/30548 is fixed - fn := addVW_g - if len(z) > 32 { - fn = addVWlarge - } - return fn(z, x, y) +func lshVU(z, x []Word, s uint) (c Word) { + return lshVU_g(z, x, s) } -func subVW(z, x []Word, y Word) (c Word) { - // TODO: remove indirect function call when golang.org/issue/30548 is fixed - fn := subVW_g - if len(z) > 32 { - fn = subVWlarge - } - return fn(z, x, y) -} - -func shlVU(z, x []Word, s uint) (c Word) { - return shlVU_g(z, x, s) -} - -func shrVU(z, x []Word, s uint) (c Word) { - return shrVU_g(z, x, s) +func rshVU(z, x []Word, s uint) (c Word) { + return rshVU_g(z, x, s) } func mulAddVWW(z, x []Word, y, r Word) (c Word) { return mulAddVWW_g(z, x, y, r) } -func addMulVVW(z, x []Word, y Word) (c Word) { - return addMulVVW_g(z, x, y) +func addMulVVWW(z, x, y []Word, m, a Word) (c Word) { + return addMulVVWW_g(z, x, y, m, a) } diff --git a/src/math/big/arith_decl_s390x.go b/src/math/big/arith_decl_s390x.go deleted file mode 100644 index 653916687812bf..00000000000000 --- a/src/math/big/arith_decl_s390x.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !math_big_pure_go - -package big - -import "internal/cpu" - -func addVV_check(z, x, y []Word) (c Word) -func addVV_vec(z, x, y []Word) (c Word) -func addVV_novec(z, x, y []Word) (c Word) -func subVV_check(z, x, y []Word) (c Word) -func subVV_vec(z, x, y []Word) (c Word) -func subVV_novec(z, x, y []Word) (c Word) - -var hasVX = cpu.S390X.HasVX diff --git a/src/math/big/arith_loong64.s b/src/math/big/arith_loong64.s index 847e3127fbf410..b2af9251245a7e 100644 --- a/src/math/big/arith_loong64.s +++ b/src/math/big/arith_loong64.s @@ -1,34 +1,457 @@ -// Copyright 2022 The Go Authors. All rights reserved. +// Copyright 2025 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !math_big_pure_go && loong64 +// Code generated by 'go generate' (with ./internal/asmgen). DO NOT EDIT. -#include "textflag.h" - -// This file provides fast assembly versions for the elementary -// arithmetic operations on vectors implemented in arith.go. +//go:build !math_big_pure_go -TEXT ·addVV(SB),NOSPLIT,$0 - JMP ·addVV_g(SB) - -TEXT ·subVV(SB),NOSPLIT,$0 - JMP ·subVV_g(SB) +#include "textflag.h" -TEXT ·addVW(SB),NOSPLIT,$0 - JMP ·addVW_g(SB) +// func addVV(z, x, y []Word) (c Word) +TEXT ·addVV(SB), NOSPLIT, $0 + MOVV z_len+8(FP), R4 + MOVV x_base+24(FP), R5 + MOVV y_base+48(FP), R6 + MOVV z_base+0(FP), R7 + // compute unrolled loop lengths + AND $3, R4, R8 + SRLV $2, R4 + XOR R28, R28 // clear carry +loop1: + BEQ R8, loop1done +loop1cont: + // unroll 1X + MOVV 0(R5), R9 + MOVV 0(R6), R10 + ADDVU R10, R9 // ADCS R10, R9, R9 (cr=R28) + SGTU R10, R9, R30 // ... + ADDVU R28, R9 // ... + SGTU R28, R9, R28 // ... + ADDVU R30, R28 // ... + MOVV R9, 0(R7) + ADDVU $8, R5 + ADDVU $8, R6 + ADDVU $8, R7 + SUBVU $1, R8 + BNE R8, loop1cont +loop1done: +loop4: + BEQ R4, loop4done +loop4cont: + // unroll 4X + MOVV 0(R5), R8 + MOVV 8(R5), R9 + MOVV 16(R5), R10 + MOVV 24(R5), R11 + MOVV 0(R6), R12 + MOVV 8(R6), R13 + MOVV 16(R6), R14 + MOVV 24(R6), R15 + ADDVU R12, R8 // ADCS R12, R8, R8 (cr=R28) + SGTU R12, R8, R30 // ... + ADDVU R28, R8 // ... + SGTU R28, R8, R28 // ... + ADDVU R30, R28 // ... + ADDVU R13, R9 // ADCS R13, R9, R9 (cr=R28) + SGTU R13, R9, R30 // ... + ADDVU R28, R9 // ... + SGTU R28, R9, R28 // ... + ADDVU R30, R28 // ... + ADDVU R14, R10 // ADCS R14, R10, R10 (cr=R28) + SGTU R14, R10, R30 // ... + ADDVU R28, R10 // ... + SGTU R28, R10, R28 // ... + ADDVU R30, R28 // ... + ADDVU R15, R11 // ADCS R15, R11, R11 (cr=R28) + SGTU R15, R11, R30 // ... + ADDVU R28, R11 // ... + SGTU R28, R11, R28 // ... + ADDVU R30, R28 // ... + MOVV R8, 0(R7) + MOVV R9, 8(R7) + MOVV R10, 16(R7) + MOVV R11, 24(R7) + ADDVU $32, R5 + ADDVU $32, R6 + ADDVU $32, R7 + SUBVU $1, R4 + BNE R4, loop4cont +loop4done: + MOVV R28, c+72(FP) + RET -TEXT ·subVW(SB),NOSPLIT,$0 - JMP ·subVW_g(SB) +// func subVV(z, x, y []Word) (c Word) +TEXT ·subVV(SB), NOSPLIT, $0 + MOVV z_len+8(FP), R4 + MOVV x_base+24(FP), R5 + MOVV y_base+48(FP), R6 + MOVV z_base+0(FP), R7 + // compute unrolled loop lengths + AND $3, R4, R8 + SRLV $2, R4 + XOR R28, R28 // clear carry +loop1: + BEQ R8, loop1done +loop1cont: + // unroll 1X + MOVV 0(R5), R9 + MOVV 0(R6), R10 + SGTU R28, R9, R30 // SBCS R10, R9, R9 + SUBVU R28, R9 // ... + SGTU R10, R9, R28 // ... + SUBVU R10, R9 // ... + ADDVU R30, R28 // ... + MOVV R9, 0(R7) + ADDVU $8, R5 + ADDVU $8, R6 + ADDVU $8, R7 + SUBVU $1, R8 + BNE R8, loop1cont +loop1done: +loop4: + BEQ R4, loop4done +loop4cont: + // unroll 4X + MOVV 0(R5), R8 + MOVV 8(R5), R9 + MOVV 16(R5), R10 + MOVV 24(R5), R11 + MOVV 0(R6), R12 + MOVV 8(R6), R13 + MOVV 16(R6), R14 + MOVV 24(R6), R15 + SGTU R28, R8, R30 // SBCS R12, R8, R8 + SUBVU R28, R8 // ... + SGTU R12, R8, R28 // ... + SUBVU R12, R8 // ... + ADDVU R30, R28 // ... + SGTU R28, R9, R30 // SBCS R13, R9, R9 + SUBVU R28, R9 // ... + SGTU R13, R9, R28 // ... + SUBVU R13, R9 // ... + ADDVU R30, R28 // ... + SGTU R28, R10, R30 // SBCS R14, R10, R10 + SUBVU R28, R10 // ... + SGTU R14, R10, R28 // ... + SUBVU R14, R10 // ... + ADDVU R30, R28 // ... + SGTU R28, R11, R30 // SBCS R15, R11, R11 + SUBVU R28, R11 // ... + SGTU R15, R11, R28 // ... + SUBVU R15, R11 // ... + ADDVU R30, R28 // ... + MOVV R8, 0(R7) + MOVV R9, 8(R7) + MOVV R10, 16(R7) + MOVV R11, 24(R7) + ADDVU $32, R5 + ADDVU $32, R6 + ADDVU $32, R7 + SUBVU $1, R4 + BNE R4, loop4cont +loop4done: + MOVV R28, c+72(FP) + RET -TEXT ·shlVU(SB),NOSPLIT,$0 - JMP ·shlVU_g(SB) +// func lshVU(z, x []Word, s uint) (c Word) +TEXT ·lshVU(SB), NOSPLIT, $0 + MOVV z_len+8(FP), R4 + BEQ R4, ret0 + MOVV s+48(FP), R5 + MOVV x_base+24(FP), R6 + MOVV z_base+0(FP), R7 + // run loop backward + SLLV $3, R4, R8 + ADDVU R8, R6 + SLLV $3, R4, R8 + ADDVU R8, R7 + // shift first word into carry + MOVV -8(R6), R8 + MOVV $64, R9 + SUBVU R5, R9 + SRLV R9, R8, R10 + SLLV R5, R8 + MOVV R10, c+56(FP) + // shift remaining words + SUBVU $1, R4 + // compute unrolled loop lengths + AND $3, R4, R10 + SRLV $2, R4 +loop1: + BEQ R10, loop1done +loop1cont: + // unroll 1X + MOVV -16(R6), R11 + SRLV R9, R11, R12 + OR R8, R12 + SLLV R5, R11, R8 + MOVV R12, -8(R7) + ADDVU $-8, R6 + ADDVU $-8, R7 + SUBVU $1, R10 + BNE R10, loop1cont +loop1done: +loop4: + BEQ R4, loop4done +loop4cont: + // unroll 4X + MOVV -16(R6), R10 + MOVV -24(R6), R11 + MOVV -32(R6), R12 + MOVV -40(R6), R13 + SRLV R9, R10, R14 + OR R8, R14 + SLLV R5, R10, R8 + SRLV R9, R11, R10 + OR R8, R10 + SLLV R5, R11, R8 + SRLV R9, R12, R11 + OR R8, R11 + SLLV R5, R12, R8 + SRLV R9, R13, R12 + OR R8, R12 + SLLV R5, R13, R8 + MOVV R14, -8(R7) + MOVV R10, -16(R7) + MOVV R11, -24(R7) + MOVV R12, -32(R7) + ADDVU $-32, R6 + ADDVU $-32, R7 + SUBVU $1, R4 + BNE R4, loop4cont +loop4done: + // store final shifted bits + MOVV R8, -8(R7) + RET +ret0: + MOVV R0, c+56(FP) + RET -TEXT ·shrVU(SB),NOSPLIT,$0 - JMP ·shrVU_g(SB) +// func rshVU(z, x []Word, s uint) (c Word) +TEXT ·rshVU(SB), NOSPLIT, $0 + MOVV z_len+8(FP), R4 + BEQ R4, ret0 + MOVV s+48(FP), R5 + MOVV x_base+24(FP), R6 + MOVV z_base+0(FP), R7 + // shift first word into carry + MOVV 0(R6), R8 + MOVV $64, R9 + SUBVU R5, R9 + SLLV R9, R8, R10 + SRLV R5, R8 + MOVV R10, c+56(FP) + // shift remaining words + SUBVU $1, R4 + // compute unrolled loop lengths + AND $3, R4, R10 + SRLV $2, R4 +loop1: + BEQ R10, loop1done +loop1cont: + // unroll 1X + MOVV 8(R6), R11 + SLLV R9, R11, R12 + OR R8, R12 + SRLV R5, R11, R8 + MOVV R12, 0(R7) + ADDVU $8, R6 + ADDVU $8, R7 + SUBVU $1, R10 + BNE R10, loop1cont +loop1done: +loop4: + BEQ R4, loop4done +loop4cont: + // unroll 4X + MOVV 8(R6), R10 + MOVV 16(R6), R11 + MOVV 24(R6), R12 + MOVV 32(R6), R13 + SLLV R9, R10, R14 + OR R8, R14 + SRLV R5, R10, R8 + SLLV R9, R11, R10 + OR R8, R10 + SRLV R5, R11, R8 + SLLV R9, R12, R11 + OR R8, R11 + SRLV R5, R12, R8 + SLLV R9, R13, R12 + OR R8, R12 + SRLV R5, R13, R8 + MOVV R14, 0(R7) + MOVV R10, 8(R7) + MOVV R11, 16(R7) + MOVV R12, 24(R7) + ADDVU $32, R6 + ADDVU $32, R7 + SUBVU $1, R4 + BNE R4, loop4cont +loop4done: + // store final shifted bits + MOVV R8, 0(R7) + RET +ret0: + MOVV R0, c+56(FP) + RET -TEXT ·mulAddVWW(SB),NOSPLIT,$0 - JMP ·mulAddVWW_g(SB) +// func mulAddVWW(z, x []Word, m, a Word) (c Word) +TEXT ·mulAddVWW(SB), NOSPLIT, $0 + MOVV m+48(FP), R4 + MOVV a+56(FP), R5 + MOVV z_len+8(FP), R6 + MOVV x_base+24(FP), R7 + MOVV z_base+0(FP), R8 + // compute unrolled loop lengths + AND $3, R6, R9 + SRLV $2, R6 +loop1: + BEQ R9, loop1done +loop1cont: + // unroll 1X + MOVV 0(R7), R10 + // synthetic carry, one column at a time + MULV R4, R10, R11 + MULHVU R4, R10, R12 + ADDVU R5, R11, R10 // ADDS R5, R11, R10 (cr=R28) + SGTU R5, R10, R28 // ... + ADDVU R28, R12, R5 // ADC $0, R12, R5 + MOVV R10, 0(R8) + ADDVU $8, R7 + ADDVU $8, R8 + SUBVU $1, R9 + BNE R9, loop1cont +loop1done: +loop4: + BEQ R6, loop4done +loop4cont: + // unroll 4X + MOVV 0(R7), R9 + MOVV 8(R7), R10 + MOVV 16(R7), R11 + MOVV 24(R7), R12 + // synthetic carry, one column at a time + MULV R4, R9, R13 + MULHVU R4, R9, R14 + ADDVU R5, R13, R9 // ADDS R5, R13, R9 (cr=R28) + SGTU R5, R9, R28 // ... + ADDVU R28, R14, R5 // ADC $0, R14, R5 + MULV R4, R10, R13 + MULHVU R4, R10, R14 + ADDVU R5, R13, R10 // ADDS R5, R13, R10 (cr=R28) + SGTU R5, R10, R28 // ... + ADDVU R28, R14, R5 // ADC $0, R14, R5 + MULV R4, R11, R13 + MULHVU R4, R11, R14 + ADDVU R5, R13, R11 // ADDS R5, R13, R11 (cr=R28) + SGTU R5, R11, R28 // ... + ADDVU R28, R14, R5 // ADC $0, R14, R5 + MULV R4, R12, R13 + MULHVU R4, R12, R14 + ADDVU R5, R13, R12 // ADDS R5, R13, R12 (cr=R28) + SGTU R5, R12, R28 // ... + ADDVU R28, R14, R5 // ADC $0, R14, R5 + MOVV R9, 0(R8) + MOVV R10, 8(R8) + MOVV R11, 16(R8) + MOVV R12, 24(R8) + ADDVU $32, R7 + ADDVU $32, R8 + SUBVU $1, R6 + BNE R6, loop4cont +loop4done: + MOVV R5, c+64(FP) + RET -TEXT ·addMulVVW(SB),NOSPLIT,$0 - JMP ·addMulVVW_g(SB) +// func addMulVVWW(z, x, y []Word, m, a Word) (c Word) +TEXT ·addMulVVWW(SB), NOSPLIT, $0 + MOVV m+72(FP), R4 + MOVV a+80(FP), R5 + MOVV z_len+8(FP), R6 + MOVV x_base+24(FP), R7 + MOVV y_base+48(FP), R8 + MOVV z_base+0(FP), R9 + // compute unrolled loop lengths + AND $3, R6, R10 + SRLV $2, R6 +loop1: + BEQ R10, loop1done +loop1cont: + // unroll 1X + MOVV 0(R7), R11 + MOVV 0(R8), R12 + // synthetic carry, one column at a time + MULV R4, R12, R13 + MULHVU R4, R12, R14 + ADDVU R11, R13 // ADDS R11, R13, R13 (cr=R28) + SGTU R11, R13, R28 // ... + ADDVU R28, R14 // ADC $0, R14, R14 + ADDVU R5, R13, R12 // ADDS R5, R13, R12 (cr=R28) + SGTU R5, R12, R28 // ... + ADDVU R28, R14, R5 // ADC $0, R14, R5 + MOVV R12, 0(R9) + ADDVU $8, R7 + ADDVU $8, R8 + ADDVU $8, R9 + SUBVU $1, R10 + BNE R10, loop1cont +loop1done: +loop4: + BEQ R6, loop4done +loop4cont: + // unroll 4X + MOVV 0(R7), R10 + MOVV 8(R7), R11 + MOVV 16(R7), R12 + MOVV 24(R7), R13 + MOVV 0(R8), R14 + MOVV 8(R8), R15 + MOVV 16(R8), R16 + MOVV 24(R8), R17 + // synthetic carry, one column at a time + MULV R4, R14, R18 + MULHVU R4, R14, R19 + ADDVU R10, R18 // ADDS R10, R18, R18 (cr=R28) + SGTU R10, R18, R28 // ... + ADDVU R28, R19 // ADC $0, R19, R19 + ADDVU R5, R18, R14 // ADDS R5, R18, R14 (cr=R28) + SGTU R5, R14, R28 // ... + ADDVU R28, R19, R5 // ADC $0, R19, R5 + MULV R4, R15, R18 + MULHVU R4, R15, R19 + ADDVU R11, R18 // ADDS R11, R18, R18 (cr=R28) + SGTU R11, R18, R28 // ... + ADDVU R28, R19 // ADC $0, R19, R19 + ADDVU R5, R18, R15 // ADDS R5, R18, R15 (cr=R28) + SGTU R5, R15, R28 // ... + ADDVU R28, R19, R5 // ADC $0, R19, R5 + MULV R4, R16, R18 + MULHVU R4, R16, R19 + ADDVU R12, R18 // ADDS R12, R18, R18 (cr=R28) + SGTU R12, R18, R28 // ... + ADDVU R28, R19 // ADC $0, R19, R19 + ADDVU R5, R18, R16 // ADDS R5, R18, R16 (cr=R28) + SGTU R5, R16, R28 // ... + ADDVU R28, R19, R5 // ADC $0, R19, R5 + MULV R4, R17, R18 + MULHVU R4, R17, R19 + ADDVU R13, R18 // ADDS R13, R18, R18 (cr=R28) + SGTU R13, R18, R28 // ... + ADDVU R28, R19 // ADC $0, R19, R19 + ADDVU R5, R18, R17 // ADDS R5, R18, R17 (cr=R28) + SGTU R5, R17, R28 // ... + ADDVU R28, R19, R5 // ADC $0, R19, R5 + MOVV R14, 0(R9) + MOVV R15, 8(R9) + MOVV R16, 16(R9) + MOVV R17, 24(R9) + ADDVU $32, R7 + ADDVU $32, R8 + ADDVU $32, R9 + SUBVU $1, R6 + BNE R6, loop4cont +loop4done: + MOVV R5, c+88(FP) + RET diff --git a/src/math/big/arith_mips64x.s b/src/math/big/arith_mips64x.s index 393a3efb9bcef2..0c0fc2db35da52 100644 --- a/src/math/big/arith_mips64x.s +++ b/src/math/big/arith_mips64x.s @@ -1,35 +1,467 @@ -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2025 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Code generated by 'go generate' (with ./internal/asmgen). DO NOT EDIT. + //go:build !math_big_pure_go && (mips64 || mips64le) #include "textflag.h" -// This file provides fast assembly versions for the elementary -// arithmetic operations on vectors implemented in arith.go. - -TEXT ·addVV(SB),NOSPLIT,$0 - JMP ·addVV_g(SB) - -TEXT ·subVV(SB),NOSPLIT,$0 - JMP ·subVV_g(SB) - -TEXT ·addVW(SB),NOSPLIT,$0 - JMP ·addVW_g(SB) - -TEXT ·subVW(SB),NOSPLIT,$0 - JMP ·subVW_g(SB) +// func addVV(z, x, y []Word) (c Word) +TEXT ·addVV(SB), NOSPLIT, $0 + MOVV z_len+8(FP), R1 + MOVV x_base+24(FP), R2 + MOVV y_base+48(FP), R3 + MOVV z_base+0(FP), R4 + // compute unrolled loop lengths + AND $3, R1, R5 + SRLV $2, R1 + XOR R24, R24 // clear carry +loop1: + BEQ R5, loop1done +loop1cont: + // unroll 1X + MOVV 0(R2), R6 + MOVV 0(R3), R7 + ADDVU R7, R6 // ADCS R7, R6, R6 (cr=R24) + SGTU R7, R6, R23 // ... + ADDVU R24, R6 // ... + SGTU R24, R6, R24 // ... + ADDVU R23, R24 // ... + MOVV R6, 0(R4) + ADDVU $8, R2 + ADDVU $8, R3 + ADDVU $8, R4 + SUBVU $1, R5 + BNE R5, loop1cont +loop1done: +loop4: + BEQ R1, loop4done +loop4cont: + // unroll 4X + MOVV 0(R2), R5 + MOVV 8(R2), R6 + MOVV 16(R2), R7 + MOVV 24(R2), R8 + MOVV 0(R3), R9 + MOVV 8(R3), R10 + MOVV 16(R3), R11 + MOVV 24(R3), R12 + ADDVU R9, R5 // ADCS R9, R5, R5 (cr=R24) + SGTU R9, R5, R23 // ... + ADDVU R24, R5 // ... + SGTU R24, R5, R24 // ... + ADDVU R23, R24 // ... + ADDVU R10, R6 // ADCS R10, R6, R6 (cr=R24) + SGTU R10, R6, R23 // ... + ADDVU R24, R6 // ... + SGTU R24, R6, R24 // ... + ADDVU R23, R24 // ... + ADDVU R11, R7 // ADCS R11, R7, R7 (cr=R24) + SGTU R11, R7, R23 // ... + ADDVU R24, R7 // ... + SGTU R24, R7, R24 // ... + ADDVU R23, R24 // ... + ADDVU R12, R8 // ADCS R12, R8, R8 (cr=R24) + SGTU R12, R8, R23 // ... + ADDVU R24, R8 // ... + SGTU R24, R8, R24 // ... + ADDVU R23, R24 // ... + MOVV R5, 0(R4) + MOVV R6, 8(R4) + MOVV R7, 16(R4) + MOVV R8, 24(R4) + ADDVU $32, R2 + ADDVU $32, R3 + ADDVU $32, R4 + SUBVU $1, R1 + BNE R1, loop4cont +loop4done: + MOVV R24, c+72(FP) + RET -TEXT ·shlVU(SB),NOSPLIT,$0 - JMP ·shlVU_g(SB) +// func subVV(z, x, y []Word) (c Word) +TEXT ·subVV(SB), NOSPLIT, $0 + MOVV z_len+8(FP), R1 + MOVV x_base+24(FP), R2 + MOVV y_base+48(FP), R3 + MOVV z_base+0(FP), R4 + // compute unrolled loop lengths + AND $3, R1, R5 + SRLV $2, R1 + XOR R24, R24 // clear carry +loop1: + BEQ R5, loop1done +loop1cont: + // unroll 1X + MOVV 0(R2), R6 + MOVV 0(R3), R7 + SGTU R24, R6, R23 // SBCS R7, R6, R6 + SUBVU R24, R6 // ... + SGTU R7, R6, R24 // ... + SUBVU R7, R6 // ... + ADDVU R23, R24 // ... + MOVV R6, 0(R4) + ADDVU $8, R2 + ADDVU $8, R3 + ADDVU $8, R4 + SUBVU $1, R5 + BNE R5, loop1cont +loop1done: +loop4: + BEQ R1, loop4done +loop4cont: + // unroll 4X + MOVV 0(R2), R5 + MOVV 8(R2), R6 + MOVV 16(R2), R7 + MOVV 24(R2), R8 + MOVV 0(R3), R9 + MOVV 8(R3), R10 + MOVV 16(R3), R11 + MOVV 24(R3), R12 + SGTU R24, R5, R23 // SBCS R9, R5, R5 + SUBVU R24, R5 // ... + SGTU R9, R5, R24 // ... + SUBVU R9, R5 // ... + ADDVU R23, R24 // ... + SGTU R24, R6, R23 // SBCS R10, R6, R6 + SUBVU R24, R6 // ... + SGTU R10, R6, R24 // ... + SUBVU R10, R6 // ... + ADDVU R23, R24 // ... + SGTU R24, R7, R23 // SBCS R11, R7, R7 + SUBVU R24, R7 // ... + SGTU R11, R7, R24 // ... + SUBVU R11, R7 // ... + ADDVU R23, R24 // ... + SGTU R24, R8, R23 // SBCS R12, R8, R8 + SUBVU R24, R8 // ... + SGTU R12, R8, R24 // ... + SUBVU R12, R8 // ... + ADDVU R23, R24 // ... + MOVV R5, 0(R4) + MOVV R6, 8(R4) + MOVV R7, 16(R4) + MOVV R8, 24(R4) + ADDVU $32, R2 + ADDVU $32, R3 + ADDVU $32, R4 + SUBVU $1, R1 + BNE R1, loop4cont +loop4done: + MOVV R24, c+72(FP) + RET -TEXT ·shrVU(SB),NOSPLIT,$0 - JMP ·shrVU_g(SB) +// func lshVU(z, x []Word, s uint) (c Word) +TEXT ·lshVU(SB), NOSPLIT, $0 + MOVV z_len+8(FP), R1 + BEQ R1, ret0 + MOVV s+48(FP), R2 + MOVV x_base+24(FP), R3 + MOVV z_base+0(FP), R4 + // run loop backward + SLLV $3, R1, R5 + ADDVU R5, R3 + SLLV $3, R1, R5 + ADDVU R5, R4 + // shift first word into carry + MOVV -8(R3), R5 + MOVV $64, R6 + SUBVU R2, R6 + SRLV R6, R5, R7 + SLLV R2, R5 + MOVV R7, c+56(FP) + // shift remaining words + SUBVU $1, R1 + // compute unrolled loop lengths + AND $3, R1, R7 + SRLV $2, R1 +loop1: + BEQ R7, loop1done +loop1cont: + // unroll 1X + MOVV -16(R3), R8 + SRLV R6, R8, R9 + OR R5, R9 + SLLV R2, R8, R5 + MOVV R9, -8(R4) + ADDVU $-8, R3 + ADDVU $-8, R4 + SUBVU $1, R7 + BNE R7, loop1cont +loop1done: +loop4: + BEQ R1, loop4done +loop4cont: + // unroll 4X + MOVV -16(R3), R7 + MOVV -24(R3), R8 + MOVV -32(R3), R9 + MOVV -40(R3), R10 + SRLV R6, R7, R11 + OR R5, R11 + SLLV R2, R7, R5 + SRLV R6, R8, R7 + OR R5, R7 + SLLV R2, R8, R5 + SRLV R6, R9, R8 + OR R5, R8 + SLLV R2, R9, R5 + SRLV R6, R10, R9 + OR R5, R9 + SLLV R2, R10, R5 + MOVV R11, -8(R4) + MOVV R7, -16(R4) + MOVV R8, -24(R4) + MOVV R9, -32(R4) + ADDVU $-32, R3 + ADDVU $-32, R4 + SUBVU $1, R1 + BNE R1, loop4cont +loop4done: + // store final shifted bits + MOVV R5, -8(R4) + RET +ret0: + MOVV R0, c+56(FP) + RET -TEXT ·mulAddVWW(SB),NOSPLIT,$0 - JMP ·mulAddVWW_g(SB) +// func rshVU(z, x []Word, s uint) (c Word) +TEXT ·rshVU(SB), NOSPLIT, $0 + MOVV z_len+8(FP), R1 + BEQ R1, ret0 + MOVV s+48(FP), R2 + MOVV x_base+24(FP), R3 + MOVV z_base+0(FP), R4 + // shift first word into carry + MOVV 0(R3), R5 + MOVV $64, R6 + SUBVU R2, R6 + SLLV R6, R5, R7 + SRLV R2, R5 + MOVV R7, c+56(FP) + // shift remaining words + SUBVU $1, R1 + // compute unrolled loop lengths + AND $3, R1, R7 + SRLV $2, R1 +loop1: + BEQ R7, loop1done +loop1cont: + // unroll 1X + MOVV 8(R3), R8 + SLLV R6, R8, R9 + OR R5, R9 + SRLV R2, R8, R5 + MOVV R9, 0(R4) + ADDVU $8, R3 + ADDVU $8, R4 + SUBVU $1, R7 + BNE R7, loop1cont +loop1done: +loop4: + BEQ R1, loop4done +loop4cont: + // unroll 4X + MOVV 8(R3), R7 + MOVV 16(R3), R8 + MOVV 24(R3), R9 + MOVV 32(R3), R10 + SLLV R6, R7, R11 + OR R5, R11 + SRLV R2, R7, R5 + SLLV R6, R8, R7 + OR R5, R7 + SRLV R2, R8, R5 + SLLV R6, R9, R8 + OR R5, R8 + SRLV R2, R9, R5 + SLLV R6, R10, R9 + OR R5, R9 + SRLV R2, R10, R5 + MOVV R11, 0(R4) + MOVV R7, 8(R4) + MOVV R8, 16(R4) + MOVV R9, 24(R4) + ADDVU $32, R3 + ADDVU $32, R4 + SUBVU $1, R1 + BNE R1, loop4cont +loop4done: + // store final shifted bits + MOVV R5, 0(R4) + RET +ret0: + MOVV R0, c+56(FP) + RET -TEXT ·addMulVVW(SB),NOSPLIT,$0 - JMP ·addMulVVW_g(SB) +// func mulAddVWW(z, x []Word, m, a Word) (c Word) +TEXT ·mulAddVWW(SB), NOSPLIT, $0 + MOVV m+48(FP), R1 + MOVV a+56(FP), R2 + MOVV z_len+8(FP), R3 + MOVV x_base+24(FP), R4 + MOVV z_base+0(FP), R5 + // compute unrolled loop lengths + AND $3, R3, R6 + SRLV $2, R3 +loop1: + BEQ R6, loop1done +loop1cont: + // unroll 1X + MOVV 0(R4), R7 + // synthetic carry, one column at a time + MULVU R1, R7 + MOVV LO, R8 + MOVV HI, R9 + ADDVU R2, R8, R7 // ADDS R2, R8, R7 (cr=R24) + SGTU R2, R7, R24 // ... + ADDVU R24, R9, R2 // ADC $0, R9, R2 + MOVV R7, 0(R5) + ADDVU $8, R4 + ADDVU $8, R5 + SUBVU $1, R6 + BNE R6, loop1cont +loop1done: +loop4: + BEQ R3, loop4done +loop4cont: + // unroll 4X + MOVV 0(R4), R6 + MOVV 8(R4), R7 + MOVV 16(R4), R8 + MOVV 24(R4), R9 + // synthetic carry, one column at a time + MULVU R1, R6 + MOVV LO, R10 + MOVV HI, R11 + ADDVU R2, R10, R6 // ADDS R2, R10, R6 (cr=R24) + SGTU R2, R6, R24 // ... + ADDVU R24, R11, R2 // ADC $0, R11, R2 + MULVU R1, R7 + MOVV LO, R10 + MOVV HI, R11 + ADDVU R2, R10, R7 // ADDS R2, R10, R7 (cr=R24) + SGTU R2, R7, R24 // ... + ADDVU R24, R11, R2 // ADC $0, R11, R2 + MULVU R1, R8 + MOVV LO, R10 + MOVV HI, R11 + ADDVU R2, R10, R8 // ADDS R2, R10, R8 (cr=R24) + SGTU R2, R8, R24 // ... + ADDVU R24, R11, R2 // ADC $0, R11, R2 + MULVU R1, R9 + MOVV LO, R10 + MOVV HI, R11 + ADDVU R2, R10, R9 // ADDS R2, R10, R9 (cr=R24) + SGTU R2, R9, R24 // ... + ADDVU R24, R11, R2 // ADC $0, R11, R2 + MOVV R6, 0(R5) + MOVV R7, 8(R5) + MOVV R8, 16(R5) + MOVV R9, 24(R5) + ADDVU $32, R4 + ADDVU $32, R5 + SUBVU $1, R3 + BNE R3, loop4cont +loop4done: + MOVV R2, c+64(FP) + RET +// func addMulVVWW(z, x, y []Word, m, a Word) (c Word) +TEXT ·addMulVVWW(SB), NOSPLIT, $0 + MOVV m+72(FP), R1 + MOVV a+80(FP), R2 + MOVV z_len+8(FP), R3 + MOVV x_base+24(FP), R4 + MOVV y_base+48(FP), R5 + MOVV z_base+0(FP), R6 + // compute unrolled loop lengths + AND $3, R3, R7 + SRLV $2, R3 +loop1: + BEQ R7, loop1done +loop1cont: + // unroll 1X + MOVV 0(R4), R8 + MOVV 0(R5), R9 + // synthetic carry, one column at a time + MULVU R1, R9 + MOVV LO, R10 + MOVV HI, R11 + ADDVU R8, R10 // ADDS R8, R10, R10 (cr=R24) + SGTU R8, R10, R24 // ... + ADDVU R24, R11 // ADC $0, R11, R11 + ADDVU R2, R10, R9 // ADDS R2, R10, R9 (cr=R24) + SGTU R2, R9, R24 // ... + ADDVU R24, R11, R2 // ADC $0, R11, R2 + MOVV R9, 0(R6) + ADDVU $8, R4 + ADDVU $8, R5 + ADDVU $8, R6 + SUBVU $1, R7 + BNE R7, loop1cont +loop1done: +loop4: + BEQ R3, loop4done +loop4cont: + // unroll 4X + MOVV 0(R4), R7 + MOVV 8(R4), R8 + MOVV 16(R4), R9 + MOVV 24(R4), R10 + MOVV 0(R5), R11 + MOVV 8(R5), R12 + MOVV 16(R5), R13 + MOVV 24(R5), R14 + // synthetic carry, one column at a time + MULVU R1, R11 + MOVV LO, R15 + MOVV HI, R16 + ADDVU R7, R15 // ADDS R7, R15, R15 (cr=R24) + SGTU R7, R15, R24 // ... + ADDVU R24, R16 // ADC $0, R16, R16 + ADDVU R2, R15, R11 // ADDS R2, R15, R11 (cr=R24) + SGTU R2, R11, R24 // ... + ADDVU R24, R16, R2 // ADC $0, R16, R2 + MULVU R1, R12 + MOVV LO, R15 + MOVV HI, R16 + ADDVU R8, R15 // ADDS R8, R15, R15 (cr=R24) + SGTU R8, R15, R24 // ... + ADDVU R24, R16 // ADC $0, R16, R16 + ADDVU R2, R15, R12 // ADDS R2, R15, R12 (cr=R24) + SGTU R2, R12, R24 // ... + ADDVU R24, R16, R2 // ADC $0, R16, R2 + MULVU R1, R13 + MOVV LO, R15 + MOVV HI, R16 + ADDVU R9, R15 // ADDS R9, R15, R15 (cr=R24) + SGTU R9, R15, R24 // ... + ADDVU R24, R16 // ADC $0, R16, R16 + ADDVU R2, R15, R13 // ADDS R2, R15, R13 (cr=R24) + SGTU R2, R13, R24 // ... + ADDVU R24, R16, R2 // ADC $0, R16, R2 + MULVU R1, R14 + MOVV LO, R15 + MOVV HI, R16 + ADDVU R10, R15 // ADDS R10, R15, R15 (cr=R24) + SGTU R10, R15, R24 // ... + ADDVU R24, R16 // ADC $0, R16, R16 + ADDVU R2, R15, R14 // ADDS R2, R15, R14 (cr=R24) + SGTU R2, R14, R24 // ... + ADDVU R24, R16, R2 // ADC $0, R16, R2 + MOVV R11, 0(R6) + MOVV R12, 8(R6) + MOVV R13, 16(R6) + MOVV R14, 24(R6) + ADDVU $32, R4 + ADDVU $32, R5 + ADDVU $32, R6 + SUBVU $1, R3 + BNE R3, loop4cont +loop4done: + MOVV R2, c+88(FP) + RET diff --git a/src/math/big/arith_mipsx.s b/src/math/big/arith_mipsx.s index cdb4bbcab63336..90737ba2a54322 100644 --- a/src/math/big/arith_mipsx.s +++ b/src/math/big/arith_mipsx.s @@ -1,35 +1,467 @@ -// Copyright 2016 The Go Authors. All rights reserved. +// Copyright 2025 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Code generated by 'go generate' (with ./internal/asmgen). DO NOT EDIT. + //go:build !math_big_pure_go && (mips || mipsle) #include "textflag.h" -// This file provides fast assembly versions for the elementary -// arithmetic operations on vectors implemented in arith.go. - -TEXT ·addVV(SB),NOSPLIT,$0 - JMP ·addVV_g(SB) - -TEXT ·subVV(SB),NOSPLIT,$0 - JMP ·subVV_g(SB) - -TEXT ·addVW(SB),NOSPLIT,$0 - JMP ·addVW_g(SB) - -TEXT ·subVW(SB),NOSPLIT,$0 - JMP ·subVW_g(SB) +// func addVV(z, x, y []Word) (c Word) +TEXT ·addVV(SB), NOSPLIT, $0 + MOVW z_len+4(FP), R1 + MOVW x_base+12(FP), R2 + MOVW y_base+24(FP), R3 + MOVW z_base+0(FP), R4 + // compute unrolled loop lengths + AND $3, R1, R5 + SRL $2, R1 + XOR R24, R24 // clear carry +loop1: + BEQ R5, loop1done +loop1cont: + // unroll 1X + MOVW 0(R2), R6 + MOVW 0(R3), R7 + ADDU R7, R6 // ADCS R7, R6, R6 (cr=R24) + SGTU R7, R6, R23 // ... + ADDU R24, R6 // ... + SGTU R24, R6, R24 // ... + ADDU R23, R24 // ... + MOVW R6, 0(R4) + ADDU $4, R2 + ADDU $4, R3 + ADDU $4, R4 + SUBU $1, R5 + BNE R5, loop1cont +loop1done: +loop4: + BEQ R1, loop4done +loop4cont: + // unroll 4X + MOVW 0(R2), R5 + MOVW 4(R2), R6 + MOVW 8(R2), R7 + MOVW 12(R2), R8 + MOVW 0(R3), R9 + MOVW 4(R3), R10 + MOVW 8(R3), R11 + MOVW 12(R3), R12 + ADDU R9, R5 // ADCS R9, R5, R5 (cr=R24) + SGTU R9, R5, R23 // ... + ADDU R24, R5 // ... + SGTU R24, R5, R24 // ... + ADDU R23, R24 // ... + ADDU R10, R6 // ADCS R10, R6, R6 (cr=R24) + SGTU R10, R6, R23 // ... + ADDU R24, R6 // ... + SGTU R24, R6, R24 // ... + ADDU R23, R24 // ... + ADDU R11, R7 // ADCS R11, R7, R7 (cr=R24) + SGTU R11, R7, R23 // ... + ADDU R24, R7 // ... + SGTU R24, R7, R24 // ... + ADDU R23, R24 // ... + ADDU R12, R8 // ADCS R12, R8, R8 (cr=R24) + SGTU R12, R8, R23 // ... + ADDU R24, R8 // ... + SGTU R24, R8, R24 // ... + ADDU R23, R24 // ... + MOVW R5, 0(R4) + MOVW R6, 4(R4) + MOVW R7, 8(R4) + MOVW R8, 12(R4) + ADDU $16, R2 + ADDU $16, R3 + ADDU $16, R4 + SUBU $1, R1 + BNE R1, loop4cont +loop4done: + MOVW R24, c+36(FP) + RET -TEXT ·shlVU(SB),NOSPLIT,$0 - JMP ·shlVU_g(SB) +// func subVV(z, x, y []Word) (c Word) +TEXT ·subVV(SB), NOSPLIT, $0 + MOVW z_len+4(FP), R1 + MOVW x_base+12(FP), R2 + MOVW y_base+24(FP), R3 + MOVW z_base+0(FP), R4 + // compute unrolled loop lengths + AND $3, R1, R5 + SRL $2, R1 + XOR R24, R24 // clear carry +loop1: + BEQ R5, loop1done +loop1cont: + // unroll 1X + MOVW 0(R2), R6 + MOVW 0(R3), R7 + SGTU R24, R6, R23 // SBCS R7, R6, R6 + SUBU R24, R6 // ... + SGTU R7, R6, R24 // ... + SUBU R7, R6 // ... + ADDU R23, R24 // ... + MOVW R6, 0(R4) + ADDU $4, R2 + ADDU $4, R3 + ADDU $4, R4 + SUBU $1, R5 + BNE R5, loop1cont +loop1done: +loop4: + BEQ R1, loop4done +loop4cont: + // unroll 4X + MOVW 0(R2), R5 + MOVW 4(R2), R6 + MOVW 8(R2), R7 + MOVW 12(R2), R8 + MOVW 0(R3), R9 + MOVW 4(R3), R10 + MOVW 8(R3), R11 + MOVW 12(R3), R12 + SGTU R24, R5, R23 // SBCS R9, R5, R5 + SUBU R24, R5 // ... + SGTU R9, R5, R24 // ... + SUBU R9, R5 // ... + ADDU R23, R24 // ... + SGTU R24, R6, R23 // SBCS R10, R6, R6 + SUBU R24, R6 // ... + SGTU R10, R6, R24 // ... + SUBU R10, R6 // ... + ADDU R23, R24 // ... + SGTU R24, R7, R23 // SBCS R11, R7, R7 + SUBU R24, R7 // ... + SGTU R11, R7, R24 // ... + SUBU R11, R7 // ... + ADDU R23, R24 // ... + SGTU R24, R8, R23 // SBCS R12, R8, R8 + SUBU R24, R8 // ... + SGTU R12, R8, R24 // ... + SUBU R12, R8 // ... + ADDU R23, R24 // ... + MOVW R5, 0(R4) + MOVW R6, 4(R4) + MOVW R7, 8(R4) + MOVW R8, 12(R4) + ADDU $16, R2 + ADDU $16, R3 + ADDU $16, R4 + SUBU $1, R1 + BNE R1, loop4cont +loop4done: + MOVW R24, c+36(FP) + RET -TEXT ·shrVU(SB),NOSPLIT,$0 - JMP ·shrVU_g(SB) +// func lshVU(z, x []Word, s uint) (c Word) +TEXT ·lshVU(SB), NOSPLIT, $0 + MOVW z_len+4(FP), R1 + BEQ R1, ret0 + MOVW s+24(FP), R2 + MOVW x_base+12(FP), R3 + MOVW z_base+0(FP), R4 + // run loop backward + SLL $2, R1, R5 + ADDU R5, R3 + SLL $2, R1, R5 + ADDU R5, R4 + // shift first word into carry + MOVW -4(R3), R5 + MOVW $32, R6 + SUBU R2, R6 + SRL R6, R5, R7 + SLL R2, R5 + MOVW R7, c+28(FP) + // shift remaining words + SUBU $1, R1 + // compute unrolled loop lengths + AND $3, R1, R7 + SRL $2, R1 +loop1: + BEQ R7, loop1done +loop1cont: + // unroll 1X + MOVW -8(R3), R8 + SRL R6, R8, R9 + OR R5, R9 + SLL R2, R8, R5 + MOVW R9, -4(R4) + ADDU $-4, R3 + ADDU $-4, R4 + SUBU $1, R7 + BNE R7, loop1cont +loop1done: +loop4: + BEQ R1, loop4done +loop4cont: + // unroll 4X + MOVW -8(R3), R7 + MOVW -12(R3), R8 + MOVW -16(R3), R9 + MOVW -20(R3), R10 + SRL R6, R7, R11 + OR R5, R11 + SLL R2, R7, R5 + SRL R6, R8, R7 + OR R5, R7 + SLL R2, R8, R5 + SRL R6, R9, R8 + OR R5, R8 + SLL R2, R9, R5 + SRL R6, R10, R9 + OR R5, R9 + SLL R2, R10, R5 + MOVW R11, -4(R4) + MOVW R7, -8(R4) + MOVW R8, -12(R4) + MOVW R9, -16(R4) + ADDU $-16, R3 + ADDU $-16, R4 + SUBU $1, R1 + BNE R1, loop4cont +loop4done: + // store final shifted bits + MOVW R5, -4(R4) + RET +ret0: + MOVW R0, c+28(FP) + RET -TEXT ·mulAddVWW(SB),NOSPLIT,$0 - JMP ·mulAddVWW_g(SB) +// func rshVU(z, x []Word, s uint) (c Word) +TEXT ·rshVU(SB), NOSPLIT, $0 + MOVW z_len+4(FP), R1 + BEQ R1, ret0 + MOVW s+24(FP), R2 + MOVW x_base+12(FP), R3 + MOVW z_base+0(FP), R4 + // shift first word into carry + MOVW 0(R3), R5 + MOVW $32, R6 + SUBU R2, R6 + SLL R6, R5, R7 + SRL R2, R5 + MOVW R7, c+28(FP) + // shift remaining words + SUBU $1, R1 + // compute unrolled loop lengths + AND $3, R1, R7 + SRL $2, R1 +loop1: + BEQ R7, loop1done +loop1cont: + // unroll 1X + MOVW 4(R3), R8 + SLL R6, R8, R9 + OR R5, R9 + SRL R2, R8, R5 + MOVW R9, 0(R4) + ADDU $4, R3 + ADDU $4, R4 + SUBU $1, R7 + BNE R7, loop1cont +loop1done: +loop4: + BEQ R1, loop4done +loop4cont: + // unroll 4X + MOVW 4(R3), R7 + MOVW 8(R3), R8 + MOVW 12(R3), R9 + MOVW 16(R3), R10 + SLL R6, R7, R11 + OR R5, R11 + SRL R2, R7, R5 + SLL R6, R8, R7 + OR R5, R7 + SRL R2, R8, R5 + SLL R6, R9, R8 + OR R5, R8 + SRL R2, R9, R5 + SLL R6, R10, R9 + OR R5, R9 + SRL R2, R10, R5 + MOVW R11, 0(R4) + MOVW R7, 4(R4) + MOVW R8, 8(R4) + MOVW R9, 12(R4) + ADDU $16, R3 + ADDU $16, R4 + SUBU $1, R1 + BNE R1, loop4cont +loop4done: + // store final shifted bits + MOVW R5, 0(R4) + RET +ret0: + MOVW R0, c+28(FP) + RET -TEXT ·addMulVVW(SB),NOSPLIT,$0 - JMP ·addMulVVW_g(SB) +// func mulAddVWW(z, x []Word, m, a Word) (c Word) +TEXT ·mulAddVWW(SB), NOSPLIT, $0 + MOVW m+24(FP), R1 + MOVW a+28(FP), R2 + MOVW z_len+4(FP), R3 + MOVW x_base+12(FP), R4 + MOVW z_base+0(FP), R5 + // compute unrolled loop lengths + AND $3, R3, R6 + SRL $2, R3 +loop1: + BEQ R6, loop1done +loop1cont: + // unroll 1X + MOVW 0(R4), R7 + // synthetic carry, one column at a time + MULU R1, R7 + MOVW LO, R8 + MOVW HI, R9 + ADDU R2, R8, R7 // ADDS R2, R8, R7 (cr=R24) + SGTU R2, R7, R24 // ... + ADDU R24, R9, R2 // ADC $0, R9, R2 + MOVW R7, 0(R5) + ADDU $4, R4 + ADDU $4, R5 + SUBU $1, R6 + BNE R6, loop1cont +loop1done: +loop4: + BEQ R3, loop4done +loop4cont: + // unroll 4X + MOVW 0(R4), R6 + MOVW 4(R4), R7 + MOVW 8(R4), R8 + MOVW 12(R4), R9 + // synthetic carry, one column at a time + MULU R1, R6 + MOVW LO, R10 + MOVW HI, R11 + ADDU R2, R10, R6 // ADDS R2, R10, R6 (cr=R24) + SGTU R2, R6, R24 // ... + ADDU R24, R11, R2 // ADC $0, R11, R2 + MULU R1, R7 + MOVW LO, R10 + MOVW HI, R11 + ADDU R2, R10, R7 // ADDS R2, R10, R7 (cr=R24) + SGTU R2, R7, R24 // ... + ADDU R24, R11, R2 // ADC $0, R11, R2 + MULU R1, R8 + MOVW LO, R10 + MOVW HI, R11 + ADDU R2, R10, R8 // ADDS R2, R10, R8 (cr=R24) + SGTU R2, R8, R24 // ... + ADDU R24, R11, R2 // ADC $0, R11, R2 + MULU R1, R9 + MOVW LO, R10 + MOVW HI, R11 + ADDU R2, R10, R9 // ADDS R2, R10, R9 (cr=R24) + SGTU R2, R9, R24 // ... + ADDU R24, R11, R2 // ADC $0, R11, R2 + MOVW R6, 0(R5) + MOVW R7, 4(R5) + MOVW R8, 8(R5) + MOVW R9, 12(R5) + ADDU $16, R4 + ADDU $16, R5 + SUBU $1, R3 + BNE R3, loop4cont +loop4done: + MOVW R2, c+32(FP) + RET +// func addMulVVWW(z, x, y []Word, m, a Word) (c Word) +TEXT ·addMulVVWW(SB), NOSPLIT, $0 + MOVW m+36(FP), R1 + MOVW a+40(FP), R2 + MOVW z_len+4(FP), R3 + MOVW x_base+12(FP), R4 + MOVW y_base+24(FP), R5 + MOVW z_base+0(FP), R6 + // compute unrolled loop lengths + AND $3, R3, R7 + SRL $2, R3 +loop1: + BEQ R7, loop1done +loop1cont: + // unroll 1X + MOVW 0(R4), R8 + MOVW 0(R5), R9 + // synthetic carry, one column at a time + MULU R1, R9 + MOVW LO, R10 + MOVW HI, R11 + ADDU R8, R10 // ADDS R8, R10, R10 (cr=R24) + SGTU R8, R10, R24 // ... + ADDU R24, R11 // ADC $0, R11, R11 + ADDU R2, R10, R9 // ADDS R2, R10, R9 (cr=R24) + SGTU R2, R9, R24 // ... + ADDU R24, R11, R2 // ADC $0, R11, R2 + MOVW R9, 0(R6) + ADDU $4, R4 + ADDU $4, R5 + ADDU $4, R6 + SUBU $1, R7 + BNE R7, loop1cont +loop1done: +loop4: + BEQ R3, loop4done +loop4cont: + // unroll 4X + MOVW 0(R4), R7 + MOVW 4(R4), R8 + MOVW 8(R4), R9 + MOVW 12(R4), R10 + MOVW 0(R5), R11 + MOVW 4(R5), R12 + MOVW 8(R5), R13 + MOVW 12(R5), R14 + // synthetic carry, one column at a time + MULU R1, R11 + MOVW LO, R15 + MOVW HI, R16 + ADDU R7, R15 // ADDS R7, R15, R15 (cr=R24) + SGTU R7, R15, R24 // ... + ADDU R24, R16 // ADC $0, R16, R16 + ADDU R2, R15, R11 // ADDS R2, R15, R11 (cr=R24) + SGTU R2, R11, R24 // ... + ADDU R24, R16, R2 // ADC $0, R16, R2 + MULU R1, R12 + MOVW LO, R15 + MOVW HI, R16 + ADDU R8, R15 // ADDS R8, R15, R15 (cr=R24) + SGTU R8, R15, R24 // ... + ADDU R24, R16 // ADC $0, R16, R16 + ADDU R2, R15, R12 // ADDS R2, R15, R12 (cr=R24) + SGTU R2, R12, R24 // ... + ADDU R24, R16, R2 // ADC $0, R16, R2 + MULU R1, R13 + MOVW LO, R15 + MOVW HI, R16 + ADDU R9, R15 // ADDS R9, R15, R15 (cr=R24) + SGTU R9, R15, R24 // ... + ADDU R24, R16 // ADC $0, R16, R16 + ADDU R2, R15, R13 // ADDS R2, R15, R13 (cr=R24) + SGTU R2, R13, R24 // ... + ADDU R24, R16, R2 // ADC $0, R16, R2 + MULU R1, R14 + MOVW LO, R15 + MOVW HI, R16 + ADDU R10, R15 // ADDS R10, R15, R15 (cr=R24) + SGTU R10, R15, R24 // ... + ADDU R24, R16 // ADC $0, R16, R16 + ADDU R2, R15, R14 // ADDS R2, R15, R14 (cr=R24) + SGTU R2, R14, R24 // ... + ADDU R24, R16, R2 // ADC $0, R16, R2 + MOVW R11, 0(R6) + MOVW R12, 4(R6) + MOVW R13, 8(R6) + MOVW R14, 12(R6) + ADDU $16, R4 + ADDU $16, R5 + ADDU $16, R6 + SUBU $1, R3 + BNE R3, loop4cont +loop4done: + MOVW R2, c+44(FP) + RET diff --git a/src/math/big/arith_ppc64x.s b/src/math/big/arith_ppc64x.s index 82aa7fb51e443e..1bcd30d7e5c90f 100644 --- a/src/math/big/arith_ppc64x.s +++ b/src/math/big/arith_ppc64x.s @@ -1,668 +1,386 @@ -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2025 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Code generated by 'go generate' (with ./internal/asmgen). DO NOT EDIT. + //go:build !math_big_pure_go && (ppc64 || ppc64le) #include "textflag.h" -// This file provides fast assembly versions for the elementary -// arithmetic operations on vectors implemented in arith.go. - -// func addVV(z, y, y []Word) (c Word) -// z[i] = x[i] + y[i] for all i, carrying +// func addVV(z, x, y []Word) (c Word) TEXT ·addVV(SB), NOSPLIT, $0 - MOVD z_len+8(FP), R7 // R7 = z_len - MOVD x+24(FP), R8 // R8 = x[] - MOVD y+48(FP), R9 // R9 = y[] - MOVD z+0(FP), R10 // R10 = z[] - - // If z_len = 0, we are done - CMP R7, $0 - MOVD R0, R4 - BEQ done - - // Process the first iteration out of the loop so we can - // use MOVDU and avoid 3 index registers updates. - MOVD 0(R8), R11 // R11 = x[i] - MOVD 0(R9), R12 // R12 = y[i] - ADD $-1, R7 // R7 = z_len - 1 - ADDC R12, R11, R15 // R15 = x[i] + y[i], set CA - CMP R7, $0 - MOVD R15, 0(R10) // z[i] - BEQ final // If z_len was 1, we are done - - SRD $2, R7, R5 // R5 = z_len/4 - CMP R5, $0 - MOVD R5, CTR // Set up loop counter - BEQ tail // If R5 = 0, we can't use the loop - - // Process 4 elements per iteration. Unrolling this loop - // means a performance trade-off: we will lose performance - // for small values of z_len (0.90x in the worst case), but - // gain significant performance as z_len increases (up to - // 1.45x). - - PCALIGN $16 -loop: - MOVD 8(R8), R11 // R11 = x[i] - MOVD 16(R8), R12 // R12 = x[i+1] - MOVD 24(R8), R14 // R14 = x[i+2] - MOVDU 32(R8), R15 // R15 = x[i+3] - MOVD 8(R9), R16 // R16 = y[i] - MOVD 16(R9), R17 // R17 = y[i+1] - MOVD 24(R9), R18 // R18 = y[i+2] - MOVDU 32(R9), R19 // R19 = y[i+3] - ADDE R11, R16, R20 // R20 = x[i] + y[i] + CA - ADDE R12, R17, R21 // R21 = x[i+1] + y[i+1] + CA - ADDE R14, R18, R22 // R22 = x[i+2] + y[i+2] + CA - ADDE R15, R19, R23 // R23 = x[i+3] + y[i+3] + CA - MOVD R20, 8(R10) // z[i] - MOVD R21, 16(R10) // z[i+1] - MOVD R22, 24(R10) // z[i+2] - MOVDU R23, 32(R10) // z[i+3] - ADD $-4, R7 // R7 = z_len - 4 - BDNZ loop - - // We may have more elements to read - CMP R7, $0 - BEQ final - - // Process the remaining elements, one at a time -tail: - MOVDU 8(R8), R11 // R11 = x[i] - MOVDU 8(R9), R16 // R16 = y[i] - ADD $-1, R7 // R7 = z_len - 1 - ADDE R11, R16, R20 // R20 = x[i] + y[i] + CA - CMP R7, $0 - MOVDU R20, 8(R10) // z[i] - BEQ final // If R7 = 0, we are done - - MOVDU 8(R8), R11 - MOVDU 8(R9), R16 - ADD $-1, R7 - ADDE R11, R16, R20 - CMP R7, $0 - MOVDU R20, 8(R10) - BEQ final - - MOVD 8(R8), R11 - MOVD 8(R9), R16 - ADDE R11, R16, R20 - MOVD R20, 8(R10) - -final: - ADDZE R4 // Capture CA - -done: - MOVD R4, c+72(FP) + MOVD z_len+8(FP), R3 + MOVD x_base+24(FP), R4 + MOVD y_base+48(FP), R5 + MOVD z_base+0(FP), R6 + // compute unrolled loop lengths + ANDCC $3, R3, R7 + SRD $2, R3 + ADDC R0, R3 // clear carry +loop1: + CMP R7, $0; BEQ loop1done; MOVD R7, CTR +loop1cont: + // unroll 1X + MOVD 0(R4), R8 + MOVD 0(R5), R9 + ADDE R9, R8 + MOVD R8, 0(R6) + ADD $8, R4 + ADD $8, R5 + ADD $8, R6 + BDNZ loop1cont +loop1done: +loop4: + CMP R3, $0; BEQ loop4done; MOVD R3, CTR +loop4cont: + // unroll 4X + MOVD 0(R4), R7 + MOVD 8(R4), R8 + MOVD 16(R4), R9 + MOVD 24(R4), R10 + MOVD 0(R5), R11 + MOVD 8(R5), R12 + MOVD 16(R5), R14 + MOVD 24(R5), R15 + ADDE R11, R7 + ADDE R12, R8 + ADDE R14, R9 + ADDE R15, R10 + MOVD R7, 0(R6) + MOVD R8, 8(R6) + MOVD R9, 16(R6) + MOVD R10, 24(R6) + ADD $32, R4 + ADD $32, R5 + ADD $32, R6 + BDNZ loop4cont +loop4done: + ADDE R0, R0, R4 // save & convert add carry + MOVD R4, c+72(FP) RET // func subVV(z, x, y []Word) (c Word) -// z[i] = x[i] - y[i] for all i, carrying TEXT ·subVV(SB), NOSPLIT, $0 - MOVD z_len+8(FP), R7 // R7 = z_len - MOVD x+24(FP), R8 // R8 = x[] - MOVD y+48(FP), R9 // R9 = y[] - MOVD z+0(FP), R10 // R10 = z[] - - // If z_len = 0, we are done - CMP R7, $0 - MOVD R0, R4 - BEQ done - - // Process the first iteration out of the loop so we can - // use MOVDU and avoid 3 index registers updates. - MOVD 0(R8), R11 // R11 = x[i] - MOVD 0(R9), R12 // R12 = y[i] - ADD $-1, R7 // R7 = z_len - 1 - SUBC R12, R11, R15 // R15 = x[i] - y[i], set CA - CMP R7, $0 - MOVD R15, 0(R10) // z[i] - BEQ final // If z_len was 1, we are done - - SRD $2, R7, R5 // R5 = z_len/4 - CMP R5, $0 - MOVD R5, CTR // Set up loop counter - BEQ tail // If R5 = 0, we can't use the loop - - // Process 4 elements per iteration. Unrolling this loop - // means a performance trade-off: we will lose performance - // for small values of z_len (0.92x in the worst case), but - // gain significant performance as z_len increases (up to - // 1.45x). - - PCALIGN $16 -loop: - MOVD 8(R8), R11 // R11 = x[i] - MOVD 16(R8), R12 // R12 = x[i+1] - MOVD 24(R8), R14 // R14 = x[i+2] - MOVDU 32(R8), R15 // R15 = x[i+3] - MOVD 8(R9), R16 // R16 = y[i] - MOVD 16(R9), R17 // R17 = y[i+1] - MOVD 24(R9), R18 // R18 = y[i+2] - MOVDU 32(R9), R19 // R19 = y[i+3] - SUBE R16, R11, R20 // R20 = x[i] - y[i] + CA - SUBE R17, R12, R21 // R21 = x[i+1] - y[i+1] + CA - SUBE R18, R14, R22 // R22 = x[i+2] - y[i+2] + CA - SUBE R19, R15, R23 // R23 = x[i+3] - y[i+3] + CA - MOVD R20, 8(R10) // z[i] - MOVD R21, 16(R10) // z[i+1] - MOVD R22, 24(R10) // z[i+2] - MOVDU R23, 32(R10) // z[i+3] - ADD $-4, R7 // R7 = z_len - 4 - BDNZ loop - - // We may have more elements to read - CMP R7, $0 - BEQ final - - // Process the remaining elements, one at a time -tail: - MOVDU 8(R8), R11 // R11 = x[i] - MOVDU 8(R9), R16 // R16 = y[i] - ADD $-1, R7 // R7 = z_len - 1 - SUBE R16, R11, R20 // R20 = x[i] - y[i] + CA - CMP R7, $0 - MOVDU R20, 8(R10) // z[i] - BEQ final // If R7 = 0, we are done - - MOVDU 8(R8), R11 - MOVDU 8(R9), R16 - ADD $-1, R7 - SUBE R16, R11, R20 - CMP R7, $0 - MOVDU R20, 8(R10) - BEQ final - - MOVD 8(R8), R11 - MOVD 8(R9), R16 - SUBE R16, R11, R20 - MOVD R20, 8(R10) - -final: - ADDZE R4 - XOR $1, R4 - -done: - MOVD R4, c+72(FP) - RET - -// func addVW(z, x []Word, y Word) (c Word) -TEXT ·addVW(SB), NOSPLIT, $0 - MOVD z+0(FP), R10 // R10 = z[] - MOVD x+24(FP), R8 // R8 = x[] - MOVD y+48(FP), R4 // R4 = y = c - MOVD z_len+8(FP), R11 // R11 = z_len - - CMP R11, $0 // If z_len is zero, return - BEQ done - - // We will process the first iteration out of the loop so we capture - // the value of c. In the subsequent iterations, we will rely on the - // value of CA set here. - MOVD 0(R8), R20 // R20 = x[i] - ADD $-1, R11 // R11 = z_len - 1 - ADDC R20, R4, R6 // R6 = x[i] + c - CMP R11, $0 // If z_len was 1, we are done - MOVD R6, 0(R10) // z[i] - BEQ final - - // We will read 4 elements per iteration - SRDCC $2, R11, R9 // R9 = z_len/4 - DCBT (R8) - MOVD R9, CTR // Set up the loop counter - BEQ tail // If R9 = 0, we can't use the loop - PCALIGN $16 - -loop: - MOVD 8(R8), R20 // R20 = x[i] - MOVD 16(R8), R21 // R21 = x[i+1] - MOVD 24(R8), R22 // R22 = x[i+2] - MOVDU 32(R8), R23 // R23 = x[i+3] - ADDZE R20, R24 // R24 = x[i] + CA - ADDZE R21, R25 // R25 = x[i+1] + CA - ADDZE R22, R26 // R26 = x[i+2] + CA - ADDZE R23, R27 // R27 = x[i+3] + CA - MOVD R24, 8(R10) // z[i] - MOVD R25, 16(R10) // z[i+1] - MOVD R26, 24(R10) // z[i+2] - MOVDU R27, 32(R10) // z[i+3] - ADD $-4, R11 // R11 = z_len - 4 - BDNZ loop - - // We may have some elements to read - CMP R11, $0 - BEQ final - -tail: - MOVDU 8(R8), R20 - ADDZE R20, R24 - ADD $-1, R11 - MOVDU R24, 8(R10) - CMP R11, $0 - BEQ final - - MOVDU 8(R8), R20 - ADDZE R20, R24 - ADD $-1, R11 - MOVDU R24, 8(R10) - CMP R11, $0 - BEQ final - - MOVD 8(R8), R20 - ADDZE R20, R24 - MOVD R24, 8(R10) - -final: - ADDZE R0, R4 // c = CA -done: - MOVD R4, c+56(FP) + MOVD z_len+8(FP), R3 + MOVD x_base+24(FP), R4 + MOVD y_base+48(FP), R5 + MOVD z_base+0(FP), R6 + // compute unrolled loop lengths + ANDCC $3, R3, R7 + SRD $2, R3 + SUBC R0, R3 // clear carry +loop1: + CMP R7, $0; BEQ loop1done; MOVD R7, CTR +loop1cont: + // unroll 1X + MOVD 0(R4), R8 + MOVD 0(R5), R9 + SUBE R9, R8 + MOVD R8, 0(R6) + ADD $8, R4 + ADD $8, R5 + ADD $8, R6 + BDNZ loop1cont +loop1done: +loop4: + CMP R3, $0; BEQ loop4done; MOVD R3, CTR +loop4cont: + // unroll 4X + MOVD 0(R4), R7 + MOVD 8(R4), R8 + MOVD 16(R4), R9 + MOVD 24(R4), R10 + MOVD 0(R5), R11 + MOVD 8(R5), R12 + MOVD 16(R5), R14 + MOVD 24(R5), R15 + SUBE R11, R7 + SUBE R12, R8 + SUBE R14, R9 + SUBE R15, R10 + MOVD R7, 0(R6) + MOVD R8, 8(R6) + MOVD R9, 16(R6) + MOVD R10, 24(R6) + ADD $32, R4 + ADD $32, R5 + ADD $32, R6 + BDNZ loop4cont +loop4done: + SUBE R4, R4 // save carry + SUB R4, R0, R4 // convert sub carry + MOVD R4, c+72(FP) RET -// func subVW(z, x []Word, y Word) (c Word) -TEXT ·subVW(SB), NOSPLIT, $0 - MOVD z+0(FP), R10 // R10 = z[] - MOVD x+24(FP), R8 // R8 = x[] - MOVD y+48(FP), R4 // R4 = y = c - MOVD z_len+8(FP), R11 // R11 = z_len - - CMP R11, $0 // If z_len is zero, return - BEQ done - - // We will process the first iteration out of the loop so we capture - // the value of c. In the subsequent iterations, we will rely on the - // value of CA set here. - MOVD 0(R8), R20 // R20 = x[i] - ADD $-1, R11 // R11 = z_len - 1 - SUBC R4, R20, R6 // R6 = x[i] - c - CMP R11, $0 // If z_len was 1, we are done - MOVD R6, 0(R10) // z[i] - BEQ final - - // We will read 4 elements per iteration - SRDCC $2, R11, R9 // R9 = z_len/4 - DCBT (R8) - MOVD R9, CTR // Set up the loop counter - BEQ tail // If R9 = 0, we can't use the loop - - // The loop here is almost the same as the one used in s390x, but - // we don't need to capture CA every iteration because we've already - // done that above. - - PCALIGN $16 -loop: - MOVD 8(R8), R20 - MOVD 16(R8), R21 - MOVD 24(R8), R22 - MOVDU 32(R8), R23 - SUBE R0, R20 - SUBE R0, R21 - SUBE R0, R22 - SUBE R0, R23 - MOVD R20, 8(R10) - MOVD R21, 16(R10) - MOVD R22, 24(R10) - MOVDU R23, 32(R10) - ADD $-4, R11 - BDNZ loop - - // We may have some elements to read - CMP R11, $0 - BEQ final - -tail: - MOVDU 8(R8), R20 - SUBE R0, R20 - ADD $-1, R11 - MOVDU R20, 8(R10) - CMP R11, $0 - BEQ final - - MOVDU 8(R8), R20 - SUBE R0, R20 - ADD $-1, R11 - MOVDU R20, 8(R10) - CMP R11, $0 - BEQ final - - MOVD 8(R8), R20 - SUBE R0, R20 - MOVD R20, 8(R10) - -final: - // Capture CA - SUBE R4, R4 - NEG R4, R4 - -done: - MOVD R4, c+56(FP) +// func lshVU(z, x []Word, s uint) (c Word) +TEXT ·lshVU(SB), NOSPLIT, $0 + MOVD z_len+8(FP), R3 + CMP R3, $0; BEQ ret0 + MOVD s+48(FP), R4 + MOVD x_base+24(FP), R5 + MOVD z_base+0(FP), R6 + // run loop backward + SLD $3, R3, R7 + ADD R7, R5 + SLD $3, R3, R7 + ADD R7, R6 + // shift first word into carry + MOVD -8(R5), R7 + MOVD $64, R8 + SUB R4, R8 + SRD R8, R7, R9 + SLD R4, R7 + MOVD R9, c+56(FP) + // shift remaining words + SUB $1, R3 + // compute unrolled loop lengths + ANDCC $3, R3, R9 + SRD $2, R3 +loop1: + CMP R9, $0; BEQ loop1done; MOVD R9, CTR +loop1cont: + // unroll 1X + MOVD -16(R5), R10 + SRD R8, R10, R11 + OR R7, R11 + SLD R4, R10, R7 + MOVD R11, -8(R6) + ADD $-8, R5 + ADD $-8, R6 + BDNZ loop1cont +loop1done: +loop4: + CMP R3, $0; BEQ loop4done; MOVD R3, CTR +loop4cont: + // unroll 4X + MOVD -16(R5), R9 + MOVD -24(R5), R10 + MOVD -32(R5), R11 + MOVD -40(R5), R12 + SRD R8, R9, R14 + OR R7, R14 + SLD R4, R9, R7 + SRD R8, R10, R9 + OR R7, R9 + SLD R4, R10, R7 + SRD R8, R11, R10 + OR R7, R10 + SLD R4, R11, R7 + SRD R8, R12, R11 + OR R7, R11 + SLD R4, R12, R7 + MOVD R14, -8(R6) + MOVD R9, -16(R6) + MOVD R10, -24(R6) + MOVD R11, -32(R6) + ADD $-32, R5 + ADD $-32, R6 + BDNZ loop4cont +loop4done: + // store final shifted bits + MOVD R7, -8(R6) RET - -//func shlVU(z, x []Word, s uint) (c Word) -TEXT ·shlVU(SB), NOSPLIT, $0 - MOVD z+0(FP), R3 - MOVD x+24(FP), R6 - MOVD s+48(FP), R9 - MOVD z_len+8(FP), R4 - MOVD x_len+32(FP), R7 - CMP R9, $0 // s==0 copy(z,x) - BEQ zeroshift - CMP R4, $0 // len(z)==0 return - BEQ done - - ADD $-1, R4, R5 // len(z)-1 - SUBC R9, $64, R4 // ŝ=_W-s, we skip & by _W-1 as the caller ensures s < _W(64) - SLD $3, R5, R7 - ADD R6, R7, R15 // save starting address &x[len(z)-1] - ADD R3, R7, R16 // save starting address &z[len(z)-1] - MOVD (R6)(R7), R14 - SRD R4, R14, R7 // compute x[len(z)-1]>>ŝ into R7 - CMP R5, $0 // iterate from i=len(z)-1 to 0 - BEQ loopexit // Already at end? - MOVD 0(R15),R10 // x[i] - PCALIGN $16 -shloop: - SLD R9, R10, R10 // x[i]<>ŝ - OR R11, R10, R10 - MOVD R10, 0(R16) // z[i-1]=x[i]<>ŝ - MOVD R14, R10 // reuse x[i-1] for next iteration - ADD $-8, R16 // i-- - CMP R15, R6 // &x[i-1]>&x[0]? - BGT shloop -loopexit: - MOVD 0(R6), R4 - SLD R9, R4, R4 - MOVD R4, 0(R3) // z[0]=x[0]<>ŝ into c +ret0: + MOVD R0, c+56(FP) RET -zeroshift: - CMP R6, $0 // x is null, nothing to copy - BEQ done - CMP R6, R3 // if x is same as z, nothing to copy - BEQ done - CMP R7, R4 - ISEL $0, R7, R4, R7 // Take the lower bound of lengths of x,z - SLD $3, R7, R7 - SUB R6, R3, R11 // dest - src - CMPU R11, R7, CR2 // < len? - BLT CR2, backward // there is overlap, copy backwards - MOVD $0, R14 - // shlVU processes backwards, but added a forward copy option - // since its faster on POWER -repeat: - MOVD (R6)(R14), R15 // Copy 8 bytes at a time - MOVD R15, (R3)(R14) - ADD $8, R14 - CMP R14, R7 // More 8 bytes left? - BLT repeat - BR done -backward: - ADD $-8,R7, R14 -repeatback: - MOVD (R6)(R14), R15 // copy x into z backwards - MOVD R15, (R3)(R14) // copy 8 bytes at a time - SUB $8, R14 - CMP R14, $-8 // More 8 bytes left? - BGT repeatback - -done: - MOVD R0, c+56(FP) // c=0 +// func rshVU(z, x []Word, s uint) (c Word) +TEXT ·rshVU(SB), NOSPLIT, $0 + MOVD z_len+8(FP), R3 + CMP R3, $0; BEQ ret0 + MOVD s+48(FP), R4 + MOVD x_base+24(FP), R5 + MOVD z_base+0(FP), R6 + // shift first word into carry + MOVD 0(R5), R7 + MOVD $64, R8 + SUB R4, R8 + SLD R8, R7, R9 + SRD R4, R7 + MOVD R9, c+56(FP) + // shift remaining words + SUB $1, R3 + // compute unrolled loop lengths + ANDCC $3, R3, R9 + SRD $2, R3 +loop1: + CMP R9, $0; BEQ loop1done; MOVD R9, CTR +loop1cont: + // unroll 1X + MOVD 8(R5), R10 + SLD R8, R10, R11 + OR R7, R11 + SRD R4, R10, R7 + MOVD R11, 0(R6) + ADD $8, R5 + ADD $8, R6 + BDNZ loop1cont +loop1done: +loop4: + CMP R3, $0; BEQ loop4done; MOVD R3, CTR +loop4cont: + // unroll 4X + MOVD 8(R5), R9 + MOVD 16(R5), R10 + MOVD 24(R5), R11 + MOVD 32(R5), R12 + SLD R8, R9, R14 + OR R7, R14 + SRD R4, R9, R7 + SLD R8, R10, R9 + OR R7, R9 + SRD R4, R10, R7 + SLD R8, R11, R10 + OR R7, R10 + SRD R4, R11, R7 + SLD R8, R12, R11 + OR R7, R11 + SRD R4, R12, R7 + MOVD R14, 0(R6) + MOVD R9, 8(R6) + MOVD R10, 16(R6) + MOVD R11, 24(R6) + ADD $32, R5 + ADD $32, R6 + BDNZ loop4cont +loop4done: + // store final shifted bits + MOVD R7, 0(R6) RET - -//func shrVU(z, x []Word, s uint) (c Word) -TEXT ·shrVU(SB), NOSPLIT, $0 - MOVD z+0(FP), R3 - MOVD x+24(FP), R6 - MOVD s+48(FP), R9 - MOVD z_len+8(FP), R4 - MOVD x_len+32(FP), R7 - - CMP R9, $0 // s==0, copy(z,x) - BEQ zeroshift - CMP R4, $0 // len(z)==0 return - BEQ done - SUBC R9, $64, R5 // ŝ=_W-s, we skip & by _W-1 as the caller ensures s < _W(64) - - MOVD 0(R6), R7 - SLD R5, R7, R7 // compute x[0]<<ŝ - MOVD $1, R8 // iterate from i=1 to i=3, else jump to scalar loop - CMP R4, $3 - BLT scalar - MTVSRD R9, VS38 // s - VSPLTB $7, V6, V4 - MTVSRD R5, VS39 // ŝ - VSPLTB $7, V7, V2 - ADD $-2, R4, R16 - PCALIGN $16 -loopback: - ADD $-1, R8, R10 - SLD $3, R10 - LXVD2X (R6)(R10), VS32 // load x[i-1], x[i] - SLD $3, R8, R12 - LXVD2X (R6)(R12), VS33 // load x[i], x[i+1] - - VSRD V0, V4, V3 // x[i-1]>>s, x[i]>>s - VSLD V1, V2, V5 // x[i]<<ŝ, x[i+1]<<ŝ - VOR V3, V5, V5 // Or(|) the two registers together - STXVD2X VS37, (R3)(R10) // store into z[i-1] and z[i] - ADD $2, R8 // Done processing 2 entries, i and i+1 - CMP R8, R16 // Are there at least a couple of more entries left? - BLE loopback - CMP R8, R4 // Are we at the last element? - BEQ loopexit -scalar: - ADD $-1, R8, R10 - SLD $3, R10 - MOVD (R6)(R10),R11 - SRD R9, R11, R11 // x[len(z)-2] >> s - SLD $3, R8, R12 - MOVD (R6)(R12), R12 - SLD R5, R12, R12 // x[len(z)-1]<<ŝ - OR R12, R11, R11 // x[len(z)-2]>>s | x[len(z)-1]<<ŝ - MOVD R11, (R3)(R10) // z[len(z)-2]=x[len(z)-2]>>s | x[len(z)-1]<<ŝ -loopexit: - ADD $-1, R4 - SLD $3, R4 - MOVD (R6)(R4), R5 - SRD R9, R5, R5 // x[len(z)-1]>>s - MOVD R5, (R3)(R4) // z[len(z)-1]=x[len(z)-1]>>s - MOVD R7, c+56(FP) // store pre-computed x[0]<<ŝ into c - RET - -zeroshift: - CMP R6, $0 // x is null, nothing to copy - BEQ done - CMP R6, R3 // if x is same as z, nothing to copy - BEQ done - CMP R7, R4 - ISEL $0, R7, R4, R7 // Take the lower bounds of lengths of x, z - SLD $3, R7, R7 - MOVD $0, R14 -repeat: - MOVD (R6)(R14), R15 // copy 8 bytes at a time - MOVD R15, (R3)(R14) // shrVU processes bytes only forwards - ADD $8, R14 - CMP R14, R7 // More 8 bytes left? - BLT repeat -done: - MOVD R0, c+56(FP) +ret0: + MOVD R0, c+56(FP) RET -// func mulAddVWW(z, x []Word, y, r Word) (c Word) +// func mulAddVWW(z, x []Word, m, a Word) (c Word) TEXT ·mulAddVWW(SB), NOSPLIT, $0 - MOVD z+0(FP), R10 // R10 = z[] - MOVD x+24(FP), R8 // R8 = x[] - MOVD y+48(FP), R9 // R9 = y - MOVD r+56(FP), R4 // R4 = r = c - MOVD z_len+8(FP), R11 // R11 = z_len - - CMP R11, $0 - BEQ done - - MOVD 0(R8), R20 - ADD $-1, R11 - MULLD R9, R20, R6 // R6 = z0 = Low-order(x[i]*y) - MULHDU R9, R20, R7 // R7 = z1 = High-order(x[i]*y) - ADDC R4, R6 // R6 = z0 + r - ADDZE R7, R4 // R4 = z1 + CA - CMP R11, $0 - MOVD R6, 0(R10) // z[i] - BEQ done - - // We will read 4 elements per iteration - SRDCC $2, R11, R14 // R14 = z_len/4 - DCBT (R8) - MOVD R14, CTR // Set up the loop counter - BEQ tail // If R9 = 0, we can't use the loop - PCALIGN $16 - -loop: - MOVD 8(R8), R20 // R20 = x[i] - MOVD 16(R8), R21 // R21 = x[i+1] - MOVD 24(R8), R22 // R22 = x[i+2] - MOVDU 32(R8), R23 // R23 = x[i+3] - MULLD R9, R20, R24 // R24 = z0[i] - MULHDU R9, R20, R20 // R20 = z1[i] - ADDC R4, R24 // R24 = z0[i] + c - MULLD R9, R21, R25 - MULHDU R9, R21, R21 - ADDE R20, R25 - MULLD R9, R22, R26 - MULHDU R9, R22, R22 - MULLD R9, R23, R27 - MULHDU R9, R23, R23 - ADDE R21, R26 - MOVD R24, 8(R10) // z[i] - MOVD R25, 16(R10) // z[i+1] - ADDE R22, R27 - ADDZE R23,R4 // update carry - MOVD R26, 24(R10) // z[i+2] - MOVDU R27, 32(R10) // z[i+3] - ADD $-4, R11 // R11 = z_len - 4 - BDNZ loop - - // We may have some elements to read - CMP R11, $0 - BEQ done - - // Process the remaining elements, one at a time -tail: - MOVDU 8(R8), R20 // R20 = x[i] - MULLD R9, R20, R24 // R24 = z0[i] - MULHDU R9, R20, R25 // R25 = z1[i] - ADD $-1, R11 // R11 = z_len - 1 - ADDC R4, R24 - ADDZE R25, R4 - MOVDU R24, 8(R10) // z[i] - CMP R11, $0 - BEQ done // If R11 = 0, we are done - - MOVDU 8(R8), R20 - MULLD R9, R20, R24 - MULHDU R9, R20, R25 - ADD $-1, R11 - ADDC R4, R24 - ADDZE R25, R4 - MOVDU R24, 8(R10) - CMP R11, $0 - BEQ done - - MOVD 8(R8), R20 - MULLD R9, R20, R24 - MULHDU R9, R20, R25 - ADD $-1, R11 - ADDC R4, R24 - ADDZE R25,R4 - MOVD R24, 8(R10) - -done: - MOVD R4, c+64(FP) + MOVD m+48(FP), R3 + MOVD a+56(FP), R4 + MOVD z_len+8(FP), R5 + MOVD x_base+24(FP), R6 + MOVD z_base+0(FP), R7 + // compute unrolled loop lengths + ANDCC $3, R5, R8 + SRD $2, R5 +loop1: + CMP R8, $0; BEQ loop1done; MOVD R8, CTR +loop1cont: + // unroll 1X + MOVD 0(R6), R9 + // multiply + MULHDU R3, R9, R10 + MULLD R3, R9 + ADDC R4, R9 + ADDE R0, R10, R4 + MOVD R9, 0(R7) + ADD $8, R6 + ADD $8, R7 + BDNZ loop1cont +loop1done: +loop4: + CMP R5, $0; BEQ loop4done; MOVD R5, CTR +loop4cont: + // unroll 4X + MOVD 0(R6), R8 + MOVD 8(R6), R9 + MOVD 16(R6), R10 + MOVD 24(R6), R11 + // multiply + MULHDU R3, R8, R12 + MULLD R3, R8 + ADDC R4, R8 + MULHDU R3, R9, R14 + MULLD R3, R9 + ADDE R12, R9 + MULHDU R3, R10, R12 + MULLD R3, R10 + ADDE R14, R10 + MULHDU R3, R11, R14 + MULLD R3, R11 + ADDE R12, R11 + ADDE R0, R14, R4 + MOVD R8, 0(R7) + MOVD R9, 8(R7) + MOVD R10, 16(R7) + MOVD R11, 24(R7) + ADD $32, R6 + ADD $32, R7 + BDNZ loop4cont +loop4done: + MOVD R4, c+64(FP) RET -// func addMulVVW(z, x []Word, y Word) (c Word) -TEXT ·addMulVVW(SB), NOSPLIT, $0 - MOVD z+0(FP), R3 // R3 = z[] - MOVD x+24(FP), R4 // R4 = x[] - MOVD y+48(FP), R5 // R5 = y - MOVD z_len+8(FP), R6 // R6 = z_len - - CMP R6, $4 - MOVD R0, R9 // R9 = c = 0 - BLT tail - SRD $2, R6, R7 - MOVD R7, CTR // Initialize loop counter - PCALIGN $16 - -loop: - MOVD 0(R4), R14 // x[i] - MOVD 8(R4), R16 // x[i+1] - MOVD 16(R4), R18 // x[i+2] - MOVD 24(R4), R20 // x[i+3] - MOVD 0(R3), R15 // z[i] - MOVD 8(R3), R17 // z[i+1] - MOVD 16(R3), R19 // z[i+2] - MOVD 24(R3), R21 // z[i+3] - MULLD R5, R14, R10 // low x[i]*y - MULHDU R5, R14, R11 // high x[i]*y - ADDC R15, R10 - ADDZE R11 - ADDC R9, R10 - ADDZE R11, R9 - MULLD R5, R16, R14 // low x[i+1]*y - MULHDU R5, R16, R15 // high x[i+1]*y - ADDC R17, R14 - ADDZE R15 - ADDC R9, R14 - ADDZE R15, R9 - MULLD R5, R18, R16 // low x[i+2]*y - MULHDU R5, R18, R17 // high x[i+2]*y - ADDC R19, R16 - ADDZE R17 - ADDC R9, R16 - ADDZE R17, R9 - MULLD R5, R20, R18 // low x[i+3]*y - MULHDU R5, R20, R19 // high x[i+3]*y - ADDC R21, R18 - ADDZE R19 - ADDC R9, R18 - ADDZE R19, R9 - MOVD R10, 0(R3) // z[i] - MOVD R14, 8(R3) // z[i+1] - MOVD R16, 16(R3) // z[i+2] - MOVD R18, 24(R3) // z[i+3] - ADD $32, R3 - ADD $32, R4 - BDNZ loop - - ANDCC $3, R6 -tail: - CMP R6, $0 - BEQ done - MOVD R6, CTR - PCALIGN $16 -tailloop: - MOVD 0(R4), R14 - MOVD 0(R3), R15 - MULLD R5, R14, R10 - MULHDU R5, R14, R11 - ADDC R15, R10 - ADDZE R11 - ADDC R9, R10 - ADDZE R11, R9 - MOVD R10, 0(R3) - ADD $8, R3 - ADD $8, R4 - BDNZ tailloop - -done: - MOVD R9, c+56(FP) +// func addMulVVWW(z, x, y []Word, m, a Word) (c Word) +TEXT ·addMulVVWW(SB), NOSPLIT, $0 + MOVD m+72(FP), R3 + MOVD a+80(FP), R4 + MOVD z_len+8(FP), R5 + MOVD x_base+24(FP), R6 + MOVD y_base+48(FP), R7 + MOVD z_base+0(FP), R8 + // compute unrolled loop lengths + ANDCC $3, R5, R9 + SRD $2, R5 +loop1: + CMP R9, $0; BEQ loop1done; MOVD R9, CTR +loop1cont: + // unroll 1X + MOVD 0(R6), R10 + MOVD 0(R7), R11 + // multiply + MULHDU R3, R11, R12 + MULLD R3, R11 + ADDC R4, R11 + ADDE R0, R12, R4 + // add + ADDC R10, R11 + ADDE R0, R4 + MOVD R11, 0(R8) + ADD $8, R6 + ADD $8, R7 + ADD $8, R8 + BDNZ loop1cont +loop1done: +loop4: + CMP R5, $0; BEQ loop4done; MOVD R5, CTR +loop4cont: + // unroll 4X + MOVD 0(R6), R9 + MOVD 8(R6), R10 + MOVD 16(R6), R11 + MOVD 24(R6), R12 + MOVD 0(R7), R14 + MOVD 8(R7), R15 + MOVD 16(R7), R16 + MOVD 24(R7), R17 + // multiply + MULHDU R3, R14, R18 + MULLD R3, R14 + ADDC R4, R14 + MULHDU R3, R15, R19 + MULLD R3, R15 + ADDE R18, R15 + MULHDU R3, R16, R18 + MULLD R3, R16 + ADDE R19, R16 + MULHDU R3, R17, R19 + MULLD R3, R17 + ADDE R18, R17 + ADDE R0, R19, R4 + // add + ADDC R9, R14 + ADDE R10, R15 + ADDE R11, R16 + ADDE R12, R17 + ADDE R0, R4 + MOVD R14, 0(R8) + MOVD R15, 8(R8) + MOVD R16, 16(R8) + MOVD R17, 24(R8) + ADD $32, R6 + ADD $32, R7 + ADD $32, R8 + BDNZ loop4cont +loop4done: + MOVD R4, c+88(FP) RET - diff --git a/src/math/big/arith_riscv64.s b/src/math/big/arith_riscv64.s index bad32497b71c67..8817b1c8265d33 100644 --- a/src/math/big/arith_riscv64.s +++ b/src/math/big/arith_riscv64.s @@ -1,35 +1,457 @@ -// Copyright 2020 The Go Authors. All rights reserved. +// Copyright 2025 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !math_big_pure_go && riscv64 +// Code generated by 'go generate' (with ./internal/asmgen). DO NOT EDIT. -#include "textflag.h" - -// This file provides fast assembly versions for the elementary -// arithmetic operations on vectors implemented in arith.go. - -TEXT ·addVV(SB),NOSPLIT,$0 - JMP ·addVV_g(SB) +//go:build !math_big_pure_go -TEXT ·subVV(SB),NOSPLIT,$0 - JMP ·subVV_g(SB) - -TEXT ·addVW(SB),NOSPLIT,$0 - JMP ·addVW_g(SB) +#include "textflag.h" -TEXT ·subVW(SB),NOSPLIT,$0 - JMP ·subVW_g(SB) +// func addVV(z, x, y []Word) (c Word) +TEXT ·addVV(SB), NOSPLIT, $0 + MOV z_len+8(FP), X5 + MOV x_base+24(FP), X6 + MOV y_base+48(FP), X7 + MOV z_base+0(FP), X8 + // compute unrolled loop lengths + AND $3, X5, X9 + SRL $2, X5 + XOR X28, X28 // clear carry +loop1: + BEQZ X9, loop1done +loop1cont: + // unroll 1X + MOV 0(X6), X10 + MOV 0(X7), X11 + ADD X11, X10 // ADCS X11, X10, X10 (cr=X28) + SLTU X11, X10, X31 // ... + ADD X28, X10 // ... + SLTU X28, X10, X28 // ... + ADD X31, X28 // ... + MOV X10, 0(X8) + ADD $8, X6 + ADD $8, X7 + ADD $8, X8 + SUB $1, X9 + BNEZ X9, loop1cont +loop1done: +loop4: + BEQZ X5, loop4done +loop4cont: + // unroll 4X + MOV 0(X6), X9 + MOV 8(X6), X10 + MOV 16(X6), X11 + MOV 24(X6), X12 + MOV 0(X7), X13 + MOV 8(X7), X14 + MOV 16(X7), X15 + MOV 24(X7), X16 + ADD X13, X9 // ADCS X13, X9, X9 (cr=X28) + SLTU X13, X9, X31 // ... + ADD X28, X9 // ... + SLTU X28, X9, X28 // ... + ADD X31, X28 // ... + ADD X14, X10 // ADCS X14, X10, X10 (cr=X28) + SLTU X14, X10, X31 // ... + ADD X28, X10 // ... + SLTU X28, X10, X28 // ... + ADD X31, X28 // ... + ADD X15, X11 // ADCS X15, X11, X11 (cr=X28) + SLTU X15, X11, X31 // ... + ADD X28, X11 // ... + SLTU X28, X11, X28 // ... + ADD X31, X28 // ... + ADD X16, X12 // ADCS X16, X12, X12 (cr=X28) + SLTU X16, X12, X31 // ... + ADD X28, X12 // ... + SLTU X28, X12, X28 // ... + ADD X31, X28 // ... + MOV X9, 0(X8) + MOV X10, 8(X8) + MOV X11, 16(X8) + MOV X12, 24(X8) + ADD $32, X6 + ADD $32, X7 + ADD $32, X8 + SUB $1, X5 + BNEZ X5, loop4cont +loop4done: + MOV X28, c+72(FP) + RET -TEXT ·shlVU(SB),NOSPLIT,$0 - JMP ·shlVU_g(SB) +// func subVV(z, x, y []Word) (c Word) +TEXT ·subVV(SB), NOSPLIT, $0 + MOV z_len+8(FP), X5 + MOV x_base+24(FP), X6 + MOV y_base+48(FP), X7 + MOV z_base+0(FP), X8 + // compute unrolled loop lengths + AND $3, X5, X9 + SRL $2, X5 + XOR X28, X28 // clear carry +loop1: + BEQZ X9, loop1done +loop1cont: + // unroll 1X + MOV 0(X6), X10 + MOV 0(X7), X11 + SLTU X28, X10, X31 // SBCS X11, X10, X10 + SUB X28, X10 // ... + SLTU X11, X10, X28 // ... + SUB X11, X10 // ... + ADD X31, X28 // ... + MOV X10, 0(X8) + ADD $8, X6 + ADD $8, X7 + ADD $8, X8 + SUB $1, X9 + BNEZ X9, loop1cont +loop1done: +loop4: + BEQZ X5, loop4done +loop4cont: + // unroll 4X + MOV 0(X6), X9 + MOV 8(X6), X10 + MOV 16(X6), X11 + MOV 24(X6), X12 + MOV 0(X7), X13 + MOV 8(X7), X14 + MOV 16(X7), X15 + MOV 24(X7), X16 + SLTU X28, X9, X31 // SBCS X13, X9, X9 + SUB X28, X9 // ... + SLTU X13, X9, X28 // ... + SUB X13, X9 // ... + ADD X31, X28 // ... + SLTU X28, X10, X31 // SBCS X14, X10, X10 + SUB X28, X10 // ... + SLTU X14, X10, X28 // ... + SUB X14, X10 // ... + ADD X31, X28 // ... + SLTU X28, X11, X31 // SBCS X15, X11, X11 + SUB X28, X11 // ... + SLTU X15, X11, X28 // ... + SUB X15, X11 // ... + ADD X31, X28 // ... + SLTU X28, X12, X31 // SBCS X16, X12, X12 + SUB X28, X12 // ... + SLTU X16, X12, X28 // ... + SUB X16, X12 // ... + ADD X31, X28 // ... + MOV X9, 0(X8) + MOV X10, 8(X8) + MOV X11, 16(X8) + MOV X12, 24(X8) + ADD $32, X6 + ADD $32, X7 + ADD $32, X8 + SUB $1, X5 + BNEZ X5, loop4cont +loop4done: + MOV X28, c+72(FP) + RET -TEXT ·shrVU(SB),NOSPLIT,$0 - JMP ·shrVU_g(SB) +// func lshVU(z, x []Word, s uint) (c Word) +TEXT ·lshVU(SB), NOSPLIT, $0 + MOV z_len+8(FP), X5 + BEQZ X5, ret0 + MOV s+48(FP), X6 + MOV x_base+24(FP), X7 + MOV z_base+0(FP), X8 + // run loop backward + SLL $3, X5, X9 + ADD X9, X7 + SLL $3, X5, X9 + ADD X9, X8 + // shift first word into carry + MOV -8(X7), X9 + MOV $64, X10 + SUB X6, X10 + SRL X10, X9, X11 + SLL X6, X9 + MOV X11, c+56(FP) + // shift remaining words + SUB $1, X5 + // compute unrolled loop lengths + AND $3, X5, X11 + SRL $2, X5 +loop1: + BEQZ X11, loop1done +loop1cont: + // unroll 1X + MOV -16(X7), X12 + SRL X10, X12, X13 + OR X9, X13 + SLL X6, X12, X9 + MOV X13, -8(X8) + ADD $-8, X7 + ADD $-8, X8 + SUB $1, X11 + BNEZ X11, loop1cont +loop1done: +loop4: + BEQZ X5, loop4done +loop4cont: + // unroll 4X + MOV -16(X7), X11 + MOV -24(X7), X12 + MOV -32(X7), X13 + MOV -40(X7), X14 + SRL X10, X11, X15 + OR X9, X15 + SLL X6, X11, X9 + SRL X10, X12, X11 + OR X9, X11 + SLL X6, X12, X9 + SRL X10, X13, X12 + OR X9, X12 + SLL X6, X13, X9 + SRL X10, X14, X13 + OR X9, X13 + SLL X6, X14, X9 + MOV X15, -8(X8) + MOV X11, -16(X8) + MOV X12, -24(X8) + MOV X13, -32(X8) + ADD $-32, X7 + ADD $-32, X8 + SUB $1, X5 + BNEZ X5, loop4cont +loop4done: + // store final shifted bits + MOV X9, -8(X8) + RET +ret0: + MOV X0, c+56(FP) + RET -TEXT ·mulAddVWW(SB),NOSPLIT,$0 - JMP ·mulAddVWW_g(SB) +// func rshVU(z, x []Word, s uint) (c Word) +TEXT ·rshVU(SB), NOSPLIT, $0 + MOV z_len+8(FP), X5 + BEQZ X5, ret0 + MOV s+48(FP), X6 + MOV x_base+24(FP), X7 + MOV z_base+0(FP), X8 + // shift first word into carry + MOV 0(X7), X9 + MOV $64, X10 + SUB X6, X10 + SLL X10, X9, X11 + SRL X6, X9 + MOV X11, c+56(FP) + // shift remaining words + SUB $1, X5 + // compute unrolled loop lengths + AND $3, X5, X11 + SRL $2, X5 +loop1: + BEQZ X11, loop1done +loop1cont: + // unroll 1X + MOV 8(X7), X12 + SLL X10, X12, X13 + OR X9, X13 + SRL X6, X12, X9 + MOV X13, 0(X8) + ADD $8, X7 + ADD $8, X8 + SUB $1, X11 + BNEZ X11, loop1cont +loop1done: +loop4: + BEQZ X5, loop4done +loop4cont: + // unroll 4X + MOV 8(X7), X11 + MOV 16(X7), X12 + MOV 24(X7), X13 + MOV 32(X7), X14 + SLL X10, X11, X15 + OR X9, X15 + SRL X6, X11, X9 + SLL X10, X12, X11 + OR X9, X11 + SRL X6, X12, X9 + SLL X10, X13, X12 + OR X9, X12 + SRL X6, X13, X9 + SLL X10, X14, X13 + OR X9, X13 + SRL X6, X14, X9 + MOV X15, 0(X8) + MOV X11, 8(X8) + MOV X12, 16(X8) + MOV X13, 24(X8) + ADD $32, X7 + ADD $32, X8 + SUB $1, X5 + BNEZ X5, loop4cont +loop4done: + // store final shifted bits + MOV X9, 0(X8) + RET +ret0: + MOV X0, c+56(FP) + RET -TEXT ·addMulVVW(SB),NOSPLIT,$0 - JMP ·addMulVVW_g(SB) +// func mulAddVWW(z, x []Word, m, a Word) (c Word) +TEXT ·mulAddVWW(SB), NOSPLIT, $0 + MOV m+48(FP), X5 + MOV a+56(FP), X6 + MOV z_len+8(FP), X7 + MOV x_base+24(FP), X8 + MOV z_base+0(FP), X9 + // compute unrolled loop lengths + AND $3, X7, X10 + SRL $2, X7 +loop1: + BEQZ X10, loop1done +loop1cont: + // unroll 1X + MOV 0(X8), X11 + // synthetic carry, one column at a time + MUL X5, X11, X12 + MULHU X5, X11, X13 + ADD X6, X12, X11 // ADDS X6, X12, X11 (cr=X28) + SLTU X6, X11, X28 // ... + ADD X28, X13, X6 // ADC $0, X13, X6 + MOV X11, 0(X9) + ADD $8, X8 + ADD $8, X9 + SUB $1, X10 + BNEZ X10, loop1cont +loop1done: +loop4: + BEQZ X7, loop4done +loop4cont: + // unroll 4X + MOV 0(X8), X10 + MOV 8(X8), X11 + MOV 16(X8), X12 + MOV 24(X8), X13 + // synthetic carry, one column at a time + MUL X5, X10, X14 + MULHU X5, X10, X15 + ADD X6, X14, X10 // ADDS X6, X14, X10 (cr=X28) + SLTU X6, X10, X28 // ... + ADD X28, X15, X6 // ADC $0, X15, X6 + MUL X5, X11, X14 + MULHU X5, X11, X15 + ADD X6, X14, X11 // ADDS X6, X14, X11 (cr=X28) + SLTU X6, X11, X28 // ... + ADD X28, X15, X6 // ADC $0, X15, X6 + MUL X5, X12, X14 + MULHU X5, X12, X15 + ADD X6, X14, X12 // ADDS X6, X14, X12 (cr=X28) + SLTU X6, X12, X28 // ... + ADD X28, X15, X6 // ADC $0, X15, X6 + MUL X5, X13, X14 + MULHU X5, X13, X15 + ADD X6, X14, X13 // ADDS X6, X14, X13 (cr=X28) + SLTU X6, X13, X28 // ... + ADD X28, X15, X6 // ADC $0, X15, X6 + MOV X10, 0(X9) + MOV X11, 8(X9) + MOV X12, 16(X9) + MOV X13, 24(X9) + ADD $32, X8 + ADD $32, X9 + SUB $1, X7 + BNEZ X7, loop4cont +loop4done: + MOV X6, c+64(FP) + RET +// func addMulVVWW(z, x, y []Word, m, a Word) (c Word) +TEXT ·addMulVVWW(SB), NOSPLIT, $0 + MOV m+72(FP), X5 + MOV a+80(FP), X6 + MOV z_len+8(FP), X7 + MOV x_base+24(FP), X8 + MOV y_base+48(FP), X9 + MOV z_base+0(FP), X10 + // compute unrolled loop lengths + AND $3, X7, X11 + SRL $2, X7 +loop1: + BEQZ X11, loop1done +loop1cont: + // unroll 1X + MOV 0(X8), X12 + MOV 0(X9), X13 + // synthetic carry, one column at a time + MUL X5, X13, X14 + MULHU X5, X13, X15 + ADD X12, X14 // ADDS X12, X14, X14 (cr=X28) + SLTU X12, X14, X28 // ... + ADD X28, X15 // ADC $0, X15, X15 + ADD X6, X14, X13 // ADDS X6, X14, X13 (cr=X28) + SLTU X6, X13, X28 // ... + ADD X28, X15, X6 // ADC $0, X15, X6 + MOV X13, 0(X10) + ADD $8, X8 + ADD $8, X9 + ADD $8, X10 + SUB $1, X11 + BNEZ X11, loop1cont +loop1done: +loop4: + BEQZ X7, loop4done +loop4cont: + // unroll 4X + MOV 0(X8), X11 + MOV 8(X8), X12 + MOV 16(X8), X13 + MOV 24(X8), X14 + MOV 0(X9), X15 + MOV 8(X9), X16 + MOV 16(X9), X17 + MOV 24(X9), X18 + // synthetic carry, one column at a time + MUL X5, X15, X19 + MULHU X5, X15, X20 + ADD X11, X19 // ADDS X11, X19, X19 (cr=X28) + SLTU X11, X19, X28 // ... + ADD X28, X20 // ADC $0, X20, X20 + ADD X6, X19, X15 // ADDS X6, X19, X15 (cr=X28) + SLTU X6, X15, X28 // ... + ADD X28, X20, X6 // ADC $0, X20, X6 + MUL X5, X16, X19 + MULHU X5, X16, X20 + ADD X12, X19 // ADDS X12, X19, X19 (cr=X28) + SLTU X12, X19, X28 // ... + ADD X28, X20 // ADC $0, X20, X20 + ADD X6, X19, X16 // ADDS X6, X19, X16 (cr=X28) + SLTU X6, X16, X28 // ... + ADD X28, X20, X6 // ADC $0, X20, X6 + MUL X5, X17, X19 + MULHU X5, X17, X20 + ADD X13, X19 // ADDS X13, X19, X19 (cr=X28) + SLTU X13, X19, X28 // ... + ADD X28, X20 // ADC $0, X20, X20 + ADD X6, X19, X17 // ADDS X6, X19, X17 (cr=X28) + SLTU X6, X17, X28 // ... + ADD X28, X20, X6 // ADC $0, X20, X6 + MUL X5, X18, X19 + MULHU X5, X18, X20 + ADD X14, X19 // ADDS X14, X19, X19 (cr=X28) + SLTU X14, X19, X28 // ... + ADD X28, X20 // ADC $0, X20, X20 + ADD X6, X19, X18 // ADDS X6, X19, X18 (cr=X28) + SLTU X6, X18, X28 // ... + ADD X28, X20, X6 // ADC $0, X20, X6 + MOV X15, 0(X10) + MOV X16, 8(X10) + MOV X17, 16(X10) + MOV X18, 24(X10) + ADD $32, X8 + ADD $32, X9 + ADD $32, X10 + SUB $1, X7 + BNEZ X7, loop4cont +loop4done: + MOV X6, c+88(FP) + RET diff --git a/src/math/big/arith_s390x.s b/src/math/big/arith_s390x.s index 01a7bb2d512629..b81ed92480b635 100644 --- a/src/math/big/arith_s390x.s +++ b/src/math/big/arith_s390x.s @@ -1,786 +1,426 @@ -// Copyright 2016 The Go Authors. All rights reserved. +// Copyright 2025 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Code generated by 'go generate' (with ./internal/asmgen). DO NOT EDIT. + //go:build !math_big_pure_go #include "textflag.h" -// This file provides fast assembly versions for the elementary -// arithmetic operations on vectors implemented in arith.go. - -// DI = R3, CX = R4, SI = r10, r8 = r8, r9=r9, r10 = r2, r11 = r5, r12 = r6, r13 = r7, r14 = r1 (R0 set to 0) + use R11 // func addVV(z, x, y []Word) (c Word) - TEXT ·addVV(SB), NOSPLIT, $0 - MOVD addvectorfacility+0x00(SB), R1 - BR (R1) - -TEXT ·addVV_check(SB), NOSPLIT, $0 - MOVB ·hasVX(SB), R1 - CMPBEQ R1, $1, vectorimpl // vectorfacility = 1, vector supported - MOVD $addvectorfacility+0x00(SB), R1 - MOVD $·addVV_novec(SB), R2 - MOVD R2, 0(R1) - - // MOVD $·addVV_novec(SB), 0(R1) - BR ·addVV_novec(SB) - -vectorimpl: - MOVD $addvectorfacility+0x00(SB), R1 - MOVD $·addVV_vec(SB), R2 - MOVD R2, 0(R1) - - // MOVD $·addVV_vec(SB), 0(R1) - BR ·addVV_vec(SB) - -GLOBL addvectorfacility+0x00(SB), NOPTR, $8 -DATA addvectorfacility+0x00(SB)/8, $·addVV_check(SB) - -TEXT ·addVV_vec(SB), NOSPLIT, $0 - MOVD z_len+8(FP), R3 - MOVD x+24(FP), R8 - MOVD y+48(FP), R9 - MOVD z+0(FP), R2 - - MOVD $0, R4 // c = 0 - MOVD $0, R0 // make sure it's zero - MOVD $0, R10 // i = 0 - - // s/JL/JMP/ below to disable the unrolled loop - SUB $4, R3 - BLT v1 - SUB $12, R3 // n -= 16 - BLT A1 // if n < 0 goto A1 - - MOVD R8, R5 - MOVD R9, R6 - MOVD R2, R7 - - // n >= 0 - // regular loop body unrolled 16x - VZERO V0 // c = 0 - -UU1: - VLM 0(R5), V1, V4 // 64-bytes into V1..V8 - ADD $64, R5 - VPDI $0x4, V1, V1, V1 // flip the doublewords to big-endian order - VPDI $0x4, V2, V2, V2 // flip the doublewords to big-endian order - - VLM 0(R6), V9, V12 // 64-bytes into V9..V16 - ADD $64, R6 - VPDI $0x4, V9, V9, V9 // flip the doublewords to big-endian order - VPDI $0x4, V10, V10, V10 // flip the doublewords to big-endian order - - VACCCQ V1, V9, V0, V25 - VACQ V1, V9, V0, V17 - VACCCQ V2, V10, V25, V26 - VACQ V2, V10, V25, V18 - - VLM 0(R5), V5, V6 // 32-bytes into V1..V8 - VLM 0(R6), V13, V14 // 32-bytes into V9..V16 - ADD $32, R5 - ADD $32, R6 - - VPDI $0x4, V3, V3, V3 // flip the doublewords to big-endian order - VPDI $0x4, V4, V4, V4 // flip the doublewords to big-endian order - VPDI $0x4, V11, V11, V11 // flip the doublewords to big-endian order - VPDI $0x4, V12, V12, V12 // flip the doublewords to big-endian order - - VACCCQ V3, V11, V26, V27 - VACQ V3, V11, V26, V19 - VACCCQ V4, V12, V27, V28 - VACQ V4, V12, V27, V20 - - VLM 0(R5), V7, V8 // 32-bytes into V1..V8 - VLM 0(R6), V15, V16 // 32-bytes into V9..V16 - ADD $32, R5 - ADD $32, R6 - - VPDI $0x4, V5, V5, V5 // flip the doublewords to big-endian order - VPDI $0x4, V6, V6, V6 // flip the doublewords to big-endian order - VPDI $0x4, V13, V13, V13 // flip the doublewords to big-endian order - VPDI $0x4, V14, V14, V14 // flip the doublewords to big-endian order - - VACCCQ V5, V13, V28, V29 - VACQ V5, V13, V28, V21 - VACCCQ V6, V14, V29, V30 - VACQ V6, V14, V29, V22 - - VPDI $0x4, V7, V7, V7 // flip the doublewords to big-endian order - VPDI $0x4, V8, V8, V8 // flip the doublewords to big-endian order - VPDI $0x4, V15, V15, V15 // flip the doublewords to big-endian order - VPDI $0x4, V16, V16, V16 // flip the doublewords to big-endian order - - VACCCQ V7, V15, V30, V31 - VACQ V7, V15, V30, V23 - VACCCQ V8, V16, V31, V0 // V0 has carry-over - VACQ V8, V16, V31, V24 - - VPDI $0x4, V17, V17, V17 // flip the doublewords to big-endian order - VPDI $0x4, V18, V18, V18 // flip the doublewords to big-endian order - VPDI $0x4, V19, V19, V19 // flip the doublewords to big-endian order - VPDI $0x4, V20, V20, V20 // flip the doublewords to big-endian order - VPDI $0x4, V21, V21, V21 // flip the doublewords to big-endian order - VPDI $0x4, V22, V22, V22 // flip the doublewords to big-endian order - VPDI $0x4, V23, V23, V23 // flip the doublewords to big-endian order - VPDI $0x4, V24, V24, V24 // flip the doublewords to big-endian order - VSTM V17, V24, 0(R7) // 128-bytes into z - ADD $128, R7 - ADD $128, R10 // i += 16 - SUB $16, R3 // n -= 16 - BGE UU1 // if n >= 0 goto U1 - VLGVG $1, V0, R4 // put cf into R4 - NEG R4, R4 // save cf - -A1: - ADD $12, R3 // n += 16 - - // s/JL/JMP/ below to disable the unrolled loop - BLT v1 // if n < 0 goto v1 - -U1: // n >= 0 - // regular loop body unrolled 4x - MOVD 0(R8)(R10*1), R5 - MOVD 8(R8)(R10*1), R6 - MOVD 16(R8)(R10*1), R7 - MOVD 24(R8)(R10*1), R1 - ADDC R4, R4 // restore CF - MOVD 0(R9)(R10*1), R11 - ADDE R11, R5 - MOVD 8(R9)(R10*1), R11 - ADDE R11, R6 - MOVD 16(R9)(R10*1), R11 - ADDE R11, R7 - MOVD 24(R9)(R10*1), R11 - ADDE R11, R1 - MOVD R0, R4 - ADDE R4, R4 // save CF - NEG R4, R4 - MOVD R5, 0(R2)(R10*1) - MOVD R6, 8(R2)(R10*1) - MOVD R7, 16(R2)(R10*1) - MOVD R1, 24(R2)(R10*1) - - ADD $32, R10 // i += 4 - SUB $4, R3 // n -= 4 - BGE U1 // if n >= 0 goto U1 - -v1: - ADD $4, R3 // n += 4 - BLE E1 // if n <= 0 goto E1 - -L1: // n > 0 - ADDC R4, R4 // restore CF - MOVD 0(R8)(R10*1), R5 - MOVD 0(R9)(R10*1), R11 - ADDE R11, R5 - MOVD R5, 0(R2)(R10*1) - MOVD R0, R4 - ADDE R4, R4 // save CF - NEG R4, R4 - - ADD $8, R10 // i++ - SUB $1, R3 // n-- - BGT L1 // if n > 0 goto L1 - -E1: - NEG R4, R4 - MOVD R4, c+72(FP) // return c - RET - -TEXT ·addVV_novec(SB), NOSPLIT, $0 + MOVB ·hasVX(SB), R1 + CMPBEQ R1, $0, novec + JMP ·addVVvec(SB) novec: - MOVD z_len+8(FP), R3 - MOVD x+24(FP), R8 - MOVD y+48(FP), R9 - MOVD z+0(FP), R2 - - MOVD $0, R4 // c = 0 - MOVD $0, R0 // make sure it's zero - MOVD $0, R10 // i = 0 - - // s/JL/JMP/ below to disable the unrolled loop - SUB $4, R3 // n -= 4 - BLT v1n // if n < 0 goto v1n - -U1n: // n >= 0 - // regular loop body unrolled 4x - MOVD 0(R8)(R10*1), R5 - MOVD 8(R8)(R10*1), R6 - MOVD 16(R8)(R10*1), R7 - MOVD 24(R8)(R10*1), R1 - ADDC R4, R4 // restore CF - MOVD 0(R9)(R10*1), R11 - ADDE R11, R5 - MOVD 8(R9)(R10*1), R11 - ADDE R11, R6 - MOVD 16(R9)(R10*1), R11 - ADDE R11, R7 - MOVD 24(R9)(R10*1), R11 - ADDE R11, R1 - MOVD R0, R4 - ADDE R4, R4 // save CF - NEG R4, R4 - MOVD R5, 0(R2)(R10*1) - MOVD R6, 8(R2)(R10*1) - MOVD R7, 16(R2)(R10*1) - MOVD R1, 24(R2)(R10*1) - - ADD $32, R10 // i += 4 - SUB $4, R3 // n -= 4 - BGE U1n // if n >= 0 goto U1n - -v1n: - ADD $4, R3 // n += 4 - BLE E1n // if n <= 0 goto E1n - -L1n: // n > 0 - ADDC R4, R4 // restore CF - MOVD 0(R8)(R10*1), R5 - MOVD 0(R9)(R10*1), R11 - ADDE R11, R5 - MOVD R5, 0(R2)(R10*1) - MOVD R0, R4 - ADDE R4, R4 // save CF - NEG R4, R4 - - ADD $8, R10 // i++ - SUB $1, R3 // n-- - BGT L1n // if n > 0 goto L1n - -E1n: - NEG R4, R4 - MOVD R4, c+72(FP) // return c - RET - -TEXT ·subVV(SB), NOSPLIT, $0 - MOVD subvectorfacility+0x00(SB), R1 - BR (R1) - -TEXT ·subVV_check(SB), NOSPLIT, $0 - MOVB ·hasVX(SB), R1 - CMPBEQ R1, $1, vectorimpl // vectorfacility = 1, vector supported - MOVD $subvectorfacility+0x00(SB), R1 - MOVD $·subVV_novec(SB), R2 - MOVD R2, 0(R1) - - // MOVD $·subVV_novec(SB), 0(R1) - BR ·subVV_novec(SB) - -vectorimpl: - MOVD $subvectorfacility+0x00(SB), R1 - MOVD $·subVV_vec(SB), R2 - MOVD R2, 0(R1) - - // MOVD $·subVV_vec(SB), 0(R1) - BR ·subVV_vec(SB) - -GLOBL subvectorfacility+0x00(SB), NOPTR, $8 -DATA subvectorfacility+0x00(SB)/8, $·subVV_check(SB) - -// DI = R3, CX = R4, SI = r10, r8 = r8, r9=r9, r10 = r2, r11 = r5, r12 = r6, r13 = r7, r14 = r1 (R0 set to 0) + use R11 -// func subVV(z, x, y []Word) (c Word) -// (same as addVV except for SUBC/SUBE instead of ADDC/ADDE and label names) -TEXT ·subVV_vec(SB), NOSPLIT, $0 - MOVD z_len+8(FP), R3 - MOVD x+24(FP), R8 - MOVD y+48(FP), R9 - MOVD z+0(FP), R2 - MOVD $0, R4 // c = 0 - MOVD $0, R0 // make sure it's zero - MOVD $0, R10 // i = 0 - - // s/JL/JMP/ below to disable the unrolled loop - SUB $4, R3 // n -= 4 - BLT v1 // if n < 0 goto v1 - SUB $12, R3 // n -= 16 - BLT A1 // if n < 0 goto A1 - - MOVD R8, R5 - MOVD R9, R6 - MOVD R2, R7 - - // n >= 0 - // regular loop body unrolled 16x - VZERO V0 // cf = 0 - MOVD $1, R4 // for 390 subtraction cf starts as 1 (no borrow) - VLVGG $1, R4, V0 // put carry into V0 - -UU1: - VLM 0(R5), V1, V4 // 64-bytes into V1..V8 - ADD $64, R5 - VPDI $0x4, V1, V1, V1 // flip the doublewords to big-endian order - VPDI $0x4, V2, V2, V2 // flip the doublewords to big-endian order - - VLM 0(R6), V9, V12 // 64-bytes into V9..V16 - ADD $64, R6 - VPDI $0x4, V9, V9, V9 // flip the doublewords to big-endian order - VPDI $0x4, V10, V10, V10 // flip the doublewords to big-endian order - - VSBCBIQ V1, V9, V0, V25 - VSBIQ V1, V9, V0, V17 - VSBCBIQ V2, V10, V25, V26 - VSBIQ V2, V10, V25, V18 - - VLM 0(R5), V5, V6 // 32-bytes into V1..V8 - VLM 0(R6), V13, V14 // 32-bytes into V9..V16 - ADD $32, R5 - ADD $32, R6 - - VPDI $0x4, V3, V3, V3 // flip the doublewords to big-endian order - VPDI $0x4, V4, V4, V4 // flip the doublewords to big-endian order - VPDI $0x4, V11, V11, V11 // flip the doublewords to big-endian order - VPDI $0x4, V12, V12, V12 // flip the doublewords to big-endian order - - VSBCBIQ V3, V11, V26, V27 - VSBIQ V3, V11, V26, V19 - VSBCBIQ V4, V12, V27, V28 - VSBIQ V4, V12, V27, V20 - - VLM 0(R5), V7, V8 // 32-bytes into V1..V8 - VLM 0(R6), V15, V16 // 32-bytes into V9..V16 - ADD $32, R5 - ADD $32, R6 - - VPDI $0x4, V5, V5, V5 // flip the doublewords to big-endian order - VPDI $0x4, V6, V6, V6 // flip the doublewords to big-endian order - VPDI $0x4, V13, V13, V13 // flip the doublewords to big-endian order - VPDI $0x4, V14, V14, V14 // flip the doublewords to big-endian order - - VSBCBIQ V5, V13, V28, V29 - VSBIQ V5, V13, V28, V21 - VSBCBIQ V6, V14, V29, V30 - VSBIQ V6, V14, V29, V22 - - VPDI $0x4, V7, V7, V7 // flip the doublewords to big-endian order - VPDI $0x4, V8, V8, V8 // flip the doublewords to big-endian order - VPDI $0x4, V15, V15, V15 // flip the doublewords to big-endian order - VPDI $0x4, V16, V16, V16 // flip the doublewords to big-endian order - - VSBCBIQ V7, V15, V30, V31 - VSBIQ V7, V15, V30, V23 - VSBCBIQ V8, V16, V31, V0 // V0 has carry-over - VSBIQ V8, V16, V31, V24 - - VPDI $0x4, V17, V17, V17 // flip the doublewords to big-endian order - VPDI $0x4, V18, V18, V18 // flip the doublewords to big-endian order - VPDI $0x4, V19, V19, V19 // flip the doublewords to big-endian order - VPDI $0x4, V20, V20, V20 // flip the doublewords to big-endian order - VPDI $0x4, V21, V21, V21 // flip the doublewords to big-endian order - VPDI $0x4, V22, V22, V22 // flip the doublewords to big-endian order - VPDI $0x4, V23, V23, V23 // flip the doublewords to big-endian order - VPDI $0x4, V24, V24, V24 // flip the doublewords to big-endian order - VSTM V17, V24, 0(R7) // 128-bytes into z - ADD $128, R7 - ADD $128, R10 // i += 16 - SUB $16, R3 // n -= 16 - BGE UU1 // if n >= 0 goto U1 - VLGVG $1, V0, R4 // put cf into R4 - SUB $1, R4 // save cf - -A1: - ADD $12, R3 // n += 16 - BLT v1 // if n < 0 goto v1 - -U1: // n >= 0 - // regular loop body unrolled 4x - MOVD 0(R8)(R10*1), R5 - MOVD 8(R8)(R10*1), R6 - MOVD 16(R8)(R10*1), R7 - MOVD 24(R8)(R10*1), R1 - MOVD R0, R11 - SUBC R4, R11 // restore CF - MOVD 0(R9)(R10*1), R11 - SUBE R11, R5 - MOVD 8(R9)(R10*1), R11 - SUBE R11, R6 - MOVD 16(R9)(R10*1), R11 - SUBE R11, R7 - MOVD 24(R9)(R10*1), R11 - SUBE R11, R1 - MOVD R0, R4 - SUBE R4, R4 // save CF - MOVD R5, 0(R2)(R10*1) - MOVD R6, 8(R2)(R10*1) - MOVD R7, 16(R2)(R10*1) - MOVD R1, 24(R2)(R10*1) - - ADD $32, R10 // i += 4 - SUB $4, R3 // n -= 4 - BGE U1 // if n >= 0 goto U1n - -v1: - ADD $4, R3 // n += 4 - BLE E1 // if n <= 0 goto E1 - -L1: // n > 0 - MOVD R0, R11 - SUBC R4, R11 // restore CF - MOVD 0(R8)(R10*1), R5 - MOVD 0(R9)(R10*1), R11 - SUBE R11, R5 - MOVD R5, 0(R2)(R10*1) - MOVD R0, R4 - SUBE R4, R4 // save CF - - ADD $8, R10 // i++ - SUB $1, R3 // n-- - BGT L1 // if n > 0 goto L1n - -E1: - NEG R4, R4 - MOVD R4, c+72(FP) // return c + MOVD $0, R0 + MOVD z_len+8(FP), R1 + MOVD x_base+24(FP), R2 + MOVD y_base+48(FP), R3 + MOVD z_base+0(FP), R4 + // compute unrolled loop lengths + MOVD R1, R5 + AND $3, R5 + SRD $2, R1 + ADDC R0, R1 // clear carry +loop1: + CMPBEQ R5, $0, loop1done +loop1cont: + // unroll 1X + MOVD 0(R2), R6 + MOVD 0(R3), R7 + ADDE R7, R6 + MOVD R6, 0(R4) + LAY 8(R2), R2 // ADD $8, R2 + LAY 8(R3), R3 // ADD $8, R3 + LAY 8(R4), R4 // ADD $8, R4 + LAY -1(R5), R5 // ADD $-1, R5 + CMPBNE R5, $0, loop1cont +loop1done: +loop4: + CMPBEQ R1, $0, loop4done +loop4cont: + // unroll 4X in batches of 2 + MOVD 0(R2), R5 + MOVD 8(R2), R6 + MOVD 0(R3), R7 + MOVD 8(R3), R8 + ADDE R7, R5 + ADDE R8, R6 + MOVD R5, 0(R4) + MOVD R6, 8(R4) + MOVD 16(R2), R5 + MOVD 24(R2), R6 + MOVD 16(R3), R7 + MOVD 24(R3), R8 + ADDE R7, R5 + ADDE R8, R6 + MOVD R5, 16(R4) + MOVD R6, 24(R4) + LAY 32(R2), R2 // ADD $32, R2 + LAY 32(R3), R3 // ADD $32, R3 + LAY 32(R4), R4 // ADD $32, R4 + LAY -1(R1), R1 // ADD $-1, R1 + CMPBNE R1, $0, loop4cont +loop4done: + ADDE R0, R0, R2 // save & convert add carry + MOVD R2, c+72(FP) RET -// DI = R3, CX = R4, SI = r10, r8 = r8, r9=r9, r10 = r2, r11 = r5, r12 = r6, r13 = r7, r14 = r1 (R0 set to 0) + use R11 // func subVV(z, x, y []Word) (c Word) -// (same as addVV except for SUBC/SUBE instead of ADDC/ADDE and label names) -TEXT ·subVV_novec(SB), NOSPLIT, $0 - MOVD z_len+8(FP), R3 - MOVD x+24(FP), R8 - MOVD y+48(FP), R9 - MOVD z+0(FP), R2 - - MOVD $0, R4 // c = 0 - MOVD $0, R0 // make sure it's zero - MOVD $0, R10 // i = 0 - - // s/JL/JMP/ below to disable the unrolled loop - SUB $4, R3 // n -= 4 - BLT v1 // if n < 0 goto v1 - -U1: // n >= 0 - // regular loop body unrolled 4x - MOVD 0(R8)(R10*1), R5 - MOVD 8(R8)(R10*1), R6 - MOVD 16(R8)(R10*1), R7 - MOVD 24(R8)(R10*1), R1 - MOVD R0, R11 - SUBC R4, R11 // restore CF - MOVD 0(R9)(R10*1), R11 - SUBE R11, R5 - MOVD 8(R9)(R10*1), R11 - SUBE R11, R6 - MOVD 16(R9)(R10*1), R11 - SUBE R11, R7 - MOVD 24(R9)(R10*1), R11 - SUBE R11, R1 - MOVD R0, R4 - SUBE R4, R4 // save CF - MOVD R5, 0(R2)(R10*1) - MOVD R6, 8(R2)(R10*1) - MOVD R7, 16(R2)(R10*1) - MOVD R1, 24(R2)(R10*1) - - ADD $32, R10 // i += 4 - SUB $4, R3 // n -= 4 - BGE U1 // if n >= 0 goto U1 - -v1: - ADD $4, R3 // n += 4 - BLE E1 // if n <= 0 goto E1 - -L1: // n > 0 - MOVD R0, R11 - SUBC R4, R11 // restore CF - MOVD 0(R8)(R10*1), R5 - MOVD 0(R9)(R10*1), R11 - SUBE R11, R5 - MOVD R5, 0(R2)(R10*1) - MOVD R0, R4 - SUBE R4, R4 // save CF - - ADD $8, R10 // i++ - SUB $1, R3 // n-- - BGT L1 // if n > 0 goto L1 - -E1: - NEG R4, R4 - MOVD R4, c+72(FP) // return c - RET - -TEXT ·addVW(SB), NOSPLIT, $0 - MOVD z_len+8(FP), R5 // length of z - MOVD x+24(FP), R6 - MOVD y+48(FP), R7 // c = y - MOVD z+0(FP), R8 - - CMPBEQ R5, $0, returnC // if len(z) == 0, we can have an early return - - // Add the first two words, and determine which path (copy path or loop path) to take based on the carry flag. - ADDC 0(R6), R7 - MOVD R7, 0(R8) - CMPBEQ R5, $1, returnResult // len(z) == 1 - MOVD $0, R9 - ADDE 8(R6), R9 - MOVD R9, 8(R8) - CMPBEQ R5, $2, returnResult // len(z) == 2 - - // Update the counters - MOVD $16, R12 // i = 2 - MOVD $-2(R5), R5 // n = n - 2 - -loopOverEachWord: - BRC $12, copySetup // carry = 0, copy the rest - MOVD $1, R9 - - // Originally we used the carry flag generated in the previous iteration - // (i.e: ADDE could be used here to do the addition). However, since we - // already know carry is 1 (otherwise we will go to copy section), we can use - // ADDC here so the current iteration does not depend on the carry flag - // generated in the previous iteration. This could be useful when branch prediction happens. - ADDC 0(R6)(R12*1), R9 - MOVD R9, 0(R8)(R12*1) // z[i] = x[i] + c - - MOVD $8(R12), R12 // i++ - BRCTG R5, loopOverEachWord // n-- - -// Return the current carry value -returnResult: +TEXT ·subVV(SB), NOSPLIT, $0 + MOVB ·hasVX(SB), R1 + CMPBEQ R1, $0, novec + JMP ·subVVvec(SB) +novec: MOVD $0, R0 - ADDE R0, R0 - MOVD R0, c+56(FP) - RET - -// Update position of x(R6) and z(R8) based on the current counter value and perform copying. -// With the assumption that x and z will not overlap with each other or x and z will -// point to same memory region, we can use a faster version of copy using only MVC here. -// In the following implementation, we have three copy loops, each copying a word, 4 words, and -// 32 words at a time. Via benchmarking, this implementation is faster than calling runtime·memmove. -copySetup: - ADD R12, R6 - ADD R12, R8 - - CMPBGE R5, $4, mediumLoop - -smallLoop: // does a loop unrolling to copy word when n < 4 - CMPBEQ R5, $0, returnZero - MVC $8, 0(R6), 0(R8) - CMPBEQ R5, $1, returnZero - MVC $8, 8(R6), 8(R8) - CMPBEQ R5, $2, returnZero - MVC $8, 16(R6), 16(R8) - -returnZero: - MOVD $0, c+56(FP) // return 0 as carry + MOVD z_len+8(FP), R1 + MOVD x_base+24(FP), R2 + MOVD y_base+48(FP), R3 + MOVD z_base+0(FP), R4 + // compute unrolled loop lengths + MOVD R1, R5 + AND $3, R5 + SRD $2, R1 + SUBC R0, R1 // clear carry +loop1: + CMPBEQ R5, $0, loop1done +loop1cont: + // unroll 1X + MOVD 0(R2), R6 + MOVD 0(R3), R7 + SUBE R7, R6 + MOVD R6, 0(R4) + LAY 8(R2), R2 // ADD $8, R2 + LAY 8(R3), R3 // ADD $8, R3 + LAY 8(R4), R4 // ADD $8, R4 + LAY -1(R5), R5 // ADD $-1, R5 + CMPBNE R5, $0, loop1cont +loop1done: +loop4: + CMPBEQ R1, $0, loop4done +loop4cont: + // unroll 4X in batches of 2 + MOVD 0(R2), R5 + MOVD 8(R2), R6 + MOVD 0(R3), R7 + MOVD 8(R3), R8 + SUBE R7, R5 + SUBE R8, R6 + MOVD R5, 0(R4) + MOVD R6, 8(R4) + MOVD 16(R2), R5 + MOVD 24(R2), R6 + MOVD 16(R3), R7 + MOVD 24(R3), R8 + SUBE R7, R5 + SUBE R8, R6 + MOVD R5, 16(R4) + MOVD R6, 24(R4) + LAY 32(R2), R2 // ADD $32, R2 + LAY 32(R3), R3 // ADD $32, R3 + LAY 32(R4), R4 // ADD $32, R4 + LAY -1(R1), R1 // ADD $-1, R1 + CMPBNE R1, $0, loop4cont +loop4done: + SUBE R2, R2 // save carry + NEG R2 // convert sub carry + MOVD R2, c+72(FP) RET -mediumLoop: - CMPBLT R5, $4, smallLoop - CMPBLT R5, $32, mediumLoopBody - -largeLoop: // Copying 256 bytes at a time. - MVC $256, 0(R6), 0(R8) - MOVD $256(R6), R6 - MOVD $256(R8), R8 - MOVD $-32(R5), R5 - CMPBGE R5, $32, largeLoop - BR mediumLoop - -mediumLoopBody: // Copying 32 bytes at a time - MVC $32, 0(R6), 0(R8) - MOVD $32(R6), R6 - MOVD $32(R8), R8 - MOVD $-4(R5), R5 - CMPBGE R5, $4, mediumLoopBody - BR smallLoop - -returnC: +// func lshVU(z, x []Word, s uint) (c Word) +TEXT ·lshVU(SB), NOSPLIT, $0 + MOVD $0, R0 + MOVD z_len+8(FP), R1 + CMPBEQ R1, $0, ret0 + MOVD s+48(FP), R2 + MOVD x_base+24(FP), R3 + MOVD z_base+0(FP), R4 + // run loop backward + SLD $3, R1, R5 + LAY (R5)(R3), R3 // ADD R5, R3 + SLD $3, R1, R5 + LAY (R5)(R4), R4 // ADD R5, R4 + // shift first word into carry + MOVD -8(R3), R5 + MOVD $64, R6 + SUBC R2, R6 + SRD R6, R5, R7 + SLD R2, R5 MOVD R7, c+56(FP) + // shift remaining words + SUBC $1, R1 + // compute unrolled loop lengths + MOVD R1, R7 + AND $3, R7 + SRD $2, R1 +loop1: + CMPBEQ R7, $0, loop1done +loop1cont: + // unroll 1X + MOVD -16(R3), R8 + SRD R6, R8, R9 + OR R5, R9 + SLD R2, R8, R5 + MOVD R9, -8(R4) + LAY -8(R3), R3 // ADD $-8, R3 + LAY -8(R4), R4 // ADD $-8, R4 + LAY -1(R7), R7 // ADD $-1, R7 + CMPBNE R7, $0, loop1cont +loop1done: +loop4: + CMPBEQ R1, $0, loop4done +loop4cont: + // unroll 4X in batches of 2 + MOVD -16(R3), R7 + MOVD -24(R3), R8 + SRD R6, R7, R9 + OR R5, R9 + SLD R2, R7, R5 + SRD R6, R8, R7 + OR R5, R7 + SLD R2, R8, R5 + MOVD R9, -8(R4) + MOVD R7, -16(R4) + MOVD -32(R3), R7 + MOVD -40(R3), R8 + SRD R6, R7, R9 + OR R5, R9 + SLD R2, R7, R5 + SRD R6, R8, R7 + OR R5, R7 + SLD R2, R8, R5 + MOVD R9, -24(R4) + MOVD R7, -32(R4) + LAY -32(R3), R3 // ADD $-32, R3 + LAY -32(R4), R4 // ADD $-32, R4 + LAY -1(R1), R1 // ADD $-1, R1 + CMPBNE R1, $0, loop4cont +loop4done: + // store final shifted bits + MOVD R5, -8(R4) RET - -TEXT ·subVW(SB), NOSPLIT, $0 - MOVD z_len+8(FP), R5 - MOVD x+24(FP), R6 - MOVD y+48(FP), R7 // The borrow bit passed in - MOVD z+0(FP), R8 - MOVD $0, R0 // R0 is a temporary variable used during computation. Ensure it has zero in it. - - CMPBEQ R5, $0, returnC // len(z) == 0, have an early return - - // Subtract the first two words, and determine which path (copy path or loop path) to take based on the borrow flag - MOVD 0(R6), R9 - SUBC R7, R9 - MOVD R9, 0(R8) - CMPBEQ R5, $1, returnResult - MOVD 8(R6), R9 - SUBE R0, R9 - MOVD R9, 8(R8) - CMPBEQ R5, $2, returnResult - - // Update the counters - MOVD $16, R12 // i = 2 - MOVD $-2(R5), R5 // n = n - 2 - -loopOverEachWord: - BRC $3, copySetup // no borrow, copy the rest - MOVD 0(R6)(R12*1), R9 - - // Originally we used the borrow flag generated in the previous iteration - // (i.e: SUBE could be used here to do the subtraction). However, since we - // already know borrow is 1 (otherwise we will go to copy section), we can - // use SUBC here so the current iteration does not depend on the borrow flag - // generated in the previous iteration. This could be useful when branch prediction happens. - SUBC $1, R9 - MOVD R9, 0(R8)(R12*1) // z[i] = x[i] - 1 - - MOVD $8(R12), R12 // i++ - BRCTG R5, loopOverEachWord // n-- - -// return the current borrow value -returnResult: - SUBE R0, R0 - NEG R0, R0 +ret0: MOVD R0, c+56(FP) RET -// Update position of x(R6) and z(R8) based on the current counter value and perform copying. -// With the assumption that x and z will not overlap with each other or x and z will -// point to same memory region, we can use a faster version of copy using only MVC here. -// In the following implementation, we have three copy loops, each copying a word, 4 words, and -// 32 words at a time. Via benchmarking, this implementation is faster than calling runtime·memmove. -copySetup: - ADD R12, R6 - ADD R12, R8 - - CMPBGE R5, $4, mediumLoop - -smallLoop: // does a loop unrolling to copy word when n < 4 - CMPBEQ R5, $0, returnZero - MVC $8, 0(R6), 0(R8) - CMPBEQ R5, $1, returnZero - MVC $8, 8(R6), 8(R8) - CMPBEQ R5, $2, returnZero - MVC $8, 16(R6), 16(R8) - -returnZero: - MOVD $0, c+56(FP) // return 0 as borrow - RET - -mediumLoop: - CMPBLT R5, $4, smallLoop - CMPBLT R5, $32, mediumLoopBody - -largeLoop: // Copying 256 bytes at a time - MVC $256, 0(R6), 0(R8) - MOVD $256(R6), R6 - MOVD $256(R8), R8 - MOVD $-32(R5), R5 - CMPBGE R5, $32, largeLoop - BR mediumLoop - -mediumLoopBody: // Copying 32 bytes at a time - MVC $32, 0(R6), 0(R8) - MOVD $32(R6), R6 - MOVD $32(R8), R8 - MOVD $-4(R5), R5 - CMPBGE R5, $4, mediumLoopBody - BR smallLoop - -returnC: +// func rshVU(z, x []Word, s uint) (c Word) +TEXT ·rshVU(SB), NOSPLIT, $0 + MOVD $0, R0 + MOVD z_len+8(FP), R1 + CMPBEQ R1, $0, ret0 + MOVD s+48(FP), R2 + MOVD x_base+24(FP), R3 + MOVD z_base+0(FP), R4 + // shift first word into carry + MOVD 0(R3), R5 + MOVD $64, R6 + SUBC R2, R6 + SLD R6, R5, R7 + SRD R2, R5 MOVD R7, c+56(FP) + // shift remaining words + SUBC $1, R1 + // compute unrolled loop lengths + MOVD R1, R7 + AND $3, R7 + SRD $2, R1 +loop1: + CMPBEQ R7, $0, loop1done +loop1cont: + // unroll 1X + MOVD 8(R3), R8 + SLD R6, R8, R9 + OR R5, R9 + SRD R2, R8, R5 + MOVD R9, 0(R4) + LAY 8(R3), R3 // ADD $8, R3 + LAY 8(R4), R4 // ADD $8, R4 + LAY -1(R7), R7 // ADD $-1, R7 + CMPBNE R7, $0, loop1cont +loop1done: +loop4: + CMPBEQ R1, $0, loop4done +loop4cont: + // unroll 4X in batches of 2 + MOVD 8(R3), R7 + MOVD 16(R3), R8 + SLD R6, R7, R9 + OR R5, R9 + SRD R2, R7, R5 + SLD R6, R8, R7 + OR R5, R7 + SRD R2, R8, R5 + MOVD R9, 0(R4) + MOVD R7, 8(R4) + MOVD 24(R3), R7 + MOVD 32(R3), R8 + SLD R6, R7, R9 + OR R5, R9 + SRD R2, R7, R5 + SLD R6, R8, R7 + OR R5, R7 + SRD R2, R8, R5 + MOVD R9, 16(R4) + MOVD R7, 24(R4) + LAY 32(R3), R3 // ADD $32, R3 + LAY 32(R4), R4 // ADD $32, R4 + LAY -1(R1), R1 // ADD $-1, R1 + CMPBNE R1, $0, loop4cont +loop4done: + // store final shifted bits + MOVD R5, 0(R4) + RET +ret0: + MOVD R0, c+56(FP) RET -// func shlVU(z, x []Word, s uint) (c Word) -TEXT ·shlVU(SB), NOSPLIT, $0 - BR ·shlVU_g(SB) - -// func shrVU(z, x []Word, s uint) (c Word) -TEXT ·shrVU(SB), NOSPLIT, $0 - BR ·shrVU_g(SB) - -// CX = R4, r8 = r8, r9=r9, r10 = r2, r11 = r5, DX = r3, AX = r6, BX = R1, (R0 set to 0) + use R11 + use R7 for i -// func mulAddVWW(z, x []Word, y, r Word) (c Word) +// func mulAddVWW(z, x []Word, m, a Word) (c Word) TEXT ·mulAddVWW(SB), NOSPLIT, $0 - MOVD z+0(FP), R2 - MOVD x+24(FP), R8 - MOVD y+48(FP), R9 - MOVD r+56(FP), R4 // c = r - MOVD z_len+8(FP), R5 - MOVD $0, R1 // i = 0 - MOVD $0, R7 // i*8 = 0 - MOVD $0, R0 // make sure it's zero - BR E5 - -L5: - MOVD (R8)(R1*1), R6 - MULHDU R9, R6 - ADDC R4, R11 // add to low order bits - ADDE R0, R6 - MOVD R11, (R2)(R1*1) - MOVD R6, R4 - ADD $8, R1 // i*8 + 8 - ADD $1, R7 // i++ - -E5: - CMPBLT R7, R5, L5 // i < n - - MOVD R4, c+64(FP) + MOVD $0, R0 + MOVD m+48(FP), R1 + MOVD a+56(FP), R2 + MOVD z_len+8(FP), R3 + MOVD x_base+24(FP), R4 + MOVD z_base+0(FP), R5 + // compute unrolled loop lengths + MOVD R3, R6 + AND $3, R6 + SRD $2, R3 +loop1: + CMPBEQ R6, $0, loop1done +loop1cont: + // unroll 1X in batches of 1 + MOVD 0(R4), R11 + // multiply + MLGR R1, R10 + ADDC R2, R11 + ADDE R0, R10, R2 + MOVD R11, 0(R5) + LAY 8(R4), R4 // ADD $8, R4 + LAY 8(R5), R5 // ADD $8, R5 + LAY -1(R6), R6 // ADD $-1, R6 + CMPBNE R6, $0, loop1cont +loop1done: +loop4: + CMPBEQ R3, $0, loop4done +loop4cont: + // unroll 4X in batches of 1 + MOVD 0(R4), R11 + // multiply + MLGR R1, R10 + ADDC R2, R11 + ADDE R0, R10, R2 + MOVD R11, 0(R5) + MOVD 8(R4), R11 + // multiply + MLGR R1, R10 + ADDC R2, R11 + ADDE R0, R10, R2 + MOVD R11, 8(R5) + MOVD 16(R4), R11 + // multiply + MLGR R1, R10 + ADDC R2, R11 + ADDE R0, R10, R2 + MOVD R11, 16(R5) + MOVD 24(R4), R11 + // multiply + MLGR R1, R10 + ADDC R2, R11 + ADDE R0, R10, R2 + MOVD R11, 24(R5) + LAY 32(R4), R4 // ADD $32, R4 + LAY 32(R5), R5 // ADD $32, R5 + LAY -1(R3), R3 // ADD $-1, R3 + CMPBNE R3, $0, loop4cont +loop4done: + MOVD R2, c+64(FP) RET -// func addMulVVW(z, x []Word, y Word) (c Word) -// CX = R4, r8 = r8, r9=r9, r10 = r2, r11 = r5, AX = r11, DX = R6, r12=r12, BX = R1, (R0 set to 0) + use R11 + use R7 for i -TEXT ·addMulVVW(SB), NOSPLIT, $0 - MOVD z+0(FP), R2 - MOVD x+24(FP), R8 - MOVD y+48(FP), R9 - MOVD z_len+8(FP), R5 - - MOVD $0, R1 // i*8 = 0 - MOVD $0, R7 // i = 0 - MOVD $0, R0 // make sure it's zero - MOVD $0, R4 // c = 0 - - MOVD R5, R12 - AND $-2, R12 - CMPBGE R5, $2, A6 - BR E6 - -A6: - MOVD (R8)(R1*1), R6 - MULHDU R9, R6 - MOVD (R2)(R1*1), R10 - ADDC R10, R11 // add to low order bits - ADDE R0, R6 - ADDC R4, R11 - ADDE R0, R6 - MOVD R6, R4 - MOVD R11, (R2)(R1*1) - - MOVD (8)(R8)(R1*1), R6 - MULHDU R9, R6 - MOVD (8)(R2)(R1*1), R10 - ADDC R10, R11 // add to low order bits - ADDE R0, R6 - ADDC R4, R11 - ADDE R0, R6 - MOVD R6, R4 - MOVD R11, (8)(R2)(R1*1) - - ADD $16, R1 // i*8 + 8 - ADD $2, R7 // i++ - - CMPBLT R7, R12, A6 - BR E6 - -L6: - MOVD (R8)(R1*1), R6 - MULHDU R9, R6 - MOVD (R2)(R1*1), R10 - ADDC R10, R11 // add to low order bits - ADDE R0, R6 - ADDC R4, R11 - ADDE R0, R6 - MOVD R6, R4 - MOVD R11, (R2)(R1*1) - - ADD $8, R1 // i*8 + 8 - ADD $1, R7 // i++ - -E6: - CMPBLT R7, R5, L6 // i < n - - MOVD R4, c+56(FP) +// func addMulVVWW(z, x, y []Word, m, a Word) (c Word) +TEXT ·addMulVVWW(SB), NOSPLIT, $0 + MOVD $0, R0 + MOVD m+72(FP), R1 + MOVD a+80(FP), R2 + MOVD z_len+8(FP), R3 + MOVD x_base+24(FP), R4 + MOVD y_base+48(FP), R5 + MOVD z_base+0(FP), R6 + // compute unrolled loop lengths + MOVD R3, R7 + AND $3, R7 + SRD $2, R3 +loop1: + CMPBEQ R7, $0, loop1done +loop1cont: + // unroll 1X in batches of 1 + MOVD 0(R4), R8 + MOVD 0(R5), R11 + // multiply + MLGR R1, R10 + ADDC R2, R11 + ADDE R0, R10, R2 + // add + ADDC R8, R11 + ADDE R0, R2 + MOVD R11, 0(R6) + LAY 8(R4), R4 // ADD $8, R4 + LAY 8(R5), R5 // ADD $8, R5 + LAY 8(R6), R6 // ADD $8, R6 + LAY -1(R7), R7 // ADD $-1, R7 + CMPBNE R7, $0, loop1cont +loop1done: +loop4: + CMPBEQ R3, $0, loop4done +loop4cont: + // unroll 4X in batches of 1 + MOVD 0(R4), R7 + MOVD 0(R5), R11 + // multiply + MLGR R1, R10 + ADDC R2, R11 + ADDE R0, R10, R2 + // add + ADDC R7, R11 + ADDE R0, R2 + MOVD R11, 0(R6) + MOVD 8(R4), R7 + MOVD 8(R5), R11 + // multiply + MLGR R1, R10 + ADDC R2, R11 + ADDE R0, R10, R2 + // add + ADDC R7, R11 + ADDE R0, R2 + MOVD R11, 8(R6) + MOVD 16(R4), R7 + MOVD 16(R5), R11 + // multiply + MLGR R1, R10 + ADDC R2, R11 + ADDE R0, R10, R2 + // add + ADDC R7, R11 + ADDE R0, R2 + MOVD R11, 16(R6) + MOVD 24(R4), R7 + MOVD 24(R5), R11 + // multiply + MLGR R1, R10 + ADDC R2, R11 + ADDE R0, R10, R2 + // add + ADDC R7, R11 + ADDE R0, R2 + MOVD R11, 24(R6) + LAY 32(R4), R4 // ADD $32, R4 + LAY 32(R5), R5 // ADD $32, R5 + LAY 32(R6), R6 // ADD $32, R6 + LAY -1(R3), R3 // ADD $-1, R3 + CMPBNE R3, $0, loop4cont +loop4done: + MOVD R2, c+88(FP) RET - diff --git a/src/math/big/arith_s390x_test.go b/src/math/big/arith_s390x_test.go index 0b91cc1393fb28..9db2fd4101a175 100644 --- a/src/math/big/arith_s390x_test.go +++ b/src/math/big/arith_s390x_test.go @@ -2,31 +2,18 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build s390x && !math_big_pure_go +//go:build !math_big_pure_go package big -import ( - "testing" -) +import "testing" -// Tests whether the non vector routines are working, even when the tests are run on a -// vector-capable machine - -func TestFunVVnovec(t *testing.T) { - if hasVX { - for _, a := range sumVV { - arg := a - testFunVV(t, "addVV_novec", addVV_novec, arg) - - arg = argVV{a.z, a.y, a.x, a.c} - testFunVV(t, "addVV_novec symmetric", addVV_novec, arg) - - arg = argVV{a.x, a.z, a.y, a.c} - testFunVV(t, "subVV_novec", subVV_novec, arg) +func TestAddVVNoVec(t *testing.T) { + setDuringTest(t, &hasVX, false) + TestAddVV(t) +} - arg = argVV{a.y, a.z, a.x, a.c} - testFunVV(t, "subVV_novec symmetric", subVV_novec, arg) - } - } +func TestSubVVNoVec(t *testing.T) { + setDuringTest(t, &hasVX, false) + TestSubVV(t) } diff --git a/src/math/big/arith_test.go b/src/math/big/arith_test.go index 64225bbd53661c..bd9f96870b1d6d 100644 --- a/src/math/big/arith_test.go +++ b/src/math/big/arith_test.go @@ -2,233 +2,484 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// This file defines tests of consistent behavior between assembly and Go versions of basic operators, +// as well as tests of pure Go implementations. + package big import ( "fmt" "internal/testenv" + "iter" "math/bits" - "math/rand" + "math/rand/v2" + "slices" "strings" "testing" ) var isRaceBuilder = strings.HasSuffix(testenv.Builder(), "-race") -type funVV func(z, x, y []Word) (c Word) -type argVV struct { - z, x, y nat - c Word -} - -var sumVV = []argVV{ - {}, - {nat{0}, nat{0}, nat{0}, 0}, - {nat{1}, nat{1}, nat{0}, 0}, - {nat{0}, nat{_M}, nat{1}, 1}, - {nat{80235}, nat{12345}, nat{67890}, 0}, - {nat{_M - 1}, nat{_M}, nat{_M}, 1}, - {nat{0, 0, 0, 0}, nat{_M, _M, _M, _M}, nat{1, 0, 0, 0}, 1}, - {nat{0, 0, 0, _M}, nat{_M, _M, _M, _M - 1}, nat{1, 0, 0, 0}, 0}, - {nat{0, 0, 0, 0}, nat{_M, 0, _M, 0}, nat{1, _M, 0, _M}, 1}, -} - -func testFunVV(t *testing.T, msg string, f funVV, a argVV) { - z := make(nat, len(a.z)) - c := f(z, a.x, a.y) - for i, zi := range z { - if zi != a.z[i] { - t.Errorf("%s%+v\n\tgot z[%d] = %#x; want %#x", msg, a, i, zi, a.z[i]) - break +var words4 = []Word{0, 1, _M - 1, _M} +var words2 = []Word{0, _M} +var muls = []Word{0, 1, 2, 3, 4, 5, _M / 4, _M / 2, _M - 3, _M - 2, _M - 1, _M} +var adds = []Word{0, 1, _M - 1, _M} +var shifts = []uint{1, 2, 3, _W/4 - 1, _W / 4, _W/4 + 1, _W/2 - 1, _W / 2, _W/2 + 1, _W - 3, _W - 2, _W - 1} + +func TestAddVV(t *testing.T) { testVV(t, "addVV", addVV, addVV_g) } +func TestSubVV(t *testing.T) { testVV(t, "subVV", subVV, subVV_g) } +func TestAddVW(t *testing.T) { testVW(t, "addVW", addVW, addVW_ref, words4) } +func TestSubVW(t *testing.T) { testVW(t, "subVW", subVW, subVW_ref, words4) } +func TestLshVU(t *testing.T) { testVU(t, "lshVU", lshVU, lshVU_g, shifts) } +func TestRshVU(t *testing.T) { testVU(t, "rshVU", rshVU, rshVU_g, shifts) } +func TestMulAddVWW(t *testing.T) { testVWW(t, "mulAddVWW", mulAddVWW, mulAddVWW_g, muls) } +func TestAddMulVVWW(t *testing.T) { testVVWW(t, "addMulVVWW", addMulVVWW, addMulVVWW_g, muls, adds) } + +// Note: It would be nice to avoid all the duplication of these test variants, +// but the only obvious way is to use reflection. These tests are already +// pretty expensive, and hitting them with reflect call overhead would +// reduce the amount of exhaustive testing it's reasonable to do, so instead +// we put up with the duplication. + +func testVV(t *testing.T, name string, fn, ref func(z, x, y []Word) (c Word)) { + for size := range 100 { + xx := make([]Word, 1+size+1) + yy := make([]Word, 1+size+1) + zz := make([]Word, 1+size+1) + words := words4 + if size > 5 { + words = words2 + } + if size > 10 { + words = nil // random + } + for x := range nats(words, size) { + for y := range nats(words, size) { + wantZ := make([]Word, size) + wantC := ref(wantZ, x, y) + + for _, inplace := range []bool{false, true} { + name := name + if inplace { + name = "in-place " + name + } + setSlice(xx, 1, x) + setSlice(yy, 2, y) + zz := zz + if inplace { + zz = xx + } else { + for i := range zz { + zz[i] = 0x9876 + } + } + setSlice(zz, 3, nil) + c := fn(zz[1:1+size], xx[1:1+size], yy[1:1+size]) + if !slices.Equal(zz[1:1+size], wantZ) || c != wantC { + t.Errorf("%s(%#x, %#x) = %#x, %#x, want %#x, %#x", name, x, y, zz[1:1+size], c, wantZ, wantC) + } + if !inplace { + checkSlice(t, name, "x", xx, 1, x) + } + checkSlice(t, name, "y", yy, 2, y) + checkSlice(t, name, "z", zz, 3, nil) + if t.Failed() { + t.FailNow() + } + } + } } - } - if c != a.c { - t.Errorf("%s%+v\n\tgot c = %#x; want %#x", msg, a, c, a.c) } } -func TestFunVV(t *testing.T) { - for _, a := range sumVV { - arg := a - testFunVV(t, "addVV_g", addVV_g, arg) - testFunVV(t, "addVV", addVV, arg) - - arg = argVV{a.z, a.y, a.x, a.c} - testFunVV(t, "addVV_g symmetric", addVV_g, arg) - testFunVV(t, "addVV symmetric", addVV, arg) - - arg = argVV{a.x, a.z, a.y, a.c} - testFunVV(t, "subVV_g", subVV_g, arg) - testFunVV(t, "subVV", subVV, arg) - - arg = argVV{a.y, a.z, a.x, a.c} - testFunVV(t, "subVV_g symmetric", subVV_g, arg) - testFunVV(t, "subVV symmetric", subVV, arg) +func testVV2(t *testing.T, name string, fn, ref func(z1, z2, x, y []Word) (c1, c2 Word)) { + for size := range 100 { + xx := make([]Word, 1+size+1) + yy := make([]Word, 1+size+1) + zz1 := make([]Word, 1+size+1) + zz2 := make([]Word, 1+size+1) + words := words4 + if size > 5 { + words = words2 + } + if size > 10 { + words = nil // random + } + for x := range nats(words, size) { + for y := range nats(words, size) { + wantZ1 := make([]Word, size) + wantZ2 := make([]Word, size) + wantC1, wantC2 := ref(wantZ1, wantZ2, x, y) + + for _, inplace := range []bool{false, true} { + name := name + if inplace { + name = "in-place " + name + } + setSlice(xx, 1, x) + setSlice(yy, 2, y) + zz1 := zz1 + zz2 := zz2 + if inplace { + zz1 = xx + zz2 = yy + } else { + for i := range zz1 { + zz1[i] = 0x9876 + } + for i := range zz2 { + zz2[i] = 0x8765 + } + } + setSlice(zz1, 3, nil) + setSlice(zz2, 4, nil) + c1, c2 := fn(zz1[1:1+size], zz2[1:1+size], xx[1:1+size], yy[1:1+size]) + if !slices.Equal(zz1[1:1+size], wantZ1) || !slices.Equal(zz2[1:1+size], wantZ2) || c1 != wantC1 || c2 != wantC2 { + t.Errorf("%s(%#x, %#x) = %#x, %#x, %#x, %#x, want %#x, %#x, %#x, %#x", name, x, y, zz1[1:1+size], zz2[1:1+size], c1, c2, wantZ1, wantZ2, wantC1, wantC2) + } + if !inplace { + checkSlice(t, name, "x", xx, 1, x) + checkSlice(t, name, "y", yy, 2, y) + } + checkSlice(t, name, "z1", zz1, 3, nil) + checkSlice(t, name, "z2", zz2, 4, nil) + if t.Failed() { + t.FailNow() + } + } + } + } } } -// Always the same seed for reproducible results. -var rnd = rand.New(rand.NewSource(0)) +func testVW(t *testing.T, name string, fn, ref func(z, x []Word, w Word) (c Word), ws []Word) { + const ( + magic0 = 0x123450 + magic1 = 0x543210 + ) -func rndW() Word { - return Word(rnd.Int63()<<1 | rnd.Int63n(2)) + for size := range 100 { + xx := make([]Word, 1+size+1) + zz := make([]Word, 1+size+1) + words := words4 + if size > 5 { + words = words2 + } + if size > 10 { + words = nil // random + } + for x := range nats(words, size) { + for _, w := range ws { + wantZ := make([]Word, size) + wantC := ref(wantZ, x, w) + + copy(xx[1:], x) + for _, inplace := range []bool{false, true} { + name := name + if inplace { + name = "in-place " + name + } + setSlice(xx, 1, x) + zz := zz + if inplace { + zz = xx + } else { + for i := range zz { + zz[i] = 0x9876 + } + } + setSlice(zz, 2, nil) + c := fn(zz[1:1+size], xx[1:1+size], w) + if !slices.Equal(zz[1:1+size], wantZ) || c != wantC { + t.Errorf("%s(%#x, %#x) = %#x, %#x, want %#x, %#x", name, x, w, zz[1:1+size], c, wantZ, wantC) + } + if !inplace { + checkSlice(t, name, "x", xx, 1, x) + } + checkSlice(t, name, "z", zz, 2, nil) + if t.Failed() { + t.FailNow() + } + } + } + } + } } -func rndV(n int) []Word { - v := make([]Word, n) - for i := range v { - v[i] = rndW() +func testVU(t *testing.T, name string, fn, ref func(z, x []Word, y uint) (c Word), ys []uint) { + wys := make([]Word, len(ys)) + for i, y := range ys { + wys[i] = Word(y) } - return v + testVW(t, name, + func(z, x []Word, y Word) Word { return fn(z, x, uint(y)) }, + func(z, x []Word, y Word) Word { return ref(z, x, uint(y)) }, + wys) } -var benchSizes = []int{1, 2, 3, 4, 5, 1e1, 1e2, 1e3, 1e4, 1e5} +func testVWW(t *testing.T, name string, fn, ref func(z, x []Word, y, r Word) (c Word), ys []Word) { + const ( + magic0 = 0x123450 + magic1 = 0x543210 + ) -func BenchmarkAddVV(b *testing.B) { - for _, n := range benchSizes { - if isRaceBuilder && n > 1e3 { - continue + for size := range 100 { + xx := make([]Word, 1+size+1) + zz := make([]Word, 1+size+1) + words := words4 + if size > 5 { + words = words2 } - x := rndV(n) - y := rndV(n) - z := make([]Word, n) - b.Run(fmt.Sprint(n), func(b *testing.B) { - b.SetBytes(int64(n * _W)) - for i := 0; i < b.N; i++ { - addVV(z, x, y) + if size > 10 { + words = nil // random + } + for x := range nats(words, size) { + for _, y := range ys { + for _, r := range ys { + wantZ := make([]Word, size) + wantC := ref(wantZ, x, y, r) + + copy(xx[1:], x) + for _, inplace := range []bool{false, true} { + name := name + if inplace { + name = "in-place " + name + } + setSlice(xx, 1, x) + zz := zz + if inplace { + zz = xx + } else { + for i := range zz { + zz[i] = 0x9876 + } + } + setSlice(zz, 2, nil) + c := fn(zz[1:1+size], xx[1:1+size], y, r) + if !slices.Equal(zz[1:1+size], wantZ) || c != wantC { + t.Errorf("%s(%#x, %#x, %#x) = %#x, %#x, want %#x, %#x", name, x, y, r, zz[1:1+size], c, wantZ, wantC) + } + if !inplace { + checkSlice(t, name, "x", xx, 1, x) + } + checkSlice(t, name, "z", zz, 2, nil) + if t.Failed() { + t.FailNow() + } + } + } } - }) + } } } -func BenchmarkSubVV(b *testing.B) { - for _, n := range benchSizes { - if isRaceBuilder && n > 1e3 { - continue +func testVVU(t *testing.T, name string, fn, ref func(z, x, y []Word, s uint) (c Word), shifts []uint) { + for size := range 100 { + xx := make([]Word, 1+size+1) + yy := make([]Word, 1+size+1) + zz := make([]Word, 1+size+1) + words := words4 + if size > 5 { + words = words2 } - x := rndV(n) - y := rndV(n) - z := make([]Word, n) - b.Run(fmt.Sprint(n), func(b *testing.B) { - b.SetBytes(int64(n * _W)) - for i := 0; i < b.N; i++ { - subVV(z, x, y) + if size > 10 { + words = nil // random + } + for x := range nats(words, size) { + for y := range nats(words, size) { + for _, s := range shifts { + wantZ := make([]Word, size) + wantC := ref(wantZ, x, y, s) + + for _, inplace := range []bool{false, true} { + name := name + if inplace { + name = "in-place " + name + } + setSlice(xx, 1, x) + setSlice(yy, 2, y) + zz := zz + if inplace { + zz = xx + } else { + for i := range zz { + zz[i] = 0x9876 + } + } + setSlice(zz, 3, nil) + c := fn(zz[1:1+size], xx[1:1+size], yy[1:1+size], s) + if !slices.Equal(zz[1:1+size], wantZ) || c != wantC { + t.Errorf("%s(%#x, %#x, %#x) = %#x, %#x, want %#x, %#x", name, x, y, s, zz[1:1+size], c, wantZ, wantC) + } + if !inplace { + checkSlice(t, name, "x", xx, 1, x) + } + checkSlice(t, name, "y", yy, 2, y) + checkSlice(t, name, "z", zz, 3, nil) + if t.Failed() { + t.FailNow() + } + } + } } - }) + } } } -type funVW func(z, x []Word, y Word) (c Word) -type argVW struct { - z, x nat - y Word - c Word -} - -var sumVW = []argVW{ - {}, - {nil, nil, 2, 2}, - {nat{0}, nat{0}, 0, 0}, - {nat{1}, nat{0}, 1, 0}, - {nat{1}, nat{1}, 0, 0}, - {nat{0}, nat{_M}, 1, 1}, - {nat{0, 0, 0, 0}, nat{_M, _M, _M, _M}, 1, 1}, - {nat{585}, nat{314}, 271, 0}, -} - -var lshVW = []argVW{ - {}, - {nat{0}, nat{0}, 0, 0}, - {nat{0}, nat{0}, 1, 0}, - {nat{0}, nat{0}, 20, 0}, - - {nat{_M}, nat{_M}, 0, 0}, - {nat{_M << 1 & _M}, nat{_M}, 1, 1}, - {nat{_M << 20 & _M}, nat{_M}, 20, _M >> (_W - 20)}, - - {nat{_M, _M, _M}, nat{_M, _M, _M}, 0, 0}, - {nat{_M << 1 & _M, _M, _M}, nat{_M, _M, _M}, 1, 1}, - {nat{_M << 20 & _M, _M, _M}, nat{_M, _M, _M}, 20, _M >> (_W - 20)}, +func testVVWW(t *testing.T, name string, fn, ref func(z, x, y []Word, m, a Word) (c Word), ms, as []Word) { + for size := range 100 { + zz := make([]Word, 1+size+1) + xx := make([]Word, 1+size+1) + yy := make([]Word, 1+size+1) + words := words4 + if size > 3 { + words = words2 + } + if size > 7 { + words = nil // random + } + for x := range nats(words, size) { + for y := range nats(words, size) { + for _, m := range ms { + for _, a := range as { + wantZ := make([]Word, size) + wantC := ref(wantZ, x, y, m, a) + + for _, inplace := range []bool{false, true} { + name := name + if inplace { + name = "in-place " + name + } + setSlice(xx, 1, x) + setSlice(yy, 2, y) + zz := zz + if inplace { + zz = xx + } else { + for i := range zz { + zz[i] = 0x9876 + } + } + setSlice(zz, 3, nil) + c := fn(zz[1:1+size], xx[1:1+size], yy[1:1+size], m, a) + if !slices.Equal(zz[1:1+size], wantZ) || c != wantC { + t.Errorf("%s(%#x, %#x, %#x, %#x) = %#x, %#x, want %#x, %#x", name, x, y, m, a, zz[1:1+size], c, wantZ, wantC) + } + if !inplace { + checkSlice(t, name, "x", xx, 1, x) + } + checkSlice(t, name, "y", yy, 2, y) + checkSlice(t, name, "z", zz, 3, nil) + if t.Failed() { + t.FailNow() + } + } + } + } + } + } + } } -var rshVW = []argVW{ - {}, - {nat{0}, nat{0}, 0, 0}, - {nat{0}, nat{0}, 1, 0}, - {nat{0}, nat{0}, 20, 0}, - - {nat{_M}, nat{_M}, 0, 0}, - {nat{_M >> 1}, nat{_M}, 1, _M << (_W - 1) & _M}, - {nat{_M >> 20}, nat{_M}, 20, _M << (_W - 20) & _M}, +const ( + magic0 = 0x123450 + magic1 = 0x543210 +) - {nat{_M, _M, _M}, nat{_M, _M, _M}, 0, 0}, - {nat{_M, _M, _M >> 1}, nat{_M, _M, _M}, 1, _M << (_W - 1) & _M}, - {nat{_M, _M, _M >> 20}, nat{_M, _M, _M}, 20, _M << (_W - 20) & _M}, +// setSlice sets x[1:len(x)-1] to orig, leaving magic values in x[0] and x[len(x)-1] +// so that we can tell if routines accidentally write before or after the data. +func setSlice(x []Word, id Word, orig []Word) { + x[0] = magic0 + id + copy(x[1:len(x)-1], orig) + x[len(x)-1] = magic1 + id } -func testFunVW(t *testing.T, msg string, f funVW, a argVW) { - z := make(nat, len(a.z)) - c := f(z, a.x, a.y) - for i, zi := range z { - if zi != a.z[i] { - t.Errorf("%s%+v\n\tgot z[%d] = %#x; want %#x", msg, a, i, zi, a.z[i]) - break - } +// checkSlice checks that the magic values left by setSlices are still there. +// If orig != nil, it also checks that the actual data in x is unmodified since setSlice. +func checkSlice(t *testing.T, name, val string, x []Word, id Word, orig []Word) { + if x[0] != magic0+id { + t.Errorf("%s smashed %s[-1]", name, val) } - if c != a.c { - t.Errorf("%s%+v\n\tgot c = %#x; want %#x", msg, a, c, a.c) + if x[len(x)-1] != magic1+id { + t.Errorf("%s smashed %s[len(%s)]", name, val, val) + } + if orig != nil && !slices.Equal(x[1:len(x)-1], orig) { + t.Errorf("%s smashed %s: have %d, want %d", name, val, x[1:len(x)-1], orig) } } -func testFunVWext(t *testing.T, msg string, f funVW, f_g funVW, a argVW) { - // using the result of addVW_g/subVW_g as golden - z_g := make(nat, len(a.z)) - c_g := f_g(z_g, a.x, a.y) - c := f(a.z, a.x, a.y) +// nats returns a sequence of interesting nats of the given size: +// +// - all 0 +// - all ^0 +// - all possible combinations of words +// - ten random values +func nats(words []Word, size int) iter.Seq[[]Word] { + return func(yield func([]Word) bool) { + if size == 0 { + yield(nil) + return + } + w := make([]Word, size) - for i, zi := range a.z { - if zi != z_g[i] { - t.Errorf("%s\n\tgot z[%d] = %#x; want %#x", msg, i, zi, z_g[i]) - break + // all 0 + for i := range w { + w[i] = 0 + } + if !yield(w) { + return } - } - if c != c_g { - t.Errorf("%s\n\tgot c = %#x; want %#x", msg, c, c_g) - } -} -func makeFunVW(f func(z, x []Word, s uint) (c Word)) funVW { - return func(z, x []Word, s Word) (c Word) { - return f(z, x, uint(s)) - } -} + // all ^0 + for i := range w { + w[i] = _M + } + if !yield(w) { + return + } -func TestFunVW(t *testing.T) { - for _, a := range sumVW { - arg := a - testFunVW(t, "addVW_g", addVW_g, arg) - testFunVW(t, "addVW", addVW, arg) + // all possible combinations of words + var generate func(int) bool + generate = func(i int) bool { + if i >= len(w) { + return yield(w) + } + for _, w[i] = range words { + if !generate(i + 1) { + return false + } + } + return true + } + if !generate(0) { + return + } - arg = argVW{a.x, a.z, a.y, a.c} - testFunVW(t, "subVW_g", subVW_g, arg) - testFunVW(t, "subVW", subVW, arg) + // ten random values + for range 10 { + for i := range w { + w[i] = Word(rnd.Uint()) + } + if !yield(w) { + return + } + } } +} - shlVW_g := makeFunVW(shlVU_g) - shlVW := makeFunVW(shlVU) - for _, a := range lshVW { - arg := a - testFunVW(t, "shlVU_g", shlVW_g, arg) - testFunVW(t, "shlVU", shlVW, arg) - } +// Always the same seed for reproducible results. +var rnd = rand.New(rand.NewPCG(1, 2)) - shrVW_g := makeFunVW(shrVU_g) - shrVW := makeFunVW(shrVU) - for _, a := range rshVW { - arg := a - testFunVW(t, "shrVU_g", shrVW_g, arg) - testFunVW(t, "shrVU", shrVW, arg) +func rndW() Word { + return Word(rnd.Uint()) +} + +func rndV(n int) []Word { + v := make([]Word, n) + for i := range v { + v[i] = rndW() } + return v } // Construct a vector comprising the same word, usually '0' or 'maximum uint' @@ -240,40 +491,6 @@ func makeWordVec(e Word, n int) []Word { return v } -// Extended testing to addVW and subVW using various kinds of input data. -// We utilize the results of addVW_g and subVW_g as golden reference to check -// correctness. -func TestFunVWExt(t *testing.T) { - // 32 is the current threshold that triggers an optimized version of - // calculation for large-sized vector, ensure we have sizes around it tested. - var vwSizes = []int{0, 1, 3, 4, 5, 8, 9, 23, 31, 32, 33, 34, 35, 36, 50, 120} - for _, n := range vwSizes { - // vector of random numbers, using the result of addVW_g/subVW_g as golden - x := rndV(n) - y := rndW() - z := make(nat, n) - arg := argVW{z, x, y, 0} - testFunVWext(t, "addVW, random inputs", addVW, addVW_g, arg) - testFunVWext(t, "subVW, random inputs", subVW, subVW_g, arg) - - // vector of random numbers, but make 'x' and 'z' share storage - arg = argVW{x, x, y, 0} - testFunVWext(t, "addVW, random inputs, sharing storage", addVW, addVW_g, arg) - testFunVWext(t, "subVW, random inputs, sharing storage", subVW, subVW_g, arg) - - // vector of maximum uint, to force carry flag set in each 'add' - y = ^Word(0) - x = makeWordVec(y, n) - arg = argVW{z, x, y, 0} - testFunVWext(t, "addVW, vector of max uint", addVW, addVW_g, arg) - - // vector of '0', to force carry flag set in each 'sub' - x = makeWordVec(0, n) - arg = argVW{z, x, 1, 0} - testFunVWext(t, "subVW, vector of zero", subVW, subVW_g, arg) - } -} - type argVU struct { d []Word // d is a Word slice, the input parameters x and z come from this array. l uint // l is the length of the input parameters x and z. @@ -285,56 +502,48 @@ type argVU struct { m string // message. } -var argshlVUIn = []Word{1, 2, 4, 8, 16, 32, 64, 0, 0, 0} -var argshlVUr0 = []Word{1, 2, 4, 8, 16, 32, 64} -var argshlVUr1 = []Word{2, 4, 8, 16, 32, 64, 128} -var argshlVUrWm1 = []Word{1 << (_W - 1), 0, 1, 2, 4, 8, 16} - -var argshlVU = []argVU{ - // test cases for shlVU - {[]Word{1, _M, _M, _M, _M, _M, 3 << (_W - 2), 0}, 7, 0, 0, 1, []Word{2, _M - 1, _M, _M, _M, _M, 1<<(_W-1) + 1}, 1, "complete overlap of shlVU"}, - {[]Word{1, _M, _M, _M, _M, _M, 3 << (_W - 2), 0, 0, 0, 0}, 7, 0, 3, 1, []Word{2, _M - 1, _M, _M, _M, _M, 1<<(_W-1) + 1}, 1, "partial overlap by half of shlVU"}, - {[]Word{1, _M, _M, _M, _M, _M, 3 << (_W - 2), 0, 0, 0, 0, 0, 0, 0}, 7, 0, 6, 1, []Word{2, _M - 1, _M, _M, _M, _M, 1<<(_W-1) + 1}, 1, "partial overlap by 1 Word of shlVU"}, - {[]Word{1, _M, _M, _M, _M, _M, 3 << (_W - 2), 0, 0, 0, 0, 0, 0, 0, 0}, 7, 0, 7, 1, []Word{2, _M - 1, _M, _M, _M, _M, 1<<(_W-1) + 1}, 1, "no overlap of shlVU"}, - // additional test cases with shift values of 0, 1 and (_W-1) - {argshlVUIn, 7, 0, 0, 0, argshlVUr0, 0, "complete overlap of shlVU and shift of 0"}, - {argshlVUIn, 7, 0, 0, 1, argshlVUr1, 0, "complete overlap of shlVU and shift of 1"}, - {argshlVUIn, 7, 0, 0, _W - 1, argshlVUrWm1, 32, "complete overlap of shlVU and shift of _W - 1"}, - {argshlVUIn, 7, 0, 1, 0, argshlVUr0, 0, "partial overlap by 6 Words of shlVU and shift of 0"}, - {argshlVUIn, 7, 0, 1, 1, argshlVUr1, 0, "partial overlap by 6 Words of shlVU and shift of 1"}, - {argshlVUIn, 7, 0, 1, _W - 1, argshlVUrWm1, 32, "partial overlap by 6 Words of shlVU and shift of _W - 1"}, - {argshlVUIn, 7, 0, 2, 0, argshlVUr0, 0, "partial overlap by 5 Words of shlVU and shift of 0"}, - {argshlVUIn, 7, 0, 2, 1, argshlVUr1, 0, "partial overlap by 5 Words of shlVU and shift of 1"}, - {argshlVUIn, 7, 0, 2, _W - 1, argshlVUrWm1, 32, "partial overlap by 5 Words of shlVU abd shift of _W - 1"}, - {argshlVUIn, 7, 0, 3, 0, argshlVUr0, 0, "partial overlap by 4 Words of shlVU and shift of 0"}, - {argshlVUIn, 7, 0, 3, 1, argshlVUr1, 0, "partial overlap by 4 Words of shlVU and shift of 1"}, - {argshlVUIn, 7, 0, 3, _W - 1, argshlVUrWm1, 32, "partial overlap by 4 Words of shlVU and shift of _W - 1"}, -} - -var argshrVUIn = []Word{0, 0, 0, 1, 2, 4, 8, 16, 32, 64} -var argshrVUr0 = []Word{1, 2, 4, 8, 16, 32, 64} -var argshrVUr1 = []Word{0, 1, 2, 4, 8, 16, 32} -var argshrVUrWm1 = []Word{4, 8, 16, 32, 64, 128, 0} - -var argshrVU = []argVU{ - // test cases for shrVU - {[]Word{0, 3, _M, _M, _M, _M, _M, 1 << (_W - 1)}, 7, 1, 1, 1, []Word{1<<(_W-1) + 1, _M, _M, _M, _M, _M >> 1, 1 << (_W - 2)}, 1 << (_W - 1), "complete overlap of shrVU"}, - {[]Word{0, 0, 0, 0, 3, _M, _M, _M, _M, _M, 1 << (_W - 1)}, 7, 4, 1, 1, []Word{1<<(_W-1) + 1, _M, _M, _M, _M, _M >> 1, 1 << (_W - 2)}, 1 << (_W - 1), "partial overlap by half of shrVU"}, - {[]Word{0, 0, 0, 0, 0, 0, 0, 3, _M, _M, _M, _M, _M, 1 << (_W - 1)}, 7, 7, 1, 1, []Word{1<<(_W-1) + 1, _M, _M, _M, _M, _M >> 1, 1 << (_W - 2)}, 1 << (_W - 1), "partial overlap by 1 Word of shrVU"}, - {[]Word{0, 0, 0, 0, 0, 0, 0, 0, 3, _M, _M, _M, _M, _M, 1 << (_W - 1)}, 7, 8, 1, 1, []Word{1<<(_W-1) + 1, _M, _M, _M, _M, _M >> 1, 1 << (_W - 2)}, 1 << (_W - 1), "no overlap of shrVU"}, +var arglshVUIn = []Word{1, 2, 4, 8, 16, 32, 64, 0, 0, 0} +var arglshVUr0 = []Word{1, 2, 4, 8, 16, 32, 64} +var arglshVUr1 = []Word{2, 4, 8, 16, 32, 64, 128} +var arglshVUrWm1 = []Word{1 << (_W - 1), 0, 1, 2, 4, 8, 16} + +var arglshVU = []argVU{ + // test cases for lshVU + {[]Word{1, _M, _M, _M, _M, _M, 3 << (_W - 2), 0}, 7, 0, 0, 1, []Word{2, _M - 1, _M, _M, _M, _M, 1<<(_W-1) + 1}, 1, "complete overlap of lshVU"}, + {[]Word{1, _M, _M, _M, _M, _M, 3 << (_W - 2), 0, 0, 0, 0}, 7, 0, 3, 1, []Word{2, _M - 1, _M, _M, _M, _M, 1<<(_W-1) + 1}, 1, "partial overlap by half of lshVU"}, + {[]Word{1, _M, _M, _M, _M, _M, 3 << (_W - 2), 0, 0, 0, 0, 0, 0, 0}, 7, 0, 6, 1, []Word{2, _M - 1, _M, _M, _M, _M, 1<<(_W-1) + 1}, 1, "partial overlap by 1 Word of lshVU"}, + {[]Word{1, _M, _M, _M, _M, _M, 3 << (_W - 2), 0, 0, 0, 0, 0, 0, 0, 0}, 7, 0, 7, 1, []Word{2, _M - 1, _M, _M, _M, _M, 1<<(_W-1) + 1}, 1, "no overlap of lshVU"}, + // additional test cases with shift values of 1 and (_W-1) + {arglshVUIn, 7, 0, 0, 1, arglshVUr1, 0, "complete overlap of lshVU and shift of 1"}, + {arglshVUIn, 7, 0, 0, _W - 1, arglshVUrWm1, 32, "complete overlap of lshVU and shift of _W - 1"}, + {arglshVUIn, 7, 0, 1, 1, arglshVUr1, 0, "partial overlap by 6 Words of lshVU and shift of 1"}, + {arglshVUIn, 7, 0, 1, _W - 1, arglshVUrWm1, 32, "partial overlap by 6 Words of lshVU and shift of _W - 1"}, + {arglshVUIn, 7, 0, 2, 1, arglshVUr1, 0, "partial overlap by 5 Words of lshVU and shift of 1"}, + {arglshVUIn, 7, 0, 2, _W - 1, arglshVUrWm1, 32, "partial overlap by 5 Words of lshVU abd shift of _W - 1"}, + {arglshVUIn, 7, 0, 3, 1, arglshVUr1, 0, "partial overlap by 4 Words of lshVU and shift of 1"}, + {arglshVUIn, 7, 0, 3, _W - 1, arglshVUrWm1, 32, "partial overlap by 4 Words of lshVU and shift of _W - 1"}, +} + +var argrshVUIn = []Word{0, 0, 0, 1, 2, 4, 8, 16, 32, 64} +var argrshVUr0 = []Word{1, 2, 4, 8, 16, 32, 64} +var argrshVUr1 = []Word{0, 1, 2, 4, 8, 16, 32} +var argrshVUrWm1 = []Word{4, 8, 16, 32, 64, 128, 0} + +var argrshVU = []argVU{ + // test cases for rshVU + {[]Word{0, 3, _M, _M, _M, _M, _M, 1 << (_W - 1)}, 7, 1, 1, 1, []Word{1<<(_W-1) + 1, _M, _M, _M, _M, _M >> 1, 1 << (_W - 2)}, 1 << (_W - 1), "complete overlap of rshVU"}, + {[]Word{0, 0, 0, 0, 3, _M, _M, _M, _M, _M, 1 << (_W - 1)}, 7, 4, 1, 1, []Word{1<<(_W-1) + 1, _M, _M, _M, _M, _M >> 1, 1 << (_W - 2)}, 1 << (_W - 1), "partial overlap by half of rshVU"}, + {[]Word{0, 0, 0, 0, 0, 0, 0, 3, _M, _M, _M, _M, _M, 1 << (_W - 1)}, 7, 7, 1, 1, []Word{1<<(_W-1) + 1, _M, _M, _M, _M, _M >> 1, 1 << (_W - 2)}, 1 << (_W - 1), "partial overlap by 1 Word of rshVU"}, + {[]Word{0, 0, 0, 0, 0, 0, 0, 0, 3, _M, _M, _M, _M, _M, 1 << (_W - 1)}, 7, 8, 1, 1, []Word{1<<(_W-1) + 1, _M, _M, _M, _M, _M >> 1, 1 << (_W - 2)}, 1 << (_W - 1), "no overlap of rshVU"}, // additional test cases with shift values of 0, 1 and (_W-1) - {argshrVUIn, 7, 3, 3, 0, argshrVUr0, 0, "complete overlap of shrVU and shift of 0"}, - {argshrVUIn, 7, 3, 3, 1, argshrVUr1, 1 << (_W - 1), "complete overlap of shrVU and shift of 1"}, - {argshrVUIn, 7, 3, 3, _W - 1, argshrVUrWm1, 2, "complete overlap of shrVU and shift of _W - 1"}, - {argshrVUIn, 7, 3, 2, 0, argshrVUr0, 0, "partial overlap by 6 Words of shrVU and shift of 0"}, - {argshrVUIn, 7, 3, 2, 1, argshrVUr1, 1 << (_W - 1), "partial overlap by 6 Words of shrVU and shift of 1"}, - {argshrVUIn, 7, 3, 2, _W - 1, argshrVUrWm1, 2, "partial overlap by 6 Words of shrVU and shift of _W - 1"}, - {argshrVUIn, 7, 3, 1, 0, argshrVUr0, 0, "partial overlap by 5 Words of shrVU and shift of 0"}, - {argshrVUIn, 7, 3, 1, 1, argshrVUr1, 1 << (_W - 1), "partial overlap by 5 Words of shrVU and shift of 1"}, - {argshrVUIn, 7, 3, 1, _W - 1, argshrVUrWm1, 2, "partial overlap by 5 Words of shrVU and shift of _W - 1"}, - {argshrVUIn, 7, 3, 0, 0, argshrVUr0, 0, "partial overlap by 4 Words of shrVU and shift of 0"}, - {argshrVUIn, 7, 3, 0, 1, argshrVUr1, 1 << (_W - 1), "partial overlap by 4 Words of shrVU and shift of 1"}, - {argshrVUIn, 7, 3, 0, _W - 1, argshrVUrWm1, 2, "partial overlap by 4 Words of shrVU and shift of _W - 1"}, + {argrshVUIn, 7, 3, 3, 1, argrshVUr1, 1 << (_W - 1), "complete overlap of rshVU and shift of 1"}, + {argrshVUIn, 7, 3, 3, _W - 1, argrshVUrWm1, 2, "complete overlap of rshVU and shift of _W - 1"}, + {argrshVUIn, 7, 3, 2, 1, argrshVUr1, 1 << (_W - 1), "partial overlap by 6 Words of rshVU and shift of 1"}, + {argrshVUIn, 7, 3, 2, _W - 1, argrshVUrWm1, 2, "partial overlap by 6 Words of rshVU and shift of _W - 1"}, + {argrshVUIn, 7, 3, 1, 1, argrshVUr1, 1 << (_W - 1), "partial overlap by 5 Words of rshVU and shift of 1"}, + {argrshVUIn, 7, 3, 1, _W - 1, argrshVUrWm1, 2, "partial overlap by 5 Words of rshVU and shift of _W - 1"}, + {argrshVUIn, 7, 3, 0, 1, argrshVUr1, 1 << (_W - 1), "partial overlap by 4 Words of rshVU and shift of 1"}, + {argrshVUIn, 7, 3, 0, _W - 1, argrshVUrWm1, 2, "partial overlap by 4 Words of rshVU and shift of _W - 1"}, } func testShiftFunc(t *testing.T, f func(z, x []Word, s uint) Word, a argVU) { @@ -346,36 +555,39 @@ func testShiftFunc(t *testing.T, f func(z, x []Word, s uint) Word, a argVU) { c := f(z, x, a.s) for i, zi := range z { if zi != a.r[i] { - t.Errorf("d := %v, %s(d[%d:%d], d[%d:%d], %d)\n\tgot z[%d] = %#x; want %#x", a.d, a.m, a.zp, a.zp+a.l, a.xp, a.xp+a.l, a.s, i, zi, a.r[i]) + t.Errorf("d := %v, %s (d[%d:%d], d[%d:%d], %d)\n\tgot z[%d] = %#x; want %#x", a.d, a.m, a.zp, a.zp+a.l, a.xp, a.xp+a.l, a.s, i, zi, a.r[i]) break } } if c != a.c { - t.Errorf("d := %v, %s(d[%d:%d], d[%d:%d], %d)\n\tgot c = %#x; want %#x", a.d, a.m, a.zp, a.zp+a.l, a.xp, a.xp+a.l, a.s, c, a.c) + t.Errorf("d := %v, %s (d[%d:%d], d[%d:%d], %d)\n\tgot c = %#x; want %#x", a.d, a.m, a.zp, a.zp+a.l, a.xp, a.xp+a.l, a.s, c, a.c) } } func TestShiftOverlap(t *testing.T) { - for _, a := range argshlVU { + for _, a := range arglshVU { arg := a - testShiftFunc(t, shlVU, arg) + testShiftFunc(t, lshVU, arg) } - for _, a := range argshrVU { + for _, a := range argrshVU { arg := a - testShiftFunc(t, shrVU, arg) + testShiftFunc(t, rshVU, arg) } } func TestIssue31084(t *testing.T) { + stk := getStack() + defer stk.free() + // compute 10^n via 5^n << n. const n = 165 - p := nat(nil).expNN(nat{5}, nat{n}, nil, false) - p = p.shl(p, n) + p := nat(nil).expNN(stk, nat{5}, nat{n}, nil, false) + p = p.lsh(p, n) got := string(p.utoa(10)) want := "1" + strings.Repeat("0", n) if got != want { - t.Errorf("shl(%v, %v)\n\tgot %s\n\twant %s", p, n, got, want) + t.Errorf("lsh(%v, %v)\n\tgot %s\n\twant %s", p, n, got, want) } } @@ -384,81 +596,11 @@ const issue42838Value = "1593091911132452277028880397767711805591104555192618786 func TestIssue42838(t *testing.T) { const s = 192 z, _, _, _ := nat(nil).scan(strings.NewReader(issue42838Value), 0, false) - z = z.shl(z, s) + z = z.lsh(z, s) got := string(z.utoa(10)) want := "1" + strings.Repeat("0", s) if got != want { - t.Errorf("shl(%v, %v)\n\tgot %s\n\twant %s", z, s, got, want) - } -} - -func BenchmarkAddVW(b *testing.B) { - for _, n := range benchSizes { - if isRaceBuilder && n > 1e3 { - continue - } - x := rndV(n) - y := rndW() - z := make([]Word, n) - b.Run(fmt.Sprint(n), func(b *testing.B) { - b.SetBytes(int64(n * _S)) - for i := 0; i < b.N; i++ { - addVW(z, x, y) - } - }) - } -} - -// Benchmarking addVW using vector of maximum uint to force carry flag set -func BenchmarkAddVWext(b *testing.B) { - for _, n := range benchSizes { - if isRaceBuilder && n > 1e3 { - continue - } - y := ^Word(0) - x := makeWordVec(y, n) - z := make([]Word, n) - b.Run(fmt.Sprint(n), func(b *testing.B) { - b.SetBytes(int64(n * _S)) - for i := 0; i < b.N; i++ { - addVW(z, x, y) - } - }) - } -} - -func BenchmarkSubVW(b *testing.B) { - for _, n := range benchSizes { - if isRaceBuilder && n > 1e3 { - continue - } - x := rndV(n) - y := rndW() - z := make([]Word, n) - b.Run(fmt.Sprint(n), func(b *testing.B) { - b.SetBytes(int64(n * _S)) - for i := 0; i < b.N; i++ { - subVW(z, x, y) - } - }) - } -} - -// Benchmarking subVW using vector of zero to force carry flag set -func BenchmarkSubVWext(b *testing.B) { - for _, n := range benchSizes { - if isRaceBuilder && n > 1e3 { - continue - } - x := makeWordVec(0, n) - y := Word(1) - z := make([]Word, n) - b.Run(fmt.Sprint(n), func(b *testing.B) { - b.SetBytes(int64(n * _S)) - for i := 0; i < b.N; i++ { - subVW(z, x, y) - } - }) + t.Errorf("lsh(%v, %v)\n\tgot %s\n\twant %s", z, s, got, want) } } @@ -524,14 +666,10 @@ type argWVW struct { func testFunWVW(t *testing.T, msg string, f funWVW, a argWVW) { z := make(nat, len(a.z)) r := f(z, a.xn, a.x, a.y) - for i, zi := range z { - if zi != a.z[i] { - t.Errorf("%s%+v\n\tgot z[%d] = %#x; want %#x", msg, a, i, zi, a.z[i]) - break - } - } - if r != a.r { - t.Errorf("%s%+v\n\tgot r = %#x; want %#x", msg, a, r, a.r) + if !slices.Equal(z, a.z) || r != a.r { + t.Errorf("%s%+v\nhave %v, %v\nwant %v, %v", msg, a, z, r, a.z, a.r) + } else { + t.Logf("%s%+v\ngood %v, %v", msg, a, z, r) } } @@ -621,77 +759,182 @@ func TestDivWW(t *testing.T) { } } -func BenchmarkMulAddVWW(b *testing.B) { +// benchSizes are the benchmark word sizes. +var benchSizes = []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16, 32, 64, 100, 1000, 10_000, 100_000} + +// A benchFunc is a function to be benchmarked. +// It takes one output buffer and two input buffers, +// but it does not have to use any of them. +type benchFunc func(z, x, y []Word) + +// bench runs benchmarks of fn for a variety of word sizes. +// It adds the given suffix (for example "/impl=go") to the benchmark names it creates, +// after a "/words=N" parameter. Putting words first makes it easier to run +// all benchmarks with a specific word size +// (go test -run=NONE '-bench=V/words=100$') +// even if different benchmarks have different numbers of other parameters. +func bench(b *testing.B, suffix string, fn benchFunc) { for _, n := range benchSizes { if isRaceBuilder && n > 1e3 { continue } - z := make([]Word, n+1) - x := rndV(n) - y := rndW() - r := rndW() - b.Run(fmt.Sprint(n), func(b *testing.B) { - b.SetBytes(int64(n * _W)) - for i := 0; i < b.N; i++ { - mulAddVWW(z, x, y, r) + var z, x, y []Word + b.Run(fmt.Sprintf("words=%d%s", n, suffix), func(b *testing.B) { + if z == nil { + z = make([]Word, n) + x = rndV(n) + y = rndV(n) + } + b.SetBytes(int64(n * _S)) + for b.Loop() { + fn(z, x, y) } }) } } -func BenchmarkAddMulVVW(b *testing.B) { - for _, n := range benchSizes { - if isRaceBuilder && n > 1e3 { - continue - } - x := rndV(n) - y := rndW() - z := make([]Word, n) - b.Run(fmt.Sprint(n), func(b *testing.B) { - b.SetBytes(int64(n * _W)) - for i := 0; i < b.N; i++ { - addMulVVW(z, x, y) - } - }) - } +// Benchmark basic I/O and arithmetic processing speed, +// to help estimate the upper bounds on other operations. + +func BenchmarkCopyVV(b *testing.B) { bench(b, "", benchVV(copyVV)) } + +func copyVV(z, x, y []Word) Word { + copy(z, x) + return 0 } -func BenchmarkDivWVW(b *testing.B) { - for _, n := range benchSizes { - if isRaceBuilder && n > 1e3 { - continue + +// Note: This benchmark consistently runs faster (even up to 2X faster on MB/s) +// with words=10 and words=100 than larger amounts like words=1000 or words=10000. +// The reason appears to that if you run 100-word addition loops repeatedly, +// they are independent calculations, and the processor speculates/pipelines/whatever +// to such a deep level that it can overlap the repeated loops. +// In contrast, if you run 1000-word or 10000-word loops repeatedly, +// the dependency chains are so long that the processor cannot overlap them. +// If we change arithVV to take the starting value of s and pass in the result +// from the previous arithVV, then even the 10-word or 100-loops become +// a single long dependency chain and the 2X disappears. But since we are +// using BenchmarkArithVV for a given word size to estimate the upper bound +// of, say, BenchmarkAddVV for that same word size, we actually want the +// dependency chain-length variation in BenchmarkArithVV too. +// It's just mysterious to see until you understand what is causing it. + +func BenchmarkArithVV(b *testing.B) { bench(b, "", benchVV(arithVV)) } + +func arithVV(z, x, y []Word) Word { + var a, b, c, d, e, f, g, h, i, j Word + if len(z) >= 8 { + a, b, c, d, e, f, g, h, i, j = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 + } + if len(z) < 10 { + // We don't really care about the speed here, but + // do something so that the small word counts aren't all the same. + s := Word(0) + for _, zi := range z { + s += zi } - x := rndV(n) - y := rndW() - z := make([]Word, n) - b.Run(fmt.Sprint(n), func(b *testing.B) { - b.SetBytes(int64(n * _W)) - for i := 0; i < b.N; i++ { - divWVW(z, 0, x, y) - } - }) + return s } + s := Word(0) + for range len(z) / 10 { + s += a + s += b + s += c + s += d + s += e + s += f + s += g + s += h + s += i + s += j + } + return s } -func BenchmarkNonZeroShifts(b *testing.B) { - for _, n := range benchSizes { - if isRaceBuilder && n > 1e3 { - continue +func BenchmarkAddVV(b *testing.B) { + bench(b, "/impl=asm", benchVV(addVV)) + bench(b, "/impl=go", benchVV(addVV_g)) +} + +func BenchmarkSubVV(b *testing.B) { + bench(b, "/impl=asm", benchVV(subVV)) + bench(b, "/impl=go", benchVV(subVV_g)) +} + +func benchVV(fn func(z, x, y []Word) Word) benchFunc { + return func(z, x, y []Word) { fn(z, x, y) } +} + +func BenchmarkAddVW(b *testing.B) { + bench(b, "/data=random", benchVW(addVW, 123)) + bench(b, "/data=carry", benchCarryVW(addVW, ^Word(0), 1)) + bench(b, "/data=shortcut", benchShortVW(addVW, 123)) +} + +func BenchmarkSubVW(b *testing.B) { + bench(b, "/data=random", benchVW(subVW, 123)) + bench(b, "/data=carry", benchCarryVW(subVW, 0, 1)) + bench(b, "/data=shortcut", benchShortVW(subVW, 123)) +} + +func benchVW(fn func(z, x []Word, w Word) Word, w Word) benchFunc { + return func(z, x, y []Word) { fn(z, x, w) } +} + +func benchCarryVW(fn func(z, x []Word, w Word) Word, xi, w Word) benchFunc { + return func(z, x, y []Word) { + // Fill x with xi the first time we are called with a given x. + // Otherwise x is random, so checking the first two elements is good enough. + // Assume this is the warmup, so we don't need to worry about it taking longer. + if x[0] != w || len(x) >= 2 && x[1] != w { + for i := range x { + x[i] = xi + } } - x := rndV(n) - s := uint(rand.Int63n(_W-2)) + 1 // avoid 0 and over-large shifts - z := make([]Word, n) - b.Run(fmt.Sprint(n), func(b *testing.B) { - b.SetBytes(int64(n * _W)) - b.Run("shrVU", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _ = shrVU(z, x, s) - } - }) - b.Run("shlVU", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _ = shlVU(z, x, s) - } - }) - }) + fn(z, x, w) } } + +func benchShortVW(fn func(z, x []Word, w Word) Word, w Word) benchFunc { + // Note: calling fn with x not z, to benchmark in-place overwriting. + return func(z, x, y []Word) { fn(x, x, w) } +} + +func BenchmarkLshVU(b *testing.B) { + bench(b, "/impl=asm", benchVU(lshVU, 3)) + bench(b, "/impl=go", benchVU(lshVU_g, 3)) +} + +func BenchmarkRshVU(b *testing.B) { + bench(b, "/impl=asm", benchVU(rshVU, 3)) + bench(b, "/impl=go", benchVU(rshVU_g, 3)) +} + +func benchVU(fn func(z, x []Word, s uint) Word, s uint) benchFunc { + return func(z, x, y []Word) { fn(z, x, s) } +} + +func BenchmarkMulAddVWW(b *testing.B) { + bench(b, "/impl=asm", benchVWW(mulAddVWW, 42, 100)) + bench(b, "/impl=go", benchVWW(mulAddVWW_g, 42, 100)) +} + +func benchVWW(fn func(z, x []Word, w1, w2 Word) Word, w1, w2 Word) benchFunc { + return func(z, x, y []Word) { fn(z, x, w1, w2) } +} + +func BenchmarkAddMulVVWW(b *testing.B) { + bench(b, "/impl=asm", benchVVWW(addMulVVWW, 42, 100)) + bench(b, "/impl=go", benchVVWW(addMulVVWW_g, 42, 100)) +} + +func benchVVWW(fn func(z, x, y []Word, w1, w2 Word) Word, w1, w2 Word) benchFunc { + return func(z, x, y []Word) { fn(z, x, y, w1, w2) } +} + +func BenchmarkDivWVW(b *testing.B) { + bench(b, "", benchWVW(divWVW, 100, 200)) +} + +func benchWVW(fn func(z []Word, w1 Word, x []Word, w2 Word) Word, w1, w2 Word) benchFunc { + return func(z, x, y []Word) { fn(z, w1, x, w2) } +} diff --git a/src/math/big/arith_wasm.s b/src/math/big/arith_wasm.s index fd51031d8a0246..3a9aa4ddcb2dca 100644 --- a/src/math/big/arith_wasm.s +++ b/src/math/big/arith_wasm.s @@ -12,21 +12,15 @@ TEXT ·addVV(SB),NOSPLIT,$0 TEXT ·subVV(SB),NOSPLIT,$0 JMP ·subVV_g(SB) -TEXT ·addVW(SB),NOSPLIT,$0 - JMP ·addVW_g(SB) +TEXT ·lshVU(SB),NOSPLIT,$0 + JMP ·lshVU_g(SB) -TEXT ·subVW(SB),NOSPLIT,$0 - JMP ·subVW_g(SB) - -TEXT ·shlVU(SB),NOSPLIT,$0 - JMP ·shlVU_g(SB) - -TEXT ·shrVU(SB),NOSPLIT,$0 - JMP ·shrVU_g(SB) +TEXT ·rshVU(SB),NOSPLIT,$0 + JMP ·rshVU_g(SB) TEXT ·mulAddVWW(SB),NOSPLIT,$0 JMP ·mulAddVWW_g(SB) -TEXT ·addMulVVW(SB),NOSPLIT,$0 - JMP ·addMulVVW_g(SB) +TEXT ·addMulVVWW(SB),NOSPLIT,$0 + JMP ·addMulVVWW_g(SB) diff --git a/src/math/big/arithvec_s390x.go b/src/math/big/arithvec_s390x.go new file mode 100644 index 00000000000000..d1703596c2fd5f --- /dev/null +++ b/src/math/big/arithvec_s390x.go @@ -0,0 +1,14 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !math_big_pure_go + +package big + +import "internal/cpu" + +var hasVX = cpu.S390X.HasVX + +func addVVvec(z, x, y []Word) (c Word) +func subVVvec(z, x, y []Word) (c Word) diff --git a/src/math/big/arithvec_s390x.s b/src/math/big/arithvec_s390x.s new file mode 100644 index 00000000000000..3ea51fde1e9d9f --- /dev/null +++ b/src/math/big/arithvec_s390x.s @@ -0,0 +1,310 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !math_big_pure_go + +#include "textflag.h" + +TEXT ·addVVvec(SB), NOSPLIT, $0 + MOVD z_len+8(FP), R3 + MOVD x+24(FP), R8 + MOVD y+48(FP), R9 + MOVD z+0(FP), R2 + + MOVD $0, R4 // c = 0 + MOVD $0, R0 // make sure it's zero + MOVD $0, R10 // i = 0 + + // s/JL/JMP/ below to disable the unrolled loop + SUB $4, R3 + BLT v1 + SUB $12, R3 // n -= 16 + BLT A1 // if n < 0 goto A1 + + MOVD R8, R5 + MOVD R9, R6 + MOVD R2, R7 + + // n >= 0 + // regular loop body unrolled 16x + VZERO V0 // c = 0 + +UU1: + VLM 0(R5), V1, V4 // 64-bytes into V1..V8 + ADD $64, R5 + VPDI $0x4, V1, V1, V1 // flip the doublewords to big-endian order + VPDI $0x4, V2, V2, V2 // flip the doublewords to big-endian order + + VLM 0(R6), V9, V12 // 64-bytes into V9..V16 + ADD $64, R6 + VPDI $0x4, V9, V9, V9 // flip the doublewords to big-endian order + VPDI $0x4, V10, V10, V10 // flip the doublewords to big-endian order + + VACCCQ V1, V9, V0, V25 + VACQ V1, V9, V0, V17 + VACCCQ V2, V10, V25, V26 + VACQ V2, V10, V25, V18 + + VLM 0(R5), V5, V6 // 32-bytes into V1..V8 + VLM 0(R6), V13, V14 // 32-bytes into V9..V16 + ADD $32, R5 + ADD $32, R6 + + VPDI $0x4, V3, V3, V3 // flip the doublewords to big-endian order + VPDI $0x4, V4, V4, V4 // flip the doublewords to big-endian order + VPDI $0x4, V11, V11, V11 // flip the doublewords to big-endian order + VPDI $0x4, V12, V12, V12 // flip the doublewords to big-endian order + + VACCCQ V3, V11, V26, V27 + VACQ V3, V11, V26, V19 + VACCCQ V4, V12, V27, V28 + VACQ V4, V12, V27, V20 + + VLM 0(R5), V7, V8 // 32-bytes into V1..V8 + VLM 0(R6), V15, V16 // 32-bytes into V9..V16 + ADD $32, R5 + ADD $32, R6 + + VPDI $0x4, V5, V5, V5 // flip the doublewords to big-endian order + VPDI $0x4, V6, V6, V6 // flip the doublewords to big-endian order + VPDI $0x4, V13, V13, V13 // flip the doublewords to big-endian order + VPDI $0x4, V14, V14, V14 // flip the doublewords to big-endian order + + VACCCQ V5, V13, V28, V29 + VACQ V5, V13, V28, V21 + VACCCQ V6, V14, V29, V30 + VACQ V6, V14, V29, V22 + + VPDI $0x4, V7, V7, V7 // flip the doublewords to big-endian order + VPDI $0x4, V8, V8, V8 // flip the doublewords to big-endian order + VPDI $0x4, V15, V15, V15 // flip the doublewords to big-endian order + VPDI $0x4, V16, V16, V16 // flip the doublewords to big-endian order + + VACCCQ V7, V15, V30, V31 + VACQ V7, V15, V30, V23 + VACCCQ V8, V16, V31, V0 // V0 has carry-over + VACQ V8, V16, V31, V24 + + VPDI $0x4, V17, V17, V17 // flip the doublewords to big-endian order + VPDI $0x4, V18, V18, V18 // flip the doublewords to big-endian order + VPDI $0x4, V19, V19, V19 // flip the doublewords to big-endian order + VPDI $0x4, V20, V20, V20 // flip the doublewords to big-endian order + VPDI $0x4, V21, V21, V21 // flip the doublewords to big-endian order + VPDI $0x4, V22, V22, V22 // flip the doublewords to big-endian order + VPDI $0x4, V23, V23, V23 // flip the doublewords to big-endian order + VPDI $0x4, V24, V24, V24 // flip the doublewords to big-endian order + VSTM V17, V24, 0(R7) // 128-bytes into z + ADD $128, R7 + ADD $128, R10 // i += 16 + SUB $16, R3 // n -= 16 + BGE UU1 // if n >= 0 goto U1 + VLGVG $1, V0, R4 // put cf into R4 + NEG R4, R4 // save cf + +A1: + ADD $12, R3 // n += 16 + + // s/JL/JMP/ below to disable the unrolled loop + BLT v1 // if n < 0 goto v1 + +U1: // n >= 0 + // regular loop body unrolled 4x + MOVD 0(R8)(R10*1), R5 + MOVD 8(R8)(R10*1), R6 + MOVD 16(R8)(R10*1), R7 + MOVD 24(R8)(R10*1), R1 + ADDC R4, R4 // restore CF + MOVD 0(R9)(R10*1), R11 + ADDE R11, R5 + MOVD 8(R9)(R10*1), R11 + ADDE R11, R6 + MOVD 16(R9)(R10*1), R11 + ADDE R11, R7 + MOVD 24(R9)(R10*1), R11 + ADDE R11, R1 + MOVD R0, R4 + ADDE R4, R4 // save CF + NEG R4, R4 + MOVD R5, 0(R2)(R10*1) + MOVD R6, 8(R2)(R10*1) + MOVD R7, 16(R2)(R10*1) + MOVD R1, 24(R2)(R10*1) + + ADD $32, R10 // i += 4 + SUB $4, R3 // n -= 4 + BGE U1 // if n >= 0 goto U1 + +v1: + ADD $4, R3 // n += 4 + BLE E1 // if n <= 0 goto E1 + +L1: // n > 0 + ADDC R4, R4 // restore CF + MOVD 0(R8)(R10*1), R5 + MOVD 0(R9)(R10*1), R11 + ADDE R11, R5 + MOVD R5, 0(R2)(R10*1) + MOVD R0, R4 + ADDE R4, R4 // save CF + NEG R4, R4 + + ADD $8, R10 // i++ + SUB $1, R3 // n-- + BGT L1 // if n > 0 goto L1 + +E1: + NEG R4, R4 + MOVD R4, c+72(FP) // return c + RET + +TEXT ·subVVvec(SB), NOSPLIT, $0 + MOVD z_len+8(FP), R3 + MOVD x+24(FP), R8 + MOVD y+48(FP), R9 + MOVD z+0(FP), R2 + MOVD $0, R4 // c = 0 + MOVD $0, R0 // make sure it's zero + MOVD $0, R10 // i = 0 + + // s/JL/JMP/ below to disable the unrolled loop + SUB $4, R3 // n -= 4 + BLT v1 // if n < 0 goto v1 + SUB $12, R3 // n -= 16 + BLT A1 // if n < 0 goto A1 + + MOVD R8, R5 + MOVD R9, R6 + MOVD R2, R7 + + // n >= 0 + // regular loop body unrolled 16x + VZERO V0 // cf = 0 + MOVD $1, R4 // for 390 subtraction cf starts as 1 (no borrow) + VLVGG $1, R4, V0 // put carry into V0 + +UU1: + VLM 0(R5), V1, V4 // 64-bytes into V1..V8 + ADD $64, R5 + VPDI $0x4, V1, V1, V1 // flip the doublewords to big-endian order + VPDI $0x4, V2, V2, V2 // flip the doublewords to big-endian order + + VLM 0(R6), V9, V12 // 64-bytes into V9..V16 + ADD $64, R6 + VPDI $0x4, V9, V9, V9 // flip the doublewords to big-endian order + VPDI $0x4, V10, V10, V10 // flip the doublewords to big-endian order + + VSBCBIQ V1, V9, V0, V25 + VSBIQ V1, V9, V0, V17 + VSBCBIQ V2, V10, V25, V26 + VSBIQ V2, V10, V25, V18 + + VLM 0(R5), V5, V6 // 32-bytes into V1..V8 + VLM 0(R6), V13, V14 // 32-bytes into V9..V16 + ADD $32, R5 + ADD $32, R6 + + VPDI $0x4, V3, V3, V3 // flip the doublewords to big-endian order + VPDI $0x4, V4, V4, V4 // flip the doublewords to big-endian order + VPDI $0x4, V11, V11, V11 // flip the doublewords to big-endian order + VPDI $0x4, V12, V12, V12 // flip the doublewords to big-endian order + + VSBCBIQ V3, V11, V26, V27 + VSBIQ V3, V11, V26, V19 + VSBCBIQ V4, V12, V27, V28 + VSBIQ V4, V12, V27, V20 + + VLM 0(R5), V7, V8 // 32-bytes into V1..V8 + VLM 0(R6), V15, V16 // 32-bytes into V9..V16 + ADD $32, R5 + ADD $32, R6 + + VPDI $0x4, V5, V5, V5 // flip the doublewords to big-endian order + VPDI $0x4, V6, V6, V6 // flip the doublewords to big-endian order + VPDI $0x4, V13, V13, V13 // flip the doublewords to big-endian order + VPDI $0x4, V14, V14, V14 // flip the doublewords to big-endian order + + VSBCBIQ V5, V13, V28, V29 + VSBIQ V5, V13, V28, V21 + VSBCBIQ V6, V14, V29, V30 + VSBIQ V6, V14, V29, V22 + + VPDI $0x4, V7, V7, V7 // flip the doublewords to big-endian order + VPDI $0x4, V8, V8, V8 // flip the doublewords to big-endian order + VPDI $0x4, V15, V15, V15 // flip the doublewords to big-endian order + VPDI $0x4, V16, V16, V16 // flip the doublewords to big-endian order + + VSBCBIQ V7, V15, V30, V31 + VSBIQ V7, V15, V30, V23 + VSBCBIQ V8, V16, V31, V0 // V0 has carry-over + VSBIQ V8, V16, V31, V24 + + VPDI $0x4, V17, V17, V17 // flip the doublewords to big-endian order + VPDI $0x4, V18, V18, V18 // flip the doublewords to big-endian order + VPDI $0x4, V19, V19, V19 // flip the doublewords to big-endian order + VPDI $0x4, V20, V20, V20 // flip the doublewords to big-endian order + VPDI $0x4, V21, V21, V21 // flip the doublewords to big-endian order + VPDI $0x4, V22, V22, V22 // flip the doublewords to big-endian order + VPDI $0x4, V23, V23, V23 // flip the doublewords to big-endian order + VPDI $0x4, V24, V24, V24 // flip the doublewords to big-endian order + VSTM V17, V24, 0(R7) // 128-bytes into z + ADD $128, R7 + ADD $128, R10 // i += 16 + SUB $16, R3 // n -= 16 + BGE UU1 // if n >= 0 goto U1 + VLGVG $1, V0, R4 // put cf into R4 + SUB $1, R4 // save cf + +A1: + ADD $12, R3 // n += 16 + BLT v1 // if n < 0 goto v1 + +U1: // n >= 0 + // regular loop body unrolled 4x + MOVD 0(R8)(R10*1), R5 + MOVD 8(R8)(R10*1), R6 + MOVD 16(R8)(R10*1), R7 + MOVD 24(R8)(R10*1), R1 + MOVD R0, R11 + SUBC R4, R11 // restore CF + MOVD 0(R9)(R10*1), R11 + SUBE R11, R5 + MOVD 8(R9)(R10*1), R11 + SUBE R11, R6 + MOVD 16(R9)(R10*1), R11 + SUBE R11, R7 + MOVD 24(R9)(R10*1), R11 + SUBE R11, R1 + MOVD R0, R4 + SUBE R4, R4 // save CF + MOVD R5, 0(R2)(R10*1) + MOVD R6, 8(R2)(R10*1) + MOVD R7, 16(R2)(R10*1) + MOVD R1, 24(R2)(R10*1) + + ADD $32, R10 // i += 4 + SUB $4, R3 // n -= 4 + BGE U1 // if n >= 0 goto U1n + +v1: + ADD $4, R3 // n += 4 + BLE E1 // if n <= 0 goto E1 + +L1: // n > 0 + MOVD R0, R11 + SUBC R4, R11 // restore CF + MOVD 0(R8)(R10*1), R5 + MOVD 0(R9)(R10*1), R11 + SUBE R11, R5 + MOVD R5, 0(R2)(R10*1) + MOVD R0, R4 + SUBE R4, R4 // save CF + + ADD $8, R10 // i++ + SUB $1, R3 // n-- + BGT L1 // if n > 0 goto L1n + +E1: + NEG R4, R4 + MOVD R4, c+72(FP) // return c + RET diff --git a/src/math/big/calibrate.md b/src/math/big/calibrate.md new file mode 100644 index 00000000000000..ff0b4ea137622b --- /dev/null +++ b/src/math/big/calibrate.md @@ -0,0 +1,180 @@ +# Calibration of Algorithm Thresholds + +This document describes the approach to calibration of algorithmic thresholds in +`math/big`, implemented in [calibrate_test.go](calibrate_test.go). + +Basic operations like multiplication and division have many possible implementations. +Most algorithms that are better asymptotically have overheads that make them +run slower for small inputs. When presented with an operation to run, `math/big` +must decide which algorithm to use. + +For example, for small inputs, multiplication using the “grade school algorithm” is fastest. +Given multi-digit x, y and a target z: clear z, and then for each digit y[i], z[i:] += x\*y[i]. +That last operation, adding a vector times a digit to another vector (including carrying up +the vector during the multiplication and addition), can be implemented in a tight assembly loop. +The overall speed is O(N\*\*2) where N is the number of digits in x and y (assume they match), +but the tight inner loop performs well for small inputs. + +[Karatsuba's algorithm](https://en.wikipedia.org/wiki/Karatsuba_algorithm) +multiplies two N-digit numbers by splitting them in half, computing +three N/2-digit products, and then reconstructing the final product using a few more +additions and subtractions. It runs in O(N\*\*log₂ 3) = O(N\*\*1.58) time. +The grade school loop runs faster for small inputs, +but eventually Karatsuba's smaller asymptotic run time wins. + +The multiplication implementation must decide which to use. +Under the assumption that once Karatsuba is faster for some N, +it will be larger for all larger N as well, +the rule is to use Karatsuba's algorithm when the input length N ≥ karatsubaThreshold. + +Calibration is the process of determining what karatsubaThreshold should be set to. +It doesn't sound like it should be that hard, but it is: +- Theoretical analysis does not help: the answer depends on the actual machines +and the actual constant factors in the two implementations. +- We are picking a single karatsubaThreshold for all systems, +despite them having different relative execution speeds for the operations +in the two algorithms. +(We could in theory pick different thresholds for different architectures, +but there can still be significant variation within a given architecture.) +- The assumption that there is a single N where +an asymptotically better algorithm becomes faster and stays faster +is not true in general. +- Recursive algorithms like Karatsuba's may have different optimal +thresholds for different large input sizes. +- Thresholds can interfere. For example, changing the karatsubaThreshold makes +multiplication faster or slower, which in turn affects the best divRecursiveThreshold +(because divisions use multiplication). + +The best we can do is measure the performance of the overall multiplication +algorithm across a variety of inputs and thresholds and look for a threshold +that balances all these concerns reasonably well, +setting thresholds in dependency order (for example, multiplication before division). + +The code in `calibrate_test.go` does this measurement of a variety of input sizes +and threshold values and prints the timing results as a CSV file. +The code in `calibrate_graph.go` reads the CSV and writes out an SVG file plotting the data. +For example: + + go test -run=Calibrate/KaratsubaMul -timeout=1h -calibrate >kmul.csv + go run calibrate_graph.go kmul.csv >kmul.svg + +Any particular input is sensitive to only a few transitions in threshold. +For example, an input of size 320 recurses on inputs of size 160, +which recurses on inputs of size 80, +which recurses on inputs of size 40, +and so on, until falling below the Karatsuba threshold. +Here is what the timing looks like for an input of size 320, +normalized so that 1.0 is the fastest timing observed: + +![KaratsubaThreshold on an Apple M3 Pro, N=320 only](https://swtch.com/math/big/_calibrate/KaratsubaMul/cal.mac320.svg) + +For this input, all thresholds from 21 to 40 perform optimally and identically: they all mean “recurse at N=40 but not at N=20”. +From the single input of size N=320, we cannot decide which of these 20 thresholds is best. + +Other inputs exercise other decision points. For example, here is the timing for N=240: + +![KaratsubaThreshold on an Apple M3 Pro, N=240 only](https://swtch.com/math/big/_calibrate/KaratsubaMul/cal.mac240.svg) + +In this case, all the thresholds from 31 to 60 perform optimally and identically, recursing at N=60 but not N=30. + +If we combine these two into a single graph and then plot the geometric mean of the two lines in blue, +the optimal range becomes a little clearer: + +![KaratsubaThreshold on an Apple M3 Pro](https://swtch.com/math/big/_calibrate/KaratsubaMul/cal.mac240+320.svg) + +The actual calibration runs all possible inputs from size N=200 to N=400, in increments of 8, +plotting all 26 lines in a faded gray (note the changed y-axis scale, zooming in near 1.0). + +![KaratsubaThreshold on an Apple M3 Pro](https://swtch.com/math/big/_calibrate/KaratsubaMul/cal.mac.svg) + +Now the optimal value is clear: the best threshold on this chip, with these algorithmic implementations, is 40. + +Unfortunately, other chips are different. Here is an Intel Xeon server chip: + +![KaratsubaThreshold on an Apple M3 Pro](https://swtch.com/math/big/_calibrate/KaratsubaMul/cal.c2s16.svg) + +On this chip, the best threshold is closer to 60. Luckily, 40 is not a terrible choice either: it is only about 2% slower on average. + +The rest of this document presents the timings measured for the `math/big` thresholds on a variety of machines +and justifies the final thresholds. The timings used these machines: + +- The `gotip-linux-amd64_c3h88-perf_vs_release` gomote, a Google Cloud c3-high-88 machine using an Intel Xeon Platinum 8481C CPU (Emerald Rapids). +- The `gotip-linux-amd64_c2s16-perf_vs_release` gomote, a Google Cloud c2-standard-16 machine using an Intel Xeon Gold 6253CL CPU (Cascade Lake). +- A home server built with an AMD Ryzen 9 7950X CPU. +- The `gotip-linux-arm64_c4as16-perf_vs_release` gomote, a Google Cloud c4a-standard-16 machine using Google's Axiom Arm CPU. +- An Apple MacBook Pro with an Apple M3 Pro CPU. + +In general, we break ties in favor of the newer c3h88 x86 perf gomote, then the c4as16 arm64 perf gomote, and then the others. + +## Karatsuba Multiplication + +Here are the full results for the Karatsuba multiplication threshold. + +![KaratsubaThreshold on an Intel Xeon Platium 8481C](https://swtch.com/math/big/_calibrate/KaratsubaMul/cal.c3h88.svg) +![KaratsubaThreshold on an Intel Xeon Gold 6253CL](https://swtch.com/math/big/_calibrate/KaratsubaMul/cal.c2s16.svg) +![KaratsubaThreshold on an AMD Ryzen 9 7950X](https://swtch.com/math/big/_calibrate/KaratsubaMul/cal.s7.svg) +![KaratsubaThreshold on an Axiom Arm](https://swtch.com/math/big/_calibrate/KaratsubaMul/cal.c4as16.svg) +![KaratsubaThreshold on an Apple M3 Pro](https://swtch.com/math/big/_calibrate/KaratsubaMul/cal.mac.svg) + +The majority of systems have optimum thresholds near 40, so we chose karatsubaThreshold = 40. + +## Basic Squaring + +For squaring a number (`z.Mul(x, x)`), math/big uses grade school multiplication +up to basicSqrThreshold, where it switches to a customized algorithm that is +still quadratic but avoids half the word-by-word multiplies +since the two arguments are identical. +That algorithm's inner loops are not as tight as the grade school multiplication, +so it is slower for small inputs. How small? + +Here are the timings: + +![BasicSqrThreshold on an Intel Xeon Platium 8481C](https://swtch.com/math/big/_calibrate/BasicSqr/cal.c3h88.svg) +![BasicSqrThreshold on an Intel Xeon Gold 6253CL](https://swtch.com/math/big/_calibrate/BasicSqr/cal.c2s16.svg) +![BasicSqrThreshold on an AMD Ryzen 9 7950X](https://swtch.com/math/big/_calibrate/BasicSqr/cal.s7.svg) +![BasicSqrThreshold on an Axiom Arm](https://swtch.com/math/big/_calibrate/BasicSqr/cal.c4as16.svg) +![BasicSqrThreshold on an Apple M3 Pro](https://swtch.com/math/big/_calibrate/BasicSqr/cal.mac.svg) + +These inputs are so small that the calibration times batches of 100 instead of individual operations. +There is no one best threshold, even on a single system, because some of the sizes seem to run +the grade school algorithm faster than others. +For example, on the AMD CPU, +for N=14, basic squaring is 4% faster than basic multiplication, +suggesting the threshold has been crossed, +but for N=16, basic multiplication is 9% faster than basic squaring, +probably because the tight assembly can use larger chunks. + +It is unclear why the Axiom Arm timings are so incredibly noisy. + +We chose basicSqrThreshold = 12. + +## Karatsuba Squaring + +Beyond the basic squaring threshold, at some point a customized Karatsuba can take over. +It uses three half-sized squarings instead of three half-sized multiplies. +Here are the timings: + +![KaratsubaSqrThreshold on an Intel Xeon Platium 8481C](https://swtch.com/math/big/_calibrate/KaratsubaSqr/cal.c3h88.svg) +![KaratsubaSqrThreshold on an Intel Xeon Gold 6253CL](https://swtch.com/math/big/_calibrate/KaratsubaSqr/cal.c2s16.svg) +![KaratsubaSqrThreshold on an AMD Ryzen 9 7950X](https://swtch.com/math/big/_calibrate/KaratsubaSqr/cal.s7.svg) +![KaratsubaSqrThreshold on an Axiom Arm](https://swtch.com/math/big/_calibrate/KaratsubaSqr/cal.c4as16.svg) +![KaratsubaSqrThreshold on an Apple M3 Pro](https://swtch.com/math/big/_calibrate/KaratsubaSqr/cal.mac.svg) + +The majority of chips preferred a lower threshold, around 60-70, +but the older Intel Xeon and the AMD prefer a threshold around 100-120. + +We chose karatsubaSqrThreshold = 80, which is within 2% of optimal on all the chips. + +## Recursive Division + +Division uses a recursive divide-and-conquer algorithm for large inputs, +eventually falling back to a more traditional grade-school whole-input trial-and-error division. +Here are the timings for the threshold between the two: + +![DivRecursiveThreshold on an Intel Xeon Platium 8481C](https://swtch.com/math/big/_calibrate/DivRecursive/cal.c3h88.svg) +![DivRecursiveThreshold on an Intel Xeon Gold 6253CL](https://swtch.com/math/big/_calibrate/DivRecursive/cal.c2s16.svg) +![DivRecursiveThreshold on an AMD Ryzen 9 7950X](https://swtch.com/math/big/_calibrate/DivRecursive/cal.s7.svg) +![DivRecursiveThreshold on an Axiom Arm](https://swtch.com/math/big/_calibrate/DivRecursive/cal.c4as16.svg) +![DivRecursiveThreshold on an Apple M3 Pro](https://swtch.com/math/big/_calibrate/DivRecursive/cal.mac.svg) + +We chose divRecursiveThreshold = 40. diff --git a/src/math/big/calibrate_graph.go b/src/math/big/calibrate_graph.go new file mode 100644 index 00000000000000..37596195a1e5b5 --- /dev/null +++ b/src/math/big/calibrate_graph.go @@ -0,0 +1,321 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ignore + +// This program converts CSV calibration data printed by +// +// go test -run=Calibrate/Name -calibrate >file.csv +// +// into an SVG file. Invoke as: +// +// go run calibrate_graph.go file.csv >file.svg +// +// See calibrate.md for more details. + +package main + +import ( + "bytes" + "encoding/csv" + "flag" + "fmt" + "log" + "math" + "os" + "strconv" +) + +func usage() { + fmt.Fprintf(os.Stderr, "usage: go run calibrate_graph.go file.csv >file.svg\n") + os.Exit(2) +} + +// A Point is an X, Y coordinate in the data being plotted. +type Point struct { + X, Y float64 +} + +// A Graph is a graph to draw as SVG. +type Graph struct { + Title string // title above graph + Geomean []Point // geomean line + Lines [][]Point // normalized data lines + XAxis string // x-axis label + YAxis string // y-axis label + Min Point // min point of data display + Max Point // max point of data display +} + +var yMax = flag.Float64("ymax", 1.2, "maximum y axis value") +var alphaNorm = flag.Float64("alphanorm", 0.1, "alpha for a single norm line") + +func main() { + flag.Usage = usage + flag.Parse() + if flag.NArg() != 1 { + usage() + } + + // Read CSV. It may be enclosed in + // -- name.csv -- + // ... + // -- eof -- + // framing, in which case remove the framing. + fdata, err := os.ReadFile(flag.Arg(0)) + if err != nil { + log.Fatal(err) + } + if _, after, ok := bytes.Cut(fdata, []byte(".csv --\n")); ok { + fdata = after + } + if before, _, ok := bytes.Cut(fdata, []byte("-- eof --\n")); ok { + fdata = before + } + rd := csv.NewReader(bytes.NewReader(fdata)) + rd.FieldsPerRecord = -1 + records, err := rd.ReadAll() + if err != nil { + log.Fatal(err) + } + + // Construct graph from loaded CSV. + // CSV starts with metadata lines like + // goos,darwin + // and then has two tables of timings. + // Each table looks like + // size \ threshold,10,20,30,40 + // 100,1,2,3,4 + // 200,2,3,4,5 + // 300,3,4,5,6 + // 400,4,5,6,7 + // 500,5,6,7,8 + // The header line gives the threshold values and then each row + // gives an input size and the timings for each threshold. + // Omitted timings are empty strings and turn into infinities when parsing. + // The first table gives raw nanosecond timings. + // The second table gives timings normalized relative to the fastest + // possible threshold for a given input size. + // We only want the second table. + // The tables are followed by a list of geomeans of all the normalized + // timings for each threshold: + // geomean,1.2,1.1,1.0,1.4 + // We turn each normalized timing row into a line in the graph, + // and we turn the geomean into an overlaid thick line. + // The metadata is used for preparing the titles. + g := &Graph{ + YAxis: "Relative Slowdown", + Min: Point{0, 1}, + Max: Point{1, 1.2}, + } + meta := make(map[string]string) + table := 0 // number of table headers seen + var thresholds []float64 + maxNorm := 0.0 + for _, rec := range records { + if len(rec) == 0 { + continue + } + if len(rec) == 2 { + meta[rec[0]] = rec[1] + continue + } + if rec[0] == `size \ threshold` { + table++ + if table == 2 { + thresholds = parseFloats(rec) + g.Min.X = thresholds[0] + g.Max.X = thresholds[len(thresholds)-1] + } + continue + } + if rec[0] == "geomean" { + table = 3 // end of norms table + geomeans := parseFloats(rec) + g.Geomean = floatsToLine(thresholds, geomeans) + continue + } + if table == 2 { + if _, err := strconv.Atoi(rec[0]); err != nil { // size + log.Fatalf("invalid table line: %q", rec) + } + norms := parseFloats(rec) + if len(norms) > len(thresholds) { + log.Fatalf("too many timings (%d > %d): %q", len(norms), len(thresholds), rec) + } + g.Lines = append(g.Lines, floatsToLine(thresholds, norms)) + for _, y := range norms { + maxNorm = max(maxNorm, y) + } + continue + } + } + + g.Max.Y = min(*yMax, math.Ceil(maxNorm*100)/100) + g.XAxis = meta["calibrate"] + "Threshold" + g.Title = meta["goos"] + "/" + meta["goarch"] + " " + meta["cpu"] + + os.Stdout.Write(g.SVG()) +} + +// parseFloats parses rec[1:] as floating point values. +// If a field is the empty string, it is represented as +Inf. +func parseFloats(rec []string) []float64 { + floats := make([]float64, 0, len(rec)-1) + for _, v := range rec[1:] { + if v == "" { + floats = append(floats, math.Inf(+1)) + continue + } + f, err := strconv.ParseFloat(v, 64) + if err != nil { + log.Fatalf("invalid record: %q (%v)", rec, err) + } + floats = append(floats, f) + } + return floats +} + +// floatsToLine converts a sequence of floats into a line, ignoring missing (infinite) values. +func floatsToLine(x, y []float64) []Point { + var line []Point + for i, yi := range y { + if !math.IsInf(yi, 0) { + line = append(line, Point{x[i], yi}) + } + } + return line +} + +const svgHeader = ` + + + + +` + +// Layout constants for drawing graph +const ( + DX = 600 // width of graphed data + DY = 150 // height of graphed data + ML = 80 // margin left + MT = 30 // margin top + MR = 10 // margin right + MB = 50 // margin bottom + PS = 14 // point size of text + W = ML + DX + MR // width of overall graph + H = MT + DY + MB // height of overall graph + Tick = 5 // axis tick length +) + +// An SVGPoint is a point in the SVG image, in pixel units, +// with Y increasing down the page. +type SVGPoint struct { + X, Y int +} + +func (p SVGPoint) String() string { + return fmt.Sprintf("%d,%d", p.X, p.Y) +} + +// pt converts an x, y data value (such as from a Point) to an SVGPoint. +func (g *Graph) pt(x, y float64) SVGPoint { + return SVGPoint{ + X: ML + int((x-g.Min.X)/(g.Max.X-g.Min.X)*DX), + Y: H - MB - int((y-g.Min.Y)/(g.Max.Y-g.Min.Y)*DY), + } +} + +// SVG returns the SVG text for the graph. +func (g *Graph) SVG() []byte { + + var svg bytes.Buffer + fmt.Fprintf(&svg, svgHeader, W, H, PS, PS, *alphaNorm) + + // Draw data, clipped. + fmt.Fprintf(&svg, "\n", + g.pt(g.Min.X, g.Min.Y), g.pt(g.Max.X, g.Min.Y), g.pt(g.Max.X, g.Max.Y), g.pt(g.Min.X, g.Max.Y)) + fmt.Fprintf(&svg, "\n") + for _, line := range g.Lines { + if len(line) == 0 { + continue + } + fmt.Fprintf(&svg, "\n") + } + // Draw geomean. + if len(g.Geomean) > 0 { + line := g.Geomean + fmt.Fprintf(&svg, "\n") + } + fmt.Fprintf(&svg, "\n") + + // Draw axes and major and minor tick marks. + fmt.Fprintf(&svg, " 0.5 { + yscale = 10 + } + for y := int(math.Ceil(g.Min.Y * yscale)); float64(y) <= g.Max.Y*yscale; y++ { + if y%5 != 0 { + fmt.Fprintf(&svg, " M %v l -%d,0", g.pt(g.Min.X, float64(y)/yscale), Tick) + } else { + fmt.Fprintf(&svg, " M %v l -%d,0", g.pt(g.Min.X, float64(y)/yscale), 2*Tick) + } + } + fmt.Fprintf(&svg, "\"/>\n") + + // Draw tick labels on major marks. + for x := int(math.Ceil(g.Min.X / xscale)); float64(x)*xscale <= g.Max.X; x++ { + if x%5 == 0 { + p := g.pt(float64(x)*xscale, g.Min.Y) + fmt.Fprintf(&svg, "%d\n", p.X, p.Y+2*Tick+PS, x*int(xscale)) + } + } + for y := int(math.Ceil(g.Min.Y * yscale)); float64(y) <= g.Max.Y*yscale; y++ { + if y%5 == 0 { + p := g.pt(g.Min.X, float64(y)/yscale) + fmt.Fprintf(&svg, "%.2f\n", p.X-2*Tick-Tick, p.Y+PS/3, float64(y)/yscale) + } + } + + // Draw graph title and axis titles. + fmt.Fprintf(&svg, "%s\n", ML+DX/2, MT-PS/3, g.Title) + fmt.Fprintf(&svg, "%s\n", ML+DX/2, MT+DY+2*Tick+2*PS+PS/2, g.XAxis) + fmt.Fprintf(&svg, "%s\n", ML-Tick-Tick-3*PS, MT+DY/2, g.YAxis) + + fmt.Fprintf(&svg, "\n") + return svg.Bytes() +} diff --git a/src/math/big/calibrate_test.go b/src/math/big/calibrate_test.go index d85833aedef619..7d44c2ed0f02ed 100644 --- a/src/math/big/calibrate_test.go +++ b/src/math/big/calibrate_test.go @@ -2,172 +2,266 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Calibration used to determine thresholds for using -// different algorithms. Ideally, this would be converted -// to go generate to create thresholds.go - -// This file prints execution times for the Mul benchmark -// given different Karatsuba thresholds. The result may be -// used to manually fine-tune the threshold constant. The -// results are somewhat fragile; use repeated runs to get -// a clear picture. - -// Calculates lower and upper thresholds for when basicSqr -// is faster than standard multiplication. - -// Usage: go test -run='^TestCalibrate$' -v -calibrate +// TestCalibrate determines appropriate thresholds for when to use +// different calculation algorithms. To run it, use: +// +// go test -run=Calibrate -calibrate >cal.log +// +// Calibration data is printed in CSV format, along with the normal test output. +// See calibrate.md for more details about using the output. package big import ( "flag" "fmt" + "internal/sysinfo" + "math" + "runtime" + "slices" + "strings" + "sync" "testing" "time" ) var calibrate = flag.Bool("calibrate", false, "run calibration test") - -const ( - sqrModeMul = "mul(x, x)" - sqrModeBasic = "basicSqr(x)" - sqrModeKaratsuba = "karatsubaSqr(x)" -) +var calibrateOnce sync.Once func TestCalibrate(t *testing.T) { if !*calibrate { return } - computeKaratsubaThresholds() + t.Run("KaratsubaMul", computeKaratsubaThreshold) + t.Run("BasicSqr", computeBasicSqrThreshold) + t.Run("KaratsubaSqr", computeKaratsubaSqrThreshold) + t.Run("DivRecursive", computeDivRecursiveThreshold) +} + +func computeKaratsubaThreshold(t *testing.T) { + set := func(n int) { karatsubaThreshold = n } + computeThreshold(t, "karatsuba", set, 0, 4, 200, benchMul, 200, 8, 400) +} - // compute basicSqrThreshold where overhead becomes negligible - minSqr := computeSqrThreshold(10, 30, 1, 3, sqrModeMul, sqrModeBasic) - // compute karatsubaSqrThreshold where karatsuba is faster - maxSqr := computeSqrThreshold(200, 500, 10, 3, sqrModeBasic, sqrModeKaratsuba) - if minSqr != 0 { - fmt.Printf("found basicSqrThreshold = %d\n", minSqr) - } else { - fmt.Println("no basicSqrThreshold found") +func benchMul(size int) func() { + x := rndNat(size) + y := rndNat(size) + var z nat + return func() { + z.mul(nil, x, y) } - if maxSqr != 0 { - fmt.Printf("found karatsubaSqrThreshold = %d\n", maxSqr) - } else { - fmt.Println("no karatsubaSqrThreshold found") +} + +func computeBasicSqrThreshold(t *testing.T) { + setDuringTest(t, &karatsubaSqrThreshold, 1e9) + set := func(n int) { basicSqrThreshold = n } + computeThreshold(t, "basicSqr", set, 2, 1, 40, benchBasicSqr, 1, 1, 40) +} + +func benchBasicSqr(size int) func() { + x := rndNat(size) + var z nat + return func() { + // Run 100 squarings because 1 is too fast at the small sizes we consider. + // Some systems don't even have precise enough clocks to measure it accurately. + for range 100 { + z.sqr(nil, x) + } } } -func karatsubaLoad(b *testing.B) { - BenchmarkMul(b) +func computeKaratsubaSqrThreshold(t *testing.T) { + set := func(n int) { karatsubaSqrThreshold = n } + computeThreshold(t, "karatsubaSqr", set, 0, 4, 200, benchSqr, 200, 8, 400) } -// measureKaratsuba returns the time to run a Karatsuba-relevant benchmark -// given Karatsuba threshold th. -func measureKaratsuba(th int) time.Duration { - th, karatsubaThreshold = karatsubaThreshold, th - res := testing.Benchmark(karatsubaLoad) - karatsubaThreshold = th - return time.Duration(res.NsPerOp()) +func benchSqr(size int) func() { + x := rndNat(size) + var z nat + return func() { + z.sqr(nil, x) + } } -func computeKaratsubaThresholds() { - fmt.Printf("Multiplication times for varying Karatsuba thresholds\n") - fmt.Printf("(run repeatedly for good results)\n") +func computeDivRecursiveThreshold(t *testing.T) { + set := func(n int) { divRecursiveThreshold = n } + computeThreshold(t, "divRecursive", set, 4, 4, 200, benchDiv, 200, 8, 400) +} - // determine Tk, the work load execution time using basic multiplication - Tb := measureKaratsuba(1e9) // th == 1e9 => Karatsuba multiplication disabled - fmt.Printf("Tb = %10s\n", Tb) +func benchDiv(size int) func() { + divx := rndNat(2 * size) + divy := rndNat(size) + var z, r nat + return func() { + z.div(nil, r, divx, divy) + } +} - // thresholds - th := 4 - th1 := -1 - th2 := -1 +func computeThreshold(t *testing.T, name string, set func(int), thresholdLo, thresholdStep, thresholdHi int, bench func(int) func(), sizeLo, sizeStep, sizeHi int) { + // Start CSV output; wrapped in txtar framing to separate CSV from other test ouptut. + fmt.Printf("-- calibrate-%s.csv --\n", name) + defer fmt.Printf("-- eof --\n") - var deltaOld time.Duration - for count := -1; count != 0 && th < 128; count-- { - // determine Tk, the work load execution time using Karatsuba multiplication - Tk := measureKaratsuba(th) + fmt.Printf("goos,%s\n", runtime.GOOS) + fmt.Printf("goarch,%s\n", runtime.GOARCH) + fmt.Printf("cpu,%s\n", sysinfo.CPUName()) + fmt.Printf("calibrate,%s\n", name) - // improvement over Tb - delta := (Tb - Tk) * 100 / Tb + // Expand lists of sizes and thresholds we will test. + var sizes, thresholds []int + for size := sizeLo; size <= sizeHi; size += sizeStep { + sizes = append(sizes, size) + } + for thresh := thresholdLo; thresh <= thresholdHi; thresh += thresholdStep { + thresholds = append(thresholds, thresh) + } - fmt.Printf("th = %3d Tk = %10s %4d%%", th, Tk, delta) + fmt.Printf("%s\n", csv("size \\ threshold", thresholds)) - // determine break-even point - if Tk < Tb && th1 < 0 { - th1 = th - fmt.Print(" break-even point") + // Track minimum time observed for each size, threshold pair. + times := make([][]float64, len(sizes)) + for i := range sizes { + times[i] = make([]float64, len(thresholds)) + for j := range thresholds { + times[i][j] = math.Inf(+1) } + } - // determine diminishing return - if 0 < delta && delta < deltaOld && th2 < 0 { - th2 = th - fmt.Print(" diminishing return") - } - deltaOld = delta + // For each size, run at most MaxRounds of considering every threshold. + // If we run a threshold Stable times in a row without seeing more + // than a 1% improvement in the observed minimum, move on to the next one. + // After we run Converged rounds (not necessarily in a row) + // without seeing any threshold improve by more than 1%, stop. + const ( + MaxRounds = 1600 + Stable = 20 + Converged = 200 + ) - fmt.Println() + for i, size := range sizes { + b := bench(size) + same := 0 + for range MaxRounds { + better := false + for j, threshold := range thresholds { + // No point if threshold is far beyond size + if false && threshold > size+2*sizeStep { + continue + } - // trigger counter - if th1 >= 0 && th2 >= 0 && count < 0 { - count = 10 // this many extra measurements after we got both thresholds + // BasicSqr is different from the recursive thresholds: it either applies or not, + // without any question of recursive subproblems. Only try the thresholds + // size-1, size, size+1, size+2 + // to get two data points using basic multiplication and two using basic squaring. + // This avoids gathering many redundant data points. + // (The others have redundant data points as well, but for them the math is less trivial + // and best not duplicated in the calibration code.) + if false && name == "basicSqr" && (threshold < size-1 || threshold > size+3) { + continue + } + + set(threshold) + b() // warm up + b() + tmin := times[i][j] + for k := 0; k < Stable; k++ { + start := time.Now() + b() + t := float64(time.Since(start)) + if t < tmin { + if t < tmin*99/100 { + better = true + k = 0 + } + tmin = t + } + } + times[i][j] = tmin + } + if !better { + if same++; same >= Converged { + break + } + } } - th++ + fmt.Printf("%s\n", csv(fmt.Sprint(size), times[i])) } -} -func measureSqr(words, nruns int, mode string) time.Duration { - // more runs for better statistics - initBasicSqr, initKaratsubaSqr := basicSqrThreshold, karatsubaSqrThreshold - - switch mode { - case sqrModeMul: - basicSqrThreshold = words + 1 - case sqrModeBasic: - basicSqrThreshold, karatsubaSqrThreshold = words-1, words+1 - case sqrModeKaratsuba: - karatsubaSqrThreshold = words - 1 + // For each size, normalize timings by the minimum achieved for that size. + fmt.Printf("%s\n", csv("size \\ threshold", thresholds)) + norms := make([][]float64, len(sizes)) + for i, times := range times { + m := min(1e100, slices.Min(times)) // make finite so divide preserves inf values + norms[i] = make([]float64, len(times)) + for j, d := range times { + norms[i][j] = d / m + } + fmt.Printf("%s\n", csv(fmt.Sprint(sizes[i]), norms[i])) } - var testval int64 - for i := 0; i < nruns; i++ { - res := testing.Benchmark(func(b *testing.B) { benchmarkNatSqr(b, words) }) - testval += res.NsPerOp() + // For each threshold, compute geomean of normalized timings across all sizes. + geomeans := make([]float64, len(thresholds)) + for j := range thresholds { + p := 1.0 + n := 0 + for i := range sizes { + if v := norms[i][j]; !math.IsInf(v, +1) { + p *= v + n++ + } + } + if n == 0 { + geomeans[j] = math.Inf(+1) + } else { + geomeans[j] = math.Pow(p, 1/float64(n)) + } } - testval /= int64(nruns) + fmt.Printf("%s\n", csv("geomean", geomeans)) - basicSqrThreshold, karatsubaSqrThreshold = initBasicSqr, initKaratsubaSqr + // Add best threshold and smallest, largest within 10% and 5% of best. + var lo10, lo5, best, hi5, hi10 int + for i, g := range geomeans { + if g < geomeans[best] { + best = i + } + } + lo5 = best + for lo5 > 0 && geomeans[lo5-1] <= 1.05 { + lo5-- + } + lo10 = lo5 + for lo10 > 0 && geomeans[lo10-1] <= 1.10 { + lo10-- + } + hi5 = best + for hi5+1 < len(geomeans) && geomeans[hi5+1] <= 1.05 { + hi5++ + } + hi10 = hi5 + for hi10+1 < len(geomeans) && geomeans[hi10+1] <= 1.10 { + hi10++ + } + fmt.Printf("lo10%%,%d\n", thresholds[lo10]) + fmt.Printf("lo5%%,%d\n", thresholds[lo5]) + fmt.Printf("min,%d\n", thresholds[best]) + fmt.Printf("hi5%%,%d\n", thresholds[hi5]) + fmt.Printf("hi10%%,%d\n", thresholds[hi10]) - return time.Duration(testval) + set(thresholds[best]) } -func computeSqrThreshold(from, to, step, nruns int, lower, upper string) int { - fmt.Printf("Calibrating threshold between %s and %s\n", lower, upper) - fmt.Printf("Looking for a timing difference for x between %d - %d words by %d step\n", from, to, step) - var initPos bool - var threshold int - for i := from; i <= to; i += step { - baseline := measureSqr(i, nruns, lower) - testval := measureSqr(i, nruns, upper) - pos := baseline > testval - delta := baseline - testval - percent := delta * 100 / baseline - fmt.Printf("words = %3d deltaT = %10s (%4d%%) is %s better: %v", i, delta, percent, upper, pos) - if i == from { - initPos = pos - } - if threshold == 0 && pos != initPos { - threshold = i - fmt.Printf(" threshold found") +// csv returns a single csv line starting with name and followed by the values. +// Values that are float64 +infinity, denoting missing data, are replaced by an empty string. +func csv[T int | float64](name string, values []T) string { + line := []string{name} + for _, v := range values { + if math.IsInf(float64(v), +1) { + line = append(line, "") + } else { + line = append(line, fmt.Sprint(v)) } - fmt.Println() - - } - if threshold != 0 { - fmt.Printf("Found threshold = %d between %d - %d\n", threshold, from, to) - } else { - fmt.Printf("Found NO threshold between %d - %d\n", from, to) } - return threshold + return strings.Join(line, ",") } diff --git a/src/math/big/decimal.go b/src/math/big/decimal.go index 716f03bfa43398..9e391adef94fc2 100644 --- a/src/math/big/decimal.go +++ b/src/math/big/decimal.go @@ -69,13 +69,13 @@ func (x *decimal) init(m nat, shift int) { if s >= ntz { s = ntz // shift at most ntz bits } - m = nat(nil).shr(m, s) + m = nat(nil).rsh(m, s) shift += int(s) } // Do any shift left in binary representation. if shift > 0 { - m = nat(nil).shl(m, uint(shift)) + m = nat(nil).lsh(m, uint(shift)) shift = 0 } @@ -93,15 +93,15 @@ func (x *decimal) init(m nat, shift int) { // Do any (remaining) shift right in decimal representation. if shift < 0 { for shift < -maxShift { - shr(x, maxShift) + rsh(x, maxShift) shift += maxShift } - shr(x, uint(-shift)) + rsh(x, uint(-shift)) } } -// shr implements x >> s, for s <= maxShift. -func shr(x *decimal, s uint) { +// rsh implements x >> s, for s <= maxShift. +func rsh(x *decimal, s uint) { // Division by 1<= 1 { + t.Errorf("Mul(len %d, len %d) allocates %.2f objects", size, size, n) + } + } +} + +func TestSqrAlloc(t *testing.T) { + r := rand.New(rand.NewSource(1234)) + sizes := []int{basicSqrThreshold / 2, basicSqrThreshold, karatsubaSqrThreshold} + for _, size := range sizes { + x := randInt(r, uint(size)) + z := &Int{abs: make(nat, 2*uint(size))} + if n := testing.AllocsPerRun(10, func() { z.Mul(x, x) }); n >= 1 { + t.Errorf("Mul(len %d with itself) allocates %.2f objects", size, n) + } + } +} diff --git a/src/math/big/float.go b/src/math/big/float.go index 813c4ebfa7477f..b99088a733f5df 100644 --- a/src/math/big/float.go +++ b/src/math/big/float.go @@ -488,7 +488,7 @@ func (z *Float) round(sbit uint) { } z.exp++ // adjust mantissa: divide by 2 to compensate for exponent adjustment - shrVU(z.mant, z.mant, 1) + rshVU(z.mant, z.mant, 1) // set msb == carry == 1 from the mantissa overflow above const msb = 1 << (_W - 1) z.mant[n-1] |= msb @@ -585,9 +585,9 @@ func fnorm(m nat) int64 { } s := nlz(m[len(m)-1]) if s > 0 { - c := shlVU(m, m, s) + c := lshVU(m, m, s) if debugFloat && c != 0 { - panic("nlz or shlVU incorrect") + panic("nlz or lshVU incorrect") } } return int64(s) @@ -602,7 +602,7 @@ func (z *Float) SetInt(x *Int) *Float { // are many trailing 0's. bits := uint32(x.BitLen()) if z.prec == 0 { - z.prec = umax32(bits, 64) + z.prec = max(bits, 64) } z.acc = Exact z.neg = x.neg @@ -628,7 +628,7 @@ func (z *Float) SetRat(x *Rat) *Float { a.SetInt(x.Num()) b.SetInt(x.Denom()) if z.prec == 0 { - z.prec = umax32(a.prec, b.prec) + z.prec = max(a.prec, b.prec) } return z.Quo(&a, &b) } @@ -1110,11 +1110,11 @@ func (x *Float) Int(z *Int) (*Int, Accuracy) { z.neg = x.neg switch { case exp > allBits: - z.abs = z.abs.shl(x.mant, exp-allBits) + z.abs = z.abs.lsh(x.mant, exp-allBits) default: z.abs = z.abs.set(x.mant) case exp < allBits: - z.abs = z.abs.shr(x.mant, allBits-exp) + z.abs = z.abs.rsh(x.mant, allBits-exp) } return z, acc @@ -1150,7 +1150,7 @@ func (x *Float) Rat(z *Rat) (*Rat, Accuracy) { z.a.neg = x.neg switch { case x.exp > allBits: - z.a.abs = z.a.abs.shl(x.mant, uint(x.exp-allBits)) + z.a.abs = z.a.abs.lsh(x.mant, uint(x.exp-allBits)) z.b.abs = z.b.abs[:0] // == 1 (see Rat) // z already in normal form default: @@ -1160,7 +1160,7 @@ func (x *Float) Rat(z *Rat) (*Rat, Accuracy) { case x.exp < allBits: z.a.abs = z.a.abs.set(x.mant) t := z.b.abs.setUint64(1) - z.b.abs = t.shl(t, uint(allBits-x.exp)) + z.b.abs = t.lsh(t, uint(allBits-x.exp)) z.norm() } return z, Exact @@ -1234,10 +1234,10 @@ func (z *Float) uadd(x, y *Float) { switch { case ex < ey: if al { - t := nat(nil).shl(y.mant, uint(ey-ex)) + t := nat(nil).lsh(y.mant, uint(ey-ex)) z.mant = z.mant.add(x.mant, t) } else { - z.mant = z.mant.shl(y.mant, uint(ey-ex)) + z.mant = z.mant.lsh(y.mant, uint(ey-ex)) z.mant = z.mant.add(x.mant, z.mant) } default: @@ -1245,10 +1245,10 @@ func (z *Float) uadd(x, y *Float) { z.mant = z.mant.add(x.mant, y.mant) case ex > ey: if al { - t := nat(nil).shl(x.mant, uint(ex-ey)) + t := nat(nil).lsh(x.mant, uint(ex-ey)) z.mant = z.mant.add(t, y.mant) } else { - z.mant = z.mant.shl(x.mant, uint(ex-ey)) + z.mant = z.mant.lsh(x.mant, uint(ex-ey)) z.mant = z.mant.add(z.mant, y.mant) } ex = ey @@ -1279,10 +1279,10 @@ func (z *Float) usub(x, y *Float) { switch { case ex < ey: if al { - t := nat(nil).shl(y.mant, uint(ey-ex)) + t := nat(nil).lsh(y.mant, uint(ey-ex)) z.mant = t.sub(x.mant, t) } else { - z.mant = z.mant.shl(y.mant, uint(ey-ex)) + z.mant = z.mant.lsh(y.mant, uint(ey-ex)) z.mant = z.mant.sub(x.mant, z.mant) } default: @@ -1290,10 +1290,10 @@ func (z *Float) usub(x, y *Float) { z.mant = z.mant.sub(x.mant, y.mant) case ex > ey: if al { - t := nat(nil).shl(x.mant, uint(ex-ey)) + t := nat(nil).lsh(x.mant, uint(ex-ey)) z.mant = t.sub(t, y.mant) } else { - z.mant = z.mant.shl(x.mant, uint(ex-ey)) + z.mant = z.mant.lsh(x.mant, uint(ex-ey)) z.mant = z.mant.sub(z.mant, y.mant) } ex = ey @@ -1327,9 +1327,9 @@ func (z *Float) umul(x, y *Float) { e := int64(x.exp) + int64(y.exp) if x == y { - z.mant = z.mant.sqr(x.mant) + z.mant = z.mant.sqr(nil, x.mant) } else { - z.mant = z.mant.mul(x.mant, y.mant) + z.mant = z.mant.mul(nil, x.mant, y.mant) } z.setExpAndRound(e-fnorm(z.mant), 0) } @@ -1363,8 +1363,10 @@ func (z *Float) uquo(x, y *Float) { d := len(xadj) - len(y.mant) // divide + stk := getStack() + defer stk.free() var r nat - z.mant, r = z.mant.div(nil, xadj, y.mant) + z.mant, r = z.mant.div(stk, nil, xadj, y.mant) e := int64(x.exp) - int64(y.exp) - int64(d-len(z.mant))*_W // The result is long enough to include (at least) the rounding bit. @@ -1451,7 +1453,7 @@ func (z *Float) Add(x, y *Float) *Float { } if z.prec == 0 { - z.prec = umax32(x.prec, y.prec) + z.prec = max(x.prec, y.prec) } if x.form == finite && y.form == finite { @@ -1525,7 +1527,7 @@ func (z *Float) Sub(x, y *Float) *Float { } if z.prec == 0 { - z.prec = umax32(x.prec, y.prec) + z.prec = max(x.prec, y.prec) } if x.form == finite && y.form == finite { @@ -1592,7 +1594,7 @@ func (z *Float) Mul(x, y *Float) *Float { } if z.prec == 0 { - z.prec = umax32(x.prec, y.prec) + z.prec = max(x.prec, y.prec) } z.neg = x.neg != y.neg @@ -1637,7 +1639,7 @@ func (z *Float) Quo(x, y *Float) *Float { } if z.prec == 0 { - z.prec = umax32(x.prec, y.prec) + z.prec = max(x.prec, y.prec) } z.neg = x.neg != y.neg @@ -1724,10 +1726,3 @@ func (x *Float) ord() int { } return m } - -func umax32(x, y uint32) uint32 { - if x > y { - return x - } - return y -} diff --git a/src/math/big/floatmarsh.go b/src/math/big/floatmarsh.go index 16be94697147ed..e220cbc91a8789 100644 --- a/src/math/big/floatmarsh.go +++ b/src/math/big/floatmarsh.go @@ -48,10 +48,10 @@ func (x *Float) GobEncode() ([]byte, error) { b |= 1 } buf[1] = b - byteorder.BePutUint32(buf[2:], x.prec) + byteorder.BEPutUint32(buf[2:], x.prec) if x.form == finite { - byteorder.BePutUint32(buf[6:], uint32(x.exp)) + byteorder.BEPutUint32(buf[6:], uint32(x.exp)) x.mant[len(x.mant)-n:].bytes(buf[10:]) // cut off unused trailing words } @@ -84,13 +84,13 @@ func (z *Float) GobDecode(buf []byte) error { z.acc = Accuracy((b>>3)&3) - 1 z.form = form((b >> 1) & 3) z.neg = b&1 != 0 - z.prec = byteorder.BeUint32(buf[2:]) + z.prec = byteorder.BEUint32(buf[2:]) if z.form == finite { if len(buf) < 10 { return errors.New("Float.GobDecode: buffer too small for finite form float") } - z.exp = int32(byteorder.BeUint32(buf[6:])) + z.exp = int32(byteorder.BEUint32(buf[6:])) z.mant = z.mant.setBytes(buf[10:]) } @@ -106,15 +106,21 @@ func (z *Float) GobDecode(buf []byte) error { return nil } -// MarshalText implements the [encoding.TextMarshaler] interface. +// AppendText implements the [encoding.TextAppender] interface. // Only the [Float] value is marshaled (in full precision), other // attributes such as precision or accuracy are ignored. -func (x *Float) MarshalText() (text []byte, err error) { +func (x *Float) AppendText(b []byte) ([]byte, error) { if x == nil { - return []byte(""), nil + return append(b, ""...), nil } - var buf []byte - return x.Append(buf, 'g', -1), nil + return x.Append(b, 'g', -1), nil +} + +// MarshalText implements the [encoding.TextMarshaler] interface. +// Only the [Float] value is marshaled (in full precision), other +// attributes such as precision or accuracy are ignored. +func (x *Float) MarshalText() (text []byte, err error) { + return x.AppendText(nil) } // UnmarshalText implements the [encoding.TextUnmarshaler] interface. diff --git a/src/math/big/floatmarsh_test.go b/src/math/big/floatmarsh_test.go index 20def68a6d3c61..339cb537642c1b 100644 --- a/src/math/big/floatmarsh_test.go +++ b/src/math/big/floatmarsh_test.go @@ -171,3 +171,46 @@ func TestFloatGobDecodeInvalid(t *testing.T) { } } } + +func TestFloatAppendText(t *testing.T) { + for _, test := range floatVals { + for _, sign := range []string{"", "+", "-"} { + for _, prec := range []uint{0, 1, 2, 10, 53, 64, 100, 1000} { + if prec > 53 && testing.Short() { + continue + } + x := sign + test + var tx Float + _, _, err := tx.SetPrec(prec).Parse(x, 0) + if err != nil { + t.Errorf("parsing of %s (prec = %d) failed (invalid test case): %v", x, prec, err) + continue + } + buf := make([]byte, 4, 32) + b, err := tx.AppendText(buf) + if err != nil { + t.Errorf("marshaling of %v (prec = %d) failed: %v", &tx, prec, err) + continue + } + var rx Float + rx.SetPrec(prec) + if err := rx.UnmarshalText(b[4:]); err != nil { + t.Errorf("unmarshaling of %v (prec = %d) failed: %v", &tx, prec, err) + continue + } + if rx.Cmp(&tx) != 0 { + t.Errorf("AppendText of %v (prec = %d) failed: got %v want %v", &tx, prec, &rx, &tx) + } + } + } + } +} + +func TestFloatAppendTextNil(t *testing.T) { + var x *Float + buf := make([]byte, 4, 16) + data, _ := x.AppendText(buf) + if string(data[4:]) != "" { + t.Errorf("got %q, want ", data[4:]) + } +} diff --git a/src/math/big/ftoa.go b/src/math/big/ftoa.go index f7a4345d3acf98..c94fbeee440189 100644 --- a/src/math/big/ftoa.go +++ b/src/math/big/ftoa.go @@ -188,9 +188,9 @@ func roundShortest(d *decimal, x *Float) { s := mant.bitLen() - int(x.prec+1) switch { case s < 0: - mant = mant.shl(mant, uint(-s)) + mant = mant.lsh(mant, uint(-s)) case s > 0: - mant = mant.shr(mant, uint(+s)) + mant = mant.rsh(mant, uint(+s)) } exp += s // x = mant * 2**exp with lsb(mant) == 1/2 ulp of x.prec @@ -309,7 +309,7 @@ func fmtF(buf []byte, prec int, d decimal) []byte { } // fmtB appends the string of x in the format mantissa "p" exponent -// with a decimal mantissa and a binary exponent, or 0" if x is zero, +// with a decimal mantissa and a binary exponent, or "0" if x is zero, // and returns the extended buffer. // The mantissa is normalized such that is uses x.Prec() bits in binary // representation. @@ -329,9 +329,9 @@ func (x *Float) fmtB(buf []byte) []byte { m := x.mant switch w := uint32(len(x.mant)) * _W; { case w < x.prec: - m = nat(nil).shl(m, uint(x.prec-w)) + m = nat(nil).lsh(m, uint(x.prec-w)) case w > x.prec: - m = nat(nil).shr(m, uint(w-x.prec)) + m = nat(nil).rsh(m, uint(w-x.prec)) } buf = append(buf, m.utoa(10)...) @@ -380,9 +380,9 @@ func (x *Float) fmtX(buf []byte, prec int) []byte { m := x.mant switch w := uint(len(x.mant)) * _W; { case w < n: - m = nat(nil).shl(m, n-w) + m = nat(nil).lsh(m, n-w) case w > n: - m = nat(nil).shr(m, w-n) + m = nat(nil).rsh(m, w-n) } exp64 := int64(x.exp) - 1 // avoid wrap-around diff --git a/src/math/big/int.go b/src/math/big/int.go index 944b70c0621afe..8eb0db6c5870ff 100644 --- a/src/math/big/int.go +++ b/src/math/big/int.go @@ -181,18 +181,25 @@ func (z *Int) Sub(x, y *Int) *Int { // Mul sets z to the product x*y and returns z. func (z *Int) Mul(x, y *Int) *Int { + z.mul(nil, x, y) + return z +} + +// mul is like Mul but takes an explicit stack to use, for internal use. +// It does not return a *Int because doing so makes the stack-allocated Ints +// used in natmul.go escape to the heap (even though the result is unused). +func (z *Int) mul(stk *stack, x, y *Int) { // x * y == x * y // x * (-y) == -(x * y) // (-x) * y == -(x * y) // (-x) * (-y) == x * y if x == y { - z.abs = z.abs.sqr(x.abs) + z.abs = z.abs.sqr(stk, x.abs) z.neg = false - return z + return } - z.abs = z.abs.mul(x.abs, y.abs) + z.abs = z.abs.mul(stk, x.abs, y.abs) z.neg = len(z.abs) > 0 && x.neg != y.neg // 0 has no sign - return z } // MulRange sets z to the product of all integers @@ -213,7 +220,7 @@ func (z *Int) MulRange(a, b int64) *Int { a, b = -b, -a } - z.abs = z.abs.mulRange(uint64(a), uint64(b)) + z.abs = z.abs.mulRange(nil, uint64(a), uint64(b)) z.neg = neg return z } @@ -264,7 +271,7 @@ func (z *Int) Binomial(n, k int64) *Int { // If y == 0, a division-by-zero run-time panic occurs. // Quo implements truncated division (like Go); see [Int.QuoRem] for more details. func (z *Int) Quo(x, y *Int) *Int { - z.abs, _ = z.abs.div(nil, x.abs, y.abs) + z.abs, _ = z.abs.div(nil, nil, x.abs, y.abs) z.neg = len(z.abs) > 0 && x.neg != y.neg // 0 has no sign return z } @@ -273,7 +280,7 @@ func (z *Int) Quo(x, y *Int) *Int { // If y == 0, a division-by-zero run-time panic occurs. // Rem implements truncated modulus (like Go); see [Int.QuoRem] for more details. func (z *Int) Rem(x, y *Int) *Int { - _, z.abs = nat(nil).div(z.abs, x.abs, y.abs) + _, z.abs = nat(nil).div(nil, z.abs, x.abs, y.abs) z.neg = len(z.abs) > 0 && x.neg // 0 has no sign return z } @@ -288,9 +295,9 @@ func (z *Int) Rem(x, y *Int) *Int { // r = x - y*q // // (See Daan Leijen, “Division and Modulus for Computer Scientists”.) -// See [DivMod] for Euclidean division and modulus (unlike Go). +// See [Int.DivMod] for Euclidean division and modulus (unlike Go). func (z *Int) QuoRem(x, y, r *Int) (*Int, *Int) { - z.abs, r.abs = z.abs.div(r.abs, x.abs, y.abs) + z.abs, r.abs = z.abs.div(nil, r.abs, x.abs, y.abs) z.neg, r.neg = len(z.abs) > 0 && x.neg != y.neg, len(r.abs) > 0 && x.neg // 0 has no sign return z, r } @@ -589,7 +596,7 @@ func (z *Int) exp(x, y, m *Int, slow bool) *Int { mWords = m.abs // m.abs may be nil for m == 0 } - z.abs = z.abs.expNN(xWords, yWords, mWords, slow) + z.abs = z.abs.expNN(nil, xWords, yWords, mWords, slow) z.neg = len(z.abs) > 0 && x.neg && len(yWords) > 0 && yWords[0]&1 == 1 // 0 has no sign if z.neg && len(mWords) > 0 { // make modulus result positive @@ -707,42 +714,36 @@ func lehmerSimulate(A, B *Int) (u0, u1, v0, v1 Word, even bool) { // For even == true: u0, v1 >= 0 && u1, v0 <= 0 // For even == false: u0, v1 <= 0 && u1, v0 >= 0 // q, r, s, t are temporary variables to avoid allocations in the multiplication. -func lehmerUpdate(A, B, q, r, s, t *Int, u0, u1, v0, v1 Word, even bool) { - - t.abs = t.abs.setWord(u0) - s.abs = s.abs.setWord(v0) - t.neg = !even - s.neg = even - - t.Mul(A, t) - s.Mul(B, s) - - r.abs = r.abs.setWord(u1) - q.abs = q.abs.setWord(v1) - r.neg = even - q.neg = !even - - r.Mul(A, r) - q.Mul(B, q) +func lehmerUpdate(A, B, q, r *Int, u0, u1, v0, v1 Word, even bool) { + mulW(q, B, even, v0) + mulW(r, A, even, u1) + mulW(A, A, !even, u0) + mulW(B, B, !even, v1) + A.Add(A, q) + B.Add(B, r) +} - A.Add(t, s) - B.Add(r, q) +// mulW sets z = x * (-?)w +// where the minus sign is present when neg is true. +func mulW(z, x *Int, neg bool, w Word) { + z.abs = z.abs.mulAddWW(x.abs, w, 0) + z.neg = x.neg != neg } // euclidUpdate performs a single step of the Euclidean GCD algorithm // if extended is true, it also updates the cosequence Ua, Ub. -func euclidUpdate(A, B, Ua, Ub, q, r, s, t *Int, extended bool) { - q, r = q.QuoRem(A, B, r) - - *A, *B, *r = *B, *r, *A +// q and r are used as temporaries; the initial values are ignored. +func euclidUpdate(A, B, Ua, Ub, q, r *Int, extended bool) (nA, nB, nr, nUa, nUb *Int) { + q.QuoRem(A, B, r) if extended { - // Ua, Ub = Ub, Ua - q*Ub - t.Set(Ub) - s.Mul(Ub, q) - Ub.Sub(Ua, s) - Ua.Set(t) + // Ua, Ub = Ub, Ua-q*Ub + q.Mul(q, Ub) + Ua, Ub = Ub, Ua + Ub.Sub(Ub, q) } + + return B, r, A, Ua, Ub } // lehmerGCD sets z to the greatest common divisor of a and b, @@ -772,8 +773,6 @@ func (z *Int) lehmerGCD(x, y, a, b *Int) *Int { // temp variables for multiprecision update q := new(Int) r := new(Int) - s := new(Int) - t := new(Int) // ensure A >= B if A.abs.cmp(B.abs) < 0 { @@ -791,18 +790,18 @@ func (z *Int) lehmerGCD(x, y, a, b *Int) *Int { // Simulate the effect of the single-precision steps using the cosequences. // A = u0*A + v0*B // B = u1*A + v1*B - lehmerUpdate(A, B, q, r, s, t, u0, u1, v0, v1, even) + lehmerUpdate(A, B, q, r, u0, u1, v0, v1, even) if extended { // Ua = u0*Ua + v0*Ub // Ub = u1*Ua + v1*Ub - lehmerUpdate(Ua, Ub, q, r, s, t, u0, u1, v0, v1, even) + lehmerUpdate(Ua, Ub, q, r, u0, u1, v0, v1, even) } } else { // Single-digit calculations failed to simulate any quotients. // Do a standard Euclidean step. - euclidUpdate(A, B, Ua, Ub, q, r, s, t, extended) + A, B, r, Ua, Ub = euclidUpdate(A, B, Ua, Ub, q, r, extended) } } @@ -810,7 +809,7 @@ func (z *Int) lehmerGCD(x, y, a, b *Int) *Int { // extended Euclidean algorithm base case if B is a single Word if len(A.abs) > 1 { // A is longer than a single Word, so one update is needed. - euclidUpdate(A, B, Ua, Ub, q, r, s, t, extended) + A, B, r, Ua, Ub = euclidUpdate(A, B, Ua, Ub, q, r, extended) } if len(B.abs) > 0 { // A and B are both a single Word. @@ -828,15 +827,9 @@ func (z *Int) lehmerGCD(x, y, a, b *Int) *Int { even = !even } - t.abs = t.abs.setWord(ua) - s.abs = s.abs.setWord(va) - t.neg = !even - s.neg = even - - t.Mul(Ua, t) - s.Mul(Ub, s) - - Ua.Add(t, s) + mulW(Ua, Ua, !even, ua) + mulW(Ub, Ub, even, va) + Ua.Add(Ua, Ub) } else { for bWord != 0 { aWord, bWord = bWord, aWord%bWord @@ -863,13 +856,13 @@ func (z *Int) lehmerGCD(x, y, a, b *Int) *Int { } if x != nil { - *x = *Ua + x.Set(Ua) if negA { x.neg = !x.neg } } - *z = *A + z.Set(A) return z } @@ -1104,7 +1097,7 @@ func (z *Int) ModSqrt(x, p *Int) *Int { // Lsh sets z = x << n and returns z. func (z *Int) Lsh(x *Int, n uint) *Int { - z.abs = z.abs.shl(x.abs, n) + z.abs = z.abs.lsh(x.abs, n) z.neg = x.neg return z } @@ -1114,13 +1107,13 @@ func (z *Int) Rsh(x *Int, n uint) *Int { if x.neg { // (-x) >> s == ^(x-1) >> s == ^((x-1) >> s) == -(((x-1) >> s) + 1) t := z.abs.sub(x.abs, natOne) // no underflow because |x| > 0 - t = t.shr(t, n) + t = t.rsh(t, n) z.abs = t.add(t, natOne) z.neg = true // z cannot be zero if x is negative return z } - z.abs = z.abs.shr(x.abs, n) + z.abs = z.abs.rsh(x.abs, n) z.neg = false return z } @@ -1312,6 +1305,6 @@ func (z *Int) Sqrt(x *Int) *Int { panic("square root of negative number") } z.neg = false - z.abs = z.abs.sqrt(x.abs) + z.abs = z.abs.sqrt(nil, x.abs) return z } diff --git a/src/math/big/int_test.go b/src/math/big/int_test.go index f701652f1b9689..eb5c177be0f53a 100644 --- a/src/math/big/int_test.go +++ b/src/math/big/int_test.go @@ -1614,7 +1614,7 @@ func TestModInverse(t *testing.T) { func BenchmarkModInverse(b *testing.B) { p := new(Int).SetInt64(1) // Mersenne prime 2**1279 -1 - p.abs = p.abs.shl(p.abs, 1279) + p.abs = p.abs.lsh(p.abs, 1279) p.Sub(p, intOne) x := new(Int).Sub(p, intOne) z := new(Int) diff --git a/src/math/big/intconv_test.go b/src/math/big/intconv_test.go index 5ba29263a6e0a9..cf337db63a7d08 100644 --- a/src/math/big/intconv_test.go +++ b/src/math/big/intconv_test.go @@ -7,6 +7,7 @@ package big import ( "bytes" "fmt" + "math/rand/v2" "testing" ) @@ -389,12 +390,14 @@ func TestFormat(t *testing.T) { } } -var scanTests = []struct { +type scanTest struct { input string format string output string remaining int -}{ +} + +var scanTests = []scanTest{ {"1010", "%b", "10", 0}, {"0b1010", "%v", "10", 0}, {"12", "%o", "10", 0}, @@ -410,6 +413,25 @@ var scanTests = []struct { {"0 ", "%v", "0", 1}, {"2+3", "%v", "2", 2}, {"0XABC 12", "%v", "2748", 3}, + + {"10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffff00000000000022222223333333333444444444", "%x", "72999049881955123498258745691204661198291656115976958889267080286388402675338838184094604981077942396458276955120179409196748346461468914795561487752253275293347599221664790586512596660792869956", 0}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff33377fffffffffffffffffffffffffffffffffffffffffffff0000000000022222eee1", "%x", "1167984798111281975972139931059274579172666497855631342228273284582214442805421410945513679697247078343332431249286160621687557589604464869034163736183926240549918956767671325412748661204059352801", 0}, + {"5c0d52f451aec609b15da8e5e5626c4eaa88723bdeac9d25ca9b961269400410ca208a16af9c2fb07d7a11c7772cba02c22f9711078d51a3797eb18e691295293284d988e349fa6deba46b25a4ecd9f715", "%x", "419981998319789881681348172155240145539175961318447822049735313481433836043208347786919222066492311384432264836938599791362288343314139526391998172436831830624710446410781662672086936222288181013", 0}, + {"92fcad4b5c0d52f451aec609b15da8e5e5626c4eaa88723bdeac9d25ca9b961269400410ca208a16af9c2fb07d799c32fe2f3cc5422f9711078d51a3797eb18e691295293284d8f5e69caf6decddfe1df6", "%x", "670619546945481998414061201992255225716434798957375727890607516800039934374391281275121813279544891602026798031004764406015624866771554937391445093144221697436880587924204655403711377861305572854", 0}, + {"10000000000000000000000200000000000000000000003000000000000000000000040000000000000000000000500000000000000000000006", "%d", "10000000000000000000000200000000000000000000003000000000000000000000040000000000000000000000500000000000000000000006", 0}, +} + +func init() { + for i := range 200 { + d := make([]byte, i+1) + for j := range d { + d[j] = '0' + rand.N(byte(10)) + } + if d[0] == '0' { + d[0] = '1' + } + scanTests = append(scanTests, scanTest{input: string(d), format: "%d", output: string(d)}) + } } func TestScan(t *testing.T) { diff --git a/src/math/big/internal/asmgen/386.go b/src/math/big/internal/asmgen/386.go new file mode 100644 index 00000000000000..f8f67ba5269728 --- /dev/null +++ b/src/math/big/internal/asmgen/386.go @@ -0,0 +1,58 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +import "fmt" + +var Arch386 = &Arch{ + Name: "386", + WordBits: 32, + WordBytes: 4, + + regs: []string{ + "BX", "SI", "DI", "BP", + "CX", "DX", "AX", // last, to leave available for hinted allocation + }, + op3: x86Op3, + hint: x86Hint, + memOK: true, + subCarryIsBorrow: true, + maxColumns: 1, // not enough registers for more + + // Note: It would be nice to not set memIndex and then + // delete all the code in pipe.go that supports it. + // But a few routines, notably lshVU and mulAddVWW, + // benefit dramatically from the use of index registers. + // Perhaps some day we will decide 386 performance + // does not matter enough to keep this code. + memIndex: _386MemIndex, + + mov: "MOVL", + adds: "ADDL", + adcs: "ADCL", + subs: "SUBL", + sbcs: "SBBL", + lsh: "SHLL", + lshd: "SHLL", + rsh: "SHRL", + rshd: "SHRL", + and: "ANDL", + or: "ORL", + xor: "XORL", + neg: "NEGL", + lea: "LEAL", + mulWideF: x86MulWide, + + addWords: "LEAL (%[2]s)(%[1]s*4), %[3]s", + + jmpZero: "TESTL %[1]s, %[1]s; JZ %[2]s", + jmpNonZero: "TESTL %[1]s, %[1]s; JNZ %[2]s", + loopBottom: "SUBL $1, %[1]s; JNZ %[2]s", + loopBottomNeg: "ADDL $1, %[1]s; JNZ %[2]s", +} + +func _386MemIndex(a *Asm, off int, ix Reg, p RegPtr) Reg { + return Reg{fmt.Sprintf("%d(%s)(%s*%d)", off, p, ix, a.Arch.WordBytes)} +} diff --git a/src/math/big/internal/asmgen/add.go b/src/math/big/internal/asmgen/add.go new file mode 100644 index 00000000000000..ee15e3a96fed75 --- /dev/null +++ b/src/math/big/internal/asmgen/add.go @@ -0,0 +1,57 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +// addOrSubVV generates addVV or subVV, +// which do z, c = x ± y. +// The caller guarantees that len(z) == len(x) == len(y). +func addOrSubVV(a *Asm, name string) { + f := a.Func("func " + name + "(z, x, y []Word) (c Word)") + + add := a.Add + which := AddCarry + if name == "subVV" { + add = a.Sub + which = SubCarry + } + + n := f.Arg("z_len") + p := f.Pipe() + p.SetHint("y", HintMemOK) // allow y to be used from memory on x86 + p.Start(n, 1, 4) + var c Reg + if !a.Arch.CarrySafeLoop { + // Carry smashed by loop tests; allocate and save in register + // around unrolled blocks. + c = a.Reg() + a.Mov(a.Imm(0), c) + a.EOL("clear saved carry") + p.AtUnrollStart(func() { a.RestoreCarry(c); a.Free(c) }) + p.AtUnrollEnd(func() { a.Unfree(c); a.SaveCarry(c) }) + } else { + // Carry preserved by loop; clear now, ahead of loop + // (but after Start, which may have modified it). + a.ClearCarry(which) + } + p.Loop(func(in, out [][]Reg) { + for i, x := range in[0] { + y := in[1][i] + add(y, x, x, SetCarry|UseCarry) + } + p.StoreN(in[:1]) + }) + p.Done() + + // Copy carry to output. + if c.Valid() { + a.ConvertCarry(which, c) + } else { + c = a.RegHint(HintCarry) + a.SaveConvertCarry(which, c) + } + f.StoreArg(c, "c") + a.Free(c) + a.Ret() +} diff --git a/src/math/big/internal/asmgen/amd64.go b/src/math/big/internal/asmgen/amd64.go new file mode 100644 index 00000000000000..36b1b5844b1074 --- /dev/null +++ b/src/math/big/internal/asmgen/amd64.go @@ -0,0 +1,146 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +var ArchAMD64 = &Arch{ + Name: "amd64", + WordBits: 64, + WordBytes: 8, + + regs: []string{ + "BX", "SI", "DI", + "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15", + "AX", "DX", "CX", // last to leave available for hinted allocation + }, + op3: x86Op3, + hint: x86Hint, + memOK: true, + subCarryIsBorrow: true, + + // Note: Not setting memIndex, because code generally runs faster + // if we avoid the use of scaled-index memory references, + // particularly in ADX instructions. + + options: map[Option]func(*Asm, string){ + OptionAltCarry: amd64JmpADX, + }, + + mov: "MOVQ", + adds: "ADDQ", + adcs: "ADCQ", + subs: "SUBQ", + sbcs: "SBBQ", + lsh: "SHLQ", + lshd: "SHLQ", + rsh: "SHRQ", + rshd: "SHRQ", + and: "ANDQ", + or: "ORQ", + xor: "XORQ", + neg: "NEGQ", + lea: "LEAQ", + addF: amd64Add, + mulWideF: x86MulWide, + + addWords: "LEAQ (%[2]s)(%[1]s*8), %[3]s", + + jmpZero: "TESTQ %[1]s, %[1]s; JZ %[2]s", + jmpNonZero: "TESTQ %[1]s, %[1]s; JNZ %[2]s", + loopBottom: "SUBQ $1, %[1]s; JNZ %[2]s", + loopBottomNeg: "ADDQ $1, %[1]s; JNZ %[2]s", +} + +func amd64JmpADX(a *Asm, label string) { + a.Printf("\tCMPB ·hasADX(SB), $0; JNZ %s\n", label) +} + +func amd64Add(a *Asm, src1, src2 Reg, dst Reg, carry Carry) bool { + if a.Enabled(OptionAltCarry) { + // If OptionAltCarry is enabled, the generator is emitting ADD instructions + // both with and without the AltCarry flag set; the AltCarry flag means to + // use ADOX. Otherwise we have to use ADCX. + // Using regular ADD/ADC would smash both carry flags, + // so we reject anything we can't handled with ADCX/ADOX. + if carry&UseCarry != 0 && carry&(SetCarry|SmashCarry) != 0 { + if carry&AltCarry != 0 { + a.op3("ADOXQ", src1, src2, dst) + } else { + a.op3("ADCXQ", src1, src2, dst) + } + return true + } + if carry&(SetCarry|UseCarry) == SetCarry && a.IsZero(src1) && src2 == dst { + // Clearing carry flag. Caller will add EOL comment. + a.Printf("\tTESTQ AX, AX\n") + return true + } + if carry != KeepCarry { + a.Fatalf("unsupported carry") + } + } + return false +} + +// The x86-prefixed functions are shared with Arch386 in 386.go. + +func x86Op3(name string) bool { + // As far as a.op3 is concerned, there are no 3-op instructions. + // (We print instructions like MULX ourselves.) + return false +} + +func x86Hint(a *Asm, h Hint) string { + switch h { + case HintShiftCount: + return "CX" + case HintMulSrc: + if a.Enabled(OptionAltCarry) { // using MULX + return "DX" + } + return "AX" + case HintMulHi: + if a.Enabled(OptionAltCarry) { // using MULX + return "" + } + return "DX" + } + return "" +} + +func x86Suffix(a *Asm) string { + // Note: Not using a.Arch == Arch386 to avoid init cycle. + if a.Arch.Name == "386" { + return "L" + } + return "Q" +} + +func x86MulWide(a *Asm, src1, src2, dstlo, dsthi Reg) { + if a.Enabled(OptionAltCarry) { + // Using ADCX/ADOX; use MULX to avoid clearing carry flag. + if src1.name != "DX" { + if src2.name != "DX" { + a.Fatalf("mul src1 or src2 must be DX") + } + src2 = src1 + } + a.Printf("\tMULXQ %s, %s, %s\n", src2, dstlo, dsthi) + return + } + + if src1.name != "AX" { + if src2.name != "AX" { + a.Fatalf("mulwide src1 or src2 must be AX") + } + src2 = src1 + } + if dstlo.name != "AX" { + a.Fatalf("mulwide dstlo must be AX") + } + if dsthi.name != "DX" { + a.Fatalf("mulwide dsthi must be DX") + } + a.Printf("\tMUL%s %s\n", x86Suffix(a), src2) +} diff --git a/src/math/big/internal/asmgen/arch.go b/src/math/big/internal/asmgen/arch.go new file mode 100644 index 00000000000000..adfcff93840323 --- /dev/null +++ b/src/math/big/internal/asmgen/arch.go @@ -0,0 +1,236 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +import ( + "fmt" + "strings" +) + +// Note: Exported fields and methods are expected to be used +// by function generators (like the ones in add.go and so on). +// Unexported fields and methods should not be. + +// An Arch defines how to generate assembly for a specific architecture. +type Arch struct { + Name string // name of architecture + Build string // build tag + WordBits int // length of word in bits (32 or 64) + WordBytes int // length of word in bytes (4 or 8) + CarrySafeLoop bool // whether loops preserve carry flag across iterations + + // Registers. + regs []string // usable general registers, in allocation order + reg0 string // dedicated zero register + regCarry string // dedicated carry register, for systems with no hardware carry bits + regAltCarry string // dedicated secondary carry register, for systems with no hardware carry bits + regTmp string // dedicated temporary register + + // regShift indicates that the architecture supports + // using REG1>>REG2 and REG1< src1), for carry-less systems + lea string // load effective address + + // addF and subF implement a.Add and a.Sub + // on systems where the situation is more complicated than + // the six basic instructions (add, adds, adcs, sub, subs, sbcs). + // They return a boolean indicating whether the operation was handled. + addF func(a *Asm, src1, src2, dst Reg, carry Carry) bool + subF func(a *Asm, src1, src2, dst Reg, carry Carry) bool + + // mulF and mulWideF implement Mul and MulWide. + // They call Fatalf if the operation is unsupported. + // An architecture can set the mul field instead of mulF. + // mulWide is optional, but otherwise mulhi should be set. + mulWideF func(a *Asm, src1, src2, dstlo, dsthi Reg) + + // addWords is a printf format taking src1, src2, dst + // and sets dst = WordBytes*src1+src2. + // It may modify the carry flag. + addWords string + + // subCarryIsBorrow is true when the actual processor carry bit used in subtraction + // is really a “borrow” bit, meaning 1 means borrow and 0 means no borrow. + // In contrast, most systems (except x86) use a carry bit with the opposite + // meaning: 0 means a borrow happened, and 1 means it didn't. + subCarryIsBorrow bool + + // Jump instruction printf formats. + // jmpZero and jmpNonZero are printf formats taking src, label + // and jump to label if src is zero / non-zero. + jmpZero string + jmpNonZero string + + // loopTop is a printf format taking src, label that should + // jump to label if src is zero, or else set up for a loop. + // If loopTop is not set, jmpZero is used. + loopTop string + + // loopBottom is a printf format taking dst, label that should + // decrement dst and then jump to label if src is non-zero. + // If loopBottom is not set, a subtraction is used followed by + // use of jmpNonZero. + loopBottom string + + // loopBottomNeg is like loopBottom but used in negative-index + // loops, which only happen memIndex is also set (only on 386). + // It increments dst instead of decrementing it. + loopBottomNeg string + + // Indexed memory access. + // If set, memIndex returns a memory reference for a mov instruction + // addressing off(ptr)(ix*WordBytes). + // Using memIndex costs an extra register but allows the end-of-loop + // to do a single increment/decrement instead of advancing two or three pointers. + // This is particularly important on 386. + memIndex func(a *Asm, off int, ix Reg, ptr RegPtr) Reg + + // Incrementing/decrementing memory access. + // loadIncN loads memory at ptr into regs, incrementing ptr by WordBytes after each reg. + // loadDecN loads memory at ptr into regs, decrementing ptr by WordBytes before each reg. + // storeIncN and storeDecN are the same, but storing from regs instead of loading into regs. + // If missing, the assembler accesses memory and advances pointers using separate instructions. + loadIncN func(a *Asm, ptr RegPtr, regs []Reg) + loadDecN func(a *Asm, ptr RegPtr, regs []Reg) + storeIncN func(a *Asm, ptr RegPtr, regs []Reg) + storeDecN func(a *Asm, ptr RegPtr, regs []Reg) + + // options is a map from optional CPU features to functions that test for them. + // The test function should jump to label if the feature is available. + options map[Option]func(a *Asm, label string) +} + +// HasShiftWide reports whether the Arch has working LshWide/RshWide instructions. +// If not, calling them will panic. +func (a *Arch) HasShiftWide() bool { + return a.lshd != "" +} + +// A Hint is a hint about what a register will be used for, +// so that an appropriate one can be selected. +type Hint uint + +const ( + HintNone Hint = iota + HintShiftCount // shift count (CX on x86) + HintMulSrc // mul source operand (AX on x86) + HintMulHi // wide mul high output (DX on x86) + HintMemOK // a memory reference is okay + HintCarry // carry flag + HintAltCarry // secondary carry flag +) + +// A Reg is an allocated register or other assembly operand. +// (For example, a constant might have name "$123" +// and a memory reference might have name "0(R8)".) +type Reg struct{ name string } + +// IsImm reports whether r is an immediate value. +func (r Reg) IsImm() bool { return strings.HasPrefix(r.name, "$") } + +// IsMem reports whether r is a memory value. +func (r Reg) IsMem() bool { return strings.HasSuffix(r.name, ")") } + +// String returns the assembly syntax for r. +func (r Reg) String() string { return r.name } + +// Valid reports whether is valid, meaning r is not the zero value of Reg (a register with no name). +func (r Reg) Valid() bool { return r.name != "" } + +// A RegPtr is like a Reg but expected to hold a pointer. +// The separate Go type helps keeps pointers and scalars separate and avoid mistakes; +// it is okay to convert to Reg as needed to use specific routines. +type RegPtr struct{ name string } + +// String returns the assembly syntax for r. +func (r RegPtr) String() string { return r.name } + +// Valid reports whether is valid, meaning r is not the zero value of RegPtr (a register with no name). +func (r RegPtr) Valid() bool { return r.name != "" } + +// mem returns a memory reference to off bytes from the pointer r. +func (r *RegPtr) mem(off int) Reg { return Reg{fmt.Sprintf("%d(%s)", off, r)} } + +// A Carry is a flag field explaining how an instruction sets and uses the carry flags. +// Different operations expect different sets of bits. +// Add and Sub expect: UseCarry or 0, SetCarry, KeepCarry, or SmashCarry; and AltCarry or 0. +// ClearCarry, SaveCarry, and ConvertCarry expect: AddCarry or SubCarry; and AltCarry or 0. +type Carry uint + +const ( + SetCarry Carry = 1 << iota // sets carry + UseCarry // uses carry + KeepCarry // must preserve carry + SmashCarry // can modify carry or not, whatever is easiest + + AltCarry // use the secondary carry flag + AddCarry // use add carry flag semantics (for ClearCarry, ConvertCarry) + SubCarry // use sub carry flag semantics (for ClearCarry, ConvertCarry) +) + +// An Option denotes an optional CPU feature that can be tested at runtime. +type Option int + +const ( + _ Option = iota + + // OptionAltCarry checks whether there is an add instruction + // that uses a secondary carry flag, so that two different sums + // can be accumulated in parallel with independent carry flags. + // Some architectures (MIPS, Loong64, RISC-V) provide this + // functionality natively, indicated by asm.Carry().Valid() being true. + OptionAltCarry +) diff --git a/src/math/big/internal/asmgen/arm.go b/src/math/big/internal/asmgen/arm.go new file mode 100644 index 00000000000000..191ae410a1c58e --- /dev/null +++ b/src/math/big/internal/asmgen/arm.go @@ -0,0 +1,76 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +var ArchARM = &Arch{ + Name: "arm", + WordBits: 32, + WordBytes: 4, + CarrySafeLoop: true, + + regs: []string{ + // R10 is g. + // R11 is the assembler/linker temporary (but we use it as a regular register). + // R13 is SP. + // R14 is LR. + // R15 is PC. + "R0", "R1", "R2", "R3", "R4", "R5", "R6", "R7", "R8", "R9", "R11", "R12", + }, + regShift: true, + + mov: "MOVW", + add: "ADD", + adds: "ADD.S", + adc: "ADC", + adcs: "ADC.S", + sub: "SUB", + subs: "SUB.S", + sbc: "SBC", + sbcs: "SBC.S", + rsb: "RSB", + and: "AND", + or: "ORR", + xor: "EOR", + + mulWideF: armMulWide, + + addWords: "ADD %s<<2, %s, %s", + + jmpZero: "TEQ $0, %s; BEQ %s", + jmpNonZero: "TEQ $0, %s; BNE %s", + + loadIncN: armLoadIncN, + loadDecN: armLoadDecN, + storeIncN: armStoreIncN, + storeDecN: armStoreDecN, +} + +func armMulWide(a *Asm, src1, src2, dstlo, dsthi Reg) { + a.Printf("\tMULLU %s, %s, (%s, %s)\n", src1, src2, dsthi, dstlo) +} + +func armLoadIncN(a *Asm, p RegPtr, regs []Reg) { + for _, r := range regs { + a.Printf("\tMOVW.P %d(%s), %s\n", a.Arch.WordBytes, p, r) + } +} + +func armLoadDecN(a *Asm, p RegPtr, regs []Reg) { + for _, r := range regs { + a.Printf("\tMOVW.W %d(%s), %s\n", -a.Arch.WordBytes, p, r) + } +} + +func armStoreIncN(a *Asm, p RegPtr, regs []Reg) { + for _, r := range regs { + a.Printf("\tMOVW.P %s, %d(%s)\n", r, a.Arch.WordBytes, p) + } +} + +func armStoreDecN(a *Asm, p RegPtr, regs []Reg) { + for _, r := range regs { + a.Printf("\tMOVW.W %s, %d(%s)\n", r, -a.Arch.WordBytes, p) + } +} diff --git a/src/math/big/internal/asmgen/arm64.go b/src/math/big/internal/asmgen/arm64.go new file mode 100644 index 00000000000000..ce70d5a1f7d133 --- /dev/null +++ b/src/math/big/internal/asmgen/arm64.go @@ -0,0 +1,111 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +var ArchARM64 = &Arch{ + Name: "arm64", + WordBits: 64, + WordBytes: 8, + CarrySafeLoop: true, + + regs: []string{ + // R18 is the platform register. + // R27 is the assembler/linker temporary (which we could potentially use but don't). + // R28 is g. + // R29 is FP. + // R30 is LR. + "R0", "R1", "R2", "R3", "R4", "R5", "R6", "R7", "R8", "R9", + "R10", "R11", "R12", "R13", "R14", "R15", "R16", "R17", "R19", + "R20", "R21", "R22", "R23", "R24", "R25", "R26", + }, + reg0: "ZR", + + mov: "MOVD", + add: "ADD", + adds: "ADDS", + adc: "ADC", + adcs: "ADCS", + sub: "SUB", + subs: "SUBS", + sbc: "SBC", + sbcs: "SBCS", + mul: "MUL", + mulhi: "UMULH", + lsh: "LSL", + rsh: "LSR", + and: "AND", + or: "ORR", + xor: "EOR", + + addWords: "ADD %[1]s<<3, %[2]s, %[3]s", + + jmpZero: "CBZ %s, %s", + jmpNonZero: "CBNZ %s, %s", + + loadIncN: arm64LoadIncN, + loadDecN: arm64LoadDecN, + storeIncN: arm64StoreIncN, + storeDecN: arm64StoreDecN, +} + +func arm64LoadIncN(a *Asm, p RegPtr, regs []Reg) { + if len(regs) == 1 { + a.Printf("\tMOVD.P %d(%s), %s\n", a.Arch.WordBytes, p, regs[0]) + return + } + a.Printf("\tLDP.P %d(%s), (%s, %s)\n", len(regs)*a.Arch.WordBytes, p, regs[0], regs[1]) + var i int + for i = 2; i+2 <= len(regs); i += 2 { + a.Printf("\tLDP %d(%s), (%s, %s)\n", (i-len(regs))*a.Arch.WordBytes, p, regs[i], regs[i+1]) + } + if i < len(regs) { + a.Printf("\tMOVD %d(%s), %s\n", -1*a.Arch.WordBytes, p, regs[i]) + } +} + +func arm64LoadDecN(a *Asm, p RegPtr, regs []Reg) { + if len(regs) == 1 { + a.Printf("\tMOVD.W -%d(%s), %s\n", a.Arch.WordBytes, p, regs[0]) + return + } + a.Printf("\tLDP.W %d(%s), (%s, %s)\n", -len(regs)*a.Arch.WordBytes, p, regs[len(regs)-1], regs[len(regs)-2]) + var i int + for i = 2; i+2 <= len(regs); i += 2 { + a.Printf("\tLDP %d(%s), (%s, %s)\n", i*a.Arch.WordBytes, p, regs[len(regs)-1-i], regs[len(regs)-2-i]) + } + if i < len(regs) { + a.Printf("\tMOVD %d(%s), %s\n", i*a.Arch.WordBytes, p, regs[0]) + } +} + +func arm64StoreIncN(a *Asm, p RegPtr, regs []Reg) { + if len(regs) == 1 { + a.Printf("\tMOVD.P %s, %d(%s)\n", regs[0], a.Arch.WordBytes, p) + return + } + a.Printf("\tSTP.P (%s, %s), %d(%s)\n", regs[0], regs[1], len(regs)*a.Arch.WordBytes, p) + var i int + for i = 2; i+2 <= len(regs); i += 2 { + a.Printf("\tSTP (%s, %s), %d(%s)\n", regs[i], regs[i+1], (i-len(regs))*a.Arch.WordBytes, p) + } + if i < len(regs) { + a.Printf("\tMOVD %s, %d(%s)\n", regs[i], -1*a.Arch.WordBytes, p) + } +} + +func arm64StoreDecN(a *Asm, p RegPtr, regs []Reg) { + if len(regs) == 1 { + a.Printf("\tMOVD.W %s, -%d(%s)\n", regs[0], a.Arch.WordBytes, p) + return + } + a.Printf("\tSTP.W (%s, %s), %d(%s)\n", regs[len(regs)-1], regs[len(regs)-2], -len(regs)*a.Arch.WordBytes, p) + var i int + for i = 2; i+2 <= len(regs); i += 2 { + a.Printf("\tSTP (%s, %s), %d(%s)\n", regs[len(regs)-1-i], regs[len(regs)-2-i], i*a.Arch.WordBytes, p) + } + if i < len(regs) { + a.Printf("\tMOVD %s, %d(%s)\n", regs[0], i*a.Arch.WordBytes, p) + } +} diff --git a/src/math/big/internal/asmgen/asm.go b/src/math/big/internal/asmgen/asm.go new file mode 100644 index 00000000000000..d1d8309c8f2c10 --- /dev/null +++ b/src/math/big/internal/asmgen/asm.go @@ -0,0 +1,804 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +import ( + "bytes" + "cmp" + "fmt" + "math/bits" + "slices" + "strings" +) + +// Note: Exported fields and methods are expected to be used +// by function generators (like the ones in add.go and so on). +// Unexported fields and methods should not be. + +// An Asm is an assembly file being written. +type Asm struct { + Arch *Arch // architecture + out bytes.Buffer // output buffer + regavail uint64 // bitmap of available registers + enabled map[Option]bool // enabled optional CPU features +} + +// NewAsm returns a new Asm preparing assembly +// for the given architecture to be written to file. +func NewAsm(arch *Arch) *Asm { + a := &Asm{Arch: arch, enabled: make(map[Option]bool)} + buildTag := "" + if arch.Build != "" { + buildTag = " && (" + arch.Build + ")" + } + a.Printf(asmHeader, buildTag) + return a +} + +// Note: Using Copyright 2025, not the current year, to avoid test failures +// on January 1 and spurious diffs when regenerating assembly. +// The generator was written in 2025; that's good enough. +// (As a matter of policy the Go project does not update copyright +// notices every year, since copyright terms are so long anyway.) + +var asmHeader = `// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated by 'go generate' (with ./internal/asmgen). DO NOT EDIT. + +//go:build !math_big_pure_go%s + +#include "textflag.h" +` + +// Fatalf reports a fatal error by panicking. +// Panicking is appropriate because there is a bug in the generator, +// and panicking will show the exact source lines leading to that bug. +func (a *Asm) Fatalf(format string, args ...any) { + text := a.out.String() + i := strings.LastIndex(text, "\nTEXT") + text = text[i+1:] + panic("[" + a.Arch.Name + "] asmgen internal error: " + fmt.Sprintf(format, args...) + "\n" + text) +} + +// hint returns the register name for the given hint. +func (a *Asm) hint(h Hint) string { + if h == HintCarry && a.Arch.regCarry != "" { + return a.Arch.regCarry + } + if h == HintAltCarry && a.Arch.regAltCarry != "" { + return a.Arch.regAltCarry + } + if h == HintNone || a.Arch.hint == nil { + return "" + } + return a.Arch.hint(a, h) +} + +// ZR returns the zero register (the specific register guaranteed to hold the integer 0), +// or else the zero Reg (Reg{}, which has r.Valid() == false). +func (a *Asm) ZR() Reg { + return Reg{a.Arch.reg0} +} + +// tmp returns the temporary register, or else the zero Reg. +// The temporary register is one available for use implementing logical instructions +// that compile into multiple actual instructions on a given system. +// The assembler sometimes uses it for that purpose, as do we. +// Of course, if we are using it, we'd better not emit an instruction that +// will cause the assembler to smash it while we want it to be holding +// a live value. In general it is the architecture implementation's responsibility +// not to suggest the use of any such pseudo-instructions in situations +// where they would cause problems. +func (a *Asm) tmp() Reg { + return Reg{a.Arch.regTmp} +} + +// Carry returns the carry register, or else the zero Reg. +func (a *Asm) Carry() Reg { + return Reg{a.Arch.regCarry} +} + +// AltCarry returns the secondary carry register, or else the zero Reg. +func (a *Asm) AltCarry() Reg { + return Reg{a.Arch.regAltCarry} +} + +// Imm returns a Reg representing an immediate (constant) value. +func (a *Asm) Imm(x int) Reg { + if x == 0 && a.Arch.reg0 != "" { + return Reg{a.Arch.reg0} + } + return Reg{fmt.Sprintf("$%d", x)} +} + +// IsZero reports whether r is a zero immediate or the zero register. +func (a *Asm) IsZero(r Reg) bool { + return r.name == "$0" || a.Arch.reg0 != "" && r.name == a.Arch.reg0 +} + +// Reg allocates a new register. +func (a *Asm) Reg() Reg { + i := bits.TrailingZeros64(a.regavail) + if i == 64 { + a.Fatalf("out of registers") + } + a.regavail ^= 1 << i + return Reg{a.Arch.regs[i]} +} + +// RegHint allocates a new register, with a hint as to its purpose. +func (a *Asm) RegHint(hint Hint) Reg { + if name := a.hint(hint); name != "" { + i := slices.Index(a.Arch.regs, name) + if i < 0 { + return Reg{name} + } + if a.regavail&(1< 0 && bytes[len(bytes)-1] == '\n' { + a.out.Truncate(a.out.Len() - 1) + } + a.Comment(format, args...) +} + +// JmpEnable emits a test for the optional CPU feature that jumps to label if the feature is present. +// If JmpEnable returns false, the feature is not available on this architecture and no code was emitted. +func (a *Asm) JmpEnable(option Option, label string) bool { + jmpEnable := a.Arch.options[option] + if jmpEnable == nil { + return false + } + jmpEnable(a, label) + return true +} + +// Enabled reports whether the optional CPU feature is considered +// to be enabled at this point in the assembly output. +func (a *Asm) Enabled(option Option) bool { + return a.enabled[option] +} + +// SetOption changes whether the optional CPU feature should be +// considered to be enabled. +func (a *Asm) SetOption(option Option, on bool) { + a.enabled[option] = on +} + +// op3 emits a 3-operand instruction op src1, src2, dst, +// taking care to handle 2-operand machines and also +// to simplify the printout when src2==dst. +func (a *Asm) op3(op string, src1, src2, dst Reg) { + if op == "" { + a.Fatalf("missing instruction") + } + if src2 == dst { + // src2 and dst are same; print as 2-op form. + a.Printf("\t%s %s, %s\n", op, src1, dst) + } else if a.Arch.op3 != nil && !a.Arch.op3(op) { + // Machine does not have 3-op form for op; convert to 2-op. + if src1 == dst { + a.Fatalf("implicit mov %s, %s would smash src1", src2, dst) + } + a.Mov(src2, dst) + a.Printf("\t%s %s, %s\n", op, src1, dst) + } else { + // Full 3-op form. + a.Printf("\t%s %s, %s, %s\n", op, src1, src2, dst) + } +} + +// Mov emits dst = src. +func (a *Asm) Mov(src, dst Reg) { + if src != dst { + a.Printf("\t%s %s, %s\n", a.Arch.mov, src, dst) + } +} + +// AddWords emits dst = src1*WordBytes + src2. +// It does not set or use the carry flag. +func (a *Asm) AddWords(src1 Reg, src2, dst RegPtr) { + if a.Arch.addWords == "" { + // Note: Assuming that Lsh does not clobber the carry flag. + // Architectures where this is not true (x86) need to provide Arch.addWords. + t := a.Reg() + a.Lsh(a.Imm(bits.TrailingZeros(uint(a.Arch.WordBytes))), src1, t) + a.Add(t, Reg(src2), Reg(dst), KeepCarry) + a.Free(t) + return + } + a.Printf("\t"+a.Arch.addWords+"\n", src1, src2, dst) +} + +// And emits dst = src1 & src2 +// It may modify the carry flag. +func (a *Asm) And(src1, src2, dst Reg) { + a.op3(a.Arch.and, src1, src2, dst) +} + +// Or emits dst = src1 | src2 +// It may modify the carry flag. +func (a *Asm) Or(src1, src2, dst Reg) { + a.op3(a.Arch.or, src1, src2, dst) +} + +// Xor emits dst = src1 ^ src2 +// It may modify the carry flag. +func (a *Asm) Xor(src1, src2, dst Reg) { + a.op3(a.Arch.xor, src1, src2, dst) +} + +// Neg emits dst = -src. +// It may modify the carry flag. +func (a *Asm) Neg(src, dst Reg) { + if a.Arch.neg == "" { + if a.Arch.rsb != "" { + a.Printf("\t%s $0, %s, %s\n", a.Arch.rsb, src, dst) + return + } + if a.Arch.sub != "" && a.Arch.reg0 != "" { + a.Printf("\t%s %s, %s, %s\n", a.Arch.sub, src, a.Arch.reg0, dst) + return + } + a.Fatalf("missing neg") + } + if src == dst { + a.Printf("\t%s %s\n", a.Arch.neg, dst) + } else { + a.Printf("\t%s %s, %s\n", a.Arch.neg, src, dst) + } +} + +// HasRegShift reports whether the architecture can use shift expressions as operands. +func (a *Asm) HasRegShift() bool { + return a.Arch.regShift +} + +// LshReg returns a shift-expression operand src<>shift. +// If a.HasRegShift() == false, RshReg panics. +func (a *Asm) RshReg(shift, src Reg) Reg { + if !a.HasRegShift() { + a.Fatalf("no reg shift") + } + return Reg{fmt.Sprintf("%s>>%s", src, strings.TrimPrefix(shift.name, "$"))} +} + +// Rsh emits dst = src >> shift. +// It may modify the carry flag. +func (a *Asm) Rsh(shift, src, dst Reg) { + if need := a.hint(HintShiftCount); need != "" && shift.name != need && !shift.IsImm() { + a.Fatalf("shift count not in %s", need) + } + if a.HasRegShift() { + a.Mov(a.RshReg(shift, src), dst) + return + } + a.op3(a.Arch.rsh, shift, src, dst) +} + +// RshWide emits dst = src >> shift with high bits shifted from adj. +// It may modify the carry flag. +func (a *Asm) RshWide(shift, adj, src, dst Reg) { + if a.Arch.lshd == "" { + a.Fatalf("no rshwide on %s", a.Arch.Name) + } + if need := a.hint(HintShiftCount); need != "" && shift.name != need && !shift.IsImm() { + a.Fatalf("shift count not in %s", need) + } + a.op3(fmt.Sprintf("%s %s,", a.Arch.rshd, shift), adj, src, dst) +} + +// SLTU emits dst = src2 < src1 (0 or 1), using an unsigned comparison. +func (a *Asm) SLTU(src1, src2, dst Reg) { + switch { + default: + a.Fatalf("arch has no sltu/sgtu") + case a.Arch.sltu != "": + a.Printf("\t%s %s, %s, %s\n", a.Arch.sltu, src1, src2, dst) + case a.Arch.sgtu != "": + a.Printf("\t%s %s, %s, %s\n", a.Arch.sgtu, src2, src1, dst) + } +} + +// Add emits dst = src1+src2, with the specified carry behavior. +func (a *Asm) Add(src1, src2, dst Reg, carry Carry) { + switch { + default: + a.Fatalf("unsupported carry behavior") + case a.Arch.addF != nil && a.Arch.addF(a, src1, src2, dst, carry): + // handled + case a.Arch.add != "" && (carry == KeepCarry || carry == SmashCarry): + a.op3(a.Arch.add, src1, src2, dst) + case a.Arch.adds != "" && (carry == SetCarry || carry == SmashCarry): + a.op3(a.Arch.adds, src1, src2, dst) + case a.Arch.adc != "" && (carry == UseCarry || carry == UseCarry|SmashCarry): + a.op3(a.Arch.adc, src1, src2, dst) + case a.Arch.adcs != "" && (carry == UseCarry|SetCarry || carry == UseCarry|SmashCarry): + a.op3(a.Arch.adcs, src1, src2, dst) + case a.Arch.lea != "" && (carry == KeepCarry || carry == SmashCarry): + if src1.IsImm() { + a.Printf("\t%s %s(%s), %s\n", a.Arch.lea, src1.name[1:], src2, dst) // name[1:] removes $ + } else { + a.Printf("\t%s (%s)(%s), %s\n", a.Arch.lea, src1, src2, dst) + } + if src2 == dst { + a.EOL("ADD %s, %s", src1, dst) + } else { + a.EOL("ADD %s, %s, %s", src1, src2, dst) + } + + case a.Arch.add != "" && a.Arch.regCarry != "": + // Machine has no carry flag; instead we've dedicated a register + // and use SLTU/SGTU (set less-than/greater-than unsigned) + // to compute the carry flags as needed. + // For ADD x, y, z, SLTU x/y, z, c computes the carry (borrow) bit. + // Either of x or y can be used as the second argument, provided + // it is not aliased to z. + // To make the output less of a wall of instructions, + // we comment the “higher-level” operation, with ... marking + // continued instructions implementing the operation. + cr := a.Carry() + if carry&AltCarry != 0 { + cr = a.AltCarry() + if !cr.Valid() { + a.Fatalf("alt carry not supported") + } + carry &^= AltCarry + } + tmp := a.tmp() + if !tmp.Valid() { + a.Fatalf("cannot simulate sub carry without regTmp") + } + switch carry { + default: + a.Fatalf("unsupported carry behavior") + case UseCarry, UseCarry | SmashCarry: + // Easy case, just add the carry afterward. + if a.IsZero(src1) { + // Only here to use the carry. + a.Add(cr, src2, dst, KeepCarry) + a.EOL("ADC $0, %s, %s", src2, dst) + break + } + a.Add(src1, src2, dst, KeepCarry) + a.EOL("ADC %s, %s, %s (cr=%s)", src1, src2, dst, cr) + a.Add(cr, dst, dst, KeepCarry) + a.EOL("...") + + case SetCarry: + if a.IsZero(src1) && src2 == dst { + // Only here to clear the carry flag. (Caller will comment.) + a.Xor(cr, cr, cr) + break + } + var old Reg // old is a src distinct from dst + switch { + case dst != src1: + old = src1 + case dst != src2: + old = src2 + default: + // src1 == src2 == dst. + // Overflows if and only if the high bit is set, so copy high bit to carry. + a.Rsh(a.Imm(a.Arch.WordBits-1), src1, cr) + a.EOL("ADDS %s, %s, %s (cr=%s)", src1, src2, dst, cr) + a.Add(src1, src2, dst, KeepCarry) + a.EOL("...") + return + } + a.Add(src1, src2, dst, KeepCarry) + a.EOL("ADDS %s, %s, %s (cr=%s)", src1, src2, dst, cr) + a.SLTU(old, dst, cr) // dst < old (one of the src) implies carry + a.EOL("...") + + case UseCarry | SetCarry: + if a.IsZero(src1) { + // Only here to use and then set the carry. + // Easy since carry is not aliased to dst. + a.Add(cr, src2, dst, KeepCarry) + a.EOL("ADCS $0, %s, %s (cr=%s)", src2, dst, cr) + a.SLTU(cr, dst, cr) // dst < cr implies carry + a.EOL("...") + break + } + // General case. Need to do two different adds (src1 + src2 + cr), + // computing carry bits for both, and add'ing them together. + // Start with src1+src2. + var old Reg // old is a src distinct from dst + switch { + case dst != src1: + old = src1 + case dst != src2: + old = src2 + } + if old.Valid() { + a.Add(src1, src2, dst, KeepCarry) + a.EOL("ADCS %s, %s, %s (cr=%s)", src1, src2, dst, cr) + a.SLTU(old, dst, tmp) // // dst < old (one of the src) implies carry + a.EOL("...") + } else { + // src1 == src2 == dst, like above. Sign bit is carry bit, + // but we copy it into tmp, not cr. + a.Rsh(a.Imm(a.Arch.WordBits-1), src1, tmp) + a.EOL("ADCS %s, %s, %s (cr=%s)", src1, src2, dst, cr) + a.Add(src1, src2, dst, KeepCarry) + a.EOL("...") + } + // Add cr to dst. + a.Add(cr, dst, dst, KeepCarry) + a.EOL("...") + a.SLTU(cr, dst, cr) // sum < cr implies carry + a.EOL("...") + // Add the two carry bits (at most one can be set, because (2⁶⁴-1)+(2⁶⁴-1)+1 < 2·2⁶⁴). + a.Add(tmp, cr, cr, KeepCarry) + a.EOL("...") + } + } +} + +// Sub emits dst = src2-src1, with the specified carry behavior. +func (a *Asm) Sub(src1, src2, dst Reg, carry Carry) { + switch { + default: + a.Fatalf("unsupported carry behavior") + case a.Arch.subF != nil && a.Arch.subF(a, src1, src2, dst, carry): + // handled + case a.Arch.sub != "" && (carry == KeepCarry || carry == SmashCarry): + a.op3(a.Arch.sub, src1, src2, dst) + case a.Arch.subs != "" && (carry == SetCarry || carry == SmashCarry): + a.op3(a.Arch.subs, src1, src2, dst) + case a.Arch.sbc != "" && (carry == UseCarry || carry == UseCarry|SmashCarry): + a.op3(a.Arch.sbc, src1, src2, dst) + case a.Arch.sbcs != "" && (carry == UseCarry|SetCarry || carry == UseCarry|SmashCarry): + a.op3(a.Arch.sbcs, src1, src2, dst) + case strings.HasPrefix(src1.name, "$") && (carry == KeepCarry || carry == SmashCarry): + // Running out of options; if this is an immediate + // and we don't need to worry about carry semantics, + // try adding the negation. + if strings.HasPrefix(src1.name, "$-") { + src1.name = "$" + src1.name[2:] + } else { + src1.name = "$-" + src1.name[1:] + } + a.Add(src1, src2, dst, carry) + + case a.Arch.sub != "" && a.Arch.regCarry != "": + // Machine has no carry flag; instead we've dedicated a register + // and use SLTU/SGTU (set less-than/greater-than unsigned) + // to compute the carry bits as needed. + // For SUB x, y, z, SLTU x, y, c computes the carry (borrow) bit. + // To make the output less of a wall of instructions, + // we comment the “higher-level” operation, with ... marking + // continued instructions implementing the operation. + // Be careful! Subtract and add have different overflow behaviors, + // so the details here are NOT the same as in Add above. + cr := a.Carry() + if carry&AltCarry != 0 { + a.Fatalf("alt carry not supported") + } + tmp := a.tmp() + if !tmp.Valid() { + a.Fatalf("cannot simulate carry without regTmp") + } + switch carry { + default: + a.Fatalf("unsupported carry behavior") + case UseCarry, UseCarry | SmashCarry: + // Easy case, just subtract the carry afterward. + if a.IsZero(src1) { + // Only here to use the carry. + a.Sub(cr, src2, dst, KeepCarry) + a.EOL("SBC $0, %s, %s", src2, dst) + break + } + a.Sub(src1, src2, dst, KeepCarry) + a.EOL("SBC %s, %s, %s", src1, src2, dst) + a.Sub(cr, dst, dst, KeepCarry) + a.EOL("...") + + case SetCarry: + if a.IsZero(src1) && src2 == dst { + // Only here to clear the carry flag. + a.Xor(cr, cr, cr) + break + } + // Compute the new carry first, in case dst is src1 or src2. + a.SLTU(src1, src2, cr) + a.EOL("SUBS %s, %s, %s", src1, src2, dst) + a.Sub(src1, src2, dst, KeepCarry) + a.EOL("...") + + case UseCarry | SetCarry: + if a.IsZero(src1) { + // Only here to use and then set the carry. + if src2 == dst { + // Unfortunate case. Using src2==dst is common (think x -= y) + // and also more efficient on two-operand machines (like x86), + // but here subtracting from dst will smash src2, making it + // impossible to recover the carry information after the SUB. + // But we want to use the carry, so we can't compute it before + // the SUB either. Compute into a temporary and MOV. + a.SLTU(cr, src2, tmp) + a.EOL("SBCS $0, %s, %s", src2, dst) + a.Sub(cr, src2, dst, KeepCarry) + a.EOL("...") + a.Mov(tmp, cr) + a.EOL("...") + break + } + a.Sub(cr, src2, dst, KeepCarry) // src2 not dst, so src2 preserved + a.SLTU(cr, src2, cr) + break + } + // General case. Need to do two different subtracts (src2 - cr - src1), + // computing carry bits for both, and add'ing them together. + // Doing src2 - cr first frees up cr to store the carry from the sub of src1. + a.SLTU(cr, src2, tmp) + a.EOL("SBCS %s, %s, %s", src1, src2, dst) + a.Sub(cr, src2, dst, KeepCarry) + a.EOL("...") + a.SLTU(src1, dst, cr) + a.EOL("...") + a.Sub(src1, dst, dst, KeepCarry) + a.EOL("...") + a.Add(tmp, cr, cr, KeepCarry) + a.EOL("...") + } + } +} + +// ClearCarry clears the carry flag. +// The ‘which’ parameter must be AddCarry or SubCarry to specify how the flag will be used. +// (On some systems, the sub carry's actual processor bit is inverted from its usual value.) +func (a *Asm) ClearCarry(which Carry) { + dst := Reg{a.Arch.regs[0]} // not actually modified + switch which & (AddCarry | SubCarry) { + default: + a.Fatalf("bad carry") + case AddCarry: + a.Add(a.Imm(0), dst, dst, SetCarry|which&AltCarry) + case SubCarry: + a.Sub(a.Imm(0), dst, dst, SetCarry|which&AltCarry) + } + a.EOL("clear carry") +} + +// SaveCarry saves the carry flag into dst. +// The meaning of the bits in dst is architecture-dependent. +// The carry flag is left in an undefined state. +func (a *Asm) SaveCarry(dst Reg) { + // Note: As implemented here, the carry flag is actually left unmodified, + // but we say it is in an undefined state in case that changes in the future. + // (The SmashCarry could be changed to SetCarry if so.) + if cr := a.Carry(); cr.Valid() { + if cr == dst { + return // avoid EOL + } + a.Mov(cr, dst) + } else { + a.Sub(dst, dst, dst, UseCarry|SmashCarry) + } + a.EOL("save carry") +} + +// RestoreCarry restores the carry flag from src. +// src is left in an undefined state. +func (a *Asm) RestoreCarry(src Reg) { + if cr := a.Carry(); cr.Valid() { + if cr == src { + return // avoid EOL + } + a.Mov(src, cr) + } else if a.Arch.subCarryIsBorrow { + a.Add(src, src, src, SetCarry) + } else { + // SaveCarry saved the sub carry flag with an encoding of 0, 1 -> 0, ^0. + // Restore it by subtracting from a value less than ^0, which will carry if src != 0. + // If there is no zero register, the SP register is guaranteed to be less than ^0. + // (This may seem too clever, but on GOARCH=arm we have no other good options.) + a.Sub(src, cmp.Or(a.ZR(), Reg{"SP"}), src, SetCarry) + } + a.EOL("restore carry") +} + +// ConvertCarry converts the carry flag in dst from the internal format to a 0 or 1. +// The carry flag is left in an undefined state. +func (a *Asm) ConvertCarry(which Carry, dst Reg) { + if a.Carry().Valid() { // already 0 or 1 + return + } + switch which { + case AddCarry: + if a.Arch.subCarryIsBorrow { + a.Neg(dst, dst) + } else { + a.Add(a.Imm(1), dst, dst, SmashCarry) + } + a.EOL("convert add carry") + case SubCarry: + a.Neg(dst, dst) + a.EOL("convert sub carry") + } +} + +// SaveConvertCarry saves and converts the carry flag into dst: 0 unset, 1 set. +// The carry flag is left in an undefined state. +func (a *Asm) SaveConvertCarry(which Carry, dst Reg) { + switch which { + default: + a.Fatalf("bad carry") + case AddCarry: + if (a.Arch.adc != "" || a.Arch.adcs != "") && a.ZR().Valid() { + a.Add(a.ZR(), a.ZR(), dst, UseCarry|SmashCarry) + a.EOL("save & convert add carry") + return + } + case SubCarry: + // no special cases + } + a.SaveCarry(dst) + a.ConvertCarry(which, dst) +} + +// MulWide emits dstlo = src1 * src2 and dsthi = (src1 * src2) >> WordBits. +// The carry flag is left in an undefined state. +// If dstlo or dsthi is the zero Reg, then those outputs are discarded. +func (a *Asm) MulWide(src1, src2, dstlo, dsthi Reg) { + switch { + default: + a.Fatalf("mulwide not available") + case a.Arch.mulWideF != nil: + a.Arch.mulWideF(a, src1, src2, dstlo, dsthi) + case a.Arch.mul != "" && !dsthi.Valid(): + a.op3(a.Arch.mul, src1, src2, dstlo) + case a.Arch.mulhi != "" && !dstlo.Valid(): + a.op3(a.Arch.mulhi, src1, src2, dsthi) + case a.Arch.mul != "" && a.Arch.mulhi != "" && dstlo != src1 && dstlo != src2: + a.op3(a.Arch.mul, src1, src2, dstlo) + a.op3(a.Arch.mulhi, src1, src2, dsthi) + case a.Arch.mul != "" && a.Arch.mulhi != "" && dsthi != src1 && dsthi != src2: + a.op3(a.Arch.mulhi, src1, src2, dsthi) + a.op3(a.Arch.mul, src1, src2, dstlo) + } +} + +// Jmp jumps to the label. +func (a *Asm) Jmp(label string) { + // Note: Some systems prefer the spelling B or BR, but all accept JMP. + a.Printf("\tJMP %s\n", label) +} + +// JmpZero jumps to the label if src is zero. +// It may modify the carry flag unless a.Arch.CarrySafeLoop is true. +func (a *Asm) JmpZero(src Reg, label string) { + a.Printf("\t"+a.Arch.jmpZero+"\n", src, label) +} + +// JmpNonZero jumps to the label if src is non-zero. +// It may modify the carry flag unless a.Arch,CarrySafeLoop is true. +func (a *Asm) JmpNonZero(src Reg, label string) { + a.Printf("\t"+a.Arch.jmpNonZero+"\n", src, label) +} + +// Label emits a label with the given name. +func (a *Asm) Label(name string) { + a.Printf("%s:\n", name) +} + +// Ret returns. +func (a *Asm) Ret() { + a.Printf("\tRET\n") +} diff --git a/src/math/big/internal/asmgen/cheat.go b/src/math/big/internal/asmgen/cheat.go new file mode 100644 index 00000000000000..9faf6d0483bcc9 --- /dev/null +++ b/src/math/big/internal/asmgen/cheat.go @@ -0,0 +1,55 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ignore + +// This program can be compiled with -S to produce a “cheat sheet” +// for filling out a new Arch: the compiler will show you how to implement +// the various operations. +// +// Usage (replace TARGET with your target architecture): +// +// GOOS=linux GOARCH=TARGET go build -gcflags='-p=cheat -S' cheat.go + +package p + +import "math/bits" + +func mov(x, y uint) uint { return y } +func zero() uint { return 0 } +func add(x, y uint) uint { return x + y } +func adds(x, y, c uint) (uint, uint) { return bits.Add(x, y, 0) } +func adcs(x, y, c uint) (uint, uint) { return bits.Add(x, y, c) } +func sub(x, y uint) uint { return x + y } +func subs(x, y uint) (uint, uint) { return bits.Sub(x, y, 0) } +func sbcs(x, y, c uint) (uint, uint) { return bits.Sub(x, y, c) } +func mul(x, y uint) uint { return x * y } +func mulWide(x, y uint) (uint, uint) { return bits.Mul(x, y) } +func lsh(x, s uint) uint { return x << s } +func rsh(x, s uint) uint { return x >> s } +func and(x, y uint) uint { return x & y } +func or(x, y uint) uint { return x | y } +func xor(x, y uint) uint { return x ^ y } +func neg(x uint) uint { return -x } +func loop(x int) int { + s := 0 + for i := 1; i < x; i++ { + s += i + if s == 98 { // useful for jmpEqual + return 99 + } + if s == 99 { + return 100 + } + if s == 0 { // useful for jmpZero + return 101 + } + if s != 0 { // useful for jmpNonZero + s *= 3 + } + s += 2 // keep last condition from being inverted + } + return s +} +func mem(x *[10]struct{ a, b uint }, i int) uint { return x[i].b } diff --git a/src/math/big/internal/asmgen/func.go b/src/math/big/internal/asmgen/func.go new file mode 100644 index 00000000000000..8a762febce09b3 --- /dev/null +++ b/src/math/big/internal/asmgen/func.go @@ -0,0 +1,138 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +import ( + "fmt" + "slices" + "strings" +) + +// Note: Exported fields and methods are expected to be used +// by function generators (like the ones in add.go and so on). +// Unexported fields and methods should not be. + +// A Func represents a single assembly function. +type Func struct { + Name string + Asm *Asm + inputs []string // name of input slices (not beginning with z) + outputs []string // names of output slices (beginning with z) + args map[string]int // offsets of args, results on stack +} + +// Func starts a new function in the assembly output. +func (a *Asm) Func(decl string) *Func { + d, ok := strings.CutPrefix(decl, "func ") + if !ok { + a.Fatalf("func decl does not begin with 'func '") + } + name, d, ok := strings.Cut(d, "(") + if !ok { + a.Fatalf("func decl does not have func arg list") + } + f := &Func{ + Name: name, + Asm: a, + args: make(map[string]int), + } + a.FreeAll() + + // Parse argument names and types. Quick and dirty. + // Convert (args) (results) into args, results. + d = strings.ReplaceAll(d, ") (", ", ") + d = strings.TrimSuffix(d, ")") + args := strings.Split(d, ",") + + // Assign implicit types to all arguments (x, y int -> x int, y int). + typ := "" + for i, arg := range slices.Backward(args) { + arg = strings.TrimSpace(arg) + if !strings.Contains(arg, " ") { + if typ == "" { + a.Fatalf("missing argument type") + } + arg += " " + typ + } else { + _, typ, _ = strings.Cut(arg, " ") + } + args[i] = arg + } + + // Record mapping from names to offsets. + off := 0 + for _, arg := range args { + name, typ, _ := strings.Cut(arg, " ") + switch typ { + default: + a.Fatalf("unknown type %s", typ) + case "Word", "uint", "int": + f.args[name] = off + off += a.Arch.WordBytes + case "[]Word": + if strings.HasPrefix(name, "z") { + f.outputs = append(f.outputs, name) + } else { + f.inputs = append(f.inputs, name) + } + f.args[name+"_base"] = off + f.args[name+"_len"] = off + a.Arch.WordBytes + f.args[name+"_cap"] = off + 2*a.Arch.WordBytes + off += 3 * a.Arch.WordBytes + } + } + + a.Printf("\n") + a.Printf("// %s\n", decl) + a.Printf("TEXT ·%s(SB), NOSPLIT, $0\n", name) + if a.Arch.setup != nil { + a.Arch.setup(f) + } + return f +} + +// Arg allocates a new register, copies the named argument (or result) into it, +// and returns that register. +func (f *Func) Arg(name string) Reg { + return f.ArgHint(name, HintNone) +} + +// ArgHint is like Arg but uses a register allocation hint. +func (f *Func) ArgHint(name string, hint Hint) Reg { + off, ok := f.args[name] + if !ok { + f.Asm.Fatalf("unknown argument %s", name) + } + mem := Reg{fmt.Sprintf("%s+%d(FP)", name, off)} + if hint == HintMemOK && f.Asm.Arch.memOK { + return mem + } + r := f.Asm.RegHint(hint) + f.Asm.Mov(mem, r) + return r +} + +// ArgPtr is like Arg but returns a RegPtr. +func (f *Func) ArgPtr(name string) RegPtr { + return RegPtr(f.Arg(name)) +} + +// StoreArg stores src into the named argument (or result). +func (f *Func) StoreArg(src Reg, name string) { + off, ok := f.args[name] + if !ok { + f.Asm.Fatalf("unknown argument %s", name) + } + a := f.Asm + mem := Reg{fmt.Sprintf("%s+%d(FP)", name, off)} + if src.IsImm() && !a.Arch.memOK { + r := a.Reg() + a.Mov(src, r) + a.Mov(r, mem) + a.Free(r) + return + } + a.Mov(src, mem) +} diff --git a/src/math/big/internal/asmgen/loong64.go b/src/math/big/internal/asmgen/loong64.go new file mode 100644 index 00000000000000..e2d05690ab4de4 --- /dev/null +++ b/src/math/big/internal/asmgen/loong64.go @@ -0,0 +1,45 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +var ArchLoong64 = &Arch{ + Name: "loong64", + WordBits: 64, + WordBytes: 8, + CarrySafeLoop: true, + + regs: []string{ + // R0 is set to 0. + // R1 is LR. + // R2 is ??? + // R3 is SP. + // R22 is g. + // R28 and R29 are our virtual carry flags. + // R30 is the linker/assembler temp, which we use too. + "R4", "R5", "R6", "R7", "R8", "R9", + "R10", "R11", "R12", "R13", "R14", "R15", "R16", "R17", "R18", "R19", + "R20", "R21", "R23", "R24", "R25", "R26", "R27", + "R31", + }, + reg0: "R0", + regCarry: "R28", + regAltCarry: "R29", + regTmp: "R30", + + mov: "MOVV", + add: "ADDVU", + sub: "SUBVU", + sltu: "SGTU", + mul: "MULV", + mulhi: "MULHVU", + lsh: "SLLV", + rsh: "SRLV", + and: "AND", + or: "OR", + xor: "XOR", + + jmpZero: "BEQ %s, %s", + jmpNonZero: "BNE %s, %s", +} diff --git a/src/math/big/internal/asmgen/main.go b/src/math/big/internal/asmgen/main.go new file mode 100644 index 00000000000000..9ca231f24b757d --- /dev/null +++ b/src/math/big/internal/asmgen/main.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Asmgen generates math/big assembly. +// +// Usage: +// +// cd go/src/math/big +// go test ./internal/asmgen -generate +// +// Or: +// +// go generate math/big +package asmgen + +var arches = []*Arch{ + Arch386, + ArchAMD64, + ArchARM, + ArchARM64, + ArchLoong64, + ArchMIPS, + ArchMIPS64x, + ArchPPC64x, + ArchRISCV64, + ArchS390X, +} + +// generate returns the file name and content of the generated assembly for the given architecture. +func generate(arch *Arch) (file string, data []byte) { + file = "arith_" + arch.Name + ".s" + a := NewAsm(arch) + addOrSubVV(a, "addVV") + addOrSubVV(a, "subVV") + shiftVU(a, "lshVU") + shiftVU(a, "rshVU") + mulAddVWW(a) + addMulVVWW(a) + return file, a.out.Bytes() +} diff --git a/src/math/big/internal/asmgen/main_test.go b/src/math/big/internal/asmgen/main_test.go new file mode 100644 index 00000000000000..adb0e0d4e84b69 --- /dev/null +++ b/src/math/big/internal/asmgen/main_test.go @@ -0,0 +1,37 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +import ( + "bytes" + "flag" + "internal/diff" + "os" + "testing" +) + +var generateFlag = flag.Bool("generate", false, "generate files") + +func Test(t *testing.T) { + for _, arch := range arches { + t.Run(arch.Name, func(t *testing.T) { + file, data := generate(arch) + old, err := os.ReadFile("../../" + file) + if err == nil && bytes.Equal(old, data) { + return + } + if *generateFlag { + if err := os.WriteFile("../../"+file, data, 0o666); err != nil { + t.Fatal(err) + } + return + } + if err != nil { + t.Fatal(err) + } + t.Fatalf("generated assembly differs:\n%s\n", diff.Diff("../../"+file, old, "regenerated", data)) + }) + } +} diff --git a/src/math/big/internal/asmgen/mips.go b/src/math/big/internal/asmgen/mips.go new file mode 100644 index 00000000000000..54734881dd6acc --- /dev/null +++ b/src/math/big/internal/asmgen/mips.go @@ -0,0 +1,48 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +var ArchMIPS = &Arch{ + Name: "mipsx", + Build: "mips || mipsle", + WordBits: 32, + WordBytes: 4, + CarrySafeLoop: true, + + regs: []string{ + // R0 is 0 + // R23 is the assembler/linker temporary (which we use too). + // R24 and R25 are our virtual carry flags. + // R28 is SB. + // R29 is SP. + // R30 is g. + // R31 is LR. + "R1", "R2", "R3", "R4", "R5", "R6", "R7", "R8", "R9", + "R10", "R11", "R12", "R13", "R14", "R15", "R16", "R17", "R18", "R19", + "R20", "R21", "R22", "R24", "R25", + }, + reg0: "R0", + regTmp: "R23", + regCarry: "R24", + regAltCarry: "R25", + + mov: "MOVW", + add: "ADDU", + sltu: "SGTU", // SGTU args are swapped, so it's really SLTU + sub: "SUBU", + mulWideF: mipsMulWide, + lsh: "SLL", + rsh: "SRL", + and: "AND", + or: "OR", + xor: "XOR", + + jmpZero: "BEQ %s, %s", + jmpNonZero: "BNE %s, %s", +} + +func mipsMulWide(a *Asm, src1, src2, dstlo, dsthi Reg) { + a.Printf("\tMULU %s, %s\n\tMOVW LO, %s\n\tMOVW HI, %s\n", src1, src2, dstlo, dsthi) +} diff --git a/src/math/big/internal/asmgen/mips64.go b/src/math/big/internal/asmgen/mips64.go new file mode 100644 index 00000000000000..0ad7af505e926c --- /dev/null +++ b/src/math/big/internal/asmgen/mips64.go @@ -0,0 +1,48 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +var ArchMIPS64x = &Arch{ + Name: "mips64x", + Build: "mips64 || mips64le", + WordBits: 64, + WordBytes: 8, + CarrySafeLoop: true, + + regs: []string{ + // R0 is 0 + // R23 is the assembler/linker temporary (which we use too). + // R24 and R25 are our virtual carry flags. + // R28 is SB. + // R29 is SP. + // R30 is g. + // R31 is LR. + "R1", "R2", "R3", "R4", "R5", "R6", "R7", "R8", "R9", + "R10", "R11", "R12", "R13", "R14", "R15", "R16", "R17", "R18", "R19", + "R20", "R21", "R22", "R24", "R25", + }, + reg0: "R0", + regTmp: "R23", + regCarry: "R24", + regAltCarry: "R25", + + mov: "MOVV", + add: "ADDVU", + sltu: "SGTU", // SGTU args are swapped, so it's really SLTU + sub: "SUBVU", + mulWideF: mips64MulWide, + lsh: "SLLV", + rsh: "SRLV", + and: "AND", + or: "OR", + xor: "XOR", + + jmpZero: "BEQ %s, %s", + jmpNonZero: "BNE %s, %s", +} + +func mips64MulWide(a *Asm, src1, src2, dstlo, dsthi Reg) { + a.Printf("\tMULVU %s, %s\n\tMOVV LO, %s\n\tMOVV HI, %s\n", src1, src2, dstlo, dsthi) +} diff --git a/src/math/big/internal/asmgen/mul.go b/src/math/big/internal/asmgen/mul.go new file mode 100644 index 00000000000000..007bfc9060f805 --- /dev/null +++ b/src/math/big/internal/asmgen/mul.go @@ -0,0 +1,320 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +// mulAddVWW generates mulAddVWW, which does z, c = x*m + a. +func mulAddVWW(a *Asm) { + f := a.Func("func mulAddVWW(z, x []Word, m, a Word) (c Word)") + + if a.AltCarry().Valid() { + addMulVirtualCarry(f, 0) + return + } + addMul(f, "", "x", 0) +} + +// addMulVVWW generates addMulVVWW which does z, c = x + y*m + a. +// (A more pedantic name would be addMulAddVVWW.) +func addMulVVWW(a *Asm) { + f := a.Func("func addMulVVWW(z, x, y []Word, m, a Word) (c Word)") + + // If the architecture has virtual carries, emit that version unconditionally. + if a.AltCarry().Valid() { + addMulVirtualCarry(f, 1) + return + } + + // If the architecture optionally has two carries, test and emit both versions. + if a.JmpEnable(OptionAltCarry, "altcarry") { + regs := a.RegsUsed() + addMul(f, "x", "y", 1) + a.Label("altcarry") + a.SetOption(OptionAltCarry, true) + a.SetRegsUsed(regs) + addMulAlt(f) + a.SetOption(OptionAltCarry, false) + return + } + + // Otherwise emit the one-carry form. + addMul(f, "x", "y", 1) +} + +// Computing z = addsrc + m*mulsrc + a, we need: +// +// for i := range z { +// lo, hi := m * mulsrc[i] +// lo, carry = bits.Add(lo, a, 0) +// lo, carryAlt = bits.Add(lo, addsrc[i], 0) +// z[i] = lo +// a = hi + carry + carryAlt // cannot overflow +// } +// +// The final addition cannot overflow because after processing N words, +// the maximum possible value is (for a 64-bit system): +// +// (2**64N - 1) + (2**64 - 1)*(2**64N - 1) + (2**64 - 1) +// = (2**64)*(2**64N - 1) + (2**64 - 1) +// = 2**64(N+1) - 1, +// +// which fits in N+1 words (the high order one being the new value of a). +// +// (For example, with 3 decimal words, 999 + 9*999 + 9 = 999*10 + 9 = 9999.) +// +// If we unroll the loop a bit, then we can chain the carries in two passes. +// Consider: +// +// lo0, hi0 := m * mulsrc[i] +// lo0, carry = bits.Add(lo0, a, 0) +// lo0, carryAlt = bits.Add(lo0, addsrc[i], 0) +// z[i] = lo0 +// a = hi + carry + carryAlt // cannot overflow +// +// lo1, hi1 := m * mulsrc[i] +// lo1, carry = bits.Add(lo1, a, 0) +// lo1, carryAlt = bits.Add(lo1, addsrc[i], 0) +// z[i] = lo1 +// a = hi + carry + carryAlt // cannot overflow +// +// lo2, hi2 := m * mulsrc[i] +// lo2, carry = bits.Add(lo2, a, 0) +// lo2, carryAlt = bits.Add(lo2, addsrc[i], 0) +// z[i] = lo2 +// a = hi + carry + carryAlt // cannot overflow +// +// lo3, hi3 := m * mulsrc[i] +// lo3, carry = bits.Add(lo3, a, 0) +// lo3, carryAlt = bits.Add(lo3, addsrc[i], 0) +// z[i] = lo3 +// a = hi + carry + carryAlt // cannot overflow +// +// There are three ways we can optimize this sequence. +// +// (1) Reordering, we can chain carries so that we can use one hardware carry flag +// but amortize the cost of saving and restoring it across multiple instructions: +// +// // multiply +// lo0, hi0 := m * mulsrc[i] +// lo1, hi1 := m * mulsrc[i+1] +// lo2, hi2 := m * mulsrc[i+2] +// lo3, hi3 := m * mulsrc[i+3] +// +// lo0, carry = bits.Add(lo0, a, 0) +// lo1, carry = bits.Add(lo1, hi0, carry) +// lo2, carry = bits.Add(lo2, hi1, carry) +// lo3, carry = bits.Add(lo3, hi2, carry) +// a = hi3 + carry // cannot overflow +// +// // add +// lo0, carryAlt = bits.Add(lo0, addsrc[i], 0) +// lo1, carryAlt = bits.Add(lo1, addsrc[i+1], carryAlt) +// lo2, carryAlt = bits.Add(lo2, addsrc[i+2], carryAlt) +// lo3, carryAlt = bits.Add(lo3, addrsc[i+3], carryAlt) +// a = a + carryAlt // cannot overflow +// +// z[i] = lo0 +// z[i+1] = lo1 +// z[i+2] = lo2 +// z[i+3] = lo3 +// +// addMul takes this approach, using the hardware carry flag +// first for carry and then for carryAlt. +// +// (2) addMulAlt assumes there are two hardware carry flags available. +// It dedicates one each to carry and carryAlt, so that a multi-block +// unrolling can keep the flags in hardware across all the blocks. +// So even if the block size is 1, the code can do: +// +// // multiply and add +// lo0, hi0 := m * mulsrc[i] +// lo0, carry = bits.Add(lo0, a, 0) +// lo0, carryAlt = bits.Add(lo0, addsrc[i], 0) +// z[i] = lo0 +// +// lo1, hi1 := m * mulsrc[i+1] +// lo1, carry = bits.Add(lo1, hi0, carry) +// lo1, carryAlt = bits.Add(lo1, addsrc[i+1], carryAlt) +// z[i+1] = lo1 +// +// lo2, hi2 := m * mulsrc[i+2] +// lo2, carry = bits.Add(lo2, hi1, carry) +// lo2, carryAlt = bits.Add(lo2, addsrc[i+2], carryAlt) +// z[i+2] = lo2 +// +// lo3, hi3 := m * mulsrc[i+3] +// lo3, carry = bits.Add(lo3, hi2, carry) +// lo3, carryAlt = bits.Add(lo3, addrsc[i+3], carryAlt) +// z[i+3] = lo2 +// +// a = hi3 + carry + carryAlt // cannot overflow +// +// (3) addMulVirtualCarry optimizes for systems with explicitly computed carry bits +// (loong64, mips, riscv64), cutting the number of actual instructions almost by half. +// Look again at the original word-at-a-time version: +// +// lo1, hi1 := m * mulsrc[i] +// lo1, carry = bits.Add(lo1, a, 0) +// lo1, carryAlt = bits.Add(lo1, addsrc[i], 0) +// z[i] = lo1 +// a = hi + carry + carryAlt // cannot overflow +// +// Although it uses four adds per word, those are cheap adds: the two bits.Add adds +// use two instructions each (ADD+SLTU) and the final + adds only use one ADD each, +// for a total of 6 instructions per word. In contrast, the middle stanzas in (2) use +// only two “adds” per word, but these are SetCarry|UseCarry adds, which compile to +// five instruction each, for a total of 10 instructions per word. So the word-at-a-time +// loop is actually better. And we can reorder things slightly to use only a single carry bit: +// +// lo1, hi1 := m * mulsrc[i] +// lo1, carry = bits.Add(lo1, a, 0) +// a = hi + carry +// lo1, carry = bits.Add(lo1, addsrc[i], 0) +// a = a + carry +// z[i] = lo1 +func addMul(f *Func, addsrc, mulsrc string, mulIndex int) { + a := f.Asm + mh := HintNone + if a.Arch == Arch386 && addsrc != "" { + mh = HintMemOK // too few registers otherwise + } + m := f.ArgHint("m", mh) + c := f.Arg("a") + n := f.Arg("z_len") + + p := f.Pipe() + if addsrc != "" { + p.SetHint(addsrc, HintMemOK) + } + p.SetHint(mulsrc, HintMulSrc) + unroll := []int{1, 4} + switch a.Arch { + case Arch386: + unroll = []int{1} // too few registers + case ArchARM: + p.SetMaxColumns(2) // too few registers (but more than 386) + case ArchARM64: + unroll = []int{1, 8} // 5% speedup on c4as16 + } + + // See the large comment above for an explanation of the code being generated. + // This is optimization strategy 1. + p.Start(n, unroll...) + p.Loop(func(in, out [][]Reg) { + a.Comment("multiply") + prev := c + flag := SetCarry + for i, x := range in[mulIndex] { + hi := a.RegHint(HintMulHi) + a.MulWide(m, x, x, hi) + a.Add(prev, x, x, flag) + flag = UseCarry | SetCarry + if prev != c { + a.Free(prev) + } + out[0][i] = x + prev = hi + } + a.Add(a.Imm(0), prev, c, UseCarry|SmashCarry) + if addsrc != "" { + a.Comment("add") + flag := SetCarry + for i, x := range in[0] { + a.Add(x, out[0][i], out[0][i], flag) + flag = UseCarry | SetCarry + } + a.Add(a.Imm(0), c, c, UseCarry|SmashCarry) + } + p.StoreN(out) + }) + + f.StoreArg(c, "c") + a.Ret() +} + +func addMulAlt(f *Func) { + a := f.Asm + m := f.ArgHint("m", HintMulSrc) + c := f.Arg("a") + n := f.Arg("z_len") + + // On amd64, we need a non-immediate for the AtUnrollEnd adds. + r0 := a.ZR() + if !r0.Valid() { + r0 = a.Reg() + a.Mov(a.Imm(0), r0) + } + + p := f.Pipe() + p.SetLabel("alt") + p.SetHint("x", HintMemOK) + p.SetHint("y", HintMemOK) + if a.Arch == ArchAMD64 { + p.SetMaxColumns(2) + } + + // See the large comment above for an explanation of the code being generated. + // This is optimization strategy (2). + var hi Reg + prev := c + p.Start(n, 1, 8) + p.AtUnrollStart(func() { + a.Comment("multiply and add") + a.ClearCarry(AddCarry | AltCarry) + a.ClearCarry(AddCarry) + hi = a.Reg() + }) + p.AtUnrollEnd(func() { + a.Add(r0, prev, c, UseCarry|SmashCarry) + a.Add(r0, c, c, UseCarry|SmashCarry|AltCarry) + prev = c + }) + p.Loop(func(in, out [][]Reg) { + for i, y := range in[1] { + x := in[0][i] + lo := y + if lo.IsMem() { + lo = a.Reg() + } + a.MulWide(m, y, lo, hi) + a.Add(prev, lo, lo, UseCarry|SetCarry) + a.Add(x, lo, lo, UseCarry|SetCarry|AltCarry) + out[0][i] = lo + prev, hi = hi, prev + } + p.StoreN(out) + }) + + f.StoreArg(c, "c") + a.Ret() +} + +func addMulVirtualCarry(f *Func, mulIndex int) { + a := f.Asm + m := f.Arg("m") + c := f.Arg("a") + n := f.Arg("z_len") + + // See the large comment above for an explanation of the code being generated. + // This is optimization strategy (3). + p := f.Pipe() + p.Start(n, 1, 4) + p.Loop(func(in, out [][]Reg) { + a.Comment("synthetic carry, one column at a time") + lo, hi := a.Reg(), a.Reg() + for i, x := range in[mulIndex] { + a.MulWide(m, x, lo, hi) + if mulIndex == 1 { + a.Add(in[0][i], lo, lo, SetCarry) + a.Add(a.Imm(0), hi, hi, UseCarry|SmashCarry) + } + a.Add(c, lo, x, SetCarry) + a.Add(a.Imm(0), hi, c, UseCarry|SmashCarry) + out[0][i] = x + } + p.StoreN(out) + }) + f.StoreArg(c, "c") + a.Ret() +} diff --git a/src/math/big/internal/asmgen/pipe.go b/src/math/big/internal/asmgen/pipe.go new file mode 100644 index 00000000000000..743e15f3f87d7c --- /dev/null +++ b/src/math/big/internal/asmgen/pipe.go @@ -0,0 +1,569 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +import ( + "fmt" + "math/bits" + "slices" +) + +// Note: Exported fields and methods are expected to be used +// by function generators (like the ones in add.go and so on). +// Unexported fields and methods should not be. + +// A Pipe manages the input and output data pipelines for a function's +// memory operations. +// +// The input is one or more equal-length slices of words, so collectively +// it can be viewed as a matrix, in which each slice is a row and each column +// is a set of corresponding words from the different slices. +// The output can be viewed the same way, although it is often just one row. +type Pipe struct { + f *Func // function being generated + label string // prefix for loop labels (default "loop") + backward bool // processing columns in reverse + started bool // Start has been called + loaded bool // LoadPtrs has been called + inPtr []RegPtr // input slice pointers + hints []Hint // for each inPtr, a register hint to use for its data + outPtr []RegPtr // output slice pointers + index Reg // index register, if in use + useIndexCounter bool // index counter requested + indexCounter int // index is also counter (386); 0 no, -1 negative counter, +1 positive counter + readOff int // read offset not yet added to index + writeOff int // write offset not yet added to index + factors []int // unrolling factors + counts []Reg // iterations for each factor + needWrite bool // need a write call during Loop1/LoopN + maxColumns int // maximum columns during unrolled loop + unrollStart func() // emit code at start of unrolled body + unrollEnd func() // emit code end of unrolled body +} + +// Pipe creates and returns a new pipe for use in the function f. +func (f *Func) Pipe() *Pipe { + a := f.Asm + p := &Pipe{ + f: f, + label: "loop", + maxColumns: 10000000, + } + if m := a.Arch.maxColumns; m != 0 { + p.maxColumns = m + } + return p +} + +// SetBackward sets the pipe to process the input and output columns in reverse order. +// This is needed for left shifts, which might otherwise overwrite data they will read later. +func (p *Pipe) SetBackward() { + if p.loaded { + p.f.Asm.Fatalf("SetBackward after Start/LoadPtrs") + } + p.backward = true +} + +// SetUseIndexCounter sets the pipe to use an index counter if possible, +// meaning the loop counter is also used as an index for accessing the slice data. +// This clever trick is slower on modern processors, but it is still necessary on 386. +// On non-386 systems, SetUseIndexCounter is a no-op. +func (p *Pipe) SetUseIndexCounter() { + if p.f.Asm.Arch.memIndex == nil { // need memIndex (only 386 provides it) + return + } + p.useIndexCounter = true +} + +// SetLabel sets the label prefix for the loops emitted by the pipe. +// The default prefix is "loop". +func (p *Pipe) SetLabel(label string) { + p.label = label +} + +// SetMaxColumns sets the maximum number of +// columns processed in a single loop body call. +func (p *Pipe) SetMaxColumns(m int) { + p.maxColumns = m +} + +// SetHint records that the inputs from the named vector +// should be allocated with the given register hint. +// +// If the hint indicates a single register on the target architecture, +// then SetHint calls SetMaxColumns(1), since the hinted register +// can only be used for one value at a time. +func (p *Pipe) SetHint(name string, hint Hint) { + if hint == HintMemOK && !p.f.Asm.Arch.memOK { + return + } + i := slices.Index(p.f.inputs, name) + if i < 0 { + p.f.Asm.Fatalf("unknown input name %s", name) + } + if p.f.Asm.hint(hint) != "" { + p.SetMaxColumns(1) + } + for len(p.hints) <= i { + p.hints = append(p.hints, HintNone) + } + p.hints[i] = hint +} + +// LoadPtrs loads the slice pointer arguments into registers, +// assuming that the slice length n has already been loaded +// into the register n. +// +// Start will call LoadPtrs if it has not been called already. +// LoadPtrs only needs to be called explicitly when code needs +// to use LoadN before Start, like when the shift.go generators +// read an initial word before the loop. +func (p *Pipe) LoadPtrs(n Reg) { + a := p.f.Asm + if p.loaded { + a.Fatalf("pointers already loaded") + } + + // Load the actual pointers. + p.loaded = true + for _, name := range p.f.inputs { + p.inPtr = append(p.inPtr, RegPtr(p.f.Arg(name+"_base"))) + } + for _, name := range p.f.outputs { + p.outPtr = append(p.outPtr, RegPtr(p.f.Arg(name+"_base"))) + } + + // Decide the memory access strategy for LoadN and StoreN. + switch { + case p.backward && p.useIndexCounter: + // Generator wants an index counter, meaning when the iteration counter + // is AX, we will access the slice with pointer BX using (BX)(AX*WordBytes). + // The loop is moving backward through the slice, but the counter + // is also moving backward, so not much to do. + a.Comment("run loop backward, using counter as positive index") + p.indexCounter = +1 + p.index = n + + case !p.backward && p.useIndexCounter: + // Generator wants an index counter, but the loop is moving forward. + // To make the counter move in the direction of data access, + // we negate the counter, counting up from -len(z) to -1. + // To make the index access the right words, we add len(z)*WordBytes + // to each of the pointers. + // See comment below about the garbage collector (non-)implications + // of pointing beyond the slice bounds. + a.Comment("use counter as negative index") + p.indexCounter = -1 + p.index = n + for _, ptr := range p.inPtr { + a.AddWords(n, ptr, ptr) + } + for _, ptr := range p.outPtr { + a.AddWords(n, ptr, ptr) + } + a.Neg(n, n) + + case p.backward: + // Generator wants to run the loop backward. + // We'll decrement the pointers before using them, + // so position them at the very end of the slices. + // If we had precise pointer information for assembly, + // these pointers would cause problems with the garbage collector, + // since they no longer point into the allocated slice, + // but the garbage collector ignores unexpected values in assembly stacks, + // and the actual slice pointers are still in the argument stack slots, + // so the slices won't be collected early. + // If we switched to the register ABI, we might have to rethink this. + // (The same thing happens by the end of forward loops, + // but it's less important since once the pointers go off the slice + // in a forward loop, the loop is over and the slice won't be accessed anymore.) + a.Comment("run loop backward") + for _, ptr := range p.inPtr { + a.AddWords(n, ptr, ptr) + } + for _, ptr := range p.outPtr { + a.AddWords(n, ptr, ptr) + } + + case !p.backward: + // Nothing to do! + } +} + +// LoadN returns the next n columns of input words as a slice of rows. +// Regs for inputs that have been marked using p.SetMemOK will be direct memory references. +// Regs for other inputs will be newly allocated registers and must be freed. +func (p *Pipe) LoadN(n int) [][]Reg { + a := p.f.Asm + regs := make([][]Reg, len(p.inPtr)) + for i, ptr := range p.inPtr { + regs[i] = make([]Reg, n) + switch { + case a.Arch.loadIncN != nil: + // Load from memory and advance pointers at the same time. + for j := range regs[i] { + regs[i][j] = p.f.Asm.Reg() + } + if p.backward { + a.Arch.loadDecN(a, ptr, regs[i]) + } else { + a.Arch.loadIncN(a, ptr, regs[i]) + } + + default: + // Load from memory using offsets. + // We'll advance the pointers or the index counter later. + for j := range n { + off := p.readOff + j + if p.backward { + off = -(off + 1) + } + var mem Reg + if p.indexCounter != 0 { + mem = a.Arch.memIndex(a, off*a.Arch.WordBytes, p.index, ptr) + } else { + mem = ptr.mem(off * a.Arch.WordBytes) + } + h := HintNone + if i < len(p.hints) { + h = p.hints[i] + } + if h == HintMemOK { + regs[i][j] = mem + } else { + r := p.f.Asm.RegHint(h) + a.Mov(mem, r) + regs[i][j] = r + } + } + } + } + p.readOff += n + return regs +} + +// StoreN writes regs (a slice of rows) to the next n columns of output, where n = len(regs[0]). +func (p *Pipe) StoreN(regs [][]Reg) { + p.needWrite = false + a := p.f.Asm + if len(regs) != len(p.outPtr) { + p.f.Asm.Fatalf("wrong number of output rows") + } + n := len(regs[0]) + for i, ptr := range p.outPtr { + switch { + case a.Arch.storeIncN != nil: + // Store to memory and advance pointers at the same time. + if p.backward { + a.Arch.storeDecN(a, ptr, regs[i]) + } else { + a.Arch.storeIncN(a, ptr, regs[i]) + } + + default: + // Store to memory using offsets. + // We'll advance the pointers or the index counter later. + for j, r := range regs[i] { + off := p.writeOff + j + if p.backward { + off = -(off + 1) + } + var mem Reg + if p.indexCounter != 0 { + mem = a.Arch.memIndex(a, off*a.Arch.WordBytes, p.index, ptr) + } else { + mem = ptr.mem(off * a.Arch.WordBytes) + } + a.Mov(r, mem) + } + } + } + p.writeOff += n +} + +// advancePtrs advances the pointers by step +// or handles bookkeeping for an imminent index advance by step +// that the caller will do. +func (p *Pipe) advancePtrs(step int) { + a := p.f.Asm + switch { + case a.Arch.loadIncN != nil: + // nothing to do + + default: + // Adjust read/write offsets for pointer advance (or imminent index advance). + p.readOff -= step + p.writeOff -= step + + if p.indexCounter == 0 { + // Advance pointers. + if p.backward { + step = -step + } + for _, ptr := range p.inPtr { + a.Add(a.Imm(step*a.Arch.WordBytes), Reg(ptr), Reg(ptr), KeepCarry) + } + for _, ptr := range p.outPtr { + a.Add(a.Imm(step*a.Arch.WordBytes), Reg(ptr), Reg(ptr), KeepCarry) + } + } + } +} + +// DropInput deletes the named input from the pipe, +// usually because it has been exhausted. +// (This is not used yet but will be used in a future generator.) +func (p *Pipe) DropInput(name string) { + i := slices.Index(p.f.inputs, name) + if i < 0 { + p.f.Asm.Fatalf("unknown input %s", name) + } + ptr := p.inPtr[i] + p.f.Asm.Free(Reg(ptr)) + p.inPtr = slices.Delete(p.inPtr, i, i+1) + p.f.inputs = slices.Delete(p.f.inputs, i, i+1) + if len(p.hints) > i { + p.hints = slices.Delete(p.hints, i, i+1) + } +} + +// Start prepares to loop over n columns. +// The factors give a sequence of unrolling factors to use, +// which must be either strictly increasing or strictly decreasing +// and must include 1. +// For example, 4, 1 means to process 4 elements at a time +// and then 1 at a time for the final 0-3; specifying 1,4 instead +// handles 0-3 elements first and then 4 at a time. +// Similarly, 32, 4, 1 means to process 32 at a time, +// then 4 at a time, then 1 at a time. +// +// One benefit of using 1, 4 instead of 4, 1 is that the body +// processing 4 at a time needs more registers, and if it is +// the final body, the register holding the fragment count (0-3) +// has been freed and is available for use. +// +// Start may modify the carry flag. +// +// Start must be followed by a call to Loop1 or LoopN, +// but it is permitted to emit other instructions first, +// for example to set an initial carry flag. +func (p *Pipe) Start(n Reg, factors ...int) { + a := p.f.Asm + if p.started { + a.Fatalf("loop already started") + } + if p.useIndexCounter && len(factors) > 1 { + a.Fatalf("cannot call SetUseIndexCounter and then use Start with factors != [1]; have factors = %v", factors) + } + p.started = true + if !p.loaded { + if len(factors) == 1 { + p.SetUseIndexCounter() + } + p.LoadPtrs(n) + } + + // If there were calls to LoadN between LoadPtrs and Start, + // adjust the loop not to scan those columns, assuming that + // either the code already called an equivalent StoreN or else + // that it will do so after the loop. + if off := p.readOff; off != 0 { + if p.indexCounter < 0 { + // Index is negated, so add off instead of subtracting. + a.Add(a.Imm(off), n, n, SmashCarry) + } else { + a.Sub(a.Imm(off), n, n, SmashCarry) + } + if p.indexCounter != 0 { + // n is also the index we are using, so adjust readOff and writeOff + // to continue to point at the same positions as before we changed n. + p.readOff -= off + p.writeOff -= off + } + } + + p.Restart(n, factors...) +} + +// Restart prepares to loop over an additional n columns, +// beyond a previous loop run by p.Start/p.Loop. +func (p *Pipe) Restart(n Reg, factors ...int) { + a := p.f.Asm + if !p.started { + a.Fatalf("pipe not started") + } + p.factors = factors + p.counts = make([]Reg, len(factors)) + if len(factors) == 0 { + factors = []int{1} + } + + // Compute the loop lengths for each unrolled section into separate registers. + // We compute them all ahead of time in case the computation would smash + // a carry flag that the loop bodies need preserved. + if len(factors) > 1 { + a.Comment("compute unrolled loop lengths") + } + switch { + default: + a.Fatalf("invalid factors %v", factors) + + case factors[0] == 1: + // increasing loop factors + div := 1 + for i, f := range factors[1:] { + if f <= factors[i] { + a.Fatalf("non-increasing factors %v", factors) + } + if f&(f-1) != 0 { + a.Fatalf("non-power-of-two factors %v", factors) + } + t := p.f.Asm.Reg() + f /= div + a.And(a.Imm(f-1), n, t) + a.Rsh(a.Imm(bits.TrailingZeros(uint(f))), n, n) + div *= f + p.counts[i] = t + } + p.counts[len(p.counts)-1] = n + + case factors[len(factors)-1] == 1: + // decreasing loop factors + for i, f := range factors[:len(factors)-1] { + if f <= factors[i+1] { + a.Fatalf("non-decreasing factors %v", factors) + } + if f&(f-1) != 0 { + a.Fatalf("non-power-of-two factors %v", factors) + } + t := p.f.Asm.Reg() + a.Rsh(a.Imm(bits.TrailingZeros(uint(f))), n, t) + a.And(a.Imm(f-1), n, n) + p.counts[i] = t + } + p.counts[len(p.counts)-1] = n + } +} + +// Done frees all the registers allocated by the pipe. +func (p *Pipe) Done() { + for _, ptr := range p.inPtr { + p.f.Asm.Free(Reg(ptr)) + } + p.inPtr = nil + for _, ptr := range p.outPtr { + p.f.Asm.Free(Reg(ptr)) + } + p.outPtr = nil + p.index = Reg{} +} + +// Loop emits code for the loop, calling block repeatedly to emit code that +// handles a block of N input columns (for arbitrary N = len(in[0]) chosen by p). +// block must call p.StoreN(out) to write N output columns. +// The out slice is a pre-allocated matrix of uninitialized Reg values. +// block is expected to set each entry to the Reg that should be written +// before calling p.StoreN(out). +// +// For example, if the loop is to be unrolled 4x in blocks of 2 columns each, +// the sequence of calls to emit the unrolled loop body is: +// +// start() // set by pAtUnrollStart +// ... reads for 2 columns ... +// block() +// ... writes for 2 columns ... +// ... reads for 2 columns ... +// block() +// ... writes for 2 columns ... +// end() // set by p.AtUnrollEnd +// +// Any registers allocated during block are freed automatically when block returns. +func (p *Pipe) Loop(block func(in, out [][]Reg)) { + if p.factors == nil { + p.f.Asm.Fatalf("Pipe.Start not called") + } + for i, factor := range p.factors { + n := p.counts[i] + p.unroll(n, factor, block) + if i < len(p.factors)-1 { + p.f.Asm.Free(n) + } + } + p.factors = nil +} + +// AtUnrollStart sets a function to call at the start of an unrolled sequence. +// See [Pipe.Loop] for details. +func (p *Pipe) AtUnrollStart(start func()) { + p.unrollStart = start +} + +// AtUnrollEnd sets a function to call at the end of an unrolled sequence. +// See [Pipe.Loop] for details. +func (p *Pipe) AtUnrollEnd(end func()) { + p.unrollEnd = end +} + +// unroll emits a single unrolled loop for the given factor, iterating n times. +func (p *Pipe) unroll(n Reg, factor int, block func(in, out [][]Reg)) { + a := p.f.Asm + label := fmt.Sprintf("%s%d", p.label, factor) + + // Top of loop control flow. + a.Label(label) + if a.Arch.loopTop != "" { + a.Printf("\t"+a.Arch.loopTop+"\n", n, label+"done") + } else { + a.JmpZero(n, label+"done") + } + a.Label(label + "cont") + + // Unrolled loop body. + if factor < p.maxColumns { + a.Comment("unroll %dX", factor) + } else { + a.Comment("unroll %dX in batches of %d", factor, p.maxColumns) + } + if p.unrollStart != nil { + p.unrollStart() + } + for done := 0; done < factor; { + batch := min(factor-done, p.maxColumns) + regs := a.RegsUsed() + out := make([][]Reg, len(p.outPtr)) + for i := range out { + out[i] = make([]Reg, batch) + } + in := p.LoadN(batch) + p.needWrite = true + block(in, out) + if p.needWrite && len(p.outPtr) > 0 { + a.Fatalf("missing p.Write1 or p.StoreN") + } + a.SetRegsUsed(regs) // free anything block allocated + done += batch + } + if p.unrollEnd != nil { + p.unrollEnd() + } + p.advancePtrs(factor) + + // Bottom of loop control flow. + switch { + case p.indexCounter >= 0 && a.Arch.loopBottom != "": + a.Printf("\t"+a.Arch.loopBottom+"\n", n, label+"cont") + + case p.indexCounter >= 0: + a.Sub(a.Imm(1), n, n, KeepCarry) + a.JmpNonZero(n, label+"cont") + + case p.indexCounter < 0 && a.Arch.loopBottomNeg != "": + a.Printf("\t"+a.Arch.loopBottomNeg+"\n", n, label+"cont") + + case p.indexCounter < 0: + a.Add(a.Imm(1), n, n, KeepCarry) + } + a.Label(label + "done") +} diff --git a/src/math/big/internal/asmgen/ppc64.go b/src/math/big/internal/asmgen/ppc64.go new file mode 100644 index 00000000000000..e2cf7229a3f571 --- /dev/null +++ b/src/math/big/internal/asmgen/ppc64.go @@ -0,0 +1,64 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +var ArchPPC64x = &Arch{ + Name: "ppc64x", + Build: "ppc64 || ppc64le", + WordBits: 64, + WordBytes: 8, + CarrySafeLoop: true, + + // Note: The old, hand-written ppc64x assembly used MOVDU + // to avoid explicit pointer updates in a few routines, but the new + // generated code runs just as fast, so we haven't bothered to try + // to add that back. (It's not trivial; you'd have to keep the pointers + // shifted one word in order to make the semantics work.) + // + // The old assembly also used some complex vector instructions + // to implement lshVU and rshVU, but the generated code that uses + // ordinary integer instructions is much faster than the vector code was, + // at least on the power10 gomote. + + regs: []string{ + // R0 is 0 by convention. + // R1 is SP. + // R2 is TOC. + // R30 is g. + // R31 is the assembler/linker temporary (which we use too). + "R3", "R4", "R5", "R6", "R7", "R8", "R9", + "R10", "R11", "R12" /*R13 is TLS*/, "R14", "R15", "R16", "R17", "R18", "R19", + "R20", "R21", "R22", "R23", "R24", "R25", "R26", "R27", "R28", "R29", + }, + reg0: "R0", + regTmp: "R31", + + // Note: Could write an addF and subF to use ADDZE and SUBZE, + // but we have R0 so it doesn't seem to matter much. + + mov: "MOVD", + add: "ADD", + adds: "ADDC", + adcs: "ADDE", + sub: "SUB", + subs: "SUBC", + sbcs: "SUBE", + mul: "MULLD", + mulhi: "MULHDU", + lsh: "SLD", + rsh: "SRD", + and: "ANDCC", // regular AND does not accept immediates + or: "OR", + xor: "XOR", + + jmpZero: "CMP %[1]s, $0; BEQ %[2]s", + jmpNonZero: "CMP %s, $0; BNE %s", + + // Note: Using CTR means that we could free the count register + // during the loop body, but the portable logic doesn't know that, + // and we're not hurting for registers. + loopTop: "CMP %[1]s, $0; BEQ %[2]s; MOVD %[1]s, CTR", + loopBottom: "BDNZ %[2]s", +} diff --git a/src/math/big/internal/asmgen/riscv64.go b/src/math/big/internal/asmgen/riscv64.go new file mode 100644 index 00000000000000..8995c4c1592ca7 --- /dev/null +++ b/src/math/big/internal/asmgen/riscv64.go @@ -0,0 +1,47 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +var ArchRISCV64 = &Arch{ + Name: "riscv64", + WordBits: 64, + WordBytes: 8, + CarrySafeLoop: true, + + regs: []string{ + // X0 is zero. + // X1 is LR. + // X2 is SP. + // X3 is SB. + // X4 is TP. + // X27 is g. + // X28 and X29 are our virtual carry flags. + // X31 is the assembler/linker temporary (which we use too). + "X5", "X6", "X7", "X8", "X9", + "X10", "X11", "X12", "X13", "X14", "X15", "X16", "X17", "X18", "X19", + "X20", "X21", "X22", "X23", "X24", "X25", "X26", + "X30", + }, + + reg0: "X0", + regCarry: "X28", + regAltCarry: "X29", + regTmp: "X31", + + mov: "MOV", + add: "ADD", + sub: "SUB", + mul: "MUL", + mulhi: "MULHU", + lsh: "SLL", + rsh: "SRL", + and: "AND", + or: "OR", + xor: "XOR", + sltu: "SLTU", + + jmpZero: "BEQZ %s, %s", + jmpNonZero: "BNEZ %s, %s", +} diff --git a/src/math/big/internal/asmgen/s390x.go b/src/math/big/internal/asmgen/s390x.go new file mode 100644 index 00000000000000..71c9b165c62537 --- /dev/null +++ b/src/math/big/internal/asmgen/s390x.go @@ -0,0 +1,100 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +var ArchS390X = &Arch{ + Name: "s390x", + WordBits: 64, + WordBytes: 8, + CarrySafeLoop: true, + + regs: []string{ + // R0 is 0 by convention in this code (see setup). + // R10 is the assembler/linker temporary. + // R11 is a second assembler/linker temporary, for wide multiply. + // We allow allocating R10 and R11 so that we can use them as + // direct multiplication targets while tracking whether they're in use. + // R13 is g. + // R14 is LR. + // R15 is SP. + "R1", "R2", "R3", "R4", "R5", "R6", "R7", "R8", "R9", + "R10", "R11", "R12", + }, + reg0: "R0", + regTmp: "R10", + setup: s390xSetup, + maxColumns: 2, + op3: s390xOp3, + hint: s390xHint, + + // Instruction reference: chapter 7 of + // https://www.ibm.com/docs/en/SSQ2R2_15.0.0/com.ibm.tpf.toolkit.hlasm.doc/dz9zr006.pdf + + mov: "MOVD", + adds: "ADDC", // ADD is an alias for ADDC, sets carry + adcs: "ADDE", + subs: "SUBC", // SUB is an alias for SUBC, sets carry + sbcs: "SUBE", + mulWideF: s390MulWide, + lsh: "SLD", + rsh: "SRD", + and: "AND", + or: "OR", + xor: "XOR", + neg: "NEG", + lea: "LAY", // LAY because LA only accepts positive offsets + + jmpZero: "CMPBEQ %s, $0, %s", + jmpNonZero: "CMPBNE %s, $0, %s", +} + +func s390xSetup(f *Func) { + a := f.Asm + if f.Name == "addVV" || f.Name == "subVV" { + // S390x, unlike every other system, has vector instructions + // that can propagate carry bits during parallel adds (VACC). + // Instead of trying to generate that for this one system, + // jump to the hand-written code in arithvec_s390x.s. + a.Printf("\tMOVB ·hasVX(SB), R1\n") + a.Printf("\tCMPBEQ R1, $0, novec\n") + a.Printf("\tJMP ·%svec(SB)\n", f.Name) + a.Printf("novec:\n") + } + a.Printf("\tMOVD $0, R0\n") +} + +func s390xOp3(name string) bool { + if name == "AND" { // AND with immediate only takes imm, reg; not imm, reg, reg. + return false + } + return true +} + +func s390xHint(_ *Asm, h Hint) string { + switch h { + case HintMulSrc: + return "R11" + case HintMulHi: + return "R10" + } + return "" +} + +func s390MulWide(a *Asm, src1, src2, dstlo, dsthi Reg) { + if src1.name != "R11" && src2.name != "R11" { + a.Fatalf("mulWide src1 or src2 must be R11") + } + if dstlo.name != "R11" { + a.Fatalf("mulWide dstlo must be R11") + } + if dsthi.name != "R10" { + a.Fatalf("mulWide dsthi must be R10") + } + src := src1 + if src.name == "R11" { + src = src2 + } + a.Printf("\tMLGR %s, R10\n", src) +} diff --git a/src/math/big/internal/asmgen/shift.go b/src/math/big/internal/asmgen/shift.go new file mode 100644 index 00000000000000..6ece599a4b54ef --- /dev/null +++ b/src/math/big/internal/asmgen/shift.go @@ -0,0 +1,135 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package asmgen + +// shiftVU generates lshVU and rshVU, which do +// z, c = x << s and z, c = x >> s, for 0 < s < _W. +func shiftVU(a *Asm, name string) { + // Because these routines can be called for z.Lsh(z, N) and z.Rsh(z, N), + // the input and output slices may be aliased at different offsets. + // For example (on 64-bit systems), during z.Lsh(z, 65), &z[0] == &x[1], + // and during z.Rsh(z, 65), &z[1] == &x[0]. + // For left shift, we must process the slices from len(z)-1 down to 0, + // so that we don't overwrite a word before we need to read it. + // For right shift, we must process the slices from 0 up to len(z)-1. + // The different traversals at least make the two cases more consistent, + // since we're always delaying the output by one word compared + // to the input. + + f := a.Func("func " + name + "(z, x []Word, s uint) (c Word)") + + // Check for no input early, since we need to start by reading 1 word. + n := f.Arg("z_len") + a.JmpZero(n, "ret0") + + // Start loop by reading first input word. + s := f.ArgHint("s", HintShiftCount) + p := f.Pipe() + if name == "lshVU" { + p.SetBackward() + } + unroll := []int{1, 4} + if a.Arch == Arch386 { + unroll = []int{1} // too few registers for more + p.SetUseIndexCounter() + } + p.LoadPtrs(n) + a.Comment("shift first word into carry") + prev := p.LoadN(1)[0][0] + + // Decide how to shift. On systems with a wide shift (x86), use that. + // Otherwise, we need shift by s and negative (reverse) shift by 64-s or 32-s. + shift := a.Lsh + shiftWide := a.LshWide + negShift := a.Rsh + negShiftReg := a.RshReg + if name == "rshVU" { + shift = a.Rsh + shiftWide = a.RshWide + negShift = a.Lsh + negShiftReg = a.LshReg + } + if a.Arch.HasShiftWide() { + // Use wide shift to avoid needing negative shifts. + // The invariant is that prev holds the previous word (not shifted at all), + // to be used as input into the wide shift. + // After the loop finishes, prev holds the final output word to be written. + c := a.Reg() + shiftWide(s, prev, a.Imm(0), c) + f.StoreArg(c, "c") + a.Free(c) + a.Comment("shift remaining words") + p.Start(n, unroll...) + p.Loop(func(in [][]Reg, out [][]Reg) { + // We reuse the input registers as output, delayed one cycle; prev is the first output. + // After writing the outputs to memory, we can copy the final x value into prev + // for the next iteration. + old := prev + for i, x := range in[0] { + shiftWide(s, x, old, old) + out[0][i] = old + old = x + } + p.StoreN(out) + a.Mov(old, prev) + }) + a.Comment("store final shifted bits") + shift(s, prev, prev) + } else { + // Construct values from x << s and x >> (64-s). + // After the first word has been processed, the invariant is that + // prev holds x << s, to be used as the high bits of the next output word, + // once we find the low bits after reading the next input word. + // After the loop finishes, prev holds the final output word to be written. + sNeg := a.Reg() + a.Mov(a.Imm(a.Arch.WordBits), sNeg) + a.Sub(s, sNeg, sNeg, SmashCarry) + c := a.Reg() + negShift(sNeg, prev, c) + shift(s, prev, prev) + f.StoreArg(c, "c") + a.Free(c) + a.Comment("shift remaining words") + p.Start(n, unroll...) + p.Loop(func(in, out [][]Reg) { + if a.HasRegShift() { + // ARM (32-bit) allows shifts in most arithmetic expressions, + // including OR, letting us combine the negShift and a.Or. + // The simplest way to manage the registers is to do StoreN for + // one output at a time, and since we don't use multi-register + // stores on ARM, that doesn't hurt us. + out[0] = out[0][:1] + for _, x := range in[0] { + a.Or(negShiftReg(sNeg, x), prev, prev) + out[0][0] = prev + p.StoreN(out) + shift(s, x, prev) + } + return + } + // We reuse the input registers as output, delayed one cycle; z0 is the first output. + z0 := a.Reg() + z := z0 + for i, x := range in[0] { + negShift(sNeg, x, z) + a.Or(prev, z, z) + shift(s, x, prev) + out[0][i] = z + z = x + } + p.StoreN(out) + }) + a.Comment("store final shifted bits") + } + p.StoreN([][]Reg{{prev}}) + p.Done() + a.Free(s) + a.Ret() + + // Return 0, used from above. + a.Label("ret0") + f.StoreArg(a.Imm(0), "c") + a.Ret() +} diff --git a/src/math/big/intmarsh.go b/src/math/big/intmarsh.go index 56eeefb884d8c2..858ca0faba5890 100644 --- a/src/math/big/intmarsh.go +++ b/src/math/big/intmarsh.go @@ -45,12 +45,14 @@ func (z *Int) GobDecode(buf []byte) error { return nil } +// AppendText implements the [encoding.TextAppender] interface. +func (x *Int) AppendText(b []byte) (text []byte, err error) { + return x.Append(b, 10), nil +} + // MarshalText implements the [encoding.TextMarshaler] interface. func (x *Int) MarshalText() (text []byte, err error) { - if x == nil { - return []byte(""), nil - } - return x.abs.itoa(x.neg, 10), nil + return x.AppendText(nil) } // UnmarshalText implements the [encoding.TextUnmarshaler] interface. diff --git a/src/math/big/intmarsh_test.go b/src/math/big/intmarsh_test.go index 8e7d29f9ddc719..681f3dd9468e4a 100644 --- a/src/math/big/intmarsh_test.go +++ b/src/math/big/intmarsh_test.go @@ -132,3 +132,36 @@ func TestIntXMLEncoding(t *testing.T) { } } } + +func TestIntAppendText(t *testing.T) { + for _, test := range encodingTests { + for _, sign := range []string{"", "+", "-"} { + x := sign + test + var tx Int + tx.SetString(x, 10) + buf := make([]byte, 4, 32) + b, err := tx.AppendText(buf) + if err != nil { + t.Errorf("marshaling of %s failed: %s", &tx, err) + continue + } + var rx Int + if err := rx.UnmarshalText(b[4:]); err != nil { + t.Errorf("unmarshaling of %s failed: %s", &tx, err) + continue + } + if rx.Cmp(&tx) != 0 { + t.Errorf("AppendText of %s failed: got %s want %s", &tx, &rx, &tx) + } + } + } +} + +func TestIntAppendTextNil(t *testing.T) { + var x *Int + buf := make([]byte, 4, 16) + data, _ := x.AppendText(buf) + if string(data[4:]) != "" { + t.Errorf("got %q, want ", data[4:]) + } +} diff --git a/src/math/big/nat.go b/src/math/big/nat.go index 23b2a0b8ddf034..43e36d309389f5 100644 --- a/src/math/big/nat.go +++ b/src/math/big/nat.go @@ -17,6 +17,7 @@ import ( "internal/byteorder" "math/bits" "math/rand" + "slices" "sync" ) @@ -110,7 +111,7 @@ func (z nat) add(x, y nat) nat { // m > 0 z = z.make(m + 1) - c := addVV(z[0:n], x, y) + c := addVV(z[:n], x[:n], y[:n]) if m > n { c = addVW(z[n:m], x[n:], c) } @@ -136,7 +137,7 @@ func (z nat) sub(x, y nat) nat { // m > 0 z = z.make(m) - c := subVV(z[0:n], x, y) + c := subVV(z[:n], x[:n], y[:n]) if m > n { c = subVW(z[n:], x[n:], c) } @@ -174,30 +175,6 @@ func (x nat) cmp(y nat) (r int) { return } -func (z nat) mulAddWW(x nat, y, r Word) nat { - m := len(x) - if m == 0 || y == 0 { - return z.setWord(r) // result is r - } - // m > 0 - - z = z.make(m + 1) - z[m] = mulAddVWW(z[0:m], x, y, r) - - return z.norm() -} - -// basicMul multiplies x and y and leaves the result in z. -// The (non-normalized) result is placed in z[0 : len(x) + len(y)]. -func basicMul(z, x, y nat) { - clear(z[0 : len(x)+len(y)]) // initialize z - for i, d := range y { - if d != 0 { - z[len(x)+i] = addMulVVW(z[i:i+len(x)], x, d) - } - } -} - // montgomery computes z mod m = x*y*2**(-n*_W) mod m, // assuming k = -1/m mod 2**_W. // z is used for storing the result which is returned; @@ -220,9 +197,9 @@ func (z nat) montgomery(x, y, m nat, k Word, n int) nat { var c Word for i := 0; i < n; i++ { d := y[i] - c2 := addMulVVW(z[i:n+i], x, d) + c2 := addMulVVWW(z[i:n+i], z[i:n+i], x, d, 0) t := z[i] * k - c3 := addMulVVW(z[i:n+i], m, t) + c3 := addMulVVWW(z[i:n+i], z[i:n+i], m, t, 0) cx := c + c2 cy := cx + c3 z[n+i] = cy @@ -240,125 +217,6 @@ func (z nat) montgomery(x, y, m nat, k Word, n int) nat { return z[:n] } -// Fast version of z[0:n+n>>1].add(z[0:n+n>>1], x[0:n]) w/o bounds checks. -// Factored out for readability - do not use outside karatsuba. -func karatsubaAdd(z, x nat, n int) { - if c := addVV(z[0:n], z, x); c != 0 { - addVW(z[n:n+n>>1], z[n:], c) - } -} - -// Like karatsubaAdd, but does subtract. -func karatsubaSub(z, x nat, n int) { - if c := subVV(z[0:n], z, x); c != 0 { - subVW(z[n:n+n>>1], z[n:], c) - } -} - -// Operands that are shorter than karatsubaThreshold are multiplied using -// "grade school" multiplication; for longer operands the Karatsuba algorithm -// is used. -var karatsubaThreshold = 40 // computed by calibrate_test.go - -// karatsuba multiplies x and y and leaves the result in z. -// Both x and y must have the same length n and n must be a -// power of 2. The result vector z must have len(z) >= 6*n. -// The (non-normalized) result is placed in z[0 : 2*n]. -func karatsuba(z, x, y nat) { - n := len(y) - - // Switch to basic multiplication if numbers are odd or small. - // (n is always even if karatsubaThreshold is even, but be - // conservative) - if n&1 != 0 || n < karatsubaThreshold || n < 2 { - basicMul(z, x, y) - return - } - // n&1 == 0 && n >= karatsubaThreshold && n >= 2 - - // Karatsuba multiplication is based on the observation that - // for two numbers x and y with: - // - // x = x1*b + x0 - // y = y1*b + y0 - // - // the product x*y can be obtained with 3 products z2, z1, z0 - // instead of 4: - // - // x*y = x1*y1*b*b + (x1*y0 + x0*y1)*b + x0*y0 - // = z2*b*b + z1*b + z0 - // - // with: - // - // xd = x1 - x0 - // yd = y0 - y1 - // - // z1 = xd*yd + z2 + z0 - // = (x1-x0)*(y0 - y1) + z2 + z0 - // = x1*y0 - x1*y1 - x0*y0 + x0*y1 + z2 + z0 - // = x1*y0 - z2 - z0 + x0*y1 + z2 + z0 - // = x1*y0 + x0*y1 - - // split x, y into "digits" - n2 := n >> 1 // n2 >= 1 - x1, x0 := x[n2:], x[0:n2] // x = x1*b + y0 - y1, y0 := y[n2:], y[0:n2] // y = y1*b + y0 - - // z is used for the result and temporary storage: - // - // 6*n 5*n 4*n 3*n 2*n 1*n 0*n - // z = [z2 copy|z0 copy| xd*yd | yd:xd | x1*y1 | x0*y0 ] - // - // For each recursive call of karatsuba, an unused slice of - // z is passed in that has (at least) half the length of the - // caller's z. - - // compute z0 and z2 with the result "in place" in z - karatsuba(z, x0, y0) // z0 = x0*y0 - karatsuba(z[n:], x1, y1) // z2 = x1*y1 - - // compute xd (or the negative value if underflow occurs) - s := 1 // sign of product xd*yd - xd := z[2*n : 2*n+n2] - if subVV(xd, x1, x0) != 0 { // x1-x0 - s = -s - subVV(xd, x0, x1) // x0-x1 - } - - // compute yd (or the negative value if underflow occurs) - yd := z[2*n+n2 : 3*n] - if subVV(yd, y0, y1) != 0 { // y0-y1 - s = -s - subVV(yd, y1, y0) // y1-y0 - } - - // p = (x1-x0)*(y0-y1) == x1*y0 - x1*y1 - x0*y0 + x0*y1 for s > 0 - // p = (x0-x1)*(y0-y1) == x0*y0 - x0*y1 - x1*y0 + x1*y1 for s < 0 - p := z[n*3:] - karatsuba(p, xd, yd) - - // save original z2:z0 - // (ok to use upper half of z since we're done recurring) - r := z[n*4:] - copy(r, z[:n*2]) - - // add up all partial products - // - // 2*n n 0 - // z = [ z2 | z0 ] - // + [ z0 ] - // + [ z2 ] - // + [ p ] - // - karatsubaAdd(z[n2:], r, n) - karatsubaAdd(z[n2:], r[n:], n) - if s > 0 { - karatsubaAdd(z[n2:], p, n) - } else { - karatsubaSub(z[n2:], p, n) - } -} - // alias reports whether x and y share the same base array. // // Note: alias assumes that the capacity of underlying arrays @@ -369,282 +227,106 @@ func alias(x, y nat) bool { return cap(x) > 0 && cap(y) > 0 && &x[0:cap(x)][cap(x)-1] == &y[0:cap(y)][cap(y)-1] } -// addAt implements z += x<<(_W*i); z must be long enough. +// addTo implements z += x; z must be long enough. // (we don't use nat.add because we need z to stay the same // slice, and we don't need to normalize z after each addition) -func addAt(z, x nat, i int) { +func addTo(z, x nat) { if n := len(x); n > 0 { - if c := addVV(z[i:i+n], z[i:], x); c != 0 { - j := i + n - if j < len(z) { - addVW(z[j:], z[j:], c) + if c := addVV(z[:n], z[:n], x[:n]); c != 0 { + if n < len(z) { + addVW(z[n:], z[n:], c) } } } } -// karatsubaLen computes an approximation to the maximum k <= n such that -// k = p<= 0. Thus, the -// result is the largest number that can be divided repeatedly by 2 before -// becoming about the value of threshold. -func karatsubaLen(n, threshold int) int { - i := uint(0) - for n > threshold { - n >>= 1 - i++ - } - return n << i -} - -func (z nat) mul(x, y nat) nat { - m := len(x) - n := len(y) - +// mulRange computes the product of all the unsigned integers in the +// range [a, b] inclusively. If a > b (empty range), the result is 1. +// The caller may pass stk == nil to request that mulRange obtain and release one itself. +func (z nat) mulRange(stk *stack, a, b uint64) nat { switch { - case m < n: - return z.mul(y, x) - case m == 0 || n == 0: - return z[:0] - case n == 1: - return z.mulAddWW(x, y[0], 0) - } - // m >= n > 1 - - // determine if z can be reused - if alias(z, x) || alias(z, y) { - z = nil // z is an alias for x or y - cannot reuse - } - - // use basic multiplication if the numbers are small - if n < karatsubaThreshold { - z = z.make(m + n) - basicMul(z, x, y) - return z.norm() + case a == 0: + // cut long ranges short (optimization) + return z.setUint64(0) + case a > b: + return z.setUint64(1) + case a == b: + return z.setUint64(a) + case a+1 == b: + return z.mul(stk, nat(nil).setUint64(a), nat(nil).setUint64(b)) } - // m >= n && n >= karatsubaThreshold && n >= 2 - - // determine Karatsuba length k such that - // - // x = xh*b + x0 (0 <= x0 < b) - // y = yh*b + y0 (0 <= y0 < b) - // b = 1<<(_W*k) ("base" of digits xi, yi) - // - k := karatsubaLen(n, karatsubaThreshold) - // k <= n - - // multiply x0 and y0 via Karatsuba - x0 := x[0:k] // x0 is not normalized - y0 := y[0:k] // y0 is not normalized - z = z.make(max(6*k, m+n)) // enough space for karatsuba of x0*y0 and full result of x*y - karatsuba(z, x0, y0) - z = z[0 : m+n] // z has final length but may be incomplete - clear(z[2*k:]) // upper portion of z is garbage (and 2*k <= m+n since k <= n <= m) - - // If xh != 0 or yh != 0, add the missing terms to z. For - // - // xh = xi*b^i + ... + x2*b^2 + x1*b (0 <= xi < b) - // yh = y1*b (0 <= y1 < b) - // - // the missing terms are - // - // x0*y1*b and xi*y0*b^i, xi*y1*b^(i+1) for i > 0 - // - // since all the yi for i > 1 are 0 by choice of k: If any of them - // were > 0, then yh >= b^2 and thus y >= b^2. Then k' = k*2 would - // be a larger valid threshold contradicting the assumption about k. - // - if k < n || m != n { - tp := getNat(3 * k) - t := *tp - - // add x0*y1*b - x0 := x0.norm() - y1 := y[k:] // y1 is normalized because y is - t = t.mul(x0, y1) // update t so we don't lose t's underlying array - addAt(z, t, k) - - // add xi*y0< k { - xi = xi[:k] - } - xi = xi.norm() - t = t.mul(xi, y0) - addAt(z, t, i) - t = t.mul(xi, y1) - addAt(z, t, i+k) - } - putNat(tp) + if stk == nil { + stk = getStack() + defer stk.free() } - return z.norm() + m := a + (b-a)/2 // avoid overflow + return z.mul(stk, nat(nil).mulRange(stk, a, m), nat(nil).mulRange(stk, m+1, b)) } -// basicSqr sets z = x*x and is asymptotically faster than basicMul -// by about a factor of 2, but slower for small arguments due to overhead. -// Requirements: len(x) > 0, len(z) == 2*len(x) -// The (non-normalized) result is placed in z. -func basicSqr(z, x nat) { - n := len(x) - tp := getNat(2 * n) - t := *tp // temporary variable to hold the products - clear(t) - z[1], z[0] = mulWW(x[0], x[0]) // the initial square - for i := 1; i < n; i++ { - d := x[i] - // z collects the squares x[i] * x[i] - z[2*i+1], z[2*i] = mulWW(d, d) - // t collects the products x[i] * x[j] where j < i - t[2*i] = addMulVVW(t[i:2*i], x[0:i], d) - } - t[2*n-1] = shlVU(t[1:2*n-1], t[1:2*n-1], 1) // double the j < i products - addVV(z, z, t) // combine the result - putNat(tp) +// A stack provides temporary storage for complex calculations +// such as multiplication and division. +// The stack is a simple slice of words, extended as needed +// to hold all the temporary storage for a calculation. +// In general, if a function takes a *stack, it expects a non-nil *stack. +// However, certain functions may allow passing a nil *stack instead, +// so that they can handle trivial stack-free cases without forcing the +// caller to obtain and free a stack that will be unused. These functions +// document that they accept a nil *stack in their doc comments. +type stack struct { + w []Word } -// karatsubaSqr squares x and leaves the result in z. -// len(x) must be a power of 2 and len(z) >= 6*len(x). -// The (non-normalized) result is placed in z[0 : 2*len(x)]. -// -// The algorithm and the layout of z are the same as for karatsuba. -func karatsubaSqr(z, x nat) { - n := len(x) +var stackPool sync.Pool - if n&1 != 0 || n < karatsubaSqrThreshold || n < 2 { - basicSqr(z[:2*n], x) - return +// getStack returns a temporary stack. +// The caller must call [stack.free] to give up use of the stack when finished. +func getStack() *stack { + s, _ := stackPool.Get().(*stack) + if s == nil { + s = new(stack) } - - n2 := n >> 1 - x1, x0 := x[n2:], x[0:n2] - - karatsubaSqr(z, x0) - karatsubaSqr(z[n:], x1) - - // s = sign(xd*yd) == -1 for xd != 0; s == 1 for xd == 0 - xd := z[2*n : 2*n+n2] - if subVV(xd, x1, x0) != 0 { - subVV(xd, x0, x1) - } - - p := z[n*3:] - karatsubaSqr(p, xd) - - r := z[n*4:] - copy(r, z[:n*2]) - - karatsubaAdd(z[n2:], r, n) - karatsubaAdd(z[n2:], r[n:], n) - karatsubaSub(z[n2:], p, n) // s == -1 for p != 0; s == 1 for p == 0 + return s } -// Operands that are shorter than basicSqrThreshold are squared using -// "grade school" multiplication; for operands longer than karatsubaSqrThreshold -// we use the Karatsuba algorithm optimized for x == y. -var basicSqrThreshold = 20 // computed by calibrate_test.go -var karatsubaSqrThreshold = 260 // computed by calibrate_test.go - -// z = x*x -func (z nat) sqr(x nat) nat { - n := len(x) - switch { - case n == 0: - return z[:0] - case n == 1: - d := x[0] - z = z.make(2) - z[1], z[0] = mulWW(d, d) - return z.norm() - } - - if alias(z, x) { - z = nil // z is an alias for x - cannot reuse - } - - if n < basicSqrThreshold { - z = z.make(2 * n) - basicMul(z, x, x) - return z.norm() - } - if n < karatsubaSqrThreshold { - z = z.make(2 * n) - basicSqr(z, x) - return z.norm() - } - - // Use Karatsuba multiplication optimized for x == y. - // The algorithm and layout of z are the same as for mul. - - // z = (x1*b + x0)^2 = x1^2*b^2 + 2*x1*x0*b + x0^2 - - k := karatsubaLen(n, karatsubaSqrThreshold) - - x0 := x[0:k] - z = z.make(max(6*k, 2*n)) - karatsubaSqr(z, x0) // z = x0^2 - z = z[0 : 2*n] - clear(z[2*k:]) - - if k < n { - tp := getNat(2 * k) - t := *tp - x0 := x0.norm() - x1 := x[k:] - t = t.mul(x0, x1) - addAt(z, t, k) - addAt(z, t, k) // z = 2*x1*x0*b + x0^2 - t = t.sqr(x1) - addAt(z, t, 2*k) // z = x1^2*b^2 + 2*x1*x0*b + x0^2 - putNat(tp) - } +// free returns the stack for use by another calculation. +func (s *stack) free() { + s.w = s.w[:0] + stackPool.Put(s) +} - return z.norm() +// save returns the current stack pointer. +// A future call to restore with the same value +// frees any temporaries allocated on the stack after the call to save. +func (s *stack) save() int { + return len(s.w) } -// mulRange computes the product of all the unsigned integers in the -// range [a, b] inclusively. If a > b (empty range), the result is 1. -func (z nat) mulRange(a, b uint64) nat { - switch { - case a == 0: - // cut long ranges short (optimization) - return z.setUint64(0) - case a > b: - return z.setUint64(1) - case a == b: - return z.setUint64(a) - case a+1 == b: - return z.mul(nat(nil).setUint64(a), nat(nil).setUint64(b)) - } - m := a + (b-a)/2 // avoid overflow - return z.mul(nat(nil).mulRange(a, m), nat(nil).mulRange(m+1, b)) +// restore restores the stack pointer to n. +// It is almost always invoked as +// +// defer stk.restore(stk.save()) +// +// which makes sure to pop any temporaries allocated in the current function +// from the stack before returning. +func (s *stack) restore(n int) { + s.w = s.w[:n] } -// getNat returns a *nat of len n. The contents may not be zero. -// The pool holds *nat to avoid allocation when converting to interface{}. -func getNat(n int) *nat { - var z *nat - if v := natPool.Get(); v != nil { - z = v.(*nat) - } - if z == nil { - z = new(nat) - } - *z = z.make(n) +// nat returns a nat of n words, allocated on the stack. +func (s *stack) nat(n int) nat { + nr := (n + 3) &^ 3 // round up to multiple of 4 + off := len(s.w) + s.w = slices.Grow(s.w, nr) + s.w = s.w[:off+nr] + x := s.w[off : off+n : off+n] if n > 0 { - (*z)[0] = 0xfedcb // break code expecting zero + x[0] = 0xfedcb } - return z -} - -func putNat(x *nat) { - natPool.Put(x) + return x } -var natPool sync.Pool - // bitLen returns the length of x in bits. // Unlike most methods, it works even if x is not normalized. func (x nat) bitLen() int { @@ -698,7 +380,7 @@ func same(x, y nat) bool { } // z = x << s -func (z nat) shl(x nat, s uint) nat { +func (z nat) lsh(x nat, s uint) nat { if s == 0 { if same(z, x) { return z @@ -716,14 +398,19 @@ func (z nat) shl(x nat, s uint) nat { n := m + int(s/_W) z = z.make(n + 1) - z[n] = shlVU(z[n-m:n], x, s%_W) + if s %= _W; s == 0 { + copy(z[n-m:n], x) + z[n] = 0 + } else { + z[n] = lshVU(z[n-m:n], x, s) + } clear(z[0 : n-m]) return z.norm() } // z = x >> s -func (z nat) shr(x nat, s uint) nat { +func (z nat) rsh(x nat, s uint) nat { if s == 0 { if same(z, x) { return z @@ -741,7 +428,11 @@ func (z nat) shr(x nat, s uint) nat { // n > 0 z = z.make(n) - shrVU(z, x[m-n:], s%_W) + if s %= _W; s == 0 { + copy(z, x[m-n:]) + } else { + rshVU(z, x[m-n:], s) + } return z.norm() } @@ -930,7 +621,8 @@ func (z nat) random(rand *rand.Rand, limit nat, n int) nat { // If m != 0 (i.e., len(m) != 0), expNN sets z to x**y mod m; // otherwise it sets z to x**y. The result is the value of z. -func (z nat) expNN(x, y, m nat, slow bool) nat { +// The caller may pass stk == nil to request that expNN obtain and release one itself. +func (z nat) expNN(stk *stack, x, y, m nat, slow bool) nat { if alias(z, x) || alias(z, y) { // We cannot allow in-place modification of x or y. z = nil @@ -961,12 +653,17 @@ func (z nat) expNN(x, y, m nat, slow bool) nat { // x > 1 // x**1 == x - if len(y) == 1 && y[0] == 1 { - if len(m) != 0 { - return z.rem(x, m) - } + if len(y) == 1 && y[0] == 1 && len(m) == 0 { return z.set(x) } + if stk == nil { + stk = getStack() + defer stk.free() + } + if len(y) == 1 && y[0] == 1 { // len(m) > 0 + return z.rem(stk, x, m) + } + // y > 1 if len(m) != 0 { @@ -980,12 +677,12 @@ func (z nat) expNN(x, y, m nat, slow bool) nat { // instance of each of the first two cases). if len(y) > 1 && !slow { if m[0]&1 == 1 { - return z.expNNMontgomery(x, y, m) + return z.expNNMontgomery(stk, x, y, m) } if logM, ok := m.isPow2(); ok { - return z.expNNWindowed(x, y, logM) + return z.expNNWindowed(stk, x, y, logM) } - return z.expNNMontgomeryEven(x, y, m) + return z.expNNMontgomeryEven(stk, x, y, m) } } @@ -1006,16 +703,16 @@ func (z nat) expNN(x, y, m nat, slow bool) nat { // otherwise the arguments would alias. var zz, r nat for j := 0; j < w; j++ { - zz = zz.sqr(z) + zz = zz.sqr(stk, z) zz, z = z, zz if v&mask != 0 { - zz = zz.mul(z, x) + zz = zz.mul(stk, z, x) zz, z = z, zz } if len(m) != 0 { - zz, r = zz.div(r, z, m) + zz, r = zz.div(stk, r, z, m) zz, r, q, z = q, z, zz, r } @@ -1026,16 +723,16 @@ func (z nat) expNN(x, y, m nat, slow bool) nat { v = y[i] for j := 0; j < _W; j++ { - zz = zz.sqr(z) + zz = zz.sqr(stk, z) zz, z = z, zz if v&mask != 0 { - zz = zz.mul(z, x) + zz = zz.mul(stk, z, x) zz, z = z, zz } if len(m) != 0 { - zz, r = zz.div(r, z, m) + zz, r = zz.div(stk, r, z, m) zz, r, q, z = q, z, zz, r } @@ -1054,11 +751,11 @@ func (z nat) expNN(x, y, m nat, slow bool) nat { // For more details, see Ç. K. Koç, “Montgomery Reduction with Even Modulus”, // IEE Proceedings: Computers and Digital Techniques, 141(5) 314-316, September 1994. // http://www.people.vcu.edu/~jwang3/CMSC691/j34monex.pdf -func (z nat) expNNMontgomeryEven(x, y, m nat) nat { +func (z nat) expNNMontgomeryEven(stk *stack, x, y, m nat) nat { // Split m = m₁ × m₂ where m₁ = 2ⁿ n := m.trailingZeroBits() - m1 := nat(nil).shl(natOne, n) - m2 := nat(nil).shr(m, n) + m1 := nat(nil).lsh(natOne, n) + m2 := nat(nil).rsh(m, n) // We want z = x**y mod m. // z₁ = x**y mod m1 = (x**y mod m) mod m1 = z mod m1 @@ -1066,8 +763,8 @@ func (z nat) expNNMontgomeryEven(x, y, m nat) nat { // (We are using the math/big convention for names here, // where the computation is z = x**y mod m, so its parts are z1 and z2. // The paper is computing x = a**e mod n; it refers to these as x2 and z1.) - z1 := nat(nil).expNN(x, y, m1, false) - z2 := nat(nil).expNN(x, y, m2, false) + z1 := nat(nil).expNN(stk, x, y, m1, false) + z2 := nat(nil).expNN(stk, x, y, m2, false) // Reconstruct z from z₁, z₂ using CRT, using algorithm from paper, // which uses only a single modInverse (and an easy one at that). @@ -1086,18 +783,18 @@ func (z nat) expNNMontgomeryEven(x, y, m nat) nat { // Reuse z2 for p = (z₁ - z₂) [in z1] * m2⁻¹ (mod m₁ [= 2ⁿ]). m2inv := nat(nil).modInverse(m2, m1) - z2 = z2.mul(z1, m2inv) + z2 = z2.mul(stk, z1, m2inv) z2 = z2.trunc(z2, n) // Reuse z1 for p * m2. - z = z.add(z, z1.mul(z2, m2)) + z = z.add(z, z1.mul(stk, z2, m2)) return z } // expNNWindowed calculates x**y mod m using a fixed, 4-bit window, // where m = 2**logM. -func (z nat) expNNWindowed(x, y nat, logM uint) nat { +func (z nat) expNNWindowed(stk *stack, x, y nat, logM uint) nat { if len(y) <= 1 { panic("big: misuse of expNNWindowed") } @@ -1112,23 +809,23 @@ func (z nat) expNNWindowed(x, y nat, logM uint) nat { // zz is used to avoid allocating in mul as otherwise // the arguments would alias. + defer stk.restore(stk.save()) w := int((logM + _W - 1) / _W) - zzp := getNat(w) - zz := *zzp + zz := stk.nat(w) const n = 4 // powers[i] contains x^i. - var powers [1 << n]*nat + var powers [1 << n]nat for i := range powers { - powers[i] = getNat(w) + powers[i] = stk.nat(w) } - *powers[0] = powers[0].set(natOne) - *powers[1] = powers[1].trunc(x, logM) + powers[0] = powers[0].set(natOne) + powers[1] = powers[1].trunc(x, logM) for i := 2; i < 1<>(_W-n)]) + zz = zz.mul(stk, z, powers[yi>>(_W-n)]) zz, z = z, zz z = z.trunc(z, logM) @@ -1185,24 +882,18 @@ func (z nat) expNNWindowed(x, y nat, logM uint) nat { } } - *zzp = zz - putNat(zzp) - for i := range powers { - putNat(powers[i]) - } - return z.norm() } // expNNMontgomery calculates x**y mod m using a fixed, 4-bit window. // Uses Montgomery representation. -func (z nat) expNNMontgomery(x, y, m nat) nat { +func (z nat) expNNMontgomery(stk *stack, x, y, m nat) nat { numWords := len(m) // We want the lengths of x and m to be equal. // It is OK if x >= m as long as len(x) == len(m). if len(x) > numWords { - _, x = nat(nil).div(nil, x, m) + _, x = nat(nil).div(stk, nil, x, m) // Note: now len(x) <= numWords, not guaranteed ==. } if len(x) < numWords { @@ -1224,8 +915,8 @@ func (z nat) expNNMontgomery(x, y, m nat) nat { // RR = 2**(2*_W*len(m)) mod m RR := nat(nil).setWord(1) - zz := nat(nil).shl(RR, uint(2*numWords*_W)) - _, RR = nat(nil).div(RR, zz, m) + zz := nat(nil).lsh(RR, uint(2*numWords*_W)) + _, RR = nat(nil).div(stk, RR, zz, m) if len(RR) < numWords { zz = zz.make(numWords) copy(zz, RR) @@ -1280,7 +971,7 @@ func (z nat) expNNMontgomery(x, y, m nat) nat { // The div is not expected to be reached. zz = zz.sub(zz, m) if zz.cmp(m) >= 0 { - _, zz = nat(nil).div(nil, zz, m) + _, zz = nat(nil).div(stk, nil, zz, m) } } @@ -1321,9 +1012,9 @@ func (z nat) bytes(buf []byte) (i int) { // bigEndianWord returns the contents of buf interpreted as a big-endian encoded Word value. func bigEndianWord(buf []byte) Word { if _W == 64 { - return Word(byteorder.BeUint64(buf)) + return Word(byteorder.BEUint64(buf)) } - return Word(byteorder.BeUint32(buf)) + return Word(byteorder.BEUint32(buf)) } // setBytes interprets buf as the bytes of a big-endian unsigned @@ -1349,7 +1040,8 @@ func (z nat) setBytes(buf []byte) nat { } // sqrt sets z = ⌊√x⌋ -func (z nat) sqrt(x nat) nat { +// The caller may pass stk == nil to request that sqrt obtain and release one itself. +func (z nat) sqrt(stk *stack, x nat) nat { if x.cmp(natOne) <= 0 { return z.set(x) } @@ -1357,6 +1049,11 @@ func (z nat) sqrt(x nat) nat { z = nil } + if stk == nil { + stk = getStack() + defer stk.free() + } + // Start with value known to be too large and repeat "z = ⌊(z + ⌊x/z⌋)/2⌋" until it stops getting smaller. // See Brent and Zimmermann, Modern Computer Arithmetic, Algorithm 1.13 (SqrtInt). // https://members.loria.fr/PZimmermann/mca/pub226.html @@ -1365,11 +1062,11 @@ func (z nat) sqrt(x nat) nat { var z1, z2 nat z1 = z z1 = z1.setUint64(1) - z1 = z1.shl(z1, uint(x.bitLen()+1)/2) // must be ≥ √x + z1 = z1.lsh(z1, uint(x.bitLen()+1)/2) // must be ≥ √x for n := 0; ; n++ { - z2, _ = z2.div(nil, x, z1) + z2, _ = z2.div(stk, nil, x, z1) z2 = z2.add(z2, z1) - z2 = z2.shr(z2, 1) + z2 = z2.rsh(z2, 1) if z2.cmp(z1) >= 0 { // z1 is answer. // Figure out whether z1 or z2 is currently aliased to z by looking at loop count. diff --git a/src/math/big/nat_test.go b/src/math/big/nat_test.go index 4722548fa965fa..333d33f4528a49 100644 --- a/src/math/big/nat_test.go +++ b/src/math/big/nat_test.go @@ -7,6 +7,8 @@ package big import ( "fmt" "math" + "math/bits" + "math/rand/v2" "runtime" "strings" "testing" @@ -42,6 +44,7 @@ func TestCmp(t *testing.T) { } type funNN func(z, x, y nat) nat +type funSNN func(z nat, stk *stack, x, y nat) nat type argNN struct { z, x, y nat } @@ -55,12 +58,84 @@ var sumNN = []argNN{ {nat{0, 0, 0, 1}, nat{0, 0, _M}, nat{0, 0, 1}}, } -var prodNN = []argNN{ - {}, - {nil, nil, nil}, +var prodNN = append(prodTests(), prodNNExtra...) + +func permute[E any](x []E) { + out := make([]E, len(x)) + for i, j := range rand.Perm(len(x)) { + out[i] = x[j] + } + copy(x, out) +} + +// testMul returns the product of x and y using the grade-school algorithm, +// as a reference implementation. +func testMul(x, y nat) nat { + z := make(nat, len(x)+len(y)) + for i, xi := range x { + for j, yj := range y { + hi, lo := bits.Mul(uint(xi), uint(yj)) + k := i + j + s, c := bits.Add(uint(z[k]), lo, 0) + z[k] = Word(s) + k++ + for hi != 0 || c != 0 { + s, c = bits.Add(uint(z[k]), hi, c) + hi = 0 + z[k] = Word(s) + k++ + } + } + } + return z.norm() +} + +func prodTests() []argNN { + var tests []argNN + for size := range 10 { + var x, y nat + for i := range size { + x = append(x, Word(i+1)) + y = append(y, Word(i+1+size)) + } + permute(x) + permute(y) + x = x.norm() + y = y.norm() + tests = append(tests, argNN{testMul(x, y), x, y}) + } + + words := []Word{0, 1, 2, 3, 4, ^Word(0), ^Word(1), ^Word(2), ^Word(3)} + for size := range 10 { + if size == 0 { + continue // already tested the only 0-length possibility above + } + for range 10 { + x := make(nat, size) + y := make(nat, size) + for i := range size { + x[i] = words[rand.N(len(words))] + y[i] = words[rand.N(len(words))] + } + x = x.norm() + y = y.norm() + tests = append(tests, argNN{testMul(x, y), x, y}) + } + } + return tests +} + +var prodNNExtra = []argNN{ {nil, nat{991}, nil}, {nat{991}, nat{991}, nat{1}}, {nat{991 * 991}, nat{991}, nat{991}}, + {nat{8, 22, 15}, nat{2, 3}, nat{4, 5}}, + {nat{10, 27, 52, 45, 28}, nat{2, 3, 4}, nat{5, 6, 7}}, + {nat{12, 32, 61, 100, 94, 76, 45}, nat{2, 3, 4, 5}, nat{6, 7, 8, 9}}, + {nat{12, 32, 61, 100, 94, 76, 45}, nat{2, 3, 4, 5}, nat{6, 7, 8, 9}}, + {nat{14, 37, 70, 114, 170, 166, 148, 115, 66}, nat{2, 3, 4, 5, 6}, nat{7, 8, 9, 10, 11}}, + {nat{991 * 991, 991 * 2, 1}, nat{991, 1}, nat{991, 1}}, + {nat{991 * 991, 991 * 777 * 2, 777 * 777}, nat{991, 777}, nat{991, 777}}, {nat{0, 0, 991 * 991}, nat{0, 991}, nat{0, 991}}, {nat{1 * 991, 2 * 991, 3 * 991, 4 * 991}, nat{1, 2, 3, 4}, nat{991}}, {nat{4, 11, 20, 30, 20, 11, 4}, nat{1, 2, 3, 4}, nat{4, 3, 2, 1}}, @@ -112,30 +187,114 @@ func testFunNN(t *testing.T, msg string, f funNN, a argNN) { } } -func TestFunNN(t *testing.T) { - for _, a := range sumNN { - arg := a - testFunNN(t, "add", nat.add, arg) +func testFunSNN(t *testing.T, msg string, f funSNN, a argNN) { + t.Helper() + stk := getStack() + defer stk.free() + z := f(nil, stk, a.x, a.y) + if z.cmp(a.z) != 0 { + t.Fatalf("%s%+v\n\tgot z = %v; want %v", msg, a, z, a.z) + } +} - arg = argNN{a.z, a.y, a.x} - testFunNN(t, "add symmetric", nat.add, arg) +func setDuringTest[V any](t *testing.T, p *V, v V) { + old := *p + *p = v + t.Cleanup(func() { *p = old }) +} - arg = argNN{a.x, a.z, a.y} - testFunNN(t, "sub", nat.sub, arg) +func TestAdd(t *testing.T) { + for _, a := range sumNN { + testFunNN(t, "add", nat.add, a) + a.x, a.y = a.y, a.x + testFunNN(t, "add", nat.add, a) + } +} - arg = argNN{a.y, a.z, a.x} - testFunNN(t, "sub symmetric", nat.sub, arg) +func TestSub(t *testing.T) { + for _, a := range sumNN { + a.x, a.z = a.z, a.x + testFunNN(t, "sub", nat.sub, a) + + a.y, a.z = a.z, a.y + testFunNN(t, "sub", nat.sub, a) } +} - for _, a := range prodNN { - arg := a - testFunNN(t, "mul", nat.mul, arg) +func TestNatMul(t *testing.T) { + t.Run("Basic", func(t *testing.T) { + setDuringTest(t, &karatsubaThreshold, 1e9) + for _, a := range prodNN { + if len(a.z) >= 100 { + continue + } + testFunSNN(t, "mul", nat.mul, a) + a.x, a.y = a.y, a.x + testFunSNN(t, "mul", nat.mul, a) + } + }) + t.Run("Karatsuba", func(t *testing.T) { + setDuringTest(t, &karatsubaThreshold, 2) + for _, a := range prodNN { + testFunSNN(t, "mul", nat.mul, a) + a.x, a.y = a.y, a.x + testFunSNN(t, "mul", nat.mul, a) + } + }) - arg = argNN{a.z, a.y, a.x} - testFunNN(t, "mul symmetric", nat.mul, arg) + t.Run("Mul", func(t *testing.T) { + for _, a := range prodNN { + testFunSNN(t, "mul", nat.mul, a) + a.x, a.y = a.y, a.x + testFunSNN(t, "mul", nat.mul, a) + } + }) +} + +func testSqr(t *testing.T, x nat) { + stk := getStack() + defer stk.free() + + got := make(nat, 2*len(x)) + want := make(nat, 2*len(x)) + got = got.sqr(stk, x) + want = want.mul(stk, x, x) + if got.cmp(want) != 0 { + t.Errorf("basicSqr(%v), got %v, want %v", x, got, want) } } +func TestNatSqr(t *testing.T) { + t.Run("Basic", func(t *testing.T) { + setDuringTest(t, &basicSqrThreshold, 0) + setDuringTest(t, &karatsubaSqrThreshold, 1e9) + for _, a := range prodNN { + if len(a.z) >= 100 { + continue + } + testSqr(t, a.x) + testSqr(t, a.y) + testSqr(t, a.z) + } + }) + t.Run("Karatsuba", func(t *testing.T) { + setDuringTest(t, &basicSqrThreshold, 2) + setDuringTest(t, &karatsubaSqrThreshold, 2) + for _, a := range prodNN { + testSqr(t, a.x) + testSqr(t, a.y) + testSqr(t, a.z) + } + }) + t.Run("Sqr", func(t *testing.T) { + for _, a := range prodNN { + testSqr(t, a.x) + testSqr(t, a.y) + testSqr(t, a.z) + } + }) +} + var mulRangesN = []struct { a, b uint64 prod string @@ -163,8 +322,11 @@ var mulRangesN = []struct { } func TestMulRangeN(t *testing.T) { + stk := getStack() + defer stk.free() + for i, r := range mulRangesN { - prod := string(nat(nil).mulRange(r.a, r.b).utoa(10)) + prod := string(nat(nil).mulRange(stk, r.a, r.b).utoa(10)) if prod != r.prod { t.Errorf("#%d: got %s; want %s", i, prod, r.prod) } @@ -185,11 +347,14 @@ func allocBytes(f func()) uint64 { // does not cause deep recursion and in turn allocate too much memory. // Test case for issue 3807. func TestMulUnbalanced(t *testing.T) { + stk := getStack() + defer stk.free() + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1)) x := rndNat(50000) y := rndNat(40) allocSize := allocBytes(func() { - nat(nil).mul(x, y) + nat(nil).mul(stk, x, y) }) inputSize := uint64(len(x)+len(y)) * _S if ratio := allocSize / uint64(inputSize); ratio > 10 { @@ -213,23 +378,14 @@ func rndNat1(n int) nat { return x } -func BenchmarkMul(b *testing.B) { - mulx := rndNat(1e4) - muly := rndNat(1e4) - b.ResetTimer() - for i := 0; i < b.N; i++ { - var z nat - z.mul(mulx, muly) - } -} - func benchmarkNatMul(b *testing.B, nwords int) { x := rndNat(nwords) y := rndNat(nwords) var z nat b.ResetTimer() + b.ReportAllocs() for i := 0; i < b.N; i++ { - z.mul(x, y) + z.mul(nil, x, y) } } @@ -274,7 +430,7 @@ var leftShiftTests = []shiftTest{ func TestShiftLeft(t *testing.T) { for i, test := range leftShiftTests { var z nat - z = z.shl(test.in, test.shift) + z = z.lsh(test.in, test.shift) for j, d := range test.out { if j >= len(z) || z[j] != d { t.Errorf("#%d: got: %v want: %v", i, z, test.out) @@ -297,7 +453,7 @@ var rightShiftTests = []shiftTest{ func TestShiftRight(t *testing.T) { for i, test := range rightShiftTests { var z nat - z = z.shr(test.in, test.shift) + z = z.rsh(test.in, test.shift) for j, d := range test.out { if j >= len(z) || z[j] != d { t.Errorf("#%d: got: %v want: %v", i, z, test.out) @@ -310,27 +466,27 @@ func TestShiftRight(t *testing.T) { func BenchmarkZeroShifts(b *testing.B) { x := rndNat(800) - b.Run("Shl", func(b *testing.B) { + b.Run("Lsh", func(b *testing.B) { for i := 0; i < b.N; i++ { var z nat - z.shl(x, 0) + z.lsh(x, 0) } }) - b.Run("ShlSame", func(b *testing.B) { + b.Run("LshSame", func(b *testing.B) { for i := 0; i < b.N; i++ { - x.shl(x, 0) + x.lsh(x, 0) } }) - b.Run("Shr", func(b *testing.B) { + b.Run("Rsh", func(b *testing.B) { for i := 0; i < b.N; i++ { var z nat - z.shr(x, 0) + z.rsh(x, 0) } }) - b.Run("ShrSame", func(b *testing.B) { + b.Run("RshSame", func(b *testing.B) { for i := 0; i < b.N; i++ { - x.shr(x, 0) + x.rsh(x, 0) } }) } @@ -443,6 +599,9 @@ var montgomeryTests = []struct { } func TestMontgomery(t *testing.T) { + stk := getStack() + defer stk.free() + one := NewInt(1) _B := new(Int).Lsh(one, _W) for i, test := range montgomeryTests { @@ -457,11 +616,11 @@ func TestMontgomery(t *testing.T) { } if x.cmp(m) > 0 { - _, r := nat(nil).div(nil, x, m) + _, r := nat(nil).div(stk, nil, x, m) t.Errorf("#%d: x > m (0x%s > 0x%s; use 0x%s)", i, x.utoa(16), m.utoa(16), r.utoa(16)) } if y.cmp(m) > 0 { - _, r := nat(nil).div(nil, x, m) + _, r := nat(nil).div(stk, nil, x, m) t.Errorf("#%d: y > m (0x%s > 0x%s; use 0x%s)", i, y.utoa(16), m.utoa(16), r.utoa(16)) } @@ -537,6 +696,9 @@ var expNNTests = []struct { } func TestExpNN(t *testing.T) { + stk := getStack() + defer stk.free() + for i, test := range expNNTests { x := natFromString(test.x) y := natFromString(test.y) @@ -547,7 +709,7 @@ func TestExpNN(t *testing.T) { m = natFromString(test.m) } - z := nat(nil).expNN(x, y, m, false) + z := nat(nil).expNN(stk, x, y, m, false) if z.cmp(out) != 0 { t.Errorf("#%d got %s want %s", i, z.utoa(10), out.utoa(10)) } @@ -571,6 +733,9 @@ func FuzzExpMont(f *testing.F) { } func BenchmarkExp3Power(b *testing.B) { + stk := getStack() + defer stk.free() + const x = 3 for _, y := range []Word{ 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, @@ -578,7 +743,7 @@ func BenchmarkExp3Power(b *testing.B) { b.Run(fmt.Sprintf("%#x", y), func(b *testing.B) { var z nat for i := 0; i < b.N; i++ { - z.expWW(x, y) + z.expWW(stk, x, y) } }) } @@ -710,36 +875,13 @@ func TestSticky(t *testing.T) { } } -func testSqr(t *testing.T, x nat) { - got := make(nat, 2*len(x)) - want := make(nat, 2*len(x)) - got = got.sqr(x) - want = want.mul(x, x) - if got.cmp(want) != 0 { - t.Errorf("basicSqr(%v), got %v, want %v", x, got, want) - } -} - -func TestSqr(t *testing.T) { - for _, a := range prodNN { - if a.x != nil { - testSqr(t, a.x) - } - if a.y != nil { - testSqr(t, a.y) - } - if a.z != nil { - testSqr(t, a.z) - } - } -} - func benchmarkNatSqr(b *testing.B, nwords int) { x := rndNat(nwords) var z nat b.ResetTimer() + b.ReportAllocs() for i := 0; i < b.N; i++ { - z.sqr(x) + z.sqr(nil, x) } } @@ -828,6 +970,9 @@ func BenchmarkNatSetBytes(b *testing.B) { } func TestNatDiv(t *testing.T) { + stk := getStack() + defer stk.free() + sizes := []int{ 1, 2, 5, 8, 15, 25, 40, 65, 100, 200, 500, 800, 1500, 2500, 4000, 6500, 10000, @@ -847,11 +992,11 @@ func TestNatDiv(t *testing.T) { c = c.norm() } // compute x = a*b+c - x := nat(nil).mul(a, b) + x := nat(nil).mul(stk, a, b) x = x.add(x, c) var q, r nat - q, r = q.div(r, x, b) + q, r = q.div(stk, r, x, b) if q.cmp(a) != 0 { t.Fatalf("wrong quotient: got %s; want %s for %s/%s", q.utoa(10), a.utoa(10), x.utoa(10), b.utoa(10)) } @@ -866,6 +1011,9 @@ func TestNatDiv(t *testing.T) { // the inaccurate estimate of the first word's quotient // happens at the very beginning of the loop. func TestIssue37499(t *testing.T) { + stk := getStack() + defer stk.free() + // Choose u and v such that v is slightly larger than u >> N. // This tricks divBasic into choosing 1 as the first word // of the quotient. This works in both 32-bit and 64-bit settings. @@ -873,7 +1021,7 @@ func TestIssue37499(t *testing.T) { v := natFromString("0x2b6c385a05be027f5c22005b63c42a1165b79ff510e1706c") q := nat(nil).make(8) - q.divBasic(u, v) + q.divBasic(stk, u, v) q = q.norm() if s := string(q.utoa(16)); s != "fffffffffffffffffffffffffffffffffffffffffffffffb" { t.Fatalf("incorrect quotient: %s", s) @@ -884,8 +1032,11 @@ func TestIssue37499(t *testing.T) { // where the first division loop is never entered, and correcting // the remainder takes exactly two iterations in the final loop. func TestIssue42552(t *testing.T) { + stk := getStack() + defer stk.free() + u := natFromString("0xc23b166884c3869092a520eceedeced2b00847bd256c9cf3b2c5e2227c15bd5e6ee7ef8a2f49236ad0eedf2c8a3b453cf6e0706f64285c526b372c4b1321245519d430540804a50b7ca8b6f1b34a2ec05cdbc24de7599af112d3e3c8db347e8799fe70f16e43c6566ba3aeb169463a3ecc486172deb2d9b80a3699c776e44fef20036bd946f1b4d054dd88a2c1aeb986199b0b2b7e58c42288824b74934d112fe1fc06e06b4d99fe1c5e725946b23210521e209cd507cce90b5f39a523f27e861f9e232aee50c3f585208b4573dcc0b897b6177f2ba20254fd5c50a033e849dee1b3a93bd2dc44ba8ca836cab2c2ae50e50b126284524fa0187af28628ff0face68d87709200329db1392852c8b8963fbe3d05fb1efe19f0ed5ca9fadc2f96f82187c24bb2512b2e85a66333a7e176605695211e1c8e0b9b9e82813e50654964945b1e1e66a90840396c7d10e23e47f364d2d3f660fa54598e18d1ca2ea4fe4f35a40a11f69f201c80b48eaee3e2e9b0eda63decf92bec08a70f731587d4ed0f218d5929285c8b2ccbc497e20db42de73885191fa453350335990184d8df805072f958d5354debda38f5421effaaafd6cb9b721ace74be0892d77679f62a4a126697cd35797f6858193da4ba1770c06aea2e5c59ec04b8ea26749e61b72ecdde403f3bc7e5e546cd799578cc939fa676dfd5e648576d4a06cbadb028adc2c0b461f145b2321f42e5e0f3b4fb898ecd461df07a6f5154067787bf74b5cc5c03704a1ce47494961931f0263b0aac32505102595957531a2de69dd71aac51f8a49902f81f21283dbe8e21e01e5d82517868826f86acf338d935aa6b4d5a25c8d540389b277dd9d64569d68baf0f71bd03dba45b92a7fc052601d1bd011a2fc6790a23f97c6fa5caeea040ab86841f268d39ce4f7caf01069df78bba098e04366492f0c2ac24f1bf16828752765fa523c9a4d42b71109d123e6be8c7b1ab3ccf8ea03404075fe1a9596f1bba1d267f9a7879ceece514818316c9c0583469d2367831fc42b517ea028a28df7c18d783d16ea2436cee2b15d52db68b5dfdee6b4d26f0905f9b030c911a04d078923a4136afea96eed6874462a482917353264cc9bee298f167ac65a6db4e4eda88044b39cc0b33183843eaa946564a00c3a0ab661f2c915e70bf0bb65bfbb6fa2eea20aed16bf2c1a1d00ec55fb4ff2f76b8e462ea70c19efa579c9ee78194b86708fdae66a9ce6e2cf3d366037798cfb50277ba6d2fd4866361022fd788ab7735b40b8b61d55e32243e06719e53992e9ac16c9c4b6e6933635c3c47c8f7e73e17dd54d0dd8aeba5d76de46894e7b3f9d3ec25ad78ee82297ba69905ea0fa094b8667faa2b8885e2187b3da80268aa1164761d7b0d6de206b676777348152b8ae1d4afed753bc63c739a5ca8ce7afb2b241a226bd9e502baba391b5b13f5054f070b65a9cf3a67063bfaa803ba390732cd03888f664023f888741d04d564e0b5674b0a183ace81452001b3fbb4214c77d42ca75376742c471e58f67307726d56a1032bd236610cbcbcd03d0d7a452900136897dc55bb3ce959d10d4e6a10fb635006bd8c41cd9ded2d3dfdd8f2e229590324a7370cb2124210b2330f4c56155caa09a2564932ceded8d92c79664dcdeb87faad7d3da006cc2ea267ee3df41e9677789cc5a8cc3b83add6491561b3047919e0648b1b2e97d7ad6f6c2aa80cab8e9ae10e1f75b1fdd0246151af709d259a6a0ed0b26bd711024965ecad7c41387de45443defce53f66612948694a6032279131c257119ed876a8e805dfb49576ef5c563574115ee87050d92d191bc761ef51d966918e2ef925639400069e3959d8fe19f36136e947ff430bf74e71da0aa5923b00000000") v := natFromString("0x838332321d443a3d30373d47301d47073847473a383d3030f25b3d3d3e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002e00000000000000000041603038331c3d32f5303441e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e01c0a5459bfc7b9be9fcbb9d2383840464319434707303030f43a32f53034411c0a5459413820878787878787878787878787878787878787878787878787878787878787878787870630303a3a30334036605b923a6101f83638413943413960204337602043323801526040523241846038414143015238604060328452413841413638523c0240384141364036605b923a6101f83638413943413960204334602043323801526040523241846038414143015238604060328452413841413638523c02403841413638433030f25a8b83838383838383838383838383838383837d838383ffffffffffffffff838383838383838383000000000000000000030000007d26e27c7c8b83838383838383838383838383838383837d838383ffffffffffffffff83838383838383838383838383838383838383838383435960f535073030f3343200000000000000011881301938343030fa398383300000002300000000000000000000f11af4600c845252904141364138383c60406032414443095238010241414303364443434132305b595a15434160b042385341ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff47476043410536613603593a6005411c437405fcfcfcfcfcfcfc0000000000005a3b075815054359000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") q := nat(nil).make(16) - q.div(q, u, v) + q.div(stk, q, u, v) } diff --git a/src/math/big/natconv.go b/src/math/big/natconv.go index ce94f2cf72e9a1..96cba37c06d2f0 100644 --- a/src/math/big/natconv.go +++ b/src/math/big/natconv.go @@ -12,6 +12,7 @@ import ( "io" "math" "math/bits" + "slices" "sync" ) @@ -106,7 +107,7 @@ var ( // is set, only), and -count is the number of fractional digits found. // In this case, the actual value of the scanned number is res * b**count. func (z nat) scan(r io.ByteScanner, base int, fracOk bool) (res nat, b, count int, err error) { - // reject invalid bases + // Reject invalid bases. baseOk := base == 0 || !fracOk && 2 <= base && base <= MaxBase || fracOk && (base == 2 || base == 8 || base == 10 || base == 16) @@ -124,10 +125,10 @@ func (z nat) scan(r io.ByteScanner, base int, fracOk bool) (res nat, b, count in // one char look-ahead ch, err := r.ReadByte() - // determine actual base + // Determine actual base. b, prefix := base, 0 if base == 0 { - // actual base is 10 unless there's a base prefix + // Actual base is 10 unless there's a base prefix. b = 10 if err == nil && ch == '0' { prev = '0' @@ -157,16 +158,32 @@ func (z nat) scan(r io.ByteScanner, base int, fracOk bool) (res nat, b, count in } } - // convert string - // Algorithm: Collect digits in groups of at most n digits in di - // and then use mulAddWW for every such group to add them to the - // result. + // Convert string. + // Algorithm: Collect digits in groups of at most n digits in di. + // For bases that pack exactly into words (2, 4, 16), append di's + // directly to the int representation and then reverse at the end (bn==0 marks this case). + // For other bases, use mulAddWW for every such group to shift + // z up one group and add di to the result. + // With more cleverness we could also handle binary bases like 8 and 32 + // (corresponding to 3-bit and 5-bit chunks) that don't pack nicely into + // words, but those are not too important. z = z[:0] b1 := Word(b) - bn, n := maxPow(b1) // at most n digits in base b1 fit into Word - di := Word(0) // 0 <= di < b1**i < bn - i := 0 // 0 <= i < n - dp := -1 // position of decimal point + var bn Word // b1**n (or 0 for the special bit-packing cases b=2,4,16) + var n int // max digits that fit into Word + switch b { + case 2: // 1 bit per digit + n = _W + case 4: // 2 bits per digit + n = _W / 2 + case 16: // 4 bits per digit + n = _W / 4 + default: + bn, n = maxPow(b1) + } + di := Word(0) // 0 <= di < b1**i < bn + i := 0 // 0 <= i < n + dp := -1 // position of decimal point for err == nil { if ch == '.' && fracOk { fracOk = false @@ -210,7 +227,11 @@ func (z nat) scan(r io.ByteScanner, base int, fracOk bool) (res nat, b, count in // if di is "full", add it to the result if i == n { - z = z.mulAddWW(z, bn, di) + if bn == 0 { + z = append(z, di) + } else { + z = z.mulAddWW(z, bn, di) + } di = 0 i = 0 } @@ -238,11 +259,24 @@ func (z nat) scan(r io.ByteScanner, base int, fracOk bool) (res nat, b, count in err = errNoDigits // fall through; result will be 0 } - // add remaining digits to result - if i > 0 { - z = z.mulAddWW(z, pow(b1, i), di) + if bn == 0 { + if i > 0 { + // Add remaining digit chunk to result. + // Left-justify group's digits; will shift back down after reverse. + z = append(z, di*pow(b1, n-i)) + } + slices.Reverse(z) + z = z.norm() + if i > 0 { + z = z.rsh(z, uint(n-i)*uint(_W/n)) + } + } else { + if i > 0 { + // Add remaining digit chunk to result. + z = z.mulAddWW(z, pow(b1, i), di) + } } - res = z.norm() + res = z // adjust count for fraction, if any if dp >= 0 { @@ -321,17 +355,20 @@ func (x nat) itoa(neg bool, base int) []byte { } } else { + stk := getStack() + defer stk.free() + bb, ndigits := maxPow(b) // construct table of successive squares of bb*leafSize to use in subdivisions // result (table != nil) <=> (len(x) > leafSize > 0) - table := divisors(len(x), b, ndigits, bb) + table := divisors(stk, len(x), b, ndigits, bb) // preserve x, create local copy for use by convertWords q := nat(nil).set(x) // convert q to string s in base b - q.convertWords(s, b, ndigits, bb, table) + q.convertWords(stk, s, b, ndigits, bb, table) // strip leading zeros // (x != 0; thus s must contain at least one non-zero digit @@ -365,7 +402,7 @@ func (x nat) itoa(neg bool, base int) []byte { // range 2..64 shows that values of 8 and 16 work well, with a 4x speedup at medium lengths and // ~30x for 20000 digits. Use nat_test.go's BenchmarkLeafSize tests to optimize leafSize for // specific hardware. -func (q nat) convertWords(s []byte, b Word, ndigits int, bb Word, table []divisor) { +func (q nat) convertWords(stk *stack, s []byte, b Word, ndigits int, bb Word, table []divisor) { // split larger blocks recursively if table != nil { // len(q) > leafSize > 0 @@ -386,12 +423,12 @@ func (q nat) convertWords(s []byte, b Word, ndigits int, bb Word, table []diviso } // split q into the two digit number (q'*bbb + r) to form independent subblocks - q, r = q.div(r, q, table[index].bbb) + q, r = q.div(stk, r, q, table[index].bbb) // convert subblocks and collect results in s[:h] and s[h:] h := len(s) - table[index].ndigits - r.convertWords(s[h:], b, ndigits, bb, table[0:index]) - s = s[:h] // == q.convertWords(s, b, ndigits, bb, table[0:index+1]) + r.convertWords(stk, s[h:], b, ndigits, bb, table[0:index]) + s = s[:h] // == q.convertWords(stk, s, b, ndigits, bb, table[0:index+1]) } } @@ -451,12 +488,12 @@ var cacheBase10 struct { } // expWW computes x**y -func (z nat) expWW(x, y Word) nat { - return z.expNN(nat(nil).setWord(x), nat(nil).setWord(y), nil, false) +func (z nat) expWW(stk *stack, x, y Word) nat { + return z.expNN(stk, nat(nil).setWord(x), nat(nil).setWord(y), nil, false) } // construct table of powers of bb*leafSize to use in subdivisions. -func divisors(m int, b Word, ndigits int, bb Word) []divisor { +func divisors(stk *stack, m int, b Word, ndigits int, bb Word) []divisor { // only compute table when recursive conversion is enabled and x is large if leafSize == 0 || m <= leafSize { return nil @@ -484,10 +521,10 @@ func divisors(m int, b Word, ndigits int, bb Word) []divisor { for i := 0; i < k; i++ { if table[i].ndigits == 0 { if i == 0 { - table[0].bbb = nat(nil).expWW(bb, Word(leafSize)) + table[0].bbb = nat(nil).expWW(stk, bb, Word(leafSize)) table[0].ndigits = ndigits * leafSize } else { - table[i].bbb = nat(nil).sqr(table[i-1].bbb) + table[i].bbb = nat(nil).sqr(stk, table[i-1].bbb) table[i].ndigits = 2 * table[i-1].ndigits } diff --git a/src/math/big/natconv_test.go b/src/math/big/natconv_test.go index d3902721085152..670dc5fdb7e7de 100644 --- a/src/math/big/natconv_test.go +++ b/src/math/big/natconv_test.go @@ -350,7 +350,10 @@ func BenchmarkStringPiParallel(b *testing.B) { } func BenchmarkScan(b *testing.B) { - const x = 10 + stk := getStack() + defer stk.free() + + const x = 9 // avoid tested bases, in case runs of 0s are handled specially for _, base := range []int{2, 8, 10, 16} { for _, y := range []Word{10, 100, 1000, 10000, 100000} { if isRaceBuilder && y > 1000 { @@ -359,7 +362,7 @@ func BenchmarkScan(b *testing.B) { b.Run(fmt.Sprintf("%d/Base%d", y, base), func(b *testing.B) { b.StopTimer() var z nat - z = z.expWW(x, y) + z = z.expWW(stk, x, y) s := z.utoa(base) if t := itoa(z, base); !bytes.Equal(s, t) { @@ -376,6 +379,9 @@ func BenchmarkScan(b *testing.B) { } func BenchmarkString(b *testing.B) { + stk := getStack() + defer stk.free() + const x = 10 for _, base := range []int{2, 8, 10, 16} { for _, y := range []Word{10, 100, 1000, 10000, 100000} { @@ -385,7 +391,7 @@ func BenchmarkString(b *testing.B) { b.Run(fmt.Sprintf("%d/Base%d", y, base), func(b *testing.B) { b.StopTimer() var z nat - z = z.expWW(x, y) + z = z.expWW(stk, x, y) z.utoa(base) // warm divisor cache b.StartTimer() @@ -416,9 +422,11 @@ func LeafSizeHelper(b *testing.B, base, size int) { for d := 1; d <= 10000; d *= 10 { b.StopTimer() + stk := getStack() var z nat - z = z.expWW(Word(base), Word(d)) // build target number - _ = z.utoa(base) // warm divisor cache + z = z.expWW(stk, Word(base), Word(d)) // build target number + _ = z.utoa(base) // warm divisor cache + stk.free() b.StartTimer() for i := 0; i < b.N; i++ { @@ -443,13 +451,16 @@ func resetTable(table []divisor) { } func TestStringPowers(t *testing.T) { + stk := getStack() + defer stk.free() + var p Word for b := 2; b <= 16; b++ { for p = 0; p <= 512; p++ { if testing.Short() && p > 10 { break } - x := nat(nil).expWW(Word(b), p) + x := nat(nil).expWW(stk, Word(b), p) xs := x.utoa(b) xs2 := itoa(x, b) if !bytes.Equal(xs, xs2) { diff --git a/src/math/big/natdiv.go b/src/math/big/natdiv.go index b55f9990cd5975..88cb5d9e2e322c 100644 --- a/src/math/big/natdiv.go +++ b/src/math/big/natdiv.go @@ -502,30 +502,24 @@ import "math/bits" // rem returns r such that r = u%v. // It uses z as the storage for r. -func (z nat) rem(u, v nat) (r nat) { +func (z nat) rem(stk *stack, u, v nat) (r nat) { if alias(z, u) { z = nil } - qp := getNat(0) - q, r := qp.div(z, u, v) - *qp = q - putNat(qp) + defer stk.restore(stk.save()) + q := stk.nat(max(1, len(u)-(len(v)-1))) + _, r = q.div(stk, z, u, v) return r } // div returns q, r such that q = ⌊u/v⌋ and r = u%v = u - q·v. // It uses z and z2 as the storage for q and r. -func (z nat) div(z2, u, v nat) (q, r nat) { +// The caller may pass stk == nil to request that div obtain and release one itself. +func (z nat) div(stk *stack, z2, u, v nat) (q, r nat) { if len(v) == 0 { panic("division by zero") } - if u.cmp(v) < 0 { - q = z[:0] - r = z2.set(u) - return - } - if len(v) == 1 { // Short division: long optimized for a single-word divisor. // In that case, the 2-by-1 guess is all we need at each step. @@ -535,7 +529,18 @@ func (z nat) div(z2, u, v nat) (q, r nat) { return } - q, r = z.divLarge(z2, u, v) + if u.cmp(v) < 0 { + q = z[:0] + r = z2.set(u) + return + } + + if stk == nil { + stk = getStack() + defer stk.free() + } + + q, r = z.divLarge(stk, z2, u, v) return } @@ -589,7 +594,7 @@ func divWVW(z []Word, xn Word, x []Word, y Word) (r Word) { // It uses z and u as the storage for q and r. // The caller must ensure that len(vIn) ≥ 2 (use divW otherwise) // and that len(uIn) ≥ len(vIn) (the answer is 0, uIn otherwise). -func (z nat) divLarge(u, uIn, vIn nat) (q, r nat) { +func (z nat) divLarge(stk *stack, u, uIn, vIn nat) (q, r nat) { n := len(vIn) m := len(uIn) - n @@ -597,12 +602,18 @@ func (z nat) divLarge(u, uIn, vIn nat) (q, r nat) { // vIn is treated as a read-only input (it may be in use by another // goroutine), so we must make a copy. // uIn is copied to u. + defer stk.restore(stk.save()) shift := nlz(vIn[n-1]) - vp := getNat(n) - v := *vp - shlVU(v, vIn, shift) + v := stk.nat(n) u = u.make(len(uIn) + 1) - u[len(uIn)] = shlVU(u[0:len(uIn)], uIn, shift) + if shift == 0 { + copy(v, vIn) + copy(u[:len(uIn)], uIn) + u[len(uIn)] = 0 + } else { + lshVU(v, vIn, shift) + u[len(uIn)] = lshVU(u[:len(uIn)], uIn, shift) + } // The caller should not pass aliased z and u, since those are // the two different outputs, but correct just in case. @@ -613,16 +624,17 @@ func (z nat) divLarge(u, uIn, vIn nat) (q, r nat) { // Use basic or recursive long division depending on size. if n < divRecursiveThreshold { - q.divBasic(u, v) + q.divBasic(stk, u, v) } else { - q.divRecursive(u, v) + q.divRecursive(stk, u, v) } - putNat(vp) q = q.norm() // Undo scaling of remainder. - shrVU(u, u, shift) + if shift != 0 { + rshVU(u, u, shift) + } r = u.norm() return q, r @@ -631,26 +643,25 @@ func (z nat) divLarge(u, uIn, vIn nat) (q, r nat) { // divBasic implements long division as described above. // It overwrites q with ⌊u/v⌋ and overwrites u with the remainder r. // q must be large enough to hold ⌊u/v⌋. -func (q nat) divBasic(u, v nat) { +func (q nat) divBasic(stk *stack, u, v nat) { n := len(v) m := len(u) - n - qhatvp := getNat(n + 1) - qhatv := *qhatvp + defer stk.restore(stk.save()) + qhatv := stk.nat(n + 1) // Set up for divWW below, precomputing reciprocal argument. vn1 := v[n-1] rec := reciprocalWord(vn1) + // Invent a leading 0 for u, for the first iteration. + // Invariant: ujn == u[j+n] in each iteration. + ujn := Word(0) + // Compute each digit of quotient. for j := m; j >= 0; j-- { // Compute the 2-by-1 guess q̂. - // The first iteration must invent a leading 0 for u. qhat := Word(_M) - var ujn Word - if j+n < len(u) { - ujn = u[j+n] - } // ujn ≤ vn1, or else q̂ would be more than one digit. // For ujn == vn1, we set q̂ to the max digit M above. @@ -688,9 +699,9 @@ func (q nat) divBasic(u, v nat) { // Subtract q̂·v from the current section of u. // If it underflows, q̂·v > u, which we fix up // by decrementing q̂ and adding v back. - c := subVV(u[j:j+qhl], u[j:], qhatv) + c := subVV(u[j:j+qhl], u[j:j+qhl], qhatv[:qhl]) if c != 0 { - c := addVV(u[j:j+n], u[j:], v) + c := addVV(u[j:j+n], u[j:j+n], v) // If n == qhl, the carry from subVV and the carry from addVV // cancel out and don't affect u[j+n]. if n < qhl { @@ -699,6 +710,8 @@ func (q nat) divBasic(u, v nat) { qhat-- } + ujn = u[j+n-1] + // Save quotient digit. // Caller may know the top digit is zero and not leave room for it. if j == m && m == len(q) && qhat == 0 { @@ -706,8 +719,6 @@ func (q nat) divBasic(u, v nat) { } q[j] = qhat } - - putNat(qhatvp) } // greaterThan reports whether the two digit numbers x1 x2 > y1 y2. @@ -719,31 +730,16 @@ func greaterThan(x1, x2, y1, y2 Word) bool { // divRecursiveThreshold is the number of divisor digits // at which point divRecursive is faster than divBasic. -const divRecursiveThreshold = 100 +var divRecursiveThreshold = 40 // see calibrate_test.go // divRecursive implements recursive division as described above. // It overwrites z with ⌊u/v⌋ and overwrites u with the remainder r. // z must be large enough to hold ⌊u/v⌋. // This function is just for allocating and freeing temporaries // around divRecursiveStep, the real implementation. -func (z nat) divRecursive(u, v nat) { - // Recursion depth is (much) less than 2 log₂(len(v)). - // Allocate a slice of temporaries to be reused across recursion, - // plus one extra temporary not live across the recursion. - recDepth := 2 * bits.Len(uint(len(v))) - tmp := getNat(3 * len(v)) - temps := make([]*nat, recDepth) - +func (z nat) divRecursive(stk *stack, u, v nat) { clear(z) - z.divRecursiveStep(u, v, 0, tmp, temps) - - // Free temporaries. - for _, n := range temps { - if n != nil { - putNat(n) - } - } - putNat(tmp) + z.divRecursiveStep(stk, u, v, 0) } // divRecursiveStep is the actual implementation of recursive division. @@ -751,7 +747,7 @@ func (z nat) divRecursive(u, v nat) { // z must be large enough to hold ⌊u/v⌋. // It uses temps[depth] (allocating if needed) as a temporary live across // the recursive call. It also uses tmp, but not live across the recursion. -func (z nat) divRecursiveStep(u, v nat, depth int, tmp *nat, temps []*nat) { +func (z nat) divRecursiveStep(stk *stack, u, v nat, depth int) { // u is a subsection of the original and may have leading zeros. // TODO(rsc): The v = v.norm() is useless and should be removed. // We know (and require) that v's top digit is ≥ B/2. @@ -765,7 +761,7 @@ func (z nat) divRecursiveStep(u, v nat, depth int, tmp *nat, temps []*nat) { // Fall back to basic division if the problem is now small enough. n := len(v) if n < divRecursiveThreshold { - z.divBasic(u, v) + z.divBasic(stk, u, v) return } @@ -784,11 +780,8 @@ func (z nat) divRecursiveStep(u, v nat, depth int, tmp *nat, temps []*nat) { B := n / 2 // Allocate a nat for qhat below. - if temps[depth] == nil { - temps[depth] = getNat(n) // TODO(rsc): Can be just B+1. - } else { - *temps[depth] = temps[depth].make(B + 1) - } + defer stk.restore(stk.save()) + qhat0 := stk.nat(B + 1) // Compute each wide digit of the quotient. // @@ -815,9 +808,9 @@ func (z nat) divRecursiveStep(u, v nat, depth int, tmp *nat, temps []*nat) { uu := u[j-B:] // Compute the 2-by-1 guess q̂, leaving r̂ in uu[s:B+n]. - qhat := *temps[depth] + qhat := qhat0 clear(qhat) - qhat.divRecursiveStep(uu[s:B+n], v[s:], depth+1, tmp, temps) + qhat.divRecursiveStep(stk, uu[s:B+n], v[s:], depth+1) qhat = qhat.norm() // Extend to a 3-by-2 quotient and remainder. @@ -832,9 +825,10 @@ func (z nat) divRecursiveStep(u, v nat, depth int, tmp *nat, temps []*nat) { // q̂·vₙ₋₂ and decrementing q̂ until that product is ≤ u. // But we can do the subtraction directly, as in the comment above // and in long division, because we know that q̂ is wrong by at most one. - qhatv := tmp.make(3 * n) + mark := stk.save() + qhatv := stk.nat(3 * n) clear(qhatv) - qhatv = qhatv.mul(qhat, v[:s]) + qhatv = qhatv.mul(stk, qhat, v[:s]) for i := 0; i < 2; i++ { e := qhatv.cmp(uu.norm()) if e <= 0 { @@ -845,7 +839,7 @@ func (z nat) divRecursiveStep(u, v nat, depth int, tmp *nat, temps []*nat) { if len(qhatv) > s { subVW(qhatv[s:], qhatv[s:], c) } - addAt(uu[s:], v[s:], 0) + addTo(uu[s:], v[s:]) } if qhatv.cmp(uu.norm()) > 0 { panic("impossible") @@ -854,8 +848,9 @@ func (z nat) divRecursiveStep(u, v nat, depth int, tmp *nat, temps []*nat) { if c > 0 { subVW(uu[len(qhatv):], uu[len(qhatv):], c) } - addAt(z, qhat, j-B) + addTo(z[j-B:], qhat) j -= B + stk.restore(mark) } // TODO(rsc): Rewrite loop as described above and delete all this code. @@ -863,13 +858,13 @@ func (z nat) divRecursiveStep(u, v nat, depth int, tmp *nat, temps []*nat) { // Now u < (v< 0 { @@ -878,13 +873,13 @@ func (z nat) divRecursiveStep(u, v nat, depth int, tmp *nat, temps []*nat) { if len(qhatv) > s { subVW(qhatv[s:], qhatv[s:], c) } - addAt(u[s:], v[s:], 0) + addTo(u[s:], v[s:]) } } if qhatv.cmp(u.norm()) > 0 { panic("impossible") } - c := subVV(u[0:len(qhatv)], u[0:len(qhatv)], qhatv) + c := subVV(u[:len(qhatv)], u[:len(qhatv)], qhatv) if c > 0 { c = subVW(u[len(qhatv):], u[len(qhatv):], c) } @@ -893,5 +888,5 @@ func (z nat) divRecursiveStep(u, v nat, depth int, tmp *nat, temps []*nat) { } // Done! - addAt(z, qhat.norm(), 0) + addTo(z, qhat.norm()) } diff --git a/src/math/big/natmul.go b/src/math/big/natmul.go new file mode 100644 index 00000000000000..175ce7fcab4e0b --- /dev/null +++ b/src/math/big/natmul.go @@ -0,0 +1,355 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Multiplication. + +package big + +// Operands that are shorter than karatsubaThreshold are multiplied using +// "grade school" multiplication; for longer operands the Karatsuba algorithm +// is used. +var karatsubaThreshold = 40 // see calibrate_test.go + +// mul sets z = x*y, using stk for temporary storage. +// The caller may pass stk == nil to request that mul obtain and release one itself. +func (z nat) mul(stk *stack, x, y nat) nat { + m := len(x) + n := len(y) + + switch { + case m < n: + return z.mul(stk, y, x) + case m == 0 || n == 0: + return z[:0] + case n == 1: + return z.mulAddWW(x, y[0], 0) + } + // m >= n > 1 + + // determine if z can be reused + if alias(z, x) || alias(z, y) { + z = nil // z is an alias for x or y - cannot reuse + } + z = z.make(m + n) + + // use basic multiplication if the numbers are small + if n < karatsubaThreshold { + basicMul(z, x, y) + return z.norm() + } + + if stk == nil { + stk = getStack() + defer stk.free() + } + + // Let x = x1:x0 where x0 is the same length as y. + // Compute z = x0*y and then add in x1*y in sections + // if needed. + karatsuba(stk, z[:2*n], x[:n], y) + + if n < m { + clear(z[2*n:]) + defer stk.restore(stk.save()) + t := stk.nat(2 * n) + for i := n; i < m; i += n { + t = t.mul(stk, x[i:min(i+n, len(x))], y) + addTo(z[i:], t) + } + } + + return z.norm() +} + +// Operands that are shorter than basicSqrThreshold are squared using +// "grade school" multiplication; for operands longer than karatsubaSqrThreshold +// we use the Karatsuba algorithm optimized for x == y. +var basicSqrThreshold = 12 // see calibrate_test.go +var karatsubaSqrThreshold = 80 // see calibrate_test.go + +// sqr sets z = x*x, using stk for temporary storage. +// The caller may pass stk == nil to request that sqr obtain and release one itself. +func (z nat) sqr(stk *stack, x nat) nat { + n := len(x) + switch { + case n == 0: + return z[:0] + case n == 1: + d := x[0] + z = z.make(2) + z[1], z[0] = mulWW(d, d) + return z.norm() + } + + if alias(z, x) { + z = nil // z is an alias for x - cannot reuse + } + z = z.make(2 * n) + + if n < basicSqrThreshold && n < karatsubaSqrThreshold { + basicMul(z, x, x) + return z.norm() + } + + if stk == nil { + stk = getStack() + defer stk.free() + } + + if n < karatsubaSqrThreshold { + basicSqr(stk, z, x) + return z.norm() + } + + karatsubaSqr(stk, z, x) + return z.norm() +} + +// basicSqr sets z = x*x and is asymptotically faster than basicMul +// by about a factor of 2, but slower for small arguments due to overhead. +// Requirements: len(x) > 0, len(z) == 2*len(x) +// The (non-normalized) result is placed in z. +func basicSqr(stk *stack, z, x nat) { + n := len(x) + if n < basicSqrThreshold { + basicMul(z, x, x) + return + } + + defer stk.restore(stk.save()) + t := stk.nat(2 * n) + clear(t) + z[1], z[0] = mulWW(x[0], x[0]) // the initial square + for i := 1; i < n; i++ { + d := x[i] + // z collects the squares x[i] * x[i] + z[2*i+1], z[2*i] = mulWW(d, d) + // t collects the products x[i] * x[j] where j < i + t[2*i] = addMulVVWW(t[i:2*i], t[i:2*i], x[0:i], d, 0) + } + t[2*n-1] = lshVU(t[1:2*n-1], t[1:2*n-1], 1) // double the j < i products + addVV(z, z, t) // combine the result +} + +// mulAddWW returns z = x*y + r. +func (z nat) mulAddWW(x nat, y, r Word) nat { + m := len(x) + if m == 0 || y == 0 { + return z.setWord(r) // result is r + } + // m > 0 + + z = z.make(m + 1) + z[m] = mulAddVWW(z[0:m], x, y, r) + + return z.norm() +} + +// basicMul multiplies x and y and leaves the result in z. +// The (non-normalized) result is placed in z[0 : len(x) + len(y)]. +func basicMul(z, x, y nat) { + clear(z[0 : len(x)+len(y)]) // initialize z + for i, d := range y { + if d != 0 { + z[len(x)+i] = addMulVVWW(z[i:i+len(x)], z[i:i+len(x)], x, d, 0) + } + } +} + +// karatsuba multiplies x and y, +// writing the (non-normalized) result to z. +// x and y must have the same length n, +// and z must have length twice that. +func karatsuba(stk *stack, z, x, y nat) { + n := len(y) + if len(x) != n || len(z) != 2*n { + panic("bad karatsuba length") + } + + // Fall back to basic algorithm if small enough. + if n < karatsubaThreshold || n < 2 { + basicMul(z, x, y) + return + } + + // Let the notation x1:x0 denote the nat (x1< D { + s, t = s[:len(s)-D], s[len(s)-D:]+"_"+t + } + return neg + s + t +} + +// trace prints a single debug value. +func trace(name string, x *Int) { + print(name, "=", ifmt(x), "\n") +} diff --git a/src/math/big/prime.go b/src/math/big/prime.go index 26688bbd64e9f2..1739b03e93131c 100644 --- a/src/math/big/prime.go +++ b/src/math/big/prime.go @@ -75,7 +75,9 @@ func (x *Int) ProbablyPrime(n int) bool { return false } - return x.abs.probablyPrimeMillerRabin(n+1, true) && x.abs.probablyPrimeLucas() + stk := getStack() + defer stk.free() + return x.abs.probablyPrimeMillerRabin(stk, n+1, true) && x.abs.probablyPrimeLucas(stk) } // probablyPrimeMillerRabin reports whether n passes reps rounds of the @@ -83,11 +85,11 @@ func (x *Int) ProbablyPrime(n int) bool { // If force2 is true, one of the rounds is forced to use base 2. // See Handbook of Applied Cryptography, p. 139, Algorithm 4.24. // The number n is known to be non-zero. -func (n nat) probablyPrimeMillerRabin(reps int, force2 bool) bool { +func (n nat) probablyPrimeMillerRabin(stk *stack, reps int, force2 bool) bool { nm1 := nat(nil).sub(n, natOne) // determine q, k such that nm1 = q << k k := nm1.trailingZeroBits() - q := nat(nil).shr(nm1, k) + q := nat(nil).rsh(nm1, k) nm3 := nat(nil).sub(nm1, natTwo) rand := rand.New(rand.NewSource(int64(n[0]))) @@ -103,13 +105,13 @@ NextRandom: x = x.random(rand, nm3, nm3Len) x = x.add(x, natTwo) } - y = y.expNN(x, q, n, false) + y = y.expNN(stk, x, q, n, false) if y.cmp(natOne) == 0 || y.cmp(nm1) == 0 { continue } for j := uint(1); j < k; j++ { - y = y.sqr(y) - quotient, y = quotient.div(y, y, n) + y = y.sqr(stk, y) + quotient, y = quotient.div(stk, y, y, n) if y.cmp(nm1) == 0 { continue NextRandom } @@ -147,7 +149,7 @@ NextRandom: // // Crandall and Pomerance, Prime Numbers: A Computational Perspective, 2nd ed. // Springer, 2005. -func (n nat) probablyPrimeLucas() bool { +func (n nat) probablyPrimeLucas(stk *stack) bool { // Discard 0, 1. if len(n) == 0 || n.cmp(natOne) == 0 { return false @@ -193,8 +195,8 @@ func (n nat) probablyPrimeLucas() bool { // We'll never find (d/n) = -1 if n is a square. // If n is a non-square we expect to find a d in just a few attempts on average. // After 40 attempts, take a moment to check if n is indeed a square. - t1 = t1.sqrt(n) - t1 = t1.sqr(t1) + t1 = t1.sqrt(stk, n) + t1 = t1.sqr(stk, t1) if t1.cmp(n) == 0 { return false } @@ -215,7 +217,7 @@ func (n nat) probablyPrimeLucas() bool { // Arrange s = (n - Jacobi(Δ, n)) / 2^r = (n+1) / 2^r. s := nat(nil).add(n, natOne) r := int(s.trailingZeroBits()) - s = s.shr(s, uint(r)) + s = s.rsh(s, uint(r)) nm2 := nat(nil).sub(n, natTwo) // n-2 // We apply the "almost extra strong" test, which checks the above conditions @@ -254,25 +256,25 @@ func (n nat) probablyPrimeLucas() bool { if s.bit(uint(i)) != 0 { // k' = 2k+1 // V(k') = V(2k+1) = V(k) V(k+1) - P. - t1 = t1.mul(vk, vk1) + t1 = t1.mul(stk, vk, vk1) t1 = t1.add(t1, n) t1 = t1.sub(t1, natP) - t2, vk = t2.div(vk, t1, n) + t2, vk = t2.div(stk, vk, t1, n) // V(k'+1) = V(2k+2) = V(k+1)² - 2. - t1 = t1.sqr(vk1) + t1 = t1.sqr(stk, vk1) t1 = t1.add(t1, nm2) - t2, vk1 = t2.div(vk1, t1, n) + t2, vk1 = t2.div(stk, vk1, t1, n) } else { // k' = 2k // V(k'+1) = V(2k+1) = V(k) V(k+1) - P. - t1 = t1.mul(vk, vk1) + t1 = t1.mul(stk, vk, vk1) t1 = t1.add(t1, n) t1 = t1.sub(t1, natP) - t2, vk1 = t2.div(vk1, t1, n) + t2, vk1 = t2.div(stk, vk1, t1, n) // V(k') = V(2k) = V(k)² - 2 - t1 = t1.sqr(vk) + t1 = t1.sqr(stk, vk) t1 = t1.add(t1, nm2) - t2, vk = t2.div(vk, t1, n) + t2, vk = t2.div(stk, vk, t1, n) } } @@ -285,8 +287,8 @@ func (n nat) probablyPrimeLucas() bool { // // Since we are checking for U(k) == 0 it suffices to check 2 V(k+1) == P V(k) mod n, // or P V(k) - 2 V(k+1) == 0 mod n. - t1 := t1.mul(vk, natP) - t2 := t2.shl(vk1, 1) + t1 := t1.mul(stk, vk, natP) + t2 := t2.lsh(vk1, 1) if t1.cmp(t2) < 0 { t1, t2 = t2, t1 } @@ -294,7 +296,7 @@ func (n nat) probablyPrimeLucas() bool { t3 := vk1 // steal vk1, no longer needed below vk1 = nil _ = vk1 - t2, t3 = t2.div(t3, t1, n) + t2, t3 = t2.div(stk, t3, t1, n) if len(t3) == 0 { return true } @@ -312,9 +314,9 @@ func (n nat) probablyPrimeLucas() bool { } // k' = 2k // V(k') = V(2k) = V(k)² - 2 - t1 = t1.sqr(vk) + t1 = t1.sqr(stk, vk) t1 = t1.sub(t1, natTwo) - t2, vk = t2.div(vk, t1, n) + t2, vk = t2.div(stk, vk, t1, n) } return false } diff --git a/src/math/big/prime_test.go b/src/math/big/prime_test.go index 8596e33a13b864..2b1995bcb224ec 100644 --- a/src/math/big/prime_test.go +++ b/src/math/big/prime_test.go @@ -159,6 +159,9 @@ func TestProbablyPrime(t *testing.T) { } func BenchmarkProbablyPrime(b *testing.B) { + stk := getStack() + defer stk.free() + p, _ := new(Int).SetString("203956878356401977405765866929034577280193993314348263094772646453283062722701277632936616063144088173312372882677123879538709400158306567338328279154499698366071906766440037074217117805690872792848149112022286332144876183376326512083574821647933992961249917319836219304274280243803104015000563790123", 10) for _, n := range []int{0, 1, 5, 10, 20} { b.Run(fmt.Sprintf("n=%d", n), func(b *testing.B) { @@ -170,26 +173,32 @@ func BenchmarkProbablyPrime(b *testing.B) { b.Run("Lucas", func(b *testing.B) { for i := 0; i < b.N; i++ { - p.abs.probablyPrimeLucas() + p.abs.probablyPrimeLucas(stk) } }) b.Run("MillerRabinBase2", func(b *testing.B) { for i := 0; i < b.N; i++ { - p.abs.probablyPrimeMillerRabin(1, true) + p.abs.probablyPrimeMillerRabin(stk, 1, true) } }) } func TestMillerRabinPseudoprimes(t *testing.T) { + stk := getStack() + defer stk.free() + testPseudoprimes(t, "probablyPrimeMillerRabin", - func(n nat) bool { return n.probablyPrimeMillerRabin(1, true) && !n.probablyPrimeLucas() }, + func(n nat) bool { return n.probablyPrimeMillerRabin(stk, 1, true) && !n.probablyPrimeLucas(stk) }, // https://oeis.org/A001262 []int{2047, 3277, 4033, 4681, 8321, 15841, 29341, 42799, 49141, 52633, 65281, 74665, 80581, 85489, 88357, 90751}) } func TestLucasPseudoprimes(t *testing.T) { + stk := getStack() + defer stk.free() + testPseudoprimes(t, "probablyPrimeLucas", - func(n nat) bool { return n.probablyPrimeLucas() && !n.probablyPrimeMillerRabin(1, true) }, + func(n nat) bool { return n.probablyPrimeLucas(stk) && !n.probablyPrimeMillerRabin(stk, 1, true) }, // https://oeis.org/A217719 []int{989, 3239, 5777, 10877, 27971, 29681, 30739, 31631, 39059, 72389, 73919, 75077}) } diff --git a/src/math/big/rat.go b/src/math/big/rat.go index e58433ecea333c..c7f79a56661b12 100644 --- a/src/math/big/rat.go +++ b/src/math/big/rat.go @@ -74,7 +74,7 @@ func (z *Rat) SetFloat64(f float64) *Rat { // nearest to the quotient a/b, using round-to-even in // halfway cases. It does not mutate its arguments. // Preconditions: b is non-zero; a and b have no common factors. -func quotToFloat32(a, b nat) (f float32, exact bool) { +func quotToFloat32(stk *stack, a, b nat) (f float32, exact bool) { const ( // float size in bits Fsize = 32 @@ -112,16 +112,16 @@ func quotToFloat32(a, b nat) (f float32, exact bool) { a2 = a2.set(a) b2 = b2.set(b) if shift := Msize2 - exp; shift > 0 { - a2 = a2.shl(a2, uint(shift)) + a2 = a2.lsh(a2, uint(shift)) } else if shift < 0 { - b2 = b2.shl(b2, uint(-shift)) + b2 = b2.lsh(b2, uint(-shift)) } // 2. Compute quotient and remainder (q, r). NB: due to the // extra shift, the low-order bit of q is logically the // high-order bit of r. var q nat - q, r := q.div(a2, a2, b2) // (recycle a2) + q, r := q.div(stk, a2, a2, b2) // (recycle a2) mantissa := low32(q) haveRem := len(r) > 0 // mantissa&1 && !haveRem => remainder is exactly half @@ -172,7 +172,7 @@ func quotToFloat32(a, b nat) (f float32, exact bool) { // nearest to the quotient a/b, using round-to-even in // halfway cases. It does not mutate its arguments. // Preconditions: b is non-zero; a and b have no common factors. -func quotToFloat64(a, b nat) (f float64, exact bool) { +func quotToFloat64(stk *stack, a, b nat) (f float64, exact bool) { const ( // float size in bits Fsize = 64 @@ -210,16 +210,16 @@ func quotToFloat64(a, b nat) (f float64, exact bool) { a2 = a2.set(a) b2 = b2.set(b) if shift := Msize2 - exp; shift > 0 { - a2 = a2.shl(a2, uint(shift)) + a2 = a2.lsh(a2, uint(shift)) } else if shift < 0 { - b2 = b2.shl(b2, uint(-shift)) + b2 = b2.lsh(b2, uint(-shift)) } // 2. Compute quotient and remainder (q, r). NB: due to the // extra shift, the low-order bit of q is logically the // high-order bit of r. var q nat - q, r := q.div(a2, a2, b2) // (recycle a2) + q, r := q.div(stk, a2, a2, b2) // (recycle a2) mantissa := low64(q) haveRem := len(r) > 0 // mantissa&1 && !haveRem => remainder is exactly half @@ -275,7 +275,9 @@ func (x *Rat) Float32() (f float32, exact bool) { if len(b) == 0 { b = natOne } - f, exact = quotToFloat32(x.a.abs, b) + stk := getStack() + defer stk.free() + f, exact = quotToFloat32(stk, x.a.abs, b) if x.a.neg { f = -f } @@ -291,7 +293,9 @@ func (x *Rat) Float64() (f float64, exact bool) { if len(b) == 0 { b = natOne } - f, exact = quotToFloat64(x.a.abs, b) + stk := getStack() + defer stk.free() + f, exact = quotToFloat64(stk, x.a.abs, b) if x.a.neg { f = -f } @@ -437,12 +441,14 @@ func (z *Rat) norm() *Rat { z.b.abs = z.b.abs.setWord(1) default: // z is fraction; normalize numerator and denominator + stk := getStack() + defer stk.free() neg := z.a.neg z.a.neg = false z.b.neg = false if f := NewInt(0).lehmerGCD(nil, nil, &z.a, &z.b); f.Cmp(intOne) != 0 { - z.a.abs, _ = z.a.abs.div(nil, z.a.abs, f.abs) - z.b.abs, _ = z.b.abs.div(nil, z.b.abs, f.abs) + z.a.abs, _ = z.a.abs.div(stk, nil, z.a.abs, f.abs) + z.b.abs, _ = z.b.abs.div(stk, nil, z.b.abs, f.abs) } z.a.neg = neg } @@ -452,7 +458,7 @@ func (z *Rat) norm() *Rat { // mulDenom sets z to the denominator product x*y (by taking into // account that 0 values for x or y must be interpreted as 1) and // returns z. -func mulDenom(z, x, y nat) nat { +func mulDenom(stk *stack, z, x, y nat) nat { switch { case len(x) == 0 && len(y) == 0: return z.setWord(1) @@ -461,17 +467,17 @@ func mulDenom(z, x, y nat) nat { case len(y) == 0: return z.set(x) } - return z.mul(x, y) + return z.mul(stk, x, y) } // scaleDenom sets z to the product x*f. // If f == 0 (zero value of denominator), z is set to (a copy of) x. -func (z *Int) scaleDenom(x *Int, f nat) { +func (z *Int) scaleDenom(stk *stack, x *Int, f nat) { if len(f) == 0 { z.Set(x) return } - z.abs = z.abs.mul(x.abs, f) + z.abs = z.abs.mul(stk, x.abs, f) z.neg = x.neg } @@ -481,58 +487,73 @@ func (z *Int) scaleDenom(x *Int, f nat) { // - +1 if x > y. func (x *Rat) Cmp(y *Rat) int { var a, b Int - a.scaleDenom(&x.a, y.b.abs) - b.scaleDenom(&y.a, x.b.abs) + stk := getStack() + defer stk.free() + a.scaleDenom(stk, &x.a, y.b.abs) + b.scaleDenom(stk, &y.a, x.b.abs) return a.Cmp(&b) } // Add sets z to the sum x+y and returns z. func (z *Rat) Add(x, y *Rat) *Rat { + stk := getStack() + defer stk.free() + var a1, a2 Int - a1.scaleDenom(&x.a, y.b.abs) - a2.scaleDenom(&y.a, x.b.abs) + a1.scaleDenom(stk, &x.a, y.b.abs) + a2.scaleDenom(stk, &y.a, x.b.abs) z.a.Add(&a1, &a2) - z.b.abs = mulDenom(z.b.abs, x.b.abs, y.b.abs) + z.b.abs = mulDenom(stk, z.b.abs, x.b.abs, y.b.abs) return z.norm() } // Sub sets z to the difference x-y and returns z. func (z *Rat) Sub(x, y *Rat) *Rat { + stk := getStack() + defer stk.free() + var a1, a2 Int - a1.scaleDenom(&x.a, y.b.abs) - a2.scaleDenom(&y.a, x.b.abs) + a1.scaleDenom(stk, &x.a, y.b.abs) + a2.scaleDenom(stk, &y.a, x.b.abs) z.a.Sub(&a1, &a2) - z.b.abs = mulDenom(z.b.abs, x.b.abs, y.b.abs) + z.b.abs = mulDenom(stk, z.b.abs, x.b.abs, y.b.abs) return z.norm() } // Mul sets z to the product x*y and returns z. func (z *Rat) Mul(x, y *Rat) *Rat { + stk := getStack() + defer stk.free() + if x == y { // a squared Rat is positive and can't be reduced (no need to call norm()) z.a.neg = false - z.a.abs = z.a.abs.sqr(x.a.abs) + z.a.abs = z.a.abs.sqr(stk, x.a.abs) if len(x.b.abs) == 0 { z.b.abs = z.b.abs.setWord(1) } else { - z.b.abs = z.b.abs.sqr(x.b.abs) + z.b.abs = z.b.abs.sqr(stk, x.b.abs) } return z } - z.a.Mul(&x.a, &y.a) - z.b.abs = mulDenom(z.b.abs, x.b.abs, y.b.abs) + + z.a.mul(stk, &x.a, &y.a) + z.b.abs = mulDenom(stk, z.b.abs, x.b.abs, y.b.abs) return z.norm() } // Quo sets z to the quotient x/y and returns z. // If y == 0, Quo panics. func (z *Rat) Quo(x, y *Rat) *Rat { + stk := getStack() + defer stk.free() + if len(y.a.abs) == 0 { panic("division by zero") } var a, b Int - a.scaleDenom(&x.a, y.b.abs) - b.scaleDenom(&y.a, x.b.abs) + a.scaleDenom(stk, &x.a, y.b.abs) + b.scaleDenom(stk, &y.a, x.b.abs) z.a.abs = a.abs z.b.abs = b.abs z.a.neg = a.neg != b.neg diff --git a/src/math/big/ratconv.go b/src/math/big/ratconv.go index dd99aecdc04af0..229f31b8f0eef4 100644 --- a/src/math/big/ratconv.go +++ b/src/math/big/ratconv.go @@ -163,6 +163,9 @@ func (z *Rat) SetString(s string) (*Rat, bool) { } // exp consumed - not needed anymore + stk := getStack() + defer stk.free() + // apply exp5 contributions // (start with exp5 so the numbers to multiply are smaller) if exp5 != 0 { @@ -178,9 +181,9 @@ func (z *Rat) SetString(s string) (*Rat, bool) { if n > 1e6 { return nil, false // avoid excessively large exponents } - pow5 := z.b.abs.expNN(natFive, nat(nil).setWord(Word(n)), nil, false) // use underlying array of z.b.abs + pow5 := z.b.abs.expNN(stk, natFive, nat(nil).setWord(Word(n)), nil, false) // use underlying array of z.b.abs if exp5 > 0 { - z.a.abs = z.a.abs.mul(z.a.abs, pow5) + z.a.abs = z.a.abs.mul(stk, z.a.abs, pow5) z.b.abs = z.b.abs.setWord(1) } else { z.b.abs = pow5 @@ -194,9 +197,9 @@ func (z *Rat) SetString(s string) (*Rat, bool) { return nil, false // avoid excessively large exponents } if exp2 > 0 { - z.a.abs = z.a.abs.shl(z.a.abs, uint(exp2)) + z.a.abs = z.a.abs.lsh(z.a.abs, uint(exp2)) } else if exp2 < 0 { - z.b.abs = z.b.abs.shl(z.b.abs, uint(-exp2)) + z.b.abs = z.b.abs.lsh(z.b.abs, uint(-exp2)) } z.a.neg = neg && len(z.a.abs) > 0 // 0 has no sign @@ -299,12 +302,13 @@ func scanExponent(r io.ByteScanner, base2ok, sepOk bool) (exp int64, base int, e // String returns a string representation of x in the form "a/b" (even if b == 1). func (x *Rat) String() string { - return string(x.marshal()) + return string(x.marshal(nil)) } -// marshal implements String returning a slice of bytes -func (x *Rat) marshal() []byte { - var buf []byte +// marshal implements [Rat.String] returning a slice of bytes. +// It appends the string representation of x in the form "a/b" (even if b == 1) to buf, +// and returns the extended buffer. +func (x *Rat) marshal(buf []byte) []byte { buf = x.a.Append(buf, 10) buf = append(buf, '/') if len(x.b.abs) != 0 { @@ -342,15 +346,17 @@ func (x *Rat) FloatString(prec int) string { } // x.b.abs != 0 - q, r := nat(nil).div(nat(nil), x.a.abs, x.b.abs) + stk := getStack() + defer stk.free() + q, r := nat(nil).div(stk, nat(nil), x.a.abs, x.b.abs) p := natOne if prec > 0 { - p = nat(nil).expNN(natTen, nat(nil).setUint64(uint64(prec)), nil, false) + p = nat(nil).expNN(stk, natTen, nat(nil).setUint64(uint64(prec)), nil, false) } - r = r.mul(r, p) - r, r2 := r.div(nat(nil), r, x.b.abs) + r = r.mul(stk, r, p) + r, r2 := r.div(stk, nat(nil), r, x.b.abs) // see if we need to round up r2 = r2.add(r2, r2) @@ -397,6 +403,9 @@ func (x *Rat) FloatString(prec int) string { // 1/4 2 true 0.25 // 1/6 1 false 0.2 (0.166... rounded) func (x *Rat) FloatPrec() (n int, exact bool) { + stk := getStack() + defer stk.free() + // Determine q and largest p2, p5 such that d = q·2^p2·5^p5. // The results n, exact are: // @@ -412,7 +421,7 @@ func (x *Rat) FloatPrec() (n int, exact bool) { // Do this first to reduce q as much as possible. var q nat p2 := d.trailingZeroBits() - q = q.shr(d, p2) + q = q.rsh(d, p2) // Determine p5 by counting factors of 5. // Build a table starting with an initial power of 5, @@ -424,11 +433,11 @@ func (x *Rat) FloatPrec() (n int, exact bool) { f := nat{1220703125} // == 5^fp (must fit into a uint32 Word) var t, r nat // temporaries for { - if _, r = t.div(r, q, f); len(r) != 0 { + if _, r = t.div(stk, r, q, f); len(r) != 0 { break // f doesn't divide q evenly } tab = append(tab, f) - f = nat(nil).sqr(f) // nat(nil) to ensure a new f for each table entry + f = nat(nil).sqr(stk, f) // nat(nil) to ensure a new f for each table entry } // Factor q using the table entries, if any. @@ -440,7 +449,7 @@ func (x *Rat) FloatPrec() (n int, exact bool) { // The same reasoning applies to the subsequent factors. var p5 uint for i := len(tab) - 1; i >= 0; i-- { - if t, r = t.div(r, q, tab[i]); len(r) == 0 { + if t, r = t.div(stk, r, q, tab[i]); len(r) == 0 { p5 += fp * (1 << i) // tab[i] == 5^(fp·2^i) q = q.set(t) } @@ -448,7 +457,7 @@ func (x *Rat) FloatPrec() (n int, exact bool) { // If fp != 1, we may still have multiples of 5 left. for { - if t, r = t.div(r, q, natFive); len(r) != 0 { + if t, r = t.div(stk, r, q, natFive); len(r) != 0 { break } p5++ diff --git a/src/math/big/ratmarsh.go b/src/math/big/ratmarsh.go index 69628294531ff8..26ded1d9d286f3 100644 --- a/src/math/big/ratmarsh.go +++ b/src/math/big/ratmarsh.go @@ -29,7 +29,7 @@ func (x *Rat) GobEncode() ([]byte, error) { // this should never happen return nil, errors.New("Rat.GobEncode: numerator too large") } - byteorder.BePutUint32(buf[j-4:j], uint32(n)) + byteorder.BEPutUint32(buf[j-4:j], uint32(n)) j -= 1 + 4 b := ratGobVersion << 1 // make space for sign bit if x.a.neg { @@ -54,7 +54,7 @@ func (z *Rat) GobDecode(buf []byte) error { return fmt.Errorf("Rat.GobDecode: encoding version %d not supported", b>>1) } const j = 1 + 4 - ln := byteorder.BeUint32(buf[j-4 : j]) + ln := byteorder.BEUint32(buf[j-4 : j]) if uint64(ln) > math.MaxInt-j { return errors.New("Rat.GobDecode: invalid length") } @@ -68,12 +68,17 @@ func (z *Rat) GobDecode(buf []byte) error { return nil } -// MarshalText implements the [encoding.TextMarshaler] interface. -func (x *Rat) MarshalText() (text []byte, err error) { +// AppendText implements the [encoding.TextAppender] interface. +func (x *Rat) AppendText(b []byte) ([]byte, error) { if x.IsInt() { - return x.a.MarshalText() + return x.a.AppendText(b) } - return x.marshal(), nil + return x.marshal(b), nil +} + +// MarshalText implements the [encoding.TextMarshaler] interface. +func (x *Rat) MarshalText() (text []byte, err error) { + return x.AppendText(nil) } // UnmarshalText implements the [encoding.TextUnmarshaler] interface. diff --git a/src/math/big/ratmarsh_test.go b/src/math/big/ratmarsh_test.go index 15c933efa6d43b..7d139bcad8efa7 100644 --- a/src/math/big/ratmarsh_test.go +++ b/src/math/big/ratmarsh_test.go @@ -136,3 +136,26 @@ func TestRatGobDecodeShortBuffer(t *testing.T) { } } } + +func TestRatAppendText(t *testing.T) { + for _, num := range ratNums { + for _, denom := range ratDenoms { + var tx Rat + tx.SetString(num + "/" + denom) + buf := make([]byte, 4, 32) + b, err := tx.AppendText(buf) + if err != nil { + t.Errorf("marshaling of %s failed: %s", &tx, err) + continue + } + var rx Rat + if err := rx.UnmarshalText(b[4:]); err != nil { + t.Errorf("unmarshaling of %s failed: %s", &tx, err) + continue + } + if rx.Cmp(&tx) != 0 { + t.Errorf("AppendText of %s failed: got %s want %s", &tx, &rx, &tx) + } + } + } +} diff --git a/src/math/bits.go b/src/math/bits.go index c5cb93b15945d4..3716a411f4f1b8 100644 --- a/src/math/bits.go +++ b/src/math/bits.go @@ -48,7 +48,12 @@ func IsInf(f float64, sign int) bool { // To avoid the floating-point hardware, could use: // x := Float64bits(f); // return sign >= 0 && x == uvinf || sign <= 0 && x == uvneginf; - return sign >= 0 && f > MaxFloat64 || sign <= 0 && f < -MaxFloat64 + if sign == 0 { + f = Abs(f) + } else if sign < 0 { + f = -f + } + return f > MaxFloat64 } // normalize returns a normal number y and exponent exp diff --git a/src/math/bits/bits.go b/src/math/bits/bits.go index 235d63e85bc53d..76ed1d03fc416b 100644 --- a/src/math/bits/bits.go +++ b/src/math/bits/bits.go @@ -38,7 +38,7 @@ func LeadingZeros64(x uint64) int { return 64 - Len64(x) } // --- TrailingZeros --- -// See http://supertech.csail.mit.edu/papers/debruijn.pdf +// See http://keithandkatie.com/keith/papers/debruijn.html const deBruijn32 = 0x077CB531 var deBruijn32tab = [32]byte{ diff --git a/src/math/bits/bits_test.go b/src/math/bits/bits_test.go index 23b4539fcd6ca4..6f6e1c2202f42a 100644 --- a/src/math/bits/bits_test.go +++ b/src/math/bits/bits_test.go @@ -1109,7 +1109,7 @@ func TestDiv64PanicZero(t *testing.T) { } func TestRem32(t *testing.T) { - // Sanity check: for non-oveflowing dividends, the result is the + // Sanity check: for non-overflowing dividends, the result is the // same as the rem returned by Div32 hi, lo, y := uint32(510510), uint32(9699690), uint32(510510+1) // ensure hi < y for i := 0; i < 1000; i++ { @@ -1136,7 +1136,7 @@ func TestRem32Overflow(t *testing.T) { } func TestRem64(t *testing.T) { - // Sanity check: for non-oveflowing dividends, the result is the + // Sanity check: for non-overflowing dividends, the result is the // same as the rem returned by Div64 hi, lo, y := uint64(510510), uint64(9699690), uint64(510510+1) // ensure hi < y for i := 0; i < 1000; i++ { diff --git a/src/math/dim_asm.go b/src/math/dim_asm.go index f4adbd0ae5e11c..a1d23dd0962b07 100644 --- a/src/math/dim_asm.go +++ b/src/math/dim_asm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build amd64 || arm64 || riscv64 || s390x +//go:build amd64 || arm64 || loong64 || riscv64 || s390x package math diff --git a/src/math/dim_loong64.s b/src/math/dim_loong64.s new file mode 100644 index 00000000000000..1484bf76381123 --- /dev/null +++ b/src/math/dim_loong64.s @@ -0,0 +1,77 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +#define PosInf 0x7FF0000000000000 +#define NaN 0x7FF8000000000001 +#define NegInf 0xFFF0000000000000 + +TEXT ·archMax(SB),NOSPLIT,$0 + MOVD x+0(FP), F0 + MOVD y+8(FP), F1 + FCLASSD F0, F2 + FCLASSD F1, F3 + + // combine x and y categories together to judge + MOVV F2, R4 + MOVV F3, R5 + OR R5, R4 + + // +Inf special cases + AND $64, R4, R5 + BNE R5, isPosInf + + // NaN special cases + AND $2, R4, R5 + BNE R5, isMaxNaN + + // normal case + FMAXD F0, F1, F0 + MOVD F0, ret+16(FP) + RET + +isMaxNaN: + MOVV $NaN, R6 + MOVV R6, ret+16(FP) + RET + +isPosInf: + MOVV $PosInf, R6 + MOVV R6, ret+16(FP) + RET + +TEXT ·archMin(SB),NOSPLIT,$0 + MOVD x+0(FP), F0 + MOVD y+8(FP), F1 + FCLASSD F0, F2 + FCLASSD F1, F3 + + // combine x and y categories together to judge + MOVV F2, R4 + MOVV F3, R5 + OR R5, R4 + + // -Inf special cases + AND $4, R4, R5 + BNE R5, isNegInf + + // NaN special cases + AND $2, R4, R5 + BNE R5, isMinNaN + + // normal case + FMIND F0, F1, F0 + MOVD F0, ret+16(FP) + RET + +isMinNaN: + MOVV $NaN, R6 + MOVV R6, ret+16(FP) + RET + +isNegInf: + MOVV $NegInf, R6 + MOVV R6, ret+16(FP) + RET diff --git a/src/math/dim_noasm.go b/src/math/dim_noasm.go index 5b9e06fed33d03..6f4917b8e8247b 100644 --- a/src/math/dim_noasm.go +++ b/src/math/dim_noasm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !amd64 && !arm64 && !riscv64 && !s390x +//go:build !amd64 && !arm64 && !loong64 && !riscv64 && !s390x package math diff --git a/src/math/erfc_s390x.s b/src/math/erfc_s390x.s index 7e9d469cc6e0d2..e03a7749ff290c 100644 --- a/src/math/erfc_s390x.s +++ b/src/math/erfc_s390x.s @@ -355,7 +355,7 @@ L10: WFMADB V0, V5, V3, V5 BR L11 L35: - WORD $0xB3130010 //lcdbr %f1,%f0 + LCDBR F0, F1 BR L9 L36: FMOVD 304(R9), F3 @@ -436,7 +436,7 @@ L30: L20: FMOVD 88(R9), F0 WFMADB V7, V2, V0, V2 - WORD $0xB3130022 //lcdbr %f2,%f2 + LCDBR F2, F2 FMOVD F2, ret+8(FP) RET L13: diff --git a/src/math/exp.go b/src/math/exp.go index 050e0ee9d88239..029a4f8163698f 100644 --- a/src/math/exp.go +++ b/src/math/exp.go @@ -109,13 +109,11 @@ func exp(x float64) float64 { // special cases switch { - case IsNaN(x) || IsInf(x, 1): + case IsNaN(x): return x - case IsInf(x, -1): - return 0 - case x > Overflow: + case x > Overflow: // handles case where x is +∞ return Inf(1) - case x < Underflow: + case x < Underflow: // handles case where x is -∞ return 0 case -NearZero < x && x < NearZero: return 1 + x @@ -157,13 +155,11 @@ func exp2(x float64) float64 { // special cases switch { - case IsNaN(x) || IsInf(x, 1): + case IsNaN(x): return x - case IsInf(x, -1): - return 0 - case x > Overflow: + case x > Overflow: // handles case where x is +∞ return Inf(1) - case x < Underflow: + case x < Underflow: // handles case where x is -∞ return 0 } diff --git a/src/math/exp2_asm.go b/src/math/exp2_asm.go index c26b2c3fab67c7..1e7875937432e8 100644 --- a/src/math/exp2_asm.go +++ b/src/math/exp2_asm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build arm64 +//go:build arm64 || loong64 package math diff --git a/src/math/exp2_noasm.go b/src/math/exp2_noasm.go index c2b409329f1e1f..847138b622ce67 100644 --- a/src/math/exp2_noasm.go +++ b/src/math/exp2_noasm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !arm64 +//go:build !arm64 && !loong64 package math diff --git a/src/math/exp_asm.go b/src/math/exp_asm.go index 424442845bb07e..125529fca360a2 100644 --- a/src/math/exp_asm.go +++ b/src/math/exp_asm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build amd64 || arm64 || s390x +//go:build amd64 || arm64 || loong64 || s390x package math diff --git a/src/math/exp_loong64.s b/src/math/exp_loong64.s new file mode 100644 index 00000000000000..bf2823f888510b --- /dev/null +++ b/src/math/exp_loong64.s @@ -0,0 +1,236 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +#define NearZero 0x3e30000000000000 // 2**-28 +#define PosInf 0x7ff0000000000000 +#define FracMask 0x000fffffffffffff +#define C1 0x3cb0000000000000 // 2**-52 + +DATA exprodata<>+0(SB)/8, $0.0 +DATA exprodata<>+8(SB)/8, $0.5 +DATA exprodata<>+16(SB)/8, $1.0 +DATA exprodata<>+24(SB)/8, $2.0 +DATA exprodata<>+32(SB)/8, $6.93147180369123816490e-01 // Ln2Hi +DATA exprodata<>+40(SB)/8, $1.90821492927058770002e-10 // Ln2Lo +DATA exprodata<>+48(SB)/8, $1.44269504088896338700e+00 // Log2e +DATA exprodata<>+56(SB)/8, $7.09782712893383973096e+02 // Overflow +DATA exprodata<>+64(SB)/8, $-7.45133219101941108420e+02 // Underflow +DATA exprodata<>+72(SB)/8, $1.0239999999999999e+03 // Overflow2 +DATA exprodata<>+80(SB)/8, $-1.0740e+03 // Underflow2 +DATA exprodata<>+88(SB)/8, $3.7252902984619141e-09 // NearZero +GLOBL exprodata<>+0(SB), NOPTR|RODATA, $96 + +DATA expmultirodata<>+0(SB)/8, $1.66666666666666657415e-01 // P1 +DATA expmultirodata<>+8(SB)/8, $-2.77777777770155933842e-03 // P2 +DATA expmultirodata<>+16(SB)/8, $6.61375632143793436117e-05 // P3 +DATA expmultirodata<>+24(SB)/8, $-1.65339022054652515390e-06 // P4 +DATA expmultirodata<>+32(SB)/8, $4.13813679705723846039e-08 // P5 +GLOBL expmultirodata<>+0(SB), NOPTR|RODATA, $40 + +// Exp returns e**x, the base-e exponential of x. +// This is an assembly implementation of the method used for function Exp in file exp.go. +// +// func Exp(x float64) float64 +TEXT ·archExp(SB),$0-16 + MOVD x+0(FP), F0 // F0 = x + + MOVV $exprodata<>+0(SB), R10 + MOVD 56(R10), F1 // Overflow + MOVD 64(R10), F2 // Underflow + MOVD 88(R10), F3 // NearZero + MOVD 16(R10), F17 // 1.0 + + CMPEQD F0, F0, FCC0 + BFPF isNaN // x = NaN, return NaN + + CMPGTD F0, F1, FCC0 + BFPT overflow // x > Overflow, return PosInf + + CMPGTD F2, F0, FCC0 + BFPT underflow // x < Underflow, return 0 + + ABSD F0, F5 + CMPGTD F3, F5, FCC0 + BFPT nearzero // fabs(x) < NearZero, return 1 + x + + // argument reduction, x = k*ln2 + r, |r| <= 0.5*ln2 + // computed as r = hi - lo for extra precision. + MOVD 0(R10), F5 + MOVD 8(R10), F3 + MOVD 48(R10), F2 + CMPGTD F0, F5, FCC0 + BFPT add // x > 0 +sub: + FMSUBD F3, F2, F0, F3 // Log2e*x - 0.5 + JMP 2(PC) +add: + FMADDD F3, F2, F0, F3 // Log2e*x + 0.5 + + FTINTRZVD F3, F4 // float64 -> int64 + MOVV F4, R5 // R5 = int(k) + FFINTDV F4, F3 // int64 -> float64 + + MOVD 32(R10), F4 + MOVD 40(R10), F5 + FNMSUBD F0, F3, F4, F4 + MULD F3, F5, F5 + SUBD F5, F4, F6 + MULD F6, F6, F7 + + // compute c + MOVV $expmultirodata<>+0(SB), R11 + MOVD 32(R11), F8 + MOVD 24(R11), F9 + FMADDD F9, F8, F7, F13 + MOVD 16(R11), F10 + FMADDD F10, F13, F7, F13 + MOVD 8(R11), F11 + FMADDD F11, F13, F7, F13 + MOVD 0(R11), F12 + FMADDD F12, F13, F7, F13 + FNMSUBD F6, F13, F7, F13 + + // compute y + MOVD 24(R10), F14 + SUBD F13, F14, F14 + MULD F6, F13, F15 + DIVD F14, F15, F15 + SUBD F15, F5, F15 + SUBD F4, F15, F15 + SUBD F15, F17, F16 + + // inline Ldexp(y, k), benefit: + // 1, no parameter pass overhead. + // 2, skip unnecessary checks for Inf/NaN/Zero + MOVV F16, R4 + MOVV $FracMask, R9 + AND R9, R4, R6 // fraction + SRLV $52, R4, R7 // exponent + ADDV R5, R7 + MOVV $1, R12 + BGE R7, R12, normal + ADDV $52, R7 // denormal + MOVV $C1, R8 + MOVV R8, F17 +normal: + SLLV $52, R7 + OR R7, R6, R4 + MOVV R4, F0 + MULD F17, F0 // return m * x + MOVD F0, ret+8(FP) + RET +nearzero: + ADDD F17, F0, F0 +isNaN: + MOVD F0, ret+8(FP) + RET +underflow: + MOVV R0, ret+8(FP) + RET +overflow: + MOVV $PosInf, R4 + MOVV R4, ret+8(FP) + RET + + +// Exp2 returns 2**x, the base-2 exponential of x. +// This is an assembly implementation of the method used for function Exp2 in file exp.go. +// +// func Exp2(x float64) float64 +TEXT ·archExp2(SB),$0-16 + MOVD x+0(FP), F0 // F0 = x + + MOVV $exprodata<>+0(SB), R10 + MOVD 72(R10), F1 // Overflow2 + MOVD 80(R10), F2 // Underflow2 + MOVD 88(R10), F3 // NearZero + + CMPEQD F0, F0, FCC0 + BFPF isNaN // x = NaN, return NaN + + CMPGTD F0, F1, FCC0 + BFPT overflow // x > Overflow, return PosInf + + CMPGTD F2, F0, FCC0 + BFPT underflow // x < Underflow, return 0 + + // argument reduction; x = r*lg(e) + k with |r| <= ln(2)/2 + // computed as r = hi - lo for extra precision. + MOVD 0(R10), F10 + MOVD 8(R10), F2 + CMPGTD F0, F10, FCC0 + BFPT add +sub: + SUBD F2, F0, F3 // x - 0.5 + JMP 2(PC) +add: + ADDD F2, F0, F3 // x + 0.5 + + FTINTRZVD F3, F4 + MOVV F4, R5 + FFINTDV F4, F3 + + MOVD 32(R10), F4 + MOVD 40(R10), F5 + SUBD F3, F0, F3 + MULD F3, F4 + FNMSUBD F10, F3, F5, F5 + SUBD F5, F4, F6 + MULD F6, F6, F7 + + // compute c + MOVV $expmultirodata<>+0(SB), R11 + MOVD 32(R11), F8 + MOVD 24(R11), F9 + FMADDD F9, F8, F7, F13 + MOVD 16(R11), F10 + FMADDD F10, F13, F7, F13 + MOVD 8(R11), F11 + FMADDD F11, F13, F7, F13 + MOVD 0(R11), F12 + FMADDD F12, F13, F7, F13 + FNMSUBD F6, F13, F7, F13 + + // compute y + MOVD 24(R10), F14 + SUBD F13, F14, F14 + MULD F6, F13, F15 + DIVD F14, F15 + + MOVD 16(R10), F17 + SUBD F15, F5, F15 + SUBD F4, F15, F15 + SUBD F15, F17, F16 + + // inline Ldexp(y, k), benefit: + // 1, no parameter pass overhead. + // 2, skip unnecessary checks for Inf/NaN/Zero + MOVV F16, R4 + MOVV $FracMask, R9 + SRLV $52, R4, R7 // exponent + AND R9, R4, R6 // fraction + ADDV R5, R7 + MOVV $1, R12 + BGE R7, R12, normal + + ADDV $52, R7 // denormal + MOVV $C1, R8 + MOVV R8, F17 +normal: + SLLV $52, R7 + OR R7, R6, R4 + MOVV R4, F0 + MULD F17, F0 +isNaN: + MOVD F0, ret+8(FP) + RET +underflow: + MOVV R0, ret+8(FP) + RET +overflow: + MOVV $PosInf, R4 + MOVV R4, ret+8(FP) + RET diff --git a/src/math/exp_noasm.go b/src/math/exp_noasm.go index bd3f02412a28a2..bf5e84b736b26f 100644 --- a/src/math/exp_noasm.go +++ b/src/math/exp_noasm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !amd64 && !arm64 && !s390x +//go:build !amd64 && !arm64 && !loong64 && !s390x package math diff --git a/src/math/exp_s390x.s b/src/math/exp_s390x.s index e0ec8230738ad9..baf0e985a1e293 100644 --- a/src/math/exp_s390x.s +++ b/src/math/exp_s390x.s @@ -116,7 +116,7 @@ L16: FMOVD F0, ret+8(FP) RET L20: - WORD $0xB3130020 //lcdbr %f2,%f0 + LCDBR F0, F2 BR L2 L6: MOVD $·expxaddexp<>+0(SB), R1 diff --git a/src/math/expm1_s390x.s b/src/math/expm1_s390x.s index 16c861bb18b605..c193edc7cb8794 100644 --- a/src/math/expm1_s390x.s +++ b/src/math/expm1_s390x.s @@ -100,14 +100,14 @@ L2: FMOVD 16(R5), F6 WFMADB V0, V2, V6, V2 RISBGZ $57, $60, $3, R1, R3 - WORD $0xB3130022 //lcdbr %f2,%f2 + LCDBR F2, F2 MOVD $·expm1tab<>+0(SB), R2 WORD $0x68432000 //ld %f4,0(%r3,%r2) FMADD F4, F0, F0 SLD $48, R1, R2 WFMSDB V2, V0, V4, V0 LDGR R2, F4 - WORD $0xB3130000 //lcdbr %f0,%f0 + LCDBR F0, F0 FSUB F4, F6 WFMSDB V0, V4, V6, V0 FMOVD F0, ret+8(FP) @@ -126,7 +126,7 @@ L16: FMOVD F0, ret+8(FP) RET L20: - WORD $0xB3130020 //lcdbr %f2,%f0 + LCDBR F0, F2 BR L2 L6: MOVD $·expm1xaddexp<>+0(SB), R1 @@ -154,13 +154,13 @@ L6: FMOVD 16(R5), F6 FMADD F4, F1, F6 LGDR F5, R1 - WORD $0xB3130066 //lcdbr %f6,%f6 + LCDBR F6, F6 RISBGZ $57, $60, $3, R1, R3 WORD $0x68432000 //ld %f4,0(%r3,%r2) FMADD F4, F1, F1 MOVD $0x4086000000000000, R2 FMSUB F1, F6, F4 - WORD $0xB3130044 //lcdbr %f4,%f4 + LCDBR F4, F4 WFCHDBS V2, V0, V0 BEQ L21 ADDW $0xF000, R1 diff --git a/src/math/floor.go b/src/math/floor.go index cb5856424b4e76..20a764112478c8 100644 --- a/src/math/floor.go +++ b/src/math/floor.go @@ -66,11 +66,18 @@ func Trunc(x float64) float64 { } func trunc(x float64) float64 { - if x == 0 || IsNaN(x) || IsInf(x, 0) { - return x + if Abs(x) < 1 { + return Copysign(0, x) } - d, _ := Modf(x) - return d + + b := Float64bits(x) + e := uint(b>>shift)&mask - bias + + // Keep the top 12+e bits, the integer part; clear the rest. + if e < 64-12 { + b &^= 1<<(64-12-e) - 1 + } + return Float64frombits(b) } // Round returns the nearest integer, rounding half away from zero. diff --git a/src/math/floor_asm.go b/src/math/floor_asm.go index fb419d6da2f6bf..1b06b8def9c650 100644 --- a/src/math/floor_asm.go +++ b/src/math/floor_asm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build 386 || amd64 || arm64 || ppc64 || ppc64le || s390x || wasm +//go:build 386 || amd64 || arm64 || loong64 || ppc64 || ppc64le || riscv64 || s390x || wasm package math diff --git a/src/math/floor_loong64.s b/src/math/floor_loong64.s new file mode 100644 index 00000000000000..0df7deee60588d --- /dev/null +++ b/src/math/floor_loong64.s @@ -0,0 +1,41 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// +// derived from math/floor_riscv64.s + +#include "textflag.h" + +#define ROUNDFN(NAME, FUNC) \ +TEXT NAME(SB),NOSPLIT,$0; \ + MOVD x+0(FP), F0; \ + MOVV F0, R11; \ + /* 1023: bias of exponent, [-2^53, 2^53]: exactly integer represent range */; \ + MOVV $1023+53, R12; \ + /* Drop all fraction bits */; \ + SRLV $52, R11, R11; \ + /* Remove sign bit */; \ + AND $0x7FF, R11, R11; \ + BLTU R12, R11, isExtremum; \ +normal:; \ + FUNC F0, F2; \ + MOVV F2, R10; \ + BEQ R10, R0, is0; \ + FFINTDV F2, F0; \ +/* Return either input is +-Inf, NaN(0x7FF) or out of precision limitation */; \ +isExtremum:; \ + MOVD F0, ret+8(FP); \ + RET; \ +is0:; \ + FCOPYSGD F0, F2, F2; \ + MOVD F2, ret+8(FP); \ + RET + +// func archFloor(x float64) float64 +ROUNDFN(·archFloor, FTINTRMVD) + +// func archCeil(x float64) float64 +ROUNDFN(·archCeil, FTINTRPVD) + +// func archTrunc(x float64) float64 +ROUNDFN(·archTrunc, FTINTRZVD) diff --git a/src/math/floor_noasm.go b/src/math/floor_noasm.go index 5641c7ea0a49ca..34bd292f0e7751 100644 --- a/src/math/floor_noasm.go +++ b/src/math/floor_noasm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !386 && !amd64 && !arm64 && !ppc64 && !ppc64le && !s390x && !wasm +//go:build !386 && !amd64 && !arm64 && !loong64 && !ppc64 && !ppc64le && !riscv64 && !s390x && !wasm package math diff --git a/src/math/floor_riscv64.s b/src/math/floor_riscv64.s new file mode 100644 index 00000000000000..d9fe0ed8e2828a --- /dev/null +++ b/src/math/floor_riscv64.s @@ -0,0 +1,48 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// RISC-V offered floating-point (FP) rounding by FP conversion instructions (FCVT) +// with rounding mode field. +// As Go spec expects FP rounding result in FP, we have to use FCVT integer +// back to FP (fp -> int -> fp). +// RISC-V only set Inexact flag during invalid FP-integer conversion without changing any data, +// on the other hand, RISC-V sets out of integer represent range yet valid FP into NaN. +// When it comes to integer-FP conversion, invalid FP like NaN, +-Inf will be +// converted into the closest valid FP, for example: +// +// `Floor(-Inf) -> int64(0x7fffffffffffffff) -> float64(9.22e+18)` +// `Floor(18446744073709549568.0) -> int64(0x7fffffffffffffff) -> float64(9.22e+18)` +// +// This ISA conversion limitation requires we skip all invalid or out of range FP +// before any normal rounding operations. + +#define ROUNDFN(NAME, MODE) \ +TEXT NAME(SB),NOSPLIT,$0; \ + MOVD x+0(FP), F10; \ + FMVXD F10, X10; \ + /* Drop all fraction bits */;\ + SRL $52, X10, X12; \ + /* Remove sign bit */; \ + AND $0x7FF, X12, X12;\ + /* Return either input is +-Inf, NaN(0x7FF) or out of precision limitation */;\ + /* 1023: bias of exponent, [-2^53, 2^53]: exactly integer represent range */;\ + MOV $1023+53, X11; \ + BLTU X11, X12, 4(PC);\ + FCVTLD.MODE F10, X11; \ + FCVTDL X11, F11; \ + /* RISC-V rounds negative values to +0, restore original sign */;\ + FSGNJD F10, F11, F10; \ + MOVD F10, ret+8(FP); \ + RET + +// func archFloor(x float64) float64 +ROUNDFN(·archFloor, RDN) + +// func archCeil(x float64) float64 +ROUNDFN(·archCeil, RUP) + +// func archTrunc(x float64) float64 +ROUNDFN(·archTrunc, RTZ) diff --git a/src/math/fma.go b/src/math/fma.go index ba03fbe8a93b27..c806b914dab5b6 100644 --- a/src/math/fma.go +++ b/src/math/fma.go @@ -96,9 +96,16 @@ func FMA(x, y, z float64) float64 { bx, by, bz := Float64bits(x), Float64bits(y), Float64bits(z) // Inf or NaN or zero involved. At most one rounding will occur. - if x == 0.0 || y == 0.0 || z == 0.0 || bx&uvinf == uvinf || by&uvinf == uvinf { + if x == 0.0 || y == 0.0 || bx&uvinf == uvinf || by&uvinf == uvinf { return x*y + z } + // Handle z == 0.0 separately. + // Adding zero usually does not change the original value. + // However, there is an exception with negative zero. (e.g. (-0) + (+0) = (+0)) + // This applies when x * y is negative and underflows. + if z == 0.0 { + return x * y + } // Handle non-finite z separately. Evaluating x*y+z where // x and y are finite, but z is infinite, should always result in z. if bz&uvinf == uvinf { diff --git a/src/math/log1p_s390x.s b/src/math/log1p_s390x.s index 00eb374996391f..98fe82c9c39002 100644 --- a/src/math/log1p_s390x.s +++ b/src/math/log1p_s390x.s @@ -114,7 +114,7 @@ TEXT ·log1pAsm(SB), NOSPLIT, $0-16 MOVD $·log1pxzero<>+0(SB), R1 FMOVD 0(R1), F2 BVS LEXITTAGlog1p - WORD $0xB3130044 // lcdbr %f4,%f4 + LCDBR F4, F4 WFCEDBS V2, V4, V6 BEQ L9 WFCHDBS V4, V2, V2 @@ -129,7 +129,7 @@ L8: FSUB F4, F3 FMADD F2, F4, F1 MOVD $·log1pc4<>+0(SB), R2 - WORD $0xB3130041 // lcdbr %f4,%f1 + LCDBR F1, F4 FMOVD 0(R2), F7 FSUB F3, F0 MOVD $·log1pc3<>+0(SB), R2 diff --git a/src/math/log_s390x.s b/src/math/log_s390x.s index 4b514f3dd42eb1..215d7a03a7509f 100644 --- a/src/math/log_s390x.s +++ b/src/math/log_s390x.s @@ -142,7 +142,7 @@ L2: BYTE $0x10 BYTE $0x1F MOVD ·logxl2<>+0(SB), R1 - WORD $0xB3130001 //lcdbr %f0,%f1 + LCDBR F1, F0 LDGR R1, F4 WFMADB V0, V4, V2, V0 L1: diff --git a/src/math/modf.go b/src/math/modf.go index 613a75fc9a6864..12630958e969b7 100644 --- a/src/math/modf.go +++ b/src/math/modf.go @@ -11,33 +11,8 @@ package math // // Modf(±Inf) = ±Inf, NaN // Modf(NaN) = NaN, NaN -func Modf(f float64) (int float64, frac float64) { - if haveArchModf { - return archModf(f) - } - return modf(f) -} - -func modf(f float64) (int float64, frac float64) { - if f < 1 { - switch { - case f < 0: - int, frac = Modf(-f) - return -int, -frac - case f == 0: - return f, f // Return -0, -0 when f == -0 - } - return 0, f - } - - x := Float64bits(f) - e := uint(x>>shift)&mask - bias - - // Keep the top 12+e bits, the integer part; clear the rest. - if e < 64-12 { - x &^= 1<<(64-12-e) - 1 - } - int = Float64frombits(x) - frac = f - int +func Modf(f float64) (integer float64, fractional float64) { + integer = Trunc(f) + fractional = Copysign(f-integer, f) return } diff --git a/src/math/modf_arm64.s b/src/math/modf_arm64.s deleted file mode 100644 index 1e4a329a4be78e..00000000000000 --- a/src/math/modf_arm64.s +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "textflag.h" - -// func archModf(f float64) (int float64, frac float64) -TEXT ·archModf(SB),NOSPLIT,$0 - MOVD f+0(FP), R0 - FMOVD R0, F0 - FRINTZD F0, F1 - FMOVD F1, int+8(FP) - FSUBD F1, F0 - FMOVD F0, R1 - AND $(1<<63), R0 - ORR R0, R1 // must have same sign - MOVD R1, frac+16(FP) - RET diff --git a/src/math/modf_asm.go b/src/math/modf_asm.go deleted file mode 100644 index c63be6cf361a5f..00000000000000 --- a/src/math/modf_asm.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build arm64 || ppc64 || ppc64le - -package math - -const haveArchModf = true - -func archModf(f float64) (int float64, frac float64) diff --git a/src/math/modf_noasm.go b/src/math/modf_noasm.go deleted file mode 100644 index 55c6a7f6e20693..00000000000000 --- a/src/math/modf_noasm.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !arm64 && !ppc64 && !ppc64le - -package math - -const haveArchModf = false - -func archModf(f float64) (int float64, frac float64) { - panic("not implemented") -} diff --git a/src/math/modf_ppc64x.s b/src/math/modf_ppc64x.s deleted file mode 100644 index 410b523c0e815c..00000000000000 --- a/src/math/modf_ppc64x.s +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build ppc64 || ppc64le - -#include "textflag.h" - -// func archModf(f float64) (int float64, frac float64) -TEXT ·archModf(SB),NOSPLIT,$0 - FMOVD f+0(FP), F0 - FRIZ F0, F1 - FMOVD F1, int+8(FP) - FSUB F1, F0, F2 - FCPSGN F2, F0, F2 - FMOVD F2, frac+16(FP) - RET diff --git a/src/math/pow_s390x.s b/src/math/pow_s390x.s index c8758fc5f8598f..97cd48d96ec850 100644 --- a/src/math/pow_s390x.s +++ b/src/math/pow_s390x.s @@ -492,10 +492,10 @@ L9: FMOVD F2, F1 BR L1 L46: - WORD $0xB3130040 //lcdbr %f4,%f0 + LCDBR F0, F4 BR L3 L44: - WORD $0xB3130030 //lcdbr %f3,%f0 + LCDBR F0, F3 BR L7 L35: FMOVD F0, F1 @@ -590,7 +590,7 @@ L32: FMOVD 8(R9), F4 BR L17 L50: - WORD $0xB3130042 //lcdbr %f4,%f2 + LCDBR F2, F4 BR L14 xIsOne: // Pow(1, y) = 1 for any y yIsOne: // Pow(x, 1) = x for any x diff --git a/src/math/rand/default_test.go b/src/math/rand/default_test.go index 19fd75dfd1770a..25c24244c46a7b 100644 --- a/src/math/rand/default_test.go +++ b/src/math/rand/default_test.go @@ -34,14 +34,9 @@ func TestDefaultRace(t *testing.T) { t.Parallel() for i := 0; i < 6; i++ { - i := i t.Run(strconv.Itoa(i), func(t *testing.T) { t.Parallel() - exe, err := os.Executable() - if err != nil { - exe = os.Args[0] - } - cmd := testenv.Command(t, exe, "-test.run=TestDefaultRace") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^TestDefaultRace$") cmd = testenv.CleanCmdEnv(cmd) cmd.Env = append(cmd.Env, fmt.Sprintf("GO_RAND_TEST_HELPER_CODE=%d", i/2)) if i%2 != 0 { diff --git a/src/math/rand/rand.go b/src/math/rand/rand.go index 61ff5c1b387a7c..4be1ca208a86ce 100644 --- a/src/math/rand/rand.go +++ b/src/math/rand/rand.go @@ -313,6 +313,9 @@ var globalRandGenerator atomic.Pointer[Rand] var randautoseed = godebug.New("randautoseed") +// randseednop controls whether the global Seed is a no-op. +var randseednop = godebug.New("randseednop") + // globalRand returns the generator to use for the top-level convenience // functions. func globalRand() *Rand { @@ -391,7 +394,15 @@ func (fs *runtimeSource) read(p []byte, readVal *int64, readPos *int8) (n int, e // a random value. Programs that call Seed with a known value to get // a specific sequence of results should use New(NewSource(seed)) to // obtain a local random generator. +// +// As of Go 1.24 [Seed] is a no-op. To restore the previous behavior set +// GODEBUG=randseednop=0. func Seed(seed int64) { + if randseednop.Value() != "0" { + return + } + randseednop.IncNonDefault() + orig := globalRandGenerator.Load() // If we are already using a lockedSource, we can just re-seed it. diff --git a/src/math/rand/rand_test.go b/src/math/rand/rand_test.go index 7906f296743fe6..1e1fad79cec719 100644 --- a/src/math/rand/rand_test.go +++ b/src/math/rand/rand_test.go @@ -556,6 +556,42 @@ func TestUniformFactorial(t *testing.T) { } } +func TestSeedNop(t *testing.T) { + // If the global Seed takes effect, then resetting it to a certain value + // should provide predictable output to functions using it. + t.Run("randseednop=0", func(t *testing.T) { + t.Setenv("GODEBUG", "randseednop=0") + Seed(1) + before := Int63() + Seed(1) + after := Int63() + if before != after { + t.Fatal("global Seed should take effect") + } + }) + // If calls to the global Seed are no-op then functions using it should + // provide different output, even if it was reset to the same value. + t.Run("randseednop=1", func(t *testing.T) { + t.Setenv("GODEBUG", "randseednop=1") + Seed(1) + before := Int63() + Seed(1) + after := Int63() + if before == after { + t.Fatal("global Seed should be a no-op") + } + }) + t.Run("GODEBUG unset", func(t *testing.T) { + Seed(1) + before := Int63() + Seed(1) + after := Int63() + if before == after { + t.Fatal("global Seed should default to being a no-op") + } + }) +} + // Benchmarks func BenchmarkInt63Threadsafe(b *testing.B) { diff --git a/src/math/rand/v2/chacha8.go b/src/math/rand/v2/chacha8.go index f9eaacf6017f25..eb43e65915fe4c 100644 --- a/src/math/rand/v2/chacha8.go +++ b/src/math/rand/v2/chacha8.go @@ -58,19 +58,19 @@ func (c *ChaCha8) Read(p []byte) (n int, err error) { p = p[n:] } for len(p) >= 8 { - byteorder.LePutUint64(p, c.Uint64()) + byteorder.LEPutUint64(p, c.Uint64()) p = p[8:] n += 8 } if len(p) > 0 { - byteorder.LePutUint64(c.readBuf[:], c.Uint64()) + byteorder.LEPutUint64(c.readBuf[:], c.Uint64()) n += copy(p, c.readBuf[:]) c.readLen = 8 - len(p) } return } -// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. +// UnmarshalBinary implements the [encoding.BinaryUnmarshaler] interface. func (c *ChaCha8) UnmarshalBinary(data []byte) error { data, ok := cutPrefix(data, []byte("readbuf:")) if ok { @@ -98,13 +98,18 @@ func readUint8LengthPrefixed(b []byte) (buf, rest []byte, ok bool) { return b[1 : 1+b[0]], b[1+b[0]:], true } -// MarshalBinary implements the encoding.BinaryMarshaler interface. -func (c *ChaCha8) MarshalBinary() ([]byte, error) { +// AppendBinary implements the [encoding.BinaryAppender] interface. +func (c *ChaCha8) AppendBinary(b []byte) ([]byte, error) { if c.readLen > 0 { - out := []byte("readbuf:") - out = append(out, uint8(c.readLen)) - out = append(out, c.readBuf[len(c.readBuf)-c.readLen:]...) - return append(out, chacha8rand.Marshal(&c.state)...), nil + b = append(b, "readbuf:"...) + b = append(b, uint8(c.readLen)) + b = append(b, c.readBuf[len(c.readBuf)-c.readLen:]...) } - return chacha8rand.Marshal(&c.state), nil + return append(b, chacha8rand.Marshal(&c.state)...), nil +} + +// MarshalBinary implements the [encoding.BinaryMarshaler] interface. +func (c *ChaCha8) MarshalBinary() ([]byte, error) { + // the maximum length of (chacha8rand.Marshal + c.readBuf + "readbuf:") is 64 + return c.AppendBinary(make([]byte, 0, 64)) } diff --git a/src/math/rand/v2/chacha8_test.go b/src/math/rand/v2/chacha8_test.go index 50e83ea19a1a84..ba11b7cc45d466 100644 --- a/src/math/rand/v2/chacha8_test.go +++ b/src/math/rand/v2/chacha8_test.go @@ -98,6 +98,22 @@ func TestChaCha8Read(t *testing.T) { } } +func BenchmarkChaCha8MarshalBinary(b *testing.B) { + p := NewChaCha8(chacha8seed) + for range b.N { + p.MarshalBinary() + } +} + +func BenchmarkChaCha8MarshalBinaryRead(b *testing.B) { + p := NewChaCha8(chacha8seed) + buf := make([]byte, 1) + for range b.N { + p.MarshalBinary() + p.Read(buf) + } +} + func TestChaCha8Marshal(t *testing.T) { p := NewChaCha8(chacha8seed) for i, x := range chacha8output { @@ -108,6 +124,17 @@ func TestChaCha8Marshal(t *testing.T) { if string(enc) != chacha8marshal[i] { t.Errorf("#%d: MarshalBinary=%q, want %q", i, enc, chacha8marshal[i]) } + + b := make([]byte, 4, 32) + b, err = p.AppendBinary(b) + encAppend := b[4:] + if err != nil { + t.Fatalf("#%d: AppendBinary: %v", i, err) + } + if string(encAppend) != chacha8marshal[i] { + t.Errorf("#%d: AppendBinary=%q, want %q", i, encAppend, chacha8marshal[i]) + } + *p = ChaCha8{} if err := p.UnmarshalBinary(enc); err != nil { t.Fatalf("#%d: UnmarshalBinary: %v", i, err) @@ -128,6 +155,17 @@ func TestChaCha8MarshalRead(t *testing.T) { if string(enc) != chacha8marshalread[i] { t.Errorf("#%d: MarshalBinary=%q, want %q", i, enc, chacha8marshalread[i]) } + + b := make([]byte, 4, 32) + b, err = p.AppendBinary(b) + encAppend := b[4:] + if err != nil { + t.Fatalf("#%d: AppendBinary: %v", i, err) + } + if string(encAppend) != chacha8marshalread[i] { + t.Errorf("#%d: AppendBinary=%q, want %q", i, encAppend, chacha8marshalread[i]) + } + *p = ChaCha8{} if err := p.UnmarshalBinary(enc); err != nil { t.Fatalf("#%d: UnmarshalBinary: %v", i, err) diff --git a/src/math/rand/v2/pcg.go b/src/math/rand/v2/pcg.go index 4ccd5e320b9716..9cd7d11ae31140 100644 --- a/src/math/rand/v2/pcg.go +++ b/src/math/rand/v2/pcg.go @@ -31,24 +31,28 @@ func (p *PCG) Seed(seed1, seed2 uint64) { p.lo = seed2 } -// MarshalBinary implements the encoding.BinaryMarshaler interface. -func (p *PCG) MarshalBinary() ([]byte, error) { - b := make([]byte, 20) - copy(b, "pcg:") - byteorder.BePutUint64(b[4:], p.hi) - byteorder.BePutUint64(b[4+8:], p.lo) +// AppendBinary implements the [encoding.BinaryAppender] interface. +func (p *PCG) AppendBinary(b []byte) ([]byte, error) { + b = append(b, "pcg:"...) + b = byteorder.BEAppendUint64(b, p.hi) + b = byteorder.BEAppendUint64(b, p.lo) return b, nil } +// MarshalBinary implements the [encoding.BinaryMarshaler] interface. +func (p *PCG) MarshalBinary() ([]byte, error) { + return p.AppendBinary(make([]byte, 0, 20)) +} + var errUnmarshalPCG = errors.New("invalid PCG encoding") -// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. +// UnmarshalBinary implements the [encoding.BinaryUnmarshaler] interface. func (p *PCG) UnmarshalBinary(data []byte) error { if len(data) != 20 || string(data[:4]) != "pcg:" { return errUnmarshalPCG } - p.hi = byteorder.BeUint64(data[4:]) - p.lo = byteorder.BeUint64(data[4+8:]) + p.hi = byteorder.BEUint64(data[4:]) + p.lo = byteorder.BEUint64(data[4+8:]) return nil } diff --git a/src/math/rand/v2/pcg_test.go b/src/math/rand/v2/pcg_test.go index db866c8c856db0..6558dab77b7c01 100644 --- a/src/math/rand/v2/pcg_test.go +++ b/src/math/rand/v2/pcg_test.go @@ -21,9 +21,10 @@ func BenchmarkPCG_DXSM(b *testing.B) { func TestPCGMarshal(t *testing.T) { var p PCG const ( - seed1 = 0x123456789abcdef0 - seed2 = 0xfedcba9876543210 - want = "pcg:\x12\x34\x56\x78\x9a\xbc\xde\xf0\xfe\xdc\xba\x98\x76\x54\x32\x10" + seed1 = 0x123456789abcdef0 + seed2 = 0xfedcba9876543210 + want = "pcg:\x12\x34\x56\x78\x9a\xbc\xde\xf0\xfe\xdc\xba\x98\x76\x54\x32\x10" + wantAppend = "\x00\x00\x00\x00" + want ) p.Seed(seed1, seed2) data, err := p.MarshalBinary() @@ -31,6 +32,12 @@ func TestPCGMarshal(t *testing.T) { t.Errorf("MarshalBinary() = %q, %v, want %q, nil", data, err, want) } + dataAppend := make([]byte, 4, 32) + dataAppend, err = p.AppendBinary(dataAppend) + if string(dataAppend) != wantAppend || err != nil { + t.Errorf("AppendBinary() = %q, %v, want %q, nil", dataAppend, err, wantAppend) + } + q := PCG{} if err := q.UnmarshalBinary([]byte(want)); err != nil { t.Fatalf("UnmarshalBinary(): %v", err) diff --git a/src/math/rand/v2/rand.go b/src/math/rand/v2/rand.go index fea1e3a2ba200b..f6b2e4754fa773 100644 --- a/src/math/rand/v2/rand.go +++ b/src/math/rand/v2/rand.go @@ -275,12 +275,12 @@ func Uint32() uint32 { return globalRand.Uint32() } // Uint64N returns, as a uint64, a pseudo-random number in the half-open interval [0,n) // from the default Source. -// It panics if n <= 0. +// It panics if n == 0. func Uint64N(n uint64) uint64 { return globalRand.Uint64N(n) } // Uint32N returns, as a uint32, a pseudo-random number in the half-open interval [0,n) // from the default Source. -// It panics if n <= 0. +// It panics if n == 0. func Uint32N(n uint32) uint32 { return globalRand.Uint32N(n) } // Uint64 returns a pseudo-random 64-bit value as a uint64 @@ -314,7 +314,7 @@ func IntN(n int) int { return globalRand.IntN(n) } // UintN returns, as a uint, a pseudo-random number in the half-open interval [0,n) // from the default Source. -// It panics if n <= 0. +// It panics if n == 0. func UintN(n uint) uint { return globalRand.UintN(n) } // N returns a pseudo-random number in the half-open interval [0,n) from the default Source. diff --git a/src/math/signbit.go b/src/math/signbit.go index f6e61d660e27d9..fe6b5a22af69ee 100644 --- a/src/math/signbit.go +++ b/src/math/signbit.go @@ -6,5 +6,5 @@ package math // Signbit reports whether x is negative or negative zero. func Signbit(x float64) bool { - return Float64bits(x)&(1<<63) != 0 + return int64(Float64bits(x)) < 0 } diff --git a/src/math/tan_s390x.s b/src/math/tan_s390x.s index 6a4c449b0dfcb6..48cd2841e881e0 100644 --- a/src/math/tan_s390x.s +++ b/src/math/tan_s390x.s @@ -87,7 +87,7 @@ L2: WFDDB V0, V1, V0 WFMDB V2, V16, V2 WFMADB V2, V0, V4, V0 - WORD $0xB3130000 //lcdbr %f0,%f0 + LCDBR F0, F0 FMOVD F0, ret+8(FP) RET L12: @@ -102,7 +102,7 @@ L11: FMOVD F0, ret+8(FP) RET L10: - WORD $0xB3130020 //lcdbr %f2,%f0 + LCDBR F0, F2 BR L2 L9: BR ·tan(SB) diff --git a/src/mime/encodedword.go b/src/mime/encodedword.go index 856433f8f3b393..c4afad043a2908 100644 --- a/src/mime/encodedword.go +++ b/src/mime/encodedword.go @@ -226,7 +226,7 @@ func (d *WordDecoder) Decode(word string) (string, error) { } // DecodeHeader decodes all encoded-words of the given string. It returns an -// error if and only if WordDecoder.CharsetReader of d returns an error. +// error if and only if [WordDecoder.CharsetReader] of d returns an error. func (d *WordDecoder) DecodeHeader(header string) (string, error) { // If there is no encoded-word, returns before creating a buffer. i := strings.Index(header, "=?") diff --git a/src/mime/grammar.go b/src/mime/grammar.go index 6a6f71dbd40ed8..cc578fbcfd4168 100644 --- a/src/mime/grammar.go +++ b/src/mime/grammar.go @@ -4,22 +4,68 @@ package mime -import ( - "strings" -) - -// isTSpecial reports whether rune is in 'tspecials' as defined by RFC +// isTSpecial reports whether c is in 'tspecials' as defined by RFC // 1521 and RFC 2045. -func isTSpecial(r rune) bool { - return strings.ContainsRune(`()<>@,;:\"/[]?=`, r) +func isTSpecial(c byte) bool { + // tspecials := "(" / ")" / "<" / ">" / "@" / + // "," / ";" / ":" / "\" / <"> + // "/" / "[" / "]" / "?" / "=" + // + // mask is a 128-bit bitmap with 1s for allowed bytes, + // so that the byte c can be tested with a shift and an and. + // If c >= 128, then 1<' | + 1<<'@' | + 1<<',' | + 1<<';' | + 1<<':' | + 1<<'\\' | + 1<<'"' | + 1<<'/' | + 1<<'[' | + 1<<']' | + 1<<'?' | + 1<<'=' + return ((uint64(1)<>64)) != 0 } -// isTokenChar reports whether rune is in 'token' as defined by RFC +// isTokenChar reports whether c is in 'token' as defined by RFC // 1521 and RFC 2045. -func isTokenChar(r rune) bool { +func isTokenChar(c byte) bool { // token := 1* - return r > 0x20 && r < 0x7f && !isTSpecial(r) + // + // mask is a 128-bit bitmap with 1s for allowed bytes, + // so that the byte c can be tested with a shift and an and. + // If c >= 128, then 1<>64)) != 0 } // isToken reports whether s is a 'token' as defined by RFC 1521 @@ -28,5 +74,10 @@ func isToken(s string) bool { if s == "" { return false } - return strings.IndexFunc(s, isNotTokenChar) < 0 + for _, c := range []byte(s) { + if !isTokenChar(c) { + return false + } + } + return true } diff --git a/src/mime/mediatype.go b/src/mime/mediatype.go index 97f3563a2d01df..c6006b614f319e 100644 --- a/src/mime/mediatype.go +++ b/src/mime/mediatype.go @@ -7,6 +7,7 @@ package mime import ( "errors" "fmt" + "maps" "slices" "strings" "unicode" @@ -33,13 +34,7 @@ func FormatMediaType(t string, param map[string]string) string { b.WriteString(strings.ToLower(sub)) } - attrs := make([]string, 0, len(param)) - for a := range param { - attrs = append(attrs, a) - } - slices.Sort(attrs) - - for _, attribute := range attrs { + for _, attribute := range slices.Sorted(maps.Keys(param)) { value := param[attribute] b.WriteByte(';') b.WriteByte(' ') @@ -65,7 +60,7 @@ func FormatMediaType(t string, param map[string]string) string { // attribute-char := if ch <= ' ' || ch >= 0x7F || ch == '*' || ch == '\'' || ch == '%' || - isTSpecial(rune(ch)) { + isTSpecial(ch) { b.WriteString(value[offset:index]) offset = index + 1 @@ -103,24 +98,32 @@ func FormatMediaType(t string, param map[string]string) string { func checkMediaTypeDisposition(s string) error { typ, rest := consumeToken(s) if typ == "" { - return errors.New("mime: no media type") + return errNoMediaType } if rest == "" { return nil } - if !strings.HasPrefix(rest, "/") { - return errors.New("mime: expected slash after first token") + var ok bool + if rest, ok = strings.CutPrefix(rest, "/"); !ok { + return errNoSlashAfterFirstToken } - subtype, rest := consumeToken(rest[1:]) + subtype, rest := consumeToken(rest) if subtype == "" { - return errors.New("mime: expected token after slash") + return errNoTokenAfterSlash } if rest != "" { - return errors.New("mime: unexpected content after media subtype") + return errUnexpectedContentAfterMediaSubtype } return nil } +var ( + errNoMediaType = errors.New("mime: no media type") + errNoSlashAfterFirstToken = errors.New("mime: expected slash after first token") + errNoTokenAfterSlash = errors.New("mime: expected token after slash") + errUnexpectedContentAfterMediaSubtype = errors.New("mime: unexpected content after media subtype") +) + // ErrInvalidMediaParameter is returned by [ParseMediaType] if // the media type value was found but there was an error parsing // the optional parameters @@ -174,7 +177,6 @@ func ParseMediaType(v string) (mediatype string, params map[string]string, err e if continuation == nil { continuation = make(map[string]map[string]string) } - var ok bool if pmap, ok = continuation[baseName]; !ok { continuation[baseName] = make(map[string]string) pmap = continuation[baseName] @@ -182,7 +184,7 @@ func ParseMediaType(v string) (mediatype string, params map[string]string, err e } if v, exists := pmap[key]; exists && v != value { // Duplicate parameter names are incorrect, but we allow them if they are equal. - return "", nil, errors.New("mime: duplicate parameter name") + return "", nil, errDuplicateParamName } pmap[key] = value v = rest @@ -232,31 +234,28 @@ func ParseMediaType(v string) (mediatype string, params map[string]string, err e return } +var errDuplicateParamName = errors.New("mime: duplicate parameter name") + func decode2231Enc(v string) (string, bool) { - sv := strings.SplitN(v, "'", 3) - if len(sv) != 3 { + charset, v, ok := strings.Cut(v, "'") + if !ok { return "", false } - // TODO: ignoring lang in sv[1] for now. If anybody needs it we'll + // TODO: ignoring the language part for now. If anybody needs it, we'll // need to decide how to expose it in the API. But I'm not sure // anybody uses it in practice. - charset := strings.ToLower(sv[0]) - if len(charset) == 0 { + _, extOtherVals, ok := strings.Cut(v, "'") + if !ok { return "", false } - if charset != "us-ascii" && charset != "utf-8" { - // TODO: unsupported encoding + charset = strings.ToLower(charset) + switch charset { + case "us-ascii", "utf-8": + default: + // Empty or unsupported encoding. return "", false } - encv, err := percentHexUnescape(sv[2]) - if err != nil { - return "", false - } - return encv, true -} - -func isNotTokenChar(r rune) bool { - return !isTokenChar(r) + return percentHexUnescape(extOtherVals) } // consumeToken consumes a token from the beginning of provided @@ -264,14 +263,12 @@ func isNotTokenChar(r rune) bool { // the token consumed and the rest of the string. Returns ("", v) on // failure to consume at least one character. func consumeToken(v string) (token, rest string) { - notPos := strings.IndexFunc(v, isNotTokenChar) - if notPos == -1 { - return v, "" - } - if notPos == 0 { - return "", v + for i := range len(v) { + if !isTokenChar(v[i]) { + return v[:i], v[i:] + } } - return v[0:notPos], v[notPos:] + return v, "" } // consumeValue consumes a "value" per RFC 2045, where a value is @@ -304,7 +301,7 @@ func consumeValue(v string) (value, rest string) { // and intended as a literal backslash. This makes Go servers deal better // with MSIE without affecting the way they handle conforming MIME // generators. - if r == '\\' && i+1 < len(v) && isTSpecial(rune(v[i+1])) { + if r == '\\' && i+1 < len(v) && isTSpecial(v[i+1]) { buffer.WriteByte(v[i+1]) i++ continue @@ -320,11 +317,11 @@ func consumeValue(v string) (value, rest string) { func consumeMediaParam(v string) (param, value, rest string) { rest = strings.TrimLeftFunc(v, unicode.IsSpace) - if !strings.HasPrefix(rest, ";") { + var ok bool + if rest, ok = strings.CutPrefix(rest, ";"); !ok { return "", "", v } - rest = rest[1:] // consume semicolon rest = strings.TrimLeftFunc(rest, unicode.IsSpace) param, rest = consumeToken(rest) param = strings.ToLower(param) @@ -333,10 +330,9 @@ func consumeMediaParam(v string) (param, value, rest string) { } rest = strings.TrimLeftFunc(rest, unicode.IsSpace) - if !strings.HasPrefix(rest, "=") { + if rest, ok = strings.CutPrefix(rest, "="); !ok { return "", "", v } - rest = rest[1:] // consume equals sign rest = strings.TrimLeftFunc(rest, unicode.IsSpace) value, rest2 := consumeValue(rest) if value == "" && rest2 == rest { @@ -346,7 +342,7 @@ func consumeMediaParam(v string) (param, value, rest string) { return param, value, rest } -func percentHexUnescape(s string) (string, error) { +func percentHexUnescape(s string) (string, bool) { // Count %, check that they're well-formed. percents := 0 for i := 0; i < len(s); { @@ -356,16 +352,12 @@ func percentHexUnescape(s string) (string, error) { } percents++ if i+2 >= len(s) || !ishex(s[i+1]) || !ishex(s[i+2]) { - s = s[i:] - if len(s) > 3 { - s = s[0:3] - } - return "", fmt.Errorf("mime: bogus characters after %%: %q", s) + return "", false } i += 3 } if percents == 0 { - return s, nil + return s, true } t := make([]byte, len(s)-2*percents) @@ -382,7 +374,7 @@ func percentHexUnescape(s string) (string, error) { i++ } } - return string(t), nil + return string(t), true } func ishex(c byte) bool { diff --git a/src/mime/mediatype_test.go b/src/mime/mediatype_test.go index 1731f7361e186a..251df8d6691ab9 100644 --- a/src/mime/mediatype_test.go +++ b/src/mime/mediatype_test.go @@ -96,7 +96,9 @@ type mediaTypeTest struct { p map[string]string } -func TestParseMediaType(t *testing.T) { +var parseMediaTypeTests []mediaTypeTest + +func init() { // Convenience map initializer m := func(s ...string) map[string]string { sm := make(map[string]string) @@ -107,7 +109,7 @@ func TestParseMediaType(t *testing.T) { } nameFoo := map[string]string{"name": "foo"} - tests := []mediaTypeTest{ + parseMediaTypeTests = []mediaTypeTest{ {`form-data; name="foo"`, "form-data", nameFoo}, {` form-data ; name=foo`, "form-data", nameFoo}, {`FORM-DATA;name="foo"`, "form-data", nameFoo}, @@ -412,7 +414,10 @@ func TestParseMediaType(t *testing.T) { {`text; charset=utf-8; charset=utf-8; format=fixed`, "text", m("charset", "utf-8", "format", "fixed")}, {`text; charset=utf-8; format=flowed; charset=utf-8`, "text", m("charset", "utf-8", "format", "flowed")}, } - for _, test := range tests { +} + +func TestParseMediaType(t *testing.T) { + for _, test := range parseMediaTypeTests { mt, params, err := ParseMediaType(test.in) if err != nil { if test.t != "" { @@ -438,6 +443,14 @@ func TestParseMediaType(t *testing.T) { } } +func BenchmarkParseMediaType(b *testing.B) { + for range b.N { + for _, test := range parseMediaTypeTests { + ParseMediaType(test.in) + } + } +} + type badMediaTypeTest struct { in string mt string @@ -486,6 +499,14 @@ func TestParseMediaTypeBogus(t *testing.T) { } } +func BenchmarkParseMediaTypeBogus(b *testing.B) { + for range b.N { + for _, test := range badMediaTypeTests { + ParseMediaType(test.in) + } + } +} + type formatTest struct { typ string params map[string]string diff --git a/src/mime/multipart/writer.go b/src/mime/multipart/writer.go index 5e589c499b339e..5d140693699ee4 100644 --- a/src/mime/multipart/writer.go +++ b/src/mime/multipart/writer.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "maps" "net/textproto" "slices" "strings" @@ -107,12 +108,7 @@ func (w *Writer) CreatePart(header textproto.MIMEHeader) (io.Writer, error) { fmt.Fprintf(&b, "--%s\r\n", w.boundary) } - keys := make([]string, 0, len(header)) - for k := range header { - keys = append(keys, k) - } - slices.Sort(keys) - for _, k := range keys { + for _, k := range slices.Sorted(maps.Keys(header)) { for _, v := range header[k] { fmt.Fprintf(&b, "%s: %s\r\n", k, v) } @@ -129,8 +125,20 @@ func (w *Writer) CreatePart(header textproto.MIMEHeader) (io.Writer, error) { return p, nil } -var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") +var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"", "\r", "%0D", "\n", "%0A") +// escapeQuotes escapes special characters in field parameter values. +// +// For historical reasons, this uses \ escaping for " and \ characters, +// and percent encoding for CR and LF. +// +// The WhatWG specification for form data encoding suggests that we should +// use percent encoding for " (%22), and should not escape \. +// https://html.spec.whatwg.org/multipage/form-control-infrastructure.html#multipart/form-data-encoding-algorithm +// +// Empirically, as of the time this comment was written, it is necessary +// to escape \ characters or else Chrome (and possibly other browsers) will +// interpet the unescaped \ as an escape. func escapeQuotes(s string) string { return quoteEscaper.Replace(s) } @@ -139,9 +147,7 @@ func escapeQuotes(s string) string { // a new form-data header with the provided field name and file name. func (w *Writer) CreateFormFile(fieldname, filename string) (io.Writer, error) { h := make(textproto.MIMEHeader) - h.Set("Content-Disposition", - fmt.Sprintf(`form-data; name="%s"; filename="%s"`, - escapeQuotes(fieldname), escapeQuotes(filename))) + h.Set("Content-Disposition", FileContentDisposition(fieldname, filename)) h.Set("Content-Type", "application/octet-stream") return w.CreatePart(h) } @@ -155,6 +161,13 @@ func (w *Writer) CreateFormField(fieldname string) (io.Writer, error) { return w.CreatePart(h) } +// FileContentDisposition returns the value of a Content-Disposition header +// with the provided field name and file name. +func FileContentDisposition(fieldname, filename string) string { + return fmt.Sprintf(`form-data; name="%s"; filename="%s"`, + escapeQuotes(fieldname), escapeQuotes(filename)) +} + // WriteField calls [Writer.CreateFormField] and then writes the given value. func (w *Writer) WriteField(fieldname, value string) error { p, err := w.CreateFormField(fieldname) diff --git a/src/mime/multipart/writer_test.go b/src/mime/multipart/writer_test.go index 9e0f1314c98350..c234b961080c84 100644 --- a/src/mime/multipart/writer_test.go +++ b/src/mime/multipart/writer_test.go @@ -172,3 +172,23 @@ func TestSortedHeader(t *testing.T) { t.Fatalf("\n got: %q\nwant: %q\n", buf.String(), want) } } + +func TestFileContentDisposition(t *testing.T) { + tests := []struct { + fieldname string + filename string + want string + }{ + {"somefield", "somefile.txt", `form-data; name="somefield"; filename="somefile.txt"`}, + {`field"withquotes"`, "somefile.txt", `form-data; name="field\"withquotes\""; filename="somefile.txt"`}, + {`somefield`, `somefile"withquotes".txt`, `form-data; name="somefield"; filename="somefile\"withquotes\".txt"`}, + {`somefield\withbackslash`, "somefile.txt", `form-data; name="somefield\\withbackslash"; filename="somefile.txt"`}, + {"somefield", `somefile\withbackslash.txt`, `form-data; name="somefield"; filename="somefile\\withbackslash.txt"`}, + {"a\rb\nc", "e\rf\ng", `form-data; name="a%0Db%0Ac"; filename="e%0Df%0Ag"`}, + } + for i, tt := range tests { + if found := FileContentDisposition(tt.fieldname, tt.filename); found != tt.want { + t.Errorf(`%d. found: "%s"; want: "%s"`, i, found, tt.want) + } + } +} diff --git a/src/mime/quotedprintable/reader.go b/src/mime/quotedprintable/reader.go index 4239625402a2f7..9e70a1bb3d06d8 100644 --- a/src/mime/quotedprintable/reader.go +++ b/src/mime/quotedprintable/reader.go @@ -66,6 +66,7 @@ var ( crlf = []byte("\r\n") lf = []byte("\n") softSuffix = []byte("=") + lwspChar = " \t" ) // Read reads and decodes quoted-printable data from the underlying reader. @@ -92,7 +93,7 @@ func (r *Reader) Read(p []byte) (n int, err error) { wholeLine := r.line r.line = bytes.TrimRightFunc(wholeLine, isQPDiscardWhitespace) if bytes.HasSuffix(r.line, softSuffix) { - rightStripped := wholeLine[len(r.line):] + rightStripped := bytes.TrimLeft(wholeLine[len(r.line):], lwspChar) r.line = r.line[:len(r.line)-1] if !bytes.HasPrefix(rightStripped, lf) && !bytes.HasPrefix(rightStripped, crlf) && !(len(rightStripped) == 0 && len(r.line) > 0 && r.rerr == io.EOF) { diff --git a/src/mime/quotedprintable/reader_test.go b/src/mime/quotedprintable/reader_test.go index 504bd5ef292547..1ff858a69c1799 100644 --- a/src/mime/quotedprintable/reader_test.go +++ b/src/mime/quotedprintable/reader_test.go @@ -66,6 +66,10 @@ func TestReader(t *testing.T) { want: "Now's the time for all folk to come to the aid of their country."}, {in: "accept UTF-8 right quotation mark: ’", want: "accept UTF-8 right quotation mark: ’"}, + + // Transport padding + {in: "foo= \r\nbar", want: "foobar"}, + {in: "foo=\t \r\nbar", want: "foobar"}, } for _, tt := range tests { var buf strings.Builder @@ -199,13 +203,13 @@ func TestExhaustive(t *testing.T) { } slices.Sort(outcomes) got := strings.Join(outcomes, "\n") - want := `OK: 28934 -invalid bytes after =: 3949 -quotedprintable: invalid hex byte 0x0d: 2048 + want := `OK: 30638 +invalid bytes after =: 2243 +quotedprintable: invalid hex byte 0x0d: 2050 unexpected EOF: 194` if testing.Short() { - want = `OK: 896 -invalid bytes after =: 100 + want = `OK: 935 +invalid bytes after =: 61 quotedprintable: invalid hex byte 0x0d: 26 unexpected EOF: 3` } diff --git a/src/mime/type.go b/src/mime/type.go index c86ebd3442c1dc..ac7b0447da3cf9 100644 --- a/src/mime/type.go +++ b/src/mime/type.go @@ -17,7 +17,7 @@ var ( mimeTypesLower sync.Map // map[string]string; ".z" => "application/x-compress" // extensions maps from MIME type to list of lowercase file - // extensions: "image/jpeg" => [".jpg", ".jpeg"] + // extensions: "image/jpeg" => [".jfif", ".jpg", ".jpeg", ".pjp", ".pjpeg"] extensionsMu sync.Mutex // Guards stores (but not loads) on extensions. extensions sync.Map // map[string][]string; slice values are append-only. ) @@ -50,23 +50,82 @@ func setMimeTypes(lowerExt, mixExt map[string]string) { } } +// A type is listed here if both Firefox and Chrome included them in their own +// lists. In the case where they contradict they are deconflicted using IANA's +// listed media types https://www.iana.org/assignments/media-types/media-types.xhtml +// +// Chrome's MIME mappings to file extensions are defined at +// https://chromium.googlesource.com/chromium/src.git/+/refs/heads/main/net/base/mime_util.cc +// +// Firefox's MIME types can be found at +// https://github.com/mozilla-firefox/firefox/blob/main/netwerk/mime/nsMimeTypes.h +// and the mappings to file extensions at +// https://github.com/mozilla-firefox/firefox/blob/main/uriloader/exthandler/nsExternalHelperAppService.cpp var builtinTypesLower = map[string]string{ - ".avif": "image/avif", - ".css": "text/css; charset=utf-8", - ".gif": "image/gif", - ".htm": "text/html; charset=utf-8", - ".html": "text/html; charset=utf-8", - ".jpeg": "image/jpeg", - ".jpg": "image/jpeg", - ".js": "text/javascript; charset=utf-8", - ".json": "application/json", - ".mjs": "text/javascript; charset=utf-8", - ".pdf": "application/pdf", - ".png": "image/png", - ".svg": "image/svg+xml", - ".wasm": "application/wasm", - ".webp": "image/webp", - ".xml": "text/xml; charset=utf-8", + ".ai": "application/postscript", + ".apk": "application/vnd.android.package-archive", + ".apng": "image/apng", + ".avif": "image/avif", + ".bin": "application/octet-stream", + ".bmp": "image/bmp", + ".com": "application/octet-stream", + ".css": "text/css; charset=utf-8", + ".csv": "text/csv; charset=utf-8", + ".doc": "application/msword", + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".ehtml": "text/html; charset=utf-8", + ".eml": "message/rfc822", + ".eps": "application/postscript", + ".exe": "application/octet-stream", + ".flac": "audio/flac", + ".gif": "image/gif", + ".gz": "application/gzip", + ".htm": "text/html; charset=utf-8", + ".html": "text/html; charset=utf-8", + ".ico": "image/vnd.microsoft.icon", + ".ics": "text/calendar; charset=utf-8", + ".jfif": "image/jpeg", + ".jpeg": "image/jpeg", + ".jpg": "image/jpeg", + ".js": "text/javascript; charset=utf-8", + ".json": "application/json", + ".m4a": "audio/mp4", + ".mjs": "text/javascript; charset=utf-8", + ".mp3": "audio/mpeg", + ".mp4": "video/mp4", + ".oga": "audio/ogg", + ".ogg": "audio/ogg", + ".ogv": "video/ogg", + ".opus": "audio/ogg", + ".pdf": "application/pdf", + ".pjp": "image/jpeg", + ".pjpeg": "image/jpeg", + ".png": "image/png", + ".ppt": "application/vnd.ms-powerpoint", + ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ".ps": "application/postscript", + ".rdf": "application/rdf+xml", + ".rtf": "application/rtf", + ".shtml": "text/html; charset=utf-8", + ".svg": "image/svg+xml", + ".text": "text/plain; charset=utf-8", + ".tif": "image/tiff", + ".tiff": "image/tiff", + ".txt": "text/plain; charset=utf-8", + ".vtt": "text/vtt; charset=utf-8", + ".wasm": "application/wasm", + ".wav": "audio/wav", + ".webm": "audio/webm", + ".webp": "image/webp", + ".xbl": "text/xml; charset=utf-8", + ".xbm": "image/x-xbitmap", + ".xht": "application/xhtml+xml", + ".xhtml": "application/xhtml+xml", + ".xls": "application/vnd.ms-excel", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".xml": "text/xml; charset=utf-8", + ".xsl": "text/xml; charset=utf-8", + ".zip": "application/zip", } var once sync.Once // guards initMime diff --git a/src/mime/type_test.go b/src/mime/type_test.go index 2e55468dd7c100..f4ec8c8754e69f 100644 --- a/src/mime/type_test.go +++ b/src/mime/type_test.go @@ -5,6 +5,7 @@ package mime import ( + "internal/asan" "slices" "strings" "sync" @@ -143,6 +144,9 @@ func TestExtensionsByType(t *testing.T) { } func TestLookupMallocs(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } n := testing.AllocsPerRun(10000, func() { TypeByExtension(".html") TypeByExtension(".HtML") @@ -204,7 +208,51 @@ func TestExtensionsByType2(t *testing.T) { typ string want []string }{ - {typ: "image/jpeg", want: []string{".jpeg", ".jpg"}}, + {typ: "application/postscript", want: []string{".ai", ".eps", ".ps"}}, + {typ: "application/vnd.android.package-archive", want: []string{".apk"}}, + {typ: "image/apng", want: []string{".apng"}}, + {typ: "image/avif", want: []string{".avif"}}, + {typ: "application/octet-stream", want: []string{".bin", ".com", ".exe"}}, + {typ: "image/bmp", want: []string{".bmp"}}, + {typ: "text/css; charset=utf-8", want: []string{".css"}}, + {typ: "text/csv; charset=utf-8", want: []string{".csv"}}, + {typ: "application/msword", want: []string{".doc"}}, + {typ: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", want: []string{".docx"}}, + {typ: "text/html; charset=utf-8", want: []string{".ehtml", ".htm", ".html", ".shtml"}}, + {typ: "message/rfc822", want: []string{".eml"}}, + {typ: "audio/flac", want: []string{".flac"}}, + {typ: "image/gif", want: []string{".gif"}}, + {typ: "application/gzip", want: []string{".gz"}}, + {typ: "image/vnd.microsoft.icon", want: []string{".ico"}}, + {typ: "text/calendar; charset=utf-8", want: []string{".ics"}}, + {typ: "image/jpeg", want: []string{".jfif", ".jpeg", ".jpg", ".pjp", ".pjpeg"}}, + {typ: "text/javascript; charset=utf-8", want: []string{".js", ".mjs"}}, + {typ: "application/json", want: []string{".json"}}, + {typ: "audio/mp4", want: []string{".m4a"}}, + {typ: "audio/mpeg", want: []string{".mp3"}}, + {typ: "video/mp4", want: []string{".mp4"}}, + {typ: "audio/ogg", want: []string{".oga", ".ogg", ".opus"}}, + {typ: "video/ogg", want: []string{".ogv"}}, + {typ: "application/pdf", want: []string{".pdf"}}, + {typ: "image/png", want: []string{".png"}}, + {typ: "application/vnd.ms-powerpoint", want: []string{".ppt"}}, + {typ: "application/vnd.openxmlformats-officedocument.presentationml.presentation", want: []string{".pptx"}}, + {typ: "application/rdf+xml", want: []string{".rdf"}}, + {typ: "application/rtf", want: []string{".rtf"}}, + {typ: "image/svg+xml", want: []string{".svg"}}, + {typ: "text/plain; charset=utf-8", want: []string{".text", ".txt"}}, + {typ: "image/tiff", want: []string{".tif", ".tiff"}}, + {typ: "text/vtt; charset=utf-8", want: []string{".vtt"}}, + {typ: "application/wasm", want: []string{".wasm"}}, + {typ: "audio/wav", want: []string{".wav"}}, + {typ: "audio/webm", want: []string{".webm"}}, + {typ: "image/webp", want: []string{".webp"}}, + {typ: "text/xml; charset=utf-8", want: []string{".xbl", ".xml", ".xsl"}}, + {typ: "image/x-xbitmap", want: []string{".xbm"}}, + {typ: "application/xhtml+xml", want: []string{".xht", ".xhtml"}}, + {typ: "application/vnd.ms-excel", want: []string{".xls"}}, + {typ: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", want: []string{".xlsx"}}, + {typ: "application/zip", want: []string{".zip"}}, } for _, tt := range tests { diff --git a/src/net/addrselect.go b/src/net/addrselect.go index caff09b3772ff7..0ff8ec37c8de35 100644 --- a/src/net/addrselect.go +++ b/src/net/addrselect.go @@ -8,7 +8,7 @@ package net import ( "net/netip" - "sort" + "slices" ) func sortByRFC6724(addrs []IPAddr) { @@ -22,19 +22,20 @@ func sortByRFC6724withSrcs(addrs []IPAddr, srcs []netip.Addr) { if len(addrs) != len(srcs) { panic("internal error") } - addrAttr := make([]ipAttr, len(addrs)) - srcAttr := make([]ipAttr, len(srcs)) + addrInfos := make([]byRFC6724Info, len(addrs)) for i, v := range addrs { addrAttrIP, _ := netip.AddrFromSlice(v.IP) - addrAttr[i] = ipAttrOf(addrAttrIP) - srcAttr[i] = ipAttrOf(srcs[i]) + addrInfos[i] = byRFC6724Info{ + addr: addrs[i], + addrAttr: ipAttrOf(addrAttrIP), + src: srcs[i], + srcAttr: ipAttrOf(srcs[i]), + } + } + slices.SortStableFunc(addrInfos, compareByRFC6724) + for i := range addrInfos { + addrs[i] = addrInfos[i].addr } - sort.Stable(&byRFC6724{ - addrs: addrs, - addrAttr: addrAttr, - srcs: srcs, - srcAttr: srcAttr, - }) } // srcAddrs tries to UDP-connect to each address to see if it has a @@ -75,45 +76,36 @@ func ipAttrOf(ip netip.Addr) ipAttr { } } -type byRFC6724 struct { - addrs []IPAddr // addrs to sort - addrAttr []ipAttr - srcs []netip.Addr // or not valid addr if unreachable - srcAttr []ipAttr -} - -func (s *byRFC6724) Len() int { return len(s.addrs) } - -func (s *byRFC6724) Swap(i, j int) { - s.addrs[i], s.addrs[j] = s.addrs[j], s.addrs[i] - s.srcs[i], s.srcs[j] = s.srcs[j], s.srcs[i] - s.addrAttr[i], s.addrAttr[j] = s.addrAttr[j], s.addrAttr[i] - s.srcAttr[i], s.srcAttr[j] = s.srcAttr[j], s.srcAttr[i] +type byRFC6724Info struct { + addr IPAddr + addrAttr ipAttr + src netip.Addr + srcAttr ipAttr } -// Less reports whether i is a better destination address for this -// host than j. -// -// The algorithm and variable names comes from RFC 6724 section 6. -func (s *byRFC6724) Less(i, j int) bool { - DA := s.addrs[i].IP - DB := s.addrs[j].IP - SourceDA := s.srcs[i] - SourceDB := s.srcs[j] - attrDA := &s.addrAttr[i] - attrDB := &s.addrAttr[j] - attrSourceDA := &s.srcAttr[i] - attrSourceDB := &s.srcAttr[j] - - const preferDA = true - const preferDB = false +// compareByRFC6724 compares two byRFC6724Info records and returns an integer +// indicating the order. It follows the algorithm and variable names from +// RFC 6724 section 6. Returns -1 if a is preferred, 1 if b is preferred, +// and 0 if they are equal. +func compareByRFC6724(a, b byRFC6724Info) int { + DA := a.addr.IP + DB := b.addr.IP + SourceDA := a.src + SourceDB := b.src + attrDA := &a.addrAttr + attrDB := &b.addrAttr + attrSourceDA := &a.srcAttr + attrSourceDB := &b.srcAttr + + const preferDA = -1 + const preferDB = 1 // Rule 1: Avoid unusable destinations. // If DB is known to be unreachable or if Source(DB) is undefined, then // prefer DA. Similarly, if DA is known to be unreachable or if // Source(DA) is undefined, then prefer DB. if !SourceDA.IsValid() && !SourceDB.IsValid() { - return false // "equal" + return 0 // "equal" } if !SourceDB.IsValid() { return preferDA @@ -212,7 +204,7 @@ func (s *byRFC6724) Less(i, j int) bool { // Rule 10: Otherwise, leave the order unchanged. // If DA preceded DB in the original list, prefer DA. // Otherwise, prefer DB. - return false // "equal" + return 0 // "equal" } type policyTableEntry struct { diff --git a/src/net/cgo_solaris.go b/src/net/cgo_solaris.go index cde9c957fee53c..98a0e819565d3f 100644 --- a/src/net/cgo_solaris.go +++ b/src/net/cgo_solaris.go @@ -7,7 +7,7 @@ package net /* -#cgo LDFLAGS: -lsocket -lnsl -lsendfile +#cgo LDFLAGS: -lsocket -lnsl #include */ import "C" diff --git a/src/net/cgo_stub.go b/src/net/cgo_stub.go index a4f6b4b0e8d8b7..640f5a4fd65a55 100644 --- a/src/net/cgo_stub.go +++ b/src/net/cgo_stub.go @@ -31,7 +31,7 @@ func cgoLookupIP(ctx context.Context, network, name string) (addrs []IPAddr, err panic("cgo stub: cgo not available") } -func cgoLookupCNAME(ctx context.Context, name string) (cname string, err error, completed bool) { +func cgoLookupCNAME(ctx context.Context, name string) (cname string, err error) { panic("cgo stub: cgo not available") } diff --git a/src/net/cgo_unix.go b/src/net/cgo_unix.go index bc374c2c7688b4..51efee6f872c53 100644 --- a/src/net/cgo_unix.go +++ b/src/net/cgo_unix.go @@ -16,6 +16,7 @@ import ( "errors" "internal/bytealg" "net/netip" + "runtime" "syscall" "unsafe" @@ -46,11 +47,7 @@ func (eai addrinfoErrno) isAddrinfoErrno() {} func doBlockingWithCtx[T any](ctx context.Context, lookupName string, blocking func() (T, error)) (T, error) { if err := acquireThread(ctx); err != nil { var zero T - return zero, &DNSError{ - Name: lookupName, - Err: mapErr(err).Error(), - IsTimeout: err == context.DeadlineExceeded, - } + return zero, newDNSError(mapErr(err), lookupName, "") } if ctx.Done() == nil { @@ -76,11 +73,7 @@ func doBlockingWithCtx[T any](ctx context.Context, lookupName string, blocking f return r.res, r.err case <-ctx.Done(): var zero T - return zero, &DNSError{ - Name: lookupName, - Err: mapErr(ctx.Err()).Error(), - IsTimeout: ctx.Err() == context.DeadlineExceeded, - } + return zero, newDNSError(mapErr(ctx.Err()), lookupName, "") } } @@ -195,6 +188,16 @@ func cgoLookupHostIP(network, name string) (addrs []IPAddr, err error) { return nil, newDNSError(err, name, "") case _C_EAI_NONAME, _C_EAI_NODATA: return nil, newDNSError(errNoSuchHost, name, "") + case _C_EAI_ADDRFAMILY: + if runtime.GOOS == "freebsd" { + // FreeBSD began returning EAI_ADDRFAMILY for valid hosts without + // an A record in 13.2. We previously returned "no such host" for + // this case. + // + // https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=273912 + return nil, newDNSError(errNoSuchHost, name, "") + } + fallthrough default: return nil, newDNSError(addrinfoErrno(gerrno), name, "") } @@ -294,16 +297,16 @@ func cgoSockaddr(ip IP, zone string) (*_C_struct_sockaddr, _C_socklen_t) { return nil, 0 } -func cgoLookupCNAME(ctx context.Context, name string) (cname string, err error, completed bool) { +func cgoLookupCNAME(ctx context.Context, name string) (cname string, err error) { resources, err := resSearch(ctx, name, int(dnsmessage.TypeCNAME), int(dnsmessage.ClassINET)) if err != nil { - return + return "", err } cname, err = parseCNAMEFromResources(resources) if err != nil { - return "", err, false + return "", err } - return cname, nil, true + return cname, nil } // resSearch will make a call to the 'res_nsearch' routine in the C library @@ -348,7 +351,7 @@ func cgoResSearch(hostname string, rtype, class int) ([]dnsmessage.Resource, err var size int for { - size := _C_res_nsearch(state, (*_C_char)(unsafe.Pointer(s)), class, rtype, buf, bufSize) + size = _C_res_nsearch(state, (*_C_char)(unsafe.Pointer(s)), class, rtype, buf, bufSize) if size <= 0 || size > 0xffff { return nil, errors.New("res_nsearch failure") } diff --git a/src/net/cgo_unix_cgo.go b/src/net/cgo_unix_cgo.go index d38ae0a84f2bc3..c4b8197c620458 100644 --- a/src/net/cgo_unix_cgo.go +++ b/src/net/cgo_unix_cgo.go @@ -22,6 +22,11 @@ package net #define EAI_NODATA -5 #endif +// If nothing else defined EAI_ADDRFAMILY, make sure it has a value. +#ifndef EAI_ADDRFAMILY +#define EAI_ADDRFAMILY -9 +#endif + // If nothing else defined EAI_OVERFLOW, make sure it has a value. #ifndef EAI_OVERFLOW #define EAI_OVERFLOW -12 @@ -31,19 +36,20 @@ import "C" import "unsafe" const ( - _C_AF_INET = C.AF_INET - _C_AF_INET6 = C.AF_INET6 - _C_AF_UNSPEC = C.AF_UNSPEC - _C_EAI_AGAIN = C.EAI_AGAIN - _C_EAI_NODATA = C.EAI_NODATA - _C_EAI_NONAME = C.EAI_NONAME - _C_EAI_SERVICE = C.EAI_SERVICE - _C_EAI_OVERFLOW = C.EAI_OVERFLOW - _C_EAI_SYSTEM = C.EAI_SYSTEM - _C_IPPROTO_TCP = C.IPPROTO_TCP - _C_IPPROTO_UDP = C.IPPROTO_UDP - _C_SOCK_DGRAM = C.SOCK_DGRAM - _C_SOCK_STREAM = C.SOCK_STREAM + _C_AF_INET = C.AF_INET + _C_AF_INET6 = C.AF_INET6 + _C_AF_UNSPEC = C.AF_UNSPEC + _C_EAI_ADDRFAMILY = C.EAI_ADDRFAMILY + _C_EAI_AGAIN = C.EAI_AGAIN + _C_EAI_NODATA = C.EAI_NODATA + _C_EAI_NONAME = C.EAI_NONAME + _C_EAI_SERVICE = C.EAI_SERVICE + _C_EAI_OVERFLOW = C.EAI_OVERFLOW + _C_EAI_SYSTEM = C.EAI_SYSTEM + _C_IPPROTO_TCP = C.IPPROTO_TCP + _C_IPPROTO_UDP = C.IPPROTO_UDP + _C_SOCK_DGRAM = C.SOCK_DGRAM + _C_SOCK_STREAM = C.SOCK_STREAM ) type ( diff --git a/src/net/cgo_unix_syscall.go b/src/net/cgo_unix_syscall.go index 735dcdfe368b12..6b7ef74bd2b945 100644 --- a/src/net/cgo_unix_syscall.go +++ b/src/net/cgo_unix_syscall.go @@ -14,19 +14,20 @@ import ( ) const ( - _C_AF_INET = syscall.AF_INET - _C_AF_INET6 = syscall.AF_INET6 - _C_AF_UNSPEC = syscall.AF_UNSPEC - _C_EAI_AGAIN = unix.EAI_AGAIN - _C_EAI_NONAME = unix.EAI_NONAME - _C_EAI_SERVICE = unix.EAI_SERVICE - _C_EAI_NODATA = unix.EAI_NODATA - _C_EAI_OVERFLOW = unix.EAI_OVERFLOW - _C_EAI_SYSTEM = unix.EAI_SYSTEM - _C_IPPROTO_TCP = syscall.IPPROTO_TCP - _C_IPPROTO_UDP = syscall.IPPROTO_UDP - _C_SOCK_DGRAM = syscall.SOCK_DGRAM - _C_SOCK_STREAM = syscall.SOCK_STREAM + _C_AF_INET = syscall.AF_INET + _C_AF_INET6 = syscall.AF_INET6 + _C_AF_UNSPEC = syscall.AF_UNSPEC + _C_EAI_ADDRFAMILY = unix.EAI_ADDRFAMILY + _C_EAI_AGAIN = unix.EAI_AGAIN + _C_EAI_NONAME = unix.EAI_NONAME + _C_EAI_SERVICE = unix.EAI_SERVICE + _C_EAI_NODATA = unix.EAI_NODATA + _C_EAI_OVERFLOW = unix.EAI_OVERFLOW + _C_EAI_SYSTEM = unix.EAI_SYSTEM + _C_IPPROTO_TCP = syscall.IPPROTO_TCP + _C_IPPROTO_UDP = syscall.IPPROTO_UDP + _C_SOCK_DGRAM = syscall.SOCK_DGRAM + _C_SOCK_STREAM = syscall.SOCK_STREAM ) type ( @@ -83,8 +84,7 @@ func _C_res_nclose(state *_C_struct___res_state) { } func cgoNameinfoPTR(b []byte, sa *syscall.RawSockaddr, salen int) (int, error) { - gerrno, err := unix.Getnameinfo(sa, salen, &b[0], len(b), nil, 0, unix.NI_NAMEREQD) - return int(gerrno), err + return unix.Getnameinfo(sa, salen, &b[0], len(b), nil, 0, unix.NI_NAMEREQD) } func cgoSockaddrInet4(ip IP) *syscall.RawSockaddr { diff --git a/src/net/cgo_unix_test.go b/src/net/cgo_unix_test.go index d8233dfaf22960..9aaeccc5f05709 100644 --- a/src/net/cgo_unix_test.go +++ b/src/net/cgo_unix_test.go @@ -8,6 +8,7 @@ package net import ( "context" + "internal/testenv" "testing" ) @@ -67,3 +68,12 @@ func TestCgoLookupPTRWithCancel(t *testing.T) { t.Error(err) } } + +func TestCgoLookupCNAME(t *testing.T) { + mustHaveExternalNetwork(t) + testenv.SkipFlakyNet(t) + defer dnsWaitGroup.Wait() + if _, err := cgoLookupCNAME(t.Context(), "www.iana.org."); err != nil { + t.Error(err) + } +} diff --git a/src/net/conf.go b/src/net/conf.go index 358f5434c4de36..92c5618d1ec221 100644 --- a/src/net/conf.go +++ b/src/net/conf.go @@ -13,7 +13,6 @@ import ( "os" "runtime" "sync" - "syscall" ) // The net package's name resolution is rather complicated. @@ -94,19 +93,30 @@ func initConfVal() { if confVal.dnsDebugLevel > 1 { println("go package net: confVal.netCgo =", confVal.netCgo, " netGo =", confVal.netGo) } + if dnsMode != "go" && dnsMode != "cgo" && dnsMode != "" { + println("go package net: GODEBUG=netdns contains an invalid dns mode, ignoring it") + } switch { - case confVal.netGo: - if netGoBuildTag { - println("go package net: built with netgo build tag; using Go's DNS resolver") + case netGoBuildTag || !cgoAvailable: + if dnsMode == "cgo" { + println("go package net: ignoring GODEBUG=netdns=cgo as the binary was compiled without support for the cgo resolver") + } else { + println("go package net: using the Go DNS resolver") + } + case netCgoBuildTag: + if dnsMode == "go" { + println("go package net: GODEBUG setting forcing use of the Go resolver") } else { - println("go package net: GODEBUG setting forcing use of Go's resolver") + println("go package net: using the cgo DNS resolver") } - case !cgoAvailable: - println("go package net: cgo resolver not supported; using Go's DNS resolver") - case confVal.netCgo || confVal.preferCgo: - println("go package net: using cgo DNS resolver") default: - println("go package net: dynamic selection of DNS resolver") + if dnsMode == "go" { + println("go package net: GODEBUG setting forcing use of the Go resolver") + } else if dnsMode == "cgo" { + println("go package net: GODEBUG setting forcing use of the cgo resolver") + } else { + println("go package net: dynamic selection of DNS resolver") + } } }() } @@ -138,7 +148,7 @@ func initConfVal() { // prefer the cgo resolver. // Note that LOCALDOMAIN can change behavior merely by being // specified with the empty string. - _, localDomainDefined := syscall.Getenv("LOCALDOMAIN") + _, localDomainDefined := os.LookupEnv("LOCALDOMAIN") if localDomainDefined || os.Getenv("RES_OPTIONS") != "" || os.Getenv("HOSTALIASES") != "" { confVal.preferCgo = true return diff --git a/src/net/conn_test.go b/src/net/conn_test.go index d1e1e7bf1cdf32..87097e10ee3281 100644 --- a/src/net/conn_test.go +++ b/src/net/conn_test.go @@ -19,7 +19,6 @@ const someTimeout = 1 * time.Hour func TestConnAndListener(t *testing.T) { for i, network := range []string{"tcp", "unix", "unixpacket"} { - i, network := i, network t.Run(network, func(t *testing.T) { if !testableNetwork(network) { t.Skipf("skipping %s test", network) diff --git a/src/net/dial.go b/src/net/dial.go index 28f346a372a780..a87c57603a813c 100644 --- a/src/net/dial.go +++ b/src/net/dial.go @@ -9,6 +9,7 @@ import ( "internal/bytealg" "internal/godebug" "internal/nettrace" + "net/netip" "syscall" "time" ) @@ -25,46 +26,93 @@ const ( // defaultTCPKeepAliveCount is a default constant value for TCP_KEEPCNT. defaultTCPKeepAliveCount = 9 - // For the moment, MultiPath TCP is not used by default + // For the moment, MultiPath TCP is used by default with listeners, if + // available, but not with dialers. // See go.dev/issue/56539 - defaultMPTCPEnabled = false + defaultMPTCPEnabledListen = true + defaultMPTCPEnabledDial = false ) +// The type of service offered +// +// 0 == MPTCP disabled +// 1 == MPTCP enabled +// 2 == MPTCP enabled on listeners only +// 3 == MPTCP enabled on dialers only var multipathtcp = godebug.New("multipathtcp") -// mptcpStatus is a tristate for Multipath TCP, see go.dev/issue/56539 -type mptcpStatus uint8 +// mptcpStatusDial is a tristate for Multipath TCP on clients, +// see go.dev/issue/56539 +type mptcpStatusDial uint8 const ( - // The value 0 is the system default, linked to defaultMPTCPEnabled - mptcpUseDefault mptcpStatus = iota - mptcpEnabled - mptcpDisabled + // The value 0 is the system default, linked to defaultMPTCPEnabledDial + mptcpUseDefaultDial mptcpStatusDial = iota + mptcpEnabledDial + mptcpDisabledDial ) -func (m *mptcpStatus) get() bool { +func (m *mptcpStatusDial) get() bool { switch *m { - case mptcpEnabled: + case mptcpEnabledDial: return true - case mptcpDisabled: + case mptcpDisabledDial: return false } // If MPTCP is forced via GODEBUG=multipathtcp=1 - if multipathtcp.Value() == "1" { + if multipathtcp.Value() == "1" || multipathtcp.Value() == "3" { multipathtcp.IncNonDefault() return true } - return defaultMPTCPEnabled + return defaultMPTCPEnabledDial } -func (m *mptcpStatus) set(use bool) { +func (m *mptcpStatusDial) set(use bool) { if use { - *m = mptcpEnabled + *m = mptcpEnabledDial } else { - *m = mptcpDisabled + *m = mptcpDisabledDial + } +} + +// mptcpStatusListen is a tristate for Multipath TCP on servers, +// see go.dev/issue/56539 +type mptcpStatusListen uint8 + +const ( + // The value 0 is the system default, linked to defaultMPTCPEnabledListen + mptcpUseDefaultListen mptcpStatusListen = iota + mptcpEnabledListen + mptcpDisabledListen +) + +func (m *mptcpStatusListen) get() bool { + switch *m { + case mptcpEnabledListen: + return true + case mptcpDisabledListen: + return false + } + + // If MPTCP is disabled via GODEBUG=multipathtcp=0 or only + // enabled on dialers, but not on listeners. + if multipathtcp.Value() == "0" || multipathtcp.Value() == "3" { + multipathtcp.IncNonDefault() + + return false + } + + return defaultMPTCPEnabledListen +} + +func (m *mptcpStatusListen) set(use bool) { + if use { + *m = mptcpEnabledListen + } else { + *m = mptcpDisabledListen } } @@ -156,8 +204,10 @@ type Dialer struct { // connection but before actually dialing. // // Network and address parameters passed to Control function are not - // necessarily the ones passed to Dial. For example, passing "tcp" to Dial - // will cause the Control function to be called with "tcp4" or "tcp6". + // necessarily the ones passed to Dial. Calling Dial with TCP networks + // will cause the Control function to be called with "tcp4" or "tcp6", + // UDP networks become "udp4" or "udp6", IP networks become "ip4" or "ip6", + // and other known networks are passed as-is. // // Control is ignored if ControlContext is not nil. Control func(network, address string, c syscall.RawConn) error @@ -166,8 +216,10 @@ type Dialer struct { // connection but before actually dialing. // // Network and address parameters passed to ControlContext function are not - // necessarily the ones passed to Dial. For example, passing "tcp" to Dial - // will cause the ControlContext function to be called with "tcp4" or "tcp6". + // necessarily the ones passed to Dial. Calling Dial with TCP networks + // will cause the ControlContext function to be called with "tcp4" or "tcp6", + // UDP networks become "udp4" or "udp6", IP networks become "ip4" or "ip6", + // and other known networks are passed as-is. // // If ControlContext is not nil, Control is ignored. ControlContext func(ctx context.Context, network, address string, c syscall.RawConn) error @@ -175,7 +227,7 @@ type Dialer struct { // If mptcpStatus is set to a value allowing Multipath TCP (MPTCP) to be // used, any call to Dial with "tcp(4|6)" as network will use MPTCP if // supported by the operating system. - mptcpStatus mptcpStatus + mptcpStatus mptcpStatusDial } func (d *Dialer) dualStack() bool { return d.FallbackDelay >= 0 } @@ -472,30 +524,8 @@ func (d *Dialer) Dial(network, address string) (Conn, error) { // See func [Dial] for a description of the network and address // parameters. func (d *Dialer) DialContext(ctx context.Context, network, address string) (Conn, error) { - if ctx == nil { - panic("nil context") - } - deadline := d.deadline(ctx, time.Now()) - if !deadline.IsZero() { - testHookStepTime() - if d, ok := ctx.Deadline(); !ok || deadline.Before(d) { - subCtx, cancel := context.WithDeadline(ctx, deadline) - defer cancel() - ctx = subCtx - } - } - if oldCancel := d.Cancel; oldCancel != nil { - subCtx, cancel := context.WithCancel(ctx) - defer cancel() - go func() { - select { - case <-oldCancel: - cancel() - case <-subCtx.Done(): - } - }() - ctx = subCtx - } + ctx, cancel := d.dialCtx(ctx) + defer cancel() // Shadow the nettrace (if any) during resolve so Connect events don't fire for DNS lookups. resolveCtx := ctx @@ -527,6 +557,97 @@ func (d *Dialer) DialContext(ctx context.Context, network, address string) (Conn return sd.dialParallel(ctx, primaries, fallbacks) } +func (d *Dialer) dialCtx(ctx context.Context) (context.Context, context.CancelFunc) { + if ctx == nil { + panic("nil context") + } + deadline := d.deadline(ctx, time.Now()) + var cancel1, cancel2 context.CancelFunc + if !deadline.IsZero() { + testHookStepTime() + if d, ok := ctx.Deadline(); !ok || deadline.Before(d) { + var subCtx context.Context + subCtx, cancel1 = context.WithDeadline(ctx, deadline) + ctx = subCtx + } + } + if oldCancel := d.Cancel; oldCancel != nil { + subCtx, cancel2 := context.WithCancel(ctx) + go func() { + select { + case <-oldCancel: + cancel2() + case <-subCtx.Done(): + } + }() + ctx = subCtx + } + return ctx, func() { + if cancel1 != nil { + cancel1() + } + if cancel2 != nil { + cancel2() + } + } +} + +// DialTCP acts like Dial for TCP networks using the provided context. +// +// The provided Context must be non-nil. If the context expires before +// the connection is complete, an error is returned. Once successfully +// connected, any expiration of the context will not affect the +// connection. +// +// The network must be a TCP network name; see func Dial for details. +func (d *Dialer) DialTCP(ctx context.Context, network string, laddr netip.AddrPort, raddr netip.AddrPort) (*TCPConn, error) { + ctx, cancel := d.dialCtx(ctx) + defer cancel() + return dialTCP(ctx, d, network, TCPAddrFromAddrPort(laddr), TCPAddrFromAddrPort(raddr)) +} + +// DialUDP acts like Dial for UDP networks using the provided context. +// +// The provided Context must be non-nil. If the context expires before +// the connection is complete, an error is returned. Once successfully +// connected, any expiration of the context will not affect the +// connection. +// +// The network must be a UDP network name; see func Dial for details. +func (d *Dialer) DialUDP(ctx context.Context, network string, laddr netip.AddrPort, raddr netip.AddrPort) (*UDPConn, error) { + ctx, cancel := d.dialCtx(ctx) + defer cancel() + return dialUDP(ctx, d, network, UDPAddrFromAddrPort(laddr), UDPAddrFromAddrPort(raddr)) +} + +// DialIP acts like Dial for IP networks using the provided context. +// +// The provided Context must be non-nil. If the context expires before +// the connection is complete, an error is returned. Once successfully +// connected, any expiration of the context will not affect the +// connection. +// +// The network must be an IP network name; see func Dial for details. +func (d *Dialer) DialIP(ctx context.Context, network string, laddr netip.Addr, raddr netip.Addr) (*IPConn, error) { + ctx, cancel := d.dialCtx(ctx) + defer cancel() + return dialIP(ctx, d, network, ipAddrFromAddr(laddr), ipAddrFromAddr(raddr)) +} + +// DialUnix acts like Dial for Unix networks using the provided context. +// +// The provided Context must be non-nil. If the context expires before +// the connection is complete, an error is returned. Once successfully +// connected, any expiration of the context will not affect the +// connection. +// +// The network must be a Unix network name; see func Dial for details. +func (d *Dialer) DialUnix(ctx context.Context, network string, laddr *UnixAddr, raddr *UnixAddr) (*UnixConn, error) { + ctx, cancel := d.dialCtx(ctx) + defer cancel() + return dialUnix(ctx, d, network, laddr, raddr) +} + // dialParallel races two copies of dialSerial, giving the first a // head start. It returns the first established connection and // closes the others. Otherwise it returns an error from the first @@ -692,9 +813,11 @@ type ListenConfig struct { // If Control is not nil, it is called after creating the network // connection but before binding it to the operating system. // - // Network and address parameters passed to Control method are not - // necessarily the ones passed to Listen. For example, passing "tcp" to - // Listen will cause the Control function to be called with "tcp4" or "tcp6". + // Network and address parameters passed to Control function are not + // necessarily the ones passed to Listen. Calling Listen with TCP networks + // will cause the Control function to be called with "tcp4" or "tcp6", + // UDP networks become "udp4" or "udp6", IP networks become "ip4" or "ip6", + // and other known networks are passed as-is. Control func(network, address string, c syscall.RawConn) error // KeepAlive specifies the keep-alive period for network @@ -720,7 +843,7 @@ type ListenConfig struct { // If mptcpStatus is set to a value allowing Multipath TCP (MPTCP) to be // used, any call to Listen with "tcp(4|6)" as network will use MPTCP if // supported by the operating system. - mptcpStatus mptcpStatus + mptcpStatus mptcpStatusListen } // MultipathTCP reports whether MPTCP will be used. @@ -745,6 +868,9 @@ func (lc *ListenConfig) SetMultipathTCP(use bool) { // // See func Listen for a description of the network and address // parameters. +// +// The ctx argument is used while resolving the address on which to listen; +// it does not affect the returned Listener. func (lc *ListenConfig) Listen(ctx context.Context, network, address string) (Listener, error) { addrs, err := DefaultResolver.resolveAddrList(ctx, "listen", network, address, nil) if err != nil { @@ -779,6 +905,9 @@ func (lc *ListenConfig) Listen(ctx context.Context, network, address string) (Li // // See func ListenPacket for a description of the network and address // parameters. +// +// The ctx argument is used while resolving the address on which to listen; +// it does not affect the returned PacketConn. func (lc *ListenConfig) ListenPacket(ctx context.Context, network, address string) (PacketConn, error) { addrs, err := DefaultResolver.resolveAddrList(ctx, "listen", network, address, nil) if err != nil { diff --git a/src/net/dial_test.go b/src/net/dial_test.go index b3bedb2fa275c3..07a9b46ddb69ef 100644 --- a/src/net/dial_test.go +++ b/src/net/dial_test.go @@ -11,6 +11,7 @@ import ( "fmt" "internal/testenv" "io" + "net/netip" "os" "runtime" "strings" @@ -231,7 +232,6 @@ func TestDialParallel(t *testing.T) { } for i, tt := range testCases { - i, tt := i, tt t.Run(fmt.Sprint(i), func(t *testing.T) { dialTCP := func(ctx context.Context, network string, laddr, raddr *TCPAddr) (*TCPConn, error) { n := "tcp6" @@ -1064,6 +1064,99 @@ func TestDialerControlContext(t *testing.T) { }) } +func TestDialContext(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("not supported on %s", runtime.GOOS) + case "js", "wasip1": + t.Skipf("skipping: fake net does not support Dialer.ControlContext") + } + + t.Run("StreamDial", func(t *testing.T) { + var err error + for i, network := range []string{"tcp", "tcp4", "tcp6", "unix", "unixpacket"} { + if !testableNetwork(network) { + continue + } + ln := newLocalListener(t, network) + defer ln.Close() + var id int + d := Dialer{ControlContext: func(ctx context.Context, network string, address string, c syscall.RawConn) error { + id = ctx.Value("id").(int) + return controlOnConnSetup(network, address, c) + }} + var c Conn + switch network { + case "tcp", "tcp4", "tcp6": + raddr, err := netip.ParseAddrPort(ln.Addr().String()) + if err != nil { + t.Error(err) + continue + } + c, err = d.DialTCP(context.WithValue(context.Background(), "id", i+1), network, (*TCPAddr)(nil).AddrPort(), raddr) + case "unix", "unixpacket": + raddr, err := ResolveUnixAddr(network, ln.Addr().String()) + if err != nil { + t.Error(err) + continue + } + c, err = d.DialUnix(context.WithValue(context.Background(), "id", i+1), network, nil, raddr) + } + if err != nil { + t.Error(err) + continue + } + if id != i+1 { + t.Errorf("%s: got id %d, want %d", network, id, i+1) + } + c.Close() + } + }) + t.Run("PacketDial", func(t *testing.T) { + var err error + for i, network := range []string{"udp", "udp4", "udp6", "unixgram"} { + if !testableNetwork(network) { + continue + } + c1 := newLocalPacketListener(t, network) + if network == "unixgram" { + defer os.Remove(c1.LocalAddr().String()) + } + defer c1.Close() + var id int + d := Dialer{ControlContext: func(ctx context.Context, network string, address string, c syscall.RawConn) error { + id = ctx.Value("id").(int) + return controlOnConnSetup(network, address, c) + }} + var c2 Conn + switch network { + case "udp", "udp4", "udp6": + raddr, err := netip.ParseAddrPort(c1.LocalAddr().String()) + if err != nil { + t.Error(err) + continue + } + c2, err = d.DialUDP(context.WithValue(context.Background(), "id", i+1), network, (*UDPAddr)(nil).AddrPort(), raddr) + case "unixgram": + raddr, err := ResolveUnixAddr(network, c1.LocalAddr().String()) + if err != nil { + t.Error(err) + continue + } + c2, err = d.DialUnix(context.WithValue(context.Background(), "id", i+1), network, nil, raddr) + } + if err != nil { + t.Error(err) + continue + } + if id != i+1 { + t.Errorf("%s: got id %d, want %d", network, id, i+1) + } + c2.Close() + } + }) +} + // mustHaveExternalNetwork is like testenv.MustHaveExternalNetwork // except on non-Linux, non-mobile builders it permits the test to // run in -short mode. diff --git a/src/net/dnsclient_unix.go b/src/net/dnsclient_unix.go index 54c7dc83bac665..940fcccf7f2139 100644 --- a/src/net/dnsclient_unix.go +++ b/src/net/dnsclient_unix.go @@ -18,6 +18,7 @@ import ( "internal/bytealg" "internal/godebug" "internal/itoa" + "internal/stringslite" "io" "os" "runtime" @@ -487,9 +488,7 @@ func avoidDNS(name string) bool { if name == "" { return true } - if name[len(name)-1] == '.' { - name = name[:len(name)-1] - } + name = stringslite.TrimSuffix(name, ".") return stringsHasSuffixFold(name, ".onion") } @@ -843,8 +842,7 @@ func (r *Resolver) goLookupPTR(ctx context.Context, addr string, order hostLooku } p, server, err := r.lookup(ctx, arpa, dnsmessage.TypePTR, conf) if err != nil { - var dnsErr *DNSError - if errors.As(err, &dnsErr) && dnsErr.IsNotFound { + if dnsErr, ok := errors.AsType[*DNSError](err); ok && dnsErr.IsNotFound { if order == hostLookupDNSFiles { names := lookupStaticAddr(addr) if len(names) > 0 { diff --git a/src/net/dnsclient_unix_test.go b/src/net/dnsclient_unix_test.go index c4e5194a340845..fc1d40f18b6f9a 100644 --- a/src/net/dnsclient_unix_test.go +++ b/src/net/dnsclient_unix_test.go @@ -2028,6 +2028,50 @@ func TestCVE202133195(t *testing.T) { MX: dnsmessage.MustNewName("good.golang.org."), }, }, + dnsmessage.Resource{ + Header: dnsmessage.ResourceHeader{ + Name: dnsmessage.MustNewName("127.0.0.1."), + Type: dnsmessage.TypeMX, + Class: dnsmessage.ClassINET, + Length: 4, + }, + Body: &dnsmessage.MXResource{ + MX: dnsmessage.MustNewName("127.0.0.1."), + }, + }, + dnsmessage.Resource{ + Header: dnsmessage.ResourceHeader{ + Name: dnsmessage.MustNewName("1.2.3.4.5."), + Type: dnsmessage.TypeMX, + Class: dnsmessage.ClassINET, + Length: 4, + }, + Body: &dnsmessage.MXResource{ + MX: dnsmessage.MustNewName("1.2.3.4.5."), + }, + }, + dnsmessage.Resource{ + Header: dnsmessage.ResourceHeader{ + Name: dnsmessage.MustNewName("2001:4860:0:2001::68."), + Type: dnsmessage.TypeMX, + Class: dnsmessage.ClassINET, + Length: 4, + }, + Body: &dnsmessage.MXResource{ + MX: dnsmessage.MustNewName("2001:4860:0:2001::68."), + }, + }, + dnsmessage.Resource{ + Header: dnsmessage.ResourceHeader{ + Name: dnsmessage.MustNewName("2001:4860:0:2001::68%zone."), + Type: dnsmessage.TypeMX, + Class: dnsmessage.ClassINET, + Length: 4, + }, + Body: &dnsmessage.MXResource{ + MX: dnsmessage.MustNewName("2001:4860:0:2001::68%zone."), + }, + }, ) case dnsmessage.TypeNS: r.Answers = append(r.Answers, @@ -2152,25 +2196,37 @@ func TestCVE202133195(t *testing.T) { { name: "MX", f: func(t *testing.T) { - expected := []*MX{ - { - Host: "good.golang.org.", - }, + expected := []string{ + "127.0.0.1.", + "2001:4860:0:2001::68.", + "good.golang.org.", } expectedErr := &DNSError{Err: errMalformedDNSRecordsDetail, Name: "golang.org"} records, err := r.LookupMX(context.Background(), "golang.org") if err.Error() != expectedErr.Error() { t.Fatalf("unexpected error: %s", err) } - if !reflect.DeepEqual(records, expected) { - t.Error("Unexpected record set") + + hosts := func(records []*MX) []string { + var got []string + for _, mx := range records { + got = append(got, mx.Host) + } + slices.Sort(got) + return got + } + + got := hosts(records) + if !slices.Equal(got, expected) { + t.Errorf("Unexpected record set: got %v, want %v", got, expected) } records, err = LookupMX("golang.org") if err.Error() != expectedErr.Error() { t.Fatalf("unexpected error: %s", err) } - if !reflect.DeepEqual(records, expected) { - t.Error("Unexpected record set") + got = hosts(records) + if !slices.Equal(got, expected) { + t.Errorf("Unexpected record set: got %v, want %v", got, expected) } }, }, @@ -2571,8 +2627,7 @@ func TestLongDNSNames(t *testing.T) { } expectedErr := DNSError{Err: errNoSuchHost.Error(), Name: v.req, IsNotFound: true} - var dnsErr *DNSError - errors.As(err, &dnsErr) + dnsErr, _ := errors.AsType[*DNSError](err) if dnsErr == nil || *dnsErr != expectedErr { t.Errorf("%v: Lookup%v: unexpected error: %v", i, testName, err) } @@ -2764,8 +2819,7 @@ func TestLookupOrderFilesNoSuchHost(t *testing.T) { } expectedErr := DNSError{Err: errNoSuchHost.Error(), Name: testName, IsNotFound: true} - var dnsErr *DNSError - errors.As(err, &dnsErr) + dnsErr, _ := errors.AsType[*DNSError](err) if dnsErr == nil || *dnsErr != expectedErr { t.Errorf("Lookup%v: unexpected error: %v", v.name, err) } @@ -2797,8 +2851,7 @@ func TestExtendedRCode(t *testing.T) { r := &Resolver{PreferGo: true, Dial: fake.DialContext} _, _, err := r.tryOneName(context.Background(), getSystemDNSConfig(), "go.dev.", dnsmessage.TypeA) - var dnsErr *DNSError - if !(errors.As(err, &dnsErr) && dnsErr.Err == errServerMisbehaving.Error()) { + if dnsErr, ok := errors.AsType[*DNSError](err); !ok || dnsErr.Err != errServerMisbehaving.Error() { t.Fatalf("r.tryOneName(): unexpected error: %v", err) } } diff --git a/src/net/error_test.go b/src/net/error_test.go index f82e8633464176..8026144c3da7f5 100644 --- a/src/net/error_test.go +++ b/src/net/error_test.go @@ -155,7 +155,6 @@ func TestDialError(t *testing.T) { d := Dialer{Timeout: someTimeout} for i, tt := range dialErrorTests { - i, tt := i, tt t.Run(fmt.Sprint(i), func(t *testing.T) { c, err := d.Dial(tt.network, tt.address) if err == nil { @@ -736,11 +735,6 @@ third: } func TestFileError(t *testing.T) { - switch runtime.GOOS { - case "windows": - t.Skipf("not supported on %s", runtime.GOOS) - } - f, err := os.CreateTemp("", "go-nettest") if err != nil { t.Fatal(err) diff --git a/src/net/example_test.go b/src/net/example_test.go index 2c045d73a24bcb..12c83970940ab0 100644 --- a/src/net/example_test.go +++ b/src/net/example_test.go @@ -334,7 +334,7 @@ func ExampleIP_To16() { // 10.255.0.0 } -func ExampleIP_to4() { +func ExampleIP_To4() { ipv6 := net.IP{0xfc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} ipv4 := net.IPv4(10, 255, 0, 0) diff --git a/src/net/external_test.go b/src/net/external_test.go index 38788efc3d39b7..9b918f8986a1f5 100644 --- a/src/net/external_test.go +++ b/src/net/external_test.go @@ -90,7 +90,6 @@ var ( "www.google.com:80", "%d.%d.%d.%d:http", "www.google.com:http", - "%03d.%03d.%03d.%03d:0080", "[::ffff:%d.%d.%d.%d]:80", "[::ffff:%02x%02x:%02x%02x]:80", "[0:0:0:0:0000:ffff:%d.%d.%d.%d]:80", diff --git a/src/net/fd_fake.go b/src/net/fd_fake.go index ae567acc6994e6..946805ab940b7a 100644 --- a/src/net/fd_fake.go +++ b/src/net/fd_fake.go @@ -104,6 +104,7 @@ func (fd *netFD) accept() (netfd *netFD, err error) { func (fd *netFD) setAddr(laddr, raddr Addr) { fd.laddr = laddr fd.raddr = raddr + // TODO Replace with runtime.AddCleanup. runtime.SetFinalizer(fd, (*netFD).Close) } @@ -111,6 +112,7 @@ func (fd *netFD) Close() error { if fd.fakeNetFD != nil { return fd.fakeNetFD.Close() } + // TODO Replace with runtime.AddCleanup. runtime.SetFinalizer(fd, nil) return fd.pfd.Close() } diff --git a/src/net/fd_plan9.go b/src/net/fd_plan9.go index da41bc0c34cac5..66a12e7d7dabdc 100644 --- a/src/net/fd_plan9.go +++ b/src/net/fd_plan9.go @@ -126,6 +126,11 @@ func (fd *netFD) Close() error { return err } } + if fd.net == "udp" { + // The following line is required to unblock Reads. + // See https://go.dev/issue/72770. + fd.SetReadDeadline(time.Now().Add(-time.Hour)) + } err := fd.ctl.Close() if fd.data != nil { if err1 := fd.data.Close(); err1 != nil && err == nil { diff --git a/src/net/fd_posix.go b/src/net/fd_posix.go index ffb9bcf8b9e8ca..023cd534e4206f 100644 --- a/src/net/fd_posix.go +++ b/src/net/fd_posix.go @@ -26,13 +26,26 @@ type netFD struct { raddr Addr } +func (fd *netFD) name() string { + var ls, rs string + if fd.laddr != nil { + ls = fd.laddr.String() + } + if fd.raddr != nil { + rs = fd.raddr.String() + } + return fd.net + ":" + ls + "->" + rs +} + func (fd *netFD) setAddr(laddr, raddr Addr) { fd.laddr = laddr fd.raddr = raddr + // TODO Replace with runtime.AddCleanup. runtime.SetFinalizer(fd, (*netFD).Close) } func (fd *netFD) Close() error { + // TODO Replace with runtime.AddCleanup. runtime.SetFinalizer(fd, nil) return fd.pfd.Close() } diff --git a/src/net/fd_unix.go b/src/net/fd_unix.go index a8d3a253a97b10..0d4303e2cc0cbe 100644 --- a/src/net/fd_unix.go +++ b/src/net/fd_unix.go @@ -41,17 +41,6 @@ func (fd *netFD) init() error { return fd.pfd.Init(fd.net, true) } -func (fd *netFD) name() string { - var ls, rs string - if fd.laddr != nil { - ls = fd.laddr.String() - } - if fd.raddr != nil { - rs = fd.raddr.String() - } - return fd.net + ":" + ls + "->" + rs -} - func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) (rsa syscall.Sockaddr, ret error) { // Do not need to call fd.writeLock here, // because fd is not yet accessible to user, @@ -85,46 +74,35 @@ func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) (rsa sysc if err := fd.pfd.Init(fd.net, true); err != nil { return nil, err } - if deadline, hasDeadline := ctx.Deadline(); hasDeadline { - fd.pfd.SetWriteDeadline(deadline) - defer fd.pfd.SetWriteDeadline(noDeadline) - } - // Start the "interrupter" goroutine, if this context might be canceled. - // - // The interrupter goroutine waits for the context to be done and - // interrupts the dial (by altering the fd's write deadline, which - // wakes up waitWrite). ctxDone := ctx.Done() if ctxDone != nil { - // Wait for the interrupter goroutine to exit before returning - // from connect. - done := make(chan struct{}) - interruptRes := make(chan error) + if deadline, hasDeadline := ctx.Deadline(); hasDeadline { + fd.pfd.SetWriteDeadline(deadline) + defer fd.pfd.SetWriteDeadline(noDeadline) + } + + // Load the hook function synchronously to prevent a race + // with test code that restores the old value. + testHookCanceledDial := testHookCanceledDial + stop := context.AfterFunc(ctx, func() { + // Force the runtime's poller to immediately give up + // waiting for writability, unblocking waitWrite + // below. + _ = fd.pfd.SetWriteDeadline(aLongTimeAgo) + testHookCanceledDial() + }) defer func() { - close(done) - if ctxErr := <-interruptRes; ctxErr != nil && ret == nil { - // The interrupter goroutine called SetWriteDeadline, - // but the connect code below had returned from - // waitWrite already and did a successful connect (ret - // == nil). Because we've now poisoned the connection - // by making it unwritable, don't return a successful - // dial. This was issue 16523. - ret = mapErr(ctxErr) - fd.Close() // prevent a leak - } - }() - go func() { - select { - case <-ctxDone: - // Force the runtime's poller to immediately give up - // waiting for writability, unblocking waitWrite - // below. - fd.pfd.SetWriteDeadline(aLongTimeAgo) - testHookCanceledDial() - interruptRes <- ctx.Err() - case <-done: - interruptRes <- nil + if !stop() && ret == nil { + // The context.AfterFunc has called or is about to call + // SetWriteDeadline, but the connect code below had + // returned from waitWrite already and did a successful + // connect (ret == nil). Because we've now poisoned the + // connection by making it unwritable, don't return a + // successful dial. This was issue 16523. + ret = mapErr(ctx.Err()) + // The caller closes fd on error, so there's no need to + // wait for the SetWriteDeadline call to return. } }() } diff --git a/src/net/fd_windows.go b/src/net/fd_windows.go index 5d7a1d54c31a32..52985be8e65403 100644 --- a/src/net/fd_windows.go +++ b/src/net/fd_windows.go @@ -53,11 +53,31 @@ func newFD(sysfd syscall.Handle, family, sotype int, net string) (*netFD, error) } func (fd *netFD) init() error { - errcall, err := fd.pfd.Init(fd.net, true) - if errcall != "" { - err = wrapSyscallError(errcall, err) + if err := fd.pfd.Init(fd.net, true); err != nil { + return err } - return err + switch fd.net { + case "udp", "udp4", "udp6": + // Disable reporting of PORT_UNREACHABLE errors. + // See https://go.dev/issue/5834. + ret := uint32(0) + flag := uint32(0) + size := uint32(unsafe.Sizeof(flag)) + err := syscall.WSAIoctl(fd.pfd.Sysfd, syscall.SIO_UDP_CONNRESET, (*byte)(unsafe.Pointer(&flag)), size, nil, 0, &ret, nil, 0) + if err != nil { + return wrapSyscallError("wsaioctl", err) + } + // Disable reporting of NET_UNREACHABLE errors. + // See https://go.dev/issue/68614. + ret = 0 + flag = 0 + size = uint32(unsafe.Sizeof(flag)) + err = syscall.WSAIoctl(fd.pfd.Sysfd, windows.SIO_UDP_NETRESET, (*byte)(unsafe.Pointer(&flag)), size, nil, 0, &ret, nil, 0) + if err != nil { + return wrapSyscallError("wsaioctl", err) + } + } + return nil } // Always returns nil for connected peer address result. @@ -213,9 +233,29 @@ func (fd *netFD) accept() (*netFD, error) { return netfd, nil } -// Unimplemented functions. +// Defined in os package. +func newWindowsFile(h syscall.Handle, name string) *os.File func (fd *netFD) dup() (*os.File, error) { - // TODO: Implement this, perhaps using internal/poll.DupCloseOnExec. - return nil, syscall.EWINDOWS + // Disassociate the IOCP from the socket, + // it is not safe to share a duplicated handle + // that is associated with IOCP. + if err := fd.pfd.DisassociateIOCP(); err != nil { + return nil, err + } + var h syscall.Handle + var syserr error + err := fd.pfd.RawControl(func(fd uintptr) { + h, syserr = dupSocket(syscall.Handle(fd)) + }) + if err != nil { + err = syserr + } + if err != nil { + return nil, err + } + // All WSASocket calls must be match with a syscall.Closesocket call, + // but os.NewFile calls syscall.CloseHandle instead. We need to use + // a hidden function so that the returned file is aware of this fact. + return newWindowsFile(h, fd.name()), nil } diff --git a/src/net/file.go b/src/net/file.go index c13332c188aeef..3e33c9afad6584 100644 --- a/src/net/file.go +++ b/src/net/file.go @@ -6,7 +6,7 @@ package net import "os" -// BUG(mikio): On JS and Windows, the FileConn, FileListener and +// BUG(mikio): On JS, the FileConn, FileListener and // FilePacketConn functions are not implemented. type fileAddr string diff --git a/src/net/file_posix.go b/src/net/file_posix.go new file mode 100644 index 00000000000000..132d03e9e3eda0 --- /dev/null +++ b/src/net/file_posix.go @@ -0,0 +1,104 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix || windows + +package net + +import ( + "internal/poll" + "os" + "syscall" +) + +func newFileFD(f *os.File) (*netFD, error) { + s, err := dupFileSocket(f) + if err != nil { + return nil, err + } + family := syscall.AF_UNSPEC + sotype, err := syscall.GetsockoptInt(s, syscall.SOL_SOCKET, _SO_TYPE) + if err != nil { + poll.CloseFunc(s) + return nil, os.NewSyscallError("getsockopt", err) + } + lsa, _ := syscall.Getsockname(s) + rsa, _ := syscall.Getpeername(s) + switch lsa.(type) { + case *syscall.SockaddrInet4: + family = syscall.AF_INET + case *syscall.SockaddrInet6: + family = syscall.AF_INET6 + case *syscall.SockaddrUnix: + family = syscall.AF_UNIX + default: + poll.CloseFunc(s) + return nil, syscall.EPROTONOSUPPORT + } + fd, err := newFD(s, family, sotype, "") + if err != nil { + poll.CloseFunc(s) + return nil, err + } + laddr := fd.addrFunc()(lsa) + raddr := fd.addrFunc()(rsa) + fd.net = laddr.Network() + if err := fd.init(); err != nil { + fd.Close() + return nil, err + } + fd.setAddr(laddr, raddr) + return fd, nil +} + +func fileConn(f *os.File) (Conn, error) { + fd, err := newFileFD(f) + if err != nil { + return nil, err + } + switch fd.laddr.(type) { + case *TCPAddr: + return newTCPConn(fd, defaultTCPKeepAliveIdle, KeepAliveConfig{}, testPreHookSetKeepAlive, testHookSetKeepAlive), nil + case *UDPAddr: + return newUDPConn(fd), nil + case *IPAddr: + return newIPConn(fd), nil + case *UnixAddr: + return newUnixConn(fd), nil + } + fd.Close() + return nil, syscall.EINVAL +} + +func fileListener(f *os.File) (Listener, error) { + fd, err := newFileFD(f) + if err != nil { + return nil, err + } + switch laddr := fd.laddr.(type) { + case *TCPAddr: + return &TCPListener{fd: fd}, nil + case *UnixAddr: + return &UnixListener{fd: fd, path: laddr.Name, unlink: false}, nil + } + fd.Close() + return nil, syscall.EINVAL +} + +func filePacketConn(f *os.File) (PacketConn, error) { + fd, err := newFileFD(f) + if err != nil { + return nil, err + } + switch fd.laddr.(type) { + case *UDPAddr: + return newUDPConn(fd), nil + case *IPAddr: + return newIPConn(fd), nil + case *UnixAddr: + return newUnixConn(fd), nil + } + fd.Close() + return nil, syscall.EINVAL +} diff --git a/src/net/file_test.go b/src/net/file_test.go index c517af50c5ee2e..51e54ff506bd1f 100644 --- a/src/net/file_test.go +++ b/src/net/file_test.go @@ -29,94 +29,95 @@ var fileConnTests = []struct { func TestFileConn(t *testing.T) { switch runtime.GOOS { - case "plan9", "windows", "js", "wasip1": + case "plan9", "js", "wasip1": t.Skipf("not supported on %s", runtime.GOOS) } for _, tt := range fileConnTests { - if !testableNetwork(tt.network) { - t.Logf("skipping %s test", tt.network) - continue - } + t.Run(tt.network, func(t *testing.T) { + if !testableNetwork(tt.network) { + t.Skipf("skipping %s test", tt.network) + } - var network, address string - switch tt.network { - case "udp": - c := newLocalPacketListener(t, tt.network) - defer c.Close() - network = c.LocalAddr().Network() - address = c.LocalAddr().String() - default: - handler := func(ls *localServer, ln Listener) { - c, err := ln.Accept() - if err != nil { - return - } + var network, address string + switch tt.network { + case "udp": + c := newLocalPacketListener(t, tt.network) defer c.Close() - var b [1]byte - c.Read(b[:]) + network = c.LocalAddr().Network() + address = c.LocalAddr().String() + default: + handler := func(ls *localServer, ln Listener) { + c, err := ln.Accept() + if err != nil { + return + } + defer c.Close() + var b [1]byte + c.Read(b[:]) + } + ls := newLocalServer(t, tt.network) + defer ls.teardown() + if err := ls.buildup(handler); err != nil { + t.Fatal(err) + } + network = ls.Listener.Addr().Network() + address = ls.Listener.Addr().String() } - ls := newLocalServer(t, tt.network) - defer ls.teardown() - if err := ls.buildup(handler); err != nil { + + c1, err := Dial(network, address) + if err != nil { + if perr := parseDialError(err); perr != nil { + t.Error(perr) + } t.Fatal(err) } - network = ls.Listener.Addr().Network() - address = ls.Listener.Addr().String() - } + addr := c1.LocalAddr() - c1, err := Dial(network, address) - if err != nil { - if perr := parseDialError(err); perr != nil { - t.Error(perr) + var f *os.File + switch c1 := c1.(type) { + case *TCPConn: + f, err = c1.File() + case *UDPConn: + f, err = c1.File() + case *UnixConn: + f, err = c1.File() } - t.Fatal(err) - } - addr := c1.LocalAddr() - - var f *os.File - switch c1 := c1.(type) { - case *TCPConn: - f, err = c1.File() - case *UDPConn: - f, err = c1.File() - case *UnixConn: - f, err = c1.File() - } - if err := c1.Close(); err != nil { - if perr := parseCloseError(err, false); perr != nil { - t.Error(perr) + if err := c1.Close(); err != nil { + if perr := parseCloseError(err, false); perr != nil { + t.Error(perr) + } + t.Error(err) } - t.Error(err) - } - if err != nil { - if perr := parseCommonError(err); perr != nil { - t.Error(perr) + if err != nil { + if perr := parseCommonError(err); perr != nil { + t.Error(perr) + } + t.Fatal(err) } - t.Fatal(err) - } - c2, err := FileConn(f) - if err := f.Close(); err != nil { - t.Error(err) - } - if err != nil { - if perr := parseCommonError(err); perr != nil { - t.Error(perr) + c2, err := FileConn(f) + if err := f.Close(); err != nil { + t.Error(err) } - t.Fatal(err) - } - defer c2.Close() + if err != nil { + if perr := parseCommonError(err); perr != nil { + t.Error(perr) + } + t.Fatal(err) + } + defer c2.Close() - if _, err := c2.Write([]byte("FILECONN TEST")); err != nil { - if perr := parseWriteError(err); perr != nil { - t.Error(perr) + if _, err := c2.Write([]byte("FILECONN TEST")); err != nil { + if perr := parseWriteError(err); perr != nil { + t.Error(perr) + } + t.Fatal(err) } - t.Fatal(err) - } - if !reflect.DeepEqual(c2.LocalAddr(), addr) { - t.Fatalf("got %#v; want %#v", c2.LocalAddr(), addr) - } + if !reflect.DeepEqual(c2.LocalAddr(), addr) { + t.Fatalf("got %#v; want %#v", c2.LocalAddr(), addr) + } + }) } } @@ -130,86 +131,87 @@ var fileListenerTests = []struct { func TestFileListener(t *testing.T) { switch runtime.GOOS { - case "plan9", "windows", "js", "wasip1": + case "plan9", "js", "wasip1": t.Skipf("not supported on %s", runtime.GOOS) } for _, tt := range fileListenerTests { - if !testableNetwork(tt.network) { - t.Logf("skipping %s test", tt.network) - continue - } + t.Run(tt.network, func(t *testing.T) { + if !testableNetwork(tt.network) { + t.Skipf("skipping %s test", tt.network) + } - ln1 := newLocalListener(t, tt.network) - switch tt.network { - case "unix", "unixpacket": - defer os.Remove(ln1.Addr().String()) - } - addr := ln1.Addr() + ln1 := newLocalListener(t, tt.network) + switch tt.network { + case "unix", "unixpacket": + defer os.Remove(ln1.Addr().String()) + } + addr := ln1.Addr() - var ( - f *os.File - err error - ) - switch ln1 := ln1.(type) { - case *TCPListener: - f, err = ln1.File() - case *UnixListener: - f, err = ln1.File() - } - switch tt.network { - case "unix", "unixpacket": - defer ln1.Close() // UnixListener.Close calls syscall.Unlink internally - default: - if err := ln1.Close(); err != nil { - t.Error(err) + var ( + f *os.File + err error + ) + switch ln1 := ln1.(type) { + case *TCPListener: + f, err = ln1.File() + case *UnixListener: + f, err = ln1.File() } - } - if err != nil { - if perr := parseCommonError(err); perr != nil { - t.Error(perr) + switch tt.network { + case "unix", "unixpacket": + defer ln1.Close() // UnixListener.Close calls syscall.Unlink internally + default: + if err := ln1.Close(); err != nil { + t.Error(err) + } + } + if err != nil { + if perr := parseCommonError(err); perr != nil { + t.Error(perr) + } + t.Fatal(err) } - t.Fatal(err) - } - ln2, err := FileListener(f) - if err := f.Close(); err != nil { - t.Error(err) - } - if err != nil { - if perr := parseCommonError(err); perr != nil { - t.Error(perr) + ln2, err := FileListener(f) + if err := f.Close(); err != nil { + t.Error(err) } - t.Fatal(err) - } - defer ln2.Close() + if err != nil { + if perr := parseCommonError(err); perr != nil { + t.Error(perr) + } + t.Fatal(err) + } + defer ln2.Close() - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - c, err := Dial(ln2.Addr().Network(), ln2.Addr().String()) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + c, err := Dial(ln2.Addr().Network(), ln2.Addr().String()) + if err != nil { + if perr := parseDialError(err); perr != nil { + t.Error(perr) + } + t.Error(err) + return + } + c.Close() + }() + c, err := ln2.Accept() if err != nil { - if perr := parseDialError(err); perr != nil { + if perr := parseAcceptError(err); perr != nil { t.Error(perr) } - t.Error(err) - return + t.Fatal(err) } c.Close() - }() - c, err := ln2.Accept() - if err != nil { - if perr := parseAcceptError(err); perr != nil { - t.Error(perr) + wg.Wait() + if !reflect.DeepEqual(ln2.Addr(), addr) { + t.Fatalf("got %#v; want %#v", ln2.Addr(), addr) } - t.Fatal(err) - } - c.Close() - wg.Wait() - if !reflect.DeepEqual(ln2.Addr(), addr) { - t.Fatalf("got %#v; want %#v", ln2.Addr(), addr) - } + }) } } @@ -222,74 +224,75 @@ var filePacketConnTests = []struct { func TestFilePacketConn(t *testing.T) { switch runtime.GOOS { - case "plan9", "windows", "js", "wasip1": + case "plan9", "js", "wasip1": t.Skipf("not supported on %s", runtime.GOOS) } for _, tt := range filePacketConnTests { - if !testableNetwork(tt.network) { - t.Logf("skipping %s test", tt.network) - continue - } + t.Run(tt.network, func(t *testing.T) { + if !testableNetwork(tt.network) { + t.Skipf("skipping %s test", tt.network) + } - c1 := newLocalPacketListener(t, tt.network) - switch tt.network { - case "unixgram": - defer os.Remove(c1.LocalAddr().String()) - } - addr := c1.LocalAddr() + c1 := newLocalPacketListener(t, tt.network) + switch tt.network { + case "unixgram": + defer os.Remove(c1.LocalAddr().String()) + } + addr := c1.LocalAddr() - var ( - f *os.File - err error - ) - switch c1 := c1.(type) { - case *UDPConn: - f, err = c1.File() - case *UnixConn: - f, err = c1.File() - } - if err := c1.Close(); err != nil { - if perr := parseCloseError(err, false); perr != nil { - t.Error(perr) + var ( + f *os.File + err error + ) + switch c1 := c1.(type) { + case *UDPConn: + f, err = c1.File() + case *UnixConn: + f, err = c1.File() } - t.Error(err) - } - if err != nil { - if perr := parseCommonError(err); perr != nil { - t.Error(perr) + if err := c1.Close(); err != nil { + if perr := parseCloseError(err, false); perr != nil { + t.Error(perr) + } + t.Error(err) + } + if err != nil { + if perr := parseCommonError(err); perr != nil { + t.Error(perr) + } + t.Fatal(err) } - t.Fatal(err) - } - c2, err := FilePacketConn(f) - if err := f.Close(); err != nil { - t.Error(err) - } - if err != nil { - if perr := parseCommonError(err); perr != nil { - t.Error(perr) + c2, err := FilePacketConn(f) + if err := f.Close(); err != nil { + t.Error(err) } - t.Fatal(err) - } - defer c2.Close() + if err != nil { + if perr := parseCommonError(err); perr != nil { + t.Error(perr) + } + t.Fatal(err) + } + defer c2.Close() - if _, err := c2.WriteTo([]byte("FILEPACKETCONN TEST"), addr); err != nil { - if perr := parseWriteError(err); perr != nil { - t.Error(perr) + if _, err := c2.WriteTo([]byte("FILEPACKETCONN TEST"), addr); err != nil { + if perr := parseWriteError(err); perr != nil { + t.Error(perr) + } + t.Fatal(err) } - t.Fatal(err) - } - if !reflect.DeepEqual(c2.LocalAddr(), addr) { - t.Fatalf("got %#v; want %#v", c2.LocalAddr(), addr) - } + if !reflect.DeepEqual(c2.LocalAddr(), addr) { + t.Fatalf("got %#v; want %#v", c2.LocalAddr(), addr) + } + }) } } // Issue 24483. func TestFileCloseRace(t *testing.T) { switch runtime.GOOS { - case "plan9", "windows", "js", "wasip1": + case "plan9", "js", "wasip1": t.Skipf("not supported on %s", runtime.GOOS) } if !testableNetwork("tcp") { diff --git a/src/net/file_unix.go b/src/net/file_unix.go index c0212cef65dba6..d56da9003747ab 100644 --- a/src/net/file_unix.go +++ b/src/net/file_unix.go @@ -12,7 +12,9 @@ import ( "syscall" ) -func dupSocket(f *os.File) (int, error) { +const _SO_TYPE = syscall.SO_TYPE + +func dupFileSocket(f *os.File) (int, error) { s, call, err := poll.DupCloseOnExec(int(f.Fd())) if err != nil { if call != "" { @@ -26,94 +28,3 @@ func dupSocket(f *os.File) (int, error) { } return s, nil } - -func newFileFD(f *os.File) (*netFD, error) { - s, err := dupSocket(f) - if err != nil { - return nil, err - } - family := syscall.AF_UNSPEC - sotype, err := syscall.GetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_TYPE) - if err != nil { - poll.CloseFunc(s) - return nil, os.NewSyscallError("getsockopt", err) - } - lsa, _ := syscall.Getsockname(s) - rsa, _ := syscall.Getpeername(s) - switch lsa.(type) { - case *syscall.SockaddrInet4: - family = syscall.AF_INET - case *syscall.SockaddrInet6: - family = syscall.AF_INET6 - case *syscall.SockaddrUnix: - family = syscall.AF_UNIX - default: - poll.CloseFunc(s) - return nil, syscall.EPROTONOSUPPORT - } - fd, err := newFD(s, family, sotype, "") - if err != nil { - poll.CloseFunc(s) - return nil, err - } - laddr := fd.addrFunc()(lsa) - raddr := fd.addrFunc()(rsa) - fd.net = laddr.Network() - if err := fd.init(); err != nil { - fd.Close() - return nil, err - } - fd.setAddr(laddr, raddr) - return fd, nil -} - -func fileConn(f *os.File) (Conn, error) { - fd, err := newFileFD(f) - if err != nil { - return nil, err - } - switch fd.laddr.(type) { - case *TCPAddr: - return newTCPConn(fd, defaultTCPKeepAliveIdle, KeepAliveConfig{}, testPreHookSetKeepAlive, testHookSetKeepAlive), nil - case *UDPAddr: - return newUDPConn(fd), nil - case *IPAddr: - return newIPConn(fd), nil - case *UnixAddr: - return newUnixConn(fd), nil - } - fd.Close() - return nil, syscall.EINVAL -} - -func fileListener(f *os.File) (Listener, error) { - fd, err := newFileFD(f) - if err != nil { - return nil, err - } - switch laddr := fd.laddr.(type) { - case *TCPAddr: - return &TCPListener{fd: fd}, nil - case *UnixAddr: - return &UnixListener{fd: fd, path: laddr.Name, unlink: false}, nil - } - fd.Close() - return nil, syscall.EINVAL -} - -func filePacketConn(f *os.File) (PacketConn, error) { - fd, err := newFileFD(f) - if err != nil { - return nil, err - } - switch fd.laddr.(type) { - case *UDPAddr: - return newUDPConn(fd), nil - case *IPAddr: - return newIPConn(fd), nil - case *UnixAddr: - return newUnixConn(fd), nil - } - fd.Close() - return nil, syscall.EINVAL -} diff --git a/src/net/file_wasip1_test.go b/src/net/file_wasip1_test.go index 4f4259069d1f73..91f57eecdacf6e 100644 --- a/src/net/file_wasip1_test.go +++ b/src/net/file_wasip1_test.go @@ -20,7 +20,7 @@ import ( // socket extensions with the net package using net.FileConn/net.FileListener. // // Note that the creation of net.Conn and net.Listener values for TCP sockets -// has an end-to-end test in src/runtime/internal/wasitest, here we are only +// has an end-to-end test in src/internal/runtime/wasitest, here we are only // verifying the code paths specific to UDP, and error handling for invalid use // of the functions. diff --git a/src/net/file_windows.go b/src/net/file_windows.go index 241fa17617c2bf..b4eb00e56448a5 100644 --- a/src/net/file_windows.go +++ b/src/net/file_windows.go @@ -5,21 +5,45 @@ package net import ( + "internal/syscall/windows" "os" "syscall" ) -func fileConn(f *os.File) (Conn, error) { - // TODO: Implement this - return nil, syscall.EWINDOWS -} +const _SO_TYPE = windows.SO_TYPE -func fileListener(f *os.File) (Listener, error) { - // TODO: Implement this - return nil, syscall.EWINDOWS +func dupSocket(h syscall.Handle) (syscall.Handle, error) { + var info syscall.WSAProtocolInfo + err := windows.WSADuplicateSocket(h, uint32(syscall.Getpid()), &info) + if err != nil { + return 0, err + } + return windows.WSASocket(-1, -1, -1, &info, 0, windows.WSA_FLAG_OVERLAPPED|windows.WSA_FLAG_NO_HANDLE_INHERIT) } -func filePacketConn(f *os.File) (PacketConn, error) { - // TODO: Implement this - return nil, syscall.EWINDOWS +func dupFileSocket(f *os.File) (syscall.Handle, error) { + // Call Fd to disassociate the IOCP from the handle, + // it is not safe to share a duplicated handle + // that is associated with IOCP. + // Don't use the returned fd, as it might be closed + // if f happens to be the last reference to the file. + f.Fd() + + sc, err := f.SyscallConn() + if err != nil { + return 0, err + } + + var h syscall.Handle + var syserr error + err = sc.Control(func(fd uintptr) { + h, syserr = dupSocket(syscall.Handle(fd)) + }) + if err != nil { + err = syserr + } + if err != nil { + return 0, err + } + return h, nil } diff --git a/src/net/http/async_test.go b/src/net/http/async_test.go new file mode 100644 index 00000000000000..545cbcf544a5e5 --- /dev/null +++ b/src/net/http/async_test.go @@ -0,0 +1,52 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http_test + +import ( + "errors" + "internal/synctest" +) + +var errStillRunning = errors.New("async op still running") + +type asyncResult[T any] struct { + donec chan struct{} + res T + err error +} + +// runAsync runs f in a new goroutine. +// It returns an asyncResult which acts as a future. +// +// Must be called from within a synctest bubble. +func runAsync[T any](f func() (T, error)) *asyncResult[T] { + r := &asyncResult[T]{ + donec: make(chan struct{}), + } + go func() { + defer close(r.donec) + r.res, r.err = f() + }() + synctest.Wait() + return r +} + +// done reports whether the function has returned. +func (r *asyncResult[T]) done() bool { + _, err := r.result() + return err != errStillRunning +} + +// result returns the result of the function. +// If the function hasn't completed yet, it returns errStillRunning. +func (r *asyncResult[T]) result() (T, error) { + select { + case <-r.donec: + return r.res, r.err + default: + var zero T + return zero, errStillRunning + } +} diff --git a/src/net/http/cgi/cgi_main.go b/src/net/http/cgi/cgi_main.go index 033036d07fb17a..1435f1b779c468 100644 --- a/src/net/http/cgi/cgi_main.go +++ b/src/net/http/cgi/cgi_main.go @@ -7,6 +7,7 @@ package cgi import ( "fmt" "io" + "maps" "net/http" "os" "path" @@ -63,22 +64,12 @@ func testCGI() { fmt.Printf("test=Hello CGI\r\n") - keys := make([]string, 0, len(params)) - for k := range params { - keys = append(keys, k) - } - slices.Sort(keys) - for _, key := range keys { + for _, key := range slices.Sorted(maps.Keys(params)) { fmt.Printf("param-%s=%s\r\n", key, params.Get(key)) } envs := envMap(os.Environ()) - keys = make([]string, 0, len(envs)) - for k := range envs { - keys = append(keys, k) - } - slices.Sort(keys) - for _, key := range keys { + for _, key := range slices.Sorted(maps.Keys(envs)) { fmt.Printf("env-%s=%s\r\n", key, envs[key]) } diff --git a/src/net/http/client.go b/src/net/http/client.go index cbf7c545019b8c..d6a801073553f7 100644 --- a/src/net/http/client.go +++ b/src/net/http/client.go @@ -172,8 +172,13 @@ func refererForURL(lastReq, newReq *url.URL, explicitRef string) string { // didTimeout is non-nil only if err != nil. func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { + cookieURL := req.URL + if req.Host != "" { + cookieURL = cloneURL(cookieURL) + cookieURL.Host = req.Host + } if c.Jar != nil { - for _, cookie := range c.Jar.Cookies(req.URL) { + for _, cookie := range c.Jar.Cookies(cookieURL) { req.AddCookie(cookie) } } @@ -183,7 +188,7 @@ func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTime } if c.Jar != nil { if rc := resp.Cookies(); len(rc) > 0 { - c.Jar.SetCookies(req.URL, rc) + c.Jar.SetCookies(cookieURL, rc) } } return resp, nil, nil @@ -388,15 +393,12 @@ func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTi } stopTimerCh := make(chan struct{}) - var once sync.Once - stopTimer = func() { - once.Do(func() { - close(stopTimerCh) - if cancelCtx != nil { - cancelCtx() - } - }) - } + stopTimer = sync.OnceFunc(func() { + close(stopTimerCh) + if cancelCtx != nil { + cancelCtx() + } + }) timer := time.NewTimer(time.Until(deadline)) var timedOut atomic.Bool @@ -613,8 +615,9 @@ func (c *Client) do(req *Request) (retres *Response, reterr error) { reqBodyClosed = false // have we closed the current req.Body? // Redirect behavior: - redirectMethod string - includeBody bool + redirectMethod string + includeBody = true + stripSensitiveHeaders = false ) uerr := func(err error) error { // the body may have been closed already by c.send() @@ -674,6 +677,7 @@ func (c *Client) do(req *Request) (retres *Response, reterr error) { resp.closeBody() return nil, uerr(err) } + req.GetBody = ireq.GetBody req.ContentLength = ireq.ContentLength } @@ -681,8 +685,12 @@ func (c *Client) do(req *Request) (retres *Response, reterr error) { // in case the user set Referer on their first request. // If they really want to override, they can do it in // their CheckRedirect func. - copyHeaders(req) - + if !stripSensitiveHeaders && reqs[0].URL.Host != req.URL.Host { + if !shouldCopyHeaderOnRedirect(reqs[0].URL, req.URL) { + stripSensitiveHeaders = true + } + } + copyHeaders(req, stripSensitiveHeaders, !includeBody) // Add the Referer header from the most recent // request URL to the new one, if it's not https->http: if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL, req.Header.Get("Referer")); ref != "" { @@ -731,11 +739,16 @@ func (c *Client) do(req *Request) (retres *Response, reterr error) { return nil, uerr(err) } - var shouldRedirect bool - redirectMethod, shouldRedirect, includeBody = redirectBehavior(req.Method, resp, reqs[0]) + var shouldRedirect, includeBodyOnHop bool + redirectMethod, shouldRedirect, includeBodyOnHop = redirectBehavior(req.Method, resp, reqs[0]) if !shouldRedirect { return resp, nil } + if !includeBodyOnHop { + // Once a hop drops the body, we never send it again + // (because we're now handling a redirect for a request with no body). + includeBody = false + } req.closeBody() } @@ -744,7 +757,7 @@ func (c *Client) do(req *Request) (retres *Response, reterr error) { // makeHeadersCopier makes a function that copies headers from the // initial Request, ireq. For every redirect, this function must be called // so that it can copy headers into the upcoming Request. -func (c *Client) makeHeadersCopier(ireq *Request) func(*Request) { +func (c *Client) makeHeadersCopier(ireq *Request) func(req *Request, stripSensitiveHeaders, stripBodyHeaders bool) { // The headers to copy are from the very initial request. // We use a closured callback to keep a reference to these original headers. var ( @@ -758,8 +771,7 @@ func (c *Client) makeHeadersCopier(ireq *Request) func(*Request) { } } - preq := ireq // The previous request - return func(req *Request) { + return func(req *Request, stripSensitiveHeaders, stripBodyHeaders bool) { // If Jar is present and there was some initial cookies provided // via the request header, then we may need to alter the initial // cookies as we follow redirects since each redirect may end up @@ -796,12 +808,25 @@ func (c *Client) makeHeadersCopier(ireq *Request) func(*Request) { // Copy the initial request's Header values // (at least the safe ones). for k, vv := range ireqhdr { - if shouldCopyHeaderOnRedirect(k, preq.URL, req.URL) { + sensitive := false + body := false + switch CanonicalHeaderKey(k) { + case "Authorization", "Www-Authenticate", "Cookie", "Cookie2", + "Proxy-Authorization", "Proxy-Authenticate": + sensitive = true + + case "Content-Encoding", "Content-Language", "Content-Location", + "Content-Type": + // Headers relating to the body which is removed for + // POST to GET redirects + // https://fetch.spec.whatwg.org/#http-redirect-fetch + body = true + + } + if !(sensitive && stripSensitiveHeaders) && !(body && stripBodyHeaders) { req.Header[k] = vv } } - - preq = req // Update previous Request with the current request } } @@ -844,7 +869,7 @@ func Post(url, contentType string, body io.Reader) (resp *Response, err error) { // To make a request with a specified context.Context, use [NewRequestWithContext] // and [Client.Do]. // -// See the Client.Do method documentation for details on how redirects +// See the [Client.Do] method documentation for details on how redirects // are handled. func (c *Client) Post(url, contentType string, body io.Reader) (resp *Response, err error) { req, err := NewRequest("POST", url, body) @@ -884,7 +909,7 @@ func PostForm(url string, data url.Values) (resp *Response, err error) { // When err is nil, resp always contains a non-nil resp.Body. // Caller should close resp.Body when done reading from it. // -// See the Client.Do method documentation for details on how redirects +// See the [Client.Do] method documentation for details on how redirects // are handled. // // To make a request with a specified context.Context, use [NewRequestWithContext] @@ -977,28 +1002,23 @@ func (b *cancelTimerBody) Close() error { return err } -func shouldCopyHeaderOnRedirect(headerKey string, initial, dest *url.URL) bool { - switch CanonicalHeaderKey(headerKey) { - case "Authorization", "Www-Authenticate", "Cookie", "Cookie2": - // Permit sending auth/cookie headers from "foo.com" - // to "sub.foo.com". - - // Note that we don't send all cookies to subdomains - // automatically. This function is only used for - // Cookies set explicitly on the initial outgoing - // client request. Cookies automatically added via the - // CookieJar mechanism continue to follow each - // cookie's scope as set by Set-Cookie. But for - // outgoing requests with the Cookie header set - // directly, we don't know their scope, so we assume - // it's for *.domain.com. - - ihost := idnaASCIIFromURL(initial) - dhost := idnaASCIIFromURL(dest) - return isDomainOrSubdomain(dhost, ihost) - } - // All other headers are copied: - return true +func shouldCopyHeaderOnRedirect(initial, dest *url.URL) bool { + // Permit sending auth/cookie headers from "foo.com" + // to "sub.foo.com". + + // Note that we don't send all cookies to subdomains + // automatically. This function is only used for + // Cookies set explicitly on the initial outgoing + // client request. Cookies automatically added via the + // CookieJar mechanism continue to follow each + // cookie's scope as set by Set-Cookie. But for + // outgoing requests with the Cookie header set + // directly, we don't know their scope, so we assume + // it's for *.domain.com. + + ihost := idnaASCIIFromURL(initial) + dhost := idnaASCIIFromURL(dest) + return isDomainOrSubdomain(dhost, ihost) } // isDomainOrSubdomain reports whether sub is a subdomain (or exact diff --git a/src/net/http/client_test.go b/src/net/http/client_test.go index 1faa1516479e75..d184f720319ce4 100644 --- a/src/net/http/client_test.go +++ b/src/net/http/client_test.go @@ -346,7 +346,7 @@ func TestPostRedirects(t *testing.T) { `POST /?code=307&next=303,308,302 "c307"`, `POST /?code=303&next=308,302 "c307"`, `GET /?code=308&next=302 ""`, - `GET /?code=302 "c307"`, + `GET /?code=302 ""`, `GET / ""`, `POST /?code=308&next=302,301 "c308"`, `POST /?code=302&next=301 "c308"`, @@ -376,7 +376,7 @@ func TestDeleteRedirects(t *testing.T) { `DELETE /?code=301&next=302,308 "c301"`, `GET /?code=302&next=308 ""`, `GET /?code=308 ""`, - `GET / "c301"`, + `GET / ""`, `DELETE /?code=302&next=302 "c302"`, `GET /?code=302 ""`, `GET / ""`, @@ -385,7 +385,7 @@ func TestDeleteRedirects(t *testing.T) { `DELETE /?code=307&next=301,308,303,302,304 "c307"`, `DELETE /?code=301&next=308,303,302,304 "c307"`, `GET /?code=308&next=303,302,304 ""`, - `GET /?code=303&next=302,304 "c307"`, + `GET /?code=303&next=302,304 ""`, `GET /?code=302&next=304 ""`, `GET /?code=304 ""`, `DELETE /?code=308&next=307 "c308"`, @@ -585,6 +585,36 @@ var echoCookiesRedirectHandler = HandlerFunc(func(w ResponseWriter, r *Request) } }) +func TestHostMismatchCookies(t *testing.T) { run(t, testHostMismatchCookies) } +func testHostMismatchCookies(t *testing.T, mode testMode) { + ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + for _, c := range r.Cookies() { + c.Value = "SetOnServer" + SetCookie(w, c) + } + })).ts + + reqURL, _ := url.Parse(ts.URL) + hostURL := *reqURL + hostURL.Host = "cookies.example.com" + + c := ts.Client() + c.Jar = new(TestJar) + c.Jar.SetCookies(reqURL, []*Cookie{{Name: "First", Value: "SetOnClient"}}) + c.Jar.SetCookies(&hostURL, []*Cookie{{Name: "Second", Value: "SetOnClient"}}) + + req, _ := NewRequest("GET", ts.URL, NoBody) + req.Host = hostURL.Host + resp, err := c.Do(req) + if err != nil { + t.Fatalf("Get: %v", err) + } + resp.Body.Close() + + matchReturnedCookies(t, []*Cookie{{Name: "First", Value: "SetOnClient"}}, c.Jar.Cookies(reqURL)) + matchReturnedCookies(t, []*Cookie{{Name: "Second", Value: "SetOnServer"}}, c.Jar.Cookies(&hostURL)) +} + func TestClientSendsCookieFromJar(t *testing.T) { defer afterTest(t) tr := &recordingTransport{} @@ -751,7 +781,7 @@ func testStreamingGet(t *testing.T, mode testMode) { var buf [10]byte for _, str := range []string{"i", "am", "also", "known", "as", "comet"} { say <- str - n, err := io.ReadFull(res.Body, buf[0:len(str)]) + n, err := io.ReadFull(res.Body, buf[:len(str)]) if err != nil { t.Fatalf("ReadFull on %q: %v", str, err) } @@ -834,8 +864,11 @@ func testClientInsecureTransport(t *testing.T, mode testMode) { for _, insecure := range []bool{true, false} { c.Transport.(*Transport).TLSClientConfig = &tls.Config{ InsecureSkipVerify: insecure, + NextProtos: cst.tr.TLSClientConfig.NextProtos, } - res, err := c.Get(ts.URL) + req, _ := NewRequest("GET", ts.URL, nil) + req.Header.Set("Connection", "close") // don't reuse this connection + res, err := c.Do(req) if (err == nil) != insecure { t.Errorf("insecure=%v: got unexpected err=%v", insecure, err) } @@ -1224,7 +1257,7 @@ func testClientTimeout(t *testing.T, mode testMode) { t.Logf("timeout before response received") continue } - if runtime.GOOS == "windows" && strings.HasPrefix(runtime.GOARCH, "arm") { + if runtime.GOOS == "windows" && runtime.GOARCH == "arm64" { testenv.SkipFlaky(t, 43120) } t.Fatal(err) @@ -1253,7 +1286,7 @@ func testClientTimeout(t *testing.T, mode testMode) { t.Errorf("ReadAll error = %q; expected some context.DeadlineExceeded", err) } if got := ne.Error(); !strings.Contains(got, "(Client.Timeout") { - if runtime.GOOS == "windows" && strings.HasPrefix(runtime.GOARCH, "arm") { + if runtime.GOOS == "windows" && runtime.GOARCH == "arm64" { testenv.SkipFlaky(t, 43120) } t.Errorf("error string = %q; missing timeout substring", got) @@ -1299,7 +1332,7 @@ func testClientTimeout_Headers(t *testing.T, mode testMode) { t.Errorf("ReadAll error = %q; expected some context.DeadlineExceeded", err) } if got := ne.Error(); !strings.Contains(got, "Client.Timeout exceeded") { - if runtime.GOOS == "windows" && strings.HasPrefix(runtime.GOARCH, "arm") { + if runtime.GOOS == "windows" && runtime.GOARCH == "arm64" { testenv.SkipFlaky(t, 43120) } t.Errorf("error string = %q; missing timeout substring", got) @@ -1536,6 +1569,91 @@ func testClientCopyHeadersOnRedirect(t *testing.T, mode testMode) { } } +// Issue #70530: Once we strip a header on a redirect to a different host, +// the header should stay stripped across any further redirects. +func TestClientStripHeadersOnRepeatedRedirect(t *testing.T) { + run(t, testClientStripHeadersOnRepeatedRedirect) +} +func testClientStripHeadersOnRepeatedRedirect(t *testing.T, mode testMode) { + var proto string + ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + if r.Host+r.URL.Path != "a.example.com/" { + if h := r.Header.Get("Authorization"); h != "" { + t.Errorf("on request to %v%v, Authorization=%q, want no header", r.Host, r.URL.Path, h) + } else if h := r.Header.Get("Proxy-Authorization"); h != "" { + t.Errorf("on request to %v%v, Proxy-Authorization=%q, want no header", r.Host, r.URL.Path, h) + } + } + // Follow a chain of redirects from a to b and back to a. + // The Authorization header is stripped on the first redirect to b, + // and stays stripped even if we're sent back to a. + switch r.Host + r.URL.Path { + case "a.example.com/": + Redirect(w, r, proto+"://b.example.com/", StatusFound) + case "b.example.com/": + Redirect(w, r, proto+"://b.example.com/redirect", StatusFound) + case "b.example.com/redirect": + Redirect(w, r, proto+"://a.example.com/redirect", StatusFound) + case "a.example.com/redirect": + w.Header().Set("X-Done", "true") + default: + t.Errorf("unexpected request to %v", r.URL) + } + })).ts + proto, _, _ = strings.Cut(ts.URL, ":") + + c := ts.Client() + c.Transport.(*Transport).Dial = func(_ string, _ string) (net.Conn, error) { + return net.Dial("tcp", ts.Listener.Addr().String()) + } + + req, _ := NewRequest("GET", proto+"://a.example.com/", nil) + req.Header.Add("Cookie", "foo=bar") + req.Header.Add("Authorization", "secretpassword") + req.Header.Add("Proxy-Authorization", "secretpassword") + res, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if res.Header.Get("X-Done") != "true" { + t.Fatalf("response missing expected header: X-Done=true") + } +} + +func TestClientStripHeadersOnPostToGetRedirect(t *testing.T) { + run(t, testClientStripHeadersOnPostToGetRedirect) +} +func testClientStripHeadersOnPostToGetRedirect(t *testing.T, mode testMode) { + ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + if r.Method == "POST" { + Redirect(w, r, "/redirected", StatusFound) + return + } else if r.Method != "GET" { + t.Errorf("unexpected request method: %v", r.Method) + return + } + for key, val := range r.Header { + if strings.HasPrefix(key, "Content-") { + t.Errorf("unexpected request body header after redirect: %v: %v", key, val) + } + } + })).ts + + c := ts.Client() + + req, _ := NewRequest("POST", ts.URL, strings.NewReader("hello world")) + req.Header.Set("Content-Encoding", "a") + req.Header.Set("Content-Language", "b") + req.Header.Set("Content-Length", "c") + req.Header.Set("Content-Type", "d") + res, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() +} + // Issue 22233: copy host when Client follows a relative redirect. func TestClientCopyHostOnRedirect(t *testing.T) { run(t, testClientCopyHostOnRedirect) } func testClientCopyHostOnRedirect(t *testing.T, mode testMode) { @@ -1702,43 +1820,39 @@ func testClientAltersCookiesOnRedirect(t *testing.T, mode testMode) { // Part of Issue 4800 func TestShouldCopyHeaderOnRedirect(t *testing.T) { tests := []struct { - header string initialURL string destURL string want bool }{ - {"User-Agent", "http://foo.com/", "http://bar.com/", true}, - {"X-Foo", "http://foo.com/", "http://bar.com/", true}, - // Sensitive headers: - {"cookie", "http://foo.com/", "http://bar.com/", false}, - {"cookie2", "http://foo.com/", "http://bar.com/", false}, - {"authorization", "http://foo.com/", "http://bar.com/", false}, - {"authorization", "http://foo.com/", "https://foo.com/", true}, - {"authorization", "http://foo.com:1234/", "http://foo.com:4321/", true}, - {"www-authenticate", "http://foo.com/", "http://bar.com/", false}, - {"authorization", "http://foo.com/", "http://[::1%25.foo.com]/", false}, + {"http://foo.com/", "http://bar.com/", false}, + {"http://foo.com/", "http://bar.com/", false}, + {"http://foo.com/", "http://bar.com/", false}, + {"http://foo.com/", "https://foo.com/", true}, + {"http://foo.com:1234/", "http://foo.com:4321/", true}, + {"http://foo.com/", "http://bar.com/", false}, + {"http://foo.com/", "http://[::1%25.foo.com]/", false}, // But subdomains should work: - {"www-authenticate", "http://foo.com/", "http://foo.com/", true}, - {"www-authenticate", "http://foo.com/", "http://sub.foo.com/", true}, - {"www-authenticate", "http://foo.com/", "http://notfoo.com/", false}, - {"www-authenticate", "http://foo.com/", "https://foo.com/", true}, - {"www-authenticate", "http://foo.com/", "http://foo.com/", true}, - {"www-authenticate", "http://foo.com/", "http://sub.foo.com/", true}, - {"www-authenticate", "http://foo.com:443/", "https://foo.com/", true}, - {"www-authenticate", "http://foo.com:443/", "https://sub.foo.com/", true}, - {"www-authenticate", "http://foo.com:1234/", "http://foo.com/", true}, - - {"authorization", "http://foo.com/", "http://foo.com/", true}, - {"authorization", "http://foo.com/", "http://sub.foo.com/", true}, - {"authorization", "http://foo.com/", "http://notfoo.com/", false}, - {"authorization", "http://foo.com/", "https://foo.com/", true}, - {"authorization", "http://foo.com/", "http://foo.com/", true}, - {"authorization", "http://foo.com/", "http://sub.foo.com/", true}, - {"authorization", "http://foo.com:443/", "https://foo.com/", true}, - {"authorization", "http://foo.com:443/", "https://sub.foo.com/", true}, - {"authorization", "http://foo.com:1234/", "http://foo.com/", true}, + {"http://foo.com/", "http://foo.com/", true}, + {"http://foo.com/", "http://sub.foo.com/", true}, + {"http://foo.com/", "http://notfoo.com/", false}, + {"http://foo.com/", "https://foo.com/", true}, + {"http://foo.com/", "http://foo.com/", true}, + {"http://foo.com/", "http://sub.foo.com/", true}, + {"http://foo.com:443/", "https://foo.com/", true}, + {"http://foo.com:443/", "https://sub.foo.com/", true}, + {"http://foo.com:1234/", "http://foo.com/", true}, + + {"http://foo.com/", "http://foo.com/", true}, + {"http://foo.com/", "http://sub.foo.com/", true}, + {"http://foo.com/", "http://notfoo.com/", false}, + {"http://foo.com/", "https://foo.com/", true}, + {"http://foo.com/", "http://foo.com/", true}, + {"http://foo.com/", "http://sub.foo.com/", true}, + {"http://foo.com:443/", "https://foo.com/", true}, + {"http://foo.com:443/", "https://sub.foo.com/", true}, + {"http://foo.com:1234/", "http://foo.com/", true}, } for i, tt := range tests { u0, err := url.Parse(tt.initialURL) @@ -1751,10 +1865,10 @@ func TestShouldCopyHeaderOnRedirect(t *testing.T) { t.Errorf("%d. dest URL %q parse error: %v", i, tt.destURL, err) continue } - got := Export_shouldCopyHeaderOnRedirect(tt.header, u0, u1) + got := Export_shouldCopyHeaderOnRedirect(u0, u1) if got != tt.want { - t.Errorf("%d. shouldCopyHeaderOnRedirect(%q, %q => %q) = %v; want %v", - i, tt.header, tt.initialURL, tt.destURL, got, tt.want) + t.Errorf("%d. shouldCopyHeaderOnRedirect(%q => %q) = %v; want %v", + i, tt.initialURL, tt.destURL, got, tt.want) } } } @@ -1914,6 +2028,61 @@ func testTransportBodyReadError(t *testing.T, mode testMode) { } } +// Make sure the retries copies the GetBody in the request. +func TestRedirectGetBody(t *testing.T) { run(t, testRedirectGetBody) } + +func testRedirectGetBody(t *testing.T, mode testMode) { + ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + b, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + if err = r.Body.Close(); err != nil { + t.Error(err) + } + if s := string(b); s != "hello" { + t.Errorf("expected hello, got %s", s) + } + if r.URL.Path == "/first" { + Redirect(w, r, "/second", StatusTemporaryRedirect) + return + } + w.Write([]byte("world")) + })).ts + c := ts.Client() + c.Transport = &roundTripperGetBody{c.Transport, t} + req, err := NewRequest("POST", ts.URL+"/first", strings.NewReader("hello")) + if err != nil { + t.Fatal(err) + } + res, err := c.Do(req.WithT(t)) + if err != nil { + t.Fatal(err) + } + b, err := io.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + if err = res.Body.Close(); err != nil { + t.Fatal(err) + } + if s := string(b); s != "world" { + t.Fatalf("expected world, got %s", s) + } +} + +type roundTripperGetBody struct { + Transport RoundTripper + t *testing.T +} + +func (r *roundTripperGetBody) RoundTrip(req *Request) (*Response, error) { + if req.GetBody == nil { + r.t.Error("missing Request.GetBody") + } + return r.Transport.RoundTrip(req) +} + type roundTripperWithoutCloseIdle struct{} func (roundTripperWithoutCloseIdle) RoundTrip(*Request) (*Response, error) { panic("unused") } diff --git a/src/net/http/clientserver_test.go b/src/net/http/clientserver_test.go index 0c2142a0639c74..8665bae38ad0f2 100644 --- a/src/net/http/clientserver_test.go +++ b/src/net/http/clientserver_test.go @@ -17,6 +17,7 @@ import ( "hash" "io" "log" + "maps" "net" . "net/http" "net/http/httptest" @@ -32,15 +33,17 @@ import ( "sync" "sync/atomic" "testing" + "testing/synctest" "time" ) type testMode string const ( - http1Mode = testMode("h1") // HTTP/1.1 - https1Mode = testMode("https1") // HTTPS/1.1 - http2Mode = testMode("h2") // HTTP/2 + http1Mode = testMode("h1") // HTTP/1.1 + https1Mode = testMode("https1") // HTTPS/1.1 + http2Mode = testMode("h2") // HTTP/2 + http2UnencryptedMode = testMode("h2unencrypted") // HTTP/2 ) type testNotParallelOpt struct{} @@ -92,6 +95,17 @@ func run[T TBRun[T]](t T, f func(t T, mode testMode), opts ...any) { } } +// runSynctest is run combined with synctest.Run. +// +// The TB passed to f arranges for cleanup functions to be run in the synctest bubble. +func runSynctest(t *testing.T, f func(t *testing.T, mode testMode), opts ...any) { + run(t, func(t *testing.T, mode testMode) { + synctest.Test(t, func(t *testing.T) { + f(t, mode) + }) + }, opts...) +} + type clientServerTest struct { t testing.TB h2 bool @@ -99,6 +113,7 @@ type clientServerTest struct { ts *httptest.Server tr *Transport c *Client + li *fakeNetListener } func (t *clientServerTest) close() { @@ -136,6 +151,8 @@ func optWithServerLog(lg *log.Logger) func(*httptest.Server) { } } +var optFakeNet = new(struct{}) + // newClientServerTest creates and starts an httptest.Server. // // The mode parameter selects the implementation to test: @@ -147,6 +164,9 @@ func optWithServerLog(lg *log.Logger) func(*httptest.Server) { // // func(*httptest.Server) // run before starting the server // func(*http.Transport) +// +// The optFakeNet option configures the server and client to use a fake network implementation, +// suitable for use in testing/synctest tests. func newClientServerTest(t testing.TB, mode testMode, h Handler, opts ...any) *clientServerTest { if mode == http2Mode { CondSkipHTTP2(t) @@ -156,15 +176,39 @@ func newClientServerTest(t testing.TB, mode testMode, h Handler, opts ...any) *c h2: mode == http2Mode, h: h, } - cst.ts = httptest.NewUnstartedServer(h) var transportFuncs []func(*Transport) + + if idx := slices.Index(opts, any(optFakeNet)); idx >= 0 { + opts = slices.Delete(opts, idx, idx+1) + cst.li = fakeNetListen() + cst.ts = &httptest.Server{ + Config: &Server{Handler: h}, + Listener: cst.li, + } + transportFuncs = append(transportFuncs, func(tr *Transport) { + tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + return cst.li.connect(), nil + } + }) + } else { + cst.ts = httptest.NewUnstartedServer(h) + } + + if mode == http2UnencryptedMode { + p := &Protocols{} + p.SetUnencryptedHTTP2(true) + cst.ts.Config.Protocols = p + } + for _, opt := range opts { switch opt := opt.(type) { case func(*Transport): transportFuncs = append(transportFuncs, opt) case func(*httptest.Server): opt(cst.ts) + case func(*Server): + opt(cst.ts.Config) default: t.Fatalf("unhandled option type %T", opt) } @@ -179,6 +223,9 @@ func newClientServerTest(t testing.TB, mode testMode, h Handler, opts ...any) *c cst.ts.Start() case https1Mode: cst.ts.StartTLS() + case http2UnencryptedMode: + ExportHttp2ConfigureServer(cst.ts.Config, nil) + cst.ts.Start() case http2Mode: ExportHttp2ConfigureServer(cst.ts.Config, nil) cst.ts.TLS = cst.ts.Config.TLSConfig @@ -188,7 +235,7 @@ func newClientServerTest(t testing.TB, mode testMode, h Handler, opts ...any) *c } cst.c = cst.ts.Client() cst.tr = cst.c.Transport.(*Transport) - if mode == http2Mode { + if mode == http2Mode || mode == http2UnencryptedMode { if err := ExportHttp2ConfigureTransport(cst.tr); err != nil { t.Fatal(err) } @@ -196,6 +243,13 @@ func newClientServerTest(t testing.TB, mode testMode, h Handler, opts ...any) *c for _, f := range transportFuncs { f(cst.tr) } + + if mode == http2UnencryptedMode { + p := &Protocols{} + p.SetUnencryptedHTTP2(true) + cst.tr.Protocols = p + } + t.Cleanup(func() { cst.close() }) @@ -213,9 +267,19 @@ func (w testLogWriter) Write(b []byte) (int, error) { // Testing the newClientServerTest helper itself. func TestNewClientServerTest(t *testing.T) { - run(t, testNewClientServerTest, []testMode{http1Mode, https1Mode, http2Mode}) + modes := []testMode{http1Mode, https1Mode, http2Mode} + t.Run("realnet", func(t *testing.T) { + run(t, func(t *testing.T, mode testMode) { + testNewClientServerTest(t, mode) + }, modes) + }) + t.Run("synctest", func(t *testing.T) { + runSynctest(t, func(t *testing.T, mode testMode) { + testNewClientServerTest(t, mode, optFakeNet) + }, modes) + }) } -func testNewClientServerTest(t *testing.T, mode testMode) { +func testNewClientServerTest(t *testing.T, mode testMode, opts ...any) { var got struct { sync.Mutex proto string @@ -227,7 +291,7 @@ func testNewClientServerTest(t *testing.T, mode testMode) { got.proto = r.Proto got.hasTLS = r.TLS != nil }) - cst := newClientServerTest(t, mode, h) + cst := newClientServerTest(t, mode, h, opts...) if _, err := cst.c.Head(cst.ts.URL); err != nil { t.Fatal(err) } @@ -689,12 +753,6 @@ func testCancelRequestMidBody(t *testing.T, mode testMode) { func TestTrailersClientToServer(t *testing.T) { run(t, testTrailersClientToServer) } func testTrailersClientToServer(t *testing.T, mode testMode) { cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { - var decl []string - for k := range r.Trailer { - decl = append(decl, k) - } - slices.Sort(decl) - slurp, err := io.ReadAll(r.Body) if err != nil { t.Errorf("Server reading request body: %v", err) @@ -705,6 +763,7 @@ func testTrailersClientToServer(t *testing.T, mode testMode) { if r.Trailer == nil { io.WriteString(w, "nil Trailer") } else { + decl := slices.Sorted(maps.Keys(r.Trailer)) fmt.Fprintf(w, "decl: %v, vals: %s, %s", decl, r.Trailer.Get("Client-Trailer-A"), @@ -1176,7 +1235,7 @@ func testTransportGCRequest(t *testing.T, mode testMode, body bool) { (func() { body := strings.NewReader("some body") req, _ := NewRequest("POST", cst.ts.URL, body) - runtime.SetFinalizer(req, func(*Request) { close(didGC) }) + runtime.AddCleanup(req, func(ch chan struct{}) { close(ch) }, didGC) res, err := cst.c.Do(req) if err != nil { t.Fatal(err) @@ -1602,6 +1661,7 @@ func testBidiStreamReverseProxy(t *testing.T, mode testMode) { _, err := io.CopyN(io.MultiWriter(h, pw), rand.Reader, size) go pw.Close() if err != nil { + t.Errorf("body copy: %v", err) bodyRes <- err } else { bodyRes <- h diff --git a/src/net/http/clone.go b/src/net/http/clone.go index 71f424227313d1..0c2daf85524a90 100644 --- a/src/net/http/clone.go +++ b/src/net/http/clone.go @@ -68,7 +68,7 @@ func cloneMultipartForm(f *multipart.Form) *multipart.Form { Value: (map[string][]string)(Header(f.Value).Clone()), } if f.File != nil { - m := make(map[string][]*multipart.FileHeader) + m := make(map[string][]*multipart.FileHeader, len(f.File)) for k, vv := range f.File { vv2 := make([]*multipart.FileHeader, len(vv)) for i, v := range vv { diff --git a/src/net/http/cookie.go b/src/net/http/cookie.go index 3483e1638190a3..f74bc1043c509e 100644 --- a/src/net/http/cookie.go +++ b/src/net/http/cookie.go @@ -7,6 +7,7 @@ package http import ( "errors" "fmt" + "internal/godebug" "log" "net" "net/http/internal/ascii" @@ -16,6 +17,8 @@ import ( "time" ) +var httpcookiemaxnum = godebug.New("httpcookiemaxnum") + // A Cookie represents an HTTP cookie as sent in the Set-Cookie header of an // HTTP response or the Cookie header of an HTTP request. // @@ -58,16 +61,37 @@ const ( ) var ( - errBlankCookie = errors.New("http: blank cookie") - errEqualNotFoundInCookie = errors.New("http: '=' not found in cookie") - errInvalidCookieName = errors.New("http: invalid cookie name") - errInvalidCookieValue = errors.New("http: invalid cookie value") + errBlankCookie = errors.New("http: blank cookie") + errEqualNotFoundInCookie = errors.New("http: '=' not found in cookie") + errInvalidCookieName = errors.New("http: invalid cookie name") + errInvalidCookieValue = errors.New("http: invalid cookie value") + errCookieNumLimitExceeded = errors.New("http: number of cookies exceeded limit") ) +const defaultCookieMaxNum = 3000 + +func cookieNumWithinMax(cookieNum int) bool { + withinDefaultMax := cookieNum <= defaultCookieMaxNum + if httpcookiemaxnum.Value() == "" { + return withinDefaultMax + } + if customMax, err := strconv.Atoi(httpcookiemaxnum.Value()); err == nil { + withinCustomMax := customMax == 0 || cookieNum <= customMax + if withinDefaultMax != withinCustomMax { + httpcookiemaxnum.IncNonDefault() + } + return withinCustomMax + } + return withinDefaultMax +} + // ParseCookie parses a Cookie header value and returns all the cookies // which were set in it. Since the same cookie name can appear multiple times // the returned Values can contain more than one value for a given key. func ParseCookie(line string) ([]*Cookie, error) { + if !cookieNumWithinMax(strings.Count(line, ";") + 1) { + return nil, errCookieNumLimitExceeded + } parts := strings.Split(textproto.TrimString(line), ";") if len(parts) == 1 && parts[0] == "" { return nil, errBlankCookie @@ -79,7 +103,7 @@ func ParseCookie(line string) ([]*Cookie, error) { if !found { return nil, errEqualNotFoundInCookie } - if !isCookieNameValid(name) { + if !isToken(name) { return nil, errInvalidCookieName } value, quoted, found := parseCookieValue(value, true) @@ -104,7 +128,7 @@ func ParseSetCookie(line string) (*Cookie, error) { return nil, errEqualNotFoundInCookie } name = textproto.TrimString(name) - if !isCookieNameValid(name) { + if !isToken(name) { return nil, errInvalidCookieName } value, quoted, ok := parseCookieValue(value, true) @@ -197,11 +221,21 @@ func ParseSetCookie(line string) (*Cookie, error) { // readSetCookies parses all "Set-Cookie" values from // the header h and returns the successfully parsed Cookies. +// +// If the amount of cookies exceeds CookieNumLimit, and httpcookielimitnum +// GODEBUG option is not explicitly turned off, this function will silently +// fail and return an empty slice. func readSetCookies(h Header) []*Cookie { cookieCount := len(h["Set-Cookie"]) if cookieCount == 0 { return []*Cookie{} } + // Cookie limit was unfortunately introduced at a later point in time. + // As such, we can only fail by returning an empty slice rather than + // explicit error. + if !cookieNumWithinMax(cookieCount) { + return []*Cookie{} + } cookies := make([]*Cookie, 0, cookieCount) for _, line := range h["Set-Cookie"] { if cookie, err := ParseSetCookie(line); err == nil { @@ -225,7 +259,7 @@ func SetCookie(w ResponseWriter, cookie *Cookie) { // header (if other fields are set). // If c is nil or c.Name is invalid, the empty string is returned. func (c *Cookie) String() string { - if c == nil || !isCookieNameValid(c.Name) { + if c == nil || !isToken(c.Name) { return "" } // extraCookieLength derived from typical length of cookie attributes @@ -295,7 +329,7 @@ func (c *Cookie) Valid() error { if c == nil { return errors.New("http: nil Cookie") } - if !isCookieNameValid(c.Name) { + if !isToken(c.Name) { return errors.New("http: invalid Cookie.Name") } if !c.Expires.IsZero() && !validCookieExpires(c.Expires) { @@ -329,13 +363,28 @@ func (c *Cookie) Valid() error { // readCookies parses all "Cookie" values from the header h and // returns the successfully parsed Cookies. // -// if filter isn't empty, only cookies of that name are returned. +// If filter isn't empty, only cookies of that name are returned. +// +// If the amount of cookies exceeds CookieNumLimit, and httpcookielimitnum +// GODEBUG option is not explicitly turned off, this function will silently +// fail and return an empty slice. func readCookies(h Header, filter string) []*Cookie { lines := h["Cookie"] if len(lines) == 0 { return []*Cookie{} } + // Cookie limit was unfortunately introduced at a later point in time. + // As such, we can only fail by returning an empty slice rather than + // explicit error. + cookieCount := 0 + for _, line := range lines { + cookieCount += strings.Count(line, ";") + 1 + } + if !cookieNumWithinMax(cookieCount) { + return []*Cookie{} + } + cookies := make([]*Cookie, 0, len(lines)+strings.Count(lines[0], ";")) for _, line := range lines { line = textproto.TrimString(line) @@ -349,7 +398,7 @@ func readCookies(h Header, filter string) []*Cookie { } name, val, _ := strings.Cut(part, "=") name = textproto.TrimString(name) - if !isCookieNameValid(name) { + if !isToken(name) { continue } if filter != "" && filter != name { @@ -459,9 +508,6 @@ func sanitizeCookieName(n string) string { // See https://golang.org/issue/7243 for the discussion. func sanitizeCookieValue(v string, quoted bool) string { v = sanitizeOrWarn("Cookie.Value", validCookieValueByte, v) - if len(v) == 0 { - return v - } if strings.ContainsAny(v, " ,") || quoted { return `"` + v + `"` } @@ -526,10 +572,3 @@ func parseCookieValue(raw string, allowDoubleQuote bool) (value string, quoted, } return raw, quoted, true } - -func isCookieNameValid(raw string) bool { - if raw == "" { - return false - } - return strings.IndexFunc(raw, isNotToken) < 0 -} diff --git a/src/net/http/cookie_test.go b/src/net/http/cookie_test.go index aac69563624fcd..f452b4ec76830f 100644 --- a/src/net/http/cookie_test.go +++ b/src/net/http/cookie_test.go @@ -11,6 +11,7 @@ import ( "log" "os" "reflect" + "slices" "strings" "testing" "time" @@ -255,16 +256,17 @@ func TestAddCookie(t *testing.T) { } var readSetCookiesTests = []struct { - Header Header - Cookies []*Cookie + header Header + cookies []*Cookie + godebug string }{ { - Header{"Set-Cookie": {"Cookie-1=v$1"}}, - []*Cookie{{Name: "Cookie-1", Value: "v$1", Raw: "Cookie-1=v$1"}}, + header: Header{"Set-Cookie": {"Cookie-1=v$1"}}, + cookies: []*Cookie{{Name: "Cookie-1", Value: "v$1", Raw: "Cookie-1=v$1"}}, }, { - Header{"Set-Cookie": {"NID=99=YsDT5i3E-CXax-; expires=Wed, 23-Nov-2011 01:05:03 GMT; path=/; domain=.google.ch; HttpOnly"}}, - []*Cookie{{ + header: Header{"Set-Cookie": {"NID=99=YsDT5i3E-CXax-; expires=Wed, 23-Nov-2011 01:05:03 GMT; path=/; domain=.google.ch; HttpOnly"}}, + cookies: []*Cookie{{ Name: "NID", Value: "99=YsDT5i3E-CXax-", Path: "/", @@ -276,8 +278,8 @@ var readSetCookiesTests = []struct { }}, }, { - Header{"Set-Cookie": {".ASPXAUTH=7E3AA; expires=Wed, 07-Mar-2012 14:25:06 GMT; path=/; HttpOnly"}}, - []*Cookie{{ + header: Header{"Set-Cookie": {".ASPXAUTH=7E3AA; expires=Wed, 07-Mar-2012 14:25:06 GMT; path=/; HttpOnly"}}, + cookies: []*Cookie{{ Name: ".ASPXAUTH", Value: "7E3AA", Path: "/", @@ -288,8 +290,8 @@ var readSetCookiesTests = []struct { }}, }, { - Header{"Set-Cookie": {"ASP.NET_SessionId=foo; path=/; HttpOnly"}}, - []*Cookie{{ + header: Header{"Set-Cookie": {"ASP.NET_SessionId=foo; path=/; HttpOnly"}}, + cookies: []*Cookie{{ Name: "ASP.NET_SessionId", Value: "foo", Path: "/", @@ -298,8 +300,8 @@ var readSetCookiesTests = []struct { }}, }, { - Header{"Set-Cookie": {"samesitedefault=foo; SameSite"}}, - []*Cookie{{ + header: Header{"Set-Cookie": {"samesitedefault=foo; SameSite"}}, + cookies: []*Cookie{{ Name: "samesitedefault", Value: "foo", SameSite: SameSiteDefaultMode, @@ -307,8 +309,8 @@ var readSetCookiesTests = []struct { }}, }, { - Header{"Set-Cookie": {"samesiteinvalidisdefault=foo; SameSite=invalid"}}, - []*Cookie{{ + header: Header{"Set-Cookie": {"samesiteinvalidisdefault=foo; SameSite=invalid"}}, + cookies: []*Cookie{{ Name: "samesiteinvalidisdefault", Value: "foo", SameSite: SameSiteDefaultMode, @@ -316,8 +318,8 @@ var readSetCookiesTests = []struct { }}, }, { - Header{"Set-Cookie": {"samesitelax=foo; SameSite=Lax"}}, - []*Cookie{{ + header: Header{"Set-Cookie": {"samesitelax=foo; SameSite=Lax"}}, + cookies: []*Cookie{{ Name: "samesitelax", Value: "foo", SameSite: SameSiteLaxMode, @@ -325,8 +327,8 @@ var readSetCookiesTests = []struct { }}, }, { - Header{"Set-Cookie": {"samesitestrict=foo; SameSite=Strict"}}, - []*Cookie{{ + header: Header{"Set-Cookie": {"samesitestrict=foo; SameSite=Strict"}}, + cookies: []*Cookie{{ Name: "samesitestrict", Value: "foo", SameSite: SameSiteStrictMode, @@ -334,8 +336,8 @@ var readSetCookiesTests = []struct { }}, }, { - Header{"Set-Cookie": {"samesitenone=foo; SameSite=None"}}, - []*Cookie{{ + header: Header{"Set-Cookie": {"samesitenone=foo; SameSite=None"}}, + cookies: []*Cookie{{ Name: "samesitenone", Value: "foo", SameSite: SameSiteNoneMode, @@ -345,47 +347,66 @@ var readSetCookiesTests = []struct { // Make sure we can properly read back the Set-Cookie headers we create // for values containing spaces or commas: { - Header{"Set-Cookie": {`special-1=a z`}}, - []*Cookie{{Name: "special-1", Value: "a z", Raw: `special-1=a z`}}, + header: Header{"Set-Cookie": {`special-1=a z`}}, + cookies: []*Cookie{{Name: "special-1", Value: "a z", Raw: `special-1=a z`}}, }, { - Header{"Set-Cookie": {`special-2=" z"`}}, - []*Cookie{{Name: "special-2", Value: " z", Quoted: true, Raw: `special-2=" z"`}}, + header: Header{"Set-Cookie": {`special-2=" z"`}}, + cookies: []*Cookie{{Name: "special-2", Value: " z", Quoted: true, Raw: `special-2=" z"`}}, }, { - Header{"Set-Cookie": {`special-3="a "`}}, - []*Cookie{{Name: "special-3", Value: "a ", Quoted: true, Raw: `special-3="a "`}}, + header: Header{"Set-Cookie": {`special-3="a "`}}, + cookies: []*Cookie{{Name: "special-3", Value: "a ", Quoted: true, Raw: `special-3="a "`}}, }, { - Header{"Set-Cookie": {`special-4=" "`}}, - []*Cookie{{Name: "special-4", Value: " ", Quoted: true, Raw: `special-4=" "`}}, + header: Header{"Set-Cookie": {`special-4=" "`}}, + cookies: []*Cookie{{Name: "special-4", Value: " ", Quoted: true, Raw: `special-4=" "`}}, }, { - Header{"Set-Cookie": {`special-5=a,z`}}, - []*Cookie{{Name: "special-5", Value: "a,z", Raw: `special-5=a,z`}}, + header: Header{"Set-Cookie": {`special-5=a,z`}}, + cookies: []*Cookie{{Name: "special-5", Value: "a,z", Raw: `special-5=a,z`}}, }, { - Header{"Set-Cookie": {`special-6=",z"`}}, - []*Cookie{{Name: "special-6", Value: ",z", Quoted: true, Raw: `special-6=",z"`}}, + header: Header{"Set-Cookie": {`special-6=",z"`}}, + cookies: []*Cookie{{Name: "special-6", Value: ",z", Quoted: true, Raw: `special-6=",z"`}}, }, { - Header{"Set-Cookie": {`special-7=a,`}}, - []*Cookie{{Name: "special-7", Value: "a,", Raw: `special-7=a,`}}, + header: Header{"Set-Cookie": {`special-7=a,`}}, + cookies: []*Cookie{{Name: "special-7", Value: "a,", Raw: `special-7=a,`}}, }, { - Header{"Set-Cookie": {`special-8=","`}}, - []*Cookie{{Name: "special-8", Value: ",", Quoted: true, Raw: `special-8=","`}}, + header: Header{"Set-Cookie": {`special-8=","`}}, + cookies: []*Cookie{{Name: "special-8", Value: ",", Quoted: true, Raw: `special-8=","`}}, }, // Make sure we can properly read back the Set-Cookie headers // for names containing spaces: { - Header{"Set-Cookie": {`special-9 =","`}}, - []*Cookie{{Name: "special-9", Value: ",", Quoted: true, Raw: `special-9 =","`}}, + header: Header{"Set-Cookie": {`special-9 =","`}}, + cookies: []*Cookie{{Name: "special-9", Value: ",", Quoted: true, Raw: `special-9 =","`}}, }, // Quoted values (issue #46443) { - Header{"Set-Cookie": {`cookie="quoted"`}}, - []*Cookie{{Name: "cookie", Value: "quoted", Quoted: true, Raw: `cookie="quoted"`}}, + header: Header{"Set-Cookie": {`cookie="quoted"`}}, + cookies: []*Cookie{{Name: "cookie", Value: "quoted", Quoted: true, Raw: `cookie="quoted"`}}, + }, + { + header: Header{"Set-Cookie": slices.Repeat([]string{"a="}, defaultCookieMaxNum+1)}, + cookies: []*Cookie{}, + }, + { + header: Header{"Set-Cookie": slices.Repeat([]string{"a="}, 10)}, + cookies: []*Cookie{}, + godebug: "httpcookiemaxnum=5", + }, + { + header: Header{"Set-Cookie": strings.Split(strings.Repeat(";a=", defaultCookieMaxNum+1)[1:], ";")}, + cookies: slices.Repeat([]*Cookie{{Name: "a", Value: "", Quoted: false, Raw: "a="}}, defaultCookieMaxNum+1), + godebug: "httpcookiemaxnum=0", + }, + { + header: Header{"Set-Cookie": strings.Split(strings.Repeat(";a=", defaultCookieMaxNum+1)[1:], ";")}, + cookies: slices.Repeat([]*Cookie{{Name: "a", Value: "", Quoted: false, Raw: "a="}}, defaultCookieMaxNum+1), + godebug: fmt.Sprintf("httpcookiemaxnum=%v", defaultCookieMaxNum+1), }, // TODO(bradfitz): users have reported seeing this in the @@ -405,79 +426,103 @@ func toJSON(v any) string { func TestReadSetCookies(t *testing.T) { for i, tt := range readSetCookiesTests { + t.Setenv("GODEBUG", tt.godebug) for n := 0; n < 2; n++ { // to verify readSetCookies doesn't mutate its input - c := readSetCookies(tt.Header) - if !reflect.DeepEqual(c, tt.Cookies) { - t.Errorf("#%d readSetCookies: have\n%s\nwant\n%s\n", i, toJSON(c), toJSON(tt.Cookies)) + c := readSetCookies(tt.header) + if !reflect.DeepEqual(c, tt.cookies) { + t.Errorf("#%d readSetCookies: have\n%s\nwant\n%s\n", i, toJSON(c), toJSON(tt.cookies)) } } } } var readCookiesTests = []struct { - Header Header - Filter string - Cookies []*Cookie + header Header + filter string + cookies []*Cookie + godebug string }{ { - Header{"Cookie": {"Cookie-1=v$1", "c2=v2"}}, - "", - []*Cookie{ + header: Header{"Cookie": {"Cookie-1=v$1", "c2=v2"}}, + filter: "", + cookies: []*Cookie{ {Name: "Cookie-1", Value: "v$1"}, {Name: "c2", Value: "v2"}, }, }, { - Header{"Cookie": {"Cookie-1=v$1", "c2=v2"}}, - "c2", - []*Cookie{ + header: Header{"Cookie": {"Cookie-1=v$1", "c2=v2"}}, + filter: "c2", + cookies: []*Cookie{ {Name: "c2", Value: "v2"}, }, }, { - Header{"Cookie": {"Cookie-1=v$1; c2=v2"}}, - "", - []*Cookie{ + header: Header{"Cookie": {"Cookie-1=v$1; c2=v2"}}, + filter: "", + cookies: []*Cookie{ {Name: "Cookie-1", Value: "v$1"}, {Name: "c2", Value: "v2"}, }, }, { - Header{"Cookie": {"Cookie-1=v$1; c2=v2"}}, - "c2", - []*Cookie{ + header: Header{"Cookie": {"Cookie-1=v$1; c2=v2"}}, + filter: "c2", + cookies: []*Cookie{ {Name: "c2", Value: "v2"}, }, }, { - Header{"Cookie": {`Cookie-1="v$1"; c2="v2"`}}, - "", - []*Cookie{ + header: Header{"Cookie": {`Cookie-1="v$1"; c2="v2"`}}, + filter: "", + cookies: []*Cookie{ {Name: "Cookie-1", Value: "v$1", Quoted: true}, {Name: "c2", Value: "v2", Quoted: true}, }, }, { - Header{"Cookie": {`Cookie-1="v$1"; c2=v2;`}}, - "", - []*Cookie{ + header: Header{"Cookie": {`Cookie-1="v$1"; c2=v2;`}}, + filter: "", + cookies: []*Cookie{ {Name: "Cookie-1", Value: "v$1", Quoted: true}, {Name: "c2", Value: "v2"}, }, }, { - Header{"Cookie": {``}}, - "", - []*Cookie{}, + header: Header{"Cookie": {``}}, + filter: "", + cookies: []*Cookie{}, + }, + // GODEBUG=httpcookiemaxnum should work regardless if all cookies are sent + // via one "Cookie" field, or multiple fields. + { + header: Header{"Cookie": {strings.Repeat(";a=", defaultCookieMaxNum+1)[1:]}}, + cookies: []*Cookie{}, + }, + { + header: Header{"Cookie": slices.Repeat([]string{"a="}, 10)}, + cookies: []*Cookie{}, + godebug: "httpcookiemaxnum=5", + }, + { + header: Header{"Cookie": {strings.Repeat(";a=", defaultCookieMaxNum+1)[1:]}}, + cookies: slices.Repeat([]*Cookie{{Name: "a", Value: "", Quoted: false}}, defaultCookieMaxNum+1), + godebug: "httpcookiemaxnum=0", + }, + { + header: Header{"Cookie": slices.Repeat([]string{"a="}, defaultCookieMaxNum+1)}, + cookies: slices.Repeat([]*Cookie{{Name: "a", Value: "", Quoted: false}}, defaultCookieMaxNum+1), + godebug: fmt.Sprintf("httpcookiemaxnum=%v", defaultCookieMaxNum+1), }, } func TestReadCookies(t *testing.T) { for i, tt := range readCookiesTests { + t.Setenv("GODEBUG", tt.godebug) for n := 0; n < 2; n++ { // to verify readCookies doesn't mutate its input - c := readCookies(tt.Header, tt.Filter) - if !reflect.DeepEqual(c, tt.Cookies) { - t.Errorf("#%d readCookies:\nhave: %s\nwant: %s\n", i, toJSON(c), toJSON(tt.Cookies)) + c := readCookies(tt.header, tt.filter) + if !reflect.DeepEqual(c, tt.cookies) { + t.Errorf("#%d readCookies:\nhave: %s\nwant: %s\n", i, toJSON(c), toJSON(tt.cookies)) } } } @@ -530,6 +575,7 @@ func TestCookieSanitizeValue(t *testing.T) { {"a,z", false, `"a,z"`}, {",z", false, `",z"`}, {"a,", false, `"a,"`}, + {"", true, `""`}, } for _, tt := range tests { if got := sanitizeCookieValue(tt.in, tt.quoted); got != tt.want { @@ -689,6 +735,7 @@ func TestParseCookie(t *testing.T) { line string cookies []*Cookie err error + godebug string }{ { line: "Cookie-1=v$1", @@ -722,8 +769,28 @@ func TestParseCookie(t *testing.T) { line: "k1=\\", err: errInvalidCookieValue, }, + { + line: strings.Repeat(";a=", defaultCookieMaxNum+1)[1:], + err: errCookieNumLimitExceeded, + }, + { + line: strings.Repeat(";a=", 10)[1:], + err: errCookieNumLimitExceeded, + godebug: "httpcookiemaxnum=5", + }, + { + line: strings.Repeat(";a=", defaultCookieMaxNum+1)[1:], + cookies: slices.Repeat([]*Cookie{{Name: "a", Value: "", Quoted: false}}, defaultCookieMaxNum+1), + godebug: "httpcookiemaxnum=0", + }, + { + line: strings.Repeat(";a=", defaultCookieMaxNum+1)[1:], + cookies: slices.Repeat([]*Cookie{{Name: "a", Value: "", Quoted: false}}, defaultCookieMaxNum+1), + godebug: fmt.Sprintf("httpcookiemaxnum=%v", defaultCookieMaxNum+1), + }, } for i, tt := range tests { + t.Setenv("GODEBUG", tt.godebug) gotCookies, gotErr := ParseCookie(tt.line) if !errors.Is(gotErr, tt.err) { t.Errorf("#%d ParseCookie got error %v, want error %v", i, gotErr, tt.err) diff --git a/src/net/http/cookiejar/jar.go b/src/net/http/cookiejar/jar.go index 2eec1a3e745b42..edf14d03ad3809 100644 --- a/src/net/http/cookiejar/jar.go +++ b/src/net/http/cookiejar/jar.go @@ -500,9 +500,7 @@ func (j *Jar) domainAndType(host, domain string) (string, bool, error) { // From here on: If the cookie is valid, it is a domain cookie (with // the one exception of a public suffix below). // See RFC 6265 section 5.2.3. - if domain[0] == '.' { - domain = domain[1:] - } + domain = strings.TrimPrefix(domain, ".") if len(domain) == 0 || domain[0] == '.' { // Received either "Domain=." or "Domain=..some.thing", diff --git a/src/net/http/csrf.go b/src/net/http/csrf.go new file mode 100644 index 00000000000000..0b71b403899a2a --- /dev/null +++ b/src/net/http/csrf.go @@ -0,0 +1,218 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "errors" + "fmt" + "net/url" + "sync" + "sync/atomic" +) + +// CrossOriginProtection implements protections against [Cross-Site Request +// Forgery (CSRF)] by rejecting non-safe cross-origin browser requests. +// +// Cross-origin requests are currently detected with the [Sec-Fetch-Site] +// header, available in all browsers since 2023, or by comparing the hostname of +// the [Origin] header with the Host header. +// +// The GET, HEAD, and OPTIONS methods are [safe methods] and are always allowed. +// It's important that applications do not perform any state changing actions +// due to requests with safe methods. +// +// Requests without Sec-Fetch-Site or Origin headers are currently assumed to be +// either same-origin or non-browser requests, and are allowed. +// +// The zero value of CrossOriginProtection is valid and has no trusted origins +// or bypass patterns. +// +// [Sec-Fetch-Site]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Sec-Fetch-Site +// [Origin]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Origin +// [Cross-Site Request Forgery (CSRF)]: https://developer.mozilla.org/en-US/docs/Web/Security/Attacks/CSRF +// [safe methods]: https://developer.mozilla.org/en-US/docs/Glossary/Safe/HTTP +type CrossOriginProtection struct { + bypass atomic.Pointer[ServeMux] + trustedMu sync.RWMutex + trusted map[string]bool + deny atomic.Pointer[Handler] +} + +// NewCrossOriginProtection returns a new [CrossOriginProtection] value. +func NewCrossOriginProtection() *CrossOriginProtection { + return &CrossOriginProtection{} +} + +// AddTrustedOrigin allows all requests with an [Origin] header +// which exactly matches the given value. +// +// Origin header values are of the form "scheme://host[:port]". +// +// AddTrustedOrigin can be called concurrently with other methods +// or request handling, and applies to future requests. +// +// [Origin]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Origin +func (c *CrossOriginProtection) AddTrustedOrigin(origin string) error { + u, err := url.Parse(origin) + if err != nil { + return fmt.Errorf("invalid origin %q: %w", origin, err) + } + if u.Scheme == "" { + return fmt.Errorf("invalid origin %q: scheme is required", origin) + } + if u.Host == "" { + return fmt.Errorf("invalid origin %q: host is required", origin) + } + if u.Path != "" || u.RawQuery != "" || u.Fragment != "" { + return fmt.Errorf("invalid origin %q: path, query, and fragment are not allowed", origin) + } + c.trustedMu.Lock() + defer c.trustedMu.Unlock() + if c.trusted == nil { + c.trusted = make(map[string]bool) + } + c.trusted[origin] = true + return nil +} + +type noopHandler struct{} + +func (noopHandler) ServeHTTP(ResponseWriter, *Request) {} + +var sentinelHandler Handler = &noopHandler{} + +// AddInsecureBypassPattern permits all requests that match the given pattern. +// +// The pattern syntax and precedence rules are the same as [ServeMux]. Only +// requests that match the pattern directly are permitted. Those that ServeMux +// would redirect to a pattern (e.g. after cleaning the path or adding a +// trailing slash) are not. +// +// AddInsecureBypassPattern panics if the pattern conflicts with one already +// registered, or if the pattern is syntactically invalid (for example, an +// improperly formed wildcard). +// +// AddInsecureBypassPattern can be called concurrently with other methods or +// request handling, and applies to future requests. +func (c *CrossOriginProtection) AddInsecureBypassPattern(pattern string) { + var bypass *ServeMux + + // Lazily initialize c.bypass + for { + bypass = c.bypass.Load() + if bypass != nil { + break + } + bypass = NewServeMux() + if c.bypass.CompareAndSwap(nil, bypass) { + break + } + } + + bypass.Handle(pattern, sentinelHandler) +} + +// SetDenyHandler sets a handler to invoke when a request is rejected. +// The default error handler responds with a 403 Forbidden status. +// +// SetDenyHandler can be called concurrently with other methods +// or request handling, and applies to future requests. +// +// Check does not call the error handler. +func (c *CrossOriginProtection) SetDenyHandler(h Handler) { + if h == nil { + c.deny.Store(nil) + return + } + c.deny.Store(&h) +} + +// Check applies cross-origin checks to a request. +// It returns an error if the request should be rejected. +func (c *CrossOriginProtection) Check(req *Request) error { + switch req.Method { + case "GET", "HEAD", "OPTIONS": + // Safe methods are always allowed. + return nil + } + + switch req.Header.Get("Sec-Fetch-Site") { + case "": + // No Sec-Fetch-Site header is present. + // Fallthrough to check the Origin header. + case "same-origin", "none": + return nil + default: + if c.isRequestExempt(req) { + return nil + } + return errCrossOriginRequest + } + + origin := req.Header.Get("Origin") + if origin == "" { + // Neither Sec-Fetch-Site nor Origin headers are present. + // Either the request is same-origin or not a browser request. + return nil + } + + if o, err := url.Parse(origin); err == nil && o.Host == req.Host { + // The Origin header matches the Host header. Note that the Host header + // doesn't include the scheme, so we don't know if this might be an + // HTTP→HTTPS cross-origin request. We fail open, since all modern + // browsers support Sec-Fetch-Site since 2023, and running an older + // browser makes a clear security trade-off already. Sites can mitigate + // this with HTTP Strict Transport Security (HSTS). + return nil + } + + if c.isRequestExempt(req) { + return nil + } + return errCrossOriginRequestFromOldBrowser +} + +var ( + errCrossOriginRequest = errors.New("cross-origin request detected from Sec-Fetch-Site header") + errCrossOriginRequestFromOldBrowser = errors.New("cross-origin request detected, and/or browser is out of date: " + + "Sec-Fetch-Site is missing, and Origin does not match Host") +) + +// isRequestExempt checks the bypasses which require taking a lock, and should +// be deferred until the last moment. +func (c *CrossOriginProtection) isRequestExempt(req *Request) bool { + if bypass := c.bypass.Load(); bypass != nil { + if h, _ := bypass.Handler(req); h == sentinelHandler { + // The request matches a bypass pattern. + return true + } + } + + c.trustedMu.RLock() + defer c.trustedMu.RUnlock() + origin := req.Header.Get("Origin") + // The request matches a trusted origin. + return origin != "" && c.trusted[origin] +} + +// Handler returns a handler that applies cross-origin checks +// before invoking the handler h. +// +// If a request fails cross-origin checks, the request is rejected +// with a 403 Forbidden status or handled with the handler passed +// to [CrossOriginProtection.SetDenyHandler]. +func (c *CrossOriginProtection) Handler(h Handler) Handler { + return HandlerFunc(func(w ResponseWriter, r *Request) { + if err := c.Check(r); err != nil { + if deny := c.deny.Load(); deny != nil { + (*deny).ServeHTTP(w, r) + return + } + Error(w, err.Error(), StatusForbidden) + return + } + h.ServeHTTP(w, r) + }) +} diff --git a/src/net/http/csrf_test.go b/src/net/http/csrf_test.go new file mode 100644 index 00000000000000..a29e3ae16dff6b --- /dev/null +++ b/src/net/http/csrf_test.go @@ -0,0 +1,345 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http_test + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// httptestNewRequest works around https://go.dev/issue/73151. +func httptestNewRequest(method, target string) *http.Request { + req := httptest.NewRequest(method, target, nil) + req.URL.Scheme = "" + req.URL.Host = "" + return req +} + +var okHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +}) + +func TestCrossOriginProtectionSecFetchSite(t *testing.T) { + protection := http.NewCrossOriginProtection() + handler := protection.Handler(okHandler) + + tests := []struct { + name string + method string + secFetchSite string + origin string + expectedStatus int + }{ + {"same-origin allowed", "POST", "same-origin", "", http.StatusOK}, + {"none allowed", "POST", "none", "", http.StatusOK}, + {"cross-site blocked", "POST", "cross-site", "", http.StatusForbidden}, + {"same-site blocked", "POST", "same-site", "", http.StatusForbidden}, + + {"no header with no origin", "POST", "", "", http.StatusOK}, + {"no header with matching origin", "POST", "", "https://example.com/", http.StatusOK}, + {"no header with mismatched origin", "POST", "", "https://attacker.example/", http.StatusForbidden}, + {"no header with null origin", "POST", "", "null", http.StatusForbidden}, + + {"GET allowed", "GET", "cross-site", "", http.StatusOK}, + {"HEAD allowed", "HEAD", "cross-site", "", http.StatusOK}, + {"OPTIONS allowed", "OPTIONS", "cross-site", "", http.StatusOK}, + {"PUT blocked", "PUT", "cross-site", "", http.StatusForbidden}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + req := httptestNewRequest(tc.method, "https://example.com/") + if tc.secFetchSite != "" { + req.Header.Set("Sec-Fetch-Site", tc.secFetchSite) + } + if tc.origin != "" { + req.Header.Set("Origin", tc.origin) + } + + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != tc.expectedStatus { + t.Errorf("got status %d, want %d", w.Code, tc.expectedStatus) + } + }) + } +} + +func TestCrossOriginProtectionTrustedOriginBypass(t *testing.T) { + protection := http.NewCrossOriginProtection() + err := protection.AddTrustedOrigin("https://trusted.example/") + if err != nil { + t.Fatalf("AddTrustedOrigin: %v", err) + } + handler := protection.Handler(okHandler) + + tests := []struct { + name string + origin string + secFetchSite string + expectedStatus int + }{ + {"trusted origin without sec-fetch-site", "https://trusted.example/", "", http.StatusOK}, + {"trusted origin with cross-site", "https://trusted.example/", "cross-site", http.StatusOK}, + {"untrusted origin without sec-fetch-site", "https://attacker.example/", "", http.StatusForbidden}, + {"untrusted origin with cross-site", "https://attacker.example/", "cross-site", http.StatusForbidden}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + req := httptestNewRequest("POST", "https://example.com/") + req.Header.Set("Origin", tc.origin) + if tc.secFetchSite != "" { + req.Header.Set("Sec-Fetch-Site", tc.secFetchSite) + } + + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != tc.expectedStatus { + t.Errorf("got status %d, want %d", w.Code, tc.expectedStatus) + } + }) + } +} + +func TestCrossOriginProtectionPatternBypass(t *testing.T) { + protection := http.NewCrossOriginProtection() + protection.AddInsecureBypassPattern("/bypass/") + protection.AddInsecureBypassPattern("/only/{foo}") + protection.AddInsecureBypassPattern("/no-trailing") + protection.AddInsecureBypassPattern("/yes-trailing/") + protection.AddInsecureBypassPattern("PUT /put-only/") + protection.AddInsecureBypassPattern("GET /get-only/") + protection.AddInsecureBypassPattern("POST /post-only/") + handler := protection.Handler(okHandler) + + tests := []struct { + name string + path string + secFetchSite string + expectedStatus int + }{ + {"bypass path without sec-fetch-site", "/bypass/", "", http.StatusOK}, + {"bypass path with cross-site", "/bypass/", "cross-site", http.StatusOK}, + {"non-bypass path without sec-fetch-site", "/api/", "", http.StatusForbidden}, + {"non-bypass path with cross-site", "/api/", "cross-site", http.StatusForbidden}, + + {"redirect to bypass path without ..", "/foo/../bypass/bar", "", http.StatusForbidden}, + {"redirect to bypass path with trailing slash", "/bypass", "", http.StatusForbidden}, + {"redirect to non-bypass path with ..", "/foo/../api/bar", "", http.StatusForbidden}, + {"redirect to non-bypass path with trailing slash", "/api", "", http.StatusForbidden}, + + {"wildcard bypass", "/only/123", "", http.StatusOK}, + {"non-wildcard", "/only/123/foo", "", http.StatusForbidden}, + + // https://go.dev/issue/75054 + {"no trailing slash exact match", "/no-trailing", "", http.StatusOK}, + {"no trailing slash with slash", "/no-trailing/", "", http.StatusForbidden}, + {"yes trailing slash exact match", "/yes-trailing/", "", http.StatusOK}, + {"yes trailing slash without slash", "/yes-trailing", "", http.StatusForbidden}, + + {"method-specific hit", "/post-only/", "", http.StatusOK}, + {"method-specific miss (PUT)", "/put-only/", "", http.StatusForbidden}, + {"method-specific miss (GET)", "/get-only/", "", http.StatusForbidden}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + req := httptestNewRequest("POST", "https://example.com/"+tc.path) + req.Header.Set("Origin", "https://attacker.example/") + if tc.secFetchSite != "" { + req.Header.Set("Sec-Fetch-Site", tc.secFetchSite) + } + + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != tc.expectedStatus { + t.Errorf("got status %d, want %d", w.Code, tc.expectedStatus) + } + }) + } +} + +func TestCrossOriginProtectionSetDenyHandler(t *testing.T) { + protection := http.NewCrossOriginProtection() + + handler := protection.Handler(okHandler) + + req := httptestNewRequest("POST", "https://example.com/") + req.Header.Set("Sec-Fetch-Site", "cross-site") + + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusForbidden { + t.Errorf("got status %d, want %d", w.Code, http.StatusForbidden) + } + + customErrHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusTeapot) + io.WriteString(w, "custom error") + }) + protection.SetDenyHandler(customErrHandler) + + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusTeapot { + t.Errorf("got status %d, want %d", w.Code, http.StatusTeapot) + } + + if !strings.Contains(w.Body.String(), "custom error") { + t.Errorf("expected custom error message, got: %q", w.Body.String()) + } + + req = httptestNewRequest("GET", "https://example.com/") + + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("got status %d, want %d", w.Code, http.StatusOK) + } + + protection.SetDenyHandler(nil) + + req = httptestNewRequest("POST", "https://example.com/") + req.Header.Set("Sec-Fetch-Site", "cross-site") + + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusForbidden { + t.Errorf("got status %d, want %d", w.Code, http.StatusForbidden) + } +} + +func TestCrossOriginProtectionAddTrustedOriginErrors(t *testing.T) { + protection := http.NewCrossOriginProtection() + + tests := []struct { + name string + origin string + wantErr bool + }{ + {"valid origin", "https://example.com/", false}, + {"valid origin with port", "https://example.com:8080/", false}, + {"http origin", "http://example.com/", false}, + {"missing scheme", "example.com", true}, + {"missing host", "https://", true}, + {"trailing slash", "https://example.com/", true}, + {"with path", "https://example.com/path", true}, + {"with query", "https://example.com/?query=value", true}, + {"with fragment", "https://example.com/#fragment", true}, + {"invalid url", "https://ex ample.com", true}, + {"empty string", "", true}, + {"null", "null", true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := protection.AddTrustedOrigin(tc.origin) + if (err != nil) != tc.wantErr { + t.Errorf("AddTrustedOrigin(%q) error = %v, wantErr %v", tc.origin, err, tc.wantErr) + } + }) + } +} + +func TestCrossOriginProtectionAddingBypassesConcurrently(t *testing.T) { + protection := http.NewCrossOriginProtection() + handler := protection.Handler(okHandler) + + req := httptestNewRequest("POST", "https://example.com/") + req.Header.Set("Origin", "https://concurrent.example/") + req.Header.Set("Sec-Fetch-Site", "cross-site") + + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusForbidden { + t.Errorf("got status %d, want %d", w.Code, http.StatusForbidden) + } + + start := make(chan struct{}) + done := make(chan struct{}) + go func() { + close(start) + defer close(done) + for range 10 { + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + } + }() + + // Add bypasses while the requests are in flight. + <-start + protection.AddTrustedOrigin("https://concurrent.example/") + protection.AddInsecureBypassPattern("/foo/") + <-done + + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("After concurrent bypass addition, got status %d, want %d", w.Code, http.StatusOK) + } +} + +func TestCrossOriginProtectionServer(t *testing.T) { + protection := http.NewCrossOriginProtection() + protection.AddTrustedOrigin("https://trusted.example/") + protection.AddInsecureBypassPattern("/bypass/") + handler := protection.Handler(okHandler) + + ts := httptest.NewServer(handler) + defer ts.Close() + + tests := []struct { + name string + method string + url string + origin string + secFetchSite string + expectedStatus int + }{ + {"cross-site", "POST", ts.URL, "https://attacker.example/", "cross-site", http.StatusForbidden}, + {"same-origin", "POST", ts.URL, "", "same-origin", http.StatusOK}, + {"origin matches host", "POST", ts.URL, ts.URL, "", http.StatusOK}, + {"trusted origin", "POST", ts.URL, "https://trusted.example/", "", http.StatusOK}, + {"untrusted origin", "POST", ts.URL, "https://attacker.example/", "", http.StatusForbidden}, + {"bypass path", "POST", ts.URL + "/bypass/", "https://attacker.example/", "", http.StatusOK}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + req, err := http.NewRequest(tc.method, tc.url, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + if tc.origin != "" { + req.Header.Set("Origin", tc.origin) + } + if tc.secFetchSite != "" { + req.Header.Set("Sec-Fetch-Site", tc.secFetchSite) + } + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Do: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != tc.expectedStatus { + t.Errorf("got status %d, want %d", resp.StatusCode, tc.expectedStatus) + } + }) + } +} diff --git a/src/net/http/doc.go b/src/net/http/doc.go index f7ad3ae762fb6c..24e07352ca726d 100644 --- a/src/net/http/doc.go +++ b/src/net/http/doc.go @@ -84,27 +84,26 @@ custom Server: # HTTP/2 -Starting with Go 1.6, the http package has transparent support for the -HTTP/2 protocol when using HTTPS. Programs that must disable HTTP/2 -can do so by setting [Transport.TLSNextProto] (for clients) or -[Server.TLSNextProto] (for servers) to a non-nil, empty -map. Alternatively, the following GODEBUG settings are -currently supported: +The http package has transparent support for the HTTP/2 protocol. + +[Server] and [DefaultTransport] automatically enable HTTP/2 support +when using HTTPS. [Transport] does not enable HTTP/2 by default. + +To enable or disable support for HTTP/1, HTTP/2, and/or unencrypted HTTP/2, +see the [Server.Protocols] and [Transport.Protocols] configuration fields. + +To configure advanced HTTP/2 features, see the [Server.HTTP2] and +[Transport.HTTP2] configuration fields. + +Alternatively, the following GODEBUG settings are currently supported: GODEBUG=http2client=0 # disable HTTP/2 client support GODEBUG=http2server=0 # disable HTTP/2 server support GODEBUG=http2debug=1 # enable verbose HTTP/2 debug logs GODEBUG=http2debug=2 # ... even more verbose, with frame dumps -Please report any issues before disabling HTTP/2 support: https://golang.org/s/http2bug - -The http package's [Transport] and [Server] both automatically enable -HTTP/2 support for simple configurations. To enable HTTP/2 for more -complex configurations, to use lower-level HTTP/2 features, or to use -a newer version of Go's http2 package, import "golang.org/x/net/http2" -directly and use its ConfigureTransport and/or ConfigureServer -functions. Manually configuring HTTP/2 via the golang.org/x/net/http2 -package takes precedence over the net/http package's built-in HTTP/2 -support. +The "omithttp2" build tag may be used to disable the HTTP/2 implementation +contained in the http package. */ + package http diff --git a/src/net/http/example_filesystem_test.go b/src/net/http/example_filesystem_test.go index 0e81458a071966..da1f0df8902f4d 100644 --- a/src/net/http/example_filesystem_test.go +++ b/src/net/http/example_filesystem_test.go @@ -5,6 +5,7 @@ package http_test import ( + "io" "io/fs" "log" "net/http" @@ -40,6 +41,9 @@ func (f dotFileHidingFile) Readdir(n int) (fis []fs.FileInfo, err error) { fis = append(fis, file) } } + if err == nil && n > 0 && len(fis) == 0 { + err = io.EOF + } return } @@ -61,7 +65,7 @@ func (fsys dotFileHidingFileSystem) Open(name string) (http.File, error) { if err != nil { return nil, err } - return dotFileHidingFile{file}, err + return dotFileHidingFile{file}, nil } func ExampleFileServer_dotFileHiding() { diff --git a/src/net/http/example_test.go b/src/net/http/example_test.go index 2f411d1d2ec23b..acb96bba5178c8 100644 --- a/src/net/http/example_test.go +++ b/src/net/http/example_test.go @@ -12,6 +12,7 @@ import ( "net/http" "os" "os/signal" + "time" ) func ExampleHijacker() { @@ -193,3 +194,50 @@ func ExampleNotFoundHandler() { log.Fatal(http.ListenAndServe(":8080", mux)) } + +func ExampleProtocols_http1() { + srv := http.Server{ + Addr: ":8443", + } + + // Serve only HTTP/1. + srv.Protocols = new(http.Protocols) + srv.Protocols.SetHTTP1(true) + + log.Fatal(srv.ListenAndServeTLS("cert.pem", "key.pem")) +} + +func ExampleProtocols_http1or2() { + t := http.DefaultTransport.(*http.Transport).Clone() + + // Use either HTTP/1 and HTTP/2. + t.Protocols = new(http.Protocols) + t.Protocols.SetHTTP1(true) + t.Protocols.SetHTTP2(true) + + cli := &http.Client{Transport: t} + res, err := cli.Get("http://www.google.com/robots.txt") + if err != nil { + log.Fatal(err) + } + res.Body.Close() +} + +func ExampleCrossOriginProtection() { + mux := http.NewServeMux() + + mux.HandleFunc("/hello", func(w http.ResponseWriter, req *http.Request) { + io.WriteString(w, "request allowed\n") + }) + + srv := http.Server{ + Addr: ":8080", + ReadTimeout: 15 * time.Second, + WriteTimeout: 15 * time.Second, + // Use CrossOriginProtection.Handler to block all non-safe cross-origin + // browser requests to mux. + Handler: http.NewCrossOriginProtection().Handler(mux), + } + + log.Fatal(srv.ListenAndServe()) +} diff --git a/src/net/http/export_test.go b/src/net/http/export_test.go index 56ebda180bb085..f2aa663a990d5f 100644 --- a/src/net/http/export_test.go +++ b/src/net/http/export_test.go @@ -21,7 +21,6 @@ import ( var ( DefaultUserAgent = defaultUserAgent NewLoggingConn = newLoggingConn - ExportAppendTime = appendTime ExportRefererForURL = refererForURL ExportServerNewConn = (*Server).newConn ExportCloseWriteAndWait = (*conn).closeWriteAndWait diff --git a/src/net/http/fs.go b/src/net/http/fs.go index 3a716fbd2cc7f8..92bd94f72dfb68 100644 --- a/src/net/http/fs.go +++ b/src/net/http/fs.go @@ -67,6 +67,11 @@ func mapOpenError(originalErr error, name string, sep rune, stat func(string) (f return originalErr } +// errInvalidUnsafePath is returned by Dir.Open when the call to +// filepath.Localize fails. filepath.Localize returns an error if the path +// cannot be represented by the operating system. +var errInvalidUnsafePath = errors.New("http: invalid or unsafe file path") + // Open implements [FileSystem] using [os.Open], opening files for reading rooted // and relative to the directory d. func (d Dir) Open(name string) (File, error) { @@ -76,7 +81,7 @@ func (d Dir) Open(name string) (File, error) { } path, err := filepath.Localize(path) if err != nil { - return nil, errors.New("http: invalid or unsafe file path") + return nil, errInvalidUnsafePath } dir := string(d) if dir == "" { @@ -768,6 +773,9 @@ func toHTTPError(err error) (msg string, httpStatus int) { if errors.Is(err, fs.ErrPermission) { return "403 Forbidden", StatusForbidden } + if errors.Is(err, errInvalidUnsafePath) { + return "404 page not found", StatusNotFound + } // Default: return "500 Internal Server Error", StatusInternalServerError } @@ -854,7 +862,7 @@ func containsDotDot(v string) bool { if !strings.Contains(v, "..") { return false } - for _, ent := range strings.FieldsFunc(v, isSlashRune) { + for ent := range strings.FieldsFuncSeq(v, isSlashRune) { if ent == ".." { return true } @@ -1014,7 +1022,7 @@ func parseRange(s string, size int64) ([]httpRange, error) { } var ranges []httpRange noOverlap := false - for _, ra := range strings.Split(s[len(b):], ",") { + for ra := range strings.SplitSeq(s[len(b):], ",") { ra = textproto.TrimString(ra) if ra == "" { continue diff --git a/src/net/http/fs_test.go b/src/net/http/fs_test.go index 3149ca35acd2a2..32fb696fee792a 100644 --- a/src/net/http/fs_test.go +++ b/src/net/http/fs_test.go @@ -733,6 +733,27 @@ func testFileServerZeroByte(t *testing.T, mode testMode) { } } +func TestFileServerNullByte(t *testing.T) { run(t, testFileServerNullByte) } +func testFileServerNullByte(t *testing.T, mode testMode) { + ts := newClientServerTest(t, mode, FileServer(Dir("testdata"))).ts + + for _, path := range []string{ + "/file%00", + "/%00", + "/file/qwe/%00", + } { + res, err := ts.Client().Get(ts.URL + path) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if res.StatusCode != 404 { + t.Errorf("Get(%q): got status %v, want 404", path, res.StatusCode) + } + + } +} + func TestFileServerNamesEscape(t *testing.T) { run(t, testFileServerNamesEscape) } func testFileServerNamesEscape(t *testing.T, mode testMode) { ts := newClientServerTest(t, mode, FileServer(Dir("testdata"))).ts @@ -1519,7 +1540,6 @@ func testServeFileRejectsInvalidSuffixLengths(t *testing.T, mode testMode) { } for _, tt := range tests { - tt := tt t.Run(tt.r, func(t *testing.T) { req, err := NewRequest("GET", cst.URL+"/index.html", nil) if err != nil { diff --git a/src/net/http/h2_bundle.go b/src/net/http/h2_bundle.go index af86c1430dbf6c..0df276321cf09e 100644 --- a/src/net/http/h2_bundle.go +++ b/src/net/http/h2_bundle.go @@ -1,7 +1,7 @@ //go:build !nethttpomithttp2 // Code generated by golang.org/x/tools/cmd/bundle. DO NOT EDIT. -// $ bundle -o=h2_bundle.go -prefix=http2 -tags=!nethttpomithttp2 golang.org/x/net/http2 +// $ bundle -o=h2_bundle.go -prefix=http2 -tags=!nethttpomithttp2 -import=golang.org/x/net/internal/httpcommon=net/http/internal/httpcommon golang.org/x/net/http2 // Package http2 implements the HTTP/2 protocol. // @@ -13,12 +13,6 @@ // // See https://http2.github.io/ for more information on HTTP/2. // -// See https://http2.golang.org/ for a test server running this code. -// -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. -// package http @@ -40,6 +34,7 @@ import ( mathrand "math/rand" "net" "net/http/httptrace" + "net/http/internal/httpcommon" "net/textproto" "net/url" "os" @@ -883,7 +878,7 @@ func (c *http2dialCall) dial(ctx context.Context, addr string) { // This code decides which ones live or die. // The return value used is whether c was used. // c is never closed. -func (p *http2clientConnPool) addConnIfNeeded(key string, t *http2Transport, c *tls.Conn) (used bool, err error) { +func (p *http2clientConnPool) addConnIfNeeded(key string, t *http2Transport, c net.Conn) (used bool, err error) { p.mu.Lock() for _, cc := range p.conns[key] { if cc.CanTakeNewRequest() { @@ -919,8 +914,8 @@ type http2addConnCall struct { err error } -func (c *http2addConnCall) run(t *http2Transport, key string, tc *tls.Conn) { - cc, err := t.NewClientConn(tc) +func (c *http2addConnCall) run(t *http2Transport, key string, nc net.Conn) { + cc, err := t.NewClientConn(nc) p := c.p p.mu.Lock() @@ -1035,6 +1030,168 @@ func http2shouldRetryDial(call *http2dialCall, req *Request) bool { return call.ctx.Err() != nil } +// http2Config is a package-internal version of net/http.HTTP2Config. +// +// http.HTTP2Config was added in Go 1.24. +// When running with a version of net/http that includes HTTP2Config, +// we merge the configuration with the fields in Transport or Server +// to produce an http2Config. +// +// Zero valued fields in http2Config are interpreted as in the +// net/http.HTTPConfig documentation. +// +// Precedence order for reconciling configurations is: +// +// - Use the net/http.{Server,Transport}.HTTP2Config value, when non-zero. +// - Otherwise use the http2.{Server.Transport} value. +// - If the resulting value is zero or out of range, use a default. +type http2http2Config struct { + MaxConcurrentStreams uint32 + StrictMaxConcurrentRequests bool + MaxDecoderHeaderTableSize uint32 + MaxEncoderHeaderTableSize uint32 + MaxReadFrameSize uint32 + MaxUploadBufferPerConnection int32 + MaxUploadBufferPerStream int32 + SendPingTimeout time.Duration + PingTimeout time.Duration + WriteByteTimeout time.Duration + PermitProhibitedCipherSuites bool + CountError func(errType string) +} + +// configFromServer merges configuration settings from +// net/http.Server.HTTP2Config and http2.Server. +func http2configFromServer(h1 *Server, h2 *http2Server) http2http2Config { + conf := http2http2Config{ + MaxConcurrentStreams: h2.MaxConcurrentStreams, + MaxEncoderHeaderTableSize: h2.MaxEncoderHeaderTableSize, + MaxDecoderHeaderTableSize: h2.MaxDecoderHeaderTableSize, + MaxReadFrameSize: h2.MaxReadFrameSize, + MaxUploadBufferPerConnection: h2.MaxUploadBufferPerConnection, + MaxUploadBufferPerStream: h2.MaxUploadBufferPerStream, + SendPingTimeout: h2.ReadIdleTimeout, + PingTimeout: h2.PingTimeout, + WriteByteTimeout: h2.WriteByteTimeout, + PermitProhibitedCipherSuites: h2.PermitProhibitedCipherSuites, + CountError: h2.CountError, + } + http2fillNetHTTPConfig(&conf, h1.HTTP2) + http2setConfigDefaults(&conf, true) + return conf +} + +// configFromTransport merges configuration settings from h2 and h2.t1.HTTP2 +// (the net/http Transport). +func http2configFromTransport(h2 *http2Transport) http2http2Config { + conf := http2http2Config{ + StrictMaxConcurrentRequests: h2.StrictMaxConcurrentStreams, + MaxEncoderHeaderTableSize: h2.MaxEncoderHeaderTableSize, + MaxDecoderHeaderTableSize: h2.MaxDecoderHeaderTableSize, + MaxReadFrameSize: h2.MaxReadFrameSize, + SendPingTimeout: h2.ReadIdleTimeout, + PingTimeout: h2.PingTimeout, + WriteByteTimeout: h2.WriteByteTimeout, + } + + // Unlike most config fields, where out-of-range values revert to the default, + // Transport.MaxReadFrameSize clips. + if conf.MaxReadFrameSize < http2minMaxFrameSize { + conf.MaxReadFrameSize = http2minMaxFrameSize + } else if conf.MaxReadFrameSize > http2maxFrameSize { + conf.MaxReadFrameSize = http2maxFrameSize + } + + if h2.t1 != nil { + http2fillNetHTTPConfig(&conf, h2.t1.HTTP2) + } + http2setConfigDefaults(&conf, false) + return conf +} + +func http2setDefault[T ~int | ~int32 | ~uint32 | ~int64](v *T, minval, maxval, defval T) { + if *v < minval || *v > maxval { + *v = defval + } +} + +func http2setConfigDefaults(conf *http2http2Config, server bool) { + http2setDefault(&conf.MaxConcurrentStreams, 1, math.MaxUint32, http2defaultMaxStreams) + http2setDefault(&conf.MaxEncoderHeaderTableSize, 1, math.MaxUint32, http2initialHeaderTableSize) + http2setDefault(&conf.MaxDecoderHeaderTableSize, 1, math.MaxUint32, http2initialHeaderTableSize) + if server { + http2setDefault(&conf.MaxUploadBufferPerConnection, http2initialWindowSize, math.MaxInt32, 1<<20) + } else { + http2setDefault(&conf.MaxUploadBufferPerConnection, http2initialWindowSize, math.MaxInt32, http2transportDefaultConnFlow) + } + if server { + http2setDefault(&conf.MaxUploadBufferPerStream, 1, math.MaxInt32, 1<<20) + } else { + http2setDefault(&conf.MaxUploadBufferPerStream, 1, math.MaxInt32, http2transportDefaultStreamFlow) + } + http2setDefault(&conf.MaxReadFrameSize, http2minMaxFrameSize, http2maxFrameSize, http2defaultMaxReadFrameSize) + http2setDefault(&conf.PingTimeout, 1, math.MaxInt64, 15*time.Second) +} + +// adjustHTTP1MaxHeaderSize converts a limit in bytes on the size of an HTTP/1 header +// to an HTTP/2 MAX_HEADER_LIST_SIZE value. +func http2adjustHTTP1MaxHeaderSize(n int64) int64 { + // http2's count is in a slightly different unit and includes 32 bytes per pair. + // So, take the net/http.Server value and pad it up a bit, assuming 10 headers. + const perFieldOverhead = 32 // per http2 spec + const typicalHeaders = 10 // conservative + return n + typicalHeaders*perFieldOverhead +} + +func http2fillNetHTTPConfig(conf *http2http2Config, h2 *HTTP2Config) { + if h2 == nil { + return + } + if h2.MaxConcurrentStreams != 0 { + conf.MaxConcurrentStreams = uint32(h2.MaxConcurrentStreams) + } + if http2http2ConfigStrictMaxConcurrentRequests(h2) { + conf.StrictMaxConcurrentRequests = true + } + if h2.MaxEncoderHeaderTableSize != 0 { + conf.MaxEncoderHeaderTableSize = uint32(h2.MaxEncoderHeaderTableSize) + } + if h2.MaxDecoderHeaderTableSize != 0 { + conf.MaxDecoderHeaderTableSize = uint32(h2.MaxDecoderHeaderTableSize) + } + if h2.MaxConcurrentStreams != 0 { + conf.MaxConcurrentStreams = uint32(h2.MaxConcurrentStreams) + } + if h2.MaxReadFrameSize != 0 { + conf.MaxReadFrameSize = uint32(h2.MaxReadFrameSize) + } + if h2.MaxReceiveBufferPerConnection != 0 { + conf.MaxUploadBufferPerConnection = int32(h2.MaxReceiveBufferPerConnection) + } + if h2.MaxReceiveBufferPerStream != 0 { + conf.MaxUploadBufferPerStream = int32(h2.MaxReceiveBufferPerStream) + } + if h2.SendPingTimeout != 0 { + conf.SendPingTimeout = h2.SendPingTimeout + } + if h2.PingTimeout != 0 { + conf.PingTimeout = h2.PingTimeout + } + if h2.WriteByteTimeout != 0 { + conf.WriteByteTimeout = h2.WriteByteTimeout + } + if h2.PermitProhibitedCipherSuites { + conf.PermitProhibitedCipherSuites = true + } + if h2.CountError != nil { + conf.CountError = h2.CountError + } +} + +func http2http2ConfigStrictMaxConcurrentRequests(h2 *HTTP2Config) bool { + return h2.StrictMaxConcurrentRequests +} + // Buffer chunks are allocated from a pool to reduce pressure on GC. // The maximum wasted space per dataBuffer is 2x the largest size class, // which happens when the dataBuffer has multiple chunks and there is @@ -1444,7 +1601,7 @@ const ( http2FrameContinuation http2FrameType = 0x9 ) -var http2frameName = map[http2FrameType]string{ +var http2frameNames = [...]string{ http2FrameData: "DATA", http2FrameHeaders: "HEADERS", http2FramePriority: "PRIORITY", @@ -1458,10 +1615,10 @@ var http2frameName = map[http2FrameType]string{ } func (t http2FrameType) String() string { - if s, ok := http2frameName[t]; ok { - return s + if int(t) < len(http2frameNames) { + return http2frameNames[t] } - return fmt.Sprintf("UNKNOWN_FRAME_TYPE_%d", uint8(t)) + return fmt.Sprintf("UNKNOWN_FRAME_TYPE_%d", t) } // Flags is a bitmask of HTTP/2 flags. @@ -1529,7 +1686,7 @@ var http2flagName = map[http2FrameType]map[http2Flags]string{ // might be 0). type http2frameParser func(fc *http2frameCache, fh http2FrameHeader, countError func(string), payload []byte) (http2Frame, error) -var http2frameParsers = map[http2FrameType]http2frameParser{ +var http2frameParsers = [...]http2frameParser{ http2FrameData: http2parseDataFrame, http2FrameHeaders: http2parseHeadersFrame, http2FramePriority: http2parsePriorityFrame, @@ -1543,8 +1700,8 @@ var http2frameParsers = map[http2FrameType]http2frameParser{ } func http2typeFrameParser(t http2FrameType) http2frameParser { - if f := http2frameParsers[t]; f != nil { - return f + if int(t) < len(http2frameParsers) { + return http2frameParsers[t] } return http2parseUnknownFrame } @@ -1630,6 +1787,11 @@ var http2fhBytes = sync.Pool{ }, } +func http2invalidHTTP1LookingFrameHeader() http2FrameHeader { + fh, _ := http2readFrameHeader(make([]byte, http2frameHeaderLen), strings.NewReader("HTTP/1.1 ")) + return fh +} + // ReadFrameHeader reads 9 bytes from r and returns a FrameHeader. // Most users should use Framer.ReadFrame instead. func http2ReadFrameHeader(r io.Reader) (http2FrameHeader, error) { @@ -1747,7 +1909,7 @@ func (fr *http2Framer) maxHeaderListSize() uint32 { func (f *http2Framer) startWrite(ftype http2FrameType, flags http2Flags, streamID uint32) { // Write the FrameHeader. f.wbuf = append(f.wbuf[:0], - 0, // 3 bytes of length, filled in in endWrite + 0, // 3 bytes of length, filled in endWrite 0, 0, byte(ftype), @@ -1911,10 +2073,16 @@ func (fr *http2Framer) ReadFrame() (http2Frame, error) { return nil, err } if fh.Length > fr.maxReadSize { + if fh == http2invalidHTTP1LookingFrameHeader() { + return nil, fmt.Errorf("http2: failed reading the frame payload: %w, note that the frame header looked like an HTTP/1.1 header", http2ErrFrameTooLarge) + } return nil, http2ErrFrameTooLarge } payload := fr.getReadBuf(fh.Length) if _, err := io.ReadFull(fr.r, payload); err != nil { + if fh == http2invalidHTTP1LookingFrameHeader() { + return nil, fmt.Errorf("http2: failed reading the frame payload: %w, note that the frame header looked like an HTTP/1.1 header", err) + } return nil, err } f, err := http2typeFrameParser(fh.Type)(fr.frameCache, fh, fr.countError, payload) @@ -2549,6 +2717,15 @@ type http2PriorityFrame struct { http2PriorityParam } +var http2defaultRFC9218Priority = http2PriorityParam{ + incremental: 0, + urgency: 3, +} + +// Note that HTTP/2 has had two different prioritization schemes, and +// PriorityParam struct below is a superset of both schemes. The exported +// symbols are from RFC 7540 and the non-exported ones are from RFC 9218. + // PriorityParam are the stream prioritzation parameters. type http2PriorityParam struct { // StreamDep is a 31-bit stream identifier for the @@ -2564,6 +2741,20 @@ type http2PriorityParam struct { // the spec, "Add one to the value to obtain a weight between // 1 and 256." Weight uint8 + + // "The urgency (u) parameter value is Integer (see Section 3.3.1 of + // [STRUCTURED-FIELDS]), between 0 and 7 inclusive, in descending order of + // priority. The default is 3." + urgency uint8 + + // "The incremental (i) parameter value is Boolean (see Section 3.3.6 of + // [STRUCTURED-FIELDS]). It indicates if an HTTP response can be processed + // incrementally, i.e., provide some meaningful output as chunks of the + // response arrive." + // + // We use uint8 (i.e. 0 is false, 1 is true) instead of bool so we can + // avoid unnecessary type conversions and because either type takes 1 byte. + incremental uint8 } func (p http2PriorityParam) IsZero() bool { @@ -2898,7 +3089,7 @@ func (mh *http2MetaHeadersFrame) checkPseudos() error { pf := mh.PseudoFields() for i, hf := range pf { switch hf.Name { - case ":method", ":path", ":scheme", ":authority": + case ":method", ":path", ":scheme", ":authority", ":protocol": isRequest = true case ":status": isResponse = true @@ -2906,7 +3097,7 @@ func (mh *http2MetaHeadersFrame) checkPseudos() error { return http2pseudoHeaderError(hf.Name) } // Check for duplicates. - // This would be a bad algorithm, but N is 4. + // This would be a bad algorithm, but N is 5. // And this doesn't allocate. for _, hf2 := range pf[:i] { if hf.Name == hf2.Name { @@ -3100,17 +3291,27 @@ func http2summarizeFrame(f http2Frame) string { var http2DebugGoroutines = os.Getenv("DEBUG_HTTP2_GOROUTINES") == "1" +// Setting DebugGoroutines to false during a test to disable goroutine debugging +// results in race detector complaints when a test leaves goroutines running before +// returning. Tests shouldn't do this, of course, but when they do it generally shows +// up as infrequent, hard-to-debug flakes. (See #66519.) +// +// Disable goroutine debugging during individual tests with an atomic bool. +// (Note that it's safe to enable/disable debugging mid-test, so the actual race condition +// here is harmless.) +var http2disableDebugGoroutines atomic.Bool + type http2goroutineLock uint64 func http2newGoroutineLock() http2goroutineLock { - if !http2DebugGoroutines { + if !http2DebugGoroutines || http2disableDebugGoroutines.Load() { return 0 } return http2goroutineLock(http2curGoroutineID()) } func (g http2goroutineLock) check() { - if !http2DebugGoroutines { + if !http2DebugGoroutines || http2disableDebugGoroutines.Load() { return } if http2curGoroutineID() != uint64(g) { @@ -3119,7 +3320,7 @@ func (g http2goroutineLock) check() { } func (g http2goroutineLock) checkNotOn() { - if !http2DebugGoroutines { + if !http2DebugGoroutines || http2disableDebugGoroutines.Load() { return } if http2curGoroutineID() == uint64(g) { @@ -3250,106 +3451,19 @@ func http2cutoff64(base int) uint64 { return (1<<64-1)/uint64(base) + 1 } -var ( - http2commonBuildOnce sync.Once - http2commonLowerHeader map[string]string // Go-Canonical-Case -> lower-case - http2commonCanonHeader map[string]string // lower-case -> Go-Canonical-Case -) - -func http2buildCommonHeaderMapsOnce() { - http2commonBuildOnce.Do(http2buildCommonHeaderMaps) -} - -func http2buildCommonHeaderMaps() { - common := []string{ - "accept", - "accept-charset", - "accept-encoding", - "accept-language", - "accept-ranges", - "age", - "access-control-allow-credentials", - "access-control-allow-headers", - "access-control-allow-methods", - "access-control-allow-origin", - "access-control-expose-headers", - "access-control-max-age", - "access-control-request-headers", - "access-control-request-method", - "allow", - "authorization", - "cache-control", - "content-disposition", - "content-encoding", - "content-language", - "content-length", - "content-location", - "content-range", - "content-type", - "cookie", - "date", - "etag", - "expect", - "expires", - "from", - "host", - "if-match", - "if-modified-since", - "if-none-match", - "if-unmodified-since", - "last-modified", - "link", - "location", - "max-forwards", - "origin", - "proxy-authenticate", - "proxy-authorization", - "range", - "referer", - "refresh", - "retry-after", - "server", - "set-cookie", - "strict-transport-security", - "trailer", - "transfer-encoding", - "user-agent", - "vary", - "via", - "www-authenticate", - "x-forwarded-for", - "x-forwarded-proto", - } - http2commonLowerHeader = make(map[string]string, len(common)) - http2commonCanonHeader = make(map[string]string, len(common)) - for _, v := range common { - chk := CanonicalHeaderKey(v) - http2commonLowerHeader[chk] = v - http2commonCanonHeader[v] = chk - } -} - -func http2lowerHeader(v string) (lower string, ascii bool) { - http2buildCommonHeaderMapsOnce() - if s, ok := http2commonLowerHeader[v]; ok { - return s, true - } - return http2asciiToLower(v) -} - -func http2canonicalHeader(v string) string { - http2buildCommonHeaderMapsOnce() - if s, ok := http2commonCanonHeader[v]; ok { - return s - } - return CanonicalHeaderKey(v) -} - var ( http2VerboseLogs bool http2logFrameWrites bool http2logFrameReads bool - http2inTests bool + + // Enabling extended CONNECT by causes browsers to attempt to use + // WebSockets-over-HTTP/2. This results in problems when the server's websocket + // package doesn't support extended CONNECT. + // + // Disable extended CONNECT by default for now. + // + // Issue #71128. + http2disableExtendedConnectProtocol = true ) func init() { @@ -3362,6 +3476,9 @@ func init() { http2logFrameWrites = true http2logFrameReads = true } + if strings.Contains(e, "http2xconnect=1") { + http2disableExtendedConnectProtocol = false + } } const ( @@ -3453,6 +3570,10 @@ func (s http2Setting) Valid() error { if s.Val < 16384 || s.Val > 1<<24-1 { return http2ConnectionError(http2ErrCodeProtocol) } + case http2SettingEnableConnectProtocol: + if s.Val != 1 && s.Val != 0 { + return http2ConnectionError(http2ErrCodeProtocol) + } } return nil } @@ -3462,21 +3583,23 @@ func (s http2Setting) Valid() error { type http2SettingID uint16 const ( - http2SettingHeaderTableSize http2SettingID = 0x1 - http2SettingEnablePush http2SettingID = 0x2 - http2SettingMaxConcurrentStreams http2SettingID = 0x3 - http2SettingInitialWindowSize http2SettingID = 0x4 - http2SettingMaxFrameSize http2SettingID = 0x5 - http2SettingMaxHeaderListSize http2SettingID = 0x6 + http2SettingHeaderTableSize http2SettingID = 0x1 + http2SettingEnablePush http2SettingID = 0x2 + http2SettingMaxConcurrentStreams http2SettingID = 0x3 + http2SettingInitialWindowSize http2SettingID = 0x4 + http2SettingMaxFrameSize http2SettingID = 0x5 + http2SettingMaxHeaderListSize http2SettingID = 0x6 + http2SettingEnableConnectProtocol http2SettingID = 0x8 ) var http2settingName = map[http2SettingID]string{ - http2SettingHeaderTableSize: "HEADER_TABLE_SIZE", - http2SettingEnablePush: "ENABLE_PUSH", - http2SettingMaxConcurrentStreams: "MAX_CONCURRENT_STREAMS", - http2SettingInitialWindowSize: "INITIAL_WINDOW_SIZE", - http2SettingMaxFrameSize: "MAX_FRAME_SIZE", - http2SettingMaxHeaderListSize: "MAX_HEADER_LIST_SIZE", + http2SettingHeaderTableSize: "HEADER_TABLE_SIZE", + http2SettingEnablePush: "ENABLE_PUSH", + http2SettingMaxConcurrentStreams: "MAX_CONCURRENT_STREAMS", + http2SettingInitialWindowSize: "INITIAL_WINDOW_SIZE", + http2SettingMaxFrameSize: "MAX_FRAME_SIZE", + http2SettingMaxHeaderListSize: "MAX_HEADER_LIST_SIZE", + http2SettingEnableConnectProtocol: "ENABLE_CONNECT_PROTOCOL", } func (s http2SettingID) String() string { @@ -3550,13 +3673,17 @@ func (cw http2closeWaiter) Wait() { // Its buffered writer is lazily allocated as needed, to minimize // idle memory usage with many connections. type http2bufferedWriter struct { - _ http2incomparable - w io.Writer // immutable - bw *bufio.Writer // non-nil when data is buffered + _ http2incomparable + conn net.Conn // immutable + bw *bufio.Writer // non-nil when data is buffered + byteTimeout time.Duration // immutable, WriteByteTimeout } -func http2newBufferedWriter(w io.Writer) *http2bufferedWriter { - return &http2bufferedWriter{w: w} +func http2newBufferedWriter(conn net.Conn, timeout time.Duration) *http2bufferedWriter { + return &http2bufferedWriter{ + conn: conn, + byteTimeout: timeout, + } } // bufWriterPoolBufferSize is the size of bufio.Writer's @@ -3583,7 +3710,7 @@ func (w *http2bufferedWriter) Available() int { func (w *http2bufferedWriter) Write(p []byte) (n int, err error) { if w.bw == nil { bw := http2bufWriterPool.Get().(*bufio.Writer) - bw.Reset(w.w) + bw.Reset((*http2bufferedWriterTimeoutWriter)(w)) w.bw = bw } return w.bw.Write(p) @@ -3601,6 +3728,32 @@ func (w *http2bufferedWriter) Flush() error { return err } +type http2bufferedWriterTimeoutWriter http2bufferedWriter + +func (w *http2bufferedWriterTimeoutWriter) Write(p []byte) (n int, err error) { + return http2writeWithByteTimeout(w.conn, w.byteTimeout, p) +} + +// writeWithByteTimeout writes to conn. +// If more than timeout passes without any bytes being written to the connection, +// the write fails. +func http2writeWithByteTimeout(conn net.Conn, timeout time.Duration, p []byte) (n int, err error) { + if timeout <= 0 { + return conn.Write(p) + } + for { + conn.SetWriteDeadline(time.Now().Add(timeout)) + nn, err := conn.Write(p[n:]) + n += nn + if n == len(p) || nn == 0 || !errors.Is(err, os.ErrDeadlineExceeded) { + // Either we finished the write, made no progress, or hit the deadline. + // Whichever it is, we're done now. + conn.SetWriteDeadline(time.Time{}) + return n, err + } + } +} + func http2mustUint31(v int32) uint32 { if v < 0 || v > 2147483647 { panic("out of range") @@ -3675,39 +3828,11 @@ func (s *http2sorter) SortStrings(ss []string) { s.v = save } -// validPseudoPath reports whether v is a valid :path pseudo-header -// value. It must be either: -// -// - a non-empty string starting with '/' -// - the string '*', for OPTIONS requests. -// -// For now this is only used a quick check for deciding when to clean -// up Opaque URLs before sending requests from the Transport. -// See golang.org/issue/16847 -// -// We used to enforce that the path also didn't start with "//", but -// Google's GFE accepts such paths and Chrome sends them, so ignore -// that part of the spec. See golang.org/issue/19103. -func http2validPseudoPath(v string) bool { - return (len(v) > 0 && v[0] == '/') || v == "*" -} - // incomparable is a zero-width, non-comparable type. Adding it to a struct // makes that struct also non-comparable, and generally doesn't add // any size (as long as it's first). type http2incomparable [0]func() -// synctestGroupInterface is the methods of synctestGroup used by Server and Transport. -// It's defined as an interface here to let us keep synctestGroup entirely test-only -// and not a part of non-test builds. -type http2synctestGroupInterface interface { - Join() - Now() time.Time - NewTimer(d time.Duration) http2timer - AfterFunc(d time.Duration, f func()) http2timer - ContextWithTimeout(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) -} - // pipe is a goroutine-safe io.Reader/io.Writer pair. It's like // io.Pipe except there are no PipeReader/PipeWriter halves, and the // underlying buffer is an interface. (io.Pipe is always unbuffered) @@ -3882,10 +4007,14 @@ func (p *http2pipe) Done() <-chan struct{} { } const ( - http2prefaceTimeout = 10 * time.Second - http2firstSettingsTimeout = 2 * time.Second // should be in-flight with preface anyway - http2handlerChunkWriteSize = 4 << 10 - http2defaultMaxStreams = 250 // TODO: make this 100 as the GFE seems to? + http2prefaceTimeout = 10 * time.Second + http2firstSettingsTimeout = 2 * time.Second // should be in-flight with preface anyway + http2handlerChunkWriteSize = 4 << 10 + http2defaultMaxStreams = 250 // TODO: make this 100 as the GFE seems to? + + // maxQueuedControlFrames is the maximum number of control frames like + // SETTINGS, PING and RST_STREAM that will be queued for writing before + // the connection is closed to prevent memory exhaustion attacks. http2maxQueuedControlFrames = 10000 ) @@ -3957,6 +4086,22 @@ type http2Server struct { // If zero or negative, there is no timeout. IdleTimeout time.Duration + // ReadIdleTimeout is the timeout after which a health check using a ping + // frame will be carried out if no frame is received on the connection. + // If zero, no health check is performed. + ReadIdleTimeout time.Duration + + // PingTimeout is the timeout after which the connection will be closed + // if a response to a ping is not received. + // If zero, a default of 15 seconds is used. + PingTimeout time.Duration + + // WriteByteTimeout is the timeout after which a connection will be + // closed if no data can be written to it. The timeout begins when data is + // available to write, and is extended whenever any bytes are written. + // If zero or negative, there is no timeout. + WriteByteTimeout time.Duration + // MaxUploadBufferPerConnection is the size of the initial flow // control window for each connections. The HTTP/2 spec does not // allow this to be smaller than 65535 or larger than 2^32-1. @@ -3984,95 +4129,15 @@ type http2Server struct { // so that we don't embed a Mutex in this struct, which will make the // struct non-copyable, which might break some callers. state *http2serverInternalState - - // Synchronization group used for testing. - // Outside of tests, this is nil. - group http2synctestGroupInterface -} - -func (s *http2Server) markNewGoroutine() { - if s.group != nil { - s.group.Join() - } -} - -func (s *http2Server) now() time.Time { - if s.group != nil { - return s.group.Now() - } - return time.Now() -} - -// newTimer creates a new time.Timer, or a synthetic timer in tests. -func (s *http2Server) newTimer(d time.Duration) http2timer { - if s.group != nil { - return s.group.NewTimer(d) - } - return http2timeTimer{time.NewTimer(d)} -} - -// afterFunc creates a new time.AfterFunc timer, or a synthetic timer in tests. -func (s *http2Server) afterFunc(d time.Duration, f func()) http2timer { - if s.group != nil { - return s.group.AfterFunc(d, f) - } - return http2timeTimer{time.AfterFunc(d, f)} -} - -func (s *http2Server) initialConnRecvWindowSize() int32 { - if s.MaxUploadBufferPerConnection >= http2initialWindowSize { - return s.MaxUploadBufferPerConnection - } - return 1 << 20 -} - -func (s *http2Server) initialStreamRecvWindowSize() int32 { - if s.MaxUploadBufferPerStream > 0 { - return s.MaxUploadBufferPerStream - } - return 1 << 20 -} - -func (s *http2Server) maxReadFrameSize() uint32 { - if v := s.MaxReadFrameSize; v >= http2minMaxFrameSize && v <= http2maxFrameSize { - return v - } - return http2defaultMaxReadFrameSize -} - -func (s *http2Server) maxConcurrentStreams() uint32 { - if v := s.MaxConcurrentStreams; v > 0 { - return v - } - return http2defaultMaxStreams -} - -func (s *http2Server) maxDecoderHeaderTableSize() uint32 { - if v := s.MaxDecoderHeaderTableSize; v > 0 { - return v - } - return http2initialHeaderTableSize -} - -func (s *http2Server) maxEncoderHeaderTableSize() uint32 { - if v := s.MaxEncoderHeaderTableSize; v > 0 { - return v - } - return http2initialHeaderTableSize -} - -// maxQueuedControlFrames is the maximum number of control frames like -// SETTINGS, PING and RST_STREAM that will be queued for writing before -// the connection is closed to prevent memory exhaustion attacks. -func (s *http2Server) maxQueuedControlFrames() int { - // TODO: if anybody asks, add a Server field, and remember to define the - // behavior of negative values. - return http2maxQueuedControlFrames } type http2serverInternalState struct { mu sync.Mutex activeConns map[*http2serverConn]struct{} + + // Pool of error channels. This is per-Server rather than global + // because channels can't be reused across synctest bubbles. + errChanPool sync.Pool } func (s *http2serverInternalState) registerConn(sc *http2serverConn) { @@ -4104,6 +4169,27 @@ func (s *http2serverInternalState) startGracefulShutdown() { s.mu.Unlock() } +// Global error channel pool used for uninitialized Servers. +// We use a per-Server pool when possible to avoid using channels across synctest bubbles. +var http2errChanPool = sync.Pool{ + New: func() any { return make(chan error, 1) }, +} + +func (s *http2serverInternalState) getErrChan() chan error { + if s == nil { + return http2errChanPool.Get().(chan error) // Server used without calling ConfigureServer + } + return s.errChanPool.Get().(chan error) +} + +func (s *http2serverInternalState) putErrChan(ch chan error) { + if s == nil { + http2errChanPool.Put(ch) // Server used without calling ConfigureServer + return + } + s.errChanPool.Put(ch) +} + // ConfigureServer adds HTTP/2 support to a net/http Server. // // The configuration conf may be nil. @@ -4116,7 +4202,10 @@ func http2ConfigureServer(s *Server, conf *http2Server) error { if conf == nil { conf = new(http2Server) } - conf.state = &http2serverInternalState{activeConns: make(map[*http2serverConn]struct{})} + conf.state = &http2serverInternalState{ + activeConns: make(map[*http2serverConn]struct{}), + errChanPool: sync.Pool{New: func() any { return make(chan error, 1) }}, + } if h1, h2 := s, conf; h2.IdleTimeout == 0 { if h1.IdleTimeout != 0 { h2.IdleTimeout = h1.IdleTimeout @@ -4166,7 +4255,7 @@ func http2ConfigureServer(s *Server, conf *http2Server) error { if s.TLSNextProto == nil { s.TLSNextProto = map[string]func(*Server, *tls.Conn, Handler){} } - protoHandler := func(hs *Server, c *tls.Conn, h Handler) { + protoHandler := func(hs *Server, c net.Conn, h Handler, sawClientPreface bool) { if http2testHookOnConn != nil { http2testHookOnConn() } @@ -4183,12 +4272,31 @@ func http2ConfigureServer(s *Server, conf *http2Server) error { ctx = bc.BaseContext() } conf.ServeConn(c, &http2ServeConnOpts{ - Context: ctx, - Handler: h, - BaseConfig: hs, + Context: ctx, + Handler: h, + BaseConfig: hs, + SawClientPreface: sawClientPreface, }) } - s.TLSNextProto[http2NextProtoTLS] = protoHandler + s.TLSNextProto[http2NextProtoTLS] = func(hs *Server, c *tls.Conn, h Handler) { + protoHandler(hs, c, h, false) + } + // The "unencrypted_http2" TLSNextProto key is used to pass off non-TLS HTTP/2 conns. + // + // A connection passed in this method has already had the HTTP/2 preface read from it. + s.TLSNextProto[http2nextProtoUnencryptedHTTP2] = func(hs *Server, c *tls.Conn, h Handler) { + nc, err := http2unencryptedNetConnFromTLSConn(c) + if err != nil { + if lg := hs.ErrorLog; lg != nil { + lg.Print(err) + } else { + log.Print(err) + } + go c.Close() + return + } + protoHandler(hs, nc, h, true) + } return nil } @@ -4263,6 +4371,9 @@ func (o *http2ServeConnOpts) handler() Handler { // // The opts parameter is optional. If nil, default values are used. func (s *http2Server) ServeConn(c net.Conn, opts *http2ServeConnOpts) { + if opts == nil { + opts = &http2ServeConnOpts{} + } s.serveConn(c, opts, nil) } @@ -4270,13 +4381,15 @@ func (s *http2Server) serveConn(c net.Conn, opts *http2ServeConnOpts, newf func( baseCtx, cancel := http2serverConnBaseContext(c, opts) defer cancel() + http1srv := opts.baseConfig() + conf := http2configFromServer(http1srv, s) sc := &http2serverConn{ srv: s, - hs: opts.baseConfig(), + hs: http1srv, conn: c, baseCtx: baseCtx, remoteAddrStr: c.RemoteAddr().String(), - bw: http2newBufferedWriter(c), + bw: http2newBufferedWriter(c, conf.WriteByteTimeout), handler: opts.handler(), streams: make(map[uint32]*http2stream), readFrameCh: make(chan http2readFrameResult), @@ -4286,9 +4399,12 @@ func (s *http2Server) serveConn(c net.Conn, opts *http2ServeConnOpts, newf func( bodyReadCh: make(chan http2bodyReadMsg), // buffering doesn't matter either way doneServing: make(chan struct{}), clientMaxStreams: math.MaxUint32, // Section 6.5.2: "Initially, there is no limit to this value" - advMaxStreams: s.maxConcurrentStreams(), + advMaxStreams: conf.MaxConcurrentStreams, initialStreamSendWindowSize: http2initialWindowSize, + initialStreamRecvWindowSize: conf.MaxUploadBufferPerStream, maxFrameSize: http2initialMaxFrameSize, + pingTimeout: conf.PingTimeout, + countErrorFunc: conf.CountError, serveG: http2newGoroutineLock(), pushEnabled: true, sawClientPreface: opts.SawClientPreface, @@ -4321,15 +4437,15 @@ func (s *http2Server) serveConn(c net.Conn, opts *http2ServeConnOpts, newf func( sc.flow.add(http2initialWindowSize) sc.inflow.init(http2initialWindowSize) sc.hpackEncoder = hpack.NewEncoder(&sc.headerWriteBuf) - sc.hpackEncoder.SetMaxDynamicTableSizeLimit(s.maxEncoderHeaderTableSize()) + sc.hpackEncoder.SetMaxDynamicTableSizeLimit(conf.MaxEncoderHeaderTableSize) fr := http2NewFramer(sc.bw, c) - if s.CountError != nil { - fr.countError = s.CountError + if conf.CountError != nil { + fr.countError = conf.CountError } - fr.ReadMetaHeaders = hpack.NewDecoder(s.maxDecoderHeaderTableSize(), nil) + fr.ReadMetaHeaders = hpack.NewDecoder(conf.MaxDecoderHeaderTableSize, nil) fr.MaxHeaderListSize = sc.maxHeaderListSize() - fr.SetMaxReadFrameSize(s.maxReadFrameSize()) + fr.SetMaxReadFrameSize(conf.MaxReadFrameSize) sc.framer = fr if tc, ok := c.(http2connectionStater); ok { @@ -4362,7 +4478,7 @@ func (s *http2Server) serveConn(c net.Conn, opts *http2ServeConnOpts, newf func( // So for now, do nothing here again. } - if !s.PermitProhibitedCipherSuites && http2isBadCipher(sc.tlsState.CipherSuite) { + if !conf.PermitProhibitedCipherSuites && http2isBadCipher(sc.tlsState.CipherSuite) { // "Endpoints MAY choose to generate a connection error // (Section 5.4.1) of type INADEQUATE_SECURITY if one of // the prohibited cipher suites are negotiated." @@ -4399,7 +4515,7 @@ func (s *http2Server) serveConn(c net.Conn, opts *http2ServeConnOpts, newf func( opts.UpgradeRequest = nil } - sc.serve() + sc.serve(conf) } func http2serverConnBaseContext(c net.Conn, opts *http2ServeConnOpts) (ctx context.Context, cancel func()) { @@ -4439,6 +4555,7 @@ type http2serverConn struct { tlsState *tls.ConnectionState // shared by all handlers, like net/http remoteAddrStr string writeSched http2WriteScheduler + countErrorFunc func(errType string) // Everything following is owned by the serve loop; use serveG.check(): serveG http2goroutineLock // used to verify funcs are on serve() @@ -4458,6 +4575,7 @@ type http2serverConn struct { streams map[uint32]*http2stream unstartedHandlers []http2unstartedHandler initialStreamSendWindowSize int32 + initialStreamRecvWindowSize int32 maxFrameSize int32 peerMaxHeaderListSize uint32 // zero means unknown (default) canonHeader map[string]string // http2-lower-case -> Go-Canonical-Case @@ -4468,9 +4586,14 @@ type http2serverConn struct { inGoAway bool // we've started to or sent GOAWAY inFrameScheduleLoop bool // whether we're in the scheduleFrameWrite loop needToSendGoAway bool // we need to schedule a GOAWAY frame write + pingSent bool + sentPingData [8]byte goAwayCode http2ErrCode - shutdownTimer http2timer // nil until used - idleTimer http2timer // nil if unused + shutdownTimer *time.Timer // nil until used + idleTimer *time.Timer // nil if unused + readIdleTimeout time.Duration + pingTimeout time.Duration + readIdleTimer *time.Timer // nil if unused // Owned by the writeFrameAsync goroutine: headerWriteBuf bytes.Buffer @@ -4485,11 +4608,7 @@ func (sc *http2serverConn) maxHeaderListSize() uint32 { if n <= 0 { n = DefaultMaxHeaderBytes } - // http2's count is in a slightly different unit and includes 32 bytes per pair. - // So, take the net/http.Server value and pad it up a bit, assuming 10 headers. - const perFieldOverhead = 32 // per http2 spec - const typicalHeaders = 10 // conservative - return uint32(n + typicalHeaders*perFieldOverhead) + return uint32(http2adjustHTTP1MaxHeaderSize(int64(n))) } func (sc *http2serverConn) curOpenStreams() uint32 { @@ -4519,12 +4638,12 @@ type http2stream struct { flow http2outflow // limits writing from Handler to client inflow http2inflow // what the client is allowed to POST/etc to us state http2streamState - resetQueued bool // RST_STREAM queued for write; set by sc.resetStream - gotTrailerHeader bool // HEADER frame for trailers was seen - wroteHeaders bool // whether we wrote headers (not status 100) - readDeadline http2timer // nil if unused - writeDeadline http2timer // nil if unused - closeErr error // set before cw is closed + resetQueued bool // RST_STREAM queued for write; set by sc.resetStream + gotTrailerHeader bool // HEADER frame for trailers was seen + wroteHeaders bool // whether we wrote headers (not status 100) + readDeadline *time.Timer // nil if unused + writeDeadline *time.Timer // nil if unused + closeErr error // set before cw is closed trailer Header // accumulated trailers reqTrailer Header // handler's Request.Trailer @@ -4648,8 +4767,7 @@ const http2maxCachedCanonicalHeadersKeysSize = 2048 func (sc *http2serverConn) canonicalHeader(v string) string { sc.serveG.check() - http2buildCommonHeaderMapsOnce() - cv, ok := http2commonCanonHeader[v] + cv, ok := httpcommon.CachedCanonicalHeader(v) if ok { return cv } @@ -4684,7 +4802,6 @@ type http2readFrameResult struct { // consumer is done with the frame. // It's run on its own goroutine. func (sc *http2serverConn) readFrames() { - sc.srv.markNewGoroutine() gate := make(chan struct{}) gateDone := func() { gate <- struct{}{} } for { @@ -4717,7 +4834,6 @@ type http2frameWriteResult struct { // At most one goroutine can be running writeFrameAsync at a time per // serverConn. func (sc *http2serverConn) writeFrameAsync(wr http2FrameWriteRequest, wd *http2writeData) { - sc.srv.markNewGoroutine() var err error if wd == nil { err = wr.write.writeFrame(sc) @@ -4756,7 +4872,7 @@ func (sc *http2serverConn) notePanic() { } } -func (sc *http2serverConn) serve() { +func (sc *http2serverConn) serve(conf http2http2Config) { sc.serveG.check() defer sc.notePanic() defer sc.conn.Close() @@ -4768,20 +4884,24 @@ func (sc *http2serverConn) serve() { sc.vlogf("http2: server connection from %v on %p", sc.conn.RemoteAddr(), sc.hs) } + settings := http2writeSettings{ + {http2SettingMaxFrameSize, conf.MaxReadFrameSize}, + {http2SettingMaxConcurrentStreams, sc.advMaxStreams}, + {http2SettingMaxHeaderListSize, sc.maxHeaderListSize()}, + {http2SettingHeaderTableSize, conf.MaxDecoderHeaderTableSize}, + {http2SettingInitialWindowSize, uint32(sc.initialStreamRecvWindowSize)}, + } + if !http2disableExtendedConnectProtocol { + settings = append(settings, http2Setting{http2SettingEnableConnectProtocol, 1}) + } sc.writeFrame(http2FrameWriteRequest{ - write: http2writeSettings{ - {http2SettingMaxFrameSize, sc.srv.maxReadFrameSize()}, - {http2SettingMaxConcurrentStreams, sc.advMaxStreams}, - {http2SettingMaxHeaderListSize, sc.maxHeaderListSize()}, - {http2SettingHeaderTableSize, sc.srv.maxDecoderHeaderTableSize()}, - {http2SettingInitialWindowSize, uint32(sc.srv.initialStreamRecvWindowSize())}, - }, + write: settings, }) sc.unackedSettings++ // Each connection starts with initialWindowSize inflow tokens. // If a higher value is configured, we add more tokens. - if diff := sc.srv.initialConnRecvWindowSize() - http2initialWindowSize; diff > 0 { + if diff := conf.MaxUploadBufferPerConnection - http2initialWindowSize; diff > 0 { sc.sendWindowUpdate(nil, int(diff)) } @@ -4797,15 +4917,22 @@ func (sc *http2serverConn) serve() { sc.setConnState(StateIdle) if sc.srv.IdleTimeout > 0 { - sc.idleTimer = sc.srv.afterFunc(sc.srv.IdleTimeout, sc.onIdleTimer) + sc.idleTimer = time.AfterFunc(sc.srv.IdleTimeout, sc.onIdleTimer) defer sc.idleTimer.Stop() } + if conf.SendPingTimeout > 0 { + sc.readIdleTimeout = conf.SendPingTimeout + sc.readIdleTimer = time.AfterFunc(conf.SendPingTimeout, sc.onReadIdleTimer) + defer sc.readIdleTimer.Stop() + } + go sc.readFrames() // closed by defer sc.conn.Close above - settingsTimer := sc.srv.afterFunc(http2firstSettingsTimeout, sc.onSettingsTimer) + settingsTimer := time.AfterFunc(http2firstSettingsTimeout, sc.onSettingsTimer) defer settingsTimer.Stop() + lastFrameTime := time.Now() loopNum := 0 for { loopNum++ @@ -4819,6 +4946,7 @@ func (sc *http2serverConn) serve() { case res := <-sc.wroteFrameCh: sc.wroteFrame(res) case res := <-sc.readFrameCh: + lastFrameTime = time.Now() // Process any written frames before reading new frames from the client since a // written frame could have triggered a new stream to be started. if sc.writingFrameAsync { @@ -4850,6 +4978,8 @@ func (sc *http2serverConn) serve() { case http2idleTimerMsg: sc.vlogf("connection is idle") sc.goAway(http2ErrCodeNo) + case http2readIdleTimerMsg: + sc.handlePingTimer(lastFrameTime) case http2shutdownTimerMsg: sc.vlogf("GOAWAY close timer fired; closing conn from %v", sc.conn.RemoteAddr()) return @@ -4872,7 +5002,7 @@ func (sc *http2serverConn) serve() { // If the peer is causing us to generate a lot of control frames, // but not reading them from us, assume they are trying to make us // run out of memory. - if sc.queuedControlFrames > sc.srv.maxQueuedControlFrames() { + if sc.queuedControlFrames > http2maxQueuedControlFrames { sc.vlogf("http2: too many control frames in send queue, closing connection") return } @@ -4888,12 +5018,42 @@ func (sc *http2serverConn) serve() { } } +func (sc *http2serverConn) handlePingTimer(lastFrameReadTime time.Time) { + if sc.pingSent { + sc.logf("timeout waiting for PING response") + if f := sc.countErrorFunc; f != nil { + f("conn_close_lost_ping") + } + sc.conn.Close() + return + } + + pingAt := lastFrameReadTime.Add(sc.readIdleTimeout) + now := time.Now() + if pingAt.After(now) { + // We received frames since arming the ping timer. + // Reset it for the next possible timeout. + sc.readIdleTimer.Reset(pingAt.Sub(now)) + return + } + + sc.pingSent = true + // Ignore crypto/rand.Read errors: It generally can't fail, and worse case if it does + // is we send a PING frame containing 0s. + _, _ = rand.Read(sc.sentPingData[:]) + sc.writeFrame(http2FrameWriteRequest{ + write: &http2writePing{data: sc.sentPingData}, + }) + sc.readIdleTimer.Reset(sc.pingTimeout) +} + type http2serverMessage int // Message values sent to serveMsgCh. var ( http2settingsTimerMsg = new(http2serverMessage) http2idleTimerMsg = new(http2serverMessage) + http2readIdleTimerMsg = new(http2serverMessage) http2shutdownTimerMsg = new(http2serverMessage) http2gracefulShutdownMsg = new(http2serverMessage) http2handlerDoneMsg = new(http2serverMessage) @@ -4903,6 +5063,8 @@ func (sc *http2serverConn) onSettingsTimer() { sc.sendServeMsg(http2settingsTime func (sc *http2serverConn) onIdleTimer() { sc.sendServeMsg(http2idleTimerMsg) } +func (sc *http2serverConn) onReadIdleTimer() { sc.sendServeMsg(http2readIdleTimerMsg) } + func (sc *http2serverConn) onShutdownTimer() { sc.sendServeMsg(http2shutdownTimerMsg) } func (sc *http2serverConn) sendServeMsg(msg interface{}) { @@ -4934,10 +5096,10 @@ func (sc *http2serverConn) readPreface() error { errc <- nil } }() - timer := sc.srv.newTimer(http2prefaceTimeout) // TODO: configurable on *Server? + timer := time.NewTimer(http2prefaceTimeout) // TODO: configurable on *Server? defer timer.Stop() select { - case <-timer.C(): + case <-timer.C: return http2errPrefaceTimeout case err := <-errc: if err == nil { @@ -4949,10 +5111,6 @@ func (sc *http2serverConn) readPreface() error { } } -var http2errChanPool = sync.Pool{ - New: func() interface{} { return make(chan error, 1) }, -} - var http2writeDataPool = sync.Pool{ New: func() interface{} { return new(http2writeData) }, } @@ -4960,7 +5118,7 @@ var http2writeDataPool = sync.Pool{ // writeDataFromHandler writes DATA response frames from a handler on // the given stream. func (sc *http2serverConn) writeDataFromHandler(stream *http2stream, data []byte, endStream bool) error { - ch := http2errChanPool.Get().(chan error) + ch := sc.srv.state.getErrChan() writeArg := http2writeDataPool.Get().(*http2writeData) *writeArg = http2writeData{stream.id, data, endStream} err := sc.writeFrameFromHandler(http2FrameWriteRequest{ @@ -4992,7 +5150,7 @@ func (sc *http2serverConn) writeDataFromHandler(stream *http2stream, data []byte return http2errStreamClosed } } - http2errChanPool.Put(ch) + sc.srv.state.putErrChan(ch) if frameWriteDone { http2writeDataPool.Put(writeArg) } @@ -5155,6 +5313,10 @@ func (sc *http2serverConn) wroteFrame(res http2frameWriteResult) { sc.writingFrame = false sc.writingFrameAsync = false + if res.err != nil { + sc.conn.Close() + } + wr := res.wr if http2writeEndsStream(wr.write) { @@ -5302,7 +5464,7 @@ func (sc *http2serverConn) goAway(code http2ErrCode) { func (sc *http2serverConn) shutDownIn(d time.Duration) { sc.serveG.check() - sc.shutdownTimer = sc.srv.afterFunc(d, sc.onShutdownTimer) + sc.shutdownTimer = time.AfterFunc(d, sc.onShutdownTimer) } func (sc *http2serverConn) resetStream(se http2StreamError) { @@ -5429,6 +5591,11 @@ func (sc *http2serverConn) processFrame(f http2Frame) error { func (sc *http2serverConn) processPing(f *http2PingFrame) error { sc.serveG.check() if f.IsAck() { + if sc.pingSent && sc.sentPingData == f.Data { + // This is a response to a PING we sent. + sc.pingSent = false + sc.readIdleTimer.Reset(sc.readIdleTimeout) + } // 6.7 PING: " An endpoint MUST NOT respond to PING frames // containing this flag." return nil @@ -5592,6 +5759,9 @@ func (sc *http2serverConn) processSetting(s http2Setting) error { sc.maxFrameSize = int32(s.Val) // the maximum valid s.Val is < 2^31 case http2SettingMaxHeaderListSize: sc.peerMaxHeaderListSize = s.Val + case http2SettingEnableConnectProtocol: + // Receipt of this parameter by a server does not + // have any impact default: // Unknown setting: "An endpoint that receives a SETTINGS // frame with any unknown or unsupported identifier MUST @@ -5899,7 +6069,7 @@ func (sc *http2serverConn) processHeaders(f *http2MetaHeadersFrame) error { // (in Go 1.8), though. That's a more sane option anyway. if sc.hs.ReadTimeout > 0 { sc.conn.SetReadDeadline(time.Time{}) - st.readDeadline = sc.srv.afterFunc(sc.hs.ReadTimeout, st.onReadTimeout) + st.readDeadline = time.AfterFunc(sc.hs.ReadTimeout, st.onReadTimeout) } return sc.scheduleHandler(id, rw, req, handler) @@ -5995,9 +6165,9 @@ func (sc *http2serverConn) newStream(id, pusherID uint32, state http2streamState st.cw.Init() st.flow.conn = &sc.flow // link to conn-level counter st.flow.add(sc.initialStreamSendWindowSize) - st.inflow.init(sc.srv.initialStreamRecvWindowSize()) + st.inflow.init(sc.initialStreamRecvWindowSize) if sc.hs.WriteTimeout > 0 { - st.writeDeadline = sc.srv.afterFunc(sc.hs.WriteTimeout, st.onWriteTimeout) + st.writeDeadline = time.AfterFunc(sc.hs.WriteTimeout, st.onWriteTimeout) } sc.streams[id] = st @@ -6017,19 +6187,25 @@ func (sc *http2serverConn) newStream(id, pusherID uint32, state http2streamState func (sc *http2serverConn) newWriterAndRequest(st *http2stream, f *http2MetaHeadersFrame) (*http2responseWriter, *Request, error) { sc.serveG.check() - rp := http2requestParam{ - method: f.PseudoValue("method"), - scheme: f.PseudoValue("scheme"), - authority: f.PseudoValue("authority"), - path: f.PseudoValue("path"), + rp := httpcommon.ServerRequestParam{ + Method: f.PseudoValue("method"), + Scheme: f.PseudoValue("scheme"), + Authority: f.PseudoValue("authority"), + Path: f.PseudoValue("path"), + Protocol: f.PseudoValue("protocol"), + } + + // extended connect is disabled, so we should not see :protocol + if http2disableExtendedConnectProtocol && rp.Protocol != "" { + return nil, nil, sc.countError("bad_connect", http2streamError(f.StreamID, http2ErrCodeProtocol)) } - isConnect := rp.method == "CONNECT" + isConnect := rp.Method == "CONNECT" if isConnect { - if rp.path != "" || rp.scheme != "" || rp.authority == "" { + if rp.Protocol == "" && (rp.Path != "" || rp.Scheme != "" || rp.Authority == "") { return nil, nil, sc.countError("bad_connect", http2streamError(f.StreamID, http2ErrCodeProtocol)) } - } else if rp.method == "" || rp.path == "" || (rp.scheme != "https" && rp.scheme != "http") { + } else if rp.Method == "" || rp.Path == "" || (rp.Scheme != "https" && rp.Scheme != "http") { // See 8.1.2.6 Malformed Requests and Responses: // // Malformed requests or responses that are detected @@ -6043,12 +6219,16 @@ func (sc *http2serverConn) newWriterAndRequest(st *http2stream, f *http2MetaHead return nil, nil, sc.countError("bad_path_method", http2streamError(f.StreamID, http2ErrCodeProtocol)) } - rp.header = make(Header) + header := make(Header) + rp.Header = header for _, hf := range f.RegularFields() { - rp.header.Add(sc.canonicalHeader(hf.Name), hf.Value) + header.Add(sc.canonicalHeader(hf.Name), hf.Value) } - if rp.authority == "" { - rp.authority = rp.header.Get("Host") + if rp.Authority == "" { + rp.Authority = header.Get("Host") + } + if rp.Protocol != "" { + header.Set(":protocol", rp.Protocol) } rw, req, err := sc.newWriterAndRequestNoBody(st, rp) @@ -6057,7 +6237,7 @@ func (sc *http2serverConn) newWriterAndRequest(st *http2stream, f *http2MetaHead } bodyOpen := !f.StreamEnded() if bodyOpen { - if vv, ok := rp.header["Content-Length"]; ok { + if vv, ok := rp.Header["Content-Length"]; ok { if cl, err := strconv.ParseUint(vv[0], 10, 63); err == nil { req.ContentLength = int64(cl) } else { @@ -6073,83 +6253,38 @@ func (sc *http2serverConn) newWriterAndRequest(st *http2stream, f *http2MetaHead return rw, req, nil } -type http2requestParam struct { - method string - scheme, authority, path string - header Header -} - -func (sc *http2serverConn) newWriterAndRequestNoBody(st *http2stream, rp http2requestParam) (*http2responseWriter, *Request, error) { +func (sc *http2serverConn) newWriterAndRequestNoBody(st *http2stream, rp httpcommon.ServerRequestParam) (*http2responseWriter, *Request, error) { sc.serveG.check() var tlsState *tls.ConnectionState // nil if not scheme https - if rp.scheme == "https" { + if rp.Scheme == "https" { tlsState = sc.tlsState } - needsContinue := httpguts.HeaderValuesContainsToken(rp.header["Expect"], "100-continue") - if needsContinue { - rp.header.Del("Expect") - } - // Merge Cookie headers into one "; "-delimited value. - if cookies := rp.header["Cookie"]; len(cookies) > 1 { - rp.header.Set("Cookie", strings.Join(cookies, "; ")) - } - - // Setup Trailers - var trailer Header - for _, v := range rp.header["Trailer"] { - for _, key := range strings.Split(v, ",") { - key = CanonicalHeaderKey(textproto.TrimString(key)) - switch key { - case "Transfer-Encoding", "Trailer", "Content-Length": - // Bogus. (copy of http1 rules) - // Ignore. - default: - if trailer == nil { - trailer = make(Header) - } - trailer[key] = nil - } - } - } - delete(rp.header, "Trailer") - - var url_ *url.URL - var requestURI string - if rp.method == "CONNECT" { - url_ = &url.URL{Host: rp.authority} - requestURI = rp.authority // mimic HTTP/1 server behavior - } else { - var err error - url_, err = url.ParseRequestURI(rp.path) - if err != nil { - return nil, nil, sc.countError("bad_path", http2streamError(st.id, http2ErrCodeProtocol)) - } - requestURI = rp.path + res := httpcommon.NewServerRequest(rp) + if res.InvalidReason != "" { + return nil, nil, sc.countError(res.InvalidReason, http2streamError(st.id, http2ErrCodeProtocol)) } body := &http2requestBody{ conn: sc, stream: st, - needsContinue: needsContinue, + needsContinue: res.NeedsContinue, } - req := &Request{ - Method: rp.method, - URL: url_, + req := (&Request{ + Method: rp.Method, + URL: res.URL, RemoteAddr: sc.remoteAddrStr, - Header: rp.header, - RequestURI: requestURI, + Header: rp.Header, + RequestURI: res.RequestURI, Proto: "HTTP/2.0", ProtoMajor: 2, ProtoMinor: 0, TLS: tlsState, - Host: rp.authority, + Host: rp.Authority, Body: body, - Trailer: trailer, - } - req = req.WithContext(st.ctx) - + Trailer: res.Trailer, + }).WithContext(st.ctx) rw := sc.newResponseWriter(st, req) return rw, req, nil } @@ -6221,7 +6356,6 @@ func (sc *http2serverConn) handlerDone() { // Run on its own goroutine. func (sc *http2serverConn) runHandler(rw *http2responseWriter, req *Request, handler func(ResponseWriter, *Request)) { - sc.srv.markNewGoroutine() defer sc.sendServeMsg(http2handlerDoneMsg) didPanic := true defer func() { @@ -6270,7 +6404,7 @@ func (sc *http2serverConn) writeHeaders(st *http2stream, headerData *http2writeR // waiting for this frame to be written, so an http.Flush mid-handler // writes out the correct value of keys, before a handler later potentially // mutates it. - errc = http2errChanPool.Get().(chan error) + errc = sc.srv.state.getErrChan() } if err := sc.writeFrameFromHandler(http2FrameWriteRequest{ write: headerData, @@ -6282,7 +6416,7 @@ func (sc *http2serverConn) writeHeaders(st *http2stream, headerData *http2writeR if errc != nil { select { case err := <-errc: - http2errChanPool.Put(errc) + sc.srv.state.putErrChan(errc) return err case <-sc.doneServing: return http2errClientDisconnected @@ -6389,7 +6523,7 @@ func (b *http2requestBody) Read(p []byte) (n int, err error) { if err == io.EOF { b.sawEOF = true } - if b.conn == nil && http2inTests { + if b.conn == nil { return } b.conn.noteBodyReadFromHandler(b.stream, n, err) @@ -6518,7 +6652,7 @@ func (rws *http2responseWriterState) writeChunk(p []byte) (n int, err error) { var date string if _, ok := rws.snapHeader["Date"]; !ok { // TODO(bradfitz): be faster here, like net/http? measure. - date = rws.conn.srv.now().UTC().Format(TimeFormat) + date = time.Now().UTC().Format(TimeFormat) } for _, v := range rws.snapHeader["Trailer"] { @@ -6640,7 +6774,7 @@ func (rws *http2responseWriterState) promoteUndeclaredTrailers() { func (w *http2responseWriter) SetReadDeadline(deadline time.Time) error { st := w.rws.stream - if !deadline.IsZero() && deadline.Before(w.rws.conn.srv.now()) { + if !deadline.IsZero() && deadline.Before(time.Now()) { // If we're setting a deadline in the past, reset the stream immediately // so writes after SetWriteDeadline returns will fail. st.onReadTimeout() @@ -6656,9 +6790,9 @@ func (w *http2responseWriter) SetReadDeadline(deadline time.Time) error { if deadline.IsZero() { st.readDeadline = nil } else if st.readDeadline == nil { - st.readDeadline = sc.srv.afterFunc(deadline.Sub(sc.srv.now()), st.onReadTimeout) + st.readDeadline = time.AfterFunc(deadline.Sub(time.Now()), st.onReadTimeout) } else { - st.readDeadline.Reset(deadline.Sub(sc.srv.now())) + st.readDeadline.Reset(deadline.Sub(time.Now())) } }) return nil @@ -6666,7 +6800,7 @@ func (w *http2responseWriter) SetReadDeadline(deadline time.Time) error { func (w *http2responseWriter) SetWriteDeadline(deadline time.Time) error { st := w.rws.stream - if !deadline.IsZero() && deadline.Before(w.rws.conn.srv.now()) { + if !deadline.IsZero() && deadline.Before(time.Now()) { // If we're setting a deadline in the past, reset the stream immediately // so writes after SetWriteDeadline returns will fail. st.onWriteTimeout() @@ -6682,14 +6816,19 @@ func (w *http2responseWriter) SetWriteDeadline(deadline time.Time) error { if deadline.IsZero() { st.writeDeadline = nil } else if st.writeDeadline == nil { - st.writeDeadline = sc.srv.afterFunc(deadline.Sub(sc.srv.now()), st.onWriteTimeout) + st.writeDeadline = time.AfterFunc(deadline.Sub(time.Now()), st.onWriteTimeout) } else { - st.writeDeadline.Reset(deadline.Sub(sc.srv.now())) + st.writeDeadline.Reset(deadline.Sub(time.Now())) } }) return nil } +func (w *http2responseWriter) EnableFullDuplex() error { + // We always support full duplex responses, so this is a no-op. + return nil +} + func (w *http2responseWriter) Flush() { w.FlushError() } @@ -6958,7 +7097,7 @@ func (w *http2responseWriter) Push(target string, opts *PushOptions) error { method: opts.Method, url: u, header: http2cloneHeader(opts.Header), - done: http2errChanPool.Get().(chan error), + done: sc.srv.state.getErrChan(), } select { @@ -6975,7 +7114,7 @@ func (w *http2responseWriter) Push(target string, opts *PushOptions) error { case <-st.cw: return http2errStreamClosed case err := <-msg.done: - http2errChanPool.Put(msg.done) + sc.srv.state.putErrChan(msg.done) return err } } @@ -7039,12 +7178,12 @@ func (sc *http2serverConn) startPush(msg *http2startPushRequest) { // we start in "half closed (remote)" for simplicity. // See further comments at the definition of stateHalfClosedRemote. promised := sc.newStream(promisedID, msg.parent.id, http2stateHalfClosedRemote) - rw, req, err := sc.newWriterAndRequestNoBody(promised, http2requestParam{ - method: msg.method, - scheme: msg.url.Scheme, - authority: msg.url.Host, - path: msg.url.RequestURI(), - header: http2cloneHeader(msg.header), // clone since handler runs concurrently with writing the PUSH_PROMISE + rw, req, err := sc.newWriterAndRequestNoBody(promised, httpcommon.ServerRequestParam{ + Method: msg.method, + Scheme: msg.url.Scheme, + Authority: msg.url.Host, + Path: msg.url.RequestURI(), + Header: http2cloneHeader(msg.header), // clone since handler runs concurrently with writing the PUSH_PROMISE }) if err != nil { // Should not happen, since we've already validated msg.url. @@ -7136,7 +7275,7 @@ func (sc *http2serverConn) countError(name string, err error) error { if sc == nil || sc.srv == nil { return err } - f := sc.srv.CountError + f := sc.countErrorFunc if f == nil { return err } @@ -7160,20 +7299,6 @@ func (sc *http2serverConn) countError(name string, err error) error { return err } -// A timer is a time.Timer, as an interface which can be replaced in tests. -type http2timer = interface { - C() <-chan time.Time - Reset(d time.Duration) bool - Stop() bool -} - -// timeTimer adapts a time.Timer to the timer interface. -type http2timeTimer struct { - *time.Timer -} - -func (t http2timeTimer) C() <-chan time.Time { return t.Timer.C } - const ( // transportDefaultConnFlow is how many connection-level flow control // tokens we give the server at start-up, past the default 64k. @@ -7330,73 +7455,29 @@ type http2Transport struct { type http2transportTestHooks struct { newclientconn func(*http2ClientConn) - group http2synctestGroupInterface -} - -func (t *http2Transport) markNewGoroutine() { - if t != nil && t.http2transportTestHooks != nil { - t.http2transportTestHooks.group.Join() - } -} - -// newTimer creates a new time.Timer, or a synthetic timer in tests. -func (t *http2Transport) newTimer(d time.Duration) http2timer { - if t.http2transportTestHooks != nil { - return t.http2transportTestHooks.group.NewTimer(d) - } - return http2timeTimer{time.NewTimer(d)} -} - -// afterFunc creates a new time.AfterFunc timer, or a synthetic timer in tests. -func (t *http2Transport) afterFunc(d time.Duration, f func()) http2timer { - if t.http2transportTestHooks != nil { - return t.http2transportTestHooks.group.AfterFunc(d, f) - } - return http2timeTimer{time.AfterFunc(d, f)} -} - -func (t *http2Transport) contextWithTimeout(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) { - if t.http2transportTestHooks != nil { - return t.http2transportTestHooks.group.ContextWithTimeout(ctx, d) - } - return context.WithTimeout(ctx, d) } func (t *http2Transport) maxHeaderListSize() uint32 { - if t.MaxHeaderListSize == 0 { + n := int64(t.MaxHeaderListSize) + if t.t1 != nil && t.t1.MaxResponseHeaderBytes != 0 { + n = t.t1.MaxResponseHeaderBytes + if n > 0 { + n = http2adjustHTTP1MaxHeaderSize(n) + } + } + if n <= 0 { return 10 << 20 } - if t.MaxHeaderListSize == 0xffffffff { + if n >= 0xffffffff { return 0 } - return t.MaxHeaderListSize -} - -func (t *http2Transport) maxFrameReadSize() uint32 { - if t.MaxReadFrameSize == 0 { - return 0 // use the default provided by the peer - } - if t.MaxReadFrameSize < http2minMaxFrameSize { - return http2minMaxFrameSize - } - if t.MaxReadFrameSize > http2maxFrameSize { - return http2maxFrameSize - } - return t.MaxReadFrameSize + return uint32(n) } func (t *http2Transport) disableCompression() bool { return t.DisableCompression || (t.t1 != nil && t.t1.DisableCompression) } -func (t *http2Transport) pingTimeout() time.Duration { - if t.PingTimeout == 0 { - return 15 * time.Second - } - return t.PingTimeout - -} - // ConfigureTransport configures a net/http HTTP/1 Transport to use HTTP/2. // It returns an error if t1 has already been HTTP/2-enabled. // @@ -7432,8 +7513,8 @@ func http2configureTransports(t1 *Transport) (*http2Transport, error) { if !http2strSliceContains(t1.TLSClientConfig.NextProtos, "http/1.1") { t1.TLSClientConfig.NextProtos = append(t1.TLSClientConfig.NextProtos, "http/1.1") } - upgradeFn := func(authority string, c *tls.Conn) RoundTripper { - addr := http2authorityAddr("https", authority) + upgradeFn := func(scheme, authority string, c net.Conn) RoundTripper { + addr := http2authorityAddr(scheme, authority) if used, err := connPool.addConnIfNeeded(addr, t2, c); err != nil { go c.Close() return http2erringRoundTripper{err} @@ -7444,18 +7525,37 @@ func http2configureTransports(t1 *Transport) (*http2Transport, error) { // was unknown) go c.Close() } + if scheme == "http" { + return (*http2unencryptedTransport)(t2) + } return t2 } - if m := t1.TLSNextProto; len(m) == 0 { - t1.TLSNextProto = map[string]func(string, *tls.Conn) RoundTripper{ - "h2": upgradeFn, + if t1.TLSNextProto == nil { + t1.TLSNextProto = make(map[string]func(string, *tls.Conn) RoundTripper) + } + t1.TLSNextProto[http2NextProtoTLS] = func(authority string, c *tls.Conn) RoundTripper { + return upgradeFn("https", authority, c) + } + // The "unencrypted_http2" TLSNextProto key is used to pass off non-TLS HTTP/2 conns. + t1.TLSNextProto[http2nextProtoUnencryptedHTTP2] = func(authority string, c *tls.Conn) RoundTripper { + nc, err := http2unencryptedNetConnFromTLSConn(c) + if err != nil { + go c.Close() + return http2erringRoundTripper{err} } - } else { - m["h2"] = upgradeFn + return upgradeFn("http", authority, nc) } return t2, nil } +// unencryptedTransport is a Transport with a RoundTrip method that +// always permits http:// URLs. +type http2unencryptedTransport http2Transport + +func (t *http2unencryptedTransport) RoundTrip(req *Request) (*Response, error) { + return (*http2Transport)(t).RoundTripOpt(req, http2RoundTripOpt{allowHTTP: true}) +} + func (t *http2Transport) connPool() http2ClientConnPool { t.connPoolOnce.Do(t.initConnPool) return t.connPoolOrDef @@ -7475,7 +7575,7 @@ type http2ClientConn struct { t *http2Transport tconn net.Conn // usually *tls.Conn, except specialized impls tlsState *tls.ConnectionState // nil only for specialized impls - reused uint32 // whether conn is being reused; atomic + atomicReused uint32 // whether conn is being reused; atomic singleUse bool // whether being used for a single http.Request getConnCalled bool // used by clientConnPool @@ -7484,33 +7584,58 @@ type http2ClientConn struct { readerErr error // set before readerDone is closed idleTimeout time.Duration // or 0 for never - idleTimer http2timer - - mu sync.Mutex // guards following - cond *sync.Cond // hold mu; broadcast on flow/closed changes - flow http2outflow // our conn-level flow control quota (cs.outflow is per stream) - inflow http2inflow // peer's conn-level flow control - doNotReuse bool // whether conn is marked to not be reused for any future requests - closing bool - closed bool - seenSettings bool // true if we've seen a settings frame, false otherwise - wantSettingsAck bool // we sent a SETTINGS frame and haven't heard back - goAway *http2GoAwayFrame // if non-nil, the GoAwayFrame we received - goAwayDebug string // goAway frame's debug data, retained as a string - streams map[uint32]*http2clientStream // client-initiated - streamsReserved int // incr by ReserveNewRequest; decr on RoundTrip - nextStreamID uint32 - pendingRequests int // requests blocked and waiting to be sent because len(streams) == maxConcurrentStreams - pings map[[8]byte]chan struct{} // in flight ping data to notification channel - br *bufio.Reader - lastActive time.Time - lastIdle time.Time // time last idle + idleTimer *time.Timer + + mu sync.Mutex // guards following + cond *sync.Cond // hold mu; broadcast on flow/closed changes + flow http2outflow // our conn-level flow control quota (cs.outflow is per stream) + inflow http2inflow // peer's conn-level flow control + doNotReuse bool // whether conn is marked to not be reused for any future requests + closing bool + closed bool + closedOnIdle bool // true if conn was closed for idleness + seenSettings bool // true if we've seen a settings frame, false otherwise + seenSettingsChan chan struct{} // closed when seenSettings is true or frame reading fails + wantSettingsAck bool // we sent a SETTINGS frame and haven't heard back + goAway *http2GoAwayFrame // if non-nil, the GoAwayFrame we received + goAwayDebug string // goAway frame's debug data, retained as a string + streams map[uint32]*http2clientStream // client-initiated + streamsReserved int // incr by ReserveNewRequest; decr on RoundTrip + nextStreamID uint32 + pendingRequests int // requests blocked and waiting to be sent because len(streams) == maxConcurrentStreams + pings map[[8]byte]chan struct{} // in flight ping data to notification channel + br *bufio.Reader + lastActive time.Time + lastIdle time.Time // time last idle // Settings from peer: (also guarded by wmu) - maxFrameSize uint32 - maxConcurrentStreams uint32 - peerMaxHeaderListSize uint64 - peerMaxHeaderTableSize uint32 - initialWindowSize uint32 + maxFrameSize uint32 + maxConcurrentStreams uint32 + peerMaxHeaderListSize uint64 + peerMaxHeaderTableSize uint32 + initialWindowSize uint32 + initialStreamRecvWindowSize int32 + readIdleTimeout time.Duration + pingTimeout time.Duration + extendedConnectAllowed bool + strictMaxConcurrentStreams bool + + // rstStreamPingsBlocked works around an unfortunate gRPC behavior. + // gRPC strictly limits the number of PING frames that it will receive. + // The default is two pings per two hours, but the limit resets every time + // the gRPC endpoint sends a HEADERS or DATA frame. See golang/go#70575. + // + // rstStreamPingsBlocked is set after receiving a response to a PING frame + // bundled with an RST_STREAM (see pendingResets below), and cleared after + // receiving a HEADERS or DATA frame. + rstStreamPingsBlocked bool + + // pendingResets is the number of RST_STREAM frames we have sent to the peer, + // without confirming that the peer has received them. When we send a RST_STREAM, + // we bundle it with a PING frame, unless a PING is already in flight. We count + // the reset stream against the connection's concurrency limit until we get + // a PING response. This limits the number of requests we'll try to send to a + // completely unresponsive connection. + pendingResets int // reqHeaderMu is a 1-element semaphore channel controlling access to sending new requests. // Write to reqHeaderMu to lock it, read from it to unlock. @@ -7568,12 +7693,12 @@ type http2clientStream struct { sentHeaders bool // owned by clientConnReadLoop: - firstByte bool // got the first response byte - pastHeaders bool // got first MetaHeadersFrame (actual headers) - pastTrailers bool // got optional second MetaHeadersFrame (trailers) - num1xx uint8 // number of 1xx responses seen - readClosed bool // peer sent an END_STREAM flag - readAborted bool // read loop reset the stream + firstByte bool // got the first response byte + pastHeaders bool // got first MetaHeadersFrame (actual headers) + pastTrailers bool // got optional second MetaHeadersFrame (trailers) + readClosed bool // peer sent an END_STREAM flag + readAborted bool // read loop reset the stream + totalHeaderSize int64 // total size of 1xx headers seen trailer Header // accumulated trailers resTrailer *Header // client's Response.Trailer @@ -7628,7 +7753,6 @@ func (cs *http2clientStream) closeReqBodyLocked() { cs.reqBodyClosed = make(chan struct{}) reqBodyClosed := cs.reqBodyClosed go func() { - cs.cc.t.markNewGoroutine() cs.reqBody.Close() close(reqBodyClosed) }() @@ -7644,22 +7768,9 @@ func (sew http2stickyErrWriter) Write(p []byte) (n int, err error) { if *sew.err != nil { return 0, *sew.err } - for { - if sew.timeout != 0 { - sew.conn.SetWriteDeadline(time.Now().Add(sew.timeout)) - } - nn, err := sew.conn.Write(p[n:]) - n += nn - if n < len(p) && nn > 0 && errors.Is(err, os.ErrDeadlineExceeded) { - // Keep extending the deadline so long as we're making progress. - continue - } - if sew.timeout != 0 { - sew.conn.SetWriteDeadline(time.Time{}) - } - *sew.err = err - return n, err - } + n, err = http2writeWithByteTimeout(sew.conn, sew.timeout, p) + *sew.err = err + return n, err } // noCachedConnError is the concrete type of ErrNoCachedConn, which @@ -7691,6 +7802,8 @@ type http2RoundTripOpt struct { // no cached connection is available, RoundTripOpt // will return ErrNoCachedConn. OnlyCachedConn bool + + allowHTTP bool // allow http:// URLs } func (t *http2Transport) RoundTrip(req *Request) (*Response, error) { @@ -7723,7 +7836,14 @@ func http2authorityAddr(scheme string, authority string) (addr string) { // RoundTripOpt is like RoundTrip, but takes options. func (t *http2Transport) RoundTripOpt(req *Request, opt http2RoundTripOpt) (*Response, error) { - if !(req.URL.Scheme == "https" || (req.URL.Scheme == "http" && t.AllowHTTP)) { + switch req.URL.Scheme { + case "https": + // Always okay. + case "http": + if !t.AllowHTTP && !opt.allowHTTP { + return nil, errors.New("http2: unencrypted HTTP/2 not enabled") + } + default: return nil, errors.New("http2: unsupported scheme") } @@ -7734,7 +7854,7 @@ func (t *http2Transport) RoundTripOpt(req *Request, opt http2RoundTripOpt) (*Res t.vlogf("http2: Transport failed to get client conn for %s: %v", addr, err) return nil, err } - reused := !atomic.CompareAndSwapUint32(&cc.reused, 0, 1) + reused := !atomic.CompareAndSwapUint32(&cc.atomicReused, 0, 1) http2traceGotConn(req, cc, reused) res, err := cc.RoundTrip(req) if err != nil && retry <= 6 { @@ -7748,9 +7868,9 @@ func (t *http2Transport) RoundTripOpt(req *Request, opt http2RoundTripOpt) (*Res backoff := float64(uint(1) << (uint(retry) - 1)) backoff += backoff * (0.1 * mathrand.Float64()) d := time.Second * time.Duration(backoff) - tm := t.newTimer(d) + tm := time.NewTimer(d) select { - case <-tm.C(): + case <-tm.C: t.vlogf("RoundTrip retrying after failure: %v", roundTripErr) continue case <-req.Context().Done(): @@ -7759,6 +7879,22 @@ func (t *http2Transport) RoundTripOpt(req *Request, opt http2RoundTripOpt) (*Res } } } + if err == http2errClientConnNotEstablished { + // This ClientConn was created recently, + // this is the first request to use it, + // and the connection is closed and not usable. + // + // In this state, cc.idleTimer will remove the conn from the pool + // when it fires. Stop the timer and remove it here so future requests + // won't try to use this connection. + // + // If the timer has already fired and we're racing it, the redundant + // call to MarkDead is harmless. + if cc.idleTimer != nil { + cc.idleTimer.Stop() + } + t.connPool().MarkDead(cc) + } if err != nil { t.vlogf("RoundTrip failure: %v", err) return nil, err @@ -7777,9 +7913,11 @@ func (t *http2Transport) CloseIdleConnections() { } var ( - http2errClientConnClosed = errors.New("http2: client conn is closed") - http2errClientConnUnusable = errors.New("http2: client conn not usable") - http2errClientConnGotGoAway = errors.New("http2: Transport received Server's graceful shutdown GOAWAY") + http2errClientConnClosed = errors.New("http2: client conn is closed") + http2errClientConnUnusable = errors.New("http2: client conn not usable") + http2errClientConnNotEstablished = errors.New("http2: client conn could not be established") + http2errClientConnGotGoAway = errors.New("http2: Transport received Server's graceful shutdown GOAWAY") + http2errClientConnForceClosed = errors.New("http2: client connection force closed via ClientConn.Close") ) // shouldRetryRequest is called by RoundTrip when a request fails to get @@ -7895,42 +8033,34 @@ func (t *http2Transport) expectContinueTimeout() time.Duration { return t.t1.ExpectContinueTimeout } -func (t *http2Transport) maxDecoderHeaderTableSize() uint32 { - if v := t.MaxDecoderHeaderTableSize; v > 0 { - return v - } - return http2initialHeaderTableSize -} - -func (t *http2Transport) maxEncoderHeaderTableSize() uint32 { - if v := t.MaxEncoderHeaderTableSize; v > 0 { - return v - } - return http2initialHeaderTableSize -} - func (t *http2Transport) NewClientConn(c net.Conn) (*http2ClientConn, error) { return t.newClientConn(c, t.disableKeepAlives()) } func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2ClientConn, error) { + conf := http2configFromTransport(t) cc := &http2ClientConn{ - t: t, - tconn: c, - readerDone: make(chan struct{}), - nextStreamID: 1, - maxFrameSize: 16 << 10, // spec default - initialWindowSize: 65535, // spec default - maxConcurrentStreams: http2initialMaxConcurrentStreams, // "infinite", per spec. Use a smaller value until we have received server settings. - peerMaxHeaderListSize: 0xffffffffffffffff, // "infinite", per spec. Use 2^64-1 instead. - streams: make(map[uint32]*http2clientStream), - singleUse: singleUse, - wantSettingsAck: true, - pings: make(map[[8]byte]chan struct{}), - reqHeaderMu: make(chan struct{}, 1), + t: t, + tconn: c, + readerDone: make(chan struct{}), + nextStreamID: 1, + maxFrameSize: 16 << 10, // spec default + initialWindowSize: 65535, // spec default + initialStreamRecvWindowSize: conf.MaxUploadBufferPerStream, + maxConcurrentStreams: http2initialMaxConcurrentStreams, // "infinite", per spec. Use a smaller value until we have received server settings. + strictMaxConcurrentStreams: conf.StrictMaxConcurrentRequests, + peerMaxHeaderListSize: 0xffffffffffffffff, // "infinite", per spec. Use 2^64-1 instead. + streams: make(map[uint32]*http2clientStream), + singleUse: singleUse, + seenSettingsChan: make(chan struct{}), + wantSettingsAck: true, + readIdleTimeout: conf.SendPingTimeout, + pingTimeout: conf.PingTimeout, + pings: make(map[[8]byte]chan struct{}), + reqHeaderMu: make(chan struct{}, 1), + lastActive: time.Now(), } if t.http2transportTestHooks != nil { - t.markNewGoroutine() t.http2transportTestHooks.newclientconn(cc) c = cc.tconn } @@ -7945,23 +8075,21 @@ func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2Client // MTU + crypto/tls record padding. cc.bw = bufio.NewWriter(http2stickyErrWriter{ conn: c, - timeout: t.WriteByteTimeout, + timeout: conf.WriteByteTimeout, err: &cc.werr, }) cc.br = bufio.NewReader(c) cc.fr = http2NewFramer(cc.bw, cc.br) - if t.maxFrameReadSize() != 0 { - cc.fr.SetMaxReadFrameSize(t.maxFrameReadSize()) - } + cc.fr.SetMaxReadFrameSize(conf.MaxReadFrameSize) if t.CountError != nil { cc.fr.countError = t.CountError } - maxHeaderTableSize := t.maxDecoderHeaderTableSize() + maxHeaderTableSize := conf.MaxDecoderHeaderTableSize cc.fr.ReadMetaHeaders = hpack.NewDecoder(maxHeaderTableSize, nil) cc.fr.MaxHeaderListSize = t.maxHeaderListSize() cc.henc = hpack.NewEncoder(&cc.hbuf) - cc.henc.SetMaxDynamicTableSizeLimit(t.maxEncoderHeaderTableSize()) + cc.henc.SetMaxDynamicTableSizeLimit(conf.MaxEncoderHeaderTableSize) cc.peerMaxHeaderTableSize = http2initialHeaderTableSize if cs, ok := c.(http2connectionStater); ok { @@ -7971,11 +8099,9 @@ func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2Client initialSettings := []http2Setting{ {ID: http2SettingEnablePush, Val: 0}, - {ID: http2SettingInitialWindowSize, Val: http2transportDefaultStreamFlow}, - } - if max := t.maxFrameReadSize(); max != 0 { - initialSettings = append(initialSettings, http2Setting{ID: http2SettingMaxFrameSize, Val: max}) + {ID: http2SettingInitialWindowSize, Val: uint32(cc.initialStreamRecvWindowSize)}, } + initialSettings = append(initialSettings, http2Setting{ID: http2SettingMaxFrameSize, Val: conf.MaxReadFrameSize}) if max := t.maxHeaderListSize(); max != 0 { initialSettings = append(initialSettings, http2Setting{ID: http2SettingMaxHeaderListSize, Val: max}) } @@ -7985,8 +8111,8 @@ func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2Client cc.bw.Write(http2clientPreface) cc.fr.WriteSettings(initialSettings...) - cc.fr.WriteWindowUpdate(0, http2transportDefaultConnFlow) - cc.inflow.init(http2transportDefaultConnFlow + http2initialWindowSize) + cc.fr.WriteWindowUpdate(0, uint32(conf.MaxUploadBufferPerConnection)) + cc.inflow.init(conf.MaxUploadBufferPerConnection + http2initialWindowSize) cc.bw.Flush() if cc.werr != nil { cc.Close() @@ -7996,7 +8122,7 @@ func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2Client // Start the idle timer after the connection is fully initialized. if d := t.idleConnTimeout(); d != 0 { cc.idleTimeout = d - cc.idleTimer = t.afterFunc(d, cc.onIdleTimeout) + cc.idleTimer = time.AfterFunc(d, cc.onIdleTimeout) } go cc.readLoop() @@ -8004,10 +8130,10 @@ func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2Client } func (cc *http2ClientConn) healthCheck() { - pingTimeout := cc.t.pingTimeout() + pingTimeout := cc.pingTimeout // We don't need to periodically ping in the health check, because the readLoop of ClientConn will // trigger the healthCheck again if there is no frame received. - ctx, cancel := cc.t.contextWithTimeout(context.Background(), pingTimeout) + ctx, cancel := context.WithTimeout(context.Background(), pingTimeout) defer cancel() cc.vlogf("http2: Transport sending health check") err := cc.Ping(ctx) @@ -8132,7 +8258,7 @@ func (cc *http2ClientConn) State() http2ClientConnState { return http2ClientConnState{ Closed: cc.closed, Closing: cc.closing || cc.singleUse || cc.doNotReuse || cc.goAway != nil, - StreamsActive: len(cc.streams), + StreamsActive: len(cc.streams) + cc.pendingResets, StreamsReserved: cc.streamsReserved, StreamsPending: cc.pendingRequests, LastIdle: cc.lastIdle, @@ -8157,23 +8283,47 @@ func (cc *http2ClientConn) idleStateLocked() (st http2clientConnIdleState) { return } var maxConcurrentOkay bool - if cc.t.StrictMaxConcurrentStreams { + if cc.strictMaxConcurrentStreams { // We'll tell the caller we can take a new request to // prevent the caller from dialing a new TCP // connection, but then we'll block later before // writing it. maxConcurrentOkay = true } else { - maxConcurrentOkay = int64(len(cc.streams)+cc.streamsReserved+1) <= int64(cc.maxConcurrentStreams) + // We can take a new request if the total of + // - active streams; + // - reservation slots for new streams; and + // - streams for which we have sent a RST_STREAM and a PING, + // but received no subsequent frame + // is less than the concurrency limit. + maxConcurrentOkay = cc.currentRequestCountLocked() < int(cc.maxConcurrentStreams) } st.canTakeNewRequest = cc.goAway == nil && !cc.closed && !cc.closing && maxConcurrentOkay && !cc.doNotReuse && int64(cc.nextStreamID)+2*int64(cc.pendingRequests) < math.MaxInt32 && !cc.tooIdleLocked() + + // If this connection has never been used for a request and is closed, + // then let it take a request (which will fail). + // If the conn was closed for idleness, we're racing the idle timer; + // don't try to use the conn. (Issue #70515.) + // + // This avoids a situation where an error early in a connection's lifetime + // goes unreported. + if cc.nextStreamID == 1 && cc.streamsReserved == 0 && cc.closed && !cc.closedOnIdle { + st.canTakeNewRequest = true + } + return } +// currentRequestCountLocked reports the number of concurrency slots currently in use, +// including active streams, reserved slots, and reset streams waiting for acknowledgement. +func (cc *http2ClientConn) currentRequestCountLocked() int { + return len(cc.streams) + cc.streamsReserved + cc.pendingResets +} + func (cc *http2ClientConn) canTakeNewRequestLocked() bool { st := cc.idleStateLocked() return st.canTakeNewRequest @@ -8224,6 +8374,7 @@ func (cc *http2ClientConn) closeIfIdle() { return } cc.closed = true + cc.closedOnIdle = true nextID := cc.nextStreamID // TODO: do clients send GOAWAY too? maybe? Just Close: cc.mu.Unlock() @@ -8251,7 +8402,6 @@ func (cc *http2ClientConn) Shutdown(ctx context.Context) error { done := make(chan struct{}) cancelled := false // guarded by cc.mu go func() { - cc.t.markNewGoroutine() cc.mu.Lock() defer cc.mu.Unlock() for { @@ -8322,8 +8472,7 @@ func (cc *http2ClientConn) closeForError(err error) { // // In-flight requests are interrupted. For a graceful shutdown, use Shutdown instead. func (cc *http2ClientConn) Close() error { - err := errors.New("http2: client connection force closed via ClientConn.Close") - cc.closeForError(err) + cc.closeForError(http2errClientConnForceClosed) return nil } @@ -8340,23 +8489,6 @@ func (cc *http2ClientConn) closeForLostPing() { // exported. At least they'll be DeepEqual for h1-vs-h2 comparisons tests. var http2errRequestCanceled = errors.New("net/http: request canceled") -func http2commaSeparatedTrailers(req *Request) (string, error) { - keys := make([]string, 0, len(req.Trailer)) - for k := range req.Trailer { - k = http2canonicalHeader(k) - switch k { - case "Transfer-Encoding", "Trailer", "Content-Length": - return "", fmt.Errorf("invalid Trailer key %q", k) - } - keys = append(keys, k) - } - if len(keys) > 0 { - sort.Strings(keys) - return strings.Join(keys, ","), nil - } - return "", nil -} - func (cc *http2ClientConn) responseHeaderTimeout() time.Duration { if cc.t.t1 != nil { return cc.t.t1.ResponseHeaderTimeout @@ -8368,22 +8500,6 @@ func (cc *http2ClientConn) responseHeaderTimeout() time.Duration { return 0 } -// checkConnHeaders checks whether req has any invalid connection-level headers. -// per RFC 7540 section 8.1.2.2: Connection-Specific Header Fields. -// Certain headers are special-cased as okay but not transmitted later. -func http2checkConnHeaders(req *Request) error { - if v := req.Header.Get("Upgrade"); v != "" { - return fmt.Errorf("http2: invalid Upgrade request header: %q", req.Header["Upgrade"]) - } - if vv := req.Header["Transfer-Encoding"]; len(vv) > 0 && (len(vv) > 1 || vv[0] != "" && vv[0] != "chunked") { - return fmt.Errorf("http2: invalid Transfer-Encoding request header: %q", vv) - } - if vv := req.Header["Connection"]; len(vv) > 0 && (len(vv) > 1 || vv[0] != "" && !http2asciiEqualFold(vv[0], "close") && !http2asciiEqualFold(vv[0], "keep-alive")) { - return fmt.Errorf("http2: invalid Connection request header: %q", vv) - } - return nil -} - // actualContentLength returns a sanitized version of // req.ContentLength, where 0 actually means zero (not unknown) and -1 // means unknown. @@ -8429,25 +8545,7 @@ func (cc *http2ClientConn) roundTrip(req *Request, streamf func(*http2clientStre donec: make(chan struct{}), } - // TODO(bradfitz): this is a copy of the logic in net/http. Unify somewhere? - if !cc.t.disableCompression() && - req.Header.Get("Accept-Encoding") == "" && - req.Header.Get("Range") == "" && - !cs.isHead { - // Request gzip only, not deflate. Deflate is ambiguous and - // not as universally supported anyway. - // See: https://zlib.net/zlib_faq.html#faq39 - // - // Note that we don't request this for HEAD requests, - // due to a bug in nginx: - // http://trac.nginx.org/nginx/ticket/358 - // https://golang.org/issue/5522 - // - // We don't request gzip if the request is for a range, since - // auto-decoding a portion of a gzipped document will just fail - // anyway. See https://golang.org/issue/8923 - cs.requestedGzip = true - } + cs.requestedGzip = httpcommon.IsRequestGzip(req.Method, req.Header, cc.t.disableCompression()) go cs.doRequest(req, streamf) @@ -8543,11 +8641,12 @@ func (cc *http2ClientConn) roundTrip(req *Request, streamf func(*http2clientStre // // It sends the request and performs post-request cleanup (closing Request.Body, etc.). func (cs *http2clientStream) doRequest(req *Request, streamf func(*http2clientStream)) { - cs.cc.t.markNewGoroutine() err := cs.writeRequest(req, streamf) cs.cleanupWriteRequest(err) } +var http2errExtendedConnectNotSupported = errors.New("net/http: extended connect not supported by peer") + // writeRequest sends a request. // // It returns nil after the request is written, the response read, @@ -8559,8 +8658,11 @@ func (cs *http2clientStream) writeRequest(req *Request, streamf func(*http2clien cc := cs.cc ctx := cs.ctx - if err := http2checkConnHeaders(req); err != nil { - return err + // wait for setting frames to be received, a server can change this value later, + // but we just wait for the first settings frame + var isExtendedConnect bool + if req.Method == "CONNECT" && req.Header.Get(":protocol") != "" { + isExtendedConnect = true } // Acquire the new-request lock by writing to reqHeaderMu. @@ -8569,6 +8671,18 @@ func (cs *http2clientStream) writeRequest(req *Request, streamf func(*http2clien if cc.reqHeaderMu == nil { panic("RoundTrip on uninitialized ClientConn") // for tests } + if isExtendedConnect { + select { + case <-cs.reqCancel: + return http2errRequestCanceled + case <-ctx.Done(): + return ctx.Err() + case <-cc.seenSettingsChan: + if !cc.extendedConnectAllowed { + return http2errExtendedConnectNotSupported + } + } + } select { case cc.reqHeaderMu <- struct{}{}: case <-cs.reqCancel: @@ -8657,9 +8771,9 @@ func (cs *http2clientStream) writeRequest(req *Request, streamf func(*http2clien var respHeaderTimer <-chan time.Time var respHeaderRecv chan struct{} if d := cc.responseHeaderTimeout(); d != 0 { - timer := cc.t.newTimer(d) + timer := time.NewTimer(d) defer timer.Stop() - respHeaderTimer = timer.C() + respHeaderTimer = timer.C respHeaderRecv = cs.respHeaderRecv } // Wait until the peer half-closes its end of the stream, @@ -8707,26 +8821,39 @@ func (cs *http2clientStream) encodeAndWriteHeaders(req *Request) error { // we send: HEADERS{1}, CONTINUATION{0,} + DATA{0,} (DATA is // sent by writeRequestBody below, along with any Trailers, // again in form HEADERS{1}, CONTINUATION{0,}) - trailers, err := http2commaSeparatedTrailers(req) - if err != nil { - return err - } - hasTrailers := trailers != "" - contentLen := http2actualContentLength(req) - hasBody := contentLen != 0 - hdrs, err := cc.encodeHeaders(req, cs.requestedGzip, trailers, contentLen) + cc.hbuf.Reset() + res, err := http2encodeRequestHeaders(req, cs.requestedGzip, cc.peerMaxHeaderListSize, func(name, value string) { + cc.writeHeader(name, value) + }) if err != nil { - return err + return fmt.Errorf("http2: %w", err) } + hdrs := cc.hbuf.Bytes() // Write the request. - endStream := !hasBody && !hasTrailers + endStream := !res.HasBody && !res.HasTrailers cs.sentHeaders = true err = cc.writeHeaders(cs.ID, endStream, int(cc.maxFrameSize), hdrs) http2traceWroteHeaders(cs.trace) return err } +func http2encodeRequestHeaders(req *Request, addGzipHeader bool, peerMaxHeaderListSize uint64, headerf func(name, value string)) (httpcommon.EncodeHeadersResult, error) { + return httpcommon.EncodeHeaders(req.Context(), httpcommon.EncodeHeadersParam{ + Request: httpcommon.Request{ + Header: req.Header, + Trailer: req.Trailer, + URL: req.URL, + Host: req.Host, + Method: req.Method, + ActualContentLength: http2actualContentLength(req), + }, + AddGzipHeader: addGzipHeader, + PeerMaxHeaderListSize: peerMaxHeaderListSize, + DefaultUserAgent: http2defaultUserAgent, + }, headerf) +} + // cleanupWriteRequest performs post-request tasks. // // If err (the result of writeRequest) is non-nil and the stream is not closed, @@ -8750,6 +8877,7 @@ func (cs *http2clientStream) cleanupWriteRequest(err error) { cs.reqBodyClosed = make(chan struct{}) } bodyClosed := cs.reqBodyClosed + closeOnIdle := cc.singleUse || cc.doNotReuse || cc.t.disableKeepAlives() || cc.goAway != nil cc.mu.Unlock() if mustCloseBody { cs.reqBody.Close() @@ -8774,16 +8902,44 @@ func (cs *http2clientStream) cleanupWriteRequest(err error) { if cs.sentHeaders { if se, ok := err.(http2StreamError); ok { if se.Cause != http2errFromPeer { - cc.writeStreamReset(cs.ID, se.Code, err) + cc.writeStreamReset(cs.ID, se.Code, false, err) } } else { - cc.writeStreamReset(cs.ID, http2ErrCodeCancel, err) + // We're cancelling an in-flight request. + // + // This could be due to the server becoming unresponsive. + // To avoid sending too many requests on a dead connection, + // we let the request continue to consume a concurrency slot + // until we can confirm the server is still responding. + // We do this by sending a PING frame along with the RST_STREAM + // (unless a ping is already in flight). + // + // For simplicity, we don't bother tracking the PING payload: + // We reset cc.pendingResets any time we receive a PING ACK. + // + // We skip this if the conn is going to be closed on idle, + // because it's short lived and will probably be closed before + // we get the ping response. + ping := false + if !closeOnIdle { + cc.mu.Lock() + // rstStreamPingsBlocked works around a gRPC behavior: + // see comment on the field for details. + if !cc.rstStreamPingsBlocked { + if cc.pendingResets == 0 { + ping = true + } + cc.pendingResets++ + } + cc.mu.Unlock() + } + cc.writeStreamReset(cs.ID, http2ErrCodeCancel, ping, err) } } cs.bufPipe.CloseWithError(err) // no-op if already closed } else { if cs.sentHeaders && !cs.sentEndStream { - cc.writeStreamReset(cs.ID, http2ErrCodeNo, nil) + cc.writeStreamReset(cs.ID, http2ErrCodeNo, false, nil) } cs.bufPipe.CloseWithError(http2errRequestCanceled) } @@ -8805,12 +8961,17 @@ func (cs *http2clientStream) cleanupWriteRequest(err error) { // Must hold cc.mu. func (cc *http2ClientConn) awaitOpenSlotForStreamLocked(cs *http2clientStream) error { for { + if cc.closed && cc.nextStreamID == 1 && cc.streamsReserved == 0 { + // This is the very first request sent to this connection. + // Return a fatal error which aborts the retry loop. + return http2errClientConnNotEstablished + } cc.lastActive = time.Now() if cc.closed || !cc.canTakeNewRequestLocked() { return http2errClientConnUnusable } cc.lastIdle = time.Time{} - if int64(len(cc.streams)) < int64(cc.maxConcurrentStreams) { + if cc.currentRequestCountLocked() < int(cc.maxConcurrentStreams) { return nil } cc.pendingRequests++ @@ -9081,214 +9242,6 @@ func (cs *http2clientStream) awaitFlowControl(maxBytes int) (taken int32, err er } } -func http2validateHeaders(hdrs Header) string { - for k, vv := range hdrs { - if !httpguts.ValidHeaderFieldName(k) { - return fmt.Sprintf("name %q", k) - } - for _, v := range vv { - if !httpguts.ValidHeaderFieldValue(v) { - // Don't include the value in the error, - // because it may be sensitive. - return fmt.Sprintf("value for header %q", k) - } - } - } - return "" -} - -var http2errNilRequestURL = errors.New("http2: Request.URI is nil") - -// requires cc.wmu be held. -func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trailers string, contentLength int64) ([]byte, error) { - cc.hbuf.Reset() - if req.URL == nil { - return nil, http2errNilRequestURL - } - - host := req.Host - if host == "" { - host = req.URL.Host - } - host, err := httpguts.PunycodeHostPort(host) - if err != nil { - return nil, err - } - if !httpguts.ValidHostHeader(host) { - return nil, errors.New("http2: invalid Host header") - } - - var path string - if req.Method != "CONNECT" { - path = req.URL.RequestURI() - if !http2validPseudoPath(path) { - orig := path - path = strings.TrimPrefix(path, req.URL.Scheme+"://"+host) - if !http2validPseudoPath(path) { - if req.URL.Opaque != "" { - return nil, fmt.Errorf("invalid request :path %q from URL.Opaque = %q", orig, req.URL.Opaque) - } else { - return nil, fmt.Errorf("invalid request :path %q", orig) - } - } - } - } - - // Check for any invalid headers+trailers and return an error before we - // potentially pollute our hpack state. (We want to be able to - // continue to reuse the hpack encoder for future requests) - if err := http2validateHeaders(req.Header); err != "" { - return nil, fmt.Errorf("invalid HTTP header %s", err) - } - if err := http2validateHeaders(req.Trailer); err != "" { - return nil, fmt.Errorf("invalid HTTP trailer %s", err) - } - - enumerateHeaders := func(f func(name, value string)) { - // 8.1.2.3 Request Pseudo-Header Fields - // The :path pseudo-header field includes the path and query parts of the - // target URI (the path-absolute production and optionally a '?' character - // followed by the query production, see Sections 3.3 and 3.4 of - // [RFC3986]). - f(":authority", host) - m := req.Method - if m == "" { - m = MethodGet - } - f(":method", m) - if req.Method != "CONNECT" { - f(":path", path) - f(":scheme", req.URL.Scheme) - } - if trailers != "" { - f("trailer", trailers) - } - - var didUA bool - for k, vv := range req.Header { - if http2asciiEqualFold(k, "host") || http2asciiEqualFold(k, "content-length") { - // Host is :authority, already sent. - // Content-Length is automatic, set below. - continue - } else if http2asciiEqualFold(k, "connection") || - http2asciiEqualFold(k, "proxy-connection") || - http2asciiEqualFold(k, "transfer-encoding") || - http2asciiEqualFold(k, "upgrade") || - http2asciiEqualFold(k, "keep-alive") { - // Per 8.1.2.2 Connection-Specific Header - // Fields, don't send connection-specific - // fields. We have already checked if any - // are error-worthy so just ignore the rest. - continue - } else if http2asciiEqualFold(k, "user-agent") { - // Match Go's http1 behavior: at most one - // User-Agent. If set to nil or empty string, - // then omit it. Otherwise if not mentioned, - // include the default (below). - didUA = true - if len(vv) < 1 { - continue - } - vv = vv[:1] - if vv[0] == "" { - continue - } - } else if http2asciiEqualFold(k, "cookie") { - // Per 8.1.2.5 To allow for better compression efficiency, the - // Cookie header field MAY be split into separate header fields, - // each with one or more cookie-pairs. - for _, v := range vv { - for { - p := strings.IndexByte(v, ';') - if p < 0 { - break - } - f("cookie", v[:p]) - p++ - // strip space after semicolon if any. - for p+1 <= len(v) && v[p] == ' ' { - p++ - } - v = v[p:] - } - if len(v) > 0 { - f("cookie", v) - } - } - continue - } - - for _, v := range vv { - f(k, v) - } - } - if http2shouldSendReqContentLength(req.Method, contentLength) { - f("content-length", strconv.FormatInt(contentLength, 10)) - } - if addGzipHeader { - f("accept-encoding", "gzip") - } - if !didUA { - f("user-agent", http2defaultUserAgent) - } - } - - // Do a first pass over the headers counting bytes to ensure - // we don't exceed cc.peerMaxHeaderListSize. This is done as a - // separate pass before encoding the headers to prevent - // modifying the hpack state. - hlSize := uint64(0) - enumerateHeaders(func(name, value string) { - hf := hpack.HeaderField{Name: name, Value: value} - hlSize += uint64(hf.Size()) - }) - - if hlSize > cc.peerMaxHeaderListSize { - return nil, http2errRequestHeaderListSize - } - - trace := httptrace.ContextClientTrace(req.Context()) - traceHeaders := http2traceHasWroteHeaderField(trace) - - // Header list size is ok. Write the headers. - enumerateHeaders(func(name, value string) { - name, ascii := http2lowerHeader(name) - if !ascii { - // Skip writing invalid headers. Per RFC 7540, Section 8.1.2, header - // field names have to be ASCII characters (just as in HTTP/1.x). - return - } - cc.writeHeader(name, value) - if traceHeaders { - http2traceWroteHeaderField(trace, name, value) - } - }) - - return cc.hbuf.Bytes(), nil -} - -// shouldSendReqContentLength reports whether the http2.Transport should send -// a "content-length" request header. This logic is basically a copy of the net/http -// transferWriter.shouldSendContentLength. -// The contentLength is the corrected contentLength (so 0 means actually 0, not unknown). -// -1 means unknown. -func http2shouldSendReqContentLength(method string, contentLength int64) bool { - if contentLength > 0 { - return true - } - if contentLength < 0 { - return false - } - // For zero bodies, whether we send a content-length depends on the method. - // It also kinda doesn't matter for http2 either way, with END_STREAM. - switch method { - case "POST", "PUT", "PATCH": - return true - default: - return false - } -} - // requires cc.wmu be held. func (cc *http2ClientConn) encodeTrailers(trailer Header) ([]byte, error) { cc.hbuf.Reset() @@ -9305,7 +9258,7 @@ func (cc *http2ClientConn) encodeTrailers(trailer Header) ([]byte, error) { } for k, vv := range trailer { - lowKey, ascii := http2lowerHeader(k) + lowKey, ascii := httpcommon.LowerHeader(k) if !ascii { // Skip writing invalid headers. Per RFC 7540, Section 8.1.2, header // field names have to be ASCII characters (just as in HTTP/1.x). @@ -9337,7 +9290,7 @@ type http2resAndError struct { func (cc *http2ClientConn) addStreamLocked(cs *http2clientStream) { cs.flow.add(int32(cc.initialWindowSize)) cs.flow.setConnFlow(&cc.flow) - cs.inflow.init(http2transportDefaultStreamFlow) + cs.inflow.init(cc.initialStreamRecvWindowSize) cs.ID = cc.nextStreamID cc.nextStreamID += 2 cc.streams[cs.ID] = cs @@ -9382,7 +9335,6 @@ type http2clientConnReadLoop struct { // readLoop runs in its own goroutine and reads and dispatches frames. func (cc *http2ClientConn) readLoop() { - cc.t.markNewGoroutine() rl := &http2clientConnReadLoop{cc: cc} defer rl.cleanup() cc.readerErr = rl.run() @@ -9416,7 +9368,6 @@ func http2isEOFOrNetReadError(err error) bool { func (rl *http2clientConnReadLoop) cleanup() { cc := rl.cc - cc.t.connPool().MarkDead(cc) defer cc.closeConn() defer close(cc.readerDone) @@ -9440,6 +9391,27 @@ func (rl *http2clientConnReadLoop) cleanup() { } cc.closed = true + // If the connection has never been used, and has been open for only a short time, + // leave it in the connection pool for a little while. + // + // This avoids a situation where new connections are constantly created, + // added to the pool, fail, and are removed from the pool, without any error + // being surfaced to the user. + unusedWaitTime := 5 * time.Second + if cc.idleTimeout > 0 && unusedWaitTime > cc.idleTimeout { + unusedWaitTime = cc.idleTimeout + } + idleTime := time.Now().Sub(cc.lastActive) + if atomic.LoadUint32(&cc.atomicReused) == 0 && idleTime < unusedWaitTime && !cc.closedOnIdle { + cc.idleTimer = time.AfterFunc(unusedWaitTime-idleTime, func() { + cc.t.connPool().MarkDead(cc) + }) + } else { + cc.mu.Unlock() // avoid any deadlocks in MarkDead + cc.t.connPool().MarkDead(cc) + cc.mu.Lock() + } + for _, cs := range cc.streams { select { case <-cs.peerClosed: @@ -9451,6 +9423,13 @@ func (rl *http2clientConnReadLoop) cleanup() { } cc.cond.Broadcast() cc.mu.Unlock() + + if !cc.seenSettings { + // If we have a pending request that wants extended CONNECT, + // let it continue and fail with the connection error. + cc.extendedConnectAllowed = true + close(cc.seenSettingsChan) + } } // countReadFrameError calls Transport.CountError with a string @@ -9483,10 +9462,10 @@ func (cc *http2ClientConn) countReadFrameError(err error) { func (rl *http2clientConnReadLoop) run() error { cc := rl.cc gotSettings := false - readIdleTimeout := cc.t.ReadIdleTimeout - var t http2timer + readIdleTimeout := cc.readIdleTimeout + var t *time.Timer if readIdleTimeout != 0 { - t = cc.t.afterFunc(readIdleTimeout, cc.healthCheck) + t = time.AfterFunc(readIdleTimeout, cc.healthCheck) } for { f, err := cc.fr.ReadFrame() @@ -9497,7 +9476,7 @@ func (rl *http2clientConnReadLoop) run() error { cc.vlogf("http2: Transport readFrame error on conn %p: (%T) %v", cc, err, err) } if se, ok := err.(http2StreamError); ok { - if cs := rl.streamByID(se.StreamID); cs != nil { + if cs := rl.streamByID(se.StreamID, http2notHeaderOrDataFrame); cs != nil { if se.Cause == nil { se.Cause = cc.fr.errDetail } @@ -9549,7 +9528,7 @@ func (rl *http2clientConnReadLoop) run() error { } func (rl *http2clientConnReadLoop) processHeaders(f *http2MetaHeadersFrame) error { - cs := rl.streamByID(f.StreamID) + cs := rl.streamByID(f.StreamID, http2headerOrDataFrame) if cs == nil { // We'd get here if we canceled a request while the // server had its response still in flight. So if this @@ -9637,7 +9616,7 @@ func (rl *http2clientConnReadLoop) handleResponse(cs *http2clientStream, f *http Status: status + " " + StatusText(statusCode), } for _, hf := range regularFields { - key := http2canonicalHeader(hf.Name) + key := httpcommon.CanonicalHeader(hf.Name) if key == "Trailer" { t := res.Trailer if t == nil { @@ -9645,7 +9624,7 @@ func (rl *http2clientConnReadLoop) handleResponse(cs *http2clientStream, f *http res.Trailer = t } http2foreachHeaderElement(hf.Value, func(v string) { - t[http2canonicalHeader(v)] = nil + t[httpcommon.CanonicalHeader(v)] = nil }) } else { vv := header[key] @@ -9667,15 +9646,34 @@ func (rl *http2clientConnReadLoop) handleResponse(cs *http2clientStream, f *http if f.StreamEnded() { return nil, errors.New("1xx informational response with END_STREAM flag") } - cs.num1xx++ - const max1xxResponses = 5 // arbitrary bound on number of informational responses, same as net/http - if cs.num1xx > max1xxResponses { - return nil, errors.New("http2: too many 1xx informational responses") - } if fn := cs.get1xxTraceFunc(); fn != nil { + // If the 1xx response is being delivered to the user, + // then they're responsible for limiting the number + // of responses. if err := fn(statusCode, textproto.MIMEHeader(header)); err != nil { return nil, err } + } else { + // If the user didn't examine the 1xx response, then we + // limit the size of all 1xx headers. + // + // This differs a bit from the HTTP/1 implementation, which + // limits the size of all 1xx headers plus the final response. + // Use the larger limit of MaxHeaderListSize and + // net/http.Transport.MaxResponseHeaderBytes. + limit := int64(cs.cc.t.maxHeaderListSize()) + if t1 := cs.cc.t.t1; t1 != nil && t1.MaxResponseHeaderBytes > limit { + limit = t1.MaxResponseHeaderBytes + } + for _, h := range f.Fields { + cs.totalHeaderSize += int64(h.Size()) + } + if cs.totalHeaderSize > limit { + if http2VerboseLogs { + log.Printf("http2: 1xx informational responses too large") + } + return nil, errors.New("header list too large") + } } if statusCode == 100 { http2traceGot100Continue(cs.trace) @@ -9750,7 +9748,7 @@ func (rl *http2clientConnReadLoop) processTrailers(cs *http2clientStream, f *htt trailer := make(Header) for _, hf := range f.RegularFields() { - key := http2canonicalHeader(hf.Name) + key := httpcommon.CanonicalHeader(hf.Name) trailer[key] = append(trailer[key], hf.Value) } cs.trailer = trailer @@ -9859,7 +9857,7 @@ func (b http2transportResponseBody) Close() error { func (rl *http2clientConnReadLoop) processData(f *http2DataFrame) error { cc := rl.cc - cs := rl.streamByID(f.StreamID) + cs := rl.streamByID(f.StreamID, http2headerOrDataFrame) data := f.Data() if cs == nil { cc.mu.Lock() @@ -9994,9 +9992,22 @@ func (rl *http2clientConnReadLoop) endStreamError(cs *http2clientStream, err err cs.abortStream(err) } -func (rl *http2clientConnReadLoop) streamByID(id uint32) *http2clientStream { +// Constants passed to streamByID for documentation purposes. +const ( + http2headerOrDataFrame = true + http2notHeaderOrDataFrame = false +) + +// streamByID returns the stream with the given id, or nil if no stream has that id. +// If headerOrData is true, it clears rst.StreamPingsBlocked. +func (rl *http2clientConnReadLoop) streamByID(id uint32, headerOrData bool) *http2clientStream { rl.cc.mu.Lock() defer rl.cc.mu.Unlock() + if headerOrData { + // Work around an unfortunate gRPC behavior. + // See comment on ClientConn.rstStreamPingsBlocked for details. + rl.cc.rstStreamPingsBlocked = false + } cs := rl.cc.streams[id] if cs != nil && !cs.readAborted { return cs @@ -10090,6 +10101,21 @@ func (rl *http2clientConnReadLoop) processSettingsNoWrite(f *http2SettingsFrame) case http2SettingHeaderTableSize: cc.henc.SetMaxDynamicTableSize(s.Val) cc.peerMaxHeaderTableSize = s.Val + case http2SettingEnableConnectProtocol: + if err := s.Valid(); err != nil { + return err + } + // If the peer wants to send us SETTINGS_ENABLE_CONNECT_PROTOCOL, + // we require that it do so in the first SETTINGS frame. + // + // When we attempt to use extended CONNECT, we wait for the first + // SETTINGS frame to see if the server supports it. If we let the + // server enable the feature with a later SETTINGS frame, then + // users will see inconsistent results depending on whether we've + // seen that frame or not. + if !cc.seenSettings { + cc.extendedConnectAllowed = s.Val == 1 + } default: cc.vlogf("Unhandled Setting: %v", s) } @@ -10107,6 +10133,7 @@ func (rl *http2clientConnReadLoop) processSettingsNoWrite(f *http2SettingsFrame) // connection can establish to our default. cc.maxConcurrentStreams = http2defaultMaxConcurrentStreams } + close(cc.seenSettingsChan) cc.seenSettings = true } @@ -10115,7 +10142,7 @@ func (rl *http2clientConnReadLoop) processSettingsNoWrite(f *http2SettingsFrame) func (rl *http2clientConnReadLoop) processWindowUpdate(f *http2WindowUpdateFrame) error { cc := rl.cc - cs := rl.streamByID(f.StreamID) + cs := rl.streamByID(f.StreamID, http2notHeaderOrDataFrame) if f.StreamID != 0 && cs == nil { return nil } @@ -10144,7 +10171,7 @@ func (rl *http2clientConnReadLoop) processWindowUpdate(f *http2WindowUpdateFrame } func (rl *http2clientConnReadLoop) processResetStream(f *http2RSTStreamFrame) error { - cs := rl.streamByID(f.StreamID) + cs := rl.streamByID(f.StreamID, http2notHeaderOrDataFrame) if cs == nil { // TODO: return error if server tries to RST_STREAM an idle stream return nil @@ -10184,7 +10211,6 @@ func (cc *http2ClientConn) Ping(ctx context.Context) error { var pingError error errc := make(chan struct{}) go func() { - cc.t.markNewGoroutine() cc.wmu.Lock() defer cc.wmu.Unlock() if pingError = cc.fr.WritePing(false, p); pingError != nil { @@ -10219,6 +10245,12 @@ func (rl *http2clientConnReadLoop) processPing(f *http2PingFrame) error { close(c) delete(cc.pings, f.Data) } + if cc.pendingResets > 0 { + // See clientStream.cleanupWriteRequest. + cc.pendingResets = 0 + cc.rstStreamPingsBlocked = true + cc.cond.Broadcast() + } return nil } cc := rl.cc @@ -10241,20 +10273,27 @@ func (rl *http2clientConnReadLoop) processPushPromise(f *http2PushPromiseFrame) return http2ConnectionError(http2ErrCodeProtocol) } -func (cc *http2ClientConn) writeStreamReset(streamID uint32, code http2ErrCode, err error) { +// writeStreamReset sends a RST_STREAM frame. +// When ping is true, it also sends a PING frame with a random payload. +func (cc *http2ClientConn) writeStreamReset(streamID uint32, code http2ErrCode, ping bool, err error) { // TODO: map err to more interesting error codes, once the // HTTP community comes up with some. But currently for // RST_STREAM there's no equivalent to GOAWAY frame's debug // data, and the error codes are all pretty vague ("cancel"). cc.wmu.Lock() cc.fr.WriteRSTStream(streamID, code) + if ping { + var payload [8]byte + rand.Read(payload[:]) + cc.fr.WritePing(false, payload) + } cc.bw.Flush() cc.wmu.Unlock() } var ( http2errResponseHeaderListSize = errors.New("http2: response header list larger than advertised limit") - http2errRequestHeaderListSize = errors.New("http2: request header list larger than peer's advertised limit") + http2errRequestHeaderListSize = httpcommon.ErrRequestHeaderListSize ) func (cc *http2ClientConn) logf(format string, args ...interface{}) { @@ -10441,16 +10480,6 @@ func http2traceFirstResponseByte(trace *httptrace.ClientTrace) { } } -func http2traceHasWroteHeaderField(trace *httptrace.ClientTrace) bool { - return trace != nil && trace.WroteHeaderField != nil -} - -func http2traceWroteHeaderField(trace *httptrace.ClientTrace, k, v string) { - if trace != nil && trace.WroteHeaderField != nil { - trace.WroteHeaderField(k, []string{v}) - } -} - func http2traceGot1xxResponseFunc(trace *httptrace.ClientTrace) func(int, textproto.MIMEHeader) error { if trace != nil { return trace.Got1xxResponse @@ -10472,6 +10501,27 @@ func (t *http2Transport) dialTLSWithContext(ctx context.Context, network, addr s return tlsCn, nil } +const http2nextProtoUnencryptedHTTP2 = "unencrypted_http2" + +// unencryptedNetConnFromTLSConn retrieves a net.Conn wrapped in a *tls.Conn. +// +// TLSNextProto functions accept a *tls.Conn. +// +// When passing an unencrypted HTTP/2 connection to a TLSNextProto function, +// we pass a *tls.Conn with an underlying net.Conn containing the unencrypted connection. +// To be extra careful about mistakes (accidentally dropping TLS encryption in a place +// where we want it), the tls.Conn contains a net.Conn with an UnencryptedNetConn method +// that returns the actual connection we want to use. +func http2unencryptedNetConnFromTLSConn(tc *tls.Conn) (net.Conn, error) { + conner, ok := tc.NetConn().(interface { + UnencryptedNetConn() net.Conn + }) + if !ok { + return nil, errors.New("http2: TLS conn unexpectedly found in unencrypted handoff") + } + return conner.UnencryptedNetConn(), nil +} + // writeFramer is implemented by any type that is used to write frames. type http2writeFramer interface { writeFrame(http2writeContext) error @@ -10588,6 +10638,18 @@ func (se http2StreamError) writeFrame(ctx http2writeContext) error { func (se http2StreamError) staysWithinBuffer(max int) bool { return http2frameHeaderLen+4 <= max } +type http2writePing struct { + data [8]byte +} + +func (w http2writePing) writeFrame(ctx http2writeContext) error { + return ctx.Framer().WritePing(false, w.data) +} + +func (w http2writePing) staysWithinBuffer(max int) bool { + return http2frameHeaderLen+len(w.data) <= max +} + type http2writePingAck struct{ pf *http2PingFrame } func (w http2writePingAck) writeFrame(ctx http2writeContext) error { @@ -10800,7 +10862,7 @@ func http2encodeHeaders(enc *hpack.Encoder, h Header, keys []string) { } for _, k := range keys { vv := h[k] - k, ascii := http2lowerHeader(k) + k, ascii := httpcommon.LowerHeader(k) if !ascii { // Skip writing invalid headers. Per RFC 7540, Section 8.1.2, header // field names have to be ASCII characters (just as in HTTP/1.x). @@ -10864,6 +10926,8 @@ type http2OpenStreamOptions struct { // PusherID is zero if the stream was initiated by the client. Otherwise, // PusherID names the stream that pushed the newly opened stream. PusherID uint32 + // priority is used to set the priority of the newly opened stream. + priority http2PriorityParam } // FrameWriteRequest is a request to write a frame. @@ -11075,7 +11139,7 @@ func (p *http2writeQueuePool) get() *http2writeQueue { } // RFC 7540, Section 5.3.5: the default weight is 16. -const http2priorityDefaultWeight = 15 // 16 = 15 + 1 +const http2priorityDefaultWeightRFC7540 = 15 // 16 = 15 + 1 // PriorityWriteSchedulerConfig configures a priorityWriteScheduler. type http2PriorityWriteSchedulerConfig struct { @@ -11130,8 +11194,8 @@ func http2NewPriorityWriteScheduler(cfg *http2PriorityWriteSchedulerConfig) http } } - ws := &http2priorityWriteScheduler{ - nodes: make(map[uint32]*http2priorityNode), + ws := &http2priorityWriteSchedulerRFC7540{ + nodes: make(map[uint32]*http2priorityNodeRFC7540), maxClosedNodesInTree: cfg.MaxClosedNodesInTree, maxIdleNodesInTree: cfg.MaxIdleNodesInTree, enableWriteThrottle: cfg.ThrottleOutOfOrderWrites, @@ -11145,32 +11209,32 @@ func http2NewPriorityWriteScheduler(cfg *http2PriorityWriteSchedulerConfig) http return ws } -type http2priorityNodeState int +type http2priorityNodeStateRFC7540 int const ( - http2priorityNodeOpen http2priorityNodeState = iota - http2priorityNodeClosed - http2priorityNodeIdle + http2priorityNodeOpenRFC7540 http2priorityNodeStateRFC7540 = iota + http2priorityNodeClosedRFC7540 + http2priorityNodeIdleRFC7540 ) -// priorityNode is a node in an HTTP/2 priority tree. +// priorityNodeRFC7540 is a node in an HTTP/2 priority tree. // Each node is associated with a single stream ID. // See RFC 7540, Section 5.3. -type http2priorityNode struct { - q http2writeQueue // queue of pending frames to write - id uint32 // id of the stream, or 0 for the root of the tree - weight uint8 // the actual weight is weight+1, so the value is in [1,256] - state http2priorityNodeState // open | closed | idle - bytes int64 // number of bytes written by this node, or 0 if closed - subtreeBytes int64 // sum(node.bytes) of all nodes in this subtree +type http2priorityNodeRFC7540 struct { + q http2writeQueue // queue of pending frames to write + id uint32 // id of the stream, or 0 for the root of the tree + weight uint8 // the actual weight is weight+1, so the value is in [1,256] + state http2priorityNodeStateRFC7540 // open | closed | idle + bytes int64 // number of bytes written by this node, or 0 if closed + subtreeBytes int64 // sum(node.bytes) of all nodes in this subtree // These links form the priority tree. - parent *http2priorityNode - kids *http2priorityNode // start of the kids list - prev, next *http2priorityNode // doubly-linked list of siblings + parent *http2priorityNodeRFC7540 + kids *http2priorityNodeRFC7540 // start of the kids list + prev, next *http2priorityNodeRFC7540 // doubly-linked list of siblings } -func (n *http2priorityNode) setParent(parent *http2priorityNode) { +func (n *http2priorityNodeRFC7540) setParent(parent *http2priorityNodeRFC7540) { if n == parent { panic("setParent to self") } @@ -11205,7 +11269,7 @@ func (n *http2priorityNode) setParent(parent *http2priorityNode) { } } -func (n *http2priorityNode) addBytes(b int64) { +func (n *http2priorityNodeRFC7540) addBytes(b int64) { n.bytes += b for ; n != nil; n = n.parent { n.subtreeBytes += b @@ -11218,7 +11282,7 @@ func (n *http2priorityNode) addBytes(b int64) { // // f(n, openParent) takes two arguments: the node to visit, n, and a bool that is true // if any ancestor p of n is still open (ignoring the root node). -func (n *http2priorityNode) walkReadyInOrder(openParent bool, tmp *[]*http2priorityNode, f func(*http2priorityNode, bool) bool) bool { +func (n *http2priorityNodeRFC7540) walkReadyInOrder(openParent bool, tmp *[]*http2priorityNodeRFC7540, f func(*http2priorityNodeRFC7540, bool) bool) bool { if !n.q.empty() && f(n, openParent) { return true } @@ -11229,7 +11293,7 @@ func (n *http2priorityNode) walkReadyInOrder(openParent bool, tmp *[]*http2prior // Don't consider the root "open" when updating openParent since // we can't send data frames on the root stream (only control frames). if n.id != 0 { - openParent = openParent || (n.state == http2priorityNodeOpen) + openParent = openParent || (n.state == http2priorityNodeOpenRFC7540) } // Common case: only one kid or all kids have the same weight. @@ -11259,7 +11323,7 @@ func (n *http2priorityNode) walkReadyInOrder(openParent bool, tmp *[]*http2prior *tmp = append(*tmp, n.kids) n.kids.setParent(nil) } - sort.Sort(http2sortPriorityNodeSiblings(*tmp)) + sort.Sort(http2sortPriorityNodeSiblingsRFC7540(*tmp)) for i := len(*tmp) - 1; i >= 0; i-- { (*tmp)[i].setParent(n) // setParent inserts at the head of n.kids } @@ -11271,13 +11335,13 @@ func (n *http2priorityNode) walkReadyInOrder(openParent bool, tmp *[]*http2prior return false } -type http2sortPriorityNodeSiblings []*http2priorityNode +type http2sortPriorityNodeSiblingsRFC7540 []*http2priorityNodeRFC7540 -func (z http2sortPriorityNodeSiblings) Len() int { return len(z) } +func (z http2sortPriorityNodeSiblingsRFC7540) Len() int { return len(z) } -func (z http2sortPriorityNodeSiblings) Swap(i, k int) { z[i], z[k] = z[k], z[i] } +func (z http2sortPriorityNodeSiblingsRFC7540) Swap(i, k int) { z[i], z[k] = z[k], z[i] } -func (z http2sortPriorityNodeSiblings) Less(i, k int) bool { +func (z http2sortPriorityNodeSiblingsRFC7540) Less(i, k int) bool { // Prefer the subtree that has sent fewer bytes relative to its weight. // See sections 5.3.2 and 5.3.4. wi, bi := float64(z[i].weight+1), float64(z[i].subtreeBytes) @@ -11291,13 +11355,13 @@ func (z http2sortPriorityNodeSiblings) Less(i, k int) bool { return bi/bk <= wi/wk } -type http2priorityWriteScheduler struct { +type http2priorityWriteSchedulerRFC7540 struct { // root is the root of the priority tree, where root.id = 0. // The root queues control frames that are not associated with any stream. - root http2priorityNode + root http2priorityNodeRFC7540 // nodes maps stream ids to priority tree nodes. - nodes map[uint32]*http2priorityNode + nodes map[uint32]*http2priorityNodeRFC7540 // maxID is the maximum stream id in nodes. maxID uint32 @@ -11305,7 +11369,7 @@ type http2priorityWriteScheduler struct { // lists of nodes that have been closed or are idle, but are kept in // the tree for improved prioritization. When the lengths exceed either // maxClosedNodesInTree or maxIdleNodesInTree, old nodes are discarded. - closedNodes, idleNodes []*http2priorityNode + closedNodes, idleNodes []*http2priorityNodeRFC7540 // From the config. maxClosedNodesInTree int @@ -11314,19 +11378,19 @@ type http2priorityWriteScheduler struct { enableWriteThrottle bool // tmp is scratch space for priorityNode.walkReadyInOrder to reduce allocations. - tmp []*http2priorityNode + tmp []*http2priorityNodeRFC7540 // pool of empty queues for reuse. queuePool http2writeQueuePool } -func (ws *http2priorityWriteScheduler) OpenStream(streamID uint32, options http2OpenStreamOptions) { +func (ws *http2priorityWriteSchedulerRFC7540) OpenStream(streamID uint32, options http2OpenStreamOptions) { // The stream may be currently idle but cannot be opened or closed. if curr := ws.nodes[streamID]; curr != nil { - if curr.state != http2priorityNodeIdle { + if curr.state != http2priorityNodeIdleRFC7540 { panic(fmt.Sprintf("stream %d already opened", streamID)) } - curr.state = http2priorityNodeOpen + curr.state = http2priorityNodeOpenRFC7540 return } @@ -11338,11 +11402,11 @@ func (ws *http2priorityWriteScheduler) OpenStream(streamID uint32, options http2 if parent == nil { parent = &ws.root } - n := &http2priorityNode{ + n := &http2priorityNodeRFC7540{ q: *ws.queuePool.get(), id: streamID, - weight: http2priorityDefaultWeight, - state: http2priorityNodeOpen, + weight: http2priorityDefaultWeightRFC7540, + state: http2priorityNodeOpenRFC7540, } n.setParent(parent) ws.nodes[streamID] = n @@ -11351,19 +11415,19 @@ func (ws *http2priorityWriteScheduler) OpenStream(streamID uint32, options http2 } } -func (ws *http2priorityWriteScheduler) CloseStream(streamID uint32) { +func (ws *http2priorityWriteSchedulerRFC7540) CloseStream(streamID uint32) { if streamID == 0 { panic("violation of WriteScheduler interface: cannot close stream 0") } if ws.nodes[streamID] == nil { panic(fmt.Sprintf("violation of WriteScheduler interface: unknown stream %d", streamID)) } - if ws.nodes[streamID].state != http2priorityNodeOpen { + if ws.nodes[streamID].state != http2priorityNodeOpenRFC7540 { panic(fmt.Sprintf("violation of WriteScheduler interface: stream %d already closed", streamID)) } n := ws.nodes[streamID] - n.state = http2priorityNodeClosed + n.state = http2priorityNodeClosedRFC7540 n.addBytes(-n.bytes) q := n.q @@ -11376,7 +11440,7 @@ func (ws *http2priorityWriteScheduler) CloseStream(streamID uint32) { } } -func (ws *http2priorityWriteScheduler) AdjustStream(streamID uint32, priority http2PriorityParam) { +func (ws *http2priorityWriteSchedulerRFC7540) AdjustStream(streamID uint32, priority http2PriorityParam) { if streamID == 0 { panic("adjustPriority on root") } @@ -11390,11 +11454,11 @@ func (ws *http2priorityWriteScheduler) AdjustStream(streamID uint32, priority ht return } ws.maxID = streamID - n = &http2priorityNode{ + n = &http2priorityNodeRFC7540{ q: *ws.queuePool.get(), id: streamID, - weight: http2priorityDefaultWeight, - state: http2priorityNodeIdle, + weight: http2priorityDefaultWeightRFC7540, + state: http2priorityNodeIdleRFC7540, } n.setParent(&ws.root) ws.nodes[streamID] = n @@ -11406,7 +11470,7 @@ func (ws *http2priorityWriteScheduler) AdjustStream(streamID uint32, priority ht parent := ws.nodes[priority.StreamDep] if parent == nil { n.setParent(&ws.root) - n.weight = http2priorityDefaultWeight + n.weight = http2priorityDefaultWeightRFC7540 return } @@ -11447,8 +11511,8 @@ func (ws *http2priorityWriteScheduler) AdjustStream(streamID uint32, priority ht n.weight = priority.Weight } -func (ws *http2priorityWriteScheduler) Push(wr http2FrameWriteRequest) { - var n *http2priorityNode +func (ws *http2priorityWriteSchedulerRFC7540) Push(wr http2FrameWriteRequest) { + var n *http2priorityNodeRFC7540 if wr.isControl() { n = &ws.root } else { @@ -11467,8 +11531,8 @@ func (ws *http2priorityWriteScheduler) Push(wr http2FrameWriteRequest) { n.q.push(wr) } -func (ws *http2priorityWriteScheduler) Pop() (wr http2FrameWriteRequest, ok bool) { - ws.root.walkReadyInOrder(false, &ws.tmp, func(n *http2priorityNode, openParent bool) bool { +func (ws *http2priorityWriteSchedulerRFC7540) Pop() (wr http2FrameWriteRequest, ok bool) { + ws.root.walkReadyInOrder(false, &ws.tmp, func(n *http2priorityNodeRFC7540, openParent bool) bool { limit := int32(math.MaxInt32) if openParent { limit = ws.writeThrottleLimit @@ -11494,7 +11558,7 @@ func (ws *http2priorityWriteScheduler) Pop() (wr http2FrameWriteRequest, ok bool return wr, ok } -func (ws *http2priorityWriteScheduler) addClosedOrIdleNode(list *[]*http2priorityNode, maxSize int, n *http2priorityNode) { +func (ws *http2priorityWriteSchedulerRFC7540) addClosedOrIdleNode(list *[]*http2priorityNodeRFC7540, maxSize int, n *http2priorityNodeRFC7540) { if maxSize == 0 { return } @@ -11508,7 +11572,7 @@ func (ws *http2priorityWriteScheduler) addClosedOrIdleNode(list *[]*http2priorit *list = append(*list, n) } -func (ws *http2priorityWriteScheduler) removeNode(n *http2priorityNode) { +func (ws *http2priorityWriteSchedulerRFC7540) removeNode(n *http2priorityNodeRFC7540) { for n.kids != nil { n.kids.setParent(n.parent) } @@ -11516,6 +11580,205 @@ func (ws *http2priorityWriteScheduler) removeNode(n *http2priorityNode) { delete(ws.nodes, n.id) } +type http2streamMetadata struct { + location *http2writeQueue + priority http2PriorityParam +} + +type http2priorityWriteSchedulerRFC9218 struct { + // control contains control frames (SETTINGS, PING, etc.). + control http2writeQueue + + // heads contain the head of a circular list of streams. + // We put these heads within a nested array that represents urgency and + // incremental, as defined in + // https://www.rfc-editor.org/rfc/rfc9218.html#name-priority-parameters. + // 8 represents u=0 up to u=7, and 2 represents i=false and i=true. + heads [8][2]*http2writeQueue + + // streams contains a mapping between each stream ID and their metadata, so + // we can quickly locate them when needing to, for example, adjust their + // priority. + streams map[uint32]http2streamMetadata + + // queuePool are empty queues for reuse. + queuePool http2writeQueuePool + + // prioritizeIncremental is used to determine whether we should prioritize + // incremental streams or not, when urgency is the same in a given Pop() + // call. + prioritizeIncremental bool +} + +func http2newPriorityWriteSchedulerRFC9128() http2WriteScheduler { + ws := &http2priorityWriteSchedulerRFC9218{ + streams: make(map[uint32]http2streamMetadata), + } + return ws +} + +func (ws *http2priorityWriteSchedulerRFC9218) OpenStream(streamID uint32, opt http2OpenStreamOptions) { + if ws.streams[streamID].location != nil { + panic(fmt.Errorf("stream %d already opened", streamID)) + } + q := ws.queuePool.get() + ws.streams[streamID] = http2streamMetadata{ + location: q, + priority: opt.priority, + } + + u, i := opt.priority.urgency, opt.priority.incremental + if ws.heads[u][i] == nil { + ws.heads[u][i] = q + q.next = q + q.prev = q + } else { + // Queues are stored in a ring. + // Insert the new stream before ws.head, putting it at the end of the list. + q.prev = ws.heads[u][i].prev + q.next = ws.heads[u][i] + q.prev.next = q + q.next.prev = q + } +} + +func (ws *http2priorityWriteSchedulerRFC9218) CloseStream(streamID uint32) { + metadata := ws.streams[streamID] + q, u, i := metadata.location, metadata.priority.urgency, metadata.priority.incremental + if q == nil { + return + } + if q.next == q { + // This was the only open stream. + ws.heads[u][i] = nil + } else { + q.prev.next = q.next + q.next.prev = q.prev + if ws.heads[u][i] == q { + ws.heads[u][i] = q.next + } + } + delete(ws.streams, streamID) + ws.queuePool.put(q) +} + +func (ws *http2priorityWriteSchedulerRFC9218) AdjustStream(streamID uint32, priority http2PriorityParam) { + metadata := ws.streams[streamID] + q, u, i := metadata.location, metadata.priority.urgency, metadata.priority.incremental + if q == nil { + return + } + + // Remove stream from current location. + if q.next == q { + // This was the only open stream. + ws.heads[u][i] = nil + } else { + q.prev.next = q.next + q.next.prev = q.prev + if ws.heads[u][i] == q { + ws.heads[u][i] = q.next + } + } + + // Insert stream to the new queue. + u, i = priority.urgency, priority.incremental + if ws.heads[u][i] == nil { + ws.heads[u][i] = q + q.next = q + q.prev = q + } else { + // Queues are stored in a ring. + // Insert the new stream before ws.head, putting it at the end of the list. + q.prev = ws.heads[u][i].prev + q.next = ws.heads[u][i] + q.prev.next = q + q.next.prev = q + } + + // Update the metadata. + ws.streams[streamID] = http2streamMetadata{ + location: q, + priority: priority, + } +} + +func (ws *http2priorityWriteSchedulerRFC9218) Push(wr http2FrameWriteRequest) { + if wr.isControl() { + ws.control.push(wr) + return + } + q := ws.streams[wr.StreamID()].location + if q == nil { + // This is a closed stream. + // wr should not be a HEADERS or DATA frame. + // We push the request onto the control queue. + if wr.DataSize() > 0 { + panic("add DATA on non-open stream") + } + ws.control.push(wr) + return + } + q.push(wr) +} + +func (ws *http2priorityWriteSchedulerRFC9218) Pop() (http2FrameWriteRequest, bool) { + // Control and RST_STREAM frames first. + if !ws.control.empty() { + return ws.control.shift(), true + } + + // On the next Pop(), we want to prioritize incremental if we prioritized + // non-incremental request of the same urgency this time. Vice-versa. + // i.e. when there are incremental and non-incremental requests at the same + // priority, we give 50% of our bandwidth to the incremental ones in + // aggregate and 50% to the first non-incremental one (since + // non-incremental streams do not use round-robin writes). + ws.prioritizeIncremental = !ws.prioritizeIncremental + + // Always prioritize lowest u (i.e. highest urgency level). + for u := range ws.heads { + for i := range ws.heads[u] { + // When we want to prioritize incremental, we try to pop i=true + // first before i=false when u is the same. + if ws.prioritizeIncremental { + i = (i + 1) % 2 + } + q := ws.heads[u][i] + if q == nil { + continue + } + for { + if wr, ok := q.consume(math.MaxInt32); ok { + if i == 1 { + // For incremental streams, we update head to q.next so + // we can round-robin between multiple streams that can + // immediately benefit from partial writes. + ws.heads[u][i] = q.next + } else { + // For non-incremental streams, we try to finish one to + // completion rather than doing round-robin. However, + // we update head here so that if q.consume() is !ok + // (e.g. the stream has no more frame to consume), head + // is updated to the next q that has frames to consume + // on future iterations. This way, we do not prioritize + // writing to unavailable stream on next Pop() calls, + // preventing head-of-line blocking. + ws.heads[u][i] = q + } + return wr, true + } + q = q.next + if q == ws.heads[u][i] { + break + } + } + + } + } + return http2FrameWriteRequest{}, false +} + // NewRandomWriteScheduler constructs a WriteScheduler that ignores HTTP/2 // priorities. Control frames like SETTINGS and PING are written before DATA // frames, but if no control frames are queued and multiple streams have queued @@ -11602,7 +11865,7 @@ type http2roundRobinWriteScheduler struct { } // newRoundRobinWriteScheduler constructs a new write scheduler. -// The round robin scheduler priorizes control frames +// The round robin scheduler prioritizes control frames // like SETTINGS and PING over DATA frames. // When there are no control frames to send, it performs a round-robin // selection from the ready streams. diff --git a/src/net/http/h2_error_test.go b/src/net/http/h2_error_test.go index 5e400683b415e7..e71825451a8e32 100644 --- a/src/net/http/h2_error_test.go +++ b/src/net/http/h2_error_test.go @@ -25,19 +25,18 @@ func (e externalStreamError) Error() string { } func TestStreamError(t *testing.T) { - var target externalStreamError streamErr := http2streamError(42, http2ErrCodeProtocol) - ok := errors.As(streamErr, &target) + extStreamErr, ok := errors.AsType[externalStreamError](streamErr) if !ok { - t.Fatalf("errors.As failed") + t.Fatalf("errors.AsType failed") } - if target.StreamID != streamErr.StreamID { - t.Errorf("got StreamID %v, expected %v", target.StreamID, streamErr.StreamID) + if extStreamErr.StreamID != streamErr.StreamID { + t.Errorf("got StreamID %v, expected %v", extStreamErr.StreamID, streamErr.StreamID) } - if target.Cause != streamErr.Cause { - t.Errorf("got Cause %v, expected %v", target.Cause, streamErr.Cause) + if extStreamErr.Cause != streamErr.Cause { + t.Errorf("got Cause %v, expected %v", extStreamErr.Cause, streamErr.Cause) } - if uint32(target.Code) != uint32(streamErr.Code) { - t.Errorf("got Code %v, expected %v", target.Code, streamErr.Code) + if uint32(extStreamErr.Code) != uint32(streamErr.Code) { + t.Errorf("got Code %v, expected %v", extStreamErr.Code, streamErr.Code) } } diff --git a/src/net/http/http.go b/src/net/http/http.go index 6e2259adbf3529..e7959fa3b6045b 100644 --- a/src/net/http/http.go +++ b/src/net/http/http.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:generate bundle -o=h2_bundle.go -prefix=http2 -tags=!nethttpomithttp2 golang.org/x/net/http2 +//go:generate bundle -o=h2_bundle.go -prefix=http2 -tags=!nethttpomithttp2 -import=golang.org/x/net/internal/httpcommon=net/http/internal/httpcommon golang.org/x/net/http2 package http @@ -16,6 +16,67 @@ import ( "golang.org/x/net/http/httpguts" ) +// Protocols is a set of HTTP protocols. +// The zero value is an empty set of protocols. +// +// The supported protocols are: +// +// - HTTP1 is the HTTP/1.0 and HTTP/1.1 protocols. +// HTTP1 is supported on both unsecured TCP and secured TLS connections. +// +// - HTTP2 is the HTTP/2 protcol over a TLS connection. +// +// - UnencryptedHTTP2 is the HTTP/2 protocol over an unsecured TCP connection. +type Protocols struct { + bits uint8 +} + +const ( + protoHTTP1 = 1 << iota + protoHTTP2 + protoUnencryptedHTTP2 +) + +// HTTP1 reports whether p includes HTTP/1. +func (p Protocols) HTTP1() bool { return p.bits&protoHTTP1 != 0 } + +// SetHTTP1 adds or removes HTTP/1 from p. +func (p *Protocols) SetHTTP1(ok bool) { p.setBit(protoHTTP1, ok) } + +// HTTP2 reports whether p includes HTTP/2. +func (p Protocols) HTTP2() bool { return p.bits&protoHTTP2 != 0 } + +// SetHTTP2 adds or removes HTTP/2 from p. +func (p *Protocols) SetHTTP2(ok bool) { p.setBit(protoHTTP2, ok) } + +// UnencryptedHTTP2 reports whether p includes unencrypted HTTP/2. +func (p Protocols) UnencryptedHTTP2() bool { return p.bits&protoUnencryptedHTTP2 != 0 } + +// SetUnencryptedHTTP2 adds or removes unencrypted HTTP/2 from p. +func (p *Protocols) SetUnencryptedHTTP2(ok bool) { p.setBit(protoUnencryptedHTTP2, ok) } + +func (p *Protocols) setBit(bit uint8, ok bool) { + if ok { + p.bits |= bit + } else { + p.bits &^= bit + } +} + +func (p Protocols) String() string { + var s []string + if p.HTTP1() { + s = append(s, "HTTP1") + } + if p.HTTP2() { + s = append(s, "HTTP2") + } + if p.UnencryptedHTTP2() { + s = append(s, "UnencryptedHTTP2") + } + return "{" + strings.Join(s, ",") + "}" +} + // incomparable is a zero-width, non-comparable type. Adding it to a struct // makes that struct also non-comparable, and generally doesn't add // any size (as long as it's first). @@ -62,6 +123,12 @@ func isNotToken(r rune) bool { return !httpguts.IsTokenRune(r) } +// isToken reports whether v is a valid token (https://www.rfc-editor.org/rfc/rfc2616#section-2.2). +func isToken(v string) bool { + // For historical reasons, this function is called ValidHeaderFieldName (see issue #67031). + return httpguts.ValidHeaderFieldName(v) +} + // stringContainsCTLByte reports whether s contains any ASCII control character. func stringContainsCTLByte(s string) bool { for i := 0; i < len(s); i++ { @@ -163,3 +230,81 @@ type Pusher interface { // is not supported on the underlying connection. Push(target string, opts *PushOptions) error } + +// HTTP2Config defines HTTP/2 configuration parameters common to +// both [Transport] and [Server]. +type HTTP2Config struct { + // MaxConcurrentStreams optionally specifies the number of + // concurrent streams that a client may have open at a time. + // If zero, MaxConcurrentStreams defaults to at least 100. + // + // This parameter only applies to Servers. + MaxConcurrentStreams int + + // StrictMaxConcurrentRequests controls whether an HTTP/2 server's + // concurrency limit should be respected across all connections + // to that server. + // If true, new requests sent when a connection's concurrency limit + // has been exceeded will block until an existing request completes. + // If false, an additional connection will be opened if all + // existing connections are at their limit. + // + // This parameter only applies to Transports. + StrictMaxConcurrentRequests bool + + // MaxDecoderHeaderTableSize optionally specifies an upper limit for the + // size of the header compression table used for decoding headers sent + // by the peer. + // A valid value is less than 4MiB. + // If zero or invalid, a default value is used. + MaxDecoderHeaderTableSize int + + // MaxEncoderHeaderTableSize optionally specifies an upper limit for the + // header compression table used for sending headers to the peer. + // A valid value is less than 4MiB. + // If zero or invalid, a default value is used. + MaxEncoderHeaderTableSize int + + // MaxReadFrameSize optionally specifies the largest frame + // this endpoint is willing to read. + // A valid value is between 16KiB and 16MiB, inclusive. + // If zero or invalid, a default value is used. + MaxReadFrameSize int + + // MaxReceiveBufferPerConnection is the maximum size of the + // flow control window for data received on a connection. + // A valid value is at least 64KiB and less than 4MiB. + // If invalid, a default value is used. + MaxReceiveBufferPerConnection int + + // MaxReceiveBufferPerStream is the maximum size of + // the flow control window for data received on a stream (request). + // A valid value is less than 4MiB. + // If zero or invalid, a default value is used. + MaxReceiveBufferPerStream int + + // SendPingTimeout is the timeout after which a health check using a ping + // frame will be carried out if no frame is received on a connection. + // If zero, no health check is performed. + SendPingTimeout time.Duration + + // PingTimeout is the timeout after which a connection will be closed + // if a response to a ping is not received. + // If zero, a default of 15 seconds is used. + PingTimeout time.Duration + + // WriteByteTimeout is the timeout after which a connection will be + // closed if no data can be written to it. The timeout begins when data is + // available to write, and is extended whenever any bytes are written. + WriteByteTimeout time.Duration + + // PermitProhibitedCipherSuites, if true, permits the use of + // cipher suites prohibited by the HTTP/2 spec. + PermitProhibitedCipherSuites bool + + // CountError, if non-nil, is called on HTTP/2 errors. + // It is intended to increment a metric for monitoring. + // The errType contains only lowercase letters, digits, and underscores + // (a-z, 0-9, _). + CountError func(errType string) +} diff --git a/src/net/http/http_test.go b/src/net/http/http_test.go index df9812fc94747c..c12bbedac986db 100644 --- a/src/net/http/http_test.go +++ b/src/net/http/http_test.go @@ -151,9 +151,7 @@ var forbiddenStringsFunctions = map[string]bool{ // strings and bytes package functions. HTTP is mostly ASCII based, and doing // Unicode-aware case folding or space stripping can introduce vulnerabilities. func TestNoUnicodeStrings(t *testing.T) { - if !testenv.HasSrc() { - t.Skip("source code not available") - } + testenv.MustHaveSource(t) re := regexp.MustCompile(`(strings|bytes).([A-Za-z]+)`) if err := fs.WalkDir(os.DirFS("."), ".", func(path string, d fs.DirEntry, err error) error { @@ -166,7 +164,9 @@ func TestNoUnicodeStrings(t *testing.T) { } if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") || - path == "h2_bundle.go" || d.IsDir() { + path == "h2_bundle.go" || + path == "internal/httpcommon/httpcommon.go" || + d.IsDir() { return nil } @@ -189,6 +189,28 @@ func TestNoUnicodeStrings(t *testing.T) { } } +func TestProtocols(t *testing.T) { + var p Protocols + if p.HTTP1() { + t.Errorf("zero-value protocols: p.HTTP1() = true, want false") + } + p.SetHTTP1(true) + p.SetHTTP2(true) + if !p.HTTP1() { + t.Errorf("initialized protocols: p.HTTP1() = false, want true") + } + if !p.HTTP2() { + t.Errorf("initialized protocols: p.HTTP2() = false, want true") + } + p.SetHTTP1(false) + if p.HTTP1() { + t.Errorf("after unsetting HTTP1: p.HTTP1() = true, want false") + } + if !p.HTTP2() { + t.Errorf("after unsetting HTTP1: p.HTTP2() = false, want true") + } +} + const redirectURL = "/thisaredirect细雪withasciilettersのけぶabcdefghijk.html" func BenchmarkHexEscapeNonASCII(b *testing.B) { diff --git a/src/net/http/httptest/httptest.go b/src/net/http/httptest/httptest.go index 0c0dbb40e89bc5..7fe7107a9a191c 100644 --- a/src/net/http/httptest/httptest.go +++ b/src/net/http/httptest/httptest.go @@ -34,9 +34,9 @@ func NewRequest(method, target string, body io.Reader) *http.Request { // // An empty method means "GET". // -// The provided body may be nil. If the body is of type *bytes.Reader, -// *strings.Reader, or *bytes.Buffer, the Request.ContentLength is -// set. +// The provided body may be nil. If the body is of type [bytes.Reader], +// [strings.Reader], [bytes.Buffer], or the value [http.NoBody], +// the Request.ContentLength is set. // // NewRequest panics on error for ease of use in testing, where a // panic is acceptable. @@ -69,6 +69,9 @@ func NewRequestWithContext(ctx context.Context, method, target string, body io.R default: req.ContentLength = -1 } + if body == http.NoBody { + req.ContentLength = 0 + } if rc, ok := body.(io.ReadCloser); ok { req.Body = rc } else { diff --git a/src/net/http/httptest/httptest_test.go b/src/net/http/httptest/httptest_test.go index d5a4c3dc9d19a1..5f2215cfc6cdc1 100644 --- a/src/net/http/httptest/httptest_test.go +++ b/src/net/http/httptest/httptest_test.go @@ -156,6 +156,24 @@ func TestNewRequestWithContext(t *testing.T) { wantBody: "foo", }, + { + name: "Post with NoBody", + method: "POST", + uri: "/", + body: http.NoBody, + want: &http.Request{ + Method: "POST", + Host: "example.com", + URL: &url.URL{Path: "/"}, + Header: http.Header{}, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + RemoteAddr: "192.0.2.1:1234", + RequestURI: "/", + }, + }, + { name: "OPTIONS *", method: "OPTIONS", diff --git a/src/net/http/httptest/recorder.go b/src/net/http/httptest/recorder.go index dd51901b0d3b94..4006f4406d27a5 100644 --- a/src/net/http/httptest/recorder.go +++ b/src/net/http/httptest/recorder.go @@ -105,23 +105,45 @@ func (rw *ResponseRecorder) writeHeader(b []byte, str string) { // Write implements http.ResponseWriter. The data in buf is written to // rw.Body, if not nil. func (rw *ResponseRecorder) Write(buf []byte) (int, error) { + // Record the write, even if we're going to return an error. rw.writeHeader(buf, "") if rw.Body != nil { rw.Body.Write(buf) } + if !bodyAllowedForStatus(rw.Code) { + return 0, http.ErrBodyNotAllowed + } return len(buf), nil } // WriteString implements [io.StringWriter]. The data in str is written // to rw.Body, if not nil. func (rw *ResponseRecorder) WriteString(str string) (int, error) { + // Record the write, even if we're going to return an error. rw.writeHeader(nil, str) if rw.Body != nil { rw.Body.WriteString(str) } + if !bodyAllowedForStatus(rw.Code) { + return 0, http.ErrBodyNotAllowed + } return len(str), nil } +// bodyAllowedForStatus reports whether a given response status code +// permits a body. See RFC 7230, section 3.3. +func bodyAllowedForStatus(status int) bool { + switch { + case status >= 100 && status <= 199: + return false + case status == 204: + return false + case status == 304: + return false + } + return true +} + func checkWriteHeaderCode(code int) { // Issue 22880: require valid WriteHeader status codes. // For now we only enforce that it's three digits. @@ -207,7 +229,7 @@ func (rw *ResponseRecorder) Result() *http.Response { if trailers, ok := rw.snapHeader["Trailer"]; ok { res.Trailer = make(http.Header, len(trailers)) for _, k := range trailers { - for _, k := range strings.Split(k, ",") { + for k := range strings.SplitSeq(k, ",") { k = http.CanonicalHeaderKey(textproto.TrimString(k)) if !httpguts.ValidTrailerHeader(k) { // Ignore since forbidden by RFC 7230, section 4.1.2. diff --git a/src/net/http/httptest/recorder_test.go b/src/net/http/httptest/recorder_test.go index 4782eced43e6ce..9d1c4430c9b01c 100644 --- a/src/net/http/httptest/recorder_test.go +++ b/src/net/http/httptest/recorder_test.go @@ -5,6 +5,8 @@ package httptest import ( + "bytes" + "errors" "fmt" "io" "net/http" @@ -309,6 +311,26 @@ func TestRecorder(t *testing.T) { } } +func TestBodyNotAllowed(t *testing.T) { + rw := NewRecorder() + rw.Body = new(bytes.Buffer) + rw.WriteHeader(204) + + _, err := rw.Write([]byte("hello ")) + if !errors.Is(err, http.ErrBodyNotAllowed) { + t.Errorf("expected BodyNotAllowed for Write after 204, got: %v", err) + } + + _, err = rw.WriteString("world") + if !errors.Is(err, http.ErrBodyNotAllowed) { + t.Errorf("expected BodyNotAllowed for WriteString after 204, got: %v", err) + } + + if got, want := rw.Body.String(), "hello world"; got != want { + t.Errorf("got Body=%q, want %q", got, want) + } +} + // issue 39017 - disallow Content-Length values such as "+3" func TestParseContentLength(t *testing.T) { tests := []struct { @@ -352,7 +374,6 @@ func TestRecorderPanicsOnNonXXXStatusCode(t *testing.T) { -100, 0, 99, 1000, 20000, } for _, badCode := range badCodes { - badCode := badCode t.Run(fmt.Sprintf("Code=%d", badCode), func(t *testing.T) { defer func() { if r := recover(); r == nil { diff --git a/src/net/http/httptest/server.go b/src/net/http/httptest/server.go index fa549231796925..7ae2561b71971d 100644 --- a/src/net/http/httptest/server.go +++ b/src/net/http/httptest/server.go @@ -7,6 +7,7 @@ package httptest import ( + "context" "crypto/tls" "crypto/x509" "flag" @@ -126,8 +127,24 @@ func (s *Server) Start() { if s.URL != "" { panic("Server already started") } + if s.client == nil { - s.client = &http.Client{Transport: &http.Transport{}} + tr := &http.Transport{} + dialer := net.Dialer{} + // User code may set either of Dial or DialContext, with DialContext taking precedence. + // We set DialContext here to preserve any context values that are passed in, + // but fall back to Dial if the user has set it. + tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + if tr.Dial != nil { + return tr.Dial(network, addr) + } + if addr == "example.com:80" || strings.HasSuffix(addr, ".example.com:80") { + addr = s.Listener.Addr().String() + } + return dialer.DialContext(ctx, network, addr) + } + s.client = &http.Client{Transport: tr} + } s.URL = "http://" + s.Listener.Addr().String() s.wrap() @@ -173,12 +190,23 @@ func (s *Server) StartTLS() { } certpool := x509.NewCertPool() certpool.AddCert(s.certificate) - s.client.Transport = &http.Transport{ + tr := &http.Transport{ TLSClientConfig: &tls.Config{ RootCAs: certpool, }, ForceAttemptHTTP2: s.EnableHTTP2, } + dialer := net.Dialer{} + tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + if tr.Dial != nil { + return tr.Dial(network, addr) + } + if addr == "example.com:443" || strings.HasSuffix(addr, ".example.com:443") { + addr = s.Listener.Addr().String() + } + return dialer.DialContext(ctx, network, addr) + } + s.client.Transport = tr s.Listener = tls.NewListener(s.Listener, s.TLS) s.URL = "https://" + s.Listener.Addr().String() s.wrap() @@ -300,6 +328,8 @@ func (s *Server) Certificate() *x509.Certificate { // It is configured to trust the server's TLS test certificate and will // close its idle connections on [Server.Close]. // Use Server.URL as the base URL to send requests to the server. +// The returned client will also redirect any requests to "example.com" +// or its subdomains to the server. func (s *Server) Client() *http.Client { return s.client } diff --git a/src/net/http/httptest/server_test.go b/src/net/http/httptest/server_test.go index c96a0ff3379b4d..f3cfa7c2dbdb76 100644 --- a/src/net/http/httptest/server_test.go +++ b/src/net/http/httptest/server_test.go @@ -293,3 +293,40 @@ func TestTLSServerWithHTTP2(t *testing.T) { }) } } + +func TestClientExampleCom(t *testing.T) { + modes := []struct { + proto string + host string + }{ + {"http", "example.com"}, + {"http", "foo.example.com"}, + {"https", "example.com"}, + {"https", "foo.example.com"}, + } + + for _, tt := range modes { + t.Run(tt.proto+" "+tt.host, func(t *testing.T) { + cst := NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("requested-hostname", r.Host) + })) + switch tt.proto { + case "https": + cst.EnableHTTP2 = true + cst.StartTLS() + default: + cst.Start() + } + + defer cst.Close() + + res, err := cst.Client().Get(tt.proto + "://" + tt.host) + if err != nil { + t.Fatalf("Failed to make request: %v", err) + } + if got, want := res.Header.Get("requested-hostname"), tt.host; got != want { + t.Fatalf("Requested hostname mismatch\ngot: %q\nwant: %q", got, want) + } + }) + } +} diff --git a/src/net/http/httptrace/trace.go b/src/net/http/httptrace/trace.go index 706a4329578ef7..cee13d2da8345d 100644 --- a/src/net/http/httptrace/trace.go +++ b/src/net/http/httptrace/trace.go @@ -76,7 +76,7 @@ func WithClientTrace(ctx context.Context, trace *ClientTrace) context.Context { // during a single round trip and has no hooks that span a series // of redirected requests. // -// See https://blog.golang.org/http-tracing for more. +// See https://go.dev/blog/http-tracing for more. type ClientTrace struct { // GetConn is called before a connection is created or // retrieved from an idle pool. The hostPort is the diff --git a/src/net/http/httputil/dump.go b/src/net/http/httputil/dump.go index 2edb9bc98d3bc8..23918e2ad4bbd5 100644 --- a/src/net/http/httputil/dump.go +++ b/src/net/http/httputil/dump.go @@ -147,7 +147,6 @@ func DumpRequestOut(req *http.Request, body bool) ([]byte, error) { req.Body = save if err != nil { - pw.Close() dr.err = err close(quitReadCh) return nil, err diff --git a/src/net/http/httputil/reverseproxy.go b/src/net/http/httputil/reverseproxy.go index 04248d5f531e23..9d8784cd2bc7a9 100644 --- a/src/net/http/httputil/reverseproxy.go +++ b/src/net/http/httputil/reverseproxy.go @@ -42,6 +42,8 @@ type ProxyRequest struct { // SetURL routes the outbound request to the scheme, host, and base path // provided in target. If the target's path is "/base" and the incoming // request was for "/dir", the target request will be for "/base/dir". +// To route requests without joining the incoming path, +// set r.Out.URL directly. // // SetURL rewrites the outbound Host header to match the target's host. // To preserve the inbound request's Host header (the default behavior @@ -100,6 +102,13 @@ func (r *ProxyRequest) SetXForwarded() { // // 1xx responses are forwarded to the client if the underlying // transport supports ClientTrace.Got1xxResponse. +// +// Hop-by-hop headers (see RFC 9110, section 7.6.1), including +// Connection, Proxy-Connection, Keep-Alive, Proxy-Authenticate, +// Proxy-Authorization, TE, Trailer, Transfer-Encoding, and Upgrade, +// are removed from client requests and backend responses. +// The Rewrite function may be used to add hop-by-hop headers to the request, +// and the ModifyResponse function may be used to remove them from the response. type ReverseProxy struct { // Rewrite must be a function which modifies // the request into a new request to be sent @@ -124,36 +133,6 @@ type ReverseProxy struct { // At most one of Rewrite or Director may be set. Rewrite func(*ProxyRequest) - // Director is a function which modifies - // the request into a new request to be sent - // using Transport. Its response is then copied - // back to the original client unmodified. - // Director must not access the provided Request - // after returning. - // - // By default, the X-Forwarded-For header is set to the - // value of the client IP address. If an X-Forwarded-For - // header already exists, the client IP is appended to the - // existing values. As a special case, if the header - // exists in the Request.Header map but has a nil value - // (such as when set by the Director func), the X-Forwarded-For - // header is not modified. - // - // To prevent IP spoofing, be sure to delete any pre-existing - // X-Forwarded-For header coming from the client or - // an untrusted proxy. - // - // Hop-by-hop headers are removed from the request after - // Director returns, which can remove headers added by - // Director. Use a Rewrite function instead to ensure - // modifications to the request are preserved. - // - // Unparsable query parameters are removed from the outbound - // request if Request.Form is set after Director returns. - // - // At most one of Rewrite or Director may be set. - Director func(*http.Request) - // The transport used to perform proxy requests. // If nil, http.DefaultTransport is used. Transport http.RoundTripper @@ -186,6 +165,10 @@ type ReverseProxy struct { // If the backend is unreachable, the optional ErrorHandler is // called without any call to ModifyResponse. // + // Hop-by-hop headers are removed from the response before + // calling ModifyResponse. ModifyResponse may need to remove + // additional headers to fit its deployment model, such as Alt-Svc. + // // If ModifyResponse returns an error, ErrorHandler is called // with its error value. If ErrorHandler is nil, its default // implementation is used. @@ -197,6 +180,88 @@ type ReverseProxy struct { // If nil, the default is to log the provided error and return // a 502 Status Bad Gateway response. ErrorHandler func(http.ResponseWriter, *http.Request, error) + + // Director is deprecated. Use Rewrite instead. + // + // This function is insecure: + // + // - Hop-by-hop headers are removed from the request after Director + // returns, which can remove headers added by Director. + // A client can designate headers as hop-by-hop by listing them + // in the Connection header, so this permits a malicious client + // to remove any headers that may be added by Director. + // + // - X-Forwarded-For, X-Forwarded-Host, and X-Forwarded-Proto + // headers in inbound requests are preserved by default, + // which can permit IP spoofing if the Director function is + // not careful to remove these headers. + // + // Rewrite addresses these issues. + // + // As an example of converting a Director function to Rewrite: + // + // // ReverseProxy with a Director function. + // proxy := &httputil.ReverseProxy{ + // Director: func(req *http.Request) { + // req.URL.Scheme = "https" + // req.URL.Host = proxyHost + // + // // A malicious client can remove this header. + // req.Header.Set("Some-Header", "some-header-value") + // + // // X-Forwarded-* headers sent by the client are preserved, + // // since Director did not remove them. + // }, + // } + // + // // ReverseProxy with a Rewrite function. + // proxy := &httputil.ReverseProxy{ + // Rewrite: func(preq *httputil.ProxyRequest) { + // // See also ProxyRequest.SetURL. + // preq.Out.URL.Scheme = "https" + // preq.Out.URL.Host = proxyHost + // + // // This header cannot be affected by a malicious client. + // preq.Out.Header.Set("Some-Header", "some-header-value") + // + // // X-Forwarded- headers sent by the client have been + // // removed from preq.Out. + // // ProxyRequest.SetXForwarded optionally adds new ones. + // preq.SetXForwarded() + // }, + // } + // + // Director is a function which modifies + // the request into a new request to be sent + // using Transport. Its response is then copied + // back to the original client unmodified. + // Director must not access the provided Request + // after returning. + // + // By default, the X-Forwarded-For header is set to the + // value of the client IP address. If an X-Forwarded-For + // header already exists, the client IP is appended to the + // existing values. As a special case, if the header + // exists in the Request.Header map but has a nil value + // (such as when set by the Director func), the X-Forwarded-For + // header is not modified. + // + // To prevent IP spoofing, be sure to delete any pre-existing + // X-Forwarded-For header coming from the client or + // an untrusted proxy. + // + // Hop-by-hop headers are removed from the request after + // Director returns, which can remove headers added by + // Director. Use a Rewrite function instead to ensure + // modifications to the request are preserved. + // + // Unparsable query parameters are removed from the outbound + // request if Request.Form is set after Director returns. + // + // At most one of Rewrite or Director may be set. + // + // Deprecated: Use Rewrite instead. + Director func(*http.Request) } // A BufferPool is an interface for getting and returning temporary @@ -246,6 +311,10 @@ func joinURLPath(a, b *url.URL) (path, rawpath string) { // // NewSingleHostReverseProxy does not rewrite the Host header. // +// For backwards compatibility reasons, NewSingleHostReverseProxy +// returns a ReverseProxy using the deprecated Director function. +// This proxy preserves X-Forwarded-* headers sent by the client. +// // To customize the ReverseProxy behavior beyond what // NewSingleHostReverseProxy provides, use ReverseProxy directly // with a Rewrite function. The ProxyRequest SetURL method @@ -577,7 +646,7 @@ func shouldPanicOnCopyError(req *http.Request) bool { func removeHopByHopHeaders(h http.Header) { // RFC 7230, section 6.1: Remove headers listed in the "Connection" header. for _, f := range h["Connection"] { - for _, sf := range strings.Split(f, ",") { + for sf := range strings.SplitSeq(f, ",") { if sf = textproto.TrimString(sf); sf != "" { h.Del(sf) } @@ -739,6 +808,7 @@ func (p *ReverseProxy) handleUpgradeResponse(rw http.ResponseWriter, req *http.R resUpType := upgradeType(res.Header) if !ascii.IsPrint(resUpType) { // We know reqUpType is ASCII, it's checked by the caller. p.getErrorHandler()(rw, req, fmt.Errorf("backend tried to switch to invalid protocol %q", resUpType)) + return } if !ascii.EqualFold(reqUpType, resUpType) { p.getErrorHandler()(rw, req, fmt.Errorf("backend tried to switch protocol %q when %q was requested", resUpType, reqUpType)) @@ -792,9 +862,17 @@ func (p *ReverseProxy) handleUpgradeResponse(rw http.ResponseWriter, req *http.R spc := switchProtocolCopier{user: conn, backend: backConn} go spc.copyToBackend(errc) go spc.copyFromBackend(errc) - <-errc + + // Wait until both copy functions have sent on the error channel, + // or until one fails. + err := <-errc + if err == nil { + err = <-errc + } } +var errCopyDone = errors.New("hijacked connection copy complete") + // switchProtocolCopier exists so goroutines proxying data back and // forth have nice names in stacks. type switchProtocolCopier struct { @@ -802,13 +880,33 @@ type switchProtocolCopier struct { } func (c switchProtocolCopier) copyFromBackend(errc chan<- error) { - _, err := io.Copy(c.user, c.backend) - errc <- err + if _, err := io.Copy(c.user, c.backend); err != nil { + errc <- err + return + } + + // backend conn has reached EOF so propogate close write to user conn + if wc, ok := c.user.(interface{ CloseWrite() error }); ok { + errc <- wc.CloseWrite() + return + } + + errc <- errCopyDone } func (c switchProtocolCopier) copyToBackend(errc chan<- error) { - _, err := io.Copy(c.backend, c.user) - errc <- err + if _, err := io.Copy(c.backend, c.user); err != nil { + errc <- err + return + } + + // user conn has reached EOF so propogate close write to backend conn + if wc, ok := c.backend.(interface{ CloseWrite() error }); ok { + errc <- wc.CloseWrite() + return + } + + errc <- errCopyDone } func cleanQueryParams(s string) string { diff --git a/src/net/http/httputil/reverseproxy_test.go b/src/net/http/httputil/reverseproxy_test.go index 67d0e50593a2eb..62c93fb261a6e5 100644 --- a/src/net/http/httputil/reverseproxy_test.go +++ b/src/net/http/httputil/reverseproxy_test.go @@ -14,6 +14,7 @@ import ( "fmt" "io" "log" + "net" "net/http" "net/http/httptest" "net/http/httptrace" @@ -22,6 +23,7 @@ import ( "net/url" "os" "reflect" + "runtime" "slices" "strconv" "strings" @@ -137,6 +139,7 @@ func TestReverseProxy(t *testing.T) { if g, e := res.Trailer.Get("X-Unannounced-Trailer"), "unannounced_trailer_value"; g != e { t.Errorf("Trailer(X-Unannounced-Trailer) = %q ; want %q", g, e) } + res.Body.Close() // Test that a backend failing to be reached or one which doesn't return // a response results in a StatusBadGateway. @@ -196,7 +199,7 @@ func TestReverseProxyStripHeadersPresentInConnection(t *testing.T) { c := r.Header["Connection"] var cf []string for _, f := range c { - for _, sf := range strings.Split(f, ",") { + for sf := range strings.SplitSeq(f, ",") { if sf = strings.TrimSpace(sf); sf != "" { cf = append(cf, sf) } @@ -328,6 +331,7 @@ func TestXForwardedFor(t *testing.T) { if err != nil { t.Fatalf("Get: %v", err) } + defer res.Body.Close() if g, e := res.StatusCode, backendStatus; g != e { t.Errorf("got res.StatusCode %d; expected %d", g, e) } @@ -801,6 +805,7 @@ func TestReverseProxy_Post(t *testing.T) { if err != nil { t.Fatalf("Do: %v", err) } + defer res.Body.Close() if g, e := res.StatusCode, backendStatus; g != e { t.Errorf("got res.StatusCode %d; expected %d", g, e) } @@ -1548,6 +1553,202 @@ func TestReverseProxyWebSocketCancellation(t *testing.T) { } } +func TestReverseProxyWebSocketHalfTCP(t *testing.T) { + // Issue #35892: support TCP half-close when HTTP is upgraded in the ReverseProxy. + // Specifically testing: + // - the communication through the reverse proxy when the client or server closes + // either the read or write streams + // - that closing the write stream is propagated through the proxy and results in reading + // EOF at the other end of the connection + + switch runtime.GOOS { + case "plan9": + t.Skipf("not supported on %s", runtime.GOOS) + } + + mustRead := func(t *testing.T, conn *net.TCPConn, msg string) { + b := make([]byte, len(msg)) + if _, err := conn.Read(b); err != nil { + t.Errorf("failed to read: %v", err) + } + + if got, want := string(b), msg; got != want { + t.Errorf("got %#q, want %#q", got, want) + } + } + + mustReadError := func(t *testing.T, conn *net.TCPConn, e error) { + b := make([]byte, 1) + if _, err := conn.Read(b); !errors.Is(err, e) { + t.Errorf("failed to read error: %v", err) + } + } + + mustWrite := func(t *testing.T, conn *net.TCPConn, msg string) { + if _, err := conn.Write([]byte(msg)); err != nil { + t.Errorf("failed to write: %v", err) + } + } + + mustCloseRead := func(t *testing.T, conn *net.TCPConn) { + if err := conn.CloseRead(); err != nil { + t.Errorf("failed to CloseRead: %v", err) + } + } + + mustCloseWrite := func(t *testing.T, conn *net.TCPConn) { + if err := conn.CloseWrite(); err != nil { + t.Errorf("failed to CloseWrite: %v", err) + } + } + + tests := map[string]func(t *testing.T, cli, srv *net.TCPConn){ + "server close read": func(t *testing.T, cli, srv *net.TCPConn) { + mustCloseRead(t, srv) + mustWrite(t, srv, "server sends") + mustRead(t, cli, "server sends") + }, + "server close write": func(t *testing.T, cli, srv *net.TCPConn) { + mustCloseWrite(t, srv) + mustWrite(t, cli, "client sends") + mustRead(t, srv, "client sends") + mustReadError(t, cli, io.EOF) + }, + "client close read": func(t *testing.T, cli, srv *net.TCPConn) { + mustCloseRead(t, cli) + mustWrite(t, cli, "client sends") + mustRead(t, srv, "client sends") + }, + "client close write": func(t *testing.T, cli, srv *net.TCPConn) { + mustCloseWrite(t, cli) + mustWrite(t, srv, "server sends") + mustRead(t, cli, "server sends") + mustReadError(t, srv, io.EOF) + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + var srv *net.TCPConn + + backendServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if g, ws := upgradeType(r.Header), "websocket"; g != ws { + t.Fatalf("Unexpected upgrade type %q, want %q", g, ws) + } + + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + conn.Close() + t.Fatalf("hijack failed: %v", err) + } + + var ok bool + if srv, ok = conn.(*net.TCPConn); !ok { + conn.Close() + t.Fatal("conn is not a TCPConn") + } + + upgradeMsg := "HTTP/1.1 101 Switching Protocols\r\nConnection: upgrade\r\nUpgrade: WebSocket\r\n\r\n" + if _, err := io.WriteString(srv, upgradeMsg); err != nil { + srv.Close() + t.Fatalf("backend upgrade failed: %v", err) + } + })) + defer backendServer.Close() + + backendURL, _ := url.Parse(backendServer.URL) + rproxy := NewSingleHostReverseProxy(backendURL) + rproxy.ErrorLog = log.New(io.Discard, "", 0) // quiet for tests + frontendProxy := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + rproxy.ServeHTTP(rw, req) + })) + defer frontendProxy.Close() + + frontendURL, _ := url.Parse(frontendProxy.URL) + addr, err := net.ResolveTCPAddr("tcp", frontendURL.Host) + if err != nil { + t.Fatalf("failed to resolve TCP address: %v", err) + } + cli, err := net.DialTCP("tcp", nil, addr) + if err != nil { + t.Fatalf("failed to dial TCP address: %v", err) + } + defer cli.Close() + + req, _ := http.NewRequest("GET", frontendProxy.URL, nil) + req.Header.Set("Connection", "Upgrade") + req.Header.Set("Upgrade", "websocket") + if err := req.Write(cli); err != nil { + t.Fatalf("failed to write request: %v", err) + } + + br := bufio.NewReader(cli) + resp, err := http.ReadResponse(br, &http.Request{Method: "GET"}) + if err != nil { + t.Fatalf("failed to read response: %v", err) + } + if resp.StatusCode != 101 { + t.Fatalf("status code not 101: %v", resp.StatusCode) + } + if strings.ToLower(resp.Header.Get("Upgrade")) != "websocket" || + strings.ToLower(resp.Header.Get("Connection")) != "upgrade" { + t.Fatalf("frontend upgrade failed") + } + defer srv.Close() + + test(t, cli, srv) + }) + } +} + +func TestReverseProxyUpgradeNoCloseWrite(t *testing.T) { + // The backend hijacks the connection, + // reads all data from the client, + // and returns. + backendDone := make(chan struct{}) + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Connection", "upgrade") + w.Header().Set("Upgrade", "u") + w.WriteHeader(101) + conn, _, err := http.NewResponseController(w).Hijack() + if err != nil { + t.Errorf("Hijack: %v", err) + } + io.Copy(io.Discard, conn) + close(backendDone) + })) + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + + // The proxy includes a ModifyResponse function which replaces the response body + // with its own wrapper, dropping the original body's CloseWrite method. + proxyHandler := NewSingleHostReverseProxy(backendURL) + proxyHandler.ModifyResponse = func(resp *http.Response) error { + type readWriteCloserOnly struct { + io.ReadWriteCloser + } + resp.Body = readWriteCloserOnly{resp.Body.(io.ReadWriteCloser)} + return nil + } + frontend := httptest.NewServer(proxyHandler) + defer frontend.Close() + + // The client sends a request and closes the connection. + req, _ := http.NewRequest("GET", frontend.URL, nil) + req.Header.Set("Connection", "upgrade") + req.Header.Set("Upgrade", "u") + resp, err := frontend.Client().Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + + // We expect that the client's closure of the connection is propagated to the backend. + <-backendDone +} + func TestUnannouncedTrailer(t *testing.T) { backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) @@ -1571,7 +1772,7 @@ func TestUnannouncedTrailer(t *testing.T) { } io.ReadAll(res.Body) - + res.Body.Close() if g, w := res.Trailer.Get("X-Unannounced-Trailer"), "unannounced_trailer_value"; g != w { t.Errorf("Trailer(X-Unannounced-Trailer) = %q; want %q", g, w) } @@ -1903,10 +2104,56 @@ func testReverseProxyQueryParameterSmuggling(t *testing.T, wantCleanQuery bool, } } +// Issue #72954: We should not call WriteHeader on a ResponseWriter after hijacking +// the connection. +func TestReverseProxyHijackCopyError(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Upgrade", "someproto") + w.WriteHeader(http.StatusSwitchingProtocols) + })) + defer backend.Close() + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + proxyHandler := &ReverseProxy{ + Rewrite: func(r *ProxyRequest) { + r.SetURL(backendURL) + }, + ModifyResponse: func(resp *http.Response) error { + resp.Body = &testReadWriteCloser{ + read: func([]byte) (int, error) { + return 0, errors.New("read error") + }, + } + return nil + }, + } + + hijacked := false + rw := &testResponseWriter{ + writeHeader: func(statusCode int) { + if hijacked { + t.Errorf("WriteHeader(%v) called after Hijack", statusCode) + } + }, + hijack: func() (net.Conn, *bufio.ReadWriter, error) { + hijacked = true + cli, srv := net.Pipe() + go io.Copy(io.Discard, cli) + return srv, bufio.NewReadWriter(bufio.NewReader(srv), bufio.NewWriter(srv)), nil + }, + } + req, _ := http.NewRequest("GET", "http://example.tld/", nil) + req.Header.Set("Upgrade", "someproto") + proxyHandler.ServeHTTP(rw, req) +} + type testResponseWriter struct { h http.Header writeHeader func(int) write func([]byte) (int, error) + hijack func() (net.Conn, *bufio.ReadWriter, error) } func (rw *testResponseWriter) Header() http.Header { @@ -1928,3 +2175,37 @@ func (rw *testResponseWriter) Write(p []byte) (int, error) { } return len(p), nil } + +func (rw *testResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + if rw.hijack != nil { + return rw.hijack() + } + return nil, nil, errors.ErrUnsupported +} + +type testReadWriteCloser struct { + read func([]byte) (int, error) + write func([]byte) (int, error) + close func() error +} + +func (rc *testReadWriteCloser) Read(p []byte) (int, error) { + if rc.read != nil { + return rc.read(p) + } + return 0, io.EOF +} + +func (rc *testReadWriteCloser) Write(p []byte) (int, error) { + if rc.write != nil { + return rc.write(p) + } + return len(p), nil +} + +func (rc *testReadWriteCloser) Close() error { + if rc.close != nil { + return rc.close() + } + return nil +} diff --git a/src/net/http/internal/chunked.go b/src/net/http/internal/chunked.go index 196b5d892589ab..0b08a97a0831ec 100644 --- a/src/net/http/internal/chunked.go +++ b/src/net/http/internal/chunked.go @@ -164,6 +164,19 @@ func readChunkLine(b *bufio.Reader) ([]byte, error) { } return nil, err } + + // RFC 9112 permits parsers to accept a bare \n as a line ending in headers, + // but not in chunked encoding lines. See https://www.rfc-editor.org/errata/eid7633, + // which explicitly rejects a clarification permitting \n as a chunk terminator. + // + // Verify that the line ends in a CRLF, and that no CRs appear before the end. + if idx := bytes.IndexByte(p, '\r'); idx == -1 { + return nil, errors.New("chunked line ends with bare LF") + } else if idx != len(p)-2 { + return nil, errors.New("invalid CR in chunked line") + } + p = p[:len(p)-2] // trim CRLF + if len(p) >= maxLineLength { return nil, ErrLineTooLong } @@ -171,14 +184,14 @@ func readChunkLine(b *bufio.Reader) ([]byte, error) { } func trimTrailingWhitespace(b []byte) []byte { - for len(b) > 0 && isASCIISpace(b[len(b)-1]) { + for len(b) > 0 && isOWS(b[len(b)-1]) { b = b[:len(b)-1] } return b } -func isASCIISpace(b byte) bool { - return b == ' ' || b == '\t' || b == '\n' || b == '\r' +func isOWS(b byte) bool { + return b == ' ' || b == '\t' } var semi = []byte(";") diff --git a/src/net/http/internal/chunked_test.go b/src/net/http/internal/chunked_test.go index af79711781a7ed..312f1734a6385c 100644 --- a/src/net/http/internal/chunked_test.go +++ b/src/net/http/internal/chunked_test.go @@ -280,6 +280,33 @@ func TestChunkReaderByteAtATime(t *testing.T) { } } +func TestChunkInvalidInputs(t *testing.T) { + for _, test := range []struct { + name string + b string + }{{ + name: "bare LF in chunk size", + b: "1\na\r\n0\r\n", + }, { + name: "extra LF in chunk size", + b: "1\r\r\na\r\n0\r\n", + }, { + name: "bare LF in chunk data", + b: "1\r\na\n0\r\n", + }, { + name: "bare LF in chunk extension", + b: "1;\na\r\n0\r\n", + }} { + t.Run(test.name, func(t *testing.T) { + r := NewChunkedReader(strings.NewReader(test.b)) + got, err := io.ReadAll(r) + if err == nil { + t.Fatalf("unexpectedly parsed invalid chunked data:\n%q", got) + } + }) + } +} + type funcReader struct { f func(iteration int) ([]byte, error) i int diff --git a/src/net/http/internal/httpcommon/httpcommon.go b/src/net/http/internal/httpcommon/httpcommon.go new file mode 100644 index 00000000000000..e3f8ec79e094d3 --- /dev/null +++ b/src/net/http/internal/httpcommon/httpcommon.go @@ -0,0 +1,618 @@ +// Code generated by golang.org/x/tools/cmd/bundle. DO NOT EDIT. +//go:generate bundle -prefix= -o=httpcommon.go golang.org/x/net/internal/httpcommon + +package httpcommon + +import ( + "context" + "errors" + "fmt" + "net/http/httptrace" + "net/textproto" + "net/url" + "sort" + "strconv" + "strings" + "sync" + + "golang.org/x/net/http/httpguts" + "golang.org/x/net/http2/hpack" +) + +// The HTTP protocols are defined in terms of ASCII, not Unicode. This file +// contains helper functions which may use Unicode-aware functions which would +// otherwise be unsafe and could introduce vulnerabilities if used improperly. + +// asciiEqualFold is strings.EqualFold, ASCII only. It reports whether s and t +// are equal, ASCII-case-insensitively. +func asciiEqualFold(s, t string) bool { + if len(s) != len(t) { + return false + } + for i := 0; i < len(s); i++ { + if lower(s[i]) != lower(t[i]) { + return false + } + } + return true +} + +// lower returns the ASCII lowercase version of b. +func lower(b byte) byte { + if 'A' <= b && b <= 'Z' { + return b + ('a' - 'A') + } + return b +} + +// isASCIIPrint returns whether s is ASCII and printable according to +// https://tools.ietf.org/html/rfc20#section-4.2. +func isASCIIPrint(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] < ' ' || s[i] > '~' { + return false + } + } + return true +} + +// asciiToLower returns the lowercase version of s if s is ASCII and printable, +// and whether or not it was. +func asciiToLower(s string) (lower string, ok bool) { + if !isASCIIPrint(s) { + return "", false + } + return strings.ToLower(s), true +} + +var ( + commonBuildOnce sync.Once + commonLowerHeader map[string]string // Go-Canonical-Case -> lower-case + commonCanonHeader map[string]string // lower-case -> Go-Canonical-Case +) + +func buildCommonHeaderMapsOnce() { + commonBuildOnce.Do(buildCommonHeaderMaps) +} + +func buildCommonHeaderMaps() { + common := []string{ + "accept", + "accept-charset", + "accept-encoding", + "accept-language", + "accept-ranges", + "age", + "access-control-allow-credentials", + "access-control-allow-headers", + "access-control-allow-methods", + "access-control-allow-origin", + "access-control-expose-headers", + "access-control-max-age", + "access-control-request-headers", + "access-control-request-method", + "allow", + "authorization", + "cache-control", + "content-disposition", + "content-encoding", + "content-language", + "content-length", + "content-location", + "content-range", + "content-type", + "cookie", + "date", + "etag", + "expect", + "expires", + "from", + "host", + "if-match", + "if-modified-since", + "if-none-match", + "if-unmodified-since", + "last-modified", + "link", + "location", + "max-forwards", + "origin", + "proxy-authenticate", + "proxy-authorization", + "range", + "referer", + "refresh", + "retry-after", + "server", + "set-cookie", + "strict-transport-security", + "trailer", + "transfer-encoding", + "user-agent", + "vary", + "via", + "www-authenticate", + "x-forwarded-for", + "x-forwarded-proto", + } + commonLowerHeader = make(map[string]string, len(common)) + commonCanonHeader = make(map[string]string, len(common)) + for _, v := range common { + chk := textproto.CanonicalMIMEHeaderKey(v) + commonLowerHeader[chk] = v + commonCanonHeader[v] = chk + } +} + +// LowerHeader returns the lowercase form of a header name, +// used on the wire for HTTP/2 and HTTP/3 requests. +func LowerHeader(v string) (lower string, ascii bool) { + buildCommonHeaderMapsOnce() + if s, ok := commonLowerHeader[v]; ok { + return s, true + } + return asciiToLower(v) +} + +// CanonicalHeader canonicalizes a header name. (For example, "host" becomes "Host".) +func CanonicalHeader(v string) string { + buildCommonHeaderMapsOnce() + if s, ok := commonCanonHeader[v]; ok { + return s + } + return textproto.CanonicalMIMEHeaderKey(v) +} + +// CachedCanonicalHeader returns the canonical form of a well-known header name. +func CachedCanonicalHeader(v string) (string, bool) { + buildCommonHeaderMapsOnce() + s, ok := commonCanonHeader[v] + return s, ok +} + +var ( + ErrRequestHeaderListSize = errors.New("request header list larger than peer's advertised limit") +) + +// Request is a subset of http.Request. +// It'd be simpler to pass an *http.Request, of course, but we can't depend on net/http +// without creating a dependency cycle. +type Request struct { + URL *url.URL + Method string + Host string + Header map[string][]string + Trailer map[string][]string + ActualContentLength int64 // 0 means 0, -1 means unknown +} + +// EncodeHeadersParam is parameters to EncodeHeaders. +type EncodeHeadersParam struct { + Request Request + + // AddGzipHeader indicates that an "accept-encoding: gzip" header should be + // added to the request. + AddGzipHeader bool + + // PeerMaxHeaderListSize, when non-zero, is the peer's MAX_HEADER_LIST_SIZE setting. + PeerMaxHeaderListSize uint64 + + // DefaultUserAgent is the User-Agent header to send when the request + // neither contains a User-Agent nor disables it. + DefaultUserAgent string +} + +// EncodeHeadersResult is the result of EncodeHeaders. +type EncodeHeadersResult struct { + HasBody bool + HasTrailers bool +} + +// EncodeHeaders constructs request headers common to HTTP/2 and HTTP/3. +// It validates a request and calls headerf with each pseudo-header and header +// for the request. +// The headerf function is called with the validated, canonicalized header name. +func EncodeHeaders(ctx context.Context, param EncodeHeadersParam, headerf func(name, value string)) (res EncodeHeadersResult, _ error) { + req := param.Request + + // Check for invalid connection-level headers. + if err := checkConnHeaders(req.Header); err != nil { + return res, err + } + + if req.URL == nil { + return res, errors.New("Request.URL is nil") + } + + host := req.Host + if host == "" { + host = req.URL.Host + } + host, err := httpguts.PunycodeHostPort(host) + if err != nil { + return res, err + } + if !httpguts.ValidHostHeader(host) { + return res, errors.New("invalid Host header") + } + + // isNormalConnect is true if this is a non-extended CONNECT request. + isNormalConnect := false + var protocol string + if vv := req.Header[":protocol"]; len(vv) > 0 { + protocol = vv[0] + } + if req.Method == "CONNECT" && protocol == "" { + isNormalConnect = true + } else if protocol != "" && req.Method != "CONNECT" { + return res, errors.New("invalid :protocol header in non-CONNECT request") + } + + // Validate the path, except for non-extended CONNECT requests which have no path. + var path string + if !isNormalConnect { + path = req.URL.RequestURI() + if !validPseudoPath(path) { + orig := path + path = strings.TrimPrefix(path, req.URL.Scheme+"://"+host) + if !validPseudoPath(path) { + if req.URL.Opaque != "" { + return res, fmt.Errorf("invalid request :path %q from URL.Opaque = %q", orig, req.URL.Opaque) + } else { + return res, fmt.Errorf("invalid request :path %q", orig) + } + } + } + } + + // Check for any invalid headers+trailers and return an error before we + // potentially pollute our hpack state. (We want to be able to + // continue to reuse the hpack encoder for future requests) + if err := validateHeaders(req.Header); err != "" { + return res, fmt.Errorf("invalid HTTP header %s", err) + } + if err := validateHeaders(req.Trailer); err != "" { + return res, fmt.Errorf("invalid HTTP trailer %s", err) + } + + trailers, err := commaSeparatedTrailers(req.Trailer) + if err != nil { + return res, err + } + + enumerateHeaders := func(f func(name, value string)) { + // 8.1.2.3 Request Pseudo-Header Fields + // The :path pseudo-header field includes the path and query parts of the + // target URI (the path-absolute production and optionally a '?' character + // followed by the query production, see Sections 3.3 and 3.4 of + // [RFC3986]). + f(":authority", host) + m := req.Method + if m == "" { + m = "GET" + } + f(":method", m) + if !isNormalConnect { + f(":path", path) + f(":scheme", req.URL.Scheme) + } + if protocol != "" { + f(":protocol", protocol) + } + if trailers != "" { + f("trailer", trailers) + } + + var didUA bool + for k, vv := range req.Header { + if asciiEqualFold(k, "host") || asciiEqualFold(k, "content-length") { + // Host is :authority, already sent. + // Content-Length is automatic, set below. + continue + } else if asciiEqualFold(k, "connection") || + asciiEqualFold(k, "proxy-connection") || + asciiEqualFold(k, "transfer-encoding") || + asciiEqualFold(k, "upgrade") || + asciiEqualFold(k, "keep-alive") { + // Per 8.1.2.2 Connection-Specific Header + // Fields, don't send connection-specific + // fields. We have already checked if any + // are error-worthy so just ignore the rest. + continue + } else if asciiEqualFold(k, "user-agent") { + // Match Go's http1 behavior: at most one + // User-Agent. If set to nil or empty string, + // then omit it. Otherwise if not mentioned, + // include the default (below). + didUA = true + if len(vv) < 1 { + continue + } + vv = vv[:1] + if vv[0] == "" { + continue + } + } else if asciiEqualFold(k, "cookie") { + // Per 8.1.2.5 To allow for better compression efficiency, the + // Cookie header field MAY be split into separate header fields, + // each with one or more cookie-pairs. + for _, v := range vv { + for { + p := strings.IndexByte(v, ';') + if p < 0 { + break + } + f("cookie", v[:p]) + p++ + // strip space after semicolon if any. + for p+1 <= len(v) && v[p] == ' ' { + p++ + } + v = v[p:] + } + if len(v) > 0 { + f("cookie", v) + } + } + continue + } else if k == ":protocol" { + // :protocol pseudo-header was already sent above. + continue + } + + for _, v := range vv { + f(k, v) + } + } + if shouldSendReqContentLength(req.Method, req.ActualContentLength) { + f("content-length", strconv.FormatInt(req.ActualContentLength, 10)) + } + if param.AddGzipHeader { + f("accept-encoding", "gzip") + } + if !didUA { + f("user-agent", param.DefaultUserAgent) + } + } + + // Do a first pass over the headers counting bytes to ensure + // we don't exceed cc.peerMaxHeaderListSize. This is done as a + // separate pass before encoding the headers to prevent + // modifying the hpack state. + if param.PeerMaxHeaderListSize > 0 { + hlSize := uint64(0) + enumerateHeaders(func(name, value string) { + hf := hpack.HeaderField{Name: name, Value: value} + hlSize += uint64(hf.Size()) + }) + + if hlSize > param.PeerMaxHeaderListSize { + return res, ErrRequestHeaderListSize + } + } + + trace := httptrace.ContextClientTrace(ctx) + + // Header list size is ok. Write the headers. + enumerateHeaders(func(name, value string) { + name, ascii := LowerHeader(name) + if !ascii { + // Skip writing invalid headers. Per RFC 7540, Section 8.1.2, header + // field names have to be ASCII characters (just as in HTTP/1.x). + return + } + + headerf(name, value) + + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField(name, []string{value}) + } + }) + + res.HasBody = req.ActualContentLength != 0 + res.HasTrailers = trailers != "" + return res, nil +} + +// IsRequestGzip reports whether we should add an Accept-Encoding: gzip header +// for a request. +func IsRequestGzip(method string, header map[string][]string, disableCompression bool) bool { + // TODO(bradfitz): this is a copy of the logic in net/http. Unify somewhere? + if !disableCompression && + len(header["Accept-Encoding"]) == 0 && + len(header["Range"]) == 0 && + method != "HEAD" { + // Request gzip only, not deflate. Deflate is ambiguous and + // not as universally supported anyway. + // See: https://zlib.net/zlib_faq.html#faq39 + // + // Note that we don't request this for HEAD requests, + // due to a bug in nginx: + // http://trac.nginx.org/nginx/ticket/358 + // https://golang.org/issue/5522 + // + // We don't request gzip if the request is for a range, since + // auto-decoding a portion of a gzipped document will just fail + // anyway. See https://golang.org/issue/8923 + return true + } + return false +} + +// checkConnHeaders checks whether req has any invalid connection-level headers. +// +// https://www.rfc-editor.org/rfc/rfc9114.html#section-4.2-3 +// https://www.rfc-editor.org/rfc/rfc9113.html#section-8.2.2-1 +// +// Certain headers are special-cased as okay but not transmitted later. +// For example, we allow "Transfer-Encoding: chunked", but drop the header when encoding. +func checkConnHeaders(h map[string][]string) error { + if vv := h["Upgrade"]; len(vv) > 0 && (vv[0] != "" && vv[0] != "chunked") { + return fmt.Errorf("invalid Upgrade request header: %q", vv) + } + if vv := h["Transfer-Encoding"]; len(vv) > 0 && (len(vv) > 1 || vv[0] != "" && vv[0] != "chunked") { + return fmt.Errorf("invalid Transfer-Encoding request header: %q", vv) + } + if vv := h["Connection"]; len(vv) > 0 && (len(vv) > 1 || vv[0] != "" && !asciiEqualFold(vv[0], "close") && !asciiEqualFold(vv[0], "keep-alive")) { + return fmt.Errorf("invalid Connection request header: %q", vv) + } + return nil +} + +func commaSeparatedTrailers(trailer map[string][]string) (string, error) { + keys := make([]string, 0, len(trailer)) + for k := range trailer { + k = CanonicalHeader(k) + switch k { + case "Transfer-Encoding", "Trailer", "Content-Length": + return "", fmt.Errorf("invalid Trailer key %q", k) + } + keys = append(keys, k) + } + if len(keys) > 0 { + sort.Strings(keys) + return strings.Join(keys, ","), nil + } + return "", nil +} + +// validPseudoPath reports whether v is a valid :path pseudo-header +// value. It must be either: +// +// - a non-empty string starting with '/' +// - the string '*', for OPTIONS requests. +// +// For now this is only used a quick check for deciding when to clean +// up Opaque URLs before sending requests from the Transport. +// See golang.org/issue/16847 +// +// We used to enforce that the path also didn't start with "//", but +// Google's GFE accepts such paths and Chrome sends them, so ignore +// that part of the spec. See golang.org/issue/19103. +func validPseudoPath(v string) bool { + return (len(v) > 0 && v[0] == '/') || v == "*" +} + +func validateHeaders(hdrs map[string][]string) string { + for k, vv := range hdrs { + if !httpguts.ValidHeaderFieldName(k) && k != ":protocol" { + return fmt.Sprintf("name %q", k) + } + for _, v := range vv { + if !httpguts.ValidHeaderFieldValue(v) { + // Don't include the value in the error, + // because it may be sensitive. + return fmt.Sprintf("value for header %q", k) + } + } + } + return "" +} + +// shouldSendReqContentLength reports whether we should send +// a "content-length" request header. This logic is basically a copy of the net/http +// transferWriter.shouldSendContentLength. +// The contentLength is the corrected contentLength (so 0 means actually 0, not unknown). +// -1 means unknown. +func shouldSendReqContentLength(method string, contentLength int64) bool { + if contentLength > 0 { + return true + } + if contentLength < 0 { + return false + } + // For zero bodies, whether we send a content-length depends on the method. + // It also kinda doesn't matter for http2 either way, with END_STREAM. + switch method { + case "POST", "PUT", "PATCH": + return true + default: + return false + } +} + +// ServerRequestParam is parameters to NewServerRequest. +type ServerRequestParam struct { + Method string + Scheme, Authority, Path string + Protocol string + Header map[string][]string +} + +// ServerRequestResult is the result of NewServerRequest. +type ServerRequestResult struct { + // Various http.Request fields. + URL *url.URL + RequestURI string + Trailer map[string][]string + + NeedsContinue bool // client provided an "Expect: 100-continue" header + + // If the request should be rejected, this is a short string suitable for passing + // to the http2 package's CountError function. + // It might be a bit odd to return errors this way rather than returning an error, + // but this ensures we don't forget to include a CountError reason. + InvalidReason string +} + +func NewServerRequest(rp ServerRequestParam) ServerRequestResult { + needsContinue := httpguts.HeaderValuesContainsToken(rp.Header["Expect"], "100-continue") + if needsContinue { + delete(rp.Header, "Expect") + } + // Merge Cookie headers into one "; "-delimited value. + if cookies := rp.Header["Cookie"]; len(cookies) > 1 { + rp.Header["Cookie"] = []string{strings.Join(cookies, "; ")} + } + + // Setup Trailers + var trailer map[string][]string + for _, v := range rp.Header["Trailer"] { + for _, key := range strings.Split(v, ",") { + key = textproto.CanonicalMIMEHeaderKey(textproto.TrimString(key)) + switch key { + case "Transfer-Encoding", "Trailer", "Content-Length": + // Bogus. (copy of http1 rules) + // Ignore. + default: + if trailer == nil { + trailer = make(map[string][]string) + } + trailer[key] = nil + } + } + } + delete(rp.Header, "Trailer") + + // "':authority' MUST NOT include the deprecated userinfo subcomponent + // for "http" or "https" schemed URIs." + // https://www.rfc-editor.org/rfc/rfc9113.html#section-8.3.1-2.3.8 + if strings.IndexByte(rp.Authority, '@') != -1 && (rp.Scheme == "http" || rp.Scheme == "https") { + return ServerRequestResult{ + InvalidReason: "userinfo_in_authority", + } + } + + var url_ *url.URL + var requestURI string + if rp.Method == "CONNECT" && rp.Protocol == "" { + url_ = &url.URL{Host: rp.Authority} + requestURI = rp.Authority // mimic HTTP/1 server behavior + } else { + var err error + url_, err = url.ParseRequestURI(rp.Path) + if err != nil { + return ServerRequestResult{ + InvalidReason: "bad_path", + } + } + requestURI = rp.Path + } + + return ServerRequestResult{ + URL: url_, + NeedsContinue: needsContinue, + RequestURI: requestURI, + Trailer: trailer, + } +} diff --git a/src/net/http/internal/testcert/testcert.go b/src/net/http/internal/testcert/testcert.go index d510e791d617cb..78ce42e2282679 100644 --- a/src/net/http/internal/testcert/testcert.go +++ b/src/net/http/internal/testcert/testcert.go @@ -10,56 +10,56 @@ import "strings" // LocalhostCert is a PEM-encoded TLS cert with SAN IPs // "127.0.0.1" and "[::1]", expiring at Jan 29 16:00:00 2084 GMT. // generated from src/crypto/tls: -// go run generate_cert.go --rsa-bits 2048 --host 127.0.0.1,::1,example.com --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h +// go run generate_cert.go --rsa-bits 2048 --host 127.0.0.1,::1,example.com,*.example.com --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h var LocalhostCert = []byte(`-----BEGIN CERTIFICATE----- -MIIDOTCCAiGgAwIBAgIQSRJrEpBGFc7tNb1fb5pKFzANBgkqhkiG9w0BAQsFADAS +MIIDSDCCAjCgAwIBAgIQEP/md970HysdBTpuzDOf0DANBgkqhkiG9w0BAQsFADAS MRAwDgYDVQQKEwdBY21lIENvMCAXDTcwMDEwMTAwMDAwMFoYDzIwODQwMTI5MTYw MDAwWjASMRAwDgYDVQQKEwdBY21lIENvMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEA6Gba5tHV1dAKouAaXO3/ebDUU4rvwCUg/CNaJ2PT5xLD4N1Vcb8r -bFSW2HXKq+MPfVdwIKR/1DczEoAGf/JWQTW7EgzlXrCd3rlajEX2D73faWJekD0U -aUgz5vtrTXZ90BQL7WvRICd7FlEZ6FPOcPlumiyNmzUqtwGhO+9ad1W5BqJaRI6P -YfouNkwR6Na4TzSj5BrqUfP0FwDizKSJ0XXmh8g8G9mtwxOSN3Ru1QFc61Xyeluk -POGKBV/q6RBNklTNe0gI8usUMlYyoC7ytppNMW7X2vodAelSu25jgx2anj9fDVZu -h7AXF5+4nJS4AAt0n1lNY7nGSsdZas8PbQIDAQABo4GIMIGFMA4GA1UdDwEB/wQE +MIIBCgKCAQEAxcl69ROJdxjN+MJZnbFrYxyQooADCsJ6VDkuMyNQIix/Hk15Nk/u +FyBX1Me++aEpGmY3RIY4fUvELqT/srvAHsTXwVVSttMcY8pcAFmXSqo3x4MuUTG/ +jCX3Vftj0r3EM5M8ImY1rzA/jqTTLJg00rD+DmuDABcqQvoXw/RV8w1yTRi5BPoH +DFD/AWTt/YgMvk1l2Yq/xI8VbMUIpjBoGXxWsSevQ5i2s1mk9/yZzu0Ysp1tTlzD +qOPa4ysFjBitdXiwfxjxtv5nXqOCP5rheKO0sWLk0fetMp1OV5JSJMAJw6c2ZMkl +U2WMqAEpRjdE/vHfIuNg+yGaRRqI07NZRQIDAQABo4GXMIGUMA4GA1UdDwEB/wQE AwICpDATBgNVHSUEDDAKBggrBgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud -DgQWBBStsdjh3/JCXXYlQryOrL4Sh7BW5TAuBgNVHREEJzAlggtleGFtcGxlLmNv -bYcEfwAAAYcQAAAAAAAAAAAAAAAAAAAAATANBgkqhkiG9w0BAQsFAAOCAQEAxWGI -5NhpF3nwwy/4yB4i/CwwSpLrWUa70NyhvprUBC50PxiXav1TeDzwzLx/o5HyNwsv -cxv3HdkLW59i/0SlJSrNnWdfZ19oTcS+6PtLoVyISgtyN6DpkKpdG1cOkW3Cy2P2 -+tK/tKHRP1Y/Ra0RiDpOAmqn0gCOFGz8+lqDIor/T7MTpibL3IxqWfPrvfVRHL3B -grw/ZQTTIVjjh4JBSW3WyWgNo/ikC1lrVxzl4iPUGptxT36Cr7Zk2Bsg0XqwbOvK -5d+NTDREkSnUbie4GeutujmX3Dsx88UiV6UY/4lHJa6I5leHUNOHahRbpbWeOfs/ -WkBKOclmOV2xlTVuPw== +DgQWBBQR5QIzmacmw78ZI1C4MXw7Q0wJ1jA9BgNVHREENjA0ggtleGFtcGxlLmNv +bYINKi5leGFtcGxlLmNvbYcEfwAAAYcQAAAAAAAAAAAAAAAAAAAAATANBgkqhkiG +9w0BAQsFAAOCAQEACrRNgiioUDzxQftd0fwOa6iRRcPampZRDtuaF68yNHoNWbOu +LUwc05eOWxRq3iABGSk2xg+FXM3DDeW4HhAhCFptq7jbVZ+4Jj6HeJG9mYRatAxR +Y/dEpa0D0EHhDxxVg6UzKOXB355n0IetGE/aWvyTV9SiDs6QsaC57Q9qq1/mitx5 +2GFBoapol9L5FxCc77bztzK8CpLujkBi25Vk6GAFbl27opLfpyxkM+rX/T6MXCPO +6/YBacNZ7ff1/57Etg4i5mNA6ubCpuc4Gi9oYqCNNohftr2lkJr7REdDR6OW0lsL +rF7r4gUnKeC7mYIH1zypY7laskopiLFAfe96Kg== -----END CERTIFICATE-----`) // LocalhostKey is the private key for LocalhostCert. var LocalhostKey = []byte(testingKey(`-----BEGIN RSA TESTING KEY----- -MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDoZtrm0dXV0Aqi -4Bpc7f95sNRTiu/AJSD8I1onY9PnEsPg3VVxvytsVJbYdcqr4w99V3AgpH/UNzMS -gAZ/8lZBNbsSDOVesJ3euVqMRfYPvd9pYl6QPRRpSDPm+2tNdn3QFAvta9EgJ3sW -URnoU85w+W6aLI2bNSq3AaE771p3VbkGolpEjo9h+i42TBHo1rhPNKPkGupR8/QX -AOLMpInRdeaHyDwb2a3DE5I3dG7VAVzrVfJ6W6Q84YoFX+rpEE2SVM17SAjy6xQy -VjKgLvK2mk0xbtfa+h0B6VK7bmODHZqeP18NVm6HsBcXn7iclLgAC3SfWU1jucZK -x1lqzw9tAgMBAAECggEABWzxS1Y2wckblnXY57Z+sl6YdmLV+gxj2r8Qib7g4ZIk -lIlWR1OJNfw7kU4eryib4fc6nOh6O4AWZyYqAK6tqNQSS/eVG0LQTLTTEldHyVJL -dvBe+MsUQOj4nTndZW+QvFzbcm2D8lY5n2nBSxU5ypVoKZ1EqQzytFcLZpTN7d89 -EPj0qDyrV4NZlWAwL1AygCwnlwhMQjXEalVF1ylXwU3QzyZ/6MgvF6d3SSUlh+sq -XefuyigXw484cQQgbzopv6niMOmGP3of+yV4JQqUSb3IDmmT68XjGd2Dkxl4iPki -6ZwXf3CCi+c+i/zVEcufgZ3SLf8D99kUGE7v7fZ6AQKBgQD1ZX3RAla9hIhxCf+O -3D+I1j2LMrdjAh0ZKKqwMR4JnHX3mjQI6LwqIctPWTU8wYFECSh9klEclSdCa64s -uI/GNpcqPXejd0cAAdqHEEeG5sHMDt0oFSurL4lyud0GtZvwlzLuwEweuDtvT9cJ -Wfvl86uyO36IW8JdvUprYDctrQKBgQDycZ697qutBieZlGkHpnYWUAeImVA878sJ -w44NuXHvMxBPz+lbJGAg8Cn8fcxNAPqHIraK+kx3po8cZGQywKHUWsxi23ozHoxo -+bGqeQb9U661TnfdDspIXia+xilZt3mm5BPzOUuRqlh4Y9SOBpSWRmEhyw76w4ZP -OPxjWYAgwQKBgA/FehSYxeJgRjSdo+MWnK66tjHgDJE8bYpUZsP0JC4R9DL5oiaA -brd2fI6Y+SbyeNBallObt8LSgzdtnEAbjIH8uDJqyOmknNePRvAvR6mP4xyuR+Bv -m+Lgp0DMWTw5J9CKpydZDItc49T/mJ5tPhdFVd+am0NAQnmr1MCZ6nHxAoGABS3Y -LkaC9FdFUUqSU8+Chkd/YbOkuyiENdkvl6t2e52jo5DVc1T7mLiIrRQi4SI8N9bN -/3oJWCT+uaSLX2ouCtNFunblzWHBrhxnZzTeqVq4SLc8aESAnbslKL4i8/+vYZlN -s8xtiNcSvL+lMsOBORSXzpj/4Ot8WwTkn1qyGgECgYBKNTypzAHeLE6yVadFp3nQ -Ckq9yzvP/ib05rvgbvrne00YeOxqJ9gtTrzgh7koqJyX1L4NwdkEza4ilDWpucn0 -xiUZS4SoaJq6ZvcBYS62Yr1t8n09iG47YL8ibgtmH3L+svaotvpVxVK+d7BLevA/ -ZboOWVe3icTy64BT3OQhmg== +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDFyXr1E4l3GM34 +wlmdsWtjHJCigAMKwnpUOS4zI1AiLH8eTXk2T+4XIFfUx775oSkaZjdEhjh9S8Qu +pP+yu8AexNfBVVK20xxjylwAWZdKqjfHgy5RMb+MJfdV+2PSvcQzkzwiZjWvMD+O +pNMsmDTSsP4Oa4MAFypC+hfD9FXzDXJNGLkE+gcMUP8BZO39iAy+TWXZir/EjxVs +xQimMGgZfFaxJ69DmLazWaT3/JnO7RiynW1OXMOo49rjKwWMGK11eLB/GPG2/mde +o4I/muF4o7SxYuTR960ynU5XklIkwAnDpzZkySVTZYyoASlGN0T+8d8i42D7IZpF +GojTs1lFAgMBAAECggEAIYthUi1lFBDd5gG4Rzlu+BlBIn5JhcqkCqLEBiJIFfOr +/4yuMRrvS3bNzqWt6xJ9MSAC4ZlN/VobRLnxL/QNymoiGYUKCT3Ww8nvPpPzR9OE +sE68TUL9tJw/zZJcRMKwgvrGqSLimfq53MxxkE+kLdOc0v9C8YH8Re26mB5ZcWYa +7YFyZQpKsQYnsmu/05cMbpOQrQWhtmIqRoyn8mG/par2s3NzjtpSE9NINyz26uFc +k/3ovFJQIHkUmTS7KHD3BgY5vuCqP98HramYnOysJ0WoYgvSDNCWw3037s5CCwJT +gCKuM+Ow6liFrj83RrdKBpm5QUGjfNpYP31o+QNP4QKBgQDSrUQ2XdgtAnibAV7u +7kbxOxro0EhIKso0Y/6LbDQgcXgxLqltkmeqZgG8nC3Z793lhlSasz2snhzzooV5 +5fTy1y8ikXqjhG0nNkInFyOhsI0auE28CFoDowaQd+5cmCatpN4Grqo5PNRXxm1w +HktfPEgoP11NNCFHvvN5fEKbbQKBgQDwVlOaV20IvW3IPq7cXZyiyabouFF9eTRo +VJka1Uv+JtyvL2P0NKkjYHOdN8gRblWqxQtJoTNk020rVA4UP1heiXALy50gvj/p +hMcybPTLYSPOhAGx838KIcvGR5oskP1aUCmFbFQzGELxhJ9diVVjxUtbG2DuwPKd +tD9TLxT2OQKBgQCcdlHSjp+dzdgERmBa0ludjGfPv9/uuNizUBAbO6D690psPFtY +JQMYaemgSd1DngEOFVWADt4e9M5Lose+YCoqr+UxpxmNlyv5kzJOFcFAs/4XeglB +PHKdgNW/NVKxMc6H54l9LPr+x05sYdGlEtqnP/3W5jhEvhJ5Vjc8YiyVgQKBgQCl +zwjyrGo+42GACy7cPYE5FeIfIDqoVByB9guC5bD98JXEDu/opQQjsgFRcBCJZhOY +M0UsURiB8ROaFu13rpQq9KrmmF0ZH+g8FSzQbzcbsTLg4VXCDXmR5esOKowFPypr +Sm667BfTAGP++D5ya7MLmCv6+RKQ5XD8uEQQAaV2kQKBgAD8qeJuWIXZT0VKkQrn +nIhgtzGERF/6sZdQGW2LxTbUDWG74AfFkkEbeBfwEkCZXY/xmnYqYABhvlSex8jU +supU6Eea21esIxIub2zv/Np0ojUb6rlqTPS4Ox1E27D787EJ3VOXpriSD10vyNnZ +jel6uj2FOP9g54s+GzlSVg/T -----END RSA TESTING KEY-----`)) func testingKey(s string) string { return strings.ReplaceAll(s, "TESTING KEY", "PRIVATE KEY") } diff --git a/src/net/http/main_test.go b/src/net/http/main_test.go index 4c1832071704e2..0c58a94f20fc24 100644 --- a/src/net/http/main_test.go +++ b/src/net/http/main_test.go @@ -31,7 +31,7 @@ func TestMain(m *testing.M) { func interestingGoroutines() (gs []string) { buf := make([]byte, 2<<20) buf = buf[:runtime.Stack(buf, true)] - for _, g := range strings.Split(string(buf), "\n\n") { + for g := range strings.SplitSeq(string(buf), "\n\n") { _, stack, _ := strings.Cut(g, "\n") stack = strings.TrimSpace(stack) if stack == "" || diff --git a/src/net/http/netconn_test.go b/src/net/http/netconn_test.go new file mode 100644 index 00000000000000..52b8069f8b33e1 --- /dev/null +++ b/src/net/http/netconn_test.go @@ -0,0 +1,439 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http_test + +import ( + "bytes" + "context" + "internal/synctest" + "io" + "math" + "net" + "net/netip" + "os" + "sync" + "time" +) + +func fakeNetListen() *fakeNetListener { + li := &fakeNetListener{ + setc: make(chan struct{}, 1), + unsetc: make(chan struct{}, 1), + addr: netip.MustParseAddrPort("127.0.0.1:8000"), + locPort: 10000, + } + li.unsetc <- struct{}{} + return li +} + +type fakeNetListener struct { + setc, unsetc chan struct{} + queue []net.Conn + closed bool + addr netip.AddrPort + locPort uint16 + + onDial func() // called when making a new connection + onClose func(*fakeNetConn) // called when closing a connection + + trackConns bool // set this to record all created conns + conns []*fakeNetConn +} + +func (li *fakeNetListener) lock() { + select { + case <-li.setc: + case <-li.unsetc: + } +} + +func (li *fakeNetListener) unlock() { + if li.closed || len(li.queue) > 0 { + li.setc <- struct{}{} + } else { + li.unsetc <- struct{}{} + } +} + +func (li *fakeNetListener) connect() *fakeNetConn { + if li.onDial != nil { + li.onDial() + } + li.lock() + defer li.unlock() + locAddr := netip.AddrPortFrom(netip.AddrFrom4([4]byte{127, 0, 0, 1}), li.locPort) + li.locPort++ + c0, c1 := fakeNetPipe(li.addr, locAddr) + c0.onClose = li.onClose + c1.onClose = li.onClose + li.queue = append(li.queue, c0) + if li.trackConns { + li.conns = append(li.conns, c0) + } + return c1 +} + +func (li *fakeNetListener) Accept() (net.Conn, error) { + <-li.setc + defer li.unlock() + if li.closed { + return nil, net.ErrClosed + } + c := li.queue[0] + li.queue = li.queue[1:] + return c, nil +} + +func (li *fakeNetListener) Close() error { + li.lock() + defer li.unlock() + li.closed = true + return nil +} + +func (li *fakeNetListener) Addr() net.Addr { + return net.TCPAddrFromAddrPort(li.addr) +} + +// fakeNetPipe creates an in-memory, full duplex network connection. +// +// Unlike net.Pipe, the connection is not synchronous. +// Writes are made to a buffer, and return immediately. +// By default, the buffer size is unlimited. +func fakeNetPipe(s1ap, s2ap netip.AddrPort) (r, w *fakeNetConn) { + s1addr := net.TCPAddrFromAddrPort(s1ap) + s2addr := net.TCPAddrFromAddrPort(s2ap) + s1 := newSynctestNetConnHalf(s1addr) + s2 := newSynctestNetConnHalf(s2addr) + c1 := &fakeNetConn{loc: s1, rem: s2} + c2 := &fakeNetConn{loc: s2, rem: s1} + c1.peer = c2 + c2.peer = c1 + return c1, c2 +} + +// A fakeNetConn is one endpoint of the connection created by fakeNetPipe. +type fakeNetConn struct { + // local and remote connection halves. + // Each half contains a buffer. + // Reads pull from the local buffer, and writes push to the remote buffer. + loc, rem *fakeNetConnHalf + + // When set, synctest.Wait is automatically called before reads and after writes. + autoWait bool + + // peer is the other endpoint. + peer *fakeNetConn + + onClose func(*fakeNetConn) // called when closing +} + +// Read reads data from the connection. +func (c *fakeNetConn) Read(b []byte) (n int, err error) { + if c.autoWait { + synctest.Wait() + } + return c.loc.read(b) +} + +// Peek returns the available unread read buffer, +// without consuming its contents. +func (c *fakeNetConn) Peek() []byte { + if c.autoWait { + synctest.Wait() + } + return c.loc.peek() +} + +// Write writes data to the connection. +func (c *fakeNetConn) Write(b []byte) (n int, err error) { + if c.autoWait { + defer synctest.Wait() + } + return c.rem.write(b) +} + +// IsClosed reports whether the peer has closed its end of the connection. +func (c *fakeNetConn) IsClosedByPeer() bool { + if c.autoWait { + synctest.Wait() + } + c.rem.lock() + defer c.rem.unlock() + // If the remote half of the conn is returning ErrClosed, + // the peer has closed the connection. + return c.rem.readErr == net.ErrClosed +} + +// Close closes the connection. +func (c *fakeNetConn) Close() error { + if c.onClose != nil { + c.onClose(c) + } + // Local half of the conn is now closed. + c.loc.lock() + c.loc.writeErr = net.ErrClosed + c.loc.readErr = net.ErrClosed + c.loc.buf.Reset() + c.loc.unlock() + // Remote half of the connection reads EOF after reading any remaining data. + c.rem.lock() + if c.rem.readErr != nil { + c.rem.readErr = io.EOF + } + c.rem.unlock() + if c.autoWait { + synctest.Wait() + } + return nil +} + +// LocalAddr returns the (fake) local network address. +func (c *fakeNetConn) LocalAddr() net.Addr { + return c.loc.addr +} + +// LocalAddr returns the (fake) remote network address. +func (c *fakeNetConn) RemoteAddr() net.Addr { + return c.rem.addr +} + +// SetDeadline sets the read and write deadlines for the connection. +func (c *fakeNetConn) SetDeadline(t time.Time) error { + c.SetReadDeadline(t) + c.SetWriteDeadline(t) + return nil +} + +// SetReadDeadline sets the read deadline for the connection. +func (c *fakeNetConn) SetReadDeadline(t time.Time) error { + c.loc.rctx.setDeadline(t) + return nil +} + +// SetWriteDeadline sets the write deadline for the connection. +func (c *fakeNetConn) SetWriteDeadline(t time.Time) error { + c.rem.wctx.setDeadline(t) + return nil +} + +// SetReadBufferSize sets the read buffer limit for the connection. +// Writes by the peer will block so long as the buffer is full. +func (c *fakeNetConn) SetReadBufferSize(size int) { + c.loc.setReadBufferSize(size) +} + +// fakeNetConnHalf is one data flow in the connection created by fakeNetPipe. +// Each half contains a buffer. Writes to the half push to the buffer, and reads pull from it. +type fakeNetConnHalf struct { + addr net.Addr + + // Read and write timeouts. + rctx, wctx deadlineContext + + // A half can be readable and/or writable. + // + // These four channels act as a lock, + // and allow waiting for readability/writability. + // When the half is unlocked, exactly one channel contains a value. + // When the half is locked, all channels are empty. + lockr chan struct{} // readable + lockw chan struct{} // writable + lockrw chan struct{} // readable and writable + lockc chan struct{} // neither readable nor writable + + bufMax int // maximum buffer size + buf bytes.Buffer + readErr error // error returned by reads + writeErr error // error returned by writes +} + +func newSynctestNetConnHalf(addr net.Addr) *fakeNetConnHalf { + h := &fakeNetConnHalf{ + addr: addr, + lockw: make(chan struct{}, 1), + lockr: make(chan struct{}, 1), + lockrw: make(chan struct{}, 1), + lockc: make(chan struct{}, 1), + bufMax: math.MaxInt, // unlimited + } + h.unlock() + return h +} + +// lock locks h. +func (h *fakeNetConnHalf) lock() { + select { + case <-h.lockw: // writable + case <-h.lockr: // readable + case <-h.lockrw: // readable and writable + case <-h.lockc: // neither readable nor writable + } +} + +// h unlocks h. +func (h *fakeNetConnHalf) unlock() { + canRead := h.readErr != nil || h.buf.Len() > 0 + canWrite := h.writeErr != nil || h.bufMax > h.buf.Len() + switch { + case canRead && canWrite: + h.lockrw <- struct{}{} // readable and writable + case canRead: + h.lockr <- struct{}{} // readable + case canWrite: + h.lockw <- struct{}{} // writable + default: + h.lockc <- struct{}{} // neither readable nor writable + } +} + +// waitAndLockForRead waits until h is readable and locks it. +func (h *fakeNetConnHalf) waitAndLockForRead() error { + // First a non-blocking select to see if we can make immediate progress. + // This permits using a canceled context for a non-blocking operation. + select { + case <-h.lockr: + return nil // readable + case <-h.lockrw: + return nil // readable and writable + default: + } + ctx := h.rctx.context() + select { + case <-h.lockr: + return nil // readable + case <-h.lockrw: + return nil // readable and writable + case <-ctx.Done(): + return context.Cause(ctx) + } +} + +// waitAndLockForWrite waits until h is writable and locks it. +func (h *fakeNetConnHalf) waitAndLockForWrite() error { + // First a non-blocking select to see if we can make immediate progress. + // This permits using a canceled context for a non-blocking operation. + select { + case <-h.lockw: + return nil // writable + case <-h.lockrw: + return nil // readable and writable + default: + } + ctx := h.wctx.context() + select { + case <-h.lockw: + return nil // writable + case <-h.lockrw: + return nil // readable and writable + case <-ctx.Done(): + return context.Cause(ctx) + } +} + +func (h *fakeNetConnHalf) peek() []byte { + h.lock() + defer h.unlock() + return h.buf.Bytes() +} + +func (h *fakeNetConnHalf) read(b []byte) (n int, err error) { + if err := h.waitAndLockForRead(); err != nil { + return 0, err + } + defer h.unlock() + if h.buf.Len() == 0 && h.readErr != nil { + return 0, h.readErr + } + return h.buf.Read(b) +} + +func (h *fakeNetConnHalf) setReadBufferSize(size int) { + h.lock() + defer h.unlock() + h.bufMax = size +} + +func (h *fakeNetConnHalf) write(b []byte) (n int, err error) { + for n < len(b) { + nn, err := h.writePartial(b[n:]) + n += nn + if err != nil { + return n, err + } + } + return n, nil +} + +func (h *fakeNetConnHalf) writePartial(b []byte) (n int, err error) { + if err := h.waitAndLockForWrite(); err != nil { + return 0, err + } + defer h.unlock() + if h.writeErr != nil { + return 0, h.writeErr + } + writeMax := h.bufMax - h.buf.Len() + if writeMax < len(b) { + b = b[:writeMax] + } + return h.buf.Write(b) +} + +// deadlineContext converts a changable deadline (as in net.Conn.SetDeadline) into a Context. +type deadlineContext struct { + mu sync.Mutex + ctx context.Context + cancel context.CancelCauseFunc + timer *time.Timer +} + +// context returns a Context which expires when the deadline does. +func (t *deadlineContext) context() context.Context { + t.mu.Lock() + defer t.mu.Unlock() + if t.ctx == nil { + t.ctx, t.cancel = context.WithCancelCause(context.Background()) + } + return t.ctx +} + +// setDeadline sets the current deadline. +func (t *deadlineContext) setDeadline(deadline time.Time) { + t.mu.Lock() + defer t.mu.Unlock() + // If t.ctx is non-nil and t.cancel is nil, then t.ctx was canceled + // and we should create a new one. + if t.ctx == nil || t.cancel == nil { + t.ctx, t.cancel = context.WithCancelCause(context.Background()) + } + // Stop any existing deadline from expiring. + if t.timer != nil { + t.timer.Stop() + } + if deadline.IsZero() { + // No deadline. + return + } + now := time.Now() + if !deadline.After(now) { + // Deadline has already expired. + t.cancel(os.ErrDeadlineExceeded) + t.cancel = nil + return + } + if t.timer != nil { + // Reuse existing deadline timer. + t.timer.Reset(deadline.Sub(now)) + return + } + // Create a new timer to cancel the context at the deadline. + t.timer = time.AfterFunc(deadline.Sub(now), func() { + t.mu.Lock() + defer t.mu.Unlock() + t.cancel(os.ErrDeadlineExceeded) + t.cancel = nil + }) +} diff --git a/src/net/http/pprof/pprof.go b/src/net/http/pprof/pprof.go index cf4b8415ca3aaf..71aade67d32046 100644 --- a/src/net/http/pprof/pprof.go +++ b/src/net/http/pprof/pprof.go @@ -67,7 +67,7 @@ // in your browser. // // For a study of the facility in action, visit -// https://blog.golang.org/2011/06/profiling-go-programs.html. +// https://go.dev/blog/pprof. package pprof import ( @@ -77,6 +77,7 @@ import ( "fmt" "html" "internal/godebug" + "internal/goexperiment" "internal/profile" "io" "log" @@ -86,7 +87,7 @@ import ( "runtime" "runtime/pprof" "runtime/trace" - "sort" + "slices" "strconv" "strings" "time" @@ -351,12 +352,13 @@ func collectProfile(p *pprof.Profile) (*profile.Profile, error) { } var profileSupportsDelta = map[handler]bool{ - "allocs": true, - "block": true, - "goroutine": true, - "heap": true, - "mutex": true, - "threadcreate": true, + "allocs": true, + "block": true, + "goroutineleak": true, + "goroutine": true, + "heap": true, + "mutex": true, + "threadcreate": true, } var profileDescriptions = map[string]string{ @@ -367,10 +369,17 @@ var profileDescriptions = map[string]string{ "heap": "A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.", "mutex": "Stack traces of holders of contended mutexes", "profile": "CPU profile. You can specify the duration in the seconds GET parameter. After you get the profile file, use the go tool pprof command to investigate the profile.", + "symbol": "Maps given program counters to function names. Counters can be specified in a GET raw query or POST body, multiple counters are separated by '+'.", "threadcreate": "Stack traces that led to the creation of new OS threads", "trace": "A trace of execution of the current program. You can specify the duration in the seconds GET parameter. After you get the trace file, use the go tool trace command to investigate the trace.", } +func init() { + if goexperiment.GoroutineLeakProfile { + profileDescriptions["goroutineleak"] = "Stack traces of all leaked goroutines. Use debug=2 as a query parameter to export in the same format as an unrecovered panic." + } +} + type profileEntry struct { Name string Href string @@ -404,7 +413,7 @@ func Index(w http.ResponseWriter, r *http.Request) { } // Adding other profiles exposed from within this package - for _, p := range []string{"cmdline", "profile", "trace"} { + for _, p := range []string{"cmdline", "profile", "symbol", "trace"} { profiles = append(profiles, profileEntry{ Name: p, Href: p, @@ -412,8 +421,8 @@ func Index(w http.ResponseWriter, r *http.Request) { }) } - sort.Slice(profiles, func(i, j int) bool { - return profiles[i].Name < profiles[j].Name + slices.SortFunc(profiles, func(a, b profileEntry) int { + return strings.Compare(a.Name, b.Name) }) if err := indexTmplExecute(w, profiles); err != nil { diff --git a/src/net/http/proxy_test.go b/src/net/http/proxy_test.go index 0dd57b41253ee6..6090506f427d96 100644 --- a/src/net/http/proxy_test.go +++ b/src/net/http/proxy_test.go @@ -10,9 +10,6 @@ import ( "testing" ) -// TODO(mattn): -// test ProxyAuth - var cacheKeysTests = []struct { proxy string scheme string @@ -48,3 +45,42 @@ func ResetProxyEnv() { } ResetCachedEnvironment() } + +var proxyAuthTests = []struct { + proxy string + key string +}{ + { + "", + "", + }, + { + "http://bar.com/", + "", + }, + { + "http://foo@bar.com/", + "Basic Zm9vOg==", + }, + { + "http://foo:bar@bar.com/", + "Basic Zm9vOmJhcg==", + }, +} + +func TestProxyAuthKeys(t *testing.T) { + for _, tt := range proxyAuthTests { + var proxy *url.URL + if tt.proxy != "" { + u, err := url.Parse(tt.proxy) + if err != nil { + t.Fatal(err) + } + proxy = u + } + cm := connectMethod{proxyURL: proxy} + if got := cm.proxyAuth(); got != tt.key { + t.Fatalf("{%q} proxyAuth key = %q; want %q", tt.proxy, got, tt.key) + } + } +} diff --git a/src/net/http/request.go b/src/net/http/request.go index ad1b5a620b07ae..167cff585af34a 100644 --- a/src/net/http/request.go +++ b/src/net/http/request.go @@ -15,6 +15,7 @@ import ( "errors" "fmt" "io" + "maps" "mime" "mime/multipart" "net/http/httptrace" @@ -390,12 +391,8 @@ func (r *Request) Clone(ctx context.Context) *Request { *r2 = *r r2.ctx = ctx r2.URL = cloneURL(r.URL) - if r.Header != nil { - r2.Header = r.Header.Clone() - } - if r.Trailer != nil { - r2.Trailer = r.Trailer.Clone() - } + r2.Header = r.Header.Clone() + r2.Trailer = r.Trailer.Clone() if s := r.TransferEncoding; s != nil { s2 := make([]string, len(s)) copy(s2, s) @@ -411,13 +408,7 @@ func (r *Request) Clone(ctx context.Context) *Request { copy(s2, s) r2.matches = s2 } - if s := r.otherValues; s != nil { - s2 := make(map[string]string, len(s)) - for k, v := range s { - s2[k] = v - } - r2.otherValues = s2 - } + r2.otherValues = maps.Clone(r.otherValues) return r2 } @@ -864,7 +855,7 @@ func validMethod(method string) bool { extension-method = token token = 1* */ - return len(method) > 0 && strings.IndexFunc(method, isNotToken) == -1 + return isToken(method) } // NewRequest wraps [NewRequestWithContext] using [context.Background]. @@ -882,12 +873,12 @@ func NewRequest(method, url string, body io.Reader) (*Request, error) { // // NewRequestWithContext returns a Request suitable for use with // [Client.Do] or [Transport.RoundTrip]. To create a request for use with -// testing a Server Handler, either use the [NewRequest] function in the -// net/http/httptest package, use [ReadRequest], or manually update the -// Request fields. For an outgoing client request, the context +// testing a Server Handler, either use the [net/http/httptest.NewRequest] function, +// use [ReadRequest], or manually update the Request fields. +// For an outgoing client request, the context // controls the entire lifetime of a request and its response: // obtaining a connection, sending the request, and reading the -// response headers and body. See the Request type's documentation for +// response headers and body. See the [Request] type's documentation for // the difference between inbound and outbound request fields. // // If body is of type [*bytes.Buffer], [*bytes.Reader], or @@ -1071,7 +1062,7 @@ func ReadRequest(b *bufio.Reader) (*Request, error) { } delete(req.Header, "Host") - return req, err + return req, nil } // readRequest should be an internal detail, diff --git a/src/net/http/requestwrite_test.go b/src/net/http/requestwrite_test.go index 380ae9dec3244d..8b097cd5e15d1f 100644 --- a/src/net/http/requestwrite_test.go +++ b/src/net/http/requestwrite_test.go @@ -15,6 +15,7 @@ import ( "strings" "testing" "testing/iotest" + "testing/synctest" "time" ) @@ -667,6 +668,13 @@ func TestRequestWrite(t *testing.T) { func TestRequestWriteTransport(t *testing.T) { t.Parallel() + // Run this test in a synctest bubble, since it relies on the transport + // successfully probing the request body within 200ms + // (see transferWriter.probeRequestBody). + // This occasionally flakes on slow builders (#52575) if we don't use a fake clock. + synctest.Test(t, testRequestWriteTransport) +} +func testRequestWriteTransport(t *testing.T) { matchSubstr := func(substr string) func(string) error { return func(written string) error { if !strings.Contains(written, substr) { diff --git a/src/net/http/response_test.go b/src/net/http/response_test.go index a63aac95ac1f69..3ccbb9b0f2d4b1 100644 --- a/src/net/http/response_test.go +++ b/src/net/http/response_test.go @@ -21,9 +21,10 @@ import ( ) type respTest struct { - Raw string - Resp Response - Body string + Raw string + RawOut string + Resp Response + Body string } func dummyReq(method string) *Request { @@ -42,6 +43,11 @@ var respTests = []respTest{ "\r\n" + "Body here\n", + "HTTP/1.0 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n" + + "Body here\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -66,6 +72,11 @@ var respTests = []respTest{ "\r\n" + "Body here\n", + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n" + + "Body here\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -87,6 +98,9 @@ var respTests = []respTest{ "\r\n" + "Body should not be read!\n", + "HTTP/1.1 204 No Content\r\n" + + "\r\n", + Response{ Status: "204 No Content", StatusCode: 204, @@ -110,6 +124,12 @@ var respTests = []respTest{ "\r\n" + "Body here\n", + "HTTP/1.0 200 OK\r\n" + + "Content-Length: 10\r\n" + + "Connection: close\r\n" + + "\r\n" + + "Body here\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -140,6 +160,14 @@ var respTests = []respTest{ "0\r\n" + "\r\n", + "HTTP/1.1 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n" + + "\r\n" + + "13\r\n" + + "Body here\ncontinued\r\n" + + "0\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -165,6 +193,12 @@ var respTests = []respTest{ "\r\n" + "Body here\n", + "HTTP/1.0 200 OK\r\n" + + "Content-Length: 10\r\n" + + "Connection: close\r\n" + + "\r\n" + + "Body here\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -195,6 +229,14 @@ var respTests = []respTest{ "0\r\n" + "\r\n", + "HTTP/1.1 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n" + + "\r\n" + + "a\r\n" + + "Body here\n\r\n" + + "0\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -217,6 +259,10 @@ var respTests = []respTest{ "Transfer-Encoding: chunked\r\n" + "\r\n", + "HTTP/1.1 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -239,6 +285,11 @@ var respTests = []respTest{ "Content-Length: 256\r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + + "Connection: close\r\n" + + "Content-Length: 256\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -261,6 +312,10 @@ var respTests = []respTest{ "Content-Length: 256\r\n" + "\r\n", + "HTTP/1.1 200 OK\r\n" + + "Content-Length: 256\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -282,6 +337,10 @@ var respTests = []respTest{ "HTTP/1.0 200 OK\r\n" + "\r\n", + "HTTP/1.0 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -304,6 +363,10 @@ var respTests = []respTest{ "Content-Length: 0\r\n" + "\r\n", + "HTTP/1.1 200 OK\r\n" + + "Content-Length: 0\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -325,6 +388,11 @@ var respTests = []respTest{ // (permitted by RFC 7230, section 3.1.2) { "HTTP/1.0 303 \r\n\r\n", + + "HTTP/1.0 303 \r\n" + + "Connection: close\r\n" + + "\r\n", + Response{ Status: "303 ", StatusCode: 303, @@ -344,6 +412,11 @@ var respTests = []respTest{ // (not permitted by RFC 7230, but we'll accept it anyway) { "HTTP/1.0 303\r\n\r\n", + + "HTTP/1.0 303 303\r\n" + + "Connection: close\r\n" + + "\r\n", + Response{ Status: "303", StatusCode: 303, @@ -366,6 +439,13 @@ Connection: close Content-Type: multipart/byteranges; boundary=18a75608c8f47cef some body`, + + "HTTP/1.1 206 Partial Content\r\n" + + "Connection: close\r\n" + + "Content-Type: multipart/byteranges; boundary=18a75608c8f47cef\r\n" + + "\r\n" + + "some body", + Response{ Status: "206 Partial Content", StatusCode: 206, @@ -390,6 +470,11 @@ some body`, "\r\n" + "Body here\n", + "HTTP/1.0 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n" + + "Body here\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -415,6 +500,14 @@ some body`, "Content-Length: 6\r\n\r\n" + "foobar", + "HTTP/1.1 206 Partial Content\r\n" + + "Content-Length: 6\r\n" + + "Accept-Ranges: bytes\r\n" + + "Content-Range: bytes 0-5/1862\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + + "\r\n" + + "foobar", + Response{ Status: "206 Partial Content", StatusCode: 206, @@ -441,6 +534,11 @@ some body`, "Connection: keep-alive, close\r\n" + "\r\n", + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "Content-Length: 256\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -467,6 +565,11 @@ some body`, "Connection: close\r\n" + "\r\n", + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "Content-Length: 256\r\n" + + "\r\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -493,6 +596,11 @@ some body`, "\r\n" + "Body here\n", + "HTTP/1.0 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n" + + "Body here\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -517,6 +625,12 @@ some body`, "\r\n" + "Body here\n", + "HTTP/1.0 200 OK\r\n" + + "Connection: close\r\n" + + "Content-Length: 10\r\n" + + "\r\n" + + "Body here\n", + Response{ Status: "200 OK", StatusCode: 200, @@ -541,6 +655,14 @@ some body`, "Connection: keep-alive\r\n" + "Keep-Alive: timeout=7200\r\n\r\n" + "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\x00s\xf3\xf7\a\x00\xab'\xd4\x1a\x03\x00\x00\x00", + + "HTTP/1.1 200 OK\r\n" + + "Content-Length: 23\r\n" + + "Connection: keep-alive\r\n" + + "Content-Encoding: gzip\r\n" + + "Keep-Alive: timeout=7200\r\n\r\n" + + "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\x00s\xf3\xf7\a\x00\xab'\xd4\x1a\x03\x00\x00\x00", + Response{ Status: "200 OK", StatusCode: 200, @@ -566,6 +688,14 @@ some body`, "Content-type: text/html\r\n" + "WWW-Authenticate: Basic realm=\"\"\r\n\r\n" + "Your Authentication failed.\r\n", + + "HTTP/1.0 401 Unauthorized\r\n" + + "Connection: close\r\n" + + "Content-Type: text/html\r\n" + + "Www-Authenticate: Basic realm=\"\"\r\n" + + "\r\n" + + "Your Authentication failed.\r\n", + Response{ Status: "401 Unauthorized", StatusCode: 401, @@ -619,11 +749,18 @@ func TestWriteResponse(t *testing.T) { t.Errorf("#%d: %v", i, err) continue } - err = resp.Write(io.Discard) + var buf bytes.Buffer + err = resp.Write(&buf) if err != nil { t.Errorf("#%d: %v", i, err) continue } + if got, want := buf.String(), tt.RawOut; got != want { + t.Errorf("#%d: response differs; got:\n----\n%v\n----\nwant:\n----\n%v\n----\n", + i, + strings.ReplaceAll(got, "\r", "\\r"), + strings.ReplaceAll(want, "\r", "\\r")) + } } } diff --git a/src/net/http/responsecontroller.go b/src/net/http/responsecontroller.go index f3f24c1273ee32..785fa21d17d45d 100644 --- a/src/net/http/responsecontroller.go +++ b/src/net/http/responsecontroller.go @@ -62,7 +62,7 @@ func (c *ResponseController) Flush() error { } // Hijack lets the caller take over the connection. -// See the Hijacker interface for details. +// See the [Hijacker] interface for details. func (c *ResponseController) Hijack() (net.Conn, *bufio.ReadWriter, error) { rw := c.rw for { diff --git a/src/net/http/roundtrip.go b/src/net/http/roundtrip.go index 6674b8419f3d32..d2e29a33d990db 100644 --- a/src/net/http/roundtrip.go +++ b/src/net/http/roundtrip.go @@ -27,5 +27,8 @@ func badRoundTrip(*Transport, *Request) (*Response, error) // Like the RoundTripper interface, the error types returned // by RoundTrip are unspecified. func (t *Transport) RoundTrip(req *Request) (*Response, error) { + if t == nil { + panic("transport is nil") + } return t.roundTrip(req) } diff --git a/src/net/http/roundtrip_js.go b/src/net/http/roundtrip_js.go index 04c241eb4c006a..7ae94617bcfb58 100644 --- a/src/net/http/roundtrip_js.go +++ b/src/net/http/roundtrip_js.go @@ -236,6 +236,14 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { if !ac.IsUndefined() { // Abort the Fetch request. ac.Call("abort") + + // Wait for fetch promise to be rejected prior to exiting. See + // https://github.com/golang/go/issues/57098 for more details. + select { + case resp := <-respCh: + resp.Body.Close() + case <-errCh: + } } return nil, req.Context().Err() case resp := <-respCh: diff --git a/src/net/http/serve_test.go b/src/net/http/serve_test.go index cc485d3b89471c..4a16ba02af6cee 100644 --- a/src/net/http/serve_test.go +++ b/src/net/http/serve_test.go @@ -12,10 +12,13 @@ import ( "compress/gzip" "compress/zlib" "context" + crand "crypto/rand" "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" + "internal/synctest" "internal/testenv" "io" "log" @@ -389,6 +392,19 @@ func TestServeMuxHandler(t *testing.T) { } } +// Issue 73688 +func TestServeMuxHandlerTrailingSlash(t *testing.T) { + setParallel(t) + mux := NewServeMux() + const original = "/{x}/" + mux.Handle(original, NotFoundHandler()) + r, _ := NewRequest("POST", "/foo", nil) + _, p := mux.Handler(r) + if p != original { + t.Errorf("got %q, want %q", p, original) + } +} + // Issue 24297 func TestServeMuxHandleFuncWithNilHandler(t *testing.T) { setParallel(t) @@ -1510,7 +1526,7 @@ func testHeadResponses(t *testing.T, mode testMode) { } // Also exercise the ReaderFrom path - _, err = io.Copy(w, strings.NewReader("789a")) + _, err = io.Copy(w, struct{ io.Reader }{strings.NewReader("789a")}) if err != nil { t.Errorf("Copy(ResponseWriter, ...): %v", err) } @@ -1537,6 +1553,34 @@ func testHeadResponses(t *testing.T, mode testMode) { } } +// Ensure ResponseWriter.ReadFrom doesn't write a body in response to a HEAD request. +// https://go.dev/issue/68609 +func TestHeadReaderFrom(t *testing.T) { run(t, testHeadReaderFrom, []testMode{http1Mode}) } +func testHeadReaderFrom(t *testing.T, mode testMode) { + // Body is large enough to exceed the content-sniffing length. + wantBody := strings.Repeat("a", 4096) + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + w.(io.ReaderFrom).ReadFrom(strings.NewReader(wantBody)) + })) + res, err := cst.c.Head(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + res, err = cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + gotBody, err := io.ReadAll(res.Body) + res.Body.Close() + if err != nil { + t.Fatal(err) + } + if string(gotBody) != wantBody { + t.Errorf("got unexpected body len=%v, want %v", len(gotBody), len(wantBody)) + } +} + func TestTLSHandshakeTimeout(t *testing.T) { run(t, testTLSHandshakeTimeout, []testMode{https1Mode, http2Mode}) } @@ -1615,6 +1659,53 @@ func testTLSServer(t *testing.T, mode testMode) { } } +type fakeConnectionStateConn struct { + net.Conn +} + +func (fcsc *fakeConnectionStateConn) ConnectionState() tls.ConnectionState { + return tls.ConnectionState{ + ServerName: "example.com", + } +} + +func TestTLSServerWithoutTLSConn(t *testing.T) { + //set up + pr, pw := net.Pipe() + c := make(chan int) + listener := &oneConnListener{&fakeConnectionStateConn{pr}} + server := &Server{ + Handler: HandlerFunc(func(writer ResponseWriter, request *Request) { + if request.TLS == nil { + t.Fatal("request.TLS is nil, expected not nil") + } + if request.TLS.ServerName != "example.com" { + t.Fatalf("request.TLS.ServerName is %s, expected %s", request.TLS.ServerName, "example.com") + } + writer.Header().Set("X-TLS-ServerName", "example.com") + }), + } + + // write request and read response + go func() { + req, _ := NewRequest(MethodGet, "https://example.com/", nil) + req.Write(pw) + + resp, _ := ReadResponse(bufio.NewReader(pw), req) + if hdr := resp.Header.Get("X-TLS-ServerName"); hdr != "example.com" { + t.Errorf("response header X-TLS-ServerName is %s, expected %s", hdr, "example.com") + } + close(c) + pw.Close() + }() + + server.Serve(listener) + + // oneConnListener returns error after one accept, wait util response is read + <-c + pr.Close() +} + func TestServeTLS(t *testing.T) { CondSkipHTTP2(t) // Not parallel: uses global test hooks. @@ -4273,19 +4364,6 @@ func TestResponseWriterWriteString(t *testing.T) { } } -func TestAppendTime(t *testing.T) { - var b [len(TimeFormat)]byte - t1 := time.Date(2013, 9, 21, 15, 41, 0, 0, time.FixedZone("CEST", 2*60*60)) - res := ExportAppendTime(b[:0], t1) - t2, err := ParseTime(string(res)) - if err != nil { - t.Fatalf("Error parsing time: %s", err) - } - if !t1.Equal(t2) { - t.Fatalf("Times differ; expected: %v, got %v (%s)", t1, t2, string(res)) - } -} - func TestServerConnState(t *testing.T) { run(t, testServerConnState, []testMode{http1Mode}) } func testServerConnState(t *testing.T, mode testMode) { handler := map[string]func(w ResponseWriter, r *Request){ @@ -5204,8 +5282,8 @@ func benchmarkClientServerParallel(b *testing.B, parallelism int, mode testMode) func BenchmarkServer(b *testing.B) { b.ReportAllocs() // Child process mode; - if url := os.Getenv("TEST_BENCH_SERVER_URL"); url != "" { - n, err := strconv.Atoi(os.Getenv("TEST_BENCH_CLIENT_N")) + if url := os.Getenv("GO_TEST_BENCH_SERVER_URL"); url != "" { + n, err := strconv.Atoi(os.Getenv("GO_TEST_BENCH_CLIENT_N")) if err != nil { panic(err) } @@ -5239,8 +5317,8 @@ func BenchmarkServer(b *testing.B) { cmd := testenv.Command(b, os.Args[0], "-test.run=^$", "-test.bench=^BenchmarkServer$") cmd.Env = append([]string{ - fmt.Sprintf("TEST_BENCH_CLIENT_N=%d", b.N), - fmt.Sprintf("TEST_BENCH_SERVER_URL=%s", ts.URL), + fmt.Sprintf("GO_TEST_BENCH_CLIENT_N=%d", b.N), + fmt.Sprintf("GO_TEST_BENCH_SERVER_URL=%s", ts.URL), }, os.Environ()...) out, err := cmd.CombinedOutput() if err != nil { @@ -5261,30 +5339,54 @@ func getNoBody(urlStr string) (*Response, error) { // A benchmark for profiling the client without the HTTP server code. // The server code runs in a subprocess. func BenchmarkClient(b *testing.B) { + var data = []byte("Hello world.\n") + + url := startClientBenchmarkServer(b, HandlerFunc(func(w ResponseWriter, _ *Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Write(data) + })) + + // Do b.N requests to the server. + b.StartTimer() + for i := 0; i < b.N; i++ { + res, err := Get(url) + if err != nil { + b.Fatalf("Get: %v", err) + } + body, err := io.ReadAll(res.Body) + res.Body.Close() + if err != nil { + b.Fatalf("ReadAll: %v", err) + } + if !bytes.Equal(body, data) { + b.Fatalf("Got body: %q", body) + } + } + b.StopTimer() +} + +func startClientBenchmarkServer(b *testing.B, handler Handler) string { b.ReportAllocs() b.StopTimer() - defer afterTest(b) - var data = []byte("Hello world.\n") - if server := os.Getenv("TEST_BENCH_SERVER"); server != "" { + if server := os.Getenv("GO_TEST_BENCH_SERVER"); server != "" { // Server process mode. - port := os.Getenv("TEST_BENCH_SERVER_PORT") // can be set by user + port := os.Getenv("GO_TEST_BENCH_SERVER_PORT") // can be set by user if port == "" { port = "0" } ln, err := net.Listen("tcp", "localhost:"+port) if err != nil { - fmt.Fprintln(os.Stderr, err.Error()) - os.Exit(1) + log.Fatal(err) } fmt.Println(ln.Addr().String()) + HandleFunc("/", func(w ResponseWriter, r *Request) { r.ParseForm() if r.Form.Get("stop") != "" { os.Exit(0) } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - w.Write(data) + handler.ServeHTTP(w, r) }) var srv Server log.Fatal(srv.Serve(ln)) @@ -5292,8 +5394,8 @@ func BenchmarkClient(b *testing.B) { // Start server process. ctx, cancel := context.WithCancel(context.Background()) - cmd := testenv.CommandContext(b, ctx, os.Args[0], "-test.run=^$", "-test.bench=^BenchmarkClient$") - cmd.Env = append(cmd.Environ(), "TEST_BENCH_SERVER=yes") + cmd := testenv.CommandContext(b, ctx, os.Args[0], "-test.run=^$", "-test.bench=^"+b.Name()+"$") + cmd.Env = append(cmd.Environ(), "GO_TEST_BENCH_SERVER=yes") cmd.Stderr = os.Stderr stdout, err := cmd.StdoutPipe() if err != nil { @@ -5308,10 +5410,6 @@ func BenchmarkClient(b *testing.B) { done <- cmd.Wait() close(done) }() - defer func() { - cancel() - <-done - }() // Wait for the server in the child process to respond and tell us // its listening address, once it's started listening: @@ -5324,6 +5422,39 @@ func BenchmarkClient(b *testing.B) { b.Fatalf("initial probe of child process failed: %v", err) } + // Instruct server process to stop. + b.Cleanup(func() { + getNoBody(url + "?stop=yes") + if err := <-done; err != nil { + b.Fatalf("subprocess failed: %v", err) + } + + cancel() + <-done + + afterTest(b) + }) + + return url +} + +func BenchmarkClientGzip(b *testing.B) { + const responseSize = 1024 * 1024 + + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + if _, err := io.CopyN(gz, crand.Reader, responseSize); err != nil { + b.Fatal(err) + } + gz.Close() + + data := buf.Bytes() + + url := startClientBenchmarkServer(b, HandlerFunc(func(w ResponseWriter, _ *Request) { + w.Header().Set("Content-Encoding", "gzip") + w.Write(data) + })) + // Do b.N requests to the server. b.StartTimer() for i := 0; i < b.N; i++ { @@ -5331,22 +5462,16 @@ func BenchmarkClient(b *testing.B) { if err != nil { b.Fatalf("Get: %v", err) } - body, err := io.ReadAll(res.Body) + n, err := io.Copy(io.Discard, res.Body) res.Body.Close() if err != nil { b.Fatalf("ReadAll: %v", err) } - if !bytes.Equal(body, data) { - b.Fatalf("Got body: %q", body) + if n != responseSize { + b.Fatalf("ReadAll: expected %d bytes, got %d", responseSize, n) } } b.StopTimer() - - // Instruct server process to stop. - getNoBody(url + "?stop=yes") - if err := <-done; err != nil { - b.Fatalf("subprocess failed: %v", err) - } } func BenchmarkServerFakeConnNoKeepAlive(b *testing.B) { @@ -5777,70 +5902,59 @@ func testServerShutdown(t *testing.T, mode testMode) { } } -func TestServerShutdownStateNew(t *testing.T) { run(t, testServerShutdownStateNew) } +func TestServerShutdownStateNew(t *testing.T) { runSynctest(t, testServerShutdownStateNew) } func testServerShutdownStateNew(t *testing.T, mode testMode) { if testing.Short() { t.Skip("test takes 5-6 seconds; skipping in short mode") } - var connAccepted sync.WaitGroup + listener := fakeNetListen() + defer listener.Close() + ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { // nothing. }), func(ts *httptest.Server) { - ts.Config.ConnState = func(conn net.Conn, state ConnState) { - if state == StateNew { - connAccepted.Done() - } - } + ts.Listener.Close() + ts.Listener = listener + // Ignore irrelevant error about TLS handshake failure. + ts.Config.ErrorLog = log.New(io.Discard, "", 0) }).ts // Start a connection but never write to it. - connAccepted.Add(1) - c, err := net.Dial("tcp", ts.Listener.Addr().String()) - if err != nil { - t.Fatal(err) - } + c := listener.connect() defer c.Close() + synctest.Wait() - // Wait for the connection to be accepted by the server. Otherwise, if - // Shutdown happens to run first, the server will be closed when - // encountering the connection, in which case it will be rejected - // immediately. - connAccepted.Wait() - - shutdownRes := make(chan error, 1) - go func() { - shutdownRes <- ts.Config.Shutdown(context.Background()) - }() - readRes := make(chan error, 1) - go func() { - _, err := c.Read([]byte{0}) - readRes <- err - }() + shutdownRes := runAsync(func() (struct{}, error) { + return struct{}{}, ts.Config.Shutdown(context.Background()) + }) // TODO(#59037): This timeout is hard-coded in closeIdleConnections. // It is undocumented, and some users may find it surprising. // Either document it, or switch to a less surprising behavior. const expectTimeout = 5 * time.Second - t0 := time.Now() - select { - case got := <-shutdownRes: - d := time.Since(t0) - if got != nil { - t.Fatalf("shutdown error after %v: %v", d, err) - } - if d < expectTimeout/2 { - t.Errorf("shutdown too soon after %v", d) - } - case <-time.After(expectTimeout * 3 / 2): - t.Fatalf("timeout waiting for shutdown") + // Wait until just before the expected timeout. + time.Sleep(expectTimeout - 1) + synctest.Wait() + if shutdownRes.done() { + t.Fatal("shutdown too soon") + } + if c.IsClosedByPeer() { + t.Fatal("connection was closed by server too soon") } - // Wait for c.Read to unblock; should be already done at this point, - // or within a few milliseconds. - if err := <-readRes; err == nil { - t.Error("expected error from Read") + // closeIdleConnections isn't precise about its actual shutdown time. + // Wait long enough for it to definitely have shut down. + // + // (It would be good to make closeIdleConnections less sloppy.) + time.Sleep(2 * time.Second) + synctest.Wait() + if _, err := shutdownRes.result(); err != nil { + t.Fatalf("Shutdown() = %v, want complete", err) + } + if !c.IsClosedByPeer() { + t.Fatalf("connection was not closed by server after shutdown") } } @@ -6133,6 +6247,50 @@ func testServerHijackGetsBackgroundByte(t *testing.T, mode testMode) { <-done } +// Test that the bufio.Reader returned by Hijack yields the entire body. +func TestServerHijackGetsFullBody(t *testing.T) { + run(t, testServerHijackGetsFullBody, []testMode{http1Mode}) +} +func testServerHijackGetsFullBody(t *testing.T, mode testMode) { + if runtime.GOOS == "plan9" { + t.Skip("skipping test; see https://golang.org/issue/18657") + } + done := make(chan struct{}) + needle := strings.Repeat("x", 100*1024) // assume: larger than net/http bufio size + ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + defer close(done) + + conn, buf, err := w.(Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + defer conn.Close() + + got := make([]byte, len(needle)) + n, err := io.ReadFull(buf.Reader, got) + if n != len(needle) || string(got) != needle || err != nil { + t.Errorf("Peek = %q, %v; want 'x'*4096, nil", got, err) + } + })).ts + + cn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer cn.Close() + buf := []byte("GET / HTTP/1.1\r\nHost: e.com\r\n\r\n") + buf = append(buf, []byte(needle)...) + if _, err := cn.Write(buf); err != nil { + t.Fatal(err) + } + + if err := cn.(*net.TCPConn).CloseWrite(); err != nil { + t.Fatal(err) + } + <-done +} + // Like TestServerHijackGetsBackgroundByte above but sending a // immediate 1MB of data to the server to fill up the server's 4KB // buffer. @@ -6543,7 +6701,6 @@ func testTimeoutHandlerSuperfluousLogs(t *testing.T, mode testMode) { } for _, tt := range tests { - tt := tt t.Run(tt.name, func(t *testing.T) { exitHandler := make(chan bool, 1) defer close(exitHandler) @@ -7123,10 +7280,6 @@ func testHeadBody(t *testing.T, mode testMode, chunked bool, method string) { // or disabled when the header is set to nil. func TestDisableContentLength(t *testing.T) { run(t, testDisableContentLength) } func testDisableContentLength(t *testing.T, mode testMode) { - if mode == http2Mode { - t.Skip("skipping until h2_bundle.go is updated; see https://go-review.googlesource.com/c/net/+/471535") - } - noCL := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { w.Header()["Content-Length"] = nil // disable the default Content-Length response fmt.Fprintf(w, "OK") @@ -7285,3 +7438,120 @@ func testServerReadAfterHandlerAbort100Continue(t *testing.T, mode testMode) { readyc <- struct{}{} // server starts reading from the request body readyc <- struct{}{} // server finishes reading from the request body } + +func TestInvalidChunkedBodies(t *testing.T) { + for _, test := range []struct { + name string + b string + }{{ + name: "bare LF in chunk size", + b: "1\na\r\n0\r\n\r\n", + }, { + name: "bare LF at body end", + b: "1\r\na\r\n0\r\n\n", + }} { + t.Run(test.name, func(t *testing.T) { + reqc := make(chan error) + ts := newClientServerTest(t, http1Mode, HandlerFunc(func(w ResponseWriter, r *Request) { + got, err := io.ReadAll(r.Body) + if err == nil { + t.Logf("read body: %q", got) + } + reqc <- err + })).ts + + serverURL, err := url.Parse(ts.URL) + if err != nil { + t.Fatal(err) + } + + conn, err := net.Dial("tcp", serverURL.Host) + if err != nil { + t.Fatal(err) + } + + if _, err := conn.Write([]byte( + "POST / HTTP/1.1\r\n" + + "Host: localhost\r\n" + + "Transfer-Encoding: chunked\r\n" + + "Connection: close\r\n" + + "\r\n" + + test.b)); err != nil { + t.Fatal(err) + } + conn.(*net.TCPConn).CloseWrite() + + if err := <-reqc; err == nil { + t.Errorf("server handler: io.ReadAll(r.Body) succeeded, want error") + } + }) + } +} + +// Issue #72100: Verify that we don't modify the caller's TLS.Config.NextProtos slice. +func TestServerTLSNextProtos(t *testing.T) { + run(t, testServerTLSNextProtos, []testMode{https1Mode, http2Mode}) +} +func testServerTLSNextProtos(t *testing.T, mode testMode) { + CondSkipHTTP2(t) + + cert, err := tls.X509KeyPair(testcert.LocalhostCert, testcert.LocalhostKey) + if err != nil { + t.Fatal(err) + } + leafCert, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + t.Fatal(err) + } + certpool := x509.NewCertPool() + certpool.AddCert(leafCert) + + protos := new(Protocols) + switch mode { + case https1Mode: + protos.SetHTTP1(true) + case http2Mode: + protos.SetHTTP2(true) + } + + wantNextProtos := []string{"http/1.1", "h2", "other"} + nextProtos := slices.Clone(wantNextProtos) + + // We don't use httptest here because it overrides the tls.Config. + srv := &Server{ + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{cert}, + NextProtos: nextProtos, + }, + Handler: HandlerFunc(func(w ResponseWriter, req *Request) {}), + Protocols: protos, + } + tr := &Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certpool, + NextProtos: nextProtos, + }, + Protocols: protos, + } + + listener := newLocalListener(t) + srvc := make(chan error, 1) + go func() { + srvc <- srv.ServeTLS(listener, "", "") + }() + t.Cleanup(func() { + srv.Close() + <-srvc + }) + + client := &Client{Transport: tr} + resp, err := client.Get("https://" + listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + + if !slices.Equal(nextProtos, wantNextProtos) { + t.Fatalf("after running test: original NextProtos slice = %v, want %v", nextProtos, wantNextProtos) + } +} diff --git a/src/net/http/server.go b/src/net/http/server.go index 1ff72a04550c42..02554d1a201afd 100644 --- a/src/net/http/server.go +++ b/src/net/http/server.go @@ -261,7 +261,7 @@ type conn struct { // rwc is the underlying network connection. // This is never wrapped by other types and is the value given out - // to CloseNotifier callers. It is usually of type *net.TCPConn or + // to [Hijacker] callers. It is usually of type *net.TCPConn or // *tls.Conn. rwc net.Conn @@ -324,12 +324,14 @@ func (c *conn) hijackLocked() (rwc net.Conn, buf *bufio.ReadWriter, err error) { rwc = c.rwc rwc.SetDeadline(time.Time{}) - buf = bufio.NewReadWriter(c.bufr, bufio.NewWriter(rwc)) if c.r.hasByte { if _, err := c.bufr.Peek(c.bufr.Buffered() + 1); err != nil { return nil, nil, fmt.Errorf("unexpected Peek failure reading buffered byte: %v", err) } } + c.bufw.Reset(rwc) + buf = bufio.NewReadWriter(c.bufr, c.bufw) + c.setState(rwc, StateHijacked, runHooks) return } @@ -486,11 +488,12 @@ type response struct { clenBuf [10]byte statusBuf [3]byte + // lazyCloseNotifyMu protects closeNotifyCh and closeNotifyTriggered. + lazyCloseNotifyMu sync.Mutex // closeNotifyCh is the channel returned by CloseNotify. - // TODO(bradfitz): this is currently (for Go 1.8) always - // non-nil. Make this lazily-created again as it used to be? - closeNotifyCh chan bool - didCloseNotify atomic.Bool // atomic (only false->true winner should send) + closeNotifyCh chan bool + // closeNotifyTriggered tracks prior closeNotify calls. + closeNotifyTriggered bool } func (c *response) SetReadDeadline(deadline time.Time) error { @@ -611,7 +614,7 @@ func (w *response) ReadFrom(src io.Reader) (n int64, err error) { w.cw.flush() // make sure Header is written; flush data to rwc // Now that cw has been flushed, its chunking field is guaranteed initialized. - if !w.cw.chunking && w.bodyAllowed() { + if !w.cw.chunking && w.bodyAllowed() && w.req.Method != "HEAD" { n0, err := rf.ReadFrom(src) n += n0 w.written += n0 @@ -628,9 +631,9 @@ func (w *response) ReadFrom(src io.Reader) (n int64, err error) { const debugServerConnections = false // Create new connection from rwc. -func (srv *Server) newConn(rwc net.Conn) *conn { +func (s *Server) newConn(rwc net.Conn) *conn { c := &conn{ - server: srv, + server: s, rwc: rwc, } if debugServerConnections { @@ -651,10 +654,13 @@ type readResult struct { // read sizes) with support for selectively keeping an io.Reader.Read // call blocked in a background goroutine to wait for activity and // trigger a CloseNotifier channel. +// After a Handler has hijacked the conn and exited, connReader behaves like a +// proxy for the net.Conn and the aforementioned behavior is bypassed. type connReader struct { - conn *conn + rwc net.Conn // rwc is the underlying network connection. mu sync.Mutex // guards following + conn *conn // conn is nil after handler exit. hasByte bool byteBuf [1]byte cond *sync.Cond @@ -672,6 +678,12 @@ func (cr *connReader) lock() { func (cr *connReader) unlock() { cr.mu.Unlock() } +func (cr *connReader) releaseConn() { + cr.lock() + defer cr.unlock() + cr.conn = nil +} + func (cr *connReader) startBackgroundRead() { cr.lock() defer cr.unlock() @@ -682,12 +694,12 @@ func (cr *connReader) startBackgroundRead() { return } cr.inRead = true - cr.conn.rwc.SetReadDeadline(time.Time{}) + cr.rwc.SetReadDeadline(time.Time{}) go cr.backgroundRead() } func (cr *connReader) backgroundRead() { - n, err := cr.conn.rwc.Read(cr.byteBuf[:]) + n, err := cr.rwc.Read(cr.byteBuf[:]) cr.lock() if n == 1 { cr.hasByte = true @@ -718,7 +730,7 @@ func (cr *connReader) backgroundRead() { // Ignore this error. It's the expected error from // another goroutine calling abortPendingRead. } else if err != nil { - cr.handleReadError(err) + cr.handleReadErrorLocked(err) } cr.aborted = false cr.inRead = false @@ -733,18 +745,18 @@ func (cr *connReader) abortPendingRead() { return } cr.aborted = true - cr.conn.rwc.SetReadDeadline(aLongTimeAgo) + cr.rwc.SetReadDeadline(aLongTimeAgo) for cr.inRead { cr.cond.Wait() } - cr.conn.rwc.SetReadDeadline(time.Time{}) + cr.rwc.SetReadDeadline(time.Time{}) } func (cr *connReader) setReadLimit(remain int64) { cr.remain = remain } func (cr *connReader) setInfiniteReadLimit() { cr.remain = maxInt64 } func (cr *connReader) hitReadLimit() bool { return cr.remain <= 0 } -// handleReadError is called whenever a Read from the client returns a +// handleReadErrorLocked is called whenever a Read from the client returns a // non-nil error. // // The provided non-nil err is almost always io.EOF or a "use of @@ -753,25 +765,27 @@ func (cr *connReader) hitReadLimit() bool { return cr.remain <= 0 } // development. Any error means the connection is dead and we should // down its context. // -// It may be called from multiple goroutines. -func (cr *connReader) handleReadError(_ error) { +// The caller must hold connReader.mu. +func (cr *connReader) handleReadErrorLocked(_ error) { + if cr.conn == nil { + return + } cr.conn.cancelCtx() - cr.closeNotify() -} - -// may be called from multiple goroutines. -func (cr *connReader) closeNotify() { - res := cr.conn.curReq.Load() - if res != nil && !res.didCloseNotify.Swap(true) { - res.closeNotifyCh <- true + if res := cr.conn.curReq.Load(); res != nil { + res.closeNotify() } } func (cr *connReader) Read(p []byte) (n int, err error) { cr.lock() + if cr.conn == nil { + cr.unlock() + return cr.rwc.Read(p) + } if cr.inRead { + hijacked := cr.conn.hijacked() cr.unlock() - if cr.conn.hijacked() { + if hijacked { panic("invalid Body.Read call. After hijacked, the original Request must not be used") } panic("invalid concurrent Body.Read call") @@ -795,12 +809,12 @@ func (cr *connReader) Read(p []byte) (n int, err error) { } cr.inRead = true cr.unlock() - n, err = cr.conn.rwc.Read(p) + n, err = cr.rwc.Read(p) cr.lock() cr.inRead = false if err != nil { - cr.handleReadError(err) + cr.handleReadErrorLocked(err) } cr.remain -= int64(n) cr.unlock() @@ -822,6 +836,7 @@ var copyBufPool = sync.Pool{New: func() any { return new([copyBufPoolSize]byte) func getCopyBuf() []byte { return copyBufPool.Get().(*[copyBufPoolSize]byte)[:] } + func putCopyBuf(b []byte) { if len(b) != copyBufPoolSize { panic("trying to put back buffer of the wrong size in the copyBufPool") @@ -915,15 +930,15 @@ func putBufioWriter(bw *bufio.Writer) { // This can be overridden by setting [Server.MaxHeaderBytes]. const DefaultMaxHeaderBytes = 1 << 20 // 1 MB -func (srv *Server) maxHeaderBytes() int { - if srv.MaxHeaderBytes > 0 { - return srv.MaxHeaderBytes +func (s *Server) maxHeaderBytes() int { + if s.MaxHeaderBytes > 0 { + return s.MaxHeaderBytes } return DefaultMaxHeaderBytes } -func (srv *Server) initialReadLimitSize() int64 { - return int64(srv.maxHeaderBytes()) + 4096 // bufio slop +func (s *Server) initialReadLimitSize() int64 { + return int64(s.maxHeaderBytes()) + 4096 // bufio slop } // tlsHandshakeTimeout returns the time limit permitted for the TLS @@ -931,12 +946,12 @@ func (srv *Server) initialReadLimitSize() int64 { // // It returns the minimum of any positive ReadHeaderTimeout, // ReadTimeout, or WriteTimeout. -func (srv *Server) tlsHandshakeTimeout() time.Duration { +func (s *Server) tlsHandshakeTimeout() time.Duration { var ret time.Duration for _, v := range [...]time.Duration{ - srv.ReadHeaderTimeout, - srv.ReadTimeout, - srv.WriteTimeout, + s.ReadHeaderTimeout, + s.ReadTimeout, + s.WriteTimeout, } { if v <= 0 { continue @@ -991,28 +1006,6 @@ func (ecr *expectContinueReader) Close() error { // For parsing this time format, see [ParseTime]. const TimeFormat = "Mon, 02 Jan 2006 15:04:05 GMT" -// appendTime is a non-allocating version of []byte(t.UTC().Format(TimeFormat)) -func appendTime(b []byte, t time.Time) []byte { - const days = "SunMonTueWedThuFriSat" - const months = "JanFebMarAprMayJunJulAugSepOctNovDec" - - t = t.UTC() - yy, mm, dd := t.Date() - hh, mn, ss := t.Clock() - day := days[3*t.Weekday():] - mon := months[3*(mm-1):] - - return append(b, - day[0], day[1], day[2], ',', ' ', - byte('0'+dd/10), byte('0'+dd%10), ' ', - mon[0], mon[1], mon[2], ' ', - byte('0'+yy/1000), byte('0'+(yy/100)%10), byte('0'+(yy/10)%10), byte('0'+yy%10), ' ', - byte('0'+hh/10), byte('0'+hh%10), ':', - byte('0'+mn/10), byte('0'+mn%10), ':', - byte('0'+ss/10), byte('0'+ss%10), ' ', - 'G', 'M', 'T') -} - var errTooLarge = errors.New("http: request too large") // Read next request from connection. @@ -1100,7 +1093,6 @@ func (c *conn) readRequest(ctx context.Context) (w *response, err error) { reqBody: req.Body, handlerHeader: make(Header), contentLength: -1, - closeNotifyCh: make(chan bool, 1), // We populate these ahead of time so we're not // reading from req.Header after their Handler starts @@ -1506,7 +1498,7 @@ func (cw *chunkWriter) writeHeader(p []byte) { } if !header.has("Date") { - setHeader.date = appendTime(cw.res.dateBuf[:0], time.Now()) + setHeader.date = time.Now().UTC().AppendFormat(cw.res.dateBuf[:0], TimeFormat) } if hasCL && hasTE && te != "identity" { @@ -1590,7 +1582,7 @@ func foreachHeaderElement(v string, fn func(string)) { fn(v) return } - for _, f := range strings.Split(v, ",") { + for f := range strings.SplitSeq(v, ",") { if f = textproto.TrimString(f); f != "" { fn(f) } @@ -1622,7 +1614,7 @@ func writeStatusLine(bw *bufio.Writer, is11 bool, code int, scratch []byte) { // It's illegal to call this before the header has been flushed. func (w *response) bodyAllowed() bool { if !w.wroteHeader { - panic("") + panic("net/http: bodyAllowed called before the header was written") } return bodyAllowedForStatus(w.status) } @@ -1933,6 +1925,10 @@ func isCommonNetReadError(err error) bool { return false } +type connectionStater interface { + ConnectionState() tls.ConnectionState +} + // Serve a new connection. func (c *conn) serve(ctx context.Context) { if ra := c.rwc.RemoteAddr(); ra != nil { @@ -2005,20 +2001,41 @@ func (c *conn) serve(ctx context.Context) { // HTTP/1.x from here on. + // Set Request.TLS if the conn is not a *tls.Conn, but implements ConnectionState. + if c.tlsState == nil { + if tc, ok := c.rwc.(connectionStater); ok { + c.tlsState = new(tls.ConnectionState) + *c.tlsState = tc.ConnectionState() + } + } + ctx, cancelCtx := context.WithCancel(ctx) c.cancelCtx = cancelCtx defer cancelCtx() - c.r = &connReader{conn: c} + c.r = &connReader{conn: c, rwc: c.rwc} c.bufr = newBufioReader(c.r) c.bufw = newBufioWriterSize(checkConnErrorWriter{c}, 4<<10) + protos := c.server.protocols() + if c.tlsState == nil && protos.UnencryptedHTTP2() { + if c.maybeServeUnencryptedHTTP2(ctx) { + return + } + } + if !protos.HTTP1() { + return + } + for { w, err := c.readRequest(ctx) if c.r.remain != c.server.initialReadLimitSize() { // If we read any bytes off the wire, we're active. c.setState(c.rwc, StateActive, runHooks) } + if c.server.shuttingDown() { + return + } if err != nil { const errorHeaders = "\r\nContent-Type: text/plain; charset=utf-8\r\nConnection: close\r\n\r\n" @@ -2093,6 +2110,7 @@ func (c *conn) serve(ctx context.Context) { inFlightResponse = nil w.cancelCtx() if c.hijacked() { + c.r.releaseConn() return } w.finishRequest() @@ -2132,6 +2150,70 @@ func (c *conn) serve(ctx context.Context) { } } +// unencryptedHTTP2Request is an HTTP handler that initializes +// certain uninitialized fields in its *Request. +// +// It's the unencrypted version of initALPNRequest. +type unencryptedHTTP2Request struct { + ctx context.Context + c net.Conn + h serverHandler +} + +func (h unencryptedHTTP2Request) BaseContext() context.Context { return h.ctx } + +func (h unencryptedHTTP2Request) ServeHTTP(rw ResponseWriter, req *Request) { + if req.Body == nil { + req.Body = NoBody + } + if req.RemoteAddr == "" { + req.RemoteAddr = h.c.RemoteAddr().String() + } + h.h.ServeHTTP(rw, req) +} + +// unencryptedNetConnInTLSConn is used to pass an unencrypted net.Conn to +// functions that only accept a *tls.Conn. +type unencryptedNetConnInTLSConn struct { + net.Conn // panic on all net.Conn methods + conn net.Conn +} + +func (c unencryptedNetConnInTLSConn) UnencryptedNetConn() net.Conn { + return c.conn +} + +func unencryptedTLSConn(c net.Conn) *tls.Conn { + return tls.Client(unencryptedNetConnInTLSConn{conn: c}, nil) +} + +// TLSNextProto key to use for unencrypted HTTP/2 connections. +// Not actually a TLS-negotiated protocol. +const nextProtoUnencryptedHTTP2 = "unencrypted_http2" + +func (c *conn) maybeServeUnencryptedHTTP2(ctx context.Context) bool { + fn, ok := c.server.TLSNextProto[nextProtoUnencryptedHTTP2] + if !ok { + return false + } + hasPreface := func(c *conn, preface []byte) bool { + c.r.setReadLimit(int64(len(preface)) - int64(c.bufr.Buffered())) + got, err := c.bufr.Peek(len(preface)) + c.r.setInfiniteReadLimit() + return err == nil && bytes.Equal(got, preface) + } + if !hasPreface(c, []byte("PRI * HTTP/2.0")) { + return false + } + if !hasPreface(c, []byte("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n")) { + return false + } + c.setState(c.rwc, StateActive, skipHooks) + h := unencryptedHTTP2Request{ctx, c.rwc, serverHandler{c.server}} + fn(c.server, unencryptedTLSConn(c.rwc), h) + return true +} + func (w *response) sendExpectationFailed() { // TODO(bradfitz): let ServeHTTP handlers handle // requests with non-standard expectation[s]? Seems @@ -2176,12 +2258,32 @@ func (w *response) Hijack() (rwc net.Conn, buf *bufio.ReadWriter, err error) { } func (w *response) CloseNotify() <-chan bool { + w.lazyCloseNotifyMu.Lock() + defer w.lazyCloseNotifyMu.Unlock() if w.handlerDone.Load() { panic("net/http: CloseNotify called after ServeHTTP finished") } + if w.closeNotifyCh == nil { + w.closeNotifyCh = make(chan bool, 1) + if w.closeNotifyTriggered { + w.closeNotifyCh <- true // action prior closeNotify call + } + } return w.closeNotifyCh } +func (w *response) closeNotify() { + w.lazyCloseNotifyMu.Lock() + defer w.lazyCloseNotifyMu.Unlock() + if w.closeNotifyTriggered { + return // already triggered + } + w.closeNotifyTriggered = true + if w.closeNotifyCh != nil { + w.closeNotifyCh <- true + } +} + func registerOnHitEOF(rc io.ReadCloser, fn func()) { switch v := rc.(type) { case *expectContinueReader: @@ -2288,6 +2390,8 @@ func StripPrefix(prefix string, h Handler) Handler { // Redirect replies to the request with a redirect to url, // which may be a path relative to the request path. +// Any non-ASCII characters in url will be percent-encoded, +// but existing percent encodings will not be changed. // // The provided code should be in the 3xx range and is usually // [StatusMovedPermanently], [StatusFound] or [StatusSeeOther]. @@ -2480,6 +2584,8 @@ func RedirectHandler(url string, code int) Handler { // ServeMux also takes care of sanitizing the URL request path and the Host // header, stripping the port number and redirecting any request containing . or // .. segments or repeated slashes to an equivalent, cleaner URL. +// Escaped path elements such as "%2e" for "." and "%2f" for "/" are preserved +// and aren't considered separators for request routing. // // # Compatibility // @@ -2503,11 +2609,10 @@ func RedirectHandler(url string, code int) Handler { // This change mostly affects how paths with %2F escapes adjacent to slashes are treated. // See https://go.dev/issue/21955 for details. type ServeMux struct { - mu sync.RWMutex - tree routingNode - index routingIndex - patterns []*pattern // TODO(jba): remove if possible - mux121 serveMux121 // used only when GODEBUG=httpmuxgo121=1 + mu sync.RWMutex + tree routingNode + index routingIndex + mux121 serveMux121 // used only when GODEBUG=httpmuxgo121=1 } // NewServeMux allocates and returns a new [ServeMux]. @@ -2569,7 +2674,12 @@ func stripHostPort(h string) string { // the path that will match after following the redirect. // // If there is no registered handler that applies to the request, -// Handler returns a “page not found” handler and an empty pattern. +// Handler returns a “page not found” or “method not supported” +// handler and an empty pattern. +// +// Handler does not modify its argument. In particular, it does not +// populate named path wildcards, so r.PathValue will always return +// the empty string. func (mux *ServeMux) Handler(r *Request) (h Handler, pattern string) { if use121 { return mux.mux121.findHandler(r) @@ -2610,7 +2720,7 @@ func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string, _ *patte var u *url.URL n, matches, u = mux.matchOrRedirect(host, r.Method, path, r.URL) if u != nil { - return RedirectHandler(u.String(), StatusMovedPermanently), u.Path, nil, nil + return RedirectHandler(u.String(), StatusMovedPermanently), n.pattern.String(), nil, nil } if path != escapedPath { // Redirect to cleaned path. @@ -2649,14 +2759,21 @@ func (mux *ServeMux) matchOrRedirect(host, method, path string, u *url.URL) (_ * defer mux.mu.RUnlock() n, matches := mux.tree.match(host, method, path) - // If we have an exact match, or we were asked not to try trailing-slash redirection, - // or the URL already has a trailing slash, then we're done. - if !exactMatch(n, path) && u != nil && !strings.HasSuffix(path, "/") { + // We can terminate here if any of the following is true: + // - We have an exact match already. + // - We were asked not to try trailing slash redirection. + // - The URL already has a trailing slash. + // - The URL is an empty string. + if !exactMatch(n, path) && u != nil && !strings.HasSuffix(path, "/") && path != "" { // If there is an exact match with a trailing slash, then redirect. path += "/" n2, _ := mux.tree.match(host, method, path) if exactMatch(n2, path) { - return nil, nil, &url.URL{Path: cleanPath(u.Path) + "/", RawQuery: u.RawQuery} + // It is safe to return n2 here: it is used only in the second RedirectHandler case + // of findHandler, and that method returns before it does the "n == nil" check where + // the first return value matters. We return it here only to make the pattern available + // to findHandler. + return n2, nil, &url.URL{Path: cleanPath(u.Path) + "/", RawQuery: u.RawQuery} } } return n, matches, nil @@ -2751,8 +2868,10 @@ func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { // always refers to user code. // Handle registers the handler for the given pattern. -// If the given pattern conflicts, with one that is already registered, Handle -// panics. +// If the given pattern conflicts with one that is already registered +// or if the pattern is invalid, Handle panics. +// +// See [ServeMux] for details on valid patterns and conflict rules. func (mux *ServeMux) Handle(pattern string, handler Handler) { if use121 { mux.mux121.handle(pattern, handler) @@ -2762,8 +2881,10 @@ func (mux *ServeMux) Handle(pattern string, handler Handler) { } // HandleFunc registers the handler function for the given pattern. -// If the given pattern conflicts, with one that is already registered, HandleFunc -// panics. +// If the given pattern conflicts with one that is already registered +// or if the pattern is invalid, HandleFunc panics. +// +// See [ServeMux] for details on valid patterns and conflict rules. func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { if use121 { mux.mux121.handleFunc(pattern, handler) @@ -2838,7 +2959,6 @@ func (mux *ServeMux) registerErr(patstr string, handler Handler) error { } mux.tree.addPattern(pat, handler) mux.index.addPattern(pat) - mux.patterns = append(mux.patterns, pat) return nil } @@ -2946,6 +3066,9 @@ type Server struct { // automatically closed when the function returns. // If TLSNextProto is not nil, HTTP/2 support is not enabled // automatically. + // + // Historically, TLSNextProto was used to disable HTTP/2 support. + // The Server.Protocols field now provides a simpler way to do this. TLSNextProto map[string]func(*Server, *tls.Conn, Handler) // ConnState specifies an optional callback function that is @@ -2973,6 +3096,20 @@ type Server struct { // value. ConnContext func(ctx context.Context, c net.Conn) context.Context + // HTTP2 configures HTTP/2 connections. + HTTP2 *HTTP2Config + + // Protocols is the set of protocols accepted by the server. + // + // If Protocols includes UnencryptedHTTP2, the server will accept + // unencrypted HTTP/2 connections. The server can serve both + // HTTP/1 and unencrypted HTTP/2 on the same address and port. + // + // If Protocols is nil, the default is usually HTTP/1 and HTTP/2. + // If TLSNextProto is non-nil and does not contain an "h2" entry, + // the default is HTTP/1 only. + Protocols *Protocols + inShutdown atomic.Bool // true when server is in shutdown disableKeepAlives atomic.Bool @@ -2996,23 +3133,23 @@ type Server struct { // // Close returns any error returned from closing the [Server]'s // underlying Listener(s). -func (srv *Server) Close() error { - srv.inShutdown.Store(true) - srv.mu.Lock() - defer srv.mu.Unlock() - err := srv.closeListenersLocked() - - // Unlock srv.mu while waiting for listenerGroup. - // The group Add and Done calls are made with srv.mu held, +func (s *Server) Close() error { + s.inShutdown.Store(true) + s.mu.Lock() + defer s.mu.Unlock() + err := s.closeListenersLocked() + + // Unlock s.mu while waiting for listenerGroup. + // The group Add and Done calls are made with s.mu held, // to avoid adding a new listener in the window between // us setting inShutdown above and waiting here. - srv.mu.Unlock() - srv.listenerGroup.Wait() - srv.mu.Lock() + s.mu.Unlock() + s.listenerGroup.Wait() + s.mu.Lock() - for c := range srv.activeConn { + for c := range s.activeConn { c.rwc.Close() - delete(srv.activeConn, c) + delete(s.activeConn, c) } return err } @@ -3034,7 +3171,7 @@ const shutdownPollIntervalMax = 500 * time.Millisecond // Shutdown returns the context's error, otherwise it returns any // error returned from closing the [Server]'s underlying Listener(s). // -// When Shutdown is called, [Serve], [ListenAndServe], and +// When Shutdown is called, [Serve], [ServeTLS], [ListenAndServe], and // [ListenAndServeTLS] immediately return [ErrServerClosed]. Make sure the // program doesn't exit and waits instead for Shutdown to return. // @@ -3046,16 +3183,16 @@ const shutdownPollIntervalMax = 500 * time.Millisecond // // Once Shutdown has been called on a server, it may not be reused; // future calls to methods such as Serve will return ErrServerClosed. -func (srv *Server) Shutdown(ctx context.Context) error { - srv.inShutdown.Store(true) +func (s *Server) Shutdown(ctx context.Context) error { + s.inShutdown.Store(true) - srv.mu.Lock() - lnerr := srv.closeListenersLocked() - for _, f := range srv.onShutdown { + s.mu.Lock() + lnerr := s.closeListenersLocked() + for _, f := range s.onShutdown { go f() } - srv.mu.Unlock() - srv.listenerGroup.Wait() + s.mu.Unlock() + s.listenerGroup.Wait() pollIntervalBase := time.Millisecond nextPollInterval := func() time.Duration { @@ -3072,7 +3209,7 @@ func (srv *Server) Shutdown(ctx context.Context) error { timer := time.NewTimer(nextPollInterval()) defer timer.Stop() for { - if srv.closeIdleConns() { + if s.closeIdleConns() { return lnerr } select { @@ -3089,10 +3226,10 @@ func (srv *Server) Shutdown(ctx context.Context) error { // undergone ALPN protocol upgrade or that have been hijacked. // This function should start protocol-specific graceful shutdown, // but should not wait for shutdown to complete. -func (srv *Server) RegisterOnShutdown(f func()) { - srv.mu.Lock() - srv.onShutdown = append(srv.onShutdown, f) - srv.mu.Unlock() +func (s *Server) RegisterOnShutdown(f func()) { + s.mu.Lock() + s.onShutdown = append(s.onShutdown, f) + s.mu.Unlock() } // closeIdleConns closes all idle connections and reports whether the @@ -3236,19 +3373,19 @@ func AllowQuerySemicolons(h Handler) Handler { }) } -// ListenAndServe listens on the TCP network address srv.Addr and then +// ListenAndServe listens on the TCP network address s.Addr and then // calls [Serve] to handle requests on incoming connections. // Accepted connections are configured to enable TCP keep-alives. // -// If srv.Addr is blank, ":http" is used. +// If s.Addr is blank, ":http" is used. // // ListenAndServe always returns a non-nil error. After [Server.Shutdown] or [Server.Close], // the returned error is [ErrServerClosed]. -func (srv *Server) ListenAndServe() error { - if srv.shuttingDown() { +func (s *Server) ListenAndServe() error { + if s.shuttingDown() { return ErrServerClosed } - addr := srv.Addr + addr := s.Addr if addr == "" { addr = ":http" } @@ -3256,23 +3393,26 @@ func (srv *Server) ListenAndServe() error { if err != nil { return err } - return srv.Serve(ln) + return s.Serve(ln) } var testHookServerServe func(*Server, net.Listener) // used if non-nil // shouldConfigureHTTP2ForServe reports whether Server.Serve should configure -// automatic HTTP/2. (which sets up the srv.TLSNextProto map) -func (srv *Server) shouldConfigureHTTP2ForServe() bool { - if srv.TLSConfig == nil { +// automatic HTTP/2. (which sets up the s.TLSNextProto map) +func (s *Server) shouldConfigureHTTP2ForServe() bool { + if s.TLSConfig == nil { // Compatibility with Go 1.6: // If there's no TLSConfig, it's possible that the user just // didn't set it on the http.Server, but did pass it to // tls.NewListener and passed that listener to Serve. - // So we should configure HTTP/2 (to set up srv.TLSNextProto) + // So we should configure HTTP/2 (to set up s.TLSNextProto) // in case the listener returns an "h2" *tls.Conn. return true } + if s.protocols().UnencryptedHTTP2() { + return true + } // The user specified a TLSConfig on their http.Server. // In this, case, only configure HTTP/2 if their tls.Config // explicitly mentions "h2". Otherwise http2.ConfigureServer @@ -3280,7 +3420,7 @@ func (srv *Server) shouldConfigureHTTP2ForServe() bool { // passed this tls.Config to tls.NewListener. And if they did, // it's too late anyway to fix it. It would only be potentially racy. // See Issue 15908. - return slices.Contains(srv.TLSConfig.NextProtos, http2NextProtoTLS) + return slices.Contains(s.TLSConfig.NextProtos, http2NextProtoTLS) } // ErrServerClosed is returned by the [Server.Serve], [ServeTLS], [ListenAndServe], @@ -3289,7 +3429,7 @@ var ErrServerClosed = errors.New("http: Server closed") // Serve accepts incoming connections on the Listener l, creating a // new service goroutine for each. The service goroutines read requests and -// then call srv.Handler to reply to them. +// then call s.Handler to reply to them. // // HTTP/2 support is only enabled if the Listener returns [*tls.Conn] // connections and they were configured with "h2" in the TLS @@ -3297,27 +3437,27 @@ var ErrServerClosed = errors.New("http: Server closed") // // Serve always returns a non-nil error and closes l. // After [Server.Shutdown] or [Server.Close], the returned error is [ErrServerClosed]. -func (srv *Server) Serve(l net.Listener) error { +func (s *Server) Serve(l net.Listener) error { if fn := testHookServerServe; fn != nil { - fn(srv, l) // call hook with unwrapped listener + fn(s, l) // call hook with unwrapped listener } origListener := l l = &onceCloseListener{Listener: l} defer l.Close() - if err := srv.setupHTTP2_Serve(); err != nil { + if err := s.setupHTTP2_Serve(); err != nil { return err } - if !srv.trackListener(&l, true) { + if !s.trackListener(&l, true) { return ErrServerClosed } - defer srv.trackListener(&l, false) + defer s.trackListener(&l, false) baseCtx := context.Background() - if srv.BaseContext != nil { - baseCtx = srv.BaseContext(origListener) + if s.BaseContext != nil { + baseCtx = s.BaseContext(origListener) if baseCtx == nil { panic("BaseContext returned a nil context") } @@ -3325,11 +3465,11 @@ func (srv *Server) Serve(l net.Listener) error { var tempDelay time.Duration // how long to sleep on accept failure - ctx := context.WithValue(baseCtx, ServerContextKey, srv) + ctx := context.WithValue(baseCtx, ServerContextKey, s) for { rw, err := l.Accept() if err != nil { - if srv.shuttingDown() { + if s.shuttingDown() { return ErrServerClosed } if ne, ok := err.(net.Error); ok && ne.Temporary() { @@ -3341,21 +3481,21 @@ func (srv *Server) Serve(l net.Listener) error { if max := 1 * time.Second; tempDelay > max { tempDelay = max } - srv.logf("http: Accept error: %v; retrying in %v", err, tempDelay) + s.logf("http: Accept error: %v; retrying in %v", err, tempDelay) time.Sleep(tempDelay) continue } return err } connCtx := ctx - if cc := srv.ConnContext; cc != nil { + if cc := s.ConnContext; cc != nil { connCtx = cc(connCtx, rw) if connCtx == nil { panic("ConnContext returned nil") } } tempDelay = 0 - c := srv.newConn(rw) + c := s.newConn(rw) c.setState(c.rwc, StateNew, runHooks) // before Serve can return go c.serve(connCtx) } @@ -3363,7 +3503,7 @@ func (srv *Server) Serve(l net.Listener) error { // ServeTLS accepts incoming connections on the Listener l, creating a // new service goroutine for each. The service goroutines perform TLS -// setup and then read requests, calling srv.Handler to reply to them. +// setup and then read requests, calling s.Handler to reply to them. // // Files containing a certificate and matching private key for the // server must be provided if neither the [Server]'s @@ -3375,17 +3515,15 @@ func (srv *Server) Serve(l net.Listener) error { // // ServeTLS always returns a non-nil error. After [Server.Shutdown] or [Server.Close], the // returned error is [ErrServerClosed]. -func (srv *Server) ServeTLS(l net.Listener, certFile, keyFile string) error { - // Setup HTTP/2 before srv.Serve, to initialize srv.TLSConfig +func (s *Server) ServeTLS(l net.Listener, certFile, keyFile string) error { + // Setup HTTP/2 before s.Serve, to initialize s.TLSConfig // before we clone it and create the TLS Listener. - if err := srv.setupHTTP2_ServeTLS(); err != nil { + if err := s.setupHTTP2_ServeTLS(); err != nil { return err } - config := cloneTLSConfig(srv.TLSConfig) - if !slices.Contains(config.NextProtos, "http/1.1") { - config.NextProtos = append(config.NextProtos, "http/1.1") - } + config := cloneTLSConfig(s.TLSConfig) + config.NextProtos = adjustNextProtos(config.NextProtos, s.protocols()) configHasCert := len(config.Certificates) > 0 || config.GetCertificate != nil || config.GetConfigForClient != nil if !configHasCert || certFile != "" || keyFile != "" { @@ -3398,7 +3536,66 @@ func (srv *Server) ServeTLS(l net.Listener, certFile, keyFile string) error { } tlsListener := tls.NewListener(l, config) - return srv.Serve(tlsListener) + return s.Serve(tlsListener) +} + +func (s *Server) protocols() Protocols { + if s.Protocols != nil { + return *s.Protocols // user-configured set + } + + // The historic way of disabling HTTP/2 is to set TLSNextProto to + // a non-nil map with no "h2" entry. + _, hasH2 := s.TLSNextProto["h2"] + http2Disabled := s.TLSNextProto != nil && !hasH2 + + // If GODEBUG=http2server=0, then HTTP/2 is disabled unless + // the user has manually added an "h2" entry to TLSNextProto + // (probably by using x/net/http2 directly). + if http2server.Value() == "0" && !hasH2 { + http2Disabled = true + } + + var p Protocols + p.SetHTTP1(true) // default always includes HTTP/1 + if !http2Disabled { + p.SetHTTP2(true) + } + return p +} + +// adjustNextProtos adds or removes "http/1.1" and "h2" entries from +// a tls.Config.NextProtos list, according to the set of protocols in protos. +func adjustNextProtos(nextProtos []string, protos Protocols) []string { + // Make a copy of NextProtos since it might be shared with some other tls.Config. + // (tls.Config.Clone doesn't do a deep copy.) + // + // We could avoid an allocation in the common case by checking to see if the slice + // is already in order, but this is just one small allocation per connection. + nextProtos = slices.Clone(nextProtos) + var have Protocols + nextProtos = slices.DeleteFunc(nextProtos, func(s string) bool { + switch s { + case "http/1.1": + if !protos.HTTP1() { + return true + } + have.SetHTTP1(true) + case "h2": + if !protos.HTTP2() { + return true + } + have.SetHTTP2(true) + } + return false + }) + if protos.HTTP2() && !have.HTTP2() { + nextProtos = append(nextProtos, "h2") + } + if protos.HTTP1() && !have.HTTP1() { + nextProtos = append(nextProtos, "http/1.1") + } + return nextProtos } // trackListener adds or removes a net.Listener to the set of tracked @@ -3469,15 +3666,15 @@ func (s *Server) shuttingDown() bool { // By default, keep-alives are always enabled. Only very // resource-constrained environments or servers in the process of // shutting down should disable them. -func (srv *Server) SetKeepAlivesEnabled(v bool) { +func (s *Server) SetKeepAlivesEnabled(v bool) { if v { - srv.disableKeepAlives.Store(false) + s.disableKeepAlives.Store(false) return } - srv.disableKeepAlives.Store(true) + s.disableKeepAlives.Store(true) // Close idle HTTP/1 conns: - srv.closeIdleConns() + s.closeIdleConns() // TODO: Issue 26303: close HTTP/2 conns as soon as they become idle. } @@ -3524,7 +3721,7 @@ func ListenAndServeTLS(addr, certFile, keyFile string, handler Handler) error { return server.ListenAndServeTLS(certFile, keyFile) } -// ListenAndServeTLS listens on the TCP network address srv.Addr and +// ListenAndServeTLS listens on the TCP network address s.Addr and // then calls [ServeTLS] to handle requests on incoming TLS connections. // Accepted connections are configured to enable TCP keep-alives. // @@ -3535,15 +3732,15 @@ func ListenAndServeTLS(addr, certFile, keyFile string, handler Handler) error { // concatenation of the server's certificate, any intermediates, and // the CA's certificate. // -// If srv.Addr is blank, ":https" is used. +// If s.Addr is blank, ":https" is used. // // ListenAndServeTLS always returns a non-nil error. After [Server.Shutdown] or // [Server.Close], the returned error is [ErrServerClosed]. -func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { - if srv.shuttingDown() { +func (s *Server) ListenAndServeTLS(certFile, keyFile string) error { + if s.shuttingDown() { return ErrServerClosed } - addr := srv.Addr + addr := s.Addr if addr == "" { addr = ":https" } @@ -3555,55 +3752,61 @@ func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { defer ln.Close() - return srv.ServeTLS(ln, certFile, keyFile) + return s.ServeTLS(ln, certFile, keyFile) } // setupHTTP2_ServeTLS conditionally configures HTTP/2 on -// srv and reports whether there was an error setting it up. If it is +// s and reports whether there was an error setting it up. If it is // not configured for policy reasons, nil is returned. -func (srv *Server) setupHTTP2_ServeTLS() error { - srv.nextProtoOnce.Do(srv.onceSetNextProtoDefaults) - return srv.nextProtoErr +func (s *Server) setupHTTP2_ServeTLS() error { + s.nextProtoOnce.Do(s.onceSetNextProtoDefaults) + return s.nextProtoErr } // setupHTTP2_Serve is called from (*Server).Serve and conditionally -// configures HTTP/2 on srv using a more conservative policy than +// configures HTTP/2 on s using a more conservative policy than // setupHTTP2_ServeTLS because Serve is called after tls.Listen, // and may be called concurrently. See shouldConfigureHTTP2ForServe. // // The tests named TestTransportAutomaticHTTP2* and // TestConcurrentServerServe in server_test.go demonstrate some // of the supported use cases and motivations. -func (srv *Server) setupHTTP2_Serve() error { - srv.nextProtoOnce.Do(srv.onceSetNextProtoDefaults_Serve) - return srv.nextProtoErr +func (s *Server) setupHTTP2_Serve() error { + s.nextProtoOnce.Do(s.onceSetNextProtoDefaults_Serve) + return s.nextProtoErr } -func (srv *Server) onceSetNextProtoDefaults_Serve() { - if srv.shouldConfigureHTTP2ForServe() { - srv.onceSetNextProtoDefaults() +func (s *Server) onceSetNextProtoDefaults_Serve() { + if s.shouldConfigureHTTP2ForServe() { + s.onceSetNextProtoDefaults() } } var http2server = godebug.New("http2server") // onceSetNextProtoDefaults configures HTTP/2, if the user hasn't -// configured otherwise. (by setting srv.TLSNextProto non-nil) -// It must only be called via srv.nextProtoOnce (use srv.setupHTTP2_*). -func (srv *Server) onceSetNextProtoDefaults() { +// configured otherwise. (by setting s.TLSNextProto non-nil) +// It must only be called via s.nextProtoOnce (use s.setupHTTP2_*). +func (s *Server) onceSetNextProtoDefaults() { if omitBundledHTTP2 { return } + p := s.protocols() + if !p.HTTP2() && !p.UnencryptedHTTP2() { + return + } if http2server.Value() == "0" { http2server.IncNonDefault() return } - // Enable HTTP/2 by default if the user hasn't otherwise - // configured their TLSNextProto map. - if srv.TLSNextProto == nil { - conf := &http2Server{} - srv.nextProtoErr = http2ConfigureServer(srv, conf) + if _, ok := s.TLSNextProto["h2"]; ok { + // TLSNextProto already contains an HTTP/2 implementation. + // The user probably called golang.org/x/net/http2.ConfigureServer + // to add it. + return } + conf := &http2Server{} + s.nextProtoErr = http2ConfigureServer(s, conf) } // TimeoutHandler returns a [Handler] that runs h with the given time limit. @@ -3677,9 +3880,7 @@ func (h *timeoutHandler) ServeHTTP(w ResponseWriter, r *Request) { tw.mu.Lock() defer tw.mu.Unlock() dst := w.Header() - for k, vv := range tw.h { - dst[k] = vv - } + maps.Copy(dst, tw.h) if !tw.wroteHeader { tw.code = StatusOK } diff --git a/src/net/http/server_test.go b/src/net/http/server_test.go index f4aafc853bd5d6..832f9688b63d9c 100644 --- a/src/net/http/server_test.go +++ b/src/net/http/server_test.go @@ -97,6 +97,7 @@ func TestFindHandler(t *testing.T) { {"GET", "/foo/x", "&http.handler{i:2}"}, {"GET", "/bar/x", "&http.handler{i:4}"}, {"GET", "/bar", `&http.redirectHandler{url:"/bar/", code:301}`}, + {"CONNECT", "", "(http.HandlerFunc)(.*)"}, {"CONNECT", "/", "&http.handler{i:1}"}, {"CONNECT", "//", "&http.handler{i:1}"}, {"CONNECT", "//foo", "&http.handler{i:5}"}, @@ -112,7 +113,7 @@ func TestFindHandler(t *testing.T) { r.URL = &url.URL{Path: test.path} gotH, _, _, _ := mux.findHandler(&r) got := fmt.Sprintf("%#v", gotH) - if got != test.wantHandler { + if !regexp.MustCompile(test.wantHandler).MatchString(got) { t.Errorf("%s %q: got %q, want %q", test.method, test.path, got, test.wantHandler) } } diff --git a/src/net/http/socks_bundle.go b/src/net/http/socks_bundle.go index 776b03d941a405..862e5fa2a5165d 100644 --- a/src/net/http/socks_bundle.go +++ b/src/net/http/socks_bundle.go @@ -453,7 +453,7 @@ func (up *socksUsernamePassword) Authenticate(ctx context.Context, rw io.ReadWri b = append(b, up.Username...) b = append(b, byte(len(up.Password))) b = append(b, up.Password...) - // TODO(mikio): handle IO deadlines and cancelation if + // TODO(mikio): handle IO deadlines and cancellation if // necessary if _, err := rw.Write(b); err != nil { return err diff --git a/src/net/http/transfer.go b/src/net/http/transfer.go index 5a3c6ceff57808..675551287fa3d6 100644 --- a/src/net/http/transfer.go +++ b/src/net/http/transfer.go @@ -11,6 +11,7 @@ import ( "fmt" "internal/godebug" "io" + "maps" "net/http/httptrace" "net/http/internal" "net/http/internal/ascii" @@ -350,7 +351,7 @@ func (t *transferWriter) writeBody(w io.Writer) (err error) { // nopCloser or readTrackingBody. This is to ensure that we can take advantage of // OS-level optimizations in the event that the body is an // *os.File. - if t.Body != nil { + if !t.ResponseToHEAD && t.Body != nil { var body = t.unwrapBody() if chunked(t.TransferEncoding) { if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse { @@ -392,7 +393,7 @@ func (t *transferWriter) writeBody(w io.Writer) (err error) { t.ContentLength, ncopy) } - if chunked(t.TransferEncoding) { + if !t.ResponseToHEAD && chunked(t.TransferEncoding) { // Write Trailer header if t.Trailer != nil { if err := t.Trailer.Write(w); err != nil { @@ -954,9 +955,7 @@ func mergeSetHeader(dst *Header, src Header) { *dst = src return } - for k, vv := range src { - (*dst)[k] = vv - } + maps.Copy(*dst, src) } // unreadDataSizeLocked returns the number of bytes of unread input. diff --git a/src/net/http/transport.go b/src/net/http/transport.go index da9163a27ae6ff..a560765d331d65 100644 --- a/src/net/http/transport.go +++ b/src/net/http/transport.go @@ -11,6 +11,7 @@ package http import ( "bufio" + "compress/flate" "compress/gzip" "container/list" "context" @@ -20,6 +21,7 @@ import ( "internal/godebug" "io" "log" + "maps" "net" "net/http/httptrace" "net/http/internal/ascii" @@ -75,8 +77,7 @@ const DefaultMaxIdleConnsPerHost = 2 // Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2 // for HTTPS URLs, depending on whether the server supports HTTP/2, // and how the Transport is configured. The [DefaultTransport] supports HTTP/2. -// To explicitly enable HTTP/2 on a transport, use golang.org/x/net/http2 -// and call ConfigureTransport. See the package docs for more about HTTP/2. +// To explicitly enable HTTP/2 on a transport, set [Transport.Protocols]. // // Responses with status codes in the 1xx range are either handled // automatically (100 expect-continue) or ignored. The one @@ -248,6 +249,9 @@ type Transport struct { // must return a RoundTripper that then handles the request. // If TLSNextProto is not nil, HTTP/2 support is not enabled // automatically. + // + // Historically, TLSNextProto was used to disable HTTP/2 support. + // The Transport.Protocols field now provides a simpler way to do this. TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper // ProxyConnectHeader optionally specifies headers to send to @@ -293,6 +297,19 @@ type Transport struct { // To use a custom dialer or TLS config and still attempt HTTP/2 // upgrades, set this to true. ForceAttemptHTTP2 bool + + // HTTP2 configures HTTP/2 connections. + HTTP2 *HTTP2Config + + // Protocols is the set of protocols supported by the transport. + // + // If Protocols includes UnencryptedHTTP2 and does not include HTTP1, + // the transport will use unencrypted HTTP/2 for requests for http:// URLs. + // + // If Protocols is nil, the default is usually HTTP/1 only. + // If ForceAttemptHTTP2 is true, or if TLSNextProto contains an "h2" entry, + // the default is HTTP/1 and HTTP/2. + Protocols *Protocols } func (t *Transport) writeBufferSize() int { @@ -309,6 +326,13 @@ func (t *Transport) readBufferSize() int { return 4 << 10 } +func (t *Transport) maxHeaderResponseSize() int64 { + if t.MaxResponseHeaderBytes > 0 { + return t.MaxResponseHeaderBytes + } + return 10 << 20 // conservative default; same as http2 +} + // Clone returns a deep copy of t's exported fields. func (t *Transport) Clone() *Transport { t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) @@ -338,10 +362,18 @@ func (t *Transport) Clone() *Transport { if t.TLSClientConfig != nil { t2.TLSClientConfig = t.TLSClientConfig.Clone() } + if t.HTTP2 != nil { + t2.HTTP2 = &HTTP2Config{} + *t2.HTTP2 = *t.HTTP2 + } + if t.Protocols != nil { + t2.Protocols = &Protocols{} + *t2.Protocols = *t.Protocols + } if !t.tlsNextProtoWasNil { - npm := map[string]func(authority string, c *tls.Conn) RoundTripper{} - for k, v := range t.TLSNextProto { - npm[k] = v + npm := maps.Clone(t.TLSNextProto) + if npm == nil { + npm = make(map[string]func(authority string, c *tls.Conn) RoundTripper) } t2.TLSNextProto = npm } @@ -388,18 +420,12 @@ func (t *Transport) onceSetNextProtoDefaults() { } } - if t.TLSNextProto != nil { - // This is the documented way to disable http2 on a - // Transport. + if _, ok := t.TLSNextProto["h2"]; ok { + // There's an existing HTTP/2 implementation installed. return } - if !t.ForceAttemptHTTP2 && (t.TLSClientConfig != nil || t.Dial != nil || t.DialContext != nil || t.hasCustomTLSDialer()) { - // Be conservative and don't automatically enable - // http2 if they've specified a custom TLS config or - // custom dialers. Let them opt-in themselves via - // http2.ConfigureTransport so we don't surprise them - // by modifying their tls.Config. Issue 14275. - // However, if ForceAttemptHTTP2 is true, it overrides the above checks. + protocols := t.protocols() + if !protocols.HTTP2() && !protocols.UnencryptedHTTP2() { return } if omitBundledHTTP2 { @@ -426,6 +452,40 @@ func (t *Transport) onceSetNextProtoDefaults() { t2.MaxHeaderListSize = uint32(limit1) } } + + // Server.ServeTLS clones the tls.Config before modifying it. + // Transport doesn't. We may want to make the two consistent some day. + // + // http2configureTransport will have already set NextProtos, but adjust it again + // here to remove HTTP/1.1 if the user has disabled it. + t.TLSClientConfig.NextProtos = adjustNextProtos(t.TLSClientConfig.NextProtos, protocols) +} + +func (t *Transport) protocols() Protocols { + if t.Protocols != nil { + return *t.Protocols // user-configured set + } + var p Protocols + p.SetHTTP1(true) // default always includes HTTP/1 + switch { + case t.TLSNextProto != nil: + // Setting TLSNextProto to an empty map is a documented way + // to disable HTTP/2 on a Transport. + if t.TLSNextProto["h2"] != nil { + p.SetHTTP2(true) + } + case !t.ForceAttemptHTTP2 && (t.TLSClientConfig != nil || t.Dial != nil || t.DialContext != nil || t.hasCustomTLSDialer()): + // Be conservative and don't automatically enable + // http2 if they've specified a custom TLS config or + // custom dialers. Let them opt-in themselves via + // Transport.Protocols.SetHTTP2(true) so we don't surprise them + // by modifying their tls.Config. Issue 14275. + // However, if ForceAttemptHTTP2 is true, it overrides the above checks. + case http2client.Value() == "0": + default: + p.SetHTTP2(true) + } + return p } // ProxyFromEnvironment returns the URL of the proxy to use for a @@ -670,7 +730,7 @@ func (t *Transport) roundTrip(req *Request) (_ *Response, err error) { if e, ok := err.(transportReadFromServerError); ok { err = e.err } - if b, ok := req.Body.(*readTrackingBody); ok && !b.didClose { + if b, ok := req.Body.(*readTrackingBody); ok && !b.didClose.Load() { // Issue 49621: Close the request body if pconn.roundTrip // didn't do so already. This can happen if the pconn // write loop exits without reading the write request. @@ -700,8 +760,8 @@ var errCannotRewind = errors.New("net/http: cannot rewind body after connection type readTrackingBody struct { io.ReadCloser - didRead bool - didClose bool + didRead bool // not atomic.Bool because only one goroutine (the user's) should be accessing + didClose atomic.Bool } func (r *readTrackingBody) Read(data []byte) (int, error) { @@ -710,7 +770,9 @@ func (r *readTrackingBody) Read(data []byte) (int, error) { } func (r *readTrackingBody) Close() error { - r.didClose = true + if !r.didClose.CompareAndSwap(false, true) { + return nil + } return r.ReadCloser.Close() } @@ -732,10 +794,10 @@ func setupRewindBody(req *Request) *Request { // rewindBody takes care of closing req.Body when appropriate // (in all cases except when rewindBody returns req unmodified). func rewindBody(req *Request) (rewound *Request, err error) { - if req.Body == nil || req.Body == NoBody || (!req.Body.(*readTrackingBody).didRead && !req.Body.(*readTrackingBody).didClose) { + if req.Body == nil || req.Body == NoBody || (!req.Body.(*readTrackingBody).didRead && !req.Body.(*readTrackingBody).didClose.Load()) { return req, nil // nothing to rewind } - if !req.Body.(*readTrackingBody).didClose { + if !req.Body.(*readTrackingBody).didClose.Load() { req.closeBody() } if req.GetBody == nil { @@ -820,9 +882,9 @@ func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) { if _, exists := oldMap[scheme]; exists { panic("protocol " + scheme + " already registered") } - newMap := make(map[string]RoundTripper) - for k, v := range oldMap { - newMap[k] = v + newMap := maps.Clone(oldMap) + if newMap == nil { + newMap = make(map[string]RoundTripper) } newMap[scheme] = rt t.altProto.Store(newMap) @@ -1306,7 +1368,7 @@ func (w *wantConn) tryDeliver(pc *persistConn, err error, idleAt time.Time) bool // cancel marks w as no longer wanting a result (for example, due to cancellation). // If a connection has been delivered already, cancel returns it with t.putOrCloseIdleConn. -func (w *wantConn) cancel(t *Transport, err error) { +func (w *wantConn) cancel(t *Transport) { w.mu.Lock() var pc *persistConn if w.done { @@ -1320,7 +1382,10 @@ func (w *wantConn) cancel(t *Transport, err error) { w.done = true w.mu.Unlock() - if pc != nil { + // HTTP/2 connections (pc.alt != nil) aren't removed from the idle pool on use, + // and should not be added back here. If the pconn isn't in the idle pool, + // it's because we removed it due to an error. + if pc != nil && pc.alt == nil { t.putOrCloseIdleConn(pc) } } @@ -1455,7 +1520,7 @@ func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (_ *persis } defer func() { if err != nil { - w.cancel(t, err) + w.cancel(t) } }() @@ -1817,7 +1882,7 @@ func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (pconn *pers } // Okay to use and discard buffered reader here, because // TLS server will not speak until spoken to. - br := bufio.NewReader(conn) + br := bufio.NewReader(&io.LimitedReader{R: conn, N: t.maxHeaderResponseSize()}) resp, err = ReadResponse(br, connectReq) }() select { @@ -1857,6 +1922,24 @@ func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (pconn *pers } } + // Possible unencrypted HTTP/2 with prior knowledge. + unencryptedHTTP2 := pconn.tlsState == nil && + t.Protocols != nil && + t.Protocols.UnencryptedHTTP2() && + !t.Protocols.HTTP1() + if unencryptedHTTP2 { + next, ok := t.TLSNextProto[nextProtoUnencryptedHTTP2] + if !ok { + return nil, errors.New("http: Transport does not support unencrypted HTTP/2") + } + alt := next(cm.targetAddr, unencryptedTLSConn(pconn.conn)) + if e, ok := alt.(erringRoundTripper); ok { + // pconn.conn was closed by next (http2configureTransports.upgradeFn). + return nil, e.RoundTripErr() + } + return &persistConn{t: t, cacheKey: pconn.cacheKey, alt: alt}, nil + } + if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" { if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok { alt := next(cm.targetAddr, pconn.conn.(*tls.Conn)) @@ -2036,10 +2119,7 @@ type persistConn struct { } func (pc *persistConn) maxHeaderResponseSize() int64 { - if v := pc.t.MaxResponseHeaderBytes; v != 0 { - return v - } - return 10 << 20 // conservative default; same as http2 + return pc.t.maxHeaderResponseSize() } func (pc *persistConn) Read(p []byte) (n int, err error) { @@ -2387,8 +2467,6 @@ func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTr trace.GotFirstResponseByte() } } - num1xx := 0 // number of informational 1xx headers received - const max1xxResponses = 5 // arbitrary bound on number of informational responses continueCh := rc.continueCh for { @@ -2408,15 +2486,18 @@ func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTr // treat 101 as a terminal status, see issue 26161 is1xxNonTerminal := is1xx && resCode != StatusSwitchingProtocols if is1xxNonTerminal { - num1xx++ - if num1xx > max1xxResponses { - return nil, errors.New("net/http: too many 1xx informational responses") - } - pc.readLimit = pc.maxHeaderResponseSize() // reset the limit if trace != nil && trace.Got1xxResponse != nil { if err := trace.Got1xxResponse(resCode, textproto.MIMEHeader(resp.Header)); err != nil { return nil, err } + // If the 1xx response was delivered to the user, + // then they're responsible for limiting the number of + // responses. Reset the header limit. + // + // If the user didn't examine the 1xx response, then we + // limit the size of all headers (including both 1xx + // and the final response) to maxHeaderResponseSize. + pc.readLimit = pc.maxHeaderResponseSize() // reset the limit } continue } @@ -2504,6 +2585,13 @@ func (b *readWriteCloserBody) Read(p []byte) (n int, err error) { return b.ReadWriteCloser.Read(p) } +func (b *readWriteCloserBody) CloseWrite() error { + if cw, ok := b.ReadWriteCloser.(interface{ CloseWrite() error }); ok { + return cw.CloseWrite() + } + return fmt.Errorf("CloseWrite: %w", ErrNotSupported) +} + // nothingWrittenError wraps a write errors which ended up writing zero bytes. type nothingWrittenError struct { error @@ -2853,11 +2941,17 @@ func (pc *persistConn) closeLocked(err error) { pc.mutateHeaderFunc = nil } -var portMap = map[string]string{ - "http": "80", - "https": "443", - "socks5": "1080", - "socks5h": "1080", +func schemePort(scheme string) string { + switch scheme { + case "http": + return "80" + case "https": + return "443" + case "socks5", "socks5h": + return "1080" + default: + return "" + } } func idnaASCIIFromURL(url *url.URL) string { @@ -2872,7 +2966,7 @@ func idnaASCIIFromURL(url *url.URL) string { func canonicalAddr(url *url.URL) string { port := url.Port() if port == "" { - port = portMap[url.Scheme] + port = schemePort(url.Scheme) } return net.JoinHostPort(idnaASCIIFromURL(url), port) } @@ -2898,6 +2992,7 @@ type bodyEOFSignal struct { } var errReadOnClosedResBody = errors.New("http: read on closed response body") +var errConcurrentReadOnResBody = errors.New("http: concurrent read on response body") func (es *bodyEOFSignal) Read(p []byte) (n int, err error) { es.mu.Lock() @@ -2947,37 +3042,98 @@ func (es *bodyEOFSignal) condfn(err error) error { } // gzipReader wraps a response body so it can lazily -// call gzip.NewReader on the first call to Read +// get gzip.Reader from the pool on the first call to Read. +// After Close is called it puts gzip.Reader to the pool immediately +// if there is no Read in progress or later when Read completes. type gzipReader struct { _ incomparable body *bodyEOFSignal // underlying HTTP/1 response body framing - zr *gzip.Reader // lazily-initialized gzip reader - zerr error // any error from gzip.NewReader; sticky + mu sync.Mutex // guards zr and zerr + zr *gzip.Reader // stores gzip reader from the pool between reads + zerr error // sticky gzip reader init error or sentinel value to detect concurrent read and read after close } -func (gz *gzipReader) Read(p []byte) (n int, err error) { +type eofReader struct{} + +func (eofReader) Read([]byte) (int, error) { return 0, io.EOF } +func (eofReader) ReadByte() (byte, error) { return 0, io.EOF } + +var gzipPool = sync.Pool{New: func() any { return new(gzip.Reader) }} + +// gzipPoolGet gets a gzip.Reader from the pool and resets it to read from r. +func gzipPoolGet(r io.Reader) (*gzip.Reader, error) { + zr := gzipPool.Get().(*gzip.Reader) + if err := zr.Reset(r); err != nil { + gzipPoolPut(zr) + return nil, err + } + return zr, nil +} + +// gzipPoolPut puts a gzip.Reader back into the pool. +func gzipPoolPut(zr *gzip.Reader) { + // Reset will allocate bufio.Reader if we pass it anything + // other than a flate.Reader, so ensure that it's getting one. + var r flate.Reader = eofReader{} + zr.Reset(r) + gzipPool.Put(zr) +} + +// acquire returns a gzip.Reader for reading response body. +// The reader must be released after use. +func (gz *gzipReader) acquire() (*gzip.Reader, error) { + gz.mu.Lock() + defer gz.mu.Unlock() + if gz.zerr != nil { + return nil, gz.zerr + } if gz.zr == nil { - if gz.zerr == nil { - gz.zr, gz.zerr = gzip.NewReader(gz.body) - } + gz.zr, gz.zerr = gzipPoolGet(gz.body) if gz.zerr != nil { - return 0, gz.zerr + return nil, gz.zerr } } + ret := gz.zr + gz.zr, gz.zerr = nil, errConcurrentReadOnResBody + return ret, nil +} + +// release returns the gzip.Reader to the pool if Close was called during Read. +func (gz *gzipReader) release(zr *gzip.Reader) { + gz.mu.Lock() + defer gz.mu.Unlock() + if gz.zerr == errConcurrentReadOnResBody { + gz.zr, gz.zerr = zr, nil + } else { // errReadOnClosedResBody + gzipPoolPut(zr) + } +} - gz.body.mu.Lock() - if gz.body.closed { - err = errReadOnClosedResBody +// close returns the gzip.Reader to the pool immediately or +// signals release to do so after Read completes. +func (gz *gzipReader) close() { + gz.mu.Lock() + defer gz.mu.Unlock() + if gz.zerr == nil && gz.zr != nil { + gzipPoolPut(gz.zr) + gz.zr = nil } - gz.body.mu.Unlock() + gz.zerr = errReadOnClosedResBody +} +func (gz *gzipReader) Read(p []byte) (n int, err error) { + zr, err := gz.acquire() if err != nil { return 0, err } - return gz.zr.Read(p) + defer gz.release(zr) + + return zr.Read(p) } func (gz *gzipReader) Close() error { + gz.close() + return gz.body.Close() } diff --git a/src/net/http/transport_dial_test.go b/src/net/http/transport_dial_test.go index 39e35cec55620e..086039ece913d2 100644 --- a/src/net/http/transport_dial_test.go +++ b/src/net/http/transport_dial_test.go @@ -6,73 +6,360 @@ package http_test import ( "context" + "crypto/tls" + "errors" "io" "net" "net/http" "net/http/httptrace" + "strings" + "sync" "testing" + "testing/synctest" ) +// Successive requests use the same HTTP/1 connection. func TestTransportPoolConnReusePriorConnection(t *testing.T) { - dt := newTransportDialTester(t, http1Mode) + synctest.Test(t, func(t *testing.T) { + dt := newTransportDialTester(t, http1Mode) - // First request creates a new connection. - rt1 := dt.roundTrip() - c1 := dt.wantDial() - c1.finish(nil) - rt1.wantDone(c1) - rt1.finish() + // First request creates a new connection. + rt1 := dt.roundTrip() + c1 := dt.wantDial() + c1.finish(nil) + rt1.wantDone(c1, "HTTP/1.1") + rt1.finish() - // Second request reuses the first connection. - rt2 := dt.roundTrip() - rt2.wantDone(c1) - rt2.finish() + // Second request reuses the first connection. + rt2 := dt.roundTrip() + rt2.wantDone(c1, "HTTP/1.1") + rt2.finish() + }) } +// Two HTTP/1 requests made at the same time use different connections. func TestTransportPoolConnCannotReuseConnectionInUse(t *testing.T) { - dt := newTransportDialTester(t, http1Mode) + synctest.Test(t, func(t *testing.T) { + dt := newTransportDialTester(t, http1Mode) + + // First request creates a new connection. + rt1 := dt.roundTrip() + c1 := dt.wantDial() + c1.finish(nil) + rt1.wantDone(c1, "HTTP/1.1") + + // Second request is made while the first request is still using its connection, + // so it goes on a new connection. + rt2 := dt.roundTrip() + c2 := dt.wantDial() + c2.finish(nil) + rt2.wantDone(c2, "HTTP/1.1") + }) +} + +// When an HTTP/2 connection is at its stream limit +// a new request is made on a new connection. +func testTransportPoolConnHTTP2NoStrictMaxConcurrentRequests(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + dt := newTransportDialTester(t, http2Mode, func(srv *http.Server) { + srv.HTTP2 = &http.HTTP2Config{ + MaxConcurrentStreams: 2, + } + }) + + // First request dials an HTTP/2 connection. + rt1 := dt.roundTrip() + c1 := dt.wantDial() + c1.finish(nil) + rt1.wantDone(c1, "HTTP/2.0") + + // Second request uses the existing connection. + rt2 := dt.roundTrip() + rt2.wantDone(c1, "HTTP/2.0") + + // Third request creates a new connection + rt3 := dt.roundTrip() + c2 := dt.wantDial() + c2.finish(nil) + rt3.wantDone(c2, "HTTP/2.0") + + rt1.finish() + rt2.finish() + rt3.finish() + + // With slots available on both connections, we prefer the oldest. + rt4 := dt.roundTrip() + rt4.wantDone(c1, "HTTP/2.0") + rt5 := dt.roundTrip() + rt5.wantDone(c1, "HTTP/2.0") + rt6 := dt.roundTrip() + rt6.wantDone(c2, "HTTP/2.0") + rt4.finish() + rt5.finish() + rt6.finish() + }) +} - // First request creates a new connection. - rt1 := dt.roundTrip() - c1 := dt.wantDial() - c1.finish(nil) - rt1.wantDone(c1) +// When an HTTP/2 connection is at its stream limit +// and StrictMaxConcurrentRequests = true, +// a new request waits for a slot on the existing connection. +func TestTransportPoolConnHTTP2StrictMaxConcurrentRequests(t *testing.T) { + t.Skip("skipped until h2_bundle.go includes support for StrictMaxConcurrentRequests") - // Second request is made while the first request is still using its connection, - // so it goes on a new connection. - rt2 := dt.roundTrip() - c2 := dt.wantDial() - c2.finish(nil) - rt2.wantDone(c2) + synctest.Test(t, func(t *testing.T) { + dt := newTransportDialTester(t, http2Mode, func(srv *http.Server) { + srv.HTTP2.MaxConcurrentStreams = 2 + }, func(tr *http.Transport) { + tr.HTTP2 = &http.HTTP2Config{ + StrictMaxConcurrentRequests: true, + } + }) + + // First request dials an HTTP/2 connection. + rt1 := dt.roundTrip() + c1 := dt.wantDial() + c1.finish(nil) + rt1.wantDone(c1, "HTTP/2.0") + + // Second request uses the existing connection. + rt2 := dt.roundTrip() + rt2.wantDone(c1, "HTTP/2.0") + + // Third request blocks waiting for a slot on the existing connection. + rt3 := dt.roundTrip() + + // First request finishing unblocks the thirrd. + rt1.finish() + rt3.wantDone(c1, "HTTP/2.0") + + rt2.finish() + rt3.finish() + }) } +// A new request made while an HTTP/2 dial is in progress will start a second dial. +func TestTransportPoolConnHTTP2Startup(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + dt := newTransportDialTester(t, http2Mode, func(srv *http.Server) {}) + + // Two requests start. + // Since the second request starts before the first dial finishes, it starts a second dial. + rt1 := dt.roundTrip() + rt2 := dt.roundTrip() + c1 := dt.wantDial() + c2 := dt.wantDial() + + // Both requests use the conn of the first dial to complete. + c1.finish(nil) + rt1.wantDone(c1, "HTTP/2.0") + rt2.wantDone(c1, "HTTP/2.0") + + rt1.finish() + rt2.finish() + c2.finish(nil) + }) +} + +// When a request finishes using an HTTP/1 connection, +// a pending request attempting to dial a new connection will use the newly-available one. func TestTransportPoolConnConnectionBecomesAvailableDuringDial(t *testing.T) { - dt := newTransportDialTester(t, http1Mode) - - // First request creates a new connection. - rt1 := dt.roundTrip() - c1 := dt.wantDial() - c1.finish(nil) - rt1.wantDone(c1) - - // Second request is made while the first request is still using its connection. - // The first connection completes while the second Dial is in progress, so the - // second request uses the first connection. - rt2 := dt.roundTrip() - c2 := dt.wantDial() - rt1.finish() - rt2.wantDone(c1) - - // This section is a bit overfitted to the current Transport implementation: - // A third request starts. We have an in-progress dial that was started by rt2, - // but this new request (rt3) is going to ignore it and make a dial of its own. - // rt3 will use the first of these dials that completes. - rt3 := dt.roundTrip() - c3 := dt.wantDial() - c2.finish(nil) - rt3.wantDone(c2) - - c3.finish(nil) + synctest.Test(t, func(t *testing.T) { + dt := newTransportDialTester(t, http1Mode) + + // First request creates a new connection. + rt1 := dt.roundTrip() + c1 := dt.wantDial() + c1.finish(nil) + rt1.wantDone(c1, "HTTP/1.1") + + // Second request is made while the first request is still using its connection. + // The first connection completes while the second Dial is in progress, so the + // second request uses the first connection. + rt2 := dt.roundTrip() + c2 := dt.wantDial() + rt1.finish() + rt2.wantDone(c1, "HTTP/1.1") + + // This section is a bit overfitted to the current Transport implementation: + // A third request starts. We have an in-progress dial that was started by rt2, + // but this new request (rt3) is going to ignore it and make a dial of its own. + // rt3 will use the first of these dials that completes. + rt3 := dt.roundTrip() + c3 := dt.wantDial() + c2.finish(nil) + rt3.wantDone(c2, "HTTP/1.1") + + c3.finish(nil) + }) +} + +// Connections are not reused when DisableKeepAlives = true. +func TestTransportPoolDisableKeepAlives(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + dt := newTransportDialTester(t, http1Mode, func(tr *http.Transport) { + tr.DisableKeepAlives = true + }) + + // Two requests, each uses a separate connection. + for range 2 { + rt := dt.roundTrip() + c := dt.wantDial() + c.finish(nil) + rt.wantDone(c, "HTTP/1.1") + rt.finish() + } + }) +} + +// Canceling a request before its connection is created returns the conn to the pool. +func TestTransportPoolCancelRequestReusesConn(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + dt := newTransportDialTester(t, http1Mode) + + // First request is canceled before its connection is created. + rt1 := dt.roundTrip() + c1 := dt.wantDial() + rt1.cancel() + rt1.wantError() + + // Second request uses the first connection. + rt2 := dt.roundTrip() + c2 := dt.wantDial() + c1.finish(nil) // first dial finishes + rt2.wantDone(c1, "HTTP/1.1") + rt2.finish() + + c2.finish(nil) // second dial finishes + }) +} + +// Connections are not reused when DisableKeepAlives = true. +func TestTransportPoolCancelRequestWithDisableKeepAlives(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + dt := newTransportDialTester(t, http1Mode, func(tr *http.Transport) { + tr.DisableKeepAlives = true + }) + + // First request is canceled before its connection is created. + rt1 := dt.roundTrip() + c1 := dt.wantDial() + rt1.cancel() + rt1.wantError() + + // Dial finishes. DisableKeepAlives = true, so we discard the connection. + c1.finish(nil) + + // Second request is made on a new connection. + rt2 := dt.roundTrip() + c2 := dt.wantDial() + c2.finish(nil) + rt2.wantDone(c2, "HTTP/1.1") + rt2.finish() + }) +} + +// Connections are not reused after an error. +func TestTransportPoolConnectionBroken(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + dt := newTransportDialTester(t, http1Mode) + + // First request creates a new connection. + // The connection breaks while sending the response. + rt1 := dt.roundTrip() + c1 := dt.wantDial() + c1.finish(nil) + rt1.wantDone(c1, "HTTP/1.1") + c1.fakeNetConn.Close() // break the connection + rt1.finish() + + // Second request is made on a new connection, since the first is broken. + rt2 := dt.roundTrip() + c2 := dt.wantDial() + c2.finish(nil) + rt2.wantDone(c2, "HTTP/1.1") + c2.fakeNetConn.Close() + rt2.finish() + }) +} + +// MaxIdleConnsPerHost limits the number of idle connections. +func TestTransportPoolClosesConnsPastMaxIdleConnsPerHost(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + dt := newTransportDialTester(t, http1Mode, func(tr *http.Transport) { + tr.MaxIdleConnsPerHost = 1 + }) + + // First request creates a new connection. + rt1 := dt.roundTrip("host1.fake.tld") + c1 := dt.wantDial() + c1.finish(nil) + rt1.wantDone(c1, "HTTP/1.1") + + // Second request also creates a new connection. + rt2 := dt.roundTrip("host1.fake.tld") + c2 := dt.wantDial() + c2.finish(nil) + rt2.wantDone(c2, "HTTP/1.1") + + // Third request is to a different host. + rt3 := dt.roundTrip("host2.fake.tld") + c3 := dt.wantDial() + c3.finish(nil) + rt3.wantDone(c3, "HTTP/1.1") + + // All requests finish. One conn is in excess of MaxIdleConnsPerHost, and is closed. + rt3.finish() + rt2.finish() + rt1.finish() + c1.wantClosed() + + // Additional requests reuse the remaining connections. + rt4 := dt.roundTrip("host1.fake.tld") + rt4.wantDone(c2, "HTTP/1.1") + rt4.finish() + rt5 := dt.roundTrip("host2.fake.tld") + rt5.wantDone(c3, "HTTP/1.1") + rt5.finish() + }) +} + +// Current (but probably wrong) behavior: +// MaxIdleConnsPerHost doesn't apply to HTTP/2 connections. +func TestTransportPoolMaxIdleConnsPerHostHTTP2(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + t.Skip("skipped until h2_bundle.go includes support for MaxConcurrentStreams") + + dt := newTransportDialTester(t, http2Mode, func(srv *http.Server) { + srv.HTTP2 = &http.HTTP2Config{ + MaxConcurrentStreams: 1, + } + }, func(tr *http.Transport) { + tr.MaxIdleConnsPerHost = 1 + }) + + // First request creates a new connection. + rt1 := dt.roundTrip() + c1 := dt.wantDial() + c1.finish(nil) + rt1.wantDone(c1, "HTTP/2.0") + + // Second request also creates a new connection. + rt2 := dt.roundTrip() + c2 := dt.wantDial() + c2.finish(nil) + rt2.wantDone(c2, "HTTP/2.0") + + // Both requests finish. + // We have two idle conns for this host, but we keep them both. + rt1.finish() + rt2.finish() + + // Two new requests use the existing connections. + rt3 := dt.roundTrip() + rt3.wantDone(c1, "HTTP/2.0") + rt4 := dt.roundTrip() + rt4.wantDone(c2, "HTTP/2.0") + }) } // A transportDialTester manages a test of a connection's Dials. @@ -80,7 +367,8 @@ type transportDialTester struct { t *testing.T cst *clientServerTest - dials chan *transportDialTesterConn // each new conn is sent to this channel + dialsMu sync.Mutex + dials []*transportDialTesterConn roundTripCount int dialCount int @@ -90,12 +378,12 @@ type transportDialTester struct { type transportDialTesterRoundTrip struct { t *testing.T - roundTripID int // distinguishes RoundTrips in logs - cancel context.CancelFunc // cancels the Request context - reqBody io.WriteCloser // write half of the Request.Body - finished bool + roundTripID int // distinguishes RoundTrips in logs + cancel context.CancelFunc // cancels the Request context + reqBody io.WriteCloser // write half of the Request.Body + respBodyClosed bool // set when the user calls Response.Body.Close + returned bool // set when RoundTrip returns - done chan struct{} // closed when RoundTrip returns:w res *http.Response err error conn *transportDialTesterConn @@ -108,15 +396,46 @@ type transportDialTesterConn struct { connID int // distinguished Dials in logs ready chan error // sent on to complete the Dial + protos []string + closed chan struct{} - net.Conn + *fakeNetConn } -func newTransportDialTester(t *testing.T, mode testMode) *transportDialTester { +func newTransportDialTester(t *testing.T, mode testMode, opts ...any) *transportDialTester { t.Helper() dt := &transportDialTester{ - t: t, - dials: make(chan *transportDialTesterConn), + t: t, + } + dialContext := func(_ context.Context, network, address string) (*transportDialTesterConn, error) { + c := &transportDialTesterConn{ + t: t, + ready: make(chan error), + closed: make(chan struct{}), + } + // Notify the test that a Dial has started, + // and wait for the test to notify us that it should complete. + dt.dialsMu.Lock() + dt.dials = append(dt.dials, c) + dt.dialsMu.Unlock() + + select { + case err := <-c.ready: + if err != nil { + return nil, err + } + case <-t.Context().Done(): + t.Errorf("test finished with dial in progress") + return nil, errors.New("test finished") + } + + c.fakeNetConn = dt.cst.li.connect() + t.Cleanup(func() { + c.fakeNetConn.Close() + }) + // Use the *transportDialTesterConn as the net.Conn, + // to let tests associate requests with connections. + return c, nil } dt.cst = newClientServerTest(t, mode, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Write response headers when we receive a request. @@ -126,45 +445,54 @@ func newTransportDialTester(t *testing.T, mode testMode) *transportDialTester { // Wait for the client to send the request body, // to synchronize with the rest of the test. io.ReadAll(r.Body) - }), func(tr *http.Transport) { - tr.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { - c := &transportDialTesterConn{ - t: t, - ready: make(chan error), - } - // Notify the test that a Dial has started, - // and wait for the test to notify us that it should complete. - dt.dials <- c - if err := <-c.ready; err != nil { + }), append([]any{optFakeNet, func(tr *http.Transport) { + tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + return dialContext(ctx, network, dt.cst.ts.Listener.Addr().String()) + } + tr.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + conn, err := dialContext(ctx, network, dt.cst.ts.Listener.Addr().String()) + if err != nil { return nil, err } - nc, err := net.Dial(network, address) - if err != nil { + config := &tls.Config{ + InsecureSkipVerify: true, + NextProtos: []string{"h2", "http/1.1"}, + } + if conn.protos != nil { + config.NextProtos = conn.protos + } + tc := tls.Client(conn, config) + if err := tc.Handshake(); err != nil { return nil, err } - // Use the *transportDialTesterConn as the net.Conn, - // to let tests associate requests with connections. - c.Conn = nc - return c, err + return tc, nil } - }) + }}, opts...)...) return dt } // roundTrip starts a RoundTrip. // It returns immediately, without waiting for the RoundTrip call to complete. -func (dt *transportDialTester) roundTrip() *transportDialTesterRoundTrip { +func (dt *transportDialTester) roundTrip(opts ...any) *transportDialTesterRoundTrip { dt.t.Helper() + host := "fake.tld" + for _, o := range opts { + switch o := o.(type) { + case string: + host = o + default: + dt.t.Fatalf("unknown option type %T", o) + } + } ctx, cancel := context.WithCancel(context.Background()) pr, pw := io.Pipe() + dt.roundTripCount++ rt := &transportDialTesterRoundTrip{ t: dt.t, roundTripID: dt.roundTripCount, - done: make(chan struct{}), reqBody: pw, cancel: cancel, } - dt.roundTripCount++ dt.t.Logf("RoundTrip %v: started", rt.roundTripID) dt.t.Cleanup(func() { rt.cancel() @@ -173,28 +501,54 @@ func (dt *transportDialTester) roundTrip() *transportDialTesterRoundTrip { go func() { ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{ GotConn: func(info httptrace.GotConnInfo) { - rt.conn = info.Conn.(*transportDialTesterConn) + c := info.Conn + if tlsConn, ok := c.(*tls.Conn); ok { + c = tlsConn.NetConn() + } + rt.conn = c.(*transportDialTesterConn) }, }) - req, _ := http.NewRequestWithContext(ctx, "POST", dt.cst.ts.URL, pr) + proto, _, _ := strings.Cut(dt.cst.ts.URL, ":") + req, _ := http.NewRequestWithContext(ctx, "POST", proto+"://"+host, pr) req.Header.Set("Content-Type", "text/plain") rt.res, rt.err = dt.cst.tr.RoundTrip(req) dt.t.Logf("RoundTrip %v: done (err:%v)", rt.roundTripID, rt.err) - close(rt.done) + rt.returned = true }() return rt } // wantDone indicates that a RoundTrip should have returned. -func (rt *transportDialTesterRoundTrip) wantDone(c *transportDialTesterConn) { +func (rt *transportDialTesterRoundTrip) wantDone(c *transportDialTesterConn, wantProto string) { rt.t.Helper() - <-rt.done + synctest.Wait() + if !rt.returned { + rt.t.Fatalf("RoundTrip %v: still running, want to have returned", rt.roundTripID) + } if rt.err != nil { rt.t.Fatalf("RoundTrip %v: want success, got err %v", rt.roundTripID, rt.err) } if rt.conn != c { rt.t.Fatalf("RoundTrip %v: want on conn %v, got conn %v", rt.roundTripID, c.connID, rt.conn.connID) } + if got, want := rt.conn, c; got != want { + rt.t.Fatalf("RoundTrip %v: sent on conn %v, want conn %v", rt.roundTripID, got.connID, want.connID) + } + if got, want := rt.res.Proto, wantProto; got != want { + rt.t.Fatalf("RoundTrip %v: got protocol %q, want %q", rt.roundTripID, got, want) + } +} + +// wantError indicates that a RoundTrip should have returned with an error. +func (rt *transportDialTesterRoundTrip) wantError() { + rt.t.Helper() + synctest.Wait() + if !rt.returned { + rt.t.Fatalf("RoundTrip %v: still running, want to have returned", rt.roundTripID) + } + if rt.err == nil { + rt.t.Fatalf("RoundTrip %v: success, want error", rt.roundTripID) + } } // finish completes a RoundTrip by sending the request body, consuming the response body, @@ -202,16 +556,18 @@ func (rt *transportDialTesterRoundTrip) wantDone(c *transportDialTesterConn) { func (rt *transportDialTesterRoundTrip) finish() { rt.t.Helper() - if rt.finished { + synctest.Wait() + if !rt.returned { + rt.t.Fatalf("RoundTrip %v: still running, want to have returned", rt.roundTripID) + } + if rt.err != nil { return } - rt.finished = true - - <-rt.done - if rt.err != nil { + if rt.respBodyClosed { return } + rt.respBodyClosed = true rt.reqBody.Close() io.ReadAll(rt.res.Body) rt.res.Body.Close() @@ -220,16 +576,40 @@ func (rt *transportDialTesterRoundTrip) finish() { // wantDial waits for the Transport to start a Dial. func (dt *transportDialTester) wantDial() *transportDialTesterConn { - c := <-dt.dials - c.connID = dt.dialCount + dt.t.Helper() + synctest.Wait() + dt.dialsMu.Lock() + defer dt.dialsMu.Unlock() + if len(dt.dials) == 0 { + dt.t.Fatalf("no dial started, want one") + } + c := dt.dials[0] + dt.dials = dt.dials[1:] dt.dialCount++ + c.connID = dt.dialCount dt.t.Logf("Dial %v: started", c.connID) return c } // finish completes a Dial. func (c *transportDialTesterConn) finish(err error) { + c.t.Helper() c.t.Logf("Dial %v: finished (err:%v)", c.connID, err) c.ready <- err close(c.ready) } + +func (c *transportDialTesterConn) wantClosed() { + c.t.Helper() + <-c.closed +} + +func (c *transportDialTesterConn) Close() error { + select { + case <-c.closed: + default: + c.t.Logf("Conn %v: closed", c.connID) + close(c.closed) + } + return nil +} diff --git a/src/net/http/transport_test.go b/src/net/http/transport_test.go index 2389284249a9b0..8ab4107fb7b6b9 100644 --- a/src/net/http/transport_test.go +++ b/src/net/http/transport_test.go @@ -22,6 +22,7 @@ import ( "fmt" "go/token" "internal/nettrace" + "internal/synctest" "io" "log" mrand "math/rand" @@ -1549,6 +1550,72 @@ func TestTransportProxy(t *testing.T) { } } +// Issue 74633: verify that a client will not indefinitely read a response from +// a proxy server that writes an infinite byte of stream, rather than +// responding with 200 OK. +func TestProxyWithInfiniteHeader(t *testing.T) { + defer afterTest(t) + + ln := newLocalListener(t) + defer ln.Close() + cancelc := make(chan struct{}) + defer close(cancelc) + + // Simulate a malicious / misbehaving proxy that writes an unlimited number + // of bytes rather than responding with 200 OK. + go func() { + c, err := ln.Accept() + if err != nil { + t.Errorf("Accept: %v", err) + return + } + defer c.Close() + // Read the CONNECT request + br := bufio.NewReader(c) + cr, err := ReadRequest(br) + if err != nil { + t.Errorf("proxy server failed to read CONNECT request") + return + } + if cr.Method != "CONNECT" { + t.Errorf("unexpected method %q", cr.Method) + return + } + + // Keep writing bytes until the test exits. + for { + // runtime.Gosched() is needed here. Otherwise, this test might + // livelock in environments like WASM, where the one single thread + // we have could be hogged by the infinite loop of writing bytes. + runtime.Gosched() + select { + case <-cancelc: + return + default: + c.Write([]byte("infinite stream of bytes")) + } + } + }() + + c := &Client{ + Transport: &Transport{ + Proxy: func(*Request) (*url.URL, error) { + return url.Parse("http://" + ln.Addr().String()) + }, + // Limit MaxResponseHeaderBytes so the test returns quicker. + MaxResponseHeaderBytes: 1024, + }, + } + req, err := NewRequest("GET", "https://golang.fake.tld/", nil) + if err != nil { + t.Fatal(err) + } + _, err = c.Do(req) + if err == nil { + t.Errorf("unexpected Get success") + } +} + func TestOnProxyConnectResponse(t *testing.T) { var tcases = []struct { @@ -2033,7 +2100,7 @@ func (d *countingDialer) DialContext(ctx context.Context, network, address strin d.total++ d.live++ - runtime.SetFinalizer(counted, d.decrement) + runtime.AddCleanup(counted, func(dd *countingDialer) { dd.decrement(nil) }, d) return counted, nil } @@ -2105,7 +2172,7 @@ func (cc *contextCounter) Track(ctx context.Context) context.Context { cc.mu.Lock() defer cc.mu.Unlock() cc.live++ - runtime.SetFinalizer(counted, cc.decrement) + runtime.AddCleanup(counted, func(c *contextCounter) { cc.decrement(nil) }, cc) return counted } @@ -2586,8 +2653,8 @@ func runCancelTestTransport(t *testing.T, mode testMode, f func(t *testing.T, te // runCancelTestChannel uses Request.Cancel. func runCancelTestChannel(t *testing.T, mode testMode, f func(t *testing.T, test cancelTest)) { - var cancelOnce sync.Once cancelc := make(chan struct{}) + cancelOnce := sync.OnceFunc(func() { close(cancelc) }) f(t, cancelTest{ mode: mode, newReq: func(req *Request) *Request { @@ -2595,9 +2662,7 @@ func runCancelTestChannel(t *testing.T, mode testMode, f func(t *testing.T, test return req }, cancel: func(tr *Transport, req *Request) { - cancelOnce.Do(func() { - close(cancelc) - }) + cancelOnce() }, checkErr: func(when string, err error) { if !errors.Is(err, ExportErrRequestCanceled) && !errors.Is(err, ExportErrRequestCanceledConn) { @@ -3260,29 +3325,68 @@ func testTransportIgnore1xxResponses(t *testing.T, mode testMode) { } } -func TestTransportLimits1xxResponses(t *testing.T) { - run(t, testTransportLimits1xxResponses, []testMode{http1Mode}) -} +func TestTransportLimits1xxResponses(t *testing.T) { run(t, testTransportLimits1xxResponses) } func testTransportLimits1xxResponses(t *testing.T, mode testMode) { cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { - conn, buf, _ := w.(Hijacker).Hijack() + w.Header().Add("X-Header", strings.Repeat("a", 100)) for i := 0; i < 10; i++ { - buf.Write([]byte("HTTP/1.1 123 OneTwoThree\r\n\r\n")) + w.WriteHeader(123) } - buf.Write([]byte("HTTP/1.1 204 No Content\r\n\r\n")) - buf.Flush() - conn.Close() + w.WriteHeader(204) })) cst.tr.DisableKeepAlives = true // prevent log spam; our test server is hanging up anyway + cst.tr.MaxResponseHeaderBytes = 1000 res, err := cst.c.Get(cst.ts.URL) - if res != nil { - defer res.Body.Close() + if err == nil { + res.Body.Close() + t.Fatalf("RoundTrip succeeded; want error") + } + for _, want := range []string{ + "response headers exceeded", + "too many 1xx", + "header list too large", + } { + if strings.Contains(err.Error(), want) { + return + } + } + t.Errorf(`got error %q; want "response headers exceeded" or "too many 1xx"`, err) +} + +func TestTransportDoesNotLimitDelivered1xxResponses(t *testing.T) { + run(t, testTransportDoesNotLimitDelivered1xxResponses) +} +func testTransportDoesNotLimitDelivered1xxResponses(t *testing.T, mode testMode) { + if mode == http2Mode { + t.Skip("skip until x/net/http2 updated") + } + const num1xx = 10 + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Add("X-Header", strings.Repeat("a", 100)) + for i := 0; i < 10; i++ { + w.WriteHeader(123) + } + w.WriteHeader(204) + })) + cst.tr.DisableKeepAlives = true // prevent log spam; our test server is hanging up anyway + cst.tr.MaxResponseHeaderBytes = 1000 + + got1xx := 0 + ctx := httptrace.WithClientTrace(context.Background(), &httptrace.ClientTrace{ + Got1xxResponse: func(code int, header textproto.MIMEHeader) error { + got1xx++ + return nil + }, + }) + req, _ := NewRequestWithContext(ctx, "GET", cst.ts.URL, nil) + res, err := cst.c.Do(req) + if err != nil { + t.Fatal(err) } - got := fmt.Sprint(err) - wantSub := "too many 1xx informational responses" - if !strings.Contains(got, wantSub) { - t.Errorf("Get error = %v; want substring %q", err, wantSub) + res.Body.Close() + if got1xx != num1xx { + t.Errorf("Got %v 1xx responses, want %x", got1xx, num1xx) } } @@ -4182,53 +4286,175 @@ func TestTransportTraceGotConnH2IdleConns(t *testing.T) { wantIdle("after round trip", 1) } -func TestTransportRemovesH2ConnsAfterIdle(t *testing.T) { - run(t, testTransportRemovesH2ConnsAfterIdle, []testMode{http2Mode}) +// https://go.dev/issue/70515 +// +// When the first request on a new connection fails, we do not retry the request. +// If the first request on a connection races with IdleConnTimeout, +// we should not fail the request. +func TestTransportIdleConnRacesRequest(t *testing.T) { + // Use unencrypted HTTP/2, since the *tls.Conn interfers with our ability to + // block the connection closing. + runSynctest(t, testTransportIdleConnRacesRequest, []testMode{http1Mode, http2UnencryptedMode}) +} +func testTransportIdleConnRacesRequest(t *testing.T, mode testMode) { + if mode == http2UnencryptedMode { + t.Skip("remove skip when #70515 is fixed") + } + timeout := 1 * time.Millisecond + trFunc := func(tr *Transport) { + tr.IdleConnTimeout = timeout + } + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + }), trFunc, optFakeNet) + cst.li.trackConns = true + + // We want to put a connection into the pool which has never had a request made on it. + // + // Make a request and cancel it before the dial completes. + // Then complete the dial. + dialc := make(chan struct{}) + cst.li.onDial = func() { + <-dialc + } + closec := make(chan struct{}) + cst.li.onClose = func(*fakeNetConn) { + <-closec + } + ctx, cancel := context.WithCancel(context.Background()) + req1c := make(chan error) + go func() { + req, _ := NewRequestWithContext(ctx, "GET", cst.ts.URL, nil) + resp, err := cst.c.Do(req) + if err == nil { + resp.Body.Close() + } + req1c <- err + }() + // Wait for the connection attempt to start. + synctest.Wait() + // Cancel the request. + cancel() + synctest.Wait() + if err := <-req1c; err == nil { + t.Fatal("expected request to fail, but it succeeded") + } + // Unblock the dial, placing a new, unused connection into the Transport's pool. + close(dialc) + + // We want IdleConnTimeout to race with a new request. + // + // There's no perfect way to do this, but the following exercises the bug in #70515: + // Block net.Conn.Close, wait until IdleConnTimeout occurs, and make a request while + // the connection close is still blocked. + // + // First: Wait for IdleConnTimeout. The net.Conn.Close blocks. + synctest.Wait() + time.Sleep(timeout) + synctest.Wait() + // Make a request, which will use a new connection (since the existing one is closing). + req2c := make(chan error) + go func() { + resp, err := cst.c.Get(cst.ts.URL) + if err == nil { + resp.Body.Close() + } + req2c <- err + }() + // Don't synctest.Wait here: The HTTP/1 transport closes the idle conn + // with a mutex held, and we'll end up in a deadlock. + close(closec) + if err := <-req2c; err != nil { + t.Fatalf("Get: %v", err) + } +} + +func TestTransportRemovesConnsAfterIdle(t *testing.T) { + runSynctest(t, testTransportRemovesConnsAfterIdle) } -func testTransportRemovesH2ConnsAfterIdle(t *testing.T, mode testMode) { +func testTransportRemovesConnsAfterIdle(t *testing.T, mode testMode) { if testing.Short() { t.Skip("skipping in short mode") } - timeout := 1 * time.Millisecond - retry := true - for retry { - trFunc := func(tr *Transport) { - tr.MaxConnsPerHost = 1 - tr.MaxIdleConnsPerHost = 1 - tr.IdleConnTimeout = timeout - } - cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) {}), trFunc) - - retry = false - tooShort := func(err error) bool { - if err == nil || !strings.Contains(err.Error(), "use of closed network connection") { - return false - } - if !retry { - t.Helper() - t.Logf("idle conn timeout %v may be too short; retrying with longer", timeout) - timeout *= 2 - retry = true - cst.close() - } - return true - } + timeout := 1 * time.Second + trFunc := func(tr *Transport) { + tr.MaxConnsPerHost = 1 + tr.MaxIdleConnsPerHost = 1 + tr.IdleConnTimeout = timeout + } + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("X-Addr", r.RemoteAddr) + }), trFunc, optFakeNet) - if _, err := cst.c.Get(cst.ts.URL); err != nil { - if tooShort(err) { - continue - } + // makeRequest returns the local address a request was made from + // (unique for each connection). + makeRequest := func() string { + resp, err := cst.c.Get(cst.ts.URL) + if err != nil { t.Fatalf("got error: %s", err) } + resp.Body.Close() + return resp.Header.Get("X-Addr") + } - time.Sleep(10 * timeout) - if _, err := cst.c.Get(cst.ts.URL); err != nil { - if tooShort(err) { - continue - } + addr1 := makeRequest() + + time.Sleep(timeout / 2) + synctest.Wait() + addr2 := makeRequest() + if addr1 != addr2 { + t.Fatalf("two requests made within IdleConnTimeout should have used the same conn, but used %v, %v", addr1, addr2) + } + + time.Sleep(timeout) + synctest.Wait() + addr3 := makeRequest() + if addr1 == addr3 { + t.Fatalf("two requests made more than IdleConnTimeout apart should have used different conns, but used %v, %v", addr1, addr3) + } +} + +func TestTransportRemovesConnsAfterBroken(t *testing.T) { + runSynctest(t, testTransportRemovesConnsAfterBroken) +} +func testTransportRemovesConnsAfterBroken(t *testing.T, mode testMode) { + if testing.Short() { + t.Skip("skipping in short mode") + } + + trFunc := func(tr *Transport) { + tr.MaxConnsPerHost = 1 + tr.MaxIdleConnsPerHost = 1 + } + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("X-Addr", r.RemoteAddr) + }), trFunc, optFakeNet) + cst.li.trackConns = true + + // makeRequest returns the local address a request was made from + // (unique for each connection). + makeRequest := func() string { + resp, err := cst.c.Get(cst.ts.URL) + if err != nil { t.Fatalf("got error: %s", err) } + resp.Body.Close() + return resp.Header.Get("X-Addr") + } + + addr1 := makeRequest() + addr2 := makeRequest() + if addr1 != addr2 { + t.Fatalf("successive requests should have used the same conn, but used %v, %v", addr1, addr2) + } + + // The connection breaks. + synctest.Wait() + cst.li.conns[0].peer.Close() + synctest.Wait() + addr3 := makeRequest() + if addr1 == addr3 { + t.Fatalf("successive requests made with conn broken between should have used different conns, but used %v, %v", addr1, addr3) } } @@ -4300,7 +4526,6 @@ func TestTransportContentEncodingCaseInsensitive(t *testing.T) { } func testTransportContentEncodingCaseInsensitive(t *testing.T, mode testMode) { for _, ce := range []string{"gzip", "GZIP"} { - ce := ce t.Run(ce, func(t *testing.T) { const encodedString = "Hello Gopher" ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { @@ -5114,20 +5339,16 @@ func testTransportEventTraceTLSVerify(t *testing.T, mode testMode) { } } -var ( - isDNSHijackedOnce sync.Once - isDNSHijacked bool -) +var isDNSHijacked = sync.OnceValue(func() bool { + addrs, _ := net.LookupHost("dns-should-not-resolve.golang") + return len(addrs) != 0 +}) func skipIfDNSHijacked(t *testing.T) { // Skip this test if the user is using a shady/ISP // DNS server hijacking queries. // See issues 16732, 16716. - isDNSHijackedOnce.Do(func() { - addrs, _ := net.LookupHost("dns-should-not-resolve.golang") - isDNSHijacked = len(addrs) != 0 - }) - if isDNSHijacked { + if isDNSHijacked() { t.Skip("skipping; test requires non-hijacking DNS server") } } @@ -5344,7 +5565,9 @@ timeoutLoop: return false } } - res.Body.Close() + if err == nil { + res.Body.Close() + } conns := idleConns() if len(conns) != 1 { if len(conns) == 0 { @@ -5463,7 +5686,7 @@ func TestTransportReturnsPeekError(t *testing.T) { errValue := errors.New("specific error value") wrote := make(chan struct{}) - var wroteOnce sync.Once + wroteOnce := sync.OnceFunc(func() { close(wrote) }) tr := &Transport{ Dial: func(network, addr string) (net.Conn, error) { @@ -5473,7 +5696,7 @@ func TestTransportReturnsPeekError(t *testing.T) { return 0, errValue }, write: func(p []byte) (int, error) { - wroteOnce.Do(func() { close(wrote) }) + wroteOnce() return len(p), nil }, } @@ -6328,12 +6551,16 @@ func TestTransportClone(t *testing.T) { GetProxyConnectHeader: func(context.Context, *url.URL, string) (Header, error) { return nil, nil }, MaxResponseHeaderBytes: 1, ForceAttemptHTTP2: true, + HTTP2: &HTTP2Config{MaxConcurrentStreams: 1}, + Protocols: &Protocols{}, TLSNextProto: map[string]func(authority string, c *tls.Conn) RoundTripper{ "foo": func(authority string, c *tls.Conn) RoundTripper { panic("") }, }, ReadBufferSize: 1, WriteBufferSize: 1, } + tr.Protocols.SetHTTP1(true) + tr.Protocols.SetHTTP2(true) tr2 := tr.Clone() rv := reflect.ValueOf(tr2).Elem() rt := rv.Type() @@ -7134,3 +7361,298 @@ func testValidateClientRequestTrailers(t *testing.T, mode testMode) { }) } } + +func TestTransportServerProtocols(t *testing.T) { + CondSkipHTTP2(t) + DefaultTransport.(*Transport).CloseIdleConnections() + + cert, err := tls.X509KeyPair(testcert.LocalhostCert, testcert.LocalhostKey) + if err != nil { + t.Fatal(err) + } + leafCert, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + t.Fatal(err) + } + certpool := x509.NewCertPool() + certpool.AddCert(leafCert) + + for _, test := range []struct { + name string + scheme string + setup func(t *testing.T) + transport func(*Transport) + server func(*Server) + want string + }{{ + name: "http default", + scheme: "http", + want: "HTTP/1.1", + }, { + name: "https default", + scheme: "https", + transport: func(tr *Transport) { + // Transport default is HTTP/1. + }, + want: "HTTP/1.1", + }, { + name: "https transport protocols include HTTP2", + scheme: "https", + transport: func(tr *Transport) { + // Server default is to support HTTP/2, so if the Transport enables + // HTTP/2 we get it. + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + tr.Protocols.SetHTTP2(true) + }, + want: "HTTP/2.0", + }, { + name: "https transport protocols only include HTTP1", + scheme: "https", + transport: func(tr *Transport) { + // Explicitly enable only HTTP/1. + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + }, + want: "HTTP/1.1", + }, { + name: "https transport ForceAttemptHTTP2", + scheme: "https", + transport: func(tr *Transport) { + // Pre-Protocols-field way of enabling HTTP/2. + tr.ForceAttemptHTTP2 = true + }, + want: "HTTP/2.0", + }, { + name: "https transport protocols override TLSNextProto", + scheme: "https", + transport: func(tr *Transport) { + // Setting TLSNextProto to an empty map is the historical way + // of disabling HTTP/2. Explicitly enabling HTTP2 in the Protocols + // field takes precedence. + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + tr.Protocols.SetHTTP2(true) + tr.TLSNextProto = map[string]func(string, *tls.Conn) RoundTripper{} + }, + want: "HTTP/2.0", + }, { + name: "https server disables HTTP2 with TLSNextProto", + scheme: "https", + server: func(srv *Server) { + // Disable HTTP/2 on the server with TLSNextProto, + // use default Protocols value. + srv.TLSNextProto = map[string]func(*Server, *tls.Conn, Handler){} + }, + want: "HTTP/1.1", + }, { + name: "https server Protocols overrides empty TLSNextProto", + scheme: "https", + server: func(srv *Server) { + // Explicitly enabling HTTP2 in the Protocols field takes precedence + // over setting an empty TLSNextProto. + srv.Protocols = &Protocols{} + srv.Protocols.SetHTTP1(true) + srv.Protocols.SetHTTP2(true) + srv.TLSNextProto = map[string]func(*Server, *tls.Conn, Handler){} + }, + want: "HTTP/2.0", + }, { + name: "https server protocols only include HTTP1", + scheme: "https", + server: func(srv *Server) { + srv.Protocols = &Protocols{} + srv.Protocols.SetHTTP1(true) + }, + want: "HTTP/1.1", + }, { + name: "https server protocols include HTTP2", + scheme: "https", + server: func(srv *Server) { + srv.Protocols = &Protocols{} + srv.Protocols.SetHTTP1(true) + srv.Protocols.SetHTTP2(true) + }, + want: "HTTP/2.0", + }, { + name: "GODEBUG disables HTTP2 client", + scheme: "https", + setup: func(t *testing.T) { + t.Setenv("GODEBUG", "http2client=0") + }, + transport: func(tr *Transport) { + // Server default is to support HTTP/2, so if the Transport enables + // HTTP/2 we get it. + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + tr.Protocols.SetHTTP2(true) + }, + want: "HTTP/1.1", + }, { + name: "GODEBUG disables HTTP2 server", + scheme: "https", + setup: func(t *testing.T) { + t.Setenv("GODEBUG", "http2server=0") + }, + transport: func(tr *Transport) { + // Server default is to support HTTP/2, so if the Transport enables + // HTTP/2 we get it. + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + tr.Protocols.SetHTTP2(true) + }, + want: "HTTP/1.1", + }, { + name: "unencrypted HTTP2 with prior knowledge", + scheme: "http", + transport: func(tr *Transport) { + tr.Protocols = &Protocols{} + tr.Protocols.SetUnencryptedHTTP2(true) + }, + server: func(srv *Server) { + srv.Protocols = &Protocols{} + srv.Protocols.SetHTTP1(true) + srv.Protocols.SetUnencryptedHTTP2(true) + }, + want: "HTTP/2.0", + }, { + name: "unencrypted HTTP2 only on server", + scheme: "http", + transport: func(tr *Transport) { + tr.Protocols = &Protocols{} + tr.Protocols.SetUnencryptedHTTP2(true) + }, + server: func(srv *Server) { + srv.Protocols = &Protocols{} + srv.Protocols.SetUnencryptedHTTP2(true) + }, + want: "HTTP/2.0", + }, { + name: "unencrypted HTTP2 with no server support", + scheme: "http", + transport: func(tr *Transport) { + tr.Protocols = &Protocols{} + tr.Protocols.SetUnencryptedHTTP2(true) + }, + server: func(srv *Server) { + srv.Protocols = &Protocols{} + srv.Protocols.SetHTTP1(true) + }, + want: "error", + }, { + name: "HTTP1 with no server support", + scheme: "http", + transport: func(tr *Transport) { + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + }, + server: func(srv *Server) { + srv.Protocols = &Protocols{} + srv.Protocols.SetUnencryptedHTTP2(true) + }, + want: "error", + }, { + name: "HTTPS1 with no server support", + scheme: "https", + transport: func(tr *Transport) { + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + }, + server: func(srv *Server) { + srv.Protocols = &Protocols{} + srv.Protocols.SetHTTP2(true) + }, + want: "error", + }} { + t.Run(test.name, func(t *testing.T) { + // We don't use httptest here because it makes its own decisions + // about how to enable/disable HTTP/2. + srv := &Server{ + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{cert}, + }, + Handler: HandlerFunc(func(w ResponseWriter, req *Request) { + w.Header().Set("X-Proto", req.Proto) + }), + } + tr := &Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certpool, + }, + } + + if test.setup != nil { + test.setup(t) + } + if test.server != nil { + test.server(srv) + } + if test.transport != nil { + test.transport(tr) + } else { + tr.Protocols = &Protocols{} + tr.Protocols.SetHTTP1(true) + tr.Protocols.SetHTTP2(true) + } + + listener := newLocalListener(t) + srvc := make(chan error, 1) + go func() { + switch test.scheme { + case "http": + srvc <- srv.Serve(listener) + case "https": + srvc <- srv.ServeTLS(listener, "", "") + } + }() + t.Cleanup(func() { + srv.Close() + <-srvc + }) + + client := &Client{Transport: tr} + resp, err := client.Get(test.scheme + "://" + listener.Addr().String()) + if err != nil { + if test.want == "error" { + return + } + t.Fatal(err) + } + if got := resp.Header.Get("X-Proto"); got != test.want { + t.Fatalf("request proto %q, want %q", got, test.want) + } + }) + } +} + +func TestIssue61474(t *testing.T) { + run(t, testIssue61474, []testMode{http2Mode}) +} +func testIssue61474(t *testing.T, mode testMode) { + if testing.Short() { + return + } + + // This test reliably exercises the condition causing #61474, + // but requires many iterations to do so. + // Keep the test around for now, but don't run it by default. + t.Skip("test is too large") + + cst := newClientServerTest(t, mode, HandlerFunc(func(rw ResponseWriter, req *Request) { + }), func(tr *Transport) { + tr.MaxConnsPerHost = 1 + }) + var wg sync.WaitGroup + defer wg.Wait() + for range 100000 { + wg.Go(func() { + ctx, cancel := context.WithTimeout(t.Context(), 1*time.Millisecond) + defer cancel() + req, _ := NewRequestWithContext(ctx, "GET", cst.ts.URL, nil) + resp, err := cst.c.Do(req) + if err == nil { + resp.Body.Close() + } + }) + } +} diff --git a/src/net/interface.go b/src/net/interface.go index 74bb4f0e1c6122..b6057780c4a98f 100644 --- a/src/net/interface.go +++ b/src/net/interface.go @@ -42,7 +42,7 @@ var ( type Interface struct { Index int // positive integer that starts at one, zero is never used MTU int // maximum transmission unit - Name string // e.g., "en0", "lo0", "eth0.100" + Name string // e.g., "en0", "lo0", "eth0.100"; may be the empty string HardwareAddr HardwareAddr // IEEE MAC-48, EUI-48 and EUI-64 form Flags Flags // e.g., FlagUp, FlagLoopback, FlagMulticast } @@ -221,9 +221,11 @@ func (zc *ipv6ZoneCache) update(ift []Interface, force bool) (updated bool) { zc.toIndex = make(map[string]int, len(ift)) zc.toName = make(map[int]string, len(ift)) for _, ifi := range ift { - zc.toIndex[ifi.Name] = ifi.Index - if _, ok := zc.toName[ifi.Index]; !ok { - zc.toName[ifi.Index] = ifi.Name + if ifi.Name != "" { + zc.toIndex[ifi.Name] = ifi.Index + if _, ok := zc.toName[ifi.Index]; !ok { + zc.toName[ifi.Index] = ifi.Name + } } } return true diff --git a/src/net/interface_bsd.go b/src/net/interface_bsd.go index 9b2b42addbb609..0861610f168996 100644 --- a/src/net/interface_bsd.go +++ b/src/net/interface_bsd.go @@ -7,9 +7,8 @@ package net import ( + "internal/routebsd" "syscall" - - "golang.org/x/net/route" ) // If the ifindex is zero, interfaceTable returns mappings of all @@ -28,23 +27,18 @@ func interfaceTable(ifindex int) ([]Interface, error) { n = 0 for _, m := range msgs { switch m := m.(type) { - case *route.InterfaceMessage: + case *routebsd.InterfaceMessage: if ifindex != 0 && ifindex != m.Index { continue } ift[n].Index = m.Index ift[n].Name = m.Name ift[n].Flags = linkFlags(m.Flags) - if sa, ok := m.Addrs[syscall.RTAX_IFP].(*route.LinkAddr); ok && len(sa.Addr) > 0 { + if sa, ok := m.Addrs[syscall.RTAX_IFP].(*routebsd.LinkAddr); ok && len(sa.Addr) > 0 { ift[n].HardwareAddr = make([]byte, len(sa.Addr)) copy(ift[n].HardwareAddr, sa.Addr) } - for _, sys := range m.Sys() { - if imx, ok := sys.(*route.InterfaceMetrics); ok { - ift[n].MTU = imx.MTU - break - } - } + ift[n].MTU = m.MTU() n++ if ifindex == m.Index { return ift[:n], nil @@ -92,27 +86,35 @@ func interfaceAddrTable(ifi *Interface) ([]Addr, error) { ifat := make([]Addr, 0, len(msgs)) for _, m := range msgs { switch m := m.(type) { - case *route.InterfaceAddrMessage: + case *routebsd.InterfaceAddrMessage: if index != 0 && index != m.Index { continue } var mask IPMask switch sa := m.Addrs[syscall.RTAX_NETMASK].(type) { - case *route.Inet4Addr: - mask = IPv4Mask(sa.IP[0], sa.IP[1], sa.IP[2], sa.IP[3]) - case *route.Inet6Addr: - mask = make(IPMask, IPv6len) - copy(mask, sa.IP[:]) + case *routebsd.InetAddr: + if sa.IP.Is4() { + a := sa.IP.As4() + mask = IPv4Mask(a[0], a[1], a[2], a[3]) + } else if sa.IP.Is6() { + a := sa.IP.As16() + mask = make(IPMask, IPv6len) + copy(mask, a[:]) + } } var ip IP switch sa := m.Addrs[syscall.RTAX_IFA].(type) { - case *route.Inet4Addr: - ip = IPv4(sa.IP[0], sa.IP[1], sa.IP[2], sa.IP[3]) - case *route.Inet6Addr: - ip = make(IP, IPv6len) - copy(ip, sa.IP[:]) + case *routebsd.InetAddr: + if sa.IP.Is4() { + a := sa.IP.As4() + ip = IPv4(a[0], a[1], a[2], a[3]) + } else if sa.IP.Is6() { + a := sa.IP.As16() + ip = make(IP, IPv6len) + copy(ip, a[:]) + } } - if ip != nil && mask != nil { // NetBSD may contain route.LinkAddr + if ip != nil && mask != nil { // NetBSD may contain routebsd.LinkAddr ifat = append(ifat, &IPNet{IP: ip, Mask: mask}) } } diff --git a/src/net/interface_bsdvar.go b/src/net/interface_bsdvar.go index e9bea3d3798a06..dc6c293a8e4b1a 100644 --- a/src/net/interface_bsdvar.go +++ b/src/net/interface_bsdvar.go @@ -7,17 +7,12 @@ package net import ( + "internal/routebsd" "syscall" - - "golang.org/x/net/route" ) -func interfaceMessages(ifindex int) ([]route.Message, error) { - rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_IFLIST, ifindex) - if err != nil { - return nil, err - } - return route.ParseRIB(syscall.NET_RT_IFLIST, rib) +func interfaceMessages(ifindex int) ([]routebsd.Message, error) { + return routebsd.FetchRIBMessages(syscall.NET_RT_IFLIST, ifindex) } // interfaceMulticastAddrTable returns addresses for a specific diff --git a/src/net/interface_darwin.go b/src/net/interface_darwin.go index bb4fd73a987670..95c44e68e63a1e 100644 --- a/src/net/interface_darwin.go +++ b/src/net/interface_darwin.go @@ -5,44 +5,39 @@ package net import ( + "internal/routebsd" "syscall" - - "golang.org/x/net/route" ) -func interfaceMessages(ifindex int) ([]route.Message, error) { - rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_IFLIST, ifindex) - if err != nil { - return nil, err - } - return route.ParseRIB(syscall.NET_RT_IFLIST, rib) +func interfaceMessages(ifindex int) ([]routebsd.Message, error) { + return routebsd.FetchRIBMessages(syscall.NET_RT_IFLIST, ifindex) } // interfaceMulticastAddrTable returns addresses for a specific // interface. func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { - rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_IFLIST2, ifi.Index) - if err != nil { - return nil, err - } - msgs, err := route.ParseRIB(syscall.NET_RT_IFLIST2, rib) + msgs, err := routebsd.FetchRIBMessages(syscall.NET_RT_IFLIST2, ifi.Index) if err != nil { return nil, err } ifmat := make([]Addr, 0, len(msgs)) for _, m := range msgs { switch m := m.(type) { - case *route.InterfaceMulticastAddrMessage: + case *routebsd.InterfaceMulticastAddrMessage: if ifi.Index != m.Index { continue } var ip IP switch sa := m.Addrs[syscall.RTAX_IFA].(type) { - case *route.Inet4Addr: - ip = IPv4(sa.IP[0], sa.IP[1], sa.IP[2], sa.IP[3]) - case *route.Inet6Addr: - ip = make(IP, IPv6len) - copy(ip, sa.IP[:]) + case *routebsd.InetAddr: + if sa.IP.Is4() { + a := sa.IP.As4() + ip = IPv4(a[0], a[1], a[2], a[3]) + } else if sa.IP.Is6() { + a := sa.IP.As16() + ip = make(IP, IPv6len) + copy(ip, a[:]) + } } if ip != nil { ifmat = append(ifmat, &IPAddr{IP: ip}) diff --git a/src/net/interface_freebsd.go b/src/net/interface_freebsd.go index 8536bd3cf6482b..301faa47baef79 100644 --- a/src/net/interface_freebsd.go +++ b/src/net/interface_freebsd.go @@ -5,44 +5,39 @@ package net import ( + "internal/routebsd" "syscall" - - "golang.org/x/net/route" ) -func interfaceMessages(ifindex int) ([]route.Message, error) { - rib, err := route.FetchRIB(syscall.AF_UNSPEC, route.RIBTypeInterface, ifindex) - if err != nil { - return nil, err - } - return route.ParseRIB(route.RIBTypeInterface, rib) +func interfaceMessages(ifindex int) ([]routebsd.Message, error) { + return routebsd.FetchRIBMessages(syscall.NET_RT_IFLIST, ifindex) } // interfaceMulticastAddrTable returns addresses for a specific // interface. func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { - rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_IFMALIST, ifi.Index) - if err != nil { - return nil, err - } - msgs, err := route.ParseRIB(syscall.NET_RT_IFMALIST, rib) + msgs, err := routebsd.FetchRIBMessages(syscall.NET_RT_IFMALIST, ifi.Index) if err != nil { return nil, err } ifmat := make([]Addr, 0, len(msgs)) for _, m := range msgs { switch m := m.(type) { - case *route.InterfaceMulticastAddrMessage: + case *routebsd.InterfaceMulticastAddrMessage: if ifi.Index != m.Index { continue } var ip IP switch sa := m.Addrs[syscall.RTAX_IFA].(type) { - case *route.Inet4Addr: - ip = IPv4(sa.IP[0], sa.IP[1], sa.IP[2], sa.IP[3]) - case *route.Inet6Addr: - ip = make(IP, IPv6len) - copy(ip, sa.IP[:]) + case *routebsd.InetAddr: + if sa.IP.Is4() { + a := sa.IP.As4() + ip = IPv4(a[0], a[1], a[2], a[3]) + } else if sa.IP.Is6() { + a := sa.IP.As16() + ip = make(IP, IPv6len) + copy(ip, a[:]) + } } if ip != nil { ifmat = append(ifmat, &IPAddr{IP: ip}) diff --git a/src/net/interface_linux.go b/src/net/interface_linux.go index 9112ecc854c74c..7856dae8fc878b 100644 --- a/src/net/interface_linux.go +++ b/src/net/interface_linux.go @@ -129,22 +129,14 @@ func interfaceAddrTable(ifi *Interface) ([]Addr, error) { if err != nil { return nil, os.NewSyscallError("parsenetlinkmessage", err) } - var ift []Interface - if ifi == nil { - var err error - ift, err = interfaceTable(0) - if err != nil { - return nil, err - } - } - ifat, err := addrTable(ift, ifi, msgs) + ifat, err := addrTable(ifi, msgs) if err != nil { return nil, err } return ifat, nil } -func addrTable(ift []Interface, ifi *Interface, msgs []syscall.NetlinkMessage) ([]Addr, error) { +func addrTable(ifi *Interface, msgs []syscall.NetlinkMessage) ([]Addr, error) { var ifat []Addr loop: for _, m := range msgs { @@ -153,14 +145,7 @@ loop: break loop case syscall.RTM_NEWADDR: ifam := (*syscall.IfAddrmsg)(unsafe.Pointer(&m.Data[0])) - if len(ift) != 0 || ifi.Index == int(ifam.Index) { - if len(ift) != 0 { - var err error - ifi, err = interfaceByIndex(ift, int(ifam.Index)) - if err != nil { - return nil, err - } - } + if ifi == nil || ifi.Index == int(ifam.Index) { attrs, err := syscall.ParseNetlinkRouteAttr(&m) if err != nil { return nil, os.NewSyscallError("parsenetlinkrouteattr", err) diff --git a/src/net/interface_plan9.go b/src/net/interface_plan9.go index 7c44566acf0165..93c783b56eec6e 100644 --- a/src/net/interface_plan9.go +++ b/src/net/interface_plan9.go @@ -57,6 +57,17 @@ func readInterface(i int) (*Interface, error) { } fields := getFields(line) + + // If the interface has no device file then we see two spaces between "device" and + // "maxtu" and getFields treats the two spaces as one delimiter. + // Insert a gap for the missing device name. + // See https://go.dev/issue/72060. + if stringslite.HasPrefix(line, "device maxtu ") { + fields = append(fields, "") + copy(fields[2:], fields[1:]) + fields[1] = "" + } + if len(fields) < 4 { return nil, errors.New("invalid interface status file: " + ifcstat) } @@ -163,7 +174,7 @@ func interfaceAddrTable(ifi *Interface) ([]Addr, error) { for line, ok := statusf.readLine(); ok; line, ok = statusf.readLine() { fields := getFields(line) if len(fields) < 1 { - return nil, errors.New("cannot parse IP address for interface: " + status) + continue } addr := fields[0] ip := ParseIP(addr) diff --git a/src/net/interface_test.go b/src/net/interface_test.go index a97d675e7e0bd6..72befca0d89c95 100644 --- a/src/net/interface_test.go +++ b/src/net/interface_test.go @@ -68,12 +68,14 @@ func TestInterfaces(t *testing.T) { t.Errorf("got %v; want %v", ifxi, ifi) } } - ifxn, err := InterfaceByName(ifi.Name) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(ifxn, &ifi) { - t.Errorf("got %v; want %v", ifxn, ifi) + if ifi.Name != "" { + ifxn, err := InterfaceByName(ifi.Name) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(ifxn, &ifi) { + t.Errorf("got %v; want %v", ifxn, ifi) + } } t.Logf("%s: flags=%v index=%d mtu=%d hwaddr=%v", ifi.Name, ifi.Flags, ifi.Index, ifi.MTU, ifi.HardwareAddr) } diff --git a/src/net/interface_windows.go b/src/net/interface_windows.go index 1b487dc474825f..7a5cb5723f1b6b 100644 --- a/src/net/interface_windows.go +++ b/src/net/interface_windows.go @@ -21,7 +21,7 @@ func adapterAddresses() ([]*windows.IpAdapterAddresses, error) { for { b = make([]byte, l) const flags = windows.GAA_FLAG_INCLUDE_PREFIX | windows.GAA_FLAG_INCLUDE_GATEWAYS - err := windows.GetAdaptersAddresses(syscall.AF_UNSPEC, flags, 0, (*windows.IpAdapterAddresses)(unsafe.Pointer(&b[0])), &l) + err := windows.GetAdaptersAddresses(syscall.AF_UNSPEC, flags, nil, (*windows.IpAdapterAddresses)(unsafe.Pointer(&b[0])), &l) if err == nil { if l == 0 { return nil, nil diff --git a/src/net/internal/cgotest/resstate.go b/src/net/internal/cgotest/resstate.go index 1b4871109eaa06..62cfa24e46fd0c 100644 --- a/src/net/internal/cgotest/resstate.go +++ b/src/net/internal/cgotest/resstate.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build cgo && darwin +//go:build !netgo && cgo && darwin package cgotest diff --git a/src/net/ip.go b/src/net/ip.go index 3e0e85e168a72f..e3ee6ca70a32dc 100644 --- a/src/net/ip.go +++ b/src/net/ip.go @@ -301,11 +301,18 @@ func (ip IP) String() string { if len(ip) != IPv4len && len(ip) != IPv6len { return "?" + hexString(ip) } - // If IPv4, use dotted notation. - if p4 := ip.To4(); len(p4) == IPv4len { - return netip.AddrFrom4([4]byte(p4)).String() + + var buf []byte + switch len(ip) { + case IPv4len: + const maxCap = len("255.255.255.255") + buf = make([]byte, 0, maxCap) + case IPv6len: + const maxCap = len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff") + buf = make([]byte, 0, maxCap) } - return netip.AddrFrom16([16]byte(ip)).String() + buf = ip.appendTo(buf) + return string(buf) } func hexString(b []byte) string { @@ -325,17 +332,41 @@ func ipEmptyString(ip IP) string { return ip.String() } -// MarshalText implements the [encoding.TextMarshaler] interface. +// appendTo appends the string representation of ip to b and returns the expanded b +// If len(ip) != IPv4len or IPv6len, it appends nothing. +func (ip IP) appendTo(b []byte) []byte { + // If IPv4, use dotted notation. + if p4 := ip.To4(); len(p4) == IPv4len { + ip = p4 + } + addr, _ := netip.AddrFromSlice(ip) + return addr.AppendTo(b) +} + +// AppendText implements the [encoding.TextAppender] interface. // The encoding is the same as returned by [IP.String], with one exception: -// When len(ip) is zero, it returns an empty slice. -func (ip IP) MarshalText() ([]byte, error) { +// When len(ip) is zero, it appends nothing. +func (ip IP) AppendText(b []byte) ([]byte, error) { if len(ip) == 0 { - return []byte(""), nil + return b, nil } if len(ip) != IPv4len && len(ip) != IPv6len { - return nil, &AddrError{Err: "invalid IP address", Addr: hexString(ip)} + return b, &AddrError{Err: "invalid IP address", Addr: hexString(ip)} + } + + return ip.appendTo(b), nil +} + +// MarshalText implements the [encoding.TextMarshaler] interface. +// The encoding is the same as returned by [IP.String], with one exception: +// When len(ip) is zero, it returns an empty slice. +func (ip IP) MarshalText() ([]byte, error) { + // 24 is satisfied with all IPv4 addresses and short IPv6 addresses + b, err := ip.AppendText(make([]byte, 0, 24)) + if err != nil { + return nil, err } - return []byte(ip.String()), nil + return b, nil } // UnmarshalText implements the [encoding.TextUnmarshaler] interface. diff --git a/src/net/ip_test.go b/src/net/ip_test.go index 11c0b752463a11..55c66fdf31e116 100644 --- a/src/net/ip_test.go +++ b/src/net/ip_test.go @@ -149,6 +149,15 @@ func TestMarshalEmptyIP(t *testing.T) { if !reflect.DeepEqual(got, []byte("")) { t.Errorf(`got %#v, want []byte("")`, got) } + + buf := make([]byte, 4) + got, err = ip.AppendText(buf) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, []byte("\x00\x00\x00\x00")) { + t.Errorf(`got %#v, want []byte("\x00\x00\x00\x00")`, got) + } } var ipStringTests = []*struct { @@ -266,9 +275,54 @@ func TestIPString(t *testing.T) { if out, err := tt.in.MarshalText(); !bytes.Equal(out, tt.byt) || !reflect.DeepEqual(err, tt.error) { t.Errorf("IP.MarshalText(%v) = %v, %v, want %v, %v", tt.in, out, err, tt.byt, tt.error) } + buf := make([]byte, 4, 32) + if out, err := tt.in.AppendText(buf); !bytes.Equal(out[4:], tt.byt) || !reflect.DeepEqual(err, tt.error) { + t.Errorf("IP.AppendText(%v) = %v, %v, want %v, %v", tt.in, out[4:], err, tt.byt, tt.error) + } + } +} + +func TestIPAppendTextNoAllocs(t *testing.T) { + // except the invalid IP + for _, tt := range ipStringTests[:len(ipStringTests)-1] { + allocs := int(testing.AllocsPerRun(1000, func() { + buf := make([]byte, 0, 64) + _, _ = tt.in.AppendText(buf) + })) + if allocs != 0 { + t.Errorf("IP(%q) AppendText allocs: %d times, want 0", tt.in, allocs) + } } } +func BenchmarkIPMarshalText(b *testing.B) { + b.Run("IPv4", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + ip := IP{192, 0, 2, 1} + for range b.N { + _, _ = ip.MarshalText() + } + }) + b.Run("IPv6", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + ip := IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0xa, 0, 0xb, 0, 0xc, 0, 0xd} + for range b.N { + _, _ = ip.MarshalText() + } + }) + b.Run("IPv6_long", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + // fd7a:115c:a1e0:ab12:4843:cd96:626b:430b + ip := IP{253, 122, 17, 92, 161, 224, 171, 18, 72, 67, 205, 150, 98, 107, 67, 11} + for range b.N { + _, _ = ip.MarshalText() + } + }) +} + var sink string func BenchmarkIPString(b *testing.B) { diff --git a/src/net/iprawsock.go b/src/net/iprawsock.go index 4c06b1b5aca446..26134d7e76a2ec 100644 --- a/src/net/iprawsock.go +++ b/src/net/iprawsock.go @@ -6,6 +6,7 @@ package net import ( "context" + "net/netip" "syscall" ) @@ -24,8 +25,18 @@ import ( // BUG(mikio): On JS and Plan 9, methods and functions related // to IPConn are not implemented. -// BUG(mikio): On Windows, the File method of IPConn is not -// implemented. +// BUG: On Windows, raw IP sockets are restricted by the operating system. +// Sending TCP data, sending UDP data with invalid source addresses, +// and calling bind with TCP protocol don't work. +// +// See Winsock reference for details. + +func ipAddrFromAddr(addr netip.Addr) *IPAddr { + return &IPAddr{ + IP: addr.AsSlice(), + Zone: addr.Zone(), + } +} // IPAddr represents the address of an IP end point. type IPAddr struct { @@ -209,11 +220,18 @@ func newIPConn(fd *netFD) *IPConn { return &IPConn{conn{fd}} } // If the IP field of raddr is nil or an unspecified IP address, the // local system is assumed. func DialIP(network string, laddr, raddr *IPAddr) (*IPConn, error) { + return dialIP(context.Background(), nil, network, laddr, raddr) +} + +func dialIP(ctx context.Context, dialer *Dialer, network string, laddr, raddr *IPAddr) (*IPConn, error) { if raddr == nil { return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} } sd := &sysDialer{network: network, address: raddr.String()} - c, err := sd.dialIP(context.Background(), laddr, raddr) + if dialer != nil { + sd.Dialer = *dialer + } + c, err := sd.dialIP(ctx, laddr, raddr) if err != nil { return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} } diff --git a/src/net/ipsock_plan9.go b/src/net/ipsock_plan9.go index c8d01804361177..6ae9cf3cc105f0 100644 --- a/src/net/ipsock_plan9.go +++ b/src/net/ipsock_plan9.go @@ -10,6 +10,7 @@ import ( "internal/itoa" "io/fs" "os" + "strconv" "syscall" ) @@ -128,6 +129,7 @@ func startPlan9(ctx context.Context, net string, addr Addr) (ctl *os.File, dest, clone, dest, err := queryCS1(ctx, proto, ip, port) if err != nil { + err = handlePlan9DNSError(err, net+":"+ip.String()+":"+strconv.Itoa(port)) return } f, err := os.OpenFile(clone, os.O_RDWR, 0) diff --git a/src/net/ipsock_posix.go b/src/net/ipsock_posix.go index 2aeabd44873f22..1ed37d7e6eaf25 100644 --- a/src/net/ipsock_posix.go +++ b/src/net/ipsock_posix.go @@ -237,8 +237,12 @@ func ipToSockaddr(family int, ip IP, port int, zone string) (syscall.Sockaddr, e func addrPortToSockaddrInet4(ap netip.AddrPort) (syscall.SockaddrInet4, error) { // ipToSockaddrInet4 has special handling here for zero length slices. // We do not, because netip has no concept of a generic zero IP address. + // + // addr is allowed to be an IPv4-mapped IPv6 address. + // As4 will unmap it to an IPv4 address. + // The error message is kept consistent with ipToSockaddrInet4. addr := ap.Addr() - if !addr.Is4() { + if !addr.Is4() && !addr.Is4In6() { return syscall.SockaddrInet4{}, &AddrError{Err: "non-IPv4 address", Addr: addr.String()} } sa := syscall.SockaddrInet4{ @@ -256,9 +260,6 @@ func addrPortToSockaddrInet6(ap netip.AddrPort) (syscall.SockaddrInet6, error) { // to an IPv4-mapped IPv6 address. // The error message is kept consistent with ipToSockaddrInet6. addr := ap.Addr() - if !addr.IsValid() { - return syscall.SockaddrInet6{}, &AddrError{Err: "non-IPv6 address", Addr: addr.String()} - } sa := syscall.SockaddrInet6{ Addr: addr.As16(), Port: int(ap.Port()), diff --git a/src/net/listen_test.go b/src/net/listen_test.go index 9100b3d9f7becd..59c1277a97966c 100644 --- a/src/net/listen_test.go +++ b/src/net/listen_test.go @@ -610,7 +610,8 @@ func TestIPv6MulticastListener(t *testing.T) { if !supportsIPv6() { t.Skip("IPv6 is not supported") } - if os.Getuid() != 0 { + // On Windows, the test can be run by non-admin users. + if runtime.GOOS != "windows" && os.Getuid() != 0 { t.Skip("must be root") } @@ -669,7 +670,7 @@ func checkMulticastListener(c *UDPConn, ip IP) error { func multicastRIBContains(ip IP) (bool, error) { switch runtime.GOOS { - case "aix", "dragonfly", "netbsd", "openbsd", "plan9", "solaris", "illumos", "windows": + case "aix", "dragonfly", "netbsd", "openbsd", "plan9", "solaris", "illumos": return true, nil // not implemented yet case "linux": if runtime.GOARCH == "arm" || runtime.GOARCH == "alpha" { diff --git a/src/net/lookup.go b/src/net/lookup.go index b04dfa23b9877b..d4be8eaa0e10c3 100644 --- a/src/net/lookup.go +++ b/src/net/lookup.go @@ -9,6 +9,7 @@ import ( "errors" "internal/nettrace" "internal/singleflight" + "internal/stringslite" "net/netip" "sync" @@ -535,9 +536,9 @@ func (r *Resolver) LookupSRV(ctx context.Context, service, proto, name string) ( // LookupMX returns the DNS MX records for the given domain name sorted by preference. // // The returned mail server names are validated to be properly -// formatted presentation-format domain names. If the response contains -// invalid names, those records are filtered out and an error -// will be returned alongside the remaining results, if any. +// formatted presentation-format domain names, or numeric IP addresses. +// If the response contains invalid names, those records are filtered out +// and an error will be returned alongside the remaining results, if any. // // LookupMX uses [context.Background] internally; to specify the context, use // [Resolver.LookupMX]. @@ -548,9 +549,9 @@ func LookupMX(name string) ([]*MX, error) { // LookupMX returns the DNS MX records for the given domain name sorted by preference. // // The returned mail server names are validated to be properly -// formatted presentation-format domain names. If the response contains -// invalid names, those records are filtered out and an error -// will be returned alongside the remaining results, if any. +// formatted presentation-format domain names, or numeric IP addresses. +// If the response contains invalid names, those records are filtered out +// and an error will be returned alongside the remaining results, if any. func (r *Resolver) LookupMX(ctx context.Context, name string) ([]*MX, error) { records, err := r.lookupMX(ctx, name) if err != nil { @@ -562,7 +563,12 @@ func (r *Resolver) LookupMX(ctx context.Context, name string) ([]*MX, error) { continue } if !isDomainName(mx.Host) { - continue + // Check for IP address. In practice we observe + // these with a trailing dot, so strip that. + ip, err := netip.ParseAddr(stringslite.TrimSuffix(mx.Host, ".")) + if err != nil || ip.Zone() != "" { + continue + } } filteredMX = append(filteredMX, mx) } @@ -614,6 +620,9 @@ func (r *Resolver) LookupNS(ctx context.Context, name string) ([]*NS, error) { // LookupTXT returns the DNS TXT records for the given domain name. // +// If a DNS TXT record holds multiple strings, they are concatenated as a +// single string. +// // LookupTXT uses [context.Background] internally; to specify the context, use // [Resolver.LookupTXT]. func LookupTXT(name string) ([]string, error) { @@ -621,6 +630,9 @@ func LookupTXT(name string) ([]string, error) { } // LookupTXT returns the DNS TXT records for the given domain name. +// +// If a DNS TXT record holds multiple strings, they are concatenated as a +// single string. func (r *Resolver) LookupTXT(ctx context.Context, name string) ([]string, error) { return r.lookupTXT(ctx, name) } diff --git a/src/net/lookup_plan9.go b/src/net/lookup_plan9.go index e3e371611f4465..c9bab29aded7ba 100644 --- a/src/net/lookup_plan9.go +++ b/src/net/lookup_plan9.go @@ -65,11 +65,7 @@ func query(ctx context.Context, filename, query string, bufSize int) (addrs []st case r := <-ch: return r.addrs, r.err case <-ctx.Done(): - return nil, &DNSError{ - Name: query, - Err: ctx.Err().Error(), - IsTimeout: ctx.Err() == context.DeadlineExceeded, - } + return nil, mapErr(ctx.Err()) } } @@ -143,7 +139,7 @@ func toLower(in string) string { func lookupProtocol(ctx context.Context, name string) (proto int, err error) { lines, err := query(ctx, netdir+"/cs", "!protocol="+toLower(name), 128) if err != nil { - return 0, err + return 0, newDNSError(err, name, "") } if len(lines) == 0 { return 0, UnknownNetworkError(name) @@ -229,16 +225,16 @@ func (*Resolver) lookupPortWithNetwork(ctx context.Context, network, errNetwork, lines, err := queryCS(ctx, network, "127.0.0.1", toLower(service)) if err != nil { if stringslite.HasSuffix(err.Error(), "can't translate service") { - return 0, &DNSError{Err: "unknown port", Name: errNetwork + "/" + service, IsNotFound: true} + return 0, newDNSError(errUnknownPort, errNetwork+"/"+service, "") } - return + return 0, newDNSError(err, errNetwork+"/"+service, "") } if len(lines) == 0 { - return 0, &DNSError{Err: "unknown port", Name: errNetwork + "/" + service, IsNotFound: true} + return 0, newDNSError(errUnknownPort, errNetwork+"/"+service, "") } f := getFields(lines[0]) if len(f) < 2 { - return 0, &DNSError{Err: "unknown port", Name: errNetwork + "/" + service, IsNotFound: true} + return 0, newDNSError(errUnknownPort, errNetwork+"/"+service, "") } s := f[1] if i := bytealg.IndexByteString(s, '!'); i >= 0 { @@ -247,7 +243,7 @@ func (*Resolver) lookupPortWithNetwork(ctx context.Context, network, errNetwork, if n, _, ok := dtoi(s); ok { return n, nil } - return 0, &DNSError{Err: "unknown port", Name: errNetwork + "/" + service, IsNotFound: true} + return 0, newDNSError(errUnknownPort, errNetwork+"/"+service, "") } func (r *Resolver) lookupCNAME(ctx context.Context, name string) (cname string, err error) { @@ -269,7 +265,7 @@ func (r *Resolver) lookupCNAME(ctx context.Context, name string) (cname string, return f[2] + ".", nil } } - return "", errors.New("bad response from ndb/dns") + return "", &DNSError{Err: "bad response from ndb/dns", Name: name} } func (r *Resolver) lookupSRV(ctx context.Context, service, proto, name string) (cname string, addrs []*SRV, err error) { diff --git a/src/net/lookup_test.go b/src/net/lookup_test.go index d106f98eef18a9..2a774100a8ec67 100644 --- a/src/net/lookup_test.go +++ b/src/net/lookup_test.go @@ -246,14 +246,10 @@ func TestLookupGmailTXT(t *testing.T) { if len(txts) == 0 { t.Error("got no record") } - found := false - for _, txt := range txts { - if strings.Contains(txt, tt.txt) && (strings.HasSuffix(txt, tt.host) || strings.HasSuffix(txt, tt.host+".")) { - found = true - break - } - } - if !found { + + if !slices.ContainsFunc(txts, func(txt string) bool { + return strings.Contains(txt, tt.txt) && (strings.HasSuffix(txt, tt.host) || strings.HasSuffix(txt, tt.host+".")) + }) { t.Errorf("got %v; want a record containing %s, %s", txts, tt.txt, tt.host) } } @@ -302,14 +298,7 @@ func TestLookupIPv6LinkLocalAddr(t *testing.T) { if err != nil { t.Fatal(err) } - found := false - for _, addr := range addrs { - if addr == "fe80::1%lo0" { - found = true - break - } - } - if !found { + if !slices.Contains(addrs, "fe80::1%lo0") { t.Skipf("not supported on %s", runtime.GOOS) } if _, err := LookupAddr("fe80::1%lo0"); err != nil { @@ -1431,8 +1420,8 @@ func testLookupNoData(t *testing.T, prefix string) { return } - var dnsErr *DNSError - if errors.As(err, &dnsErr) { + dnsErr, ok := errors.AsType[*DNSError](err) + if ok { succeeded := true if !dnsErr.IsNotFound { succeeded = false @@ -1466,8 +1455,7 @@ func testLookupNoData(t *testing.T, prefix string) { func TestLookupPortNotFound(t *testing.T) { allResolvers(t, func(t *testing.T) { _, err := LookupPort("udp", "_-unknown-service-") - var dnsErr *DNSError - if !errors.As(err, &dnsErr) || !dnsErr.IsNotFound { + if dnsErr, ok := errors.AsType[*DNSError](err); !ok || !dnsErr.IsNotFound { t.Fatalf("unexpected error: %v", err) } }) @@ -1486,8 +1474,7 @@ var tcpOnlyService = func() string { func TestLookupPortDifferentNetwork(t *testing.T) { allResolvers(t, func(t *testing.T) { _, err := LookupPort("udp", tcpOnlyService) - var dnsErr *DNSError - if !errors.As(err, &dnsErr) || !dnsErr.IsNotFound { + if dnsErr, ok := errors.AsType[*DNSError](err); !ok || !dnsErr.IsNotFound { t.Fatalf("unexpected error: %v", err) } }) diff --git a/src/net/lookup_unix.go b/src/net/lookup_unix.go index 382a2d44bb5cfd..86108939cd03d6 100644 --- a/src/net/lookup_unix.go +++ b/src/net/lookup_unix.go @@ -12,11 +12,9 @@ import ( "sync" ) -var onceReadProtocols sync.Once - -// readProtocols loads contents of /etc/protocols into protocols map +// readProtocolsOnce loads contents of /etc/protocols into protocols map // for quick access. -func readProtocols() { +var readProtocolsOnce = sync.OnceFunc(func() { file, err := open("/etc/protocols") if err != nil { return @@ -43,12 +41,12 @@ func readProtocols() { } } } -} +}) // lookupProtocol looks up IP protocol name in /etc/protocols and // returns correspondent protocol number. func lookupProtocol(_ context.Context, name string) (int, error) { - onceReadProtocols.Do(readProtocols) + readProtocolsOnce() return lookupProtocolMap(name) } @@ -89,8 +87,8 @@ func (r *Resolver) lookupPort(ctx context.Context, network, service string) (int func (r *Resolver) lookupCNAME(ctx context.Context, name string) (string, error) { order, conf := systemConf().hostLookupOrder(r, name) if order == hostLookupCgo { - if cname, err, ok := cgoLookupCNAME(ctx, name); ok { - return cname, err + if cname, err := cgoLookupCNAME(ctx, name); err == nil { + return cname, nil } } return r.goLookupCNAME(ctx, name, order, conf) diff --git a/src/net/lookup_windows.go b/src/net/lookup_windows.go index 7d415bee4f07b6..38034dc6014c93 100644 --- a/src/net/lookup_windows.go +++ b/src/net/lookup_windows.go @@ -52,7 +52,7 @@ func lookupProtocol(ctx context.Context, name string) (int, error) { proto int err error } - ch := make(chan result) // unbuffered + ch := make(chan result, 1) // buffer so that next goroutine never blocks go func() { if err := acquireThread(ctx); err != nil { ch <- result{err: mapErr(err)} @@ -62,10 +62,7 @@ func lookupProtocol(ctx context.Context, name string) (int, error) { runtime.LockOSThread() defer runtime.UnlockOSThread() proto, err := getprotobyname(name) - select { - case ch <- result{proto: proto, err: err}: - case <-ctx.Done(): - } + ch <- result{proto: proto, err: err} }() select { case r := <-ch: @@ -77,7 +74,7 @@ func lookupProtocol(ctx context.Context, name string) (int, error) { } return r.proto, r.err case <-ctx.Done(): - return 0, mapErr(ctx.Err()) + return 0, newDNSError(mapErr(ctx.Err()), name, "") } } @@ -110,11 +107,7 @@ func (r *Resolver) lookupIP(ctx context.Context, network, name string) ([]IPAddr getaddr := func() ([]IPAddr, error) { if err := acquireThread(ctx); err != nil { - return nil, &DNSError{ - Name: name, - Err: mapErr(err).Error(), - IsTimeout: ctx.Err() == context.DeadlineExceeded, - } + return nil, newDNSError(mapErr(err), name, "") } defer releaseThread() hints := syscall.AddrinfoW{ @@ -197,11 +190,7 @@ func (r *Resolver) lookupPort(ctx context.Context, network, service string) (int // TODO(bradfitz): finish ctx plumbing if err := acquireThread(ctx); err != nil { - return 0, &DNSError{ - Name: network + "/" + service, - Err: mapErr(err).Error(), - IsTimeout: ctx.Err() == context.DeadlineExceeded, - } + return 0, newDNSError(mapErr(err), network+"/"+service, "") } defer releaseThread() @@ -265,11 +254,7 @@ func (r *Resolver) lookupCNAME(ctx context.Context, name string) (string, error) // TODO(bradfitz): finish ctx plumbing if err := acquireThread(ctx); err != nil { - return "", &DNSError{ - Name: name, - Err: mapErr(err).Error(), - IsTimeout: ctx.Err() == context.DeadlineExceeded, - } + return "", newDNSError(mapErr(err), name, "") } defer releaseThread() var rec *syscall.DNSRecord @@ -295,11 +280,7 @@ func (r *Resolver) lookupSRV(ctx context.Context, service, proto, name string) ( } // TODO(bradfitz): finish ctx plumbing if err := acquireThread(ctx); err != nil { - return "", nil, &DNSError{ - Name: name, - Err: mapErr(err).Error(), - IsTimeout: ctx.Err() == context.DeadlineExceeded, - } + return "", nil, newDNSError(mapErr(err), name, "") } defer releaseThread() var target string @@ -330,11 +311,7 @@ func (r *Resolver) lookupMX(ctx context.Context, name string) ([]*MX, error) { } // TODO(bradfitz): finish ctx plumbing. if err := acquireThread(ctx); err != nil { - return nil, &DNSError{ - Name: name, - Err: mapErr(err).Error(), - IsTimeout: ctx.Err() == context.DeadlineExceeded, - } + return nil, newDNSError(mapErr(err), name, "") } defer releaseThread() var rec *syscall.DNSRecord @@ -359,11 +336,7 @@ func (r *Resolver) lookupNS(ctx context.Context, name string) ([]*NS, error) { } // TODO(bradfitz): finish ctx plumbing. if err := acquireThread(ctx); err != nil { - return nil, &DNSError{ - Name: name, - Err: mapErr(err).Error(), - IsTimeout: ctx.Err() == context.DeadlineExceeded, - } + return nil, newDNSError(mapErr(err), name, "") } defer releaseThread() var rec *syscall.DNSRecord @@ -387,11 +360,7 @@ func (r *Resolver) lookupTXT(ctx context.Context, name string) ([]string, error) } // TODO(bradfitz): finish ctx plumbing. if err := acquireThread(ctx); err != nil { - return nil, &DNSError{ - Name: name, - Err: mapErr(err).Error(), - IsTimeout: ctx.Err() == context.DeadlineExceeded, - } + return nil, newDNSError(mapErr(err), name, "") } defer releaseThread() var rec *syscall.DNSRecord @@ -420,11 +389,7 @@ func (r *Resolver) lookupAddr(ctx context.Context, addr string) ([]string, error // TODO(bradfitz): finish ctx plumbing. if err := acquireThread(ctx); err != nil { - return nil, &DNSError{ - Name: addr, - Err: mapErr(err).Error(), - IsTimeout: ctx.Err() == context.DeadlineExceeded, - } + return nil, newDNSError(mapErr(err), addr, "") } defer releaseThread() arpa, err := reverseaddr(addr) diff --git a/src/net/mail/example_test.go b/src/net/mail/example_test.go index d325dc791f9319..9fadda2463abe0 100644 --- a/src/net/mail/example_test.go +++ b/src/net/mail/example_test.go @@ -10,6 +10,7 @@ import ( "log" "net/mail" "strings" + "time" ) func ExampleParseAddressList() { @@ -75,3 +76,17 @@ Message body // Subject: Gophers at Gophercon // Message body } + +func ExampleParseDate() { + dateStr := "Wed, 09 Oct 2024 09:55:06 -0700" + + t, err := mail.ParseDate(dateStr) + if err != nil { + log.Fatalf("Failed to parse date: %v", err) + } + + fmt.Println(t.Format(time.RFC3339)) + + // Output: + // 2024-10-09T09:55:06-07:00 +} diff --git a/src/net/mail/message.go b/src/net/mail/message.go index 21b075e78affbe..1502b3596252ba 100644 --- a/src/net/mail/message.go +++ b/src/net/mail/message.go @@ -115,12 +115,7 @@ func readHeader(r *textproto.Reader) (map[string][]string, error) { // Layouts suitable for passing to time.Parse. // These are tried in order. -var ( - dateLayoutsBuildOnce sync.Once - dateLayouts []string -) - -func buildDateLayouts() { +var dateLayouts = sync.OnceValue(func() []string { // Generate layouts based on RFC 5322, section 3.3. dows := [...]string{"", "Mon, "} // day-of-week @@ -130,23 +125,27 @@ func buildDateLayouts() { // "-0700 (MST)" is not in RFC 5322, but is common. zones := [...]string{"-0700", "MST", "UT"} // zone = (("+" / "-") 4DIGIT) / "UT" / "GMT" / ... + total := len(dows) * len(days) * len(years) * len(seconds) * len(zones) + layouts := make([]string, 0, total) + for _, dow := range dows { for _, day := range days { for _, year := range years { for _, second := range seconds { for _, zone := range zones { s := dow + day + " Jan " + year + " 15:04" + second + " " + zone - dateLayouts = append(dateLayouts, s) + layouts = append(layouts, s) } } } } } -} + + return layouts +}) // ParseDate parses an RFC 5322 date string. func ParseDate(date string) (time.Time, error) { - dateLayoutsBuildOnce.Do(buildDateLayouts) // CR and LF must match and are tolerated anywhere in the date field. date = strings.ReplaceAll(date, "\r\n", "") if strings.Contains(date, "\r") { @@ -184,7 +183,7 @@ func ParseDate(date string) (time.Time, error) { if !p.skipCFWS() { return time.Time{}, errors.New("mail: misformatted parenthetical comment") } - for _, layout := range dateLayouts { + for _, layout := range dateLayouts() { t, err := time.Parse(layout, date) if err == nil { return t, nil @@ -725,7 +724,8 @@ func (p *addrParser) consumeDomainLiteral() (string, error) { } // Parse the dtext - var dtext string + dtext := p.s + dtextLen := 0 for { if p.empty() { return "", errors.New("mail: unclosed domain-literal") @@ -742,9 +742,10 @@ func (p *addrParser) consumeDomainLiteral() (string, error) { return "", fmt.Errorf("mail: bad character in domain-literal: %q", r) } - dtext += p.s[:size] + dtextLen += size p.s = p.s[size:] } + dtext = dtext[:dtextLen] // Skip the trailing ] if !p.consume(']') { diff --git a/src/net/main_plan9_test.go b/src/net/main_plan9_test.go index 2bc5be88be6481..ca5f19adc149ac 100644 --- a/src/net/main_plan9_test.go +++ b/src/net/main_plan9_test.go @@ -4,6 +4,8 @@ package net +import "os/exec" + func installTestHooks() {} func uninstallTestHooks() {} @@ -14,3 +16,5 @@ func forceCloseSockets() {} func enableSocketConnect() {} func disableSocketConnect(network string) {} + +func addCmdInheritedHandle(cmd *exec.Cmd, fd uintptr) {} diff --git a/src/net/main_test.go b/src/net/main_test.go index e5767f7c7c12f6..66735962f10373 100644 --- a/src/net/main_test.go +++ b/src/net/main_test.go @@ -185,7 +185,7 @@ func runningGoroutines() []string { var gss []string b := make([]byte, 2<<20) b = b[:runtime.Stack(b, true)] - for _, s := range strings.Split(string(b), "\n\n") { + for s := range strings.SplitSeq(string(b), "\n\n") { _, stack, _ := strings.Cut(s, "\n") stack = strings.TrimSpace(stack) if !strings.Contains(stack, "created by net") { diff --git a/src/net/main_unix_test.go b/src/net/main_unix_test.go index e7a5b4fe9ad4ef..49d15b963e2630 100644 --- a/src/net/main_unix_test.go +++ b/src/net/main_unix_test.go @@ -6,7 +6,10 @@ package net -import "internal/poll" +import ( + "internal/poll" + "os/exec" +) var ( // Placeholders for saving original socket system calls. @@ -53,3 +56,5 @@ func forceCloseSockets() { poll.CloseFunc(s) } } + +func addCmdInheritedHandle(cmd *exec.Cmd, fd uintptr) {} diff --git a/src/net/main_wasm_test.go b/src/net/main_wasm_test.go index b8196bb283dc10..2dcfdabb3b444b 100644 --- a/src/net/main_wasm_test.go +++ b/src/net/main_wasm_test.go @@ -6,8 +6,12 @@ package net +import "os/exec" + func installTestHooks() {} func uninstallTestHooks() {} func forceCloseSockets() {} + +func addCmdInheritedHandle(cmd *exec.Cmd, fd uintptr) {} diff --git a/src/net/main_windows_test.go b/src/net/main_windows_test.go index bc024c0bbd82d0..425030133535ca 100644 --- a/src/net/main_windows_test.go +++ b/src/net/main_windows_test.go @@ -4,7 +4,11 @@ package net -import "internal/poll" +import ( + "internal/poll" + "os/exec" + "syscall" +) var ( // Placeholders for saving original socket system calls. @@ -40,3 +44,14 @@ func forceCloseSockets() { poll.CloseFunc(s) } } + +func addCmdInheritedHandle(cmd *exec.Cmd, fd uintptr) { + // Inherited handles are not inherited by default in Windows. + // We need to set the handle inheritance flag explicitly. + // See https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessa#parameters + // for more details. + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.AdditionalInheritedHandles = append(cmd.SysProcAttr.AdditionalInheritedHandles, syscall.Handle(fd)) +} diff --git a/src/net/mockserver_test.go b/src/net/mockserver_test.go index 4d5e79a5926529..66b8cbe40da536 100644 --- a/src/net/mockserver_test.go +++ b/src/net/mockserver_test.go @@ -509,6 +509,10 @@ func packetTransceiver(c PacketConn, wb []byte, dst Addr, ch chan<- error) { func spawnTestSocketPair(t testing.TB, net string) (client, server Conn) { t.Helper() + if !testableNetwork(net) { + t.Skipf("network %q not supported", net) + } + ln := newLocalListener(t, net) defer ln.Close() var cerr, serr error @@ -536,19 +540,12 @@ func spawnTestSocketPair(t testing.TB, net string) (client, server Conn) { func startTestSocketPeer(t testing.TB, conn Conn, op string, chunkSize, totalSize int) (func(t testing.TB), error) { t.Helper() - - if runtime.GOOS == "windows" { - // TODO(panjf2000): Windows has not yet implemented FileConn, - // remove this when it's implemented in https://go.dev/issues/9503. - t.Fatalf("startTestSocketPeer is not supported on %s", runtime.GOOS) - } - f, err := conn.(interface{ File() (*os.File, error) }).File() if err != nil { return nil, err } - cmd := testenv.Command(t, os.Args[0]) + cmd := testenv.Command(t, testenv.Executable(t)) cmd.Env = []string{ "GO_NET_TEST_TRANSFER=1", "GO_NET_TEST_TRANSFER_OP=" + op, @@ -556,7 +553,14 @@ func startTestSocketPeer(t testing.TB, conn Conn, op string, chunkSize, totalSiz "GO_NET_TEST_TRANSFER_TOTAL_SIZE=" + strconv.Itoa(totalSize), "TMPDIR=" + os.Getenv("TMPDIR"), } - cmd.ExtraFiles = append(cmd.ExtraFiles, f) + if runtime.GOOS == "windows" { + // Windows doesn't support ExtraFiles + fd := f.Fd() + cmd.Env = append(cmd.Env, "GO_NET_TEST_TRANSFER_FD="+strconv.FormatUint(uint64(fd), 10)) + addCmdInheritedHandle(cmd, fd) + } else { + cmd.ExtraFiles = append(cmd.ExtraFiles, f) + } cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -586,7 +590,17 @@ func init() { } defer os.Exit(0) - f := os.NewFile(uintptr(3), "splice-test-conn") + var fd uintptr + if runtime.GOOS == "windows" { + v, err := strconv.ParseUint(os.Getenv("GO_NET_TEST_TRANSFER_FD"), 10, 0) + if err != nil { + log.Fatal(err) + } + fd = uintptr(v) + } else { + fd = uintptr(3) + } + f := os.NewFile(fd, "splice-test-conn") defer f.Close() conn, err := FileConn(f) diff --git a/src/net/mptcpsock_linux.go b/src/net/mptcpsock_linux.go index b2ac3ee7182a09..5b58418653fbf5 100644 --- a/src/net/mptcpsock_linux.go +++ b/src/net/mptcpsock_linux.go @@ -16,7 +16,7 @@ import ( var ( mptcpOnce sync.Once mptcpAvailable bool - hasSOLMPTCP bool + hasSOLMPTCP bool // only valid if mptcpAvailable is true ) // These constants aren't in the syscall package, which is frozen @@ -34,10 +34,17 @@ func supportsMultipathTCP() bool { // Check that MPTCP is supported by attempting to create an MPTCP socket and by // looking at the returned error if any. func initMPTCPavailable() { - s, err := sysSocket(syscall.AF_INET, syscall.SOCK_STREAM, _IPPROTO_MPTCP) + family := syscall.AF_INET + if !supportsIPv4() { + family = syscall.AF_INET6 + } + s, err := sysSocket(family, syscall.SOCK_STREAM, _IPPROTO_MPTCP) + switch { case errors.Is(err, syscall.EPROTONOSUPPORT): // Not supported: >= v5.6 + return case errors.Is(err, syscall.EINVAL): // Not supported: < v5.6 + return case err == nil: // Supported and no error poll.CloseFunc(s) fallthrough @@ -46,9 +53,8 @@ func initMPTCPavailable() { mptcpAvailable = true } - major, minor := unix.KernelVersion() - // SOL_MPTCP only supported from kernel 5.16 - hasSOLMPTCP = major > 5 || (major == 5 && minor >= 16) + // SOL_MPTCP only supported from kernel 5.16. + hasSOLMPTCP = unix.KernelVersionGE(5, 16) } func (sd *sysDialer) dialMPTCP(ctx context.Context, laddr, raddr *TCPAddr) (*TCPConn, error) { @@ -119,6 +125,10 @@ func isUsingMPTCPProto(fd *netFD) bool { // Please look at the description of hasFallenBack (kernel >=5.16) and // isUsingMPTCPProto methods for more details about what is being checked here. func isUsingMultipathTCP(fd *netFD) bool { + if !supportsMultipathTCP() { + return false + } + if hasSOLMPTCP { return !hasFallenBack(fd) } diff --git a/src/net/net.go b/src/net/net.go index f8b5834acba56a..72f57721550982 100644 --- a/src/net/net.go +++ b/src/net/net.go @@ -68,6 +68,11 @@ GODEBUG environment variable (see package runtime) to go or cgo, as in: The decision can also be forced while building the Go source tree by setting the netgo or netcgo build tag. +The netgo build tag disables entirely the use of the native (CGO) resolver, +meaning the Go resolver is the only one that can be used. +With the netcgo build tag the native and the pure Go resolver are compiled into the binary, +but the native (CGO) resolver is preferred over the Go resolver. +With netcgo, the Go resolver can still be forced at runtime with GODEBUG=netdns=go. A numeric netdns setting, as in GODEBUG=netdns=1, causes the resolver to print debugging information about its decisions. @@ -129,6 +134,8 @@ type Conn interface { // Close closes the connection. // Any blocked Read or Write operations will be unblocked and return errors. + // Close may or may not block until any buffered data is sent; + // for TCP connections see [*TCPConn.SetLinger]. Close() error // LocalAddr returns the local network address, if known. @@ -301,6 +308,9 @@ func (c *conn) SetWriteBuffer(bytes int) error { // The returned os.File's file descriptor is different from the connection's. // Attempting to change properties of the original using this duplicate // may or may not have the desired effect. +// +// On Windows, the returned os.File's file descriptor is not usable +// on other processes. func (c *conn) File() (f *os.File, err error) { f, err = c.fd.dup() if err != nil { diff --git a/src/net/net_fake.go b/src/net/net_fake.go index f7eb28e01ac37d..cc35d90a26742f 100644 --- a/src/net/net_fake.go +++ b/src/net/net_fake.go @@ -1082,11 +1082,19 @@ func (ffd *fakeNetFD) writeMsg(p []byte, oob []byte, sa syscall.Sockaddr) (n int return n, 0, err } -func (ffd *fakeNetFD) writeMsgInet4(p []byte, oob []byte, sa *syscall.SockaddrInet4) (n int, oobn int, err error) { +func (ffd *fakeNetFD) writeMsgInet4(p []byte, oob []byte, sa4 *syscall.SockaddrInet4) (n int, oobn int, err error) { + var sa syscall.Sockaddr + if sa4 != nil { + sa = sa4 + } return ffd.writeMsg(p, oob, sa) } -func (ffd *fakeNetFD) writeMsgInet6(p []byte, oob []byte, sa *syscall.SockaddrInet6) (n int, oobn int, err error) { +func (ffd *fakeNetFD) writeMsgInet6(p []byte, oob []byte, sa6 *syscall.SockaddrInet6) (n int, oobn int, err error) { + var sa syscall.Sockaddr + if sa6 != nil { + sa = sa6 + } return ffd.writeMsg(p, oob, sa) } diff --git a/src/net/net_test.go b/src/net/net_test.go index 4a5dc3b73a6b44..637c95540f41bc 100644 --- a/src/net/net_test.go +++ b/src/net/net_test.go @@ -11,6 +11,7 @@ import ( "net/internal/socktest" "os" "runtime" + "sync" "testing" "time" ) @@ -23,7 +24,6 @@ func TestCloseRead(t *testing.T) { t.Parallel() for _, network := range []string{"tcp", "unix", "unixpacket"} { - network := network t.Run(network, func(t *testing.T) { if !testableNetwork(network) { t.Skipf("network %s is not testable on the current platform", network) @@ -82,7 +82,6 @@ func TestCloseWrite(t *testing.T) { } for _, network := range []string{"tcp", "unix", "unixpacket"} { - network := network t.Run(network, func(t *testing.T) { if !testableNetwork(network) { t.Skipf("network %s is not testable on the current platform", network) @@ -97,12 +96,12 @@ func TestCloseWrite(t *testing.T) { } // Workaround for https://go.dev/issue/49352. - // On arm64 macOS (current as of macOS 12.4), + // On Windows and arm64 macOS (current as of macOS 12.4), // reading from a socket at the same time as the client // is closing it occasionally hangs for 60 seconds before // returning ECONNRESET. Sleep for a bit to give the // socket time to close before trying to read from it. - if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + if runtime.GOOS == "windows" || (runtime.GOOS == "darwin" && runtime.GOARCH == "arm64") { time.Sleep(10 * time.Millisecond) } @@ -184,7 +183,6 @@ func TestCloseWrite(t *testing.T) { func TestConnClose(t *testing.T) { t.Parallel() for _, network := range []string{"tcp", "unix", "unixpacket"} { - network := network t.Run(network, func(t *testing.T) { if !testableNetwork(network) { t.Skipf("network %s is not testable on the current platform", network) @@ -226,7 +224,6 @@ func TestConnClose(t *testing.T) { func TestListenerClose(t *testing.T) { t.Parallel() for _, network := range []string{"tcp", "unix", "unixpacket"} { - network := network t.Run(network, func(t *testing.T) { if !testableNetwork(network) { t.Skipf("network %s is not testable on the current platform", network) @@ -264,7 +261,6 @@ func TestListenerClose(t *testing.T) { func TestPacketConnClose(t *testing.T) { t.Parallel() for _, network := range []string{"udp", "unixgram"} { - network := network t.Run(network, func(t *testing.T) { if !testableNetwork(network) { t.Skipf("network %s is not testable on the current platform", network) @@ -348,7 +344,6 @@ func TestAcceptIgnoreAbortedConnRequest(t *testing.T) { func TestZeroByteRead(t *testing.T) { t.Parallel() for _, network := range []string{"tcp", "unix", "unixpacket"} { - network := network t.Run(network, func(t *testing.T) { if !testableNetwork(network) { t.Skipf("network %s is not testable on the current platform", network) @@ -508,6 +503,46 @@ func TestCloseUnblocksRead(t *testing.T) { withTCPConnPair(t, client, server) } +// Issue 72770: verify that a blocked UDP read is woken up by a Close. +func TestCloseUnblocksReadUDP(t *testing.T) { + t.Parallel() + var ( + mu sync.Mutex + done bool + ) + defer func() { + mu.Lock() + defer mu.Unlock() + done = true + }() + pc, err := ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + time.AfterFunc(250*time.Millisecond, func() { + mu.Lock() + defer mu.Unlock() + if done { + return + } + t.Logf("closing conn...") + pc.Close() + }) + timer := time.AfterFunc(time.Second*10, func() { + panic("timeout waiting for Close") + }) + defer timer.Stop() + + n, src, err := pc.(*UDPConn).ReadFromUDPAddrPort([]byte{}) + + // Check for n > 0. Checking err == nil alone isn't enough; + // on macOS, it returns (n=0, src=0.0.0.0:0, err=nil). + if n > 0 { + t.Fatalf("unexpected Read success from ReadFromUDPAddrPort; read %d bytes from %v, err=%v", n, src, err) + } + t.Logf("got expected UDP read error") +} + // Issue 24808: verify that ECONNRESET is not temporary for read. func TestNotTemporaryRead(t *testing.T) { t.Parallel() diff --git a/src/net/net_windows_test.go b/src/net/net_windows_test.go index 50554c05c5eb2a..0a5c77f032527e 100644 --- a/src/net/net_windows_test.go +++ b/src/net/net_windows_test.go @@ -100,7 +100,7 @@ func TestAcceptIgnoreSomeErrors(t *testing.T) { defer ln.Close() // Start child process that connects to our listener. - cmd := exec.Command(os.Args[0], "-test.run=TestAcceptIgnoreSomeErrors") + cmd := exec.Command(testenv.Executable(t), "-test.run=^TestAcceptIgnoreSomeErrors$") cmd.Env = append(os.Environ(), "GOTEST_DIAL_ADDR="+ln.Addr().String()) stdout, err := cmd.StdoutPipe() if err != nil { @@ -240,8 +240,7 @@ func netshInterfaceIPShowInterface(ipver string, ifaces map[string]bool) error { //Metric : 10 //... var name string - lines := bytes.Split(out, []byte{'\r', '\n'}) - for _, line := range lines { + for line := range bytes.SplitSeq(out, []byte{'\r', '\n'}) { if bytes.HasPrefix(line, []byte("Interface ")) && bytes.HasSuffix(line, []byte(" Parameters")) { f := line[len("Interface "):] f = f[:len(f)-len(" Parameters")] @@ -303,7 +302,7 @@ func TestInterfacesWithNetsh(t *testing.T) { } slices.Sort(want) - if strings.Join(want, "/") != strings.Join(have, "/") { + if !slices.Equal(want, have) { t.Fatalf("unexpected interface list %q, want %q", have, want) } } @@ -330,8 +329,7 @@ func netshInterfaceIPv4ShowAddress(name string, netshOutput []byte) []string { addrs := make([]string, 0) var addr, subnetprefix string var processingOurInterface bool - lines := bytes.Split(netshOutput, []byte{'\r', '\n'}) - for _, line := range lines { + for line := range bytes.SplitSeq(netshOutput, []byte{'\r', '\n'}) { if !processingOurInterface { if !bytes.HasPrefix(line, []byte("Configuration for interface")) { continue @@ -398,8 +396,7 @@ func netshInterfaceIPv6ShowAddress(name string, netshOutput []byte) []string { // TODO: need to test ipv6 netmask too, but netsh does not outputs it var addr string addrs := make([]string, 0) - lines := bytes.Split(netshOutput, []byte{'\r', '\n'}) - for _, line := range lines { + for line := range bytes.SplitSeq(netshOutput, []byte{'\r', '\n'}) { if addr != "" { if len(line) == 0 { addr = "" @@ -490,7 +487,7 @@ func TestInterfaceAddrsWithNetsh(t *testing.T) { want = append(want, wantIPv6...) slices.Sort(want) - if strings.Join(want, "/") != strings.Join(have, "/") { + if !slices.Equal(want, have) { t.Errorf("%s: unexpected addresses list %q, want %q", ifi.Name, have, want) } } @@ -584,8 +581,7 @@ func TestInterfaceHardwareAddrWithGetmac(t *testing.T) { want[cname] = addr group = make(map[string]string) } - lines := bytes.Split(out, []byte{'\r', '\n'}) - for _, line := range lines { + for line := range bytes.SplitSeq(out, []byte{'\r', '\n'}) { if len(line) == 0 { processGroup() continue diff --git a/src/net/netip/export_test.go b/src/net/netip/export_test.go index b2fae1aa47eedc..777a76a6b26401 100644 --- a/src/net/netip/export_test.go +++ b/src/net/netip/export_test.go @@ -34,5 +34,3 @@ var TestAppendToMarshal = testAppendToMarshal func (a Addr) IsZero() bool { return a.isZero() } func (p Prefix) IsZero() bool { return p.isZero() } - -func (p Prefix) Compare(p2 Prefix) int { return p.compare(p2) } diff --git a/src/net/netip/netip.go b/src/net/netip/netip.go index a1e93cb29bfbd9..b1b15b47287de2 100644 --- a/src/net/netip/netip.go +++ b/src/net/netip/netip.go @@ -102,8 +102,8 @@ func AddrFrom4(addr [4]byte) Addr { func AddrFrom16(addr [16]byte) Addr { return Addr{ addr: uint128{ - byteorder.BeUint64(addr[:8]), - byteorder.BeUint64(addr[8:]), + byteorder.BEUint64(addr[:8]), + byteorder.BEUint64(addr[8:]), }, z: z6noz, } @@ -462,7 +462,9 @@ func (ip Addr) Is4() bool { return ip.z == z4 } -// Is4In6 reports whether ip is an IPv4-mapped IPv6 address. +// Is4In6 reports whether ip is an "IPv4-mapped IPv6 address" +// as defined by RFC 4291. +// That is, it reports whether ip is in ::ffff:0:0/96. func (ip Addr) Is4In6() bool { return ip.Is6() && ip.addr.hi == 0 && ip.addr.lo>>32 == 0xffff } @@ -700,8 +702,8 @@ func (ip Addr) Prefix(b int) (Prefix, error) { // [Addr.Zone] method to get it). // The ip zero value returns all zeroes. func (ip Addr) As16() (a16 [16]byte) { - byteorder.BePutUint64(a16[:8], ip.addr.hi) - byteorder.BePutUint64(a16[8:], ip.addr.lo) + byteorder.BEPutUint64(a16[:8], ip.addr.hi) + byteorder.BEPutUint64(a16[8:], ip.addr.lo) return a16 } @@ -710,7 +712,7 @@ func (ip Addr) As16() (a16 [16]byte) { // Note that 0.0.0.0 is not the zero Addr. func (ip Addr) As4() (a4 [4]byte) { if ip.z == z4 || ip.Is4In6() { - byteorder.BePutUint32(a4[:], uint32(ip.addr.lo)) + byteorder.BEPutUint32(a4[:], uint32(ip.addr.lo)) return a4 } if ip.z == z0 { @@ -726,12 +728,12 @@ func (ip Addr) AsSlice() []byte { return nil case z4: var ret [4]byte - byteorder.BePutUint32(ret[:], uint32(ip.addr.lo)) + byteorder.BEPutUint32(ret[:], uint32(ip.addr.lo)) return ret[:] default: var ret [16]byte - byteorder.BePutUint64(ret[:8], ip.addr.hi) - byteorder.BePutUint64(ret[8:], ip.addr.lo) + byteorder.BEPutUint64(ret[:8], ip.addr.hi) + byteorder.BEPutUint64(ret[8:], ip.addr.lo) return ret[:] } } @@ -966,27 +968,32 @@ func (ip Addr) StringExpanded() string { return string(ret) } +// AppendText implements the [encoding.TextAppender] interface, +// It is the same as [Addr.AppendTo]. +func (ip Addr) AppendText(b []byte) ([]byte, error) { + return ip.AppendTo(b), nil +} + // MarshalText implements the [encoding.TextMarshaler] interface, // The encoding is the same as returned by [Addr.String], with one exception: // If ip is the zero [Addr], the encoding is the empty string. func (ip Addr) MarshalText() ([]byte, error) { + buf := []byte{} switch ip.z { case z0: - return []byte(""), nil case z4: - max := len("255.255.255.255") - b := make([]byte, 0, max) - return ip.appendTo4(b), nil + const maxCap = len("255.255.255.255") + buf = make([]byte, 0, maxCap) default: if ip.Is4In6() { - max := len("::ffff:255.255.255.255%enp5s0") - b := make([]byte, 0, max) - return ip.appendTo4In6(b), nil + const maxCap = len("::ffff:255.255.255.255%enp5s0") + buf = make([]byte, 0, maxCap) + break } - max := len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0") - b := make([]byte, 0, max) - return ip.appendTo6(b), nil + const maxCap = len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0") + buf = make([]byte, 0, maxCap) } + return ip.AppendText(buf) } // UnmarshalText implements the encoding.TextUnmarshaler interface. @@ -1004,22 +1011,29 @@ func (ip *Addr) UnmarshalText(text []byte) error { return err } -func (ip Addr) marshalBinaryWithTrailingBytes(trailingBytes int) []byte { - var b []byte +// AppendBinary implements the [encoding.BinaryAppender] interface. +func (ip Addr) AppendBinary(b []byte) ([]byte, error) { switch ip.z { case z0: - b = make([]byte, trailingBytes) case z4: - b = make([]byte, 4+trailingBytes) - byteorder.BePutUint32(b, uint32(ip.addr.lo)) + b = byteorder.BEAppendUint32(b, uint32(ip.addr.lo)) default: - z := ip.Zone() - b = make([]byte, 16+len(z)+trailingBytes) - byteorder.BePutUint64(b[:8], ip.addr.hi) - byteorder.BePutUint64(b[8:], ip.addr.lo) - copy(b[16:], z) + b = byteorder.BEAppendUint64(b, ip.addr.hi) + b = byteorder.BEAppendUint64(b, ip.addr.lo) + b = append(b, ip.Zone()...) + } + return b, nil +} + +func (ip Addr) marshalBinarySize() int { + switch ip.z { + case z0: + return 0 + case z4: + return 4 + default: + return 16 + len(ip.Zone()) } - return b } // MarshalBinary implements the [encoding.BinaryMarshaler] interface. @@ -1027,7 +1041,7 @@ func (ip Addr) marshalBinaryWithTrailingBytes(trailingBytes int) []byte { // the 4-byte form for an IPv4 address, // and the 16-byte form with zone appended for an IPv6 address. func (ip Addr) MarshalBinary() ([]byte, error) { - return ip.marshalBinaryWithTrailingBytes(0), nil + return ip.AppendBinary(make([]byte, 0, ip.marshalBinarySize())) } // UnmarshalBinary implements the [encoding.BinaryUnmarshaler] interface. @@ -1198,21 +1212,27 @@ func (p AddrPort) AppendTo(b []byte) []byte { return b } +// AppendText implements the [encoding.TextAppender] interface. The +// encoding is the same as returned by [AddrPort.AppendTo]. +func (p AddrPort) AppendText(b []byte) ([]byte, error) { + return p.AppendTo(b), nil +} + // MarshalText implements the [encoding.TextMarshaler] interface. The // encoding is the same as returned by [AddrPort.String], with one exception: if // p.Addr() is the zero [Addr], the encoding is the empty string. func (p AddrPort) MarshalText() ([]byte, error) { - var max int + buf := []byte{} switch p.ip.z { case z0: case z4: - max = len("255.255.255.255:65535") + const maxCap = len("255.255.255.255:65535") + buf = make([]byte, 0, maxCap) default: - max = len("[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0]:65535") + const maxCap = len("[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0]:65535") + buf = make([]byte, 0, maxCap) } - b := make([]byte, 0, max) - b = p.AppendTo(b) - return b, nil + return p.AppendText(buf) } // UnmarshalText implements the encoding.TextUnmarshaler @@ -1228,13 +1248,22 @@ func (p *AddrPort) UnmarshalText(text []byte) error { return err } +// AppendBinary implements the [encoding.BinaryAppendler] interface. +// It returns [Addr.AppendBinary] with an additional two bytes appended +// containing the port in little-endian. +func (p AddrPort) AppendBinary(b []byte) ([]byte, error) { + b, err := p.Addr().AppendBinary(b) + if err != nil { + return nil, err + } + return byteorder.LEAppendUint16(b, p.Port()), nil +} + // MarshalBinary implements the [encoding.BinaryMarshaler] interface. // It returns [Addr.MarshalBinary] with an additional two bytes appended // containing the port in little-endian. func (p AddrPort) MarshalBinary() ([]byte, error) { - b := p.Addr().marshalBinaryWithTrailingBytes(2) - byteorder.LePutUint16(b[len(b)-2:], p.Port()) - return b, nil + return p.AppendBinary(make([]byte, 0, p.Addr().marshalBinarySize()+2)) } // UnmarshalBinary implements the [encoding.BinaryUnmarshaler] interface. @@ -1248,7 +1277,7 @@ func (p *AddrPort) UnmarshalBinary(b []byte) error { if err != nil { return err } - *p = AddrPortFrom(addr, byteorder.LeUint16(b[len(b)-2:])) + *p = AddrPortFrom(addr, byteorder.LEUint16(b[len(b)-2:])) return nil } @@ -1301,21 +1330,23 @@ func (p Prefix) isZero() bool { return p == Prefix{} } // IsSingleIP reports whether p contains exactly one IP. func (p Prefix) IsSingleIP() bool { return p.IsValid() && p.Bits() == p.ip.BitLen() } -// compare returns an integer comparing two prefixes. +// Compare returns an integer comparing two prefixes. // The result will be 0 if p == p2, -1 if p < p2, and +1 if p > p2. // Prefixes sort first by validity (invalid before valid), then -// address family (IPv4 before IPv6), then prefix length, then -// address. -// -// Unexported for Go 1.22 because we may want to compare by p.Addr first. -// See post-acceptance discussion on go.dev/issue/61642. -func (p Prefix) compare(p2 Prefix) int { - if c := cmp.Compare(p.Addr().BitLen(), p2.Addr().BitLen()); c != 0 { +// address family (IPv4 before IPv6), then masked prefix address, then +// prefix length, then unmasked address. +func (p Prefix) Compare(p2 Prefix) int { + // Aside from sorting based on the masked address, this use of + // Addr.Compare also enforces the valid vs. invalid and address + // family ordering for the prefix. + if c := p.Masked().Addr().Compare(p2.Masked().Addr()); c != 0 { return c } + if c := cmp.Compare(p.Bits(), p2.Bits()); c != 0 { return c } + return p.Addr().Compare(p2.Addr()) } @@ -1487,21 +1518,27 @@ func (p Prefix) AppendTo(b []byte) []byte { return b } +// AppendText implements the [encoding.TextAppender] interface. +// It is the same as [Prefix.AppendTo]. +func (p Prefix) AppendText(b []byte) ([]byte, error) { + return p.AppendTo(b), nil +} + // MarshalText implements the [encoding.TextMarshaler] interface, // The encoding is the same as returned by [Prefix.String], with one exception: // If p is the zero value, the encoding is the empty string. func (p Prefix) MarshalText() ([]byte, error) { - var max int + buf := []byte{} switch p.ip.z { case z0: case z4: - max = len("255.255.255.255/32") + const maxCap = len("255.255.255.255/32") + buf = make([]byte, 0, maxCap) default: - max = len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0/128") + const maxCap = len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0/128") + buf = make([]byte, 0, maxCap) } - b := make([]byte, 0, max) - b = p.AppendTo(b) - return b, nil + return p.AppendText(buf) } // UnmarshalText implements the encoding.TextUnmarshaler interface. @@ -1517,13 +1554,23 @@ func (p *Prefix) UnmarshalText(text []byte) error { return err } +// AppendBinary implements the [encoding.AppendMarshaler] interface. +// It returns [Addr.AppendBinary] with an additional byte appended +// containing the prefix bits. +func (p Prefix) AppendBinary(b []byte) ([]byte, error) { + b, err := p.Addr().withoutZone().AppendBinary(b) + if err != nil { + return nil, err + } + return append(b, uint8(p.Bits())), nil +} + // MarshalBinary implements the [encoding.BinaryMarshaler] interface. // It returns [Addr.MarshalBinary] with an additional byte appended // containing the prefix bits. func (p Prefix) MarshalBinary() ([]byte, error) { - b := p.Addr().withoutZone().marshalBinaryWithTrailingBytes(1) - b[len(b)-1] = uint8(p.Bits()) - return b, nil + // without the zone the max length is 16, plus an additional byte is 17 + return p.AppendBinary(make([]byte, 0, p.Addr().withoutZone().marshalBinarySize()+1)) } // UnmarshalBinary implements the [encoding.BinaryUnmarshaler] interface. diff --git a/src/net/netip/netip_test.go b/src/net/netip/netip_test.go index e1a0a83f64511d..71e39021ca8969 100644 --- a/src/net/netip/netip_test.go +++ b/src/net/netip/netip_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "flag" "fmt" + "internal/asan" "internal/testenv" "net" . "net/netip" @@ -351,6 +352,32 @@ func TestIPv4Constructors(t *testing.T) { } } +func TestAddrAppendText(t *testing.T) { + tests := []struct { + ip Addr + want string + }{ + {Addr{}, ""}, // zero IP + {mustIP("1.2.3.4"), "1.2.3.4"}, + {mustIP("fd7a:115c:a1e0:ab12:4843:cd96:626b:430b"), "fd7a:115c:a1e0:ab12:4843:cd96:626b:430b"}, + {mustIP("::ffff:192.168.140.255"), "::ffff:192.168.140.255"}, + {mustIP("::ffff:192.168.140.255%en0"), "::ffff:192.168.140.255%en0"}, + } + for i, tc := range tests { + ip := tc.ip + + mtAppend := make([]byte, 4, 32) + mtAppend, err := ip.AppendText(mtAppend) + mtAppend = mtAppend[4:] + if err != nil { + t.Fatal(err) + } + if string(mtAppend) != tc.want { + t.Errorf("%d. for (%v) AppendText = %q; want %q", i, ip, mtAppend, tc.want) + } + } +} + func TestAddrMarshalUnmarshalBinary(t *testing.T) { tests := []struct { ip string @@ -381,6 +408,23 @@ func TestAddrMarshalUnmarshalBinary(t *testing.T) { if ip != ip2 { t.Fatalf("got %v; want %v", ip2, ip) } + + bAppend := make([]byte, 4, 32) + bAppend, err = ip.AppendBinary(bAppend) + bAppend = bAppend[4:] + if err != nil { + t.Fatal(err) + } + if len(bAppend) != tc.wantSize { + t.Fatalf("%q encoded to size %d; want %d", tc.ip, len(bAppend), tc.wantSize) + } + var ip3 Addr + if err := ip3.UnmarshalBinary(bAppend); err != nil { + t.Fatal(err) + } + if ip != ip3 { + t.Fatalf("got %v; want %v", ip3, ip) + } } // Cannot unmarshal from unexpected IP length. @@ -416,6 +460,17 @@ func TestAddrPortMarshalTextString(t *testing.T) { if string(mt) != tt.want { t.Errorf("%d. for (%v, %v) MarshalText = %q; want %q", i, tt.in.Addr(), tt.in.Port(), mt, tt.want) } + + mtAppend := make([]byte, 4, 32) + mtAppend, err = tt.in.AppendText(mtAppend) + mtAppend = mtAppend[4:] + if err != nil { + t.Errorf("%d. for (%v, %v) AppendText error: %v", i, tt.in.Addr(), tt.in.Port(), err) + continue + } + if string(mtAppend) != tt.want { + t.Errorf("%d. for (%v, %v) AppendText = %q; want %q", i, tt.in.Addr(), tt.in.Port(), mtAppend, tt.want) + } } } @@ -448,6 +503,23 @@ func TestAddrPortMarshalUnmarshalBinary(t *testing.T) { if ipport != ipport2 { t.Fatalf("got %v; want %v", ipport2, ipport) } + + bAppend := make([]byte, 4, 32) + bAppend, err = ipport.AppendBinary(bAppend) + bAppend = bAppend[4:] + if err != nil { + t.Fatal(err) + } + if len(bAppend) != tc.wantSize { + t.Fatalf("%q encoded to size %d; want %d", tc.ipport, len(bAppend), tc.wantSize) + } + var ipport3 AddrPort + if err := ipport3.UnmarshalBinary(bAppend); err != nil { + t.Fatal(err) + } + if ipport != ipport3 { + t.Fatalf("got %v; want %v", ipport3, ipport) + } } // Cannot unmarshal from unexpected lengths. @@ -482,6 +554,17 @@ func TestPrefixMarshalTextString(t *testing.T) { if string(mt) != tt.want { t.Errorf("%d. for %v MarshalText = %q; want %q", i, tt.in, mt, tt.want) } + + mtAppend := make([]byte, 4, 64) + mtAppend, err = tt.in.AppendText(mtAppend) + mtAppend = mtAppend[4:] + if err != nil { + t.Errorf("%d. for %v AppendText error: %v", i, tt.in, err) + continue + } + if string(mtAppend) != tt.want { + t.Errorf("%d. for %v AppendText = %q; want %q", i, tt.in, mtAppend, tt.want) + } } } @@ -515,6 +598,23 @@ func TestPrefixMarshalUnmarshalBinary(t *testing.T) { if prefix != prefix2 { t.Fatalf("got %v; want %v", prefix2, prefix) } + + bAppend := make([]byte, 4, 32) + bAppend, err = prefix.AppendBinary(bAppend) + bAppend = bAppend[4:] + if err != nil { + t.Fatal(err) + } + if len(bAppend) != tc.wantSize { + t.Fatalf("%q encoded to size %d; want %d", tc.prefix, len(bAppend), tc.wantSize) + } + var prefix3 Prefix + if err := prefix3.UnmarshalBinary(bAppend); err != nil { + t.Fatal(err) + } + if prefix != prefix3 { + t.Fatalf("got %v; want %v", prefix3, prefix) + } } // Cannot unmarshal from unexpected lengths. @@ -1023,6 +1123,9 @@ func TestPrefixCompare(t *testing.T) { {mustPrefix("fe80::/48"), mustPrefix("fe80::/64"), -1}, {mustPrefix("1.2.3.0/24"), mustPrefix("fe80::/8"), -1}, + + {mustPrefix("1.2.3.0/24"), mustPrefix("1.2.3.4/24"), -1}, + {mustPrefix("1.2.3.0/24"), mustPrefix("1.2.3.0/28"), -1}, } for _, tt := range tests { got := tt.a.Compare(tt.b) @@ -1048,10 +1151,70 @@ func TestPrefixCompare(t *testing.T) { Prefix{}, mustPrefix("fe80::/48"), mustPrefix("1.2.0.0/24"), + mustPrefix("1.2.3.4/24"), + mustPrefix("1.2.3.0/28"), } slices.SortFunc(values, Prefix.Compare) got := fmt.Sprintf("%s", values) - want := `[invalid Prefix 1.2.0.0/16 1.2.0.0/24 1.2.3.0/24 fe80::/48 fe80::/64 fe90::/64]` + want := `[invalid Prefix 1.2.0.0/16 1.2.0.0/24 1.2.3.0/24 1.2.3.4/24 1.2.3.0/28 fe80::/48 fe80::/64 fe90::/64]` + if got != want { + t.Errorf("unexpected sort\n got: %s\nwant: %s\n", got, want) + } + + // Lists from + // https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml and + // https://www.iana.org/assignments/ipv6-address-space/ipv6-address-space.xhtml, + // to verify that the sort order matches IANA's conventional + // ordering. + values = []Prefix{ + mustPrefix("0.0.0.0/8"), + mustPrefix("127.0.0.0/8"), + mustPrefix("10.0.0.0/8"), + mustPrefix("203.0.113.0/24"), + mustPrefix("169.254.0.0/16"), + mustPrefix("192.0.0.0/24"), + mustPrefix("240.0.0.0/4"), + mustPrefix("192.0.2.0/24"), + mustPrefix("192.0.0.170/32"), + mustPrefix("198.18.0.0/15"), + mustPrefix("192.0.0.8/32"), + mustPrefix("0.0.0.0/32"), + mustPrefix("192.0.0.9/32"), + mustPrefix("198.51.100.0/24"), + mustPrefix("192.168.0.0/16"), + mustPrefix("192.0.0.10/32"), + mustPrefix("192.175.48.0/24"), + mustPrefix("192.52.193.0/24"), + mustPrefix("100.64.0.0/10"), + mustPrefix("255.255.255.255/32"), + mustPrefix("192.31.196.0/24"), + mustPrefix("172.16.0.0/12"), + mustPrefix("192.0.0.0/29"), + mustPrefix("192.88.99.0/24"), + mustPrefix("fec0::/10"), + mustPrefix("6000::/3"), + mustPrefix("fe00::/9"), + mustPrefix("8000::/3"), + mustPrefix("0000::/8"), + mustPrefix("0400::/6"), + mustPrefix("f800::/6"), + mustPrefix("e000::/4"), + mustPrefix("ff00::/8"), + mustPrefix("a000::/3"), + mustPrefix("fc00::/7"), + mustPrefix("1000::/4"), + mustPrefix("0800::/5"), + mustPrefix("4000::/3"), + mustPrefix("0100::/8"), + mustPrefix("c000::/3"), + mustPrefix("fe80::/10"), + mustPrefix("0200::/7"), + mustPrefix("f000::/5"), + mustPrefix("2000::/3"), + } + slices.SortFunc(values, func(a, b Prefix) int { return a.Compare(b) }) + got = fmt.Sprintf("%s", values) + want = `[0.0.0.0/8 0.0.0.0/32 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12 192.0.0.0/24 192.0.0.0/29 192.0.0.8/32 192.0.0.9/32 192.0.0.10/32 192.0.0.170/32 192.0.2.0/24 192.31.196.0/24 192.52.193.0/24 192.88.99.0/24 192.168.0.0/16 192.175.48.0/24 198.18.0.0/15 198.51.100.0/24 203.0.113.0/24 240.0.0.0/4 255.255.255.255/32 ::/8 100::/8 200::/7 400::/6 800::/5 1000::/4 2000::/3 4000::/3 6000::/3 8000::/3 a000::/3 c000::/3 e000::/4 f000::/5 f800::/6 fc00::/7 fe00::/9 fe80::/10 fec0::/10 ff00::/8]` if got != want { t.Errorf("unexpected sort\n got: %s\nwant: %s\n", got, want) } @@ -2033,6 +2196,10 @@ var ( ) func TestNoAllocs(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } + // Wrappers that panic on error, to prove that our alloc-free // methods are returning successfully. panicIP := func(ip Addr, err error) Addr { @@ -2179,11 +2346,14 @@ func TestPrefixString(t *testing.T) { } } -func TestInvalidAddrPortString(t *testing.T) { +func TestAddrPortString(t *testing.T) { tests := []struct { ipp AddrPort want string }{ + {MustParseAddrPort("127.0.0.1:80"), "127.0.0.1:80"}, + {MustParseAddrPort("[0000::0]:8080"), "[::]:8080"}, + {MustParseAddrPort("[FFFF::1]:8080"), "[ffff::1]:8080"}, {AddrPort{}, "invalid AddrPort"}, {AddrPortFrom(Addr{}, 80), "invalid AddrPort"}, } diff --git a/src/net/platform_plan9_test.go b/src/net/platform_plan9_test.go new file mode 100644 index 00000000000000..e1ca0aadae1238 --- /dev/null +++ b/src/net/platform_plan9_test.go @@ -0,0 +1,9 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +func supportsUnixSocket() bool { + return false +} diff --git a/src/net/platform_test.go b/src/net/platform_test.go index 709d4a3eb7b7a7..f8317104d03ed1 100644 --- a/src/net/platform_test.go +++ b/src/net/platform_test.go @@ -7,30 +7,11 @@ package net import ( "internal/testenv" "os" - "os/exec" "runtime" - "strconv" "strings" "testing" ) -var unixEnabledOnAIX bool - -func init() { - if runtime.GOOS == "aix" { - // Unix network isn't properly working on AIX 7.2 with - // Technical Level < 2. - // The information is retrieved only once in this init() - // instead of everytime testableNetwork is called. - out, _ := exec.Command("oslevel", "-s").Output() - if len(out) >= len("7200-XX-ZZ-YYMM") { // AIX 7.2, Tech Level XX, Service Pack ZZ, date YYMM - aixVer := string(out[:4]) - tl, _ := strconv.Atoi(string(out[5:7])) - unixEnabledOnAIX = aixVer > "7200" || (aixVer == "7200" && tl >= 2) - } - } -} - // testableNetwork reports whether network is testable on the current // platform configuration. func testableNetwork(network string) bool { @@ -46,13 +27,15 @@ func testableNetwork(network string) bool { return false } } - case "unix", "unixgram": + case "unixgram": switch runtime.GOOS { - case "android", "ios", "plan9", "windows": + case "windows": return false - case "aix": - return unixEnabledOnAIX + default: + return supportsUnixSocket() } + case "unix": + return supportsUnixSocket() case "unixpacket": switch runtime.GOOS { case "aix", "android", "darwin", "ios", "plan9", "windows": diff --git a/src/net/platform_unix_test.go b/src/net/platform_unix_test.go new file mode 100644 index 00000000000000..b6b3a5549de865 --- /dev/null +++ b/src/net/platform_unix_test.go @@ -0,0 +1,40 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix || js || wasip1 + +package net + +import ( + "os/exec" + "runtime" + "strconv" +) + +var unixEnabledOnAIX bool + +func init() { + if runtime.GOOS == "aix" { + // Unix network isn't properly working on AIX 7.2 with + // Technical Level < 2. + // The information is retrieved only once in this init() + // instead of everytime testableNetwork is called. + out, _ := exec.Command("oslevel", "-s").Output() + if len(out) >= len("7200-XX-ZZ-YYMM") { // AIX 7.2, Tech Level XX, Service Pack ZZ, date YYMM + aixVer := string(out[:4]) + tl, _ := strconv.Atoi(string(out[5:7])) + unixEnabledOnAIX = aixVer > "7200" || (aixVer == "7200" && tl >= 2) + } + } +} + +func supportsUnixSocket() bool { + switch runtime.GOOS { + case "android", "ios": + return false + case "aix": + return unixEnabledOnAIX + } + return true +} diff --git a/src/net/platform_windows_test.go b/src/net/platform_windows_test.go new file mode 100644 index 00000000000000..ec2c861e80142e --- /dev/null +++ b/src/net/platform_windows_test.go @@ -0,0 +1,11 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import "internal/syscall/windows" + +func supportsUnixSocket() bool { + return windows.SupportUnixSocket() +} diff --git a/src/net/rawconn_windows_test.go b/src/net/rawconn_windows_test.go index 5febf08f774b18..ebfec3499a2fdd 100644 --- a/src/net/rawconn_windows_test.go +++ b/src/net/rawconn_windows_test.go @@ -95,6 +95,11 @@ func controlOnConnSetup(network string, address string, c syscall.RawConn) error switch network { case "tcp", "udp", "ip": return errors.New("ambiguous network: " + network) + case "unix", "unixpacket", "unixgram": + fn = func(s uintptr) { + const SO_ERROR = 0x1007 + _, operr = syscall.GetsockoptInt(syscall.Handle(s), syscall.SOL_SOCKET, SO_ERROR) + } default: switch network[len(network)-1] { case '4': diff --git a/src/net/rpc/jsonrpc/client.go b/src/net/rpc/jsonrpc/client.go index 1beba0f364a571..c0f383445dfb88 100644 --- a/src/net/rpc/jsonrpc/client.go +++ b/src/net/rpc/jsonrpc/client.go @@ -120,5 +120,5 @@ func Dial(network, address string) (*rpc.Client, error) { if err != nil { return nil, err } - return NewClient(conn), err + return NewClient(conn), nil } diff --git a/src/net/rpc/server.go b/src/net/rpc/server.go index 1771726a932eda..4233a426fefb1f 100644 --- a/src/net/rpc/server.go +++ b/src/net/rpc/server.go @@ -4,7 +4,11 @@ /* Package rpc provides access to the exported methods of an object across a -network or other I/O connection. A server registers an object, making it visible +network or other I/O connection. + +The net/rpc package is frozen and is not accepting new features. + +A server registers an object, making it visible as a service with the name of the type of the object. After registration, exported methods of the object will be accessible remotely. A server may register multiple objects (services) of different types but it is an error to register multiple @@ -121,8 +125,6 @@ or A server implementation will often provide a simple, type-safe wrapper for the client. - -The net/rpc package is frozen and is not accepting new features. */ package rpc diff --git a/src/net/sendfile.go b/src/net/sendfile.go new file mode 100644 index 00000000000000..0e0fcc40fff4a3 --- /dev/null +++ b/src/net/sendfile.go @@ -0,0 +1,60 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux || (darwin && !ios) || dragonfly || freebsd || solaris || windows + +package net + +import ( + "internal/poll" + "io" + "syscall" +) + +// sendFile copies the contents of r to c using the sendfile +// system call to minimize copies. +// +// if handled == true, sendFile returns the number (potentially zero) of bytes +// copied and any non-EOF error. +// +// if handled == false, sendFile performed no work. +func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { + if !supportsSendfile() { + return 0, nil, false + } + var remain int64 = 0 // 0 writes the entire file + lr, ok := r.(*io.LimitedReader) + if ok { + remain, r = lr.N, lr.R + if remain <= 0 { + return 0, nil, true + } + } + // r might be an *os.File or an os.fileWithoutWriteTo. + // Type assert to an interface rather than *os.File directly to handle the latter case. + f, ok := r.(syscall.Conn) + if !ok { + return 0, nil, false + } + + sc, err := f.SyscallConn() + if err != nil { + return 0, nil, false + } + + var werr error + err = sc.Read(func(fd uintptr) bool { + written, werr, handled = poll.SendFile(&c.pfd, fd, remain) + return true + }) + if err == nil { + err = werr + } + + if lr != nil { + lr.N = remain - written + } + + return written, wrapSyscallError("sendfile", err), handled +} diff --git a/src/net/sendfile_linux.go b/src/net/sendfile_linux.go deleted file mode 100644 index f8a7bec8d38169..00000000000000 --- a/src/net/sendfile_linux.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -import ( - "internal/poll" - "io" - "os" -) - -const supportsSendfile = true - -// sendFile copies the contents of r to c using the sendfile -// system call to minimize copies. -// -// if handled == true, sendFile returns the number (potentially zero) of bytes -// copied and any non-EOF error. -// -// if handled == false, sendFile performed no work. -func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { - var remain int64 = 1<<63 - 1 // by default, copy until EOF - - lr, ok := r.(*io.LimitedReader) - if ok { - remain, r = lr.N, lr.R - if remain <= 0 { - return 0, nil, true - } - } - f, ok := r.(*os.File) - if !ok { - return 0, nil, false - } - - sc, err := f.SyscallConn() - if err != nil { - return 0, nil, false - } - - var werr error - err = sc.Read(func(fd uintptr) bool { - written, werr, handled = poll.SendFile(&c.pfd, int(fd), remain) - return true - }) - if err == nil { - err = werr - } - - if lr != nil { - lr.N = remain - written - } - return written, wrapSyscallError("sendfile", err), handled -} diff --git a/src/net/sendfile_nonwindows.go b/src/net/sendfile_nonwindows.go new file mode 100644 index 00000000000000..2106d378951e80 --- /dev/null +++ b/src/net/sendfile_nonwindows.go @@ -0,0 +1,12 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux || (darwin && !ios) || dragonfly || freebsd || solaris + +package net + +// Always true except for workstation and client versions of Windows +func supportsSendfile() bool { + return true +} diff --git a/src/net/sendfile_stub.go b/src/net/sendfile_stub.go index 7f31cc63e1ed6a..17d8d5448f77d9 100644 --- a/src/net/sendfile_stub.go +++ b/src/net/sendfile_stub.go @@ -8,7 +8,9 @@ package net import "io" -const supportsSendfile = false +func supportsSendfile() bool { + return false +} func sendFile(c *netFD, r io.Reader) (n int64, err error, handled bool) { return 0, nil, false diff --git a/src/net/sendfile_test.go b/src/net/sendfile_test.go index 4f3411565bb6f3..437d181508045e 100644 --- a/src/net/sendfile_test.go +++ b/src/net/sendfile_test.go @@ -13,6 +13,7 @@ import ( "fmt" "internal/poll" "io" + "math/rand" "os" "runtime" "strconv" @@ -30,11 +31,11 @@ const ( // expectSendfile runs f, and verifies that internal/poll.SendFile successfully handles // a write to wantConn during f's execution. // -// On platforms where supportsSendfile is false, expectSendfile runs f but does not +// On platforms where supportsSendfile() is false, expectSendfile runs f but does not // expect a call to SendFile. func expectSendfile(t *testing.T, wantConn Conn, f func()) { t.Helper() - if !supportsSendfile { + if !supportsSendfile() { f() return } @@ -46,14 +47,16 @@ func expectSendfile(t *testing.T, wantConn Conn, f func()) { called bool gotHandled bool gotFD *poll.FD + gotErr error ) - poll.TestHookDidSendFile = func(dstFD *poll.FD, src int, written int64, err error, handled bool) { + poll.TestHookDidSendFile = func(dstFD *poll.FD, src uintptr, written int64, err error, handled bool) { if called { t.Error("internal/poll.SendFile called multiple times, want one call") } called = true gotHandled = handled gotFD = dstFD + gotErr = err } f() if !called { @@ -61,7 +64,7 @@ func expectSendfile(t *testing.T, wantConn Conn, f func()) { return } if !gotHandled { - t.Error("internal/poll.SendFile did not handle the write, want it to") + t.Error("internal/poll.SendFile did not handle the write, want it to, error:", gotErr) return } if &wantConn.(*TCPConn).fd.pfd != gotFD { @@ -69,7 +72,33 @@ func expectSendfile(t *testing.T, wantConn Conn, f func()) { } } -func TestSendfile(t *testing.T) { +func TestSendfile(t *testing.T) { testSendfile(t, newton, newtonSHA256, newtonLen, 0) } +func TestSendfileWithExactLimit(t *testing.T) { + testSendfile(t, newton, newtonSHA256, newtonLen, newtonLen) +} +func TestSendfileWithLimitLargerThanFile(t *testing.T) { + testSendfile(t, newton, newtonSHA256, newtonLen, newtonLen*2) +} +func TestSendfileWithLargeFile(t *testing.T) { + // Some platforms are not capable of handling large files with sendfile + // due to limited system resource, so we only run this test on amd64 and + // arm64 for the moment. + if runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" { + t.Skip("skipping on non-amd64 and non-arm64 platforms") + } + // Also skip it during short testing. + if testing.Short() { + t.Skip("Skip it during short testing") + } + + // We're using 1<<31 - 1 as the chunk size for sendfile currently, + // make an edge case file that is 1 byte bigger than that. + f := createTempFile(t, 1<<31) + // For big file like this, only verify the transmission of the file, + // skip the content check. + testSendfile(t, f.Name(), "", 1<<31, 0) +} +func testSendfile(t *testing.T, filePath, fileHash string, size, limit int64) { ln := newLocalListener(t, "tcp") defer ln.Close() @@ -87,7 +116,7 @@ func TestSendfile(t *testing.T) { defer close(errc) defer conn.Close() - f, err := os.Open(newton) + f, err := os.Open(filePath) if err != nil { errc <- err return @@ -97,23 +126,23 @@ func TestSendfile(t *testing.T) { // Return file data using io.Copy, which should use // sendFile if available. var sbytes int64 - switch runtime.GOOS { - case "windows": - // Windows is not using sendfile for some reason: - // https://go.dev/issue/67042 - sbytes, err = io.Copy(conn, f) - default: - expectSendfile(t, conn, func() { + expectSendfile(t, conn, func() { + if limit > 0 { + sbytes, err = io.CopyN(conn, f, limit) + if err == io.EOF && limit > size { + err = nil + } + } else { sbytes, err = io.Copy(conn, f) - }) - } + } + }) if err != nil { errc <- err return } - if sbytes != newtonLen { - errc <- fmt.Errorf("sent %d bytes; expected %d", sbytes, newtonLen) + if sbytes != size { + errc <- fmt.Errorf("sent %d bytes; expected %d", sbytes, size) return } }() @@ -133,11 +162,11 @@ func TestSendfile(t *testing.T) { t.Error(err) } - if rbytes != newtonLen { - t.Errorf("received %d bytes; expected %d", rbytes, newtonLen) + if rbytes != size { + t.Errorf("received %d bytes; expected %d", rbytes, size) } - if res := hex.EncodeToString(h.Sum(nil)); res != newtonSHA256 { + if len(fileHash) > 0 && hex.EncodeToString(h.Sum(nil)) != newtonSHA256 { t.Error("retrieved data hash did not match") } @@ -542,7 +571,7 @@ type sendFileBench struct { func (bench sendFileBench) benchSendFile(b *testing.B) { fileSize := b.N * bench.chunkSize - f := createTempFile(b, fileSize) + f := createTempFile(b, int64(fileSize)) client, server := spawnTestSocketPair(b, bench.proto) defer server.Close() @@ -568,25 +597,27 @@ func (bench sendFileBench) benchSendFile(b *testing.B) { } } -func createTempFile(b *testing.B, size int) *os.File { - f, err := os.CreateTemp(b.TempDir(), "sendfile-bench") +func createTempFile(tb testing.TB, size int64) *os.File { + f, err := os.CreateTemp(tb.TempDir(), "sendfile-bench") if err != nil { - b.Fatalf("failed to create temporary file: %v", err) + tb.Fatalf("failed to create temporary file: %v", err) } - b.Cleanup(func() { + tb.Cleanup(func() { f.Close() }) - data := make([]byte, size) - if _, err := f.Write(data); err != nil { - b.Fatalf("failed to create and feed the file: %v", err) - } - if err := f.Sync(); err != nil { - b.Fatalf("failed to save the file: %v", err) + if _, err := io.CopyN(f, newRandReader(tb), size); err != nil { + tb.Fatalf("failed to fill the file with random data: %v", err) } if _, err := f.Seek(0, io.SeekStart); err != nil { - b.Fatalf("failed to rewind the file: %v", err) + tb.Fatalf("failed to rewind the file: %v", err) } return f } + +func newRandReader(tb testing.TB) io.Reader { + seed := time.Now().UnixNano() + tb.Logf("Deterministic RNG seed based on timestamp: 0x%x", seed) + return rand.New(rand.NewSource(seed)) +} diff --git a/src/net/sendfile_unix_alt.go b/src/net/sendfile_unix_alt.go deleted file mode 100644 index 9e46c4e607d4d8..00000000000000 --- a/src/net/sendfile_unix_alt.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build (darwin && !ios) || dragonfly || freebsd || solaris - -package net - -import ( - "internal/poll" - "io" - "io/fs" - "syscall" -) - -const supportsSendfile = true - -// sendFile copies the contents of r to c using the sendfile -// system call to minimize copies. -// -// if handled == true, sendFile returns the number (potentially zero) of bytes -// copied and any non-EOF error. -// -// if handled == false, sendFile performed no work. -func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { - // Darwin, FreeBSD, DragonFly and Solaris use 0 as the "until EOF" value. - // If you pass in more bytes than the file contains, it will - // loop back to the beginning ad nauseam until it's sent - // exactly the number of bytes told to. As such, we need to - // know exactly how many bytes to send. - var remain int64 = 0 - - lr, ok := r.(*io.LimitedReader) - if ok { - remain, r = lr.N, lr.R - if remain <= 0 { - return 0, nil, true - } - } - // r might be an *os.File or an os.fileWithoutWriteTo. - // Type assert to an interface rather than *os.File directly to handle the latter case. - f, ok := r.(interface { - fs.File - io.Seeker - syscall.Conn - }) - if !ok { - return 0, nil, false - } - - if remain == 0 { - fi, err := f.Stat() - if err != nil { - return 0, err, false - } - - remain = fi.Size() - } - - // The other quirk with Darwin/FreeBSD/DragonFly/Solaris's sendfile - // implementation is that it doesn't use the current position - // of the file -- if you pass it offset 0, it starts from - // offset 0. There's no way to tell it "start from current - // position", so we have to manage that explicitly. - pos, err := f.Seek(0, io.SeekCurrent) - if err != nil { - return 0, err, false - } - - sc, err := f.SyscallConn() - if err != nil { - return 0, nil, false - } - - var werr error - err = sc.Read(func(fd uintptr) bool { - written, werr, handled = poll.SendFile(&c.pfd, int(fd), pos, remain) - return true - }) - if err == nil { - err = werr - } - - if lr != nil { - lr.N = remain - written - } - - _, err1 := f.Seek(written, io.SeekCurrent) - if err1 != nil && err == nil { - return written, err1, handled - } - - return written, wrapSyscallError("sendfile", err), handled -} diff --git a/src/net/sendfile_unix_test.go b/src/net/sendfile_unix_test.go new file mode 100644 index 00000000000000..79fb23b31010d5 --- /dev/null +++ b/src/net/sendfile_unix_test.go @@ -0,0 +1,86 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix + +package net + +import ( + "internal/testpty" + "io" + "os" + "sync" + "syscall" + "testing" +) + +// Issue 70763: test that we don't fail on sendfile from a tty. +func TestCopyFromTTY(t *testing.T) { + pty, ttyName, err := testpty.Open() + if err != nil { + t.Skipf("skipping test because pty open failed: %v", err) + } + defer pty.Close() + + // Use syscall.Open so that the tty is blocking. + ttyFD, err := syscall.Open(ttyName, syscall.O_RDWR, 0) + if err != nil { + t.Skipf("skipping test because tty open failed: %v", err) + } + defer syscall.Close(ttyFD) + + tty := os.NewFile(uintptr(ttyFD), "tty") + defer tty.Close() + + ln := newLocalListener(t, "tcp") + defer ln.Close() + + ch := make(chan bool) + + const data = "data\n" + + var wg sync.WaitGroup + defer wg.Wait() + + wg.Add(1) + go func() { + defer wg.Done() + conn, err := ln.Accept() + if err != nil { + t.Error(err) + return + } + defer conn.Close() + + buf := make([]byte, len(data)) + if _, err := io.ReadFull(conn, buf); err != nil { + t.Error(err) + } + + ch <- true + }() + + conn, err := Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + wg.Add(1) + go func() { + defer wg.Done() + if _, err := pty.Write([]byte(data)); err != nil { + t.Error(err) + } + <-ch + if err := pty.Close(); err != nil { + t.Error(err) + } + }() + + lr := io.LimitReader(tty, int64(len(data))) + if _, err := io.Copy(conn, lr); err != nil { + t.Error(err) + } +} diff --git a/src/net/sendfile_windows.go b/src/net/sendfile_windows.go index 0377a485daa025..44ddb421a1b1ba 100644 --- a/src/net/sendfile_windows.go +++ b/src/net/sendfile_windows.go @@ -1,49 +1,16 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2025 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net -import ( - "internal/poll" - "io" - "os" - "syscall" -) +import "internal/syscall/windows" -const supportsSendfile = true - -// sendFile copies the contents of r to c using the TransmitFile -// system call to minimize copies. -// -// if handled == true, sendFile returns the number of bytes copied and any -// non-EOF error. -// -// if handled == false, sendFile performed no work. -func sendFile(fd *netFD, r io.Reader) (written int64, err error, handled bool) { - var n int64 = 0 // by default, copy until EOF. - - lr, ok := r.(*io.LimitedReader) - if ok { - n, r = lr.N, lr.R - if n <= 0 { - return 0, nil, true - } - } - - f, ok := r.(*os.File) - if !ok { - return 0, nil, false - } - - written, err = poll.SendFile(&fd.pfd, syscall.Handle(f.Fd()), n) - if err != nil { - err = wrapSyscallError("transmitfile", err) - } - - // If any byte was copied, regardless of any error - // encountered mid-way, handled must be set to true. - handled = written > 0 - - return +// Workstation and client versions of Windows limit the number +// of concurrent TransmitFile operations allowed on the system +// to a maximum of two. Please see: +// https://learn.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-transmitfile +// https://golang.org/issue/73746 +func supportsSendfile() bool { + return windows.SupportUnlimitedTransmitFile() } diff --git a/src/net/server_test.go b/src/net/server_test.go index eb6b111f1f5ffb..cc9e9857099af2 100644 --- a/src/net/server_test.go +++ b/src/net/server_test.go @@ -250,7 +250,6 @@ var udpServerTests = []struct { func TestUDPServer(t *testing.T) { for i, tt := range udpServerTests { - i, tt := i, tt t.Run(fmt.Sprint(i), func(t *testing.T) { if !testableListenArgs(tt.snet, tt.saddr, tt.taddr) { t.Skipf("skipping %s %s<-%s test", tt.snet, tt.saddr, tt.taddr) @@ -340,7 +339,6 @@ func TestUnixgramServer(t *testing.T) { } for i, tt := range unixgramServerTests { - i, tt := i, tt t.Run(fmt.Sprint(i), func(t *testing.T) { if !testableListenArgs("unixgram", tt.saddr, "") { t.Skipf("skipping unixgram %s<-%s test", tt.saddr, tt.caddr) diff --git a/src/net/smtp/smtp.go b/src/net/smtp/smtp.go index d750a2854cbf90..522d80e4eb57cf 100644 --- a/src/net/smtp/smtp.go +++ b/src/net/smtp/smtp.go @@ -413,9 +413,7 @@ func (c *Client) Noop() error { // Quit sends the QUIT command and closes the connection to the server. func (c *Client) Quit() error { - if err := c.hello(); err != nil { - return err - } + c.hello() // ignore error; we're quitting anyhow _, _, err := c.cmd(221, "QUIT") if err != nil { return err diff --git a/src/net/smtp/smtp_test.go b/src/net/smtp/smtp_test.go index c91c99b1f53111..427ed0f7d9153a 100644 --- a/src/net/smtp/smtp_test.go +++ b/src/net/smtp/smtp_test.go @@ -288,6 +288,37 @@ Goodbye. QUIT ` +func TestHELOFailed(t *testing.T) { + serverLines := `502 EH? +502 EH? +221 OK +` + clientLines := `EHLO localhost +HELO localhost +QUIT +` + + server := strings.Join(strings.Split(serverLines, "\n"), "\r\n") + client := strings.Join(strings.Split(clientLines, "\n"), "\r\n") + var cmdbuf strings.Builder + bcmdbuf := bufio.NewWriter(&cmdbuf) + var fake faker + fake.ReadWriter = bufio.NewReadWriter(bufio.NewReader(strings.NewReader(server)), bcmdbuf) + c := &Client{Text: textproto.NewConn(fake), localName: "localhost"} + + if err := c.Hello("localhost"); err == nil { + t.Fatal("expected EHLO to fail") + } + if err := c.Quit(); err != nil { + t.Errorf("QUIT failed: %s", err) + } + bcmdbuf.Flush() + actual := cmdbuf.String() + if client != actual { + t.Errorf("Got:\n%s\nWant:\n%s", actual, client) + } +} + func TestExtensions(t *testing.T) { fake := func(server string) (c *Client, bcmdbuf *bufio.Writer, cmdbuf *strings.Builder) { server = strings.Join(strings.Split(server, "\n"), "\r\n") @@ -1105,40 +1136,58 @@ func sendMail(hostPort string) error { // localhostCert is a PEM-encoded TLS cert generated from src/crypto/tls: // -// go run generate_cert.go --rsa-bits 1024 --host 127.0.0.1,::1,example.com \ +// go run generate_cert.go --rsa-bits 2048 --host 127.0.0.1,::1,example.com \ // --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h var localhostCert = []byte(` -----BEGIN CERTIFICATE----- -MIICFDCCAX2gAwIBAgIRAK0xjnaPuNDSreeXb+z+0u4wDQYJKoZIhvcNAQELBQAw -EjEQMA4GA1UEChMHQWNtZSBDbzAgFw03MDAxMDEwMDAwMDBaGA8yMDg0MDEyOTE2 -MDAwMFowEjEQMA4GA1UEChMHQWNtZSBDbzCBnzANBgkqhkiG9w0BAQEFAAOBjQAw -gYkCgYEA0nFbQQuOWsjbGtejcpWz153OlziZM4bVjJ9jYruNw5n2Ry6uYQAffhqa -JOInCmmcVe2siJglsyH9aRh6vKiobBbIUXXUU1ABd56ebAzlt0LobLlx7pZEMy30 -LqIi9E6zmL3YvdGzpYlkFRnRrqwEtWYbGBf3znO250S56CCWH2UCAwEAAaNoMGYw -DgYDVR0PAQH/BAQDAgKkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQF -MAMBAf8wLgYDVR0RBCcwJYILZXhhbXBsZS5jb22HBH8AAAGHEAAAAAAAAAAAAAAA -AAAAAAEwDQYJKoZIhvcNAQELBQADgYEAbZtDS2dVuBYvb+MnolWnCNqvw1w5Gtgi -NmvQQPOMgM3m+oQSCPRTNGSg25e1Qbo7bgQDv8ZTnq8FgOJ/rbkyERw2JckkHpD4 -n4qcK27WkEDBtQFlPihIM8hLIuzWoi/9wygiElTy/tVL3y7fGCvY2/k1KBthtZGF -tN8URjVmyEo= +MIIDFDCCAfygAwIBAgIRAPV4ktbcY/mn0oRRjnGAGJgwDQYJKoZIhvcNAQELBQAw +EjEQMA4GA1UEChMHQWNtZSBDbzAeFw0yNTAzMTgxOTI3NTRaFw0yNjAzMTgxOTI3 +NTRaMBIxEDAOBgNVBAoTB0FjbWUgQ28wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQDbsEfk1bK7ozwZlcQM8rBUikC4gwnnw0J1PUlGDGu1Y84dKtulbdWj +yrh88D4fSdtmxFbXE7fhYUJTBmEHSUk9OLHh/Tr+nSC3SfH0I/9y6l9j9vVVYhYJ +C07Z1mZZKVb+gmbbB7LEavGMNaFHjvRJAwBX2TMDbXJceZ9jU/iihILkZbrbG40r +n1mctYVmcR3YqOzI/ynLje97FEvxtsg99OUjzzXyFMqfAl0J3Gc6tzvAER3N+ovK +nudsnMB5Y+InQHHmPeizG4mFyeBYesXNwX6cmI30c8KFiAlKHcsxjJsuoBZ3bSwv +vFdK2hnuCO05HEgCzAQKUlY6Q2F0xJblAgMBAAGjZTBjMA4GA1UdDwEB/wQEAwIF +oDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMC4GA1UdEQQnMCWC +C2V4YW1wbGUuY29thwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMA0GCSqGSIb3DQEB +CwUAA4IBAQBnfO4lYRXR9AdMidpgdITqMEKJik8MvCkpQ+EKQLq3CIGXPt5lkHLs +ysbF9f3VxioKNYzkakJGVGyu51hqyhGqGQ4M7IpOBQkmY24IExWPVEk2wkIV+HTU ++oQVZOIrHF+s9IIFOIh3SIPIsXNvx7rUc5sgF4P+eAnAcv3o1zL7YjGJZ8e27Ai2 +uF8iG/po/0Vd93OSB8Tj/Nvg99SSucy7nBYTreSdhUjZWRI0W1oYJX49/fhWljR9 +8+f2GqUfLc7iCjcV3wxlfBqEKCdpjXsiqtsb1KrAx7AEOj7XfDjJjyCL4bshLp9x +PbV+kBFCN151iWYtfzhKEplrZFYNXlX2 -----END CERTIFICATE-----`) // localhostKey is the private key for localhostCert. var localhostKey = []byte(testingKey(` -----BEGIN RSA TESTING KEY----- -MIICXgIBAAKBgQDScVtBC45ayNsa16NylbPXnc6XOJkzhtWMn2Niu43DmfZHLq5h -AB9+Gpok4icKaZxV7ayImCWzIf1pGHq8qKhsFshRddRTUAF3np5sDOW3QuhsuXHu -lkQzLfQuoiL0TrOYvdi90bOliWQVGdGurAS1ZhsYF/fOc7bnRLnoIJYfZQIDAQAB -AoGBAMst7OgpKyFV6c3JwyI/jWqxDySL3caU+RuTTBaodKAUx2ZEmNJIlx9eudLA -kucHvoxsM/eRxlxkhdFxdBcwU6J+zqooTnhu/FE3jhrT1lPrbhfGhyKnUrB0KKMM -VY3IQZyiehpxaeXAwoAou6TbWoTpl9t8ImAqAMY8hlULCUqlAkEA+9+Ry5FSYK/m -542LujIcCaIGoG1/Te6Sxr3hsPagKC2rH20rDLqXwEedSFOpSS0vpzlPAzy/6Rbb -PHTJUhNdwwJBANXkA+TkMdbJI5do9/mn//U0LfrCR9NkcoYohxfKz8JuhgRQxzF2 -6jpo3q7CdTuuRixLWVfeJzcrAyNrVcBq87cCQFkTCtOMNC7fZnCTPUv+9q1tcJyB -vNjJu3yvoEZeIeuzouX9TJE21/33FaeDdsXbRhQEj23cqR38qFHsF1qAYNMCQQDP -QXLEiJoClkR2orAmqjPLVhR3t2oB3INcnEjLNSq8LHyQEfXyaFfu4U9l5+fRPL2i -jiC0k/9L5dHUsF0XZothAkEA23ddgRs+Id/HxtojqqUT27B8MT/IGNrYsp4DvS/c -qgkeluku4GjxRlDMBuXk94xOBEinUs+p/hwP1Alll80Tpg== +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDbsEfk1bK7ozwZ +lcQM8rBUikC4gwnnw0J1PUlGDGu1Y84dKtulbdWjyrh88D4fSdtmxFbXE7fhYUJT +BmEHSUk9OLHh/Tr+nSC3SfH0I/9y6l9j9vVVYhYJC07Z1mZZKVb+gmbbB7LEavGM +NaFHjvRJAwBX2TMDbXJceZ9jU/iihILkZbrbG40rn1mctYVmcR3YqOzI/ynLje97 +FEvxtsg99OUjzzXyFMqfAl0J3Gc6tzvAER3N+ovKnudsnMB5Y+InQHHmPeizG4mF +yeBYesXNwX6cmI30c8KFiAlKHcsxjJsuoBZ3bSwvvFdK2hnuCO05HEgCzAQKUlY6 +Q2F0xJblAgMBAAECggEACzZIOQraBB8M3G5rEtEZBDuJGZGgggpSXDrsQC22mouV +M6JiEuOT5Xfdagz10rF5h9lp6DCqsA8/bA7ViWJpYT1BQNwkdGWvC4Oz3EaxDRue +kjLCqyCmKMCBvfbmAtNsC/G6T5/pNQKTQNlk2YrXd1l2nUUpyBlAHq2bX52jwSGD +bFy5hyzSrzjeLpLUNZ56W/uXCvP0l6PAEvXRn/KG89XLZCtMBvVDMCfjIe77Q1U9 +/XzIrnb67RzQwiDelvX+biMkBrjeYw/Gvdo9hNCOfbOZ+SpnfDOLEfAha/XPmr3+ +5EeF4emeEhCODvfe7wy/4h1gHEG2N435S61DcV3gQQKBgQD92EJidwriPGDTUSM8 +nJrPQ5xwPMKz5hWpfI0zxIYZyqA37eRC5Q9WD3rDbrEZiLCInFh+Ci899iLzEpFZ +dFQAUiRam+zFpDCQcGHr/uytRoTH/nxh2MrYPq8cA5ZGU6oMH+Yl4TynqJm2KN7e +0ocE07QjyK/9nIvEtdibEiFEwQKBgQDdjcgoqHaM49YJ4yxGpjuRdc5a3iuKzZU6 +BON4GKqYQ9u/o8/NPaOSQ3vKhwzTjiEoOZImn+eX1cRP0ZskmQ+LyzsdVAHMDydz +9I23dbIywtCXGhKOJRwt9O++8ataWIxi1frjj6BcI+TzGl8LM2lYIfUHzVzfswwE +1EK8ikxnJQKBgBqPKvr0a54aJSNXBPHNjOEMuOyBXvnFpBSUpI17DXDbY4IWkOBy +6PTfL8AM79i1FYtlmFivphu8ihGWqsCKTFOwRH96ev5+3FnweD5h8M98Zl4qgUcX +kLmpbVboBSwcitkz6TejZl5AZLzLb+4uZtQZdmqcD9XgMDuHrz8iWXrBAoGBAMJO +Z34pCRfVddFkGF+5yMJw5FLTSLLKTJb+1JRuZad21BIF0+i3p25OmxHrUXd0zmWd +4CzZzt5eD3bFaOA3EOhUi/rTw2O44qwSjfuZUHiuXQw4RI+/wjAYAe+fud1ZjX3d +FtVfEI/etxvyQ+rp4vj1hxWZqVtThzXxBrqePBW1AoGBAOTC19rFQXtVf9A+8c/w +2ryAY2W9qNKe0xMivTAqau0Kdy2/2toJekR/5qOy3tOF7JasOgG+y3m3gLF47EFF +v75eW4FkiCFvsyl/qv4CO1eKnHlvkRoDsnMb+dA5czst58rO6BK40QvPqwXaSxj1 +ee8ReNCDhC0Zidczajm63O1G -----END RSA TESTING KEY-----`)) func testingKey(s string) string { return strings.ReplaceAll(s, "TESTING KEY", "PRIVATE KEY") } diff --git a/src/net/sock_linux.go b/src/net/sock_linux.go index cffe9a236fc7f1..380a8880d1a779 100644 --- a/src/net/sock_linux.go +++ b/src/net/sock_linux.go @@ -18,9 +18,8 @@ import ( // // See issue 5030 and 41470. func maxAckBacklog(n int) int { - major, minor := unix.KernelVersion() size := 16 - if major > 4 || (major == 4 && minor >= 1) { + if unix.KernelVersionGE(4, 1) { size = 32 } diff --git a/src/net/sock_linux_test.go b/src/net/sock_linux_test.go index 11303cfff1e646..9696cddce9ff53 100644 --- a/src/net/sock_linux_test.go +++ b/src/net/sock_linux_test.go @@ -11,13 +11,12 @@ import ( func TestMaxAckBacklog(t *testing.T) { n := 196602 - major, minor := unix.KernelVersion() backlog := maxAckBacklog(n) expected := 1<<16 - 1 - if major > 4 || (major == 4 && minor >= 1) { + if unix.KernelVersionGE(4, 1) { expected = n } if backlog != expected { - t.Fatalf(`Kernel version: "%d.%d", sk_max_ack_backlog mismatch, got %d, want %d`, major, minor, backlog, expected) + t.Fatalf(`sk_max_ack_backlog mismatch, got %d, want %d`, backlog, expected) } } diff --git a/src/net/sockopt_posix.go b/src/net/sockopt_posix.go index a380c7719b484c..2452f06b0a6e32 100644 --- a/src/net/sockopt_posix.go +++ b/src/net/sockopt_posix.go @@ -7,7 +7,6 @@ package net import ( - "internal/bytealg" "runtime" "syscall" ) @@ -43,35 +42,6 @@ func interfaceToIPv4Addr(ifi *Interface) (IP, error) { return nil, errNoSuchInterface } -func setIPv4MreqToInterface(mreq *syscall.IPMreq, ifi *Interface) error { - if ifi == nil { - return nil - } - ifat, err := ifi.Addrs() - if err != nil { - return err - } - for _, ifa := range ifat { - switch v := ifa.(type) { - case *IPAddr: - if a := v.IP.To4(); a != nil { - copy(mreq.Interface[:], a) - goto done - } - case *IPNet: - if a := v.IP.To4(); a != nil { - copy(mreq.Interface[:], a) - goto done - } - } - } -done: - if bytealg.Equal(mreq.Multiaddr[:], IPv4zero.To4()) { - return errNoSuchMulticastInterface - } - return nil -} - func setReadBuffer(fd *netFD, bytes int) error { err := fd.pfd.SetsockoptInt(syscall.SOL_SOCKET, syscall.SO_RCVBUF, bytes) runtime.KeepAlive(fd) diff --git a/src/net/sockoptip_bsdvar.go b/src/net/sockoptip4_bsdvar.go similarity index 100% rename from src/net/sockoptip_bsdvar.go rename to src/net/sockoptip4_bsdvar.go diff --git a/src/net/sockoptip4_linux.go b/src/net/sockoptip4_linux.go new file mode 100644 index 00000000000000..8b953ebdc6b438 --- /dev/null +++ b/src/net/sockoptip4_linux.go @@ -0,0 +1,37 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "runtime" + "syscall" +) + +func joinIPv4Group(fd *netFD, ifi *Interface, ip IP) error { + mreq := &syscall.IPMreqn{Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]}} + if ifi != nil { + mreq.Ifindex = int32(ifi.Index) + } + err := fd.pfd.SetsockoptIPMreqn(syscall.IPPROTO_IP, syscall.IP_ADD_MEMBERSHIP, mreq) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) +} + +func setIPv4MulticastInterface(fd *netFD, ifi *Interface) error { + var v int32 + if ifi != nil { + v = int32(ifi.Index) + } + mreq := &syscall.IPMreqn{Ifindex: v} + err := fd.pfd.SetsockoptIPMreqn(syscall.IPPROTO_IP, syscall.IP_MULTICAST_IF, mreq) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) +} + +func setIPv4MulticastLoopback(fd *netFD, v bool) error { + err := fd.pfd.SetsockoptInt(syscall.IPPROTO_IP, syscall.IP_MULTICAST_LOOP, boolint(v)) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) +} diff --git a/src/net/sockoptip4_posix_nonlinux.go b/src/net/sockoptip4_posix_nonlinux.go new file mode 100644 index 00000000000000..85e8c6dcfe4116 --- /dev/null +++ b/src/net/sockoptip4_posix_nonlinux.go @@ -0,0 +1,52 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (unix && !linux) || windows + +package net + +import ( + "internal/bytealg" + "runtime" + "syscall" +) + +func joinIPv4Group(fd *netFD, ifi *Interface, ip IP) error { + mreq := &syscall.IPMreq{Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]}} + if err := setIPv4MreqToInterface(mreq, ifi); err != nil { + return err + } + err := fd.pfd.SetsockoptIPMreq(syscall.IPPROTO_IP, syscall.IP_ADD_MEMBERSHIP, mreq) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) +} + +func setIPv4MreqToInterface(mreq *syscall.IPMreq, ifi *Interface) error { + if ifi == nil { + return nil + } + ifat, err := ifi.Addrs() + if err != nil { + return err + } + for _, ifa := range ifat { + switch v := ifa.(type) { + case *IPAddr: + if a := v.IP.To4(); a != nil { + copy(mreq.Interface[:], a) + goto done + } + case *IPNet: + if a := v.IP.To4(); a != nil { + copy(mreq.Interface[:], a) + goto done + } + } + } +done: + if bytealg.Equal(mreq.Interface[:], IPv4zero.To4()) { + return errNoSuchMulticastInterface + } + return nil +} diff --git a/src/net/sockoptip_windows.go b/src/net/sockoptip4_windows.go similarity index 100% rename from src/net/sockoptip_windows.go rename to src/net/sockoptip4_windows.go diff --git a/src/net/sockoptip6_posix.go b/src/net/sockoptip6_posix.go new file mode 100644 index 00000000000000..5bbc609f7b1a4b --- /dev/null +++ b/src/net/sockoptip6_posix.go @@ -0,0 +1,39 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix || windows + +package net + +import ( + "runtime" + "syscall" +) + +func setIPv6MulticastInterface(fd *netFD, ifi *Interface) error { + var v int + if ifi != nil { + v = ifi.Index + } + err := fd.pfd.SetsockoptInt(syscall.IPPROTO_IPV6, syscall.IPV6_MULTICAST_IF, v) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) +} + +func setIPv6MulticastLoopback(fd *netFD, v bool) error { + err := fd.pfd.SetsockoptInt(syscall.IPPROTO_IPV6, syscall.IPV6_MULTICAST_LOOP, boolint(v)) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) +} + +func joinIPv6Group(fd *netFD, ifi *Interface, ip IP) error { + mreq := &syscall.IPv6Mreq{} + copy(mreq.Multiaddr[:], ip) + if ifi != nil { + mreq.Interface = uint32(ifi.Index) + } + err := fd.pfd.SetsockoptIPv6Mreq(syscall.IPPROTO_IPV6, syscall.IPV6_JOIN_GROUP, mreq) + runtime.KeepAlive(fd) + return wrapSyscallError("setsockopt", err) +} diff --git a/src/net/sockoptip_linux.go b/src/net/sockoptip_linux.go deleted file mode 100644 index bd7d8344258e24..00000000000000 --- a/src/net/sockoptip_linux.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -import ( - "runtime" - "syscall" -) - -func setIPv4MulticastInterface(fd *netFD, ifi *Interface) error { - var v int32 - if ifi != nil { - v = int32(ifi.Index) - } - mreq := &syscall.IPMreqn{Ifindex: v} - err := fd.pfd.SetsockoptIPMreqn(syscall.IPPROTO_IP, syscall.IP_MULTICAST_IF, mreq) - runtime.KeepAlive(fd) - return wrapSyscallError("setsockopt", err) -} - -func setIPv4MulticastLoopback(fd *netFD, v bool) error { - err := fd.pfd.SetsockoptInt(syscall.IPPROTO_IP, syscall.IP_MULTICAST_LOOP, boolint(v)) - runtime.KeepAlive(fd) - return wrapSyscallError("setsockopt", err) -} diff --git a/src/net/sockoptip_posix.go b/src/net/sockoptip_posix.go deleted file mode 100644 index 572ea455c09586..00000000000000 --- a/src/net/sockoptip_posix.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build unix || windows - -package net - -import ( - "runtime" - "syscall" -) - -func joinIPv4Group(fd *netFD, ifi *Interface, ip IP) error { - mreq := &syscall.IPMreq{Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]}} - if err := setIPv4MreqToInterface(mreq, ifi); err != nil { - return err - } - err := fd.pfd.SetsockoptIPMreq(syscall.IPPROTO_IP, syscall.IP_ADD_MEMBERSHIP, mreq) - runtime.KeepAlive(fd) - return wrapSyscallError("setsockopt", err) -} - -func setIPv6MulticastInterface(fd *netFD, ifi *Interface) error { - var v int - if ifi != nil { - v = ifi.Index - } - err := fd.pfd.SetsockoptInt(syscall.IPPROTO_IPV6, syscall.IPV6_MULTICAST_IF, v) - runtime.KeepAlive(fd) - return wrapSyscallError("setsockopt", err) -} - -func setIPv6MulticastLoopback(fd *netFD, v bool) error { - err := fd.pfd.SetsockoptInt(syscall.IPPROTO_IPV6, syscall.IPV6_MULTICAST_LOOP, boolint(v)) - runtime.KeepAlive(fd) - return wrapSyscallError("setsockopt", err) -} - -func joinIPv6Group(fd *netFD, ifi *Interface, ip IP) error { - mreq := &syscall.IPv6Mreq{} - copy(mreq.Multiaddr[:], ip) - if ifi != nil { - mreq.Interface = uint32(ifi.Index) - } - err := fd.pfd.SetsockoptIPv6Mreq(syscall.IPPROTO_IPV6, syscall.IPV6_JOIN_GROUP, mreq) - runtime.KeepAlive(fd) - return wrapSyscallError("setsockopt", err) -} diff --git a/src/net/tcpconn_keepalive_test.go b/src/net/tcpconn_keepalive_test.go index 53d0be034fb554..4bf2f9ef206420 100644 --- a/src/net/tcpconn_keepalive_test.go +++ b/src/net/tcpconn_keepalive_test.go @@ -22,7 +22,7 @@ func TestTCPConnKeepAliveConfigDialer(t *testing.T) { oldCfg KeepAliveConfig ) testPreHookSetKeepAlive = func(nfd *netFD) { - oldCfg, errHook = getCurrentKeepAliveSettings(fdType(nfd.pfd.Sysfd)) + oldCfg, errHook = getCurrentKeepAliveSettings(nfd.pfd.Sysfd) } handler := func(ls *localServer, ln Listener) { @@ -80,7 +80,7 @@ func TestTCPConnKeepAliveConfigListener(t *testing.T) { oldCfg KeepAliveConfig ) testPreHookSetKeepAlive = func(nfd *netFD) { - oldCfg, errHook = getCurrentKeepAliveSettings(fdType(nfd.pfd.Sysfd)) + oldCfg, errHook = getCurrentKeepAliveSettings(nfd.pfd.Sysfd) } ch := make(chan Conn, 1) diff --git a/src/net/tcpsock.go b/src/net/tcpsock.go index f5df502f0fd000..376bf238c70d07 100644 --- a/src/net/tcpsock.go +++ b/src/net/tcpsock.go @@ -14,7 +14,7 @@ import ( "time" ) -// BUG(mikio): On JS and Windows, the File method of TCPConn and +// BUG(mikio): On JS, the File method of TCPConn and // TCPListener is not implemented. // TCPAddr represents the address of a TCP end point. @@ -315,6 +315,10 @@ func newTCPConn(fd *netFD, keepAliveIdle time.Duration, keepAliveCfg KeepAliveCo // If the IP field of raddr is nil or an unspecified IP address, the // local system is assumed. func DialTCP(network string, laddr, raddr *TCPAddr) (*TCPConn, error) { + return dialTCP(context.Background(), nil, network, laddr, raddr) +} + +func dialTCP(ctx context.Context, dialer *Dialer, network string, laddr, raddr *TCPAddr) (*TCPConn, error) { switch network { case "tcp", "tcp4", "tcp6": default: @@ -324,7 +328,18 @@ func DialTCP(network string, laddr, raddr *TCPAddr) (*TCPConn, error) { return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} } sd := &sysDialer{network: network, address: raddr.String()} - c, err := sd.dialTCP(context.Background(), laddr, raddr) + var ( + c *TCPConn + err error + ) + if dialer != nil { + sd.Dialer = *dialer + } + if sd.MultipathTCP() { + c, err = sd.dialMPTCP(ctx, laddr, raddr) + } else { + c, err = sd.dialTCP(ctx, laddr, raddr) + } if err != nil { return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} } @@ -409,6 +424,9 @@ func (l *TCPListener) SetDeadline(t time.Time) error { // The returned os.File's file descriptor is different from the // connection's. Attempting to change properties of the original // using this duplicate may or may not have the desired effect. +// +// On Windows, the returned os.File's file descriptor is not +// usable on other processes. func (l *TCPListener) File() (f *os.File, err error) { if !l.ok() { return nil, syscall.EINVAL @@ -439,7 +457,15 @@ func ListenTCP(network string, laddr *TCPAddr) (*TCPListener, error) { laddr = &TCPAddr{} } sl := &sysListener{network: network, address: laddr.String()} - ln, err := sl.listenTCP(context.Background(), laddr) + var ( + ln *TCPListener + err error + ) + if sl.MultipathTCP() { + ln, err = sl.listenMPTCP(context.Background(), laddr) + } else { + ln, err = sl.listenTCP(context.Background(), laddr) + } if err != nil { return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err} } diff --git a/src/net/tcpsock_test.go b/src/net/tcpsock_test.go index 9ed49a925b4b39..085989c7499b60 100644 --- a/src/net/tcpsock_test.go +++ b/src/net/tcpsock_test.go @@ -475,6 +475,9 @@ func TestTCPReadWriteAllocs(t *testing.T) { t.Skipf("not supported on %s", runtime.GOOS) } + // Optimizations are required to remove the allocs. + testenv.SkipIfOptimizationOff(t) + ln, err := Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) @@ -509,7 +512,7 @@ func TestTCPReadWriteAllocs(t *testing.T) { } }) if allocs > 0 { - t.Fatalf("got %v; want 0", allocs) + t.Errorf("got %v; want 0", allocs) } var bufwrt [128]byte @@ -531,7 +534,7 @@ func TestTCPReadWriteAllocs(t *testing.T) { } }) if allocs > 0 { - t.Fatalf("got %v; want 0", allocs) + t.Errorf("got %v; want 0", allocs) } } diff --git a/src/net/textproto/reader.go b/src/net/textproto/reader.go index f98e05bd1d86af..6df3a630917d78 100644 --- a/src/net/textproto/reader.go +++ b/src/net/textproto/reader.go @@ -110,11 +110,12 @@ func trim(s []byte) []byte { for i < len(s) && (s[i] == ' ' || s[i] == '\t') { i++ } - n := len(s) - for n > i && (s[n-1] == ' ' || s[n-1] == '\t') { + s = s[i:] + n := len(s) - 1 + for n >= 0 && (s[n] == ' ' || s[n] == '\t') { n-- } - return s[i:n] + return s[:n+1] } // ReadContinuedLineBytes is like [Reader.ReadContinuedLine] but @@ -284,8 +285,10 @@ func (r *Reader) ReadCodeLine(expectCode int) (code int, message string, err err // // An expectCode <= 0 disables the check of the status code. func (r *Reader) ReadResponse(expectCode int) (code int, message string, err error) { - code, continued, message, err := r.readCodeLine(expectCode) + code, continued, first, err := r.readCodeLine(expectCode) multi := continued + var messageBuilder strings.Builder + messageBuilder.WriteString(first) for continued { line, err := r.ReadLine() if err != nil { @@ -296,12 +299,15 @@ func (r *Reader) ReadResponse(expectCode int) (code int, message string, err err var moreMessage string code2, continued, moreMessage, err = parseCodeLine(line, 0) if err != nil || code2 != code { - message += "\n" + strings.TrimRight(line, "\r\n") + messageBuilder.WriteByte('\n') + messageBuilder.WriteString(strings.TrimRight(line, "\r\n")) continued = true continue } - message += "\n" + moreMessage + messageBuilder.WriteByte('\n') + messageBuilder.WriteString(moreMessage) } + message = messageBuilder.String() if err != nil && multi && message != "" { // replace one line error message with all lines (full message) err = &Error{code, message} @@ -642,8 +648,8 @@ func (r *Reader) upcomingHeaderKeys() (n int) { // the rest are converted to lowercase. For example, the // canonical key for "accept-encoding" is "Accept-Encoding". // MIME header keys are assumed to be ASCII only. -// If s contains a space or invalid header field bytes, it is -// returned without modifications. +// If s contains a space or invalid header field bytes as +// defined by RFC 9112, it is returned without modifications. func CanonicalMIMEHeaderKey(s string) string { // Quick check for canonical encoding. upper := true diff --git a/src/net/textproto/textproto.go b/src/net/textproto/textproto.go index 4ae3ecff747093..00dc8cbee52b48 100644 --- a/src/net/textproto/textproto.go +++ b/src/net/textproto/textproto.go @@ -5,6 +5,9 @@ // Package textproto implements generic support for text-based request/response // protocols in the style of HTTP, NNTP, and SMTP. // +// This package enforces the HTTP/1.1 character set defined by +// RFC 9112 for header keys and values. +// // The package provides: // // [Error], which represents a numeric error response from diff --git a/src/net/timeout_test.go b/src/net/timeout_test.go index 09adb9bdca4cd7..b7f8c613b4b1a2 100644 --- a/src/net/timeout_test.go +++ b/src/net/timeout_test.go @@ -180,7 +180,6 @@ func TestAcceptTimeout(t *testing.T) { } for _, timeout := range timeouts { - timeout := timeout t.Run(fmt.Sprintf("%v", timeout), func(t *testing.T) { t.Parallel() @@ -730,10 +729,7 @@ func nextTimeout(actual time.Duration) (next time.Duration, ok bool) { // duration by any significant margin. Try the next attempt with an arbitrary // factor above that, so that our growth curve is at least exponential. next = actual * 5 / 4 - if next > maxDynamicTimeout { - return maxDynamicTimeout, true - } - return next, true + return min(next, maxDynamicTimeout), true } // There is a very similar copy of this in os/timeout_test.go. diff --git a/src/net/udpsock.go b/src/net/udpsock.go index 56aabffa3180e4..f9a3bee867d340 100644 --- a/src/net/udpsock.go +++ b/src/net/udpsock.go @@ -14,9 +14,6 @@ import ( // BUG(mikio): On Plan 9, the ReadMsgUDP and // WriteMsgUDP methods of UDPConn are not implemented. -// BUG(mikio): On Windows, the File method of UDPConn is not -// implemented. - // BUG(mikio): On JS, methods and functions related to UDPConn are not // implemented. @@ -288,6 +285,10 @@ func newUDPConn(fd *netFD) *UDPConn { return &UDPConn{conn{fd}} } // If the IP field of raddr is nil or an unspecified IP address, the // local system is assumed. func DialUDP(network string, laddr, raddr *UDPAddr) (*UDPConn, error) { + return dialUDP(context.Background(), nil, network, laddr, raddr) +} + +func dialUDP(ctx context.Context, dialer *Dialer, network string, laddr, raddr *UDPAddr) (*UDPConn, error) { switch network { case "udp", "udp4", "udp6": default: @@ -297,7 +298,10 @@ func DialUDP(network string, laddr, raddr *UDPAddr) (*UDPConn, error) { return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} } sd := &sysDialer{network: network, address: raddr.String()} - c, err := sd.dialUDP(context.Background(), laddr, raddr) + if dialer != nil { + sd.Dialer = *dialer + } + c, err := sd.dialUDP(ctx, laddr, raddr) if err != nil { return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} } diff --git a/src/net/udpsock_posix.go b/src/net/udpsock_posix.go index 3cd1d0a7624165..99e4359e719d34 100644 --- a/src/net/udpsock_posix.go +++ b/src/net/udpsock_posix.go @@ -186,17 +186,25 @@ func (c *UDPConn) writeMsgAddrPort(b, oob []byte, addr netip.AddrPort) (n, oobn switch c.fd.family { case syscall.AF_INET: - sa, err := addrPortToSockaddrInet4(addr) - if err != nil { - return 0, 0, err + var sap *syscall.SockaddrInet4 + if addr.IsValid() { + sa, err := addrPortToSockaddrInet4(addr) + if err != nil { + return 0, 0, err + } + sap = &sa } - return c.fd.writeMsgInet4(b, oob, &sa) + return c.fd.writeMsgInet4(b, oob, sap) case syscall.AF_INET6: - sa, err := addrPortToSockaddrInet6(addr) - if err != nil { - return 0, 0, err + var sap *syscall.SockaddrInet6 + if addr.IsValid() { + sa, err := addrPortToSockaddrInet6(addr) + if err != nil { + return 0, 0, err + } + sap = &sa } - return c.fd.writeMsgInet6(b, oob, &sa) + return c.fd.writeMsgInet6(b, oob, sap) default: return 0, 0, &AddrError{Err: "invalid address family", Addr: addr.Addr().String()} } diff --git a/src/net/udpsock_test.go b/src/net/udpsock_test.go index 43065d06dab771..a79e9f83c11098 100644 --- a/src/net/udpsock_test.go +++ b/src/net/udpsock_test.go @@ -7,6 +7,8 @@ package net import ( "errors" "fmt" + "internal/asan" + "internal/race" "internal/testenv" "net/netip" "os" @@ -140,36 +142,37 @@ func testWriteToConn(t *testing.T, raddr string) { if err != nil { t.Fatal(err) } + rap := ra.AddrPort() + + assertErrWriteToConnected := func(t *testing.T, err error) { + t.Helper() + if e, ok := err.(*OpError); !ok || e.Err != ErrWriteToConnected { + t.Errorf("got %v; want ErrWriteToConnected", err) + } + } b := []byte("CONNECTED-MODE SOCKET") + _, err = c.(*UDPConn).WriteToUDPAddrPort(b, rap) + assertErrWriteToConnected(t, err) _, err = c.(*UDPConn).WriteToUDP(b, ra) - if err == nil { - t.Fatal("should fail") - } - if err != nil && err.(*OpError).Err != ErrWriteToConnected { - t.Fatalf("should fail as ErrWriteToConnected: %v", err) - } + assertErrWriteToConnected(t, err) _, err = c.(*UDPConn).WriteTo(b, ra) - if err == nil { - t.Fatal("should fail") - } - if err != nil && err.(*OpError).Err != ErrWriteToConnected { - t.Fatalf("should fail as ErrWriteToConnected: %v", err) - } + assertErrWriteToConnected(t, err) _, err = c.Write(b) if err != nil { - t.Fatal(err) + t.Errorf("c.Write(b) = %v; want nil", err) } _, _, err = c.(*UDPConn).WriteMsgUDP(b, nil, ra) - if err == nil { - t.Fatal("should fail") - } - if err != nil && err.(*OpError).Err != ErrWriteToConnected { - t.Fatalf("should fail as ErrWriteToConnected: %v", err) - } + assertErrWriteToConnected(t, err) _, _, err = c.(*UDPConn).WriteMsgUDP(b, nil, nil) if err != nil { - t.Fatal(err) + t.Errorf("c.WriteMsgUDP(b, nil, nil) = %v; want nil", err) + } + _, _, err = c.(*UDPConn).WriteMsgUDPAddrPort(b, nil, rap) + assertErrWriteToConnected(t, err) + _, _, err = c.(*UDPConn).WriteMsgUDPAddrPort(b, nil, netip.AddrPort{}) + if err != nil { + t.Errorf("c.WriteMsgUDPAddrPort(b, nil, netip.AddrPort{}) = %v; want nil", err) } } @@ -489,10 +492,19 @@ func TestAllocs(t *testing.T) { case "plan9", "js", "wasip1": // These implementations have not been optimized. t.Skipf("skipping on %v", runtime.GOOS) + case "windows": + if race.Enabled { + // The Windows implementation make use of sync.Pool, + // which randomly drops cached items when race is enabled. + t.Skip("skipping test in race") + } } if !testableNetwork("udp4") { t.Skipf("skipping: udp4 not available") } + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } // Optimizations are required to remove the allocs. testenv.SkipIfOptimizationOff(t) @@ -701,3 +713,40 @@ func TestIPv6WriteMsgUDPAddrPortTargetAddrIPVersion(t *testing.T) { t.Fatal(err) } } + +// TestIPv4WriteMsgUDPAddrPortTargetAddrIPVersion verifies that +// WriteMsgUDPAddrPort accepts IPv4 and IPv4-mapped IPv6 destination addresses, +// and rejects IPv6 destination addresses on a "udp4" connection. +func TestIPv4WriteMsgUDPAddrPortTargetAddrIPVersion(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("not supported on %s", runtime.GOOS) + } + + if !testableNetwork("udp4") { + t.Skipf("skipping: udp4 not available") + } + + conn, err := ListenUDP("udp4", &UDPAddr{IP: IPv4(127, 0, 0, 1)}) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + daddr4 := netip.AddrPortFrom(netip.MustParseAddr("127.0.0.1"), 12345) + daddr4in6 := netip.AddrPortFrom(netip.MustParseAddr("::ffff:127.0.0.1"), 12345) + daddr6 := netip.AddrPortFrom(netip.MustParseAddr("::1"), 12345) + buf := make([]byte, 8) + + if _, _, err = conn.WriteMsgUDPAddrPort(buf, nil, daddr4); err != nil { + t.Errorf("conn.WriteMsgUDPAddrPort(buf, nil, daddr4) failed: %v", err) + } + + if _, _, err = conn.WriteMsgUDPAddrPort(buf, nil, daddr4in6); err != nil { + t.Errorf("conn.WriteMsgUDPAddrPort(buf, nil, daddr4in6) failed: %v", err) + } + + if _, _, err = conn.WriteMsgUDPAddrPort(buf, nil, daddr6); err == nil { + t.Errorf("conn.WriteMsgUDPAddrPort(buf, nil, daddr6) should have failed, but got no error") + } +} diff --git a/src/net/unixsock.go b/src/net/unixsock.go index 821be7bf741972..0ee79f35dec8a4 100644 --- a/src/net/unixsock.go +++ b/src/net/unixsock.go @@ -79,7 +79,7 @@ func (c *UnixConn) SyscallConn() (syscall.RawConn, error) { } // CloseRead shuts down the reading side of the Unix domain connection. -// Most callers should just use Close. +// Most callers should just use [UnixConn.Close]. func (c *UnixConn) CloseRead() error { if !c.ok() { return syscall.EINVAL @@ -91,7 +91,7 @@ func (c *UnixConn) CloseRead() error { } // CloseWrite shuts down the writing side of the Unix domain connection. -// Most callers should just use Close. +// Most callers should just use [UnixConn.Close]. func (c *UnixConn) CloseWrite() error { if !c.ok() { return syscall.EINVAL @@ -114,7 +114,7 @@ func (c *UnixConn) ReadFromUnix(b []byte) (int, *UnixAddr, error) { return n, addr, err } -// ReadFrom implements the [PacketConn] ReadFrom method. +// ReadFrom implements the [PacketConn].ReadFrom method. func (c *UnixConn) ReadFrom(b []byte) (int, Addr, error) { if !c.ok() { return 0, nil, syscall.EINVAL @@ -159,7 +159,7 @@ func (c *UnixConn) WriteToUnix(b []byte, addr *UnixAddr) (int, error) { return n, err } -// WriteTo implements the [PacketConn] WriteTo method. +// WriteTo implements the [PacketConn].WriteTo method. func (c *UnixConn) WriteTo(b []byte, addr Addr) (int, error) { if !c.ok() { return 0, syscall.EINVAL @@ -196,18 +196,25 @@ func newUnixConn(fd *netFD) *UnixConn { return &UnixConn{conn{fd}} } // DialUnix acts like [Dial] for Unix networks. // -// The network must be a Unix network name; see func Dial for details. +// The network must be a Unix network name; see func [Dial] for details. // // If laddr is non-nil, it is used as the local address for the // connection. func DialUnix(network string, laddr, raddr *UnixAddr) (*UnixConn, error) { + return dialUnix(context.Background(), nil, network, laddr, raddr) +} + +func dialUnix(ctx context.Context, dialer *Dialer, network string, laddr, raddr *UnixAddr) (*UnixConn, error) { switch network { case "unix", "unixgram", "unixpacket": default: return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(network)} } sd := &sysDialer{network: network, address: raddr.String()} - c, err := sd.dialUnix(context.Background(), laddr, raddr) + if dialer != nil { + sd.Dialer = *dialer + } + c, err := sd.dialUnix(ctx, laddr, raddr) if err != nil { return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} } @@ -229,7 +236,7 @@ func (ln *UnixListener) ok() bool { return ln != nil && ln.fd != nil } // SyscallConn returns a raw network connection. // This implements the [syscall.Conn] interface. // -// The returned RawConn only supports calling Control. Read and +// The returned [syscall.RawConn] only supports calling Control. Read and // Write return an error. func (l *UnixListener) SyscallConn() (syscall.RawConn, error) { if !l.ok() { @@ -277,7 +284,7 @@ func (l *UnixListener) Close() error { } // Addr returns the listener's network address. -// The Addr returned is shared by all invocations of Addr, so +// The [Addr] returned is shared by all invocations of Addr, so // do not modify it. func (l *UnixListener) Addr() Addr { return l.fd.laddr } @@ -294,9 +301,12 @@ func (l *UnixListener) SetDeadline(t time.Time) error { // It is the caller's responsibility to close f when finished. // Closing l does not affect f, and closing f does not affect l. // -// The returned os.File's file descriptor is different from the +// The returned [os.File]'s file descriptor is different from the // connection's. Attempting to change properties of the original // using this duplicate may or may not have the desired effect. +// +// On Windows, the returned os.File's file descriptor is not +// usable on other processes. func (l *UnixListener) File() (f *os.File, err error) { if !l.ok() { return nil, syscall.EINVAL diff --git a/src/net/unixsock_linux_test.go b/src/net/unixsock_linux_test.go index d04007cef38b2c..fa50ba7bd51abd 100644 --- a/src/net/unixsock_linux_test.go +++ b/src/net/unixsock_linux_test.go @@ -49,6 +49,23 @@ func TestUnixAutobindClose(t *testing.T) { ln.Close() } +func TestUnixAbstractLongNameNulStart(t *testing.T) { + // Create an abstract socket name that starts with a null byte ("\x00") + // whose length is the maximum of RawSockaddrUnix Path len + paddedAddr := make([]byte, len(syscall.RawSockaddrUnix{}.Path)) + copy(paddedAddr, "\x00abstract_test") + + la, err := ResolveUnixAddr("unix", string(paddedAddr)) + if err != nil { + t.Fatal(err) + } + c, err := ListenUnix("unix", la) + if err != nil { + t.Fatal(err) + } + c.Close() +} + func TestUnixgramLinuxAbstractLongName(t *testing.T) { if !testableNetwork("unixgram") { t.Skip("abstract unix socket long name test") diff --git a/src/net/unixsock_test.go b/src/net/unixsock_test.go index 6906ecc0466a9d..f6c5679f4291b3 100644 --- a/src/net/unixsock_test.go +++ b/src/net/unixsock_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !plan9 && !windows +//go:build !plan9 package net @@ -247,7 +247,6 @@ func TestUnixConnLocalAndRemoteNames(t *testing.T) { handler := func(ls *localServer, ln Listener) {} for _, laddr := range []string{"", testUnixAddr(t)} { - laddr := laddr taddr := testUnixAddr(t) ta, err := ResolveUnixAddr("unix", taddr) if err != nil { @@ -282,7 +281,7 @@ func TestUnixConnLocalAndRemoteNames(t *testing.T) { } switch runtime.GOOS { - case "android", "linux": + case "android", "linux", "windows": if laddr == "" { laddr = "@" // autobind feature } @@ -306,7 +305,6 @@ func TestUnixgramConnLocalAndRemoteNames(t *testing.T) { } for _, laddr := range []string{"", testUnixAddr(t)} { - laddr := laddr taddr := testUnixAddr(t) ta, err := ResolveUnixAddr("unixgram", taddr) if err != nil { diff --git a/src/net/unixsock_windows_test.go b/src/net/unixsock_windows_test.go index 511ba6f2df203e..4c1a8d7e7e04c0 100644 --- a/src/net/unixsock_windows_test.go +++ b/src/net/unixsock_windows_test.go @@ -10,6 +10,7 @@ import ( "internal/syscall/windows" "os" "reflect" + "syscall" "testing" ) @@ -69,6 +70,27 @@ func TestUnixConnLocalWindows(t *testing.T) { } } +func TestUnixAbstractLongNameNulStart(t *testing.T) { + if !windows.SupportUnixSocket() { + t.Skip("unix test") + } + + // Create an abstract socket name that starts with a null byte ("\x00") + // whose length is the maximum of RawSockaddrUnix Path len + paddedAddr := make([]byte, len(syscall.RawSockaddrUnix{}.Path)) + copy(paddedAddr, "\x00abstract_test") + + la, err := ResolveUnixAddr("unix", string(paddedAddr)) + if err != nil { + t.Fatal(err) + } + c, err := ListenUnix("unix", la) + if err != nil { + t.Fatal(err) + } + c.Close() +} + func TestModeSocket(t *testing.T) { if !windows.SupportUnixSocket() { t.Skip("unix test") diff --git a/src/net/url/encoding_table.go b/src/net/url/encoding_table.go new file mode 100644 index 00000000000000..60b3564948e3ce --- /dev/null +++ b/src/net/url/encoding_table.go @@ -0,0 +1,114 @@ +// Code generated from gen_encoding_table.go using 'go generate'; DO NOT EDIT. + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package url + +type encoding uint8 + +const ( + encodePath encoding = 1 << iota + encodePathSegment + encodeHost + encodeZone + encodeUserPassword + encodeQueryComponent + encodeFragment + + // hexChar is actually NOT an encoding mode, but there are only seven + // encoding modes. We might as well abuse the otherwise unused most + // significant bit in uint8 to indicate whether a character is + // hexadecimal. + hexChar +) + +var table = [256]encoding{ + '!': encodeFragment | encodeZone | encodeHost, + '"': encodeZone | encodeHost, + '$': encodeFragment | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '&': encodeFragment | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '\'': encodeZone | encodeHost, + '(': encodeFragment | encodeZone | encodeHost, + ')': encodeFragment | encodeZone | encodeHost, + '*': encodeFragment | encodeZone | encodeHost, + '+': encodeFragment | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + ',': encodeFragment | encodeUserPassword | encodeZone | encodeHost | encodePath, + '-': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '.': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '/': encodeFragment | encodePath, + '0': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '1': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '2': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '3': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '4': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '5': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '6': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '7': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '8': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '9': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + ':': encodeFragment | encodeZone | encodeHost | encodePathSegment | encodePath, + ';': encodeFragment | encodeUserPassword | encodeZone | encodeHost | encodePath, + '<': encodeZone | encodeHost, + '=': encodeFragment | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '>': encodeZone | encodeHost, + '?': encodeFragment, + '@': encodeFragment | encodePathSegment | encodePath, + 'A': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'B': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'C': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'D': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'E': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'F': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'G': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'H': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'I': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'J': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'K': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'L': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'M': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'N': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'O': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'P': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'Q': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'R': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'S': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'T': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'U': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'V': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'W': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'X': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'Y': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'Z': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '[': encodeZone | encodeHost, + ']': encodeZone | encodeHost, + '_': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'a': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'b': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'c': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'd': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'e': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'f': hexChar | encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'g': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'h': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'i': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'j': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'k': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'l': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'm': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'n': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'o': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'p': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'q': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'r': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 's': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 't': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'u': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'v': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'w': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'x': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'y': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + 'z': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, + '~': encodeFragment | encodeQueryComponent | encodeUserPassword | encodeZone | encodeHost | encodePathSegment | encodePath, +} diff --git a/src/net/url/example_test.go b/src/net/url/example_test.go index a1913508f72fce..311ba5c329b442 100644 --- a/src/net/url/example_test.go +++ b/src/net/url/example_test.go @@ -253,6 +253,18 @@ func ExampleURL_IsAbs() { // true } +func ExampleURL_JoinPath() { + u, err := url.Parse("https://example.com/foo/bar") + if err != nil { + log.Fatal(err) + } + + fmt.Println(u.JoinPath("baz", "qux")) + + // Output: + // https://example.com/foo/bar/baz/qux +} + func ExampleURL_MarshalBinary() { u, _ := url.Parse("https://example.org/") b, err := u.MarshalBinary() diff --git a/src/net/url/gen_encoding_table.go b/src/net/url/gen_encoding_table.go new file mode 100644 index 00000000000000..5defe5046bb292 --- /dev/null +++ b/src/net/url/gen_encoding_table.go @@ -0,0 +1,234 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ignore + +package main + +import ( + "bytes" + _ "embed" + "fmt" + "go/format" + "io" + "log" + "maps" + "os" + "slices" + "strconv" + "strings" +) + +// We embed this source file in the resulting code-generation program in order +// to extract the definitions of the encoding type and constants from it and +// include them in the generated file. +// +//go:embed gen_encoding_table.go +var genSource string + +const filename = "encoding_table.go" + +func main() { + var out bytes.Buffer + fmt.Fprintln(&out, "// Code generated from gen_encoding_table.go using 'go generate'; DO NOT EDIT.") + fmt.Fprintln(&out) + fmt.Fprintln(&out, "// Copyright 2025 The Go Authors. All rights reserved.") + fmt.Fprintln(&out, "// Use of this source code is governed by a BSD-style") + fmt.Fprintln(&out, "// license that can be found in the LICENSE file.") + fmt.Fprintln(&out) + fmt.Fprintln(&out, "package url") + fmt.Fprintln(&out) + generateEnc(&out, genSource) + generateTable(&out) + + formatted, err := format.Source(out.Bytes()) + if err != nil { + log.Fatal("format:", err) + } + + err = os.WriteFile(filename, formatted, 0644) + if err != nil { + log.Fatal("WriteFile:", err) + } +} + +func generateEnc(w io.Writer, src string) { + var writeLine bool + for line := range strings.Lines(src) { + if strings.HasPrefix(line, "// START encoding") { + writeLine = true + continue + } + if strings.HasPrefix(line, "// END encoding") { + return + } + if writeLine { + fmt.Fprint(w, line) + } + } +} + +func generateTable(w io.Writer) { + fmt.Fprintln(w, "var table = [256]encoding{") + + // Sort the encodings (in decreasing order) to guarantee a stable output. + sortedEncs := slices.Sorted(maps.Keys(encNames)) + slices.Reverse(sortedEncs) + + for i := range 256 { + c := byte(i) + var lineBuf bytes.Buffer + + // Write key to line buffer. + lineBuf.WriteString(strconv.QuoteRune(rune(c))) + + lineBuf.WriteByte(':') + + // Write value to line buffer. + blankVal := true + if ishex(c) { + // Set the hexChar bit if this char is hexadecimal. + lineBuf.WriteString("hexChar") + blankVal = false + } + for _, enc := range sortedEncs { + if !shouldEscape(c, enc) { + if !blankVal { + lineBuf.WriteByte('|') + } + // Set this encoding mode's bit if this char should NOT be + // escaped. + name := encNames[enc] + lineBuf.WriteString(name) + blankVal = false + } + } + + if !blankVal { + lineBuf.WriteString(",\n") + w.Write(lineBuf.Bytes()) + } + } + fmt.Fprintln(w, "}") +} + +// START encoding (keep this marker comment in sync with genEnc) +type encoding uint8 + +const ( + encodePath encoding = 1 << iota + encodePathSegment + encodeHost + encodeZone + encodeUserPassword + encodeQueryComponent + encodeFragment + + // hexChar is actually NOT an encoding mode, but there are only seven + // encoding modes. We might as well abuse the otherwise unused most + // significant bit in uint8 to indicate whether a character is + // hexadecimal. + hexChar +) + +// END encoding (keep this marker comment in sync with genEnc) + +// Keep this in sync with the definitions of encoding mode constants. +var encNames = map[encoding]string{ + encodePath: "encodePath", + encodePathSegment: "encodePathSegment", + encodeHost: "encodeHost", + encodeZone: "encodeZone", + encodeUserPassword: "encodeUserPassword", + encodeQueryComponent: "encodeQueryComponent", + encodeFragment: "encodeFragment", +} + +// Return true if the specified character should be escaped when +// appearing in a URL string, according to RFC 3986. +// +// Please be informed that for now shouldEscape does not check all +// reserved characters correctly. See golang.org/issue/5684. +func shouldEscape(c byte, mode encoding) bool { + // §2.3 Unreserved characters (alphanum) + if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' { + return false + } + + if mode == encodeHost || mode == encodeZone { + // §3.2.2 Host allows + // sub-delims = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "=" + // as part of reg-name. + // We add : because we include :port as part of host. + // We add [ ] because we include [ipv6]:port as part of host. + // We add < > because they're the only characters left that + // we could possibly allow, and Parse will reject them if we + // escape them (because hosts can't use %-encoding for + // ASCII bytes). + switch c { + case '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=', ':', '[', ']', '<', '>', '"': + return false + } + } + + switch c { + case '-', '_', '.', '~': // §2.3 Unreserved characters (mark) + return false + + case '$', '&', '+', ',', '/', ':', ';', '=', '?', '@': // §2.2 Reserved characters (reserved) + // Different sections of the URL allow a few of + // the reserved characters to appear unescaped. + switch mode { + case encodePath: // §3.3 + // The RFC allows : @ & = + $ but saves / ; , for assigning + // meaning to individual path segments. This package + // only manipulates the path as a whole, so we allow those + // last three as well. That leaves only ? to escape. + return c == '?' + + case encodePathSegment: // §3.3 + // The RFC allows : @ & = + $ but saves / ; , for assigning + // meaning to individual path segments. + return c == '/' || c == ';' || c == ',' || c == '?' + + case encodeUserPassword: // §3.2.1 + // The RFC allows ';', ':', '&', '=', '+', '$', and ',' in + // userinfo, so we must escape only '@', '/', and '?'. + // The parsing of userinfo treats ':' as special so we must escape + // that too. + return c == '@' || c == '/' || c == '?' || c == ':' + + case encodeQueryComponent: // §3.4 + // The RFC reserves (so we must escape) everything. + return true + + case encodeFragment: // §4.1 + // The RFC text is silent but the grammar allows + // everything, so escape nothing. + return false + } + } + + if mode == encodeFragment { + // RFC 3986 §2.2 allows not escaping sub-delims. A subset of sub-delims are + // included in reserved from RFC 2396 §2.2. The remaining sub-delims do not + // need to be escaped. To minimize potential breakage, we apply two restrictions: + // (1) we always escape sub-delims outside of the fragment, and (2) we always + // escape single quote to avoid breaking callers that had previously assumed that + // single quotes would be escaped. See issue #19917. + switch c { + case '!', '(', ')', '*': + return false + } + } + + // Everything else must be escaped. + return true +} + +func ishex(c byte) bool { + return '0' <= c && c <= '9' || + 'a' <= c && c <= 'f' || + 'A' <= c && c <= 'F' +} diff --git a/src/net/url/url.go b/src/net/url/url.go index 7beaef1ba66311..71fd8f59b3f451 100644 --- a/src/net/url/url.go +++ b/src/net/url/url.go @@ -3,16 +3,22 @@ // license that can be found in the LICENSE file. // Package url parses URLs and implements query escaping. +// +// See RFC 3986. This package generally follows RFC 3986, except where +// it deviates for compatibility reasons. +// RFC 6874 followed for IPv6 zone literals. + +//go:generate go run gen_encoding_table.go + package url -// See RFC 3986. This package generally follows RFC 3986, except where -// it deviates for compatibility reasons. When sending changes, first -// search old issues for history on decisions. Unit tests should also -// contain references to issue numbers with details. +// When sending changes, first search old issues for history on decisions. +// Unit tests should also contain references to issue numbers with details. import ( "errors" "fmt" + "net/netip" "path" "slices" "strconv" @@ -47,15 +53,7 @@ func (e *Error) Temporary() bool { const upperhex = "0123456789ABCDEF" func ishex(c byte) bool { - switch { - case '0' <= c && c <= '9': - return true - case 'a' <= c && c <= 'f': - return true - case 'A' <= c && c <= 'F': - return true - } - return false + return table[c]&hexChar != 0 } func unhex(c byte) byte { @@ -66,22 +64,11 @@ func unhex(c byte) byte { return c - 'a' + 10 case 'A' <= c && c <= 'F': return c - 'A' + 10 + default: + panic("invalid hex character") } - return 0 } -type encoding int - -const ( - encodePath encoding = 1 + iota - encodePathSegment - encodeHost - encodeZone - encodeUserPassword - encodeQueryComponent - encodeFragment -) - type EscapeError string func (e EscapeError) Error() string { @@ -94,86 +81,9 @@ func (e InvalidHostError) Error() string { return "invalid character " + strconv.Quote(string(e)) + " in host name" } -// Return true if the specified character should be escaped when -// appearing in a URL string, according to RFC 3986. -// -// Please be informed that for now shouldEscape does not check all -// reserved characters correctly. See golang.org/issue/5684. +// See the reference implementation in gen_encoding_table.go. func shouldEscape(c byte, mode encoding) bool { - // §2.3 Unreserved characters (alphanum) - if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' { - return false - } - - if mode == encodeHost || mode == encodeZone { - // §3.2.2 Host allows - // sub-delims = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "=" - // as part of reg-name. - // We add : because we include :port as part of host. - // We add [ ] because we include [ipv6]:port as part of host. - // We add < > because they're the only characters left that - // we could possibly allow, and Parse will reject them if we - // escape them (because hosts can't use %-encoding for - // ASCII bytes). - switch c { - case '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=', ':', '[', ']', '<', '>', '"': - return false - } - } - - switch c { - case '-', '_', '.', '~': // §2.3 Unreserved characters (mark) - return false - - case '$', '&', '+', ',', '/', ':', ';', '=', '?', '@': // §2.2 Reserved characters (reserved) - // Different sections of the URL allow a few of - // the reserved characters to appear unescaped. - switch mode { - case encodePath: // §3.3 - // The RFC allows : @ & = + $ but saves / ; , for assigning - // meaning to individual path segments. This package - // only manipulates the path as a whole, so we allow those - // last three as well. That leaves only ? to escape. - return c == '?' - - case encodePathSegment: // §3.3 - // The RFC allows : @ & = + $ but saves / ; , for assigning - // meaning to individual path segments. - return c == '/' || c == ';' || c == ',' || c == '?' - - case encodeUserPassword: // §3.2.1 - // The RFC allows ';', ':', '&', '=', '+', '$', and ',' in - // userinfo, so we must escape only '@', '/', and '?'. - // The parsing of userinfo treats ':' as special so we must escape - // that too. - return c == '@' || c == '/' || c == '?' || c == ':' - - case encodeQueryComponent: // §3.4 - // The RFC reserves (so we must escape) everything. - return true - - case encodeFragment: // §4.1 - // The RFC text is silent but the grammar allows - // everything, so escape nothing. - return false - } - } - - if mode == encodeFragment { - // RFC 3986 §2.2 allows not escaping sub-delims. A subset of sub-delims are - // included in reserved from RFC 2396 §2.2. The remaining sub-delims do not - // need to be escaped. To minimize potential breakage, we apply two restrictions: - // (1) we always escape sub-delims outside of the fragment, and (2) we always - // escape single quote to avoid breaking callers that had previously assumed that - // single quotes would be escaped. See issue #19917. - switch c { - case '!', '(', ')', '*': - return false - } - } - - // Everything else must be escaped. - return true + return table[c]&mode == 0 } // QueryUnescape does the inverse transformation of [QueryEscape], @@ -360,25 +270,41 @@ func escape(s string, mode encoding) string { // A consequence is that it is impossible to tell which slashes in the Path were // slashes in the raw URL and which were %2f. This distinction is rarely important, // but when it is, the code should use the [URL.EscapedPath] method, which preserves -// the original encoding of Path. +// the original encoding of Path. The Fragment field is also stored in decoded form, +// use [URL.EscapedFragment] to retrieve the original encoding. // -// The RawPath field is an optional field which is only set when the default -// encoding of Path is different from the escaped path. See the EscapedPath method -// for more details. -// -// URL's String method uses the EscapedPath method to obtain the path. +// The [URL.String] method uses the [URL.EscapedPath] method to obtain the path. type URL struct { - Scheme string - Opaque string // encoded opaque data - User *Userinfo // username and password information - Host string // host or host:port (see Hostname and Port methods) - Path string // path (relative paths may omit leading slash) - RawPath string // encoded path hint (see EscapedPath method) - OmitHost bool // do not emit empty host (authority) - ForceQuery bool // append a query ('?') even if RawQuery is empty - RawQuery string // encoded query values, without '?' - Fragment string // fragment for references, without '#' - RawFragment string // encoded fragment hint (see EscapedFragment method) + Scheme string + Opaque string // encoded opaque data + User *Userinfo // username and password information + Host string // "host" or "host:port" (see Hostname and Port methods) + Path string // path (relative paths may omit leading slash) + Fragment string // fragment for references (without '#') + + // RawQuery contains the encoded query values, without the initial '?'. + // Use URL.Query to decode the query. + RawQuery string + + // RawPath is an optional field containing an encoded path hint. + // See the EscapedPath method for more details. + // + // In general, code should call EscapedPath instead of reading RawPath. + RawPath string + + // RawFragment is an optional field containing an encoded fragment hint. + // See the EscapedFragment method for more details. + // + // In general, code should call EscapedFragment instead of reading RawFragment. + RawFragment string + + // ForceQuery indicates whether the original URL contained a query ('?') character. + // When set, the String method will include a trailing '?', even when RawQuery is empty. + ForceQuery bool + + // OmitHost indicates the URL has an empty host (authority). + // When set, the String method will not include the host when it is empty. + OmitHost bool } // User returns a [Userinfo] containing the provided username @@ -622,40 +548,61 @@ func parseAuthority(authority string) (user *Userinfo, host string, err error) { // parseHost parses host as an authority without user // information. That is, as host[:port]. func parseHost(host string) (string, error) { - if strings.HasPrefix(host, "[") { + if openBracketIdx := strings.LastIndex(host, "["); openBracketIdx != -1 { // Parse an IP-Literal in RFC 3986 and RFC 6874. // E.g., "[fe80::1]", "[fe80::1%25en0]", "[fe80::1]:80". - i := strings.LastIndex(host, "]") - if i < 0 { + closeBracketIdx := strings.LastIndex(host, "]") + if closeBracketIdx < 0 { return "", errors.New("missing ']' in host") } - colonPort := host[i+1:] + + colonPort := host[closeBracketIdx+1:] if !validOptionalPort(colonPort) { return "", fmt.Errorf("invalid port %q after host", colonPort) } + unescapedColonPort, err := unescape(colonPort, encodeHost) + if err != nil { + return "", err + } + hostname := host[openBracketIdx+1 : closeBracketIdx] + var unescapedHostname string // RFC 6874 defines that %25 (%-encoded percent) introduces // the zone identifier, and the zone identifier can use basically // any %-encoding it likes. That's different from the host, which // can only %-encode non-ASCII bytes. // We do impose some restrictions on the zone, to avoid stupidity // like newlines. - zone := strings.Index(host[:i], "%25") - if zone >= 0 { - host1, err := unescape(host[:zone], encodeHost) + zoneIdx := strings.Index(hostname, "%25") + if zoneIdx >= 0 { + hostPart, err := unescape(hostname[:zoneIdx], encodeHost) if err != nil { return "", err } - host2, err := unescape(host[zone:i], encodeZone) + zonePart, err := unescape(hostname[zoneIdx:], encodeZone) if err != nil { return "", err } - host3, err := unescape(host[i:], encodeHost) + unescapedHostname = hostPart + zonePart + } else { + var err error + unescapedHostname, err = unescape(hostname, encodeHost) if err != nil { return "", err } - return host1 + host2 + host3, nil } + + // Per RFC 3986, only a host identified by a valid + // IPv6 address can be enclosed by square brackets. + // This excludes any IPv4, but notably not IPv4-mapped addresses. + addr, err := netip.ParseAddr(unescapedHostname) + if err != nil { + return "", fmt.Errorf("invalid host: %w", err) + } + if addr.Is4() { + return "", errors.New("invalid IP-literal") + } + return "[" + unescapedHostname + "]" + unescapedColonPort, nil } else if i := strings.LastIndex(host, ":"); i != -1 { colonPort := host[i:] if !validOptionalPort(colonPort) { @@ -1004,9 +951,13 @@ func (v Values) Encode() string { return "" } var buf strings.Builder - keys := make([]string, 0, len(v)) + // To minimize allocations, we eschew iterators and pre-size the slice in + // which we collect v's keys. + keys := make([]string, len(v)) + var i int for k := range v { - keys = append(keys, k) + keys[i] = k + i++ } slices.Sort(keys) for _, k := range keys { @@ -1219,7 +1170,11 @@ func splitHostPort(hostPort string) (host, port string) { // Would like to implement MarshalText/UnmarshalText but that will change the JSON representation of URLs. func (u *URL) MarshalBinary() (text []byte, err error) { - return []byte(u.String()), nil + return u.AppendBinary(nil) +} + +func (u *URL) AppendBinary(b []byte) ([]byte, error) { + return append(b, u.String()...), nil } func (u *URL) UnmarshalBinary(text []byte) error { @@ -1277,7 +1232,18 @@ func validUserinfo(s string) bool { } switch r { case '-', '.', '_', ':', '~', '!', '$', '&', '\'', - '(', ')', '*', '+', ',', ';', '=', '%', '@': + '(', ')', '*', '+', ',', ';', '=', '%': + continue + case '@': + // `RFC 3986 section 3.2.1` does not allow '@' in userinfo. + // It is a delimiter between userinfo and host. + // However, URLs are diverse, and in some cases, + // the userinfo may contain an '@' character, + // for example, in "http://username:p%40ssword@google.com/", + // the string "username:p@ssword" should be treated as valid userinfo. + // Ref: + // https://go.dev/issue/3439 + // https://go.dev/issue/22655 continue default: return false diff --git a/src/net/url/url_test.go b/src/net/url/url_test.go index 68219c3df1081d..501558403ac771 100644 --- a/src/net/url/url_test.go +++ b/src/net/url/url_test.go @@ -383,6 +383,16 @@ var urltests = []URLTest{ }, "", }, + // valid IPv6 host with port and path + { + "https://[2001:db8::1]:8443/test/path", + &URL{ + Scheme: "https", + Host: "[2001:db8::1]:8443", + Path: "/test/path", + }, + "", + }, // host subcomponent; IPv6 address with zone identifier in RFC 6874 { "http://[fe80::1%25en0]/", // alphanum zone identifier @@ -707,6 +717,24 @@ var parseRequestURLTests = []struct { // RFC 6874. {"http://[fe80::1%en0]/", false}, {"http://[fe80::1%en0]:8080/", false}, + + // Tests exercising RFC 3986 compliance + {"https://[1:2:3:4:5:6:7:8]/", true}, // full IPv6 address + {"https://[2001:db8::a:b:c:d]/", true}, // compressed IPv6 address + {"https://[fe80::1%25eth0]", true}, // link-local address with zone ID (interface name) + {"https://[fe80::abc:def%254]", true}, // link-local address with zone ID (interface index) + {"https://[2001:db8::1]/path", true}, // compressed IPv6 address with path + {"https://[fe80::1%25eth0]/path?query=1", true}, // link-local with zone, path, and query + + {"https://[::ffff:c000:201]/", true}, + {"https://[:1] ", false}, + {"https://[1:2:3:4:5:6:7:8:9]", false}, + {"https://[1::1::1]", false}, + {"https://[1:2:3:]", false}, + {"https://[ffff::127.0.0.4000]", false}, + {"https://[0:0::test.com]:80", false}, + {"https://[2001:db8::test.com]", false}, + {"https://[test.com]", false}, } func TestParseRequestURI(t *testing.T) { @@ -1080,6 +1108,17 @@ var encodeQueryTests = []EncodeQueryTest{ "b": {"b1", "b2", "b3"}, "c": {"c1", "c2", "c3"}, }, "a=a1&a=a2&a=a3&b=b1&b=b2&b=b3&c=c1&c=c2&c=c3"}, + {Values{ + "a": {"a"}, + "b": {"b"}, + "c": {"c"}, + "d": {"d"}, + "e": {"e"}, + "f": {"f"}, + "g": {"g"}, + "h": {"h"}, + "i": {"i"}, + }, "a=a&b=b&c=c&d=d&e=e&f=f&g=g&h=h&i=i"}, } func TestEncodeQuery(t *testing.T) { @@ -1090,6 +1129,17 @@ func TestEncodeQuery(t *testing.T) { } } +func BenchmarkEncodeQuery(b *testing.B) { + for _, tt := range encodeQueryTests { + b.Run(tt.expected, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + tt.m.Encode() + } + }) + } +} + var resolvePathTests = []struct { base, ref, expected string }{ @@ -1643,6 +1693,18 @@ func TestParseErrors(t *testing.T) { {"cache_object:foo", true}, {"cache_object:foo/bar", true}, {"cache_object/:foo/bar", false}, + + {"http://[192.168.0.1]/", true}, // IPv4 in brackets + {"http://[192.168.0.1]:8080/", true}, // IPv4 in brackets with port + {"http://[::ffff:c0a8:1]/", false}, // IPv4-mapped IPv6 in brackets + {"http://[::ffff:192.168.0.1000]/", true}, // Out of range IPv4-mapped IPv6 in brackets + {"http://[::ffff:c0a8:1]:8080/", false}, // IPv4-mapped IPv6 in brackets with port + {"http://[::ffff:c0a8:1]/", false}, // IPv4-mapped IPv6 in brackets (hex) + {"http://[not-an-ip]/", true}, // invalid IP string in brackets + {"http://[fe80::1%foo]/", true}, // invalid zone format in brackets + {"http://[fe80::1", true}, // missing closing bracket + {"http://fe80::1]/", true}, // missing opening bracket + {"http://[test.com]/", true}, // domain name in brackets } for _, tt := range tests { u, err := Parse(tt.in) @@ -1866,6 +1928,7 @@ func TestURLHostnameAndPort(t *testing.T) { var _ encodingPkg.BinaryMarshaler = (*URL)(nil) var _ encodingPkg.BinaryUnmarshaler = (*URL)(nil) +var _ encodingPkg.BinaryAppender = (*URL)(nil) func TestJSON(t *testing.T) { u, err := Parse("https://www.google.com/x?y=z") diff --git a/src/os/copy_test.go b/src/os/copy_test.go new file mode 100644 index 00000000000000..6fe7f6e53b8a5e --- /dev/null +++ b/src/os/copy_test.go @@ -0,0 +1,254 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os_test + +import ( + "bytes" + "errors" + "fmt" + "io" + "math/rand/v2" + "net" + "os" + "runtime" + "sync" + "testing" + + "golang.org/x/net/nettest" +) + +// Exercise sendfile/splice fast paths with a moderately large file. +// +// https://go.dev/issue/70000 + +func TestLargeCopyViaNetwork(t *testing.T) { + const size = 10 * 1024 * 1024 + dir := t.TempDir() + + src, err := os.Create(dir + "/src") + if err != nil { + t.Fatal(err) + } + defer src.Close() + if _, err := io.CopyN(src, newRandReader(), size); err != nil { + t.Fatal(err) + } + if _, err := src.Seek(0, 0); err != nil { + t.Fatal(err) + } + + dst, err := os.Create(dir + "/dst") + if err != nil { + t.Fatal(err) + } + defer dst.Close() + + client, server := createSocketPair(t, "tcp") + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + if n, err := io.Copy(dst, server); n != size || err != nil { + t.Errorf("copy to destination = %v, %v; want %v, nil", n, err, size) + } + }() + go func() { + defer wg.Done() + defer client.Close() + if n, err := io.Copy(client, src); n != size || err != nil { + t.Errorf("copy from source = %v, %v; want %v, nil", n, err, size) + } + }() + wg.Wait() + + if _, err := dst.Seek(0, 0); err != nil { + t.Fatal(err) + } + if err := compareReaders(dst, io.LimitReader(newRandReader(), size)); err != nil { + t.Fatal(err) + } +} + +func TestCopyFileToFile(t *testing.T) { + const size = 1 * 1024 * 1024 + dir := t.TempDir() + + src, err := os.Create(dir + "/src") + if err != nil { + t.Fatal(err) + } + defer src.Close() + if _, err := io.CopyN(src, newRandReader(), size); err != nil { + t.Fatal(err) + } + if _, err := src.Seek(0, 0); err != nil { + t.Fatal(err) + } + + mustSeek := func(f *os.File, offset int64, whence int) int64 { + ret, err := f.Seek(offset, whence) + if err != nil { + t.Fatal(err) + } + return ret + } + + for _, srcStart := range []int64{0, 100, size} { + remaining := size - srcStart + for _, dstStart := range []int64{0, 200} { + for _, limit := range []int64{remaining, remaining - 100, size * 2, 0} { + if limit < 0 { + continue + } + name := fmt.Sprintf("srcStart=%v/dstStart=%v/limit=%v", srcStart, dstStart, limit) + t.Run(name, func(t *testing.T) { + dst, err := os.CreateTemp(dir, "dst") + if err != nil { + t.Fatal(err) + } + defer dst.Close() + defer os.Remove(dst.Name()) + + mustSeek(src, srcStart, io.SeekStart) + if _, err := io.CopyN(dst, zeroReader{}, dstStart); err != nil { + t.Fatal(err) + } + + var copied int64 + if limit == 0 { + copied, err = io.Copy(dst, src) + } else { + copied, err = io.CopyN(dst, src, limit) + } + if limit > remaining { + if err != io.EOF { + t.Errorf("Copy: %v; want io.EOF", err) + } + } else { + if err != nil { + t.Errorf("Copy: %v; want nil", err) + } + } + + wantCopied := remaining + if limit != 0 { + wantCopied = min(limit, wantCopied) + } + if copied != wantCopied { + t.Errorf("copied %v bytes, want %v", copied, wantCopied) + } + + srcPos := mustSeek(src, 0, io.SeekCurrent) + wantSrcPos := srcStart + wantCopied + if srcPos != wantSrcPos { + t.Errorf("source position = %v, want %v", srcPos, wantSrcPos) + } + + dstPos := mustSeek(dst, 0, io.SeekCurrent) + wantDstPos := dstStart + wantCopied + if dstPos != wantDstPos { + t.Errorf("destination position = %v, want %v", dstPos, wantDstPos) + } + + mustSeek(dst, 0, io.SeekStart) + rr := newRandReader() + io.CopyN(io.Discard, rr, srcStart) + wantReader := io.MultiReader( + io.LimitReader(zeroReader{}, dstStart), + io.LimitReader(rr, wantCopied), + ) + if err := compareReaders(dst, wantReader); err != nil { + t.Fatal(err) + } + }) + + } + } + } +} + +func compareReaders(a, b io.Reader) error { + bufa := make([]byte, 4096) + bufb := make([]byte, 4096) + off := 0 + for { + na, erra := io.ReadFull(a, bufa) + if erra != nil && erra != io.EOF && erra != io.ErrUnexpectedEOF { + return erra + } + nb, errb := io.ReadFull(b, bufb) + if errb != nil && errb != io.EOF && errb != io.ErrUnexpectedEOF { + return errb + } + if !bytes.Equal(bufa[:na], bufb[:nb]) { + return errors.New("contents mismatch") + } + if erra != nil && errb != nil { + break + } + off += len(bufa) + } + return nil +} + +type zeroReader struct{} + +func (r zeroReader) Read(p []byte) (int, error) { + clear(p) + return len(p), nil +} + +type randReader struct { + rand *rand.Rand +} + +func newRandReader() *randReader { + return &randReader{rand.New(rand.NewPCG(0, 0))} +} + +func (r *randReader) Read(p []byte) (int, error) { + for i := range p { + p[i] = byte(r.rand.Uint32() & 0xff) + } + return len(p), nil +} + +func createSocketPair(t *testing.T, proto string) (client, server net.Conn) { + t.Helper() + if !nettest.TestableNetwork(proto) { + t.Skipf("%s does not support %q", runtime.GOOS, proto) + } + + ln, err := nettest.NewLocalListener(proto) + if err != nil { + t.Fatalf("NewLocalListener error: %v", err) + } + t.Cleanup(func() { + if ln != nil { + ln.Close() + } + if client != nil { + client.Close() + } + if server != nil { + server.Close() + } + }) + ch := make(chan struct{}) + go func() { + var err error + server, err = ln.Accept() + if err != nil { + t.Errorf("Accept new connection error: %v", err) + } + ch <- struct{}{} + }() + client, err = net.Dial(proto, ln.Addr().String()) + <-ch + if err != nil { + t.Fatalf("Dial new connection error: %v", err) + } + return client, server +} diff --git a/src/os/dir.go b/src/os/dir.go index dab75b5d436ce5..fb71d88e3e30b9 100644 --- a/src/os/dir.go +++ b/src/os/dir.go @@ -106,10 +106,6 @@ func (f *File) ReadDir(n int) ([]DirEntry, error) { return dirents, err } -// testingForceReadDirLstat forces ReadDir to call Lstat, for testing that code path. -// This can be difficult to provoke on some Unix systems otherwise. -var testingForceReadDirLstat bool - // ReadDir reads the named directory, // returning all its directory entries sorted by filename. // If an error occurs reading the directory, @@ -136,14 +132,15 @@ func ReadDir(name string) ([]DirEntry, error) { // from the source, and directories are created with mode 0o777 // (before umask). // -// CopyFS will not overwrite existing files, and returns an error -// if a file name in fsys already exists in the destination. -// -// Symbolic links in fsys are not supported. A *PathError with Err set -// to ErrInvalid is returned when copying from a symbolic link. +// CopyFS will not overwrite existing files. If a file name in fsys +// already exists in the destination, CopyFS will return an error +// such that errors.Is(err, fs.ErrExist) will be true. // // Symbolic links in dir are followed. // +// New files added to fsys (including if dir is a subdirectory of fsys) +// while CopyFS is running are not guaranteed to be copied. +// // Copying stops at and returns the first error encountered. func CopyFS(dir string, fsys fs.FS) error { return fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error { @@ -156,35 +153,38 @@ func CopyFS(dir string, fsys fs.FS) error { return err } newPath := joinPath(dir, fpath) - if d.IsDir() { - return MkdirAll(newPath, 0777) - } - // TODO(panjf2000): handle symlinks with the help of fs.ReadLinkFS - // once https://go.dev/issue/49580 is done. - // we also need filepathlite.IsLocal from https://go.dev/cl/564295. - if !d.Type().IsRegular() { + switch d.Type() { + case ModeDir: + return MkdirAll(newPath, 0777) + case ModeSymlink: + target, err := fs.ReadLink(fsys, path) + if err != nil { + return err + } + return Symlink(target, newPath) + case 0: + r, err := fsys.Open(path) + if err != nil { + return err + } + defer r.Close() + info, err := r.Stat() + if err != nil { + return err + } + w, err := OpenFile(newPath, O_CREATE|O_EXCL|O_WRONLY, 0666|info.Mode()&0777) + if err != nil { + return err + } + + if _, err := io.Copy(w, r); err != nil { + w.Close() + return &PathError{Op: "Copy", Path: newPath, Err: err} + } + return w.Close() + default: return &PathError{Op: "CopyFS", Path: path, Err: ErrInvalid} } - - r, err := fsys.Open(path) - if err != nil { - return err - } - defer r.Close() - info, err := r.Stat() - if err != nil { - return err - } - w, err := OpenFile(newPath, O_CREATE|O_TRUNC|O_WRONLY, 0666|info.Mode()&0777) - if err != nil { - return err - } - - if _, err := io.Copy(w, r); err != nil { - w.Close() - return &PathError{Op: "Copy", Path: newPath, Err: err} - } - return w.Close() }) } diff --git a/src/os/dir_plan9.go b/src/os/dir_plan9.go index ab5c1efce5b075..9d28bd7dda4a89 100644 --- a/src/os/dir_plan9.go +++ b/src/os/dir_plan9.go @@ -11,12 +11,19 @@ import ( ) func (file *File) readdir(n int, mode readdirMode) (names []string, dirents []DirEntry, infos []FileInfo, err error) { - // If this file has no dirinfo, create one. - d := file.dirinfo.Load() - if d == nil { - d = new(dirInfo) - file.dirinfo.Store(d) + var d *dirInfo + for { + d = file.dirinfo.Load() + if d != nil { + break + } + newD := new(dirInfo) + if file.dirinfo.CompareAndSwap(nil, newD) { + d = newD + break + } } + d.mu.Lock() defer d.mu.Unlock() diff --git a/src/os/dir_unix.go b/src/os/dir_unix.go index d8b4faa05788f0..6a0135b70b073b 100644 --- a/src/os/dir_unix.go +++ b/src/os/dir_unix.go @@ -46,11 +46,19 @@ func (d *dirInfo) close() { func (f *File) readdir(n int, mode readdirMode) (names []string, dirents []DirEntry, infos []FileInfo, err error) { // If this file has no dirInfo, create one. - d := f.dirinfo.Load() - if d == nil { - d = new(dirInfo) - f.dirinfo.Store(d) + var d *dirInfo + for { + d = f.dirinfo.Load() + if d != nil { + break + } + newD := new(dirInfo) + if f.dirinfo.CompareAndSwap(nil, newD) { + d = newD + break + } } + d.mu.Lock() defer d.mu.Unlock() if d.buf == nil { @@ -175,11 +183,11 @@ func readIntBE(b []byte, size uintptr) uint64 { case 1: return uint64(b[0]) case 2: - return uint64(byteorder.BeUint16(b)) + return uint64(byteorder.BEUint16(b)) case 4: - return uint64(byteorder.BeUint32(b)) + return uint64(byteorder.BEUint32(b)) case 8: - return uint64(byteorder.BeUint64(b)) + return uint64(byteorder.BEUint64(b)) default: panic("syscall: readInt with unsupported size") } @@ -190,11 +198,11 @@ func readIntLE(b []byte, size uintptr) uint64 { case 1: return uint64(b[0]) case 2: - return uint64(byteorder.LeUint16(b)) + return uint64(byteorder.LEUint16(b)) case 4: - return uint64(byteorder.LeUint32(b)) + return uint64(byteorder.LEUint32(b)) case 8: - return uint64(byteorder.LeUint64(b)) + return uint64(byteorder.LEUint64(b)) default: panic("syscall: readInt with unsupported size") } diff --git a/src/os/eloop_netbsd.go b/src/os/eloop_netbsd.go new file mode 100644 index 00000000000000..670c88a8359812 --- /dev/null +++ b/src/os/eloop_netbsd.go @@ -0,0 +1,19 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build netbsd + +package os + +import "syscall" + +// isNoFollowErr reports whether err may result from O_NOFOLLOW blocking an open operation. +func isNoFollowErr(err error) bool { + // NetBSD returns EFTYPE, but check the other possibilities as well. + switch err { + case syscall.ELOOP, syscall.EMLINK, syscall.EFTYPE: + return true + } + return false +} diff --git a/src/os/eloop_other.go b/src/os/eloop_other.go new file mode 100644 index 00000000000000..aace57bd39124e --- /dev/null +++ b/src/os/eloop_other.go @@ -0,0 +1,27 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || darwin || dragonfly || freebsd || linux || openbsd || solaris || wasip1 + +package os + +import ( + "runtime" + "syscall" +) + +// isNoFollowErr reports whether err may result from O_NOFOLLOW blocking an open operation. +func isNoFollowErr(err error) bool { + switch err { + case syscall.ELOOP, syscall.EMLINK: + return true + } + if runtime.GOOS == "dragonfly" { + // Dragonfly appears to return EINVAL from openat in this case. + if err == syscall.EINVAL { + return true + } + } + return false +} diff --git a/src/os/env_test.go b/src/os/env_test.go index e3de64196abdbc..2515881db85ffe 100644 --- a/src/os/env_test.go +++ b/src/os/env_test.go @@ -189,17 +189,13 @@ func TestEnvironConsistency(t *testing.T) { k := kv[:i] v := kv[i+1:] v2, ok := LookupEnv(k) - if ok && v == v2 { - t.Logf("LookupEnv(%q) = %q, %t", k, v2, ok) - } else { + if !ok || v != v2 { t.Errorf("Environ contains %q, but LookupEnv(%q) = %q, %t", kv, k, v2, ok) } // Since k=v is already present in the environment, // setting it should be a no-op. - if err := Setenv(k, v); err == nil { - t.Logf("Setenv(%q, %q)", k, v) - } else { + if err := Setenv(k, v); err != nil { t.Errorf("Environ contains %q, but SetEnv(%q, %q) = %q", kv, k, v, err) } } diff --git a/src/os/error_errno.go b/src/os/error_errno.go index c8140461a4dd9f..662258338bd929 100644 --- a/src/os/error_errno.go +++ b/src/os/error_errno.go @@ -9,3 +9,9 @@ package os import "syscall" type syscallErrorType = syscall.Errno + +const ( + errENOSYS = syscall.ENOSYS + errERANGE = syscall.ERANGE + errENOMEM = syscall.ENOMEM +) diff --git a/src/os/error_plan9.go b/src/os/error_plan9.go index af6065db568edc..35026554c27bcb 100644 --- a/src/os/error_plan9.go +++ b/src/os/error_plan9.go @@ -7,3 +7,7 @@ package os import "syscall" type syscallErrorType = syscall.ErrorString + +var errENOSYS = syscall.NewError("function not implemented") +var errERANGE = syscall.NewError("out of range") +var errENOMEM = syscall.NewError("cannot allocate memory") diff --git a/src/os/exec.go b/src/os/exec.go index 1220761df5ee09..8b164ad6673bb9 100644 --- a/src/os/exec.go +++ b/src/os/exec.go @@ -14,239 +14,192 @@ import ( "time" ) -// ErrProcessDone indicates a [Process] has finished. -var ErrProcessDone = errors.New("os: process already finished") - -type processMode uint8 - -const ( - // modePID means that Process operations such use the raw PID from the - // Pid field. handle is not used. - // - // This may be due to the host not supporting handles, or because - // Process was created as a literal, leaving handle unset. - // - // This must be the zero value so Process literals get modePID. - modePID processMode = iota - - // modeHandle means that Process operations use handle, which is - // initialized with an OS process handle. - // - // Note that Release and Wait will deactivate and eventually close the - // handle, so acquire may fail, indicating the reason. - modeHandle +var ( + // ErrProcessDone indicates a [Process] has finished. + ErrProcessDone = errors.New("os: process already finished") + // errProcessReleased indicates a [Process] has been released. + errProcessReleased = errors.New("os: process already released") + // ErrNoHandle indicates a [Process] does not have a handle. + ErrNoHandle = errors.New("os: process handle unavailable") ) -type processStatus uint64 +// processStatus describes the status of a [Process]. +type processStatus uint32 const ( - // PID/handle OK to use. - statusOK processStatus = 0 + // statusOK means that the Process is ready to use. + statusOK processStatus = iota // statusDone indicates that the PID/handle should not be used because // the process is done (has been successfully Wait'd on). - statusDone processStatus = 1 << 62 + statusDone // statusReleased indicates that the PID/handle should not be used // because the process is released. - statusReleased processStatus = 1 << 63 - - processStatusMask = 0x3 << 62 + statusReleased ) // Process stores the information about a process created by [StartProcess]. type Process struct { Pid int - mode processMode - - // State contains the atomic process state. - // - // In modePID, this consists only of the processStatus fields, which - // indicate if the process is done/released. - // - // In modeHandle, the lower bits also contain a reference count for the - // handle field. - // - // The Process itself initially holds 1 persistent reference. Any - // operation that uses the handle with a system call temporarily holds - // an additional transient reference. This prevents the handle from - // being closed prematurely, which could result in the OS allocating a - // different handle with the same value, leading to Process' methods - // operating on the wrong process. - // - // Release and Wait both drop the Process' persistent reference, but - // other concurrent references may delay actually closing the handle - // because they hold a transient reference. + // state contains the atomic process state. // - // Regardless, we want new method calls to immediately treat the handle - // as unavailable after Release or Wait to avoid extending this delay. - // This is achieved by setting either processStatus flag when the - // Process' persistent reference is dropped. The only difference in the - // flags is the reason the handle is unavailable, which affects the - // errors returned by concurrent calls. - state atomic.Uint64 - - // Used only in modePID. + // This consists of the processStatus fields, + // which indicate if the process is done/released. + state atomic.Uint32 + + // Used only when handle is nil sigMu sync.RWMutex // avoid race between wait and signal - // handle is the OS handle for process actions, used only in - // modeHandle. - // - // handle must be accessed only via the handleTransientAcquire method - // (or during closeHandle), not directly! handle is immutable. + // handle, if not nil, is a pointer to a struct + // that holds the OS-specific process handle. + // This pointer is set when Process is created, + // and never changed afterward. + // This is a pointer to a separate memory allocation + // so that we can use runtime.AddCleanup. + handle *processHandle + + // cleanup is used to clean up the process handle. + cleanup runtime.Cleanup +} + +// processHandle holds an operating system handle to a process. +// This is only used on systems that support that concept, +// currently Linux and Windows. +// This maintains a reference count to the handle, +// and closes the handle when the reference drops to zero. +type processHandle struct { + // The actual handle. This field should not be used directly. + // Instead, use the acquire and release methods. // - // On Windows, it is a handle from OpenProcess. - // On Linux, it is a pidfd. - // It is unused on other GOOSes. + // On Windows this is a handle returned by OpenProcess. + // On Linux this is a pidfd. handle uintptr + + // Number of active references. When this drops to zero + // the handle is closed. + refs atomic.Int32 +} + +// acquire adds a reference and returns the handle. +// The bool result reports whether acquire succeeded; +// it fails if the handle is already closed. +// Every successful call to acquire should be paired with a call to release. +func (ph *processHandle) acquire() (uintptr, bool) { + for { + refs := ph.refs.Load() + if refs < 0 { + panic("internal error: negative process handle reference count") + } + if refs == 0 { + return 0, false + } + if ph.refs.CompareAndSwap(refs, refs+1) { + return ph.handle, true + } + } } +// release releases a reference to the handle. +func (ph *processHandle) release() { + for { + refs := ph.refs.Load() + if refs <= 0 { + panic("internal error: too many releases of process handle") + } + if ph.refs.CompareAndSwap(refs, refs-1) { + if refs == 1 { + ph.closeHandle() + } + return + } + } +} + +// newPIDProcess returns a [Process] for the given PID. func newPIDProcess(pid int) *Process { p := &Process{ - Pid: pid, - mode: modePID, + Pid: pid, } - runtime.SetFinalizer(p, (*Process).Release) return p } +// newHandleProcess returns a [Process] with the given PID and handle. func newHandleProcess(pid int, handle uintptr) *Process { + ph := &processHandle{ + handle: handle, + } + + // Start the reference count as 1, + // meaning the reference from the returned Process. + ph.refs.Store(1) + p := &Process{ Pid: pid, - mode: modeHandle, - handle: handle, + handle: ph, } - p.state.Store(1) // 1 persistent reference - runtime.SetFinalizer(p, (*Process).Release) + + p.cleanup = runtime.AddCleanup(p, (*processHandle).release, ph) + return p } +// newDoneProcess returns a [Process] for the given PID +// that is already marked as done. This is used on Unix systems +// if the process is known to not exist. func newDoneProcess(pid int) *Process { p := &Process{ - Pid: pid, - mode: modeHandle, - // N.B Since we set statusDone, handle will never actually be - // used, so its value doesn't matter. + Pid: pid, } - p.state.Store(uint64(statusDone)) // No persistent reference, as there is no handle. - runtime.SetFinalizer(p, (*Process).Release) + p.state.Store(uint32(statusDone)) // No persistent reference, as there is no handle. return p } +// handleTransientAcquire returns the process handle or, +// if the process is not ready, the current status. func (p *Process) handleTransientAcquire() (uintptr, processStatus) { - if p.mode != modeHandle { + if p.handle == nil { panic("handleTransientAcquire called in invalid mode") } - for { - refs := p.state.Load() - if refs&processStatusMask != 0 { - return 0, processStatus(refs & processStatusMask) - } - new := refs + 1 - if !p.state.CompareAndSwap(refs, new) { - continue - } - return p.handle, statusOK + status := processStatus(p.state.Load()) + if status != statusOK { + return 0, status } -} - -func (p *Process) handleTransientRelease() { - if p.mode != modeHandle { - panic("handleTransientRelease called in invalid mode") + h, ok := p.handle.acquire() + if ok { + return h, statusOK } - for { - state := p.state.Load() - refs := state &^ processStatusMask - status := processStatus(state & processStatusMask) - if refs == 0 { - // This should never happen because - // handleTransientRelease is always paired with - // handleTransientAcquire. - panic("release of handle with refcount 0") - } - if refs == 1 && status == statusOK { - // Process holds a persistent reference and always sets - // a status when releasing that reference - // (handlePersistentRelease). Thus something has gone - // wrong if this is the last release but a status has - // not always been set. - panic("final release of handle without processStatus") - } - new := state - 1 - if !p.state.CompareAndSwap(state, new) { - continue - } - if new&^processStatusMask == 0 { - p.closeHandle() - } - return + // This case means that the handle has been closed. + // We always set the status to non-zero before closing the handle. + // If we get here the status must have been set non-zero after + // we just checked it above. + status = processStatus(p.state.Load()) + if status == statusOK { + panic("inconsistent process status") } + return 0, status } -// Drop the Process' persistent reference on the handle, deactivating future -// Wait/Signal calls with the passed reason. -// -// Returns the status prior to this call. If this is not statusOK, then the -// reference was not dropped or status changed. -func (p *Process) handlePersistentRelease(reason processStatus) processStatus { - if p.mode != modeHandle { - panic("handlePersistentRelease called in invalid mode") - } - - for { - refs := p.state.Load() - status := processStatus(refs & processStatusMask) - if status != statusOK { - // Both Release and successful Wait will drop the - // Process' persistent reference on the handle. We - // can't allow concurrent calls to drop the reference - // twice, so we use the status as a guard to ensure the - // reference is dropped exactly once. - return status - } - if refs == 0 { - // This should never happen because dropping the - // persistent reference always sets a status. - panic("release of handle with refcount 0") - } - new := (refs - 1) | uint64(reason) - if !p.state.CompareAndSwap(refs, new) { - continue - } - if new&^processStatusMask == 0 { - p.closeHandle() - } - return status +// handleTransientRelease releases a handle returned by handleTransientAcquire. +func (p *Process) handleTransientRelease() { + if p.handle == nil { + panic("handleTransientRelease called in invalid mode") } + p.handle.release() } +// pidStatus returns the current process status. func (p *Process) pidStatus() processStatus { - if p.mode != modePID { + if p.handle != nil { panic("pidStatus called in invalid mode") } return processStatus(p.state.Load()) } -func (p *Process) pidDeactivate(reason processStatus) { - if p.mode != modePID { - panic("pidDeactivate called in invalid mode") - } - - // Both Release and successful Wait will deactivate the PID. Only one - // of those should win, so nothing left to do here if the compare - // fails. - // - // N.B. This means that results can be inconsistent. e.g., with a - // racing Release and Wait, Wait may successfully wait on the process, - // returning the wait status, while future calls error with "process - // released" rather than "process done". - p.state.CompareAndSwap(0, uint64(reason)) -} - // ProcAttr holds the attributes that will be applied to a new process // started by StartProcess. type ProcAttr struct { @@ -323,23 +276,58 @@ func StartProcess(name string, argv []string, attr *ProcAttr) (*Process, error) // rendering it unusable in the future. // Release only needs to be called if [Process.Wait] is not. func (p *Process) Release() error { - // Note to future authors: the Release API is cursed. - // - // On Unix and Plan 9, Release sets p.Pid = -1. This is the only part of the - // Process API that is not thread-safe, but it can't be changed now. - // - // On Windows, Release does _not_ modify p.Pid. - // - // On Windows, Wait calls Release after successfully waiting to - // proactively clean up resources. - // - // On Unix and Plan 9, Wait also proactively cleans up resources, but - // can not call Release, as Wait does not set p.Pid = -1. - // - // On Unix and Plan 9, calling Release a second time has no effect. - // - // On Windows, calling Release a second time returns EINVAL. - return p.release() + // Unfortunately, for historical reasons, on systems other + // than Windows, Release sets the Pid field to -1. + // This causes the race detector to report a problem + // on concurrent calls to Release, but we can't change it now. + if runtime.GOOS != "windows" { + p.Pid = -1 + } + + oldStatus := p.doRelease(statusReleased) + + // For backward compatibility, on Windows only, + // we return EINVAL on a second call to Release. + if runtime.GOOS == "windows" { + if oldStatus == statusReleased { + return syscall.EINVAL + } + } + + return nil +} + +// doRelease releases a [Process], setting the status to newStatus. +// If the previous status is not statusOK, this does nothing. +// It returns the previous status. +func (p *Process) doRelease(newStatus processStatus) processStatus { + for { + state := p.state.Load() + oldStatus := processStatus(state) + if oldStatus != statusOK { + return oldStatus + } + + if !p.state.CompareAndSwap(state, uint32(newStatus)) { + continue + } + + // We have successfully released the Process. + // If it has a handle, release the reference we + // created in newHandleProcess. + if p.handle != nil { + // No need for more cleanup. + // We must stop the cleanup before calling release; + // otherwise the cleanup might run concurrently + // with the release, which would cause the reference + // counts to be invalid, causing a panic. + p.cleanup.Stop() + + p.handle.release() + } + + return statusOK + } } // Kill causes the [Process] to exit immediately. Kill does not wait until @@ -364,6 +352,18 @@ func (p *Process) Signal(sig Signal) error { return p.signal(sig) } +// WithHandle calls a supplied function f with a valid process handle +// as an argument. The handle is guaranteed to refer to process p +// until f returns, even if p terminates. This function cannot be used +// after [Process.Release] or [Process.Wait]. +// +// If process handles are not supported or a handle is not available, +// it returns [ErrNoHandle]. Currently, process handles are supported +// on Linux 5.4 or later (pidfd) and Windows. +func (p *Process) WithHandle(f func(handle uintptr)) error { + return p.withHandle(f) +} + // UserTime returns the user CPU time of the exited process and its children. func (p *ProcessState) UserTime() time.Duration { return p.userTime() diff --git a/src/os/exec/dot_test.go b/src/os/exec/dot_test.go index ed4bad23b1cca2..b95639e6c82068 100644 --- a/src/os/exec/dot_test.go +++ b/src/os/exec/dot_test.go @@ -38,8 +38,7 @@ func TestLookPath(t *testing.T) { if err := os.WriteFile(filepath.Join(tmpDir, executable), []byte{1, 2, 3}, 0777); err != nil { t.Fatal(err) } - chdir(t, tmpDir) - t.Setenv("PWD", tmpDir) + t.Chdir(tmpDir) t.Logf(". is %#q", tmpDir) origPath := os.Getenv(pathVar) @@ -178,4 +177,48 @@ func TestLookPath(t *testing.T) { } } }) + + checker := func(test string) func(t *testing.T) { + return func(t *testing.T) { + t.Helper() + t.Logf("PATH=%s", os.Getenv("PATH")) + p, err := LookPath(test) + if err == nil { + t.Errorf("%q: error expected, got nil", test) + } + if p != "" { + t.Errorf("%q: path returned should be \"\". Got %q", test, p) + } + } + } + + // Reference behavior for the next test + t.Run(pathVar+"=$OTHER2", func(t *testing.T) { + t.Run("empty", checker("")) + t.Run("dot", checker(".")) + t.Run("dotdot1", checker("abc/..")) + t.Run("dotdot2", checker("..")) + }) + + // Test the behavior when PATH contains an executable file which is not a directory + t.Run(pathVar+"=exe", func(t *testing.T) { + // Inject an executable file (not a directory) in PATH. + // Use our own binary os.Args[0]. + t.Setenv(pathVar, testenv.Executable(t)) + t.Run("empty", checker("")) + t.Run("dot", checker(".")) + t.Run("dotdot1", checker("abc/..")) + t.Run("dotdot2", checker("..")) + }) + + // Test the behavior when PATH contains an executable file which is not a directory + t.Run(pathVar+"=exe/xx", func(t *testing.T) { + // Inject an executable file (not a directory) in PATH. + // Use our own binary os.Args[0]. + t.Setenv(pathVar, filepath.Join(testenv.Executable(t), "xx")) + t.Run("empty", checker("")) + t.Run("dot", checker(".")) + t.Run("dotdot1", checker("abc/..")) + t.Run("dotdot2", checker("..")) + }) } diff --git a/src/os/exec/exec.go b/src/os/exec/exec.go index da9f68fe28f14b..38354a5244322f 100644 --- a/src/os/exec/exec.go +++ b/src/os/exec/exec.go @@ -17,7 +17,7 @@ // // Note that the examples in this package assume a Unix system. // They may not run on Windows, and they do not run in the Go Playground -// used by golang.org and godoc.org. +// used by go.dev and pkg.go.dev. // // # Executables in the current directory // @@ -166,11 +166,27 @@ type Cmd struct { // value in the slice for each duplicate key is used. // As a special case on Windows, SYSTEMROOT is always added if // missing and not explicitly set to the empty string. + // + // See also the Dir field, which may set PWD in the environment. Env []string // Dir specifies the working directory of the command. // If Dir is the empty string, Run runs the command in the // calling process's current directory. + // + // On Unix systems, the value of Dir also determines the + // child process's PWD environment variable if not otherwise + // specified. A Unix process represents its working directory + // not by name but as an implicit reference to a node in the + // file tree. So, if the child process obtains its working + // directory by calling a function such as C's getcwd, which + // computes the canonical name by walking up the file tree, it + // will not recover the original value of Dir if that value + // was an alias involving symbolic links. However, if the + // child process calls Go's [os.Getwd] or GNU C's + // get_current_dir_name, and the value of PWD is an alias for + // the current directory, those functions will return the + // value of PWD, which matches the value of Dir. Dir string // Stdin specifies the process's standard input. @@ -984,7 +1000,9 @@ func (c *Cmd) awaitGoroutines(timer *time.Timer) error { // Output runs the command and returns its standard output. // Any returned error will usually be of type [*ExitError]. -// If c.Stderr was nil, Output populates [ExitError.Stderr]. +// If c.Stderr was nil and the returned error is of type +// [*ExitError], Output populates the Stderr field of the +// returned error. func (c *Cmd) Output() ([]byte, error) { if c.Stdout != nil { return nil, errors.New("exec: Stdout already set") @@ -1310,3 +1328,13 @@ func addCriticalEnv(env []string) []string { // Code should use errors.Is(err, ErrDot), not err == ErrDot, // to test whether a returned error err is due to this condition. var ErrDot = errors.New("cannot run executable found relative to current directory") + +// validateLookPath excludes paths that can't be valid +// executable names. See issue #74466 and CVE-2025-47906. +func validateLookPath(s string) error { + switch s { + case "", ".", "..": + return ErrNotFound + } + return nil +} diff --git a/src/os/exec/exec_posix_test.go b/src/os/exec/exec_posix_test.go index 45604203dd0c46..0711fac90e7c14 100644 --- a/src/os/exec/exec_posix_test.go +++ b/src/os/exec/exec_posix_test.go @@ -11,12 +11,15 @@ import ( "internal/testenv" "io" "os" + "os/exec" + "os/signal" "os/user" "path/filepath" "runtime" "slices" "strconv" "strings" + "sync" "syscall" "testing" "time" @@ -24,6 +27,7 @@ import ( func init() { registerHelperCommand("pwd", cmdPwd) + registerHelperCommand("signaltest", cmdSignalTest) } func cmdPwd(...string) { @@ -161,7 +165,6 @@ func TestImplicitPWD(t *testing.T) { } for _, tc := range cases { - tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -238,7 +241,6 @@ func TestExplicitPWD(t *testing.T) { // contain symlinks preserved from the PWD value in the test's environment. } for _, tc := range cases { - tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -274,3 +276,55 @@ func TestExplicitPWD(t *testing.T) { }) } } + +// Issue 71828. +func TestSIGCHLD(t *testing.T) { + cmd := helperCommand(t, "signaltest") + out, err := cmd.CombinedOutput() + t.Logf("%s", out) + if err != nil { + t.Error(err) + } +} + +// cmdSignaltest is for TestSIGCHLD. +// This runs in a separate process because the bug only happened +// the first time that a child process was started. +func cmdSignalTest(...string) { + chSig := make(chan os.Signal, 1) + signal.Notify(chSig, syscall.SIGCHLD) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + c := 0 + for range chSig { + c++ + fmt.Printf("SIGCHLD %d\n", c) + if c > 1 { + fmt.Println("too many SIGCHLD signals") + os.Exit(1) + } + } + }() + defer func() { + signal.Reset(syscall.SIGCHLD) + close(chSig) + wg.Wait() + }() + + exe, err := os.Executable() + if err != nil { + fmt.Printf("os.Executable failed: %v\n", err) + os.Exit(1) + } + + cmd := exec.Command(exe, "hang", "200ms") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Printf("failed to run child process: %v\n", err) + os.Exit(1) + } +} diff --git a/src/os/exec/exec_test.go b/src/os/exec/exec_test.go index a0bb89e203ddf1..1decebdc222d23 100644 --- a/src/os/exec/exec_test.go +++ b/src/os/exec/exec_test.go @@ -159,64 +159,16 @@ func helperCommandContext(t *testing.T, ctx context.Context, name string, args . helperCommandUsed.LoadOrStore(name, true) t.Helper() - testenv.MustHaveExec(t) - + exe := testenv.Executable(t) cs := append([]string{name}, args...) if ctx != nil { - cmd = exec.CommandContext(ctx, exePath(t), cs...) + cmd = exec.CommandContext(ctx, exe, cs...) } else { - cmd = exec.Command(exePath(t), cs...) + cmd = exec.Command(exe, cs...) } return cmd } -// exePath returns the path to the running executable. -func exePath(t testing.TB) string { - exeOnce.Do(func() { - // Use os.Executable instead of os.Args[0] in case the caller modifies - // cmd.Dir: if the test binary is invoked like "./exec.test", it should - // not fail spuriously. - exeOnce.path, exeOnce.err = os.Executable() - }) - - if exeOnce.err != nil { - if t == nil { - panic(exeOnce.err) - } - t.Fatal(exeOnce.err) - } - - return exeOnce.path -} - -var exeOnce struct { - path string - err error - sync.Once -} - -func chdir(t *testing.T, dir string) { - t.Helper() - - prev, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - if err := os.Chdir(dir); err != nil { - t.Fatal(err) - } - t.Logf("Chdir(%#q)", dir) - - t.Cleanup(func() { - if err := os.Chdir(prev); err != nil { - // Couldn't chdir back to the original working directory. - // panic instead of t.Fatal so that we don't run other tests - // in an unexpected location. - panic("couldn't restore working directory: " + err.Error()) - } - }) -} - var helperCommandUsed sync.Map var helperCommands = map[string]func(...string){ @@ -731,7 +683,10 @@ func TestExtraFiles(t *testing.T) { // This test runs with cgo disabled. External linking needs cgo, so // it doesn't work if external linking is required. - testenv.MustInternalLink(t, false) + // + // N.B. go build below explictly doesn't pass through + // -asan/-msan/-race, so we don't care about those. + testenv.MustInternalLink(t, testenv.NoSpecialBuildTypes) if runtime.GOOS == "windows" { t.Skipf("skipping test on %q", runtime.GOOS) @@ -1201,7 +1156,7 @@ func cmdHang(args ...string) { pid := os.Getpid() if *subsleep != 0 { - cmd := exec.Command(exePath(nil), "hang", subsleep.String(), "-read=true", "-probe="+probe.String()) + cmd := exec.Command(testenv.Executable(nil), "hang", subsleep.String(), "-read=true", "-probe="+probe.String()) cmd.Stdin = os.Stdin cmd.Stderr = os.Stderr out, err := cmd.StdoutPipe() @@ -1423,8 +1378,8 @@ func TestWaitInterrupt(t *testing.T) { // The child process should be reported as failed, // and the grandchild will exit (or die by SIGPIPE) once the // stderr pipe is closed. - if ee := new(*exec.ExitError); !errors.As(err, ee) { - t.Errorf("Wait error = %v; want %T", err, *ee) + if ee, ok := errors.AsType[*exec.ExitError](err); !ok { + t.Errorf("Wait error = %v; want %T", err, ee) } }) @@ -1468,8 +1423,8 @@ func TestWaitInterrupt(t *testing.T) { // This command ignores SIGINT, sleeping until it is killed. // Wait should return the usual error for a killed process. - if ee := new(*exec.ExitError); !errors.As(err, ee) { - t.Errorf("Wait error = %v; want %T", err, *ee) + if ee, ok := errors.AsType[*exec.ExitError](err); !ok { + t.Errorf("Wait error = %v; want %T", err, ee) } }) @@ -1516,7 +1471,7 @@ func TestWaitInterrupt(t *testing.T) { t.Logf("stderr:\n%s", cmd.Stderr) t.Logf("[%d] %v", cmd.Process.Pid, err) - if ee := new(*exec.ExitError); !errors.As(err, ee) { + if _, ok := errors.AsType[*exec.ExitError](err); !ok { t.Errorf("Wait error = %v; want %v", err, ctx.Err()) } diff --git a/src/os/exec/lp_linux_test.go b/src/os/exec/lp_linux_test.go index a7f9aa24b8cdbc..1436763038a73d 100644 --- a/src/os/exec/lp_linux_test.go +++ b/src/os/exec/lp_linux_test.go @@ -19,7 +19,7 @@ func TestFindExecutableVsNoexec(t *testing.T) { t.Parallel() // This test case relies on faccessat2(2) syscall, which appeared in Linux v5.8. - if major, minor := unix.KernelVersion(); major < 5 || (major == 5 && minor < 8) { + if !unix.KernelVersionGE(5, 8) { t.Skip("requires Linux kernel v5.8 with faccessat2(2) syscall") } diff --git a/src/os/exec/lp_plan9.go b/src/os/exec/lp_plan9.go index 87359b3551d32f..f713a6905cfbdc 100644 --- a/src/os/exec/lp_plan9.go +++ b/src/os/exec/lp_plan9.go @@ -36,6 +36,10 @@ func findExecutable(file string) error { // As of Go 1.19, LookPath will instead return that path along with an error satisfying // [errors.Is](err, [ErrDot]). See the package documentation for more details. func LookPath(file string) (string, error) { + if err := validateLookPath(filepath.Clean(file)); err != nil { + return "", &Error{file, err} + } + // skip the path lookup for these prefixes skip := []string{"/", "#", "./", "../"} diff --git a/src/os/exec/lp_unix.go b/src/os/exec/lp_unix.go index 8617d45e983e6e..e5fddbafe21b94 100644 --- a/src/os/exec/lp_unix.go +++ b/src/os/exec/lp_unix.go @@ -54,6 +54,10 @@ func LookPath(file string) (string, error) { // (only bypass the path if file begins with / or ./ or ../) // but that would not match all the Unix shells. + if err := validateLookPath(file); err != nil { + return "", &Error{file, err} + } + if strings.Contains(file, "/") { err := findExecutable(file) if err == nil { diff --git a/src/os/exec/lp_unix_test.go b/src/os/exec/lp_unix_test.go index 1503ddae93309e..ea7ec11cc70758 100644 --- a/src/os/exec/lp_unix_test.go +++ b/src/os/exec/lp_unix_test.go @@ -16,7 +16,7 @@ func TestLookPathUnixEmptyPath(t *testing.T) { // Not parallel: uses Chdir and Setenv. tmp := t.TempDir() - chdir(t, tmp) + t.Chdir(tmp) f, err := os.OpenFile("exec_me", os.O_CREATE|os.O_EXCL, 0700) if err != nil { diff --git a/src/os/exec/lp_windows.go b/src/os/exec/lp_windows.go index 0e058d41b0c11f..74537dec687c81 100644 --- a/src/os/exec/lp_windows.go +++ b/src/os/exec/lp_windows.go @@ -10,7 +10,6 @@ import ( "os" "path/filepath" "strings" - "syscall" ) // ErrNotFound is the error resulting if a path search failed to find an executable file. @@ -68,6 +67,10 @@ func findExecutable(file string, exts []string) (string, error) { // As of Go 1.19, LookPath will instead return that path along with an error satisfying // [errors.Is](err, [ErrDot]). See the package documentation for more details. func LookPath(file string) (string, error) { + if err := validateLookPath(file); err != nil { + return "", &Error{file, err} + } + return lookPath(file, pathExt()) } @@ -81,6 +84,10 @@ func LookPath(file string) (string, error) { // "C:\foo\example.com" would be returned as-is even if the // program is actually "C:\foo\example.com.exe". func lookExtensions(path, dir string) (string, error) { + if err := validateLookPath(path); err != nil { + return "", &Error{path, err} + } + if filepath.Base(path) == path { path = "." + string(filepath.Separator) + path } @@ -116,7 +123,7 @@ func pathExt() []string { var exts []string x := os.Getenv(`PATHEXT`) if x != "" { - for _, e := range strings.Split(strings.ToLower(x), `;`) { + for e := range strings.SplitSeq(strings.ToLower(x), `;`) { if e == "" { continue } @@ -154,7 +161,7 @@ func lookPath(file string, exts []string) (string, error) { dotf string dotErr error ) - if _, found := syscall.Getenv("NoDefaultCurrentDirectoryInExePath"); !found { + if _, found := os.LookupEnv("NoDefaultCurrentDirectoryInExePath"); !found { if f, err := findExecutable(filepath.Join(".", file), exts); err == nil { if execerrdot.Value() == "0" { execerrdot.IncNonDefault() diff --git a/src/os/exec/lp_windows_test.go b/src/os/exec/lp_windows_test.go index a92a29799f35f1..01eda04c7509fd 100644 --- a/src/os/exec/lp_windows_test.go +++ b/src/os/exec/lp_windows_test.go @@ -25,13 +25,8 @@ func init() { registerHelperCommand("printpath", cmdPrintPath) } -func cmdPrintPath(args ...string) { - exe, err := os.Executable() - if err != nil { - fmt.Fprintf(os.Stderr, "Executable: %v\n", err) - os.Exit(1) - } - fmt.Println(exe) +func cmdPrintPath(_ ...string) { + fmt.Println(testenv.Executable(nil)) } // makePATH returns a PATH variable referring to the @@ -82,7 +77,7 @@ func installProgs(t *testing.T, root string, files []string) { // (We use a copy instead of just a symlink to ensure that os.Executable // always reports an unambiguous path, regardless of how it is implemented.) func installExe(t *testing.T, dstPath string) { - src, err := os.Open(exePath(t)) + src, err := os.Open(testenv.Executable(t)) if err != nil { t.Fatal(err) } @@ -319,7 +314,7 @@ func TestLookPathWindows(t *testing.T) { t.Setenv("PATH", pathVar) t.Logf("set PATH=%s", pathVar) - chdir(t, root) + t.Chdir(root) if !testing.Short() && !(tt.skipCmdExeCheck || errors.Is(tt.wantErr, exec.ErrDot)) { // Check that cmd.exe, which is our source of ground truth, @@ -554,7 +549,7 @@ func TestCommand(t *testing.T) { t.Setenv("PATH", pathVar) t.Logf("set PATH=%s", pathVar) - chdir(t, root) + t.Chdir(root) cmd := exec.Command(tt.arg0, "printpath") cmd.Dir = filepath.Join(root, tt.dir) diff --git a/src/os/exec/read3.go b/src/os/exec/read3.go index 8327d73e514bea..3ccf1cacc83f2d 100644 --- a/src/os/exec/read3.go +++ b/src/os/exec/read3.go @@ -20,7 +20,6 @@ import ( "os/exec" "os/exec/internal/fdtest" "runtime" - "strings" ) func main() { @@ -81,7 +80,7 @@ func main() { cmd := exec.Command(ofcmd, args...) out, err := cmd.CombinedOutput() if err != nil { - fmt.Fprintf(os.Stderr, "%s failed: %v\n", strings.Join(cmd.Args, " "), err) + fmt.Fprintf(os.Stderr, "%#q failed: %v\n", cmd, err) } fmt.Printf("%s", out) os.Exit(1) diff --git a/src/os/exec_linux.go b/src/os/exec_linux.go index b47c6cb191986e..aaa022cb96d028 100644 --- a/src/os/exec_linux.go +++ b/src/os/exec_linux.go @@ -8,6 +8,6 @@ import ( "syscall" ) -func (p *Process) closeHandle() { - syscall.Close(int(p.handle)) +func (ph *processHandle) closeHandle() { + syscall.Close(int(ph.handle)) } diff --git a/src/os/exec_nohandle.go b/src/os/exec_nohandle.go index d06f4091c3eb77..0f70d21ccd8607 100644 --- a/src/os/exec_nohandle.go +++ b/src/os/exec_nohandle.go @@ -6,4 +6,6 @@ package os -func (p *Process) closeHandle() {} +func (ph *processHandle) closeHandle() { + panic("internal error: unexpected call to closeHandle") +} diff --git a/src/os/exec_nohandle_test.go b/src/os/exec_nohandle_test.go new file mode 100644 index 00000000000000..73a8ada7eb6bde --- /dev/null +++ b/src/os/exec_nohandle_test.go @@ -0,0 +1,40 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !linux && !windows + +package os_test + +import ( + "internal/testenv" + . "os" + "testing" + "time" +) + +func TestProcessWithHandleUnsupported(t *testing.T) { + const envVar = "OSTEST_PROCESS_WITH_HANDLE" + if Getenv(envVar) != "" { + time.Sleep(1 * time.Minute) + return + } + + cmd := testenv.CommandContext(t, t.Context(), testenv.Executable(t), "-test.run=^"+t.Name()+"$") + cmd = testenv.CleanCmdEnv(cmd) + cmd.Env = append(cmd.Env, envVar+"=1") + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { + cmd.Process.Kill() + cmd.Wait() + }() + + err := cmd.Process.WithHandle(func(handle uintptr) { + t.Errorf("WithHandle: callback called unexpectedly with handle=%v", handle) + }) + if err != ErrNoHandle { + t.Fatalf("WithHandle: got error %v, want %v", err, ErrNoHandle) + } +} diff --git a/src/os/exec_plan9.go b/src/os/exec_plan9.go index bc7a9cdcbc5f74..a3d363b344061f 100644 --- a/src/os/exec_plan9.go +++ b/src/os/exec_plan9.go @@ -6,7 +6,6 @@ package os import ( "internal/itoa" - "runtime" "syscall" "time" ) @@ -81,7 +80,7 @@ func (p *Process) wait() (ps *ProcessState, err error) { return nil, NewSyscallError("wait", err) } - p.pidDeactivate(statusDone) + p.doRelease(statusDone) ps = &ProcessState{ pid: waitmsg.Pid, status: &waitmsg, @@ -89,15 +88,8 @@ func (p *Process) wait() (ps *ProcessState, err error) { return ps, nil } -func (p *Process) release() error { - p.Pid = -1 - - // Just mark the PID unusable. - p.pidDeactivate(statusReleased) - - // no need for a finalizer anymore - runtime.SetFinalizer(p, nil) - return nil +func (p *Process) withHandle(_ func(handle uintptr)) error { + return ErrNoHandle } func findProcess(pid int) (p *Process, err error) { diff --git a/src/os/exec_posix.go b/src/os/exec_posix.go index cba2e151673aba..6b6977ab785591 100644 --- a/src/os/exec_posix.go +++ b/src/os/exec_posix.go @@ -35,10 +35,11 @@ func startProcess(name string, argv []string, attr *ProcAttr) (p *Process, err e } } + attrSys, shouldDupPidfd := ensurePidfd(attr.Sys) sysattr := &syscall.ProcAttr{ Dir: attr.Dir, Env: attr.Env, - Sys: ensurePidfd(attr.Sys), + Sys: attrSys, } if sysattr.Env == nil { sysattr.Env, err = execenv.Default(sysattr.Sys) @@ -63,7 +64,7 @@ func startProcess(name string, argv []string, attr *ProcAttr) (p *Process, err e // For Windows, syscall.StartProcess above already returned a process handle. if runtime.GOOS != "windows" { var ok bool - h, ok = getPidfd(sysattr.Sys) + h, ok = getPidfd(sysattr.Sys, shouldDupPidfd) if !ok { return newPIDProcess(pid), nil } @@ -76,6 +77,23 @@ func (p *Process) kill() error { return p.Signal(Kill) } +func (p *Process) withHandle(f func(handle uintptr)) error { + if p.handle == nil { + return ErrNoHandle + } + handle, status := p.handleTransientAcquire() + switch status { + case statusDone: + return ErrProcessDone + case statusReleased: + return errProcessReleased + } + defer p.handleTransientRelease() + f(handle) + + return nil +} + // ProcessState stores information about a process, as reported by Wait. type ProcessState struct { pid int // The process's id. diff --git a/src/os/exec_unix.go b/src/os/exec_unix.go index ba6146ada11b8e..d2ceb0ceb5aeeb 100644 --- a/src/os/exec_unix.go +++ b/src/os/exec_unix.go @@ -8,7 +8,6 @@ package os import ( "errors" - "runtime" "syscall" "time" ) @@ -21,15 +20,12 @@ const ( func (p *Process) wait() (ps *ProcessState, err error) { // Which type of Process do we have? - switch p.mode { - case modeHandle: + if p.handle != nil { // pidfd return p.pidfdWait() - case modePID: + } else { // Regular PID return p.pidWait() - default: - panic("unreachable") } } @@ -53,7 +49,7 @@ func (p *Process) pidWait() (*ProcessState, error) { if ready { // Mark the process done now, before the call to Wait4, // so that Process.pidSignal will not send a signal. - p.pidDeactivate(statusDone) + p.doRelease(statusDone) // Acquire a write lock on sigMu to wait for any // active call to the signal method to complete. p.sigMu.Lock() @@ -63,19 +59,14 @@ func (p *Process) pidWait() (*ProcessState, error) { var ( status syscall.WaitStatus rusage syscall.Rusage - pid1 int - e error ) - for { - pid1, e = syscall.Wait4(p.Pid, &status, 0, &rusage) - if e != syscall.EINTR { - break - } - } - if e != nil { - return nil, NewSyscallError("wait", e) + pid1, err := ignoringEINTR2(func() (int, error) { + return syscall.Wait4(p.Pid, &status, 0, &rusage) + }) + if err != nil { + return nil, NewSyscallError("wait", err) } - p.pidDeactivate(statusDone) + p.doRelease(statusDone) return &ProcessState{ pid: pid1, status: status, @@ -90,21 +81,18 @@ func (p *Process) signal(sig Signal) error { } // Which type of Process do we have? - switch p.mode { - case modeHandle: + if p.handle != nil { // pidfd return p.pidfdSendSignal(s) - case modePID: + } else { // Regular PID return p.pidSignal(s) - default: - panic("unreachable") } } func (p *Process) pidSignal(s syscall.Signal) error { if p.Pid == pidReleased { - return errors.New("os: process already released") + return errProcessReleased } if p.Pid == pidUnset { return errors.New("os: process not initialized") @@ -117,7 +105,7 @@ func (p *Process) pidSignal(s syscall.Signal) error { case statusDone: return ErrProcessDone case statusReleased: - return errors.New("os: process already released") + return errProcessReleased } return convertESRCH(syscall.Kill(p.Pid, s)) @@ -130,29 +118,6 @@ func convertESRCH(err error) error { return err } -func (p *Process) release() error { - // We clear the Pid field only for API compatibility. On Unix, Release - // has always set Pid to -1. Internally, the implementation relies - // solely on statusReleased to determine that the Process is released. - p.Pid = pidReleased - - switch p.mode { - case modeHandle: - // Drop the Process' reference and mark handle unusable for - // future calls. - // - // Ignore the return value: we don't care if this was a no-op - // racing with Wait, or a double Release. - p.handlePersistentRelease(statusReleased) - case modePID: - // Just mark the PID unusable. - p.pidDeactivate(statusReleased) - } - // no need for a finalizer anymore - runtime.SetFinalizer(p, nil) - return nil -} - func findProcess(pid int) (p *Process, err error) { h, err := pidfdFind(pid) if err == ErrProcessDone { diff --git a/src/os/exec_windows.go b/src/os/exec_windows.go index ab2dae1d718548..68c7064d2dd0e5 100644 --- a/src/os/exec_windows.go +++ b/src/os/exec_windows.go @@ -12,7 +12,7 @@ import ( "time" ) -// Note that Process.mode is always modeHandle because Windows always requires +// Note that Process.handle is never nil because Windows always requires // a handle. A manually-created Process literal is not valid. func (p *Process) wait() (ps *ProcessState, err error) { @@ -44,7 +44,11 @@ func (p *Process) wait() (ps *ProcessState, err error) { if e != nil { return nil, NewSyscallError("GetProcessTimes", e) } - defer p.Release() + + // For compatibility we use statusReleased here rather + // than statusDone. + p.doRelease(statusReleased) + return &ProcessState{p.Pid, syscall.WaitStatus{ExitCode: ec}, &u}, nil } @@ -73,23 +77,8 @@ func (p *Process) signal(sig Signal) error { return syscall.Errno(syscall.EWINDOWS) } -func (p *Process) release() error { - // Drop the Process' reference and mark handle unusable for - // future calls. - // - // The API on Windows expects EINVAL if Release is called multiple - // times. - if old := p.handlePersistentRelease(statusReleased); old == statusReleased { - return syscall.EINVAL - } - - // no need for a finalizer anymore - runtime.SetFinalizer(p, nil) - return nil -} - -func (p *Process) closeHandle() { - syscall.CloseHandle(syscall.Handle(p.handle)) +func (ph *processHandle) closeHandle() { + syscall.CloseHandle(syscall.Handle(ph.handle)) } func findProcess(pid int) (p *Process, err error) { diff --git a/src/os/exec_windows_test.go b/src/os/exec_windows_test.go index f8ed4cdf1c9266..272395ca929837 100644 --- a/src/os/exec_windows_test.go +++ b/src/os/exec_windows_test.go @@ -12,7 +12,9 @@ import ( . "os" "path/filepath" "sync" + "syscall" "testing" + "time" ) func TestRemoveAllWithExecutedProcess(t *testing.T) { @@ -81,3 +83,39 @@ func TestRemoveAllWithExecutedProcess(t *testing.T) { } wg.Wait() } + +func TestProcessWithHandleWindows(t *testing.T) { + const envVar = "OSTEST_PROCESS_WITH_HANDLE" + if Getenv(envVar) != "" { + time.Sleep(1 * time.Minute) + return + } + + cmd := testenv.CommandContext(t, t.Context(), testenv.Executable(t), "-test.run=^"+t.Name()+"$") + cmd = testenv.CleanCmdEnv(cmd) + cmd.Env = append(cmd.Env, envVar+"=1") + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { + cmd.Process.Kill() + cmd.Wait() + }() + + called := false + err := cmd.Process.WithHandle(func(handle uintptr) { + called = true + // Check the handle is valid. + var u syscall.Rusage + e := syscall.GetProcessTimes(syscall.Handle(handle), &u.CreationTime, &u.ExitTime, &u.KernelTime, &u.UserTime) + if e != nil { + t.Errorf("Using process handle failed: %v", NewSyscallError("GetProcessTimes", e)) + } + }) + if err != nil { + t.Fatalf("WithHandle: got error %v, want nil", err) + } + if !called { + t.Fatal("WithHandle did not call the callback function") + } +} diff --git a/src/os/executable_dragonfly.go b/src/os/executable_dragonfly.go index 19c2ae890f9a53..939c6f6ebbac21 100644 --- a/src/os/executable_dragonfly.go +++ b/src/os/executable_dragonfly.go @@ -10,3 +10,5 @@ const ( _KERN_PROC = 14 _KERN_PROC_PATHNAME = 9 ) + +var executableMIB = [4]int32{_CTL_KERN, _KERN_PROC, _KERN_PROC_PATHNAME, -1} diff --git a/src/os/executable_freebsd.go b/src/os/executable_freebsd.go index 95f1a93cb9512c..da40fcb32ab913 100644 --- a/src/os/executable_freebsd.go +++ b/src/os/executable_freebsd.go @@ -10,3 +10,5 @@ const ( _KERN_PROC = 14 _KERN_PROC_PATHNAME = 12 ) + +var executableMIB = [4]int32{_CTL_KERN, _KERN_PROC, _KERN_PROC_PATHNAME, -1} diff --git a/src/os/executable_netbsd.go b/src/os/executable_netbsd.go new file mode 100644 index 00000000000000..fd075390e6c763 --- /dev/null +++ b/src/os/executable_netbsd.go @@ -0,0 +1,14 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +// From NetBSD's +const ( + _CTL_KERN = 1 + _KERN_PROC_ARGS = 48 + _KERN_PROC_PATHNAME = 5 +) + +var executableMIB = [4]int32{_CTL_KERN, _KERN_PROC_ARGS, -1, _KERN_PROC_PATHNAME} diff --git a/src/os/executable_procfs.go b/src/os/executable_procfs.go index 6a2cd10be7c402..a52631c0bb2fa1 100644 --- a/src/os/executable_procfs.go +++ b/src/os/executable_procfs.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build linux || netbsd +//go:build linux package os @@ -19,8 +19,6 @@ func executable() (string, error) { return "", errors.New("Executable not implemented for " + runtime.GOOS) case "linux", "android": procfn = "/proc/self/exe" - case "netbsd": - procfn = "/proc/curproc/exe" } path, err := Readlink(procfn) diff --git a/src/os/executable_sysctl.go b/src/os/executable_sysctl.go index 3c2aeacf7da53f..8b52e92c413cb1 100644 --- a/src/os/executable_sysctl.go +++ b/src/os/executable_sysctl.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build freebsd || dragonfly +//go:build freebsd || dragonfly || netbsd package os @@ -12,11 +12,9 @@ import ( ) func executable() (string, error) { - mib := [4]int32{_CTL_KERN, _KERN_PROC, _KERN_PROC_PATHNAME, -1} - n := uintptr(0) // get length - _, _, err := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0) + _, _, err := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&executableMIB[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0) if err != 0 { return "", err } @@ -24,7 +22,7 @@ func executable() (string, error) { return "", nil } buf := make([]byte, n) - _, _, err = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0) + _, _, err = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&executableMIB[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0) if err != 0 { return "", err } diff --git a/src/os/executable_test.go b/src/os/executable_test.go index 98b72d7d5e42d0..1770843c7af8e7 100644 --- a/src/os/executable_test.go +++ b/src/os/executable_test.go @@ -13,16 +13,30 @@ import ( "testing" ) -const executable_EnvVar = "OSTEST_OUTPUT_EXECPATH" - func TestExecutable(t *testing.T) { - testenv.MustHaveExec(t) - t.Parallel() + const helperEnvVar = "OSTEST_OUTPUT_EXECPATH" - ep, err := os.Executable() - if err != nil { - t.Fatalf("Executable failed: %v", err) + if os.Getenv(helperEnvVar) != "" { + // First chdir to another path. + dir := "/" + if runtime.GOOS == "windows" { + cwd, err := os.Getwd() + if err != nil { + panic(err) + } + dir = filepath.VolumeName(cwd) + } + os.Chdir(dir) + if ep, err := os.Executable(); err != nil { + fmt.Fprint(os.Stderr, "ERROR: ", err) + } else { + fmt.Fprint(os.Stderr, ep) + } + os.Exit(0) } + + t.Parallel() + ep := testenv.Executable(t) // we want fn to be of the form "dir/prog" dir := filepath.Dir(filepath.Dir(ep)) fn, err := filepath.Rel(dir, ep) @@ -30,7 +44,7 @@ func TestExecutable(t *testing.T) { t.Fatalf("filepath.Rel: %v", err) } - cmd := testenv.Command(t, fn, "-test.run=^$") + cmd := testenv.Command(t, fn, "-test.run=^"+t.Name()+"$") // make child start with a relative program path cmd.Dir = dir cmd.Path = fn @@ -41,7 +55,7 @@ func TestExecutable(t *testing.T) { // get real path of the executable without influenced by argv[0]. cmd.Args[0] = "-" } - cmd.Env = append(cmd.Environ(), fmt.Sprintf("%s=1", executable_EnvVar)) + cmd.Env = append(cmd.Environ(), fmt.Sprintf("%s=1", helperEnvVar)) out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("exec(self) failed: %v", err) @@ -67,27 +81,6 @@ func sameFile(fn1, fn2 string) bool { return os.SameFile(fi1, fi2) } -func init() { - if e := os.Getenv(executable_EnvVar); e != "" { - // first chdir to another path - dir := "/" - if runtime.GOOS == "windows" { - cwd, err := os.Getwd() - if err != nil { - panic(err) - } - dir = filepath.VolumeName(cwd) - } - os.Chdir(dir) - if ep, err := os.Executable(); err != nil { - fmt.Fprint(os.Stderr, "ERROR: ", err) - } else { - fmt.Fprint(os.Stderr, ep) - } - os.Exit(0) - } -} - func TestExecutableDeleted(t *testing.T) { testenv.MustHaveGoBuild(t) switch runtime.GOOS { diff --git a/src/os/export_freebsd_test.go b/src/os/export_freebsd_test.go new file mode 100644 index 00000000000000..56bfcc6c715aec --- /dev/null +++ b/src/os/export_freebsd_test.go @@ -0,0 +1,9 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +var ( + PollCopyFileRangeP = &pollCopyFileRange +) diff --git a/src/os/export_linux_test.go b/src/os/export_linux_test.go index 12434cb42671ef..4ace32bb5b2c0b 100644 --- a/src/os/export_linux_test.go +++ b/src/os/export_linux_test.go @@ -14,5 +14,5 @@ var ( const StatusDone = statusDone func (p *Process) Status() processStatus { - return processStatus(p.state.Load() & processStatusMask) + return processStatus(p.state.Load()) } diff --git a/src/os/export_test.go b/src/os/export_test.go index dc7caae267d704..93b10898e0b4ba 100644 --- a/src/os/export_test.go +++ b/src/os/export_test.go @@ -9,9 +9,10 @@ package os var Atime = atime var LstatP = &lstat var ErrWriteAtInAppendMode = errWriteAtInAppendMode -var TestingForceReadDirLstat = &testingForceReadDirLstat var ErrPatternHasSeparator = errPatternHasSeparator func init() { checkWrapErr = true } + +var ExportReadFileContents = readFileContents diff --git a/src/os/export_windows_test.go b/src/os/export_windows_test.go index aefbe4033e3738..5b939b4c25f307 100644 --- a/src/os/export_windows_test.go +++ b/src/os/export_windows_test.go @@ -11,4 +11,5 @@ var ( NewConsoleFile = newConsoleFile CommandLineToArgv = commandLineToArgv AllowReadDirFileID = &allowReadDirFileID + SplitPath = splitPath ) diff --git a/src/os/fifo_test.go b/src/os/fifo_test.go index 3b7e5eac197c25..111dd2aaeb616f 100644 --- a/src/os/fifo_test.go +++ b/src/os/fifo_test.go @@ -161,10 +161,7 @@ func TestNonPollable(t *testing.T) { // Issue 60211. func TestOpenFileNonBlocking(t *testing.T) { - exe, err := os.Executable() - if err != nil { - t.Skipf("can't find executable: %v", err) - } + exe := testenv.Executable(t) f, err := os.OpenFile(exe, os.O_RDONLY|syscall.O_NONBLOCK, 0666) if err != nil { t.Fatal(err) diff --git a/src/os/file.go b/src/os/file.go index 541b4320819c3f..66269c199e7d95 100644 --- a/src/os/file.go +++ b/src/os/file.go @@ -51,6 +51,7 @@ import ( "io" "io/fs" "runtime" + "slices" "syscall" "time" "unsafe" @@ -114,6 +115,25 @@ func (e *LinkError) Unwrap() error { return e.Err } +// NewFile returns a new [File] with the given file descriptor and name. +// The returned value will be nil if fd is not a valid file descriptor. +// +// NewFile's behavior differs on some platforms: +// +// - On Unix, if fd is in non-blocking mode, NewFile will attempt to return a pollable file. +// - On Windows, if fd is opened for asynchronous I/O (that is, [syscall.FILE_FLAG_OVERLAPPED] +// has been specified in the [syscall.CreateFile] call), NewFile will attempt to return a pollable +// file by associating fd with the Go runtime I/O completion port. +// The I/O operations will be performed synchronously if the association fails. +// +// Only pollable files support [File.SetDeadline], [File.SetReadDeadline], and [File.SetWriteDeadline]. +// +// After passing it to NewFile, fd may become invalid under the same conditions described +// in the comments of [File.Fd], and the same constraints apply. +func NewFile(fd uintptr, name string) *File { + return newFileFromNewFile(fd, name) +} + // Read reads up to len(b) bytes from the File and stores them in b. // It returns the number of bytes read and any error encountered. // At end of file, Read returns 0, io.EOF. @@ -215,7 +235,7 @@ var errWriteAtInAppendMode = errors.New("os: invalid use of WriteAt on file open // It returns the number of bytes written and an error, if any. // WriteAt returns a non-nil error when n != len(b). // -// If file was opened with the O_APPEND flag, WriteAt returns an error. +// If file was opened with the [O_APPEND] flag, WriteAt returns an error. func (f *File) WriteAt(b []byte, off int64) (n int, err error) { if err := f.checkValid("write"); err != nil { return 0, err @@ -279,7 +299,7 @@ func genericWriteTo(f *File, w io.Writer) (int64, error) { // according to whence: 0 means relative to the origin of the file, 1 means // relative to the current offset, and 2 means relative to the end. // It returns the new offset and an error, if any. -// The behavior of Seek on a file opened with O_APPEND is not specified. +// The behavior of Seek on a file opened with [O_APPEND] is not specified. func (f *File) Seek(offset int64, whence int) (ret int64, err error) { if err := f.checkValid("seek"); err != nil { return 0, err @@ -303,7 +323,7 @@ func (f *File) WriteString(s string) (n int, err error) { // Mkdir creates a new directory with the specified name and permission // bits (before umask). -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. func Mkdir(name string, perm FileMode) error { longName := fixLongPath(name) e := ignoringEINTR(func() error { @@ -337,15 +357,20 @@ func setStickyBit(name string) error { } // Chdir changes the current working directory to the named directory. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. func Chdir(dir string) error { if e := syscall.Chdir(dir); e != nil { testlog.Open(dir) // observe likely non-existent directory return &PathError{Op: "chdir", Path: dir, Err: e} } if runtime.GOOS == "windows" { + abs := filepathlite.IsAbs(dir) getwdCache.Lock() - getwdCache.dir = dir + if abs { + getwdCache.dir = dir + } else { + getwdCache.dir = "" + } getwdCache.Unlock() } if log := testlog.Logger(); log != nil { @@ -359,8 +384,8 @@ func Chdir(dir string) error { // Open opens the named file for reading. If successful, methods on // the returned file can be used for reading; the associated file -// descriptor has mode O_RDONLY. -// If there is an error, it will be of type *PathError. +// descriptor has mode [O_RDONLY]. +// If there is an error, it will be of type [*PathError]. func Open(name string) (*File, error) { return OpenFile(name, O_RDONLY, 0) } @@ -368,18 +393,20 @@ func Open(name string) (*File, error) { // Create creates or truncates the named file. If the file already exists, // it is truncated. If the file does not exist, it is created with mode 0o666 // (before umask). If successful, methods on the returned File can -// be used for I/O; the associated file descriptor has mode O_RDWR. -// If there is an error, it will be of type *PathError. +// be used for I/O; the associated file descriptor has mode [O_RDWR]. +// The directory containing the file must already exist. +// If there is an error, it will be of type [*PathError]. func Create(name string) (*File, error) { return OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666) } // OpenFile is the generalized open call; most users will use Open // or Create instead. It opens the named file with specified flag -// (O_RDONLY etc.). If the file does not exist, and the O_CREATE flag -// is passed, it is created with mode perm (before umask). If successful, +// ([O_RDONLY] etc.). If the file does not exist, and the [O_CREATE] flag +// is passed, it is created with mode perm (before umask); +// the containing directory must exist. If successful, // methods on the returned File can be used for I/O. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. func OpenFile(name string, flag int, perm FileMode) (*File, error) { testlog.Open(name) f, err := openFileNolog(name, flag, perm) @@ -391,6 +418,8 @@ func OpenFile(name string, flag int, perm FileMode) (*File, error) { return f, nil } +var errPathEscapes = errors.New("path escapes from parent") + // openDir opens a file which is assumed to be a directory. As such, it skips // the syscalls that make the file descriptor non-blocking as these take time // and will fail on file descriptors for directories. @@ -404,6 +433,7 @@ var lstat = Lstat // Rename renames (moves) oldpath to newpath. // If newpath already exists and is not a directory, Rename replaces it. +// If newpath already exists and is a directory, Rename returns an error. // OS-specific restrictions may apply when oldpath and newpath are in different directories. // Even within the same directory, on non-Unix platforms Rename is not an atomic operation. // If there is an error, it will be of type *LinkError. @@ -412,7 +442,7 @@ func Rename(oldpath, newpath string) error { } // Readlink returns the destination of the named symbolic link. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. // // If the link destination is relative, Readlink returns the relative path // without resolving it to an absolute one. @@ -586,7 +616,7 @@ func UserHomeDir() (string, error) { if v := Getenv(env); v != "" { return v, nil } - // On some geese the home directory is not always defined. + // On some operating systems the home directory is not always defined. switch runtime.GOOS { case "android": return "/sdcard", nil @@ -598,13 +628,13 @@ func UserHomeDir() (string, error) { // Chmod changes the mode of the named file to mode. // If the file is a symbolic link, it changes the mode of the link's target. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. // // A different subset of the mode bits are used, depending on the // operating system. // -// On Unix, the mode's permission bits, ModeSetuid, ModeSetgid, and -// ModeSticky are used. +// On Unix, the mode's permission bits, [ModeSetuid], [ModeSetgid], and +// [ModeSticky] are used. // // On Windows, only the 0o200 bit (owner writable) of mode is used; it // controls whether the file's read-only attribute is set or cleared. @@ -612,12 +642,12 @@ func UserHomeDir() (string, error) { // and earlier, use a non-zero mode. Use mode 0o400 for a read-only // file and 0o600 for a readable+writable file. // -// On Plan 9, the mode's permission bits, ModeAppend, ModeExclusive, -// and ModeTemporary are used. +// On Plan 9, the mode's permission bits, [ModeAppend], [ModeExclusive], +// and [ModeTemporary] are used. func Chmod(name string, mode FileMode) error { return chmod(name, mode) } // Chmod changes the mode of the file to mode. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. func (f *File) Chmod(mode FileMode) error { return f.chmod(mode) } // SetDeadline sets the read and write deadlines for a File. @@ -675,6 +705,27 @@ func (f *File) SyscallConn() (syscall.RawConn, error) { return newRawConn(f) } +// Fd returns the system file descriptor or handle referencing the open file. +// If f is closed, the descriptor becomes invalid. +// If f is garbage collected, a finalizer may close the descriptor, +// making it invalid; see [runtime.SetFinalizer] for more information on when +// a finalizer might be run. +// +// Do not close the returned descriptor; that could cause a later +// close of f to close an unrelated descriptor. +// +// Fd's behavior differs on some platforms: +// +// - On Unix and Windows, [File.SetDeadline] methods will stop working. +// - On Windows, the file descriptor will be disassociated from the +// Go runtime I/O completion port if there are no concurrent I/O +// operations on the file. +// +// For most uses prefer the f.SyscallConn method. +func (f *File) Fd() uintptr { + return f.fd() +} + // DirFS returns a file system (an fs.FS) for the tree of files rooted at the directory dir. // // Note that DirFS("/prefix") only guarantees that the Open calls it makes to the @@ -686,14 +737,21 @@ func (f *File) SyscallConn() (syscall.RawConn, error) { // a general substitute for a chroot-style security mechanism when the directory tree // contains arbitrary content. // +// Use [Root.FS] to obtain a fs.FS that prevents escapes from the tree via symbolic links. +// // The directory dir must not be "". // -// The result implements [io/fs.StatFS], [io/fs.ReadFileFS] and -// [io/fs.ReadDirFS]. +// The result implements [io/fs.StatFS], [io/fs.ReadFileFS], [io/fs.ReadDirFS], and +// [io/fs.ReadLinkFS]. func DirFS(dir string) fs.FS { return dirFS(dir) } +var _ fs.StatFS = dirFS("") +var _ fs.ReadFileFS = dirFS("") +var _ fs.ReadDirFS = dirFS("") +var _ fs.ReadLinkFS = dirFS("") + type dirFS string func (dir dirFS) Open(name string) (fs.File, error) { @@ -765,6 +823,28 @@ func (dir dirFS) Stat(name string) (fs.FileInfo, error) { return f, nil } +func (dir dirFS) Lstat(name string) (fs.FileInfo, error) { + fullname, err := dir.join(name) + if err != nil { + return nil, &PathError{Op: "lstat", Path: name, Err: err} + } + f, err := Lstat(fullname) + if err != nil { + // See comment in dirFS.Open. + err.(*PathError).Path = name + return nil, err + } + return f, nil +} + +func (dir dirFS) ReadLink(name string) (string, error) { + fullname, err := dir.join(name) + if err != nil { + return "", &PathError{Op: "readlink", Path: name, Err: err} + } + return Readlink(fullname) +} + // join returns the path for name in dir. func (dir dirFS) join(name string) (string, error) { if dir == "" { @@ -791,26 +871,44 @@ func ReadFile(name string) ([]byte, error) { } defer f.Close() + return readFileContents(statOrZero(f), f.Read) +} + +func statOrZero(f *File) int64 { + if fi, err := f.Stat(); err == nil { + return fi.Size() + } + return 0 +} + +// readFileContents reads the contents of a file using the provided read function +// (*os.File.Read, except in tests) one or more times, until an error is seen. +// +// The provided size is the stat size of the file, which might be 0 for a +// /proc-like file that doesn't report a size. +func readFileContents(statSize int64, read func([]byte) (int, error)) ([]byte, error) { + zeroSize := statSize == 0 + + // Figure out how big to make the initial slice. For files with known size + // that fit in memory, use that size + 1. Otherwise, use a small buffer and + // we'll grow. var size int - if info, err := f.Stat(); err == nil { - size64 := info.Size() - if int64(int(size64)) == size64 { - size = int(size64) - } + if int64(int(statSize)) == statSize { + size = int(statSize) } size++ // one byte for final read at EOF - // If a file claims a small size, read at least 512 bytes. - // In particular, files in Linux's /proc claim size 0 but - // then do not work right if read in small pieces, - // so an initial read of 1 byte would not work correctly. - if size < 512 { - size = 512 + const minBuf = 512 + // If a file claims a small size, read at least 512 bytes. In particular, + // files in Linux's /proc claim size 0 but then do not work right if read in + // small pieces, so an initial read of 1 byte would not work correctly. + if size < minBuf { + size = minBuf } data := make([]byte, 0, size) for { - n, err := f.Read(data[len(data):cap(data)]) + n, err := read(data[len(data):cap(data)]) data = data[:len(data)+n] if err != nil { if err == io.EOF { @@ -819,9 +917,12 @@ func ReadFile(name string) ([]byte, error) { return data, err } - if len(data) >= cap(data) { - d := append(data[:cap(data)], 0) - data = d[:len(data)] + // If we're either out of capacity or if the file was a /proc-like zero + // sized file, grow the buffer. Per Issue 72080, we always want to issue + // Read calls on zero-length files with a non-tiny buffer size. + capRemain := cap(data) - len(data) + if capRemain == 0 || (zeroSize && capRemain < minBuf) { + data = slices.Grow(data, minBuf) } } } diff --git a/src/os/file_plan9.go b/src/os/file_plan9.go index ef277deccce022..e563d123efa2c8 100644 --- a/src/os/file_plan9.go +++ b/src/os/file_plan9.go @@ -27,36 +27,27 @@ func fixLongPath(path string) string { // to close the wrong file descriptor. type file struct { fdmu poll.FDMutex - fd int + sysfd int name string dirinfo atomic.Pointer[dirInfo] // nil unless directory being read appendMode bool // whether file is opened for appending } -// Fd returns the integer Plan 9 file descriptor referencing the open file. -// If f is closed, the file descriptor becomes invalid. -// If f is garbage collected, a finalizer may close the file descriptor, -// making it invalid; see [runtime.SetFinalizer] for more information on when -// a finalizer might be run. On Unix systems this will cause the [File.SetDeadline] -// methods to stop working. -// -// As an alternative, see the f.SyscallConn method. -func (f *File) Fd() uintptr { +// fd is the Plan 9 implementation of Fd. +func (f *File) fd() uintptr { if f == nil { return ^(uintptr(0)) } - return uintptr(f.fd) + return uintptr(f.sysfd) } -// NewFile returns a new File with the given file descriptor and -// name. The returned value will be nil if fd is not a valid file -// descriptor. -func NewFile(fd uintptr, name string) *File { +// newFileFromNewFile is called by [NewFile]. +func newFileFromNewFile(fd uintptr, name string) *File { fdi := int(fd) if fdi < 0 { return nil } - f := &File{&file{fd: fdi, name: name}} + f := &File{&file{sysfd: fdi, name: name}} runtime.SetFinalizer(f.file, (*file).close) return f } @@ -144,7 +135,20 @@ func openFileNolog(name string, flag int, perm FileMode) (*File, error) { } func openDirNolog(name string) (*File, error) { - return openFileNolog(name, O_RDONLY, 0) + f, e := openFileNolog(name, O_RDONLY, 0) + if e != nil { + return nil, e + } + d, e := f.Stat() + if e != nil { + f.Close() + return nil, e + } + if !d.IsDir() { + f.Close() + return nil, &PathError{Op: "open", Path: name, Err: syscall.ENOTDIR} + } + return f, nil } // Close closes the File, rendering it unusable for I/O. @@ -178,14 +182,14 @@ func (file *file) close() error { // and writeUnlock methods. func (file *file) destroy() error { var err error - if e := syscall.Close(file.fd); e != nil { + if e := syscall.Close(file.sysfd); e != nil { err = &PathError{Op: "close", Path: file.name, Err: e} } return err } // Stat returns the FileInfo structure describing file. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. func (f *File) Stat() (FileInfo, error) { if f == nil { return nil, ErrInvalid @@ -199,7 +203,7 @@ func (f *File) Stat() (FileInfo, error) { // Truncate changes the size of the file. // It does not change the I/O offset. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. func (f *File) Truncate(size int64) error { if f == nil { return ErrInvalid @@ -220,7 +224,7 @@ func (f *File) Truncate(size int64) error { } defer f.decref() - if err = syscall.Fwstat(f.fd, buf[:n]); err != nil { + if err = syscall.Fwstat(f.sysfd, buf[:n]); err != nil { return &PathError{Op: "truncate", Path: f.name, Err: err} } return nil @@ -252,7 +256,7 @@ func (f *File) chmod(mode FileMode) error { } defer f.decref() - if err = syscall.Fwstat(f.fd, buf[:n]); err != nil { + if err = syscall.Fwstat(f.sysfd, buf[:n]); err != nil { return &PathError{Op: "chmod", Path: f.name, Err: err} } return nil @@ -279,7 +283,7 @@ func (f *File) Sync() error { } defer f.decref() - if err = syscall.Fwstat(f.fd, buf[:n]); err != nil { + if err = syscall.Fwstat(f.sysfd, buf[:n]); err != nil { return &PathError{Op: "sync", Path: f.name, Err: err} } return nil @@ -292,7 +296,7 @@ func (f *File) read(b []byte) (n int, err error) { return 0, err } defer f.readUnlock() - n, e := fixCount(syscall.Read(f.fd, b)) + n, e := fixCount(syscall.Read(f.sysfd, b)) if n == 0 && len(b) > 0 && e == nil { return 0, io.EOF } @@ -307,7 +311,7 @@ func (f *File) pread(b []byte, off int64) (n int, err error) { return 0, err } defer f.readUnlock() - n, e := fixCount(syscall.Pread(f.fd, b, off)) + n, e := fixCount(syscall.Pread(f.sysfd, b, off)) if n == 0 && len(b) > 0 && e == nil { return 0, io.EOF } @@ -326,7 +330,7 @@ func (f *File) write(b []byte) (n int, err error) { if len(b) == 0 { return 0, nil } - return fixCount(syscall.Write(f.fd, b)) + return fixCount(syscall.Write(f.sysfd, b)) } // pwrite writes len(b) bytes to the File starting at byte offset off. @@ -341,7 +345,7 @@ func (f *File) pwrite(b []byte, off int64) (n int, err error) { if len(b) == 0 { return 0, nil } - return fixCount(syscall.Pwrite(f.fd, b, off)) + return fixCount(syscall.Pwrite(f.sysfd, b, off)) } // seek sets the offset for the next Read or Write on file to offset, interpreted @@ -356,12 +360,12 @@ func (f *File) seek(offset int64, whence int) (ret int64, err error) { // Free cached dirinfo, so we allocate a new one if we // access this file as a directory again. See #35767 and #37161. f.dirinfo.Store(nil) - return syscall.Seek(f.fd, offset, whence) + return syscall.Seek(f.sysfd, offset, whence) } // Truncate changes the size of the named file. // If the file is a symbolic link, it changes the size of the link's target. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. func Truncate(name string, size int64) error { var d syscall.Dir @@ -380,7 +384,7 @@ func Truncate(name string, size int64) error { } // Remove removes the named file or directory. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. func Remove(name string) error { if e := syscall.Remove(name); e != nil { return &PathError{Op: "remove", Path: name, Err: e} @@ -453,7 +457,7 @@ func chmod(name string, mode FileMode) error { // // The underlying filesystem may truncate or round the values to a // less precise time unit. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. func Chtimes(name string, atime time.Time, mtime time.Time) error { var d syscall.Dir @@ -513,23 +517,23 @@ func readlink(name string) (string, error) { // Chown changes the numeric uid and gid of the named file. // If the file is a symbolic link, it changes the uid and gid of the link's target. // A uid or gid of -1 means to not change that value. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. // -// On Windows or Plan 9, Chown always returns the syscall.EWINDOWS or -// EPLAN9 error, wrapped in *PathError. +// On Windows or Plan 9, Chown always returns the [syscall.EWINDOWS] or +// [syscall.EPLAN9] error, wrapped in [*PathError]. func Chown(name string, uid, gid int) error { return &PathError{Op: "chown", Path: name, Err: syscall.EPLAN9} } // Lchown changes the numeric uid and gid of the named file. // If the file is a symbolic link, it changes the uid and gid of the link itself. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. func Lchown(name string, uid, gid int) error { return &PathError{Op: "lchown", Path: name, Err: syscall.EPLAN9} } // Chown changes the numeric uid and gid of the named file. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. func (f *File) Chown(uid, gid int) error { if f == nil { return ErrInvalid @@ -547,13 +551,13 @@ func tempDir() string { // Chdir changes the current working directory to the file, // which must be a directory. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. func (f *File) Chdir() error { if err := f.incref("chdir"); err != nil { return err } defer f.decref() - if e := syscall.Fchdir(f.fd); e != nil { + if e := syscall.Fchdir(f.sysfd); e != nil { return &PathError{Op: "chdir", Path: f.name, Err: e} } return nil @@ -617,3 +621,7 @@ func newRawConn(file *File) (*rawConn, error) { func ignoringEINTR(fn func() error) error { return fn() } + +func ignoringEINTR2[T any](fn func() (T, error)) (T, error) { + return fn() +} diff --git a/src/os/file_posix.go b/src/os/file_posix.go index 8ff0ada4629033..68904f62e1ffca 100644 --- a/src/os/file_posix.go +++ b/src/os/file_posix.go @@ -101,7 +101,7 @@ func (f *File) chmod(mode FileMode) error { // If there is an error, it will be of type [*PathError]. // // On Windows or Plan 9, Chown always returns the [syscall.EWINDOWS] or -// EPLAN9 error, wrapped in *PathError. +// [syscall.EPLAN9] error, wrapped in [*PathError]. func Chown(name string, uid, gid int) error { e := ignoringEINTR(func() error { return syscall.Chown(name, uid, gid) @@ -117,7 +117,7 @@ func Chown(name string, uid, gid int) error { // If there is an error, it will be of type [*PathError]. // // On Windows, it always returns the [syscall.EWINDOWS] error, wrapped -// in *PathError. +// in [*PathError]. func Lchown(name string, uid, gid int) error { e := ignoringEINTR(func() error { return syscall.Lchown(name, uid, gid) @@ -132,7 +132,7 @@ func Lchown(name string, uid, gid int) error { // If there is an error, it will be of type [*PathError]. // // On Windows, it always returns the [syscall.EWINDOWS] error, wrapped -// in *PathError. +// in [*PathError]. func (f *File) Chown(uid, gid int) error { if err := f.checkValid("chown"); err != nil { return err @@ -177,6 +177,14 @@ func (f *File) Sync() error { // less precise time unit. // If there is an error, it will be of type [*PathError]. func Chtimes(name string, atime time.Time, mtime time.Time) error { + utimes := chtimesUtimes(atime, mtime) + if e := syscall.UtimesNano(fixLongPath(name), utimes[0:]); e != nil { + return &PathError{Op: "chtimes", Path: name, Err: e} + } + return nil +} + +func chtimesUtimes(atime, mtime time.Time) [2]syscall.Timespec { var utimes [2]syscall.Timespec set := func(i int, t time.Time) { if t.IsZero() { @@ -187,10 +195,7 @@ func Chtimes(name string, atime time.Time, mtime time.Time) error { } set(0, atime) set(1, mtime) - if e := syscall.UtimesNano(fixLongPath(name), utimes[0:]); e != nil { - return &PathError{Op: "chtimes", Path: name, Err: e} - } - return nil + return utimes } // Chdir changes the current working directory to the file, @@ -254,3 +259,13 @@ func ignoringEINTR(fn func() error) error { } } } + +// ignoringEINTR2 is ignoringEINTR, but returning an additional value. +func ignoringEINTR2[T any](fn func() (T, error)) (T, error) { + for { + v, err := fn() + if err != syscall.EINTR { + return v, err + } + } +} diff --git a/src/os/file_test.go b/src/os/file_test.go new file mode 100644 index 00000000000000..f56a34da3e080f --- /dev/null +++ b/src/os/file_test.go @@ -0,0 +1,156 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os_test + +import ( + "internal/testenv" + "io/fs" + . "os" + "path/filepath" + "testing" +) + +func TestDirFSReadLink(t *testing.T) { + testenv.MustHaveSymlink(t) + + root := t.TempDir() + subdir := filepath.Join(root, "dir") + if err := Mkdir(subdir, 0o777); err != nil { + t.Fatal(err) + } + links := map[string]string{ + filepath.Join(root, "parent-link"): filepath.Join("..", "foo"), + filepath.Join(root, "sneaky-parent-link"): filepath.Join("dir", "..", "..", "foo"), + filepath.Join(root, "abs-link"): filepath.Join(root, "foo"), + filepath.Join(root, "rel-link"): "foo", + filepath.Join(root, "rel-sub-link"): filepath.Join("dir", "foo"), + filepath.Join(subdir, "parent-link"): filepath.Join("..", "foo"), + } + for newname, oldname := range links { + if err := Symlink(oldname, newname); err != nil { + t.Fatal(err) + } + } + + fsys := DirFS(root) + want := map[string]string{ + "rel-link": "foo", + "rel-sub-link": filepath.Join("dir", "foo"), + "dir/parent-link": filepath.Join("..", "foo"), + "parent-link": filepath.Join("..", "foo"), + "sneaky-parent-link": filepath.Join("dir", "..", "..", "foo"), + "abs-link": filepath.Join(root, "foo"), + } + for name, want := range want { + got, err := fs.ReadLink(fsys, name) + if got != want || err != nil { + t.Errorf("fs.ReadLink(fsys, %q) = %q, %v; want %q, ", name, got, err, want) + } + } +} + +func TestDirFSLstat(t *testing.T) { + testenv.MustHaveSymlink(t) + + root := t.TempDir() + subdir := filepath.Join(root, "dir") + if err := Mkdir(subdir, 0o777); err != nil { + t.Fatal(err) + } + if err := Symlink("dir", filepath.Join(root, "link")); err != nil { + t.Fatal(err) + } + + fsys := DirFS(root) + want := map[string]fs.FileMode{ + "link": fs.ModeSymlink, + "dir": fs.ModeDir, + } + for name, want := range want { + info, err := fs.Lstat(fsys, name) + var got fs.FileMode + if info != nil { + got = info.Mode().Type() + } + if got != want || err != nil { + t.Errorf("fs.Lstat(fsys, %q).Mode().Type() = %v, %v; want %v, ", name, got, err, want) + } + } +} + +func TestDirFSWalkDir(t *testing.T) { + testenv.MustHaveSymlink(t) + + root := t.TempDir() + subdir := filepath.Join(root, "dir") + if err := Mkdir(subdir, 0o777); err != nil { + t.Fatal(err) + } + if err := Symlink("dir", filepath.Join(root, "link")); err != nil { + t.Fatal(err) + } + if err := WriteFile(filepath.Join(root, "dir", "a"), nil, 0o666); err != nil { + t.Fatal(err) + } + fsys := DirFS(root) + + t.Run("SymlinkRoot", func(t *testing.T) { + wantTypes := map[string]fs.FileMode{ + "link": fs.ModeDir, + "link/a": 0, + } + marks := make(map[string]int) + err := fs.WalkDir(fsys, "link", func(path string, entry fs.DirEntry, err error) error { + marks[path]++ + if want, ok := wantTypes[path]; !ok { + t.Errorf("Unexpected path %q in walk", path) + } else if got := entry.Type(); got != want { + t.Errorf("%s entry type = %v; want %v", path, got, want) + } + if err != nil { + t.Errorf("%s: %v", path, err) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + for path := range wantTypes { + if got := marks[path]; got != 1 { + t.Errorf("%s visited %d times; expected 1", path, got) + } + } + }) + + t.Run("SymlinkPresent", func(t *testing.T) { + wantTypes := map[string]fs.FileMode{ + ".": fs.ModeDir, + "dir": fs.ModeDir, + "dir/a": 0, + "link": fs.ModeSymlink, + } + marks := make(map[string]int) + err := fs.WalkDir(fsys, ".", func(path string, entry fs.DirEntry, err error) error { + marks[path]++ + if want, ok := wantTypes[path]; !ok { + t.Errorf("Unexpected path %q in walk", path) + } else if got := entry.Type(); got != want { + t.Errorf("%s entry type = %v; want %v", path, got, want) + } + if err != nil { + t.Errorf("%s: %v", path, err) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + for path := range wantTypes { + if got := marks[path]; got != 1 { + t.Errorf("%s visited %d times; expected 1", path, got) + } + } + }) +} diff --git a/src/os/file_unix.go b/src/os/file_unix.go index 37bfaa1a72dfae..2074df70febc2e 100644 --- a/src/os/file_unix.go +++ b/src/os/file_unix.go @@ -65,19 +65,8 @@ type file struct { appendMode bool // whether file is opened for appending } -// Fd returns the integer Unix file descriptor referencing the open file. -// If f is closed, the file descriptor becomes invalid. -// If f is garbage collected, a finalizer may close the file descriptor, -// making it invalid; see [runtime.SetFinalizer] for more information on when -// a finalizer might be run. On Unix systems this will cause the [File.SetDeadline] -// methods to stop working. -// Because file descriptors can be reused, the returned file descriptor may -// only be closed through the [File.Close] method of f, or by its finalizer during -// garbage collection. Otherwise, during garbage collection the finalizer -// may close an unrelated file descriptor with the same (reused) number. -// -// As an alternative, see the f.SyscallConn method. -func (f *File) Fd() uintptr { +// fd is the Unix implementation of Fd. +func (f *File) fd() uintptr { if f == nil { return ^(uintptr(0)) } @@ -94,16 +83,8 @@ func (f *File) Fd() uintptr { return uintptr(f.pfd.Sysfd) } -// NewFile returns a new File with the given file descriptor and -// name. The returned value will be nil if fd is not a valid file -// descriptor. On Unix systems, if the file descriptor is in -// non-blocking mode, NewFile will attempt to return a pollable File -// (one for which the SetDeadline methods work). -// -// After passing it to NewFile, fd may become invalid under the same -// conditions described in the comments of the Fd method, and the same -// constraints apply. -func NewFile(fd uintptr, name string) *File { +// newFileFromNewFile is called by [NewFile]. +func newFileFromNewFile(fd uintptr, name string) *File { fdi := int(fd) if fdi < 0 { return nil @@ -306,7 +287,7 @@ func openDirNolog(name string) (*File, error) { e error ) ignoringEINTR(func() error { - r, s, e = open(name, O_RDONLY|syscall.O_CLOEXEC, 0) + r, s, e = open(name, O_RDONLY|syscall.O_CLOEXEC|syscall.O_DIRECTORY, 0) return e }) if e != nil { @@ -359,7 +340,7 @@ func (f *File) seek(offset int64, whence int) (ret int64, err error) { // Truncate changes the size of the named file. // If the file is a symbolic link, it changes the size of the link's target. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. func Truncate(name string, size int64) error { e := ignoringEINTR(func() error { return syscall.Truncate(name, size) @@ -371,7 +352,7 @@ func Truncate(name string, size int64) error { } // Remove removes the named file or (empty) directory. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. func Remove(name string) error { // System call interface forces us to know // whether name is a file or directory. @@ -446,22 +427,15 @@ func Symlink(oldname, newname string) error { func readlink(name string) (string, error) { for len := 128; ; len *= 2 { b := make([]byte, len) - var ( - n int - e error - ) - for { - n, e = fixCount(syscall.Readlink(name, b)) - if e != syscall.EINTR { - break - } - } + n, err := ignoringEINTR2(func() (int, error) { + return fixCount(syscall.Readlink(name, b)) + }) // buffer too small - if (runtime.GOOS == "aix" || runtime.GOOS == "wasip1") && e == syscall.ERANGE { + if (runtime.GOOS == "aix" || runtime.GOOS == "wasip1") && err == syscall.ERANGE { continue } - if e != nil { - return "", &PathError{Op: "readlink", Path: name, Err: e} + if err != nil { + return "", &PathError{Op: "readlink", Path: name, Err: err} } if n < len { return string(b[0:n]), nil @@ -497,7 +471,7 @@ func newUnixDirent(parent, name string, typ FileMode) (DirEntry, error) { name: name, typ: typ, } - if typ != ^FileMode(0) && !testingForceReadDirLstat { + if typ != ^FileMode(0) { return ude, nil } diff --git a/src/os/file_windows.go b/src/os/file_windows.go index cf652ca1bb3f17..83f5fde7f9de46 100644 --- a/src/os/file_windows.go +++ b/src/os/file_windows.go @@ -31,28 +31,31 @@ type file struct { appendMode bool // whether file is opened for appending } -// Fd returns the Windows handle referencing the open file. -// If f is closed, the file descriptor becomes invalid. -// If f is garbage collected, a finalizer may close the file descriptor, -// making it invalid; see [runtime.SetFinalizer] for more information on when -// a finalizer might be run. On Unix systems this will cause the [File.SetDeadline] -// methods to stop working. -func (file *File) Fd() uintptr { +// fd is the Windows implementation of Fd. +func (file *File) fd() uintptr { if file == nil { return uintptr(syscall.InvalidHandle) } + // Try to disassociate the file from the runtime poller. + // File.Fd doesn't return an error, so we don't have a way to + // report it. We just ignore it. It's up to the caller to call + // it when there are no concurrent IO operations. + _ = file.pfd.DisassociateIOCP() return uintptr(file.pfd.Sysfd) } // newFile returns a new File with the given file handle and name. // Unlike NewFile, it does not check that h is syscall.InvalidHandle. -func newFile(h syscall.Handle, name string, kind string) *File { +// If nonBlocking is true, it tries to add the file to the runtime poller. +func newFile(h syscall.Handle, name string, kind string, nonBlocking bool) *File { if kind == "file" { - var m uint32 - if syscall.GetConsoleMode(h, &m) == nil { - kind = "console" - } - if t, err := syscall.GetFileType(h); err == nil && t == syscall.FILE_TYPE_PIPE { + t, err := syscall.GetFileType(h) + if err != nil || t == syscall.FILE_TYPE_CHAR { + var m uint32 + if syscall.GetConsoleMode(h, &m) == nil { + kind = "console" + } + } else if t == syscall.FILE_TYPE_PIPE { kind = "pipe" } } @@ -69,25 +72,35 @@ func newFile(h syscall.Handle, name string, kind string) *File { // Ignore initialization errors. // Assume any problems will show up in later I/O. - f.pfd.Init(kind, false) - + f.pfd.Init(kind, nonBlocking) return f } // newConsoleFile creates new File that will be used as console. func newConsoleFile(h syscall.Handle, name string) *File { - return newFile(h, name, "console") + return newFile(h, name, "console", false) } -// NewFile returns a new File with the given file descriptor and -// name. The returned value will be nil if fd is not a valid file -// descriptor. -func NewFile(fd uintptr, name string) *File { +// newFileFromNewFile is called by [NewFile]. +func newFileFromNewFile(fd uintptr, name string) *File { h := syscall.Handle(fd) if h == syscall.InvalidHandle { return nil } - return newFile(h, name, "file") + nonBlocking, _ := windows.IsNonblock(syscall.Handle(fd)) + return newFile(h, name, "file", nonBlocking) +} + +// net_newWindowsFile is a hidden entry point called by net.conn.File. +// This is used so that the File.pfd.close method calls [syscall.Closesocket] +// instead of [syscall.CloseHandle]. +// +//go:linkname net_newWindowsFile net.newWindowsFile +func net_newWindowsFile(h syscall.Handle, name string) *File { + if h == syscall.InvalidHandle { + panic("invalid FD") + } + return newFile(h, name, "file+net", true) } func epipecheck(file *File, e error) { @@ -103,26 +116,16 @@ func openFileNolog(name string, flag int, perm FileMode) (*File, error) { return nil, &PathError{Op: "open", Path: name, Err: syscall.ENOENT} } path := fixLongPath(name) - r, e := syscall.Open(path, flag|syscall.O_CLOEXEC, syscallMode(perm)) - if e != nil { - // We should return EISDIR when we are trying to open a directory with write access. - if e == syscall.ERROR_ACCESS_DENIED && (flag&O_WRONLY != 0 || flag&O_RDWR != 0) { - pathp, e1 := syscall.UTF16PtrFromString(path) - if e1 == nil { - var fa syscall.Win32FileAttributeData - e1 = syscall.GetFileAttributesEx(pathp, syscall.GetFileExInfoStandard, (*byte)(unsafe.Pointer(&fa))) - if e1 == nil && fa.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 { - e = syscall.EISDIR - } - } - } - return nil, &PathError{Op: "open", Path: name, Err: e} + r, err := syscall.Open(path, flag|syscall.O_CLOEXEC, syscallMode(perm)) + if err != nil { + return nil, &PathError{Op: "open", Path: name, Err: err} } - return newFile(r, name, "file"), nil + nonblocking := flag&windows.O_FILE_FLAG_OVERLAPPED != 0 + return newFile(r, name, "file", nonblocking), nil } func openDirNolog(name string) (*File, error) { - return openFileNolog(name, O_RDONLY, 0) + return openFileNolog(name, O_RDONLY|windows.O_DIRECTORY, 0) } func (file *file) close() error { @@ -176,7 +179,7 @@ func Truncate(name string, size int64) error { } // Remove removes the named file or directory. -// If there is an error, it will be of type *PathError. +// If there is an error, it will be of type [*PathError]. func Remove(name string) error { p, e := syscall.UTF16PtrFromString(fixLongPath(name)) if e != nil { @@ -231,20 +234,17 @@ func Pipe() (r *File, w *File, err error) { if e != nil { return nil, nil, NewSyscallError("pipe", e) } - return newFile(p[0], "|0", "pipe"), newFile(p[1], "|1", "pipe"), nil + // syscall.Pipe always returns a non-blocking handle. + return newFile(p[0], "|0", "pipe", false), newFile(p[1], "|1", "pipe", false), nil } -var ( - useGetTempPath2Once sync.Once - useGetTempPath2 bool -) +var useGetTempPath2 = sync.OnceValue(func() bool { + return windows.ErrorLoadingGetTempPath2() == nil +}) func tempDir() string { - useGetTempPath2Once.Do(func() { - useGetTempPath2 = (windows.ErrorLoadingGetTempPath2() == nil) - }) getTempPath := syscall.GetTempPath - if useGetTempPath2 { + if useGetTempPath2() { getTempPath = windows.GetTempPath2 } n := uint32(syscall.MAX_PATH) @@ -430,10 +430,13 @@ func readReparseLink(path string) (string, error) { return "", err } defer syscall.CloseHandle(h) + return readReparseLinkHandle(h) +} +func readReparseLinkHandle(h syscall.Handle) (string, error) { rdbbuf := make([]byte, syscall.MAXIMUM_REPARSE_DATA_BUFFER_SIZE) var bytesReturned uint32 - err = syscall.DeviceIoControl(h, syscall.FSCTL_GET_REPARSE_POINT, nil, 0, &rdbbuf[0], uint32(len(rdbbuf)), &bytesReturned, nil) + err := syscall.DeviceIoControl(h, syscall.FSCTL_GET_REPARSE_POINT, nil, 0, &rdbbuf[0], uint32(len(rdbbuf)), &bytesReturned, nil) if err != nil { return "", err } diff --git a/src/os/getwd.go b/src/os/getwd.go index 90604cf2f4b18e..5ce948faf5d57d 100644 --- a/src/os/getwd.go +++ b/src/os/getwd.go @@ -15,45 +15,61 @@ var getwdCache struct { dir string } -// Getwd returns a rooted path name corresponding to the +// Getwd returns an absolute path name corresponding to the // current directory. If the current directory can be // reached via multiple paths (due to symbolic links), // Getwd may return any one of them. +// +// On Unix platforms, if the environment variable PWD +// provides an absolute name, and it is a name of the +// current directory, it is returned. func Getwd() (dir string, err error) { if runtime.GOOS == "windows" || runtime.GOOS == "plan9" { - return syscall.Getwd() + // Use syscall.Getwd directly for + // - plan9: see reasons in CL 89575; + // - windows: syscall implementation is sufficient, + // and we should not rely on $PWD. + dir, err = syscall.Getwd() + return dir, NewSyscallError("getwd", err) } // Clumsy but widespread kludge: // if $PWD is set and matches ".", use it. - dot, err := statNolog(".") - if err != nil { - return "", err - } + var dot FileInfo dir = Getenv("PWD") if len(dir) > 0 && dir[0] == '/' { + dot, err = statNolog(".") + if err != nil { + return "", err + } d, err := statNolog(dir) if err == nil && SameFile(dot, d) { return dir, nil } + // If err is ENAMETOOLONG here, the syscall.Getwd below will + // fail with the same error, too, but let's give it a try + // anyway as the fallback code is much slower. } // If the operating system provides a Getwd call, use it. - // Otherwise, we're trying to find our way back to ".". if syscall.ImplementsGetwd { - var ( - s string - e error - ) - for { - s, e = syscall.Getwd() - if e != syscall.EINTR { - break - } + dir, err = ignoringEINTR2(syscall.Getwd) + // Linux returns ENAMETOOLONG if the result is too long. + // Some BSD systems appear to return EINVAL. + // FreeBSD systems appear to use ENOMEM + // Solaris appears to use ERANGE. + if err != syscall.ENAMETOOLONG && err != syscall.EINVAL && err != errERANGE && err != errENOMEM { + return dir, NewSyscallError("getwd", err) } - return s, NewSyscallError("getwd", e) } + // We're trying to find our way back to ".". + if dot == nil { + dot, err = statNolog(".") + if err != nil { + return "", err + } + } // Apply same kludge but to cached dir instead of $PWD. getwdCache.Lock() dir = getwdCache.dir @@ -82,9 +98,9 @@ func Getwd() (dir string, err error) { dir = "" for parent := ".."; ; parent = "../" + parent { if len(parent) >= 1024 { // Sanity check - return "", syscall.ENAMETOOLONG + return "", NewSyscallError("getwd", syscall.ENAMETOOLONG) } - fd, err := openFileNolog(parent, O_RDONLY, 0) + fd, err := openDirNolog(parent) if err != nil { return "", err } @@ -93,7 +109,14 @@ func Getwd() (dir string, err error) { names, err := fd.Readdirnames(100) if err != nil { fd.Close() - return "", err + // Readdirnames can return io.EOF or other error. + // In any case, we're here because syscall.Getwd + // is not implemented or failed with ENAMETOOLONG, + // so return the most sensible error. + if syscall.ImplementsGetwd { + return "", NewSyscallError("getwd", syscall.ENAMETOOLONG) + } + return "", NewSyscallError("getwd", errENOSYS) } for _, name := range names { d, _ := lstatNolog(parent + "/" + name) diff --git a/src/os/getwd_unix_test.go b/src/os/getwd_unix_test.go new file mode 100644 index 00000000000000..3f3067b5d59c3b --- /dev/null +++ b/src/os/getwd_unix_test.go @@ -0,0 +1,113 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix + +package os_test + +import ( + "errors" + . "os" + "path/filepath" + "runtime" + "strings" + "syscall" + "testing" +) + +func TestGetwdDeep(t *testing.T) { + testGetwdDeep(t, false) +} + +func TestGetwdDeepWithPWDSet(t *testing.T) { + testGetwdDeep(t, true) +} + +// testGetwdDeep checks that os.Getwd is able to return paths +// longer than syscall.PathMax (with or without PWD set). +func testGetwdDeep(t *testing.T, setPWD bool) { + tempDir := t.TempDir() + + dir := tempDir + t.Chdir(dir) + + if setPWD { + t.Setenv("PWD", dir) + } else { + // When testing os.Getwd, setting PWD to empty string + // is the same as unsetting it, but the latter would + // be more complicated since we don't have t.Unsetenv. + t.Setenv("PWD", "") + } + + name := strings.Repeat("a", 200) + for { + if err := Mkdir(name, 0o700); err != nil { + t.Fatal(err) + } + if err := Chdir(name); err != nil { + t.Fatal(err) + } + if setPWD { + dir += "/" + name + if err := Setenv("PWD", dir); err != nil { + t.Fatal(err) + } + t.Logf(" $PWD len: %d", len(dir)) + } + + wd, err := Getwd() + t.Logf("Getwd len: %d", len(wd)) + if err != nil { + // We can get an EACCES error if we can't read up + // to root, which happens on the Android builders. + if errors.Is(err, syscall.EACCES) { + t.Logf("ignoring EACCES error: %v", err) + break + } + t.Fatal(err) + } + if setPWD && wd != dir { + // It's possible for the stat of PWD to fail + // with ENAMETOOLONG, and for getwd to fail for + // the same reason, and it's possible for $TMPDIR + // to contain a symlink. In that case the fallback + // code will not return the same directory. + if len(dir) > 1000 { + symDir, err := filepath.EvalSymlinks(tempDir) + if err == nil && symDir != tempDir { + t.Logf("EvalSymlinks(%q) = %q", tempDir, symDir) + if strings.Replace(dir, tempDir, symDir, 1) == wd { + // Symlink confusion is OK. + break + } + } + } + + t.Fatalf("Getwd: got %q, want same value as $PWD: %q", wd, dir) + } + // Ideally the success criterion should be len(wd) > syscall.PathMax, + // but the latter is not public for some platforms, so use Stat(wd). + // When it fails with ENAMETOOLONG, it means: + // - wd is longer than PathMax; + // - Getwd have used the slow fallback code. + // + // To avoid an endless loop here in case Stat keeps working, + // check if len(wd) is above the largest known PathMax among + // all Unix platforms (4096, on Linux). + if _, err := Stat(wd); err != nil || len(wd) > 4096 { + t.Logf("Done; len(wd)=%d", len(wd)) + // Most systems return ENAMETOOLONG. + // Dragonfly returns EFAULT. + switch { + case err == nil: + case errors.Is(err, syscall.ENAMETOOLONG): + case runtime.GOOS == "dragonfly" && errors.Is(err, syscall.EFAULT): + default: + t.Fatalf("unexpected Stat error: %v", err) + } + break + } + } +} diff --git a/src/os/os_test.go b/src/os/os_test.go index 46c4f138c3819c..536734901baff6 100644 --- a/src/os/os_test.go +++ b/src/os/os_test.go @@ -19,6 +19,7 @@ import ( "runtime" "runtime/debug" "slices" + "strconv" "strings" "sync" "syscall" @@ -195,7 +196,7 @@ func TestStat(t *testing.T) { } func TestStatError(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) path := "no-such-file" @@ -232,8 +233,7 @@ func TestStatError(t *testing.T) { func TestStatSymlinkLoop(t *testing.T) { testenv.MustHaveSymlink(t) - - defer chtmpdir(t)() + t.Chdir(t.TempDir()) err := Symlink("x", "y") if err != nil { @@ -840,8 +840,7 @@ func TestReaddirOfFile(t *testing.T) { if err == nil { t.Error("Readdirnames succeeded; want non-nil error") } - var pe *PathError - if !errors.As(err, &pe) || pe.Path != f.Name() { + if pe, ok := errors.AsType[*PathError](err); !ok || pe.Path != f.Name() { t.Errorf("Readdirnames returned %q; want a PathError with path %q", err, f.Name()) } if len(names) > 0 { @@ -850,34 +849,49 @@ func TestReaddirOfFile(t *testing.T) { } func TestHardLink(t *testing.T) { + testMaybeRooted(t, testHardLink) +} +func testHardLink(t *testing.T, root *Root) { testenv.MustHaveLink(t) - defer chtmpdir(t)() + var ( + create = Create + link = Link + stat = Stat + op = "link" + ) + if root != nil { + create = root.Create + link = root.Link + stat = root.Stat + op = "linkat" + } + from, to := "hardlinktestfrom", "hardlinktestto" - file, err := Create(to) + file, err := create(to) if err != nil { t.Fatalf("open %q failed: %v", to, err) } if err = file.Close(); err != nil { t.Errorf("close %q failed: %v", to, err) } - err = Link(to, from) + err = link(to, from) if err != nil { t.Fatalf("link %q, %q failed: %v", to, from, err) } none := "hardlinktestnone" - err = Link(none, none) + err = link(none, none) // Check the returned error is well-formed. if lerr, ok := err.(*LinkError); !ok || lerr.Error() == "" { t.Errorf("link %q, %q failed to return a valid error", none, none) } - tostat, err := Stat(to) + tostat, err := stat(to) if err != nil { t.Fatalf("stat %q failed: %v", to, err) } - fromstat, err := Stat(from) + fromstat, err := stat(from) if err != nil { t.Fatalf("stat %q failed: %v", from, err) } @@ -885,11 +899,11 @@ func TestHardLink(t *testing.T) { t.Errorf("link %q, %q did not create hard link", to, from) } // We should not be able to perform the same Link() a second time - err = Link(to, from) + err = link(to, from) switch err := err.(type) { case *LinkError: - if err.Op != "link" { - t.Errorf("Link(%q, %q) err.Op = %q; want %q", to, from, err.Op, "link") + if err.Op != op { + t.Errorf("Link(%q, %q) err.Op = %q; want %q", to, from, err.Op, op) } if err.Old != to { t.Errorf("Link(%q, %q) err.Old = %q; want %q", to, from, err.Old, to) @@ -907,66 +921,63 @@ func TestHardLink(t *testing.T) { } } -// chtmpdir changes the working directory to a new temporary directory and -// provides a cleanup function. -func chtmpdir(t *testing.T) func() { - oldwd, err := Getwd() - if err != nil { - t.Fatalf("chtmpdir: %v", err) - } - d, err := MkdirTemp("", "test") - if err != nil { - t.Fatalf("chtmpdir: %v", err) - } - if err := Chdir(d); err != nil { - t.Fatalf("chtmpdir: %v", err) - } - return func() { - if err := Chdir(oldwd); err != nil { - t.Fatalf("chtmpdir: %v", err) - } - RemoveAll(d) - } -} - func TestSymlink(t *testing.T) { + testMaybeRooted(t, testSymlink) +} +func testSymlink(t *testing.T, root *Root) { testenv.MustHaveSymlink(t) - defer chtmpdir(t)() + var ( + create = Create + open = Open + symlink = Symlink + stat = Stat + lstat = Lstat + readlink = Readlink + ) + if root != nil { + create = root.Create + open = root.Open + symlink = root.Symlink + stat = root.Stat + lstat = root.Lstat + readlink = root.Readlink + } + from, to := "symlinktestfrom", "symlinktestto" - file, err := Create(to) + file, err := create(to) if err != nil { t.Fatalf("Create(%q) failed: %v", to, err) } if err = file.Close(); err != nil { t.Errorf("Close(%q) failed: %v", to, err) } - err = Symlink(to, from) + err = symlink(to, from) if err != nil { t.Fatalf("Symlink(%q, %q) failed: %v", to, from, err) } - tostat, err := Lstat(to) + tostat, err := lstat(to) if err != nil { t.Fatalf("Lstat(%q) failed: %v", to, err) } if tostat.Mode()&ModeSymlink != 0 { t.Fatalf("Lstat(%q).Mode()&ModeSymlink = %v, want 0", to, tostat.Mode()&ModeSymlink) } - fromstat, err := Stat(from) + fromstat, err := stat(from) if err != nil { t.Fatalf("Stat(%q) failed: %v", from, err) } if !SameFile(tostat, fromstat) { t.Errorf("Symlink(%q, %q) did not create symlink", to, from) } - fromstat, err = Lstat(from) + fromstat, err = lstat(from) if err != nil { t.Fatalf("Lstat(%q) failed: %v", from, err) } if fromstat.Mode()&ModeSymlink == 0 { t.Fatalf("Lstat(%q).Mode()&ModeSymlink = 0, want %v", from, ModeSymlink) } - fromstat, err = Stat(from) + fromstat, err = stat(from) if err != nil { t.Fatalf("Stat(%q) failed: %v", from, err) } @@ -976,14 +987,14 @@ func TestSymlink(t *testing.T) { if fromstat.Mode()&ModeSymlink != 0 { t.Fatalf("Stat(%q).Mode()&ModeSymlink = %v, want 0", from, fromstat.Mode()&ModeSymlink) } - s, err := Readlink(from) + s, err := readlink(from) if err != nil { t.Fatalf("Readlink(%q) failed: %v", from, err) } if s != to { t.Fatalf("Readlink(%q) = %q, want %q", from, s, to) } - file, err = Open(from) + file, err = open(from) if err != nil { t.Fatalf("Open(%q) failed: %v", from, err) } @@ -992,8 +1003,8 @@ func TestSymlink(t *testing.T) { func TestLongSymlink(t *testing.T) { testenv.MustHaveSymlink(t) + t.Chdir(t.TempDir()) - defer chtmpdir(t)() s := "0123456789abcdef" // Long, but not too long: a common limit is 255. s = s + s + s + s + s + s + s + s + s + s + s + s + s + s + s @@ -1012,7 +1023,7 @@ func TestLongSymlink(t *testing.T) { } func TestRename(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) from, to := "renamefrom", "renameto" file, err := Create(from) @@ -1033,7 +1044,7 @@ func TestRename(t *testing.T) { } func TestRenameOverwriteDest(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) from, to := "renamefrom", "renameto" toData := []byte("to") @@ -1070,7 +1081,7 @@ func TestRenameOverwriteDest(t *testing.T) { } func TestRenameFailed(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) from, to := "renamefrom", "renameto" err := Rename(from, to) @@ -1093,7 +1104,7 @@ func TestRenameFailed(t *testing.T) { } func TestRenameNotExisting(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) from, to := "doesnt-exist", "dest" Mkdir(to, 0777) @@ -1104,7 +1115,7 @@ func TestRenameNotExisting(t *testing.T) { } func TestRenameToDirFailed(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) from, to := "renamefrom", "renameto" Mkdir(from, 0777) @@ -1149,7 +1160,7 @@ func TestRenameCaseDifference(pt *testing.T) { for _, test := range tests { pt.Run(test.name, func(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) if err := test.create(); err != nil { t.Fatalf("failed to create test file: %s", err) @@ -1367,6 +1378,9 @@ var hasNoatime = sync.OnceValue(func() bool { // but the syscall is OS-specific and is not even wired into Go stdlib. // // Only used on NetBSD (which ignores explicit atime updates with noatime). + if runtime.GOOS != "netbsd" { + return false + } mounts, _ := ReadFile("/proc/mounts") return bytes.Contains(mounts, []byte("noatime")) }) @@ -1375,8 +1389,7 @@ func TestChtimes(t *testing.T) { t.Parallel() f := newFile(t) - - f.Write([]byte("hello, world\n")) + // This should be an empty file (see #68687, #68663). f.Close() testChtimes(t, f.Name()) @@ -1394,12 +1407,9 @@ func TestChtimesOmit(t *testing.T) { func testChtimesOmit(t *testing.T, omitAt, omitMt bool) { t.Logf("omit atime: %v, mtime: %v", omitAt, omitMt) file := newFile(t) - _, err := file.Write([]byte("hello, world\n")) - if err != nil { - t.Fatal(err) - } + // This should be an empty file (see #68687, #68663). name := file.Name() - err = file.Close() + err := file.Close() if err != nil { t.Error(err) } @@ -1533,10 +1543,10 @@ func testChtimes(t *testing.T, name string) { t.Log(errormsg) t.Log("Known NetBSD issue (atime not changed on fs mounted with noatime); ignoring.") } else { - t.Errorf(errormsg) + t.Error(errormsg) } default: - t.Errorf(errormsg) + t.Error(errormsg) } } @@ -1575,7 +1585,7 @@ func TestFileChdir(t *testing.T) { if err != nil { t.Fatalf("Getwd: %s", err) } - defer Chdir(wd) + t.Chdir(".") // Ensure wd is restored after the test. fd, err := Open(".") if err != nil { @@ -1610,10 +1620,8 @@ func TestFileChdir(t *testing.T) { } func TestChdirAndGetwd(t *testing.T) { - fd, err := Open(".") - if err != nil { - t.Fatalf("Open .: %s", err) - } + t.Chdir(t.TempDir()) // Ensure wd is restored after the test. + // These are chosen carefully not to be symlinks on a Mac // (unlike, say, /var, /etc), except /tmp, which we handle below. dirs := []string{"/", "/usr/bin", "/tmp"} @@ -1627,16 +1635,16 @@ func TestChdirAndGetwd(t *testing.T) { dirs = nil for _, dir := range []string{t.TempDir(), t.TempDir()} { // Expand symlinks so path equality tests work. - dir, err = filepath.EvalSymlinks(dir) + dir, err := filepath.EvalSymlinks(dir) if err != nil { t.Fatalf("EvalSymlinks: %v", err) } dirs = append(dirs, dir) } } - oldwd := Getenv("PWD") for mode := 0; mode < 2; mode++ { for _, d := range dirs { + var err error if mode == 0 { err = Chdir(d) } else { @@ -1652,30 +1660,17 @@ func TestChdirAndGetwd(t *testing.T) { Setenv("PWD", "/tmp") } pwd, err1 := Getwd() - Setenv("PWD", oldwd) - err2 := fd.Chdir() - if err2 != nil { - // We changed the current directory and cannot go back. - // Don't let the tests continue; they'll scribble - // all over some other directory. - fmt.Fprintf(Stderr, "fchdir back to dot failed: %s\n", err2) - Exit(1) - } if err != nil { - fd.Close() t.Fatalf("Chdir %s: %s", d, err) } if err1 != nil { - fd.Close() t.Fatalf("Getwd in %s: %s", d, err1) } if !equal(pwd, d) { - fd.Close() t.Fatalf("Getwd returned %q want %q", pwd, d) } } } - fd.Close() } // Test that Chdir+Getwd is program-wide. @@ -1686,17 +1681,7 @@ func TestProgWideChdir(t *testing.T) { done := make(chan struct{}) d := t.TempDir() - oldwd, err := Getwd() - if err != nil { - t.Fatalf("Getwd: %v", err) - } - defer func() { - if err := Chdir(oldwd); err != nil { - // It's not safe to continue with tests if we can't get back to - // the original working directory. - panic(err) - } - }() + t.Chdir(d) // Note the deferred Wait must be called after the deferred close(done), // to ensure the N goroutines have been released even if the main goroutine @@ -1751,6 +1736,7 @@ func TestProgWideChdir(t *testing.T) { } }(i) } + var err error if err = Chdir(d); err != nil { t.Fatalf("Chdir: %v", err) } @@ -1796,13 +1782,6 @@ func TestSeek(t *testing.T) { for i, tt := range tests { off, err := f.Seek(tt.in, tt.whence) if off != tt.out || err != nil { - if e, ok := err.(*PathError); ok && e.Err == syscall.EINVAL && tt.out > 1<<32 && runtime.GOOS == "linux" { - mounts, _ := ReadFile("/proc/mounts") - if strings.Contains(string(mounts), "reiserfs") { - // Reiserfs rejects the big seeks. - t.Skipf("skipping test known to fail on reiserfs; https://golang.org/issue/91") - } - } t.Errorf("#%d: Seek(%v, %v) = %v, %v want %v, nil", i, tt.in, tt.whence, off, err, tt.out) } } @@ -1835,43 +1814,62 @@ func TestSeekError(t *testing.T) { } } -type openErrorTest struct { - path string - mode int - error error +func TestOpenError(t *testing.T) { + t.Parallel() + dir := makefs(t, []string{ + "is-a-file", + "is-a-dir/", + }) + t.Run("NoRoot", func(t *testing.T) { testOpenError(t, dir, false) }) + t.Run("InRoot", func(t *testing.T) { testOpenError(t, dir, true) }) } - -var openErrorTests = []openErrorTest{ - { - sfdir + "/no-such-file", +func testOpenError(t *testing.T, dir string, rooted bool) { + t.Parallel() + var r *Root + if rooted { + var err error + r, err = OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer r.Close() + } + for _, tt := range []struct { + path string + mode int + error error + }{{ + "no-such-file", O_RDONLY, syscall.ENOENT, - }, - { - sfdir, + }, { + "is-a-dir", O_WRONLY, syscall.EISDIR, - }, - { - sfdir + "/" + sfname + "/no-such-file", + }, { + "is-a-file/no-such-file", O_WRONLY, syscall.ENOTDIR, - }, -} - -func TestOpenError(t *testing.T) { - t.Parallel() - - for _, tt := range openErrorTests { - f, err := OpenFile(tt.path, tt.mode, 0) + }} { + var f *File + var err error + var name string + if rooted { + name = fmt.Sprintf("Root(%q).OpenFile(%q, %d)", dir, tt.path, tt.mode) + f, err = r.OpenFile(tt.path, tt.mode, 0) + } else { + path := filepath.Join(dir, tt.path) + name = fmt.Sprintf("OpenFile(%q, %d)", path, tt.mode) + f, err = OpenFile(path, tt.mode, 0) + } if err == nil { - t.Errorf("Open(%q, %d) succeeded", tt.path, tt.mode) + t.Errorf("%v succeeded", name) f.Close() continue } perr, ok := err.(*PathError) if !ok { - t.Errorf("Open(%q, %d) returns error of %T type; want *PathError", tt.path, tt.mode, err) + t.Errorf("%v returns error of %T type; want *PathError", name, err) } if perr.Err != tt.error { if runtime.GOOS == "plan9" { @@ -1879,12 +1877,14 @@ func TestOpenError(t *testing.T) { expectedErrStr := strings.Replace(tt.error.Error(), "file ", "", 1) if !strings.HasSuffix(syscallErrStr, expectedErrStr) { // Some Plan 9 file servers incorrectly return - // EACCES rather than EISDIR when a directory is + // EPERM or EACCES rather than EISDIR when a directory is // opened for write. - if tt.error == syscall.EISDIR && strings.HasSuffix(syscallErrStr, syscall.EACCES.Error()) { + if tt.error == syscall.EISDIR && + (strings.HasSuffix(syscallErrStr, syscall.EPERM.Error()) || + strings.HasSuffix(syscallErrStr, syscall.EACCES.Error())) { continue } - t.Errorf("Open(%q, %d) = _, %q; want suffix %q", tt.path, tt.mode, syscallErrStr, expectedErrStr) + t.Errorf("%v = _, %q; want suffix %q", name, syscallErrStr, expectedErrStr) } continue } @@ -1895,7 +1895,7 @@ func TestOpenError(t *testing.T) { continue } } - t.Errorf("Open(%q, %d) = _, %q; want %q", tt.path, tt.mode, perr.Err.Error(), tt.error.Error()) + t.Errorf("%v = _, %q; want %q", name, perr.Err.Error(), tt.error.Error()) } } } @@ -2081,20 +2081,58 @@ func TestWriteAt(t *testing.T) { f := newFile(t) - const data = "hello, world\n" + const data = "hello, world" io.WriteString(f, data) - n, err := f.WriteAt([]byte("WORLD"), 7) - if err != nil || n != 5 { + n, err := f.WriteAt([]byte("WOR"), 7) + if err != nil || n != 3 { t.Fatalf("WriteAt 7: %d, %v", n, err) } + n, err = io.WriteString(f, "!") // test that WriteAt doesn't change the file offset + if err != nil || n != 1 { + t.Fatal(err) + } + + got, err := ReadFile(f.Name()) + if err != nil { + t.Fatalf("ReadFile %s: %v", f.Name(), err) + } + want := "hello, WORld!" + if string(got) != want { + t.Fatalf("after write: have %q want %q", string(got), want) + } +} + +func TestWriteAtConcurrent(t *testing.T) { + t.Parallel() + + f := newFile(t) + io.WriteString(f, "0000000000") + + var wg sync.WaitGroup + for i := range 10 { + wg.Add(1) + go func() { + defer wg.Done() + n, err := f.WriteAt([]byte(strconv.Itoa(i)), int64(i)) + if err != nil || n != 1 { + t.Errorf("WriteAt %d: %d, %v", i, n, err) + } + n, err = io.WriteString(f, "!") // test that WriteAt doesn't change the file offset + if err != nil || n != 1 { + t.Error(err) + } + }() + } + wg.Wait() - b, err := ReadFile(f.Name()) + got, err := ReadFile(f.Name()) if err != nil { t.Fatalf("ReadFile %s: %v", f.Name(), err) } - if string(b) != "hello, WORLD\n" { - t.Fatalf("after write: have %q want %q", string(b), "hello, WORLD\n") + want := "0123456789!!!!!!!!!!" + if string(got) != want { + t.Fatalf("after write: have %q want %q", string(got), want) } } @@ -2114,7 +2152,7 @@ func TestWriteAtNegativeOffset(t *testing.T) { // Verify that WriteAt doesn't work in append mode. func TestWriteAtInAppendMode(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) f, err := OpenFile("write_at_in_append_mode.txt", O_APPEND|O_CREATE, 0666) if err != nil { t.Fatalf("OpenFile: %v", err) @@ -2127,8 +2165,15 @@ func TestWriteAtInAppendMode(t *testing.T) { } } -func writeFile(t *testing.T, fname string, flag int, text string) string { - f, err := OpenFile(fname, flag, 0666) +func writeFile(t *testing.T, r *Root, fname string, flag int, text string) string { + t.Helper() + var f *File + var err error + if r == nil { + f, err = OpenFile(fname, flag, 0666) + } else { + f, err = r.OpenFile(fname, flag, 0666) + } if err != nil { t.Fatalf("Open: %v", err) } @@ -2145,35 +2190,206 @@ func writeFile(t *testing.T, fname string, flag int, text string) string { } func TestAppend(t *testing.T) { - defer chtmpdir(t)() - const f = "append.txt" - s := writeFile(t, f, O_CREATE|O_TRUNC|O_RDWR, "new") - if s != "new" { - t.Fatalf("writeFile: have %q want %q", s, "new") - } - s = writeFile(t, f, O_APPEND|O_RDWR, "|append") - if s != "new|append" { - t.Fatalf("writeFile: have %q want %q", s, "new|append") - } - s = writeFile(t, f, O_CREATE|O_APPEND|O_RDWR, "|append") - if s != "new|append|append" { - t.Fatalf("writeFile: have %q want %q", s, "new|append|append") - } - err := Remove(f) - if err != nil { - t.Fatalf("Remove: %v", err) - } - s = writeFile(t, f, O_CREATE|O_APPEND|O_RDWR, "new&append") - if s != "new&append" { - t.Fatalf("writeFile: after append have %q want %q", s, "new&append") + testMaybeRooted(t, func(t *testing.T, r *Root) { + const f = "append.txt" + s := writeFile(t, r, f, O_CREATE|O_TRUNC|O_RDWR, "new") + if s != "new" { + t.Fatalf("writeFile: have %q want %q", s, "new") + } + s = writeFile(t, r, f, O_APPEND|O_RDWR, "|append") + if s != "new|append" { + t.Fatalf("writeFile: have %q want %q", s, "new|append") + } + s = writeFile(t, r, f, O_CREATE|O_APPEND|O_RDWR, "|append") + if s != "new|append|append" { + t.Fatalf("writeFile: have %q want %q", s, "new|append|append") + } + err := Remove(f) + if err != nil { + t.Fatalf("Remove: %v", err) + } + s = writeFile(t, r, f, O_CREATE|O_APPEND|O_RDWR, "new&append") + if s != "new&append" { + t.Fatalf("writeFile: after append have %q want %q", s, "new&append") + } + s = writeFile(t, r, f, O_CREATE|O_RDWR, "old") + if s != "old&append" { + t.Fatalf("writeFile: after create have %q want %q", s, "old&append") + } + s = writeFile(t, r, f, O_CREATE|O_TRUNC|O_RDWR, "new") + if s != "new" { + t.Fatalf("writeFile: after truncate have %q want %q", s, "new") + } + }) +} + +// TestFilePermissions tests setting Unix permission bits on file creation. +func TestFilePermissions(t *testing.T) { + if Getuid() == 0 { + t.Skip("skipping test when running as root") } - s = writeFile(t, f, O_CREATE|O_RDWR, "old") - if s != "old&append" { - t.Fatalf("writeFile: after create have %q want %q", s, "old&append") + for _, test := range []struct { + name string + mode FileMode + }{ + {"r", 0o444}, + {"w", 0o222}, + {"rw", 0o666}, + } { + t.Run(test.name, func(t *testing.T) { + switch runtime.GOOS { + case "windows": + if test.mode&0444 == 0 { + t.Skip("write-only files not supported on " + runtime.GOOS) + } + case "wasip1": + t.Skip("file permissions not supported on " + runtime.GOOS) + } + testMaybeRooted(t, func(t *testing.T, r *Root) { + const filename = "f" + var f *File + var err error + if r == nil { + f, err = OpenFile(filename, O_RDWR|O_CREATE|O_EXCL, test.mode) + } else { + f, err = r.OpenFile(filename, O_RDWR|O_CREATE|O_EXCL, test.mode) + } + if err != nil { + t.Fatal(err) + } + f.Close() + b, err := ReadFile(filename) + if test.mode&0o444 != 0 { + if err != nil { + t.Errorf("ReadFile = %v; want success", err) + } + } else { + if err == nil { + t.Errorf("ReadFile = %q, ; want failure", string(b)) + } + } + _, err = Stat(filename) + if err != nil { + t.Errorf("Stat = %v; want success", err) + } + err = WriteFile(filename, nil, 0666) + if test.mode&0o222 != 0 { + if err != nil { + t.Errorf("WriteFile = %v; want success", err) + b, err := ReadFile(filename) + t.Errorf("ReadFile: %v", err) + t.Errorf("file contents: %q", b) + } + } else { + if err == nil { + t.Errorf("WriteFile(%q) = ; want failure", filename) + st, err := Stat(filename) + if err == nil { + t.Errorf("mode: %s", st.Mode()) + } + b, err := ReadFile(filename) + t.Errorf("ReadFile: %v", err) + t.Errorf("file contents: %q", b) + } + } + }) + }) } - s = writeFile(t, f, O_CREATE|O_TRUNC|O_RDWR, "new") - if s != "new" { - t.Fatalf("writeFile: after truncate have %q want %q", s, "new") + +} + +func TestOpenFileCreateExclDanglingSymlink(t *testing.T) { + testenv.MustHaveSymlink(t) + testMaybeRooted(t, func(t *testing.T, r *Root) { + const link = "link" + if err := Symlink("does_not_exist", link); err != nil { + t.Fatal(err) + } + var f *File + var err error + if r == nil { + f, err = OpenFile(link, O_WRONLY|O_CREATE|O_EXCL, 0o444) + } else { + f, err = r.OpenFile(link, O_WRONLY|O_CREATE|O_EXCL, 0o444) + } + if err == nil { + f.Close() + } + if !errors.Is(err, ErrExist) { + t.Errorf("OpenFile of a dangling symlink with O_CREATE|O_EXCL = %v, want ErrExist", err) + } + if _, err := Stat(link); err == nil { + t.Errorf("OpenFile of a dangling symlink with O_CREATE|O_EXCL created a file") + } + }) +} + +// TestFileRDWRFlags tests the O_RDONLY, O_WRONLY, and O_RDWR flags. +func TestFileRDWRFlags(t *testing.T) { + for _, test := range []struct { + name string + flag int + }{ + {"O_RDONLY", O_RDONLY}, + {"O_WRONLY", O_WRONLY}, + {"O_RDWR", O_RDWR}, + } { + t.Run(test.name, func(t *testing.T) { + testMaybeRooted(t, func(t *testing.T, r *Root) { + const filename = "f" + content := []byte("content") + if err := WriteFile(filename, content, 0666); err != nil { + t.Fatal(err) + } + var f *File + var err error + if r == nil { + f, err = OpenFile(filename, test.flag, 0) + } else { + f, err = r.OpenFile(filename, test.flag, 0) + } + if err != nil { + t.Fatal(err) + } + defer f.Close() + got, err := io.ReadAll(f) + if test.flag == O_WRONLY { + if err == nil { + t.Errorf("read file: %q, %v; want error", got, err) + } + } else { + if err != nil || !bytes.Equal(got, content) { + t.Errorf("read file: %q, %v; want %q, ", got, err, content) + } + } + if _, err := f.Seek(0, 0); err != nil { + t.Fatalf("f.Seek: %v", err) + } + newcontent := []byte("CONTENT") + _, err = f.Write(newcontent) + if test.flag == O_RDONLY { + if err == nil { + t.Errorf("write file: succeeded, want error") + } + } else { + if err != nil { + t.Errorf("write file: %v, want success", err) + } + } + f.Close() + got, err = ReadFile(filename) + if err != nil { + t.Fatal(err) + } + want := content + if test.flag != O_RDONLY { + want = newcontent + } + if !bytes.Equal(got, want) { + t.Fatalf("after write, file contains %q, want %q", got, want) + } + }) + }) } } @@ -2204,7 +2420,7 @@ func TestNilProcessStateString(t *testing.T) { } func TestSameFile(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) fa, err := Create("a") if err != nil { t.Fatalf("Create(a): %v", err) @@ -2347,13 +2563,8 @@ func TestStatStdin(t *testing.T) { Exit(0) } - exe, err := Executable() - if err != nil { - t.Skipf("can't find executable: %v", err) - } - - testenv.MustHaveExec(t) t.Parallel() + exe := testenv.Executable(t) fi, err := Stdin.Stat() if err != nil { @@ -2519,11 +2730,10 @@ func TestLongPath(t *testing.T) { } func testKillProcess(t *testing.T, processKiller func(p *Process)) { - testenv.MustHaveExec(t) t.Parallel() // Re-exec the test binary to start a process that hangs until stdin is closed. - cmd := testenv.Command(t, Args[0]) + cmd := testenv.Command(t, testenv.Executable(t)) cmd.Env = append(cmd.Environ(), "GO_OS_TEST_DRAIN_STDIN=1") stdout, err := cmd.StdoutPipe() if err != nil { @@ -2572,10 +2782,9 @@ func TestGetppid(t *testing.T) { Exit(0) } - testenv.MustHaveExec(t) t.Parallel() - cmd := testenv.Command(t, Args[0], "-test.run=^TestGetppid$") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^TestGetppid$") cmd.Env = append(Environ(), "GO_WANT_HELPER_PROCESS=1") // verify that Getppid() from the forked process reports our process id @@ -2668,10 +2877,7 @@ func TestRemoveAllRace(t *testing.T) { n := runtime.GOMAXPROCS(16) defer runtime.GOMAXPROCS(n) - root, err := MkdirTemp("", "issue") - if err != nil { - t.Fatal(err) - } + root := t.TempDir() mkdirTree(t, root, 1, 6) hold := make(chan struct{}) var wg sync.WaitGroup @@ -2693,6 +2899,8 @@ func TestRemoveAllRace(t *testing.T) { // Test that reading from a pipe doesn't use up a thread. func TestPipeThreads(t *testing.T) { switch runtime.GOOS { + case "aix": + t.Skip("skipping on aix; issue 70131") case "illumos", "solaris": t.Skip("skipping on Solaris and illumos; issue 19111") case "windows": @@ -2707,11 +2915,6 @@ func TestPipeThreads(t *testing.T) { threads := 100 - // OpenBSD has a low default for max number of files. - if runtime.GOOS == "openbsd" { - threads = 50 - } - r := make([]*File, threads) w := make([]*File, threads) for i := 0; i < threads; i++ { @@ -3007,6 +3210,22 @@ func isDeadlineExceeded(err error) bool { // Test that opening a file does not change its permissions. Issue 38225. func TestOpenFileKeepsPermissions(t *testing.T) { + t.Run("OpenFile", func(t *testing.T) { + testOpenFileKeepsPermissions(t, OpenFile) + }) + t.Run("RootOpenFile", func(t *testing.T) { + testOpenFileKeepsPermissions(t, func(name string, flag int, perm FileMode) (*File, error) { + dir, file := filepath.Split(name) + r, err := OpenRoot(dir) + if err != nil { + return nil, err + } + defer r.Close() + return r.OpenFile(file, flag, perm) + }) + }) +} +func testOpenFileKeepsPermissions(t *testing.T, openf func(name string, flag int, perm FileMode) (*File, error)) { t.Parallel() dir := t.TempDir() @@ -3018,7 +3237,7 @@ func TestOpenFileKeepsPermissions(t *testing.T) { if err := f.Close(); err != nil { t.Error(err) } - f, err = OpenFile(name, O_WRONLY|O_CREATE|O_TRUNC, 0) + f, err = openf(name, O_WRONLY|O_CREATE|O_TRUNC, 0) if err != nil { t.Fatal(err) } @@ -3072,10 +3291,22 @@ func forceMFTUpdateOnWindows(t *testing.T, path string) { func TestDirFS(t *testing.T) { t.Parallel() + testDirFS(t, DirFS("./testdata/dirfs")) +} +func TestRootDirFS(t *testing.T) { + t.Parallel() + r, err := OpenRoot("./testdata/dirfs") + if err != nil { + t.Fatal(err) + } + defer r.Close() + testDirFS(t, r.FS()) +} + +func testDirFS(t *testing.T, fsys fs.FS) { forceMFTUpdateOnWindows(t, "./testdata/dirfs") - fsys := DirFS("./testdata/dirfs") if err := fstest.TestFS(fsys, "a", "b", "dir/x"); err != nil { t.Fatal(err) } @@ -3390,27 +3621,18 @@ func TestCopyFS(t *testing.T) { if err := fstest.TestFS(tmpFsys, "a", "b", "dir/x"); err != nil { t.Fatal("TestFS:", err) } - if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error { - if d.IsDir() { - return nil - } - - data, err := fs.ReadFile(fsys, path) - if err != nil { - return err - } - newData, err := fs.ReadFile(tmpFsys, path) - if err != nil { - return err - } - if !bytes.Equal(data, newData) { - return errors.New("file " + path + " contents differ") - } - return nil - }); err != nil { + if err := verifyCopyFS(t, fsys, tmpFsys); err != nil { t.Fatal("comparing two directories:", err) } + // Test whether CopyFS disallows copying for disk filesystem when there is any + // existing file in the destination directory. + if err := CopyFS(tmpDir, fsys); !errors.Is(err, fs.ErrExist) { + t.Errorf("CopyFS should have failed and returned error when there is"+ + "any existing file in the destination directory (in disk filesystem), "+ + "got: %v, expected any error that indicates ", err) + } + // Test with memory filesystem. fsys = fstest.MapFS{ "william": {Data: []byte("Shakespeare\n")}, @@ -3428,26 +3650,111 @@ func TestCopyFS(t *testing.T) { if err := fstest.TestFS(tmpFsys, "william", "carl", "daVinci", "einstein", "dir/newton"); err != nil { t.Fatal("TestFS:", err) } - if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error { + if err := verifyCopyFS(t, fsys, tmpFsys); err != nil { + t.Fatal("comparing two directories:", err) + } + + // Test whether CopyFS disallows copying for memory filesystem when there is any + // existing file in the destination directory. + if err := CopyFS(tmpDir, fsys); !errors.Is(err, fs.ErrExist) { + t.Errorf("CopyFS should have failed and returned error when there is"+ + "any existing file in the destination directory (in memory filesystem), "+ + "got: %v, expected any error that indicates ", err) + } +} + +// verifyCopyFS checks the content and permission of each file inside copied FS to ensure +// the copied files satisfy the convention stipulated in CopyFS. +func verifyCopyFS(t *testing.T, originFS, copiedFS fs.FS) error { + testDir := filepath.Join(t.TempDir(), "test") + // umask doesn't apply to the wasip and windows and there is no general way to get masked perm, + // so create a dir and a file to compare the permission after umask if any + if err := Mkdir(testDir, ModePerm); err != nil { + return fmt.Errorf("mkdir %q failed: %v", testDir, err) + } + dirStat, err := Stat(testDir) + if err != nil { + return fmt.Errorf("stat dir %q failed: %v", testDir, err) + } + wantDirMode := dirStat.Mode() + + f, err := Create(filepath.Join(testDir, "tmp")) + if err != nil { + return fmt.Errorf("open %q failed: %v", filepath.Join(testDir, "tmp"), err) + } + defer f.Close() + wantFileRWStat, err := f.Stat() + if err != nil { + return fmt.Errorf("stat file %q failed: %v", f.Name(), err) + } + wantFileRWMode := wantFileRWStat.Mode() + + return fs.WalkDir(originFS, ".", func(path string, d fs.DirEntry, err error) error { if d.IsDir() { + // the dir . is not the dir created by CopyFS so skip checking its permission + if d.Name() == "." { + return nil + } + + dinfo, err := fs.Stat(copiedFS, path) + if err != nil { + return err + } + + if dinfo.Mode() != wantDirMode { + return fmt.Errorf("dir %q mode is %v, want %v", + d.Name(), dinfo.Mode(), wantDirMode) + } return nil } - data, err := fs.ReadFile(fsys, path) + fInfo, err := originFS.Open(path) if err != nil { return err } - newData, err := fs.ReadFile(tmpFsys, path) + defer fInfo.Close() + copiedInfo, err := copiedFS.Open(path) + if err != nil { + return err + } + defer copiedInfo.Close() + + // verify the file contents are the same + data, err := io.ReadAll(fInfo) + if err != nil { + return err + } + newData, err := io.ReadAll(copiedInfo) if err != nil { return err } if !bytes.Equal(data, newData) { - return errors.New("file " + path + " contents differ") + return fmt.Errorf("file %q content is %s, want %s", path, newData, data) + } + + fStat, err := fInfo.Stat() + if err != nil { + return err + } + copiedStat, err := copiedInfo.Stat() + if err != nil { + return err + } + + // check whether the execute permission is inherited from original FS + + if copiedStat.Mode()&0111&wantFileRWMode != fStat.Mode()&0111&wantFileRWMode { + return fmt.Errorf("file %q execute mode is %v, want %v", + path, copiedStat.Mode()&0111, fStat.Mode()&0111) + } + + rwMode := copiedStat.Mode() &^ 0111 // unset the executable permission from file mode + if rwMode != wantFileRWMode { + return fmt.Errorf("file %q rw mode is %v, want %v", + path, rwMode, wantFileRWStat.Mode()) } return nil - }); err != nil { - t.Fatal("comparing two directories:", err) - } + }) } func TestCopyFSWithSymlinks(t *testing.T) { @@ -3456,9 +3763,9 @@ func TestCopyFSWithSymlinks(t *testing.T) { // Create a directory and file outside. tmpDir := t.TempDir() - outsideDir, err := MkdirTemp(tmpDir, "copyfs_out_") - if err != nil { - t.Fatalf("MkdirTemp: %v", err) + outsideDir := filepath.Join(tmpDir, "copyfs_out") + if err := Mkdir(outsideDir, 0755); err != nil { + t.Fatalf("Mkdir: %v", err) } outsideFile := filepath.Join(outsideDir, "file.out.txt") @@ -3467,9 +3774,9 @@ func TestCopyFSWithSymlinks(t *testing.T) { } // Create a directory and file inside. - insideDir, err := MkdirTemp(tmpDir, "copyfs_in_") - if err != nil { - t.Fatalf("MkdirTemp: %v", err) + insideDir := filepath.Join(tmpDir, "copyfs_in") + if err := Mkdir(insideDir, 0755); err != nil { + t.Fatalf("Mkdir: %v", err) } insideFile := filepath.Join(insideDir, "file.in.txt") if err := WriteFile(insideFile, []byte("Testing CopyFS inside"), 0644); err != nil { @@ -3515,18 +3822,14 @@ func TestCopyFSWithSymlinks(t *testing.T) { // Copy the directory tree and verify. forceMFTUpdateOnWindows(t, insideDir) fsys := DirFS(insideDir) - tmpDupDir, err := MkdirTemp(tmpDir, "copyfs_dup_") - if err != nil { - t.Fatalf("MkdirTemp: %v", err) + tmpDupDir := filepath.Join(tmpDir, "copyfs_dup") + if err := Mkdir(tmpDupDir, 0755); err != nil { + t.Fatalf("Mkdir: %v", err) } - // TODO(panjf2000): symlinks are currently not supported, and a specific error - // will be returned. Verify that error and skip the subsequent test, - // revisit this once #49580 is closed. - if err := CopyFS(tmpDupDir, fsys); !errors.Is(err, ErrInvalid) { - t.Fatalf("got %v, want ErrInvalid", err) + if err := CopyFS(tmpDupDir, fsys); err != nil { + t.Fatalf("CopyFS: %v", err) } - t.Skip("skip the subsequent test and wait for #49580") forceMFTUpdateOnWindows(t, tmpDupDir) tmpFsys := DirFS(tmpDupDir) @@ -3589,3 +3892,164 @@ func TestCopyFSWithSymlinks(t *testing.T) { t.Fatal("comparing two directories:", err) } } + +func TestAppendDoesntOverwrite(t *testing.T) { + testMaybeRooted(t, func(t *testing.T, r *Root) { + name := "file" + if err := WriteFile(name, []byte("hello"), 0666); err != nil { + t.Fatal(err) + } + var f *File + var err error + if r == nil { + f, err = OpenFile(name, O_APPEND|O_WRONLY, 0) + } else { + f, err = r.OpenFile(name, O_APPEND|O_WRONLY, 0) + } + if err != nil { + t.Fatal(err) + } + if _, err := f.Write([]byte(" world")); err != nil { + f.Close() + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + got, err := ReadFile(name) + if err != nil { + t.Fatal(err) + } + want := "hello world" + if string(got) != want { + t.Fatalf("got %q, want %q", got, want) + } + }) +} + +func TestRemoveReadOnlyFile(t *testing.T) { + testMaybeRooted(t, func(t *testing.T, r *Root) { + if err := WriteFile("file", []byte("1"), 0); err != nil { + t.Fatal(err) + } + var err error + if r == nil { + err = Remove("file") + } else { + err = r.Remove("file") + } + if err != nil { + t.Fatalf("Remove read-only file: %v", err) + } + if _, err := Stat("file"); !IsNotExist(err) { + t.Fatalf("Stat read-only file after removal: %v (want IsNotExist)", err) + } + }) +} + +func TestOpenFileDevNull(t *testing.T) { + // See https://go.dev/issue/71752. + t.Parallel() + + f, err := OpenFile(DevNull, O_WRONLY|O_CREATE|O_TRUNC, 0o644) + if err != nil { + t.Fatalf("OpenFile(DevNull): %v", err) + } + f.Close() +} + +func TestReadFileContents(t *testing.T) { + type readStep struct { + bufSize int // non-zero to check length of buf to Read + retN int // result of Read call + retErr error // error result of Read call + } + errFoo := errors.New("foo") + tests := []struct { + name string + statSize int64 // size of file to read, per stat (may be 0 for /proc files) + wantSize int // wanted length of []byte from readFileContents + wantErr error // wanted error from readFileContents + reads []readStep + }{ + { + name: "big-file", + statSize: 2000, + wantSize: 2000, + reads: []readStep{ + {bufSize: 2001, retN: 21, retErr: nil}, + {bufSize: 1980, retN: 1979, retErr: io.EOF}, + }, + }, + { + name: "small-file", + statSize: 100, + wantSize: 100, + reads: []readStep{ + {bufSize: 512, retN: 100, retErr: io.EOF}, + }, + }, + { + name: "returning-error", + statSize: 1000, + wantSize: 50, + wantErr: errFoo, + reads: []readStep{ + {bufSize: 1001, retN: 25, retErr: nil}, + {retN: 25, retErr: errFoo}, + }, + }, + { + name: "proc-file", + statSize: 0, + wantSize: 1023, + reads: []readStep{ + {bufSize: 512, retN: 512, retErr: nil}, + {retN: 511, retErr: io.EOF}, + }, + }, + { + name: "plan9-iproute-file", // Issue 72080 + statSize: 0, + wantSize: 1032, + reads: []readStep{ + {bufSize: 512, retN: 511, retErr: nil}, + {retN: 511, retErr: nil}, + {retN: 10, retErr: io.EOF}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + remain := tt.reads + i := -1 + got, err := ExportReadFileContents(tt.statSize, func(buf []byte) (int, error) { + i++ + t.Logf("read[%d] with buf size %d", i, len(buf)) + if len(remain) == 0 { + t.Fatalf("unexpected read of length %d after %d expected reads", len(buf), len(tt.reads)) + } + if tt.statSize == 0 && len(buf) < 512 { + // Issue 72080: readFileContents should not do /proc reads with buffers + // smaller than 512. + t.Fatalf("read[%d] with buf size %d; want at least 512 for 0-sized file", i, len(buf)) + } + step := remain[0] + remain = remain[1:] + if step.bufSize != 0 && len(buf) != step.bufSize { + t.Fatalf("read[%d] has buffer size %d; want %d", i, len(buf), step.bufSize) + } + return step.retN, step.retErr + }) + if len(remain) > 0 { + t.Fatalf("expected %d reads, got %d", len(tt.reads), i+1) + } + if fmt.Sprint(err) != fmt.Sprint(tt.wantErr) { + t.Errorf("got error %v; want %v", err, tt.wantErr) + } + if len(got) != tt.wantSize { + t.Errorf("got size %d; want %d", len(got), tt.wantSize) + } + }) + } +} diff --git a/src/os/os_unix_test.go b/src/os/os_unix_test.go index fcc75e5ee61360..41feaf77e2cea4 100644 --- a/src/os/os_unix_test.go +++ b/src/os/os_unix_test.go @@ -64,7 +64,7 @@ func TestChown(t *testing.T) { // Then try all the auxiliary groups. groups, err := Getgroups() if err != nil { - t.Fatalf("getgroups: %s", err) + t.Fatal(err) } t.Log("groups: ", groups) for _, g := range groups { @@ -112,7 +112,7 @@ func TestFileChown(t *testing.T) { // Then try all the auxiliary groups. groups, err := Getgroups() if err != nil { - t.Fatalf("getgroups: %s", err) + t.Fatal(err) } t.Log("groups: ", groups) for _, g := range groups { @@ -170,7 +170,7 @@ func TestLchown(t *testing.T) { // Then try all the auxiliary groups. groups, err := Getgroups() if err != nil { - t.Fatalf("getgroups: %s", err) + t.Fatal(err) } t.Log("groups: ", groups) for _, g := range groups { @@ -233,7 +233,9 @@ func TestMkdirStickyUmask(t *testing.T) { if runtime.GOOS == "wasip1" { t.Skip("file permissions not supported on " + runtime.GOOS) } - t.Parallel() + // Issue #69788: This test temporarily changes the umask for testing purposes, + // so it shouldn't be run in parallel with other test cases + // to avoid other tests (e.g., TestCopyFS) creating files with an unintended umask. const umask = 0077 dir := t.TempDir() @@ -372,7 +374,7 @@ func TestSplitPath(t *testing.T) { // // Regression test for go.dev/issue/60181 func TestIssue60181(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) want := "hello gopher" diff --git a/src/os/os_windows_test.go b/src/os/os_windows_test.go index 4a8d0d07ef75af..cd2413d26d4a53 100644 --- a/src/os/os_windows_test.go +++ b/src/os/os_windows_test.go @@ -5,6 +5,7 @@ package os_test import ( + "bytes" "errors" "fmt" "internal/godebug" @@ -19,9 +20,13 @@ import ( "path/filepath" "runtime" "slices" + "strconv" "strings" + "sync" + "sync/atomic" "syscall" "testing" + "time" "unicode/utf16" "unsafe" ) @@ -32,28 +37,8 @@ var winreadlinkvolume = godebug.New("winreadlinkvolume") // For TestRawConnReadWrite. type syscallDescriptor = syscall.Handle -// chdir changes the current working directory to the named directory, -// and then restore the original working directory at the end of the test. -func chdir(t *testing.T, dir string) { - olddir, err := os.Getwd() - if err != nil { - t.Fatalf("chdir: %v", err) - } - if err := os.Chdir(dir); err != nil { - t.Fatalf("chdir %s: %v", dir, err) - } - - t.Cleanup(func() { - if err := os.Chdir(olddir); err != nil { - t.Errorf("chdir to original working directory %s: %v", olddir, err) - os.Exit(1) - } - }) -} - func TestSameWindowsFile(t *testing.T) { - temp := t.TempDir() - chdir(t, temp) + t.Chdir(t.TempDir()) f, err := os.Create("a") if err != nil { @@ -79,9 +64,6 @@ func TestSameWindowsFile(t *testing.T) { } p := filepath.VolumeName(path) + filepath.Base(path) - if err != nil { - t.Fatal(err) - } ia3, err := os.Stat(p) if err != nil { t.Fatal(err) @@ -99,7 +81,7 @@ type dirLinkTest struct { func testDirLinks(t *testing.T, tests []dirLinkTest) { tmpdir := t.TempDir() - chdir(t, tmpdir) + t.Chdir(tmpdir) dir := filepath.Join(tmpdir, "dir") err := os.Mkdir(dir, 0777) @@ -458,7 +440,7 @@ func TestNetworkSymbolicLink(t *testing.T) { const _NERR_ServerNotStarted = syscall.Errno(2114) dir := t.TempDir() - chdir(t, dir) + t.Chdir(dir) pid := os.Getpid() shareName := fmt.Sprintf("GoSymbolicLinkTestShare%d", pid) @@ -561,8 +543,7 @@ func TestStatLxSymLink(t *testing.T) { t.Skip("skipping: WSL not detected") } - temp := t.TempDir() - chdir(t, temp) + t.Chdir(t.TempDir()) const target = "target" const link = "link" @@ -629,7 +610,7 @@ func TestBadNetPathError(t *testing.T) { } func TestStatDir(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) f, err := os.Open(".") if err != nil { @@ -659,7 +640,7 @@ func TestStatDir(t *testing.T) { func TestOpenVolumeName(t *testing.T) { tmpdir := t.TempDir() - chdir(t, tmpdir) + t.Chdir(tmpdir) want := []string{"file1", "file2", "file3", "gopher.txt"} slices.Sort(want) @@ -682,7 +663,7 @@ func TestOpenVolumeName(t *testing.T) { } slices.Sort(have) - if strings.Join(want, "/") != strings.Join(have, "/") { + if !slices.Equal(want, have) { t.Fatalf("unexpected file list %q, want %q", have, want) } } @@ -952,7 +933,9 @@ func findOneDriveDir() (string, error) { return "", fmt.Errorf("reading UserFolder failed: %v", err) } - if valtype == registry.EXPAND_SZ { + // REG_SZ values may also contain environment variables that need to be expanded. + // It's recommended but not required to use REG_EXPAND_SZ for paths that contain environment variables. + if valtype == registry.EXPAND_SZ || valtype == registry.SZ { expanded, err := registry.ExpandString(path) if err != nil { return "", fmt.Errorf("expanding UserFolder failed: %v", err) @@ -1011,6 +994,8 @@ func TestFileStatNUL(t *testing.T) { if err != nil { t.Fatal(err) } + defer f.Close() + fi, err := f.Stat() if err != nil { t.Fatal(err) @@ -1036,8 +1021,8 @@ func TestStatNUL(t *testing.T) { // works on Windows when developer mode is active. // This is supported starting Windows 10 (1703, v10.0.14972). func TestSymlinkCreation(t *testing.T) { - if !testenv.HasSymlink() && !isWindowsDeveloperModeActive() { - t.Skip("Windows developer mode is not active") + if !testenv.HasSymlink() { + t.Skip("skipping test; no symlink support") } t.Parallel() @@ -1053,23 +1038,6 @@ func TestSymlinkCreation(t *testing.T) { } } -// isWindowsDeveloperModeActive checks whether or not the developer mode is active on Windows 10. -// Returns false for prior Windows versions. -// see https://docs.microsoft.com/en-us/windows/uwp/get-started/enable-your-device-for-development -func isWindowsDeveloperModeActive() bool { - key, err := registry.OpenKey(registry.LOCAL_MACHINE, "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\AppModelUnlock", registry.READ) - if err != nil { - return false - } - - val, _, err := key.GetIntegerValue("AllowDevelopmentWithoutDevLicense") - if err != nil { - return false - } - - return val != 0 -} - // TestRootRelativeDirSymlink verifies that symlinks to paths relative to the // drive root (beginning with "\" but no volume name) are created with the // correct symlink type. @@ -1129,14 +1097,7 @@ func TestWorkingDirectoryRelativeSymlink(t *testing.T) { if err != nil { t.Fatal(err) } - defer func() { - if err := os.Chdir(oldwd); err != nil { - t.Fatal(err) - } - }() - if err := os.Chdir(temp); err != nil { - t.Fatal(err) - } + t.Chdir(temp) t.Logf("Chdir(%#q)", temp) wdRelDir := filepath.VolumeName(temp) + `dir\sub` // no backslash after volume. @@ -1221,10 +1182,7 @@ func TestRootDirAsTemp(t *testing.T) { testenv.MustHaveExec(t) t.Parallel() - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } + exe := testenv.Executable(t) newtmp, err := findUnusedDriveLetter() if err != nil { @@ -1300,6 +1258,9 @@ func TestReadlink(t *testing.T) { } t.Run(name, func(t *testing.T) { + if !tt.junction { + testenv.MustHaveSymlink(t) + } if !tt.relative { t.Parallel() } @@ -1327,7 +1288,7 @@ func TestReadlink(t *testing.T) { } else { want = relTarget } - chdir(t, tmpdir) + t.Chdir(tmpdir) link = filepath.Base(link) target = relTarget } else { @@ -1604,3 +1565,713 @@ func TestReadDirNoFileID(t *testing.T) { t.Errorf("SameFile(%v, %v) = false; want true", f2, f2s) } } + +func TestReadWriteFileOverlapped(t *testing.T) { + // See https://go.dev/issue/15388. + t.Parallel() + + name := filepath.Join(t.TempDir(), "test.txt") + f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|windows.O_FILE_FLAG_OVERLAPPED, 0666) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + data := []byte("test") + n, err := f.Write(data) + if err != nil { + t.Fatal(err) + } + if n != len(data) { + t.Fatalf("Write = %d; want %d", n, len(data)) + } + + if _, err := f.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } + + got, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Fatalf("Read = %q; want %q", got, data) + } +} + +func TestStdinOverlappedPipe(t *testing.T) { + // Test that we can read from a named pipe open with FILE_FLAG_OVERLAPPED. + // See https://go.dev/issue/15388. + if os.Getenv("GO_WANT_HELPER_PROCESS") == "1" { + var buf string + _, err := fmt.Scanln(&buf) + if err != nil { + fmt.Print(err) + os.Exit(1) + } + fmt.Println(buf) + os.Exit(0) + } + + t.Parallel() + name := pipeName() + + // Create the read handle inherited by the child process. + r := newPipe(t, name, false, true) + defer r.Close() + + // Create a write handle. + w, err := os.OpenFile(name, os.O_WRONLY, 0666) + if err != nil { + t.Fatal(err) + } + defer w.Close() + + // Write some data to the pipe. The child process will read it. + want := []byte("test\n") + if _, err := w.Write(want); err != nil { + t.Fatal(err) + } + + // Create a child process that will read from the pipe + // and write the data to stdout. + cmd := testenv.Command(t, testenv.Executable(t), fmt.Sprintf("-test.run=^%s$", t.Name()), "-test.v") + cmd = testenv.CleanCmdEnv(cmd) + cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=1") + cmd.Stdin = r + got, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("running %q failed: %v\n%s", cmd, err, got) + } + + if !bytes.Contains(got, want) { + t.Fatalf("output %q does not contain %q", got, want) + } +} + +func newFileOverlapped(t testing.TB, name string, overlapped bool) *os.File { + flags := os.O_RDWR | os.O_CREATE + if overlapped { + flags |= windows.O_FILE_FLAG_OVERLAPPED + } + f, err := os.OpenFile(name, flags, 0666) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := f.Close(); err != nil && !errors.Is(err, os.ErrClosed) { + t.Fatal(err) + } + }) + return f +} + +var currentProcess = sync.OnceValue(func() string { + // Convert the process ID to a string. + return strconv.FormatUint(uint64(os.Getpid()), 10) +}) + +var pipeCounter atomic.Uint64 + +func newBytePipe(t testing.TB, name string, overlapped bool) *os.File { + return newPipe(t, name, false, overlapped) +} + +func newMessagePipe(t testing.TB, name string, overlapped bool) *os.File { + return newPipe(t, name, true, overlapped) +} + +func pipeName() string { + return `\\.\pipe\go-os-test-` + currentProcess() + `-` + strconv.FormatUint(pipeCounter.Add(1), 10) +} + +func newPipe(t testing.TB, name string, message, overlapped bool) *os.File { + wname, err := syscall.UTF16PtrFromString(name) + if err != nil { + t.Fatal(err) + } + // Create the read handle. + flags := windows.PIPE_ACCESS_DUPLEX + if overlapped { + flags |= windows.O_FILE_FLAG_OVERLAPPED + } + typ := windows.PIPE_TYPE_BYTE | windows.PIPE_READMODE_BYTE + if message { + typ = windows.PIPE_TYPE_MESSAGE | windows.PIPE_READMODE_MESSAGE + } + h, err := windows.CreateNamedPipe(wname, uint32(flags), uint32(typ), 1, 4096, 4096, 0, nil) + if err != nil { + t.Fatal(err) + } + f := os.NewFile(uintptr(h), name) + t.Cleanup(func() { + if err := f.Close(); err != nil && !errors.Is(err, os.ErrClosed) { + t.Fatal(err) + } + }) + return f +} + +func testReadWrite(t *testing.T, fdr, fdw *os.File) { + write := make(chan string, 1) + read := make(chan struct{}, 1) + go func() { + for s := range write { + n, err := fdw.Write([]byte(s)) + read <- struct{}{} + if err != nil { + t.Error(err) + } + if n != len(s) { + t.Errorf("expected to write %d bytes, got %d", len(s), n) + } + } + }() + for i := range 10 { + s := strconv.Itoa(i) + write <- s + <-read + buf := make([]byte, len(s)) + _, err := io.ReadFull(fdr, buf) + if err != nil { + t.Fatalf("read failed: %v", err) + } + if !bytes.Equal(buf, []byte(s)) { + t.Fatalf("expected %q, got %q", s, buf) + } + } + close(read) + close(write) +} + +func testPreadPwrite(t *testing.T, fdr, fdw *os.File) { + type op struct { + s string + off int64 + } + write := make(chan op, 1) + read := make(chan struct{}, 1) + go func() { + for o := range write { + n, err := fdw.WriteAt([]byte(o.s), o.off) + read <- struct{}{} + if err != nil { + t.Error(err) + } + if n != len(o.s) { + t.Errorf("expected to write %d bytes, got %d", len(o.s), n) + } + } + }() + for i := range 10 { + off := int64(i % 3) // exercise some back and forth + s := strconv.Itoa(i) + write <- op{s, off} + <-read + buf := make([]byte, len(s)) + n, err := fdr.ReadAt(buf, off) + if err != nil { + t.Fatal(err) + } + if n != len(s) { + t.Fatalf("expected to read %d bytes, got %d", len(s), n) + } + if !bytes.Equal(buf, []byte(s)) { + t.Fatalf("expected %q, got %q", s, buf) + } + } + close(read) + close(write) +} + +func testFileReadEOF(t *testing.T, f *os.File) { + end, err := f.Seek(0, io.SeekEnd) + if err != nil { + t.Fatal(err) + } + var buf [1]byte + n, err := f.Read(buf[:]) + if err != nil && err != io.EOF { + t.Errorf("expected EOF, got %v", err) + } + if n != 0 { + t.Errorf("expected 0 bytes, got %d", n) + } + + n, err = f.ReadAt(buf[:], end) + if err != nil && err != io.EOF { + t.Errorf("expected EOF, got %v", err) + } + if n != 0 { + t.Errorf("expected 0 bytes, got %d", n) + } +} + +func TestFile(t *testing.T) { + t.Parallel() + tests := []struct { + name string + overlappedRead bool + overlappedWrite bool + }{ + {"overlapped", true, true}, + {"overlapped-read", true, false}, + {"overlapped-write", false, true}, + {"sync", false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + name := filepath.Join(t.TempDir(), "foo") + rh := newFileOverlapped(t, name, tt.overlappedRead) + wh := newFileOverlapped(t, name, tt.overlappedWrite) + testReadWrite(t, rh, wh) + testPreadPwrite(t, rh, wh) + testFileReadEOF(t, rh) + }) + } +} + +func TestFileOverlappedSeek(t *testing.T) { + t.Parallel() + name := filepath.Join(t.TempDir(), "foo") + f := newFileOverlapped(t, name, true) + content := []byte("foo") + if _, err := f.Write(content); err != nil { + t.Fatal(err) + } + // Check that the file pointer is at the expected offset. + n, err := f.Seek(0, io.SeekCurrent) + if err != nil { + t.Fatal(err) + } + if n != int64(len(content)) { + t.Errorf("expected file pointer to be at offset %d, got %d", len(content), n) + } + // Set the file pointer to the start of the file. + if _, err := f.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } + // Read the first byte. + var buf [1]byte + if _, err := f.Read(buf[:]); err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf[:], content[:len(buf)]) { + t.Errorf("expected %q, got %q", content[:len(buf)], buf[:]) + } + // Check that the file pointer is at the expected offset. + n, err = f.Seek(0, io.SeekCurrent) + if err != nil { + t.Fatal(err) + } + if n != int64(len(buf)) { + t.Errorf("expected file pointer to be at offset %d, got %d", len(buf), n) + } + if n, err = f.Seek(1, io.SeekStart); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Errorf("expected file pointer to be at offset %d, got %d", 1, n) + } + if n, err = f.Seek(-1, io.SeekEnd); err != nil { + t.Fatal(err) + } else if n != int64(len(content)-1) { + t.Errorf("expected file pointer to be at offset %d, got %d", len(content)-1, n) + } + if _, err := f.Seek(-1, io.SeekStart); !errors.Is(err, windows.ERROR_NEGATIVE_SEEK) { + t.Errorf("expected ERROR_NEGATIVE_SEEK, got %v", err) + } + if _, err := f.Seek(0, -1); !errors.Is(err, windows.ERROR_INVALID_PARAMETER) { + t.Errorf("expected ERROR_INVALID_PARAMETER, got %v", err) + } +} + +func TestFileOverlappedReadAtSeekVolume(t *testing.T) { + // Test that we can use File.ReadAt and File.Seek with an overlapped volume handle. + // See https://go.dev/issues/74951. + t.Parallel() + name := `\\.\` + filepath.VolumeName(t.TempDir()) + f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|windows.O_FILE_FLAG_OVERLAPPED, 0666) + if err != nil { + if errors.Is(err, syscall.ERROR_ACCESS_DENIED) { + t.Skip("skipping test: access denied") + } + t.Fatal(err) + } + defer f.Close() + + var buf [0]byte + if _, err := f.ReadAt(buf[:], 0); err != nil { + t.Fatal(err) + } + if _, err := f.Seek(0, io.SeekCurrent); err != nil { + t.Fatal(err) + } +} + +func TestPipe(t *testing.T) { + t.Parallel() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer func() { + if err := r.Close(); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + }() + testReadWrite(t, r, w) +} + +func TestNamedPipe(t *testing.T) { + t.Parallel() + tests := []struct { + name string + overlappedRead bool + overlappedWrite bool + }{ + {"overlapped", true, true}, + {"overlapped-write", false, true}, + {"overlapped-read", true, false}, + {"sync", false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + name := pipeName() + pipe := newBytePipe(t, name, tt.overlappedWrite) + file := newFileOverlapped(t, name, tt.overlappedRead) + testReadWrite(t, pipe, file) + }) + } +} + +func TestPipeMessageReadEOF(t *testing.T) { + t.Parallel() + name := pipeName() + pipe := newMessagePipe(t, name, true) + file := newFileOverlapped(t, name, true) + + _, err := pipe.Write(nil) + if err != nil { + t.Error(err) + } + + var buf [10]byte + n, err := file.Read(buf[:]) + if err != io.EOF { + t.Errorf("expected EOF, got %v", err) + } + if n != 0 { + t.Errorf("expected 0 bytes, got %d", n) + } +} + +func TestPipeClosedEOF(t *testing.T) { + t.Parallel() + name := pipeName() + pipe := newBytePipe(t, name, true) + file := newFileOverlapped(t, name, true) + + pipe.Close() + + var buf [10]byte + n, err := file.Read(buf[:]) + if err != io.EOF { + t.Errorf("expected EOF, got %v", err) + } + if n != 0 { + t.Errorf("expected 0 bytes, got %d", n) + } +} + +func TestPipeReadTimeout(t *testing.T) { + t.Parallel() + name := pipeName() + _ = newBytePipe(t, name, true) + file := newFileOverlapped(t, name, true) + + err := file.SetReadDeadline(time.Now().Add(time.Millisecond)) + if err != nil { + t.Fatal(err) + } + + var buf [10]byte + _, err = file.Read(buf[:]) + if !errors.Is(err, os.ErrDeadlineExceeded) { + t.Errorf("expected deadline exceeded, got %v", err) + } +} + +func TestPipeCanceled(t *testing.T) { + t.Parallel() + name := pipeName() + _ = newBytePipe(t, name, true) + file := newFileOverlapped(t, name, true) + ch := make(chan struct{}, 1) + go func() { + for { + select { + case <-ch: + return + default: + sc, err := file.SyscallConn() + if err != nil { + t.Error(err) + return + } + if err := sc.Control(func(fd uintptr) { + syscall.CancelIoEx(syscall.Handle(fd), nil) + }); err != nil { + t.Error(err) + return + } + time.Sleep(100 * time.Millisecond) + } + } + }() + var tmp [1]byte + // Read will block until the cancel is complete. + _, err := file.Read(tmp[:]) + ch <- struct{}{} + if errors.Is(err, os.ErrDeadlineExceeded) { + t.Skip("took too long to cancel") + } + if !errors.Is(err, syscall.ERROR_OPERATION_ABORTED) { + t.Errorf("expected ERROR_OPERATION_ABORTED, got %v", err) + } +} + +func iocpAssociateFile(f *os.File, iocp syscall.Handle) error { + _, err := windows.CreateIoCompletionPort(syscall.Handle(f.Fd()), iocp, 0, 0) + return err +} + +func TestFileAssociatedWithExternalIOCP(t *testing.T) { + // Test that a caller can associate an overlapped handle to an external IOCP + // after the handle has been passed to os.NewFile. + // Also test that the File can perform I/O after it is associated with the + // external IOCP and that those operations do not post to the external IOCP. + t.Parallel() + name := pipeName() + pipe := newMessagePipe(t, name, true) + _ = newFileOverlapped(t, name, true) // just open a pipe client + + // Use a file to exercise WriteAt. + file := newFileOverlapped(t, filepath.Join(t.TempDir(), "a"), true) + + iocp, err := windows.CreateIoCompletionPort(syscall.InvalidHandle, 0, 0, 0) + if err != nil { + t.Fatal(err) + } + defer func() { + if iocp == syscall.InvalidHandle { + // Already closed at the end of the test. + return + } + if err := syscall.CloseHandle(iocp); err != nil { + t.Fatal(err) + } + }() + + ch := make(chan error, 1) + go func() { + var bytes, key uint32 + var overlapped *syscall.Overlapped + err := syscall.GetQueuedCompletionStatus(syscall.Handle(iocp), &bytes, &key, &overlapped, syscall.INFINITE) + ch <- err + }() + + if err := iocpAssociateFile(pipe, iocp); err != nil { + t.Fatal(err) + } + if err := iocpAssociateFile(file, iocp); err != nil { + t.Fatal(err) + } + + if _, err := pipe.Write([]byte("hello")); err != nil { + t.Fatal(err) + } + if _, err := file.Write([]byte("hello")); err != nil { + t.Fatal(err) + } + if _, err := file.WriteAt([]byte("hello"), 0); err != nil { + t.Fatal(err) + } + + // Wait fot he goroutine to call GetQueuedCompletionStatus. + time.Sleep(100 * time.Millisecond) + + // Trigger ERROR_ABANDONED_WAIT_0. + if err := syscall.CloseHandle(iocp); err != nil { + t.Fatal(err) + } + + // Wait for the completion to be posted to the IOCP. + err = <-ch + iocp = syscall.InvalidHandle + const ERROR_ABANDONED_WAIT_0 = syscall.Errno(735) + switch err { + case ERROR_ABANDONED_WAIT_0: + // This is what we expect. + case nil: + t.Error("unexpected queued completion") + default: + t.Error(err) + } +} + +func TestFileWriteFdRace(t *testing.T) { + t.Parallel() + + f := newFileOverlapped(t, filepath.Join(t.TempDir(), "a"), true) + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + n, err := f.Write([]byte("hi")) + if err != nil { + // We look at error strings as the + // expected errors are OS-specific. + switch { + case errors.Is(err, windows.ERROR_INVALID_HANDLE): + // Ignore an expected error. + default: + // Unexpected error. + t.Error(err) + } + return + } + if n != 2 { + t.Errorf("wrote %d bytes, expected 2", n) + return + } + }() + go func() { + defer wg.Done() + f.Fd() + }() + wg.Wait() + + iocp, err := windows.CreateIoCompletionPort(syscall.InvalidHandle, 0, 0, 0) + if err != nil { + t.Fatal(err) + } + defer syscall.CloseHandle(iocp) + if err := iocpAssociateFile(f, iocp); err != nil { + t.Fatal(err) + } + + if _, err := f.Write([]byte("hi")); err != nil { + t.Error(err) + } +} + +func TestSplitPath(t *testing.T) { + t.Parallel() + for _, tt := range []struct{ path, wantDir, wantBase string }{ + {`a`, `.`, `a`}, + {`a\`, `.`, `a`}, + {`a\\`, `.`, `a`}, + {`a\b`, `a`, `b`}, + {`a\\b`, `a`, `b`}, + {`a\b\`, `a`, `b`}, + {`a\b\c`, `a\b`, `c`}, + {`\a`, `\`, `a`}, + {`\a\`, `\`, `a`}, + {`\a\b`, `\a`, `b`}, + {`\a\b\`, `\a`, `b`}, + {`\a\b\c`, `\a\b`, `c`}, + {`\\a`, `\\a`, `.`}, + {`\\a\`, `\\a\`, `.`}, + {`\\\a`, `\\\a`, `.`}, + {`\\\a\`, `\\\a`, `.`}, + {`\\a\b\c`, `\\a\b`, `c`}, + {`c:`, `c:`, `.`}, + {`c:\`, `c:\`, `.`}, + {`c:\a`, `c:\`, `a`}, + {`c:a`, `c:`, `a`}, + {`c:a\b\`, `c:a`, `b`}, + {`c:base`, `c:`, `base`}, + {`a/b/c`, `a/b`, `c`}, + {`a/b/c/`, `a/b`, `c`}, + {`\\?\c:\a`, `\\?\c:\`, `a`}, + } { + if dir, base := os.SplitPath(tt.path); dir != tt.wantDir || base != tt.wantBase { + t.Errorf("splitPath(%q) = %q, %q, want %q, %q", tt.path, dir, base, tt.wantDir, tt.wantBase) + } + } +} + +func TestOpenFileFlags(t *testing.T) { + t.Parallel() + + // The only way to retrieve some of the flags passed in CreateFile + // is using NtQueryInformationFile, which returns the file flags + // NT equivalent. Note that FILE_SYNCHRONOUS_IO_NONALERT is always + // set when FILE_FLAG_OVERLAPPED is not passed. + // The flags that can't be retrieved using NtQueryInformationFile won't + // be tested in here, but we at least know that the logic to handle them is correct. + tests := []struct { + flag uint32 + wantMode uint32 + }{ + {0, windows.FILE_SYNCHRONOUS_IO_NONALERT}, + {windows.O_FILE_FLAG_OVERLAPPED, 0}, + {windows.O_FILE_FLAG_NO_BUFFERING, windows.FILE_NO_INTERMEDIATE_BUFFERING | windows.FILE_SYNCHRONOUS_IO_NONALERT}, + {windows.O_FILE_FLAG_NO_BUFFERING | windows.O_FILE_FLAG_OVERLAPPED, windows.FILE_NO_INTERMEDIATE_BUFFERING}, + {windows.O_FILE_FLAG_SEQUENTIAL_SCAN, windows.FILE_SEQUENTIAL_ONLY | windows.FILE_SYNCHRONOUS_IO_NONALERT}, + {windows.O_FILE_FLAG_WRITE_THROUGH, windows.FILE_WRITE_THROUGH | windows.FILE_SYNCHRONOUS_IO_NONALERT}, + } + for i, tt := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + t.Parallel() + f, err := os.OpenFile(filepath.Join(t.TempDir(), "test.txt"), syscall.O_RDWR|syscall.O_CREAT|int(tt.flag), 0666) + if err != nil { + t.Fatal(err) + } + defer f.Close() + var info windows.FILE_MODE_INFORMATION + if err := windows.NtQueryInformationFile(syscall.Handle(f.Fd()), &windows.IO_STATUS_BLOCK{}, + unsafe.Pointer(&info), uint32(unsafe.Sizeof(info)), windows.FileModeInformation); err != nil { + t.Fatal(err) + } + if info.Mode != tt.wantMode { + t.Errorf("file mode = 0x%x; want 0x%x", info.Mode, tt.wantMode) + } + }) + } +} + +func TestOpenFileDeleteOnClose(t *testing.T) { + t.Parallel() + name := filepath.Join(t.TempDir(), "test.txt") + f, err := os.OpenFile(name, syscall.O_RDWR|syscall.O_CREAT|windows.O_FILE_FLAG_DELETE_ON_CLOSE, 0666) + if err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + // The file should be deleted after closing. + if _, err := os.Stat(name); !errors.Is(err, os.ErrNotExist) { + t.Errorf("expected file to be deleted, got %v", err) + } +} + +func TestOpenFileFlagInvalid(t *testing.T) { + t.Parallel() + // invalidFileFlag is the only value in the file flag range that is not supported, + // as it is not defined in the Windows API. + const invalidFileFlag = 0x00400000 + f, err := os.OpenFile(filepath.Join(t.TempDir(), "test.txt"), syscall.O_RDWR|syscall.O_CREAT|invalidFileFlag, 0666) + if !errors.Is(err, os.ErrInvalid) { + t.Fatalf("expected os.ErrInvalid, got %v", err) + } + f.Close() +} diff --git a/src/os/path_test.go b/src/os/path_test.go index 2a4e9565dc26db..563f7753bdbf05 100644 --- a/src/os/path_test.go +++ b/src/os/path_test.go @@ -16,63 +16,84 @@ import ( var isReadonlyError = func(error) bool { return false } func TestMkdirAll(t *testing.T) { - t.Parallel() + testMaybeRooted(t, func(t *testing.T, r *Root) { + mkdirAll := MkdirAll + create := Create + if r != nil { + mkdirAll = r.MkdirAll + create = r.Create + } - tmpDir := TempDir() - path := tmpDir + "/_TestMkdirAll_/dir/./dir2" - err := MkdirAll(path, 0777) - if err != nil { - t.Fatalf("MkdirAll %q: %s", path, err) - } - defer RemoveAll(tmpDir + "/_TestMkdirAll_") + path := "_TestMkdirAll_/dir/./dir2" + err := mkdirAll(path, 0777) + if err != nil { + t.Fatalf("MkdirAll %q: %s", path, err) + } - // Already exists, should succeed. - err = MkdirAll(path, 0777) - if err != nil { - t.Fatalf("MkdirAll %q (second time): %s", path, err) - } + // Already exists, should succeed. + err = mkdirAll(path, 0777) + if err != nil { + t.Fatalf("MkdirAll %q (second time): %s", path, err) + } - // Make file. - fpath := path + "/file" - f, err := Create(fpath) - if err != nil { - t.Fatalf("create %q: %s", fpath, err) - } - defer f.Close() + // Make file. + fpath := path + "/file" + f, err := create(fpath) + if err != nil { + t.Fatalf("create %q: %s", fpath, err) + } + defer f.Close() - // Can't make directory named after file. - err = MkdirAll(fpath, 0777) - if err == nil { - t.Fatalf("MkdirAll %q: no error", fpath) - } - perr, ok := err.(*PathError) - if !ok { - t.Fatalf("MkdirAll %q returned %T, not *PathError", fpath, err) - } - if filepath.Clean(perr.Path) != filepath.Clean(fpath) { - t.Fatalf("MkdirAll %q returned wrong error path: %q not %q", fpath, filepath.Clean(perr.Path), filepath.Clean(fpath)) - } + // Can't make directory named after file. + err = mkdirAll(fpath, 0777) + if err == nil { + t.Fatalf("MkdirAll %q: no error", fpath) + } + perr, ok := err.(*PathError) + if !ok { + t.Fatalf("MkdirAll %q returned %T, not *PathError", fpath, err) + } + if filepath.Clean(perr.Path) != filepath.Clean(fpath) { + t.Fatalf("MkdirAll %q returned wrong error path: %q not %q", fpath, filepath.Clean(perr.Path), filepath.Clean(fpath)) + } - // Can't make subdirectory of file. - ffpath := fpath + "/subdir" - err = MkdirAll(ffpath, 0777) - if err == nil { - t.Fatalf("MkdirAll %q: no error", ffpath) - } - perr, ok = err.(*PathError) - if !ok { - t.Fatalf("MkdirAll %q returned %T, not *PathError", ffpath, err) - } - if filepath.Clean(perr.Path) != filepath.Clean(fpath) { - t.Fatalf("MkdirAll %q returned wrong error path: %q not %q", ffpath, filepath.Clean(perr.Path), filepath.Clean(fpath)) - } + // Can't make subdirectory of file. + ffpath := fpath + "/subdir" + err = mkdirAll(ffpath, 0777) + if err == nil { + t.Fatalf("MkdirAll %q: no error", ffpath) + } + perr, ok = err.(*PathError) + if !ok { + t.Fatalf("MkdirAll %q returned %T, not *PathError", ffpath, err) + } + if filepath.Clean(perr.Path) != filepath.Clean(fpath) { + t.Fatalf("MkdirAll %q returned wrong error path: %q not %q", ffpath, filepath.Clean(perr.Path), filepath.Clean(fpath)) + } - if runtime.GOOS == "windows" { - path := tmpDir + `\_TestMkdirAll_\dir\.\dir2\` - err := MkdirAll(path, 0777) - if err != nil { - t.Fatalf("MkdirAll %q: %s", path, err) + if runtime.GOOS == "windows" { + path := `_TestMkdirAll_\dir\.\dir2\` + err := mkdirAll(path, 0777) + if err != nil { + t.Fatalf("MkdirAll %q: %s", path, err) + } } + }) +} + +func TestMkdirAllAbsPath(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "/a/b/c") + if err := MkdirAll(path, 0o777); err != nil { + t.Fatal(err) + } + st, err := Stat(path) + if err != nil { + t.Fatal(err) + } + if !st.IsDir() { + t.Fatalf("after MkdirAll(%q, 0o777), %q is not a directory", path, path) } } diff --git a/src/os/path_windows.go b/src/os/path_windows.go index f585aa5ee6ded4..8273d8b0472770 100644 --- a/src/os/path_windows.go +++ b/src/os/path_windows.go @@ -21,6 +21,56 @@ func IsPathSeparator(c uint8) bool { return c == '\\' || c == '/' } +// splitPath returns the base name and parent directory. +func splitPath(path string) (string, string) { + if path == "" { + return ".", "." + } + + // The first prefixlen bytes are part of the parent directory. + // The prefix consists of the volume name (if any) and the first \ (if significant). + prefixlen := filepathlite.VolumeNameLen(path) + if len(path) > prefixlen && IsPathSeparator(path[prefixlen]) { + if prefixlen == 0 { + // This is a path relative to the current volume, like \foo. + // Include the initial \ in the prefix. + prefixlen = 1 + } else if path[prefixlen-1] == ':' { + // This is an absolute path on a named drive, like c:\foo. + // Include the initial \ in the prefix. + prefixlen++ + } + } + + i := len(path) - 1 + + // Remove trailing slashes. + for i >= prefixlen && IsPathSeparator(path[i]) { + i-- + } + path = path[:i+1] + + // Find the last path separator. The basename is what follows. + for i >= prefixlen && !IsPathSeparator(path[i]) { + i-- + } + basename := path[i+1:] + if basename == "" { + basename = "." + } + + // Remove trailing slashes. The remainder is dirname. + for i >= prefixlen && IsPathSeparator(path[i]) { + i-- + } + dirname := path[:i+1] + if dirname == "" { + dirname = "." + } + + return dirname, basename +} + func dirname(path string) string { vol := filepathlite.VolumeName(path) i := len(path) - 1 diff --git a/src/os/path_windows_test.go b/src/os/path_windows_test.go index 0b5d7099f65788..eea2b58ee0a886 100644 --- a/src/os/path_windows_test.go +++ b/src/os/path_windows_test.go @@ -224,7 +224,7 @@ func TestRemoveAllLongPathRelative(t *testing.T) { // Test that RemoveAll doesn't hang with long relative paths. // See go.dev/issue/36375. tmp := t.TempDir() - chdir(t, tmp) + t.Chdir(tmp) dir := filepath.Join(tmp, "foo", "bar", strings.Repeat("a", 150), strings.Repeat("b", 150)) err := os.MkdirAll(dir, 0755) if err != nil { @@ -236,6 +236,23 @@ func TestRemoveAllLongPathRelative(t *testing.T) { } } +func TestRemoveAllFallback(t *testing.T) { + windows.TestDeleteatFallback = true + t.Cleanup(func() { windows.TestDeleteatFallback = false }) + + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "file1"), []byte{}, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "file2"), []byte{}, 0400); err != nil { // read-only file + t.Fatal(err) + } + + if err := os.RemoveAll(dir); err != nil { + t.Fatal(err) + } +} + func testLongPathAbs(t *testing.T, target string) { t.Helper() testWalkFn := func(path string, info os.FileInfo, err error) error { @@ -265,7 +282,7 @@ func TestLongPathAbs(t *testing.T) { } func TestLongPathRel(t *testing.T) { - chdir(t, t.TempDir()) + t.Chdir(t.TempDir()) target := strings.Repeat("b\\", 300) testLongPathAbs(t, target) diff --git a/src/os/pidfd_linux.go b/src/os/pidfd_linux.go index 0404c4ff64b72e..796d8c018c7f2a 100644 --- a/src/os/pidfd_linux.go +++ b/src/os/pidfd_linux.go @@ -8,20 +8,27 @@ // v5.3: pidfd_open syscall, clone3 syscall; // v5.4: P_PIDFD idtype support for waitid syscall; // v5.6: pidfd_getfd syscall. +// +// N.B. Alternative Linux implementations may not follow this ordering. e.g., +// QEMU user mode 7.2 added pidfd_open, but CLONE_PIDFD was not added until +// 8.0. package os import ( - "errors" "internal/syscall/unix" + "runtime" "sync" "syscall" - "unsafe" + _ "unsafe" // for linkname ) -func ensurePidfd(sysAttr *syscall.SysProcAttr) *syscall.SysProcAttr { +// ensurePidfd initializes the PidFD field in sysAttr if it is not already set. +// It returns the original or modified SysProcAttr struct and a flag indicating +// whether the PidFD should be duplicated before using. +func ensurePidfd(sysAttr *syscall.SysProcAttr) (*syscall.SysProcAttr, bool) { if !pidfdWorks() { - return sysAttr + return sysAttr, false } var pidfd int @@ -29,25 +36,36 @@ func ensurePidfd(sysAttr *syscall.SysProcAttr) *syscall.SysProcAttr { if sysAttr == nil { return &syscall.SysProcAttr{ PidFD: &pidfd, - } + }, false } if sysAttr.PidFD == nil { newSys := *sysAttr // copy newSys.PidFD = &pidfd - return &newSys + return &newSys, false } - return sysAttr + return sysAttr, true } -func getPidfd(sysAttr *syscall.SysProcAttr) (uintptr, bool) { +// getPidfd returns the value of sysAttr.PidFD (or its duplicate if needDup is +// set) and a flag indicating whether the value can be used. +func getPidfd(sysAttr *syscall.SysProcAttr, needDup bool) (uintptr, bool) { if !pidfdWorks() { return 0, false } - return uintptr(*sysAttr.PidFD), true + h := *sysAttr.PidFD + if needDup { + dupH, e := unix.Fcntl(h, syscall.F_DUPFD_CLOEXEC, 0) + if e != nil { + return 0, false + } + h = dupH + } + return uintptr(h), true } +// pidfdFind returns the process handle for pid. func pidfdFind(pid int) (uintptr, error) { if !pidfdWorks() { return 0, syscall.ENOSYS @@ -60,9 +78,8 @@ func pidfdFind(pid int) (uintptr, error) { return h, nil } -// _P_PIDFD is used as idtype argument to waitid syscall. -const _P_PIDFD = 3 - +// pidfdWait waits for the process to complete, +// and updates the process status to done. func (p *Process) pidfdWait() (*ProcessState, error) { // When pidfd is used, there is no wait/kill race (described in CL 23967) // because the PID recycle issue doesn't exist (IOW, pidfd, unlike PID, @@ -86,20 +103,18 @@ func (p *Process) pidfdWait() (*ProcessState, error) { var ( info unix.SiginfoChild rusage syscall.Rusage - e syscall.Errno ) - for { - _, _, e = syscall.Syscall6(syscall.SYS_WAITID, _P_PIDFD, handle, uintptr(unsafe.Pointer(&info)), syscall.WEXITED, uintptr(unsafe.Pointer(&rusage)), 0) - if e != syscall.EINTR { - break - } - } - if e != 0 { - return nil, NewSyscallError("waitid", e) + err := ignoringEINTR(func() error { + return unix.Waitid(unix.P_PIDFD, int(handle), &info, syscall.WEXITED, &rusage) + }) + if err != nil { + return nil, NewSyscallError("waitid", err) } - // Release the Process' handle reference, in addition to the reference - // we took above. - p.handlePersistentRelease(statusDone) + + // Update the Process status to statusDone. + // This also releases a reference to the handle. + p.doRelease(statusDone) + return &ProcessState{ pid: int(info.Pid), status: info.WaitStatus(), @@ -107,33 +122,43 @@ func (p *Process) pidfdWait() (*ProcessState, error) { }, nil } +// pidfdSendSignal sends a signal to the process. func (p *Process) pidfdSendSignal(s syscall.Signal) error { handle, status := p.handleTransientAcquire() switch status { case statusDone: return ErrProcessDone case statusReleased: - return errors.New("os: process already released") + return errProcessReleased } defer p.handleTransientRelease() return convertESRCH(unix.PidFDSendSignal(handle, s)) } +// pidfdWorks returns whether we can use pidfd on this system. func pidfdWorks() bool { return checkPidfdOnce() == nil } +// checkPidfdOnce is used to only check whether pidfd works once. var checkPidfdOnce = sync.OnceValue(checkPidfd) -// checkPidfd checks whether all required pidfd-related syscalls work. -// This consists of pidfd_open and pidfd_send_signal syscalls, and waitid -// syscall with idtype of P_PIDFD. +// checkPidfd checks whether all required pidfd-related syscalls work. This +// consists of pidfd_open and pidfd_send_signal syscalls, waitid syscall with +// idtype of P_PIDFD, and clone(CLONE_PIDFD). // // Reasons for non-working pidfd syscalls include an older kernel and an // execution environment in which the above system calls are restricted by // seccomp or a similar technology. func checkPidfd() error { + // In Android version < 12, pidfd-related system calls are not allowed + // by seccomp and trigger the SIGSYS signal. See issue #69065. + if runtime.GOOS == "android" { + ignoreSIGSYS() + defer restoreSIGSYS() + } + // Get a pidfd of the current process (opening of "/proc/self" won't // work for waitid). fd, err := unix.PidFDOpen(syscall.Getpid(), 0) @@ -143,12 +168,12 @@ func checkPidfd() error { defer syscall.Close(int(fd)) // Check waitid(P_PIDFD) works. - for { - _, _, err = syscall.Syscall6(syscall.SYS_WAITID, _P_PIDFD, fd, 0, syscall.WEXITED, 0, 0) - if err != syscall.EINTR { - break - } - } + err = ignoringEINTR(func() error { + var info unix.SiginfoChild + // We don't actually care about the info, but passing a nil pointer + // makes valgrind complain because 0x0 is unaddressable. + return unix.Waitid(unix.P_PIDFD, int(fd), &info, syscall.WEXITED, nil) + }) // Expect ECHILD from waitid since we're not our own parent. if err != syscall.ECHILD { return NewSyscallError("pidfd_wait", err) @@ -159,5 +184,27 @@ func checkPidfd() error { return NewSyscallError("pidfd_send_signal", err) } + // Verify that clone(CLONE_PIDFD) works. + // + // This shouldn't be necessary since pidfd_open was added in Linux 5.3, + // after CLONE_PIDFD in Linux 5.2, but some alternative Linux + // implementations may not adhere to this ordering. + if err := checkClonePidfd(); err != nil { + return err + } + return nil } + +// Provided by syscall. +// +//go:linkname checkClonePidfd +func checkClonePidfd() error + +// Provided by runtime. +// +//go:linkname ignoreSIGSYS +func ignoreSIGSYS() + +//go:linkname restoreSIGSYS +func restoreSIGSYS() diff --git a/src/os/pidfd_linux_test.go b/src/os/pidfd_linux_test.go index 837593706bae8e..0210850c0450e6 100644 --- a/src/os/pidfd_linux_test.go +++ b/src/os/pidfd_linux_test.go @@ -6,10 +6,13 @@ package os_test import ( "errors" + "internal/syscall/unix" "internal/testenv" "os" + "os/exec" "syscall" "testing" + "time" ) func TestFindProcessViaPidfd(t *testing.T) { @@ -57,3 +60,159 @@ func TestFindProcessViaPidfd(t *testing.T) { t.Fatalf("Release: got %v, want ", err) } } + +func TestStartProcessWithPidfd(t *testing.T) { + testenv.MustHaveGoBuild(t) + t.Parallel() + + if err := os.CheckPidfdOnce(); err != nil { + // Non-pidfd code paths tested in exec_unix_test.go. + t.Skipf("skipping: pidfd not available: %v", err) + } + + var pidfd int + p, err := os.StartProcess(testenv.GoToolPath(t), []string{"go"}, &os.ProcAttr{ + Sys: &syscall.SysProcAttr{ + PidFD: &pidfd, + }, + }) + if err != nil { + t.Fatalf("starting test process: %v", err) + } + defer syscall.Close(pidfd) + + if _, err := p.Wait(); err != nil { + t.Fatalf("Wait: got %v, want ", err) + } + + // Check the pidfd is still valid + err = unix.PidFDSendSignal(uintptr(pidfd), syscall.Signal(0)) + if !errors.Is(err, syscall.ESRCH) { + t.Errorf("SendSignal: got %v, want %v", err, syscall.ESRCH) + } +} + +// Issue #69284 +func TestPidfdLeak(t *testing.T) { + exe := testenv.Executable(t) + + // Find the next 10 descriptors. + // We need to get more than one descriptor in practice; + // the pidfd winds up not being the next descriptor. + const count = 10 + want := make([]int, count) + for i := range count { + var err error + want[i], err = syscall.Open(exe, syscall.O_RDONLY, 0) + if err != nil { + t.Fatal(err) + } + } + + // Close the descriptors. + for _, d := range want { + syscall.Close(d) + } + + // Start a process 10 times. + for range 10 { + // For testing purposes this has to be an absolute path. + // Otherwise we will fail finding the executable + // and won't start a process at all. + cmd := exec.Command("/noSuchExecutable") + cmd.Run() + } + + // Open the next 10 descriptors again. + got := make([]int, count) + for i := range count { + var err error + got[i], err = syscall.Open(exe, syscall.O_RDONLY, 0) + if err != nil { + t.Fatal(err) + } + } + + // Close the descriptors + for _, d := range got { + syscall.Close(d) + } + + t.Logf("got %v", got) + t.Logf("want %v", want) + + // Allow some slack for runtime epoll descriptors and the like. + if got[count-1] > want[count-1]+5 { + t.Errorf("got descriptor %d, want %d", got[count-1], want[count-1]) + } +} + +func TestProcessWithHandleLinux(t *testing.T) { + t.Parallel() + havePidfd := os.CheckPidfdOnce() == nil + + const envVar = "OSTEST_PROCESS_WITH_HANDLE" + if os.Getenv(envVar) != "" { + time.Sleep(1 * time.Minute) + return + } + + cmd := testenv.CommandContext(t, t.Context(), testenv.Executable(t), "-test.run=^"+t.Name()+"$") + cmd = testenv.CleanCmdEnv(cmd) + cmd.Env = append(cmd.Env, envVar+"=1") + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { + cmd.Process.Kill() + cmd.Wait() + }() + + const sig = syscall.SIGINT + called := false + err := cmd.Process.WithHandle(func(pidfd uintptr) { + called = true + // Check the provided pidfd is valid, and terminate the child. + err := unix.PidFDSendSignal(pidfd, sig) + if err != nil { + t.Errorf("PidFDSendSignal: got error %v, want nil", err) + } + }) + // If pidfd is not supported, WithHandle should fail. + if !havePidfd && err == nil { + t.Fatal("WithHandle: got nil, want error") + } + // If pidfd is supported, WithHandle should succeed. + if havePidfd && err != nil { + t.Fatalf("WithHandle: got error %v, want nil", err) + } + // If pidfd is supported, function should have been called, and vice versa. + if havePidfd != called { + t.Fatalf("WithHandle: havePidfd is %v, but called is %v", havePidfd, called) + } + // If pidfd is supported, wait on the child process to check it worked as intended. + if called { + err := cmd.Wait() + if err == nil { + t.Fatal("Wait: want error, got nil") + } + st := cmd.ProcessState.Sys().(syscall.WaitStatus) + if !st.Signaled() { + t.Fatal("ProcessState: want Signaled, got", err) + } + if gotSig := st.Signal(); sig != gotSig { + t.Fatalf("ProcessState.Signal: want %v, got %v", sig, gotSig) + } + // Finally, check that WithHandle now returns ErrProcessDone. + called = false + err = cmd.Process.WithHandle(func(_ uintptr) { + called = true + }) + if err != os.ErrProcessDone { + t.Fatalf("WithHandle: want os.ErrProcessDone, got %v", err) + } + if called { + t.Fatal("called: want false, got true") + } + } +} diff --git a/src/os/pidfd_other.go b/src/os/pidfd_other.go index 57804327790199..7a282307db2d01 100644 --- a/src/os/pidfd_other.go +++ b/src/os/pidfd_other.go @@ -8,11 +8,11 @@ package os import "syscall" -func ensurePidfd(sysAttr *syscall.SysProcAttr) *syscall.SysProcAttr { - return sysAttr +func ensurePidfd(sysAttr *syscall.SysProcAttr) (*syscall.SysProcAttr, bool) { + return sysAttr, false } -func getPidfd(_ *syscall.SysProcAttr) (uintptr, bool) { +func getPidfd(_ *syscall.SysProcAttr, _ bool) (uintptr, bool) { return 0, false } diff --git a/src/os/pipe_test.go b/src/os/pipe_test.go index a9e0c8bc8a9d2c..ccae6f61bf4c93 100644 --- a/src/os/pipe_test.go +++ b/src/os/pipe_test.go @@ -118,7 +118,7 @@ func TestStdPipe(t *testing.T) { // all writes should fail with EPIPE and then exit 0. for _, sig := range []bool{false, true} { for dest := 1; dest < 4; dest++ { - cmd := testenv.Command(t, os.Args[0], "-test.run", "TestStdPipe") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run", "TestStdPipe") cmd.Stdout = w cmd.Stderr = w cmd.ExtraFiles = []*os.File{w} @@ -145,7 +145,7 @@ func TestStdPipe(t *testing.T) { } // Test redirecting stdout but not stderr. Issue 40076. - cmd := testenv.Command(t, os.Args[0], "-test.run", "TestStdPipe") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run", "TestStdPipe") cmd.Stdout = w var stderr bytes.Buffer cmd.Stderr = &stderr @@ -263,7 +263,7 @@ func TestReadNonblockingFd(t *testing.T) { } defer r.Close() defer w.Close() - cmd := testenv.Command(t, os.Args[0], "-test.run=^"+t.Name()+"$") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^"+t.Name()+"$") cmd.Env = append(cmd.Environ(), "GO_WANT_READ_NONBLOCKING_FD=1") cmd.Stdin = r output, err := cmd.CombinedOutput() diff --git a/src/os/read_test.go b/src/os/read_test.go index 1f79e89bafda49..e5fd941d0fc1f6 100644 --- a/src/os/read_test.go +++ b/src/os/read_test.go @@ -6,9 +6,11 @@ package os_test import ( "bytes" + "errors" . "os" "path/filepath" "runtime" + "syscall" "testing" ) @@ -78,16 +80,11 @@ func TestReadOnlyWriteFile(t *testing.T) { t.Parallel() // We don't want to use CreateTemp directly, since that opens a file for us as 0600. - tempDir, err := MkdirTemp("", t.Name()) - if err != nil { - t.Fatal(err) - } - defer RemoveAll(tempDir) - filename := filepath.Join(tempDir, "blurp.txt") + filename := filepath.Join(t.TempDir(), "blurp.txt") shmorp := []byte("shmorp") florp := []byte("florp") - err = WriteFile(filename, shmorp, 0444) + err := WriteFile(filename, shmorp, 0444) if err != nil { t.Fatalf("WriteFile %s: %v", filename, err) } @@ -108,11 +105,20 @@ func TestReadDir(t *testing.T) { t.Parallel() dirname := "rumpelstilzchen" - _, err := ReadDir(dirname) - if err == nil { + if _, err := ReadDir(dirname); err == nil { t.Fatalf("ReadDir %s: error expected, none found", dirname) } + filename := filepath.Join(t.TempDir(), "foo") + f, err := Create(filename) + if err != nil { + t.Fatal(err) + } + f.Close() + if list, err := ReadDir(filename); list != nil || !errors.Is(err, syscall.ENOTDIR) { + t.Fatalf("ReadDir %s: (nil, ENOTDIR) expected, got (%v, %v)", filename, list, err) + } + dirname = "." list, err := ReadDir(dirname) if err != nil { diff --git a/src/os/readfrom_freebsd_test.go b/src/os/readfrom_freebsd_test.go new file mode 100644 index 00000000000000..186049951f0ac2 --- /dev/null +++ b/src/os/readfrom_freebsd_test.go @@ -0,0 +1,57 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os_test + +import ( + "internal/poll" + . "os" + "testing" +) + +var ( + copyFileTests = []copyFileTestFunc{newCopyFileRangeTest} + copyFileHooks = []copyFileTestHook{hookCopyFileRange} +) + +func testCopyFiles(t *testing.T, size, limit int64) { + testCopyFileRange(t, size, limit) +} + +func testCopyFileRange(t *testing.T, size int64, limit int64) { + dst, src, data, hook, name := newCopyFileRangeTest(t, size) + testCopyFile(t, dst, src, data, hook, limit, name) +} + +// newCopyFileRangeTest initializes a new test for copy_file_range. +// It hooks package os' call to poll.CopyFileRange and returns the hook, +// so it can be inspected. +func newCopyFileRangeTest(t *testing.T, size int64) (dst, src *File, data []byte, hook *copyFileHook, name string) { + t.Helper() + + name = "newCopyFileRangeTest" + + dst, src, data = newCopyFileTest(t, size) + hook, _ = hookCopyFileRange(t) + + return +} + +func hookCopyFileRange(t *testing.T) (hook *copyFileHook, name string) { + name = "hookCopyFileRange" + + hook = new(copyFileHook) + orig := *PollCopyFileRangeP + t.Cleanup(func() { + *PollCopyFileRangeP = orig + }) + *PollCopyFileRangeP = func(dst, src *poll.FD, remain int64) (int64, bool, error) { + hook.called = true + hook.dstfd = dst.Sysfd + hook.srcfd = src.Sysfd + hook.written, hook.handled, hook.err = orig(dst, src, remain) + return hook.written, hook.handled, hook.err + } + return +} diff --git a/src/os/readfrom_linux_test.go b/src/os/readfrom_linux_test.go index 8dcb9cb2172882..d33f9cf9c984f6 100644 --- a/src/os/readfrom_linux_test.go +++ b/src/os/readfrom_linux_test.go @@ -14,251 +14,13 @@ import ( "net" . "os" "path/filepath" - "runtime" "strconv" - "strings" "sync" "syscall" "testing" "time" - - "golang.org/x/net/nettest" ) -func TestCopyFileRange(t *testing.T) { - sizes := []int{ - 1, - 42, - 1025, - syscall.Getpagesize() + 1, - 32769, - } - t.Run("Basic", func(t *testing.T) { - for _, size := range sizes { - t.Run(strconv.Itoa(size), func(t *testing.T) { - testCopyFileRange(t, int64(size), -1) - }) - } - }) - t.Run("Limited", func(t *testing.T) { - t.Run("OneLess", func(t *testing.T) { - for _, size := range sizes { - t.Run(strconv.Itoa(size), func(t *testing.T) { - testCopyFileRange(t, int64(size), int64(size)-1) - }) - } - }) - t.Run("Half", func(t *testing.T) { - for _, size := range sizes { - t.Run(strconv.Itoa(size), func(t *testing.T) { - testCopyFileRange(t, int64(size), int64(size)/2) - }) - } - }) - t.Run("More", func(t *testing.T) { - for _, size := range sizes { - t.Run(strconv.Itoa(size), func(t *testing.T) { - testCopyFileRange(t, int64(size), int64(size)+7) - }) - } - }) - }) - t.Run("DoesntTryInAppendMode", func(t *testing.T) { - dst, src, data, hook := newCopyFileRangeTest(t, 42) - - dst2, err := OpenFile(dst.Name(), O_RDWR|O_APPEND, 0755) - if err != nil { - t.Fatal(err) - } - defer dst2.Close() - - if _, err := io.Copy(dst2, src); err != nil { - t.Fatal(err) - } - if hook.called { - t.Fatal("called poll.CopyFileRange for destination in O_APPEND mode") - } - mustSeekStart(t, dst2) - mustContainData(t, dst2, data) // through traditional means - }) - t.Run("CopyFileItself", func(t *testing.T) { - hook := hookCopyFileRange(t) - - f, err := CreateTemp("", "file-readfrom-itself-test") - if err != nil { - t.Fatalf("failed to create tmp file: %v", err) - } - t.Cleanup(func() { - f.Close() - Remove(f.Name()) - }) - - data := []byte("hello world!") - if _, err := f.Write(data); err != nil { - t.Fatalf("failed to create and feed the file: %v", err) - } - - if err := f.Sync(); err != nil { - t.Fatalf("failed to save the file: %v", err) - } - - // Rewind it. - if _, err := f.Seek(0, io.SeekStart); err != nil { - t.Fatalf("failed to rewind the file: %v", err) - } - - // Read data from the file itself. - if _, err := io.Copy(f, f); err != nil { - t.Fatalf("failed to read from the file: %v", err) - } - - if !hook.called || hook.written != 0 || hook.handled || hook.err != nil { - t.Fatalf("poll.CopyFileRange should be called and return the EINVAL error, but got hook.called=%t, hook.err=%v", hook.called, hook.err) - } - - // Rewind it. - if _, err := f.Seek(0, io.SeekStart); err != nil { - t.Fatalf("failed to rewind the file: %v", err) - } - - data2, err := io.ReadAll(f) - if err != nil { - t.Fatalf("failed to read from the file: %v", err) - } - - // It should wind up a double of the original data. - if strings.Repeat(string(data), 2) != string(data2) { - t.Fatalf("data mismatch: %s != %s", string(data), string(data2)) - } - }) - t.Run("NotRegular", func(t *testing.T) { - t.Run("BothPipes", func(t *testing.T) { - hook := hookCopyFileRange(t) - - pr1, pw1, err := Pipe() - if err != nil { - t.Fatal(err) - } - defer pr1.Close() - defer pw1.Close() - - pr2, pw2, err := Pipe() - if err != nil { - t.Fatal(err) - } - defer pr2.Close() - defer pw2.Close() - - // The pipe is empty, and PIPE_BUF is large enough - // for this, by (POSIX) definition, so there is no - // need for an additional goroutine. - data := []byte("hello") - if _, err := pw1.Write(data); err != nil { - t.Fatal(err) - } - pw1.Close() - - n, err := io.Copy(pw2, pr1) - if err != nil { - t.Fatal(err) - } - if n != int64(len(data)) { - t.Fatalf("transferred %d, want %d", n, len(data)) - } - if !hook.called { - t.Fatalf("should have called poll.CopyFileRange") - } - pw2.Close() - mustContainData(t, pr2, data) - }) - t.Run("DstPipe", func(t *testing.T) { - dst, src, data, hook := newCopyFileRangeTest(t, 255) - dst.Close() - - pr, pw, err := Pipe() - if err != nil { - t.Fatal(err) - } - defer pr.Close() - defer pw.Close() - - n, err := io.Copy(pw, src) - if err != nil { - t.Fatal(err) - } - if n != int64(len(data)) { - t.Fatalf("transferred %d, want %d", n, len(data)) - } - if !hook.called { - t.Fatalf("should have called poll.CopyFileRange") - } - pw.Close() - mustContainData(t, pr, data) - }) - t.Run("SrcPipe", func(t *testing.T) { - dst, src, data, hook := newCopyFileRangeTest(t, 255) - src.Close() - - pr, pw, err := Pipe() - if err != nil { - t.Fatal(err) - } - defer pr.Close() - defer pw.Close() - - // The pipe is empty, and PIPE_BUF is large enough - // for this, by (POSIX) definition, so there is no - // need for an additional goroutine. - if _, err := pw.Write(data); err != nil { - t.Fatal(err) - } - pw.Close() - - n, err := io.Copy(dst, pr) - if err != nil { - t.Fatal(err) - } - if n != int64(len(data)) { - t.Fatalf("transferred %d, want %d", n, len(data)) - } - if !hook.called { - t.Fatalf("should have called poll.CopyFileRange") - } - mustSeekStart(t, dst) - mustContainData(t, dst, data) - }) - }) - t.Run("Nil", func(t *testing.T) { - var nilFile *File - anyFile, err := CreateTemp("", "") - if err != nil { - t.Fatal(err) - } - defer Remove(anyFile.Name()) - defer anyFile.Close() - - if _, err := io.Copy(nilFile, nilFile); err != ErrInvalid { - t.Errorf("io.Copy(nilFile, nilFile) = %v, want %v", err, ErrInvalid) - } - if _, err := io.Copy(anyFile, nilFile); err != ErrInvalid { - t.Errorf("io.Copy(anyFile, nilFile) = %v, want %v", err, ErrInvalid) - } - if _, err := io.Copy(nilFile, anyFile); err != ErrInvalid { - t.Errorf("io.Copy(nilFile, anyFile) = %v, want %v", err, ErrInvalid) - } - - if _, err := nilFile.ReadFrom(nilFile); err != ErrInvalid { - t.Errorf("nilFile.ReadFrom(nilFile) = %v, want %v", err, ErrInvalid) - } - if _, err := anyFile.ReadFrom(nilFile); err != ErrInvalid { - t.Errorf("anyFile.ReadFrom(nilFile) = %v, want %v", err, ErrInvalid) - } - if _, err := nilFile.ReadFrom(anyFile); err != ErrInvalid { - t.Errorf("nilFile.ReadFrom(anyFile) = %v, want %v", err, ErrInvalid) - } - }) -} - func TestSpliceFile(t *testing.T) { sizes := []int{ 1, @@ -479,110 +241,33 @@ func testSpliceToTTY(t *testing.T, proto string, size int64) { } } -func testCopyFileRange(t *testing.T, size int64, limit int64) { - dst, src, data, hook := newCopyFileRangeTest(t, size) - - // If we have a limit, wrap the reader. - var ( - realsrc io.Reader - lr *io.LimitedReader - ) - if limit >= 0 { - lr = &io.LimitedReader{N: limit, R: src} - realsrc = lr - if limit < int64(len(data)) { - data = data[:limit] - } - } else { - realsrc = src - } - - // Now call ReadFrom (through io.Copy), which will hopefully call - // poll.CopyFileRange. - n, err := io.Copy(dst, realsrc) - if err != nil { - t.Fatal(err) - } - - // If we didn't have a limit, we should have called poll.CopyFileRange - // with the right file descriptor arguments. - if limit > 0 && !hook.called { - t.Fatal("never called poll.CopyFileRange") - } - if hook.called && hook.dstfd != int(dst.Fd()) { - t.Fatalf("wrong destination file descriptor: got %d, want %d", hook.dstfd, dst.Fd()) - } - if hook.called && hook.srcfd != int(src.Fd()) { - t.Fatalf("wrong source file descriptor: got %d, want %d", hook.srcfd, src.Fd()) - } +var ( + copyFileTests = []copyFileTestFunc{newCopyFileRangeTest} + copyFileHooks = []copyFileTestHook{hookCopyFileRange} +) - // Check that the offsets after the transfer make sense, that the size - // of the transfer was reported correctly, and that the destination - // file contains exactly the bytes we expect it to contain. - dstoff, err := dst.Seek(0, io.SeekCurrent) - if err != nil { - t.Fatal(err) - } - srcoff, err := src.Seek(0, io.SeekCurrent) - if err != nil { - t.Fatal(err) - } - if dstoff != srcoff { - t.Errorf("offsets differ: dstoff = %d, srcoff = %d", dstoff, srcoff) - } - if dstoff != int64(len(data)) { - t.Errorf("dstoff = %d, want %d", dstoff, len(data)) - } - if n != int64(len(data)) { - t.Errorf("short ReadFrom: wrote %d bytes, want %d", n, len(data)) - } - mustSeekStart(t, dst) - mustContainData(t, dst, data) +func testCopyFiles(t *testing.T, size, limit int64) { + testCopyFileRange(t, size, limit) +} - // If we had a limit, check that it was updated. - if lr != nil { - if want := limit - n; lr.N != want { - t.Fatalf("didn't update limit correctly: got %d, want %d", lr.N, want) - } - } +func testCopyFileRange(t *testing.T, size int64, limit int64) { + dst, src, data, hook, name := newCopyFileRangeTest(t, size) + testCopyFile(t, dst, src, data, hook, limit, name) } // newCopyFileRangeTest initializes a new test for copy_file_range. // -// It creates source and destination files, and populates the source file -// with random data of the specified size. It also hooks package os' call -// to poll.CopyFileRange and returns the hook so it can be inspected. -func newCopyFileRangeTest(t *testing.T, size int64) (dst, src *File, data []byte, hook *copyFileRangeHook) { +// It hooks package os' call to poll.CopyFileRange and returns the hook, +// so it can be inspected. +func newCopyFileRangeTest(t *testing.T, size int64) (dst, src *File, data []byte, hook *copyFileHook, name string) { t.Helper() - hook = hookCopyFileRange(t) - tmp := t.TempDir() + name = "newCopyFileRangeTest" - src, err := Create(filepath.Join(tmp, "src")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { src.Close() }) + dst, src, data = newCopyFileTest(t, size) + hook, _ = hookCopyFileRange(t) - dst, err = Create(filepath.Join(tmp, "dst")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { dst.Close() }) - - // Populate the source file with data, then rewind it, so it can be - // consumed by copy_file_range(2). - prng := rand.New(rand.NewSource(time.Now().Unix())) - data = make([]byte, size) - prng.Read(data) - if _, err := src.Write(data); err != nil { - t.Fatal(err) - } - if _, err := src.Seek(0, io.SeekStart); err != nil { - t.Fatal(err) - } - - return dst, src, data, hook + return } // newSpliceFileTest initializes a new test for splice. @@ -619,63 +304,22 @@ func newSpliceFileTest(t *testing.T, proto string, size int64) (*File, net.Conn, return dst, server, data, hook, func() { <-done } } -// mustContainData ensures that the specified file contains exactly the -// specified data. -func mustContainData(t *testing.T, f *File, data []byte) { - t.Helper() +func hookCopyFileRange(t *testing.T) (hook *copyFileHook, name string) { + name = "hookCopyFileRange" - got := make([]byte, len(data)) - if _, err := io.ReadFull(f, got); err != nil { - t.Fatal(err) - } - if !bytes.Equal(got, data) { - t.Fatalf("didn't get the same data back from %s", f.Name()) - } - if _, err := f.Read(make([]byte, 1)); err != io.EOF { - t.Fatalf("not at EOF") - } -} - -func mustSeekStart(t *testing.T, f *File) { - if _, err := f.Seek(0, io.SeekStart); err != nil { - t.Fatal(err) - } -} - -func hookCopyFileRange(t *testing.T) *copyFileRangeHook { - h := new(copyFileRangeHook) - h.install() - t.Cleanup(h.uninstall) - return h -} - -type copyFileRangeHook struct { - called bool - dstfd int - srcfd int - remain int64 - - written int64 - handled bool - err error - - original func(dst, src *poll.FD, remain int64) (int64, bool, error) -} - -func (h *copyFileRangeHook) install() { - h.original = *PollCopyFileRangeP + hook = new(copyFileHook) + orig := *PollCopyFileRangeP + t.Cleanup(func() { + *PollCopyFileRangeP = orig + }) *PollCopyFileRangeP = func(dst, src *poll.FD, remain int64) (int64, bool, error) { - h.called = true - h.dstfd = dst.Sysfd - h.srcfd = src.Sysfd - h.remain = remain - h.written, h.handled, h.err = h.original(dst, src, remain) - return h.written, h.handled, h.err + hook.called = true + hook.dstfd = dst.Sysfd + hook.srcfd = src.Sysfd + hook.written, hook.handled, hook.err = orig(dst, src, remain) + return hook.written, hook.handled, hook.err } -} - -func (h *copyFileRangeHook) uninstall() { - *PollCopyFileRangeP = h.original + return } func hookSpliceFile(t *testing.T) *spliceFileHook { @@ -784,41 +428,3 @@ func testGetPollFDAndNetwork(t *testing.T, proto string) { t.Fatalf("server Control error: %v", err) } } - -func createSocketPair(t *testing.T, proto string) (client, server net.Conn) { - t.Helper() - if !nettest.TestableNetwork(proto) { - t.Skipf("%s does not support %q", runtime.GOOS, proto) - } - - ln, err := nettest.NewLocalListener(proto) - if err != nil { - t.Fatalf("NewLocalListener error: %v", err) - } - t.Cleanup(func() { - if ln != nil { - ln.Close() - } - if client != nil { - client.Close() - } - if server != nil { - server.Close() - } - }) - ch := make(chan struct{}) - go func() { - var err error - server, err = ln.Accept() - if err != nil { - t.Errorf("Accept new connection error: %v", err) - } - ch <- struct{}{} - }() - client, err = net.Dial(proto, ln.Addr().String()) - <-ch - if err != nil { - t.Fatalf("Dial new connection error: %v", err) - } - return client, server -} diff --git a/src/os/readfrom_sendfile_test.go b/src/os/readfrom_sendfile_test.go new file mode 100644 index 00000000000000..86ef71ee0293c2 --- /dev/null +++ b/src/os/readfrom_sendfile_test.go @@ -0,0 +1,55 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build solaris + +package os_test + +import ( + "io" + . "os" + "testing" +) + +func BenchmarkSendFile(b *testing.B) { + hook := hookSendFileTB(b) + + // 1 GiB file size for copy. + const fileSize = 1 << 30 + + src, _ := createTempFile(b, "benchmark-sendfile-src", int64(fileSize)) + dst, err := CreateTemp(b.TempDir(), "benchmark-sendfile-dst") + if err != nil { + b.Fatalf("failed to create temporary file of destination: %v", err) + } + b.Cleanup(func() { + dst.Close() + }) + + b.ReportAllocs() + b.SetBytes(int64(fileSize)) + b.ResetTimer() + + for i := 0; i <= b.N; i++ { + sent, err := io.Copy(dst, src) + + if err != nil { + b.Fatalf("failed to copy data: %v", err) + } + if !hook.called { + b.Fatalf("should have called the sendfile(2)") + } + if sent != int64(fileSize) { + b.Fatalf("sent %d bytes, want %d", sent, fileSize) + } + + // Rewind the files for the next iteration. + if _, err := src.Seek(0, io.SeekStart); err != nil { + b.Fatalf("failed to rewind the source file: %v", err) + } + if _, err := dst.Seek(0, io.SeekStart); err != nil { + b.Fatalf("failed to rewind the destination file: %v", err) + } + } +} diff --git a/src/os/readfrom_solaris_test.go b/src/os/readfrom_solaris_test.go new file mode 100644 index 00000000000000..b460f4c113e9f6 --- /dev/null +++ b/src/os/readfrom_solaris_test.go @@ -0,0 +1,60 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os_test + +import ( + "internal/poll" + . "os" + "testing" +) + +var ( + copyFileTests = []copyFileTestFunc{newSendfileTest} + copyFileHooks = []copyFileTestHook{hookSendFile} +) + +func testCopyFiles(t *testing.T, size, limit int64) { + testSendfile(t, size, limit) +} + +func testSendfile(t *testing.T, size int64, limit int64) { + dst, src, data, hook, name := newSendfileTest(t, size) + testCopyFile(t, dst, src, data, hook, limit, name) +} + +// newSendFileTest initializes a new test for sendfile over copy_file_range. +// It hooks package os' call to poll.SendFile and returns the hook, +// so it can be inspected. +func newSendfileTest(t *testing.T, size int64) (dst, src *File, data []byte, hook *copyFileHook, name string) { + t.Helper() + + name = "newSendfileTest" + + dst, src, data = newCopyFileTest(t, size) + hook, _ = hookSendFile(t) + + return +} + +func hookSendFile(t *testing.T) (*copyFileHook, string) { + return hookSendFileTB(t), "hookSendFile" +} + +func hookSendFileTB(tb testing.TB) *copyFileHook { + hook := new(copyFileHook) + orig := poll.TestHookDidSendFile + tb.Cleanup(func() { + poll.TestHookDidSendFile = orig + }) + poll.TestHookDidSendFile = func(dstFD *poll.FD, src uintptr, written int64, err error, handled bool) { + hook.called = true + hook.dstfd = dstFD.Sysfd + hook.srcfd = int(src) + hook.written = written + hook.err = err + hook.handled = handled + } + return hook +} diff --git a/src/os/readfrom_unix_test.go b/src/os/readfrom_unix_test.go new file mode 100644 index 00000000000000..dbe2b683a7d7ed --- /dev/null +++ b/src/os/readfrom_unix_test.go @@ -0,0 +1,467 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build freebsd || linux || solaris + +package os_test + +import ( + "bytes" + "io" + "math/rand" + . "os" + "runtime" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +type ( + copyFileTestFunc func(*testing.T, int64) (*File, *File, []byte, *copyFileHook, string) + copyFileTestHook func(*testing.T) (*copyFileHook, string) +) + +func TestCopyFile(t *testing.T) { + sizes := []int{ + 1, + 42, + 1025, + syscall.Getpagesize() + 1, + 32769, + } + t.Run("Basic", func(t *testing.T) { + for _, size := range sizes { + t.Run(strconv.Itoa(size), func(t *testing.T) { + testCopyFiles(t, int64(size), -1) + }) + } + }) + t.Run("Limited", func(t *testing.T) { + t.Run("OneLess", func(t *testing.T) { + for _, size := range sizes { + t.Run(strconv.Itoa(size), func(t *testing.T) { + testCopyFiles(t, int64(size), int64(size)-1) + }) + } + }) + t.Run("Half", func(t *testing.T) { + for _, size := range sizes { + t.Run(strconv.Itoa(size), func(t *testing.T) { + testCopyFiles(t, int64(size), int64(size)/2) + }) + } + }) + t.Run("More", func(t *testing.T) { + for _, size := range sizes { + t.Run(strconv.Itoa(size), func(t *testing.T) { + testCopyFiles(t, int64(size), int64(size)+7) + }) + } + }) + }) + t.Run("DoesntTryInAppendMode", func(t *testing.T) { + for _, newTest := range copyFileTests { + dst, src, data, hook, testName := newTest(t, 42) + + dst2, err := OpenFile(dst.Name(), O_RDWR|O_APPEND, 0755) + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + defer dst2.Close() + + if _, err := io.Copy(dst2, src); err != nil { + t.Fatalf("%s: %v", testName, err) + } + switch runtime.GOOS { + case "illumos", "solaris": // sendfile() on SunOS allows target file with O_APPEND set. + if !hook.called { + t.Fatalf("%s: should have called the hook even with destination in O_APPEND mode", testName) + } + default: + if hook.called { + t.Fatalf("%s: hook shouldn't be called with destination in O_APPEND mode", testName) + } + } + mustSeekStart(t, dst2) + mustContainData(t, dst2, data) // through traditional means + } + }) + t.Run("CopyFileItself", func(t *testing.T) { + for _, hookFunc := range copyFileHooks { + hook, testName := hookFunc(t) + + f, err := CreateTemp("", "file-readfrom-itself-test") + if err != nil { + t.Fatalf("%s: failed to create tmp file: %v", testName, err) + } + t.Cleanup(func() { + f.Close() + Remove(f.Name()) + }) + + data := []byte("hello world!") + if _, err := f.Write(data); err != nil { + t.Fatalf("%s: failed to create and feed the file: %v", testName, err) + } + + if err := f.Sync(); err != nil { + t.Fatalf("%s: failed to save the file: %v", testName, err) + } + + // Rewind it. + if _, err := f.Seek(0, io.SeekStart); err != nil { + t.Fatalf("%s: failed to rewind the file: %v", testName, err) + } + + // Read data from the file itself. + if _, err := io.Copy(f, f); err != nil { + t.Fatalf("%s: failed to read from the file: %v", testName, err) + } + + if hook.written != 0 || hook.handled || hook.err != nil { + t.Fatalf("%s: File.readFrom is expected not to use any zero-copy techniques when copying itself."+ + "got hook.written=%d, hook.handled=%t, hook.err=%v; expected hook.written=0, hook.handled=false, hook.err=nil", + testName, hook.written, hook.handled, hook.err) + } + + switch testName { + case "hookCopyFileRange": + // For copy_file_range(2), it fails and returns EINVAL when the source and target + // refer to the same file and their ranges overlap. The hook should be called to + // get the returned error and fall back to generic copy. + if !hook.called { + t.Fatalf("%s: should have called the hook", testName) + } + case "hookSendFile", "hookSendFileOverCopyFileRange": + // For sendfile(2), it allows the source and target to refer to the same file and overlap. + // The hook should not be called and just fall back to generic copy directly. + if hook.called { + t.Fatalf("%s: shouldn't have called the hook", testName) + } + default: + t.Fatalf("%s: unexpected test", testName) + } + + // Rewind it. + if _, err := f.Seek(0, io.SeekStart); err != nil { + t.Fatalf("%s: failed to rewind the file: %v", testName, err) + } + + data2, err := io.ReadAll(f) + if err != nil { + t.Fatalf("%s: failed to read from the file: %v", testName, err) + } + + // It should wind up a double of the original data. + if s := strings.Repeat(string(data), 2); s != string(data2) { + t.Fatalf("%s: file contained %s, expected %s", testName, data2, s) + } + } + }) + t.Run("NotRegular", func(t *testing.T) { + t.Run("BothPipes", func(t *testing.T) { + for _, hookFunc := range copyFileHooks { + hook, testName := hookFunc(t) + + pr1, pw1, err := Pipe() + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + defer pr1.Close() + defer pw1.Close() + + pr2, pw2, err := Pipe() + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + defer pr2.Close() + defer pw2.Close() + + // The pipe is empty, and PIPE_BUF is large enough + // for this, by (POSIX) definition, so there is no + // need for an additional goroutine. + data := []byte("hello") + if _, err := pw1.Write(data); err != nil { + t.Fatalf("%s: %v", testName, err) + } + pw1.Close() + + n, err := io.Copy(pw2, pr1) + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + if n != int64(len(data)) { + t.Fatalf("%s: transferred %d, want %d", testName, n, len(data)) + } + switch runtime.GOOS { + case "illumos", "solaris": + // On solaris, We rely on File.Stat to get the size of the source file, + // which doesn't work for pipe. + // On illumos, We skip anything other than regular files conservatively + // for the target file, therefore the hook shouldn't have been called. + if hook.called { + t.Fatalf("%s: shouldn't have called the hook with a source or a destination of pipe", testName) + } + default: + if !hook.called { + t.Fatalf("%s: should have called the hook with both source and destination of pipe", testName) + } + } + pw2.Close() + mustContainData(t, pr2, data) + } + }) + t.Run("DstPipe", func(t *testing.T) { + for _, newTest := range copyFileTests { + dst, src, data, hook, testName := newTest(t, 255) + dst.Close() + + pr, pw, err := Pipe() + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + defer pr.Close() + defer pw.Close() + + n, err := io.Copy(pw, src) + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + if n != int64(len(data)) { + t.Fatalf("%s: transferred %d, want %d", testName, n, len(data)) + } + switch runtime.GOOS { + case "illumos": + // On illumos, We skip anything other than regular files conservatively + // for the target file, therefore the hook shouldn't have been called. + if hook.called { + t.Fatalf("%s: shouldn't have called the hook with a destination of pipe", testName) + } + default: + if !hook.called { + t.Fatalf("%s: should have called the hook with a destination of pipe", testName) + } + } + pw.Close() + mustContainData(t, pr, data) + } + }) + t.Run("SrcPipe", func(t *testing.T) { + for _, newTest := range copyFileTests { + dst, src, data, hook, testName := newTest(t, 255) + src.Close() + + pr, pw, err := Pipe() + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + defer pr.Close() + defer pw.Close() + + // The pipe is empty, and PIPE_BUF is large enough + // for this, by (POSIX) definition, so there is no + // need for an additional goroutine. + if _, err := pw.Write(data); err != nil { + t.Fatalf("%s: %v", testName, err) + } + pw.Close() + + n, err := io.Copy(dst, pr) + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + if n != int64(len(data)) { + t.Fatalf("%s: transferred %d, want %d", testName, n, len(data)) + } + switch runtime.GOOS { + case "illumos", "solaris": + // On SunOS, We rely on File.Stat to get the size of the source file, + // which doesn't work for pipe. + if hook.called { + t.Fatalf("%s: shouldn't have called the hook with a source of pipe", testName) + } + default: + if !hook.called { + t.Fatalf("%s: should have called the hook with a source of pipe", testName) + } + } + mustSeekStart(t, dst) + mustContainData(t, dst, data) + } + }) + }) + t.Run("Nil", func(t *testing.T) { + var nilFile *File + anyFile, err := CreateTemp("", "") + if err != nil { + t.Fatal(err) + } + defer Remove(anyFile.Name()) + defer anyFile.Close() + + if _, err := io.Copy(nilFile, nilFile); err != ErrInvalid { + t.Errorf("io.Copy(nilFile, nilFile) = %v, want %v", err, ErrInvalid) + } + if _, err := io.Copy(anyFile, nilFile); err != ErrInvalid { + t.Errorf("io.Copy(anyFile, nilFile) = %v, want %v", err, ErrInvalid) + } + if _, err := io.Copy(nilFile, anyFile); err != ErrInvalid { + t.Errorf("io.Copy(nilFile, anyFile) = %v, want %v", err, ErrInvalid) + } + + if _, err := nilFile.ReadFrom(nilFile); err != ErrInvalid { + t.Errorf("nilFile.ReadFrom(nilFile) = %v, want %v", err, ErrInvalid) + } + if _, err := anyFile.ReadFrom(nilFile); err != ErrInvalid { + t.Errorf("anyFile.ReadFrom(nilFile) = %v, want %v", err, ErrInvalid) + } + if _, err := nilFile.ReadFrom(anyFile); err != ErrInvalid { + t.Errorf("nilFile.ReadFrom(anyFile) = %v, want %v", err, ErrInvalid) + } + }) +} + +func testCopyFile(t *testing.T, dst, src *File, data []byte, hook *copyFileHook, limit int64, testName string) { + // If we have a limit, wrap the reader. + var ( + realsrc io.Reader + lr *io.LimitedReader + ) + if limit >= 0 { + lr = &io.LimitedReader{N: limit, R: src} + realsrc = lr + if limit < int64(len(data)) { + data = data[:limit] + } + } else { + realsrc = src + } + + // Now call ReadFrom (through io.Copy), which will hopefully call + // poll.CopyFileRange or poll.SendFile. + n, err := io.Copy(dst, realsrc) + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + + // If we didn't have a limit or had a positive limit, we should have called + // poll.CopyFileRange or poll.SendFile with the right file descriptor arguments. + if limit != 0 && !hook.called { + t.Fatalf("%s: never called the hook", testName) + } + if hook.called && hook.dstfd != int(dst.Fd()) { + t.Fatalf("%s: wrong destination file descriptor: got %d, want %d", testName, hook.dstfd, dst.Fd()) + } + if hook.called && hook.srcfd != int(src.Fd()) { + t.Fatalf("%s: wrong source file descriptor: got %d, want %d", testName, hook.srcfd, src.Fd()) + } + + // Check that the offsets after the transfer make sense, that the size + // of the transfer was reported correctly, and that the destination + // file contains exactly the bytes we expect it to contain. + dstoff, err := dst.Seek(0, io.SeekCurrent) + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + srcoff, err := src.Seek(0, io.SeekCurrent) + if err != nil { + t.Fatalf("%s: %v", testName, err) + } + if dstoff != srcoff { + t.Errorf("%s: offsets differ: dstoff = %d, srcoff = %d", testName, dstoff, srcoff) + } + if dstoff != int64(len(data)) { + t.Errorf("%s: dstoff = %d, want %d", testName, dstoff, len(data)) + } + if n != int64(len(data)) { + t.Errorf("%s: short ReadFrom: wrote %d bytes, want %d", testName, n, len(data)) + } + mustSeekStart(t, dst) + mustContainData(t, dst, data) + + // If we had a limit, check that it was updated. + if lr != nil { + if want := limit - n; lr.N != want { + t.Fatalf("%s: didn't update limit correctly: got %d, want %d", testName, lr.N, want) + } + } +} + +// mustContainData ensures that the specified file contains exactly the +// specified data. +func mustContainData(t *testing.T, f *File, data []byte) { + t.Helper() + + got := make([]byte, len(data)) + if _, err := io.ReadFull(f, got); err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Fatalf("didn't get the same data back from %s", f.Name()) + } + if _, err := f.Read(make([]byte, 1)); err != io.EOF { + t.Fatalf("not at EOF") + } +} + +func mustSeekStart(t *testing.T, f *File) { + if _, err := f.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } +} + +// newCopyFileTest initializes a new test for copying data between files. +// It creates source and destination files, and populates the source file +// with random data of the specified size, then rewind it, so it can be +// consumed by copy_file_range(2) or sendfile(2). +func newCopyFileTest(t *testing.T, size int64) (dst, src *File, data []byte) { + src, data = createTempFile(t, "test-copy-file-src", size) + + dst, err := CreateTemp(t.TempDir(), "test-copy-file-dst") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { dst.Close() }) + + return +} + +type copyFileHook struct { + called bool + dstfd int + srcfd int + + written int64 + handled bool + err error +} + +func createTempFile(tb testing.TB, name string, size int64) (*File, []byte) { + f, err := CreateTemp(tb.TempDir(), name) + if err != nil { + tb.Fatalf("failed to create temporary file: %v", err) + } + tb.Cleanup(func() { + f.Close() + }) + + randSeed := time.Now().Unix() + tb.Logf("random data seed: %d\n", randSeed) + prng := rand.New(rand.NewSource(randSeed)) + data := make([]byte, size) + prng.Read(data) + if _, err := f.Write(data); err != nil { + tb.Fatalf("failed to create and feed the file: %v", err) + } + if err := f.Sync(); err != nil { + tb.Fatalf("failed to save the file: %v", err) + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + tb.Fatalf("failed to rewind the file: %v", err) + } + + return f, data +} diff --git a/src/os/removeall_at.go b/src/os/removeall_at.go index cc254e0043fc09..5ddc1ade6134e5 100644 --- a/src/os/removeall_at.go +++ b/src/os/removeall_at.go @@ -2,13 +2,13 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build unix +//go:build unix || wasip1 || windows package os import ( - "internal/syscall/unix" "io" + "runtime" "syscall" ) @@ -35,7 +35,15 @@ func removeAll(path string) error { // its parent directory parentDir, base := splitPath(path) - parent, err := Open(parentDir) + flag := O_RDONLY + if runtime.GOOS == "windows" { + // On Windows, the process might not have read permission on the parent directory, + // but still can delete files in it. See https://go.dev/issue/74134. + // We can open a file even if we don't have read permission by passing the + // O_WRONLY | O_RDWR flag, which is mapped to FILE_READ_ATTRIBUTES. + flag = O_WRONLY | O_RDWR + } + parent, err := OpenFile(parentDir, flag, 0) if IsNotExist(err) { // If parent does not exist, base cannot exist. Fail silently return nil @@ -45,7 +53,7 @@ func removeAll(path string) error { } defer parent.Close() - if err := removeAllFrom(parent, base); err != nil { + if err := removeAllFrom(sysfdType(parent.Fd()), base); err != nil { if pathErr, ok := err.(*PathError); ok { pathErr.Path = parentDir + string(PathSeparator) + pathErr.Path err = pathErr @@ -55,12 +63,9 @@ func removeAll(path string) error { return nil } -func removeAllFrom(parent *File, base string) error { - parentFd := int(parent.Fd()) +func removeAllFrom(parentFd sysfdType, base string) error { // Simple case: if Unlink (aka remove) works, we're done. - err := ignoringEINTR(func() error { - return unix.Unlinkat(parentFd, base, 0) - }) + err := removefileat(parentFd, base) if err == nil || IsNotExist(err) { return nil } @@ -82,13 +87,13 @@ func removeAllFrom(parent *File, base string) error { const reqSize = 1024 var respSize int - // Open the directory to recurse into + // Open the directory to recurse into. file, err := openDirAt(parentFd, base) if err != nil { if IsNotExist(err) { return nil } - if err == syscall.ENOTDIR || err == unix.NoFollowErrno { + if err == syscall.ENOTDIR || isErrNoFollow(err) { // Not a directory; return the error from the unix.Unlinkat. return &PathError{Op: "unlinkat", Path: base, Err: uErr} } @@ -111,7 +116,7 @@ func removeAllFrom(parent *File, base string) error { respSize = len(names) for _, name := range names { - err := removeAllFrom(file, name) + err := removeAllFrom(sysfdType(file.Fd()), name) if err != nil { if pathErr, ok := err.(*PathError); ok { pathErr.Path = base + string(PathSeparator) + pathErr.Path @@ -144,9 +149,7 @@ func removeAllFrom(parent *File, base string) error { } // Remove the directory itself. - unlinkError := ignoringEINTR(func() error { - return unix.Unlinkat(parentFd, base, unix.AT_REMOVEDIR) - }) + unlinkError := removedirat(parentFd, base) if unlinkError == nil || IsNotExist(unlinkError) { return nil } @@ -165,27 +168,10 @@ func removeAllFrom(parent *File, base string) error { // This acts like openFileNolog rather than OpenFile because // we are going to (try to) remove the file. // The contents of this file are not relevant for test caching. -func openDirAt(dirfd int, name string) (*File, error) { - var r int - for { - var e error - r, e = unix.Openat(dirfd, name, O_RDONLY|syscall.O_CLOEXEC|syscall.O_DIRECTORY|syscall.O_NOFOLLOW, 0) - if e == nil { - break - } - - // See comment in openFileNolog. - if e == syscall.EINTR { - continue - } - - return nil, e - } - - if !supportsCloseOnExec { - syscall.CloseOnExec(r) +func openDirAt(dirfd sysfdType, name string) (*File, error) { + fd, err := rootOpenDir(dirfd, name) + if err != nil { + return nil, err } - - // We use kindNoPoll because we know that this is a directory. - return newFile(r, name, kindNoPoll, false), nil + return newDirFile(fd, name) } diff --git a/src/os/removeall_noat.go b/src/os/removeall_noat.go index 2b8a7727f4f3e1..395a1503d49442 100644 --- a/src/os/removeall_noat.go +++ b/src/os/removeall_noat.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !unix +//go:build (js && wasm) || plan9 package os diff --git a/src/os/removeall_test.go b/src/os/removeall_test.go index 6aca98021fb393..bc439e4a5eb762 100644 --- a/src/os/removeall_test.go +++ b/src/os/removeall_test.go @@ -159,40 +159,25 @@ func TestRemoveAllLongPath(t *testing.T) { t.Skip("skipping for not implemented platforms") } - prevDir, err := Getwd() - if err != nil { - t.Fatalf("Could not get wd: %s", err) - } - - startPath, err := MkdirTemp("", "TestRemoveAllLongPath-") - if err != nil { - t.Fatalf("Could not create TempDir: %s", err) - } - defer RemoveAll(startPath) - - err = Chdir(startPath) - if err != nil { - t.Fatalf("Could not chdir %s: %s", startPath, err) - } + startPath := t.TempDir() + t.Chdir(startPath) - // Removing paths with over 4096 chars commonly fails + // Removing paths with over 4096 chars commonly fails. + name := strings.Repeat("a", 100) for i := 0; i < 41; i++ { - name := strings.Repeat("a", 100) - - err = Mkdir(name, 0755) - if err != nil { + if err := Mkdir(name, 0755); err != nil { t.Fatalf("Could not mkdir %s: %s", name, err) } - - err = Chdir(name) - if err != nil { + if err := Chdir(name); err != nil { t.Fatalf("Could not chdir %s: %s", name, err) } } - err = Chdir(prevDir) + // Chdir out of startPath before attempting to remove it, + // otherwise RemoveAll fails on aix, illumos and solaris. + err := Chdir(filepath.Join(startPath, "..")) if err != nil { - t.Fatalf("Could not chdir %s: %s", prevDir, err) + t.Fatalf("Could not chdir: %s", err) } err = RemoveAll(startPath) @@ -202,30 +187,11 @@ func TestRemoveAllLongPath(t *testing.T) { } func TestRemoveAllDot(t *testing.T) { - prevDir, err := Getwd() - if err != nil { - t.Fatalf("Could not get wd: %s", err) - } - tempDir, err := MkdirTemp("", "TestRemoveAllDot-") - if err != nil { - t.Fatalf("Could not create TempDir: %s", err) - } - defer RemoveAll(tempDir) - - err = Chdir(tempDir) - if err != nil { - t.Fatalf("Could not chdir to tempdir: %s", err) - } + t.Chdir(t.TempDir()) - err = RemoveAll(".") - if err == nil { + if err := RemoveAll("."); err == nil { t.Errorf("RemoveAll succeed to remove .") } - - err = Chdir(prevDir) - if err != nil { - t.Fatalf("Could not chdir %s: %s", prevDir, err) - } } func TestRemoveAllDotDot(t *testing.T) { @@ -505,6 +471,27 @@ func TestRemoveAllNoFcntl(t *testing.T) { } } +func TestRemoveAllTrailingSlash(t *testing.T) { + slashes := []string{"/"} + if runtime.GOOS == "windows" { + slashes = append(slashes, `\`) + } + for _, slash := range slashes { + dir := makefs(t, []string{ + "dir/a/file1", + "dir/a/file2", + "dir/file3", + }) + path := dir + "/dir" + if err := RemoveAll(path + slash); err != nil { + t.Fatal(err) + } + if _, err := Stat(path); !IsNotExist(err) { + t.Errorf("after RemoveAll(%q), directory still exists", path+slash) + } + } +} + func BenchmarkRemoveAll(b *testing.B) { tmpDir := filepath.Join(b.TempDir(), "target") b.ReportAllocs() diff --git a/src/os/removeall_unix.go b/src/os/removeall_unix.go new file mode 100644 index 00000000000000..287fc81fa9c237 --- /dev/null +++ b/src/os/removeall_unix.go @@ -0,0 +1,20 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix || wasip1 + +package os + +import ( + "internal/syscall/unix" +) + +func isErrNoFollow(err error) bool { + return err == unix.NoFollowErrno +} + +func newDirFile(fd int, name string) (*File, error) { + // We use kindNoPoll because we know that this is a directory. + return newFile(fd, name, kindNoPoll, false), nil +} diff --git a/src/os/removeall_windows.go b/src/os/removeall_windows.go new file mode 100644 index 00000000000000..5cbc5fb0367a92 --- /dev/null +++ b/src/os/removeall_windows.go @@ -0,0 +1,17 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build windows + +package os + +import "syscall" + +func isErrNoFollow(err error) bool { + return err == syscall.ELOOP +} + +func newDirFile(fd syscall.Handle, name string) (*File, error) { + return newFile(fd, name, "file", false), nil +} diff --git a/src/os/root.go b/src/os/root.go new file mode 100644 index 00000000000000..d759727ce7a89b --- /dev/null +++ b/src/os/root.go @@ -0,0 +1,451 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +import ( + "errors" + "internal/bytealg" + "internal/stringslite" + "internal/testlog" + "io/fs" + "runtime" + "slices" + "time" +) + +// OpenInRoot opens the file name in the directory dir. +// It is equivalent to OpenRoot(dir) followed by opening the file in the root. +// +// OpenInRoot returns an error if any component of the name +// references a location outside of dir. +// +// See [Root] for details and limitations. +func OpenInRoot(dir, name string) (*File, error) { + r, err := OpenRoot(dir) + if err != nil { + return nil, err + } + defer r.Close() + return r.Open(name) +} + +// Root may be used to only access files within a single directory tree. +// +// Methods on Root can only access files and directories beneath a root directory. +// If any component of a file name passed to a method of Root references a location +// outside the root, the method returns an error. +// File names may reference the directory itself (.). +// +// Methods on Root will follow symbolic links, but symbolic links may not +// reference a location outside the root. +// Symbolic links must not be absolute. +// +// Methods on Root do not prohibit traversal of filesystem boundaries, +// Linux bind mounts, /proc special files, or access to Unix device files. +// +// Methods on Root are safe to be used from multiple goroutines simultaneously. +// +// On most platforms, creating a Root opens a file descriptor or handle referencing +// the directory. If the directory is moved, methods on Root reference the original +// directory in its new location. +// +// Root's behavior differs on some platforms: +// +// - When GOOS=windows, file names may not reference Windows reserved device names +// such as NUL and COM1. +// - On Unix, [Root.Chmod], [Root.Chown], and [Root.Chtimes] are vulnerable to a race condition. +// If the target of the operation is changed from a regular file to a symlink +// while the operation is in progress, the operation may be performed on the link +// rather than the link target. +// - When GOOS=js, Root is vulnerable to TOCTOU (time-of-check-time-of-use) +// attacks in symlink validation, and cannot ensure that operations will not +// escape the root. +// - When GOOS=plan9 or GOOS=js, Root does not track directories across renames. +// On these platforms, a Root references a directory name, not a file descriptor. +// - WASI preview 1 (GOOS=wasip1) does not support [Root.Chmod]. +type Root struct { + root *root +} + +const ( + // Maximum number of symbolic links we will follow when resolving a file in a root. + // 8 is __POSIX_SYMLOOP_MAX (the minimum allowed value for SYMLOOP_MAX), + // and a common limit. + rootMaxSymlinks = 8 +) + +// OpenRoot opens the named directory. +// It follows symbolic links in the directory name. +// If there is an error, it will be of type [*PathError]. +func OpenRoot(name string) (*Root, error) { + testlog.Open(name) + return openRootNolog(name) +} + +// Name returns the name of the directory presented to OpenRoot. +// +// It is safe to call Name after [Close]. +func (r *Root) Name() string { + return r.root.Name() +} + +// Close closes the Root. +// After Close is called, methods on Root return errors. +func (r *Root) Close() error { + return r.root.Close() +} + +// Open opens the named file in the root for reading. +// See [Open] for more details. +func (r *Root) Open(name string) (*File, error) { + return r.OpenFile(name, O_RDONLY, 0) +} + +// Create creates or truncates the named file in the root. +// See [Create] for more details. +func (r *Root) Create(name string) (*File, error) { + return r.OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666) +} + +// OpenFile opens the named file in the root. +// See [OpenFile] for more details. +// +// If perm contains bits other than the nine least-significant bits (0o777), +// OpenFile returns an error. +func (r *Root) OpenFile(name string, flag int, perm FileMode) (*File, error) { + if perm&0o777 != perm { + return nil, &PathError{Op: "openat", Path: name, Err: errors.New("unsupported file mode")} + } + r.logOpen(name) + rf, err := rootOpenFileNolog(r, name, flag, perm) + if err != nil { + return nil, err + } + rf.appendMode = flag&O_APPEND != 0 + return rf, nil +} + +// OpenRoot opens the named directory in the root. +// If there is an error, it will be of type [*PathError]. +func (r *Root) OpenRoot(name string) (*Root, error) { + r.logOpen(name) + return openRootInRoot(r, name) +} + +// Chmod changes the mode of the named file in the root to mode. +// See [Chmod] for more details. +func (r *Root) Chmod(name string, mode FileMode) error { + return rootChmod(r, name, mode) +} + +// Mkdir creates a new directory in the root +// with the specified name and permission bits (before umask). +// See [Mkdir] for more details. +// +// If perm contains bits other than the nine least-significant bits (0o777), +// Mkdir returns an error. +func (r *Root) Mkdir(name string, perm FileMode) error { + if perm&0o777 != perm { + return &PathError{Op: "mkdirat", Path: name, Err: errors.New("unsupported file mode")} + } + return rootMkdir(r, name, perm) +} + +// MkdirAll creates a new directory in the root, along with any necessary parents. +// See [MkdirAll] for more details. +// +// If perm contains bits other than the nine least-significant bits (0o777), +// MkdirAll returns an error. +func (r *Root) MkdirAll(name string, perm FileMode) error { + if perm&0o777 != perm { + return &PathError{Op: "mkdirat", Path: name, Err: errors.New("unsupported file mode")} + } + return rootMkdirAll(r, name, perm) +} + +// Chown changes the numeric uid and gid of the named file in the root. +// See [Chown] for more details. +func (r *Root) Chown(name string, uid, gid int) error { + return rootChown(r, name, uid, gid) +} + +// Lchown changes the numeric uid and gid of the named file in the root. +// See [Lchown] for more details. +func (r *Root) Lchown(name string, uid, gid int) error { + return rootLchown(r, name, uid, gid) +} + +// Chtimes changes the access and modification times of the named file in the root. +// See [Chtimes] for more details. +func (r *Root) Chtimes(name string, atime time.Time, mtime time.Time) error { + return rootChtimes(r, name, atime, mtime) +} + +// Remove removes the named file or (empty) directory in the root. +// See [Remove] for more details. +func (r *Root) Remove(name string) error { + return rootRemove(r, name) +} + +// RemoveAll removes the named file or directory and any children that it contains. +// See [RemoveAll] for more details. +func (r *Root) RemoveAll(name string) error { + return rootRemoveAll(r, name) +} + +// Stat returns a [FileInfo] describing the named file in the root. +// See [Stat] for more details. +func (r *Root) Stat(name string) (FileInfo, error) { + r.logStat(name) + return rootStat(r, name, false) +} + +// Lstat returns a [FileInfo] describing the named file in the root. +// If the file is a symbolic link, the returned FileInfo +// describes the symbolic link. +// See [Lstat] for more details. +func (r *Root) Lstat(name string) (FileInfo, error) { + r.logStat(name) + return rootStat(r, name, true) +} + +// Readlink returns the destination of the named symbolic link in the root. +// See [Readlink] for more details. +func (r *Root) Readlink(name string) (string, error) { + return rootReadlink(r, name) +} + +// Rename renames (moves) oldname to newname. +// Both paths are relative to the root. +// See [Rename] for more details. +func (r *Root) Rename(oldname, newname string) error { + return rootRename(r, oldname, newname) +} + +// Link creates newname as a hard link to the oldname file. +// Both paths are relative to the root. +// See [Link] for more details. +// +// If oldname is a symbolic link, Link creates new link to oldname and not its target. +// This behavior may differ from that of [Link] on some platforms. +// +// When GOOS=js, Link returns an error if oldname is a symbolic link. +func (r *Root) Link(oldname, newname string) error { + return rootLink(r, oldname, newname) +} + +// Symlink creates newname as a symbolic link to oldname. +// See [Symlink] for more details. +// +// Symlink does not validate oldname, +// which may reference a location outside the root. +// +// On Windows, a directory link is created if oldname references +// a directory within the root. Otherwise a file link is created. +func (r *Root) Symlink(oldname, newname string) error { + return rootSymlink(r, oldname, newname) +} + +// ReadFile reads the named file in the root and returns its contents. +// See [ReadFile] for more details. +func (r *Root) ReadFile(name string) ([]byte, error) { + f, err := r.Open(name) + if err != nil { + return nil, err + } + defer f.Close() + return readFileContents(statOrZero(f), f.Read) +} + +// WriteFile writes data to the named file in the root, creating it if necessary. +// See [WriteFile] for more details. +func (r *Root) WriteFile(name string, data []byte, perm FileMode) error { + f, err := r.OpenFile(name, O_WRONLY|O_CREATE|O_TRUNC, perm) + if err != nil { + return err + } + _, err = f.Write(data) + if err1 := f.Close(); err == nil { + err = err1 + } + return err +} + +func (r *Root) logOpen(name string) { + if log := testlog.Logger(); log != nil { + // This won't be right if r's name has changed since it was opened, + // but it's the best we can do. + log.Open(joinPath(r.Name(), name)) + } +} + +func (r *Root) logStat(name string) { + if log := testlog.Logger(); log != nil { + // This won't be right if r's name has changed since it was opened, + // but it's the best we can do. + log.Stat(joinPath(r.Name(), name)) + } +} + +// splitPathInRoot splits a path into components +// and joins it with the given prefix and suffix. +// +// The path is relative to a Root, and must not be +// absolute, volume-relative, or "". +// +// "." components are removed, except in the last component. +// +// Path separators following the last component are returned in suffixSep. +func splitPathInRoot(s string, prefix, suffix []string) (_ []string, suffixSep string, err error) { + if len(s) == 0 { + return nil, "", errors.New("empty path") + } + if IsPathSeparator(s[0]) { + return nil, "", errPathEscapes + } + + if runtime.GOOS == "windows" { + // Windows cleans paths before opening them. + s, err = rootCleanPath(s, prefix, suffix) + if err != nil { + return nil, "", err + } + prefix = nil + suffix = nil + } + + parts := slices.Clone(prefix) + i, j := 0, 1 + for { + if j < len(s) && !IsPathSeparator(s[j]) { + // Keep looking for the end of this component. + j++ + continue + } + parts = append(parts, s[i:j]) + // Advance to the next component, or end of the path. + partEnd := j + for j < len(s) && IsPathSeparator(s[j]) { + j++ + } + if j == len(s) { + // If this is the last path component, + // preserve any trailing path separators. + suffixSep = s[partEnd:] + break + } + if parts[len(parts)-1] == "." { + // Remove "." components, except at the end. + parts = parts[:len(parts)-1] + } + i = j + } + if len(suffix) > 0 && len(parts) > 0 && parts[len(parts)-1] == "." { + // Remove a trailing "." component if we're joining to a suffix. + parts = parts[:len(parts)-1] + } + parts = append(parts, suffix...) + return parts, suffixSep, nil +} + +// FS returns a file system (an fs.FS) for the tree of files in the root. +// +// The result implements [io/fs.StatFS], [io/fs.ReadFileFS], +// [io/fs.ReadDirFS], and [io/fs.ReadLinkFS]. +func (r *Root) FS() fs.FS { + return (*rootFS)(r) +} + +type rootFS Root + +func (rfs *rootFS) Open(name string) (fs.File, error) { + r := (*Root)(rfs) + if !isValidRootFSPath(name) { + return nil, &PathError{Op: "open", Path: name, Err: ErrInvalid} + } + f, err := r.Open(name) + if err != nil { + return nil, err + } + return f, nil +} + +func (rfs *rootFS) ReadDir(name string) ([]DirEntry, error) { + r := (*Root)(rfs) + if !isValidRootFSPath(name) { + return nil, &PathError{Op: "readdir", Path: name, Err: ErrInvalid} + } + + // This isn't efficient: We just open a regular file and ReadDir it. + // Ideally, we would skip creating a *File entirely and operate directly + // on the file descriptor, but that will require some extensive reworking + // of directory reading in general. + // + // This suffices for the moment. + f, err := r.Open(name) + if err != nil { + return nil, err + } + defer f.Close() + dirs, err := f.ReadDir(-1) + slices.SortFunc(dirs, func(a, b DirEntry) int { + return bytealg.CompareString(a.Name(), b.Name()) + }) + return dirs, err +} + +func (rfs *rootFS) ReadFile(name string) ([]byte, error) { + r := (*Root)(rfs) + if !isValidRootFSPath(name) { + return nil, &PathError{Op: "readfile", Path: name, Err: ErrInvalid} + } + f, err := r.Open(name) + if err != nil { + return nil, err + } + defer f.Close() + return readFileContents(statOrZero(f), f.Read) +} + +func (rfs *rootFS) ReadLink(name string) (string, error) { + r := (*Root)(rfs) + if !isValidRootFSPath(name) { + return "", &PathError{Op: "readlink", Path: name, Err: ErrInvalid} + } + return r.Readlink(name) +} + +func (rfs *rootFS) Stat(name string) (FileInfo, error) { + r := (*Root)(rfs) + if !isValidRootFSPath(name) { + return nil, &PathError{Op: "stat", Path: name, Err: ErrInvalid} + } + return r.Stat(name) +} + +func (rfs *rootFS) Lstat(name string) (FileInfo, error) { + r := (*Root)(rfs) + if !isValidRootFSPath(name) { + return nil, &PathError{Op: "lstat", Path: name, Err: ErrInvalid} + } + return r.Lstat(name) +} + +// isValidRootFSPath reports whether name is a valid filename to pass a Root.FS method. +func isValidRootFSPath(name string) bool { + if !fs.ValidPath(name) { + return false + } + if runtime.GOOS == "windows" { + // fs.FS paths are /-separated. + // On Windows, reject the path if it contains any \ separators. + // Other forms of invalid path (for example, "NUL") are handled by + // Root's usual file lookup mechanisms. + if stringslite.IndexByte(name, '\\') >= 0 { + return false + } + } + return true +} diff --git a/src/os/root_js.go b/src/os/root_js.go new file mode 100644 index 00000000000000..56a37dafe1626e --- /dev/null +++ b/src/os/root_js.go @@ -0,0 +1,113 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build js && wasm + +package os + +import ( + "errors" + "slices" + "syscall" +) + +// checkPathEscapes reports whether name escapes the root. +// +// Due to the lack of openat, checkPathEscapes is subject to TOCTOU races +// when symlinks change during the resolution process. +func checkPathEscapes(r *Root, name string) error { + return checkPathEscapesInternal(r, name, false) +} + +// checkPathEscapesLstat reports whether name escapes the root. +// It does not resolve symlinks in the final path component. +// +// Due to the lack of openat, checkPathEscapes is subject to TOCTOU races +// when symlinks change during the resolution process. +func checkPathEscapesLstat(r *Root, name string) error { + return checkPathEscapesInternal(r, name, true) +} + +func checkPathEscapesInternal(r *Root, name string, lstat bool) error { + if r.root.closed.Load() { + return ErrClosed + } + parts, suffixSep, err := splitPathInRoot(name, nil, nil) + if err != nil { + return err + } + + i := 0 + symlinks := 0 + base := r.root.name + for i < len(parts) { + if parts[i] == ".." { + // Resolve one or more parent ("..") path components. + end := i + 1 + for end < len(parts) && parts[end] == ".." { + end++ + } + count := end - i + if count > i { + return errPathEscapes + } + parts = slices.Delete(parts, i-count, end) + i -= count + base = r.root.name + for j := range i { + base = joinPath(base, parts[j]) + } + continue + } + + part := parts[i] + if i == len(parts)-1 { + if lstat { + break + } + part += suffixSep + } + + next := joinPath(base, part) + fi, err := Lstat(next) + if err != nil { + if IsNotExist(err) { + return nil + } + return underlyingError(err) + } + if fi.Mode()&ModeSymlink != 0 { + link, err := Readlink(next) + if err != nil { + return errPathEscapes + } + symlinks++ + if symlinks > rootMaxSymlinks { + return errors.New("too many symlinks") + } + newparts, newSuffixSep, err := splitPathInRoot(link, parts[:i], parts[i+1:]) + if err != nil { + return err + } + if i == len(parts) { + // suffixSep contains any trailing path separator characters + // in the link target. + // If we are replacing the remainder of the path, retain these. + // If we're replacing some intermediate component of the path, + // ignore them, since intermediate components must always be + // directories. + suffixSep = newSuffixSep + } + parts = newparts + continue + } + if !fi.IsDir() && i < len(parts)-1 { + return syscall.ENOTDIR + } + + base = next + i++ + } + return nil +} diff --git a/src/os/root_nonwindows.go b/src/os/root_nonwindows.go new file mode 100644 index 00000000000000..e40ce4d7d63e13 --- /dev/null +++ b/src/os/root_nonwindows.go @@ -0,0 +1,11 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !windows + +package os + +func rootCleanPath(s string, prefix, suffix []string) (string, error) { + return s, nil +} diff --git a/src/os/root_noopenat.go b/src/os/root_noopenat.go new file mode 100644 index 00000000000000..59f1abe91b0b2e --- /dev/null +++ b/src/os/root_noopenat.go @@ -0,0 +1,260 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (js && wasm) || plan9 + +package os + +import ( + "errors" + "internal/filepathlite" + "internal/stringslite" + "sync/atomic" + "syscall" + "time" +) + +// root implementation for platforms with no openat. +// Currently plan9 and js. +type root struct { + name string + closed atomic.Bool +} + +// openRootNolog is OpenRoot. +func openRootNolog(name string) (*Root, error) { + r, err := newRoot(name) + if err != nil { + return nil, &PathError{Op: "open", Path: name, Err: err} + } + return r, nil +} + +// openRootInRoot is Root.OpenRoot. +func openRootInRoot(r *Root, name string) (*Root, error) { + if err := checkPathEscapes(r, name); err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: err} + } + r, err := newRoot(joinPath(r.root.name, name)) + if err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: err} + } + return r, nil +} + +// newRoot returns a new Root. +// If fd is not a directory, it closes it and returns an error. +func newRoot(name string) (*Root, error) { + fi, err := Stat(name) + if err != nil { + return nil, err.(*PathError).Err + } + if !fi.IsDir() { + return nil, errors.New("not a directory") + } + return &Root{&root{name: name}}, nil +} + +func (r *root) Close() error { + // For consistency with platforms where Root.Close closes a handle, + // mark the Root as closed and return errors from future calls. + r.closed.Store(true) + return nil +} + +func (r *root) Name() string { + return r.name +} + +// rootOpenFileNolog is Root.OpenFile. +func rootOpenFileNolog(r *Root, name string, flag int, perm FileMode) (*File, error) { + if err := checkPathEscapes(r, name); err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: err} + } + f, err := openFileNolog(joinPath(r.root.name, name), flag, perm) + if err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: underlyingError(err)} + } + return f, nil +} + +func rootStat(r *Root, name string, lstat bool) (FileInfo, error) { + var fi FileInfo + var err error + if lstat { + err = checkPathEscapesLstat(r, name) + if err == nil { + fi, err = Lstat(joinPath(r.root.name, name)) + } + } else { + err = checkPathEscapes(r, name) + if err == nil { + fi, err = Stat(joinPath(r.root.name, name)) + } + } + if err != nil { + return nil, &PathError{Op: "statat", Path: name, Err: underlyingError(err)} + } + return fi, nil +} + +func rootChmod(r *Root, name string, mode FileMode) error { + if err := checkPathEscapes(r, name); err != nil { + return &PathError{Op: "chmodat", Path: name, Err: err} + } + if err := Chmod(joinPath(r.root.name, name), mode); err != nil { + return &PathError{Op: "chmodat", Path: name, Err: underlyingError(err)} + } + return nil +} + +func rootChown(r *Root, name string, uid, gid int) error { + if err := checkPathEscapes(r, name); err != nil { + return &PathError{Op: "chownat", Path: name, Err: err} + } + if err := Chown(joinPath(r.root.name, name), uid, gid); err != nil { + return &PathError{Op: "chownat", Path: name, Err: underlyingError(err)} + } + return nil +} + +func rootLchown(r *Root, name string, uid, gid int) error { + if err := checkPathEscapesLstat(r, name); err != nil { + return &PathError{Op: "lchownat", Path: name, Err: err} + } + if err := Lchown(joinPath(r.root.name, name), uid, gid); err != nil { + return &PathError{Op: "lchownat", Path: name, Err: underlyingError(err)} + } + return nil +} + +func rootChtimes(r *Root, name string, atime time.Time, mtime time.Time) error { + if err := checkPathEscapes(r, name); err != nil { + return &PathError{Op: "chtimesat", Path: name, Err: err} + } + if err := Chtimes(joinPath(r.root.name, name), atime, mtime); err != nil { + return &PathError{Op: "chtimesat", Path: name, Err: underlyingError(err)} + } + return nil +} + +func rootMkdir(r *Root, name string, perm FileMode) error { + if err := checkPathEscapes(r, name); err != nil { + return &PathError{Op: "mkdirat", Path: name, Err: err} + } + if err := Mkdir(joinPath(r.root.name, name), perm); err != nil { + return &PathError{Op: "mkdirat", Path: name, Err: underlyingError(err)} + } + return nil +} + +func rootMkdirAll(r *Root, name string, perm FileMode) error { + // We only check for errPathEscapes here. + // For errors such as ENOTDIR (a non-directory file appeared somewhere along the path), + // we let MkdirAll generate the error. + // MkdirAll will return a PathError referencing the exact location of the error, + // and we want to preserve that property. + if err := checkPathEscapes(r, name); err == errPathEscapes { + return &PathError{Op: "mkdirat", Path: name, Err: err} + } + prefix := r.root.name + string(PathSeparator) + if err := MkdirAll(prefix+name, perm); err != nil { + if pe, ok := err.(*PathError); ok { + pe.Op = "mkdirat" + pe.Path = stringslite.TrimPrefix(pe.Path, prefix) + return pe + } + return &PathError{Op: "mkdirat", Path: name, Err: underlyingError(err)} + } + return nil +} + +func rootRemove(r *Root, name string) error { + if err := checkPathEscapesLstat(r, name); err != nil { + return &PathError{Op: "removeat", Path: name, Err: err} + } + if endsWithDot(name) { + // We don't want to permit removing the root itself, so check for that. + if filepathlite.Clean(name) == "." { + return &PathError{Op: "removeat", Path: name, Err: errPathEscapes} + } + } + if err := Remove(joinPath(r.root.name, name)); err != nil { + return &PathError{Op: "removeat", Path: name, Err: underlyingError(err)} + } + return nil +} + +func rootRemoveAll(r *Root, name string) error { + if endsWithDot(name) { + // Consistency with os.RemoveAll: Return EINVAL when trying to remove . + return &PathError{Op: "RemoveAll", Path: name, Err: syscall.EINVAL} + } + if err := checkPathEscapesLstat(r, name); err != nil { + if err == syscall.ENOTDIR { + // Some intermediate path component is not a directory. + // RemoveAll treats this as success (since the target doesn't exist). + return nil + } + return &PathError{Op: "RemoveAll", Path: name, Err: err} + } + if err := RemoveAll(joinPath(r.root.name, name)); err != nil { + return &PathError{Op: "RemoveAll", Path: name, Err: underlyingError(err)} + } + return nil +} + +func rootReadlink(r *Root, name string) (string, error) { + if err := checkPathEscapesLstat(r, name); err != nil { + return "", &PathError{Op: "readlinkat", Path: name, Err: err} + } + name, err := Readlink(joinPath(r.root.name, name)) + if err != nil { + return "", &PathError{Op: "readlinkat", Path: name, Err: underlyingError(err)} + } + return name, nil +} + +func rootRename(r *Root, oldname, newname string) error { + if err := checkPathEscapesLstat(r, oldname); err != nil { + return &PathError{Op: "renameat", Path: oldname, Err: err} + } + if err := checkPathEscapesLstat(r, newname); err != nil { + return &PathError{Op: "renameat", Path: newname, Err: err} + } + err := Rename(joinPath(r.root.name, oldname), joinPath(r.root.name, newname)) + if err != nil { + return &LinkError{"renameat", oldname, newname, underlyingError(err)} + } + return nil +} + +func rootLink(r *Root, oldname, newname string) error { + if err := checkPathEscapesLstat(r, oldname); err != nil { + return &PathError{Op: "linkat", Path: oldname, Err: err} + } + fullOldName := joinPath(r.root.name, oldname) + if fs, err := Lstat(fullOldName); err == nil && fs.Mode()&ModeSymlink != 0 { + return &PathError{Op: "linkat", Path: oldname, Err: errors.New("cannot create a hard link to a symlink")} + } + if err := checkPathEscapesLstat(r, newname); err != nil { + return &PathError{Op: "linkat", Path: newname, Err: err} + } + err := Link(fullOldName, joinPath(r.root.name, newname)) + if err != nil { + return &LinkError{"linkat", oldname, newname, underlyingError(err)} + } + return nil +} + +func rootSymlink(r *Root, oldname, newname string) error { + if err := checkPathEscapesLstat(r, newname); err != nil { + return &PathError{Op: "symlinkat", Path: newname, Err: err} + } + err := Symlink(oldname, joinPath(r.root.name, newname)) + if err != nil { + return &LinkError{"symlinkat", oldname, newname, underlyingError(err)} + } + return nil +} diff --git a/src/os/root_openat.go b/src/os/root_openat.go new file mode 100644 index 00000000000000..83bde5ef14160d --- /dev/null +++ b/src/os/root_openat.go @@ -0,0 +1,406 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix || windows || wasip1 + +package os + +import ( + "runtime" + "slices" + "sync" + "syscall" + "time" +) + +// root implementation for platforms with a function to open a file +// relative to a directory. +type root struct { + name string + + // refs is incremented while an operation is using fd. + // closed is set when Close is called. + // fd is closed when closed is true and refs is 0. + mu sync.Mutex + fd sysfdType + refs int // number of active operations + closed bool // set when closed +} + +func (r *root) Close() error { + r.mu.Lock() + defer r.mu.Unlock() + if !r.closed && r.refs == 0 { + syscall.Close(r.fd) + } + r.closed = true + runtime.SetFinalizer(r, nil) // no need for a finalizer any more + return nil +} + +func (r *root) incref() error { + r.mu.Lock() + defer r.mu.Unlock() + if r.closed { + return ErrClosed + } + r.refs++ + return nil +} + +func (r *root) decref() { + r.mu.Lock() + defer r.mu.Unlock() + if r.refs <= 0 { + panic("bad Root refcount") + } + r.refs-- + if r.closed && r.refs == 0 { + syscall.Close(r.fd) + } +} + +func (r *root) Name() string { + return r.name +} + +func rootChmod(r *Root, name string, mode FileMode) error { + _, err := doInRoot(r, name, nil, func(parent sysfdType, name string) (struct{}, error) { + return struct{}{}, chmodat(parent, name, mode) + }) + if err != nil { + return &PathError{Op: "chmodat", Path: name, Err: err} + } + return nil +} + +func rootChown(r *Root, name string, uid, gid int) error { + _, err := doInRoot(r, name, nil, func(parent sysfdType, name string) (struct{}, error) { + return struct{}{}, chownat(parent, name, uid, gid) + }) + if err != nil { + return &PathError{Op: "chownat", Path: name, Err: err} + } + return nil +} + +func rootLchown(r *Root, name string, uid, gid int) error { + _, err := doInRoot(r, name, nil, func(parent sysfdType, name string) (struct{}, error) { + return struct{}{}, lchownat(parent, name, uid, gid) + }) + if err != nil { + return &PathError{Op: "lchownat", Path: name, Err: err} + } + return err +} + +func rootChtimes(r *Root, name string, atime time.Time, mtime time.Time) error { + _, err := doInRoot(r, name, nil, func(parent sysfdType, name string) (struct{}, error) { + return struct{}{}, chtimesat(parent, name, atime, mtime) + }) + if err != nil { + return &PathError{Op: "chtimesat", Path: name, Err: err} + } + return err +} + +func rootMkdir(r *Root, name string, perm FileMode) error { + _, err := doInRoot(r, name, nil, func(parent sysfdType, name string) (struct{}, error) { + return struct{}{}, mkdirat(parent, name, perm) + }) + if err != nil { + return &PathError{Op: "mkdirat", Path: name, Err: err} + } + return nil +} + +func rootMkdirAll(r *Root, fullname string, perm FileMode) error { + // doInRoot opens each path element in turn. + // + // openDirFunc opens all but the last path component. + // The usual default openDirFunc just opens directories with O_DIRECTORY. + // We replace it here with one that creates missing directories along the way. + openDirFunc := func(parent sysfdType, name string) (sysfdType, error) { + for try := range 2 { + fd, err := rootOpenDir(parent, name) + switch err.(type) { + case nil, errSymlink: + return fd, err + } + if try > 0 || !IsNotExist(err) { + return 0, &PathError{Op: "openat", Err: err} + } + // Try again on EEXIST, because the directory may have been created + // by another process or thread between the rootOpenDir and mkdirat calls. + if err := mkdirat(parent, name, perm); err != nil && err != syscall.EEXIST { + return 0, &PathError{Op: "mkdirat", Err: err} + } + } + panic("unreachable") + } + // openLastComponentFunc opens the last path component. + openLastComponentFunc := func(parent sysfdType, name string) (struct{}, error) { + err := mkdirat(parent, name, perm) + if err == syscall.EEXIST { + mode, e := modeAt(parent, name) + if e == nil { + if mode.IsDir() { + // The target of MkdirAll is an existing directory. + err = nil + } else if mode&ModeSymlink != 0 { + // The target of MkdirAll is a symlink. + // For consistency with os.MkdirAll, + // succeed if the link resolves to a directory. + // We don't return errSymlink here, because we don't + // want to create the link target if it doesn't exist. + fi, e := r.Stat(fullname) + if e == nil && fi.Mode().IsDir() { + err = nil + } + } + } + } + switch err.(type) { + case nil, errSymlink: + return struct{}{}, err + } + return struct{}{}, &PathError{Op: "mkdirat", Err: err} + } + _, err := doInRoot(r, fullname, openDirFunc, openLastComponentFunc) + if err != nil { + if _, ok := err.(*PathError); !ok { + err = &PathError{Op: "mkdirat", Path: fullname, Err: err} + } + } + return err +} + +func rootReadlink(r *Root, name string) (string, error) { + target, err := doInRoot(r, name, nil, func(parent sysfdType, name string) (string, error) { + return readlinkat(parent, name) + }) + if err != nil { + return "", &PathError{Op: "readlinkat", Path: name, Err: err} + } + return target, nil +} + +func rootRemove(r *Root, name string) error { + _, err := doInRoot(r, name, nil, func(parent sysfdType, name string) (struct{}, error) { + return struct{}{}, removeat(parent, name) + }) + if err != nil { + return &PathError{Op: "removeat", Path: name, Err: err} + } + return nil +} + +func rootRemoveAll(r *Root, name string) error { + // Consistency with os.RemoveAll: Strip trailing /s from the name, + // so RemoveAll("not_a_directory/") succeeds. + for len(name) > 0 && IsPathSeparator(name[len(name)-1]) { + name = name[:len(name)-1] + } + if endsWithDot(name) { + // Consistency with os.RemoveAll: Return EINVAL when trying to remove . + return &PathError{Op: "RemoveAll", Path: name, Err: syscall.EINVAL} + } + _, err := doInRoot(r, name, nil, func(parent sysfdType, name string) (struct{}, error) { + return struct{}{}, removeAllFrom(parent, name) + }) + if IsNotExist(err) { + return nil + } + if err != nil { + return &PathError{Op: "RemoveAll", Path: name, Err: underlyingError(err)} + } + return err +} + +func rootRename(r *Root, oldname, newname string) error { + _, err := doInRoot(r, oldname, nil, func(oldparent sysfdType, oldname string) (struct{}, error) { + _, err := doInRoot(r, newname, nil, func(newparent sysfdType, newname string) (struct{}, error) { + return struct{}{}, renameat(oldparent, oldname, newparent, newname) + }) + return struct{}{}, err + }) + if err != nil { + return &LinkError{"renameat", oldname, newname, err} + } + return err +} + +func rootLink(r *Root, oldname, newname string) error { + _, err := doInRoot(r, oldname, nil, func(oldparent sysfdType, oldname string) (struct{}, error) { + _, err := doInRoot(r, newname, nil, func(newparent sysfdType, newname string) (struct{}, error) { + return struct{}{}, linkat(oldparent, oldname, newparent, newname) + }) + return struct{}{}, err + }) + if err != nil { + return &LinkError{"linkat", oldname, newname, err} + } + return err +} + +// doInRoot performs an operation on a path in a Root. +// +// It calls f with the FD or handle for the directory containing the last +// path element, and the name of the last path element. +// +// For example, given the path a/b/c it calls f with the FD for a/b and the name "c". +// +// If openDirFunc is non-nil, it is called to open intermediate path elements. +// For example, given the path a/b/c openDirFunc will be called to open a and a/b in turn. +// +// f or openDirFunc may return errSymlink to indicate that the path element is a symlink +// which should be followed. Note that this can result in f being called multiple times +// with different names. For example, give the path "link" which is a symlink to "target", +// f is called with the path "link", returns errSymlink("target"), and is called again with +// the path "target". +// +// If f or openDirFunc return a *PathError, doInRoot will set PathError.Path to the +// full path which caused the error. +func doInRoot[T any](r *Root, name string, openDirFunc func(parent sysfdType, name string) (sysfdType, error), f func(parent sysfdType, name string) (T, error)) (ret T, err error) { + if err := r.root.incref(); err != nil { + return ret, err + } + defer r.root.decref() + + parts, suffixSep, err := splitPathInRoot(name, nil, nil) + if err != nil { + return ret, err + } + if openDirFunc == nil { + openDirFunc = rootOpenDir + } + + rootfd := r.root.fd + dirfd := rootfd + defer func() { + if dirfd != rootfd { + syscall.Close(dirfd) + } + }() + + // When resolving .. path components, we restart path resolution from the root. + // (We can't openat(dir, "..") to move up to the parent directory, + // because dir may have moved since we opened it.) + // To limit how many opens a malicious path can cause us to perform, we set + // a limit on the total number of path steps and the total number of restarts + // caused by .. components. If *both* limits are exceeded, we halt the operation. + const maxSteps = 255 + const maxRestarts = 8 + + i := 0 + steps := 0 + restarts := 0 + symlinks := 0 +Loop: + for { + steps++ + if steps > maxSteps && restarts > maxRestarts { + return ret, syscall.ENAMETOOLONG + } + + if parts[i] == ".." { + // Resolve one or more parent ("..") path components. + // + // Rewrite the original path, + // removing the elements eliminated by ".." components, + // and start over from the beginning. + restarts++ + end := i + 1 + for end < len(parts) && parts[end] == ".." { + end++ + } + count := end - i + if count > i { + return ret, errPathEscapes + } + parts = slices.Delete(parts, i-count, end) + if len(parts) == 0 { + parts = []string{"."} + } + i = 0 + if dirfd != rootfd { + syscall.Close(dirfd) + } + dirfd = rootfd + continue + } + + if i == len(parts)-1 { + // This is the last path element. + // Call f to decide what to do with it. + // If f returns errSymlink, this element is a symlink + // which should be followed. + // suffixSep contains any trailing separator characters + // which we rejoin to the final part at this time. + ret, err = f(dirfd, parts[i]+suffixSep) + if err == nil { + return + } + } else { + var fd sysfdType + fd, err = openDirFunc(dirfd, parts[i]) + if err == nil { + if dirfd != rootfd { + syscall.Close(dirfd) + } + dirfd = fd + } + } + + switch e := err.(type) { + case nil: + case errSymlink: + symlinks++ + if symlinks > rootMaxSymlinks { + return ret, syscall.ELOOP + } + newparts, newSuffixSep, err := splitPathInRoot(string(e), parts[:i], parts[i+1:]) + if err != nil { + return ret, err + } + if i == len(parts)-1 { + // suffixSep contains any trailing path separator characters + // in the link target. + // If we are replacing the remainder of the path, retain these. + // If we're replacing some intermediate component of the path, + // ignore them, since intermediate components must always be + // directories. + suffixSep = newSuffixSep + } + if len(newparts) < i || !slices.Equal(parts[:i], newparts[:i]) { + // Some component in the path which we have already traversed + // has changed. We need to restart parsing from the root. + i = 0 + if dirfd != rootfd { + syscall.Close(dirfd) + } + dirfd = rootfd + } + parts = newparts + continue Loop + case *PathError: + // This is strings.Join(parts[:i+1], PathSeparator). + e.Path = parts[0] + for _, part := range parts[1 : i+1] { + e.Path += string(PathSeparator) + part + } + return ret, e + default: + return ret, err + } + + i++ + } +} + +// errSymlink reports that a file being operated on is actually a symlink, +// and the target of that symlink. +type errSymlink string + +func (errSymlink) Error() string { panic("errSymlink is not user-visible") } diff --git a/src/os/root_plan9.go b/src/os/root_plan9.go new file mode 100644 index 00000000000000..08005accb54be1 --- /dev/null +++ b/src/os/root_plan9.go @@ -0,0 +1,25 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build plan9 + +package os + +import ( + "internal/filepathlite" +) + +func checkPathEscapes(r *Root, name string) error { + if r.root.closed.Load() { + return ErrClosed + } + if !filepathlite.IsLocal(name) { + return errPathEscapes + } + return nil +} + +func checkPathEscapesLstat(r *Root, name string) error { + return checkPathEscapes(r, name) +} diff --git a/src/os/root_test.go b/src/os/root_test.go new file mode 100644 index 00000000000000..f9fbd11575e6a0 --- /dev/null +++ b/src/os/root_test.go @@ -0,0 +1,1954 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os_test + +import ( + "bytes" + "errors" + "fmt" + "internal/testenv" + "io" + "io/fs" + "net" + "os" + "path" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + "time" +) + +// testMaybeRooted calls f in two subtests, +// one with a Root and one with a nil r. +func testMaybeRooted(t *testing.T, f func(t *testing.T, r *os.Root)) { + t.Run("NoRoot", func(t *testing.T) { + t.Chdir(t.TempDir()) + f(t, nil) + }) + t.Run("InRoot", func(t *testing.T) { + t.Chdir(t.TempDir()) + r, err := os.OpenRoot(".") + if err != nil { + t.Fatal(err) + } + defer r.Close() + f(t, r) + }) +} + +// makefs creates a test filesystem layout and returns the path to its root. +// +// Each entry in the slice is a file, directory, or symbolic link to create: +// +// - "d/": directory d +// - "f": file f with contents f +// - "a => b": symlink a with target b +// +// The directory containing the filesystem is always named ROOT. +// $ABS is replaced with the absolute path of the directory containing the filesystem. +// +// Parent directories are automatically created as needed. +// +// makefs calls t.Skip if the layout contains features not supported by the current GOOS. +func makefs(t *testing.T, fs []string) string { + root := path.Join(t.TempDir(), "ROOT") + if err := os.Mkdir(root, 0o777); err != nil { + t.Fatal(err) + } + for _, ent := range fs { + ent = strings.ReplaceAll(ent, "$ABS", root) + base, link, isLink := strings.Cut(ent, " => ") + if isLink { + if runtime.GOOS == "wasip1" && path.IsAbs(link) { + t.Skip("absolute link targets not supported on " + runtime.GOOS) + } + if runtime.GOOS == "plan9" { + t.Skip("symlinks not supported on " + runtime.GOOS) + } + ent = base + } + if err := os.MkdirAll(path.Join(root, path.Dir(base)), 0o777); err != nil { + t.Fatal(err) + } + if isLink { + if err := os.Symlink(link, path.Join(root, base)); err != nil { + t.Fatal(err) + } + } else if strings.HasSuffix(ent, "/") { + if err := os.MkdirAll(path.Join(root, ent), 0o777); err != nil { + t.Fatal(err) + } + } else { + if err := os.WriteFile(path.Join(root, ent), []byte(ent), 0o666); err != nil { + t.Fatal(err) + } + } + } + return root +} + +// A rootTest is a test case for os.Root. +type rootTest struct { + name string + + // fs is the test filesystem layout. See makefs above. + fs []string + + // open is the filename to access in the test. + open string + + // target is the filename that we expect to be accessed, after resolving all symlinks. + // For test cases where the operation fails due to an escaping path such as ../ROOT/x, + // the target is the filename that should not have been opened. + target string + + // ltarget is the filename that we expect to accessed, after resolving all symlinks + // except the last one. This is the file we expect to be removed by Remove or statted + // by Lstat. + // + // If the last path component in open is not a symlink, ltarget should be "". + ltarget string + + // wantError is true if accessing the file should fail. + wantError bool + + // alwaysFails is true if the open operation is expected to fail + // even when using non-openat operations. + // + // This lets us check that tests that are expected to fail because (for example) + // a path escapes the directory root will succeed when the escaping checks are not + // performed. + alwaysFails bool +} + +// run sets up the test filesystem layout, os.OpenDirs the root, and calls f. +func (test *rootTest) run(t *testing.T, f func(t *testing.T, target string, d *os.Root)) { + t.Run(test.name, func(t *testing.T) { + root := makefs(t, test.fs) + d, err := os.OpenRoot(root) + if err != nil { + t.Fatal(err) + } + defer d.Close() + // The target is a file that will be accessed, + // or a file that should not be accessed + // (because doing so escapes the root). + target := test.target + if test.target != "" { + target = filepath.Join(root, test.target) + } + f(t, target, d) + }) +} + +// errEndsTest checks the error result of a test, +// verifying that it succeeded or failed as expected. +// +// It returns true if the test is done due to encountering an expected error. +// false if the test should continue. +func errEndsTest(t *testing.T, err error, wantError bool, format string, args ...any) bool { + t.Helper() + if wantError { + if err == nil { + op := fmt.Sprintf(format, args...) + t.Fatalf("%v = nil; want error", op) + } + return true + } else { + if err != nil { + op := fmt.Sprintf(format, args...) + t.Fatalf("%v = %v; want success", op, err) + } + return false + } +} + +var rootTestCases = []rootTest{{ + name: "plain path", + fs: []string{}, + open: "target", + target: "target", +}, { + name: "path in directory", + fs: []string{ + "a/b/c/", + }, + open: "a/b/c/target", + target: "a/b/c/target", +}, { + name: "symlink", + fs: []string{ + "link => target", + }, + open: "link", + target: "target", + ltarget: "link", +}, { + name: "symlink dotdot slash", + fs: []string{ + "link => ../", + }, + open: "link", + ltarget: "link", + wantError: true, +}, { + name: "symlink ending in slash", + fs: []string{ + "dir/", + "link => dir/", + }, + open: "link/target", + target: "dir/target", +}, { + name: "symlink dotdot dotdot slash", + fs: []string{ + "dir/link => ../../", + }, + open: "dir/link", + ltarget: "dir/link", + wantError: true, +}, { + name: "symlink chain", + fs: []string{ + "link => a/b/c/target", + "a/b => e", + "a/e => ../f", + "f => g/h/i", + "g/h/i => ..", + "g/c/", + }, + open: "link", + target: "g/c/target", + ltarget: "link", +}, { + name: "path with dot", + fs: []string{ + "a/b/", + }, + open: "./a/./b/./target", + target: "a/b/target", +}, { + name: "path with dotdot", + fs: []string{ + "a/b/", + }, + open: "a/../a/b/../../a/b/../b/target", + target: "a/b/target", +}, { + name: "path with dotdot slash", + fs: []string{}, + open: "../", + wantError: true, +}, { + name: "path with dotdot dotdot slash", + fs: []string{}, + open: "a/../../", + wantError: true, +}, { + name: "dotdot no symlink", + fs: []string{ + "a/", + }, + open: "a/../target", + target: "target", +}, { + name: "dotdot after symlink", + fs: []string{ + "a => b/c", + "b/c/", + }, + open: "a/../target", + target: func() string { + if runtime.GOOS == "windows" { + // On Windows, the path is cleaned before symlink resolution. + return "target" + } + return "b/target" + }(), +}, { + name: "dotdot before symlink", + fs: []string{ + "a => b/c", + "b/c/", + }, + open: "b/../a/target", + target: "b/c/target", +}, { + name: "symlink ends in dot", + fs: []string{ + "a => b/.", + "b/", + }, + open: "a/target", + target: "b/target", +}, { + name: "directory does not exist", + fs: []string{}, + open: "a/file", + wantError: true, + alwaysFails: true, +}, { + name: "empty path", + fs: []string{}, + open: "", + wantError: true, + alwaysFails: true, +}, { + name: "symlink cycle", + fs: []string{ + "a => a", + }, + open: "a", + ltarget: "a", + wantError: true, + alwaysFails: true, +}, { + name: "path escapes", + fs: []string{}, + open: "../ROOT/target", + target: "target", + wantError: true, +}, { + name: "long path escapes", + fs: []string{ + "a/", + }, + open: "a/../../ROOT/target", + target: "target", + wantError: true, +}, { + name: "absolute symlink", + fs: []string{ + "link => $ABS/target", + }, + open: "link", + ltarget: "link", + target: "target", + wantError: true, +}, { + name: "relative symlink", + fs: []string{ + "link => ../ROOT/target", + }, + open: "link", + target: "target", + ltarget: "link", + wantError: true, +}, { + name: "symlink chain escapes", + fs: []string{ + "link => a/b/c/target", + "a/b => e", + "a/e => ../../ROOT", + "c/", + }, + open: "link", + target: "c/target", + ltarget: "link", + wantError: true, +}} + +func TestRootOpen_File(t *testing.T) { + want := []byte("target") + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + if target != "" { + if err := os.WriteFile(target, want, 0o666); err != nil { + t.Fatal(err) + } + } + f, err := root.Open(test.open) + if errEndsTest(t, err, test.wantError, "root.Open(%q)", test.open) { + return + } + defer f.Close() + got, err := io.ReadAll(f) + if err != nil || !bytes.Equal(got, want) { + t.Errorf(`Dir.Open(%q): read content %q, %v; want %q`, test.open, string(got), err, string(want)) + } + }) + } +} + +func TestRootOpen_Directory(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + if target != "" { + if err := os.Mkdir(target, 0o777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target+"/found", nil, 0o666); err != nil { + t.Fatal(err) + } + } + f, err := root.Open(test.open) + if errEndsTest(t, err, test.wantError, "root.Open(%q)", test.open) { + return + } + defer f.Close() + got, err := f.Readdirnames(-1) + if err != nil { + t.Errorf(`Dir.Open(%q).Readdirnames: %v`, test.open, err) + } + if want := []string{"found"}; !slices.Equal(got, want) { + t.Errorf(`Dir.Open(%q).Readdirnames: %q, want %q`, test.open, got, want) + } + }) + } +} + +func TestRootCreate(t *testing.T) { + want := []byte("target") + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + f, err := root.Create(test.open) + if errEndsTest(t, err, test.wantError, "root.Create(%q)", test.open) { + return + } + if _, err := f.Write(want); err != nil { + t.Fatal(err) + } + f.Close() + got, err := os.ReadFile(target) + if err != nil { + t.Fatalf(`reading file created with root.Create(%q): %v`, test.open, err) + } + if !bytes.Equal(got, want) { + t.Fatalf(`reading file created with root.Create(%q): got %q; want %q`, test.open, got, want) + } + }) + } +} + +func TestRootChmod(t *testing.T) { + if runtime.GOOS == "wasip1" { + t.Skip("Chmod not supported on " + runtime.GOOS) + } + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + if target != "" { + // Create a file with no read/write permissions, + // to ensure we can use Chmod on an inaccessible file. + if err := os.WriteFile(target, nil, 0o000); err != nil { + t.Fatal(err) + } + } + if runtime.GOOS == "windows" { + // On Windows, Chmod("symlink") affects the link, not its target. + // See issue 71492. + fi, err := root.Lstat(test.open) + if err == nil && !fi.Mode().IsRegular() { + t.Skip("https://go.dev/issue/71492") + } + } + want := os.FileMode(0o666) + err := root.Chmod(test.open, want) + if errEndsTest(t, err, test.wantError, "root.Chmod(%q)", test.open) { + return + } + st, err := os.Stat(target) + if err != nil { + t.Fatalf("os.Stat(%q) = %v", target, err) + } + if got := st.Mode(); got != want { + t.Errorf("after root.Chmod(%q, %v): file mode = %v, want %v", test.open, want, got, want) + } + }) + } +} + +func TestRootChtimes(t *testing.T) { + // Don't check atimes if the fs is mounted noatime, + // or on Plan 9 which does not permit changing atimes to arbitrary values. + checkAtimes := !hasNoatime() && runtime.GOOS != "plan9" + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + if target != "" { + if err := os.WriteFile(target, nil, 0o666); err != nil { + t.Fatal(err) + } + } + for _, times := range []struct { + atime, mtime time.Time + }{{ + atime: time.Now().Add(-1 * time.Minute), + mtime: time.Now().Add(-1 * time.Minute), + }, { + atime: time.Now().Add(1 * time.Minute), + mtime: time.Now().Add(1 * time.Minute), + }, { + atime: time.Time{}, + mtime: time.Now(), + }, { + atime: time.Now(), + mtime: time.Time{}, + }} { + switch runtime.GOOS { + case "js", "plan9": + times.atime = times.atime.Truncate(1 * time.Second) + times.mtime = times.mtime.Truncate(1 * time.Second) + case "illumos": + times.atime = times.atime.Truncate(1 * time.Microsecond) + times.mtime = times.mtime.Truncate(1 * time.Microsecond) + } + + err := root.Chtimes(test.open, times.atime, times.mtime) + if errEndsTest(t, err, test.wantError, "root.Chtimes(%q)", test.open) { + return + } + st, err := os.Stat(target) + if err != nil { + t.Fatalf("os.Stat(%q) = %v", target, err) + } + if got := st.ModTime(); !times.mtime.IsZero() && !got.Equal(times.mtime) { + t.Errorf("after root.Chtimes(%q, %v, %v): got mtime=%v, want %v", test.open, times.atime, times.mtime, got, times.mtime) + } + if checkAtimes { + if got := os.Atime(st); !times.atime.IsZero() && !got.Equal(times.atime) { + t.Errorf("after root.Chtimes(%q, %v, %v): got atime=%v, want %v", test.open, times.atime, times.mtime, got, times.atime) + } + } + } + }) + } +} + +func TestRootMkdir(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + wantError := test.wantError + if !wantError { + fi, err := os.Lstat(filepath.Join(root.Name(), test.open)) + if err == nil && fi.Mode().Type() == fs.ModeSymlink { + // This case is trying to mkdir("some symlink"), + // which is an error. + wantError = true + } + } + + err := root.Mkdir(test.open, 0o777) + if errEndsTest(t, err, wantError, "root.Create(%q)", test.open) { + return + } + fi, err := os.Lstat(target) + if err != nil { + t.Fatalf(`stat file created with Root.Mkdir(%q): %v`, test.open, err) + } + if !fi.IsDir() { + t.Fatalf(`stat file created with Root.Mkdir(%q): not a directory`, test.open) + } + if mode := fi.Mode(); mode&0o777 == 0 { + // Issue #73559: We're not going to worry about the exact + // mode bits (which will have been modified by umask), + // but there should be mode bits. + t.Fatalf(`stat file created with Root.Mkdir(%q): mode=%v, want non-zero`, test.open, mode) + } + }) + } +} + +func TestRootMkdirAll(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + wantError := test.wantError + if !wantError { + fi, err := os.Lstat(filepath.Join(root.Name(), test.open)) + if err == nil && fi.Mode().Type() == fs.ModeSymlink { + // This case is trying to mkdir("some symlink"), + // which is an error. + wantError = true + } + } + + err := root.Mkdir(test.open, 0o777) + if errEndsTest(t, err, wantError, "root.MkdirAll(%q)", test.open) { + return + } + fi, err := os.Lstat(target) + if err != nil { + t.Fatalf(`stat file created with Root.MkdirAll(%q): %v`, test.open, err) + } + if !fi.IsDir() { + t.Fatalf(`stat file created with Root.MkdirAll(%q): not a directory`, test.open) + } + if mode := fi.Mode(); mode&0o777 == 0 { + // Issue #73559: We're not going to worry about the exact + // mode bits (which will have been modified by umask), + // but there should be mode bits. + t.Fatalf(`stat file created with Root.MkdirAll(%q): mode=%v, want non-zero`, test.open, mode) + } + }) + } +} + +func TestRootOpenRoot(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + if target != "" { + if err := os.Mkdir(target, 0o777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target+"/f", nil, 0o666); err != nil { + t.Fatal(err) + } + } + rr, err := root.OpenRoot(test.open) + if errEndsTest(t, err, test.wantError, "root.OpenRoot(%q)", test.open) { + return + } + defer rr.Close() + f, err := rr.Open("f") + if err != nil { + t.Fatalf(`root.OpenRoot(%q).Open("f") = %v`, test.open, err) + } + f.Close() + }) + } +} + +func TestRootRemoveFile(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + wantError := test.wantError + if test.ltarget != "" { + // Remove doesn't follow symlinks in the final path component, + // so it will successfully remove ltarget. + wantError = false + target = filepath.Join(root.Name(), test.ltarget) + } else if target != "" { + if err := os.WriteFile(target, nil, 0o666); err != nil { + t.Fatal(err) + } + } + + err := root.Remove(test.open) + if errEndsTest(t, err, wantError, "root.Remove(%q)", test.open) { + return + } + _, err = os.Lstat(target) + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf(`stat file removed with Root.Remove(%q): %v, want ErrNotExist`, test.open, err) + } + }) + } +} + +func TestRootRemoveDirectory(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + wantError := test.wantError + if test.ltarget != "" { + // Remove doesn't follow symlinks in the final path component, + // so it will successfully remove ltarget. + wantError = false + target = filepath.Join(root.Name(), test.ltarget) + } else if target != "" { + if err := os.Mkdir(target, 0o777); err != nil { + t.Fatal(err) + } + } + + err := root.Remove(test.open) + if errEndsTest(t, err, wantError, "root.Remove(%q)", test.open) { + return + } + _, err = os.Lstat(target) + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf(`stat file removed with Root.Remove(%q): %v, want ErrNotExist`, test.open, err) + } + }) + } +} + +func TestRootRemoveAll(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + wantError := test.wantError + if test.ltarget != "" { + // Remove doesn't follow symlinks in the final path component, + // so it will successfully remove ltarget. + wantError = false + target = filepath.Join(root.Name(), test.ltarget) + } else if target != "" { + if err := os.Mkdir(target, 0o777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(target, "file"), nil, 0o666); err != nil { + t.Fatal(err) + } + } + targetExists := true + if _, err := root.Lstat(test.open); errors.Is(err, os.ErrNotExist) { + // If the target doesn't exist, RemoveAll succeeds rather + // than returning ErrNotExist. + targetExists = false + wantError = false + } + + err := root.RemoveAll(test.open) + if errEndsTest(t, err, wantError, "root.RemoveAll(%q)", test.open) { + return + } + if !targetExists { + return + } + _, err = os.Lstat(target) + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf(`stat file removed with Root.Remove(%q): %v, want ErrNotExist`, test.open, err) + } + }) + } +} + +func TestRootOpenFileAsRoot(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "target") + if err := os.WriteFile(target, nil, 0o666); err != nil { + t.Fatal(err) + } + r, err := os.OpenRoot(target) + if err == nil { + r.Close() + t.Fatal("os.OpenRoot(file) succeeded; want failure") + } + r, err = os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer r.Close() + rr, err := r.OpenRoot("target") + if err == nil { + rr.Close() + t.Fatal("Root.OpenRoot(file) succeeded; want failure") + } +} + +func TestRootStat(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + const content = "content" + if target != "" { + if err := os.WriteFile(target, []byte(content), 0o666); err != nil { + t.Fatal(err) + } + } + + fi, err := root.Stat(test.open) + if errEndsTest(t, err, test.wantError, "root.Stat(%q)", test.open) { + return + } + if got, want := fi.Name(), filepath.Base(test.open); got != want { + t.Errorf("root.Stat(%q).Name() = %q, want %q", test.open, got, want) + } + if got, want := fi.Size(), int64(len(content)); got != want { + t.Errorf("root.Stat(%q).Size() = %v, want %v", test.open, got, want) + } + }) + } +} + +func TestRootLstat(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + const content = "content" + wantError := test.wantError + if test.ltarget != "" { + // Lstat will stat the final link, rather than following it. + wantError = false + } else if target != "" { + if err := os.WriteFile(target, []byte(content), 0o666); err != nil { + t.Fatal(err) + } + } + + fi, err := root.Lstat(test.open) + if errEndsTest(t, err, wantError, "root.Stat(%q)", test.open) { + return + } + if got, want := fi.Name(), filepath.Base(test.open); got != want { + t.Errorf("root.Stat(%q).Name() = %q, want %q", test.open, got, want) + } + if test.ltarget == "" { + if got := fi.Mode(); got&os.ModeSymlink != 0 { + t.Errorf("root.Stat(%q).Mode() = %v, want non-symlink", test.open, got) + } + if got, want := fi.Size(), int64(len(content)); got != want { + t.Errorf("root.Stat(%q).Size() = %v, want %v", test.open, got, want) + } + } else { + if got := fi.Mode(); got&os.ModeSymlink == 0 { + t.Errorf("root.Stat(%q).Mode() = %v, want symlink", test.open, got) + } + } + }) + } +} + +func TestRootReadlink(t *testing.T) { + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + const content = "content" + wantError := test.wantError + if test.ltarget != "" { + // Readlink will read the final link, rather than following it. + wantError = false + } else { + // Readlink fails on non-link targets. + wantError = true + } + + got, err := root.Readlink(test.open) + if errEndsTest(t, err, wantError, "root.Readlink(%q)", test.open) { + return + } + + want, err := os.Readlink(filepath.Join(root.Name(), test.ltarget)) + if err != nil { + t.Fatalf("os.Readlink(%q) = %v, want success", test.ltarget, err) + } + if got != want { + t.Errorf("root.Readlink(%q) = %q, want %q", test.open, got, want) + } + }) + } +} + +// TestRootRenameFrom tests renaming the test case target to a known-good path. +func TestRootRenameFrom(t *testing.T) { + testRootMoveFrom(t, true) +} + +// TestRootRenameFrom tests linking the test case target to a known-good path. +func TestRootLinkFrom(t *testing.T) { + testenv.MustHaveLink(t) + testRootMoveFrom(t, false) +} + +func testRootMoveFrom(t *testing.T, rename bool) { + want := []byte("target") + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + if target != "" { + if err := os.WriteFile(target, want, 0o666); err != nil { + t.Fatal(err) + } + } + wantError := test.wantError + var linkTarget string + if test.ltarget != "" { + // Rename will rename the link, not the file linked to. + wantError = false + var err error + linkTarget, err = root.Readlink(test.ltarget) + if err != nil { + t.Fatalf("root.Readlink(%q) = %v, want success", test.ltarget, err) + } + + // When GOOS=js, creating a hard link to a symlink fails. + if !rename && runtime.GOOS == "js" { + wantError = true + } + + // Windows allows creating a hard link to a file symlink, + // but not to a directory symlink. + // + // This uses os.Stat to check the link target, because this + // is easier than figuring out whether the link itself is a + // directory link. The link was created with os.Symlink, + // which creates directory links when the target is a directory, + // so this is good enough for a test. + if !rename && runtime.GOOS == "windows" { + st, err := os.Stat(filepath.Join(root.Name(), test.ltarget)) + if err == nil && st.IsDir() { + wantError = true + } + } + } + + const dstPath = "destination" + + // Plan 9 doesn't allow cross-directory renames. + if runtime.GOOS == "plan9" && strings.Contains(test.open, "/") { + wantError = true + } + + var op string + var err error + if rename { + op = "Rename" + err = root.Rename(test.open, dstPath) + } else { + op = "Link" + err = root.Link(test.open, dstPath) + } + if errEndsTest(t, err, wantError, "root.%v(%q, %q)", op, test.open, dstPath) { + return + } + + origPath := target + if test.ltarget != "" { + origPath = filepath.Join(root.Name(), test.ltarget) + } + _, err = os.Lstat(origPath) + if rename { + if !errors.Is(err, os.ErrNotExist) { + t.Errorf("after renaming file, Lstat(%q) = %v, want ErrNotExist", origPath, err) + } + } else { + if err != nil { + t.Errorf("after linking file, error accessing original: %v", err) + } + } + + dstFullPath := filepath.Join(root.Name(), dstPath) + if test.ltarget != "" { + got, err := os.Readlink(dstFullPath) + if err != nil || got != linkTarget { + t.Errorf("os.Readlink(%q) = %q, %v, want %q", dstFullPath, got, err, linkTarget) + } + } else { + got, err := os.ReadFile(dstFullPath) + if err != nil || !bytes.Equal(got, want) { + t.Errorf(`os.ReadFile(%q): read content %q, %v; want %q`, dstFullPath, string(got), err, string(want)) + } + st, err := os.Lstat(dstFullPath) + if err != nil || st.Mode()&fs.ModeSymlink != 0 { + t.Errorf(`os.Lstat(%q) = %v, %v; want non-symlink`, dstFullPath, st.Mode(), err) + } + + } + }) + } +} + +// TestRootRenameTo tests renaming a known-good path to the test case target. +func TestRootRenameTo(t *testing.T) { + testRootMoveTo(t, true) +} + +// TestRootLinkTo tests renaming a known-good path to the test case target. +func TestRootLinkTo(t *testing.T) { + testenv.MustHaveLink(t) + testRootMoveTo(t, true) +} + +func testRootMoveTo(t *testing.T, rename bool) { + want := []byte("target") + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + const srcPath = "source" + if err := os.WriteFile(filepath.Join(root.Name(), srcPath), want, 0o666); err != nil { + t.Fatal(err) + } + + target = test.target + wantError := test.wantError + if test.ltarget != "" { + // Rename will overwrite the final link rather than follow it. + target = test.ltarget + wantError = false + } + + // Plan 9 doesn't allow cross-directory renames. + if runtime.GOOS == "plan9" && strings.Contains(test.open, "/") { + wantError = true + } + + var err error + var op string + if rename { + op = "Rename" + err = root.Rename(srcPath, test.open) + } else { + op = "Link" + err = root.Link(srcPath, test.open) + } + if errEndsTest(t, err, wantError, "root.%v(%q, %q)", op, srcPath, test.open) { + return + } + + _, err = os.Lstat(filepath.Join(root.Name(), srcPath)) + if rename { + if !errors.Is(err, os.ErrNotExist) { + t.Errorf("after renaming file, Lstat(%q) = %v, want ErrNotExist", srcPath, err) + } + } else { + if err != nil { + t.Errorf("after linking file, error accessing original: %v", err) + } + } + + got, err := os.ReadFile(filepath.Join(root.Name(), target)) + if err != nil || !bytes.Equal(got, want) { + t.Errorf(`os.ReadFile(%q): read content %q, %v; want %q`, target, string(got), err, string(want)) + } + }) + } +} + +func TestRootSymlink(t *testing.T) { + testenv.MustHaveSymlink(t) + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + wantError := test.wantError + if test.ltarget != "" { + // We can't create a symlink over an existing symlink. + wantError = true + } + + const wantTarget = "linktarget" + err := root.Symlink(wantTarget, test.open) + if errEndsTest(t, err, wantError, "root.Symlink(%q)", test.open) { + return + } + got, err := os.Readlink(target) + if err != nil || got != wantTarget { + t.Fatalf("ReadLink(%q) = %q, %v; want %q, nil", target, got, err, wantTarget) + } + }) + } +} + +// A rootConsistencyTest is a test case comparing os.Root behavior with +// the corresponding non-Root function. +// +// These tests verify that, for example, Root.Open("file/./") and os.Open("file/./") +// have the same result, although the specific result may vary by platform. +type rootConsistencyTest struct { + name string + + // fs is the test filesystem layout. See makefs above. + // fsFunc is called to modify the test filesystem, or replace it. + fs []string + fsFunc func(t *testing.T, dir string) string + + // open is the filename to access in the test. + open string + + // detailedErrorMismatch indicates that os.Root and the corresponding non-Root + // function return different errors for this test. + detailedErrorMismatch func(t *testing.T) bool + + // check is called before the test starts, and may t.Skip if necessary. + check func(t *testing.T) +} + +var rootConsistencyTestCases = []rootConsistencyTest{{ + name: "file", + fs: []string{ + "target", + }, + open: "target", +}, { + name: "dir slash dot", + fs: []string{ + "target/file", + }, + open: "target/.", +}, { + name: "dot", + fs: []string{ + "file", + }, + open: ".", +}, { + name: "file slash dot", + fs: []string{ + "target", + }, + open: "target/.", + detailedErrorMismatch: func(t *testing.T) bool { + // FreeBSD returns EPERM in the non-Root case. + return runtime.GOOS == "freebsd" && strings.HasPrefix(t.Name(), "TestRootConsistencyRemove") + }, +}, { + name: "dir slash", + fs: []string{ + "target/file", + }, + open: "target/", +}, { + name: "dot slash", + fs: []string{ + "file", + }, + open: "./", +}, { + name: "file slash", + fs: []string{ + "target", + }, + open: "target/", + detailedErrorMismatch: func(t *testing.T) bool { + // os.Create returns ENOTDIR or EISDIR depending on the platform. + return runtime.GOOS == "js" + }, +}, { + name: "file in path", + fs: []string{ + "file", + }, + open: "file/target", +}, { + name: "directory in path missing", + open: "dir/target", +}, { + name: "target does not exist", + open: "target", +}, { + name: "symlink slash", + fs: []string{ + "target/file", + "link => target", + }, + open: "link/", +}, { + name: "symlink slash dot", + fs: []string{ + "target/file", + "link => target", + }, + open: "link/.", +}, { + name: "file symlink slash", + fs: []string{ + "target", + "link => target", + }, + open: "link/", + detailedErrorMismatch: func(t *testing.T) bool { + // os.Create returns ENOTDIR or EISDIR depending on the platform. + return runtime.GOOS == "js" + }, +}, { + name: "unresolved symlink", + fs: []string{ + "link => target", + }, + open: "link", +}, { + name: "resolved symlink", + fs: []string{ + "link => target", + "target", + }, + open: "link", +}, { + name: "dotdot in path after symlink", + fs: []string{ + "a => b/c", + "b/c/", + "b/target", + }, + open: "a/../target", +}, { + name: "symlink to dir ends in slash", + fs: []string{ + "dir/", + "link => dir/", + }, + open: "link", +}, { + name: "symlink to file ends in slash", + fs: []string{ + "file", + "link => file/", + }, + open: "link", +}, { + name: "long file name", + open: strings.Repeat("a", 500), +}, { + name: "unreadable directory", + fs: []string{ + "dir/target", + }, + fsFunc: func(t *testing.T, dir string) string { + os.Chmod(filepath.Join(dir, "dir"), 0) + t.Cleanup(func() { + os.Chmod(filepath.Join(dir, "dir"), 0o700) + }) + return dir + }, + open: "dir/target", +}, { + name: "unix domain socket target", + fsFunc: func(t *testing.T, dir string) string { + return tempDirWithUnixSocket(t, "a") + }, + open: "a", +}, { + name: "unix domain socket in path", + fsFunc: func(t *testing.T, dir string) string { + return tempDirWithUnixSocket(t, "a") + }, + open: "a/b", + detailedErrorMismatch: func(t *testing.T) bool { + // On Windows, os.Root.Open returns "The directory name is invalid." + // and os.Open returns "The file cannot be accessed by the system.". + return runtime.GOOS == "windows" + }, + check: func(t *testing.T) { + if runtime.GOOS == "windows" && strings.HasPrefix(t.Name(), "TestRootConsistencyRemoveAll/") { + // Root.RemoveAll notices that a/ is not a directory, + // and returns success. + // os.RemoveAll tries to open a/ and fails because + // it is not a regular file. + // The inconsistency here isn't worth fixing, so just skip this test. + t.Skip("known inconsistency on windows") + } + }, +}, { + name: "question mark", + open: "?", +}, { + name: "nul byte", + open: "\x00", +}} + +func tempDirWithUnixSocket(t *testing.T, name string) string { + dir, err := os.MkdirTemp("", "") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }) + addr, err := net.ResolveUnixAddr("unix", filepath.Join(dir, name)) + if err != nil { + t.Skipf("net.ResolveUnixAddr: %v", err) + } + conn, err := net.ListenUnix("unix", addr) + if err != nil { + t.Skipf("net.ListenUnix: %v", err) + } + t.Cleanup(func() { + conn.Close() + }) + return dir +} + +func (test rootConsistencyTest) run(t *testing.T, f func(t *testing.T, path string, r *os.Root) (string, error)) { + if runtime.GOOS == "wasip1" { + // On wasip, non-Root functions clean paths before opening them, + // resulting in inconsistent behavior. + // https://go.dev/issue/69509 + t.Skip("#69509: inconsistent results on wasip1") + } + + t.Run(test.name, func(t *testing.T) { + if test.check != nil { + test.check(t) + } + + dir1 := makefs(t, test.fs) + dir2 := makefs(t, test.fs) + if test.fsFunc != nil { + dir1 = test.fsFunc(t, dir1) + dir2 = test.fsFunc(t, dir2) + } + + r, err := os.OpenRoot(dir1) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + res1, err1 := f(t, test.open, r) + res2, err2 := f(t, dir2+"/"+test.open, nil) + + if res1 != res2 || ((err1 == nil) != (err2 == nil)) { + t.Errorf("with root: res=%v", res1) + t.Errorf(" err=%v", err1) + t.Errorf("without root: res=%v", res2) + t.Errorf(" err=%v", err2) + t.Errorf("want consistent results, got mismatch") + } + + if err1 != nil || err2 != nil { + underlyingError := func(how string, err error) error { + switch e := err1.(type) { + case *os.PathError: + return e.Err + case *os.LinkError: + return e.Err + default: + t.Fatalf("%v, expected PathError or LinkError; got: %v", how, err) + } + return nil + } + e1 := underlyingError("with root", err1) + e2 := underlyingError("without root", err1) + detailedErrorMismatch := false + if f := test.detailedErrorMismatch; f != nil { + detailedErrorMismatch = f(t) + } + if runtime.GOOS == "plan9" { + // Plan9 syscall errors aren't comparable. + detailedErrorMismatch = true + } + if !detailedErrorMismatch && e1 != e2 { + t.Errorf("with root: err=%v", e1) + t.Errorf("without root: err=%v", e2) + t.Errorf("want consistent results, got mismatch") + } + } + }) +} + +func TestRootConsistencyOpen(t *testing.T) { + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + var f *os.File + var err error + if r == nil { + f, err = os.Open(path) + } else { + f, err = r.Open(path) + } + if err != nil { + return "", err + } + defer f.Close() + fi, err := f.Stat() + if err == nil && !fi.IsDir() { + b, err := io.ReadAll(f) + return string(b), err + } else { + names, err := f.Readdirnames(-1) + slices.Sort(names) + return fmt.Sprintf("%q", names), err + } + }) + } +} + +func TestRootConsistencyCreate(t *testing.T) { + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + var f *os.File + var err error + if r == nil { + f, err = os.Create(path) + } else { + f, err = r.Create(path) + } + if err == nil { + f.Write([]byte("file contents")) + f.Close() + } + return "", err + }) + } +} + +func TestRootConsistencyChmod(t *testing.T) { + if runtime.GOOS == "wasip1" { + t.Skip("Chmod not supported on " + runtime.GOOS) + } + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + chmod := os.Chmod + lstat := os.Lstat + if r != nil { + chmod = r.Chmod + lstat = r.Lstat + } + + var m1, m2 os.FileMode + if err := chmod(path, 0o555); err != nil { + return "chmod 0o555", err + } + fi, err := lstat(path) + if err == nil { + m1 = fi.Mode() + } + if err = chmod(path, 0o777); err != nil { + return "chmod 0o777", err + } + fi, err = lstat(path) + if err == nil { + m2 = fi.Mode() + } + return fmt.Sprintf("%v %v", m1, m2), err + }) + } +} + +func TestRootConsistencyMkdir(t *testing.T) { + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + var err error + if r == nil { + err = os.Mkdir(path, 0o777) + } else { + err = r.Mkdir(path, 0o777) + } + return "", err + }) + } +} + +func TestRootConsistencyMkdirAll(t *testing.T) { + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + var err error + if r == nil { + err = os.MkdirAll(path, 0o777) + } else { + err = r.MkdirAll(path, 0o777) + } + return "", err + }) + } +} + +func TestRootConsistencyRemove(t *testing.T) { + for _, test := range rootConsistencyTestCases { + if test.open == "." || test.open == "./" { + continue // can't remove the root itself + } + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + var err error + if r == nil { + err = os.Remove(path) + } else { + err = r.Remove(path) + } + return "", err + }) + } +} + +func TestRootConsistencyRemoveAll(t *testing.T) { + for _, test := range rootConsistencyTestCases { + if test.open == "." || test.open == "./" { + continue // can't remove the root itself + } + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + var err error + if r == nil { + err = os.RemoveAll(path) + } else { + err = r.RemoveAll(path) + } + return "", err + }) + } +} + +func TestRootConsistencyStat(t *testing.T) { + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + var fi os.FileInfo + var err error + if r == nil { + fi, err = os.Stat(path) + } else { + fi, err = r.Stat(path) + } + if err != nil { + return "", err + } + return fmt.Sprintf("name:%q size:%v mode:%v isdir:%v", fi.Name(), fi.Size(), fi.Mode(), fi.IsDir()), nil + }) + } +} + +func TestRootConsistencyLstat(t *testing.T) { + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + var fi os.FileInfo + var err error + if r == nil { + fi, err = os.Lstat(path) + } else { + fi, err = r.Lstat(path) + } + if err != nil { + return "", err + } + return fmt.Sprintf("name:%q size:%v mode:%v isdir:%v", fi.Name(), fi.Size(), fi.Mode(), fi.IsDir()), nil + }) + } +} + +func TestRootConsistencyReadlink(t *testing.T) { + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + if r == nil { + return os.Readlink(path) + } else { + return r.Readlink(path) + } + }) + } +} + +func TestRootConsistencyRename(t *testing.T) { + testRootConsistencyMove(t, true) +} + +func TestRootConsistencyLink(t *testing.T) { + testenv.MustHaveLink(t) + testRootConsistencyMove(t, false) +} + +func testRootConsistencyMove(t *testing.T, rename bool) { + if runtime.GOOS == "plan9" { + // This test depends on moving files between directories. + t.Skip("Plan 9 does not support cross-directory renames") + } + // Run this test in two directions: + // Renaming the test path to a known-good path (from), + // and renaming a known-good path to the test path (to). + for _, name := range []string{"from", "to"} { + t.Run(name, func(t *testing.T) { + for _, test := range rootConsistencyTestCases { + if runtime.GOOS == "windows" { + // On Windows, Rename("/path/to/.", x) succeeds, + // because Windows cleans the path to just "/path/to". + // Root.Rename(".", x) fails as expected. + // Don't run this consistency test on Windows. + if test.open == "." || test.open == "./" { + continue + } + } + + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + var move func(oldname, newname string) error + switch { + case rename && r == nil: + move = os.Rename + case rename && r != nil: + move = r.Rename + case !rename && r == nil: + move = os.Link + case !rename && r != nil: + move = r.Link + } + lstat := os.Lstat + if r != nil { + lstat = r.Lstat + } + + otherPath := "other" + if r == nil { + otherPath = filepath.Join(t.TempDir(), otherPath) + } + + var srcPath, dstPath string + if name == "from" { + srcPath = path + dstPath = otherPath + } else { + srcPath = otherPath + dstPath = path + } + + if !rename { + // When the source is a symlink, Root.Link creates + // a hard link to the symlink. + // os.Link does whatever the link syscall does, + // which varies between operating systems and + // their versions. + // Skip running the consistency test when + // the source is a symlink. + fi, err := lstat(srcPath) + if err == nil && fi.Mode()&os.ModeSymlink != 0 { + return "", nil + } + } + + if err := move(srcPath, dstPath); err != nil { + return "", err + } + fi, err := lstat(dstPath) + if err != nil { + t.Errorf("stat(%q) after successful copy: %v", dstPath, err) + return "stat error", err + } + return fmt.Sprintf("name:%q size:%v mode:%v isdir:%v", fi.Name(), fi.Size(), fi.Mode(), fi.IsDir()), nil + }) + } + }) + } +} + +func TestRootConsistencySymlink(t *testing.T) { + testenv.MustHaveSymlink(t) + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + const target = "linktarget" + var err error + var got string + if r == nil { + err = os.Symlink(target, path) + got, _ = os.Readlink(target) + } else { + err = r.Symlink(target, path) + got, _ = r.Readlink(target) + } + return got, err + }) + } +} + +func TestRootRenameAfterOpen(t *testing.T) { + switch runtime.GOOS { + case "windows": + t.Skip("renaming open files not supported on " + runtime.GOOS) + case "js", "plan9": + t.Skip("openat not supported on " + runtime.GOOS) + case "wasip1": + if os.Getenv("GOWASIRUNTIME") == "wazero" { + t.Skip("wazero does not track renamed directories") + } + } + + dir := t.TempDir() + + // Create directory "a" and open it. + if err := os.Mkdir(filepath.Join(dir, "a"), 0o777); err != nil { + t.Fatal(err) + } + dirf, err := os.OpenRoot(filepath.Join(dir, "a")) + if err != nil { + t.Fatal(err) + } + defer dirf.Close() + + // Rename "a" => "b", and create "b/f". + if err := os.Rename(filepath.Join(dir, "a"), filepath.Join(dir, "b")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "b/f"), []byte("hello"), 0o666); err != nil { + t.Fatal(err) + } + + // Open "f", and confirm that we see it. + f, err := dirf.OpenFile("f", os.O_RDONLY, 0) + if err != nil { + t.Fatalf("reading file after renaming parent: %v", err) + } + defer f.Close() + b, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + if got, want := string(b), "hello"; got != want { + t.Fatalf("file contents: %q, want %q", got, want) + } + + // f.Name reflects the original path we opened the directory under (".../a"), not "b". + if got, want := f.Name(), dirf.Name()+string(os.PathSeparator)+"f"; got != want { + t.Errorf("f.Name() = %q, want %q", got, want) + } +} + +func TestRootNonPermissionMode(t *testing.T) { + r, err := os.OpenRoot(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + if _, err := r.OpenFile("file", os.O_RDWR|os.O_CREATE, 0o1777); err == nil { + t.Errorf("r.OpenFile(file, O_RDWR|O_CREATE, 0o1777) succeeded; want error") + } + if err := r.Mkdir("file", 0o1777); err == nil { + t.Errorf("r.Mkdir(file, 0o1777) succeeded; want error") + } +} + +func TestRootUseAfterClose(t *testing.T) { + r, err := os.OpenRoot(t.TempDir()) + if err != nil { + t.Fatal(err) + } + r.Close() + for _, test := range []struct { + name string + f func(r *os.Root, filename string) error + }{{ + name: "Open", + f: func(r *os.Root, filename string) error { + _, err := r.Open(filename) + return err + }, + }, { + name: "Create", + f: func(r *os.Root, filename string) error { + _, err := r.Create(filename) + return err + }, + }, { + name: "OpenFile", + f: func(r *os.Root, filename string) error { + _, err := r.OpenFile(filename, os.O_RDWR, 0o666) + return err + }, + }, { + name: "OpenRoot", + f: func(r *os.Root, filename string) error { + _, err := r.OpenRoot(filename) + return err + }, + }, { + name: "Mkdir", + f: func(r *os.Root, filename string) error { + return r.Mkdir(filename, 0o777) + }, + }} { + err := test.f(r, "target") + pe, ok := err.(*os.PathError) + if !ok || pe.Path != "target" || pe.Err != os.ErrClosed { + t.Errorf(`r.%v = %v; want &PathError{Path: "target", Err: ErrClosed}`, test.name, err) + } + } +} + +func TestRootConcurrentClose(t *testing.T) { + r, err := os.OpenRoot(t.TempDir()) + if err != nil { + t.Fatal(err) + } + ch := make(chan error, 1) + go func() { + defer close(ch) + first := true + for { + f, err := r.OpenFile("file", os.O_RDWR|os.O_CREATE, 0o666) + if err != nil { + ch <- err + return + } + if first { + ch <- nil + first = false + } + f.Close() + if runtime.GOARCH == "wasm" { + // TODO(go.dev/issue/71134) can lead to goroutine starvation. + runtime.Gosched() + } + } + }() + if err := <-ch; err != nil { + t.Errorf("OpenFile: %v, want success", err) + } + r.Close() + if err := <-ch; !errors.Is(err, os.ErrClosed) { + t.Errorf("OpenFile: %v, want ErrClosed", err) + } +} + +// TestRootRaceRenameDir attempts to escape a Root by renaming a path component mid-parse. +// +// We create a deeply nested directory: +// +// base/a/a/a/a/ [...] /a +// +// And a path that descends into the tree, then returns to the top using ..: +// +// base/a/a/a/a/ [...] /a/../../../ [..] /../a/f +// +// While opening this file, we rename base/a/a to base/b. +// A naive lookup operation will resolve the path to base/f. +func TestRootRaceRenameDir(t *testing.T) { + dir := t.TempDir() + r, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + const depth = 4 + + os.MkdirAll(dir+"/base/"+strings.Repeat("/a", depth), 0o777) + + path := "base/" + strings.Repeat("a/", depth) + strings.Repeat("../", depth) + "a/f" + os.WriteFile(dir+"/f", []byte("secret"), 0o666) + os.WriteFile(dir+"/base/a/f", []byte("public"), 0o666) + + // Compute how long it takes to open the path in the common case. + const tries = 10 + var total time.Duration + for range tries { + start := time.Now() + f, err := r.Open(path) + if err != nil { + t.Fatal(err) + } + b, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + if string(b) != "public" { + t.Fatalf("read %q, want %q", b, "public") + } + f.Close() + total += time.Since(start) + } + avg := total / tries + + // We're trying to exploit a race, so try this a number of times. + for range 100 { + // Start a goroutine to open the file. + gotc := make(chan []byte) + go func() { + f, err := r.Open(path) + if err != nil { + gotc <- nil + } + defer f.Close() + b, _ := io.ReadAll(f) + gotc <- b + }() + + // Wait for the open operation to partially complete, + // and then rename a directory near the root. + time.Sleep(avg / 4) + if err := os.Rename(dir+"/base/a", dir+"/b"); err != nil { + // Windows and Plan9 won't let us rename a directory if we have + // an open handle for it, so an error here is expected. + switch runtime.GOOS { + case "windows", "plan9": + default: + t.Fatal(err) + } + } + + got := <-gotc + os.Rename(dir+"/b", dir+"/base/a") + if len(got) > 0 && string(got) != "public" { + t.Errorf("read file: %q; want error or 'public'", got) + } + } +} + +func TestRootSymlinkToRoot(t *testing.T) { + dir := makefs(t, []string{ + "d/d => ..", + }) + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer root.Close() + if err := root.Mkdir("d/d/new", 0777); err != nil { + t.Fatal(err) + } + f, err := root.Open("d/d") + if err != nil { + t.Fatal(err) + } + defer f.Close() + names, err := f.Readdirnames(-1) + if err != nil { + t.Fatal(err) + } + slices.Sort(names) + if got, want := names, []string{"d", "new"}; !slices.Equal(got, want) { + t.Errorf("root contains: %q, want %q", got, want) + } +} + +func TestOpenInRoot(t *testing.T) { + dir := makefs(t, []string{ + "file", + "link => ../ROOT/file", + }) + f, err := os.OpenInRoot(dir, "file") + if err != nil { + t.Fatalf("OpenInRoot(`file`) = %v, want success", err) + } + f.Close() + for _, name := range []string{ + "link", + "../ROOT/file", + dir + "/file", + } { + f, err := os.OpenInRoot(dir, name) + if err == nil { + f.Close() + t.Fatalf("OpenInRoot(%q) = nil, want error", name) + } + } +} + +func TestRootRemoveDot(t *testing.T) { + dir := t.TempDir() + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer root.Close() + if err := root.Remove("."); err == nil { + t.Errorf(`root.Remove(".") = %v, want error`, err) + } + if err := root.RemoveAll("."); err == nil { + t.Errorf(`root.RemoveAll(".") = %v, want error`, err) + } + if _, err := os.Stat(dir); err != nil { + t.Error(`root.Remove(All)?(".") removed the root`) + } +} + +func TestRootWriteReadFile(t *testing.T) { + dir := t.TempDir() + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + name := "filename" + want := []byte("file contents") + if err := root.WriteFile(name, want, 0o666); err != nil { + t.Fatalf("root.WriteFile(%q, %q, 0o666) = %v; want nil", name, want, err) + } + + got, err := root.ReadFile(name) + if err != nil { + t.Fatalf("root.ReadFile(%q) = %q, %v; want %q, nil", name, got, err, want) + } +} + +func TestRootName(t *testing.T) { + dir := t.TempDir() + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer root.Close() + if got, want := root.Name(), dir; got != want { + t.Errorf("root.Name() = %q, want %q", got, want) + } + + f, err := root.Create("file") + if err != nil { + t.Fatal(err) + } + defer f.Close() + if got, want := f.Name(), filepath.Join(dir, "file"); got != want { + t.Errorf(`root.Create("file").Name() = %q, want %q`, got, want) + } + + if err := root.Mkdir("dir", 0o777); err != nil { + t.Fatal(err) + } + subroot, err := root.OpenRoot("dir") + if err != nil { + t.Fatal(err) + } + defer subroot.Close() + if got, want := subroot.Name(), filepath.Join(dir, "dir"); got != want { + t.Errorf(`root.OpenRoot("dir").Name() = %q, want %q`, got, want) + } +} diff --git a/src/os/root_unix.go b/src/os/root_unix.go new file mode 100644 index 00000000000000..c891e81b793bb5 --- /dev/null +++ b/src/os/root_unix.go @@ -0,0 +1,302 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix || wasip1 + +package os + +import ( + "errors" + "internal/syscall/unix" + "runtime" + "syscall" + "time" +) + +// sysfdType is the native type of a file handle +// (int on Unix, syscall.Handle on Windows), +// permitting helper functions to be written portably. +type sysfdType = int + +// openRootNolog is OpenRoot. +func openRootNolog(name string) (*Root, error) { + var fd int + err := ignoringEINTR(func() error { + var err error + fd, _, err = open(name, syscall.O_CLOEXEC, 0) + return err + }) + if err != nil { + return nil, &PathError{Op: "open", Path: name, Err: err} + } + return newRoot(fd, name) +} + +// newRoot returns a new Root. +// If fd is not a directory, it closes it and returns an error. +func newRoot(fd int, name string) (*Root, error) { + var fs fileStat + err := ignoringEINTR(func() error { + return syscall.Fstat(fd, &fs.sys) + }) + fillFileStatFromSys(&fs, name) + if err == nil && !fs.IsDir() { + syscall.Close(fd) + return nil, &PathError{Op: "open", Path: name, Err: errors.New("not a directory")} + } + + // There's a race here with fork/exec, which we are + // content to live with. See ../syscall/exec_unix.go. + if !supportsCloseOnExec { + syscall.CloseOnExec(fd) + } + + r := &Root{&root{ + fd: fd, + name: name, + }} + runtime.SetFinalizer(r.root, (*root).Close) + return r, nil +} + +// openRootInRoot is Root.OpenRoot. +func openRootInRoot(r *Root, name string) (*Root, error) { + fd, err := doInRoot(r, name, nil, func(parent int, name string) (fd int, err error) { + ignoringEINTR(func() error { + fd, err = unix.Openat(parent, name, syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0) + if isNoFollowErr(err) { + err = checkSymlink(parent, name, err) + } + return err + }) + return fd, err + }) + if err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: err} + } + return newRoot(fd, joinPath(r.Name(), name)) +} + +// rootOpenFileNolog is Root.OpenFile. +func rootOpenFileNolog(root *Root, name string, flag int, perm FileMode) (*File, error) { + fd, err := doInRoot(root, name, nil, func(parent int, name string) (fd int, err error) { + ignoringEINTR(func() error { + fd, err = unix.Openat(parent, name, syscall.O_NOFOLLOW|syscall.O_CLOEXEC|flag, uint32(perm)) + if err != nil { + // Never follow symlinks when O_CREATE|O_EXCL, no matter + // what error the OS returns. + isCreateExcl := flag&(O_CREATE|O_EXCL) == (O_CREATE | O_EXCL) + if !isCreateExcl && (isNoFollowErr(err) || err == syscall.ENOTDIR) { + err = checkSymlink(parent, name, err) + } + // AIX returns ELOOP instead of EEXIST for a dangling symlink. + // Convert this to EEXIST so it matches ErrExists. + if isCreateExcl && err == syscall.ELOOP { + err = syscall.EEXIST + } + } + return err + }) + return fd, err + }) + if err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: err} + } + f := newFile(fd, joinPath(root.Name(), name), kindOpenFile, unix.HasNonblockFlag(flag)) + return f, nil +} + +func rootOpenDir(parent int, name string) (int, error) { + var ( + fd int + err error + ) + ignoringEINTR(func() error { + fd, err = unix.Openat(parent, name, syscall.O_NOFOLLOW|syscall.O_CLOEXEC|syscall.O_DIRECTORY, 0) + if isNoFollowErr(err) || err == syscall.ENOTDIR { + err = checkSymlink(parent, name, err) + } else if err == syscall.ENOTSUP || err == syscall.EOPNOTSUPP { + // ENOTSUP and EOPNOTSUPP are often, but not always, the same errno. + // Translate both to ENOTDIR, since this indicates a non-terminal + // path component was not a directory. + err = syscall.ENOTDIR + } + return err + }) + return fd, err +} + +func rootStat(r *Root, name string, lstat bool) (FileInfo, error) { + fi, err := doInRoot(r, name, nil, func(parent sysfdType, n string) (FileInfo, error) { + var fs fileStat + if err := unix.Fstatat(parent, n, &fs.sys, unix.AT_SYMLINK_NOFOLLOW); err != nil { + return nil, err + } + fillFileStatFromSys(&fs, name) + if !lstat && fs.Mode()&ModeSymlink != 0 { + return nil, checkSymlink(parent, n, syscall.ELOOP) + } + return &fs, nil + }) + if err != nil { + return nil, &PathError{Op: "statat", Path: name, Err: err} + } + return fi, nil +} + +func rootSymlink(r *Root, oldname, newname string) error { + _, err := doInRoot(r, newname, nil, func(parent sysfdType, name string) (struct{}, error) { + return struct{}{}, symlinkat(oldname, parent, name) + }) + if err != nil { + return &LinkError{"symlinkat", oldname, newname, err} + } + return nil +} + +// On systems which use fchmodat, fchownat, etc., we have a race condition: +// When "name" is a symlink, Root.Chmod("name") should act on the target of that link. +// However, fchmodat doesn't allow us to chmod a file only if it is not a symlink; +// the AT_SYMLINK_NOFOLLOW parameter causes the operation to act on the symlink itself. +// +// We do the best we can by first checking to see if the target of the operation is a symlink, +// and only attempting the fchmodat if it is not. If the target is replaced between the check +// and the fchmodat, we will chmod the symlink rather than following it. +// +// This race condition is unfortunate, but does not permit escaping a root: +// We may act on the wrong file, but that file will be contained within the root. +func afterResolvingSymlink(parent int, name string, f func() error) error { + if err := checkSymlink(parent, name, nil); err != nil { + return err + } + return f() +} + +func chmodat(parent int, name string, mode FileMode) error { + return afterResolvingSymlink(parent, name, func() error { + return ignoringEINTR(func() error { + return unix.Fchmodat(parent, name, syscallMode(mode), unix.AT_SYMLINK_NOFOLLOW) + }) + }) +} + +func chownat(parent int, name string, uid, gid int) error { + return afterResolvingSymlink(parent, name, func() error { + return ignoringEINTR(func() error { + return unix.Fchownat(parent, name, uid, gid, unix.AT_SYMLINK_NOFOLLOW) + }) + }) +} + +func lchownat(parent int, name string, uid, gid int) error { + return ignoringEINTR(func() error { + return unix.Fchownat(parent, name, uid, gid, unix.AT_SYMLINK_NOFOLLOW) + }) +} + +func chtimesat(parent int, name string, atime time.Time, mtime time.Time) error { + return afterResolvingSymlink(parent, name, func() error { + return ignoringEINTR(func() error { + utimes := chtimesUtimes(atime, mtime) + return unix.Utimensat(parent, name, &utimes, unix.AT_SYMLINK_NOFOLLOW) + }) + }) +} + +func mkdirat(fd int, name string, perm FileMode) error { + return ignoringEINTR(func() error { + return unix.Mkdirat(fd, name, syscallMode(perm)) + }) +} + +func removeat(fd int, name string) error { + // The system call interface forces us to know whether + // we are removing a file or directory. Try both. + e := ignoringEINTR(func() error { + return unix.Unlinkat(fd, name, 0) + }) + if e == nil { + return nil + } + e1 := ignoringEINTR(func() error { + return unix.Unlinkat(fd, name, unix.AT_REMOVEDIR) + }) + if e1 == nil { + return nil + } + // Both failed. See comment in Remove for how we decide which error to return. + if e1 != syscall.ENOTDIR { + return e1 + } + return e +} + +func removefileat(fd int, name string) error { + return ignoringEINTR(func() error { + return unix.Unlinkat(fd, name, 0) + }) +} + +func removedirat(fd int, name string) error { + return ignoringEINTR(func() error { + return unix.Unlinkat(fd, name, unix.AT_REMOVEDIR) + }) +} + +func renameat(oldfd int, oldname string, newfd int, newname string) error { + return unix.Renameat(oldfd, oldname, newfd, newname) +} + +func linkat(oldfd int, oldname string, newfd int, newname string) error { + return unix.Linkat(oldfd, oldname, newfd, newname, 0) +} + +func symlinkat(oldname string, newfd int, newname string) error { + return unix.Symlinkat(oldname, newfd, newname) +} + +func modeAt(parent int, name string) (FileMode, error) { + var fs fileStat + if err := unix.Fstatat(parent, name, &fs.sys, unix.AT_SYMLINK_NOFOLLOW); err != nil { + return 0, err + } + fillFileStatFromSys(&fs, name) + return fs.mode, nil +} + +// checkSymlink resolves the symlink name in parent, +// and returns errSymlink with the link contents. +// +// If name is not a symlink, return origError. +func checkSymlink(parent int, name string, origError error) error { + link, err := readlinkat(parent, name) + if err != nil { + return origError + } + return errSymlink(link) +} + +func readlinkat(fd int, name string) (string, error) { + for len := 128; ; len *= 2 { + b := make([]byte, len) + var ( + n int + e error + ) + ignoringEINTR(func() error { + n, e = unix.Readlinkat(fd, name, b) + return e + }) + if e == syscall.ERANGE { + continue + } + if e != nil { + return "", e + } + n = max(n, 0) + if n < len { + return string(b[0:n]), nil + } + } +} diff --git a/src/os/root_unix_test.go b/src/os/root_unix_test.go new file mode 100644 index 00000000000000..b4b37c2be9dea2 --- /dev/null +++ b/src/os/root_unix_test.go @@ -0,0 +1,164 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix || (js && wasm) || wasip1 + +package os_test + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "syscall" + "testing" +) + +func TestRootChown(t *testing.T) { + if runtime.GOOS == "wasip1" { + t.Skip("Chown not supported on " + runtime.GOOS) + } + + // Look up the current default uid/gid. + f := newFile(t) + dir, err := f.Stat() + if err != nil { + t.Fatal(err) + } + sys := dir.Sys().(*syscall.Stat_t) + + groups, err := os.Getgroups() + if err != nil { + t.Fatal(err) + } + groups = append(groups, os.Getgid()) + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + if target != "" { + if err := os.WriteFile(target, nil, 0o666); err != nil { + t.Fatal(err) + } + } + for _, gid := range groups { + err := root.Chown(test.open, -1, gid) + if errEndsTest(t, err, test.wantError, "root.Chown(%q, -1, %v)", test.open, gid) { + return + } + checkUidGid(t, target, int(sys.Uid), gid) + } + }) + } +} + +func TestRootLchown(t *testing.T) { + if runtime.GOOS == "wasip1" { + t.Skip("Lchown not supported on " + runtime.GOOS) + } + + // Look up the current default uid/gid. + f := newFile(t) + dir, err := f.Stat() + if err != nil { + t.Fatal(err) + } + sys := dir.Sys().(*syscall.Stat_t) + + groups, err := os.Getgroups() + if err != nil { + t.Fatal(err) + } + groups = append(groups, os.Getgid()) + for _, test := range rootTestCases { + test.run(t, func(t *testing.T, target string, root *os.Root) { + wantError := test.wantError + if test.ltarget != "" { + wantError = false + target = filepath.Join(root.Name(), test.ltarget) + } else if target != "" { + if err := os.WriteFile(target, nil, 0o666); err != nil { + t.Fatal(err) + } + } + for _, gid := range groups { + err := root.Lchown(test.open, -1, gid) + if errEndsTest(t, err, wantError, "root.Lchown(%q, -1, %v)", test.open, gid) { + return + } + checkUidGid(t, target, int(sys.Uid), gid) + } + }) + } +} + +func TestRootConsistencyChown(t *testing.T) { + if runtime.GOOS == "wasip1" { + t.Skip("Chown not supported on " + runtime.GOOS) + } + groups, err := os.Getgroups() + if err != nil { + t.Fatal(err) + } + var gid int + if len(groups) == 0 { + gid = os.Getgid() + } else { + gid = groups[0] + } + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + chown := os.Chown + lstat := os.Lstat + if r != nil { + chown = r.Chown + lstat = r.Lstat + } + err := chown(path, -1, gid) + if err != nil { + return "", err + } + fi, err := lstat(path) + if err != nil { + return "", err + } + sys := fi.Sys().(*syscall.Stat_t) + return fmt.Sprintf("%v %v", sys.Uid, sys.Gid), nil + }) + } +} + +func TestRootConsistencyLchown(t *testing.T) { + if runtime.GOOS == "wasip1" { + t.Skip("Lchown not supported on " + runtime.GOOS) + } + groups, err := os.Getgroups() + if err != nil { + t.Fatal(err) + } + var gid int + if len(groups) == 0 { + gid = os.Getgid() + } else { + gid = groups[0] + } + for _, test := range rootConsistencyTestCases { + test.run(t, func(t *testing.T, path string, r *os.Root) (string, error) { + lchown := os.Lchown + lstat := os.Lstat + if r != nil { + lchown = r.Lchown + lstat = r.Lstat + } + err := lchown(path, -1, gid) + if err != nil { + return "", err + } + fi, err := lstat(path) + if err != nil { + return "", err + } + sys := fi.Sys().(*syscall.Stat_t) + return fmt.Sprintf("%v %v", sys.Uid, sys.Gid), nil + }) + } +} diff --git a/src/os/root_windows.go b/src/os/root_windows.go new file mode 100644 index 00000000000000..033a119862d292 --- /dev/null +++ b/src/os/root_windows.go @@ -0,0 +1,404 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build windows + +package os + +import ( + "errors" + "internal/filepathlite" + "internal/stringslite" + "internal/syscall/windows" + "runtime" + "syscall" + "time" + "unsafe" +) + +// rootCleanPath uses GetFullPathName to perform lexical path cleaning. +// +// On Windows, file names are lexically cleaned at the start of a file operation. +// For example, on Windows the path `a\..\b` is exactly equivalent to `b` alone, +// even if `a` does not exist or is not a directory. +// +// We use the Windows API function GetFullPathName to perform this cleaning. +// We could do this ourselves, but there are a number of subtle behaviors here, +// and deferring to the OS maintains consistency. +// (For example, `a\.\` cleans to `a\`.) +// +// GetFullPathName operates on absolute paths, and our input path is relative. +// We make the path absolute by prepending a fixed prefix of \\?\?\. +// +// We want to detect paths which use .. components to escape the root. +// We do this by ensuring the cleaned path still begins with \\?\?\. +// We catch the corner case of a path which includes a ..\?\. component +// by rejecting any input paths which contain a ?, which is not a valid character +// in a Windows filename. +func rootCleanPath(s string, prefix, suffix []string) (string, error) { + // Reject paths which include a ? component (see above). + if stringslite.IndexByte(s, '?') >= 0 { + return "", windows.ERROR_INVALID_NAME + } + + const fixedPrefix = `\\?\?` + buf := []byte(fixedPrefix) + for _, p := range prefix { + buf = append(buf, '\\') + buf = append(buf, []byte(p)...) + } + buf = append(buf, '\\') + buf = append(buf, []byte(s)...) + for _, p := range suffix { + buf = append(buf, '\\') + buf = append(buf, []byte(p)...) + } + s = string(buf) + + s, err := syscall.FullPath(s) + if err != nil { + return "", err + } + + s, ok := stringslite.CutPrefix(s, fixedPrefix) + if !ok { + return "", errPathEscapes + } + s = stringslite.TrimPrefix(s, `\`) + if s == "" { + s = "." + } + + if !filepathlite.IsLocal(s) { + return "", errPathEscapes + } + + return s, nil +} + +// sysfdType is the native type of a file handle +// (int on Unix, syscall.Handle on Windows), +// permitting helper functions to be written portably. +type sysfdType = syscall.Handle + +// openRootNolog is OpenRoot. +func openRootNolog(name string) (*Root, error) { + if name == "" { + return nil, &PathError{Op: "open", Path: name, Err: syscall.ENOENT} + } + path := fixLongPath(name) + fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_CLOEXEC, 0) + if err != nil { + return nil, &PathError{Op: "open", Path: name, Err: err} + } + return newRoot(fd, name) +} + +// newRoot returns a new Root. +// If fd is not a directory, it closes it and returns an error. +func newRoot(fd syscall.Handle, name string) (*Root, error) { + // Check that this is a directory. + // + // If we get any errors here, ignore them; worst case we create a Root + // which returns errors when you try to use it. + var fi syscall.ByHandleFileInformation + err := syscall.GetFileInformationByHandle(fd, &fi) + if err == nil && fi.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY == 0 { + syscall.CloseHandle(fd) + return nil, &PathError{Op: "open", Path: name, Err: errors.New("not a directory")} + } + + r := &Root{&root{ + fd: fd, + name: name, + }} + runtime.SetFinalizer(r.root, (*root).Close) + return r, nil +} + +// openRootInRoot is Root.OpenRoot. +func openRootInRoot(r *Root, name string) (*Root, error) { + fd, err := doInRoot(r, name, nil, rootOpenDir) + if err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: err} + } + return newRoot(fd, joinPath(r.Name(), name)) +} + +// rootOpenFileNolog is Root.OpenFile. +func rootOpenFileNolog(root *Root, name string, flag int, perm FileMode) (*File, error) { + fd, err := doInRoot(root, name, nil, func(parent syscall.Handle, name string) (syscall.Handle, error) { + return openat(parent, name, uint64(flag), perm) + }) + if err != nil { + return nil, &PathError{Op: "openat", Path: name, Err: err} + } + // openat always returns a non-blocking handle. + return newFile(fd, joinPath(root.Name(), name), "file", false), nil +} + +func openat(dirfd syscall.Handle, name string, flag uint64, perm FileMode) (syscall.Handle, error) { + h, err := windows.Openat(dirfd, name, flag|syscall.O_CLOEXEC|windows.O_NOFOLLOW_ANY, syscallMode(perm)) + if err == syscall.ELOOP || err == syscall.ENOTDIR { + if link, err := readReparseLinkAt(dirfd, name); err == nil { + return syscall.InvalidHandle, errSymlink(link) + } + } + return h, err +} + +func readReparseLinkAt(dirfd syscall.Handle, name string) (string, error) { + objectName, err := windows.NewNTUnicodeString(name) + if err != nil { + return "", err + } + objAttrs := &windows.OBJECT_ATTRIBUTES{ + ObjectName: objectName, + } + if dirfd != syscall.InvalidHandle { + objAttrs.RootDirectory = dirfd + } + objAttrs.Length = uint32(unsafe.Sizeof(*objAttrs)) + var h syscall.Handle + err = windows.NtCreateFile( + &h, + windows.FILE_GENERIC_READ, + objAttrs, + &windows.IO_STATUS_BLOCK{}, + nil, + uint32(syscall.FILE_ATTRIBUTE_NORMAL), + syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, + windows.FILE_OPEN, + windows.FILE_SYNCHRONOUS_IO_NONALERT|windows.FILE_OPEN_REPARSE_POINT, + nil, + 0, + ) + if err != nil { + return "", err + } + defer syscall.CloseHandle(h) + return readReparseLinkHandle(h) +} + +func rootOpenDir(parent syscall.Handle, name string) (syscall.Handle, error) { + h, err := openat(parent, name, syscall.O_RDONLY|syscall.O_CLOEXEC|windows.O_DIRECTORY, 0) + if err == syscall.ERROR_FILE_NOT_FOUND { + // Windows returns: + // - ERROR_PATH_NOT_FOUND if any path compoenent before the leaf + // does not exist or is not a directory. + // - ERROR_FILE_NOT_FOUND if the leaf does not exist. + // + // This differs from Unix behavior, which is: + // - ENOENT if any path component does not exist, including the leaf. + // - ENOTDIR if any path component before the leaf is not a directory. + // + // We map syscall.ENOENT to ERROR_FILE_NOT_FOUND and syscall.ENOTDIR + // to ERROR_PATH_NOT_FOUND, but the Windows errors don't quite match. + // + // For consistency with os.Open, convert ERROR_FILE_NOT_FOUND here into + // ERROR_PATH_NOT_FOUND, since we're opening a non-leaf path component. + err = syscall.ERROR_PATH_NOT_FOUND + } + return h, err +} + +func rootStat(r *Root, name string, lstat bool) (FileInfo, error) { + if len(name) > 0 && IsPathSeparator(name[len(name)-1]) { + // When a filename ends with a path separator, + // Lstat behaves like Stat. + // + // This behavior is not based on a principled decision here, + // merely the empirical evidence that Lstat behaves this way. + lstat = false + } + fi, err := doInRoot(r, name, nil, func(parent syscall.Handle, n string) (FileInfo, error) { + fd, err := openat(parent, n, windows.O_OPEN_REPARSE, 0) + if err != nil { + return nil, err + } + defer syscall.CloseHandle(fd) + fi, err := statHandle(name, fd) + if err != nil { + return nil, err + } + if !lstat && fi.(*fileStat).isReparseTagNameSurrogate() { + link, err := readReparseLinkHandle(fd) + if err != nil { + return nil, err + } + return nil, errSymlink(link) + } + return fi, nil + }) + if err != nil { + return nil, &PathError{Op: "statat", Path: name, Err: err} + } + return fi, nil +} + +func rootSymlink(r *Root, oldname, newname string) error { + if oldname == "" { + return syscall.EINVAL + } + + // CreateSymbolicLinkW converts volume-relative paths into absolute ones. + // Do the same. + if filepathlite.VolumeNameLen(oldname) > 0 && !filepathlite.IsAbs(oldname) { + p, err := syscall.FullPath(oldname) + if err == nil { + oldname = p + } + } + + // If oldname can be resolved to a directory in the root, create a directory link. + // Otherwise, create a file link. + var flags windows.SymlinkatFlags + if filepathlite.VolumeNameLen(oldname) == 0 && !IsPathSeparator(oldname[0]) { + // oldname is a path relative to the directory containing newname. + // Prepend newname's directory to it to make a path relative to the root. + // For example, if oldname=old and newname=a\new, destPath=a\old. + destPath := oldname + if dir := dirname(newname); dir != "." { + destPath = dir + `\` + oldname + } + fi, err := r.Stat(destPath) + if err == nil && fi.IsDir() { + flags |= windows.SYMLINKAT_DIRECTORY + } + } + + // Empirically, CreateSymbolicLinkW appears to set the relative flag iff + // the target does not contain a volume name. + if filepathlite.VolumeNameLen(oldname) == 0 { + flags |= windows.SYMLINKAT_RELATIVE + } + + _, err := doInRoot(r, newname, nil, func(parent sysfdType, name string) (struct{}, error) { + return struct{}{}, windows.Symlinkat(oldname, parent, name, flags) + }) + if err != nil { + return &LinkError{"symlinkat", oldname, newname, err} + } + return nil +} + +func chmodat(parent syscall.Handle, name string, mode FileMode) error { + // Currently, on Windows os.Chmod("symlink") will act on "symlink", + // not on any file it points to. + // + // This may or may not be the desired behavior: https://go.dev/issue/71492 + // + // For now, be consistent with os.Symlink. + // Passing O_OPEN_REPARSE causes us to open the named file itself, + // not any file that it links to. + // + // If we want to change this in the future, pass O_NOFOLLOW_ANY instead + // and return errSymlink when encountering a symlink: + // + // if err == syscall.ELOOP || err == syscall.ENOTDIR { + // if link, err := readReparseLinkAt(parent, name); err == nil { + // return errSymlink(link) + // } + // } + h, err := windows.Openat(parent, name, syscall.O_CLOEXEC|windows.O_OPEN_REPARSE|windows.O_WRITE_ATTRS, 0) + if err != nil { + return err + } + defer syscall.CloseHandle(h) + + var d syscall.ByHandleFileInformation + if err := syscall.GetFileInformationByHandle(h, &d); err != nil { + return err + } + attrs := d.FileAttributes + + if mode&syscall.S_IWRITE != 0 { + attrs &^= syscall.FILE_ATTRIBUTE_READONLY + } else { + attrs |= syscall.FILE_ATTRIBUTE_READONLY + } + if attrs == d.FileAttributes { + return nil + } + + var fbi windows.FILE_BASIC_INFO + fbi.FileAttributes = attrs + return windows.SetFileInformationByHandle(h, windows.FileBasicInfo, unsafe.Pointer(&fbi), uint32(unsafe.Sizeof(fbi))) +} + +func chownat(parent syscall.Handle, name string, uid, gid int) error { + return syscall.EWINDOWS // matches syscall.Chown +} + +func lchownat(parent syscall.Handle, name string, uid, gid int) error { + return syscall.EWINDOWS // matches syscall.Lchown +} + +func mkdirat(dirfd syscall.Handle, name string, perm FileMode) error { + return windows.Mkdirat(dirfd, name, syscallMode(perm)) +} + +func removeat(dirfd syscall.Handle, name string) error { + return windows.Deleteat(dirfd, name, 0) +} + +func removefileat(dirfd syscall.Handle, name string) error { + return windows.Deleteat(dirfd, name, windows.FILE_NON_DIRECTORY_FILE) +} + +func removedirat(dirfd syscall.Handle, name string) error { + return windows.Deleteat(dirfd, name, windows.FILE_DIRECTORY_FILE) +} + +func chtimesat(dirfd syscall.Handle, name string, atime time.Time, mtime time.Time) error { + h, err := windows.Openat(dirfd, name, syscall.O_CLOEXEC|windows.O_NOFOLLOW_ANY|windows.O_WRITE_ATTRS, 0) + if err == syscall.ELOOP || err == syscall.ENOTDIR { + if link, err := readReparseLinkAt(dirfd, name); err == nil { + return errSymlink(link) + } + } + if err != nil { + return err + } + defer syscall.CloseHandle(h) + a := syscall.Filetime{} + w := syscall.Filetime{} + if !atime.IsZero() { + a = syscall.NsecToFiletime(atime.UnixNano()) + } + if !mtime.IsZero() { + w = syscall.NsecToFiletime(mtime.UnixNano()) + } + return syscall.SetFileTime(h, nil, &a, &w) +} + +func renameat(oldfd syscall.Handle, oldname string, newfd syscall.Handle, newname string) error { + return windows.Renameat(oldfd, oldname, newfd, newname) +} + +func linkat(oldfd syscall.Handle, oldname string, newfd syscall.Handle, newname string) error { + return windows.Linkat(oldfd, oldname, newfd, newname) +} + +func readlinkat(dirfd syscall.Handle, name string) (string, error) { + fd, err := openat(dirfd, name, windows.O_OPEN_REPARSE, 0) + if err != nil { + return "", err + } + defer syscall.CloseHandle(fd) + return readReparseLinkHandle(fd) +} + +func modeAt(parent syscall.Handle, name string) (FileMode, error) { + fd, err := openat(parent, name, windows.O_OPEN_REPARSE|windows.O_DIRECTORY, 0) + if err != nil { + return 0, err + } + defer syscall.CloseHandle(fd) + fi, err := statHandle(name, fd) + if err != nil { + return 0, err + } + return fi.Mode(), nil +} diff --git a/src/os/root_windows_test.go b/src/os/root_windows_test.go new file mode 100644 index 00000000000000..8ae6f0c9d34d74 --- /dev/null +++ b/src/os/root_windows_test.go @@ -0,0 +1,230 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build windows + +package os_test + +import ( + "errors" + "fmt" + "internal/syscall/windows" + "os" + "path/filepath" + "syscall" + "testing" + "unsafe" +) + +// Verify that Root.Open rejects Windows reserved names. +func TestRootWindowsDeviceNames(t *testing.T) { + r, err := os.OpenRoot(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + if f, err := r.Open("NUL"); err == nil { + t.Errorf(`r.Open("NUL") succeeded; want error"`) + f.Close() + } +} + +// Verify that Root.Open is case-insensitive. +// (The wrong options to NtOpenFile could make operations case-sensitive, +// so this is worth checking.) +func TestRootWindowsCaseInsensitivity(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "file"), nil, 0666); err != nil { + t.Fatal(err) + } + r, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer r.Close() + f, err := r.Open("FILE") + if err != nil { + t.Fatal(err) + } + f.Close() + if err := r.Remove("FILE"); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dir, "file")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("os.Stat(file) after deletion: %v, want ErrNotFound", err) + } +} + +// TestRootSymlinkRelativity tests that symlinks created using Root.Symlink have the +// same SYMLINK_FLAG_RELATIVE value as ones creates using os.Symlink. +func TestRootSymlinkRelativity(t *testing.T) { + dir := t.TempDir() + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + for i, test := range []struct { + name string + target string + }{{ + name: "relative", + target: `foo`, + }, { + name: "absolute", + target: `C:\foo`, + }, { + name: "current working directory-relative", + target: `C:foo`, + }, { + name: "root-relative", + target: `\foo`, + }, { + name: "question prefix", + target: `\\?\foo`, + }, { + name: "relative with dot dot", + target: `a\..\b`, // could be cleaned (but isn't) + }} { + t.Run(test.name, func(t *testing.T) { + name := fmt.Sprintf("symlink_%v", i) + if err := os.Symlink(test.target, filepath.Join(dir, name)); err != nil { + t.Fatal(err) + } + if err := root.Symlink(test.target, name+"_at"); err != nil { + t.Fatal(err) + } + + osRDB, err := readSymlinkReparseData(filepath.Join(dir, name)) + if err != nil { + t.Fatal(err) + } + rootRDB, err := readSymlinkReparseData(filepath.Join(dir, name+"_at")) + if err != nil { + t.Fatal(err) + } + if osRDB.Flags != rootRDB.Flags { + t.Errorf("symlink target %q: Symlink flags = %x, Root.Symlink flags = %x", test.target, osRDB.Flags, rootRDB.Flags) + } + + // Compare the link target. + // os.Symlink converts current working directory-relative links + // such as c:foo into absolute links. + osTarget, err := os.Readlink(filepath.Join(dir, name)) + if err != nil { + t.Fatal(err) + } + rootTarget, err := os.Readlink(filepath.Join(dir, name+"_at")) + if err != nil { + t.Fatal(err) + } + if osTarget != rootTarget { + t.Errorf("symlink created with target %q: Symlink target = %q, Root.Symlink target = %q", test.target, osTarget, rootTarget) + } + }) + } +} + +func readSymlinkReparseData(name string) (*windows.SymbolicLinkReparseBuffer, error) { + nameu16, err := syscall.UTF16FromString(name) + if err != nil { + return nil, err + } + h, err := syscall.CreateFile(&nameu16[0], syscall.GENERIC_READ, 0, nil, syscall.OPEN_EXISTING, + syscall.FILE_FLAG_OPEN_REPARSE_POINT|syscall.FILE_FLAG_BACKUP_SEMANTICS, 0) + if err != nil { + return nil, err + } + defer syscall.CloseHandle(h) + + var rdbbuf [syscall.MAXIMUM_REPARSE_DATA_BUFFER_SIZE]byte + var bytesReturned uint32 + err = syscall.DeviceIoControl(h, syscall.FSCTL_GET_REPARSE_POINT, nil, 0, &rdbbuf[0], uint32(len(rdbbuf)), &bytesReturned, nil) + if err != nil { + return nil, err + } + + rdb := (*windows.REPARSE_DATA_BUFFER)(unsafe.Pointer(&rdbbuf[0])) + if rdb.ReparseTag != syscall.IO_REPARSE_TAG_SYMLINK { + return nil, fmt.Errorf("%q: not a symlink", name) + } + + bufoff := unsafe.Offsetof(rdb.DUMMYUNIONNAME) + symlinkBuf := (*windows.SymbolicLinkReparseBuffer)(unsafe.Pointer(&rdbbuf[bufoff])) + + return symlinkBuf, nil +} + +// TestRootSymlinkToDirectory tests that Root.Symlink creates directory links +// when the target is a directory contained within the root. +func TestRootSymlinkToDirectory(t *testing.T) { + dir := t.TempDir() + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + defer root.Close() + + if err := os.Mkdir(filepath.Join(dir, "dir"), 0777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "file"), nil, 0666); err != nil { + t.Fatal(err) + } + + dir2 := t.TempDir() + + for i, test := range []struct { + name string + target string + wantDir bool + }{{ + name: "directory outside root", + target: dir2, + wantDir: false, + }, { + name: "directory inside root", + target: "dir", + wantDir: true, + }, { + name: "file inside root", + target: "file", + wantDir: false, + }, { + name: "nonexistent inside root", + target: "nonexistent", + wantDir: false, + }} { + t.Run(test.name, func(t *testing.T) { + name := fmt.Sprintf("symlink_%v", i) + if err := root.Symlink(test.target, name); err != nil { + t.Fatal(err) + } + + // Lstat strips the directory mode bit from reparse points, + // so we need to use GetFileInformationByHandle directly to + // determine if this is a directory link. + nameu16, err := syscall.UTF16PtrFromString(filepath.Join(dir, name)) + if err != nil { + t.Fatal(err) + } + h, err := syscall.CreateFile(nameu16, 0, 0, nil, syscall.OPEN_EXISTING, + syscall.FILE_FLAG_OPEN_REPARSE_POINT|syscall.FILE_FLAG_BACKUP_SEMANTICS, 0) + if err != nil { + t.Fatal(err) + } + defer syscall.CloseHandle(h) + var fi syscall.ByHandleFileInformation + if err := syscall.GetFileInformationByHandle(h, &fi); err != nil { + t.Fatal(err) + } + gotDir := fi.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 + + if got, want := gotDir, test.wantDir; got != want { + t.Errorf("link target %q: isDir = %v, want %v", test.target, got, want) + } + }) + } +} diff --git a/src/os/signal/doc.go b/src/os/signal/doc.go index 1d3e6eb573a552..df942b842898de 100644 --- a/src/os/signal/doc.go +++ b/src/os/signal/doc.go @@ -98,12 +98,13 @@ the behavior depends on the file descriptor number. A write to a broken pipe on file descriptors 1 or 2 (standard output or standard error) will cause the program to exit with a SIGPIPE signal. A write to a broken pipe on some other file descriptor will take no action on -the SIGPIPE signal, and the write will fail with an EPIPE error. +the SIGPIPE signal, and the write will fail with a [syscall.EPIPE] +error. If the program has called Notify to receive SIGPIPE signals, the file descriptor number does not matter. The SIGPIPE signal will be -delivered to the Notify channel, and the write will fail with an EPIPE -error. +delivered to the Notify channel, and the write will fail with a +[syscall.EPIPE] error. This means that, by default, command line programs will behave like typical Unix command line programs, while other programs will not diff --git a/src/os/signal/signal.go b/src/os/signal/signal.go index 9a4cd64fb75e0e..b9fe16baa50b1f 100644 --- a/src/os/signal/signal.go +++ b/src/os/signal/signal.go @@ -7,6 +7,7 @@ package signal import ( "context" "os" + "slices" "sync" ) @@ -217,7 +218,7 @@ func Stop(c chan<- os.Signal) { for i, s := range handlers.stopping { if s.c == c { - handlers.stopping = append(handlers.stopping[:i], handlers.stopping[i+1:]...) + handlers.stopping = slices.Delete(handlers.stopping, i, i+1) break } } diff --git a/src/os/signal/signal_test.go b/src/os/signal/signal_test.go index d54787bc199a3b..8d3f230178ece5 100644 --- a/src/os/signal/signal_test.go +++ b/src/os/signal/signal_test.go @@ -304,7 +304,7 @@ func TestDetectNohup(t *testing.T) { // We have no intention of reading from c. c := make(chan os.Signal, 1) Notify(c, syscall.SIGHUP) - if out, err := testenv.Command(t, os.Args[0], "-test.run=^TestDetectNohup$", "-check_sighup_ignored").CombinedOutput(); err == nil { + if out, err := testenv.Command(t, testenv.Executable(t), "-test.run=^TestDetectNohup$", "-check_sighup_ignored").CombinedOutput(); err == nil { t.Errorf("ran test with -check_sighup_ignored and it succeeded: expected failure.\nOutput:\n%s", out) } Stop(c) @@ -316,7 +316,7 @@ func TestDetectNohup(t *testing.T) { } Ignore(syscall.SIGHUP) os.Remove("nohup.out") - out, err := testenv.Command(t, "/usr/bin/nohup", os.Args[0], "-test.run=^TestDetectNohup$", "-check_sighup_ignored").CombinedOutput() + out, err := testenv.Command(t, "/usr/bin/nohup", testenv.Executable(t), "-test.run=^TestDetectNohup$", "-check_sighup_ignored").CombinedOutput() data, _ := os.ReadFile("nohup.out") os.Remove("nohup.out") @@ -347,7 +347,6 @@ func TestStop(t *testing.T) { } for _, sig := range sigs { - sig := sig t.Run(fmt.Sprint(sig), func(t *testing.T) { // When calling Notify with a specific signal, // independent signals should not interfere with each other, @@ -441,7 +440,6 @@ func TestNohup(t *testing.T) { subTimeout -= subTimeout / 10 // Leave 10% headroom for propagating output. } for i := 1; i <= 2; i++ { - i := i t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { t.Parallel() @@ -454,7 +452,7 @@ func TestNohup(t *testing.T) { if subTimeout != 0 { args = append(args, fmt.Sprintf("-test.timeout=%v", subTimeout)) } - out, err := testenv.Command(t, os.Args[0], args...).CombinedOutput() + out, err := testenv.Command(t, testenv.Executable(t), args...).CombinedOutput() if err == nil { t.Errorf("ran test with -send_uncaught_sighup=%d and it succeeded: expected failure.\nOutput:\n%s", i, out) @@ -484,7 +482,6 @@ func TestNohup(t *testing.T) { subTimeout -= subTimeout / 10 // Leave 10% headroom for propagating output. } for i := 1; i <= 2; i++ { - i := i t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { t.Parallel() @@ -562,7 +559,7 @@ func TestAtomicStop(t *testing.T) { if deadline, ok := t.Deadline(); ok { timeout = time.Until(deadline).String() } - cmd := testenv.Command(t, os.Args[0], "-test.run=^TestAtomicStop$", "-test.timeout="+timeout) + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^TestAtomicStop$", "-test.timeout="+timeout) cmd.Env = append(os.Environ(), "GO_TEST_ATOMIC_STOP=1") out, err := cmd.CombinedOutput() if err == nil { @@ -743,7 +740,6 @@ func TestNotifyContextNotifications(t *testing.T) { {"multiple", 10}, } for _, tc := range testCases { - tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -765,7 +761,7 @@ func TestNotifyContextNotifications(t *testing.T) { if subTimeout != 0 { args = append(args, fmt.Sprintf("-test.timeout=%v", subTimeout)) } - out, err := testenv.Command(t, os.Args[0], args...).CombinedOutput() + out, err := testenv.Command(t, testenv.Executable(t), args...).CombinedOutput() if err != nil { t.Errorf("ran test with -check_notify_ctx_notification and it failed with %v.\nOutput:\n%s", err, out) } diff --git a/src/os/stat_plan9.go b/src/os/stat_plan9.go index a5e9901379aeac..e9fba17e9d1b51 100644 --- a/src/os/stat_plan9.go +++ b/src/os/stat_plan9.go @@ -59,7 +59,7 @@ func dirstat(arg any) (*syscall.Dir, error) { if err := a.incref("fstat"); err != nil { return nil, err } - n, err = syscall.Fstat(a.fd, buf) + n, err = syscall.Fstat(a.sysfd, buf) a.decref() case string: name = a diff --git a/src/os/stat_test.go b/src/os/stat_test.go index 36da573f0c0280..92ecabb787a1fe 100644 --- a/src/os/stat_test.go +++ b/src/os/stat_test.go @@ -361,3 +361,18 @@ func TestClosedStat(t *testing.T) { t.Errorf("error from Stat on closed file did not match ErrClosed: %q, type %T", err, err) } } + +func TestStatNotExist(t *testing.T) { + t.Parallel() + name := filepath.Join(t.TempDir(), "notfound") + _, err := os.Stat(name) + if !errors.Is(err, fs.ErrNotExist) { + t.Errorf("os.Stat(%q) = %v; want fs.ErrNotExist", name, err) + } + + name = filepath.Join(t.TempDir(), "notfounddir", "notfound") + _, err = os.Stat(name) + if !errors.Is(err, fs.ErrNotExist) { + t.Errorf("os.Stat(%q) = %v; want fs.ErrNotExist", name, err) + } +} diff --git a/src/os/stat_wasip1.go b/src/os/stat_wasip1.go index 85a36498899269..dcf2e38ddec2d7 100644 --- a/src/os/stat_wasip1.go +++ b/src/os/stat_wasip1.go @@ -15,7 +15,6 @@ import ( func fillFileStatFromSys(fs *fileStat, name string) { fs.name = filepathlite.Base(name) fs.size = int64(fs.sys.Size) - fs.mode = FileMode(fs.sys.Mode) fs.modTime = time.Unix(0, int64(fs.sys.Mtime)) switch fs.sys.Filetype { @@ -32,6 +31,15 @@ func fillFileStatFromSys(fs *fileStat, name string) { case syscall.FILETYPE_SYMBOLIC_LINK: fs.mode |= ModeSymlink } + + // WASI does not support unix-like permissions, but Go programs are likely + // to expect the permission bits to not be zero so we set defaults to help + // avoid breaking applications that are migrating to WASM. + if fs.sys.Filetype == syscall.FILETYPE_DIRECTORY { + fs.mode |= 0700 + } else { + fs.mode |= 0600 + } } // For testing. diff --git a/src/os/stat_windows.go b/src/os/stat_windows.go index 160a3893ce56f1..e2ed58a3a0b6e9 100644 --- a/src/os/stat_windows.go +++ b/src/os/stat_windows.go @@ -5,6 +5,7 @@ package os import ( + "errors" "internal/filepathlite" "internal/syscall/windows" "syscall" @@ -34,6 +35,9 @@ func stat(funcname, name string, followSurrogates bool) (FileInfo, error) { // See https://golang.org/issues/19922#issuecomment-300031421 for details. var fa syscall.Win32FileAttributeData err = syscall.GetFileAttributesEx(namep, syscall.GetFileExInfoStandard, (*byte)(unsafe.Pointer(&fa))) + if errors.Is(err, ErrNotExist) { + return nil, &PathError{Op: "GetFileAttributesEx", Path: name, Err: err} + } if err == nil && fa.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT == 0 { // Not a surrogate for another named entity, because it isn't any kind of reparse point. // The information we got from GetFileAttributesEx is good enough for now. @@ -115,7 +119,7 @@ func statHandle(name string, h syscall.Handle) (FileInfo, error) { return nil, err } fs.filetype = ft - return fs, err + return fs, nil } // statNolog implements Stat for Windows. diff --git a/src/os/tempfile.go b/src/os/tempfile.go index af70b360b3a0f0..428dc965b7a017 100644 --- a/src/os/tempfile.go +++ b/src/os/tempfile.go @@ -105,7 +105,7 @@ func MkdirTemp(dir, pattern string) (string, error) { if try++; try < 10000 { continue } - return "", &PathError{Op: "mkdirtemp", Path: dir + string(PathSeparator) + prefix + "*" + suffix, Err: ErrExist} + return "", &PathError{Op: "mkdirtemp", Path: prefix + "*" + suffix, Err: ErrExist} } if IsNotExist(err) { if _, err := Stat(dir); IsNotExist(err) { diff --git a/src/os/tempfile_test.go b/src/os/tempfile_test.go index 82f0aabda07fdc..f2b4ffa7500cac 100644 --- a/src/os/tempfile_test.go +++ b/src/os/tempfile_test.go @@ -17,13 +17,7 @@ import ( func TestCreateTemp(t *testing.T) { t.Parallel() - dir, err := MkdirTemp("", "TestCreateTempBadDir") - if err != nil { - t.Fatal(err) - } - defer RemoveAll(dir) - - nonexistentDir := filepath.Join(dir, "_not_exists_") + nonexistentDir := filepath.Join(t.TempDir(), "_not_exists_") f, err := CreateTemp(nonexistentDir, "foo") if f != nil || err == nil { t.Errorf("CreateTemp(%q, `foo`) = %v, %v", nonexistentDir, f, err) @@ -57,11 +51,7 @@ func TestCreateTempPattern(t *testing.T) { func TestCreateTempBadPattern(t *testing.T) { t.Parallel() - tmpDir, err := MkdirTemp("", t.Name()) - if err != nil { - t.Fatal(err) - } - defer RemoveAll(tmpDir) + tmpDir := t.TempDir() const sep = string(PathSeparator) tests := []struct { @@ -152,14 +142,8 @@ func TestMkdirTemp(t *testing.T) { func TestMkdirTempBadDir(t *testing.T) { t.Parallel() - dir, err := MkdirTemp("", "MkdirTempBadDir") - if err != nil { - t.Fatal(err) - } - defer RemoveAll(dir) - - badDir := filepath.Join(dir, "not-exist") - _, err = MkdirTemp(badDir, "foo") + badDir := filepath.Join(t.TempDir(), "not-exist") + _, err := MkdirTemp(badDir, "foo") if pe, ok := err.(*fs.PathError); !ok || !IsNotExist(err) || pe.Path != badDir { t.Errorf("TempDir error = %#v; want PathError for path %q satisfying IsNotExist", err, badDir) } @@ -168,11 +152,7 @@ func TestMkdirTempBadDir(t *testing.T) { func TestMkdirTempBadPattern(t *testing.T) { t.Parallel() - tmpDir, err := MkdirTemp("", t.Name()) - if err != nil { - t.Fatal(err) - } - defer RemoveAll(tmpDir) + tmpDir := t.TempDir() const sep = string(PathSeparator) tests := []struct { diff --git a/src/os/testdata/dirfs/dir/x b/src/os/testdata/dirfs/dir/x old mode 100644 new mode 100755 diff --git a/src/os/timeout_test.go b/src/os/timeout_test.go index e0d2328ba14898..a8a38d24a19166 100644 --- a/src/os/timeout_test.go +++ b/src/os/timeout_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !js && !plan9 && !wasip1 && !windows +//go:build !js && !plan9 && !wasip1 package os_test @@ -11,18 +11,16 @@ import ( "io" "math/rand" "os" - "os/signal" "runtime" "sync" - "syscall" "testing" "time" ) func TestNonpollableDeadline(t *testing.T) { // On BSD systems regular files seem to be pollable, - // so just run this test on Linux. - if runtime.GOOS != "linux" { + // so just run this test on Linux and Windows. + if runtime.GOOS != "linux" && runtime.GOOS != "windows" { t.Skipf("skipping on %s", runtime.GOOS) } t.Parallel() @@ -45,6 +43,13 @@ func TestNonpollableDeadline(t *testing.T) { } } +type pipeDeadlineTest struct { + name string + create func(t *testing.T) (r, w *os.File) +} + +var pipeDeadlinesTestCases []pipeDeadlineTest + // noDeadline is a zero time.Time value, which cancels a deadline. var noDeadline time.Time @@ -63,40 +68,43 @@ var readTimeoutTests = []struct { func TestReadTimeout(t *testing.T) { t.Parallel() - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - defer r.Close() - defer w.Close() + for _, tc := range pipeDeadlinesTestCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() - if _, err := w.Write([]byte("READ TIMEOUT TEST")); err != nil { - t.Fatal(err) - } + r, w := tc.create(t) + defer r.Close() + defer w.Close() - for i, tt := range readTimeoutTests { - if err := r.SetReadDeadline(time.Now().Add(tt.timeout)); err != nil { - t.Fatalf("#%d: %v", i, err) - } - var b [1]byte - for j, xerr := range tt.xerrs { - for { - n, err := r.Read(b[:]) - if xerr != nil { - if !isDeadlineExceeded(err) { - t.Fatalf("#%d/%d: %v", i, j, err) - } - } - if err == nil { - time.Sleep(tt.timeout / 3) - continue + if _, err := w.Write([]byte("READ TIMEOUT TEST")); err != nil { + t.Fatal(err) + } + + for i, tt := range readTimeoutTests { + if err := r.SetReadDeadline(time.Now().Add(tt.timeout)); err != nil { + t.Fatalf("#%d: %v", i, err) } - if n != 0 { - t.Fatalf("#%d/%d: read %d; want 0", i, j, n) + var b [1]byte + for j, xerr := range tt.xerrs { + for { + n, err := r.Read(b[:]) + if xerr != nil { + if !isDeadlineExceeded(err) { + t.Fatalf("#%d/%d: %v", i, j, err) + } + } + if err == nil { + time.Sleep(tt.timeout / 3) + continue + } + if n != 0 { + t.Fatalf("#%d/%d: read %d; want 0", i, j, n) + } + break + } } - break } - } + }) } } @@ -104,40 +112,43 @@ func TestReadTimeout(t *testing.T) { func TestReadTimeoutMustNotReturn(t *testing.T) { t.Parallel() - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - defer r.Close() - defer w.Close() - - max := time.NewTimer(100 * time.Millisecond) - defer max.Stop() - ch := make(chan error) - go func() { - if err := r.SetDeadline(time.Now().Add(-5 * time.Second)); err != nil { - t.Error(err) - } - if err := r.SetWriteDeadline(time.Now().Add(-5 * time.Second)); err != nil { - t.Error(err) - } - if err := r.SetReadDeadline(noDeadline); err != nil { - t.Error(err) - } - var b [1]byte - _, err := r.Read(b[:]) - ch <- err - }() - - select { - case err := <-ch: - t.Fatalf("expected Read to not return, but it returned with %v", err) - case <-max.C: - w.Close() - err := <-ch // wait for tester goroutine to stop - if os.IsTimeout(err) { - t.Fatal(err) - } + for _, tc := range pipeDeadlinesTestCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + r, w := tc.create(t) + defer r.Close() + defer w.Close() + + max := time.NewTimer(100 * time.Millisecond) + defer max.Stop() + ch := make(chan error) + go func() { + if err := r.SetDeadline(time.Now().Add(-5 * time.Second)); err != nil { + t.Error(err) + } + if err := r.SetWriteDeadline(time.Now().Add(-5 * time.Second)); err != nil { + t.Error(err) + } + if err := r.SetReadDeadline(noDeadline); err != nil { + t.Error(err) + } + var b [1]byte + _, err := r.Read(b[:]) + ch <- err + }() + + select { + case err := <-ch: + t.Fatalf("expected Read to not return, but it returned with %v", err) + case <-max.C: + w.Close() + err := <-ch // wait for tester goroutine to stop + if os.IsTimeout(err) { + t.Fatal(err) + } + } + }) } } @@ -156,35 +167,38 @@ var writeTimeoutTests = []struct { func TestWriteTimeout(t *testing.T) { t.Parallel() - for i, tt := range writeTimeoutTests { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - defer r.Close() - defer w.Close() + for _, tc := range pipeDeadlinesTestCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() - if err := w.SetWriteDeadline(time.Now().Add(tt.timeout)); err != nil { - t.Fatalf("%v", err) - } - for j, xerr := range tt.xerrs { - for { - n, err := w.Write([]byte("WRITE TIMEOUT TEST")) - if xerr != nil { - if !isDeadlineExceeded(err) { - t.Fatalf("%d: %v", j, err) - } - } - if err == nil { - time.Sleep(tt.timeout / 3) - continue + for i, tt := range writeTimeoutTests { + t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { + r, w := tc.create(t) + defer r.Close() + defer w.Close() + + if err := w.SetWriteDeadline(time.Now().Add(tt.timeout)); err != nil { + t.Fatalf("%v", err) } - if n != 0 { - t.Fatalf("%d: wrote %d; want 0", j, n) + for j, xerr := range tt.xerrs { + for { + n, err := w.Write([]byte("WRITE TIMEOUT TEST")) + if xerr != nil { + if !isDeadlineExceeded(err) { + t.Fatalf("%d: %v", j, err) + } + } + if err == nil { + time.Sleep(tt.timeout / 3) + continue + } + if n != 0 { + t.Fatalf("%d: wrote %d; want 0", j, n) + } + break + } } - break - } + }) } }) } @@ -194,44 +208,47 @@ func TestWriteTimeout(t *testing.T) { func TestWriteTimeoutMustNotReturn(t *testing.T) { t.Parallel() - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - defer r.Close() - defer w.Close() - - max := time.NewTimer(100 * time.Millisecond) - defer max.Stop() - ch := make(chan error) - go func() { - if err := w.SetDeadline(time.Now().Add(-5 * time.Second)); err != nil { - t.Error(err) - } - if err := w.SetReadDeadline(time.Now().Add(-5 * time.Second)); err != nil { - t.Error(err) - } - if err := w.SetWriteDeadline(noDeadline); err != nil { - t.Error(err) - } - var b [1]byte - for { - if _, err := w.Write(b[:]); err != nil { - ch <- err - break + for _, tc := range pipeDeadlinesTestCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + r, w := tc.create(t) + defer r.Close() + defer w.Close() + + max := time.NewTimer(100 * time.Millisecond) + defer max.Stop() + ch := make(chan error) + go func() { + if err := w.SetDeadline(time.Now().Add(-5 * time.Second)); err != nil { + t.Error(err) + } + if err := w.SetReadDeadline(time.Now().Add(-5 * time.Second)); err != nil { + t.Error(err) + } + if err := w.SetWriteDeadline(noDeadline); err != nil { + t.Error(err) + } + var b [1]byte + for { + if _, err := w.Write(b[:]); err != nil { + ch <- err + break + } + } + }() + + select { + case err := <-ch: + t.Fatalf("expected Write to not return, but it returned with %v", err) + case <-max.C: + r.Close() + err := <-ch // wait for tester goroutine to stop + if os.IsTimeout(err) { + t.Fatal(err) + } } - } - }() - - select { - case err := <-ch: - t.Fatalf("expected Write to not return, but it returned with %v", err) - case <-max.C: - r.Close() - err := <-ch // wait for tester goroutine to stop - if os.IsTimeout(err) { - t.Fatal(err) - } + }) } } @@ -282,60 +299,60 @@ func nextTimeout(actual time.Duration) (next time.Duration, ok bool) { // duration by any significant margin. Try the next attempt with an arbitrary // factor above that, so that our growth curve is at least exponential. next = actual * 5 / 4 - if next > maxDynamicTimeout { - return maxDynamicTimeout, true - } - return next, true + return min(next, maxDynamicTimeout), true } // There is a very similar copy of this in net/timeout_test.go. func TestReadTimeoutFluctuation(t *testing.T) { t.Parallel() - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - defer r.Close() - defer w.Close() + for _, tc := range pipeDeadlinesTestCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() - d := minDynamicTimeout - b := make([]byte, 256) - for { - t.Logf("SetReadDeadline(+%v)", d) - t0 := time.Now() - deadline := t0.Add(d) - if err = r.SetReadDeadline(deadline); err != nil { - t.Fatalf("SetReadDeadline(%v): %v", deadline, err) - } - var n int - n, err = r.Read(b) - t1 := time.Now() + r, w := tc.create(t) + defer r.Close() + defer w.Close() - if n != 0 || err == nil || !isDeadlineExceeded(err) { - t.Errorf("Read did not return (0, timeout): (%d, %v)", n, err) - } + d := minDynamicTimeout + b := make([]byte, 256) + for { + t.Logf("SetReadDeadline(+%v)", d) + t0 := time.Now() + deadline := t0.Add(d) + if err := r.SetReadDeadline(deadline); err != nil { + t.Fatalf("SetReadDeadline(%v): %v", deadline, err) + } + var n int + n, err := r.Read(b) + t1 := time.Now() - actual := t1.Sub(t0) - if t1.Before(deadline) { - t.Errorf("Read took %s; expected at least %s", actual, d) - } - if t.Failed() { - return - } - if want := timeoutUpperBound(d); actual > want { - next, ok := nextTimeout(actual) - if !ok { - t.Fatalf("Read took %s; expected at most %v", actual, want) - } - // Maybe this machine is too slow to reliably schedule goroutines within - // the requested duration. Increase the timeout and try again. - t.Logf("Read took %s (expected %s); trying with longer timeout", actual, d) - d = next - continue - } + if n != 0 || err == nil || !isDeadlineExceeded(err) { + t.Errorf("Read did not return (0, timeout): (%d, %v)", n, err) + } + + actual := t1.Sub(t0) + if t1.Before(deadline) { + t.Errorf("Read took %s; expected at least %s", actual, d) + } + if t.Failed() { + return + } + if want := timeoutUpperBound(d); actual > want { + next, ok := nextTimeout(actual) + if !ok { + t.Fatalf("Read took %s; expected at most %v", actual, want) + } + // Maybe this machine is too slow to reliably schedule goroutines within + // the requested duration. Increase the timeout and try again. + t.Logf("Read took %s (expected %s); trying with longer timeout", actual, d) + d = next + continue + } - break + break + } + }) } } @@ -343,76 +360,84 @@ func TestReadTimeoutFluctuation(t *testing.T) { func TestWriteTimeoutFluctuation(t *testing.T) { t.Parallel() - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - defer r.Close() - defer w.Close() + for _, tc := range pipeDeadlinesTestCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() - d := minDynamicTimeout - for { - t.Logf("SetWriteDeadline(+%v)", d) - t0 := time.Now() - deadline := t0.Add(d) - if err := w.SetWriteDeadline(deadline); err != nil { - t.Fatalf("SetWriteDeadline(%v): %v", deadline, err) - } - var n int64 - var err error - for { - var dn int - dn, err = w.Write([]byte("TIMEOUT TRANSMITTER")) - n += int64(dn) - if err != nil { - break - } - } - t1 := time.Now() - // Inv: err != nil - if !isDeadlineExceeded(err) { - t.Fatalf("Write did not return (any, timeout): (%d, %v)", n, err) - } + r, w := tc.create(t) + defer r.Close() + defer w.Close() - actual := t1.Sub(t0) - if t1.Before(deadline) { - t.Errorf("Write took %s; expected at least %s", actual, d) - } - if t.Failed() { - return - } - if want := timeoutUpperBound(d); actual > want { - if n > 0 { - // SetWriteDeadline specifies a time “after which I/O operations fail - // instead of blocking”. However, the kernel's send buffer is not yet - // full, we may be able to write some arbitrary (but finite) number of - // bytes to it without blocking. - t.Logf("Wrote %d bytes into send buffer; retrying until buffer is full", n) - if d <= maxDynamicTimeout/2 { - // We don't know how long the actual write loop would have taken if - // the buffer were full, so just guess and double the duration so that - // the next attempt can make twice as much progress toward filling it. - d *= 2 + d := minDynamicTimeout + for { + t.Logf("SetWriteDeadline(+%v)", d) + t0 := time.Now() + deadline := t0.Add(d) + if err := w.SetWriteDeadline(deadline); err != nil { + t.Fatalf("SetWriteDeadline(%v): %v", deadline, err) + } + var n int64 + var err error + for { + var dn int + dn, err = w.Write([]byte("TIMEOUT TRANSMITTER")) + n += int64(dn) + if err != nil { + break + } + } + t1 := time.Now() + // Inv: err != nil + if !isDeadlineExceeded(err) { + t.Fatalf("Write did not return (any, timeout): (%d, %v)", n, err) + } + + actual := t1.Sub(t0) + if t1.Before(deadline) { + t.Errorf("Write took %s; expected at least %s", actual, d) + } + if t.Failed() { + return + } + if want := timeoutUpperBound(d); actual > want { + if n > 0 { + // SetWriteDeadline specifies a time “after which I/O operations fail + // instead of blocking”. However, the kernel's send buffer is not yet + // full, we may be able to write some arbitrary (but finite) number of + // bytes to it without blocking. + t.Logf("Wrote %d bytes into send buffer; retrying until buffer is full", n) + if d <= maxDynamicTimeout/2 { + // We don't know how long the actual write loop would have taken if + // the buffer were full, so just guess and double the duration so that + // the next attempt can make twice as much progress toward filling it. + d *= 2 + } + } else if next, ok := nextTimeout(actual); !ok { + t.Fatalf("Write took %s; expected at most %s", actual, want) + } else { + // Maybe this machine is too slow to reliably schedule goroutines within + // the requested duration. Increase the timeout and try again. + t.Logf("Write took %s (expected %s); trying with longer timeout", actual, d) + d = next + } + continue } - } else if next, ok := nextTimeout(actual); !ok { - t.Fatalf("Write took %s; expected at most %s", actual, want) - } else { - // Maybe this machine is too slow to reliably schedule goroutines within - // the requested duration. Increase the timeout and try again. - t.Logf("Write took %s (expected %s); trying with longer timeout", actual, d) - d = next - } - continue - } - break + break + } + }) } } // There is a very similar copy of this in net/timeout_test.go. func TestVariousDeadlines(t *testing.T) { t.Parallel() - testVariousDeadlines(t) + for _, tc := range pipeDeadlinesTestCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + testVariousDeadlines(t, tc.create) + }) + } } // There is a very similar copy of this in net/timeout_test.go. @@ -422,7 +447,12 @@ func TestVariousDeadlines1Proc(t *testing.T) { t.Skip("skipping in short mode") } defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1)) - testVariousDeadlines(t) + for _, tc := range pipeDeadlinesTestCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + testVariousDeadlines(t, tc.create) + }) + } } // There is a very similar copy of this in net/timeout_test.go. @@ -432,7 +462,12 @@ func TestVariousDeadlines4Proc(t *testing.T) { t.Skip("skipping in short mode") } defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(4)) - testVariousDeadlines(t) + for _, tc := range pipeDeadlinesTestCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + testVariousDeadlines(t, tc.create) + }) + } } type neverEnding byte @@ -444,7 +479,7 @@ func (b neverEnding) Read(p []byte) (int, error) { return len(p), nil } -func testVariousDeadlines(t *testing.T) { +func testVariousDeadlines(t *testing.T, create func(t *testing.T) (r, w *os.File)) { type result struct { n int64 err error @@ -490,10 +525,7 @@ func testVariousDeadlines(t *testing.T) { } for run := 0; run < numRuns; run++ { t.Run(fmt.Sprintf("%v-%d", timeout, run+1), func(t *testing.T) { - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } + r, w := create(t) defer r.Close() defer w.Close() @@ -517,7 +549,7 @@ func testVariousDeadlines(t *testing.T) { select { case res := <-actvch: - if !isDeadlineExceeded(err) { + if isDeadlineExceeded(res.err) { t.Logf("good client timeout after %v, reading %d bytes", res.d, res.n) } else { t.Fatalf("client Copy = %d, %v; want timeout", res.n, res.err) @@ -546,50 +578,53 @@ func TestReadWriteDeadlineRace(t *testing.T) { N = 50 } - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) + for _, tc := range pipeDeadlinesTestCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + r, w := tc.create(t) + defer r.Close() + defer w.Close() + + var wg sync.WaitGroup + wg.Add(3) + go func() { + defer wg.Done() + tic := time.NewTicker(2 * time.Microsecond) + defer tic.Stop() + for i := 0; i < N; i++ { + if err := r.SetReadDeadline(time.Now().Add(2 * time.Microsecond)); err != nil { + break + } + if err := w.SetWriteDeadline(time.Now().Add(2 * time.Microsecond)); err != nil { + break + } + <-tic.C + } + }() + go func() { + defer wg.Done() + var b [1]byte + for i := 0; i < N; i++ { + _, err := r.Read(b[:]) + if err != nil && !isDeadlineExceeded(err) { + t.Error("Read returned non-timeout error", err) + } + } + }() + go func() { + defer wg.Done() + var b [1]byte + for i := 0; i < N; i++ { + _, err := w.Write(b[:]) + if err != nil && !isDeadlineExceeded(err) { + t.Error("Write returned non-timeout error", err) + } + } + }() + wg.Wait() // wait for tester goroutine to stop + }) } - defer r.Close() - defer w.Close() - - var wg sync.WaitGroup - wg.Add(3) - go func() { - defer wg.Done() - tic := time.NewTicker(2 * time.Microsecond) - defer tic.Stop() - for i := 0; i < N; i++ { - if err := r.SetReadDeadline(time.Now().Add(2 * time.Microsecond)); err != nil { - break - } - if err := w.SetWriteDeadline(time.Now().Add(2 * time.Microsecond)); err != nil { - break - } - <-tic.C - } - }() - go func() { - defer wg.Done() - var b [1]byte - for i := 0; i < N; i++ { - _, err := r.Read(b[:]) - if err != nil && !isDeadlineExceeded(err) { - t.Error("Read returned non-timeout error", err) - } - } - }() - go func() { - defer wg.Done() - var b [1]byte - for i := 0; i < N; i++ { - _, err := w.Write(b[:]) - if err != nil && !isDeadlineExceeded(err) { - t.Error("Write returned non-timeout error", err) - } - } - }() - wg.Wait() // wait for tester goroutine to stop } // TestRacyRead tests that it is safe to mutate the input Read buffer @@ -597,37 +632,40 @@ func TestReadWriteDeadlineRace(t *testing.T) { func TestRacyRead(t *testing.T) { t.Parallel() - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - defer r.Close() - defer w.Close() - - var wg sync.WaitGroup - defer wg.Wait() - - go io.Copy(w, rand.New(rand.NewSource(0))) - - r.SetReadDeadline(time.Now().Add(time.Millisecond)) - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - - b1 := make([]byte, 1024) - b2 := make([]byte, 1024) - for j := 0; j < 100; j++ { - _, err := r.Read(b1) - copy(b1, b2) // Mutate b1 to trigger potential race - if err != nil { - if !isDeadlineExceeded(err) { - t.Error(err) + for _, tc := range pipeDeadlinesTestCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + r, w := tc.create(t) + defer r.Close() + defer w.Close() + + var wg sync.WaitGroup + defer wg.Wait() + + go io.Copy(w, rand.New(rand.NewSource(0))) + + r.SetReadDeadline(time.Now().Add(time.Millisecond)) + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + + b1 := make([]byte, 1024) + b2 := make([]byte, 1024) + for j := 0; j < 100; j++ { + _, err := r.Read(b1) + copy(b1, b2) // Mutate b1 to trigger potential race + if err != nil { + if !isDeadlineExceeded(err) { + t.Error(err) + } + r.SetReadDeadline(time.Now().Add(time.Millisecond)) + } } - r.SetReadDeadline(time.Now().Add(time.Millisecond)) - } + }() } - }() + }) } } @@ -636,73 +674,39 @@ func TestRacyRead(t *testing.T) { func TestRacyWrite(t *testing.T) { t.Parallel() - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - defer r.Close() - defer w.Close() - - var wg sync.WaitGroup - defer wg.Wait() - - go io.Copy(io.Discard, r) - - w.SetWriteDeadline(time.Now().Add(time.Millisecond)) - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - - b1 := make([]byte, 1024) - b2 := make([]byte, 1024) - for j := 0; j < 100; j++ { - _, err := w.Write(b1) - copy(b1, b2) // Mutate b1 to trigger potential race - if err != nil { - if !isDeadlineExceeded(err) { - t.Error(err) - } - w.SetWriteDeadline(time.Now().Add(time.Millisecond)) - } - } - }() - } -} + for _, tc := range pipeDeadlinesTestCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() -// Closing a TTY while reading from it should not hang. Issue 23943. -func TestTTYClose(t *testing.T) { - // Ignore SIGTTIN in case we are running in the background. - signal.Ignore(syscall.SIGTTIN) - defer signal.Reset(syscall.SIGTTIN) + r, w := tc.create(t) + defer r.Close() + defer w.Close() - f, err := os.Open("/dev/tty") - if err != nil { - t.Skipf("skipping because opening /dev/tty failed: %v", err) - } + var wg sync.WaitGroup + defer wg.Wait() - go func() { - var buf [1]byte - f.Read(buf[:]) - }() - - // Give the goroutine a chance to enter the read. - // It doesn't matter much if it occasionally fails to do so, - // we won't be testing what we want to test but the test will pass. - time.Sleep(time.Millisecond) - - c := make(chan bool) - go func() { - defer close(c) - f.Close() - }() - - select { - case <-c: - case <-time.After(time.Second): - t.Error("timed out waiting for close") - } + go io.Copy(io.Discard, r) - // On some systems the goroutines may now be hanging. - // There's not much we can do about that. + w.SetWriteDeadline(time.Now().Add(time.Millisecond)) + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + + b1 := make([]byte, 1024) + b2 := make([]byte, 1024) + for j := 0; j < 100; j++ { + _, err := w.Write(b1) + copy(b1, b2) // Mutate b1 to trigger potential race + if err != nil { + if !isDeadlineExceeded(err) { + t.Error(err) + } + w.SetWriteDeadline(time.Now().Add(time.Millisecond)) + } + } + }() + } + }) + } } diff --git a/src/os/timeout_unix_test.go b/src/os/timeout_unix_test.go new file mode 100644 index 00000000000000..bc8fd4a54f4b03 --- /dev/null +++ b/src/os/timeout_unix_test.go @@ -0,0 +1,65 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !js && !plan9 && !wasip1 && !windows + +package os_test + +import ( + "os" + "os/signal" + "syscall" + "testing" + "time" +) + +func init() { + pipeDeadlinesTestCases = []pipeDeadlineTest{{ + "anonymous pipe", + func(t *testing.T) (r, w *os.File) { + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + return r, w + }, + }} +} + +// Closing a TTY while reading from it should not hang. Issue 23943. +func TestTTYClose(t *testing.T) { + // Ignore SIGTTIN in case we are running in the background. + signal.Ignore(syscall.SIGTTIN) + defer signal.Reset(syscall.SIGTTIN) + + f, err := os.Open("/dev/tty") + if err != nil { + t.Skipf("skipping because opening /dev/tty failed: %v", err) + } + + go func() { + var buf [1]byte + f.Read(buf[:]) + }() + + // Give the goroutine a chance to enter the read. + // It doesn't matter much if it occasionally fails to do so, + // we won't be testing what we want to test but the test will pass. + time.Sleep(time.Millisecond) + + c := make(chan bool) + go func() { + defer close(c) + f.Close() + }() + + select { + case <-c: + case <-time.After(time.Second): + t.Error("timed out waiting for close") + } + + // On some systems the goroutines may now be hanging. + // There's not much we can do about that. +} diff --git a/src/os/timeout_windows_test.go b/src/os/timeout_windows_test.go new file mode 100644 index 00000000000000..ea1f5e5c9872bc --- /dev/null +++ b/src/os/timeout_windows_test.go @@ -0,0 +1,24 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os_test + +import ( + "os" + "testing" +) + +func init() { + pipeDeadlinesTestCases = []pipeDeadlineTest{ + { + "named overlapped pipe", + func(t *testing.T) (r, w *os.File) { + name := pipeName() + w = newBytePipe(t, name, true) + r = newFileOverlapped(t, name, true) + return + }, + }, + } +} diff --git a/src/os/user/cgo_listgroups_unix.go b/src/os/user/cgo_listgroups_unix.go index 59636954b2ab87..bbf1886b3c0277 100644 --- a/src/os/user/cgo_listgroups_unix.go +++ b/src/os/user/cgo_listgroups_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (cgo || darwin) && !osusergo && (darwin || dragonfly || freebsd || (linux && !android) || netbsd || openbsd || (solaris && !illumos)) +//go:build (cgo || darwin) && !osusergo && unix && !android && !aix package user diff --git a/src/os/user/cgo_lookup_unix.go b/src/os/user/cgo_lookup_unix.go index 458d8cd453ac3a..1c9a289672133a 100644 --- a/src/os/user/cgo_lookup_unix.go +++ b/src/os/user/cgo_lookup_unix.go @@ -37,7 +37,7 @@ func lookupUser(username string) (*User, error) { if err != nil { return nil, fmt.Errorf("user: lookup username %s: %v", username, err) } - return buildUser(&pwd), err + return buildUser(&pwd), nil } func lookupUserId(uid string) (*User, error) { diff --git a/src/os/user/cgo_lookup_unix_test.go b/src/os/user/cgo_lookup_unix_test.go new file mode 100644 index 00000000000000..cd225eb8df4f74 --- /dev/null +++ b/src/os/user/cgo_lookup_unix_test.go @@ -0,0 +1,23 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (cgo || darwin) && !osusergo && unix && !android + +package user + +import ( + "testing" +) + +// Issue 22739 +func TestNegativeUid(t *testing.T) { + sp := structPasswdForNegativeTest() + u := buildUser(&sp) + if g, w := u.Uid, "4294967294"; g != w { + t.Errorf("Uid = %q; want %q", g, w) + } + if g, w := u.Gid, "4294967293"; g != w { + t.Errorf("Gid = %q; want %q", g, w) + } +} diff --git a/src/os/user/cgo_unix_test.go b/src/os/user/cgo_unix_test.go deleted file mode 100644 index 6d16aa20b30ec8..00000000000000 --- a/src/os/user/cgo_unix_test.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build (darwin || dragonfly || freebsd || (!android && linux) || netbsd || openbsd || solaris) && cgo && !osusergo - -package user - -import ( - "testing" -) - -// Issue 22739 -func TestNegativeUid(t *testing.T) { - sp := structPasswdForNegativeTest() - u := buildUser(&sp) - if g, w := u.Uid, "4294967294"; g != w { - t.Errorf("Uid = %q; want %q", g, w) - } - if g, w := u.Gid, "4294967293"; g != w { - t.Errorf("Gid = %q; want %q", g, w) - } -} diff --git a/src/os/user/getgrouplist_unix.go b/src/os/user/getgrouplist_unix.go index fb482d35baee0c..9b033b89815e31 100644 --- a/src/os/user/getgrouplist_unix.go +++ b/src/os/user/getgrouplist_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build cgo && !osusergo && (dragonfly || freebsd || (!android && linux) || netbsd || openbsd || (solaris && !illumos)) +//go:build cgo && !osusergo && unix && !android && !aix && !darwin package user diff --git a/src/os/user/listgroups_unix.go b/src/os/user/listgroups_unix.go index 67bd8a776e2bbe..f370ec19d7d8d3 100644 --- a/src/os/user/listgroups_unix.go +++ b/src/os/user/listgroups_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build ((darwin || dragonfly || freebsd || (js && wasm) || wasip1 || (!android && linux) || netbsd || openbsd || solaris) && ((!cgo && !darwin) || osusergo)) || aix || illumos +//go:build (((unix && !android) || (js && wasm) || wasip1) && ((!cgo && !darwin) || osusergo)) || aix package user diff --git a/src/os/user/listgroups_unix_test.go b/src/os/user/listgroups_unix_test.go index 9619462cec983a..2dbfa225e8d161 100644 --- a/src/os/user/listgroups_unix_test.go +++ b/src/os/user/listgroups_unix_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build ((darwin || dragonfly || freebsd || (js && wasm) || wasip1 || (!android && linux) || netbsd || openbsd || solaris) && ((!cgo && !darwin) || osusergo)) || aix || illumos +//go:build (((unix && !android) || (js && wasm) || wasip1) && ((!cgo && !darwin) || osusergo)) || aix package user diff --git a/src/os/user/lookup_windows.go b/src/os/user/lookup_windows.go index a48fc89720b1cb..e0e77f3ea75a55 100644 --- a/src/os/user/lookup_windows.go +++ b/src/os/user/lookup_windows.go @@ -5,9 +5,11 @@ package user import ( + "errors" "fmt" "internal/syscall/windows" "internal/syscall/windows/registry" + "runtime" "syscall" "unsafe" ) @@ -84,16 +86,73 @@ func getProfilesDirectory() (string, error) { } } +func isServiceAccount(sid *syscall.SID) bool { + if !windows.IsValidSid(sid) { + // We don't accept SIDs from the public API, so this should never happen. + // Better be on the safe side and validate anyway. + return false + } + // The following RIDs are considered service user accounts as per + // https://learn.microsoft.com/en-us/windows/win32/secauthz/well-known-sids and + // https://learn.microsoft.com/en-us/windows/win32/services/service-user-accounts: + // - "S-1-5-18": LocalSystem + // - "S-1-5-19": LocalService + // - "S-1-5-20": NetworkService + if windows.GetSidSubAuthorityCount(sid) != windows.SID_REVISION || + windows.GetSidIdentifierAuthority(sid) != windows.SECURITY_NT_AUTHORITY { + return false + } + switch windows.GetSidSubAuthority(sid, 0) { + case windows.SECURITY_LOCAL_SYSTEM_RID, + windows.SECURITY_LOCAL_SERVICE_RID, + windows.SECURITY_NETWORK_SERVICE_RID: + return true + } + return false +} + +func isValidUserAccountType(sid *syscall.SID, sidType uint32) bool { + switch sidType { + case syscall.SidTypeUser: + return true + case syscall.SidTypeWellKnownGroup: + return isServiceAccount(sid) + } + return false +} + +func isValidGroupAccountType(sidType uint32) bool { + switch sidType { + case syscall.SidTypeGroup: + return true + case syscall.SidTypeWellKnownGroup: + // Some well-known groups are also considered service accounts, + // so isValidUserAccountType would return true for them. + // We have historically allowed them in LookupGroup and LookupGroupId, + // so don't treat them as invalid here. + return true + case syscall.SidTypeAlias: + // https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-samr/7b2aeb27-92fc-41f6-8437-deb65d950921#gt_0387e636-5654-4910-9519-1f8326cf5ec0 + // SidTypeAlias should also be treated as a group type next to SidTypeGroup + // and SidTypeWellKnownGroup: + // "alias object -> resource group: A group object..." + // + // Tests show that "Administrators" can be considered of type SidTypeAlias. + return true + } + return false +} + // lookupUsernameAndDomain obtains the username and domain for usid. -func lookupUsernameAndDomain(usid *syscall.SID) (username, domain string, e error) { - username, domain, t, e := usid.LookupAccount("") +func lookupUsernameAndDomain(usid *syscall.SID) (username, domain string, sidType uint32, e error) { + username, domain, sidType, e = usid.LookupAccount("") if e != nil { - return "", "", e + return "", "", 0, e } - if t != syscall.SidTypeUser { - return "", "", fmt.Errorf("user: should be user account type, not %d", t) + if !isValidUserAccountType(usid, sidType) { + return "", "", 0, fmt.Errorf("user: should be user account type, not %d", sidType) } - return username, domain, nil + return username, domain, sidType, nil } // findHomeDirInRegistry finds the user home path based on the uid. @@ -116,13 +175,7 @@ func lookupGroupName(groupname string) (string, error) { if e != nil { return "", e } - // https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-samr/7b2aeb27-92fc-41f6-8437-deb65d950921#gt_0387e636-5654-4910-9519-1f8326cf5ec0 - // SidTypeAlias should also be treated as a group type next to SidTypeGroup - // and SidTypeWellKnownGroup: - // "alias object -> resource group: A group object..." - // - // Tests show that "Administrators" can be considered of type SidTypeAlias. - if t != syscall.SidTypeGroup && t != syscall.SidTypeWellKnownGroup && t != syscall.SidTypeAlias { + if !isValidGroupAccountType(t) { return "", fmt.Errorf("lookupGroupName: should be group account type, not %d", t) } return sid.String() @@ -149,17 +202,13 @@ func listGroupsForUsernameAndDomain(username, domain string) ([]string, error) { // NetUserGetLocalGroups() would return a list of LocalGroupUserInfo0 // elements which hold the names of local groups where the user participates. // The list does not follow any sorting order. - // - // If no groups can be found for this user, NetUserGetLocalGroups() should - // always return the SID of a single group called "None", which - // also happens to be the primary group for the local user. err = windows.NetUserGetLocalGroups(nil, q, 0, windows.LG_INCLUDE_INDIRECT, &p0, windows.MAX_PREFERRED_LENGTH, &entriesRead, &totalEntries) if err != nil { return nil, err } defer syscall.NetApiBufferFree(p0) if entriesRead == 0 { - return nil, fmt.Errorf("listGroupsForUsernameAndDomain: NetUserGetLocalGroups() returned an empty list for domain: %s, username: %s", domain, username) + return nil, nil } entries := (*[1024]windows.LocalGroupUserInfo0)(unsafe.Pointer(p0))[:entriesRead:entriesRead] var sids []string @@ -200,36 +249,103 @@ var ( ) func current() (*User, error) { - t, e := syscall.OpenCurrentProcessToken() - if e != nil { - return nil, e - } - defer t.Close() - u, e := t.GetTokenUser() - if e != nil { - return nil, e - } - pg, e := t.GetTokenPrimaryGroup() - if e != nil { - return nil, e - } - uid, e := u.User.Sid.String() - if e != nil { - return nil, e - } - gid, e := pg.PrimaryGroup.String() - if e != nil { - return nil, e - } - dir, e := t.GetUserProfileDirectory() - if e != nil { - return nil, e + // Use runAsProcessOwner to ensure that we can access the process token + // when calling syscall.OpenCurrentProcessToken if the current thread + // is impersonating a different user. See https://go.dev/issue/68647. + var usr *User + err := runAsProcessOwner(func() error { + t, e := syscall.OpenCurrentProcessToken() + if e != nil { + return e + } + defer t.Close() + u, e := t.GetTokenUser() + if e != nil { + return e + } + pg, e := t.GetTokenPrimaryGroup() + if e != nil { + return e + } + uid, e := u.User.Sid.String() + if e != nil { + return e + } + gid, e := pg.PrimaryGroup.String() + if e != nil { + return e + } + dir, e := t.GetUserProfileDirectory() + if e != nil { + return e + } + username, e := windows.GetUserName(syscall.NameSamCompatible) + if e != nil { + return e + } + displayName, e := windows.GetUserName(syscall.NameDisplay) + if e != nil { + // Historically, the username is used as fallback + // when the display name can't be retrieved. + displayName = username + } + usr = &User{ + Uid: uid, + Gid: gid, + Username: username, + Name: displayName, + HomeDir: dir, + } + return nil + }) + return usr, err +} + +// runAsProcessOwner runs f in the context of the current process owner, +// that is, removing any impersonation that may be in effect before calling f, +// and restoring the impersonation afterwards. +func runAsProcessOwner(f func() error) error { + var impersonationRollbackErr error + runtime.LockOSThread() + defer func() { + // If impersonation failed, the thread is running with the wrong token, + // so it's better to terminate it. + // This is achieved by not calling runtime.UnlockOSThread. + if impersonationRollbackErr != nil { + println("os/user: failed to revert to previous token:", impersonationRollbackErr.Error()) + runtime.Goexit() + } else { + runtime.UnlockOSThread() + } + }() + prevToken, isProcessToken, err := getCurrentToken() + if err != nil { + return fmt.Errorf("os/user: failed to get current token: %w", err) } - username, domain, e := lookupUsernameAndDomain(u.User.Sid) - if e != nil { - return nil, e + defer prevToken.Close() + if !isProcessToken { + if err = windows.RevertToSelf(); err != nil { + return fmt.Errorf("os/user: failed to revert to self: %w", err) + } + defer func() { + impersonationRollbackErr = windows.ImpersonateLoggedOnUser(prevToken) + }() } - return newUser(uid, gid, dir, username, domain) + return f() +} + +// getCurrentToken returns the current thread token, or +// the process token if the thread doesn't have a token. +func getCurrentToken() (t syscall.Token, isProcessToken bool, err error) { + thread, _ := windows.GetCurrentThread() + // Need TOKEN_DUPLICATE and TOKEN_IMPERSONATE to use the token in ImpersonateLoggedOnUser. + err = windows.OpenThreadToken(thread, syscall.TOKEN_QUERY|syscall.TOKEN_DUPLICATE|syscall.TOKEN_IMPERSONATE, true, &t) + if errors.Is(err, windows.ERROR_NO_TOKEN) { + // Not impersonating, use the process token. + isProcessToken = true + t, err = syscall.OpenCurrentProcessToken() + } + return t, isProcessToken, err } // lookupUserPrimaryGroup obtains the primary group SID for a user using this method: @@ -290,11 +406,7 @@ func lookupUserPrimaryGroup(username, domain string) (string, error) { } func newUserFromSid(usid *syscall.SID) (*User, error) { - username, domain, e := lookupUsernameAndDomain(usid) - if e != nil { - return nil, e - } - gid, e := lookupUserPrimaryGroup(username, domain) + username, domain, sidType, e := lookupUsernameAndDomain(usid) if e != nil { return nil, e } @@ -302,6 +414,19 @@ func newUserFromSid(usid *syscall.SID) (*User, error) { if e != nil { return nil, e } + var gid string + if sidType == syscall.SidTypeWellKnownGroup { + // The SID does not contain a domain; this function's domain variable has + // been populated with the SID's identifier authority. This happens with + // special service user accounts such as "NT AUTHORITY\LocalSystem". + // In this case, gid is the same as the user SID. + gid = uid + } else { + gid, e = lookupUserPrimaryGroup(username, domain) + if e != nil { + return nil, e + } + } // If this user has logged in at least once their home path should be stored // in the registry under the specified SID. References: // https://social.technet.microsoft.com/wiki/contents/articles/13895.how-to-remove-a-corrupted-user-profile-from-the-registry.aspx @@ -331,7 +456,7 @@ func lookupUser(username string) (*User, error) { if e != nil { return nil, e } - if t != syscall.SidTypeUser { + if !isValidUserAccountType(sid, t) { return nil, fmt.Errorf("user: should be user account type, not %d", t) } return newUserFromSid(sid) @@ -362,24 +487,52 @@ func lookupGroupId(gid string) (*Group, error) { if err != nil { return nil, err } - if t != syscall.SidTypeGroup && t != syscall.SidTypeWellKnownGroup && t != syscall.SidTypeAlias { + if !isValidGroupAccountType(t) { return nil, fmt.Errorf("lookupGroupId: should be group account type, not %d", t) } return &Group{Name: groupname, Gid: gid}, nil } func listGroups(user *User) ([]string, error) { - sid, err := syscall.StringToSid(user.Uid) - if err != nil { - return nil, err - } - username, domain, err := lookupUsernameAndDomain(sid) - if err != nil { - return nil, err - } - sids, err := listGroupsForUsernameAndDomain(username, domain) - if err != nil { - return nil, err + var sids []string + if u, err := Current(); err == nil && u.Uid == user.Uid { + // It is faster and more reliable to get the groups + // of the current user from the current process token. + err := runAsProcessOwner(func() error { + t, err := syscall.OpenCurrentProcessToken() + if err != nil { + return err + } + defer t.Close() + groups, err := windows.GetTokenGroups(t) + if err != nil { + return err + } + for _, g := range groups.AllGroups() { + sid, err := g.Sid.String() + if err != nil { + return err + } + sids = append(sids, sid) + } + return nil + }) + if err != nil { + return nil, err + } + } else { + sid, err := syscall.StringToSid(user.Uid) + if err != nil { + return nil, err + } + username, domain, _, err := lookupUsernameAndDomain(sid) + if err != nil { + return nil, err + } + sids, err = listGroupsForUsernameAndDomain(username, domain) + if err != nil { + return nil, err + } } // Add the primary group of the user to the list if it is not already there. // This is done only to comply with the POSIX concept of a primary group. diff --git a/src/os/user/user_test.go b/src/os/user/user_test.go index fa597b78ece7c8..0e06369bf5a5da 100644 --- a/src/os/user/user_test.go +++ b/src/os/user/user_test.go @@ -6,6 +6,7 @@ package user import ( "os" + "slices" "testing" ) @@ -45,8 +46,9 @@ func TestCurrent(t *testing.T) { } func BenchmarkCurrent(b *testing.B) { + // Benchmark current instead of Current because Current caches the result. for i := 0; i < b.N; i++ { - Current() + current() } } @@ -177,16 +179,7 @@ func TestGroupIds(t *testing.T) { if err != nil { t.Fatalf("%+v.GroupIds(): %v", user, err) } - if !containsID(gids, user.Gid) { + if !slices.Contains(gids, user.Gid) { t.Errorf("%+v.GroupIds() = %v; does not contain user GID %s", user, gids, user.Gid) } } - -func containsID(ids []string, id string) bool { - for _, x := range ids { - if x == id { - return true - } - } - return false -} diff --git a/src/os/user/user_windows_test.go b/src/os/user/user_windows_test.go new file mode 100644 index 00000000000000..c95811594d1050 --- /dev/null +++ b/src/os/user/user_windows_test.go @@ -0,0 +1,360 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package user + +import ( + "crypto/rand" + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "internal/syscall/windows" + "internal/testenv" + "os" + "os/exec" + "runtime" + "slices" + "strconv" + "strings" + "sync" + "syscall" + "testing" + "unicode" + "unicode/utf8" + "unsafe" +) + +// addUserAccount creates a local user account. +// It returns the name and password of the new account. +// Multiple programs or goroutines calling addUserAccount simultaneously will not choose the same directory. +func addUserAccount(t *testing.T) (name, password string) { + t.TempDir() + pattern := t.Name() + // Windows limits the user name to 20 characters, + // leave space for a 4 digits random suffix. + const maxNameLen, suffixLen = 20, 4 + pattern = pattern[:min(len(pattern), maxNameLen-suffixLen)] + // Drop unusual characters from the account name. + mapper := func(r rune) rune { + if r < utf8.RuneSelf { + if '0' <= r && r <= '9' || + 'a' <= r && r <= 'z' || + 'A' <= r && r <= 'Z' { + return r + } + } else if unicode.IsLetter(r) || unicode.IsNumber(r) { + return r + } + return -1 + } + pattern = strings.Map(mapper, pattern) + + // Generate a long random password. + var pwd [33]byte + rand.Read(pwd[:]) + // Add special chars to ensure it satisfies password requirements. + password = base64.StdEncoding.EncodeToString(pwd[:]) + "_-As@!%*(1)4#2" + password16, err := syscall.UTF16PtrFromString(password) + if err != nil { + t.Fatal(err) + } + + try := 0 + for { + // Calculate a random suffix to append to the user name. + var suffix [2]byte + rand.Read(suffix[:]) + suffixStr := strconv.FormatUint(uint64(binary.LittleEndian.Uint16(suffix[:])), 10) + name := pattern + suffixStr[:min(len(suffixStr), suffixLen)] + name16, err := syscall.UTF16PtrFromString(name) + if err != nil { + t.Fatal(err) + } + // Create user. + userInfo := windows.UserInfo1{ + Name: name16, + Password: password16, + Priv: windows.USER_PRIV_USER, + } + err = windows.NetUserAdd(nil, 1, (*byte)(unsafe.Pointer(&userInfo)), nil) + if errors.Is(err, syscall.ERROR_ACCESS_DENIED) { + t.Skip("skipping test; don't have permission to create user") + } + // If the user already exists, try again with a different name. + if errors.Is(err, windows.NERR_UserExists) { + if try++; try < 1000 { + t.Log("user already exists, trying again with a different name") + continue + } + } + if err != nil { + t.Fatalf("NetUserAdd failed: %v", err) + } + // Delete the user when the test is done. + t.Cleanup(func() { + if err := windows.NetUserDel(nil, name16); err != nil { + if !errors.Is(err, windows.NERR_UserNotFound) { + t.Fatal(err) + } + } + }) + return name, password + } +} + +// windowsTestAccount creates a test user and returns a token for that user. +// If the user already exists, it will be deleted and recreated. +// The caller is responsible for closing the token. +func windowsTestAccount(t *testing.T) (syscall.Token, *User) { + if testenv.Builder() == "" { + // Adding and deleting users requires special permissions. + // Even if we have them, we don't want to create users on + // on dev machines, as they may not be cleaned up. + // See https://dev.go/issue/70396. + t.Skip("skipping non-hermetic test outside of Go builders") + } + name, password := addUserAccount(t) + name16, err := syscall.UTF16PtrFromString(name) + if err != nil { + t.Fatal(err) + } + pwd16, err := syscall.UTF16PtrFromString(password) + if err != nil { + t.Fatal(err) + } + domain, err := syscall.UTF16PtrFromString(".") + if err != nil { + t.Fatal(err) + } + const LOGON32_PROVIDER_DEFAULT = 0 + const LOGON32_LOGON_INTERACTIVE = 2 + var token syscall.Token + if err = windows.LogonUser(name16, domain, pwd16, LOGON32_LOGON_INTERACTIVE, LOGON32_PROVIDER_DEFAULT, &token); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + token.Close() + }) + usr, err := Lookup(name) + if err != nil { + t.Fatal(err) + } + return token, usr +} + +func TestImpersonatedSelf(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + want, err := current() + if err != nil { + t.Fatal(err) + } + + levels := []uint32{ + windows.SecurityAnonymous, + windows.SecurityIdentification, + windows.SecurityImpersonation, + windows.SecurityDelegation, + } + for _, level := range levels { + t.Run(strconv.Itoa(int(level)), func(t *testing.T) { + if err = windows.ImpersonateSelf(level); err != nil { + t.Fatal(err) + } + defer windows.RevertToSelf() + + got, err := current() + if level == windows.SecurityAnonymous { + // We can't get the process token when using an anonymous token, + // so we expect an error here. + if err == nil { + t.Fatal("expected error") + } + return + } + if err != nil { + t.Fatal(err) + } + compare(t, want, got) + }) + } +} + +func TestImpersonated(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + want, err := current() + if err != nil { + t.Fatal(err) + } + + // Create a test user and log in as that user. + token, _ := windowsTestAccount(t) + + // Impersonate the test user. + if err = windows.ImpersonateLoggedOnUser(token); err != nil { + t.Fatal(err) + } + defer func() { + err = windows.RevertToSelf() + if err != nil { + // If we can't revert to self, we can't continue testing. + panic(err) + } + }() + + got, err := current() + if err != nil { + t.Fatal(err) + } + compare(t, want, got) +} + +func TestCurrentNetapi32(t *testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") == "1" { + // Test that Current does not load netapi32.dll. + // First call Current. + Current() + + // Then check if netapi32.dll is loaded. + netapi32, err := syscall.UTF16PtrFromString("netapi32.dll") + if err != nil { + fmt.Fprintf(os.Stderr, "error: %s\n", err.Error()) + os.Exit(9) + return + } + mod, _ := windows.GetModuleHandle(netapi32) + if mod != 0 { + fmt.Fprintf(os.Stderr, "netapi32.dll is loaded\n") + os.Exit(9) + return + } + os.Exit(0) + return + } + exe := testenv.Executable(t) + cmd := testenv.CleanCmdEnv(exec.Command(exe, "-test.run=^TestCurrentNetapi32$")) + cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=1") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%v\n%s", err, out) + } +} + +func TestGroupIdsTestUser(t *testing.T) { + // Create a test user and log in as that user. + _, user := windowsTestAccount(t) + + gids, err := user.GroupIds() + if err != nil { + t.Fatal(err) + } + + if err != nil { + t.Fatalf("%+v.GroupIds(): %v", user, err) + } + if !slices.Contains(gids, user.Gid) { + t.Errorf("%+v.GroupIds() = %v; does not contain user GID %s", user, gids, user.Gid) + } +} + +var isSystemDefaultLCIDEnglish = sync.OnceValue(func() bool { + // GetSystemDefaultLCID() + // https://learn.microsoft.com/en-us/windows/win32/api/winnls/nf-winnls-getsystemdefaultlcid + r, _, _ := syscall.MustLoadDLL("kernel32.dll").MustFindProc("GetSystemDefaultLCID").Call() + lcid := uint32(r) + + lcidLow := lcid & 0xFF + // 0x0409 is en-US + // 0x1000 is "Locale without assigned LCID" + return lcidLow == 0x00 || lcidLow == 0x09 +}) + +var serviceAccounts = []struct { + sid string + name string // name on english Windows +}{ + {"S-1-5-18", "NT AUTHORITY\\SYSTEM"}, + {"S-1-5-19", "NT AUTHORITY\\LOCAL SERVICE"}, + {"S-1-5-20", "NT AUTHORITY\\NETWORK SERVICE"}, +} + +func TestLookupServiceAccount(t *testing.T) { + t.Parallel() + for _, tt := range serviceAccounts { + t.Run(tt.name, func(t *testing.T) { + u, err := Lookup(tt.name) + if err != nil { + t.Logf("Lookup(%q): %v", tt.name, err) + if !isSystemDefaultLCIDEnglish() { + t.Skipf("test not supported on non-English Windows") + } + t.Fail() + return + } + if u.Uid != tt.sid { + t.Errorf("unexpected uid for %q; got %q, want %q", u.Name, u.Uid, tt.sid) + } + t.Logf("Lookup(%q): %q", tt.name, u.Username) + }) + } +} + +func TestLookupIdServiceAccount(t *testing.T) { + t.Parallel() + for _, tt := range serviceAccounts { + u, err := LookupId(tt.sid) + if err != nil { + t.Errorf("LookupId(%q): %v", tt.sid, err) + continue + } + if u.Gid != tt.sid { + t.Errorf("unexpected gid for %q; got %q, want %q", u.Name, u.Gid, tt.sid) + } + if u.Username != tt.name { + if isSystemDefaultLCIDEnglish() { + t.Errorf("unexpected user name for %q; got %q, want %q", u.Gid, u.Username, tt.name) + } else { + t.Logf("user name for %q: %q", u.Gid, u.Username) + } + } + } +} + +func TestLookupGroupServiceAccount(t *testing.T) { + t.Parallel() + for _, tt := range serviceAccounts { + t.Run(tt.name, func(t *testing.T) { + g, err := LookupGroup(tt.name) + if err != nil { + t.Logf("LookupGroup(%q): %v", tt.name, err) + if !isSystemDefaultLCIDEnglish() { + t.Skipf("test not supported on non-English Windows") + } + t.Fail() + return + } + if g.Gid != tt.sid { + t.Errorf("unexpected gid for %q; got %q, want %q", g.Name, g.Gid, tt.sid) + } + }) + } +} + +func TestLookupGroupIdServiceAccount(t *testing.T) { + t.Parallel() + for _, tt := range serviceAccounts { + u, err := LookupGroupId(tt.sid) + if err != nil { + t.Errorf("LookupGroupId(%q): %v", tt.sid, err) + continue + } + if u.Gid != tt.sid { + t.Errorf("unexpected gid for %q; got %q, want %q", u.Name, u.Gid, tt.sid) + } + } +} diff --git a/src/os/wait_wait6.go b/src/os/wait_wait6.go index 1031428826ca66..113535e5bdeb38 100644 --- a/src/os/wait_wait6.go +++ b/src/os/wait_wait6.go @@ -15,18 +15,18 @@ import ( // succeed immediately, and reports whether it has done so. // It does not actually call p.Wait. func (p *Process) blockUntilWaitable() (bool, error) { - var errno syscall.Errno - for { - _, errno = wait6(_P_PID, p.Pid, syscall.WEXITED|syscall.WNOWAIT) - if errno != syscall.EINTR { - break + err := ignoringEINTR(func() error { + _, errno := wait6(_P_PID, p.Pid, syscall.WEXITED|syscall.WNOWAIT) + if errno != 0 { + return errno } - } + return nil + }) runtime.KeepAlive(p) - if errno == syscall.ENOSYS { + if err == syscall.ENOSYS { return false, nil - } else if errno != 0 { - return false, NewSyscallError("wait6", errno) + } else if err != nil { + return false, NewSyscallError("wait6", err) } return true, nil } diff --git a/src/os/wait_waitid.go b/src/os/wait_waitid.go index cd078f35220676..832dbe907b963c 100644 --- a/src/os/wait_waitid.go +++ b/src/os/wait_waitid.go @@ -10,39 +10,28 @@ package os import ( + "internal/syscall/unix" "runtime" "syscall" - "unsafe" ) -const _P_PID = 1 - // blockUntilWaitable attempts to block until a call to p.Wait will // succeed immediately, and reports whether it has done so. // It does not actually call p.Wait. func (p *Process) blockUntilWaitable() (bool, error) { - // The waitid system call expects a pointer to a siginfo_t, - // which is 128 bytes on all Linux systems. - // On darwin/amd64, it requires 104 bytes. - // We don't care about the values it returns. - var siginfo [16]uint64 - psig := &siginfo[0] - var e syscall.Errno - for { - _, _, e = syscall.Syscall6(syscall.SYS_WAITID, _P_PID, uintptr(p.Pid), uintptr(unsafe.Pointer(psig)), syscall.WEXITED|syscall.WNOWAIT, 0, 0) - if e != syscall.EINTR { - break - } - } + var info unix.SiginfoChild + err := ignoringEINTR(func() error { + return unix.Waitid(unix.P_PID, p.Pid, &info, syscall.WEXITED|syscall.WNOWAIT, nil) + }) runtime.KeepAlive(p) - if e != 0 { + if err != nil { // waitid has been available since Linux 2.6.9, but // reportedly is not available in Ubuntu on Windows. // See issue 16610. - if e == syscall.ENOSYS { + if err == syscall.ENOSYS { return false, nil } - return false, NewSyscallError("waitid", e) + return false, NewSyscallError("waitid", err) } return true, nil } diff --git a/src/os/writeto_linux_test.go b/src/os/writeto_linux_test.go index e3900631baa1ee..7d11bda74f7ad0 100644 --- a/src/os/writeto_linux_test.go +++ b/src/os/writeto_linux_test.go @@ -8,13 +8,11 @@ import ( "bytes" "internal/poll" "io" - "math/rand" "net" . "os" "strconv" "syscall" "testing" - "time" ) func TestSendFile(t *testing.T) { @@ -102,7 +100,7 @@ func newSendFileTest(t *testing.T, proto string, size int64) (net.Conn, *File, n hook := hookSendFile(t) client, server := createSocketPair(t, proto) - tempFile, data := createTempFile(t, size) + tempFile, data := createTempFile(t, "writeto-sendfile-to-socket", size) return client, tempFile, server, data, hook } @@ -113,10 +111,10 @@ func hookSendFile(t *testing.T) *sendFileHook { t.Cleanup(func() { poll.TestHookDidSendFile = orig }) - poll.TestHookDidSendFile = func(dstFD *poll.FD, src int, written int64, err error, handled bool) { + poll.TestHookDidSendFile = func(dstFD *poll.FD, src uintptr, written int64, err error, handled bool) { h.called = true h.dstfd = dstFD.Sysfd - h.srcfd = src + h.srcfd = int(src) h.written = written h.err = err h.handled = handled @@ -133,30 +131,3 @@ type sendFileHook struct { handled bool err error } - -func createTempFile(t *testing.T, size int64) (*File, []byte) { - f, err := CreateTemp(t.TempDir(), "writeto-sendfile-to-socket") - if err != nil { - t.Fatalf("failed to create temporary file: %v", err) - } - t.Cleanup(func() { - f.Close() - }) - - randSeed := time.Now().Unix() - t.Logf("random data seed: %d\n", randSeed) - prng := rand.New(rand.NewSource(randSeed)) - data := make([]byte, size) - prng.Read(data) - if _, err := f.Write(data); err != nil { - t.Fatalf("failed to create and feed the file: %v", err) - } - if err := f.Sync(); err != nil { - t.Fatalf("failed to save the file: %v", err) - } - if _, err := f.Seek(0, io.SeekStart); err != nil { - t.Fatalf("failed to rewind the file: %v", err) - } - - return f, data -} diff --git a/src/os/zero_copy_freebsd.go b/src/os/zero_copy_freebsd.go new file mode 100644 index 00000000000000..aacfe8610571ac --- /dev/null +++ b/src/os/zero_copy_freebsd.go @@ -0,0 +1,57 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +import ( + "internal/poll" + "io" +) + +var pollCopyFileRange = poll.CopyFileRange + +func (f *File) writeTo(w io.Writer) (written int64, handled bool, err error) { + return 0, false, nil +} + +func (f *File) readFrom(r io.Reader) (written int64, handled bool, err error) { + // copy_file_range(2) doesn't support destinations opened with + // O_APPEND, so don't bother to try zero-copy with these system calls. + // + // Visit https://man.freebsd.org/cgi/man.cgi?copy_file_range(2)#ERRORS for details. + if f.appendMode { + return 0, false, nil + } + + var ( + remain int64 + lr *io.LimitedReader + ) + if lr, r, remain = tryLimitedReader(r); remain <= 0 { + return 0, true, nil + } + + var src *File + switch v := r.(type) { + case *File: + src = v + case fileWithoutWriteTo: + src = v.File + default: + return 0, false, nil + } + + if src.checkValid("ReadFrom") != nil { + // Avoid returning the error as we report handled as false, + // leave further error handling as the responsibility of the caller. + return 0, false, nil + } + + written, handled, err = pollCopyFileRange(&f.pfd, &src.pfd, remain) + if lr != nil { + lr.N -= written + } + + return written, handled, wrapSyscallError("copy_file_range", err) +} diff --git a/src/os/zero_copy_linux.go b/src/os/zero_copy_linux.go index 0afc19e125a24e..af30a68168af17 100644 --- a/src/os/zero_copy_linux.go +++ b/src/os/zero_copy_linux.go @@ -15,15 +15,6 @@ var ( pollSplice = poll.Splice ) -// wrapSyscallError takes an error and a syscall name. If the error is -// a syscall.Errno, it wraps it in an os.SyscallError using the syscall name. -func wrapSyscallError(name string, err error) error { - if _, ok := err.(syscall.Errno); ok { - err = NewSyscallError(name, err) - } - return err -} - func (f *File) writeTo(w io.Writer) (written int64, handled bool, err error) { pfd, network := getPollFDAndNetwork(w) // TODO(panjf2000): same as File.spliceToFile. @@ -37,7 +28,7 @@ func (f *File) writeTo(w io.Writer) (written int64, handled bool, err error) { } rerr := sc.Read(func(fd uintptr) (done bool) { - written, err, handled = poll.SendFile(pfd, int(fd), 1<<63-1) + written, err, handled = poll.SendFile(pfd, fd, 0) return true }) @@ -149,21 +140,6 @@ func getPollFDAndNetwork(i any) (*poll.FD, poll.String) { return irc.PollFD(), irc.Network() } -// tryLimitedReader tries to assert the io.Reader to io.LimitedReader, it returns the io.LimitedReader, -// the underlying io.Reader and the remaining amount of bytes if the assertion succeeds, -// otherwise it just returns the original io.Reader and the theoretical unlimited remaining amount of bytes. -func tryLimitedReader(r io.Reader) (*io.LimitedReader, io.Reader, int64) { - var remain int64 = 1<<63 - 1 // by default, copy until EOF - - lr, ok := r.(*io.LimitedReader) - if !ok { - return nil, r, remain - } - - remain = lr.N - return lr, lr.R, remain -} - func isUnixOrTCP(network string) bool { switch network { case "tcp", "tcp4", "tcp6", "unix": diff --git a/src/os/zero_copy_posix.go b/src/os/zero_copy_posix.go new file mode 100644 index 00000000000000..161144fe23465b --- /dev/null +++ b/src/os/zero_copy_posix.go @@ -0,0 +1,36 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix || js || wasip1 || windows + +package os + +import ( + "io" + "syscall" +) + +// wrapSyscallError takes an error and a syscall name. If the error is +// a syscall.Errno, it wraps it in an os.SyscallError using the syscall name. +func wrapSyscallError(name string, err error) error { + if _, ok := err.(syscall.Errno); ok { + err = NewSyscallError(name, err) + } + return err +} + +// tryLimitedReader tries to assert the io.Reader to io.LimitedReader, it returns the io.LimitedReader, +// the underlying io.Reader and the remaining amount of bytes if the assertion succeeds, +// otherwise it just returns the original io.Reader and the theoretical unlimited remaining amount of bytes. +func tryLimitedReader(r io.Reader) (*io.LimitedReader, io.Reader, int64) { + var remain int64 = 1<<63 - 1 // by default, copy until EOF + + lr, ok := r.(*io.LimitedReader) + if !ok { + return nil, r, remain + } + + remain = lr.N + return lr, lr.R, remain +} diff --git a/src/os/zero_copy_solaris.go b/src/os/zero_copy_solaris.go new file mode 100644 index 00000000000000..6000700fce76a5 --- /dev/null +++ b/src/os/zero_copy_solaris.go @@ -0,0 +1,92 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +import ( + "internal/poll" + "io" + "runtime" + "syscall" +) + +func (f *File) writeTo(w io.Writer) (written int64, handled bool, err error) { + return 0, false, nil +} + +// readFrom is basically a refactor of net.sendFile, but adapted to work for the target of *File. +func (f *File) readFrom(r io.Reader) (written int64, handled bool, err error) { + var remain int64 = 0 // 0 indicates sending until EOF + lr, ok := r.(*io.LimitedReader) + if ok { + remain, r = lr.N, lr.R + if remain <= 0 { + return 0, true, nil + } + } + + var src *File + switch v := r.(type) { + case *File: + src = v + case fileWithoutWriteTo: + src = v.File + default: + return 0, false, nil + } + + if src.checkValid("ReadFrom") != nil { + // Avoid returning the error as we report handled as false, + // leave further error handling as the responsibility of the caller. + return 0, false, nil + } + + // If fd_in and fd_out refer to the same file and the source and target ranges overlap, + // sendfile(2) on SunOS will allow this kind of overlapping and work like a memmove, + // in this case the file content remains the same after copying, which is not what we want. + // Thus, we just bail out here and leave it to generic copy when it's a file copying itself. + if f.pfd.Sysfd == src.pfd.Sysfd { + return 0, false, nil + } + + // sendfile() on illumos seems to incur intermittent failures when the + // target file is a standard stream (stdout/stderr), we hereby skip any + // anything other than regular files conservatively and leave them to generic copy. + // Check out https://go.dev/issue/68863 for more details. + if runtime.GOOS == "illumos" { + fi, err := f.Stat() + if err != nil { + return 0, false, nil + } + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, false, nil + } + if typ := st.Mode & syscall.S_IFMT; typ != syscall.S_IFREG { + return 0, false, nil + } + } + + sc, err := src.SyscallConn() + if err != nil { + return + } + + // System call sendfile()s on Solaris and illumos support file-to-file copying. + // Check out https://docs.oracle.com/cd/E86824_01/html/E54768/sendfile-3ext.html and + // https://docs.oracle.com/cd/E88353_01/html/E37843/sendfile-3c.html and + // https://illumos.org/man/3EXT/sendfile for more details. + rerr := sc.Read(func(fd uintptr) bool { + written, err, handled = poll.SendFile(&f.pfd, fd, remain) + return true + }) + if lr != nil { + lr.N = remain - written + } + if err == nil { + err = rerr + } + + return written, handled, wrapSyscallError("sendfile", err) +} diff --git a/src/os/zero_copy_stub.go b/src/os/zero_copy_stub.go index 9ec5808101889d..0470a205efefd2 100644 --- a/src/os/zero_copy_stub.go +++ b/src/os/zero_copy_stub.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !linux +//go:build !freebsd && !linux && !solaris package os diff --git a/src/path/filepath/match.go b/src/path/filepath/match.go index b93e89adb03a31..7ccf71ee0c3dd8 100644 --- a/src/path/filepath/match.go +++ b/src/path/filepath/match.go @@ -94,16 +94,12 @@ func scanChunk(pattern string) (star bool, chunk, rest string) { star = true } inrange := false - var i int -Scan: - for i = 0; i < len(pattern); i++ { + for i := 0; i < len(pattern); i++ { switch pattern[i] { case '\\': - if runtime.GOOS != "windows" { - // error check handled in matchChunk: bad pattern. - if i+1 < len(pattern) { - i++ - } + // error check handled in matchChunk: bad pattern. + if runtime.GOOS != "windows" && i+1 < len(pattern) { + i++ } case '[': inrange = true @@ -111,11 +107,11 @@ Scan: inrange = false case '*': if !inrange { - break Scan + return star, pattern[:i], pattern[i:] } } } - return star, pattern[0:i], pattern[i:] + return star, pattern, "" } // matchChunk checks whether chunk matches the beginning of s. @@ -127,9 +123,7 @@ func matchChunk(chunk, s string) (rest string, ok bool, err error) { // checking that the pattern is well-formed but no longer reading s. failed := false for len(chunk) > 0 { - if !failed && len(s) == 0 { - failed = true - } + failed = failed || len(s) == 0 switch chunk[0] { case '[': // character class @@ -164,20 +158,14 @@ func matchChunk(chunk, s string) (rest string, ok bool, err error) { return "", false, err } } - if lo <= r && r <= hi { - match = true - } + match = match || lo <= r && r <= hi nrange++ } - if match == negated { - failed = true - } + failed = failed || match == negated case '?': if !failed { - if s[0] == Separator { - failed = true - } + failed = s[0] == Separator _, n := utf8.DecodeRuneInString(s) s = s[n:] } @@ -194,9 +182,7 @@ func matchChunk(chunk, s string) (rest string, ok bool, err error) { default: if !failed { - if chunk[0] != s[0] { - failed = true - } + failed = chunk[0] != s[0] s = s[1:] } chunk = chunk[1:] diff --git a/src/path/filepath/match_test.go b/src/path/filepath/match_test.go index 3cee92f8ae0009..51ccaecaa99fff 100644 --- a/src/path/filepath/match_test.go +++ b/src/path/filepath/match_test.go @@ -106,6 +106,23 @@ func TestMatch(t *testing.T) { } } +func BenchmarkMatch(b *testing.B) { + for _, tt := range matchTests { + name := fmt.Sprintf("%q %q", tt.pattern, tt.s) + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for range b.N { + bSink, errSink = Match(tt.pattern, tt.s) + } + }) + } +} + +var ( + bSink bool + errSink error +) + var globTests = []struct { pattern, result string }{ @@ -231,7 +248,7 @@ func (test *globTest) globAbs(root, rootPattern string) error { } slices.Sort(have) want := test.buildWant(root + `\`) - if strings.Join(want, "_") == strings.Join(have, "_") { + if slices.Equal(want, have) { return nil } return fmt.Errorf("Glob(%q) returns %q, but %q expected", p, have, want) @@ -245,12 +262,12 @@ func (test *globTest) globRel(root string) error { } slices.Sort(have) want := test.buildWant(root) - if strings.Join(want, "_") == strings.Join(have, "_") { + if slices.Equal(want, have) { return nil } // try also matching version without root prefix wantWithNoRoot := test.buildWant("") - if strings.Join(wantWithNoRoot, "_") == strings.Join(have, "_") { + if slices.Equal(wantWithNoRoot, have) { return nil } return fmt.Errorf("Glob(%q) returns %q, but %q expected", p, have, want) @@ -326,20 +343,7 @@ func TestWindowsGlob(t *testing.T) { } // test relative paths - wd, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - err = os.Chdir(tmpDir) - if err != nil { - t.Fatal(err) - } - defer func() { - err := os.Chdir(wd) - if err != nil { - t.Fatal(err) - } - }() + t.Chdir(tmpDir) for _, test := range tests { err := test.globRel("") if err != nil { diff --git a/src/path/filepath/path.go b/src/path/filepath/path.go index 5ffd9f0b6c3fba..ecf201ac000bde 100644 --- a/src/path/filepath/path.go +++ b/src/path/filepath/path.go @@ -173,19 +173,20 @@ func unixAbs(path string) (string, error) { return Join(wd, path), nil } -// Rel returns a relative path that is lexically equivalent to targpath when -// joined to basepath with an intervening separator. That is, -// [Join](basepath, Rel(basepath, targpath)) is equivalent to targpath itself. -// On success, the returned path will always be relative to basepath, -// even if basepath and targpath share no elements. -// An error is returned if targpath can't be made relative to basepath or if -// knowing the current working directory would be necessary to compute it. -// Rel calls [Clean] on the result. -func Rel(basepath, targpath string) (string, error) { - baseVol := VolumeName(basepath) - targVol := VolumeName(targpath) - base := Clean(basepath) - targ := Clean(targpath) +// Rel returns a relative path that is lexically equivalent to targPath when +// joined to basePath with an intervening separator. That is, +// [Join](basePath, Rel(basePath, targPath)) is equivalent to targPath itself. +// +// The returned path will always be relative to basePath, even if basePath and +// targPath share no elements. Rel calls [Clean] on the result. +// +// An error is returned if targPath can't be made relative to basePath +// or if knowing the current working directory would be necessary to compute it. +func Rel(basePath, targPath string) (string, error) { + baseVol := VolumeName(basePath) + targVol := VolumeName(targPath) + base := Clean(basePath) + targ := Clean(targPath) if sameWord(targ, base) { return ".", nil } @@ -194,7 +195,7 @@ func Rel(basepath, targpath string) (string, error) { if base == "." { base = "" } else if base == "" && filepathlite.VolumeNameLen(baseVol) > 2 /* isUNC */ { - // Treat any targetpath matching `\\host\share` basepath as absolute path. + // Treat any targetpath matching `\\host\share` basePath as absolute path. base = string(Separator) } @@ -202,7 +203,7 @@ func Rel(basepath, targpath string) (string, error) { baseSlashed := len(base) > 0 && base[0] == Separator targSlashed := len(targ) > 0 && targ[0] == Separator if baseSlashed != targSlashed || !sameWord(baseVol, targVol) { - return "", errors.New("Rel: can't make " + targpath + " relative to " + basepath) + return "", errors.New("Rel: can't make " + targPath + " relative to " + basePath) } // Position base[b0:bi] and targ[t0:ti] at the first differing elements. bl := len(base) @@ -228,7 +229,7 @@ func Rel(basepath, targpath string) (string, error) { t0 = ti } if base[b0:bi] == ".." { - return "", errors.New("Rel: can't make " + targpath + " relative to " + basepath) + return "", errors.New("Rel: can't make " + targPath + " relative to " + basePath) } if b0 != bl { // Base elements left. Must go up before going down. @@ -248,7 +249,7 @@ func Rel(basepath, targpath string) (string, error) { buf[n] = Separator copy(buf[n+1:], targ[t0:]) } - return string(buf), nil + return Clean(string(buf)), nil } return targ[t0:], nil } diff --git a/src/path/filepath/path_test.go b/src/path/filepath/path_test.go index 5d3cbc991fefae..ad99f70287f74c 100644 --- a/src/path/filepath/path_test.go +++ b/src/path/filepath/path_test.go @@ -597,45 +597,6 @@ func mark(d fs.DirEntry, err error, errors *[]error, clear bool) error { return nil } -// chdir changes the current working directory to the named directory, -// and then restore the original working directory at the end of the test. -func chdir(t *testing.T, dir string) { - olddir, err := os.Getwd() - if err != nil { - t.Fatalf("getwd %s: %v", dir, err) - } - if err := os.Chdir(dir); err != nil { - t.Fatalf("chdir %s: %v", dir, err) - } - - t.Cleanup(func() { - if err := os.Chdir(olddir); err != nil { - t.Errorf("restore original working directory %s: %v", olddir, err) - os.Exit(1) - } - }) -} - -func chtmpdir(t *testing.T) (restore func()) { - oldwd, err := os.Getwd() - if err != nil { - t.Fatalf("chtmpdir: %v", err) - } - d, err := os.MkdirTemp("", "test") - if err != nil { - t.Fatalf("chtmpdir: %v", err) - } - if err := os.Chdir(d); err != nil { - t.Fatalf("chtmpdir: %v", err) - } - return func() { - if err := os.Chdir(oldwd); err != nil { - t.Fatalf("chtmpdir: %v", err) - } - os.RemoveAll(d) - } -} - // tempDirCanonical returns a temporary directory for the test to use, ensuring // that the returned path does not contain symlinks. func tempDirCanonical(t *testing.T) string { @@ -663,21 +624,7 @@ func TestWalkDir(t *testing.T) { } func testWalk(t *testing.T, walk func(string, fs.WalkDirFunc) error, errVisit int) { - if runtime.GOOS == "ios" { - restore := chtmpdir(t) - defer restore() - } - - tmpDir := t.TempDir() - - origDir, err := os.Getwd() - if err != nil { - t.Fatal("finding working dir:", err) - } - if err = os.Chdir(tmpDir); err != nil { - t.Fatal("entering temp dir:", err) - } - defer os.Chdir(origDir) + t.Chdir(t.TempDir()) makeTree(t) errors := make([]error, 0, 10) @@ -686,7 +633,7 @@ func testWalk(t *testing.T, walk func(string, fs.WalkDirFunc) error, errVisit in return mark(d, err, &errors, clear) } // Expect no errors. - err = walk(tree.name, markFn) + err := walk(tree.name, markFn) if err != nil { t.Fatalf("no error expected, found: %s", err) } @@ -989,7 +936,6 @@ func TestWalkSymlinkRoot(t *testing.T) { buggyGOOS: []string{"darwin", "ios"}, // https://go.dev/issue/59586 }, } { - tt := tt t.Run(tt.desc, func(t *testing.T) { var walked []string err := filepath.Walk(tt.root, func(path string, info fs.FileInfo, err error) error { @@ -1225,22 +1171,7 @@ func testEvalSymlinks(t *testing.T, path, want string) { } func testEvalSymlinksAfterChdir(t *testing.T, wd, path, want string) { - cwd, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - defer func() { - err := os.Chdir(cwd) - if err != nil { - t.Fatal(err) - } - }() - - err = os.Chdir(wd) - if err != nil { - t.Fatal(err) - } - + t.Chdir(wd) have, err := filepath.EvalSymlinks(path) if err != nil { t.Errorf("EvalSymlinks(%q) in %q directory error: %v", path, wd, err) @@ -1314,8 +1245,7 @@ func TestEvalSymlinks(t *testing.T) { func TestEvalSymlinksIsNotExist(t *testing.T) { testenv.MustHaveSymlink(t) - - defer chtmpdir(t)() + t.Chdir(t.TempDir()) _, err := filepath.EvalSymlinks("notexist") if !os.IsNotExist(err) { @@ -1396,10 +1326,10 @@ func TestIssue13582(t *testing.T) { // Issue 57905. func TestRelativeSymlinkToAbsolute(t *testing.T) { testenv.MustHaveSymlink(t) - // Not parallel: uses os.Chdir. + // Not parallel: uses t.Chdir. tmpDir := t.TempDir() - chdir(t, tmpDir) + t.Chdir(tmpDir) // Create "link" in the current working directory as a symlink to an arbitrary // absolute path. On macOS, this path is likely to begin with a symlink @@ -1452,18 +1382,10 @@ var absTests = []string{ func TestAbs(t *testing.T) { root := t.TempDir() - wd, err := os.Getwd() - if err != nil { - t.Fatal("getwd failed: ", err) - } - err = os.Chdir(root) - if err != nil { - t.Fatal("chdir failed: ", err) - } - defer os.Chdir(wd) + t.Chdir(root) for _, dir := range absTestDirs { - err = os.Mkdir(dir, 0777) + err := os.Mkdir(dir, 0777) if err != nil { t.Fatal("Mkdir failed: ", err) } @@ -1485,7 +1407,7 @@ func TestAbs(t *testing.T) { tests = append(slices.Clip(tests), extra...) } - err = os.Chdir(absTestDirs[0]) + err := os.Chdir(absTestDirs[0]) if err != nil { t.Fatal("chdir failed: ", err) } @@ -1521,16 +1443,7 @@ func TestAbs(t *testing.T) { // a valid path, so it can't be used with os.Stat. func TestAbsEmptyString(t *testing.T) { root := t.TempDir() - - wd, err := os.Getwd() - if err != nil { - t.Fatal("getwd failed: ", err) - } - err = os.Chdir(root) - if err != nil { - t.Fatal("chdir failed: ", err) - } - defer os.Chdir(wd) + t.Chdir(root) info, err := os.Stat(root) if err != nil { @@ -1592,6 +1505,7 @@ var reltests = []RelTests{ {"/../../a/b", "/../../a/b/c/d", "c/d"}, {".", "a/b", "a/b"}, {".", "..", ".."}, + {"", "../../.", "../.."}, // can't do purely lexically {"..", ".", "err"}, @@ -1757,19 +1671,9 @@ func TestBug3486(t *testing.T) { // https://golang.org/issue/3486 func testWalkSymlink(t *testing.T, mklink func(target, link string) error) { tmpdir := t.TempDir() + t.Chdir(tmpdir) - wd, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - defer os.Chdir(wd) - - err = os.Chdir(tmpdir) - if err != nil { - t.Fatal(err) - } - - err = mklink(tmpdir, "link") + err := mklink(tmpdir, "link") if err != nil { t.Fatal(err) } @@ -1882,13 +1786,7 @@ func TestEvalSymlinksAboveRoot(t *testing.T) { // Issue 30520 part 2. func TestEvalSymlinksAboveRootChdir(t *testing.T) { testenv.MustHaveSymlink(t) - - tmpDir, err := os.MkdirTemp("", "TestEvalSymlinksAboveRootChdir") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpDir) - chdir(t, tmpDir) + t.Chdir(t.TempDir()) subdir := filepath.Join("a", "b") if err := os.MkdirAll(subdir, 0777); err != nil { @@ -1956,12 +1854,11 @@ func TestIssue51617(t *testing.T) { } func TestEscaping(t *testing.T) { - dir1 := t.TempDir() - dir2 := t.TempDir() - chdir(t, dir1) + dir := t.TempDir() + t.Chdir(t.TempDir()) for _, p := range []string{ - filepath.Join(dir2, "x"), + filepath.Join(dir, "x"), } { if !filepath.IsLocal(p) { continue @@ -1970,7 +1867,7 @@ func TestEscaping(t *testing.T) { if err != nil { f.Close() } - ents, err := os.ReadDir(dir2) + ents, err := os.ReadDir(dir) if err != nil { t.Fatal(err) } @@ -1992,3 +1889,18 @@ func TestEvalSymlinksTooManyLinks(t *testing.T) { t.Fatal("expected error, got nil") } } + +func BenchmarkIsLocal(b *testing.B) { + tests := islocaltests + if runtime.GOOS == "windows" { + tests = append(tests, winislocaltests...) + } + if runtime.GOOS == "plan9" { + tests = append(tests, plan9islocaltests...) + } + for b.Loop() { + for _, test := range tests { + filepath.IsLocal(test.path) + } + } +} diff --git a/src/path/filepath/path_windows.go b/src/path/filepath/path_windows.go index d53f87f1ac1a36..d0eb42c5f6db2a 100644 --- a/src/path/filepath/path_windows.go +++ b/src/path/filepath/path_windows.go @@ -1,3 +1,7 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package filepath import ( diff --git a/src/path/filepath/path_windows_test.go b/src/path/filepath/path_windows_test.go index 603b179405d01b..d60903f62ebb53 100644 --- a/src/path/filepath/path_windows_test.go +++ b/src/path/filepath/path_windows_test.go @@ -408,12 +408,7 @@ func TestToNorm(t *testing.T) { if err != nil { t.Fatal(err) } - defer func() { - err := os.Chdir(cwd) - if err != nil { - t.Fatal(err) - } - }() + t.Chdir(".") // Ensure cwd is restored after the test. tmpVol := filepath.VolumeName(ctmp) if len(tmpVol) != 2 { diff --git a/src/path/filepath/symlink_plan9.go b/src/path/filepath/symlink_plan9.go index 820d150d976e19..b29abe1dfd4660 100644 --- a/src/path/filepath/symlink_plan9.go +++ b/src/path/filepath/symlink_plan9.go @@ -16,8 +16,8 @@ func evalSymlinks(path string) (string, error) { // Check validity of path _, err := os.Lstat(path) if err != nil { - // Return the same error value as on other operating systems - if strings.HasSuffix(err.Error(), "not a directory") { + // Return the same error value as on other operating systems. + if strings.Contains(err.Error(), "not a directory") { err = syscall.ENOTDIR } return "", err diff --git a/src/path/match.go b/src/path/match.go index d8b6809568fbdc..0678effd6e5fba 100644 --- a/src/path/match.go +++ b/src/path/match.go @@ -95,9 +95,7 @@ func scanChunk(pattern string) (star bool, chunk, rest string) { star = true } inrange := false - var i int -Scan: - for i = 0; i < len(pattern); i++ { + for i := 0; i < len(pattern); i++ { switch pattern[i] { case '\\': // error check handled in matchChunk: bad pattern. @@ -110,11 +108,11 @@ Scan: inrange = false case '*': if !inrange { - break Scan + return star, pattern[:i], pattern[i:] } } } - return star, pattern[0:i], pattern[i:] + return star, pattern, "" } // matchChunk checks whether chunk matches the beginning of s. @@ -126,9 +124,7 @@ func matchChunk(chunk, s string) (rest string, ok bool, err error) { // checking that the pattern is well-formed but no longer reading s. failed := false for len(chunk) > 0 { - if !failed && len(s) == 0 { - failed = true - } + failed = failed || len(s) == 0 switch chunk[0] { case '[': // character class @@ -163,20 +159,14 @@ func matchChunk(chunk, s string) (rest string, ok bool, err error) { return "", false, err } } - if lo <= r && r <= hi { - match = true - } + match = match || lo <= r && r <= hi nrange++ } - if match == negated { - failed = true - } + failed = failed || match == negated case '?': if !failed { - if s[0] == '/' { - failed = true - } + failed = s[0] == '/' _, n := utf8.DecodeRuneInString(s) s = s[n:] } @@ -191,9 +181,7 @@ func matchChunk(chunk, s string) (rest string, ok bool, err error) { default: if !failed { - if chunk[0] != s[0] { - failed = true - } + failed = chunk[0] != s[0] s = s[1:] } chunk = chunk[1:] diff --git a/src/path/match_test.go b/src/path/match_test.go index 996bd691eb82b6..90ca19a6f457a3 100644 --- a/src/path/match_test.go +++ b/src/path/match_test.go @@ -5,6 +5,7 @@ package path_test import ( + "fmt" . "path" "testing" ) @@ -82,3 +83,20 @@ func TestMatch(t *testing.T) { } } } + +func BenchmarkMatch(b *testing.B) { + for _, tt := range matchTests { + name := fmt.Sprintf("%q %q", tt.pattern, tt.s) + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for range b.N { + bSink, errSink = Match(tt.pattern, tt.s) + } + }) + } +} + +var ( + bSink bool + errSink error +) diff --git a/src/path/path_test.go b/src/path/path_test.go index a57286f6b8468b..6bd7a2cdad885c 100644 --- a/src/path/path_test.go +++ b/src/path/path_test.go @@ -5,6 +5,7 @@ package path_test import ( + "path" . "path" "runtime" "testing" @@ -234,3 +235,14 @@ func TestIsAbs(t *testing.T) { } } } + +func BenchmarkJoin(b *testing.B) { + b.ReportAllocs() + parts := []string{"one", "two", "three", "four"} + s := parts[0] + for b.Loop() { + parts[0] = s + s = path.Join(parts...) + s = s[:3] + } +} diff --git a/src/plugin/plugin.go b/src/plugin/plugin.go index b4b1697b52c82d..a4ff694eb57077 100644 --- a/src/plugin/plugin.go +++ b/src/plugin/plugin.go @@ -29,6 +29,10 @@ // macOS, making them unsuitable for applications intended to be // portable. // +// - Plugins are poorly supported by the Go race detector. Even simple +// race conditions may not be automatically detected. See +// https://go.dev/issue/24245 for more information. +// // - Applications that use plugins may require careful configuration // to ensure that the various parts of the program be made available // in the correct location in the file system (or container image). diff --git a/src/race.bash b/src/race.bash index f1a168bfbb7313..3ce91a345fdbea 100755 --- a/src/race.bash +++ b/src/race.bash @@ -9,7 +9,7 @@ set -e function usage { - echo 'race detector is only supported on linux/amd64, linux/ppc64le, linux/arm64, linux/s390x, freebsd/amd64, netbsd/amd64, openbsd/amd64, darwin/amd64, and darwin/arm64' 1>&2 + echo 'race detector is only supported on linux/amd64, linux/ppc64le, linux/arm64, linux/loong64, linux/riscv64, linux/s390x, freebsd/amd64, netbsd/amd64, openbsd/amd64, darwin/amd64, and darwin/arm64' 1>&2 exit 1 } @@ -19,6 +19,8 @@ case $(uname -s -m) in "Linux x86_64") ;; "Linux ppc64le") ;; "Linux aarch64") ;; + "Linux loongarch64") ;; + "Linux riscv64") ;; "Linux s390x") ;; "FreeBSD amd64") ;; "NetBSD amd64") ;; diff --git a/src/race.bat b/src/race.bat index d395e19f9741dd..206d4126eea648 100644 --- a/src/race.bat +++ b/src/race.bat @@ -9,43 +9,16 @@ setlocal -if exist make.bat goto ok -echo race.bat must be run from go\src -:: cannot exit: would kill parent command interpreter -goto end -:ok - -set GOROOT=%CD%\.. -call .\make.bat --dist-tool >NUL -if errorlevel 1 goto fail -.\cmd\dist\dist.exe env -w -p >env.bat -if errorlevel 1 goto fail -call .\env.bat -del env.bat - -if %GOHOSTARCH% == amd64 goto continue -echo Race detector is only supported on windows/amd64. -goto fail - -:continue -call .\make.bat --no-banner --no-local -if %GOBUILDFAIL%==1 goto end -echo # go install -race std -go install -race std -if errorlevel 1 goto fail - -go tool dist test -race - -if errorlevel 1 goto fail -goto succ - -:fail -set GOBUILDFAIL=1 -echo Fail. -goto end - -:succ -echo All tests passed. - -:end -if x%GOBUILDEXIT%==x1 exit %GOBUILDFAIL% +if not exist make.bat ( + echo race.bat must be run from go\src + exit /b 1 +) + +if not "%PROCESSOR_ARCHITECTURE%"=="AMD64" ( + echo Race detector is only supported on windows/amd64. + exit /b 1 +) + +call .\make.bat --no-banner || exit /b 1 +go install -race std || exit /b 1 +go tool dist test -race || exit /b 1 diff --git a/src/reflect/abi.go b/src/reflect/abi.go index b67d8217430d68..bcd0a95f1b6c62 100644 --- a/src/reflect/abi.go +++ b/src/reflect/abi.go @@ -166,7 +166,7 @@ func (a *abiSeq) addRcvr(rcvr *abi.Type) (*abiStep, bool) { // The receiver is always one word. a.valueStart = append(a.valueStart, len(a.steps)) var ok, ptr bool - if rcvr.IfaceIndir() || rcvr.Pointers() { + if !rcvr.IsDirectIface() || rcvr.Pointers() { ok = a.assignIntN(0, goarch.PtrSize, 1, 0b1) ptr = true } else { diff --git a/src/reflect/all_test.go b/src/reflect/all_test.go index a57f17f8c30f12..1fa850a27562ce 100644 --- a/src/reflect/all_test.go +++ b/src/reflect/all_test.go @@ -10,8 +10,10 @@ import ( "flag" "fmt" "go/token" - "internal/abi" + "internal/asan" "internal/goarch" + "internal/msan" + "internal/race" "internal/testenv" "io" "math" @@ -22,6 +24,7 @@ import ( "reflect/internal/example1" "reflect/internal/example2" "runtime" + "runtime/debug" "slices" "strconv" "strings" @@ -32,8 +35,6 @@ import ( "unsafe" ) -const bucketCount = abi.MapBucketCount - var sink any func TestBool(t *testing.T) { @@ -1134,13 +1135,15 @@ var deepEqualTests = []DeepEqualTest{ } func TestDeepEqual(t *testing.T) { - for _, test := range deepEqualTests { - if test.b == (self{}) { - test.b = test.a - } - if r := DeepEqual(test.a, test.b); r != test.eq { - t.Errorf("DeepEqual(%#v, %#v) = %v, want %v", test.a, test.b, r, test.eq) - } + for i, test := range deepEqualTests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + if test.b == (self{}) { + test.b = test.a + } + if r := DeepEqual(test.a, test.b); r != test.eq { + t.Errorf("DeepEqual(%#v, %#v) = %v, want %v", test.a, test.b, r, test.eq) + } + }) } } @@ -1273,6 +1276,10 @@ var deepEqualPerfTests = []struct { } func TestDeepEqualAllocs(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } + for _, tt := range deepEqualPerfTests { t.Run(ValueOf(tt.x).Type().String(), func(t *testing.T) { got := testing.AllocsPerRun(100, func() { @@ -2230,18 +2237,18 @@ func TestCallReturnsEmpty(t *testing.T) { // Issue 21717: past-the-end pointer write in Call with // nonzero-sized frame and zero-sized return value. runtime.GC() - var finalized uint32 + var cleanedUp atomic.Uint32 f := func() (emptyStruct, *[2]int64) { - i := new([2]int64) // big enough to not be tinyalloc'd, so finalizer always runs when i dies - runtime.SetFinalizer(i, func(*[2]int64) { atomic.StoreUint32(&finalized, 1) }) + i := new([2]int64) // big enough to not be tinyalloc'd, so cleanup always runs when i dies + runtime.AddCleanup(i, func(cu *atomic.Uint32) { cu.Store(uint32(1)) }, &cleanedUp) return emptyStruct{}, i } - v := ValueOf(f).Call(nil)[0] // out[0] should not alias out[1]'s memory, so the finalizer should run. + v := ValueOf(f).Call(nil)[0] // out[0] should not alias out[1]'s memory, so the cleanup should run. timeout := time.After(5 * time.Second) - for atomic.LoadUint32(&finalized) == 0 { + for cleanedUp.Load() == 0 { select { case <-timeout: - t.Fatal("finalizer did not run") + t.Fatal("cleanup did not run") default: } runtime.Gosched() @@ -3533,6 +3540,14 @@ func TestAllocations(t *testing.T) { panic("wrong result") } }) + if runtime.GOOS != "js" && runtime.GOOS != "wasip1" { + typ := TypeFor[struct{ f int }]() + noAlloc(t, 100, func(int) { + if typ.Field(0).Index[0] != 0 { + panic("wrong field index") + } + }) + } } func TestSmallNegativeInt(t *testing.T) { @@ -6183,19 +6198,6 @@ func TestChanOfDir(t *testing.T) { } func TestChanOfGC(t *testing.T) { - done := make(chan bool, 1) - go func() { - select { - case <-done: - case <-time.After(5 * time.Second): - panic("deadlock in TestChanOfGC") - } - }() - - defer func() { - done <- true - }() - type T *uintptr tt := TypeOf(T(nil)) ct := ChanOf(BothDir, tt) @@ -6291,6 +6293,32 @@ func TestMapOfGCKeys(t *testing.T) { } } +// Test assignment and access to a map with keys larger than word size. +func TestMapOfGCBigKey(t *testing.T) { + type KV struct { + i int64 + j int64 + } + + kvTyp := TypeFor[KV]() + mt := MapOf(kvTyp, kvTyp) + + const n = 100 + m := MakeMap(mt) + for i := 0; i < n; i++ { + kv := KV{int64(i), int64(i + 1)} + m.SetMapIndex(ValueOf(kv), ValueOf(kv)) + } + + for i := 0; i < n; i++ { + kv := KV{int64(i), int64(i + 1)} + elem := m.MapIndex(ValueOf(kv)).Interface().(KV) + if elem != kv { + t.Errorf("lost m[%v] = %v, want %v", kv, elem, kv) + } + } +} + func TestMapOfGCValues(t *testing.T) { type T *uintptr tt := TypeOf(T(nil)) @@ -6823,7 +6851,7 @@ func TestInvalid(t *testing.T) { } // Issue 8917. -func TestLargeGCProg(t *testing.T) { +func TestLarge(t *testing.T) { fv := ValueOf(func([256]*byte) {}) fv.Call([]Value{ValueOf([256]*byte{})}) } @@ -6863,6 +6891,25 @@ func TestTypeFieldOutOfRangePanic(t *testing.T) { } } +func TestTypeFieldReadOnly(t *testing.T) { + if runtime.GOOS == "js" || runtime.GOOS == "wasip1" { + // This is OK because we don't use the optimization + // for js or wasip1. + t.Skip("test does not fault on GOOS=js") + } + + // It's important that changing one StructField.Index + // value not affect other StructField.Index values. + // Right now StructField.Index is read-only; + // that saves allocations but is otherwise not important. + typ := TypeFor[struct{ f int }]() + f := typ.Field(0) + defer debug.SetPanicOnFault(debug.SetPanicOnFault(true)) + shouldPanic("", func() { + f.Index[0] = 1 + }) +} + // Issue 9179. func TestCallGC(t *testing.T) { f := func(a, b, c, d, e string) { @@ -7144,60 +7191,61 @@ func verifyGCBitsSlice(t *testing.T, typ Type, cap int, bits []byte) { t.Errorf("line %d: heapBits incorrect for make(%v, 0, %v)\nhave %v\nwant %v", line, typ, cap, heapBits, bits) } -func TestGCBits(t *testing.T) { - verifyGCBits(t, TypeOf((*byte)(nil)), []byte{1}) +// Building blocks for types seen by the compiler (like [2]Xscalar). +// The compiler will create the type structures for the derived types, +// including their GC metadata. +type Xscalar struct{ x uintptr } +type Xptr struct{ x *byte } +type Xptrscalar struct { + *byte + uintptr +} +type Xscalarptr struct { + uintptr + *byte +} +type Xbigptrscalar struct { + _ [100]*byte + _ [100]uintptr +} + +var Tscalar, Tint64, Tptr, Tscalarptr, Tptrscalar, Tbigptrscalar Type - // Building blocks for types seen by the compiler (like [2]Xscalar). - // The compiler will create the type structures for the derived types, - // including their GC metadata. - type Xscalar struct{ x uintptr } - type Xptr struct{ x *byte } - type Xptrscalar struct { +func init() { + // Building blocks for types constructed by reflect. + // This code is in a separate block so that code below + // cannot accidentally refer to these. + // The compiler must NOT see types derived from these + // (for example, [2]Scalar must NOT appear in the program), + // or else reflect will use it instead of having to construct one. + // The goal is to test the construction. + type Scalar struct{ x uintptr } + type Ptr struct{ x *byte } + type Ptrscalar struct { *byte uintptr } - type Xscalarptr struct { + type Scalarptr struct { uintptr *byte } - type Xbigptrscalar struct { + type Bigptrscalar struct { _ [100]*byte _ [100]uintptr } + type Int64 int64 + Tscalar = TypeOf(Scalar{}) + Tint64 = TypeOf(Int64(0)) + Tptr = TypeOf(Ptr{}) + Tscalarptr = TypeOf(Scalarptr{}) + Tptrscalar = TypeOf(Ptrscalar{}) + Tbigptrscalar = TypeOf(Bigptrscalar{}) +} - var Tscalar, Tint64, Tptr, Tscalarptr, Tptrscalar, Tbigptrscalar Type - { - // Building blocks for types constructed by reflect. - // This code is in a separate block so that code below - // cannot accidentally refer to these. - // The compiler must NOT see types derived from these - // (for example, [2]Scalar must NOT appear in the program), - // or else reflect will use it instead of having to construct one. - // The goal is to test the construction. - type Scalar struct{ x uintptr } - type Ptr struct{ x *byte } - type Ptrscalar struct { - *byte - uintptr - } - type Scalarptr struct { - uintptr - *byte - } - type Bigptrscalar struct { - _ [100]*byte - _ [100]uintptr - } - type Int64 int64 - Tscalar = TypeOf(Scalar{}) - Tint64 = TypeOf(Int64(0)) - Tptr = TypeOf(Ptr{}) - Tscalarptr = TypeOf(Scalarptr{}) - Tptrscalar = TypeOf(Ptrscalar{}) - Tbigptrscalar = TypeOf(Bigptrscalar{}) - } - - empty := []byte{} +var empty = []byte{} + +func TestGCBits(t *testing.T) { + verifyGCBits(t, TypeOf((*byte)(nil)), []byte{1}) verifyGCBits(t, TypeOf(Xscalar{}), empty) verifyGCBits(t, Tscalar, empty) @@ -7277,47 +7325,8 @@ func TestGCBits(t *testing.T) { verifyGCBits(t, TypeOf(([][10000]Xscalar)(nil)), lit(1)) verifyGCBits(t, SliceOf(ArrayOf(10000, Tscalar)), lit(1)) - hdr := make([]byte, bucketCount/goarch.PtrSize) - - verifyMapBucket := func(t *testing.T, k, e Type, m any, want []byte) { - verifyGCBits(t, MapBucketOf(k, e), want) - verifyGCBits(t, CachedBucketOf(TypeOf(m)), want) - } - verifyMapBucket(t, - Tscalar, Tptr, - map[Xscalar]Xptr(nil), - join(hdr, rep(bucketCount, lit(0)), rep(bucketCount, lit(1)), lit(1))) - verifyMapBucket(t, - Tscalarptr, Tptr, - map[Xscalarptr]Xptr(nil), - join(hdr, rep(bucketCount, lit(0, 1)), rep(bucketCount, lit(1)), lit(1))) - verifyMapBucket(t, Tint64, Tptr, - map[int64]Xptr(nil), - join(hdr, rep(bucketCount, rep(8/goarch.PtrSize, lit(0))), rep(bucketCount, lit(1)), lit(1))) - verifyMapBucket(t, - Tscalar, Tscalar, - map[Xscalar]Xscalar(nil), - empty) - verifyMapBucket(t, - ArrayOf(2, Tscalarptr), ArrayOf(3, Tptrscalar), - map[[2]Xscalarptr][3]Xptrscalar(nil), - join(hdr, rep(bucketCount*2, lit(0, 1)), rep(bucketCount*3, lit(1, 0)), lit(1))) - verifyMapBucket(t, - ArrayOf(64/goarch.PtrSize, Tscalarptr), ArrayOf(64/goarch.PtrSize, Tptrscalar), - map[[64 / goarch.PtrSize]Xscalarptr][64 / goarch.PtrSize]Xptrscalar(nil), - join(hdr, rep(bucketCount*64/goarch.PtrSize, lit(0, 1)), rep(bucketCount*64/goarch.PtrSize, lit(1, 0)), lit(1))) - verifyMapBucket(t, - ArrayOf(64/goarch.PtrSize+1, Tscalarptr), ArrayOf(64/goarch.PtrSize, Tptrscalar), - map[[64/goarch.PtrSize + 1]Xscalarptr][64 / goarch.PtrSize]Xptrscalar(nil), - join(hdr, rep(bucketCount, lit(1)), rep(bucketCount*64/goarch.PtrSize, lit(1, 0)), lit(1))) - verifyMapBucket(t, - ArrayOf(64/goarch.PtrSize, Tscalarptr), ArrayOf(64/goarch.PtrSize+1, Tptrscalar), - map[[64 / goarch.PtrSize]Xscalarptr][64/goarch.PtrSize + 1]Xptrscalar(nil), - join(hdr, rep(bucketCount*64/goarch.PtrSize, lit(0, 1)), rep(bucketCount, lit(1)), lit(1))) - verifyMapBucket(t, - ArrayOf(64/goarch.PtrSize+1, Tscalarptr), ArrayOf(64/goarch.PtrSize+1, Tptrscalar), - map[[64/goarch.PtrSize + 1]Xscalarptr][64/goarch.PtrSize + 1]Xptrscalar(nil), - join(hdr, rep(bucketCount, lit(1)), rep(bucketCount, lit(1)), lit(1))) + // For maps, we don't manually construct GC data, instead using the + // public reflect API in groupAndSlotOf. } func rep(n int, b []byte) []byte { return bytes.Repeat(b, n) } @@ -7359,6 +7368,9 @@ func TestPtrToMethods(t *testing.T) { } func TestMapAlloc(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } m := ValueOf(make(map[int]int, 10)) k := ValueOf(5) v := ValueOf(7) @@ -7389,6 +7401,9 @@ func TestMapAlloc(t *testing.T) { } func TestChanAlloc(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } // Note: for a chan int, the return Value must be allocated, so we // use a chan *int instead. c := ValueOf(make(chan *int, 1)) @@ -7500,7 +7515,6 @@ func TestTypeStrings(t *testing.T) { func TestOffsetLock(t *testing.T) { var wg sync.WaitGroup for i := 0; i < 4; i++ { - i := i wg.Add(1) go func() { for j := 0; j < 50; j++ { @@ -7751,11 +7765,14 @@ func TestMapIterReset(t *testing.T) { } // Reset should not allocate. + // + // Except with -asan, where there are additional allocations. + // See #70079. n := int(testing.AllocsPerRun(10, func() { iter.Reset(ValueOf(m2)) iter.Reset(Value{}) })) - if n > 0 { + if !asan.Enabled && n > 0 { t.Errorf("MapIter.Reset allocated %d times", n) } } @@ -8517,7 +8534,6 @@ func TestClear(t *testing.T) { } for _, tc := range tests { - tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() if !tc.testFunc(tc.value) { @@ -8551,7 +8567,6 @@ func TestValuePointerAndUnsafePointer(t *testing.T) { } for _, tc := range tests { - tc := tc t.Run(tc.name, func(t *testing.T) { if got := tc.val.Pointer(); got != uintptr(tc.wantUnsafePointer) { t.Errorf("unexpected uintptr result, got %#x, want %#x", got, uintptr(tc.wantUnsafePointer)) @@ -8603,3 +8618,188 @@ func TestSliceAt(t *testing.T) { // _ = SliceAt(typ, unsafe.Pointer(last), 1) shouldPanic("", func() { _ = SliceAt(typ, unsafe.Pointer(last), 2) }) } + +// Test that maps created with MapOf properly updates keys on overwrite as +// expected (i.e., it sets the key update flag in the map). +// +// This test is based on runtime.TestNegativeZero. +func TestMapOfKeyUpdate(t *testing.T) { + m := MakeMap(MapOf(TypeFor[float64](), TypeFor[bool]())) + + zero := float64(0.0) + negZero := math.Copysign(zero, -1.0) + + m.SetMapIndex(ValueOf(zero), ValueOf(true)) + m.SetMapIndex(ValueOf(negZero), ValueOf(true)) + + if m.Len() != 1 { + t.Errorf("map length got %d want 1", m.Len()) + } + + iter := m.MapRange() + for iter.Next() { + k := iter.Key().Float() + if math.Copysign(1.0, k) > 0 { + t.Errorf("map key %f has positive sign", k) + } + } +} + +// Test that maps created with MapOf properly panic on unhashable keys, even if +// the map is empty. (i.e., it sets the hash might panic flag in the map). +// +// This test is a simplified version of runtime.TestEmptyMapWithInterfaceKey +// for reflect. +func TestMapOfKeyPanic(t *testing.T) { + defer func() { + r := recover() + if r == nil { + t.Errorf("didn't panic") + } + }() + + m := MakeMap(MapOf(TypeFor[any](), TypeFor[bool]())) + + var slice []int + m.MapIndex(ValueOf(slice)) +} + +func TestTypeAssert(t *testing.T) { + testTypeAssert(t, int(123456789), int(123456789), true) + testTypeAssert(t, int(-123456789), int(-123456789), true) + testTypeAssert(t, int32(123456789), int32(123456789), true) + testTypeAssert(t, int8(-123), int8(-123), true) + testTypeAssert(t, [2]int{1234, -5678}, [2]int{1234, -5678}, true) + testTypeAssert(t, "test value", "test value", true) + testTypeAssert(t, any("test value"), any("test value"), true) + + v := 123456789 + testTypeAssert(t, &v, &v, true) + + testTypeAssert(t, int(123), uint(0), false) + + testTypeAssert[any](t, 1, 1, true) + testTypeAssert[fmt.Stringer](t, 1, nil, false) + + vv := testTypeWithMethod{"test"} + testTypeAssert[any](t, vv, vv, true) + testTypeAssert[any](t, &vv, &vv, true) + testTypeAssert[fmt.Stringer](t, vv, vv, true) + testTypeAssert[fmt.Stringer](t, &vv, &vv, true) + testTypeAssert[interface{ A() }](t, vv, nil, false) + testTypeAssert[interface{ A() }](t, &vv, nil, false) + testTypeAssert(t, any(vv), any(vv), true) + testTypeAssert(t, fmt.Stringer(vv), fmt.Stringer(vv), true) + + testTypeAssert(t, fmt.Stringer(vv), any(vv), true) + testTypeAssert(t, any(vv), fmt.Stringer(vv), true) + testTypeAssert(t, fmt.Stringer(vv), interface{ M() }(vv), true) + testTypeAssert(t, interface{ M() }(vv), fmt.Stringer(vv), true) + + testTypeAssert(t, any(int(1)), int(1), true) + testTypeAssert(t, any(int(1)), byte(0), false) + testTypeAssert(t, fmt.Stringer(vv), vv, true) + + testTypeAssert(t, any(nil), any(nil), false) + testTypeAssert(t, any(nil), error(nil), false) + testTypeAssert(t, error(nil), any(nil), false) + testTypeAssert(t, error(nil), error(nil), false) +} + +func testTypeAssert[T comparable, V any](t *testing.T, val V, wantVal T, wantOk bool) { + t.Helper() + + v, ok := TypeAssert[T](ValueOf(&val).Elem()) + if v != wantVal || ok != wantOk { + t.Errorf("TypeAssert[%v](%#v) = (%#v, %v); want = (%#v, %v)", TypeFor[T](), val, v, ok, wantVal, wantOk) + } + + // Additionally make sure that TypeAssert[T](v) behaves in the same way as v.Interface().(T). + v2, ok2 := ValueOf(&val).Elem().Interface().(T) + if v != v2 || ok != ok2 { + t.Errorf("reflect.ValueOf(%#v).Interface().(%v) = (%#v, %v); want = (%#v, %v)", val, TypeFor[T](), v2, ok2, v, ok) + } +} + +type testTypeWithMethod struct{ val string } + +func (v testTypeWithMethod) String() string { return v.val } +func (v testTypeWithMethod) M() {} + +func TestTypeAssertMethod(t *testing.T) { + method := ValueOf(&testTypeWithMethod{val: "test value"}).MethodByName("String") + f, ok := TypeAssert[func() string](method) + if !ok { + t.Fatalf(`TypeAssert[func() string](method) = (,false); want = (,true)`) + } + + out := f() + if out != "test value" { + t.Fatalf(`TypeAssert[func() string](method)() = %q; want "test value"`, out) + } +} + +func TestTypeAssertPanic(t *testing.T) { + t.Run("zero val", func(t *testing.T) { + defer func() { recover() }() + TypeAssert[int](Value{}) + t.Fatalf("TypeAssert did not panic") + }) + t.Run("read only", func(t *testing.T) { + defer func() { recover() }() + TypeAssert[int](ValueOf(&testTypeWithMethod{}).FieldByName("val")) + t.Fatalf("TypeAssert did not panic") + }) +} + +func TestTypeAssertAllocs(t *testing.T) { + if race.Enabled || asan.Enabled || msan.Enabled { + t.Skip("instrumentation breaks this optimization") + } + typeAssertAllocs[[128]int](t, ValueOf([128]int{}), 0) + typeAssertAllocs[any](t, ValueOf([128]int{}), 0) + + val := 123 + typeAssertAllocs[any](t, ValueOf(val), 0) + typeAssertAllocs[any](t, ValueOf(&val).Elem(), 1) // must allocate, so that Set() does not modify the returned inner iface value. + typeAssertAllocs[int](t, ValueOf(val), 0) + typeAssertAllocs[int](t, ValueOf(&val).Elem(), 0) + + typeAssertAllocs[time.Time](t, ValueOf(new(time.Time)).Elem(), 0) + typeAssertAllocs[time.Time](t, ValueOf(*new(time.Time)), 0) + + type I interface{ foo() } + typeAssertAllocs[I](t, ValueOf(new(string)).Elem(), 0) // assert fail doesn't alloc +} + +func typeAssertAllocs[T any](t *testing.T, val Value, wantAllocs int) { + t.Helper() + allocs := testing.AllocsPerRun(10, func() { + TypeAssert[T](val) + }) + if allocs != float64(wantAllocs) { + t.Errorf("TypeAssert[%v](%v) unexpected amount of allocations = %v; want = %v", TypeFor[T](), val.Type(), allocs, wantAllocs) + } +} + +func BenchmarkTypeAssert(b *testing.B) { + benchmarkTypeAssert[int](b, ValueOf(int(1))) + benchmarkTypeAssert[byte](b, ValueOf(int(1))) + + benchmarkTypeAssert[fmt.Stringer](b, ValueOf(testTypeWithMethod{})) + benchmarkTypeAssert[fmt.Stringer](b, ValueOf(&testTypeWithMethod{})) + benchmarkTypeAssert[any](b, ValueOf(int(1))) + benchmarkTypeAssert[any](b, ValueOf(testTypeWithMethod{})) + + benchmarkTypeAssert[time.Time](b, ValueOf(*new(time.Time))) + + benchmarkTypeAssert[func() string](b, ValueOf(time.Now()).MethodByName("String")) +} + +func benchmarkTypeAssert[T any](b *testing.B, val Value) { + b.Run(fmt.Sprintf("TypeAssert[%v](%v)", TypeFor[T](), val.Type()), func(b *testing.B) { + for b.Loop() { + TypeAssert[T](val) + } + }) +} diff --git a/src/reflect/badlinkname.go b/src/reflect/badlinkname.go index eb701bff033d14..8a9bea6721de6b 100644 --- a/src/reflect/badlinkname.go +++ b/src/reflect/badlinkname.go @@ -27,7 +27,7 @@ import ( // //go:linkname unusedIfaceIndir reflect.ifaceIndir func unusedIfaceIndir(t *abi.Type) bool { - return t.Kind_&abi.KindDirectIface == 0 + return !t.IsDirectIface() } //go:linkname valueInterface diff --git a/src/reflect/benchmark_test.go b/src/reflect/benchmark_test.go index 2e701b062edf6f..6b2f9ce7a0381a 100644 --- a/src/reflect/benchmark_test.go +++ b/src/reflect/benchmark_test.go @@ -9,6 +9,7 @@ import ( . "reflect" "strconv" "testing" + "time" ) var sourceAll = struct { @@ -196,6 +197,57 @@ func BenchmarkSetZero(b *testing.B) { } } +// BenchmarkZero overlaps some with BenchmarkSetZero, +// but the inputs are set up differently to exercise +// different optimizations. +func BenchmarkZero(b *testing.B) { + type bm struct { + name string + zero Value + nonZero Value + size int + } + type Small struct { + A int64 + B, C bool + } + type Big struct { + A int64 + B, C bool + D [1008]byte + } + entry := func(name string, zero any, nonZero any) bm { + return bm{name, ValueOf(zero), ValueOf(nonZero).Elem(), int(TypeOf(zero).Size())} + } + nonZeroTime := func() *time.Time { t := time.Now(); return &t } + + bms := []bm{ + entry("ByteArray", [16]byte{}, &[16]byte{1}), + entry("ByteArray", [64]byte{}, &[64]byte{1}), + entry("ByteArray", [1024]byte{}, &[1024]byte{1}), + entry("BigStruct", Big{}, &Big{A: 1}), + entry("SmallStruct", Small{}, &Small{A: 1}), + entry("SmallStructArray", [4]Small{}, &[4]Small{0: {A: 1}}), + entry("SmallStructArray", [64]Small{}, &[64]Small{0: {A: 1}}), + entry("Time", time.Time{}, nonZeroTime()), + } + + for _, bm := range bms { + b.Run(fmt.Sprintf("IsZero/%s/size=%d", bm.name, bm.size), func(b *testing.B) { + for i := 0; i < b.N; i++ { + bm.zero.IsZero() + } + }) + } + for _, bm := range bms { + b.Run(fmt.Sprintf("SetZero/%s/size=%d", bm.name, bm.size), func(b *testing.B) { + for i := 0; i < b.N; i++ { + bm.nonZero.Set(bm.zero) + } + }) + } +} + func BenchmarkSelect(b *testing.B) { channel := make(chan int) close(channel) diff --git a/src/reflect/export_test.go b/src/reflect/export_test.go index 30a0e823afe5f4..fc209fdfbab190 100644 --- a/src/reflect/export_test.go +++ b/src/reflect/export_test.go @@ -58,9 +58,6 @@ func FuncLayout(t Type, rcvr Type) (frametype Type, argSize, retOffset uintptr, inReg = append(inReg, bool2byte(abid.inRegPtrs.Get(i))) outReg = append(outReg, bool2byte(abid.outRegPtrs.Get(i))) } - if ft.Kind_&abi.KindGCProg != 0 { - panic("can't handle gc programs") - } // Expand frame type's GC bitmap into byte-map. ptrs = ft.Pointers() @@ -91,19 +88,6 @@ var GCBits = gcbits func gcbits(any) []byte // provided by runtime -func MapBucketOf(x, y Type) Type { - return toType(bucketOf(x.common(), y.common())) -} - -func CachedBucketOf(m Type) Type { - t := m.(*rtype) - if Kind(t.t.Kind_&abi.KindMask) != Map { - panic("not map") - } - tt := (*mapType)(unsafe.Pointer(t)) - return toType(tt.Bucket) -} - type EmbedWithUnexpMeth struct{} func (EmbedWithUnexpMeth) f() {} @@ -168,3 +152,8 @@ var MethodValueCallCodePtr = methodValueCallCodePtr var InternalIsZero = isZero var IsRegularMemory = isRegularMemory + +func MapGroupOf(x, y Type) Type { + grp, _ := groupAndSlotOf(x, y) + return grp +} diff --git a/src/reflect/iter.go b/src/reflect/iter.go index 36472013cb7c12..03df87b17882ba 100644 --- a/src/reflect/iter.go +++ b/src/reflect/iter.go @@ -4,15 +4,24 @@ package reflect -import "iter" +import ( + "iter" +) func rangeNum[T int8 | int16 | int32 | int64 | int | uint8 | uint16 | uint32 | uint64 | uint | - uintptr, N int64 | uint64](v N) iter.Seq[Value] { + uintptr, N int64 | uint64](num N, t Type) iter.Seq[Value] { return func(yield func(v Value) bool) { + convert := t.PkgPath() != "" // cannot use range T(v) because no core type. - for i := T(0); i < T(v); i++ { - if !yield(ValueOf(i)) { + for i := T(0); i < T(num); i++ { + tmp := ValueOf(i) + // if the iteration value type is define by + // type T built-in type. + if convert { + tmp = tmp.Convert(t) + } + if !yield(tmp) { return } } @@ -27,7 +36,7 @@ func rangeNum[T int8 | int16 | int32 | int64 | int | // Uint, Uint8, Uint16, Uint32, Uint64, Uintptr, // Array, Chan, Map, Slice, or String. func (v Value) Seq() iter.Seq[Value] { - if canRangeFunc(v.typ()) { + if canRangeFunc(v.abiType()) { return func(yield func(Value) bool) { rf := MakeFunc(v.Type().In(0), func(in []Value) []Value { return []Value{ValueOf(yield(in[0]))} @@ -35,29 +44,29 @@ func (v Value) Seq() iter.Seq[Value] { v.Call([]Value{rf}) } } - switch v.Kind() { + switch v.kind() { case Int: - return rangeNum[int](v.Int()) + return rangeNum[int](v.Int(), v.Type()) case Int8: - return rangeNum[int8](v.Int()) + return rangeNum[int8](v.Int(), v.Type()) case Int16: - return rangeNum[int16](v.Int()) + return rangeNum[int16](v.Int(), v.Type()) case Int32: - return rangeNum[int32](v.Int()) + return rangeNum[int32](v.Int(), v.Type()) case Int64: - return rangeNum[int64](v.Int()) + return rangeNum[int64](v.Int(), v.Type()) case Uint: - return rangeNum[uint](v.Uint()) + return rangeNum[uint](v.Uint(), v.Type()) case Uint8: - return rangeNum[uint8](v.Uint()) + return rangeNum[uint8](v.Uint(), v.Type()) case Uint16: - return rangeNum[uint16](v.Uint()) + return rangeNum[uint16](v.Uint(), v.Type()) case Uint32: - return rangeNum[uint32](v.Uint()) + return rangeNum[uint32](v.Uint(), v.Type()) case Uint64: - return rangeNum[uint64](v.Uint()) + return rangeNum[uint64](v.Uint(), v.Type()) case Uintptr: - return rangeNum[uintptr](v.Uint()) + return rangeNum[uintptr](v.Uint(), v.Type()) case Pointer: if v.Elem().kind() != Array { break @@ -113,7 +122,7 @@ func (v Value) Seq() iter.Seq[Value] { // If v's kind is Pointer, the pointer element type must have kind Array. // Otherwise v's kind must be Array, Map, Slice, or String. func (v Value) Seq2() iter.Seq2[Value, Value] { - if canRangeFunc2(v.typ()) { + if canRangeFunc2(v.abiType()) { return func(yield func(Value, Value) bool) { rf := MakeFunc(v.Type().In(0), func(in []Value) []Value { return []Value{ValueOf(yield(in[0], in[1]))} diff --git a/src/reflect/iter_test.go b/src/reflect/iter_test.go index 9b78fcf7247f5f..668d6652802801 100644 --- a/src/reflect/iter_test.go +++ b/src/reflect/iter_test.go @@ -7,10 +7,13 @@ package reflect_test import ( "iter" "maps" + "reflect" . "reflect" "testing" ) +type N int8 + func TestValueSeq(t *testing.T) { m := map[string]int{ "1": 1, @@ -114,10 +117,10 @@ func TestValueSeq(t *testing.T) { }}, {"string", ValueOf("12语言"), func(t *testing.T, s iter.Seq[Value]) { i := int64(0) - indexs := []int64{0, 1, 2, 5} + indexes := []int64{0, 1, 2, 5} for v := range s { - if v.Int() != indexs[i] { - t.Fatalf("got %d, want %d", v.Int(), indexs[i]) + if v.Int() != indexes[i] { + t.Fatalf("got %d, want %d", v.Int(), indexes[i]) } i++ } @@ -126,14 +129,12 @@ func TestValueSeq(t *testing.T) { } }}, {"map[string]int", ValueOf(m), func(t *testing.T, s iter.Seq[Value]) { - i := int64(0) copy := maps.Clone(m) for v := range s { if _, ok := copy[v.String()]; !ok { t.Fatalf("unexpected %v", v.Interface()) } delete(copy, v.String()) - i++ } if len(copy) != 0 { t.Fatalf("should loop four times") @@ -175,6 +176,33 @@ func TestValueSeq(t *testing.T) { t.Fatalf("should loop four times") } }}, + {"method", ValueOf(methodIter{}).MethodByName("Seq"), func(t *testing.T, s iter.Seq[Value]) { + i := int64(0) + for v := range s { + if v.Int() != i { + t.Fatalf("got %d, want %d", v.Int(), i) + } + i++ + } + if i != 4 { + t.Fatalf("should loop four times") + } + }}, + {"type N int8", ValueOf(N(4)), func(t *testing.T, s iter.Seq[Value]) { + i := N(0) + for v := range s { + if v.Int() != int64(i) { + t.Fatalf("got %d, want %d", v.Int(), i) + } + i++ + if v.Type() != reflect.TypeOf(i) { + t.Fatalf("got %s, want %s", v.Type(), reflect.TypeOf(i)) + } + } + if i != 4 { + t.Fatalf("should loop four times") + } + }}, } for _, tc := range tests { seq := tc.val.Seq() @@ -240,11 +268,10 @@ func TestValueSeq2(t *testing.T) { } }}, {"string", ValueOf("12语言"), func(t *testing.T, s iter.Seq2[Value, Value]) { - i := int64(0) - str := "12语言" next, stop := iter.Pull2(s) defer stop() - for j, s := range str { + i := int64(0) + for j, s := range "12语言" { v1, v2, ok := next() if !ok { t.Fatalf("should loop four times") @@ -296,9 +323,90 @@ func TestValueSeq2(t *testing.T) { t.Fatalf("should loop four times") } }}, + {"method", ValueOf(methodIter2{}).MethodByName("Seq2"), func(t *testing.T, s iter.Seq2[Value, Value]) { + i := int64(0) + for v1, v2 := range s { + if v1.Int() != i { + t.Fatalf("got %d, want %d", v1.Int(), i) + } + i++ + if v2.Int() != i { + t.Fatalf("got %d, want %d", v2.Int(), i) + } + } + if i != 4 { + t.Fatalf("should loop four times") + } + }}, + {"[4]N", ValueOf([4]N{0, 1, 2, 3}), func(t *testing.T, s iter.Seq2[Value, Value]) { + i := N(0) + for v1, v2 := range s { + if v1.Int() != int64(i) { + t.Fatalf("got %d, want %d", v1.Int(), i) + } + if v2.Int() != int64(i) { + t.Fatalf("got %d, want %d", v2.Int(), i) + } + i++ + if v2.Type() != reflect.TypeOf(i) { + t.Fatalf("got %s, want %s", v2.Type(), reflect.TypeOf(i)) + } + } + if i != 4 { + t.Fatalf("should loop four times") + } + }}, + {"[]N", ValueOf([]N{1, 2, 3, 4}), func(t *testing.T, s iter.Seq2[Value, Value]) { + i := N(0) + for v1, v2 := range s { + if v1.Int() != int64(i) { + t.Fatalf("got %d, want %d", v1.Int(), i) + } + i++ + if v2.Int() != int64(i) { + t.Fatalf("got %d, want %d", v2.Int(), i) + } + if v2.Type() != reflect.TypeOf(i) { + t.Fatalf("got %s, want %s", v2.Type(), reflect.TypeOf(i)) + } + } + if i != 4 { + t.Fatalf("should loop four times") + } + }}, } for _, tc := range tests { seq := tc.val.Seq2() tc.check(t, seq) } } + +// methodIter is a type from which we can derive a method +// value that is an iter.Seq. +type methodIter struct{} + +func (methodIter) Seq(yield func(int) bool) { + for i := range 4 { + if !yield(i) { + return + } + } +} + +// For Type.CanSeq test. +func (methodIter) NonSeq(yield func(int)) {} + +// methodIter2 is a type from which we can derive a method +// value that is an iter.Seq2. +type methodIter2 struct{} + +func (methodIter2) Seq2(yield func(int, int) bool) { + for i := range 4 { + if !yield(i, i+1) { + return + } + } +} + +// For Type.CanSeq2 test. +func (methodIter2) NonSeq2(yield func(int, int)) {} diff --git a/src/reflect/makefunc.go b/src/reflect/makefunc.go index 5da6cd2ec7d4ea..d35c92a14c7317 100644 --- a/src/reflect/makefunc.go +++ b/src/reflect/makefunc.go @@ -8,6 +8,7 @@ package reflect import ( "internal/abi" + "internal/goarch" "unsafe" ) @@ -164,13 +165,18 @@ func moveMakeFuncArgPtrs(ctxt *makeFuncCtxt, args *abi.RegArgs) { for i, arg := range args.Ints { // Avoid write barriers! Because our write barrier enqueues what // was there before, we might enqueue garbage. + // Also avoid bounds checks, we don't have the stack space for it. + // (Normally the prove pass removes them, but for -N builds we + // use too much stack.) + // ptr := &args.Ptrs[i] (but cast from *unsafe.Pointer to *uintptr) + ptr := (*uintptr)(add(unsafe.Pointer(unsafe.SliceData(args.Ptrs[:])), uintptr(i)*goarch.PtrSize, "always in [0:IntArgRegs]")) if ctxt.regPtrs.Get(i) { - *(*uintptr)(unsafe.Pointer(&args.Ptrs[i])) = arg + *ptr = arg } else { // We *must* zero this space ourselves because it's defined in // assembly code and the GC will scan these pointers. Otherwise, // there will be garbage here. - *(*uintptr)(unsafe.Pointer(&args.Ptrs[i])) = 0 + *ptr = 0 } } } diff --git a/src/reflect/map.go b/src/reflect/map.go new file mode 100644 index 00000000000000..9d25b1818c10ea --- /dev/null +++ b/src/reflect/map.go @@ -0,0 +1,449 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package reflect + +import ( + "internal/abi" + "internal/race" + "internal/runtime/maps" + "internal/runtime/sys" + "unsafe" +) + +func (t *rtype) Key() Type { + if t.Kind() != Map { + panic("reflect: Key of non-map type " + t.String()) + } + tt := (*abi.MapType)(unsafe.Pointer(t)) + return toType(tt.Key) +} + +// MapOf returns the map type with the given key and element types. +// For example, if k represents int and e represents string, +// MapOf(k, e) represents map[int]string. +// +// If the key type is not a valid map key type (that is, if it does +// not implement Go's == operator), MapOf panics. +func MapOf(key, elem Type) Type { + ktyp := key.common() + etyp := elem.common() + + if ktyp.Equal == nil { + panic("reflect.MapOf: invalid key type " + stringFor(ktyp)) + } + + // Look in cache. + ckey := cacheKey{Map, ktyp, etyp, 0} + if mt, ok := lookupCache.Load(ckey); ok { + return mt.(Type) + } + + // Look in known types. + s := "map[" + stringFor(ktyp) + "]" + stringFor(etyp) + for _, tt := range typesByString(s) { + mt := (*abi.MapType)(unsafe.Pointer(tt)) + if mt.Key == ktyp && mt.Elem == etyp { + ti, _ := lookupCache.LoadOrStore(ckey, toRType(tt)) + return ti.(Type) + } + } + + group, slot := groupAndSlotOf(key, elem) + + // Make a map type. + // Note: flag values must match those used in the TMAP case + // in ../cmd/compile/internal/reflectdata/reflect.go:writeType. + var imap any = (map[unsafe.Pointer]unsafe.Pointer)(nil) + mt := **(**abi.MapType)(unsafe.Pointer(&imap)) + mt.Str = resolveReflectName(newName(s, "", false, false)) + mt.TFlag = abi.TFlagDirectIface + mt.Hash = fnv1(etyp.Hash, 'm', byte(ktyp.Hash>>24), byte(ktyp.Hash>>16), byte(ktyp.Hash>>8), byte(ktyp.Hash)) + mt.Key = ktyp + mt.Elem = etyp + mt.Group = group.common() + mt.Hasher = func(p unsafe.Pointer, seed uintptr) uintptr { + return typehash(ktyp, p, seed) + } + mt.GroupSize = mt.Group.Size() + mt.SlotSize = slot.Size() + mt.ElemOff = slot.Field(1).Offset + mt.Flags = 0 + if needKeyUpdate(ktyp) { + mt.Flags |= abi.MapNeedKeyUpdate + } + if hashMightPanic(ktyp) { + mt.Flags |= abi.MapHashMightPanic + } + if ktyp.Size_ > abi.MapMaxKeyBytes { + mt.Flags |= abi.MapIndirectKey + } + if etyp.Size_ > abi.MapMaxKeyBytes { + mt.Flags |= abi.MapIndirectElem + } + mt.PtrToThis = 0 + + ti, _ := lookupCache.LoadOrStore(ckey, toRType(&mt.Type)) + return ti.(Type) +} + +func groupAndSlotOf(ktyp, etyp Type) (Type, Type) { + // type group struct { + // ctrl uint64 + // slots [abi.MapGroupSlots]struct { + // key keyType + // elem elemType + // } + // } + + if ktyp.Size() > abi.MapMaxKeyBytes { + ktyp = PointerTo(ktyp) + } + if etyp.Size() > abi.MapMaxElemBytes { + etyp = PointerTo(etyp) + } + + fields := []StructField{ + { + Name: "Key", + Type: ktyp, + }, + { + Name: "Elem", + Type: etyp, + }, + } + slot := StructOf(fields) + + fields = []StructField{ + { + Name: "Ctrl", + Type: TypeFor[uint64](), + }, + { + Name: "Slots", + Type: ArrayOf(abi.MapGroupSlots, slot), + }, + } + group := StructOf(fields) + return group, slot +} + +var stringType = rtypeOf("") + +// MapIndex returns the value associated with key in the map v. +// It panics if v's Kind is not [Map]. +// It returns the zero Value if key is not found in the map or if v represents a nil map. +// As in Go, the key's value must be assignable to the map's key type. +func (v Value) MapIndex(key Value) Value { + v.mustBe(Map) + tt := (*abi.MapType)(unsafe.Pointer(v.typ())) + + // Do not require key to be exported, so that DeepEqual + // and other programs can use all the keys returned by + // MapKeys as arguments to MapIndex. If either the map + // or the key is unexported, though, the result will be + // considered unexported. This is consistent with the + // behavior for structs, which allow read but not write + // of unexported fields. + + var e unsafe.Pointer + if (tt.Key == stringType || key.kind() == String) && tt.Key == key.typ() && tt.Elem.Size() <= abi.MapMaxElemBytes { + k := *(*string)(key.ptr) + e = mapaccess_faststr(v.typ(), v.pointer(), k) + } else { + key = key.assignTo("reflect.Value.MapIndex", tt.Key, nil) + var k unsafe.Pointer + if key.flag&flagIndir != 0 { + k = key.ptr + } else { + k = unsafe.Pointer(&key.ptr) + } + e = mapaccess(v.typ(), v.pointer(), k) + } + if e == nil { + return Value{} + } + typ := tt.Elem + fl := (v.flag | key.flag).ro() + fl |= flag(typ.Kind()) + return copyVal(typ, fl, e) +} + +// Equivalent to runtime.mapIterStart. +// +//go:noinline +func mapIterStart(t *abi.MapType, m *maps.Map, it *maps.Iter) { + if race.Enabled && m != nil { + callerpc := sys.GetCallerPC() + race.ReadPC(unsafe.Pointer(m), callerpc, abi.FuncPCABIInternal(mapIterStart)) + } + + it.Init(t, m) + it.Next() +} + +// Equivalent to runtime.mapIterNext. +// +//go:noinline +func mapIterNext(it *maps.Iter) { + if race.Enabled { + callerpc := sys.GetCallerPC() + race.ReadPC(unsafe.Pointer(it.Map()), callerpc, abi.FuncPCABIInternal(mapIterNext)) + } + + it.Next() +} + +// MapKeys returns a slice containing all the keys present in the map, +// in unspecified order. +// It panics if v's Kind is not [Map]. +// It returns an empty slice if v represents a nil map. +func (v Value) MapKeys() []Value { + v.mustBe(Map) + tt := (*abi.MapType)(unsafe.Pointer(v.typ())) + keyType := tt.Key + + fl := v.flag.ro() | flag(keyType.Kind()) + + // Escape analysis can't see that the map doesn't escape. It sees an + // escape from maps.IterStart, via assignment into it, even though it + // doesn't escape this function. + mptr := abi.NoEscape(v.pointer()) + m := (*maps.Map)(mptr) + mlen := int(0) + if m != nil { + mlen = maplen(mptr) + } + var it maps.Iter + mapIterStart(tt, m, &it) + a := make([]Value, mlen) + var i int + for i = 0; i < len(a); i++ { + key := it.Key() + if key == nil { + // Someone deleted an entry from the map since we + // called maplen above. It's a data race, but nothing + // we can do about it. + break + } + a[i] = copyVal(keyType, fl, key) + mapIterNext(&it) + } + return a[:i] +} + +// A MapIter is an iterator for ranging over a map. +// See [Value.MapRange]. +type MapIter struct { + m Value + hiter maps.Iter +} + +// Key returns the key of iter's current map entry. +func (iter *MapIter) Key() Value { + if !iter.hiter.Initialized() { + panic("MapIter.Key called before Next") + } + iterkey := iter.hiter.Key() + if iterkey == nil { + panic("MapIter.Key called on exhausted iterator") + } + + t := (*abi.MapType)(unsafe.Pointer(iter.m.typ())) + ktype := t.Key + return copyVal(ktype, iter.m.flag.ro()|flag(ktype.Kind()), iterkey) +} + +// SetIterKey assigns to v the key of iter's current map entry. +// It is equivalent to v.Set(iter.Key()), but it avoids allocating a new Value. +// As in Go, the key must be assignable to v's type and +// must not be derived from an unexported field. +// It panics if [Value.CanSet] returns false. +func (v Value) SetIterKey(iter *MapIter) { + if !iter.hiter.Initialized() { + panic("reflect: Value.SetIterKey called before Next") + } + iterkey := iter.hiter.Key() + if iterkey == nil { + panic("reflect: Value.SetIterKey called on exhausted iterator") + } + + v.mustBeAssignable() + var target unsafe.Pointer + if v.kind() == Interface { + target = v.ptr + } + + t := (*abi.MapType)(unsafe.Pointer(iter.m.typ())) + ktype := t.Key + + iter.m.mustBeExported() // do not let unexported m leak + key := Value{ktype, iterkey, iter.m.flag | flag(ktype.Kind()) | flagIndir} + key = key.assignTo("reflect.MapIter.SetKey", v.typ(), target) + typedmemmove(v.typ(), v.ptr, key.ptr) +} + +// Value returns the value of iter's current map entry. +func (iter *MapIter) Value() Value { + if !iter.hiter.Initialized() { + panic("MapIter.Value called before Next") + } + iterelem := iter.hiter.Elem() + if iterelem == nil { + panic("MapIter.Value called on exhausted iterator") + } + + t := (*abi.MapType)(unsafe.Pointer(iter.m.typ())) + vtype := t.Elem + return copyVal(vtype, iter.m.flag.ro()|flag(vtype.Kind()), iterelem) +} + +// SetIterValue assigns to v the value of iter's current map entry. +// It is equivalent to v.Set(iter.Value()), but it avoids allocating a new Value. +// As in Go, the value must be assignable to v's type and +// must not be derived from an unexported field. +// It panics if [Value.CanSet] returns false. +func (v Value) SetIterValue(iter *MapIter) { + if !iter.hiter.Initialized() { + panic("reflect: Value.SetIterValue called before Next") + } + iterelem := iter.hiter.Elem() + if iterelem == nil { + panic("reflect: Value.SetIterValue called on exhausted iterator") + } + + v.mustBeAssignable() + var target unsafe.Pointer + if v.kind() == Interface { + target = v.ptr + } + + t := (*abi.MapType)(unsafe.Pointer(iter.m.typ())) + vtype := t.Elem + + iter.m.mustBeExported() // do not let unexported m leak + elem := Value{vtype, iterelem, iter.m.flag | flag(vtype.Kind()) | flagIndir} + elem = elem.assignTo("reflect.MapIter.SetValue", v.typ(), target) + typedmemmove(v.typ(), v.ptr, elem.ptr) +} + +// Next advances the map iterator and reports whether there is another +// entry. It returns false when iter is exhausted; subsequent +// calls to [MapIter.Key], [MapIter.Value], or [MapIter.Next] will panic. +func (iter *MapIter) Next() bool { + if !iter.m.IsValid() { + panic("MapIter.Next called on an iterator that does not have an associated map Value") + } + if !iter.hiter.Initialized() { + t := (*abi.MapType)(unsafe.Pointer(iter.m.typ())) + m := (*maps.Map)(iter.m.pointer()) + mapIterStart(t, m, &iter.hiter) + } else { + if iter.hiter.Key() == nil { + panic("MapIter.Next called on exhausted iterator") + } + mapIterNext(&iter.hiter) + } + return iter.hiter.Key() != nil +} + +// Reset modifies iter to iterate over v. +// It panics if v's Kind is not [Map] and v is not the zero Value. +// Reset(Value{}) causes iter to not to refer to any map, +// which may allow the previously iterated-over map to be garbage collected. +func (iter *MapIter) Reset(v Value) { + if v.IsValid() { + v.mustBe(Map) + } + iter.m = v + iter.hiter = maps.Iter{} +} + +// MapRange returns a range iterator for a map. +// It panics if v's Kind is not [Map]. +// +// Call [MapIter.Next] to advance the iterator, and [MapIter.Key]/[MapIter.Value] to access each entry. +// [MapIter.Next] returns false when the iterator is exhausted. +// MapRange follows the same iteration semantics as a range statement. +// +// Example: +// +// iter := reflect.ValueOf(m).MapRange() +// for iter.Next() { +// k := iter.Key() +// v := iter.Value() +// ... +// } +func (v Value) MapRange() *MapIter { + // This is inlinable to take advantage of "function outlining". + // The allocation of MapIter can be stack allocated if the caller + // does not allow it to escape. + // See https://blog.filippo.io/efficient-go-apis-with-the-inliner/ + if v.kind() != Map { + v.panicNotMap() + } + return &MapIter{m: v} +} + +// SetMapIndex sets the element associated with key in the map v to elem. +// It panics if v's Kind is not [Map]. +// If elem is the zero Value, SetMapIndex deletes the key from the map. +// Otherwise if v holds a nil map, SetMapIndex will panic. +// As in Go, key's elem must be assignable to the map's key type, +// and elem's value must be assignable to the map's elem type. +func (v Value) SetMapIndex(key, elem Value) { + v.mustBe(Map) + v.mustBeExported() + key.mustBeExported() + tt := (*abi.MapType)(unsafe.Pointer(v.typ())) + + if (tt.Key == stringType || key.kind() == String) && tt.Key == key.typ() && tt.Elem.Size() <= abi.MapMaxElemBytes { + k := *(*string)(key.ptr) + if elem.typ() == nil { + mapdelete_faststr(v.typ(), v.pointer(), k) + return + } + elem.mustBeExported() + elem = elem.assignTo("reflect.Value.SetMapIndex", tt.Elem, nil) + var e unsafe.Pointer + if elem.flag&flagIndir != 0 { + e = elem.ptr + } else { + e = unsafe.Pointer(&elem.ptr) + } + mapassign_faststr(v.typ(), v.pointer(), k, e) + return + } + + key = key.assignTo("reflect.Value.SetMapIndex", tt.Key, nil) + var k unsafe.Pointer + if key.flag&flagIndir != 0 { + k = key.ptr + } else { + k = unsafe.Pointer(&key.ptr) + } + if elem.typ() == nil { + mapdelete(v.typ(), v.pointer(), k) + return + } + elem.mustBeExported() + elem = elem.assignTo("reflect.Value.SetMapIndex", tt.Elem, nil) + var e unsafe.Pointer + if elem.flag&flagIndir != 0 { + e = elem.ptr + } else { + e = unsafe.Pointer(&elem.ptr) + } + mapassign(v.typ(), v.pointer(), k, e) +} + +// Force slow panicking path not inlined, so it won't add to the +// inlining budget of the caller. +// TODO: undo when the inliner is no longer bottom-up only. +// +//go:noinline +func (f flag) panicNotMap() { + f.mustBe(Map) +} diff --git a/src/reflect/map_test.go b/src/reflect/map_test.go new file mode 100644 index 00000000000000..621b5fdd7306bc --- /dev/null +++ b/src/reflect/map_test.go @@ -0,0 +1,23 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package reflect_test + +import ( + "reflect" + "testing" +) + +// See also runtime_test.TestGroupSizeZero. +func TestGroupSizeZero(t *testing.T) { + st := reflect.TypeFor[struct{}]() + grp := reflect.MapGroupOf(st, st) + + // internal/runtime/maps when create pointers to slots, even if slots + // are size 0. We should have reserved an extra word to ensure that + // pointers to the zero-size type at the end of group are valid. + if grp.Size() <= 8 { + t.Errorf("Group size got %d want >8", grp.Size()) + } +} diff --git a/src/reflect/type.go b/src/reflect/type.go index 07e2bf16447bbb..fc6edb1e106751 100644 --- a/src/reflect/type.go +++ b/src/reflect/type.go @@ -18,6 +18,7 @@ package reflect import ( "internal/abi" "internal/goarch" + "runtime" "strconv" "sync" "unicode" @@ -58,6 +59,9 @@ type Type interface { // method signature, without a receiver, and the Func field is nil. // // Methods are sorted in lexicographic order. + // + // Calling this method will force the linker to retain all exported methods in all packages. + // This may make the executable binary larger but will not affect execution time. Method(int) Method // MethodByName returns the method with that name in the type's @@ -68,6 +72,10 @@ type Type interface { // // For an interface type, the returned Method's Type field gives the // method signature, without a receiver, and the Func field is nil. + // + // Calling this method will cause the linker to retain all methods with this name in all packages. + // If the linker can't determine the name, it will retain all exported methods. + // This may make the executable binary larger but will not affect execution time. MethodByName(string) (Method, bool) // NumMethod returns the number of methods accessible using Method. @@ -262,7 +270,7 @@ type Type interface { /* * These data structures are known to the compiler (../cmd/compile/internal/reflectdata/reflect.go). * A few are known to ../runtime/type.go to convey to debuggers. - * They are also known to ../runtime/type.go. + * They are also known to ../internal/abi/type.go. */ // A Kind represents the specific kind of type that a [Type] represents. @@ -300,6 +308,8 @@ const ( ) // Ptr is the old name for the [Pointer] kind. +// +//go:fix inline const Ptr = Pointer // uncommonType is present only for defined types or types with methods @@ -388,11 +398,6 @@ func (t *interfaceType) uncommon() *abi.UncommonType { return t.Uncommon() } -// mapType represents a map type. -type mapType struct { - abi.MapType -} - // ptrType represents a pointer type. type ptrType struct { abi.PtrType @@ -773,14 +778,6 @@ func (t *rtype) FieldByNameFunc(match func(string) bool) (StructField, bool) { return tt.FieldByNameFunc(match) } -func (t *rtype) Key() Type { - if t.Kind() != Map { - panic("reflect: Key of non-map type " + t.String()) - } - tt := (*mapType)(unsafe.Pointer(t)) - return toType(tt.Key) -} - func (t *rtype) Len() int { if t.Kind() != Array { panic("reflect: Len of non-array type " + t.String()) @@ -1127,17 +1124,35 @@ func (t *structType) Field(i int) (f StructField) { } f.Offset = p.Offset - // NOTE(rsc): This is the only allocation in the interface - // presented by a reflect.Type. It would be nice to avoid, - // at least in the common cases, but we need to make sure - // that misbehaving clients of reflect cannot affect other - // uses of reflect. One possibility is CL 5371098, but we - // postponed that ugliness until there is a demonstrated - // need for the performance. This is issue 2320. - f.Index = []int{i} + // We can't safely use this optimization on js or wasi, + // which do not appear to support read-only data. + if i < 256 && runtime.GOOS != "js" && runtime.GOOS != "wasip1" { + staticuint64s := getStaticuint64s() + p := unsafe.Pointer(&(*staticuint64s)[i]) + if unsafe.Sizeof(int(0)) == 4 && goarch.BigEndian { + p = unsafe.Add(p, 4) + } + f.Index = unsafe.Slice((*int)(p), 1) + } else { + // NOTE(rsc): This is the only allocation in the interface + // presented by a reflect.Type. It would be nice to avoid, + // but we need to make sure that misbehaving clients of + // reflect cannot affect other uses of reflect. + // One possibility is CL 5371098, but we postponed that + // ugliness until there is a demonstrated + // need for the performance. This is issue 2320. + f.Index = []int{i} + } return } +// getStaticuint64s returns a pointer to an array of 256 uint64 values, +// defined in the runtime package in read-only memory. +// staticuint64s[0] == 0, staticuint64s[1] == 1, and so forth. +// +//go:linkname getStaticuint64s runtime.getStaticuint64s +func getStaticuint64s() *[256]uint64 + // TODO(gri): Should there be an error/bool indicator if the index // is wrong for FieldByIndex? @@ -1297,6 +1312,12 @@ func TypeOf(i any) Type { return toType(abi.TypeOf(i)) } +// TypeFor returns the [Type] that represents the type argument T. +func TypeFor[T any]() Type { + // toRType is safe to use here; type is never nil as T is statically known. + return toRType(abi.TypeFor[T]()) +} + // rtypeOf directly extracts the *rtype of the provided value. func rtypeOf(i any) *abi.Type { return abi.TypeOf(i) @@ -1312,6 +1333,8 @@ var ptrMap sync.Map // map[*rtype]*ptrType // The two functions behave identically. // // Deprecated: Superseded by [PointerTo]. +// +//go:fix inline func PtrTo(t Type) Type { return PointerTo(t) } // PointerTo returns the pointer type with element t. @@ -1791,7 +1814,7 @@ func ChanOf(dir ChanDir, t Type) Type { var ichan any = (chan unsafe.Pointer)(nil) prototype := *(**chanType)(unsafe.Pointer(&ichan)) ch := *prototype - ch.TFlag = abi.TFlagRegularMemory + ch.TFlag = abi.TFlagRegularMemory | abi.TFlagDirectIface ch.Dir = abi.ChanDir(dir) ch.Str = resolveReflectName(newName(s, "", false, false)) ch.Hash = fnv1(typ.Hash, 'c', byte(dir)) @@ -1801,79 +1824,6 @@ func ChanOf(dir ChanDir, t Type) Type { return ti.(Type) } -// MapOf returns the map type with the given key and element types. -// For example, if k represents int and e represents string, -// MapOf(k, e) represents map[int]string. -// -// If the key type is not a valid map key type (that is, if it does -// not implement Go's == operator), MapOf panics. -func MapOf(key, elem Type) Type { - ktyp := key.common() - etyp := elem.common() - - if ktyp.Equal == nil { - panic("reflect.MapOf: invalid key type " + stringFor(ktyp)) - } - - // Look in cache. - ckey := cacheKey{Map, ktyp, etyp, 0} - if mt, ok := lookupCache.Load(ckey); ok { - return mt.(Type) - } - - // Look in known types. - s := "map[" + stringFor(ktyp) + "]" + stringFor(etyp) - for _, tt := range typesByString(s) { - mt := (*mapType)(unsafe.Pointer(tt)) - if mt.Key == ktyp && mt.Elem == etyp { - ti, _ := lookupCache.LoadOrStore(ckey, toRType(tt)) - return ti.(Type) - } - } - - // Make a map type. - // Note: flag values must match those used in the TMAP case - // in ../cmd/compile/internal/reflectdata/reflect.go:writeType. - var imap any = (map[unsafe.Pointer]unsafe.Pointer)(nil) - mt := **(**mapType)(unsafe.Pointer(&imap)) - mt.Str = resolveReflectName(newName(s, "", false, false)) - mt.TFlag = 0 - mt.Hash = fnv1(etyp.Hash, 'm', byte(ktyp.Hash>>24), byte(ktyp.Hash>>16), byte(ktyp.Hash>>8), byte(ktyp.Hash)) - mt.Key = ktyp - mt.Elem = etyp - mt.Bucket = bucketOf(ktyp, etyp) - mt.Hasher = func(p unsafe.Pointer, seed uintptr) uintptr { - return typehash(ktyp, p, seed) - } - mt.Flags = 0 - if ktyp.Size_ > abi.MapMaxKeyBytes { - mt.KeySize = uint8(goarch.PtrSize) - mt.Flags |= 1 // indirect key - } else { - mt.KeySize = uint8(ktyp.Size_) - } - if etyp.Size_ > abi.MapMaxElemBytes { - mt.ValueSize = uint8(goarch.PtrSize) - mt.Flags |= 2 // indirect value - } else { - mt.MapType.ValueSize = uint8(etyp.Size_) - } - mt.MapType.BucketSize = uint16(mt.Bucket.Size_) - if isReflexive(ktyp) { - mt.Flags |= 4 - } - if needKeyUpdate(ktyp) { - mt.Flags |= 8 - } - if hashMightPanic(ktyp) { - mt.Flags |= 16 - } - mt.PtrToThis = 0 - - ti, _ := lookupCache.LoadOrStore(ckey, toRType(&mt.Type)) - return ti.(Type) -} - var funcTypes []Type var funcTypesMutex sync.Mutex @@ -1945,7 +1895,7 @@ func FuncOf(in, out []Type, variadic bool) Type { hash = fnv1(hash, byte(t.t.Hash>>24), byte(t.t.Hash>>16), byte(t.t.Hash>>8), byte(t.t.Hash)) } - ft.TFlag = 0 + ft.TFlag = abi.TFlagDirectIface ft.Hash = hash ft.InCount = uint16(len(in)) ft.OutCount = uint16(len(out)) @@ -2106,79 +2056,9 @@ func hashMightPanic(t *abi.Type) bool { } } -func bucketOf(ktyp, etyp *abi.Type) *abi.Type { - if ktyp.Size_ > abi.MapMaxKeyBytes { - ktyp = ptrTo(ktyp) - } - if etyp.Size_ > abi.MapMaxElemBytes { - etyp = ptrTo(etyp) - } - - // Prepare GC data if any. - // A bucket is at most bucketSize*(1+maxKeySize+maxValSize)+ptrSize bytes, - // or 2064 bytes, or 258 pointer-size words, or 33 bytes of pointer bitmap. - // Note that since the key and value are known to be <= 128 bytes, - // they're guaranteed to have bitmaps instead of GC programs. - var gcdata *byte - var ptrdata uintptr - - size := abi.MapBucketCount*(1+ktyp.Size_+etyp.Size_) + goarch.PtrSize - if size&uintptr(ktyp.Align_-1) != 0 || size&uintptr(etyp.Align_-1) != 0 { - panic("reflect: bad size computation in MapOf") - } - - if ktyp.Pointers() || etyp.Pointers() { - nptr := (abi.MapBucketCount*(1+ktyp.Size_+etyp.Size_) + goarch.PtrSize) / goarch.PtrSize - n := (nptr + 7) / 8 - - // Runtime needs pointer masks to be a multiple of uintptr in size. - n = (n + goarch.PtrSize - 1) &^ (goarch.PtrSize - 1) - mask := make([]byte, n) - base := uintptr(abi.MapBucketCount / goarch.PtrSize) - - if ktyp.Pointers() { - emitGCMask(mask, base, ktyp, abi.MapBucketCount) - } - base += abi.MapBucketCount * ktyp.Size_ / goarch.PtrSize - - if etyp.Pointers() { - emitGCMask(mask, base, etyp, abi.MapBucketCount) - } - base += abi.MapBucketCount * etyp.Size_ / goarch.PtrSize - - word := base - mask[word/8] |= 1 << (word % 8) - gcdata = &mask[0] - ptrdata = (word + 1) * goarch.PtrSize - - // overflow word must be last - if ptrdata != size { - panic("reflect: bad layout computation in MapOf") - } - } - - b := &abi.Type{ - Align_: goarch.PtrSize, - Size_: size, - Kind_: abi.Struct, - PtrBytes: ptrdata, - GCData: gcdata, - } - s := "bucket(" + stringFor(ktyp) + "," + stringFor(etyp) + ")" - b.Str = resolveReflectName(newName(s, "", false, false)) - return b -} - -func (t *rtype) gcSlice(begin, end uintptr) []byte { - return (*[1 << 30]byte)(unsafe.Pointer(t.t.GCData))[begin:end:end] -} - // emitGCMask writes the GC mask for [n]typ into out, starting at bit // offset base. func emitGCMask(out []byte, base uintptr, typ *abi.Type, n uintptr) { - if typ.Kind_&abi.KindGCProg != 0 { - panic("reflect: unexpected GC program") - } ptrs := typ.PtrBytes / goarch.PtrSize words := typ.Size_ / goarch.PtrSize mask := typ.GcSlice(0, (ptrs+7)/8) @@ -2192,32 +2072,6 @@ func emitGCMask(out []byte, base uintptr, typ *abi.Type, n uintptr) { } } -// appendGCProg appends the GC program for the first ptrdata bytes of -// typ to dst and returns the extended slice. -func appendGCProg(dst []byte, typ *abi.Type) []byte { - if typ.Kind_&abi.KindGCProg != 0 { - // Element has GC program; emit one element. - n := uintptr(*(*uint32)(unsafe.Pointer(typ.GCData))) - prog := typ.GcSlice(4, 4+n-1) - return append(dst, prog...) - } - - // Element is small with pointer mask; use as literal bits. - ptrs := typ.PtrBytes / goarch.PtrSize - mask := typ.GcSlice(0, (ptrs+7)/8) - - // Emit 120-bit chunks of full bytes (max is 127 but we avoid using partial bytes). - for ; ptrs > 120; ptrs -= 120 { - dst = append(dst, 120) - dst = append(dst, mask[:15]...) - mask = mask[15:] - } - - dst = append(dst, byte(ptrs)) - dst = append(dst, mask...) - return dst -} - // SliceOf returns the slice type with element type t. // For example, if t represents int, SliceOf(t) represents []int. func SliceOf(t Type) Type { @@ -2356,8 +2210,6 @@ func StructOf(fields []StructField) Type { fs = make([]structField, len(fields)) repr = make([]byte, 0, 64) fset = map[string]struct{}{} // fields' names - - hasGCProg = false // records whether a struct-field type has a GCProg ) lastzero := uintptr(0) @@ -2375,9 +2227,6 @@ func StructOf(fields []StructField) Type { } f, fpkgpath := runtimeStructField(field) ft := f.Typ - if ft.Kind_&abi.KindGCProg != 0 { - hasGCProg = true - } if fpkgpath != "" { if pkgpath == "" { pkgpath = fpkgpath @@ -2465,7 +2314,7 @@ func StructOf(fields []StructField) Type { // Issue 15924. panic("reflect: embedded type with methods not implemented if type is not first field") } - if len(fields) > 1 && ft.Kind_&abi.KindDirectIface != 0 { + if len(fields) > 1 && ft.IsDirectIface() { panic("reflect: embedded type with methods not implemented for non-pointer type") } for _, m := range unt.Methods() { @@ -2648,51 +2497,19 @@ func StructOf(fields []StructField) Type { typ.TFlag |= abi.TFlagUncommon } - if hasGCProg { - lastPtrField := 0 - for i, ft := range fs { - if ft.Typ.Pointers() { - lastPtrField = i - } - } - prog := []byte{0, 0, 0, 0} // will be length of prog - var off uintptr - for i, ft := range fs { - if i > lastPtrField { - // gcprog should not include anything for any field after - // the last field that contains pointer data - break - } - if !ft.Typ.Pointers() { - // Ignore pointerless fields. - continue - } - // Pad to start of this field with zeros. - if ft.Offset > off { - n := (ft.Offset - off) / goarch.PtrSize - prog = append(prog, 0x01, 0x00) // emit a 0 bit - if n > 1 { - prog = append(prog, 0x81) // repeat previous bit - prog = appendVarint(prog, n-1) // n-1 times - } - off = ft.Offset - } - - prog = appendGCProg(prog, ft.Typ) - off += ft.Typ.PtrBytes - } - prog = append(prog, 0) - *(*uint32)(unsafe.Pointer(&prog[0])) = uint32(len(prog) - 4) - typ.Kind_ |= abi.KindGCProg - typ.GCData = &prog[0] - } else { - typ.Kind_ &^= abi.KindGCProg + if typ.PtrBytes == 0 { + typ.GCData = nil + } else if typ.PtrBytes <= abi.MaxPtrmaskBytes*8*goarch.PtrSize { bv := new(bitVector) addTypeBits(bv, 0, &typ.Type) - if len(bv.data) > 0 { - typ.GCData = &bv.data[0] - } + typ.GCData = &bv.data[0] + } else { + // Runtime will build the mask if needed. We just need to allocate + // space to store it. + typ.TFlag |= abi.TFlagGCMaskOnDemand + typ.GCData = (*byte)(unsafe.Pointer(new(uintptr))) } + typ.Equal = nil if comparable { typ.Equal = func(p, q unsafe.Pointer) bool { @@ -2708,11 +2525,11 @@ func StructOf(fields []StructField) Type { } switch { - case len(fs) == 1 && !fs[0].Typ.IfaceIndir(): + case len(fs) == 1 && fs[0].Typ.IsDirectIface(): // structs of 1 direct iface type can be direct - typ.Kind_ |= abi.KindDirectIface + typ.TFlag |= abi.TFlagDirectIface default: - typ.Kind_ &^= abi.KindDirectIface + typ.TFlag &^= abi.TFlagDirectIface } return addToCache(toType(&typ.Type)) @@ -2824,6 +2641,8 @@ func ArrayOf(length int, elem Type) Type { array.Size_ = typ.Size_ * uintptr(length) if length > 0 && typ.Pointers() { array.PtrBytes = typ.Size_*uintptr(length-1) + typ.PtrBytes + } else { + array.PtrBytes = 0 } array.Align_ = typ.Align_ array.FieldAlign_ = typ.FieldAlign_ @@ -2831,21 +2650,18 @@ func ArrayOf(length int, elem Type) Type { array.Slice = &(SliceOf(elem).(*rtype).t) switch { - case !typ.Pointers() || array.Size_ == 0: + case array.PtrBytes == 0: // No pointers. array.GCData = nil - array.PtrBytes = 0 case length == 1: // In memory, 1-element array looks just like the element. - array.Kind_ |= typ.Kind_ & abi.KindGCProg + // We share the bitmask with the element type. + array.TFlag |= typ.TFlag & abi.TFlagGCMaskOnDemand array.GCData = typ.GCData - array.PtrBytes = typ.PtrBytes - case typ.Kind_&abi.KindGCProg == 0 && array.Size_ <= abi.MaxPtrmaskBytes*8*goarch.PtrSize: - // Element is small with pointer mask; array is still small. - // Create direct pointer mask by turning each 1 bit in elem - // into length 1 bits in larger mask. + case array.PtrBytes <= abi.MaxPtrmaskBytes*8*goarch.PtrSize: + // Create pointer mask by repeating the element bitmask Len times. n := (array.PtrBytes/goarch.PtrSize + 7) / 8 // Runtime needs pointer masks to be a multiple of uintptr in size. n = (n + goarch.PtrSize - 1) &^ (goarch.PtrSize - 1) @@ -2854,34 +2670,10 @@ func ArrayOf(length int, elem Type) Type { array.GCData = &mask[0] default: - // Create program that emits one element - // and then repeats to make the array. - prog := []byte{0, 0, 0, 0} // will be length of prog - prog = appendGCProg(prog, typ) - // Pad from ptrdata to size. - elemPtrs := typ.PtrBytes / goarch.PtrSize - elemWords := typ.Size_ / goarch.PtrSize - if elemPtrs < elemWords { - // Emit literal 0 bit, then repeat as needed. - prog = append(prog, 0x01, 0x00) - if elemPtrs+1 < elemWords { - prog = append(prog, 0x81) - prog = appendVarint(prog, elemWords-elemPtrs-1) - } - } - // Repeat length-1 times. - if elemWords < 0x80 { - prog = append(prog, byte(elemWords|0x80)) - } else { - prog = append(prog, 0x80) - prog = appendVarint(prog, elemWords) - } - prog = appendVarint(prog, uintptr(length)-1) - prog = append(prog, 0) - *(*uint32)(unsafe.Pointer(&prog[0])) = uint32(len(prog) - 4) - array.Kind_ |= abi.KindGCProg - array.GCData = &prog[0] - array.PtrBytes = array.Size_ // overestimate but ok; must match program + // Runtime will build the mask if needed. We just need to allocate + // space to store it. + array.TFlag |= abi.TFlagGCMaskOnDemand + array.GCData = (*byte)(unsafe.Pointer(new(uintptr))) } etyp := typ @@ -2903,11 +2695,11 @@ func ArrayOf(length int, elem Type) Type { } switch { - case length == 1 && !typ.IfaceIndir(): + case length == 1 && typ.IsDirectIface(): // array of 1 direct iface type can be direct - array.Kind_ |= abi.KindDirectIface + array.TFlag |= abi.TFlagDirectIface default: - array.Kind_ &^= abi.KindDirectIface + array.TFlag &^= abi.TFlagDirectIface } ti, _ := lookupCache.LoadOrStore(ckey, toRType(&array.Type)) @@ -3043,7 +2835,7 @@ func addTypeBits(bv *bitVector, offset uintptr, t *abi.Type) { return } - switch Kind(t.Kind_ & abi.KindMask) { + switch Kind(t.Kind()) { case Chan, Func, Map, Pointer, Slice, String, UnsafePointer: // 1 pointer at start of representation for bv.n < uint32(offset/goarch.PtrSize) { @@ -3075,12 +2867,3 @@ func addTypeBits(bv *bitVector, offset uintptr, t *abi.Type) { } } } - -// TypeFor returns the [Type] that represents the type argument T. -func TypeFor[T any]() Type { - var v T - if t := TypeOf(v); t != nil { - return t // optimize for T being a non-interface kind - } - return TypeOf((*T)(nil)).Elem() // only for an interface kind -} diff --git a/src/reflect/type_test.go b/src/reflect/type_test.go index 51abc0776cd4f1..fc76a4fb9858e5 100644 --- a/src/reflect/type_test.go +++ b/src/reflect/type_test.go @@ -126,6 +126,8 @@ func TestType_CanSeq(t *testing.T) { }{ {"func(func(int) bool)", reflect.TypeOf(func(func(int) bool) {}), true}, {"func(func(int))", reflect.TypeOf(func(func(int)) {}), false}, + {"methodIter.Seq", reflect.ValueOf(methodIter{}).MethodByName("Seq").Type(), true}, + {"methodIter.NonSeq", reflect.ValueOf(methodIter{}).MethodByName("NonSeq").Type(), false}, {"int64", reflect.TypeOf(int64(1)), true}, {"uint64", reflect.TypeOf(uint64(1)), true}, {"*[4]int", reflect.TypeOf(&[4]int{}), true}, @@ -151,6 +153,8 @@ func TestType_CanSeq2(t *testing.T) { }{ {"func(func(int, int) bool)", reflect.TypeOf(func(func(int, int) bool) {}), true}, {"func(func(int, int))", reflect.TypeOf(func(func(int, int)) {}), false}, + {"methodIter2.Seq2", reflect.ValueOf(methodIter2{}).MethodByName("Seq2").Type(), true}, + {"methodIter2.NonSeq2", reflect.ValueOf(methodIter2{}).MethodByName("NonSeq2").Type(), false}, {"int64", reflect.TypeOf(int64(1)), false}, {"uint64", reflect.TypeOf(uint64(1)), false}, {"*[4]int", reflect.TypeOf(&[4]int{}), true}, diff --git a/src/reflect/value.go b/src/reflect/value.go index 8df7d13114d3de..c0ac45de77be85 100644 --- a/src/reflect/value.go +++ b/src/reflect/value.go @@ -55,7 +55,7 @@ type Value struct { // - flagIndir: val holds a pointer to the data // - flagAddr: v.CanAddr is true (implies flagIndir and ptr is non-nil) // - flagMethod: v is a method value. - // If ifaceIndir(typ), code can assume that flagIndir is set. + // If !typ.IsDirectIface(), code can assume that flagIndir is set. // // The remaining 22+ bits give a method number for method values. // If flag.kind() != Func, code can assume that flagMethod is unset. @@ -93,6 +93,9 @@ func (f flag) ro() flag { return 0 } +// typ returns the *abi.Type stored in the Value. This method is fast, +// but it doesn't always return the correct type for the Value. +// See abiType and Type, which do return the correct type. func (v Value) typ() *abi.Type { // Types are either static (for compiler-created types) or // heap-allocated but always reachable (for reflection-created @@ -117,12 +120,18 @@ func (v Value) pointer() unsafe.Pointer { // packEface converts v to the empty interface. func packEface(v Value) any { + return *(*any)(unsafe.Pointer(&abi.EmptyInterface{ + Type: v.typ(), + Data: packEfaceData(v), + })) +} + +// packEfaceData is a helper that packs the Data part of an interface, +// if v were to be stored in an interface. +func packEfaceData(v Value) unsafe.Pointer { t := v.typ() - var i any - e := (*abi.EmptyInterface)(unsafe.Pointer(&i)) - // First, fill in the data portion of the interface. switch { - case t.IfaceIndir(): + case !t.IsDirectIface(): if v.flag&flagIndir == 0 { panic("bad indir") } @@ -133,33 +142,26 @@ func packEface(v Value) any { typedmemmove(t, c, ptr) ptr = c } - e.Data = ptr + return ptr case v.flag&flagIndir != 0: // Value is indirect, but interface is direct. We need // to load the data at v.ptr into the interface data word. - e.Data = *(*unsafe.Pointer)(v.ptr) + return *(*unsafe.Pointer)(v.ptr) default: // Value is direct, and so is the interface. - e.Data = v.ptr + return v.ptr } - // Now, fill in the type portion. We're very careful here not - // to have any operation between the e.word and e.typ assignments - // that would let the garbage collector observe the partially-built - // interface value. - e.Type = t - return i } // unpackEface converts the empty interface i to a Value. func unpackEface(i any) Value { e := (*abi.EmptyInterface)(unsafe.Pointer(&i)) - // NOTE: don't read e.word until we know whether it is really a pointer or not. t := e.Type if t == nil { return Value{} } f := flag(t.Kind()) - if t.IfaceIndir() { + if !t.IsDirectIface() { f |= flagIndir } return Value{t, e.Data, f} @@ -624,7 +626,7 @@ func (v Value) call(op string, in []Value) []Value { } // Handle pointers passed in registers. - if !tv.IfaceIndir() { + if tv.IsDirectIface() { // Pointer-valued data gets put directly // into v.ptr. if steps[0].kind != abiStepPointer { @@ -714,7 +716,7 @@ func callReflect(ctxt *makeFuncImpl, frame unsafe.Pointer, retValid *bool, regs v := Value{typ, nil, flag(typ.Kind())} steps := abid.call.stepsForValue(i) if st := steps[0]; st.kind == abiStepStack { - if typ.IfaceIndir() { + if !typ.IsDirectIface() { // value cannot be inlined in interface data. // Must make a copy, because f might keep a reference to it, // and we cannot let f keep a reference to the stack frame @@ -728,7 +730,7 @@ func callReflect(ctxt *makeFuncImpl, frame unsafe.Pointer, retValid *bool, regs v.ptr = *(*unsafe.Pointer)(add(ptr, st.stkOff, "1-ptr")) } } else { - if typ.IfaceIndir() { + if !typ.IsDirectIface() { // All that's left is values passed in registers that we need to // create space for the values. v.flag |= flagIndir @@ -914,7 +916,7 @@ func storeRcvr(v Value, p unsafe.Pointer) { // the interface data word becomes the receiver word iface := (*nonEmptyInterface)(v.ptr) *(*unsafe.Pointer)(p) = iface.word - } else if v.flag&flagIndir != 0 && !t.IfaceIndir() { + } else if v.flag&flagIndir != 0 && t.IsDirectIface() { *(*unsafe.Pointer)(p) = *(*unsafe.Pointer)(v.ptr) } else { *(*unsafe.Pointer)(p) = v.ptr @@ -1216,15 +1218,7 @@ func (v Value) Elem() Value { k := v.kind() switch k { case Interface: - var eface any - if v.typ().NumMethod() == 0 { - eface = *(*any)(v.ptr) - } else { - eface = (any)(*(*interface { - M() - })(v.ptr)) - } - x := unpackEface(eface) + x := unpackEface(packIfaceValueIntoEmptyIface(v)) if x.flag != 0 { x.flag |= v.flag.ro() } @@ -1232,7 +1226,7 @@ func (v Value) Elem() Value { case Pointer: ptr := v.ptr if v.flag&flagIndir != 0 { - if v.typ().IfaceIndir() { + if !v.typ().IsDirectIface() { // This is a pointer to a not-in-heap object. ptr points to a uintptr // in the heap. That uintptr is the address of a not-in-heap object. // In general, pointers to not-in-heap objects can be total junk. @@ -1497,19 +1491,101 @@ func valueInterface(v Value, safe bool) any { if v.kind() == Interface { // Special case: return the element inside the interface. - // Empty interface has one layout, all interfaces with - // methods have a second layout. - if v.NumMethod() == 0 { - return *(*any)(v.ptr) - } - return *(*interface { - M() - })(v.ptr) + return packIfaceValueIntoEmptyIface(v) } return packEface(v) } +// TypeAssert is semantically equivalent to: +// +// v2, ok := v.Interface().(T) +func TypeAssert[T any](v Value) (T, bool) { + if v.flag == 0 { + panic(&ValueError{"reflect.TypeAssert", Invalid}) + } + if v.flag&flagRO != 0 { + // Do not allow access to unexported values via TypeAssert, + // because they might be pointers that should not be + // writable or methods or function that should not be callable. + panic("reflect.TypeAssert: cannot return value obtained from unexported field or method") + } + + if v.flag&flagMethod != 0 { + v = makeMethodValue("TypeAssert", v) + } + + typ := abi.TypeFor[T]() + + // If v is an interface, return the element inside the interface. + // + // T is a concrete type and v is an interface. For example: + // + // var v any = int(1) + // val := ValueOf(&v).Elem() + // TypeAssert[int](val) == val.Interface().(int) + // + // T is a interface and v is a non-nil interface value. For example: + // + // var v any = &someError{} + // val := ValueOf(&v).Elem() + // TypeAssert[error](val) == val.Interface().(error) + // + // T is a interface and v is a nil interface value. For example: + // + // var v error = nil + // val := ValueOf(&v).Elem() + // TypeAssert[error](val) == val.Interface().(error) + if v.kind() == Interface { + v, ok := packIfaceValueIntoEmptyIface(v).(T) + return v, ok + } + + // If T is an interface and v is a concrete type. For example: + // + // TypeAssert[any](ValueOf(1)) == ValueOf(1).Interface().(any) + // TypeAssert[error](ValueOf(&someError{})) == ValueOf(&someError{}).Interface().(error) + if typ.Kind() == abi.Interface { + // To avoid allocating memory, in case the type assertion fails, + // first do the type assertion with a nil Data pointer. + iface := *(*any)(unsafe.Pointer(&abi.EmptyInterface{Type: v.typ(), Data: nil})) + if out, ok := iface.(T); ok { + // Now populate the Data field properly, we update the Data ptr + // directly to avoid an additional type asertion. We can re-use the + // itab we already got from the runtime (through the previous type assertion). + (*abi.CommonInterface)(unsafe.Pointer(&out)).Data = packEfaceData(v) + return out, true + } + var zero T + return zero, false + } + + // Both v and T must be concrete types. + // The only way for an type-assertion to match is if the types are equal. + if typ != v.typ() { + var zero T + return zero, false + } + if v.flag&flagIndir == 0 { + return *(*T)(unsafe.Pointer(&v.ptr)), true + } + return *(*T)(v.ptr), true +} + +// packIfaceValueIntoEmptyIface converts an interface Value into an empty interface. +// +// Precondition: v.kind() == Interface +func packIfaceValueIntoEmptyIface(v Value) any { + // Empty interface has one layout, all interfaces with + // methods have a second layout. + if v.NumMethod() == 0 { + return *(*any)(v.ptr) + } + return *(*interface { + M() + })(v.ptr) +} + // InterfaceData returns a pair of unspecified uintptr values. // It panics if v's Kind is not Interface. // @@ -1585,6 +1661,9 @@ func (v Value) IsZero() bool { if v.flag&flagIndir == 0 { return v.ptr == nil } + if v.ptr == unsafe.Pointer(&zeroVal[0]) { + return true + } typ := (*abi.ArrayType)(unsafe.Pointer(v.typ())) // If the type is comparable, then compare directly with zero. if typ.Equal != nil && typ.Size() <= abi.ZeroValSize { @@ -1613,6 +1692,9 @@ func (v Value) IsZero() bool { if v.flag&flagIndir == 0 { return v.ptr == nil } + if v.ptr == unsafe.Pointer(&zeroVal[0]) { + return true + } typ := (*abi.StructType)(unsafe.Pointer(v.typ())) // If the type is comparable, then compare directly with zero. if typ.Equal != nil && typ.Size() <= abi.ZeroValSize { @@ -1779,269 +1861,10 @@ func (v Value) lenNonSlice() int { panic(&ValueError{"reflect.Value.Len", v.kind()}) } -var stringType = rtypeOf("") - -// MapIndex returns the value associated with key in the map v. -// It panics if v's Kind is not [Map]. -// It returns the zero Value if key is not found in the map or if v represents a nil map. -// As in Go, the key's value must be assignable to the map's key type. -func (v Value) MapIndex(key Value) Value { - v.mustBe(Map) - tt := (*mapType)(unsafe.Pointer(v.typ())) - - // Do not require key to be exported, so that DeepEqual - // and other programs can use all the keys returned by - // MapKeys as arguments to MapIndex. If either the map - // or the key is unexported, though, the result will be - // considered unexported. This is consistent with the - // behavior for structs, which allow read but not write - // of unexported fields. - - var e unsafe.Pointer - if (tt.Key == stringType || key.kind() == String) && tt.Key == key.typ() && tt.Elem.Size() <= abi.MapMaxElemBytes { - k := *(*string)(key.ptr) - e = mapaccess_faststr(v.typ(), v.pointer(), k) - } else { - key = key.assignTo("reflect.Value.MapIndex", tt.Key, nil) - var k unsafe.Pointer - if key.flag&flagIndir != 0 { - k = key.ptr - } else { - k = unsafe.Pointer(&key.ptr) - } - e = mapaccess(v.typ(), v.pointer(), k) - } - if e == nil { - return Value{} - } - typ := tt.Elem - fl := (v.flag | key.flag).ro() - fl |= flag(typ.Kind()) - return copyVal(typ, fl, e) -} - -// MapKeys returns a slice containing all the keys present in the map, -// in unspecified order. -// It panics if v's Kind is not [Map]. -// It returns an empty slice if v represents a nil map. -func (v Value) MapKeys() []Value { - v.mustBe(Map) - tt := (*mapType)(unsafe.Pointer(v.typ())) - keyType := tt.Key - - fl := v.flag.ro() | flag(keyType.Kind()) - - m := v.pointer() - mlen := int(0) - if m != nil { - mlen = maplen(m) - } - var it hiter - mapiterinit(v.typ(), m, &it) - a := make([]Value, mlen) - var i int - for i = 0; i < len(a); i++ { - key := mapiterkey(&it) - if key == nil { - // Someone deleted an entry from the map since we - // called maplen above. It's a data race, but nothing - // we can do about it. - break - } - a[i] = copyVal(keyType, fl, key) - mapiternext(&it) - } - return a[:i] -} - -// hiter's structure matches runtime.hiter's structure. -// Having a clone here allows us to embed a map iterator -// inside type MapIter so that MapIters can be re-used -// without doing any allocations. -type hiter struct { - key unsafe.Pointer - elem unsafe.Pointer - t unsafe.Pointer - h unsafe.Pointer - buckets unsafe.Pointer - bptr unsafe.Pointer - overflow *[]unsafe.Pointer - oldoverflow *[]unsafe.Pointer - startBucket uintptr - offset uint8 - wrapped bool - B uint8 - i uint8 - bucket uintptr - checkBucket uintptr -} - -func (h *hiter) initialized() bool { - return h.t != nil -} - -// A MapIter is an iterator for ranging over a map. -// See [Value.MapRange]. -type MapIter struct { - m Value - hiter hiter -} - -// Key returns the key of iter's current map entry. -func (iter *MapIter) Key() Value { - if !iter.hiter.initialized() { - panic("MapIter.Key called before Next") - } - iterkey := mapiterkey(&iter.hiter) - if iterkey == nil { - panic("MapIter.Key called on exhausted iterator") - } - - t := (*mapType)(unsafe.Pointer(iter.m.typ())) - ktype := t.Key - return copyVal(ktype, iter.m.flag.ro()|flag(ktype.Kind()), iterkey) -} - -// SetIterKey assigns to v the key of iter's current map entry. -// It is equivalent to v.Set(iter.Key()), but it avoids allocating a new Value. -// As in Go, the key must be assignable to v's type and -// must not be derived from an unexported field. -func (v Value) SetIterKey(iter *MapIter) { - if !iter.hiter.initialized() { - panic("reflect: Value.SetIterKey called before Next") - } - iterkey := mapiterkey(&iter.hiter) - if iterkey == nil { - panic("reflect: Value.SetIterKey called on exhausted iterator") - } - - v.mustBeAssignable() - var target unsafe.Pointer - if v.kind() == Interface { - target = v.ptr - } - - t := (*mapType)(unsafe.Pointer(iter.m.typ())) - ktype := t.Key - - iter.m.mustBeExported() // do not let unexported m leak - key := Value{ktype, iterkey, iter.m.flag | flag(ktype.Kind()) | flagIndir} - key = key.assignTo("reflect.MapIter.SetKey", v.typ(), target) - typedmemmove(v.typ(), v.ptr, key.ptr) -} - -// Value returns the value of iter's current map entry. -func (iter *MapIter) Value() Value { - if !iter.hiter.initialized() { - panic("MapIter.Value called before Next") - } - iterelem := mapiterelem(&iter.hiter) - if iterelem == nil { - panic("MapIter.Value called on exhausted iterator") - } - - t := (*mapType)(unsafe.Pointer(iter.m.typ())) - vtype := t.Elem - return copyVal(vtype, iter.m.flag.ro()|flag(vtype.Kind()), iterelem) -} - -// SetIterValue assigns to v the value of iter's current map entry. -// It is equivalent to v.Set(iter.Value()), but it avoids allocating a new Value. -// As in Go, the value must be assignable to v's type and -// must not be derived from an unexported field. -func (v Value) SetIterValue(iter *MapIter) { - if !iter.hiter.initialized() { - panic("reflect: Value.SetIterValue called before Next") - } - iterelem := mapiterelem(&iter.hiter) - if iterelem == nil { - panic("reflect: Value.SetIterValue called on exhausted iterator") - } - - v.mustBeAssignable() - var target unsafe.Pointer - if v.kind() == Interface { - target = v.ptr - } - - t := (*mapType)(unsafe.Pointer(iter.m.typ())) - vtype := t.Elem - - iter.m.mustBeExported() // do not let unexported m leak - elem := Value{vtype, iterelem, iter.m.flag | flag(vtype.Kind()) | flagIndir} - elem = elem.assignTo("reflect.MapIter.SetValue", v.typ(), target) - typedmemmove(v.typ(), v.ptr, elem.ptr) -} - -// Next advances the map iterator and reports whether there is another -// entry. It returns false when iter is exhausted; subsequent -// calls to [MapIter.Key], [MapIter.Value], or [MapIter.Next] will panic. -func (iter *MapIter) Next() bool { - if !iter.m.IsValid() { - panic("MapIter.Next called on an iterator that does not have an associated map Value") - } - if !iter.hiter.initialized() { - mapiterinit(iter.m.typ(), iter.m.pointer(), &iter.hiter) - } else { - if mapiterkey(&iter.hiter) == nil { - panic("MapIter.Next called on exhausted iterator") - } - mapiternext(&iter.hiter) - } - return mapiterkey(&iter.hiter) != nil -} - -// Reset modifies iter to iterate over v. -// It panics if v's Kind is not [Map] and v is not the zero Value. -// Reset(Value{}) causes iter to not to refer to any map, -// which may allow the previously iterated-over map to be garbage collected. -func (iter *MapIter) Reset(v Value) { - if v.IsValid() { - v.mustBe(Map) - } - iter.m = v - iter.hiter = hiter{} -} - -// MapRange returns a range iterator for a map. -// It panics if v's Kind is not [Map]. -// -// Call [MapIter.Next] to advance the iterator, and [MapIter.Key]/[MapIter.Value] to access each entry. -// [MapIter.Next] returns false when the iterator is exhausted. -// MapRange follows the same iteration semantics as a range statement. -// -// Example: -// -// iter := reflect.ValueOf(m).MapRange() -// for iter.Next() { -// k := iter.Key() -// v := iter.Value() -// ... -// } -func (v Value) MapRange() *MapIter { - // This is inlinable to take advantage of "function outlining". - // The allocation of MapIter can be stack allocated if the caller - // does not allow it to escape. - // See https://blog.filippo.io/efficient-go-apis-with-the-inliner/ - if v.kind() != Map { - v.panicNotMap() - } - return &MapIter{m: v} -} - -// Force slow panicking path not inlined, so it won't add to the -// inlining budget of the caller. -// TODO: undo when the inliner is no longer bottom-up only. -// -//go:noinline -func (f flag) panicNotMap() { - f.mustBe(Map) -} - // copyVal returns a Value containing the map key or value at ptr, // allocating a new variable as needed. func copyVal(typ *abi.Type, fl flag, ptr unsafe.Pointer) Value { - if typ.IfaceIndir() { + if !typ.IsDirectIface() { // Copy result so future changes to the map // won't change the underlying value. c := unsafe_New(typ) @@ -2055,6 +1878,9 @@ func copyVal(typ *abi.Type, fl flag, ptr unsafe.Pointer) Value { // The arguments to a Call on the returned function should not include // a receiver; the returned function will always use v as the receiver. // Method panics if i is out of range or if v is a nil interface value. +// +// Calling this method will force the linker to retain all exported methods in all packages. +// This may make the executable binary larger but will not affect execution time. func (v Value) Method(i int) Value { if v.typ() == nil { panic(&ValueError{"reflect.Value.Method", Invalid}) @@ -2091,6 +1917,10 @@ func (v Value) NumMethod() int { // The arguments to a Call on the returned function should not include // a receiver; the returned function will always use v as the receiver. // It returns the zero Value if no method was found. +// +// Calling this method will cause the linker to retain all methods with this name in all packages. +// If the linker can't determine the name, it will retain all exported methods. +// This may make the executable binary larger but will not affect execution time. func (v Value) MethodByName(name string) Value { if v.typ() == nil { panic(&ValueError{"reflect.Value.MethodByName", Invalid}) @@ -2258,7 +2088,7 @@ func (v Value) recv(nb bool) (val Value, ok bool) { t := tt.Elem val = Value{t, nil, flag(t.Kind())} var p unsafe.Pointer - if t.IfaceIndir() { + if !t.IsDirectIface() { p = unsafe_New(t) val.ptr = p val.flag |= flagIndir @@ -2331,7 +2161,8 @@ func (v Value) SetBool(x bool) { } // SetBytes sets v's underlying value. -// It panics if v's underlying value is not a slice of bytes. +// It panics if v's underlying value is not a slice of bytes +// or if [Value.CanSet] returns false. func (v Value) SetBytes(x []byte) { v.mustBeAssignable() v.mustBe(Slice) @@ -2342,7 +2173,8 @@ func (v Value) SetBytes(x []byte) { } // setRunes sets v's underlying value. -// It panics if v's underlying value is not a slice of runes (int32s). +// It panics if v's underlying value is not a slice of runes (int32s) +// or if [Value.CanSet] returns false. func (v Value) setRunes(x []rune) { v.mustBeAssignable() v.mustBe(Slice) @@ -2353,7 +2185,8 @@ func (v Value) setRunes(x []rune) { } // SetComplex sets v's underlying value to x. -// It panics if v's Kind is not [Complex64] or [Complex128], or if [Value.CanSet] returns false. +// It panics if v's Kind is not [Complex64] or [Complex128], +// or if [Value.CanSet] returns false. func (v Value) SetComplex(x complex128) { v.mustBeAssignable() switch k := v.kind(); k { @@ -2367,7 +2200,8 @@ func (v Value) SetComplex(x complex128) { } // SetFloat sets v's underlying value to x. -// It panics if v's Kind is not [Float32] or [Float64], or if [Value.CanSet] returns false. +// It panics if v's Kind is not [Float32] or [Float64], +// or if [Value.CanSet] returns false. func (v Value) SetFloat(x float64) { v.mustBeAssignable() switch k := v.kind(); k { @@ -2381,7 +2215,8 @@ func (v Value) SetFloat(x float64) { } // SetInt sets v's underlying value to x. -// It panics if v's Kind is not [Int], [Int8], [Int16], [Int32], or [Int64], or if [Value.CanSet] returns false. +// It panics if v's Kind is not [Int], [Int8], [Int16], [Int32], or [Int64], +// or if [Value.CanSet] returns false. func (v Value) SetInt(x int64) { v.mustBeAssignable() switch k := v.kind(); k { @@ -2401,8 +2236,9 @@ func (v Value) SetInt(x int64) { } // SetLen sets v's length to n. -// It panics if v's Kind is not [Slice] or if n is negative or -// greater than the capacity of the slice. +// It panics if v's Kind is not [Slice], or if n is negative or +// greater than the capacity of the slice, +// or if [Value.CanSet] returns false. func (v Value) SetLen(n int) { v.mustBeAssignable() v.mustBe(Slice) @@ -2414,8 +2250,9 @@ func (v Value) SetLen(n int) { } // SetCap sets v's capacity to n. -// It panics if v's Kind is not [Slice] or if n is smaller than the length or -// greater than the capacity of the slice. +// It panics if v's Kind is not [Slice], or if n is smaller than the length or +// greater than the capacity of the slice, +// or if [Value.CanSet] returns false. func (v Value) SetCap(n int) { v.mustBeAssignable() v.mustBe(Slice) @@ -2426,60 +2263,9 @@ func (v Value) SetCap(n int) { s.Cap = n } -// SetMapIndex sets the element associated with key in the map v to elem. -// It panics if v's Kind is not [Map]. -// If elem is the zero Value, SetMapIndex deletes the key from the map. -// Otherwise if v holds a nil map, SetMapIndex will panic. -// As in Go, key's elem must be assignable to the map's key type, -// and elem's value must be assignable to the map's elem type. -func (v Value) SetMapIndex(key, elem Value) { - v.mustBe(Map) - v.mustBeExported() - key.mustBeExported() - tt := (*mapType)(unsafe.Pointer(v.typ())) - - if (tt.Key == stringType || key.kind() == String) && tt.Key == key.typ() && tt.Elem.Size() <= abi.MapMaxElemBytes { - k := *(*string)(key.ptr) - if elem.typ() == nil { - mapdelete_faststr(v.typ(), v.pointer(), k) - return - } - elem.mustBeExported() - elem = elem.assignTo("reflect.Value.SetMapIndex", tt.Elem, nil) - var e unsafe.Pointer - if elem.flag&flagIndir != 0 { - e = elem.ptr - } else { - e = unsafe.Pointer(&elem.ptr) - } - mapassign_faststr(v.typ(), v.pointer(), k, e) - return - } - - key = key.assignTo("reflect.Value.SetMapIndex", tt.Key, nil) - var k unsafe.Pointer - if key.flag&flagIndir != 0 { - k = key.ptr - } else { - k = unsafe.Pointer(&key.ptr) - } - if elem.typ() == nil { - mapdelete(v.typ(), v.pointer(), k) - return - } - elem.mustBeExported() - elem = elem.assignTo("reflect.Value.SetMapIndex", tt.Elem, nil) - var e unsafe.Pointer - if elem.flag&flagIndir != 0 { - e = elem.ptr - } else { - e = unsafe.Pointer(&elem.ptr) - } - mapassign(v.typ(), v.pointer(), k, e) -} - // SetUint sets v's underlying value to x. -// It panics if v's Kind is not [Uint], [Uintptr], [Uint8], [Uint16], [Uint32], or [Uint64], or if [Value.CanSet] returns false. +// It panics if v's Kind is not [Uint], [Uintptr], [Uint8], [Uint16], [Uint32], or [Uint64], +// or if [Value.CanSet] returns false. func (v Value) SetUint(x uint64) { v.mustBeAssignable() switch k := v.kind(); k { @@ -2501,7 +2287,8 @@ func (v Value) SetUint(x uint64) { } // SetPointer sets the [unsafe.Pointer] value v to x. -// It panics if v's Kind is not [UnsafePointer]. +// It panics if v's Kind is not [UnsafePointer] +// or if [Value.CanSet] returns false. func (v Value) SetPointer(x unsafe.Pointer) { v.mustBeAssignable() v.mustBe(UnsafePointer) @@ -2682,14 +2469,26 @@ func (v Value) Type() Type { return v.typeSlow() } +//go:noinline func (v Value) typeSlow() Type { + return toRType(v.abiTypeSlow()) +} + +func (v Value) abiType() *abi.Type { + if v.flag != 0 && v.flag&flagMethod == 0 { + return v.typ() + } + return v.abiTypeSlow() +} + +func (v Value) abiTypeSlow() *abi.Type { if v.flag == 0 { panic(&ValueError{"reflect.Value.Type", Invalid}) } typ := v.typ() if v.flag&flagMethod == 0 { - return toRType(v.typ()) + return v.typ() } // Method value. @@ -2702,7 +2501,7 @@ func (v Value) typeSlow() Type { panic("reflect: internal error: invalid method index") } m := &tt.Methods[i] - return toRType(typeOffFor(typ, m.Typ)) + return typeOffFor(typ, m.Typ) } // Method on concrete type. ms := typ.ExportedMethods() @@ -2710,7 +2509,7 @@ func (v Value) typeSlow() Type { panic("reflect: internal error: invalid method index") } m := ms[i] - return toRType(typeOffFor(typ, m.Mtyp)) + return typeOffFor(typ, m.Mtyp) } // CanUint reports whether [Value.Uint] can be used without panicking. @@ -2869,8 +2668,8 @@ func arrayAt(p unsafe.Pointer, i int, eltSize uintptr, whySafe string) unsafe.Po // another n elements. After Grow(n), at least n elements can be appended // to the slice without another allocation. // -// It panics if v's Kind is not a [Slice] or if n is negative or too large to -// allocate the memory. +// It panics if v's Kind is not a [Slice], or if n is negative or too large to +// allocate the memory, or if [Value.CanSet] returns false. func (v Value) Grow(n int) { v.mustBeAssignable() v.mustBe(Slice) @@ -2958,6 +2757,7 @@ func AppendSlice(s, t Value) Value { // It returns the number of elements copied. // Dst and src each must have kind [Slice] or [Array], and // dst and src must have the same element type. +// It dst is an [Array], it panics if [Value.CanSet] returns false. // // As a special case, src can have kind [String] if the element type of dst is kind [Uint8]. func Copy(dst, src Value) int { @@ -3164,7 +2964,7 @@ func Select(cases []SelectCase) (chosen int, recv Value, recvOK bool) { t := tt.Elem p := runcases[chosen].val fl := flag(t.Kind()) - if t.IfaceIndir() { + if !t.IsDirectIface() { recv = Value{t, p, fl | flagIndir} } else { recv = Value{t, *(*unsafe.Pointer)(p), fl} @@ -3277,7 +3077,7 @@ func Zero(typ Type) Value { } t := &typ.(*rtype).t fl := flag(t.Kind()) - if t.IfaceIndir() { + if !t.IsDirectIface() { var p unsafe.Pointer if t.Size() <= abi.ZeroValSize { p = unsafe.Pointer(&zeroVal[0]) @@ -3300,7 +3100,7 @@ func New(typ Type) Value { } t := &typ.(*rtype).t pt := ptrTo(t) - if pt.IfaceIndir() { + if !pt.IsDirectIface() { // This is a pointer to a not-in-heap type. panic("reflect: New of type that may not be allocated in heap (possibly undefined cgo C type)") } @@ -3904,18 +3704,6 @@ func mapdelete(t *abi.Type, m unsafe.Pointer, key unsafe.Pointer) //go:noescape func mapdelete_faststr(t *abi.Type, m unsafe.Pointer, key string) -//go:noescape -func mapiterinit(t *abi.Type, m unsafe.Pointer, it *hiter) - -//go:noescape -func mapiterkey(it *hiter) (key unsafe.Pointer) - -//go:noescape -func mapiterelem(it *hiter) (elem unsafe.Pointer) - -//go:noescape -func mapiternext(it *hiter) - //go:noescape func maplen(m unsafe.Pointer) int diff --git a/src/reflect/visiblefields_test.go b/src/reflect/visiblefields_test.go index 66d545dd1f7c2c..eca0925f966283 100644 --- a/src/reflect/visiblefields_test.go +++ b/src/reflect/visiblefields_test.go @@ -292,7 +292,6 @@ type Rec2 struct { func TestFields(t *testing.T) { for _, test := range fieldsTests { - test := test t.Run(test.testName, func(t *testing.T) { typ := TypeOf(test.val) fields := VisibleFields(typ) diff --git a/src/regexp/all_test.go b/src/regexp/all_test.go index c9c046b61d09be..ead184d2868b02 100644 --- a/src/regexp/all_test.go +++ b/src/regexp/all_test.go @@ -965,6 +965,21 @@ func TestUnmarshalText(t *testing.T) { if unmarshaled.String() != goodRe[i] { t.Errorf("UnmarshalText returned unexpected value: %s", unmarshaled.String()) } + + buf := make([]byte, 4, 32) + marshalAppend, err := re.AppendText(buf) + if err != nil { + t.Errorf("regexp %#q failed to marshal: %s", re, err) + continue + } + marshalAppend = marshalAppend[4:] + if err := unmarshaled.UnmarshalText(marshalAppend); err != nil { + t.Errorf("regexp %#q failed to unmarshal: %s", re, err) + continue + } + if unmarshaled.String() != goodRe[i] { + t.Errorf("UnmarshalText returned unexpected value: %s", unmarshaled.String()) + } } t.Run("invalid pattern", func(t *testing.T) { re := new(Regexp) diff --git a/src/regexp/find_test.go b/src/regexp/find_test.go index 2edbe9b86e6154..49e9619cef9bda 100644 --- a/src/regexp/find_test.go +++ b/src/regexp/find_test.go @@ -98,6 +98,8 @@ var findTests = []FindTest{ {`\B`, "x y", nil}, {`\B`, "xx yy", build(2, 1, 1, 4, 4)}, {`(|a)*`, "aa", build(3, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2)}, + {`0A|0[aA]`, "0a", build(1, 0, 2)}, + {`0[aA]|0A`, "0a", build(1, 0, 2)}, // RE2 tests {`[^\S\s]`, "abcd", nil}, diff --git a/src/regexp/regexp.go b/src/regexp/regexp.go index e06099425ebcaf..66c73693995a42 100644 --- a/src/regexp/regexp.go +++ b/src/regexp/regexp.go @@ -384,10 +384,6 @@ type inputString struct { func (i *inputString) step(pos int) (rune, int) { if pos < len(i.str) { - c := i.str[pos] - if c < utf8.RuneSelf { - return rune(c), 1 - } return utf8.DecodeRuneInString(i.str[pos:]) } return endOfText, 0 @@ -409,17 +405,11 @@ func (i *inputString) context(pos int) lazyFlag { r1, r2 := endOfText, endOfText // 0 < pos && pos <= len(i.str) if uint(pos-1) < uint(len(i.str)) { - r1 = rune(i.str[pos-1]) - if r1 >= utf8.RuneSelf { - r1, _ = utf8.DecodeLastRuneInString(i.str[:pos]) - } + r1, _ = utf8.DecodeLastRuneInString(i.str[:pos]) } // 0 <= pos && pos < len(i.str) if uint(pos) < uint(len(i.str)) { - r2 = rune(i.str[pos]) - if r2 >= utf8.RuneSelf { - r2, _ = utf8.DecodeRuneInString(i.str[pos:]) - } + r2, _ = utf8.DecodeRuneInString(i.str[pos:]) } return newLazyFlag(r1, r2) } @@ -431,10 +421,6 @@ type inputBytes struct { func (i *inputBytes) step(pos int) (rune, int) { if pos < len(i.str) { - c := i.str[pos] - if c < utf8.RuneSelf { - return rune(c), 1 - } return utf8.DecodeRune(i.str[pos:]) } return endOfText, 0 @@ -456,17 +442,11 @@ func (i *inputBytes) context(pos int) lazyFlag { r1, r2 := endOfText, endOfText // 0 < pos && pos <= len(i.str) if uint(pos-1) < uint(len(i.str)) { - r1 = rune(i.str[pos-1]) - if r1 >= utf8.RuneSelf { - r1, _ = utf8.DecodeLastRune(i.str[:pos]) - } + r1, _ = utf8.DecodeLastRune(i.str[:pos]) } // 0 <= pos && pos < len(i.str) if uint(pos) < uint(len(i.str)) { - r2 = rune(i.str[pos]) - if r2 >= utf8.RuneSelf { - r2, _ = utf8.DecodeRune(i.str[pos:]) - } + r2, _ = utf8.DecodeRune(i.str[pos:]) } return newLazyFlag(r1, r2) } @@ -1277,14 +1257,22 @@ func (re *Regexp) Split(s string, n int) []string { return strings } -// MarshalText implements [encoding.TextMarshaler]. The output +// AppendText implements [encoding.TextAppender]. The output // matches that of calling the [Regexp.String] method. // // Note that the output is lossy in some cases: This method does not indicate // POSIX regular expressions (i.e. those compiled by calling [CompilePOSIX]), or // those for which the [Regexp.Longest] method has been called. +func (re *Regexp) AppendText(b []byte) ([]byte, error) { + return append(b, re.String()...), nil +} + +// MarshalText implements [encoding.TextMarshaler]. The output +// matches that of calling the [Regexp.AppendText] method. +// +// See [Regexp.AppendText] for more information. func (re *Regexp) MarshalText() ([]byte, error) { - return []byte(re.String()), nil + return re.AppendText(nil) } // UnmarshalText implements [encoding.TextUnmarshaler] by calling diff --git a/src/regexp/syntax/doc.go b/src/regexp/syntax/doc.go index 877f1043ddda8b..8a7d9992a2cfb8 100644 --- a/src/regexp/syntax/doc.go +++ b/src/regexp/syntax/doc.go @@ -137,6 +137,7 @@ ASCII character classes: [[:word:]] word characters (== [0-9A-Za-z_]) [[:xdigit:]] hex digit (== [0-9A-Fa-f]) -Unicode character classes are those in [unicode.Categories] and [unicode.Scripts]. +Unicode character classes are those in [unicode.Categories], +[unicode.CategoryAliases], and [unicode.Scripts]. */ package syntax diff --git a/src/regexp/syntax/parse.go b/src/regexp/syntax/parse.go index 26242902f1eb26..b77a7dab8e2e60 100644 --- a/src/regexp/syntax/parse.go +++ b/src/regexp/syntax/parse.go @@ -7,6 +7,7 @@ package syntax import ( "sort" "strings" + "sync" "unicode" "unicode/utf8" ) @@ -621,7 +622,7 @@ func (p *parser) factor(sub []*Regexp) []*Regexp { } // Found end of a run with common leading literal string: - // sub[start:i] all begin with str[0:len(str)], but sub[i] + // sub[start:i] all begin with str[:len(str)], but sub[i] // does not even begin with str[0]. // // Factor out common string and append factored expression to out. @@ -1639,20 +1640,109 @@ var anyTable = &unicode.RangeTable{ R32: []unicode.Range32{{Lo: 1 << 16, Hi: unicode.MaxRune, Stride: 1}}, } +var asciiTable = &unicode.RangeTable{ + R16: []unicode.Range16{{Lo: 0, Hi: 0x7F, Stride: 1}}, +} + +var asciiFoldTable = &unicode.RangeTable{ + R16: []unicode.Range16{ + {Lo: 0, Hi: 0x7F, Stride: 1}, + {Lo: 0x017F, Hi: 0x017F, Stride: 1}, // Old English long s (ſ), folds to S/s. + {Lo: 0x212A, Hi: 0x212A, Stride: 1}, // Kelvin K, folds to K/k. + }, +} + +// categoryAliases is a lazily constructed copy of unicode.CategoryAliases +// but with the keys passed through canonicalName, to support inexact matches. +var categoryAliases struct { + once sync.Once + m map[string]string +} + +// initCategoryAliases initializes categoryAliases by canonicalizing unicode.CategoryAliases. +func initCategoryAliases() { + categoryAliases.m = make(map[string]string) + for name, actual := range unicode.CategoryAliases { + categoryAliases.m[canonicalName(name)] = actual + } +} + +// canonicalName returns the canonical lookup string for name. +// The canonical name has a leading uppercase letter and then lowercase letters, +// and it omits all underscores, spaces, and hyphens. +// (We could have used all lowercase, but this way most package unicode +// map keys are already canonical.) +func canonicalName(name string) string { + var b []byte + first := true + for i := range len(name) { + c := name[i] + switch { + case c == '_' || c == '-' || c == ' ': + c = ' ' + case first: + if 'a' <= c && c <= 'z' { + c -= 'a' - 'A' + } + first = false + default: + if 'A' <= c && c <= 'Z' { + c += 'a' - 'A' + } + } + if b == nil { + if c == name[i] && c != ' ' { + // No changes so far, avoid allocating b. + continue + } + b = make([]byte, i, len(name)) + copy(b, name[:i]) + } + if c == ' ' { + continue + } + b = append(b, c) + } + if b == nil { + return name + } + return string(b) +} + // unicodeTable returns the unicode.RangeTable identified by name // and the table of additional fold-equivalent code points. -func unicodeTable(name string) (*unicode.RangeTable, *unicode.RangeTable) { - // Special case: "Any" means any. - if name == "Any" { - return anyTable, anyTable +// If sign < 0, the result should be inverted. +func unicodeTable(name string) (tab, fold *unicode.RangeTable, sign int) { + name = canonicalName(name) + + // Special cases: Any, Assigned, and ASCII. + // Also LC is the only non-canonical Categories key, so handle it here. + switch name { + case "Any": + return anyTable, anyTable, +1 + case "Assigned": + return unicode.Cn, unicode.Cn, -1 // invert Cn (unassigned) + case "Ascii": + return asciiTable, asciiFoldTable, +1 + case "Lc": + return unicode.Categories["LC"], unicode.FoldCategory["LC"], +1 } if t := unicode.Categories[name]; t != nil { - return t, unicode.FoldCategory[name] + return t, unicode.FoldCategory[name], +1 } if t := unicode.Scripts[name]; t != nil { - return t, unicode.FoldScript[name] + return t, unicode.FoldScript[name], +1 + } + + // unicode.CategoryAliases makes liberal use of underscores in its names + // (they are defined that way by Unicode), but we want to match ignoring + // the underscores, so make our own map with canonical names. + categoryAliases.once.Do(initCategoryAliases) + if actual := categoryAliases.m[name]; actual != "" { + t := unicode.Categories[actual] + return t, unicode.FoldCategory[actual], +1 } - return nil, nil + return nil, nil, 0 } // parseUnicodeClass parses a leading Unicode character class like \p{Han} @@ -1700,10 +1790,13 @@ func (p *parser) parseUnicodeClass(s string, r []rune) (out []rune, rest string, name = name[1:] } - tab, fold := unicodeTable(name) + tab, fold, tsign := unicodeTable(name) if tab == nil { return nil, "", &Error{ErrInvalidCharRange, seq} } + if tsign < 0 { + sign = -sign + } if p.flags&FoldCase == 0 || fold == nil { if sign > 0 { diff --git a/src/regexp/syntax/parse_test.go b/src/regexp/syntax/parse_test.go index 0f885bd5c8149f..9d2f698e258d19 100644 --- a/src/regexp/syntax/parse_test.go +++ b/src/regexp/syntax/parse_test.go @@ -107,10 +107,16 @@ var parseTests = []parseTest{ {`[\P{^Braille}]`, `cc{0x2800-0x28ff}`}, {`[\pZ]`, `cc{0x20 0xa0 0x1680 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}`}, {`\p{Lu}`, mkCharClass(unicode.IsUpper)}, + {`\p{Uppercase_Letter}`, mkCharClass(unicode.IsUpper)}, + {`\p{upper case-let ter}`, mkCharClass(unicode.IsUpper)}, + {`\p{__upper case-let ter}`, mkCharClass(unicode.IsUpper)}, {`[\p{Lu}]`, mkCharClass(unicode.IsUpper)}, {`(?i)[\p{Lu}]`, mkCharClass(isUpperFold)}, {`\p{Any}`, `dot{}`}, {`\p{^Any}`, `cc{}`}, + {`(?i)\p{ascii}`, `cc{0x0-0x7f 0x17f 0x212a}`}, + {`\p{Assigned}`, mkCharClass(func(r rune) bool { return !unicode.In(r, unicode.Cn) })}, + {`\p{^Assigned}`, mkCharClass(func(r rune) bool { return unicode.In(r, unicode.Cn) })}, // Hex, octal. {`[\012-\234]\141`, `cat{cc{0xa-0x9c}lit{a}}`}, diff --git a/src/regexp/syntax/regexp.go b/src/regexp/syntax/regexp.go index f15d2051230464..499492884ef28e 100644 --- a/src/regexp/syntax/regexp.go +++ b/src/regexp/syntax/regexp.go @@ -76,7 +76,7 @@ func (x *Regexp) Equal(y *Regexp) bool { } case OpLiteral, OpCharClass: - return slices.Equal(x.Rune, y.Rune) + return x.Flags&FoldCase == y.Flags&FoldCase && slices.Equal(x.Rune, y.Rune) case OpAlternate, OpConcat: return slices.EqualFunc(x.Sub, y.Sub, (*Regexp).Equal) diff --git a/src/run.bat b/src/run.bat index 35c8ead8cb2247..b6a101b2ff0fae 100644 --- a/src/run.bat +++ b/src/run.bat @@ -4,39 +4,17 @@ @echo off -if exist ..\bin\go.exe goto ok -echo Must run run.bat from Go src directory after installing cmd/go. -goto fail -:ok +if not exist ..\bin\go.exe ( + echo Must run run.bat from Go src directory after installing cmd/go. + exit /b 1 +) -:: Keep environment variables within this script -:: unless invoked with --no-local. -if x%1==x--no-local goto nolocal -if x%2==x--no-local goto nolocal setlocal -:nolocal - -set GOBUILDFAIL=0 set GOENV=off -..\bin\go tool dist env > env.bat -if errorlevel 1 goto fail +..\bin\go tool dist env > env.bat || exit /b 1 call .\env.bat del env.bat set GOPATH=c:\nonexist-gopath - -if x%1==x--no-rebuild goto norebuild -..\bin\go tool dist test --rebuild -if errorlevel 1 goto fail -goto end - -:norebuild -..\bin\go tool dist test -if errorlevel 1 goto fail -goto end - -:fail -set GOBUILDFAIL=1 - -:end +..\bin\go tool dist test --rebuild %* || exit /b 1 diff --git a/src/runtime/HACKING.md b/src/runtime/HACKING.md index f0c60f3af9e131..c53779a5882f4c 100644 --- a/src/runtime/HACKING.md +++ b/src/runtime/HACKING.md @@ -266,6 +266,153 @@ If memory is already in a type-safe state and is simply being set to the zero value, this must be done using regular writes, `typedmemclr`, or `memclrHasPointers`. This performs write barriers. +Linkname conventions +==================== + +``` +//go:linkname localname [importpath.name] +``` + +`//go:linkname` specifies the symbol name (`importpath.name`) used to a +reference a local identifier (`localname`). The target symbol name is an +arbitrary ELF/macho/etc symbol name, but by convention we typically use a +package-prefixed symbol name to keep things organized. + +The full generality of `//go:linkname` is very flexible, so as a convention to +simplify things, we define three standard forms of `//go:linkname` directives. + +When possible, always prefer to use the linkname "handshake" described below. + +"Push linkname" +--------------- + +A "push" linkname gives a local _definition_ a final symbol name in a different +package. This effectively "pushes" the symbol to the other package. + +``` +//go:linkname foo otherpkg.foo +func foo() { + // impl +} +``` + +The other package needs a _declaration_ to use the symbol from Go, or it can +directly reference the symbol in assembly. Typically this is an "export +linkname" declaration (below). + +"Pull linkname" +--------------- + +A "pull" linkname gives references to a local _declaration_ a final symbol name +in a different package. This effectively "pulls" the symbol from the other +package. + +``` +//go:linkname foo otherpkg.foo +func foo() +``` + +The other package simply needs to define the symbol, but typically this is a +"export linkname" definition (below). + +"Export linkname" +----------------- + +The second argument to `//go:linkname` is the target symbol name. If it is +omitted, the toolchain uses the default symbol name. In other words, this is a +linkname to itself. This seems to be a no-op, but it is used to mean that this +symbol is "exported" for use with another linkname. + +``` +//go:linkname foo +func foo() { + // impl +} +``` + +When applied to a definition, an export linkname indicates that another package +has a pull linkname targeting this symbol. This has a few effects: + +- The compiler avoids generates ABI wrappers for ABI0 and/or ABIInternal, so a + symbol defined in Go can be referenced from assembly in another package, or + vice versa. +- The linker will allow pull linknames to this symbol even with + `-checklinkname=true` (see "Handshake" section below). + +``` +//go:linkname foo +func foo() +``` + +When applied to a declaration, an export linkname indicates that another package +has a push linkname targeting this symbol. Other than documentation, the only +effect this has on the toolchain is that the compiler will not require a `.s` +file in the package (normally the compiler requires a `.s` file when there are +function declarations without a body). + +Handshake +--------- + +We always prefer to use push linknames rather than pull linknames. With a push +linkname, the package with the definition is aware it is publishing an API to +another package. On the other hand, with a pull linkname, the definition +package may be completely unaware of the dependency and may unintentionally +break users. + +The preferred form for a linkname is to use a push linkname in the defining +package, and a target linkname in the receiving package. The latter is not +strictly required, but serves as documentation. By convention, the receiving +package names the symbol containing the source package to further aid +documentation. + +``` +package runtime + +//go:linkname foo otherpkg.runtime_foo +func foo() { + // impl +} +``` + +``` +package otherpkg + +//go:linkname runtime_foo +func runtime_foo() +``` + +As of Go 1.23, the linker forbids pull linknames of symbols in the standard +library unless they participate in a handshake. Since many third-party packages +already have pull linknames to standard library functions, for backwards +compatibility, standard library symbols that are the target of external pull +linknames must use a target linkname to signal to the linker that pull +linknames are acceptable. + +``` +package runtime + +//go:linkname fastrand +func fastrand() { + // impl +} +``` + +Note that linker enforcement can be disabled with the `-checklinkname=false` +flag. + +Variables +--------- + +All of the examples above use `//go:linkname` on functions. It is also possible +to use it on global variables as well, though this is much less common. + +Variables don't have a clear distinction between definition and declaration. As +a rule, only one side should have a non-zero initial value. That side is the +"definition" and the other is the "declaration". + +Both sides should have the same type, including size. Though if one side is +larger than another, the linker allocates space for the larger size. + Runtime-only compiler directives ================================ @@ -330,3 +477,69 @@ transitive calls) to prevent stack growth. The conversion from pointer to uintptr must appear in the argument list of any call to this function. This directive is used for some low-level system call implementations. + +Execution tracer +================ + +The execution tracer is a way for users to see what their goroutines are doing, +but they're also useful for runtime hacking. + +Using execution traces to debug runtime problems +------------------------------------------------ + +Execution traces contain a wealth of information about what the runtime is +doing. They contain all goroutine scheduling actions, data about time spent in +the scheduler (P running without a G), data about time spent in the garbage +collector, and more. Use `go tool trace` or [gotraceui](https://gotraceui.dev) +to inspect traces. + +Traces are especially useful for debugging latency issues, and especially if you +can catch the problem in the act. Consider using the flight recorder to help +with this. + +Turn on CPU profiling when you take a trace. This will put the CPU profiling +samples as timestamped events into the trace, allowing you to see execution with +greater detail. If you see CPU profiling sample events appear at a rate that does +not match the sample rate, consider that the OS or platform might be taking away +CPU time from the process, and that you might not be debugging a Go issue. + +If you're really stuck on a problem, adding new instrumentation with the tracer +might help, especially if it's helpful to see events in relation to other +scheduling events. See the next section on modifying the execution tracer. +However, consider using `debuglog` for additional instrumentation first, as that +is far easier to get started with. + +Notes on modifying the execution tracer +--------------------------------------- + +The execution tracer lives in the files whose names start with "trace." +The parser for the execution trace format lives in the `internal/trace` package. + +If you plan on adding new trace events, consider starting with a [trace +experiment](../internal/trace/tracev2/EXPERIMENTS.md). + +If you plan to add new trace instrumentation to the runtime, wrap whatever operation +you're tracing in `traceAcquire` and `traceRelease` fully. These functions mark a +critical section that appears atomic to the execution tracer (but nothing else). + +debuglog +======== + +`debuglog` is a powerful runtime-only debugging tool. Think of it as an +ultra-low-overhead `println` that works just about anywhere in the runtime. +These properties are invaluable when debugging subtle problems in tricky parts +of the codebase. `println` can often perturb code enough to stop data races from +happening, while `debuglog` perturbs execution far less. + +`debuglog` accumulates log messages in a ring buffer on each M, and dumps out +the contents, ordering it by timestamp, on certain kinds of crashes. Some messages +might be lost if the ring buffer gets full, in which case consider increasing the +size, or just work with a partial log. + +1. Add `debuglog` instrumentation to the runtime. Don't forget to call `end`! + Example: `dlog().s("hello world").u32(5).end()` +2. By default, `debuglog` only dumps its contents in certain kinds of crashes. + Consider adding more calls to `printDebugLog` if you're not getting any output. +3. Build the program you wish to debug with the `debuglog` build tag. + +`debuglog` is lower level than execution traces, and much easier to set up. diff --git a/src/runtime/_mkmalloc/astutil/clone.go b/src/runtime/_mkmalloc/astutil/clone.go new file mode 100644 index 00000000000000..16ea7163ca0cb8 --- /dev/null +++ b/src/runtime/_mkmalloc/astutil/clone.go @@ -0,0 +1,73 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file is a copy of golang.org/x/tools/internal/astutil/clone.go + +package astutil + +import ( + "go/ast" + "reflect" +) + +// CloneNode returns a deep copy of a Node. +// It omits pointers to ast.{Scope,Object} variables. +func CloneNode[T ast.Node](n T) T { + return cloneNode(n).(T) +} + +func cloneNode(n ast.Node) ast.Node { + var clone func(x reflect.Value) reflect.Value + set := func(dst, src reflect.Value) { + src = clone(src) + if src.IsValid() { + dst.Set(src) + } + } + clone = func(x reflect.Value) reflect.Value { + switch x.Kind() { + case reflect.Pointer: + if x.IsNil() { + return x + } + // Skip fields of types potentially involved in cycles. + switch x.Interface().(type) { + case *ast.Object, *ast.Scope: + return reflect.Zero(x.Type()) + } + y := reflect.New(x.Type().Elem()) + set(y.Elem(), x.Elem()) + return y + + case reflect.Struct: + y := reflect.New(x.Type()).Elem() + for i := 0; i < x.Type().NumField(); i++ { + set(y.Field(i), x.Field(i)) + } + return y + + case reflect.Slice: + if x.IsNil() { + return x + } + y := reflect.MakeSlice(x.Type(), x.Len(), x.Cap()) + for i := 0; i < x.Len(); i++ { + set(y.Index(i), x.Index(i)) + } + return y + + case reflect.Interface: + y := reflect.New(x.Type()).Elem() + set(y, x.Elem()) + return y + + case reflect.Array, reflect.Chan, reflect.Func, reflect.Map, reflect.UnsafePointer: + panic(x) // unreachable in AST + + default: + return x // bool, string, number + } + } + return clone(reflect.ValueOf(n)).Interface().(ast.Node) +} diff --git a/src/runtime/_mkmalloc/constants.go b/src/runtime/_mkmalloc/constants.go new file mode 100644 index 00000000000000..ad20c7b52be151 --- /dev/null +++ b/src/runtime/_mkmalloc/constants.go @@ -0,0 +1,29 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +const ( + // Constants that we use and will transfer to the runtime. + minHeapAlign = 8 + maxSmallSize = 32 << 10 + smallSizeDiv = 8 + smallSizeMax = 1024 + largeSizeDiv = 128 + pageShift = 13 + tinySize = 16 + + // Derived constants. + pageSize = 1 << pageShift +) + +const ( + maxPtrSize = max(4, 8) + maxPtrBits = 8 * maxPtrSize + + // Maximum size smallScanNoHeader would be called for, which is the + // maximum value gc.MinSizeForMallocHeader can have on any platform. + // gc.MinSizeForMallocHeader is defined as goarch.PtrSize * goarch.PtrBits. + smallScanNoHeaderMax = maxPtrSize * maxPtrBits +) diff --git a/src/runtime/_mkmalloc/go.mod b/src/runtime/_mkmalloc/go.mod new file mode 100644 index 00000000000000..623c3417691d16 --- /dev/null +++ b/src/runtime/_mkmalloc/go.mod @@ -0,0 +1,5 @@ +module runtime/_mkmalloc + +go 1.24 + +require golang.org/x/tools v0.33.0 diff --git a/src/runtime/_mkmalloc/go.sum b/src/runtime/_mkmalloc/go.sum new file mode 100644 index 00000000000000..bead5223ca8f31 --- /dev/null +++ b/src/runtime/_mkmalloc/go.sum @@ -0,0 +1,2 @@ +golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= +golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= diff --git a/src/runtime/_mkmalloc/mkmalloc.go b/src/runtime/_mkmalloc/mkmalloc.go new file mode 100644 index 00000000000000..986b0aa9f85d68 --- /dev/null +++ b/src/runtime/_mkmalloc/mkmalloc.go @@ -0,0 +1,605 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "log" + "os" + "strings" + + "golang.org/x/tools/go/ast/astutil" + + internalastutil "runtime/_mkmalloc/astutil" +) + +var stdout = flag.Bool("stdout", false, "write sizeclasses source to stdout instead of sizeclasses.go") + +func makeSizeToSizeClass(classes []class) []uint8 { + sc := uint8(0) + ret := make([]uint8, smallScanNoHeaderMax+1) + for i := range ret { + if i > classes[sc].size { + sc++ + } + ret[i] = sc + } + return ret +} + +func main() { + log.SetFlags(0) + log.SetPrefix("mkmalloc: ") + + classes := makeClasses() + sizeToSizeClass := makeSizeToSizeClass(classes) + + if *stdout { + if _, err := os.Stdout.Write(mustFormat(generateSizeClasses(classes))); err != nil { + log.Fatal(err) + } + return + } + + sizeclasesesfile := "../../internal/runtime/gc/sizeclasses.go" + if err := os.WriteFile(sizeclasesesfile, mustFormat(generateSizeClasses(classes)), 0666); err != nil { + log.Fatal(err) + } + + outfile := "../malloc_generated.go" + if err := os.WriteFile(outfile, mustFormat(inline(specializedMallocConfig(classes, sizeToSizeClass))), 0666); err != nil { + log.Fatal(err) + } + + tablefile := "../malloc_tables_generated.go" + if err := os.WriteFile(tablefile, mustFormat(generateTable(sizeToSizeClass)), 0666); err != nil { + log.Fatal(err) + } +} + +// withLineNumbers returns b with line numbers added to help debugging. +func withLineNumbers(b []byte) []byte { + var buf bytes.Buffer + i := 1 + for line := range bytes.Lines(b) { + fmt.Fprintf(&buf, "%d: %s", i, line) + i++ + } + return buf.Bytes() +} + +// mustFormat formats the input source, or exits if there's an error. +func mustFormat(b []byte) []byte { + formatted, err := format.Source(b) + if err != nil { + log.Fatalf("error formatting source: %v\nsource:\n%s\n", err, withLineNumbers(b)) + } + return formatted +} + +// generatorConfig is the configuration for the generator. It uses the given file to find +// its templates, and generates each of the functions specified by specs. +type generatorConfig struct { + file string + specs []spec +} + +// spec is the specification for a function for the inliner to produce. The function gets +// the given name, and is produced by starting with the function with the name given by +// templateFunc and applying each of the ops. +type spec struct { + name string + templateFunc string + ops []op +} + +// replacementKind specifies the operation to ben done by a op. +type replacementKind int + +const ( + inlineFunc = replacementKind(iota) + subBasicLit +) + +// op is a single inlining operation for the inliner. Any calls to the function +// from are replaced with the inlined body of to. For non-functions, uses of from are +// replaced with the basic literal expression given by to. +type op struct { + kind replacementKind + from string + to string +} + +func smallScanNoHeaderSCFuncName(sc, scMax uint8) string { + if sc == 0 || sc > scMax { + return "mallocPanic" + } + return fmt.Sprintf("mallocgcSmallScanNoHeaderSC%d", sc) +} + +func tinyFuncName(size uintptr) string { + if size == 0 || size > smallScanNoHeaderMax { + return "mallocPanic" + } + return fmt.Sprintf("mallocTiny%d", size) +} + +func smallNoScanSCFuncName(sc, scMax uint8) string { + if sc < 2 || sc > scMax { + return "mallocPanic" + } + return fmt.Sprintf("mallocgcSmallNoScanSC%d", sc) +} + +// specializedMallocConfig produces an inlining config to stamp out the definitions of the size-specialized +// malloc functions to be written by mkmalloc. +func specializedMallocConfig(classes []class, sizeToSizeClass []uint8) generatorConfig { + config := generatorConfig{file: "../malloc_stubs.go"} + + // Only generate specialized functions for sizes that don't have + // a header on 64-bit platforms. (They may have a header on 32-bit, but + // we will fall back to the non-specialized versions in that case) + scMax := sizeToSizeClass[smallScanNoHeaderMax] + + str := fmt.Sprint + + // allocations with pointer bits + { + const noscan = 0 + for sc := uint8(0); sc <= scMax; sc++ { + if sc == 0 { + continue + } + name := smallScanNoHeaderSCFuncName(sc, scMax) + elemsize := classes[sc].size + config.specs = append(config.specs, spec{ + templateFunc: "mallocStub", + name: name, + ops: []op{ + {inlineFunc, "inlinedMalloc", "smallScanNoHeaderStub"}, + {inlineFunc, "heapSetTypeNoHeaderStub", "heapSetTypeNoHeaderStub"}, + {inlineFunc, "nextFreeFastStub", "nextFreeFastStub"}, + {inlineFunc, "writeHeapBitsSmallStub", "writeHeapBitsSmallStub"}, + {subBasicLit, "elemsize_", str(elemsize)}, + {subBasicLit, "sizeclass_", str(sc)}, + {subBasicLit, "noscanint_", str(noscan)}, + }, + }) + } + } + + // allocations without pointer bits + { + const noscan = 1 + + // tiny + tinySizeClass := sizeToSizeClass[tinySize] + for s := range uintptr(16) { + if s == 0 { + continue + } + name := tinyFuncName(s) + elemsize := classes[tinySizeClass].size + config.specs = append(config.specs, spec{ + templateFunc: "mallocStub", + name: name, + ops: []op{ + {inlineFunc, "inlinedMalloc", "tinyStub"}, + {inlineFunc, "nextFreeFastTiny", "nextFreeFastTiny"}, + {subBasicLit, "elemsize_", str(elemsize)}, + {subBasicLit, "sizeclass_", str(tinySizeClass)}, + {subBasicLit, "size_", str(s)}, + {subBasicLit, "noscanint_", str(noscan)}, + }, + }) + } + + // non-tiny + for sc := uint8(tinySizeClass); sc <= scMax; sc++ { + name := smallNoScanSCFuncName(sc, scMax) + elemsize := classes[sc].size + config.specs = append(config.specs, spec{ + templateFunc: "mallocStub", + name: name, + ops: []op{ + {inlineFunc, "inlinedMalloc", "smallNoScanStub"}, + {inlineFunc, "nextFreeFastStub", "nextFreeFastStub"}, + {subBasicLit, "elemsize_", str(elemsize)}, + {subBasicLit, "sizeclass_", str(sc)}, + {subBasicLit, "noscanint_", str(noscan)}, + }, + }) + } + } + + return config +} + +// inline applies the inlining operations given by the config. +func inline(config generatorConfig) []byte { + var out bytes.Buffer + + // Read the template file in. + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, config.file, nil, 0) + if err != nil { + log.Fatalf("parsing %s: %v", config.file, err) + } + + // Collect the function and import declarations. The function + // declarations in the template file provide both the templates + // that will be stamped out, and the functions that will be inlined + // into them. The imports from the template file will be copied + // straight to the output. + funcDecls := map[string]*ast.FuncDecl{} + importDecls := []*ast.GenDecl{} + for _, decl := range f.Decls { + switch decl := decl.(type) { + case *ast.FuncDecl: + funcDecls[decl.Name.Name] = decl + case *ast.GenDecl: + if decl.Tok.String() == "import" { + importDecls = append(importDecls, decl) + continue + } + } + } + + // Write out the package and import declarations. + out.WriteString("// Code generated by mkmalloc.go; DO NOT EDIT.\n\n") + out.WriteString("package " + f.Name.Name + "\n\n") + for _, importDecl := range importDecls { + out.Write(mustFormatNode(fset, importDecl)) + out.WriteString("\n\n") + } + + // Produce each of the inlined functions specified by specs. + for _, spec := range config.specs { + // Start with a renamed copy of the template function. + containingFuncCopy := internalastutil.CloneNode(funcDecls[spec.templateFunc]) + if containingFuncCopy == nil { + log.Fatal("did not find", spec.templateFunc) + } + containingFuncCopy.Name.Name = spec.name + + // Apply each of the ops given by the specs + stamped := ast.Node(containingFuncCopy) + for _, repl := range spec.ops { + if toDecl, ok := funcDecls[repl.to]; ok { + stamped = inlineFunction(stamped, repl.from, toDecl) + } else { + stamped = substituteWithBasicLit(stamped, repl.from, repl.to) + } + } + + out.Write(mustFormatNode(fset, stamped)) + out.WriteString("\n\n") + } + + return out.Bytes() +} + +// substituteWithBasicLit recursively renames identifiers in the provided AST +// according to 'from' and 'to'. +func substituteWithBasicLit(node ast.Node, from, to string) ast.Node { + // The op is a substitution of an identifier with an basic literal. + toExpr, err := parser.ParseExpr(to) + if err != nil { + log.Fatalf("parsing expr %q: %v", to, err) + } + if _, ok := toExpr.(*ast.BasicLit); !ok { + log.Fatalf("op 'to' expr %q is not a basic literal", to) + } + return astutil.Apply(node, func(cursor *astutil.Cursor) bool { + if isIdentWithName(cursor.Node(), from) { + cursor.Replace(toExpr) + } + return true + }, nil) +} + +// inlineFunction recursively replaces calls to the function 'from' with the body of the function +// 'toDecl'. All calls to 'from' must appear in assignment statements. +// The replacement is very simple: it doesn't substitute the arguments for the parameters, so the +// arguments to the function call must be the same identifier as the parameters to the function +// declared by 'toDecl'. If there are any calls to from where that's not the case there will be a fatal error. +func inlineFunction(node ast.Node, from string, toDecl *ast.FuncDecl) ast.Node { + return astutil.Apply(node, func(cursor *astutil.Cursor) bool { + switch node := cursor.Node().(type) { + case *ast.AssignStmt: + // TODO(matloob) CHECK function args have same name + // as parameters (or parameter is "_"). + if len(node.Rhs) == 1 && isCallTo(node.Rhs[0], from) { + args := node.Rhs[0].(*ast.CallExpr).Args + if !argsMatchParameters(args, toDecl.Type.Params) { + log.Fatalf("applying op: arguments to %v don't match parameter names of %v: %v", from, toDecl.Name, debugPrint(args...)) + } + replaceAssignment(cursor, node, toDecl) + } + return false + case *ast.CallExpr: + // double check that all calls to from appear within an assignment + if isCallTo(node, from) { + if _, ok := cursor.Parent().(*ast.AssignStmt); !ok { + log.Fatalf("applying op: all calls to function %q being replaced must appear in an assignment statement, appears in %T", from, cursor.Parent()) + } + } + } + return true + }, nil) +} + +// argsMatchParameters reports whether the arguments given by args are all identifiers +// whose names are the same as the corresponding parameters in params. +func argsMatchParameters(args []ast.Expr, params *ast.FieldList) bool { + var paramIdents []*ast.Ident + for _, f := range params.List { + paramIdents = append(paramIdents, f.Names...) + } + + if len(args) != len(paramIdents) { + return false + } + + for i := range args { + if !isIdentWithName(args[i], paramIdents[i].Name) { + return false + } + } + + return true +} + +// isIdentWithName reports whether the expression is an identifier with the given name. +func isIdentWithName(expr ast.Node, name string) bool { + ident, ok := expr.(*ast.Ident) + if !ok { + return false + } + return ident.Name == name +} + +// isCallTo reports whether the expression is a call expression to the function with the given name. +func isCallTo(expr ast.Expr, name string) bool { + callexpr, ok := expr.(*ast.CallExpr) + if !ok { + return false + } + return isIdentWithName(callexpr.Fun, name) +} + +// replaceAssignment replaces an assignment statement where the right hand side is a function call +// whose arguments have the same names as the parameters to funcdecl with the body of funcdecl. +// It sets the left hand side of the assignment to the return values of the function. +func replaceAssignment(cursor *astutil.Cursor, assign *ast.AssignStmt, funcdecl *ast.FuncDecl) { + if !hasTerminatingReturn(funcdecl.Body) { + log.Fatal("function being inlined must have a return at the end") + } + + body := internalastutil.CloneNode(funcdecl.Body) + if hasTerminatingAndNonterminatingReturn(funcdecl.Body) { + // The function has multiple return points. Add the code that we'd continue with in the caller + // after each of the return points. The calling function must have a terminating return + // so we don't continue execution in the replaced function after we finish executing the + // continue block that we add. + body = addContinues(cursor, assign, body, everythingFollowingInParent(cursor)).(*ast.BlockStmt) + } + + if len(body.List) < 1 { + log.Fatal("replacing with empty bodied function") + } + + // The op happens in two steps: first we insert the body of the function being inlined (except for + // the final return) before the assignment, and then we change the assignment statement to replace the function call + // with the expressions being returned. + + // Determine the expressions being returned. + beforeReturn, ret := body.List[:len(body.List)-1], body.List[len(body.List)-1] + returnStmt, ok := ret.(*ast.ReturnStmt) + if !ok { + log.Fatal("last stmt in function we're replacing with should be a return") + } + results := returnStmt.Results + + // Insert the body up to the final return. + for _, stmt := range beforeReturn { + cursor.InsertBefore(stmt) + } + + // Rewrite the assignment statement. + replaceWithAssignment(cursor, assign.Lhs, results, assign.Tok) +} + +// hasTerminatingReturn reparts whether the block ends in a return statement. +func hasTerminatingReturn(block *ast.BlockStmt) bool { + _, ok := block.List[len(block.List)-1].(*ast.ReturnStmt) + return ok +} + +// hasTerminatingAndNonterminatingReturn reports whether the block ends in a return +// statement, and also has a return elsewhere in it. +func hasTerminatingAndNonterminatingReturn(block *ast.BlockStmt) bool { + if !hasTerminatingReturn(block) { + return false + } + var ret bool + for i := range block.List[:len(block.List)-1] { + ast.Inspect(block.List[i], func(node ast.Node) bool { + _, ok := node.(*ast.ReturnStmt) + if ok { + ret = true + return false + } + return true + }) + } + return ret +} + +// everythingFollowingInParent returns a block with everything in the parent block node of the cursor after +// the cursor itself. The cursor must point to an element in a block node's list. +func everythingFollowingInParent(cursor *astutil.Cursor) *ast.BlockStmt { + parent := cursor.Parent() + block, ok := parent.(*ast.BlockStmt) + if !ok { + log.Fatal("internal error: in everythingFollowingInParent, cursor doesn't point to element in block list") + } + + blockcopy := internalastutil.CloneNode(block) // get a clean copy + blockcopy.List = blockcopy.List[cursor.Index()+1:] // and remove everything before and including stmt + + if _, ok := blockcopy.List[len(blockcopy.List)-1].(*ast.ReturnStmt); !ok { + log.Printf("%s", mustFormatNode(token.NewFileSet(), blockcopy)) + log.Fatal("internal error: parent doesn't end in a return") + } + return blockcopy +} + +// in the case that there's a return in the body being inlined (toBlock), addContinues +// replaces those returns that are not at the end of the function with the code in the +// caller after the function call that execution would continue with after the return. +// The block being added must end in a return. +func addContinues(cursor *astutil.Cursor, assignNode *ast.AssignStmt, toBlock *ast.BlockStmt, continueBlock *ast.BlockStmt) ast.Node { + if !hasTerminatingReturn(continueBlock) { + log.Fatal("the block being continued to in addContinues must end in a return") + } + applyFunc := func(cursor *astutil.Cursor) bool { + ret, ok := cursor.Node().(*ast.ReturnStmt) + if !ok { + return true + } + + if cursor.Parent() == toBlock && cursor.Index() == len(toBlock.List)-1 { + return false + } + + // This is the opposite of replacing a function call with the body. First + // we replace the return statement with the assignment from the caller, and + // then add the code we continue with. + replaceWithAssignment(cursor, assignNode.Lhs, ret.Results, assignNode.Tok) + cursor.InsertAfter(internalastutil.CloneNode(continueBlock)) + + return false + } + return astutil.Apply(toBlock, applyFunc, nil) +} + +// debugPrint prints out the expressions given by nodes for debugging. +func debugPrint(nodes ...ast.Expr) string { + var b strings.Builder + for i, node := range nodes { + b.Write(mustFormatNode(token.NewFileSet(), node)) + if i != len(nodes)-1 { + b.WriteString(", ") + } + } + return b.String() +} + +// mustFormatNode produces the formatted Go code for the given node. +func mustFormatNode(fset *token.FileSet, node any) []byte { + var buf bytes.Buffer + format.Node(&buf, fset, node) + return buf.Bytes() +} + +// mustMatchExprs makes sure that the expression lists have the same length, +// and returns the lists of the expressions on the lhs and rhs where the +// identifiers are not the same. These are used to produce assignment statements +// where the expressions on the right are assigned to the identifiers on the left. +func mustMatchExprs(lhs []ast.Expr, rhs []ast.Expr) ([]ast.Expr, []ast.Expr) { + if len(lhs) != len(rhs) { + log.Fatal("exprs don't match", debugPrint(lhs...), debugPrint(rhs...)) + } + + var newLhs, newRhs []ast.Expr + for i := range lhs { + lhsIdent, ok1 := lhs[i].(*ast.Ident) + rhsIdent, ok2 := rhs[i].(*ast.Ident) + if ok1 && ok2 && lhsIdent.Name == rhsIdent.Name { + continue + } + newLhs = append(newLhs, lhs[i]) + newRhs = append(newRhs, rhs[i]) + } + + return newLhs, newRhs +} + +// replaceWithAssignment replaces the node pointed to by the cursor with an assignment of the +// left hand side to the righthand side, removing any redundant assignments of a variable to itself, +// and replacing an assignment to a single basic literal with a constant declaration. +func replaceWithAssignment(cursor *astutil.Cursor, lhs, rhs []ast.Expr, tok token.Token) { + newLhs, newRhs := mustMatchExprs(lhs, rhs) + if len(newLhs) == 0 { + cursor.Delete() + return + } + if len(newRhs) == 1 { + if lit, ok := newRhs[0].(*ast.BasicLit); ok { + constDecl := &ast.DeclStmt{ + Decl: &ast.GenDecl{ + Tok: token.CONST, + Specs: []ast.Spec{ + &ast.ValueSpec{ + Names: []*ast.Ident{newLhs[0].(*ast.Ident)}, + Values: []ast.Expr{lit}, + }, + }, + }, + } + cursor.Replace(constDecl) + return + } + } + newAssignment := &ast.AssignStmt{ + Lhs: newLhs, + Rhs: newRhs, + Tok: tok, + } + cursor.Replace(newAssignment) +} + +// generateTable generates the file with the jump tables for the specialized malloc functions. +func generateTable(sizeToSizeClass []uint8) []byte { + scMax := sizeToSizeClass[smallScanNoHeaderMax] + + var b bytes.Buffer + fmt.Fprintln(&b, `// Code generated by mkmalloc.go; DO NOT EDIT. +//go:build !plan9 + +package runtime + +import "unsafe" + +var mallocScanTable = [513]func(size uintptr, typ *_type, needzero bool) unsafe.Pointer{`) + + for i := range uintptr(smallScanNoHeaderMax + 1) { + fmt.Fprintf(&b, "%s,\n", smallScanNoHeaderSCFuncName(sizeToSizeClass[i], scMax)) + } + + fmt.Fprintln(&b, ` +} + +var mallocNoScanTable = [513]func(size uintptr, typ *_type, needzero bool) unsafe.Pointer{`) + for i := range uintptr(smallScanNoHeaderMax + 1) { + if i < 16 { + fmt.Fprintf(&b, "%s,\n", tinyFuncName(i)) + } else { + fmt.Fprintf(&b, "%s,\n", smallNoScanSCFuncName(sizeToSizeClass[i], scMax)) + } + } + + fmt.Fprintln(&b, ` +}`) + + return b.Bytes() +} diff --git a/src/runtime/_mkmalloc/mkmalloc_test.go b/src/runtime/_mkmalloc/mkmalloc_test.go new file mode 100644 index 00000000000000..bd15c3226ab6d2 --- /dev/null +++ b/src/runtime/_mkmalloc/mkmalloc_test.go @@ -0,0 +1,36 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "os" + "testing" +) + +func TestNoChange(t *testing.T) { + classes := makeClasses() + sizeToSizeClass := makeSizeToSizeClass(classes) + + outfile := "../malloc_generated.go" + want, err := os.ReadFile(outfile) + if err != nil { + t.Fatal(err) + } + got := mustFormat(inline(specializedMallocConfig(classes, sizeToSizeClass))) + if !bytes.Equal(want, got) { + t.Fatalf("want:\n%s\ngot:\n%s\n", withLineNumbers(want), withLineNumbers(got)) + } + + tablefile := "../malloc_tables_generated.go" + wanttable, err := os.ReadFile(tablefile) + if err != nil { + t.Fatal(err) + } + gotTable := mustFormat(generateTable(sizeToSizeClass)) + if !bytes.Equal(wanttable, gotTable) { + t.Fatalf("want:\n%s\ngot:\n%s\n", withLineNumbers(wanttable), withLineNumbers(gotTable)) + } +} diff --git a/src/runtime/mksizeclasses.go b/src/runtime/_mkmalloc/mksizeclasses.go similarity index 83% rename from src/runtime/mksizeclasses.go rename to src/runtime/_mkmalloc/mksizeclasses.go index bb06ba1eddc32c..2c39617c6b3ef7 100644 --- a/src/runtime/mksizeclasses.go +++ b/src/runtime/_mkmalloc/mksizeclasses.go @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build ignore - // Generate tables for small malloc size classes. // // See malloc.go for overview. @@ -33,59 +31,29 @@ import ( "bytes" "flag" "fmt" - "go/format" "io" - "log" "math" "math/bits" - "os" ) -// Generate msize.go - -var stdout = flag.Bool("stdout", false, "write to stdout instead of sizeclasses.go") +// Generate internal/runtime/gc/msize.go -func main() { +func generateSizeClasses(classes []class) []byte { flag.Parse() var b bytes.Buffer fmt.Fprintln(&b, "// Code generated by mksizeclasses.go; DO NOT EDIT.") - fmt.Fprintln(&b, "//go:generate go run mksizeclasses.go") + fmt.Fprintln(&b, "//go:generate go -C ../../../runtime/_mkmalloc run mksizeclasses.go") fmt.Fprintln(&b) - fmt.Fprintln(&b, "package runtime") - classes := makeClasses() + fmt.Fprintln(&b, "package gc") printComment(&b, classes) printClasses(&b, classes) - out, err := format.Source(b.Bytes()) - if err != nil { - log.Fatal(err) - } - if *stdout { - _, err = os.Stdout.Write(out) - } else { - err = os.WriteFile("sizeclasses.go", out, 0666) - } - if err != nil { - log.Fatal(err) - } + return b.Bytes() } -const ( - // Constants that we use and will transfer to the runtime. - minHeapAlign = 8 - maxSmallSize = 32 << 10 - smallSizeDiv = 8 - smallSizeMax = 1024 - largeSizeDiv = 128 - pageShift = 13 - - // Derived constants. - pageSize = 1 << pageShift -) - type class struct { size int // max size npages int // number of pages @@ -287,31 +255,51 @@ func maxObjsPerSpan(classes []class) int { return most } +func maxNPages(classes []class) int { + most := 0 + for _, c := range classes[1:] { + most = max(most, c.npages) + } + return most +} + func printClasses(w io.Writer, classes []class) { + sizeToSizeClass := func(size int) int { + for j, c := range classes { + if c.size >= size { + return j + } + } + panic("unreachable") + } + fmt.Fprintln(w, "const (") - fmt.Fprintf(w, "minHeapAlign = %d\n", minHeapAlign) - fmt.Fprintf(w, "_MaxSmallSize = %d\n", maxSmallSize) - fmt.Fprintf(w, "smallSizeDiv = %d\n", smallSizeDiv) - fmt.Fprintf(w, "smallSizeMax = %d\n", smallSizeMax) - fmt.Fprintf(w, "largeSizeDiv = %d\n", largeSizeDiv) - fmt.Fprintf(w, "_NumSizeClasses = %d\n", len(classes)) - fmt.Fprintf(w, "_PageShift = %d\n", pageShift) - fmt.Fprintf(w, "maxObjsPerSpan = %d\n", maxObjsPerSpan(classes)) + fmt.Fprintf(w, "MinHeapAlign = %d\n", minHeapAlign) + fmt.Fprintf(w, "MaxSmallSize = %d\n", maxSmallSize) + fmt.Fprintf(w, "SmallSizeDiv = %d\n", smallSizeDiv) + fmt.Fprintf(w, "SmallSizeMax = %d\n", smallSizeMax) + fmt.Fprintf(w, "LargeSizeDiv = %d\n", largeSizeDiv) + fmt.Fprintf(w, "NumSizeClasses = %d\n", len(classes)) + fmt.Fprintf(w, "PageShift = %d\n", pageShift) + fmt.Fprintf(w, "MaxObjsPerSpan = %d\n", maxObjsPerSpan(classes)) + fmt.Fprintf(w, "MaxSizeClassNPages = %d\n", maxNPages(classes)) + fmt.Fprintf(w, "TinySize = %d\n", tinySize) + fmt.Fprintf(w, "TinySizeClass = %d\n", sizeToSizeClass(tinySize)) fmt.Fprintln(w, ")") - fmt.Fprint(w, "var class_to_size = [_NumSizeClasses]uint16 {") + fmt.Fprint(w, "var SizeClassToSize = [NumSizeClasses]uint16 {") for _, c := range classes { fmt.Fprintf(w, "%d,", c.size) } fmt.Fprintln(w, "}") - fmt.Fprint(w, "var class_to_allocnpages = [_NumSizeClasses]uint8 {") + fmt.Fprint(w, "var SizeClassToNPages = [NumSizeClasses]uint8 {") for _, c := range classes { fmt.Fprintf(w, "%d,", c.npages) } fmt.Fprintln(w, "}") - fmt.Fprint(w, "var class_to_divmagic = [_NumSizeClasses]uint32 {") + fmt.Fprint(w, "var SizeClassToDivMagic = [NumSizeClasses]uint32 {") for _, c := range classes { if c.size == 0 { fmt.Fprintf(w, "0,") @@ -325,14 +313,9 @@ func printClasses(w io.Writer, classes []class) { sc := make([]int, smallSizeMax/smallSizeDiv+1) for i := range sc { size := i * smallSizeDiv - for j, c := range classes { - if c.size >= size { - sc[i] = j - break - } - } + sc[i] = sizeToSizeClass(size) } - fmt.Fprint(w, "var size_to_class8 = [smallSizeMax/smallSizeDiv+1]uint8 {") + fmt.Fprint(w, "var SizeToSizeClass8 = [SmallSizeMax/SmallSizeDiv+1]uint8 {") for _, v := range sc { fmt.Fprintf(w, "%d,", v) } @@ -342,14 +325,9 @@ func printClasses(w io.Writer, classes []class) { sc = make([]int, (maxSmallSize-smallSizeMax)/largeSizeDiv+1) for i := range sc { size := smallSizeMax + i*largeSizeDiv - for j, c := range classes { - if c.size >= size { - sc[i] = j - break - } - } + sc[i] = sizeToSizeClass(size) } - fmt.Fprint(w, "var size_to_class128 = [(_MaxSmallSize-smallSizeMax)/largeSizeDiv+1]uint8 {") + fmt.Fprint(w, "var SizeToSizeClass128 = [(MaxSmallSize-SmallSizeMax)/LargeSizeDiv+1]uint8 {") for _, v := range sc { fmt.Fprintf(w, "%d,", v) } diff --git a/src/runtime/abi_test.go b/src/runtime/abi_test.go index d2e79c6dc4072b..5f8e44a171fde1 100644 --- a/src/runtime/abi_test.go +++ b/src/runtime/abi_test.go @@ -44,12 +44,10 @@ type TintPointer struct { func (*TintPointer) m() {} func TestFinalizerRegisterABI(t *testing.T) { - testenv.MustHaveExec(t) - // Actually run the test in a subprocess because we don't want // finalizers from other tests interfering. if os.Getenv("TEST_FINALIZER_REGABI") != "1" { - cmd := testenv.CleanCmdEnv(exec.Command(os.Args[0], "-test.run=^TestFinalizerRegisterABI$", "-test.v")) + cmd := testenv.CleanCmdEnv(exec.Command(testenv.Executable(t), "-test.run=^TestFinalizerRegisterABI$", "-test.v")) cmd.Env = append(cmd.Env, "TEST_FINALIZER_REGABI=1") out, err := cmd.CombinedOutput() if !strings.Contains(string(out), "PASS\n") || err != nil { @@ -68,6 +66,9 @@ func TestFinalizerRegisterABI(t *testing.T) { runtime.GC() runtime.GC() + // Make sure the finalizer goroutine is running. + runtime.SetFinalizer(new(TintPointer), func(_ *TintPointer) {}) + // fing will only pick the new IntRegArgs up if it's currently // sleeping and wakes up, so wait for it to go to sleep. success := false diff --git a/src/runtime/alg.go b/src/runtime/alg.go index bfb9fa1d297bca..c5951dc20b8a53 100644 --- a/src/runtime/alg.go +++ b/src/runtime/alg.go @@ -6,16 +6,31 @@ package runtime import ( "internal/abi" + "internal/byteorder" "internal/cpu" "internal/goarch" + "internal/runtime/sys" "unsafe" ) const ( - c0 = uintptr((8-goarch.PtrSize)/4*2860486313 + (goarch.PtrSize-4)/4*33054211828000289) - c1 = uintptr((8-goarch.PtrSize)/4*3267000013 + (goarch.PtrSize-4)/4*23344194077549503) + // We use 32-bit hash on Wasm, see hash32.go. + hashSize = (1-goarch.IsWasm)*goarch.PtrSize + goarch.IsWasm*4 + c0 = uintptr((8-hashSize)/4*2860486313 + (hashSize-4)/4*33054211828000289) + c1 = uintptr((8-hashSize)/4*3267000013 + (hashSize-4)/4*23344194077549503) ) +func trimHash(h uintptr) uintptr { + if goarch.IsWasm != 0 { + // On Wasm, we use 32-bit hash, despite that uintptr is 64-bit. + // memhash* always returns a uintptr with high 32-bit being 0 + // (see hash32.go). We trim the hash in other places where we + // compute the hash manually, e.g. in interhash. + return uintptr(uint32(h)) + } + return h +} + func memhash0(p unsafe.Pointer, h uintptr) uintptr { return h } @@ -34,7 +49,7 @@ func memhash128(p unsafe.Pointer, h uintptr) uintptr { //go:nosplit func memhash_varlen(p unsafe.Pointer, h uintptr) uintptr { - ptr := getclosureptr() + ptr := sys.GetClosurePtr() size := *(*uintptr)(unsafe.Pointer(ptr + unsafe.Sizeof(h))) return memhash(p, h, size) } @@ -56,8 +71,6 @@ var useAeshash bool // - github.com/outcaste-io/ristretto // - github.com/puzpuzpuz/xsync/v2 // - github.com/puzpuzpuz/xsync/v3 -// - github.com/segmentio/parquet-go -// - github.com/parquet-go/parquet-go // - github.com/authzed/spicedb // - github.com/pingcap/badger // @@ -67,28 +80,8 @@ var useAeshash bool //go:linkname memhash func memhash(p unsafe.Pointer, h, s uintptr) uintptr -// memhash32 should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/segmentio/parquet-go -// - github.com/parquet-go/parquet-go -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname memhash32 func memhash32(p unsafe.Pointer, h uintptr) uintptr -// memhash64 should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/segmentio/parquet-go -// - github.com/parquet-go/parquet-go -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname memhash64 func memhash64(p unsafe.Pointer, h uintptr) uintptr // strhash should be an internal detail, @@ -97,7 +90,6 @@ func memhash64(p unsafe.Pointer, h uintptr) uintptr // - github.com/aristanetworks/goarista // - github.com/bytedance/sonic // - github.com/bytedance/go-tagexpr/v2 -// - github.com/cloudwego/frugal // - github.com/cloudwego/dynamicgo // - github.com/v2fly/v2ray-core/v5 // @@ -121,9 +113,9 @@ func f32hash(p unsafe.Pointer, h uintptr) uintptr { f := *(*float32)(p) switch { case f == 0: - return c1 * (c0 ^ h) // +0, -0 + return trimHash(c1 * (c0 ^ h)) // +0, -0 case f != f: - return c1 * (c0 ^ h ^ uintptr(rand())) // any kind of NaN + return trimHash(c1 * (c0 ^ h ^ uintptr(rand()))) // any kind of NaN default: return memhash(p, h, 4) } @@ -133,9 +125,9 @@ func f64hash(p unsafe.Pointer, h uintptr) uintptr { f := *(*float64)(p) switch { case f == 0: - return c1 * (c0 ^ h) // +0, -0 + return trimHash(c1 * (c0 ^ h)) // +0, -0 case f != f: - return c1 * (c0 ^ h ^ uintptr(rand())) // any kind of NaN + return trimHash(c1 * (c0 ^ h ^ uintptr(rand()))) // any kind of NaN default: return memhash(p, h, 8) } @@ -165,10 +157,10 @@ func interhash(p unsafe.Pointer, h uintptr) uintptr { // we want to report the struct, not the slice). panic(errorString("hash of unhashable type " + toRType(t).string())) } - if isDirectIface(t) { - return c1 * typehash(t, unsafe.Pointer(&a.data), h^c0) + if t.IsDirectIface() { + return trimHash(c1 * typehash(t, unsafe.Pointer(&a.data), h^c0)) } else { - return c1 * typehash(t, a.data, h^c0) + return trimHash(c1 * typehash(t, a.data, h^c0)) } } @@ -192,10 +184,10 @@ func nilinterhash(p unsafe.Pointer, h uintptr) uintptr { // See comment in interhash above. panic(errorString("hash of unhashable type " + toRType(t).string())) } - if isDirectIface(t) { - return c1 * typehash(t, unsafe.Pointer(&a.data), h^c0) + if t.IsDirectIface() { + return trimHash(c1 * typehash(t, unsafe.Pointer(&a.data), h^c0)) } else { - return c1 * typehash(t, a.data, h^c0) + return trimHash(c1 * typehash(t, a.data, h^c0)) } } @@ -232,7 +224,7 @@ func typehash(t *_type, p unsafe.Pointer, h uintptr) uintptr { return memhash(p, h, t.Size_) } } - switch t.Kind_ & abi.KindMask { + switch t.Kind() { case abi.Float32: return f32hash(p, h) case abi.Float64: @@ -271,74 +263,6 @@ func typehash(t *_type, p unsafe.Pointer, h uintptr) uintptr { } } -func mapKeyError(t *maptype, p unsafe.Pointer) error { - if !t.HashMightPanic() { - return nil - } - return mapKeyError2(t.Key, p) -} - -func mapKeyError2(t *_type, p unsafe.Pointer) error { - if t.TFlag&abi.TFlagRegularMemory != 0 { - return nil - } - switch t.Kind_ & abi.KindMask { - case abi.Float32, abi.Float64, abi.Complex64, abi.Complex128, abi.String: - return nil - case abi.Interface: - i := (*interfacetype)(unsafe.Pointer(t)) - var t *_type - var pdata *unsafe.Pointer - if len(i.Methods) == 0 { - a := (*eface)(p) - t = a._type - if t == nil { - return nil - } - pdata = &a.data - } else { - a := (*iface)(p) - if a.tab == nil { - return nil - } - t = a.tab.Type - pdata = &a.data - } - - if t.Equal == nil { - return errorString("hash of unhashable type " + toRType(t).string()) - } - - if isDirectIface(t) { - return mapKeyError2(t, unsafe.Pointer(pdata)) - } else { - return mapKeyError2(t, *pdata) - } - case abi.Array: - a := (*arraytype)(unsafe.Pointer(t)) - for i := uintptr(0); i < a.Len; i++ { - if err := mapKeyError2(a.Elem, add(p, i*a.Elem.Size_)); err != nil { - return err - } - } - return nil - case abi.Struct: - s := (*structtype)(unsafe.Pointer(t)) - for _, f := range s.Fields { - if f.Name.IsBlank() { - continue - } - if err := mapKeyError2(f.Typ, add(p, f.Offset)); err != nil { - return err - } - } - return nil - default: - // Should never happen, keep this case for robustness. - return errorString("hash of unhashable type " + toRType(t).string()) - } -} - //go:linkname reflect_typehash reflect.typehash func reflect_typehash(t *_type, p unsafe.Pointer, h uintptr) uintptr { return typehash(t, p, h) @@ -395,7 +319,7 @@ func efaceeq(t *_type, x, y unsafe.Pointer) bool { if eq == nil { panic(errorString("comparing uncomparable type " + toRType(t).string())) } - if isDirectIface(t) { + if t.IsDirectIface() { // Direct interface types are ptr, chan, map, func, and single-element structs/arrays thereof. // Maps and funcs are not comparable, so they can't reach here. // Ptrs, chans, and single-element items can be compared directly using ==. @@ -412,7 +336,7 @@ func ifaceeq(tab *itab, x, y unsafe.Pointer) bool { if eq == nil { panic(errorString("comparing uncomparable type " + toRType(t).string())) } - if isDirectIface(t) { + if t.IsDirectIface() { // See comment in efaceeq. return x == y } @@ -496,16 +420,15 @@ func initAlgAES() { func readUnaligned32(p unsafe.Pointer) uint32 { q := (*[4]byte)(p) if goarch.BigEndian { - return uint32(q[3]) | uint32(q[2])<<8 | uint32(q[1])<<16 | uint32(q[0])<<24 + return byteorder.BEUint32(q[:]) } - return uint32(q[0]) | uint32(q[1])<<8 | uint32(q[2])<<16 | uint32(q[3])<<24 + return byteorder.LEUint32(q[:]) } func readUnaligned64(p unsafe.Pointer) uint64 { q := (*[8]byte)(p) if goarch.BigEndian { - return uint64(q[7]) | uint64(q[6])<<8 | uint64(q[5])<<16 | uint64(q[4])<<24 | - uint64(q[3])<<32 | uint64(q[2])<<40 | uint64(q[1])<<48 | uint64(q[0])<<56 + return byteorder.BEUint64(q[:]) } - return uint64(q[0]) | uint64(q[1])<<8 | uint64(q[2])<<16 | uint64(q[3])<<24 | uint64(q[4])<<32 | uint64(q[5])<<40 | uint64(q[6])<<48 | uint64(q[7])<<56 + return byteorder.LEUint64(q[:]) } diff --git a/src/runtime/align_runtime_test.go b/src/runtime/align_runtime_test.go index 6d77e0d3d4a2fb..4bcb49db2f51dc 100644 --- a/src/runtime/align_runtime_test.go +++ b/src/runtime/align_runtime_test.go @@ -28,7 +28,6 @@ var AtomicFields = []uintptr{ unsafe.Offsetof(heapStatsDelta{}.released), unsafe.Offsetof(heapStatsDelta{}.inHeap), unsafe.Offsetof(heapStatsDelta{}.inStacks), - unsafe.Offsetof(heapStatsDelta{}.inPtrScalarBits), unsafe.Offsetof(heapStatsDelta{}.inWorkBufs), unsafe.Offsetof(lfnode{}.next), unsafe.Offsetof(mstats{}.last_gc_nanotime), diff --git a/src/runtime/arena.go b/src/runtime/arena.go index 936e3604bf3e57..52a2a99d6cae87 100644 --- a/src/runtime/arena.go +++ b/src/runtime/arena.go @@ -111,7 +111,7 @@ func arena_newArena() unsafe.Pointer { //go:linkname arena_arena_New arena.runtime_arena_arena_New func arena_arena_New(arena unsafe.Pointer, typ any) any { t := (*_type)(efaceOf(&typ).data) - if t.Kind_&abi.KindMask != abi.Pointer { + if t.Kind() != abi.Pointer { throw("arena_New: non-pointer type") } te := (*ptrtype)(unsafe.Pointer(t)).Elem @@ -145,7 +145,7 @@ func arena_heapify(s any) any { var v unsafe.Pointer e := efaceOf(&s) t := e._type - switch t.Kind_ & abi.KindMask { + switch t.Kind() { case abi.String: v = stringStructOf((*string)(e.data)).str case abi.Slice: @@ -162,7 +162,7 @@ func arena_heapify(s any) any { } // Heap-allocate storage for a copy. var x any - switch t.Kind_ & abi.KindMask { + switch t.Kind() { case abi.String: s1 := s.(string) s2, b := rawstring(len(s1)) @@ -293,11 +293,11 @@ func (a *userArena) slice(sl any, cap int) { } i := efaceOf(&sl) typ := i._type - if typ.Kind_&abi.KindMask != abi.Pointer { + if typ.Kind() != abi.Pointer { panic("slice result of non-ptr type") } typ = (*ptrtype)(unsafe.Pointer(typ)).Elem - if typ.Kind_&abi.KindMask != abi.Slice { + if typ.Kind() != abi.Slice { panic("slice of non-ptr-to-slice type") } typ = (*slicetype)(unsafe.Pointer(typ)).Elem @@ -554,13 +554,7 @@ func userArenaHeapBitsSetType(typ *_type, ptr unsafe.Pointer, s *mspan) { base := s.base() h := s.writeUserArenaHeapBits(uintptr(ptr)) - p := typ.GCData // start of 1-bit pointer mask (or GC program) - var gcProgBits uintptr - if typ.Kind_&abi.KindGCProg != 0 { - // Expand gc program, using the object itself for storage. - gcProgBits = runGCProg(addb(p, 4), (*byte)(ptr)) - p = (*byte)(ptr) - } + p := getGCMask(typ) // start of 1-bit pointer mask nb := typ.PtrBytes / goarch.PtrSize for i := uintptr(0); i < nb; i += ptrBits { @@ -585,11 +579,6 @@ func userArenaHeapBitsSetType(typ *_type, ptr unsafe.Pointer, s *mspan) { h = h.pad(s, typ.Size_-typ.PtrBytes) h.flush(s, uintptr(ptr), typ.Size_) - if typ.Kind_&abi.KindGCProg != 0 { - // Zero out temporary ptrmask buffer inside object. - memclrNoHeapPointers(ptr, (gcProgBits+7)/8) - } - // Update the PtrBytes value in the type information. After this // point, the GC will observe the new bitmap. s.largeType.PtrBytes = uintptr(ptr) - base + typ.PtrBytes @@ -756,7 +745,9 @@ func newUserArenaChunk() (unsafe.Pointer, *mspan) { // does represent additional work for the GC, but we also have no idea // what that looks like until we actually allocate things into the // arena). - deductAssistCredit(userArenaChunkBytes) + if gcBlackenEnabled != 0 { + deductAssistCredit(userArenaChunkBytes) + } // Set mp.mallocing to keep from being preempted by GC. mp := acquirem() @@ -798,11 +789,8 @@ func newUserArenaChunk() (unsafe.Pointer, *mspan) { if asanenabled { // TODO(mknyszek): Track individual objects. - rzSize := computeRZlog(span.elemsize) - span.elemsize -= rzSize - span.largeType.Size_ = span.elemsize + // N.B. span.elemsize includes a redzone already. rzStart := span.base() + span.elemsize - span.userArenaChunkFree = makeAddrRange(span.base(), rzStart) asanpoison(unsafe.Pointer(rzStart), span.limit-rzStart) asanunpoison(unsafe.Pointer(span.base()), span.elemsize) } @@ -813,8 +801,8 @@ func newUserArenaChunk() (unsafe.Pointer, *mspan) { throw("newUserArenaChunk called without a P or outside bootstrapping") } // Note cache c only valid while m acquired; see #47302 - if rate != 1 && userArenaChunkBytes < c.nextSample { - c.nextSample -= userArenaChunkBytes + if rate != 1 && int64(userArenaChunkBytes) < c.nextSample { + c.nextSample -= int64(userArenaChunkBytes) } else { profilealloc(mp, unsafe.Pointer(span.base()), userArenaChunkBytes) } @@ -964,6 +952,9 @@ func freeUserArenaChunk(s *mspan, x unsafe.Pointer) { if asanenabled { asanpoison(unsafe.Pointer(s.base()), s.elemsize) } + if valgrindenabled { + valgrindFree(unsafe.Pointer(s.base())) + } // Make ourselves non-preemptible as we manipulate state and statistics. // @@ -1022,7 +1013,7 @@ func (h *mheap) allocUserArenaChunk() *mspan { // is mapped contiguously. hintList = &h.arenaHints } - v, size := h.sysAlloc(userArenaChunkBytes, hintList, false) + v, size := h.sysAlloc(userArenaChunkBytes, hintList, &mheap_.userArenaArenas) if size%userArenaChunkBytes != 0 { throw("sysAlloc size is not divisible by userArenaChunkBytes") } @@ -1055,7 +1046,7 @@ func (h *mheap) allocUserArenaChunk() *mspan { // // Unlike (*mheap).grow, just map in everything that we // asked for. We're likely going to use it all. - sysMap(unsafe.Pointer(base), userArenaChunkBytes, &gcController.heapReleased) + sysMap(unsafe.Pointer(base), userArenaChunkBytes, &gcController.heapReleased, "user arena chunk") sysUsed(unsafe.Pointer(base), userArenaChunkBytes, userArenaChunkBytes) // Model the user arena as a heap span for a large object. @@ -1063,10 +1054,23 @@ func (h *mheap) allocUserArenaChunk() *mspan { h.initSpan(s, spanAllocHeap, spc, base, userArenaChunkPages) s.isUserArenaChunk = true s.elemsize -= userArenaChunkReserveBytes() - s.limit = s.base() + s.elemsize s.freeindex = 1 s.allocCount = 1 + // Adjust s.limit down to the object-containing part of the span. + // + // This is just to create a slightly tighter bound on the limit. + // It's totally OK if the garbage collector, in particular + // conservative scanning, can temporarily observes an inflated + // limit. It will simply mark the whole chunk or just skip it + // since we're in the mark phase anyway. + s.limit = s.base() + s.elemsize + + // Adjust size to include redzone. + if asanenabled { + s.elemsize -= redZoneSize(s.elemsize) + } + // Account for this new arena chunk memory. gcController.heapInUse.add(int64(userArenaChunkBytes)) gcController.heapReleased.add(-int64(userArenaChunkBytes)) @@ -1088,7 +1092,7 @@ func (h *mheap) allocUserArenaChunk() *mspan { // This must clear the entire heap bitmap so that it's safe // to allocate noscan data without writing anything out. - s.initHeapBits(true) + s.initHeapBits() // Clear the span preemptively. It's an arena chunk, so let's assume // everything is going to be used. diff --git a/src/runtime/asan.go b/src/runtime/asan.go index d79637a334db51..ee070d3270aa8f 100644 --- a/src/runtime/asan.go +++ b/src/runtime/asan.go @@ -7,24 +7,26 @@ package runtime import ( + "internal/runtime/sys" "unsafe" ) // Public address sanitizer API. func ASanRead(addr unsafe.Pointer, len int) { - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() doasanread(addr, uintptr(len), sp, pc) } func ASanWrite(addr unsafe.Pointer, len int) { - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() doasanwrite(addr, uintptr(len), sp, pc) } // Private interface for the runtime. const asanenabled = true +const asanenabledBit = 1 // asan{read,write} are nosplit because they may be called between // fork and exec, when the stack must not grow. See issue #50391. @@ -32,16 +34,16 @@ const asanenabled = true //go:linkname asanread //go:nosplit func asanread(addr unsafe.Pointer, sz uintptr) { - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() doasanread(addr, sz, sp, pc) } //go:linkname asanwrite //go:nosplit func asanwrite(addr unsafe.Pointer, sz uintptr) { - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() doasanwrite(addr, sz, sp, pc) } @@ -60,6 +62,14 @@ func asanpoison(addr unsafe.Pointer, sz uintptr) //go:noescape func asanregisterglobals(addr unsafe.Pointer, n uintptr) +//go:noescape +func lsanregisterrootregion(addr unsafe.Pointer, n uintptr) + +//go:noescape +func lsanunregisterrootregion(addr unsafe.Pointer, n uintptr) + +func lsandoleakcheck() + // These are called from asan_GOARCH.s // //go:cgo_import_static __asan_read_go @@ -67,3 +77,6 @@ func asanregisterglobals(addr unsafe.Pointer, n uintptr) //go:cgo_import_static __asan_unpoison_go //go:cgo_import_static __asan_poison_go //go:cgo_import_static __asan_register_globals_go +//go:cgo_import_static __lsan_register_root_region_go +//go:cgo_import_static __lsan_unregister_root_region_go +//go:cgo_import_static __lsan_do_leak_check_go diff --git a/src/runtime/asan/asan.go b/src/runtime/asan/asan.go index ef70b0145b7f88..fefc82b2787d33 100644 --- a/src/runtime/asan/asan.go +++ b/src/runtime/asan/asan.go @@ -13,6 +13,7 @@ package asan #include #include #include +#include void __asan_read_go(void *addr, uintptr_t sz, void *sp, void *pc) { if (__asan_region_is_poisoned(addr, sz)) { @@ -34,6 +35,18 @@ void __asan_poison_go(void *addr, uintptr_t sz) { __asan_poison_memory_region(addr, sz); } +void __lsan_register_root_region_go(void *addr, uintptr_t sz) { + __lsan_register_root_region(addr, sz); +} + +void __lsan_unregister_root_region_go(void *addr, uintptr_t sz) { + __lsan_unregister_root_region(addr, sz); +} + +void __lsan_do_leak_check_go(void) { + __lsan_do_leak_check(); +} + // Keep in sync with the definition in compiler-rt // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/asan/asan_interface_internal.h#L41 // This structure is used to describe the source location of diff --git a/src/runtime/asan0.go b/src/runtime/asan0.go index bcfd96f1ab9160..f20eae0af773be 100644 --- a/src/runtime/asan0.go +++ b/src/runtime/asan0.go @@ -13,6 +13,7 @@ import ( ) const asanenabled = false +const asanenabledBit = 0 // Because asanenabled is false, none of these functions should be called. @@ -21,3 +22,6 @@ func asanwrite(addr unsafe.Pointer, sz uintptr) { throw("asan") } func asanunpoison(addr unsafe.Pointer, sz uintptr) { throw("asan") } func asanpoison(addr unsafe.Pointer, sz uintptr) { throw("asan") } func asanregisterglobals(addr unsafe.Pointer, sz uintptr) { throw("asan") } +func lsanregisterrootregion(unsafe.Pointer, uintptr) { throw("asan") } +func lsanunregisterrootregion(unsafe.Pointer, uintptr) { throw("asan") } +func lsandoleakcheck() { throw("asan") } diff --git a/src/runtime/asan_amd64.s b/src/runtime/asan_amd64.s index 195faf4e6da1b4..12f14ecf8815ff 100644 --- a/src/runtime/asan_amd64.s +++ b/src/runtime/asan_amd64.s @@ -69,6 +69,28 @@ TEXT runtime·asanregisterglobals(SB), NOSPLIT, $0-16 MOVQ $__asan_register_globals_go(SB), AX JMP asancall<>(SB) +// func runtime·lsanregisterrootregion(addr unsafe.Pointer, n uintptr) +TEXT runtime·lsanregisterrootregion(SB), NOSPLIT, $0-16 + MOVQ addr+0(FP), RARG0 + MOVQ n+8(FP), RARG1 + // void __lsan_register_root_region_go(void *addr, uintptr_t sz) + MOVQ $__lsan_register_root_region_go(SB), AX + JMP asancall<>(SB) + +// func runtime·lsanunregisterrootregion(addr unsafe.Pointer, n uintptr) +TEXT runtime·lsanunregisterrootregion(SB), NOSPLIT, $0-16 + MOVQ addr+0(FP), RARG0 + MOVQ n+8(FP), RARG1 + // void __lsan_unregister_root_region_go(void *addr, uintptr_t sz) + MOVQ $__lsan_unregister_root_region_go(SB), AX + JMP asancall<>(SB) + +// func runtime·lsandoleakcheck() +TEXT runtime·lsandoleakcheck(SB), NOSPLIT, $0-0 + // void __lsan_do_leak_check_go(void); + MOVQ $__lsan_do_leak_check_go(SB), AX + JMP asancall<>(SB) + // Switches SP to g0 stack and calls (AX). Arguments already set. TEXT asancall<>(SB), NOSPLIT, $0-0 get_tls(R12) @@ -78,7 +100,12 @@ TEXT asancall<>(SB), NOSPLIT, $0-0 JE call // no g; still on a system stack MOVQ g_m(R14), R13 - // Switch to g0 stack. + + // Switch to g0 stack if we aren't already on g0 or gsignal. + MOVQ m_gsignal(R13), R10 + CMPQ R10, R14 + JE call // already on gsignal + MOVQ m_g0(R13), R10 CMPQ R10, R14 JE call // already on g0 diff --git a/src/runtime/asan_arm64.s b/src/runtime/asan_arm64.s index dfa3f81bf23f8e..64417552a9ca00 100644 --- a/src/runtime/asan_arm64.s +++ b/src/runtime/asan_arm64.s @@ -58,19 +58,47 @@ TEXT runtime·asanregisterglobals(SB), NOSPLIT, $0-16 MOVD $__asan_register_globals_go(SB), FARG JMP asancall<>(SB) +// func runtime·lsanregisterrootregion(addr unsafe.Pointer, n uintptr) +TEXT runtime·lsanregisterrootregion(SB), NOSPLIT, $0-16 + MOVD addr+0(FP), RARG0 + MOVD n+8(FP), RARG1 + // void __lsan_register_root_region_go(void *addr, uintptr_t n); + MOVD $__lsan_register_root_region_go(SB), FARG + JMP asancall<>(SB) + +// func runtime·lsanunregisterrootregion(addr unsafe.Pointer, n uintptr) +TEXT runtime·lsanunregisterrootregion(SB), NOSPLIT, $0-16 + MOVD addr+0(FP), RARG0 + MOVD n+8(FP), RARG1 + // void __lsan_unregister_root_region_go(void *addr, uintptr_t n); + MOVD $__lsan_unregister_root_region_go(SB), FARG + JMP asancall<>(SB) + +// func runtime·lsandoleakcheck() +TEXT runtime·lsandoleakcheck(SB), NOSPLIT, $0-0 + // void __lsan_do_leak_check_go(void); + MOVD $__lsan_do_leak_check_go(SB), FARG + JMP asancall<>(SB) + // Switches SP to g0 stack and calls (FARG). Arguments already set. TEXT asancall<>(SB), NOSPLIT, $0-0 MOVD RSP, R19 // callee-saved - CBZ g, g0stack // no g, still on a system stack + CBZ g, call // no g, still on a system stack MOVD g_m(g), R10 + + // Switch to g0 stack if we aren't already on g0 or gsignal. + MOVD m_gsignal(R10), R11 + CMP R11, g + BEQ call + MOVD m_g0(R10), R11 CMP R11, g - BEQ g0stack + BEQ call MOVD (g_sched+gobuf_sp)(R11), R5 MOVD R5, RSP -g0stack: +call: BL (FARG) MOVD R19, RSP RET diff --git a/src/runtime/asan_loong64.s b/src/runtime/asan_loong64.s index 0034a316876ee3..13e4a99ac8b97f 100644 --- a/src/runtime/asan_loong64.s +++ b/src/runtime/asan_loong64.s @@ -58,18 +58,45 @@ TEXT runtime·asanregisterglobals(SB), NOSPLIT, $0-16 MOVV $__asan_register_globals_go(SB), FARG JMP asancall<>(SB) +// func runtime·lsanregisterrootregion(addr unsafe.Pointer, n uintptr) +TEXT runtime·lsanregisterrootregion(SB), NOSPLIT, $0-16 + MOVV addr+0(FP), RARG0 + MOVV n+8(FP), RARG1 + // void __lsan_register_root_region_go(void *addr, uintptr_t n); + MOVV $__lsan_register_root_region_go(SB), FARG + JMP asancall<>(SB) + +// func runtime·lsanunregisterrootregion(addr unsafe.Pointer, n uintptr) +TEXT runtime·lsanunregisterrootregion(SB), NOSPLIT, $0-16 + MOVV addr+0(FP), RARG0 + MOVV n+8(FP), RARG1 + // void __lsan_unregister_root_region_go(void *addr, uintptr_t n); + MOVV $__lsan_unregister_root_region_go(SB), FARG + JMP asancall<>(SB) + +// func runtime·lsandoleakcheck() +TEXT runtime·lsandoleakcheck(SB), NOSPLIT, $0-0 + // void __lsan_do_leak_check_go(void); + MOVV $__lsan_do_leak_check_go(SB), FARG + JMP asancall<>(SB) + // Switches SP to g0 stack and calls (FARG). Arguments already set. TEXT asancall<>(SB), NOSPLIT, $0-0 MOVV R3, R23 // callee-saved - BEQ g, g0stack // no g, still on a system stack + BEQ g, call // no g, still on a system stack MOVV g_m(g), R14 + + // Switch to g0 stack if we aren't already on g0 or gsignal. + MOVV m_gsignal(R14), R15 + BEQ R15, g, call + MOVV m_g0(R14), R15 - BEQ R15, g, g0stack + BEQ R15, g, call MOVV (g_sched+gobuf_sp)(R15), R9 MOVV R9, R3 -g0stack: +call: JAL (FARG) MOVV R23, R3 RET diff --git a/src/runtime/asan_ppc64le.s b/src/runtime/asan_ppc64le.s index d13301a1b1d249..b8a38ea47eb70c 100644 --- a/src/runtime/asan_ppc64le.s +++ b/src/runtime/asan_ppc64le.s @@ -58,6 +58,28 @@ TEXT runtime·asanregisterglobals(SB),NOSPLIT|NOFRAME,$0-16 MOVD $__asan_register_globals_go(SB), FARG BR asancall<>(SB) +// func runtime·lsanregisterrootregion(addr unsafe.Pointer, n uintptr) +TEXT runtime·lsanregisterrootregion(SB),NOSPLIT|NOFRAME,$0-16 + MOVD addr+0(FP), RARG0 + MOVD n+8(FP), RARG1 + // void __lsan_register_root_region_go(void *addr, uintptr_t n); + MOVD $__lsan_register_root_region_go(SB), FARG + BR asancall<>(SB) + +// func runtime·lsanunregisterrootregion(addr unsafe.Pointer, n uintptr) +TEXT runtime·lsanunregisterrootregion(SB),NOSPLIT|NOFRAME,$0-16 + MOVD addr+0(FP), RARG0 + MOVD n+8(FP), RARG1 + // void __lsan_unregister_root_region_go(void *addr, uintptr_t n); + MOVD $__lsan_unregister_root_region_go(SB), FARG + BR asancall<>(SB) + +// func runtime·lsandoleakcheck() +TEXT runtime·lsandoleakcheck(SB), NOSPLIT|NOFRAME, $0-0 + // void __lsan_do_leak_check_go(void); + MOVD $__lsan_do_leak_check_go(SB), FARG + BR asancall<>(SB) + // Switches SP to g0 stack and calls (FARG). Arguments already set. TEXT asancall<>(SB), NOSPLIT, $0-0 // LR saved in generated prologue @@ -66,10 +88,18 @@ TEXT asancall<>(SB), NOSPLIT, $0-0 MOVD 0(R10), g MOVD g_m(g), R7 // m for g MOVD R1, R16 // callee-saved, preserved across C call - MOVD m_g0(R7), R10 // g0 for m - CMP R10, g // same g0? - BEQ call // already on g0 + + // Switch to g0 stack if we aren't already on g0 or gsignal. + MOVD m_gsignal(R7), R10 + CMP R10, g + BEQ call + + MOVD m_g0(R7), R10 + CMP R10, g + BEQ call + MOVD (g_sched+gobuf_sp)(R10), R1 // switch R1 + call: // prepare frame for C ABI SUB $32, R1 // create frame for callee saving LR, CR, R2 etc. diff --git a/src/runtime/asan_riscv64.s b/src/runtime/asan_riscv64.s index 6fcd94d4b1fcf0..5a333361dde90d 100644 --- a/src/runtime/asan_riscv64.s +++ b/src/runtime/asan_riscv64.s @@ -52,17 +52,44 @@ TEXT runtime·asanregisterglobals(SB), NOSPLIT, $0-16 MOV $__asan_register_globals_go(SB), X14 JMP asancall<>(SB) +// func runtime·lsanregisterrootregion(addr unsafe.Pointer, n uintptr) +TEXT runtime·lsanregisterrootregion(SB), NOSPLIT, $0-16 + MOV addr+0(FP), X10 + MOV n+8(FP), X11 + // void __lsan_register_root_region_go(void *addr, uintptr_t n); + MOV $__lsan_register_root_region_go(SB), X14 + JMP asancall<>(SB) + +// func runtime·lsanunregisterrootregion(addr unsafe.Pointer, n uintptr) +TEXT runtime·lsanunregisterrootregion(SB), NOSPLIT, $0-16 + MOV addr+0(FP), X10 + MOV n+8(FP), X11 + // void __lsan_unregister_root_region_go(void *addr, uintptr_t n); + MOV $__lsan_unregister_root_region_go(SB), X14 + JMP asancall<>(SB) + +// func runtime·lsandoleakcheck() +TEXT runtime·lsandoleakcheck(SB), NOSPLIT, $0-0 + // void __lsan_do_leak_check_go(void); + MOV $__lsan_do_leak_check_go(SB), X14 + JMP asancall<>(SB) + // Switches SP to g0 stack and calls (X14). Arguments already set. TEXT asancall<>(SB), NOSPLIT, $0-0 MOV X2, X8 // callee-saved - BEQZ g, g0stack // no g, still on a system stack + BEQZ g, call // no g, still on a system stack MOV g_m(g), X21 - MOV m_g0(X21), X21 - BEQ X21, g, g0stack - MOV (g_sched+gobuf_sp)(X21), X2 + // Switch to g0 stack if we aren't already on g0 or gsignal. + MOV m_gsignal(X21), X22 + BEQ X22, g, call + + MOV m_g0(X21), X22 + BEQ X22, g, call + + MOV (g_sched+gobuf_sp)(X22), X2 -g0stack: +call: JALR RA, X14 MOV X8, X2 RET diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index 5aafe14be9a107..03f1a46b55913f 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -291,10 +291,8 @@ TEXT gogo<>(SB), NOSPLIT, $0 get_tls(CX) MOVL DX, g(CX) MOVL gobuf_sp(BX), SP // restore SP - MOVL gobuf_ret(BX), AX MOVL gobuf_ctxt(BX), DX MOVL $0, gobuf_sp(BX) // clear to help garbage collector - MOVL $0, gobuf_ret(BX) MOVL $0, gobuf_ctxt(BX) MOVL gobuf_pc(BX), BX JMP BX @@ -599,12 +597,15 @@ CALLFN(·call268435456, 268435456) CALLFN(·call536870912, 536870912) CALLFN(·call1073741824, 1073741824) -TEXT runtime·procyield(SB),NOSPLIT,$0-0 +TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0 MOVL cycles+0(FP), AX + TESTL AX, AX + JZ done again: PAUSE SUBL $1, AX JNZ again +done: RET TEXT ·publicationBarrier(SB),NOSPLIT,$0-0 @@ -625,7 +626,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT,$0 MOVL AX, (g_sched+gobuf_sp)(BX) MOVL $runtime·systemstack_switch(SB), AX MOVL AX, (g_sched+gobuf_pc)(BX) - MOVL $0, (g_sched+gobuf_ret)(BX) // Assert ctxt is zero. See func save. MOVL (g_sched+gobuf_ctxt)(BX), AX TESTL AX, AX @@ -1373,10 +1373,6 @@ TEXT ·checkASM(SB),NOSPLIT,$0-1 SETEQ ret+0(FP) RET -TEXT runtime·return0(SB), NOSPLIT, $0 - MOVL $0, AX - RET - // Called from cgo wrappers, this function returns g->m->curg.stack.hi. // Must obey the gcc calling convention. TEXT _cgo_topofstack(SB),NOSPLIT,$0 @@ -1516,161 +1512,47 @@ TEXT runtime·gcWriteBarrier8(SB),NOSPLIT,$0 MOVL $32, DI JMP gcWriteBarrier<>(SB) -// Note: these functions use a special calling convention to save generated code space. -// Arguments are passed in registers, but the space for those arguments are allocated -// in the caller's stack frame. These stubs write the args into that stack space and -// then tail call to the corresponding runtime handler. -// The tail call makes these stubs disappear in backtraces. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-8 - MOVL AX, x+0(FP) - MOVL CX, y+4(FP) - JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-8 - MOVL AX, x+0(FP) - MOVL CX, y+4(FP) - JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-8 - MOVL CX, x+0(FP) - MOVL DX, y+4(FP) - JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-8 - MOVL CX, x+0(FP) - MOVL DX, y+4(FP) - JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-8 - MOVL CX, x+0(FP) - MOVL DX, y+4(FP) - JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-8 - MOVL CX, x+0(FP) - MOVL DX, y+4(FP) - JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-8 - MOVL AX, x+0(FP) - MOVL CX, y+4(FP) - JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-8 - MOVL AX, x+0(FP) - MOVL CX, y+4(FP) - JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-8 - MOVL DX, x+0(FP) - MOVL BX, y+4(FP) - JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-8 - MOVL DX, x+0(FP) - MOVL BX, y+4(FP) - JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-8 - MOVL DX, x+0(FP) - MOVL BX, y+4(FP) - JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-8 - MOVL DX, x+0(FP) - MOVL BX, y+4(FP) - JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-8 - MOVL CX, x+0(FP) - MOVL DX, y+4(FP) - JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-8 - MOVL CX, x+0(FP) - MOVL DX, y+4(FP) - JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-8 - MOVL AX, x+0(FP) - MOVL CX, y+4(FP) - JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-8 - MOVL AX, x+0(FP) - MOVL CX, y+4(FP) - JMP runtime·goPanicSlice3CU(SB) -TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-8 - MOVL DX, x+0(FP) - MOVL BX, y+4(FP) - JMP runtime·goPanicSliceConvert(SB) - -// Extended versions for 64-bit indexes. -TEXT runtime·panicExtendIndex(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL AX, lo+4(FP) - MOVL CX, y+8(FP) - JMP runtime·goPanicExtendIndex(SB) -TEXT runtime·panicExtendIndexU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL AX, lo+4(FP) - MOVL CX, y+8(FP) - JMP runtime·goPanicExtendIndexU(SB) -TEXT runtime·panicExtendSliceAlen(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL CX, lo+4(FP) - MOVL DX, y+8(FP) - JMP runtime·goPanicExtendSliceAlen(SB) -TEXT runtime·panicExtendSliceAlenU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL CX, lo+4(FP) - MOVL DX, y+8(FP) - JMP runtime·goPanicExtendSliceAlenU(SB) -TEXT runtime·panicExtendSliceAcap(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL CX, lo+4(FP) - MOVL DX, y+8(FP) - JMP runtime·goPanicExtendSliceAcap(SB) -TEXT runtime·panicExtendSliceAcapU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL CX, lo+4(FP) - MOVL DX, y+8(FP) - JMP runtime·goPanicExtendSliceAcapU(SB) -TEXT runtime·panicExtendSliceB(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL AX, lo+4(FP) - MOVL CX, y+8(FP) - JMP runtime·goPanicExtendSliceB(SB) -TEXT runtime·panicExtendSliceBU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL AX, lo+4(FP) - MOVL CX, y+8(FP) - JMP runtime·goPanicExtendSliceBU(SB) -TEXT runtime·panicExtendSlice3Alen(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL DX, lo+4(FP) - MOVL BX, y+8(FP) - JMP runtime·goPanicExtendSlice3Alen(SB) -TEXT runtime·panicExtendSlice3AlenU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL DX, lo+4(FP) - MOVL BX, y+8(FP) - JMP runtime·goPanicExtendSlice3AlenU(SB) -TEXT runtime·panicExtendSlice3Acap(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL DX, lo+4(FP) - MOVL BX, y+8(FP) - JMP runtime·goPanicExtendSlice3Acap(SB) -TEXT runtime·panicExtendSlice3AcapU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL DX, lo+4(FP) - MOVL BX, y+8(FP) - JMP runtime·goPanicExtendSlice3AcapU(SB) -TEXT runtime·panicExtendSlice3B(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL CX, lo+4(FP) - MOVL DX, y+8(FP) - JMP runtime·goPanicExtendSlice3B(SB) -TEXT runtime·panicExtendSlice3BU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL CX, lo+4(FP) - MOVL DX, y+8(FP) - JMP runtime·goPanicExtendSlice3BU(SB) -TEXT runtime·panicExtendSlice3C(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL AX, lo+4(FP) - MOVL CX, y+8(FP) - JMP runtime·goPanicExtendSlice3C(SB) -TEXT runtime·panicExtendSlice3CU(SB),NOSPLIT,$0-12 - MOVL SI, hi+0(FP) - MOVL AX, lo+4(FP) - MOVL CX, y+8(FP) - JMP runtime·goPanicExtendSlice3CU(SB) +TEXT runtime·panicBounds(SB),NOSPLIT,$40-0 + NO_LOCAL_POINTERS + // Save all int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + MOVL AX, 8(SP) + MOVL CX, 12(SP) + MOVL DX, 16(SP) + MOVL BX, 20(SP) + // skip SP @ 24(SP) + MOVL BP, 28(SP) + MOVL SI, 32(SP) + MOVL DI, 36(SP) + + MOVL SP, AX // hide SP read from vet + MOVL 40(AX), AX // PC immediately after call to panicBounds + MOVL AX, 0(SP) + LEAL 8(SP), AX + MOVL AX, 4(SP) + CALL runtime·panicBounds32(SB) + RET + +TEXT runtime·panicExtend(SB),NOSPLIT,$40-0 + NO_LOCAL_POINTERS + // Save all int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + MOVL AX, 8(SP) + MOVL CX, 12(SP) + MOVL DX, 16(SP) + MOVL BX, 20(SP) + // skip SP @ 24(SP) + MOVL BP, 28(SP) + MOVL SI, 32(SP) + MOVL DI, 36(SP) + + MOVL SP, AX // hide SP read from vet + MOVL 40(AX), AX // PC immediately after call to panicExtend + MOVL AX, 0(SP) + LEAL 8(SP), AX + MOVL AX, 4(SP) + CALL runtime·panicBounds32X(SB) + RET #ifdef GOOS_android // Use the free TLS_SLOT_APP slot #2 on Android Q. diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index cdf9874a7f930e..a4c6c53a900ce0 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -37,7 +37,12 @@ TEXT _rt0_amd64_lib(SB),NOSPLIT|NOFRAME,$0 MOVQ SI, _rt0_amd64_lib_argv<>(SB) // Synchronous initialization. +#ifndef GOOS_windows + // Avoid calling it on Windows because it is not used + // and it would crash the application due to the autogenerated + // ABI wrapper trying to access a non-existent TLS slot. CALL runtime·libpreinit(SB) +#endif // Create a new thread to finish Go runtime initialization. MOVQ _cgo_sys_thread_create(SB), AX @@ -45,12 +50,23 @@ TEXT _rt0_amd64_lib(SB),NOSPLIT|NOFRAME,$0 JZ nocgo // We're calling back to C. - // Align stack per ELF ABI requirements. + // Align stack per C ABI requirements. MOVQ SP, BX // Callee-save in C ABI ANDQ $~15, SP MOVQ $_rt0_amd64_lib_go(SB), DI MOVQ $0, SI +#ifdef GOOS_windows + // For Windows ABI + MOVQ DI, CX + MOVQ SI, DX + // Leave space for four words on the stack as required + // by the Windows amd64 calling convention. + ADJSP $32 +#endif CALL AX +#ifdef GOOS_windows + ADJSP $-32 // just to make the assembler not complain about unbalanced stack +#endif MOVQ BX, SP JMP restore @@ -412,11 +428,9 @@ TEXT gogo<>(SB), NOSPLIT, $0 MOVQ DX, g(CX) MOVQ DX, R14 // set the g register MOVQ gobuf_sp(BX), SP // restore SP - MOVQ gobuf_ret(BX), AX MOVQ gobuf_ctxt(BX), DX MOVQ gobuf_bp(BX), BP MOVQ $0, gobuf_sp(BX) // clear to help garbage collector - MOVQ $0, gobuf_ret(BX) MOVQ $0, gobuf_ctxt(BX) MOVQ $0, gobuf_bp(BX) MOVQ gobuf_pc(BX), BX @@ -454,6 +468,7 @@ goodm: get_tls(CX) // Set G in TLS MOVQ R14, g(CX) MOVQ (g_sched+gobuf_sp)(R14), SP // sp = g0.sched.sp + MOVQ $0, BP // clear frame pointer, as caller may execute on another M PUSHQ AX // open up space for fn's arg spill slot MOVQ 0(DX), R12 CALL R12 // fn(g) @@ -617,7 +632,7 @@ TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0 MOVQ m_g0(BX), BX MOVQ BX, g(CX) MOVQ (g_sched+gobuf_sp)(BX), SP - MOVQ (g_sched+gobuf_bp)(BX), BP + MOVQ $0, BP // clear frame pointer, as caller may execute on another M CALL runtime·newstack(SB) CALL runtime·abort(SB) // crash if newstack returns RET @@ -800,12 +815,15 @@ CALLFN(·call268435456, 268435456) CALLFN(·call536870912, 536870912) CALLFN(·call1073741824, 1073741824) -TEXT runtime·procyield(SB),NOSPLIT,$0-0 +TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0 MOVL cycles+0(FP), AX + TESTL AX, AX + JZ done again: PAUSE SUBL $1, AX JNZ again +done: RET @@ -828,7 +846,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOVQ R9, (g_sched+gobuf_pc)(R14) LEAQ 8(SP), R9 MOVQ R9, (g_sched+gobuf_sp)(R14) - MOVQ $0, (g_sched+gobuf_ret)(R14) MOVQ BP, (g_sched+gobuf_bp)(R14) // Assert ctxt is zero. See func save. MOVQ (g_sched+gobuf_ctxt)(R14), R9 @@ -1679,11 +1696,6 @@ DATA shifts<>+0xf0(SB)/8, $0x0807060504030201 DATA shifts<>+0xf8(SB)/8, $0xff0f0e0d0c0b0a09 GLOBL shifts<>(SB),RODATA,$256 -TEXT runtime·return0(SB), NOSPLIT, $0 - MOVL $0, AX - RET - - // Called from cgo wrappers, this function returns g->m->curg.stack.hi. // Must obey the gcc calling convention. TEXT _cgo_topofstack(SB),NOSPLIT,$0 @@ -1718,9 +1730,7 @@ TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0 TEXT ·sigpanic0(SB),NOSPLIT,$0-0 get_tls(R14) MOVQ g(R14), R14 -#ifndef GOOS_plan9 XORPS X15, X15 -#endif JMP ·sigpanic(SB) // gcWriteBarrier informs the GC about heap pointer writes. @@ -2033,69 +2043,32 @@ TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16 BYTE $0xcc RET -// Note: these functions use a special calling convention to save generated code space. -// Arguments are passed in registers, but the space for those arguments are allocated -// in the caller's stack frame. These stubs write the args into that stack space and -// then tail call to the corresponding runtime handler. -// The tail call makes these stubs disappear in backtraces. -// Defined as ABIInternal since they do not use the stack-based Go ABI. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 - MOVQ CX, BX - JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 - MOVQ CX, BX - JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 - MOVQ CX, AX - MOVQ DX, BX - JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 - MOVQ CX, AX - MOVQ DX, BX - JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 - MOVQ CX, AX - MOVQ DX, BX - JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 - MOVQ CX, AX - MOVQ DX, BX - JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 - MOVQ CX, BX - JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 - MOVQ CX, BX - JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 - MOVQ DX, AX - JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 - MOVQ DX, AX - JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 - MOVQ DX, AX - JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 - MOVQ DX, AX - JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 - MOVQ CX, AX - MOVQ DX, BX - JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 - MOVQ CX, AX - MOVQ DX, BX - JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 - MOVQ CX, BX - JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 - MOVQ CX, BX - JMP runtime·goPanicSlice3CU(SB) -TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16 - MOVQ DX, AX - JMP runtime·goPanicSliceConvert(SB) +TEXT runtime·panicBounds(SB),NOSPLIT,$144-0 + NO_LOCAL_POINTERS + // Save all 14 int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + MOVQ AX, 16(SP) + MOVQ CX, 24(SP) + MOVQ DX, 32(SP) + MOVQ BX, 40(SP) + // skip SP @ 48(SP) + MOVQ BP, 56(SP) + MOVQ SI, 64(SP) + MOVQ DI, 72(SP) + MOVQ R8, 80(SP) + MOVQ R9, 88(SP) + MOVQ R10, 96(SP) + MOVQ R11, 104(SP) + MOVQ R12, 112(SP) + MOVQ R13, 120(SP) + // skip R14 @ 128(SP) (aka G) + MOVQ R15, 136(SP) + + MOVQ SP, AX // hide SP read from vet + MOVQ 152(AX), AX // PC immediately after call to panicBounds + LEAQ 16(SP), BX + CALL runtime·panicBounds64(SB) + RET #ifdef GOOS_android // Use the free TLS_SLOT_APP slot #2 on Android Q. diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index 4d57ec6062ce14..9373846c74a742 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -226,11 +226,9 @@ TEXT gogo<>(SB),NOSPLIT|NOFRAME,$0 BL setg<>(SB) MOVW gobuf_sp(R1), R13 // restore SP==R13 MOVW gobuf_lr(R1), LR - MOVW gobuf_ret(R1), R0 MOVW gobuf_ctxt(R1), R7 MOVW $0, R11 MOVW R11, gobuf_sp(R1) // clear to help garbage collector - MOVW R11, gobuf_ret(R1) MOVW R11, gobuf_lr(R1) MOVW R11, gobuf_ctxt(R1) MOVW gobuf_pc(R1), R11 @@ -550,7 +548,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOVW R13, (g_sched+gobuf_sp)(g) MOVW $0, R11 MOVW R11, (g_sched+gobuf_lr)(g) - MOVW R11, (g_sched+gobuf_ret)(g) // Assert ctxt is zero. See func save. MOVW (g_sched+gobuf_ctxt)(g), R11 TST R11, R11 @@ -797,9 +794,6 @@ TEXT setg<>(SB),NOSPLIT|NOFRAME,$0-0 MOVW R0, g // Save g to thread-local storage. -#ifdef GOOS_windows - B runtime·save_g(SB) -#else #ifdef GOOS_openbsd B runtime·save_g(SB) #else @@ -811,7 +805,6 @@ TEXT setg<>(SB),NOSPLIT|NOFRAME,$0-0 MOVW g, R0 RET #endif -#endif TEXT runtime·emptyfunc(SB),0,$0-0 RET @@ -846,11 +839,7 @@ TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-12 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-12 JMP runtime·memhash64Fallback(SB) -TEXT runtime·return0(SB),NOSPLIT,$0 - MOVW $0, R0 - RET - -TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0 +TEXT runtime·procyieldAsm(SB),NOSPLIT|NOFRAME,$0 MOVW cycles+0(FP), R1 MOVW $0, R0 yieldloop: @@ -998,158 +987,56 @@ TEXT runtime·gcWriteBarrier8(SB),NOSPLIT,$0 MOVW $32, R8 JMP gcWriteBarrier<>(SB) -// Note: these functions use a special calling convention to save generated code space. -// Arguments are passed in registers, but the space for those arguments are allocated -// in the caller's stack frame. These stubs write the args into that stack space and -// then tail call to the corresponding runtime handler. -// The tail call makes these stubs disappear in backtraces. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-8 - MOVW R0, x+0(FP) - MOVW R1, y+4(FP) - JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-8 - MOVW R0, x+0(FP) - MOVW R1, y+4(FP) - JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-8 - MOVW R0, x+0(FP) - MOVW R1, y+4(FP) - JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-8 - MOVW R0, x+0(FP) - MOVW R1, y+4(FP) - JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-8 - MOVW R0, x+0(FP) - MOVW R1, y+4(FP) - JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-8 - MOVW R0, x+0(FP) - MOVW R1, y+4(FP) - JMP runtime·goPanicSlice3CU(SB) -TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSliceConvert(SB) - -// Extended versions for 64-bit indexes. -TEXT runtime·panicExtendIndex(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R0, lo+4(FP) - MOVW R1, y+8(FP) - JMP runtime·goPanicExtendIndex(SB) -TEXT runtime·panicExtendIndexU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R0, lo+4(FP) - MOVW R1, y+8(FP) - JMP runtime·goPanicExtendIndexU(SB) -TEXT runtime·panicExtendSliceAlen(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSliceAlen(SB) -TEXT runtime·panicExtendSliceAlenU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSliceAlenU(SB) -TEXT runtime·panicExtendSliceAcap(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSliceAcap(SB) -TEXT runtime·panicExtendSliceAcapU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSliceAcapU(SB) -TEXT runtime·panicExtendSliceB(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R0, lo+4(FP) - MOVW R1, y+8(FP) - JMP runtime·goPanicExtendSliceB(SB) -TEXT runtime·panicExtendSliceBU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R0, lo+4(FP) - MOVW R1, y+8(FP) - JMP runtime·goPanicExtendSliceBU(SB) -TEXT runtime·panicExtendSlice3Alen(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R2, lo+4(FP) - MOVW R3, y+8(FP) - JMP runtime·goPanicExtendSlice3Alen(SB) -TEXT runtime·panicExtendSlice3AlenU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R2, lo+4(FP) - MOVW R3, y+8(FP) - JMP runtime·goPanicExtendSlice3AlenU(SB) -TEXT runtime·panicExtendSlice3Acap(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R2, lo+4(FP) - MOVW R3, y+8(FP) - JMP runtime·goPanicExtendSlice3Acap(SB) -TEXT runtime·panicExtendSlice3AcapU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R2, lo+4(FP) - MOVW R3, y+8(FP) - JMP runtime·goPanicExtendSlice3AcapU(SB) -TEXT runtime·panicExtendSlice3B(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSlice3B(SB) -TEXT runtime·panicExtendSlice3BU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSlice3BU(SB) -TEXT runtime·panicExtendSlice3C(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R0, lo+4(FP) - MOVW R1, y+8(FP) - JMP runtime·goPanicExtendSlice3C(SB) -TEXT runtime·panicExtendSlice3CU(SB),NOSPLIT,$0-12 - MOVW R4, hi+0(FP) - MOVW R0, lo+4(FP) - MOVW R1, y+8(FP) - JMP runtime·goPanicExtendSlice3CU(SB) +TEXT runtime·panicBounds(SB),NOSPLIT,$72-0 + NO_LOCAL_POINTERS + // Save all int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + MOVW R0, 12(R13) + MOVW R1, 16(R13) + MOVW R2, 20(R13) + MOVW R3, 24(R13) + MOVW R4, 28(R13) + MOVW R5, 32(R13) + MOVW R6, 36(R13) + MOVW R7, 40(R13) + MOVW R8, 44(R13) + MOVW R9, 48(R13) + // skip R10 aka G @ 52(R13) + // skip R11 aka tmp @ 56(R13) + MOVW R12, 60(R13) + // skip R13 aka SP @ 64(R13) + MOVW R14, 68(R13) + // skip R15 aka PC @ 72(R13) + + MOVW R14, 4(R13) // PC immediately after call to panicBounds + ADD $12, R13, R0 // pointer to save area + MOVW R0, 8(R13) + CALL runtime·panicBounds32(SB) + RET + +TEXT runtime·panicExtend(SB),NOSPLIT,$72-0 + NO_LOCAL_POINTERS + // Save all int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + MOVW R0, 12(R13) + MOVW R1, 16(R13) + MOVW R2, 20(R13) + MOVW R3, 24(R13) + MOVW R4, 28(R13) + MOVW R5, 32(R13) + MOVW R6, 36(R13) + MOVW R7, 40(R13) + MOVW R8, 44(R13) + MOVW R9, 48(R13) + // skip R10 aka G @ 52(R13) + // skip R11 aka tmp @ 56(R13) + MOVW R12, 60(R13) + // skip R13 aka SP @ 64(R13) + // skip R14 aka LR @ 68(R13) + // skip R15 aka PC @ 72(R13) + + MOVW R14, 4(R13) // PC immediately after call to panicExtend + ADD $12, R13, R0 // pointer to save area + MOVW R0, 8(R13) + CALL runtime·panicBounds32X(SB) + RET diff --git a/src/runtime/asm_arm64.s b/src/runtime/asm_arm64.s index 64a1880589390d..902a7066aaa113 100644 --- a/src/runtime/asm_arm64.s +++ b/src/runtime/asm_arm64.s @@ -7,6 +7,100 @@ #include "tls_arm64.h" #include "funcdata.h" #include "textflag.h" +#include "cgo/abi_arm64.h" + +// _rt0_arm64 is common startup code for most arm64 systems when using +// internal linking. This is the entry point for the program from the +// kernel for an ordinary -buildmode=exe program. The stack holds the +// number of arguments and the C-style argv. +TEXT _rt0_arm64(SB),NOSPLIT,$0 + MOVD 0(RSP), R0 // argc + ADD $8, RSP, R1 // argv + JMP runtime·rt0_go(SB) + +// main is common startup code for most amd64 systems when using +// external linking. The C startup code will call the symbol "main" +// passing argc and argv in the usual C ABI registers R0 and R1. +TEXT main(SB),NOSPLIT,$0 + JMP runtime·rt0_go(SB) + +// _rt0_arm64_lib is common startup code for most arm64 systems when +// using -buildmode=c-archive or -buildmode=c-shared. The linker will +// arrange to invoke this function as a global constructor (for +// c-archive) or when the shared library is loaded (for c-shared). +// We expect argc and argv to be passed in the usual C ABI registers +// R0 and R1. +TEXT _rt0_arm64_lib(SB),NOSPLIT,$184 + // Preserve callee-save registers. + SAVE_R19_TO_R28(24) + SAVE_F8_TO_F15(104) + + // Initialize g as null in case of using g later e.g. sigaction in cgo_sigaction.go + MOVD ZR, g + + MOVD R0, _rt0_arm64_lib_argc<>(SB) + MOVD R1, _rt0_arm64_lib_argv<>(SB) + + // Synchronous initialization. + MOVD $runtime·libpreinit(SB), R4 + BL (R4) + + // Create a new thread to do the runtime initialization and return. + MOVD _cgo_sys_thread_create(SB), R4 + CBZ R4, nocgo + MOVD $_rt0_arm64_lib_go(SB), R0 + MOVD $0, R1 + SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved. + BL (R4) + ADD $16, RSP + B restore + +nocgo: + MOVD $0x800000, R0 // stacksize = 8192KB + MOVD $_rt0_arm64_lib_go(SB), R1 + MOVD R0, 8(RSP) + MOVD R1, 16(RSP) + MOVD $runtime·newosproc0(SB),R4 + BL (R4) + +restore: + // Restore callee-save registers. + RESTORE_R19_TO_R28(24) + RESTORE_F8_TO_F15(104) + RET + +TEXT _rt0_arm64_lib_go(SB),NOSPLIT,$0 + MOVD _rt0_arm64_lib_argc<>(SB), R0 + MOVD _rt0_arm64_lib_argv<>(SB), R1 + MOVD $runtime·rt0_go(SB),R4 + B (R4) + +DATA _rt0_arm64_lib_argc<>(SB)/8, $0 +GLOBL _rt0_arm64_lib_argc<>(SB),NOPTR, $8 +DATA _rt0_arm64_lib_argv<>(SB)/8, $0 +GLOBL _rt0_arm64_lib_argv<>(SB),NOPTR, $8 + +#ifdef GOARM64_LSE +DATA no_lse_msg<>+0x00(SB)/64, $"This program can only run on ARM64 processors with LSE support.\n" +GLOBL no_lse_msg<>(SB), RODATA, $64 +#endif + +// We know for sure that Linux and FreeBSD allow to read instruction set +// attribute registers (while some others OSes, like OpenBSD and Darwin, +// are not). Let's be conservative and allow code reading such registers +// only when we sure this won't lead to sigill. +#ifdef GOOS_linux +#define ISA_REGS_READABLE +#endif +#ifdef GOOS_freebsd +#define ISA_REGS_READABLE +#endif + +#ifdef GOARM64_LSE +#ifdef ISA_REGS_READABLE +#define CHECK_GOARM64_LSE +#endif +#endif TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 // SP = stack; R0 = argc; R1 = argv @@ -77,6 +171,19 @@ nocgo: BL runtime·wintls(SB) #endif + // Check that CPU we use for execution supports instructions targeted during compile-time. +#ifdef CHECK_GOARM64_LSE + // Read the ID_AA64ISAR0_EL1 register + MRS ID_AA64ISAR0_EL1, R0 + + // Extract the LSE field (bits [23:20]) + LSR $20, R0, R0 + AND $0xf, R0, R0 + + // LSE support is indicated by a non-zero value + CBZ R0, no_lse +#endif + MOVW 8(RSP), R0 // copy argc MOVW R0, -8(RSP) MOVD 16(RSP), R0 // copy argv @@ -95,6 +202,21 @@ nocgo: // start this M BL runtime·mstart(SB) + UNDEF + +#ifdef CHECK_GOARM64_LSE +no_lse: + MOVD $1, R0 // stderr + MOVD R0, 8(RSP) + MOVD $no_lse_msg<>(SB), R1 // message address + MOVD R1, 16(RSP) + MOVD $64, R2 // message length + MOVD R2, 24(RSP) + CALL runtime·write(SB) + CALL runtime·exit(SB) + CALL runtime·abort(SB) + RET +#endif // Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are // intended to be called by debuggers. @@ -149,11 +271,9 @@ TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 MOVD R0, RSP MOVD gobuf_bp(R5), R29 MOVD gobuf_lr(R5), LR - MOVD gobuf_ret(R5), R0 MOVD gobuf_ctxt(R5), R26 MOVD $0, gobuf_sp(R5) MOVD $0, gobuf_bp(R5) - MOVD $0, gobuf_ret(R5) MOVD $0, gobuf_lr(R5) MOVD $0, gobuf_ctxt(R5) CMP ZR, ZR // set condition codes for == test, needed by stack split @@ -185,7 +305,7 @@ TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8 MOVD (g_sched+gobuf_sp)(g), R0 MOVD R0, RSP // sp = m->g0->sched.sp - MOVD (g_sched+gobuf_bp)(g), R29 + MOVD $0, R29 // clear frame pointer, as caller may execute on another M MOVD R3, R0 // arg = g MOVD $0, -16(RSP) // dummy LR SUB $16, RSP @@ -228,7 +348,10 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8 B runtime·abort(SB) switch: - // save our state in g->sched. Pretend to + // Switch stacks. + // The original frame pointer is stored in R29, + // which is useful for stack unwinding. + // Save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. BL gosave_systemstack_switch<>(SB) @@ -237,7 +360,6 @@ switch: BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R3 MOVD R3, RSP - MOVD (g_sched+gobuf_bp)(g), R29 // call target function MOVD 0(R26), R3 // code pointer @@ -337,7 +459,7 @@ TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0 BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R0 MOVD R0, RSP - MOVD (g_sched+gobuf_bp)(g), R29 + MOVD $0, R29 // clear frame pointer, as caller may execute on another M MOVD.W $0, -16(RSP) // create a call frame on g0 (saved LR; keep 16-aligned) BL runtime·newstack(SB) @@ -914,12 +1036,61 @@ aesloop: VMOV V0.D[0], R0 RET -TEXT runtime·procyield(SB),NOSPLIT,$0-0 +// The Arm architecture provides a user space accessible counter-timer which +// is incremented at a fixed but machine-specific rate. Software can (spin) +// wait until the counter-timer reaches some desired value. +// +// Armv8.7-A introduced the WFET (FEAT_WFxT) instruction, which allows the +// processor to enter a low power state for a set time, or until an event is +// received. +// +// However, WFET is not used here because it is only available on newer hardware, +// and we aim to maintain compatibility with older Armv8-A platforms that do not +// support this feature. +// +// As a fallback, we can instead use the ISB instruction to decrease processor +// activity and thus power consumption between checks of the counter-timer. +// Note that we do not depend on the latency of the ISB instruction which is +// implementation specific. Actual delay comes from comparing against a fresh +// read of the counter-timer value. +// +// Read more in this Arm blog post: +// https://community.arm.com/arm-community-blogs/b/architectures-and-processors-blog/posts/multi-threaded-applications-arm + +TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0 MOVWU cycles+0(FP), R0 -again: - YIELD - SUBW $1, R0 - CBNZ R0, again + CBZ R0, done + //Prevent speculation of subsequent counter/timer reads and memory accesses. + ISB $15 + // If the delay is very short, just return. + // Hardcode 18ns as the first ISB delay. + CMP $18, R0 + BLS done + // Adjust for overhead of initial ISB. + SUB $18, R0, R0 + // Convert the delay from nanoseconds to counter/timer ticks. + // Read the counter/timer frequency. + // delay_ticks = (delay * CNTFRQ_EL0) / 1e9 + // With the below simplifications and adjustments, + // we are usually within 2% of the correct value: + // delay_ticks = (delay + delay / 16) * CNTFRQ_EL0 >> 30 + MRS CNTFRQ_EL0, R1 + ADD R0>>4, R0, R0 + MUL R1, R0, R0 + LSR $30, R0, R0 + CBZ R0, done + // start = current counter/timer value + MRS CNTVCT_EL0, R2 +delay: + // Delay using ISB for all ticks. + ISB $15 + // Substract and compare to handle counter roll-over. + // counter_read() - start < delay_ticks + MRS CNTVCT_EL0, R1 + SUB R2, R1, R1 + CMP R0, R1 + BCC delay +done: RET // Save state of caller into g->sched, @@ -935,7 +1106,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOVD R0, (g_sched+gobuf_sp)(g) MOVD R29, (g_sched+gobuf_bp)(g) MOVD $0, (g_sched+gobuf_lr)(g) - MOVD $0, (g_sched+gobuf_ret)(g) // Assert ctxt is zero. See func save. MOVD (g_sched+gobuf_ctxt)(g), R0 CBZ R0, 2(PC) @@ -1213,10 +1383,6 @@ TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0 MOVD (R0), R0 UNDEF -TEXT runtime·return0(SB), NOSPLIT, $0 - MOVW $0, R0 - RET - // The top-most function running on a goroutine // returns to goexit+PCQuantum. TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0 @@ -1529,70 +1695,22 @@ TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16 BREAK RET -// Note: these functions use a special calling convention to save generated code space. -// Arguments are passed in registers, but the space for those arguments are allocated -// in the caller's stack frame. These stubs write the args into that stack space and -// then tail call to the corresponding runtime handler. -// The tail call makes these stubs disappear in backtraces. -// -// Defined as ABIInternal since the compiler generates ABIInternal -// calls to it directly and it does not use the stack-based Go ABI. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 - JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 - JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 - MOVD R1, R0 - MOVD R2, R1 - JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 - MOVD R1, R0 - MOVD R2, R1 - JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 - MOVD R1, R0 - MOVD R2, R1 - JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 - MOVD R1, R0 - MOVD R2, R1 - JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 - JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 - JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 - MOVD R2, R0 - MOVD R3, R1 - JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 - MOVD R2, R0 - MOVD R3, R1 - JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 - MOVD R2, R0 - MOVD R3, R1 - JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 - MOVD R2, R0 - MOVD R3, R1 - JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 - MOVD R1, R0 - MOVD R2, R1 - JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 - MOVD R1, R0 - MOVD R2, R1 - JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 - JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 - JMP runtime·goPanicSlice3CU(SB) -TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16 - MOVD R2, R0 - MOVD R3, R1 - JMP runtime·goPanicSliceConvert(SB) +TEXT runtime·panicBounds(SB),NOSPLIT,$144-0 + NO_LOCAL_POINTERS + // Save all 16 int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + STP (R0, R1), 24(RSP) + STP (R2, R3), 40(RSP) + STP (R4, R5), 56(RSP) + STP (R6, R7), 72(RSP) + STP (R8, R9), 88(RSP) + STP (R10, R11), 104(RSP) + STP (R12, R13), 120(RSP) + STP (R14, R15), 136(RSP) + MOVD LR, R0 // PC immediately after call to panicBounds + ADD $24, RSP, R1 // pointer to save area + CALL runtime·panicBounds64(SB) + RET TEXT ·getfp(SB),NOSPLIT|NOFRAME,$0 MOVD R29, R0 diff --git a/src/runtime/asm_loong64.s b/src/runtime/asm_loong64.s index c16b27a0f27669..a6a5519afb6f7d 100644 --- a/src/runtime/asm_loong64.s +++ b/src/runtime/asm_loong64.s @@ -6,6 +6,57 @@ #include "go_tls.h" #include "funcdata.h" #include "textflag.h" +#include "cgo/abi_loong64.h" + +// When building with -buildmode=c-shared, this symbol is called when the shared +// library is loaded. +TEXT _rt0_loong64_lib(SB),NOSPLIT,$168 + // Preserve callee-save registers. + SAVE_R22_TO_R31(3*8) + SAVE_F24_TO_F31(13*8) + + // Initialize g as nil in case of using g later e.g. sigaction in cgo_sigaction.go + MOVV R0, g + + MOVV R4, _rt0_loong64_lib_argc<>(SB) + MOVV R5, _rt0_loong64_lib_argv<>(SB) + + // Synchronous initialization. + MOVV $runtime·libpreinit(SB), R19 + JAL (R19) + + // Create a new thread to do the runtime initialization and return. + MOVV _cgo_sys_thread_create(SB), R19 + BEQ R19, nocgo + MOVV $_rt0_loong64_lib_go(SB), R4 + MOVV $0, R5 + JAL (R19) + JMP restore + +nocgo: + MOVV $0x800000, R4 // stacksize = 8192KB + MOVV $_rt0_loong64_lib_go(SB), R5 + MOVV R4, 8(R3) + MOVV R5, 16(R3) + MOVV $runtime·newosproc0(SB), R19 + JAL (R19) + +restore: + // Restore callee-save registers. + RESTORE_R22_TO_R31(3*8) + RESTORE_F24_TO_F31(13*8) + RET + +TEXT _rt0_loong64_lib_go(SB),NOSPLIT,$0 + MOVV _rt0_loong64_lib_argc<>(SB), R4 + MOVV _rt0_loong64_lib_argv<>(SB), R5 + MOVV $runtime·rt0_go(SB),R19 + JMP (R19) + +DATA _rt0_loong64_lib_argc<>(SB)/8, $0 +GLOBL _rt0_loong64_lib_argc<>(SB),NOPTR, $8 +DATA _rt0_loong64_lib_argv<>(SB)/8, $0 +GLOBL _rt0_loong64_lib_argv<>(SB),NOPTR, $8 #define REGCTXT R29 @@ -37,6 +88,7 @@ TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 JAL (R25) nocgo: + JAL runtime·save_g(SB) // update stackguard after _cgo_init MOVV (g_stack+stack_lo)(g), R19 ADDV $const_stackGuard, R19 @@ -69,6 +121,11 @@ nocgo: // start this M JAL runtime·mstart(SB) + // Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are + // intended to be called by debuggers. + MOVV $runtime·debugPinnerV1(SB), R0 + MOVV $runtime·debugCallV2(SB), R0 + MOVV R0, 1(R0) RET @@ -87,9 +144,8 @@ TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 RET // not reached // func cputicks() int64 -TEXT runtime·cputicks(SB),NOSPLIT,$0-8 +TEXT runtime·cputicks(SB),NOSPLIT,$0-8 RDTIMED R0, R4 - MOVV R4, ret+0(FP) RET /* @@ -110,10 +166,8 @@ TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 MOVV gobuf_sp(R4), R3 MOVV gobuf_lr(R4), R1 - MOVV gobuf_ret(R4), R19 MOVV gobuf_ctxt(R4), REGCTXT MOVV R0, gobuf_sp(R4) - MOVV R0, gobuf_ret(R4) MOVV R0, gobuf_lr(R4) MOVV R0, gobuf_ctxt(R4) MOVV gobuf_pc(R4), R6 @@ -209,19 +263,19 @@ noswitch: JMP (R4) // func switchToCrashStack0(fn func()) -TEXT runtime·switchToCrashStack0(SB), NOSPLIT, $0-8 - MOVV fn+0(FP), REGCTXT // context register - MOVV g_m(g), R4 // curm +TEXT runtime·switchToCrashStack0(SB),NOSPLIT,$0-8 + MOVV R4, REGCTXT // context register + MOVV g_m(g), R5 // curm // set g to gcrash MOVV $runtime·gcrash(SB), g // g = &gcrash JAL runtime·save_g(SB) - MOVV R4, g_m(g) // g.m = curm - MOVV g, m_g0(R4) // curm.g0 = g + MOVV R5, g_m(g) // g.m = curm + MOVV g, m_g0(R5) // curm.g0 = g // switch to crashstack - MOVV (g_stack+stack_hi)(g), R4 - ADDV $(-4*8), R4, R3 + MOVV (g_stack+stack_hi)(g), R5 + ADDV $(-4*8), R5, R3 // call target function MOVV 0(REGCTXT), R6 @@ -344,32 +398,65 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \ NO_LOCAL_POINTERS; \ /* copy arguments to stack */ \ MOVV arg+16(FP), R4; \ - MOVWU argsize+24(FP), R5; \ - MOVV R3, R12; \ + MOVWU argsize+24(FP), R5; \ + MOVV R3, R12; \ + MOVV $16, R13; \ + ADDV $8, R12; \ + BLT R5, R13, check8; \ + /* copy 16 bytes a time */ \ + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R16; \ + BEQ R16, copy16_again; \ +loop16:; \ + VMOVQ (R4), V0; \ + ADDV $16, R4; \ + ADDV $-16, R5; \ + VMOVQ V0, (R12); \ + ADDV $16, R12; \ + BGE R5, R13, loop16; \ + JMP check8; \ +copy16_again:; \ + MOVV (R4), R14; \ + MOVV 8(R4), R15; \ + ADDV $16, R4; \ + ADDV $-16, R5; \ + MOVV R14, (R12); \ + MOVV R15, 8(R12); \ + ADDV $16, R12; \ + BGE R5, R13, copy16_again; \ +check8:; \ + /* R13 = 8 */; \ + SRLV $1, R13; \ + BLT R5, R13, 6(PC); \ + /* copy 8 bytes a time */ \ + MOVV (R4), R14; \ + ADDV $8, R4; \ + ADDV $-8, R5; \ + MOVV R14, (R12); \ ADDV $8, R12; \ - ADDV R12, R5; \ - BEQ R12, R5, 6(PC); \ - MOVBU (R4), R6; \ - ADDV $1, R4; \ - MOVBU R6, (R12); \ - ADDV $1, R12; \ - JMP -5(PC); \ + BEQ R5, R0, 7(PC); \ + /* copy 1 byte a time for the rest */ \ + MOVBU (R4), R14; \ + ADDV $1, R4; \ + ADDV $-1, R5; \ + MOVBU R14, (R12); \ + ADDV $1, R12; \ + JMP -6(PC); \ /* set up argument registers */ \ MOVV regArgs+40(FP), R25; \ JAL ·unspillArgs(SB); \ /* call function */ \ - MOVV f+8(FP), REGCTXT; \ + MOVV f+8(FP), REGCTXT; \ MOVV (REGCTXT), R25; \ PCDATA $PCDATA_StackMapIndex, $0; \ JAL (R25); \ /* copy return values back */ \ MOVV regArgs+40(FP), R25; \ - JAL ·spillArgs(SB); \ + JAL ·spillArgs(SB); \ MOVV argtype+0(FP), R7; \ MOVV arg+16(FP), R4; \ MOVWU n+24(FP), R5; \ MOVWU retoffset+28(FP), R6; \ - ADDV $8, R3, R12; \ + ADDV $8, R3, R12; \ ADDV R6, R12; \ ADDV R6, R4; \ SUBVU R6, R5; \ @@ -418,7 +505,7 @@ CALLFN(·call268435456, 268435456) CALLFN(·call536870912, 536870912) CALLFN(·call1073741824, 1073741824) -TEXT runtime·procyield(SB),NOSPLIT,$0-0 +TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0 RET // Save state of caller into g->sched. @@ -432,7 +519,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOVV R19, (g_sched+gobuf_pc)(g) MOVV R3, (g_sched+gobuf_sp)(g) MOVV R0, (g_sched+gobuf_lr)(g) - MOVV R0, (g_sched+gobuf_ret)(g) // Assert ctxt is zero. See func save. MOVV (g_sched+gobuf_ctxt)(g), R19 BEQ R19, 2(PC) @@ -623,9 +709,9 @@ TEXT runtime·setg(SB), NOSPLIT, $0-8 JAL runtime·save_g(SB) RET -// void setg_gcc(G*); set g called from gcc with g in R19 +// void setg_gcc(G*); set g called from gcc with g in R4 TEXT setg_gcc<>(SB),NOSPLIT,$0-0 - MOVV R19, g + MOVV R4, g JAL runtime·save_g(SB) RET @@ -643,10 +729,6 @@ TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24 JMP runtime·memhash64Fallback(SB) -TEXT runtime·return0(SB), NOSPLIT, $0 - MOVW $0, R19 - RET - // Called from cgo wrappers, this function returns g->m->curg.stack.hi. // Must obey the gcc calling convention. TEXT _cgo_topofstack(SB),NOSPLIT,$16 @@ -882,76 +964,252 @@ TEXT runtime·gcWriteBarrier8(SB),NOSPLIT,$0 MOVV $64, R29 JMP gcWriteBarrier<>(SB) -// Note: these functions use a special calling convention to save generated code space. -// Arguments are passed in registers, but the space for those arguments are allocated -// in the caller's stack frame. These stubs write the args into that stack space and -// then tail call to the corresponding runtime handler. -// The tail call makes these stubs disappear in backtraces. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 - MOVV R20, R4 - MOVV R21, R5 - JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 - MOVV R20, R4 - MOVV R21, R5 - JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 - MOVV R21, R4 - MOVV R23, R5 - JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 - MOVV R21, R4 - MOVV R23, R5 - JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 - MOVV R21, R4 - MOVV R23, R5 - JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 - MOVV R21, R4 - MOVV R23, R5 - JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 - MOVV R20, R4 - MOVV R21, R5 - JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 - MOVV R20, R4 - MOVV R21, R5 - JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 - MOVV R23, R4 - MOVV R24, R5 - JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 - MOVV R23, R4 - MOVV R24, R5 - JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 - MOVV R23, R4 - MOVV R24, R5 - JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 - MOVV R23, R4 - MOVV R24, R5 - JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 - MOVV R21, R4 - MOVV R23, R5 - JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 - MOVV R21, R4 - MOVV R23, R5 - JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 - MOVV R20, R4 - MOVV R21, R5 - JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 - MOVV R20, R4 - MOVV R21, R5 - JMP runtime·goPanicSlice3CU(SB) -TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16 - MOVV R23, R4 - MOVV R24, R5 - JMP runtime·goPanicSliceConvert(SB) +DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large" +GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below + +// debugCallV2 is the entry point for debugger-injected function +// calls on running goroutines. It informs the runtime that a +// debug call has been injected and creates a call frame for the +// debugger to fill in. +// +// To inject a function call, a debugger should: +// 1. Check that the goroutine is in state _Grunning and that +// there are at least 280 bytes free on the stack. +// 2. Set SP as SP-8. +// 3. Store the current LR in (SP) (using the SP after step 2). +// 4. Store the current PC in the LR register. +// 5. Write the desired argument frame size at SP-8 +// 6. Save all machine registers so they can be restored later by the debugger. +// 7. Set the PC to debugCallV2 and resume execution. +// +// If the goroutine is in state _Grunnable, then it's not generally +// safe to inject a call because it may return out via other runtime +// operations. Instead, the debugger should unwind the stack to find +// the return to non-runtime code, add a temporary breakpoint there, +// and inject the call once that breakpoint is hit. +// +// If the goroutine is in any other state, it's not safe to inject a call. +// +// This function communicates back to the debugger by setting R19 and +// invoking BREAK to raise a breakpoint signal. Note that the signal PC of +// the signal triggered by the BREAK instruction is the PC where the signal +// is trapped, not the next PC, so to resume execution, the debugger needs +// to set the signal PC to PC+4. See the comments in the implementation for +// the protocol the debugger is expected to follow. InjectDebugCall in the +// runtime tests demonstrates this protocol. +// +// The debugger must ensure that any pointers passed to the function +// obey escape analysis requirements. Specifically, it must not pass +// a stack pointer to an escaping argument. debugCallV2 cannot check +// this invariant. +// +// This is ABIInternal because Go code injects its PC directly into new +// goroutine stacks. +TEXT runtime·debugCallV2(SB),NOSPLIT|NOFRAME,$0-0 + MOVV R1, -272(R3) + ADDV $-272, R3 + + // We can't do anything that might clobber any of these + // registers before this. + MOVV R2, (4*8)(R3) + MOVV R4, (5*8)(R3) + MOVV R5, (6*8)(R3) + MOVV R6, (7*8)(R3) + MOVV R7, (8*8)(R3) + MOVV R8, (9*8)(R3) + MOVV R9, (10*8)(R3) + MOVV R10, (11*8)(R3) + MOVV R11, (12*8)(R3) + MOVV R12, (13*8)(R3) + MOVV R13, (14*8)(R3) + MOVV R14, (15*8)(R3) + MOVV R15, (16*8)(R3) + MOVV R16, (17*8)(R3) + MOVV R17, (18*8)(R3) + MOVV R18, (19*8)(R3) + MOVV R19, (20*8)(R3) + MOVV R20, (21*8)(R3) + MOVV R21, (22*8)(R3) + MOVV g, (23*8)(R3) + MOVV R23, (24*8)(R3) + MOVV R24, (25*8)(R3) + MOVV R25, (26*8)(R3) + MOVV R26, (27*8)(R3) + MOVV R27, (28*8)(R3) + MOVV R28, (29*8)(R3) + MOVV R29, (30*8)(R3) + MOVV R30, (31*8)(R3) + MOVV R31, (32*8)(R3) + + // Perform a safe-point check. + MOVV R1, 8(R3) + CALL runtime·debugCallCheck(SB) + MOVV 16(R3), R30 + BEQ R30, good + + // The safety check failed. Put the reason string at the top + // of the stack. + MOVV R30, 8(R3) + + MOVV 24(R3), R30 + MOVV R30, 16(R3) + + MOVV $8, R19 + BREAK + JMP restore + +good: + // Registers are saved and it's safe to make a call. + // Open up a call frame, moving the stack if necessary. + // + // Once the frame is allocated, this will set R19 to 0 and + // invoke BREAK. The debugger should write the argument + // frame for the call at SP+8, set up argument registers, + // set the LR as the signal PC + 4, set the PC to the function + // to call, set R29 to point to the closure (if a closure call), + // and resume execution. + // + // If the function returns, this will set R19 to 1 and invoke + // BREAK. The debugger can then inspect any return value saved + // on the stack at SP+8 and in registers. To resume execution, + // the debugger should restore the LR from (SP). + // + // If the function panics, this will set R19 to 2 and invoke BREAK. + // The interface{} value of the panic will be at SP+8. The debugger + // can inspect the panic value and resume execution again. +#define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \ + MOVV $MAXSIZE, R27; \ + BLT R27, R30, 5(PC); \ + MOVV $NAME(SB), R28; \ + MOVV R28, 8(R3); \ + CALL runtime·debugCallWrap(SB); \ + JMP restore + + MOVV 264(R3), R30 // the argument frame size + DEBUG_CALL_DISPATCH(debugCall32<>, 32) + DEBUG_CALL_DISPATCH(debugCall64<>, 64) + DEBUG_CALL_DISPATCH(debugCall128<>, 128) + DEBUG_CALL_DISPATCH(debugCall256<>, 256) + DEBUG_CALL_DISPATCH(debugCall512<>, 512) + DEBUG_CALL_DISPATCH(debugCall1024<>, 1024) + DEBUG_CALL_DISPATCH(debugCall2048<>, 2048) + DEBUG_CALL_DISPATCH(debugCall4096<>, 4096) + DEBUG_CALL_DISPATCH(debugCall8192<>, 8192) + DEBUG_CALL_DISPATCH(debugCall16384<>, 16384) + DEBUG_CALL_DISPATCH(debugCall32768<>, 32768) + DEBUG_CALL_DISPATCH(debugCall65536<>, 65536) + // The frame size is too large. Report the error. + MOVV $debugCallFrameTooLarge<>(SB), R30 + MOVV R30, 8(R3) + MOVV $20, R30 + MOVV R30, 16(R3) // length of debugCallFrameTooLarge string + MOVV $8, R19 + BREAK + JMP restore + +restore: + // Calls and failures resume here. + // + // Set R19 to 16 and invoke BREAK. The debugger should restore + // all registers except for PC and SP and resume execution. + MOVV $16, R19 + BREAK + // We must not modify flags after this point. + + // Restore pointer-containing registers, which may have been + // modified from the debugger's copy by stack copying. + MOVV (4*8)(R3), R2 + MOVV (5*8)(R3), R4 + MOVV (6*8)(R3), R5 + MOVV (7*8)(R3), R6 + MOVV (8*8)(R3), R7 + MOVV (9*8)(R3), R8 + MOVV (10*8)(R3), R9 + MOVV (11*8)(R3), R10 + MOVV (12*8)(R3), R11 + MOVV (13*8)(R3), R12 + MOVV (14*8)(R3), R13 + MOVV (15*8)(R3), R14 + MOVV (16*8)(R3), R15 + MOVV (17*8)(R3), R16 + MOVV (18*8)(R3), R17 + MOVV (19*8)(R3), R18 + MOVV (20*8)(R3), R19 + MOVV (21*8)(R3), R20 + MOVV (22*8)(R3), R21 + MOVV (23*8)(R3), g + MOVV (24*8)(R3), R23 + MOVV (25*8)(R3), R24 + MOVV (26*8)(R3), R25 + MOVV (27*8)(R3), R26 + MOVV (28*8)(R3), R27 + MOVV (29*8)(R3), R28 + MOVV (30*8)(R3), R29 + MOVV (31*8)(R3), R30 + MOVV (32*8)(R3), R31 + + MOVV 0(R3), R30 + ADDV $280, R3 // Add 8 more bytes, see saveSigContext + MOVV -8(R3), R1 + JMP (R30) + +// runtime.debugCallCheck assumes that functions defined with the +// DEBUG_CALL_FN macro are safe points to inject calls. +#define DEBUG_CALL_FN(NAME,MAXSIZE) \ +TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \ + NO_LOCAL_POINTERS; \ + MOVV $0, R19; \ + BREAK; \ + MOVV $1, R19; \ + BREAK; \ + RET +DEBUG_CALL_FN(debugCall32<>, 32) +DEBUG_CALL_FN(debugCall64<>, 64) +DEBUG_CALL_FN(debugCall128<>, 128) +DEBUG_CALL_FN(debugCall256<>, 256) +DEBUG_CALL_FN(debugCall512<>, 512) +DEBUG_CALL_FN(debugCall1024<>, 1024) +DEBUG_CALL_FN(debugCall2048<>, 2048) +DEBUG_CALL_FN(debugCall4096<>, 4096) +DEBUG_CALL_FN(debugCall8192<>, 8192) +DEBUG_CALL_FN(debugCall16384<>, 16384) +DEBUG_CALL_FN(debugCall32768<>, 32768) +DEBUG_CALL_FN(debugCall65536<>, 65536) + +// func debugCallPanicked(val interface{}) +TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16 + // Copy the panic value to the top of stack at SP+8. + MOVV val_type+0(FP), R30 + MOVV R30, 8(R3) + MOVV val_data+8(FP), R30 + MOVV R30, 16(R3) + MOVV $2, R19 + BREAK + RET + +TEXT runtime·panicBounds(SB),NOSPLIT,$144-0 + NO_LOCAL_POINTERS + // Save all 16 int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + // Skip R0 aka ZERO, R1 aka LR, R2 aka thread pointer, R3 aka SP. + MOVV R4, 24(R3) + MOVV R5, 32(R3) + MOVV R6, 40(R3) + MOVV R7, 48(R3) + MOVV R8, 56(R3) + MOVV R9, 64(R3) + MOVV R10, 72(R3) + MOVV R11, 80(R3) + MOVV R12, 88(R3) + MOVV R13, 96(R3) + MOVV R14, 104(R3) + MOVV R15, 112(R3) + MOVV R16, 120(R3) + MOVV R17, 128(R3) + MOVV R18, 136(R3) + MOVV R19, 144(R3) + + MOVV R1, R4 // PC immediately after call to panicBounds + ADDV $24, R3, R5 // pointer to save area + CALL runtime·panicBounds64(SB) + RET diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s index 80cd87c4af335f..532eca752ffc8a 100644 --- a/src/runtime/asm_mips64x.s +++ b/src/runtime/asm_mips64x.s @@ -107,10 +107,8 @@ TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 MOVV 0(g), R2 MOVV gobuf_sp(R3), R29 MOVV gobuf_lr(R3), R31 - MOVV gobuf_ret(R3), R1 MOVV gobuf_ctxt(R3), REGCTXT MOVV R0, gobuf_sp(R3) - MOVV R0, gobuf_ret(R3) MOVV R0, gobuf_lr(R3) MOVV R0, gobuf_ctxt(R3) MOVV gobuf_pc(R3), R4 @@ -410,7 +408,7 @@ CALLFN(·call268435456, 268435456) CALLFN(·call536870912, 536870912) CALLFN(·call1073741824, 1073741824) -TEXT runtime·procyield(SB),NOSPLIT,$0-0 +TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0 RET // Save state of caller into g->sched, @@ -424,7 +422,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOVV R1, (g_sched+gobuf_pc)(g) MOVV R29, (g_sched+gobuf_sp)(g) MOVV R0, (g_sched+gobuf_lr)(g) - MOVV R0, (g_sched+gobuf_ret)(g) // Assert ctxt is zero. See func save. MOVV (g_sched+gobuf_ctxt)(g), R1 BEQ R1, 2(PC) @@ -644,10 +641,6 @@ TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24 JMP runtime·memhash64Fallback(SB) -TEXT runtime·return0(SB), NOSPLIT, $0 - MOVW $0, R1 - RET - // Called from cgo wrappers, this function returns g->m->curg.stack.hi. // Must obey the gcc calling convention. TEXT _cgo_topofstack(SB),NOSPLIT,$16 @@ -798,76 +791,30 @@ TEXT runtime·gcWriteBarrier8(SB),NOSPLIT,$0 MOVV $64, R25 JMP gcWriteBarrier<>(SB) -// Note: these functions use a special calling convention to save generated code space. -// Arguments are passed in registers, but the space for those arguments are allocated -// in the caller's stack frame. These stubs write the args into that stack space and -// then tail call to the corresponding runtime handler. -// The tail call makes these stubs disappear in backtraces. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 - MOVV R1, x+0(FP) - MOVV R2, y+8(FP) - JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 - MOVV R1, x+0(FP) - MOVV R2, y+8(FP) - JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 - MOVV R2, x+0(FP) - MOVV R3, y+8(FP) - JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 - MOVV R2, x+0(FP) - MOVV R3, y+8(FP) - JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 - MOVV R2, x+0(FP) - MOVV R3, y+8(FP) - JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 - MOVV R2, x+0(FP) - MOVV R3, y+8(FP) - JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 - MOVV R1, x+0(FP) - MOVV R2, y+8(FP) - JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 - MOVV R1, x+0(FP) - MOVV R2, y+8(FP) - JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 - MOVV R3, x+0(FP) - MOVV R4, y+8(FP) - JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 - MOVV R3, x+0(FP) - MOVV R4, y+8(FP) - JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 - MOVV R3, x+0(FP) - MOVV R4, y+8(FP) - JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 - MOVV R3, x+0(FP) - MOVV R4, y+8(FP) - JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 - MOVV R2, x+0(FP) - MOVV R3, y+8(FP) - JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 - MOVV R2, x+0(FP) - MOVV R3, y+8(FP) - JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 - MOVV R1, x+0(FP) - MOVV R2, y+8(FP) - JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 - MOVV R1, x+0(FP) - MOVV R2, y+8(FP) - JMP runtime·goPanicSlice3CU(SB) -TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16 - MOVV R3, x+0(FP) - MOVV R4, y+8(FP) - JMP runtime·goPanicSliceConvert(SB) +TEXT runtime·panicBounds(SB),NOSPLIT,$144-0 + NO_LOCAL_POINTERS + // Save all 16 int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + // Skip R0 aka ZERO. + MOVV R1, 24(R29) + MOVV R2, 32(R29) + MOVV R3, 40(R29) + MOVV R4, 48(R29) + MOVV R5, 56(R29) + MOVV R6, 64(R29) + MOVV R7, 72(R29) + MOVV R8, 80(R29) + MOVV R9, 88(R29) + MOVV R10, 96(R29) + MOVV R11, 104(R29) + MOVV R12, 112(R29) + MOVV R13, 120(R29) + MOVV R14, 128(R29) + MOVV R15, 136(R29) + MOVV R16, 144(R29) + + MOVV R31, 8(R29) // PC immediately after call to panicBounds + ADDV $24, R29, R1 // pointer to save area + MOVV R1, 16(R29) + CALL runtime·panicBounds64(SB) + RET diff --git a/src/runtime/asm_mipsx.s b/src/runtime/asm_mipsx.s index ca95f22bd67f46..2dc3f1c3ad729a 100644 --- a/src/runtime/asm_mipsx.s +++ b/src/runtime/asm_mipsx.s @@ -106,10 +106,8 @@ TEXT gogo<>(SB),NOSPLIT|NOFRAME,$0 JAL runtime·save_g(SB) MOVW gobuf_sp(R3), R29 MOVW gobuf_lr(R3), R31 - MOVW gobuf_ret(R3), R1 MOVW gobuf_ctxt(R3), REGCTXT MOVW R0, gobuf_sp(R3) - MOVW R0, gobuf_ret(R3) MOVW R0, gobuf_lr(R3) MOVW R0, gobuf_ctxt(R3) MOVW gobuf_pc(R3), R4 @@ -408,7 +406,7 @@ CALLFN(·call268435456, 268435456) CALLFN(·call536870912, 536870912) CALLFN(·call1073741824, 1073741824) -TEXT runtime·procyield(SB),NOSPLIT,$0-4 +TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-4 RET // Save state of caller into g->sched, @@ -422,7 +420,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOVW R1, (g_sched+gobuf_pc)(g) MOVW R29, (g_sched+gobuf_sp)(g) MOVW R0, (g_sched+gobuf_lr)(g) - MOVW R0, (g_sched+gobuf_ret)(g) // Assert ctxt is zero. See func save. MOVW (g_sched+gobuf_ctxt)(g), R1 BEQ R1, 2(PC) @@ -634,10 +631,6 @@ TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-12 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-12 JMP runtime·memhash64Fallback(SB) -TEXT runtime·return0(SB),NOSPLIT,$0 - MOVW $0, R1 - RET - // Called from cgo wrappers, this function returns g->m->curg.stack.hi. // Must obey the gcc calling convention. TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0 @@ -794,158 +787,58 @@ TEXT runtime·gcWriteBarrier8(SB),NOSPLIT,$0 MOVW $32, R25 JMP gcWriteBarrier<>(SB) -// Note: these functions use a special calling convention to save generated code space. -// Arguments are passed in registers, but the space for those arguments are allocated -// in the caller's stack frame. These stubs write the args into that stack space and -// then tail call to the corresponding runtime handler. -// The tail call makes these stubs disappear in backtraces. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-8 - MOVW R3, x+0(FP) - MOVW R4, y+4(FP) - JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-8 - MOVW R3, x+0(FP) - MOVW R4, y+4(FP) - JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-8 - MOVW R3, x+0(FP) - MOVW R4, y+4(FP) - JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-8 - MOVW R3, x+0(FP) - MOVW R4, y+4(FP) - JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-8 - MOVW R2, x+0(FP) - MOVW R3, y+4(FP) - JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-8 - MOVW R1, x+0(FP) - MOVW R2, y+4(FP) - JMP runtime·goPanicSlice3CU(SB) -TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-8 - MOVW R3, x+0(FP) - MOVW R4, y+4(FP) - JMP runtime·goPanicSliceConvert(SB) - -// Extended versions for 64-bit indexes. -TEXT runtime·panicExtendIndex(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendIndex(SB) -TEXT runtime·panicExtendIndexU(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendIndexU(SB) -TEXT runtime·panicExtendSliceAlen(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R2, lo+4(FP) - MOVW R3, y+8(FP) - JMP runtime·goPanicExtendSliceAlen(SB) -TEXT runtime·panicExtendSliceAlenU(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R2, lo+4(FP) - MOVW R3, y+8(FP) - JMP runtime·goPanicExtendSliceAlenU(SB) -TEXT runtime·panicExtendSliceAcap(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R2, lo+4(FP) - MOVW R3, y+8(FP) - JMP runtime·goPanicExtendSliceAcap(SB) -TEXT runtime·panicExtendSliceAcapU(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R2, lo+4(FP) - MOVW R3, y+8(FP) - JMP runtime·goPanicExtendSliceAcapU(SB) -TEXT runtime·panicExtendSliceB(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSliceB(SB) -TEXT runtime·panicExtendSliceBU(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSliceBU(SB) -TEXT runtime·panicExtendSlice3Alen(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R3, lo+4(FP) - MOVW R4, y+8(FP) - JMP runtime·goPanicExtendSlice3Alen(SB) -TEXT runtime·panicExtendSlice3AlenU(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R3, lo+4(FP) - MOVW R4, y+8(FP) - JMP runtime·goPanicExtendSlice3AlenU(SB) -TEXT runtime·panicExtendSlice3Acap(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R3, lo+4(FP) - MOVW R4, y+8(FP) - JMP runtime·goPanicExtendSlice3Acap(SB) -TEXT runtime·panicExtendSlice3AcapU(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R3, lo+4(FP) - MOVW R4, y+8(FP) - JMP runtime·goPanicExtendSlice3AcapU(SB) -TEXT runtime·panicExtendSlice3B(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R2, lo+4(FP) - MOVW R3, y+8(FP) - JMP runtime·goPanicExtendSlice3B(SB) -TEXT runtime·panicExtendSlice3BU(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R2, lo+4(FP) - MOVW R3, y+8(FP) - JMP runtime·goPanicExtendSlice3BU(SB) -TEXT runtime·panicExtendSlice3C(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSlice3C(SB) -TEXT runtime·panicExtendSlice3CU(SB),NOSPLIT,$0-12 - MOVW R5, hi+0(FP) - MOVW R1, lo+4(FP) - MOVW R2, y+8(FP) - JMP runtime·goPanicExtendSlice3CU(SB) +TEXT runtime·panicBounds(SB),NOSPLIT,$72-0 + NO_LOCAL_POINTERS + // Save all 16 int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + // Skip R0 aka ZERO. + MOVW R1, 12(R29) + MOVW R2, 16(R29) + MOVW R3, 20(R29) + MOVW R4, 24(R29) + MOVW R5, 28(R29) + MOVW R6, 32(R29) + MOVW R7, 36(R29) + MOVW R8, 40(R29) + MOVW R9, 44(R29) + MOVW R10, 48(R29) + MOVW R11, 52(R29) + MOVW R12, 56(R29) + MOVW R13, 60(R29) + MOVW R14, 64(R29) + MOVW R15, 68(R29) + MOVW R16, 72(R29) + + MOVW R31, 4(R29) // PC immediately after call to panicBounds + ADD $12, R29, R1 // pointer to save area + MOVW R1, 8(R29) + CALL runtime·panicBounds32(SB) + RET + +TEXT runtime·panicExtend(SB),NOSPLIT,$72-0 + NO_LOCAL_POINTERS + // Save all 16 int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + // Skip R0 aka ZERO. + MOVW R1, 12(R29) + MOVW R2, 16(R29) + MOVW R3, 20(R29) + MOVW R4, 24(R29) + MOVW R5, 28(R29) + MOVW R6, 32(R29) + MOVW R7, 36(R29) + MOVW R8, 40(R29) + MOVW R9, 44(R29) + MOVW R10, 48(R29) + MOVW R11, 52(R29) + MOVW R12, 56(R29) + MOVW R13, 60(R29) + MOVW R14, 64(R29) + MOVW R15, 68(R29) + MOVW R16, 72(R29) + + MOVW R31, 4(R29) // PC immediately after call to panicBounds + ADD $12, R29, R1 // pointer to save area + MOVW R1, 8(R29) + CALL runtime·panicBounds32X(SB) + RET diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s index 2b8c4d42a3deaf..aaa2e4346c7a41 100644 --- a/src/runtime/asm_ppc64x.s +++ b/src/runtime/asm_ppc64x.s @@ -9,6 +9,85 @@ #include "funcdata.h" #include "textflag.h" #include "asm_ppc64x.h" +#include "cgo/abi_ppc64x.h" + +// This is called using the host ABI. argc and argv arguments +// should be in R3 and R4 respectively. +TEXT _rt0_ppc64x_lib(SB),NOSPLIT|NOFRAME,$0 + // Start with standard C stack frame layout and linkage, allocate + // 16 bytes of argument space, save callee-save regs, and set R0 to $0. + // Allocate an extra 16 bytes to account for the larger fixed frame size + // of aix/elfv1 (48 vs 32) to ensure 16 bytes of parameter save space. + STACK_AND_SAVE_HOST_TO_GO_ABI(32) + // The above will not preserve R2 (TOC). Save it in case Go is + // compiled without a TOC pointer (e.g -buildmode=default). + MOVD R2, 24(R1) + + MOVD R3, _rt0_ppc64x_lib_argc<>(SB) + MOVD R4, _rt0_ppc64x_lib_argv<>(SB) + + // Synchronous initialization. + MOVD $runtime·reginit(SB), R12 + MOVD R12, CTR + BL (CTR) + MOVD $runtime·libpreinit(SB), R12 + MOVD R12, CTR + BL (CTR) + +#ifdef GOOS_aix + // See runtime/cgo/gcc_aix_ppc64.c + MOVBZ runtime·isarchive(SB), R3 // Check buildmode = c-archive + CMP $0, R3 + BEQ done +#endif + + // Create a new thread to do the runtime initialization and return. + // _cgo_sys_thread_create is a C function. + MOVD _cgo_sys_thread_create(SB), R12 + CMP $0, R12 + BEQ nocgo + MOVD $_rt0_ppc64x_lib_go(SB), R3 + MOVD $0, R4 +#ifdef GO_PPC64X_HAS_FUNCDESC + // Load the real entry address from the first slot of the function descriptor. + MOVD 8(R12), R2 + MOVD (R12), R12 +#endif + MOVD R12, CTR + BL (CTR) + MOVD 24(R1), R2 // Restore the old frame, and R2. + BR done + +nocgo: + MOVD $0x800000, R12 // stacksize = 8192KB + MOVD R12, 8+FIXED_FRAME(R1) + MOVD $_rt0_ppc64x_lib_go(SB), R12 + MOVD R12, 16+FIXED_FRAME(R1) + MOVD $runtime·newosproc0(SB),R12 + MOVD R12, CTR + BL (CTR) + +done: + UNSTACK_AND_RESTORE_GO_TO_HOST_ABI(32) + RET + +#ifdef GO_PPC64X_HAS_FUNCDESC +DEFINE_PPC64X_FUNCDESC(_rt0_ppc64x_lib_go, __rt0_ppc64x_lib_go) +TEXT __rt0_ppc64x_lib_go(SB),NOSPLIT,$0 +#else +TEXT _rt0_ppc64x_lib_go(SB),NOSPLIT,$0 +#endif + MOVD _rt0_ppc64x_lib_argc<>(SB), R3 + MOVD _rt0_ppc64x_lib_argv<>(SB), R4 + MOVD $runtime·rt0_go(SB), R12 + MOVD R12, CTR + BR (CTR) + +DATA _rt0_ppc64x_lib_argc<>(SB)/8, $0 +GLOBL _rt0_ppc64x_lib_argc<>(SB),NOPTR, $8 +DATA _rt0_ppc64x_lib_argv<>(SB)/8, $0 +GLOBL _rt0_ppc64x_lib_argv<>(SB),NOPTR, $8 + #ifdef GOOS_aix #define cgoCalleeStackSize 48 @@ -154,10 +233,8 @@ TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 MOVD 24(R1), R2 // restore R2 #endif MOVD R31, LR - MOVD gobuf_ret(R5), R3 MOVD gobuf_ctxt(R5), R11 MOVD R0, gobuf_sp(R5) - MOVD R0, gobuf_ret(R5) MOVD R0, gobuf_lr(R5) MOVD R0, gobuf_ctxt(R5) CMP R0, R0 // set condition codes for == test, needed by stack split @@ -535,8 +612,10 @@ CALLFN(·call268435456, 268435456) CALLFN(·call536870912, 536870912) CALLFN(·call1073741824, 1073741824) -TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0-4 +TEXT runtime·procyieldAsm(SB),NOSPLIT|NOFRAME,$0-4 MOVW cycles+0(FP), R7 + CMP $0, R7 + BEQ done // POWER does not have a pause/yield instruction equivalent. // Instead, we can lower the program priority by setting the // Program Priority Register prior to the wait loop and set it @@ -548,6 +627,7 @@ again: CMP $0, R7 BNE again OR R6, R6, R6 // Set PPR priority back to medium-low +done: RET // Save state of caller into g->sched, @@ -561,7 +641,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOVD R31, (g_sched+gobuf_pc)(g) MOVD R1, (g_sched+gobuf_sp)(g) MOVD R0, (g_sched+gobuf_lr)(g) - MOVD R0, (g_sched+gobuf_ret)(g) // Assert ctxt is zero. See func save. MOVD (g_sched+gobuf_ctxt)(g), R31 CMP R31, $0 @@ -980,10 +1059,6 @@ TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24 JMP runtime·memhash64Fallback(SB) -TEXT runtime·return0(SB), NOSPLIT, $0 - MOVW $0, R3 - RET - // Called from cgo wrappers, this function returns g->m->curg.stack.hi. // Must obey the gcc calling convention. #ifdef GOOS_aix @@ -1356,67 +1431,29 @@ TEXT runtime·debugCallPanicked(SB),NOSPLIT,$32-16 TW $31, R0, R0 RET #endif -// Note: these functions use a special calling convention to save generated code space. -// Arguments are passed in registers, but the space for those arguments are allocated -// in the caller's stack frame. These stubs write the args into that stack space and -// then tail call to the corresponding runtime handler. -// The tail call makes these stubs disappear in backtraces. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 - JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 - JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 - MOVD R4, R3 - MOVD R5, R4 - JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 - MOVD R4, R3 - MOVD R5, R4 - JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 - MOVD R4, R3 - MOVD R5, R4 - JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 - MOVD R4, R3 - MOVD R5, R4 - JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 - JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 - JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 - MOVD R5, R3 - MOVD R6, R4 - JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 - MOVD R5, R3 - MOVD R6, R4 - JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 - MOVD R5, R3 - MOVD R6, R4 - JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 - MOVD R5, R3 - MOVD R6, R4 - JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 - MOVD R4, R3 - MOVD R5, R4 - JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 - MOVD R4, R3 - MOVD R5, R4 - JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 - JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 - JMP runtime·goPanicSlice3CU(SB) -TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16 - MOVD R5, R3 - MOVD R6, R4 - JMP runtime·goPanicSliceConvert(SB) + +TEXT runtime·panicBounds(SB),NOSPLIT,$88-0 + // Note: frame size is 16 bytes larger than necessary + // in order to pacify vet. Vet doesn't understand ppc64 + // layout properly. + NO_LOCAL_POINTERS + // Save all 7 int registers that could have an index in them. + // They may be pointers, but if so they are dead. + // Skip R0 aka ZERO, R1 aka SP, R2 aka SB + MOVD R3, 48(R1) + MOVD R4, 56(R1) + MOVD R5, 64(R1) + MOVD R6, 72(R1) + MOVD R7, 80(R1) + MOVD R8, 88(R1) + MOVD R9, 96(R1) + // Note: we only save 7 registers to keep under nosplit stack limit + // Also, R11 is clobbered in dynamic linking situations + + MOVD LR, R3 // PC immediately after call to panicBounds + ADD $48, R1, R4 // pointer to save area + CALL runtime·panicBounds64(SB) + RET // These functions are used when internal linking cgo with external // objects compiled with the -Os on gcc. They reduce prologue/epilogue diff --git a/src/runtime/asm_riscv64.h b/src/runtime/asm_riscv64.h new file mode 100644 index 00000000000000..2414b9f067675a --- /dev/null +++ b/src/runtime/asm_riscv64.h @@ -0,0 +1,21 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Define features that are guaranteed to be supported by setting the GORISCV64 variable. +// If a feature is supported, there's no need to check it at runtime every time. + +#ifdef GORISCV64_rva22u64 +#define hasZba +#define hasZbb +#define hasZbs +#endif + +#ifdef GORISCV64_rva23u64 +#define hasV +#define hasZba +#define hasZbb +#define hasZbs +#define hasZfa +#define hasZicond +#endif diff --git a/src/runtime/asm_riscv64.s b/src/runtime/asm_riscv64.s index 491635b1cf1a3d..5bd16181ee2b87 100644 --- a/src/runtime/asm_riscv64.s +++ b/src/runtime/asm_riscv64.s @@ -6,6 +6,104 @@ #include "funcdata.h" #include "textflag.h" + +// When building with -buildmode=c-shared, this symbol is called when the shared +// library is loaded. +TEXT _rt0_riscv64_lib(SB),NOSPLIT,$224 + // Preserve callee-save registers, along with X1 (LR). + MOV X1, (8*3)(X2) + MOV X8, (8*4)(X2) + MOV X9, (8*5)(X2) + MOV X18, (8*6)(X2) + MOV X19, (8*7)(X2) + MOV X20, (8*8)(X2) + MOV X21, (8*9)(X2) + MOV X22, (8*10)(X2) + MOV X23, (8*11)(X2) + MOV X24, (8*12)(X2) + MOV X25, (8*13)(X2) + MOV X26, (8*14)(X2) + MOV g, (8*15)(X2) + MOVD F8, (8*16)(X2) + MOVD F9, (8*17)(X2) + MOVD F18, (8*18)(X2) + MOVD F19, (8*19)(X2) + MOVD F20, (8*20)(X2) + MOVD F21, (8*21)(X2) + MOVD F22, (8*22)(X2) + MOVD F23, (8*23)(X2) + MOVD F24, (8*24)(X2) + MOVD F25, (8*25)(X2) + MOVD F26, (8*26)(X2) + MOVD F27, (8*27)(X2) + + // Initialize g as nil in case of using g later e.g. sigaction in cgo_sigaction.go + MOV X0, g + + MOV A0, _rt0_riscv64_lib_argc<>(SB) + MOV A1, _rt0_riscv64_lib_argv<>(SB) + + // Synchronous initialization. + MOV $runtime·libpreinit(SB), T0 + JALR RA, T0 + + // Create a new thread to do the runtime initialization and return. + MOV _cgo_sys_thread_create(SB), T0 + BEQZ T0, nocgo + MOV $_rt0_riscv64_lib_go(SB), A0 + MOV $0, A1 + JALR RA, T0 + JMP restore + +nocgo: + MOV $0x800000, A0 // stacksize = 8192KB + MOV $_rt0_riscv64_lib_go(SB), A1 + MOV A0, 8(X2) + MOV A1, 16(X2) + MOV $runtime·newosproc0(SB), T0 + JALR RA, T0 + +restore: + // Restore callee-save registers, along with X1 (LR). + MOV (8*3)(X2), X1 + MOV (8*4)(X2), X8 + MOV (8*5)(X2), X9 + MOV (8*6)(X2), X18 + MOV (8*7)(X2), X19 + MOV (8*8)(X2), X20 + MOV (8*9)(X2), X21 + MOV (8*10)(X2), X22 + MOV (8*11)(X2), X23 + MOV (8*12)(X2), X24 + MOV (8*13)(X2), X25 + MOV (8*14)(X2), X26 + MOV (8*15)(X2), g + MOVD (8*16)(X2), F8 + MOVD (8*17)(X2), F9 + MOVD (8*18)(X2), F18 + MOVD (8*19)(X2), F19 + MOVD (8*20)(X2), F20 + MOVD (8*21)(X2), F21 + MOVD (8*22)(X2), F22 + MOVD (8*23)(X2), F23 + MOVD (8*24)(X2), F24 + MOVD (8*25)(X2), F25 + MOVD (8*26)(X2), F26 + MOVD (8*27)(X2), F27 + + RET + +TEXT _rt0_riscv64_lib_go(SB),NOSPLIT,$0 + MOV _rt0_riscv64_lib_argc<>(SB), A0 + MOV _rt0_riscv64_lib_argv<>(SB), A1 + MOV $runtime·rt0_go(SB), T0 + JALR ZERO, T0 + +DATA _rt0_riscv64_lib_argc<>(SB)/8, $0 +GLOBL _rt0_riscv64_lib_argc<>(SB),NOPTR, $8 +DATA _rt0_riscv64_lib_argv<>(SB)/8, $0 +GLOBL _rt0_riscv64_lib_argv<>(SB),NOPTR, $8 + // func rt0_go() TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 // X2 = stack; A0 = argc; A1 = argv @@ -80,12 +178,11 @@ TEXT setg_gcc<>(SB),NOSPLIT,$0-0 RET // func cputicks() int64 -TEXT runtime·cputicks(SB),NOSPLIT,$0-8 +TEXT runtime·cputicks(SB),NOSPLIT,$0-0 // RDTIME to emulate cpu ticks // RDCYCLE reads counter that is per HART(core) based // according to the riscv manual, see issue 46737 - RDTIME A0 - MOV A0, ret+0(FP) + RDTIME X10 RET // systemstack_switch is a dummy routine that systemstack leaves at the bottom @@ -248,11 +345,6 @@ TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24 JMP runtime·memhash64Fallback(SB) -// func return0() -TEXT runtime·return0(SB), NOSPLIT, $0 - MOV $0, A0 - RET - // restore state from Gobuf; longjmp // func gogo(buf *gobuf) @@ -268,17 +360,15 @@ TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 MOV gobuf_sp(T0), X2 MOV gobuf_lr(T0), RA - MOV gobuf_ret(T0), A0 MOV gobuf_ctxt(T0), CTXT MOV ZERO, gobuf_sp(T0) - MOV ZERO, gobuf_ret(T0) MOV ZERO, gobuf_lr(T0) MOV ZERO, gobuf_ctxt(T0) MOV gobuf_pc(T0), T0 JALR ZERO, T0 -// func procyield(cycles uint32) -TEXT runtime·procyield(SB),NOSPLIT,$0-0 +// func procyieldAsm(cycles uint32) +TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0 RET // Switch to m->g0's stack, call fn(g). @@ -321,7 +411,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOV X31, (g_sched+gobuf_pc)(g) MOV X2, (g_sched+gobuf_sp)(g) MOV ZERO, (g_sched+gobuf_lr)(g) - MOV ZERO, (g_sched+gobuf_ret)(g) // Assert ctxt is zero. See func save. MOV (g_sched+gobuf_ctxt)(g), X31 BEQ ZERO, X31, 2(PC) @@ -542,6 +631,15 @@ TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0 // traceback from goexit1 must hit code range of goexit MOV ZERO, ZERO // NOP + +// This is called from .init_array and follows the platform, not the Go ABI. +TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0 + // Use X31 as it is a scratch register in both the Go ABI and psABI. + MOV runtime·lastmoduledatap(SB), X31 + MOV X10, moduledata_next(X31) + MOV X10, runtime·lastmoduledatap(SB) + RET + // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. TEXT ·cgocallback(SB),NOSPLIT,$24-24 @@ -884,80 +982,32 @@ TEXT runtime·gcWriteBarrier8(SB),NOSPLIT,$0 MOV $64, X24 JMP gcWriteBarrier<>(SB) -// Note: these functions use a special calling convention to save generated code space. -// Arguments are passed in registers (ssa/gen/RISCV64Ops.go), but the space for those -// arguments are allocated in the caller's stack frame. -// These stubs write the args into that stack space and then tail call to the -// corresponding runtime handler. -// The tail call makes these stubs disappear in backtraces. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 - MOV T0, X10 - MOV T1, X11 - JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 - MOV T0, X10 - MOV T1, X11 - JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 - MOV T1, X10 - MOV T2, X11 - JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 - MOV T1, X10 - MOV T2, X11 - JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 - MOV T1, X10 - MOV T2, X11 - JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 - MOV T1, X10 - MOV T2, X11 - JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 - MOV T0, X10 - MOV T1, X11 - JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 - MOV T0, X10 - MOV T1, X11 - JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 - MOV T2, X10 - MOV T3, X11 - JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 - MOV T2, X10 - MOV T3, X11 - JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 - MOV T2, X10 - MOV T3, X11 - JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 - MOV T2, X10 - MOV T3, X11 - JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 - MOV T1, X10 - MOV T2, X11 - JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 - MOV T1, X10 - MOV T2, X11 - JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 - MOV T0, X10 - MOV T1, X11 - JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 - MOV T0, X10 - MOV T1, X11 - JMP runtime·goPanicSlice3CU(SB) -TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16 - MOV T2, X10 - MOV T3, X11 - JMP runtime·goPanicSliceConvert(SB) +TEXT runtime·panicBounds(SB),NOSPLIT,$144-0 + NO_LOCAL_POINTERS + // Save all 16 int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + // Skip X0 aka ZERO, X1 aka LR, X2 aka SP, X3 aka GP, X4 aka TP. + MOV X5, 24(X2) + MOV X6, 32(X2) + MOV X7, 40(X2) + MOV X8, 48(X2) + MOV X9, 56(X2) + MOV X10, 64(X2) + MOV X11, 72(X2) + MOV X12, 80(X2) + MOV X13, 88(X2) + MOV X14, 96(X2) + MOV X15, 104(X2) + MOV X16, 112(X2) + MOV X17, 120(X2) + MOV X18, 128(X2) + MOV X19, 136(X2) + MOV X20, 144(X2) + + MOV X1, X10 // PC immediately after call to panicBounds + ADD $24, X2, X11 // pointer to save area + CALL runtime·panicBounds64(SB) + RET DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) GLOBL runtime·mainPC(SB),RODATA,$8 diff --git a/src/runtime/asm_s390x.s b/src/runtime/asm_s390x.s index f2354a6d53614a..bb29845f5839c0 100644 --- a/src/runtime/asm_s390x.s +++ b/src/runtime/asm_s390x.s @@ -193,10 +193,8 @@ TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 MOVD 0(g), R4 MOVD gobuf_sp(R5), R15 MOVD gobuf_lr(R5), LR - MOVD gobuf_ret(R5), R3 MOVD gobuf_ctxt(R5), R12 MOVD $0, gobuf_sp(R5) - MOVD $0, gobuf_ret(R5) MOVD $0, gobuf_lr(R5) MOVD $0, gobuf_ctxt(R5) CMP R0, R0 // set condition codes for == test, needed by stack split @@ -508,7 +506,7 @@ CALLFN(·call1073741824, 1073741824) TEXT callfnMVC<>(SB),NOSPLIT|NOFRAME,$0-0 MVC $1, 0(R4), 0(R6) -TEXT runtime·procyield(SB),NOSPLIT,$0-0 +TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0 RET // Save state of caller into g->sched, @@ -522,7 +520,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOVD R1, (g_sched+gobuf_pc)(g) MOVD R15, (g_sched+gobuf_sp)(g) MOVD $0, (g_sched+gobuf_lr)(g) - MOVD $0, (g_sched+gobuf_ret)(g) // Assert ctxt is zero. See func save. MOVD (g_sched+gobuf_ctxt)(g), R1 CMPBEQ R1, $0, 2(PC) @@ -767,10 +764,6 @@ TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24 JMP runtime·memhash64Fallback(SB) -TEXT runtime·return0(SB), NOSPLIT, $0 - MOVW $0, R3 - RET - // Called from cgo wrappers, this function returns g->m->curg.stack.hi. // Must obey the gcc calling convention. TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0 @@ -899,76 +892,18 @@ TEXT runtime·gcWriteBarrier8(SB),NOSPLIT,$0 MOVD $64, R9 JMP gcWriteBarrier<>(SB) -// Note: these functions use a special calling convention to save generated code space. -// Arguments are passed in registers, but the space for those arguments are allocated -// in the caller's stack frame. These stubs write the args into that stack space and -// then tail call to the corresponding runtime handler. -// The tail call makes these stubs disappear in backtraces. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 - MOVD R0, x+0(FP) - MOVD R1, y+8(FP) - JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 - MOVD R0, x+0(FP) - MOVD R1, y+8(FP) - JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 - MOVD R1, x+0(FP) - MOVD R2, y+8(FP) - JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 - MOVD R1, x+0(FP) - MOVD R2, y+8(FP) - JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 - MOVD R1, x+0(FP) - MOVD R2, y+8(FP) - JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 - MOVD R1, x+0(FP) - MOVD R2, y+8(FP) - JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 - MOVD R0, x+0(FP) - MOVD R1, y+8(FP) - JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 - MOVD R0, x+0(FP) - MOVD R1, y+8(FP) - JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 - MOVD R2, x+0(FP) - MOVD R3, y+8(FP) - JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 - MOVD R2, x+0(FP) - MOVD R3, y+8(FP) - JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 - MOVD R2, x+0(FP) - MOVD R3, y+8(FP) - JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 - MOVD R2, x+0(FP) - MOVD R3, y+8(FP) - JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 - MOVD R1, x+0(FP) - MOVD R2, y+8(FP) - JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 - MOVD R1, x+0(FP) - MOVD R2, y+8(FP) - JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 - MOVD R0, x+0(FP) - MOVD R1, y+8(FP) - JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 - MOVD R0, x+0(FP) - MOVD R1, y+8(FP) - JMP runtime·goPanicSlice3CU(SB) -TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16 - MOVD R2, x+0(FP) - MOVD R3, y+8(FP) - JMP runtime·goPanicSliceConvert(SB) +TEXT runtime·panicBounds(SB),NOSPLIT,$144-0 + NO_LOCAL_POINTERS + // Save all 16 int registers that could have an index in them. + // They may be pointers, but if they are they are dead. + STMG R0, R12, 24(R15) + // Note that R10 @ 104 is not needed, it is an assembler temp + // skip R13 aka G @ 128 + // skip R14 aka LR @ 136 + // skip R15 aka SP @ 144 + + MOVD R14, 8(R15) // PC immediately after call to panicBounds + ADD $24, R15, R0 // pointer to save area + MOVD R0, 16(R15) + CALL runtime·panicBounds64(SB) + RET diff --git a/src/runtime/asm_wasm.s b/src/runtime/asm_wasm.s index b44a4f7dd4983d..c46cb4ae46413c 100644 --- a/src/runtime/asm_wasm.s +++ b/src/runtime/asm_wasm.s @@ -51,11 +51,9 @@ TEXT runtime·gogo(SB), NOSPLIT, $0-8 I64Load gobuf_pc(R0) I64Store $0 - MOVD gobuf_ret(R0), RET0 MOVD gobuf_ctxt(R0), CTXT // clear to help garbage collector MOVD $0, gobuf_sp(R0) - MOVD $0, gobuf_ret(R0) MOVD $0, gobuf_ctxt(R0) I32Const $1 @@ -195,10 +193,6 @@ TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24 JMP runtime·memhash64Fallback(SB) -TEXT runtime·return0(SB), NOSPLIT, $0-0 - MOVD $0, RET0 - RET - TEXT runtime·asminit(SB), NOSPLIT, $0-0 // No per-thread init. RET @@ -206,7 +200,7 @@ TEXT runtime·asminit(SB), NOSPLIT, $0-0 TEXT ·publicationBarrier(SB), NOSPLIT, $0-0 RET -TEXT runtime·procyield(SB), NOSPLIT, $0-0 // FIXME +TEXT runtime·procyieldAsm(SB), NOSPLIT, $0-0 // FIXME RET TEXT runtime·breakpoint(SB), NOSPLIT, $0-0 @@ -538,7 +532,7 @@ TEXT wasm_pc_f_loop(SB),NOSPLIT,$0 Get SP I32Const $8 I32Sub - I32Load16U $2 // PC_F + I32Load $2 // PC_F CallIndirect $0 Drop @@ -554,5 +548,73 @@ TEXT wasm_pc_f_loop(SB),NOSPLIT,$0 Return +// wasm_pc_f_loop_export is like wasm_pc_f_loop, except that this takes an +// argument (on Wasm stack) that is a PC_F, and the loop stops when we get +// to that PC in a normal return (not unwinding). +// This is for handling an wasmexport function when it needs to switch the +// stack. +TEXT wasm_pc_f_loop_export(SB),NOSPLIT,$0 + Get PAUSE + I32Eqz +outer: + If + // R1 is whether a function return normally (0) or unwinding (1). + // Start with unwinding. + I32Const $1 + Set R1 + loop: + Loop + // Get PC_F & PC_B from -8(SP) + Get SP + I32Const $8 + I32Sub + I32Load $2 // PC_F + Tee R2 + + Get R0 + I32Eq + If // PC_F == R0, we're at the stop PC + Get R1 + I32Eqz + // Break if it is a normal return + BrIf outer // actually jump to after the corresponding End + End + + Get SP + I32Const $8 + I32Sub + I32Load16U $0 // PC_B + + Get R2 // PC_F + CallIndirect $0 + Set R1 // save return/unwinding state for next iteration + + Get PAUSE + I32Eqz + BrIf loop + End + End + + I32Const $0 + Set PAUSE + + Return + TEXT wasm_export_lib(SB),NOSPLIT,$0 UNDEF + +TEXT runtime·pause(SB), NOSPLIT, $0-8 + MOVD newsp+0(FP), SP + I32Const $1 + Set PAUSE + RETUNWIND + +// Called if a wasmexport function is called before runtime initialization +TEXT runtime·notInitialized(SB), NOSPLIT, $0 + MOVD $runtime·wasmStack+(m0Stack__size-16-8)(SB), SP + I32Const $0 // entry PC_B + Call runtime·notInitialized1(SB) + Drop + I32Const $0 // entry PC_B + Call runtime·abort(SB) + UNDEF diff --git a/src/runtime/atomic_loong64.s b/src/runtime/atomic_loong64.s index 4818a827de4d20..5332d36fadb1fb 100644 --- a/src/runtime/atomic_loong64.s +++ b/src/runtime/atomic_loong64.s @@ -5,5 +5,5 @@ #include "textflag.h" TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0 - DBAR + DBAR $0x1A // StoreStore barrier RET diff --git a/src/runtime/atomic_pointer.go b/src/runtime/atomic_pointer.go index df067ede77f8eb..311ef7d0bda300 100644 --- a/src/runtime/atomic_pointer.go +++ b/src/runtime/atomic_pointer.go @@ -49,7 +49,7 @@ func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) { atomic.StorepNoWB(noescape(ptr), new) } -// atomic_storePointer is the implementation of runtime/internal/UnsafePointer.Store +// atomic_storePointer is the implementation of internal/runtime/atomic.UnsafePointer.Store // (like StoreNoWB but with the write barrier). // //go:nosplit @@ -58,7 +58,7 @@ func atomic_storePointer(ptr *unsafe.Pointer, new unsafe.Pointer) { atomicstorep(unsafe.Pointer(ptr), new) } -// atomic_casPointer is the implementation of runtime/internal/UnsafePointer.CompareAndSwap +// atomic_casPointer is the implementation of internal/runtime/atomic.UnsafePointer.CompareAndSwap // (like CompareAndSwapNoWB but with the write barrier). // //go:nosplit diff --git a/src/runtime/bitcursor_test.go b/src/runtime/bitcursor_test.go new file mode 100644 index 00000000000000..3a4c7d0fc37523 --- /dev/null +++ b/src/runtime/bitcursor_test.go @@ -0,0 +1,49 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + . "runtime" + "testing" +) + +func TestBitCursor(t *testing.T) { + ones := [5]byte{0xff, 0xff, 0xff, 0xff, 0xff} + zeros := [5]byte{0, 0, 0, 0, 0} + + for start := uintptr(0); start < 16; start++ { + for end := start + 1; end < 32; end++ { + buf := zeros + NewBitCursor(&buf[0]).Offset(start).Write(&ones[0], end-start) + + for i := uintptr(0); i < uintptr(len(buf)*8); i++ { + bit := buf[i/8] >> (i % 8) & 1 + if bit == 0 && i >= start && i < end { + t.Errorf("bit %d not set in [%d:%d]", i, start, end) + } + if bit == 1 && (i < start || i >= end) { + t.Errorf("bit %d is set outside [%d:%d]", i, start, end) + } + } + } + } + + for start := uintptr(0); start < 16; start++ { + for end := start + 1; end < 32; end++ { + buf := ones + NewBitCursor(&buf[0]).Offset(start).Write(&zeros[0], end-start) + + for i := uintptr(0); i < uintptr(len(buf)*8); i++ { + bit := buf[i/8] >> (i % 8) & 1 + if bit == 1 && i >= start && i < end { + t.Errorf("bit %d not cleared in [%d:%d]", i, start, end) + } + if bit == 0 && (i < start || i >= end) { + t.Errorf("bit %d cleared outside [%d:%d]", i, start, end) + } + } + } + } +} diff --git a/src/runtime/cgo.go b/src/runtime/cgo.go index 8285d87fcf673b..60f2403d739f34 100644 --- a/src/runtime/cgo.go +++ b/src/runtime/cgo.go @@ -15,7 +15,9 @@ import "unsafe" //go:linkname _cgo_sys_thread_create _cgo_sys_thread_create //go:linkname _cgo_notify_runtime_init_done _cgo_notify_runtime_init_done //go:linkname _cgo_callers _cgo_callers -//go:linkname _cgo_set_context_function _cgo_set_context_function +//go:linkname _cgo_set_traceback_functions _cgo_set_traceback_functions +//go:linkname _cgo_call_traceback_function _cgo_call_traceback_function +//go:linkname _cgo_call_symbolizer_function _cgo_call_symbolizer_function //go:linkname _cgo_yield _cgo_yield //go:linkname _cgo_pthread_key_created _cgo_pthread_key_created //go:linkname _cgo_bindm _cgo_bindm @@ -27,7 +29,9 @@ var ( _cgo_sys_thread_create unsafe.Pointer _cgo_notify_runtime_init_done unsafe.Pointer _cgo_callers unsafe.Pointer - _cgo_set_context_function unsafe.Pointer + _cgo_set_traceback_functions unsafe.Pointer + _cgo_call_traceback_function unsafe.Pointer + _cgo_call_symbolizer_function unsafe.Pointer _cgo_yield unsafe.Pointer _cgo_pthread_key_created unsafe.Pointer _cgo_bindm unsafe.Pointer @@ -72,11 +76,20 @@ var cgoHasExtraM bool // cgoUse should not actually be called (see cgoAlwaysFalse). func cgoUse(any) { throw("cgoUse should not be called") } +// cgoKeepAlive is called by cgo-generated code (using go:linkname to get at +// an unexported name). This call keeps its argument alive until the call site; +// cgo emits the call after the last possible use of the argument by C code. +// cgoKeepAlive is marked in the cgo-generated code as //go:noescape, so +// unlike cgoUse it does not force the argument to escape to the heap. +// This is used to implement the #cgo noescape directive. +func cgoKeepAlive(any) { throw("cgoKeepAlive should not be called") } + // cgoAlwaysFalse is a boolean value that is always false. -// The cgo-generated code says if cgoAlwaysFalse { cgoUse(p) }. +// The cgo-generated code says if cgoAlwaysFalse { cgoUse(p) }, +// or if cgoAlwaysFalse { cgoKeepAlive(p) }. // The compiler cannot see that cgoAlwaysFalse is always false, // so it emits the test and keeps the call, giving the desired -// escape analysis result. The test is cheaper than the call. +// escape/alive analysis result. The test is cheaper than the call. var cgoAlwaysFalse bool var cgo_yield = &_cgo_yield diff --git a/src/runtime/cgo/callbacks.go b/src/runtime/cgo/callbacks.go index 3c246a88b6ca91..986f61914f2670 100644 --- a/src/runtime/cgo/callbacks.go +++ b/src/runtime/cgo/callbacks.go @@ -121,13 +121,30 @@ var _cgo_bindm = &x_cgo_bindm var x_cgo_notify_runtime_init_done byte var _cgo_notify_runtime_init_done = &x_cgo_notify_runtime_init_done -// Sets the traceback context function. See runtime.SetCgoTraceback. - -//go:cgo_import_static x_cgo_set_context_function -//go:linkname x_cgo_set_context_function x_cgo_set_context_function -//go:linkname _cgo_set_context_function _cgo_set_context_function -var x_cgo_set_context_function byte -var _cgo_set_context_function = &x_cgo_set_context_function +// Sets the traceback, context, and symbolizer functions. See +// runtime.SetCgoTraceback. + +//go:cgo_import_static x_cgo_set_traceback_functions +//go:linkname x_cgo_set_traceback_functions x_cgo_set_traceback_functions +//go:linkname _cgo_set_traceback_functions _cgo_set_traceback_functions +var x_cgo_set_traceback_functions byte +var _cgo_set_traceback_functions = &x_cgo_set_traceback_functions + +// Call the traceback function registered with x_cgo_set_traceback_functions. + +//go:cgo_import_static x_cgo_call_traceback_function +//go:linkname x_cgo_call_traceback_function x_cgo_call_traceback_function +//go:linkname _cgo_call_traceback_function _cgo_call_traceback_function +var x_cgo_call_traceback_function byte +var _cgo_call_traceback_function = &x_cgo_call_traceback_function + +// Call the symbolizer function registered with x_cgo_set_symbolizer_functions. + +//go:cgo_import_static x_cgo_call_symbolizer_function +//go:linkname x_cgo_call_symbolizer_function x_cgo_call_symbolizer_function +//go:linkname _cgo_call_symbolizer_function _cgo_call_symbolizer_function +var x_cgo_call_symbolizer_function byte +var _cgo_call_symbolizer_function = &x_cgo_call_symbolizer_function // Calls a libc function to execute background work injected via libc // interceptors, such as processing pending signals under the thread diff --git a/src/runtime/cgo/cgo.go b/src/runtime/cgo/cgo.go index c37135fbbe4b16..c90fb424ac29ff 100644 --- a/src/runtime/cgo/cgo.go +++ b/src/runtime/cgo/cgo.go @@ -25,7 +25,8 @@ package cgo // Use -fno-stack-protector to avoid problems locating the // proper support functions. See issues #52919, #54313, #58385. -#cgo CFLAGS: -Wall -Werror -fno-stack-protector +// Use -Wdeclaration-after-statement because some CI builds use it. +#cgo CFLAGS: -Wall -Werror -fno-stack-protector -Wdeclaration-after-statement #cgo solaris CPPFLAGS: -D_POSIX_PTHREAD_SEMANTICS diff --git a/src/runtime/cgo/clearenv.go b/src/runtime/cgo/clearenv.go new file mode 100644 index 00000000000000..6d605e5f1ad316 --- /dev/null +++ b/src/runtime/cgo/clearenv.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux + +package cgo + +import _ "unsafe" // for go:linkname + +//go:cgo_import_static x_cgo_clearenv +//go:linkname x_cgo_clearenv x_cgo_clearenv +//go:linkname _cgo_clearenv runtime._cgo_clearenv +var x_cgo_clearenv byte +var _cgo_clearenv = &x_cgo_clearenv diff --git a/src/runtime/cgo/gcc_clearenv.c b/src/runtime/cgo/gcc_clearenv.c new file mode 100644 index 00000000000000..7657e3562d439c --- /dev/null +++ b/src/runtime/cgo/gcc_clearenv.c @@ -0,0 +1,18 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux + +#include "libcgo.h" + +#include + +/* Stub for calling clearenv */ +void +x_cgo_clearenv(void **_unused) +{ + _cgo_tsan_acquire(); + clearenv(); + _cgo_tsan_release(); +} diff --git a/src/runtime/cgo/gcc_context.c b/src/runtime/cgo/gcc_context.c index ad586928219609..b647c99a982539 100644 --- a/src/runtime/cgo/gcc_context.c +++ b/src/runtime/cgo/gcc_context.c @@ -8,11 +8,11 @@ // Releases the cgo traceback context. void _cgo_release_context(uintptr_t ctxt) { - void (*pfn)(struct context_arg*); + void (*pfn)(struct cgoContextArg*); pfn = _cgo_get_context_function(); if (ctxt != 0 && pfn != nil) { - struct context_arg arg; + struct cgoContextArg arg; arg.Context = ctxt; (*pfn)(&arg); diff --git a/src/runtime/cgo/gcc_darwin_amd64.c b/src/runtime/cgo/gcc_darwin_amd64.c index 5b5e369f26938b..1ffbbd75f26877 100644 --- a/src/runtime/cgo/gcc_darwin_amd64.c +++ b/src/runtime/cgo/gcc_darwin_amd64.c @@ -34,6 +34,7 @@ _cgo_sys_thread_start(ThreadStart *ts) size = pthread_get_stacksize_np(pthread_self()); pthread_attr_init(&attr); pthread_attr_setstacksize(&attr, size); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); diff --git a/src/runtime/cgo/gcc_darwin_arm64.c b/src/runtime/cgo/gcc_darwin_arm64.c index f1344de8e19be3..e800e9303c0a7a 100644 --- a/src/runtime/cgo/gcc_darwin_arm64.c +++ b/src/runtime/cgo/gcc_darwin_arm64.c @@ -39,6 +39,7 @@ _cgo_sys_thread_start(ThreadStart *ts) size = pthread_get_stacksize_np(pthread_self()); pthread_attr_init(&attr); pthread_attr_setstacksize(&attr, size); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); @@ -75,19 +76,27 @@ threadentry(void *v) static void init_working_dir() { - CFBundleRef bundle = CFBundleGetMainBundle(); + CFBundleRef bundle; + CFURLRef url_ref; + CFStringRef url_str_ref; + char buf[MAXPATHLEN]; + Boolean res; + int url_len; + char *dir; + CFStringRef wd_ref; + + bundle = CFBundleGetMainBundle(); if (bundle == NULL) { fprintf(stderr, "runtime/cgo: no main bundle\n"); return; } - CFURLRef url_ref = CFBundleCopyResourceURL(bundle, CFSTR("Info"), CFSTR("plist"), NULL); + url_ref = CFBundleCopyResourceURL(bundle, CFSTR("Info"), CFSTR("plist"), NULL); if (url_ref == NULL) { // No Info.plist found. It can happen on Corellium virtual devices. return; } - CFStringRef url_str_ref = CFURLGetString(url_ref); - char buf[MAXPATHLEN]; - Boolean res = CFStringGetCString(url_str_ref, buf, sizeof(buf), kCFStringEncodingUTF8); + url_str_ref = CFURLGetString(url_ref); + res = CFStringGetCString(url_str_ref, buf, sizeof(buf), kCFStringEncodingUTF8); CFRelease(url_ref); if (!res) { fprintf(stderr, "runtime/cgo: cannot get URL string\n"); @@ -96,13 +105,13 @@ init_working_dir() // url is of the form "file:///path/to/Info.plist". // strip it down to the working directory "/path/to". - int url_len = strlen(buf); + url_len = strlen(buf); if (url_len < sizeof("file://")+sizeof("/Info.plist")) { fprintf(stderr, "runtime/cgo: bad URL: %s\n", buf); return; } buf[url_len-sizeof("/Info.plist")+1] = 0; - char *dir = &buf[0] + sizeof("file://")-1; + dir = &buf[0] + sizeof("file://")-1; if (chdir(dir) != 0) { fprintf(stderr, "runtime/cgo: chdir(%s) failed\n", dir); @@ -110,7 +119,7 @@ init_working_dir() // The test harness in go_ios_exec passes the relative working directory // in the GoExecWrapperWorkingDirectory property of the app bundle. - CFStringRef wd_ref = CFBundleGetValueForInfoDictionaryKey(bundle, CFSTR("GoExecWrapperWorkingDirectory")); + wd_ref = CFBundleGetValueForInfoDictionaryKey(bundle, CFSTR("GoExecWrapperWorkingDirectory")); if (wd_ref != NULL) { if (!CFStringGetCString(wd_ref, buf, sizeof(buf), kCFStringEncodingUTF8)) { fprintf(stderr, "runtime/cgo: cannot get GoExecWrapperWorkingDirectory string\n"); diff --git a/src/runtime/cgo/gcc_dragonfly_amd64.c b/src/runtime/cgo/gcc_dragonfly_amd64.c index 009d4b4fb99e9c..646e9e653a1bff 100644 --- a/src/runtime/cgo/gcc_dragonfly_amd64.c +++ b/src/runtime/cgo/gcc_dragonfly_amd64.c @@ -33,6 +33,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. diff --git a/src/runtime/cgo/gcc_freebsd.c b/src/runtime/cgo/gcc_freebsd.c index a94121138d5902..1cc582a3bbc37f 100644 --- a/src/runtime/cgo/gcc_freebsd.c +++ b/src/runtime/cgo/gcc_freebsd.c @@ -45,6 +45,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_freebsd_amd64.c b/src/runtime/cgo/gcc_freebsd_amd64.c index 31905f2aa3c4ef..ec23a00fee38e8 100644 --- a/src/runtime/cgo/gcc_freebsd_amd64.c +++ b/src/runtime/cgo/gcc_freebsd_amd64.c @@ -43,6 +43,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_libinit.c b/src/runtime/cgo/gcc_libinit.c index 33a9ff93cad527..05998fadf81456 100644 --- a/src/runtime/cgo/gcc_libinit.c +++ b/src/runtime/cgo/gcc_libinit.c @@ -32,13 +32,24 @@ static void pthread_key_destructor(void* g); uintptr_t x_cgo_pthread_key_created; void (*x_crosscall2_ptr)(void (*fn)(void *), void *, int, size_t); +// The traceback function, used when tracing C calls. +static void (*cgo_traceback_function)(struct cgoTracebackArg*); + // The context function, used when tracing back C calls into Go. -static void (*cgo_context_function)(struct context_arg*); +static void (*cgo_context_function)(struct cgoContextArg*); + +// The symbolizer function, used when symbolizing C frames. +static void (*cgo_symbolizer_function)(struct cgoSymbolizerArg*); void x_cgo_sys_thread_create(void* (*func)(void*), void* arg) { + pthread_attr_t attr; pthread_t p; - int err = _cgo_try_pthread_create(&p, NULL, func, arg); + int err; + + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + err = _cgo_try_pthread_create(&p, &attr, func, arg); if (err != 0) { fprintf(stderr, "pthread_create failed: %s", strerror(err)); abort(); @@ -47,10 +58,12 @@ x_cgo_sys_thread_create(void* (*func)(void*), void* arg) { uintptr_t _cgo_wait_runtime_init_done(void) { - void (*pfn)(struct context_arg*); + void (*pfn)(struct cgoContextArg*); + int done; + pfn = __atomic_load_n(&cgo_context_function, __ATOMIC_CONSUME); - int done = 2; + done = 2; if (__atomic_load_n(&runtime_init_done, __ATOMIC_CONSUME) != done) { pthread_mutex_lock(&runtime_init_mu); while (__atomic_load_n(&runtime_init_done, __ATOMIC_CONSUME) == 0) { @@ -63,7 +76,6 @@ _cgo_wait_runtime_init_done(void) { x_cgo_pthread_key_created = 1; } - // TODO(iant): For the case of a new C thread calling into Go, such // as when using -buildmode=c-archive, we know that Go runtime // initialization is complete but we do not know that all Go init @@ -80,7 +92,7 @@ _cgo_wait_runtime_init_done(void) { } if (pfn != nil) { - struct context_arg arg; + struct cgoContextArg arg; arg.Context = 0; (*pfn)(&arg); @@ -131,17 +143,71 @@ x_cgo_notify_runtime_init_done(void* dummy __attribute__ ((unused))) { pthread_mutex_unlock(&runtime_init_mu); } -// Sets the context function to call to record the traceback context -// when calling a Go function from C code. Called from runtime.SetCgoTraceback. -void x_cgo_set_context_function(void (*context)(struct context_arg*)) { - __atomic_store_n(&cgo_context_function, context, __ATOMIC_RELEASE); +// Sets the traceback, context, and symbolizer functions. Called from +// runtime.SetCgoTraceback. +void x_cgo_set_traceback_functions(struct cgoSetTracebackFunctionsArg* arg) { + __atomic_store_n(&cgo_traceback_function, arg->Traceback, __ATOMIC_RELEASE); + __atomic_store_n(&cgo_context_function, arg->Context, __ATOMIC_RELEASE); + __atomic_store_n(&cgo_symbolizer_function, arg->Symbolizer, __ATOMIC_RELEASE); +} + +// Gets the traceback function to call to trace C calls. +void (*(_cgo_get_traceback_function(void)))(struct cgoTracebackArg*) { + return __atomic_load_n(&cgo_traceback_function, __ATOMIC_CONSUME); } -// Gets the context function. -void (*(_cgo_get_context_function(void)))(struct context_arg*) { +// Call the traceback function registered with x_cgo_set_traceback_functions. +// +// The traceback function is an arbitrary user C function which may be built +// with TSAN, and thus must be wrapped with TSAN acquire/release calls. For +// normal cgo calls, cmd/cgo automatically inserts TSAN acquire/release calls. +// Since the traceback, context, and symbolizer functions are registered at +// startup and called via the runtime, they do not get automatic TSAN +// acquire/release calls. +// +// The only purpose of this wrapper is to perform TSAN acquire/release. +// Alternatively, if the runtime arranged to safely call TSAN acquire/release, +// it could perform the call directly. +void x_cgo_call_traceback_function(struct cgoTracebackArg* arg) { + void (*pfn)(struct cgoTracebackArg*); + + pfn = _cgo_get_traceback_function(); + if (pfn == nil) { + return; + } + + _cgo_tsan_acquire(); + (*pfn)(arg); + _cgo_tsan_release(); +} + +// Gets the context function to call to record the traceback context +// when calling a Go function from C code. +void (*(_cgo_get_context_function(void)))(struct cgoContextArg*) { return __atomic_load_n(&cgo_context_function, __ATOMIC_CONSUME); } +// Gets the symbolizer function to call to symbolize C frames. +void (*(_cgo_get_symbolizer_function(void)))(struct cgoSymbolizerArg*) { + return __atomic_load_n(&cgo_symbolizer_function, __ATOMIC_CONSUME); +} + +// Call the symbolizer function registered with x_cgo_set_traceback_functions. +// +// See comment on x_cgo_call_traceback_function. +void x_cgo_call_symbolizer_function(struct cgoSymbolizerArg* arg) { + void (*pfn)(struct cgoSymbolizerArg*); + + pfn = _cgo_get_symbolizer_function(); + if (pfn == nil) { + return; + } + + _cgo_tsan_acquire(); + (*pfn)(arg); + _cgo_tsan_release(); +} + // _cgo_try_pthread_create retries pthread_create if it fails with // EAGAIN. int @@ -153,7 +219,6 @@ _cgo_try_pthread_create(pthread_t* thread, const pthread_attr_t* attr, void* (*p for (tries = 0; tries < 20; tries++) { err = pthread_create(thread, attr, pfn, arg); if (err == 0) { - pthread_detach(*thread); return 0; } if (err != EAGAIN) { diff --git a/src/runtime/cgo/gcc_libinit_windows.c b/src/runtime/cgo/gcc_libinit_windows.c index 9a8c65ea291ad8..7e7ff3e667266f 100644 --- a/src/runtime/cgo/gcc_libinit_windows.c +++ b/src/runtime/cgo/gcc_libinit_windows.c @@ -2,26 +2,19 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#ifdef __CYGWIN__ +#error "don't use the cygwin compiler to build native Windows programs; use MinGW instead" +#endif + #define WIN32_LEAN_AND_MEAN #include -#include #include #include -#include #include "libcgo.h" #include "libcgo_windows.h" -// Ensure there's one symbol marked __declspec(dllexport). -// If there are no exported symbols, the unfortunate behavior of -// the binutils linker is to also strip the relocations table, -// resulting in non-PIE binary. The other option is the -// --export-all-symbols flag, but we don't need to export all symbols -// and this may overflow the export table (#40795). -// See https://sourceware.org/bugzilla/show_bug.cgi?id=19011 -__declspec(dllexport) int _cgo_dummy_export; - static volatile LONG runtime_init_once_gate = 0; static volatile LONG runtime_init_once_done = 0; @@ -30,6 +23,7 @@ static CRITICAL_SECTION runtime_init_cs; static HANDLE runtime_init_wait; static int runtime_init_done; +// No pthreads on Windows, these are always zero. uintptr_t x_cgo_pthread_key_created; void (*x_crosscall2_ptr)(void (*fn)(void *), void *, int, size_t); @@ -63,21 +57,23 @@ _cgo_maybe_run_preinit() { } void -x_cgo_sys_thread_create(void (*func)(void*), void* arg) { +x_cgo_sys_thread_create(unsigned long (__stdcall *func)(void*), void* arg) { _cgo_beginthread(func, arg); } int _cgo_is_runtime_initialized() { + int status; + EnterCriticalSection(&runtime_init_cs); - int status = runtime_init_done; + status = runtime_init_done; LeaveCriticalSection(&runtime_init_cs); return status; } uintptr_t _cgo_wait_runtime_init_done(void) { - void (*pfn)(struct context_arg*); + void (*pfn)(struct cgoContextArg*); _cgo_maybe_run_preinit(); while (!_cgo_is_runtime_initialized()) { @@ -85,7 +81,7 @@ _cgo_wait_runtime_init_done(void) { } pfn = _cgo_get_context_function(); if (pfn != nil) { - struct context_arg arg; + struct cgoContextArg arg; arg.Context = 0; (*pfn)(&arg); @@ -114,20 +110,54 @@ x_cgo_notify_runtime_init_done(void* dummy) { } } +// The traceback function, used when tracing C calls. +static void (*cgo_traceback_function)(struct cgoTracebackArg*); + // The context function, used when tracing back C calls into Go. -static void (*cgo_context_function)(struct context_arg*); +static void (*cgo_context_function)(struct cgoContextArg*); -// Sets the context function to call to record the traceback context -// when calling a Go function from C code. Called from runtime.SetCgoTraceback. -void x_cgo_set_context_function(void (*context)(struct context_arg*)) { +// The symbolizer function, used when symbolizing C frames. +static void (*cgo_symbolizer_function)(struct cgoSymbolizerArg*); + +// Sets the traceback, context, and symbolizer functions. Called from +// runtime.SetCgoTraceback. +void x_cgo_set_traceback_functions(struct cgoSetTracebackFunctionsArg* arg) { EnterCriticalSection(&runtime_init_cs); - cgo_context_function = context; + cgo_traceback_function = arg->Traceback; + cgo_context_function = arg->Context; + cgo_symbolizer_function = arg->Symbolizer; LeaveCriticalSection(&runtime_init_cs); } -// Gets the context function. -void (*(_cgo_get_context_function(void)))(struct context_arg*) { - void (*ret)(struct context_arg*); +// Gets the traceback function to call to trace C calls. +void (*(_cgo_get_traceback_function(void)))(struct cgoTracebackArg*) { + void (*ret)(struct cgoTracebackArg*); + + EnterCriticalSection(&runtime_init_cs); + ret = cgo_traceback_function; + LeaveCriticalSection(&runtime_init_cs); + return ret; +} + +// Call the traceback function registered with x_cgo_set_traceback_functions. +// +// On other platforms, this coordinates with C/C++ TSAN. On Windows, there is +// no C/C++ TSAN. +void x_cgo_call_traceback_function(struct cgoTracebackArg* arg) { + void (*pfn)(struct cgoTracebackArg*); + + pfn = _cgo_get_traceback_function(); + if (pfn == nil) { + return; + } + + (*pfn)(arg); +} + +// Gets the context function to call to record the traceback context +// when calling a Go function from C code. +void (*(_cgo_get_context_function(void)))(struct cgoContextArg*) { + void (*ret)(struct cgoContextArg*); EnterCriticalSection(&runtime_init_cs); ret = cgo_context_function; @@ -135,24 +165,50 @@ void (*(_cgo_get_context_function(void)))(struct context_arg*) { return ret; } -void _cgo_beginthread(void (*func)(void*), void* arg) { +// Gets the symbolizer function to call to symbolize C frames. +void (*(_cgo_get_symbolizer_function(void)))(struct cgoSymbolizerArg*) { + void (*ret)(struct cgoSymbolizerArg*); + + EnterCriticalSection(&runtime_init_cs); + ret = cgo_symbolizer_function; + LeaveCriticalSection(&runtime_init_cs); + return ret; +} + +// Call the symbolizer function registered with x_cgo_set_symbolizer_functions. +// +// On other platforms, this coordinates with C/C++ TSAN. On Windows, there is +// no C/C++ TSAN. +void x_cgo_call_symbolizer_function(struct cgoSymbolizerArg* arg) { + void (*pfn)(struct cgoSymbolizerArg*); + + pfn = _cgo_get_symbolizer_function(); + if (pfn == nil) { + return; + } + + (*pfn)(arg); +} + +void _cgo_beginthread(unsigned long (__stdcall *func)(void*), void* arg) { int tries; - uintptr_t thandle; + HANDLE thandle; for (tries = 0; tries < 20; tries++) { - thandle = _beginthread(func, 0, arg); - if (thandle == -1 && errno == EACCES) { + thandle = CreateThread(NULL, 0, func, arg, 0, NULL); + if (thandle == 0 && GetLastError() == ERROR_ACCESS_DENIED) { // "Insufficient resources", try again in a bit. // // Note that the first Sleep(0) is a yield. Sleep(tries); // milliseconds continue; - } else if (thandle == -1) { + } else if (thandle == 0) { break; } + CloseHandle(thandle); return; // Success! } - fprintf(stderr, "runtime: failed to create new OS thread (%d)\n", errno); + fprintf(stderr, "runtime: failed to create new OS thread (%lu)\n", GetLastError()); abort(); } diff --git a/src/runtime/cgo/gcc_linux.c b/src/runtime/cgo/gcc_linux.c index 9624df596feffe..fbdb5e652a2142 100644 --- a/src/runtime/cgo/gcc_linux.c +++ b/src/runtime/cgo/gcc_linux.c @@ -40,6 +40,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_linux_amd64.c b/src/runtime/cgo/gcc_linux_amd64.c index dcb596e213750c..c81fde2887d518 100644 --- a/src/runtime/cgo/gcc_linux_amd64.c +++ b/src/runtime/cgo/gcc_linux_amd64.c @@ -63,6 +63,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_linux_arm64.c b/src/runtime/cgo/gcc_linux_arm64.c index 0dcff2c0901406..ce46804ba29738 100644 --- a/src/runtime/cgo/gcc_linux_arm64.c +++ b/src/runtime/cgo/gcc_linux_arm64.c @@ -28,6 +28,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_linux_s390x.c b/src/runtime/cgo/gcc_linux_s390x.c index 4b9f76c39ef839..1fae4e7d44142a 100644 --- a/src/runtime/cgo/gcc_linux_s390x.c +++ b/src/runtime/cgo/gcc_linux_s390x.c @@ -33,6 +33,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_loong64.S b/src/runtime/cgo/gcc_loong64.S index 6b7668f2889bec..d2b062f49f9ce5 100644 --- a/src/runtime/cgo/gcc_loong64.S +++ b/src/runtime/cgo/gcc_loong64.S @@ -8,23 +8,23 @@ * void crosscall1(void (*fn)(void), void (*setg_gcc)(void *g), void *g) * * Calling into the gc tool chain, where all registers are caller save. - * Called from standard lp64d ABI, where $r1, $r3, $r23-$r30, and $f24-$f31 + * Called from standard lp64d ABI, where $r1, $r3, $r22-$r31, and $f24-$f31 * are callee-save, so they must be saved explicitly, along with $r1 (LR). */ .globl crosscall1 crosscall1: addi.d $r3, $r3, -160 st.d $r1, $r3, 0 - st.d $r23, $r3, 8 - st.d $r24, $r3, 16 - st.d $r25, $r3, 24 - st.d $r26, $r3, 32 - st.d $r27, $r3, 40 - st.d $r28, $r3, 48 - st.d $r29, $r3, 56 - st.d $r30, $r3, 64 - st.d $r2, $r3, 72 - st.d $r22, $r3, 80 + st.d $r22, $r3, 8 + st.d $r23, $r3, 16 + st.d $r24, $r3, 24 + st.d $r25, $r3, 32 + st.d $r26, $r3, 40 + st.d $r27, $r3, 48 + st.d $r28, $r3, 56 + st.d $r29, $r3, 64 + st.d $r30, $r3, 72 + st.d $r31, $r3, 80 fst.d $f24, $r3, 88 fst.d $f25, $r3, 96 fst.d $f26, $r3, 104 @@ -34,21 +34,22 @@ crosscall1: fst.d $f30, $r3, 136 fst.d $f31, $r3, 144 - move $r18, $r4 // save R4 - move $r19, $r6 + // r4 = *fn, r5 = *setg_gcc, r6 = *g + move $r23, $r4 // save R4 + move $r4, $r6 jirl $r1, $r5, 0 // call setg_gcc (clobbers R4) - jirl $r1, $r18, 0 // call fn + jirl $r1, $r23, 0 // call fn - ld.d $r23, $r3, 8 - ld.d $r24, $r3, 16 - ld.d $r25, $r3, 24 - ld.d $r26, $r3, 32 - ld.d $r27, $r3, 40 - ld.d $r28, $r3, 48 - ld.d $r29, $r3, 56 - ld.d $r30, $r3, 64 - ld.d $r2, $r3, 72 - ld.d $r22, $r3, 80 + ld.d $r22, $r3, 8 + ld.d $r23, $r3, 16 + ld.d $r24, $r3, 24 + ld.d $r25, $r3, 32 + ld.d $r26, $r3, 40 + ld.d $r27, $r3, 48 + ld.d $r28, $r3, 56 + ld.d $r29, $r3, 64 + ld.d $r30, $r3, 72 + ld.d $r31, $r3, 80 fld.d $f24, $r3, 88 fld.d $f25, $r3, 96 fld.d $f26, $r3, 104 diff --git a/src/runtime/cgo/gcc_netbsd.c b/src/runtime/cgo/gcc_netbsd.c index 16819ce8b47077..98b629e805b365 100644 --- a/src/runtime/cgo/gcc_netbsd.c +++ b/src/runtime/cgo/gcc_netbsd.c @@ -35,6 +35,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_openbsd.c b/src/runtime/cgo/gcc_openbsd.c index 3a4e545496603b..b31c312f590e9c 100644 --- a/src/runtime/cgo/gcc_openbsd.c +++ b/src/runtime/cgo/gcc_openbsd.c @@ -34,6 +34,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. diff --git a/src/runtime/cgo/gcc_ppc64x.c b/src/runtime/cgo/gcc_ppc64x.c index 98a654957b189a..e1a5c14ff94f05 100644 --- a/src/runtime/cgo/gcc_ppc64x.c +++ b/src/runtime/cgo/gcc_ppc64x.c @@ -35,6 +35,7 @@ _cgo_sys_thread_start(ThreadStart *ts) pthread_sigmask(SIG_SETMASK, &ign, &oset); pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); pthread_attr_getstacksize(&attr, &size); // Leave stacklo=0 and set stackhi=size; mstart will do the rest. ts->g->stackhi = size; diff --git a/src/runtime/cgo/gcc_sigaction.c b/src/runtime/cgo/gcc_sigaction.c index 374909bf9768e1..ad48a88dc16112 100644 --- a/src/runtime/cgo/gcc_sigaction.c +++ b/src/runtime/cgo/gcc_sigaction.c @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build linux && (amd64 || arm64 || ppc64le) +//go:build linux && (386 || amd64 || arm64 || loong64 || ppc64le) #include #include @@ -13,13 +13,18 @@ #include "libcgo.h" // go_sigaction_t is a C version of the sigactiont struct from -// defs_linux_amd64.go. This definition — and its conversion to and from struct -// sigaction — are specific to linux/amd64. +// defs_${goos}_${goarch}.go. This definition — and its conversion +// to and from struct sigaction — are specific to ${goos}/${goarch}. typedef struct { uintptr_t handler; - uint64_t flags; + unsigned long flags; +#ifdef __loongarch__ + uint64_t mask; + uintptr_t restorer; +#else uintptr_t restorer; uint64_t mask; +#endif } go_sigaction_t; // SA_RESTORER is part of the kernel interface. @@ -52,7 +57,7 @@ x_cgo_sigaction(intptr_t signum, const go_sigaction_t *goact, go_sigaction_t *ol sigaddset(&act.sa_mask, (int)(i+1)); } } - act.sa_flags = (int)(goact->flags & ~(uint64_t)SA_RESTORER); + act.sa_flags = (int)(goact->flags & ~(unsigned long)SA_RESTORER); } ret = sigaction((int)signum, goact ? &act : NULL, oldgoact ? &oldact : NULL); @@ -74,7 +79,7 @@ x_cgo_sigaction(intptr_t signum, const go_sigaction_t *goact, go_sigaction_t *ol oldgoact->mask |= (uint64_t)(1)<flags = (uint64_t)oldact.sa_flags; + oldgoact->flags = (unsigned long)oldact.sa_flags; } _cgo_tsan_release(); diff --git a/src/runtime/cgo/gcc_stack_unix.c b/src/runtime/cgo/gcc_stack_unix.c index fcb03d0dea7e34..9550cd783941a8 100644 --- a/src/runtime/cgo/gcc_stack_unix.c +++ b/src/runtime/cgo/gcc_stack_unix.c @@ -21,7 +21,7 @@ x_cgo_getstackbound(uintptr bounds[2]) // Needed before pthread_getattr_np, too, since before glibc 2.32 // it did not call pthread_attr_init in all cases (see #65625). pthread_attr_init(&attr); -#if defined(__GLIBC__) || (defined(__sun) && !defined(__illumos__)) +#if defined(__GLIBC__) || defined(__BIONIC__) || (defined(__sun) && !defined(__illumos__)) // pthread_getattr_np is a GNU extension supported in glibc. // Solaris is not glibc but does support pthread_getattr_np // (and the fallback doesn't work...). Illumos does not. @@ -31,10 +31,11 @@ x_cgo_getstackbound(uintptr bounds[2]) pthread_attr_get_np(pthread_self(), &attr); pthread_attr_getstack(&attr, &addr, &size); // low address #else - // We don't know how to get the current stacks, so assume they are the - // same as the default stack bounds. - pthread_attr_getstacksize(&attr, &size); - addr = __builtin_frame_address(0) + 4096 - size; + // We don't know how to get the current stacks, leave it as + // 0 and the caller will use an estimate based on the current + // SP. + addr = 0; + size = 0; #endif pthread_attr_destroy(&attr); diff --git a/src/runtime/cgo/gcc_windows_386.c b/src/runtime/cgo/gcc_windows_386.c index 983e14b7c8ff3b..d394dc6acb9318 100644 --- a/src/runtime/cgo/gcc_windows_386.c +++ b/src/runtime/cgo/gcc_windows_386.c @@ -11,7 +11,7 @@ #include "libcgo.h" #include "libcgo_windows.h" -static void threadentry(void*); +static unsigned long __stdcall threadentry(void*); static void (*setg_gcc)(void*); static DWORD *tls_g; @@ -29,7 +29,8 @@ _cgo_sys_thread_start(ThreadStart *ts) } extern void crosscall1(void (*fn)(void), void (*setg_gcc)(void*), void *g); -static void +static unsigned long +__stdcall threadentry(void *v) { ThreadStart ts; @@ -50,4 +51,5 @@ threadentry(void *v) ); crosscall1(ts.fn, setg_gcc, ts.g); + return 0; } diff --git a/src/runtime/cgo/gcc_windows_amd64.c b/src/runtime/cgo/gcc_windows_amd64.c index e26887a1727cae..455b6b7cb29f28 100644 --- a/src/runtime/cgo/gcc_windows_amd64.c +++ b/src/runtime/cgo/gcc_windows_amd64.c @@ -11,7 +11,7 @@ #include "libcgo.h" #include "libcgo_windows.h" -static void threadentry(void*); +static unsigned long __stdcall threadentry(void*); static void (*setg_gcc)(void*); static DWORD *tls_g; @@ -30,7 +30,8 @@ _cgo_sys_thread_start(ThreadStart *ts) } extern void crosscall1(void (*fn)(void), void (*setg_gcc)(void*), void *g); -static void +static unsigned long +__stdcall threadentry(void *v) { ThreadStart ts; @@ -49,4 +50,5 @@ threadentry(void *v) ); crosscall1(ts.fn, setg_gcc, (void*)ts.g); + return 0; } diff --git a/src/runtime/cgo/gcc_windows_arm64.c b/src/runtime/cgo/gcc_windows_arm64.c index 8f113cc3b16727..7f02d4bd5c048b 100644 --- a/src/runtime/cgo/gcc_windows_arm64.c +++ b/src/runtime/cgo/gcc_windows_arm64.c @@ -11,7 +11,7 @@ #include "libcgo.h" #include "libcgo_windows.h" -static void threadentry(void*); +static unsigned long __stdcall threadentry(void*); static void (*setg_gcc)(void*); void @@ -28,7 +28,8 @@ _cgo_sys_thread_start(ThreadStart *ts) extern void crosscall1(void (*fn)(void), void (*setg_gcc)(void*), void *g); -static void +static unsigned long +__stdcall threadentry(void *v) { ThreadStart ts; @@ -37,4 +38,5 @@ threadentry(void *v) free(v); crosscall1(ts.fn, setg_gcc, (void *)ts.g); + return 0; } diff --git a/src/runtime/cgo/handle.go b/src/runtime/cgo/handle.go index 59b65da2b6d763..7d6dd9146c30d3 100644 --- a/src/runtime/cgo/handle.go +++ b/src/runtime/cgo/handle.go @@ -61,10 +61,16 @@ import ( // } // // Some C functions accept a void* argument that points to an arbitrary -// data value supplied by the caller. It is not safe to coerce a [cgo.Handle] +// data value supplied by the caller. It is not safe to coerce a Handle // (an integer) to a Go [unsafe.Pointer], but instead we can pass the address // of the cgo.Handle to the void* parameter, as in this variant of the -// previous example: +// previous example. +// +// Note that, as described in the [cmd/cgo] documentation, +// the C code must not keep a copy of the Go pointer that it receives, +// unless the memory is explicitly pinned using [runtime.Pinner]. +// This example is OK because the C function myprint does not keep +// a copy of the pointer. // // package main // diff --git a/src/runtime/cgo/libcgo.h b/src/runtime/cgo/libcgo.h index 26da68fadb67d0..aa0b57d6d7a04c 100644 --- a/src/runtime/cgo/libcgo.h +++ b/src/runtime/cgo/libcgo.h @@ -89,15 +89,7 @@ void darwin_arm_init_thread_exception_port(void); void darwin_arm_init_mach_exception_handler(void); /* - * The cgo context function. See runtime.SetCgoTraceback. - */ -struct context_arg { - uintptr_t Context; -}; -extern void (*(_cgo_get_context_function(void)))(struct context_arg*); - -/* - * The argument for the cgo traceback callback. See runtime.SetCgoTraceback. + * The cgo traceback callback. See runtime.SetCgoTraceback. */ struct cgoTracebackArg { uintptr_t Context; @@ -105,6 +97,38 @@ struct cgoTracebackArg { uintptr_t* Buf; uintptr_t Max; }; +extern void (*(_cgo_get_traceback_function(void)))(struct cgoTracebackArg*); + +/* + * The cgo context callback. See runtime.SetCgoTraceback. + */ +struct cgoContextArg { + uintptr_t Context; +}; +extern void (*(_cgo_get_context_function(void)))(struct cgoContextArg*); + +/* + * The argument for the cgo symbolizer callback. See runtime.SetCgoTraceback. + */ +struct cgoSymbolizerArg { + uintptr_t PC; + const char* File; + uintptr_t Lineno; + const char* Func; + uintptr_t Entry; + uintptr_t More; + uintptr_t Data; +}; +extern void (*(_cgo_get_symbolizer_function(void)))(struct cgoSymbolizerArg*); + +/* + * The argument for x_cgo_set_traceback_functions. See runtime.SetCgoTraceback. + */ +struct cgoSetTracebackFunctionsArg { + void (*Traceback)(struct cgoTracebackArg*); + void (*Context)(struct cgoContextArg*); + void (*Symbolizer)(struct cgoSymbolizerArg*); +}; /* * TSAN support. This is only useful when building with @@ -121,11 +145,21 @@ struct cgoTracebackArg { #ifdef CGO_TSAN +// _cgo_tsan_acquire tells C/C++ TSAN that we are acquiring a dummy lock. We +// call this when calling from Go to C. This is necessary because TSAN cannot +// see the synchronization in Go. Note that C/C++ code built with TSAN is not +// the same as the Go race detector. +// +// cmd/cgo generates calls to _cgo_tsan_acquire and _cgo_tsan_release. For +// other cgo calls, manual calls are required. +// // These must match the definitions in yesTsanProlog in cmd/cgo/out.go. // In general we should call _cgo_tsan_acquire when we enter C code, // and call _cgo_tsan_release when we return to Go code. +// // This is only necessary when calling code that might be instrumented // by TSAN, which mostly means system library calls that TSAN intercepts. +// // See the comment in cmd/cgo/out.go for more details. long long _cgo_sync __attribute__ ((common)); diff --git a/src/runtime/cgo/libcgo_windows.h b/src/runtime/cgo/libcgo_windows.h index 33d7637fece0ec..682b7bdbbf2c1c 100644 --- a/src/runtime/cgo/libcgo_windows.h +++ b/src/runtime/cgo/libcgo_windows.h @@ -3,4 +3,4 @@ // license that can be found in the LICENSE file. // Call _beginthread, aborting on failure. -void _cgo_beginthread(void (*func)(void*), void* arg); +void _cgo_beginthread(unsigned long (__stdcall *func)(void*), void* arg); diff --git a/src/runtime/cgo/sigaction.go b/src/runtime/cgo/sigaction.go index dc714f7ef4a3b0..90034bad32ffb2 100644 --- a/src/runtime/cgo/sigaction.go +++ b/src/runtime/cgo/sigaction.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (linux && amd64) || (freebsd && amd64) || (linux && arm64) || (linux && ppc64le) +//go:build (linux && (386 || amd64 || arm64 || loong64 || ppc64le)) || (freebsd && amd64) package cgo diff --git a/src/runtime/cgo/windows.go b/src/runtime/cgo/windows.go new file mode 100644 index 00000000000000..7ba61753dffda2 --- /dev/null +++ b/src/runtime/cgo/windows.go @@ -0,0 +1,22 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build windows + +package cgo + +import _ "unsafe" // for go:linkname + +// _cgo_stub_export is only used to ensure there's at least one symbol +// in the .def file passed to the external linker. +// If there are no exported symbols, the unfortunate behavior of +// the binutils linker is to also strip the relocations table, +// resulting in non-PIE binary. The other option is the +// --export-all-symbols flag, but we don't need to export all symbols +// and this may overflow the export table (#40795). +// See https://sourceware.org/bugzilla/show_bug.cgi?id=19011 +// +//go:cgo_export_static _cgo_stub_export +//go:linkname _cgo_stub_export _cgo_stub_export +var _cgo_stub_export uintptr diff --git a/src/runtime/cgo_sigaction.go b/src/runtime/cgo_sigaction.go index 9500c522059580..f725dbef4d5bd4 100644 --- a/src/runtime/cgo_sigaction.go +++ b/src/runtime/cgo_sigaction.go @@ -3,8 +3,10 @@ // license that can be found in the LICENSE file. // Support for sanitizers. See runtime/cgo/sigaction.go. +// Also used on linux/386 to clear the SA_RESTORER flag +// when using cgo; see issue #75253. -//go:build (linux && amd64) || (freebsd && amd64) || (linux && arm64) || (linux && ppc64le) +//go:build (linux && (386 || amd64 || arm64 || loong64 || ppc64le)) || (freebsd && amd64) package runtime @@ -42,6 +44,8 @@ func sigaction(sig uint32, new, old *sigactiont) { var ret int32 + fixSigactionForCgo(new) + var g *g if mainStarted { g = getg() diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go index ae91627972d2ea..9b9a47b87eab75 100644 --- a/src/runtime/cgocall.go +++ b/src/runtime/cgocall.go @@ -33,7 +33,7 @@ // // To make it possible for gcc-compiled C code to call a Go function p.GoF, // cgo writes a gcc-compiled function named GoF (not p.GoF, since gcc doesn't -// know about packages). The gcc-compiled C function f calls GoF. +// know about packages). The gcc-compiled C function f calls GoF. // // GoF initializes "frame", a structure containing all of its // arguments and slots for p.GoF's results. It calls @@ -58,10 +58,10 @@ // m.g0 stack, so that it can be restored later. // // runtime.cgocallbackg (below) is now running on a real goroutine -// stack (not an m.g0 stack). First it calls runtime.exitsyscall, which will +// stack (not an m.g0 stack). First it calls runtime.exitsyscall, which will // block until the $GOMAXPROCS limit allows running this goroutine. // Once exitsyscall has returned, it is safe to do things like call the memory -// allocator or invoke the Go callback function. runtime.cgocallbackg +// allocator or invoke the Go callback function. runtime.cgocallbackg // first defers a function to unwind m.g0.sched.sp, so that if p.GoF // panics, m.g0.sched.sp will be restored to its old value: the m.g0 stack // and the m.curg stack will be unwound in lock step. @@ -191,8 +191,8 @@ func cgocall(fn, arg unsafe.Pointer) int32 { osPreemptExtExit(mp) - // Save current syscall parameters, so m.winsyscall can be - // used again if callback decide to make syscall. + // After exitsyscall we can be rescheduled on a different M, + // so we need to restore the original M's winsyscall. winsyscall := mp.winsyscall exitsyscall() @@ -231,34 +231,6 @@ func cgocall(fn, arg unsafe.Pointer) int32 { func callbackUpdateSystemStack(mp *m, sp uintptr, signal bool) { g0 := mp.g0 - inBound := sp > g0.stack.lo && sp <= g0.stack.hi - if mp.ncgo > 0 && !inBound { - // ncgo > 0 indicates that this M was in Go further up the stack - // (it called C and is now receiving a callback). - // - // !inBound indicates that we were called with SP outside the - // expected system stack bounds (C changed the stack out from - // under us between the cgocall and cgocallback?). - // - // It is not safe for the C call to change the stack out from - // under us, so throw. - - // Note that this case isn't possible for signal == true, as - // that is always passing a new M from needm. - - // Stack is bogus, but reset the bounds anyway so we can print. - hi := g0.stack.hi - lo := g0.stack.lo - g0.stack.hi = sp + 1024 - g0.stack.lo = sp - 32*1024 - g0.stackguard0 = g0.stack.lo + stackGuard - g0.stackguard1 = g0.stackguard0 - - print("M ", mp.id, " procid ", mp.procid, " runtime: cgocallback with sp=", hex(sp), " out of bounds [", hex(lo), ", ", hex(hi), "]") - print("\n") - exit(2) - } - if !mp.isextra { // We allocated the stack for standard Ms. Don't replace the // stack bounds with estimated ones when we already initialized @@ -266,26 +238,37 @@ func callbackUpdateSystemStack(mp *m, sp uintptr, signal bool) { return } - // This M does not have Go further up the stack. However, it may have - // previously called into Go, initializing the stack bounds. Between - // that call returning and now the stack may have changed (perhaps the - // C thread is running a coroutine library). We need to update the - // stack bounds for this case. + inBound := sp > g0.stack.lo && sp <= g0.stack.hi + if inBound && mp.g0StackAccurate { + // This M has called into Go before and has the stack bounds + // initialized. We have the accurate stack bounds, and the SP + // is in bounds. We expect it continues to run within the same + // bounds. + return + } + + // We don't have an accurate stack bounds (either it never calls + // into Go before, or we couldn't get the accurate bounds), or the + // current SP is not within the previous bounds (the stack may have + // changed between calls). We need to update the stack bounds. // // N.B. we need to update the stack bounds even if SP appears to - // already be in bounds. Our "bounds" may actually be estimated dummy - // bounds (below). The actual stack bounds could have shifted but still - // have partial overlap with our dummy bounds. If we failed to update - // in that case, we could find ourselves seemingly called near the - // bottom of the stack bounds, where we quickly run out of space. + // already be in bounds, if our bounds are estimated dummy bounds + // (below). We may be in a different region within the same actual + // stack bounds, but our estimates were not accurate. Or the actual + // stack bounds could have shifted but still have partial overlap with + // our dummy bounds. If we failed to update in that case, we could find + // ourselves seemingly called near the bottom of the stack bounds, where + // we quickly run out of space. // Set the stack bounds to match the current stack. If we don't // actually know how big the stack is, like we don't know how big any // scheduling stack is, but we assume there's at least 32 kB. If we // can get a more accurate stack bound from pthread, use that, provided - // it actually contains SP.. + // it actually contains SP. g0.stack.hi = sp + 1024 g0.stack.lo = sp - 32*1024 + mp.g0StackAccurate = false if !signal && _cgo_getstackbound != nil { // Don't adjust if called from the signal handler. // We are on the signal stack, not the pthread stack. @@ -296,12 +279,16 @@ func callbackUpdateSystemStack(mp *m, sp uintptr, signal bool) { asmcgocall(_cgo_getstackbound, unsafe.Pointer(&bounds)) // getstackbound is an unsupported no-op on Windows. // + // On Unix systems, if the API to get accurate stack bounds is + // not available, it returns zeros. + // // Don't use these bounds if they don't contain SP. Perhaps we // were called by something not using the standard thread // stack. if bounds[0] != 0 && sp > bounds[0] && sp <= bounds[1] { g0.stack.lo = bounds[0] g0.stack.hi = bounds[1] + mp.g0StackAccurate = true } } g0.stackguard0 = g0.stack.lo + stackGuard @@ -319,6 +306,8 @@ func cgocallbackg(fn, frame unsafe.Pointer, ctxt uintptr) { } sp := gp.m.g0.sched.sp // system sp saved by cgocallback. + oldStack := gp.m.g0.stack + oldAccurate := gp.m.g0StackAccurate callbackUpdateSystemStack(gp.m, sp, false) // The call from C is on gp.m's g0 stack, so we must ensure @@ -338,9 +327,14 @@ func cgocallbackg(fn, frame unsafe.Pointer, ctxt uintptr) { // stack. However, since we're returning to an earlier stack frame and // need to pair with the entersyscall() call made by cgocall, we must // save syscall* and let reentersyscall restore them. + // + // Note: savedsp and savedbp MUST be held in locals as an unsafe.Pointer. + // When we call into Go, the stack is free to be moved. If these locals + // aren't visible in the stack maps, they won't get updated properly, + // and will end up being stale when restored by reentersyscall. savedsp := unsafe.Pointer(gp.syscallsp) savedpc := gp.syscallpc - savedbp := gp.syscallbp + savedbp := unsafe.Pointer(gp.syscallbp) exitsyscall() // coming out of cgo call gp.m.incgo = false if gp.m.isextra { @@ -361,7 +355,9 @@ func cgocallbackg(fn, frame unsafe.Pointer, ctxt uintptr) { gp.m.incgo = true unlockOSThread() - if gp.m.isextra { + if gp.m.isextra && gp.m.ncgo == 0 { + // There are no active cgocalls above this frame (ncgo == 0), + // thus there can't be more Go frames above this frame. gp.m.isExtraInC = true } @@ -372,9 +368,15 @@ func cgocallbackg(fn, frame unsafe.Pointer, ctxt uintptr) { osPreemptExtEnter(gp.m) // going back to cgo call - reentersyscall(savedpc, uintptr(savedsp), savedbp) + reentersyscall(savedpc, uintptr(savedsp), uintptr(savedbp)) gp.m.winsyscall = winsyscall + + // Restore the old g0 stack bounds + gp.m.g0.stack = oldStack + gp.m.g0.stackguard0 = oldStack.lo + stackGuard + gp.m.g0.stackguard1 = gp.m.g0.stackguard0 + gp.m.g0StackAccurate = oldAccurate } func cgocallbackg1(fn, frame unsafe.Pointer, ctxt uintptr) { @@ -391,7 +393,7 @@ func cgocallbackg1(fn, frame unsafe.Pointer, ctxt uintptr) { // Now we need to set gp.cgoCtxt = s, but we could get // a SIGPROF signal while manipulating the slice, and // the SIGPROF handler could pick up gp.cgoCtxt while - // tracing up the stack. We need to ensure that the + // tracing up the stack. We need to ensure that the // handler always sees a valid slice, so set the // values in an order such that it always does. p := (*slice)(unsafe.Pointer(&gp.cgoCtxt)) @@ -425,6 +427,13 @@ func cgocallbackg1(fn, frame unsafe.Pointer, ctxt uintptr) { restore := true defer unwindm(&restore) + var ditAlreadySet bool + if debug.dataindependenttiming == 1 && gp.m.isextra { + // We only need to enable DIT for threads that were created by C, as it + // should already by enabled on threads that were created by Go. + ditAlreadySet = sys.EnableDIT() + } + if raceenabled { raceacquire(unsafe.Pointer(&racecgosync)) } @@ -440,6 +449,11 @@ func cgocallbackg1(fn, frame unsafe.Pointer, ctxt uintptr) { racereleasemerge(unsafe.Pointer(&racecgosync)) } + if debug.dataindependenttiming == 1 && !ditAlreadySet { + // Only unset DIT if it wasn't already enabled when cgocallback was called. + sys.DisableDIT() + } + // Do not unwind m->g0->sched.sp. // Our caller, cgocallback, will do that. restore = false @@ -529,18 +543,18 @@ func cgoCheckPointer(ptr any, arg any) { t := ep._type top := true - if arg != nil && (t.Kind_&abi.KindMask == abi.Pointer || t.Kind_&abi.KindMask == abi.UnsafePointer) { + if arg != nil && (t.Kind() == abi.Pointer || t.Kind() == abi.UnsafePointer) { p := ep.data - if t.Kind_&abi.KindDirectIface == 0 { + if !t.IsDirectIface() { p = *(*unsafe.Pointer)(p) } if p == nil || !cgoIsGoPointer(p) { return } aep := efaceOf(&arg) - switch aep._type.Kind_ & abi.KindMask { + switch aep._type.Kind() { case abi.Bool: - if t.Kind_&abi.KindMask == abi.UnsafePointer { + if t.Kind() == abi.UnsafePointer { // We don't know the type of the element. break } @@ -558,20 +572,31 @@ func cgoCheckPointer(ptr any, arg any) { ep = aep t = ep._type top = false + case abi.Pointer: + // The Go code is indexing into a pointer to an array, + // and we have been passed the pointer-to-array. + // Check the array rather than the pointer. + pt := (*abi.PtrType)(unsafe.Pointer(aep._type)) + t = pt.Elem + if t.Kind() != abi.Array { + throw("can't happen") + } + ep = aep + top = false default: throw("can't happen") } } - cgoCheckArg(t, ep.data, t.Kind_&abi.KindDirectIface == 0, top, cgoCheckPointerFail) + cgoCheckArg(t, ep.data, !t.IsDirectIface(), top, cgoCheckPointerFail) } const cgoCheckPointerFail = "cgo argument has Go pointer to unpinned Go pointer" const cgoResultFail = "cgo result is unpinned Go pointer or points to unpinned Go pointer" -// cgoCheckArg is the real work of cgoCheckPointer. The argument p -// is either a pointer to the value (of type t), or the value itself, -// depending on indir. The top parameter is whether we are at the top +// cgoCheckArg is the real work of cgoCheckPointer and cgoCheckResult. +// The argument p is either a pointer to the value (of type t), or the value +// itself, depending on indir. The top parameter is whether we are at the top // level, where Go pointers are allowed. Go pointers to pinned objects are // allowed as long as they don't reference other unpinned pointers. func cgoCheckArg(t *_type, p unsafe.Pointer, indir, top bool, msg string) { @@ -580,7 +605,7 @@ func cgoCheckArg(t *_type, p unsafe.Pointer, indir, top bool, msg string) { return } - switch t.Kind_ & abi.KindMask { + switch t.Kind() { default: throw("can't happen") case abi.Array: @@ -589,7 +614,7 @@ func cgoCheckArg(t *_type, p unsafe.Pointer, indir, top bool, msg string) { if at.Len != 1 { throw("can't happen") } - cgoCheckArg(at.Elem, p, at.Elem.Kind_&abi.KindDirectIface == 0, top, msg) + cgoCheckArg(at.Elem, p, !at.Elem.IsDirectIface(), top, msg) return } for i := uintptr(0); i < at.Len; i++ { @@ -627,7 +652,7 @@ func cgoCheckArg(t *_type, p unsafe.Pointer, indir, top bool, msg string) { if !top && !isPinned(p) { panic(errorString(msg)) } - cgoCheckArg(it, p, it.Kind_&abi.KindDirectIface == 0, false, msg) + cgoCheckArg(it, p, !it.IsDirectIface(), false, msg) case abi.Slice: st := (*slicetype)(unsafe.Pointer(t)) s := (*slice)(p) @@ -659,7 +684,7 @@ func cgoCheckArg(t *_type, p unsafe.Pointer, indir, top bool, msg string) { if len(st.Fields) != 1 { throw("can't happen") } - cgoCheckArg(st.Fields[0].Typ, p, st.Fields[0].Typ.Kind_&abi.KindDirectIface == 0, top, msg) + cgoCheckArg(st.Fields[0].Typ, p, !st.Fields[0].Typ.IsDirectIface(), top, msg) return } for _, f := range st.Fields { @@ -767,5 +792,5 @@ func cgoCheckResult(val any) { ep := efaceOf(&val) t := ep._type - cgoCheckArg(t, ep.data, t.Kind_&abi.KindDirectIface == 0, false, cgoResultFail) + cgoCheckArg(t, ep.data, !t.IsDirectIface(), false, cgoResultFail) } diff --git a/src/runtime/cgocheck.go b/src/runtime/cgocheck.go index 3f2c271953db66..ab804a5a36ed95 100644 --- a/src/runtime/cgocheck.go +++ b/src/runtime/cgocheck.go @@ -8,7 +8,6 @@ package runtime import ( - "internal/abi" "internal/goarch" "unsafe" ) @@ -142,52 +141,7 @@ func cgoCheckTypedBlock(typ *_type, src unsafe.Pointer, off, size uintptr) { size = ptrdataSize } - if typ.Kind_&abi.KindGCProg == 0 { - cgoCheckBits(src, typ.GCData, off, size) - return - } - - // The type has a GC program. Try to find GC bits somewhere else. - for _, datap := range activeModules() { - if cgoInRange(src, datap.data, datap.edata) { - doff := uintptr(src) - datap.data - cgoCheckBits(add(src, -doff), datap.gcdatamask.bytedata, off+doff, size) - return - } - if cgoInRange(src, datap.bss, datap.ebss) { - boff := uintptr(src) - datap.bss - cgoCheckBits(add(src, -boff), datap.gcbssmask.bytedata, off+boff, size) - return - } - } - - s := spanOfUnchecked(uintptr(src)) - if s.state.get() == mSpanManual { - // There are no heap bits for value stored on the stack. - // For a channel receive src might be on the stack of some - // other goroutine, so we can't unwind the stack even if - // we wanted to. - // We can't expand the GC program without extra storage - // space we can't easily get. - // Fortunately we have the type information. - systemstack(func() { - cgoCheckUsingType(typ, src, off, size) - }) - return - } - - // src must be in the regular heap. - tp := s.typePointersOf(uintptr(src), size) - for { - var addr uintptr - if tp, addr = tp.next(uintptr(src) + size); addr == 0 { - break - } - v := *(*unsafe.Pointer)(unsafe.Pointer(addr)) - if cgoIsGoPointer(v) && !isPinned(v) { - throw(cgoWriteBarrierFail) - } - } + cgoCheckBits(src, getGCMask(typ), off, size) } // cgoCheckBits checks the block of memory at src, for up to size @@ -245,48 +199,5 @@ func cgoCheckUsingType(typ *_type, src unsafe.Pointer, off, size uintptr) { size = ptrdataSize } - if typ.Kind_&abi.KindGCProg == 0 { - cgoCheckBits(src, typ.GCData, off, size) - return - } - switch typ.Kind_ & abi.KindMask { - default: - throw("can't happen") - case abi.Array: - at := (*arraytype)(unsafe.Pointer(typ)) - for i := uintptr(0); i < at.Len; i++ { - if off < at.Elem.Size_ { - cgoCheckUsingType(at.Elem, src, off, size) - } - src = add(src, at.Elem.Size_) - skipped := off - if skipped > at.Elem.Size_ { - skipped = at.Elem.Size_ - } - checked := at.Elem.Size_ - skipped - off -= skipped - if size <= checked { - return - } - size -= checked - } - case abi.Struct: - st := (*structtype)(unsafe.Pointer(typ)) - for _, f := range st.Fields { - if off < f.Typ.Size_ { - cgoCheckUsingType(f.Typ, src, off, size) - } - src = add(src, f.Typ.Size_) - skipped := off - if skipped > f.Typ.Size_ { - skipped = f.Typ.Size_ - } - checked := f.Typ.Size_ - skipped - off -= skipped - if size <= checked { - return - } - size -= checked - } - } + cgoCheckBits(src, getGCMask(typ), off, size) } diff --git a/src/runtime/cgroup_linux.go b/src/runtime/cgroup_linux.go new file mode 100644 index 00000000000000..73e7363eb44632 --- /dev/null +++ b/src/runtime/cgroup_linux.go @@ -0,0 +1,119 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "internal/runtime/cgroup" +) + +// cgroup-aware GOMAXPROCS default +// +// At startup (defaultGOMAXPROCSInit), we read /proc/self/cgroup and /proc/self/mountinfo +// to find our current CPU cgroup and open its limit file(s), which remain open +// for the entire process lifetime. We periodically read the current limit by +// rereading the limit file(s) from the beginning. +// +// This makes reading updated limits simple, but has a few downsides: +// +// 1. We only read the limit from the leaf cgroup that actually contains this +// process. But a parent cgroup may have a tighter limit. That tighter limit +// would be our effective limit. That said, container runtimes tend to hide +// parent cgroups from the container anyway. +// +// 2. If the process is migrated to another cgroup while it is running it will +// not notice, as we only check which cgroup we are in once at startup. +var ( + // We can't allocate during early initialization when we need to find + // the cgroup. Simply use a fixed global as a scratch parsing buffer. + cgroupScratch [cgroup.ScratchSize]byte + + cgroupOK bool + cgroupCPU cgroup.CPU + + // defaultGOMAXPROCSInit runs before internal/godebug init, so we can't + // directly update the GODEBUG counter. Store the result until after + // init runs. + containermaxprocsNonDefault bool + containermaxprocs = &godebugInc{name: "containermaxprocs"} +) + +// Prepare for defaultGOMAXPROCS. +// +// Must run after parsedebugvars. +func defaultGOMAXPROCSInit() { + c, err := cgroup.OpenCPU(cgroupScratch[:]) + if err != nil { + // Likely cgroup.ErrNoCgroup. + return + } + + if debug.containermaxprocs > 0 { + // Normal operation. + cgroupCPU = c + cgroupOK = true + return + } + + // cgroup-aware GOMAXPROCS is disabled. We still check the cgroup once + // at startup to see if enabling the GODEBUG would result in a + // different default GOMAXPROCS. If so, we increment runtime/metrics + // /godebug/non-default-behavior/cgroupgomaxprocs:events. + procs := getCPUCount() + cgroupProcs := adjustCgroupGOMAXPROCS(procs, c) + if procs != cgroupProcs { + containermaxprocsNonDefault = true + } + + // Don't need the cgroup for remaining execution. + c.Close() +} + +// defaultGOMAXPROCSUpdateGODEBUG updates the internal/godebug counter for +// container GOMAXPROCS, once internal/godebug is initialized. +func defaultGOMAXPROCSUpdateGODEBUG() { + if containermaxprocsNonDefault { + containermaxprocs.IncNonDefault() + } +} + +// Return the default value for GOMAXPROCS when it has not been set explicitly. +// +// ncpu is the optional precomputed value of getCPUCount. If passed as 0, +// defaultGOMAXPROCS will call getCPUCount. +func defaultGOMAXPROCS(ncpu int32) int32 { + // GOMAXPROCS is the minimum of: + // + // 1. Total number of logical CPUs available from sched_getaffinity. + // + // 2. The average CPU cgroup throughput limit (average throughput = + // quota/period). A limit less than 2 is rounded up to 2, and any + // fractional component is rounded up. + // + // TODO: add rationale. + + procs := ncpu + if procs <= 0 { + procs = getCPUCount() + } + if !cgroupOK { + // No cgroup, or disabled by debug.containermaxprocs. + return procs + } + + return adjustCgroupGOMAXPROCS(procs, cgroupCPU) +} + +// Lower procs as necessary for the current cgroup CPU limit. +func adjustCgroupGOMAXPROCS(procs int32, cpu cgroup.CPU) int32 { + limit, ok, err := cgroup.ReadCPULimit(cpu) + if err == nil && ok { + limit = ceil(limit) + limit = max(limit, 2) + if int32(limit) < procs { + procs = int32(limit) + } + } + return procs +} diff --git a/src/runtime/cgroup_linux_test.go b/src/runtime/cgroup_linux_test.go new file mode 100644 index 00000000000000..0b060572b6912b --- /dev/null +++ b/src/runtime/cgroup_linux_test.go @@ -0,0 +1,325 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "fmt" + "internal/cgrouptest" + "runtime" + "strings" + "syscall" + "testing" + "unsafe" +) + +func mustHaveFourCPUs(t *testing.T) { + // If NumCPU is lower than the cgroup limit, GOMAXPROCS will use + // NumCPU. + // + // cgroup GOMAXPROCS also have a minimum of 2. We need some room above + // that to test interesting properies. + if runtime.NumCPU() < 4 { + t.Helper() + t.Skip("skipping test: fewer than 4 CPUs") + } +} + +func TestCgroupGOMAXPROCS(t *testing.T) { + mustHaveFourCPUs(t) + + exe, err := buildTestProg(t, "testprog") + if err != nil { + t.Fatal(err) + } + + tests := []struct { + godebug int + want int + }{ + // With containermaxprocs=1, GOMAXPROCS should use the cgroup + // limit. + { + godebug: 1, + want: 3, + }, + // With containermaxprocs=0, it should be ignored. + { + godebug: 0, + want: runtime.NumCPU(), + }, + } + for _, tc := range tests { + t.Run(fmt.Sprintf("containermaxprocs=%d", tc.godebug), func(t *testing.T) { + cgrouptest.InCgroupV2(t, func(c *cgrouptest.CgroupV2) { + if err := c.SetCPUMax(300000, 100000); err != nil { + t.Fatalf("unable to set CPU limit: %v", err) + } + + got := runBuiltTestProg(t, exe, "PrintGOMAXPROCS", fmt.Sprintf("GODEBUG=containermaxprocs=%d", tc.godebug)) + want := fmt.Sprintf("%d\n", tc.want) + if got != want { + t.Fatalf("output got %q want %q", got, want) + } + }) + }) + } +} + +// Without a cgroup limit, GOMAXPROCS uses NumCPU. +func TestCgroupGOMAXPROCSNoLimit(t *testing.T) { + exe, err := buildTestProg(t, "testprog") + if err != nil { + t.Fatal(err) + } + + cgrouptest.InCgroupV2(t, func(c *cgrouptest.CgroupV2) { + if err := c.SetCPUMax(-1, 100000); err != nil { + t.Fatalf("unable to set CPU limit: %v", err) + } + + got := runBuiltTestProg(t, exe, "PrintGOMAXPROCS") + want := fmt.Sprintf("%d\n", runtime.NumCPU()) + if got != want { + t.Fatalf("output got %q want %q", got, want) + } + }) +} + +// If the cgroup limit is higher than NumCPU, GOMAXPROCS uses NumCPU. +func TestCgroupGOMAXPROCSHigherThanNumCPU(t *testing.T) { + exe, err := buildTestProg(t, "testprog") + if err != nil { + t.Fatal(err) + } + + cgrouptest.InCgroupV2(t, func(c *cgrouptest.CgroupV2) { + if err := c.SetCPUMax(2*int64(runtime.NumCPU())*100000, 100000); err != nil { + t.Fatalf("unable to set CPU limit: %v", err) + } + + got := runBuiltTestProg(t, exe, "PrintGOMAXPROCS") + want := fmt.Sprintf("%d\n", runtime.NumCPU()) + if got != want { + t.Fatalf("output got %q want %q", got, want) + } + }) +} + +func TestCgroupGOMAXPROCSRound(t *testing.T) { + mustHaveFourCPUs(t) + + exe, err := buildTestProg(t, "testprog") + if err != nil { + t.Fatal(err) + } + + tests := []struct { + quota int64 + want int + }{ + // We always round the fractional component up. + { + quota: 200001, + want: 3, + }, + { + quota: 250000, + want: 3, + }, + { + quota: 299999, + want: 3, + }, + // Anything less than two rounds up to a minimum of 2. + { + quota: 50000, // 0.5 + want: 2, + }, + { + quota: 100000, + want: 2, + }, + { + quota: 150000, + want: 2, + }, + } + for _, tc := range tests { + t.Run(fmt.Sprintf("%d", tc.quota), func(t *testing.T) { + cgrouptest.InCgroupV2(t, func(c *cgrouptest.CgroupV2) { + if err := c.SetCPUMax(tc.quota, 100000); err != nil { + t.Fatalf("unable to set CPU limit: %v", err) + } + + got := runBuiltTestProg(t, exe, "PrintGOMAXPROCS") + want := fmt.Sprintf("%d\n", tc.want) + if got != want { + t.Fatalf("output got %q want %q", got, want) + } + }) + }) + } +} + +// Environment variable takes precedence over defaults. +func TestCgroupGOMAXPROCSEnvironment(t *testing.T) { + mustHaveFourCPUs(t) + + exe, err := buildTestProg(t, "testprog") + if err != nil { + t.Fatal(err) + } + + cgrouptest.InCgroupV2(t, func(c *cgrouptest.CgroupV2) { + if err := c.SetCPUMax(200000, 100000); err != nil { + t.Fatalf("unable to set CPU limit: %v", err) + } + + got := runBuiltTestProg(t, exe, "PrintGOMAXPROCS", "GOMAXPROCS=3") + want := "3\n" + if got != want { + t.Fatalf("output got %q want %q", got, want) + } + }) +} + +// CPU affinity takes priority if lower than cgroup limit. +func TestCgroupGOMAXPROCSSchedAffinity(t *testing.T) { + exe, err := buildTestProg(t, "testprog") + if err != nil { + t.Fatal(err) + } + + cgrouptest.InCgroupV2(t, func(c *cgrouptest.CgroupV2) { + if err := c.SetCPUMax(300000, 100000); err != nil { + t.Fatalf("unable to set CPU limit: %v", err) + } + + // CPU affinity is actually a per-thread attribute. + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + const maxCPUs = 64 * 1024 + var orig [maxCPUs / 8]byte + _, _, errno := syscall.Syscall6(syscall.SYS_SCHED_GETAFFINITY, 0, unsafe.Sizeof(orig), uintptr(unsafe.Pointer(&orig[0])), 0, 0, 0) + if errno != 0 { + t.Fatalf("unable to get CPU affinity: %v", errno) + } + + // We're going to restrict to CPUs 0 and 1. Make sure those are already available. + if orig[0]&0b11 != 0b11 { + t.Skipf("skipping test: CPUs 0 and 1 not available") + } + + var mask [maxCPUs / 8]byte + mask[0] = 0b11 + _, _, errno = syscall.Syscall6(syscall.SYS_SCHED_SETAFFINITY, 0, unsafe.Sizeof(mask), uintptr(unsafe.Pointer(&mask[0])), 0, 0, 0) + if errno != 0 { + t.Fatalf("unable to set CPU affinity: %v", errno) + } + defer func() { + _, _, errno = syscall.Syscall6(syscall.SYS_SCHED_SETAFFINITY, 0, unsafe.Sizeof(orig), uintptr(unsafe.Pointer(&orig[0])), 0, 0, 0) + if errno != 0 { + t.Fatalf("unable to restore CPU affinity: %v", errno) + } + }() + + got := runBuiltTestProg(t, exe, "PrintGOMAXPROCS") + want := "2\n" + if got != want { + t.Fatalf("output got %q want %q", got, want) + } + }) +} + +func TestCgroupGOMAXPROCSSetDefault(t *testing.T) { + mustHaveFourCPUs(t) + + exe, err := buildTestProg(t, "testprog") + if err != nil { + t.Fatal(err) + } + + tests := []struct { + godebug int + want int + }{ + // With containermaxprocs=1, SetDefaultGOMAXPROCS should observe + // the cgroup limit. + { + godebug: 1, + want: 3, + }, + // With containermaxprocs=0, it should be ignored. + { + godebug: 0, + want: runtime.NumCPU(), + }, + } + for _, tc := range tests { + t.Run(fmt.Sprintf("containermaxprocs=%d", tc.godebug), func(t *testing.T) { + cgrouptest.InCgroupV2(t, func(c *cgrouptest.CgroupV2) { + env := []string{ + fmt.Sprintf("GO_TEST_CPU_MAX_PATH=%s", c.CPUMaxPath()), + "GO_TEST_CPU_MAX_QUOTA=300000", + fmt.Sprintf("GODEBUG=containermaxprocs=%d", tc.godebug), + } + got := runBuiltTestProg(t, exe, "SetLimitThenDefaultGOMAXPROCS", env...) + want := fmt.Sprintf("%d\n", tc.want) + if got != want { + t.Fatalf("output got %q want %q", got, want) + } + }) + }) + } +} + +func TestCgroupGOMAXPROCSUpdate(t *testing.T) { + mustHaveFourCPUs(t) + + if testing.Short() { + t.Skip("skipping test: long sleeps") + } + + exe, err := buildTestProg(t, "testprog") + if err != nil { + t.Fatal(err) + } + + cgrouptest.InCgroupV2(t, func(c *cgrouptest.CgroupV2) { + got := runBuiltTestProg(t, exe, "UpdateGOMAXPROCS", fmt.Sprintf("GO_TEST_CPU_MAX_PATH=%s", c.CPUMaxPath())) + if !strings.Contains(got, "OK") { + t.Fatalf("output got %q want OK", got) + } + }) +} + +func TestCgroupGOMAXPROCSDontUpdate(t *testing.T) { + mustHaveFourCPUs(t) + + if testing.Short() { + t.Skip("skipping test: long sleeps") + } + + exe, err := buildTestProg(t, "testprog") + if err != nil { + t.Fatal(err) + } + + // Two ways to disable updates: explicit GOMAXPROCS or GODEBUG for + // update feature. + for _, v := range []string{"GOMAXPROCS=4", "GODEBUG=updatemaxprocs=0"} { + t.Run(v, func(t *testing.T) { + cgrouptest.InCgroupV2(t, func(c *cgrouptest.CgroupV2) { + got := runBuiltTestProg(t, exe, "DontUpdateGOMAXPROCS", + fmt.Sprintf("GO_TEST_CPU_MAX_PATH=%s", c.CPUMaxPath()), + v) + if !strings.Contains(got, "OK") { + t.Fatalf("output got %q want OK", got) + } + }) + }) + } +} diff --git a/src/runtime/cgroup_stubs.go b/src/runtime/cgroup_stubs.go new file mode 100644 index 00000000000000..1f37b1783bd0fd --- /dev/null +++ b/src/runtime/cgroup_stubs.go @@ -0,0 +1,24 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !linux + +package runtime + +func defaultGOMAXPROCSInit() {} +func defaultGOMAXPROCSUpdateGODEBUG() {} + +func defaultGOMAXPROCS(ncpu int32) int32 { + // Use the total number of logical CPUs available now, as CPU affinity + // may change after start. + // + // TODO(prattmic): On some GOOS getCPUCount can never change. Don't + // bother calling over and over. + + procs := ncpu + if procs <= 0 { + procs = getCPUCount() + } + return procs +} diff --git a/src/runtime/chan.go b/src/runtime/chan.go index 4c854c72bc9842..3320c248b458b6 100644 --- a/src/runtime/chan.go +++ b/src/runtime/chan.go @@ -21,6 +21,7 @@ import ( "internal/abi" "internal/runtime/atomic" "internal/runtime/math" + "internal/runtime/sys" "unsafe" ) @@ -42,6 +43,7 @@ type hchan struct { recvx uint // receive index recvq waitq // list of recv waiters sendq waitq // list of send waiters + bubble *synctestBubble // lock protects all fields in hchan, as well as several // fields in sudogs blocked on this channel. @@ -111,6 +113,9 @@ func makechan(t *chantype, size int) *hchan { c.elemsize = uint16(elem.Size_) c.elemtype = elem c.dataqsiz = uint(size) + if b := getg().bubble; b != nil { + c.bubble = b + } lockInit(&c.lock, lockRankHchan) if debugChan { @@ -153,7 +158,7 @@ func full(c *hchan) bool { // //go:nosplit func chansend1(c *hchan, elem unsafe.Pointer) { - chansend(c, elem, true, getcallerpc()) + chansend(c, elem, true, sys.GetCallerPC()) } /* @@ -185,6 +190,10 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { racereadpc(c.raceaddr(), callerpc, abi.FuncPCABIInternal(chansend)) } + if c.bubble != nil && getg().bubble != c.bubble { + fatal("send on synctest channel from outside bubble") + } + // Fast path: check for failed non-blocking operation without acquiring the lock. // // After observing that the channel is not closed, we observe that the channel is @@ -254,11 +263,11 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { } // No stack splits between assigning elem and enqueuing mysg // on gp.waiting where copystack can find it. - mysg.elem = ep + mysg.elem.set(ep) mysg.waitlink = nil mysg.g = gp mysg.isSelect = false - mysg.c = c + mysg.c.set(c) gp.waiting = mysg gp.param = nil c.sendq.enqueue(mysg) @@ -267,7 +276,11 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { // changes and when we set gp.activeStackChans is not safe for // stack shrinking. gp.parkingOnChan.Store(true) - gopark(chanparkcommit, unsafe.Pointer(&c.lock), waitReasonChanSend, traceBlockChanSend, 2) + reason := waitReasonChanSend + if c.bubble != nil { + reason = waitReasonSynctestChanSend + } + gopark(chanparkcommit, unsafe.Pointer(&c.lock), reason, traceBlockChanSend, 2) // Ensure the value being sent is kept alive until the // receiver copies it out. The sudog has a pointer to the // stack object, but sudogs aren't considered as roots of the @@ -285,7 +298,7 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { if mysg.releasetime > 0 { blockevent(mysg.releasetime-t0, 2) } - mysg.c = nil + mysg.c.set(nil) releaseSudog(mysg) if closed { if c.closed == 0 { @@ -303,6 +316,10 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { // sg must already be dequeued from c. // ep must be non-nil and point to the heap or the caller's stack. func send(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func(), skip int) { + if c.bubble != nil && getg().bubble != c.bubble { + unlockf() + fatal("send on synctest channel from outside bubble") + } if raceenabled { if c.dataqsiz == 0 { racesync(c, sg) @@ -319,9 +336,9 @@ func send(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func(), skip int) { c.sendx = c.recvx // c.sendx = (c.sendx+1) % c.dataqsiz } } - if sg.elem != nil { + if sg.elem.get() != nil { sendDirect(c.elemtype, sg, ep) - sg.elem = nil + sg.elem.set(nil) } gp := sg.g unlockf() @@ -378,7 +395,7 @@ func sendDirect(t *_type, sg *sudog, src unsafe.Pointer) { // Once we read sg.elem out of sg, it will no longer // be updated if the destination's stack gets copied (shrunk). // So make sure that no preemption points can happen between read & use. - dst := sg.elem + dst := sg.elem.get() typeBitsBulkBarrier(t, uintptr(dst), uintptr(src), t.Size_) // No need for cgo write barrier checks because dst is always // Go memory. @@ -389,7 +406,7 @@ func recvDirect(t *_type, sg *sudog, dst unsafe.Pointer) { // dst is on our stack or the heap, src is on another stack. // The channel is locked, so src will not move during this // operation. - src := sg.elem + src := sg.elem.get() typeBitsBulkBarrier(t, uintptr(dst), uintptr(src), t.Size_) memmove(dst, src, t.Size_) } @@ -398,6 +415,9 @@ func closechan(c *hchan) { if c == nil { panic(plainError("close of nil channel")) } + if c.bubble != nil && getg().bubble != c.bubble { + fatal("close of synctest channel from outside bubble") + } lock(&c.lock) if c.closed != 0 { @@ -406,7 +426,7 @@ func closechan(c *hchan) { } if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racewritepc(c.raceaddr(), callerpc, abi.FuncPCABIInternal(closechan)) racerelease(c.raceaddr()) } @@ -421,9 +441,9 @@ func closechan(c *hchan) { if sg == nil { break } - if sg.elem != nil { - typedmemclr(c.elemtype, sg.elem) - sg.elem = nil + if sg.elem.get() != nil { + typedmemclr(c.elemtype, sg.elem.get()) + sg.elem.set(nil) } if sg.releasetime != 0 { sg.releasetime = cputicks() @@ -443,7 +463,7 @@ func closechan(c *hchan) { if sg == nil { break } - sg.elem = nil + sg.elem.set(nil) if sg.releasetime != 0 { sg.releasetime = cputicks() } @@ -477,7 +497,7 @@ func empty(c *hchan) bool { // c.timer is also immutable (it is set after make(chan) but before any channel operations). // All timer channels have dataqsiz > 0. if c.timer != nil { - c.timer.maybeRunChan() + c.timer.maybeRunChan(c) } return atomic.Loaduint(&c.qcount) == 0 } @@ -517,8 +537,12 @@ func chanrecv(c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) throw("unreachable") } + if c.bubble != nil && getg().bubble != c.bubble { + fatal("receive on synctest channel from outside bubble") + } + if c.timer != nil { - c.timer.maybeRunChan() + c.timer.maybeRunChan(c) } // Fast path: check for failed non-blocking operation without acquiring the lock. @@ -618,13 +642,13 @@ func chanrecv(c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) } // No stack splits between assigning elem and enqueuing mysg // on gp.waiting where copystack can find it. - mysg.elem = ep + mysg.elem.set(ep) mysg.waitlink = nil gp.waiting = mysg mysg.g = gp mysg.isSelect = false - mysg.c = c + mysg.c.set(c) gp.param = nil c.recvq.enqueue(mysg) if c.timer != nil { @@ -636,7 +660,11 @@ func chanrecv(c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) // changes and when we set gp.activeStackChans is not safe for // stack shrinking. gp.parkingOnChan.Store(true) - gopark(chanparkcommit, unsafe.Pointer(&c.lock), waitReasonChanReceive, traceBlockChanRecv, 2) + reason := waitReasonChanReceive + if c.bubble != nil { + reason = waitReasonSynctestChanReceive + } + gopark(chanparkcommit, unsafe.Pointer(&c.lock), reason, traceBlockChanRecv, 2) // someone woke us up if mysg != gp.waiting { @@ -652,7 +680,7 @@ func chanrecv(c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) } success := mysg.success gp.param = nil - mysg.c = nil + mysg.c.set(nil) releaseSudog(mysg) return true, success } @@ -672,6 +700,10 @@ func chanrecv(c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) // sg must already be dequeued from c. // A non-nil ep must point to the heap or the caller's stack. func recv(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func(), skip int) { + if c.bubble != nil && getg().bubble != c.bubble { + unlockf() + fatal("receive on synctest channel from outside bubble") + } if c.dataqsiz == 0 { if raceenabled { racesync(c, sg) @@ -695,14 +727,14 @@ func recv(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func(), skip int) { typedmemmove(c.elemtype, ep, qp) } // copy data from sender to queue - typedmemmove(c.elemtype, qp, sg.elem) + typedmemmove(c.elemtype, qp, sg.elem.get()) c.recvx++ if c.recvx == c.dataqsiz { c.recvx = 0 } c.sendx = c.recvx // c.sendx = (c.sendx+1) % c.dataqsiz } - sg.elem = nil + sg.elem.set(nil) gp := sg.g unlockf() gp.param = unsafe.Pointer(sg) @@ -750,7 +782,7 @@ func chanparkcommit(gp *g, chanLock unsafe.Pointer) bool { // ... bar // } func selectnbsend(c *hchan, elem unsafe.Pointer) (selected bool) { - return chansend(c, elem, false, getcallerpc()) + return chansend(c, elem, false, sys.GetCallerPC()) } // compiler implements @@ -775,7 +807,7 @@ func selectnbrecv(elem unsafe.Pointer, c *hchan) (selected, received bool) { //go:linkname reflect_chansend reflect.chansend0 func reflect_chansend(c *hchan, elem unsafe.Pointer, nb bool) (selected bool) { - return chansend(c, elem, !nb, getcallerpc()) + return chansend(c, elem, !nb, sys.GetCallerPC()) } //go:linkname reflect_chanrecv reflect.chanrecv @@ -789,7 +821,7 @@ func chanlen(c *hchan) int { } async := debug.asynctimerchan.Load() != 0 if c.timer != nil && async { - c.timer.maybeRunChan() + c.timer.maybeRunChan(c) } if c.timer != nil && !async { // timer channels have a buffered implementation @@ -875,8 +907,11 @@ func (q *waitq) dequeue() *sudog { // We use a flag in the G struct to tell us when someone // else has won the race to signal this goroutine but the goroutine // hasn't removed itself from the queue yet. - if sgp.isSelect && !sgp.g.selectDone.CompareAndSwap(0, 1) { - continue + if sgp.isSelect { + if !sgp.g.selectDone.CompareAndSwap(0, 1) { + // We lost the race to wake this goroutine. + continue + } } return sgp diff --git a/src/runtime/chan_test.go b/src/runtime/chan_test.go index 526d45bb430136..5a1ca52a8c3b03 100644 --- a/src/runtime/chan_test.go +++ b/src/runtime/chan_test.go @@ -309,7 +309,6 @@ func TestSelfSelect(t *testing.T) { wg.Add(2) c := make(chan int, chanCap) for p := 0; p < 2; p++ { - p := p go func() { defer wg.Done() for i := 0; i < 1000; i++ { @@ -359,7 +358,6 @@ func TestSelectStress(t *testing.T) { var wg sync.WaitGroup wg.Add(10) for k := 0; k < 4; k++ { - k := k go func() { for i := 0; i < N; i++ { c[k] <- 0 diff --git a/src/runtime/checkptr_test.go b/src/runtime/checkptr_test.go index 811c0f03553420..d08b0524499b62 100644 --- a/src/runtime/checkptr_test.go +++ b/src/runtime/checkptr_test.go @@ -35,6 +35,7 @@ func TestCheckPtr(t *testing.T) { {"CheckPtrAlignmentNilPtr", ""}, {"CheckPtrArithmetic", "fatal error: checkptr: pointer arithmetic result points to invalid allocation\n"}, {"CheckPtrArithmetic2", "fatal error: checkptr: pointer arithmetic result points to invalid allocation\n"}, + {"CheckPtrArithmeticUnsafeAdd", "fatal error: checkptr: pointer arithmetic result points to invalid allocation\n"}, {"CheckPtrSize", "fatal error: checkptr: converted pointer straddles multiple allocations\n"}, {"CheckPtrSmall", "fatal error: checkptr: pointer arithmetic computed bad pointer value\n"}, {"CheckPtrSliceOK", ""}, @@ -44,7 +45,6 @@ func TestCheckPtr(t *testing.T) { } for _, tc := range testCases { - tc := tc t.Run(tc.cmd, func(t *testing.T) { t.Parallel() got, err := testenv.CleanCmdEnv(exec.Command(exe, tc.cmd)).CombinedOutput() @@ -87,7 +87,6 @@ func TestCheckPtr2(t *testing.T) { } for _, tc := range testCases { - tc := tc t.Run(tc.cmd, func(t *testing.T) { t.Parallel() got, err := testenv.CleanCmdEnv(exec.Command(exe, tc.cmd)).CombinedOutput() diff --git a/src/runtime/conv_wasm_test.go b/src/runtime/conv_wasm_test.go index 5054fca04dc40a..3979a7b618028b 100644 --- a/src/runtime/conv_wasm_test.go +++ b/src/runtime/conv_wasm_test.go @@ -11,6 +11,8 @@ import ( var res int64 var ures uint64 +// TODO: This test probably should be in a different place. + func TestFloatTruncation(t *testing.T) { testdata := []struct { input float64 @@ -21,36 +23,37 @@ func TestFloatTruncation(t *testing.T) { // max +- 1 { input: 0x7fffffffffffffff, - convInt64: -0x8000000000000000, + convInt64: 0x7fffffffffffffff, convUInt64: 0x8000000000000000, }, // For out-of-bounds conversion, the result is implementation-dependent. - // This test verifies the implementation of wasm architecture. + // This test verifies the implementation of wasm architecture, which is, + // saturating to the min/max value. { input: 0x8000000000000000, - convInt64: -0x8000000000000000, + convInt64: 0x7fffffffffffffff, convUInt64: 0x8000000000000000, }, { input: 0x7ffffffffffffffe, - convInt64: -0x8000000000000000, + convInt64: 0x7fffffffffffffff, convUInt64: 0x8000000000000000, }, // neg max +- 1 { input: -0x8000000000000000, convInt64: -0x8000000000000000, - convUInt64: 0x8000000000000000, + convUInt64: 0, }, { input: -0x8000000000000001, convInt64: -0x8000000000000000, - convUInt64: 0x8000000000000000, + convUInt64: 0, }, { input: -0x7fffffffffffffff, convInt64: -0x8000000000000000, - convUInt64: 0x8000000000000000, + convUInt64: 0, }, // trunc point +- 1 { @@ -60,7 +63,7 @@ func TestFloatTruncation(t *testing.T) { }, { input: 0x7ffffffffffffe00, - convInt64: -0x8000000000000000, + convInt64: 0x7fffffffffffffff, convUInt64: 0x8000000000000000, }, { @@ -72,48 +75,48 @@ func TestFloatTruncation(t *testing.T) { { input: -0x7ffffffffffffdff, convInt64: -0x7ffffffffffffc00, - convUInt64: 0x8000000000000000, + convUInt64: 0, }, { input: -0x7ffffffffffffe00, convInt64: -0x8000000000000000, - convUInt64: 0x8000000000000000, + convUInt64: 0, }, { input: -0x7ffffffffffffdfe, convInt64: -0x7ffffffffffffc00, - convUInt64: 0x8000000000000000, + convUInt64: 0, }, // umax +- 1 { input: 0xffffffffffffffff, - convInt64: -0x8000000000000000, - convUInt64: 0x8000000000000000, + convInt64: 0x7fffffffffffffff, + convUInt64: 0xffffffffffffffff, }, { input: 0x10000000000000000, - convInt64: -0x8000000000000000, - convUInt64: 0x8000000000000000, + convInt64: 0x7fffffffffffffff, + convUInt64: 0xffffffffffffffff, }, { input: 0xfffffffffffffffe, - convInt64: -0x8000000000000000, - convUInt64: 0x8000000000000000, + convInt64: 0x7fffffffffffffff, + convUInt64: 0xffffffffffffffff, }, // umax trunc +- 1 { input: 0xfffffffffffffbff, - convInt64: -0x8000000000000000, + convInt64: 0x7fffffffffffffff, convUInt64: 0xfffffffffffff800, }, { input: 0xfffffffffffffc00, - convInt64: -0x8000000000000000, - convUInt64: 0x8000000000000000, + convInt64: 0x7fffffffffffffff, + convUInt64: 0xffffffffffffffff, }, { input: 0xfffffffffffffbfe, - convInt64: -0x8000000000000000, + convInt64: 0x7fffffffffffffff, convUInt64: 0xfffffffffffff800, }, } diff --git a/src/runtime/coro.go b/src/runtime/coro.go index 30ada455e4985e..40d4e47fbee79d 100644 --- a/src/runtime/coro.go +++ b/src/runtime/coro.go @@ -4,7 +4,10 @@ package runtime -import "unsafe" +import ( + "internal/runtime/sys" + "unsafe" +) // A coro represents extra concurrency without extra parallelism, // as would be needed for a coroutine implementation. @@ -39,7 +42,7 @@ type coro struct { func newcoro(f func(*coro)) *coro { c := new(coro) c.f = f - pc := getcallerpc() + pc := sys.GetCallerPC() gp := getg() systemstack(func() { mp := gp.m @@ -134,6 +137,16 @@ func coroswitch_m(gp *g) { // emitting an event for every single transition. trace := traceAcquire() + canCAS := true + bubble := gp.bubble + if bubble != nil { + // If we're in a synctest group, always use casgstatus (which tracks + // group idleness) rather than directly CASing. Mark the group as active + // while we're in the process of transferring control. + canCAS = false + bubble.incActive() + } + if locked { // Detach the goroutine from the thread; we'll attach to the goroutine we're // switching to before returning. @@ -152,7 +165,7 @@ func coroswitch_m(gp *g) { // If we can CAS ourselves directly from running to waiting, so do, // keeping the control transfer as lightweight as possible. gp.waitreason = waitReasonCoroutine - if !gp.atomicstatus.CompareAndSwap(_Grunning, _Gwaiting) { + if !canCAS || !gp.atomicstatus.CompareAndSwap(_Grunning, _Gwaiting) { // The CAS failed: use casgstatus, which will take care of // coordinating with the garbage collector about the state change. casgstatus(gp, _Grunning, _Gwaiting) @@ -208,7 +221,19 @@ func coroswitch_m(gp *g) { // directly if possible. setGNoWB(&mp.curg, gnext) setMNoWB(&gnext.m, mp) - if !gnext.atomicstatus.CompareAndSwap(_Gwaiting, _Grunning) { + + // Synchronize with any out-standing goroutine profile. We're about to start + // executing, and an invariant of the profiler is that we tryRecordGoroutineProfile + // whenever a goroutine is about to start running. + // + // N.B. We must do this before transitioning to _Grunning but after installing gnext + // in curg, so that we have a valid curg for allocation (tryRecordGoroutineProfile + // may allocate). + if goroutineProfile.active { + tryRecordGoroutineProfile(gnext, nil, osyield) + } + + if !canCAS || !gnext.atomicstatus.CompareAndSwap(_Gwaiting, _Grunning) { // The CAS failed: use casgstatus, which will take care of // coordinating with the garbage collector about the state change. casgstatus(gnext, _Gwaiting, _Grunnable) @@ -226,6 +251,10 @@ func coroswitch_m(gp *g) { traceRelease(trace) } + if bubble != nil { + bubble.decActive() + } + // Switch to gnext. Does not return. gogo(&gnext.sched) } diff --git a/src/runtime/coro_test.go b/src/runtime/coro_test.go index 10b5e1ea08d8d9..ddc734a17f5228 100644 --- a/src/runtime/coro_test.go +++ b/src/runtime/coro_test.go @@ -5,6 +5,7 @@ package runtime_test import ( + "internal/race" "internal/testenv" "runtime" "strings" @@ -35,6 +36,9 @@ func TestCoroCgoCallback(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("coro cgo callback tests not supported on Windows") } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } for _, test := range []string{ "CoroCgoIterCallback", "CoroCgoIterCallbackYield", diff --git a/src/runtime/coverage/coverage.go b/src/runtime/coverage/coverage.go index 6b99a0bce6e638..c9f725e3efcfbb 100644 --- a/src/runtime/coverage/coverage.go +++ b/src/runtime/coverage/coverage.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Package coverage contains APIs for writing coverage profile data at runtime +// from long-running and/or server programs that do not terminate via [os.Exit]. package coverage import ( diff --git a/src/runtime/cpuflags.go b/src/runtime/cpuflags.go index bbe93c5bea2d7e..6452364b68ec32 100644 --- a/src/runtime/cpuflags.go +++ b/src/runtime/cpuflags.go @@ -13,12 +13,16 @@ import ( const ( offsetX86HasAVX = unsafe.Offsetof(cpu.X86.HasAVX) offsetX86HasAVX2 = unsafe.Offsetof(cpu.X86.HasAVX2) + offsetX86HasAVX512 = unsafe.Offsetof(cpu.X86.HasAVX512) // F+CD+BW+DQ+VL offsetX86HasERMS = unsafe.Offsetof(cpu.X86.HasERMS) offsetX86HasRDTSCP = unsafe.Offsetof(cpu.X86.HasRDTSCP) offsetARMHasIDIVA = unsafe.Offsetof(cpu.ARM.HasIDIVA) offsetMIPS64XHasMSA = unsafe.Offsetof(cpu.MIPS64X.HasMSA) + + offsetLOONG64HasLSX = unsafe.Offsetof(cpu.Loong64.HasLSX) + offsetLOONG64HasLASX = unsafe.Offsetof(cpu.Loong64.HasLASX) ) var ( @@ -31,4 +35,10 @@ var ( armHasVFPv4 bool arm64HasATOMICS bool + + loong64HasLAMCAS bool + loong64HasLAM_BH bool + loong64HasLSX bool + + riscv64HasZbb bool ) diff --git a/src/runtime/crash_cgo_test.go b/src/runtime/crash_cgo_test.go index 57111c9aefa3be..baf4523a7ae109 100644 --- a/src/runtime/crash_cgo_test.go +++ b/src/runtime/crash_cgo_test.go @@ -8,8 +8,10 @@ package runtime_test import ( "fmt" + "internal/asan" "internal/goos" - "internal/platform" + "internal/msan" + "internal/race" "internal/testenv" "os" "os/exec" @@ -33,6 +35,9 @@ func TestCgoSignalDeadlock(t *testing.T) { if testing.Short() && runtime.GOOS == "windows" { t.Skip("Skipping in short mode") // takes up to 64 seconds } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "CgoSignalDeadlock") want := "OK\n" if got != want { @@ -41,6 +46,10 @@ func TestCgoSignalDeadlock(t *testing.T) { } func TestCgoTraceback(t *testing.T) { + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } + t.Parallel() got := runTestProg(t, "testprogcgo", "CgoTraceback") want := "OK\n" @@ -55,6 +64,9 @@ func TestCgoCallbackGC(t *testing.T) { case "plan9", "windows": t.Skipf("no pthreads on %s", runtime.GOOS) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } if testing.Short() { switch { case runtime.GOOS == "dragonfly": @@ -72,11 +84,33 @@ func TestCgoCallbackGC(t *testing.T) { } } +func TestCgoCallbackPprof(t *testing.T) { + t.Parallel() + switch runtime.GOOS { + case "plan9", "windows": + t.Skipf("no pthreads on %s", runtime.GOOS) + } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } + if testenv.CPUProfilingBroken() { + t.Skip("skipping on platform with broken profiling") + } + + got := runTestProg(t, "testprogcgo", "CgoCallbackPprof") + if want := "OK\n"; got != want { + t.Fatalf("expected %q, but got:\n%s", want, got) + } +} + func TestCgoExternalThreadPanic(t *testing.T) { t.Parallel() if runtime.GOOS == "plan9" { t.Skipf("no pthreads on %s", runtime.GOOS) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "CgoExternalThreadPanic") want := "panic: BOOM" if !strings.Contains(got, want) { @@ -91,6 +125,9 @@ func TestCgoExternalThreadSIGPROF(t *testing.T) { case "plan9", "windows": t.Skipf("no pthreads on %s", runtime.GOOS) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "CgoExternalThreadSIGPROF", "GO_START_SIGPROF_THREAD=1") if want := "OK\n"; got != want { @@ -105,6 +142,9 @@ func TestCgoExternalThreadSignal(t *testing.T) { case "plan9", "windows": t.Skipf("no pthreads on %s", runtime.GOOS) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "CgoExternalThreadSignal") if want := "OK\n"; got != want { @@ -120,6 +160,9 @@ func TestCgoDLLImports(t *testing.T) { if runtime.GOOS != "windows" { t.Skip("skipping windows specific test") } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "CgoDLLImportsMain") want := "OK\n" if got != want { @@ -134,6 +177,9 @@ func TestCgoExecSignalMask(t *testing.T) { case "windows", "plan9": t.Skipf("skipping signal mask test on %s", runtime.GOOS) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "CgoExecSignalMask", "GOTRACEBACK=system") want := "OK\n" if got != want { @@ -148,6 +194,9 @@ func TestEnsureDropM(t *testing.T) { case "windows", "plan9": t.Skipf("skipping dropm test on %s", runtime.GOOS) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "EnsureDropM") want := "OK\n" if got != want { @@ -162,6 +211,9 @@ func TestCgoCheckBytes(t *testing.T) { t.Parallel() // Make sure we don't count the build time as part of the run time. testenv.MustHaveGoBuild(t) + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } exe, err := buildTestProg(t, "testprogcgo") if err != nil { t.Fatal(err) @@ -200,6 +252,9 @@ func TestCgoCheckBytes(t *testing.T) { func TestCgoPanicDeadlock(t *testing.T) { t.Parallel() + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } // test issue 14432 got := runTestProg(t, "testprogcgo", "CgoPanicDeadlock") want := "panic: cgo error\n\n" @@ -210,6 +265,9 @@ func TestCgoPanicDeadlock(t *testing.T) { func TestCgoCCodeSIGPROF(t *testing.T) { t.Parallel() + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "CgoCCodeSIGPROF") want := "OK\n" if got != want { @@ -225,6 +283,9 @@ func TestCgoPprofCallback(t *testing.T) { case "windows", "plan9": t.Skipf("skipping cgo pprof callback test on %s", runtime.GOOS) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "CgoPprofCallback") want := "OK\n" if got != want { @@ -238,14 +299,21 @@ func TestCgoCrashTraceback(t *testing.T) { case "darwin/amd64": case "linux/amd64": case "linux/arm64": + case "linux/loong64": case "linux/ppc64le": default: t.Skipf("not yet supported on %s", platform) } + if asan.Enabled || msan.Enabled { + t.Skip("skipping test on ASAN/MSAN: triggers SIGSEGV in sanitizer runtime") + } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "CrashTraceback") for i := 1; i <= 3; i++ { if !strings.Contains(got, fmt.Sprintf("cgo symbolizer:%d", i)) { - t.Errorf("missing cgo symbolizer:%d", i) + t.Errorf("missing cgo symbolizer:%d in %s", i, got) } } } @@ -256,10 +324,14 @@ func TestCgoCrashTracebackGo(t *testing.T) { case "darwin/amd64": case "linux/amd64": case "linux/arm64": + case "linux/loong64": case "linux/ppc64le": default: t.Skipf("not yet supported on %s", platform) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "CrashTracebackGo") for i := 1; i <= 3; i++ { want := fmt.Sprintf("main.h%d", i) @@ -271,6 +343,9 @@ func TestCgoCrashTracebackGo(t *testing.T) { func TestCgoTracebackContext(t *testing.T) { t.Parallel() + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "TracebackContext") want := "OK\n" if got != want { @@ -280,6 +355,9 @@ func TestCgoTracebackContext(t *testing.T) { func TestCgoTracebackContextPreemption(t *testing.T) { t.Parallel() + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "TracebackContextPreemption") want := "OK\n" if got != want { @@ -287,14 +365,33 @@ func TestCgoTracebackContextPreemption(t *testing.T) { } } +func TestCgoTracebackContextProfile(t *testing.T) { + t.Parallel() + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } + got := runTestProg(t, "testprogcgo", "TracebackContextProfile") + want := "OK\n" + if got != want { + t.Errorf("expected %q got %v", want, got) + } +} + func testCgoPprof(t *testing.T, buildArg, runArg, top, bottom string) { t.Parallel() - if runtime.GOOS != "linux" || (runtime.GOARCH != "amd64" && runtime.GOARCH != "ppc64le" && runtime.GOARCH != "arm64") { + if runtime.GOOS != "linux" || (runtime.GOARCH != "amd64" && runtime.GOARCH != "ppc64le" && runtime.GOARCH != "arm64" && runtime.GOARCH != "loong64") { t.Skipf("not yet supported on %s/%s", runtime.GOOS, runtime.GOARCH) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } testenv.MustHaveGoRun(t) - exe, err := buildTestProg(t, "testprogcgo", buildArg) + var args []string + if buildArg != "" { + args = append(args, buildArg) + } + exe, err := buildTestProg(t, "testprogcgo", args...) if err != nil { t.Fatal(err) } @@ -355,6 +452,9 @@ func TestCgoPprof(t *testing.T) { } func TestCgoPprofPIE(t *testing.T) { + if race.Enabled { + t.Skip("skipping test: -race + PIE not supported") + } testCgoPprof(t, "-buildmode=pie", "CgoPprof", "cpuHog", "runtime.main") } @@ -367,23 +467,20 @@ func TestCgoPprofThreadNoTraceback(t *testing.T) { } func TestRaceProf(t *testing.T) { - if !platform.RaceDetectorSupported(runtime.GOOS, runtime.GOARCH) { - t.Skipf("skipping on %s/%s because race detector not supported", runtime.GOOS, runtime.GOARCH) + if !race.Enabled { + t.Skip("skipping: race detector not enabled") } if runtime.GOOS == "windows" { t.Skipf("skipping: test requires pthread support") // TODO: Can this test be rewritten to use the C11 thread API instead? } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } testenv.MustHaveGoRun(t) - // This test requires building various packages with -race, so - // it's somewhat slow. - if testing.Short() { - t.Skip("skipping test in -short mode") - } - - exe, err := buildTestProg(t, "testprogcgo", "-race") + exe, err := buildTestProg(t, "testprogcgo") if err != nil { t.Fatal(err) } @@ -399,8 +496,8 @@ func TestRaceProf(t *testing.T) { } func TestRaceSignal(t *testing.T) { - if !platform.RaceDetectorSupported(runtime.GOOS, runtime.GOARCH) { - t.Skipf("skipping on %s/%s because race detector not supported", runtime.GOOS, runtime.GOARCH) + if !race.Enabled { + t.Skip("skipping: race detector not enabled") } if runtime.GOOS == "windows" { t.Skipf("skipping: test requires pthread support") @@ -409,18 +506,15 @@ func TestRaceSignal(t *testing.T) { if runtime.GOOS == "darwin" || runtime.GOOS == "ios" { testenv.SkipFlaky(t, 60316) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } t.Parallel() testenv.MustHaveGoRun(t) - // This test requires building various packages with -race, so - // it's somewhat slow. - if testing.Short() { - t.Skip("skipping test in -short mode") - } - - exe, err := buildTestProg(t, "testprogcgo", "-race") + exe, err := buildTestProg(t, "testprogcgo") if err != nil { t.Fatal(err) } @@ -441,6 +535,9 @@ func TestCgoNumGoroutine(t *testing.T) { case "windows", "plan9": t.Skipf("skipping numgoroutine test on %s", runtime.GOOS) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } t.Parallel() got := runTestProg(t, "testprogcgo", "NumGoroutine") want := "OK\n" @@ -459,6 +556,9 @@ func TestCatchPanic(t *testing.T) { t.Skipf("crash() on darwin/amd64 doesn't raise SIGABRT") } } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } testenv.MustHaveGoRun(t) @@ -486,6 +586,9 @@ func TestCgoLockOSThreadExit(t *testing.T) { case "plan9", "windows": t.Skipf("no pthreads on %s", runtime.GOOS) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } t.Parallel() testLockOSThreadExit(t, "testprogcgo") } @@ -494,6 +597,9 @@ func TestWindowsStackMemoryCgo(t *testing.T) { if runtime.GOOS != "windows" { t.Skip("skipping windows specific test") } + if race.Enabled { + t.Skip("skipping test: race mode uses more stack memory") + } testenv.SkipFlaky(t, 22575) o := runTestProg(t, "testprogcgo", "StackMemory") stackUsage, err := strconv.Atoi(o) @@ -510,6 +616,9 @@ func TestSigStackSwapping(t *testing.T) { case "plan9", "windows": t.Skipf("no sigaltstack on %s", runtime.GOOS) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } t.Parallel() got := runTestProg(t, "testprogcgo", "SigStack") want := "OK\n" @@ -527,6 +636,12 @@ func TestCgoTracebackSigpanic(t *testing.T) { // than injecting a sigpanic. t.Skip("no sigpanic in C on windows") } + if asan.Enabled || msan.Enabled { + t.Skip("skipping test on ASAN/MSAN: triggers SIGSEGV in sanitizer runtime") + } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } if runtime.GOOS == "ios" { testenv.SkipFlaky(t, 59912) } @@ -554,6 +669,9 @@ func TestCgoTracebackSigpanic(t *testing.T) { } func TestCgoPanicCallback(t *testing.T) { + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } t.Parallel() got := runTestProg(t, "testprogcgo", "PanicCallback") t.Log(got) @@ -629,10 +747,11 @@ func TestSegv(t *testing.T) { case "plan9", "windows": t.Skipf("no signals on %s", runtime.GOOS) } + if race.Enabled || asan.Enabled || msan.Enabled { + t.Skip("skipping test on race/ASAN/MSAN: triggers SIGSEGV in sanitizer runtime") + } for _, test := range []string{"Segv", "SegvInCgo", "TgkillSegv", "TgkillSegvInCgo"} { - test := test - // The tgkill variants only run on Linux. if runtime.GOOS != "linux" && strings.HasPrefix(test, "Tgkill") { continue @@ -642,6 +761,9 @@ func TestSegv(t *testing.T) { if test == "SegvInCgo" && runtime.GOOS == "ios" { testenv.SkipFlaky(t, 59947) // Don't even try, in case it times out. } + if strings.HasSuffix(test, "InCgo") && runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } t.Parallel() prog := "testprog" @@ -694,6 +816,9 @@ func TestAbortInCgo(t *testing.T) { // without going through the runtime at all. t.Skipf("no signals on %s", runtime.GOOS) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } t.Parallel() got := runTestProg(t, "testprogcgo", "Abort") @@ -715,17 +840,9 @@ func TestEINTR(t *testing.T) { switch runtime.GOOS { case "plan9", "windows": t.Skipf("no EINTR on %s", runtime.GOOS) - case "linux": - if runtime.GOARCH == "386" { - // On linux-386 the Go signal handler sets - // a restorer function that is not preserved - // by the C sigaction call in the test, - // causing the signal handler to crash when - // returning the normal code. The test is not - // architecture-specific, so just skip on 386 - // rather than doing a complicated workaround. - t.Skip("skipping on linux-386; C sigaction does not preserve Go restorer") - } + } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") } t.Parallel() @@ -742,6 +859,9 @@ func TestNeedmDeadlock(t *testing.T) { case "plan9", "windows": t.Skipf("no signals on %s", runtime.GOOS) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } output := runTestProg(t, "testprogcgo", "NeedmDeadlock") want := "OK\n" if output != want { @@ -750,7 +870,9 @@ func TestNeedmDeadlock(t *testing.T) { } func TestCgoNoCallback(t *testing.T) { - t.Skip("TODO(#56378): enable in Go 1.23") + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "CgoNoCallback") want := "function marked with #cgo nocallback called back into Go" if !strings.Contains(got, want) { @@ -759,7 +881,12 @@ func TestCgoNoCallback(t *testing.T) { } func TestCgoNoEscape(t *testing.T) { - t.Skip("TODO(#56378): enable in Go 1.23") + if asan.Enabled { + t.Skip("skipping test: ASAN forces extra heap allocations") + } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "CgoNoEscape") want := "OK\n" if got != want { @@ -767,7 +894,22 @@ func TestCgoNoEscape(t *testing.T) { } } +// Issue #63739. +func TestCgoEscapeWithMultiplePointers(t *testing.T) { + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } + got := runTestProg(t, "testprogcgo", "CgoEscapeWithMultiplePointers") + want := "OK\n" + if got != want { + t.Fatalf("output is %s; want %s", got, want) + } +} + func TestCgoTracebackGoroutineProfile(t *testing.T) { + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } output := runTestProg(t, "testprogcgo", "GoroutineProfile") want := "OK\n" if output != want { @@ -780,6 +922,9 @@ func TestCgoSigfwd(t *testing.T) { if !goos.IsUnix { t.Skipf("no signals on %s", runtime.GOOS) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "CgoSigfwd", "GO_TEST_CGOSIGFWD=1") if want := "OK\n"; got != want { @@ -788,6 +933,9 @@ func TestCgoSigfwd(t *testing.T) { } func TestDestructorCallback(t *testing.T) { + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } t.Parallel() got := runTestProg(t, "testprogcgo", "DestructorCallback") if want := "OK\n"; got != want { @@ -795,40 +943,15 @@ func TestDestructorCallback(t *testing.T) { } } -func TestDestructorCallbackRace(t *testing.T) { - // This test requires building with -race, - // so it's somewhat slow. - if testing.Short() { - t.Skip("skipping test in -short mode") - } - - if !platform.RaceDetectorSupported(runtime.GOOS, runtime.GOARCH) { - t.Skipf("skipping on %s/%s because race detector not supported", runtime.GOOS, runtime.GOARCH) - } - - t.Parallel() - - exe, err := buildTestProg(t, "testprogcgo", "-race") - if err != nil { - t.Fatal(err) - } - - got, err := testenv.CleanCmdEnv(exec.Command(exe, "DestructorCallback")).CombinedOutput() - if err != nil { - t.Fatal(err) - } - - if want := "OK\n"; string(got) != want { - t.Errorf("expected %q, but got:\n%s", want, got) - } -} - func TestEnsureBindM(t *testing.T) { t.Parallel() switch runtime.GOOS { case "windows", "plan9": t.Skipf("skipping bindm test on %s", runtime.GOOS) } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "EnsureBindM") want := "OK\n" if got != want { @@ -842,6 +965,13 @@ func TestStackSwitchCallback(t *testing.T) { case "windows", "plan9", "android", "ios", "openbsd": // no getcontext t.Skipf("skipping test on %s", runtime.GOOS) } + if asan.Enabled { + // ASAN prints this as a warning. + t.Skip("skipping test on ASAN because ASAN doesn't fully support makecontext/swapcontext functions") + } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } got := runTestProg(t, "testprogcgo", "StackSwitchCallback") skip := "SKIP\n" if got == skip { @@ -852,3 +982,16 @@ func TestStackSwitchCallback(t *testing.T) { t.Errorf("expected %q, got %v", want, got) } } + +func TestCgoToGoCallGoexit(t *testing.T) { + if runtime.GOOS == "plan9" || runtime.GOOS == "windows" { + t.Skipf("no pthreads on %s", runtime.GOOS) + } + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } + output := runTestProg(t, "testprogcgo", "CgoToGoCallGoexit") + if !strings.Contains(output, "runtime.Goexit called in a thread that was not created by the Go runtime") { + t.Fatalf("output should contain %s, got %s", "runtime.Goexit called in a thread that was not created by the Go runtime", output) + } +} diff --git a/src/runtime/crash_test.go b/src/runtime/crash_test.go index 69e1034ff825af..2b8ca549ad84f2 100644 --- a/src/runtime/crash_test.go +++ b/src/runtime/crash_test.go @@ -10,6 +10,10 @@ import ( "errors" "flag" "fmt" + "internal/asan" + "internal/msan" + "internal/profile" + "internal/race" "internal/testenv" traceparse "internal/trace" "io" @@ -32,8 +36,11 @@ const entrypointVar = "RUNTIME_TEST_ENTRYPOINT" func TestMain(m *testing.M) { switch entrypoint := os.Getenv(entrypointVar); entrypoint { - case "crash": - crash() + case "panic": + crashViaPanic() + panic("unreachable") + case "trap": + crashViaTrap() panic("unreachable") default: log.Fatalf("invalid %s: %q", entrypointVar, entrypoint) @@ -162,6 +169,16 @@ func buildTestProg(t *testing.T, binary string, flags ...string) (string, error) // Don't get confused if testenv.GoToolPath calls t.Skip. target.err = errors.New("building test called t.Skip") + if asan.Enabled { + flags = append(flags, "-asan") + } + if msan.Enabled { + flags = append(flags, "-msan") + } + if race.Enabled { + flags = append(flags, "-race") + } + exe := filepath.Join(dir, name+".exe") start := time.Now() @@ -170,18 +187,19 @@ func buildTestProg(t *testing.T, binary string, flags ...string) (string, error) cmd.Dir = "testdata/" + binary cmd = testenv.CleanCmdEnv(cmd) - // Add the rangefunc GOEXPERIMENT unconditionally since some tests depend on it. - // TODO(61405): Remove this once it's enabled by default. + // If tests need any experimental flags, add them here. + // + // TODO(vsaioc): Remove `goroutineleakprofile` once the feature is no longer experimental. edited := false for i := range cmd.Env { e := cmd.Env[i] if _, vars, ok := strings.Cut(e, "GOEXPERIMENT="); ok { - cmd.Env[i] = "GOEXPERIMENT=" + vars + ",rangefunc" - edited = true + cmd.Env[i] = "GOEXPERIMENT=" + vars + ",goroutineleakprofile" + edited, _ = true, vars } } if !edited { - cmd.Env = append(cmd.Env, "GOEXPERIMENT=rangefunc") + cmd.Env = append(cmd.Env, "GOEXPERIMENT=goroutineleakprofile") } out, err := cmd.CombinedOutput() @@ -212,6 +230,9 @@ func testCrashHandler(t *testing.T, cgo bool) { } var output string if cgo { + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } output = runTestProg(t, "testprogcgo", "Crash") } else { output = runTestProg(t, "testprog", "Crash") @@ -226,9 +247,17 @@ func TestCrashHandler(t *testing.T) { testCrashHandler(t, false) } +var deadlockBuildTypes = testenv.SpecialBuildTypes{ + // External linking brings in cgo, causing deadlock detection not working. + Cgo: false, + Asan: asan.Enabled, + Msan: msan.Enabled, + Race: race.Enabled, +} + func testDeadlock(t *testing.T, name string) { // External linking brings in cgo, causing deadlock detection not working. - testenv.MustInternalLink(t, false) + testenv.MustInternalLink(t, deadlockBuildTypes) output := runTestProg(t, "testprog", name) want := "fatal error: all goroutines are asleep - deadlock!\n" @@ -255,7 +284,7 @@ func TestLockedDeadlock2(t *testing.T) { func TestGoexitDeadlock(t *testing.T) { // External linking brings in cgo, causing deadlock detection not working. - testenv.MustInternalLink(t, false) + testenv.MustInternalLink(t, deadlockBuildTypes) output := runTestProg(t, "testprog", "GoexitDeadlock") want := "no goroutines (main called runtime.Goexit) - deadlock!" @@ -353,9 +382,40 @@ panic: third panic } +func TestRepanickedPanic(t *testing.T) { + output := runTestProg(t, "testprog", "RepanickedPanic") + want := `panic: message [recovered, repanicked] +` + if !strings.HasPrefix(output, want) { + t.Fatalf("output does not start with %q:\n%s", want, output) + } +} + +func TestRepanickedMiddlePanic(t *testing.T) { + output := runTestProg(t, "testprog", "RepanickedMiddlePanic") + want := `panic: inner [recovered] + panic: middle [recovered, repanicked] + panic: outer +` + if !strings.HasPrefix(output, want) { + t.Fatalf("output does not start with %q:\n%s", want, output) + } +} + +func TestRepanickedPanicSandwich(t *testing.T) { + output := runTestProg(t, "testprog", "RepanickedPanicSandwich") + want := `panic: outer [recovered] + panic: inner [recovered] + panic: outer +` + if !strings.HasPrefix(output, want) { + t.Fatalf("output does not start with %q:\n%s", want, output) + } +} + func TestGoexitCrash(t *testing.T) { // External linking brings in cgo, causing deadlock detection not working. - testenv.MustInternalLink(t, false) + testenv.MustInternalLink(t, deadlockBuildTypes) output := runTestProg(t, "testprog", "GoexitExit") want := "no goroutines (main called runtime.Goexit) - deadlock!" @@ -416,7 +476,7 @@ func TestBreakpoint(t *testing.T) { func TestGoexitInPanic(t *testing.T) { // External linking brings in cgo, causing deadlock detection not working. - testenv.MustInternalLink(t, false) + testenv.MustInternalLink(t, deadlockBuildTypes) // see issue 8774: this code used to trigger an infinite recursion output := runTestProg(t, "testprog", "GoexitInPanic") @@ -483,7 +543,7 @@ func TestPanicAfterGoexit(t *testing.T) { func TestRecoveredPanicAfterGoexit(t *testing.T) { // External linking brings in cgo, causing deadlock detection not working. - testenv.MustInternalLink(t, false) + testenv.MustInternalLink(t, deadlockBuildTypes) output := runTestProg(t, "testprog", "RecoveredPanicAfterGoexit") want := "fatal error: no goroutines (main called runtime.Goexit) - deadlock!" @@ -494,7 +554,7 @@ func TestRecoveredPanicAfterGoexit(t *testing.T) { func TestRecoverBeforePanicAfterGoexit(t *testing.T) { // External linking brings in cgo, causing deadlock detection not working. - testenv.MustInternalLink(t, false) + testenv.MustInternalLink(t, deadlockBuildTypes) t.Parallel() output := runTestProg(t, "testprog", "RecoverBeforePanicAfterGoexit") @@ -506,7 +566,7 @@ func TestRecoverBeforePanicAfterGoexit(t *testing.T) { func TestRecoverBeforePanicAfterGoexit2(t *testing.T) { // External linking brings in cgo, causing deadlock detection not working. - testenv.MustInternalLink(t, false) + testenv.MustInternalLink(t, deadlockBuildTypes) t.Parallel() output := runTestProg(t, "testprog", "RecoverBeforePanicAfterGoexit2") @@ -619,10 +679,16 @@ func TestConcurrentMapWrites(t *testing.T) { if !*concurrentMapTest { t.Skip("skipping without -run_concurrent_map_tests") } + if race.Enabled { + t.Skip("skipping test: -race will catch the race, this test is for the built-in race detection") + } testenv.MustHaveGoRun(t) output := runTestProg(t, "testprog", "concurrentMapWrites") - want := "fatal error: concurrent map writes" - if !strings.HasPrefix(output, want) { + want := "fatal error: concurrent map writes\n" + // Concurrent writes can corrupt the map in a way that we + // detect with a separate throw. + want2 := "fatal error: small map with no empty slot (concurrent map writes?)\n" + if !strings.HasPrefix(output, want) && !strings.HasPrefix(output, want2) { t.Fatalf("output does not start with %q:\n%s", want, output) } } @@ -630,10 +696,16 @@ func TestConcurrentMapReadWrite(t *testing.T) { if !*concurrentMapTest { t.Skip("skipping without -run_concurrent_map_tests") } + if race.Enabled { + t.Skip("skipping test: -race will catch the race, this test is for the built-in race detection") + } testenv.MustHaveGoRun(t) output := runTestProg(t, "testprog", "concurrentMapReadWrite") - want := "fatal error: concurrent map read and map write" - if !strings.HasPrefix(output, want) { + want := "fatal error: concurrent map read and map write\n" + // Concurrent writes can corrupt the map in a way that we + // detect with a separate throw. + want2 := "fatal error: small map with no empty slot (concurrent map writes?)\n" + if !strings.HasPrefix(output, want) && !strings.HasPrefix(output, want2) { t.Fatalf("output does not start with %q:\n%s", want, output) } } @@ -641,14 +713,50 @@ func TestConcurrentMapIterateWrite(t *testing.T) { if !*concurrentMapTest { t.Skip("skipping without -run_concurrent_map_tests") } + if race.Enabled { + t.Skip("skipping test: -race will catch the race, this test is for the built-in race detection") + } testenv.MustHaveGoRun(t) output := runTestProg(t, "testprog", "concurrentMapIterateWrite") - want := "fatal error: concurrent map iteration and map write" - if !strings.HasPrefix(output, want) { + want := "fatal error: concurrent map iteration and map write\n" + // Concurrent writes can corrupt the map in a way that we + // detect with a separate throw. + want2 := "fatal error: small map with no empty slot (concurrent map writes?)\n" + if !strings.HasPrefix(output, want) && !strings.HasPrefix(output, want2) { t.Fatalf("output does not start with %q:\n%s", want, output) } } +func TestConcurrentMapWritesIssue69447(t *testing.T) { + testenv.MustHaveGoRun(t) + if race.Enabled { + t.Skip("skipping test: -race will catch the race, this test is for the built-in race detection") + } + exe, err := buildTestProg(t, "testprog") + if err != nil { + t.Fatal(err) + } + for i := 0; i < 200; i++ { + output := runBuiltTestProg(t, exe, "concurrentMapWrites") + if output == "" { + // If we didn't detect an error, that's ok. + // This case makes this test not flaky like + // the other ones above. + // (More correctly, this case makes this test flaky + // in the other direction, in that it might not + // detect a problem even if there is one.) + continue + } + want := "fatal error: concurrent map writes\n" + // Concurrent writes can corrupt the map in a way that we + // detect with a separate throw. + want2 := "fatal error: small map with no empty slot (concurrent map writes?)\n" + if !strings.HasPrefix(output, want) && !strings.HasPrefix(output, want2) { + t.Fatalf("output does not start with %q:\n%s", want, output) + } + } +} + type point struct { x, y *int } @@ -724,6 +832,9 @@ retry: } func TestBadTraceback(t *testing.T) { + if asan.Enabled || msan.Enabled || race.Enabled { + t.Skip("skipped test: checkptr mode catches the corruption") + } output := runTestProg(t, "testprog", "BadTraceback") for _, want := range []string{ "unexpected return pc", @@ -744,6 +855,9 @@ func TestTimePprof(t *testing.T) { case "aix", "darwin", "illumos", "openbsd", "solaris": t.Skipf("skipping on %s because nanotime calls libc", runtime.GOOS) } + if race.Enabled || asan.Enabled || msan.Enabled { + t.Skip("skipping on sanitizers because the sanitizer runtime is external code") + } // Pass GOTRACEBACK for issue #41120 to try to get more // information on timeout. @@ -819,8 +933,7 @@ func init() { } func TestRuntimePanic(t *testing.T) { - testenv.MustHaveExec(t) - cmd := testenv.CleanCmdEnv(exec.Command(os.Args[0], "-test.run=^TestRuntimePanic$")) + cmd := testenv.CleanCmdEnv(exec.Command(testenv.Executable(t), "-test.run=^TestRuntimePanic$")) cmd.Env = append(cmd.Env, "GO_TEST_RUNTIME_PANIC=1") out, err := cmd.CombinedOutput() t.Logf("%s", out) @@ -832,8 +945,7 @@ func TestRuntimePanic(t *testing.T) { } func TestTracebackRuntimeFunction(t *testing.T) { - testenv.MustHaveExec(t) - cmd := testenv.CleanCmdEnv(exec.Command(os.Args[0], "-test.run=TestTracebackRuntimeFunction")) + cmd := testenv.CleanCmdEnv(exec.Command(testenv.Executable(t), "-test.run=^TestTracebackRuntimeFunction$")) cmd.Env = append(cmd.Env, "GO_TEST_RUNTIME_NPE_READMEMSTATS=1") out, err := cmd.CombinedOutput() t.Logf("%s", out) @@ -845,8 +957,7 @@ func TestTracebackRuntimeFunction(t *testing.T) { } func TestTracebackRuntimeMethod(t *testing.T) { - testenv.MustHaveExec(t) - cmd := testenv.CleanCmdEnv(exec.Command(os.Args[0], "-test.run=TestTracebackRuntimeMethod")) + cmd := testenv.CleanCmdEnv(exec.Command(testenv.Executable(t), "-test.run=^TestTracebackRuntimeMethod$")) cmd.Env = append(cmd.Env, "GO_TEST_RUNTIME_NPE_FUNCMETHOD=1") out, err := cmd.CombinedOutput() t.Logf("%s", out) @@ -859,14 +970,12 @@ func TestTracebackRuntimeMethod(t *testing.T) { // Test that g0 stack overflows are handled gracefully. func TestG0StackOverflow(t *testing.T) { - testenv.MustHaveExec(t) - if runtime.GOOS == "ios" { testenv.SkipFlaky(t, 62671) } if os.Getenv("TEST_G0_STACK_OVERFLOW") != "1" { - cmd := testenv.CleanCmdEnv(testenv.Command(t, os.Args[0], "-test.run=^TestG0StackOverflow$", "-test.v")) + cmd := testenv.CleanCmdEnv(testenv.Command(t, testenv.Executable(t), "-test.run=^TestG0StackOverflow$", "-test.v")) cmd.Env = append(cmd.Env, "TEST_G0_STACK_OVERFLOW=1") out, err := cmd.CombinedOutput() t.Logf("output:\n%s", out) @@ -907,7 +1016,7 @@ func init() { func TestCrashWhileTracing(t *testing.T) { testenv.MustHaveExec(t) - cmd := testenv.CleanCmdEnv(testenv.Command(t, os.Args[0])) + cmd := testenv.CleanCmdEnv(testenv.Command(t, testenv.Executable(t))) cmd.Env = append(cmd.Env, "TEST_CRASH_WHILE_TRACING=1") stdOut, err := cmd.StdoutPipe() var errOut bytes.Buffer @@ -920,7 +1029,8 @@ func TestCrashWhileTracing(t *testing.T) { if err != nil { t.Fatalf("could not create trace.NewReader: %v", err) } - var seen, seenSync bool + var seen bool + nSync := 0 i := 1 loop: for ; ; i++ { @@ -935,7 +1045,7 @@ loop: } switch ev.Kind() { case traceparse.EventSync: - seenSync = true + nSync = ev.Sync().N case traceparse.EventLog: v := ev.Log() if v.Category == "xyzzy-cat" && v.Message == "xyzzy-msg" { @@ -949,7 +1059,7 @@ loop: if err := cmd.Wait(); err == nil { t.Error("the process should have panicked") } - if !seenSync { + if nSync <= 1 { t.Errorf("expected at least one full generation to have been emitted before the trace was considered broken") } if !seen { @@ -1020,7 +1130,9 @@ func TestPanicWhilePanicking(t *testing.T) { func TestPanicOnUnsafeSlice(t *testing.T) { output := runTestProg(t, "testprog", "panicOnNilAndEleSizeIsZero") - want := "panic: runtime error: unsafe.Slice: ptr is nil and len is not zero" + // Note: This is normally a panic, but is a throw when checkptr is + // enabled. + want := "unsafe.Slice: ptr is nil and len is not zero" if !strings.Contains(output, want) { t.Errorf("output does not contain %q:\n%s", want, output) } @@ -1034,3 +1146,103 @@ func TestNetpollWaiters(t *testing.T) { t.Fatalf("output is not %q\n%s", want, output) } } + +func TestFinalizerOrCleanupDeadlock(t *testing.T) { + t.Parallel() + + for _, useCleanup := range []bool{false, true} { + progName := "Finalizer" + want := "runtime.runFinalizers" + if useCleanup { + progName = "Cleanup" + want = "runtime.runCleanups" + } + + // The runtime.runFinalizers/runtime.runCleanups frame should appear in panics, even if + // runtime frames are normally hidden (GOTRACEBACK=all). + t.Run("Panic", func(t *testing.T) { + t.Parallel() + output := runTestProg(t, "testprog", progName+"Deadlock", "GOTRACEBACK=all", "GO_TEST_FINALIZER_DEADLOCK=panic") + want := want + "()" + if !strings.Contains(output, want) { + t.Errorf("output does not contain %q:\n%s", want, output) + } + }) + + // The runtime.runFinalizers/runtime.Cleanups frame should appear in runtime.Stack, + // even though runtime frames are normally hidden. + t.Run("Stack", func(t *testing.T) { + t.Parallel() + output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=stack") + want := want + "()" + if !strings.Contains(output, want) { + t.Errorf("output does not contain %q:\n%s", want, output) + } + }) + + // The runtime.runFinalizers/runtime.Cleanups frame should appear in goroutine + // profiles. + t.Run("PprofProto", func(t *testing.T) { + t.Parallel() + output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=pprof_proto") + + p, err := profile.Parse(strings.NewReader(output)) + if err != nil { + // Logging the binary proto data is not very nice, but it might + // be a text error message instead. + t.Logf("Output: %s", output) + t.Fatalf("Error parsing proto output: %v", err) + } + for _, s := range p.Sample { + for _, loc := range s.Location { + for _, line := range loc.Line { + if line.Function.Name == want { + // Done! + return + } + } + } + } + t.Errorf("Profile does not contain %q:\n%s", want, p) + }) + + // The runtime.runFinalizers/runtime.runCleanups frame should appear in goroutine + // profiles (debug=1). + t.Run("PprofDebug1", func(t *testing.T) { + t.Parallel() + output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=pprof_debug1") + want := want + "+" + if !strings.Contains(output, want) { + t.Errorf("output does not contain %q:\n%s", want, output) + } + }) + + // The runtime.runFinalizers/runtime.runCleanups frame should appear in goroutine + // profiles (debug=2). + t.Run("PprofDebug2", func(t *testing.T) { + t.Parallel() + output := runTestProg(t, "testprog", progName+"Deadlock", "GO_TEST_FINALIZER_DEADLOCK=pprof_debug2") + want := want + "()" + if !strings.Contains(output, want) { + t.Errorf("output does not contain %q:\n%s", want, output) + } + }) + } +} + +func TestSynctestCondSignalFromNoBubble(t *testing.T) { + for _, test := range []string{ + "SynctestCond/signal/no_bubble", + "SynctestCond/broadcast/no_bubble", + "SynctestCond/signal/other_bubble", + "SynctestCond/broadcast/other_bubble", + } { + t.Run(test, func(t *testing.T) { + output := runTestProg(t, "testprog", test) + want := "fatal error: semaphore wake of synctest goroutine from outside bubble" + if !strings.Contains(output, want) { + t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want) + } + }) + } +} diff --git a/src/runtime/crash_unix_test.go b/src/runtime/crash_unix_test.go index 123a462423db4a..f9c12d54047485 100644 --- a/src/runtime/crash_unix_test.go +++ b/src/runtime/crash_unix_test.go @@ -65,7 +65,7 @@ func TestCrashDumpsAllThreads(t *testing.T) { t.Skipf("skipping; not supported on %v", runtime.GOOS) } - if runtime.GOOS == "openbsd" && (runtime.GOARCH == "arm" || runtime.GOARCH == "mips64" || runtime.GOARCH == "ppc64") { + if runtime.GOOS == "openbsd" && (runtime.GOARCH == "arm" || runtime.GOARCH == "ppc64") { // This may be ncpu < 2 related... t.Skipf("skipping; test fails on %s/%s - see issue #42464", runtime.GOOS, runtime.GOARCH) } @@ -163,7 +163,7 @@ func TestPanicSystemstack(t *testing.T) { } t.Parallel() - cmd := exec.Command(os.Args[0], "testPanicSystemstackInternal") + cmd := exec.Command(testenv.Executable(t), "testPanicSystemstackInternal") cmd = testenv.CleanCmdEnv(cmd) cmd.Dir = t.TempDir() // put any core file in tempdir cmd.Env = append(cmd.Env, "GOTRACEBACK=crash") diff --git a/src/runtime/debug.go b/src/runtime/debug.go index c477e2b9f6e4a0..56b766f11fa37d 100644 --- a/src/runtime/debug.go +++ b/src/runtime/debug.go @@ -10,9 +10,63 @@ import ( ) // GOMAXPROCS sets the maximum number of CPUs that can be executing -// simultaneously and returns the previous setting. It defaults to -// the value of [runtime.NumCPU]. If n < 1, it does not change the current setting. -// This call will go away when the scheduler improves. +// simultaneously and returns the previous setting. If n < 1, it does not change +// the current setting. +// +// # Default +// +// If the GOMAXPROCS environment variable is set to a positive whole number, +// GOMAXPROCS defaults to that value. +// +// Otherwise, the Go runtime selects an appropriate default value from a combination of +// - the number of logical CPUs on the machine, +// - the process’s CPU affinity mask, +// - and, on Linux, the process’s average CPU throughput limit based on cgroup CPU +// quota, if any. +// +// If GODEBUG=containermaxprocs=0 is set and GOMAXPROCS is not set by the +// environment variable, then GOMAXPROCS instead defaults to the value of +// [runtime.NumCPU]. Note that GODEBUG=containermaxprocs=0 is [default] for +// language version 1.24 and below. +// +// # Updates +// +// The Go runtime periodically updates the default value based on changes to +// the total logical CPU count, the CPU affinity mask, or cgroup quota. Setting +// a custom value with the GOMAXPROCS environment variable or by calling +// GOMAXPROCS disables automatic updates. The default value and automatic +// updates can be restored by calling [SetDefaultGOMAXPROCS]. +// +// If GODEBUG=updatemaxprocs=0 is set, the Go runtime does not perform +// automatic GOMAXPROCS updating. Note that GODEBUG=updatemaxprocs=0 is +// [default] for language version 1.24 and below. +// +// # Compatibility +// +// Note that the default GOMAXPROCS behavior may change as the scheduler +// improves, especially the implementation detail below. +// +// # Implementation details +// +// When computing default GOMAXPROCS via cgroups, the Go runtime computes the +// "average CPU throughput limit" as the cgroup CPU quota / period. In cgroup +// v2, these values come from the cpu.max file. In cgroup v1, they come from +// cpu.cfs_quota_us and cpu.cfs_period_us, respectively. In container runtimes +// that allow configuring CPU limits, this value usually corresponds to the +// "CPU limit" option, not "CPU request". +// +// The Go runtime typically selects the default GOMAXPROCS as the minimum of +// the logical CPU count, the CPU affinity mask count, or the cgroup CPU +// throughput limit. However, it will never set GOMAXPROCS less than 2 unless +// the logical CPU count or CPU affinity mask count are below 2. +// +// If the cgroup CPU throughput limit is not a whole number, the Go runtime +// rounds up to the next whole number. +// +// GOMAXPROCS updates are performed up to once per second, or less if the +// application is idle. +// +// [default]: https://go.dev/doc/godebug#default func GOMAXPROCS(n int) int { if GOARCH == "wasm" && n > 1 { n = 1 // WebAssembly has no threads yet, so only one CPU is possible. @@ -20,27 +74,85 @@ func GOMAXPROCS(n int) int { lock(&sched.lock) ret := int(gomaxprocs) + if n <= 0 { + unlock(&sched.lock) + return ret + } + // Set early so we can wait for sysmon befor STW. See comment on + // computeMaxProcsLock. + sched.customGOMAXPROCS = true unlock(&sched.lock) - if n <= 0 || n == ret { + + // Wait for sysmon to complete running defaultGOMAXPROCS. + lock(&computeMaxProcsLock) + unlock(&computeMaxProcsLock) + + if n == ret { + // sched.customGOMAXPROCS set, but no need to actually STW + // since the gomaxprocs itself isn't changing. return ret } stw := stopTheWorldGC(stwGOMAXPROCS) // newprocs will be processed by startTheWorld + // + // TODO(prattmic): this could use a nicer API. Perhaps add it to the + // stw parameter? newprocs = int32(n) startTheWorldGC(stw) return ret } +// SetDefaultGOMAXPROCS updates the GOMAXPROCS setting to the runtime +// default, as described by [GOMAXPROCS], ignoring the GOMAXPROCS +// environment variable. +// +// SetDefaultGOMAXPROCS can be used to enable the default automatic updating +// GOMAXPROCS behavior if it has been disabled by the GOMAXPROCS +// environment variable or a prior call to [GOMAXPROCS], or to force an immediate +// update if the caller is aware of a change to the total logical CPU count, CPU +// affinity mask or cgroup quota. +func SetDefaultGOMAXPROCS() { + // SetDefaultGOMAXPROCS conceptually means "[re]do what the runtime + // would do at startup if the GOMAXPROCS environment variable were + // unset." It still respects GODEBUG. + + procs := defaultGOMAXPROCS(0) + + lock(&sched.lock) + curr := gomaxprocs + custom := sched.customGOMAXPROCS + unlock(&sched.lock) + + if !custom && procs == curr { + // Nothing to do if we're already using automatic GOMAXPROCS + // and the limit is unchanged. + return + } + + stw := stopTheWorldGC(stwGOMAXPROCS) + + // newprocs will be processed by startTheWorld + // + // TODO(prattmic): this could use a nicer API. Perhaps add it to the + // stw parameter? + newprocs = procs + lock(&sched.lock) + sched.customGOMAXPROCS = false + unlock(&sched.lock) + + startTheWorldGC(stw) +} + // NumCPU returns the number of logical CPUs usable by the current process. // // The set of available CPUs is checked by querying the operating system // at process startup. Changes to operating system CPU allocation after // process startup are not reflected. func NumCPU() int { - return int(ncpu) + return int(numCPUStartup) } // NumCgoCall returns the number of cgo calls made by the current process. @@ -65,7 +177,7 @@ func totalMutexWaitTimeNanos() int64 { // NumGoroutine returns the number of goroutines that currently exist. func NumGoroutine() int { - return int(gcount()) + return int(gcount(false)) } //go:linkname debug_modinfo runtime/debug.modinfo diff --git a/src/runtime/debug/example_monitor_test.go b/src/runtime/debug/example_monitor_test.go index b077e7adb3139b..ed6757df855d08 100644 --- a/src/runtime/debug/example_monitor_test.go +++ b/src/runtime/debug/example_monitor_test.go @@ -83,8 +83,11 @@ func monitor() { if err != nil { log.Fatal(err) } - cmd := exec.Command(exe, "-test.run=ExampleSetCrashOutput_monitor") - cmd.Env = append(os.Environ(), monitorVar+"=1") + cmd := exec.Command(exe, "-test.run=^ExampleSetCrashOutput_monitor$") + // Be selective in which variables we allow the child to inherit. + // Depending on the application, some may be necessary, + // while others (e.g. GOGC, GOMEMLIMIT) may be harmful; see #73490. + cmd.Env = []string{monitorVar + "=1"} cmd.Stderr = os.Stderr cmd.Stdout = os.Stderr pipe, err := cmd.StdinPipe() diff --git a/src/runtime/debug/garbage_test.go b/src/runtime/debug/garbage_test.go index cd91782d27c1ce..506f698ad72a8e 100644 --- a/src/runtime/debug/garbage_test.go +++ b/src/runtime/debug/garbage_test.go @@ -18,7 +18,7 @@ func TestReadGCStats(t *testing.T) { var stats GCStats var mstats runtime.MemStats - var min, max time.Duration + var minimum, maximum time.Duration // First ReadGCStats will allocate, second should not, // especially if we follow up with an explicit garbage collection. @@ -52,11 +52,11 @@ func TestReadGCStats(t *testing.T) { if dt != time.Duration(mstats.PauseNs[off]) { t.Errorf("stats.Pause[%d] = %d, want %d", i, dt, mstats.PauseNs[off]) } - if max < dt { - max = dt - } - if min > dt || i == 0 { - min = dt + maximum = max(maximum, dt) + if i == 0 { + minimum = dt + } else { + minimum = min(minimum, dt) } off = (off + len(mstats.PauseNs) - 1) % len(mstats.PauseNs) } @@ -64,8 +64,8 @@ func TestReadGCStats(t *testing.T) { q := stats.PauseQuantiles nq := len(q) - if q[0] != min || q[nq-1] != max { - t.Errorf("stats.PauseQuantiles = [%d, ..., %d], want [%d, ..., %d]", q[0], q[nq-1], min, max) + if q[0] != minimum || q[nq-1] != maximum { + t.Errorf("stats.PauseQuantiles = [%d, ..., %d], want [%d, ..., %d]", q[0], q[nq-1], minimum, maximum) } for i := 0; i < nq-1; i++ { diff --git a/src/runtime/debug/mod.go b/src/runtime/debug/mod.go index a4705605b859a1..34227d85448ace 100644 --- a/src/runtime/debug/mod.go +++ b/src/runtime/debug/mod.go @@ -41,29 +41,29 @@ func ReadBuildInfo() (info *BuildInfo, ok bool) { type BuildInfo struct { // GoVersion is the version of the Go toolchain that built the binary // (for example, "go1.19.2"). - GoVersion string + GoVersion string `json:",omitempty"` // Path is the package path of the main package for the binary // (for example, "golang.org/x/tools/cmd/stringer"). - Path string + Path string `json:",omitempty"` // Main describes the module that contains the main package for the binary. - Main Module + Main Module `json:""` // Deps describes all the dependency modules, both direct and indirect, // that contributed packages to the build of this binary. - Deps []*Module + Deps []*Module `json:",omitempty"` // Settings describes the build settings used to build the binary. - Settings []BuildSetting + Settings []BuildSetting `json:",omitempty"` } // A Module describes a single module included in a build. type Module struct { - Path string // module path - Version string // module version - Sum string // checksum - Replace *Module // replaced by this module + Path string `json:",omitempty"` // module path + Version string `json:",omitempty"` // module version + Sum string `json:",omitempty"` // checksum + Replace *Module `json:",omitempty"` // replaced by this module } // A BuildSetting is a key-value pair describing one setting that influenced a build. @@ -77,9 +77,11 @@ type Module struct { // - CGO_CPPFLAGS: the effective CGO_CPPFLAGS environment variable // - CGO_CXXFLAGS: the effective CGO_CXXFLAGS environment variable // - CGO_LDFLAGS: the effective CGO_LDFLAGS environment variable +// - DefaultGODEBUG: the effective GODEBUG settings // - GOARCH: the architecture target // - GOAMD64/GOARM/GO386/etc: the architecture feature level for GOARCH // - GOOS: the operating system target +// - GOFIPS140: the frozen FIPS 140-3 module version, if any // - vcs: the version control system for the source tree where the build ran // - vcs.revision: the revision identifier for the current commit or checkout // - vcs.time: the modification time associated with vcs.revision, in RFC3339 format @@ -87,8 +89,9 @@ type Module struct { type BuildSetting struct { // Key and Value describe the build setting. // Key must not contain an equals sign, space, tab, or newline. + Key string `json:",omitempty"` // Value must not contain newlines ('\n'). - Key, Value string + Value string `json:",omitempty"` } // quoteKey reports whether key is required to be quoted. @@ -101,6 +104,7 @@ func quoteValue(value string) bool { return strings.ContainsAny(value, " \t\r\n\"`") } +// String returns a string representation of a [BuildInfo]. func (bi *BuildInfo) String() string { buf := new(strings.Builder) if bi.GoVersion != "" { @@ -146,6 +150,12 @@ func (bi *BuildInfo) String() string { return buf.String() } +// ParseBuildInfo parses the string returned by [*BuildInfo.String], +// restoring the original BuildInfo, +// except that the GoVersion field is not set. +// Programs should normally not call this function, +// but instead call [ReadBuildInfo], [debug/buildinfo.ReadFile], +// or [debug/buildinfo.Read]. func ParseBuildInfo(data string) (bi *BuildInfo, err error) { lineNum := 1 defer func() { @@ -154,7 +164,7 @@ func ParseBuildInfo(data string) (bi *BuildInfo, err error) { } }() - var ( + const ( pathLine = "path\t" modLine = "mod\t" depLine = "dep\t" @@ -195,7 +205,7 @@ func ParseBuildInfo(data string) (bi *BuildInfo, err error) { switch { case strings.HasPrefix(line, pathLine): elem := line[len(pathLine):] - bi.Path = string(elem) + bi.Path = elem case strings.HasPrefix(line, modLine): elem := strings.Split(line[len(modLine):], tab) last = &bi.Main @@ -220,9 +230,9 @@ func ParseBuildInfo(data string) (bi *BuildInfo, err error) { return nil, fmt.Errorf("replacement with no module on previous line") } last.Replace = &Module{ - Path: string(elem[0]), - Version: string(elem[1]), - Sum: string(elem[2]), + Path: elem[0], + Version: elem[1], + Sum: elem[2], } last = nil case strings.HasPrefix(line, buildLine): diff --git a/src/runtime/debug/stack.go b/src/runtime/debug/stack.go index d7a860b7dc941b..c4c3be141c7b85 100644 --- a/src/runtime/debug/stack.go +++ b/src/runtime/debug/stack.go @@ -52,7 +52,7 @@ func SetCrashOutput(f *os.File, opts CrashOptions) error { // The runtime will write to this file descriptor from // low-level routines during a panic, possibly without // a G, so we must call f.Fd() eagerly. This creates a - // danger that that the file descriptor is no longer + // danger that the file descriptor is no longer // valid at the time of the write, because the caller // (incorrectly) called f.Close() and the kernel // reissued the fd in a later call to open(2), leading diff --git a/src/runtime/debug/stack_test.go b/src/runtime/debug/stack_test.go index e1559303f05fe0..88d4ad0c0fe757 100644 --- a/src/runtime/debug/stack_test.go +++ b/src/runtime/debug/stack_test.go @@ -87,12 +87,7 @@ func TestStack(t *testing.T) { // initial (not current) environment. Spawn a subprocess to determine the // real baked-in GOROOT. t.Logf("found GOROOT %q from environment; checking embedded GOROOT value", envGoroot) - testenv.MustHaveExec(t) - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - cmd := exec.Command(exe) + cmd := exec.Command(testenv.Executable(t)) cmd.Env = append(os.Environ(), "GOROOT=", "GO_RUNTIME_DEBUG_TEST_ENTRYPOINT=dumpgoroot") out, err := cmd.Output() if err != nil { @@ -137,18 +132,12 @@ func TestStack(t *testing.T) { } func TestSetCrashOutput(t *testing.T) { - testenv.MustHaveExec(t) - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - crashOutput := filepath.Join(t.TempDir(), "crash.out") - cmd := exec.Command(exe) + cmd := exec.Command(testenv.Executable(t)) cmd.Stderr = new(strings.Builder) cmd.Env = append(os.Environ(), "GO_RUNTIME_DEBUG_TEST_ENTRYPOINT=setcrashoutput", "CRASHOUTPUT="+crashOutput) - err = cmd.Run() + err := cmd.Run() stderr := fmt.Sprint(cmd.Stderr) if err == nil { t.Fatalf("child process succeeded unexpectedly (stderr: %s)", stderr) diff --git a/src/runtime/debug_test.go b/src/runtime/debug_test.go index 1c00d2fb0d918b..37093cd87e6c90 100644 --- a/src/runtime/debug_test.go +++ b/src/runtime/debug_test.go @@ -9,13 +9,15 @@ // spends all of its time in the race runtime, which isn't a safe // point. -//go:build (amd64 || arm64 || ppc64le) && linux && !race +//go:build (amd64 || arm64 || loong64 || ppc64le) && linux && !race package runtime_test import ( "fmt" "internal/abi" + "internal/asan" + "internal/msan" "math" "os" "regexp" @@ -32,6 +34,14 @@ func startDebugCallWorker(t *testing.T) (g *runtime.G, after func()) { // a debugger. skipUnderDebugger(t) + // asan/msan instrumentation interferes with tests since we might + // inject debugCallV2 while in the asan/msan runtime. This is a + // problem for doing things like running the GC or taking stack + // traces. Not sure why this is happening yet, but skip for now. + if msan.Enabled || asan.Enabled { + t.Skip("debugCallV2 is injected erroneously during asan/msan runtime calls; skipping") + } + // This can deadlock if there aren't enough threads or if a GC // tries to interrupt an atomic loop (see issue #10958). Execute // an extra GC to ensure even the sweep phase is done (out of diff --git a/src/runtime/debugcall.go b/src/runtime/debugcall.go index fee4116aa5e707..e6554475c094b4 100644 --- a/src/runtime/debugcall.go +++ b/src/runtime/debugcall.go @@ -5,12 +5,13 @@ // Though the debug call function feature is not enabled on // ppc64, inserted ppc64 to avoid missing Go declaration error // for debugCallPanicked while building runtime.test -//go:build amd64 || arm64 || ppc64le || ppc64 +//go:build amd64 || arm64 || loong64 || ppc64le || ppc64 package runtime import ( "internal/abi" + "internal/runtime/sys" "unsafe" ) @@ -34,7 +35,7 @@ func debugCallCheck(pc uintptr) string { if getg() != getg().m.curg { return debugCallSystemStack } - if sp := getcallersp(); !(getg().stack.lo < sp && sp <= getg().stack.hi) { + if sp := sys.GetCallerSP(); !(getg().stack.lo < sp && sp <= getg().stack.hi) { // Fast syscalls (nanotime) and racecall switch to the // g0 stack without switching g. We can't safely make // a call in this state. (We can't even safely @@ -106,7 +107,7 @@ func debugCallCheck(pc uintptr) string { //go:nosplit func debugCallWrap(dispatch uintptr) { var lockedExt uint32 - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() gp := getg() // Lock ourselves to the OS thread. diff --git a/src/runtime/debuglog.go b/src/runtime/debuglog.go index 25186af7e04252..e993e396c14e9f 100644 --- a/src/runtime/debuglog.go +++ b/src/runtime/debuglog.go @@ -12,11 +12,22 @@ // // This facility can be enabled by passing -tags debuglog when // building. Without this tag, dlog calls compile to nothing. +// +// Implementation notes +// +// There are two implementations of the dlog interface: dloggerImpl and +// dloggerFake. dloggerFake is a no-op implementation. dlogger is type-aliased +// to one or the other depending on the debuglog build tag. However, both types +// always exist and are always built. This helps ensure we compile as much of +// the implementation as possible in the default build configuration, while also +// enabling us to achieve good test coverage of the real debuglog implementation +// even when the debuglog build tag is not set. package runtime import ( "internal/abi" + "internal/byteorder" "internal/runtime/atomic" "internal/runtime/sys" "unsafe" @@ -48,11 +59,20 @@ const debugLogStringLimit = debugLogBytes / 8 // //go:nosplit //go:nowritebarrierrec -func dlog() *dlogger { - if !dlogEnabled { - return nil - } +func dlog() dlogger { + // dlog1 is defined to either dlogImpl or dlogFake. + return dlog1() +} + +//go:nosplit +//go:nowritebarrierrec +func dlogFake() dloggerFake { + return dloggerFake{} +} +//go:nosplit +//go:nowritebarrierrec +func dlogImpl() *dloggerImpl { // Get the time. tick, nano := uint64(cputicks()), uint64(nanotime()) @@ -63,7 +83,7 @@ func dlog() *dlogger { // global pool. if l == nil { allp := (*uintptr)(unsafe.Pointer(&allDloggers)) - all := (*dlogger)(unsafe.Pointer(atomic.Loaduintptr(allp))) + all := (*dloggerImpl)(unsafe.Pointer(atomic.Loaduintptr(allp))) for l1 := all; l1 != nil; l1 = l1.allLink { if l1.owned.Load() == 0 && l1.owned.CompareAndSwap(0, 1) { l = l1 @@ -76,7 +96,7 @@ func dlog() *dlogger { if l == nil { // Use sysAllocOS instead of sysAlloc because we want to interfere // with the runtime as little as possible, and sysAlloc updates accounting. - l = (*dlogger)(sysAllocOS(unsafe.Sizeof(dlogger{}))) + l = (*dloggerImpl)(sysAllocOS(unsafe.Sizeof(dloggerImpl{}), "debug log")) if l == nil { throw("failed to allocate debug log") } @@ -87,7 +107,7 @@ func dlog() *dlogger { headp := (*uintptr)(unsafe.Pointer(&allDloggers)) for { head := atomic.Loaduintptr(headp) - l.allLink = (*dlogger)(unsafe.Pointer(head)) + l.allLink = (*dloggerImpl)(unsafe.Pointer(head)) if atomic.Casuintptr(headp, head, uintptr(unsafe.Pointer(l))) { break } @@ -119,16 +139,16 @@ func dlog() *dlogger { return l } -// A dlogger writes to the debug log. +// A dloggerImpl writes to the debug log. // -// To obtain a dlogger, call dlog(). When done with the dlogger, call +// To obtain a dloggerImpl, call dlog(). When done with the dloggerImpl, call // end(). -type dlogger struct { +type dloggerImpl struct { _ sys.NotInHeap w debugLogWriter // allLink is the next dlogger in the allDloggers list. - allLink *dlogger + allLink *dloggerImpl // owned indicates that this dlogger is owned by an M. This is // accessed atomically. @@ -138,14 +158,16 @@ type dlogger struct { // allDloggers is a list of all dloggers, linked through // dlogger.allLink. This is accessed atomically. This is prepend only, // so it doesn't need to protect against ABA races. -var allDloggers *dlogger +var allDloggers *dloggerImpl + +// A dloggerFake is a no-op implementation of dlogger. +type dloggerFake struct{} //go:nosplit -func (l *dlogger) end() { - if !dlogEnabled { - return - } +func (l dloggerFake) end() {} +//go:nosplit +func (l *dloggerImpl) end() { // Fill in framing header. size := l.w.write - l.w.r.end if !l.w.writeFrameAt(l.w.r.end, size) { @@ -181,10 +203,10 @@ const ( ) //go:nosplit -func (l *dlogger) b(x bool) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) b(x bool) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) b(x bool) *dloggerImpl { if x { l.w.byte(debugLogBoolTrue) } else { @@ -194,91 +216,118 @@ func (l *dlogger) b(x bool) *dlogger { } //go:nosplit -func (l *dlogger) i(x int) *dlogger { +func (l dloggerFake) i(x int) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) i(x int) *dloggerImpl { return l.i64(int64(x)) } //go:nosplit -func (l *dlogger) i8(x int8) *dlogger { +func (l dloggerFake) i8(x int8) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) i8(x int8) *dloggerImpl { return l.i64(int64(x)) } //go:nosplit -func (l *dlogger) i16(x int16) *dlogger { +func (l dloggerFake) i16(x int16) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) i16(x int16) *dloggerImpl { return l.i64(int64(x)) } //go:nosplit -func (l *dlogger) i32(x int32) *dlogger { +func (l dloggerFake) i32(x int32) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) i32(x int32) *dloggerImpl { return l.i64(int64(x)) } //go:nosplit -func (l *dlogger) i64(x int64) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) i64(x int64) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) i64(x int64) *dloggerImpl { l.w.byte(debugLogInt) l.w.varint(x) return l } //go:nosplit -func (l *dlogger) u(x uint) *dlogger { +func (l dloggerFake) u(x uint) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) u(x uint) *dloggerImpl { return l.u64(uint64(x)) } //go:nosplit -func (l *dlogger) uptr(x uintptr) *dlogger { +func (l dloggerFake) uptr(x uintptr) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) uptr(x uintptr) *dloggerImpl { return l.u64(uint64(x)) } //go:nosplit -func (l *dlogger) u8(x uint8) *dlogger { +func (l dloggerFake) u8(x uint8) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) u8(x uint8) *dloggerImpl { return l.u64(uint64(x)) } //go:nosplit -func (l *dlogger) u16(x uint16) *dlogger { +func (l dloggerFake) u16(x uint16) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) u16(x uint16) *dloggerImpl { return l.u64(uint64(x)) } //go:nosplit -func (l *dlogger) u32(x uint32) *dlogger { +func (l dloggerFake) u32(x uint32) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) u32(x uint32) *dloggerImpl { return l.u64(uint64(x)) } //go:nosplit -func (l *dlogger) u64(x uint64) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) u64(x uint64) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) u64(x uint64) *dloggerImpl { l.w.byte(debugLogUint) l.w.uvarint(x) return l } //go:nosplit -func (l *dlogger) hex(x uint64) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) hex(x uint64) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) hex(x uint64) *dloggerImpl { l.w.byte(debugLogHex) l.w.uvarint(x) return l } //go:nosplit -func (l *dlogger) p(x any) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) p(x any) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) p(x any) *dloggerImpl { l.w.byte(debugLogPtr) if x == nil { l.w.uvarint(0) } else { v := efaceOf(&x) - switch v._type.Kind_ & abi.KindMask { + switch v._type.Kind() { case abi.Chan, abi.Func, abi.Map, abi.Pointer, abi.UnsafePointer: l.w.uvarint(uint64(uintptr(v.data))) default: @@ -289,11 +338,10 @@ func (l *dlogger) p(x any) *dlogger { } //go:nosplit -func (l *dlogger) s(x string) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) s(x string) dloggerFake { return l } +//go:nosplit +func (l *dloggerImpl) s(x string) *dloggerImpl { strData := unsafe.StringData(x) datap := &firstmoduledata if len(x) > 4 && datap.etext <= uintptr(unsafe.Pointer(strData)) && uintptr(unsafe.Pointer(strData)) < datap.end { @@ -325,20 +373,20 @@ func (l *dlogger) s(x string) *dlogger { } //go:nosplit -func (l *dlogger) pc(x uintptr) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) pc(x uintptr) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) pc(x uintptr) *dloggerImpl { l.w.byte(debugLogPC) l.w.uvarint(uint64(x)) return l } //go:nosplit -func (l *dlogger) traceback(x []uintptr) *dlogger { - if !dlogEnabled { - return l - } +func (l dloggerFake) traceback(x []uintptr) dloggerFake { return l } + +//go:nosplit +func (l *dloggerImpl) traceback(x []uintptr) *dloggerImpl { l.w.byte(debugLogTraceback) l.w.uvarint(uint64(len(x))) for _, pc := range x { @@ -430,14 +478,7 @@ func (l *debugLogWriter) writeSync(tick, nano uint64) { //go:nosplit func (l *debugLogWriter) writeUint64LE(x uint64) { var b [8]byte - b[0] = byte(x) - b[1] = byte(x >> 8) - b[2] = byte(x >> 16) - b[3] = byte(x >> 24) - b[4] = byte(x >> 32) - b[5] = byte(x >> 40) - b[6] = byte(x >> 48) - b[7] = byte(x >> 56) + byteorder.LEPutUint64(b[:], x) l.bytes(b[:]) } @@ -529,10 +570,7 @@ func (r *debugLogReader) readUint64LEAt(pos uint64) uint64 { b[i] = r.data.b[pos%uint64(len(r.data.b))] pos++ } - return uint64(b[0]) | uint64(b[1])<<8 | - uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | - uint64(b[6])<<48 | uint64(b[7])<<56 + return byteorder.LEUint64(b[:]) } func (r *debugLogReader) peek() (tick uint64) { @@ -693,10 +731,12 @@ func (r *debugLogReader) printVal() bool { // printDebugLog prints the debug log. func printDebugLog() { - if !dlogEnabled { - return + if dlogEnabled { + printDebugLogImpl() } +} +func printDebugLogImpl() { // This function should not panic or throw since it is used in // the fatal panic path and this may deadlock. @@ -704,7 +744,7 @@ func printDebugLog() { // Get the list of all debug logs. allp := (*uintptr)(unsafe.Pointer(&allDloggers)) - all := (*dlogger)(unsafe.Pointer(atomic.Loaduintptr(allp))) + all := (*dloggerImpl)(unsafe.Pointer(atomic.Loaduintptr(allp))) // Count the logs. n := 0 @@ -725,7 +765,7 @@ func printDebugLog() { } // Use sysAllocOS instead of sysAlloc because we want to interfere // with the runtime as little as possible, and sysAlloc updates accounting. - state1 := sysAllocOS(unsafe.Sizeof(readState{}) * uintptr(n)) + state1 := sysAllocOS(unsafe.Sizeof(readState{})*uintptr(n), "debug log") if state1 == nil { println("failed to allocate read state for", n, "logs") printunlock() diff --git a/src/runtime/debuglog_off.go b/src/runtime/debuglog_off.go index fa3be39c70f834..4eb59fa683e301 100644 --- a/src/runtime/debuglog_off.go +++ b/src/runtime/debuglog_off.go @@ -8,12 +8,18 @@ package runtime const dlogEnabled = false +type dlogger = dloggerFake + +func dlog1() dloggerFake { + return dlogFake() +} + type dlogPerM struct{} -func getCachedDlogger() *dlogger { +func getCachedDlogger() *dloggerImpl { return nil } -func putCachedDlogger(l *dlogger) bool { +func putCachedDlogger(l *dloggerImpl) bool { return false } diff --git a/src/runtime/debuglog_on.go b/src/runtime/debuglog_on.go index b8150202251b36..99773129ab19d0 100644 --- a/src/runtime/debuglog_on.go +++ b/src/runtime/debuglog_on.go @@ -8,21 +8,32 @@ package runtime const dlogEnabled = true +// dlogger is the underlying implementation of the dlogger interface, selected +// at build time. +// +// We use a type alias instead of struct embedding so that the dlogger type is +// identical to the type returned by method chaining on the methods of this type. +type dlogger = *dloggerImpl + +func dlog1() *dloggerImpl { + return dlogImpl() +} + // dlogPerM is the per-M debug log data. This is embedded in the m // struct. type dlogPerM struct { - dlogCache *dlogger + dlogCache *dloggerImpl } // getCachedDlogger returns a cached dlogger if it can do so // efficiently, or nil otherwise. The returned dlogger will be owned. -func getCachedDlogger() *dlogger { +func getCachedDlogger() *dloggerImpl { mp := acquirem() // We don't return a cached dlogger if we're running on the // signal stack in case the signal arrived while in // get/putCachedDlogger. (Too bad we don't have non-atomic // exchange!) - var l *dlogger + var l *dloggerImpl if getg() != mp.gsignal { l = mp.dlogCache mp.dlogCache = nil @@ -33,7 +44,7 @@ func getCachedDlogger() *dlogger { // putCachedDlogger attempts to return l to the local cache. It // returns false if this fails. -func putCachedDlogger(l *dlogger) bool { +func putCachedDlogger(l *dloggerImpl) bool { mp := acquirem() if getg() != mp.gsignal && mp.dlogCache == nil { mp.dlogCache = l diff --git a/src/runtime/debuglog_test.go b/src/runtime/debuglog_test.go index 18c54a81b93642..d958c037400ff3 100644 --- a/src/runtime/debuglog_test.go +++ b/src/runtime/debuglog_test.go @@ -24,18 +24,16 @@ package runtime_test import ( "fmt" - "internal/testenv" "regexp" "runtime" "strings" "sync" - "sync/atomic" "testing" ) func skipDebugLog(t *testing.T) { - if !runtime.DlogEnabled { - t.Skip("debug log disabled (rebuild with -tags debuglog)") + if runtime.DlogEnabled { + t.Skip("debug log tests disabled to avoid collisions with real debug logs") } } @@ -83,28 +81,63 @@ func TestDebugLogSym(t *testing.T) { func TestDebugLogInterleaving(t *testing.T) { skipDebugLog(t) runtime.ResetDebugLog() + + n1 := runtime.CountDebugLog() + t.Logf("number of log shards at start: %d", n1) + + const limit = 1000 + const concurrency = 10 + + // Start several goroutines writing to the log simultaneously. var wg sync.WaitGroup - done := int32(0) - wg.Add(1) - go func() { - // Encourage main goroutine to move around to - // different Ms and Ps. - for atomic.LoadInt32(&done) == 0 { - runtime.Gosched() - } - wg.Done() - }() - var want strings.Builder - for i := 0; i < 1000; i++ { - runtime.Dlog().I(i).End() - fmt.Fprintf(&want, "[] %d\n", i) - runtime.Gosched() + i := 0 + chans := make([]chan bool, concurrency) + for gid := range concurrency { + chans[gid] = make(chan bool) + wg.Add(1) + go func() { + defer wg.Done() + var log *runtime.Dlogger + for { + <-chans[gid] + if log != nil { + log.End() + } + next := chans[(gid+1)%len(chans)] + if i >= limit { + close(next) + break + } + // Log an entry, but *don't* release the log shard until its our + // turn again. This should result in at least n=concurrency log + // shards. + log = runtime.Dlog().I(i) + i++ + // Wake up the next logger goroutine. + next <- true + } + }() } - atomic.StoreInt32(&done, 1) - wg.Wait() + // Start the chain reaction. + chans[0] <- true + // Wait for them to finish and get the log. + wg.Wait() gotFull := runtime.DumpDebugLog() got := dlogCanonicalize(gotFull) + + n2 := runtime.CountDebugLog() + t.Logf("number of log shards at end: %d", n2) + if n2 < concurrency { + t.Errorf("created %d log shards, expected >= %d", n2, concurrency) + } + + // Construct the desired output. + var want strings.Builder + for i := 0; i < limit; i++ { + fmt.Fprintf(&want, "[] %d\n", i) + } + if got != want.String() { // Since the timestamps are useful in understand // failures of this test, we print the uncanonicalized @@ -156,14 +189,3 @@ func TestDebugLogLongString(t *testing.T) { t.Fatalf("want %q, got %q", want, got) } } - -// TestDebugLogBuild verifies that the runtime builds with -tags=debuglog. -func TestDebugLogBuild(t *testing.T) { - testenv.MustHaveGoBuild(t) - - // It doesn't matter which program we build, anything will rebuild the - // runtime. - if _, err := buildTestProg(t, "testprog", "-tags=debuglog"); err != nil { - t.Fatal(err) - } -} diff --git a/src/runtime/decoratemappings_test.go b/src/runtime/decoratemappings_test.go new file mode 100644 index 00000000000000..7d1121c125df08 --- /dev/null +++ b/src/runtime/decoratemappings_test.go @@ -0,0 +1,72 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "os" + "regexp" + "runtime" + "testing" +) + +func validateMapLabels(t *testing.T, labels []string) { + // These are the specific region labels that need get added during the + // runtime phase. Hence they are the ones that need to be confirmed as + // present at the time the test reads its own region labels, which + // is sufficient to validate that the default `decoratemappings` value + // (enabled) was set early enough in the init process. + regions := map[string]bool{ + "allspans array": false, + "gc bits": false, + "heap": false, + "heap index": false, + "heap reservation": false, + "immortal metadata": false, + "page alloc": false, + "page alloc index": false, + "page summary": false, + "scavenge index": false, + } + for _, label := range labels { + if _, ok := regions[label]; !ok { + t.Logf("unexpected region label found: \"%s\"", label) + } + regions[label] = true + } + for label, found := range regions { + if !found { + t.Logf("region label missing: \"%s\"", label) + } + } +} + +func TestDecorateMappings(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("decoratemappings is only supported on Linux") + // /proc/self/maps is also Linux-specific + } + + var labels []string + if rawMaps, err := os.ReadFile("/proc/self/maps"); err != nil { + t.Fatalf("failed to read /proc/self/maps: %v", err) + } else { + t.Logf("maps:%s\n", string(rawMaps)) + matches := regexp.MustCompile("[^[]+ \\[anon: Go: (.+)\\]\n").FindAllSubmatch(rawMaps, -1) + for _, match_pair := range matches { + // match_pair consists of the matching substring and the parenthesized group + labels = append(labels, string(match_pair[1])) + } + } + t.Logf("DebugDecorateMappings: %v", *runtime.DebugDecorateMappings) + if *runtime.DebugDecorateMappings != 0 && runtime.SetVMANameSupported() { + validateMapLabels(t, labels) + } else { + if len(labels) > 0 { + t.Errorf("unexpected mapping labels present: %v", labels) + } else { + t.Skip("mapping labels absent as expected") + } + } +} diff --git a/src/runtime/defs2_linux.go b/src/runtime/defs2_linux.go index 5d6730a7ad7fd8..597073c39d3d78 100644 --- a/src/runtime/defs2_linux.go +++ b/src/runtime/defs2_linux.go @@ -48,6 +48,7 @@ const ( EINTR = C.EINTR EAGAIN = C.EAGAIN ENOMEM = C.ENOMEM + ENOSYS = C.ENOSYS PROT_NONE = C.PROT_NONE PROT_READ = C.PROT_READ diff --git a/src/runtime/defs_linux.go b/src/runtime/defs_linux.go index 296fcb4bfd6a7a..d2b619ecabc0e8 100644 --- a/src/runtime/defs_linux.go +++ b/src/runtime/defs_linux.go @@ -37,6 +37,7 @@ const ( EINTR = C.EINTR EAGAIN = C.EAGAIN ENOMEM = C.ENOMEM + ENOSYS = C.ENOSYS PROT_NONE = C.PROT_NONE PROT_READ = C.PROT_READ diff --git a/src/runtime/defs_linux_386.go b/src/runtime/defs_linux_386.go index 5fef55610f39d2..c6c7d7d6d880c0 100644 --- a/src/runtime/defs_linux_386.go +++ b/src/runtime/defs_linux_386.go @@ -9,6 +9,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 @@ -136,16 +137,30 @@ type fpstate struct { anon0 [48]byte } -type timespec struct { +// The timespec structs and types are defined in Linux in +// include/uapi/linux/time_types.h and include/uapi/asm-generic/posix_types.h. +type timespec32 struct { tv_sec int32 tv_nsec int32 } //go:nosplit -func (ts *timespec) setNsec(ns int64) { +func (ts *timespec32) setNsec(ns int64) { ts.tv_sec = timediv(ns, 1e9, &ts.tv_nsec) } +type timespec struct { + tv_sec int64 + tv_nsec int64 +} + +//go:nosplit +func (ts *timespec) setNsec(ns int64) { + var newNS int32 + ts.tv_sec = int64(timediv(ns, 1e9, &newNS)) + ts.tv_nsec = int64(newNS) +} + type timeval struct { tv_sec int32 tv_usec int32 @@ -222,6 +237,10 @@ type ucontext struct { uc_sigmask uint32 } +type itimerspec32 struct { + it_interval timespec32 + it_value timespec32 +} type itimerspec struct { it_interval timespec it_value timespec diff --git a/src/runtime/defs_linux_amd64.go b/src/runtime/defs_linux_amd64.go index dce7799b6adc9c..9a908c94000a58 100644 --- a/src/runtime/defs_linux_amd64.go +++ b/src/runtime/defs_linux_amd64.go @@ -9,6 +9,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 diff --git a/src/runtime/defs_linux_arm.go b/src/runtime/defs_linux_arm.go index 71cf8c6d50e760..ff879fad89faa1 100644 --- a/src/runtime/defs_linux_arm.go +++ b/src/runtime/defs_linux_arm.go @@ -11,6 +11,7 @@ const ( _EINTR = 0x4 _ENOMEM = 0xc _EAGAIN = 0xb + _ENOSYS = 0x26 _PROT_NONE = 0 _PROT_READ = 0x1 @@ -95,16 +96,30 @@ const ( _SOCK_DGRAM = 0x2 ) -type timespec struct { +// The timespec structs and types are defined in Linux in +// include/uapi/linux/time_types.h and include/uapi/asm-generic/posix_types.h. +type timespec32 struct { tv_sec int32 tv_nsec int32 } //go:nosplit -func (ts *timespec) setNsec(ns int64) { +func (ts *timespec32) setNsec(ns int64) { ts.tv_sec = timediv(ns, 1e9, &ts.tv_nsec) } +type timespec struct { + tv_sec int64 + tv_nsec int64 +} + +//go:nosplit +func (ts *timespec) setNsec(ns int64) { + var newNS int32 + ts.tv_sec = int64(timediv(ns, 1e9, &newNS)) + ts.tv_nsec = int64(newNS) +} + type stackt struct { ss_sp *byte ss_flags int32 @@ -154,6 +169,11 @@ func (tv *timeval) set_usec(x int32) { tv.tv_usec = x } +type itimerspec32 struct { + it_interval timespec32 + it_value timespec32 +} + type itimerspec struct { it_interval timespec it_value timespec diff --git a/src/runtime/defs_linux_arm64.go b/src/runtime/defs_linux_arm64.go index 606cd70494e37a..4992e91ea68b8a 100644 --- a/src/runtime/defs_linux_arm64.go +++ b/src/runtime/defs_linux_arm64.go @@ -9,6 +9,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 diff --git a/src/runtime/defs_linux_loong64.go b/src/runtime/defs_linux_loong64.go index 692d8c78e9a4d2..670d4c318dd6cc 100644 --- a/src/runtime/defs_linux_loong64.go +++ b/src/runtime/defs_linux_loong64.go @@ -10,6 +10,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 @@ -184,6 +185,7 @@ type sigcontext struct { sc_pc uint64 sc_regs [32]uint64 sc_flags uint32 + sc_pad0 [1]uint32 sc_extcontext [0]uint64 } diff --git a/src/runtime/defs_linux_mips64x.go b/src/runtime/defs_linux_mips64x.go index 8a0af41234f226..7449d2cfac96d7 100644 --- a/src/runtime/defs_linux_mips64x.go +++ b/src/runtime/defs_linux_mips64x.go @@ -12,6 +12,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 diff --git a/src/runtime/defs_linux_mipsx.go b/src/runtime/defs_linux_mipsx.go index 8322beab2b1c20..5b10b910dbebf2 100644 --- a/src/runtime/defs_linux_mipsx.go +++ b/src/runtime/defs_linux_mipsx.go @@ -12,6 +12,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 @@ -93,16 +94,30 @@ const ( _SIGEV_THREAD_ID = 0x4 ) -type timespec struct { +// The timespec structs and types are defined in Linux in +// include/uapi/linux/time_types.h and include/uapi/asm-generic/posix_types.h. +type timespec32 struct { tv_sec int32 tv_nsec int32 } //go:nosplit -func (ts *timespec) setNsec(ns int64) { +func (ts *timespec32) setNsec(ns int64) { ts.tv_sec = timediv(ns, 1e9, &ts.tv_nsec) } +type timespec struct { + tv_sec int64 + tv_nsec int64 +} + +//go:nosplit +func (ts *timespec) setNsec(ns int64) { + var newNS int32 + ts.tv_sec = int64(timediv(ns, 1e9, &newNS)) + ts.tv_nsec = int64(newNS) +} + type timeval struct { tv_sec int32 tv_usec int32 @@ -137,6 +152,11 @@ type siginfo struct { _ [_si_max_size - unsafe.Sizeof(siginfoFields{})]byte } +type itimerspec32 struct { + it_interval timespec32 + it_value timespec32 +} + type itimerspec struct { it_interval timespec it_value timespec diff --git a/src/runtime/defs_linux_ppc64.go b/src/runtime/defs_linux_ppc64.go index f87924affe65ef..dc45f37b7c3b25 100644 --- a/src/runtime/defs_linux_ppc64.go +++ b/src/runtime/defs_linux_ppc64.go @@ -9,6 +9,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 diff --git a/src/runtime/defs_linux_ppc64le.go b/src/runtime/defs_linux_ppc64le.go index f87924affe65ef..dc45f37b7c3b25 100644 --- a/src/runtime/defs_linux_ppc64le.go +++ b/src/runtime/defs_linux_ppc64le.go @@ -9,6 +9,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 diff --git a/src/runtime/defs_linux_riscv64.go b/src/runtime/defs_linux_riscv64.go index 29b1ef2a50a94c..b73e208ac3491e 100644 --- a/src/runtime/defs_linux_riscv64.go +++ b/src/runtime/defs_linux_riscv64.go @@ -10,6 +10,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 diff --git a/src/runtime/defs_linux_s390x.go b/src/runtime/defs_linux_s390x.go index b0280213b3d12f..c03d0f21171a61 100644 --- a/src/runtime/defs_linux_s390x.go +++ b/src/runtime/defs_linux_s390x.go @@ -10,6 +10,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 diff --git a/src/runtime/defs_openbsd.go b/src/runtime/defs_openbsd.go index d93c087a8194b1..9564a3354cd832 100644 --- a/src/runtime/defs_openbsd.go +++ b/src/runtime/defs_openbsd.go @@ -11,7 +11,6 @@ GOARCH=amd64 go tool cgo -godefs defs_openbsd.go GOARCH=386 go tool cgo -godefs defs_openbsd.go GOARCH=arm go tool cgo -godefs defs_openbsd.go GOARCH=arm64 go tool cgo -godefs defs_openbsd.go -GOARCH=mips64 go tool cgo -godefs defs_openbsd.go */ package runtime diff --git a/src/runtime/defs_openbsd_mips64.go b/src/runtime/defs_openbsd_mips64.go deleted file mode 100644 index 7958044d0422d8..00000000000000 --- a/src/runtime/defs_openbsd_mips64.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Generated from: -// -// GOARCH=mips64 go tool cgo -godefs defs_openbsd.go -// -// Then converted to the form used by the runtime. - -package runtime - -import "unsafe" - -const ( - _EINTR = 0x4 - _EFAULT = 0xe - _EAGAIN = 0x23 - _ETIMEDOUT = 0x3c - - _O_WRONLY = 0x1 - _O_NONBLOCK = 0x4 - _O_CREAT = 0x200 - _O_TRUNC = 0x400 - _O_CLOEXEC = 0x10000 - - _PROT_NONE = 0x0 - _PROT_READ = 0x1 - _PROT_WRITE = 0x2 - _PROT_EXEC = 0x4 - - _MAP_ANON = 0x1000 - _MAP_PRIVATE = 0x2 - _MAP_FIXED = 0x10 - _MAP_STACK = 0x4000 - - _MADV_DONTNEED = 0x4 - _MADV_FREE = 0x6 - - _SA_SIGINFO = 0x40 - _SA_RESTART = 0x2 - _SA_ONSTACK = 0x1 - - _SIGHUP = 0x1 - _SIGINT = 0x2 - _SIGQUIT = 0x3 - _SIGILL = 0x4 - _SIGTRAP = 0x5 - _SIGABRT = 0x6 - _SIGEMT = 0x7 - _SIGFPE = 0x8 - _SIGKILL = 0x9 - _SIGBUS = 0xa - _SIGSEGV = 0xb - _SIGSYS = 0xc - _SIGPIPE = 0xd - _SIGALRM = 0xe - _SIGTERM = 0xf - _SIGURG = 0x10 - _SIGSTOP = 0x11 - _SIGTSTP = 0x12 - _SIGCONT = 0x13 - _SIGCHLD = 0x14 - _SIGTTIN = 0x15 - _SIGTTOU = 0x16 - _SIGIO = 0x17 - _SIGXCPU = 0x18 - _SIGXFSZ = 0x19 - _SIGVTALRM = 0x1a - _SIGPROF = 0x1b - _SIGWINCH = 0x1c - _SIGINFO = 0x1d - _SIGUSR1 = 0x1e - _SIGUSR2 = 0x1f - - _FPE_INTDIV = 0x1 - _FPE_INTOVF = 0x2 - _FPE_FLTDIV = 0x3 - _FPE_FLTOVF = 0x4 - _FPE_FLTUND = 0x5 - _FPE_FLTRES = 0x6 - _FPE_FLTINV = 0x7 - _FPE_FLTSUB = 0x8 - - _BUS_ADRALN = 0x1 - _BUS_ADRERR = 0x2 - _BUS_OBJERR = 0x3 - - _SEGV_MAPERR = 0x1 - _SEGV_ACCERR = 0x2 - - _ITIMER_REAL = 0x0 - _ITIMER_VIRTUAL = 0x1 - _ITIMER_PROF = 0x2 - - _EV_ADD = 0x1 - _EV_DELETE = 0x2 - _EV_CLEAR = 0x20 - _EV_ERROR = 0x4000 - _EV_EOF = 0x8000 - _EVFILT_READ = -0x1 - _EVFILT_WRITE = -0x2 -) - -type tforkt struct { - tf_tcb unsafe.Pointer - tf_tid *int32 - tf_stack uintptr -} - -type sigcontext struct { - sc_cookie uint64 - sc_mask uint64 - sc_pc uint64 - sc_regs [32]uint64 - mullo uint64 - mulhi uint64 - sc_fpregs [33]uint64 - sc_fpused uint64 - sc_fpc_eir uint64 - _xxx [8]int64 -} - -type siginfo struct { - si_signo int32 - si_code int32 - si_errno int32 - pad_cgo_0 [4]byte - _data [120]byte -} - -type stackt struct { - ss_sp uintptr - ss_size uintptr - ss_flags int32 - pad_cgo_0 [4]byte -} - -type timespec struct { - tv_sec int64 - tv_nsec int64 -} - -//go:nosplit -func (ts *timespec) setNsec(ns int64) { - ts.tv_sec = ns / 1e9 - ts.tv_nsec = ns % 1e9 -} - -type timeval struct { - tv_sec int64 - tv_usec int64 -} - -func (tv *timeval) set_usec(x int32) { - tv.tv_usec = int64(x) -} - -type itimerval struct { - it_interval timeval - it_value timeval -} - -type keventt struct { - ident uint64 - filter int16 - flags uint16 - fflags uint32 - data int64 - udata *byte -} diff --git a/src/runtime/defs_windows.go b/src/runtime/defs_windows.go deleted file mode 100644 index 2f09afbe1fef4c..00000000000000 --- a/src/runtime/defs_windows.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Windows architecture-independent definitions. - -package runtime - -const ( - _PROT_NONE = 0 - _PROT_READ = 1 - _PROT_WRITE = 2 - _PROT_EXEC = 4 - - _MAP_ANON = 1 - _MAP_PRIVATE = 2 - - _DUPLICATE_SAME_ACCESS = 0x2 - _THREAD_PRIORITY_HIGHEST = 0x2 - - _SIGINT = 0x2 - _SIGTERM = 0xF - _CTRL_C_EVENT = 0x0 - _CTRL_BREAK_EVENT = 0x1 - _CTRL_CLOSE_EVENT = 0x2 - _CTRL_LOGOFF_EVENT = 0x5 - _CTRL_SHUTDOWN_EVENT = 0x6 - - _EXCEPTION_ACCESS_VIOLATION = 0xc0000005 - _EXCEPTION_IN_PAGE_ERROR = 0xc0000006 - _EXCEPTION_BREAKPOINT = 0x80000003 - _EXCEPTION_ILLEGAL_INSTRUCTION = 0xc000001d - _EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d - _EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e - _EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f - _EXCEPTION_FLT_OVERFLOW = 0xc0000091 - _EXCEPTION_FLT_UNDERFLOW = 0xc0000093 - _EXCEPTION_INT_DIVIDE_BY_ZERO = 0xc0000094 - _EXCEPTION_INT_OVERFLOW = 0xc0000095 - - _INFINITE = 0xffffffff - _WAIT_TIMEOUT = 0x102 - - _EXCEPTION_CONTINUE_EXECUTION = -0x1 - _EXCEPTION_CONTINUE_SEARCH = 0x0 - _EXCEPTION_CONTINUE_SEARCH_SEH = 0x1 -) - -type systeminfo struct { - anon0 [4]byte - dwpagesize uint32 - lpminimumapplicationaddress *byte - lpmaximumapplicationaddress *byte - dwactiveprocessormask uintptr - dwnumberofprocessors uint32 - dwprocessortype uint32 - dwallocationgranularity uint32 - wprocessorlevel uint16 - wprocessorrevision uint16 -} - -type exceptionpointers struct { - record *exceptionrecord - context *context -} - -type exceptionrecord struct { - exceptioncode uint32 - exceptionflags uint32 - exceptionrecord *exceptionrecord - exceptionaddress uintptr - numberparameters uint32 - exceptioninformation [15]uintptr -} - -type overlapped struct { - internal uintptr - internalhigh uintptr - anon0 [8]byte - hevent *byte -} - -type memoryBasicInformation struct { - baseAddress uintptr - allocationBase uintptr - allocationProtect uint32 - regionSize uintptr - state uint32 - protect uint32 - type_ uint32 -} - -// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_osversioninfow -type _OSVERSIONINFOW struct { - osVersionInfoSize uint32 - majorVersion uint32 - minorVersion uint32 - buildNumber uint32 - platformId uint32 - csdVersion [128]uint16 -} diff --git a/src/runtime/defs_windows_386.go b/src/runtime/defs_windows_386.go deleted file mode 100644 index 8cf2bfc307f0de..00000000000000 --- a/src/runtime/defs_windows_386.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -const _CONTEXT_CONTROL = 0x10001 - -type floatingsavearea struct { - controlword uint32 - statusword uint32 - tagword uint32 - erroroffset uint32 - errorselector uint32 - dataoffset uint32 - dataselector uint32 - registerarea [80]uint8 - cr0npxstate uint32 -} - -type context struct { - contextflags uint32 - dr0 uint32 - dr1 uint32 - dr2 uint32 - dr3 uint32 - dr6 uint32 - dr7 uint32 - floatsave floatingsavearea - seggs uint32 - segfs uint32 - seges uint32 - segds uint32 - edi uint32 - esi uint32 - ebx uint32 - edx uint32 - ecx uint32 - eax uint32 - ebp uint32 - eip uint32 - segcs uint32 - eflags uint32 - esp uint32 - segss uint32 - extendedregisters [512]uint8 -} - -func (c *context) ip() uintptr { return uintptr(c.eip) } -func (c *context) sp() uintptr { return uintptr(c.esp) } - -// 386 does not have link register, so this returns 0. -func (c *context) lr() uintptr { return 0 } -func (c *context) set_lr(x uintptr) {} - -func (c *context) set_ip(x uintptr) { c.eip = uint32(x) } -func (c *context) set_sp(x uintptr) { c.esp = uint32(x) } - -// 386 does not have frame pointer register. -func (c *context) set_fp(x uintptr) {} - -func prepareContextForSigResume(c *context) { - c.edx = c.esp - c.ecx = c.eip -} - -func dumpregs(r *context) { - print("eax ", hex(r.eax), "\n") - print("ebx ", hex(r.ebx), "\n") - print("ecx ", hex(r.ecx), "\n") - print("edx ", hex(r.edx), "\n") - print("edi ", hex(r.edi), "\n") - print("esi ", hex(r.esi), "\n") - print("ebp ", hex(r.ebp), "\n") - print("esp ", hex(r.esp), "\n") - print("eip ", hex(r.eip), "\n") - print("eflags ", hex(r.eflags), "\n") - print("cs ", hex(r.segcs), "\n") - print("fs ", hex(r.segfs), "\n") - print("gs ", hex(r.seggs), "\n") -} - -// _DISPATCHER_CONTEXT is not defined on 386. -type _DISPATCHER_CONTEXT struct{} - -func (c *_DISPATCHER_CONTEXT) ctx() *context { - return nil -} diff --git a/src/runtime/defs_windows_amd64.go b/src/runtime/defs_windows_amd64.go deleted file mode 100644 index 9dbfb40e63ee40..00000000000000 --- a/src/runtime/defs_windows_amd64.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -const _CONTEXT_CONTROL = 0x100001 - -type m128a struct { - low uint64 - high int64 -} - -type context struct { - p1home uint64 - p2home uint64 - p3home uint64 - p4home uint64 - p5home uint64 - p6home uint64 - contextflags uint32 - mxcsr uint32 - segcs uint16 - segds uint16 - seges uint16 - segfs uint16 - seggs uint16 - segss uint16 - eflags uint32 - dr0 uint64 - dr1 uint64 - dr2 uint64 - dr3 uint64 - dr6 uint64 - dr7 uint64 - rax uint64 - rcx uint64 - rdx uint64 - rbx uint64 - rsp uint64 - rbp uint64 - rsi uint64 - rdi uint64 - r8 uint64 - r9 uint64 - r10 uint64 - r11 uint64 - r12 uint64 - r13 uint64 - r14 uint64 - r15 uint64 - rip uint64 - anon0 [512]byte - vectorregister [26]m128a - vectorcontrol uint64 - debugcontrol uint64 - lastbranchtorip uint64 - lastbranchfromrip uint64 - lastexceptiontorip uint64 - lastexceptionfromrip uint64 -} - -func (c *context) ip() uintptr { return uintptr(c.rip) } -func (c *context) sp() uintptr { return uintptr(c.rsp) } - -// AMD64 does not have link register, so this returns 0. -func (c *context) lr() uintptr { return 0 } -func (c *context) set_lr(x uintptr) {} - -func (c *context) set_ip(x uintptr) { c.rip = uint64(x) } -func (c *context) set_sp(x uintptr) { c.rsp = uint64(x) } -func (c *context) set_fp(x uintptr) { c.rbp = uint64(x) } - -func prepareContextForSigResume(c *context) { - c.r8 = c.rsp - c.r9 = c.rip -} - -func dumpregs(r *context) { - print("rax ", hex(r.rax), "\n") - print("rbx ", hex(r.rbx), "\n") - print("rcx ", hex(r.rcx), "\n") - print("rdx ", hex(r.rdx), "\n") - print("rdi ", hex(r.rdi), "\n") - print("rsi ", hex(r.rsi), "\n") - print("rbp ", hex(r.rbp), "\n") - print("rsp ", hex(r.rsp), "\n") - print("r8 ", hex(r.r8), "\n") - print("r9 ", hex(r.r9), "\n") - print("r10 ", hex(r.r10), "\n") - print("r11 ", hex(r.r11), "\n") - print("r12 ", hex(r.r12), "\n") - print("r13 ", hex(r.r13), "\n") - print("r14 ", hex(r.r14), "\n") - print("r15 ", hex(r.r15), "\n") - print("rip ", hex(r.rip), "\n") - print("rflags ", hex(r.eflags), "\n") - print("cs ", hex(r.segcs), "\n") - print("fs ", hex(r.segfs), "\n") - print("gs ", hex(r.seggs), "\n") -} - -type _DISPATCHER_CONTEXT struct { - controlPc uint64 - imageBase uint64 - functionEntry uintptr - establisherFrame uint64 - targetIp uint64 - context *context - languageHandler uintptr - handlerData uintptr -} - -func (c *_DISPATCHER_CONTEXT) ctx() *context { - return c.context -} diff --git a/src/runtime/defs_windows_arm.go b/src/runtime/defs_windows_arm.go deleted file mode 100644 index 861a88430ea3b8..00000000000000 --- a/src/runtime/defs_windows_arm.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -// NOTE(rsc): _CONTEXT_CONTROL is actually 0x200001 and should include PC, SP, and LR. -// However, empirically, LR doesn't come along on Windows 10 -// unless you also set _CONTEXT_INTEGER (0x200002). -// Without LR, we skip over the next-to-bottom function in profiles -// when the bottom function is frameless. -// So we set both here, to make a working _CONTEXT_CONTROL. -const _CONTEXT_CONTROL = 0x200003 - -type neon128 struct { - low uint64 - high int64 -} - -type context struct { - contextflags uint32 - r0 uint32 - r1 uint32 - r2 uint32 - r3 uint32 - r4 uint32 - r5 uint32 - r6 uint32 - r7 uint32 - r8 uint32 - r9 uint32 - r10 uint32 - r11 uint32 - r12 uint32 - - spr uint32 - lrr uint32 - pc uint32 - cpsr uint32 - - fpscr uint32 - padding uint32 - - floatNeon [16]neon128 - - bvr [8]uint32 - bcr [8]uint32 - wvr [1]uint32 - wcr [1]uint32 - padding2 [2]uint32 -} - -func (c *context) ip() uintptr { return uintptr(c.pc) } -func (c *context) sp() uintptr { return uintptr(c.spr) } -func (c *context) lr() uintptr { return uintptr(c.lrr) } - -func (c *context) set_ip(x uintptr) { c.pc = uint32(x) } -func (c *context) set_sp(x uintptr) { c.spr = uint32(x) } -func (c *context) set_lr(x uintptr) { c.lrr = uint32(x) } - -// arm does not have frame pointer register. -func (c *context) set_fp(x uintptr) {} - -func prepareContextForSigResume(c *context) { - c.r0 = c.spr - c.r1 = c.pc -} - -func dumpregs(r *context) { - print("r0 ", hex(r.r0), "\n") - print("r1 ", hex(r.r1), "\n") - print("r2 ", hex(r.r2), "\n") - print("r3 ", hex(r.r3), "\n") - print("r4 ", hex(r.r4), "\n") - print("r5 ", hex(r.r5), "\n") - print("r6 ", hex(r.r6), "\n") - print("r7 ", hex(r.r7), "\n") - print("r8 ", hex(r.r8), "\n") - print("r9 ", hex(r.r9), "\n") - print("r10 ", hex(r.r10), "\n") - print("r11 ", hex(r.r11), "\n") - print("r12 ", hex(r.r12), "\n") - print("sp ", hex(r.spr), "\n") - print("lr ", hex(r.lrr), "\n") - print("pc ", hex(r.pc), "\n") - print("cpsr ", hex(r.cpsr), "\n") -} - -func stackcheck() { - // TODO: not implemented on ARM -} - -type _DISPATCHER_CONTEXT struct { - controlPc uint32 - imageBase uint32 - functionEntry uintptr - establisherFrame uint32 - targetIp uint32 - context *context - languageHandler uintptr - handlerData uintptr -} - -func (c *_DISPATCHER_CONTEXT) ctx() *context { - return c.context -} diff --git a/src/runtime/defs_windows_arm64.go b/src/runtime/defs_windows_arm64.go deleted file mode 100644 index 70e28d2ae2875e..00000000000000 --- a/src/runtime/defs_windows_arm64.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -// NOTE(rsc): _CONTEXT_CONTROL is actually 0x400001 and should include PC, SP, and LR. -// However, empirically, LR doesn't come along on Windows 10 -// unless you also set _CONTEXT_INTEGER (0x400002). -// Without LR, we skip over the next-to-bottom function in profiles -// when the bottom function is frameless. -// So we set both here, to make a working _CONTEXT_CONTROL. -const _CONTEXT_CONTROL = 0x400003 - -type neon128 struct { - low uint64 - high int64 -} - -// See https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-arm64_nt_context -type context struct { - contextflags uint32 - cpsr uint32 - x [31]uint64 // fp is x[29], lr is x[30] - xsp uint64 - pc uint64 - v [32]neon128 - fpcr uint32 - fpsr uint32 - bcr [8]uint32 - bvr [8]uint64 - wcr [2]uint32 - wvr [2]uint64 -} - -func (c *context) ip() uintptr { return uintptr(c.pc) } -func (c *context) sp() uintptr { return uintptr(c.xsp) } -func (c *context) lr() uintptr { return uintptr(c.x[30]) } - -func (c *context) set_ip(x uintptr) { c.pc = uint64(x) } -func (c *context) set_sp(x uintptr) { c.xsp = uint64(x) } -func (c *context) set_lr(x uintptr) { c.x[30] = uint64(x) } -func (c *context) set_fp(x uintptr) { c.x[29] = uint64(x) } - -func prepareContextForSigResume(c *context) { - c.x[0] = c.xsp - c.x[1] = c.pc -} - -func dumpregs(r *context) { - print("r0 ", hex(r.x[0]), "\n") - print("r1 ", hex(r.x[1]), "\n") - print("r2 ", hex(r.x[2]), "\n") - print("r3 ", hex(r.x[3]), "\n") - print("r4 ", hex(r.x[4]), "\n") - print("r5 ", hex(r.x[5]), "\n") - print("r6 ", hex(r.x[6]), "\n") - print("r7 ", hex(r.x[7]), "\n") - print("r8 ", hex(r.x[8]), "\n") - print("r9 ", hex(r.x[9]), "\n") - print("r10 ", hex(r.x[10]), "\n") - print("r11 ", hex(r.x[11]), "\n") - print("r12 ", hex(r.x[12]), "\n") - print("r13 ", hex(r.x[13]), "\n") - print("r14 ", hex(r.x[14]), "\n") - print("r15 ", hex(r.x[15]), "\n") - print("r16 ", hex(r.x[16]), "\n") - print("r17 ", hex(r.x[17]), "\n") - print("r18 ", hex(r.x[18]), "\n") - print("r19 ", hex(r.x[19]), "\n") - print("r20 ", hex(r.x[20]), "\n") - print("r21 ", hex(r.x[21]), "\n") - print("r22 ", hex(r.x[22]), "\n") - print("r23 ", hex(r.x[23]), "\n") - print("r24 ", hex(r.x[24]), "\n") - print("r25 ", hex(r.x[25]), "\n") - print("r26 ", hex(r.x[26]), "\n") - print("r27 ", hex(r.x[27]), "\n") - print("r28 ", hex(r.x[28]), "\n") - print("r29 ", hex(r.x[29]), "\n") - print("lr ", hex(r.x[30]), "\n") - print("sp ", hex(r.xsp), "\n") - print("pc ", hex(r.pc), "\n") - print("cpsr ", hex(r.cpsr), "\n") -} - -func stackcheck() { - // TODO: not implemented on ARM -} - -type _DISPATCHER_CONTEXT struct { - controlPc uint64 - imageBase uint64 - functionEntry uintptr - establisherFrame uint64 - targetIp uint64 - context *context - languageHandler uintptr - handlerData uintptr -} - -func (c *_DISPATCHER_CONTEXT) ctx() *context { - return c.context -} diff --git a/src/runtime/duff_amd64.s b/src/runtime/duff_amd64.s deleted file mode 100644 index 69e9980a30e7ac..00000000000000 --- a/src/runtime/duff_amd64.s +++ /dev/null @@ -1,427 +0,0 @@ -// Code generated by mkduff.go; DO NOT EDIT. -// Run go generate from src/runtime to update. -// See mkduff.go for comments. - -#include "textflag.h" - -TEXT runtime·duffzero(SB), NOSPLIT|NOFRAME, $0-0 - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - MOVUPS X15,(DI) - MOVUPS X15,16(DI) - MOVUPS X15,32(DI) - MOVUPS X15,48(DI) - LEAQ 64(DI),DI - - RET - -TEXT runtime·duffcopy(SB), NOSPLIT|NOFRAME, $0-0 - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - MOVUPS (SI), X0 - ADDQ $16, SI - MOVUPS X0, (DI) - ADDQ $16, DI - - RET diff --git a/src/runtime/duff_arm64.s b/src/runtime/duff_arm64.s deleted file mode 100644 index 33c4905078d242..00000000000000 --- a/src/runtime/duff_arm64.s +++ /dev/null @@ -1,267 +0,0 @@ -// Code generated by mkduff.go; DO NOT EDIT. -// Run go generate from src/runtime to update. -// See mkduff.go for comments. - -#include "textflag.h" - -TEXT runtime·duffzero(SB), NOSPLIT|NOFRAME, $0-0 - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP.P (ZR, ZR), 16(R20) - STP (ZR, ZR), (R20) - RET - -TEXT runtime·duffcopy(SB), NOSPLIT|NOFRAME, $0-0 - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - LDP.P 16(R20), (R26, R27) - STP.P (R26, R27), 16(R21) - - RET diff --git a/src/runtime/duff_loong64.s b/src/runtime/duff_loong64.s deleted file mode 100644 index b05502d91db2ef..00000000000000 --- a/src/runtime/duff_loong64.s +++ /dev/null @@ -1,907 +0,0 @@ -// Code generated by mkduff.go; DO NOT EDIT. -// Run go generate from src/runtime to update. -// See mkduff.go for comments. - -#include "textflag.h" - -TEXT runtime·duffzero(SB), NOSPLIT|NOFRAME, $0-0 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - MOVV R0, (R20) - ADDV $8, R20 - RET - -TEXT runtime·duffcopy(SB), NOSPLIT|NOFRAME, $0-0 - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - MOVV (R20), R30 - ADDV $8, R20 - MOVV R30, (R21) - ADDV $8, R21 - - RET diff --git a/src/runtime/duff_riscv64.s b/src/runtime/duff_riscv64.s deleted file mode 100644 index ec447677ad0c17..00000000000000 --- a/src/runtime/duff_riscv64.s +++ /dev/null @@ -1,907 +0,0 @@ -// Code generated by mkduff.go; DO NOT EDIT. -// Run go generate from src/runtime to update. -// See mkduff.go for comments. - -#include "textflag.h" - -TEXT runtime·duffzero(SB), NOSPLIT|NOFRAME, $0-0 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - MOV ZERO, (X25) - ADD $8, X25 - RET - -TEXT runtime·duffcopy(SB), NOSPLIT|NOFRAME, $0-0 - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - MOV (X24), X31 - ADD $8, X24 - MOV X31, (X25) - ADD $8, X25 - - RET diff --git a/src/runtime/ehooks_test.go b/src/runtime/ehooks_test.go index 4beb20b0bec510..380d7098760a5d 100644 --- a/src/runtime/ehooks_test.go +++ b/src/runtime/ehooks_test.go @@ -5,89 +5,74 @@ package runtime_test import ( - "internal/platform" - "internal/testenv" "os/exec" - "runtime" "strings" "testing" ) func TestExitHooks(t *testing.T) { - bmodes := []string{""} if testing.Short() { t.Skip("skipping due to -short") } - // Note the HasCGO() test below; this is to prevent the test - // running if CGO_ENABLED=0 is in effect. - haverace := platform.RaceDetectorSupported(runtime.GOOS, runtime.GOARCH) - if haverace && testenv.HasCGO() { - bmodes = append(bmodes, "-race") - } - for _, bmode := range bmodes { - scenarios := []struct { - mode string - expected string - musthave []string - }{ - { - mode: "simple", - expected: "bar foo", - }, - { - mode: "goodexit", - expected: "orange apple", - }, - { - mode: "badexit", - expected: "blub blix", - }, - { - mode: "panics", - musthave: []string{ - "fatal error: exit hook invoked panic", - "main.testPanics", - }, - }, - { - mode: "callsexit", - musthave: []string{ - "fatal error: exit hook invoked exit", - }, + + scenarios := []struct { + mode string + expected string + musthave []string + }{ + { + mode: "simple", + expected: "bar foo", + }, + { + mode: "goodexit", + expected: "orange apple", + }, + { + mode: "badexit", + expected: "blub blix", + }, + { + mode: "panics", + musthave: []string{ + "fatal error: exit hook invoked panic", + "main.testPanics", }, - { - mode: "exit2", - expected: "", + }, + { + mode: "callsexit", + musthave: []string{ + "fatal error: exit hook invoked exit", }, - } + }, + { + mode: "exit2", + expected: "", + }, + } - exe, err := buildTestProg(t, "testexithooks", bmode) - if err != nil { - t.Fatal(err) - } + exe, err := buildTestProg(t, "testexithooks") + if err != nil { + t.Fatal(err) + } - bt := "" - if bmode != "" { - bt = " bmode: " + bmode + for _, s := range scenarios { + cmd := exec.Command(exe, []string{"-mode", s.mode}...) + out, _ := cmd.CombinedOutput() + outs := strings.ReplaceAll(string(out), "\n", " ") + outs = strings.TrimSpace(outs) + if s.expected != "" && s.expected != outs { + t.Fatalf("failed %s: wanted %q\noutput:\n%s", + s.mode, s.expected, outs) } - for _, s := range scenarios { - cmd := exec.Command(exe, []string{"-mode", s.mode}...) - out, _ := cmd.CombinedOutput() - outs := strings.ReplaceAll(string(out), "\n", " ") - outs = strings.TrimSpace(outs) - if s.expected != "" && s.expected != outs { - t.Fatalf("failed%s mode %s: wanted %q\noutput:\n%s", bt, - s.mode, s.expected, outs) - } - for _, need := range s.musthave { - if !strings.Contains(outs, need) { - t.Fatalf("failed mode %s: output does not contain %q\noutput:\n%s", - s.mode, need, outs) - } - } - if s.expected == "" && s.musthave == nil && outs != "" { - t.Errorf("failed mode %s: wanted no output\noutput:\n%s", s.mode, outs) + for _, need := range s.musthave { + if !strings.Contains(outs, need) { + t.Fatalf("failed mode %s: output does not contain %q\noutput:\n%s", + s.mode, need, outs) } } + if s.expected == "" && s.musthave == nil && outs != "" { + t.Errorf("failed mode %s: wanted no output\noutput:\n%s", s.mode, outs) + } } } diff --git a/src/runtime/env_plan9.go b/src/runtime/env_plan9.go index d206c5dbba4b1f..5622cb4eac256b 100644 --- a/src/runtime/env_plan9.go +++ b/src/runtime/env_plan9.go @@ -30,7 +30,7 @@ const ( func goenvs() { buf := make([]byte, envBufSize) copy(buf, envDir) - dirfd := open(&buf[0], _OREAD, 0) + dirfd := open(&buf[0], _OREAD|_OCEXEC, 0) if dirfd < 0 { return } @@ -40,7 +40,7 @@ func goenvs() { buf = buf[:len(envDir)] copy(buf, envDir) buf = append(buf, name...) - fd := open(&buf[0], _OREAD, 0) + fd := open(&buf[0], _OREAD|_OCEXEC, 0) if fd < 0 { return } diff --git a/src/runtime/error.go b/src/runtime/error.go index 406f36ca5f9d37..f95b14d7808189 100644 --- a/src/runtime/error.go +++ b/src/runtime/error.go @@ -7,16 +7,27 @@ package runtime import ( "internal/abi" "internal/bytealg" + "internal/runtime/sys" ) -// The Error interface identifies a run time error. +// Error identifies a runtime error used in panic. +// +// The Go runtime triggers panics for a variety of cases, as described by the +// Go Language Spec, such as out-of-bounds slice/array access, close of nil +// channels, type assertion failures, etc. +// +// When these cases occur, the Go runtime panics with an error that implements +// Error. This can be useful when recovering from panics to distinguish between +// custom application panics and fundamental runtime panics. +// +// Packages outside of the Go standard library should not implement Error. type Error interface { error // RuntimeError is a no-op function but - // serves to distinguish types that are run time + // serves to distinguish types that are runtime // errors from ordinary errors: a type is a - // run time error if it has a RuntimeError method. + // runtime error if it has a RuntimeError method. RuntimeError() } @@ -121,52 +132,34 @@ type boundsError struct { // Instead, we keep track of whether x should be interpreted as signed or unsigned. // y is known to be nonnegative and to fit in an int. signed bool - code boundsErrorCode + code abi.BoundsErrorCode } -type boundsErrorCode uint8 - -const ( - boundsIndex boundsErrorCode = iota // s[x], 0 <= x < len(s) failed - - boundsSliceAlen // s[?:x], 0 <= x <= len(s) failed - boundsSliceAcap // s[?:x], 0 <= x <= cap(s) failed - boundsSliceB // s[x:y], 0 <= x <= y failed (but boundsSliceA didn't happen) - - boundsSlice3Alen // s[?:?:x], 0 <= x <= len(s) failed - boundsSlice3Acap // s[?:?:x], 0 <= x <= cap(s) failed - boundsSlice3B // s[?:x:y], 0 <= x <= y failed (but boundsSlice3A didn't happen) - boundsSlice3C // s[x:y:?], 0 <= x <= y failed (but boundsSlice3A/B didn't happen) - - boundsConvert // (*[x]T)(s), 0 <= x <= len(s) failed - // Note: in the above, len(s) and cap(s) are stored in y -) - // boundsErrorFmts provide error text for various out-of-bounds panics. // Note: if you change these strings, you should adjust the size of the buffer // in boundsError.Error below as well. var boundsErrorFmts = [...]string{ - boundsIndex: "index out of range [%x] with length %y", - boundsSliceAlen: "slice bounds out of range [:%x] with length %y", - boundsSliceAcap: "slice bounds out of range [:%x] with capacity %y", - boundsSliceB: "slice bounds out of range [%x:%y]", - boundsSlice3Alen: "slice bounds out of range [::%x] with length %y", - boundsSlice3Acap: "slice bounds out of range [::%x] with capacity %y", - boundsSlice3B: "slice bounds out of range [:%x:%y]", - boundsSlice3C: "slice bounds out of range [%x:%y:]", - boundsConvert: "cannot convert slice with length %y to array or pointer to array with length %x", + abi.BoundsIndex: "index out of range [%x] with length %y", + abi.BoundsSliceAlen: "slice bounds out of range [:%x] with length %y", + abi.BoundsSliceAcap: "slice bounds out of range [:%x] with capacity %y", + abi.BoundsSliceB: "slice bounds out of range [%x:%y]", + abi.BoundsSlice3Alen: "slice bounds out of range [::%x] with length %y", + abi.BoundsSlice3Acap: "slice bounds out of range [::%x] with capacity %y", + abi.BoundsSlice3B: "slice bounds out of range [:%x:%y]", + abi.BoundsSlice3C: "slice bounds out of range [%x:%y:]", + abi.BoundsConvert: "cannot convert slice with length %y to array or pointer to array with length %x", } // boundsNegErrorFmts are overriding formats if x is negative. In this case there's no need to report y. var boundsNegErrorFmts = [...]string{ - boundsIndex: "index out of range [%x]", - boundsSliceAlen: "slice bounds out of range [:%x]", - boundsSliceAcap: "slice bounds out of range [:%x]", - boundsSliceB: "slice bounds out of range [%x:]", - boundsSlice3Alen: "slice bounds out of range [::%x]", - boundsSlice3Acap: "slice bounds out of range [::%x]", - boundsSlice3B: "slice bounds out of range [:%x:]", - boundsSlice3C: "slice bounds out of range [%x::]", + abi.BoundsIndex: "index out of range [%x]", + abi.BoundsSliceAlen: "slice bounds out of range [:%x]", + abi.BoundsSliceAcap: "slice bounds out of range [:%x]", + abi.BoundsSliceB: "slice bounds out of range [%x:]", + abi.BoundsSlice3Alen: "slice bounds out of range [::%x]", + abi.BoundsSlice3Acap: "slice bounds out of range [::%x]", + abi.BoundsSlice3B: "slice bounds out of range [:%x:]", + abi.BoundsSlice3C: "slice bounds out of range [%x::]", } func (e boundsError) RuntimeError() {} @@ -267,7 +260,7 @@ func printanycustomtype(i any) { eface := efaceOf(&i) typestring := toRType(eface._type).string() - switch eface._type.Kind_ { + switch eface._type.Kind() { case abi.String: print(typestring, `("`) printindented(*(*string)(eface.data)) @@ -329,7 +322,7 @@ func printindented(s string) { // // It is called from the generated wrapper code. func panicwrap() { - pc := getcallerpc() + pc := sys.GetCallerPC() name := funcNameForPrint(funcname(findfunc(pc))) // name is something like "main.(*T).F". // We want to extract pkg ("main"), typ ("T"), and meth ("F"). diff --git a/src/runtime/example_test.go b/src/runtime/example_test.go index dcb8f7798e2905..8fa8122b000589 100644 --- a/src/runtime/example_test.go +++ b/src/runtime/example_test.go @@ -6,6 +6,7 @@ package runtime_test import ( "fmt" + "os" "runtime" "strings" ) @@ -32,15 +33,14 @@ func ExampleFrames() { for { frame, more := frames.Next() - // Process this frame. - // - // To keep this example's output stable - // even if there are changes in the testing package, - // stop unwinding when we leave package runtime. - if !strings.Contains(frame.File, "runtime/") { + // Canonicalize function name and skip callers of this function + // for predictable example output. + // You probably don't need this in your own code. + function := strings.ReplaceAll(frame.Function, "main.main", "runtime_test.ExampleFrames") + fmt.Printf("- more:%v | %s\n", more, function) + if function == "runtime_test.ExampleFrames" { break } - fmt.Printf("- more:%v | %s\n", more, frame.Function) // Check whether there are more frames to process after this one. if !more { @@ -60,3 +60,36 @@ func ExampleFrames() { // - more:true | runtime_test.ExampleFrames.func3 // - more:true | runtime_test.ExampleFrames } + +func ExampleAddCleanup() { + tempFile, err := os.CreateTemp(os.TempDir(), "file.*") + if err != nil { + fmt.Println("failed to create temp file:", err) + return + } + + ch := make(chan struct{}) + + // Attach a cleanup function to the file object. + runtime.AddCleanup(&tempFile, func(fileName string) { + if err := os.Remove(fileName); err == nil { + fmt.Println("temp file has been removed") + } + ch <- struct{}{} + }, tempFile.Name()) + + if err := tempFile.Close(); err != nil { + fmt.Println("failed to close temp file:", err) + return + } + + // Run the garbage collector to reclaim unreachable objects + // and enqueue their cleanup functions. + runtime.GC() + + // Wait until cleanup function is done. + <-ch + + // Output: + // temp file has been removed +} diff --git a/src/runtime/export_debug_loong64_test.go b/src/runtime/export_debug_loong64_test.go new file mode 100644 index 00000000000000..eaaf3598923c7c --- /dev/null +++ b/src/runtime/export_debug_loong64_test.go @@ -0,0 +1,227 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build loong64 && linux + +package runtime + +import ( + "internal/abi" + "internal/goarch" + "unsafe" +) + +type sigContext struct { + savedRegs sigcontext +} + +func sigctxtSetContextRegister(ctxt *sigctxt, x uint64) { + ctxt.regs().sc_regs[29] = x +} + +func sigctxtAtTrapInstruction(ctxt *sigctxt) bool { + return *(*uint32)(unsafe.Pointer(ctxt.sigpc())) == 0x002a0000 // BREAK 0 +} + +func sigctxtStatus(ctxt *sigctxt) uint64 { + return ctxt.r19() +} + +func (h *debugCallHandler) saveSigContext(ctxt *sigctxt) { + sp := ctxt.sp() + sp -= goarch.PtrSize + ctxt.set_sp(sp) + *(*uint64)(unsafe.Pointer(uintptr(sp))) = ctxt.link() // save the current lr + ctxt.set_link(ctxt.pc()) // set new lr to the current pc + // Write the argument frame size. + *(*uintptr)(unsafe.Pointer(uintptr(sp - 8))) = h.argSize + // Save current registers. + h.sigCtxt.savedRegs = *ctxt.regs() +} + +// case 0 +func (h *debugCallHandler) debugCallRun(ctxt *sigctxt) { + sp := ctxt.sp() + memmove(unsafe.Pointer(uintptr(sp)+8), h.argp, h.argSize) + if h.regArgs != nil { + storeRegArgs(ctxt.regs(), h.regArgs) + } + // Push return PC, which should be the signal PC+4, because + // the signal PC is the PC of the trap instruction itself. + ctxt.set_link(ctxt.pc() + 4) + // Set PC to call and context register. + ctxt.set_pc(uint64(h.fv.fn)) + sigctxtSetContextRegister(ctxt, uint64(uintptr(unsafe.Pointer(h.fv)))) +} + +// case 1 +func (h *debugCallHandler) debugCallReturn(ctxt *sigctxt) { + sp := ctxt.sp() + memmove(h.argp, unsafe.Pointer(uintptr(sp)+8), h.argSize) + if h.regArgs != nil { + loadRegArgs(h.regArgs, ctxt.regs()) + } + // Restore the old lr from *sp + olr := *(*uint64)(unsafe.Pointer(uintptr(sp))) + ctxt.set_link(olr) + pc := ctxt.pc() + ctxt.set_pc(pc + 4) // step to next instruction +} + +// case 2 +func (h *debugCallHandler) debugCallPanicOut(ctxt *sigctxt) { + sp := ctxt.sp() + memmove(unsafe.Pointer(&h.panic), unsafe.Pointer(uintptr(sp)+8), 2*goarch.PtrSize) + ctxt.set_pc(ctxt.pc() + 4) +} + +// case 8 +func (h *debugCallHandler) debugCallUnsafe(ctxt *sigctxt) { + sp := ctxt.sp() + reason := *(*string)(unsafe.Pointer(uintptr(sp) + 8)) + h.err = plainError(reason) + ctxt.set_pc(ctxt.pc() + 4) +} + +// case 16 +func (h *debugCallHandler) restoreSigContext(ctxt *sigctxt) { + // Restore all registers except for pc and sp + pc, sp := ctxt.pc(), ctxt.sp() + *ctxt.regs() = h.sigCtxt.savedRegs + ctxt.set_pc(pc + 4) + ctxt.set_sp(sp) +} + +func getVal32(base uintptr, off uintptr) uint32 { + return *(*uint32)(unsafe.Pointer(base + off)) +} + +func getVal64(base uintptr, off uintptr) uint64 { + return *(*uint64)(unsafe.Pointer(base + off)) +} + +func setVal64(base uintptr, off uintptr, val uint64) { + *(*uint64)(unsafe.Pointer(base + off)) = val +} + +// Layout for sigcontext on linux/loong64: arch/loongarch/include/uapi/asm/sigcontext.h +// +// sc_extcontext | sctx_info +// ------------------------------------------ +// | {fpu,lsx,lasx}_context +// --------------------------- +// | sctx_info +// --------------------------- +// | lbt_context +// + +const ( + INVALID_MAGIC uint32 = 0 + FPU_CTX_MAGIC = 0x46505501 + LSX_CTX_MAGIC = 0x53580001 + LASX_CTX_MAGIC = 0x41535801 + LBT_CTX_MAGIC = 0x42540001 +) + +const ( + SCTX_INFO_SIZE = 4 + 4 + 8 + FPU_CTX_SIZE = 8*32 + 8 + 4 // fpu context size + LSX_CTX_SIZE = 8*64 + 8 + 4 // lsx context size + LASX_CTX_SIZE = 8*128 + 8 + 4 // lasx context size + LBT_CTX_SIZE = 8*4 + 4 + 4 // lbt context size +) + +// storeRegArgs sets up argument registers in the signal context state +// from an abi.RegArgs. +// +// Both src and dst must be non-nil. +func storeRegArgs(dst *sigcontext, src *abi.RegArgs) { + // R4..R19 are used to pass int arguments in registers on loong64 + for i := 0; i < abi.IntArgRegs; i++ { + dst.sc_regs[i+4] = (uint64)(src.Ints[i]) + } + + // F0..F15 are used to pass float arguments in registers on loong64 + offset := (uintptr)(0) + baseAddr := (uintptr)(unsafe.Pointer(&dst.sc_extcontext)) + + for { + magic := getVal32(baseAddr, offset) + size := getVal32(baseAddr, offset+4) + + switch magic { + case INVALID_MAGIC: + return + + case FPU_CTX_MAGIC: + offset += SCTX_INFO_SIZE + for i := 0; i < abi.FloatArgRegs; i++ { + setVal64(baseAddr, ((uintptr)(i*8) + offset), src.Floats[i]) + } + return + + case LSX_CTX_MAGIC: + offset += SCTX_INFO_SIZE + for i := 0; i < abi.FloatArgRegs; i++ { + setVal64(baseAddr, ((uintptr)(i*16) + offset), src.Floats[i]) + } + return + + case LASX_CTX_MAGIC: + offset += SCTX_INFO_SIZE + for i := 0; i < abi.FloatArgRegs; i++ { + setVal64(baseAddr, ((uintptr)(i*32) + offset), src.Floats[i]) + } + return + + case LBT_CTX_MAGIC: + offset += uintptr(size) + } + } +} + +func loadRegArgs(dst *abi.RegArgs, src *sigcontext) { + // R4..R19 are used to pass int arguments in registers on loong64 + for i := 0; i < abi.IntArgRegs; i++ { + dst.Ints[i] = uintptr(src.sc_regs[i+4]) + } + + // F0..F15 are used to pass float arguments in registers on loong64 + offset := (uintptr)(0) + baseAddr := (uintptr)(unsafe.Pointer(&src.sc_extcontext)) + + for { + magic := getVal32(baseAddr, offset) + size := getVal32(baseAddr, (offset + 4)) + + switch magic { + case INVALID_MAGIC: + return + + case FPU_CTX_MAGIC: + offset += SCTX_INFO_SIZE + for i := 0; i < abi.FloatArgRegs; i++ { + dst.Floats[i] = getVal64(baseAddr, (uintptr(i*8) + offset)) + } + return + + case LSX_CTX_MAGIC: + offset += SCTX_INFO_SIZE + for i := 0; i < abi.FloatArgRegs; i++ { + dst.Floats[i] = getVal64(baseAddr, (uintptr(i*16) + offset)) + } + return + + case LASX_CTX_MAGIC: + offset += SCTX_INFO_SIZE + for i := 0; i < abi.FloatArgRegs; i++ { + dst.Floats[i] = getVal64(baseAddr, (uintptr(i*32) + offset)) + } + return + + case LBT_CTX_MAGIC: + offset += uintptr(size) + } + } +} diff --git a/src/runtime/export_debug_test.go b/src/runtime/export_debug_test.go index 4e0a4ef97e9d07..94dc97480420c7 100644 --- a/src/runtime/export_debug_test.go +++ b/src/runtime/export_debug_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (amd64 || arm64 || ppc64le) && linux +//go:build (amd64 || arm64 || loong64 || ppc64le) && linux package runtime @@ -33,13 +33,13 @@ func InjectDebugCall(gp *g, fn any, regArgs *abi.RegArgs, stackArgs any, tkill f } f := efaceOf(&fn) - if f._type == nil || f._type.Kind_&abi.KindMask != abi.Func { + if f._type == nil || f._type.Kind() != abi.Func { return nil, plainError("fn must be a function") } fv := (*funcval)(f.data) a := efaceOf(&stackArgs) - if a._type != nil && a._type.Kind_&abi.KindMask != abi.Pointer { + if a._type != nil && a._type.Kind() != abi.Pointer { return nil, plainError("args must be a pointer or nil") } argp := a.data diff --git a/src/runtime/export_debuglog_test.go b/src/runtime/export_debuglog_test.go index 04ac79f35791c7..c370a7933611a5 100644 --- a/src/runtime/export_debuglog_test.go +++ b/src/runtime/export_debuglog_test.go @@ -12,22 +12,26 @@ const DebugLogBytes = debugLogBytes const DebugLogStringLimit = debugLogStringLimit -var Dlog = dlog - -func (l *dlogger) End() { l.end() } -func (l *dlogger) B(x bool) *dlogger { return l.b(x) } -func (l *dlogger) I(x int) *dlogger { return l.i(x) } -func (l *dlogger) I16(x int16) *dlogger { return l.i16(x) } -func (l *dlogger) U64(x uint64) *dlogger { return l.u64(x) } -func (l *dlogger) Hex(x uint64) *dlogger { return l.hex(x) } -func (l *dlogger) P(x any) *dlogger { return l.p(x) } -func (l *dlogger) S(x string) *dlogger { return l.s(x) } -func (l *dlogger) PC(x uintptr) *dlogger { return l.pc(x) } +type Dlogger = dloggerImpl + +func Dlog() *Dlogger { + return dlogImpl() +} + +func (l *dloggerImpl) End() { l.end() } +func (l *dloggerImpl) B(x bool) *dloggerImpl { return l.b(x) } +func (l *dloggerImpl) I(x int) *dloggerImpl { return l.i(x) } +func (l *dloggerImpl) I16(x int16) *dloggerImpl { return l.i16(x) } +func (l *dloggerImpl) U64(x uint64) *dloggerImpl { return l.u64(x) } +func (l *dloggerImpl) Hex(x uint64) *dloggerImpl { return l.hex(x) } +func (l *dloggerImpl) P(x any) *dloggerImpl { return l.p(x) } +func (l *dloggerImpl) S(x string) *dloggerImpl { return l.s(x) } +func (l *dloggerImpl) PC(x uintptr) *dloggerImpl { return l.pc(x) } func DumpDebugLog() string { gp := getg() gp.writebuf = make([]byte, 0, 1<<20) - printDebugLog() + printDebugLogImpl() buf := gp.writebuf gp.writebuf = nil @@ -44,3 +48,13 @@ func ResetDebugLog() { } startTheWorld(stw) } + +func CountDebugLog() int { + stw := stopTheWorld(stwForTestResetDebugLog) + i := 0 + for l := allDloggers; l != nil; l = l.allLink { + i++ + } + startTheWorld(stw) + return i +} diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index 759463755ac5d9..9f2fcacc30ee5c 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -11,6 +11,7 @@ import ( "internal/goarch" "internal/goos" "internal/runtime/atomic" + "internal/runtime/gc" "internal/runtime/sys" "unsafe" ) @@ -34,11 +35,11 @@ var ReadRandomFailed = &readRandomFailed var Fastlog2 = fastlog2 -var Atoi = atoi -var Atoi32 = atoi32 var ParseByteCount = parseByteCount var Nanotime = nanotime +var Cputicks = cputicks +var CyclesPerSecond = pprof_cyclesPerSecond var NetpollBreak = netpollBreak var Usleep = usleep @@ -57,9 +58,6 @@ const CrashStackImplemented = crashStackImplemented const TracebackInnerFrames = tracebackInnerFrames const TracebackOuterFrames = tracebackOuterFrames -var MapKeys = keys -var MapValues = values - var LockPartialOrder = lockPartialOrder type TimeTimer = timeTimer @@ -94,9 +92,9 @@ func Netpoll(delta int64) { }) } -func GCMask(x any) (ret []byte) { +func PointerMask(x any) (ret []byte) { systemstack(func() { - ret = getgcmask(x) + ret = pointerMask(x) }) return } @@ -363,7 +361,7 @@ func ReadMemStatsSlow() (base, slow MemStats) { slow.Mallocs = 0 slow.Frees = 0 slow.HeapReleased = 0 - var bySize [_NumSizeClasses]struct { + var bySize [gc.NumSizeClasses]struct { Mallocs, Frees uint64 } @@ -391,11 +389,11 @@ func ReadMemStatsSlow() (base, slow MemStats) { // Collect per-sizeclass free stats. var smallFree uint64 - for i := 0; i < _NumSizeClasses; i++ { + for i := 0; i < gc.NumSizeClasses; i++ { slow.Frees += m.smallFreeCount[i] bySize[i].Frees += m.smallFreeCount[i] bySize[i].Mallocs += m.smallFreeCount[i] - smallFree += m.smallFreeCount[i] * uint64(class_to_size[i]) + smallFree += m.smallFreeCount[i] * uint64(gc.SizeClassToSize[i]) } slow.Frees += m.tinyAllocCount + m.largeFreeCount slow.Mallocs += slow.Frees @@ -416,7 +414,8 @@ func ReadMemStatsSlow() (base, slow MemStats) { slow.HeapReleased += uint64(pg) * pageSize } for _, p := range allp { - pg := sys.OnesCount64(p.pcache.scav) + // Only count scav bits for pages in the cache + pg := sys.OnesCount64(p.pcache.cache & p.pcache.scav) slow.HeapReleased += uint64(pg) * pageSize } @@ -481,22 +480,6 @@ func (rw *RWMutex) Unlock() { rw.rw.unlock() } -const RuntimeHmapSize = unsafe.Sizeof(hmap{}) - -func MapBucketsCount(m map[int]int) int { - h := *(**hmap)(unsafe.Pointer(&m)) - return 1 << h.B -} - -func MapBucketsPointerIsNil(m map[int]int) bool { - h := *(**hmap)(unsafe.Pointer(&m)) - return h.buckets == nil -} - -func OverLoadFactor(count int, B uint8) bool { - return overLoadFactor(count, B) -} - func LockOSCounts() (external, internal uint32) { gp := getg() if gp.m.lockedExt+gp.m.lockedInt == 0 { @@ -514,7 +497,7 @@ func LockOSCounts() (external, internal uint32) { //go:noinline func TracebackSystemstack(stk []uintptr, i int) int { if i == 0 { - pc, sp := getcallerpc(), getcallersp() + pc, sp := sys.GetCallerPC(), sys.GetCallerSP() var u unwinder u.initAt(pc, sp, 0, getg(), unwindJumpStack) // Don't ignore errors, for testing return tracebackPCs(&u, 0, stk) @@ -552,7 +535,7 @@ func MapNextArenaHint() (start, end uintptr, ok bool) { } else { start, end = addr, addr+heapArenaBytes } - got := sysReserve(unsafe.Pointer(addr), physPageSize) + got := sysReserve(unsafe.Pointer(addr), physPageSize, "") ok = (addr == uintptr(got)) if !ok { // We were unable to get the requested reservation. @@ -570,6 +553,8 @@ type G = g type Sudog = sudog +type XRegPerG = xRegPerG + func Getg() *G { return getg() } @@ -597,7 +582,7 @@ func unexportedPanicForTesting(b []byte, i int) byte { func G0StackOverflow() { systemstack(func() { g0 := getg() - sp := getcallersp() + sp := sys.GetCallerSP() // The stack bounds for g0 stack is not always precise. // Use an artificially small stack, to trigger a stack overflow // without actually run out of the system stack (which may seg fault). @@ -614,42 +599,6 @@ func stackOverflow(x *byte) { stackOverflow(&buf[0]) } -func MapTombstoneCheck(m map[int]int) { - // Make sure emptyOne and emptyRest are distributed correctly. - // We should have a series of filled and emptyOne cells, followed by - // a series of emptyRest cells. - h := *(**hmap)(unsafe.Pointer(&m)) - i := any(m) - t := *(**maptype)(unsafe.Pointer(&i)) - - for x := 0; x < 1<= n && b.tophash[i] != emptyRest { - panic("late non-emptyRest") - } - if k == n-1 && b.tophash[i] == emptyOne { - panic("last non-emptyRest entry is emptyOne") - } - k++ - } - } - } -} - func RunGetgThreadSwitchTest() { // Test that getg works correctly with thread switch. // With gccgo, if we generate getg inlined, the backend @@ -1171,12 +1120,14 @@ func CheckScavengedBitsCleared(mismatches []BitsMismatch) (n int, ok bool) { // Lock so that we can safely access the bitmap. lock(&mheap_.lock) + chunkLoop: for i := mheap_.pages.start; i < mheap_.pages.end; i++ { chunk := mheap_.pages.tryChunkOf(i) if chunk == nil { continue } + cb := chunkBase(i) for j := 0; j < pallocChunkPages/64; j++ { // Run over each 64-bit bitmap section and ensure // scavenged is being cleared properly on allocation. @@ -1191,7 +1142,7 @@ func CheckScavengedBitsCleared(mismatches []BitsMismatch) (n int, ok bool) { break chunkLoop } mismatches[n] = BitsMismatch{ - Base: chunkBase(i) + uintptr(j)*64*pageSize, + Base: cb + uintptr(j)*64*pageSize, Got: got, Want: want, } @@ -1203,6 +1154,37 @@ func CheckScavengedBitsCleared(mismatches []BitsMismatch) (n int, ok bool) { getg().m.mallocing-- }) + + if randomizeHeapBase && len(mismatches) > 0 { + // When goexperiment.RandomizedHeapBase64 is set we use a series of + // padding pages to generate randomized heap base address which have + // both the alloc and scav bits set. Because of this we expect exactly + // one arena will have mismatches, so check for that explicitly and + // remove the mismatches if that property holds. If we see more than one + // arena with this property, that is an indication something has + // actually gone wrong, so return the mismatches. + // + // We do this, instead of ignoring the mismatches in the chunkLoop, because + // it's not easy to determine which arena we added the padding pages to + // programmatically, without explicitly recording the base address somewhere + // in a global variable (which we'd rather not do as the address of that variable + // is likely to be somewhat predictable, potentially defeating the purpose + // of our randomization). + affectedArenas := map[arenaIdx]bool{} + for _, mismatch := range mismatches { + if mismatch.Base > 0 { + affectedArenas[arenaIndex(mismatch.Base)] = true + } + } + if len(affectedArenas) == 1 { + ok = true + // zero the mismatches + for i := range n { + mismatches[i] = BitsMismatch{} + } + } + } + return } @@ -1223,6 +1205,9 @@ func PageCachePagesLeaked() (leaked uintptr) { return } +var ProcYield = procyield +var OSYield = osyield + type Mutex = mutex var Lock = lock @@ -1280,6 +1265,7 @@ func AllocMSpan() *MSpan { systemstack(func() { lock(&mheap_.lock) s = (*mspan)(mheap_.spanalloc.alloc()) + s.init(0, 0) unlock(&mheap_.lock) }) return (*MSpan)(s) @@ -1313,7 +1299,7 @@ const ( type TimeHistogram timeHistogram -// Counts returns the counts for the given bucket, subBucket indices. +// Count returns the counts for the given bucket, subBucket indices. // Returns true if the bucket was valid, otherwise returns the counts // for the overflow bucket if bucket > 0 or the underflow bucket if // bucket < 0, and false. @@ -1782,7 +1768,7 @@ func NewUserArena() *UserArena { func (a *UserArena) New(out *any) { i := efaceOf(out) typ := i._type - if typ.Kind_&abi.KindMask != abi.Pointer { + if typ.Kind() != abi.Pointer { panic("new result of non-ptr type") } typ = (*ptrtype)(unsafe.Pointer(typ)).Elem @@ -1819,16 +1805,22 @@ func BlockUntilEmptyFinalizerQueue(timeout int64) bool { return blockUntilEmptyFinalizerQueue(timeout) } +func BlockUntilEmptyCleanupQueue(timeout int64) bool { + return gcCleanups.blockUntilEmpty(timeout) +} + func FrameStartLine(f *Frame) int { return f.startLine } // PersistentAlloc allocates some memory that lives outside the Go heap. // This memory will never be freed; use sparingly. -func PersistentAlloc(n uintptr) unsafe.Pointer { - return persistentalloc(n, 0, &memstats.other_sys) +func PersistentAlloc(n, align uintptr) unsafe.Pointer { + return persistentalloc(n, align, &memstats.other_sys) } +const TagAlign = tagAlign + // FPCallers works like Callers and uses frame pointer unwinding to populate // pcBuf with the return addresses of the physical frames on the stack. func FPCallers(pcBuf []uintptr) int { @@ -1886,3 +1878,65 @@ func (m *TraceMap) PutString(s string) (uint64, bool) { func (m *TraceMap) Reset() { m.traceMap.reset() } + +func SetSpinInGCMarkDone(spin bool) { + gcDebugMarkDone.spinAfterRaggedBarrier.Store(spin) +} + +func GCMarkDoneRestarted() bool { + // Only read this outside of the GC. If we're running during a GC, just report false. + mp := acquirem() + if gcphase != _GCoff { + releasem(mp) + return false + } + restarted := gcDebugMarkDone.restartedDueTo27993 + releasem(mp) + return restarted +} + +func GCMarkDoneResetRestartFlag() { + mp := acquirem() + for gcphase != _GCoff { + releasem(mp) + Gosched() + mp = acquirem() + } + gcDebugMarkDone.restartedDueTo27993 = false + releasem(mp) +} + +type BitCursor struct { + b bitCursor +} + +func NewBitCursor(buf *byte) BitCursor { + return BitCursor{b: bitCursor{ptr: buf, n: 0}} +} + +func (b BitCursor) Write(data *byte, cnt uintptr) { + b.b.write(data, cnt) +} +func (b BitCursor) Offset(cnt uintptr) BitCursor { + return BitCursor{b: b.b.offset(cnt)} +} + +const ( + BubbleAssocUnbubbled = bubbleAssocUnbubbled + BubbleAssocCurrentBubble = bubbleAssocCurrentBubble + BubbleAssocOtherBubble = bubbleAssocOtherBubble +) + +type TraceStackTable traceStackTable + +func (t *TraceStackTable) Reset() { + t.tab.reset() +} + +func TraceStack(gp *G, tab *TraceStackTable) { + traceStack(0, gp, (*traceStackTable)(tab)) +} + +var DebugDecorateMappings = &debug.decoratemappings + +func SetVMANameSupported() bool { return setVMANameSupported() } diff --git a/src/runtime/export_vdso_linux_test.go b/src/runtime/export_vdso_linux_test.go new file mode 100644 index 00000000000000..cd339c6038f717 --- /dev/null +++ b/src/runtime/export_vdso_linux_test.go @@ -0,0 +1,29 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (386 || amd64 || arm || arm64 || loong64 || mips64 || mips64le || ppc64 || ppc64le || riscv64 || s390x) + +package runtime + +type VDSOSymbolKey vdsoSymbolKey + +func (v VDSOSymbolKey) Name() string { + return v.name +} + +func (v VDSOSymbolKey) SymHash() uint32 { + return v.symHash +} + +func (v VDSOSymbolKey) GNUHash() uint32 { + return v.gnuHash +} + +func VDSOSymbolKeys() []VDSOSymbolKey { + keys := make([]VDSOSymbolKey, 0, len(vdsoSymbolKeys)) + for _, k := range vdsoSymbolKeys { + keys = append(keys, VDSOSymbolKey(k)) + } + return keys +} diff --git a/src/runtime/export_windows_test.go b/src/runtime/export_windows_test.go index 4880e62a550ee0..f44e24bbacf396 100644 --- a/src/runtime/export_windows_test.go +++ b/src/runtime/export_windows_test.go @@ -6,9 +6,10 @@ package runtime -import "unsafe" - -const MaxArgs = maxArgs +import ( + "internal/runtime/syscall/windows" + "unsafe" +) var ( OsYield = osyield @@ -16,23 +17,11 @@ var ( ) func NumberOfProcessors() int32 { - var info systeminfo - stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info))) - return int32(info.dwnumberofprocessors) -} - -type ContextStub struct { - context -} - -func (c ContextStub) GetPC() uintptr { - return c.ip() + var info windows.SystemInfo + stdcall(_GetSystemInfo, uintptr(unsafe.Pointer(&info))) + return int32(info.NumberOfProcessors) } -func NewContextStub() *ContextStub { - var ctx context - ctx.set_ip(getcallerpc()) - ctx.set_sp(getcallersp()) - ctx.set_fp(getcallerfp()) - return &ContextStub{ctx} +func GetCallerFp() uintptr { + return getcallerfp() } diff --git a/src/runtime/extern.go b/src/runtime/extern.go index 2019be4ddec4f3..c35aef06624f54 100644 --- a/src/runtime/extern.go +++ b/src/runtime/extern.go @@ -52,6 +52,28 @@ It is a comma-separated list of name=val pairs setting these named variables: cgocheck mode can be enabled using GOEXPERIMENT (which requires a rebuild), see https://pkg.go.dev/internal/goexperiment for details. + checkfinalizers: setting checkfinalizers=1 causes the garbage collector to run + multiple partial non-parallel stop-the-world collections to identify common issues with + finalizers and cleanups, like those listed at + https://go.dev/doc/gc-guide#Finalizers_cleanups_and_weak_pointers. If a potential issue + is found, the program will terminate with a description of all potential issues, the + associated values, and a list of those values' finalizers and cleanups, including where + they were created. It also adds tracking for tiny blocks to help diagnose issues with + those as well. The analysis performed during the partial collection is conservative. + Notably, it flags any path back to the original object from the cleanup function, + cleanup arguments, or finalizer function as a potential issue, even if that path might + be severed sometime later during execution (though this is not a recommended pattern). + This mode also produces one line of output to stderr every GC cycle with information + about the finalizer and cleanup queue lengths. Lines produced by this mode start with + "checkfinalizers:". + + decoratemappings: controls whether the Go runtime annotates OS + anonymous memory mappings with context about their purpose. These + annotations appear in /proc/self/maps and /proc/self/smaps as + "[anon: Go: ...]". This setting is only used on Linux. For Go 1.25, it + defaults to `decoratemappings=1`, enabling annotations. Using + `decoratemappings=0` reverts to the pre-Go 1.25 behavior. + disablethp: setting disablethp=1 on Linux disables transparent huge pages for the heap. It has no effect on other platforms. disablethp is meant for compatibility with versions of Go before 1.21, which stopped working around a Linux kernel default that can result @@ -149,25 +171,9 @@ It is a comma-separated list of name=val pairs setting these named variables: silently default to 1024. Future versions of Go may remove this limitation and extend profstackdepth to apply to the CPU profiler and execution tracer. - pagetrace: setting pagetrace=/path/to/file will write out a trace of page events - that can be viewed, analyzed, and visualized using the x/debug/cmd/pagetrace tool. - Build your program with GOEXPERIMENT=pagetrace to enable this functionality. Do not - enable this functionality if your program is a setuid binary as it introduces a security - risk in that scenario. Currently not supported on Windows, plan9 or js/wasm. Setting this - option for some applications can produce large traces, so use with care. - panicnil: setting panicnil=1 disables the runtime error when calling panic with nil interface value or an untyped nil. - runtimecontentionstacks: setting runtimecontentionstacks=1 enables inclusion of call stacks - related to contention on runtime-internal locks in the "mutex" profile, subject to the - MutexProfileFraction setting. When runtimecontentionstacks=0, contention on - runtime-internal locks will report as "runtime._LostContendedRuntimeLock". When - runtimecontentionstacks=1, the call stacks will correspond to the unlock call that released - the lock. But instead of the value corresponding to the amount of contention that call - stack caused, it corresponds to the amount of time the caller of unlock had to wait in its - original call to lock. A future release is expected to align those and remove this setting. - invalidptr: invalidptr=1 (the default) causes the garbage collector and stack copier to crash the program if an invalid pointer value (for example, 1) is found in a pointer-typed location. Setting invalidptr=0 disables this check. @@ -294,10 +300,11 @@ import ( // Caller reports file and line number information about function invocations on // the calling goroutine's stack. The argument skip is the number of stack frames -// to ascend, with 0 identifying the caller of Caller. (For historical reasons the -// meaning of skip differs between Caller and [Callers].) The return values report the -// program counter, file name, and line number within the file of the corresponding -// call. The boolean ok is false if it was not possible to recover the information. +// to ascend, with 0 identifying the caller of Caller. (For historical reasons the +// meaning of skip differs between Caller and [Callers].) The return values report +// the program counter, the file name (using forward slashes as path separator, even +// on Windows), and the line number within the file of the corresponding call. +// The boolean ok is false if it was not possible to recover the information. func Caller(skip int) (pc uintptr, file string, line int, ok bool) { rpc := make([]uintptr, 1) n := callers(skip+1, rpc) @@ -336,6 +343,11 @@ var defaultGOROOT string // set by cmd/link // GOROOT returns the root of the Go tree. It uses the // GOROOT environment variable, if set at process start, // or else the root used during the Go build. +// +// Deprecated: The root used during the Go build will not be +// meaningful if the binary is copied to another machine. +// Use the system path to locate the “go” binary, and use +// “go env GOROOT” to find its GOROOT. func GOROOT() string { s := gogetenv("GOROOT") if s != "" { diff --git a/src/runtime/float.go b/src/runtime/float.go index 9f281c404505ae..d8573c103b325f 100644 --- a/src/runtime/float.go +++ b/src/runtime/float.go @@ -6,6 +6,12 @@ package runtime import "unsafe" +const ( + float64Mask = 0x7FF + float64Shift = 64 - 11 - 1 + float64Bias = 1023 +) + var inf = float64frombits(0x7FF0000000000000) // isNaN reports whether f is an IEEE 754 “not-a-number” value. @@ -52,3 +58,76 @@ func float64bits(f float64) uint64 { func float64frombits(b uint64) float64 { return *(*float64)(unsafe.Pointer(&b)) } + +// floor returns the greatest integer value less than or equal to x. +// +// Special cases are: +// +// floor(±0) = ±0 +// floor(±Inf) = ±Inf +// floor(NaN) = NaN +// +// N.B. Portable floor copied from math. math also has optimized arch-specific +// implementations. +func floor(x float64) float64 { + if x == 0 || isNaN(x) || isInf(x) { + return x + } + if x < 0 { + d, fract := modf(-x) + if fract != 0.0 { + d = d + 1 + } + return -d + } + d, _ := modf(x) + return d +} + +// ceil returns the least integer value greater than or equal to x. +// +// Special cases are: +// +// Ceil(±0) = ±0 +// Ceil(±Inf) = ±Inf +// Ceil(NaN) = NaN +// +// N.B. Portable ceil copied from math. math also has optimized arch-specific +// implementations. +func ceil(x float64) float64 { + return -floor(-x) +} + +// modf returns integer and fractional floating-point numbers +// that sum to f. Both values have the same sign as f. +// +// Special cases are: +// +// Modf(±Inf) = ±Inf, NaN +// Modf(NaN) = NaN, NaN +// +// N.B. Portable modf copied from math. math also has optimized arch-specific +// implementations. +func modf(f float64) (int float64, frac float64) { + if f < 1 { + switch { + case f < 0: + int, frac = modf(-f) + return -int, -frac + case f == 0: + return f, f // Return -0, -0 when f == -0 + } + return 0, f + } + + x := float64bits(f) + e := uint(x>>float64Shift)&float64Mask - float64Bias + + // Keep the top 12+e bits, the integer part; clear the rest. + if e < 64-12 { + x &^= 1<<(64-12-e) - 1 + } + int = float64frombits(x) + frac = f - int + return +} diff --git a/src/runtime/gc_test.go b/src/runtime/gc_test.go index 908f6322466b17..be4d3451f36a9c 100644 --- a/src/runtime/gc_test.go +++ b/src/runtime/gc_test.go @@ -6,6 +6,10 @@ package runtime_test import ( "fmt" + "internal/asan" + "internal/msan" + "internal/race" + "internal/testenv" "math/bits" "math/rand" "os" @@ -19,6 +23,7 @@ import ( "testing" "time" "unsafe" + "weak" ) func TestGcSys(t *testing.T) { @@ -196,6 +201,9 @@ func TestPeriodicGC(t *testing.T) { } func TestGcZombieReporting(t *testing.T) { + if asan.Enabled || msan.Enabled || race.Enabled { + t.Skip("skipped test: checkptr mode catches the issue before getting to zombie reporting") + } // This test is somewhat sensitive to how the allocator works. // Pointers in zombies slice may cross-span, thus we // add invalidptr=0 for avoiding the badPointer check. @@ -208,6 +216,9 @@ func TestGcZombieReporting(t *testing.T) { } func TestGCTestMoveStackOnNextCall(t *testing.T) { + if asan.Enabled { + t.Skip("extra allocations with -asan causes this to fail; see #70079") + } t.Parallel() var onStack int // GCTestMoveStackOnNextCall can fail in rare cases if there's @@ -298,6 +309,9 @@ var pointerClassBSS *int var pointerClassData = 42 func TestGCTestPointerClass(t *testing.T) { + if asan.Enabled { + t.Skip("extra allocations cause this test to fail; see #70079") + } t.Parallel() check := func(p unsafe.Pointer, want string) { t.Helper() @@ -734,7 +748,7 @@ func BenchmarkMSpanCountAlloc(b *testing.B) { // always rounded up 8 bytes. for _, n := range []int{8, 16, 32, 64, 128} { b.Run(fmt.Sprintf("bits=%d", n*8), func(b *testing.B) { - // Initialize a new byte slice with pseduo-random data. + // Initialize a new byte slice with pseudo-random data. bits := make([]byte, n) rand.Read(bits) @@ -787,3 +801,104 @@ func TestMemoryLimitNoGCPercent(t *testing.T) { func TestMyGenericFunc(t *testing.T) { runtime.MyGenericFunc[int]() } + +func TestWeakToStrongMarkTermination(t *testing.T) { + testenv.MustHaveParallelism(t) + + type T struct { + a *int + b int + } + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(2)) + defer debug.SetGCPercent(debug.SetGCPercent(-1)) + w := make([]weak.Pointer[T], 2048) + + // Make sure there's no out-standing GC from a previous test. + runtime.GC() + + // Create many objects with a weak pointers to them. + for i := range w { + x := new(T) + x.a = new(int) + w[i] = weak.Make(x) + } + + // Reset the restart flag. + runtime.GCMarkDoneResetRestartFlag() + + // Prevent mark termination from completing. + runtime.SetSpinInGCMarkDone(true) + + // Start a GC, and wait a little bit to get something spinning in mark termination. + // Simultaneously, fire off another goroutine to disable spinning. If everything's + // working correctly, then weak.Value will block, so we need to make sure something + // prevents the GC from continuing to spin. + done := make(chan struct{}) + go func() { + runtime.GC() + done <- struct{}{} + }() + go func() { + // Usleep here instead of time.Sleep. time.Sleep + // can allocate, and if we get unlucky, then it + // can end up stuck in gcMarkDone with nothing to + // wake it. + runtime.Usleep(100000) // 100ms + + // Let mark termination continue. + runtime.SetSpinInGCMarkDone(false) + }() + time.Sleep(10 * time.Millisecond) + + // Perform many weak->strong conversions in the critical window. + var wg sync.WaitGroup + for _, wp := range w { + wg.Add(1) + go func() { + defer wg.Done() + wp.Value() + }() + } + + // Make sure the GC completes. + <-done + + // Make sure all the weak->strong conversions finish. + wg.Wait() + + // The bug is triggered if there's still mark work after gcMarkDone stops the world. + // + // This can manifest in one of two ways today: + // - An exceedingly rare crash in mark termination. + // - gcMarkDone restarts, as if issue #27993 is at play. + // + // Check for the latter. This is a fairly controlled environment, so #27993 is very + // unlikely to happen (it's already rare to begin with) but we'll always _appear_ to + // trigger the same bug if weak->strong conversions aren't properly coordinated with + // mark termination. + if runtime.GCMarkDoneRestarted() { + t.Errorf("gcMarkDone restarted") + } +} + +func TestDetectFinalizerAndCleanupLeaks(t *testing.T) { + got := runTestProg(t, "testprog", "DetectFinalizerAndCleanupLeaks", "GODEBUG=checkfinalizers=1") + sp := strings.SplitN(got, "detected possible issues with cleanups and/or finalizers", 2) + if len(sp) != 2 { + t.Fatalf("expected the runtime to throw, got:\n%s", got) + } + if strings.Count(sp[0], "is reachable from") != 2 { + t.Fatalf("expected exactly two leaked cleanups and/or finalizers, got:\n%s", got) + } + // N.B. Disable in race mode and in asan mode. Both disable the tiny allocator. + wantSymbolizedLocations := 2 + if !race.Enabled && !asan.Enabled { + if strings.Count(sp[0], "is in a tiny block") != 1 { + t.Fatalf("expected exactly one report for allocation in a tiny block, got:\n%s", got) + } + wantSymbolizedLocations++ + } + if strings.Count(sp[0], "main.DetectFinalizerAndCleanupLeaks()") != wantSymbolizedLocations { + t.Fatalf("expected %d symbolized locations, got:\n%s", wantSymbolizedLocations, got) + } +} diff --git a/src/runtime/gcinfo_test.go b/src/runtime/gcinfo_test.go index 5f72caf0ecebfb..02457f682e28fb 100644 --- a/src/runtime/gcinfo_test.go +++ b/src/runtime/gcinfo_test.go @@ -90,7 +90,7 @@ func TestGCInfo(t *testing.T) { } func verifyGCInfo(t *testing.T, name string, p any, mask0 []byte) { - mask := runtime.GCMask(p) + mask := runtime.PointerMask(p) if bytes.HasPrefix(mask, mask0) { // Just the prefix matching is OK. // diff --git a/src/runtime/gomaxprocs_windows_test.go b/src/runtime/gomaxprocs_windows_test.go new file mode 100644 index 00000000000000..34e6ba5840dbfa --- /dev/null +++ b/src/runtime/gomaxprocs_windows_test.go @@ -0,0 +1,44 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "strings" + "testing" +) + +func TestGOMAXPROCSUpdate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test: long sleeps") + } + + got := runTestProg(t, "testprog", "WindowsUpdateGOMAXPROCS") + if strings.Contains(got, "SKIP") { + t.Skip(got) + } + if !strings.Contains(got, "OK") { + t.Fatalf("output got %q want OK", got) + } +} + +func TestCgroupGOMAXPROCSDontUpdate(t *testing.T) { + if testing.Short() { + t.Skip("skipping test: long sleeps") + } + + // Two ways to disable updates: explicit GOMAXPROCS or GODEBUG for + // update feature. + for _, v := range []string{"GOMAXPROCS=4", "GODEBUG=updatemaxprocs=0"} { + t.Run(v, func(t *testing.T) { + got := runTestProg(t, "testprog", "WindowsDontUpdateGOMAXPROCS", v) + if strings.Contains(got, "SKIP") { + t.Skip(got) + } + if !strings.Contains(got, "OK") { + t.Fatalf("output got %q want OK", got) + } + }) + } +} diff --git a/src/runtime/goroutineleakprofile_test.go b/src/runtime/goroutineleakprofile_test.go new file mode 100644 index 00000000000000..f5d2dd6372e54e --- /dev/null +++ b/src/runtime/goroutineleakprofile_test.go @@ -0,0 +1,603 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "fmt" + "internal/testenv" + "os" + "regexp" + "strings" + "testing" +) + +func TestGoroutineLeakProfile(t *testing.T) { + if strings.Contains(os.Getenv("GOFLAGS"), "mayMoreStackPreempt") { + // Some tests have false negatives under mayMoreStackPreempt. This may be a test-only issue, + // but needs more investigation. + testenv.SkipFlaky(t, 75729) + } + + // Goroutine leak test case. + // + // Test cases can be configured with test name, the name of the entry point function, + // a set of expected leaks identified by regular expressions, and the number of times + // the test should be repeated. + // + // Repeated runs reduce flakiness in some tests. + type testCase struct { + name string + simple bool + repetitions int + expectedLeaks map[*regexp.Regexp]bool + + // flakyLeaks are goroutine leaks that are too flaky to be reliably detected. + // Still, they might pop up every once in a while. The test will pass regardless + // if they occur or nor, as they are not unexpected. + // + // Note that all flaky leaks are true positives, i.e. real goroutine leaks, + // and it is only their detection that is unreliable due to scheduling + // non-determinism. + flakyLeaks map[*regexp.Regexp]struct{} + } + + // makeAnyTest is a short-hand for creating test cases. + // Each of the leaks in the list is identified by a regular expression. + // If a leak is flaky, it is added to the flakyLeaks map. + makeAnyTest := func(name string, flaky bool, repetitions int, leaks ...string) testCase { + tc := testCase{ + name: name, + expectedLeaks: make(map[*regexp.Regexp]bool, len(leaks)), + flakyLeaks: make(map[*regexp.Regexp]struct{}, len(leaks)), + // Make sure the test is repeated at least once. + repetitions: repetitions | 1, + } + + for _, leak := range leaks { + if !flaky { + tc.expectedLeaks[regexp.MustCompile(leak)] = false + } else { + tc.flakyLeaks[regexp.MustCompile(leak)] = struct{}{} + } + } + + return tc + } + + // makeTest is a short-hand for creating non-flaky test cases. + makeTest := func(name string, leaks ...string) testCase { + tcase := makeAnyTest(name, false, 2, leaks...) + tcase.simple = true + return tcase + } + + // makeFlakyTest is a short-hand for creating flaky test cases. + makeFlakyTest := func(name string, leaks ...string) testCase { + if testing.Short() { + return makeAnyTest(name, true, 2, leaks...) + } + return makeAnyTest(name, true, 10, leaks...) + } + + goroutineHeader := regexp.MustCompile(`goroutine \d+ \[`) + + // extractLeaks takes the output of a test and splits it into a + // list of strings denoting goroutine leaks. + // + // If the input is: + // + // goroutine 1 [wait reason (leaked)]: + // main.leaked() + // ./testdata/testgoroutineleakprofile/foo.go:37 +0x100 + // created by main.main() + // ./testdata/testgoroutineleakprofile/main.go:10 +0x20 + // + // goroutine 2 [wait reason (leaked)]: + // main.leaked2() + // ./testdata/testgoroutineleakprofile/foo.go:37 +0x100 + // created by main.main() + // ./testdata/testgoroutineleakprofile/main.go:10 +0x20 + // + // The output is (as a list of strings): + // + // leaked() [wait reason] + // leaked2() [wait reason] + extractLeaks := func(output string) []string { + stacks := strings.Split(output, "\n\ngoroutine") + var leaks []string + for _, stack := range stacks { + lines := strings.Split(stack, "\n") + if len(lines) < 5 { + // Expecting at least the following lines (where n=len(lines)-1): + // + // [0] goroutine n [wait reason (leaked)] + // ... + // [n-3] bottom.leak.frame(...) + // [n-2] ./bottom/leak/frame/source.go:line + // [n-1] created by go.instruction() + // [n] ./go/instruction/source.go:line + continue + } + + if !strings.Contains(lines[0], "(leaked)") { + // Ignore non-leaked goroutines. + continue + } + + // Get the wait reason from the goroutine header. + header := lines[0] + waitReason := goroutineHeader.ReplaceAllString(header, "[") + waitReason = strings.ReplaceAll(waitReason, " (leaked)", "") + + // Get the function name from the stack trace (should be two lines above `created by`). + var funcName string + for i := len(lines) - 1; i >= 0; i-- { + if strings.Contains(lines[i], "created by") { + funcName = strings.TrimPrefix(lines[i-2], "main.") + break + } + } + if funcName == "" { + t.Fatalf("failed to extract function name from stack trace: %s", lines) + } + + leaks = append(leaks, funcName+" "+waitReason) + } + return leaks + } + + // Micro tests involve very simple leaks for each type of concurrency primitive operation. + microTests := []testCase{ + makeTest("NilRecv", + `NilRecv\.func1\(.* \[chan receive \(nil chan\)\]`, + ), + makeTest("NilSend", + `NilSend\.func1\(.* \[chan send \(nil chan\)\]`, + ), + makeTest("SelectNoCases", + `SelectNoCases\.func1\(.* \[select \(no cases\)\]`, + ), + makeTest("ChanRecv", + `ChanRecv\.func1\(.* \[chan receive\]`, + ), + makeTest("ChanSend", + `ChanSend\.func1\(.* \[chan send\]`, + ), + makeTest("Select", + `Select\.func1\(.* \[select\]`, + ), + makeTest("WaitGroup", + `WaitGroup\.func1\(.* \[sync\.WaitGroup\.Wait\]`, + ), + makeTest("MutexStack", + `MutexStack\.func1\(.* \[sync\.Mutex\.Lock\]`, + ), + makeTest("MutexHeap", + `MutexHeap\.func1.1\(.* \[sync\.Mutex\.Lock\]`, + ), + makeTest("Cond", + `Cond\.func1\(.* \[sync\.Cond\.Wait\]`, + ), + makeTest("RWMutexRLock", + `RWMutexRLock\.func1\(.* \[sync\.RWMutex\.RLock\]`, + ), + makeTest("RWMutexLock", + `RWMutexLock\.func1\(.* \[sync\.(RW)?Mutex\.Lock\]`, + ), + makeTest("Mixed", + `Mixed\.func1\(.* \[sync\.WaitGroup\.Wait\]`, + `Mixed\.func1.1\(.* \[chan send\]`, + ), + makeTest("NoLeakGlobal"), + } + + // Stress tests are flaky and we do not strictly care about their output. + // They are only intended to stress the goroutine leak detector and profiling + // infrastructure in interesting ways. + stressTestCases := []testCase{ + makeFlakyTest("SpawnGC", + `spawnGC.func1\(.* \[chan receive\]`, + ), + makeTest("DaisyChain"), + } + + // Common goroutine leak patterns. + // Extracted from "Unveiling and Vanquishing Goroutine Leaks in Enterprise Microservices: A Dynamic Analysis Approach" + // doi:10.1109/CGO57630.2024.10444835 + patternTestCases := []testCase{ + makeTest("NoCloseRange", + `noCloseRange\(.* \[chan send\]`, + `noCloseRange\.func1\(.* \[chan receive\]`, + ), + makeTest("MethodContractViolation", + `worker\.Start\.func1\(.* \[select\]`, + ), + makeTest("DoubleSend", + `DoubleSend\.func3\(.* \[chan send\]`, + ), + makeTest("EarlyReturn", + `earlyReturn\.func1\(.* \[chan send\]`, + ), + makeTest("NCastLeak", + `nCastLeak\.func1\(.* \[chan send\]`, + `NCastLeak\.func2\(.* \[chan receive\]`, + ), + makeTest("Timeout", + // (vsaioc): Timeout is *theoretically* flaky, but the + // pseudo-random choice for select case branches makes it + // practically impossible for it to fail. + `timeout\.func1\(.* \[chan send\]`, + ), + } + + // GoKer tests from "GoBench: A Benchmark Suite of Real-World Go Concurrency Bugs". + // Refer to testdata/testgoroutineleakprofile/goker/README.md. + // + // This list is curated for tests that are not excessively flaky. + // Some tests are also excluded because they are redundant. + // + // TODO(vsaioc): Some of these might be removable (their patterns may overlap). + gokerTestCases := []testCase{ + makeFlakyTest("Cockroach584", + `Cockroach584\.func2\(.* \[sync\.Mutex\.Lock\]`, + ), + makeFlakyTest("Cockroach1055", + `Cockroach1055\.func2\(.* \[chan receive\]`, + `Cockroach1055\.func2\.2\(.* \[sync\.WaitGroup\.Wait\]`, + `Cockroach1055\.func2\.1\(.* \[chan receive\]`, + `Cockroach1055\.func2\.1\(.* \[sync\.Mutex\.Lock\]`, + ), + makeFlakyTest("Cockroach1462", + `\(\*Stopper_cockroach1462\)\.RunWorker\.func1\(.* \[chan send\]`, + `Cockroach1462\.func2\(.* \[sync\.WaitGroup\.Wait\]`, + ), + makeFlakyTest("Cockroach2448", + `\(\*Store_cockroach2448\)\.processRaft\(.* \[select\]`, + `\(\*state_cockroach2448\)\.start\(.* \[select\]`, + ), + makeFlakyTest("Cockroach3710", + `\(\*Store_cockroach3710\)\.ForceRaftLogScanAndProcess\(.* \[sync\.RWMutex\.RLock\]`, + `\(\*Store_cockroach3710\)\.processRaft\.func1\(.* \[sync\.RWMutex\.Lock\]`, + ), + makeFlakyTest("Cockroach6181", + `testRangeCacheCoalescedRequests_cockroach6181\(.* \[sync\.WaitGroup\.Wait\]`, + `testRangeCacheCoalescedRequests_cockroach6181\.func1\.1\(.* \[sync\.(RW)?Mutex\.Lock\]`, + `testRangeCacheCoalescedRequests_cockroach6181\.func1\.1\(.* \[sync\.RWMutex\.RLock\]`, + ), + makeTest("Cockroach7504", + `Cockroach7504\.func2\.1.* \[sync\.Mutex\.Lock\]`, + `Cockroach7504\.func2\.2.* \[sync\.Mutex\.Lock\]`, + ), + makeFlakyTest("Cockroach9935", + `\(\*loggingT_cockroach9935\)\.outputLogEntry\(.* \[sync\.Mutex\.Lock\]`, + ), + makeFlakyTest("Cockroach10214", + `\(*Store_cockroach10214\)\.sendQueuedHeartbeats\(.* \[sync\.Mutex\.Lock\]`, + `\(*Replica_cockroach10214\)\.tick\(.* \[sync\.Mutex\.Lock\]`, + ), + makeFlakyTest("Cockroach10790", + `\(\*Replica_cockroach10790\)\.beginCmds\.func1\(.* \[chan receive\]`, + ), + makeTest("Cockroach13197", + `\(\*Tx_cockroach13197\)\.awaitDone\(.* \[chan receive\]`, + ), + makeTest("Cockroach13755", + `\(\*Rows_cockroach13755\)\.awaitDone\(.* \[chan receive\]`, + ), + makeFlakyTest("Cockroach16167", + `Cockroach16167\.func2\(.* \[sync\.RWMutex\.RLock\]`, + `\(\*Executor_cockroach16167\)\.Start\(.* \[sync\.RWMutex\.Lock\]`, + ), + makeFlakyTest("Cockroach18101", + `restore_cockroach18101\.func1\(.* \[chan send\]`, + ), + makeTest("Cockroach24808", + `Cockroach24808\.func2\(.* \[chan send\]`, + ), + makeTest("Cockroach25456", + `Cockroach25456\.func2\(.* \[chan receive\]`, + ), + makeTest("Cockroach35073", + `Cockroach35073\.func2.1\(.* \[chan send\]`, + `Cockroach35073\.func2\(.* \[chan send\]`, + ), + makeTest("Cockroach35931", + `Cockroach35931\.func2\(.* \[chan send\]`, + ), + makeTest("Etcd5509", + `Etcd5509\.func2\(.* \[sync\.RWMutex\.Lock\]`, + ), + makeTest("Etcd6708", + `Etcd6708\.func2\(.* \[sync\.RWMutex\.RLock\]`, + ), + makeFlakyTest("Etcd6857", + `\(\*node_etcd6857\)\.Status\(.* \[chan send\]`, + ), + makeFlakyTest("Etcd6873", + `\(\*watchBroadcasts_etcd6873\)\.stop\(.* \[chan receive\]`, + `newWatchBroadcasts_etcd6873\.func1\(.* \[sync\.Mutex\.Lock\]`, + ), + makeFlakyTest("Etcd7492", + `Etcd7492\.func2\(.* \[sync\.WaitGroup\.Wait\]`, + `Etcd7492\.func2\.1\(.* \[chan send\]`, + `\(\*simpleTokenTTLKeeper_etcd7492\)\.run\(.* \[sync\.Mutex\.Lock\]`, + ), + makeFlakyTest("Etcd7902", + `doRounds_etcd7902\.func1\(.* \[chan receive\]`, + `doRounds_etcd7902\.func1\(.* \[sync\.Mutex\.Lock\]`, + `runElectionFunc_etcd7902\(.* \[sync\.WaitGroup\.Wait\]`, + ), + makeTest("Etcd10492", + `Etcd10492\.func2\(.* \[sync\.Mutex\.Lock\]`, + ), + makeTest("Grpc660", + `\(\*benchmarkClient_grpc660\)\.doCloseLoopUnary\.func1\(.* \[chan send\]`, + ), + makeFlakyTest("Grpc795", + `\(\*Server_grpc795\)\.Serve\(.* \[sync\.Mutex\.Lock\]`, + `testServerGracefulStopIdempotent_grpc795\(.* \[sync\.Mutex\.Lock\]`, + ), + makeTest("Grpc862", + `DialContext_grpc862\.func2\(.* \[chan receive\]`), + makeTest("Grpc1275", + `testInflightStreamClosing_grpc1275\.func1\(.* \[chan receive\]`), + makeTest("Grpc1424", + `DialContext_grpc1424\.func1\(.* \[chan receive\]`), + makeFlakyTest("Grpc1460", + `\(\*http2Client_grpc1460\)\.keepalive\(.* \[chan receive\]`, + `\(\*http2Client_grpc1460\)\.NewStream\(.* \[sync\.Mutex\.Lock\]`, + ), + makeFlakyTest("Grpc3017", + // grpc/3017 involves a goroutine leak that also simultaneously engages many GC assists. + `Grpc3017\.func2\(.* \[chan receive\]`, + `Grpc3017\.func2\.1\(.* \[sync\.Mutex\.Lock\]`, + `\(\*lbCacheClientConn_grpc3017\)\.RemoveSubConn\.func1\(.* \[sync\.Mutex\.Lock\]`, + ), + makeFlakyTest("Hugo3251", + `Hugo3251\.func2\(.* \[sync\.WaitGroup\.Wait\]`, + `Hugo3251\.func2\.1\(.* \[sync\.Mutex\.Lock\]`, + `Hugo3251\.func2\.1\(.* \[sync\.RWMutex\.RLock\]`, + ), + makeFlakyTest("Hugo5379", + `\(\*Page_hugo5379\)\.initContent\.func1\.1\(.* \[sync\.Mutex\.Lock\]`, + `pageRenderer_hugo5379\(.* \[sync\.Mutex\.Lock\]`, + `Hugo5379\.func2\(.* \[sync\.WaitGroup\.Wait\]`, + ), + makeFlakyTest("Istio16224", + `Istio16224\.func2\(.* \[sync\.Mutex\.Lock\]`, + `\(\*controller_istio16224\)\.Run\(.* \[chan send\]`, + `\(\*controller_istio16224\)\.Run\(.* \[chan receive\]`, + ), + makeFlakyTest("Istio17860", + `\(\*agent_istio17860\)\.runWait\(.* \[chan send\]`, + ), + makeFlakyTest("Istio18454", + `\(\*Worker_istio18454\)\.Start\.func1\(.* \[chan receive\]`, + `\(\*Worker_istio18454\)\.Start\.func1\(.* \[chan send\]`, + ), + // NOTE(vsaioc): + // Kubernetes/1321 is excluded due to a race condition in the original program + // that may, in extremely rare cases, lead to nil pointer dereference crashes. + // (Reproducible even with regular GC). Only kept here for posterity. + // + // makeTest(testCase{name: "Kubernetes1321"}, + // `NewMux_kubernetes1321\.gowrap1\(.* \[chan send\]`, + // `testMuxWatcherClose_kubernetes1321\(.* \[sync\.Mutex\.Lock\]`), + makeTest("Kubernetes5316", + `finishRequest_kubernetes5316\.func1\(.* \[chan send\]`, + ), + makeFlakyTest("Kubernetes6632", + `\(\*idleAwareFramer_kubernetes6632\)\.monitor\(.* \[sync\.Mutex\.Lock\]`, + `\(\*idleAwareFramer_kubernetes6632\)\.WriteFrame\(.* \[chan send\]`, + ), + makeFlakyTest("Kubernetes10182", + `\(\*statusManager_kubernetes10182\)\.Start\.func1\(.* \[sync\.Mutex\.Lock\]`, + `\(\*statusManager_kubernetes10182\)\.SetPodStatus\(.* \[chan send\]`, + ), + makeFlakyTest("Kubernetes11298", + `After_kubernetes11298\.func1\(.* \[chan receive\]`, + `After_kubernetes11298\.func1\(.* \[sync\.Cond\.Wait\]`, + `Kubernetes11298\.func2\(.* \[chan receive\]`, + ), + makeFlakyTest("Kubernetes13135", + `Util_kubernetes13135\(.* \[sync\.Mutex\.Lock\]`, + `\(\*WatchCache_kubernetes13135\)\.Add\(.* \[sync\.Mutex\.Lock\]`, + ), + makeTest("Kubernetes25331", + `\(\*watchChan_kubernetes25331\)\.run\(.* \[chan send\]`, + ), + makeFlakyTest("Kubernetes26980", + `Kubernetes26980\.func2\(.* \[chan receive\]`, + `Kubernetes26980\.func2\.1\(.* \[sync\.Mutex\.Lock\]`, + `\(\*processorListener_kubernetes26980\)\.pop\(.* \[chan receive\]`, + ), + makeFlakyTest("Kubernetes30872", + `\(\*DelayingDeliverer_kubernetes30872\)\.StartWithHandler\.func1\(.* \[sync\.Mutex\.Lock\]`, + `\(\*Controller_kubernetes30872\)\.Run\(.* \[sync\.Mutex\.Lock\]`, + `\(\*NamespaceController_kubernetes30872\)\.Run\.func1\(.* \[sync\.Mutex\.Lock\]`, + ), + makeTest("Kubernetes38669", + `\(\*cacheWatcher_kubernetes38669\)\.process\(.* \[chan send\]`, + ), + makeFlakyTest("Kubernetes58107", + `\(\*ResourceQuotaController_kubernetes58107\)\.worker\(.* \[sync\.Cond\.Wait\]`, + `\(\*ResourceQuotaController_kubernetes58107\)\.worker\(.* \[sync\.RWMutex\.RLock\]`, + `\(\*ResourceQuotaController_kubernetes58107\)\.Sync\(.* \[sync\.RWMutex\.Lock\]`, + ), + makeFlakyTest("Kubernetes62464", + `\(\*manager_kubernetes62464\)\.reconcileState\(.* \[sync\.RWMutex\.RLock\]`, + `\(\*staticPolicy_kubernetes62464\)\.RemoveContainer\(.* \[sync\.(RW)?Mutex\.Lock\]`, + ), + makeFlakyTest("Kubernetes70277", + `Kubernetes70277\.func2\(.* \[chan receive\]`, + ), + makeFlakyTest("Moby4951", + `\(\*DeviceSet_moby4951\)\.DeleteDevice\(.* \[sync\.Mutex\.Lock\]`, + ), + makeTest("Moby7559", + `\(\*UDPProxy_moby7559\)\.Run\(.* \[sync\.Mutex\.Lock\]`, + ), + makeTest("Moby17176", + `testDevmapperLockReleasedDeviceDeletion_moby17176\.func1\(.* \[sync\.Mutex\.Lock\]`, + ), + makeFlakyTest("Moby21233", + `\(\*Transfer_moby21233\)\.Watch\.func1\(.* \[chan send\]`, + `\(\*Transfer_moby21233\)\.Watch\.func1\(.* \[select\]`, + `testTransfer_moby21233\(.* \[chan receive\]`, + ), + makeTest("Moby25348", + `\(\*Manager_moby25348\)\.init\(.* \[sync\.WaitGroup\.Wait\]`, + ), + makeFlakyTest("Moby27782", + `\(\*JSONFileLogger_moby27782\)\.readLogs\(.* \[sync\.Cond\.Wait\]`, + `\(\*Watcher_moby27782\)\.readEvents\(.* \[select\]`, + ), + makeFlakyTest("Moby28462", + `monitor_moby28462\(.* \[sync\.Mutex\.Lock\]`, + `\(\*Daemon_moby28462\)\.StateChanged\(.* \[chan send\]`, + ), + makeTest("Moby30408", + `Moby30408\.func2\(.* \[chan receive\]`, + `testActive_moby30408\.func1\(.* \[sync\.Cond\.Wait\]`, + ), + makeFlakyTest("Moby33781", + `monitor_moby33781\.func1\(.* \[chan send\]`, + ), + makeFlakyTest("Moby36114", + `\(\*serviceVM_moby36114\)\.hotAddVHDsAtStart\(.* \[sync\.Mutex\.Lock\]`, + ), + makeFlakyTest("Serving2137", + `\(\*Breaker_serving2137\)\.concurrentRequest\.func1\(.* \[chan send\]`, + `\(\*Breaker_serving2137\)\.concurrentRequest\.func1\(.* \[sync\.Mutex\.Lock\]`, + `Serving2137\.func2\(.* \[chan receive\]`, + ), + makeTest("Syncthing4829", + `Syncthing4829\.func2\(.* \[sync\.RWMutex\.RLock\]`, + ), + makeTest("Syncthing5795", + `\(\*rawConnection_syncthing5795\)\.dispatcherLoop\(.* \[chan receive\]`, + `Syncthing5795\.func2.* \[chan receive\]`, + ), + } + + // Combine all test cases into a single list. + testCases := append(microTests, stressTestCases...) + testCases = append(testCases, patternTestCases...) + + // Test cases must not panic or cause fatal exceptions. + failStates := regexp.MustCompile(`fatal|panic|DATA RACE`) + + testApp := func(exepath string, testCases []testCase) { + + // Build the test program once. + exe, err := buildTestProg(t, exepath) + if err != nil { + t.Fatal(fmt.Sprintf("building testgoroutineleakprofile failed: %v", err)) + } + + for _, tcase := range testCases { + t.Run(tcase.name, func(t *testing.T) { + t.Parallel() + + cmdEnv := []string{ + "GODEBUG=asyncpreemptoff=1", + "GOEXPERIMENT=greenteagc,goroutineleakprofile", + } + + if tcase.simple { + // If the test is simple, set GOMAXPROCS=1 in order to better + // control the behavior of the scheduler. + cmdEnv = append(cmdEnv, "GOMAXPROCS=1") + } + + var output string + for i := 0; i < tcase.repetitions; i++ { + // Run program for one repetition and get runOutput trace. + runOutput := runBuiltTestProg(t, exe, tcase.name, cmdEnv...) + if len(runOutput) == 0 { + t.Errorf("Test %s produced no output. Is the goroutine leak profile collected?", tcase.name) + } + + // Zero tolerance policy for fatal exceptions, panics, or data races. + if failStates.MatchString(runOutput) { + t.Errorf("unexpected fatal exception or panic\noutput:\n%s\n\n", runOutput) + } + + output += runOutput + "\n\n" + } + + // Extract all the goroutine leaks + foundLeaks := extractLeaks(output) + + // If the test case was not expected to produce leaks, but some were reported, + // stop the test immediately. Zero tolerance policy for false positives. + if len(tcase.expectedLeaks)+len(tcase.flakyLeaks) == 0 && len(foundLeaks) > 0 { + t.Errorf("output:\n%s\n\ngoroutines leaks detected in case with no leaks", output) + } + + unexpectedLeaks := make([]string, 0, len(foundLeaks)) + + // Parse every leak and check if it is expected (maybe as a flaky leak). + leaks: + for _, leak := range foundLeaks { + // Check if the leak is expected. + // If it is, check whether it has been encountered before. + var foundNew bool + var leakPattern *regexp.Regexp + + for expectedLeak, ok := range tcase.expectedLeaks { + if expectedLeak.MatchString(leak) { + if !ok { + foundNew = true + } + + leakPattern = expectedLeak + break + } + } + + if foundNew { + // Only bother writing if we found a new leak. + tcase.expectedLeaks[leakPattern] = true + } + + if leakPattern == nil { + // We are dealing with a leak not marked as expected. + // Check if it is a flaky leak. + for flakyLeak := range tcase.flakyLeaks { + if flakyLeak.MatchString(leak) { + // The leak is flaky. Carry on to the next line. + continue leaks + } + } + + unexpectedLeaks = append(unexpectedLeaks, leak) + } + } + + missingLeakStrs := make([]string, 0, len(tcase.expectedLeaks)) + for expectedLeak, found := range tcase.expectedLeaks { + if !found { + missingLeakStrs = append(missingLeakStrs, expectedLeak.String()) + } + } + + var errors []error + if len(unexpectedLeaks) > 0 { + errors = append(errors, fmt.Errorf("unexpected goroutine leaks:\n%s\n", strings.Join(unexpectedLeaks, "\n"))) + } + if len(missingLeakStrs) > 0 { + errors = append(errors, fmt.Errorf("missing expected leaks:\n%s\n", strings.Join(missingLeakStrs, ", "))) + } + if len(errors) > 0 { + t.Fatalf("Failed with the following errors:\n%s\n\noutput:\n%s", errors, output) + } + }) + } + } + + testApp("testgoroutineleakprofile", testCases) + testApp("testgoroutineleakprofile/goker", gokerTestCases) +} diff --git a/src/runtime/hash32.go b/src/runtime/hash32.go index 0616c7dd050751..206a308f12e25a 100644 --- a/src/runtime/hash32.go +++ b/src/runtime/hash32.go @@ -5,7 +5,7 @@ // Hashing algorithm inspired by // wyhash: https://github.com/wangyi-fudan/wyhash/blob/ceb019b530e2c1c14d70b79bfa2bc49de7d95bc1/Modern%20Non-Cryptographic%20Hash%20Function%20and%20Pseudorandom%20Number%20Generator.pdf -//go:build 386 || arm || mips || mipsle +//go:build 386 || arm || mips || mipsle || wasm || (gccgo && (ppc || s390)) package runtime diff --git a/src/runtime/hash64.go b/src/runtime/hash64.go index 124bb7d77accdd..ac26e660c6d44a 100644 --- a/src/runtime/hash64.go +++ b/src/runtime/hash64.go @@ -5,7 +5,7 @@ // Hashing algorithm inspired by // wyhash: https://github.com/wangyi-fudan/wyhash -//go:build amd64 || arm64 || loong64 || mips64 || mips64le || ppc64 || ppc64le || riscv64 || s390x || wasm +//go:build amd64 || arm64 || loong64 || mips64 || mips64le || ppc64 || ppc64le || riscv64 || s390x package runtime diff --git a/src/runtime/hash_test.go b/src/runtime/hash_test.go index 24c04b260e3841..c4e9f5ab89ba9f 100644 --- a/src/runtime/hash_test.go +++ b/src/runtime/hash_test.go @@ -7,6 +7,7 @@ package runtime_test import ( "encoding/binary" "fmt" + "internal/byteorder" "internal/race" "internal/testenv" "math" @@ -326,10 +327,7 @@ func genPerm(h *HashSet, b []byte, s []uint32, n int) { return } for _, v := range s { - b[n] = byte(v) - b[n+1] = byte(v >> 8) - b[n+2] = byte(v >> 16) - b[n+3] = byte(v >> 24) + byteorder.LEPutUint32(b[n:], v) genPerm(h, b, s, n+4) } } @@ -638,11 +636,10 @@ func TestSmhasherSeed(t *testing.T) { } func TestIssue66841(t *testing.T) { - testenv.MustHaveExec(t) if *UseAeshash && os.Getenv("TEST_ISSUE_66841") == "" { // We want to test the backup hash, so if we're running on a machine // that uses aeshash, exec ourselves while turning aes off. - cmd := testenv.CleanCmdEnv(testenv.Command(t, os.Args[0], "-test.run=^TestIssue66841$")) + cmd := testenv.CleanCmdEnv(testenv.Command(t, testenv.Executable(t), "-test.run=^TestIssue66841$")) cmd.Env = append(cmd.Env, "GODEBUG=cpu.aes=off", "TEST_ISSUE_66841=1") out, err := cmd.CombinedOutput() if err != nil { diff --git a/src/runtime/heapdump.go b/src/runtime/heapdump.go index 95fb62dc425cda..9aaee8f90c9d26 100644 --- a/src/runtime/heapdump.go +++ b/src/runtime/heapdump.go @@ -14,6 +14,7 @@ package runtime import ( "internal/abi" "internal/goarch" + "internal/runtime/gc" "unsafe" ) @@ -205,7 +206,7 @@ func dumptype(t *_type) { dwritebyte('.') dwrite(unsafe.Pointer(unsafe.StringData(name)), uintptr(len(name))) } - dumpbool(t.Kind_&abi.KindDirectIface == 0 || t.PtrBytes != 0) + dumpbool(!t.IsDirectIface() || t.Pointers()) } // dump an object. @@ -414,7 +415,7 @@ func dumpgs() { default: print("runtime: unexpected G.status ", hex(status), "\n") throw("dumpgs in STW - bad status") - case _Gdead: + case _Gdead, _Gdeadextra: // ok case _Grunnable, _Gsyscall, @@ -459,7 +460,7 @@ func dumproots() { continue } spf := (*specialfinalizer)(unsafe.Pointer(sp)) - p := unsafe.Pointer(s.base() + uintptr(spf.special.offset)) + p := unsafe.Pointer(s.base() + spf.special.offset) dumpfinalizer(p, spf.fn, spf.fint, spf.ot) } } @@ -471,7 +472,7 @@ func dumproots() { // Bit vector of free marks. // Needs to be as big as the largest number of objects per span. -var freemark [_PageSize / 8]bool +var freemark [pageSize / 8]bool func dumpobjs() { // To protect mheap_.allspans. @@ -483,7 +484,7 @@ func dumpobjs() { } p := s.base() size := s.elemsize - n := (s.npages << _PageShift) / size + n := (s.npages << gc.PageShift) / size if n > uintptr(len(freemark)) { throw("freemark array doesn't have enough entries") } @@ -535,7 +536,7 @@ func dumpparams() { dumpint(uint64(arenaEnd)) dumpstr(goarch.GOARCH) dumpstr(buildVersion) - dumpint(uint64(ncpu)) + dumpint(uint64(numCPUStartup)) } func itab_callback(tab *itab) { @@ -658,7 +659,7 @@ func dumpmemprof() { continue } spp := (*specialprofile)(unsafe.Pointer(sp)) - p := s.base() + uintptr(spp.special.offset) + p := s.base() + spp.special.offset dumpint(tagAllocSample) dumpint(uint64(p)) dumpint(uint64(uintptr(unsafe.Pointer(spp.b)))) @@ -727,7 +728,7 @@ func makeheapobjbv(p uintptr, size uintptr) bitvector { sysFree(unsafe.Pointer(&tmpbuf[0]), uintptr(len(tmpbuf)), &memstats.other_sys) } n := nptr/8 + 1 - p := sysAlloc(n, &memstats.other_sys) + p := sysAlloc(n, &memstats.other_sys, "heapdump") if p == nil { throw("heapdump: out of memory") } diff --git a/src/runtime/iface.go b/src/runtime/iface.go index 5316182f4ab24a..0665c4b9840ced 100644 --- a/src/runtime/iface.go +++ b/src/runtime/iface.go @@ -333,7 +333,7 @@ var ( // be used as the second word of an interface value. func convT(t *_type, v unsafe.Pointer) unsafe.Pointer { if raceenabled { - raceReadObjectPC(t, v, getcallerpc(), abi.FuncPCABIInternal(convT)) + raceReadObjectPC(t, v, sys.GetCallerPC(), abi.FuncPCABIInternal(convT)) } if msanenabled { msanread(v, t.Size_) @@ -348,7 +348,7 @@ func convT(t *_type, v unsafe.Pointer) unsafe.Pointer { func convTnoptr(t *_type, v unsafe.Pointer) unsafe.Pointer { // TODO: maybe take size instead of type? if raceenabled { - raceReadObjectPC(t, v, getcallerpc(), abi.FuncPCABIInternal(convTnoptr)) + raceReadObjectPC(t, v, sys.GetCallerPC(), abi.FuncPCABIInternal(convTnoptr)) } if msanenabled { msanread(v, t.Size_) @@ -474,7 +474,7 @@ func typeAssert(s *abi.TypeAssert, t *_type) *itab { tab = getitab(s.Inter, t, s.CanFail) } - if !abi.UseInterfaceSwitchCache(GOARCH) { + if !abi.UseInterfaceSwitchCache(goarch.ArchFamily) { return tab } @@ -574,7 +574,7 @@ func interfaceSwitch(s *abi.InterfaceSwitch, t *_type) (int, *itab) { } } - if !abi.UseInterfaceSwitchCache(GOARCH) { + if !abi.UseInterfaceSwitchCache(goarch.ArchFamily) { return case_, tab } @@ -692,39 +692,16 @@ func iterate_itabs(fn func(*itab)) { } // staticuint64s is used to avoid allocating in convTx for small integer values. -var staticuint64s = [...]uint64{ - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, - 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, - 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, - 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, - 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, +// staticuint64s[0] == 0, staticuint64s[1] == 1, and so forth. +// It is defined in assembler code so that it is read-only. +var staticuint64s [256]uint64 + +// getStaticuint64s is called by the reflect package to get a pointer +// to the read-only array. +// +//go:linkname getStaticuint64s +func getStaticuint64s() *[256]uint64 { + return &staticuint64s } // The linker redirects a reference of a method that it determined diff --git a/src/runtime/iface_test.go b/src/runtime/iface_test.go index 06f6eeb9524637..5bc209cfcf0dc7 100644 --- a/src/runtime/iface_test.go +++ b/src/runtime/iface_test.go @@ -60,7 +60,7 @@ func TestCmpIfaceConcreteAlloc(t *testing.T) { t.Skip("skipping on non-gc compiler") } - n := testing.AllocsPerRun(1, func() { + n := testing.AllocsPerRun(100, func() { _ = e == ts _ = i1 == ts _ = e == 1 diff --git a/src/runtime/ints.s b/src/runtime/ints.s new file mode 100644 index 00000000000000..b816a2fb76b42d --- /dev/null +++ b/src/runtime/ints.s @@ -0,0 +1,264 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +DATA ·staticuint64s+0x000(SB)/8, $0 +DATA ·staticuint64s+0x008(SB)/8, $1 +DATA ·staticuint64s+0x010(SB)/8, $2 +DATA ·staticuint64s+0x018(SB)/8, $3 +DATA ·staticuint64s+0x020(SB)/8, $4 +DATA ·staticuint64s+0x028(SB)/8, $5 +DATA ·staticuint64s+0x030(SB)/8, $6 +DATA ·staticuint64s+0x038(SB)/8, $7 +DATA ·staticuint64s+0x040(SB)/8, $8 +DATA ·staticuint64s+0x048(SB)/8, $9 +DATA ·staticuint64s+0x050(SB)/8, $10 +DATA ·staticuint64s+0x058(SB)/8, $11 +DATA ·staticuint64s+0x060(SB)/8, $12 +DATA ·staticuint64s+0x068(SB)/8, $13 +DATA ·staticuint64s+0x070(SB)/8, $14 +DATA ·staticuint64s+0x078(SB)/8, $15 +DATA ·staticuint64s+0x080(SB)/8, $16 +DATA ·staticuint64s+0x088(SB)/8, $17 +DATA ·staticuint64s+0x090(SB)/8, $18 +DATA ·staticuint64s+0x098(SB)/8, $19 +DATA ·staticuint64s+0x0a0(SB)/8, $20 +DATA ·staticuint64s+0x0a8(SB)/8, $21 +DATA ·staticuint64s+0x0b0(SB)/8, $22 +DATA ·staticuint64s+0x0b8(SB)/8, $23 +DATA ·staticuint64s+0x0c0(SB)/8, $24 +DATA ·staticuint64s+0x0c8(SB)/8, $25 +DATA ·staticuint64s+0x0d0(SB)/8, $26 +DATA ·staticuint64s+0x0d8(SB)/8, $27 +DATA ·staticuint64s+0x0e0(SB)/8, $28 +DATA ·staticuint64s+0x0e8(SB)/8, $29 +DATA ·staticuint64s+0x0f0(SB)/8, $30 +DATA ·staticuint64s+0x0f8(SB)/8, $31 +DATA ·staticuint64s+0x100(SB)/8, $32 +DATA ·staticuint64s+0x108(SB)/8, $33 +DATA ·staticuint64s+0x110(SB)/8, $34 +DATA ·staticuint64s+0x118(SB)/8, $35 +DATA ·staticuint64s+0x120(SB)/8, $36 +DATA ·staticuint64s+0x128(SB)/8, $37 +DATA ·staticuint64s+0x130(SB)/8, $38 +DATA ·staticuint64s+0x138(SB)/8, $39 +DATA ·staticuint64s+0x140(SB)/8, $40 +DATA ·staticuint64s+0x148(SB)/8, $41 +DATA ·staticuint64s+0x150(SB)/8, $42 +DATA ·staticuint64s+0x158(SB)/8, $43 +DATA ·staticuint64s+0x160(SB)/8, $44 +DATA ·staticuint64s+0x168(SB)/8, $45 +DATA ·staticuint64s+0x170(SB)/8, $46 +DATA ·staticuint64s+0x178(SB)/8, $47 +DATA ·staticuint64s+0x180(SB)/8, $48 +DATA ·staticuint64s+0x188(SB)/8, $49 +DATA ·staticuint64s+0x190(SB)/8, $50 +DATA ·staticuint64s+0x198(SB)/8, $51 +DATA ·staticuint64s+0x1a0(SB)/8, $52 +DATA ·staticuint64s+0x1a8(SB)/8, $53 +DATA ·staticuint64s+0x1b0(SB)/8, $54 +DATA ·staticuint64s+0x1b8(SB)/8, $55 +DATA ·staticuint64s+0x1c0(SB)/8, $56 +DATA ·staticuint64s+0x1c8(SB)/8, $57 +DATA ·staticuint64s+0x1d0(SB)/8, $58 +DATA ·staticuint64s+0x1d8(SB)/8, $59 +DATA ·staticuint64s+0x1e0(SB)/8, $60 +DATA ·staticuint64s+0x1e8(SB)/8, $61 +DATA ·staticuint64s+0x1f0(SB)/8, $62 +DATA ·staticuint64s+0x1f8(SB)/8, $63 +DATA ·staticuint64s+0x200(SB)/8, $64 +DATA ·staticuint64s+0x208(SB)/8, $65 +DATA ·staticuint64s+0x210(SB)/8, $66 +DATA ·staticuint64s+0x218(SB)/8, $67 +DATA ·staticuint64s+0x220(SB)/8, $68 +DATA ·staticuint64s+0x228(SB)/8, $69 +DATA ·staticuint64s+0x230(SB)/8, $70 +DATA ·staticuint64s+0x238(SB)/8, $71 +DATA ·staticuint64s+0x240(SB)/8, $72 +DATA ·staticuint64s+0x248(SB)/8, $73 +DATA ·staticuint64s+0x250(SB)/8, $74 +DATA ·staticuint64s+0x258(SB)/8, $75 +DATA ·staticuint64s+0x260(SB)/8, $76 +DATA ·staticuint64s+0x268(SB)/8, $77 +DATA ·staticuint64s+0x270(SB)/8, $78 +DATA ·staticuint64s+0x278(SB)/8, $79 +DATA ·staticuint64s+0x280(SB)/8, $80 +DATA ·staticuint64s+0x288(SB)/8, $81 +DATA ·staticuint64s+0x290(SB)/8, $82 +DATA ·staticuint64s+0x298(SB)/8, $83 +DATA ·staticuint64s+0x2a0(SB)/8, $84 +DATA ·staticuint64s+0x2a8(SB)/8, $85 +DATA ·staticuint64s+0x2b0(SB)/8, $86 +DATA ·staticuint64s+0x2b8(SB)/8, $87 +DATA ·staticuint64s+0x2c0(SB)/8, $88 +DATA ·staticuint64s+0x2c8(SB)/8, $89 +DATA ·staticuint64s+0x2d0(SB)/8, $90 +DATA ·staticuint64s+0x2d8(SB)/8, $91 +DATA ·staticuint64s+0x2e0(SB)/8, $92 +DATA ·staticuint64s+0x2e8(SB)/8, $93 +DATA ·staticuint64s+0x2f0(SB)/8, $94 +DATA ·staticuint64s+0x2f8(SB)/8, $95 +DATA ·staticuint64s+0x300(SB)/8, $96 +DATA ·staticuint64s+0x308(SB)/8, $97 +DATA ·staticuint64s+0x310(SB)/8, $98 +DATA ·staticuint64s+0x318(SB)/8, $99 +DATA ·staticuint64s+0x320(SB)/8, $100 +DATA ·staticuint64s+0x328(SB)/8, $101 +DATA ·staticuint64s+0x330(SB)/8, $102 +DATA ·staticuint64s+0x338(SB)/8, $103 +DATA ·staticuint64s+0x340(SB)/8, $104 +DATA ·staticuint64s+0x348(SB)/8, $105 +DATA ·staticuint64s+0x350(SB)/8, $106 +DATA ·staticuint64s+0x358(SB)/8, $107 +DATA ·staticuint64s+0x360(SB)/8, $108 +DATA ·staticuint64s+0x368(SB)/8, $109 +DATA ·staticuint64s+0x370(SB)/8, $110 +DATA ·staticuint64s+0x378(SB)/8, $111 +DATA ·staticuint64s+0x380(SB)/8, $112 +DATA ·staticuint64s+0x388(SB)/8, $113 +DATA ·staticuint64s+0x390(SB)/8, $114 +DATA ·staticuint64s+0x398(SB)/8, $115 +DATA ·staticuint64s+0x3a0(SB)/8, $116 +DATA ·staticuint64s+0x3a8(SB)/8, $117 +DATA ·staticuint64s+0x3b0(SB)/8, $118 +DATA ·staticuint64s+0x3b8(SB)/8, $119 +DATA ·staticuint64s+0x3c0(SB)/8, $120 +DATA ·staticuint64s+0x3c8(SB)/8, $121 +DATA ·staticuint64s+0x3d0(SB)/8, $122 +DATA ·staticuint64s+0x3d8(SB)/8, $123 +DATA ·staticuint64s+0x3e0(SB)/8, $124 +DATA ·staticuint64s+0x3e8(SB)/8, $125 +DATA ·staticuint64s+0x3f0(SB)/8, $126 +DATA ·staticuint64s+0x3f8(SB)/8, $127 +DATA ·staticuint64s+0x400(SB)/8, $128 +DATA ·staticuint64s+0x408(SB)/8, $129 +DATA ·staticuint64s+0x410(SB)/8, $130 +DATA ·staticuint64s+0x418(SB)/8, $131 +DATA ·staticuint64s+0x420(SB)/8, $132 +DATA ·staticuint64s+0x428(SB)/8, $133 +DATA ·staticuint64s+0x430(SB)/8, $134 +DATA ·staticuint64s+0x438(SB)/8, $135 +DATA ·staticuint64s+0x440(SB)/8, $136 +DATA ·staticuint64s+0x448(SB)/8, $137 +DATA ·staticuint64s+0x450(SB)/8, $138 +DATA ·staticuint64s+0x458(SB)/8, $139 +DATA ·staticuint64s+0x460(SB)/8, $140 +DATA ·staticuint64s+0x468(SB)/8, $141 +DATA ·staticuint64s+0x470(SB)/8, $142 +DATA ·staticuint64s+0x478(SB)/8, $143 +DATA ·staticuint64s+0x480(SB)/8, $144 +DATA ·staticuint64s+0x488(SB)/8, $145 +DATA ·staticuint64s+0x490(SB)/8, $146 +DATA ·staticuint64s+0x498(SB)/8, $147 +DATA ·staticuint64s+0x4a0(SB)/8, $148 +DATA ·staticuint64s+0x4a8(SB)/8, $149 +DATA ·staticuint64s+0x4b0(SB)/8, $150 +DATA ·staticuint64s+0x4b8(SB)/8, $151 +DATA ·staticuint64s+0x4c0(SB)/8, $152 +DATA ·staticuint64s+0x4c8(SB)/8, $153 +DATA ·staticuint64s+0x4d0(SB)/8, $154 +DATA ·staticuint64s+0x4d8(SB)/8, $155 +DATA ·staticuint64s+0x4e0(SB)/8, $156 +DATA ·staticuint64s+0x4e8(SB)/8, $157 +DATA ·staticuint64s+0x4f0(SB)/8, $158 +DATA ·staticuint64s+0x4f8(SB)/8, $159 +DATA ·staticuint64s+0x500(SB)/8, $160 +DATA ·staticuint64s+0x508(SB)/8, $161 +DATA ·staticuint64s+0x510(SB)/8, $162 +DATA ·staticuint64s+0x518(SB)/8, $163 +DATA ·staticuint64s+0x520(SB)/8, $164 +DATA ·staticuint64s+0x528(SB)/8, $165 +DATA ·staticuint64s+0x530(SB)/8, $166 +DATA ·staticuint64s+0x538(SB)/8, $167 +DATA ·staticuint64s+0x540(SB)/8, $168 +DATA ·staticuint64s+0x548(SB)/8, $169 +DATA ·staticuint64s+0x550(SB)/8, $170 +DATA ·staticuint64s+0x558(SB)/8, $171 +DATA ·staticuint64s+0x560(SB)/8, $172 +DATA ·staticuint64s+0x568(SB)/8, $173 +DATA ·staticuint64s+0x570(SB)/8, $174 +DATA ·staticuint64s+0x578(SB)/8, $175 +DATA ·staticuint64s+0x580(SB)/8, $176 +DATA ·staticuint64s+0x588(SB)/8, $177 +DATA ·staticuint64s+0x590(SB)/8, $178 +DATA ·staticuint64s+0x598(SB)/8, $179 +DATA ·staticuint64s+0x5a0(SB)/8, $180 +DATA ·staticuint64s+0x5a8(SB)/8, $181 +DATA ·staticuint64s+0x5b0(SB)/8, $182 +DATA ·staticuint64s+0x5b8(SB)/8, $183 +DATA ·staticuint64s+0x5c0(SB)/8, $184 +DATA ·staticuint64s+0x5c8(SB)/8, $185 +DATA ·staticuint64s+0x5d0(SB)/8, $186 +DATA ·staticuint64s+0x5d8(SB)/8, $187 +DATA ·staticuint64s+0x5e0(SB)/8, $188 +DATA ·staticuint64s+0x5e8(SB)/8, $189 +DATA ·staticuint64s+0x5f0(SB)/8, $190 +DATA ·staticuint64s+0x5f8(SB)/8, $191 +DATA ·staticuint64s+0x600(SB)/8, $192 +DATA ·staticuint64s+0x608(SB)/8, $193 +DATA ·staticuint64s+0x610(SB)/8, $194 +DATA ·staticuint64s+0x618(SB)/8, $195 +DATA ·staticuint64s+0x620(SB)/8, $196 +DATA ·staticuint64s+0x628(SB)/8, $197 +DATA ·staticuint64s+0x630(SB)/8, $198 +DATA ·staticuint64s+0x638(SB)/8, $199 +DATA ·staticuint64s+0x640(SB)/8, $200 +DATA ·staticuint64s+0x648(SB)/8, $201 +DATA ·staticuint64s+0x650(SB)/8, $202 +DATA ·staticuint64s+0x658(SB)/8, $203 +DATA ·staticuint64s+0x660(SB)/8, $204 +DATA ·staticuint64s+0x668(SB)/8, $205 +DATA ·staticuint64s+0x670(SB)/8, $206 +DATA ·staticuint64s+0x678(SB)/8, $207 +DATA ·staticuint64s+0x680(SB)/8, $208 +DATA ·staticuint64s+0x688(SB)/8, $209 +DATA ·staticuint64s+0x690(SB)/8, $210 +DATA ·staticuint64s+0x698(SB)/8, $211 +DATA ·staticuint64s+0x6a0(SB)/8, $212 +DATA ·staticuint64s+0x6a8(SB)/8, $213 +DATA ·staticuint64s+0x6b0(SB)/8, $214 +DATA ·staticuint64s+0x6b8(SB)/8, $215 +DATA ·staticuint64s+0x6c0(SB)/8, $216 +DATA ·staticuint64s+0x6c8(SB)/8, $217 +DATA ·staticuint64s+0x6d0(SB)/8, $218 +DATA ·staticuint64s+0x6d8(SB)/8, $219 +DATA ·staticuint64s+0x6e0(SB)/8, $220 +DATA ·staticuint64s+0x6e8(SB)/8, $221 +DATA ·staticuint64s+0x6f0(SB)/8, $222 +DATA ·staticuint64s+0x6f8(SB)/8, $223 +DATA ·staticuint64s+0x700(SB)/8, $224 +DATA ·staticuint64s+0x708(SB)/8, $225 +DATA ·staticuint64s+0x710(SB)/8, $226 +DATA ·staticuint64s+0x718(SB)/8, $227 +DATA ·staticuint64s+0x720(SB)/8, $228 +DATA ·staticuint64s+0x728(SB)/8, $229 +DATA ·staticuint64s+0x730(SB)/8, $230 +DATA ·staticuint64s+0x738(SB)/8, $231 +DATA ·staticuint64s+0x740(SB)/8, $232 +DATA ·staticuint64s+0x748(SB)/8, $233 +DATA ·staticuint64s+0x750(SB)/8, $234 +DATA ·staticuint64s+0x758(SB)/8, $235 +DATA ·staticuint64s+0x760(SB)/8, $236 +DATA ·staticuint64s+0x768(SB)/8, $237 +DATA ·staticuint64s+0x770(SB)/8, $238 +DATA ·staticuint64s+0x778(SB)/8, $239 +DATA ·staticuint64s+0x780(SB)/8, $240 +DATA ·staticuint64s+0x788(SB)/8, $241 +DATA ·staticuint64s+0x790(SB)/8, $242 +DATA ·staticuint64s+0x798(SB)/8, $243 +DATA ·staticuint64s+0x7a0(SB)/8, $244 +DATA ·staticuint64s+0x7a8(SB)/8, $245 +DATA ·staticuint64s+0x7b0(SB)/8, $246 +DATA ·staticuint64s+0x7b8(SB)/8, $247 +DATA ·staticuint64s+0x7c0(SB)/8, $248 +DATA ·staticuint64s+0x7c8(SB)/8, $249 +DATA ·staticuint64s+0x7d0(SB)/8, $250 +DATA ·staticuint64s+0x7d8(SB)/8, $251 +DATA ·staticuint64s+0x7e0(SB)/8, $252 +DATA ·staticuint64s+0x7e8(SB)/8, $253 +DATA ·staticuint64s+0x7f0(SB)/8, $254 +DATA ·staticuint64s+0x7f8(SB)/8, $255 + +GLOBL ·staticuint64s(SB), RODATA, $0x800 diff --git a/src/runtime/lfstack.go b/src/runtime/lfstack.go index cbec6e84472629..1e2f5a296544f8 100644 --- a/src/runtime/lfstack.go +++ b/src/runtime/lfstack.go @@ -24,10 +24,6 @@ type lfstack uint64 func (head *lfstack) push(node *lfnode) { node.pushcnt++ new := lfstackPack(node, node.pushcnt) - if node1 := lfstackUnpack(new); node1 != node { - print("runtime: lfstack.push invalid packing: node=", node, " cnt=", hex(node.pushcnt), " packed=", hex(new), " -> node=", node1, "\n") - throw("lfstack.push") - } for { old := atomic.Load64((*uint64)(head)) node.next = old @@ -38,6 +34,11 @@ func (head *lfstack) push(node *lfnode) { } func (head *lfstack) pop() unsafe.Pointer { + var backoff uint32 + // TODO: tweak backoff parameters on other architectures. + if GOARCH == "arm64" { + backoff = 128 + } for { old := atomic.Load64((*uint64)(head)) if old == 0 { @@ -48,6 +49,16 @@ func (head *lfstack) pop() unsafe.Pointer { if atomic.Cas64((*uint64)(head), old, next) { return unsafe.Pointer(node) } + + // Use a backoff approach to reduce demand to the shared memory location + // decreases memory contention and allows for other threads to make quicker + // progress. + // Read more in this Arm blog post: + // https://community.arm.com/arm-community-blogs/b/architectures-and-processors-blog/posts/multi-threaded-applications-arm + procyield(backoff) + // Increase backoff time. + backoff += backoff / 2 + } } @@ -61,15 +72,11 @@ func lfnodeValidate(node *lfnode) { if base, _, _ := findObject(uintptr(unsafe.Pointer(node)), 0, 0); base != 0 { throw("lfstack node allocated from the heap") } - if lfstackUnpack(lfstackPack(node, ^uintptr(0))) != node { - printlock() - println("runtime: bad lfnode address", hex(uintptr(unsafe.Pointer(node)))) - throw("bad lfnode address") - } + lfstackPack(node, ^uintptr(0)) } func lfstackPack(node *lfnode, cnt uintptr) uint64 { - return uint64(taggedPointerPack(unsafe.Pointer(node), cnt)) + return uint64(taggedPointerPack(unsafe.Pointer(node), cnt&(1<(SB), NOSPLIT, $0-0 + MOVV R4, R12 // fn + MOVV R5, RARG0 // hookId + MOVV R6, RARG1 // s1 + MOVV R7, RARG2 // s2 + MOVV R8, RARG3 // result + + MOVV g_m(g), R13 + + // Switch to g0 stack. + MOVV R3, R23 // callee-saved, preserved across the CALL + MOVV m_g0(R13), R14 + BEQ R14, g, call // already on g0 + MOVV (g_sched+gobuf_sp)(R14), R3 + +call: + JAL (R12) + MOVV R23, R3 + RET + +// void runtime·libfuzzerCallWithTwoByteBuffers(fn, start, end *byte) +// Calls C function fn from libFuzzer and passes 2 arguments of type *byte to it. +TEXT runtime·libfuzzerCallWithTwoByteBuffers(SB), NOSPLIT, $0-0 + MOVV R4, R12 // fn + MOVV R5, RARG0 // start + MOVV R6, RARG1 // end + + MOVV g_m(g), R13 + + // Switch to g0 stack. + MOVV R3, R23 // callee-saved, preserved across the CALL + MOVV m_g0(R13), R14 + BEQ R14, g, call // already on g0 + MOVV (g_sched+gobuf_sp)(R14), R3 + +call: + JAL (R12) + MOVV R23, R3 + RET + +// void runtime·libfuzzerCallTraceIntCmp(fn, arg0, arg1, fakePC uintptr) +// Calls C function fn from libFuzzer and passes 2 arguments to it after +// manipulating the return address so that libfuzzer's integer compare hooks +// work. +// The problem statement and solution are documented in detail in libfuzzer_amd64.s. +// See commentary there. +TEXT runtime·libfuzzerCallTraceIntCmp(SB), NOSPLIT, $0-0 + MOVV R4, R12 // fn + MOVV R5, RARG0 // arg0 + MOVV R6, RARG1 // arg1 + // Save the original return address in a local variable + MOVV R1, savedRetAddr-8(SP) + + MOVV g_m(g), R13 + + // Switch to g0 stack. + MOVV R3, R23 // callee-saved, preserved across the CALL + MOVV m_g0(R13), R14 + BEQ R14, g, call // already on g0 + MOVV (g_sched+gobuf_sp)(R14), R3 + +call: + // Load address of the ret sled into the default register for the return + // address. + MOVV $ret_sled(SB), R1 + // Clear the lowest 2 bits of fakePC. All Loong64 instructions are four + // bytes long, so we cannot get better return address granularity than + // multiples of 4. + AND $-4, R7 + // Load the address of the i'th return instruction from the return sled. + // The index is given in the fakePC argument. + ADDV R7, R1 + // Call the function by jumping to it and reusing all registers except + // for the modified return address register R1. + JMP (R12) + +// The ret sled for Loong64 consists of 128 br instructions jumping to the +// end of the function. Each instruction is 4 bytes long. The sled thus has +// the same byte length of 4 * 128 = 512 as the x86_64 sled, but coarser +// granularity. +#define RET_SLED \ + JMP end_of_function; + +TEXT ret_sled(SB), NOSPLIT, $0-0 + REPEAT_128(RET_SLED); + +end_of_function: + MOVV R23, R3 + MOVV savedRetAddr-8(SP), R1 + RET diff --git a/src/runtime/linkname.go b/src/runtime/linkname.go index dd7f67425103e3..eec190c39cfd1e 100644 --- a/src/runtime/linkname.go +++ b/src/runtime/linkname.go @@ -13,6 +13,7 @@ import _ "unsafe" //go:linkname _cgo_panic_internal //go:linkname cgoAlwaysFalse //go:linkname cgoUse +//go:linkname cgoKeepAlive //go:linkname cgoCheckPointer //go:linkname cgoCheckResult //go:linkname cgoNoCallback diff --git a/src/runtime/linkname_shim.go b/src/runtime/linkname_shim.go new file mode 100644 index 00000000000000..4ba3d1fb783cf5 --- /dev/null +++ b/src/runtime/linkname_shim.go @@ -0,0 +1,208 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "internal/abi" + "internal/runtime/maps" + "internal/runtime/sys" + "unsafe" +) + +// Legacy //go:linkname compatibility shims +// +// The functions below are unused by the toolchain, and exist only for +// compatibility with existing //go:linkname use in the ecosystem. + +// linknameIter is the it argument to mapiterinit and mapiternext. +// +// Callers of mapiterinit allocate their own iter structure, which has the +// layout of the pre-Go 1.24 hiter structure, shown here for posterity: +// +// type hiter struct { +// key unsafe.Pointer +// elem unsafe.Pointer +// t *maptype // old map abi.Type +// h *hmap +// buckets unsafe.Pointer +// bptr *bmap +// overflow *[]*bmap +// oldoverflow *[]*bmap +// startBucket uintptr +// offset uint8 +// wrapped bool +// B uint8 +// i uint8 +// bucket uintptr +// checkBucket uintptr +// } +// +// Our structure must maintain compatibility with the old structure. This +// means: +// +// - Our structure must be the same size or smaller than hiter. Otherwise we +// may write outside the caller's hiter allocation. +// - Our structure must have the same pointer layout as hiter, so that the GC +// tracks pointers properly. +// +// Based on analysis of the "hall of shame" users of these linknames: +// +// - The key and elem fields must be kept up to date with the current key/elem. +// Some users directly access the key and elem fields rather than calling +// reflect.mapiterkey/reflect.mapiterelem. +// - The t field must be non-nil after mapiterinit. gonum.org/v1/gonum uses +// this to verify the iterator is initialized. +// - github.com/segmentio/encoding and github.com/RomiChan/protobuf check if h +// is non-nil, but the code has no effect. Thus the value of h does not +// matter. See internal/runtime_reflect/map.go. +type linknameIter struct { + // Fields from hiter. + key unsafe.Pointer + elem unsafe.Pointer + typ *abi.MapType + + // The real iterator. + it *maps.Iter +} + +// mapiterinit is a compatibility wrapper for map iterator for users of +// //go:linkname from before Go 1.24. It is not used by Go itself. New users +// should use reflect or the maps package. +// +// mapiterinit should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/bytedance/sonic +// - github.com/goccy/go-json +// - github.com/RomiChan/protobuf +// - github.com/segmentio/encoding +// - github.com/ugorji/go/codec +// - github.com/wI2L/jettison +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname mapiterinit +func mapiterinit(t *abi.MapType, m *maps.Map, it *linknameIter) { + if raceenabled && m != nil { + callerpc := sys.GetCallerPC() + racereadpc(unsafe.Pointer(m), callerpc, abi.FuncPCABIInternal(mapiterinit)) + } + + it.typ = t + + it.it = new(maps.Iter) + it.it.Init(t, m) + it.it.Next() + + it.key = it.it.Key() + it.elem = it.it.Elem() +} + +// reflect_mapiterinit is a compatibility wrapper for map iterator for users of +// //go:linkname from before Go 1.24. It is not used by Go itself. New users +// should use reflect or the maps package. +// +// reflect_mapiterinit should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/modern-go/reflect2 +// - gitee.com/quant1x/gox +// - github.com/v2pro/plz +// - github.com/wI2L/jettison +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname reflect_mapiterinit reflect.mapiterinit +func reflect_mapiterinit(t *abi.MapType, m *maps.Map, it *linknameIter) { + mapiterinit(t, m, it) +} + +// mapiternext is a compatibility wrapper for map iterator for users of +// //go:linkname from before Go 1.24. It is not used by Go itself. New users +// should use reflect or the maps package. +// +// mapiternext should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/bytedance/sonic +// - github.com/RomiChan/protobuf +// - github.com/segmentio/encoding +// - github.com/ugorji/go/codec +// - gonum.org/v1/gonum +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname mapiternext +func mapiternext(it *linknameIter) { + if raceenabled { + callerpc := sys.GetCallerPC() + racereadpc(unsafe.Pointer(it.it.Map()), callerpc, abi.FuncPCABIInternal(mapiternext)) + } + + it.it.Next() + + it.key = it.it.Key() + it.elem = it.it.Elem() +} + +// reflect_mapiternext is a compatibility wrapper for map iterator for users of +// //go:linkname from before Go 1.24. It is not used by Go itself. New users +// should use reflect or the maps package. +// +// reflect_mapiternext is for package reflect, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - gitee.com/quant1x/gox +// - github.com/modern-go/reflect2 +// - github.com/goccy/go-json +// - github.com/v2pro/plz +// - github.com/wI2L/jettison +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname reflect_mapiternext reflect.mapiternext +func reflect_mapiternext(it *linknameIter) { + mapiternext(it) +} + +// reflect_mapiterkey is a compatibility wrapper for map iterator for users of +// //go:linkname from before Go 1.24. It is not used by Go itself. New users +// should use reflect or the maps package. +// +// reflect_mapiterkey should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/goccy/go-json +// - gonum.org/v1/gonum +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname reflect_mapiterkey reflect.mapiterkey +func reflect_mapiterkey(it *linknameIter) unsafe.Pointer { + return it.it.Key() +} + +// reflect_mapiterelem is a compatibility wrapper for map iterator for users of +// //go:linkname from before Go 1.24. It is not used by Go itself. New users +// should use reflect or the maps package. +// +// reflect_mapiterelem should be an internal detail, +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/goccy/go-json +// - gonum.org/v1/gonum +// +// Do not remove or change the type signature. +// See go.dev/issue/67401. +// +//go:linkname reflect_mapiterelem reflect.mapiterelem +func reflect_mapiterelem(it *linknameIter) unsafe.Pointer { + return it.it.Elem() +} diff --git a/src/runtime/lock_futex.go b/src/runtime/lock_futex.go index 58690e45e4d5a8..1cf146b43c0557 100644 --- a/src/runtime/lock_futex.go +++ b/src/runtime/lock_futex.go @@ -11,32 +11,6 @@ import ( "unsafe" ) -// This implementation depends on OS-specific implementations of -// -// futexsleep(addr *uint32, val uint32, ns int64) -// Atomically, -// if *addr == val { sleep } -// Might be woken up spuriously; that's allowed. -// Don't sleep longer than ns; ns < 0 means forever. -// -// futexwakeup(addr *uint32, cnt uint32) -// If any procs are sleeping on addr, wake up at most cnt. - -const ( - mutex_unlocked = 0 - mutex_locked = 1 - mutex_sleeping = 2 - - active_spin = 4 - active_spin_cnt = 30 - passive_spin = 1 -) - -// Possible lock states are mutex_unlocked, mutex_locked and mutex_sleeping. -// mutex_sleeping means that there is presumably at least one sleeping thread. -// Note that there can be spinning threads during all states - they do not -// affect mutex's state. - // We use the uintptr mutex.key and note.key as a uint32. // //go:nosplit @@ -44,103 +18,6 @@ func key32(p *uintptr) *uint32 { return (*uint32)(unsafe.Pointer(p)) } -func mutexContended(l *mutex) bool { - return atomic.Load(key32(&l.key)) > mutex_locked -} - -func lock(l *mutex) { - lockWithRank(l, getLockRank(l)) -} - -func lock2(l *mutex) { - gp := getg() - - if gp.m.locks < 0 { - throw("runtime·lock: lock count") - } - gp.m.locks++ - - // Speculative grab for lock. - v := atomic.Xchg(key32(&l.key), mutex_locked) - if v == mutex_unlocked { - return - } - - // wait is either MUTEX_LOCKED or MUTEX_SLEEPING - // depending on whether there is a thread sleeping - // on this mutex. If we ever change l->key from - // MUTEX_SLEEPING to some other value, we must be - // careful to change it back to MUTEX_SLEEPING before - // returning, to ensure that the sleeping thread gets - // its wakeup call. - wait := v - - timer := &lockTimer{lock: l} - timer.begin() - // On uniprocessors, no point spinning. - // On multiprocessors, spin for ACTIVE_SPIN attempts. - spin := 0 - if ncpu > 1 { - spin = active_spin - } - for { - // Try for lock, spinning. - for i := 0; i < spin; i++ { - for l.key == mutex_unlocked { - if atomic.Cas(key32(&l.key), mutex_unlocked, wait) { - timer.end() - return - } - } - procyield(active_spin_cnt) - } - - // Try for lock, rescheduling. - for i := 0; i < passive_spin; i++ { - for l.key == mutex_unlocked { - if atomic.Cas(key32(&l.key), mutex_unlocked, wait) { - timer.end() - return - } - } - osyield() - } - - // Sleep. - v = atomic.Xchg(key32(&l.key), mutex_sleeping) - if v == mutex_unlocked { - timer.end() - return - } - wait = mutex_sleeping - futexsleep(key32(&l.key), mutex_sleeping, -1) - } -} - -func unlock(l *mutex) { - unlockWithRank(l) -} - -func unlock2(l *mutex) { - v := atomic.Xchg(key32(&l.key), mutex_unlocked) - if v == mutex_unlocked { - throw("unlock of unlocked lock") - } - if v == mutex_sleeping { - futexwakeup(key32(&l.key), 1) - } - - gp := getg() - gp.m.mLockProfile.recordUnlock(l) - gp.m.locks-- - if gp.m.locks < 0 { - throw("runtime·unlock: lock count") - } - if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack - gp.stackguard0 = stackPreempt - } -} - // One-time notifications. func noteclear(n *note) { n.key = 0 @@ -254,3 +131,33 @@ func beforeIdle(int64, int64) (*g, bool) { } func checkTimeouts() {} + +//go:nosplit +func semacreate(mp *m) {} + +//go:nosplit +func semasleep(ns int64) int32 { + mp := getg().m + + for v := atomic.Xadd(&mp.waitsema, -1); ; v = atomic.Load(&mp.waitsema) { + if int32(v) >= 0 { + return 0 + } + futexsleep(&mp.waitsema, v, ns) + if ns >= 0 { + if int32(v) >= 0 { + return 0 + } else { + return -1 + } + } + } +} + +//go:nosplit +func semawakeup(mp *m) { + v := atomic.Xadd(&mp.waitsema, 1) + if v == 0 { + futexwakeup(&mp.waitsema, 1) + } +} diff --git a/src/runtime/lock_js.go b/src/runtime/lock_js.go index b6ee5ec7afe269..a40e3010850011 100644 --- a/src/runtime/lock_js.go +++ b/src/runtime/lock_js.go @@ -6,7 +6,10 @@ package runtime -import _ "unsafe" // for go:linkname +import ( + "internal/runtime/sys" + _ "unsafe" // for go:linkname +) // js/wasm has no support for threads yet. There is no preemption. @@ -23,6 +26,10 @@ const ( passive_spin = 1 ) +type mWaitList struct{} + +func lockVerifyMSize() {} + func mutexContended(l *mutex) bool { return false } @@ -63,29 +70,21 @@ func unlock2(l *mutex) { // One-time notifications. -type noteWithTimeout struct { - gp *g - deadline int64 -} - -var ( - notes = make(map[*note]*g) - notesWithTimeout = make(map[*note]noteWithTimeout) -) +// Linked list of notes with a deadline. +var allDeadlineNotes *note func noteclear(n *note) { - n.key = note_cleared + n.status = note_cleared } func notewakeup(n *note) { - // gp := getg() - if n.key == note_woken { + if n.status == note_woken { throw("notewakeup - double wakeup") } - cleared := n.key == note_cleared - n.key = note_woken + cleared := n.status == note_cleared + n.status = note_woken if cleared { - goready(notes[n], 1) + goready(n.gp, 1) } } @@ -113,48 +112,50 @@ func notetsleepg(n *note, ns int64) bool { } id := scheduleTimeoutEvent(delay) - mp := acquirem() - notes[n] = gp - notesWithTimeout[n] = noteWithTimeout{gp: gp, deadline: deadline} - releasem(mp) + + n.gp = gp + n.deadline = deadline + if allDeadlineNotes != nil { + allDeadlineNotes.allprev = n + } + n.allnext = allDeadlineNotes + allDeadlineNotes = n gopark(nil, nil, waitReasonSleep, traceBlockSleep, 1) clearTimeoutEvent(id) // note might have woken early, clear timeout - mp = acquirem() - delete(notes, n) - delete(notesWithTimeout, n) - releasem(mp) + n.gp = nil + n.deadline = 0 + if n.allprev != nil { + n.allprev.allnext = n.allnext + } + if allDeadlineNotes == n { + allDeadlineNotes = n.allnext + } + n.allprev = nil + n.allnext = nil - return n.key == note_woken + return n.status == note_woken } - for n.key != note_woken { - mp := acquirem() - notes[n] = gp - releasem(mp) + for n.status != note_woken { + n.gp = gp gopark(nil, nil, waitReasonZero, traceBlockGeneric, 1) - mp = acquirem() - delete(notes, n) - releasem(mp) + n.gp = nil } return true } // checkTimeouts resumes goroutines that are waiting on a note which has reached its deadline. -// TODO(drchase): need to understand if write barriers are really okay in this context. -// -//go:yeswritebarrierrec func checkTimeouts() { now := nanotime() - // TODO: map iteration has the write barriers in it; is that okay? - for n, nt := range notesWithTimeout { - if n.key == note_cleared && now >= nt.deadline { - n.key = note_timeout - goready(nt.gp, 1) + for n := allDeadlineNotes; n != nil; n = n.allnext { + if n.status == note_cleared && n.deadline != 0 && now >= n.deadline { + n.status = note_timeout + goready(n.gp, 1) } } } @@ -250,7 +251,7 @@ var idleStart int64 func handleAsyncEvent() { idleStart = nanotime() - pause(getcallersp() - 16) + pause(sys.GetCallerSP() - 16) } // clearIdleTimeout clears our record of the timeout started by beforeIdle. @@ -259,9 +260,6 @@ func clearIdleTimeout() { idleTimeout = nil } -// pause sets SP to newsp and pauses the execution of Go's WebAssembly code until an event is triggered. -func pause(newsp uintptr) - // scheduleTimeoutEvent tells the WebAssembly environment to trigger an event after ms milliseconds. // It returns a timer id that can be used with clearTimeoutEvent. // @@ -300,7 +298,7 @@ func handleEvent() { // return execution to JavaScript idleStart = nanotime() - pause(getcallersp() - 16) + pause(sys.GetCallerSP() - 16) } // eventHandler retrieves and executes handlers for pending JavaScript events. diff --git a/src/runtime/lock_sema.go b/src/runtime/lock_sema.go index 32d2235ad3ab90..3e1b07b918b62e 100644 --- a/src/runtime/lock_sema.go +++ b/src/runtime/lock_sema.go @@ -11,131 +11,10 @@ import ( "unsafe" ) -// This implementation depends on OS-specific implementations of -// -// func semacreate(mp *m) -// Create a semaphore for mp, if it does not already have one. -// -// func semasleep(ns int64) int32 -// If ns < 0, acquire m's semaphore and return 0. -// If ns >= 0, try to acquire m's semaphore for at most ns nanoseconds. -// Return 0 if the semaphore was acquired, -1 if interrupted or timed out. -// -// func semawakeup(mp *m) -// Wake up mp, which is or will soon be sleeping on its semaphore. const ( locked uintptr = 1 - - active_spin = 4 - active_spin_cnt = 30 - passive_spin = 1 ) -func mutexContended(l *mutex) bool { - return atomic.Loaduintptr(&l.key) > locked -} - -func lock(l *mutex) { - lockWithRank(l, getLockRank(l)) -} - -func lock2(l *mutex) { - gp := getg() - if gp.m.locks < 0 { - throw("runtime·lock: lock count") - } - gp.m.locks++ - - // Speculative grab for lock. - if atomic.Casuintptr(&l.key, 0, locked) { - return - } - semacreate(gp.m) - - timer := &lockTimer{lock: l} - timer.begin() - // On uniprocessor's, no point spinning. - // On multiprocessors, spin for ACTIVE_SPIN attempts. - spin := 0 - if ncpu > 1 { - spin = active_spin - } -Loop: - for i := 0; ; i++ { - v := atomic.Loaduintptr(&l.key) - if v&locked == 0 { - // Unlocked. Try to lock. - if atomic.Casuintptr(&l.key, v, v|locked) { - timer.end() - return - } - i = 0 - } - if i < spin { - procyield(active_spin_cnt) - } else if i < spin+passive_spin { - osyield() - } else { - // Someone else has it. - // l->waitm points to a linked list of M's waiting - // for this lock, chained through m->nextwaitm. - // Queue this M. - for { - gp.m.nextwaitm = muintptr(v &^ locked) - if atomic.Casuintptr(&l.key, v, uintptr(unsafe.Pointer(gp.m))|locked) { - break - } - v = atomic.Loaduintptr(&l.key) - if v&locked == 0 { - continue Loop - } - } - if v&locked != 0 { - // Queued. Wait. - semasleep(-1) - i = 0 - } - } - } -} - -func unlock(l *mutex) { - unlockWithRank(l) -} - -// We might not be holding a p in this code. -// -//go:nowritebarrier -func unlock2(l *mutex) { - gp := getg() - var mp *m - for { - v := atomic.Loaduintptr(&l.key) - if v == locked { - if atomic.Casuintptr(&l.key, locked, 0) { - break - } - } else { - // Other M's are waiting for the lock. - // Dequeue an M. - mp = muintptr(v &^ locked).ptr() - if atomic.Casuintptr(&l.key, v, uintptr(mp.nextwaitm)) { - // Dequeued an M. Wake it. - semawakeup(mp) - break - } - } - } - gp.m.mLockProfile.recordUnlock(l) - gp.m.locks-- - if gp.m.locks < 0 { - throw("runtime·unlock: lock count") - } - if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack - gp.stackguard0 = stackPreempt - } -} - // One-time notifications. func noteclear(n *note) { n.key = 0 diff --git a/src/runtime/lock_spinbit.go b/src/runtime/lock_spinbit.go new file mode 100644 index 00000000000000..039ea6f5656a8a --- /dev/null +++ b/src/runtime/lock_spinbit.go @@ -0,0 +1,468 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !wasm + +package runtime + +import ( + "internal/goarch" + "internal/runtime/atomic" + "internal/runtime/gc" + "unsafe" +) + +// This implementation depends on OS-specific implementations of +// +// func semacreate(mp *m) +// Create a semaphore for mp, if it does not already have one. +// +// func semasleep(ns int64) int32 +// If ns < 0, acquire m's semaphore and return 0. +// If ns >= 0, try to acquire m's semaphore for at most ns nanoseconds. +// Return 0 if the semaphore was acquired, -1 if interrupted or timed out. +// +// func semawakeup(mp *m) +// Wake up mp, which is or will soon be sleeping on its semaphore. + +// The mutex state consists of four flags and a pointer. The flag at bit 0, +// mutexLocked, represents the lock itself. Bit 1, mutexSleeping, is a hint that +// the pointer is non-nil. The fast paths for locking and unlocking the mutex +// are based on atomic 8-bit swap operations on the low byte; bits 2 through 7 +// are unused. +// +// Bit 8, mutexSpinning, is a try-lock that grants a waiting M permission to +// spin on the state word. Most other Ms must attempt to spend their time +// sleeping to reduce traffic on the cache line. This is the "spin bit" for +// which the implementation is named. (The anti-starvation mechanism also grants +// temporary permission for an M to spin.) +// +// Bit 9, mutexStackLocked, is a try-lock that grants an unlocking M permission +// to inspect the list of waiting Ms and to pop an M off of that stack. +// +// The upper bits hold a (partial) pointer to the M that most recently went to +// sleep. The sleeping Ms form a stack linked by their mWaitList.next fields. +// Because the fast paths use an 8-bit swap on the low byte of the state word, +// we'll need to reconstruct the full M pointer from the bits we have. Most Ms +// are allocated on the heap, and have a known alignment and base offset. (The +// offset is due to mallocgc's allocation headers.) The main program thread uses +// a static M value, m0. We check for m0 specifically and add a known offset +// otherwise. + +const ( + active_spin = 4 // referenced in proc.go for sync.Mutex implementation + active_spin_cnt = 30 // referenced in proc.go for sync.Mutex implementation +) + +const ( + mutexLocked = 0x001 + mutexSleeping = 0x002 + mutexSpinning = 0x100 + mutexStackLocked = 0x200 + mutexMMask = 0x3FF + mutexMOffset = gc.MallocHeaderSize // alignment of heap-allocated Ms (those other than m0) + + mutexActiveSpinCount = 4 + mutexActiveSpinSize = 30 + mutexPassiveSpinCount = 1 + + mutexTailWakePeriod = 16 +) + +//go:nosplit +func key8(p *uintptr) *uint8 { + if goarch.BigEndian { + return &(*[8]uint8)(unsafe.Pointer(p))[goarch.PtrSize/1-1] + } + return &(*[8]uint8)(unsafe.Pointer(p))[0] +} + +// mWaitList is part of the M struct, and holds the list of Ms that are waiting +// for a particular runtime.mutex. +// +// When an M is unable to immediately obtain a lock, it adds itself to the list +// of Ms waiting for the lock. It does that via this struct's next field, +// forming a singly-linked list with the mutex's key field pointing to the head +// of the list. +type mWaitList struct { + next muintptr // next m waiting for lock + startTicks int64 // when this m started waiting for the current lock holder, in cputicks +} + +// lockVerifyMSize confirms that we can recreate the low bits of the M pointer. +func lockVerifyMSize() { + size := roundupsize(unsafe.Sizeof(mPadded{}), false) + gc.MallocHeaderSize + if size&mutexMMask != 0 { + print("M structure uses sizeclass ", size, "/", hex(size), " bytes; ", + "incompatible with mutex flag mask ", hex(mutexMMask), "\n") + throw("runtime.m memory alignment too small for spinbit mutex") + } +} + +// mutexWaitListHead recovers a full muintptr that was missing its low bits. +// With the exception of the static m0 value, it requires allocating runtime.m +// values in a size class with a particular minimum alignment. The 2048-byte +// size class allows recovering the full muintptr value even after overwriting +// the low 11 bits with flags. We can use those 11 bits as 3 flags and an +// atomically-swapped byte. +// +//go:nosplit +func mutexWaitListHead(v uintptr) muintptr { + if highBits := v &^ mutexMMask; highBits == 0 { + return 0 + } else if m0bits := muintptr(unsafe.Pointer(&m0)); highBits == uintptr(m0bits)&^mutexMMask { + return m0bits + } else { + return muintptr(highBits + mutexMOffset) + } +} + +// mutexPreferLowLatency reports if this mutex prefers low latency at the risk +// of performance collapse. If so, we can allow all waiting threads to spin on +// the state word rather than go to sleep. +// +// TODO: We could have the waiting Ms each spin on their own private cache line, +// especially if we can put a bound on the on-CPU time that would consume. +// +// TODO: If there's a small set of mutex values with special requirements, they +// could make use of a more specialized lock2/unlock2 implementation. Otherwise, +// we're constrained to what we can fit within a single uintptr with no +// additional storage on the M for each lock held. +// +//go:nosplit +func mutexPreferLowLatency(l *mutex) bool { + switch l { + default: + return false + case &sched.lock: + // We often expect sched.lock to pass quickly between Ms in a way that + // each M has unique work to do: for instance when we stop-the-world + // (bringing each P to idle) or add new netpoller-triggered work to the + // global run queue. + return true + } +} + +func mutexContended(l *mutex) bool { + return atomic.Loaduintptr(&l.key)&^mutexMMask != 0 +} + +func lock(l *mutex) { + lockWithRank(l, getLockRank(l)) +} + +func lock2(l *mutex) { + gp := getg() + if gp.m.locks < 0 { + throw("runtime·lock: lock count") + } + gp.m.locks++ + + k8 := key8(&l.key) + + // Speculative grab for lock. + v8 := atomic.Xchg8(k8, mutexLocked) + if v8&mutexLocked == 0 { + if v8&mutexSleeping != 0 { + atomic.Or8(k8, mutexSleeping) + } + return + } + semacreate(gp.m) + + var startTime int64 + // On uniprocessors, no point spinning. + // On multiprocessors, spin for mutexActiveSpinCount attempts. + spin := 0 + if numCPUStartup > 1 { + spin = mutexActiveSpinCount + } + + var weSpin, atTail, haveTimers bool + v := atomic.Loaduintptr(&l.key) +tryAcquire: + for i := 0; ; i++ { + if v&mutexLocked == 0 { + if weSpin { + next := (v &^ mutexSpinning) | mutexSleeping | mutexLocked + if next&^mutexMMask == 0 { + // The fast-path Xchg8 may have cleared mutexSleeping. Fix + // the hint so unlock2 knows when to use its slow path. + next = next &^ mutexSleeping + } + if atomic.Casuintptr(&l.key, v, next) { + gp.m.mLockProfile.end(startTime) + return + } + } else { + prev8 := atomic.Xchg8(k8, mutexLocked|mutexSleeping) + if prev8&mutexLocked == 0 { + gp.m.mLockProfile.end(startTime) + return + } + } + v = atomic.Loaduintptr(&l.key) + continue tryAcquire + } + + if !weSpin && v&mutexSpinning == 0 && atomic.Casuintptr(&l.key, v, v|mutexSpinning) { + v |= mutexSpinning + weSpin = true + } + + if weSpin || atTail || mutexPreferLowLatency(l) { + if i < spin { + procyield(mutexActiveSpinSize) + v = atomic.Loaduintptr(&l.key) + continue tryAcquire + } else if i < spin+mutexPassiveSpinCount { + osyield() // TODO: Consider removing this step. See https://go.dev/issue/69268. + v = atomic.Loaduintptr(&l.key) + continue tryAcquire + } + } + + // Go to sleep + if v&mutexLocked == 0 { + throw("runtime·lock: sleeping while lock is available") + } + + // Collect times for mutex profile (seen in unlock2 only via mWaitList), + // and for "/sync/mutex/wait/total:seconds" metric (to match). + if !haveTimers { + gp.m.mWaitList.startTicks = cputicks() + startTime = gp.m.mLockProfile.start() + haveTimers = true + } + // Store the current head of the list of sleeping Ms in our gp.m.mWaitList.next field + gp.m.mWaitList.next = mutexWaitListHead(v) + + // Pack a (partial) pointer to this M with the current lock state bits + next := (uintptr(unsafe.Pointer(gp.m)) &^ mutexMMask) | v&mutexMMask | mutexSleeping + if weSpin { // If we were spinning, prepare to retire + next = next &^ mutexSpinning + } + + if atomic.Casuintptr(&l.key, v, next) { + weSpin = false + // We've pushed ourselves onto the stack of waiters. Wait. + semasleep(-1) + atTail = gp.m.mWaitList.next == 0 // we were at risk of starving + i = 0 + } + + gp.m.mWaitList.next = 0 + v = atomic.Loaduintptr(&l.key) + } +} + +func unlock(l *mutex) { + unlockWithRank(l) +} + +// We might not be holding a p in this code. +// +//go:nowritebarrier +func unlock2(l *mutex) { + gp := getg() + + var prev8 uint8 + var haveStackLock bool + var endTicks int64 + if !mutexSampleContention() { + // Not collecting a sample for the contention profile, do the quick release + prev8 = atomic.Xchg8(key8(&l.key), 0) + } else { + // If there's contention, we'll sample it. Don't allow another + // lock2/unlock2 pair to finish before us and take our blame. Prevent + // that by trading for the stack lock with a CAS. + v := atomic.Loaduintptr(&l.key) + for { + if v&^mutexMMask == 0 || v&mutexStackLocked != 0 { + // No contention, or (stack lock unavailable) no way to calculate it + prev8 = atomic.Xchg8(key8(&l.key), 0) + endTicks = 0 + break + } + + // There's contention, the stack lock appeared to be available, and + // we'd like to collect a sample for the contention profile. + if endTicks == 0 { + // Read the time before releasing the lock. The profile will be + // strictly smaller than what other threads would see by timing + // their lock calls. + endTicks = cputicks() + } + next := (v | mutexStackLocked) &^ (mutexLocked | mutexSleeping) + if atomic.Casuintptr(&l.key, v, next) { + haveStackLock = true + prev8 = uint8(v) + // The fast path of lock2 may have cleared mutexSleeping. + // Restore it so we're sure to call unlock2Wake below. + prev8 |= mutexSleeping + break + } + v = atomic.Loaduintptr(&l.key) + } + } + if prev8&mutexLocked == 0 { + throw("unlock of unlocked lock") + } + + if prev8&mutexSleeping != 0 { + unlock2Wake(l, haveStackLock, endTicks) + } + + gp.m.mLockProfile.store() + gp.m.locks-- + if gp.m.locks < 0 { + throw("runtime·unlock: lock count") + } + if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack + gp.stackguard0 = stackPreempt + } +} + +// mutexSampleContention returns whether the current mutex operation should +// report any contention it discovers. +func mutexSampleContention() bool { + if rate := int64(atomic.Load64(&mutexprofilerate)); rate <= 0 { + return false + } else { + // TODO: have SetMutexProfileFraction do the clamping + rate32 := uint32(rate) + if int64(rate32) != rate { + rate32 = ^uint32(0) + } + return cheaprandn(rate32) == 0 + } +} + +// unlock2Wake updates the list of Ms waiting on l, waking an M if necessary. +// +//go:nowritebarrier +func unlock2Wake(l *mutex, haveStackLock bool, endTicks int64) { + v := atomic.Loaduintptr(&l.key) + + // On occasion, seek out and wake the M at the bottom of the stack so it + // doesn't starve. + antiStarve := cheaprandn(mutexTailWakePeriod) == 0 + + if haveStackLock { + goto useStackLock + } + + if !(antiStarve || // avoiding starvation may require a wake + v&mutexSpinning == 0 || // no spinners means we must wake + mutexPreferLowLatency(l)) { // prefer waiters be awake as much as possible + return + } + + for { + if v&^mutexMMask == 0 || v&mutexStackLocked != 0 { + // No waiting Ms means nothing to do. + // + // If the stack lock is unavailable, its owner would make the same + // wake decisions that we would, so there's nothing for us to do. + // + // Although: This thread may have a different call stack, which + // would result in a different entry in the mutex contention profile + // (upon completion of go.dev/issue/66999). That could lead to weird + // results if a slow critical section ends but another thread + // quickly takes the lock, finishes its own critical section, + // releases the lock, and then grabs the stack lock. That quick + // thread would then take credit (blame) for the delay that this + // slow thread caused. The alternative is to have more expensive + // atomic operations (a CAS) on the critical path of unlock2. + return + } + // Other M's are waiting for the lock. + // Obtain the stack lock, and pop off an M. + next := v | mutexStackLocked + if atomic.Casuintptr(&l.key, v, next) { + break + } + v = atomic.Loaduintptr(&l.key) + } + + // We own the mutexStackLocked flag. New Ms may push themselves onto the + // stack concurrently, but we're now the only thread that can remove or + // modify the Ms that are sleeping in the list. +useStackLock: + + if endTicks != 0 { + // Find the M at the bottom of the stack of waiters, which has been + // asleep for the longest. Take the average of its wait time and the + // head M's wait time for the mutex contention profile, matching the + // estimate we do in semrelease1 (for sync.Mutex contention). + // + // We don't keep track of the tail node (we don't need it often), so do + // an O(N) walk on the list of sleeping Ms to find it. + head := mutexWaitListHead(v).ptr() + for node, n := head, 0; ; { + n++ + next := node.mWaitList.next.ptr() + if next == nil { + cycles := ((endTicks - head.mWaitList.startTicks) + (endTicks - node.mWaitList.startTicks)) / 2 + node.mWaitList.startTicks = endTicks + head.mWaitList.startTicks = endTicks + getg().m.mLockProfile.recordUnlock(cycles * int64(n)) + break + } + node = next + } + } + + var committed *m // If we choose an M within the stack, we've made a promise to wake it + for { + headM := v &^ mutexMMask + flags := v & (mutexMMask &^ mutexStackLocked) // preserve low bits, but release stack lock + + mp := mutexWaitListHead(v).ptr() + wakem := committed + if committed == nil { + if v&mutexSpinning == 0 || mutexPreferLowLatency(l) { + wakem = mp + } + if antiStarve { + // Wake the M at the bottom of the stack of waiters. (This is + // O(N) with the number of waiters.) + wakem = mp + prev := mp + for { + next := wakem.mWaitList.next.ptr() + if next == nil { + break + } + prev, wakem = wakem, next + } + if wakem != mp { + committed = wakem + prev.mWaitList.next = wakem.mWaitList.next + // An M sets its own startTicks when it first goes to sleep. + // When an unlock operation is sampled for the mutex + // contention profile, it takes blame for the entire list of + // waiting Ms but only updates the startTicks value at the + // tail. Copy any updates to the next-oldest M. + prev.mWaitList.startTicks = wakem.mWaitList.startTicks + } + } + } + + if wakem == mp { + headM = uintptr(mp.mWaitList.next) &^ mutexMMask + } + + next := headM | flags + if atomic.Casuintptr(&l.key, v, next) { + if wakem != nil { + // Claimed an M. Wake it. + semawakeup(wakem) + } + return + } + + v = atomic.Loaduintptr(&l.key) + } +} diff --git a/src/runtime/lock_wasip1.go b/src/runtime/lock_wasip1.go index acfc62acb48e90..55153c3a05f542 100644 --- a/src/runtime/lock_wasip1.go +++ b/src/runtime/lock_wasip1.go @@ -19,6 +19,10 @@ const ( active_spin_cnt = 30 ) +type mWaitList struct{} + +func lockVerifyMSize() {} + func mutexContended(l *mutex) bool { return false } diff --git a/src/runtime/lockrank.go b/src/runtime/lockrank.go index 432ace728b8269..9821e499989951 100644 --- a/src/runtime/lockrank.go +++ b/src/runtime/lockrank.go @@ -14,12 +14,17 @@ const ( lockRankSysmon lockRankScavenge lockRankForcegc + lockRankComputeMaxProcs + lockRankUpdateMaxProcsG lockRankDefer lockRankSweepWaiters lockRankAssistQueue + lockRankStrongFromWeakQueue + lockRankCleanupQueue lockRankSweep lockRankTestR lockRankTestW + lockRankVgetrandom lockRankTimerSend lockRankAllocmW lockRankExecW @@ -42,6 +47,7 @@ const ( lockRankRoot lockRankItab lockRankReflectOffs + lockRankSynctest lockRankUserArenaState // TRACEGLOBAL lockRankTraceBuf @@ -64,6 +70,7 @@ const ( lockRankHchanLeaf // WB lockRankWbufSpans + lockRankXRegAlloc lockRankMheap lockRankMheapSpecial lockRankGlobalAlloc @@ -84,64 +91,71 @@ const lockRankLeafRank lockRank = 1000 // lockNames gives the names associated with each of the above ranks. var lockNames = []string{ - lockRankSysmon: "sysmon", - lockRankScavenge: "scavenge", - lockRankForcegc: "forcegc", - lockRankDefer: "defer", - lockRankSweepWaiters: "sweepWaiters", - lockRankAssistQueue: "assistQueue", - lockRankSweep: "sweep", - lockRankTestR: "testR", - lockRankTestW: "testW", - lockRankTimerSend: "timerSend", - lockRankAllocmW: "allocmW", - lockRankExecW: "execW", - lockRankCpuprof: "cpuprof", - lockRankPollCache: "pollCache", - lockRankPollDesc: "pollDesc", - lockRankWakeableSleep: "wakeableSleep", - lockRankHchan: "hchan", - lockRankAllocmR: "allocmR", - lockRankExecR: "execR", - lockRankSched: "sched", - lockRankAllg: "allg", - lockRankAllp: "allp", - lockRankNotifyList: "notifyList", - lockRankSudog: "sudog", - lockRankTimers: "timers", - lockRankTimer: "timer", - lockRankNetpollInit: "netpollInit", - lockRankRoot: "root", - lockRankItab: "itab", - lockRankReflectOffs: "reflectOffs", - lockRankUserArenaState: "userArenaState", - lockRankTraceBuf: "traceBuf", - lockRankTraceStrings: "traceStrings", - lockRankFin: "fin", - lockRankSpanSetSpine: "spanSetSpine", - lockRankMspanSpecial: "mspanSpecial", - lockRankTraceTypeTab: "traceTypeTab", - lockRankGcBitsArenas: "gcBitsArenas", - lockRankProfInsert: "profInsert", - lockRankProfBlock: "profBlock", - lockRankProfMemActive: "profMemActive", - lockRankProfMemFuture: "profMemFuture", - lockRankGscan: "gscan", - lockRankStackpool: "stackpool", - lockRankStackLarge: "stackLarge", - lockRankHchanLeaf: "hchanLeaf", - lockRankWbufSpans: "wbufSpans", - lockRankMheap: "mheap", - lockRankMheapSpecial: "mheapSpecial", - lockRankGlobalAlloc: "globalAlloc", - lockRankTrace: "trace", - lockRankTraceStackTab: "traceStackTab", - lockRankPanic: "panic", - lockRankDeadlock: "deadlock", - lockRankRaceFini: "raceFini", - lockRankAllocmRInternal: "allocmRInternal", - lockRankExecRInternal: "execRInternal", - lockRankTestRInternal: "testRInternal", + lockRankSysmon: "sysmon", + lockRankScavenge: "scavenge", + lockRankForcegc: "forcegc", + lockRankComputeMaxProcs: "computeMaxProcs", + lockRankUpdateMaxProcsG: "updateMaxProcsG", + lockRankDefer: "defer", + lockRankSweepWaiters: "sweepWaiters", + lockRankAssistQueue: "assistQueue", + lockRankStrongFromWeakQueue: "strongFromWeakQueue", + lockRankCleanupQueue: "cleanupQueue", + lockRankSweep: "sweep", + lockRankTestR: "testR", + lockRankTestW: "testW", + lockRankVgetrandom: "vgetrandom", + lockRankTimerSend: "timerSend", + lockRankAllocmW: "allocmW", + lockRankExecW: "execW", + lockRankCpuprof: "cpuprof", + lockRankPollCache: "pollCache", + lockRankPollDesc: "pollDesc", + lockRankWakeableSleep: "wakeableSleep", + lockRankHchan: "hchan", + lockRankAllocmR: "allocmR", + lockRankExecR: "execR", + lockRankSched: "sched", + lockRankAllg: "allg", + lockRankAllp: "allp", + lockRankNotifyList: "notifyList", + lockRankSudog: "sudog", + lockRankTimers: "timers", + lockRankTimer: "timer", + lockRankNetpollInit: "netpollInit", + lockRankRoot: "root", + lockRankItab: "itab", + lockRankReflectOffs: "reflectOffs", + lockRankSynctest: "synctest", + lockRankUserArenaState: "userArenaState", + lockRankTraceBuf: "traceBuf", + lockRankTraceStrings: "traceStrings", + lockRankFin: "fin", + lockRankSpanSetSpine: "spanSetSpine", + lockRankMspanSpecial: "mspanSpecial", + lockRankTraceTypeTab: "traceTypeTab", + lockRankGcBitsArenas: "gcBitsArenas", + lockRankProfInsert: "profInsert", + lockRankProfBlock: "profBlock", + lockRankProfMemActive: "profMemActive", + lockRankProfMemFuture: "profMemFuture", + lockRankGscan: "gscan", + lockRankStackpool: "stackpool", + lockRankStackLarge: "stackLarge", + lockRankHchanLeaf: "hchanLeaf", + lockRankWbufSpans: "wbufSpans", + lockRankXRegAlloc: "xRegAlloc", + lockRankMheap: "mheap", + lockRankMheapSpecial: "mheapSpecial", + lockRankGlobalAlloc: "globalAlloc", + lockRankTrace: "trace", + lockRankTraceStackTab: "traceStackTab", + lockRankPanic: "panic", + lockRankDeadlock: "deadlock", + lockRankRaceFini: "raceFini", + lockRankAllocmRInternal: "allocmRInternal", + lockRankExecRInternal: "execRInternal", + lockRankTestRInternal: "testRInternal", } func (rank lockRank) String() string { @@ -163,62 +177,69 @@ func (rank lockRank) String() string { // // Lock ranks that allow self-cycles list themselves. var lockPartialOrder [][]lockRank = [][]lockRank{ - lockRankSysmon: {}, - lockRankScavenge: {lockRankSysmon}, - lockRankForcegc: {lockRankSysmon}, - lockRankDefer: {}, - lockRankSweepWaiters: {}, - lockRankAssistQueue: {}, - lockRankSweep: {}, - lockRankTestR: {}, - lockRankTestW: {}, - lockRankTimerSend: {}, - lockRankAllocmW: {}, - lockRankExecW: {}, - lockRankCpuprof: {}, - lockRankPollCache: {}, - lockRankPollDesc: {}, - lockRankWakeableSleep: {}, - lockRankHchan: {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankWakeableSleep, lockRankHchan}, - lockRankAllocmR: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan}, - lockRankExecR: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan}, - lockRankSched: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR}, - lockRankAllg: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched}, - lockRankAllp: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched}, - lockRankNotifyList: {}, - lockRankSudog: {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankWakeableSleep, lockRankHchan, lockRankNotifyList}, - lockRankTimers: {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankTimers}, - lockRankTimer: {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankTimers}, - lockRankNetpollInit: {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankTimers, lockRankTimer}, - lockRankRoot: {}, - lockRankItab: {}, - lockRankReflectOffs: {lockRankItab}, - lockRankUserArenaState: {}, - lockRankTraceBuf: {lockRankSysmon, lockRankScavenge}, - lockRankTraceStrings: {lockRankSysmon, lockRankScavenge, lockRankTraceBuf}, - lockRankFin: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, - lockRankSpanSetSpine: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, - lockRankMspanSpecial: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, - lockRankTraceTypeTab: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, - lockRankGcBitsArenas: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankMspanSpecial}, - lockRankProfInsert: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, - lockRankProfBlock: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, - lockRankProfMemActive: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, - lockRankProfMemFuture: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankProfMemActive}, - lockRankGscan: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture}, - lockRankStackpool: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan}, - lockRankStackLarge: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan}, - lockRankHchanLeaf: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankHchanLeaf}, - lockRankWbufSpans: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan}, - lockRankMheap: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans}, - lockRankMheapSpecial: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap}, - lockRankGlobalAlloc: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap, lockRankMheapSpecial}, - lockRankTrace: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap}, - lockRankTraceStackTab: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap, lockRankTrace}, - lockRankPanic: {}, - lockRankDeadlock: {lockRankPanic, lockRankDeadlock}, - lockRankRaceFini: {lockRankPanic}, - lockRankAllocmRInternal: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankAllocmW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR}, - lockRankExecRInternal: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankExecR}, - lockRankTestRInternal: {lockRankTestR, lockRankTestW}, + lockRankSysmon: {}, + lockRankScavenge: {lockRankSysmon}, + lockRankForcegc: {lockRankSysmon}, + lockRankComputeMaxProcs: {lockRankSysmon}, + lockRankUpdateMaxProcsG: {lockRankSysmon}, + lockRankDefer: {}, + lockRankSweepWaiters: {}, + lockRankAssistQueue: {}, + lockRankStrongFromWeakQueue: {}, + lockRankCleanupQueue: {}, + lockRankSweep: {}, + lockRankTestR: {}, + lockRankTestW: {}, + lockRankVgetrandom: {}, + lockRankTimerSend: {}, + lockRankAllocmW: {}, + lockRankExecW: {}, + lockRankCpuprof: {}, + lockRankPollCache: {}, + lockRankPollDesc: {}, + lockRankWakeableSleep: {}, + lockRankHchan: {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankWakeableSleep, lockRankHchan}, + lockRankAllocmR: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan}, + lockRankExecR: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan}, + lockRankSched: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR}, + lockRankAllg: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched}, + lockRankAllp: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched}, + lockRankNotifyList: {}, + lockRankSudog: {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankWakeableSleep, lockRankHchan, lockRankNotifyList}, + lockRankTimers: {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankTimers}, + lockRankTimer: {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankTimers}, + lockRankNetpollInit: {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankTimers, lockRankTimer}, + lockRankRoot: {}, + lockRankItab: {}, + lockRankReflectOffs: {lockRankItab}, + lockRankSynctest: {lockRankSysmon, lockRankScavenge, lockRankSweepWaiters, lockRankStrongFromWeakQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankRoot, lockRankItab, lockRankReflectOffs}, + lockRankUserArenaState: {}, + lockRankTraceBuf: {lockRankSysmon, lockRankScavenge}, + lockRankTraceStrings: {lockRankSysmon, lockRankScavenge, lockRankTraceBuf}, + lockRankFin: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, + lockRankSpanSetSpine: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, + lockRankMspanSpecial: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, + lockRankTraceTypeTab: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, + lockRankGcBitsArenas: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankMspanSpecial}, + lockRankProfInsert: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, + lockRankProfBlock: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, + lockRankProfMemActive: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings}, + lockRankProfMemFuture: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankProfMemActive}, + lockRankGscan: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture}, + lockRankStackpool: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan}, + lockRankStackLarge: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan}, + lockRankHchanLeaf: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankHchanLeaf}, + lockRankWbufSpans: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan}, + lockRankXRegAlloc: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched}, + lockRankMheap: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans}, + lockRankMheapSpecial: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap}, + lockRankGlobalAlloc: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankXRegAlloc, lockRankMheap, lockRankMheapSpecial}, + lockRankTrace: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap}, + lockRankTraceStackTab: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankVgetrandom, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollCache, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankNotifyList, lockRankSudog, lockRankTimers, lockRankTimer, lockRankNetpollInit, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankSynctest, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap, lockRankTrace}, + lockRankPanic: {}, + lockRankDeadlock: {lockRankPanic, lockRankDeadlock}, + lockRankRaceFini: {lockRankPanic}, + lockRankAllocmRInternal: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankAllocmW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankAllocmR}, + lockRankExecRInternal: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankComputeMaxProcs, lockRankUpdateMaxProcsG, lockRankSweepWaiters, lockRankAssistQueue, lockRankStrongFromWeakQueue, lockRankCleanupQueue, lockRankSweep, lockRankTestR, lockRankTimerSend, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankWakeableSleep, lockRankHchan, lockRankExecR}, + lockRankTestRInternal: {lockRankTestR, lockRankTestW}, } diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index b24ebec27d2347..fc4f21b53277ad 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -102,8 +102,10 @@ package runtime import ( "internal/goarch" + "internal/goexperiment" "internal/goos" "internal/runtime/atomic" + "internal/runtime/gc" "internal/runtime/math" "internal/runtime/sys" "unsafe" @@ -112,20 +114,21 @@ import ( const ( maxTinySize = _TinySize tinySizeClass = _TinySizeClass - maxSmallSize = _MaxSmallSize + maxSmallSize = gc.MaxSmallSize + pageSize = 1 << gc.PageShift + pageMask = pageSize - 1 - pageShift = _PageShift - pageSize = _PageSize - - _PageSize = 1 << _PageShift - _PageMask = _PageSize - 1 + // Unused. Left for viewcore. + _PageSize = pageSize + minSizeForMallocHeader = gc.MinSizeForMallocHeader + mallocHeaderSize = gc.MallocHeaderSize // _64bit = 1 on 64-bit systems, 0 on 32-bit systems _64bit = 1 << (^uintptr(0) >> 63) / 2 // Tiny allocator parameters, see "Tiny allocator" comment in malloc.go. - _TinySize = 16 - _TinySizeClass = int8(2) + _TinySize = gc.TinySize + _TinySizeClass = int8(gc.TinySizeClass) _FixAllocChunk = 16 << 10 // Chunk size for FixAlloc @@ -227,22 +230,25 @@ const ( // -------------- --------- ---------- ---------- ----------- // */64-bit 48 64MB 1 4M (32MB) // windows/64-bit 48 4MB 64 1M (8MB) - // ios/arm64 33 4MB 1 2048 (8KB) + // ios/arm64 40 4MB 1 256K (2MB) // */32-bit 32 4MB 1 1024 (4KB) // */mips(le) 31 4MB 1 512 (2KB) + // wasm 32 512KB 1 8192 (64KB) // heapArenaBytes is the size of a heap arena. The heap // consists of mappings of size heapArenaBytes, aligned to // heapArenaBytes. The initial heap mapping is one arena. // - // This is currently 64MB on 64-bit non-Windows and 4MB on - // 32-bit and on Windows. We use smaller arenas on Windows - // because all committed memory is charged to the process, - // even if it's not touched. Hence, for processes with small - // heaps, the mapped arena space needs to be commensurate. - // This is particularly important with the race detector, - // since it significantly amplifies the cost of committed - // memory. + // This is currently 64MB on 64-bit non-Windows, 4MB on + // 32-bit and on Windows, and 512KB on Wasm. We use smaller + // arenas on Windows because all committed memory is charged + // to the process, even if it's not touched. Hence, for + // processes with small heaps, the mapped arena space needs + // to be commensurate. This is particularly important with + // the race detector, since it significantly amplifies the + // cost of committed memory. We use smaller arenas on Wasm + // because some Wasm programs have very small heap, and + // everything in the Wasm linear memory is charged. heapArenaBytes = 1 << logHeapArenaBytes heapArenaWords = heapArenaBytes / goarch.PtrSize @@ -250,7 +256,7 @@ const ( // logHeapArenaBytes is log_2 of heapArenaBytes. For clarity, // prefer using heapArenaBytes where possible (we need the // constant to compute some other constants). - logHeapArenaBytes = (6+20)*(_64bit*(1-goos.IsWindows)*(1-goarch.IsWasm)*(1-goos.IsIos*goarch.IsArm64)) + (2+20)*(_64bit*goos.IsWindows) + (2+20)*(1-_64bit) + (2+20)*goarch.IsWasm + (2+20)*goos.IsIos*goarch.IsArm64 + logHeapArenaBytes = (6+20)*(_64bit*(1-goos.IsWindows)*(1-goarch.IsWasm)*(1-goos.IsIos*goarch.IsArm64)) + (2+20)*(_64bit*goos.IsWindows) + (2+20)*(1-_64bit) + (9+10)*goarch.IsWasm + (2+20)*goos.IsIos*goarch.IsArm64 // heapArenaBitmapWords is the size of each heap arena's bitmap in uintptrs. heapArenaBitmapWords = heapArenaWords / (8 * goarch.PtrSize) @@ -343,6 +349,14 @@ const ( // metadata mappings back to the OS. That would be quite complex to do in general // as the heap is likely fragmented after a reduction in heap size. minHeapForMetadataHugePages = 1 << 30 + + // randomizeHeapBase indicates if the heap base address should be randomized. + // See comment in mallocinit for how the randomization is performed. + randomizeHeapBase = goexperiment.RandomizedHeapBase64 && goarch.PtrSize == 8 && !isSbrkPlatform && !raceenabled && !msanenabled && !asanenabled + + // randHeapBasePrefixMask is used to extract the top byte of the randomized + // heap base address. + randHeapBasePrefixMask = ^uintptr(0xff << (heapAddrBits - 8)) ) // physPageSize is the size in bytes of the OS's physical pages. @@ -370,8 +384,26 @@ var ( physHugePageShift uint ) +var ( + // heapRandSeed is a random value that is populated in mallocinit if + // randomizeHeapBase is set. It is used in mallocinit, and mheap.grow, to + // randomize the base heap address. + heapRandSeed uintptr + heapRandSeedBitsRemaining int +) + +func nextHeapRandBits(bits int) uintptr { + if bits > heapRandSeedBitsRemaining { + throw("not enough heapRandSeed bits remaining") + } + r := heapRandSeed >> (64 - bits) + heapRandSeed <<= bits + heapRandSeedBitsRemaining -= bits + return r +} + func mallocinit() { - if class_to_size[_TinySizeClass] != _TinySize { + if gc.SizeClassToSize[tinySizeClass] != maxTinySize { throw("bad TinySizeClass") } @@ -427,9 +459,16 @@ func mallocinit() { // Check that the minimum size (exclusive) for a malloc header is also // a size class boundary. This is important to making sure checks align // across different parts of the runtime. + // + // While we're here, also check to make sure all these size classes' + // span sizes are one page. Some code relies on this. minSizeForMallocHeaderIsSizeClass := false - for i := 0; i < len(class_to_size); i++ { - if minSizeForMallocHeader == uintptr(class_to_size[i]) { + sizeClassesUpToMinSizeForMallocHeaderAreOnePage := true + for i := 0; i < len(gc.SizeClassToSize); i++ { + if gc.SizeClassToNPages[i] > 1 { + sizeClassesUpToMinSizeForMallocHeaderAreOnePage = false + } + if gc.MinSizeForMallocHeader == uintptr(gc.SizeClassToSize[i]) { minSizeForMallocHeaderIsSizeClass = true break } @@ -437,14 +476,17 @@ func mallocinit() { if !minSizeForMallocHeaderIsSizeClass { throw("min size of malloc header is not a size class boundary") } + if !sizeClassesUpToMinSizeForMallocHeaderAreOnePage { + throw("expected all size classes up to min size for malloc header to fit in one-page spans") + } // Check that the pointer bitmap for all small sizes without a malloc header // fits in a word. - if minSizeForMallocHeader/goarch.PtrSize > 8*goarch.PtrSize { + if gc.MinSizeForMallocHeader/goarch.PtrSize > 8*goarch.PtrSize { throw("max pointer/scan bitmap size for headerless objects is too large") } - if minTagBits > taggedPointerBits { - throw("taggedPointerbits too small") + if minTagBits > tagBits { + throw("tagBits too small") } // Initialize the heap. @@ -460,7 +502,10 @@ func mallocinit() { lockInit(&globalAlloc.mutex, lockRankGlobalAlloc) // Create initial arena growth hints. - if goarch.PtrSize == 8 { + if isSbrkPlatform { + // Don't generate hints on sbrk platforms. We can + // only grow the break sequentially. + } else if goarch.PtrSize == 8 { // On a 64-bit machine, we pick the following hints // because: // @@ -502,9 +547,64 @@ func mallocinit() { // // In race mode we have no choice but to just use the same hints because // the race detector requires that the heap be mapped contiguously. + // + // If randomizeHeapBase is set, we attempt to randomize the base address + // as much as possible. We do this by generating a random uint64 via + // bootstrapRand and using it's bits to randomize portions of the base + // address as follows: + // * We first generate a random heapArenaBytes aligned address that we use for + // generating the hints. + // * On the first call to mheap.grow, we then generate a random PallocChunkBytes + // aligned offset into the mmap'd heap region, which we use as the base for + // the heap region. + // * We then select a page offset in that PallocChunkBytes region to start the + // heap at, and mark all the pages up to that offset as allocated. + // + // Our final randomized "heap base address" becomes the first byte of + // the first available page returned by the page allocator. This results + // in an address with at least heapAddrBits-gc.PageShift-2-(1*goarch.IsAmd64) + // bits of entropy. + + var randHeapBase uintptr + var randHeapBasePrefix byte + // heapAddrBits is 48 on most platforms, but we only use 47 of those + // bits in order to provide a good amount of room for the heap to grow + // contiguously. On amd64, there are 48 bits, but the top bit is sign + // extended, so we throw away another bit, just to be safe. + randHeapAddrBits := heapAddrBits - 1 - (goarch.IsAmd64 * 1) + if randomizeHeapBase { + // Generate a random value, and take the bottom heapAddrBits-logHeapArenaBytes + // bits, using them as the top bits for randHeapBase. + heapRandSeed, heapRandSeedBitsRemaining = uintptr(bootstrapRand()), 64 + + topBits := (randHeapAddrBits - logHeapArenaBytes) + randHeapBase = nextHeapRandBits(topBits) << (randHeapAddrBits - topBits) + randHeapBase = alignUp(randHeapBase, heapArenaBytes) + randHeapBasePrefix = byte(randHeapBase >> (randHeapAddrBits - 8)) + } + + var vmaSize int + if GOARCH == "riscv64" { + // Identify which memory layout is in use based on the system + // stack address, knowing that the bottom half of virtual memory + // is user space. This should result in 39, 48 or 57. It may be + // possible to use RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS at some + // point in the future - for now use the system stack address. + vmaSize = sys.Len64(uint64(getg().m.g0.stack.hi)) + 1 + if raceenabled && vmaSize != 39 && vmaSize != 48 { + println("vma size = ", vmaSize) + throw("riscv64 vma size is unknown and race mode is enabled") + } + } + for i := 0x7f; i >= 0; i-- { var p uintptr switch { + case raceenabled && GOARCH == "riscv64" && vmaSize == 39: + p = uintptr(i)<<28 | uintptrMask&(0x0013<<28) + if p >= uintptrMask&0x000f00000000 { + continue + } case raceenabled: // The TSAN runtime requires the heap // to be in the range [0x00c000000000, @@ -513,10 +613,15 @@ func mallocinit() { if p >= uintptrMask&0x00e000000000 { continue } + case randomizeHeapBase: + prefix := uintptr(randHeapBasePrefix+byte(i)) << (randHeapAddrBits - 8) + p = prefix | (randHeapBase & randHeapBasePrefixMask) case GOARCH == "arm64" && GOOS == "ios": p = uintptr(i)<<40 | uintptrMask&(0x0013<<28) case GOARCH == "arm64": p = uintptr(i)<<40 | uintptrMask&(0x0040<<32) + case GOARCH == "riscv64" && vmaSize == 39: + p = uintptr(i)<<32 | uintptrMask&(0x0013<<28) case GOOS == "aix": if i == 0 { // We don't use addresses directly after 0x0A00000000000000 @@ -557,7 +662,7 @@ func mallocinit() { // heap reservation. const arenaMetaSize = (1 << arenaBits) * unsafe.Sizeof(heapArena{}) - meta := uintptr(sysReserve(nil, arenaMetaSize)) + meta := uintptr(sysReserve(nil, arenaMetaSize, "heap reservation")) if meta != 0 { mheap_.heapArenaAlloc.init(meta, arenaMetaSize, true) } @@ -594,7 +699,7 @@ func mallocinit() { 128 << 20, } for _, arenaSize := range arenaSizes { - a, size := sysReserveAligned(unsafe.Pointer(p), arenaSize, heapArenaBytes) + a, size := sysReserveAligned(unsafe.Pointer(p), arenaSize, heapArenaBytes, "heap reservation") if a != nil { mheap_.arena.init(uintptr(a), size, false) p = mheap_.arena.end // For hint below @@ -615,7 +720,7 @@ func mallocinit() { } // Initialize the memory limit here because the allocator is going to look at it // but we haven't called gcinit yet and we're definitely going to allocate memory before then. - gcController.memoryLimit.Store(maxInt64) + gcController.memoryLimit.Store(math.MaxInt64) } // sysAlloc allocates heap arena space for at least n bytes. The @@ -627,14 +732,13 @@ func mallocinit() { // hintList is a list of hint addresses for where to allocate new // heap arenas. It must be non-nil. // -// register indicates whether the heap arena should be registered -// in allArenas. -// // sysAlloc returns a memory region in the Reserved state. This region must // be transitioned to Prepared and then Ready before use. // +// arenaList is the list the arena should be added to. +// // h must be locked. -func (h *mheap) sysAlloc(n uintptr, hintList **arenaHint, register bool) (v unsafe.Pointer, size uintptr) { +func (h *mheap) sysAlloc(n uintptr, hintList **arenaHint, arenaList *[]arenaIdx) (v unsafe.Pointer, size uintptr) { assertLockHeld(&h.lock) n = alignUp(n, heapArenaBytes) @@ -645,7 +749,7 @@ func (h *mheap) sysAlloc(n uintptr, hintList **arenaHint, register bool) (v unsa // // Only do this if we're using the regular heap arena hints. // This behavior is only for the heap. - v = h.arena.alloc(n, heapArenaBytes, &gcController.heapReleased) + v = h.arena.alloc(n, heapArenaBytes, &gcController.heapReleased, "heap") if v != nil { size = n goto mapped @@ -666,7 +770,7 @@ func (h *mheap) sysAlloc(n uintptr, hintList **arenaHint, register bool) (v unsa // Outside addressable heap. Can't use. v = nil } else { - v = sysReserve(unsafe.Pointer(p), n) + v = sysReserve(unsafe.Pointer(p), n, "heap reservation") } if p == uintptr(v) { // Success. Update the hint. @@ -702,7 +806,7 @@ func (h *mheap) sysAlloc(n uintptr, hintList **arenaHint, register bool) (v unsa // All of the hints failed, so we'll take any // (sufficiently aligned) address the kernel will give // us. - v, size = sysReserveAligned(nil, n, heapArenaBytes) + v, size = sysReserveAligned(nil, n, heapArenaBytes, "heap") if v == nil { return nil, 0 } @@ -740,6 +844,11 @@ func (h *mheap) sysAlloc(n uintptr, hintList **arenaHint, register bool) (v unsa } mapped: + if valgrindenabled { + valgrindCreateMempool(v) + valgrindMakeMemNoAccess(v, size) + } + // Create arena metadata. for ri := arenaIndex(uintptr(v)); ri <= arenaIndex(uintptr(v)+size-1); ri++ { l2 := h.arenas[ri.l1()] @@ -752,7 +861,7 @@ mapped: // is paged in is too expensive. Trying to account for the whole region means // that it will appear like an enormous memory overhead in statistics, even though // it is not. - l2 = (*[1 << arenaL2Bits]*heapArena)(sysAllocOS(unsafe.Sizeof(*l2))) + l2 = (*[1 << arenaL2Bits]*heapArena)(sysAllocOS(unsafe.Sizeof(*l2), "heap index")) if l2 == nil { throw("out of memory allocating heap arena map") } @@ -768,7 +877,7 @@ mapped: throw("arena already initialized") } var r *heapArena - r = (*heapArena)(h.heapArenaAlloc.alloc(unsafe.Sizeof(*r), goarch.PtrSize, &memstats.gcMiscSys)) + r = (*heapArena)(h.heapArenaAlloc.alloc(unsafe.Sizeof(*r), goarch.PtrSize, &memstats.gcMiscSys, "heap metadata")) if r == nil { r = (*heapArena)(persistentalloc(unsafe.Sizeof(*r), goarch.PtrSize, &memstats.gcMiscSys)) if r == nil { @@ -777,27 +886,25 @@ mapped: } // Register the arena in allArenas if requested. - if register { - if len(h.allArenas) == cap(h.allArenas) { - size := 2 * uintptr(cap(h.allArenas)) * goarch.PtrSize - if size == 0 { - size = physPageSize - } - newArray := (*notInHeap)(persistentalloc(size, goarch.PtrSize, &memstats.gcMiscSys)) - if newArray == nil { - throw("out of memory allocating allArenas") - } - oldSlice := h.allArenas - *(*notInHeapSlice)(unsafe.Pointer(&h.allArenas)) = notInHeapSlice{newArray, len(h.allArenas), int(size / goarch.PtrSize)} - copy(h.allArenas, oldSlice) - // Do not free the old backing array because - // there may be concurrent readers. Since we - // double the array each time, this can lead - // to at most 2x waste. + if len((*arenaList)) == cap((*arenaList)) { + size := 2 * uintptr(cap((*arenaList))) * goarch.PtrSize + if size == 0 { + size = physPageSize + } + newArray := (*notInHeap)(persistentalloc(size, goarch.PtrSize, &memstats.gcMiscSys)) + if newArray == nil { + throw("out of memory allocating allArenas") } - h.allArenas = h.allArenas[:len(h.allArenas)+1] - h.allArenas[len(h.allArenas)-1] = ri + oldSlice := (*arenaList) + *(*notInHeapSlice)(unsafe.Pointer(&(*arenaList))) = notInHeapSlice{newArray, len((*arenaList)), int(size / goarch.PtrSize)} + copy((*arenaList), oldSlice) + // Do not free the old backing array because + // there may be concurrent readers. Since we + // double the array each time, this can lead + // to at most 2x waste. } + (*arenaList) = (*arenaList)[:len((*arenaList))+1] + (*arenaList)[len((*arenaList))-1] = ri // Store atomically just in case an object from the // new heap arena becomes visible before the heap lock @@ -817,13 +924,19 @@ mapped: // sysReserveAligned is like sysReserve, but the returned pointer is // aligned to align bytes. It may reserve either n or n+align bytes, // so it returns the size that was reserved. -func sysReserveAligned(v unsafe.Pointer, size, align uintptr) (unsafe.Pointer, uintptr) { +func sysReserveAligned(v unsafe.Pointer, size, align uintptr, vmaName string) (unsafe.Pointer, uintptr) { + if isSbrkPlatform { + if v != nil { + throw("unexpected heap arena hint on sbrk platform") + } + return sysReserveAlignedSbrk(size, align) + } // Since the alignment is rather large in uses of this // function, we're not likely to get it by chance, so we ask // for a larger region and remove the parts we don't need. retries := 0 retry: - p := uintptr(sysReserve(v, size+align)) + p := uintptr(sysReserve(v, size+align, vmaName)) switch { case p == 0: return nil, 0 @@ -836,7 +949,7 @@ retry: // so we may have to try again. sysFreeOS(unsafe.Pointer(p), size+align) p = alignUp(p, align) - p2 := sysReserve(unsafe.Pointer(p), size) + p2 := sysReserve(unsafe.Pointer(p), size, vmaName) if p != uintptr(p2) { // Must have raced. Try again. sysFreeOS(p2, size) @@ -932,9 +1045,9 @@ func nextFreeFast(s *mspan) gclinkptr { // // Must run in a non-preemptible context since otherwise the owner of // c could change. -func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, shouldhelpgc bool) { +func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, checkGCTrigger bool) { s = c.alloc[spc] - shouldhelpgc = false + checkGCTrigger = false freeIndex := s.nextFreeIndex() if freeIndex == s.nelems { // The span is full. @@ -943,7 +1056,7 @@ func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, shouldhelpgc bo throw("s.allocCount != s.nelems && freeIndex == s.nelems") } c.refill(spc) - shouldhelpgc = true + checkGCTrigger = true s = c.alloc[spc] freeIndex = s.nextFreeIndex() @@ -962,6 +1075,20 @@ func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, shouldhelpgc bo return } +// doubleCheckMalloc enables a bunch of extra checks to malloc to double-check +// that various invariants are upheld. +// +// We might consider turning these on by default; many of them previously were. +// They account for a few % of mallocgc's cost though, which does matter somewhat +// at scale. +const doubleCheckMalloc = false + +// sizeSpecializedMallocEnabled is the set of conditions where we enable the size-specialized +// mallocgc implementation: the experiment must be enabled, and none of the sanitizers should +// be enabled. The tables used to select the size-specialized malloc function do not compile +// properly on plan9, so size-specialized malloc is also disabled on plan9. +const sizeSpecializedMallocEnabled = goexperiment.SizeSpecializedMalloc && GOOS != "plan9" && !asanenabled && !raceenabled && !msanenabled && !valgrindenabled + // Allocate an object of size bytes. // Small objects are allocated from the per-P cache's free lists. // Large objects (> 32 kB) are allocated straight from the heap. @@ -981,216 +1108,410 @@ func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, shouldhelpgc bo // //go:linkname mallocgc func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - if gcphase == _GCmarktermination { - throw("mallocgc called with gcphase == _GCmarktermination") + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } } + // Short-circuit zero-sized allocation requests. if size == 0 { return unsafe.Pointer(&zerobase) } + if sizeSpecializedMallocEnabled && heapBitsInSpan(size) { + if typ == nil || !typ.Pointers() { + return mallocNoScanTable[size](size, typ, needzero) + } else { + if !needzero { + throw("objects with pointers must be zeroed") + } + return mallocScanTable[size](size, typ, needzero) + } + } + // It's possible for any malloc to trigger sweeping, which may in // turn queue finalizers. Record this dynamic lock edge. + // N.B. Compiled away if lockrank experiment is not enabled. lockRankMayQueueFinalizer() - userSize := size + // Pre-malloc debug hooks. + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + // For ASAN, we allocate extra memory around each allocation called the "redzone." + // These "redzones" are marked as unaddressable. + var asanRZ uintptr if asanenabled { - // Refer to ASAN runtime library, the malloc() function allocates extra memory, - // the redzone, around the user requested memory region. And the redzones are marked - // as unaddressable. We perform the same operations in Go to detect the overflows or - // underflows. - size += computeRZlog(size) + asanRZ = redZoneSize(size) + size += asanRZ } - if debug.malloc { - if debug.sbrk != 0 { - align := uintptr(16) - if typ != nil { - // TODO(austin): This should be just - // align = uintptr(typ.align) - // but that's only 4 on 32-bit platforms, - // even if there's a uint64 field in typ (see #599). - // This causes 64-bit atomic accesses to panic. - // Hence, we use stricter alignment that matches - // the normal allocator better. - if size&7 == 0 { - align = 8 - } else if size&3 == 0 { - align = 4 - } else if size&1 == 0 { - align = 2 + // Assist the GC if needed. + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + // Actually do the allocation. + var x unsafe.Pointer + var elemsize uintptr + if sizeSpecializedMallocEnabled { + // we know that heapBitsInSpan is true. + if size <= maxSmallSize-gc.MallocHeaderSize { + if typ == nil || !typ.Pointers() { + x, elemsize = mallocgcSmallNoscan(size, typ, needzero) + } else { + if !needzero { + throw("objects with pointers must be zeroed") + } + x, elemsize = mallocgcSmallScanHeader(size, typ) + } + } else { + x, elemsize = mallocgcLarge(size, typ, needzero) + } + } else { + if size <= maxSmallSize-gc.MallocHeaderSize { + if typ == nil || !typ.Pointers() { + if size < maxTinySize { + x, elemsize = mallocgcTiny(size, typ) } else { - align = 1 + x, elemsize = mallocgcSmallNoscan(size, typ, needzero) + } + } else { + if !needzero { + throw("objects with pointers must be zeroed") + } + if heapBitsInSpan(size) { + x, elemsize = mallocgcSmallScanNoHeader(size, typ) + } else { + x, elemsize = mallocgcSmallScanHeader(size, typ) } } - return persistentalloc(size, align, &memstats.other_sys) + } else { + x, elemsize = mallocgcLarge(size, typ, needzero) } + } + + // Notify sanitizers, if enabled. + if raceenabled { + racemalloc(x, size-asanRZ) + } + if msanenabled { + msanmalloc(x, size-asanRZ) + } + if asanenabled { + // Poison the space between the end of the requested size of x + // and the end of the slot. Unpoison the requested allocation. + frag := elemsize - size + if typ != nil && typ.Pointers() && !heapBitsInSpan(elemsize) && size <= maxSmallSize-gc.MallocHeaderSize { + frag -= gc.MallocHeaderSize + } + asanpoison(unsafe.Add(x, size-asanRZ), asanRZ) + asanunpoison(x, size-asanRZ) + } + if valgrindenabled { + valgrindMalloc(x, size-asanRZ) + } - if inittrace.active && inittrace.id == getg().goid { - // Init functions are executed sequentially in a single goroutine. - inittrace.allocs += 1 + // Adjust our GC assist debt to account for internal fragmentation. + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) } } - // assistG is the G to charge for this allocation, or nil if - // GC is not currently active. - assistG := deductAssistCredit(size) + // Post-malloc debug hooks. + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} +func mallocgcTiny(size uintptr, typ *_type) (unsafe.Pointer, uintptr) { // Set mp.mallocing to keep from being preempted by GC. mp := acquirem() - if mp.mallocing != 0 { - throw("malloc deadlock") - } - if mp.gsignal == getg() { - throw("malloc during signal") + if doubleCheckMalloc { + if mp.mallocing != 0 { + throw("malloc deadlock") + } + if mp.gsignal == getg() { + throw("malloc during signal") + } + if typ != nil && typ.Pointers() { + throw("expected noscan for tiny alloc") + } } mp.mallocing = 1 - shouldhelpgc := false - dataSize := userSize + // Tiny allocator. + // + // Tiny allocator combines several tiny allocation requests + // into a single memory block. The resulting memory block + // is freed when all subobjects are unreachable. The subobjects + // must be noscan (don't have pointers), this ensures that + // the amount of potentially wasted memory is bounded. + // + // Size of the memory block used for combining (maxTinySize) is tunable. + // Current setting is 16 bytes, which relates to 2x worst case memory + // wastage (when all but one subobjects are unreachable). + // 8 bytes would result in no wastage at all, but provides less + // opportunities for combining. + // 32 bytes provides more opportunities for combining, + // but can lead to 4x worst case wastage. + // The best case winning is 8x regardless of block size. + // + // Objects obtained from tiny allocator must not be freed explicitly. + // So when an object will be freed explicitly, we ensure that + // its size >= maxTinySize. + // + // SetFinalizer has a special case for objects potentially coming + // from tiny allocator, it such case it allows to set finalizers + // for an inner byte of a memory block. + // + // The main targets of tiny allocator are small strings and + // standalone escaping variables. On a json benchmark + // the allocator reduces number of allocations by ~12% and + // reduces heap size by ~20%. c := getMCache(mp) - if c == nil { - throw("mallocgc called without a P or outside bootstrapping") + off := c.tinyoffset + // Align tiny pointer for required (conservative) alignment. + if size&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && size == 12 { + // Conservatively align 12-byte objects to 8 bytes on 32-bit + // systems so that objects whose first field is a 64-bit + // value is aligned to 8 bytes and does not cause a fault on + // atomic access. See issue 37262. + // TODO(mknyszek): Remove this workaround if/when issue 36606 + // is resolved. + off = alignUp(off, 8) + } else if size&3 == 0 { + off = alignUp(off, 4) + } else if size&1 == 0 { + off = alignUp(off, 2) + } + if off+size <= maxTinySize && c.tiny != 0 { + // The object fits into existing tiny block. + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + size + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + return x, 0 + } + // Allocate a new maxTinySize block. + checkGCTrigger := false + span := c.alloc[tinySpanClass] + v := nextFreeFast(span) + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 // Always zero + (*[2]uint64)(x)[1] = 0 + // See if we need to replace the existing tiny block with the new one + // based on amount of remaining free space. + if !raceenabled && (size < c.tinyoffset || c.tiny == 0) { + // Note: disabled when race detector is on, see comment near end of this function. + c.tiny = uintptr(x) + c.tinyoffset = size } - var span *mspan - var header **_type - var x unsafe.Pointer - noscan := typ == nil || !typ.Pointers() - // In some cases block zeroing can profitably (for latency reduction purposes) - // be delayed till preemption is possible; delayedZeroing tracks that state. - delayedZeroing := false - // Determine if it's a 'small' object that goes into a size-classed span. + + // Ensure that the stores above that initialize x to + // type-safe memory and set the heap bits occur before + // the caller can make x observable to the garbage + // collector. Otherwise, on weakly ordered machines, + // the garbage collector could follow a pointer to x, + // but see uninitialized memory or stale heap bits. + publicationBarrier() + + if writeBarrier.enabled { + // Allocate black during GC. + // All slots hold nil so no scanning is needed. + // This may be racing with GC so do it atomically if there can be + // a race marking the bit. + gcmarknewobject(span, uintptr(x)) + } else { + // Track the last free index before the mark phase. This field + // is only used by the garbage collector. During the mark phase + // this is used by the conservative scanner to filter out objects + // that are both free and recently-allocated. It's safe to do that + // because we allocate-black if the GC is enabled. The conservative + // scanner produces pointers out of thin air, so without additional + // synchronization it might otherwise observe a partially-initialized + // object, which could crash the program. + span.freeIndexForScan = span.freeindex + } + + // Note cache c only valid while m acquired; see #47302 // - // Note: This comparison looks a little strange, but it exists to smooth out - // the crossover between the largest size class and large objects that have - // their own spans. The small window of object sizes between maxSmallSize-mallocHeaderSize - // and maxSmallSize will be considered large, even though they might fit in - // a size class. In practice this is completely fine, since the largest small - // size class has a single object in it already, precisely to make the transition - // to large objects smooth. - if size <= maxSmallSize-mallocHeaderSize { - if noscan && size < maxTinySize { - // Tiny allocator. - // - // Tiny allocator combines several tiny allocation requests - // into a single memory block. The resulting memory block - // is freed when all subobjects are unreachable. The subobjects - // must be noscan (don't have pointers), this ensures that - // the amount of potentially wasted memory is bounded. - // - // Size of the memory block used for combining (maxTinySize) is tunable. - // Current setting is 16 bytes, which relates to 2x worst case memory - // wastage (when all but one subobjects are unreachable). - // 8 bytes would result in no wastage at all, but provides less - // opportunities for combining. - // 32 bytes provides more opportunities for combining, - // but can lead to 4x worst case wastage. - // The best case winning is 8x regardless of block size. - // - // Objects obtained from tiny allocator must not be freed explicitly. - // So when an object will be freed explicitly, we ensure that - // its size >= maxTinySize. - // - // SetFinalizer has a special case for objects potentially coming - // from tiny allocator, it such case it allows to set finalizers - // for an inner byte of a memory block. - // - // The main targets of tiny allocator are small strings and - // standalone escaping variables. On a json benchmark - // the allocator reduces number of allocations by ~12% and - // reduces heap size by ~20%. - off := c.tinyoffset - // Align tiny pointer for required (conservative) alignment. - if size&7 == 0 { - off = alignUp(off, 8) - } else if goarch.PtrSize == 4 && size == 12 { - // Conservatively align 12-byte objects to 8 bytes on 32-bit - // systems so that objects whose first field is a 64-bit - // value is aligned to 8 bytes and does not cause a fault on - // atomic access. See issue 37262. - // TODO(mknyszek): Remove this workaround if/when issue 36606 - // is resolved. - off = alignUp(off, 8) - } else if size&3 == 0 { - off = alignUp(off, 4) - } else if size&1 == 0 { - off = alignUp(off, 2) - } - if off+size <= maxTinySize && c.tiny != 0 { - // The object fits into existing tiny block. - x = unsafe.Pointer(c.tiny + off) - c.tinyoffset = off + size - c.tinyAllocs++ - mp.mallocing = 0 - releasem(mp) - return x - } - // Allocate a new maxTinySize block. - span = c.alloc[tinySpanClass] - v := nextFreeFast(span) - if v == 0 { - v, span, shouldhelpgc = c.nextFree(tinySpanClass) - } - x = unsafe.Pointer(v) - (*[2]uint64)(x)[0] = 0 - (*[2]uint64)(x)[1] = 0 - // See if we need to replace the existing tiny block with the new one - // based on amount of remaining free space. - if !raceenabled && (size < c.tinyoffset || c.tiny == 0) { - // Note: disabled when race detector is on, see comment near end of this function. - c.tiny = uintptr(x) - c.tinyoffset = size - } - size = maxTinySize - } else { - hasHeader := !noscan && !heapBitsInSpan(size) - if hasHeader { - size += mallocHeaderSize - } - var sizeclass uint8 - if size <= smallSizeMax-8 { - sizeclass = size_to_class8[divRoundUp(size, smallSizeDiv)] - } else { - sizeclass = size_to_class128[divRoundUp(size-smallSizeMax, largeSizeDiv)] - } - size = uintptr(class_to_size[sizeclass]) - spc := makeSpanClass(sizeclass, noscan) - span = c.alloc[spc] - v := nextFreeFast(span) - if v == 0 { - v, span, shouldhelpgc = c.nextFree(spc) - } - x = unsafe.Pointer(v) - if needzero && span.needzero != 0 { - memclrNoHeapPointers(x, size) - } - if hasHeader { - header = (**_type)(x) - x = add(x, mallocHeaderSize) - size -= mallocHeaderSize - } + // N.B. Use the full size because that matches how the GC + // will update the mem profile on the "free" side. + // + // TODO(mknyszek): We should really count the header as part + // of gc_sys or something. The code below just pretends it is + // internal fragmentation and matches the GC's accounting by + // using the whole allocation slot. + c.nextSample -= int64(span.elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, span.elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + // Pad tinysize allocations so they are aligned with the end + // of the tinyalloc region. This ensures that any arithmetic + // that goes off the top end of the object will be detectable + // by checkptr (issue 38872). + // Note that we disable tinyalloc when raceenabled for this to work. + // TODO: This padding is only performed when the race detector + // is enabled. It would be nice to enable it if any package + // was compiled with checkptr, but there's no easy way to + // detect that (especially at compile time). + // TODO: enable this padding for all allocations, not just + // tinyalloc ones. It's tricky because of pointer maps. + // Maybe just all noscan objects? + x = add(x, span.elemsize-size) + } + return x, span.elemsize +} + +func mallocgcSmallNoscan(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { + // Set mp.mallocing to keep from being preempted by GC. + mp := acquirem() + if doubleCheckMalloc { + if mp.mallocing != 0 { + throw("malloc deadlock") + } + if mp.gsignal == getg() { + throw("malloc during signal") } + if typ != nil && typ.Pointers() { + throw("expected noscan type for noscan alloc") + } + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + var sizeclass uint8 + if size <= gc.SmallSizeMax-8 { + sizeclass = gc.SizeToSizeClass8[divRoundUp(size, gc.SmallSizeDiv)] } else { - shouldhelpgc = true - // For large allocations, keep track of zeroed state so that - // bulk zeroing can be happen later in a preemptible context. - span = c.allocLarge(size, noscan) - span.freeindex = 1 - span.allocCount = 1 - size = span.elemsize - x = unsafe.Pointer(span.base()) - if needzero && span.needzero != 0 { - delayedZeroing = true + sizeclass = gc.SizeToSizeClass128[divRoundUp(size-gc.SmallSizeMax, gc.LargeSizeDiv)] + } + size = uintptr(gc.SizeClassToSize[sizeclass]) + spc := makeSpanClass(sizeclass, true) + span := c.alloc[spc] + v := nextFreeFast(span) + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, size) + } + + // Ensure that the stores above that initialize x to + // type-safe memory and set the heap bits occur before + // the caller can make x observable to the garbage + // collector. Otherwise, on weakly ordered machines, + // the garbage collector could follow a pointer to x, + // but see uninitialized memory or stale heap bits. + publicationBarrier() + + if writeBarrier.enabled { + // Allocate black during GC. + // All slots hold nil so no scanning is needed. + // This may be racing with GC so do it atomically if there can be + // a race marking the bit. + gcmarknewobject(span, uintptr(x)) + } else { + // Track the last free index before the mark phase. This field + // is only used by the garbage collector. During the mark phase + // this is used by the conservative scanner to filter out objects + // that are both free and recently-allocated. It's safe to do that + // because we allocate-black if the GC is enabled. The conservative + // scanner produces pointers out of thin air, so without additional + // synchronization it might otherwise observe a partially-initialized + // object, which could crash the program. + span.freeIndexForScan = span.freeindex + } + + // Note cache c only valid while m acquired; see #47302 + // + // N.B. Use the full size because that matches how the GC + // will update the mem profile on the "free" side. + // + // TODO(mknyszek): We should really count the header as part + // of gc_sys or something. The code below just pretends it is + // internal fragmentation and matches the GC's accounting by + // using the whole allocation slot. + c.nextSample -= int64(size) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, size) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + return x, size +} + +func mallocgcSmallScanNoHeader(size uintptr, typ *_type) (unsafe.Pointer, uintptr) { + // Set mp.mallocing to keep from being preempted by GC. + mp := acquirem() + if doubleCheckMalloc { + if mp.mallocing != 0 { + throw("malloc deadlock") } - if !noscan { - // Tell the GC not to look at this yet. - span.largeType = nil - header = &span.largeType + if mp.gsignal == getg() { + throw("malloc during signal") + } + if typ == nil || !typ.Pointers() { + throw("noscan allocated in scan-only path") + } + if !heapBitsInSpan(size) { + throw("heap bits in not in span for non-header-only path") } } - if !noscan && !delayedZeroing { - c.scanAlloc += heapSetType(uintptr(x), dataSize, typ, header, span) + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + sizeclass := gc.SizeToSizeClass8[divRoundUp(size, gc.SmallSizeDiv)] + spc := makeSpanClass(sizeclass, false) + span := c.alloc[spc] + v := nextFreeFast(span) + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, size) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + // initHeapBits already set the pointer bits for the 8-byte sizeclass + // on 64-bit platforms. + c.scanAlloc += 8 + } else { + c.scanAlloc += heapSetTypeNoHeader(uintptr(x), size, typ, span) } + size = uintptr(gc.SizeClassToSize[sizeclass]) // Ensure that the stores above that initialize x to // type-safe memory and set the heap bits occur before @@ -1199,129 +1520,296 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { // the garbage collector could follow a pointer to x, // but see uninitialized memory or stale heap bits. publicationBarrier() - // As x and the heap bits are initialized, update - // freeIndexForScan now so x is seen by the GC - // (including conservative scan) as an allocated object. - // While this pointer can't escape into user code as a - // _live_ pointer until we return, conservative scanning - // may find a dead pointer that happens to point into this - // object. Delaying this update until now ensures that - // conservative scanning considers this pointer dead until - // this point. - span.freeIndexForScan = span.freeindex - - // Allocate black during GC. - // All slots hold nil so no scanning is needed. - // This may be racing with GC so do it atomically if there can be - // a race marking the bit. - if gcphase != _GCoff { + + if writeBarrier.enabled { + // Allocate black during GC. + // All slots hold nil so no scanning is needed. + // This may be racing with GC so do it atomically if there can be + // a race marking the bit. gcmarknewobject(span, uintptr(x)) + } else { + // Track the last free index before the mark phase. This field + // is only used by the garbage collector. During the mark phase + // this is used by the conservative scanner to filter out objects + // that are both free and recently-allocated. It's safe to do that + // because we allocate-black if the GC is enabled. The conservative + // scanner produces pointers out of thin air, so without additional + // synchronization it might otherwise observe a partially-initialized + // object, which could crash the program. + span.freeIndexForScan = span.freeindex + } + + // Note cache c only valid while m acquired; see #47302 + // + // N.B. Use the full size because that matches how the GC + // will update the mem profile on the "free" side. + // + // TODO(mknyszek): We should really count the header as part + // of gc_sys or something. The code below just pretends it is + // internal fragmentation and matches the GC's accounting by + // using the whole allocation slot. + c.nextSample -= int64(size) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, size) } + mp.mallocing = 0 + releasem(mp) - if raceenabled { - racemalloc(x, size) + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } } + return x, size +} - if msanenabled { - msanmalloc(x, size) +func mallocgcSmallScanHeader(size uintptr, typ *_type) (unsafe.Pointer, uintptr) { + // Set mp.mallocing to keep from being preempted by GC. + mp := acquirem() + if doubleCheckMalloc { + if mp.mallocing != 0 { + throw("malloc deadlock") + } + if mp.gsignal == getg() { + throw("malloc during signal") + } + if typ == nil || !typ.Pointers() { + throw("noscan allocated in scan-only path") + } + if heapBitsInSpan(size) { + throw("heap bits in span for header-only path") + } } + mp.mallocing = 1 - if asanenabled { - // We should only read/write the memory with the size asked by the user. - // The rest of the allocated memory should be poisoned, so that we can report - // errors when accessing poisoned memory. - // The allocated memory is larger than required userSize, it will also include - // redzone and some other padding bytes. - rzBeg := unsafe.Add(x, userSize) - asanpoison(rzBeg, size-userSize) - asanunpoison(x, userSize) + checkGCTrigger := false + c := getMCache(mp) + size += gc.MallocHeaderSize + var sizeclass uint8 + if size <= gc.SmallSizeMax-8 { + sizeclass = gc.SizeToSizeClass8[divRoundUp(size, gc.SmallSizeDiv)] + } else { + sizeclass = gc.SizeToSizeClass128[divRoundUp(size-gc.SmallSizeMax, gc.LargeSizeDiv)] + } + size = uintptr(gc.SizeClassToSize[sizeclass]) + spc := makeSpanClass(sizeclass, false) + span := c.alloc[spc] + v := nextFreeFast(span) + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, size) + } + header := (**_type)(x) + x = add(x, gc.MallocHeaderSize) + c.scanAlloc += heapSetTypeSmallHeader(uintptr(x), size-gc.MallocHeaderSize, typ, header, span) + + // Ensure that the stores above that initialize x to + // type-safe memory and set the heap bits occur before + // the caller can make x observable to the garbage + // collector. Otherwise, on weakly ordered machines, + // the garbage collector could follow a pointer to x, + // but see uninitialized memory or stale heap bits. + publicationBarrier() + if writeBarrier.enabled { + // Allocate black during GC. + // All slots hold nil so no scanning is needed. + // This may be racing with GC so do it atomically if there can be + // a race marking the bit. + gcmarknewobject(span, uintptr(x)) + } else { + // Track the last free index before the mark phase. This field + // is only used by the garbage collector. During the mark phase + // this is used by the conservative scanner to filter out objects + // that are both free and recently-allocated. It's safe to do that + // because we allocate-black if the GC is enabled. The conservative + // scanner produces pointers out of thin air, so without additional + // synchronization it might otherwise observe a partially-initialized + // object, which could crash the program. + span.freeIndexForScan = span.freeindex + } + + // Note cache c only valid while m acquired; see #47302 + // + // N.B. Use the full size because that matches how the GC + // will update the mem profile on the "free" side. + // // TODO(mknyszek): We should really count the header as part // of gc_sys or something. The code below just pretends it is // internal fragmentation and matches the GC's accounting by // using the whole allocation slot. - fullSize := span.elemsize - if rate := MemProfileRate; rate > 0 { - // Note cache c only valid while m acquired; see #47302 - // - // N.B. Use the full size because that matches how the GC - // will update the mem profile on the "free" side. - if rate != 1 && fullSize < c.nextSample { - c.nextSample -= fullSize - } else { - profilealloc(mp, x, fullSize) + c.nextSample -= int64(size) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, size) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) } } + return x, size +} + +func mallocgcLarge(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { + // Set mp.mallocing to keep from being preempted by GC. + mp := acquirem() + if doubleCheckMalloc { + if mp.mallocing != 0 { + throw("malloc deadlock") + } + if mp.gsignal == getg() { + throw("malloc during signal") + } + } + mp.mallocing = 1 + + c := getMCache(mp) + // For large allocations, keep track of zeroed state so that + // bulk zeroing can be happen later in a preemptible context. + span := c.allocLarge(size, typ == nil || !typ.Pointers()) + span.freeindex = 1 + span.allocCount = 1 + span.largeType = nil // Tell the GC not to look at this yet. + size = span.elemsize + x := unsafe.Pointer(span.base()) + + // Ensure that the store above that sets largeType to + // nil happens before the caller can make x observable + // to the garbage collector. + // + // Otherwise, on weakly ordered machines, the garbage + // collector could follow a pointer to x, but see a stale + // largeType value. + publicationBarrier() + + if writeBarrier.enabled { + // Allocate black during GC. + // All slots hold nil so no scanning is needed. + // This may be racing with GC so do it atomically if there can be + // a race marking the bit. + gcmarknewobject(span, uintptr(x)) + } else { + // Track the last free index before the mark phase. This field + // is only used by the garbage collector. During the mark phase + // this is used by the conservative scanner to filter out objects + // that are both free and recently-allocated. It's safe to do that + // because we allocate-black if the GC is enabled. The conservative + // scanner produces pointers out of thin air, so without additional + // synchronization it might otherwise observe a partially-initialized + // object, which could crash the program. + span.freeIndexForScan = span.freeindex + } + + // Note cache c only valid while m acquired; see #47302 + // + // N.B. Use the full size because that matches how the GC + // will update the mem profile on the "free" side. + // + // TODO(mknyszek): We should really count the header as part + // of gc_sys or something. The code below just pretends it is + // internal fragmentation and matches the GC's accounting by + // using the whole allocation slot. + c.nextSample -= int64(size) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, size) + } mp.mallocing = 0 releasem(mp) + // Check to see if we need to trigger the GC. + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + // Objects can be zeroed late in a context where preemption can occur. - // If the object contains pointers, its pointer data must be cleared - // or otherwise indicate that the GC shouldn't scan it. + // // x will keep the memory alive. - if delayedZeroing { + if needzero && span.needzero != 0 { // N.B. size == fullSize always in this case. memclrNoHeapPointersChunked(size, x) // This is a possible preemption point: see #47302 - - // Finish storing the type information for this case. - if !noscan { - mp := acquirem() - getMCache(mp).scanAlloc += heapSetType(uintptr(x), dataSize, typ, header, span) - - // Publish the type information with the zeroed memory. - publicationBarrier() - releasem(mp) - } } - if debug.malloc { - if inittrace.active && inittrace.id == getg().goid { - // Init functions are executed sequentially in a single goroutine. - inittrace.bytes += uint64(fullSize) - } + // Set the type and run the publication barrier while non-preemptible. We need to make + // sure that between heapSetTypeLarge and publicationBarrier we cannot get preempted, + // otherwise the GC could potentially observe non-zeroed memory but largeType set on weak + // memory architectures. + // + // The GC can also potentially observe non-zeroed memory if conservative scanning spuriously + // observes a partially-allocated object, see the freeIndexForScan update above. This case is + // handled by synchronization inside heapSetTypeLarge. + mp = acquirem() + if typ != nil && typ.Pointers() { + // Finish storing the type information, now that we're certain the memory is zeroed. + getMCache(mp).scanAlloc += heapSetTypeLarge(uintptr(x), size, typ, span) + } + // Publish the object again, now with zeroed memory and initialized type information. + // + // Even if we didn't update any type information, this is necessary to ensure that, for example, + // x written to a global without any synchronization still results in other goroutines observing + // zeroed memory. + publicationBarrier() + releasem(mp) + return x, size +} - if traceAllocFreeEnabled() { - trace := traceAcquire() - if trace.ok() { - trace.HeapObjectAlloc(uintptr(x), typ) - traceRelease(trace) +func preMallocgcDebug(size uintptr, typ *_type) unsafe.Pointer { + if debug.sbrk != 0 { + align := uintptr(16) + if typ != nil { + // TODO(austin): This should be just + // align = uintptr(typ.align) + // but that's only 4 on 32-bit platforms, + // even if there's a uint64 field in typ (see #599). + // This causes 64-bit atomic accesses to panic. + // Hence, we use stricter alignment that matches + // the normal allocator better. + if size&7 == 0 { + align = 8 + } else if size&3 == 0 { + align = 4 + } else if size&1 == 0 { + align = 2 + } else { + align = 1 } } + return persistentalloc(size, align, &memstats.other_sys) } + if inittrace.active && inittrace.id == getg().goid { + // Init functions are executed sequentially in a single goroutine. + inittrace.allocs += 1 + } + return nil +} - if assistG != nil { - // Account for internal fragmentation in the assist - // debt now that we know it. - // - // N.B. Use the full size because that's how the rest - // of the GC accounts for bytes marked. - assistG.gcAssistBytes -= int64(fullSize - dataSize) +func postMallocgcDebug(x unsafe.Pointer, elemsize uintptr, typ *_type) { + if inittrace.active && inittrace.id == getg().goid { + // Init functions are executed sequentially in a single goroutine. + inittrace.bytes += uint64(elemsize) } - if shouldhelpgc { - if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { - gcStart(t) + if traceAllocFreeEnabled() { + trace := traceAcquire() + if trace.ok() { + trace.HeapObjectAlloc(uintptr(x), typ) + traceRelease(trace) } } - if raceenabled && noscan && dataSize < maxTinySize { - // Pad tinysize allocations so they are aligned with the end - // of the tinyalloc region. This ensures that any arithmetic - // that goes off the top end of the object will be detectable - // by checkptr (issue 38872). - // Note that we disable tinyalloc when raceenabled for this to work. - // TODO: This padding is only performed when the race detector - // is enabled. It would be nice to enable it if any package - // was compiled with checkptr, but there's no easy way to - // detect that (especially at compile time). - // TODO: enable this padding for all allocations, not just - // tinyalloc ones. It's tricky because of pointer maps. - // Maybe just all noscan objects? - x = add(x, size-dataSize) + // N.B. elemsize == 0 indicates a tiny allocation, since no new slot was + // allocated to fulfill this call to mallocgc. This means checkfinalizer + // will only flag an error if there is actually any risk. If an allocation + // has the tiny block to itself, it will not get flagged, because we won't + // mark the block as a tiny block. + if debug.checkfinalizers != 0 && elemsize == 0 { + setTinyBlockContext(unsafe.Pointer(alignDown(uintptr(x), maxTinySize))) } - - return x } // deductAssistCredit reduces the current G's assist credit @@ -1330,26 +1818,22 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { // Caller must be preemptible. // // Returns the G for which the assist credit was accounted. -func deductAssistCredit(size uintptr) *g { - var assistG *g - if gcBlackenEnabled != 0 { - // Charge the current user G for this allocation. - assistG = getg() - if assistG.m.curg != nil { - assistG = assistG.m.curg - } - // Charge the allocation against the G. We'll account - // for internal fragmentation at the end of mallocgc. - assistG.gcAssistBytes -= int64(size) - - if assistG.gcAssistBytes < 0 { - // This G is in debt. Assist the GC to correct - // this before allocating. This must happen - // before disabling preemption. - gcAssistAlloc(assistG) - } +func deductAssistCredit(size uintptr) { + // Charge the current user G for this allocation. + assistG := getg() + if assistG.m.curg != nil { + assistG = assistG.m.curg + } + // Charge the allocation against the G. We'll account + // for internal fragmentation at the end of mallocgc. + assistG.gcAssistBytes -= int64(size) + + if assistG.gcAssistBytes < 0 { + // This G is in debt. Assist the GC to correct + // this before allocating. This must happen + // before disabling preemption. + gcAssistAlloc(assistG) } - return assistG } // memclrNoHeapPointersChunked repeatedly calls memclrNoHeapPointers @@ -1386,6 +1870,11 @@ func newobject(typ *_type) unsafe.Pointer { return mallocgc(typ.Size_, typ, true) } +//go:linkname maps_newobject internal/runtime/maps.newobject +func maps_newobject(typ *_type) unsafe.Pointer { + return newobject(typ) +} + // reflect_unsafe_New is meant for package reflect, // but widely used packages access it using linkname. // Notable members of the hall of shame include: @@ -1450,11 +1939,21 @@ func reflect_unsafe_NewArray(typ *_type, n int) unsafe.Pointer { return newarray(typ, n) } +//go:linkname maps_newarray internal/runtime/maps.newarray +func maps_newarray(typ *_type, n int) unsafe.Pointer { + return newarray(typ, n) +} + +// profilealloc resets the current mcache's nextSample counter and +// records a memory profile sample. +// +// The caller must be non-preemptible and have a P. func profilealloc(mp *m, x unsafe.Pointer, size uintptr) { c := getMCache(mp) if c == nil { throw("profilealloc called without a P or outside bootstrapping") } + c.memProfRate = MemProfileRate c.nextSample = nextSample() mProf_Malloc(mp, x, size) } @@ -1466,22 +1965,16 @@ func profilealloc(mp *m, x unsafe.Pointer, size uintptr) { // processes, the distance between two samples follows the exponential // distribution (exp(MemProfileRate)), so the best return value is a random // number taken from an exponential distribution whose mean is MemProfileRate. -func nextSample() uintptr { +func nextSample() int64 { + if MemProfileRate == 0 { + // Basically never sample. + return math.MaxInt64 + } if MemProfileRate == 1 { - // Callers assign our return value to - // mcache.next_sample, but next_sample is not used - // when the rate is 1. So avoid the math below and - // just return something. + // Sample immediately. return 0 } - if GOOS == "plan9" { - // Plan 9 doesn't support floating point in note handler. - if gp := getg(); gp == gp.m.gsignal { - return nextSampleNoFP() - } - } - - return uintptr(fastexprand(MemProfileRate)) + return int64(fastexprand(MemProfileRate)) } // fastexprand returns a random number from an exponential distribution with @@ -1514,20 +2007,6 @@ func fastexprand(mean int) int32 { return int32(qlog*(minusLog2*float64(mean))) + 1 } -// nextSampleNoFP is similar to nextSample, but uses older, -// simpler code to avoid floating point. -func nextSampleNoFP() uintptr { - // Set first allocation sample size. - rate := MemProfileRate - if rate > 0x3fffffff { // make 2*rate not overflow - rate = 0x3fffffff - } - if rate != 0 { - return uintptr(cheaprandn(uint32(2 * rate))) - } - return 0 -} - type persistentAlloc struct { base *notInHeap off uintptr @@ -1556,6 +2035,10 @@ var persistentChunks *notInHeap // // Consider marking persistentalloc'd types not in heap by embedding // internal/runtime/sys.NotInHeap. +// +// nosplit because it is used during write barriers and must not be preempted. +// +//go:nosplit func persistentalloc(size, align uintptr, sysStat *sysMemStat) unsafe.Pointer { var p *notInHeap systemstack(func() { @@ -1580,7 +2063,7 @@ func persistentalloc1(size, align uintptr, sysStat *sysMemStat) *notInHeap { if align&(align-1) != 0 { throw("persistentalloc: align is not a power of 2") } - if align > _PageSize { + if align > pageSize { throw("persistentalloc: align is too large") } } else { @@ -1588,7 +2071,7 @@ func persistentalloc1(size, align uintptr, sysStat *sysMemStat) *notInHeap { } if size >= maxBlock { - return (*notInHeap)(sysAlloc(size, sysStat)) + return (*notInHeap)(sysAlloc(size, sysStat, "immortal metadata")) } mp := acquirem() @@ -1601,7 +2084,7 @@ func persistentalloc1(size, align uintptr, sysStat *sysMemStat) *notInHeap { } persistent.off = alignUp(persistent.off, align) if persistent.off+size > persistentChunkSize || persistent.base == nil { - persistent.base = (*notInHeap)(sysAlloc(persistentChunkSize, &memstats.other_sys)) + persistent.base = (*notInHeap)(sysAlloc(persistentChunkSize, &memstats.other_sys, "immortal metadata")) if persistent.base == nil { if persistent == &globalAlloc.persistentAlloc { unlock(&globalAlloc.mutex) @@ -1675,7 +2158,7 @@ func (l *linearAlloc) init(base, size uintptr, mapMemory bool) { l.mapMemory = mapMemory } -func (l *linearAlloc) alloc(size, align uintptr, sysStat *sysMemStat) unsafe.Pointer { +func (l *linearAlloc) alloc(size, align uintptr, sysStat *sysMemStat, vmaName string) unsafe.Pointer { p := alignUp(l.next, align) if p+size > l.end { return nil @@ -1685,7 +2168,7 @@ func (l *linearAlloc) alloc(size, align uintptr, sysStat *sysMemStat) unsafe.Poi if l.mapMemory { // Transition from Reserved to Prepared to Ready. n := pEnd - l.mapped - sysMap(unsafe.Pointer(l.mapped), n, sysStat) + sysMap(unsafe.Pointer(l.mapped), n, sysStat, vmaName) sysUsed(unsafe.Pointer(l.mapped), n, n) } l.mapped = pEnd @@ -1707,9 +2190,9 @@ func (p *notInHeap) add(bytes uintptr) *notInHeap { return (*notInHeap)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + bytes)) } -// computeRZlog computes the size of the redzone. +// redZoneSize computes the size of the redzone for a given allocation. // Refer to the implementation of the compiler-rt. -func computeRZlog(userSize uintptr) uintptr { +func redZoneSize(userSize uintptr) uintptr { switch { case userSize <= (64 - 16): return 16 << 0 diff --git a/src/runtime/malloc_generated.go b/src/runtime/malloc_generated.go new file mode 100644 index 00000000000000..2215dbaddb2e1c --- /dev/null +++ b/src/runtime/malloc_generated.go @@ -0,0 +1,8792 @@ +// Code generated by mkmalloc.go; DO NOT EDIT. + +package runtime + +import ( + "internal/goarch" + "internal/runtime/sys" + "unsafe" +) + +func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 1 + + const elemsize = 8 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallScanNoHeader(size, typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(0) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 8 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(8)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 8 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(16)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 16 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 24 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(24)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 24 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 32 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(32)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 32 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 48 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(48)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 48 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 64 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(64)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 64 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 80 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(80)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 80 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 96 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(96)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 96 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 112 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(112)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 112 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 128 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(128)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 128 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 144 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(144)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 144 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 160 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(160)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 160 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 176 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(176)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 176 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 192 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(192)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 192 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 208 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(208)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 208 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 224 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(224)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 224 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 240 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(240)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 240 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 256 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(256)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 256 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 288 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(288)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 288 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 320 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(320)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 320 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 352 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(352)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 352 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 384 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(384)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 384 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 416 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(416)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 416 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 448 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(448)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 448 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 480 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(480)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 480 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 512 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + + c.scanAlloc += 8 + } else { + dataSize := size + x := uintptr(x) + + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(512)) { + throw("tried to write heap bits, but no heap bits in span") + } + + src0 := readUintptr(getGCMask(typ)) + + const elemsize = 512 + + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + + x = add(x, elemsize-constsize) + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocTiny2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const constsize = 2 + + const elemsize = 16 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckTiny(constsize, typ, mp) + } + mp.mallocing = 1 + + c := getMCache(mp) + off := c.tinyoffset + + if constsize&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && constsize == 12 { + + off = alignUp(off, 8) + } else if constsize&3 == 0 { + off = alignUp(off, 4) + } else if constsize&1 == 0 { + off = alignUp(off, 2) + } + if off+constsize <= maxTinySize && c.tiny != 0 { + + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + constsize + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + const elemsize = 0 + { + + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x + } + + } + + checkGCTrigger := false + span := c.alloc[tinySpanClass] + + const nbytes = 8192 + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / + 16, + ) + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + + x = add(x, elemsize-constsize) + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocTiny3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const constsize = 3 + + const elemsize = 16 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckTiny(constsize, typ, mp) + } + mp.mallocing = 1 + + c := getMCache(mp) + off := c.tinyoffset + + if constsize&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && constsize == 12 { + + off = alignUp(off, 8) + } else if constsize&3 == 0 { + off = alignUp(off, 4) + } else if constsize&1 == 0 { + off = alignUp(off, 2) + } + if off+constsize <= maxTinySize && c.tiny != 0 { + + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + constsize + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + const elemsize = 0 + { + + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x + } + + } + + checkGCTrigger := false + span := c.alloc[tinySpanClass] + + const nbytes = 8192 + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / + 16, + ) + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + + x = add(x, elemsize-constsize) + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocTiny4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const constsize = 4 + + const elemsize = 16 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckTiny(constsize, typ, mp) + } + mp.mallocing = 1 + + c := getMCache(mp) + off := c.tinyoffset + + if constsize&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && constsize == 12 { + + off = alignUp(off, 8) + } else if constsize&3 == 0 { + off = alignUp(off, 4) + } else if constsize&1 == 0 { + off = alignUp(off, 2) + } + if off+constsize <= maxTinySize && c.tiny != 0 { + + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + constsize + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + const elemsize = 0 + { + + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x + } + + } + + checkGCTrigger := false + span := c.alloc[tinySpanClass] + + const nbytes = 8192 + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / + 16, + ) + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + + x = add(x, elemsize-constsize) + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocTiny5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const constsize = 5 + + const elemsize = 16 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckTiny(constsize, typ, mp) + } + mp.mallocing = 1 + + c := getMCache(mp) + off := c.tinyoffset + + if constsize&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && constsize == 12 { + + off = alignUp(off, 8) + } else if constsize&3 == 0 { + off = alignUp(off, 4) + } else if constsize&1 == 0 { + off = alignUp(off, 2) + } + if off+constsize <= maxTinySize && c.tiny != 0 { + + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + constsize + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + const elemsize = 0 + { + + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x + } + + } + + checkGCTrigger := false + span := c.alloc[tinySpanClass] + + const nbytes = 8192 + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / + 16, + ) + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + + x = add(x, elemsize-constsize) + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocTiny6(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const constsize = 6 + + const elemsize = 16 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckTiny(constsize, typ, mp) + } + mp.mallocing = 1 + + c := getMCache(mp) + off := c.tinyoffset + + if constsize&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && constsize == 12 { + + off = alignUp(off, 8) + } else if constsize&3 == 0 { + off = alignUp(off, 4) + } else if constsize&1 == 0 { + off = alignUp(off, 2) + } + if off+constsize <= maxTinySize && c.tiny != 0 { + + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + constsize + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + const elemsize = 0 + { + + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x + } + + } + + checkGCTrigger := false + span := c.alloc[tinySpanClass] + + const nbytes = 8192 + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / + 16, + ) + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + + x = add(x, elemsize-constsize) + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocTiny7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const constsize = 7 + + const elemsize = 16 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckTiny(constsize, typ, mp) + } + mp.mallocing = 1 + + c := getMCache(mp) + off := c.tinyoffset + + if constsize&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && constsize == 12 { + + off = alignUp(off, 8) + } else if constsize&3 == 0 { + off = alignUp(off, 4) + } else if constsize&1 == 0 { + off = alignUp(off, 2) + } + if off+constsize <= maxTinySize && c.tiny != 0 { + + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + constsize + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + const elemsize = 0 + { + + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x + } + + } + + checkGCTrigger := false + span := c.alloc[tinySpanClass] + + const nbytes = 8192 + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / + 16, + ) + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + + x = add(x, elemsize-constsize) + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocTiny8(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const constsize = 8 + + const elemsize = 16 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckTiny(constsize, typ, mp) + } + mp.mallocing = 1 + + c := getMCache(mp) + off := c.tinyoffset + + if constsize&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && constsize == 12 { + + off = alignUp(off, 8) + } else if constsize&3 == 0 { + off = alignUp(off, 4) + } else if constsize&1 == 0 { + off = alignUp(off, 2) + } + if off+constsize <= maxTinySize && c.tiny != 0 { + + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + constsize + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + const elemsize = 0 + { + + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x + } + + } + + checkGCTrigger := false + span := c.alloc[tinySpanClass] + + const nbytes = 8192 + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / + 16, + ) + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + + x = add(x, elemsize-constsize) + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocTiny9(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const constsize = 9 + + const elemsize = 16 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckTiny(constsize, typ, mp) + } + mp.mallocing = 1 + + c := getMCache(mp) + off := c.tinyoffset + + if constsize&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && constsize == 12 { + + off = alignUp(off, 8) + } else if constsize&3 == 0 { + off = alignUp(off, 4) + } else if constsize&1 == 0 { + off = alignUp(off, 2) + } + if off+constsize <= maxTinySize && c.tiny != 0 { + + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + constsize + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + const elemsize = 0 + { + + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x + } + + } + + checkGCTrigger := false + span := c.alloc[tinySpanClass] + + const nbytes = 8192 + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / + 16, + ) + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + + x = add(x, elemsize-constsize) + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocTiny10(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const constsize = 10 + + const elemsize = 16 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckTiny(constsize, typ, mp) + } + mp.mallocing = 1 + + c := getMCache(mp) + off := c.tinyoffset + + if constsize&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && constsize == 12 { + + off = alignUp(off, 8) + } else if constsize&3 == 0 { + off = alignUp(off, 4) + } else if constsize&1 == 0 { + off = alignUp(off, 2) + } + if off+constsize <= maxTinySize && c.tiny != 0 { + + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + constsize + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + const elemsize = 0 + { + + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x + } + + } + + checkGCTrigger := false + span := c.alloc[tinySpanClass] + + const nbytes = 8192 + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / + 16, + ) + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + + x = add(x, elemsize-constsize) + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocTiny11(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const constsize = 11 + + const elemsize = 16 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckTiny(constsize, typ, mp) + } + mp.mallocing = 1 + + c := getMCache(mp) + off := c.tinyoffset + + if constsize&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && constsize == 12 { + + off = alignUp(off, 8) + } else if constsize&3 == 0 { + off = alignUp(off, 4) + } else if constsize&1 == 0 { + off = alignUp(off, 2) + } + if off+constsize <= maxTinySize && c.tiny != 0 { + + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + constsize + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + const elemsize = 0 + { + + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x + } + + } + + checkGCTrigger := false + span := c.alloc[tinySpanClass] + + const nbytes = 8192 + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / + 16, + ) + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + + x = add(x, elemsize-constsize) + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocTiny12(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const constsize = 12 + + const elemsize = 16 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckTiny(constsize, typ, mp) + } + mp.mallocing = 1 + + c := getMCache(mp) + off := c.tinyoffset + + if constsize&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && constsize == 12 { + + off = alignUp(off, 8) + } else if constsize&3 == 0 { + off = alignUp(off, 4) + } else if constsize&1 == 0 { + off = alignUp(off, 2) + } + if off+constsize <= maxTinySize && c.tiny != 0 { + + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + constsize + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + const elemsize = 0 + { + + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x + } + + } + + checkGCTrigger := false + span := c.alloc[tinySpanClass] + + const nbytes = 8192 + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / + 16, + ) + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + + x = add(x, elemsize-constsize) + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocTiny13(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const constsize = 13 + + const elemsize = 16 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckTiny(constsize, typ, mp) + } + mp.mallocing = 1 + + c := getMCache(mp) + off := c.tinyoffset + + if constsize&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && constsize == 12 { + + off = alignUp(off, 8) + } else if constsize&3 == 0 { + off = alignUp(off, 4) + } else if constsize&1 == 0 { + off = alignUp(off, 2) + } + if off+constsize <= maxTinySize && c.tiny != 0 { + + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + constsize + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + const elemsize = 0 + { + + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x + } + + } + + checkGCTrigger := false + span := c.alloc[tinySpanClass] + + const nbytes = 8192 + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / + 16, + ) + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + + x = add(x, elemsize-constsize) + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocTiny14(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const constsize = 14 + + const elemsize = 16 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckTiny(constsize, typ, mp) + } + mp.mallocing = 1 + + c := getMCache(mp) + off := c.tinyoffset + + if constsize&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && constsize == 12 { + + off = alignUp(off, 8) + } else if constsize&3 == 0 { + off = alignUp(off, 4) + } else if constsize&1 == 0 { + off = alignUp(off, 2) + } + if off+constsize <= maxTinySize && c.tiny != 0 { + + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + constsize + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + const elemsize = 0 + { + + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x + } + + } + + checkGCTrigger := false + span := c.alloc[tinySpanClass] + + const nbytes = 8192 + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / + 16, + ) + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + + x = add(x, elemsize-constsize) + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocTiny15(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const constsize = 15 + + const elemsize = 16 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckTiny(constsize, typ, mp) + } + mp.mallocing = 1 + + c := getMCache(mp) + off := c.tinyoffset + + if constsize&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && constsize == 12 { + + off = alignUp(off, 8) + } else if constsize&3 == 0 { + off = alignUp(off, 4) + } else if constsize&1 == 0 { + off = alignUp(off, 2) + } + if off+constsize <= maxTinySize && c.tiny != 0 { + + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + constsize + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + const elemsize = 0 + { + + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x + } + + } + + checkGCTrigger := false + span := c.alloc[tinySpanClass] + + const nbytes = 8192 + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / + 16, + ) + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 + (*[2]uint64)(x)[1] = 0 + + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + + x = add(x, elemsize-constsize) + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 2 + + const elemsize = 16 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 16 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 3 + + const elemsize = 24 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 24 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 4 + + const elemsize = 32 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 32 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 5 + + const elemsize = 48 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 48 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 6 + + const elemsize = 64 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 64 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 7 + + const elemsize = 80 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 80 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC8(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 8 + + const elemsize = 96 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 96 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC9(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 9 + + const elemsize = 112 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 112 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC10(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 10 + + const elemsize = 128 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 128 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC11(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 11 + + const elemsize = 144 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 144 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC12(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 12 + + const elemsize = 160 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 160 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC13(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 13 + + const elemsize = 176 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 176 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC14(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 14 + + const elemsize = 192 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 192 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC15(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 15 + + const elemsize = 208 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 208 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC16(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 16 + + const elemsize = 224 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 224 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC17(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 17 + + const elemsize = 240 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 240 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC18(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 18 + + const elemsize = 256 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 256 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC19(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 19 + + const elemsize = 288 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 288 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC20(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 20 + + const elemsize = 320 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 320 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC21(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 21 + + const elemsize = 352 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 352 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC22(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 22 + + const elemsize = 384 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 384 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC23(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 23 + + const elemsize = 416 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 416 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC24(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 24 + + const elemsize = 448 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 448 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC25(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 25 + + const elemsize = 480 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 480 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +func mallocgcSmallNoScanSC26(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + lockRankMayQueueFinalizer() + + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + const sizeclass = 26 + + const elemsize = 512 + + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(1) + span := c.alloc[spc] + + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)* + 512 + + span.base()) + } + } + } + v := nextFreeFastResult + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + publicationBarrier() + + if writeBarrier.enabled { + + gcmarknewobject(span, uintptr(x)) + } else { + + span.freeIndexForScan = span.freeindex + } + + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + if valgrindenabled { + valgrindMalloc(x, size) + } + + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} diff --git a/src/runtime/malloc_stubs.go b/src/runtime/malloc_stubs.go new file mode 100644 index 00000000000000..224746f3d41124 --- /dev/null +++ b/src/runtime/malloc_stubs.go @@ -0,0 +1,595 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains stub functions that are not meant to be called directly, +// but that will be assembled together using the inlining logic in runtime/_mkmalloc +// to produce a full mallocgc function that's specialized for a span class +// or specific size in the case of the tiny allocator. +// +// To assemble a mallocgc function, the mallocStub function is cloned, and the call to +// inlinedMalloc is replaced with the inlined body of smallScanNoHeaderStub, +// smallNoScanStub or tinyStub, depending on the parameters being specialized. +// +// The size_ (for the tiny case) and elemsize_, sizeclass_, and noscanint_ (for all three cases) +// identifiers are replaced with the value of the parameter in the specialized case. +// The nextFreeFastStub, nextFreeFastTiny, heapSetTypeNoHeaderStub, and writeHeapBitsSmallStub +// functions are also inlined by _mkmalloc. + +package runtime + +import ( + "internal/goarch" + "internal/runtime/sys" + "unsafe" +) + +// These identifiers will all be replaced by the inliner. So their values don't +// really matter: they just need to be set so that the stub functions, which +// will never be used on their own, can compile. elemsize_ can't be set to +// zero because we divide by it in nextFreeFastTiny, and the compiler would +// complain about a division by zero. Its replaced value will always be greater +// than zero. +const elemsize_ = 8 +const sizeclass_ = 0 +const noscanint_ = 0 +const size_ = 0 + +func malloc0(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + // Short-circuit zero-sized allocation requests. + return unsafe.Pointer(&zerobase) +} + +func mallocPanic(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + panic("not defined for sizeclass") +} + +// WARNING: mallocStub does not do any work for sanitizers so callers need +// to steer out of this codepath early if sanitizers are enabled. +func mallocStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer { + if doubleCheckMalloc { + if gcphase == _GCmarktermination { + throw("mallocgc called with gcphase == _GCmarktermination") + } + } + + // It's possible for any malloc to trigger sweeping, which may in + // turn queue finalizers. Record this dynamic lock edge. + // N.B. Compiled away if lockrank experiment is not enabled. + lockRankMayQueueFinalizer() + + // Pre-malloc debug hooks. + if debug.malloc { + if x := preMallocgcDebug(size, typ); x != nil { + return x + } + } + + // Assist the GC if needed. + if gcBlackenEnabled != 0 { + deductAssistCredit(size) + } + + // Actually do the allocation. + x, elemsize := inlinedMalloc(size, typ, needzero) + + // Notify valgrind, if enabled. + // To allow the compiler to not know about valgrind, we do valgrind instrumentation + // unlike the other sanitizers. + if valgrindenabled { + valgrindMalloc(x, size) + } + + // Adjust our GC assist debt to account for internal fragmentation. + if gcBlackenEnabled != 0 && elemsize != 0 { + if assistG := getg().m.curg; assistG != nil { + assistG.gcAssistBytes -= int64(elemsize - size) + } + } + + // Post-malloc debug hooks. + if debug.malloc { + postMallocgcDebug(x, elemsize, typ) + } + return x +} + +// inlinedMalloc will never be called. It is defined just so that the compiler can compile +// the mallocStub function, which will also never be called, but instead used as a template +// to generate a size-specialized malloc function. The call to inlinedMalloc in mallocStub +// will be replaced with the inlined body of smallScanNoHeaderStub, smallNoScanStub, or tinyStub +// when generating the size-specialized malloc function. See the comment at the top of this +// file for more information. +func inlinedMalloc(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { + return unsafe.Pointer(uintptr(0)), 0 +} + +func doubleCheckSmallScanNoHeader(size uintptr, typ *_type, mp *m) { + if mp.mallocing != 0 { + throw("malloc deadlock") + } + if mp.gsignal == getg() { + throw("malloc during signal") + } + if typ == nil || !typ.Pointers() { + throw("noscan allocated in scan-only path") + } + if !heapBitsInSpan(size) { + throw("heap bits in not in span for non-header-only path") + } +} + +func smallScanNoHeaderStub(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { + const sizeclass = sizeclass_ + const elemsize = elemsize_ + + // Set mp.mallocing to keep from being preempted by GC. + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallScanNoHeader(size, typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(noscanint_) + span := c.alloc[spc] + v := nextFreeFastStub(span) + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + if goarch.PtrSize == 8 && sizeclass == 1 { + // initHeapBits already set the pointer bits for the 8-byte sizeclass + // on 64-bit platforms. + c.scanAlloc += 8 + } else { + dataSize := size // make the inliner happy + x := uintptr(x) + scanSize := heapSetTypeNoHeaderStub(x, dataSize, typ, span) + c.scanAlloc += scanSize + } + + // Ensure that the stores above that initialize x to + // type-safe memory and set the heap bits occur before + // the caller can make x observable to the garbage + // collector. Otherwise, on weakly ordered machines, + // the garbage collector could follow a pointer to x, + // but see uninitialized memory or stale heap bits. + publicationBarrier() + + if writeBarrier.enabled { + // Allocate black during GC. + // All slots hold nil so no scanning is needed. + // This may be racing with GC so do it atomically if there can be + // a race marking the bit. + gcmarknewobject(span, uintptr(x)) + } else { + // Track the last free index before the mark phase. This field + // is only used by the garbage collector. During the mark phase + // this is used by the conservative scanner to filter out objects + // that are both free and recently-allocated. It's safe to do that + // because we allocate-black if the GC is enabled. The conservative + // scanner produces pointers out of thin air, so without additional + // synchronization it might otherwise observe a partially-initialized + // object, which could crash the program. + span.freeIndexForScan = span.freeindex + } + + // Note cache c only valid while m acquired; see #47302 + // + // N.B. Use the full size because that matches how the GC + // will update the mem profile on the "free" side. + // + // TODO(mknyszek): We should really count the header as part + // of gc_sys or something. The code below just pretends it is + // internal fragmentation and matches the GC's accounting by + // using the whole allocation slot. + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + return x, elemsize +} + +func doubleCheckSmallNoScan(typ *_type, mp *m) { + if mp.mallocing != 0 { + throw("malloc deadlock") + } + if mp.gsignal == getg() { + throw("malloc during signal") + } + if typ != nil && typ.Pointers() { + throw("expected noscan type for noscan alloc") + } +} + +func smallNoScanStub(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { + // TODO(matloob): Add functionality to mkmalloc to allow us to inline a non-constant + // sizeclass_ and elemsize_ value (instead just set to the expressions to look up the size class + // and elemsize. We'd also need to teach mkmalloc that values that are touched by these (specifically + // spc below) should turn into vars. This would allow us to generate mallocgcSmallNoScan itself, + // so that its code could not diverge from the generated functions. + const sizeclass = sizeclass_ + const elemsize = elemsize_ + + // Set mp.mallocing to keep from being preempted by GC. + mp := acquirem() + if doubleCheckMalloc { + doubleCheckSmallNoScan(typ, mp) + } + mp.mallocing = 1 + + checkGCTrigger := false + c := getMCache(mp) + const spc = spanClass(sizeclass<<1) | spanClass(noscanint_) + span := c.alloc[spc] + v := nextFreeFastStub(span) + if v == 0 { + v, span, checkGCTrigger = c.nextFree(spc) + } + x := unsafe.Pointer(v) + if needzero && span.needzero != 0 { + memclrNoHeapPointers(x, elemsize) + } + + // Ensure that the stores above that initialize x to + // type-safe memory and set the heap bits occur before + // the caller can make x observable to the garbage + // collector. Otherwise, on weakly ordered machines, + // the garbage collector could follow a pointer to x, + // but see uninitialized memory or stale heap bits. + publicationBarrier() + + if writeBarrier.enabled { + // Allocate black during GC. + // All slots hold nil so no scanning is needed. + // This may be racing with GC so do it atomically if there can be + // a race marking the bit. + gcmarknewobject(span, uintptr(x)) + } else { + // Track the last free index before the mark phase. This field + // is only used by the garbage collector. During the mark phase + // this is used by the conservative scanner to filter out objects + // that are both free and recently-allocated. It's safe to do that + // because we allocate-black if the GC is enabled. The conservative + // scanner produces pointers out of thin air, so without additional + // synchronization it might otherwise observe a partially-initialized + // object, which could crash the program. + span.freeIndexForScan = span.freeindex + } + + // Note cache c only valid while m acquired; see #47302 + // + // N.B. Use the full size because that matches how the GC + // will update the mem profile on the "free" side. + // + // TODO(mknyszek): We should really count the header as part + // of gc_sys or something. The code below just pretends it is + // internal fragmentation and matches the GC's accounting by + // using the whole allocation slot. + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + return x, elemsize +} + +func doubleCheckTiny(size uintptr, typ *_type, mp *m) { + if mp.mallocing != 0 { + throw("malloc deadlock") + } + if mp.gsignal == getg() { + throw("malloc during signal") + } + if typ != nil && typ.Pointers() { + throw("expected noscan for tiny alloc") + } +} + +func tinyStub(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) { + const constsize = size_ + const elemsize = elemsize_ + + // Set mp.mallocing to keep from being preempted by GC. + mp := acquirem() + if doubleCheckMalloc { + doubleCheckTiny(constsize, typ, mp) + } + mp.mallocing = 1 + + // Tiny allocator. + // + // Tiny allocator combines several tiny allocation requests + // into a single memory block. The resulting memory block + // is freed when all subobjects are unreachable. The subobjects + // must be noscan (don't have pointers), this ensures that + // the amount of potentially wasted memory is bounded. + // + // Size of the memory block used for combining (maxTinySize) is tunable. + // Current setting is 16 bytes, which relates to 2x worst case memory + // wastage (when all but one subobjects are unreachable). + // 8 bytes would result in no wastage at all, but provides less + // opportunities for combining. + // 32 bytes provides more opportunities for combining, + // but can lead to 4x worst case wastage. + // The best case winning is 8x regardless of block size. + // + // Objects obtained from tiny allocator must not be freed explicitly. + // So when an object will be freed explicitly, we ensure that + // its size >= maxTinySize. + // + // SetFinalizer has a special case for objects potentially coming + // from tiny allocator, it such case it allows to set finalizers + // for an inner byte of a memory block. + // + // The main targets of tiny allocator are small strings and + // standalone escaping variables. On a json benchmark + // the allocator reduces number of allocations by ~12% and + // reduces heap size by ~20%. + c := getMCache(mp) + off := c.tinyoffset + // Align tiny pointer for required (conservative) alignment. + if constsize&7 == 0 { + off = alignUp(off, 8) + } else if goarch.PtrSize == 4 && constsize == 12 { + // Conservatively align 12-byte objects to 8 bytes on 32-bit + // systems so that objects whose first field is a 64-bit + // value is aligned to 8 bytes and does not cause a fault on + // atomic access. See issue 37262. + // TODO(mknyszek): Remove this workaround if/when issue 36606 + // is resolved. + off = alignUp(off, 8) + } else if constsize&3 == 0 { + off = alignUp(off, 4) + } else if constsize&1 == 0 { + off = alignUp(off, 2) + } + if off+constsize <= maxTinySize && c.tiny != 0 { + // The object fits into existing tiny block. + x := unsafe.Pointer(c.tiny + off) + c.tinyoffset = off + constsize + c.tinyAllocs++ + mp.mallocing = 0 + releasem(mp) + return x, 0 + } + // Allocate a new maxTinySize block. + checkGCTrigger := false + span := c.alloc[tinySpanClass] + v := nextFreeFastTiny(span) + if v == 0 { + v, span, checkGCTrigger = c.nextFree(tinySpanClass) + } + x := unsafe.Pointer(v) + (*[2]uint64)(x)[0] = 0 // Always zero + (*[2]uint64)(x)[1] = 0 + // See if we need to replace the existing tiny block with the new one + // based on amount of remaining free space. + if !raceenabled && (constsize < c.tinyoffset || c.tiny == 0) { + // Note: disabled when race detector is on, see comment near end of this function. + c.tiny = uintptr(x) + c.tinyoffset = constsize + } + + // Ensure that the stores above that initialize x to + // type-safe memory and set the heap bits occur before + // the caller can make x observable to the garbage + // collector. Otherwise, on weakly ordered machines, + // the garbage collector could follow a pointer to x, + // but see uninitialized memory or stale heap bits. + publicationBarrier() + + if writeBarrier.enabled { + // Allocate black during GC. + // All slots hold nil so no scanning is needed. + // This may be racing with GC so do it atomically if there can be + // a race marking the bit. + gcmarknewobject(span, uintptr(x)) + } else { + // Track the last free index before the mark phase. This field + // is only used by the garbage collector. During the mark phase + // this is used by the conservative scanner to filter out objects + // that are both free and recently-allocated. It's safe to do that + // because we allocate-black if the GC is enabled. The conservative + // scanner produces pointers out of thin air, so without additional + // synchronization it might otherwise observe a partially-initialized + // object, which could crash the program. + span.freeIndexForScan = span.freeindex + } + + // Note cache c only valid while m acquired; see #47302 + // + // N.B. Use the full size because that matches how the GC + // will update the mem profile on the "free" side. + // + // TODO(mknyszek): We should really count the header as part + // of gc_sys or something. The code below just pretends it is + // internal fragmentation and matches the GC's accounting by + // using the whole allocation slot. + c.nextSample -= int64(elemsize) + if c.nextSample < 0 || MemProfileRate != c.memProfRate { + profilealloc(mp, x, elemsize) + } + mp.mallocing = 0 + releasem(mp) + + if checkGCTrigger { + if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { + gcStart(t) + } + } + + if raceenabled { + // Pad tinysize allocations so they are aligned with the end + // of the tinyalloc region. This ensures that any arithmetic + // that goes off the top end of the object will be detectable + // by checkptr (issue 38872). + // Note that we disable tinyalloc when raceenabled for this to work. + // TODO: This padding is only performed when the race detector + // is enabled. It would be nice to enable it if any package + // was compiled with checkptr, but there's no easy way to + // detect that (especially at compile time). + // TODO: enable this padding for all allocations, not just + // tinyalloc ones. It's tricky because of pointer maps. + // Maybe just all noscan objects? + x = add(x, elemsize-constsize) + } + return x, elemsize +} + +// TODO(matloob): Should we let the go compiler inline this instead of using mkmalloc? +// We won't be able to use elemsize_ but that's probably ok. +func nextFreeFastTiny(span *mspan) gclinkptr { + const nbytes = 8192 + const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / elemsize_) + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) // Is there a free object in the allocCache? + result := span.freeindex + uint16(theBit) + if result < nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)*elemsize_ + span.base()) + } + } + } + return nextFreeFastResult +} + +func nextFreeFastStub(span *mspan) gclinkptr { + var nextFreeFastResult gclinkptr + if span.allocCache != 0 { + theBit := sys.TrailingZeros64(span.allocCache) // Is there a free object in the allocCache? + result := span.freeindex + uint16(theBit) + if result < span.nelems { + freeidx := result + 1 + if !(freeidx%64 == 0 && freeidx != span.nelems) { + span.allocCache >>= uint(theBit + 1) + span.freeindex = freeidx + span.allocCount++ + nextFreeFastResult = gclinkptr(uintptr(result)*elemsize_ + span.base()) + } + } + } + return nextFreeFastResult +} + +func heapSetTypeNoHeaderStub(x, dataSize uintptr, typ *_type, span *mspan) uintptr { + if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(elemsize_)) { + throw("tried to write heap bits, but no heap bits in span") + } + scanSize := writeHeapBitsSmallStub(span, x, dataSize, typ) + if doubleCheckHeapSetType { + doubleCheckHeapType(x, dataSize, typ, nil, span) + } + return scanSize +} + +// writeHeapBitsSmallStub writes the heap bits for small objects whose ptr/scalar data is +// stored as a bitmap at the end of the span. +// +// Assumes dataSize is <= ptrBits*goarch.PtrSize. x must be a pointer into the span. +// heapBitsInSpan(dataSize) must be true. dataSize must be >= typ.Size_. +// +//go:nosplit +func writeHeapBitsSmallStub(span *mspan, x, dataSize uintptr, typ *_type) uintptr { + // The objects here are always really small, so a single load is sufficient. + src0 := readUintptr(getGCMask(typ)) + + const elemsize = elemsize_ + + // Create repetitions of the bitmap if we have a small slice backing store. + scanSize := typ.PtrBytes + src := src0 + if typ.Size_ == goarch.PtrSize { + src = (1 << (dataSize / goarch.PtrSize)) - 1 + } else { + // N.B. We rely on dataSize being an exact multiple of the type size. + // The alternative is to be defensive and mask out src to the length + // of dataSize. The purpose is to save on one additional masking operation. + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } + for i := typ.Size_; i < dataSize; i += typ.Size_ { + src |= src0 << (i / goarch.PtrSize) + scanSize += typ.Size_ + } + } + + // Since we're never writing more than one uintptr's worth of bits, we're either going + // to do one or two writes. + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize) + dst := unsafe.Pointer(dstBase) + o := (x - span.base()) / goarch.PtrSize + i := o / ptrBits + j := o % ptrBits + const bits uintptr = elemsize / goarch.PtrSize + // In the if statement below, we have to do two uintptr writes if the bits + // we need to write straddle across two different memory locations. But if + // the number of bits we're writing divides evenly into the number of bits + // in the uintptr we're writing, this can never happen. Since bitsIsPowerOfTwo + // is a compile-time constant in the generated code, in the case where the size is + // a power of two less than or equal to ptrBits, the compiler can remove the + // 'two writes' branch of the if statement and always do only one write without + // the check. + const bitsIsPowerOfTwo = bits&(bits-1) == 0 + if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) { + // Two writes. + bits0 := ptrBits - j + bits1 := bits - bits0 + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) + } else { + // One write. + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)< ptrbits we always take the other branch + } + + const doubleCheck = false + if doubleCheck { + writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ) + } + return scanSize +} + +func writeHeapBitsDoubleCheck(span *mspan, x, dataSize, src, src0, i, j, bits uintptr, typ *_type) { + srcRead := span.heapBitsSmallForAddr(x) + if srcRead != src { + print("runtime: x=", hex(x), " i=", i, " j=", j, " bits=", bits, "\n") + print("runtime: dataSize=", dataSize, " typ.Size_=", typ.Size_, " typ.PtrBytes=", typ.PtrBytes, "\n") + print("runtime: src0=", hex(src0), " src="http://github.com/,%20hex(src)," srcRead=", hex(srcRead), "\n") + throw("bad pointer bits written for small object") + } +} diff --git a/src/runtime/malloc_tables_generated.go b/src/runtime/malloc_tables_generated.go new file mode 100644 index 00000000000000..36650881fe8bea --- /dev/null +++ b/src/runtime/malloc_tables_generated.go @@ -0,0 +1,1038 @@ +// Code generated by mkmalloc.go; DO NOT EDIT. +//go:build !plan9 + +package runtime + +import "unsafe" + +var mallocScanTable = [513]func(size uintptr, typ *_type, needzero bool) unsafe.Pointer{ + mallocPanic, + mallocgcSmallScanNoHeaderSC1, + mallocgcSmallScanNoHeaderSC1, + mallocgcSmallScanNoHeaderSC1, + mallocgcSmallScanNoHeaderSC1, + mallocgcSmallScanNoHeaderSC1, + mallocgcSmallScanNoHeaderSC1, + mallocgcSmallScanNoHeaderSC1, + mallocgcSmallScanNoHeaderSC1, + mallocgcSmallScanNoHeaderSC2, + mallocgcSmallScanNoHeaderSC2, + mallocgcSmallScanNoHeaderSC2, + mallocgcSmallScanNoHeaderSC2, + mallocgcSmallScanNoHeaderSC2, + mallocgcSmallScanNoHeaderSC2, + mallocgcSmallScanNoHeaderSC2, + mallocgcSmallScanNoHeaderSC2, + mallocgcSmallScanNoHeaderSC3, + mallocgcSmallScanNoHeaderSC3, + mallocgcSmallScanNoHeaderSC3, + mallocgcSmallScanNoHeaderSC3, + mallocgcSmallScanNoHeaderSC3, + mallocgcSmallScanNoHeaderSC3, + mallocgcSmallScanNoHeaderSC3, + mallocgcSmallScanNoHeaderSC3, + mallocgcSmallScanNoHeaderSC4, + mallocgcSmallScanNoHeaderSC4, + mallocgcSmallScanNoHeaderSC4, + mallocgcSmallScanNoHeaderSC4, + mallocgcSmallScanNoHeaderSC4, + mallocgcSmallScanNoHeaderSC4, + mallocgcSmallScanNoHeaderSC4, + mallocgcSmallScanNoHeaderSC4, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC5, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC6, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC7, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC8, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC9, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC10, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC11, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC12, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC13, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC14, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC15, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC16, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC17, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC18, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC19, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC20, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC21, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC22, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC23, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC24, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC25, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, + mallocgcSmallScanNoHeaderSC26, +} + +var mallocNoScanTable = [513]func(size uintptr, typ *_type, needzero bool) unsafe.Pointer{ + mallocPanic, + mallocTiny1, + mallocTiny2, + mallocTiny3, + mallocTiny4, + mallocTiny5, + mallocTiny6, + mallocTiny7, + mallocTiny8, + mallocTiny9, + mallocTiny10, + mallocTiny11, + mallocTiny12, + mallocTiny13, + mallocTiny14, + mallocTiny15, + mallocgcSmallNoScanSC2, + mallocgcSmallNoScanSC3, + mallocgcSmallNoScanSC3, + mallocgcSmallNoScanSC3, + mallocgcSmallNoScanSC3, + mallocgcSmallNoScanSC3, + mallocgcSmallNoScanSC3, + mallocgcSmallNoScanSC3, + mallocgcSmallNoScanSC3, + mallocgcSmallNoScanSC4, + mallocgcSmallNoScanSC4, + mallocgcSmallNoScanSC4, + mallocgcSmallNoScanSC4, + mallocgcSmallNoScanSC4, + mallocgcSmallNoScanSC4, + mallocgcSmallNoScanSC4, + mallocgcSmallNoScanSC4, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC5, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC6, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC7, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC8, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC9, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC10, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC11, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC12, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC13, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC14, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC15, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC16, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC17, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC18, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC19, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC20, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC21, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC22, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC23, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC24, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC25, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, + mallocgcSmallNoScanSC26, +} diff --git a/src/runtime/malloc_tables_plan9.go b/src/runtime/malloc_tables_plan9.go new file mode 100644 index 00000000000000..4d2740bbb29916 --- /dev/null +++ b/src/runtime/malloc_tables_plan9.go @@ -0,0 +1,14 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build plan9 + +package runtime + +import "unsafe" + +var ( + mallocScanTable []func(size uintptr, typ *_type, needzero bool) unsafe.Pointer + mallocNoScanTable []func(size uintptr, typ *_type, needzero bool) unsafe.Pointer +) diff --git a/src/runtime/malloc_test.go b/src/runtime/malloc_test.go index 8c162fbea4b158..bf58947bbc5856 100644 --- a/src/runtime/malloc_test.go +++ b/src/runtime/malloc_test.go @@ -7,6 +7,7 @@ package runtime_test import ( "flag" "fmt" + "internal/asan" "internal/race" "internal/testenv" "os" @@ -157,6 +158,9 @@ func TestTinyAlloc(t *testing.T) { if runtime.Raceenabled { t.Skip("tinyalloc suppressed when running in race mode") } + if asan.Enabled { + t.Skip("tinyalloc suppressed when running in asan mode due to redzone") + } const N = 16 var v [N]unsafe.Pointer for i := range v { @@ -182,6 +186,9 @@ func TestTinyAllocIssue37262(t *testing.T) { if runtime.Raceenabled { t.Skip("tinyalloc suppressed when running in race mode") } + if asan.Enabled { + t.Skip("tinyalloc suppressed when running in asan mode due to redzone") + } // Try to cause an alignment access fault // by atomically accessing the first 64-bit // value of a tiny-allocated object. @@ -263,12 +270,10 @@ type acLink struct { var arenaCollisionSink []*acLink func TestArenaCollision(t *testing.T) { - testenv.MustHaveExec(t) - // Test that mheap.sysAlloc handles collisions with other // memory mappings. if os.Getenv("TEST_ARENA_COLLISION") != "1" { - cmd := testenv.CleanCmdEnv(exec.Command(os.Args[0], "-test.run=^TestArenaCollision$", "-test.v")) + cmd := testenv.CleanCmdEnv(exec.Command(testenv.Executable(t), "-test.run=^TestArenaCollision$", "-test.v")) cmd.Env = append(cmd.Env, "TEST_ARENA_COLLISION=1") out, err := cmd.CombinedOutput() if race.Enabled { @@ -447,3 +452,13 @@ func BenchmarkGoroutineIdle(b *testing.B) { close(quit) time.Sleep(10 * time.Millisecond) } + +func TestMkmalloc(t *testing.T) { + testenv.MustHaveGoRun(t) + testenv.MustHaveExternalNetwork(t) // To download the golang.org/x/tools dependency. + output, err := exec.Command("go", "-C", "_mkmalloc", "test").CombinedOutput() + t.Logf("test output:\n%s", output) + if err != nil { + t.Errorf("_mkmalloc tests failed: %v", err) + } +} diff --git a/src/runtime/map.go b/src/runtime/map.go index 9169d5733eef4f..4a0713cfc4c05e 100644 --- a/src/runtime/map.go +++ b/src/runtime/map.go @@ -4,284 +4,31 @@ package runtime -// This file contains the implementation of Go's map type. -// -// A map is just a hash table. The data is arranged -// into an array of buckets. Each bucket contains up to -// 8 key/elem pairs. The low-order bits of the hash are -// used to select a bucket. Each bucket contains a few -// high-order bits of each hash to distinguish the entries -// within a single bucket. -// -// If more than 8 keys hash to a bucket, we chain on -// extra buckets. -// -// When the hashtable grows, we allocate a new array -// of buckets twice as big. Buckets are incrementally -// copied from the old bucket array to the new bucket array. -// -// Map iterators walk through the array of buckets and -// return the keys in walk order (bucket #, then overflow -// chain order, then bucket index). To maintain iteration -// semantics, we never move keys within their bucket (if -// we did, keys might be returned 0 or 2 times). When -// growing the table, iterators remain iterating through the -// old table and must check the new table if the bucket -// they are iterating through has been moved ("evacuated") -// to the new table. - -// Picking loadFactor: too large and we have lots of overflow -// buckets, too small and we waste a lot of space. I wrote -// a simple program to check some stats for different loads: -// (64-bit, 8 byte keys and elems) -// loadFactor %overflow bytes/entry hitprobe missprobe -// 4.00 2.13 20.77 3.00 4.00 -// 4.50 4.05 17.30 3.25 4.50 -// 5.00 6.85 14.77 3.50 5.00 -// 5.50 10.55 12.94 3.75 5.50 -// 6.00 15.27 11.67 4.00 6.00 -// 6.50 20.90 10.79 4.25 6.50 -// 7.00 27.14 10.15 4.50 7.00 -// 7.50 34.03 9.73 4.75 7.50 -// 8.00 41.10 9.40 5.00 8.00 -// -// %overflow = percentage of buckets which have an overflow bucket -// bytes/entry = overhead bytes used per key/elem pair -// hitprobe = # of entries to check when looking up a present key -// missprobe = # of entries to check when looking up an absent key -// -// Keep in mind this data is for maximally loaded tables, i.e. just -// before the table grows. Typical tables will be somewhat less loaded. - import ( "internal/abi" - "internal/goarch" - "internal/runtime/atomic" - "internal/runtime/math" + "internal/runtime/maps" + "internal/runtime/sys" "unsafe" ) const ( - // Maximum number of key/elem pairs a bucket can hold. - bucketCntBits = abi.MapBucketCountBits - - // Maximum average load of a bucket that triggers growth is bucketCnt*13/16 (about 80% full) - // Because of minimum alignment rules, bucketCnt is known to be at least 8. - // Represent as loadFactorNum/loadFactorDen, to allow integer math. - loadFactorDen = 2 - loadFactorNum = loadFactorDen * abi.MapBucketCount * 13 / 16 - - // data offset should be the size of the bmap struct, but needs to be - // aligned correctly. For amd64p32 this means 64-bit alignment - // even though pointers are 32 bit. - dataOffset = unsafe.Offsetof(struct { - b bmap - v int64 - }{}.v) - - // Possible tophash values. We reserve a few possibilities for special marks. - // Each bucket (including its overflow buckets, if any) will have either all or none of its - // entries in the evacuated* states (except during the evacuate() method, which only happens - // during map writes and thus no one else can observe the map during that time). - emptyRest = 0 // this cell is empty, and there are no more non-empty cells at higher indexes or overflows. - emptyOne = 1 // this cell is empty - evacuatedX = 2 // key/elem is valid. Entry has been evacuated to first half of larger table. - evacuatedY = 3 // same as above, but evacuated to second half of larger table. - evacuatedEmpty = 4 // cell is empty, bucket is evacuated. - minTopHash = 5 // minimum tophash for a normal filled cell. - - // flags - iterator = 1 // there may be an iterator using buckets - oldIterator = 2 // there may be an iterator using oldbuckets - hashWriting = 4 // a goroutine is writing to the map - sameSizeGrow = 8 // the current map growth is to a new map of the same size - - // sentinel bucket ID for iterator checks - noCheck = 1<<(8*goarch.PtrSize) - 1 + // TODO: remove? These are used by tests but not the actual map + loadFactorNum = 7 + loadFactorDen = 8 ) -// isEmpty reports whether the given tophash array entry represents an empty bucket entry. -func isEmpty(x uint8) bool { - return x <= emptyOne -} - -// A header for a Go map. -type hmap struct { - // Note: the format of the hmap is also encoded in cmd/compile/internal/reflectdata/reflect.go. - // Make sure this stays in sync with the compiler's definition. - count int // # live cells == size of map. Must be first (used by len() builtin) - flags uint8 - B uint8 // log_2 of # of buckets (can hold up to loadFactor * 2^B items) - noverflow uint16 // approximate number of overflow buckets; see incrnoverflow for details - hash0 uint32 // hash seed - - buckets unsafe.Pointer // array of 2^B Buckets. may be nil if count==0. - oldbuckets unsafe.Pointer // previous bucket array of half the size, non-nil only when growing - nevacuate uintptr // progress counter for evacuation (buckets less than this have been evacuated) - - extra *mapextra // optional fields -} - -// mapextra holds fields that are not present on all maps. -type mapextra struct { - // If both key and elem do not contain pointers and are inline, then we mark bucket - // type as containing no pointers. This avoids scanning such maps. - // However, bmap.overflow is a pointer. In order to keep overflow buckets - // alive, we store pointers to all overflow buckets in hmap.extra.overflow and hmap.extra.oldoverflow. - // overflow and oldoverflow are only used if key and elem do not contain pointers. - // overflow contains overflow buckets for hmap.buckets. - // oldoverflow contains overflow buckets for hmap.oldbuckets. - // The indirection allows to store a pointer to the slice in hiter. - overflow *[]*bmap - oldoverflow *[]*bmap - - // nextOverflow holds a pointer to a free overflow bucket. - nextOverflow *bmap -} - -// A bucket for a Go map. -type bmap struct { - // tophash generally contains the top byte of the hash value - // for each key in this bucket. If tophash[0] < minTopHash, - // tophash[0] is a bucket evacuation state instead. - tophash [abi.MapBucketCount]uint8 - // Followed by bucketCnt keys and then bucketCnt elems. - // NOTE: packing all the keys together and then all the elems together makes the - // code a bit more complicated than alternating key/elem/key/elem/... but it allows - // us to eliminate padding which would be needed for, e.g., map[int64]int8. - // Followed by an overflow pointer. -} - -// A hash iteration structure. -// If you modify hiter, also change cmd/compile/internal/reflectdata/reflect.go -// and reflect/value.go to match the layout of this structure. -type hiter struct { - key unsafe.Pointer // Must be in first position. Write nil to indicate iteration end (see cmd/compile/internal/walk/range.go). - elem unsafe.Pointer // Must be in second position (see cmd/compile/internal/walk/range.go). - t *maptype - h *hmap - buckets unsafe.Pointer // bucket ptr at hash_iter initialization time - bptr *bmap // current bucket - overflow *[]*bmap // keeps overflow buckets of hmap.buckets alive - oldoverflow *[]*bmap // keeps overflow buckets of hmap.oldbuckets alive - startBucket uintptr // bucket iteration started at - offset uint8 // intra-bucket offset to start from during iteration (should be big enough to hold bucketCnt-1) - wrapped bool // already wrapped around from end of bucket array to beginning - B uint8 - i uint8 - bucket uintptr - checkBucket uintptr -} - -// bucketShift returns 1<> (goarch.PtrSize*8 - 8)) - if top < minTopHash { - top += minTopHash - } - return top -} - -func evacuated(b *bmap) bool { - h := b.tophash[0] - return h > emptyOne && h < minTopHash -} - -func (b *bmap) overflow(t *maptype) *bmap { - return *(**bmap)(add(unsafe.Pointer(b), uintptr(t.BucketSize)-goarch.PtrSize)) -} - -func (b *bmap) setoverflow(t *maptype, ovf *bmap) { - *(**bmap)(add(unsafe.Pointer(b), uintptr(t.BucketSize)-goarch.PtrSize)) = ovf -} - -func (b *bmap) keys() unsafe.Pointer { - return add(unsafe.Pointer(b), dataOffset) -} - -// incrnoverflow increments h.noverflow. -// noverflow counts the number of overflow buckets. -// This is used to trigger same-size map growth. -// See also tooManyOverflowBuckets. -// To keep hmap small, noverflow is a uint16. -// When there are few buckets, noverflow is an exact count. -// When there are many buckets, noverflow is an approximate count. -func (h *hmap) incrnoverflow() { - // We trigger same-size map growth if there are - // as many overflow buckets as buckets. - // We need to be able to count to 1< maxAlloc { +func makemap(t *abi.MapType, hint int, m *maps.Map) *maps.Map { + if hint < 0 { hint = 0 } - // initialize Hmap - if h == nil { - h = new(hmap) - } - h.hash0 = uint32(rand()) - - // Find the size parameter B which will hold the requested # of elements. - // For hint < 0 overLoadFactor returns false since hint < bucketCnt. - B := uint8(0) - for overLoadFactor(hint, B) { - B++ - } - h.B = B - - // allocate initial hash table - // if B == 0, the buckets field is allocated lazily later (in mapassign) - // If hint is large zeroing this memory could take a while. - if h.B != 0 { - var nextOverflow *bmap - h.buckets, nextOverflow = makeBucketArray(t, h.B, nil) - if nextOverflow != nil { - h.extra = new(mapextra) - h.extra.nextOverflow = nextOverflow - } - } - - return h -} - -// makeBucketArray initializes a backing array for map buckets. -// 1<= 4 { - // Add on the estimated number of overflow buckets - // required to insert the median number of elements - // used with this value of b. - nbuckets += bucketShift(b - 4) - sz := t.Bucket.Size_ * nbuckets - up := roundupsize(sz, !t.Bucket.Pointers()) - if up != sz { - nbuckets = up / t.Bucket.Size_ - } - } - - if dirtyalloc == nil { - buckets = newarray(t.Bucket, int(nbuckets)) - } else { - // dirtyalloc was previously generated by - // the above newarray(t.Bucket, int(nbuckets)) - // but may not be empty. - buckets = dirtyalloc - size := t.Bucket.Size_ * nbuckets - if t.Bucket.Pointers() { - memclrHasPointers(buckets, size) - } else { - memclrNoHeapPointers(buckets, size) - } - } - - if base != nbuckets { - // We preallocated some overflow buckets. - // To keep the overhead of tracking these overflow buckets to a minimum, - // we use the convention that if a preallocated overflow bucket's overflow - // pointer is nil, then there are more available by bumping the pointer. - // We need a safe non-nil pointer for the last overflow bucket; just use buckets. - nextOverflow = (*bmap)(add(buckets, base*uintptr(t.BucketSize))) - last := (*bmap)(add(buckets, (nbuckets-1)*uintptr(t.BucketSize))) - last.setoverflow(t, (*bmap)(buckets)) - } - return buckets, nextOverflow + return maps.NewMap(t, uintptr(hint), m, maxAlloc) } // mapaccess1 returns a pointer to h[key]. Never returns nil, instead @@ -406,66 +72,12 @@ func makeBucketArray(t *maptype, b uint8, dirtyalloc unsafe.Pointer) (buckets un // the key is not in the map. // NOTE: The returned pointer may keep the whole map live, so don't // hold onto it for very long. -func mapaccess1(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer { - if raceenabled && h != nil { - callerpc := getcallerpc() - pc := abi.FuncPCABIInternal(mapaccess1) - racereadpc(unsafe.Pointer(h), callerpc, pc) - raceReadObjectPC(t.Key, key, callerpc, pc) - } - if msanenabled && h != nil { - msanread(key, t.Key.Size_) - } - if asanenabled && h != nil { - asanread(key, t.Key.Size_) - } - if h == nil || h.count == 0 { - if err := mapKeyError(t, key); err != nil { - panic(err) // see issue 23734 - } - return unsafe.Pointer(&zeroVal[0]) - } - if h.flags&hashWriting != 0 { - fatal("concurrent map read and map write") - } - hash := t.Hasher(key, uintptr(h.hash0)) - m := bucketMask(h.B) - b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.BucketSize))) - if c := h.oldbuckets; c != nil { - if !h.sameSizeGrow() { - // There used to be half as many buckets; mask down one more power of two. - m >>= 1 - } - oldb := (*bmap)(add(c, (hash&m)*uintptr(t.BucketSize))) - if !evacuated(oldb) { - b = oldb - } - } - top := tophash(hash) -bucketloop: - for ; b != nil; b = b.overflow(t) { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - if b.tophash[i] != top { - if b.tophash[i] == emptyRest { - break bucketloop - } - continue - } - k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.KeySize)) - if t.IndirectKey() { - k = *((*unsafe.Pointer)(k)) - } - if t.Key.Equal(key, k) { - e := add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+i*uintptr(t.ValueSize)) - if t.IndirectElem() { - e = *((*unsafe.Pointer)(e)) - } - return e - } - } - } - return unsafe.Pointer(&zeroVal[0]) -} +// +// mapaccess1 is pushed from internal/runtime/maps. We could just call it, but +// we want to avoid one layer of call. +// +//go:linkname mapaccess1 +func mapaccess1(t *abi.MapType, m *maps.Map, key unsafe.Pointer) unsafe.Pointer // mapaccess2 should be an internal detail, // but widely used packages access it using linkname. @@ -476,134 +88,31 @@ bucketloop: // See go.dev/issue/67401. // //go:linkname mapaccess2 -func mapaccess2(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, bool) { - if raceenabled && h != nil { - callerpc := getcallerpc() - pc := abi.FuncPCABIInternal(mapaccess2) - racereadpc(unsafe.Pointer(h), callerpc, pc) - raceReadObjectPC(t.Key, key, callerpc, pc) - } - if msanenabled && h != nil { - msanread(key, t.Key.Size_) - } - if asanenabled && h != nil { - asanread(key, t.Key.Size_) - } - if h == nil || h.count == 0 { - if err := mapKeyError(t, key); err != nil { - panic(err) // see issue 23734 - } - return unsafe.Pointer(&zeroVal[0]), false - } - if h.flags&hashWriting != 0 { - fatal("concurrent map read and map write") - } - hash := t.Hasher(key, uintptr(h.hash0)) - m := bucketMask(h.B) - b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.BucketSize))) - if c := h.oldbuckets; c != nil { - if !h.sameSizeGrow() { - // There used to be half as many buckets; mask down one more power of two. - m >>= 1 - } - oldb := (*bmap)(add(c, (hash&m)*uintptr(t.BucketSize))) - if !evacuated(oldb) { - b = oldb - } - } - top := tophash(hash) -bucketloop: - for ; b != nil; b = b.overflow(t) { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - if b.tophash[i] != top { - if b.tophash[i] == emptyRest { - break bucketloop - } - continue - } - k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.KeySize)) - if t.IndirectKey() { - k = *((*unsafe.Pointer)(k)) - } - if t.Key.Equal(key, k) { - e := add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+i*uintptr(t.ValueSize)) - if t.IndirectElem() { - e = *((*unsafe.Pointer)(e)) - } - return e, true - } - } - } - return unsafe.Pointer(&zeroVal[0]), false -} +func mapaccess2(t *abi.MapType, m *maps.Map, key unsafe.Pointer) (unsafe.Pointer, bool) -// returns both key and elem. Used by map iterator. -func mapaccessK(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, unsafe.Pointer) { - if h == nil || h.count == 0 { - return nil, nil - } - hash := t.Hasher(key, uintptr(h.hash0)) - m := bucketMask(h.B) - b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.BucketSize))) - if c := h.oldbuckets; c != nil { - if !h.sameSizeGrow() { - // There used to be half as many buckets; mask down one more power of two. - m >>= 1 - } - oldb := (*bmap)(add(c, (hash&m)*uintptr(t.BucketSize))) - if !evacuated(oldb) { - b = oldb - } - } - top := tophash(hash) -bucketloop: - for ; b != nil; b = b.overflow(t) { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - if b.tophash[i] != top { - if b.tophash[i] == emptyRest { - break bucketloop - } - continue - } - k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.KeySize)) - if t.IndirectKey() { - k = *((*unsafe.Pointer)(k)) - } - if t.Key.Equal(key, k) { - e := add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+i*uintptr(t.ValueSize)) - if t.IndirectElem() { - e = *((*unsafe.Pointer)(e)) - } - return k, e - } - } - } - return nil, nil -} - -func mapaccess1_fat(t *maptype, h *hmap, key, zero unsafe.Pointer) unsafe.Pointer { - e := mapaccess1(t, h, key) +func mapaccess1_fat(t *abi.MapType, m *maps.Map, key, zero unsafe.Pointer) unsafe.Pointer { + e := mapaccess1(t, m, key) if e == unsafe.Pointer(&zeroVal[0]) { return zero } return e } -func mapaccess2_fat(t *maptype, h *hmap, key, zero unsafe.Pointer) (unsafe.Pointer, bool) { - e := mapaccess1(t, h, key) +func mapaccess2_fat(t *abi.MapType, m *maps.Map, key, zero unsafe.Pointer) (unsafe.Pointer, bool) { + e := mapaccess1(t, m, key) if e == unsafe.Pointer(&zeroVal[0]) { return zero, false } return e, true } -// Like mapaccess, but allocates a slot for the key if it is not present in the map. +// mapassign is pushed from internal/runtime/maps. We could just call it, but +// we want to avoid one layer of call. // // mapassign should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/sonic -// - github.com/cloudwego/frugal // - github.com/RomiChan/protobuf // - github.com/segmentio/encoding // - github.com/ugorji/go/codec @@ -612,122 +121,7 @@ func mapaccess2_fat(t *maptype, h *hmap, key, zero unsafe.Pointer) (unsafe.Point // See go.dev/issue/67401. // //go:linkname mapassign -func mapassign(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer { - if h == nil { - panic(plainError("assignment to entry in nil map")) - } - if raceenabled { - callerpc := getcallerpc() - pc := abi.FuncPCABIInternal(mapassign) - racewritepc(unsafe.Pointer(h), callerpc, pc) - raceReadObjectPC(t.Key, key, callerpc, pc) - } - if msanenabled { - msanread(key, t.Key.Size_) - } - if asanenabled { - asanread(key, t.Key.Size_) - } - if h.flags&hashWriting != 0 { - fatal("concurrent map writes") - } - hash := t.Hasher(key, uintptr(h.hash0)) - - // Set hashWriting after calling t.hasher, since t.hasher may panic, - // in which case we have not actually done a write. - h.flags ^= hashWriting - - if h.buckets == nil { - h.buckets = newobject(t.Bucket) // newarray(t.Bucket, 1) - } - -again: - bucket := hash & bucketMask(h.B) - if h.growing() { - growWork(t, h, bucket) - } - b := (*bmap)(add(h.buckets, bucket*uintptr(t.BucketSize))) - top := tophash(hash) - - var inserti *uint8 - var insertk unsafe.Pointer - var elem unsafe.Pointer -bucketloop: - for { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - if b.tophash[i] != top { - if isEmpty(b.tophash[i]) && inserti == nil { - inserti = &b.tophash[i] - insertk = add(unsafe.Pointer(b), dataOffset+i*uintptr(t.KeySize)) - elem = add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+i*uintptr(t.ValueSize)) - } - if b.tophash[i] == emptyRest { - break bucketloop - } - continue - } - k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.KeySize)) - if t.IndirectKey() { - k = *((*unsafe.Pointer)(k)) - } - if !t.Key.Equal(key, k) { - continue - } - // already have a mapping for key. Update it. - if t.NeedKeyUpdate() { - typedmemmove(t.Key, k, key) - } - elem = add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+i*uintptr(t.ValueSize)) - goto done - } - ovf := b.overflow(t) - if ovf == nil { - break - } - b = ovf - } - - // Did not find mapping for key. Allocate new cell & add entry. - - // If we hit the max load factor or we have too many overflow buckets, - // and we're not already in the middle of growing, start growing. - if !h.growing() && (overLoadFactor(h.count+1, h.B) || tooManyOverflowBuckets(h.noverflow, h.B)) { - hashGrow(t, h) - goto again // Growing the table invalidates everything, so try again - } - - if inserti == nil { - // The current bucket and all the overflow buckets connected to it are full, allocate a new one. - newb := h.newoverflow(t, b) - inserti = &newb.tophash[0] - insertk = add(unsafe.Pointer(newb), dataOffset) - elem = add(insertk, abi.MapBucketCount*uintptr(t.KeySize)) - } - - // store new key/elem at insert position - if t.IndirectKey() { - kmem := newobject(t.Key) - *(*unsafe.Pointer)(insertk) = kmem - insertk = kmem - } - if t.IndirectElem() { - vmem := newobject(t.Elem) - *(*unsafe.Pointer)(elem) = vmem - } - typedmemmove(t.Key, insertk, key) - *inserti = top - h.count++ - -done: - if h.flags&hashWriting == 0 { - fatal("concurrent map writes") - } - h.flags &^= hashWriting - if t.IndirectElem() { - elem = *((*unsafe.Pointer)(elem)) - } - return elem -} +func mapassign(t *abi.MapType, m *maps.Map, key unsafe.Pointer) unsafe.Pointer // mapdelete should be an internal detail, // but widely used packages access it using linkname. @@ -738,651 +132,57 @@ done: // See go.dev/issue/67401. // //go:linkname mapdelete -func mapdelete(t *maptype, h *hmap, key unsafe.Pointer) { - if raceenabled && h != nil { - callerpc := getcallerpc() +func mapdelete(t *abi.MapType, m *maps.Map, key unsafe.Pointer) { + if raceenabled && m != nil { + callerpc := sys.GetCallerPC() pc := abi.FuncPCABIInternal(mapdelete) - racewritepc(unsafe.Pointer(h), callerpc, pc) + racewritepc(unsafe.Pointer(m), callerpc, pc) raceReadObjectPC(t.Key, key, callerpc, pc) } - if msanenabled && h != nil { + if msanenabled && m != nil { msanread(key, t.Key.Size_) } - if asanenabled && h != nil { + if asanenabled && m != nil { asanread(key, t.Key.Size_) } - if h == nil || h.count == 0 { - if err := mapKeyError(t, key); err != nil { - panic(err) // see issue 23734 - } - return - } - if h.flags&hashWriting != 0 { - fatal("concurrent map writes") - } - - hash := t.Hasher(key, uintptr(h.hash0)) - - // Set hashWriting after calling t.hasher, since t.hasher may panic, - // in which case we have not actually done a write (delete). - h.flags ^= hashWriting - - bucket := hash & bucketMask(h.B) - if h.growing() { - growWork(t, h, bucket) - } - b := (*bmap)(add(h.buckets, bucket*uintptr(t.BucketSize))) - bOrig := b - top := tophash(hash) -search: - for ; b != nil; b = b.overflow(t) { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - if b.tophash[i] != top { - if b.tophash[i] == emptyRest { - break search - } - continue - } - k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.KeySize)) - k2 := k - if t.IndirectKey() { - k2 = *((*unsafe.Pointer)(k2)) - } - if !t.Key.Equal(key, k2) { - continue - } - // Only clear key if there are pointers in it. - if t.IndirectKey() { - *(*unsafe.Pointer)(k) = nil - } else if t.Key.Pointers() { - memclrHasPointers(k, t.Key.Size_) - } - e := add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+i*uintptr(t.ValueSize)) - if t.IndirectElem() { - *(*unsafe.Pointer)(e) = nil - } else if t.Elem.Pointers() { - memclrHasPointers(e, t.Elem.Size_) - } else { - memclrNoHeapPointers(e, t.Elem.Size_) - } - b.tophash[i] = emptyOne - // If the bucket now ends in a bunch of emptyOne states, - // change those to emptyRest states. - // It would be nice to make this a separate function, but - // for loops are not currently inlineable. - if i == abi.MapBucketCount-1 { - if b.overflow(t) != nil && b.overflow(t).tophash[0] != emptyRest { - goto notLast - } - } else { - if b.tophash[i+1] != emptyRest { - goto notLast - } - } - for { - b.tophash[i] = emptyRest - if i == 0 { - if b == bOrig { - break // beginning of initial bucket, we're done. - } - // Find previous bucket, continue at its last entry. - c := b - for b = bOrig; b.overflow(t) != c; b = b.overflow(t) { - } - i = abi.MapBucketCount - 1 - } else { - i-- - } - if b.tophash[i] != emptyOne { - break - } - } - notLast: - h.count-- - // Reset the hash seed to make it more difficult for attackers to - // repeatedly trigger hash collisions. See issue 25237. - if h.count == 0 { - h.hash0 = uint32(rand()) - } - break search - } - } - if h.flags&hashWriting == 0 { - fatal("concurrent map writes") - } - h.flags &^= hashWriting + m.Delete(t, key) } -// mapiterinit initializes the hiter struct used for ranging over maps. -// The hiter struct pointed to by 'it' is allocated on the stack -// by the compilers order pass or on the heap by reflect_mapiterinit. -// Both need to have zeroed hiter since the struct contains pointers. -// -// mapiterinit should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/bytedance/sonic -// - github.com/cloudwego/frugal -// - github.com/goccy/go-json -// - github.com/RomiChan/protobuf -// - github.com/segmentio/encoding -// - github.com/ugorji/go/codec -// - github.com/wI2L/jettison -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname mapiterinit -func mapiterinit(t *maptype, h *hmap, it *hiter) { - if raceenabled && h != nil { - callerpc := getcallerpc() - racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapiterinit)) - } - - it.t = t - if h == nil || h.count == 0 { - return +// mapIterStart initializes the Iter struct used for ranging over maps and +// performs the first step of iteration. The Iter struct pointed to by 'it' is +// allocated on the stack by the compilers order pass or on the heap by +// reflect. Both need to have zeroed it since the struct contains pointers. +func mapIterStart(t *abi.MapType, m *maps.Map, it *maps.Iter) { + if raceenabled && m != nil { + callerpc := sys.GetCallerPC() + racereadpc(unsafe.Pointer(m), callerpc, abi.FuncPCABIInternal(mapIterStart)) } - if unsafe.Sizeof(hiter{})/goarch.PtrSize != 12 { - throw("hash_iter size incorrect") // see cmd/compile/internal/reflectdata/reflect.go - } - it.h = h - - // grab snapshot of bucket state - it.B = h.B - it.buckets = h.buckets - if !t.Bucket.Pointers() { - // Allocate the current slice and remember pointers to both current and old. - // This preserves all relevant overflow buckets alive even if - // the table grows and/or overflow buckets are added to the table - // while we are iterating. - h.createOverflow() - it.overflow = h.extra.overflow - it.oldoverflow = h.extra.oldoverflow - } - - // decide where to start - r := uintptr(rand()) - it.startBucket = r & bucketMask(h.B) - it.offset = uint8(r >> h.B & (abi.MapBucketCount - 1)) - - // iterator state - it.bucket = it.startBucket - - // Remember we have an iterator. - // Can run concurrently with another mapiterinit(). - if old := h.flags; old&(iterator|oldIterator) != iterator|oldIterator { - atomic.Or8(&h.flags, iterator|oldIterator) - } - - mapiternext(it) + it.Init(t, m) + it.Next() } -// mapiternext should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/bytedance/sonic -// - github.com/cloudwego/frugal -// - github.com/RomiChan/protobuf -// - github.com/segmentio/encoding -// - github.com/ugorji/go/codec -// - gonum.org/v1/gonum -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname mapiternext -func mapiternext(it *hiter) { - h := it.h +// mapIterNext performs the next step of iteration. Afterwards, the next +// key/elem are in it.Key()/it.Elem(). +func mapIterNext(it *maps.Iter) { if raceenabled { - callerpc := getcallerpc() - racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapiternext)) - } - if h.flags&hashWriting != 0 { - fatal("concurrent map iteration and map write") + callerpc := sys.GetCallerPC() + racereadpc(unsafe.Pointer(it.Map()), callerpc, abi.FuncPCABIInternal(mapIterNext)) } - t := it.t - bucket := it.bucket - b := it.bptr - i := it.i - checkBucket := it.checkBucket -next: - if b == nil { - if bucket == it.startBucket && it.wrapped { - // end of iteration - it.key = nil - it.elem = nil - return - } - if h.growing() && it.B == h.B { - // Iterator was started in the middle of a grow, and the grow isn't done yet. - // If the bucket we're looking at hasn't been filled in yet (i.e. the old - // bucket hasn't been evacuated) then we need to iterate through the old - // bucket and only return the ones that will be migrated to this bucket. - oldbucket := bucket & it.h.oldbucketmask() - b = (*bmap)(add(h.oldbuckets, oldbucket*uintptr(t.BucketSize))) - if !evacuated(b) { - checkBucket = bucket - } else { - b = (*bmap)(add(it.buckets, bucket*uintptr(t.BucketSize))) - checkBucket = noCheck - } - } else { - b = (*bmap)(add(it.buckets, bucket*uintptr(t.BucketSize))) - checkBucket = noCheck - } - bucket++ - if bucket == bucketShift(it.B) { - bucket = 0 - it.wrapped = true - } - i = 0 - } - for ; i < abi.MapBucketCount; i++ { - offi := (i + it.offset) & (abi.MapBucketCount - 1) - if isEmpty(b.tophash[offi]) || b.tophash[offi] == evacuatedEmpty { - // TODO: emptyRest is hard to use here, as we start iterating - // in the middle of a bucket. It's feasible, just tricky. - continue - } - k := add(unsafe.Pointer(b), dataOffset+uintptr(offi)*uintptr(t.KeySize)) - if t.IndirectKey() { - k = *((*unsafe.Pointer)(k)) - } - e := add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+uintptr(offi)*uintptr(t.ValueSize)) - if checkBucket != noCheck && !h.sameSizeGrow() { - // Special case: iterator was started during a grow to a larger size - // and the grow is not done yet. We're working on a bucket whose - // oldbucket has not been evacuated yet. Or at least, it wasn't - // evacuated when we started the bucket. So we're iterating - // through the oldbucket, skipping any keys that will go - // to the other new bucket (each oldbucket expands to two - // buckets during a grow). - if t.ReflexiveKey() || t.Key.Equal(k, k) { - // If the item in the oldbucket is not destined for - // the current new bucket in the iteration, skip it. - hash := t.Hasher(k, uintptr(h.hash0)) - if hash&bucketMask(it.B) != checkBucket { - continue - } - } else { - // Hash isn't repeatable if k != k (NaNs). We need a - // repeatable and randomish choice of which direction - // to send NaNs during evacuation. We'll use the low - // bit of tophash to decide which way NaNs go. - // NOTE: this case is why we need two evacuate tophash - // values, evacuatedX and evacuatedY, that differ in - // their low bit. - if checkBucket>>(it.B-1) != uintptr(b.tophash[offi]&1) { - continue - } - } - } - if (b.tophash[offi] != evacuatedX && b.tophash[offi] != evacuatedY) || - !(t.ReflexiveKey() || t.Key.Equal(k, k)) { - // This is the golden data, we can return it. - // OR - // key!=key, so the entry can't be deleted or updated, so we can just return it. - // That's lucky for us because when key!=key we can't look it up successfully. - it.key = k - if t.IndirectElem() { - e = *((*unsafe.Pointer)(e)) - } - it.elem = e - } else { - // The hash table has grown since the iterator was started. - // The golden data for this key is now somewhere else. - // Check the current hash table for the data. - // This code handles the case where the key - // has been deleted, updated, or deleted and reinserted. - // NOTE: we need to regrab the key as it has potentially been - // updated to an equal() but not identical key (e.g. +0.0 vs -0.0). - rk, re := mapaccessK(t, h, k) - if rk == nil { - continue // key has been deleted - } - it.key = rk - it.elem = re - } - it.bucket = bucket - if it.bptr != b { // avoid unnecessary write barrier; see issue 14921 - it.bptr = b - } - it.i = i + 1 - it.checkBucket = checkBucket - return - } - b = b.overflow(t) - i = 0 - goto next + it.Next() } // mapclear deletes all keys from a map. -// It is called by the compiler. -// -// mapclear should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname mapclear -func mapclear(t *maptype, h *hmap) { - if raceenabled && h != nil { - callerpc := getcallerpc() +func mapclear(t *abi.MapType, m *maps.Map) { + if raceenabled && m != nil { + callerpc := sys.GetCallerPC() pc := abi.FuncPCABIInternal(mapclear) - racewritepc(unsafe.Pointer(h), callerpc, pc) - } - - if h == nil || h.count == 0 { - return - } - - if h.flags&hashWriting != 0 { - fatal("concurrent map writes") - } - - h.flags ^= hashWriting - - // Mark buckets empty, so existing iterators can be terminated, see issue #59411. - markBucketsEmpty := func(bucket unsafe.Pointer, mask uintptr) { - for i := uintptr(0); i <= mask; i++ { - b := (*bmap)(add(bucket, i*uintptr(t.BucketSize))) - for ; b != nil; b = b.overflow(t) { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - b.tophash[i] = emptyRest - } - } - } - } - markBucketsEmpty(h.buckets, bucketMask(h.B)) - if oldBuckets := h.oldbuckets; oldBuckets != nil { - markBucketsEmpty(oldBuckets, h.oldbucketmask()) - } - - h.flags &^= sameSizeGrow - h.oldbuckets = nil - h.nevacuate = 0 - h.noverflow = 0 - h.count = 0 - - // Reset the hash seed to make it more difficult for attackers to - // repeatedly trigger hash collisions. See issue 25237. - h.hash0 = uint32(rand()) - - // Keep the mapextra allocation but clear any extra information. - if h.extra != nil { - *h.extra = mapextra{} - } - - // makeBucketArray clears the memory pointed to by h.buckets - // and recovers any overflow buckets by generating them - // as if h.buckets was newly alloced. - _, nextOverflow := makeBucketArray(t, h.B, h.buckets) - if nextOverflow != nil { - // If overflow buckets are created then h.extra - // will have been allocated during initial bucket creation. - h.extra.nextOverflow = nextOverflow - } - - if h.flags&hashWriting == 0 { - fatal("concurrent map writes") - } - h.flags &^= hashWriting -} - -func hashGrow(t *maptype, h *hmap) { - // If we've hit the load factor, get bigger. - // Otherwise, there are too many overflow buckets, - // so keep the same number of buckets and "grow" laterally. - bigger := uint8(1) - if !overLoadFactor(h.count+1, h.B) { - bigger = 0 - h.flags |= sameSizeGrow - } - oldbuckets := h.buckets - newbuckets, nextOverflow := makeBucketArray(t, h.B+bigger, nil) - - flags := h.flags &^ (iterator | oldIterator) - if h.flags&iterator != 0 { - flags |= oldIterator - } - // commit the grow (atomic wrt gc) - h.B += bigger - h.flags = flags - h.oldbuckets = oldbuckets - h.buckets = newbuckets - h.nevacuate = 0 - h.noverflow = 0 - - if h.extra != nil && h.extra.overflow != nil { - // Promote current overflow buckets to the old generation. - if h.extra.oldoverflow != nil { - throw("oldoverflow is not nil") - } - h.extra.oldoverflow = h.extra.overflow - h.extra.overflow = nil - } - if nextOverflow != nil { - if h.extra == nil { - h.extra = new(mapextra) - } - h.extra.nextOverflow = nextOverflow - } - - // the actual copying of the hash table data is done incrementally - // by growWork() and evacuate(). -} - -// overLoadFactor reports whether count items placed in 1< abi.MapBucketCount && uintptr(count) > loadFactorNum*(bucketShift(B)/loadFactorDen) -} - -// tooManyOverflowBuckets reports whether noverflow buckets is too many for a map with 1< 15 { - B = 15 - } - // The compiler doesn't see here that B < 16; mask B to generate shorter shift code. - return noverflow >= uint16(1)<<(B&15) -} - -// growing reports whether h is growing. The growth may be to the same size or bigger. -func (h *hmap) growing() bool { - return h.oldbuckets != nil -} - -// sameSizeGrow reports whether the current growth is to a map of the same size. -func (h *hmap) sameSizeGrow() bool { - return h.flags&sameSizeGrow != 0 -} - -// noldbuckets calculates the number of buckets prior to the current map growth. -func (h *hmap) noldbuckets() uintptr { - oldB := h.B - if !h.sameSizeGrow() { - oldB-- - } - return bucketShift(oldB) -} - -// oldbucketmask provides a mask that can be applied to calculate n % noldbuckets(). -func (h *hmap) oldbucketmask() uintptr { - return h.noldbuckets() - 1 -} - -func growWork(t *maptype, h *hmap, bucket uintptr) { - // make sure we evacuate the oldbucket corresponding - // to the bucket we're about to use - evacuate(t, h, bucket&h.oldbucketmask()) - - // evacuate one more oldbucket to make progress on growing - if h.growing() { - evacuate(t, h, h.nevacuate) - } -} - -func bucketEvacuated(t *maptype, h *hmap, bucket uintptr) bool { - b := (*bmap)(add(h.oldbuckets, bucket*uintptr(t.BucketSize))) - return evacuated(b) -} - -// evacDst is an evacuation destination. -type evacDst struct { - b *bmap // current destination bucket - i int // key/elem index into b - k unsafe.Pointer // pointer to current key storage - e unsafe.Pointer // pointer to current elem storage -} - -func evacuate(t *maptype, h *hmap, oldbucket uintptr) { - b := (*bmap)(add(h.oldbuckets, oldbucket*uintptr(t.BucketSize))) - newbit := h.noldbuckets() - if !evacuated(b) { - // TODO: reuse overflow buckets instead of using new ones, if there - // is no iterator using the old buckets. (If !oldIterator.) - - // xy contains the x and y (low and high) evacuation destinations. - var xy [2]evacDst - x := &xy[0] - x.b = (*bmap)(add(h.buckets, oldbucket*uintptr(t.BucketSize))) - x.k = add(unsafe.Pointer(x.b), dataOffset) - x.e = add(x.k, abi.MapBucketCount*uintptr(t.KeySize)) - - if !h.sameSizeGrow() { - // Only calculate y pointers if we're growing bigger. - // Otherwise GC can see bad pointers. - y := &xy[1] - y.b = (*bmap)(add(h.buckets, (oldbucket+newbit)*uintptr(t.BucketSize))) - y.k = add(unsafe.Pointer(y.b), dataOffset) - y.e = add(y.k, abi.MapBucketCount*uintptr(t.KeySize)) - } - - for ; b != nil; b = b.overflow(t) { - k := add(unsafe.Pointer(b), dataOffset) - e := add(k, abi.MapBucketCount*uintptr(t.KeySize)) - for i := 0; i < abi.MapBucketCount; i, k, e = i+1, add(k, uintptr(t.KeySize)), add(e, uintptr(t.ValueSize)) { - top := b.tophash[i] - if isEmpty(top) { - b.tophash[i] = evacuatedEmpty - continue - } - if top < minTopHash { - throw("bad map state") - } - k2 := k - if t.IndirectKey() { - k2 = *((*unsafe.Pointer)(k2)) - } - var useY uint8 - if !h.sameSizeGrow() { - // Compute hash to make our evacuation decision (whether we need - // to send this key/elem to bucket x or bucket y). - hash := t.Hasher(k2, uintptr(h.hash0)) - if h.flags&iterator != 0 && !t.ReflexiveKey() && !t.Key.Equal(k2, k2) { - // If key != key (NaNs), then the hash could be (and probably - // will be) entirely different from the old hash. Moreover, - // it isn't reproducible. Reproducibility is required in the - // presence of iterators, as our evacuation decision must - // match whatever decision the iterator made. - // Fortunately, we have the freedom to send these keys either - // way. Also, tophash is meaningless for these kinds of keys. - // We let the low bit of tophash drive the evacuation decision. - // We recompute a new random tophash for the next level so - // these keys will get evenly distributed across all buckets - // after multiple grows. - useY = top & 1 - top = tophash(hash) - } else { - if hash&newbit != 0 { - useY = 1 - } - } - } - - if evacuatedX+1 != evacuatedY || evacuatedX^1 != evacuatedY { - throw("bad evacuatedN") - } - - b.tophash[i] = evacuatedX + useY // evacuatedX + 1 == evacuatedY - dst := &xy[useY] // evacuation destination - - if dst.i == abi.MapBucketCount { - dst.b = h.newoverflow(t, dst.b) - dst.i = 0 - dst.k = add(unsafe.Pointer(dst.b), dataOffset) - dst.e = add(dst.k, abi.MapBucketCount*uintptr(t.KeySize)) - } - dst.b.tophash[dst.i&(abi.MapBucketCount-1)] = top // mask dst.i as an optimization, to avoid a bounds check - if t.IndirectKey() { - *(*unsafe.Pointer)(dst.k) = k2 // copy pointer - } else { - typedmemmove(t.Key, dst.k, k) // copy elem - } - if t.IndirectElem() { - *(*unsafe.Pointer)(dst.e) = *(*unsafe.Pointer)(e) - } else { - typedmemmove(t.Elem, dst.e, e) - } - dst.i++ - // These updates might push these pointers past the end of the - // key or elem arrays. That's ok, as we have the overflow pointer - // at the end of the bucket to protect against pointing past the - // end of the bucket. - dst.k = add(dst.k, uintptr(t.KeySize)) - dst.e = add(dst.e, uintptr(t.ValueSize)) - } - } - // Unlink the overflow buckets & clear key/elem to help GC. - if h.flags&oldIterator == 0 && t.Bucket.Pointers() { - b := add(h.oldbuckets, oldbucket*uintptr(t.BucketSize)) - // Preserve b.tophash because the evacuation - // state is maintained there. - ptr := add(b, dataOffset) - n := uintptr(t.BucketSize) - dataOffset - memclrHasPointers(ptr, n) - } + racewritepc(unsafe.Pointer(m), callerpc, pc) } - if oldbucket == h.nevacuate { - advanceEvacuationMark(h, t, newbit) - } -} - -func advanceEvacuationMark(h *hmap, t *maptype, newbit uintptr) { - h.nevacuate++ - // Experiments suggest that 1024 is overkill by at least an order of magnitude. - // Put it in there as a safeguard anyway, to ensure O(1) behavior. - stop := h.nevacuate + 1024 - if stop > newbit { - stop = newbit - } - for h.nevacuate != stop && bucketEvacuated(t, h, h.nevacuate) { - h.nevacuate++ - } - if h.nevacuate == newbit { // newbit == # of oldbuckets - // Growing is all done. Free old main bucket array. - h.oldbuckets = nil - // Can discard old overflow buckets as well. - // If they are still referenced by an iterator, - // then the iterator holds a pointers to the slice. - if h.extra != nil { - h.extra.oldoverflow = nil - } - h.flags &^= sameSizeGrow - } + m.Clear(t) } // Reflect stubs. Called from ../reflect/asm_*.s @@ -1401,40 +201,12 @@ func advanceEvacuationMark(h *hmap, t *maptype, newbit uintptr) { // See go.dev/issue/67401. // //go:linkname reflect_makemap reflect.makemap -func reflect_makemap(t *maptype, cap int) *hmap { +func reflect_makemap(t *abi.MapType, cap int) *maps.Map { // Check invariants and reflects math. if t.Key.Equal == nil { throw("runtime.reflect_makemap: unsupported map key type") } - if t.Key.Size_ > abi.MapMaxKeyBytes && (!t.IndirectKey() || t.KeySize != uint8(goarch.PtrSize)) || - t.Key.Size_ <= abi.MapMaxKeyBytes && (t.IndirectKey() || t.KeySize != uint8(t.Key.Size_)) { - throw("key size wrong") - } - if t.Elem.Size_ > abi.MapMaxElemBytes && (!t.IndirectElem() || t.ValueSize != uint8(goarch.PtrSize)) || - t.Elem.Size_ <= abi.MapMaxElemBytes && (t.IndirectElem() || t.ValueSize != uint8(t.Elem.Size_)) { - throw("elem size wrong") - } - if t.Key.Align_ > abi.MapBucketCount { - throw("key align too big") - } - if t.Elem.Align_ > abi.MapBucketCount { - throw("elem align too big") - } - if t.Key.Size_%uintptr(t.Key.Align_) != 0 { - throw("key size not a multiple of key align") - } - if t.Elem.Size_%uintptr(t.Elem.Align_) != 0 { - throw("elem size not a multiple of elem align") - } - if abi.MapBucketCount < 8 { - throw("bucketsize too small for proper alignment") - } - if dataOffset%uintptr(t.Key.Align_) != 0 { - throw("need padding in bucket (key)") - } - if dataOffset%uintptr(t.Elem.Align_) != 0 { - throw("need padding in bucket (elem)") - } + // TODO: other checks return makemap(t, cap, nil) } @@ -1450,8 +222,8 @@ func reflect_makemap(t *maptype, cap int) *hmap { // See go.dev/issue/67401. // //go:linkname reflect_mapaccess reflect.mapaccess -func reflect_mapaccess(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer { - elem, ok := mapaccess2(t, h, key) +func reflect_mapaccess(t *abi.MapType, m *maps.Map, key unsafe.Pointer) unsafe.Pointer { + elem, ok := mapaccess2(t, m, key) if !ok { // reflect wants nil for a missing element elem = nil @@ -1460,8 +232,8 @@ func reflect_mapaccess(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer { } //go:linkname reflect_mapaccess_faststr reflect.mapaccess_faststr -func reflect_mapaccess_faststr(t *maptype, h *hmap, key string) unsafe.Pointer { - elem, ok := mapaccess2_faststr(t, h, key) +func reflect_mapaccess_faststr(t *abi.MapType, m *maps.Map, key string) unsafe.Pointer { + elem, ok := mapaccess2_faststr(t, m, key) if !ok { // reflect wants nil for a missing element elem = nil @@ -1478,86 +250,25 @@ func reflect_mapaccess_faststr(t *maptype, h *hmap, key string) unsafe.Pointer { // Do not remove or change the type signature. // //go:linkname reflect_mapassign reflect.mapassign0 -func reflect_mapassign(t *maptype, h *hmap, key unsafe.Pointer, elem unsafe.Pointer) { - p := mapassign(t, h, key) +func reflect_mapassign(t *abi.MapType, m *maps.Map, key unsafe.Pointer, elem unsafe.Pointer) { + p := mapassign(t, m, key) typedmemmove(t.Elem, p, elem) } //go:linkname reflect_mapassign_faststr reflect.mapassign_faststr0 -func reflect_mapassign_faststr(t *maptype, h *hmap, key string, elem unsafe.Pointer) { - p := mapassign_faststr(t, h, key) +func reflect_mapassign_faststr(t *abi.MapType, m *maps.Map, key string, elem unsafe.Pointer) { + p := mapassign_faststr(t, m, key) typedmemmove(t.Elem, p, elem) } //go:linkname reflect_mapdelete reflect.mapdelete -func reflect_mapdelete(t *maptype, h *hmap, key unsafe.Pointer) { - mapdelete(t, h, key) +func reflect_mapdelete(t *abi.MapType, m *maps.Map, key unsafe.Pointer) { + mapdelete(t, m, key) } //go:linkname reflect_mapdelete_faststr reflect.mapdelete_faststr -func reflect_mapdelete_faststr(t *maptype, h *hmap, key string) { - mapdelete_faststr(t, h, key) -} - -// reflect_mapiterinit is for package reflect, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/modern-go/reflect2 -// - gitee.com/quant1x/gox -// - github.com/v2pro/plz -// - github.com/wI2L/jettison -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname reflect_mapiterinit reflect.mapiterinit -func reflect_mapiterinit(t *maptype, h *hmap, it *hiter) { - mapiterinit(t, h, it) -} - -// reflect_mapiternext is for package reflect, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - gitee.com/quant1x/gox -// - github.com/modern-go/reflect2 -// - github.com/goccy/go-json -// - github.com/v2pro/plz -// - github.com/wI2L/jettison -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname reflect_mapiternext reflect.mapiternext -func reflect_mapiternext(it *hiter) { - mapiternext(it) -} - -// reflect_mapiterkey is for package reflect, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/goccy/go-json -// - gonum.org/v1/gonum -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname reflect_mapiterkey reflect.mapiterkey -func reflect_mapiterkey(it *hiter) unsafe.Pointer { - return it.key -} - -// reflect_mapiterelem is for package reflect, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/goccy/go-json -// - gonum.org/v1/gonum -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname reflect_mapiterelem reflect.mapiterelem -func reflect_mapiterelem(it *hiter) unsafe.Pointer { - return it.elem +func reflect_mapdelete_faststr(t *abi.MapType, m *maps.Map, key string) { + mapdelete_faststr(t, m, key) } // reflect_maplen is for package reflect, @@ -1570,32 +281,32 @@ func reflect_mapiterelem(it *hiter) unsafe.Pointer { // See go.dev/issue/67401. // //go:linkname reflect_maplen reflect.maplen -func reflect_maplen(h *hmap) int { - if h == nil { +func reflect_maplen(m *maps.Map) int { + if m == nil { return 0 } if raceenabled { - callerpc := getcallerpc() - racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(reflect_maplen)) + callerpc := sys.GetCallerPC() + racereadpc(unsafe.Pointer(m), callerpc, abi.FuncPCABIInternal(reflect_maplen)) } - return h.count + return int(m.Used()) } //go:linkname reflect_mapclear reflect.mapclear -func reflect_mapclear(t *maptype, h *hmap) { - mapclear(t, h) +func reflect_mapclear(t *abi.MapType, m *maps.Map) { + mapclear(t, m) } //go:linkname reflectlite_maplen internal/reflectlite.maplen -func reflectlite_maplen(h *hmap) int { - if h == nil { +func reflectlite_maplen(m *maps.Map) int { + if m == nil { return 0 } if raceenabled { - callerpc := getcallerpc() - racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(reflect_maplen)) + callerpc := sys.GetCallerPC() + racereadpc(unsafe.Pointer(m), callerpc, abi.FuncPCABIInternal(reflect_maplen)) } - return h.count + return int(m.Used()) } // mapinitnoop is a no-op function known the Go linker; if a given global @@ -1610,288 +321,9 @@ func mapinitnoop() //go:linkname mapclone maps.clone func mapclone(m any) any { e := efaceOf(&m) - e.data = unsafe.Pointer(mapclone2((*maptype)(unsafe.Pointer(e._type)), (*hmap)(e.data))) + typ := (*abi.MapType)(unsafe.Pointer(e._type)) + map_ := (*maps.Map)(e.data) + map_ = map_.Clone(typ) + e.data = (unsafe.Pointer)(map_) return m } - -// moveToBmap moves a bucket from src to dst. It returns the destination bucket or new destination bucket if it overflows -// and the pos that the next key/value will be written, if pos == bucketCnt means needs to written in overflow bucket. -func moveToBmap(t *maptype, h *hmap, dst *bmap, pos int, src *bmap) (*bmap, int) { - for i := 0; i < abi.MapBucketCount; i++ { - if isEmpty(src.tophash[i]) { - continue - } - - for ; pos < abi.MapBucketCount; pos++ { - if isEmpty(dst.tophash[pos]) { - break - } - } - - if pos == abi.MapBucketCount { - dst = h.newoverflow(t, dst) - pos = 0 - } - - srcK := add(unsafe.Pointer(src), dataOffset+uintptr(i)*uintptr(t.KeySize)) - srcEle := add(unsafe.Pointer(src), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+uintptr(i)*uintptr(t.ValueSize)) - dstK := add(unsafe.Pointer(dst), dataOffset+uintptr(pos)*uintptr(t.KeySize)) - dstEle := add(unsafe.Pointer(dst), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+uintptr(pos)*uintptr(t.ValueSize)) - - dst.tophash[pos] = src.tophash[i] - if t.IndirectKey() { - srcK = *(*unsafe.Pointer)(srcK) - if t.NeedKeyUpdate() { - kStore := newobject(t.Key) - typedmemmove(t.Key, kStore, srcK) - srcK = kStore - } - // Note: if NeedKeyUpdate is false, then the memory - // used to store the key is immutable, so we can share - // it between the original map and its clone. - *(*unsafe.Pointer)(dstK) = srcK - } else { - typedmemmove(t.Key, dstK, srcK) - } - if t.IndirectElem() { - srcEle = *(*unsafe.Pointer)(srcEle) - eStore := newobject(t.Elem) - typedmemmove(t.Elem, eStore, srcEle) - *(*unsafe.Pointer)(dstEle) = eStore - } else { - typedmemmove(t.Elem, dstEle, srcEle) - } - pos++ - h.count++ - } - return dst, pos -} - -func mapclone2(t *maptype, src *hmap) *hmap { - dst := makemap(t, src.count, nil) - dst.hash0 = src.hash0 - dst.nevacuate = 0 - // flags do not need to be copied here, just like a new map has no flags. - - if src.count == 0 { - return dst - } - - if src.flags&hashWriting != 0 { - fatal("concurrent map clone and map write") - } - - if src.B == 0 && !(t.IndirectKey() && t.NeedKeyUpdate()) && !t.IndirectElem() { - // Quick copy for small maps. - dst.buckets = newobject(t.Bucket) - dst.count = src.count - typedmemmove(t.Bucket, dst.buckets, src.buckets) - return dst - } - - if dst.B == 0 { - dst.buckets = newobject(t.Bucket) - } - dstArraySize := int(bucketShift(dst.B)) - srcArraySize := int(bucketShift(src.B)) - for i := 0; i < dstArraySize; i++ { - dstBmap := (*bmap)(add(dst.buckets, uintptr(i*int(t.BucketSize)))) - pos := 0 - for j := 0; j < srcArraySize; j += dstArraySize { - srcBmap := (*bmap)(add(src.buckets, uintptr((i+j)*int(t.BucketSize)))) - for srcBmap != nil { - dstBmap, pos = moveToBmap(t, dst, dstBmap, pos, srcBmap) - srcBmap = srcBmap.overflow(t) - } - } - } - - if src.oldbuckets == nil { - return dst - } - - oldB := src.B - srcOldbuckets := src.oldbuckets - if !src.sameSizeGrow() { - oldB-- - } - oldSrcArraySize := int(bucketShift(oldB)) - - for i := 0; i < oldSrcArraySize; i++ { - srcBmap := (*bmap)(add(srcOldbuckets, uintptr(i*int(t.BucketSize)))) - if evacuated(srcBmap) { - continue - } - - if oldB >= dst.B { // main bucket bits in dst is less than oldB bits in src - dstBmap := (*bmap)(add(dst.buckets, (uintptr(i)&bucketMask(dst.B))*uintptr(t.BucketSize))) - for dstBmap.overflow(t) != nil { - dstBmap = dstBmap.overflow(t) - } - pos := 0 - for srcBmap != nil { - dstBmap, pos = moveToBmap(t, dst, dstBmap, pos, srcBmap) - srcBmap = srcBmap.overflow(t) - } - continue - } - - // oldB < dst.B, so a single source bucket may go to multiple destination buckets. - // Process entries one at a time. - for srcBmap != nil { - // move from oldBlucket to new bucket - for i := uintptr(0); i < abi.MapBucketCount; i++ { - if isEmpty(srcBmap.tophash[i]) { - continue - } - - if src.flags&hashWriting != 0 { - fatal("concurrent map clone and map write") - } - - srcK := add(unsafe.Pointer(srcBmap), dataOffset+i*uintptr(t.KeySize)) - if t.IndirectKey() { - srcK = *((*unsafe.Pointer)(srcK)) - } - - srcEle := add(unsafe.Pointer(srcBmap), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+i*uintptr(t.ValueSize)) - if t.IndirectElem() { - srcEle = *((*unsafe.Pointer)(srcEle)) - } - dstEle := mapassign(t, dst, srcK) - typedmemmove(t.Elem, dstEle, srcEle) - } - srcBmap = srcBmap.overflow(t) - } - } - return dst -} - -// keys for implementing maps.keys -// -//go:linkname keys maps.keys -func keys(m any, p unsafe.Pointer) { - e := efaceOf(&m) - t := (*maptype)(unsafe.Pointer(e._type)) - h := (*hmap)(e.data) - - if h == nil || h.count == 0 { - return - } - s := (*slice)(p) - r := int(rand()) - offset := uint8(r >> h.B & (abi.MapBucketCount - 1)) - if h.B == 0 { - copyKeys(t, h, (*bmap)(h.buckets), s, offset) - return - } - arraySize := int(bucketShift(h.B)) - buckets := h.buckets - for i := 0; i < arraySize; i++ { - bucket := (i + r) & (arraySize - 1) - b := (*bmap)(add(buckets, uintptr(bucket)*uintptr(t.BucketSize))) - copyKeys(t, h, b, s, offset) - } - - if h.growing() { - oldArraySize := int(h.noldbuckets()) - for i := 0; i < oldArraySize; i++ { - bucket := (i + r) & (oldArraySize - 1) - b := (*bmap)(add(h.oldbuckets, uintptr(bucket)*uintptr(t.BucketSize))) - if evacuated(b) { - continue - } - copyKeys(t, h, b, s, offset) - } - } - return -} - -func copyKeys(t *maptype, h *hmap, b *bmap, s *slice, offset uint8) { - for b != nil { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - offi := (i + uintptr(offset)) & (abi.MapBucketCount - 1) - if isEmpty(b.tophash[offi]) { - continue - } - if h.flags&hashWriting != 0 { - fatal("concurrent map read and map write") - } - k := add(unsafe.Pointer(b), dataOffset+offi*uintptr(t.KeySize)) - if t.IndirectKey() { - k = *((*unsafe.Pointer)(k)) - } - if s.len >= s.cap { - fatal("concurrent map read and map write") - } - typedmemmove(t.Key, add(s.array, uintptr(s.len)*uintptr(t.Key.Size())), k) - s.len++ - } - b = b.overflow(t) - } -} - -// values for implementing maps.values -// -//go:linkname values maps.values -func values(m any, p unsafe.Pointer) { - e := efaceOf(&m) - t := (*maptype)(unsafe.Pointer(e._type)) - h := (*hmap)(e.data) - if h == nil || h.count == 0 { - return - } - s := (*slice)(p) - r := int(rand()) - offset := uint8(r >> h.B & (abi.MapBucketCount - 1)) - if h.B == 0 { - copyValues(t, h, (*bmap)(h.buckets), s, offset) - return - } - arraySize := int(bucketShift(h.B)) - buckets := h.buckets - for i := 0; i < arraySize; i++ { - bucket := (i + r) & (arraySize - 1) - b := (*bmap)(add(buckets, uintptr(bucket)*uintptr(t.BucketSize))) - copyValues(t, h, b, s, offset) - } - - if h.growing() { - oldArraySize := int(h.noldbuckets()) - for i := 0; i < oldArraySize; i++ { - bucket := (i + r) & (oldArraySize - 1) - b := (*bmap)(add(h.oldbuckets, uintptr(bucket)*uintptr(t.BucketSize))) - if evacuated(b) { - continue - } - copyValues(t, h, b, s, offset) - } - } - return -} - -func copyValues(t *maptype, h *hmap, b *bmap, s *slice, offset uint8) { - for b != nil { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - offi := (i + uintptr(offset)) & (abi.MapBucketCount - 1) - if isEmpty(b.tophash[offi]) { - continue - } - - if h.flags&hashWriting != 0 { - fatal("concurrent map read and map write") - } - - ele := add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*uintptr(t.KeySize)+offi*uintptr(t.ValueSize)) - if t.IndirectElem() { - ele = *((*unsafe.Pointer)(ele)) - } - if s.len >= s.cap { - fatal("concurrent map read and map write") - } - typedmemmove(t.Elem, add(s.array, uintptr(s.len)*uintptr(t.Elem.Size())), ele) - s.len++ - } - b = b.overflow(t) - } -} diff --git a/src/runtime/map_benchmark_test.go b/src/runtime/map_benchmark_test.go index 43d1accbb97cd6..9e93b219f17795 100644 --- a/src/runtime/map_benchmark_test.go +++ b/src/runtime/map_benchmark_test.go @@ -5,13 +5,20 @@ package runtime_test import ( + "encoding/binary" + "flag" "fmt" "math/rand" + "runtime" + "slices" "strconv" "strings" "testing" + "unsafe" ) +var mapbench = flag.Bool("mapbench", false, "enable the full set of map benchmark variants") + const size = 10 func BenchmarkHashStringSpeed(b *testing.B) { @@ -189,10 +196,12 @@ func BenchmarkSmallStrMap(b *testing.B) { } } -func BenchmarkMapStringKeysEight_16(b *testing.B) { benchmarkMapStringKeysEight(b, 16) } -func BenchmarkMapStringKeysEight_32(b *testing.B) { benchmarkMapStringKeysEight(b, 32) } -func BenchmarkMapStringKeysEight_64(b *testing.B) { benchmarkMapStringKeysEight(b, 64) } -func BenchmarkMapStringKeysEight_1M(b *testing.B) { benchmarkMapStringKeysEight(b, 1<<20) } +func BenchmarkMapStringKeysEight_16(b *testing.B) { benchmarkMapStringKeysEight(b, 16) } +func BenchmarkMapStringKeysEight_32(b *testing.B) { benchmarkMapStringKeysEight(b, 32) } +func BenchmarkMapStringKeysEight_64(b *testing.B) { benchmarkMapStringKeysEight(b, 64) } +func BenchmarkMapStringKeysEight_128(b *testing.B) { benchmarkMapStringKeysEight(b, 128) } +func BenchmarkMapStringKeysEight_256(b *testing.B) { benchmarkMapStringKeysEight(b, 256) } +func BenchmarkMapStringKeysEight_1M(b *testing.B) { benchmarkMapStringKeysEight(b, 1<<20) } func benchmarkMapStringKeysEight(b *testing.B, keySize int) { m := make(map[string]bool) @@ -206,17 +215,6 @@ func benchmarkMapStringKeysEight(b *testing.B, keySize int) { } } -func BenchmarkIntMap(b *testing.B) { - m := make(map[int]bool) - for i := 0; i < 8; i++ { - m[i] = true - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = m[7] - } -} - func BenchmarkMapFirst(b *testing.B) { for n := 1; n <= 16; n++ { b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { @@ -260,12 +258,41 @@ func BenchmarkMapLast(b *testing.B) { } } +func cyclicPermutation(n int) []int { + // From https://crypto.stackexchange.com/questions/51787/creating-single-cycle-permutations + p := rand.New(rand.NewSource(1)).Perm(n) + inc := make([]int, n) + pInv := make([]int, n) + for i := 0; i < n; i++ { + inc[i] = (i + 1) % n + pInv[p[i]] = i + } + res := make([]int, n) + for i := 0; i < n; i++ { + res[i] = pInv[inc[p[i]]] + } + + // Test result. + j := 0 + for i := 0; i < n-1; i++ { + j = res[j] + if j == 0 { + panic("got back to 0 too early") + } + } + j = res[j] + if j != 0 { + panic("didn't get back to 0") + } + return res +} + func BenchmarkMapCycle(b *testing.B) { // Arrange map entries to be a permutation, so that // we hit all entries, and one lookup is data dependent // on the previous lookup. const N = 3127 - p := rand.New(rand.NewSource(1)).Perm(N) + p := cyclicPermutation(N) m := map[int]int{} for i := 0; i < N; i++ { m[i] = p[i] @@ -333,27 +360,6 @@ func BenchmarkNewSmallMap(b *testing.B) { } } -func BenchmarkMapIter(b *testing.B) { - m := make(map[int]bool) - for i := 0; i < 8; i++ { - m[i] = true - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - for range m { - } - } -} - -func BenchmarkMapIterEmpty(b *testing.B) { - m := make(map[int]bool) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for range m { - } - } -} - func BenchmarkSameLengthMap(b *testing.B) { // long strings, same length, differ in first few // and last few bytes. @@ -368,28 +374,6 @@ func BenchmarkSameLengthMap(b *testing.B) { } } -type BigKey [3]int64 - -func BenchmarkBigKeyMap(b *testing.B) { - m := make(map[BigKey]bool) - k := BigKey{3, 4, 5} - m[k] = true - for i := 0; i < b.N; i++ { - _ = m[k] - } -} - -type BigVal [3]int64 - -func BenchmarkBigValMap(b *testing.B) { - m := make(map[BigKey]BigVal) - k := BigKey{3, 4, 5} - m[k] = BigVal{6, 7, 8} - for i := 0; i < b.N; i++ { - _ = m[k] - } -} - func BenchmarkSmallKeyMap(b *testing.B) { m := make(map[int16]bool) m[5] = true @@ -509,7 +493,6 @@ func BenchmarkMapInterfacePtr(b *testing.B) { m := map[any]bool{} for i := 0; i < 100; i++ { - i := i m[&i] = true } @@ -538,3 +521,708 @@ func BenchmarkNewEmptyMapHintGreaterThan8(b *testing.B) { _ = make(map[int]int, hintGreaterThan8) } } + +func benchSizes(f func(b *testing.B, n int)) func(*testing.B) { + var cases = []int{ + 0, + 6, + 12, + 18, + 24, + 30, + 64, + 128, + 256, + 512, + 1024, + 2048, + 4096, + 8192, + 1 << 16, + 1 << 18, + 1 << 20, + 1 << 22, + } + + // Cases enabled by default. Set -mapbench for the remainder. + // + // With the other type combinations, there are literally thousands of + // variations. It take too long to run all of these as part of + // builders. + byDefault := map[int]bool{ + 6: true, + 64: true, + 1 << 16: true, + } + + return func(b *testing.B) { + for _, n := range cases { + b.Run("len="+strconv.Itoa(n), func(b *testing.B) { + if !*mapbench && !byDefault[n] { + b.Skip("Skipped because -mapbench=false") + } + + f(b, n) + }) + } + } +} +func smallBenchSizes(f func(b *testing.B, n int)) func(*testing.B) { + return func(b *testing.B) { + for n := 1; n <= 8; n++ { + b.Run("len="+strconv.Itoa(n), func(b *testing.B) { + f(b, n) + }) + } + } +} + +// A 16 byte type. +type smallType [16]byte + +// A 512 byte type. +type mediumType [1 << 9]byte + +// A 4KiB type. +type bigType [1 << 12]byte + +type mapBenchmarkKeyType interface { + int32 | int64 | string | smallType | mediumType | bigType | *int32 +} + +type mapBenchmarkElemType interface { + mapBenchmarkKeyType | []int32 +} + +func genIntValues[T int | int32 | int64](start, end int) []T { + vals := make([]T, 0, end-start) + for i := start; i < end; i++ { + vals = append(vals, T(i)) + } + return vals +} + +func genStringValues(start, end int) []string { + vals := make([]string, 0, end-start) + for i := start; i < end; i++ { + vals = append(vals, strconv.Itoa(i)) + } + return vals +} + +func genSmallValues(start, end int) []smallType { + vals := make([]smallType, 0, end-start) + for i := start; i < end; i++ { + var v smallType + binary.NativeEndian.PutUint64(v[:], uint64(i)) + vals = append(vals, v) + } + return vals +} + +func genMediumValues(start, end int) []mediumType { + vals := make([]mediumType, 0, end-start) + for i := start; i < end; i++ { + var v mediumType + binary.NativeEndian.PutUint64(v[:], uint64(i)) + vals = append(vals, v) + } + return vals +} + +func genBigValues(start, end int) []bigType { + vals := make([]bigType, 0, end-start) + for i := start; i < end; i++ { + var v bigType + binary.NativeEndian.PutUint64(v[:], uint64(i)) + vals = append(vals, v) + } + return vals +} + +func genPtrValues[T any](start, end int) []*T { + // Start and end don't mean much. Each pointer by definition has a + // unique identity. + vals := make([]*T, 0, end-start) + for i := start; i < end; i++ { + v := new(T) + vals = append(vals, v) + } + return vals +} + +func genIntSliceValues[T int | int32 | int64](start, end int) [][]T { + vals := make([][]T, 0, end-start) + for i := start; i < end; i++ { + vals = append(vals, []T{T(i)}) + } + return vals +} + +func genValues[T mapBenchmarkElemType](start, end int) []T { + var t T + switch any(t).(type) { + case int32: + return any(genIntValues[int32](start, end)).([]T) + case int64: + return any(genIntValues[int64](start, end)).([]T) + case string: + return any(genStringValues(start, end)).([]T) + case smallType: + return any(genSmallValues(start, end)).([]T) + case mediumType: + return any(genMediumValues(start, end)).([]T) + case bigType: + return any(genBigValues(start, end)).([]T) + case *int32: + return any(genPtrValues[int32](start, end)).([]T) + case []int32: + return any(genIntSliceValues[int32](start, end)).([]T) + default: + panic("unreachable") + } +} + +// Avoid inlining to force a heap allocation. +// +//go:noinline +func newSink[T mapBenchmarkElemType]() *T { + return new(T) +} + +// Return a new maps filled with keys and elems. Both slices must be the same length. +func fillMap[K mapBenchmarkKeyType, E mapBenchmarkElemType](keys []K, elems []E) map[K]E { + m := make(map[K]E, len(keys)) + for i := range keys { + m[keys[i]] = elems[i] + } + return m +} + +func iterCount(b *testing.B, n int) int { + // Divide b.N by n so that the ns/op reports time per element, + // not time per full map iteration. This makes benchmarks of + // different map sizes more comparable. + // + // If size is zero we still need to do iterations. + if n == 0 { + return b.N + } + return b.N / n +} + +func checkAllocSize[K, E any](b *testing.B, n int) { + var k K + size := uint64(n) * uint64(unsafe.Sizeof(k)) + var e E + size += uint64(n) * uint64(unsafe.Sizeof(e)) + + if size >= 1<<30 { + b.Skipf("Total key+elem size %d exceeds 1GiB", size) + } +} + +func benchmarkMapIter[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + iterations := iterCount(b, n) + sinkK := newSink[K]() + sinkE := newSink[E]() + b.ResetTimer() + + for i := 0; i < iterations; i++ { + for k, e := range m { + *sinkK = k + *sinkE = e + } + } +} + +func BenchmarkMapIter(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapIter[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapIter[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapIter[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapIter[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapIter[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapIter[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapIter[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapIter[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapIter[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapIter[int32, *int32])) +} + +func benchmarkMapIterLowLoad[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + // Only insert one entry regardless of map size. + k := genValues[K](0, 1) + e := genValues[E](0, 1) + + m := make(map[K]E, n) + for i := range k { + m[k[i]] = e[i] + } + + iterations := iterCount(b, n) + sinkK := newSink[K]() + sinkE := newSink[E]() + b.ResetTimer() + + for i := 0; i < iterations; i++ { + for k, e := range m { + *sinkK = k + *sinkE = e + } + } +} + +func BenchmarkMapIterLowLoad(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapIterLowLoad[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapIterLowLoad[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapIterLowLoad[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapIterLowLoad[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapIterLowLoad[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapIterLowLoad[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapIterLowLoad[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapIterLowLoad[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapIterLowLoad[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapIterLowLoad[int32, *int32])) +} + +func benchmarkMapAccessHit[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't access empty map") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + sink := newSink[E]() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + *sink = m[k[i%n]] + } +} + +func BenchmarkMapAccessHit(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAccessHit[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAccessHit[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAccessHit[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAccessHit[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAccessHit[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAccessHit[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapAccessHit[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapAccessHit[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapAccessHit[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapAccessHit[int32, *int32])) +} + +var sinkOK bool + +func benchmarkMapAccessMiss[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + if n == 0 { // Create a lookup values for empty maps. + n = 1 + } + w := genValues[K](n, 2*n) + b.ResetTimer() + + var ok bool + for i := 0; i < b.N; i++ { + _, ok = m[w[i%n]] + } + + sinkOK = ok +} + +func BenchmarkMapAccessMiss(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAccessMiss[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAccessMiss[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAccessMiss[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAccessMiss[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAccessMiss[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAccessMiss[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapAccessMiss[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapAccessMiss[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapAccessMiss[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapAccessMiss[int32, *int32])) +} + +// Assign to a key that already exists. +func benchmarkMapAssignExists[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't assign to existing keys in empty map") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + m[k[i%n]] = e[i%n] + } +} + +func BenchmarkMapAssignExists(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAssignExists[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAssignExists[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAssignExists[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAssignExists[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAssignExists[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAssignExists[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapAssignExists[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapAssignExists[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapAssignExists[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapAssignExists[int32, *int32])) +} + +// Fill a map of size n with no hint. Time is per-key. A new map is created +// every n assignments. +// +// TODO(prattmic): Results don't make much sense if b.N < n. +// TODO(prattmic): Measure distribution of assign time to reveal the grow +// latency. +func benchmarkMapAssignFillNoHint[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't create empty map via assignment") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + b.ResetTimer() + + var m map[K]E + for i := 0; i < b.N; i++ { + if i%n == 0 { + m = make(map[K]E) + } + m[k[i%n]] = e[i%n] + } +} + +func BenchmarkMapAssignFillNoHint(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAssignFillNoHint[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAssignFillNoHint[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAssignFillNoHint[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAssignFillNoHint[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAssignFillNoHint[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAssignFillNoHint[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapAssignFillNoHint[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapAssignFillNoHint[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapAssignFillNoHint[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapAssignFillNoHint[int32, *int32])) +} + +// Identical to benchmarkMapAssignFillNoHint, but additionally measures the +// latency of each mapassign to report tail latency due to map grow. +func benchmarkMapAssignGrowLatency[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't create empty map via assignment") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + + // Store the run time of each mapassign. Keeping the full data rather + // than a histogram provides higher precision. b.N tends to be <10M, so + // the memory requirement isn't too bad. + sample := make([]int64, b.N) + + b.ResetTimer() + + var m map[K]E + for i := 0; i < b.N; i++ { + if i%n == 0 { + m = make(map[K]E) + } + start := runtime.Nanotime() + m[k[i%n]] = e[i%n] + end := runtime.Nanotime() + sample[i] = end - start + } + + b.StopTimer() + + slices.Sort(sample) + // TODO(prattmic): Grow is so rare that even p99.99 often doesn't + // display a grow case. Switch to a more direct measure of grow cases + // only? + b.ReportMetric(float64(sample[int(float64(len(sample))*0.5)]), "p50-ns/op") + b.ReportMetric(float64(sample[int(float64(len(sample))*0.99)]), "p99-ns/op") + b.ReportMetric(float64(sample[int(float64(len(sample))*0.999)]), "p99.9-ns/op") + b.ReportMetric(float64(sample[int(float64(len(sample))*0.9999)]), "p99.99-ns/op") + b.ReportMetric(float64(sample[len(sample)-1]), "p100-ns/op") +} + +func BenchmarkMapAssignGrowLatency(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAssignGrowLatency[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAssignGrowLatency[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAssignGrowLatency[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAssignGrowLatency[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAssignGrowLatency[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAssignGrowLatency[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapAssignGrowLatency[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapAssignGrowLatency[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapAssignGrowLatency[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapAssignGrowLatency[int32, *int32])) +} + +// Fill a map of size n with size hint. Time is per-key. A new map is created +// every n assignments. +// +// TODO(prattmic): Results don't make much sense if b.N < n. +func benchmarkMapAssignFillHint[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't create empty map via assignment") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + b.ResetTimer() + + var m map[K]E + for i := 0; i < b.N; i++ { + if i%n == 0 { + m = make(map[K]E, n) + } + m[k[i%n]] = e[i%n] + } +} + +func BenchmarkMapAssignFillHint(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAssignFillHint[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAssignFillHint[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAssignFillHint[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAssignFillHint[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAssignFillHint[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAssignFillHint[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapAssignFillHint[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapAssignFillHint[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapAssignFillHint[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapAssignFillHint[int32, *int32])) +} + +// Fill a map of size n, reusing the same map. Time is per-key. The map is +// cleared every n assignments. +// +// TODO(prattmic): Results don't make much sense if b.N < n. +func benchmarkMapAssignFillClear[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't create empty map via assignment") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if i%n == 0 { + clear(m) + } + m[k[i%n]] = e[i%n] + } +} + +func BenchmarkMapAssignFillClear(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAssignFillClear[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAssignFillClear[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAssignFillClear[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAssignFillClear[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAssignFillClear[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAssignFillClear[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapAssignFillClear[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapAssignFillClear[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapAssignFillClear[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapAssignFillClear[int32, *int32])) +} + +// Modify values using +=. +func benchmarkMapAssignAddition[K mapBenchmarkKeyType, E int32 | int64 | string](b *testing.B, n int) { + if n == 0 { + b.Skip("can't modify empty map via assignment") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + m[k[i%n]] += e[i%n] + } +} + +func BenchmarkMapAssignAddition(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapAssignAddition[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapAssignAddition[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapAssignAddition[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapAssignAddition[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapAssignAddition[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapAssignAddition[bigType, int32])) +} + +// Modify values append. +func benchmarkMapAssignAppend[K mapBenchmarkKeyType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't modify empty map via append") + } + checkAllocSize[K, []int32](b, n) + k := genValues[K](0, n) + e := genValues[[]int32](0, n) + m := fillMap(k, e) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + m[k[i%n]] = append(m[k[i%n]], e[i%n][0]) + } +} + +func BenchmarkMapAssignAppend(b *testing.B) { + b.Run("Key=int32/Elem=[]int32", benchSizes(benchmarkMapAssignAppend[int32])) + b.Run("Key=int64/Elem=[]int32", benchSizes(benchmarkMapAssignAppend[int64])) + b.Run("Key=string/Elem=[]int32", benchSizes(benchmarkMapAssignAppend[string])) +} + +func benchmarkMapDelete[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't delete from empty map") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if len(m) == 0 { + // We'd like to StopTimer while refilling the map, but + // it is way too expensive and thus makes the benchmark + // take a long time. See https://go.dev/issue/20875. + for j := range k { + m[k[j]] = e[j] + } + } + delete(m, k[i%n]) + } +} + +func BenchmarkMapDelete(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapDelete[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapDelete[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapDelete[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapDelete[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapDelete[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapDelete[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapDelete[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapDelete[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapDelete[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapDelete[int32, *int32])) +} + +// Use iterator to pop an element. We want this to be fast, see +// https://go.dev/issue/8412. +func benchmarkMapPop[K mapBenchmarkKeyType, E mapBenchmarkElemType](b *testing.B, n int) { + if n == 0 { + b.Skip("can't delete from empty map") + } + checkAllocSize[K, E](b, n) + k := genValues[K](0, n) + e := genValues[E](0, n) + m := fillMap(k, e) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if len(m) == 0 { + // We'd like to StopTimer while refilling the map, but + // it is way too expensive and thus makes the benchmark + // take a long time. See https://go.dev/issue/20875. + for j := range k { + m[k[j]] = e[j] + } + } + for key := range m { + delete(m, key) + break + } + } +} + +func BenchmarkMapPop(b *testing.B) { + b.Run("Key=int32/Elem=int32", benchSizes(benchmarkMapPop[int32, int32])) + b.Run("Key=int64/Elem=int64", benchSizes(benchmarkMapPop[int64, int64])) + b.Run("Key=string/Elem=string", benchSizes(benchmarkMapPop[string, string])) + b.Run("Key=smallType/Elem=int32", benchSizes(benchmarkMapPop[smallType, int32])) + b.Run("Key=mediumType/Elem=int32", benchSizes(benchmarkMapPop[mediumType, int32])) + b.Run("Key=bigType/Elem=int32", benchSizes(benchmarkMapPop[bigType, int32])) + b.Run("Key=bigType/Elem=bigType", benchSizes(benchmarkMapPop[bigType, bigType])) + b.Run("Key=int32/Elem=bigType", benchSizes(benchmarkMapPop[int32, bigType])) + b.Run("Key=*int32/Elem=int32", benchSizes(benchmarkMapPop[*int32, int32])) + b.Run("Key=int32/Elem=*int32", benchSizes(benchmarkMapPop[int32, *int32])) +} + +func BenchmarkMapDeleteLargeKey(b *testing.B) { + m := map[string]int{} + for i := range 9 { + m[fmt.Sprintf("%d", i)] = i + } + key := strings.Repeat("*", 10000) + for range b.N { + delete(m, key) + } +} + +func BenchmarkMapSmallAccessHit(b *testing.B) { + b.Run("Key=int32/Elem=int32", smallBenchSizes(benchmarkMapAccessHit[int32, int32])) + b.Run("Key=int64/Elem=int64", smallBenchSizes(benchmarkMapAccessHit[int64, int64])) + b.Run("Key=string/Elem=string", smallBenchSizes(benchmarkMapAccessHit[string, string])) + b.Run("Key=smallType/Elem=int32", smallBenchSizes(benchmarkMapAccessHit[smallType, int32])) +} + +func BenchmarkMapSmallAccessMiss(b *testing.B) { + b.Run("Key=int32/Elem=int32", smallBenchSizes(benchmarkMapAccessMiss[int32, int32])) + b.Run("Key=int64/Elem=int64", smallBenchSizes(benchmarkMapAccessMiss[int64, int64])) + b.Run("Key=string/Elem=string", smallBenchSizes(benchmarkMapAccessMiss[string, string])) + b.Run("Key=smallType/Elem=int32", smallBenchSizes(benchmarkMapAccessMiss[smallType, int32])) +} + +func mapAccessZeroBenchmark[K comparable](b *testing.B) { + var m map[K]uint64 + var key K + for i := 0; i < b.N; i++ { + sink = m[key] + } +} + +func BenchmarkMapAccessZero(b *testing.B) { + b.Run("Key=int64", mapAccessZeroBenchmark[int64]) + b.Run("Key=int32", mapAccessZeroBenchmark[int32]) + b.Run("Key=string", mapAccessZeroBenchmark[string]) + b.Run("Key=mediumType", mapAccessZeroBenchmark[mediumType]) + b.Run("Key=bigType", mapAccessZeroBenchmark[bigType]) +} + +func mapAccessEmptyBenchmark[K mapBenchmarkKeyType](b *testing.B) { + m := make(map[K]uint64) + for i, v := range genValues[K](0, 1000) { + m[v] = uint64(i) + } + clear(m) + var key K + for i := 0; i < b.N; i++ { + sink = m[key] + } +} + +func BenchmarkMapAccessEmpty(b *testing.B) { + b.Run("Key=int64", mapAccessEmptyBenchmark[int64]) + b.Run("Key=int32", mapAccessEmptyBenchmark[int32]) + b.Run("Key=string", mapAccessEmptyBenchmark[string]) + b.Run("Key=mediumType", mapAccessEmptyBenchmark[mediumType]) + b.Run("Key=bigType", mapAccessEmptyBenchmark[bigType]) +} diff --git a/src/runtime/map_fast32.go b/src/runtime/map_fast32.go index 0eb8562f5102dc..17b4c31d02bdd2 100644 --- a/src/runtime/map_fast32.go +++ b/src/runtime/map_fast32.go @@ -6,49 +6,14 @@ package runtime import ( "internal/abi" - "internal/goarch" + "internal/runtime/maps" "unsafe" ) -func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer { - if raceenabled && h != nil { - callerpc := getcallerpc() - racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapaccess1_fast32)) - } - if h == nil || h.count == 0 { - return unsafe.Pointer(&zeroVal[0]) - } - if h.flags&hashWriting != 0 { - fatal("concurrent map read and map write") - } - var b *bmap - if h.B == 0 { - // One-bucket table. No need to hash. - b = (*bmap)(h.buckets) - } else { - hash := t.Hasher(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) - m := bucketMask(h.B) - b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.BucketSize))) - if c := h.oldbuckets; c != nil { - if !h.sameSizeGrow() { - // There used to be half as many buckets; mask down one more power of two. - m >>= 1 - } - oldb := (*bmap)(add(c, (hash&m)*uintptr(t.BucketSize))) - if !evacuated(oldb) { - b = oldb - } - } - } - for ; b != nil; b = b.overflow(t) { - for i, k := uintptr(0), b.keys(); i < abi.MapBucketCount; i, k = i+1, add(k, 4) { - if *(*uint32)(k) == key && !isEmpty(b.tophash[i]) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*4+i*uintptr(t.ValueSize)) - } - } - } - return unsafe.Pointer(&zeroVal[0]) -} +// Functions below pushed from internal/runtime/maps. + +//go:linkname mapaccess1_fast32 +func mapaccess1_fast32(t *abi.MapType, m *maps.Map, key uint32) unsafe.Pointer // mapaccess2_fast32 should be an internal detail, // but widely used packages access it using linkname. @@ -59,146 +24,19 @@ func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer { // See go.dev/issue/67401. // //go:linkname mapaccess2_fast32 -func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) { - if raceenabled && h != nil { - callerpc := getcallerpc() - racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapaccess2_fast32)) - } - if h == nil || h.count == 0 { - return unsafe.Pointer(&zeroVal[0]), false - } - if h.flags&hashWriting != 0 { - fatal("concurrent map read and map write") - } - var b *bmap - if h.B == 0 { - // One-bucket table. No need to hash. - b = (*bmap)(h.buckets) - } else { - hash := t.Hasher(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) - m := bucketMask(h.B) - b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.BucketSize))) - if c := h.oldbuckets; c != nil { - if !h.sameSizeGrow() { - // There used to be half as many buckets; mask down one more power of two. - m >>= 1 - } - oldb := (*bmap)(add(c, (hash&m)*uintptr(t.BucketSize))) - if !evacuated(oldb) { - b = oldb - } - } - } - for ; b != nil; b = b.overflow(t) { - for i, k := uintptr(0), b.keys(); i < abi.MapBucketCount; i, k = i+1, add(k, 4) { - if *(*uint32)(k) == key && !isEmpty(b.tophash[i]) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*4+i*uintptr(t.ValueSize)), true - } - } - } - return unsafe.Pointer(&zeroVal[0]), false -} +func mapaccess2_fast32(t *abi.MapType, m *maps.Map, key uint32) (unsafe.Pointer, bool) // mapassign_fast32 should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/sonic -// - github.com/cloudwego/frugal // - github.com/ugorji/go/codec // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname mapassign_fast32 -func mapassign_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer { - if h == nil { - panic(plainError("assignment to entry in nil map")) - } - if raceenabled { - callerpc := getcallerpc() - racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapassign_fast32)) - } - if h.flags&hashWriting != 0 { - fatal("concurrent map writes") - } - hash := t.Hasher(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) - - // Set hashWriting after calling t.hasher for consistency with mapassign. - h.flags ^= hashWriting - - if h.buckets == nil { - h.buckets = newobject(t.Bucket) // newarray(t.bucket, 1) - } - -again: - bucket := hash & bucketMask(h.B) - if h.growing() { - growWork_fast32(t, h, bucket) - } - b := (*bmap)(add(h.buckets, bucket*uintptr(t.BucketSize))) - - var insertb *bmap - var inserti uintptr - var insertk unsafe.Pointer - -bucketloop: - for { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - if isEmpty(b.tophash[i]) { - if insertb == nil { - inserti = i - insertb = b - } - if b.tophash[i] == emptyRest { - break bucketloop - } - continue - } - k := *((*uint32)(add(unsafe.Pointer(b), dataOffset+i*4))) - if k != key { - continue - } - inserti = i - insertb = b - goto done - } - ovf := b.overflow(t) - if ovf == nil { - break - } - b = ovf - } - - // Did not find mapping for key. Allocate new cell & add entry. - - // If we hit the max load factor or we have too many overflow buckets, - // and we're not already in the middle of growing, start growing. - if !h.growing() && (overLoadFactor(h.count+1, h.B) || tooManyOverflowBuckets(h.noverflow, h.B)) { - hashGrow(t, h) - goto again // Growing the table invalidates everything, so try again - } - - if insertb == nil { - // The current bucket and all the overflow buckets connected to it are full, allocate a new one. - insertb = h.newoverflow(t, b) - inserti = 0 // not necessary, but avoids needlessly spilling inserti - } - insertb.tophash[inserti&(abi.MapBucketCount-1)] = tophash(hash) // mask inserti to avoid bounds checks - - insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*4) - // store new key at insert position - *(*uint32)(insertk) = key - - h.count++ - -done: - elem := add(unsafe.Pointer(insertb), dataOffset+abi.MapBucketCount*4+inserti*uintptr(t.ValueSize)) - if h.flags&hashWriting == 0 { - fatal("concurrent map writes") - } - h.flags &^= hashWriting - return elem -} +func mapassign_fast32(t *abi.MapType, m *maps.Map, key uint32) unsafe.Pointer // mapassign_fast32ptr should be an internal detail, // but widely used packages access it using linkname. @@ -209,283 +47,7 @@ done: // See go.dev/issue/67401. // //go:linkname mapassign_fast32ptr -func mapassign_fast32ptr(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer { - if h == nil { - panic(plainError("assignment to entry in nil map")) - } - if raceenabled { - callerpc := getcallerpc() - racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapassign_fast32)) - } - if h.flags&hashWriting != 0 { - fatal("concurrent map writes") - } - hash := t.Hasher(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) - - // Set hashWriting after calling t.hasher for consistency with mapassign. - h.flags ^= hashWriting - - if h.buckets == nil { - h.buckets = newobject(t.Bucket) // newarray(t.bucket, 1) - } - -again: - bucket := hash & bucketMask(h.B) - if h.growing() { - growWork_fast32(t, h, bucket) - } - b := (*bmap)(add(h.buckets, bucket*uintptr(t.BucketSize))) - - var insertb *bmap - var inserti uintptr - var insertk unsafe.Pointer - -bucketloop: - for { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - if isEmpty(b.tophash[i]) { - if insertb == nil { - inserti = i - insertb = b - } - if b.tophash[i] == emptyRest { - break bucketloop - } - continue - } - k := *((*unsafe.Pointer)(add(unsafe.Pointer(b), dataOffset+i*4))) - if k != key { - continue - } - inserti = i - insertb = b - goto done - } - ovf := b.overflow(t) - if ovf == nil { - break - } - b = ovf - } - - // Did not find mapping for key. Allocate new cell & add entry. - - // If we hit the max load factor or we have too many overflow buckets, - // and we're not already in the middle of growing, start growing. - if !h.growing() && (overLoadFactor(h.count+1, h.B) || tooManyOverflowBuckets(h.noverflow, h.B)) { - hashGrow(t, h) - goto again // Growing the table invalidates everything, so try again - } - - if insertb == nil { - // The current bucket and all the overflow buckets connected to it are full, allocate a new one. - insertb = h.newoverflow(t, b) - inserti = 0 // not necessary, but avoids needlessly spilling inserti - } - insertb.tophash[inserti&(abi.MapBucketCount-1)] = tophash(hash) // mask inserti to avoid bounds checks - - insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*4) - // store new key at insert position - *(*unsafe.Pointer)(insertk) = key - - h.count++ - -done: - elem := add(unsafe.Pointer(insertb), dataOffset+abi.MapBucketCount*4+inserti*uintptr(t.ValueSize)) - if h.flags&hashWriting == 0 { - fatal("concurrent map writes") - } - h.flags &^= hashWriting - return elem -} - -func mapdelete_fast32(t *maptype, h *hmap, key uint32) { - if raceenabled && h != nil { - callerpc := getcallerpc() - racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapdelete_fast32)) - } - if h == nil || h.count == 0 { - return - } - if h.flags&hashWriting != 0 { - fatal("concurrent map writes") - } - - hash := t.Hasher(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) - - // Set hashWriting after calling t.hasher for consistency with mapdelete - h.flags ^= hashWriting - - bucket := hash & bucketMask(h.B) - if h.growing() { - growWork_fast32(t, h, bucket) - } - b := (*bmap)(add(h.buckets, bucket*uintptr(t.BucketSize))) - bOrig := b -search: - for ; b != nil; b = b.overflow(t) { - for i, k := uintptr(0), b.keys(); i < abi.MapBucketCount; i, k = i+1, add(k, 4) { - if key != *(*uint32)(k) || isEmpty(b.tophash[i]) { - continue - } - // Only clear key if there are pointers in it. - // This can only happen if pointers are 32 bit - // wide as 64 bit pointers do not fit into a 32 bit key. - if goarch.PtrSize == 4 && t.Key.Pointers() { - // The key must be a pointer as we checked pointers are - // 32 bits wide and the key is 32 bits wide also. - *(*unsafe.Pointer)(k) = nil - } - e := add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*4+i*uintptr(t.ValueSize)) - if t.Elem.Pointers() { - memclrHasPointers(e, t.Elem.Size_) - } else { - memclrNoHeapPointers(e, t.Elem.Size_) - } - b.tophash[i] = emptyOne - // If the bucket now ends in a bunch of emptyOne states, - // change those to emptyRest states. - if i == abi.MapBucketCount-1 { - if b.overflow(t) != nil && b.overflow(t).tophash[0] != emptyRest { - goto notLast - } - } else { - if b.tophash[i+1] != emptyRest { - goto notLast - } - } - for { - b.tophash[i] = emptyRest - if i == 0 { - if b == bOrig { - break // beginning of initial bucket, we're done. - } - // Find previous bucket, continue at its last entry. - c := b - for b = bOrig; b.overflow(t) != c; b = b.overflow(t) { - } - i = abi.MapBucketCount - 1 - } else { - i-- - } - if b.tophash[i] != emptyOne { - break - } - } - notLast: - h.count-- - // Reset the hash seed to make it more difficult for attackers to - // repeatedly trigger hash collisions. See issue 25237. - if h.count == 0 { - h.hash0 = uint32(rand()) - } - break search - } - } - - if h.flags&hashWriting == 0 { - fatal("concurrent map writes") - } - h.flags &^= hashWriting -} - -func growWork_fast32(t *maptype, h *hmap, bucket uintptr) { - // make sure we evacuate the oldbucket corresponding - // to the bucket we're about to use - evacuate_fast32(t, h, bucket&h.oldbucketmask()) - - // evacuate one more oldbucket to make progress on growing - if h.growing() { - evacuate_fast32(t, h, h.nevacuate) - } -} - -func evacuate_fast32(t *maptype, h *hmap, oldbucket uintptr) { - b := (*bmap)(add(h.oldbuckets, oldbucket*uintptr(t.BucketSize))) - newbit := h.noldbuckets() - if !evacuated(b) { - // TODO: reuse overflow buckets instead of using new ones, if there - // is no iterator using the old buckets. (If !oldIterator.) - - // xy contains the x and y (low and high) evacuation destinations. - var xy [2]evacDst - x := &xy[0] - x.b = (*bmap)(add(h.buckets, oldbucket*uintptr(t.BucketSize))) - x.k = add(unsafe.Pointer(x.b), dataOffset) - x.e = add(x.k, abi.MapBucketCount*4) - - if !h.sameSizeGrow() { - // Only calculate y pointers if we're growing bigger. - // Otherwise GC can see bad pointers. - y := &xy[1] - y.b = (*bmap)(add(h.buckets, (oldbucket+newbit)*uintptr(t.BucketSize))) - y.k = add(unsafe.Pointer(y.b), dataOffset) - y.e = add(y.k, abi.MapBucketCount*4) - } - - for ; b != nil; b = b.overflow(t) { - k := add(unsafe.Pointer(b), dataOffset) - e := add(k, abi.MapBucketCount*4) - for i := 0; i < abi.MapBucketCount; i, k, e = i+1, add(k, 4), add(e, uintptr(t.ValueSize)) { - top := b.tophash[i] - if isEmpty(top) { - b.tophash[i] = evacuatedEmpty - continue - } - if top < minTopHash { - throw("bad map state") - } - var useY uint8 - if !h.sameSizeGrow() { - // Compute hash to make our evacuation decision (whether we need - // to send this key/elem to bucket x or bucket y). - hash := t.Hasher(k, uintptr(h.hash0)) - if hash&newbit != 0 { - useY = 1 - } - } - - b.tophash[i] = evacuatedX + useY // evacuatedX + 1 == evacuatedY, enforced in makemap - dst := &xy[useY] // evacuation destination - - if dst.i == abi.MapBucketCount { - dst.b = h.newoverflow(t, dst.b) - dst.i = 0 - dst.k = add(unsafe.Pointer(dst.b), dataOffset) - dst.e = add(dst.k, abi.MapBucketCount*4) - } - dst.b.tophash[dst.i&(abi.MapBucketCount-1)] = top // mask dst.i as an optimization, to avoid a bounds check - - // Copy key. - if goarch.PtrSize == 4 && t.Key.Pointers() && writeBarrier.enabled { - // Write with a write barrier. - *(*unsafe.Pointer)(dst.k) = *(*unsafe.Pointer)(k) - } else { - *(*uint32)(dst.k) = *(*uint32)(k) - } - - typedmemmove(t.Elem, dst.e, e) - dst.i++ - // These updates might push these pointers past the end of the - // key or elem arrays. That's ok, as we have the overflow pointer - // at the end of the bucket to protect against pointing past the - // end of the bucket. - dst.k = add(dst.k, 4) - dst.e = add(dst.e, uintptr(t.ValueSize)) - } - } - // Unlink the overflow buckets & clear key/elem to help GC. - if h.flags&oldIterator == 0 && t.Bucket.Pointers() { - b := add(h.oldbuckets, oldbucket*uintptr(t.BucketSize)) - // Preserve b.tophash because the evacuation - // state is maintained there. - ptr := add(b, dataOffset) - n := uintptr(t.BucketSize) - dataOffset - memclrHasPointers(ptr, n) - } - } +func mapassign_fast32ptr(t *abi.MapType, m *maps.Map, key unsafe.Pointer) unsafe.Pointer - if oldbucket == h.nevacuate { - advanceEvacuationMark(h, t, newbit) - } -} +//go:linkname mapdelete_fast32 +func mapdelete_fast32(t *abi.MapType, m *maps.Map, key uint32) diff --git a/src/runtime/map_fast64.go b/src/runtime/map_fast64.go index aca60eb2a80b02..8640acf6a6a3eb 100644 --- a/src/runtime/map_fast64.go +++ b/src/runtime/map_fast64.go @@ -6,49 +6,14 @@ package runtime import ( "internal/abi" - "internal/goarch" + "internal/runtime/maps" "unsafe" ) -func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer { - if raceenabled && h != nil { - callerpc := getcallerpc() - racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapaccess1_fast64)) - } - if h == nil || h.count == 0 { - return unsafe.Pointer(&zeroVal[0]) - } - if h.flags&hashWriting != 0 { - fatal("concurrent map read and map write") - } - var b *bmap - if h.B == 0 { - // One-bucket table. No need to hash. - b = (*bmap)(h.buckets) - } else { - hash := t.Hasher(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) - m := bucketMask(h.B) - b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.BucketSize))) - if c := h.oldbuckets; c != nil { - if !h.sameSizeGrow() { - // There used to be half as many buckets; mask down one more power of two. - m >>= 1 - } - oldb := (*bmap)(add(c, (hash&m)*uintptr(t.BucketSize))) - if !evacuated(oldb) { - b = oldb - } - } - } - for ; b != nil; b = b.overflow(t) { - for i, k := uintptr(0), b.keys(); i < abi.MapBucketCount; i, k = i+1, add(k, 8) { - if *(*uint64)(k) == key && !isEmpty(b.tophash[i]) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*8+i*uintptr(t.ValueSize)) - } - } - } - return unsafe.Pointer(&zeroVal[0]) -} +// Functions below pushed from internal/runtime/maps. + +//go:linkname mapaccess1_fast64 +func mapaccess1_fast64(t *abi.MapType, m *maps.Map, key uint64) unsafe.Pointer // mapaccess2_fast64 should be an internal detail, // but widely used packages access it using linkname. @@ -59,443 +24,31 @@ func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer { // See go.dev/issue/67401. // //go:linkname mapaccess2_fast64 -func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) { - if raceenabled && h != nil { - callerpc := getcallerpc() - racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapaccess2_fast64)) - } - if h == nil || h.count == 0 { - return unsafe.Pointer(&zeroVal[0]), false - } - if h.flags&hashWriting != 0 { - fatal("concurrent map read and map write") - } - var b *bmap - if h.B == 0 { - // One-bucket table. No need to hash. - b = (*bmap)(h.buckets) - } else { - hash := t.Hasher(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) - m := bucketMask(h.B) - b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.BucketSize))) - if c := h.oldbuckets; c != nil { - if !h.sameSizeGrow() { - // There used to be half as many buckets; mask down one more power of two. - m >>= 1 - } - oldb := (*bmap)(add(c, (hash&m)*uintptr(t.BucketSize))) - if !evacuated(oldb) { - b = oldb - } - } - } - for ; b != nil; b = b.overflow(t) { - for i, k := uintptr(0), b.keys(); i < abi.MapBucketCount; i, k = i+1, add(k, 8) { - if *(*uint64)(k) == key && !isEmpty(b.tophash[i]) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*8+i*uintptr(t.ValueSize)), true - } - } - } - return unsafe.Pointer(&zeroVal[0]), false -} +func mapaccess2_fast64(t *abi.MapType, m *maps.Map, key uint64) (unsafe.Pointer, bool) // mapassign_fast64 should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/sonic -// - github.com/cloudwego/frugal // - github.com/ugorji/go/codec // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname mapassign_fast64 -func mapassign_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer { - if h == nil { - panic(plainError("assignment to entry in nil map")) - } - if raceenabled { - callerpc := getcallerpc() - racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapassign_fast64)) - } - if h.flags&hashWriting != 0 { - fatal("concurrent map writes") - } - hash := t.Hasher(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) - - // Set hashWriting after calling t.hasher for consistency with mapassign. - h.flags ^= hashWriting - - if h.buckets == nil { - h.buckets = newobject(t.Bucket) // newarray(t.bucket, 1) - } - -again: - bucket := hash & bucketMask(h.B) - if h.growing() { - growWork_fast64(t, h, bucket) - } - b := (*bmap)(add(h.buckets, bucket*uintptr(t.BucketSize))) - - var insertb *bmap - var inserti uintptr - var insertk unsafe.Pointer - -bucketloop: - for { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - if isEmpty(b.tophash[i]) { - if insertb == nil { - insertb = b - inserti = i - } - if b.tophash[i] == emptyRest { - break bucketloop - } - continue - } - k := *((*uint64)(add(unsafe.Pointer(b), dataOffset+i*8))) - if k != key { - continue - } - insertb = b - inserti = i - goto done - } - ovf := b.overflow(t) - if ovf == nil { - break - } - b = ovf - } - - // Did not find mapping for key. Allocate new cell & add entry. - - // If we hit the max load factor or we have too many overflow buckets, - // and we're not already in the middle of growing, start growing. - if !h.growing() && (overLoadFactor(h.count+1, h.B) || tooManyOverflowBuckets(h.noverflow, h.B)) { - hashGrow(t, h) - goto again // Growing the table invalidates everything, so try again - } - - if insertb == nil { - // The current bucket and all the overflow buckets connected to it are full, allocate a new one. - insertb = h.newoverflow(t, b) - inserti = 0 // not necessary, but avoids needlessly spilling inserti - } - insertb.tophash[inserti&(abi.MapBucketCount-1)] = tophash(hash) // mask inserti to avoid bounds checks - - insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*8) - // store new key at insert position - *(*uint64)(insertk) = key - - h.count++ - -done: - elem := add(unsafe.Pointer(insertb), dataOffset+abi.MapBucketCount*8+inserti*uintptr(t.ValueSize)) - if h.flags&hashWriting == 0 { - fatal("concurrent map writes") - } - h.flags &^= hashWriting - return elem -} +func mapassign_fast64(t *abi.MapType, m *maps.Map, key uint64) unsafe.Pointer // mapassign_fast64ptr should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/sonic -// - github.com/cloudwego/frugal // - github.com/ugorji/go/codec // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname mapassign_fast64ptr -func mapassign_fast64ptr(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer { - if h == nil { - panic(plainError("assignment to entry in nil map")) - } - if raceenabled { - callerpc := getcallerpc() - racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapassign_fast64)) - } - if h.flags&hashWriting != 0 { - fatal("concurrent map writes") - } - hash := t.Hasher(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) - - // Set hashWriting after calling t.hasher for consistency with mapassign. - h.flags ^= hashWriting - - if h.buckets == nil { - h.buckets = newobject(t.Bucket) // newarray(t.bucket, 1) - } - -again: - bucket := hash & bucketMask(h.B) - if h.growing() { - growWork_fast64(t, h, bucket) - } - b := (*bmap)(add(h.buckets, bucket*uintptr(t.BucketSize))) - - var insertb *bmap - var inserti uintptr - var insertk unsafe.Pointer - -bucketloop: - for { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - if isEmpty(b.tophash[i]) { - if insertb == nil { - insertb = b - inserti = i - } - if b.tophash[i] == emptyRest { - break bucketloop - } - continue - } - k := *((*unsafe.Pointer)(add(unsafe.Pointer(b), dataOffset+i*8))) - if k != key { - continue - } - insertb = b - inserti = i - goto done - } - ovf := b.overflow(t) - if ovf == nil { - break - } - b = ovf - } - - // Did not find mapping for key. Allocate new cell & add entry. - - // If we hit the max load factor or we have too many overflow buckets, - // and we're not already in the middle of growing, start growing. - if !h.growing() && (overLoadFactor(h.count+1, h.B) || tooManyOverflowBuckets(h.noverflow, h.B)) { - hashGrow(t, h) - goto again // Growing the table invalidates everything, so try again - } - - if insertb == nil { - // The current bucket and all the overflow buckets connected to it are full, allocate a new one. - insertb = h.newoverflow(t, b) - inserti = 0 // not necessary, but avoids needlessly spilling inserti - } - insertb.tophash[inserti&(abi.MapBucketCount-1)] = tophash(hash) // mask inserti to avoid bounds checks - - insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*8) - // store new key at insert position - *(*unsafe.Pointer)(insertk) = key - - h.count++ - -done: - elem := add(unsafe.Pointer(insertb), dataOffset+abi.MapBucketCount*8+inserti*uintptr(t.ValueSize)) - if h.flags&hashWriting == 0 { - fatal("concurrent map writes") - } - h.flags &^= hashWriting - return elem -} - -func mapdelete_fast64(t *maptype, h *hmap, key uint64) { - if raceenabled && h != nil { - callerpc := getcallerpc() - racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapdelete_fast64)) - } - if h == nil || h.count == 0 { - return - } - if h.flags&hashWriting != 0 { - fatal("concurrent map writes") - } - - hash := t.Hasher(noescape(unsafe.Pointer(&key)), uintptr(h.hash0)) - - // Set hashWriting after calling t.hasher for consistency with mapdelete - h.flags ^= hashWriting - - bucket := hash & bucketMask(h.B) - if h.growing() { - growWork_fast64(t, h, bucket) - } - b := (*bmap)(add(h.buckets, bucket*uintptr(t.BucketSize))) - bOrig := b -search: - for ; b != nil; b = b.overflow(t) { - for i, k := uintptr(0), b.keys(); i < abi.MapBucketCount; i, k = i+1, add(k, 8) { - if key != *(*uint64)(k) || isEmpty(b.tophash[i]) { - continue - } - // Only clear key if there are pointers in it. - if t.Key.Pointers() { - if goarch.PtrSize == 8 { - *(*unsafe.Pointer)(k) = nil - } else { - // There are three ways to squeeze at one or more 32 bit pointers into 64 bits. - // Just call memclrHasPointers instead of trying to handle all cases here. - memclrHasPointers(k, 8) - } - } - e := add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*8+i*uintptr(t.ValueSize)) - if t.Elem.Pointers() { - memclrHasPointers(e, t.Elem.Size_) - } else { - memclrNoHeapPointers(e, t.Elem.Size_) - } - b.tophash[i] = emptyOne - // If the bucket now ends in a bunch of emptyOne states, - // change those to emptyRest states. - if i == abi.MapBucketCount-1 { - if b.overflow(t) != nil && b.overflow(t).tophash[0] != emptyRest { - goto notLast - } - } else { - if b.tophash[i+1] != emptyRest { - goto notLast - } - } - for { - b.tophash[i] = emptyRest - if i == 0 { - if b == bOrig { - break // beginning of initial bucket, we're done. - } - // Find previous bucket, continue at its last entry. - c := b - for b = bOrig; b.overflow(t) != c; b = b.overflow(t) { - } - i = abi.MapBucketCount - 1 - } else { - i-- - } - if b.tophash[i] != emptyOne { - break - } - } - notLast: - h.count-- - // Reset the hash seed to make it more difficult for attackers to - // repeatedly trigger hash collisions. See issue 25237. - if h.count == 0 { - h.hash0 = uint32(rand()) - } - break search - } - } - - if h.flags&hashWriting == 0 { - fatal("concurrent map writes") - } - h.flags &^= hashWriting -} - -func growWork_fast64(t *maptype, h *hmap, bucket uintptr) { - // make sure we evacuate the oldbucket corresponding - // to the bucket we're about to use - evacuate_fast64(t, h, bucket&h.oldbucketmask()) - - // evacuate one more oldbucket to make progress on growing - if h.growing() { - evacuate_fast64(t, h, h.nevacuate) - } -} - -func evacuate_fast64(t *maptype, h *hmap, oldbucket uintptr) { - b := (*bmap)(add(h.oldbuckets, oldbucket*uintptr(t.BucketSize))) - newbit := h.noldbuckets() - if !evacuated(b) { - // TODO: reuse overflow buckets instead of using new ones, if there - // is no iterator using the old buckets. (If !oldIterator.) - - // xy contains the x and y (low and high) evacuation destinations. - var xy [2]evacDst - x := &xy[0] - x.b = (*bmap)(add(h.buckets, oldbucket*uintptr(t.BucketSize))) - x.k = add(unsafe.Pointer(x.b), dataOffset) - x.e = add(x.k, abi.MapBucketCount*8) - - if !h.sameSizeGrow() { - // Only calculate y pointers if we're growing bigger. - // Otherwise GC can see bad pointers. - y := &xy[1] - y.b = (*bmap)(add(h.buckets, (oldbucket+newbit)*uintptr(t.BucketSize))) - y.k = add(unsafe.Pointer(y.b), dataOffset) - y.e = add(y.k, abi.MapBucketCount*8) - } - - for ; b != nil; b = b.overflow(t) { - k := add(unsafe.Pointer(b), dataOffset) - e := add(k, abi.MapBucketCount*8) - for i := 0; i < abi.MapBucketCount; i, k, e = i+1, add(k, 8), add(e, uintptr(t.ValueSize)) { - top := b.tophash[i] - if isEmpty(top) { - b.tophash[i] = evacuatedEmpty - continue - } - if top < minTopHash { - throw("bad map state") - } - var useY uint8 - if !h.sameSizeGrow() { - // Compute hash to make our evacuation decision (whether we need - // to send this key/elem to bucket x or bucket y). - hash := t.Hasher(k, uintptr(h.hash0)) - if hash&newbit != 0 { - useY = 1 - } - } - - b.tophash[i] = evacuatedX + useY // evacuatedX + 1 == evacuatedY, enforced in makemap - dst := &xy[useY] // evacuation destination - - if dst.i == abi.MapBucketCount { - dst.b = h.newoverflow(t, dst.b) - dst.i = 0 - dst.k = add(unsafe.Pointer(dst.b), dataOffset) - dst.e = add(dst.k, abi.MapBucketCount*8) - } - dst.b.tophash[dst.i&(abi.MapBucketCount-1)] = top // mask dst.i as an optimization, to avoid a bounds check - - // Copy key. - if t.Key.Pointers() && writeBarrier.enabled { - if goarch.PtrSize == 8 { - // Write with a write barrier. - *(*unsafe.Pointer)(dst.k) = *(*unsafe.Pointer)(k) - } else { - // There are three ways to squeeze at least one 32 bit pointer into 64 bits. - // Give up and call typedmemmove. - typedmemmove(t.Key, dst.k, k) - } - } else { - *(*uint64)(dst.k) = *(*uint64)(k) - } - - typedmemmove(t.Elem, dst.e, e) - dst.i++ - // These updates might push these pointers past the end of the - // key or elem arrays. That's ok, as we have the overflow pointer - // at the end of the bucket to protect against pointing past the - // end of the bucket. - dst.k = add(dst.k, 8) - dst.e = add(dst.e, uintptr(t.ValueSize)) - } - } - // Unlink the overflow buckets & clear key/elem to help GC. - if h.flags&oldIterator == 0 && t.Bucket.Pointers() { - b := add(h.oldbuckets, oldbucket*uintptr(t.BucketSize)) - // Preserve b.tophash because the evacuation - // state is maintained there. - ptr := add(b, dataOffset) - n := uintptr(t.BucketSize) - dataOffset - memclrHasPointers(ptr, n) - } - } +func mapassign_fast64ptr(t *abi.MapType, m *maps.Map, key unsafe.Pointer) unsafe.Pointer - if oldbucket == h.nevacuate { - advanceEvacuationMark(h, t, newbit) - } -} +//go:linkname mapdelete_fast64 +func mapdelete_fast64(t *abi.MapType, m *maps.Map, key uint64) diff --git a/src/runtime/map_faststr.go b/src/runtime/map_faststr.go index 5461a9f81e8143..5a7b52d0370cff 100644 --- a/src/runtime/map_faststr.go +++ b/src/runtime/map_faststr.go @@ -6,104 +6,14 @@ package runtime import ( "internal/abi" - "internal/goarch" + "internal/runtime/maps" "unsafe" ) -func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer { - if raceenabled && h != nil { - callerpc := getcallerpc() - racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapaccess1_faststr)) - } - if h == nil || h.count == 0 { - return unsafe.Pointer(&zeroVal[0]) - } - if h.flags&hashWriting != 0 { - fatal("concurrent map read and map write") - } - key := stringStructOf(&ky) - if h.B == 0 { - // One-bucket table. - b := (*bmap)(h.buckets) - if key.len < 32 { - // short key, doing lots of comparisons is ok - for i, kptr := uintptr(0), b.keys(); i < abi.MapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { - k := (*stringStruct)(kptr) - if k.len != key.len || isEmpty(b.tophash[i]) { - if b.tophash[i] == emptyRest { - break - } - continue - } - if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)) - } - } - return unsafe.Pointer(&zeroVal[0]) - } - // long key, try not to do more comparisons than necessary - keymaybe := uintptr(abi.MapBucketCount) - for i, kptr := uintptr(0), b.keys(); i < abi.MapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { - k := (*stringStruct)(kptr) - if k.len != key.len || isEmpty(b.tophash[i]) { - if b.tophash[i] == emptyRest { - break - } - continue - } - if k.str == key.str { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)) - } - // check first 4 bytes - if *((*[4]byte)(key.str)) != *((*[4]byte)(k.str)) { - continue - } - // check last 4 bytes - if *((*[4]byte)(add(key.str, uintptr(key.len)-4))) != *((*[4]byte)(add(k.str, uintptr(key.len)-4))) { - continue - } - if keymaybe != abi.MapBucketCount { - // Two keys are potential matches. Use hash to distinguish them. - goto dohash - } - keymaybe = i - } - if keymaybe != abi.MapBucketCount { - k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+keymaybe*2*goarch.PtrSize)) - if memequal(k.str, key.str, uintptr(key.len)) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+keymaybe*uintptr(t.ValueSize)) - } - } - return unsafe.Pointer(&zeroVal[0]) - } -dohash: - hash := t.Hasher(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0)) - m := bucketMask(h.B) - b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.BucketSize))) - if c := h.oldbuckets; c != nil { - if !h.sameSizeGrow() { - // There used to be half as many buckets; mask down one more power of two. - m >>= 1 - } - oldb := (*bmap)(add(c, (hash&m)*uintptr(t.BucketSize))) - if !evacuated(oldb) { - b = oldb - } - } - top := tophash(hash) - for ; b != nil; b = b.overflow(t) { - for i, kptr := uintptr(0), b.keys(); i < abi.MapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { - k := (*stringStruct)(kptr) - if k.len != key.len || b.tophash[i] != top { - continue - } - if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)) - } - } - } - return unsafe.Pointer(&zeroVal[0]) -} +// Functions below pushed from internal/runtime/maps. + +//go:linkname mapaccess1_faststr +func mapaccess1_faststr(t *abi.MapType, m *maps.Map, ky string) unsafe.Pointer // mapaccess2_faststr should be an internal detail, // but widely used packages access it using linkname. @@ -114,392 +24,19 @@ dohash: // See go.dev/issue/67401. // //go:linkname mapaccess2_faststr -func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) { - if raceenabled && h != nil { - callerpc := getcallerpc() - racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapaccess2_faststr)) - } - if h == nil || h.count == 0 { - return unsafe.Pointer(&zeroVal[0]), false - } - if h.flags&hashWriting != 0 { - fatal("concurrent map read and map write") - } - key := stringStructOf(&ky) - if h.B == 0 { - // One-bucket table. - b := (*bmap)(h.buckets) - if key.len < 32 { - // short key, doing lots of comparisons is ok - for i, kptr := uintptr(0), b.keys(); i < abi.MapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { - k := (*stringStruct)(kptr) - if k.len != key.len || isEmpty(b.tophash[i]) { - if b.tophash[i] == emptyRest { - break - } - continue - } - if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)), true - } - } - return unsafe.Pointer(&zeroVal[0]), false - } - // long key, try not to do more comparisons than necessary - keymaybe := uintptr(abi.MapBucketCount) - for i, kptr := uintptr(0), b.keys(); i < abi.MapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { - k := (*stringStruct)(kptr) - if k.len != key.len || isEmpty(b.tophash[i]) { - if b.tophash[i] == emptyRest { - break - } - continue - } - if k.str == key.str { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)), true - } - // check first 4 bytes - if *((*[4]byte)(key.str)) != *((*[4]byte)(k.str)) { - continue - } - // check last 4 bytes - if *((*[4]byte)(add(key.str, uintptr(key.len)-4))) != *((*[4]byte)(add(k.str, uintptr(key.len)-4))) { - continue - } - if keymaybe != abi.MapBucketCount { - // Two keys are potential matches. Use hash to distinguish them. - goto dohash - } - keymaybe = i - } - if keymaybe != abi.MapBucketCount { - k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+keymaybe*2*goarch.PtrSize)) - if memequal(k.str, key.str, uintptr(key.len)) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+keymaybe*uintptr(t.ValueSize)), true - } - } - return unsafe.Pointer(&zeroVal[0]), false - } -dohash: - hash := t.Hasher(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0)) - m := bucketMask(h.B) - b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.BucketSize))) - if c := h.oldbuckets; c != nil { - if !h.sameSizeGrow() { - // There used to be half as many buckets; mask down one more power of two. - m >>= 1 - } - oldb := (*bmap)(add(c, (hash&m)*uintptr(t.BucketSize))) - if !evacuated(oldb) { - b = oldb - } - } - top := tophash(hash) - for ; b != nil; b = b.overflow(t) { - for i, kptr := uintptr(0), b.keys(); i < abi.MapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { - k := (*stringStruct)(kptr) - if k.len != key.len || b.tophash[i] != top { - continue - } - if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) { - return add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)), true - } - } - } - return unsafe.Pointer(&zeroVal[0]), false -} +func mapaccess2_faststr(t *abi.MapType, m *maps.Map, ky string) (unsafe.Pointer, bool) // mapassign_faststr should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/sonic -// - github.com/cloudwego/frugal // - github.com/ugorji/go/codec // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname mapassign_faststr -func mapassign_faststr(t *maptype, h *hmap, s string) unsafe.Pointer { - if h == nil { - panic(plainError("assignment to entry in nil map")) - } - if raceenabled { - callerpc := getcallerpc() - racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapassign_faststr)) - } - if h.flags&hashWriting != 0 { - fatal("concurrent map writes") - } - key := stringStructOf(&s) - hash := t.Hasher(noescape(unsafe.Pointer(&s)), uintptr(h.hash0)) - - // Set hashWriting after calling t.hasher for consistency with mapassign. - h.flags ^= hashWriting - - if h.buckets == nil { - h.buckets = newobject(t.Bucket) // newarray(t.bucket, 1) - } - -again: - bucket := hash & bucketMask(h.B) - if h.growing() { - growWork_faststr(t, h, bucket) - } - b := (*bmap)(add(h.buckets, bucket*uintptr(t.BucketSize))) - top := tophash(hash) - - var insertb *bmap - var inserti uintptr - var insertk unsafe.Pointer - -bucketloop: - for { - for i := uintptr(0); i < abi.MapBucketCount; i++ { - if b.tophash[i] != top { - if isEmpty(b.tophash[i]) && insertb == nil { - insertb = b - inserti = i - } - if b.tophash[i] == emptyRest { - break bucketloop - } - continue - } - k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*goarch.PtrSize)) - if k.len != key.len { - continue - } - if k.str != key.str && !memequal(k.str, key.str, uintptr(key.len)) { - continue - } - // already have a mapping for key. Update it. - inserti = i - insertb = b - // Overwrite existing key, so it can be garbage collected. - // The size is already guaranteed to be set correctly. - k.str = key.str - goto done - } - ovf := b.overflow(t) - if ovf == nil { - break - } - b = ovf - } - - // Did not find mapping for key. Allocate new cell & add entry. - - // If we hit the max load factor or we have too many overflow buckets, - // and we're not already in the middle of growing, start growing. - if !h.growing() && (overLoadFactor(h.count+1, h.B) || tooManyOverflowBuckets(h.noverflow, h.B)) { - hashGrow(t, h) - goto again // Growing the table invalidates everything, so try again - } - - if insertb == nil { - // The current bucket and all the overflow buckets connected to it are full, allocate a new one. - insertb = h.newoverflow(t, b) - inserti = 0 // not necessary, but avoids needlessly spilling inserti - } - insertb.tophash[inserti&(abi.MapBucketCount-1)] = top // mask inserti to avoid bounds checks - - insertk = add(unsafe.Pointer(insertb), dataOffset+inserti*2*goarch.PtrSize) - // store new key at insert position - *((*stringStruct)(insertk)) = *key - h.count++ - -done: - elem := add(unsafe.Pointer(insertb), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+inserti*uintptr(t.ValueSize)) - if h.flags&hashWriting == 0 { - fatal("concurrent map writes") - } - h.flags &^= hashWriting - return elem -} - -func mapdelete_faststr(t *maptype, h *hmap, ky string) { - if raceenabled && h != nil { - callerpc := getcallerpc() - racewritepc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapdelete_faststr)) - } - if h == nil || h.count == 0 { - return - } - if h.flags&hashWriting != 0 { - fatal("concurrent map writes") - } - - key := stringStructOf(&ky) - hash := t.Hasher(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0)) - - // Set hashWriting after calling t.hasher for consistency with mapdelete - h.flags ^= hashWriting - - bucket := hash & bucketMask(h.B) - if h.growing() { - growWork_faststr(t, h, bucket) - } - b := (*bmap)(add(h.buckets, bucket*uintptr(t.BucketSize))) - bOrig := b - top := tophash(hash) -search: - for ; b != nil; b = b.overflow(t) { - for i, kptr := uintptr(0), b.keys(); i < abi.MapBucketCount; i, kptr = i+1, add(kptr, 2*goarch.PtrSize) { - k := (*stringStruct)(kptr) - if k.len != key.len || b.tophash[i] != top { - continue - } - if k.str != key.str && !memequal(k.str, key.str, uintptr(key.len)) { - continue - } - // Clear key's pointer. - k.str = nil - e := add(unsafe.Pointer(b), dataOffset+abi.MapBucketCount*2*goarch.PtrSize+i*uintptr(t.ValueSize)) - if t.Elem.Pointers() { - memclrHasPointers(e, t.Elem.Size_) - } else { - memclrNoHeapPointers(e, t.Elem.Size_) - } - b.tophash[i] = emptyOne - // If the bucket now ends in a bunch of emptyOne states, - // change those to emptyRest states. - if i == abi.MapBucketCount-1 { - if b.overflow(t) != nil && b.overflow(t).tophash[0] != emptyRest { - goto notLast - } - } else { - if b.tophash[i+1] != emptyRest { - goto notLast - } - } - for { - b.tophash[i] = emptyRest - if i == 0 { - if b == bOrig { - break // beginning of initial bucket, we're done. - } - // Find previous bucket, continue at its last entry. - c := b - for b = bOrig; b.overflow(t) != c; b = b.overflow(t) { - } - i = abi.MapBucketCount - 1 - } else { - i-- - } - if b.tophash[i] != emptyOne { - break - } - } - notLast: - h.count-- - // Reset the hash seed to make it more difficult for attackers to - // repeatedly trigger hash collisions. See issue 25237. - if h.count == 0 { - h.hash0 = uint32(rand()) - } - break search - } - } - - if h.flags&hashWriting == 0 { - fatal("concurrent map writes") - } - h.flags &^= hashWriting -} - -func growWork_faststr(t *maptype, h *hmap, bucket uintptr) { - // make sure we evacuate the oldbucket corresponding - // to the bucket we're about to use - evacuate_faststr(t, h, bucket&h.oldbucketmask()) - - // evacuate one more oldbucket to make progress on growing - if h.growing() { - evacuate_faststr(t, h, h.nevacuate) - } -} - -func evacuate_faststr(t *maptype, h *hmap, oldbucket uintptr) { - b := (*bmap)(add(h.oldbuckets, oldbucket*uintptr(t.BucketSize))) - newbit := h.noldbuckets() - if !evacuated(b) { - // TODO: reuse overflow buckets instead of using new ones, if there - // is no iterator using the old buckets. (If !oldIterator.) - - // xy contains the x and y (low and high) evacuation destinations. - var xy [2]evacDst - x := &xy[0] - x.b = (*bmap)(add(h.buckets, oldbucket*uintptr(t.BucketSize))) - x.k = add(unsafe.Pointer(x.b), dataOffset) - x.e = add(x.k, abi.MapBucketCount*2*goarch.PtrSize) - - if !h.sameSizeGrow() { - // Only calculate y pointers if we're growing bigger. - // Otherwise GC can see bad pointers. - y := &xy[1] - y.b = (*bmap)(add(h.buckets, (oldbucket+newbit)*uintptr(t.BucketSize))) - y.k = add(unsafe.Pointer(y.b), dataOffset) - y.e = add(y.k, abi.MapBucketCount*2*goarch.PtrSize) - } - - for ; b != nil; b = b.overflow(t) { - k := add(unsafe.Pointer(b), dataOffset) - e := add(k, abi.MapBucketCount*2*goarch.PtrSize) - for i := 0; i < abi.MapBucketCount; i, k, e = i+1, add(k, 2*goarch.PtrSize), add(e, uintptr(t.ValueSize)) { - top := b.tophash[i] - if isEmpty(top) { - b.tophash[i] = evacuatedEmpty - continue - } - if top < minTopHash { - throw("bad map state") - } - var useY uint8 - if !h.sameSizeGrow() { - // Compute hash to make our evacuation decision (whether we need - // to send this key/elem to bucket x or bucket y). - hash := t.Hasher(k, uintptr(h.hash0)) - if hash&newbit != 0 { - useY = 1 - } - } - - b.tophash[i] = evacuatedX + useY // evacuatedX + 1 == evacuatedY, enforced in makemap - dst := &xy[useY] // evacuation destination - - if dst.i == abi.MapBucketCount { - dst.b = h.newoverflow(t, dst.b) - dst.i = 0 - dst.k = add(unsafe.Pointer(dst.b), dataOffset) - dst.e = add(dst.k, abi.MapBucketCount*2*goarch.PtrSize) - } - dst.b.tophash[dst.i&(abi.MapBucketCount-1)] = top // mask dst.i as an optimization, to avoid a bounds check - - // Copy key. - *(*string)(dst.k) = *(*string)(k) - - typedmemmove(t.Elem, dst.e, e) - dst.i++ - // These updates might push these pointers past the end of the - // key or elem arrays. That's ok, as we have the overflow pointer - // at the end of the bucket to protect against pointing past the - // end of the bucket. - dst.k = add(dst.k, 2*goarch.PtrSize) - dst.e = add(dst.e, uintptr(t.ValueSize)) - } - } - // Unlink the overflow buckets & clear key/elem to help GC. - if h.flags&oldIterator == 0 && t.Bucket.Pointers() { - b := add(h.oldbuckets, oldbucket*uintptr(t.BucketSize)) - // Preserve b.tophash because the evacuation - // state is maintained there. - ptr := add(b, dataOffset) - n := uintptr(t.BucketSize) - dataOffset - memclrHasPointers(ptr, n) - } - } +func mapassign_faststr(t *abi.MapType, m *maps.Map, s string) unsafe.Pointer - if oldbucket == h.nevacuate { - advanceEvacuationMark(h, t, newbit) - } -} +//go:linkname mapdelete_faststr +func mapdelete_faststr(t *abi.MapType, m *maps.Map, ky string) diff --git a/src/runtime/map_test.go b/src/runtime/map_test.go index ba2ea746490994..7fe8399130aac3 100644 --- a/src/runtime/map_test.go +++ b/src/runtime/map_test.go @@ -8,6 +8,7 @@ import ( "fmt" "internal/abi" "internal/goarch" + "internal/runtime/maps" "internal/testenv" "math" "os" @@ -21,17 +22,6 @@ import ( "unsafe" ) -func TestHmapSize(t *testing.T) { - // The structure of hmap is defined in runtime/map.go - // and in cmd/compile/internal/gc/reflect.go and must be in sync. - // The size of hmap should be 48 bytes on 64 bit and 28 bytes on 32 bit platforms. - var hmapSize = uintptr(8 + 5*goarch.PtrSize) - if runtime.RuntimeHmapSize != hmapSize { - t.Errorf("sizeof(runtime.hmap{})==%d, want %d", runtime.RuntimeHmapSize, hmapSize) - } - -} - // negative zero is a good test because: // 1. 0 and -0 are equal, yet have distinct representations. // 2. 0 is represented as all zeros, -0 isn't. @@ -431,6 +421,12 @@ func TestEmptyKeyAndValue(t *testing.T) { if len(a) != 1 { t.Errorf("empty value insert problem") } + if len(b) != 1 { + t.Errorf("empty key insert problem") + } + if len(c) != 1 { + t.Errorf("empty key+value insert problem") + } if b[empty{}] != 1 { t.Errorf("empty key returned wrong value") } @@ -509,43 +505,6 @@ func TestMapNanGrowIterator(t *testing.T) { } } -func TestMapIterOrder(t *testing.T) { - sizes := []int{3, 7, 9, 15} - if abi.MapBucketCountBits >= 5 { - // it gets flaky (often only one iteration order) at size 3 when abi.MapBucketCountBits >=5. - t.Fatalf("This test becomes flaky if abi.MapBucketCountBits(=%d) is 5 or larger", abi.MapBucketCountBits) - } - for _, n := range sizes { - for i := 0; i < 1000; i++ { - // Make m be {0: true, 1: true, ..., n-1: true}. - m := make(map[int]bool) - for i := 0; i < n; i++ { - m[i] = true - } - // Check that iterating over the map produces at least two different orderings. - ord := func() []int { - var s []int - for key := range m { - s = append(s, key) - } - return s - } - first := ord() - ok := false - for try := 0; try < 100; try++ { - if !slices.Equal(first, ord()) { - ok = true - break - } - } - if !ok { - t.Errorf("Map with n=%d elements had consistent iteration order: %v", n, first) - break - } - } - } -} - // Issue 8410 func TestMapSparseIterOrder(t *testing.T) { // Run several rounds to increase the probability @@ -582,6 +541,38 @@ NextRound: } } +// Map iteration must not return duplicate entries. +func TestMapIterDuplicate(t *testing.T) { + // Run several rounds to increase the probability + // of failure. One is not enough. + for range 1000 { + m := make(map[int]bool) + // Add 1000 items, remove 980. + for i := 0; i < 1000; i++ { + m[i] = true + } + for i := 20; i < 1000; i++ { + delete(m, i) + } + + var want []int + for i := 0; i < 20; i++ { + want = append(want, i) + } + + var got []int + for i := range m { + got = append(got, i) + } + + slices.Sort(got) + + if !reflect.DeepEqual(got, want) { + t.Errorf("iteration got %v want %v\n", got, want) + } + } +} + func TestMapStringBytesLookup(t *testing.T) { // Use large string keys to avoid small-allocation coalescing, // which can cause AllocsPerRun to report lower counts than it should. @@ -682,165 +673,6 @@ func TestIgnoreBogusMapHint(t *testing.T) { } } -const bs = abi.MapBucketCount - -// belowOverflow should be a pretty-full pair of buckets; -// atOverflow is 1/8 bs larger = 13/8 buckets or two buckets -// that are 13/16 full each, which is the overflow boundary. -// Adding one to that should ensure overflow to the next higher size. -const ( - belowOverflow = bs * 3 / 2 // 1.5 bs = 2 buckets @ 75% - atOverflow = belowOverflow + bs/8 // 2 buckets at 13/16 fill. -) - -var mapBucketTests = [...]struct { - n int // n is the number of map elements - noescape int // number of expected buckets for non-escaping map - escape int // number of expected buckets for escaping map -}{ - {-(1 << 30), 1, 1}, - {-1, 1, 1}, - {0, 1, 1}, - {1, 1, 1}, - {bs, 1, 1}, - {bs + 1, 2, 2}, - {belowOverflow, 2, 2}, // 1.5 bs = 2 buckets @ 75% - {atOverflow + 1, 4, 4}, // 13/8 bs + 1 == overflow to 4 - - {2 * belowOverflow, 4, 4}, // 3 bs = 4 buckets @75% - {2*atOverflow + 1, 8, 8}, // 13/4 bs + 1 = overflow to 8 - - {4 * belowOverflow, 8, 8}, // 6 bs = 8 buckets @ 75% - {4*atOverflow + 1, 16, 16}, // 13/2 bs + 1 = overflow to 16 -} - -func TestMapBuckets(t *testing.T) { - // Test that maps of different sizes have the right number of buckets. - // Non-escaping maps with small buckets (like map[int]int) never - // have a nil bucket pointer due to starting with preallocated buckets - // on the stack. Escaping maps start with a non-nil bucket pointer if - // hint size is above bucketCnt and thereby have more than one bucket. - // These tests depend on bucketCnt and loadFactor* in map.go. - t.Run("mapliteral", func(t *testing.T) { - for _, tt := range mapBucketTests { - localMap := map[int]int{} - if runtime.MapBucketsPointerIsNil(localMap) { - t.Errorf("no escape: buckets pointer is nil for non-escaping map") - } - for i := 0; i < tt.n; i++ { - localMap[i] = i - } - if got := runtime.MapBucketsCount(localMap); got != tt.noescape { - t.Errorf("no escape: n=%d want %d buckets, got %d", tt.n, tt.noescape, got) - } - escapingMap := runtime.Escape(map[int]int{}) - if count := runtime.MapBucketsCount(escapingMap); count > 1 && runtime.MapBucketsPointerIsNil(escapingMap) { - t.Errorf("escape: buckets pointer is nil for n=%d buckets", count) - } - for i := 0; i < tt.n; i++ { - escapingMap[i] = i - } - if got := runtime.MapBucketsCount(escapingMap); got != tt.escape { - t.Errorf("escape n=%d want %d buckets, got %d", tt.n, tt.escape, got) - } - } - }) - t.Run("nohint", func(t *testing.T) { - for _, tt := range mapBucketTests { - localMap := make(map[int]int) - if runtime.MapBucketsPointerIsNil(localMap) { - t.Errorf("no escape: buckets pointer is nil for non-escaping map") - } - for i := 0; i < tt.n; i++ { - localMap[i] = i - } - if got := runtime.MapBucketsCount(localMap); got != tt.noescape { - t.Errorf("no escape: n=%d want %d buckets, got %d", tt.n, tt.noescape, got) - } - escapingMap := runtime.Escape(make(map[int]int)) - if count := runtime.MapBucketsCount(escapingMap); count > 1 && runtime.MapBucketsPointerIsNil(escapingMap) { - t.Errorf("escape: buckets pointer is nil for n=%d buckets", count) - } - for i := 0; i < tt.n; i++ { - escapingMap[i] = i - } - if got := runtime.MapBucketsCount(escapingMap); got != tt.escape { - t.Errorf("escape: n=%d want %d buckets, got %d", tt.n, tt.escape, got) - } - } - }) - t.Run("makemap", func(t *testing.T) { - for _, tt := range mapBucketTests { - localMap := make(map[int]int, tt.n) - if runtime.MapBucketsPointerIsNil(localMap) { - t.Errorf("no escape: buckets pointer is nil for non-escaping map") - } - for i := 0; i < tt.n; i++ { - localMap[i] = i - } - if got := runtime.MapBucketsCount(localMap); got != tt.noescape { - t.Errorf("no escape: n=%d want %d buckets, got %d", tt.n, tt.noescape, got) - } - escapingMap := runtime.Escape(make(map[int]int, tt.n)) - if count := runtime.MapBucketsCount(escapingMap); count > 1 && runtime.MapBucketsPointerIsNil(escapingMap) { - t.Errorf("escape: buckets pointer is nil for n=%d buckets", count) - } - for i := 0; i < tt.n; i++ { - escapingMap[i] = i - } - if got := runtime.MapBucketsCount(escapingMap); got != tt.escape { - t.Errorf("escape: n=%d want %d buckets, got %d", tt.n, tt.escape, got) - } - } - }) - t.Run("makemap64", func(t *testing.T) { - for _, tt := range mapBucketTests { - localMap := make(map[int]int, int64(tt.n)) - if runtime.MapBucketsPointerIsNil(localMap) { - t.Errorf("no escape: buckets pointer is nil for non-escaping map") - } - for i := 0; i < tt.n; i++ { - localMap[i] = i - } - if got := runtime.MapBucketsCount(localMap); got != tt.noescape { - t.Errorf("no escape: n=%d want %d buckets, got %d", tt.n, tt.noescape, got) - } - escapingMap := runtime.Escape(make(map[int]int, tt.n)) - if count := runtime.MapBucketsCount(escapingMap); count > 1 && runtime.MapBucketsPointerIsNil(escapingMap) { - t.Errorf("escape: buckets pointer is nil for n=%d buckets", count) - } - for i := 0; i < tt.n; i++ { - escapingMap[i] = i - } - if got := runtime.MapBucketsCount(escapingMap); got != tt.escape { - t.Errorf("escape: n=%d want %d buckets, got %d", tt.n, tt.escape, got) - } - } - }) - -} - -func benchmarkMapPop(b *testing.B, n int) { - m := map[int]int{} - for i := 0; i < b.N; i++ { - for j := 0; j < n; j++ { - m[j] = j - } - for j := 0; j < n; j++ { - // Use iterator to pop an element. - // We want this to be fast, see issue 8412. - for k := range m { - delete(m, k) - break - } - } - } -} - -func BenchmarkMapPop100(b *testing.B) { benchmarkMapPop(b, 100) } -func BenchmarkMapPop1000(b *testing.B) { benchmarkMapPop(b, 1000) } -func BenchmarkMapPop10000(b *testing.B) { benchmarkMapPop(b, 10000) } - var testNonEscapingMapVariable int = 8 func TestNonEscapingMap(t *testing.T) { @@ -849,222 +681,30 @@ func TestNonEscapingMap(t *testing.T) { m[0] = 0 }) if n != 0 { - t.Fatalf("mapliteral: want 0 allocs, got %v", n) + t.Errorf("mapliteral: want 0 allocs, got %v", n) } n = testing.AllocsPerRun(1000, func() { m := make(map[int]int) m[0] = 0 }) if n != 0 { - t.Fatalf("no hint: want 0 allocs, got %v", n) + t.Errorf("no hint: want 0 allocs, got %v", n) } n = testing.AllocsPerRun(1000, func() { m := make(map[int]int, 8) m[0] = 0 }) if n != 0 { - t.Fatalf("with small hint: want 0 allocs, got %v", n) + t.Errorf("with small hint: want 0 allocs, got %v", n) } n = testing.AllocsPerRun(1000, func() { m := make(map[int]int, testNonEscapingMapVariable) m[0] = 0 }) if n != 0 { - t.Fatalf("with variable hint: want 0 allocs, got %v", n) - } - -} - -func benchmarkMapAssignInt32(b *testing.B, n int) { - a := make(map[int32]int) - for i := 0; i < b.N; i++ { - a[int32(i&(n-1))] = i + t.Errorf("with variable hint: want 0 allocs, got %v", n) } -} -func benchmarkMapOperatorAssignInt32(b *testing.B, n int) { - a := make(map[int32]int) - for i := 0; i < b.N; i++ { - a[int32(i&(n-1))] += i - } -} - -func benchmarkMapAppendAssignInt32(b *testing.B, n int) { - a := make(map[int32][]int) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - key := int32(i & (n - 1)) - a[key] = append(a[key], i) - } -} - -func benchmarkMapDeleteInt32(b *testing.B, n int) { - a := make(map[int32]int, n) - b.ResetTimer() - for i := 0; i < b.N; i++ { - if len(a) == 0 { - b.StopTimer() - for j := i; j < i+n; j++ { - a[int32(j)] = j - } - b.StartTimer() - } - delete(a, int32(i)) - } -} - -func benchmarkMapAssignInt64(b *testing.B, n int) { - a := make(map[int64]int) - for i := 0; i < b.N; i++ { - a[int64(i&(n-1))] = i - } -} - -func benchmarkMapOperatorAssignInt64(b *testing.B, n int) { - a := make(map[int64]int) - for i := 0; i < b.N; i++ { - a[int64(i&(n-1))] += i - } -} - -func benchmarkMapAppendAssignInt64(b *testing.B, n int) { - a := make(map[int64][]int) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - key := int64(i & (n - 1)) - a[key] = append(a[key], i) - } -} - -func benchmarkMapDeleteInt64(b *testing.B, n int) { - a := make(map[int64]int, n) - b.ResetTimer() - for i := 0; i < b.N; i++ { - if len(a) == 0 { - b.StopTimer() - for j := i; j < i+n; j++ { - a[int64(j)] = j - } - b.StartTimer() - } - delete(a, int64(i)) - } -} - -func benchmarkMapAssignStr(b *testing.B, n int) { - k := make([]string, n) - for i := 0; i < len(k); i++ { - k[i] = strconv.Itoa(i) - } - b.ResetTimer() - a := make(map[string]int) - for i := 0; i < b.N; i++ { - a[k[i&(n-1)]] = i - } -} - -func benchmarkMapOperatorAssignStr(b *testing.B, n int) { - k := make([]string, n) - for i := 0; i < len(k); i++ { - k[i] = strconv.Itoa(i) - } - b.ResetTimer() - a := make(map[string]string) - for i := 0; i < b.N; i++ { - key := k[i&(n-1)] - a[key] += key - } -} - -func benchmarkMapAppendAssignStr(b *testing.B, n int) { - k := make([]string, n) - for i := 0; i < len(k); i++ { - k[i] = strconv.Itoa(i) - } - a := make(map[string][]string) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - key := k[i&(n-1)] - a[key] = append(a[key], key) - } -} - -func benchmarkMapDeleteStr(b *testing.B, n int) { - i2s := make([]string, n) - for i := 0; i < n; i++ { - i2s[i] = strconv.Itoa(i) - } - a := make(map[string]int, n) - b.ResetTimer() - k := 0 - for i := 0; i < b.N; i++ { - if len(a) == 0 { - b.StopTimer() - for j := 0; j < n; j++ { - a[i2s[j]] = j - } - k = i - b.StartTimer() - } - delete(a, i2s[i-k]) - } -} - -func benchmarkMapDeletePointer(b *testing.B, n int) { - i2p := make([]*int, n) - for i := 0; i < n; i++ { - i2p[i] = new(int) - } - a := make(map[*int]int, n) - b.ResetTimer() - k := 0 - for i := 0; i < b.N; i++ { - if len(a) == 0 { - b.StopTimer() - for j := 0; j < n; j++ { - a[i2p[j]] = j - } - k = i - b.StartTimer() - } - delete(a, i2p[i-k]) - } -} - -func runWith(f func(*testing.B, int), v ...int) func(*testing.B) { - return func(b *testing.B) { - for _, n := range v { - b.Run(strconv.Itoa(n), func(b *testing.B) { f(b, n) }) - } - } -} - -func BenchmarkMapAssign(b *testing.B) { - b.Run("Int32", runWith(benchmarkMapAssignInt32, 1<<8, 1<<16)) - b.Run("Int64", runWith(benchmarkMapAssignInt64, 1<<8, 1<<16)) - b.Run("Str", runWith(benchmarkMapAssignStr, 1<<8, 1<<16)) -} - -func BenchmarkMapOperatorAssign(b *testing.B) { - b.Run("Int32", runWith(benchmarkMapOperatorAssignInt32, 1<<8, 1<<16)) - b.Run("Int64", runWith(benchmarkMapOperatorAssignInt64, 1<<8, 1<<16)) - b.Run("Str", runWith(benchmarkMapOperatorAssignStr, 1<<8, 1<<16)) -} - -func BenchmarkMapAppendAssign(b *testing.B) { - b.Run("Int32", runWith(benchmarkMapAppendAssignInt32, 1<<8, 1<<16)) - b.Run("Int64", runWith(benchmarkMapAppendAssignInt64, 1<<8, 1<<16)) - b.Run("Str", runWith(benchmarkMapAppendAssignStr, 1<<8, 1<<16)) -} - -func BenchmarkMapDelete(b *testing.B) { - b.Run("Int32", runWith(benchmarkMapDeleteInt32, 100, 1000, 10000)) - b.Run("Int64", runWith(benchmarkMapDeleteInt64, 100, 1000, 10000)) - b.Run("Str", runWith(benchmarkMapDeleteStr, 100, 1000, 10000)) - b.Run("Pointer", runWith(benchmarkMapDeletePointer, 100, 1000, 10000)) } func TestDeferDeleteSlow(t *testing.T) { @@ -1174,31 +814,6 @@ func TestIncrementAfterBulkClearKeyStringValueInt(t *testing.T) { } } -func TestMapTombstones(t *testing.T) { - m := map[int]int{} - const N = 10000 - // Fill a map. - for i := 0; i < N; i++ { - m[i] = i - } - runtime.MapTombstoneCheck(m) - // Delete half of the entries. - for i := 0; i < N; i += 2 { - delete(m, i) - } - runtime.MapTombstoneCheck(m) - // Add new entries to fill in holes. - for i := N; i < 3*N/2; i++ { - m[i] = i - } - runtime.MapTombstoneCheck(m) - // Delete everything. - for i := 0; i < 3*N/2; i++ { - delete(m, i) - } - runtime.MapTombstoneCheck(m) -} - type canString int func (c canString) String() string { @@ -1422,51 +1037,6 @@ func TestEmptyMapWithInterfaceKey(t *testing.T) { }) } -func TestLoadFactor(t *testing.T) { - for b := uint8(0); b < 20; b++ { - count := 13 * (1 << b) / 2 // 6.5 - if b == 0 { - count = 8 - } - if runtime.OverLoadFactor(count, b) { - t.Errorf("OverLoadFactor(%d,%d)=true, want false", count, b) - } - if !runtime.OverLoadFactor(count+1, b) { - t.Errorf("OverLoadFactor(%d,%d)=false, want true", count+1, b) - } - } -} - -func TestMapKeys(t *testing.T) { - type key struct { - s string - pad [128]byte // sizeof(key) > abi.MapMaxKeyBytes - } - m := map[key]int{{s: "a"}: 1, {s: "b"}: 2} - keys := make([]key, 0, len(m)) - runtime.MapKeys(m, unsafe.Pointer(&keys)) - for _, k := range keys { - if len(k.s) != 1 { - t.Errorf("len(k.s) == %d, want 1", len(k.s)) - } - } -} - -func TestMapValues(t *testing.T) { - type val struct { - s string - pad [128]byte // sizeof(val) > abi.MapMaxElemBytes - } - m := map[int]val{1: {s: "a"}, 2: {s: "b"}} - vals := make([]val, 0, len(m)) - runtime.MapValues(m, unsafe.Pointer(&vals)) - for _, v := range vals { - if len(v.s) != 1 { - t.Errorf("len(v.s) == %d, want 1", len(v.s)) - } - } -} - func computeHash() uintptr { var v struct{} return runtime.MemHash(unsafe.Pointer(&v), 0, unsafe.Sizeof(v)) @@ -1475,7 +1045,7 @@ func computeHash() uintptr { func subprocessHash(t *testing.T, env string) uintptr { t.Helper() - cmd := testenv.CleanCmdEnv(testenv.Command(t, os.Args[0], "-test.run=^TestMemHashGlobalSeed$")) + cmd := testenv.CleanCmdEnv(testenv.Command(t, testenv.Executable(t), "-test.run=^TestMemHashGlobalSeed$")) cmd.Env = append(cmd.Env, "GO_TEST_SUBPROCESS_HASH=1") if env != "" { cmd.Env = append(cmd.Env, env) @@ -1544,3 +1114,89 @@ func TestMemHashGlobalSeed(t *testing.T) { } }) } + +func TestMapIterDeleteReplace(t *testing.T) { + inc := 1 + if testing.Short() { + inc = 100 + } + for i := 0; i < 10000; i += inc { + t.Run(fmt.Sprint(i), func(t *testing.T) { + m := make(map[int]bool) + for j := range i { + m[j] = false + } + + // Delete and replace all entries. + for k := range m { + delete(m, k) + m[k] = true + } + + for k, v := range m { + if !v { + t.Errorf("m[%d] got false want true", k) + } + } + }) + } +} + +func TestHmapSize(t *testing.T) { + // The structure of Map is defined in internal/runtime/maps/map.go + // and in cmd/compile/internal/reflectdata/map.go and must be in sync. + // The size of Map should be 48 bytes on 64 bit and 32 bytes on 32 bit platforms. + wantSize := uintptr(2*8 + 4*goarch.PtrSize) + gotSize := unsafe.Sizeof(maps.Map{}) + if gotSize != wantSize { + t.Errorf("sizeof(maps.Map{})==%d, want %d", gotSize, wantSize) + } +} + +// See also reflect_test.TestGroupSizeZero. +func TestGroupSizeZero(t *testing.T) { + var m map[struct{}]struct{} + mTyp := abi.TypeOf(m) + mt := (*abi.MapType)(unsafe.Pointer(mTyp)) + + // internal/runtime/maps when create pointers to slots, even if slots + // are size 0. The compiler should have reserved an extra word to + // ensure that pointers to the zero-size type at the end of group are + // valid. + if mt.Group.Size() <= 8 { + t.Errorf("Group size got %d want >8", mt.Group.Size()) + } +} + +func TestMapIterOrder(t *testing.T) { + sizes := []int{3, 7, 9, 15} + for _, n := range sizes { + for i := 0; i < 1000; i++ { + // Make m be {0: true, 1: true, ..., n-1: true}. + m := make(map[int]bool) + for i := 0; i < n; i++ { + m[i] = true + } + // Check that iterating over the map produces at least two different orderings. + ord := func() []int { + var s []int + for key := range m { + s = append(s, key) + } + return s + } + first := ord() + ok := false + for try := 0; try < 100; try++ { + if !slices.Equal(first, ord()) { + ok = true + break + } + } + if !ok { + t.Errorf("Map with n=%d elements had consistent iteration order: %v", n, first) + break + } + } + } +} diff --git a/src/runtime/mbarrier.go b/src/runtime/mbarrier.go index 7dc8a1a5e56df7..a582a204b26b45 100644 --- a/src/runtime/mbarrier.go +++ b/src/runtime/mbarrier.go @@ -17,6 +17,7 @@ import ( "internal/abi" "internal/goarch" "internal/goexperiment" + "internal/runtime/sys" "unsafe" ) @@ -91,19 +92,6 @@ import ( // barriers, which will slow down both the mutator and the GC, we always grey // the ptr object regardless of the slot's color. // -// Another place where we intentionally omit memory barriers is when -// accessing mheap_.arena_used to check if a pointer points into the -// heap. On relaxed memory machines, it's possible for a mutator to -// extend the size of the heap by updating arena_used, allocate an -// object from this new region, and publish a pointer to that object, -// but for tracing running on another processor to observe the pointer -// but use the old value of arena_used. In this case, tracing will not -// mark the object, even though it's reachable. However, the mutator -// is guaranteed to execute a write barrier when it publishes the -// pointer, so it will take care of marking the object. A general -// consequence of this is that the garbage collector may cache the -// value of mheap_.arena_used. (See issue #9984.) -// // // Stack writes: // @@ -224,8 +212,8 @@ func wbMove(typ *_type, dst, src unsafe.Pointer) { //go:linkname reflect_typedmemmove reflect.typedmemmove func reflect_typedmemmove(typ *_type, dst, src unsafe.Pointer) { if raceenabled { - raceWriteObjectPC(typ, dst, getcallerpc(), abi.FuncPCABIInternal(reflect_typedmemmove)) - raceReadObjectPC(typ, src, getcallerpc(), abi.FuncPCABIInternal(reflect_typedmemmove)) + raceWriteObjectPC(typ, dst, sys.GetCallerPC(), abi.FuncPCABIInternal(reflect_typedmemmove)) + raceReadObjectPC(typ, src, sys.GetCallerPC(), abi.FuncPCABIInternal(reflect_typedmemmove)) } if msanenabled { msanwrite(dst, typ.Size_) @@ -243,6 +231,11 @@ func reflectlite_typedmemmove(typ *_type, dst, src unsafe.Pointer) { reflect_typedmemmove(typ, dst, src) } +//go:linkname maps_typedmemmove internal/runtime/maps.typedmemmove +func maps_typedmemmove(typ *_type, dst, src unsafe.Pointer) { + typedmemmove(typ, dst, src) +} + // reflectcallmove is invoked by reflectcall to copy the return values // out of the stack and into the heap, invoking the necessary write // barriers. dst, src, and size describe the return value area to @@ -294,7 +287,7 @@ func typedslicecopy(typ *_type, dstPtr unsafe.Pointer, dstLen int, srcPtr unsafe // assignment operations, it's not instrumented in the calling // code and needs its own instrumentation. if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() pc := abi.FuncPCABIInternal(slicecopy) racewriterangepc(dstPtr, uintptr(n)*typ.Size_, callerpc, pc) racereadrangepc(srcPtr, uintptr(n)*typ.Size_, callerpc, pc) @@ -375,7 +368,7 @@ func typedmemclr(typ *_type, ptr unsafe.Pointer) { memclrNoHeapPointers(ptr, typ.Size_) } -// reflect_typedslicecopy is meant for package reflect, +// reflect_typedmemclr is meant for package reflect, // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/ugorji/go/codec @@ -388,6 +381,11 @@ func reflect_typedmemclr(typ *_type, ptr unsafe.Pointer) { typedmemclr(typ, ptr) } +//go:linkname maps_typedmemclr internal/runtime/maps.typedmemclr +func maps_typedmemclr(typ *_type, ptr unsafe.Pointer) { + typedmemclr(typ, ptr) +} + //go:linkname reflect_typedmemclrpartial reflect.typedmemclrpartial func reflect_typedmemclrpartial(typ *_type, ptr unsafe.Pointer, off, size uintptr) { if writeBarrier.enabled && typ.Pointers() { diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go index a25995f46fc028..37a92c64bc6ecc 100644 --- a/src/runtime/mbitmap.go +++ b/src/runtime/mbitmap.go @@ -58,61 +58,25 @@ package runtime import ( "internal/abi" "internal/goarch" + "internal/goexperiment" "internal/runtime/atomic" + "internal/runtime/gc" "internal/runtime/sys" "unsafe" ) -const ( - // A malloc header is functionally a single type pointer, but - // we need to use 8 here to ensure 8-byte alignment of allocations - // on 32-bit platforms. It's wasteful, but a lot of code relies on - // 8-byte alignment for 8-byte atomics. - mallocHeaderSize = 8 - - // The minimum object size that has a malloc header, exclusive. - // - // The size of this value controls overheads from the malloc header. - // The minimum size is bound by writeHeapBitsSmall, which assumes that the - // pointer bitmap for objects of a size smaller than this doesn't cross - // more than one pointer-word boundary. This sets an upper-bound on this - // value at the number of bits in a uintptr, multiplied by the pointer - // size in bytes. - // - // We choose a value here that has a natural cutover point in terms of memory - // overheads. This value just happens to be the maximum possible value this - // can be. - // - // A span with heap bits in it will have 128 bytes of heap bits on 64-bit - // platforms, and 256 bytes of heap bits on 32-bit platforms. The first size - // class where malloc headers match this overhead for 64-bit platforms is - // 512 bytes (8 KiB / 512 bytes * 8 bytes-per-header = 128 bytes of overhead). - // On 32-bit platforms, this same point is the 256 byte size class - // (8 KiB / 256 bytes * 8 bytes-per-header = 256 bytes of overhead). - // - // Guaranteed to be exactly at a size class boundary. The reason this value is - // an exclusive minimum is subtle. Suppose we're allocating a 504-byte object - // and its rounded up to 512 bytes for the size class. If minSizeForMallocHeader - // is 512 and an inclusive minimum, then a comparison against minSizeForMallocHeader - // by the two values would produce different results. In other words, the comparison - // would not be invariant to size-class rounding. Eschewing this property means a - // more complex check or possibly storing additional state to determine whether a - // span has malloc headers. - minSizeForMallocHeader = goarch.PtrSize * ptrBits -) - // heapBitsInSpan returns true if the size of an object implies its ptr/scalar // data is stored at the end of the span, and is accessible via span.heapBits. // // Note: this works for both rounded-up sizes (span.elemsize) and unrounded -// type sizes because minSizeForMallocHeader is guaranteed to be at a size +// type sizes because gc.MinSizeForMallocHeader is guaranteed to be at a size // class boundary. // //go:nosplit func heapBitsInSpan(userSize uintptr) bool { - // N.B. minSizeForMallocHeader is an exclusive minimum so that this function is + // N.B. gc.MinSizeForMallocHeader is an exclusive minimum so that this function is // invariant under size-class rounding on its input. - return userSize <= minSizeForMallocHeader + return userSize <= gc.MinSizeForMallocHeader } // typePointers is an iterator over the pointers in a heap object. @@ -189,23 +153,24 @@ func (span *mspan) typePointersOfUnchecked(addr uintptr) typePointers { if spc.sizeclass() != 0 { // Pull the allocation header from the first word of the object. typ = *(**_type)(unsafe.Pointer(addr)) - addr += mallocHeaderSize + addr += gc.MallocHeaderSize } else { - typ = span.largeType + // Synchronize with allocator, in case this came from the conservative scanner. + // See heapSetTypeLarge for more details. + typ = (*_type)(atomic.Loadp(unsafe.Pointer(&span.largeType))) if typ == nil { // Allow a nil type here for delayed zeroing. See mallocgc. return typePointers{} } } - gcdata := typ.GCData - return typePointers{elem: addr, addr: addr, mask: readUintptr(gcdata), typ: typ} + gcmask := getGCMask(typ) + return typePointers{elem: addr, addr: addr, mask: readUintptr(gcmask), typ: typ} } // typePointersOfType is like typePointersOf, but assumes addr points to one or more -// contiguous instances of the provided type. The provided type must not be nil and -// it must not have its type metadata encoded as a gcprog. +// contiguous instances of the provided type. The provided type must not be nil. // -// It returns an iterator that tiles typ.GCData starting from addr. It's the caller's +// It returns an iterator that tiles typ's gcmask starting from addr. It's the caller's // responsibility to limit iteration. // // nosplit because its callers are nosplit and require all their callees to be nosplit. @@ -213,15 +178,15 @@ func (span *mspan) typePointersOfUnchecked(addr uintptr) typePointers { //go:nosplit func (span *mspan) typePointersOfType(typ *abi.Type, addr uintptr) typePointers { const doubleCheck = false - if doubleCheck && (typ == nil || typ.Kind_&abi.KindGCProg != 0) { + if doubleCheck && typ == nil { throw("bad type passed to typePointersOfType") } if span.spanclass.noscan() { return typePointers{} } // Since we have the type, pretend we have a header. - gcdata := typ.GCData - return typePointers{elem: addr, addr: addr, mask: readUintptr(gcdata), typ: typ} + gcmask := getGCMask(typ) + return typePointers{elem: addr, addr: addr, mask: readUintptr(gcmask), typ: typ} } // nextFast is the fast path of next. nextFast is written to be inlineable and, @@ -256,8 +221,13 @@ func (tp typePointers) nextFast() (typePointers, uintptr) { } else { i = sys.TrailingZeros32(uint32(tp.mask)) } - // BTCQ - tp.mask ^= uintptr(1) << (i & (ptrBits - 1)) + if GOARCH == "amd64" { + // BTCQ + tp.mask ^= uintptr(1) << (i & (ptrBits - 1)) + } else { + // SUB, AND + tp.mask &= tp.mask - 1 + } // LEAQ (XX)(XX*8) return tp, tp.addr + uintptr(i)*goarch.PtrSize } @@ -295,7 +265,7 @@ func (tp typePointers) next(limit uintptr) (typePointers, uintptr) { } // Grab more bits and try again. - tp.mask = readUintptr(addb(tp.typ.GCData, (tp.addr-tp.elem)/goarch.PtrSize/8)) + tp.mask = readUintptr(addb(getGCMask(tp.typ), (tp.addr-tp.elem)/goarch.PtrSize/8)) if tp.addr+goarch.PtrSize*ptrBits > limit { bits := (tp.addr + goarch.PtrSize*ptrBits - limit) / goarch.PtrSize tp.mask &^= ((1 << (bits)) - 1) << (ptrBits - bits) @@ -345,7 +315,7 @@ func (tp typePointers) fastForward(n, limit uintptr) typePointers { // Move up to the next element. tp.elem += tp.typ.Size_ tp.addr = tp.elem - tp.mask = readUintptr(tp.typ.GCData) + tp.mask = readUintptr(getGCMask(tp.typ)) // We may have exceeded the limit after this. Bail just like next does. if tp.addr >= limit { @@ -354,7 +324,7 @@ func (tp typePointers) fastForward(n, limit uintptr) typePointers { } else { // Grab the mask, but then clear any bits before the target address and any // bits over the limit. - tp.mask = readUintptr(addb(tp.typ.GCData, (tp.addr-tp.elem)/goarch.PtrSize/8)) + tp.mask = readUintptr(addb(getGCMask(tp.typ), (tp.addr-tp.elem)/goarch.PtrSize/8)) tp.mask &^= (1 << ((target - tp.addr) / goarch.PtrSize)) - 1 } if tp.addr+goarch.PtrSize*ptrBits > limit { @@ -457,7 +427,7 @@ func bulkBarrierPreWrite(dst, src, size uintptr, typ *abi.Type) { } var tp typePointers - if typ != nil && typ.Kind_&abi.KindGCProg == 0 { + if typ != nil { tp = s.typePointersOfType(typ, dst) } else { tp = s.typePointersOf(dst, size) @@ -518,7 +488,7 @@ func bulkBarrierPreWriteSrcOnly(dst, src, size uintptr, typ *abi.Type) { } var tp typePointers - if typ != nil && typ.Kind_&abi.KindGCProg == 0 { + if typ != nil { tp = s.typePointersOfType(typ, dst) } else { tp = s.typePointersOf(dst, size) @@ -535,15 +505,19 @@ func bulkBarrierPreWriteSrcOnly(dst, src, size uintptr, typ *abi.Type) { } // initHeapBits initializes the heap bitmap for a span. -// -// TODO(mknyszek): This should set the heap bits for single pointer -// allocations eagerly to avoid calling heapSetType at allocation time, -// just to write one bit. -func (s *mspan) initHeapBits(forceClear bool) { - if (!s.spanclass.noscan() && heapBitsInSpan(s.elemsize)) || s.isUserArenaChunk { +func (s *mspan) initHeapBits() { + if goarch.PtrSize == 8 && !s.spanclass.noscan() && s.spanclass.sizeclass() == 1 { + b := s.heapBits() + for i := range b { + b[i] = ^uintptr(0) + } + } else if (!s.spanclass.noscan() && heapBitsInSpan(s.elemsize)) || s.isUserArenaChunk { b := s.heapBits() clear(b) } + if goexperiment.GreenTeaGC && gcUsesSpanInlineMarkBits(s.elemsize) { + s.initInlineMarkBits() + } } // heapBits returns the heap ptr/scalar bits stored at the end of the span for @@ -567,7 +541,7 @@ func (span *mspan) heapBits() []uintptr { if span.spanclass.noscan() { throw("heapBits called for noscan") } - if span.elemsize > minSizeForMallocHeader { + if span.elemsize > gc.MinSizeForMallocHeader { throw("heapBits called for span class that should have a malloc header") } } @@ -576,22 +550,32 @@ func (span *mspan) heapBits() []uintptr { // Nearly every span with heap bits is exactly one page in size. Arenas are the only exception. if span.npages == 1 { // This will be inlined and constant-folded down. - return heapBitsSlice(span.base(), pageSize) + return heapBitsSlice(span.base(), pageSize, span.elemsize) } - return heapBitsSlice(span.base(), span.npages*pageSize) + return heapBitsSlice(span.base(), span.npages*pageSize, span.elemsize) } // Helper for constructing a slice for the span's heap bits. // //go:nosplit -func heapBitsSlice(spanBase, spanSize uintptr) []uintptr { - bitmapSize := spanSize / goarch.PtrSize / 8 +func heapBitsSlice(spanBase, spanSize, elemsize uintptr) []uintptr { + base, bitmapSize := spanHeapBitsRange(spanBase, spanSize, elemsize) elems := int(bitmapSize / goarch.PtrSize) var sl notInHeapSlice - sl = notInHeapSlice{(*notInHeap)(unsafe.Pointer(spanBase + spanSize - bitmapSize)), elems, elems} + sl = notInHeapSlice{(*notInHeap)(unsafe.Pointer(base)), elems, elems} return *(*[]uintptr)(unsafe.Pointer(&sl)) } +//go:nosplit +func spanHeapBitsRange(spanBase, spanSize, elemsize uintptr) (base, size uintptr) { + size = spanSize / goarch.PtrSize / 8 + base = spanBase + spanSize - size + if goexperiment.GreenTeaGC && gcUsesSpanInlineMarkBits(elemsize) { + base -= unsafe.Sizeof(spanInlineMarkBits{}) + } + return +} + // heapBitsSmallForAddr loads the heap bits for the object stored at addr from span.heapBits. // // addr must be the base pointer of an object in the span. heapBitsInSpan(span.elemsize) @@ -599,9 +583,8 @@ func heapBitsSlice(spanBase, spanSize uintptr) []uintptr { // //go:nosplit func (span *mspan) heapBitsSmallForAddr(addr uintptr) uintptr { - spanSize := span.npages * pageSize - bitmapSize := spanSize / goarch.PtrSize / 8 - hbits := (*byte)(unsafe.Pointer(span.base() + spanSize - bitmapSize)) + hbitsBase, _ := spanHeapBitsRange(span.base(), span.npages*pageSize, span.elemsize) + hbits := (*byte)(unsafe.Pointer(hbitsBase)) // These objects are always small enough that their bitmaps // fit in a single word, so just load the word or two we need. @@ -640,37 +623,51 @@ func (span *mspan) heapBitsSmallForAddr(addr uintptr) uintptr { //go:nosplit func (span *mspan) writeHeapBitsSmall(x, dataSize uintptr, typ *_type) (scanSize uintptr) { // The objects here are always really small, so a single load is sufficient. - src0 := readUintptr(typ.GCData) + src0 := readUintptr(getGCMask(typ)) - // Create repetitions of the bitmap if we have a small array. - bits := span.elemsize / goarch.PtrSize + // Create repetitions of the bitmap if we have a small slice backing store. scanSize = typ.PtrBytes src := src0 - switch typ.Size_ { - case goarch.PtrSize: + if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 - default: + } else { + // N.B. We rely on dataSize being an exact multiple of the type size. + // The alternative is to be defensive and mask out src to the length + // of dataSize. The purpose is to save on one additional masking operation. + if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { + throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") + } for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ } + if asanenabled { + // Mask src down to dataSize. dataSize is going to be a strange size because of + // the redzone required for allocations when asan is enabled. + src &= (1 << (dataSize / goarch.PtrSize)) - 1 + } } // Since we're never writing more than one uintptr's worth of bits, we're either going // to do one or two writes. - dst := span.heapBits() + dstBase, _ := spanHeapBitsRange(span.base(), pageSize, span.elemsize) + dst := unsafe.Pointer(dstBase) o := (x - span.base()) / goarch.PtrSize i := o / ptrBits j := o % ptrBits + bits := span.elemsize / goarch.PtrSize if j+bits > ptrBits { // Two writes. bits0 := ptrBits - j bits1 := bits - bits0 - dst[i+0] = dst[i+0]&(^uintptr(0)>>bits0) | (src << j) - dst[i+1] = dst[i+1]&^((1<> bits0) + dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize)) + dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize)) + *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j) + *dst1 = (*dst1)&^((1<> bits0) } else { // One write. - dst[i] = (dst[i] &^ (((1 << bits) - 1) << j)) | (src << j) + dst := (*uintptr)(add(dst, i*goarch.PtrSize)) + *dst = (*dst)&^(((1< x+maxIterBytes { - size = x + maxIterBytes - interior - } - doubleCheckHeapPointersInterior(x, interior, size, dataSize, gctyp, header, span) +func heapSetTypeLarge(x, dataSize uintptr, typ *_type, span *mspan) uintptr { + gctyp := typ + // Write out the header atomically to synchronize with the garbage collector. + // + // This atomic store is paired with an atomic load in typePointersOfUnchecked. + // This store ensures that initializing x's memory cannot be reordered after + // this store. Meanwhile the load in typePointersOfUnchecked ensures that + // reading x's memory cannot be reordered before largeType is loaded. Together, + // these two operations guarantee that the garbage collector can only see + // initialized memory if largeType is non-nil. + // + // Gory details below... + // + // Ignoring conservative scanning for a moment, this store need not be atomic + // if we have a publication barrier on our side. This is because the garbage + // collector cannot observe x unless: + // 1. It stops this goroutine and scans its stack, or + // 2. We return from mallocgc and publish the pointer somewhere. + // Either case requires a write on our side, followed by some synchronization + // followed by a read by the garbage collector. + // + // In case (1), the garbage collector can only observe a nil largeType, since it + // had to stop our goroutine when it was preemptible during zeroing. For the + // duration of the zeroing, largeType is nil and the object has nothing interesting + // for the garbage collector to look at, so the garbage collector will not access + // the object at all. + // + // In case (2), the garbage collector can also observe a nil largeType. This + // might happen if the object was newly allocated, and a new GC cycle didn't start + // (that would require a global barrier, STW). In this case, the garbage collector + // will once again ignore the object, and that's safe because objects are + // allocate-black. + // + // However, the garbage collector can also observe a non-nil largeType in case (2). + // This is still okay, since to access the object's memory, it must have first + // loaded the object's pointer from somewhere. This makes the access of the object's + // memory a data-dependent load, and our publication barrier in the allocator + // guarantees that a data-dependent load must observe a version of the object's + // data from after the publication barrier executed. + // + // Unfortunately conservative scanning is a problem. There's no guarantee of a + // data dependency as in case (2) because conservative scanning can produce pointers + // 'out of thin air' in that it need not have been written somewhere by the allocating + // thread first. It might not even be a pointer, or it could be a pointer written to + // some stack location long ago. This is the fundamental reason why we need + // explicit synchronization somewhere in this whole mess. We choose to put that + // synchronization on largeType. + // + // As described at the very top, the treating largeType as an atomic variable, on + // both the reader and writer side, is sufficient to ensure that only initialized + // memory at x will be observed if largeType is non-nil. + atomic.StorepNoWB(unsafe.Pointer(&span.largeType), unsafe.Pointer(gctyp)) + if doubleCheckHeapSetType { + doubleCheckHeapType(x, dataSize, typ, &span.largeType, span) + } + return span.elemsize +} + +func doubleCheckHeapType(x, dataSize uintptr, gctyp *_type, header **_type, span *mspan) { + doubleCheckHeapPointers(x, dataSize, gctyp, header, span) + + // To exercise the less common path more often, generate + // a random interior pointer and make sure iterating from + // that point works correctly too. + maxIterBytes := span.elemsize + if header == nil { + maxIterBytes = dataSize } - return + off := alignUp(uintptr(cheaprand())%dataSize, goarch.PtrSize) + size := dataSize - off + if size == 0 { + off -= goarch.PtrSize + size += goarch.PtrSize + } + interior := x + off + size -= alignDown(uintptr(cheaprand())%size, goarch.PtrSize) + if size == 0 { + size = goarch.PtrSize + } + // Round up the type to the size of the type. + size = (size + gctyp.Size_ - 1) / gctyp.Size_ * gctyp.Size_ + if interior+size > x+maxIterBytes { + size = x + maxIterBytes - interior + } + doubleCheckHeapPointersInterior(x, interior, size, dataSize, gctyp, header, span) } func doubleCheckHeapPointers(x, dataSize uintptr, typ *_type, header **_type, span *mspan) { @@ -794,7 +843,7 @@ func doubleCheckHeapPointers(x, dataSize uintptr, typ *_type, header **_type, sp off := i % typ.Size_ if off < typ.PtrBytes { j := off / goarch.PtrSize - want = *addb(typ.GCData, j/8)>>(j%8)&1 != 0 + want = *addb(getGCMask(typ), j/8)>>(j%8)&1 != 0 } } if want { @@ -817,7 +866,7 @@ func doubleCheckHeapPointers(x, dataSize uintptr, typ *_type, header **_type, sp } println("runtime: extra pointer:", hex(addr)) } - print("runtime: hasHeader=", header != nil, " typ.Size_=", typ.Size_, " hasGCProg=", typ.Kind_&abi.KindGCProg != 0, "\n") + print("runtime: hasHeader=", header != nil, " typ.Size_=", typ.Size_, " TFlagGCMaskOnDemaind=", typ.TFlag&abi.TFlagGCMaskOnDemand != 0, "\n") print("runtime: x=", hex(x), " dataSize=", dataSize, " elemsize=", span.elemsize, "\n") print("runtime: typ=", unsafe.Pointer(typ), " typ.PtrBytes=", typ.PtrBytes, "\n") print("runtime: limit=", hex(x+span.elemsize), "\n") @@ -851,7 +900,7 @@ func doubleCheckHeapPointersInterior(x, interior, size, dataSize uintptr, typ *_ off := i % typ.Size_ if off < typ.PtrBytes { j := off / goarch.PtrSize - want = *addb(typ.GCData, j/8)>>(j%8)&1 != 0 + want = *addb(getGCMask(typ), j/8)>>(j%8)&1 != 0 } } if want { @@ -899,7 +948,7 @@ func doubleCheckHeapPointersInterior(x, interior, size, dataSize uintptr, typ *_ off := i % typ.Size_ if off < typ.PtrBytes { j := off / goarch.PtrSize - want = *addb(typ.GCData, j/8)>>(j%8)&1 != 0 + want = *addb(getGCMask(typ), j/8)>>(j%8)&1 != 0 } } if want { @@ -915,10 +964,10 @@ func doubleCheckHeapPointersInterior(x, interior, size, dataSize uintptr, typ *_ //go:nosplit func doubleCheckTypePointersOfType(s *mspan, typ *_type, addr, size uintptr) { - if typ == nil || typ.Kind_&abi.KindGCProg != 0 { + if typ == nil { return } - if typ.Kind_&abi.KindMask == abi.Interface { + if typ.Kind() == abi.Interface { // Interfaces are unfortunately inconsistently handled // when it comes to the type pointer, so it's easy to // produce a lot of false positives here. @@ -1116,7 +1165,32 @@ func (s *mspan) nextFreeIndex() uint16 { // The caller must ensure s.state is mSpanInUse, and there must have // been no preemption points since ensuring this (which could allow a // GC transition, which would allow the state to change). +// +// Callers must ensure that the index passed here must not have been +// produced from a pointer that came from 'thin air', as might happen +// with conservative scanning. func (s *mspan) isFree(index uintptr) bool { + if index < uintptr(s.freeindex) { + return false + } + bytep, mask := s.allocBits.bitp(index) + return *bytep&mask == 0 +} + +// isFreeOrNewlyAllocated reports whether the index'th object in s is +// either unallocated or has been allocated since the beginning of the +// last mark phase. +// +// The caller must ensure s.state is mSpanInUse, and there must have +// been no preemption points since ensuring this (which could allow a +// GC transition, which would allow the state to change). +// +// Callers must ensure that the index passed here must not have been +// produced from a pointer that came from 'thin air', as might happen +// with conservative scanning, unless the GC is currently in the mark +// phase. If the GC is currently in the mark phase, this function is +// safe to call for out-of-thin-air pointers. +func (s *mspan) isFreeOrNewlyAllocated(index uintptr) bool { if index < uintptr(s.freeIndexForScan) { return false } @@ -1158,15 +1232,6 @@ func markBitsForAddr(p uintptr) markBits { return s.markBitsForIndex(objIndex) } -func (s *mspan) markBitsForIndex(objIndex uintptr) markBits { - bytep, mask := s.gcmarkBits.bitp(objIndex) - return markBits{bytep, mask, objIndex} -} - -func (s *mspan) markBitsForBase() markBits { - return markBits{&s.gcmarkBits.x, uint8(1), 0} -} - // isMarked reports whether mark bit m is set. func (m markBits) isMarked() bool { return *m.bytep&m.mask != 0 @@ -1202,6 +1267,28 @@ func markBitsForSpan(base uintptr) (mbits markBits) { return mbits } +// isMarkedOrNotInHeap returns true if a pointer is in the heap and marked, +// or if the pointer is not in the heap. Used by goroutine leak detection +// to determine if concurrency resources are reachable in memory. +func isMarkedOrNotInHeap(p unsafe.Pointer) bool { + obj, span, objIndex := findObject(uintptr(p), 0, 0) + if obj != 0 { + mbits := span.markBitsForIndex(objIndex) + return mbits.isMarked() + } + + // If we fall through to get here, the object is not in the heap. + // In this case, it is either a pointer to a stack object or a global resource. + // Treat it as reachable in memory by default, to be safe. + // + // TODO(vsaioc): we could be more precise by checking against the stacks + // of runnable goroutines. I don't think this is necessary, based on what we've seen, but + // let's keep the option open in case the runtime evolves. + // This will (naively) lead to quadratic blow-up for goroutine leak detection, + // but if it is only run on demand, maybe the extra cost is not a show-stopper. + return true +} + // advance advances the markBits to the next object in the span. func (m *markBits) advance() { if m.mask == 1<<7 { @@ -1365,9 +1452,6 @@ func bulkBarrierBitmap(dst, src, size, maskOffset uintptr, bits *uint8) { // // The type typ must correspond exactly to [src, src+size) and [dst, dst+size). // dst, src, and size must be pointer-aligned. -// The type typ must have a plain bitmap, not a GC program. -// The only use of this function is in channel sends, and the -// 64 kB channel element limit takes care of this for us. // // Must not be preempted because it typically runs right before memmove, // and the GC must observe them as an atomic action. @@ -1383,14 +1467,10 @@ func typeBitsBulkBarrier(typ *_type, dst, src, size uintptr) { println("runtime: typeBitsBulkBarrier with type ", toRType(typ).string(), " of size ", typ.Size_, " but memory size", size) throw("runtime: invalid typeBitsBulkBarrier") } - if typ.Kind_&abi.KindGCProg != 0 { - println("runtime: typeBitsBulkBarrier with type ", toRType(typ).string(), " with GC prog") - throw("runtime: invalid typeBitsBulkBarrier") - } if !writeBarrier.enabled { return } - ptrmask := typ.GCData + ptrmask := getGCMask(typ) buf := &getg().m.p.ptr().wbBuf var bits uint32 for i := uintptr(0); i < typ.PtrBytes; i += goarch.PtrSize { @@ -1475,6 +1555,9 @@ func progToPointerMask(prog *byte, size uintptr) bitvector { // 0nnnnnnn: emit n bits copied from the next (n+7)/8 bytes // 10000000 n c: repeat the previous n bits c times; n, c are varints // 1nnnnnnn c: repeat the previous n bits c times; c is a varint +// +// Currently, gc programs are only used for describing data and bss +// sections of the binary. // runGCProg returns the number of 1-bit entries written to memory. func runGCProg(prog, dst *byte) uintptr { @@ -1671,24 +1754,6 @@ Run: return totalBits } -// materializeGCProg allocates space for the (1-bit) pointer bitmask -// for an object of size ptrdata. Then it fills that space with the -// pointer bitmask specified by the program prog. -// The bitmask starts at s.startAddr. -// The result must be deallocated with dematerializeGCProg. -func materializeGCProg(ptrdata uintptr, prog *byte) *mspan { - // Each word of ptrdata needs one bit in the bitmap. - bitmapBytes := divRoundUp(ptrdata, 8*goarch.PtrSize) - // Compute the number of pages needed for bitmapBytes. - pages := divRoundUp(bitmapBytes, pageSize) - s := mheap_.allocManual(pages, spanAllocPtrScalarBits) - runGCProg(addb(prog, 4), (*byte)(unsafe.Pointer(s.startAddr))) - return s -} -func dematerializeGCProg(s *mspan) { - mheap_.freeManual(s, spanAllocPtrScalarBits) -} - func dumpGCProg(p *byte) { nptr := 0 for { @@ -1741,19 +1806,19 @@ func dumpGCProg(p *byte) { // //go:linkname reflect_gcbits reflect.gcbits func reflect_gcbits(x any) []byte { - return getgcmask(x) + return pointerMask(x) } // Returns GC type info for the pointer stored in ep for testing. // If ep points to the stack, only static live information will be returned // (i.e. not for objects which are only dynamically live stack objects). -func getgcmask(ep any) (mask []byte) { +func pointerMask(ep any) (mask []byte) { e := *efaceOf(&ep) p := e.data t := e._type var et *_type - if t.Kind_&abi.KindMask != abi.Pointer { + if t.Kind() != abi.Pointer { throw("bad argument to getgcmask: expected type to be a pointer to the value type whose mask is being queried") } et = (*ptrtype)(unsafe.Pointer(t)).Elem @@ -1823,50 +1888,48 @@ func getgcmask(ep any) (mask []byte) { maskFromHeap = maskFromHeap[:len(maskFromHeap)-1] } - if et.Kind_&abi.KindGCProg == 0 { - // Unroll again, but this time from the type information. - maskFromType := make([]byte, (limit-base)/goarch.PtrSize) - tp = s.typePointersOfType(et, base) - for { - var addr uintptr - if tp, addr = tp.next(limit); addr == 0 { - break - } - maskFromType[(addr-base)/goarch.PtrSize] = 1 + // Unroll again, but this time from the type information. + maskFromType := make([]byte, (limit-base)/goarch.PtrSize) + tp = s.typePointersOfType(et, base) + for { + var addr uintptr + if tp, addr = tp.next(limit); addr == 0 { + break } - - // Validate that the prefix of maskFromType is equal to - // maskFromHeap. maskFromType may contain more pointers than - // maskFromHeap produces because maskFromHeap may be able to - // get exact type information for certain classes of objects. - // With maskFromType, we're always just tiling the type bitmap - // through to the elemsize. - // - // It's OK if maskFromType has pointers in elemsize that extend - // past the actual populated space; we checked above that all - // that space is zeroed, so just the GC will just see nil pointers. - differs := false - for i := range maskFromHeap { - if maskFromHeap[i] != maskFromType[i] { - differs = true - break - } + maskFromType[(addr-base)/goarch.PtrSize] = 1 + } + + // Validate that the prefix of maskFromType is equal to + // maskFromHeap. maskFromType may contain more pointers than + // maskFromHeap produces because maskFromHeap may be able to + // get exact type information for certain classes of objects. + // With maskFromType, we're always just tiling the type bitmap + // through to the elemsize. + // + // It's OK if maskFromType has pointers in elemsize that extend + // past the actual populated space; we checked above that all + // that space is zeroed, so just the GC will just see nil pointers. + differs := false + for i := range maskFromHeap { + if maskFromHeap[i] != maskFromType[i] { + differs = true + break } + } - if differs { - print("runtime: heap mask=") - for _, b := range maskFromHeap { - print(b) - } - println() - print("runtime: type mask=") - for _, b := range maskFromType { - print(b) - } - println() - print("runtime: type=", toRType(et).string(), "\n") - throw("found two different masks from two different methods") + if differs { + print("runtime: heap mask=") + for _, b := range maskFromHeap { + print(b) + } + println() + print("runtime: type mask=") + for _, b := range maskFromType { + print(b) } + println() + print("runtime: type=", toRType(et).string(), "\n") + throw("found two different masks from two different methods") } // Select the heap mask to return. We may not have a type mask. diff --git a/src/runtime/mcache.go b/src/runtime/mcache.go index 97a5f70e9c78f7..a1d04d2f8a2e49 100644 --- a/src/runtime/mcache.go +++ b/src/runtime/mcache.go @@ -6,6 +6,7 @@ package runtime import ( "internal/runtime/atomic" + "internal/runtime/gc" "internal/runtime/sys" "unsafe" ) @@ -21,8 +22,9 @@ type mcache struct { // The following members are accessed on every malloc, // so they are grouped here for better caching. - nextSample uintptr // trigger heap sample after allocating this many bytes - scanAlloc uintptr // bytes of scannable heap allocated + nextSample int64 // trigger heap sample after allocating this many bytes + memProfRate int // cached mem profile rate, used to detect changes + scanAlloc uintptr // bytes of scannable heap allocated // Allocator cache for tiny objects w/o pointers. // See "Tiny allocator" comment in malloc.go. @@ -217,18 +219,18 @@ func (c *mcache) refill(spc spanClass) { // allocLarge allocates a span for a large object. func (c *mcache) allocLarge(size uintptr, noscan bool) *mspan { - if size+_PageSize < size { + if size+pageSize < size { throw("out of memory") } - npages := size >> _PageShift - if size&_PageMask != 0 { + npages := size >> gc.PageShift + if size&pageMask != 0 { npages++ } // Deduct credit for this span allocation and sweep if // necessary. mHeap_Alloc will also sweep npages, so this only // pays the debt down to npage pages. - deductSweepCredit(npages*_PageSize, npages) + deductSweepCredit(npages*pageSize, npages) spc := makeSpanClass(0, noscan) s := mheap_.alloc(npages, spc) @@ -251,8 +253,16 @@ func (c *mcache) allocLarge(size uintptr, noscan bool) *mspan { // Put the large span in the mcentral swept list so that it's // visible to the background sweeper. mheap_.central[spc].mcentral.fullSwept(mheap_.sweepgen).push(s) + + // Adjust s.limit down to the object-containing part of the span. + // + // This is just to create a slightly tighter bound on the limit. + // It's totally OK if the garbage collector, in particular + // conservative scanning, can temporarily observes an inflated + // limit. It will simply mark the whole object or just skip it + // since we're in the mark phase anyway. s.limit = s.base() + size - s.initHeapBits(false) + s.initHeapBits() return s } diff --git a/src/runtime/mcentral.go b/src/runtime/mcentral.go index 1a4819bc2c43a5..ec27ce25a88812 100644 --- a/src/runtime/mcentral.go +++ b/src/runtime/mcentral.go @@ -14,6 +14,7 @@ package runtime import ( "internal/runtime/atomic" + "internal/runtime/gc" "internal/runtime/sys" ) @@ -80,7 +81,7 @@ func (c *mcentral) fullSwept(sweepgen uint32) *spanSet { // Allocate a span to use in an mcache. func (c *mcentral) cacheSpan() *mspan { // Deduct credit for this span allocation and sweep if necessary. - spanBytes := uintptr(class_to_allocnpages[c.spanclass.sizeclass()]) * _PageSize + spanBytes := uintptr(gc.SizeClassToNPages[c.spanclass.sizeclass()]) * pageSize deductSweepCredit(spanBytes, 0) traceDone := false @@ -248,18 +249,11 @@ func (c *mcentral) uncacheSpan(s *mspan) { // grow allocates a new empty span from the heap and initializes it for c's size class. func (c *mcentral) grow() *mspan { - npages := uintptr(class_to_allocnpages[c.spanclass.sizeclass()]) - size := uintptr(class_to_size[c.spanclass.sizeclass()]) - + npages := uintptr(gc.SizeClassToNPages[c.spanclass.sizeclass()]) s := mheap_.alloc(npages, c.spanclass) if s == nil { return nil } - - // Use division by multiplication and shifts to quickly compute: - // n := (npages << _PageShift) / size - n := s.divideByElemSize(npages << _PageShift) - s.limit = s.base() + size*n - s.initHeapBits(false) + s.initHeapBits() return s } diff --git a/src/runtime/mcheckmark.go b/src/runtime/mcheckmark.go index f5560cf50f1fdc..083220f4492954 100644 --- a/src/runtime/mcheckmark.go +++ b/src/runtime/mcheckmark.go @@ -39,7 +39,7 @@ func startCheckmarks() { assertWorldStopped() // Clear all checkmarks. - for _, ai := range mheap_.allArenas { + clearCheckmarks := func(ai arenaIdx) { arena := mheap_.arenas[ai.l1()][ai.l2()] bitmap := arena.checkmarks @@ -55,13 +55,20 @@ func startCheckmarks() { clear(bitmap.b[:]) } } + for _, ai := range mheap_.heapArenas { + clearCheckmarks(ai) + } + for _, ai := range mheap_.userArenaArenas { + clearCheckmarks(ai) + } + // Enable checkmarking. useCheckmark = true } // endCheckmarks ends the checkmarks phase. func endCheckmarks() { - if gcMarkWorkAvailable(nil) { + if !gcIsMarkDone() { throw("GC work not flushed") } useCheckmark = false @@ -85,18 +92,243 @@ func setCheckmark(obj, base, off uintptr, mbits markBits) bool { getg().m.traceback = 2 throw("checkmark found unmarked object") } + bytep, mask := getCheckmark(obj) + if bytep == nil { + return false + } + if atomic.Load8(bytep)&mask != 0 { + // Already checkmarked. + return true + } + atomic.Or8(bytep, mask) + return false +} +func getCheckmark(obj uintptr) (bytep *byte, mask uint8) { ai := arenaIndex(obj) arena := mheap_.arenas[ai.l1()][ai.l2()] - arenaWord := (obj / heapArenaBytes / 8) % uintptr(len(arena.checkmarks.b)) - mask := byte(1 << ((obj / heapArenaBytes) % 8)) - bytep := &arena.checkmarks.b[arenaWord] + if arena == nil { + // Non-heap pointer. + return nil, 0 + } + wordIdx := (obj - alignDown(obj, heapArenaBytes)) / goarch.PtrSize + arenaWord := wordIdx / 8 + mask = byte(1 << (wordIdx % 8)) + bytep = &arena.checkmarks.b[arenaWord] + return bytep, mask +} - if atomic.Load8(bytep)&mask != 0 { - // Already checkmarked. +// runCheckmark runs a full non-parallel, stop-the-world mark using +// checkmark bits, to check that we didn't forget to mark anything +// during the concurrent mark process. +// +// The world must be stopped to call runCheckmark. +func runCheckmark(prepareRootSet func(*gcWork)) { + assertWorldStopped() + + // Turn off gcwaiting because that will force + // gcDrain to return early if this goroutine + // happens to have its preemption flag set. + // This is fine because the world is stopped. + // Restore it after we're done just to be safe. + sched.gcwaiting.Store(false) + startCheckmarks() + gcResetMarkState() + gcw := &getg().m.p.ptr().gcw + prepareRootSet(gcw) + gcDrain(gcw, 0) + wbBufFlush1(getg().m.p.ptr()) + gcw.dispose() + endCheckmarks() + sched.gcwaiting.Store(true) +} + +// checkFinalizersAndCleanups uses checkmarks to check for potential issues +// with the program's use of cleanups and finalizers. +func checkFinalizersAndCleanups() { + assertWorldStopped() + + const ( + reportCycle = 1 << iota + reportTiny + ) + + // Find the arena and page index into that arena for this shard. + type report struct { + issues int + ptr uintptr + sp *special + } + var reports [50]report + var nreports int + var more bool + var lastTinyBlock uintptr + + forEachSpecial(func(p uintptr, s *mspan, sp *special) bool { + // N.B. The tiny block specials are sorted first in the specials list. + if sp.kind == _KindSpecialTinyBlock { + lastTinyBlock = s.base() + sp.offset + return true + } + + // We only care about finalizers and cleanups. + if sp.kind != _KindSpecialFinalizer && sp.kind != _KindSpecialCleanup { + return true + } + + // Run a checkmark GC using this cleanup and/or finalizer as a root. + if debug.checkfinalizers > 1 { + print("Scan trace for cleanup/finalizer on ", hex(p), ":\n") + } + runCheckmark(func(gcw *gcWork) { + switch sp.kind { + case _KindSpecialFinalizer: + gcScanFinalizer((*specialfinalizer)(unsafe.Pointer(sp)), s, gcw) + case _KindSpecialCleanup: + gcScanCleanup((*specialCleanup)(unsafe.Pointer(sp)), gcw) + } + }) + if debug.checkfinalizers > 1 { + println() + } + + // Now check to see if the object the special is attached to was marked. + // The roots above do not directly mark p, so if it is marked, then p + // must be reachable from the finalizer and/or cleanup, preventing + // reclamation. + bytep, mask := getCheckmark(p) + if bytep == nil { + return true + } + var issues int + if atomic.Load8(bytep)&mask != 0 { + issues |= reportCycle + } + if p >= lastTinyBlock && p < lastTinyBlock+maxTinySize { + issues |= reportTiny + } + if issues != 0 { + if nreports >= len(reports) { + more = true + return false + } + reports[nreports] = report{issues, p, sp} + nreports++ + } return true + }) + + if nreports > 0 { + lastPtr := uintptr(0) + println("WARNING: LIKELY CLEANUP/FINALIZER ISSUES") + println() + for _, r := range reports[:nreports] { + var ctx *specialCheckFinalizer + var kind string + if r.sp.kind == _KindSpecialFinalizer { + kind = "finalizer" + ctx = getCleanupContext(r.ptr, 0) + } else { + kind = "cleanup" + ctx = getCleanupContext(r.ptr, ((*specialCleanup)(unsafe.Pointer(r.sp))).id) + } + + // N.B. reports is sorted 'enough' that cleanups/finalizers on the same pointer will + // appear consecutively because the specials list is sorted. + if lastPtr != r.ptr { + if lastPtr != 0 { + println() + } + print("Value of type ", toRType(ctx.ptrType).string(), " at ", hex(r.ptr), "\n") + if r.issues&reportCycle != 0 { + if r.sp.kind == _KindSpecialFinalizer { + println(" is reachable from finalizer") + } else { + println(" is reachable from cleanup or cleanup argument") + } + } + if r.issues&reportTiny != 0 { + println(" is in a tiny block with other (possibly long-lived) values") + } + if r.issues&reportTiny != 0 && r.issues&reportCycle != 0 { + if r.sp.kind == _KindSpecialFinalizer { + println(" may be in the same tiny block as finalizer") + } else { + println(" may be in the same tiny block as cleanup or cleanup argument") + } + } + } + println() + + println("Has", kind, "at", hex(uintptr(unsafe.Pointer(r.sp)))) + funcInfo := findfunc(ctx.funcPC) + if funcInfo.valid() { + file, line := funcline(funcInfo, ctx.funcPC) + print(" ", funcname(funcInfo), "()\n") + print(" ", file, ":", line, " +", hex(ctx.funcPC-funcInfo.entry()), "\n") + } else { + print(" \n") + } + + println("created at: ") + createInfo := findfunc(ctx.createPC) + if createInfo.valid() { + file, line := funcline(createInfo, ctx.createPC) + print(" ", funcname(createInfo), "()\n") + print(" ", file, ":", line, " +", hex(ctx.createPC-createInfo.entry()), "\n") + } else { + print(" \n") + } + + lastPtr = r.ptr + } + println() + if more { + println("... too many potential issues ...") + } + throw("detected possible issues with cleanups and/or finalizers") } +} - atomic.Or8(bytep, mask) - return false +// forEachSpecial is an iterator over all specials. +// +// Used by debug.checkfinalizers. +// +// The world must be stopped. +func forEachSpecial(yield func(p uintptr, s *mspan, sp *special) bool) { + assertWorldStopped() + + // Find the arena and page index into that arena for this shard. + for _, ai := range mheap_.markArenas { + ha := mheap_.arenas[ai.l1()][ai.l2()] + + // Construct slice of bitmap which we'll iterate over. + for i := range ha.pageSpecials[:] { + // Find set bits, which correspond to spans with specials. + specials := atomic.Load8(&ha.pageSpecials[i]) + if specials == 0 { + continue + } + for j := uint(0); j < 8; j++ { + if specials&(1<= b.n { + // Take all. + copy(dst, b.cleanups[:]) + a.n += b.n + b.n = 0 + } else { + // Partial take. Copy from the tail to avoid having + // to move more memory around. + copy(dst, b.cleanups[b.n-uint32(len(dst)):b.n]) + a.n = uint32(len(a.cleanups)) + b.n -= uint32(len(dst)) + } +} + +// cleanupQueue is a queue of ready-to-run cleanup functions. +type cleanupQueue struct { + // Stack of full cleanup blocks. + full lfstack + workUnits atomic.Uint64 // length of full; decrement before pop from full, increment after push to full + _ [cpu.CacheLinePadSize - unsafe.Sizeof(lfstack(0)) - unsafe.Sizeof(atomic.Uint64{})]byte + + // Stack of free cleanup blocks. + free lfstack + + // flushed indicates whether all local cleanupBlocks have been + // flushed, and we're in a period of time where this condition is + // stable (after the last sweeper, before the next sweep phase + // begins). + flushed atomic.Bool // Next to free because frequently accessed together. + + _ [cpu.CacheLinePadSize - unsafe.Sizeof(lfstack(0)) - 1]byte + + // Linked list of all cleanup blocks. + all atomic.UnsafePointer // *cleanupBlock + _ [cpu.CacheLinePadSize - unsafe.Sizeof(atomic.UnsafePointer{})]byte + + // Goroutine block state. + lock mutex + + // sleeping is the list of sleeping cleanup goroutines. + // + // Protected by lock. + sleeping gList + + // asleep is the number of cleanup goroutines sleeping. + // + // Read without lock, written only with the lock held. + // When the lock is held, the lock holder may only observe + // asleep.Load() == sleeping.n. + // + // To make reading without the lock safe as a signal to wake up + // a goroutine and handle new work, it must always be greater + // than or equal to sleeping.n. In the periods of time that it + // is strictly greater, it may cause spurious calls to wake. + asleep atomic.Uint32 + + // running indicates the number of cleanup goroutines actively + // executing user cleanup functions at any point in time. + // + // Read and written to without lock. + running atomic.Uint32 + + // ng is the number of cleanup goroutines. + // + // Read without lock, written only with lock held. + ng atomic.Uint32 + + // needg is the number of new cleanup goroutines that + // need to be created. + // + // Read without lock, written only with lock held. + needg atomic.Uint32 + + // Cleanup queue stats. + + // queued represents a monotonic count of queued cleanups. This is sharded across + // Ps via the field cleanupsQueued in each p, so reading just this value is insufficient. + // In practice, this value only includes the queued count of dead Ps. + // + // Writes are protected by STW. + queued uint64 + + // executed is a monotonic count of executed cleanups. + // + // Read and updated atomically. + executed atomic.Uint64 +} + +// addWork indicates that n units of parallelizable work have been added to the queue. +func (q *cleanupQueue) addWork(n int) { + q.workUnits.Add(int64(n)) +} + +// tryTakeWork is an attempt to dequeue some work by a cleanup goroutine. +// This might fail if there's no work to do. +func (q *cleanupQueue) tryTakeWork() bool { + for { + wu := q.workUnits.Load() + if wu == 0 { + return false + } + // CAS to prevent us from going negative. + if q.workUnits.CompareAndSwap(wu, wu-1) { + return true + } + } +} + +// enqueue queues a single cleanup for execution. +// +// Called by the sweeper, and only the sweeper. +func (q *cleanupQueue) enqueue(fn *funcval) { + mp := acquirem() + pp := mp.p.ptr() + b := pp.cleanups + if b == nil { + if q.flushed.Load() { + q.flushed.Store(false) + } + b = (*cleanupBlock)(q.free.pop()) + if b == nil { + b = (*cleanupBlock)(persistentalloc(cleanupBlockSize, tagAlign, &memstats.gcMiscSys)) + for { + next := (*cleanupBlock)(q.all.Load()) + b.alllink = next + if q.all.CompareAndSwap(unsafe.Pointer(next), unsafe.Pointer(b)) { + break + } + } + } + pp.cleanups = b + } + if full := b.enqueue(fn); full { + q.full.push(&b.lfnode) + pp.cleanups = nil + q.addWork(1) + } + pp.cleanupsQueued++ + releasem(mp) +} + +// dequeue pops a block of cleanups from the queue. Blocks until one is available +// and never returns nil. +func (q *cleanupQueue) dequeue() *cleanupBlock { + for { + if q.tryTakeWork() { + // Guaranteed to be non-nil. + return (*cleanupBlock)(q.full.pop()) + } + lock(&q.lock) + // Increment asleep first. We may have to undo this if we abort the sleep. + // We must update asleep first because the scheduler might not try to wake + // us up when work comes in between the last check of workUnits and when we + // go to sleep. (It may see asleep as 0.) By incrementing it here, we guarantee + // after this point that if new work comes in, someone will try to grab the + // lock and wake us. However, this also means that if we back out, we may cause + // someone to spuriously grab the lock and try to wake us up, only to fail. + // This should be very rare because the window here is incredibly small: the + // window between now and when we decrement q.asleep below. + q.asleep.Add(1) + + // Re-check workUnits under the lock and with asleep updated. If it's still zero, + // then no new work came in, and it's safe for us to go to sleep. If new work + // comes in after this point, then the scheduler will notice that we're sleeping + // and wake us up. + if q.workUnits.Load() > 0 { + // Undo the q.asleep update and try to take work again. + q.asleep.Add(-1) + unlock(&q.lock) + continue + } + q.sleeping.push(getg()) + goparkunlock(&q.lock, waitReasonCleanupWait, traceBlockSystemGoroutine, 1) + } +} + +// flush pushes all active cleanup blocks to the full list and wakes up cleanup +// goroutines to handle them. +// +// Must only be called at a point when we can guarantee that no more cleanups +// are being queued, such as after the final sweeper for the cycle is done +// but before the next mark phase. +func (q *cleanupQueue) flush() { + mp := acquirem() + flushed := 0 + emptied := 0 + missing := 0 + + // Coalesce the partially-filled blocks to present a more accurate picture of demand. + // We use the number of coalesced blocks to process as a signal for demand to create + // new cleanup goroutines. + var cb *cleanupBlock + for _, pp := range allp { + if pp == nil { + // This function is reachable via mallocgc in the + // middle of procresize, when allp has been resized, + // but the new Ps not allocated yet. + missing++ + continue + } + b := pp.cleanups + if b == nil { + missing++ + continue + } + pp.cleanups = nil + if cb == nil { + cb = b + continue + } + // N.B. After take, either cb is full, b is empty, or both. + cb.take(b) + if cb.full() { + q.full.push(&cb.lfnode) + flushed++ + cb = b + b = nil + } + if b != nil && b.empty() { + q.free.push(&b.lfnode) + emptied++ + } + } + if cb != nil { + q.full.push(&cb.lfnode) + flushed++ + } + if flushed != 0 { + q.addWork(flushed) + } + if flushed+emptied+missing != len(allp) { + throw("failed to correctly flush all P-owned cleanup blocks") + } + q.flushed.Store(true) + releasem(mp) +} + +// needsWake returns true if cleanup goroutines may need to be awoken or created to handle cleanup load. +func (q *cleanupQueue) needsWake() bool { + return q.workUnits.Load() > 0 && (q.asleep.Load() > 0 || q.ng.Load() < maxCleanupGs()) +} + +// wake wakes up one or more goroutines to process the cleanup queue. If there aren't +// enough sleeping goroutines to handle the demand, wake will arrange for new goroutines +// to be created. +func (q *cleanupQueue) wake() { + lock(&q.lock) + + // Figure out how many goroutines to wake, and how many extra goroutines to create. + // Wake one goroutine for each work unit. + var wake, extra uint32 + work := q.workUnits.Load() + asleep := uint64(q.asleep.Load()) + if work > asleep { + wake = uint32(asleep) + if work > uint64(math.MaxUint32) { + // Protect against overflow. + extra = math.MaxUint32 + } else { + extra = uint32(work - asleep) + } + } else { + wake = uint32(work) + extra = 0 + } + if extra != 0 { + // Signal that we should create new goroutines, one for each extra work unit, + // up to maxCleanupGs. + newg := min(extra, maxCleanupGs()-q.ng.Load()) + if newg > 0 { + q.needg.Add(int32(newg)) + } + } + if wake == 0 { + // Nothing to do. + unlock(&q.lock) + return + } + + // Take ownership of waking 'wake' goroutines. + // + // Nobody else will wake up these goroutines, so they're guaranteed + // to be sitting on q.sleeping, waiting for us to wake them. + q.asleep.Add(-int32(wake)) + + // Collect them and schedule them. + var list gList + for range wake { + list.push(q.sleeping.pop()) + } + unlock(&q.lock) + + injectglist(&list) + return +} + +func (q *cleanupQueue) needG() bool { + have := q.ng.Load() + if have >= maxCleanupGs() { + return false + } + if have == 0 { + // Make sure we have at least one. + return true + } + return q.needg.Load() > 0 +} + +func (q *cleanupQueue) createGs() { + lock(&q.lock) + have := q.ng.Load() + need := min(q.needg.Swap(0), maxCleanupGs()-have) + if have == 0 && need == 0 { + // Make sure we have at least one. + need = 1 + } + if need > 0 { + q.ng.Add(int32(need)) + } + unlock(&q.lock) + + for range need { + go runCleanups() + } +} + +func (q *cleanupQueue) beginRunningCleanups() { + // Update runningCleanups and running atomically with respect + // to goroutine profiles by disabling preemption. + mp := acquirem() + getg().runningCleanups.Store(true) + q.running.Add(1) + releasem(mp) +} + +func (q *cleanupQueue) endRunningCleanups() { + // Update runningCleanups and running atomically with respect + // to goroutine profiles by disabling preemption. + mp := acquirem() + getg().runningCleanups.Store(false) + q.running.Add(-1) + releasem(mp) +} + +func (q *cleanupQueue) readQueueStats() (queued, executed uint64) { + executed = q.executed.Load() + queued = q.queued + + // N.B. This is inconsistent, but that's intentional. It's just an estimate. + // Read this _after_ reading executed to decrease the chance that we observe + // an inconsistency in the statistics (executed > queued). + for _, pp := range allp { + queued += pp.cleanupsQueued + } + return +} + +func maxCleanupGs() uint32 { + // N.B. Left as a function to make changing the policy easier. + return uint32(max(gomaxprocs/4, 1)) +} + +// gcCleanups is the global cleanup queue. +var gcCleanups cleanupQueue + +// runCleanups is the entrypoint for all cleanup-running goroutines. +func runCleanups() { + for { + b := gcCleanups.dequeue() + if raceenabled { + // Approximately: adds a happens-before edge between the cleanup + // argument being mutated and the call to the cleanup below. + racefingo() + } + + gcCleanups.beginRunningCleanups() + for i := 0; i < int(b.n); i++ { + fn := b.cleanups[i] + + var racectx uintptr + if raceenabled { + // Enter a new race context so the race detector can catch + // potential races between cleanups, even if they execute on + // the same goroutine. + // + // Synchronize on fn. This would fail to find races on the + // closed-over values in fn (suppose fn is passed to multiple + // AddCleanup calls) if fn was not unique, but it is. Update + // the synchronization on fn if you intend to optimize it + // and store the cleanup function and cleanup argument on the + // queue directly. + racerelease(unsafe.Pointer(fn)) + racectx = raceEnterNewCtx() + raceacquire(unsafe.Pointer(fn)) + } + + // Execute the next cleanup. + cleanup := *(*func())(unsafe.Pointer(&fn)) + cleanup() + b.cleanups[i] = nil + + if raceenabled { + // Restore the old context. + raceRestoreCtx(racectx) + } + } + gcCleanups.endRunningCleanups() + gcCleanups.executed.Add(int64(b.n)) + + atomic.Store(&b.n, 0) // Synchronize with markroot. See comment in cleanupBlockHeader. + gcCleanups.free.push(&b.lfnode) + } +} + +// blockUntilEmpty blocks until either the cleanup queue is emptied +// and the cleanups have been executed, or the timeout is reached. +// Returns true if the cleanup queue was emptied. +// This is used by the sync and unique tests. +func (q *cleanupQueue) blockUntilEmpty(timeout int64) bool { + start := nanotime() + for nanotime()-start < timeout { + lock(&q.lock) + // The queue is empty when there's no work left to do *and* all the cleanup goroutines + // are asleep. If they're not asleep, they may be actively working on a block. + if q.flushed.Load() && q.full.empty() && uint32(q.sleeping.size) == q.ng.Load() { + unlock(&q.lock) + return true + } + unlock(&q.lock) + Gosched() + } + return false +} + +//go:linkname unique_runtime_blockUntilEmptyCleanupQueue unique.runtime_blockUntilEmptyCleanupQueue +func unique_runtime_blockUntilEmptyCleanupQueue(timeout int64) bool { + return gcCleanups.blockUntilEmpty(timeout) +} + +//go:linkname sync_test_runtime_blockUntilEmptyCleanupQueue sync_test.runtime_blockUntilEmptyCleanupQueue +func sync_test_runtime_blockUntilEmptyCleanupQueue(timeout int64) bool { + return gcCleanups.blockUntilEmpty(timeout) +} + +// raceEnterNewCtx creates a new racectx and switches the current +// goroutine to it. Returns the old racectx. +// +// Must be running on a user goroutine. nosplit to match other race +// instrumentation. +// +//go:nosplit +func raceEnterNewCtx() uintptr { + // We use the existing ctx as the spawn context, but gp.gopc + // as the spawn PC to make the error output a little nicer + // (pointing to AddCleanup, where the goroutines are created). + // + // We also need to carefully indicate to the race detector + // that the goroutine stack will only be accessed by the new + // race context, to avoid false positives on stack locations. + // We do this by marking the stack as free in the first context + // and then re-marking it as allocated in the second. Crucially, + // there must be (1) no race operations and (2) no stack changes + // in between. (1) is easy to avoid because we're in the runtime + // so there's no implicit race instrumentation. To avoid (2) we + // defensively become non-preemptible so the GC can't stop us, + // and rely on the fact that racemalloc, racefreem, and racectx + // are nosplit. + mp := acquirem() + gp := getg() + ctx := getg().racectx + racefree(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo) + getg().racectx = racectxstart(gp.gopc, ctx) + racemalloc(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo) + releasem(mp) + return ctx +} + +// raceRestoreCtx restores ctx on the goroutine. It is the inverse of +// raceenternewctx and must be called with its result. +// +// Must be running on a user goroutine. nosplit to match other race +// instrumentation. +// +//go:nosplit +func raceRestoreCtx(ctx uintptr) { + mp := acquirem() + gp := getg() + racefree(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo) + racectxend(getg().racectx) + racemalloc(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo) + getg().racectx = ctx + releasem(mp) +} diff --git a/src/runtime/mcleanup_test.go b/src/runtime/mcleanup_test.go new file mode 100644 index 00000000000000..22b9eccd20042e --- /dev/null +++ b/src/runtime/mcleanup_test.go @@ -0,0 +1,338 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "internal/runtime/atomic" + "runtime" + "sync" + "testing" + "time" + "unsafe" +) + +func TestCleanup(t *testing.T) { + ch := make(chan bool, 1) + done := make(chan bool, 1) + want := 97531 + go func() { + // allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + cleanup := func(x int) { + if x != want { + t.Errorf("cleanup %d, want %d", x, want) + } + ch <- true + } + runtime.AddCleanup(v, cleanup, 97531) + v = nil + done <- true + }() + <-done + runtime.GC() + <-ch +} + +func TestCleanupMultiple(t *testing.T) { + ch := make(chan bool, 3) + done := make(chan bool, 1) + want := 97531 + go func() { + // allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + cleanup := func(x int) { + if x != want { + t.Errorf("cleanup %d, want %d", x, want) + } + ch <- true + } + runtime.AddCleanup(v, cleanup, 97531) + runtime.AddCleanup(v, cleanup, 97531) + runtime.AddCleanup(v, cleanup, 97531) + v = nil + done <- true + }() + <-done + runtime.GC() + <-ch + <-ch + <-ch +} + +func TestCleanupZeroSizedStruct(t *testing.T) { + type Z struct{} + z := new(Z) + runtime.AddCleanup(z, func(s string) {}, "foo") +} + +func TestCleanupAfterFinalizer(t *testing.T) { + ch := make(chan int, 2) + done := make(chan bool, 1) + want := 97531 + go func() { + // allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + finalizer := func(x *int) { + ch <- 1 + } + cleanup := func(x int) { + if x != want { + t.Errorf("cleanup %d, want %d", x, want) + } + ch <- 2 + } + runtime.AddCleanup(v, cleanup, 97531) + runtime.SetFinalizer(v, finalizer) + v = nil + done <- true + }() + <-done + runtime.GC() + var result int + result = <-ch + if result != 1 { + t.Errorf("result %d, want 1", result) + } + runtime.GC() + result = <-ch + if result != 2 { + t.Errorf("result %d, want 2", result) + } +} + +func TestCleanupInteriorPointer(t *testing.T) { + ch := make(chan bool, 3) + done := make(chan bool, 1) + want := 97531 + go func() { + // Allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + p unsafe.Pointer + i int + a int + b int + c int + } + ts := new(T) + ts.a = 97531 + ts.b = 97531 + ts.c = 97531 + cleanup := func(x int) { + if x != want { + t.Errorf("cleanup %d, want %d", x, want) + } + ch <- true + } + runtime.AddCleanup(&ts.a, cleanup, 97531) + runtime.AddCleanup(&ts.b, cleanup, 97531) + runtime.AddCleanup(&ts.c, cleanup, 97531) + ts = nil + done <- true + }() + <-done + runtime.GC() + <-ch + <-ch + <-ch +} + +func TestCleanupStop(t *testing.T) { + done := make(chan bool, 1) + go func() { + // allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + cleanup := func(x int) { + t.Error("cleanup called, want no cleanup called") + } + c := runtime.AddCleanup(v, cleanup, 97531) + c.Stop() + v = nil + done <- true + }() + <-done + runtime.GC() +} + +func TestCleanupStopMultiple(t *testing.T) { + done := make(chan bool, 1) + go func() { + // allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + cleanup := func(x int) { + t.Error("cleanup called, want no cleanup called") + } + c := runtime.AddCleanup(v, cleanup, 97531) + c.Stop() + c.Stop() + c.Stop() + v = nil + done <- true + }() + <-done + runtime.GC() +} + +func TestCleanupStopinterleavedMultiple(t *testing.T) { + ch := make(chan bool, 3) + done := make(chan bool, 1) + go func() { + // allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + cleanup := func(x int) { + if x != 1 { + t.Error("cleanup called, want no cleanup called") + } + ch <- true + } + runtime.AddCleanup(v, cleanup, 1) + runtime.AddCleanup(v, cleanup, 2).Stop() + runtime.AddCleanup(v, cleanup, 1) + runtime.AddCleanup(v, cleanup, 2).Stop() + runtime.AddCleanup(v, cleanup, 1) + v = nil + done <- true + }() + <-done + runtime.GC() + <-ch + <-ch + <-ch +} + +func TestCleanupStopAfterCleanupRuns(t *testing.T) { + ch := make(chan bool, 1) + done := make(chan bool, 1) + var stop func() + go func() { + // Allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + cleanup := func(x int) { + ch <- true + } + cl := runtime.AddCleanup(v, cleanup, 97531) + v = nil + stop = cl.Stop + done <- true + }() + <-done + runtime.GC() + <-ch + stop() +} + +func TestCleanupPointerEqualsArg(t *testing.T) { + // See go.dev/issue/71316 + defer func() { + want := "runtime.AddCleanup: ptr is equal to arg, cleanup will never run" + if r := recover(); r == nil { + t.Error("want panic, test did not panic") + } else if r == want { + // do nothing + } else { + t.Errorf("wrong panic: want=%q, got=%q", want, r) + } + }() + + // allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + v := &new(T).v + *v = 97531 + runtime.AddCleanup(v, func(x *int) {}, v) + v = nil + runtime.GC() +} + +// Checks to make sure cleanups aren't lost when there are a lot of them. +func TestCleanupLost(t *testing.T) { + type T struct { + v int + p unsafe.Pointer + } + + cleanups := 10_000 + if testing.Short() { + cleanups = 100 + } + n := runtime.GOMAXPROCS(-1) + want := n * cleanups + var got atomic.Uint64 + var wg sync.WaitGroup + for i := range n { + wg.Add(1) + go func(i int) { + defer wg.Done() + + for range cleanups { + v := &new(T).v + *v = 97531 + runtime.AddCleanup(v, func(_ int) { + got.Add(1) + }, 97531) + } + }(i) + } + wg.Wait() + runtime.GC() + runtime.BlockUntilEmptyCleanupQueue(int64(10 * time.Second)) + if got := int(got.Load()); got != want { + t.Errorf("expected %d cleanups to be executed, got %d", got, want) + } +} diff --git a/src/runtime/mem.go b/src/runtime/mem.go index 22688d51d5e3fe..cd06ea323d8a7c 100644 --- a/src/runtime/mem.go +++ b/src/runtime/mem.go @@ -46,10 +46,18 @@ import "unsafe" // which prevents us from allocating more stack. // //go:nosplit -func sysAlloc(n uintptr, sysStat *sysMemStat) unsafe.Pointer { +func sysAlloc(n uintptr, sysStat *sysMemStat, vmaName string) unsafe.Pointer { sysStat.add(int64(n)) gcController.mappedReady.Add(int64(n)) - return sysAllocOS(n) + p := sysAllocOS(n, vmaName) + + // When using ASAN leak detection, we must tell ASAN about + // cases where we store pointers in mmapped memory. + if asanenabled { + lsanregisterrootregion(p, n) + } + + return p } // sysUnused transitions a memory region from Ready to Prepared. It notifies the @@ -111,6 +119,13 @@ func sysHugePageCollapse(v unsafe.Pointer, n uintptr) { // //go:nosplit func sysFree(v unsafe.Pointer, n uintptr, sysStat *sysMemStat) { + // When using ASAN leak detection, the memory being freed is + // known by the sanitizer. We need to unregister it so it's + // not accessed by it. + if asanenabled { + lsanunregisterrootregion(v, n) + } + sysStat.add(-int64(n)) gcController.mappedReady.Add(-int64(n)) sysFreeOS(v, n) @@ -142,15 +157,23 @@ func sysFault(v unsafe.Pointer, n uintptr) { // NOTE: sysReserve returns OS-aligned memory, but the heap allocator // may use larger alignment, so the caller must be careful to realign the // memory obtained by sysReserve. -func sysReserve(v unsafe.Pointer, n uintptr) unsafe.Pointer { - return sysReserveOS(v, n) +func sysReserve(v unsafe.Pointer, n uintptr, vmaName string) unsafe.Pointer { + p := sysReserveOS(v, n, vmaName) + + // When using ASAN leak detection, we must tell ASAN about + // cases where we store pointers in mmapped memory. + if asanenabled { + lsanregisterrootregion(p, n) + } + + return p } // sysMap transitions a memory region from Reserved to Prepared. It ensures the // memory region can be efficiently transitioned to Ready. // // sysStat must be non-nil. -func sysMap(v unsafe.Pointer, n uintptr, sysStat *sysMemStat) { +func sysMap(v unsafe.Pointer, n uintptr, sysStat *sysMemStat, vmaName string) { sysStat.add(int64(n)) - sysMapOS(v, n) + sysMapOS(v, n, vmaName) } diff --git a/src/runtime/mem_aix.go b/src/runtime/mem_aix.go index dff2756d971ab1..c5e4710dacfb0f 100644 --- a/src/runtime/mem_aix.go +++ b/src/runtime/mem_aix.go @@ -12,7 +12,7 @@ import ( // prevents us from allocating more stack. // //go:nosplit -func sysAllocOS(n uintptr) unsafe.Pointer { +func sysAllocOS(n uintptr, _ string) unsafe.Pointer { p, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0) if err != 0 { if err == _EACCES { @@ -56,7 +56,7 @@ func sysFaultOS(v unsafe.Pointer, n uintptr) { mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, -1, 0) } -func sysReserveOS(v unsafe.Pointer, n uintptr) unsafe.Pointer { +func sysReserveOS(v unsafe.Pointer, n uintptr, _ string) unsafe.Pointer { p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0) if err != 0 { return nil @@ -64,7 +64,7 @@ func sysReserveOS(v unsafe.Pointer, n uintptr) unsafe.Pointer { return p } -func sysMapOS(v unsafe.Pointer, n uintptr) { +func sysMapOS(v unsafe.Pointer, n uintptr, _ string) { // AIX does not allow mapping a range that is already mapped. // So, call mprotect to change permissions. // Note that sysMap is always called with a non-nil pointer diff --git a/src/runtime/mem_bsd.go b/src/runtime/mem_bsd.go index 78128aedf7b069..0c05b44c08f9c3 100644 --- a/src/runtime/mem_bsd.go +++ b/src/runtime/mem_bsd.go @@ -14,7 +14,7 @@ import ( // which prevents us from allocating more stack. // //go:nosplit -func sysAllocOS(n uintptr) unsafe.Pointer { +func sysAllocOS(n uintptr, _ string) unsafe.Pointer { v, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0) if err != 0 { return nil @@ -57,7 +57,7 @@ func sysFaultOS(v unsafe.Pointer, n uintptr) { // Indicates not to reserve swap space for the mapping. const _sunosMAP_NORESERVE = 0x40 -func sysReserveOS(v unsafe.Pointer, n uintptr) unsafe.Pointer { +func sysReserveOS(v unsafe.Pointer, n uintptr, _ string) unsafe.Pointer { flags := int32(_MAP_ANON | _MAP_PRIVATE) if GOOS == "solaris" || GOOS == "illumos" { // Be explicit that we don't want to reserve swap space @@ -75,7 +75,7 @@ func sysReserveOS(v unsafe.Pointer, n uintptr) unsafe.Pointer { const _sunosEAGAIN = 11 const _ENOMEM = 12 -func sysMapOS(v unsafe.Pointer, n uintptr) { +func sysMapOS(v unsafe.Pointer, n uintptr, _ string) { p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0) if err == _ENOMEM || ((GOOS == "solaris" || GOOS == "illumos") && err == _sunosEAGAIN) { throw("runtime: out of memory") diff --git a/src/runtime/mem_darwin.go b/src/runtime/mem_darwin.go index ae8487127cfdd7..9d4de516228bf5 100644 --- a/src/runtime/mem_darwin.go +++ b/src/runtime/mem_darwin.go @@ -12,7 +12,7 @@ import ( // which prevents us from allocating more stack. // //go:nosplit -func sysAllocOS(n uintptr) unsafe.Pointer { +func sysAllocOS(n uintptr, _ string) unsafe.Pointer { v, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0) if err != 0 { return nil @@ -54,7 +54,7 @@ func sysFaultOS(v unsafe.Pointer, n uintptr) { mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, -1, 0) } -func sysReserveOS(v unsafe.Pointer, n uintptr) unsafe.Pointer { +func sysReserveOS(v unsafe.Pointer, n uintptr, _ string) unsafe.Pointer { p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0) if err != 0 { return nil @@ -64,7 +64,7 @@ func sysReserveOS(v unsafe.Pointer, n uintptr) unsafe.Pointer { const _ENOMEM = 12 -func sysMapOS(v unsafe.Pointer, n uintptr) { +func sysMapOS(v unsafe.Pointer, n uintptr, _ string) { p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0) if err == _ENOMEM { throw("runtime: out of memory") diff --git a/src/runtime/mem_linux.go b/src/runtime/mem_linux.go index 9aaa57ac9e71d3..24e006debca94a 100644 --- a/src/runtime/mem_linux.go +++ b/src/runtime/mem_linux.go @@ -18,7 +18,7 @@ const ( // prevents us from allocating more stack. // //go:nosplit -func sysAllocOS(n uintptr) unsafe.Pointer { +func sysAllocOS(n uintptr, vmaName string) unsafe.Pointer { p, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0) if err != 0 { if err == _EACCES { @@ -31,6 +31,7 @@ func sysAllocOS(n uintptr) unsafe.Pointer { } return nil } + setVMAName(p, n, vmaName) return p } @@ -70,7 +71,10 @@ func sysUnusedOS(v unsafe.Pointer, n uintptr) { // Fall back on mmap if it's not supported. // _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE will unmap all the // pages in the old mapping, and remap the memory region. - mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0) + p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0) + if err == 0 && p != nil { + setVMAName(p, n, "unused") + } } if debug.harddecommit > 0 { @@ -78,6 +82,7 @@ func sysUnusedOS(v unsafe.Pointer, n uintptr) { if p != v || err != 0 { throw("runtime: cannot disable permissions in address space") } + setVMAName(p, n, "unused") } } @@ -90,6 +95,7 @@ func sysUsedOS(v unsafe.Pointer, n uintptr) { if p != v || err != 0 { throw("runtime: cannot remap pages in address space") } + setVMAName(p, n, "used") return } } @@ -154,15 +160,16 @@ func sysFaultOS(v unsafe.Pointer, n uintptr) { madvise(v, n, _MADV_DONTNEED) } -func sysReserveOS(v unsafe.Pointer, n uintptr) unsafe.Pointer { +func sysReserveOS(v unsafe.Pointer, n uintptr, vmaName string) unsafe.Pointer { p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0) if err != 0 { return nil } + setVMAName(p, n, vmaName) return p } -func sysMapOS(v unsafe.Pointer, n uintptr) { +func sysMapOS(v unsafe.Pointer, n uintptr, vmaName string) { p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0) if err == _ENOMEM { throw("runtime: out of memory") @@ -171,6 +178,7 @@ func sysMapOS(v unsafe.Pointer, n uintptr) { print("runtime: mmap(", v, ", ", n, ") returned ", p, ", ", err, "\n") throw("runtime: cannot map pages in arena address space") } + setVMAName(p, n, vmaName) // Disable huge pages if the GODEBUG for it is set. // diff --git a/src/runtime/mem_nonsbrk.go b/src/runtime/mem_nonsbrk.go new file mode 100644 index 00000000000000..41b7260eace4c6 --- /dev/null +++ b/src/runtime/mem_nonsbrk.go @@ -0,0 +1,15 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !plan9 && !wasm + +package runtime + +import "unsafe" + +const isSbrkPlatform = false + +func sysReserveAlignedSbrk(size, align uintptr) (unsafe.Pointer, uintptr) { + panic("unreachable") +} diff --git a/src/runtime/mem_sbrk.go b/src/runtime/mem_sbrk.go index dc0a764a2cac86..5284bbd0009865 100644 --- a/src/runtime/mem_sbrk.go +++ b/src/runtime/mem_sbrk.go @@ -8,10 +8,32 @@ package runtime import "unsafe" +const isSbrkPlatform = true + const memDebug = false -var bloc uintptr -var blocMax uintptr +// Memory management on sbrk systems (including the linear memory +// on Wasm). + +// bloc is the runtime's sense of the break, which can go up or +// down. blocMax is the system's break, also the high water mark +// of bloc. The runtime uses memory up to bloc. The memory +// between bloc and blocMax is allocated by the OS but not used +// by the runtime. +// +// When the runtime needs to grow the heap address range, it +// increases bloc. When it needs to grow beyond blocMax, it calls +// the system sbrk to allocate more memory (and therefore +// increase blocMax). +// +// When the runtime frees memory at the end of the address space, +// it decreases bloc, but does not reduces the system break (as +// the OS doesn't support it). When the runtime frees memory in +// the middle of the address space, the memory goes to a free +// list. + +var bloc uintptr // The runtime's sense of break. Can go up or down. +var blocMax uintptr // The break of the OS. Only increase. var memlock mutex type memHdr struct { @@ -26,7 +48,33 @@ type memHdrPtr uintptr func (p memHdrPtr) ptr() *memHdr { return (*memHdr)(unsafe.Pointer(p)) } func (p *memHdrPtr) set(x *memHdr) { *p = memHdrPtr(unsafe.Pointer(x)) } +// memAlloc allocates n bytes from the brk reservation, or if it's full, +// the system. +// +// memlock must be held. +// +// memAlloc must be called on the system stack, otherwise a stack growth +// could cause us to call back into it. Since memlock is held, that could +// lead to a self-deadlock. +// +//go:systemstack func memAlloc(n uintptr) unsafe.Pointer { + if p := memAllocNoGrow(n); p != nil { + return p + } + return sbrk(n) +} + +// memAllocNoGrow attempts to allocate n bytes from the existing brk. +// +// memlock must be held. +// +// memAlloc must be called on the system stack, otherwise a stack growth +// could cause us to call back into it. Since memlock is held, that could +// lead to a self-deadlock. +// +//go:systemstack +func memAllocNoGrow(n uintptr) unsafe.Pointer { n = memRound(n) var prevp *memHdr for p := memFreelist.ptr(); p != nil; p = p.next.ptr() { @@ -46,9 +94,18 @@ func memAlloc(n uintptr) unsafe.Pointer { } prevp = p } - return sbrk(n) + return nil } +// memFree makes [ap, ap+n) available for reallocation by memAlloc. +// +// memlock must be held. +// +// memAlloc must be called on the system stack, otherwise a stack growth +// could cause us to call back into it. Since memlock is held, that could +// lead to a self-deadlock. +// +//go:systemstack func memFree(ap unsafe.Pointer, n uintptr) { n = memRound(n) memclrNoHeapPointers(ap, n) @@ -93,6 +150,15 @@ func memFree(ap unsafe.Pointer, n uintptr) { } } +// memCheck checks invariants around free list management. +// +// memlock must be held. +// +// memAlloc must be called on the system stack, otherwise a stack growth +// could cause us to call back into it. Since memlock is held, that could +// lead to a self-deadlock. +// +//go:systemstack func memCheck() { if !memDebug { return @@ -128,27 +194,32 @@ func initBloc() { blocMax = bloc } -func sysAllocOS(n uintptr) unsafe.Pointer { - lock(&memlock) - p := memAlloc(n) - memCheck() - unlock(&memlock) - return p +func sysAllocOS(n uintptr, _ string) unsafe.Pointer { + var p uintptr + systemstack(func() { + lock(&memlock) + p = uintptr(memAlloc(n)) + memCheck() + unlock(&memlock) + }) + return unsafe.Pointer(p) } func sysFreeOS(v unsafe.Pointer, n uintptr) { - lock(&memlock) - if uintptr(v)+n == bloc { - // Address range being freed is at the end of memory, - // so record a new lower value for end of memory. - // Can't actually shrink address space because segment is shared. - memclrNoHeapPointers(v, n) - bloc -= n - } else { - memFree(v, n) - memCheck() - } - unlock(&memlock) + systemstack(func() { + lock(&memlock) + if uintptr(v)+n == bloc { + // Address range being freed is at the end of memory, + // so record a new lower value for end of memory. + // Can't actually shrink address space because segment is shared. + memclrNoHeapPointers(v, n) + bloc -= n + } else { + memFree(v, n) + memCheck() + } + unlock(&memlock) + }) } func sysUnusedOS(v unsafe.Pointer, n uintptr) { @@ -166,24 +237,62 @@ func sysNoHugePageOS(v unsafe.Pointer, n uintptr) { func sysHugePageCollapseOS(v unsafe.Pointer, n uintptr) { } -func sysMapOS(v unsafe.Pointer, n uintptr) { +func sysMapOS(v unsafe.Pointer, n uintptr, _ string) { } func sysFaultOS(v unsafe.Pointer, n uintptr) { } -func sysReserveOS(v unsafe.Pointer, n uintptr) unsafe.Pointer { - lock(&memlock) - var p unsafe.Pointer - if uintptr(v) == bloc { - // Address hint is the current end of memory, - // so try to extend the address space. - p = sbrk(n) - } - if p == nil && v == nil { - p = memAlloc(n) - memCheck() - } - unlock(&memlock) - return p +func sysReserveOS(v unsafe.Pointer, n uintptr, _ string) unsafe.Pointer { + var p uintptr + systemstack(func() { + lock(&memlock) + if uintptr(v) == bloc { + // Address hint is the current end of memory, + // so try to extend the address space. + p = uintptr(sbrk(n)) + } + if p == 0 && v == nil { + p = uintptr(memAlloc(n)) + memCheck() + } + unlock(&memlock) + }) + return unsafe.Pointer(p) +} + +func sysReserveAlignedSbrk(size, align uintptr) (unsafe.Pointer, uintptr) { + var p uintptr + systemstack(func() { + lock(&memlock) + if base := memAllocNoGrow(size + align); base != nil { + // We can satisfy the reservation from the free list. + // Trim off the unaligned parts. + start := alignUp(uintptr(base), align) + if startLen := start - uintptr(base); startLen > 0 { + memFree(base, startLen) + } + end := start + size + if endLen := (uintptr(base) + size + align) - end; endLen > 0 { + memFree(unsafe.Pointer(end), endLen) + } + memCheck() + unlock(&memlock) + p = start + return + } + + // Round up bloc to align, then allocate size. + p = alignUp(bloc, align) + r := sbrk(p + size - bloc) + if r == nil { + p, size = 0, 0 + } else if l := p - uintptr(r); l > 0 { + // Free the area we skipped over for alignment. + memFree(r, l) + memCheck() + } + unlock(&memlock) + }) + return unsafe.Pointer(p), size } diff --git a/src/runtime/mem_wasm.go b/src/runtime/mem_wasm.go index d9d32705bb6a06..76de88ac3cd4aa 100644 --- a/src/runtime/mem_wasm.go +++ b/src/runtime/mem_wasm.go @@ -7,14 +7,21 @@ package runtime import "unsafe" func sbrk(n uintptr) unsafe.Pointer { - grow := divRoundUp(n, physPageSize) - size := growMemory(int32(grow)) - if size < 0 { - return nil + bl := bloc + n = memRound(n) + if bl+n > blocMax { + grow := (bl + n - blocMax) / physPageSize + size := growMemory(int32(grow)) + if size < 0 { + return nil + } + resetMemoryDataView() + blocMax = bl + n } - resetMemoryDataView() - return unsafe.Pointer(uintptr(size) * physPageSize) + bloc += n + return unsafe.Pointer(bl) } // Implemented in src/runtime/sys_wasm.s func growMemory(pages int32) int32 +func currentMemory() int32 diff --git a/src/runtime/mem_windows.go b/src/runtime/mem_windows.go index 477d8988702c67..3db6fc2ba408fb 100644 --- a/src/runtime/mem_windows.go +++ b/src/runtime/mem_windows.go @@ -25,12 +25,12 @@ const ( // which prevents us from allocating more stack. // //go:nosplit -func sysAllocOS(n uintptr) unsafe.Pointer { - return unsafe.Pointer(stdcall4(_VirtualAlloc, 0, n, _MEM_COMMIT|_MEM_RESERVE, _PAGE_READWRITE)) +func sysAllocOS(n uintptr, _ string) unsafe.Pointer { + return unsafe.Pointer(stdcall(_VirtualAlloc, 0, n, _MEM_COMMIT|_MEM_RESERVE, _PAGE_READWRITE)) } func sysUnusedOS(v unsafe.Pointer, n uintptr) { - r := stdcall3(_VirtualFree, uintptr(v), n, _MEM_DECOMMIT) + r := stdcall(_VirtualFree, uintptr(v), n, _MEM_DECOMMIT) if r != 0 { return } @@ -46,7 +46,7 @@ func sysUnusedOS(v unsafe.Pointer, n uintptr) { // in the worst case, but that's fast enough. for n > 0 { small := n - for small >= 4096 && stdcall3(_VirtualFree, uintptr(v), small, _MEM_DECOMMIT) == 0 { + for small >= 4096 && stdcall(_VirtualFree, uintptr(v), small, _MEM_DECOMMIT) == 0 { small /= 2 small &^= 4096 - 1 } @@ -60,7 +60,7 @@ func sysUnusedOS(v unsafe.Pointer, n uintptr) { } func sysUsedOS(v unsafe.Pointer, n uintptr) { - p := stdcall4(_VirtualAlloc, uintptr(v), n, _MEM_COMMIT, _PAGE_READWRITE) + p := stdcall(_VirtualAlloc, uintptr(v), n, _MEM_COMMIT, _PAGE_READWRITE) if p == uintptr(v) { return } @@ -71,7 +71,7 @@ func sysUsedOS(v unsafe.Pointer, n uintptr) { k := n for k > 0 { small := k - for small >= 4096 && stdcall4(_VirtualAlloc, uintptr(v), small, _MEM_COMMIT, _PAGE_READWRITE) == 0 { + for small >= 4096 && stdcall(_VirtualAlloc, uintptr(v), small, _MEM_COMMIT, _PAGE_READWRITE) == 0 { small /= 2 small &^= 4096 - 1 } @@ -105,7 +105,7 @@ func sysHugePageCollapseOS(v unsafe.Pointer, n uintptr) { // //go:nosplit func sysFreeOS(v unsafe.Pointer, n uintptr) { - r := stdcall3(_VirtualFree, uintptr(v), 0, _MEM_RELEASE) + r := stdcall(_VirtualFree, uintptr(v), 0, _MEM_RELEASE) if r == 0 { print("runtime: VirtualFree of ", n, " bytes failed with errno=", getlasterror(), "\n") throw("runtime: failed to release pages") @@ -117,18 +117,18 @@ func sysFaultOS(v unsafe.Pointer, n uintptr) { sysUnusedOS(v, n) } -func sysReserveOS(v unsafe.Pointer, n uintptr) unsafe.Pointer { +func sysReserveOS(v unsafe.Pointer, n uintptr, _ string) unsafe.Pointer { // v is just a hint. // First try at v. // This will fail if any of [v, v+n) is already reserved. - v = unsafe.Pointer(stdcall4(_VirtualAlloc, uintptr(v), n, _MEM_RESERVE, _PAGE_READWRITE)) + v = unsafe.Pointer(stdcall(_VirtualAlloc, uintptr(v), n, _MEM_RESERVE, _PAGE_READWRITE)) if v != nil { return v } // Next let the kernel choose the address. - return unsafe.Pointer(stdcall4(_VirtualAlloc, 0, n, _MEM_RESERVE, _PAGE_READWRITE)) + return unsafe.Pointer(stdcall(_VirtualAlloc, 0, n, _MEM_RESERVE, _PAGE_READWRITE)) } -func sysMapOS(v unsafe.Pointer, n uintptr) { +func sysMapOS(v unsafe.Pointer, n uintptr, _ string) { } diff --git a/src/runtime/memclr_arm.s b/src/runtime/memclr_arm.s index f02d058ead8b15..f113a1aa2dd9ff 100644 --- a/src/runtime/memclr_arm.s +++ b/src/runtime/memclr_arm.s @@ -33,7 +33,6 @@ // See memclrNoHeapPointers Go doc for important implementation constraints. // func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) -// Also called from assembly in sys_windows_arm.s without g (but using Go stack convention). TEXT runtime·memclrNoHeapPointers(SB),NOSPLIT,$0-8 MOVW ptr+0(FP), TO MOVW n+4(FP), N diff --git a/src/runtime/memclr_arm64.s b/src/runtime/memclr_arm64.s index 1c35dfe0cf258b..3f8acdaeff670f 100644 --- a/src/runtime/memclr_arm64.s +++ b/src/runtime/memclr_arm64.s @@ -82,6 +82,7 @@ last16: last_end: RET + PCALIGN $16 no_zva: SUB $16, R0, R0 SUB $64, R1, R1 @@ -98,6 +99,7 @@ loop_64: BNE tail63 RET + PCALIGN $16 try_zva: // Try using the ZVA feature to zero entire cache lines // It is not meaningful to use ZVA if the block size is less than 64, @@ -124,6 +126,7 @@ try_zva: MOVW R5, block_size<>(SB) B no_zva + PCALIGN $16 init: MOVW $4, R9 ANDW $15, R3, R5 @@ -134,6 +137,7 @@ init: // Block size is less than 64. BNE no_zva + PCALIGN $16 zero_by_line: CMP R5, R1 // Not enough memory to reach alignment @@ -170,8 +174,9 @@ loop_zva_prolog: aligned: SUB R5, R1, R1 + PCALIGN $16 loop_zva: - WORD $0xd50b7420 // DC ZVA, R0 + DC ZVA, R0 ADD R5, R0, R0 SUBS R5, R1, R1 BHS loop_zva diff --git a/src/runtime/memclr_loong64.s b/src/runtime/memclr_loong64.s index 1d45e82d498d48..76d8fb56bfd899 100644 --- a/src/runtime/memclr_loong64.s +++ b/src/runtime/memclr_loong64.s @@ -5,36 +5,325 @@ #include "go_asm.h" #include "textflag.h" +// Register map +// +// R4: ptr +// R5: n +// R6: ptrend +// R7: tmp + +// Algorithm: +// +// 1. if lasx is enabled: +// THRESHOLD = 256, ALIGNMENTS = 32, LOOPBLOCKS = 256, +// else if lsx is enabled: +// THRESHOLD = 128, ALIGNMENTS = 16, LOOPBLOCKS = 128, +// else +// THRESHOLD = 64, ALIGNMENTS = 8, LOOPBLOCKS = 64, +// +// 2. when 'count <= THRESHOLD' bytes, memory alignment check is omitted. +// The handling is divided into distinct cases based on the size of count: +// a. clr_0, clr_1, clr_2, clr_3, clr_4, clr_5through7, clr_8, +// clr_9through16, clr_17through32, clr_33through64, +// b. lsx_clr_17through32, lsx_clr_33through64, lsx_clr_65through128, +// c. lasx_clr_17through32, lasx_clr_33through64, lsx_clr_65through128, +// lasx_clr_65through128, lasx_clr_129through256 +// +// 3. when 'count > THRESHOLD' bytes, memory alignment check is performed. Unaligned +// bytes are processed first (that is, ALIGNMENTS - (ptr & (ALIGNMENTS-1))), and then +// a LOOPBLOCKS-byte loop is executed to zero out memory. +// When the number of remaining bytes not cleared is n < LOOPBLOCKS bytes, a tail +// processing is performed, invoking the corresponding case based on the size of n. +// +// example: +// THRESHOLD = 64, ALIGNMENTS = 8, LOOPBLOCKS = 64 +// +// ptr newptr ptrend +// | |<----count after correction---->| +// |<-------------count before correction---------->| +// |<--8-(ptr&7)-->| |<---64 bytes--->| +// +------------------------------------------------+ +// | Head | Body | Tail | +// +---------------+---------------+----------------+ +// newptr = ptr - (ptr & 7) + 8 +// count = count - 8 + (ptr & 7) + // func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) TEXT runtime·memclrNoHeapPointers(SB),NOSPLIT,$0-16 + BEQ R5, clr_0 ADDV R4, R5, R6 +tail: + // <=64 bytes, clear directly, not check aligned + SGTU $2, R5, R7 + BNE R7, clr_1 + SGTU $3, R5, R7 + BNE R7, clr_2 + SGTU $4, R5, R7 + BNE R7, clr_3 + SGTU $5, R5, R7 + BNE R7, clr_4 + SGTU $8, R5, R7 + BNE R7, clr_5through7 + SGTU $9, R5, R7 + BNE R7, clr_8 + SGTU $17, R5, R7 + BNE R7, clr_9through16 - // if less than 8 bytes, do one byte at a time - SGTU $8, R5, R8 - BNE R8, out + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R7 + BNE R7, lasx_tail + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R7 + BNE R7, lsx_tail - // do one byte at a time until 8-aligned - AND $7, R4, R8 - BEQ R8, words - MOVB R0, (R4) - ADDV $1, R4 - JMP -4(PC) + SGTU $33, R5, R7 + BNE R7, clr_17through32 + SGTU $65, R5, R7 + BNE R7, clr_33through64 + JMP clr_large + +lasx_tail: + // X0 = 0 + XVXORV X0, X0, X0 + + SGTU $33, R5, R7 + BNE R7, lasx_clr_17through32 + SGTU $65, R5, R7 + BNE R7, lasx_clr_33through64 + SGTU $129, R5, R7 + BNE R7, lasx_clr_65through128 + SGTU $257, R5, R7 + BNE R7, lasx_clr_129through256 + JMP lasx_clr_large + +lsx_tail: + // V0 = 0 + VXORV V0, V0, V0 + + SGTU $33, R5, R7 + BNE R7, lsx_clr_17through32 + SGTU $65, R5, R7 + BNE R7, lsx_clr_33through64 + SGTU $129, R5, R7 + BNE R7, lsx_clr_65through128 + JMP lsx_clr_large + + // use simd 256 instructions to implement memclr + // n > 256 bytes, check 32-byte alignment +lasx_clr_large: + AND $31, R4, R7 + BEQ R7, lasx_clr_256loop + XVMOVQ X0, (R4) + SUBV R7, R4 + ADDV R7, R5 + SUBV $32, R5 // newn = n - (32 - (ptr & 31)) + ADDV $32, R4 // newptr = ptr + (32 - (ptr & 31)) + SGTU $257, R5, R7 + BNE R7, lasx_clr_129through256 +lasx_clr_256loop: + SUBV $256, R5 + SGTU $256, R5, R7 + XVMOVQ X0, 0(R4) + XVMOVQ X0, 32(R4) + XVMOVQ X0, 64(R4) + XVMOVQ X0, 96(R4) + XVMOVQ X0, 128(R4) + XVMOVQ X0, 160(R4) + XVMOVQ X0, 192(R4) + XVMOVQ X0, 224(R4) + ADDV $256, R4 + BEQ R7, lasx_clr_256loop + + // remaining_length is 0 + BEQ R5, clr_0 + + // 128 < remaining_length < 256 + SGTU $129, R5, R7 + BEQ R7, lasx_clr_129through256 + + // 64 < remaining_length <= 128 + SGTU $65, R5, R7 + BEQ R7, lasx_clr_65through128 + + // 32 < remaining_length <= 64 + SGTU $33, R5, R7 + BEQ R7, lasx_clr_33through64 -words: - // do 8 bytes at a time if there is room - ADDV $-7, R6, R5 + // 16 < remaining_length <= 32 + SGTU $17, R5, R7 + BEQ R7, lasx_clr_17through32 - PCALIGN $16 - SGTU R5, R4, R8 - BEQ R8, out + // 0 < remaining_length <= 16 + JMP tail + + // use simd 128 instructions to implement memclr + // n > 128 bytes, check 16-byte alignment +lsx_clr_large: + // check 16-byte alignment + AND $15, R4, R7 + BEQ R7, lsx_clr_128loop + VMOVQ V0, (R4) + SUBV R7, R4 + ADDV R7, R5 + SUBV $16, R5 // newn = n - (16 - (ptr & 15)) + ADDV $16, R4 // newptr = ptr + (16 - (ptr & 15)) + SGTU $129, R5, R7 + BNE R7, lsx_clr_65through128 +lsx_clr_128loop: + SUBV $128, R5 + SGTU $128, R5, R7 + VMOVQ V0, 0(R4) + VMOVQ V0, 16(R4) + VMOVQ V0, 32(R4) + VMOVQ V0, 48(R4) + VMOVQ V0, 64(R4) + VMOVQ V0, 80(R4) + VMOVQ V0, 96(R4) + VMOVQ V0, 112(R4) + ADDV $128, R4 + BEQ R7, lsx_clr_128loop + + // remaining_length is 0 + BEQ R5, clr_0 + + // 64 < remaining_length <= 128 + SGTU $65, R5, R7 + BEQ R7, lsx_clr_65through128 + + // 32 < remaining_length <= 64 + SGTU $33, R5, R7 + BEQ R7, lsx_clr_33through64 + + // 16 < remaining_length <= 32 + SGTU $17, R5, R7 + BEQ R7, lsx_clr_17through32 + + // 0 < remaining_length <= 16 + JMP tail + + // use general instructions to implement memclr + // n > 64 bytes, check 16-byte alignment +clr_large: + AND $7, R4, R7 + BEQ R7, clr_64loop + MOVV R0, (R4) + SUBV R7, R4 + ADDV R7, R5 + ADDV $8, R4 // newptr = ptr + (8 - (ptr & 7)) + SUBV $8, R5 // newn = n - (8 - (ptr & 7)) + MOVV $64, R7 + BLT R5, R7, clr_33through64 +clr_64loop: + SUBV $64, R5 + SGTU $64, R5, R7 MOVV R0, (R4) - ADDV $8, R4 - JMP -4(PC) + MOVV R0, 8(R4) + MOVV R0, 16(R4) + MOVV R0, 24(R4) + MOVV R0, 32(R4) + MOVV R0, 40(R4) + MOVV R0, 48(R4) + MOVV R0, 56(R4) + ADDV $64, R4 + BEQ R7, clr_64loop + + // remaining_length is 0 + BEQ R5, clr_0 + + // 32 < remaining_length < 64 + SGTU $33, R5, R7 + BEQ R7, clr_33through64 + + // 16 < remaining_length <= 32 + SGTU $17, R5, R7 + BEQ R7, clr_17through32 -out: - BEQ R4, R6, done + // 0 < remaining_length <= 16 + JMP tail + +clr_0: + RET +clr_1: MOVB R0, (R4) - ADDV $1, R4 - JMP -3(PC) -done: + RET +clr_2: + MOVH R0, (R4) + RET +clr_3: + MOVH R0, (R4) + MOVB R0, 2(R4) + RET +clr_4: + MOVW R0, (R4) + RET +clr_5through7: + MOVW R0, (R4) + MOVW R0, -4(R6) + RET +clr_8: + MOVV R0, (R4) + RET +clr_9through16: + MOVV R0, (R4) + MOVV R0, -8(R6) + RET +clr_17through32: + MOVV R0, (R4) + MOVV R0, 8(R4) + MOVV R0, -16(R6) + MOVV R0, -8(R6) + RET +clr_33through64: + MOVV R0, (R4) + MOVV R0, 8(R4) + MOVV R0, 16(R4) + MOVV R0, 24(R4) + MOVV R0, -32(R6) + MOVV R0, -24(R6) + MOVV R0, -16(R6) + MOVV R0, -8(R6) + RET + +lasx_clr_17through32: + VMOVQ V0, 0(R4) + VMOVQ V0, -16(R6) + RET +lasx_clr_33through64: + XVMOVQ X0, 0(R4) + XVMOVQ X0, -32(R6) + RET +lasx_clr_65through128: + XVMOVQ X0, 0(R4) + XVMOVQ X0, 32(R4) + XVMOVQ X0, -64(R6) + XVMOVQ X0, -32(R6) + RET +lasx_clr_129through256: + XVMOVQ X0, 0(R4) + XVMOVQ X0, 32(R4) + XVMOVQ X0, 64(R4) + XVMOVQ X0, 96(R4) + XVMOVQ X0, -128(R6) + XVMOVQ X0, -96(R6) + XVMOVQ X0, -64(R6) + XVMOVQ X0, -32(R6) + RET + +lsx_clr_17through32: + VMOVQ V0, 0(R4) + VMOVQ V0, -16(R6) + RET +lsx_clr_33through64: + VMOVQ V0, 0(R4) + VMOVQ V0, 16(R4) + VMOVQ V0, -32(R6) + VMOVQ V0, -16(R6) + RET +lsx_clr_65through128: + VMOVQ V0, 0(R4) + VMOVQ V0, 16(R4) + VMOVQ V0, 32(R4) + VMOVQ V0, 48(R4) + VMOVQ V0, -64(R6) + VMOVQ V0, -48(R6) + VMOVQ V0, -32(R6) + VMOVQ V0, -16(R6) RET diff --git a/src/runtime/memclr_mips64x.s b/src/runtime/memclr_mips64x.s index cf3a9c4ab4fb36..3df3728146a107 100644 --- a/src/runtime/memclr_mips64x.s +++ b/src/runtime/memclr_mips64x.s @@ -71,29 +71,93 @@ msa_large_loop: no_msa: // if less than 8 bytes, do one byte at a time SGTU $8, R2, R3 - BNE R3, out + BNE R3, check4 - // do one byte at a time until 8-aligned + // Check alignment AND $7, R1, R3 - BEQ R3, words + BEQ R3, aligned + + // Zero one byte at a time until we reach 8 byte alignment. + MOVV $8, R5 + SUBV R3, R5, R3 + SUBV R3, R2, R2 +align: + SUBV $1, R3 MOVB R0, (R1) ADDV $1, R1 - JMP -4(PC) + BNE R3, align -words: - // do 8 bytes at a time if there is room - ADDV $-7, R4, R2 +aligned: + SGTU $8, R2, R3 + BNE R3, check4 + SGTU $16, R2, R3 + BNE R3, zero8 + SGTU $32, R2, R3 + BNE R3, zero16 + SGTU $64, R2, R3 + BNE R3, zero32 +loop64: + MOVV R0, (R1) + MOVV R0, 8(R1) + MOVV R0, 16(R1) + MOVV R0, 24(R1) + MOVV R0, 32(R1) + MOVV R0, 40(R1) + MOVV R0, 48(R1) + MOVV R0, 56(R1) + ADDV $64, R1 + SUBV $64, R2 + SGTU $64, R2, R3 + BEQ R0, R3, loop64 + BEQ R2, done + +check32: + SGTU $32, R2, R3 + BNE R3, check16 +zero32: + MOVV R0, (R1) + MOVV R0, 8(R1) + MOVV R0, 16(R1) + MOVV R0, 24(R1) + ADDV $32, R1 + SUBV $32, R2 + BEQ R2, done + +check16: + SGTU $16, R2, R3 + BNE R3, check8 +zero16: + MOVV R0, (R1) + MOVV R0, 8(R1) + ADDV $16, R1 + SUBV $16, R2 + BEQ R2, done - SGTU R2, R1, R3 - BEQ R3, out +check8: + SGTU $8, R2, R3 + BNE R3, check4 +zero8: MOVV R0, (R1) ADDV $8, R1 - JMP -4(PC) + SUBV $8, R2 + BEQ R2, done -out: +check4: + SGTU $4, R2, R3 + BNE R3, loop1 +zero4: + MOVB R0, (R1) + MOVB R0, 1(R1) + MOVB R0, 2(R1) + MOVB R0, 3(R1) + ADDV $4, R1 + SUBV $4, R2 + +loop1: BEQ R1, R4, done MOVB R0, (R1) ADDV $1, R1 - JMP -3(PC) + JMP loop1 done: RET + diff --git a/src/runtime/memclr_ppc64x.s b/src/runtime/memclr_ppc64x.s index bc4b3fc2832b75..ffe40e12f66638 100644 --- a/src/runtime/memclr_ppc64x.s +++ b/src/runtime/memclr_ppc64x.s @@ -19,7 +19,7 @@ check: SRD $3, R4, R6 // R6: double words to clear CMP R6, $0, CR1 // CR1[EQ] set if no double words - BC 12, 6, nozerolarge // only single bytes + BEQ CR1, nozerolarge // only single bytes CMP R4, $512 BLT under512 // special case for < 512 ANDCC $127, R3, R8 // check for 128 alignment of address @@ -104,7 +104,7 @@ lt16gt8: #endif nozerolarge: ANDCC $7, R4, R5 // any remaining bytes - BC 4, 1, LR // ble lr + BLE CR0, LR // ble lr #ifdef GOPPC64_power10 XXLXOR VS32, VS32, VS32 // clear VS32 (V0) SLD $56, R5, R7 @@ -124,7 +124,7 @@ next2: ADD $-2, R5 next1: CMP R5, $0 - BC 12, 2, LR // beqlr + BEQ CR0, LR // beqlr MOVB R0, 0(R3) RET #endif diff --git a/src/runtime/memclr_s390x.s b/src/runtime/memclr_s390x.s index fa657ef66e6b95..392057565e8cb3 100644 --- a/src/runtime/memclr_s390x.s +++ b/src/runtime/memclr_s390x.s @@ -11,13 +11,13 @@ TEXT runtime·memclrNoHeapPointers(SB),NOSPLIT|NOFRAME,$0-16 MOVD ptr+0(FP), R4 MOVD n+8(FP), R5 + CMPBGE R5, $32, clearge32 + start: CMPBLE R5, $3, clear0to3 CMPBLE R5, $7, clear4to7 CMPBLE R5, $11, clear8to11 CMPBLE R5, $15, clear12to15 - CMP R5, $32 - BGE clearmt32 MOVD $0, 0(R4) MOVD $0, 8(R4) ADD $16, R4 @@ -102,23 +102,100 @@ clear15: MOVB $0, 14(R4) RET -clearmt32: +clearge32: + CMP R5, $4096 + BLT clear256Bto4KB + +// For size >= 4KB, XC is loop unrolled 16 times (4KB = 256B * 16) +clearge4KB: + XC $256, 0(R4), 0(R4) + XC $256, 256(R4), 256(R4) + XC $256, 512(R4), 512(R4) + XC $256, 768(R4), 768(R4) + XC $256, 1024(R4), 1024(R4) + XC $256, 1280(R4), 1280(R4) + XC $256, 1536(R4), 1536(R4) + XC $256, 1792(R4), 1792(R4) + XC $256, 2048(R4), 2048(R4) + XC $256, 2304(R4), 2304(R4) + XC $256, 2560(R4), 2560(R4) + XC $256, 2816(R4), 2816(R4) + XC $256, 3072(R4), 3072(R4) + XC $256, 3328(R4), 3328(R4) + XC $256, 3584(R4), 3584(R4) + XC $256, 3840(R4), 3840(R4) + ADD $4096, R4 + ADD $-4096, R5 + CMP R5, $4096 + BGE clearge4KB + +clear256Bto4KB: CMP R5, $256 - BLT clearlt256 + BLT clear32to255 XC $256, 0(R4), 0(R4) ADD $256, R4 ADD $-256, R5 - BR clearmt32 -clearlt256: + BR clear256Bto4KB + +clear32to255: CMPBEQ R5, $0, done - ADD $-1, R5 - EXRL $memclr_exrl_xc<>(SB), R5 -done: + CMPBLT R5, $32, start + CMPBEQ R5, $32, clear32 + CMPBLE R5, $64, clear33to64 + CMP R5, $128 + BLE clear65to128 + CMP R5, $255 + BLE clear129to255 + +clear32: + VZERO V1 + VST V1, 0(R4) + VST V1, 16(R4) RET -// DO NOT CALL - target for exrl (execute relative long) instruction. -TEXT memclr_exrl_xc<>(SB),NOSPLIT|NOFRAME,$0-0 - XC $1, 0(R4), 0(R4) - MOVD $0, 0(R0) +clear33to64: + VZERO V1 + VST V1, 0(R4) + VST V1, 16(R4) + ADD $-32, R5 + VST V1, 0(R4)(R5) + VST V1, 16(R4)(R5) + RET + +clear65to128: + VZERO V1 + VST V1, 0(R4) + VST V1, 16(R4) + VST V1, 32(R4) + VST V1, 48(R4) + ADD $-64, R5 + VST V1, 0(R4)(R5) + VST V1, 16(R4)(R5) + VST V1, 32(R4)(R5) + VST V1, 48(R4)(R5) + RET + +clear129to255: + VZERO V1 + VST V1, 0(R4) + VST V1, 16(R4) + VST V1, 32(R4) + VST V1, 48(R4) + VST V1, 64(R4) + VST V1, 80(R4) + VST V1, 96(R4) + VST V1, 112(R4) + ADD $-128, R5 + VST V1, 0(R4)(R5) + VST V1, 16(R4)(R5) + VST V1, 32(R4)(R5) + VST V1, 48(R4)(R5) + VST V1, 64(R4)(R5) + VST V1, 80(R4)(R5) + VST V1, 96(R4)(R5) + VST V1, 112(R4)(R5) + RET + +done: RET diff --git a/src/runtime/memmove_linux_amd64_test.go b/src/runtime/memmove_linux_amd64_test.go index 5f900623bee83d..c558811599cc2b 100644 --- a/src/runtime/memmove_linux_amd64_test.go +++ b/src/runtime/memmove_linux_amd64_test.go @@ -5,6 +5,7 @@ package runtime_test import ( + "internal/asan" "os" "syscall" "testing" @@ -14,6 +15,10 @@ import ( // TestMemmoveOverflow maps 3GB of memory and calls memmove on // the corresponding slice. func TestMemmoveOverflow(t *testing.T) { + if asan.Enabled { + t.Skip("appears to break asan and causes spurious failures") + } + t.Parallel() // Create a temporary file. tmp, err := os.CreateTemp("", "go-memmovetest") diff --git a/src/runtime/memmove_loong64.s b/src/runtime/memmove_loong64.s index a94cf999bc4af9..0d0af68214a14e 100644 --- a/src/runtime/memmove_loong64.s +++ b/src/runtime/memmove_loong64.s @@ -2,103 +2,537 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include "go_asm.h" #include "textflag.h" // See memmove Go doc for important implementation constraints. +// Register map +// +// to R4 +// from R5 +// n(aka count) R6 +// to-end R7 +// from-end R8 +// data R11-R18 +// tmp R9 + +// Algorithm: +// +// Memory alignment check is only performed for copy size greater +// than 64 bytes to minimize overhead. +// +// when copy size <= 64 bytes, jump to label tail, according to the +// copy size to select the appropriate case and copy directly. +// Based on the common memory access instructions of loong64, the +// currently implemented cases are: +// move_0, move_1, move_2, move_3, move_4, move_5through7, move_8, +// move_9through16, move_17through32, move_33through64 +// +// when copy size > 64 bytes, use the destination-aligned copying, +// adopt the following strategy to copy in 3 parts: +// 1. Head: do the memory alignment +// 2. Body: a 64-byte loop structure +// 3. Tail: processing of the remaining part (<= 64 bytes) +// +// forward: +// +// Dst NewDst Dstend +// | |<----count after correction---->| +// |<-------------count before correction---------->| +// |<--8-(Dst&7)-->| |<---64 bytes--->| +// +------------------------------------------------+ +// | Head | Body | Tail | +// +---------------+---------------+----------------+ +// NewDst = Dst - (Dst & 7) + 8 +// count = count - 8 + (Dst & 7) +// Src = Src - (Dst & 7) + 8 +// +// backward: +// +// Dst NewDstend Dstend +// |<-----count after correction------>| | +// |<------------count before correction--------------->| +// |<---64 bytes--->| |<---Dstend&7--->| +// +----------------------------------------------------+ +// | Tail | Body | Head | +// +----------------+------------------+----------------+ +// NewDstend = Dstend - (Dstend & 7) +// count = count - (Dstend & 7) +// Srcend = Srcend - (Dstend & 7) + // func memmove(to, from unsafe.Pointer, n uintptr) TEXT runtime·memmove(SB), NOSPLIT|NOFRAME, $0-24 - BNE R6, check - RET + BEQ R4, R5, move_0 + BEQ R6, move_0 -check: - SGTU R4, R5, R7 - BNE R7, backward + ADDV R4, R6, R7 // to-end pointer + ADDV R5, R6, R8 // from-end pointer - ADDV R4, R6, R9 // end pointer +// copy size <= 64 bytes, copy directly, not check aligned +tail: + // < 2 bytes + SGTU $2, R6, R9 + BNE R9, move_1 - // if the two pointers are not of same alignments, do byte copying - SUBVU R5, R4, R7 - AND $7, R7 - BNE R7, out + // < 3 bytes + SGTU $3, R6, R9 + BNE R9, move_2 - // if less than 8 bytes, do byte copying - SGTU $8, R6, R7 - BNE R7, out + // < 4 bytes + SGTU $4, R6, R9 + BNE R9, move_3 - // do one byte at a time until 8-aligned - AND $7, R4, R8 - BEQ R8, words - MOVB (R5), R7 - ADDV $1, R5 - MOVB R7, (R4) - ADDV $1, R4 - JMP -6(PC) + // < 5 bytes + SGTU $5, R6, R9 + BNE R9, move_4 -words: - // do 8 bytes at a time if there is room - ADDV $-7, R9, R6 // R6 is end pointer-7 + // >= 5 bytes and < 8 bytes + SGTU $8, R6, R9 + BNE R9, move_5through7 - PCALIGN $16 - SGTU R6, R4, R8 - BEQ R8, out - MOVV (R5), R7 - ADDV $8, R5 - MOVV R7, (R4) - ADDV $8, R4 - JMP -6(PC) + // < 9 bytes + SGTU $9, R6, R9 + BNE R9, move_8 + + // >= 9 bytes and < 17 bytes + SGTU $17, R6, R9 + BNE R9, move_9through16 + + // >= 17 bytes and < 33 bytes + SGTU $33, R6, R9 + BNE R9, move_17through32 + + // >= 33 bytes and < 65 bytes + SGTU $65, R6, R9 + BNE R9, move_33through64 + + // >= 65 bytes and < 256 bytes + SGTU $256, R6, R9 + BNE R9, move_large + + // >= 256 + JMP lasx_move_large + +move_0: + RET + +move_1: + MOVB (R5), R11 + MOVB R11, (R4) + RET +move_2: + MOVH (R5), R11 + MOVH R11, (R4) + RET +move_3: + MOVH (R5), R11 + MOVB -1(R8), R12 + MOVH R11, (R4) + MOVB R12, -1(R7) + RET +move_4: + MOVW (R5), R11 + MOVW R11, (R4) + RET +move_5through7: + MOVW (R5), R11 + MOVW -4(R8), R12 + MOVW R11, (R4) + MOVW R12, -4(R7) + RET +move_8: + MOVV (R5), R11 + MOVV R11, (R4) + RET +move_9through16: + MOVV (R5), R11 + MOVV -8(R8), R12 + MOVV R11, (R4) + MOVV R12, -8(R7) + RET +move_17through32: + MOVV (R5), R11 + MOVV 8(R5), R12 + MOVV -16(R8), R13 + MOVV -8(R8), R14 + MOVV R11, (R4) + MOVV R12, 8(R4) + MOVV R13, -16(R7) + MOVV R14, -8(R7) + RET +move_33through64: + MOVV (R5), R11 + MOVV 8(R5), R12 + MOVV 16(R5), R13 + MOVV 24(R5), R14 + MOVV -32(R8), R15 + MOVV -24(R8), R16 + MOVV -16(R8), R17 + MOVV -8(R8), R18 + MOVV R11, (R4) + MOVV R12, 8(R4) + MOVV R13, 16(R4) + MOVV R14, 24(R4) + MOVV R15, -32(R7) + MOVV R16, -24(R7) + MOVV R17, -16(R7) + MOVV R18, -8(R7) + RET -out: - BEQ R4, R9, done - MOVB (R5), R7 - ADDV $1, R5 - MOVB R7, (R4) - ADDV $1, R4 - JMP -5(PC) -done: +move_large: + // if (dst > src) && (dst < (src + count)) + // regarded as memory overlap + // jump to backward + // else + // jump to forward + BGEU R5, R4, forward + ADDV R5, R6, R10 + BLTU R4, R10, backward +forward: + AND $7, R4, R9 // dst & 7 + BEQ R9, forward_move_64loop +forward_unaligned: + MOVV $8, R10 + SUBV R9, R10 // head = 8 - (dst & 7) + MOVV (R5), R11 + SUBV R10, R6 // newcount = count - (8 - (dst & 7)) + ADDV R10, R5 // newsrc = src + (8 - (dst & 7)) + MOVV (R5), R12 + MOVV R11, (R4) + ADDV R10, R4 // newdst = dst + (8 - (dst & 7)) + MOVV R12, (R4) + SUBV $8, R6 + ADDV $8, R4 + ADDV $8, R5 + SGTU $65, R6, R9 + BNE R9, move_33through64 +forward_move_64loop: + SUBV $64, R6 + SGTU $64, R6, R9 + MOVV (R5), R11 + MOVV 8(R5), R12 + MOVV 16(R5), R13 + MOVV 24(R5), R14 + MOVV 32(R5), R15 + MOVV 40(R5), R16 + MOVV 48(R5), R17 + MOVV 56(R5), R18 + MOVV R11, (R4) + MOVV R12, 8(R4) + MOVV R13, 16(R4) + MOVV R14, 24(R4) + MOVV R15, 32(R4) + MOVV R16, 40(R4) + MOVV R17, 48(R4) + MOVV R18, 56(R4) + ADDV $64, R5 + ADDV $64, R4 + BEQ R9, forward_move_64loop + // 0 < remaining_length < 64 + BNE R6, tail RET +// The backward copy algorithm is the same as the forward +// copy, except for the direction. backward: - ADDV R6, R5 // from-end pointer - ADDV R4, R6, R9 // to-end pointer - - // if the two pointers are not of same alignments, do byte copying - SUBVU R9, R5, R7 - AND $7, R7 - BNE R7, out1 - - // if less than 8 bytes, do byte copying - SGTU $8, R6, R7 - BNE R7, out1 - - // do one byte at a time until 8-aligned - AND $7, R9, R8 - BEQ R8, words1 - ADDV $-1, R5 - MOVB (R5), R7 - ADDV $-1, R9 - MOVB R7, (R9) - JMP -6(PC) - -words1: - // do 8 bytes at a time if there is room - ADDV $7, R4, R6 // R6 is start pointer+7 - - PCALIGN $16 - SGTU R9, R6, R8 - BEQ R8, out1 - ADDV $-8, R5 - MOVV (R5), R7 - ADDV $-8, R9 - MOVV R7, (R9) - JMP -6(PC) - -out1: - BEQ R4, R9, done1 - ADDV $-1, R5 - MOVB (R5), R7 - ADDV $-1, R9 - MOVB R7, (R9) - JMP -5(PC) -done1: + AND $7, R7, R9 // dstend & 7 + BEQ R9, backward_move_64loop +backward_unaligned: + MOVV -8(R8), R11 + SUBV R9, R6 // newcount = count - (dstend & 7) + SUBV R9, R8 // newsrcend = srcend - (dstend & 7) + MOVV -8(R8), R12 + MOVV R11, -8(R7) + SUBV R9, R7 // newdstend = dstend - (dstend & 7) + MOVV R12, -8(R7) + SUBV $8, R6 + SUBV $8, R7 + SUBV $8, R8 + SGTU $65, R6, R9 + BNE R9, move_33through64 +backward_move_64loop: + SUBV $64, R6 + SGTU $64, R6, R9 + MOVV -8(R8), R11 + MOVV -16(R8), R12 + MOVV -24(R8), R13 + MOVV -32(R8), R14 + MOVV -40(R8), R15 + MOVV -48(R8), R16 + MOVV -56(R8), R17 + MOVV -64(R8), R18 + MOVV R11, -8(R7) + MOVV R12, -16(R7) + MOVV R13, -24(R7) + MOVV R14, -32(R7) + MOVV R15, -40(R7) + MOVV R16, -48(R7) + MOVV R17, -56(R7) + MOVV R18, -64(R7) + SUBV $64, R7 + SUBV $64, R8 + BEQ R9, backward_move_64loop + // 0 < remaining_length < 64 + BNE R6, tail + RET + +// use simd 128 instructions to implement memmove +// n >= 256 bytes, check 16-byte alignment +lsx_move_large: + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R9 + BEQ R9, move_large + + // if (dst > src) && (dst < (src + count)) + // regarded as memory overlap + // jump to lsx_backward + // else + // jump to lsx_forward + BGEU R5, R4, lsx_forward + ADDV R5, R6, R10 + BLTU R4, R10, lsx_backward +lsx_forward: + AND $15, R4, R9 // dst & 15 + BEQ R9, lsx_forward_move_128 +lsx_forward_unaligned: + MOVV $16, R10 + SUBV R9, R10 // head = 16 - (dst & 15) + VMOVQ (R5), V0 + SUBV R10, R6 // newcount = count - (16 - (dst & 15)) + ADDV R10, R5 // newsrc = src + (16 - (dst & 15)) + VMOVQ (R5), V1 + VMOVQ V0, (R4) + ADDV R10, R4 // newdst = dst + (16 - (dst & 15)) + VMOVQ V1, (R4) + SUBV $16, R6 + ADDV $16, R4 + ADDV $16, R5 +lsx_forward_move_128: + SGTU $128, R6, R9 + BNE R9, lsx_forward_move_32 +lsx_forward_move_128loop: + SUBV $128, R6 + SGTU $128, R6, R9 + VMOVQ 0(R5), V0 + VMOVQ 16(R5), V1 + VMOVQ 32(R5), V2 + VMOVQ 48(R5), V3 + VMOVQ 64(R5), V4 + VMOVQ 80(R5), V5 + VMOVQ 96(R5), V6 + VMOVQ 112(R5), V7 + VMOVQ V0, 0(R4) + VMOVQ V1, 16(R4) + VMOVQ V2, 32(R4) + VMOVQ V3, 48(R4) + VMOVQ V4, 64(R4) + VMOVQ V5, 80(R4) + VMOVQ V6, 96(R4) + VMOVQ V7, 112(R4) + ADDV $128, R5 + ADDV $128, R4 + BEQ R9, lsx_forward_move_128loop +lsx_forward_move_32: + SGTU $32, R6, R9 + BNE R9, lsx_forward_move_tail +lsx_forward_move_32loop: + SUBV $32, R6 + SGTU $32, R6, R9 + VMOVQ 0(R5), V0 + VMOVQ 16(R5), V1 + VMOVQ V0, 0(R4) + VMOVQ V1, 16(R4) + ADDV $32, R5 + ADDV $32, R4 + BEQ R9, lsx_forward_move_32loop +lsx_forward_move_tail: + // 0 < remaining_length < 64 + BNE R6, tail + RET + +lsx_backward: + AND $15, R7, R9 // dstend & 15 + BEQ R9, lsx_backward_move_128 +lsx_backward_unaligned: + VMOVQ -16(R8), V0 + SUBV R9, R6 // newcount = count - (dstend & 15) + SUBV R9, R8 // newsrcend = srcend - (dstend & 15) + VMOVQ -16(R8), V1 + VMOVQ V0, -16(R7) + SUBV R9, R7 // newdstend = dstend - (dstend & 15) + VMOVQ V1, -16(R7) + SUBV $16, R6 + SUBV $16, R7 + SUBV $16, R8 +lsx_backward_move_128: + SGTU $128, R6, R9 + BNE R9, lsx_backward_move_32 +lsx_backward_move_128loop: + SUBV $128, R6 + SGTU $128, R6, R9 + VMOVQ -16(R8), V0 + VMOVQ -32(R8), V1 + VMOVQ -48(R8), V2 + VMOVQ -64(R8), V3 + VMOVQ -80(R8), V4 + VMOVQ -96(R8), V5 + VMOVQ -112(R8), V6 + VMOVQ -128(R8), V7 + VMOVQ V0, -16(R7) + VMOVQ V1, -32(R7) + VMOVQ V2, -48(R7) + VMOVQ V3, -64(R7) + VMOVQ V4, -80(R7) + VMOVQ V5, -96(R7) + VMOVQ V6, -112(R7) + VMOVQ V7, -128(R7) + SUBV $128, R8 + SUBV $128, R7 + BEQ R9, lsx_backward_move_128loop +lsx_backward_move_32: + SGTU $32, R6, R9 + BNE R9, lsx_backward_move_tail +lsx_backward_move_32loop: + SUBV $32, R6 + SGTU $32, R6, R9 + VMOVQ -16(R8), V0 + VMOVQ -32(R8), V1 + VMOVQ V0, -16(R7) + VMOVQ V1, -32(R7) + SUBV $32, R8 + SUBV $32, R7 + BEQ R9, lsx_backward_move_32loop +lsx_backward_move_tail: + // 0 < remaining_length < 64 + BNE R6, tail + RET + +// use simd 256 instructions to implement memmove +// n >= 256 bytes, check 32-byte alignment +lasx_move_large: + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R9 + BEQ R9, lsx_move_large + + // if (dst > src) && (dst < (src + count)) + // regarded as memory overlap + // jump to lasx_backward + // else + // jump to lasx_forward + BGEU R5, R4, lasx_forward + ADDV R5, R6, R10 + BLTU R4, R10, lasx_backward +lasx_forward: + AND $31, R4, R9 // dst & 31 + BEQ R9, lasx_forward_move_256 +lasx_forward_unaligned: + MOVV $32, R10 + SUBV R9, R10 // head = 32 - (dst & 31) + XVMOVQ (R5), X0 + SUBV R10, R6 // newcount = count - (32 - (dst & 31)) + ADDV R10, R5 // newsrc = src + (32 - (dst & 31)) + XVMOVQ (R5), X1 + XVMOVQ X0, (R4) + ADDV R10, R4 // newdst = dst + (32 - (dst & 31)) + XVMOVQ X1, (R4) + SUBV $32, R6 + ADDV $32, R4 + ADDV $32, R5 +lasx_forward_move_256: + SGTU $256, R6, R9 + BNE R9, lasx_forward_move_64 +lasx_forward_move_256loop: + SUBV $256, R6 + SGTU $256, R6, R9 + XVMOVQ 0(R5), X0 + XVMOVQ 32(R5), X1 + XVMOVQ 64(R5), X2 + XVMOVQ 96(R5), X3 + XVMOVQ 128(R5), X4 + XVMOVQ 160(R5), X5 + XVMOVQ 192(R5), X6 + XVMOVQ 224(R5), X7 + XVMOVQ X0, 0(R4) + XVMOVQ X1, 32(R4) + XVMOVQ X2, 64(R4) + XVMOVQ X3, 96(R4) + XVMOVQ X4, 128(R4) + XVMOVQ X5, 160(R4) + XVMOVQ X6, 192(R4) + XVMOVQ X7, 224(R4) + ADDV $256, R5 + ADDV $256, R4 + BEQ R9, lasx_forward_move_256loop +lasx_forward_move_64: + SGTU $64, R6, R9 + BNE R9, lasx_forward_move_tail +lasx_forward_move_64loop: + SUBV $64, R6 + SGTU $64, R6, R9 + XVMOVQ (R5), X0 + XVMOVQ 32(R5), X1 + XVMOVQ X0, (R4) + XVMOVQ X1, 32(R4) + ADDV $64, R5 + ADDV $64, R4 + BEQ R9, lasx_forward_move_64loop +lasx_forward_move_tail: + // 0 < remaining_length < 64 + BNE R6, tail + RET + +lasx_backward: + AND $31, R7, R9 // dstend & 31 + BEQ R9, lasx_backward_move_256 +lasx_backward_unaligned: + XVMOVQ -32(R8), X0 + SUBV R9, R6 // newcount = count - (dstend & 31) + SUBV R9, R8 // newsrcend = srcend - (dstend & 31) + XVMOVQ -32(R8), X1 + XVMOVQ X0, -32(R7) + SUBV R9, R7 // newdstend = dstend - (dstend & 31) + XVMOVQ X1, -32(R7) + SUBV $32, R6 + SUBV $32, R7 + SUBV $32, R8 +lasx_backward_move_256: + SGTU $256, R6, R9 + BNE R9, lasx_backward_move_64 +lasx_backward_move_256loop: + SUBV $256, R6 + SGTU $256, R6, R9 + XVMOVQ -32(R8), X0 + XVMOVQ -64(R8), X1 + XVMOVQ -96(R8), X2 + XVMOVQ -128(R8), X3 + XVMOVQ -160(R8), X4 + XVMOVQ -192(R8), X5 + XVMOVQ -224(R8), X6 + XVMOVQ -256(R8), X7 + XVMOVQ X0, -32(R7) + XVMOVQ X1, -64(R7) + XVMOVQ X2, -96(R7) + XVMOVQ X3, -128(R7) + XVMOVQ X4, -160(R7) + XVMOVQ X5, -192(R7) + XVMOVQ X6, -224(R7) + XVMOVQ X7, -256(R7) + SUBV $256, R8 + SUBV $256, R7 + BEQ R9, lasx_backward_move_256loop +lasx_backward_move_64: + SGTU $64, R6, R9 + BNE R9, lasx_backward_move_tail +lasx_backward_move_64loop: + SUBV $64, R6 + SGTU $64, R6, R9 + XVMOVQ -32(R8), X0 + XVMOVQ -64(R8), X1 + XVMOVQ X0, -32(R7) + XVMOVQ X1, -64(R7) + SUBV $64, R8 + SUBV $64, R7 + BEQ R9, lasx_backward_move_64loop +lasx_backward_move_tail: + // 0 < remaining_length < 64 + BNE R6, tail RET diff --git a/src/runtime/memmove_ppc64x.s b/src/runtime/memmove_ppc64x.s index 18b9c850f240f3..9892028d50463d 100644 --- a/src/runtime/memmove_ppc64x.s +++ b/src/runtime/memmove_ppc64x.s @@ -60,11 +60,11 @@ mcopy: SUB SRC, TGT, TMP // dest - src CMPU TMP, LEN, CR2 // < len? - BC 12, 8, backward // BLT CR2 backward + BLT CR2, backward // Copying forward if no overlap. - BC 12, 6, checkbytes // BEQ CR1, checkbytes + BEQ CR1, checkbytes SRDCC $3, DWORDS, OCTWORDS // 64 byte chunks? MOVD $16, IDX16 BEQ lt64gt8 // < 64 bytes @@ -132,7 +132,7 @@ lt16: // Move 8 bytes if possible MOVD TMP, 0(TGT) ADD $8, TGT checkbytes: - BC 12, 14, LR // BEQ lr + BEQ CR3, LR #ifdef GOPPC64_power10 SLD $56, BYTES, TMP LXVL SRC, TMP, V0 @@ -157,7 +157,7 @@ lt4: // Move halfword if possible ADD $2, TGT lt2: // Move last byte if 1 left CMP BYTES, $1 - BC 12, 0, LR // ble lr + BLT CR0, LR MOVBZ 0(SRC), TMP MOVBZ TMP, 0(TGT) RET @@ -182,7 +182,7 @@ backwardtailloop: BDNZ backwardtailloop nobackwardtail: - BC 4, 5, LR // blelr cr1, return if DWORDS == 0 + BLE CR1, LR // return if DWORDS == 0 SRDCC $2,DWORDS,QWORDS // Compute number of 32B blocks and compare to 0 BNE backward32setup // If QWORDS != 0, start the 32B copy loop. @@ -190,16 +190,16 @@ backward24: // DWORDS is a value between 1-3. CMP DWORDS, $2 - MOVD -8(SRC), TMP - MOVD TMP, -8(TGT) - BC 12, 0, LR // bltlr, return if DWORDS == 1 + MOVD -8(SRC), TMP + MOVD TMP, -8(TGT) + BLT CR0, LR // return if DWORDS == 1 - MOVD -16(SRC), TMP - MOVD TMP, -16(TGT) - BC 12, 2, LR // beqlr, return if DWORDS == 2 + MOVD -16(SRC), TMP + MOVD TMP, -16(TGT) + BEQ CR0, LR // return if DWORDS == 2 - MOVD -24(SRC), TMP - MOVD TMP, -24(TGT) + MOVD -24(SRC), TMP + MOVD TMP, -24(TGT) RET backward32setup: @@ -216,5 +216,5 @@ backward32loop: STXVD2X VS32, (R0)(TGT) // store 16x2 bytes STXVD2X VS33, (IDX16)(TGT) BDNZ backward32loop - BC 12, 2, LR // beqlr, return if DWORDS == 0 + BEQ CR0, LR // return if DWORDS == 0 BR backward24 diff --git a/src/runtime/memmove_test.go b/src/runtime/memmove_test.go index 6550c759d63736..6065a845539c53 100644 --- a/src/runtime/memmove_test.go +++ b/src/runtime/memmove_test.go @@ -8,6 +8,8 @@ import ( "crypto/rand" "encoding/binary" "fmt" + "internal/asan" + "internal/msan" "internal/race" "internal/testenv" . "runtime" @@ -102,8 +104,8 @@ func TestMemmoveLarge0x180000(t *testing.T) { } t.Parallel() - if race.Enabled { - t.Skip("skipping large memmove test under race detector") + if race.Enabled || asan.Enabled || msan.Enabled { + t.Skip("skipping large memmove test under sanitizers") } testSize(t, 0x180000) } @@ -114,8 +116,8 @@ func TestMemmoveOverlapLarge0x120000(t *testing.T) { } t.Parallel() - if race.Enabled { - t.Skip("skipping large memmove test under race detector") + if race.Enabled || asan.Enabled || msan.Enabled { + t.Skip("skipping large memmove test under sanitizers") } testOverlap(t, 0x120000) } @@ -219,8 +221,6 @@ func TestMemmoveAtomicity(t *testing.T) { for _, backward := range []bool{true, false} { for _, n := range []int{3, 4, 5, 6, 7, 8, 9, 10, 15, 25, 49} { - n := n - // test copying [N]*int. sz := uintptr(n * PtrSize) name := fmt.Sprint(sz) @@ -292,6 +292,7 @@ func BenchmarkMemmove(b *testing.B) { benchmarkSizes(b, bufSizes, func(b *testing.B, n int) { x := make([]byte, n) y := make([]byte, n) + b.ResetTimer() for i := 0; i < b.N; i++ { copy(x, y) } @@ -301,6 +302,7 @@ func BenchmarkMemmove(b *testing.B) { func BenchmarkMemmoveOverlap(b *testing.B) { benchmarkSizes(b, bufSizesOverlap, func(b *testing.B, n int) { x := make([]byte, n+16) + b.ResetTimer() for i := 0; i < b.N; i++ { copy(x[16:n+16], x[:n]) } @@ -311,6 +313,7 @@ func BenchmarkMemmoveUnalignedDst(b *testing.B) { benchmarkSizes(b, bufSizes, func(b *testing.B, n int) { x := make([]byte, n+1) y := make([]byte, n) + b.ResetTimer() for i := 0; i < b.N; i++ { copy(x[1:], y) } @@ -320,6 +323,7 @@ func BenchmarkMemmoveUnalignedDst(b *testing.B) { func BenchmarkMemmoveUnalignedDstOverlap(b *testing.B) { benchmarkSizes(b, bufSizesOverlap, func(b *testing.B, n int) { x := make([]byte, n+16) + b.ResetTimer() for i := 0; i < b.N; i++ { copy(x[16:n+16], x[1:n+1]) } @@ -330,6 +334,7 @@ func BenchmarkMemmoveUnalignedSrc(b *testing.B) { benchmarkSizes(b, bufSizes, func(b *testing.B, n int) { x := make([]byte, n) y := make([]byte, n+1) + b.ResetTimer() for i := 0; i < b.N; i++ { copy(x, y[1:]) } @@ -362,6 +367,7 @@ func BenchmarkMemmoveUnalignedSrcDst(b *testing.B) { func BenchmarkMemmoveUnalignedSrcOverlap(b *testing.B) { benchmarkSizes(b, bufSizesOverlap, func(b *testing.B, n int) { x := make([]byte, n+1) + b.ResetTimer() for i := 0; i < b.N; i++ { copy(x[1:n+1], x[:n]) } @@ -450,6 +456,7 @@ func BenchmarkMemclrUnaligned(b *testing.B) { func BenchmarkGoMemclr(b *testing.B) { benchmarkSizes(b, []int{5, 16, 64, 256}, func(b *testing.B, n int) { x := make([]byte, n) + b.ResetTimer() for i := 0; i < b.N; i++ { clear(x) } @@ -511,6 +518,42 @@ func BenchmarkMemclrRange(b *testing.B) { } } +func BenchmarkClearFat3(b *testing.B) { + p := new([3]byte) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = [3]byte{} + } +} + +func BenchmarkClearFat4(b *testing.B) { + p := new([4 / 4]uint32) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = [4 / 4]uint32{} + } +} + +func BenchmarkClearFat5(b *testing.B) { + p := new([5]byte) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = [5]byte{} + } +} + +func BenchmarkClearFat6(b *testing.B) { + p := new([6]byte) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = [6]byte{} + } +} + func BenchmarkClearFat7(b *testing.B) { p := new([7]byte) Escape(p) @@ -529,6 +572,24 @@ func BenchmarkClearFat8(b *testing.B) { } } +func BenchmarkClearFat9(b *testing.B) { + p := new([9]byte) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = [9]byte{} + } +} + +func BenchmarkClearFat10(b *testing.B) { + p := new([10]byte) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = [10]byte{} + } +} + func BenchmarkClearFat11(b *testing.B) { p := new([11]byte) Escape(p) @@ -583,6 +644,24 @@ func BenchmarkClearFat16(b *testing.B) { } } +func BenchmarkClearFat18(b *testing.B) { + p := new([18]byte) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = [18]byte{} + } +} + +func BenchmarkClearFat20(b *testing.B) { + p := new([20 / 4]uint32) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = [20 / 4]uint32{} + } +} + func BenchmarkClearFat24(b *testing.B) { p := new([24 / 4]uint32) Escape(p) @@ -700,6 +779,46 @@ func BenchmarkClearFat1040(b *testing.B) { } } +func BenchmarkCopyFat3(b *testing.B) { + var x [3]byte + p := new([3]byte) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = x + } +} + +func BenchmarkCopyFat4(b *testing.B) { + var x [4 / 4]uint32 + p := new([4 / 4]uint32) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = x + } +} + +func BenchmarkCopyFat5(b *testing.B) { + var x [5]byte + p := new([5]byte) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = x + } +} + +func BenchmarkCopyFat6(b *testing.B) { + var x [6]byte + p := new([6]byte) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = x + } +} + func BenchmarkCopyFat7(b *testing.B) { var x [7]byte p := new([7]byte) @@ -720,6 +839,26 @@ func BenchmarkCopyFat8(b *testing.B) { } } +func BenchmarkCopyFat9(b *testing.B) { + var x [9]byte + p := new([9]byte) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = x + } +} + +func BenchmarkCopyFat10(b *testing.B) { + var x [10]byte + p := new([10]byte) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = x + } +} + func BenchmarkCopyFat11(b *testing.B) { var x [11]byte p := new([11]byte) @@ -780,6 +919,26 @@ func BenchmarkCopyFat16(b *testing.B) { } } +func BenchmarkCopyFat18(b *testing.B) { + var x [18]byte + p := new([18]byte) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = x + } +} + +func BenchmarkCopyFat20(b *testing.B) { + var x [20 / 4]uint32 + p := new([20 / 4]uint32) + Escape(p) + b.ResetTimer() + for i := 0; i < b.N; i++ { + *p = x + } +} + func BenchmarkCopyFat24(b *testing.B) { var x [24 / 4]uint32 p := new([24 / 4]uint32) @@ -1116,3 +1275,102 @@ func BenchmarkMemclrKnownSize512KiB(b *testing.B) { memclrSink = x[:] } + +func BenchmarkMemmoveKnownSize112(b *testing.B) { + type T struct { + x [112]int8 + } + p := &T{} + q := &T{} + + b.SetBytes(int64(unsafe.Sizeof(T{}))) + for i := 0; i < b.N; i++ { + *p = *q + } + + memclrSink = p.x[:] +} +func BenchmarkMemmoveKnownSize128(b *testing.B) { + type T struct { + x [128]int8 + } + p := &T{} + q := &T{} + + b.SetBytes(int64(unsafe.Sizeof(T{}))) + for i := 0; i < b.N; i++ { + *p = *q + } + + memclrSink = p.x[:] +} +func BenchmarkMemmoveKnownSize192(b *testing.B) { + type T struct { + x [192]int8 + } + p := &T{} + q := &T{} + + b.SetBytes(int64(unsafe.Sizeof(T{}))) + for i := 0; i < b.N; i++ { + *p = *q + } + + memclrSink = p.x[:] +} +func BenchmarkMemmoveKnownSize248(b *testing.B) { + type T struct { + x [248]int8 + } + p := &T{} + q := &T{} + + b.SetBytes(int64(unsafe.Sizeof(T{}))) + for i := 0; i < b.N; i++ { + *p = *q + } + + memclrSink = p.x[:] +} +func BenchmarkMemmoveKnownSize256(b *testing.B) { + type T struct { + x [256]int8 + } + p := &T{} + q := &T{} + + b.SetBytes(int64(unsafe.Sizeof(T{}))) + for i := 0; i < b.N; i++ { + *p = *q + } + + memclrSink = p.x[:] +} +func BenchmarkMemmoveKnownSize512(b *testing.B) { + type T struct { + x [512]int8 + } + p := &T{} + q := &T{} + + b.SetBytes(int64(unsafe.Sizeof(T{}))) + for i := 0; i < b.N; i++ { + *p = *q + } + + memclrSink = p.x[:] +} +func BenchmarkMemmoveKnownSize1024(b *testing.B) { + type T struct { + x [1024]int8 + } + p := &T{} + q := &T{} + + b.SetBytes(int64(unsafe.Sizeof(T{}))) + for i := 0; i < b.N; i++ { + *p = *q + } + + memclrSink = p.x[:] +} diff --git a/src/runtime/metrics.go b/src/runtime/metrics.go index 417f1071bb7007..6ee2c285493473 100644 --- a/src/runtime/metrics.go +++ b/src/runtime/metrics.go @@ -8,6 +8,8 @@ package runtime import ( "internal/godebugs" + "internal/runtime/atomic" + "internal/runtime/gc" "unsafe" ) @@ -62,12 +64,12 @@ func initMetrics() { return } - sizeClassBuckets = make([]float64, _NumSizeClasses, _NumSizeClasses+1) + sizeClassBuckets = make([]float64, gc.NumSizeClasses, gc.NumSizeClasses+1) // Skip size class 0 which is a stand-in for large objects, but large // objects are tracked separately (and they actually get placed in // the last bucket, not the first). sizeClassBuckets[0] = 1 // The smallest allocation is 1 byte in size. - for i := 1; i < _NumSizeClasses; i++ { + for i := 1; i < gc.NumSizeClasses; i++ { // Size classes have an inclusive upper-bound // and exclusive lower bound (e.g. 48-byte size class is // (32, 48]) whereas we want and inclusive lower-bound @@ -79,7 +81,7 @@ func initMetrics() { // value up to 2^53 and size classes are relatively small // (nowhere near 2^48 even) so this will give us exact // boundaries. - sizeClassBuckets[i] = float64(class_to_size[i] + 1) + sizeClassBuckets[i] = float64(gc.SizeClassToSize[i] + 1) } sizeClassBuckets = append(sizeClassBuckets, float64Inf()) @@ -168,6 +170,20 @@ func initMetrics() { out.scalar = float64bits(nsToSec(in.cpuStats.UserTime)) }, }, + "/gc/cleanups/executed:cleanups": { + deps: makeStatDepSet(finalStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.finalStats.cleanupsExecuted + }, + }, + "/gc/cleanups/queued:cleanups": { + deps: makeStatDepSet(finalStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.finalStats.cleanupsQueued + }, + }, "/gc/cycles/automatic:gc-cycles": { deps: makeStatDepSet(sysStatsDep), compute: func(in *statAggregate, out *metricValue) { @@ -189,6 +205,20 @@ func initMetrics() { out.scalar = in.sysStats.gcCyclesDone }, }, + "/gc/finalizers/executed:finalizers": { + deps: makeStatDepSet(finalStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.finalStats.finalizersExecuted + }, + }, + "/gc/finalizers/queued:finalizers": { + deps: makeStatDepSet(finalStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.finalStats.finalizersQueued + }, + }, "/gc/scan/globals:bytes": { deps: makeStatDepSet(gcStatsDep), compute: func(in *statAggregate, out *metricValue) { @@ -332,8 +362,7 @@ func initMetrics() { compute: func(in *statAggregate, out *metricValue) { out.kind = metricKindUint64 out.scalar = uint64(in.heapStats.committed - in.heapStats.inHeap - - in.heapStats.inStacks - in.heapStats.inWorkBufs - - in.heapStats.inPtrScalarBits) + in.heapStats.inStacks - in.heapStats.inWorkBufs) }, }, "/memory/classes/heap/objects:bytes": { @@ -396,7 +425,7 @@ func initMetrics() { deps: makeStatDepSet(heapStatsDep, sysStatsDep), compute: func(in *statAggregate, out *metricValue) { out.kind = metricKindUint64 - out.scalar = uint64(in.heapStats.inWorkBufs+in.heapStats.inPtrScalarBits) + in.sysStats.gcMiscSys + out.scalar = uint64(in.heapStats.inWorkBufs) + in.sysStats.gcMiscSys }, }, "/memory/classes/os-stacks:bytes": { @@ -437,9 +466,45 @@ func initMetrics() { }, }, "/sched/goroutines:goroutines": { - compute: func(_ *statAggregate, out *metricValue) { + deps: makeStatDepSet(schedStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.schedStats.gTotal + }, + }, + "/sched/goroutines/not-in-go:goroutines": { + deps: makeStatDepSet(schedStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.schedStats.gNonGo + }, + }, + "/sched/goroutines/running:goroutines": { + deps: makeStatDepSet(schedStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.schedStats.gRunning + }, + }, + "/sched/goroutines/runnable:goroutines": { + deps: makeStatDepSet(schedStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.schedStats.gRunnable + }, + }, + "/sched/goroutines/waiting:goroutines": { + deps: makeStatDepSet(schedStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.schedStats.gWaiting + }, + }, + "/sched/goroutines-created:goroutines": { + deps: makeStatDepSet(schedStatsDep), + compute: func(in *statAggregate, out *metricValue) { out.kind = metricKindUint64 - out.scalar = uint64(gcount()) + out.scalar = in.schedStats.gCreated }, }, "/sched/latencies:seconds": { @@ -467,6 +532,13 @@ func initMetrics() { sched.stwTotalTimeOther.write(out) }, }, + "/sched/threads/total:threads": { + deps: makeStatDepSet(schedStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.schedStats.threads + }, + }, "/sync/mutex/wait/total:seconds": { compute: func(_ *statAggregate, out *metricValue) { out.kind = metricKindFloat64 @@ -514,10 +586,12 @@ func godebug_registerMetric(name string, read func() uint64) { type statDep uint const ( - heapStatsDep statDep = iota // corresponds to heapStatsAggregate - sysStatsDep // corresponds to sysStatsAggregate - cpuStatsDep // corresponds to cpuStatsAggregate - gcStatsDep // corresponds to gcStatsAggregate + heapStatsDep statDep = iota // corresponds to heapStatsAggregate + sysStatsDep // corresponds to sysStatsAggregate + cpuStatsDep // corresponds to cpuStatsAggregate + gcStatsDep // corresponds to gcStatsAggregate + finalStatsDep // corresponds to finalStatsAggregate + schedStatsDep // corresponds to schedStatsAggregate numStatsDeps ) @@ -615,8 +689,8 @@ func (a *heapStatsAggregate) compute() { nf := a.smallFreeCount[i] a.totalAllocs += na a.totalFrees += nf - a.totalAllocated += na * uint64(class_to_size[i]) - a.totalFreed += nf * uint64(class_to_size[i]) + a.totalAllocated += na * uint64(gc.SizeClassToSize[i]) + a.totalFreed += nf * uint64(gc.SizeClassToSize[i]) } a.inObjects = a.totalAllocated - a.totalFreed a.numObjects = a.totalAllocs - a.totalFrees @@ -696,6 +770,103 @@ func (a *gcStatsAggregate) compute() { a.totalScan = a.heapScan + a.stackScan + a.globalsScan } +// finalStatsAggregate represents various finalizer/cleanup stats obtained +// from the runtime acquired together to avoid skew and inconsistencies. +type finalStatsAggregate struct { + finalizersQueued uint64 + finalizersExecuted uint64 + cleanupsQueued uint64 + cleanupsExecuted uint64 +} + +// compute populates the finalStatsAggregate with values from the runtime. +func (a *finalStatsAggregate) compute() { + a.finalizersQueued, a.finalizersExecuted = finReadQueueStats() + a.cleanupsQueued, a.cleanupsExecuted = gcCleanups.readQueueStats() +} + +// schedStatsAggregate contains stats about the scheduler, including +// an approximate count of goroutines in each state. +type schedStatsAggregate struct { + gTotal uint64 + gRunning uint64 + gRunnable uint64 + gNonGo uint64 + gWaiting uint64 + gCreated uint64 + threads uint64 +} + +// compute populates the schedStatsAggregate with values from the runtime. +func (a *schedStatsAggregate) compute() { + // Lock the scheduler so the global run queue can't change and + // the number of Ps can't change. This doesn't prevent the + // local run queues from changing, so the results are still + // approximate. + lock(&sched.lock) + + // The total count of threads owned by Go is the number of Ms + // minus extra Ms on the list or in use. + a.threads = uint64(mcount()) - uint64(extraMInUse.Load()) - uint64(extraMLength.Load()) + + // Collect running/runnable from per-P run queues. + a.gCreated += sched.goroutinesCreated.Load() + for _, p := range allp { + if p == nil || p.status == _Pdead { + break + } + a.gCreated += p.goroutinesCreated + switch p.status { + case _Prunning: + a.gRunning++ + case _Psyscall: + a.gNonGo++ + case _Pgcstop: + // The world is stopping or stopped. + // This is fine. The results will be + // slightly odd since nothing else + // is running, but it will be accurate. + } + + for { + h := atomic.Load(&p.runqhead) + t := atomic.Load(&p.runqtail) + next := atomic.Loaduintptr((*uintptr)(&p.runnext)) + runnable := int32(t - h) + if atomic.Load(&p.runqhead) != h || runnable < 0 { + continue + } + if next != 0 { + runnable++ + } + a.gRunnable += uint64(runnable) + break + } + } + + // Global run queue. + a.gRunnable += uint64(sched.runq.size) + + // Account for Gs that are in _Gsyscall without a P in _Psyscall. + nGsyscallNoP := sched.nGsyscallNoP.Load() + + // nGsyscallNoP can go negative during temporary races. + if nGsyscallNoP >= 0 { + a.gNonGo += uint64(nGsyscallNoP) + } + + // Compute the number of blocked goroutines. We have to + // include system goroutines in this count because we included + // them above. + a.gTotal = uint64(gcount(true)) + a.gWaiting = a.gTotal - (a.gRunning + a.gRunnable + a.gNonGo) + if a.gWaiting < 0 { + a.gWaiting = 0 + } + + unlock(&sched.lock) +} + // nsToSec takes a duration in nanoseconds and converts it to seconds as // a float64. func nsToSec(ns int64) float64 { @@ -708,11 +879,13 @@ func nsToSec(ns int64) float64 { // as a set of these aggregates that it has populated. The aggregates // are populated lazily by its ensure method. type statAggregate struct { - ensured statDepSet - heapStats heapStatsAggregate - sysStats sysStatsAggregate - cpuStats cpuStatsAggregate - gcStats gcStatsAggregate + ensured statDepSet + heapStats heapStatsAggregate + sysStats sysStatsAggregate + cpuStats cpuStatsAggregate + gcStats gcStatsAggregate + finalStats finalStatsAggregate + schedStats schedStatsAggregate } // ensure populates statistics aggregates determined by deps if they @@ -735,6 +908,10 @@ func (a *statAggregate) ensure(deps *statDepSet) { a.cpuStats.compute() case gcStatsDep: a.gcStats.compute() + case finalStatsDep: + a.finalStats.compute() + case schedStatsDep: + a.schedStats.compute() } } a.ensured = a.ensured.union(missing) diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go index 19a7dbf07a6a0b..b9a6ab5fea6083 100644 --- a/src/runtime/metrics/description.go +++ b/src/runtime/metrics/description.go @@ -126,7 +126,7 @@ var allDesc = []Description{ { Name: "/cpu/classes/scavenge/assist:cpu-seconds", Description: "Estimated total CPU time spent returning unused memory to the " + - "underlying platform in response eagerly in response to memory pressure. " + + "underlying platform in response eagerly to memory pressure. " + "This metric is an overestimate, and not directly comparable to " + "system CPU time measurements. Compare only with other /cpu/classes " + "metrics.", @@ -174,6 +174,22 @@ var allDesc = []Description{ Kind: KindFloat64, Cumulative: true, }, + { + Name: "/gc/cleanups/executed:cleanups", + Description: "Approximate total count of cleanup functions (created by runtime.AddCleanup) " + + "executed by the runtime. Subtract /gc/cleanups/queued:cleanups to approximate " + + "cleanup queue length. Useful for detecting slow cleanups holding up the queue.", + Kind: KindUint64, + Cumulative: true, + }, + { + Name: "/gc/cleanups/queued:cleanups", + Description: "Approximate total count of cleanup functions (created by runtime.AddCleanup) " + + "queued by the runtime for execution. Subtract from /gc/cleanups/executed:cleanups " + + "to approximate cleanup queue length. Useful for detecting slow cleanups holding up the queue.", + Kind: KindUint64, + Cumulative: true, + }, { Name: "/gc/cycles/automatic:gc-cycles", Description: "Count of completed GC cycles generated by the Go runtime.", @@ -192,6 +208,23 @@ var allDesc = []Description{ Kind: KindUint64, Cumulative: true, }, + { + Name: "/gc/finalizers/executed:finalizers", + Description: "Total count of finalizer functions (created by runtime.SetFinalizer) " + + "executed by the runtime. Subtract /gc/finalizers/queued:finalizers to approximate " + + "finalizer queue length. Useful for detecting finalizers overwhelming the queue, " + + "either by being too slow, or by there being too many of them.", + Kind: KindUint64, + Cumulative: true, + }, + { + Name: "/gc/finalizers/queued:finalizers", + Description: "Total count of finalizer functions (created by runtime.SetFinalizer) and " + + "queued by the runtime for execution. Subtract from /gc/finalizers/executed:finalizers " + + "to approximate finalizer queue length. Useful for detecting slow finalizers holding up the queue.", + Kind: KindUint64, + Cumulative: true, + }, { Name: "/gc/gogc:percent", Description: "Heap size target percentage configured by the user, otherwise 100. This " + @@ -404,6 +437,32 @@ var allDesc = []Description{ Description: "The current runtime.GOMAXPROCS setting, or the number of operating system threads that can execute user-level Go code simultaneously.", Kind: KindUint64, }, + { + Name: "/sched/goroutines-created:goroutines", + Description: "Count of goroutines created since program start.", + Cumulative: true, + Kind: KindUint64, + }, + { + Name: "/sched/goroutines/not-in-go:goroutines", + Description: "Approximate count of goroutines running or blocked in a system call or cgo call. Not guaranteed to add up to /sched/goroutines:goroutines with other goroutine metrics.", + Kind: KindUint64, + }, + { + Name: "/sched/goroutines/runnable:goroutines", + Description: "Approximate count of goroutines ready to execute, but not executing. Not guaranteed to add up to /sched/goroutines:goroutines with other goroutine metrics.", + Kind: KindUint64, + }, + { + Name: "/sched/goroutines/running:goroutines", + Description: "Approximate count of goroutines executing. Always less than or equal to /sched/gomaxprocs:threads. Not guaranteed to add up to /sched/goroutines:goroutines with other goroutine metrics.", + Kind: KindUint64, + }, + { + Name: "/sched/goroutines/waiting:goroutines", + Description: "Approximate count of goroutines waiting on a resource (I/O or sync primitives). Not guaranteed to add up to /sched/goroutines:goroutines with other goroutine metrics.", + Kind: KindUint64, + }, { Name: "/sched/goroutines:goroutines", Description: "Count of live goroutines.", @@ -439,6 +498,11 @@ var allDesc = []Description{ Kind: KindFloat64Histogram, Cumulative: true, }, + { + Name: "/sched/threads/total:threads", + Description: "The current count of live threads that are owned by the Go runtime.", + Kind: KindUint64, + }, { Name: "/sync/mutex/wait/total:seconds", Description: "Approximate cumulative time goroutines have spent blocked on a sync.Mutex, sync.RWMutex, or runtime-internal lock. This metric is useful for identifying global changes in lock contention. Collect a mutex or block profile using the runtime/pprof package for more detailed contention data.", diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go index da3d956d480beb..05646132ce4e69 100644 --- a/src/runtime/metrics/doc.go +++ b/src/runtime/metrics/doc.go @@ -104,10 +104,10 @@ Below is the full list of supported metrics, ordered lexicographically. /cpu/classes/scavenge/assist:cpu-seconds Estimated total CPU time spent returning unused memory to the - underlying platform in response eagerly in response to memory - pressure. This metric is an overestimate, and not directly - comparable to system CPU time measurements. Compare only with - other /cpu/classes metrics. + underlying platform in response eagerly to memory pressure. This + metric is an overestimate, and not directly comparable to system + CPU time measurements. Compare only with other /cpu/classes + metrics. /cpu/classes/scavenge/background:cpu-seconds Estimated total CPU time spent performing background tasks to @@ -137,6 +137,19 @@ Below is the full list of supported metrics, ordered lexicographically. to system CPU time measurements. Compare only with other /cpu/classes metrics. + /gc/cleanups/executed:cleanups + Approximate total count of cleanup functions (created + by runtime.AddCleanup) executed by the runtime. Subtract + /gc/cleanups/queued:cleanups to approximate cleanup queue + length. Useful for detecting slow cleanups holding up the queue. + + /gc/cleanups/queued:cleanups + Approximate total count of cleanup functions (created by + runtime.AddCleanup) queued by the runtime for execution. + Subtract from /gc/cleanups/executed:cleanups to approximate + cleanup queue length. Useful for detecting slow cleanups holding + up the queue. + /gc/cycles/automatic:gc-cycles Count of completed GC cycles generated by the Go runtime. @@ -146,6 +159,20 @@ Below is the full list of supported metrics, ordered lexicographically. /gc/cycles/total:gc-cycles Count of all completed GC cycles. + /gc/finalizers/executed:finalizers + Total count of finalizer functions (created by + runtime.SetFinalizer) executed by the runtime. Subtract + /gc/finalizers/queued:finalizers to approximate finalizer queue + length. Useful for detecting finalizers overwhelming the queue, + either by being too slow, or by there being too many of them. + + /gc/finalizers/queued:finalizers + Total count of finalizer functions (created by + runtime.SetFinalizer) and queued by the runtime for execution. + Subtract from /gc/finalizers/executed:finalizers to approximate + finalizer queue length. Useful for detecting slow finalizers + holding up the queue. + /gc/gogc:percent Heap size target percentage configured by the user, otherwise 100. This value is set by the GOGC environment variable, and the @@ -230,10 +257,25 @@ Below is the full list of supported metrics, ordered lexicographically. /gc/stack/starting-size:bytes The stack size of new goroutines. + /godebug/non-default-behavior/allowmultiplevcs:events + The number of non-default behaviors executed by the cmd/go + package due to a non-default GODEBUG=allowmultiplevcs=... + setting. + /godebug/non-default-behavior/asynctimerchan:events The number of non-default behaviors executed by the time package due to a non-default GODEBUG=asynctimerchan=... setting. + /godebug/non-default-behavior/containermaxprocs:events + The number of non-default behaviors executed by the runtime + package due to a non-default GODEBUG=containermaxprocs=... + setting. + + /godebug/non-default-behavior/embedfollowsymlinks:events + The number of non-default behaviors executed by the cmd/go + package due to a non-default GODEBUG=embedfollowsymlinks=... + setting. + /godebug/non-default-behavior/execerrdot:events The number of non-default behaviors executed by the os/exec package due to a non-default GODEBUG=execerrdot=... setting. @@ -250,6 +292,11 @@ Below is the full list of supported metrics, ordered lexicographically. The number of non-default behaviors executed by the cmd/go package due to a non-default GODEBUG=gocacheverify=... setting. + /godebug/non-default-behavior/gotestjsonbuildtext:events + The number of non-default behaviors executed by the cmd/go + package due to a non-default GODEBUG=gotestjsonbuildtext=... + setting. + /godebug/non-default-behavior/gotypesalias:events The number of non-default behaviors executed by the go/types package due to a non-default GODEBUG=gotypesalias=... setting. @@ -262,6 +309,11 @@ Below is the full list of supported metrics, ordered lexicographically. The number of non-default behaviors executed by the net/http package due to a non-default GODEBUG=http2server=... setting. + /godebug/non-default-behavior/httpcookiemaxnum:events + The number of non-default behaviors executed by the net/http + package due to a non-default GODEBUG=httpcookiemaxnum=... + setting. + /godebug/non-default-behavior/httplaxcontentlength:events The number of non-default behaviors executed by the net/http package due to a non-default GODEBUG=httplaxcontentlength=... @@ -306,6 +358,14 @@ Below is the full list of supported metrics, ordered lexicographically. The number of non-default behaviors executed by the math/rand package due to a non-default GODEBUG=randautoseed=... setting. + /godebug/non-default-behavior/randseednop:events + The number of non-default behaviors executed by the math/rand + package due to a non-default GODEBUG=randseednop=... setting. + + /godebug/non-default-behavior/rsa1024min:events + The number of non-default behaviors executed by the crypto/rsa + package due to a non-default GODEBUG=rsa1024min=... setting. + /godebug/non-default-behavior/tarinsecurepath:events The number of non-default behaviors executed by the archive/tar package due to a non-default GODEBUG=tarinsecurepath=... @@ -327,10 +387,18 @@ Below is the full list of supported metrics, ordered lexicographically. The number of non-default behaviors executed by the crypto/tls package due to a non-default GODEBUG=tlsrsakex=... setting. + /godebug/non-default-behavior/tlssha1:events + The number of non-default behaviors executed by the crypto/tls + package due to a non-default GODEBUG=tlssha1=... setting. + /godebug/non-default-behavior/tlsunsafeekm:events The number of non-default behaviors executed by the crypto/tls package due to a non-default GODEBUG=tlsunsafeekm=... setting. + /godebug/non-default-behavior/updatemaxprocs:events + The number of non-default behaviors executed by the runtime + package due to a non-default GODEBUG=updatemaxprocs=... setting. + /godebug/non-default-behavior/winreadlinkvolume:events The number of non-default behaviors executed by the os package due to a non-default GODEBUG=winreadlinkvolume=... setting. @@ -349,9 +417,13 @@ Below is the full list of supported metrics, ordered lexicographically. package due to a non-default GODEBUG=x509negativeserial=... setting. - /godebug/non-default-behavior/x509sha1:events + /godebug/non-default-behavior/x509rsacrt:events + The number of non-default behaviors executed by the crypto/x509 + package due to a non-default GODEBUG=x509rsacrt=... setting. + + /godebug/non-default-behavior/x509sha256skid:events The number of non-default behaviors executed by the crypto/x509 - package due to a non-default GODEBUG=x509sha1=... setting. + package due to a non-default GODEBUG=x509sha256skid=... setting. /godebug/non-default-behavior/x509usefallbackroots:events The number of non-default behaviors executed by the crypto/x509 @@ -442,6 +514,29 @@ Below is the full list of supported metrics, ordered lexicographically. operating system threads that can execute user-level Go code simultaneously. + /sched/goroutines-created:goroutines + Count of goroutines created since program start. + + /sched/goroutines/not-in-go:goroutines + Approximate count of goroutines running or blocked in + a system call or cgo call. Not guaranteed to add up to + /sched/goroutines:goroutines with other goroutine metrics. + + /sched/goroutines/runnable:goroutines + Approximate count of goroutines ready to execute, + but not executing. Not guaranteed to add up to + /sched/goroutines:goroutines with other goroutine metrics. + + /sched/goroutines/running:goroutines + Approximate count of goroutines executing. Always less than or + equal to /sched/gomaxprocs:threads. Not guaranteed to add up to + /sched/goroutines:goroutines with other goroutine metrics. + + /sched/goroutines/waiting:goroutines + Approximate count of goroutines waiting on a resource + (I/O or sync primitives). Not guaranteed to add up to + /sched/goroutines:goroutines with other goroutine metrics. + /sched/goroutines:goroutines Count of live goroutines. @@ -482,6 +577,10 @@ Below is the full list of supported metrics, ordered lexicographically. /sched/pauses/stopping/other:seconds). Bucket counts increase monotonically. + /sched/threads/total:threads + The current count of live threads that are owned by the Go + runtime. + /sync/mutex/wait/total:seconds Approximate cumulative time goroutines have spent blocked on a sync.Mutex, sync.RWMutex, or runtime-internal lock. This metric diff --git a/src/runtime/metrics_test.go b/src/runtime/metrics_test.go index 9191d86d043770..b67424301b42fa 100644 --- a/src/runtime/metrics_test.go +++ b/src/runtime/metrics_test.go @@ -6,7 +6,6 @@ package runtime_test import ( "bytes" - "fmt" "internal/abi" "internal/goexperiment" "internal/profile" @@ -500,6 +499,10 @@ func TestReadMetricsCumulative(t *testing.T) { defer wg.Done() for { // Add more things here that could influence metrics. + for i := 0; i < 10; i++ { + runtime.AddCleanup(new(*int), func(_ struct{}) {}, struct{}{}) + runtime.SetFinalizer(new(*int), func(_ **int) {}) + } for i := 0; i < len(readMetricsSink); i++ { readMetricsSink[i] = make([]byte, 1024) select { @@ -955,17 +958,6 @@ func TestRuntimeLockMetricsAndProfile(t *testing.T) { t.Fatalf("need MutexProfileRate 0, got %d", old) } - { - before := os.Getenv("GODEBUG") - for _, s := range strings.Split(before, ",") { - if strings.HasPrefix(s, "runtimecontentionstacks=") { - t.Logf("GODEBUG includes explicit setting %q", s) - } - } - defer func() { os.Setenv("GODEBUG", before) }() - os.Setenv("GODEBUG", fmt.Sprintf("%s,runtimecontentionstacks=1", before)) - } - t.Logf("NumCPU %d", runtime.NumCPU()) t.Logf("GOMAXPROCS %d", runtime.GOMAXPROCS(0)) if minCPU := 2; runtime.NumCPU() < minCPU { @@ -1020,9 +1012,9 @@ func TestRuntimeLockMetricsAndProfile(t *testing.T) { return metricGrowth, profileGrowth, p } - testcase := func(strictTiming bool, acceptStacks [][]string, workers int, fn func() bool) func(t *testing.T) (metricGrowth, profileGrowth float64, n, value int64) { - return func(t *testing.T) (metricGrowth, profileGrowth float64, n, value int64) { - metricGrowth, profileGrowth, p := measureDelta(t, func() { + testcase := func(strictTiming bool, acceptStacks [][]string, workers int, fn func() bool) func(t *testing.T) (metricGrowth float64, profileGrowth []int64, n, value int64, explain func()) { + return func(t *testing.T) (metricGrowth float64, profileGrowth []int64, n, value int64, explain func()) { + metricGrowth, totalProfileGrowth, p := measureDelta(t, func() { var started, stopped sync.WaitGroup started.Add(workers) stopped.Add(workers) @@ -1042,7 +1034,7 @@ func TestRuntimeLockMetricsAndProfile(t *testing.T) { stopped.Wait() }) - if profileGrowth == 0 { + if totalProfileGrowth == 0 { t.Errorf("no increase in mutex profile") } if metricGrowth == 0 && strictTiming { @@ -1062,7 +1054,7 @@ func TestRuntimeLockMetricsAndProfile(t *testing.T) { // together. It doesn't work as well for user-space contention, where the // involved goroutines are not _Grunnable the whole time and so need to pass // through the scheduler. - t.Logf("lock contention growth in runtime/pprof's view (%fs)", profileGrowth) + t.Logf("lock contention growth in runtime/pprof's view (%fs)", totalProfileGrowth) t.Logf("lock contention growth in runtime/metrics' view (%fs)", metricGrowth) acceptStacks = append([][]string(nil), acceptStacks...) @@ -1082,7 +1074,7 @@ func TestRuntimeLockMetricsAndProfile(t *testing.T) { } var stks [][]string - values := make([][2]int64, len(acceptStacks)) + values := make([][2]int64, len(acceptStacks)+1) for _, s := range p.Sample { var have []string for _, loc := range s.Location { @@ -1091,16 +1083,26 @@ func TestRuntimeLockMetricsAndProfile(t *testing.T) { } } stks = append(stks, have) + found := false for i, stk := range acceptStacks { if slices.Equal(have, stk) { values[i][0] += s.Value[0] values[i][1] += s.Value[1] + found = true + break } } + if !found { + values[len(values)-1][0] += s.Value[0] + values[len(values)-1][1] += s.Value[1] + } } + profileGrowth = make([]int64, len(acceptStacks)+1) + profileGrowth[len(profileGrowth)-1] = values[len(values)-1][1] for i, stk := range acceptStacks { n += values[i][0] value += values[i][1] + profileGrowth[i] = values[i][1] t.Logf("stack %v has samples totaling n=%d value=%d", stk, values[i][0], values[i][1]) } if n == 0 && value == 0 { @@ -1113,18 +1115,15 @@ func TestRuntimeLockMetricsAndProfile(t *testing.T) { } } - return metricGrowth, profileGrowth, n, value + return metricGrowth, profileGrowth, n, value, func() { + t.Logf("profile:\n%s", p) + } } } name := t.Name() t.Run("runtime.lock", func(t *testing.T) { - mus := make([]runtime.Mutex, 200) - var needContention atomic.Int64 - delay := 100 * time.Microsecond // large relative to system noise, for comparison between clocks - delayMicros := delay.Microseconds() - // The goroutine that acquires the lock will only proceed when it // detects that its partner is contended for the lock. That will lead to // live-lock if anything (such as a STW) prevents the partner goroutine @@ -1134,11 +1133,29 @@ func TestRuntimeLockMetricsAndProfile(t *testing.T) { // will end up contended twice. Instead, disable the GC. defer debug.SetGCPercent(debug.SetGCPercent(-1)) - const workers = 2 - if runtime.GOMAXPROCS(0) < workers { - t.Skipf("contention on runtime-internal locks requires GOMAXPROCS >= %d", workers) + mus := make([]runtime.Mutex, 200) + var needContention atomic.Int64 + + baseDelay := 100 * time.Microsecond // large relative to system noise, for comparison between clocks + fastDelayMicros := baseDelay.Microseconds() + slowDelayMicros := baseDelay.Microseconds() * 4 + + const ( + fastRole = iota + slowRole + workerCount + ) + if runtime.GOMAXPROCS(0) < workerCount { + t.Skipf("contention on runtime-internal locks requires GOMAXPROCS >= %d", workerCount) } + minTicks := make([][]int64, workerCount) // lower bound, known-contended time, measured by cputicks + maxTicks := make([][]int64, workerCount) // upper bound, total lock() duration, measured by cputicks + for i := range minTicks { + minTicks[i] = make([]int64, len(mus)) + maxTicks[i] = make([]int64, len(mus)) + } + var id atomic.Int32 fn := func() bool { n := int(needContention.Load()) if n < 0 { @@ -1146,41 +1163,89 @@ func TestRuntimeLockMetricsAndProfile(t *testing.T) { } mu := &mus[n] - runtime.Lock(mu) - for int(needContention.Load()) == n { - if runtime.MutexContended(mu) { - // make them wait a little while - for start := runtime.Nanotime(); (runtime.Nanotime()-start)/1000 < delayMicros; { - runtime.Usleep(uint32(delayMicros)) + // Each worker has a role: to have a fast or slow critical section. + // Rotate the role assignments as we step through the mutex slice so + // we don't end up with one M always claiming the same kind of work. + id := int(id.Add(1)) + role := (id + n) % workerCount + + marker, delayMicros := fastMarkerFrame, fastDelayMicros + if role == slowRole { + marker, delayMicros = slowMarkerFrame, slowDelayMicros + } + + // Each lock is used by two different critical sections, one fast + // and one slow, identified in profiles by their different "marker" + // functions. We expect the profile to blame each for the amount of + // delay it inflicts on other users of the lock. We run one worker + // of each kind, so any contention in one would be due to the other. + // + // We measure how long our runtime.lock call takes, which sets an + // upper bound on how much blame to expect for the other worker type + // in the profile. And if we acquire the lock first, we wait for the + // other worker to announce its contention. We measure the + // known-contended time, to use as a lower bound on how much blame + // we expect of ourselves in the profile. Then we stall for a little + // while (different amounts for "fast" versus "slow") before + // unlocking the mutex. + + marker(func() { + t0 := runtime.Cputicks() + runtime.Lock(mu) + maxTicks[role][n] = runtime.Cputicks() - t0 + minTicks[role][n] = 0 + for int(needContention.Load()) == n { + if runtime.MutexContended(mu) { + t1 := runtime.Cputicks() + // make them wait a little while + for start := runtime.Nanotime(); (runtime.Nanotime()-start)/1000 < delayMicros; { + runtime.Usleep(uint32(1 + delayMicros/8)) + } + minTicks[role][n] = runtime.Cputicks() - t1 + break } - break } - } - runtime.Unlock(mu) - needContention.Store(int64(n - 1)) + runtime.Unlock(mu) + needContention.Store(int64(n - 1)) + }) return true } - stks := [][]string{{ - "runtime.unlock", - "runtime_test." + name + ".func5.1", - "runtime_test.(*contentionWorker).run", - }} + stks := make([][]string, 2) + for i := range stks { + marker := "runtime_test.fastMarkerFrame" + if i == slowRole { + marker = "runtime_test.slowMarkerFrame" + } + + stks[i] = []string{ + "runtime.unlock", + "runtime_test." + name + ".func4.1.1", + marker, + "runtime_test." + name + ".func4.1", + "runtime_test.(*contentionWorker).run", + } + } t.Run("sample-1", func(t *testing.T) { old := runtime.SetMutexProfileFraction(1) defer runtime.SetMutexProfileFraction(old) needContention.Store(int64(len(mus) - 1)) - metricGrowth, profileGrowth, n, _ := testcase(true, stks, workers, fn)(t) + metricGrowth, profileGrowth, n, _, explain := testcase(true, stks, workerCount, fn)(t) + defer func() { + if t.Failed() { + explain() + } + }() t.Run("metric", func(t *testing.T) { // The runtime/metrics view may be sampled at 1 per // gTrackingPeriod, so we don't have a hard lower bound here. testenv.SkipFlaky(t, 64253) - if have, want := metricGrowth, delay.Seconds()*float64(len(mus)); have < want { + if have, want := metricGrowth, baseDelay.Seconds()*float64(len(mus)); have < want { // The test imposes a delay with usleep, verified with calls to // nanotime. Compare against the runtime/metrics package's view // (based on nanotime) rather than runtime/pprof's view (based @@ -1192,10 +1257,68 @@ func TestRuntimeLockMetricsAndProfile(t *testing.T) { t.Errorf("mutex profile reported contention count different from the known true count (%d != %d)", have, want) } + var slowMinTicks, fastMinTicks int64 + for role, ticks := range minTicks { + for _, delta := range ticks { + if role == slowRole { + slowMinTicks += delta + } else { + fastMinTicks += delta + } + } + } + var slowMaxTicks, fastMaxTicks int64 + for role, ticks := range maxTicks { + for _, delta := range ticks { + if role == slowRole { + slowMaxTicks += delta + } else { + fastMaxTicks += delta + } + } + } + + cpuGHz := float64(runtime.CyclesPerSecond()) / 1e9 + for _, set := range []struct { + name string + profTime int64 + minTime int64 + maxTime int64 + }{ + { + name: "slow", + profTime: profileGrowth[slowRole], + minTime: int64(float64(slowMinTicks) / cpuGHz), + maxTime: int64(float64(fastMaxTicks) / cpuGHz), + }, + { + name: "fast", + profTime: profileGrowth[fastRole], + minTime: int64(float64(fastMinTicks) / cpuGHz), + maxTime: int64(float64(slowMaxTicks) / cpuGHz), + }, + } { + t.Logf("profile's view of delays due to %q critical section: %dns", set.name, set.profTime) + t.Logf("test's view of known-contended time within %q critical section: %dns", set.name, set.minTime) + t.Logf("test's view of lock duration before critical sections other than %q: %dns", set.name, set.maxTime) + + if set.profTime < set.minTime { + t.Errorf("profile undercounted %q critical section", set.name) + } + if set.profTime > set.maxTime { + t.Errorf("profile overcounted %q critical section", set.name) + } + } + + var totalProfileGrowth float64 + for _, growth := range profileGrowth { + totalProfileGrowth += float64(growth) * time.Nanosecond.Seconds() + } + const slop = 1.5 // account for nanotime vs cputicks t.Run("compare timers", func(t *testing.T) { testenv.SkipFlaky(t, 64253) - if profileGrowth > slop*metricGrowth || metricGrowth > slop*profileGrowth { + if totalProfileGrowth > slop*metricGrowth || metricGrowth > slop*totalProfileGrowth { t.Errorf("views differ by more than %fx", slop) } }) @@ -1208,7 +1331,12 @@ func TestRuntimeLockMetricsAndProfile(t *testing.T) { defer runtime.SetMutexProfileFraction(old) needContention.Store(int64(len(mus) - 1)) - metricGrowth, profileGrowth, n, _ := testcase(true, stks, workers, fn)(t) + metricGrowth, profileGrowth, n, _, explain := testcase(true, stks, workerCount, fn)(t) + defer func() { + if t.Failed() { + explain() + } + }() // With 100 trials and profile fraction of 2, we expect to capture // 50 samples. Allow the test to pass if we get at least 20 samples; @@ -1216,7 +1344,7 @@ func TestRuntimeLockMetricsAndProfile(t *testing.T) { // 1e-9 chance of that, which is an acceptably low flakiness rate. const samplingSlop = 2.5 - if have, want := metricGrowth, delay.Seconds()*float64(len(mus)); samplingSlop*have < want { + if have, want := metricGrowth, baseDelay.Seconds()*float64(len(mus)); samplingSlop*have < want { // The test imposes a delay with usleep, verified with calls to // nanotime. Compare against the runtime/metrics package's view // (based on nanotime) rather than runtime/pprof's view (based @@ -1227,8 +1355,13 @@ func TestRuntimeLockMetricsAndProfile(t *testing.T) { t.Errorf("mutex profile reported contention count too different from the expected count (%d far from %d)", have, want) } + var totalProfileGrowth float64 + for _, growth := range profileGrowth { + totalProfileGrowth += float64(growth) * time.Nanosecond.Seconds() + } + const timerSlop = 1.5 * samplingSlop // account for nanotime vs cputicks, plus the two views' independent sampling - if profileGrowth > timerSlop*metricGrowth || metricGrowth > timerSlop*profileGrowth { + if totalProfileGrowth > timerSlop*metricGrowth || metricGrowth > timerSlop*totalProfileGrowth { t.Errorf("views differ by more than %fx", timerSlop) } }) @@ -1270,14 +1403,14 @@ func TestRuntimeLockMetricsAndProfile(t *testing.T) { { "runtime.unlock", "runtime.semrelease1", - "runtime_test.TestRuntimeLockMetricsAndProfile.func6.1", + "runtime_test.TestRuntimeLockMetricsAndProfile.func5.1", "runtime_test.(*contentionWorker).run", }, { "runtime.unlock", "runtime.semacquire1", "runtime.semacquire", - "runtime_test.TestRuntimeLockMetricsAndProfile.func6.1", + "runtime_test.TestRuntimeLockMetricsAndProfile.func5.1", "runtime_test.(*contentionWorker).run", }, } @@ -1295,6 +1428,9 @@ func TestRuntimeLockMetricsAndProfile(t *testing.T) { }) } +func slowMarkerFrame(fn func()) { fn() } +func fastMarkerFrame(fn func()) { fn() } + // contentionWorker provides cleaner call stacks for lock contention profile tests type contentionWorker struct { before func() @@ -1380,3 +1516,71 @@ func TestMetricHeapUnusedLargeObjectOverflow(t *testing.T) { done <- struct{}{} wg.Wait() } + +func TestReadMetricsCleanups(t *testing.T) { + runtime.GC() // End any in-progress GC. + runtime.BlockUntilEmptyCleanupQueue(int64(1 * time.Second)) // Flush any queued cleanups. + + var before [2]metrics.Sample + before[0].Name = "/gc/cleanups/queued:cleanups" + before[1].Name = "/gc/cleanups/executed:cleanups" + after := before + + metrics.Read(before[:]) + + const N = 10 + for i := 0; i < N; i++ { + runtime.AddCleanup(new(*int), func(_ struct{}) {}, struct{}{}) + } + + runtime.GC() + runtime.BlockUntilEmptyCleanupQueue(int64(1 * time.Second)) + + metrics.Read(after[:]) + + if v0, v1 := before[0].Value.Uint64(), after[0].Value.Uint64(); v0+N != v1 { + t.Errorf("expected %s difference to be exactly %d, got %d -> %d", before[0].Name, N, v0, v1) + } + if v0, v1 := before[1].Value.Uint64(), after[1].Value.Uint64(); v0+N != v1 { + t.Errorf("expected %s difference to be exactly %d, got %d -> %d", before[1].Name, N, v0, v1) + } +} + +func TestReadMetricsFinalizers(t *testing.T) { + runtime.GC() // End any in-progress GC. + runtime.BlockUntilEmptyFinalizerQueue(int64(1 * time.Second)) // Flush any queued finalizers. + + var before [2]metrics.Sample + before[0].Name = "/gc/finalizers/queued:finalizers" + before[1].Name = "/gc/finalizers/executed:finalizers" + after := before + + metrics.Read(before[:]) + + const N = 10 + for i := 0; i < N; i++ { + runtime.SetFinalizer(new(*int), func(_ **int) {}) + } + + runtime.GC() + runtime.GC() + runtime.BlockUntilEmptyFinalizerQueue(int64(1 * time.Second)) + + metrics.Read(after[:]) + + if v0, v1 := before[0].Value.Uint64(), after[0].Value.Uint64(); v0+N != v1 { + t.Errorf("expected %s difference to be exactly %d, got %d -> %d", before[0].Name, N, v0, v1) + } + if v0, v1 := before[1].Value.Uint64(), after[1].Value.Uint64(); v0+N != v1 { + t.Errorf("expected %s difference to be exactly %d, got %d -> %d", before[1].Name, N, v0, v1) + } +} + +func TestReadMetricsSched(t *testing.T) { + // This test is run in a subprocess to prevent other tests from polluting the metrics. + output := runTestProg(t, "testprog", "SchedMetrics") + want := "OK\n" + if output != want { + t.Fatalf("output:\n%s\n\nwanted:\n%s", output, want) + } +} diff --git a/src/runtime/mfinal.go b/src/runtime/mfinal.go index a926a8ec357c4f..bafdb016038830 100644 --- a/src/runtime/mfinal.go +++ b/src/runtime/mfinal.go @@ -10,23 +10,26 @@ import ( "internal/abi" "internal/goarch" "internal/runtime/atomic" + "internal/runtime/gc" "internal/runtime/sys" "unsafe" ) -// finblock is an array of finalizers to be executed. finblocks are -// arranged in a linked list for the finalizer queue. +const finBlockSize = 4 * 1024 + +// finBlock is an block of finalizers to be executed. finBlocks +// are arranged in a linked list for the finalizer queue. // -// finblock is allocated from non-GC'd memory, so any heap pointers +// finBlock is allocated from non-GC'd memory, so any heap pointers // must be specially handled. GC currently assumes that the finalizer // queue does not grow during marking (but it can shrink). -type finblock struct { +type finBlock struct { _ sys.NotInHeap - alllink *finblock - next *finblock + alllink *finBlock + next *finBlock cnt uint32 _ int32 - fin [(_FinBlockSize - 2*goarch.PtrSize - 2*4) / unsafe.Sizeof(finalizer{})]finalizer + fin [(finBlockSize - 2*goarch.PtrSize - 2*4) / unsafe.Sizeof(finalizer{})]finalizer } var fingStatus atomic.Uint32 @@ -40,13 +43,17 @@ const ( fingWake ) -var finlock mutex // protects the following variables -var fing *g // goroutine that runs finalizers -var finq *finblock // list of finalizers that are to be executed -var finc *finblock // cache of free blocks -var finptrmask [_FinBlockSize / goarch.PtrSize / 8]byte +var ( + finlock mutex // protects the following variables + fing *g // goroutine that runs finalizers + finq *finBlock // list of finalizers that are to be executed + finc *finBlock // cache of free blocks + finptrmask [finBlockSize / goarch.PtrSize / 8]byte + finqueued uint64 // monotonic count of queued finalizers + finexecuted uint64 // monotonic count of executed finalizers +) -var allfin *finblock // list of all blocks +var allfin *finBlock // list of all blocks // NOTE: Layout known to queuefinalizer. type finalizer struct { @@ -103,9 +110,10 @@ func queuefinalizer(p unsafe.Pointer, fn *funcval, nret uintptr, fint *_type, ot } lock(&finlock) + if finq == nil || finq.cnt == uint32(len(finq.fin)) { if finc == nil { - finc = (*finblock)(persistentalloc(_FinBlockSize, 0, &memstats.gcMiscSys)) + finc = (*finBlock)(persistentalloc(finBlockSize, 0, &memstats.gcMiscSys)) finc.alllink = allfin allfin = finc if finptrmask[0] == 0 { @@ -136,6 +144,7 @@ func queuefinalizer(p unsafe.Pointer, fn *funcval, nret uintptr, fint *_type, ot f.fint = fint f.ot = ot f.arg = p + finqueued++ unlock(&finlock) fingStatus.Or(fingWake) } @@ -160,7 +169,7 @@ func wakefing() *g { func createfing() { // start the finalizer goroutine exactly once if fingStatus.Load() == fingUninitialized && fingStatus.CompareAndSwap(fingUninitialized, fingCreated) { - go runfinq() + go runFinalizers() } } @@ -172,8 +181,16 @@ func finalizercommit(gp *g, lock unsafe.Pointer) bool { return true } +func finReadQueueStats() (queued, executed uint64) { + lock(&finlock) + queued = finqueued + executed = finexecuted + unlock(&finlock) + return +} + // This is the goroutine that runs all of the finalizers. -func runfinq() { +func runFinalizers() { var ( frame unsafe.Pointer framecap uintptr @@ -199,7 +216,8 @@ func runfinq() { racefingo() } for fb != nil { - for i := fb.cnt; i > 0; i-- { + n := fb.cnt + for i := n; i > 0; i-- { f := &fb.fin[i-1] var regs abi.RegArgs @@ -220,9 +238,8 @@ func runfinq() { frame = mallocgc(framesz, nil, true) framecap = framesz } - if f.fint == nil { - throw("missing type in runfinq") + throw("missing type in finalizer") } r := frame if argRegs > 0 { @@ -234,7 +251,7 @@ func runfinq() { // confusing the write barrier. *(*[2]uintptr)(frame) = [2]uintptr{} } - switch f.fint.Kind_ & abi.KindMask { + switch f.fint.Kind() { case abi.Pointer: // direct use of pointer *(*unsafe.Pointer)(r) = f.arg @@ -249,7 +266,7 @@ func runfinq() { (*iface)(r).tab = assertE2I(ityp, (*eface)(r)._type) } default: - throw("bad kind in runfinq") + throw("bad type kind in finalizer") } fingStatus.Or(fingRunningFinalizer) reflectcall(nil, unsafe.Pointer(f.fn), frame, uint32(framesz), uint32(framesz), uint32(framesz), ®s) @@ -266,6 +283,7 @@ func runfinq() { } next := fb.next lock(&finlock) + finexecuted += uint64(n) fb.next = finc finc = fb unlock(&finlock) @@ -302,7 +320,7 @@ func isGoPointerWithoutSpan(p unsafe.Pointer) bool { // blockUntilEmptyFinalizerQueue blocks until either the finalizer // queue is emptied (and the finalizers have executed) or the timeout // is reached. Returns true if the finalizer queue was emptied. -// This is used by the runtime and sync tests. +// This is used by the runtime, sync, and unique tests. func blockUntilEmptyFinalizerQueue(timeout int64) bool { start := nanotime() for nanotime()-start < timeout { @@ -330,6 +348,9 @@ func blockUntilEmptyFinalizerQueue(timeout int64) bool { // // SetFinalizer(obj, nil) clears any finalizer associated with obj. // +// New Go code should consider using [AddCleanup] instead, which is much +// less error-prone than SetFinalizer. +// // The argument obj must be a pointer to an object allocated by calling // new, by taking the address of a composite literal, or by taking the // address of a local variable. @@ -409,28 +430,27 @@ func blockUntilEmptyFinalizerQueue(timeout int64) bool { // need to use appropriate synchronization, such as mutexes or atomic updates, // to avoid read-write races. func SetFinalizer(obj any, finalizer any) { - if debug.sbrk != 0 { - // debug.sbrk never frees memory, so no finalizers run - // (and we don't have the data structures to record them). - return - } e := efaceOf(&obj) etyp := e._type if etyp == nil { throw("runtime.SetFinalizer: first argument is nil") } - if etyp.Kind_&abi.KindMask != abi.Pointer { + if etyp.Kind() != abi.Pointer { throw("runtime.SetFinalizer: first argument is " + toRType(etyp).string() + ", not pointer") } ot := (*ptrtype)(unsafe.Pointer(etyp)) if ot.Elem == nil { throw("nil elem type!") } - if inUserArenaChunk(uintptr(e.data)) { // Arena-allocated objects are not eligible for finalizers. throw("runtime.SetFinalizer: first argument was allocated into an arena") } + if debug.sbrk != 0 { + // debug.sbrk never frees memory, so no finalizers run + // (and we don't have the data structures to record them). + return + } // find the containing object base, span, _ := findObject(uintptr(e.data), 0, 0) @@ -444,7 +464,7 @@ func SetFinalizer(obj any, finalizer any) { // Move base forward if we've got an allocation header. if !span.spanclass.noscan() && !heapBitsInSpan(span.elemsize) && span.spanclass.sizeclass() != 0 { - base += mallocHeaderSize + base += gc.MallocHeaderSize } if uintptr(e.data) != base { @@ -461,11 +481,16 @@ func SetFinalizer(obj any, finalizer any) { // switch to system stack and remove finalizer systemstack(func() { removefinalizer(e.data) + + if debug.checkfinalizers != 0 { + clearFinalizerContext(uintptr(e.data)) + KeepAlive(e.data) + } }) return } - if ftyp.Kind_&abi.KindMask != abi.Func { + if ftyp.Kind() != abi.Func { throw("runtime.SetFinalizer: second argument is " + toRType(ftyp).string() + ", not a function") } ft := (*functype)(unsafe.Pointer(ftyp)) @@ -480,13 +505,13 @@ func SetFinalizer(obj any, finalizer any) { case fint == etyp: // ok - same type goto okarg - case fint.Kind_&abi.KindMask == abi.Pointer: + case fint.Kind() == abi.Pointer: if (fint.Uncommon() == nil || etyp.Uncommon() == nil) && (*ptrtype)(unsafe.Pointer(fint)).Elem == ot.Elem { // ok - not same type, but both pointers, // one or the other is unnamed, and same element type, so assignable. goto okarg } - case fint.Kind_&abi.KindMask == abi.Interface: + case fint.Kind() == abi.Interface: ityp := (*interfacetype)(unsafe.Pointer(fint)) if len(ityp.Methods) == 0 { // ok - satisfies empty interface @@ -508,10 +533,14 @@ okarg: // make sure we have a finalizer goroutine createfing() + callerpc := sys.GetCallerPC() systemstack(func() { if !addfinalizer(e.data, (*funcval)(f.data), nret, fint, ot) { throw("runtime.SetFinalizer: finalizer already set") } + if debug.checkfinalizers != 0 { + setFinalizerContext(e.data, ot.Elem, callerpc, (*funcval)(f.data).fn) + } }) } diff --git a/src/runtime/mfinal_test.go b/src/runtime/mfinal_test.go index 87d31c472c1b01..5c93c74cfb2f08 100644 --- a/src/runtime/mfinal_test.go +++ b/src/runtime/mfinal_test.go @@ -5,6 +5,7 @@ package runtime_test import ( + "internal/asan" "runtime" "testing" "time" @@ -165,6 +166,9 @@ func adjChunks() (*objtype, *objtype) { // Make sure an empty slice on the stack doesn't pin the next object in memory. func TestEmptySlice(t *testing.T) { + if asan.Enabled { + t.Skip("skipping with -asan: test assumes exact size class alignment, but asan redzone breaks that assumption") + } x, y := adjChunks() // the pointer inside xs points to y. @@ -194,6 +198,9 @@ func adjStringChunk() (string, *objtype) { // Make sure an empty string on the stack doesn't pin the next object in memory. func TestEmptyString(t *testing.T) { + if asan.Enabled { + t.Skip("skipping with -asan: test assumes exact size class alignment, but asan redzone breaks that assumption") + } x, y := adjStringChunk() ss := x[objsize:] // change objsize to objsize-1 and the test passes diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 2654c696582211..b4b4485629b59d 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -130,13 +130,15 @@ package runtime import ( "internal/cpu" + "internal/goarch" + "internal/goexperiment" "internal/runtime/atomic" + "internal/runtime/gc" "unsafe" ) const ( - _DebugGC = 0 - _FinBlockSize = 4 * 1024 + _DebugGC = 0 // concurrentSweep is a debug flag. Disabling this flag // ensures all spans are swept while the world is stopped. @@ -186,11 +188,18 @@ func gcinit() { // Use the environment variable GOMEMLIMIT for the initial memoryLimit value. gcController.init(readGOGC(), readGOMEMLIMIT()) + // Set up the cleanup block ptr mask. + for i := range cleanupBlockPtrMask { + cleanupBlockPtrMask[i] = 0xff + } + work.startSema = 1 work.markDoneSema = 1 lockInit(&work.sweepWaiters.lock, lockRankSweepWaiters) lockInit(&work.assistQueue.lock, lockRankAssistQueue) + lockInit(&work.strongFromWeak.lock, lockRankStrongFromWeakQueue) lockInit(&work.wbufSpans.lock, lockRankWbufSpans) + lockInit(&gcCleanups.lock, lockRankCleanupQueue) } // gcenable is called after the bulk of the runtime initialization, @@ -220,7 +229,6 @@ var gcphase uint32 // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/sonic -// - github.com/cloudwego/frugal // // Do not remove or change the type signature. // See go.dev/issue/67401. @@ -318,7 +326,7 @@ type workType struct { full lfstack // lock-free list of full blocks workbuf _ cpu.CacheLinePad // prevents false-sharing between full and empty empty lfstack // lock-free list of empty blocks workbuf - _ cpu.CacheLinePad // prevents false-sharing between empty and nproc/nwait + _ cpu.CacheLinePad // prevents false-sharing between empty and wbufSpans wbufSpans struct { lock mutex @@ -329,17 +337,36 @@ type workType struct { // one of the workbuf lists. busy mSpanList } + _ cpu.CacheLinePad // prevents false-sharing between wbufSpans and spanWorkMask + + // spanqMask is a bitmap indicating which Ps have local work worth stealing. + // Set or cleared by the owning P, cleared by stealing Ps. + // + // spanqMask is like a proxy for a global queue. An important invariant is that + // forced flushing like gcw.dispose must set this bit on any P that has local + // span work. + spanqMask pMask + _ cpu.CacheLinePad // prevents false-sharing between spanqMask and everything else + + // List of all spanSPMCs. + // + // Only used if goexperiment.GreenTeaGC. + spanSPMCs struct { + lock mutex // no lock rank because it's a leaf lock (see mklockrank.go). + all *spanSPMC + } // Restore 64-bit alignment on 32-bit. - _ uint32 + // _ uint32 // bytesMarked is the number of bytes marked this cycle. This // includes bytes blackened in scanned objects, noscan objects - // that go straight to black, and permagrey objects scanned by - // markroot during the concurrent scan phase. This is updated - // atomically during the cycle. Updates may be batched - // arbitrarily, since the value is only read at the end of the - // cycle. + // that go straight to black, objects allocated as black during + // the cycle, and permagrey objects scanned by markroot during + // the concurrent scan phase. + // + // This is updated atomically during the cycle. Updates may be batched + // arbitrarily, since the value is only read at the end of the cycle. // // Because of benign races during marking, this number may not // be the exact number of marked bytes, but it should be very @@ -349,26 +376,53 @@ type workType struct { // (and thus 8-byte alignment even on 32-bit architectures). bytesMarked uint64 - markrootNext uint32 // next markroot job - markrootJobs uint32 // number of markroot jobs + markrootNext atomic.Uint32 // next markroot job + markrootJobs atomic.Uint32 // number of markroot jobs nproc uint32 tstart int64 nwait uint32 - // Number of roots of various root types. Set by gcMarkRootPrepare. + // Number of roots of various root types. Set by gcPrepareMarkRoots. // - // nStackRoots == len(stackRoots), but we have nStackRoots for - // consistency. - nDataRoots, nBSSRoots, nSpanRoots, nStackRoots int + // During normal GC cycle, nStackRoots == nMaybeRunnableStackRoots == len(stackRoots); + // during goroutine leak detection, nMaybeRunnableStackRoots is the number of stackRoots + // scheduled for marking. + // In both variants, nStackRoots == len(stackRoots). + nDataRoots, nBSSRoots, nSpanRoots, nStackRoots, nMaybeRunnableStackRoots int + + // The following fields monitor the GC phase of the current cycle during + // goroutine leak detection. + goroutineLeak struct { + // Once set, it indicates that the GC will perform goroutine leak detection during + // the next GC cycle; it is set by goroutineLeakGC and unset during gcStart. + pending atomic.Bool + // Once set, it indicates that the GC has started a goroutine leak detection run; + // it is set during gcStart and unset during gcMarkTermination; + // + // Protected by STW. + enabled bool + // Once set, it indicates that the GC has performed goroutine leak detection during + // the current GC cycle; it is set during gcMarkDone, right after goroutine leak detection, + // and unset during gcMarkTermination; + // + // Protected by STW. + done bool + // The number of leaked goroutines during the last leak detection GC cycle. + // + // Write-protected by STW in findGoroutineLeaks. + count int + } - // Base indexes of each root type. Set by gcMarkRootPrepare. + // Base indexes of each root type. Set by gcPrepareMarkRoots. baseData, baseBSS, baseSpans, baseStacks, baseEnd uint32 - // stackRoots is a snapshot of all of the Gs that existed - // before the beginning of concurrent marking. The backing - // store of this must not be modified because it might be - // shared with allgs. + // stackRoots is a snapshot of all of the Gs that existed before the + // beginning of concurrent marking. During goroutine leak detection, stackRoots + // is partitioned into two sets; to the left of nMaybeRunnableStackRoots are stackRoots + // of running / runnable goroutines and to the right of nMaybeRunnableStackRoots are + // stackRoots of unmarked / not runnable goroutines + // The stackRoots array is re-partitioned after each marking phase iteration. stackRoots []*g // Each type of GC state transition is protected by a lock. @@ -418,6 +472,26 @@ type workType struct { list gList } + // strongFromWeak controls how the GC interacts with weak->strong + // pointer conversions. + strongFromWeak struct { + // block is a flag set during mark termination that prevents + // new weak->strong conversions from executing by blocking the + // goroutine and enqueuing it onto q. + // + // Mutated only by one goroutine at a time in gcMarkDone, + // with globally-synchronizing events like forEachP and + // stopTheWorld. + block bool + + // q is a queue of goroutines that attempted to perform a + // weak->strong conversion during mark termination. + // + // Protected by lock. + lock mutex + q gQueue + } + // cycles is the number of completed GC cycles, where a GC // cycle is sweep termination, mark, mark termination, and // sweep. This differs from memstats.numgc, which is @@ -515,6 +589,55 @@ func GC() { releasem(mp) } +// goroutineLeakGC runs a GC cycle that performs goroutine leak detection. +// +//go:linkname goroutineLeakGC runtime/pprof.runtime_goroutineLeakGC +func goroutineLeakGC() { + // Set the pending flag to true, instructing the next GC cycle to + // perform goroutine leak detection. + work.goroutineLeak.pending.Store(true) + + // Spin GC cycles until the pending flag is unset. + // This ensures that goroutineLeakGC waits for a GC cycle that + // actually performs goroutine leak detection. + // + // This is needed in case multiple concurrent calls to GC + // are simultaneously fired by the system, wherein some + // of them are dropped. + // + // In the vast majority of cases, only one loop iteration is needed; + // however, multiple concurrent calls to goroutineLeakGC could lead to + // the execution of additional GC cycles. + // + // Examples: + // + // pending? | G1 | G2 + // ---------|-------------------------|----------------------- + // - | goroutineLeakGC() | goroutineLeakGC() + // - | pending.Store(true) | . + // X | for pending.Load() | . + // X | GC() | . + // X | > gcStart() | . + // X | pending.Store(false) | . + // ... + // - | > gcMarkDone() | . + // - | . | pending.Store(true) + // ... + // X | > gcMarkTermination() | . + // X | ... + // X | < GC returns | . + // X | for pending.Load | . + // X | GC() | . + // X | . | for pending.Load() + // X | . | GC() + // ... + // The first to pick up the pending flag will start a + // leak detection cycle. + for work.goroutineLeak.pending.Load() { + GC() + } +} + // gcWaitOnMark blocks until GC finishes the Nth mark phase. If GC has // already completed this mark phase, it returns immediately. func gcWaitOnMark(n uint32) { @@ -618,6 +741,17 @@ func gcStart(trigger gcTrigger) { releasem(mp) mp = nil + if gp := getg(); gp.bubble != nil { + // Disassociate the G from its synctest bubble while allocating. + // This is less elegant than incrementing the group's active count, + // but avoids any contamination between GC and synctest. + bubble := gp.bubble + gp.bubble = nil + defer func() { + gp.bubble = bubble + }() + } + // Pick up the remaining unswept/not being swept spans concurrently // // This shouldn't happen if we're being invoked in background @@ -665,12 +799,17 @@ func gcStart(trigger gcTrigger) { traceRelease(trace) } - // Check that all Ps have finished deferred mcache flushes. + // Check and setup per-P state. for _, p := range allp { + // Check that all Ps have finished deferred mcache flushes. if fg := p.mcache.flushGen.Load(); fg != mheap_.sweepgen { println("runtime: p", p.id, "flushGen", fg, "!= sweepgen", mheap_.sweepgen) throw("p mcache not flushed") } + // Initialize ptrBuf if necessary. + if goexperiment.GreenTeaGC && p.gcw.ptrBuf == nil { + p.gcw.ptrBuf = (*[gc.PageSize / goarch.PtrSize]uintptr)(persistentalloc(gc.PageSize, goarch.PtrSize, &memstats.gcMiscSys)) + } } gcBgMarkStartWorkers() @@ -678,10 +817,10 @@ func gcStart(trigger gcTrigger) { systemstack(gcResetMarkState) work.stwprocs, work.maxprocs = gomaxprocs, gomaxprocs - if work.stwprocs > ncpu { - // This is used to compute CPU time of the STW phases, - // so it can't be more than ncpu, even if GOMAXPROCS is. - work.stwprocs = ncpu + if work.stwprocs > numCPUStartup { + // This is used to compute CPU time of the STW phases, so it + // can't be more than the CPU count, even if GOMAXPROCS is. + work.stwprocs = numCPUStartup } work.heap0 = gcController.heapLive.Load() work.pauseNS = 0 @@ -722,6 +861,15 @@ func gcStart(trigger gcTrigger) { schedEnableUser(false) } + // If goroutine leak detection is pending, enable it for this GC cycle. + if work.goroutineLeak.pending.Load() { + work.goroutineLeak.enabled = true + work.goroutineLeak.pending.Store(false) + // Set all sync objects of blocked goroutines as untraceable + // by the GC. Only set as traceable at the end of the GC cycle. + setSyncObjectsUntraceable() + } + // Enter concurrent mark phase and enable // write barriers. // @@ -739,7 +887,7 @@ func gcStart(trigger gcTrigger) { setGCPhase(_GCmark) gcBgMarkPrepare() // Must happen before assists are enabled. - gcMarkRootPrepare() + gcPrepareMarkRoots() // Mark all active tinyalloc blocks. Since we're // allocating from these, they need to be black like @@ -800,16 +948,30 @@ func gcStart(trigger gcTrigger) { // This is protected by markDoneSema. var gcMarkDoneFlushed uint32 +// gcDebugMarkDone contains fields used to debug/test mark termination. +var gcDebugMarkDone struct { + // spinAfterRaggedBarrier forces gcMarkDone to spin after it executes + // the ragged barrier. + spinAfterRaggedBarrier atomic.Bool + + // restartedDueTo27993 indicates that we restarted mark termination + // due to the bug described in issue #27993. + // + // Protected by worldsema. + restartedDueTo27993 bool +} + // gcMarkDone transitions the GC from mark to mark termination if all // reachable objects have been marked (that is, there are no grey // objects and can be no more in the future). Otherwise, it flushes // all local work to the global queues where it can be discovered by // other workers. // +// All goroutines performing GC work must call gcBeginWork to signal +// that they're executing GC work. They must call gcEndWork when done. // This should be called when all local mark work has been drained and -// there are no remaining workers. Specifically, when -// -// work.nwait == work.nproc && !gcMarkWorkAvailable(p) +// there are no remaining workers. Specifically, when gcEndWork returns +// true. // // The calling context must be preemptible. // @@ -833,7 +995,7 @@ top: // empty before performing the ragged barrier. Otherwise, // there could be global work that a P could take after the P // has passed the ragged barrier. - if !(gcphase == _GCmark && work.nwait == work.nproc && !gcMarkWorkAvailable(nil)) { + if !(gcphase == _GCmark && gcIsMarkDone()) { semrelease(&work.markDoneSema) return } @@ -842,6 +1004,10 @@ top: // stop the world later, so acquire worldsema now. semacquire(&worldsema) + // Prevent weak->strong conversions from generating additional + // GC work. forEachP will guarantee that it is observed globally. + work.strongFromWeak.block = true + // Flush all local buffers and collect flushedWork flags. gcMarkDoneFlushed = 0 forEachP(waitReasonGCMarkTermination, func(pp *p) { @@ -855,6 +1021,7 @@ top: // TODO(austin): Break up these workbufs to // better distribute work. pp.gcw.dispose() + // Collect the flushedWork flag. if pp.gcw.flushedWork { atomic.Xadd(&gcMarkDoneFlushed, 1) @@ -872,6 +1039,10 @@ top: goto top } + // For debugging/testing. + for gcDebugMarkDone.spinAfterRaggedBarrier.Load() { + } + // There was no global work, no local work, and no Ps // communicated work since we took markDoneSema. Therefore // there are no grey objects and no more objects can be @@ -909,7 +1080,21 @@ top: } } }) - if restart { + + // Check whether we need to resume the marking phase because of issue #27993 + // or because of goroutine leak detection. + if restart || (work.goroutineLeak.enabled && !work.goroutineLeak.done) { + if restart { + // Restart because of issue #27993. + gcDebugMarkDone.restartedDueTo27993 = true + } else { + // Marking has reached a fixed-point. Attempt to detect goroutine leaks. + // + // If the returned value is true, then detection already concluded for this cycle. + // Otherwise, more runnable goroutines were discovered, requiring additional mark work. + work.goroutineLeak.done = findGoroutineLeaks() + } + getg().m.preemptoff = "" systemstack(func() { // Accumulate the time we were stopped before we had to start again. @@ -936,6 +1121,11 @@ top: // start the world again. gcWakeAllAssists() + // Wake all blocked weak->strong conversions. These will run + // when we start the world again. + work.strongFromWeak.block = false + gcWakeAllStrongFromWeak() + // Likewise, release the transition lock. Blocked // workers and assists will run when we start the // world again. @@ -954,6 +1144,172 @@ top: gcMarkTermination(stw) } +// isMaybeRunnable checks whether a goroutine may still be semantically runnable. +// For goroutines which are semantically runnable, this will eventually return true +// as the GC marking phase progresses. It returns false for leaked goroutines, or for +// goroutines which are not yet computed as possibly runnable by the GC. +func (gp *g) isMaybeRunnable() bool { + // Check whether the goroutine is actually in a waiting state first. + if readgstatus(gp) != _Gwaiting { + // If the goroutine is not waiting, then clearly it is maybe runnable. + return true + } + + switch gp.waitreason { + case waitReasonSelectNoCases, + waitReasonChanSendNilChan, + waitReasonChanReceiveNilChan: + // Select with no cases or communicating on nil channels + // make goroutines unrunnable by definition. + return false + case waitReasonChanReceive, + waitReasonSelect, + waitReasonChanSend: + // Cycle all through all *sudog to check whether + // the goroutine is waiting on a marked channel. + for sg := gp.waiting; sg != nil; sg = sg.waitlink { + if isMarkedOrNotInHeap(unsafe.Pointer(sg.c.get())) { + return true + } + } + return false + case waitReasonSyncCondWait, + waitReasonSyncWaitGroupWait, + waitReasonSyncMutexLock, + waitReasonSyncRWMutexLock, + waitReasonSyncRWMutexRLock: + // If waiting on mutexes, wait groups, or condition variables, + // check if the synchronization primitive attached to the sudog is marked. + if gp.waiting != nil { + return isMarkedOrNotInHeap(gp.waiting.elem.get()) + } + } + return true +} + +// findMaybeRunnableGoroutines checks to see if more blocked but maybe-runnable goroutines exist. +// If so, it adds them into root set and increments work.markrootJobs accordingly. +// Returns true if we need to run another phase of markroots; returns false otherwise. +func findMaybeRunnableGoroutines() (moreWork bool) { + oldRootJobs := work.markrootJobs.Load() + + // To begin with we have a set of unchecked stackRoots between + // vIndex and ivIndex. During the loop, anything < vIndex should be + // valid stackRoots and anything >= ivIndex should be invalid stackRoots. + // The loop terminates when the two indices meet. + var vIndex, ivIndex int = work.nMaybeRunnableStackRoots, work.nStackRoots + // Reorder goroutine list + for vIndex < ivIndex { + if work.stackRoots[vIndex].isMaybeRunnable() { + vIndex = vIndex + 1 + continue + } + for ivIndex = ivIndex - 1; ivIndex != vIndex; ivIndex = ivIndex - 1 { + if gp := work.stackRoots[ivIndex]; gp.isMaybeRunnable() { + work.stackRoots[ivIndex] = work.stackRoots[vIndex] + work.stackRoots[vIndex] = gp + vIndex = vIndex + 1 + break + } + } + } + + newRootJobs := work.baseStacks + uint32(vIndex) + if newRootJobs > oldRootJobs { + work.nMaybeRunnableStackRoots = vIndex + work.markrootJobs.Store(newRootJobs) + } + return newRootJobs > oldRootJobs +} + +// setSyncObjectsUntraceable scans allgs and sets the elem and c fields of all sudogs to +// an untrackable pointer. This prevents the GC from marking these objects as live in memory +// by following these pointers when runnning deadlock detection. +func setSyncObjectsUntraceable() { + assertWorldStopped() + + forEachGRace(func(gp *g) { + // Set as untraceable all synchronization objects of goroutines + // blocked at concurrency operations that could leak. + switch { + case gp.waitreason.isSyncWait(): + // Synchronization primitives are reachable from the *sudog via + // via the elem field. + for sg := gp.waiting; sg != nil; sg = sg.waitlink { + sg.elem.setUntraceable() + } + case gp.waitreason.isChanWait(): + // Channels and select statements are reachable from the *sudog via the c field. + for sg := gp.waiting; sg != nil; sg = sg.waitlink { + sg.c.setUntraceable() + } + } + }) +} + +// gcRestoreSyncObjects restores the elem and c fields of all sudogs to their original values. +// Should be invoked after the goroutine leak detection phase. +func gcRestoreSyncObjects() { + assertWorldStopped() + + forEachGRace(func(gp *g) { + for sg := gp.waiting; sg != nil; sg = sg.waitlink { + sg.elem.setTraceable() + sg.c.setTraceable() + } + }) +} + +// findGoroutineLeaks scans the remaining stackRoots and marks any which are +// blocked over exclusively unreachable concurrency primitives as leaked (deadlocked). +// Returns true if the goroutine leak check was performed (or unnecessary). +// Returns false if the GC cycle has not yet computed all maybe-runnable goroutines. +func findGoroutineLeaks() bool { + assertWorldStopped() + + // Report goroutine leaks and mark them unreachable, and resume marking + // we still need to mark these unreachable *g structs as they + // get reused, but their stack won't get scanned + if work.nMaybeRunnableStackRoots == work.nStackRoots { + // nMaybeRunnableStackRoots == nStackRoots means that all goroutines are marked. + return true + } + + // Check whether any more maybe-runnable goroutines can be found by the GC. + if findMaybeRunnableGoroutines() { + // We found more work, so we need to resume the marking phase. + return false + } + + // For the remaining goroutines, mark them as unreachable and leaked. + work.goroutineLeak.count = work.nStackRoots - work.nMaybeRunnableStackRoots + + for i := work.nMaybeRunnableStackRoots; i < work.nStackRoots; i++ { + gp := work.stackRoots[i] + casgstatus(gp, _Gwaiting, _Gleaked) + + // Add the primitives causing the goroutine leaks + // to the GC work queue, to ensure they are marked. + // + // NOTE(vsaioc): these primitives should also be reachable + // from the goroutine's stack, but let's play it safe. + switch { + case gp.waitreason.isChanWait(): + for sg := gp.waiting; sg != nil; sg = sg.waitlink { + shade(sg.c.uintptr()) + } + case gp.waitreason.isSyncWait(): + for sg := gp.waiting; sg != nil; sg = sg.waitlink { + shade(sg.elem.uintptr()) + } + } + } + // Put the remaining roots as ready for marking and drain them. + work.markrootJobs.Add(int32(work.nStackRoots - work.nMaybeRunnableStackRoots)) + work.nMaybeRunnableStackRoots = work.nStackRoots + return true +} + // World must be stopped and mark assists and background workers must be // disabled. func gcMarkTermination(stw worldStop) { @@ -970,7 +1326,7 @@ func gcMarkTermination(stw worldStop) { // N.B. The execution tracer is not aware of this status // transition and handles it specially based on the // wait reason. - casGToWaitingForGC(curgp, _Grunning, waitReasonGarbageCollection) + casGToWaitingForSuspendG(curgp, _Grunning, waitReasonGarbageCollection) // Run gc on the g0 stack. We do this so that the g stack // we're currently running on will no longer change. Cuts @@ -992,17 +1348,10 @@ func gcMarkTermination(stw worldStop) { systemstack(func() { work.heap2 = work.bytesMarked if debug.gccheckmark > 0 { - // Run a full non-parallel, stop-the-world - // mark using checkmark bits, to check that we - // didn't forget to mark anything during the - // concurrent mark process. - startCheckmarks() - gcResetMarkState() - gcw := &getg().m.p.ptr().gcw - gcDrain(gcw, 0) - wbBufFlush1(getg().m.p.ptr()) - gcw.dispose() - endCheckmarks() + runCheckmark(func(_ *gcWork) { gcPrepareMarkRoots() }) + } + if debug.checkfinalizers > 0 { + checkFinalizersAndCleanups() } // marking is complete so we can turn the write barrier off @@ -1113,7 +1462,18 @@ func gcMarkTermination(stw worldStop) { throw("non-concurrent sweep failed to drain all sweep queues") } + if work.goroutineLeak.enabled { + // Restore the elem and c fields of all sudogs to their original values. + gcRestoreSyncObjects() + } + + var goroutineLeakDone bool systemstack(func() { + // Pull the GC out of goroutine leak detection mode. + work.goroutineLeak.enabled = false + goroutineLeakDone = work.goroutineLeak.done + work.goroutineLeak.done = false + // The memstats updated above must be updated with the world // stopped to ensure consistency of some values, such as // sched.idleTime and sched.totaltime. memstats also include @@ -1150,6 +1510,9 @@ func gcMarkTermination(stw worldStop) { // // Also, flush the pinner cache, to avoid leaking that memory // indefinitely. + if debug.gctrace > 1 { + clear(memstats.lastScanStats[:]) + } forEachP(waitReasonFlushProcCaches, func(pp *p) { pp.mcache.prepareForSweep() if pp.status == _Pidle { @@ -1159,6 +1522,9 @@ func gcMarkTermination(stw worldStop) { unlock(&mheap_.lock) }) } + if debug.gctrace > 1 { + pp.gcw.flushScanStats(&memstats.lastScanStats) + } pp.pinnerCache = nil }) if sl.valid { @@ -1181,7 +1547,11 @@ func gcMarkTermination(stw worldStop) { printlock() print("gc ", memstats.numgc, " @", string(itoaDiv(sbuf[:], uint64(work.tSweepTerm-runtimeInitTime)/1e6, 3)), "s ", - util, "%: ") + util, "%") + if goroutineLeakDone { + print(" (checking for goroutine leaks)") + } + print(": ") prev := work.tSweepTerm for i, ns := range []int64{work.tMark, work.tMarkTerm, work.tEnd} { if i != 0 { @@ -1216,9 +1586,26 @@ func gcMarkTermination(stw worldStop) { print(" (forced)") } print("\n") + + if debug.gctrace > 1 { + dumpScanStats() + } printunlock() } + // Print finalizer/cleanup queue length. Like gctrace, do this before the next GC starts. + // The fact that the next GC might start is not that problematic here, but acts as a convenient + // lock on printing this information (so it cannot overlap with itself from the next GC cycle). + if debug.checkfinalizers > 0 { + fq, fe := finReadQueueStats() + fn := max(int64(fq)-int64(fe), 0) + + cq, ce := gcCleanups.readQueueStats() + cn := max(int64(cq)-int64(ce), 0) + + println("checkfinalizers: queue:", fn, "finalizers +", cn, "cleanups") + } + // Set any arena chunks that were deferred to fault. lock(&userArenaState.lock) faultList := userArenaState.fault @@ -1324,6 +1711,12 @@ type gcBgMarkWorkerNode struct { // gcBgMarkWorker(). m muintptr } +type gcBgMarkWorkerNodePadded struct { + gcBgMarkWorkerNode + pad [tagAlign - unsafe.Sizeof(gcBgMarkWorkerNode{}) - gcBgMarkWorkerNodeRedZoneSize]byte +} + +const gcBgMarkWorkerNodeRedZoneSize = (16 << 2) * asanenabledBit // redZoneSize(512) func gcBgMarkWorker(ready chan struct{}) { gp := getg() @@ -1332,7 +1725,7 @@ func gcBgMarkWorker(ready chan struct{}) { // the stack (see gopark). Prevent deadlock from recursively // starting GC by disabling preemption. gp.m.preemptoff = "GC worker init" - node := new(gcBgMarkWorkerNode) + node := &new(gcBgMarkWorkerNodePadded).gcBgMarkWorkerNode // TODO: technically not allowed in the heap. See comment in tagptr.go. gp.m.preemptoff = "" node.gp.set(gp) @@ -1414,25 +1807,19 @@ func gcBgMarkWorker(ready chan struct{}) { trackLimiterEvent = pp.limiterEvent.start(limiterEventIdleMarkWork, startTime) } - decnwait := atomic.Xadd(&work.nwait, -1) - if decnwait == work.nproc { - println("runtime: work.nwait=", decnwait, "work.nproc=", work.nproc) - throw("work.nwait was > work.nproc") - } + gcBeginWork() systemstack(func() { - // Mark our goroutine preemptible so its stack - // can be scanned. This lets two mark workers - // scan each other (otherwise, they would - // deadlock). We must not modify anything on - // the G stack. However, stack shrinking is - // disabled for mark workers, so it is safe to - // read from the G stack. + // Mark our goroutine preemptible so its stack can be scanned or observed + // by the execution tracer. This, for example, lets two mark workers scan + // each other (otherwise, they would deadlock). // - // N.B. The execution tracer is not aware of this status - // transition and handles it specially based on the - // wait reason. - casGToWaitingForGC(gp, _Grunning, waitReasonGCWorkerActive) + // casGToWaitingForSuspendG marks the goroutine as ineligible for a + // stack shrink, effectively pinning the stack in memory for the duration. + // + // N.B. The execution tracer is not aware of this status transition and + // handles it specially based on the wait reason. + casGToWaitingForSuspendG(gp, _Grunning, waitReasonGCWorkerActive) switch pp.gcMarkWorkerMode { default: throw("gcBgMarkWorker: unexpected gcMarkWorkerMode") @@ -1444,9 +1831,9 @@ func gcBgMarkWorker(ready chan struct{}) { // everything out of the run // queue so it can run // somewhere else. - if drainQ, n := runqdrain(pp); n > 0 { + if drainQ := runqdrain(pp); !drainQ.empty() { lock(&sched.lock) - globrunqputbatch(&drainQ, int32(n)) + globrunqputbatch(&drainQ) unlock(&sched.lock) } } @@ -1472,24 +1859,15 @@ func gcBgMarkWorker(ready chan struct{}) { atomic.Xaddint64(&pp.gcFractionalMarkTime, duration) } - // Was this the last worker and did we run out - // of work? - incnwait := atomic.Xadd(&work.nwait, +1) - if incnwait > work.nproc { - println("runtime: p.gcMarkWorkerMode=", pp.gcMarkWorkerMode, - "work.nwait=", incnwait, "work.nproc=", work.nproc) - throw("work.nwait > work.nproc") - } - // We'll releasem after this point and thus this P may run // something else. We must clear the worker mode to avoid // attributing the mode to a different (non-worker) G in - // traceGoStart. + // tracev2.GoStart. pp.gcMarkWorkerMode = gcMarkWorkerNotWorker // If this worker reached a background mark completion // point, signal the main GC goroutine. - if incnwait == work.nproc && !gcMarkWorkAvailable(nil) { + if gcEndWork() { // We don't need the P-local buffers here, allow // preemption because we may schedule like a regular // goroutine in gcMarkDone (block on locks, etc). @@ -1501,20 +1879,40 @@ func gcBgMarkWorker(ready chan struct{}) { } } -// gcMarkWorkAvailable reports whether executing a mark worker -// on p is potentially useful. p may be nil, in which case it only -// checks the global sources of work. -func gcMarkWorkAvailable(p *p) bool { +// gcShouldScheduleWorker reports whether executing a mark worker +// on p is potentially useful. p may be nil. +func gcShouldScheduleWorker(p *p) bool { if p != nil && !p.gcw.empty() { return true } - if !work.full.empty() { - return true // global work available + return gcMarkWorkAvailable() +} + +// gcIsMarkDone reports whether the mark phase is (probably) done. +func gcIsMarkDone() bool { + return work.nwait == work.nproc && !gcMarkWorkAvailable() +} + +// gcBeginWork signals to the garbage collector that a new worker is +// about to process GC work. +func gcBeginWork() { + decnwait := atomic.Xadd(&work.nwait, -1) + if decnwait == work.nproc { + println("runtime: work.nwait=", decnwait, "work.nproc=", work.nproc) + throw("work.nwait was > work.nproc") } - if work.markrootNext < work.markrootJobs { - return true // root scan work available +} + +// gcEndWork signals to the garbage collector that a new worker has just finished +// its work. It reports whether it was the last worker and there's no more work +// to do. If it returns true, the caller must call gcMarkDone. +func gcEndWork() (last bool) { + incnwait := atomic.Xadd(&work.nwait, +1) + if incnwait > work.nproc { + println("runtime: work.nwait=", incnwait, "work.nproc=", work.nproc) + throw("work.nwait > work.nproc") } - return false + return incnwait == work.nproc && !gcMarkWorkAvailable() } // gcMark runs the mark (or, for concurrent GC, mark termination) @@ -1527,8 +1925,8 @@ func gcMark(startTime int64) { work.tstart = startTime // Check that there's no marking work remaining. - if work.full != 0 || work.markrootNext < work.markrootJobs { - print("runtime: full=", hex(work.full), " next=", work.markrootNext, " jobs=", work.markrootJobs, " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n") + if next, jobs := work.markrootNext.Load(), work.markrootJobs.Load(); work.full != 0 || next < jobs { + print("runtime: full=", hex(work.full), " next=", next, " jobs=", jobs, " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n") panic("non-empty mark queue after concurrent mark") } @@ -1624,7 +2022,7 @@ func gcSweep(mode gcMode) bool { mheap_.sweepgen += 2 sweep.active.reset() mheap_.pagesSwept.Store(0) - mheap_.sweepArenas = mheap_.allArenas + mheap_.sweepArenas = mheap_.heapArenas mheap_.reclaimIndex.Store(0) mheap_.reclaimCredit.Store(0) unlock(&mheap_.lock) @@ -1644,10 +2042,11 @@ func gcSweep(mode gcMode) bool { // Sweep all spans eagerly. for sweepone() != ^uintptr(0) { } - // Free workbufs eagerly. + // Free workbufs and span rings eagerly. prepareFreeWorkbufs() for freeSomeWbufs(false) { } + freeDeadSpanSPMCs() // All "free" events for this mark/sweep cycle have // now happened, so we can make this profile cycle // available immediately. @@ -1687,7 +2086,7 @@ func gcResetMarkState() { // Clear page marks. This is just 1MB per 64GB of heap, so the // time here is pretty trivial. lock(&mheap_.lock) - arenas := mheap_.allArenas + arenas := mheap_.heapArenas unlock(&mheap_.lock) for _, ai := range arenas { ha := mheap_.arenas[ai.l1()][ai.l2()] @@ -1701,8 +2100,7 @@ func gcResetMarkState() { // Hooks for other packages var poolcleanup func() -var boringCaches []unsafe.Pointer // for crypto/internal/boring -var uniqueMapCleanup chan struct{} // for unique +var boringCaches []unsafe.Pointer // for crypto/internal/boring // sync_runtime_registerPoolCleanup should be an internal detail, // but widely used packages access it using linkname. @@ -1723,18 +2121,6 @@ func boring_registerCache(p unsafe.Pointer) { boringCaches = append(boringCaches, p) } -//go:linkname unique_runtime_registerUniqueMapCleanup unique.runtime_registerUniqueMapCleanup -func unique_runtime_registerUniqueMapCleanup(f func()) { - // Start the goroutine in the runtime so it's counted as a system goroutine. - uniqueMapCleanup = make(chan struct{}, 1) - go func(cleanup func()) { - for { - <-uniqueMapCleanup - cleanup() - } - }(f) -} - func clearpools() { // clear sync.Pools if poolcleanup != nil { @@ -1746,14 +2132,6 @@ func clearpools() { atomicstorep(p, nil) } - // clear unique maps - if uniqueMapCleanup != nil { - select { - case uniqueMapCleanup <- struct{}{}: - default: - } - } - // Clear central sudog cache. // Leave per-P caches alone, they have strictly bounded size. // Disconnect cached list before dropping it on the floor, @@ -1859,7 +2237,7 @@ func gcTestIsReachable(ptrs ...unsafe.Pointer) (mask uint64) { s := (*specialReachable)(mheap_.specialReachableAlloc.alloc()) unlock(&mheap_.speciallock) s.special.kind = _KindSpecialReachable - if !addspecial(p, &s.special) { + if !addspecial(p, &s.special, false) { throw("already have a reachable special (duplicate pointer?)") } specials[i] = s diff --git a/src/runtime/mgclimit.go b/src/runtime/mgclimit.go index ad86fbd65bceb3..80aeb71cad6185 100644 --- a/src/runtime/mgclimit.go +++ b/src/runtime/mgclimit.go @@ -209,14 +209,12 @@ func (l *gcCPULimiterState) updateLocked(now int64) { for _, pp := range allp { typ, duration := pp.limiterEvent.consume(now) switch typ { - case limiterEventIdleMarkWork: - fallthrough case limiterEventIdle: - idleTime += duration sched.idleTime.Add(duration) - case limiterEventMarkAssist: - fallthrough - case limiterEventScavengeAssist: + idleTime += duration + case limiterEventIdleMarkWork: + idleTime += duration + case limiterEventMarkAssist, limiterEventScavengeAssist: assistTime += duration case limiterEventNone: break @@ -470,14 +468,12 @@ func (e *limiterEvent) stop(typ limiterEventType, now int64) { } // Account for the event. switch typ { - case limiterEventIdleMarkWork: - gcCPULimiter.addIdleTime(duration) case limiterEventIdle: - gcCPULimiter.addIdleTime(duration) sched.idleTime.Add(duration) - case limiterEventMarkAssist: - fallthrough - case limiterEventScavengeAssist: + gcCPULimiter.addIdleTime(duration) + case limiterEventIdleMarkWork: + gcCPULimiter.addIdleTime(duration) + case limiterEventMarkAssist, limiterEventScavengeAssist: gcCPULimiter.addAssistTime(duration) default: throw("limiterEvent.stop: invalid limiter event type found") diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 9a48d15552f32b..dd76973c623ed8 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -9,6 +9,7 @@ package runtime import ( "internal/abi" "internal/goarch" + "internal/goexperiment" "internal/runtime/atomic" "internal/runtime/sys" "unsafe" @@ -17,6 +18,7 @@ import ( const ( fixedRootFinalizers = iota fixedRootFreeGStacks + fixedRootCleanups fixedRootCount // rootBlockBytes is the number of bytes to scan per data or @@ -48,14 +50,63 @@ const ( // // Must be a multiple of the pageInUse bitmap element size and // must also evenly divide pagesPerArena. - pagesPerSpanRoot = 512 + pagesPerSpanRoot = min(512, pagesPerArena) ) -// gcMarkRootPrepare queues root scanning jobs (stacks, globals, and +// internalBlocked returns true if the goroutine is blocked due to an +// internal (non-leaking) waitReason, e.g. waiting for the netpoller or garbage collector. +// Such goroutines are never leak detection candidates according to the GC. +// +//go:nosplit +func (gp *g) internalBlocked() bool { + reason := gp.waitreason + return reason < waitReasonChanReceiveNilChan || waitReasonSyncWaitGroupWait < reason +} + +// allGsSnapshotSortedForGC takes a snapshot of allgs and returns a sorted +// array of Gs. The array is sorted by the G's status, with running Gs +// first, followed by blocked Gs. The returned index indicates the cutoff +// between runnable and blocked Gs. +// +// The world must be stopped or allglock must be held. +func allGsSnapshotSortedForGC() ([]*g, int) { + assertWorldStoppedOrLockHeld(&allglock) + + // Reset the status of leaked goroutines in order to improve + // the precision of goroutine leak detection. + for _, gp := range allgs { + gp.atomicstatus.CompareAndSwap(_Gleaked, _Gwaiting) + } + + allgsSorted := make([]*g, len(allgs)) + + // Indices cutting off runnable and blocked Gs. + var currIndex, blockedIndex = 0, len(allgsSorted) - 1 + for _, gp := range allgs { + // not sure if we need atomic load because we are stopping the world, + // but do it just to be safe for now + if status := readgstatus(gp); status != _Gwaiting || gp.internalBlocked() { + allgsSorted[currIndex] = gp + currIndex++ + } else { + allgsSorted[blockedIndex] = gp + blockedIndex-- + } + } + + // Because the world is stopped or allglock is held, allgadd + // cannot happen concurrently with this. allgs grows + // monotonically and existing entries never change, so we can + // simply return a copy of the slice header. For added safety, + // we trim everything past len because that can still change. + return allgsSorted, blockedIndex + 1 +} + +// gcPrepareMarkRoots queues root scanning jobs (stacks, globals, and // some miscellany) and initializes scanning-related state. // // The world must be stopped. -func gcMarkRootPrepare() { +func gcPrepareMarkRoots() { assertWorldStopped() // Compute how many data and BSS root blocks there are. @@ -89,9 +140,9 @@ func gcMarkRootPrepare() { // // Break up the work into arenas, and further into chunks. // - // Snapshot allArenas as markArenas. This snapshot is safe because allArenas + // Snapshot heapArenas as markArenas. This snapshot is safe because heapArenas // is append-only. - mheap_.markArenas = mheap_.allArenas[:len(mheap_.allArenas):len(mheap_.allArenas)] + mheap_.markArenas = mheap_.heapArenas[:len(mheap_.heapArenas):len(mheap_.heapArenas)] work.nSpanRoots = len(mheap_.markArenas) * (pagesPerArena / pagesPerSpanRoot) // Scan stacks. @@ -100,11 +151,20 @@ func gcMarkRootPrepare() { // ignore them because they begin life without any roots, so // there's nothing to scan, and any roots they create during // the concurrent phase will be caught by the write barrier. - work.stackRoots = allGsSnapshot() + if work.goroutineLeak.enabled { + // goroutine leak finder GC --- only prepare runnable + // goroutines for marking. + work.stackRoots, work.nMaybeRunnableStackRoots = allGsSnapshotSortedForGC() + } else { + // regular GC --- scan every goroutine + work.stackRoots = allGsSnapshot() + work.nMaybeRunnableStackRoots = len(work.stackRoots) + } + work.nStackRoots = len(work.stackRoots) - work.markrootNext = 0 - work.markrootJobs = uint32(fixedRootCount + work.nDataRoots + work.nBSSRoots + work.nSpanRoots + work.nStackRoots) + work.markrootNext.Store(0) + work.markrootJobs.Store(uint32(fixedRootCount + work.nDataRoots + work.nBSSRoots + work.nSpanRoots + work.nMaybeRunnableStackRoots)) // Calculate base indexes of each root type work.baseData = uint32(fixedRootCount) @@ -117,8 +177,8 @@ func gcMarkRootPrepare() { // gcMarkRootCheck checks that all roots have been scanned. It is // purely for debugging. func gcMarkRootCheck() { - if work.markrootNext < work.markrootJobs { - print(work.markrootNext, " of ", work.markrootJobs, " markroot jobs done\n") + if next, jobs := work.markrootNext.Load(), work.markrootJobs.Load(); next < jobs { + print(next, " of ", jobs, " markroot jobs done\n") throw("left over markroot jobs") } @@ -126,7 +186,7 @@ func gcMarkRootCheck() { // // We only check the first nStackRoots Gs that we should have scanned. // Since we don't care about newer Gs (see comment in - // gcMarkRootPrepare), no locking is required. + // gcPrepareMarkRoots), no locking is required. i := 0 forEachGRace(func(gp *g) { if i >= work.nStackRoots { @@ -186,6 +246,14 @@ func markroot(gcw *gcWork, i uint32, flushBgCredit bool) int64 { // stackfree. systemstack(markrootFreeGStacks) + case i == fixedRootCleanups: + for cb := (*cleanupBlock)(gcCleanups.all.Load()); cb != nil; cb = cb.alllink { + // N.B. This only needs to synchronize with cleanup execution, which only resets these blocks. + // All cleanup queueing happens during sweep. + n := uintptr(atomic.Load(&cb.n)) + scanblock(uintptr(unsafe.Pointer(&cb.cleanups[0])), n*goarch.PtrSize, &cleanupBlockPtrMask[0], gcw, nil) + } + case work.baseSpans <= i && i < work.baseStacks: // mark mspan.specials markrootSpans(gcw, int(i-work.baseSpans)) @@ -217,7 +285,7 @@ func markroot(gcw *gcWork, i uint32, flushBgCredit bool) int64 { userG := getg().m.curg selfScan := gp == userG && readgstatus(userG) == _Grunning if selfScan { - casGToWaitingForGC(userG, _Grunning, waitReasonGarbageCollectionScan) + casGToWaitingForSuspendG(userG, _Grunning, waitReasonGarbageCollectionScan) } // TODO: suspendG blocks (and spins) until gp @@ -299,16 +367,20 @@ func markrootFreeGStacks() { } // Free stacks. - q := gQueue{list.head, list.head} + var tail *g for gp := list.head.ptr(); gp != nil; gp = gp.schedlink.ptr() { + tail = gp stackfree(gp.stack) gp.stack.lo = 0 gp.stack.hi = 0 - // Manipulate the queue directly since the Gs are - // already all linked the right way. - q.tail.set(gp) + if valgrindenabled { + valgrindDeregisterStack(gp.valgrindStackID) + gp.valgrindStackID = 0 + } } + q := gQueue{list.head, tail.guintptr(), list.size} + // Put Gs back on the free list. lock(&sched.gFree.lock) sched.gFree.noStack.pushAll(q) @@ -382,25 +454,13 @@ func markrootSpans(gcw *gcWork, shard int) { for sp := s.specials; sp != nil; sp = sp.next { switch sp.kind { case _KindSpecialFinalizer: - // don't mark finalized object, but scan it so we - // retain everything it points to. - spf := (*specialfinalizer)(unsafe.Pointer(sp)) - // A finalizer can be set for an inner byte of an object, find object beginning. - p := s.base() + uintptr(spf.special.offset)/s.elemsize*s.elemsize - - // Mark everything that can be reached from - // the object (but *not* the object itself or - // we'll never collect it). - if !s.spanclass.noscan() { - scanobject(p, gcw) - } - - // The special itself is a root. - scanblock(uintptr(unsafe.Pointer(&spf.fn)), goarch.PtrSize, &oneptrmask[0], gcw, nil) + gcScanFinalizer((*specialfinalizer)(unsafe.Pointer(sp)), s, gcw) case _KindSpecialWeakHandle: // The special itself is a root. spw := (*specialWeakHandle)(unsafe.Pointer(sp)) scanblock(uintptr(unsafe.Pointer(&spw.handle)), goarch.PtrSize, &oneptrmask[0], gcw, nil) + case _KindSpecialCleanup: + gcScanCleanup((*specialCleanup)(unsafe.Pointer(sp)), gcw) } } unlock(&s.speciallock) @@ -408,6 +468,30 @@ func markrootSpans(gcw *gcWork, shard int) { } } +// gcScanFinalizer scans the relevant parts of a finalizer special as a root. +func gcScanFinalizer(spf *specialfinalizer, s *mspan, gcw *gcWork) { + // Don't mark finalized object, but scan it so we retain everything it points to. + + // A finalizer can be set for an inner byte of an object, find object beginning. + p := s.base() + spf.special.offset/s.elemsize*s.elemsize + + // Mark everything that can be reached from + // the object (but *not* the object itself or + // we'll never collect it). + if !s.spanclass.noscan() { + scanObject(p, gcw) + } + + // The special itself is also a root. + scanblock(uintptr(unsafe.Pointer(&spf.fn)), goarch.PtrSize, &oneptrmask[0], gcw, nil) +} + +// gcScanCleanup scans the relevant parts of a cleanup special as a root. +func gcScanCleanup(spc *specialCleanup, gcw *gcWork) { + // The special itself is a root. + scanblock(uintptr(unsafe.Pointer(&spc.fn)), goarch.PtrSize, &oneptrmask[0], gcw, nil) +} + // gcAssistAlloc performs GC work to make gp's assist debt positive. // gp must be the calling user goroutine. // @@ -422,6 +506,17 @@ func gcAssistAlloc(gp *g) { return } + if gp := getg(); gp.bubble != nil { + // Disassociate the G from its synctest bubble while allocating. + // This is less elegant than incrementing the group's active count, + // but avoids any contamination between GC assist and synctest. + bubble := gp.bubble + gp.bubble = nil + defer func() { + gp.bubble = bubble + }() + } + // This extremely verbose boolean indicates whether we've // entered mark assist from the perspective of the tracer. // @@ -629,6 +724,7 @@ func gcAssistAlloc1(gp *g, scanWork int64) { gp.gcAssistBytes = 0 return } + // Track time spent in this assist. Since we're on the // system stack, this is non-preemptible, so we can // just measure start and end time. @@ -638,14 +734,10 @@ func gcAssistAlloc1(gp *g, scanWork int64) { startTime := nanotime() trackLimiterEvent := gp.m.p.ptr().limiterEvent.start(limiterEventMarkAssist, startTime) - decnwait := atomic.Xadd(&work.nwait, -1) - if decnwait == work.nproc { - println("runtime: work.nwait =", decnwait, "work.nproc=", work.nproc) - throw("nwait > work.nprocs") - } + gcBeginWork() // gcDrainN requires the caller to be preemptible. - casGToWaitingForGC(gp, _Grunning, waitReasonGCAssistMarking) + casGToWaitingForSuspendG(gp, _Grunning, waitReasonGCAssistMarking) // drain own cached work first in the hopes that it // will be more cache friendly. @@ -665,14 +757,7 @@ func gcAssistAlloc1(gp *g, scanWork int64) { // If this is the last worker and we ran out of work, // signal a completion point. - incnwait := atomic.Xadd(&work.nwait, +1) - if incnwait > work.nproc { - println("runtime: work.nwait=", incnwait, - "work.nproc=", work.nproc) - throw("work.nwait > work.nproc") - } - - if incnwait == work.nproc && !gcMarkWorkAvailable(nil) { + if gcEndWork() { // This has reached a background completion point. Set // gp.param to a non-nil value to indicate this. It // doesn't matter what we set it to (it just has to be @@ -826,12 +911,12 @@ func scanstack(gp *g, gcw *gcWork) int64 { default: print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n") throw("mark - bad status") - case _Gdead: + case _Gdead, _Gdeadextra: return 0 case _Grunning: print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n") throw("scanstack: goroutine not stopped") - case _Grunnable, _Gsyscall, _Gwaiting: + case _Grunnable, _Gsyscall, _Gwaiting, _Gleaked: // ok } @@ -945,31 +1030,12 @@ func scanstack(gp *g, gcw *gcWork) int64 { println() printunlock() } - gcdata := r.gcdata() - var s *mspan - if r.useGCProg() { - // This path is pretty unlikely, an object large enough - // to have a GC program allocated on the stack. - // We need some space to unpack the program into a straight - // bitmask, which we allocate/free here. - // TODO: it would be nice if there were a way to run a GC - // program without having to store all its bits. We'd have - // to change from a Lempel-Ziv style program to something else. - // Or we can forbid putting objects on stacks if they require - // a gc program (see issue 27447). - s = materializeGCProg(r.ptrdata(), gcdata) - gcdata = (*byte)(unsafe.Pointer(s.startAddr)) - } - + ptrBytes, gcData := r.gcdata() b := state.stack.lo + uintptr(obj.off) if conservative { - scanConservative(b, r.ptrdata(), gcdata, gcw, &state) + scanConservative(b, ptrBytes, gcData, gcw, &state) } else { - scanblock(b, r.ptrdata(), gcdata, gcw, &state) - } - - if s != nil { - dematerializeGCProg(s) + scanblock(b, ptrBytes, gcData, gcw, &state) } } @@ -1118,6 +1184,28 @@ func gcDrainMarkWorkerFractional(gcw *gcWork) { gcDrain(gcw, gcDrainFractional|gcDrainUntilPreempt|gcDrainFlushBgCredit) } +// gcNextMarkRoot safely increments work.markrootNext and returns the +// index of the next root job. The returned boolean is true if the root job +// is valid, and false if there are no more root jobs to be claimed, +// i.e. work.markrootNext >= work.markrootJobs. +func gcNextMarkRoot() (uint32, bool) { + if !work.goroutineLeak.enabled { + // If not running goroutine leak detection, assume regular GC behavior. + job := work.markrootNext.Add(1) - 1 + return job, job < work.markrootJobs.Load() + } + + // Otherwise, use a CAS loop to increment markrootNext. + for next, jobs := work.markrootNext.Load(), work.markrootJobs.Load(); next < jobs; next = work.markrootNext.Load() { + // There is still work available at the moment. + if work.markrootNext.CompareAndSwap(next, next+1) { + // We manage to snatch a root job. Return the root index. + return next, true + } + } + return 0, false +} + // gcDrain scans roots and objects in work buffers, blackening grey // objects until it is unable to get more work. It may return before // GC is done; it's the caller's responsibility to balance work from @@ -1176,19 +1264,26 @@ func gcDrain(gcw *gcWork, flags gcDrainFlags) { } } - // Drain root marking jobs. - if work.markrootNext < work.markrootJobs { + if work.markrootNext.Load() < work.markrootJobs.Load() { // Stop if we're preemptible, if someone wants to STW, or if // someone is calling forEachP. for !(gp.preempt && (preemptible || sched.gcwaiting.Load() || pp.runSafePointFn != 0)) { - job := atomic.Xadd(&work.markrootNext, +1) - 1 - if job >= work.markrootJobs { + job, ok := gcNextMarkRoot() + if !ok { break } markroot(gcw, job, flushBgCredit) if check != nil && check() { goto done } + + // Spin up a new worker if requested. + if goexperiment.GreenTeaGC && gcw.mayNeedWorker { + gcw.mayNeedWorker = false + if gcphase == _GCmark { + gcController.enlistWorker() + } + } } } @@ -1212,22 +1307,42 @@ func gcDrain(gcw *gcWork, flags gcDrainFlags) { gcw.balance() } - b := gcw.tryGetFast() - if b == 0 { - b = gcw.tryGet() - if b == 0 { - // Flush the write barrier - // buffer; this may create - // more work. - wbBufFlush() - b = gcw.tryGet() + // See mgcwork.go for the rationale behind the order in which we check these queues. + var b uintptr + var s objptr + if b = gcw.tryGetObjFast(); b == 0 { + if s = gcw.tryGetSpanFast(); s == 0 { + if b = gcw.tryGetObj(); b == 0 { + if s = gcw.tryGetSpan(); s == 0 { + // Flush the write barrier + // buffer; this may create + // more work. + wbBufFlush() + if b = gcw.tryGetObj(); b == 0 { + if s = gcw.tryGetSpan(); s == 0 { + s = gcw.tryStealSpan() + } + } + } + } } } - if b == 0 { + if b != 0 { + scanObject(b, gcw) + } else if s != 0 { + scanSpan(s, gcw) + } else { // Unable to get work. break } - scanobject(b, gcw) + + // Spin up a new worker if requested. + if goexperiment.GreenTeaGC && gcw.mayNeedWorker { + gcw.mayNeedWorker = false + if gcphase == _GCmark { + gcController.enlistWorker() + } + } // Flush background scan work credit to the global // account if we've accumulated enough locally so @@ -1292,38 +1407,57 @@ func gcDrainN(gcw *gcWork, scanWork int64) int64 { gcw.balance() } - b := gcw.tryGetFast() - if b == 0 { - b = gcw.tryGet() - if b == 0 { - // Flush the write barrier buffer; - // this may create more work. - wbBufFlush() - b = gcw.tryGet() - } - } - - if b == 0 { - // Try to do a root job. - if work.markrootNext < work.markrootJobs { - job := atomic.Xadd(&work.markrootNext, +1) - 1 - if job < work.markrootJobs { - workFlushed += markroot(gcw, job, false) - continue + // See mgcwork.go for the rationale behind the order in which we check these queues. + var b uintptr + var s objptr + if b = gcw.tryGetObjFast(); b == 0 { + if s = gcw.tryGetSpanFast(); s == 0 { + if b = gcw.tryGetObj(); b == 0 { + if s = gcw.tryGetSpan(); s == 0 { + // Flush the write barrier + // buffer; this may create + // more work. + wbBufFlush() + if b = gcw.tryGetObj(); b == 0 { + if s = gcw.tryGetSpan(); s == 0 { + // Try to do a root job. + if work.markrootNext.Load() < work.markrootJobs.Load() { + job, ok := gcNextMarkRoot() + if ok { + workFlushed += markroot(gcw, job, false) + continue + } + } + s = gcw.tryStealSpan() + } + } + } } } - // No heap or root jobs. + } + if b != 0 { + scanObject(b, gcw) + } else if s != 0 { + scanSpan(s, gcw) + } else { + // Unable to get work. break } - scanobject(b, gcw) - // Flush background scan work credit. if gcw.heapScanWork >= gcCreditSlack { gcController.heapScanWork.Add(gcw.heapScanWork) workFlushed += gcw.heapScanWork gcw.heapScanWork = 0 } + + // Spin up a new worker if requested. + if goexperiment.GreenTeaGC && gcw.mayNeedWorker { + gcw.mayNeedWorker = false + if gcphase == _GCmark { + gcController.enlistWorker() + } + } } // Unlike gcDrain, there's no need to flush remaining work @@ -1333,7 +1467,7 @@ func gcDrainN(gcw *gcWork, scanWork int64) int64 { return workFlushed + gcw.heapScanWork } -// scanblock scans b as scanobject would, but using an explicit +// scanblock scans b as scanObject would, but using an explicit // pointer bitmap instead of the heap bitmap. // // This is used to scan non-heap roots, so it does not update @@ -1358,13 +1492,17 @@ func scanblock(b0, n0 uintptr, ptrmask *uint8, gcw *gcWork, stk *stackScanState) } for j := 0; j < 8 && i < n; j++ { if bits&1 != 0 { - // Same work as in scanobject; see comments there. + // Same work as in scanObject; see comments there. p := *(*uintptr)(unsafe.Pointer(b + i)) if p != 0 { - if obj, span, objIndex := findObject(p, b, i); obj != 0 { - greyobject(obj, b, i, span, gcw, objIndex) - } else if stk != nil && p >= stk.stack.lo && p < stk.stack.hi { + if stk != nil && p >= stk.stack.lo && p < stk.stack.hi { stk.putPtr(p, false) + } else { + if !tryDeferToSpanScan(p, gcw) { + if obj, span, objIndex := findObject(p, b, i); obj != 0 { + greyobject(obj, b, i, span, gcw, objIndex) + } + } } } } @@ -1374,102 +1512,6 @@ func scanblock(b0, n0 uintptr, ptrmask *uint8, gcw *gcWork, stk *stackScanState) } } -// scanobject scans the object starting at b, adding pointers to gcw. -// b must point to the beginning of a heap object or an oblet. -// scanobject consults the GC bitmap for the pointer mask and the -// spans for the size of the object. -// -//go:nowritebarrier -func scanobject(b uintptr, gcw *gcWork) { - // Prefetch object before we scan it. - // - // This will overlap fetching the beginning of the object with initial - // setup before we start scanning the object. - sys.Prefetch(b) - - // Find the bits for b and the size of the object at b. - // - // b is either the beginning of an object, in which case this - // is the size of the object to scan, or it points to an - // oblet, in which case we compute the size to scan below. - s := spanOfUnchecked(b) - n := s.elemsize - if n == 0 { - throw("scanobject n == 0") - } - if s.spanclass.noscan() { - // Correctness-wise this is ok, but it's inefficient - // if noscan objects reach here. - throw("scanobject of a noscan object") - } - - var tp typePointers - if n > maxObletBytes { - // Large object. Break into oblets for better - // parallelism and lower latency. - if b == s.base() { - // Enqueue the other oblets to scan later. - // Some oblets may be in b's scalar tail, but - // these will be marked as "no more pointers", - // so we'll drop out immediately when we go to - // scan those. - for oblet := b + maxObletBytes; oblet < s.base()+s.elemsize; oblet += maxObletBytes { - if !gcw.putFast(oblet) { - gcw.put(oblet) - } - } - } - - // Compute the size of the oblet. Since this object - // must be a large object, s.base() is the beginning - // of the object. - n = s.base() + s.elemsize - b - n = min(n, maxObletBytes) - tp = s.typePointersOfUnchecked(s.base()) - tp = tp.fastForward(b-tp.addr, b+n) - } else { - tp = s.typePointersOfUnchecked(b) - } - - var scanSize uintptr - for { - var addr uintptr - if tp, addr = tp.nextFast(); addr == 0 { - if tp, addr = tp.next(b + n); addr == 0 { - break - } - } - - // Keep track of farthest pointer we found, so we can - // update heapScanWork. TODO: is there a better metric, - // now that we can skip scalar portions pretty efficiently? - scanSize = addr - b + goarch.PtrSize - - // Work here is duplicated in scanblock and above. - // If you make changes here, make changes there too. - obj := *(*uintptr)(unsafe.Pointer(addr)) - - // At this point we have extracted the next potential pointer. - // Quickly filter out nil and pointers back to the current object. - if obj != 0 && obj-b >= n { - // Test if obj points into the Go heap and, if so, - // mark the object. - // - // Note that it's possible for findObject to - // fail if obj points to a just-allocated heap - // object because of a race with growing the - // heap. In this case, we know the object was - // just allocated and hence will be marked by - // allocation itself. - if obj, span, objIndex := findObject(obj, b, addr-b); obj != 0 { - greyobject(obj, b, addr-b, span, gcw, objIndex) - } - } - } - gcw.bytesMarked += uint64(n) - gcw.heapScanWork += int64(scanSize) -} - // scanConservative scans block [b, b+n) conservatively, treating any // pointer-like value in the block as a pointer. // @@ -1501,7 +1543,7 @@ func scanConservative(b, n uintptr, ptrmask *uint8, gcw *gcWork, state *stackSca return ' ' } idx := span.objIndex(val) - if span.isFree(idx) { + if span.isFreeOrNewlyAllocated(idx) { return ' ' } return '*' @@ -1554,14 +1596,19 @@ func scanConservative(b, n uintptr, ptrmask *uint8, gcw *gcWork, state *stackSca } // Check if val points to an allocated object. + // + // Ignore objects allocated during the mark phase, they've + // been allocated black. idx := span.objIndex(val) - if span.isFree(idx) { + if span.isFreeOrNewlyAllocated(idx) { continue } // val points to an allocated object. Mark it. obj := span.base() + idx*span.elemsize - greyobject(obj, b, i, span, gcw, idx) + if !tryDeferToSpanScan(obj, gcw) { + greyobject(obj, b, i, span, gcw, idx) + } } } @@ -1571,9 +1618,11 @@ func scanConservative(b, n uintptr, ptrmask *uint8, gcw *gcWork, state *stackSca // //go:nowritebarrier func shade(b uintptr) { - if obj, span, objIndex := findObject(b, 0, 0); obj != 0 { - gcw := &getg().m.p.ptr().gcw - greyobject(obj, 0, 0, span, gcw, objIndex) + gcw := &getg().m.p.ptr().gcw + if !tryDeferToSpanScan(b, gcw) { + if obj, span, objIndex := findObject(b, 0, 0); obj != 0 { + greyobject(obj, 0, 0, span, gcw, objIndex) + } } } @@ -1596,6 +1645,9 @@ func greyobject(obj, base, off uintptr, span *mspan, gcw *gcWork, objIndex uintp // Already marked. return } + if debug.checkfinalizers > 1 { + print(" mark ", hex(obj), " found at *(", hex(base), "+", hex(off), ")\n") + } } else { if debug.gccheckmark > 0 && span.isFree(objIndex) { print("runtime: marking free object ", hex(obj), " found at *(", hex(base), "+", hex(off), ")\n") @@ -1616,13 +1668,13 @@ func greyobject(obj, base, off uintptr, span *mspan, gcw *gcWork, objIndex uintp if arena.pageMarks[pageIdx]&pageMask == 0 { atomic.Or8(&arena.pageMarks[pageIdx], pageMask) } + } - // If this is a noscan object, fast-track it to black - // instead of greying it. - if span.spanclass.noscan() { - gcw.bytesMarked += uint64(span.elemsize) - return - } + // If this is a noscan object, fast-track it to black + // instead of greying it. + if span.spanclass.noscan() { + gcw.bytesMarked += uint64(span.elemsize) + return } // We're adding obj to P's local workbuf, so it's likely @@ -1631,8 +1683,8 @@ func greyobject(obj, base, off uintptr, span *mspan, gcw *gcWork, objIndex uintp // some benefit on platforms with inclusive shared caches. sys.Prefetch(obj) // Queue the obj for scanning. - if !gcw.putFast(obj) { - gcw.put(obj) + if !gcw.putObjFast(obj) { + gcw.putObj(obj) } } @@ -1694,10 +1746,18 @@ func gcmarknewobject(span *mspan, obj uintptr) { if useCheckmark { // The world should be stopped so this should not happen. throw("gcmarknewobject called while doing checkmark") } + if gcphase == _GCmarktermination { + // Check this here instead of on the hot path. + throw("mallocgc called with gcphase == _GCmarktermination") + } // Mark object. objIndex := span.objIndex(obj) span.markBitsForIndex(objIndex).setMarked() + if goexperiment.GreenTeaGC && gcUsesSpanInlineMarkBits(span.elemsize) { + // No need to scan the new object. + span.scannedBitsForIndex(objIndex).setMarked() + } // Mark span. arena, pageIdx, pageMask := pageIndexOf(span.base()) @@ -1720,8 +1780,10 @@ func gcMarkTinyAllocs() { if c == nil || c.tiny == 0 { continue } - _, span, objIndex := findObject(c.tiny, 0, 0) gcw := &p.gcw - greyobject(c.tiny, 0, 0, span, gcw, objIndex) + if !tryDeferToSpanScan(c.tiny, gcw) { + _, span, objIndex := findObject(c.tiny, 0, 0) + greyobject(c.tiny, 0, 0, span, gcw, objIndex) + } } } diff --git a/src/runtime/mgcmark_greenteagc.go b/src/runtime/mgcmark_greenteagc.go new file mode 100644 index 00000000000000..7b78611cf7b6fe --- /dev/null +++ b/src/runtime/mgcmark_greenteagc.go @@ -0,0 +1,1311 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Green Tea mark algorithm +// +// The core idea behind Green Tea is simple: achieve better locality during +// mark/scan by delaying scanning so that we can accumulate objects to scan +// within the same span, then scan the objects that have accumulated on the +// span all together. +// +// By batching objects this way, we increase the chance that adjacent objects +// will be accessed, amortize the cost of accessing object metadata, and create +// better opportunities for prefetching. We can take this even further and +// optimize the scan loop by size class (not yet completed) all the way to the +// point of applying SIMD techniques to really tear through the heap. +// +// Naturally, this depends on being able to create opportunties to batch objects +// together. The basic idea here is to have two sets of mark bits. One set is the +// regular set of mark bits ("marks"), while the other essentially says that the +// objects have been scanned already ("scans"). When we see a pointer for the first +// time we set its mark and enqueue its span. We track these spans in work queues +// with a FIFO policy, unlike workbufs which have a LIFO policy. Empirically, a +// FIFO policy appears to work best for accumulating objects to scan on a span. +// Later, when we dequeue the span, we find both the union and intersection of the +// mark and scan bitsets. The union is then written back into the scan bits, while +// the intersection is used to decide which objects need scanning, such that the GC +// is still precise. +// +// Below is the bulk of the implementation, focusing on the worst case +// for locality, small objects. Specifically, those that are smaller than +// a few cache lines in size and whose metadata is stored the same way (at the +// end of the span). + +//go:build goexperiment.greenteagc + +package runtime + +import ( + "internal/goarch" + "internal/runtime/atomic" + "internal/runtime/gc" + "internal/runtime/gc/scan" + "internal/runtime/sys" + "unsafe" +) + +const doubleCheckGreenTea = false + +// spanInlineMarkBits are mark bits that are inlined into the span +// itself. gcUsesSpanInlineMarkBits may be used to check if objects +// of a particular size use inline mark bits. +// +// Inline mark bits are a little bit more than just mark bits. They +// consist of two parts: scans and marks. Marks are like pre-mark +// bits. They're set once a pointer to an object is discovered for +// the first time. The marks allow us to scan many objects in bulk +// if we queue the whole span for scanning. Before we scan such objects +// in bulk, we copy the marks to the scans, computing a diff along the +// way. The resulting bitmap tells us which objects we should scan. +// +// The inlineMarkBits also hold state sufficient for scanning any +// object in the span, as well as state for acquiring ownership of +// the span for queuing. This avoids the need to look at the mspan when +// scanning. +type spanInlineMarkBits struct { + scans [63]uint8 // scanned bits. + owned spanScanOwnership // see the comment on spanScanOwnership. + marks [63]uint8 // mark bits. + class spanClass +} + +// spanScanOwnership indicates whether some thread has acquired +// the span for scanning, and whether there has been one or more +// attempts to acquire the span. The latter information helps to +// fast-track span scans that only apply to a single mark, skipping +// the relatively costly merge-and-diff process for scans and marks +// by allowing one to just set the mark directly. +type spanScanOwnership uint8 + +const ( + spanScanUnowned spanScanOwnership = 0 // Indicates the span is not acquired for scanning. + spanScanOneMark = 1 << iota // Indicates that only one mark bit is set relative to the scan bits. + spanScanManyMark // Indicates one or more scan bits may be set relative to the mark bits. + // "ManyMark" need not be exactly the value it has. In practice we just + // want to distinguish "none" from "one" from "many," so a comparison is + // sufficient (as opposed to a bit test) to check between these cases. +) + +// load atomically loads from a pointer to a spanScanOwnership. +func (o *spanScanOwnership) load() spanScanOwnership { + return spanScanOwnership(atomic.Load8((*uint8)(unsafe.Pointer(o)))) +} + +func (o *spanScanOwnership) or(v spanScanOwnership) spanScanOwnership { + // N.B. We round down the address and use Or32 because Or8 doesn't + // return a result, and it's strictly necessary for this protocol. + // + // Making Or8 return a result, while making the code look nicer, would + // not be strictly better on any supported platform, as an Or8 that + // returns a result is not a common instruction. On many platforms it + // would be implemented exactly as it is here, and since Or8 is + // exclusively used in the runtime and a hot function, we want to keep + // using its no-result version elsewhere for performance. + o32 := (*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(o)) &^ 0b11)) + off := (uintptr(unsafe.Pointer(o)) & 0b11) * 8 + if goarch.BigEndian { + off = 32 - off - 8 + } + return spanScanOwnership(atomic.Or32(o32, uint32(v)<> off) +} + +func (imb *spanInlineMarkBits) init(class spanClass, needzero bool) { + if imb == nil { + // This nil check and throw is almost pointless. Normally we would + // expect imb to never be nil. However, this is called on potentially + // freshly-allocated virtual memory. As of 2025, the compiler-inserted + // nil check is not a branch but a memory read that we expect to fault + // if the pointer really is nil. + // + // However, this causes a read of the page, and operating systems may + // take it as a hint to back the accessed memory with a read-only zero + // page. However, we immediately write to this memory, which can then + // force operating systems to have to update the page table and flush + // the TLB, causing a lot of churn for programs that are short-lived + // and monotonically grow in size. + // + // This nil check is thus an explicit branch instead of what the compiler + // would insert circa 2025, which is a memory read instruction. + // + // See go.dev/issue/74375 for details. + throw("runtime: span inline mark bits nil?") + } + if needzero { + // Use memclrNoHeapPointers to avoid having the compiler make a worse + // decision. We know that imb is both aligned and a nice power-of-two + // size that works well for wider SIMD instructions. The compiler likely + // has no idea that imb is aligned to 128 bytes. + memclrNoHeapPointers(unsafe.Pointer(imb), unsafe.Sizeof(spanInlineMarkBits{})) + } + imb.class = class +} + +// tryAcquire attempts to acquire the span for scanning. On success, the caller +// must queue the span for scanning or scan the span immediately. +func (imb *spanInlineMarkBits) tryAcquire() bool { + switch imb.owned.load() { + case spanScanUnowned: + // Try to mark the span as having only one object marked. + if imb.owned.or(spanScanOneMark) == spanScanUnowned { + return true + } + // If we didn't see an old value of spanScanUnowned, then we must + // have raced with someone else and seen spanScanOneMark or greater. + // Fall through and try to set spanScanManyMark. + fallthrough + case spanScanOneMark: + // We may be the first to set *any* bit on owned. In such a case, + // we still need to make sure the span is queued. + return imb.owned.or(spanScanManyMark) == spanScanUnowned + } + return false +} + +// release releases the span for scanning, allowing another thread to queue the span. +// +// Returns an upper bound on the number of mark bits set since the span was queued. The +// upper bound is described as "one" (spanScanOneMark) or "many" (spanScanManyMark, with or +// without spanScanOneMark). If the return value indicates only one mark bit was set, the +// caller can be certain that it was the same mark bit that caused the span to get queued. +// Take note of the fact that this is *only* an upper-bound. In particular, it may still +// turn out that only one mark bit was set, even if the return value indicates "many". +func (imb *spanInlineMarkBits) release() spanScanOwnership { + return spanScanOwnership(atomic.Xchg8((*uint8)(unsafe.Pointer(&imb.owned)), uint8(spanScanUnowned))) +} + +// spanInlineMarkBitsFromBase returns the spanInlineMarkBits for a span whose start address is base. +// +// The span must be gcUsesSpanInlineMarkBits(span.elemsize). +func spanInlineMarkBitsFromBase(base uintptr) *spanInlineMarkBits { + return (*spanInlineMarkBits)(unsafe.Pointer(base + gc.PageSize - unsafe.Sizeof(spanInlineMarkBits{}))) +} + +// initInlineMarkBits initializes the inlineMarkBits stored at the end of the span. +func (s *mspan) initInlineMarkBits() { + if doubleCheckGreenTea && !gcUsesSpanInlineMarkBits(s.elemsize) { + throw("expected span with inline mark bits") + } + // Zeroing is only necessary if this span wasn't just freshly allocated from the OS. + s.inlineMarkBits().init(s.spanclass, s.needzero != 0) +} + +// moveInlineMarks merges the span's inline mark bits into dst and clears them. +// +// gcUsesSpanInlineMarkBits(s.elemsize) must be true. +func (s *mspan) moveInlineMarks(dst *gcBits) { + if doubleCheckGreenTea && !gcUsesSpanInlineMarkBits(s.elemsize) { + throw("expected span with inline mark bits") + } + bytes := divRoundUp(uintptr(s.nelems), 8) + imb := s.inlineMarkBits() + imbMarks := (*gc.ObjMask)(unsafe.Pointer(&imb.marks)) + for i := uintptr(0); i < bytes; i += goarch.PtrSize { + marks := bswapIfBigEndian(imbMarks[i/goarch.PtrSize]) + if i/goarch.PtrSize == uintptr(len(imb.marks)+1)/goarch.PtrSize-1 { + marks &^= 0xff << ((goarch.PtrSize - 1) * 8) // mask out class + } + *(*uintptr)(unsafe.Pointer(dst.bytep(i))) |= bswapIfBigEndian(marks) + } + if doubleCheckGreenTea && !s.spanclass.noscan() && imb.marks != imb.scans { + throw("marks don't match scans for span with pointer") + } + + // Reset the inline mark bits. + imb.init(s.spanclass, true /* We know these bits are always dirty now. */) +} + +// inlineMarkBits returns the inline mark bits for the span. +// +// gcUsesSpanInlineMarkBits(s.elemsize) must be true. +func (s *mspan) inlineMarkBits() *spanInlineMarkBits { + if doubleCheckGreenTea && !gcUsesSpanInlineMarkBits(s.elemsize) { + throw("expected span with inline mark bits") + } + return spanInlineMarkBitsFromBase(s.base()) +} + +func (s *mspan) markBitsForIndex(objIndex uintptr) (bits markBits) { + if gcUsesSpanInlineMarkBits(s.elemsize) { + bits.bytep = &s.inlineMarkBits().marks[objIndex/8] + } else { + bits.bytep = s.gcmarkBits.bytep(objIndex / 8) + } + bits.mask = uint8(1) << (objIndex % 8) + bits.index = objIndex + return +} + +func (s *mspan) markBitsForBase() markBits { + if gcUsesSpanInlineMarkBits(s.elemsize) { + return markBits{&s.inlineMarkBits().marks[0], uint8(1), 0} + } + return markBits{&s.gcmarkBits.x, uint8(1), 0} +} + +// scannedBitsForIndex returns a markBits representing the scanned bit +// for objIndex in the inline mark bits. +func (s *mspan) scannedBitsForIndex(objIndex uintptr) markBits { + return markBits{&s.inlineMarkBits().scans[objIndex/8], uint8(1) << (objIndex % 8), objIndex} +} + +// gcUsesSpanInlineMarkBits returns true if a span holding objects of a certain size +// has inline mark bits. size must be the span's elemsize. +// +// nosplit because this is called from gcmarknewobject, which is nosplit. +// +//go:nosplit +func gcUsesSpanInlineMarkBits(size uintptr) bool { + return heapBitsInSpan(size) && size >= 16 +} + +// tryDeferToSpanScan tries to queue p on the span it points to, if it +// points to a small object span (gcUsesSpanQueue size). +func tryDeferToSpanScan(p uintptr, gcw *gcWork) bool { + if useCheckmark { + return false + } + + // Quickly to see if this is a span that has inline mark bits. + ha := heapArenaOf(p) + if ha == nil { + return false + } + pageIdx := ((p / pageSize) / 8) % uintptr(len(ha.pageInUse)) + pageMask := byte(1 << ((p / pageSize) % 8)) + if ha.pageUseSpanInlineMarkBits[pageIdx]&pageMask == 0 { + return false + } + + // Find the object's index from the span class info stored in the inline mark bits. + base := alignDown(p, gc.PageSize) + q := spanInlineMarkBitsFromBase(base) + objIndex := uint16((uint64(p-base) * uint64(gc.SizeClassToDivMagic[q.class.sizeclass()])) >> 32) + + // Set mark bit. + idx, mask := objIndex/8, uint8(1)<<(objIndex%8) + if atomic.Load8(&q.marks[idx])&mask != 0 { + return true + } + atomic.Or8(&q.marks[idx], mask) + + // Fast-track noscan objects. + if q.class.noscan() { + gcw.bytesMarked += uint64(gc.SizeClassToSize[q.class.sizeclass()]) + return true + } + + // Queue up the pointer (as a representative for its span). + if q.tryAcquire() { + if gcw.spanq.put(makeObjPtr(base, objIndex)) { + if gcphase == _GCmark { + // This is intentionally racy; the bit set here might get + // stomped on by a stealing P. See the comment in tryStealSpan + // for an explanation as to why this is OK. + if !work.spanqMask.read(uint32(gcw.id)) { + work.spanqMask.set(gcw.id) + } + gcw.mayNeedWorker = true + } + gcw.flushedWork = true + } + } + return true +} + +// tryGetSpanFast attempts to get an entire span to scan. +func (w *gcWork) tryGetSpanFast() objptr { + return w.spanq.tryGetFast() +} + +// tryGetSpan attempts to get an entire span to scan. +func (w *gcWork) tryGetSpan() objptr { + if s := w.spanq.tryGetFast(); s != 0 { + return s + } + // "Steal" from ourselves. + if s := w.spanq.steal(&w.spanq); s != 0 { + return s + } + // We failed to get any local work, so we're fresh out. + // Nobody else is going to add work for us. Clear our bit. + if work.spanqMask.read(uint32(w.id)) { + work.spanqMask.clear(w.id) + } + return 0 +} + +// spanQueue is a P-local stealable span queue. +type spanQueue struct { + // head, tail, and ring represent a local non-thread-safe ring buffer. + head, tail uint32 + ring [256]objptr + + // putsSinceDrain counts the number of put calls since the last drain. + putsSinceDrain int + + // chain contains state visible to other Ps. + // + // In particular, that means a linked chain of single-producer multi-consumer + // ring buffers where the single producer is this P only. + // + // This linked chain structure is based off the sync.Pool dequeue. + chain struct { + // head is the spanSPMC to put to. This is only accessed + // by the producer, so doesn't need to be synchronized. + head *spanSPMC + + // tail is the spanSPMC to steal from. This is accessed + // by consumers, so reads and writes must be atomic. + tail atomic.UnsafePointer // *spanSPMC + } +} + +// putFast tries to put s onto the queue, but may fail if it's full. +func (q *spanQueue) putFast(s objptr) (ok bool) { + if q.tail-q.head == uint32(len(q.ring)) { + return false + } + q.ring[q.tail%uint32(len(q.ring))] = s + q.tail++ + return true +} + +// put puts s onto the queue. +// +// Returns whether the caller should spin up a new worker. +func (q *spanQueue) put(s objptr) bool { + // The constants below define the period of and volume of + // spans we spill to the spmc chain when the local queue is + // not full. + // + // spillPeriod must be > spillMax, otherwise that sets the + // effective maximum size of our local span queue. Even if + // we have a span ring of size N, but we flush K spans every + // K puts, then K becomes our effective maximum length. When + // spillPeriod > spillMax, then we're always spilling spans + // at a slower rate than we're accumulating them. + const ( + // spillPeriod defines how often to check if we should + // spill some spans, counted in the number of calls to put. + spillPeriod = 64 + + // spillMax defines, at most, how many spans to drain with + // each spill. + spillMax = 16 + ) + + if q.putFast(s) { + // Occasionally try to spill some work to generate parallelism. + q.putsSinceDrain++ + if q.putsSinceDrain >= spillPeriod { + // Reset even if we don't drain, so we don't check every time. + q.putsSinceDrain = 0 + + // Try to drain some spans. Don't bother if there's very + // few of them or there's already spans in the spmc chain. + n := min((q.tail-q.head)/2, spillMax) + if n > 4 && q.chainEmpty() { + q.drain(n) + return true + } + } + return false + } + + // We're out of space. Drain out our local spans. + q.drain(uint32(len(q.ring)) / 2) + if !q.putFast(s) { + throw("failed putFast after drain") + } + return true +} + +// flush publishes all spans in the local queue to the spmc chain. +func (q *spanQueue) flush() { + n := q.tail - q.head + if n == 0 { + return + } + q.drain(n) +} + +// empty returns true if there's no more work on the queue. +// +// Not thread-safe. Must only be called by the owner of q. +func (q *spanQueue) empty() bool { + // Check the local queue for work. + if q.tail-q.head > 0 { + return false + } + return q.chainEmpty() +} + +// chainEmpty returns true if the spmc chain is empty. +// +// Thread-safe. +func (q *spanQueue) chainEmpty() bool { + // Check the rest of the rings for work. + r := (*spanSPMC)(q.chain.tail.Load()) + for r != nil { + if !r.empty() { + return false + } + r = (*spanSPMC)(r.prev.Load()) + } + return true +} + +// drain publishes n spans from the local queue to the spmc chain. +func (q *spanQueue) drain(n uint32) { + q.putsSinceDrain = 0 + + if q.chain.head == nil { + // N.B. We target 1024, but this may be bigger if the physical + // page size is bigger, or if we can fit more uintptrs into a + // physical page. See newSpanSPMC docs. + r := newSpanSPMC(1024) + q.chain.head = r + q.chain.tail.StoreNoWB(unsafe.Pointer(r)) + } + + // Try to drain some of the queue to the head spmc. + if q.tryDrain(q.chain.head, n) { + return + } + // No space. Create a bigger spmc and add it to the chain. + + // Double the size of the next one, up to a maximum. + // + // We double each time so we can avoid taking this slow path + // in the future, which involves a global lock. Ideally we want + // to hit a steady-state where the deepest any queue goes during + // a mark phase can fit in the ring. + // + // However, we still set a maximum on this. We set the maximum + // to something large to amortize the cost of lock acquisition, but + // still at a reasonable size for big heaps and/or a lot of Ps (which + // tend to be correlated). + // + // It's not too bad to burn relatively large-but-fixed amounts of per-P + // memory if we need to deal with really, really deep queues, since the + // constants of proportionality are small. Simultaneously, we want to + // avoid a situation where a single worker ends up queuing O(heap) + // work and then forever retains a queue of that size. + const maxCap = 1 << 20 / goarch.PtrSize + newCap := q.chain.head.cap * 2 + if newCap > maxCap { + newCap = maxCap + } + newHead := newSpanSPMC(newCap) + if !q.tryDrain(newHead, n) { + throw("failed to put span on newly-allocated spanSPMC") + } + q.chain.head.prev.StoreNoWB(unsafe.Pointer(newHead)) + q.chain.head = newHead +} + +// tryDrain attempts to drain n spans from q's local queue to the chain. +// +// Returns whether it succeeded. +func (q *spanQueue) tryDrain(r *spanSPMC, n uint32) bool { + if q.head+n > q.tail { + throw("attempt to drain too many elements") + } + h := r.head.Load() // synchronize with consumers + t := r.tail.Load() + rn := t - h + if rn+n <= r.cap { + for i := uint32(0); i < n; i++ { + *r.slot(t + i) = q.ring[(q.head+i)%uint32(len(q.ring))] + } + r.tail.Store(t + n) // Makes the items avail for consumption. + q.head += n + return true + } + return false +} + +// tryGetFast attempts to get a span from the local queue, but may fail if it's empty, +// returning false. +func (q *spanQueue) tryGetFast() objptr { + if q.tail-q.head == 0 { + return 0 + } + s := q.ring[q.head%uint32(len(q.ring))] + q.head++ + return s +} + +// steal takes some spans from the ring chain of another span queue. +// +// q == q2 is OK. +func (q *spanQueue) steal(q2 *spanQueue) objptr { + r := (*spanSPMC)(q2.chain.tail.Load()) + if r == nil { + return 0 + } + for { + // It's important that we load the next pointer + // *before* popping the tail. In general, r may be + // transiently empty, but if next is non-nil before + // the pop and the pop fails, then r is permanently + // empty, which is the only condition under which it's + // safe to drop r from the chain. + r2 := (*spanSPMC)(r.prev.Load()) + + // Try to refill from one of the rings + if s := q.refill(r); s != 0 { + return s + } + + if r2 == nil { + // This is the only ring. It's empty right + // now, but could be pushed to in the future. + return 0 + } + + // The tail of the chain has been drained, so move on + // to the next ring. Try to drop it from the chain + // so the next consumer doesn't have to look at the empty + // ring again. + if q2.chain.tail.CompareAndSwapNoWB(unsafe.Pointer(r), unsafe.Pointer(r2)) { + r.dead.Store(true) + } + + r = r2 + } +} + +// refill takes some spans from r and puts them into q's local queue. +// +// One span is removed from the stolen spans and returned on success. +// Failure to steal returns a zero objptr. +// +// steal is thread-safe with respect to r. +func (q *spanQueue) refill(r *spanSPMC) objptr { + if q.tail-q.head != 0 { + throw("steal with local work available") + } + + // Steal some spans. + var n uint32 + for { + h := r.head.Load() // load-acquire, synchronize with other consumers + t := r.tail.Load() // load-acquire, synchronize with the producer + n = t - h + n = n - n/2 + if n == 0 { + return 0 + } + if n > r.cap { // read inconsistent h and t + continue + } + n = min(n, uint32(len(q.ring)/2)) + for i := uint32(0); i < n; i++ { + q.ring[i] = *r.slot(h + i) + } + if r.head.CompareAndSwap(h, h+n) { + break + } + } + + // Update local queue head and tail to reflect new buffered values. + q.head = 0 + q.tail = n + + // Pop off the head of the queue and return it. + return q.tryGetFast() +} + +// destroy frees all chains in an empty spanQueue. +// +// Preconditions: +// - World is stopped. +// - GC is outside of the mark phase. +// - (Therefore) the queue is empty. +func (q *spanQueue) destroy() { + assertWorldStopped() + if gcphase != _GCoff { + throw("spanQueue.destroy during the mark phase") + } + if !q.empty() { + throw("spanQueue.destroy on non-empty queue") + } + + lock(&work.spanSPMCs.lock) + + // Remove and free each ring. + for r := (*spanSPMC)(q.chain.tail.Load()); r != nil; r = (*spanSPMC)(r.prev.Load()) { + prev := r.allprev + next := r.allnext + if prev != nil { + prev.allnext = next + } + if next != nil { + next.allprev = prev + } + if work.spanSPMCs.all == r { + work.spanSPMCs.all = next + } + + r.deinit() + mheap_.spanSPMCAlloc.free(unsafe.Pointer(r)) + } + + q.chain.head = nil + q.chain.tail.Store(nil) + q.putsSinceDrain = 0 + + unlock(&work.spanSPMCs.lock) +} + +// spanSPMC is a ring buffer of objptrs that represent spans. +// Accessed without a lock. +// +// Single-producer, multi-consumer. The only producer is the P that owns this +// queue, but any other P may consume from it. +// +// ## Invariants for memory management +// +// 1. All spanSPMCs are allocated from mheap_.spanSPMCAlloc. +// 2. All allocated spanSPMCs must be on the work.spanSPMCs list. +// 3. spanSPMCs may only be allocated if gcphase != _GCoff. +// 4. spanSPMCs may only be deallocated if gcphase == _GCoff. +// +// Invariants (3) and (4) ensure that we do not need to concern ourselves with +// tricky reuse issues that stem from not knowing when a thread is truly done +// with a spanSPMC. For example, two threads could load the same spanSPMC from +// the tail of the chain. One thread is then paused while the other steals the +// last few elements off of it. It's not safe to free at that point since the +// other thread will still inspect that spanSPMC, and we have no way of knowing +// without more complex and/or heavyweight synchronization. +// +// Instead, we rely on the global synchronization inherent to GC phases, and +// the fact that spanSPMCs are only ever used during the mark phase, to ensure +// memory safety. This means we temporarily waste some memory, but it's only +// until the end of the mark phase. +type spanSPMC struct { + _ sys.NotInHeap + + // allnext is the link to the next spanSPMC on the work.spanSPMCs list. + // This is used to find and free dead spanSPMCs. Protected by + // work.spanSPMCs.lock. + allnext *spanSPMC + + // allprev is the link to the previous spanSPMC on the work.spanSPMCs + // list. This is used to find and free dead spanSPMCs. Protected by + // work.spanSPMCs.lock. + allprev *spanSPMC + + // dead indicates whether the spanSPMC is no longer in use. + // Protected by the CAS to the prev field of the spanSPMC pointing + // to this spanSPMC. That is, whoever wins that CAS takes ownership + // of marking this spanSPMC as dead. See spanQueue.steal for details. + dead atomic.Bool + + // prev is the next link up a spanQueue's SPMC chain, from tail to head, + // hence the name "prev." Set by a spanQueue's producer, cleared by a + // CAS in spanQueue.steal. + prev atomic.UnsafePointer // *spanSPMC + + // head, tail, cap, and ring together represent a fixed-size SPMC lock-free + // ring buffer of size cap. The ring buffer contains objptr values. + head atomic.Uint32 + tail atomic.Uint32 + cap uint32 // cap(ring)) + ring *objptr +} + +// newSpanSPMC allocates and initializes a new spmc with the provided capacity. +// +// newSpanSPMC may override the capacity with a larger one if the provided one would +// waste memory. +func newSpanSPMC(cap uint32) *spanSPMC { + lock(&work.spanSPMCs.lock) + r := (*spanSPMC)(mheap_.spanSPMCAlloc.alloc()) + next := work.spanSPMCs.all + r.allnext = next + if next != nil { + next.allprev = r + } + work.spanSPMCs.all = r + unlock(&work.spanSPMCs.lock) + + // If cap < the capacity of a single physical page, round up. + pageCap := uint32(physPageSize / goarch.PtrSize) // capacity of a single page + if cap < pageCap { + cap = pageCap + } + if cap&(cap-1) != 0 { + throw("spmc capacity must be a power of 2") + } + + r.cap = cap + ring := sysAlloc(uintptr(cap)*unsafe.Sizeof(objptr(0)), &memstats.gcMiscSys, "GC span queue") + atomic.StorepNoWB(unsafe.Pointer(&r.ring), ring) + return r +} + +// empty returns true if the spmc is empty. +// +// empty is thread-safe. +func (r *spanSPMC) empty() bool { + h := r.head.Load() + t := r.tail.Load() + return t == h +} + +// deinit frees any resources the spanSPMC is holding onto and zeroes it. +func (r *spanSPMC) deinit() { + sysFree(unsafe.Pointer(r.ring), uintptr(r.cap)*unsafe.Sizeof(objptr(0)), &memstats.gcMiscSys) + r.ring = nil + r.dead.Store(false) + r.prev.StoreNoWB(nil) + r.head.Store(0) + r.tail.Store(0) + r.cap = 0 + r.allnext = nil + r.allprev = nil +} + +// slot returns a pointer to slot i%r.cap. +func (r *spanSPMC) slot(i uint32) *objptr { + idx := uintptr(i & (r.cap - 1)) + return (*objptr)(unsafe.Add(unsafe.Pointer(r.ring), idx*unsafe.Sizeof(objptr(0)))) +} + +// freeDeadSpanSPMCs frees dead spanSPMCs back to the OS. +func freeDeadSpanSPMCs() { + // According to the SPMC memory management invariants, we can only free + // spanSPMCs outside of the mark phase. We ensure we do this in two ways. + // + // 1. We take the work.spanSPMCs lock, which we need anyway. This ensures + // that we are non-preemptible. If this path becomes lock-free, we will + // need to become non-preemptible in some other way. + // 2. Once we are non-preemptible, we check the gcphase, and back out if + // it's not safe. + // + // This way, we ensure that we don't start freeing if we're in the wrong + // phase, and the phase can't change on us while we're freeing. + // + // TODO(go.dev/issue/75771): Due to the grow semantics in + // spanQueue.drain, we expect a steady-state of around one spanSPMC per + // P, with some spikes higher when Ps have more than one. For high + // GOMAXPROCS, or if this list otherwise gets long, it would be nice to + // have a way to batch work that allows preemption during processing. + lock(&work.spanSPMCs.lock) + if gcphase != _GCoff || work.spanSPMCs.all == nil { + unlock(&work.spanSPMCs.lock) + return + } + r := work.spanSPMCs.all + for r != nil { + next := r.allnext + if r.dead.Load() { + // It's dead. Deinitialize and free it. + prev := r.allprev + if prev != nil { + prev.allnext = next + } + if next != nil { + next.allprev = prev + } + if work.spanSPMCs.all == r { + work.spanSPMCs.all = next + } + + r.deinit() + mheap_.spanSPMCAlloc.free(unsafe.Pointer(r)) + } + r = next + } + unlock(&work.spanSPMCs.lock) +} + +// tryStealSpan attempts to steal a span from another P's local queue. +// +// Returns a non-zero objptr on success. +func (w *gcWork) tryStealSpan() objptr { + pp := getg().m.p.ptr() + + for enum := stealOrder.start(cheaprand()); !enum.done(); enum.next() { + if !work.spanqMask.read(enum.position()) { + continue + } + p2 := allp[enum.position()] + if pp == p2 { + continue + } + if s := w.spanq.steal(&p2.gcw.spanq); s != 0 { + return s + } + // N.B. This is intentionally racy. We may stomp on a mask set by + // a P that just put a bunch of work into its local queue. + // + // This is OK because the ragged barrier in gcMarkDone will set + // the bit on each P if there's local work we missed. This race + // should generally be rare, since the window between noticing + // an empty local queue and this bit being set is quite small. + work.spanqMask.clear(int32(enum.position())) + } + return 0 +} + +// objptr consists of a span base and the index of the object in the span. +type objptr uintptr + +// makeObjPtr creates an objptr from a span base address and an object index. +func makeObjPtr(spanBase uintptr, objIndex uint16) objptr { + if doubleCheckGreenTea && spanBase&((1< 1 { + gcw.stats[spanclass.sizeclass()].sparseObjsScanned++ + } + b := spanBase + uintptr(objIndex)*elemsize + scanObjectSmall(spanBase, b, elemsize, gcw) + return + } + + // Compute nelems. + divMagic := uint64(gc.SizeClassToDivMagic[spanclass.sizeclass()]) + usableSpanSize := uint64(gc.PageSize - unsafe.Sizeof(spanInlineMarkBits{})) + if !spanclass.noscan() { + usableSpanSize -= gc.PageSize / goarch.PtrSize / 8 + } + nelems := uint16((usableSpanSize * divMagic) >> 32) + + // Grey objects and return if there's nothing else to do. + var toScan gc.ObjMask + objsMarked := spanSetScans(spanBase, nelems, imb, &toScan) + if objsMarked == 0 { + return + } + gcw.bytesMarked += uint64(objsMarked) * uint64(elemsize) + + // Check if we have enough density to make a dartboard scan + // worthwhile. If not, just do what scanobject does, but + // localized to the span, using the dartboard. + if !scan.HasFastScanSpanPacked() || objsMarked < int(nelems/8) { + if debug.gctrace > 1 { + gcw.stats[spanclass.sizeclass()].spansSparseScanned++ + gcw.stats[spanclass.sizeclass()].spanObjsSparseScanned += uint64(objsMarked) + } + scanObjectsSmall(spanBase, elemsize, nelems, gcw, &toScan) + return + } + + // Scan the span. + // + // N.B. Use gcw.ptrBuf as the output buffer. This is a bit different + // from scanObjectsSmall, which puts addresses to dereference. ScanSpanPacked + // on the other hand, fills gcw.ptrBuf with already dereferenced pointers. + nptrs := scan.ScanSpanPacked( + unsafe.Pointer(spanBase), + &gcw.ptrBuf[0], + &toScan, + uintptr(spanclass.sizeclass()), + spanPtrMaskUnsafe(spanBase), + ) + gcw.heapScanWork += int64(objsMarked) * int64(elemsize) + + if debug.gctrace > 1 { + // Write down some statistics. + gcw.stats[spanclass.sizeclass()].spansDenseScanned++ + gcw.stats[spanclass.sizeclass()].spanObjsDenseScanned += uint64(objsMarked) + } + + // Process all the pointers we just got. + for _, p := range gcw.ptrBuf[:nptrs] { + if !tryDeferToSpanScan(p, gcw) { + if obj, span, objIndex := findObject(p, 0, 0); obj != 0 { + greyobject(obj, 0, 0, span, gcw, objIndex) + } + } + } +} + +// spanSetScans sets any unset mark bits that have their mark bits set in the inline mark bits. +// +// toScan is populated with bits indicating whether a particular mark bit was set. +// +// Returns the number of objects marked, which could be zero. +func spanSetScans(spanBase uintptr, nelems uint16, imb *spanInlineMarkBits, toScan *gc.ObjMask) int { + arena, pageIdx, pageMask := pageIndexOf(spanBase) + if arena.pageMarks[pageIdx]&pageMask == 0 { + atomic.Or8(&arena.pageMarks[pageIdx], pageMask) + } + + bytes := divRoundUp(uintptr(nelems), 8) + objsMarked := 0 + + // Careful: these two structures alias since ObjMask is much bigger + // than marks or scans. We do these unsafe shenanigans so that we can + // access the marks and scans by uintptrs rather than by byte. + imbMarks := (*gc.ObjMask)(unsafe.Pointer(&imb.marks)) + imbScans := (*gc.ObjMask)(unsafe.Pointer(&imb.scans)) + + // Iterate over one uintptr-sized chunks at a time, computing both + // the union and intersection of marks and scans. Store the union + // into scans, and the intersection into toScan. + for i := uintptr(0); i < bytes; i += goarch.PtrSize { + scans := atomic.Loaduintptr(&imbScans[i/goarch.PtrSize]) + marks := imbMarks[i/goarch.PtrSize] + scans = bswapIfBigEndian(scans) + marks = bswapIfBigEndian(marks) + if i/goarch.PtrSize == uintptr(len(imb.marks)+1)/goarch.PtrSize-1 { + scans &^= 0xff << ((goarch.PtrSize - 1) * 8) // mask out owned + marks &^= 0xff << ((goarch.PtrSize - 1) * 8) // mask out class + } + toGrey := marks &^ scans + toScan[i/goarch.PtrSize] = toGrey + + // If there's anything left to grey, do it. + if toGrey != 0 { + toGrey = bswapIfBigEndian(toGrey) + if goarch.PtrSize == 4 { + atomic.Or32((*uint32)(unsafe.Pointer(&imbScans[i/goarch.PtrSize])), uint32(toGrey)) + } else { + atomic.Or64((*uint64)(unsafe.Pointer(&imbScans[i/goarch.PtrSize])), uint64(toGrey)) + } + } + objsMarked += sys.OnesCount64(uint64(toGrey)) + } + return objsMarked +} + +func scanObjectSmall(spanBase, b, objSize uintptr, gcw *gcWork) { + ptrBits := heapBitsSmallForAddrInline(spanBase, b, objSize) + gcw.heapScanWork += int64(sys.Len64(uint64(ptrBits)) * goarch.PtrSize) + nptrs := 0 + n := sys.OnesCount64(uint64(ptrBits)) + for range n { + k := sys.TrailingZeros64(uint64(ptrBits)) + ptrBits &^= 1 << k + addr := b + uintptr(k)*goarch.PtrSize + + // Prefetch addr since we're about to use it. This point for prefetching + // was chosen empirically. + sys.Prefetch(addr) + + // N.B. ptrBuf is always large enough to hold pointers for an entire 1-page span. + gcw.ptrBuf[nptrs] = addr + nptrs++ + } + + // Process all the pointers we just got. + for _, p := range gcw.ptrBuf[:nptrs] { + p = *(*uintptr)(unsafe.Pointer(p)) + if p == 0 { + continue + } + if !tryDeferToSpanScan(p, gcw) { + if obj, span, objIndex := findObject(p, 0, 0); obj != 0 { + greyobject(obj, 0, 0, span, gcw, objIndex) + } + } + } +} + +func scanObjectsSmall(base, objSize uintptr, elems uint16, gcw *gcWork, scans *gc.ObjMask) { + nptrs := 0 + for i, bits := range scans { + if i*(goarch.PtrSize*8) > int(elems) { + break + } + n := sys.OnesCount64(uint64(bits)) + for range n { + j := sys.TrailingZeros64(uint64(bits)) + bits &^= 1 << j + + b := base + uintptr(i*(goarch.PtrSize*8)+j)*objSize + ptrBits := heapBitsSmallForAddrInline(base, b, objSize) + gcw.heapScanWork += int64(sys.Len64(uint64(ptrBits)) * goarch.PtrSize) + + n := sys.OnesCount64(uint64(ptrBits)) + for range n { + k := sys.TrailingZeros64(uint64(ptrBits)) + ptrBits &^= 1 << k + addr := b + uintptr(k)*goarch.PtrSize + + // Prefetch addr since we're about to use it. This point for prefetching + // was chosen empirically. + sys.Prefetch(addr) + + // N.B. ptrBuf is always large enough to hold pointers for an entire 1-page span. + gcw.ptrBuf[nptrs] = addr + nptrs++ + } + } + } + + // Process all the pointers we just got. + for _, p := range gcw.ptrBuf[:nptrs] { + p = *(*uintptr)(unsafe.Pointer(p)) + if p == 0 { + continue + } + if !tryDeferToSpanScan(p, gcw) { + if obj, span, objIndex := findObject(p, 0, 0); obj != 0 { + greyobject(obj, 0, 0, span, gcw, objIndex) + } + } + } +} + +func heapBitsSmallForAddrInline(spanBase, addr, elemsize uintptr) uintptr { + hbitsBase, _ := spanHeapBitsRange(spanBase, gc.PageSize, elemsize) + hbits := (*byte)(unsafe.Pointer(hbitsBase)) + + // These objects are always small enough that their bitmaps + // fit in a single word, so just load the word or two we need. + // + // Mirrors mspan.writeHeapBitsSmall. + // + // We should be using heapBits(), but unfortunately it introduces + // both bounds checks panics and throw which causes us to exceed + // the nosplit limit in quite a few cases. + i := (addr - spanBase) / goarch.PtrSize / ptrBits + j := (addr - spanBase) / goarch.PtrSize % ptrBits + bits := elemsize / goarch.PtrSize + word0 := (*uintptr)(unsafe.Pointer(addb(hbits, goarch.PtrSize*(i+0)))) + word1 := (*uintptr)(unsafe.Pointer(addb(hbits, goarch.PtrSize*(i+1)))) + + var read uintptr + if j+bits > ptrBits { + // Two reads. + bits0 := ptrBits - j + bits1 := bits - bits0 + read = *word0 >> j + read |= (*word1 & ((1 << bits1) - 1)) << bits0 + } else { + // One read. + read = (*word0 >> j) & ((1 << bits) - 1) + } + return read +} + +// spanPtrMaskUnsafe returns the pointer mask for a span with inline mark bits. +// +// The caller must ensure spanBase is the base of a span that: +// - 1 page in size, +// - Uses inline mark bits, +// - Contains pointers. +func spanPtrMaskUnsafe(spanBase uintptr) *gc.PtrMask { + base := spanBase + gc.PageSize - unsafe.Sizeof(gc.PtrMask{}) - unsafe.Sizeof(spanInlineMarkBits{}) + return (*gc.PtrMask)(unsafe.Pointer(base)) +} + +type sizeClassScanStats struct { + spansDenseScanned uint64 // Spans scanned with ScanSpanPacked. + spanObjsDenseScanned uint64 // Objects scanned with ScanSpanPacked. + spansSparseScanned uint64 // Spans scanned with scanObjectsSmall. + spanObjsSparseScanned uint64 // Objects scanned with scanObjectsSmall. + sparseObjsScanned uint64 // Objects scanned with scanobject or scanObjectSmall. + // Note: sparseObjsScanned is sufficient for both cases because + // a particular size class either uses scanobject or scanObjectSmall, + // not both. In the latter case, we also know that there was one + // object scanned per span, so no need for a span counter. +} + +func dumpScanStats() { + var ( + spansDenseScanned uint64 + spanObjsDenseScanned uint64 + spansSparseScanned uint64 + spanObjsSparseScanned uint64 + sparseObjsScanned uint64 + ) + for _, stats := range memstats.lastScanStats { + spansDenseScanned += stats.spansDenseScanned + spanObjsDenseScanned += stats.spanObjsDenseScanned + spansSparseScanned += stats.spansSparseScanned + spanObjsSparseScanned += stats.spanObjsSparseScanned + sparseObjsScanned += stats.sparseObjsScanned + } + totalObjs := sparseObjsScanned + spanObjsSparseScanned + spanObjsDenseScanned + totalSpans := spansSparseScanned + spansDenseScanned + print("scan: total ", sparseObjsScanned, "+", spanObjsSparseScanned, "+", spanObjsDenseScanned, "=", totalObjs, " objs") + print(", ", spansSparseScanned, "+", spansDenseScanned, "=", totalSpans, " spans\n") + for i, stats := range memstats.lastScanStats { + if stats == (sizeClassScanStats{}) { + continue + } + totalObjs := stats.sparseObjsScanned + stats.spanObjsSparseScanned + stats.spanObjsDenseScanned + totalSpans := stats.spansSparseScanned + stats.spansDenseScanned + if i == 0 { + print("scan: class L ") + } else { + print("scan: class ", gc.SizeClassToSize[i], "B ") + } + print(stats.sparseObjsScanned, "+", stats.spanObjsSparseScanned, "+", stats.spanObjsDenseScanned, "=", totalObjs, " objs") + print(", ", stats.spansSparseScanned, "+", stats.spansDenseScanned, "=", totalSpans, " spans\n") + } +} + +func (w *gcWork) flushScanStats(dst *[gc.NumSizeClasses]sizeClassScanStats) { + for i := range w.stats { + dst[i].spansDenseScanned += w.stats[i].spansDenseScanned + dst[i].spanObjsDenseScanned += w.stats[i].spanObjsDenseScanned + dst[i].spansSparseScanned += w.stats[i].spansSparseScanned + dst[i].spanObjsSparseScanned += w.stats[i].spanObjsSparseScanned + dst[i].sparseObjsScanned += w.stats[i].sparseObjsScanned + } + clear(w.stats[:]) +} + +// gcMarkWorkAvailable reports whether there's any non-local work available to do. +// +// This is a heavyweight check and must only be used for correctness, not +// as a hint. +func gcMarkWorkAvailable() bool { + if !work.full.empty() { + return true // global work available + } + if work.markrootNext.Load() < work.markrootJobs.Load() { + return true // root scan work available + } + if work.spanqMask.any() { + return true // stealable local work available + } + return false +} + +// scanObject scans the object starting at b, adding pointers to gcw. +// b must point to the beginning of a heap object or an oblet. +// scanObject consults the GC bitmap for the pointer mask and the +// spans for the size of the object. +// +// Used only for !gcUsesSpanInlineMarkBits spans, but supports all +// object sizes and is safe to be called on all heap objects. +// +//go:nowritebarrier +func scanObject(b uintptr, gcw *gcWork) { + // Prefetch object before we scan it. + // + // This will overlap fetching the beginning of the object with initial + // setup before we start scanning the object. + sys.Prefetch(b) + + // Find the bits for b and the size of the object at b. + // + // b is either the beginning of an object, in which case this + // is the size of the object to scan, or it points to an + // oblet, in which case we compute the size to scan below. + s := spanOfUnchecked(b) + n := s.elemsize + if n == 0 { + throw("scanObject n == 0") + } + if s.spanclass.noscan() { + // Correctness-wise this is ok, but it's inefficient + // if noscan objects reach here. + throw("scanObject of a noscan object") + } + + var tp typePointers + if n > maxObletBytes { + // Large object. Break into oblets for better + // parallelism and lower latency. + if b == s.base() { + // Enqueue the other oblets to scan later. + // Some oblets may be in b's scalar tail, but + // these will be marked as "no more pointers", + // so we'll drop out immediately when we go to + // scan those. + for oblet := b + maxObletBytes; oblet < s.base()+s.elemsize; oblet += maxObletBytes { + if !gcw.putObjFast(oblet) { + gcw.putObj(oblet) + } + } + } + + // Compute the size of the oblet. Since this object + // must be a large object, s.base() is the beginning + // of the object. + n = s.base() + s.elemsize - b + n = min(n, maxObletBytes) + tp = s.typePointersOfUnchecked(s.base()) + tp = tp.fastForward(b-tp.addr, b+n) + } else { + tp = s.typePointersOfUnchecked(b) + } + + var scanSize uintptr + for { + var addr uintptr + if tp, addr = tp.nextFast(); addr == 0 { + if tp, addr = tp.next(b + n); addr == 0 { + break + } + } + + // Keep track of farthest pointer we found, so we can + // update heapScanWork. TODO: is there a better metric, + // now that we can skip scalar portions pretty efficiently? + scanSize = addr - b + goarch.PtrSize + + // Work here is duplicated in scanblock and above. + // If you make changes here, make changes there too. + obj := *(*uintptr)(unsafe.Pointer(addr)) + + // At this point we have extracted the next potential pointer. + // Quickly filter out nil and pointers back to the current object. + if obj != 0 && obj-b >= n { + // Test if obj points into the Go heap and, if so, + // mark the object. + // + // Note that it's possible for findObject to + // fail if obj points to a just-allocated heap + // object because of a race with growing the + // heap. In this case, we know the object was + // just allocated and hence will be marked by + // allocation itself. + if !tryDeferToSpanScan(obj, gcw) { + if obj, span, objIndex := findObject(obj, b, addr-b); obj != 0 { + greyobject(obj, b, addr-b, span, gcw, objIndex) + } + } + } + } + gcw.bytesMarked += uint64(n) + gcw.heapScanWork += int64(scanSize) + if debug.gctrace > 1 { + gcw.stats[s.spanclass.sizeclass()].sparseObjsScanned++ + } +} diff --git a/src/runtime/mgcmark_nogreenteagc.go b/src/runtime/mgcmark_nogreenteagc.go new file mode 100644 index 00000000000000..024565ef3e250d --- /dev/null +++ b/src/runtime/mgcmark_nogreenteagc.go @@ -0,0 +1,235 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !goexperiment.greenteagc + +package runtime + +import ( + "internal/goarch" + "internal/runtime/gc" + "internal/runtime/sys" + "unsafe" +) + +func (s *mspan) markBitsForIndex(objIndex uintptr) markBits { + bytep, mask := s.gcmarkBits.bitp(objIndex) + return markBits{bytep, mask, objIndex} +} + +func (s *mspan) markBitsForBase() markBits { + return markBits{&s.gcmarkBits.x, uint8(1), 0} +} + +func tryDeferToSpanScan(p uintptr, gcw *gcWork) bool { + return false +} + +func (s *mspan) initInlineMarkBits() { +} + +func (s *mspan) moveInlineMarks(to *gcBits) { + throw("unimplemented") +} + +func gcUsesSpanInlineMarkBits(_ uintptr) bool { + return false +} + +func (s *mspan) inlineMarkBits() *spanInlineMarkBits { + return nil +} + +func (s *mspan) scannedBitsForIndex(objIndex uintptr) markBits { + throw("unimplemented") + return markBits{} +} + +type spanInlineMarkBits struct { +} + +func (q *spanInlineMarkBits) tryAcquire() bool { + return false +} + +type spanQueue struct { +} + +func (q *spanQueue) flush() { +} + +func (q *spanQueue) empty() bool { + return true +} + +func (q *spanQueue) destroy() { +} + +type spanSPMC struct { + _ sys.NotInHeap +} + +func freeDeadSpanSPMCs() { + return +} + +type objptr uintptr + +func (w *gcWork) tryGetSpanFast() objptr { + return 0 +} + +func (w *gcWork) tryGetSpan() objptr { + return 0 +} + +func (w *gcWork) tryStealSpan() objptr { + return 0 +} + +func scanSpan(p objptr, gcw *gcWork) { + throw("unimplemented") +} + +type sizeClassScanStats struct { + sparseObjsScanned uint64 +} + +func dumpScanStats() { + var sparseObjsScanned uint64 + for _, stats := range memstats.lastScanStats { + sparseObjsScanned += stats.sparseObjsScanned + } + print("scan: total ", sparseObjsScanned, " objs\n") + for i, stats := range memstats.lastScanStats { + if stats == (sizeClassScanStats{}) { + continue + } + if i == 0 { + print("scan: class L ") + } else { + print("scan: class ", gc.SizeClassToSize[i], "B ") + } + print(stats.sparseObjsScanned, " objs\n") + } +} + +func (w *gcWork) flushScanStats(dst *[gc.NumSizeClasses]sizeClassScanStats) { + for i := range w.stats { + dst[i].sparseObjsScanned += w.stats[i].sparseObjsScanned + } + clear(w.stats[:]) +} + +// gcMarkWorkAvailable reports whether there's any non-local work available to do. +func gcMarkWorkAvailable() bool { + if !work.full.empty() { + return true // global work available + } + if work.markrootNext.Load() < work.markrootJobs.Load() { + return true // root scan work available + } + return false +} + +// scanObject scans the object starting at b, adding pointers to gcw. +// b must point to the beginning of a heap object or an oblet. +// scanObject consults the GC bitmap for the pointer mask and the +// spans for the size of the object. +// +//go:nowritebarrier +func scanObject(b uintptr, gcw *gcWork) { + // Prefetch object before we scan it. + // + // This will overlap fetching the beginning of the object with initial + // setup before we start scanning the object. + sys.Prefetch(b) + + // Find the bits for b and the size of the object at b. + // + // b is either the beginning of an object, in which case this + // is the size of the object to scan, or it points to an + // oblet, in which case we compute the size to scan below. + s := spanOfUnchecked(b) + n := s.elemsize + if n == 0 { + throw("scanObject n == 0") + } + if s.spanclass.noscan() { + // Correctness-wise this is ok, but it's inefficient + // if noscan objects reach here. + throw("scanObject of a noscan object") + } + + var tp typePointers + if n > maxObletBytes { + // Large object. Break into oblets for better + // parallelism and lower latency. + if b == s.base() { + // Enqueue the other oblets to scan later. + // Some oblets may be in b's scalar tail, but + // these will be marked as "no more pointers", + // so we'll drop out immediately when we go to + // scan those. + for oblet := b + maxObletBytes; oblet < s.base()+s.elemsize; oblet += maxObletBytes { + if !gcw.putObjFast(oblet) { + gcw.putObj(oblet) + } + } + } + + // Compute the size of the oblet. Since this object + // must be a large object, s.base() is the beginning + // of the object. + n = s.base() + s.elemsize - b + n = min(n, maxObletBytes) + tp = s.typePointersOfUnchecked(s.base()) + tp = tp.fastForward(b-tp.addr, b+n) + } else { + tp = s.typePointersOfUnchecked(b) + } + + var scanSize uintptr + for { + var addr uintptr + if tp, addr = tp.nextFast(); addr == 0 { + if tp, addr = tp.next(b + n); addr == 0 { + break + } + } + + // Keep track of farthest pointer we found, so we can + // update heapScanWork. TODO: is there a better metric, + // now that we can skip scalar portions pretty efficiently? + scanSize = addr - b + goarch.PtrSize + + // Work here is duplicated in scanblock and above. + // If you make changes here, make changes there too. + obj := *(*uintptr)(unsafe.Pointer(addr)) + + // At this point we have extracted the next potential pointer. + // Quickly filter out nil and pointers back to the current object. + if obj != 0 && obj-b >= n { + // Test if obj points into the Go heap and, if so, + // mark the object. + // + // Note that it's possible for findObject to + // fail if obj points to a just-allocated heap + // object because of a race with growing the + // heap. In this case, we know the object was + // just allocated and hence will be marked by + // allocation itself. + if !tryDeferToSpanScan(obj, gcw) { + if obj, span, objIndex := findObject(obj, b, addr-b); obj != 0 { + greyobject(obj, b, addr-b, span, gcw, objIndex) + } + } + } + } + gcw.bytesMarked += uint64(n) + gcw.heapScanWork += int64(scanSize) + if debug.gctrace > 1 { + gcw.stats[s.spanclass.sizeclass()].sparseObjsScanned++ + } +} diff --git a/src/runtime/mgcpacer.go b/src/runtime/mgcpacer.go index cda87fe9484483..17e2f405e48296 100644 --- a/src/runtime/mgcpacer.go +++ b/src/runtime/mgcpacer.go @@ -8,6 +8,8 @@ import ( "internal/cpu" "internal/goexperiment" "internal/runtime/atomic" + "internal/runtime/math" + "internal/runtime/strconv" _ "unsafe" // for go:linkname ) @@ -687,21 +689,42 @@ func (c *gcControllerState) endCycle(now int64, procs int, userForced bool) { // another P if there are spare worker slots. It is used by putfull // when more work is made available. // +// If goexperiment.GreenTeaGC, the caller must not hold a G's scan bit, +// otherwise this could cause a deadlock. This is already enforced by +// the static lock ranking. +// //go:nowritebarrier func (c *gcControllerState) enlistWorker() { - // If there are idle Ps, wake one so it will run an idle worker. - // NOTE: This is suspected of causing deadlocks. See golang.org/issue/19112. + needDedicated := c.dedicatedMarkWorkersNeeded.Load() > 0 + + // Create new workers from idle Ps with goexperiment.GreenTeaGC. // - // if sched.npidle.Load() != 0 && sched.nmspinning.Load() == 0 { - // wakep() - // return - // } - - // There are no idle Ps. If we need more dedicated workers, - // try to preempt a running P so it will switch to a worker. - if c.dedicatedMarkWorkersNeeded.Load() <= 0 { + // Note: with Green Tea, this places a requirement on enlistWorker + // that it must not be called while a G's scan bit is held. + if goexperiment.GreenTeaGC { + needIdle := c.needIdleMarkWorker() + + // If we're all full on dedicated and idle workers, nothing + // to do. + if !needDedicated && !needIdle { + return + } + + // If there are idle Ps, wake one so it will run a worker + // (the scheduler will already prefer to spin up a new + // dedicated worker over an idle one). + if sched.npidle.Load() != 0 && sched.nmspinning.Load() == 0 { + wakep() // Likely to consume our worker request. + return + } + } + + // If we still need more dedicated workers, try to preempt a running P + // so it will switch to a worker. + if !needDedicated { return } + // Pick a random other P to preempt. if gomaxprocs <= 1 { return @@ -744,14 +767,25 @@ func (c *gcControllerState) findRunnableGCWorker(pp *p, now int64) (*g, int64) { gcCPULimiter.update(now) } - if !gcMarkWorkAvailable(pp) { - // No work to be done right now. This can happen at + if !gcShouldScheduleWorker(pp) { + // No good reason to schedule a worker. This can happen at // the end of the mark phase when there are still // assists tapering off. Don't bother running a worker // now because it'll just return immediately. return nil, now } + if c.dedicatedMarkWorkersNeeded.Load() <= 0 && c.fractionalUtilizationGoal == 0 { + // No current need for dedicated workers, and no need at all for + // fractional workers. Check before trying to acquire a worker; when + // GOMAXPROCS is large, that can be expensive and is often unnecessary. + // + // When a dedicated worker stops running, the gcBgMarkWorker loop notes + // the need for the worker before returning it to the pool. If we don't + // see the need now, we wouldn't have found it in the pool anyway. + return nil, now + } + // Grab a worker before we commit to running below. node := (*gcBgMarkWorkerNode)(gcBgMarkWorkerPool.pop()) if node == nil { @@ -1279,7 +1313,7 @@ func readGOGC() int32 { if p == "off" { return -1 } - if n, ok := atoi32(p); ok { + if n, ok := strconv.Atoi32(p); ok { return n } return 100 @@ -1323,7 +1357,7 @@ func setMemoryLimit(in int64) (out int64) { func readGOMEMLIMIT() int64 { p := gogetenv("GOMEMLIMIT") if p == "" || p == "off" { - return maxInt64 + return math.MaxInt64 } n, ok := parseByteCount(p) if !ok { diff --git a/src/runtime/mgcpacer_test.go b/src/runtime/mgcpacer_test.go index ef1483d629ea13..4b9cbf558909d4 100644 --- a/src/runtime/mgcpacer_test.go +++ b/src/runtime/mgcpacer_test.go @@ -603,7 +603,6 @@ func TestGcPacer(t *testing.T) { // However, it is still possible to trigger this case if something exceptional // happens between calls to revise; the framework just doesn't support this yet. } { - e := e t.Run(e.name, func(t *testing.T) { t.Parallel() diff --git a/src/runtime/mgcscavenge_test.go b/src/runtime/mgcscavenge_test.go index 9c4cf1f27793a5..4f9dbac4815dfe 100644 --- a/src/runtime/mgcscavenge_test.go +++ b/src/runtime/mgcscavenge_test.go @@ -121,13 +121,17 @@ func TestPallocDataFindScavengeCandidate(t *testing.T) { max: PallocChunkPages, want: BitRange{41, 1}, }, - "MultiMin1": { - alloc: []BitRange{{0, 63}, {65, 20}, {87, PallocChunkPages - 87}}, + } + if PallocChunkPages >= 512 { + // avoid constant overflow when PallocChunkPages is small + var pallocChunkPages uint = PallocChunkPages + tests["MultiMin1"] = test{ + alloc: []BitRange{{0, 63}, {65, 20}, {87, pallocChunkPages - 87}}, scavenged: []BitRange{{86, 1}}, min: 1, max: PallocChunkPages, want: BitRange{85, 1}, - }, + } } // Try out different page minimums. for m := uintptr(1); m <= 64; m *= 2 { @@ -162,25 +166,27 @@ func TestPallocDataFindScavengeCandidate(t *testing.T) { max: PallocChunkPages, want: BitRange{PallocChunkPages - uint(m), uint(m)}, } - tests["Straddle64"+suffix] = test{ - alloc: []BitRange{{0, 64 - uint(m)}, {64 + uint(m), PallocChunkPages - (64 + uint(m))}}, - min: m, - max: 2 * m, - want: BitRange{64 - uint(m), 2 * uint(m)}, - } - tests["BottomEdge64WithFull"+suffix] = test{ - alloc: []BitRange{{64, 64}, {128 + 3*uint(m), PallocChunkPages - (128 + 3*uint(m))}}, - scavenged: []BitRange{{1, 10}}, - min: m, - max: 3 * m, - want: BitRange{128, 3 * uint(m)}, - } - tests["BottomEdge64WithPocket"+suffix] = test{ - alloc: []BitRange{{64, 62}, {127, 1}, {128 + 3*uint(m), PallocChunkPages - (128 + 3*uint(m))}}, - scavenged: []BitRange{{1, 10}}, - min: m, - max: 3 * m, - want: BitRange{128, 3 * uint(m)}, + if PallocChunkPages >= 512 { + tests["Straddle64"+suffix] = test{ + alloc: []BitRange{{0, 64 - uint(m)}, {64 + uint(m), PallocChunkPages - (64 + uint(m))}}, + min: m, + max: 2 * m, + want: BitRange{64 - uint(m), 2 * uint(m)}, + } + tests["BottomEdge64WithFull"+suffix] = test{ + alloc: []BitRange{{64, 64}, {128 + 3*uint(m), PallocChunkPages - (128 + 3*uint(m))}}, + scavenged: []BitRange{{1, 10}}, + min: m, + max: 3 * m, + want: BitRange{128, 3 * uint(m)}, + } + tests["BottomEdge64WithPocket"+suffix] = test{ + alloc: []BitRange{{64, 62}, {127, 1}, {128 + 3*uint(m), PallocChunkPages - (128 + 3*uint(m))}}, + scavenged: []BitRange{{1, 10}}, + min: m, + max: 3 * m, + want: BitRange{128, 3 * uint(m)}, + } } tests["Max0"+suffix] = test{ scavenged: []BitRange{{0, PallocChunkPages - uint(m)}}, @@ -204,23 +210,29 @@ func TestPallocDataFindScavengeCandidate(t *testing.T) { } } if m > 1 { - tests["MaxUnaligned"+suffix] = test{ - scavenged: []BitRange{{0, PallocChunkPages - uint(m*2-1)}}, - min: m, - max: m - 2, - want: BitRange{PallocChunkPages - uint(m), uint(m)}, - } - tests["SkipSmall"+suffix] = test{ - alloc: []BitRange{{0, 64 - uint(m)}, {64, 5}, {70, 11}, {82, PallocChunkPages - 82}}, - min: m, - max: m, - want: BitRange{64 - uint(m), uint(m)}, + if PallocChunkPages >= m*2 { + tests["MaxUnaligned"+suffix] = test{ + scavenged: []BitRange{{0, PallocChunkPages - uint(m*2-1)}}, + min: m, + max: m - 2, + want: BitRange{PallocChunkPages - uint(m), uint(m)}, + } } - tests["SkipMisaligned"+suffix] = test{ - alloc: []BitRange{{0, 64 - uint(m)}, {64, 63}, {127 + uint(m), PallocChunkPages - (127 + uint(m))}}, - min: m, - max: m, - want: BitRange{64 - uint(m), uint(m)}, + if PallocChunkPages >= 512 { + // avoid constant overflow when PallocChunkPages is small + var PallocChunkPages uint = PallocChunkPages + tests["SkipSmall"+suffix] = test{ + alloc: []BitRange{{0, 64 - uint(m)}, {64, 5}, {70, 11}, {82, PallocChunkPages - 82}}, + min: m, + max: m, + want: BitRange{64 - uint(m), uint(m)}, + } + tests["SkipMisaligned"+suffix] = test{ + alloc: []BitRange{{0, 64 - uint(m)}, {64, 63}, {127 + uint(m), PallocChunkPages - (127 + uint(m))}}, + min: m, + max: m, + want: BitRange{64 - uint(m), uint(m)}, + } } tests["MaxLessThan"+suffix] = test{ scavenged: []BitRange{{0, PallocChunkPages - uint(m)}}, @@ -273,7 +285,6 @@ func TestPallocDataFindScavengeCandidate(t *testing.T) { } } for name, v := range tests { - v := v t.Run(name, func(t *testing.T) { b := makePallocData(v.alloc, v.scavenged) start, size := b.FindScavengeCandidate(PallocChunkPages-1, v.min, v.max) @@ -435,7 +446,6 @@ func TestPageAllocScavenge(t *testing.T) { } } for name, v := range tests { - v := v t.Run(name, func(t *testing.T) { b := NewPageAlloc(v.beforeAlloc, v.beforeScav) defer FreePageAlloc(b) @@ -641,7 +651,7 @@ func TestScavengeIndex(t *testing.T) { mark func(markFunc) find func(findFunc) } - for _, test := range []testCase{ + tests := []testCase{ { name: "Uninitialized", mark: func(_ markFunc) {}, @@ -692,26 +702,6 @@ func TestScavengeIndex(t *testing.T) { find(BaseChunkIdx, PallocChunkPages-1) }, }, - { - name: "TwoChunks", - mark: func(mark markFunc) { - mark(PageBase(BaseChunkIdx, 128), PageBase(BaseChunkIdx+1, 128)) - }, - find: func(find findFunc) { - find(BaseChunkIdx+1, 127) - find(BaseChunkIdx, PallocChunkPages-1) - }, - }, - { - name: "TwoChunksOffset", - mark: func(mark markFunc) { - mark(PageBase(BaseChunkIdx+7, 128), PageBase(BaseChunkIdx+8, 129)) - }, - find: func(find findFunc) { - find(BaseChunkIdx+8, 128) - find(BaseChunkIdx+7, PallocChunkPages-1) - }, - }, { name: "SevenChunksOffset", mark: func(mark markFunc) { @@ -793,8 +783,32 @@ func TestScavengeIndex(t *testing.T) { } }, }, - } { - test := test + } + if PallocChunkPages >= 512 { + tests = append(tests, + testCase{ + name: "TwoChunks", + mark: func(mark markFunc) { + mark(PageBase(BaseChunkIdx, 128), PageBase(BaseChunkIdx+1, 128)) + }, + find: func(find findFunc) { + find(BaseChunkIdx+1, 127) + find(BaseChunkIdx, PallocChunkPages-1) + }, + }, + testCase{ + name: "TwoChunksOffset", + mark: func(mark markFunc) { + mark(PageBase(BaseChunkIdx+7, 128), PageBase(BaseChunkIdx+8, 129)) + }, + find: func(find findFunc) { + find(BaseChunkIdx+8, 128) + find(BaseChunkIdx+7, PallocChunkPages-1) + }, + }, + ) + } + for _, test := range tests { t.Run("Bg/"+test.name, func(t *testing.T) { mark, find, nextGen := setup(t, false) test.mark(mark) @@ -830,8 +844,10 @@ func TestScavengeIndex(t *testing.T) { } func TestScavChunkDataPack(t *testing.T) { - if !CheckPackScavChunkData(1918237402, 512, 512, 0b11) { - t.Error("failed pack/unpack check for scavChunkData 1") + if PallocChunkPages >= 512 { + if !CheckPackScavChunkData(1918237402, 512, 512, 0b11) { + t.Error("failed pack/unpack check for scavChunkData 1") + } } if !CheckPackScavChunkData(^uint32(0), 12, 0, 0b00) { t.Error("failed pack/unpack check for scavChunkData 2") diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go index f53330a5b97656..c3d6afb90a54fe 100644 --- a/src/runtime/mgcsweep.go +++ b/src/runtime/mgcsweep.go @@ -25,7 +25,6 @@ package runtime import ( - "internal/abi" "internal/runtime/atomic" "unsafe" ) @@ -170,13 +169,16 @@ func (a *activeSweep) end(sl sweepLocker) { throw("mismatched begin/end of activeSweep") } if a.state.CompareAndSwap(state, state-1) { - if state != sweepDrainedMask { + if state-1 != sweepDrainedMask { return } + // We're the last sweeper, and there's nothing left to sweep. if debug.gcpacertrace > 0 { live := gcController.heapLive.Load() print("pacer: sweep done at heap size ", live>>20, "MB; allocated ", (live-mheap_.sweepHeapLiveBasis)>>20, "MB during sweep; swept ", mheap_.pagesSwept.Load(), " pages at ", mheap_.sweepPagesPerByte, " pages/byte\n") } + // Now that sweeping is completely done, flush remaining cleanups. + gcCleanups.flush() return } } @@ -305,12 +307,17 @@ func bgsweep(c chan int) { // N.B. freeSomeWbufs is already batched internally. goschedIfBusy() } + freeDeadSpanSPMCs() lock(&sweep.lock) if !isSweepDone() { // This can happen if a GC runs between // gosweepone returning ^0 above // and the lock being acquired. unlock(&sweep.lock) + // This goroutine must preempt when we have no work to do + // but isSweepDone returns false because of another existing sweeper. + // See issue #73499. + goschedIfBusy() continue } sweep.parked = true @@ -518,7 +525,7 @@ func (sl *sweepLocked) sweep(preserve bool) bool { trace := traceAcquire() if trace.ok() { - trace.GCSweepSpan(s.npages * _PageSize) + trace.GCSweepSpan(s.npages * pageSize) traceRelease(trace) } @@ -547,7 +554,7 @@ func (sl *sweepLocked) sweep(preserve bool) bool { siter := newSpecialsIter(s) for siter.valid() { // A finalizer can be set for an inner byte of an object, find object beginning. - objIndex := uintptr(siter.s.offset) / size + objIndex := siter.s.offset / size p := s.base() + objIndex*size mbits := s.markBitsForIndex(objIndex) if !mbits.isMarked() { @@ -555,7 +562,7 @@ func (sl *sweepLocked) sweep(preserve bool) bool { // Pass 1: see if it has a finalizer. hasFinAndRevived := false endOffset := p - s.base() + size - for tmp := siter.s; tmp != nil && uintptr(tmp.offset) < endOffset; tmp = tmp.next { + for tmp := siter.s; tmp != nil && tmp.offset < endOffset; tmp = tmp.next { if tmp.kind == _KindSpecialFinalizer { // Stop freeing of object if it has a finalizer. mbits.setMarkedNonAtomic() @@ -565,13 +572,13 @@ func (sl *sweepLocked) sweep(preserve bool) bool { } if hasFinAndRevived { // Pass 2: queue all finalizers and clear any weak handles. Weak handles are cleared - // before finalization as specified by the internal/weak package. See the documentation + // before finalization as specified by the weak package. See the documentation // for that package for more details. - for siter.valid() && uintptr(siter.s.offset) < endOffset { + for siter.valid() && siter.s.offset < endOffset { // Find the exact byte for which the special was setup // (as opposed to object beginning). special := siter.s - p := s.base() + uintptr(special.offset) + p := s.base() + special.offset if special.kind == _KindSpecialFinalizer || special.kind == _KindSpecialWeakHandle { siter.unlinkAndNext() freeSpecial(special, unsafe.Pointer(p), size) @@ -583,11 +590,11 @@ func (sl *sweepLocked) sweep(preserve bool) bool { } } else { // Pass 2: the object is truly dead, free (and handle) all specials. - for siter.valid() && uintptr(siter.s.offset) < endOffset { + for siter.valid() && siter.s.offset < endOffset { // Find the exact byte for which the special was setup // (as opposed to object beginning). special := siter.s - p := s.base() + uintptr(special.offset) + p := s.base() + special.offset siter.unlinkAndNext() freeSpecial(special, unsafe.Pointer(p), size) } @@ -635,12 +642,20 @@ func (sl *sweepLocked) sweep(preserve bool) bool { if asanenabled && !s.isUserArenaChunk { asanpoison(unsafe.Pointer(x), size) } + if valgrindenabled && !s.isUserArenaChunk { + valgrindFree(unsafe.Pointer(x)) + } } mbits.advance() abits.advance() } } + // Copy over and clear the inline mark bits if necessary. + if gcUsesSpanInlineMarkBits(s.elemsize) { + s.moveInlineMarks(s.gcmarkBits) + } + // Check for zombie objects. if s.freeindex < s.nelems { // Everything < freeindex is allocated and hence @@ -818,18 +833,6 @@ func (sl *sweepLocked) sweep(preserve bool) bool { } else { mheap_.freeSpan(s) } - if s.largeType != nil && s.largeType.TFlag&abi.TFlagUnrolledBitmap != 0 { - // The unrolled GCProg bitmap is allocated separately. - // Free the space for the unrolled bitmap. - systemstack(func() { - s := spanOf(uintptr(unsafe.Pointer(s.largeType))) - mheap_.freeManual(s, spanAllocPtrScalarBits) - }) - // Make sure to zero this pointer without putting the old - // value in a write buffer, as the old value might be an - // invalid pointer. See arena.go:(*mheap).allocUserArenaChunk. - *(*uintptr)(unsafe.Pointer(&s.largeType)) = 0 - } return true } @@ -855,7 +858,7 @@ func (sl *sweepLocked) sweep(preserve bool) bool { // pointer to that object and marked it. func (s *mspan) reportZombies() { printlock() - print("runtime: marked free object in span ", s, ", elemsize=", s.elemsize, " freeindex=", s.freeindex, " (bad use of unsafe.Pointer? try -d=checkptr)\n") + print("runtime: marked free object in span ", s, ", elemsize=", s.elemsize, " freeindex=", s.freeindex, " (bad use of unsafe.Pointer or having race conditions? try -d=checkptr or -race)\n") mbits := s.markBitsForBase() abits := s.allocBitsForIndex(0) for i := uintptr(0); i < uintptr(s.nelems); i++ { @@ -994,9 +997,9 @@ func gcPaceSweeper(trigger uint64) { // concurrent sweep are less likely to leave pages // unswept when GC starts. heapDistance -= 1024 * 1024 - if heapDistance < _PageSize { + if heapDistance < pageSize { // Avoid setting the sweep ratio extremely high - heapDistance = _PageSize + heapDistance = pageSize } pagesSwept := mheap_.pagesSwept.Load() pagesInUse := mheap_.pagesInUse.Load() diff --git a/src/runtime/mgcwork.go b/src/runtime/mgcwork.go index 2d66fa400231de..48ac94eecd06ba 100644 --- a/src/runtime/mgcwork.go +++ b/src/runtime/mgcwork.go @@ -6,7 +6,9 @@ package runtime import ( "internal/goarch" + "internal/goexperiment" "internal/runtime/atomic" + "internal/runtime/gc" "internal/runtime/sys" "unsafe" ) @@ -32,13 +34,38 @@ func init() { // Garbage collector work pool abstraction. // // This implements a producer/consumer model for pointers to grey -// objects. A grey object is one that is marked and on a work -// queue. A black object is marked and not on a work queue. +// objects. +// +// For objects in workbufs, a grey object is one that is marked and +// on a work queue. A black object is marked and not on a work queue. +// +// For objects in the span queue, a grey object is one that is marked +// and has an unset scan bit. A black object is marked and has its scan +// bit set. (Green Tea GC only.) // // Write barriers, root discovery, stack scanning, and object scanning // produce pointers to grey objects. Scanning consumes pointers to // grey objects, thus blackening them, and then scans them, // potentially producing new pointers to grey objects. +// +// Work queues must be prioritized in the following order wherever work +// is processed. +// +// +----------------------------------------------------------+ +// | Priority | Work queue | Restrictions | Function | +// |----------------------------------------------------------| +// | 1 | Workbufs | P-local | tryGetObjFast | +// | 2 | Span queue | P-local | tryGetSpanFast | [greenteagc] +// | 3 | Workbufs | None | tryGetObj | +// | 4 | Span queue | None | tryGetSpan | [greenteagc] +// | 5 | Span queue | None | tryStealSpan | [greenteagc] +// +----------------------------------------------------------+ +// +// The rationale behind this ordering comes from two insights: +// 1. It's always preferable to look for P-local work first to avoid hammering on +// global lists. +// 2. It's always preferable to scan individual objects first to increase the +// likelihood that spans will accumulate more objects to scan. // A gcWork provides the interface to produce and consume work for the // garbage collector. @@ -54,6 +81,8 @@ func init() { // gcWork may locally hold GC work buffers. This can be done by // disabling preemption (systemstack or acquirem). type gcWork struct { + id int32 // same ID as the parent P + // wbuf1 and wbuf2 are the primary and secondary work buffers. // // This can be thought of as a stack of both work buffers' @@ -74,6 +103,14 @@ type gcWork struct { // Invariant: Both wbuf1 and wbuf2 are nil or neither are. wbuf1, wbuf2 *workbuf + // spanq is a queue of spans to process. + // + // Only used if goexperiment.GreenTeaGC. + spanq spanQueue + + // ptrBuf is a temporary buffer used by span scanning. + ptrBuf *[pageSize / goarch.PtrSize]uintptr + // Bytes marked (blackened) on this gcWork. This is aggregated // into work.bytesMarked by dispose. bytesMarked uint64 @@ -88,6 +125,15 @@ type gcWork struct { // termination check. Specifically, this indicates that this // gcWork may have communicated work to another gcWork. flushedWork bool + + // mayNeedWorker is a hint that we may need to spin up a new + // worker, and that gcDrain* should call enlistWorker. This flag + // is set only if goexperiment.GreenTeaGC. If !goexperiment.GreenTeaGC, + // enlistWorker is called directly instead. + mayNeedWorker bool + + // stats are scan stats broken down by size class. + stats [gc.NumSizeClasses]sizeClassScanStats } // Most of the methods of gcWork are go:nowritebarrierrec because the @@ -106,11 +152,11 @@ func (w *gcWork) init() { w.wbuf2 = wbuf2 } -// put enqueues a pointer for the garbage collector to trace. +// putObj enqueues a pointer for the garbage collector to trace. // obj must point to the beginning of a heap object or an oblet. // //go:nowritebarrierrec -func (w *gcWork) put(obj uintptr) { +func (w *gcWork) putObj(obj uintptr) { flushed := false wbuf := w.wbuf1 // Record that this may acquire the wbufSpans or heap lock to @@ -141,15 +187,19 @@ func (w *gcWork) put(obj uintptr) { // the end of put so that w is in a consistent state, since // enlistWorker may itself manipulate w. if flushed && gcphase == _GCmark { - gcController.enlistWorker() + if goexperiment.GreenTeaGC { + w.mayNeedWorker = true + } else { + gcController.enlistWorker() + } } } -// putFast does a put and reports whether it can be done quickly +// putObjFast does a put and reports whether it can be done quickly // otherwise it returns false and the caller needs to call put. // //go:nowritebarrierrec -func (w *gcWork) putFast(obj uintptr) bool { +func (w *gcWork) putObjFast(obj uintptr) bool { wbuf := w.wbuf1 if wbuf == nil || wbuf.nobj == len(wbuf.obj) { return false @@ -160,11 +210,11 @@ func (w *gcWork) putFast(obj uintptr) bool { return true } -// putBatch performs a put on every pointer in obj. See put for +// putObjBatch performs a put on every pointer in obj. See put for // constraints on these pointers. // //go:nowritebarrierrec -func (w *gcWork) putBatch(obj []uintptr) { +func (w *gcWork) putObjBatch(obj []uintptr) { if len(obj) == 0 { return } @@ -190,18 +240,22 @@ func (w *gcWork) putBatch(obj []uintptr) { } if flushed && gcphase == _GCmark { - gcController.enlistWorker() + if goexperiment.GreenTeaGC { + w.mayNeedWorker = true + } else { + gcController.enlistWorker() + } } } -// tryGet dequeues a pointer for the garbage collector to trace. +// tryGetObj dequeues a pointer for the garbage collector to trace. // // If there are no pointers remaining in this gcWork or in the global // queue, tryGet returns 0. Note that there may still be pointers in // other gcWork instances or other caches. // //go:nowritebarrierrec -func (w *gcWork) tryGet() uintptr { +func (w *gcWork) tryGetObj() uintptr { wbuf := w.wbuf1 if wbuf == nil { w.init() @@ -226,12 +280,12 @@ func (w *gcWork) tryGet() uintptr { return wbuf.obj[wbuf.nobj] } -// tryGetFast dequeues a pointer for the garbage collector to trace +// tryGetObjFast dequeues a pointer for the garbage collector to trace // if one is readily available. Otherwise it returns 0 and // the caller is expected to call tryGet(). // //go:nowritebarrierrec -func (w *gcWork) tryGetFast() uintptr { +func (w *gcWork) tryGetObjFast() uintptr { wbuf := w.wbuf1 if wbuf == nil || wbuf.nobj == 0 { return 0 @@ -267,6 +321,20 @@ func (w *gcWork) dispose() { } w.wbuf2 = nil } + if !w.spanq.empty() { + w.spanq.flush() // Flush any local work. + + // There's globally-visible work now, so make everyone aware of it. + // + // Note that we need to make everyone aware even if flush didn't + // flush any local work. The global work was always visible, but + // the bitmap bit may have been unset. + // + // See the comment in tryStealSpan, which explains how it relies + // on this behavior. + work.spanqMask.set(w.id) + w.flushedWork = true + } if w.bytesMarked != 0 { // dispose happens relatively infrequently. If this // atomic becomes a problem, we should first try to @@ -301,7 +369,11 @@ func (w *gcWork) balance() { } // We flushed a buffer to the full list, so wake a worker. if gcphase == _GCmark { - gcController.enlistWorker() + if goexperiment.GreenTeaGC { + w.mayNeedWorker = true + } else { + gcController.enlistWorker() + } } } @@ -309,7 +381,7 @@ func (w *gcWork) balance() { // //go:nowritebarrierrec func (w *gcWork) empty() bool { - return w.wbuf1 == nil || (w.wbuf1.nobj == 0 && w.wbuf2.nobj == 0) + return (w.wbuf1 == nil || (w.wbuf1.nobj == 0 && w.wbuf2.nobj == 0)) && w.spanq.empty() } // Internally, the GC work pool is kept in arrays in work buffers. diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index afbd20c7bf4e14..f2dc3717b1bd31 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -9,9 +9,12 @@ package runtime import ( + "internal/abi" "internal/cpu" "internal/goarch" + "internal/goexperiment" "internal/runtime/atomic" + "internal/runtime/gc" "internal/runtime/sys" "unsafe" ) @@ -44,7 +47,7 @@ const ( // // Must be a multiple of the pageInUse bitmap element size and // must also evenly divide pagesPerArena. - pagesPerReclaimerChunk = 512 + pagesPerReclaimerChunk = min(512, pagesPerArena) // physPageAlignedStacks indicates whether stack allocations must be // physical page aligned. This is a requirement for MAP_STACK on @@ -108,9 +111,9 @@ type mheap struct { // Page reclaimer state - // reclaimIndex is the page index in allArenas of next page to + // reclaimIndex is the page index in heapArenas of next page to // reclaim. Specifically, it refers to page (i % - // pagesPerArena) of arena allArenas[i / pagesPerArena]. + // pagesPerArena) of arena heapArenas[i / pagesPerArena]. // // If this is >= 1<<63, the page reclaimer is done scanning // the page marks. @@ -165,22 +168,31 @@ type mheap struct { // (the actual arenas). This is only used on 32-bit. arena linearAlloc - // allArenas is the arenaIndex of every mapped arena. This can - // be used to iterate through the address space. + // heapArenas is the arenaIndex of every mapped arena mapped for the heap. + // This can be used to iterate through the heap address space. // // Access is protected by mheap_.lock. However, since this is // append-only and old backing arrays are never freed, it is // safe to acquire mheap_.lock, copy the slice header, and // then release mheap_.lock. - allArenas []arenaIdx + heapArenas []arenaIdx - // sweepArenas is a snapshot of allArenas taken at the + // userArenaArenas is the arenaIndex of every mapped arena mapped for + // user arenas. + // + // Access is protected by mheap_.lock. However, since this is + // append-only and old backing arrays are never freed, it is + // safe to acquire mheap_.lock, copy the slice header, and + // then release mheap_.lock. + userArenaArenas []arenaIdx + + // sweepArenas is a snapshot of heapArenas taken at the // beginning of the sweep cycle. This can be read safely by // simply blocking GC (by disabling preemption). sweepArenas []arenaIdx - // markArenas is a snapshot of allArenas taken at the beginning - // of the mark cycle. Because allArenas is append-only, neither + // markArenas is a snapshot of heapArenas taken at the beginning + // of the mark cycle. Because heapArenas is append-only, neither // this slice nor its contents will change during the mark, so // it can be read safely. markArenas []arenaIdx @@ -201,15 +213,20 @@ type mheap struct { pad [(cpu.CacheLinePadSize - unsafe.Sizeof(mcentral{})%cpu.CacheLinePadSize) % cpu.CacheLinePadSize]byte } - spanalloc fixalloc // allocator for span* - cachealloc fixalloc // allocator for mcache* - specialfinalizeralloc fixalloc // allocator for specialfinalizer* - specialprofilealloc fixalloc // allocator for specialprofile* - specialReachableAlloc fixalloc // allocator for specialReachable - specialPinCounterAlloc fixalloc // allocator for specialPinCounter - specialWeakHandleAlloc fixalloc // allocator for specialWeakHandle - speciallock mutex // lock for special record allocators. - arenaHintAlloc fixalloc // allocator for arenaHints + spanalloc fixalloc // allocator for span + spanSPMCAlloc fixalloc // allocator for spanSPMC, protected by work.spanSPMCs.lock + cachealloc fixalloc // allocator for mcache + specialfinalizeralloc fixalloc // allocator for specialfinalizer + specialCleanupAlloc fixalloc // allocator for specialCleanup + specialCheckFinalizerAlloc fixalloc // allocator for specialCheckFinalizer + specialTinyBlockAlloc fixalloc // allocator for specialTinyBlock + specialprofilealloc fixalloc // allocator for specialprofile + specialReachableAlloc fixalloc // allocator for specialReachable + specialPinCounterAlloc fixalloc // allocator for specialPinCounter + specialWeakHandleAlloc fixalloc // allocator for specialWeakHandle + specialBubbleAlloc fixalloc // allocator for specialBubble + speciallock mutex // lock for special record allocators. + arenaHintAlloc fixalloc // allocator for arenaHints // User arena state. // @@ -230,6 +247,16 @@ type mheap struct { readyList mSpanList } + // cleanupID is a counter which is incremented each time a cleanup special is added + // to a span. It's used to create globally unique identifiers for individual cleanup. + // cleanupID is protected by mheap_.speciallock. It must only be incremented while holding + // the lock. ID 0 is reserved. Users should increment first, then read the value. + cleanupID uint64 + + _ cpu.CacheLinePad + + immortalWeakHandles immortalWeakHandleMap + unused *specialfinalizer // never set, just here to force the specialfinalizer type into DWARF } @@ -286,8 +313,14 @@ type heapArena struct { // during marking. pageSpecials [pagesPerArena / 8]uint8 + // pageUseSpanInlineMarkBits is a bitmap where each bit corresponds + // to a span, as only spans one page in size can have inline mark bits. + // The bit indicates that the span has a spanInlineMarkBits struct + // stored directly at the top end of the span's memory. + pageUseSpanInlineMarkBits [pagesPerArena / 8]uint8 + // checkmarks stores the debug.gccheckmark state. It is only - // used if debug.gccheckmark > 0. + // used if debug.gccheckmark > 0 or debug.checkfinalizers > 0. checkmarks *checkmarksMap // zeroedBase marks the first byte of the first page in this @@ -385,13 +418,6 @@ func (b *mSpanStateBox) get() mSpanState { return mSpanState(b.s.Load()) } -// mSpanList heads a linked list of spans. -type mSpanList struct { - _ sys.NotInHeap - first *mspan // first span in list, or nil if none - last *mspan // last span in list, or nil if none -} - type mspan struct { _ sys.NotInHeap next *mspan // next span in list, or nil if none @@ -409,12 +435,12 @@ type mspan struct { // indicating a free object. freeindex is then adjusted so that subsequent scans begin // just past the newly discovered free object. // - // If freeindex == nelem, this span has no free objects. + // If freeindex == nelems, this span has no free objects. // // allocBits is a bitmap of objects in this span. // If n >= freeindex and allocBits[n/8] & (1<<(n%8)) is 0 // then object n is free; - // otherwise, object n is allocated. Bits starting at nelem are + // otherwise, object n is allocated. Bits starting at nelems are // undefined and should never be referenced. // // Object n starts at address n*elemsize + (start << pageShift). @@ -430,6 +456,12 @@ type mspan struct { // mallocgc, and issue 54596). freeIndexForScan uint16 + // Temporary storage for the object index that caused this span to + // be queued for scanning. + // + // Used only with goexperiment.GreenTeaGC. + scanIdx uint16 + // Cache of the allocBits at freeindex. allocCache is shifted // such that the lowest bit corresponds to the bit freeindex. // allocCache holds the complement of allocBits, thus allowing @@ -493,7 +525,7 @@ func (s *mspan) base() uintptr { } func (s *mspan) layout() (size, n, total uintptr) { - total = s.npages << _PageShift + total = s.npages << gc.PageShift size = s.elemsize if size > 0 { n = total / size @@ -527,7 +559,7 @@ func recordspan(vh unsafe.Pointer, p unsafe.Pointer) { } var new []*mspan sp := (*slice)(unsafe.Pointer(&new)) - sp.array = sysAlloc(uintptr(n)*goarch.PtrSize, &memstats.other_sys) + sp.array = sysAlloc(uintptr(n)*goarch.PtrSize, &memstats.other_sys, "allspans array") if sp.array == nil { throw("runtime: cannot allocate memory") } @@ -555,7 +587,7 @@ func recordspan(vh unsafe.Pointer, p unsafe.Pointer) { type spanClass uint8 const ( - numSpanClasses = _NumSizeClasses << 1 + numSpanClasses = gc.NumSizeClasses << 1 tinySpanClass = spanClass(tinySizeClass<<1 | 1) ) @@ -735,18 +767,44 @@ func pageIndexOf(p uintptr) (arena *heapArena, pageIdx uintptr, pageMask uint8) return } +// heapArenaOf returns the heap arena for p, if one exists. +func heapArenaOf(p uintptr) *heapArena { + ri := arenaIndex(p) + if arenaL1Bits == 0 { + // If there's no L1, then ri.l1() can't be out of bounds but ri.l2() can. + if ri.l2() >= uint(len(mheap_.arenas[0])) { + return nil + } + } else { + // If there's an L1, then ri.l1() can be out of bounds but ri.l2() can't. + if ri.l1() >= uint(len(mheap_.arenas)) { + return nil + } + } + l2 := mheap_.arenas[ri.l1()] + if arenaL1Bits != 0 && l2 == nil { // Should never happen if there's no L1. + return nil + } + return l2[ri.l2()] +} + // Initialize the heap. func (h *mheap) init() { lockInit(&h.lock, lockRankMheap) lockInit(&h.speciallock, lockRankMheapSpecial) h.spanalloc.init(unsafe.Sizeof(mspan{}), recordspan, unsafe.Pointer(h), &memstats.mspan_sys) + h.spanSPMCAlloc.init(unsafe.Sizeof(spanSPMC{}), nil, nil, &memstats.gcMiscSys) h.cachealloc.init(unsafe.Sizeof(mcache{}), nil, nil, &memstats.mcache_sys) h.specialfinalizeralloc.init(unsafe.Sizeof(specialfinalizer{}), nil, nil, &memstats.other_sys) + h.specialCleanupAlloc.init(unsafe.Sizeof(specialCleanup{}), nil, nil, &memstats.other_sys) + h.specialCheckFinalizerAlloc.init(unsafe.Sizeof(specialCheckFinalizer{}), nil, nil, &memstats.other_sys) + h.specialTinyBlockAlloc.init(unsafe.Sizeof(specialTinyBlock{}), nil, nil, &memstats.other_sys) h.specialprofilealloc.init(unsafe.Sizeof(specialprofile{}), nil, nil, &memstats.other_sys) h.specialReachableAlloc.init(unsafe.Sizeof(specialReachable{}), nil, nil, &memstats.other_sys) h.specialPinCounterAlloc.init(unsafe.Sizeof(specialPinCounter{}), nil, nil, &memstats.other_sys) h.specialWeakHandleAlloc.init(unsafe.Sizeof(specialWeakHandle{}), nil, nil, &memstats.gcMiscSys) + h.specialBubbleAlloc.init(unsafe.Sizeof(specialBubble{}), nil, nil, &memstats.other_sys) h.arenaHintAlloc.init(unsafe.Sizeof(arenaHint{}), nil, nil, &memstats.other_sys) // Don't zero mspan allocations. Background sweeping can @@ -765,6 +823,8 @@ func (h *mheap) init() { } h.pages.init(&h.lock, &memstats.gcMiscSys, false) + + xRegInitAlloc() } // reclaim sweeps and reclaims at least npage pages into the heap. @@ -931,10 +991,9 @@ func (h *mheap) reclaimChunk(arenas []arenaIdx, pageIdx, n uintptr) uintptr { type spanAllocType uint8 const ( - spanAllocHeap spanAllocType = iota // heap span - spanAllocStack // stack span - spanAllocPtrScalarBits // unrolled GC prog bitmap span - spanAllocWorkBuf // work buf span + spanAllocHeap spanAllocType = iota // heap span + spanAllocStack // stack span + spanAllocWorkBuf // work buf span ) // manual returns true if the span allocation is manually managed. @@ -1277,7 +1336,7 @@ HaveSpan: // that we expect to page in. inuse := gcController.mappedReady.Load() // Be careful about overflow, especially with uintptrs. Even on 32-bit platforms - // someone can set a really big memory limit that isn't maxInt64. + // someone can set a really big memory limit that isn't math.MaxInt64. if uint64(scav)+inuse > uint64(limit) { bytesToScavenge = uintptr(uint64(scav) + inuse - uint64(limit)) forceScavenge = true @@ -1337,6 +1396,10 @@ HaveSpan: // Initialize the span. h.initSpan(s, typ, spanclass, base, npages) + if valgrindenabled { + valgrindMempoolMalloc(unsafe.Pointer(arenaBase(arenaIndex(base))), unsafe.Pointer(base), npages*pageSize) + } + // Commit and account for any scavenged memory that the span now owns. nbytes := npages * pageSize if scav != 0 { @@ -1359,8 +1422,6 @@ HaveSpan: atomic.Xaddint64(&stats.inHeap, int64(nbytes)) case spanAllocStack: atomic.Xaddint64(&stats.inStacks, int64(nbytes)) - case spanAllocPtrScalarBits: - atomic.Xaddint64(&stats.inPtrScalarBits, int64(nbytes)) case spanAllocWorkBuf: atomic.Xaddint64(&stats.inWorkBufs, int64(nbytes)) } @@ -1390,7 +1451,6 @@ func (h *mheap) initSpan(s *mspan, typ spanAllocType, spanclass spanClass, base, if typ.manual() { s.manualFreeList = 0 s.nelems = 0 - s.limit = s.base() + s.npages*pageSize s.state.set(mSpanManual) } else { // We must set span properties before the span is published anywhere @@ -1401,14 +1461,27 @@ func (h *mheap) initSpan(s *mspan, typ spanAllocType, spanclass spanClass, base, s.nelems = 1 s.divMul = 0 } else { - s.elemsize = uintptr(class_to_size[sizeclass]) - if !s.spanclass.noscan() && heapBitsInSpan(s.elemsize) { - // Reserve space for the pointer/scan bitmap at the end. - s.nelems = uint16((nbytes - (nbytes / goarch.PtrSize / 8)) / s.elemsize) + s.elemsize = uintptr(gc.SizeClassToSize[sizeclass]) + if goexperiment.GreenTeaGC { + var reserve uintptr + if gcUsesSpanInlineMarkBits(s.elemsize) { + // Reserve space for the inline mark bits. + reserve += unsafe.Sizeof(spanInlineMarkBits{}) + } + if heapBitsInSpan(s.elemsize) && !s.spanclass.noscan() { + // Reserve space for the pointer/scan bitmap at the end. + reserve += nbytes / goarch.PtrSize / 8 + } + s.nelems = uint16((nbytes - reserve) / s.elemsize) } else { - s.nelems = uint16(nbytes / s.elemsize) + if !s.spanclass.noscan() && heapBitsInSpan(s.elemsize) { + // Reserve space for the pointer/scan bitmap at the end. + s.nelems = uint16((nbytes - (nbytes / goarch.PtrSize / 8)) / s.elemsize) + } else { + s.nelems = uint16(nbytes / s.elemsize) + } } - s.divMul = class_to_divmagic[sizeclass] + s.divMul = gc.SizeClassToDivMagic[sizeclass] } // Initialize mark and allocation structures. @@ -1418,6 +1491,9 @@ func (h *mheap) initSpan(s *mspan, typ spanAllocType, spanclass spanClass, base, s.gcmarkBits = newMarkBits(uintptr(s.nelems)) s.allocBits = newAllocBits(uintptr(s.nelems)) + // Adjust s.limit down to the object-containing part of the span. + s.limit = s.base() + s.elemsize*uintptr(s.nelems) + // It's safe to access h.sweepgen without the heap lock because it's // only ever updated with the world stopped and we run on the // systemstack which blocks a STW transition. @@ -1454,6 +1530,11 @@ func (h *mheap) initSpan(s *mspan, typ spanAllocType, spanclass spanClass, base, arena, pageIdx, pageMask := pageIndexOf(s.base()) atomic.Or8(&arena.pageInUse[pageIdx], pageMask) + // Mark packed span. + if gcUsesSpanInlineMarkBits(s.elemsize) { + atomic.Or8(&arena.pageUseSpanInlineMarkBits[pageIdx], pageMask) + } + // Update related page sweeper stats. h.pagesInUse.Add(npages) } @@ -1470,6 +1551,8 @@ func (h *mheap) initSpan(s *mspan, typ spanAllocType, spanclass spanClass, base, func (h *mheap) grow(npage uintptr) (uintptr, bool) { assertLockHeld(&h.lock) + firstGrow := h.curArena.base == 0 + // We must grow the heap in whole palloc chunks. // We call sysMap below but note that because we // round up to pallocChunkPages which is on the order @@ -1486,7 +1569,7 @@ func (h *mheap) grow(npage uintptr) (uintptr, bool) { // Not enough room in the current arena. Allocate more // arena space. This may not be contiguous with the // current arena, so we have to request the full ask. - av, asize := h.sysAlloc(ask, &h.arenaHints, true) + av, asize := h.sysAlloc(ask, &h.arenaHints, &h.heapArenas) if av == nil { inUse := gcController.heapFree.load() + gcController.heapReleased.load() + gcController.heapInUse.load() print("runtime: out of memory: cannot allocate ", ask, "-byte block (", inUse, " in use)\n") @@ -1505,7 +1588,7 @@ func (h *mheap) grow(npage uintptr) (uintptr, bool) { // Transition this space from Reserved to Prepared and mark it // as released since we'll be able to start using it after updating // the page allocator and releasing the lock at any time. - sysMap(unsafe.Pointer(h.curArena.base), size, &gcController.heapReleased) + sysMap(unsafe.Pointer(h.curArena.base), size, &gcController.heapReleased, "heap") // Update stats. stats := memstats.heapStats.acquire() atomic.Xaddint64(&stats.released, int64(size)) @@ -1518,6 +1601,16 @@ func (h *mheap) grow(npage uintptr) (uintptr, bool) { // Switch to the new space. h.curArena.base = uintptr(av) h.curArena.end = uintptr(av) + asize + + if firstGrow && randomizeHeapBase { + // The top heapAddrBits-logHeapArenaBytes are randomized, we now + // want to randomize the next + // logHeapArenaBytes-log2(pallocChunkBytes) bits, making sure + // h.curArena.base is aligned to pallocChunkBytes. + bits := logHeapArenaBytes - logPallocChunkBytes + offset := nextHeapRandBits(bits) + h.curArena.base = alignDown(h.curArena.base|(offset< physPageSize, so its safe to // just add directly to heapReleased. - sysMap(unsafe.Pointer(v), nBase-v, &gcController.heapReleased) + sysMap(unsafe.Pointer(v), nBase-v, &gcController.heapReleased, "heap") // The memory just allocated counts as both released // and idle, even though it's not yet backed by spans. @@ -1548,6 +1641,22 @@ func (h *mheap) grow(npage uintptr) (uintptr, bool) { // space ready for allocation. h.pages.grow(v, nBase-v) totalGrowth += nBase - v + + if firstGrow && randomizeHeapBase { + // The top heapAddrBits-log2(pallocChunkBytes) bits are now randomized, + // we finally want to randomize the next + // log2(pallocChunkBytes)-log2(pageSize) bits, while maintaining + // alignment to pageSize. We do this by calculating a random number of + // pages into the current arena, and marking them as allocated. The + // address of the next available page becomes our fully randomized base + // heap address. + randOffset := nextHeapRandBits(logPallocChunkBytes) + randNumPages := alignDown(randOffset, pageSize) / pageSize + if randNumPages != 0 { + h.pages.markRandomPaddingPages(v, randNumPages) + } + } + return totalGrowth, true } @@ -1567,15 +1676,19 @@ func (h *mheap) freeSpan(s *mspan) { if msanenabled { // Tell msan that this entire span is no longer in use. base := unsafe.Pointer(s.base()) - bytes := s.npages << _PageShift + bytes := s.npages << gc.PageShift msanfree(base, bytes) } if asanenabled { // Tell asan that this entire span is no longer in use. base := unsafe.Pointer(s.base()) - bytes := s.npages << _PageShift + bytes := s.npages << gc.PageShift asanpoison(base, bytes) } + if valgrindenabled { + base := s.base() + valgrindMempoolFree(unsafe.Pointer(arenaBase(arenaIndex(base))), unsafe.Pointer(base)) + } h.freeSpanLocked(s, spanAllocHeap) unlock(&h.lock) }) @@ -1604,6 +1717,10 @@ func (h *mheap) freeManual(s *mspan, typ spanAllocType) { s.needzero = 1 lock(&h.lock) + if valgrindenabled { + base := s.base() + valgrindMempoolFree(unsafe.Pointer(arenaBase(arenaIndex(base))), unsafe.Pointer(base)) + } h.freeSpanLocked(s, typ) unlock(&h.lock) } @@ -1629,6 +1746,11 @@ func (h *mheap) freeSpanLocked(s *mspan, typ spanAllocType) { // Clear in-use bit in arena page bitmap. arena, pageIdx, pageMask := pageIndexOf(s.base()) atomic.And8(&arena.pageInUse[pageIdx], ^pageMask) + + // Clear small heap span bit if necessary. + if gcUsesSpanInlineMarkBits(s.elemsize) { + atomic.And8(&arena.pageUseSpanInlineMarkBits[pageIdx], ^pageMask) + } default: throw("mheap.freeSpanLocked - invalid span state") } @@ -1648,8 +1770,6 @@ func (h *mheap) freeSpanLocked(s *mspan, typ spanAllocType) { atomic.Xaddint64(&stats.inHeap, -int64(nbytes)) case spanAllocStack: atomic.Xaddint64(&stats.inStacks, -int64(nbytes)) - case spanAllocPtrScalarBits: - atomic.Xaddint64(&stats.inPtrScalarBits, -int64(nbytes)) case spanAllocWorkBuf: atomic.Xaddint64(&stats.inWorkBufs, -int64(nbytes)) } @@ -1701,6 +1821,7 @@ func (span *mspan) init(base uintptr, npages uintptr) { span.list = nil span.startAddr = base span.npages = npages + span.limit = base + npages*gc.PageSize // see go.dev/issue/74288; adjusted later for heap spans span.allocCount = 0 span.spanclass = 0 span.elemsize = 0 @@ -1720,6 +1841,13 @@ func (span *mspan) inList() bool { return span.list != nil } +// mSpanList heads a linked list of spans. +type mSpanList struct { + _ sys.NotInHeap + first *mspan // first span in list, or nil if none + last *mspan // last span in list, or nil if none +} + // Initialize an empty doubly-linked list. func (list *mSpanList) init() { list.first = nil @@ -1812,24 +1940,36 @@ func (list *mSpanList) takeAll(other *mSpanList) { } const ( + // _KindSpecialTinyBlock indicates that a given allocation is a tiny block. + // Ordered before KindSpecialFinalizer and KindSpecialCleanup so that it + // always appears first in the specials list. + // Used only if debug.checkfinalizers != 0. + _KindSpecialTinyBlock = 1 // _KindSpecialFinalizer is for tracking finalizers. - _KindSpecialFinalizer = 1 + _KindSpecialFinalizer = 2 // _KindSpecialWeakHandle is used for creating weak pointers. - _KindSpecialWeakHandle = 2 + _KindSpecialWeakHandle = 3 // _KindSpecialProfile is for memory profiling. - _KindSpecialProfile = 3 + _KindSpecialProfile = 4 // _KindSpecialReachable is a special used for tracking // reachability during testing. - _KindSpecialReachable = 4 + _KindSpecialReachable = 5 // _KindSpecialPinCounter is a special used for objects that are pinned // multiple times - _KindSpecialPinCounter = 5 + _KindSpecialPinCounter = 6 + // _KindSpecialCleanup is for tracking cleanups. + _KindSpecialCleanup = 7 + // _KindSpecialCheckFinalizer adds additional context to a finalizer or cleanup. + // Used only if debug.checkfinalizers != 0. + _KindSpecialCheckFinalizer = 8 + // _KindSpecialBubble is used to associate objects with synctest bubbles. + _KindSpecialBubble = 9 ) type special struct { _ sys.NotInHeap next *special // linked list in span - offset uint16 // span offset of object + offset uintptr // span offset of object kind byte // kind of special } @@ -1849,13 +1989,13 @@ func spanHasNoSpecials(s *mspan) { atomic.And8(&ha.pageSpecials[arenaPage/8], ^(uint8(1) << (arenaPage % 8))) } -// Adds the special record s to the list of special records for +// addspecial adds the special record s to the list of special records for // the object p. All fields of s should be filled in except for // offset & next, which this routine will fill in. // Returns true if the special was successfully added, false otherwise. // (The add will fail only if a record with the same p and s->kind -// already exists.) -func addspecial(p unsafe.Pointer, s *special) bool { +// already exists unless force is set to true.) +func addspecial(p unsafe.Pointer, s *special, force bool) bool { span := spanOfHeap(uintptr(p)) if span == nil { throw("addspecial on invalid pointer") @@ -1874,9 +2014,9 @@ func addspecial(p unsafe.Pointer, s *special) bool { // Find splice point, check for existing record. iter, exists := span.specialFindSplicePoint(offset, kind) - if !exists { + if !exists || force { // Splice in record, fill in offset. - s.offset = uint16(offset) + s.offset = offset s.next = *iter *iter = s spanHasSpecials(span) @@ -1884,7 +2024,10 @@ func addspecial(p unsafe.Pointer, s *special) bool { unlock(&span.speciallock) releasem(mp) - return !exists // already exists + // We're converting p to a uintptr and looking it up, and we + // don't want it to die and get swept while we're doing so. + KeepAlive(p) + return !exists || force // already exists or addition was forced } // Removes the Special record of the given kind for the object p. @@ -1933,11 +2076,11 @@ func (span *mspan) specialFindSplicePoint(offset uintptr, kind byte) (**special, if s == nil { break } - if offset == uintptr(s.offset) && kind == s.kind { + if offset == s.offset && kind == s.kind { found = true break } - if offset < uintptr(s.offset) || (offset == uintptr(s.offset) && kind < s.kind) { + if offset < s.offset || (offset == s.offset && kind < s.kind) { break } iter = &s.next @@ -1968,7 +2111,7 @@ func addfinalizer(p unsafe.Pointer, f *funcval, nret uintptr, fint *_type, ot *p s.nret = nret s.fint = fint s.ot = ot - if addspecial(p, &s.special) { + if addspecial(p, &s.special, false) { // This is responsible for maintaining the same // GC-related invariants as markrootSpans in any // situation where it's possible that markrootSpans @@ -1980,7 +2123,7 @@ func addfinalizer(p unsafe.Pointer, f *funcval, nret uintptr, fint *_type, ot *p // Mark everything reachable from the object // so it's retained for the finalizer. if !span.spanclass.noscan() { - scanobject(base, gcw) + scanObject(base, gcw) } // Mark the finalizer itself, since the // special isn't part of the GC'd heap. @@ -2008,6 +2151,221 @@ func removefinalizer(p unsafe.Pointer) { unlock(&mheap_.speciallock) } +// The described object has a cleanup set for it. +type specialCleanup struct { + _ sys.NotInHeap + special special + fn *funcval + // Globally unique ID for the cleanup, obtained from mheap_.cleanupID. + id uint64 +} + +// addCleanup attaches a cleanup function to the object. Multiple +// cleanups are allowed on an object, and even the same pointer. +// A cleanup id is returned which can be used to uniquely identify +// the cleanup. +func addCleanup(p unsafe.Pointer, f *funcval) uint64 { + lock(&mheap_.speciallock) + s := (*specialCleanup)(mheap_.specialCleanupAlloc.alloc()) + mheap_.cleanupID++ // Increment first. ID 0 is reserved. + id := mheap_.cleanupID + unlock(&mheap_.speciallock) + s.special.kind = _KindSpecialCleanup + s.fn = f + s.id = id + + mp := acquirem() + addspecial(p, &s.special, true) + // This is responsible for maintaining the same + // GC-related invariants as markrootSpans in any + // situation where it's possible that markrootSpans + // has already run but mark termination hasn't yet. + if gcphase != _GCoff { + gcw := &mp.p.ptr().gcw + // Mark the cleanup itself, since the + // special isn't part of the GC'd heap. + scanblock(uintptr(unsafe.Pointer(&s.fn)), goarch.PtrSize, &oneptrmask[0], gcw, nil) + } + releasem(mp) + // Keep f alive. There's a window in this function where it's + // only reachable via the special while the special hasn't been + // added to the specials list yet. This is similar to a bug + // discovered for weak handles, see #70455. + KeepAlive(f) + return id +} + +// Always paired with a specialCleanup or specialfinalizer, adds context. +type specialCheckFinalizer struct { + _ sys.NotInHeap + special special + cleanupID uint64 // Needed to disambiguate cleanups. + createPC uintptr + funcPC uintptr + ptrType *_type +} + +// setFinalizerContext adds a specialCheckFinalizer to ptr. ptr must already have a +// finalizer special attached. +func setFinalizerContext(ptr unsafe.Pointer, ptrType *_type, createPC, funcPC uintptr) { + setCleanupContext(ptr, ptrType, createPC, funcPC, 0) +} + +// setCleanupContext adds a specialCheckFinalizer to ptr. ptr must already have a +// finalizer or cleanup special attached. Pass 0 for the cleanupID to indicate +// a finalizer. +func setCleanupContext(ptr unsafe.Pointer, ptrType *_type, createPC, funcPC uintptr, cleanupID uint64) { + lock(&mheap_.speciallock) + s := (*specialCheckFinalizer)(mheap_.specialCheckFinalizerAlloc.alloc()) + unlock(&mheap_.speciallock) + s.special.kind = _KindSpecialCheckFinalizer + s.cleanupID = cleanupID + s.createPC = createPC + s.funcPC = funcPC + s.ptrType = ptrType + + mp := acquirem() + addspecial(ptr, &s.special, true) + releasem(mp) + KeepAlive(ptr) +} + +func getCleanupContext(ptr uintptr, cleanupID uint64) *specialCheckFinalizer { + assertWorldStopped() + + span := spanOfHeap(ptr) + if span == nil { + return nil + } + var found *specialCheckFinalizer + offset := ptr - span.base() + iter, exists := span.specialFindSplicePoint(offset, _KindSpecialCheckFinalizer) + if exists { + for { + s := *iter + if s == nil { + // Reached the end of the linked list. Stop searching at this point. + break + } + if offset == s.offset && _KindSpecialCheckFinalizer == s.kind && + (*specialCheckFinalizer)(unsafe.Pointer(s)).cleanupID == cleanupID { + // The special is a cleanup and contains a matching cleanup id. + *iter = s.next + found = (*specialCheckFinalizer)(unsafe.Pointer(s)) + break + } + if offset < s.offset || (offset == s.offset && _KindSpecialCheckFinalizer < s.kind) { + // The special is outside the region specified for that kind of + // special. The specials are sorted by kind. + break + } + // Try the next special. + iter = &s.next + } + } + return found +} + +// clearFinalizerContext removes the specialCheckFinalizer for the given pointer, if any. +func clearFinalizerContext(ptr uintptr) { + clearCleanupContext(ptr, 0) +} + +// clearFinalizerContext removes the specialCheckFinalizer for the given pointer and cleanup ID, if any. +func clearCleanupContext(ptr uintptr, cleanupID uint64) { + // The following block removes the Special record of type cleanup for the object c.ptr. + span := spanOfHeap(ptr) + if span == nil { + return + } + // Ensure that the span is swept. + // Sweeping accesses the specials list w/o locks, so we have + // to synchronize with it. And it's just much safer. + mp := acquirem() + span.ensureSwept() + + offset := ptr - span.base() + + var found *special + lock(&span.speciallock) + + iter, exists := span.specialFindSplicePoint(offset, _KindSpecialCheckFinalizer) + if exists { + for { + s := *iter + if s == nil { + // Reached the end of the linked list. Stop searching at this point. + break + } + if offset == s.offset && _KindSpecialCheckFinalizer == s.kind && + (*specialCheckFinalizer)(unsafe.Pointer(s)).cleanupID == cleanupID { + // The special is a cleanup and contains a matching cleanup id. + *iter = s.next + found = s + break + } + if offset < s.offset || (offset == s.offset && _KindSpecialCheckFinalizer < s.kind) { + // The special is outside the region specified for that kind of + // special. The specials are sorted by kind. + break + } + // Try the next special. + iter = &s.next + } + } + if span.specials == nil { + spanHasNoSpecials(span) + } + unlock(&span.speciallock) + releasem(mp) + + if found == nil { + return + } + lock(&mheap_.speciallock) + mheap_.specialCheckFinalizerAlloc.free(unsafe.Pointer(found)) + unlock(&mheap_.speciallock) +} + +// Indicates that an allocation is a tiny block. +// Used only if debug.checkfinalizers != 0. +type specialTinyBlock struct { + _ sys.NotInHeap + special special +} + +// setTinyBlockContext marks an allocation as a tiny block to diagnostics like +// checkfinalizer. +// +// A tiny block is only marked if it actually contains more than one distinct +// value, since we're using this for debugging. +func setTinyBlockContext(ptr unsafe.Pointer) { + lock(&mheap_.speciallock) + s := (*specialTinyBlock)(mheap_.specialTinyBlockAlloc.alloc()) + unlock(&mheap_.speciallock) + s.special.kind = _KindSpecialTinyBlock + + mp := acquirem() + addspecial(ptr, &s.special, false) + releasem(mp) + KeepAlive(ptr) +} + +// inTinyBlock returns whether ptr is in a tiny alloc block, at one point grouped +// with other distinct values. +func inTinyBlock(ptr uintptr) bool { + assertWorldStopped() + + ptr = alignDown(ptr, maxTinySize) + span := spanOfHeap(ptr) + if span == nil { + return false + } + offset := ptr - span.base() + _, exists := span.specialFindSplicePoint(offset, _KindSpecialTinyBlock) + return exists +} + // The described object has a weak pointer. // // Weak pointers in the GC have the following invariants: @@ -2040,17 +2398,28 @@ type specialWeakHandle struct { handle *atomic.Uintptr } -//go:linkname internal_weak_runtime_registerWeakPointer internal/weak.runtime_registerWeakPointer +//go:linkname internal_weak_runtime_registerWeakPointer weak.runtime_registerWeakPointer func internal_weak_runtime_registerWeakPointer(p unsafe.Pointer) unsafe.Pointer { - return unsafe.Pointer(getOrAddWeakHandle(unsafe.Pointer(p))) + return unsafe.Pointer(getOrAddWeakHandle(p)) } -//go:linkname internal_weak_runtime_makeStrongFromWeak internal/weak.runtime_makeStrongFromWeak +//go:linkname internal_weak_runtime_makeStrongFromWeak weak.runtime_makeStrongFromWeak func internal_weak_runtime_makeStrongFromWeak(u unsafe.Pointer) unsafe.Pointer { handle := (*atomic.Uintptr)(u) - // Prevent preemption. We want to make sure that another GC cycle can't start. + // Prevent preemption. We want to make sure that another GC cycle can't start + // and that work.strongFromWeak.block can't change out from under us. mp := acquirem() + + // Yield to the GC if necessary. + if work.strongFromWeak.block { + releasem(mp) + + // Try to park and wait for mark termination. + // N.B. gcParkStrongFromWeak calls acquirem before returning. + mp = gcParkStrongFromWeak() + } + p := handle.Load() if p == 0 { releasem(mp) @@ -2061,7 +2430,15 @@ func internal_weak_runtime_makeStrongFromWeak(u unsafe.Pointer) unsafe.Pointer { // even if it's just some random span. span := spanOfHeap(p) if span == nil { - // The span probably got swept and released. + // If it's immortal, then just return the pointer. + // + // Stay non-preemptible so the GC can't see us convert this potentially + // completely bogus value to an unsafe.Pointer. + if isGoPointerWithoutSpan(unsafe.Pointer(p)) { + releasem(mp) + return unsafe.Pointer(p) + } + // It's heap-allocated, so the span probably just got swept and released. releasem(mp) return nil } @@ -2073,14 +2450,77 @@ func internal_weak_runtime_makeStrongFromWeak(u unsafe.Pointer) unsafe.Pointer { // Even if we just swept some random span that doesn't contain this object, because // this object is long dead and its memory has since been reused, we'll just observe nil. ptr := unsafe.Pointer(handle.Load()) + + // This is responsible for maintaining the same GC-related + // invariants as the Yuasa part of the write barrier. During + // the mark phase, it's possible that we just created the only + // valid pointer to the object pointed to by ptr. If it's only + // ever referenced from our stack, and our stack is blackened + // already, we could fail to mark it. So, mark it now. + if gcphase != _GCoff { + shade(uintptr(ptr)) + } releasem(mp) + + // Explicitly keep ptr alive. This seems unnecessary since we return ptr, + // but let's be explicit since it's important we keep ptr alive across the + // call to shade. + KeepAlive(ptr) return ptr } +// gcParkStrongFromWeak puts the current goroutine on the weak->strong queue and parks. +func gcParkStrongFromWeak() *m { + // Prevent preemption as we check strongFromWeak, so it can't change out from under us. + mp := acquirem() + + for work.strongFromWeak.block { + lock(&work.strongFromWeak.lock) + releasem(mp) // N.B. Holding the lock prevents preemption. + + // Queue ourselves up. + work.strongFromWeak.q.pushBack(getg()) + + // Park. + goparkunlock(&work.strongFromWeak.lock, waitReasonGCWeakToStrongWait, traceBlockGCWeakToStrongWait, 2) + + // Re-acquire the current M since we're going to check the condition again. + mp = acquirem() + + // Re-check condition. We may have awoken in the next GC's mark termination phase. + } + return mp +} + +// gcWakeAllStrongFromWeak wakes all currently blocked weak->strong +// conversions. This is used at the end of a GC cycle. +// +// work.strongFromWeak.block must be false to prevent woken goroutines +// from immediately going back to sleep. +func gcWakeAllStrongFromWeak() { + lock(&work.strongFromWeak.lock) + list := work.strongFromWeak.q.popList() + injectglist(&list) + unlock(&work.strongFromWeak.lock) +} + // Retrieves or creates a weak pointer handle for the object p. func getOrAddWeakHandle(p unsafe.Pointer) *atomic.Uintptr { + if debug.sbrk != 0 { + // debug.sbrk never frees memory, so it'll never go nil. However, we do still + // need a weak handle that's specific to p. Use the immortal weak handle map. + // Keep p alive across the call to getOrAdd defensively, though it doesn't + // really matter in this particular case. + handle := mheap_.immortalWeakHandles.getOrAdd(uintptr(p)) + KeepAlive(p) + return handle + } + // First try to retrieve without allocating. if handle := getWeakHandle(p); handle != nil { + // Keep p alive for the duration of the function to ensure + // that it cannot die while we're trying to do this. + KeepAlive(p) return handle } @@ -2092,7 +2532,7 @@ func getOrAddWeakHandle(p unsafe.Pointer) *atomic.Uintptr { s.special.kind = _KindSpecialWeakHandle s.handle = handle handle.Store(uintptr(p)) - if addspecial(p, &s.special) { + if addspecial(p, &s.special, false) { // This is responsible for maintaining the same // GC-related invariants as markrootSpans in any // situation where it's possible that markrootSpans @@ -2105,7 +2545,17 @@ func getOrAddWeakHandle(p unsafe.Pointer) *atomic.Uintptr { scanblock(uintptr(unsafe.Pointer(&s.handle)), goarch.PtrSize, &oneptrmask[0], gcw, nil) releasem(mp) } - return s.handle + + // Keep p alive for the duration of the function to ensure + // that it cannot die while we're trying to do this. + // + // Same for handle, which is only stored in the special. + // There's a window where it might die if we don't keep it + // alive explicitly. Returning it here is probably good enough, + // but let's be defensive and explicit. See #70455. + KeepAlive(p) + KeepAlive(handle) + return handle } // There was an existing handle. Free the special @@ -2124,14 +2574,20 @@ func getOrAddWeakHandle(p unsafe.Pointer) *atomic.Uintptr { } // Keep p alive for the duration of the function to ensure - // that it cannot die while we're trying to this. + // that it cannot die while we're trying to do this. + // + // Same for handle, just to be defensive. KeepAlive(p) + KeepAlive(handle) return handle } func getWeakHandle(p unsafe.Pointer) *atomic.Uintptr { span := spanOfHeap(uintptr(p)) if span == nil { + if isGoPointerWithoutSpan(p) { + return mheap_.immortalWeakHandles.getOrAdd(uintptr(p)) + } throw("getWeakHandle on invalid pointer") } @@ -2154,9 +2610,86 @@ func getWeakHandle(p unsafe.Pointer) *atomic.Uintptr { unlock(&span.speciallock) releasem(mp) + // Keep p alive for the duration of the function to ensure + // that it cannot die while we're trying to do this. + KeepAlive(p) return handle } +type immortalWeakHandleMap struct { + root atomic.UnsafePointer // *immortalWeakHandle (can't use generics because it's notinheap) +} + +// immortalWeakHandle is a lock-free append-only hash-trie. +// +// Key features: +// - 2-ary trie. Child nodes are indexed by the highest bit (remaining) of the hash of the address. +// - New nodes are placed at the first empty level encountered. +// - When the first child is added to a node, the existing value is not moved into a child. +// This means that we must check the value at each level, not just at the leaf. +// - No deletion or rebalancing. +// - Intentionally devolves into a linked list on hash collisions (the hash bits will all +// get shifted out during iteration, and new nodes will just be appended to the 0th child). +type immortalWeakHandle struct { + _ sys.NotInHeap + + children [2]atomic.UnsafePointer // *immortalObjectMapNode (can't use generics because it's notinheap) + ptr uintptr // &ptr is the weak handle +} + +// handle returns a canonical weak handle. +func (h *immortalWeakHandle) handle() *atomic.Uintptr { + // N.B. Since we just need an *atomic.Uintptr that never changes, we can trivially + // reference ptr to save on some memory in immortalWeakHandle and avoid extra atomics + // in getOrAdd. + return (*atomic.Uintptr)(unsafe.Pointer(&h.ptr)) +} + +// getOrAdd introduces p, which must be a pointer to immortal memory (for example, a linker-allocated +// object) and returns a weak handle. The weak handle will never become nil. +func (tab *immortalWeakHandleMap) getOrAdd(p uintptr) *atomic.Uintptr { + var newNode *immortalWeakHandle + m := &tab.root + hash := memhash(abi.NoEscape(unsafe.Pointer(&p)), 0, goarch.PtrSize) + hashIter := hash + for { + n := (*immortalWeakHandle)(m.Load()) + if n == nil { + // Try to insert a new map node. We may end up discarding + // this node if we fail to insert because it turns out the + // value is already in the map. + // + // The discard will only happen if two threads race on inserting + // the same value. Both might create nodes, but only one will + // succeed on insertion. If two threads race to insert two + // different values, then both nodes will *always* get inserted, + // because the equality checking below will always fail. + // + // Performance note: contention on insertion is likely to be + // higher for small maps, but since this data structure is + // append-only, either the map stays small because there isn't + // much activity, or the map gets big and races to insert on + // the same node are much less likely. + if newNode == nil { + newNode = (*immortalWeakHandle)(persistentalloc(unsafe.Sizeof(immortalWeakHandle{}), goarch.PtrSize, &memstats.gcMiscSys)) + newNode.ptr = p + } + if m.CompareAndSwapNoWB(nil, unsafe.Pointer(newNode)) { + return newNode.handle() + } + // Reload n. Because pointers are only stored once, + // we must have lost the race, and therefore n is not nil + // anymore. + n = (*immortalWeakHandle)(m.Load()) + } + if n.ptr == p { + return n.handle() + } + m = &n.children[hashIter>>(8*goarch.PtrSize-1)] + hashIter <<= 1 + } +} + // The described object is being heap profiled. type specialprofile struct { _ sys.NotInHeap @@ -2171,7 +2704,7 @@ func setprofilebucket(p unsafe.Pointer, b *bucket) { unlock(&mheap_.speciallock) s.special.kind = _KindSpecialProfile s.b = b - if !addspecial(p, &s.special) { + if !addspecial(p, &s.special, false) { throw("setprofilebucket: profile already set") } } @@ -2248,6 +2781,30 @@ func freeSpecial(s *special, p unsafe.Pointer, size uintptr) { lock(&mheap_.speciallock) mheap_.specialPinCounterAlloc.free(unsafe.Pointer(s)) unlock(&mheap_.speciallock) + case _KindSpecialCleanup: + sc := (*specialCleanup)(unsafe.Pointer(s)) + // Cleanups, unlike finalizers, do not resurrect the objects + // they're attached to, so we only need to pass the cleanup + // function, not the object. + gcCleanups.enqueue(sc.fn) + lock(&mheap_.speciallock) + mheap_.specialCleanupAlloc.free(unsafe.Pointer(sc)) + unlock(&mheap_.speciallock) + case _KindSpecialCheckFinalizer: + sc := (*specialCheckFinalizer)(unsafe.Pointer(s)) + lock(&mheap_.speciallock) + mheap_.specialCheckFinalizerAlloc.free(unsafe.Pointer(sc)) + unlock(&mheap_.speciallock) + case _KindSpecialTinyBlock: + st := (*specialTinyBlock)(unsafe.Pointer(s)) + lock(&mheap_.speciallock) + mheap_.specialTinyBlockAlloc.free(unsafe.Pointer(st)) + unlock(&mheap_.speciallock) + case _KindSpecialBubble: + st := (*specialBubble)(unsafe.Pointer(s)) + lock(&mheap_.speciallock) + mheap_.specialBubbleAlloc.free(unsafe.Pointer(st)) + unlock(&mheap_.speciallock) default: throw("bad special kind") panic("not reached") @@ -2413,7 +2970,7 @@ func newArenaMayUnlock() *gcBitsArena { var result *gcBitsArena if gcBitsArenas.free == nil { unlock(&gcBitsArenas.lock) - result = (*gcBitsArena)(sysAlloc(gcBitsChunkBytes, &memstats.gcMiscSys)) + result = (*gcBitsArena)(sysAlloc(gcBitsChunkBytes, &memstats.gcMiscSys, "gc bits")) if result == nil { throw("runtime: cannot allocate memory") } diff --git a/src/runtime/mkduff.go b/src/runtime/mkduff.go index b7f07b5087caf9..06d8251fae6bb4 100644 --- a/src/runtime/mkduff.go +++ b/src/runtime/mkduff.go @@ -32,14 +32,10 @@ import ( ) func main() { - gen("amd64", notags, zeroAMD64, copyAMD64) gen("386", notags, zero386, copy386) gen("arm", notags, zeroARM, copyARM) - gen("arm64", notags, zeroARM64, copyARM64) - gen("loong64", notags, zeroLOONG64, copyLOONG64) gen("ppc64x", tagsPPC64x, zeroPPC64x, copyPPC64x) gen("mips64x", tagsMIPS64x, zeroMIPS64x, copyMIPS64x) - gen("riscv64", notags, zeroRISCV64, copyRISCV64) } func gen(arch string, tags, zero, copy func(io.Writer)) { @@ -177,30 +173,6 @@ func copyARM64(w io.Writer) { fmt.Fprintln(w, "\tRET") } -func zeroLOONG64(w io.Writer) { - // R0: always zero - // R19 (aka REGRT1): ptr to memory to be zeroed - // On return, R19 points to the last zeroed dword. - fmt.Fprintln(w, "TEXT runtime·duffzero(SB), NOSPLIT|NOFRAME, $0-0") - for i := 0; i < 128; i++ { - fmt.Fprintln(w, "\tMOVV\tR0, (R20)") - fmt.Fprintln(w, "\tADDV\t$8, R20") - } - fmt.Fprintln(w, "\tRET") -} - -func copyLOONG64(w io.Writer) { - fmt.Fprintln(w, "TEXT runtime·duffcopy(SB), NOSPLIT|NOFRAME, $0-0") - for i := 0; i < 128; i++ { - fmt.Fprintln(w, "\tMOVV\t(R20), R30") - fmt.Fprintln(w, "\tADDV\t$8, R20") - fmt.Fprintln(w, "\tMOVV\tR30, (R21)") - fmt.Fprintln(w, "\tADDV\t$8, R21") - fmt.Fprintln(w) - } - fmt.Fprintln(w, "\tRET") -} - func tagsPPC64x(w io.Writer) { fmt.Fprintln(w) fmt.Fprintln(w, "//go:build ppc64 || ppc64le") @@ -257,30 +229,3 @@ func copyMIPS64x(w io.Writer) { } fmt.Fprintln(w, "\tRET") } - -func zeroRISCV64(w io.Writer) { - // ZERO: always zero - // X25: ptr to memory to be zeroed - // X25 is updated as a side effect. - fmt.Fprintln(w, "TEXT runtime·duffzero(SB), NOSPLIT|NOFRAME, $0-0") - for i := 0; i < 128; i++ { - fmt.Fprintln(w, "\tMOV\tZERO, (X25)") - fmt.Fprintln(w, "\tADD\t$8, X25") - } - fmt.Fprintln(w, "\tRET") -} - -func copyRISCV64(w io.Writer) { - // X24: ptr to source memory - // X25: ptr to destination memory - // X24 and X25 are updated as a side effect - fmt.Fprintln(w, "TEXT runtime·duffcopy(SB), NOSPLIT|NOFRAME, $0-0") - for i := 0; i < 128; i++ { - fmt.Fprintln(w, "\tMOV\t(X24), X31") - fmt.Fprintln(w, "\tADD\t$8, X24") - fmt.Fprintln(w, "\tMOV\tX31, (X25)") - fmt.Fprintln(w, "\tADD\t$8, X25") - fmt.Fprintln(w) - } - fmt.Fprintln(w, "\tRET") -} diff --git a/src/runtime/mklockrank.go b/src/runtime/mklockrank.go index 1239b4a546ea39..9c503369a35841 100644 --- a/src/runtime/mklockrank.go +++ b/src/runtime/mklockrank.go @@ -41,7 +41,7 @@ const ranks = ` # Sysmon NONE < sysmon -< scavenge, forcegc; +< scavenge, forcegc, computeMaxProcs, updateMaxProcsG; # Defer NONE < defer; @@ -50,22 +50,31 @@ NONE < defer; NONE < sweepWaiters, assistQueue, + strongFromWeakQueue, + cleanupQueue, sweep; # Test only NONE < testR, testW; +# vgetrandom +NONE < vgetrandom; + NONE < timerSend; # Scheduler, timers, netpoll NONE < allocmW, execW, cpuprof, pollCache, pollDesc, wakeableSleep; scavenge, sweep, testR, wakeableSleep, timerSend < hchan; assistQueue, + cleanupQueue, + computeMaxProcs, cpuprof, forcegc, + updateMaxProcsG, hchan, pollDesc, # pollDesc can interact with timers, which can lock sched. scavenge, + strongFromWeakQueue, sweep, sweepWaiters, testR, @@ -93,6 +102,17 @@ NONE < itab < reflectOffs; +# Synctest +hchan, + notifyList, + reflectOffs, + root, + strongFromWeakQueue, + sweepWaiters, + timer, + timers +< synctest; + # User arena state NONE < userArenaState; @@ -118,7 +138,8 @@ allg, reflectOffs, timer, traceStrings, - userArenaState + userArenaState, + vgetrandom # Above MALLOC are things that can allocate memory. < MALLOC # Below MALLOC is the malloc implementation. @@ -143,6 +164,7 @@ gcBitsArenas, profInsert, profMemFuture, spanSetSpine, + synctest, fin, root # Anything that can grow the stack can acquire STACKGROW. @@ -171,6 +193,9 @@ defer, # Below WB is the write barrier implementation. < wbufSpans; +# xRegState allocator +sched < xRegAlloc; + # Span allocator stackLarge, stackpool, @@ -183,7 +208,8 @@ stackLarge, # an mspanSpecial lock, and they're part of the malloc implementation. # Pinner bits might be freed by the span allocator. mheap, mspanSpecial < mheapSpecial; -mheap, mheapSpecial < globalAlloc; +# Fixallocs +mheap, mheapSpecial, xRegAlloc < globalAlloc; # Execution tracer events (with a P) hchan, diff --git a/src/runtime/mkpreempt.go b/src/runtime/mkpreempt.go index 08500a90d532ed..dc951cd9a99f99 100644 --- a/src/runtime/mkpreempt.go +++ b/src/runtime/mkpreempt.go @@ -9,8 +9,10 @@ package main import ( + "bytes" "flag" "fmt" + "go/format" "io" "log" "os" @@ -73,16 +75,14 @@ var regNamesAMD64 = []string{ "X15", } -var out io.Writer - -var arches = map[string]func(){ +var arches = map[string]func(g *gen){ "386": gen386, "amd64": genAMD64, "arm": genARM, "arm64": genARM64, "loong64": genLoong64, - "mips64x": func() { genMIPS(true) }, - "mipsx": func() { genMIPS(false) }, + "mips64x": func(g *gen) { genMIPS(g, true) }, + "mipsx": func(g *gen) { genMIPS(g, false) }, "ppc64x": genPPC64, "riscv64": genRISCV64, "s390x": genS390X, @@ -93,67 +93,161 @@ var beLe = map[string]bool{"mips64x": true, "mipsx": true, "ppc64x": true} func main() { flag.Parse() if flag.NArg() > 0 { - out = os.Stdout for _, arch := range flag.Args() { - gen, ok := arches[arch] + genFn, ok := arches[arch] if !ok { log.Fatalf("unknown arch %s", arch) } - header(arch) - gen() + g := gen{os.Stdout, arch} + g.asmHeader() + genFn(&g) } return } - for arch, gen := range arches { + for arch, genFn := range arches { f, err := os.Create(fmt.Sprintf("preempt_%s.s", arch)) if err != nil { log.Fatal(err) } - out = f - header(arch) - gen() + g := gen{f, arch} + g.asmHeader() + genFn(&g) if err := f.Close(); err != nil { log.Fatal(err) } } } -func header(arch string) { - fmt.Fprintf(out, "// Code generated by mkpreempt.go; DO NOT EDIT.\n\n") - if beLe[arch] { - base := arch[:len(arch)-1] - fmt.Fprintf(out, "//go:build %s || %sle\n\n", base, base) +type gen struct { + w io.Writer + goarch string +} + +func (g *gen) commonHeader() { + fmt.Fprintf(g.w, "// Code generated by mkpreempt.go; DO NOT EDIT.\n\n") + if beLe[g.goarch] { + base := g.goarch[:len(g.goarch)-1] + fmt.Fprintf(g.w, "//go:build %s || %sle\n\n", base, base) } - fmt.Fprintf(out, "#include \"go_asm.h\"\n") - if arch == "amd64" { - fmt.Fprintf(out, "#include \"asm_amd64.h\"\n") +} + +func (g *gen) asmHeader() { + g.commonHeader() + fmt.Fprintf(g.w, "#include \"go_asm.h\"\n") + if g.goarch == "amd64" { + fmt.Fprintf(g.w, "#include \"go_tls.h\"\n") + fmt.Fprintf(g.w, "#include \"asm_amd64.h\"\n") } - fmt.Fprintf(out, "#include \"textflag.h\"\n\n") - fmt.Fprintf(out, "TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0\n") + fmt.Fprintf(g.w, "#include \"textflag.h\"\n\n") + fmt.Fprintf(g.w, "TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0\n") } -func p(f string, args ...any) { +func (g *gen) p(f string, args ...any) { fmted := fmt.Sprintf(f, args...) - fmt.Fprintf(out, "\t%s\n", strings.ReplaceAll(fmted, "\n", "\n\t")) + fmt.Fprintf(g.w, "\t%s\n", strings.ReplaceAll(fmted, "\n", "\n\t")) } -func label(l string) { - fmt.Fprintf(out, "%s\n", l) +func (g *gen) label(l string) { + fmt.Fprintf(g.w, "%s\n", l) +} + +// writeXRegs writes an architecture xregs file. +func writeXRegs(arch string, l *layout) { + var code bytes.Buffer + g := gen{&code, arch} + g.commonHeader() + fmt.Fprintf(g.w, ` +package runtime + +type xRegs struct { +`) + pos := 0 + for _, seq := range l.regs { + for _, r := range seq.regs { + if r.pos != pos && !seq.fixedOffset { + log.Fatalf("padding not implemented") + } + typ := fmt.Sprintf("[%d]byte", r.size) + switch { + case r.size == 4 && r.pos%4 == 0: + typ = "uint32" + case r.size == 8 && r.pos%8 == 0: + typ = "uint64" + } + fmt.Fprintf(g.w, "\t%s %s\n", r.name, typ) + pos += r.size + } + } + fmt.Fprintf(g.w, "}\n") + + path := fmt.Sprintf("preempt_%s.go", arch) + b, err := format.Source(code.Bytes()) + if err != nil { + log.Fatalf("formatting %s: %s", path, err) + } + if err := os.WriteFile(path, b, 0666); err != nil { + log.Fatal(err) + } } type layout struct { stack int - regs []regPos + regs []regSeq sp string // stack pointer register } -type regPos struct { - pos int +type regInfo struct { + size int // register size in bytes + name string // register name + + // Some register names may require a specific suffix. + // In ARM64, a suffix called an "arrangement specifier" can be added to + // a register name. For example: + // + // V0.B16 + // + // In this case, "V0" is the register name, and ".B16" is the suffix. + suffix string + + pos int // position on stack +} +// Some save/restore operations can involve multiple registers in a single +// instruction. For example, the LDP/STP instructions in ARM64: +// +// LDP 8(RSP), (R0, R1) +// STP (R0, R1), 8(RSP) +// +// In these cases, a pair of registers (R0, R1) is used as a single argument. +type regSeq struct { saveOp string restoreOp string - reg string + regs []regInfo + + // By default, all registers are saved on the stack, and the stack pointer offset + // is calculated based on the size of each register. For example (ARM64): + // + // STP (R0, R1), 8(RSP) + // STP (R2, R3), 24(RSP) + // + // However, automatic offset calculation may not always be desirable. + // In some cases, the offset must remain fixed: + // + // VST1.P [V0.B16, V1.B16, V2.B16, V3.B16], 64(R0) + // VST1.P [V4.B16, V5.B16, V6.B16, V7.B16], 64(R0) + // + // In this example, R0 is post-incremented after each instruction, + // so the offset should not be recalculated. For such cases, + // `fixedOffset` is set to true. + fixedOffset bool + + // After conversion to a string, register names are separated by commas + // and may be wrapped in a custom pair of brackets. For example (ARM64): + // + // (R0, R1) // wrapped in parentheses + // [V0.B16, V1.B16, V2.B16, V3.B16] // wrapped in square brackets + brackets [2]string // If this register requires special save and restore, these // give those operations with a %d placeholder for the stack @@ -161,43 +255,100 @@ type regPos struct { save, restore string } -func (l *layout) add(op, reg string, size int) { - l.regs = append(l.regs, regPos{saveOp: op, restoreOp: op, reg: reg, pos: l.stack}) +func (l *layout) add(op, regname string, size int) { + l.regs = append(l.regs, regSeq{saveOp: op, restoreOp: op, regs: []regInfo{{size, regname, "", l.stack}}}) l.stack += size } -func (l *layout) add2(sop, rop, reg string, size int) { - l.regs = append(l.regs, regPos{saveOp: sop, restoreOp: rop, reg: reg, pos: l.stack}) - l.stack += size +func (l *layout) add2(sop, rop string, regs []regInfo, brackets [2]string, fixedOffset bool) { + l.regs = append(l.regs, regSeq{saveOp: sop, restoreOp: rop, regs: regs, brackets: brackets, fixedOffset: fixedOffset}) + if !fixedOffset { + for i := range regs { + regs[i].pos = l.stack + l.stack += regs[i].size + } + } } func (l *layout) addSpecial(save, restore string, size int) { - l.regs = append(l.regs, regPos{save: save, restore: restore, pos: l.stack}) + l.regs = append(l.regs, regSeq{save: save, restore: restore, regs: []regInfo{{size, "", "", l.stack}}}) l.stack += size } -func (l *layout) save() { - for _, reg := range l.regs { - if reg.save != "" { - p(reg.save, reg.pos) +func (rs *regSeq) String() string { + switch len(rs.regs) { + case 0: + log.Fatal("Register sequence must not be empty!") + case 1: + return rs.regs[0].name + default: + names := make([]string, 0) + for _, r := range rs.regs { + name := r.name + r.suffix + names = append(names, name) + } + return rs.brackets[0] + strings.Join(names, ", ") + rs.brackets[1] + } + return "" +} + +func (l *layout) save(g *gen) { + for _, seq := range l.regs { + if len(seq.regs) < 1 { + log.Fatal("Register sequence must not be empty!") + } + // When dealing with a sequence of registers, we assume that only the position + // of the first register is relevant. For example: + // + // STP (R0, R1), 8(RSP) + // STP (R2, R3), 24(RSP) + // + // Here, R0.pos is 8. While we can infer that R1.pos is 16, it doesn't need to + // be explicitly specified, as the STP instruction calculates it automatically. + pos := seq.regs[0].pos + if seq.save != "" { + g.p(seq.save, pos) } else { - p("%s %s, %d(%s)", reg.saveOp, reg.reg, reg.pos, l.sp) + name := seq.String() + g.p("%s %s, %d(%s)", seq.saveOp, name, pos, l.sp) } } } -func (l *layout) restore() { - for i := len(l.regs) - 1; i >= 0; i-- { - reg := l.regs[i] +func (l *layout) restoreInOrder(g *gen, reverse bool) { + var seq []regSeq + if reverse { + seq = make([]regSeq, 0) + for i := len(l.regs) - 1; i >= 0; i-- { + seq = append(seq, l.regs[i]) + } + } else { + seq = l.regs + } + for _, reg := range seq { + if len(reg.regs) < 1 { + log.Fatal("Register sequence must not be empty!") + } + pos := reg.regs[0].pos if reg.restore != "" { - p(reg.restore, reg.pos) + g.p(reg.restore, pos) } else { - p("%s %d(%s), %s", reg.restoreOp, reg.pos, l.sp, reg.reg) + g.p("%s %d(%s), %s", reg.restoreOp, pos, l.sp, reg.String()) } } } -func gen386() { +func (l *layout) restore(g *gen) { + l.restoreInOrder(g, true) +} + +func (l *layout) restoreDirect(g *gen) { + l.restoreInOrder(g, false) +} + +func gen386(g *gen) { + p := g.p + p("PUSHFL") // Save general purpose registers. var l = layout{sp: "SP"} @@ -218,22 +369,26 @@ func gen386() { p("ADJSP $%d", lSSE.stack) p("NOP SP") - l.save() + l.save(g) p("#ifndef %s", softfloat) - lSSE.save() + lSSE.save(g) p("#endif") p("CALL ·asyncPreempt2(SB)") p("#ifndef %s", softfloat) - lSSE.restore() + lSSE.restore(g) p("#endif") - l.restore() + l.restore(g) p("ADJSP $%d", -lSSE.stack) p("POPFL") p("RET") } -func genAMD64() { +func genAMD64(g *gen) { + const xReg = "AX" // *xRegState + + p, label := g.p, g.label + // Assign stack offsets. var l = layout{sp: "SP"} for _, reg := range regNamesAMD64 { @@ -244,37 +399,124 @@ func genAMD64() { l.add("MOVQ", reg, 8) } } - lSSE := layout{stack: l.stack, sp: "SP"} - for _, reg := range regNamesAMD64 { - if strings.HasPrefix(reg, "X") { - lSSE.add("MOVUPS", reg, 16) + // Create layouts for X, Y, and Z registers. + const ( + numXRegs = 16 + numZRegs = 16 // TODO: If we start using upper registers, change to 32 + numKRegs = 8 + ) + lZRegs := layout{sp: xReg} // Non-GP registers + lXRegs, lYRegs := lZRegs, lZRegs + for i := range numZRegs { + lZRegs.add("VMOVDQU64", fmt.Sprintf("Z%d", i), 512/8) + if i < numXRegs { + // Use SSE-only instructions for X registers. + lXRegs.add("MOVUPS", fmt.Sprintf("X%d", i), 128/8) + lYRegs.add("VMOVDQU", fmt.Sprintf("Y%d", i), 256/8) } } + for i := range numKRegs { + lZRegs.add("KMOVQ", fmt.Sprintf("K%d", i), 8) + } + // The Z layout is the most general, so we line up the others with that one. + // We don't have to do this, but it results in a nice Go type. If we split + // this into multiple types, we probably should stop doing this. + for i := range lXRegs.regs { + for j := range lXRegs.regs[i].regs { + lXRegs.regs[i].regs[j].pos = lZRegs.regs[i].regs[j].pos + lYRegs.regs[i].regs[j].pos = lZRegs.regs[i].regs[j].pos + } - // TODO: MXCSR register? + } + writeXRegs(g.goarch, &lZRegs) p("PUSHQ BP") p("MOVQ SP, BP") p("// Save flags before clobbering them") p("PUSHFQ") p("// obj doesn't understand ADD/SUB on SP, but does understand ADJSP") - p("ADJSP $%d", lSSE.stack) + p("ADJSP $%d", l.stack) p("// But vet doesn't know ADJSP, so suppress vet stack checking") p("NOP SP") - l.save() + p("// Save GPs") + l.save(g) + + // In general, the limitations on asynchronous preemption mean we only + // preempt in ABIInternal code. However, there's at least one exception to + // this: when we're in an open-coded transition between an ABIInternal + // function and an ABI0 call. We could more carefully arrange unsafe points + // to avoid ever landing in ABI0, but it's easy to just make this code not + // sensitive to the ABI we're preempting. The CALL to asyncPreempt2 will + // ensure we're in ABIInternal register state. + p("// Save extended register state to p.xRegs.scratch") + p("// Don't make assumptions about ABI register state. See mkpreempt.go") + p("get_tls(CX)") + p("MOVQ g(CX), R14") + p("MOVQ g_m(R14), %s", xReg) + p("MOVQ m_p(%s), %s", xReg, xReg) + p("LEAQ (p_xRegs+xRegPerP_scratch)(%s), %s", xReg, xReg) + + // Which registers do we need to save? + p("#ifdef GOEXPERIMENT_simd") + p("CMPB internal∕cpu·X86+const_offsetX86HasAVX512(SB), $1") + p("JE saveAVX512") + p("CMPB internal∕cpu·X86+const_offsetX86HasAVX2(SB), $1") + p("JE saveAVX2") + p("#endif") + + // No features. Assume only SSE. + label("saveSSE:") + lXRegs.save(g) + p("JMP preempt") + + label("saveAVX2:") + lYRegs.save(g) + p("JMP preempt") - lSSE.save() + label("saveAVX512:") + lZRegs.save(g) + p("JMP preempt") + + label("preempt:") p("CALL ·asyncPreempt2(SB)") - lSSE.restore() - l.restore() - p("ADJSP $%d", -lSSE.stack) + + p("// Restore non-GPs from *p.xRegs.cache") + p("MOVQ g_m(R14), %s", xReg) + p("MOVQ m_p(%s), %s", xReg, xReg) + p("MOVQ (p_xRegs+xRegPerP_cache)(%s), %s", xReg, xReg) + + p("#ifdef GOEXPERIMENT_simd") + p("CMPB internal∕cpu·X86+const_offsetX86HasAVX512(SB), $1") + p("JE restoreAVX512") + p("CMPB internal∕cpu·X86+const_offsetX86HasAVX2(SB), $1") + p("JE restoreAVX2") + p("#endif") + + label("restoreSSE:") + lXRegs.restore(g) + p("JMP restoreGPs") + + label("restoreAVX2:") + lYRegs.restore(g) + p("JMP restoreGPs") + + label("restoreAVX512:") + lZRegs.restore(g) + p("JMP restoreGPs") + + label("restoreGPs:") + p("// Restore GPs") + l.restore(g) + p("ADJSP $%d", -l.stack) p("POPFQ") p("POPQ BP") p("RET") } -func genARM() { +func genARM(g *gen) { + p := g.p + // Add integer registers R0-R12. // R13 (SP), R14 (LR), R15 (PC) are special and not saved here. var l = layout{sp: "R13", stack: 4} // add LR slot @@ -303,22 +545,24 @@ func genARM() { } p("MOVW.W R14, -%d(R13)", lfp.stack) // allocate frame, save LR - l.save() + l.save(g) p("MOVB ·goarmsoftfp(SB), R0\nCMP $0, R0\nBNE nofp") // test goarmsoftfp, and skip FP registers if goarmsoftfp!=0. - lfp.save() - label("nofp:") + lfp.save(g) + g.label("nofp:") p("CALL ·asyncPreempt2(SB)") p("MOVB ·goarmsoftfp(SB), R0\nCMP $0, R0\nBNE nofp2") // test goarmsoftfp, and skip FP registers if goarmsoftfp!=0. - lfp.restore() - label("nofp2:") - l.restore() + lfp.restore(g) + g.label("nofp2:") + l.restore(g) p("MOVW %d(R13), R14", lfp.stack) // sigctxt.pushCall pushes LR on stack, restore it p("MOVW.P %d(R13), R15", lfp.stack+4) // load PC, pop frame (including the space pushed by sigctxt.pushCall) p("UNDEF") // shouldn't get here } -func genARM64() { +func genARM64(g *gen) { + const vReg = "R0" // *xRegState + p := g.p // Add integer registers R0-R26 // R27 (REGTMP), R28 (g), R29 (FP), R30 (LR), R31 (SP) are special // and not saved here. @@ -328,8 +572,11 @@ func genARM64() { i-- continue // R18 is not used, skip } - reg := fmt.Sprintf("(R%d, R%d)", i, i+1) - l.add2("STP", "LDP", reg, 16) + regs := []regInfo{ + {name: fmt.Sprintf("R%d", i), size: 8}, + {name: fmt.Sprintf("R%d", i+1), size: 8}, + } + l.add2("STP", "LDP", regs, [2]string{"(", ")"}, false) } // Add flag registers. l.addSpecial( @@ -342,10 +589,17 @@ func genARM64() { 8) // TODO: FPCR? I don't think we'll change it, so no need to save. // Add floating point registers F0-F31. - for i := 0; i < 31; i += 2 { - reg := fmt.Sprintf("(F%d, F%d)", i, i+1) - l.add2("FSTPD", "FLDPD", reg, 16) + lVRegs := layout{sp: vReg} // Non-GP registers + for i := 0; i < 31; i += 4 { + regs := []regInfo{ + {name: fmt.Sprintf("V%d", i), suffix: ".B16", size: 16, pos: 64}, + {name: fmt.Sprintf("V%d", i+1), suffix: ".B16", size: 16, pos: 64}, + {name: fmt.Sprintf("V%d", i+2), suffix: ".B16", size: 16, pos: 64}, + {name: fmt.Sprintf("V%d", i+3), suffix: ".B16", size: 16, pos: 64}, + } + lVRegs.add2("VST1.P", "VLD1.P", regs, [2]string{"[", "]"}, true) } + writeXRegs(g.goarch, &lVRegs) if l.stack%16 != 0 { l.stack += 8 // SP needs 16-byte alignment } @@ -362,18 +616,32 @@ func genARM64() { p("MOVD R30, (RSP)") p("#endif") - l.save() + p("// Save GPs") + l.save(g) + p("// Save extended register state to p.xRegs.scratch") + p("MOVD g_m(g), %s", vReg) + p("MOVD m_p(%s), %s", vReg, vReg) + p("ADD $(p_xRegs+xRegPerP_scratch), %s, %s", vReg, vReg) + lVRegs.save(g) p("CALL ·asyncPreempt2(SB)") - l.restore() + p("// Restore non-GPs from *p.xRegs.cache") + p("MOVD g_m(g), %s", vReg) + p("MOVD m_p(%s), %s", vReg, vReg) + p("MOVD (p_xRegs+xRegPerP_cache)(%s), %s", vReg, vReg) + lVRegs.restoreDirect(g) + p("// Restore GPs") + l.restore(g) p("MOVD %d(RSP), R30", l.stack) // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it p("MOVD -8(RSP), R29") // restore frame pointer p("MOVD (RSP), R27") // load PC to REGTMP p("ADD $%d, RSP", l.stack+16) // pop frame (including the space pushed by sigctxt.pushCall) - p("JMP (R27)") + p("RET (R27)") } -func genMIPS(_64bit bool) { +func genMIPS(g *gen, _64bit bool) { + p := g.p + mov := "MOVW" movf := "MOVF" add := "ADD" @@ -428,15 +696,15 @@ func genMIPS(_64bit bool) { p(mov+" R31, -%d(R29)", lfp.stack) p(sub+" $%d, R29", lfp.stack) - l.save() + l.save(g) p("#ifndef %s", softfloat) - lfp.save() + lfp.save(g) p("#endif") p("CALL ·asyncPreempt2(SB)") p("#ifndef %s", softfloat) - lfp.restore() + lfp.restore(g) p("#endif") - l.restore() + l.restore(g) p(mov+" %d(R29), R31", lfp.stack) // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it p(mov + " (R29), R23") // load PC to REGTMP @@ -444,9 +712,12 @@ func genMIPS(_64bit bool) { p("JMP (R23)") } -func genLoong64() { +func genLoong64(g *gen) { + const xReg = "R4" // *xRegState + + p, label := g.p, g.label + mov := "MOVV" - movf := "MOVD" add := "ADDV" sub := "SUBV" regsize := 8 @@ -462,25 +733,107 @@ func genLoong64() { l.add(mov, reg, regsize) } - // Add floating point registers F0-F31. - for i := 0; i <= 31; i++ { - reg := fmt.Sprintf("F%d", i) - l.add(movf, reg, regsize) + // Add condition flag register fcc0-fcc7 + sv := "" + rs := "" + last := 7 + for i := 0; i <= last; i++ { + msb := 7 + (i * 8) + lsb := 0 + (i * 8) + + // MOVV FCCx, R4, + // BSTRINSV $msb, R4, $lsb, R5 + sv += fmt.Sprintf("%s FCC%d, R4\n", mov, i) + sv += fmt.Sprintf("BSTRINSV $%d, R4, $%d, R5\n", msb, lsb) + + // BSTRPICKV $msb, R5, $lsb, R4 + // MOVV R4, FCCx + rs += fmt.Sprintf("BSTRPICKV $%d, R5, $%d, R4\n", msb, lsb) + rs += fmt.Sprintf("%s R4, FCC%d", mov, i) + if i != last { + rs += fmt.Sprintf("\n") + } } - - // save/restore FCC0 l.addSpecial( - mov+" FCC0, R4\n"+mov+" R4, %d(R3)", - mov+" %d(R3), R4\n"+mov+" R4, FCC0", + sv+mov+" R5, %d(R3)", + mov+" %d(R3), R5\n"+rs, regsize) + // Create layouts for lasx, lsx and fp registers. + lasxRegs := layout{sp: xReg} + lsxRegs := lasxRegs + fpRegs := lasxRegs + for i := 0; i <= 31; i++ { + lasxRegs.add("XVMOVQ", fmt.Sprintf("X%d", i), 256/8) + lsxRegs.add("VMOVQ", fmt.Sprintf("V%d", i), 128/8) + fpRegs.add("MOVD", fmt.Sprintf("F%d", i), 64/8) + } + + for i := range lsxRegs.regs { + for j := range lsxRegs.regs[i].regs { + lsxRegs.regs[i].regs[j].pos = lasxRegs.regs[i].regs[j].pos + fpRegs.regs[i].regs[j].pos = lasxRegs.regs[i].regs[j].pos + } + } + writeXRegs(g.goarch, &lasxRegs) + // allocate frame, save PC of interrupted instruction (in LR) p(mov+" R1, -%d(R3)", l.stack) p(sub+" $%d, R3", l.stack) - l.save() + p("// Save GPs") + l.save(g) + + p("// Save extended register state to p.xRegs.scratch") + p("MOVV g_m(g), %s", xReg) + p("MOVV m_p(%s), %s", xReg, xReg) + p("ADDV $(p_xRegs+xRegPerP_scratch), %s, %s", xReg, xReg) + + p("MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R5") + p("BNE R5, saveLASX") + + p("MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R5") + p("BNE R5, saveLSX") + + label("saveFP:") + fpRegs.save(g) + p("JMP preempt") + + label("saveLSX:") + lsxRegs.save(g) + p("JMP preempt") + + label("saveLASX:") + lasxRegs.save(g) + + label("preempt:") p("CALL ·asyncPreempt2(SB)") - l.restore() + + p("// Restore non-GPs from *p.xRegs.cache") + p("MOVV g_m(g), %s", xReg) + p("MOVV m_p(%s), %s", xReg, xReg) + p("MOVV (p_xRegs+xRegPerP_cache)(%s), %s", xReg, xReg) + + p("MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R5") + p("BNE R5, restoreLASX") + + p("MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R5") + p("BNE R5, restoreLSX") + + label("restoreFP:") + fpRegs.restore(g) + p("JMP restoreGPs") + + label("restoreLSX:") + lsxRegs.restore(g) + p("JMP restoreGPs") + + label("restoreLASX:") + lasxRegs.restore(g) + + p("// Restore GPs") + label("restoreGPs:") + l.restore(g) p(mov+" %d(R3), R1", l.stack) // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it p(mov + " (R3), R30") // load PC to REGTMP @@ -488,7 +841,9 @@ func genLoong64() { p("JMP (R30)") } -func genPPC64() { +func genPPC64(g *gen) { + p := g.p + // Add integer registers R3-R29 // R0 (zero), R1 (SP), R30 (g) are special and not saved here. // R2 (TOC pointer in PIC mode), R12 (function entry address in PIC mode) have been saved in sigctxt.pushCall. @@ -528,9 +883,9 @@ func genPPC64() { p("MOVD LR, R31") p("MOVDU R31, -%d(R1)", l.stack) // allocate frame, save PC of interrupted instruction (in LR) - l.save() + l.save(g) p("CALL ·asyncPreempt2(SB)") - l.restore() + l.restore(g) p("MOVD %d(R1), R31", l.stack) // sigctxt.pushCall has pushed LR, R2, R12 (at interrupt) on stack, restore them p("MOVD R31, LR") @@ -543,7 +898,9 @@ func genPPC64() { p("JMP (CTR)") } -func genRISCV64() { +func genRISCV64(g *gen) { + p := g.p + // X0 (zero), X1 (LR), X2 (SP), X3 (GP), X4 (TP), X27 (g), X31 (TMP) are special. var l = layout{sp: "X2", stack: 8} @@ -564,16 +921,18 @@ func genRISCV64() { p("MOV X1, -%d(X2)", l.stack) p("SUB $%d, X2", l.stack) - l.save() + l.save(g) p("CALL ·asyncPreempt2(SB)") - l.restore() + l.restore(g) p("MOV %d(X2), X1", l.stack) p("MOV (X2), X31") p("ADD $%d, X2", l.stack+8) p("JMP (X31)") } -func genS390X() { +func genS390X(g *gen) { + p := g.p + // Add integer registers R0-R12 // R13 (g), R14 (LR), R15 (SP) are special, and not saved here. // Saving R10 (REGTMP) is not necessary, but it is saved anyway. @@ -594,9 +953,9 @@ func genS390X() { p("ADD $-%d, R15", l.stack) p("MOVW R10, 8(R15)") // save flags - l.save() + l.save(g) p("CALL ·asyncPreempt2(SB)") - l.restore() + l.restore(g) p("MOVD %d(R15), R14", l.stack) // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it p("ADD $%d, R15", l.stack+8) // pop frame (including the space pushed by sigctxt.pushCall) @@ -606,12 +965,14 @@ func genS390X() { p("JMP (R10)") } -func genWasm() { +func genWasm(g *gen) { + p := g.p p("// No async preemption on wasm") p("UNDEF") } -func notImplemented() { +func notImplemented(g *gen) { + p := g.p p("// Not implemented yet") p("JMP ·abort(SB)") } diff --git a/src/runtime/mpagealloc.go b/src/runtime/mpagealloc.go index 46d3ebacaf8af7..a30434a5343166 100644 --- a/src/runtime/mpagealloc.go +++ b/src/runtime/mpagealloc.go @@ -48,17 +48,21 @@ package runtime import ( + "internal/goarch" "internal/runtime/atomic" + "internal/runtime/gc" "unsafe" ) const ( // The size of a bitmap chunk, i.e. the amount of bits (that is, pages) to consider - // in the bitmap at once. + // in the bitmap at once. It is 4MB on most platforms, except on Wasm it is 512KB. + // We use a smaller chuck size on Wasm for the same reason as the smaller arena + // size (see heapArenaBytes). pallocChunkPages = 1 << logPallocChunkPages pallocChunkBytes = pallocChunkPages * pageSize - logPallocChunkPages = 9 - logPallocChunkBytes = logPallocChunkPages + pageShift + logPallocChunkPages = 9*(1-goarch.IsWasm) + 6*goarch.IsWasm + logPallocChunkBytes = logPallocChunkPages + gc.PageShift // The number of radix bits for each level. // @@ -81,6 +85,8 @@ const ( // there should this change. pallocChunksL2Bits = heapAddrBits - logPallocChunkBytes - pallocChunksL1Bits pallocChunksL1Shift = pallocChunksL2Bits + + vmaNamePageAllocIndex = "page alloc index" ) // maxSearchAddr returns the maximum searchAddr value, which indicates @@ -217,6 +223,7 @@ type pageAlloc struct { // heapAddrBits | L1 Bits | L2 Bits | L2 Entry Size // ------------------------------------------------ // 32 | 0 | 10 | 128 KiB + // 32 (wasm) | 0 | 13 | 128 KiB // 33 (iOS) | 0 | 11 | 256 KiB // 48 | 13 | 13 | 1 MiB // @@ -401,7 +408,7 @@ func (p *pageAlloc) grow(base, size uintptr) { if p.chunks[c.l1()] == nil { // Create the necessary l2 entry. const l2Size = unsafe.Sizeof(*p.chunks[0]) - r := sysAlloc(l2Size, p.sysStat) + r := sysAlloc(l2Size, p.sysStat, vmaNamePageAllocIndex) if r == nil { throw("pageAlloc: out of memory") } @@ -969,6 +976,45 @@ func (p *pageAlloc) free(base, npages uintptr) { p.update(base, npages, true, false) } +// markRandomPaddingPages marks the range of memory [base, base+npages*pageSize] +// as both allocated and scavenged. This is used for randomizing the base heap +// address. Both the alloc and scav bits are set so that the pages are not used +// and so the memory accounting stats are correctly calculated. +// +// Similar to allocRange, it also updates the summaries to reflect the +// newly-updated bitmap. +// +// p.mheapLock must be held. +func (p *pageAlloc) markRandomPaddingPages(base uintptr, npages uintptr) { + assertLockHeld(p.mheapLock) + + limit := base + npages*pageSize - 1 + sc, ec := chunkIndex(base), chunkIndex(limit) + si, ei := chunkPageIndex(base), chunkPageIndex(limit) + if sc == ec { + chunk := p.chunkOf(sc) + chunk.allocRange(si, ei+1-si) + p.scav.index.alloc(sc, ei+1-si) + chunk.scavenged.setRange(si, ei+1-si) + } else { + chunk := p.chunkOf(sc) + chunk.allocRange(si, pallocChunkPages-si) + p.scav.index.alloc(sc, pallocChunkPages-si) + chunk.scavenged.setRange(si, pallocChunkPages-si) + for c := sc + 1; c < ec; c++ { + chunk := p.chunkOf(c) + chunk.allocAll() + p.scav.index.alloc(c, pallocChunkPages) + chunk.scavenged.setAll() + } + chunk = p.chunkOf(ec) + chunk.allocRange(0, ei+1) + p.scav.index.alloc(ec, ei+1) + chunk.scavenged.setRange(0, ei+1) + } + p.update(base, npages, true, true) +} + const ( pallocSumBytes = unsafe.Sizeof(pallocSum(0)) diff --git a/src/runtime/mpagealloc_32bit.go b/src/runtime/mpagealloc_32bit.go index 900146e363af0e..44c7beecbc37cc 100644 --- a/src/runtime/mpagealloc_32bit.go +++ b/src/runtime/mpagealloc_32bit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build 386 || arm || mips || mipsle || wasm +//go:build 386 || arm || mips || mipsle || wasm || (gccgo && (ppc || s390)) // wasm is a treated as a 32-bit architecture for the purposes of the page // allocator, even though it has 64-bit pointers. This is because any wasm @@ -71,12 +71,12 @@ func (p *pageAlloc) sysInit(test bool) { totalSize = alignUp(totalSize, physPageSize) // Reserve memory for all levels in one go. There shouldn't be much for 32-bit. - reservation := sysReserve(nil, totalSize) + reservation := sysReserve(nil, totalSize, "page summary") if reservation == nil { throw("failed to reserve page summary memory") } // There isn't much. Just map it and mark it as used immediately. - sysMap(reservation, totalSize, p.sysStat) + sysMap(reservation, totalSize, p.sysStat, "page summary") sysUsed(reservation, totalSize, totalSize) p.summaryMappedReady += totalSize @@ -123,7 +123,7 @@ func (s *scavengeIndex) sysInit(test bool, sysStat *sysMemStat) (mappedReady uin if test { // Set up the scavenge index via sysAlloc so the test can free it later. scavIndexSize := uintptr(len(scavengeIndexArray)) * unsafe.Sizeof(atomicScavChunkData{}) - s.chunks = ((*[(1 << heapAddrBits) / pallocChunkBytes]atomicScavChunkData)(sysAlloc(scavIndexSize, sysStat)))[:] + s.chunks = ((*[(1 << heapAddrBits) / pallocChunkBytes]atomicScavChunkData)(sysAlloc(scavIndexSize, sysStat, vmaNamePageAllocIndex)))[:] mappedReady = scavIndexSize } else { // Set up the scavenge index. diff --git a/src/runtime/mpagealloc_64bit.go b/src/runtime/mpagealloc_64bit.go index 36cd222360c3ba..2e3643004bc858 100644 --- a/src/runtime/mpagealloc_64bit.go +++ b/src/runtime/mpagealloc_64bit.go @@ -76,7 +76,7 @@ func (p *pageAlloc) sysInit(test bool) { // Reserve b bytes of memory anywhere in the address space. b := alignUp(uintptr(entries)*pallocSumBytes, physPageSize) - r := sysReserve(nil, b) + r := sysReserve(nil, b, "page summary") if r == nil { throw("failed to reserve page summary memory") } @@ -176,13 +176,10 @@ func (p *pageAlloc) sysGrow(base, limit uintptr) { } // Map and commit need. - sysMap(unsafe.Pointer(need.base.addr()), need.size(), p.sysStat) + sysMap(unsafe.Pointer(need.base.addr()), need.size(), p.sysStat, "page alloc") sysUsed(unsafe.Pointer(need.base.addr()), need.size(), need.size()) p.summaryMappedReady += need.size() } - - // Update the scavenge index. - p.summaryMappedReady += p.scav.index.sysGrow(base, limit, p.sysStat) } // sysGrow increases the index's backing store in response to a heap growth. @@ -229,7 +226,7 @@ func (s *scavengeIndex) sysGrow(base, limit uintptr, sysStat *sysMemStat) uintpt // If we've got something to map, map it, and update the slice bounds. if need.size() != 0 { - sysMap(unsafe.Pointer(need.base.addr()), need.size(), sysStat) + sysMap(unsafe.Pointer(need.base.addr()), need.size(), sysStat, "scavenge index") sysUsed(unsafe.Pointer(need.base.addr()), need.size(), need.size()) // Update the indices only after the new memory is valid. if haveMax == 0 || needMin < haveMin { @@ -248,7 +245,7 @@ func (s *scavengeIndex) sysGrow(base, limit uintptr, sysStat *sysMemStat) uintpt func (s *scavengeIndex) sysInit(test bool, sysStat *sysMemStat) uintptr { n := uintptr(1<= 512 { + tests["AllFree64"] = test{ + before: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {}, + }, + scav: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{21, 1}, {63, 65}}, + }, + hits: []hit{ + {64, PageBase(BaseChunkIdx, 0), 2 * PageSize}, + {64, PageBase(BaseChunkIdx, 64), 64 * PageSize}, + {64, PageBase(BaseChunkIdx, 128), 0}, + }, + after: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{0, 192}}, + }, + } + tests["AllFree65"] = test{ + before: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {}, + }, + scav: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{129, 1}}, + }, + hits: []hit{ + {65, PageBase(BaseChunkIdx, 0), 0}, + {65, PageBase(BaseChunkIdx, 65), PageSize}, + {65, PageBase(BaseChunkIdx, 130), 0}, + }, + after: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{0, 195}}, + }, + } + tests["StraddlePallocChunkPages"] = test{ + before: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{0, PallocChunkPages / 2}}, + BaseChunkIdx + 1: {{PallocChunkPages / 2, PallocChunkPages / 2}}, + }, + scav: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {}, + BaseChunkIdx + 1: {{3, 100}}, + }, + hits: []hit{ + {PallocChunkPages, PageBase(BaseChunkIdx, PallocChunkPages/2), 100 * PageSize}, + {PallocChunkPages, 0, 0}, + {1, 0, 0}, + }, + after: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{0, PallocChunkPages}}, + BaseChunkIdx + 1: {{0, PallocChunkPages}}, + }, + } + tests["AllFreePallocChunkPages*7+5"] = test{ before: map[ChunkIdx][]BitRange{ BaseChunkIdx: {}, BaseChunkIdx + 1: {}, @@ -572,7 +551,29 @@ func TestPageAllocAlloc(t *testing.T) { BaseChunkIdx + 6: {{0, PallocChunkPages}}, BaseChunkIdx + 7: {{0, 6}}, }, - }, + } + tests["StraddlePallocChunkPages*2"] = test{ + before: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{0, PallocChunkPages / 2}}, + BaseChunkIdx + 1: {}, + BaseChunkIdx + 2: {{PallocChunkPages / 2, PallocChunkPages / 2}}, + }, + scav: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{0, 7}}, + BaseChunkIdx + 1: {{3, 5}, {121, 10}}, + BaseChunkIdx + 2: {{PallocChunkPages/2 + 12, 2}}, + }, + hits: []hit{ + {PallocChunkPages * 2, PageBase(BaseChunkIdx, PallocChunkPages/2), 15 * PageSize}, + {PallocChunkPages * 2, 0, 0}, + {1, 0, 0}, + }, + after: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{0, PallocChunkPages}}, + BaseChunkIdx + 1: {{0, PallocChunkPages}}, + BaseChunkIdx + 2: {{0, PallocChunkPages}}, + }, + } } // Disable these tests on iOS since we have a small address space. // See #46860. @@ -676,7 +677,6 @@ func TestPageAllocAlloc(t *testing.T) { } } for name, v := range tests { - v := v t.Run(name, func(t *testing.T) { b := NewPageAlloc(v.before, v.scav) defer FreePageAlloc(b) @@ -703,7 +703,6 @@ func TestPageAllocExhaust(t *testing.T) { t.Skip("skipping because virtual memory is limited; see #36210") } for _, npages := range []uintptr{1, 2, 3, 4, 5, 8, 16, 64, 1024, 1025, 2048, 2049} { - npages := npages t.Run(fmt.Sprintf("%d", npages), func(t *testing.T) { // Construct b. bDesc := make(map[ChunkIdx][]BitRange) @@ -754,12 +753,13 @@ func TestPageAllocFree(t *testing.T) { if GOOS == "openbsd" && testing.Short() { t.Skip("skipping because virtual memory is limited; see #36210") } - tests := map[string]struct { + type test struct { before map[ChunkIdx][]BitRange after map[ChunkIdx][]BitRange npages uintptr frees []uintptr - }{ + } + tests := map[string]test{ "Free1": { npages: 1, before: map[ChunkIdx][]BitRange{ @@ -840,34 +840,6 @@ func TestPageAllocFree(t *testing.T) { BaseChunkIdx: {{25, PallocChunkPages - 25}}, }, }, - "Free64": { - npages: 64, - before: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{0, PallocChunkPages}}, - }, - frees: []uintptr{ - PageBase(BaseChunkIdx, 0), - PageBase(BaseChunkIdx, 64), - PageBase(BaseChunkIdx, 128), - }, - after: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{192, PallocChunkPages - 192}}, - }, - }, - "Free65": { - npages: 65, - before: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{0, PallocChunkPages}}, - }, - frees: []uintptr{ - PageBase(BaseChunkIdx, 0), - PageBase(BaseChunkIdx, 65), - PageBase(BaseChunkIdx, 130), - }, - after: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{195, PallocChunkPages - 195}}, - }, - }, "FreePallocChunkPages": { npages: PallocChunkPages, before: map[ChunkIdx][]BitRange{ @@ -965,8 +937,39 @@ func TestPageAllocFree(t *testing.T) { }, }, } + if PallocChunkPages >= 512 { + // avoid constant overflow when PallocChunkPages is small + var PallocChunkPages uint = PallocChunkPages + tests["Free64"] = test{ + npages: 64, + before: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{0, PallocChunkPages}}, + }, + frees: []uintptr{ + PageBase(BaseChunkIdx, 0), + PageBase(BaseChunkIdx, 64), + PageBase(BaseChunkIdx, 128), + }, + after: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{192, PallocChunkPages - 192}}, + }, + } + tests["Free65"] = test{ + npages: 65, + before: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{0, PallocChunkPages}}, + }, + frees: []uintptr{ + PageBase(BaseChunkIdx, 0), + PageBase(BaseChunkIdx, 65), + PageBase(BaseChunkIdx, 130), + }, + after: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{195, PallocChunkPages - 195}}, + }, + } + } for name, v := range tests { - v := v t.Run(name, func(t *testing.T) { b := NewPageAlloc(v.before, nil) defer FreePageAlloc(b) @@ -1021,7 +1024,6 @@ func TestPageAllocAllocAndFree(t *testing.T) { }, } for name, v := range tests { - v := v t.Run(name, func(t *testing.T) { b := NewPageAlloc(v.init, nil) defer FreePageAlloc(b) diff --git a/src/runtime/mpagecache_test.go b/src/runtime/mpagecache_test.go index 6cb0620f7b883c..523a1c0b07f15e 100644 --- a/src/runtime/mpagecache_test.go +++ b/src/runtime/mpagecache_test.go @@ -164,7 +164,6 @@ func TestPageCacheAlloc(t *testing.T) { }, } for name, test := range tests { - test := test t.Run(name, func(t *testing.T) { c := test.cache for i, h := range test.hits { @@ -269,23 +268,6 @@ func TestPageAllocAllocToCache(t *testing.T) { afterScav map[ChunkIdx][]BitRange } tests := map[string]test{ - "AllFree": { - beforeAlloc: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {}, - }, - beforeScav: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{1, 1}, {64, 64}}, - }, - hits: []PageCache{ - NewPageCache(PageBase(BaseChunkIdx, 0), ^uint64(0), 0x2), - NewPageCache(PageBase(BaseChunkIdx, 64), ^uint64(0), ^uint64(0)), - NewPageCache(PageBase(BaseChunkIdx, 128), ^uint64(0), 0), - NewPageCache(PageBase(BaseChunkIdx, 192), ^uint64(0), 0), - }, - afterAlloc: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{0, 256}}, - }, - }, "ManyArena": { beforeAlloc: map[ChunkIdx][]BitRange{ BaseChunkIdx: {{0, PallocChunkPages}}, @@ -306,72 +288,92 @@ func TestPageAllocAllocToCache(t *testing.T) { BaseChunkIdx + 2: {{0, PallocChunkPages}}, }, }, - "NotContiguous": { + + "Fail": { beforeAlloc: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{0, PallocChunkPages}}, - BaseChunkIdx + 0xff: {{0, 0}}, + BaseChunkIdx: {{0, PallocChunkPages}}, + }, + hits: []PageCache{ + NewPageCache(0, 0, 0), + NewPageCache(0, 0, 0), + NewPageCache(0, 0, 0), + }, + afterAlloc: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{0, PallocChunkPages}}, + }, + }, + "RetainScavBits": { + beforeAlloc: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{0, 1}, {10, 2}}, }, beforeScav: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{0, PallocChunkPages}}, - BaseChunkIdx + 0xff: {{31, 67}}, + BaseChunkIdx: {{0, 4}, {11, 1}}, }, hits: []PageCache{ - NewPageCache(PageBase(BaseChunkIdx+0xff, 0), ^uint64(0), ((uint64(1)<<33)-1)<<31), + NewPageCache(PageBase(BaseChunkIdx, 0), ^uint64(0x1|(0x3<<10)), 0x7<<1), }, afterAlloc: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{0, PallocChunkPages}}, - BaseChunkIdx + 0xff: {{0, 64}}, + BaseChunkIdx: {{0, 64}}, }, afterScav: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{0, PallocChunkPages}}, - BaseChunkIdx + 0xff: {{64, 34}}, + BaseChunkIdx: {{0, 1}, {11, 1}}, }, }, - "First": { + } + if PallocChunkPages >= 512 { + tests["AllFree"] = test{ beforeAlloc: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{0, 32}, {33, 31}, {96, 32}}, + BaseChunkIdx: {}, }, beforeScav: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{1, 4}, {31, 5}, {66, 2}}, + BaseChunkIdx: {{1, 1}, {64, 64}}, }, hits: []PageCache{ - NewPageCache(PageBase(BaseChunkIdx, 0), 1<<32, 1<<32), - NewPageCache(PageBase(BaseChunkIdx, 64), (uint64(1)<<32)-1, 0x3<<2), + NewPageCache(PageBase(BaseChunkIdx, 0), ^uint64(0), 0x2), + NewPageCache(PageBase(BaseChunkIdx, 64), ^uint64(0), ^uint64(0)), + NewPageCache(PageBase(BaseChunkIdx, 128), ^uint64(0), 0), + NewPageCache(PageBase(BaseChunkIdx, 192), ^uint64(0), 0), }, afterAlloc: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{0, 128}}, + BaseChunkIdx: {{0, 256}}, }, - }, - "Fail": { + } + tests["NotContiguous"] = test{ beforeAlloc: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{0, PallocChunkPages}}, + BaseChunkIdx: {{0, PallocChunkPages}}, + BaseChunkIdx + 0xff: {{0, 0}}, + }, + beforeScav: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{0, PallocChunkPages}}, + BaseChunkIdx + 0xff: {{31, 67}}, }, hits: []PageCache{ - NewPageCache(0, 0, 0), - NewPageCache(0, 0, 0), - NewPageCache(0, 0, 0), + NewPageCache(PageBase(BaseChunkIdx+0xff, 0), ^uint64(0), ((uint64(1)<<33)-1)<<31), }, afterAlloc: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{0, PallocChunkPages}}, + BaseChunkIdx: {{0, PallocChunkPages}}, + BaseChunkIdx + 0xff: {{0, 64}}, }, - }, - "RetainScavBits": { + afterScav: map[ChunkIdx][]BitRange{ + BaseChunkIdx: {{0, PallocChunkPages}}, + BaseChunkIdx + 0xff: {{64, 34}}, + }, + } + tests["First"] = test{ beforeAlloc: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{0, 1}, {10, 2}}, + BaseChunkIdx: {{0, 32}, {33, 31}, {96, 32}}, }, beforeScav: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{0, 4}, {11, 1}}, + BaseChunkIdx: {{1, 4}, {31, 5}, {66, 2}}, }, hits: []PageCache{ - NewPageCache(PageBase(BaseChunkIdx, 0), ^uint64(0x1|(0x3<<10)), 0x7<<1), + NewPageCache(PageBase(BaseChunkIdx, 0), 1<<32, 1<<32), + NewPageCache(PageBase(BaseChunkIdx, 64), (uint64(1)<<32)-1, 0x3<<2), }, afterAlloc: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{0, 64}}, - }, - afterScav: map[ChunkIdx][]BitRange{ - BaseChunkIdx: {{0, 1}, {11, 1}}, + BaseChunkIdx: {{0, 128}}, }, - }, + } } // Disable these tests on iOS since we have a small address space. // See #46860. @@ -404,7 +406,6 @@ func TestPageAllocAllocToCache(t *testing.T) { } } for name, v := range tests { - v := v t.Run(name, func(t *testing.T) { b := NewPageAlloc(v.beforeAlloc, v.beforeScav) defer FreePageAlloc(b) diff --git a/src/runtime/mpallocbits_test.go b/src/runtime/mpallocbits_test.go index 5095e24220e80d..755f423f960f24 100644 --- a/src/runtime/mpallocbits_test.go +++ b/src/runtime/mpallocbits_test.go @@ -54,35 +54,39 @@ func TestPallocBitsAllocRange(t *testing.T) { want[PallocChunkPages/64-1] = 1 << 63 test(t, PallocChunkPages-1, 1, want) }) - t.Run("Inner", func(t *testing.T) { - want := new(PallocBits) - want[2] = 0x3e - test(t, 129, 5, want) - }) - t.Run("Aligned", func(t *testing.T) { - want := new(PallocBits) - want[2] = ^uint64(0) - want[3] = ^uint64(0) - test(t, 128, 128, want) - }) - t.Run("Begin", func(t *testing.T) { - want := new(PallocBits) - want[0] = ^uint64(0) - want[1] = ^uint64(0) - want[2] = ^uint64(0) - want[3] = ^uint64(0) - want[4] = ^uint64(0) - want[5] = 0x1 - test(t, 0, 321, want) - }) - t.Run("End", func(t *testing.T) { - want := new(PallocBits) - want[PallocChunkPages/64-1] = ^uint64(0) - want[PallocChunkPages/64-2] = ^uint64(0) - want[PallocChunkPages/64-3] = ^uint64(0) - want[PallocChunkPages/64-4] = 1 << 63 - test(t, PallocChunkPages-(64*3+1), 64*3+1, want) - }) + if PallocChunkPages >= 512 { + t.Run("Inner", func(t *testing.T) { + want := new(PallocBits) + want[:][2] = 0x3e + test(t, 129, 5, want) + }) + t.Run("Aligned", func(t *testing.T) { + want := new(PallocBits) + want[:][2] = ^uint64(0) + want[:][3] = ^uint64(0) + test(t, 128, 128, want) + }) + t.Run("Begin", func(t *testing.T) { + want := new(PallocBits) + want[:][0] = ^uint64(0) + want[:][1] = ^uint64(0) + want[:][2] = ^uint64(0) + want[:][3] = ^uint64(0) + want[:][4] = ^uint64(0) + want[:][5] = 0x1 + test(t, 0, 321, want) + }) + t.Run("End", func(t *testing.T) { + // avoid constant overflow when PallocChunkPages is small + var PallocChunkPages uint = PallocChunkPages + want := new(PallocBits) + want[PallocChunkPages/64-1] = ^uint64(0) + want[PallocChunkPages/64-2] = ^uint64(0) + want[PallocChunkPages/64-3] = ^uint64(0) + want[PallocChunkPages/64-4] = 1 << 63 + test(t, PallocChunkPages-(64*3+1), 64*3+1, want) + }) + } t.Run("All", func(t *testing.T) { want := new(PallocBits) for i := range want { @@ -118,10 +122,11 @@ func TestMallocBitsPopcntRange(t *testing.T) { i, n uint // bit range to popcnt over. want uint // expected popcnt result on that range. } - tests := map[string]struct { + type testCase struct { init []BitRange // bit ranges to set to 1 in the bitmap. tests []test // a set of popcnt tests to run over the bitmap. - }{ + } + tests := map[string]testCase{ "None": { tests: []test{ {0, 1, 0}, @@ -157,7 +162,9 @@ func TestMallocBitsPopcntRange(t *testing.T) { {0, PallocChunkPages, PallocChunkPages / 2}, }, }, - "OddBound": { + } + if PallocChunkPages >= 512 { + tests["OddBound"] = testCase{ init: []BitRange{{0, 111}}, tests: []test{ {0, 1, 1}, @@ -172,8 +179,8 @@ func TestMallocBitsPopcntRange(t *testing.T) { {PallocChunkPages / 2, PallocChunkPages / 2, 0}, {0, PallocChunkPages, 111}, }, - }, - "Scattered": { + } + tests["Scattered"] = testCase{ init: []BitRange{ {1, 3}, {5, 1}, {7, 1}, {10, 2}, {13, 1}, {15, 4}, {21, 1}, {23, 1}, {26, 2}, {30, 5}, {36, 2}, {40, 3}, @@ -190,10 +197,9 @@ func TestMallocBitsPopcntRange(t *testing.T) { {1, 128, 74}, {0, PallocChunkPages, 75}, }, - }, + } } for name, v := range tests { - v := v t.Run(name, func(t *testing.T) { b := makePallocBits(v.init) for _, h := range v.tests { @@ -251,23 +257,25 @@ func TestPallocBitsSummarize(t *testing.T) { PackPallocSum(11, 23, 23), }, } - tests["StartMaxEnd"] = test{ - free: []BitRange{{0, 4}, {50, 100}, {PallocChunkPages - 4, 4}}, - hits: []PallocSum{ - PackPallocSum(4, 100, 4), - }, - } - tests["OnlyMax"] = test{ - free: []BitRange{{1, 20}, {35, 241}, {PallocChunkPages - 50, 30}}, - hits: []PallocSum{ - PackPallocSum(0, 241, 0), - }, - } - tests["MultiMax"] = test{ - free: []BitRange{{35, 2}, {40, 5}, {100, 5}}, - hits: []PallocSum{ - PackPallocSum(0, 5, 0), - }, + if PallocChunkPages >= 512 { + tests["StartMaxEnd"] = test{ + free: []BitRange{{0, 4}, {50, 100}, {PallocChunkPages - 4, 4}}, + hits: []PallocSum{ + PackPallocSum(4, 100, 4), + }, + } + tests["OnlyMax"] = test{ + free: []BitRange{{1, 20}, {35, 241}, {PallocChunkPages - 50, 30}}, + hits: []PallocSum{ + PackPallocSum(0, 241, 0), + }, + } + tests["MultiMax"] = test{ + free: []BitRange{{35, 2}, {40, 5}, {100, 5}}, + hits: []PallocSum{ + PackPallocSum(0, 5, 0), + }, + } } tests["One"] = test{ free: []BitRange{{2, 1}}, @@ -282,7 +290,6 @@ func TestPallocBitsSummarize(t *testing.T) { }, } for name, v := range tests { - v := v t.Run(name, func(t *testing.T) { b := makePallocBits(v.free) // In the PallocBits we create 1's represent free spots, but in our actual @@ -329,12 +336,13 @@ func BenchmarkPallocBitsSummarize(b *testing.B) { // Ensures page allocation works. func TestPallocBitsAlloc(t *testing.T) { - tests := map[string]struct { + type test struct { before []BitRange after []BitRange npages uintptr hits []uint - }{ + } + tests := map[string]test{ "AllFree1": { npages: 1, hits: []uint{0, 1, 2, 3, 4, 5}, @@ -350,22 +358,6 @@ func TestPallocBitsAlloc(t *testing.T) { hits: []uint{0, 5, 10, 15, 20}, after: []BitRange{{0, 25}}, }, - "AllFree64": { - npages: 64, - hits: []uint{0, 64, 128}, - after: []BitRange{{0, 192}}, - }, - "AllFree65": { - npages: 65, - hits: []uint{0, 65, 130}, - after: []BitRange{{0, 195}}, - }, - "SomeFree64": { - before: []BitRange{{0, 32}, {64, 32}, {100, PallocChunkPages - 100}}, - npages: 64, - hits: []uint{^uint(0)}, - after: []BitRange{{0, 32}, {64, 32}, {100, PallocChunkPages - 100}}, - }, "NoneFree1": { before: []BitRange{{0, PallocChunkPages}}, npages: 1, @@ -408,21 +400,40 @@ func TestPallocBitsAlloc(t *testing.T) { hits: []uint{PallocChunkPages/2 - 3, ^uint(0)}, after: []BitRange{{0, PallocChunkPages}}, }, - "ExactFit65": { + } + if PallocChunkPages >= 512 { + // avoid constant overflow when PallocChunkPages is small + var PallocChunkPages uint = PallocChunkPages + tests["AllFree64"] = test{ + npages: 64, + hits: []uint{0, 64, 128}, + after: []BitRange{{0, 192}}, + } + tests["AllFree65"] = test{ + npages: 65, + hits: []uint{0, 65, 130}, + after: []BitRange{{0, 195}}, + } + tests["SomeFree64"] = test{ + before: []BitRange{{0, 32}, {64, 32}, {100, PallocChunkPages - 100}}, + npages: 64, + hits: []uint{^uint(0)}, + after: []BitRange{{0, 32}, {64, 32}, {100, PallocChunkPages - 100}}, + } + tests["ExactFit65"] = test{ before: []BitRange{{0, PallocChunkPages/2 - 31}, {PallocChunkPages/2 + 34, PallocChunkPages/2 - 34}}, npages: 65, hits: []uint{PallocChunkPages/2 - 31, ^uint(0)}, after: []BitRange{{0, PallocChunkPages}}, - }, - "SomeFree161": { + } + tests["SomeFree161"] = test{ before: []BitRange{{0, 185}, {331, 1}}, npages: 161, hits: []uint{332}, after: []BitRange{{0, 185}, {331, 162}}, - }, + } } for name, v := range tests { - v := v t.Run(name, func(t *testing.T) { b := makePallocBits(v.before) for iter, i := range v.hits { @@ -442,18 +453,13 @@ func TestPallocBitsAlloc(t *testing.T) { // Ensures page freeing works. func TestPallocBitsFree(t *testing.T) { - tests := map[string]struct { + type test struct { beforeInv []BitRange afterInv []BitRange frees []uint npages uintptr - }{ - "SomeFree": { - npages: 1, - beforeInv: []BitRange{{0, 32}, {64, 32}, {100, 1}}, - frees: []uint{32}, - afterInv: []BitRange{{0, 33}, {64, 32}, {100, 1}}, - }, + } + tests := map[string]test{ "NoneFree1": { npages: 1, frees: []uint{0, 1, 2, 3, 4, 5}, @@ -469,19 +475,26 @@ func TestPallocBitsFree(t *testing.T) { frees: []uint{0, 5, 10, 15, 20}, afterInv: []BitRange{{0, 25}}, }, - "NoneFree64": { + } + if PallocChunkPages >= 512 { + tests["SomeFree"] = test{ + npages: 1, + beforeInv: []BitRange{{0, 32}, {64, 32}, {100, 1}}, + frees: []uint{32}, + afterInv: []BitRange{{0, 33}, {64, 32}, {100, 1}}, + } + tests["NoneFree64"] = test{ npages: 64, frees: []uint{0, 64, 128}, afterInv: []BitRange{{0, 192}}, - }, - "NoneFree65": { + } + tests["NoneFree65"] = test{ npages: 65, frees: []uint{0, 65, 130}, afterInv: []BitRange{{0, 195}}, - }, + } } for name, v := range tests { - v := v t.Run(name, func(t *testing.T) { b := makePallocBits(v.beforeInv) invertPallocBits(b) diff --git a/src/runtime/mprof.go b/src/runtime/mprof.go index a9adc7b6f74566..19e90fbb337e55 100644 --- a/src/runtime/mprof.go +++ b/src/runtime/mprof.go @@ -49,7 +49,7 @@ const ( // desired maximum number of frames after expansion. // This should be at least as large as the largest skip value // used for profiling; otherwise stacks may be truncated inconsistently - maxSkip = 5 + maxSkip = 6 // maxProfStackDepth is the highest valid value for debug.profstackdepth. // It's used for the bucket.stk func. @@ -242,7 +242,7 @@ func newBucket(typ bucketType, nstk int) *bucket { return b } -// stk returns the slice in b holding the stack. The caller can asssume that the +// stk returns the slice in b holding the stack. The caller can assume that the // backing array is immutable. func (b *bucket) stk() []uintptr { stk := (*[maxProfStackDepth]uintptr)(add(unsafe.Pointer(b), unsafe.Sizeof(*b))) @@ -279,7 +279,7 @@ func stkbucket(typ bucketType, size uintptr, stk []uintptr, alloc bool) *bucket // check again under the lock bh = (*buckhashArray)(buckhash.Load()) if bh == nil { - bh = (*buckhashArray)(sysAlloc(unsafe.Sizeof(buckhashArray{}), &memstats.buckhash_sys)) + bh = (*buckhashArray)(sysAlloc(unsafe.Sizeof(buckhashArray{}), &memstats.buckhash_sys, "profiler hash buckets")) if bh == nil { throw("runtime: cannot allocate memory") } @@ -444,7 +444,7 @@ func mProf_Malloc(mp *m, p unsafe.Pointer, size uintptr) { } // Only use the part of mp.profStack we need and ignore the extra space // reserved for delayed inline expansion with frame pointer unwinding. - nstk := callers(4, mp.profStack[:debug.profstackdepth]) + nstk := callers(3, mp.profStack[:debug.profstackdepth+2]) index := (mProfCycle.read() + 2) % uint32(len(memRecord{}.future)) b := stkbucket(memProfile, size, mp.profStack[:nstk], true) @@ -617,113 +617,68 @@ func fpTracebackPartialExpand(skip int, fp unsafe.Pointer, pcBuf []uintptr) int return n } -// lockTimer assists with profiling contention on runtime-internal locks. +// mLockProfile holds information about the runtime-internal lock contention +// experienced and caused by this M, to report in metrics and profiles. // -// There are several steps between the time that an M experiences contention and -// when that contention may be added to the profile. This comes from our -// constraints: We need to keep the critical section of each lock small, -// especially when those locks are contended. The reporting code cannot acquire -// new locks until the M has released all other locks, which means no memory -// allocations and encourages use of (temporary) M-local storage. +// These measurements are subject to some notable constraints: First, the fast +// path for lock and unlock must remain very fast, with a minimal critical +// section. Second, the critical section during contention has to remain small +// too, so low levels of contention are less likely to snowball into large ones. +// The reporting code cannot acquire new locks until the M has released all +// other locks, which means no memory allocations and encourages use of +// (temporary) M-local storage. // -// The M will have space for storing one call stack that caused contention, and -// for the magnitude of that contention. It will also have space to store the -// magnitude of additional contention the M caused, since it only has space to -// remember one call stack and might encounter several contention events before -// it releases all of its locks and is thus able to transfer the local buffer -// into the profile. +// The M has space for storing one call stack that caused contention, and the +// magnitude of that contention. It also has space to store the magnitude of +// additional contention the M caused, since it might encounter several +// contention events before it releases all of its locks and is thus able to +// transfer the locally buffered call stack and magnitude into the profile. // -// The M will collect the call stack when it unlocks the contended lock. That -// minimizes the impact on the critical section of the contended lock, and -// matches the mutex profile's behavior for contention in sync.Mutex: measured -// at the Unlock method. +// The M collects the call stack when it unlocks the contended lock. The +// traceback takes place outside of the lock's critical section. // // The profile for contention on sync.Mutex blames the caller of Unlock for the // amount of contention experienced by the callers of Lock which had to wait. // When there are several critical sections, this allows identifying which of -// them is responsible. +// them is responsible. We must match that reporting behavior for contention on +// runtime-internal locks. // -// Matching that behavior for runtime-internal locks will require identifying -// which Ms are blocked on the mutex. The semaphore-based implementation is -// ready to allow that, but the futex-based implementation will require a bit -// more work. Until then, we report contention on runtime-internal locks with a -// call stack taken from the unlock call (like the rest of the user-space -// "mutex" profile), but assign it a duration value based on how long the -// previous lock call took (like the user-space "block" profile). -// -// Thus, reporting the call stacks of runtime-internal lock contention is -// guarded by GODEBUG for now. Set GODEBUG=runtimecontentionstacks=1 to enable. -// -// TODO(rhysh): plumb through the delay duration, remove GODEBUG, update comment -// -// The M will track this by storing a pointer to the lock; lock/unlock pairs for -// runtime-internal locks are always on the same M. -// -// Together, that demands several steps for recording contention. First, when -// finally acquiring a contended lock, the M decides whether it should plan to -// profile that event by storing a pointer to the lock in its "to be profiled -// upon unlock" field. If that field is already set, it uses the relative -// magnitudes to weight a random choice between itself and the other lock, with -// the loser's time being added to the "additional contention" field. Otherwise -// if the M's call stack buffer is occupied, it does the comparison against that -// sample's magnitude. -// -// Second, having unlocked a mutex the M checks to see if it should capture the -// call stack into its local buffer. Finally, when the M unlocks its last mutex, -// it transfers the local buffer into the profile. As part of that step, it also -// transfers any "additional contention" time to the profile. Any lock -// contention that it experiences while adding samples to the profile will be -// recorded later as "additional contention" and not include a call stack, to -// avoid an echo. -type lockTimer struct { - lock *mutex - timeRate int64 - timeStart int64 - tickStart int64 -} - -func (lt *lockTimer) begin() { - rate := int64(atomic.Load64(&mutexprofilerate)) - - lt.timeRate = gTrackingPeriod - if rate != 0 && rate < lt.timeRate { - lt.timeRate = rate - } - if int64(cheaprand())%lt.timeRate == 0 { - lt.timeStart = nanotime() - } - - if rate > 0 && int64(cheaprand())%rate == 0 { - lt.tickStart = cputicks() - } +// When the M unlocks its last mutex, it transfers the locally buffered call +// stack and magnitude into the profile. As part of that step, it also transfers +// any "additional contention" time to the profile. Any lock contention that it +// experiences while adding samples to the profile will be recorded later as +// "additional contention" and not include a call stack, to avoid an echo. +type mLockProfile struct { + waitTime atomic.Int64 // (nanotime) total time this M has spent waiting in runtime.lockWithRank. Read by runtime/metrics. + stack []uintptr // call stack at the point of this M's unlock call, when other Ms had to wait + cycles int64 // (cputicks) cycles attributable to "stack" + cyclesLost int64 // (cputicks) contention for which we weren't able to record a call stack + haveStack bool // stack and cycles are to be added to the mutex profile (even if cycles is 0) + disabled bool // attribute all time to "lost" } -func (lt *lockTimer) end() { - gp := getg() - - if lt.timeStart != 0 { - nowTime := nanotime() - gp.m.mLockProfile.waitTime.Add((nowTime - lt.timeStart) * lt.timeRate) - } - - if lt.tickStart != 0 { - nowTick := cputicks() - gp.m.mLockProfile.recordLock(nowTick-lt.tickStart, lt.lock) +func (prof *mLockProfile) start() int64 { + if cheaprandn(gTrackingPeriod) == 0 { + return nanotime() } + return 0 } -type mLockProfile struct { - waitTime atomic.Int64 // total nanoseconds spent waiting in runtime.lockWithRank - stack []uintptr // stack that experienced contention in runtime.lockWithRank - pending uintptr // *mutex that experienced contention (to be traceback-ed) - cycles int64 // cycles attributable to "pending" (if set), otherwise to "stack" - cyclesLost int64 // contention for which we weren't able to record a call stack - disabled bool // attribute all time to "lost" +func (prof *mLockProfile) end(start int64) { + if start != 0 { + prof.waitTime.Add((nanotime() - start) * gTrackingPeriod) + } } -func (prof *mLockProfile) recordLock(cycles int64, l *mutex) { - if cycles <= 0 { - return +// recordUnlock prepares data for later addition to the mutex contention +// profile. The M may hold arbitrary locks during this call. +// +// From unlock2, we might not be holding a p in this code. +// +//go:nowritebarrierrec +func (prof *mLockProfile) recordUnlock(cycles int64) { + if cycles < 0 { + cycles = 0 } if prof.disabled { @@ -734,17 +689,13 @@ func (prof *mLockProfile) recordLock(cycles int64, l *mutex) { return } - if uintptr(unsafe.Pointer(l)) == prof.pending { - // Optimization: we'd already planned to profile this same lock (though - // possibly from a different unlock site). - prof.cycles += cycles - return - } - if prev := prof.cycles; prev > 0 { // We can only store one call stack for runtime-internal lock contention // on this M, and we've already got one. Decide which should stay, and // add the other to the report for runtime._LostContendedRuntimeLock. + if cycles == 0 { + return + } prevScore := uint64(cheaprand64()) % uint64(prev) thisScore := uint64(cheaprand64()) % uint64(cycles) if prevScore > thisScore { @@ -754,26 +705,10 @@ func (prof *mLockProfile) recordLock(cycles int64, l *mutex) { prof.cyclesLost += prev } } - // Saving the *mutex as a uintptr is safe because: - // - lockrank_on.go does this too, which gives it regular exercise - // - the lock would only move if it's stack allocated, which means it - // cannot experience multi-M contention - prof.pending = uintptr(unsafe.Pointer(l)) + prof.captureStack() prof.cycles = cycles } -// From unlock2, we might not be holding a p in this code. -// -//go:nowritebarrierrec -func (prof *mLockProfile) recordUnlock(l *mutex) { - if uintptr(unsafe.Pointer(l)) == prof.pending { - prof.captureStack() - } - if gp := getg(); gp.m.locks == 1 && gp.m.mLockProfile.cycles != 0 { - prof.store() - } -} - func (prof *mLockProfile) captureStack() { if debug.profstackdepth == 0 { // profstackdepth is set to 0 by the user, so mp.profStack is nil and we @@ -781,7 +716,7 @@ func (prof *mLockProfile) captureStack() { return } - skip := 3 // runtime.(*mLockProfile).recordUnlock runtime.unlock2 runtime.unlockWithRank + skip := 4 // runtime.(*mLockProfile).recordUnlock runtime.unlock2Wake runtime.unlock2 runtime.unlockWithRank if staticLockRanking { // When static lock ranking is enabled, we'll always be on the system // stack at this point. There will be a runtime.unlockWithRank.func1 @@ -794,19 +729,14 @@ func (prof *mLockProfile) captureStack() { // "runtime.unlock". skip += 1 // runtime.unlockWithRank.func1 } - prof.pending = 0 + prof.haveStack = true prof.stack[0] = logicalStackSentinel - if debug.runtimeContentionStacks.Load() == 0 { - prof.stack[1] = abi.FuncPCABIInternal(_LostContendedRuntimeLock) + sys.PCQuantum - prof.stack[2] = 0 - return - } var nstk int gp := getg() - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() systemstack(func() { var u unwinder u.initAt(pc, sp, 0, gp, unwindSilentErrors|unwindJumpStack) @@ -817,7 +747,18 @@ func (prof *mLockProfile) captureStack() { } } +// store adds the M's local record to the mutex contention profile. +// +// From unlock2, we might not be holding a p in this code. +// +//go:nowritebarrierrec func (prof *mLockProfile) store() { + if gp := getg(); gp.m.locks == 1 && gp.m.mLockProfile.haveStack { + prof.storeSlow() + } +} + +func (prof *mLockProfile) storeSlow() { // Report any contention we experience within this function as "lost"; it's // important that the act of reporting a contention event not lead to a // reportable contention event. This also means we can use prof.stack @@ -835,6 +776,7 @@ func (prof *mLockProfile) store() { cycles, lost := prof.cycles, prof.cyclesLost prof.cycles, prof.cyclesLost = 0, 0 + prof.haveStack = false rate := int64(atomic.Load64(&mutexprofilerate)) saveBlockEventStack(cycles, rate, prof.stack[:nstk], mutexProfile) @@ -1074,7 +1016,7 @@ func copyMemProfileRecord(dst *MemProfileRecord, src profilerecord.MemProfileRec dst.AllocObjects = src.AllocObjects dst.FreeObjects = src.FreeObjects if raceenabled { - racewriterangepc(unsafe.Pointer(&dst.Stack0[0]), unsafe.Sizeof(dst.Stack0), getcallerpc(), abi.FuncPCABIInternal(MemProfile)) + racewriterangepc(unsafe.Pointer(&dst.Stack0[0]), unsafe.Sizeof(dst.Stack0), sys.GetCallerPC(), abi.FuncPCABIInternal(MemProfile)) } if msanenabled { msanwrite(unsafe.Pointer(&dst.Stack0[0]), unsafe.Sizeof(dst.Stack0)) @@ -1136,11 +1078,12 @@ func expandFrames(p []BlockProfileRecord) { for i := range p { cf := CallersFrames(p[i].Stack()) j := 0 - for ; j < len(expandedStack); j++ { + for j < len(expandedStack) { f, more := cf.Next() // f.PC is a "call PC", but later consumers will expect // "return PCs" expandedStack[j] = f.PC + 1 + j++ if !more { break } @@ -1187,7 +1130,7 @@ func copyBlockProfileRecord(dst *BlockProfileRecord, src profilerecord.BlockProf dst.Count = src.Count dst.Cycles = src.Cycles if raceenabled { - racewriterangepc(unsafe.Pointer(&dst.Stack0[0]), unsafe.Sizeof(dst.Stack0), getcallerpc(), abi.FuncPCABIInternal(BlockProfile)) + racewriterangepc(unsafe.Pointer(&dst.Stack0[0]), unsafe.Sizeof(dst.Stack0), sys.GetCallerPC(), abi.FuncPCABIInternal(BlockProfile)) } if msanenabled { msanwrite(unsafe.Pointer(&dst.Stack0[0]), unsafe.Sizeof(dst.Stack0)) @@ -1270,7 +1213,8 @@ func pprof_mutexProfileInternal(p []profilerecord.BlockProfileRecord) (n int, ok // of calling ThreadCreateProfile directly. func ThreadCreateProfile(p []StackRecord) (n int, ok bool) { return threadCreateProfileInternal(len(p), func(r profilerecord.StackRecord) { - copy(p[0].Stack0[:], r.Stack) + i := copy(p[0].Stack0[:], r.Stack) + clear(p[0].Stack0[i:]) p = p[1:] }) } @@ -1315,6 +1259,20 @@ func goroutineProfileWithLabels(p []profilerecord.StackRecord, labels []unsafe.P return goroutineProfileWithLabelsConcurrent(p, labels) } +//go:linkname pprof_goroutineLeakProfileWithLabels +func pprof_goroutineLeakProfileWithLabels(p []profilerecord.StackRecord, labels []unsafe.Pointer) (n int, ok bool) { + return goroutineLeakProfileWithLabelsConcurrent(p, labels) +} + +// labels may be nil. If labels is non-nil, it must have the same length as p. +func goroutineLeakProfileWithLabels(p []profilerecord.StackRecord, labels []unsafe.Pointer) (n int, ok bool) { + if labels != nil && len(labels) != len(p) { + labels = nil + } + + return goroutineLeakProfileWithLabelsConcurrent(p, labels) +} + var goroutineProfile = struct { sema uint32 active bool @@ -1358,13 +1316,55 @@ func (p *goroutineProfileStateHolder) CompareAndSwap(old, new goroutineProfileSt return (*atomic.Uint32)(p).CompareAndSwap(uint32(old), uint32(new)) } +func goroutineLeakProfileWithLabelsConcurrent(p []profilerecord.StackRecord, labels []unsafe.Pointer) (n int, ok bool) { + if len(p) == 0 { + // An empty slice is obviously too small. Return a rough + // allocation estimate. + return work.goroutineLeak.count, false + } + + // Use the same semaphore as goroutineProfileWithLabelsConcurrent, + // because ultimately we still use goroutine profiles. + semacquire(&goroutineProfile.sema) + + // Unlike in goroutineProfileWithLabelsConcurrent, we don't need to + // save the current goroutine stack, because it is obviously not leaked. + + pcbuf := makeProfStack() // see saveg() for explanation + + // Prepare a profile large enough to store all leaked goroutines. + n = work.goroutineLeak.count + + if n > len(p) { + // There's not enough space in p to store the whole profile, so (per the + // contract of runtime.GoroutineProfile) we're not allowed to write to p + // at all and must return n, false. + semrelease(&goroutineProfile.sema) + return n, false + } + + // Visit each leaked goroutine and try to record its stack. + forEachGRace(func(gp1 *g) { + if readgstatus(gp1) == _Gleaked { + doRecordGoroutineProfile(gp1, pcbuf) + } + }) + + if raceenabled { + raceacquire(unsafe.Pointer(&labelSync)) + } + + semrelease(&goroutineProfile.sema) + return n, true +} + func goroutineProfileWithLabelsConcurrent(p []profilerecord.StackRecord, labels []unsafe.Pointer) (n int, ok bool) { if len(p) == 0 { // An empty slice is obviously too small. Return a rough // allocation estimate without bothering to STW. As long as // this is close, then we'll only need to STW once (on the next // call). - return int(gcount()), false + return int(gcount(false)), false } semacquire(&goroutineProfile.sema) @@ -1379,11 +1379,12 @@ func goroutineProfileWithLabelsConcurrent(p []profilerecord.StackRecord, labels // with what we'd get from isSystemGoroutine, we need special handling for // goroutines that can vary between user and system to ensure that the count // doesn't change during the collection. So, check the finalizer goroutine - // in particular. - n = int(gcount()) + // and cleanup goroutines in particular. + n = int(gcount(false)) if fingStatus.Load()&fingRunningFinalizer != 0 { n++ } + n += int(gcCleanups.running.Load()) if n > len(p) { // There's not enough space in p to store the whole profile, so (per the @@ -1395,8 +1396,8 @@ func goroutineProfileWithLabelsConcurrent(p []profilerecord.StackRecord, labels } // Save current goroutine. - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() systemstack(func() { saveg(pc, sp, ourg, &p[0], pcbuf) }) @@ -1414,15 +1415,6 @@ func goroutineProfileWithLabelsConcurrent(p []profilerecord.StackRecord, labels goroutineProfile.active = true goroutineProfile.records = p goroutineProfile.labels = labels - // The finalizer goroutine needs special handling because it can vary over - // time between being a user goroutine (eligible for this profile) and a - // system goroutine (to be excluded). Pick one before restarting the world. - if fing != nil { - fing.goroutineProfiled.Store(goroutineProfileSatisfied) - if readgstatus(fing) != _Gdead && !isSystemGoroutine(fing, false) { - doRecordGoroutineProfile(fing, pcbuf) - } - } startTheWorld(stw) // Visit each goroutine that existed as of the startTheWorld call above. @@ -1462,7 +1454,7 @@ func goroutineProfileWithLabelsConcurrent(p []profilerecord.StackRecord, labels // were collecting the profile. But probably better to return a // truncated profile than to crash the whole process. // - // For instance, needm moves a goroutine out of the _Gdead state and so + // For instance, needm moves a goroutine out of the _Gdeadextra state and so // might be able to change the goroutine count without interacting with // the scheduler. For code like that, the race windows are small and the // combination of features is uncommon, so it's hard to be (and remain) @@ -1488,18 +1480,13 @@ func tryRecordGoroutineProfileWB(gp1 *g) { // in the current goroutine profile: either that it should not be profiled, or // that a snapshot of its call stack and labels are now in the profile. func tryRecordGoroutineProfile(gp1 *g, pcbuf []uintptr, yield func()) { - if readgstatus(gp1) == _Gdead { + if status := readgstatus(gp1); status == _Gdead || status == _Gdeadextra { // Dead goroutines should not appear in the profile. Goroutines that // start while profile collection is active will get goroutineProfiled // set to goroutineProfileSatisfied before transitioning out of _Gdead, // so here we check _Gdead first. return } - if isSystemGoroutine(gp1, true) { - // System goroutines should not appear in the profile. (The finalizer - // goroutine is marked as "already profiled".) - return - } for { prev := gp1.goroutineProfiled.Load() @@ -1537,6 +1524,17 @@ func tryRecordGoroutineProfile(gp1 *g, pcbuf []uintptr, yield func()) { // stack), or from the scheduler in preparation to execute gp1 (running on the // system stack). func doRecordGoroutineProfile(gp1 *g, pcbuf []uintptr) { + if isSystemGoroutine(gp1, false) { + // System goroutines should not appear in the profile. + // Check this here and not in tryRecordGoroutineProfile because isSystemGoroutine + // may change on a goroutine while it is executing, so while the scheduler might + // see a system goroutine, goroutineProfileWithLabelsConcurrent might not, and + // this inconsistency could cause invariants to be violated, such as trying to + // record the stack of a running goroutine below. In short, we still want system + // goroutines to participate in the same state machine on gp1.goroutineProfiled as + // everything else, we just don't record the stack in the profile. + return + } if readgstatus(gp1) == _Grunning { print("doRecordGoroutineProfile gp1=", gp1.goid, "\n") throw("cannot read stack of running goroutine") @@ -1572,7 +1570,16 @@ func goroutineProfileWithLabelsSync(p []profilerecord.StackRecord, labels []unsa isOK := func(gp1 *g) bool { // Checking isSystemGoroutine here makes GoroutineProfile // consistent with both NumGoroutine and Stack. - return gp1 != gp && readgstatus(gp1) != _Gdead && !isSystemGoroutine(gp1, false) + if gp1 == gp { + return false + } + if status := readgstatus(gp1); status == _Gdead || status == _Gdeadextra { + return false + } + if isSystemGoroutine(gp1, false) { + return false + } + return true } pcbuf := makeProfStack() // see saveg() for explanation @@ -1591,8 +1598,8 @@ func goroutineProfileWithLabelsSync(p []profilerecord.StackRecord, labels []unsa r, lbl := p, labels // Save current goroutine. - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() systemstack(func() { saveg(pc, sp, gp, &r[0], pcbuf) }) @@ -1649,7 +1656,8 @@ func GoroutineProfile(p []StackRecord) (n int, ok bool) { return } for i, mr := range records[0:n] { - copy(p[i].Stack0[:], mr.Stack) + l := copy(p[i].Stack0[:], mr.Stack) + clear(p[i].Stack0[l:]) } return } @@ -1693,8 +1701,8 @@ func Stack(buf []byte, all bool) int { n := 0 if len(buf) > 0 { gp := getg() - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() systemstack(func() { g0 := getg() // Force traceback=1 to override GOTRACEBACK setting, diff --git a/src/runtime/msan.go b/src/runtime/msan.go index cb740dc2d86ae5..04378805a81615 100644 --- a/src/runtime/msan.go +++ b/src/runtime/msan.go @@ -64,4 +64,4 @@ func msanmove(dst, src unsafe.Pointer, sz uintptr) //go:cgo_import_static __msan_write_go //go:cgo_import_static __msan_malloc_go //go:cgo_import_static __msan_free_go -//go:cgo_import_static __msan_memmove +//go:cgo_import_static __msan_memmove_go diff --git a/src/runtime/msan/msan.go b/src/runtime/msan/msan.go index 7b3e8e608d840f..8d4471b816e761 100644 --- a/src/runtime/msan/msan.go +++ b/src/runtime/msan/msan.go @@ -13,6 +13,8 @@ package msan #include #include +extern void __msan_memmove(void*, const void*, uintptr_t); + void __msan_read_go(void *addr, uintptr_t sz) { __msan_check_mem_is_initialized(addr, sz); } @@ -28,5 +30,9 @@ void __msan_malloc_go(void *addr, uintptr_t sz) { void __msan_free_go(void *addr, uintptr_t sz) { __msan_poison(addr, sz); } + +void __msan_memmove_go(void *to, const void *from, uintptr_t sz) { + __msan_memmove(to, from, sz); +} */ import "C" diff --git a/src/runtime/msan_amd64.s b/src/runtime/msan_amd64.s index a1dc38806396ce..0c3845890d9274 100644 --- a/src/runtime/msan_amd64.s +++ b/src/runtime/msan_amd64.s @@ -63,8 +63,8 @@ TEXT runtime·msanmove(SB), NOSPLIT, $0-24 MOVQ dst+0(FP), RARG0 MOVQ src+8(FP), RARG1 MOVQ sz+16(FP), RARG2 - // void __msan_memmove(void *dst, void *src, uintptr_t sz); - MOVQ $__msan_memmove(SB), AX + // void __msan_memmove_go(void *dst, void *src, uintptr_t sz); + MOVQ $__msan_memmove_go(SB), AX JMP msancall<>(SB) // Switches SP to g0 stack and calls (AX). Arguments already set. @@ -76,7 +76,12 @@ TEXT msancall<>(SB), NOSPLIT, $0-0 JE call // no g; still on a system stack MOVQ g_m(R14), R13 - // Switch to g0 stack. + + // Switch to g0 stack if we aren't already on g0 or gsignal. + MOVQ m_gsignal(R13), R10 + CMPQ R10, R14 + JE call // already on gsignal + MOVQ m_g0(R13), R10 CMPQ R10, R14 JE call // already on g0 diff --git a/src/runtime/msan_arm64.s b/src/runtime/msan_arm64.s index ce475cf44a5fc7..fe8723ea60e4a1 100644 --- a/src/runtime/msan_arm64.s +++ b/src/runtime/msan_arm64.s @@ -51,23 +51,29 @@ TEXT runtime·msanmove(SB), NOSPLIT, $0-24 MOVD dst+0(FP), RARG0 MOVD src+8(FP), RARG1 MOVD sz+16(FP), RARG2 - // void __msan_memmove(void *dst, void *src, uintptr_t sz); - MOVD $__msan_memmove(SB), FARG + // void __msan_memmove_go(void *dst, void *src, uintptr_t sz); + MOVD $__msan_memmove_go(SB), FARG JMP msancall<>(SB) // Switches SP to g0 stack and calls (FARG). Arguments already set. TEXT msancall<>(SB), NOSPLIT, $0-0 MOVD RSP, R19 // callee-saved - CBZ g, g0stack // no g, still on a system stack + CBZ g, call // no g, still on a system stack MOVD g_m(g), R10 + + // Switch to g0 stack if we aren't already on g0 or gsignal. + MOVD m_gsignal(R10), R11 + CMP R11, g + BEQ call + MOVD m_g0(R10), R11 CMP R11, g - BEQ g0stack + BEQ call MOVD (g_sched+gobuf_sp)(R11), R4 MOVD R4, RSP -g0stack: +call: BL (FARG) MOVD R19, RSP RET diff --git a/src/runtime/msan_loong64.s b/src/runtime/msan_loong64.s index b9fa5fd1204165..713201716a5ebf 100644 --- a/src/runtime/msan_loong64.s +++ b/src/runtime/msan_loong64.s @@ -51,17 +51,22 @@ TEXT runtime·msanmove(SB), NOSPLIT, $0-24 MOVV dst+0(FP), RARG0 MOVV src+8(FP), RARG1 MOVV sz+16(FP), RARG2 - // void __msan_memmove(void *dst, void *src, uintptr_t sz); - MOVV $__msan_memmove(SB), FARG + // void __msan_memmove_go(void *dst, void *src, uintptr_t sz); + MOVV $__msan_memmove_go(SB), FARG JMP msancall<>(SB) // Switches SP to g0 stack and calls (FARG). Arguments already set. TEXT msancall<>(SB), NOSPLIT, $0-0 MOVV R3, R23 // callee-saved - BEQ g, g0stack // no g, still on a system stack + BEQ g, call // no g, still on a system stack MOVV g_m(g), R14 + + // Switch to g0 stack if we aren't already on g0 or gsignal. + MOVV m_gsignal(R14), R15 + BEQ R15, g, call + MOVV m_g0(R14), R15 - BEQ R15, g, g0stack + BEQ R15, g, call MOVV (g_sched+gobuf_sp)(R15), R9 MOVV R9, R3 diff --git a/src/runtime/msize.go b/src/runtime/msize.go index 64d1531ab098fd..09da7459b24745 100644 --- a/src/runtime/msize.go +++ b/src/runtime/msize.go @@ -9,21 +9,23 @@ package runtime +import "internal/runtime/gc" + // Returns size of the memory block that mallocgc will allocate if you ask for the size, // minus any inline space for metadata. func roundupsize(size uintptr, noscan bool) (reqSize uintptr) { reqSize = size - if reqSize <= maxSmallSize-mallocHeaderSize { + if reqSize <= maxSmallSize-gc.MallocHeaderSize { // Small object. - if !noscan && reqSize > minSizeForMallocHeader { // !noscan && !heapBitsInSpan(reqSize) - reqSize += mallocHeaderSize + if !noscan && reqSize > gc.MinSizeForMallocHeader { // !noscan && !heapBitsInSpan(reqSize) + reqSize += gc.MallocHeaderSize } // (reqSize - size) is either mallocHeaderSize or 0. We need to subtract mallocHeaderSize // from the result if we have one, since mallocgc will add it back in. - if reqSize <= smallSizeMax-8 { - return uintptr(class_to_size[size_to_class8[divRoundUp(reqSize, smallSizeDiv)]]) - (reqSize - size) + if reqSize <= gc.SmallSizeMax-8 { + return uintptr(gc.SizeClassToSize[gc.SizeToSizeClass8[divRoundUp(reqSize, gc.SmallSizeDiv)]]) - (reqSize - size) } - return uintptr(class_to_size[size_to_class128[divRoundUp(reqSize-smallSizeMax, largeSizeDiv)]]) - (reqSize - size) + return uintptr(gc.SizeClassToSize[gc.SizeToSizeClass128[divRoundUp(reqSize-gc.SmallSizeMax, gc.LargeSizeDiv)]]) - (reqSize - size) } // Large object. Align reqSize up to the next page. Check for overflow. reqSize += pageSize - 1 diff --git a/src/runtime/mspanset.go b/src/runtime/mspanset.go index 3aa2b5b3937f0a..68d2dd0d1eee2f 100644 --- a/src/runtime/mspanset.go +++ b/src/runtime/mspanset.go @@ -56,7 +56,7 @@ const ( spanSetInitSpineCap = 256 // Enough for 1GB heap on 64-bit ) -type spanSetBlock struct { +type spanSetBlockHeader struct { // Free spanSetBlocks are managed via a lock-free stack. lfnode @@ -64,6 +64,15 @@ type spanSetBlock struct { // this block. This number is used to help determine when a block // may be safely recycled. popped atomic.Uint32 +} + +type spanSetBlockHeader2 struct { + spanSetBlockHeader + pad [tagAlign - unsafe.Sizeof(spanSetBlockHeader{})]byte +} + +type spanSetBlock struct { + spanSetBlockHeader2 // spans is the set of spans in this block. spans [spanSetBlockEntries]atomicMSpanPointer @@ -140,6 +149,11 @@ retry: // pop is safe to call concurrently with other pop and push operations. func (b *spanSet) pop() *mspan { var head, tail uint32 + var backoff uint32 + // TODO: tweak backoff parameters on other architectures. + if GOARCH == "arm64" { + backoff = 128 + } claimLoop: for { headtail := b.index.load() @@ -168,6 +182,14 @@ claimLoop: if b.index.cas(headtail, makeHeadTailIndex(want+1, tail)) { break claimLoop } + // Use a backoff approach to reduce demand to the shared memory location + // decreases memory contention and allows for other threads to make quicker + // progress. + // Read more in this Arm blog post: + // https://community.arm.com/arm-community-blogs/b/architectures-and-processors-blog/posts/multi-threaded-applications-arm + procyield(backoff) + // Increase backoff time. + backoff += backoff / 2 headtail = b.index.load() head, tail = headtail.split() } @@ -313,7 +335,7 @@ func (p *spanSetBlockAlloc) alloc() *spanSetBlock { if s := (*spanSetBlock)(p.stack.pop()); s != nil { return s } - return (*spanSetBlock)(persistentalloc(unsafe.Sizeof(spanSetBlock{}), cpu.CacheLineSize, &memstats.gcMiscSys)) + return (*spanSetBlock)(persistentalloc(unsafe.Sizeof(spanSetBlock{}), max(cpu.CacheLineSize, tagAlign), &memstats.gcMiscSys)) } // free returns a spanSetBlock back to the pool. @@ -398,7 +420,7 @@ func (p *atomicMSpanPointer) Load() *mspan { return (*mspan)(p.p.Load()) } -// Store stores an *mspan. +// StoreNoWB stores an *mspan. func (p *atomicMSpanPointer) StoreNoWB(s *mspan) { p.p.StoreNoWB(unsafe.Pointer(s)) } diff --git a/src/runtime/mstats.go b/src/runtime/mstats.go index c10ca402217cfb..e34f0b10ea013e 100644 --- a/src/runtime/mstats.go +++ b/src/runtime/mstats.go @@ -8,6 +8,7 @@ package runtime import ( "internal/runtime/atomic" + "internal/runtime/gc" "unsafe" ) @@ -43,6 +44,8 @@ type mstats struct { last_gc_nanotime uint64 // last gc (monotonic time) lastHeapInUse uint64 // heapInUse at mark termination of the previous GC + lastScanStats [gc.NumSizeClasses]sizeClassScanStats + enablegc bool } @@ -397,23 +400,23 @@ func readmemstats_m(stats *MemStats) { nFree := consStats.largeFreeCount // Collect per-sizeclass stats. - var bySize [_NumSizeClasses]struct { + var bySize [gc.NumSizeClasses]struct { Size uint32 Mallocs uint64 Frees uint64 } for i := range bySize { - bySize[i].Size = uint32(class_to_size[i]) + bySize[i].Size = uint32(gc.SizeClassToSize[i]) // Malloc stats. a := consStats.smallAllocCount[i] - totalAlloc += a * uint64(class_to_size[i]) + totalAlloc += a * uint64(gc.SizeClassToSize[i]) nMalloc += a bySize[i].Mallocs = a // Free stats. f := consStats.smallFreeCount[i] - totalFree += f * uint64(class_to_size[i]) + totalFree += f * uint64(gc.SizeClassToSize[i]) nFree += f bySize[i].Frees = f } @@ -431,12 +434,11 @@ func readmemstats_m(stats *MemStats) { stackInUse := uint64(consStats.inStacks) gcWorkBufInUse := uint64(consStats.inWorkBufs) - gcProgPtrScalarBitsInUse := uint64(consStats.inPtrScalarBits) totalMapped := gcController.heapInUse.load() + gcController.heapFree.load() + gcController.heapReleased.load() + memstats.stacks_sys.load() + memstats.mspan_sys.load() + memstats.mcache_sys.load() + memstats.buckhash_sys.load() + memstats.gcMiscSys.load() + memstats.other_sys.load() + - stackInUse + gcWorkBufInUse + gcProgPtrScalarBitsInUse + stackInUse + gcWorkBufInUse heapGoal := gcController.heapGoal() @@ -450,7 +452,7 @@ func readmemstats_m(stats *MemStats) { // // * memstats.heapInUse == inHeap // * memstats.heapReleased == released - // * memstats.heapInUse + memstats.heapFree == committed - inStacks - inWorkBufs - inPtrScalarBits + // * memstats.heapInUse + memstats.heapFree == committed - inStacks - inWorkBufs // * memstats.totalAlloc == totalAlloc // * memstats.totalFree == totalFree // @@ -471,7 +473,7 @@ func readmemstats_m(stats *MemStats) { throw("heapReleased and consistent stats are not equal") } heapRetained := gcController.heapInUse.load() + gcController.heapFree.load() - consRetained := uint64(consStats.committed - consStats.inStacks - consStats.inWorkBufs - consStats.inPtrScalarBits) + consRetained := uint64(consStats.committed - consStats.inStacks - consStats.inWorkBufs) if heapRetained != consRetained { print("runtime: global value=", heapRetained, "\n") print("runtime: consistent value=", consRetained, "\n") @@ -522,8 +524,8 @@ func readmemstats_m(stats *MemStats) { // // or // - // HeapSys = sys - stacks_inuse - gcWorkBufInUse - gcProgPtrScalarBitsInUse - // HeapIdle = sys - stacks_inuse - gcWorkBufInUse - gcProgPtrScalarBitsInUse - heapInUse + // HeapSys = sys - stacks_inuse - gcWorkBufInUse + // HeapIdle = sys - stacks_inuse - gcWorkBufInUse - heapInUse // // => HeapIdle = HeapSys - heapInUse = heapFree + heapReleased stats.HeapIdle = gcController.heapFree.load() + gcController.heapReleased.load() @@ -542,7 +544,7 @@ func readmemstats_m(stats *MemStats) { // MemStats defines GCSys as an aggregate of all memory related // to the memory management system, but we track this memory // at a more granular level in the runtime. - stats.GCSys = memstats.gcMiscSys.load() + gcWorkBufInUse + gcProgPtrScalarBitsInUse + stats.GCSys = memstats.gcMiscSys.load() + gcWorkBufInUse stats.OtherSys = memstats.other_sys.load() stats.NextGC = heapGoal stats.LastGC = memstats.last_gc_unix @@ -667,24 +669,23 @@ func (s *sysMemStat) add(n int64) { // consistent with one another. type heapStatsDelta struct { // Memory stats. - committed int64 // byte delta of memory committed - released int64 // byte delta of released memory generated - inHeap int64 // byte delta of memory placed in the heap - inStacks int64 // byte delta of memory reserved for stacks - inWorkBufs int64 // byte delta of memory reserved for work bufs - inPtrScalarBits int64 // byte delta of memory reserved for unrolled GC prog bits + committed int64 // byte delta of memory committed + released int64 // byte delta of released memory generated + inHeap int64 // byte delta of memory placed in the heap + inStacks int64 // byte delta of memory reserved for stacks + inWorkBufs int64 // byte delta of memory reserved for work bufs // Allocator stats. // // These are all uint64 because they're cumulative, and could quickly wrap // around otherwise. - tinyAllocCount uint64 // number of tiny allocations - largeAlloc uint64 // bytes allocated for large objects - largeAllocCount uint64 // number of large object allocations - smallAllocCount [_NumSizeClasses]uint64 // number of allocs for small objects - largeFree uint64 // bytes freed for large objects (>maxSmallSize) - largeFreeCount uint64 // number of frees for large objects (>maxSmallSize) - smallFreeCount [_NumSizeClasses]uint64 // number of frees for small objects (<=maxSmallSize) + tinyAllocCount uint64 // number of tiny allocations + largeAlloc uint64 // bytes allocated for large objects + largeAllocCount uint64 // number of large object allocations + smallAllocCount [gc.NumSizeClasses]uint64 // number of allocs for small objects + largeFree uint64 // bytes freed for large objects (>maxSmallSize) + largeFreeCount uint64 // number of frees for large objects (>maxSmallSize) + smallFreeCount [gc.NumSizeClasses]uint64 // number of frees for small objects (<=maxSmallSize) // NOTE: This struct must be a multiple of 8 bytes in size because it // is stored in an array. If it's not, atomic accesses to the above @@ -698,7 +699,6 @@ func (a *heapStatsDelta) merge(b *heapStatsDelta) { a.inHeap += b.inHeap a.inStacks += b.inStacks a.inWorkBufs += b.inWorkBufs - a.inPtrScalarBits += b.inPtrScalarBits a.tinyAllocCount += b.tinyAllocCount a.largeAlloc += b.largeAlloc @@ -834,9 +834,7 @@ func (m *consistentHeapStats) unsafeRead(out *heapStatsDelta) { func (m *consistentHeapStats) unsafeClear() { assertWorldStopped() - for i := range m.stats { - m.stats[i] = heapStatsDelta{} - } + clear(m.stats[:]) } // read takes a globally consistent snapshot of m diff --git a/src/runtime/mwbbuf.go b/src/runtime/mwbbuf.go index b998d2b2bdf5f9..e8c6064905afe6 100644 --- a/src/runtime/mwbbuf.go +++ b/src/runtime/mwbbuf.go @@ -215,7 +215,7 @@ func wbBufFlush1(pp *p) { // pointers we greyed. We use the buffer itself to temporarily // record greyed pointers. // - // TODO: Should scanobject/scanblock just stuff pointers into + // TODO: Should scanObject/scanblock just stuff pointers into // the wbBuf? Then this would become the sole greying path. // // TODO: We could avoid shading any of the "new" pointers in @@ -237,6 +237,9 @@ func wbBufFlush1(pp *p) { // path to reduce the rate of flushes? continue } + if tryDeferToSpanScan(ptr, gcw) { + continue + } obj, span, objIndex := findObject(ptr, 0, 0) if obj == 0 { continue @@ -264,7 +267,7 @@ func wbBufFlush1(pp *p) { } // Enqueue the greyed objects. - gcw.putBatch(ptrs[:pos]) + gcw.putObjBatch(ptrs[:pos]) pp.wbBuf.reset() } diff --git a/src/runtime/netpoll.go b/src/runtime/netpoll.go index 36b9edfe21780a..fab921e2d385bf 100644 --- a/src/runtime/netpoll.go +++ b/src/runtime/netpoll.go @@ -31,7 +31,7 @@ import ( // poll without blocking. If delta > 0, block for up to delta nanoseconds. // Return a list of goroutines built by calling netpollready, // and a delta to add to netpollWaiters when all goroutines are ready. -// This will never return an empty list with a non-zero delta. +// This must never return an empty list with a non-zero delta. // // func netpollBreak() // Wake up the network poller, assumed to be blocked in netpoll. @@ -302,7 +302,7 @@ func (c *pollCache) free(pd *pollDesc) { // Increment the fdseq field, so that any currently // running netpoll calls will not mark pd as ready. fdseq := pd.fdseq.Load() - fdseq = (fdseq + 1) & (1< 0: block for up to that many nanoseconds diff --git a/src/runtime/netpoll_epoll.go b/src/runtime/netpoll_epoll.go index ff6e0b5f89ccef..e658cc3d89c666 100644 --- a/src/runtime/netpoll_epoll.go +++ b/src/runtime/netpoll_epoll.go @@ -8,7 +8,7 @@ package runtime import ( "internal/runtime/atomic" - "internal/runtime/syscall" + "internal/runtime/syscall/linux" "unsafe" ) @@ -20,21 +20,21 @@ var ( func netpollinit() { var errno uintptr - epfd, errno = syscall.EpollCreate1(syscall.EPOLL_CLOEXEC) + epfd, errno = linux.EpollCreate1(linux.EPOLL_CLOEXEC) if errno != 0 { println("runtime: epollcreate failed with", errno) throw("runtime: netpollinit failed") } - efd, errno := syscall.Eventfd(0, syscall.EFD_CLOEXEC|syscall.EFD_NONBLOCK) + efd, errno := linux.Eventfd(0, linux.EFD_CLOEXEC|linux.EFD_NONBLOCK) if errno != 0 { - println("runtime: eventfd failed with", -errno) + println("runtime: eventfd failed with", errno) throw("runtime: eventfd failed") } - ev := syscall.EpollEvent{ - Events: syscall.EPOLLIN, + ev := linux.EpollEvent{ + Events: linux.EPOLLIN, } *(**uintptr)(unsafe.Pointer(&ev.Data)) = &netpollEventFd - errno = syscall.EpollCtl(epfd, syscall.EPOLL_CTL_ADD, efd, &ev) + errno = linux.EpollCtl(epfd, linux.EPOLL_CTL_ADD, efd, &ev) if errno != 0 { println("runtime: epollctl failed with", errno) throw("runtime: epollctl failed") @@ -47,16 +47,16 @@ func netpollIsPollDescriptor(fd uintptr) bool { } func netpollopen(fd uintptr, pd *pollDesc) uintptr { - var ev syscall.EpollEvent - ev.Events = syscall.EPOLLIN | syscall.EPOLLOUT | syscall.EPOLLRDHUP | syscall.EPOLLET + var ev linux.EpollEvent + ev.Events = linux.EPOLLIN | linux.EPOLLOUT | linux.EPOLLRDHUP | linux.EPOLLET tp := taggedPointerPack(unsafe.Pointer(pd), pd.fdseq.Load()) *(*taggedPointer)(unsafe.Pointer(&ev.Data)) = tp - return syscall.EpollCtl(epfd, syscall.EPOLL_CTL_ADD, int32(fd), &ev) + return linux.EpollCtl(epfd, linux.EPOLL_CTL_ADD, int32(fd), &ev) } func netpollclose(fd uintptr) uintptr { - var ev syscall.EpollEvent - return syscall.EpollCtl(epfd, syscall.EPOLL_CTL_DEL, int32(fd), &ev) + var ev linux.EpollEvent + return linux.EpollCtl(epfd, linux.EPOLL_CTL_DEL, int32(fd), &ev) } func netpollarm(pd *pollDesc, mode int) { @@ -89,7 +89,10 @@ func netpollBreak() { } // netpoll checks for ready network connections. -// Returns list of goroutines that become runnable. +// Returns a list of goroutines that become runnable, +// and a delta to add to netpollWaiters. +// This must never return an empty list with a non-zero delta. +// // delay < 0: blocks indefinitely // delay == 0: does not block, just polls // delay > 0: block for up to that many nanoseconds @@ -111,9 +114,9 @@ func netpoll(delay int64) (gList, int32) { // 1e9 ms == ~11.5 days. waitms = 1e9 } - var events [128]syscall.EpollEvent + var events [128]linux.EpollEvent retry: - n, errno := syscall.EpollWait(epfd, events[:], int32(len(events)), waitms) + n, errno := linux.EpollWait(epfd, events[:], int32(len(events)), waitms) if errno != 0 { if errno != _EINTR { println("runtime: epollwait on fd", epfd, "failed with", errno) @@ -135,7 +138,7 @@ retry: } if *(**uintptr)(unsafe.Pointer(&ev.Data)) == &netpollEventFd { - if ev.Events != syscall.EPOLLIN { + if ev.Events != linux.EPOLLIN { println("runtime: netpoll: eventfd ready for", ev.Events) throw("runtime: netpoll: eventfd ready for something unexpected") } @@ -153,10 +156,10 @@ retry: } var mode int32 - if ev.Events&(syscall.EPOLLIN|syscall.EPOLLRDHUP|syscall.EPOLLHUP|syscall.EPOLLERR) != 0 { + if ev.Events&(linux.EPOLLIN|linux.EPOLLRDHUP|linux.EPOLLHUP|linux.EPOLLERR) != 0 { mode += 'r' } - if ev.Events&(syscall.EPOLLOUT|syscall.EPOLLHUP|syscall.EPOLLERR) != 0 { + if ev.Events&(linux.EPOLLOUT|linux.EPOLLHUP|linux.EPOLLERR) != 0 { mode += 'w' } if mode != 0 { @@ -164,7 +167,7 @@ retry: pd := (*pollDesc)(tp.pointer()) tag := tp.tag() if pd.fdseq.Load() == tag { - pd.setEventErr(ev.Events == syscall.EPOLLERR, tag) + pd.setEventErr(ev.Events == linux.EPOLLERR, tag) delta += netpollready(&toRun, pd, mode) } } diff --git a/src/runtime/netpoll_kqueue.go b/src/runtime/netpoll_kqueue.go index 6cd80d5c30d16e..db4dddc2fe04c7 100644 --- a/src/runtime/netpoll_kqueue.go +++ b/src/runtime/netpoll_kqueue.go @@ -80,7 +80,10 @@ func netpollBreak() { } // netpoll checks for ready network connections. -// Returns list of goroutines that become runnable. +// Returns a list of goroutines that become runnable, +// and a delta to add to netpollWaiters. +// This must never return an empty list with a non-zero delta. +// // delay < 0: blocks indefinitely // delay == 0: does not block, just polls // delay > 0: block for up to that many nanoseconds @@ -126,10 +129,11 @@ retry: ev := &events[i] if isWakeup(ev) { - if delay != 0 { + isBlocking := delay != 0 + processWakeupEvent(kq, isBlocking) + if isBlocking { // netpollBreak could be picked up by a nonblocking poll. - // Only call drainWakeupEvent and reset the netpollWakeSig if blocking. - drainWakeupEvent(kq) + // Only reset the netpollWakeSig if blocking. netpollWakeSig.Store(0) } continue diff --git a/src/runtime/netpoll_kqueue_event.go b/src/runtime/netpoll_kqueue_event.go index d5f783e607e846..852a00a5d86fa4 100644 --- a/src/runtime/netpoll_kqueue_event.go +++ b/src/runtime/netpoll_kqueue_event.go @@ -16,7 +16,7 @@ func addWakeupEvent(kq int32) { ev := keventt{ ident: kqIdent, filter: _EVFILT_USER, - flags: _EV_ADD, + flags: _EV_ADD | _EV_CLEAR, } for { n := kevent(kq, &ev, 1, nil, 0, nil) @@ -38,7 +38,6 @@ func wakeNetpoll(kq int32) { ev := keventt{ ident: kqIdent, filter: _EVFILT_USER, - flags: _EV_ENABLE, fflags: _NOTE_TRIGGER, } for { @@ -66,13 +65,11 @@ func isWakeup(ev *keventt) bool { return false } -func drainWakeupEvent(kq int32) { - ev := keventt{ - ident: kqIdent, - filter: _EVFILT_USER, - flags: _EV_DISABLE, +func processWakeupEvent(kq int32, isBlocking bool) { + if !isBlocking { + // Got a wrong thread, relay + wakeNetpoll(kq) } - kevent(kq, &ev, 1, nil, 0, nil) } func netpollIsPollDescriptor(fd uintptr) bool { diff --git a/src/runtime/netpoll_kqueue_pipe.go b/src/runtime/netpoll_kqueue_pipe.go index 98f73e84d29fc9..cf1e2afa8371d4 100644 --- a/src/runtime/netpoll_kqueue_pipe.go +++ b/src/runtime/netpoll_kqueue_pipe.go @@ -63,7 +63,11 @@ func isWakeup(ev *keventt) bool { return false } -func drainWakeupEvent(_ int32) { +func processWakeupEvent(_ int32, isBlocking bool) { + // Only drain if blocking. + if !isBlocking { + return + } var buf [16]byte read(int32(netpollBreakRd), noescape(unsafe.Pointer(&buf[0])), int32(len(buf))) } diff --git a/src/runtime/netpoll_solaris.go b/src/runtime/netpoll_solaris.go index fddc29000b7864..90459c08df544f 100644 --- a/src/runtime/netpoll_solaris.go +++ b/src/runtime/netpoll_solaris.go @@ -215,7 +215,10 @@ func netpollBreak() { } // netpoll checks for ready network connections. -// Returns list of goroutines that become runnable. +// Returns a list of goroutines that become runnable, +// and a delta to add to netpollWaiters. +// This must never return an empty list with a non-zero delta. +// // delay < 0: blocks indefinitely // delay == 0: does not block, just polls // delay > 0: block for up to that many nanoseconds diff --git a/src/runtime/netpoll_stub.go b/src/runtime/netpoll_stub.go index c1bda3fa8b0d66..dc5d708b4c5bce 100644 --- a/src/runtime/netpoll_stub.go +++ b/src/runtime/netpoll_stub.go @@ -32,7 +32,9 @@ func netpollBreak() { } // Polls for ready network connections. -// Returns list of goroutines that become runnable. +// Returns a list of goroutines that become runnable, +// and a delta to add to netpollWaiters. +// This must never return an empty list with a non-zero delta. func netpoll(delay int64) (gList, int32) { // Implementation for platforms that do not support // integrated network poller. diff --git a/src/runtime/netpoll_wasip1.go b/src/runtime/netpoll_wasip1.go index e6b299a20fa8d4..c7e66a64dddb18 100644 --- a/src/runtime/netpoll_wasip1.go +++ b/src/runtime/netpoll_wasip1.go @@ -209,7 +209,7 @@ func netpoll(delay int64) (gList, int32) { retry: var nevents size - errno := poll_oneoff(unsafe.Pointer(&pollsubs[0]), unsafe.Pointer(&evts[0]), uint32(len(pollsubs)), unsafe.Pointer(&nevents)) + errno := poll_oneoff(&pollsubs[0], &evts[0], uint32(len(pollsubs)), &nevents) if errno != 0 { if errno != _EINTR { println("errno=", errno, " len(pollsubs)=", len(pollsubs)) diff --git a/src/runtime/netpoll_windows.go b/src/runtime/netpoll_windows.go index c3c10af723794f..db5d043506a78c 100644 --- a/src/runtime/netpoll_windows.go +++ b/src/runtime/netpoll_windows.go @@ -7,13 +7,10 @@ package runtime import ( "internal/goarch" "internal/runtime/atomic" + "internal/runtime/syscall/windows" "unsafe" ) -const _DWORD_MAX = 0xffffffff - -const _INVALID_HANDLE_VALUE = ^uintptr(0) - // Sources are used to identify the event that created an overlapped entry. // The source values are arbitrary. There is no risk of collision with user // defined values because the only way to set the key of an overlapped entry @@ -59,7 +56,7 @@ func unpackNetpollSource(key uintptr) uint8 { // Keep these in sync. type pollOperation struct { // used by windows - _ overlapped + _ windows.Overlapped // used by netpoll pd *pollDesc mode int32 @@ -90,19 +87,19 @@ func pollOperationFromOverlappedEntry(e *overlappedEntry) *pollOperation { // https://learn.microsoft.com/en-us/windows/win32/api/minwinbase/ns-minwinbase-overlapped_entry type overlappedEntry struct { key uintptr - ov *overlapped + ov *windows.Overlapped internal uintptr qty uint32 } var ( - iocphandle uintptr = _INVALID_HANDLE_VALUE // completion port io handle + iocphandle uintptr = windows.INVALID_HANDLE_VALUE // completion port io handle netpollWakeSig atomic.Uint32 // used to avoid duplicate calls of netpollBreak ) func netpollinit() { - iocphandle = stdcall4(_CreateIoCompletionPort, _INVALID_HANDLE_VALUE, 0, 0, _DWORD_MAX) + iocphandle = stdcall(_CreateIoCompletionPort, windows.INVALID_HANDLE_VALUE, 0, 0, windows.DWORD_MAX) if iocphandle == 0 { println("runtime: CreateIoCompletionPort failed (errno=", getlasterror(), ")") throw("runtime: netpollinit failed") @@ -115,7 +112,7 @@ func netpollIsPollDescriptor(fd uintptr) bool { func netpollopen(fd uintptr, pd *pollDesc) int32 { key := packNetpollKey(netpollSourceReady, pd) - if stdcall4(_CreateIoCompletionPort, fd, iocphandle, key, 0) == 0 { + if stdcall(_CreateIoCompletionPort, fd, iocphandle, key, 0) == 0 { return int32(getlasterror()) } return 0 @@ -137,19 +134,22 @@ func netpollBreak() { } key := packNetpollKey(netpollSourceBreak, nil) - if stdcall4(_PostQueuedCompletionStatus, iocphandle, 0, key, 0) == 0 { + if stdcall(_PostQueuedCompletionStatus, iocphandle, 0, key, 0) == 0 { println("runtime: netpoll: PostQueuedCompletionStatus failed (errno=", getlasterror(), ")") throw("runtime: netpoll: PostQueuedCompletionStatus failed") } } // netpoll checks for ready network connections. -// Returns list of goroutines that become runnable. +// Returns a list of goroutines that become runnable, +// and a delta to add to netpollWaiters. +// This must never return an empty list with a non-zero delta. +// // delay < 0: blocks indefinitely // delay == 0: does not block, just polls // delay > 0: block for up to that many nanoseconds func netpoll(delay int64) (gList, int32) { - if iocphandle == _INVALID_HANDLE_VALUE { + if iocphandle == windows.INVALID_HANDLE_VALUE { return gList{}, 0 } @@ -179,7 +179,7 @@ func netpoll(delay int64) (gList, int32) { } } if delay < 0 { - wait = _INFINITE + wait = windows.INFINITE } else if delay == 0 { wait = 0 } else if delay < 1e6 { @@ -194,10 +194,10 @@ func netpoll(delay int64) (gList, int32) { if delay != 0 { mp.blocked = true } - if stdcall6(_GetQueuedCompletionStatusEx, iocphandle, uintptr(unsafe.Pointer(&entries[0])), uintptr(n), uintptr(unsafe.Pointer(&n)), uintptr(wait), 0) == 0 { + if stdcall(_GetQueuedCompletionStatusEx, iocphandle, uintptr(unsafe.Pointer(&entries[0])), uintptr(n), uintptr(unsafe.Pointer(&n)), uintptr(wait), 0) == 0 { mp.blocked = false errno := getlasterror() - if errno == _WAIT_TIMEOUT { + if errno == windows.WAIT_TIMEOUT { return gList{}, 0 } println("runtime: GetQueuedCompletionStatusEx failed (errno=", errno, ")") @@ -240,11 +240,6 @@ func netpoll(delay int64) (gList, int32) { // netpollQueueTimer queues a timer to wake up the poller after the given delay. // It returns true if the timer expired during this call. func netpollQueueTimer(delay int64) (signaled bool) { - const ( - STATUS_SUCCESS = 0x00000000 - STATUS_PENDING = 0x00000103 - STATUS_CANCELLED = 0xC0000120 - ) mp := getg().m // A wait completion packet can only be associated with one timer at a time, // so we need to cancel the previous one if it exists. This wouldn't be necessary @@ -253,24 +248,24 @@ func netpollQueueTimer(delay int64) (signaled bool) { // such as a netpollBreak, so we can get to this point with a timer that hasn't // expired yet. In this case, the completion packet can still be picked up by // another thread, so defer the cancellation until it is really necessary. - errno := stdcall2(_NtCancelWaitCompletionPacket, mp.waitIocpHandle, 1) + errno := stdcall(_NtCancelWaitCompletionPacket, mp.waitIocpHandle, 1) switch errno { - case STATUS_CANCELLED: + case windows.STATUS_CANCELLED: // STATUS_CANCELLED is returned when the associated timer has already expired, // in which automatically cancels the wait completion packet. fallthrough - case STATUS_SUCCESS: + case windows.STATUS_SUCCESS: dt := -delay / 100 // relative sleep (negative), 100ns units - if stdcall6(_SetWaitableTimer, mp.waitIocpTimer, uintptr(unsafe.Pointer(&dt)), 0, 0, 0, 0) == 0 { + if stdcall(_SetWaitableTimer, mp.waitIocpTimer, uintptr(unsafe.Pointer(&dt)), 0, 0, 0, 0) == 0 { println("runtime: SetWaitableTimer failed; errno=", getlasterror()) throw("runtime: netpoll failed") } key := packNetpollKey(netpollSourceTimer, nil) - if errno := stdcall8(_NtAssociateWaitCompletionPacket, mp.waitIocpHandle, iocphandle, mp.waitIocpTimer, key, 0, 0, 0, uintptr(unsafe.Pointer(&signaled))); errno != 0 { + if errno := stdcall(_NtAssociateWaitCompletionPacket, mp.waitIocpHandle, iocphandle, mp.waitIocpTimer, key, 0, 0, 0, uintptr(unsafe.Pointer(&signaled))); errno != 0 { println("runtime: NtAssociateWaitCompletionPacket failed; errno=", errno) throw("runtime: netpoll failed") } - case STATUS_PENDING: + case windows.STATUS_PENDING: // STATUS_PENDING is returned if the wait operation can't be canceled yet. // This can happen if this thread was woken up by another event, such as a netpollBreak, // and the timer expired just while calling NtCancelWaitCompletionPacket, in which case diff --git a/src/runtime/norace_linux_test.go b/src/runtime/norace_linux_test.go deleted file mode 100644 index 3521b24655bc95..00000000000000 --- a/src/runtime/norace_linux_test.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// The file contains tests that cannot run under race detector for some reason. -// -//go:build !race - -package runtime_test - -import ( - "internal/abi" - "runtime" - "testing" - "time" - "unsafe" -) - -var newOSProcDone bool - -//go:nosplit -func newOSProcCreated() { - newOSProcDone = true -} - -// Can't be run with -race because it inserts calls into newOSProcCreated() -// that require a valid G/M. -func TestNewOSProc0(t *testing.T) { - runtime.NewOSProc0(0x800000, unsafe.Pointer(abi.FuncPCABIInternal(newOSProcCreated))) - check := time.NewTicker(100 * time.Millisecond) - defer check.Stop() - end := time.After(5 * time.Second) - for { - select { - case <-check.C: - if newOSProcDone { - return - } - case <-end: - t.Fatalf("couldn't create new OS process") - } - } -} diff --git a/src/runtime/nosan_linux_test.go b/src/runtime/nosan_linux_test.go new file mode 100644 index 00000000000000..5c99591a07872b --- /dev/null +++ b/src/runtime/nosan_linux_test.go @@ -0,0 +1,43 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The file contains tests that cannot run under race detector (or asan or msan) for some reason. +// +//go:build !race && !asan && !msan + +package runtime_test + +import ( + "internal/abi" + "runtime" + "testing" + "time" + "unsafe" +) + +var newOSProcDone bool + +//go:nosplit +func newOSProcCreated() { + newOSProcDone = true +} + +// Can't be run with -race, -asan, or -msan because it inserts calls into newOSProcCreated() +// that require a valid G/M. +func TestNewOSProc0(t *testing.T) { + runtime.NewOSProc0(0x800000, unsafe.Pointer(abi.FuncPCABIInternal(newOSProcCreated))) + check := time.NewTicker(100 * time.Millisecond) + defer check.Stop() + end := time.After(5 * time.Second) + for { + select { + case <-check.C: + if newOSProcDone { + return + } + case <-end: + t.Fatalf("couldn't create new OS process") + } + } +} diff --git a/src/runtime/note_js.go b/src/runtime/note_js.go new file mode 100644 index 00000000000000..be43fa42b0bc59 --- /dev/null +++ b/src/runtime/note_js.go @@ -0,0 +1,40 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +// sleep and wakeup on one-time events. +// before any calls to notesleep or notewakeup, +// must call noteclear to initialize the Note. +// then, exactly one thread can call notesleep +// and exactly one thread can call notewakeup (once). +// once notewakeup has been called, the notesleep +// will return. future notesleep will return immediately. +// subsequent noteclear must be called only after +// previous notesleep has returned, e.g. it's disallowed +// to call noteclear straight after notewakeup. +// +// notetsleep is like notesleep but wakes up after +// a given number of nanoseconds even if the event +// has not yet happened. if a goroutine uses notetsleep to +// wake up early, it must wait to call noteclear until it +// can be sure that no other goroutine is calling +// notewakeup. +// +// notesleep/notetsleep are generally called on g0, +// notetsleepg is similar to notetsleep but is called on user g. +type note struct { + status int32 + + // The G waiting on this note. + gp *g + + // Deadline, if any. 0 indicates no timeout. + deadline int64 + + // allprev and allnext are used to form the allDeadlineNotes linked + // list. These are unused if there is no deadline. + allprev *note + allnext *note +} diff --git a/src/runtime/note_other.go b/src/runtime/note_other.go new file mode 100644 index 00000000000000..7f62c1c6c5f301 --- /dev/null +++ b/src/runtime/note_other.go @@ -0,0 +1,33 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !js + +package runtime + +// sleep and wakeup on one-time events. +// before any calls to notesleep or notewakeup, +// must call noteclear to initialize the Note. +// then, exactly one thread can call notesleep +// and exactly one thread can call notewakeup (once). +// once notewakeup has been called, the notesleep +// will return. future notesleep will return immediately. +// subsequent noteclear must be called only after +// previous notesleep has returned, e.g. it's disallowed +// to call noteclear straight after notewakeup. +// +// notetsleep is like notesleep but wakes up after +// a given number of nanoseconds even if the event +// has not yet happened. if a goroutine uses notetsleep to +// wake up early, it must wait to call noteclear until it +// can be sure that no other goroutine is calling +// notewakeup. +// +// notesleep/notetsleep are generally called on g0, +// notetsleepg is similar to notetsleep but is called on user g. +type note struct { + // Futex-based impl treats it as uint32 key, + // while sema-based impl as M* waitm. + key uintptr +} diff --git a/src/runtime/os2_aix.go b/src/runtime/os2_aix.go index 0d200792424daf..51758bd3049545 100644 --- a/src/runtime/os2_aix.go +++ b/src/runtime/os2_aix.go @@ -11,6 +11,7 @@ package runtime import ( + "internal/runtime/sys" "unsafe" ) @@ -182,10 +183,10 @@ func syscall0(fn *libFunc) (r, err uintptr) { resetLibcall := true if mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { resetLibcall = false // See comment in sys_darwin.go:libcCall } @@ -213,10 +214,10 @@ func syscall1(fn *libFunc, a0 uintptr) (r, err uintptr) { resetLibcall := true if mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { resetLibcall = false // See comment in sys_darwin.go:libcCall } @@ -245,10 +246,10 @@ func syscall2(fn *libFunc, a0, a1 uintptr) (r, err uintptr) { resetLibcall := true if mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { resetLibcall = false // See comment in sys_darwin.go:libcCall } @@ -277,10 +278,10 @@ func syscall3(fn *libFunc, a0, a1, a2 uintptr) (r, err uintptr) { resetLibcall := true if mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { resetLibcall = false // See comment in sys_darwin.go:libcCall } @@ -309,10 +310,10 @@ func syscall4(fn *libFunc, a0, a1, a2, a3 uintptr) (r, err uintptr) { resetLibcall := true if mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { resetLibcall = false // See comment in sys_darwin.go:libcCall } @@ -341,10 +342,10 @@ func syscall5(fn *libFunc, a0, a1, a2, a3, a4 uintptr) (r, err uintptr) { resetLibcall := true if mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { resetLibcall = false // See comment in sys_darwin.go:libcCall } @@ -373,10 +374,10 @@ func syscall6(fn *libFunc, a0, a1, a2, a3, a4, a5 uintptr) (r, err uintptr) { resetLibcall := true if mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { resetLibcall = false // See comment in sys_darwin.go:libcCall } diff --git a/src/runtime/os3_solaris.go b/src/runtime/os3_solaris.go index cf163a6bf401f6..3197c66537b8cf 100644 --- a/src/runtime/os3_solaris.go +++ b/src/runtime/os3_solaris.go @@ -137,7 +137,7 @@ func osinit() { // before calling minit on m0. asmcgocall(unsafe.Pointer(abi.FuncPCABI0(miniterrno)), unsafe.Pointer(&libc____errno)) - ncpu = getncpu() + numCPUStartup = getCPUCount() if physPageSize == 0 { physPageSize = getPageSize() } @@ -234,8 +234,12 @@ func unminit() { getg().m.procid = 0 } -// Called from exitm, but not from drop, to undo the effect of thread-owned +// Called from mexit, but not from dropm, to undo the effect of thread-owned // resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +// +// This always runs without a P, so //go:nowritebarrierrec is required. +// +//go:nowritebarrierrec func mdestroy(mp *m) { } diff --git a/src/runtime/os_aix.go b/src/runtime/os_aix.go index 93464cb997f3be..4bb8576f42de0e 100644 --- a/src/runtime/os_aix.go +++ b/src/runtime/os_aix.go @@ -27,6 +27,7 @@ type funcDescriptor struct { type mOS struct { waitsema uintptr // semaphore for parking on locks perrno uintptr // pointer to tls errno + libcall libcall } //go:nosplit @@ -97,10 +98,14 @@ func osinit() { // before calling minit on m0. miniterrno() - ncpu = int32(sysconf(__SC_NPROCESSORS_ONLN)) + numCPUStartup = getCPUCount() physPageSize = sysconf(__SC_PAGE_SIZE) } +func getCPUCount() int32 { + return int32(sysconf(__SC_NPROCESSORS_ONLN)) +} + // newosproc0 is a version of newosproc that can be called before the runtime // is initialized. // @@ -186,8 +191,12 @@ func unminit() { getg().m.procid = 0 } -// Called from exitm, but not from drop, to undo the effect of thread-owned +// Called from mexit, but not from dropm, to undo the effect of thread-owned // resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +// +// This always runs without a P, so //go:nowritebarrierrec is required. +// +//go:nowritebarrierrec func mdestroy(mp *m) { } diff --git a/src/runtime/os_darwin.go b/src/runtime/os_darwin.go index 430d1865dffb41..c908809c81b2be 100644 --- a/src/runtime/os_darwin.go +++ b/src/runtime/os_darwin.go @@ -6,6 +6,7 @@ package runtime import ( "internal/abi" + "internal/stringslite" "unsafe" ) @@ -14,6 +15,11 @@ type mOS struct { mutex pthreadmutex cond pthreadcond count int + + // address of errno variable for this thread. + // This is an optimization to avoid calling libc_error + // on every syscall_rawsyscalln. + errnoAddr *int32 } func unimplemented(name string) { @@ -143,7 +149,7 @@ func osinit() { // pthread_create delayed until end of goenvs so that we // can look at the environment first. - ncpu = getncpu() + numCPUStartup = getCPUCount() physPageSize = getPageSize() osinit_hack() @@ -167,7 +173,7 @@ const ( _HW_PAGESIZE = 7 ) -func getncpu() int32 { +func getCPUCount() int32 { // Use sysctl to fetch hw.ncpu. mib := [2]uint32{_CTL_HW, _HW_NCPU} out := uint32(0) @@ -191,14 +197,10 @@ func getPageSize() uintptr { return 0 } -var urandom_dev = []byte("/dev/urandom\x00") - //go:nosplit func readRandom(r []byte) int { - fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) - n := read(fd, unsafe.Pointer(&r[0]), int32(len(r))) - closefd(fd) - return int(n) + arc4random_buf(unsafe.Pointer(&r[0]), int32(len(r))) + return len(r) } func goenvs() { @@ -333,6 +335,7 @@ func minit() { } minitSignalMask() getg().m.procid = uint64(pthread_self()) + libc_error_addr(&getg().m.errnoAddr) } // Called from dropm to undo the effect of an minit. @@ -347,8 +350,12 @@ func unminit() { getg().m.procid = 0 } -// Called from exitm, but not from drop, to undo the effect of thread-owned +// Called from mexit, but not from dropm, to undo the effect of thread-owned // resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +// +// This always runs without a P, so //go:nowritebarrierrec is required. +// +//go:nowritebarrierrec func mdestroy(mp *m) { } @@ -465,10 +472,7 @@ func sysargs(argc int32, argv **byte) { executablePath = gostringnocopy(argv_index(argv, n+1)) // strip "executable_path=" prefix if available, it's added after OS X 10.11. - const prefix = "executable_path=" - if len(executablePath) > len(prefix) && executablePath[:len(prefix)] == prefix { - executablePath = executablePath[len(prefix):] - } + executablePath = stringslite.TrimPrefix(executablePath, "executable_path=") } func signalM(mp *m, sig int) { diff --git a/src/runtime/os_dragonfly.go b/src/runtime/os_dragonfly.go index 2aeea1775523c1..fbbee64fd3847f 100644 --- a/src/runtime/os_dragonfly.go +++ b/src/runtime/os_dragonfly.go @@ -19,7 +19,9 @@ const ( _SIG_SETMASK = 3 ) -type mOS struct{} +type mOS struct { + waitsema uint32 // semaphore for parking on locks +} //go:noescape func lwp_create(param *lwpparams) int32 @@ -76,7 +78,7 @@ const ( var sigset_all = sigset{[4]uint32{^uint32(0), ^uint32(0), ^uint32(0), ^uint32(0)}} -func getncpu() int32 { +func getCPUCount() int32 { mib := [2]uint32{_CTL_HW, _HW_NCPU} out := uint32(0) nout := unsafe.Sizeof(out) @@ -172,7 +174,7 @@ func newosproc(mp *m) { } func osinit() { - ncpu = getncpu() + numCPUStartup = getCPUCount() if physPageSize == 0 { physPageSize = getPageSize() } @@ -214,8 +216,12 @@ func unminit() { getg().m.procid = 0 } -// Called from exitm, but not from drop, to undo the effect of thread-owned +// Called from mexit, but not from dropm, to undo the effect of thread-owned // resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +// +// This always runs without a P, so //go:nowritebarrierrec is required. +// +//go:nowritebarrierrec func mdestroy(mp *m) { } diff --git a/src/runtime/os_freebsd.go b/src/runtime/os_freebsd.go index d0d6f14fa0cb5a..68d895b95dc2e2 100644 --- a/src/runtime/os_freebsd.go +++ b/src/runtime/os_freebsd.go @@ -10,7 +10,9 @@ import ( "unsafe" ) -type mOS struct{} +type mOS struct { + waitsema uint32 // semaphore for parking on locks +} //go:noescape func thr_new(param *thrparam, size int32) int32 @@ -89,7 +91,7 @@ const ( func cpuset_getaffinity(level int, which int, id int64, size int, mask *byte) int32 //go:systemstack -func getncpu() int32 { +func getCPUCount() int32 { // Use a large buffer for the CPU mask. We're on the system // stack, so this is fine, and we can't allocate memory for a // dynamically-sized buffer at this point. @@ -231,7 +233,7 @@ func newosproc(mp *m) { // //go:nosplit func newosproc0(stacksize uintptr, fn unsafe.Pointer) { - stack := sysAlloc(stacksize, &memstats.stacks_sys) + stack := sysAlloc(stacksize, &memstats.stacks_sys, "OS thread stack") if stack == nil { writeErrStr(failallocatestack) exit(1) @@ -274,7 +276,7 @@ func libpreinit() { } func osinit() { - ncpu = getncpu() + numCPUStartup = getCPUCount() if physPageSize == 0 { physPageSize = getPageSize() } @@ -455,6 +457,12 @@ func sysSigaction(sig uint32, new, old *sigactiont) { } } +// fixSigactionForCgo is needed for Linux. +// +//go:nosplit +func fixSigactionForCgo(new *sigactiont) { +} + // asmSigaction is implemented in assembly. // //go:noescape diff --git a/src/runtime/os_freebsd_arm.go b/src/runtime/os_freebsd_arm.go index 5f6bf46798c9ee..5a6c60210af988 100644 --- a/src/runtime/os_freebsd_arm.go +++ b/src/runtime/os_freebsd_arm.go @@ -28,8 +28,9 @@ func checkgoarm() { exit(1) } - // osinit not called yet, so ncpu not set: must use getncpu directly. - if getncpu() > 1 && goarm < 7 { + // osinit not called yet, so numCPUStartup not set: must use + // getCPUCount directly. + if getCPUCount() > 1 && goarm < 7 { print("runtime: this system has multiple CPUs and must use\n") print("atomic synchronization instructions. Recompile using GOARM=7.\n") exit(1) diff --git a/src/runtime/os_freebsd_riscv64.go b/src/runtime/os_freebsd_riscv64.go deleted file mode 100644 index 0f2ed5096c8176..00000000000000 --- a/src/runtime/os_freebsd_riscv64.go +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -func osArchInit() {} diff --git a/src/runtime/os_illumos.go b/src/runtime/os_illumos.go index c3c3e4e6d56df5..c4bc9fe9c06997 100644 --- a/src/runtime/os_illumos.go +++ b/src/runtime/os_illumos.go @@ -87,7 +87,7 @@ func getcpucap() uint64 { return capval } -func getncpu() int32 { +func getCPUCount() int32 { n := int32(sysconf(__SC_NPROCESSORS_ONLN)) if n < 1 { return 1 diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go index 6ce656c70e146e..1b7d5731cc32ab 100644 --- a/src/runtime/os_linux.go +++ b/src/runtime/os_linux.go @@ -8,7 +8,8 @@ import ( "internal/abi" "internal/goarch" "internal/runtime/atomic" - "internal/runtime/syscall" + "internal/runtime/strconv" + "internal/runtime/syscall/linux" "unsafe" ) @@ -31,10 +32,13 @@ type mOS struct { // needPerThreadSyscall indicates that a per-thread syscall is required // for doAllThreadsSyscall. needPerThreadSyscall atomic.Uint8 -} -//go:noescape -func futex(addr unsafe.Pointer, op int32, val uint32, ts, addr2 unsafe.Pointer, val3 uint32) int32 + // This is a pointer to a chunk of memory allocated with a special + // mmap invocation in vgetrandomGetState(). + vgetrandomState uintptr + + waitsema uint32 // semaphore for parking on locks +} // Linux futex. // @@ -72,7 +76,7 @@ func futexsleep(addr *uint32, val uint32, ns int64) { var ts timespec ts.setNsec(ns) - futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, unsafe.Pointer(&ts), nil, 0) + futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, &ts, nil, 0) } // If any procs are sleeping on addr, wake up at most cnt. @@ -94,7 +98,7 @@ func futexwakeup(addr *uint32, cnt uint32) { *(*int32)(unsafe.Pointer(uintptr(0x1006))) = 0x1006 } -func getproccount() int32 { +func getCPUCount() int32 { // This buffer is huge (8 kB) but we are on the system stack // and there should be plenty of space (64 kB). // Also this is a leaf, so we're not holding up the memory for long. @@ -200,7 +204,7 @@ func newosproc(mp *m) { // //go:nosplit func newosproc0(stacksize uintptr, fn unsafe.Pointer) { - stack := sysAlloc(stacksize, &memstats.stacks_sys) + stack := sysAlloc(stacksize, &memstats.stacks_sys, "OS thread stack") if stack == nil { writeErrStr(failallocatestack) exit(1) @@ -292,13 +296,19 @@ func sysargs(argc int32, argv **byte) { var secureMode bool func sysauxv(auxv []uintptr) (pairs int) { + // Process the auxiliary vector entries provided by the kernel when the + // program is executed. See getauxval(3). var i int for ; auxv[i] != _AT_NULL; i += 2 { tag, val := auxv[i], auxv[i+1] switch tag { case _AT_RANDOM: - // The kernel provides a pointer to 16-bytes - // worth of random data. + // The kernel provides a pointer to 16 bytes of cryptographically + // random data. Note that in cgo programs this value may have + // already been used by libc at this point, and in particular glibc + // and musl use the value as-is for stack and pointer protector + // cookies from libc_start_main and/or dl_start. Also, cgo programs + // may use the value after we do. startupRand = (*[16]byte)(unsafe.Pointer(val))[:] case _AT_PAGESZ: @@ -329,7 +339,7 @@ func getHugePageSize() uintptr { return 0 } n-- // remove trailing newline - v, ok := atoi(slicebytetostringtmp((*byte)(ptr), int(n))) + v, ok := strconv.Atoi(slicebytetostringtmp((*byte)(ptr), int(n))) if !ok || v < 0 { v = 0 } @@ -341,14 +351,16 @@ func getHugePageSize() uintptr { } func osinit() { - ncpu = getproccount() + numCPUStartup = getCPUCount() physHugePageSize = getHugePageSize() - osArchInit() + vgetrandomInit() } var urandom_dev = []byte("/dev/urandom\x00") func readRandom(r []byte) int { + // Note that all supported Linux kernels should provide AT_RANDOM which + // populates startupRand, so this fallback should be unreachable. fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) n := read(fd, unsafe.Pointer(&r[0]), int32(len(r))) closefd(fd) @@ -397,8 +409,12 @@ func unminit() { getg().m.procid = 0 } -// Called from exitm, but not from drop, to undo the effect of thread-owned +// Called from mexit, but not from dropm, to undo the effect of thread-owned // resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +// +// This always runs without a P, so //go:nowritebarrierrec is required. +// +//go:nowritebarrierrec func mdestroy(mp *m) { } @@ -419,9 +435,6 @@ func setitimer(mode int32, new, old *itimerval) //go:noescape func timer_create(clockid int32, sevp *sigevent, timerid *int32) int32 -//go:noescape -func timer_settime(timerid int32, flags int32, new, old *itimerspec) int32 - //go:noescape func timer_delete(timerid int32) int32 @@ -450,7 +463,7 @@ func pipe2(flags int32) (r, w int32, errno int32) //go:nosplit func fcntl(fd, cmd, arg int32) (ret int32, errno int32) { - r, _, err := syscall.Syscall6(syscall.SYS_FCNTL, uintptr(fd), uintptr(cmd), uintptr(arg), 0, 0, 0) + r, _, err := linux.Syscall6(linux.SYS_FCNTL, uintptr(fd), uintptr(cmd), uintptr(arg), 0, 0, 0) return int32(r), int32(err) } @@ -467,7 +480,8 @@ func setsig(i uint32, fn uintptr) { sigfillset(&sa.sa_mask) // Although Linux manpage says "sa_restorer element is obsolete and // should not be used". x86_64 kernel requires it. Only use it on - // x86. + // x86. Note that on 386 this is cleared when using the C sigaction + // function via cgo; see fixSigactionForCgo. if GOARCH == "386" || GOARCH == "amd64" { sa.sa_restorer = abi.FuncPCABI0(sigreturn__sigaction) } @@ -543,6 +557,21 @@ func sysSigaction(sig uint32, new, old *sigactiont) { //go:noescape func rt_sigaction(sig uintptr, new, old *sigactiont, size uintptr) int32 +// fixSigactionForCgo is called when we are using cgo to call the +// C sigaction function. On 386 the C function does not expect the +// SA_RESTORER flag to be set, and in some cases will fail if it is set: +// it will pass the SA_RESTORER flag to the kernel without passing +// the sa_restorer field. Since the C function will handle SA_RESTORER +// for us, we need not pass it. See issue #75253. +// +//go:nosplit +func fixSigactionForCgo(new *sigactiont) { + if GOARCH == "386" && new != nil { + new.sa_flags &^= _SA_RESTORER + new.sa_restorer = 0 + } +} + func getpid() int func tgkill(tgid, tid, sig int) @@ -636,7 +665,7 @@ func setThreadCPUProfiler(hz int32) { // spend shows up as a 10% chance of one sample (for an expected value of // 0.1 samples), and so that "two and six tenths" periods of CPU spend show // up as a 60% chance of 3 samples and a 40% chance of 2 samples (for an - // expected value of 2.6). Set the initial delay to a value in the unifom + // expected value of 2.6). Set the initial delay to a value in the uniform // random distribution between 0 and the desired period. And because "0" // means "disable timer", add 1 so the half-open interval [0,period) turns // into (0,period]. @@ -753,7 +782,7 @@ func syscall_runtime_doAllThreadsSyscall(trap, a1, a2, a3, a4, a5, a6 uintptr) ( // ensuring all threads execute system calls from multiple calls in the // same order. - r1, r2, errno := syscall.Syscall6(trap, a1, a2, a3, a4, a5, a6) + r1, r2, errno := linux.Syscall6(trap, a1, a2, a3, a4, a5, a6) if GOARCH == "ppc64" || GOARCH == "ppc64le" { // TODO(https://go.dev/issue/51192 ): ppc64 doesn't use r2. r2 = 0 @@ -864,7 +893,7 @@ func runPerThreadSyscall() { } args := perThreadSyscall - r1, r2, errno := syscall.Syscall6(args.trap, args.a1, args.a2, args.a3, args.a4, args.a5, args.a6) + r1, r2, errno := linux.Syscall6(args.trap, args.a1, args.a2, args.a3, args.a4, args.a5, args.a6) if GOARCH == "ppc64" || GOARCH == "ppc64le" { // TODO(https://go.dev/issue/51192 ): ppc64 doesn't use r2. r2 = 0 @@ -879,8 +908,9 @@ func runPerThreadSyscall() { } const ( - _SI_USER = 0 - _SI_TKILL = -6 + _SI_USER = 0 + _SI_TKILL = -6 + _SYS_SECCOMP = 1 ) // sigFromUser reports whether the signal was sent because of a call @@ -892,8 +922,16 @@ func (c *sigctxt) sigFromUser() bool { return code == _SI_USER || code == _SI_TKILL } +// sigFromSeccomp reports whether the signal was sent from seccomp. +// +//go:nosplit +func (c *sigctxt) sigFromSeccomp() bool { + code := int32(c.sigcode()) + return code == _SYS_SECCOMP +} + //go:nosplit func mprotect(addr unsafe.Pointer, n uintptr, prot int32) (ret int32, errno int32) { - r, _, err := syscall.Syscall6(syscall.SYS_MPROTECT, uintptr(addr), n, uintptr(prot), 0, 0, 0) + r, _, err := linux.Syscall6(linux.SYS_MPROTECT, uintptr(addr), n, uintptr(prot), 0, 0, 0) return int32(r), int32(err) } diff --git a/src/runtime/os_linux_arm.go b/src/runtime/os_linux_arm.go index 5e1274ebab0fdf..46b2dc467cbb7a 100644 --- a/src/runtime/os_linux_arm.go +++ b/src/runtime/os_linux_arm.go @@ -48,8 +48,6 @@ func archauxv(tag, val uintptr) { } } -func osArchInit() {} - //go:nosplit func cputicks() int64 { // nanotime() is a poor approximation of CPU ticks that is enough for the profiler. diff --git a/src/runtime/os_linux_arm64.go b/src/runtime/os_linux_arm64.go index 62cead1d221c17..ccfb92f8ebdacd 100644 --- a/src/runtime/os_linux_arm64.go +++ b/src/runtime/os_linux_arm64.go @@ -15,8 +15,6 @@ func archauxv(tag, val uintptr) { } } -func osArchInit() {} - //go:nosplit func cputicks() int64 { // nanotime() is a poor approximation of CPU ticks that is enough for the profiler. diff --git a/src/runtime/os_linux_futex32.go b/src/runtime/os_linux_futex32.go new file mode 100644 index 00000000000000..c5cffa24d155ac --- /dev/null +++ b/src/runtime/os_linux_futex32.go @@ -0,0 +1,40 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (386 || arm || mips || mipsle || (gccgo && (ppc || s390))) + +package runtime + +import ( + "internal/runtime/atomic" + "unsafe" +) + +//go:noescape +func futex_time32(addr unsafe.Pointer, op int32, val uint32, ts *timespec32, addr2 unsafe.Pointer, val3 uint32) int32 + +//go:noescape +func futex_time64(addr unsafe.Pointer, op int32, val uint32, ts *timespec, addr2 unsafe.Pointer, val3 uint32) int32 + +var is32bitOnly atomic.Bool + +//go:nosplit +func futex(addr unsafe.Pointer, op int32, val uint32, ts *timespec, addr2 unsafe.Pointer, val3 uint32) int32 { + if !is32bitOnly.Load() { + ret := futex_time64(addr, op, val, ts, addr2, val3) + // futex_time64 is only supported on Linux 5.0+ + if ret != -_ENOSYS { + return ret + } + is32bitOnly.Store(true) + } + // Downgrade ts. + var ts32 timespec32 + var pts32 *timespec32 + if ts != nil { + ts32.setNsec(ts.tv_sec*1e9 + ts.tv_nsec) + pts32 = &ts32 + } + return futex_time32(addr, op, val, pts32, addr2, val3) +} diff --git a/src/runtime/os_linux_futex64.go b/src/runtime/os_linux_futex64.go new file mode 100644 index 00000000000000..2448e30cf1b000 --- /dev/null +++ b/src/runtime/os_linux_futex64.go @@ -0,0 +1,14 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && !(386 || arm || mips || mipsle || (gccgo && (ppc || s390))) + +package runtime + +import ( + "unsafe" +) + +//go:noescape +func futex(addr unsafe.Pointer, op int32, val uint32, ts *timespec, addr2 unsafe.Pointer, val3 uint32) int32 diff --git a/src/runtime/os_linux_loong64.go b/src/runtime/os_linux_loong64.go index 61213dadf85f88..179ae08e54b37b 100644 --- a/src/runtime/os_linux_loong64.go +++ b/src/runtime/os_linux_loong64.go @@ -6,6 +6,11 @@ package runtime -func archauxv(tag, val uintptr) {} +import "internal/cpu" -func osArchInit() {} +func archauxv(tag, val uintptr) { + switch tag { + case _AT_HWCAP: + cpu.HWCap = uint(val) + } +} diff --git a/src/runtime/os_linux_mips64x.go b/src/runtime/os_linux_mips64x.go index 770cc27ba78915..778db11221c9bc 100644 --- a/src/runtime/os_linux_mips64x.go +++ b/src/runtime/os_linux_mips64x.go @@ -15,8 +15,6 @@ func archauxv(tag, val uintptr) { } } -func osArchInit() {} - //go:nosplit func cputicks() int64 { // nanotime() is a poor approximation of CPU ticks that is enough for the profiler. diff --git a/src/runtime/os_linux_mipsx.go b/src/runtime/os_linux_mipsx.go index 3807e6d05103c8..ef5d0b867edab4 100644 --- a/src/runtime/os_linux_mipsx.go +++ b/src/runtime/os_linux_mipsx.go @@ -9,8 +9,6 @@ package runtime func archauxv(tag, val uintptr) { } -func osArchInit() {} - //go:nosplit func cputicks() int64 { // nanotime() is a poor approximation of CPU ticks that is enough for the profiler. diff --git a/src/runtime/os_linux_ppc64x.go b/src/runtime/os_linux_ppc64x.go index 25d7ccc0356c16..9abc9ec3d229b3 100644 --- a/src/runtime/os_linux_ppc64x.go +++ b/src/runtime/os_linux_ppc64x.go @@ -19,5 +19,3 @@ func archauxv(tag, val uintptr) { cpu.HWCap2 = uint(val) } } - -func osArchInit() {} diff --git a/src/runtime/os_linux_riscv64.go b/src/runtime/os_linux_riscv64.go index 9be88a5ad24468..f4f528c736fe3b 100644 --- a/src/runtime/os_linux_riscv64.go +++ b/src/runtime/os_linux_riscv64.go @@ -4,4 +4,32 @@ package runtime -func osArchInit() {} +import ( + "internal/runtime/syscall/linux" + "unsafe" +) + +type riscvHWProbePairs = struct { + key int64 + value uint64 +} + +// TODO: Consider whether to use the VDSO entry for riscv_hwprobe. +// There is a VDSO entry for riscv_hwprobe that should allow us to avoid the syscall +// entirely as it can handle the case where the caller only requests extensions that are +// supported on all cores, which is what we're doing here. However, as we're only calling +// this syscall once, it may not be worth the added effort to implement the VDSO call. + +//go:linkname internal_cpu_riscvHWProbe internal/cpu.riscvHWProbe +func internal_cpu_riscvHWProbe(pairs []riscvHWProbePairs, flags uint) bool { + // sys_RISCV_HWPROBE is copied from golang.org/x/sys/unix/zsysnum_linux_riscv64.go. + const sys_RISCV_HWPROBE uintptr = 258 + + if len(pairs) == 0 { + return false + } + // Passing in a cpuCount of 0 and a cpu of nil ensures that only extensions supported by all the + // cores are returned, which is the behaviour we want in internal/cpu. + _, _, e1 := linux.Syscall6(sys_RISCV_HWPROBE, uintptr(unsafe.Pointer(&pairs[0])), uintptr(len(pairs)), uintptr(0), uintptr(unsafe.Pointer(nil)), uintptr(flags), 0) + return e1 == 0 +} diff --git a/src/runtime/os_linux_s390x.go b/src/runtime/os_linux_s390x.go index 0a1d95975edd67..c5c095593c1f65 100644 --- a/src/runtime/os_linux_s390x.go +++ b/src/runtime/os_linux_s390x.go @@ -17,8 +17,6 @@ func archauxv(tag, val uintptr) { } } -func osArchInit() {} - func checkS390xCPU() { // Check if the present z-system has the hardware capability to carryout // floating point operations. Check if hwcap reflects CPU capability for the diff --git a/src/runtime/os_linux_settime32.go b/src/runtime/os_linux_settime32.go new file mode 100644 index 00000000000000..e6c5d9f95c0ee0 --- /dev/null +++ b/src/runtime/os_linux_settime32.go @@ -0,0 +1,47 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (386 || arm || mips || mipsle || (gccgo && (ppc || s390))) + +package runtime + +import "internal/runtime/atomic" + +var timer32bitOnly atomic.Bool + +//go:noescape +func timer_settime32(timerid int32, flags int32, new, old *itimerspec32) int32 + +//go:noescape +func timer_settime64(timerid int32, flags int32, new, old *itimerspec) int32 + +//go:nosplit +func timer_settime(timerid int32, flags int32, new, old *itimerspec) int32 { + if !timer32bitOnly.Load() { + ret := timer_settime64(timerid, flags, new, old) + // timer_settime64 is only supported on Linux 5.0+ + if ret != -_ENOSYS { + return ret + } + timer32bitOnly.Store(true) + } + + var newts, oldts itimerspec32 + var new32, old32 *itimerspec32 + + if new != nil { + newts.it_interval.setNsec(new.it_interval.tv_sec*1e9 + new.it_interval.tv_nsec) + newts.it_value.setNsec(new.it_value.tv_sec*1e9 + new.it_value.tv_nsec) + new32 = &newts + } + + if old != nil { + oldts.it_interval.setNsec(old.it_interval.tv_sec*1e9 + old.it_interval.tv_nsec) + oldts.it_value.setNsec(old.it_value.tv_sec*1e9 + old.it_value.tv_nsec) + old32 = &oldts + } + + // Fall back to 32-bit timer + return timer_settime32(timerid, flags, new32, old32) +} diff --git a/src/runtime/os_linux_settime64.go b/src/runtime/os_linux_settime64.go new file mode 100644 index 00000000000000..dfacf5612d8610 --- /dev/null +++ b/src/runtime/os_linux_settime64.go @@ -0,0 +1,10 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && !(386 || arm || mips || mipsle || (gccgo && (ppc || s390))) + +package runtime + +//go:noescape +func timer_settime(timerid int32, flags int32, new, old *itimerspec) int32 diff --git a/src/runtime/os_linux_x86.go b/src/runtime/os_linux_x86.go deleted file mode 100644 index c88f61fa2e99e6..00000000000000 --- a/src/runtime/os_linux_x86.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build linux && (386 || amd64) - -package runtime - -func osArchInit() {} diff --git a/src/runtime/os_netbsd.go b/src/runtime/os_netbsd.go index 735ace25adb3b3..f117253f34abb4 100644 --- a/src/runtime/os_netbsd.go +++ b/src/runtime/os_netbsd.go @@ -101,9 +101,6 @@ var sigset_all = sigset{[4]uint32{^uint32(0), ^uint32(0), ^uint32(0), ^uint32(0) // From NetBSD's const ( - _CTL_KERN = 1 - _KERN_OSREV = 3 - _CTL_HW = 6 _HW_NCPU = 3 _HW_PAGESIZE = 7 @@ -120,7 +117,7 @@ func sysctlInt(mib []uint32) (int32, bool) { return out, true } -func getncpu() int32 { +func getCPUCount() int32 { if n, ok := sysctlInt([]uint32{_CTL_HW, _HW_NCPUONLINE}); ok { return int32(n) } @@ -141,13 +138,6 @@ func getPageSize() uintptr { return 0 } -func getOSRev() int { - if osrev, ok := sysctlInt([]uint32{_CTL_KERN, _KERN_OSREV}); ok { - return int(osrev) - } - return 0 -} - //go:nosplit func semacreate(mp *m) { } @@ -264,11 +254,10 @@ func netbsdMstart0() { } func osinit() { - ncpu = getncpu() + numCPUStartup = getCPUCount() if physPageSize == 0 { physPageSize = getPageSize() } - needSysmonWorkaround = getOSRev() < 902000000 // NetBSD 9.2 } var urandom_dev = []byte("/dev/urandom\x00") @@ -320,8 +309,12 @@ func unminit() { // must continue working after unminit. } -// Called from exitm, but not from drop, to undo the effect of thread-owned +// Called from mexit, but not from dropm, to undo the effect of thread-owned // resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +// +// This always runs without a P, so //go:nowritebarrierrec is required. +// +//go:nowritebarrierrec func mdestroy(mp *m) { } diff --git a/src/runtime/os_netbsd_arm.go b/src/runtime/os_netbsd_arm.go index 7494a387e33b99..884b18bdd400ce 100644 --- a/src/runtime/os_netbsd_arm.go +++ b/src/runtime/os_netbsd_arm.go @@ -21,8 +21,9 @@ func lwp_mcontext_init(mc *mcontextt, stk unsafe.Pointer, mp *m, gp *g, fn uintp func checkgoarm() { // TODO(minux): FP checks like in os_linux_arm.go. - // osinit not called yet, so ncpu not set: must use getncpu directly. - if getncpu() > 1 && goarm < 7 { + // osinit not called yet, so numCPUStartup not set: must use + // getCPUCount directly. + if getCPUCount() > 1 && goarm < 7 { print("runtime: this system has multiple CPUs and must use\n") print("atomic synchronization instructions. Recompile using GOARM=7.\n") exit(1) diff --git a/src/runtime/os_only_solaris.go b/src/runtime/os_only_solaris.go index 0c72500674f72e..aa7b8faf007d6a 100644 --- a/src/runtime/os_only_solaris.go +++ b/src/runtime/os_only_solaris.go @@ -8,7 +8,7 @@ package runtime -func getncpu() int32 { +func getCPUCount() int32 { n := int32(sysconf(__SC_NPROCESSORS_ONLN)) if n < 1 { return 1 diff --git a/src/runtime/os_openbsd.go b/src/runtime/os_openbsd.go index 9a21d6a8d05228..39431118530b69 100644 --- a/src/runtime/os_openbsd.go +++ b/src/runtime/os_openbsd.go @@ -63,7 +63,7 @@ func internal_cpu_sysctlUint64(mib []uint32) (uint64, bool) { return sysctlUint64(mib) } -func getncpu() int32 { +func getCPUCount() int32 { // Try hw.ncpuonline first because hw.ncpu would report a number twice as // high as the actual CPUs running on OpenBSD 6.4 with hyperthreading // disabled (hw.smt=0). See https://golang.org/issue/30127 @@ -134,11 +134,62 @@ func semawakeup(mp *m) { } } +// mstart_stub provides glue code to call mstart from pthread_create. +func mstart_stub() + +// May run with m.p==nil, so write barriers are not allowed. +// +//go:nowritebarrierrec +func newosproc(mp *m) { + if false { + print("newosproc m=", mp, " g=", mp.g0, " id=", mp.id, " ostk=", &mp, "\n") + } + + // Initialize an attribute object. + var attr pthreadattr + if err := pthread_attr_init(&attr); err != 0 { + writeErrStr(failthreadcreate) + exit(1) + } + + // Find out OS stack size for our own stack guard. + var stacksize uintptr + if pthread_attr_getstacksize(&attr, &stacksize) != 0 { + writeErrStr(failthreadcreate) + exit(1) + } + mp.g0.stack.hi = stacksize // for mstart + + // Tell the pthread library we won't join with this thread. + if pthread_attr_setdetachstate(&attr, _PTHREAD_CREATE_DETACHED) != 0 { + writeErrStr(failthreadcreate) + exit(1) + } + + // Finally, create the thread. It starts at mstart_stub, which does some low-level + // setup and then calls mstart. + var oset sigset + sigprocmask(_SIG_SETMASK, &sigset_all, &oset) + err := retryOnEAGAIN(func() int32 { + return pthread_create(&attr, abi.FuncPCABI0(mstart_stub), unsafe.Pointer(mp)) + }) + sigprocmask(_SIG_SETMASK, &oset, nil) + if err != 0 { + writeErrStr(failthreadcreate) + exit(1) + } + + pthread_attr_destroy(&attr) +} + func osinit() { - ncpu = getncpu() + numCPUStartup = getCPUCount() physPageSize = getPageSize() } +// TODO(#69781): set startupRand using the .openbsd.randomdata ELF section. +// See SPECS.randomdata. + var urandom_dev = []byte("/dev/urandom\x00") //go:nosplit @@ -157,9 +208,6 @@ func goenvs() { // Called on the parent thread (main thread in case of bootstrap), can allocate memory. func mpreinit(mp *m) { gsignalSize := int32(32 * 1024) - if GOARCH == "mips64" { - gsignalSize = int32(64 * 1024) - } mp.gsignal = malg(gsignalSize) mp.gsignal.m = mp } @@ -179,8 +227,12 @@ func unminit() { getg().m.procid = 0 } -// Called from exitm, but not from drop, to undo the effect of thread-owned +// Called from mexit, but not from dropm, to undo the effect of thread-owned // resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +// +// This always runs without a P, so //go:nowritebarrierrec is required. +// +//go:nowritebarrierrec func mdestroy(mp *m) { } diff --git a/src/runtime/os_openbsd_arm.go b/src/runtime/os_openbsd_arm.go index d5dc8cb4792406..ba33d4d5bb268b 100644 --- a/src/runtime/os_openbsd_arm.go +++ b/src/runtime/os_openbsd_arm.go @@ -7,8 +7,9 @@ package runtime func checkgoarm() { // TODO(minux): FP checks like in os_linux_arm.go. - // osinit not called yet, so ncpu not set: must use getncpu directly. - if getncpu() > 1 && goarm < 7 { + // osinit not called yet, so numCPUStartup not set: must use + // getCPUCount directly. + if getCPUCount() > 1 && goarm < 7 { print("runtime: this system has multiple CPUs and must use\n") print("atomic synchronization instructions. Recompile using GOARM=7.\n") exit(1) diff --git a/src/runtime/os_openbsd_libc.go b/src/runtime/os_openbsd_libc.go deleted file mode 100644 index 201f1629d9b78b..00000000000000 --- a/src/runtime/os_openbsd_libc.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build openbsd && !mips64 - -package runtime - -import ( - "internal/abi" - "unsafe" -) - -// mstart_stub provides glue code to call mstart from pthread_create. -func mstart_stub() - -// May run with m.p==nil, so write barriers are not allowed. -// -//go:nowritebarrierrec -func newosproc(mp *m) { - if false { - print("newosproc m=", mp, " g=", mp.g0, " id=", mp.id, " ostk=", &mp, "\n") - } - - // Initialize an attribute object. - var attr pthreadattr - if err := pthread_attr_init(&attr); err != 0 { - writeErrStr(failthreadcreate) - exit(1) - } - - // Find out OS stack size for our own stack guard. - var stacksize uintptr - if pthread_attr_getstacksize(&attr, &stacksize) != 0 { - writeErrStr(failthreadcreate) - exit(1) - } - mp.g0.stack.hi = stacksize // for mstart - - // Tell the pthread library we won't join with this thread. - if pthread_attr_setdetachstate(&attr, _PTHREAD_CREATE_DETACHED) != 0 { - writeErrStr(failthreadcreate) - exit(1) - } - - // Finally, create the thread. It starts at mstart_stub, which does some low-level - // setup and then calls mstart. - var oset sigset - sigprocmask(_SIG_SETMASK, &sigset_all, &oset) - err := retryOnEAGAIN(func() int32 { - return pthread_create(&attr, abi.FuncPCABI0(mstart_stub), unsafe.Pointer(mp)) - }) - sigprocmask(_SIG_SETMASK, &oset, nil) - if err != 0 { - writeErrStr(failthreadcreate) - exit(1) - } - - pthread_attr_destroy(&attr) -} diff --git a/src/runtime/os_openbsd_mips64.go b/src/runtime/os_openbsd_mips64.go deleted file mode 100644 index e5eeb2dcd1070b..00000000000000 --- a/src/runtime/os_openbsd_mips64.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -//go:nosplit -func cputicks() int64 { - // runtime·nanotime() is a poor approximation of CPU ticks that is enough for the profiler. - return nanotime() -} diff --git a/src/runtime/os_openbsd_syscall.go b/src/runtime/os_openbsd_syscall.go deleted file mode 100644 index d784f764750ebd..00000000000000 --- a/src/runtime/os_openbsd_syscall.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build openbsd && mips64 - -package runtime - -import ( - "internal/abi" - "internal/goarch" - "unsafe" -) - -//go:noescape -func tfork(param *tforkt, psize uintptr, mm *m, gg *g, fn uintptr) int32 - -// May run with m.p==nil, so write barriers are not allowed. -// -//go:nowritebarrier -func newosproc(mp *m) { - stk := unsafe.Pointer(mp.g0.stack.hi) - if false { - print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " id=", mp.id, " ostk=", &mp, "\n") - } - - // Stack pointer must point inside stack area (as marked with MAP_STACK), - // rather than at the top of it. - param := tforkt{ - tf_tcb: unsafe.Pointer(&mp.tls[0]), - tf_tid: nil, // minit will record tid - tf_stack: uintptr(stk) - goarch.PtrSize, - } - - var oset sigset - sigprocmask(_SIG_SETMASK, &sigset_all, &oset) - ret := retryOnEAGAIN(func() int32 { - errno := tfork(¶m, unsafe.Sizeof(param), mp, mp.g0, abi.FuncPCABI0(mstart)) - // tfork returns negative errno - return -errno - }) - sigprocmask(_SIG_SETMASK, &oset, nil) - - if ret != 0 { - print("runtime: failed to create new OS thread (have ", mcount()-1, " already; errno=", ret, ")\n") - if ret == _EAGAIN { - println("runtime: may need to increase max user processes (ulimit -p)") - } - throw("runtime.newosproc") - } -} diff --git a/src/runtime/os_openbsd_syscall1.go b/src/runtime/os_openbsd_syscall1.go deleted file mode 100644 index d32894ba6a38b2..00000000000000 --- a/src/runtime/os_openbsd_syscall1.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build openbsd && mips64 - -package runtime - -//go:noescape -func thrsleep(ident uintptr, clock_id int32, tsp *timespec, lock uintptr, abort *uint32) int32 - -//go:noescape -func thrwakeup(ident uintptr, n int32) int32 - -func osyield() - -//go:nosplit -func osyield_no_g() { - osyield() -} diff --git a/src/runtime/os_openbsd_syscall2.go b/src/runtime/os_openbsd_syscall2.go deleted file mode 100644 index 072f53320dc50a..00000000000000 --- a/src/runtime/os_openbsd_syscall2.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build openbsd && mips64 - -package runtime - -import ( - "internal/runtime/atomic" - "unsafe" -) - -//go:noescape -func sigaction(sig uint32, new, old *sigactiont) - -func kqueue() int32 - -//go:noescape -func kevent(kq int32, ch *keventt, nch int32, ev *keventt, nev int32, ts *timespec) int32 - -func raiseproc(sig uint32) - -func getthrid() int32 -func thrkill(tid int32, sig int) - -// read calls the read system call. -// It returns a non-negative number of bytes written or a negative errno value. -func read(fd int32, p unsafe.Pointer, n int32) int32 - -func closefd(fd int32) int32 - -func exit(code int32) -func usleep(usec uint32) - -//go:nosplit -func usleep_no_g(usec uint32) { - usleep(usec) -} - -// write1 calls the write system call. -// It returns a non-negative number of bytes written or a negative errno value. -// -//go:noescape -func write1(fd uintptr, p unsafe.Pointer, n int32) int32 - -//go:noescape -func open(name *byte, mode, perm int32) int32 - -// return value is only set on linux to be used in osinit(). -func madvise(addr unsafe.Pointer, n uintptr, flags int32) int32 - -// exitThread terminates the current thread, writing *wait = freeMStack when -// the stack is safe to reclaim. -// -//go:noescape -func exitThread(wait *atomic.Uint32) - -//go:noescape -func obsdsigprocmask(how int32, new sigset) sigset - -//go:nosplit -//go:nowritebarrierrec -func sigprocmask(how int32, new, old *sigset) { - n := sigset(0) - if new != nil { - n = *new - } - r := obsdsigprocmask(how, n) - if old != nil { - *old = r - } -} - -func pipe2(flags int32) (r, w int32, errno int32) - -//go:noescape -func setitimer(mode int32, new, old *itimerval) - -//go:noescape -func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 - -// mmap calls the mmap system call. It is implemented in assembly. -// We only pass the lower 32 bits of file offset to the -// assembly routine; the higher bits (if required), should be provided -// by the assembly routine as 0. -// The err result is an OS error code such as ENOMEM. -func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (p unsafe.Pointer, err int) - -// munmap calls the munmap system call. It is implemented in assembly. -func munmap(addr unsafe.Pointer, n uintptr) - -func nanotime1() int64 - -//go:noescape -func sigaltstack(new, old *stackt) - -func fcntl(fd, cmd, arg int32) (ret int32, errno int32) - -func walltime() (sec int64, nsec int32) - -func issetugid() int32 diff --git a/src/runtime/os_plan9.go b/src/runtime/os_plan9.go index 2dbb42ad037d1f..72a86579854b5c 100644 --- a/src/runtime/os_plan9.go +++ b/src/runtime/os_plan9.go @@ -6,6 +6,7 @@ package runtime import ( "internal/abi" + "internal/byteorder" "internal/runtime/atomic" "internal/stringslite" "unsafe" @@ -18,6 +19,7 @@ type mOS struct { ignoreHangup bool } +func dupfd(old, new int32) int32 func closefd(fd int32) int32 //go:noescape @@ -217,16 +219,20 @@ func minit() { func unminit() { } -// Called from exitm, but not from drop, to undo the effect of thread-owned +// Called from mexit, but not from dropm, to undo the effect of thread-owned // resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +// +// This always runs without a P, so //go:nowritebarrierrec is required. +// +//go:nowritebarrierrec func mdestroy(mp *m) { } var sysstat = []byte("/dev/sysstat\x00") -func getproccount() int32 { +func getCPUCount() int32 { var buf [2048]byte - fd := open(&sysstat[0], _OREAD, 0) + fd := open(&sysstat[0], _OREAD|_OCEXEC, 0) if fd < 0 { return 1 } @@ -255,7 +261,7 @@ var pagesize = []byte(" pagesize\n") func getPageSize() uintptr { var buf [2048]byte var pos int - fd := open(&devswap[0], _OREAD, 0) + fd := open(&devswap[0], _OREAD|_OCEXEC, 0) if fd < 0 { // There's not much we can do if /dev/swap doesn't // exist. However, nothing in the memory manager uses @@ -314,11 +320,36 @@ func getpid() uint64 { return uint64(_atoi(c)) } +var ( + bintimeFD int32 = -1 + + bintimeDev = []byte("/dev/bintime\x00") + randomDev = []byte("/dev/random\x00") +) + func osinit() { physPageSize = getPageSize() initBloc() - ncpu = getproccount() + numCPUStartup = getCPUCount() getg().m.procid = getpid() + + fd := open(&bintimeDev[0], _OREAD|_OCEXEC, 0) + if fd < 0 { + fatal("cannot open /dev/bintime") + } + bintimeFD = fd + + // Move fd high up, to avoid conflicts with smaller ones + // that programs might hard code, and to make exec's job easier. + // Plan 9 allocates chunks of DELTAFD=20 fds in a row, + // so 18 is near the top of what's possible. + if bintimeFD < 18 { + if dupfd(bintimeFD, 18) < 0 { + fatal("cannot dup /dev/bintime onto 18") + } + closefd(bintimeFD) + bintimeFD = 18 + } } //go:nosplit @@ -327,6 +358,10 @@ func crash() { *(*int)(nil) = 0 } +// Don't read from /dev/random, since this device can only +// return a few hundred bits a second and would slow creation +// of Go processes down significantly. +// //go:nosplit func readRandom(r []byte) int { return 0 @@ -362,17 +397,6 @@ func usleep_no_g(usec uint32) { usleep(usec) } -//go:nosplit -func nanotime1() int64 { - var scratch int64 - ns := nsec(&scratch) - // TODO(aram): remove hack after I fix _nsec in the pc64 kernel. - if ns == 0 { - return scratch - } - return ns -} - var goexits = []byte("go: exit ") var emptystatus = []byte("\x00") var exiting uint32 @@ -530,3 +554,43 @@ func preemptM(mp *m) { // // TODO: Use a note like we use signals on POSIX OSes } + +//go:nosplit +func readtime(t *uint64, min, n int) int { + if bintimeFD < 0 { + fatal("/dev/bintime not opened") + } + const uint64size = 8 + r := pread(bintimeFD, unsafe.Pointer(t), int32(n*uint64size), 0) + if int(r) < min*uint64size { + fatal("cannot read /dev/bintime") + } + return int(r) / uint64size +} + +// timesplit returns u/1e9, u%1e9 +func timesplit(u uint64) (sec int64, nsec int32) + +func frombe(u uint64) uint64 { + b := (*[8]byte)(unsafe.Pointer(&u)) + return byteorder.BEUint64(b[:]) +} + +//go:nosplit +func nanotime1() int64 { + var t [4]uint64 + if readtime(&t[0], 1, 4) == 4 { + // long read indicates new kernel sending monotonic time + // (https://github.com/rsc/plan9/commit/baf076425). + return int64(frombe(t[3])) + } + // fall back to unix time + return int64(frombe(t[0])) +} + +//go:nosplit +func walltime() (sec int64, nsec int32) { + var t [1]uint64 + readtime(&t[0], 1, 1) + return timesplit(frombe(t[0])) +} diff --git a/src/runtime/os_solaris.go b/src/runtime/os_solaris.go index bc00698cbacf03..42b7e4a6bc2c5a 100644 --- a/src/runtime/os_solaris.go +++ b/src/runtime/os_solaris.go @@ -4,7 +4,10 @@ package runtime -import "unsafe" +import ( + "internal/runtime/sys" + "unsafe" +) type mts struct { tv_sec int64 @@ -18,9 +21,8 @@ type mscratch struct { type mOS struct { waitsema uintptr // semaphore for parking on locks perrno *int32 // pointer to tls errno - // these are here because they are too large to be on the stack - // of low-level NOSPLIT functions. - //LibCall libcall; + // This is here to avoid using the G stack so the stack can move during the call. + libcall libcall ts mts scratch mscratch } @@ -42,10 +44,10 @@ func sysvicall0(fn *libcFunc) uintptr { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { mp = nil // See comment in sys_darwin.go:libcCall } @@ -80,10 +82,10 @@ func sysvicall1Err(fn *libcFunc, a1 uintptr) (r1, err uintptr) { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { mp = nil } @@ -120,10 +122,10 @@ func sysvicall2Err(fn *libcFunc, a1, a2 uintptr) (uintptr, uintptr) { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { mp = nil } @@ -159,10 +161,10 @@ func sysvicall3Err(fn *libcFunc, a1, a2, a3 uintptr) (r1, err uintptr) { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { mp = nil } @@ -189,10 +191,10 @@ func sysvicall4(fn *libcFunc, a1, a2, a3, a4 uintptr) uintptr { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { mp = nil } @@ -219,10 +221,10 @@ func sysvicall5(fn *libcFunc, a1, a2, a3, a4, a5 uintptr) uintptr { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { mp = nil } @@ -249,10 +251,10 @@ func sysvicall6(fn *libcFunc, a1, a2, a3, a4, a5, a6 uintptr) uintptr { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { mp = nil } diff --git a/src/runtime/os_unix_nonlinux.go b/src/runtime/os_unix_nonlinux.go index b98753b8fe12b7..0e8b61c3b11aa2 100644 --- a/src/runtime/os_unix_nonlinux.go +++ b/src/runtime/os_unix_nonlinux.go @@ -13,3 +13,10 @@ package runtime func (c *sigctxt) sigFromUser() bool { return c.sigcode() == _SI_USER } + +// sigFromSeccomp reports whether the signal was sent from seccomp. +// +//go:nosplit +func (c *sigctxt) sigFromSeccomp() bool { + return false +} diff --git a/src/runtime/os_wasip1.go b/src/runtime/os_wasip1.go index acac2b3f7ad912..ed4f646a62bf0f 100644 --- a/src/runtime/os_wasip1.go +++ b/src/runtime/os_wasip1.go @@ -6,7 +6,10 @@ package runtime -import "unsafe" +import ( + "structs" + "unsafe" +) // GOARCH=wasm currently has 64 bits pointers, but the WebAssembly host expects // pointers to be 32 bits so we use this type alias to represent pointers in @@ -48,31 +51,31 @@ func exit(code int32) //go:wasmimport wasi_snapshot_preview1 args_get //go:noescape -func args_get(argv, argvBuf unsafe.Pointer) errno +func args_get(argv *uintptr32, argvBuf *byte) errno //go:wasmimport wasi_snapshot_preview1 args_sizes_get //go:noescape -func args_sizes_get(argc, argvBufLen unsafe.Pointer) errno +func args_sizes_get(argc, argvBufLen *size) errno //go:wasmimport wasi_snapshot_preview1 clock_time_get //go:noescape -func clock_time_get(clock_id clockid, precision timestamp, time unsafe.Pointer) errno +func clock_time_get(clock_id clockid, precision timestamp, time *timestamp) errno //go:wasmimport wasi_snapshot_preview1 environ_get //go:noescape -func environ_get(environ, environBuf unsafe.Pointer) errno +func environ_get(environ *uintptr32, environBuf *byte) errno //go:wasmimport wasi_snapshot_preview1 environ_sizes_get //go:noescape -func environ_sizes_get(environCount, environBufLen unsafe.Pointer) errno +func environ_sizes_get(environCount, environBufLen *size) errno //go:wasmimport wasi_snapshot_preview1 fd_write //go:noescape -func fd_write(fd int32, iovs unsafe.Pointer, iovsLen size, nwritten unsafe.Pointer) errno +func fd_write(fd int32, iovs unsafe.Pointer, iovsLen size, nwritten *size) errno //go:wasmimport wasi_snapshot_preview1 random_get //go:noescape -func random_get(buf unsafe.Pointer, bufLen size) errno +func random_get(buf *byte, bufLen size) errno type eventtype = uint8 @@ -99,6 +102,7 @@ type userdata = uint64 // struct size because errno is declared as a 32 bits type, so we declare the // error field as a plain uint16. type event struct { + _ structs.HostLayout userdata userdata error uint16 typ eventtype @@ -106,6 +110,7 @@ type event struct { } type eventFdReadwrite struct { + _ structs.HostLayout nbytes filesize flags eventrwflags } @@ -117,6 +122,7 @@ const ( ) type subscriptionClock struct { + _ structs.HostLayout id clockid timeout timestamp precision timestamp @@ -124,10 +130,12 @@ type subscriptionClock struct { } type subscriptionFdReadwrite struct { + _ structs.HostLayout fd int32 } type subscription struct { + _ structs.HostLayout userdata userdata u subscriptionUnion } @@ -148,7 +156,7 @@ func (u *subscriptionUnion) subscriptionFdReadwrite() *subscriptionFdReadwrite { //go:wasmimport wasi_snapshot_preview1 poll_oneoff //go:noescape -func poll_oneoff(in, out unsafe.Pointer, nsubscriptions size, nevents unsafe.Pointer) errno +func poll_oneoff(in *subscription, out *event, nsubscriptions size, nevents *size) errno func write1(fd uintptr, p unsafe.Pointer, n int32) int32 { iov := iovec{ @@ -156,7 +164,7 @@ func write1(fd uintptr, p unsafe.Pointer, n int32) int32 { bufLen: size(n), } var nwritten size - if fd_write(int32(fd), unsafe.Pointer(&iov), 1, unsafe.Pointer(&nwritten)) != 0 { + if fd_write(int32(fd), unsafe.Pointer(&iov), 1, &nwritten) != 0 { throw("fd_write failed") } return int32(nwritten) @@ -175,13 +183,13 @@ func usleep(usec uint32) { subscription.timeout = timestamp(usec) * 1e3 subscription.precision = 1e3 - if poll_oneoff(unsafe.Pointer(&in), unsafe.Pointer(&out), 1, unsafe.Pointer(&nevents)) != 0 { + if poll_oneoff(&in, &out, 1, &nevents) != 0 { throw("wasi_snapshot_preview1.poll_oneoff") } } func readRandom(r []byte) int { - if random_get(unsafe.Pointer(&r[0]), size(len(r))) != 0 { + if random_get(&r[0], size(len(r))) != 0 { return 0 } return len(r) @@ -191,7 +199,7 @@ func goenvs() { // arguments var argc size var argvBufLen size - if args_sizes_get(unsafe.Pointer(&argc), unsafe.Pointer(&argvBufLen)) != 0 { + if args_sizes_get(&argc, &argvBufLen) != 0 { throw("args_sizes_get failed") } @@ -199,7 +207,7 @@ func goenvs() { if argc > 0 { argv := make([]uintptr32, argc) argvBuf := make([]byte, argvBufLen) - if args_get(unsafe.Pointer(&argv[0]), unsafe.Pointer(&argvBuf[0])) != 0 { + if args_get(&argv[0], &argvBuf[0]) != 0 { throw("args_get failed") } @@ -216,7 +224,7 @@ func goenvs() { // environment var environCount size var environBufLen size - if environ_sizes_get(unsafe.Pointer(&environCount), unsafe.Pointer(&environBufLen)) != 0 { + if environ_sizes_get(&environCount, &environBufLen) != 0 { throw("environ_sizes_get failed") } @@ -224,7 +232,7 @@ func goenvs() { if environCount > 0 { environ := make([]uintptr32, environCount) environBuf := make([]byte, environBufLen) - if environ_get(unsafe.Pointer(&environ[0]), unsafe.Pointer(&environBuf[0])) != 0 { + if environ_get(&environ[0], &environBuf[0]) != 0 { throw("environ_get failed") } @@ -245,7 +253,7 @@ func walltime() (sec int64, nsec int32) { func walltime1() (sec int64, nsec int32) { var time timestamp - if clock_time_get(clockRealtime, 0, unsafe.Pointer(&time)) != 0 { + if clock_time_get(clockRealtime, 0, &time) != 0 { throw("clock_time_get failed") } return int64(time / 1000000000), int32(time % 1000000000) @@ -253,7 +261,7 @@ func walltime1() (sec int64, nsec int32) { func nanotime1() int64 { var time timestamp - if clock_time_get(clockMonotonic, 0, unsafe.Pointer(&time)) != 0 { + if clock_time_get(clockMonotonic, 0, &time) != 0 { throw("clock_time_get failed") } return int64(time) diff --git a/src/runtime/os_wasm.go b/src/runtime/os_wasm.go index fbafc319b988e6..15137cc13f3329 100644 --- a/src/runtime/os_wasm.go +++ b/src/runtime/os_wasm.go @@ -13,10 +13,15 @@ func osinit() { // https://webassembly.github.io/spec/core/exec/runtime.html#memory-instances physPageSize = 64 * 1024 initBloc() - ncpu = 1 + blocMax = uintptr(currentMemory()) * physPageSize // record the initial linear memory size + numCPUStartup = getCPUCount() getg().m.procid = 2 } +func getCPUCount() int32 { + return 1 +} + const _SIGSEGV = 0xb func sigpanic() { @@ -96,7 +101,7 @@ func signame(sig uint32) string { } func crash() { - *(*int32)(nil) = 0 + abort() } func initsig(preinit bool) { @@ -109,10 +114,10 @@ func newosproc(mp *m) { throw("newosproc: not implemented") } +// Do nothing on WASM platform, always return EPIPE to caller. +// //go:linkname os_sigpipe os.sigpipe -func os_sigpipe() { - throw("too many writes on closed pipe") -} +func os_sigpipe() {} //go:linkname syscall_now syscall.now func syscall_now() (sec int64, nsec int32) { diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index 4aabc29644e6f9..7610802e0f7be8 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -6,8 +6,9 @@ package runtime import ( "internal/abi" - "internal/goarch" "internal/runtime/atomic" + "internal/runtime/sys" + "internal/runtime/syscall/windows" "unsafe" ) @@ -40,7 +41,6 @@ const ( //go:cgo_import_dynamic runtime._GetThreadContext GetThreadContext%2 "kernel32.dll" //go:cgo_import_dynamic runtime._SetThreadContext SetThreadContext%2 "kernel32.dll" //go:cgo_import_dynamic runtime._LoadLibraryExW LoadLibraryExW%3 "kernel32.dll" -//go:cgo_import_dynamic runtime._LoadLibraryW LoadLibraryW%1 "kernel32.dll" //go:cgo_import_dynamic runtime._PostQueuedCompletionStatus PostQueuedCompletionStatus%4 "kernel32.dll" //go:cgo_import_dynamic runtime._QueryPerformanceCounter QueryPerformanceCounter%1 "kernel32.dll" //go:cgo_import_dynamic runtime._QueryPerformanceFrequency QueryPerformanceFrequency%1 "kernel32.dll" @@ -98,7 +98,6 @@ var ( _GetThreadContext, _SetThreadContext, _LoadLibraryExW, - _LoadLibraryW, _PostQueuedCompletionStatus, _QueryPerformanceCounter, _QueryPerformanceFrequency, @@ -160,6 +159,9 @@ func tstart_stdcall(newm *m) func wintls() type mOS struct { + // This is here to avoid using the G stack so the stack can move during the call. + stdCallInfo windows.StdCallInfo + threadLock mutex // protects "thread" and prevents closing thread uintptr // thread handle @@ -210,19 +212,15 @@ func read(fd int32, p unsafe.Pointer, n int32) int32 { type sigset struct{} -// Call a Windows function with stdcall conventions, -// and switch to os stack during the call. -func asmstdcall(fn unsafe.Pointer) - var asmstdcallAddr unsafe.Pointer -type winlibcall libcall +type winlibcall windows.StdCallInfo func windowsFindfunc(lib uintptr, name []byte) stdFunction { if name[len(name)-1] != 0 { throw("usage") } - f := stdcall2(_GetProcAddress, lib, uintptr(unsafe.Pointer(&name[0]))) + f := stdcall(_GetProcAddress, lib, uintptr(unsafe.Pointer(&name[0]))) return stdFunction(unsafe.Pointer(f)) } @@ -231,7 +229,7 @@ var sysDirectory [_MAX_PATH + 1]byte var sysDirectoryLen uintptr func initSysDirectory() { - l := stdcall2(_GetSystemDirectoryA, uintptr(unsafe.Pointer(&sysDirectory[0])), uintptr(len(sysDirectory)-1)) + l := stdcall(_GetSystemDirectoryA, uintptr(unsafe.Pointer(&sysDirectory[0])), uintptr(len(sysDirectory)-1)) if l == 0 || l > uintptr(len(sysDirectory)-1) { throw("Unable to determine system directory") } @@ -245,20 +243,21 @@ func windows_GetSystemDirectory() string { } func windowsLoadSystemLib(name []uint16) uintptr { - return stdcall3(_LoadLibraryExW, uintptr(unsafe.Pointer(&name[0])), 0, _LOAD_LIBRARY_SEARCH_SYSTEM32) + const _LOAD_LIBRARY_SEARCH_SYSTEM32 = 0x00000800 + return stdcall(_LoadLibraryExW, uintptr(unsafe.Pointer(&name[0])), 0, _LOAD_LIBRARY_SEARCH_SYSTEM32) } //go:linkname windows_QueryPerformanceCounter internal/syscall/windows.QueryPerformanceCounter func windows_QueryPerformanceCounter() int64 { var counter int64 - stdcall1(_QueryPerformanceCounter, uintptr(unsafe.Pointer(&counter))) + stdcall(_QueryPerformanceCounter, uintptr(unsafe.Pointer(&counter))) return counter } //go:linkname windows_QueryPerformanceFrequency internal/syscall/windows.QueryPerformanceFrequency func windows_QueryPerformanceFrequency() int64 { var frequency int64 - stdcall1(_QueryPerformanceFrequency, uintptr(unsafe.Pointer(&frequency))) + stdcall(_QueryPerformanceFrequency, uintptr(unsafe.Pointer(&frequency))) return frequency } @@ -309,7 +308,7 @@ func monitorSuspendResume() { var fn any = func(context uintptr, changeType uint32, setting uintptr) uintptr { for mp := (*m)(atomic.Loadp(unsafe.Pointer(&allm))); mp != nil; mp = mp.alllink { if mp.resumesema != 0 { - stdcall1(_SetEvent, mp.resumesema) + stdcall(_SetEvent, mp.resumesema) } } return 0 @@ -318,13 +317,13 @@ func monitorSuspendResume() { callback: compileCallback(*efaceOf(&fn), true), } handle := uintptr(0) - stdcall3(powerRegisterSuspendResumeNotification, _DEVICE_NOTIFY_CALLBACK, + stdcall(powerRegisterSuspendResumeNotification, _DEVICE_NOTIFY_CALLBACK, uintptr(unsafe.Pointer(¶ms)), uintptr(unsafe.Pointer(&handle))) } -func getproccount() int32 { +func getCPUCount() int32 { var mask, sysmask uintptr - ret := stdcall3(_GetProcessAffinityMask, currentProcess, uintptr(unsafe.Pointer(&mask)), uintptr(unsafe.Pointer(&sysmask))) + ret := stdcall(_GetProcessAffinityMask, windows.CurrentProcess, uintptr(unsafe.Pointer(&mask)), uintptr(unsafe.Pointer(&sysmask))) if ret != 0 { n := 0 maskbits := int(unsafe.Sizeof(mask) * 8) @@ -338,22 +337,17 @@ func getproccount() int32 { } } // use GetSystemInfo if GetProcessAffinityMask fails - var info systeminfo - stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info))) - return int32(info.dwnumberofprocessors) + var info windows.SystemInfo + stdcall(_GetSystemInfo, uintptr(unsafe.Pointer(&info))) + return int32(info.NumberOfProcessors) } func getPageSize() uintptr { - var info systeminfo - stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info))) - return uintptr(info.dwpagesize) + var info windows.SystemInfo + stdcall(_GetSystemInfo, uintptr(unsafe.Pointer(&info))) + return uintptr(info.PageSize) } -const ( - currentProcess = ^uintptr(0) // -1 = current process - currentThread = ^uintptr(1) // -2 = current thread -) - // in sys_windows_386.s and sys_windows_amd64.s: func getlasterror() uint32 @@ -384,9 +378,9 @@ func osRelax(relax bool) uint32 { } if relax { - return uint32(stdcall1(_timeEndPeriod, 1)) + return uint32(stdcall(_timeEndPeriod, 1)) } else { - return uint32(stdcall1(_timeBeginPeriod, 1)) + return uint32(stdcall(_timeBeginPeriod, 1)) } } @@ -406,18 +400,11 @@ var haveHighResSleep = false // resolution timer. createHighResTimer returns new timer // handle or 0, if CreateWaitableTimerEx failed. func createHighResTimer() uintptr { - const ( - // As per @jstarks, see - // https://github.com/golang/go/issues/8687#issuecomment-656259353 - _CREATE_WAITABLE_TIMER_HIGH_RESOLUTION = 0x00000002 - - _SYNCHRONIZE = 0x00100000 - _TIMER_QUERY_STATE = 0x0001 - _TIMER_MODIFY_STATE = 0x0002 - ) - return stdcall4(_CreateWaitableTimerExW, 0, 0, - _CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, - _SYNCHRONIZE|_TIMER_QUERY_STATE|_TIMER_MODIFY_STATE) + // As per @jstarks, see + // https://github.com/golang/go/issues/8687#issuecomment-656259353 + return stdcall(_CreateWaitableTimerExW, 0, 0, + windows.CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, + windows.SYNCHRONIZE|windows.TIMER_QUERY_STATE|windows.TIMER_MODIFY_STATE) } func initHighResTimer() { @@ -425,7 +412,7 @@ func initHighResTimer() { if h != 0 { haveHighResTimer = true haveHighResSleep = _NtCreateWaitCompletionPacket != nil - stdcall1(_CloseHandle, h) + stdcall(_CloseHandle, h) } else { // Only load winmm.dll if we need it. // This avoids a dependency on winmm.dll for Go programs @@ -455,24 +442,24 @@ func initLongPathSupport() { ) // Check that we're ≥ 10.0.15063. - info := _OSVERSIONINFOW{} - info.osVersionInfoSize = uint32(unsafe.Sizeof(info)) - stdcall1(_RtlGetVersion, uintptr(unsafe.Pointer(&info))) - if info.majorVersion < 10 || (info.majorVersion == 10 && info.minorVersion == 0 && info.buildNumber < 15063) { + info := windows.OSVERSIONINFOW{} + info.OSVersionInfoSize = uint32(unsafe.Sizeof(info)) + stdcall(_RtlGetVersion, uintptr(unsafe.Pointer(&info))) + if info.MajorVersion < 10 || (info.MajorVersion == 10 && info.MinorVersion == 0 && info.BuildNumber < 15063) { return } // Set the IsLongPathAwareProcess flag of the PEB's bit field. // This flag is not documented, but it's known to be used // by Windows to enable long path support. - bitField := (*byte)(unsafe.Pointer(stdcall0(_RtlGetCurrentPeb) + PebBitFieldOffset)) + bitField := (*byte)(unsafe.Pointer(stdcall(_RtlGetCurrentPeb) + PebBitFieldOffset)) *bitField |= IsLongPathAwareProcess canUseLongPaths = true } func osinit() { - asmstdcallAddr = unsafe.Pointer(abi.FuncPCABI0(asmstdcall)) + asmstdcallAddr = unsafe.Pointer(windows.AsmStdCallAddr()) loadOptionalSyscalls() @@ -486,7 +473,7 @@ func osinit() { initSysDirectory() initLongPathSupport() - ncpu = getproccount() + numCPUStartup = getCPUCount() physPageSize = getPageSize() @@ -494,13 +481,13 @@ func osinit() { // of dedicated threads -- GUI, IO, computational, etc. Go processes use // equivalent threads that all do a mix of GUI, IO, computations, etc. // In such context dynamic priority boosting does nothing but harm, so we turn it off. - stdcall2(_SetProcessPriorityBoost, currentProcess, 1) + stdcall(_SetProcessPriorityBoost, windows.CurrentProcess, 1) } //go:nosplit func readRandom(r []byte) int { n := 0 - if stdcall2(_ProcessPrng, uintptr(unsafe.Pointer(&r[0])), uintptr(len(r)))&0xff != 0 { + if stdcall(_ProcessPrng, uintptr(unsafe.Pointer(&r[0])), uintptr(len(r)))&0xff != 0 { n = len(r) } return n @@ -510,7 +497,7 @@ func goenvs() { // strings is a pointer to environment variable pairs in the form: // "envA=valA\x00envB=valB\x00\x00" (in UTF-16) // Two consecutive zero bytes end the list. - strings := unsafe.Pointer(stdcall0(_GetEnvironmentStringsW)) + strings := unsafe.Pointer(stdcall(_GetEnvironmentStringsW)) p := (*[1 << 24]uint16)(strings)[:] n := 0 @@ -534,13 +521,13 @@ func goenvs() { p = p[1:] // skip nil byte } - stdcall1(_FreeEnvironmentStringsW, uintptr(strings)) + stdcall(_FreeEnvironmentStringsW, uintptr(strings)) // We call these all the way here, late in init, so that malloc works // for the callback functions these generate. var fn any = ctrlHandler ctrlHandlerPC := compileCallback(*efaceOf(&fn), true) - stdcall2(_SetConsoleCtrlHandler, ctrlHandlerPC, 1) + stdcall(_SetConsoleCtrlHandler, ctrlHandlerPC, 1) monitorSuspendResume() } @@ -556,7 +543,7 @@ func exit(code int32) { // kills the suspending thread, and then this thread suspends. lock(&suspendLock) atomic.Store(&exiting, 1) - stdcall1(_ExitProcess, uintptr(code)) + stdcall(_ExitProcess, uintptr(code)) } // write1 must be nosplit because it's used as a last resort in @@ -572,9 +559,9 @@ func write1(fd uintptr, buf unsafe.Pointer, n int32) int32 { var handle uintptr switch fd { case 1: - handle = stdcall1(_GetStdHandle, _STD_OUTPUT_HANDLE) + handle = stdcall(_GetStdHandle, _STD_OUTPUT_HANDLE) case 2: - handle = stdcall1(_GetStdHandle, _STD_ERROR_HANDLE) + handle = stdcall(_GetStdHandle, _STD_ERROR_HANDLE) default: // assume fd is real windows handle. handle = fd @@ -590,7 +577,7 @@ func write1(fd uintptr, buf unsafe.Pointer, n int32) int32 { if !isASCII { var m uint32 - isConsole := stdcall2(_GetConsoleMode, handle, uintptr(unsafe.Pointer(&m))) != 0 + isConsole := stdcall(_GetConsoleMode, handle, uintptr(unsafe.Pointer(&m))) != 0 // If this is a console output, various non-unicode code pages can be in use. // Use the dedicated WriteConsole call to ensure unicode is printed correctly. if isConsole { @@ -598,7 +585,7 @@ func write1(fd uintptr, buf unsafe.Pointer, n int32) int32 { } } var written uint32 - stdcall5(_WriteFile, handle, uintptr(buf), uintptr(n), uintptr(unsafe.Pointer(&written)), 0) + stdcall(_WriteFile, handle, uintptr(buf), uintptr(n), uintptr(unsafe.Pointer(&written)), 0) return int32(written) } @@ -651,7 +638,7 @@ func writeConsoleUTF16(handle uintptr, b []uint16) { return } var written uint32 - stdcall5(_WriteConsoleW, + stdcall(_WriteConsoleW, handle, uintptr(unsafe.Pointer(&b[0])), uintptr(l), @@ -672,7 +659,7 @@ func semasleep(ns int64) int32 { var result uintptr if ns < 0 { - result = stdcall2(_WaitForSingleObject, getg().m.waitsema, uintptr(_INFINITE)) + result = stdcall(_WaitForSingleObject, getg().m.waitsema, uintptr(windows.INFINITE)) } else { start := nanotime() elapsed := int64(0) @@ -681,7 +668,7 @@ func semasleep(ns int64) int32 { if ms == 0 { ms = 1 } - result = stdcall4(_WaitForMultipleObjects, 2, + result = stdcall(_WaitForMultipleObjects, 2, uintptr(unsafe.Pointer(&[2]uintptr{getg().m.waitsema, getg().m.resumesema})), 0, uintptr(ms)) if result != _WAIT_OBJECT_0+1 { @@ -724,7 +711,7 @@ func semasleep(ns int64) int32 { //go:nosplit func semawakeup(mp *m) { - if stdcall1(_SetEvent, mp.waitsema) == 0 { + if stdcall(_SetEvent, mp.waitsema) == 0 { systemstack(func() { print("runtime: setevent failed; errno=", getlasterror(), "\n") throw("runtime.semawakeup") @@ -737,20 +724,20 @@ func semacreate(mp *m) { if mp.waitsema != 0 { return } - mp.waitsema = stdcall4(_CreateEventA, 0, 0, 0, 0) + mp.waitsema = stdcall(_CreateEventA, 0, 0, 0, 0) if mp.waitsema == 0 { systemstack(func() { print("runtime: createevent failed; errno=", getlasterror(), "\n") throw("runtime.semacreate") }) } - mp.resumesema = stdcall4(_CreateEventA, 0, 0, 0, 0) + mp.resumesema = stdcall(_CreateEventA, 0, 0, 0, 0) if mp.resumesema == 0 { systemstack(func() { print("runtime: createevent failed; errno=", getlasterror(), "\n") throw("runtime.semacreate") }) - stdcall1(_CloseHandle, mp.waitsema) + stdcall(_CloseHandle, mp.waitsema) mp.waitsema = 0 } } @@ -763,7 +750,7 @@ func semacreate(mp *m) { //go:nosplit func newosproc(mp *m) { // We pass 0 for the stack size to use the default for this binary. - thandle := stdcall6(_CreateThread, 0, 0, + thandle := stdcall(_CreateThread, 0, 0, abi.FuncPCABI0(tstart_stdcall), uintptr(unsafe.Pointer(mp)), 0, 0) @@ -781,7 +768,7 @@ func newosproc(mp *m) { } // Close thandle to avoid leaking the thread object if it exits. - stdcall1(_CloseHandle, thandle) + stdcall(_CloseHandle, thandle) } // Used by the C library build mode. On Linux this function would allocate a @@ -790,13 +777,14 @@ func newosproc(mp *m) { // //go:nowritebarrierrec //go:nosplit -func newosproc0(mp *m, stk unsafe.Pointer) { - // TODO: this is completely broken. The args passed to newosproc0 (in asm_amd64.s) - // are stacksize and function, not *m and stack. - // Check os_linux.go for an implementation that might actually work. +func newosproc0(stacksize uintptr, fn uintptr) { throw("bad newosproc0") } +//go:nosplit +//go:nowritebarrierrec +func libpreinit() {} + func exitThread(wait *atomic.Uint32) { // We should never reach exitThread on Windows because we let // the OS clean up threads. @@ -829,7 +817,7 @@ func sigblock(exiting bool) { // Called on the new thread, cannot allocate Go memory. func minit() { var thandle uintptr - if stdcall7(_DuplicateHandle, currentProcess, currentThread, currentProcess, uintptr(unsafe.Pointer(&thandle)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 { + if stdcall(_DuplicateHandle, windows.CurrentProcess, windows.CurrentThread, windows.CurrentProcess, uintptr(unsafe.Pointer(&thandle)), 0, 0, windows.DUPLICATE_SAME_ACCESS) == 0 { print("runtime.minit: duplicatehandle failed; errno=", getlasterror(), "\n") throw("runtime.minit: duplicatehandle failed") } @@ -837,7 +825,7 @@ func minit() { mp := getg().m lock(&mp.threadLock) mp.thread = thandle - mp.procid = uint64(stdcall0(_GetCurrentThreadId)) + mp.procid = uint64(stdcall(_GetCurrentThreadId)) // Configure usleep timer, if possible. if mp.highResTimer == 0 && haveHighResTimer { @@ -854,7 +842,7 @@ func minit() { throw("CreateWaitableTimerEx when creating timer failed") } const GENERIC_ALL = 0x10000000 - errno := stdcall3(_NtCreateWaitCompletionPacket, uintptr(unsafe.Pointer(&mp.waitIocpHandle)), GENERIC_ALL, 0) + errno := stdcall(_NtCreateWaitCompletionPacket, uintptr(unsafe.Pointer(&mp.waitIocpHandle)), GENERIC_ALL, 0) if mp.waitIocpHandle == 0 { print("runtime: NtCreateWaitCompletionPacket failed; errno=", errno, "\n") throw("NtCreateWaitCompletionPacket failed") @@ -864,8 +852,8 @@ func minit() { // Query the true stack base from the OS. Currently we're // running on a small assumed stack. - var mbi memoryBasicInformation - res := stdcall3(_VirtualQuery, uintptr(unsafe.Pointer(&mbi)), uintptr(unsafe.Pointer(&mbi)), unsafe.Sizeof(mbi)) + var mbi windows.MemoryBasicInformation + res := stdcall(_VirtualQuery, uintptr(unsafe.Pointer(&mbi)), uintptr(unsafe.Pointer(&mbi)), unsafe.Sizeof(mbi)) if res == 0 { print("runtime: VirtualQuery failed; errno=", getlasterror(), "\n") throw("VirtualQuery for stack base failed") @@ -876,7 +864,7 @@ func minit() { // calling C functions that don't have stack checks and for // lastcontinuehandler. We shouldn't be anywhere near this // bound anyway. - base := mbi.allocationBase + 16<<10 + base := mbi.AllocationBase + 16<<10 // Sanity check the stack bounds. g0 := getg() if base > g0.stack.hi || g0.stack.hi-base > 64<<20 { @@ -897,7 +885,7 @@ func unminit() { mp := getg().m lock(&mp.threadLock) if mp.thread != 0 { - stdcall1(_CloseHandle, mp.thread) + stdcall(_CloseHandle, mp.thread) mp.thread = 0 } unlock(&mp.threadLock) @@ -905,174 +893,103 @@ func unminit() { mp.procid = 0 } -// Called from exitm, but not from drop, to undo the effect of thread-owned +// Called from mexit, but not from dropm, to undo the effect of thread-owned // resources in minit, semacreate, or elsewhere. Do not take locks after calling this. // +// This always runs without a P, so //go:nowritebarrierrec is required. +// +//go:nowritebarrierrec //go:nosplit func mdestroy(mp *m) { if mp.highResTimer != 0 { - stdcall1(_CloseHandle, mp.highResTimer) + stdcall(_CloseHandle, mp.highResTimer) mp.highResTimer = 0 } if mp.waitIocpTimer != 0 { - stdcall1(_CloseHandle, mp.waitIocpTimer) + stdcall(_CloseHandle, mp.waitIocpTimer) mp.waitIocpTimer = 0 } if mp.waitIocpHandle != 0 { - stdcall1(_CloseHandle, mp.waitIocpHandle) + stdcall(_CloseHandle, mp.waitIocpHandle) mp.waitIocpHandle = 0 } if mp.waitsema != 0 { - stdcall1(_CloseHandle, mp.waitsema) + stdcall(_CloseHandle, mp.waitsema) mp.waitsema = 0 } if mp.resumesema != 0 { - stdcall1(_CloseHandle, mp.resumesema) + stdcall(_CloseHandle, mp.resumesema) mp.resumesema = 0 } } -// asmstdcall_trampoline calls asmstdcall converting from Go to C calling convention. -func asmstdcall_trampoline(args unsafe.Pointer) - -// stdcall_no_g calls asmstdcall on os stack without using g. +// stdcall_no_g is like [stdcall] but can be called without a G. // +//go:nowritebarrier //go:nosplit -func stdcall_no_g(fn stdFunction, n int, args uintptr) uintptr { - libcall := libcall{ - fn: uintptr(unsafe.Pointer(fn)), - n: uintptr(n), - args: args, +//go:uintptrkeepalive +func stdcall_no_g(fn stdFunction, args ...uintptr) uintptr { + call := windows.StdCallInfo{ + Fn: uintptr(unsafe.Pointer(fn)), + N: uintptr(len(args)), + } + if len(args) > 0 { + call.Args = uintptr(abi.NoEscape(unsafe.Pointer(&args[0]))) } - asmstdcall_trampoline(noescape(unsafe.Pointer(&libcall))) - return libcall.r1 + windows.StdCall(&call) + return call.R1 } -// Calling stdcall on os stack. +// stdcall calls fn with the given arguments using the stdcall calling convention. +// Must be called from the system stack. // May run during STW, so write barriers are not allowed. // //go:nowritebarrier //go:nosplit -func stdcall(fn stdFunction) uintptr { +//go:uintptrkeepalive +func stdcall(fn stdFunction, args ...uintptr) uintptr { gp := getg() mp := gp.m - mp.libcall.fn = uintptr(unsafe.Pointer(fn)) + mp.stdCallInfo.Fn = uintptr(unsafe.Pointer(fn)) + mp.stdCallInfo.N = uintptr(len(args)) + if len(args) > 0 { + mp.stdCallInfo.Args = uintptr(abi.NoEscape(unsafe.Pointer(&args[0]))) + } resetLibcall := false if mp.profilehz != 0 && mp.libcallsp == 0 { // leave pc/sp for cpu profiler mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() resetLibcall = true // See comment in sys_darwin.go:libcCall } - asmcgocall(asmstdcallAddr, unsafe.Pointer(&mp.libcall)) + asmcgocall(asmstdcallAddr, unsafe.Pointer(&mp.stdCallInfo)) if resetLibcall { mp.libcallsp = 0 } - return mp.libcall.r1 -} - -//go:nosplit -func stdcall0(fn stdFunction) uintptr { - mp := getg().m - mp.libcall.n = 0 - mp.libcall.args = 0 - return stdcall(fn) -} - -//go:nosplit -//go:cgo_unsafe_args -func stdcall1(fn stdFunction, a0 uintptr) uintptr { - mp := getg().m - mp.libcall.n = 1 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) - return stdcall(fn) -} - -//go:nosplit -//go:cgo_unsafe_args -func stdcall2(fn stdFunction, a0, a1 uintptr) uintptr { - mp := getg().m - mp.libcall.n = 2 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) - return stdcall(fn) -} - -//go:nosplit -//go:cgo_unsafe_args -func stdcall3(fn stdFunction, a0, a1, a2 uintptr) uintptr { - mp := getg().m - mp.libcall.n = 3 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) - return stdcall(fn) -} - -//go:nosplit -//go:cgo_unsafe_args -func stdcall4(fn stdFunction, a0, a1, a2, a3 uintptr) uintptr { - mp := getg().m - mp.libcall.n = 4 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) - return stdcall(fn) -} - -//go:nosplit -//go:cgo_unsafe_args -func stdcall5(fn stdFunction, a0, a1, a2, a3, a4 uintptr) uintptr { - mp := getg().m - mp.libcall.n = 5 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) - return stdcall(fn) -} - -//go:nosplit -//go:cgo_unsafe_args -func stdcall6(fn stdFunction, a0, a1, a2, a3, a4, a5 uintptr) uintptr { - mp := getg().m - mp.libcall.n = 6 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) - return stdcall(fn) -} - -//go:nosplit -//go:cgo_unsafe_args -func stdcall7(fn stdFunction, a0, a1, a2, a3, a4, a5, a6 uintptr) uintptr { - mp := getg().m - mp.libcall.n = 7 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) - return stdcall(fn) -} - -//go:nosplit -//go:cgo_unsafe_args -func stdcall8(fn stdFunction, a0, a1, a2, a3, a4, a5, a6, a7 uintptr) uintptr { - mp := getg().m - mp.libcall.n = 8 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) - return stdcall(fn) + return mp.stdCallInfo.R1 } // These must run on the system stack only. //go:nosplit func osyield_no_g() { - stdcall_no_g(_SwitchToThread, 0, 0) + stdcall_no_g(_SwitchToThread) } //go:nosplit func osyield() { systemstack(func() { - stdcall0(_SwitchToThread) + stdcall(_SwitchToThread) }) } //go:nosplit func usleep_no_g(us uint32) { timeout := uintptr(us) / 1000 // ms units - args := [...]uintptr{_INVALID_HANDLE_VALUE, timeout} - stdcall_no_g(_WaitForSingleObject, len(args), uintptr(noescape(unsafe.Pointer(&args[0])))) + stdcall_no_g(_WaitForSingleObject, windows.INVALID_HANDLE_VALUE, timeout) } //go:nosplit @@ -1084,13 +1001,13 @@ func usleep(us uint32) { if haveHighResTimer && getg().m.highResTimer != 0 { h = getg().m.highResTimer dt := -10 * int64(us) // relative sleep (negative), 100ns units - stdcall6(_SetWaitableTimer, h, uintptr(unsafe.Pointer(&dt)), 0, 0, 0, 0) - timeout = _INFINITE + stdcall(_SetWaitableTimer, h, uintptr(unsafe.Pointer(&dt)), 0, 0, 0, 0) + timeout = windows.INFINITE } else { - h = _INVALID_HANDLE_VALUE + h = windows.INVALID_HANDLE_VALUE timeout = uintptr(us) / 1000 // ms units } - stdcall2(_WaitForSingleObject, h, timeout) + stdcall(_WaitForSingleObject, h, timeout) }) } @@ -1098,16 +1015,16 @@ func ctrlHandler(_type uint32) uintptr { var s uint32 switch _type { - case _CTRL_C_EVENT, _CTRL_BREAK_EVENT: - s = _SIGINT - case _CTRL_CLOSE_EVENT, _CTRL_LOGOFF_EVENT, _CTRL_SHUTDOWN_EVENT: - s = _SIGTERM + case windows.CTRL_C_EVENT, windows.CTRL_BREAK_EVENT: + s = windows.SIGINT + case windows.CTRL_CLOSE_EVENT, windows.CTRL_LOGOFF_EVENT, windows.CTRL_SHUTDOWN_EVENT: + s = windows.SIGTERM default: return 0 } if sigsend(s) { - if s == _SIGTERM { + if s == windows.SIGTERM { // Windows terminates the process after this handler returns. // Block indefinitely to give signal handlers a chance to clean up, // but make sure to be properly parked first, so the rest of the @@ -1126,16 +1043,16 @@ var profiletimer uintptr func profilem(mp *m, thread uintptr) { // Align Context to 16 bytes. - var c *context + var c *windows.Context var cbuf [unsafe.Sizeof(*c) + 15]byte - c = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15)) + c = (*windows.Context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15)) - c.contextflags = _CONTEXT_CONTROL - stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(c))) + c.ContextFlags = windows.CONTEXT_CONTROL + stdcall(_GetThreadContext, thread, uintptr(unsafe.Pointer(c))) - gp := gFromSP(mp, c.sp()) + gp := gFromSP(mp, c.SP()) - sigprof(c.ip(), c.sp(), c.lr(), gp, mp) + sigprof(c.PC(), c.SP(), c.LR(), gp, mp) } func gFromSP(mp *m, sp uintptr) *g { @@ -1152,10 +1069,10 @@ func gFromSP(mp *m, sp uintptr) *g { } func profileLoop() { - stdcall2(_SetThreadPriority, currentThread, _THREAD_PRIORITY_HIGHEST) + stdcall(_SetThreadPriority, windows.CurrentThread, windows.THREAD_PRIORITY_HIGHEST) for { - stdcall2(_WaitForSingleObject, profiletimer, _INFINITE) + stdcall(_WaitForSingleObject, profiletimer, windows.INFINITE) first := (*m)(atomic.Loadp(unsafe.Pointer(&allm))) for mp := first; mp != nil; mp = mp.alllink { if mp == getg().m { @@ -1173,7 +1090,7 @@ func profileLoop() { } // Acquire our own handle to the thread. var thread uintptr - if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 { + if stdcall(_DuplicateHandle, windows.CurrentProcess, mp.thread, windows.CurrentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, windows.DUPLICATE_SAME_ACCESS) == 0 { print("runtime: duplicatehandle failed; errno=", getlasterror(), "\n") throw("duplicatehandle failed") } @@ -1183,9 +1100,9 @@ func profileLoop() { // above and the SuspendThread. The handle // will remain valid, but SuspendThread may // fail. - if int32(stdcall1(_SuspendThread, thread)) == -1 { + if int32(stdcall(_SuspendThread, thread)) == -1 { // The thread no longer exists. - stdcall1(_CloseHandle, thread) + stdcall(_CloseHandle, thread) continue } if mp.profilehz != 0 && !mp.blocked { @@ -1193,8 +1110,8 @@ func profileLoop() { // was in the process of shutting down. profilem(mp, thread) } - stdcall1(_ResumeThread, thread) - stdcall1(_CloseHandle, thread) + stdcall(_ResumeThread, thread) + stdcall(_CloseHandle, thread) } } } @@ -1205,7 +1122,7 @@ func setProcessCPUProfiler(hz int32) { if haveHighResTimer { timer = createHighResTimer() } else { - timer = stdcall3(_CreateWaitableTimerA, 0, 0, 0) + timer = stdcall(_CreateWaitableTimerA, 0, 0, 0) } atomic.Storeuintptr(&profiletimer, timer) newm(profileLoop, nil, -1) @@ -1222,7 +1139,7 @@ func setThreadCPUProfiler(hz int32) { } due = int64(ms) * -10000 } - stdcall6(_SetWaitableTimer, profiletimer, uintptr(unsafe.Pointer(&due)), uintptr(ms), 0, 0, 0) + stdcall(_SetWaitableTimer, profiletimer, uintptr(unsafe.Pointer(&due)), uintptr(ms), 0, 0, 0) atomic.Store((*uint32)(unsafe.Pointer(&getg().m.profilehz)), uint32(hz)) } @@ -1255,17 +1172,17 @@ func preemptM(mp *m) { return } var thread uintptr - if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 { + if stdcall(_DuplicateHandle, windows.CurrentProcess, mp.thread, windows.CurrentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, windows.DUPLICATE_SAME_ACCESS) == 0 { print("runtime.preemptM: duplicatehandle failed; errno=", getlasterror(), "\n") throw("runtime.preemptM: duplicatehandle failed") } unlock(&mp.threadLock) // Prepare thread context buffer. This must be aligned to 16 bytes. - var c *context + var c *windows.Context var cbuf [unsafe.Sizeof(*c) + 15]byte - c = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15)) - c.contextflags = _CONTEXT_CONTROL + c = (*windows.Context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15)) + c.ContextFlags = windows.CONTEXT_CONTROL // Serialize thread suspension. SuspendThread is asynchronous, // so it's otherwise possible for two threads to suspend each @@ -1275,9 +1192,9 @@ func preemptM(mp *m) { lock(&suspendLock) // Suspend the thread. - if int32(stdcall1(_SuspendThread, thread)) == -1 { + if int32(stdcall(_SuspendThread, thread)) == -1 { unlock(&suspendLock) - stdcall1(_CloseHandle, thread) + stdcall(_CloseHandle, thread) atomic.Store(&mp.preemptExtLock, 0) // The thread no longer exists. This shouldn't be // possible, but just acknowledge the request. @@ -1294,52 +1211,18 @@ func preemptM(mp *m) { // We have to get the thread context before inspecting the M // because SuspendThread only requests a suspend. // GetThreadContext actually blocks until it's suspended. - stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(c))) + stdcall(_GetThreadContext, thread, uintptr(unsafe.Pointer(c))) unlock(&suspendLock) // Does it want a preemption and is it safe to preempt? - gp := gFromSP(mp, c.sp()) + gp := gFromSP(mp, c.SP()) if gp != nil && wantAsyncPreempt(gp) { - if ok, newpc := isAsyncSafePoint(gp, c.ip(), c.sp(), c.lr()); ok { + if ok, resumePC := isAsyncSafePoint(gp, c.PC(), c.SP(), c.LR()); ok { // Inject call to asyncPreempt targetPC := abi.FuncPCABI0(asyncPreempt) - switch GOARCH { - default: - throw("unsupported architecture") - case "386", "amd64": - // Make it look like the thread called targetPC. - sp := c.sp() - sp -= goarch.PtrSize - *(*uintptr)(unsafe.Pointer(sp)) = newpc - c.set_sp(sp) - c.set_ip(targetPC) - - case "arm": - // Push LR. The injected call is responsible - // for restoring LR. gentraceback is aware of - // this extra slot. See sigctxt.pushCall in - // signal_arm.go, which is similar except we - // subtract 1 from IP here. - sp := c.sp() - sp -= goarch.PtrSize - c.set_sp(sp) - *(*uint32)(unsafe.Pointer(sp)) = uint32(c.lr()) - c.set_lr(newpc - 1) - c.set_ip(targetPC) - - case "arm64": - // Push LR. The injected call is responsible - // for restoring LR. gentraceback is aware of - // this extra slot. See sigctxt.pushCall in - // signal_arm64.go. - sp := c.sp() - 16 // SP needs 16-byte alignment - c.set_sp(sp) - *(*uint64)(unsafe.Pointer(sp)) = uint64(c.lr()) - c.set_lr(newpc) - c.set_ip(targetPC) - } - stdcall2(_SetThreadContext, thread, uintptr(unsafe.Pointer(c))) + c.PushCall(targetPC, resumePC) + stdcall(_SetThreadContext, thread, uintptr(unsafe.Pointer(c))) } } @@ -1348,8 +1231,8 @@ func preemptM(mp *m) { // Acknowledge the preemption. mp.preemptGen.Add(1) - stdcall1(_ResumeThread, thread) - stdcall1(_CloseHandle, thread) + stdcall(_ResumeThread, thread) + stdcall(_CloseHandle, thread) } // osPreemptExtEnter is called before entering external code that may diff --git a/src/runtime/os_windows_arm.go b/src/runtime/os_windows_arm.go deleted file mode 100644 index 10aff75e31191b..00000000000000 --- a/src/runtime/os_windows_arm.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -import "unsafe" - -//go:nosplit -func cputicks() int64 { - var counter int64 - stdcall1(_QueryPerformanceCounter, uintptr(unsafe.Pointer(&counter))) - return counter -} - -func checkgoarm() { - if goarm < 7 { - print("Need atomic synchronization instructions, coprocessor ", - "access instructions. Recompile using GOARM=7.\n") - exit(1) - } -} diff --git a/src/runtime/os_windows_arm64.go b/src/runtime/os_windows_arm64.go index 7e413445ba4302..62caea7c2c6ca3 100644 --- a/src/runtime/os_windows_arm64.go +++ b/src/runtime/os_windows_arm64.go @@ -9,6 +9,10 @@ import "unsafe" //go:nosplit func cputicks() int64 { var counter int64 - stdcall1(_QueryPerformanceCounter, uintptr(unsafe.Pointer(&counter))) + stdcall(_QueryPerformanceCounter, uintptr(unsafe.Pointer(&counter))) return counter } + +func stackcheck() { + // TODO: not implemented +} diff --git a/src/runtime/panic.go b/src/runtime/panic.go index bd1ea096aa2a01..8c91c9435abd18 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -59,7 +59,7 @@ func panicCheck1(pc uintptr, msg string) { throw(msg) } // TODO: is this redundant? How could we be in malloc - // but not in the runtime? runtime/internal/*, maybe? + // but not in the runtime? internal/runtime/*, maybe? gp := getg() if gp != nil && gp.m != nil && gp.m.mallocing != 0 { throw(msg) @@ -103,133 +103,207 @@ func panicCheck2(err string) { // these (they always look like they're called from the runtime). // Hence, for these, we just check for clearly bad runtime conditions. // -// The panic{Index,Slice} functions are implemented in assembly and tail call -// to the goPanic{Index,Slice} functions below. This is done so we can use -// a space-minimal register calling convention. +// The goPanic{Index,Slice} functions are only used by wasm. All the other architectures +// use panic{Bounds,Extend} in assembly, which then call to panicBounds{64,32,32X}. // failures in the comparisons for s[x], 0 <= x < y (y == len(s)) // //go:yeswritebarrierrec func goPanicIndex(x int, y int) { - panicCheck1(getcallerpc(), "index out of range") - panic(boundsError{x: int64(x), signed: true, y: y, code: boundsIndex}) + panicCheck1(sys.GetCallerPC(), "index out of range") + panic(boundsError{x: int64(x), signed: true, y: y, code: abi.BoundsIndex}) } //go:yeswritebarrierrec func goPanicIndexU(x uint, y int) { - panicCheck1(getcallerpc(), "index out of range") - panic(boundsError{x: int64(x), signed: false, y: y, code: boundsIndex}) + panicCheck1(sys.GetCallerPC(), "index out of range") + panic(boundsError{x: int64(x), signed: false, y: y, code: abi.BoundsIndex}) } // failures in the comparisons for s[:x], 0 <= x <= y (y == len(s) or cap(s)) // //go:yeswritebarrierrec func goPanicSliceAlen(x int, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSliceAlen}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(x), signed: true, y: y, code: abi.BoundsSliceAlen}) } //go:yeswritebarrierrec func goPanicSliceAlenU(x uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSliceAlen}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(x), signed: false, y: y, code: abi.BoundsSliceAlen}) } //go:yeswritebarrierrec func goPanicSliceAcap(x int, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSliceAcap}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(x), signed: true, y: y, code: abi.BoundsSliceAcap}) } //go:yeswritebarrierrec func goPanicSliceAcapU(x uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSliceAcap}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(x), signed: false, y: y, code: abi.BoundsSliceAcap}) } // failures in the comparisons for s[x:y], 0 <= x <= y // //go:yeswritebarrierrec func goPanicSliceB(x int, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSliceB}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(x), signed: true, y: y, code: abi.BoundsSliceB}) } //go:yeswritebarrierrec func goPanicSliceBU(x uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSliceB}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(x), signed: false, y: y, code: abi.BoundsSliceB}) } // failures in the comparisons for s[::x], 0 <= x <= y (y == len(s) or cap(s)) func goPanicSlice3Alen(x int, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3Alen}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(x), signed: true, y: y, code: abi.BoundsSlice3Alen}) } func goPanicSlice3AlenU(x uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3Alen}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(x), signed: false, y: y, code: abi.BoundsSlice3Alen}) } func goPanicSlice3Acap(x int, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3Acap}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(x), signed: true, y: y, code: abi.BoundsSlice3Acap}) } func goPanicSlice3AcapU(x uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3Acap}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(x), signed: false, y: y, code: abi.BoundsSlice3Acap}) } // failures in the comparisons for s[:x:y], 0 <= x <= y func goPanicSlice3B(x int, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3B}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(x), signed: true, y: y, code: abi.BoundsSlice3B}) } func goPanicSlice3BU(x uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3B}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(x), signed: false, y: y, code: abi.BoundsSlice3B}) } // failures in the comparisons for s[x:y:], 0 <= x <= y func goPanicSlice3C(x int, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3C}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(x), signed: true, y: y, code: abi.BoundsSlice3C}) } func goPanicSlice3CU(x uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3C}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(x), signed: false, y: y, code: abi.BoundsSlice3C}) } // failures in the conversion ([x]T)(s) or (*[x]T)(s), 0 <= x <= y, y == len(s) func goPanicSliceConvert(x int, y int) { - panicCheck1(getcallerpc(), "slice length too short to convert to array or pointer to array") - panic(boundsError{x: int64(x), signed: true, y: y, code: boundsConvert}) -} - -// Implemented in assembly, as they take arguments in registers. -// Declared here to mark them as ABIInternal. -func panicIndex(x int, y int) -func panicIndexU(x uint, y int) -func panicSliceAlen(x int, y int) -func panicSliceAlenU(x uint, y int) -func panicSliceAcap(x int, y int) -func panicSliceAcapU(x uint, y int) -func panicSliceB(x int, y int) -func panicSliceBU(x uint, y int) -func panicSlice3Alen(x int, y int) -func panicSlice3AlenU(x uint, y int) -func panicSlice3Acap(x int, y int) -func panicSlice3AcapU(x uint, y int) -func panicSlice3B(x int, y int) -func panicSlice3BU(x uint, y int) -func panicSlice3C(x int, y int) -func panicSlice3CU(x uint, y int) -func panicSliceConvert(x int, y int) + panicCheck1(sys.GetCallerPC(), "slice length too short to convert to array or pointer to array") + panic(boundsError{x: int64(x), signed: true, y: y, code: abi.BoundsConvert}) +} + +// Implemented in assembly. Declared here to mark them as ABIInternal. +func panicBounds() // in asm_GOARCH.s files, called from generated code +func panicExtend() // in asm_GOARCH.s files, called from generated code (on 32-bit archs) + +func panicBounds64(pc uintptr, regs *[16]int64) { // called from panicBounds on 64-bit archs + f := findfunc(pc) + v := pcdatavalue(f, abi.PCDATA_PanicBounds, pc-1) + + code, signed, xIsReg, yIsReg, xVal, yVal := abi.BoundsDecode(int(v)) + + if code == abi.BoundsIndex { + panicCheck1(pc, "index out of range") + } else { + panicCheck1(pc, "slice bounds out of range") + } + + var e boundsError + e.code = code + e.signed = signed + if xIsReg { + e.x = regs[xVal] + } else { + e.x = int64(xVal) + } + if yIsReg { + e.y = int(regs[yVal]) + } else { + e.y = yVal + } + panic(e) +} + +func panicBounds32(pc uintptr, regs *[16]int32) { // called from panicBounds on 32-bit archs + f := findfunc(pc) + v := pcdatavalue(f, abi.PCDATA_PanicBounds, pc-1) + + code, signed, xIsReg, yIsReg, xVal, yVal := abi.BoundsDecode(int(v)) + + if code == abi.BoundsIndex { + panicCheck1(pc, "index out of range") + } else { + panicCheck1(pc, "slice bounds out of range") + } + + var e boundsError + e.code = code + e.signed = signed + if xIsReg { + if signed { + e.x = int64(regs[xVal]) + } else { + e.x = int64(uint32(regs[xVal])) + } + } else { + e.x = int64(xVal) + } + if yIsReg { + e.y = int(regs[yVal]) + } else { + e.y = yVal + } + panic(e) +} + +func panicBounds32X(pc uintptr, regs *[16]int32) { // called from panicExtend on 32-bit archs + f := findfunc(pc) + v := pcdatavalue(f, abi.PCDATA_PanicBounds, pc-1) + + code, signed, xIsReg, yIsReg, xVal, yVal := abi.BoundsDecode(int(v)) + + if code == abi.BoundsIndex { + panicCheck1(pc, "index out of range") + } else { + panicCheck1(pc, "slice bounds out of range") + } + + var e boundsError + e.code = code + e.signed = signed + if xIsReg { + // Our 4-bit register numbers are actually 2 2-bit register numbers. + lo := xVal & 3 + hi := xVal >> 2 + e.x = int64(regs[hi])<<32 + int64(uint32(regs[lo])) + } else { + e.x = int64(xVal) + } + if yIsReg { + e.y = int(regs[yVal]) + } else { + e.y = yVal + } + panic(e) +} var shiftError = error(errorString("negative shift amount")) //go:yeswritebarrierrec func panicshift() { - panicCheck1(getcallerpc(), "negative shift amount") + panicCheck1(sys.GetCallerPC(), "negative shift amount") panic(shiftError) } @@ -280,21 +354,11 @@ func deferproc(fn func()) { d.link = gp._defer gp._defer = d d.fn = fn - d.pc = getcallerpc() - // We must not be preempted between calling getcallersp and - // storing it to d.sp because getcallersp's result is a + d.pc = sys.GetCallerPC() + // We must not be preempted between calling GetCallerSP and + // storing it to d.sp because GetCallerSP's result is a // uintptr stack pointer. - d.sp = getcallersp() - - // deferproc returns 0 normally. - // a deferred func that stops a panic - // makes the deferproc return 1. - // the code the compiler generates always - // checks the return value and jumps to the - // end of the function if deferproc returns != 0. - return0() - // No code can go here - the C return register has - // been set and must not be clobbered. + d.sp = sys.GetCallerSP() } var rangeDoneError = error(errorString("range function continued iteration after function for loop body returned false")) @@ -394,11 +458,11 @@ func deferrangefunc() any { d := newdefer() d.link = gp._defer gp._defer = d - d.pc = getcallerpc() - // We must not be preempted between calling getcallersp and - // storing it to d.sp because getcallersp's result is a + d.pc = sys.GetCallerPC() + // We must not be preempted between calling GetCallerSP and + // storing it to d.sp because GetCallerSP's result is a // uintptr stack pointer. - d.sp = getcallersp() + d.sp = sys.GetCallerSP() d.rangefunc = true d.head = new(atomic.Pointer[_defer]) @@ -416,7 +480,7 @@ func badDefer() *_defer { func deferprocat(fn func(), frame any) { head := frame.(*atomic.Pointer[_defer]) if raceenabled { - racewritepc(unsafe.Pointer(head), getcallerpc(), abi.FuncPCABIInternal(deferprocat)) + racewritepc(unsafe.Pointer(head), sys.GetCallerPC(), abi.FuncPCABIInternal(deferprocat)) } d1 := newdefer() d1.fn = fn @@ -429,9 +493,6 @@ func deferprocat(fn func(), frame any) { break } } - - // Must be last - see deferproc above. - return0() } // deferconvert converts the rangefunc defer list of d0 into an ordinary list @@ -440,7 +501,7 @@ func deferprocat(fn func(), frame any) { func deferconvert(d0 *_defer) { head := d0.head if raceenabled { - racereadpc(unsafe.Pointer(head), getcallerpc(), abi.FuncPCABIInternal(deferconvert)) + racereadpc(unsafe.Pointer(head), sys.GetCallerPC(), abi.FuncPCABIInternal(deferconvert)) } tail := d0.link d0.rangefunc = false @@ -479,13 +540,14 @@ func deferprocStack(d *_defer) { // go code on the system stack can't defer throw("defer on system stack") } + // fn is already set. // The other fields are junk on entry to deferprocStack and // are initialized here. d.heap = false d.rangefunc = false - d.sp = getcallersp() - d.pc = getcallerpc() + d.sp = sys.GetCallerSP() + d.pc = sys.GetCallerPC() // The lines below implement: // d.panic = nil // d.fd = nil @@ -501,10 +563,6 @@ func deferprocStack(d *_defer) { *(*uintptr)(unsafe.Pointer(&d.link)) = uintptr(unsafe.Pointer(gp._defer)) *(*uintptr)(unsafe.Pointer(&d.head)) = 0 *(*uintptr)(unsafe.Pointer(&gp._defer)) = uintptr(unsafe.Pointer(d)) - - return0() - // No code can go here - the C return register has - // been set and must not be clobbered. } // Each P holds a pool for defers. @@ -596,7 +654,7 @@ func deferreturn() { var p _panic p.deferreturn = true - p.start(getcallerpc(), unsafe.Pointer(getcallersp())) + p.start(sys.GetCallerPC(), unsafe.Pointer(sys.GetCallerSP())) for { fn, ok := p.nextDefer() if !ok { @@ -614,13 +672,15 @@ func deferreturn() { // without func main returning. Since func main has not returned, // the program continues execution of other goroutines. // If all other goroutines exit, the program crashes. +// +// It crashes if called from a thread not created by the Go runtime. func Goexit() { // Create a panic object for Goexit, so we can recognize when it might be // bypassed by a recover(). var p _panic p.goexit = true - p.start(getcallerpc(), unsafe.Pointer(getcallersp())) + p.start(sys.GetCallerPC(), unsafe.Pointer(sys.GetCallerSP())) for { fn, ok := p.nextDefer() if !ok { @@ -647,6 +707,13 @@ func preprintpanics(p *_panic) { } }() for p != nil { + if p.link != nil && *efaceOf(&p.link.arg) == *efaceOf(&p.arg) { + // This panic contains the same value as the next one in the chain. + // Mark it as repanicked. We will skip printing it twice in a row. + p.link.repanicked = true + p = p.link + continue + } switch v := p.arg.(type) { case error: p.arg = v.Error() @@ -662,6 +729,9 @@ func preprintpanics(p *_panic) { func printpanics(p *_panic) { if p.link != nil { printpanics(p.link) + if p.link.repanicked { + return + } if !p.link.goexit { print("\t") } @@ -671,7 +741,9 @@ func printpanics(p *_panic) { } print("panic: ") printpanicval(p.arg) - if p.recovered { + if p.repanicked { + print(" [recovered, repanicked]") + } else if p.recovered { print(" [recovered]") } print("\n") @@ -773,10 +845,11 @@ func gopanic(e any) { var p _panic p.arg = e + p.gopanicFP = unsafe.Pointer(sys.GetCallerSP()) runningPanicDefers.Add(1) - p.start(getcallerpc(), unsafe.Pointer(getcallersp())) + p.start(sys.GetCallerPC(), unsafe.Pointer(sys.GetCallerSP())) for { fn, ok := p.nextDefer() if !ok { @@ -815,8 +888,8 @@ func (p *_panic) start(pc uintptr, sp unsafe.Pointer) { // that have been recovered. Also, so that if p is from Goexit, we // can restart its defer processing loop if a recovered panic tries // to jump past it. - p.startPC = getcallerpc() - p.startSP = unsafe.Pointer(getcallersp()) + p.startPC = sys.GetCallerPC() + p.startSP = unsafe.Pointer(sys.GetCallerSP()) if p.deferreturn { p.sp = sp @@ -867,10 +940,6 @@ func (p *_panic) nextDefer() (func(), bool) { } } - // The assembler adjusts p.argp in wrapper functions that shouldn't - // be visible to recover(), so we need to restore it each iteration. - p.argp = add(p.startSP, sys.MinFrameSize) - for { for p.deferBitsPtr != nil { bits := *p.deferBitsPtr @@ -908,9 +977,6 @@ func (p *_panic) nextDefer() (func(), bool) { fn := d.fn - // TODO(mdempsky): Instead of having each deferproc call have - // its own "deferreturn(); return" sequence, we should just make - // them reuse the one we emit for open-coded defers. p.retpc = d.pc // Unlink and free. @@ -998,27 +1064,89 @@ func (p *_panic) initOpenCodedDefers(fn funcInfo, varp unsafe.Pointer) bool { } // The implementation of the predeclared function recover. -// Cannot split the stack because it needs to reliably -// find the stack segment of its caller. -// -// TODO(rsc): Once we commit to CopyStackAlways, -// this doesn't need to be nosplit. -// -//go:nosplit -func gorecover(argp uintptr) any { - // Must be in a function running as part of a deferred call during the panic. - // Must be called from the topmost function of the call - // (the function used in the defer statement). - // p.argp is the argument pointer of that topmost deferred function call. - // Compare against argp reported by caller. - // If they match, the caller is the one who can recover. +func gorecover() any { gp := getg() p := gp._panic - if p != nil && !p.goexit && !p.recovered && argp == uintptr(p.argp) { - p.recovered = true - return p.arg + if p == nil || p.goexit || p.recovered { + return nil } - return nil + + // Check to see if the function that called recover() was + // deferred directly from the panicking function. + // For code like: + // func foo() { + // defer bar() + // panic("panic") + // } + // func bar() { + // recover() + // } + // Normally the stack would look like this: + // foo + // runtime.gopanic + // bar + // runtime.gorecover + // + // However, if the function we deferred requires a wrapper + // of some sort, we need to ignore the wrapper. In that case, + // the stack looks like: + // foo + // runtime.gopanic + // wrapper + // bar + // runtime.gorecover + // And we should also successfully recover. + // + // Finally, in the weird case "defer recover()", the stack looks like: + // foo + // runtime.gopanic + // wrapper + // runtime.gorecover + // And we should not recover in that case. + // + // So our criteria is, there must be exactly one non-wrapper + // frame between gopanic and gorecover. + // + // We don't recover this: + // defer func() { func() { recover() }() } + // because there are 2 non-wrapper frames. + // + // We don't recover this: + // defer recover() + // because there are 0 non-wrapper frames. + canRecover := false + systemstack(func() { + var u unwinder + u.init(gp, 0) + u.next() // skip systemstack_switch + u.next() // skip gorecover + nonWrapperFrames := 0 + loop: + for ; u.valid(); u.next() { + for iu, f := newInlineUnwinder(u.frame.fn, u.symPC()); f.valid(); f = iu.next(f) { + sf := iu.srcFunc(f) + switch sf.funcID { + case abi.FuncIDWrapper: + continue + case abi.FuncID_gopanic: + if u.frame.fp == uintptr(p.gopanicFP) && nonWrapperFrames > 0 { + canRecover = true + } + break loop + default: + nonWrapperFrames++ + if nonWrapperFrames > 1 { + break loop + } + } + } + } + }) + if !canRecover { + return nil + } + p.recovered = true + return p.arg } //go:linkname sync_throw sync.throw @@ -1031,14 +1159,46 @@ func sync_fatal(s string) { fatal(s) } +//go:linkname rand_fatal crypto/rand.fatal +func rand_fatal(s string) { + fatal(s) +} + +//go:linkname sysrand_fatal crypto/internal/sysrand.fatal +func sysrand_fatal(s string) { + fatal(s) +} + +//go:linkname fips_fatal crypto/internal/fips140.fatal +func fips_fatal(s string) { + fatal(s) +} + +//go:linkname maps_fatal internal/runtime/maps.fatal +func maps_fatal(s string) { + fatal(s) +} + +//go:linkname internal_sync_throw internal/sync.throw +func internal_sync_throw(s string) { + throw(s) +} + +//go:linkname internal_sync_fatal internal/sync.fatal +func internal_sync_fatal(s string) { + fatal(s) +} + +//go:linkname cgroup_throw internal/runtime/cgroup.throw +func cgroup_throw(s string) { + throw(s) +} + // throw triggers a fatal error that dumps a stack trace and exits. // // throw should be used for runtime-internal fatal errors where Go itself, // rather than user code, may be at fault for the failure. // -// NOTE: temporarily marked "go:noinline" pending investigation/fix of -// issue #67274, so as to fix longtest builders. -// // throw should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: @@ -1079,6 +1239,7 @@ func throw(s string) { func fatal(s string) { // Everything fatal does should be recursively nosplit so it // can be called even when it's unsafe to grow the stack. + printlock() // Prevent multiple interleaved fatal reports. See issue 69447. systemstack(func() { print("fatal error: ") printindented(s) // logically printpanicval(s), but avoids convTstring write barrier @@ -1086,6 +1247,7 @@ func fatal(s string) { }) fatalthrow(throwTypeUser) + printunlock() } // runningPanicDefers is non-zero while running deferred functions for panic. @@ -1111,6 +1273,15 @@ func recovery(gp *g) { pc, sp, fp := p.retpc, uintptr(p.sp), uintptr(p.fp) p0, saveOpenDeferState := p, p.deferBitsPtr != nil && *p.deferBitsPtr != 0 + // The linker records the f-relative address of a call to deferreturn in f's funcInfo. + // Assuming a "normal" call to recover() inside one of f's deferred functions + // invoked for a panic, that is the desired PC for exiting f. + f := findfunc(pc) + if f.deferreturn == 0 { + throw("no deferreturn") + } + gotoPc := f.entry() + uintptr(f.deferreturn) + // Unwind the panic stack. for ; p != nil && uintptr(p.startSP) < sp; p = p.link { // Don't allow jumping past a pending Goexit. @@ -1121,7 +1292,7 @@ func recovery(gp *g) { // frames that we've already processed. // // There's a similar issue with nested panics, when the inner - // panic supercedes the outer panic. Again, we end up needing to + // panic supersedes the outer panic. Again, we end up needing to // walk the same stack frames. // // These are probably pretty rare occurrences in practice, and @@ -1133,7 +1304,7 @@ func recovery(gp *g) { // With how subtle defer handling is, this might not actually be // worthwhile though. if p.goexit { - pc, sp = p.startPC, uintptr(p.startSP) + gotoPc, sp = p.startPC, uintptr(p.startSP) saveOpenDeferState = false // goexit is unwinding the stack anyway break } @@ -1194,11 +1365,9 @@ func recovery(gp *g) { throw("bad recovery") } - // Make the deferproc for this d return again, - // this time returning 1. The calling function will - // jump to the standard return epilogue. + // branch directly to the deferreturn gp.sched.sp = sp - gp.sched.pc = pc + gp.sched.pc = gotoPc gp.sched.lr = 0 // Restore the bp on platforms that support frame pointers. // N.B. It's fine to not set anything for platforms that don't @@ -1215,7 +1384,6 @@ func recovery(gp *g) { // only gets us to the caller's fp. gp.sched.bp = sp - goarch.PtrSize } - gp.sched.ret = 1 gogo(&gp.sched) } @@ -1225,8 +1393,8 @@ func recovery(gp *g) { // //go:nosplit func fatalthrow(t throwType) { - pc := getcallerpc() - sp := getcallersp() + pc := sys.GetCallerPC() + sp := sys.GetCallerSP() gp := getg() if gp.m.throwing == throwTypeNone { @@ -1242,7 +1410,7 @@ func fatalthrow(t throwType) { startpanic_m() - if dopanic_m(gp, pc, sp) { + if dopanic_m(gp, pc, sp, nil) { // crash uses a decent amount of nosplit stack and we're already // low on stack in throw, so crash on the system stack (unlike // fatalpanic). @@ -1261,8 +1429,8 @@ func fatalthrow(t throwType) { // //go:nosplit func fatalpanic(msgs *_panic) { - pc := getcallerpc() - sp := getcallersp() + pc := sys.GetCallerPC() + sp := sys.GetCallerSP() gp := getg() var docrash bool // Switch to the system stack to avoid any stack growth, which @@ -1280,7 +1448,14 @@ func fatalpanic(msgs *_panic) { printpanics(msgs) } - docrash = dopanic_m(gp, pc, sp) + // If this panic is the result of a synctest bubble deadlock, + // print stacks for the goroutines in the bubble. + var bubble *synctestBubble + if de, ok := msgs.arg.(synctestDeadlockError); ok { + bubble = de.bubble + } + + docrash = dopanic_m(gp, pc, sp, bubble) }) if docrash { @@ -1362,7 +1537,8 @@ var deadlock mutex // gp is the crashing g running on this M, but may be a user G, while getg() is // always g0. -func dopanic_m(gp *g, pc, sp uintptr) bool { +// If bubble is non-nil, print the stacks for goroutines in this group as well. +func dopanic_m(gp *g, pc, sp uintptr, bubble *synctestBubble) bool { if gp.sig != 0 { signame := signame(gp.sig) if signame != "" { @@ -1386,10 +1562,19 @@ func dopanic_m(gp *g, pc, sp uintptr) bool { print("\nruntime stack:\n") traceback(pc, sp, 0, gp) } - if !didothers && all { - didothers = true - tracebackothers(gp) + if !didothers { + if all { + didothers = true + tracebackothers(gp) + } else if bubble != nil { + // This panic is caused by a synctest bubble deadlock. + // Print stacks for goroutines in the deadlocked bubble. + tracebacksomeothers(gp, func(other *g) bool { + return bubble == other.bubble + }) + } } + } unlock(&paniclk) diff --git a/src/runtime/panic32.go b/src/runtime/panic32.go index fa3f2bf2f8bdf1..7abc9f595bdf7d 100644 --- a/src/runtime/panic32.go +++ b/src/runtime/panic32.go @@ -2,87 +2,92 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build 386 || arm || mips || mipsle +//go:build 386 || arm || mips || mipsle || (gccgo && (ppc || s390)) package runtime +import ( + "internal/abi" + "internal/runtime/sys" +) + // Additional index/slice error paths for 32-bit platforms. // Used when the high word of a 64-bit index is not zero. // failures in the comparisons for s[x], 0 <= x < y (y == len(s)) func goPanicExtendIndex(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "index out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsIndex}) + panicCheck1(sys.GetCallerPC(), "index out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: abi.BoundsIndex}) } func goPanicExtendIndexU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "index out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsIndex}) + panicCheck1(sys.GetCallerPC(), "index out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: abi.BoundsIndex}) } // failures in the comparisons for s[:x], 0 <= x <= y (y == len(s) or cap(s)) func goPanicExtendSliceAlen(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsSliceAlen}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: abi.BoundsSliceAlen}) } func goPanicExtendSliceAlenU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsSliceAlen}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: abi.BoundsSliceAlen}) } func goPanicExtendSliceAcap(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsSliceAcap}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: abi.BoundsSliceAcap}) } func goPanicExtendSliceAcapU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsSliceAcap}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: abi.BoundsSliceAcap}) } // failures in the comparisons for s[x:y], 0 <= x <= y func goPanicExtendSliceB(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsSliceB}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: abi.BoundsSliceB}) } func goPanicExtendSliceBU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsSliceB}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: abi.BoundsSliceB}) } // failures in the comparisons for s[::x], 0 <= x <= y (y == len(s) or cap(s)) func goPanicExtendSlice3Alen(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsSlice3Alen}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: abi.BoundsSlice3Alen}) } func goPanicExtendSlice3AlenU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsSlice3Alen}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: abi.BoundsSlice3Alen}) } func goPanicExtendSlice3Acap(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsSlice3Acap}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: abi.BoundsSlice3Acap}) } func goPanicExtendSlice3AcapU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsSlice3Acap}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: abi.BoundsSlice3Acap}) } // failures in the comparisons for s[:x:y], 0 <= x <= y func goPanicExtendSlice3B(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsSlice3B}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: abi.BoundsSlice3B}) } func goPanicExtendSlice3BU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsSlice3B}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: abi.BoundsSlice3B}) } // failures in the comparisons for s[x:y:], 0 <= x <= y func goPanicExtendSlice3C(hi int, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: boundsSlice3C}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: true, y: y, code: abi.BoundsSlice3C}) } func goPanicExtendSlice3CU(hi uint, lo uint, y int) { - panicCheck1(getcallerpc(), "slice bounds out of range") - panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: boundsSlice3C}) + panicCheck1(sys.GetCallerPC(), "slice bounds out of range") + panic(boundsError{x: int64(hi)<<32 + int64(lo), signed: false, y: y, code: abi.BoundsSlice3C}) } // Implemented in assembly, as they take arguments in registers. diff --git a/src/runtime/pinner.go b/src/runtime/pinner.go index 7a9c3815803b39..424dd065efd577 100644 --- a/src/runtime/pinner.go +++ b/src/runtime/pinner.go @@ -108,7 +108,7 @@ func pinnerGetPtr(i *any) unsafe.Pointer { if etyp == nil { panic(errorString("runtime.Pinner: argument is nil")) } - if kind := etyp.Kind_ & abi.KindMask; kind != abi.Pointer && kind != abi.UnsafePointer { + if kind := etyp.Kind(); kind != abi.Pointer && kind != abi.UnsafePointer { panic(errorString("runtime.Pinner: argument is not a pointer: " + toRType(etyp).string())) } if inUserArenaChunk(uintptr(e.data)) { @@ -331,7 +331,7 @@ func (span *mspan) incPinCounter(offset uintptr) { rec = (*specialPinCounter)(mheap_.specialPinCounterAlloc.alloc()) unlock(&mheap_.speciallock) // splice in record, fill in offset. - rec.special.offset = uint16(offset) + rec.special.offset = offset rec.special.kind = _KindSpecialPinCounter rec.special.next = *ref *ref = (*special)(unsafe.Pointer(rec)) diff --git a/src/runtime/plugin.go b/src/runtime/plugin.go index 4b6821b1fbbc9b..49cf13cb64673b 100644 --- a/src/runtime/plugin.go +++ b/src/runtime/plugin.go @@ -88,7 +88,7 @@ func plugin_lastmoduleinit() (path string, syms map[string]any, initTasks []*ini (*valp)[0] = unsafe.Pointer(t) name := symName.Name() - if t.Kind_&abi.KindMask == abi.Func { + if t.Kind() == abi.Func { name = "." + name } syms[name] = val diff --git a/src/runtime/pprof/label.go b/src/runtime/pprof/label.go index 41eece2f74c0a3..4c1d8d38ce5ea2 100644 --- a/src/runtime/pprof/label.go +++ b/src/runtime/pprof/label.go @@ -27,7 +27,7 @@ type labelContextKey struct{} func labelValue(ctx context.Context) labelMap { labels, _ := ctx.Value(labelContextKey{}).(*labelMap) if labels == nil { - return labelMap(nil) + return labelMap{} } return *labels } @@ -35,7 +35,9 @@ func labelValue(ctx context.Context) labelMap { // labelMap is the representation of the label set held in the context type. // This is an initial implementation, but it will be replaced with something // that admits incremental immutable modification more efficiently. -type labelMap map[string]string +type labelMap struct { + LabelSet +} // String satisfies Stringer and returns key, value pairs in a consistent // order. @@ -43,14 +45,13 @@ func (l *labelMap) String() string { if l == nil { return "" } - keyVals := make([]string, 0, len(*l)) + keyVals := make([]string, 0, len(l.list)) - for k, v := range *l { - keyVals = append(keyVals, fmt.Sprintf("%q:%q", k, v)) + for _, lbl := range l.list { + keyVals = append(keyVals, fmt.Sprintf("%q:%q", lbl.key, lbl.value)) } slices.Sort(keyVals) - return "{" + strings.Join(keyVals, ", ") + "}" } @@ -58,17 +59,38 @@ func (l *labelMap) String() string { // A label overwrites a prior label with the same key. func WithLabels(ctx context.Context, labels LabelSet) context.Context { parentLabels := labelValue(ctx) - childLabels := make(labelMap, len(parentLabels)) - // TODO(matloob): replace the map implementation with something - // more efficient so creating a child context WithLabels doesn't need - // to clone the map. - for k, v := range parentLabels { - childLabels[k] = v + return context.WithValue(ctx, labelContextKey{}, &labelMap{mergeLabelSets(parentLabels.LabelSet, labels)}) +} + +func mergeLabelSets(left, right LabelSet) LabelSet { + if len(left.list) == 0 { + return right + } else if len(right.list) == 0 { + return left } - for _, label := range labels.list { - childLabels[label.key] = label.value + + l, r := 0, 0 + result := make([]label, 0, len(right.list)) + for l < len(left.list) && r < len(right.list) { + switch strings.Compare(left.list[l].key, right.list[r].key) { + case -1: // left key < right key + result = append(result, left.list[l]) + l++ + case 1: // right key < left key + result = append(result, right.list[r]) + r++ + case 0: // keys are equal, right value overwrites left value + result = append(result, right.list[r]) + l++ + r++ + } } - return context.WithValue(ctx, labelContextKey{}, &childLabels) + + // Append the remaining elements + result = append(result, left.list[l:]...) + result = append(result, right.list[r:]...) + + return LabelSet{list: result} } // Labels takes an even number of strings representing key-value pairs @@ -82,8 +104,25 @@ func Labels(args ...string) LabelSet { panic("uneven number of arguments to pprof.Labels") } list := make([]label, 0, len(args)/2) + sortedNoDupes := true for i := 0; i+1 < len(args); i += 2 { list = append(list, label{key: args[i], value: args[i+1]}) + sortedNoDupes = sortedNoDupes && (i < 2 || args[i] > args[i-2]) + } + if !sortedNoDupes { + // slow path: keys are unsorted, contain duplicates, or both + slices.SortStableFunc(list, func(a, b label) int { + return strings.Compare(a.key, b.key) + }) + deduped := make([]label, 0, len(list)) + for i, lbl := range list { + if i == 0 || lbl.key != list[i-1].key { + deduped = append(deduped, lbl) + } else { + deduped[len(deduped)-1] = lbl + } + } + list = deduped } return LabelSet{list: list} } @@ -92,16 +131,20 @@ func Labels(args ...string) LabelSet { // whether that label exists. func Label(ctx context.Context, key string) (string, bool) { ctxLabels := labelValue(ctx) - v, ok := ctxLabels[key] - return v, ok + for _, lbl := range ctxLabels.list { + if lbl.key == key { + return lbl.value, true + } + } + return "", false } // ForLabels invokes f with each label set on the context. // The function f should return true to continue iteration or false to stop iteration early. func ForLabels(ctx context.Context, f func(key, value string) bool) { ctxLabels := labelValue(ctx) - for k, v := range ctxLabels { - if !f(k, v) { + for _, lbl := range ctxLabels.list { + if !f(lbl.key, lbl.value) { break } } diff --git a/src/runtime/pprof/label_test.go b/src/runtime/pprof/label_test.go index 38d9e80dfc2a58..3018693c247453 100644 --- a/src/runtime/pprof/label_test.go +++ b/src/runtime/pprof/label_test.go @@ -6,6 +6,7 @@ package pprof import ( "context" + "fmt" "reflect" "slices" "strings" @@ -92,16 +93,18 @@ func TestLabelMapStringer(t *testing.T) { expected: "{}", }, { m: labelMap{ - "foo": "bar", + Labels("foo", "bar"), }, expected: `{"foo":"bar"}`, }, { m: labelMap{ - "foo": "bar", - "key1": "value1", - "key2": "value2", - "key3": "value3", - "key4WithNewline": "\nvalue4", + Labels( + "foo", "bar", + "key1", "value1", + "key2", "value2", + "key3", "value3", + "key4WithNewline", "\nvalue4", + ), }, expected: `{"foo":"bar", "key1":"value1", "key2":"value2", "key3":"value3", "key4WithNewline":"\nvalue4"}`, }, @@ -111,3 +114,75 @@ func TestLabelMapStringer(t *testing.T) { } } } + +func BenchmarkLabels(b *testing.B) { + b.Run("set-one", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + Do(context.Background(), Labels("key", "value"), func(context.Context) {}) + } + }) + + b.Run("merge-one", func(b *testing.B) { + ctx := WithLabels(context.Background(), Labels("key1", "val1")) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + Do(ctx, Labels("key2", "value2"), func(context.Context) {}) + } + }) + + b.Run("overwrite-one", func(b *testing.B) { + ctx := WithLabels(context.Background(), Labels("key", "val")) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + Do(ctx, Labels("key", "value"), func(context.Context) {}) + } + }) + + for _, scenario := range []string{"ordered", "unordered"} { + var labels []string + for i := 0; i < 10; i++ { + labels = append(labels, fmt.Sprintf("key%03d", i), fmt.Sprintf("value%03d", i)) + } + if scenario == "unordered" { + labels[0], labels[len(labels)-1] = labels[len(labels)-1], labels[0] + } + + b.Run(scenario, func(b *testing.B) { + b.Run("set-many", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + Do(context.Background(), Labels(labels...), func(context.Context) {}) + } + }) + + b.Run("merge-many", func(b *testing.B) { + ctx := WithLabels(context.Background(), Labels(labels[:len(labels)/2]...)) + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + Do(ctx, Labels(labels[len(labels)/2:]...), func(context.Context) {}) + } + }) + + b.Run("overwrite-many", func(b *testing.B) { + ctx := WithLabels(context.Background(), Labels(labels...)) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + Do(ctx, Labels(labels...), func(context.Context) {}) + } + }) + }) + } + + // TODO: hit slow path in Labels +} diff --git a/src/runtime/pprof/mprof_test.go b/src/runtime/pprof/mprof_test.go index 391588d4acd0ec..456ba9044498af 100644 --- a/src/runtime/pprof/mprof_test.go +++ b/src/runtime/pprof/mprof_test.go @@ -9,6 +9,7 @@ package pprof import ( "bytes" "fmt" + "internal/asan" "internal/profile" "reflect" "regexp" @@ -63,6 +64,10 @@ func allocateReflect() { var memoryProfilerRun = 0 func TestMemoryProfiler(t *testing.T) { + if asan.Enabled { + t.Skip("extra allocations with -asan throw off the test; see #70079") + } + // Disable sampling, otherwise it's difficult to assert anything. oldRate := runtime.MemProfileRate runtime.MemProfileRate = 1 @@ -92,32 +97,32 @@ func TestMemoryProfiler(t *testing.T) { legacy string }{{ stk: []string{"runtime/pprof.allocatePersistent1K", "runtime/pprof.TestMemoryProfiler"}, - legacy: fmt.Sprintf(`%v: %v \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ -# 0x[0-9,a-f]+ runtime/pprof\.allocatePersistent1K\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test\.go:47 -# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test\.go:82 + legacy: fmt.Sprintf(`%v: %v \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+( 0x[0-9,a-f]+ 0x[0-9,a-f]+)? +# 0x[0-9,a-f]+ runtime/pprof\.allocatePersistent1K\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test\.go:48 +# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test\.go:87 `, 32*memoryProfilerRun, 1024*memoryProfilerRun, 32*memoryProfilerRun, 1024*memoryProfilerRun), }, { stk: []string{"runtime/pprof.allocateTransient1M", "runtime/pprof.TestMemoryProfiler"}, - legacy: fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ -# 0x[0-9,a-f]+ runtime/pprof\.allocateTransient1M\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:24 -# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:79 + legacy: fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ +# 0x[0-9,a-f]+ runtime/pprof\.allocateTransient1M\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:25 +# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:84 `, (1<<10)*memoryProfilerRun, (1<<20)*memoryProfilerRun), }, { stk: []string{"runtime/pprof.allocateTransient2M", "runtime/pprof.TestMemoryProfiler"}, - legacy: fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ -# 0x[0-9,a-f]+ runtime/pprof\.allocateTransient2M\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:30 -# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:80 + legacy: fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ +# 0x[0-9,a-f]+ runtime/pprof\.allocateTransient2M\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:31 +# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:85 `, memoryProfilerRun, (2<<20)*memoryProfilerRun), }, { stk: []string{"runtime/pprof.allocateTransient2MInline", "runtime/pprof.TestMemoryProfiler"}, - legacy: fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ -# 0x[0-9,a-f]+ runtime/pprof\.allocateTransient2MInline\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:34 -# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:81 + legacy: fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ +# 0x[0-9,a-f]+ runtime/pprof\.allocateTransient2MInline\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:35 +# 0x[0-9,a-f]+ runtime/pprof\.TestMemoryProfiler\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:86 `, memoryProfilerRun, (2<<20)*memoryProfilerRun), }, { stk: []string{"runtime/pprof.allocateReflectTransient"}, legacy: fmt.Sprintf(`0: 0 \[%v: %v\] @( 0x[0-9,a-f]+)+ -# 0x[0-9,a-f]+ runtime/pprof\.allocateReflectTransient\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:55 +# 0x[0-9,a-f]+ runtime/pprof\.allocateReflectTransient\+0x[0-9,a-f]+ .*runtime/pprof/mprof_test.go:56 `, memoryProfilerRun, (2<<20)*memoryProfilerRun), }} @@ -145,7 +150,7 @@ func TestMemoryProfiler(t *testing.T) { } t.Logf("Profile = %v", p) - stks := stacks(p) + stks := profileStacks(p) for _, test := range tests { if !containsStack(stks, test.stk) { t.Fatalf("No matching stack entry for %q\n\nProfile:\n%v\n", test.stk, p) diff --git a/src/runtime/pprof/pprof.go b/src/runtime/pprof/pprof.go index 4b7a9f63c65006..b524e992b8b209 100644 --- a/src/runtime/pprof/pprof.go +++ b/src/runtime/pprof/pprof.go @@ -44,7 +44,10 @@ // } // defer f.Close() // error handling omitted for example // runtime.GC() // get up-to-date statistics -// if err := pprof.WriteHeapProfile(f); err != nil { +// // Lookup("allocs") creates a profile similar to go test -memprofile. +// // Alternatively, use Lookup("heap") for a profile +// // that has inuse_space as the default index. +// if err := pprof.Lookup("allocs").WriteTo(f, 0); err != nil { // log.Fatal("could not write memory profile: ", err) // } // } @@ -77,6 +80,7 @@ import ( "cmp" "fmt" "internal/abi" + "internal/goexperiment" "internal/profilerecord" "io" "runtime" @@ -102,12 +106,13 @@ import ( // // Each Profile has a unique name. A few profiles are predefined: // -// goroutine - stack traces of all current goroutines -// heap - a sampling of memory allocations of live objects -// allocs - a sampling of all past memory allocations -// threadcreate - stack traces that led to the creation of new OS threads -// block - stack traces that led to blocking on synchronization primitives -// mutex - stack traces of holders of contended mutexes +// goroutine - stack traces of all current goroutines +// goroutineleak - stack traces of all leaked goroutines +// allocs - a sampling of all past memory allocations +// heap - a sampling of memory allocations of live objects +// threadcreate - stack traces that led to the creation of new OS threads +// block - stack traces that led to blocking on synchronization primitives +// mutex - stack traces of holders of contended mutexes // // These predefined profiles maintain themselves and panic on an explicit // [Profile.Add] or [Profile.Remove] method call. @@ -166,12 +171,7 @@ import ( // holds a lock for 1s while 5 other goroutines are waiting for the entire // second to acquire the lock, its unlock call stack will report 5s of // contention. -// -// Runtime-internal locks are always reported at the location -// "runtime._LostContendedRuntimeLock". More detailed stack traces for -// runtime-internal locks can be obtained by setting -// `GODEBUG=runtimecontentionstacks=1` (see package [runtime] docs for -// caveats). + type Profile struct { name string mu sync.Mutex @@ -192,6 +192,12 @@ var goroutineProfile = &Profile{ write: writeGoroutine, } +var goroutineLeakProfile = &Profile{ + name: "goroutineleak", + count: runtime_goroutineleakcount, + write: writeGoroutineLeak, +} + var threadcreateProfile = &Profile{ name: "threadcreate", count: countThreadCreate, @@ -234,6 +240,9 @@ func lockProfiles() { "block": blockProfile, "mutex": mutexProfile, } + if goexperiment.GoroutineLeakProfile { + profiles.m["goroutineleak"] = goroutineLeakProfile + } } } @@ -278,6 +287,7 @@ func Profiles() []*Profile { all := make([]*Profile, 0, len(profiles.m)) for _, p := range profiles.m { + all = append(all, p) } @@ -449,8 +459,7 @@ func printCountCycleProfile(w io.Writer, countName, cycleName string, records [] locs = b.appendLocsForStack(locs[:0], expandedStack[:n]) b.pbSample(values, locs, nil) } - b.build() - return nil + return b.build() } // printCountProfile prints a countProfile at the specified debug level. @@ -513,15 +522,14 @@ func printCountProfile(w io.Writer, debug int, name string, p countProfile) erro var labels func() if p.Label(idx) != nil { labels = func() { - for k, v := range *p.Label(idx) { - b.pbLabel(tagSample_Label, k, v, 0) + for _, lbl := range p.Label(idx).list { + b.pbLabel(tagSample_Label, lbl.key, lbl.value, 0) } } } b.pbSample(values, locs, labels) } - b.build() - return nil + return b.build() } // keysByCount sorts keys with higher counts first, breaking ties by key string order. @@ -552,7 +560,7 @@ func printStackRecord(w io.Writer, stk []uintptr, allFrames bool) { if name == "" { show = true fmt.Fprintf(w, "#\t%#x\n", frame.PC) - } else if name != "runtime.goexit" && (show || !strings.HasPrefix(name, "runtime.")) { + } else if name != "runtime.goexit" && (show || !(strings.HasPrefix(name, "runtime.") || strings.HasPrefix(name, "internal/runtime/"))) { // Hide runtime.goexit and any runtime functions at the beginning. // This is useful mainly for allocation traces. show = true @@ -752,6 +760,23 @@ func writeGoroutine(w io.Writer, debug int) error { return writeRuntimeProfile(w, debug, "goroutine", pprof_goroutineProfileWithLabels) } +// writeGoroutineLeak first invokes a GC cycle that performs goroutine leak detection. +// It then writes the goroutine profile, filtering for leaked goroutines. +func writeGoroutineLeak(w io.Writer, debug int) error { + // Run the GC with leak detection first so that leaked goroutines + // may transition to the leaked state. + runtime_goroutineLeakGC() + + // If the debug flag is set sufficiently high, just defer to writing goroutine stacks + // like in a regular goroutine profile. Include non-leaked goroutines, too. + if debug >= 2 { + return writeGoroutineStacks(w) + } + + // Otherwise, write the goroutine leak profile. + return writeRuntimeProfile(w, debug, "goroutineleak", pprof_goroutineLeakProfileWithLabels) +} + func writeGoroutineStacks(w io.Writer) error { // We don't know how big the buffer needs to be to collect // all the goroutines. Start with 1 MB and try a few times, doubling each time. @@ -974,6 +999,9 @@ func writeProfileInternal(w io.Writer, debug int, name string, runtimeProfile fu //go:linkname pprof_goroutineProfileWithLabels runtime.pprof_goroutineProfileWithLabels func pprof_goroutineProfileWithLabels(p []profilerecord.StackRecord, labels []unsafe.Pointer) (n int, ok bool) +//go:linkname pprof_goroutineLeakProfileWithLabels runtime.pprof_goroutineLeakProfileWithLabels +func pprof_goroutineLeakProfileWithLabels(p []profilerecord.StackRecord, labels []unsafe.Pointer) (n int, ok bool) + //go:linkname pprof_cyclesPerSecond runtime/pprof.runtime_cyclesPerSecond func pprof_cyclesPerSecond() int64 diff --git a/src/runtime/pprof/pprof_test.go b/src/runtime/pprof/pprof_test.go index 30ef50b1c0fa7a..b816833e52285f 100644 --- a/src/runtime/pprof/pprof_test.go +++ b/src/runtime/pprof/pprof_test.go @@ -15,12 +15,14 @@ import ( "internal/syscall/unix" "internal/testenv" "io" + "iter" "math" "math/big" "os" "regexp" "runtime" "runtime/debug" + "slices" "strconv" "strings" "sync" @@ -115,10 +117,6 @@ func TestCPUProfileMultithreadMagnitude(t *testing.T) { t.Skip("issue 35057 is only confirmed on Linux") } - // Linux [5.9,5.16) has a kernel bug that can break CPU timers on newly - // created threads, breaking our CPU accounting. - major, minor := unix.KernelVersion() - t.Logf("Running on Linux %d.%d", major, minor) defer func() { if t.Failed() { t.Logf("Failure of this test may indicate that your system suffers from a known Linux kernel bug fixed on newer kernels. See https://golang.org/issue/49065.") @@ -129,9 +127,9 @@ func TestCPUProfileMultithreadMagnitude(t *testing.T) { // it enabled to potentially warn users that they are on a broken // kernel. if testenv.Builder() != "" && (runtime.GOARCH == "386" || runtime.GOARCH == "amd64") { - have59 := major > 5 || (major == 5 && minor >= 9) - have516 := major > 5 || (major == 5 && minor >= 16) - if have59 && !have516 { + // Linux [5.9,5.16) has a kernel bug that can break CPU timers on newly + // created threads, breaking our CPU accounting. + if unix.KernelVersionGE(5, 9) && !unix.KernelVersionGE(5, 16) { testenv.SkipFlaky(t, 49065) } } @@ -346,6 +344,11 @@ func (h inlineWrapper) dump(pcs []uintptr) { func inlinedWrapperCallerDump(pcs []uintptr) { var h inlineWrapperInterface + + // Take the address of h, such that h.dump() call (below) + // does not get devirtualized by the compiler. + _ = &h + h = &inlineWrapper{} h.dump(pcs) } @@ -414,27 +417,6 @@ func parseProfile(t *testing.T, valBytes []byte, f func(uintptr, []*profile.Loca return p } -func cpuProfilingBroken() bool { - switch runtime.GOOS { - case "plan9": - // Profiling unimplemented. - return true - case "aix": - // See https://golang.org/issue/45170. - return true - case "ios", "dragonfly", "netbsd", "illumos", "solaris": - // See https://golang.org/issue/13841. - return true - case "openbsd": - if runtime.GOARCH == "arm" || runtime.GOARCH == "arm64" { - // See https://golang.org/issue/13841. - return true - } - } - - return false -} - // testCPUProfile runs f under the CPU profiler, checking for some conditions specified by need, // as interpreted by matches, and returns the parsed profile. func testCPUProfile(t *testing.T, matches profileMatchFunc, f func(dur time.Duration)) *profile.Profile { @@ -452,7 +434,7 @@ func testCPUProfile(t *testing.T, matches profileMatchFunc, f func(dur time.Dura t.Skip("skipping on wasip1") } - broken := cpuProfilingBroken() + broken := testenv.CPUProfilingBroken() deadline, ok := t.Deadline() if broken || !ok { @@ -484,7 +466,7 @@ func testCPUProfile(t *testing.T, matches profileMatchFunc, f func(dur time.Dura f(duration) StopCPUProfile() - if p, ok := profileOk(t, matches, prof, duration); ok { + if p, ok := profileOk(t, matches, &prof, duration); ok { return p } @@ -520,15 +502,6 @@ func diffCPUTime(t *testing.T, f func()) (user, system time.Duration) { return 0, 0 } -func contains(slice []string, s string) bool { - for i := range slice { - if slice[i] == s { - return true - } - } - return false -} - // stackContains matches if a function named spec appears anywhere in the stack trace. func stackContains(spec string, count uintptr, stk []*profile.Location, labels map[string][]string) bool { for _, loc := range stk { @@ -543,7 +516,7 @@ func stackContains(spec string, count uintptr, stk []*profile.Location, labels m type sampleMatchFunc func(spec string, count uintptr, stk []*profile.Location, labels map[string][]string) bool -func profileOk(t *testing.T, matches profileMatchFunc, prof bytes.Buffer, duration time.Duration) (_ *profile.Profile, ok bool) { +func profileOk(t *testing.T, matches profileMatchFunc, prof *bytes.Buffer, duration time.Duration) (_ *profile.Profile, ok bool) { ok = true var samples uintptr @@ -663,10 +636,6 @@ func TestCPUProfileWithFork(t *testing.T) { // Use smaller size for Android to avoid crash. heap = 100 << 20 } - if runtime.GOOS == "windows" && runtime.GOARCH == "arm" { - // Use smaller heap for Windows/ARM to avoid crash. - heap = 100 << 20 - } if testing.Short() { heap = 100 << 20 } @@ -981,7 +950,7 @@ func TestBlockProfile(t *testing.T) { t.Fatalf("invalid profile: %v", err) } - stks := stacks(p) + stks := profileStacks(p) for _, test := range tests { if !containsStack(stks, test.stk) { t.Errorf("No matching stack entry for %v, want %+v", test.name, test.stk) @@ -991,7 +960,7 @@ func TestBlockProfile(t *testing.T) { } -func stacks(p *profile.Profile) (res [][]string) { +func profileStacks(p *profile.Profile) (res [][]string) { for _, s := range p.Sample { var stk []string for _, l := range s.Location { @@ -1004,6 +973,22 @@ func stacks(p *profile.Profile) (res [][]string) { return res } +func blockRecordStacks(records []runtime.BlockProfileRecord) (res [][]string) { + for _, record := range records { + frames := runtime.CallersFrames(record.Stack()) + var stk []string + for { + frame, more := frames.Next() + stk = append(stk, frame.Function) + if !more { + break + } + } + res = append(res, stk) + } + return res +} + func containsStack(got [][]string, want []string) bool { for _, stk := range got { if len(stk) < len(want) { @@ -1220,7 +1205,7 @@ func blockFrequentShort(rate int) { } } -// blockFrequentShort produces 10000 block events with an average duration of +// blockInfrequentLong produces 10000 block events with an average duration of // rate. func blockInfrequentLong(rate int) { for i := 0; i < 10000; i++ { @@ -1288,7 +1273,7 @@ func TestMutexProfile(t *testing.T) { t.Fatalf("invalid profile: %v", err) } - stks := stacks(p) + stks := profileStacks(p) for _, want := range [][]string{ {"sync.(*Mutex).Unlock", "runtime/pprof.blockMutexN.func1"}, } { @@ -1328,6 +1313,28 @@ func TestMutexProfile(t *testing.T) { t.Fatalf("profile samples total %v, want within range [%v, %v] (target: %v)", d, lo, hi, N*D) } }) + + t.Run("records", func(t *testing.T) { + // Record a mutex profile using the structured record API. + var records []runtime.BlockProfileRecord + for { + n, ok := runtime.MutexProfile(records) + if ok { + records = records[:n] + break + } + records = make([]runtime.BlockProfileRecord, n*2) + } + + // Check that we see the same stack trace as the proto profile. For + // historical reason we expect a runtime.goexit root frame here that is + // omitted in the proto profile. + stks := blockRecordStacks(records) + want := []string{"sync.(*Mutex).Unlock", "runtime/pprof.blockMutexN.func1", "runtime.goexit"} + if !containsStack(stks, want) { + t.Errorf("No matching stack entry for %+v", want) + } + }) } func TestMutexProfileRateAdjust(t *testing.T) { @@ -1371,7 +1378,7 @@ func TestMutexProfileRateAdjust(t *testing.T) { blockMutex(t) contentions, delay := readProfile() - if contentions == 0 || delay == 0 { + if contentions == 0 { // low-resolution timers can have delay of 0 in mutex profile t.Fatal("did not see expected function in profile") } runtime.SetMutexProfileFraction(0) @@ -1451,11 +1458,11 @@ func TestGoroutineCounts(t *testing.T) { goroutineProf.WriteTo(&w, 1) prof := w.String() - labels := labelMap{"label": "value"} + labels := labelMap{Labels("label", "value")} labelStr := "\n# labels: " + labels.String() - selfLabel := labelMap{"self-label": "self-value"} + selfLabel := labelMap{Labels("self-label", "self-value")} selfLabelStr := "\n# labels: " + selfLabel.String() - fingLabel := labelMap{"fing-label": "fing-value"} + fingLabel := labelMap{Labels("fing-label", "fing-value")} fingLabelStr := "\n# labels: " + fingLabel.String() orderedPrefix := []string{ "\n50 @ ", @@ -1567,8 +1574,8 @@ func TestGoroutineProfileConcurrency(t *testing.T) { return strings.Count(s, "\truntime/pprof.runtime_goroutineProfileWithLabels+") } - includesFinalizer := func(s string) bool { - return strings.Contains(s, "runtime.runfinq") + includesFinalizerOrCleanup := func(s string) bool { + return strings.Contains(s, "runtime.runFinalizers") || strings.Contains(s, "runtime.runCleanups") } // Concurrent calls to the goroutine profiler should not trigger data races @@ -1606,8 +1613,8 @@ func TestGoroutineProfileConcurrency(t *testing.T) { var w strings.Builder goroutineProf.WriteTo(&w, 1) prof := w.String() - if includesFinalizer(prof) { - t.Errorf("profile includes finalizer (but finalizer should be marked as system):\n%s", prof) + if includesFinalizerOrCleanup(prof) { + t.Errorf("profile includes finalizer or cleanup (but should be marked as system):\n%s", prof) } }) @@ -1638,7 +1645,7 @@ func TestGoroutineProfileConcurrency(t *testing.T) { var w strings.Builder goroutineProf.WriteTo(&w, 1) prof := w.String() - if !includesFinalizer(prof) { + if !includesFinalizerOrCleanup(prof) { t.Errorf("profile does not include finalizer (and it should be marked as user):\n%s", prof) } }) @@ -1754,6 +1761,89 @@ func TestGoroutineProfileConcurrency(t *testing.T) { } } +// Regression test for #69998. +func TestGoroutineProfileCoro(t *testing.T) { + testenv.MustHaveParallelism(t) + + goroutineProf := Lookup("goroutine") + + // Set up a goroutine to just create and run coroutine goroutines all day. + iterFunc := func() { + p, stop := iter.Pull2( + func(yield func(int, int) bool) { + for i := 0; i < 10000; i++ { + if !yield(i, i) { + return + } + } + }, + ) + defer stop() + for { + _, _, ok := p() + if !ok { + break + } + } + } + var wg sync.WaitGroup + done := make(chan struct{}) + wg.Add(1) + go func() { + defer wg.Done() + for { + iterFunc() + select { + case <-done: + default: + } + } + }() + + // Take a goroutine profile. If the bug in #69998 is present, this will crash + // with high probability. We don't care about the output for this bug. + goroutineProf.WriteTo(io.Discard, 1) +} + +// This test tries to provoke a situation wherein the finalizer goroutine is +// erroneously inspected by the goroutine profiler in such a way that could +// cause a crash. See go.dev/issue/74090. +func TestGoroutineProfileIssue74090(t *testing.T) { + testenv.MustHaveParallelism(t) + + goroutineProf := Lookup("goroutine") + + // T is a pointer type so it won't be allocated by the tiny + // allocator, which can lead to its finalizer not being called + // during this test. + type T *byte + for range 10 { + // We use finalizers for this test because finalizers transition between + // system and user goroutine on each call, since there's substantially + // more work to do to set up a finalizer call. Cleanups, on the other hand, + // transition once for a whole batch, and so are less likely to trigger + // the failure. Under stress testing conditions this test fails approximately + // 5 times every 1000 executions on a 64 core machine without the appropriate + // fix, which is not ideal but if this test crashes at all, it's a clear + // signal that something is broken. + var objs []*T + for range 10000 { + obj := new(T) + runtime.SetFinalizer(obj, func(_ interface{}) {}) + objs = append(objs, obj) + } + objs = nil + + // Queue up all the finalizers. + runtime.GC() + + // Try to run a goroutine profile concurrently with finalizer execution + // to trigger the bug. + var w strings.Builder + goroutineProf.WriteTo(&w, 1) + } +} + func BenchmarkGoroutine(b *testing.B) { withIdle := func(n int, fn func(b *testing.B)) func(b *testing.B) { return func(b *testing.B) { @@ -1877,7 +1967,7 @@ func stackContainsLabeled(spec string, count uintptr, stk []*profile.Location, l if !ok { panic("missing = in key/value spec") } - if !contains(labels[k], v) { + if !slices.Contains(labels[k], v) { return false } return stackContains(base, count, stk, labels) @@ -1994,7 +2084,7 @@ func TestLabelSystemstack(t *testing.T) { // * labelHog should always be labeled. // * The label should _only_ appear on labelHog and the Do call above. for _, s := range p.Sample { - isLabeled := s.Label != nil && contains(s.Label["key"], "value") + isLabeled := s.Label != nil && slices.Contains(s.Label["key"], "value") var ( mayBeLabeled bool mustBeLabeled string @@ -2011,7 +2101,7 @@ func TestLabelSystemstack(t *testing.T) { // which part of the function they are // at. mayBeLabeled = true - case "runtime.bgsweep", "runtime.bgscavenge", "runtime.forcegchelper", "runtime.gcBgMarkWorker", "runtime.runfinq", "runtime.sysmon": + case "runtime.bgsweep", "runtime.bgscavenge", "runtime.forcegchelper", "runtime.gcBgMarkWorker", "runtime.runFinalizers", "runtime.runCleanups", "runtime.sysmon": // Runtime system goroutines or threads // (such as those identified by // runtime.isSystemGoroutine). These @@ -2441,16 +2531,7 @@ func TestTimeVDSO(t *testing.T) { } func TestProfilerStackDepth(t *testing.T) { - // Disable sampling, otherwise it's difficult to assert anything. - oldMemRate := runtime.MemProfileRate - runtime.MemProfileRate = 1 - runtime.SetBlockProfileRate(1) - oldMutexRate := runtime.SetMutexProfileFraction(1) - t.Cleanup(func() { - runtime.MemProfileRate = oldMemRate - runtime.SetBlockProfileRate(0) - runtime.SetMutexProfileFraction(oldMutexRate) - }) + t.Cleanup(disableSampling()) const depth = 128 go produceProfileEvents(t, depth) @@ -2478,35 +2559,43 @@ func TestProfilerStackDepth(t *testing.T) { } t.Logf("Profile = %v", p) - stks := stacks(p) - var stk []string - for _, s := range stks { - if hasPrefix(s, test.prefix) { - stk = s - break + stks := profileStacks(p) + var matchedStacks [][]string + for _, stk := range stks { + if !hasPrefix(stk, test.prefix) { + continue } + // We may get multiple stacks which contain the prefix we want, but + // which might not have enough frames, e.g. if the profiler hides + // some leaf frames that would count against the stack depth limit. + // Check for at least one match + matchedStacks = append(matchedStacks, stk) + if len(stk) != depth { + continue + } + if rootFn, wantFn := stk[depth-1], "runtime/pprof.produceProfileEvents"; rootFn != wantFn { + continue + } + // Found what we wanted + return } - if len(stk) != depth { - t.Fatalf("want stack depth = %d, got %d", depth, len(stk)) - } + for _, stk := range matchedStacks { + t.Logf("matched stack=%s", stk) + if len(stk) != depth { + t.Errorf("want stack depth = %d, got %d", depth, len(stk)) + continue + } - if rootFn, wantFn := stk[depth-1], "runtime/pprof.produceProfileEvents"; rootFn != wantFn { - t.Fatalf("want stack stack root %s, got %v", wantFn, rootFn) + if rootFn, wantFn := stk[depth-1], "runtime/pprof.allocDeep"; rootFn != wantFn { + t.Errorf("want stack stack root %s, got %v", wantFn, rootFn) + } } }) } } func hasPrefix(stk []string, prefix []string) bool { - if len(prefix) > len(stk) { - return false - } - for i := range prefix { - if stk[i] != prefix[i] { - return false - } - } - return true + return len(prefix) <= len(stk) && slices.Equal(stk[:len(prefix)], prefix) } // ensure that stack records are valid map keys (comparable) @@ -2580,7 +2669,7 @@ func produceProfileEvents(t *testing.T, depth int) { goroutineDeep(t, depth-4) // -4 for produceProfileEvents, **, chanrecv1, chanrev, gopark } -func getProfileStacks(collect func([]runtime.BlockProfileRecord) (int, bool), fileLine bool) []string { +func getProfileStacks(collect func([]runtime.BlockProfileRecord) (int, bool), fileLine bool, pcs bool) []string { var n int var ok bool var p []runtime.BlockProfileRecord @@ -2599,6 +2688,9 @@ func getProfileStacks(collect func([]runtime.BlockProfileRecord) (int, bool), fi if i > 0 { stack.WriteByte('\n') } + if pcs { + fmt.Fprintf(&stack, "%x ", pc) + } // Use FuncForPC instead of CallersFrames, // because we want to see the info for exactly // the PCs returned by the mutex profile to @@ -2639,9 +2731,9 @@ func TestMutexBlockFullAggregation(t *testing.T) { wg := sync.WaitGroup{} wg.Add(workers) - for j := 0; j < workers; j++ { + for range workers { go func() { - for i := 0; i < iters; i++ { + for range iters { m.Lock() // Wait at least 1 millisecond to pass the // starvation threshold for the mutex @@ -2654,7 +2746,7 @@ func TestMutexBlockFullAggregation(t *testing.T) { wg.Wait() assertNoDuplicates := func(name string, collect func([]runtime.BlockProfileRecord) (int, bool)) { - stacks := getProfileStacks(collect, true) + stacks := getProfileStacks(collect, true, true) seen := make(map[string]struct{}) for _, s := range stacks { if _, ok := seen[s]; ok { @@ -2730,7 +2822,7 @@ runtime/pprof.inlineA`, for _, tc := range tcs { t.Run(tc.Name, func(t *testing.T) { - stacks := getProfileStacks(tc.Collect, false) + stacks := getProfileStacks(tc.Collect, false, false) for _, s := range stacks { if strings.Contains(s, tc.SubStack) { return @@ -2742,3 +2834,84 @@ runtime/pprof.inlineA`, }) } } + +func TestProfileRecordNullPadding(t *testing.T) { + // Produce events for the different profile types. + t.Cleanup(disableSampling()) + memSink = make([]byte, 1) // MemProfile + <-time.After(time.Millisecond) // BlockProfile + blockMutex(t) // MutexProfile + runtime.GC() + + // Test that all profile records are null padded. + testProfileRecordNullPadding(t, "MutexProfile", runtime.MutexProfile) + testProfileRecordNullPadding(t, "GoroutineProfile", runtime.GoroutineProfile) + testProfileRecordNullPadding(t, "BlockProfile", runtime.BlockProfile) + testProfileRecordNullPadding(t, "MemProfile/inUseZero=true", func(p []runtime.MemProfileRecord) (int, bool) { + return runtime.MemProfile(p, true) + }) + testProfileRecordNullPadding(t, "MemProfile/inUseZero=false", func(p []runtime.MemProfileRecord) (int, bool) { + return runtime.MemProfile(p, false) + }) + // Not testing ThreadCreateProfile because it is broken, see issue 6104. +} + +func testProfileRecordNullPadding[T runtime.StackRecord | runtime.MemProfileRecord | runtime.BlockProfileRecord](t *testing.T, name string, fn func([]T) (int, bool)) { + stack0 := func(sr *T) *[32]uintptr { + switch t := any(sr).(type) { + case *runtime.StackRecord: + return &t.Stack0 + case *runtime.MemProfileRecord: + return &t.Stack0 + case *runtime.BlockProfileRecord: + return &t.Stack0 + default: + panic(fmt.Sprintf("unexpected type %T", sr)) + } + } + + t.Run(name, func(t *testing.T) { + var p []T + for { + n, ok := fn(p) + if ok { + p = p[:n] + break + } + p = make([]T, n*2) + for i := range p { + s0 := stack0(&p[i]) + for j := range s0 { + // Poison the Stack0 array to identify lack of zero padding + s0[j] = ^uintptr(0) + } + } + } + + if len(p) == 0 { + t.Fatal("no records found") + } + + for _, sr := range p { + for i, v := range stack0(&sr) { + if v == ^uintptr(0) { + t.Fatalf("record p[%d].Stack0 is not null padded: %+v", i, sr) + } + } + } + }) +} + +// disableSampling configures the profilers to capture all events, otherwise +// it's difficult to assert anything. +func disableSampling() func() { + oldMemRate := runtime.MemProfileRate + runtime.MemProfileRate = 1 + runtime.SetBlockProfileRate(1) + oldMutexRate := runtime.SetMutexProfileFraction(1) + return func() { + runtime.MemProfileRate = oldMemRate + runtime.SetBlockProfileRate(0) + runtime.SetMutexProfileFraction(oldMutexRate) + } +} diff --git a/src/runtime/pprof/proto.go b/src/runtime/pprof/proto.go index 5214374bd9b087..28ceb81542110a 100644 --- a/src/runtime/pprof/proto.go +++ b/src/runtime/pprof/proto.go @@ -345,7 +345,7 @@ func (b *profileBuilder) addCPUData(data []uint64, tags []unsafe.Pointer) error } // build completes and returns the constructed profile. -func (b *profileBuilder) build() { +func (b *profileBuilder) build() error { b.end = time.Now() b.pb.int64Opt(tagProfile_TimeNanos, b.start.UnixNano()) @@ -367,8 +367,8 @@ func (b *profileBuilder) build() { var labels func() if e.tag != nil { labels = func() { - for k, v := range *(*labelMap)(e.tag) { - b.pbLabel(tagSample_Label, k, v, 0) + for _, lbl := range (*labelMap)(e.tag).list { + b.pbLabel(tagSample_Label, lbl.key, lbl.value, 0) } } } @@ -387,8 +387,11 @@ func (b *profileBuilder) build() { // TODO: Anything for tagProfile_KeepFrames? b.pb.strings(tagProfile_StringTable, b.strings) - b.zw.Write(b.pb.data) - b.zw.Close() + _, err := b.zw.Write(b.pb.data) + if err != nil { + return err + } + return b.zw.Close() } // appendLocsForStack appends the location IDs for the given stack trace to the given @@ -404,6 +407,7 @@ func (b *profileBuilder) appendLocsForStack(locs []uint64, stk []uintptr) (newLo b.deck.reset() // The last frame might be truncated. Recover lost inline frames. + origStk := stk stk = runtime_expandFinalInlineFrame(stk) for len(stk) > 0 { @@ -440,6 +444,9 @@ func (b *profileBuilder) appendLocsForStack(locs []uint64, stk []uintptr) (newLo // Even if stk was truncated due to the stack depth // limit, expandFinalInlineFrame above has already // fixed the truncation, ensuring it is long enough. + if len(l.pcs) > len(stk) { + panic(fmt.Sprintf("stack too short to match cached location; stk = %#x, l.pcs = %#x, original stk = %#x", stk, l.pcs, origStk)) + } stk = stk[len(l.pcs):] continue } diff --git a/src/runtime/pprof/proto_test.go b/src/runtime/pprof/proto_test.go index caaaa45f1257eb..b22d6e2b0366c3 100644 --- a/src/runtime/pprof/proto_test.go +++ b/src/runtime/pprof/proto_test.go @@ -7,6 +7,7 @@ package pprof import ( "bytes" "encoding/json" + "errors" "fmt" "internal/abi" "internal/profile" @@ -33,7 +34,9 @@ func translateCPUProfile(data []uint64, count int) (*profile.Profile, error) { if err := b.addCPUData(data, tags); err != nil { return nil, err } - b.build() + if err := b.build(); err != nil { + return nil, err + } return profile.Parse(&buf) } @@ -73,7 +76,10 @@ func TestConvertCPUProfileNoSamples(t *testing.T) { checkProfile(t, p, 2000*1000, periodType, sampleType, nil, "") } +//go:noinline func f1() { f1() } + +//go:noinline func f2() { f2() } // testPCs returns two PCs and two corresponding memory mappings @@ -470,3 +476,16 @@ func TestEmptyStack(t *testing.T) { t.Fatalf("translating profile: %v", err) } } + +var errWrite = errors.New("error from writer") + +type errWriter struct{} + +func (errWriter) Write(p []byte) (int, error) { return 0, errWrite } + +func TestWriteToErr(t *testing.T) { + err := Lookup("heap").WriteTo(&errWriter{}, 0) + if !errors.Is(err, errWrite) { + t.Fatalf("want error from writer, got: %v", err) + } +} diff --git a/src/runtime/pprof/proto_windows.go b/src/runtime/pprof/proto_windows.go index f4dc44bd078eac..3118e8911e28ec 100644 --- a/src/runtime/pprof/proto_windows.go +++ b/src/runtime/pprof/proto_windows.go @@ -67,8 +67,7 @@ func readMainModuleMapping() (start, end uint64, exe, buildID string, err error) func createModuleSnapshot() (syscall.Handle, error) { for { snap, err := syscall.CreateToolhelp32Snapshot(windows.TH32CS_SNAPMODULE|windows.TH32CS_SNAPMODULE32, uint32(syscall.Getpid())) - var errno syscall.Errno - if err != nil && errors.As(err, &errno) && errno == windows.ERROR_BAD_LENGTH { + if errno, ok := errors.AsType[syscall.Errno](err); ok && errno == windows.ERROR_BAD_LENGTH { // When CreateToolhelp32Snapshot(SNAPMODULE|SNAPMODULE32, ...) fails // with ERROR_BAD_LENGTH then it should be retried until it succeeds. continue diff --git a/src/runtime/pprof/protomem.go b/src/runtime/pprof/protomem.go index ab3550f43f9a39..e0d3746e364370 100644 --- a/src/runtime/pprof/protomem.go +++ b/src/runtime/pprof/protomem.go @@ -36,7 +36,7 @@ func writeHeapProto(w io.Writer, p []profilerecord.MemProfileRecord, rate int64, // what appendLocsForStack expects. if hideRuntime { for i, addr := range stk { - if f := runtime.FuncForPC(addr); f != nil && strings.HasPrefix(f.Name(), "runtime.") { + if f := runtime.FuncForPC(addr); f != nil && (strings.HasPrefix(f.Name(), "runtime.") || strings.HasPrefix(f.Name(), "internal/runtime/")) { continue } // Found non-runtime. Show any runtime uses above it. @@ -63,8 +63,7 @@ func writeHeapProto(w io.Writer, p []profilerecord.MemProfileRecord, rate int64, } }) } - b.build() - return nil + return b.build() } // scaleHeapSample adjusts the data from a heap Sample to diff --git a/src/runtime/pprof/protomem_test.go b/src/runtime/pprof/protomem_test.go index 8e9732a33156e1..6f3231d42a966b 100644 --- a/src/runtime/pprof/protomem_test.go +++ b/src/runtime/pprof/protomem_test.go @@ -7,6 +7,7 @@ package pprof import ( "bytes" "fmt" + "internal/asan" "internal/profile" "internal/profilerecord" "internal/testenv" @@ -117,8 +118,11 @@ func locationToStrings(loc *profile.Location, funcs []string) []string { return funcs } -// This is a regression test for https://go.dev/issue/64528 . +// This is a regression test for https://go.dev/issue/64528. func TestGenericsHashKeyInPprofBuilder(t *testing.T) { + if asan.Enabled { + t.Skip("extra allocations with -asan throw off the test; see #70079") + } previousRate := runtime.MemProfileRate runtime.MemProfileRate = 1 defer func() { @@ -127,7 +131,7 @@ func TestGenericsHashKeyInPprofBuilder(t *testing.T) { for _, sz := range []int{128, 256} { genericAllocFunc[uint32](sz / 4) } - for _, sz := range []int{32, 64} { + for _, sz := range []int{64, 128} { genericAllocFunc[uint64](sz / 8) } @@ -145,8 +149,8 @@ func TestGenericsHashKeyInPprofBuilder(t *testing.T) { expected := []string{ "testing.tRunner;runtime/pprof.TestGenericsHashKeyInPprofBuilder;runtime/pprof.genericAllocFunc[go.shape.uint32] [1 128 0 0]", "testing.tRunner;runtime/pprof.TestGenericsHashKeyInPprofBuilder;runtime/pprof.genericAllocFunc[go.shape.uint32] [1 256 0 0]", - "testing.tRunner;runtime/pprof.TestGenericsHashKeyInPprofBuilder;runtime/pprof.genericAllocFunc[go.shape.uint64] [1 32 0 0]", "testing.tRunner;runtime/pprof.TestGenericsHashKeyInPprofBuilder;runtime/pprof.genericAllocFunc[go.shape.uint64] [1 64 0 0]", + "testing.tRunner;runtime/pprof.TestGenericsHashKeyInPprofBuilder;runtime/pprof.genericAllocFunc[go.shape.uint64] [1 128 0 0]", } for _, l := range expected { @@ -178,6 +182,9 @@ func nonRecursiveGenericAllocFunction[CurrentOp any, OtherOp any](alloc bool) { } func TestGenericsInlineLocations(t *testing.T) { + if asan.Enabled { + t.Skip("extra allocations with -asan throw off the test; see #70079") + } if testenv.OptimizationOff() { t.Skip("skipping test with optimizations disabled") } @@ -222,3 +229,58 @@ func TestGenericsInlineLocations(t *testing.T) { t.Errorf("expected a location with at least 3 functions\n%s\ngot\n%s\n", expectedLocation, actual) } } + +func growMap() { + m := make(map[int]int) + for i := range 512 { + m[i] = i + } +} + +// Runtime frames are hidden in heap profiles. +// This is a regression test for https://go.dev/issue/71174. +func TestHeapRuntimeFrames(t *testing.T) { + previousRate := runtime.MemProfileRate + runtime.MemProfileRate = 1 + defer func() { + runtime.MemProfileRate = previousRate + }() + + growMap() + + runtime.GC() + buf := bytes.NewBuffer(nil) + if err := WriteHeapProfile(buf); err != nil { + t.Fatalf("writing profile: %v", err) + } + p, err := profile.Parse(buf) + if err != nil { + t.Fatalf("profile.Parse: %v", err) + } + + actual := profileToStrings(p) + + // We must see growMap at least once. + foundGrowMap := false + for _, l := range actual { + if !strings.Contains(l, "runtime/pprof.growMap") { + continue + } + foundGrowMap = true + + // Runtime frames like mapassign and map internals should be hidden. + if strings.Contains(l, "runtime.") { + t.Errorf("Sample got %s, want no runtime frames", l) + } + if strings.Contains(l, "internal/runtime/") { + t.Errorf("Sample got %s, want no runtime frames", l) + } + if strings.Contains(l, "mapassign") { // in case mapassign moves to a package not matching above paths. + t.Errorf("Sample got %s, want no mapassign frames", l) + } + } + + if !foundGrowMap { + t.Errorf("Profile got:\n%s\nwant sample in runtime/pprof.growMap", strings.Join(actual, "\n")) + } +} diff --git a/src/runtime/pprof/runtime.go b/src/runtime/pprof/runtime.go index 8d37c7d3add146..690cab81ab5a5a 100644 --- a/src/runtime/pprof/runtime.go +++ b/src/runtime/pprof/runtime.go @@ -29,6 +29,12 @@ func runtime_setProfLabel(labels unsafe.Pointer) // runtime_getProfLabel is defined in runtime/proflabel.go. func runtime_getProfLabel() unsafe.Pointer +// runtime_goroutineleakcount is defined in runtime/proc.go. +func runtime_goroutineleakcount() int + +// runtime_goroutineLeakGC is defined in runtime/mgc.go. +func runtime_goroutineLeakGC() + // SetGoroutineLabels sets the current goroutine's labels to match ctx. // A new goroutine inherits the labels of the goroutine that created it. // This is a lower-level API than [Do], which should be used instead when possible. diff --git a/src/runtime/pprof/runtime_test.go b/src/runtime/pprof/runtime_test.go index e77c7f2bc90d7b..353ed8a3f1d027 100644 --- a/src/runtime/pprof/runtime_test.go +++ b/src/runtime/pprof/runtime_test.go @@ -92,5 +92,9 @@ func getProfLabel() map[string]string { if l == nil { return map[string]string{} } - return *l + m := make(map[string]string, len(l.list)) + for _, lbl := range l.list { + m[lbl.key] = lbl.value + } + return m } diff --git a/src/runtime/pprof/vminfo_darwin.go b/src/runtime/pprof/vminfo_darwin.go index 35b9e6d4879c56..610de0a2f0816f 100644 --- a/src/runtime/pprof/vminfo_darwin.go +++ b/src/runtime/pprof/vminfo_darwin.go @@ -5,6 +5,7 @@ package pprof import ( + "internal/byteorder" "os" "unsafe" ) @@ -39,7 +40,7 @@ func machVMInfo(addMapping func(lo, hi, offset uint64, file, buildID string)) bo // offset is usually 0. addMapping(addr, addr+memRegionSize, - read64(&info.Offset), + byteorder.LEUint64(info.Offset[:]), regionFilename(addr), "") added = true @@ -48,11 +49,6 @@ func machVMInfo(addMapping func(lo, hi, offset uint64, file, buildID string)) bo } } -func read64(p *[8]byte) uint64 { - // all supported darwin platforms are little endian - return uint64(p[0]) | uint64(p[1])<<8 | uint64(p[2])<<16 | uint64(p[3])<<24 | uint64(p[4])<<32 | uint64(p[5])<<40 | uint64(p[6])<<48 | uint64(p[7])<<56 -} - func regionFilename(address uint64) string { buf := make([]byte, _MAXPATHLEN) r := proc_regionfilename( diff --git a/src/runtime/pprof/vminfo_darwin_test.go b/src/runtime/pprof/vminfo_darwin_test.go index 641587200c297f..6d375c5d53368a 100644 --- a/src/runtime/pprof/vminfo_darwin_test.go +++ b/src/runtime/pprof/vminfo_darwin_test.go @@ -17,6 +17,7 @@ import ( "strconv" "strings" "testing" + "time" ) func TestVMInfo(t *testing.T) { @@ -56,18 +57,35 @@ func TestVMInfo(t *testing.T) { } } +type mapping struct { + hi, lo uint64 + err error +} + func useVMMapWithRetry(t *testing.T) (hi, lo uint64, err error) { var retryable bool - for { - hi, lo, retryable, err = useVMMap(t) - if err == nil { - return hi, lo, nil - } - if !retryable { - return 0, 0, err + ch := make(chan mapping) + go func() { + for { + hi, lo, retryable, err = useVMMap(t) + if err == nil { + ch <- mapping{hi, lo, nil} + return + } + if !retryable { + ch <- mapping{0, 0, err} + return + } + t.Logf("retrying vmmap after error: %v", err) } - t.Logf("retrying vmmap after error: %v", err) + }() + select { + case m := <-ch: + return m.hi, m.lo, m.err + case <-time.After(time.Minute): + t.Skip("vmmap taking too long") } + return 0, 0, fmt.Errorf("unreachable") } func useVMMap(t *testing.T) (hi, lo uint64, retryable bool, err error) { @@ -79,7 +97,7 @@ func useVMMap(t *testing.T) (hi, lo uint64, retryable bool, err error) { t.Logf("vmmap output: %s", out) if ee, ok := cmdErr.(*exec.ExitError); ok && len(ee.Stderr) > 0 { t.Logf("%v: %v\n%s", cmd, cmdErr, ee.Stderr) - if testing.Short() && strings.Contains(string(ee.Stderr), "No process corpse slots currently available, waiting to get one") { + if testing.Short() && (strings.Contains(string(ee.Stderr), "No process corpse slots currently available, waiting to get one") || strings.Contains(string(ee.Stderr), "Failed to generate corpse from the process")) { t.Skipf("Skipping knwn flake in short test mode") } retryable = bytes.Contains(ee.Stderr, []byte("resource shortage")) diff --git a/src/runtime/preempt.go b/src/runtime/preempt.go index 45b1b5e9c7d42c..1044a6d7207f69 100644 --- a/src/runtime/preempt.go +++ b/src/runtime/preempt.go @@ -134,7 +134,7 @@ func suspendG(gp *g) suspendGState { dumpgstatus(gp) throw("invalid g status") - case _Gdead: + case _Gdead, _Gdeadextra: // Nothing to suspend. // // preemptStop may need to be cleared, but @@ -160,7 +160,7 @@ func suspendG(gp *g) suspendGState { s = _Gwaiting fallthrough - case _Grunnable, _Gsyscall, _Gwaiting: + case _Grunnable, _Gsyscall, _Gwaiting, _Gleaked: // Claim goroutine by setting scan bit. // This may race with execution or readying of gp. // The scan bit keeps it from transition state. @@ -269,6 +269,7 @@ func resumeG(state suspendGState) { case _Grunnable | _Gscan, _Gwaiting | _Gscan, + _Gleaked | _Gscan, _Gsyscall | _Gscan: casfrom_Gscanstatus(gp, s, s&^_Gscan) } @@ -292,21 +293,52 @@ func canPreemptM(mp *m) bool { // asyncPreempt saves all user registers and calls asyncPreempt2. // -// When stack scanning encounters an asyncPreempt frame, it scans that +// It saves GP registers (anything that might contain a pointer) to the G stack. +// Hence, when stack scanning encounters an asyncPreempt frame, it scans that // frame and its parent frame conservatively. // +// On some platforms, it saves large additional scalar-only register state such +// as vector registers to an "extended register state" on the P. +// // asyncPreempt is implemented in assembly. func asyncPreempt() +// asyncPreempt2 is the Go continuation of asyncPreempt. +// +// It must be deeply nosplit because there's untyped data on the stack from +// asyncPreempt. +// +// It must not have any write barriers because we need to limit the amount of +// stack it uses. +// //go:nosplit +//go:nowritebarrierrec func asyncPreempt2() { + // We can't grow the stack with untyped data from asyncPreempt, so switch to + // the system stack right away. + mcall(func(gp *g) { + gp.asyncSafePoint = true + + // Move the extended register state from the P to the G. We do this now that + // we're on the system stack to avoid stack splits. + xRegSave(gp) + + if gp.preemptStop { + preemptPark(gp) + } else { + gopreempt_m(gp) + } + // The above functions never return. + }) + + // Do not grow the stack below here! + gp := getg() - gp.asyncSafePoint = true - if gp.preemptStop { - mcall(preemptPark) - } else { - mcall(gopreempt_m) - } + + // Put the extended register state back on the M so resumption can find it. + // We can't do this in asyncPreemptM because the park calls never return. + xRegRestore(gp) + gp.asyncSafePoint = false } @@ -319,19 +351,13 @@ func init() { total := funcMaxSPDelta(f) f = findfunc(abi.FuncPCABIInternal(asyncPreempt2)) total += funcMaxSPDelta(f) + f = findfunc(abi.FuncPCABIInternal(xRegRestore)) + total += funcMaxSPDelta(f) // Add some overhead for return PCs, etc. asyncPreemptStack = uintptr(total) + 8*goarch.PtrSize if asyncPreemptStack > stackNosplit { - // We need more than the nosplit limit. This isn't - // unsafe, but it may limit asynchronous preemption. - // - // This may be a problem if we start using more - // registers. In that case, we should store registers - // in a context object. If we pre-allocate one per P, - // asyncPreempt can spill just a few registers to the - // stack, then grab its context object and spill into - // it. When it enters the runtime, it would allocate a - // new context for the P. + // We need more than the nosplit limit. This isn't unsafe, but it may + // limit asynchronous preemption. Consider moving state into xRegState. print("runtime: asyncPreemptStack=", asyncPreemptStack, "\n") throw("async stack too large") } @@ -418,15 +444,21 @@ func isAsyncSafePoint(gp *g, pc, sp, lr uintptr) (bool, uintptr) { u, uf := newInlineUnwinder(f, pc) name := u.srcFunc(uf).name() if stringslite.HasPrefix(name, "runtime.") || - stringslite.HasPrefix(name, "runtime/internal/") || + stringslite.HasPrefix(name, "internal/runtime/") || stringslite.HasPrefix(name, "reflect.") { // For now we never async preempt the runtime or // anything closely tied to the runtime. Known issues // include: various points in the scheduler ("don't // preempt between here and here"), much of the defer // implementation (untyped info on stack), bulk write - // barriers (write barrier check), - // reflect.{makeFuncStub,methodValueCall}. + // barriers (write barrier check), atomic functions in + // internal/runtime/atomic, reflect.{makeFuncStub,methodValueCall}. + // + // Note that this is a subset of the runtimePkgs in pkgspecial.go + // and these checks are theoretically redundant because the compiler + // marks "all points" in runtime functions as unsafe for async preemption. + // But for some reason, we can't eliminate these checks until https://go.dev/issue/72031 + // is resolved. // // TODO(austin): We should improve this, or opt things // in incrementally. diff --git a/src/runtime/preempt_amd64.go b/src/runtime/preempt_amd64.go new file mode 100644 index 00000000000000..88c0ddd34ade72 --- /dev/null +++ b/src/runtime/preempt_amd64.go @@ -0,0 +1,30 @@ +// Code generated by mkpreempt.go; DO NOT EDIT. + +package runtime + +type xRegs struct { + Z0 [64]byte + Z1 [64]byte + Z2 [64]byte + Z3 [64]byte + Z4 [64]byte + Z5 [64]byte + Z6 [64]byte + Z7 [64]byte + Z8 [64]byte + Z9 [64]byte + Z10 [64]byte + Z11 [64]byte + Z12 [64]byte + Z13 [64]byte + Z14 [64]byte + Z15 [64]byte + K0 uint64 + K1 uint64 + K2 uint64 + K3 uint64 + K4 uint64 + K5 uint64 + K6 uint64 + K7 uint64 +} diff --git a/src/runtime/preempt_amd64.s b/src/runtime/preempt_amd64.s index 8e3ed0d7c59dce..c35de7f3b75726 100644 --- a/src/runtime/preempt_amd64.s +++ b/src/runtime/preempt_amd64.s @@ -1,6 +1,7 @@ // Code generated by mkpreempt.go; DO NOT EDIT. #include "go_asm.h" +#include "go_tls.h" #include "asm_amd64.h" #include "textflag.h" @@ -10,9 +11,10 @@ TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 // Save flags before clobbering them PUSHFQ // obj doesn't understand ADD/SUB on SP, but does understand ADJSP - ADJSP $368 + ADJSP $112 // But vet doesn't know ADJSP, so suppress vet stack checking NOP SP + // Save GPs MOVQ AX, 0(SP) MOVQ CX, 8(SP) MOVQ DX, 16(SP) @@ -27,39 +29,157 @@ TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVQ R13, 88(SP) MOVQ R14, 96(SP) MOVQ R15, 104(SP) - MOVUPS X0, 112(SP) - MOVUPS X1, 128(SP) - MOVUPS X2, 144(SP) - MOVUPS X3, 160(SP) - MOVUPS X4, 176(SP) - MOVUPS X5, 192(SP) - MOVUPS X6, 208(SP) - MOVUPS X7, 224(SP) - MOVUPS X8, 240(SP) - MOVUPS X9, 256(SP) - MOVUPS X10, 272(SP) - MOVUPS X11, 288(SP) - MOVUPS X12, 304(SP) - MOVUPS X13, 320(SP) - MOVUPS X14, 336(SP) - MOVUPS X15, 352(SP) + // Save extended register state to p.xRegs.scratch + // Don't make assumptions about ABI register state. See mkpreempt.go + get_tls(CX) + MOVQ g(CX), R14 + MOVQ g_m(R14), AX + MOVQ m_p(AX), AX + LEAQ (p_xRegs+xRegPerP_scratch)(AX), AX + #ifdef GOEXPERIMENT_simd + CMPB internal∕cpu·X86+const_offsetX86HasAVX512(SB), $1 + JE saveAVX512 + CMPB internal∕cpu·X86+const_offsetX86HasAVX2(SB), $1 + JE saveAVX2 + #endif +saveSSE: + MOVUPS X0, 0(AX) + MOVUPS X1, 64(AX) + MOVUPS X2, 128(AX) + MOVUPS X3, 192(AX) + MOVUPS X4, 256(AX) + MOVUPS X5, 320(AX) + MOVUPS X6, 384(AX) + MOVUPS X7, 448(AX) + MOVUPS X8, 512(AX) + MOVUPS X9, 576(AX) + MOVUPS X10, 640(AX) + MOVUPS X11, 704(AX) + MOVUPS X12, 768(AX) + MOVUPS X13, 832(AX) + MOVUPS X14, 896(AX) + MOVUPS X15, 960(AX) + JMP preempt +saveAVX2: + VMOVDQU Y0, 0(AX) + VMOVDQU Y1, 64(AX) + VMOVDQU Y2, 128(AX) + VMOVDQU Y3, 192(AX) + VMOVDQU Y4, 256(AX) + VMOVDQU Y5, 320(AX) + VMOVDQU Y6, 384(AX) + VMOVDQU Y7, 448(AX) + VMOVDQU Y8, 512(AX) + VMOVDQU Y9, 576(AX) + VMOVDQU Y10, 640(AX) + VMOVDQU Y11, 704(AX) + VMOVDQU Y12, 768(AX) + VMOVDQU Y13, 832(AX) + VMOVDQU Y14, 896(AX) + VMOVDQU Y15, 960(AX) + JMP preempt +saveAVX512: + VMOVDQU64 Z0, 0(AX) + VMOVDQU64 Z1, 64(AX) + VMOVDQU64 Z2, 128(AX) + VMOVDQU64 Z3, 192(AX) + VMOVDQU64 Z4, 256(AX) + VMOVDQU64 Z5, 320(AX) + VMOVDQU64 Z6, 384(AX) + VMOVDQU64 Z7, 448(AX) + VMOVDQU64 Z8, 512(AX) + VMOVDQU64 Z9, 576(AX) + VMOVDQU64 Z10, 640(AX) + VMOVDQU64 Z11, 704(AX) + VMOVDQU64 Z12, 768(AX) + VMOVDQU64 Z13, 832(AX) + VMOVDQU64 Z14, 896(AX) + VMOVDQU64 Z15, 960(AX) + KMOVQ K0, 1024(AX) + KMOVQ K1, 1032(AX) + KMOVQ K2, 1040(AX) + KMOVQ K3, 1048(AX) + KMOVQ K4, 1056(AX) + KMOVQ K5, 1064(AX) + KMOVQ K6, 1072(AX) + KMOVQ K7, 1080(AX) + JMP preempt +preempt: CALL ·asyncPreempt2(SB) - MOVUPS 352(SP), X15 - MOVUPS 336(SP), X14 - MOVUPS 320(SP), X13 - MOVUPS 304(SP), X12 - MOVUPS 288(SP), X11 - MOVUPS 272(SP), X10 - MOVUPS 256(SP), X9 - MOVUPS 240(SP), X8 - MOVUPS 224(SP), X7 - MOVUPS 208(SP), X6 - MOVUPS 192(SP), X5 - MOVUPS 176(SP), X4 - MOVUPS 160(SP), X3 - MOVUPS 144(SP), X2 - MOVUPS 128(SP), X1 - MOVUPS 112(SP), X0 + // Restore non-GPs from *p.xRegs.cache + MOVQ g_m(R14), AX + MOVQ m_p(AX), AX + MOVQ (p_xRegs+xRegPerP_cache)(AX), AX + #ifdef GOEXPERIMENT_simd + CMPB internal∕cpu·X86+const_offsetX86HasAVX512(SB), $1 + JE restoreAVX512 + CMPB internal∕cpu·X86+const_offsetX86HasAVX2(SB), $1 + JE restoreAVX2 + #endif +restoreSSE: + MOVUPS 960(AX), X15 + MOVUPS 896(AX), X14 + MOVUPS 832(AX), X13 + MOVUPS 768(AX), X12 + MOVUPS 704(AX), X11 + MOVUPS 640(AX), X10 + MOVUPS 576(AX), X9 + MOVUPS 512(AX), X8 + MOVUPS 448(AX), X7 + MOVUPS 384(AX), X6 + MOVUPS 320(AX), X5 + MOVUPS 256(AX), X4 + MOVUPS 192(AX), X3 + MOVUPS 128(AX), X2 + MOVUPS 64(AX), X1 + MOVUPS 0(AX), X0 + JMP restoreGPs +restoreAVX2: + VMOVDQU 960(AX), Y15 + VMOVDQU 896(AX), Y14 + VMOVDQU 832(AX), Y13 + VMOVDQU 768(AX), Y12 + VMOVDQU 704(AX), Y11 + VMOVDQU 640(AX), Y10 + VMOVDQU 576(AX), Y9 + VMOVDQU 512(AX), Y8 + VMOVDQU 448(AX), Y7 + VMOVDQU 384(AX), Y6 + VMOVDQU 320(AX), Y5 + VMOVDQU 256(AX), Y4 + VMOVDQU 192(AX), Y3 + VMOVDQU 128(AX), Y2 + VMOVDQU 64(AX), Y1 + VMOVDQU 0(AX), Y0 + JMP restoreGPs +restoreAVX512: + KMOVQ 1080(AX), K7 + KMOVQ 1072(AX), K6 + KMOVQ 1064(AX), K5 + KMOVQ 1056(AX), K4 + KMOVQ 1048(AX), K3 + KMOVQ 1040(AX), K2 + KMOVQ 1032(AX), K1 + KMOVQ 1024(AX), K0 + VMOVDQU64 960(AX), Z15 + VMOVDQU64 896(AX), Z14 + VMOVDQU64 832(AX), Z13 + VMOVDQU64 768(AX), Z12 + VMOVDQU64 704(AX), Z11 + VMOVDQU64 640(AX), Z10 + VMOVDQU64 576(AX), Z9 + VMOVDQU64 512(AX), Z8 + VMOVDQU64 448(AX), Z7 + VMOVDQU64 384(AX), Z6 + VMOVDQU64 320(AX), Z5 + VMOVDQU64 256(AX), Z4 + VMOVDQU64 192(AX), Z3 + VMOVDQU64 128(AX), Z2 + VMOVDQU64 64(AX), Z1 + VMOVDQU64 0(AX), Z0 + JMP restoreGPs +restoreGPs: + // Restore GPs MOVQ 104(SP), R15 MOVQ 96(SP), R14 MOVQ 88(SP), R13 @@ -74,7 +194,7 @@ TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVQ 16(SP), DX MOVQ 8(SP), CX MOVQ 0(SP), AX - ADJSP $-368 + ADJSP $-112 POPFQ POPQ BP RET diff --git a/src/runtime/preempt_arm64.go b/src/runtime/preempt_arm64.go new file mode 100644 index 00000000000000..1b71d2713ea5d0 --- /dev/null +++ b/src/runtime/preempt_arm64.go @@ -0,0 +1,38 @@ +// Code generated by mkpreempt.go; DO NOT EDIT. + +package runtime + +type xRegs struct { + V0 [16]byte + V1 [16]byte + V2 [16]byte + V3 [16]byte + V4 [16]byte + V5 [16]byte + V6 [16]byte + V7 [16]byte + V8 [16]byte + V9 [16]byte + V10 [16]byte + V11 [16]byte + V12 [16]byte + V13 [16]byte + V14 [16]byte + V15 [16]byte + V16 [16]byte + V17 [16]byte + V18 [16]byte + V19 [16]byte + V20 [16]byte + V21 [16]byte + V22 [16]byte + V23 [16]byte + V24 [16]byte + V25 [16]byte + V26 [16]byte + V27 [16]byte + V28 [16]byte + V29 [16]byte + V30 [16]byte + V31 [16]byte +} diff --git a/src/runtime/preempt_arm64.s b/src/runtime/preempt_arm64.s index c27d475dee6ae0..9017d8815970b2 100644 --- a/src/runtime/preempt_arm64.s +++ b/src/runtime/preempt_arm64.s @@ -4,13 +4,14 @@ #include "textflag.h" TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 - MOVD R30, -496(RSP) - SUB $496, RSP + MOVD R30, -240(RSP) + SUB $240, RSP MOVD R29, -8(RSP) SUB $8, RSP, R29 #ifdef GOOS_ios MOVD R30, (RSP) #endif + // Save GPs STP (R0, R1), 8(RSP) STP (R2, R3), 24(RSP) STP (R4, R5), 40(RSP) @@ -28,39 +29,32 @@ TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVD R0, 216(RSP) MOVD FPSR, R0 MOVD R0, 224(RSP) - FSTPD (F0, F1), 232(RSP) - FSTPD (F2, F3), 248(RSP) - FSTPD (F4, F5), 264(RSP) - FSTPD (F6, F7), 280(RSP) - FSTPD (F8, F9), 296(RSP) - FSTPD (F10, F11), 312(RSP) - FSTPD (F12, F13), 328(RSP) - FSTPD (F14, F15), 344(RSP) - FSTPD (F16, F17), 360(RSP) - FSTPD (F18, F19), 376(RSP) - FSTPD (F20, F21), 392(RSP) - FSTPD (F22, F23), 408(RSP) - FSTPD (F24, F25), 424(RSP) - FSTPD (F26, F27), 440(RSP) - FSTPD (F28, F29), 456(RSP) - FSTPD (F30, F31), 472(RSP) + // Save extended register state to p.xRegs.scratch + MOVD g_m(g), R0 + MOVD m_p(R0), R0 + ADD $(p_xRegs+xRegPerP_scratch), R0, R0 + VST1.P [V0.B16, V1.B16, V2.B16, V3.B16], 64(R0) + VST1.P [V4.B16, V5.B16, V6.B16, V7.B16], 64(R0) + VST1.P [V8.B16, V9.B16, V10.B16, V11.B16], 64(R0) + VST1.P [V12.B16, V13.B16, V14.B16, V15.B16], 64(R0) + VST1.P [V16.B16, V17.B16, V18.B16, V19.B16], 64(R0) + VST1.P [V20.B16, V21.B16, V22.B16, V23.B16], 64(R0) + VST1.P [V24.B16, V25.B16, V26.B16, V27.B16], 64(R0) + VST1.P [V28.B16, V29.B16, V30.B16, V31.B16], 64(R0) CALL ·asyncPreempt2(SB) - FLDPD 472(RSP), (F30, F31) - FLDPD 456(RSP), (F28, F29) - FLDPD 440(RSP), (F26, F27) - FLDPD 424(RSP), (F24, F25) - FLDPD 408(RSP), (F22, F23) - FLDPD 392(RSP), (F20, F21) - FLDPD 376(RSP), (F18, F19) - FLDPD 360(RSP), (F16, F17) - FLDPD 344(RSP), (F14, F15) - FLDPD 328(RSP), (F12, F13) - FLDPD 312(RSP), (F10, F11) - FLDPD 296(RSP), (F8, F9) - FLDPD 280(RSP), (F6, F7) - FLDPD 264(RSP), (F4, F5) - FLDPD 248(RSP), (F2, F3) - FLDPD 232(RSP), (F0, F1) + // Restore non-GPs from *p.xRegs.cache + MOVD g_m(g), R0 + MOVD m_p(R0), R0 + MOVD (p_xRegs+xRegPerP_cache)(R0), R0 + VLD1.P 64(R0), [V0.B16, V1.B16, V2.B16, V3.B16] + VLD1.P 64(R0), [V4.B16, V5.B16, V6.B16, V7.B16] + VLD1.P 64(R0), [V8.B16, V9.B16, V10.B16, V11.B16] + VLD1.P 64(R0), [V12.B16, V13.B16, V14.B16, V15.B16] + VLD1.P 64(R0), [V16.B16, V17.B16, V18.B16, V19.B16] + VLD1.P 64(R0), [V20.B16, V21.B16, V22.B16, V23.B16] + VLD1.P 64(R0), [V24.B16, V25.B16, V26.B16, V27.B16] + VLD1.P 64(R0), [V28.B16, V29.B16, V30.B16, V31.B16] + // Restore GPs MOVD 224(RSP), R0 MOVD R0, FPSR MOVD 216(RSP), R0 @@ -78,8 +72,8 @@ TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 LDP 40(RSP), (R4, R5) LDP 24(RSP), (R2, R3) LDP 8(RSP), (R0, R1) - MOVD 496(RSP), R30 + MOVD 240(RSP), R30 MOVD -8(RSP), R29 MOVD (RSP), R27 - ADD $512, RSP - JMP (R27) + ADD $256, RSP + RET (R27) diff --git a/src/runtime/preempt_loong64.go b/src/runtime/preempt_loong64.go new file mode 100644 index 00000000000000..c7fec338f26694 --- /dev/null +++ b/src/runtime/preempt_loong64.go @@ -0,0 +1,38 @@ +// Code generated by mkpreempt.go; DO NOT EDIT. + +package runtime + +type xRegs struct { + X0 [32]byte + X1 [32]byte + X2 [32]byte + X3 [32]byte + X4 [32]byte + X5 [32]byte + X6 [32]byte + X7 [32]byte + X8 [32]byte + X9 [32]byte + X10 [32]byte + X11 [32]byte + X12 [32]byte + X13 [32]byte + X14 [32]byte + X15 [32]byte + X16 [32]byte + X17 [32]byte + X18 [32]byte + X19 [32]byte + X20 [32]byte + X21 [32]byte + X22 [32]byte + X23 [32]byte + X24 [32]byte + X25 [32]byte + X26 [32]byte + X27 [32]byte + X28 [32]byte + X29 [32]byte + X30 [32]byte + X31 [32]byte +} diff --git a/src/runtime/preempt_loong64.s b/src/runtime/preempt_loong64.s index bb9c9483650147..4bc7ea3947bd27 100644 --- a/src/runtime/preempt_loong64.s +++ b/src/runtime/preempt_loong64.s @@ -4,8 +4,9 @@ #include "textflag.h" TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 - MOVV R1, -480(R3) - SUBV $480, R3 + MOVV R1, -224(R3) + SUBV $224, R3 + // Save GPs MOVV R4, 8(R3) MOVV R5, 16(R3) MOVV R6, 24(R3) @@ -32,75 +33,262 @@ TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVV R28, 192(R3) MOVV R29, 200(R3) MOVV R31, 208(R3) - MOVD F0, 216(R3) - MOVD F1, 224(R3) - MOVD F2, 232(R3) - MOVD F3, 240(R3) - MOVD F4, 248(R3) - MOVD F5, 256(R3) - MOVD F6, 264(R3) - MOVD F7, 272(R3) - MOVD F8, 280(R3) - MOVD F9, 288(R3) - MOVD F10, 296(R3) - MOVD F11, 304(R3) - MOVD F12, 312(R3) - MOVD F13, 320(R3) - MOVD F14, 328(R3) - MOVD F15, 336(R3) - MOVD F16, 344(R3) - MOVD F17, 352(R3) - MOVD F18, 360(R3) - MOVD F19, 368(R3) - MOVD F20, 376(R3) - MOVD F21, 384(R3) - MOVD F22, 392(R3) - MOVD F23, 400(R3) - MOVD F24, 408(R3) - MOVD F25, 416(R3) - MOVD F26, 424(R3) - MOVD F27, 432(R3) - MOVD F28, 440(R3) - MOVD F29, 448(R3) - MOVD F30, 456(R3) - MOVD F31, 464(R3) MOVV FCC0, R4 - MOVV R4, 472(R3) + BSTRINSV $7, R4, $0, R5 + MOVV FCC1, R4 + BSTRINSV $15, R4, $8, R5 + MOVV FCC2, R4 + BSTRINSV $23, R4, $16, R5 + MOVV FCC3, R4 + BSTRINSV $31, R4, $24, R5 + MOVV FCC4, R4 + BSTRINSV $39, R4, $32, R5 + MOVV FCC5, R4 + BSTRINSV $47, R4, $40, R5 + MOVV FCC6, R4 + BSTRINSV $55, R4, $48, R5 + MOVV FCC7, R4 + BSTRINSV $63, R4, $56, R5 + MOVV R5, 216(R3) + // Save extended register state to p.xRegs.scratch + MOVV g_m(g), R4 + MOVV m_p(R4), R4 + ADDV $(p_xRegs+xRegPerP_scratch), R4, R4 + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R5 + BNE R5, saveLASX + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R5 + BNE R5, saveLSX +saveFP: + MOVD F0, 0(R4) + MOVD F1, 32(R4) + MOVD F2, 64(R4) + MOVD F3, 96(R4) + MOVD F4, 128(R4) + MOVD F5, 160(R4) + MOVD F6, 192(R4) + MOVD F7, 224(R4) + MOVD F8, 256(R4) + MOVD F9, 288(R4) + MOVD F10, 320(R4) + MOVD F11, 352(R4) + MOVD F12, 384(R4) + MOVD F13, 416(R4) + MOVD F14, 448(R4) + MOVD F15, 480(R4) + MOVD F16, 512(R4) + MOVD F17, 544(R4) + MOVD F18, 576(R4) + MOVD F19, 608(R4) + MOVD F20, 640(R4) + MOVD F21, 672(R4) + MOVD F22, 704(R4) + MOVD F23, 736(R4) + MOVD F24, 768(R4) + MOVD F25, 800(R4) + MOVD F26, 832(R4) + MOVD F27, 864(R4) + MOVD F28, 896(R4) + MOVD F29, 928(R4) + MOVD F30, 960(R4) + MOVD F31, 992(R4) + JMP preempt +saveLSX: + VMOVQ V0, 0(R4) + VMOVQ V1, 32(R4) + VMOVQ V2, 64(R4) + VMOVQ V3, 96(R4) + VMOVQ V4, 128(R4) + VMOVQ V5, 160(R4) + VMOVQ V6, 192(R4) + VMOVQ V7, 224(R4) + VMOVQ V8, 256(R4) + VMOVQ V9, 288(R4) + VMOVQ V10, 320(R4) + VMOVQ V11, 352(R4) + VMOVQ V12, 384(R4) + VMOVQ V13, 416(R4) + VMOVQ V14, 448(R4) + VMOVQ V15, 480(R4) + VMOVQ V16, 512(R4) + VMOVQ V17, 544(R4) + VMOVQ V18, 576(R4) + VMOVQ V19, 608(R4) + VMOVQ V20, 640(R4) + VMOVQ V21, 672(R4) + VMOVQ V22, 704(R4) + VMOVQ V23, 736(R4) + VMOVQ V24, 768(R4) + VMOVQ V25, 800(R4) + VMOVQ V26, 832(R4) + VMOVQ V27, 864(R4) + VMOVQ V28, 896(R4) + VMOVQ V29, 928(R4) + VMOVQ V30, 960(R4) + VMOVQ V31, 992(R4) + JMP preempt +saveLASX: + XVMOVQ X0, 0(R4) + XVMOVQ X1, 32(R4) + XVMOVQ X2, 64(R4) + XVMOVQ X3, 96(R4) + XVMOVQ X4, 128(R4) + XVMOVQ X5, 160(R4) + XVMOVQ X6, 192(R4) + XVMOVQ X7, 224(R4) + XVMOVQ X8, 256(R4) + XVMOVQ X9, 288(R4) + XVMOVQ X10, 320(R4) + XVMOVQ X11, 352(R4) + XVMOVQ X12, 384(R4) + XVMOVQ X13, 416(R4) + XVMOVQ X14, 448(R4) + XVMOVQ X15, 480(R4) + XVMOVQ X16, 512(R4) + XVMOVQ X17, 544(R4) + XVMOVQ X18, 576(R4) + XVMOVQ X19, 608(R4) + XVMOVQ X20, 640(R4) + XVMOVQ X21, 672(R4) + XVMOVQ X22, 704(R4) + XVMOVQ X23, 736(R4) + XVMOVQ X24, 768(R4) + XVMOVQ X25, 800(R4) + XVMOVQ X26, 832(R4) + XVMOVQ X27, 864(R4) + XVMOVQ X28, 896(R4) + XVMOVQ X29, 928(R4) + XVMOVQ X30, 960(R4) + XVMOVQ X31, 992(R4) +preempt: CALL ·asyncPreempt2(SB) - MOVV 472(R3), R4 + // Restore non-GPs from *p.xRegs.cache + MOVV g_m(g), R4 + MOVV m_p(R4), R4 + MOVV (p_xRegs+xRegPerP_cache)(R4), R4 + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLASX(SB), R5 + BNE R5, restoreLASX + MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLSX(SB), R5 + BNE R5, restoreLSX +restoreFP: + MOVD 992(R4), F31 + MOVD 960(R4), F30 + MOVD 928(R4), F29 + MOVD 896(R4), F28 + MOVD 864(R4), F27 + MOVD 832(R4), F26 + MOVD 800(R4), F25 + MOVD 768(R4), F24 + MOVD 736(R4), F23 + MOVD 704(R4), F22 + MOVD 672(R4), F21 + MOVD 640(R4), F20 + MOVD 608(R4), F19 + MOVD 576(R4), F18 + MOVD 544(R4), F17 + MOVD 512(R4), F16 + MOVD 480(R4), F15 + MOVD 448(R4), F14 + MOVD 416(R4), F13 + MOVD 384(R4), F12 + MOVD 352(R4), F11 + MOVD 320(R4), F10 + MOVD 288(R4), F9 + MOVD 256(R4), F8 + MOVD 224(R4), F7 + MOVD 192(R4), F6 + MOVD 160(R4), F5 + MOVD 128(R4), F4 + MOVD 96(R4), F3 + MOVD 64(R4), F2 + MOVD 32(R4), F1 + MOVD 0(R4), F0 + JMP restoreGPs +restoreLSX: + VMOVQ 992(R4), V31 + VMOVQ 960(R4), V30 + VMOVQ 928(R4), V29 + VMOVQ 896(R4), V28 + VMOVQ 864(R4), V27 + VMOVQ 832(R4), V26 + VMOVQ 800(R4), V25 + VMOVQ 768(R4), V24 + VMOVQ 736(R4), V23 + VMOVQ 704(R4), V22 + VMOVQ 672(R4), V21 + VMOVQ 640(R4), V20 + VMOVQ 608(R4), V19 + VMOVQ 576(R4), V18 + VMOVQ 544(R4), V17 + VMOVQ 512(R4), V16 + VMOVQ 480(R4), V15 + VMOVQ 448(R4), V14 + VMOVQ 416(R4), V13 + VMOVQ 384(R4), V12 + VMOVQ 352(R4), V11 + VMOVQ 320(R4), V10 + VMOVQ 288(R4), V9 + VMOVQ 256(R4), V8 + VMOVQ 224(R4), V7 + VMOVQ 192(R4), V6 + VMOVQ 160(R4), V5 + VMOVQ 128(R4), V4 + VMOVQ 96(R4), V3 + VMOVQ 64(R4), V2 + VMOVQ 32(R4), V1 + VMOVQ 0(R4), V0 + JMP restoreGPs +restoreLASX: + XVMOVQ 992(R4), X31 + XVMOVQ 960(R4), X30 + XVMOVQ 928(R4), X29 + XVMOVQ 896(R4), X28 + XVMOVQ 864(R4), X27 + XVMOVQ 832(R4), X26 + XVMOVQ 800(R4), X25 + XVMOVQ 768(R4), X24 + XVMOVQ 736(R4), X23 + XVMOVQ 704(R4), X22 + XVMOVQ 672(R4), X21 + XVMOVQ 640(R4), X20 + XVMOVQ 608(R4), X19 + XVMOVQ 576(R4), X18 + XVMOVQ 544(R4), X17 + XVMOVQ 512(R4), X16 + XVMOVQ 480(R4), X15 + XVMOVQ 448(R4), X14 + XVMOVQ 416(R4), X13 + XVMOVQ 384(R4), X12 + XVMOVQ 352(R4), X11 + XVMOVQ 320(R4), X10 + XVMOVQ 288(R4), X9 + XVMOVQ 256(R4), X8 + XVMOVQ 224(R4), X7 + XVMOVQ 192(R4), X6 + XVMOVQ 160(R4), X5 + XVMOVQ 128(R4), X4 + XVMOVQ 96(R4), X3 + XVMOVQ 64(R4), X2 + XVMOVQ 32(R4), X1 + XVMOVQ 0(R4), X0 + // Restore GPs +restoreGPs: + MOVV 216(R3), R5 + BSTRPICKV $7, R5, $0, R4 MOVV R4, FCC0 - MOVD 464(R3), F31 - MOVD 456(R3), F30 - MOVD 448(R3), F29 - MOVD 440(R3), F28 - MOVD 432(R3), F27 - MOVD 424(R3), F26 - MOVD 416(R3), F25 - MOVD 408(R3), F24 - MOVD 400(R3), F23 - MOVD 392(R3), F22 - MOVD 384(R3), F21 - MOVD 376(R3), F20 - MOVD 368(R3), F19 - MOVD 360(R3), F18 - MOVD 352(R3), F17 - MOVD 344(R3), F16 - MOVD 336(R3), F15 - MOVD 328(R3), F14 - MOVD 320(R3), F13 - MOVD 312(R3), F12 - MOVD 304(R3), F11 - MOVD 296(R3), F10 - MOVD 288(R3), F9 - MOVD 280(R3), F8 - MOVD 272(R3), F7 - MOVD 264(R3), F6 - MOVD 256(R3), F5 - MOVD 248(R3), F4 - MOVD 240(R3), F3 - MOVD 232(R3), F2 - MOVD 224(R3), F1 - MOVD 216(R3), F0 + BSTRPICKV $15, R5, $8, R4 + MOVV R4, FCC1 + BSTRPICKV $23, R5, $16, R4 + MOVV R4, FCC2 + BSTRPICKV $31, R5, $24, R4 + MOVV R4, FCC3 + BSTRPICKV $39, R5, $32, R4 + MOVV R4, FCC4 + BSTRPICKV $47, R5, $40, R4 + MOVV R4, FCC5 + BSTRPICKV $55, R5, $48, R4 + MOVV R4, FCC6 + BSTRPICKV $63, R5, $56, R4 + MOVV R4, FCC7 MOVV 208(R3), R31 MOVV 200(R3), R29 MOVV 192(R3), R28 @@ -127,7 +315,7 @@ TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVV 24(R3), R6 MOVV 16(R3), R5 MOVV 8(R3), R4 - MOVV 480(R3), R1 + MOVV 224(R3), R1 MOVV (R3), R30 - ADDV $488, R3 + ADDV $232, R3 JMP (R30) diff --git a/src/runtime/preempt_noxreg.go b/src/runtime/preempt_noxreg.go new file mode 100644 index 00000000000000..977bf0bcec7b34 --- /dev/null +++ b/src/runtime/preempt_noxreg.go @@ -0,0 +1,27 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !amd64 && !arm64 && !loong64 + +// This provides common support for architectures that DO NOT use extended +// register state in asynchronous preemption. + +package runtime + +type xRegPerG struct{} + +type xRegPerP struct{} + +// xRegState is defined only so the build fails if we try to define a real +// xRegState on a noxreg architecture. +type xRegState struct{} + +func xRegInitAlloc() {} + +func xRegSave(gp *g) {} + +//go:nosplit +func xRegRestore(gp *g) {} + +func (*xRegPerP) free() {} diff --git a/src/runtime/preempt_xreg.go b/src/runtime/preempt_xreg.go new file mode 100644 index 00000000000000..cc52c5f3c4ec2a --- /dev/null +++ b/src/runtime/preempt_xreg.go @@ -0,0 +1,137 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build amd64 || arm64 || loong64 + +// This provides common support for architectures that use extended register +// state in asynchronous preemption. +// +// While asynchronous preemption stores general-purpose (GP) registers on the +// preempted goroutine's own stack, extended register state can be used to save +// non-GP state off the stack. In particular, this is meant for large vector +// register files. Currently, we assume this contains only scalar data, though +// we could change this constraint by conservatively scanning this memory. +// +// For an architecture to support extended register state, it must provide a Go +// definition of an xRegState type for storing the state, and its asyncPreempt +// implementation must write this register state to p.xRegs.scratch. + +package runtime + +import ( + "internal/runtime/sys" + "unsafe" +) + +// xRegState is long-lived extended register state. It is allocated off-heap and +// manually managed. +type xRegState struct { + _ sys.NotInHeap // Allocated from xRegAlloc + regs xRegs +} + +// xRegPerG stores extended register state while a goroutine is asynchronously +// preempted. This is nil otherwise, so we can reuse a (likely small) pool of +// xRegState objects. +type xRegPerG struct { + state *xRegState +} + +type xRegPerP struct { + // scratch temporary per-P space where [asyncPreempt] saves the register + // state before entering Go. It's quickly copied to per-G state. + scratch xRegs + + // cache is a 1-element allocation cache of extended register state used by + // asynchronous preemption. On entry to preemption, this is used as a simple + // allocation cache. On exit from preemption, the G's xRegState is always + // stored here where it can be restored, and later either freed or reused + // for another preemption. On exit, this serves the dual purpose of + // delay-freeing the allocated xRegState until after we've definitely + // restored it. + cache *xRegState +} + +// xRegAlloc allocates xRegState objects. +var xRegAlloc struct { + lock mutex + alloc fixalloc +} + +func xRegInitAlloc() { + lockInit(&xRegAlloc.lock, lockRankXRegAlloc) + xRegAlloc.alloc.init(unsafe.Sizeof(xRegState{}), nil, nil, &memstats.other_sys) +} + +// xRegSave saves the extended register state on this P to gp. +// +// This must run on the system stack because it assumes the P won't change. +// +//go:systemstack +func xRegSave(gp *g) { + if gp.xRegs.state != nil { + // Double preempt? + throw("gp.xRegState.p != nil on async preempt") + } + + // Get the place to save the register state. + var dest *xRegState + pp := gp.m.p.ptr() + if pp.xRegs.cache != nil { + // Use the cached allocation. + dest = pp.xRegs.cache + pp.xRegs.cache = nil + } else { + // Allocate a new save block. + lock(&xRegAlloc.lock) + dest = (*xRegState)(xRegAlloc.alloc.alloc()) + unlock(&xRegAlloc.lock) + } + + // Copy state saved in the scratchpad to dest. + // + // If we ever need to save less state (e.g., avoid saving vector registers + // that aren't in use), we could have multiple allocation pools for + // different size states and copy only the registers we need. + dest.regs = pp.xRegs.scratch + + // Save on the G. + gp.xRegs.state = dest +} + +// xRegRestore prepares the extended register state on gp to be restored. +// +// It moves the state to gp.m.p.xRegs.cache where [asyncPreempt] expects to find +// it. This means nothing else may use the cache between this call and the +// return to asyncPreempt. This is not quite symmetric with [xRegSave], which +// uses gp.m.p.xRegs.scratch. By using cache instead, we save a block copy. +// +// This is called with asyncPreempt on the stack and thus must not grow the +// stack. +// +//go:nosplit +func xRegRestore(gp *g) { + if gp.xRegs.state == nil { + throw("gp.xRegState.p == nil on return from async preempt") + } + // If the P has a block cached on it, free that so we can replace it. + pp := gp.m.p.ptr() + if pp.xRegs.cache != nil { + // Don't grow the G stack. + systemstack(func() { + pp.xRegs.free() + }) + } + pp.xRegs.cache = gp.xRegs.state + gp.xRegs.state = nil +} + +func (xRegs *xRegPerP) free() { + if xRegs.cache != nil { + lock(&xRegAlloc.lock) + xRegAlloc.alloc.free(unsafe.Pointer(xRegs.cache)) + xRegs.cache = nil + unlock(&xRegAlloc.lock) + } +} diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 2cf8a31971f951..36949fb2cf0681 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -11,6 +11,7 @@ import ( "internal/goos" "internal/runtime/atomic" "internal/runtime/exithook" + "internal/runtime/strconv" "internal/runtime/sys" "internal/stringslite" "unsafe" @@ -209,6 +210,7 @@ func main() { }() gcenable() + defaultGOMAXPROCSUpdateEnable() // don't STW before runtime initialized. main_init_done = make(chan bool) if iscgo { @@ -250,8 +252,12 @@ func main() { // by package plugin). Run through the modules in dependency // order (the order they are initialized by the dynamic // loader, i.e. they are added to the moduledata linked list). - for m := &firstmoduledata; m != nil; m = m.next { + last := lastmoduledatap // grab before loop starts. Any added modules after this point will do their own doInit calls. + for m := &firstmoduledata; true; m = m.next { doInit(m.inittasks) + if m == last { + break + } } // Disable init tracing after main init done to avoid overhead @@ -266,15 +272,42 @@ func main() { if isarchive || islibrary { // A program compiled with -buildmode=c-archive or c-shared // has a main, but it is not executed. + if GOARCH == "wasm" { + // On Wasm, pause makes it return to the host. + // Unlike cgo callbacks where Ms are created on demand, + // on Wasm we have only one M. So we keep this M (and this + // G) for callbacks. + // Using the caller's SP unwinds this frame and backs to + // goexit. The -16 is: 8 for goexit's (fake) return PC, + // and pause's epilogue pops 8. + pause(sys.GetCallerSP() - 16) // should not return + panic("unreachable") + } return } fn := main_main // make an indirect call, as the linker doesn't know the address of the main package when laying down the runtime fn() + + exitHooksRun := false if raceenabled { runExitHooks(0) // run hooks now, since racefini does not return + exitHooksRun = true racefini() } + // Check for C memory leaks if using ASAN and we've made cgo calls, + // or if we are running as a library in a C program. + // We always make one cgo call, above, to notify_runtime_init_done, + // so we ignore that one. + // No point in leak checking if no cgo calls, since leak checking + // just looks for objects allocated using malloc and friends. + // Just checking iscgo doesn't help because asan implies iscgo. + if asanenabled && (isarchive || islibrary || NumCgoCall() > 1) { + runExitHooks(0) // lsandoleakcheck may not return + exitHooksRun = true + lsandoleakcheck() + } + // Make racy client program work: if panicking on // another goroutine at the same time as main returns, // let the other goroutine finish printing the panic trace. @@ -291,7 +324,9 @@ func main() { if panicking.Load() != 0 { gopark(nil, nil, waitReasonPanicWait, traceBlockForever, 1) } - runExitHooks(0) + if !exitHooksRun { + runExitHooks(0) + } exit(0) for { @@ -308,6 +343,11 @@ func os_beforeExit(exitCode int) { if exitCode == 0 && raceenabled { racefini() } + + // See comment in main, above. + if exitCode == 0 && asanenabled && (isarchive || islibrary || NumCgoCall() > 1) { + lsandoleakcheck() + } } func init() { @@ -477,7 +517,7 @@ func acquireSudog() *sudog { s := pp.sudogcache[n-1] pp.sudogcache[n-1] = nil pp.sudogcache = pp.sudogcache[:n-1] - if s.elem != nil { + if s.elem.get() != nil { throw("acquireSudog: found s.elem != nil in cache") } releasem(mp) @@ -486,7 +526,7 @@ func acquireSudog() *sudog { //go:nosplit func releaseSudog(s *sudog) { - if s.elem != nil { + if s.elem.get() != nil { throw("runtime: sudog with non-nil elem") } if s.isSelect { @@ -501,7 +541,7 @@ func releaseSudog(s *sudog) { if s.waitlink != nil { throw("runtime: sudog with non-nil waitlink") } - if s.c != nil { + if s.c.get() != nil { throw("runtime: sudog with non-nil c") } gp := getg() @@ -720,10 +760,6 @@ const ( // cpuinit sets up CPU feature flags and calls internal/cpu.Initialize. env should be the complete // value of the GODEBUG environment variable. func cpuinit(env string) { - switch GOOS { - case "aix", "darwin", "ios", "dragonfly", "freebsd", "netbsd", "openbsd", "illumos", "solaris", "linux": - cpu.DebugOptions = true - } cpu.Initialize(env) // Support cpu feature variables are used in code generated by the compiler @@ -739,13 +775,23 @@ func cpuinit(env string) { case "arm64": arm64HasATOMICS = cpu.ARM64.HasATOMICS + + case "loong64": + loong64HasLAMCAS = cpu.Loong64.HasLAMCAS + loong64HasLAM_BH = cpu.Loong64.HasLAM_BH + loong64HasLSX = cpu.Loong64.HasLSX + + case "riscv64": + riscv64HasZbb = cpu.RISCV64.HasZbb } } // getGodebugEarly extracts the environment variable GODEBUG from the environment on // Unix-like operating systems and returns it. This function exists to extract GODEBUG // early before much of the runtime is initialized. -func getGodebugEarly() string { +// +// Returns nil, false if OS doesn't provide env vars early in the init sequence. +func getGodebugEarly() (string, bool) { const prefix = "GODEBUG=" var env string switch GOOS { @@ -763,12 +809,16 @@ func getGodebugEarly() string { s := unsafe.String(p, findnull(p)) if stringslite.HasPrefix(s, prefix) { - env = gostring(p)[len(prefix):] + env = gostringnocopy(p)[len(prefix):] break } } + break + + default: + return "", false } - return env + return env, true } // The bootstrap sequence is: @@ -791,6 +841,7 @@ func schedinit() { lockInit(&reflectOffs.lock, lockRankReflectOffs) lockInit(&finlock, lockRankFin) lockInit(&cpuprof.lock, lockRankCpuprof) + lockInit(&computeMaxProcsLock, lockRankComputeMaxProcs) allocmLock.init(lockRankAllocmR, lockRankAllocmRInternal, lockRankAllocmW) execLock.init(lockRankExecR, lockRankExecRInternal, lockRankExecW) traceLockInit() @@ -799,6 +850,8 @@ func schedinit() { // extremely short. lockInit(&memstats.heapStats.noPLock, lockRankLeafRank) + lockVerifyMSize() + // raceinit must be the first call to race detector. // In particular, it must be done before mallocinit below calls racemapshadow. gp := getg() @@ -812,13 +865,16 @@ func schedinit() { // The world starts stopped. worldStopped() + godebug, parsedGodebug := getGodebugEarly() + if parsedGodebug { + parseRuntimeDebugVars(godebug) + } ticks.init() // run as early as possible moduledataverify() stackinit() + randinit() // must run before mallocinit, alginit, mcommoninit mallocinit() - godebug := getGodebugEarly() cpuinit(godebug) // must run before alginit - randinit() // must run before alginit, mcommoninit alginit() // maps, hash, rand must not be used before this call mcommoninit(gp.m, -1) modulesinit() // provides activeModules @@ -833,7 +889,12 @@ func schedinit() { goenvs() secure() checkfds() - parsedebugvars() + if !parsedGodebug { + // Some platforms, e.g., Windows, didn't make env vars available "early", + // so try again now. + parseRuntimeDebugVars(gogetenv("GODEBUG")) + } + finishDebugVarsSetup() gcinit() // Allocate stack space that can be used when crashing due to bad stack @@ -852,12 +913,24 @@ func schedinit() { // mcommoninit runs before parsedebugvars, so init profstacks again. mProfStackInit(gp.m) + defaultGOMAXPROCSInit() lock(&sched.lock) sched.lastpoll.Store(nanotime()) - procs := ncpu - if n, ok := atoi32(gogetenv("GOMAXPROCS")); ok && n > 0 { + var procs int32 + if n, ok := strconv.Atoi32(gogetenv("GOMAXPROCS")); ok && n > 0 { procs = n + sched.customGOMAXPROCS = true + } else { + // Use numCPUStartup for initial GOMAXPROCS for two reasons: + // + // 1. We just computed it in osinit, recomputing is (minorly) wasteful. + // + // 2. More importantly, if debug.containermaxprocs == 0 && + // debug.updatemaxprocs == 0, we want to guarantee that + // runtime.GOMAXPROCS(0) always equals runtime.NumCPU (which is + // just numCPUStartup). + procs = defaultGOMAXPROCS(numCPUStartup) } if procresize(procs) != nil { throw("unknown runnable goroutine during bootstrap") @@ -948,7 +1021,7 @@ func mcommoninit(mp *m, id int64) { // when it is just in a register or thread-local storage. mp.alllink = allm - // NumCgoCall() and others iterate over allm w/o schedlock, + // NumCgoCall and others iterate over allm w/o schedlock, // so we need to publish it safely. atomicstorep(unsafe.Pointer(&allm), unsafe.Pointer(mp)) unlock(&sched.lock) @@ -1000,6 +1073,28 @@ func (mp *m) becomeSpinning() { sched.needspinning.Store(0) } +// Take a snapshot of allp, for use after dropping the P. +// +// Must be called with a P, but the returned slice may be used after dropping +// the P. The M holds a reference on the snapshot to keep the backing array +// alive. +// +//go:yeswritebarrierrec +func (mp *m) snapshotAllp() []*p { + mp.allpSnapshot = allp + return mp.allpSnapshot +} + +// Clear the saved allp snapshot. Should be called as soon as the snapshot is +// no longer required. +// +// Must be called after reacquiring a P, as it requires a write barrier. +// +//go:yeswritebarrierrec +func (mp *m) clearAllpSnapshot() { + mp.allpSnapshot = nil +} + func (mp *m) hasCgoOnStack() bool { return mp.ncgo > 0 || mp.isextra } @@ -1127,7 +1222,9 @@ func casfrom_Gscanstatus(gp *g, oldval, newval uint32) { _Gscanwaiting, _Gscanrunning, _Gscansyscall, - _Gscanpreempted: + _Gscanleaked, + _Gscanpreempted, + _Gscandeadextra: if newval == oldval&^_Gscan { success = gp.atomicstatus.CompareAndSwap(oldval, newval) } @@ -1147,7 +1244,9 @@ func castogscanstatus(gp *g, oldval, newval uint32) bool { case _Grunnable, _Grunning, _Gwaiting, - _Gsyscall: + _Gleaked, + _Gsyscall, + _Gdeadextra: if newval == oldval|_Gscan { r := gp.atomicstatus.CompareAndSwap(oldval, newval) if r { @@ -1211,6 +1310,12 @@ func casgstatus(gp *g, oldval, newval uint32) { } } + if gp.bubble != nil { + systemstack(func() { + gp.bubble.changegstatus(gp, oldval, newval) + }) + } + if oldval == _Grunning { // Track every gTrackingPeriod time a goroutine transitions out of running. if casgstatusAlwaysTrack || gp.trackingSeq%gTrackingPeriod == 0 { @@ -1282,36 +1387,20 @@ func casGToWaiting(gp *g, old uint32, reason waitReason) { casgstatus(gp, old, _Gwaiting) } -// casGToWaitingForGC transitions gp from old to _Gwaiting, and sets the wait reason. -// The wait reason must be a valid isWaitingForGC wait reason. +// casGToWaitingForSuspendG transitions gp from old to _Gwaiting, and sets the wait reason. +// The wait reason must be a valid isWaitingForSuspendG wait reason. +// +// While a goroutine is in this state, it's stack is effectively pinned. +// The garbage collector must not shrink or otherwise mutate the goroutine's stack. // // Use this over casgstatus when possible to ensure that a waitreason is set. -func casGToWaitingForGC(gp *g, old uint32, reason waitReason) { - if !reason.isWaitingForGC() { - throw("casGToWaitingForGC with non-isWaitingForGC wait reason") +func casGToWaitingForSuspendG(gp *g, old uint32, reason waitReason) { + if !reason.isWaitingForSuspendG() { + throw("casGToWaitingForSuspendG with non-isWaitingForSuspendG wait reason") } casGToWaiting(gp, old, reason) } -// casgstatus(gp, oldstatus, Gcopystack), assuming oldstatus is Gwaiting or Grunnable. -// Returns old status. Cannot call casgstatus directly, because we are racing with an -// async wakeup that might come in from netpoll. If we see Gwaiting from the readgstatus, -// it might have become Grunnable by the time we get to the cas. If we called casgstatus, -// it would loop waiting for the status to go back to Gwaiting, which it never will. -// -//go:nosplit -func casgcopystack(gp *g) uint32 { - for { - oldstatus := readgstatus(gp) &^ _Gscan - if oldstatus != _Gwaiting && oldstatus != _Grunnable { - throw("copystack: bad status, not Gwaiting or Grunnable") - } - if gp.atomicstatus.CompareAndSwap(oldstatus, _Gcopystack) { - return oldstatus - } - } -} - // casGToPreemptScan transitions gp from _Grunning to _Gscan|_Gpreempted. // // TODO(austin): This is the only status operation that both changes @@ -1323,6 +1412,12 @@ func casGToPreemptScan(gp *g, old, new uint32) { acquireLockRankAndM(lockRankGscan) for !gp.atomicstatus.CompareAndSwap(_Grunning, _Gscan|_Gpreempted) { } + // We never notify gp.bubble that the goroutine state has moved + // from _Grunning to _Gpreempted. We call bubble.changegstatus + // after status changes happen, but doing so here would violate the + // ordering between the gscan and synctest locks. The bubble doesn't + // distinguish between _Grunning and _Gpreempted anyway, so not + // notifying it is fine. } // casGFromPreempted attempts to transition gp from _Gpreempted to @@ -1333,7 +1428,13 @@ func casGFromPreempted(gp *g, old, new uint32) bool { throw("bad g transition") } gp.waitreason = waitReasonPreempted - return gp.atomicstatus.CompareAndSwap(_Gpreempted, _Gwaiting) + if !gp.atomicstatus.CompareAndSwap(_Gpreempted, _Gwaiting) { + return false + } + if bubble := gp.bubble; bubble != nil { + bubble.changegstatus(gp, _Gpreempted, _Gwaiting) + } + return true } // stwReason is an enumeration of reasons the world is stopping. @@ -1429,23 +1530,7 @@ func stopTheWorld(reason stwReason) worldStop { gp := getg() gp.m.preemptoff = reason.String() systemstack(func() { - // Mark the goroutine which called stopTheWorld preemptible so its - // stack may be scanned. - // This lets a mark worker scan us while we try to stop the world - // since otherwise we could get in a mutual preemption deadlock. - // We must not modify anything on the G stack because a stack shrink - // may occur. A stack shrink is otherwise OK though because in order - // to return from this function (and to leave the system stack) we - // must have preempted all goroutines, including any attempting - // to scan our stack, in which case, any stack shrinking will - // have already completed by the time we exit. - // - // N.B. The execution tracer is not aware of this status - // transition and handles it specially based on the - // wait reason. - casGToWaitingForGC(gp, _Grunning, waitReasonStoppingTheWorld) stopTheWorldContext = stopTheWorldWithSema(reason) // avoid write to stack - casgstatus(gp, _Gwaiting, _Grunning) }) return stopTheWorldContext } @@ -1534,7 +1619,23 @@ var gcsema uint32 = 1 // // Returns the STW context. When starting the world, this context must be // passed to startTheWorldWithSema. +// +//go:systemstack func stopTheWorldWithSema(reason stwReason) worldStop { + // Mark the goroutine which called stopTheWorld preemptible so its + // stack may be scanned by the GC or observed by the execution tracer. + // + // This lets a mark worker scan us or the execution tracer take our + // stack while we try to stop the world since otherwise we could get + // in a mutual preemption deadlock. + // + // casGToWaitingForSuspendG marks the goroutine as ineligible for a + // stack shrink, effectively pinning the stack in memory for the duration. + // + // N.B. The execution tracer is not aware of this status transition and + // handles it specially based on the wait reason. + casGToWaitingForSuspendG(getg().m.curg, _Grunning, waitReasonStoppingTheWorld) + trace := traceAcquire() if trace.ok() { trace.STWStart(reason) @@ -1565,6 +1666,7 @@ func stopTheWorldWithSema(reason stwReason) worldStop { if trace.ok() { trace.ProcSteal(pp, false) } + sched.nGsyscallNoP.Add(1) pp.syscalltick++ pp.gcStopTime = nanotime() sched.stopwait-- @@ -1642,6 +1744,9 @@ func stopTheWorldWithSema(reason stwReason) worldStop { worldStopped() + // Switch back to _Grunning, now that the world is stopped. + casgstatus(getg().m.curg, _Gwaiting, _Grunning) + return worldStop{ reason: reason, startedStopping: start, @@ -1729,10 +1834,8 @@ func startTheWorldWithSema(now int64, w worldStop) int64 { // via libcall. func usesLibcall() bool { switch GOOS { - case "aix", "darwin", "illumos", "ios", "solaris", "windows": + case "aix", "darwin", "illumos", "ios", "openbsd", "solaris", "windows": return true - case "openbsd": - return GOARCH != "mips64" } return false } @@ -1741,10 +1844,8 @@ func usesLibcall() bool { // system-allocated stack. func mStackIsSystemAllocated() bool { switch GOOS { - case "aix", "darwin", "plan9", "illumos", "ios", "solaris", "windows": + case "aix", "darwin", "plan9", "illumos", "ios", "openbsd", "solaris", "windows": return true - case "openbsd": - return GOARCH != "mips64" } return false } @@ -1800,7 +1901,7 @@ func mstart0() { mexit(osStack) } -// The go:noinline is to guarantee the getcallerpc/getcallersp below are safe, +// The go:noinline is to guarantee the sys.GetCallerPC/sys.GetCallerSP below are safe, // so that we can set up g0.sched to return to the call of mstart1 above. // //go:noinline @@ -1818,8 +1919,8 @@ func mstart1() { // And goexit0 does a gogo that needs to return from mstart1 // and let mstart0 exit the thread. gp.sched.g = guintptr(unsafe.Pointer(gp)) - gp.sched.pc = getcallerpc() - gp.sched.sp = getcallersp() + gp.sched.pc = sys.GetCallerPC() + gp.sched.sp = sys.GetCallerSP() asminit() minit() @@ -1830,6 +1931,10 @@ func mstart1() { mstartm0() } + if debug.dataindependenttiming == 1 { + sys.EnableDIT() + } + if fn := gp.m.mstartfn; fn != nil { fn() } @@ -1907,6 +2012,10 @@ func mexit(osStack bool) { // Free the gsignal stack. if mp.gsignal != nil { stackfree(mp.gsignal.stack) + if valgrindenabled { + valgrindDeregisterStack(mp.gsignal.valgrindStackID) + mp.gsignal.valgrindStackID = 0 + } // On some platforms, when calling into VDSO (e.g. nanotime) // we store our g on the gsignal stack, if there is one. // Now the stack is freed, unlink it from the m, so we @@ -1914,6 +2023,9 @@ func mexit(osStack bool) { mp.gsignal = nil } + // Free vgetrandom state. + vgetrandomDestroy(mp) + // Remove m from allm. lock(&sched.lock) for pprev := &allm; *pprev != nil; pprev = &(*pprev).alllink { @@ -1999,15 +2111,18 @@ found: func forEachP(reason waitReason, fn func(*p)) { systemstack(func() { gp := getg().m.curg - // Mark the user stack as preemptible so that it may be scanned. - // Otherwise, our attempt to force all P's to a safepoint could - // result in a deadlock as we attempt to preempt a worker that's - // trying to preempt us (e.g. for a stack scan). + // Mark the user stack as preemptible so that it may be scanned + // by the GC or observed by the execution tracer. Otherwise, our + // attempt to force all P's to a safepoint could result in a + // deadlock as we attempt to preempt a goroutine that's trying + // to preempt us (e.g. for a stack scan). // - // N.B. The execution tracer is not aware of this status - // transition and handles it specially based on the - // wait reason. - casGToWaitingForGC(gp, _Grunning, reason) + // casGToWaitingForSuspendG marks the goroutine as ineligible for a + // stack shrink, effectively pinning the stack in memory for the duration. + // + // N.B. The execution tracer is not aware of this status transition and + // handles it specially based on the wait reason. + casGToWaitingForSuspendG(gp, _Grunning, reason) forEachPInternal(fn) casgstatus(gp, _Gwaiting, _Grunning) }) @@ -2074,6 +2189,7 @@ func forEachPInternal(fn func(*p)) { trace.ProcSteal(p2, false) traceRelease(trace) } + sched.nGsyscallNoP.Add(1) p2.syscalltick++ handoffp(p2) } else if trace.ok() { @@ -2201,6 +2317,10 @@ func allocm(pp *p, fn func(), id int64) *m { // startm. systemstack(func() { stackfree(freem.g0.stack) + if valgrindenabled { + valgrindDeregisterStack(freem.g0.valgrindStackID) + freem.g0.valgrindStackID = 0 + } }) } freem = freem.freelink @@ -2209,7 +2329,7 @@ func allocm(pp *p, fn func(), id int64) *m { unlock(&sched.lock) } - mp := new(m) + mp := &new(mPadded).m mp.mstartfn = fn mcommoninit(mp, id) @@ -2318,7 +2438,7 @@ func needm(signal bool) { // Install g (= m->g0) and set the stack bounds // to match the current stack. setg(mp.g0) - sp := getcallersp() + sp := sys.GetCallerSP() callbackUpdateSystemStack(mp, sp, signal) // Should mark we are already in Go now. @@ -2341,8 +2461,9 @@ func needm(signal bool) { } // mp.curg is now a real goroutine. - casgstatus(mp.curg, _Gdead, _Gsyscall) + casgstatus(mp.curg, _Gdeadextra, _Gsyscall) sched.ngsys.Add(-1) + sched.nGsyscallNoP.Add(1) if !signal { if trace.ok() { @@ -2396,11 +2517,10 @@ func oneNewExtraM() { gp.syscallpc = gp.sched.pc gp.syscallsp = gp.sched.sp gp.stktopsp = gp.sched.sp - // malg returns status as _Gidle. Change to _Gdead before - // adding to allg where GC can see it. We use _Gdead to hide - // this from tracebacks and stack scans since it isn't a - // "real" goroutine until needm grabs it. - casgstatus(gp, _Gidle, _Gdead) + // malg returns status as _Gidle. Change to _Gdeadextra before + // adding to allg where GC can see it. _Gdeadextra hides this + // from traceback and stack scans. + casgstatus(gp, _Gidle, _Gdeadextra) gp.m = mp mp.curg = gp mp.isextra = true @@ -2474,10 +2594,11 @@ func dropm() { trace = traceAcquire() } - // Return mp.curg to dead state. - casgstatus(mp.curg, _Gsyscall, _Gdead) + // Return mp.curg to _Gdeadextra state. + casgstatus(mp.curg, _Gsyscall, _Gdeadextra) mp.curg.preemptStop = false sched.ngsys.Add(1) + sched.nGsyscallNoP.Add(-1) if !mp.isExtraInSig { if trace.ok() { @@ -2539,6 +2660,7 @@ func dropm() { g0.stack.lo = 0 g0.stackguard0 = 0 g0.stackguard1 = 0 + mp.g0StackAccurate = false putExtraM(mp) @@ -3010,7 +3132,7 @@ func handoffp(pp *p) { // findrunnable would return a G to run on pp. // if it has local work, start it straight away - if !runqempty(pp) || sched.runqsize != 0 { + if !runqempty(pp) || !sched.runq.empty() { startm(pp, false, false) return } @@ -3020,7 +3142,7 @@ func handoffp(pp *p) { return } // if it has GC work, start it straight away - if gcBlackenEnabled != 0 && gcMarkWorkAvailable(pp) { + if gcBlackenEnabled != 0 && gcShouldScheduleWorker(pp) { startm(pp, false, false) return } @@ -3049,7 +3171,7 @@ func handoffp(pp *p) { notewakeup(&sched.safePointNote) } } - if sched.runqsize != 0 { + if !sched.runq.empty() { unlock(&sched.lock) startm(pp, false, false) return @@ -3215,10 +3337,10 @@ func execute(gp *g, inheritTime bool) { tryRecordGoroutineProfile(gp, nil, osyield) } - // Assign gp.m before entering _Grunning so running Gs have an - // M. + // Assign gp.m before entering _Grunning so running Gs have an M. mp.curg = gp gp.m = mp + gp.syncSafePoint = false // Clear the flag, which may have been set by morestack. casgstatus(gp, _Grunnable, _Grunning) gp.waitsince = 0 gp.preempt = false @@ -3254,6 +3376,11 @@ func findRunnable() (gp *g, inheritTime, tryWakeP bool) { // an M. top: + // We may have collected an allp snapshot below. The snapshot is only + // required in each loop iteration. Clear it to all GC to collect the + // slice. + mp.clearAllpSnapshot() + pp := mp.p.ptr() if sched.gcwaiting.Load() { gcstopm() @@ -3267,7 +3394,7 @@ top: // which may steal timers. It's important that between now // and then, nothing blocks, so these numbers remain mostly // relevant. - now, pollUntil, _ := pp.timers.check(0) + now, pollUntil, _ := pp.timers.check(0, nil) // Try to schedule the trace reader. if traceEnabled() || traceShuttingDown() { @@ -3295,9 +3422,9 @@ top: // Check the global runnable queue once in a while to ensure fairness. // Otherwise two goroutines can completely occupy the local runqueue // by constantly respawning each other. - if pp.schedtick%61 == 0 && sched.runqsize > 0 { + if pp.schedtick%61 == 0 && !sched.runq.empty() { lock(&sched.lock) - gp := globrunqget(pp, 1) + gp := globrunqget() unlock(&sched.lock) if gp != nil { return gp, false, false @@ -3310,6 +3437,12 @@ top: ready(gp, 0, true) } } + + // Wake up one or more cleanup Gs. + if gcCleanups.needsWake() { + gcCleanups.wake() + } + if *cgo_yield != nil { asmcgocall(*cgo_yield, nil) } @@ -3320,11 +3453,14 @@ top: } // global runq - if sched.runqsize != 0 { + if !sched.runq.empty() { lock(&sched.lock) - gp := globrunqget(pp, 0) + gp, q := globrunqgetbatch(int32(len(pp.runq)) / 2) unlock(&sched.lock) if gp != nil { + if runqputbatch(pp, &q); !q.empty() { + throw("Couldn't put Gs into empty local runq") + } return gp, false, false } } @@ -3336,8 +3472,12 @@ top: // blocked thread (e.g. it has already returned from netpoll, but does // not set lastpoll yet), this thread will do blocking netpoll below // anyway. - if netpollinited() && netpollAnyWaiters() && sched.lastpoll.Load() != 0 { - if list, delta := netpoll(0); !list.empty() { // non-blocking + // We only poll from one thread at a time to avoid kernel contention + // on machines with many cores. + if netpollinited() && netpollAnyWaiters() && sched.lastpoll.Load() != 0 && sched.pollingNet.Swap(1) == 0 { + list, delta := netpoll(0) + sched.pollingNet.Store(0) + if !list.empty() { // non-blocking gp := list.pop() injectglist(&list) netpollAdjustWaiters(delta) @@ -3383,7 +3523,7 @@ top: // // If we're in the GC mark phase, can safely scan and blacken objects, // and have work to do, run idle-time marking rather than give up the P. - if gcBlackenEnabled != 0 && gcMarkWorkAvailable(pp) && gcController.addIdleMarkWorker() { + if gcBlackenEnabled != 0 && gcShouldScheduleWorker(pp) && gcController.addIdleMarkWorker() { node := (*gcBgMarkWorkerNode)(gcBgMarkWorkerPool.pop()) if node != nil { pp.gcMarkWorkerMode = gcMarkWorkerIdleMode @@ -3422,7 +3562,11 @@ top: // which can change underfoot once we no longer block // safe-points. We don't need to snapshot the contents because // everything up to cap(allp) is immutable. - allpSnapshot := allp + // + // We clear the snapshot from the M after return via + // mp.clearAllpSnapshop (in schedule) and on each iteration of the top + // loop. + allpSnapshot := mp.snapshotAllp() // Also snapshot masks. Value changes are OK, but we can't allow // len to change out from under us. idlepMaskSnapshot := idlepMask @@ -3434,9 +3578,15 @@ top: unlock(&sched.lock) goto top } - if sched.runqsize != 0 { - gp := globrunqget(pp, 0) + if !sched.runq.empty() { + gp, q := globrunqgetbatch(int32(len(pp.runq)) / 2) unlock(&sched.lock) + if gp == nil { + throw("global runq empty with non-zero runqsize") + } + if runqputbatch(pp, &q); !q.empty() { + throw("Couldn't put Gs into empty local runq") + } return gp, false, false } if !mp.spinning && sched.needspinning.Load() == 1 { @@ -3506,14 +3656,17 @@ top: // Check global and P runqueues again. lock(&sched.lock) - if sched.runqsize != 0 { + if !sched.runq.empty() { pp, _ := pidlegetSpinning(0) if pp != nil { - gp := globrunqget(pp, 0) + gp, q := globrunqgetbatch(int32(len(pp.runq)) / 2) + unlock(&sched.lock) if gp == nil { throw("global runq empty with non-zero runqsize") } - unlock(&sched.lock) + if runqputbatch(pp, &q); !q.empty() { + throw("Couldn't put Gs into empty local runq") + } acquirep(pp) mp.becomeSpinning() return gp, false, false @@ -3554,6 +3707,9 @@ top: pollUntil = checkTimersNoP(allpSnapshot, timerpMaskSnapshot, pollUntil) } + // We don't need allp anymore at this pointer, but can't clear the + // snapshot without a P for the write barrier.. + // Poll network until next timer. if netpollinited() && (netpollAnyWaiters() || pollUntil != 0) && sched.lastpoll.Swap(0) != 0 { sched.pollUntil.Store(pollUntil) @@ -3628,7 +3784,7 @@ top: // background work loops, like idle GC. It checks a subset of the // conditions checked by the actual scheduler. func pollWork() bool { - if sched.runqsize != 0 { + if !sched.runq.empty() { return true } p := getg().m.p.ptr() @@ -3684,7 +3840,7 @@ func stealWork(now int64) (gp *g, inheritTime bool, rnow, pollUntil int64, newWo // timerpMask tells us whether the P may have timers at all. If it // can't, no need to check at all. if stealTimersOrRunNextG && timerpMask.read(enum.position()) { - tnow, w, ran := p2.timers.check(now) + tnow, w, ran := p2.timers.check(now, nil) now = tnow if w != 0 && (pollUntil == 0 || w < pollUntil) { pollUntil = w @@ -3774,7 +3930,7 @@ func checkIdleGCNoP() (*p, *g) { if atomic.Load(&gcBlackenEnabled) == 0 || !gcController.needIdleMarkWorker() { return nil, nil } - if !gcMarkWorkAvailable(nil) { + if !gcShouldScheduleWorker(nil) { return nil, nil } @@ -3872,33 +4028,28 @@ func injectglist(glist *gList) { if glist.empty() { return } - trace := traceAcquire() - if trace.ok() { - for gp := glist.head.ptr(); gp != nil; gp = gp.schedlink.ptr() { - trace.GoUnpark(gp, 0) - } - traceRelease(trace) - } // Mark all the goroutines as runnable before we put them // on the run queues. - head := glist.head.ptr() var tail *g - qsize := 0 - for gp := head; gp != nil; gp = gp.schedlink.ptr() { + trace := traceAcquire() + for gp := glist.head.ptr(); gp != nil; gp = gp.schedlink.ptr() { tail = gp - qsize++ casgstatus(gp, _Gwaiting, _Grunnable) + if trace.ok() { + trace.GoUnpark(gp, 0) + } + } + if trace.ok() { + traceRelease(trace) } // Turn the gList into a gQueue. - var q gQueue - q.head.set(head) - q.tail.set(tail) + q := gQueue{glist.head, tail.guintptr(), glist.size} *glist = gList{} - startIdle := func(n int) { - for i := 0; i < n; i++ { + startIdle := func(n int32) { + for ; n > 0; n-- { mp := acquirem() // See comment in startm. lock(&sched.lock) @@ -3917,32 +4068,32 @@ func injectglist(glist *gList) { pp := getg().m.p.ptr() if pp == nil { + n := q.size lock(&sched.lock) - globrunqputbatch(&q, int32(qsize)) + globrunqputbatch(&q) unlock(&sched.lock) - startIdle(qsize) + startIdle(n) return } - npidle := int(sched.npidle.Load()) - var ( - globq gQueue - n int - ) - for n = 0; n < npidle && !q.empty(); n++ { + var globq gQueue + npidle := sched.npidle.Load() + for ; npidle > 0 && !q.empty(); npidle-- { g := q.pop() globq.pushBack(g) } - if n > 0 { + if !globq.empty() { + n := globq.size lock(&sched.lock) - globrunqputbatch(&globq, int32(n)) + globrunqputbatch(&globq) unlock(&sched.lock) startIdle(n) - qsize -= n } - if !q.empty() { - runqputbatch(pp, &q, qsize) + if runqputbatch(pp, &q); !q.empty() { + lock(&sched.lock) + globrunqputbatch(&q) + unlock(&sched.lock) } // Some P's might have become idle after we loaded `sched.npidle` @@ -3994,6 +4145,11 @@ top: gp, inheritTime, tryWakeP := findRunnable() // blocks until work is available + // findRunnable may have collected an allp snapshot. The snapshot is + // only required within findRunnable. Clear it to all GC to collect the + // slice. + mp.clearAllpSnapshot() + if debug.dontfreezetheworld > 0 && freezing.Load() { // See comment in freezetheworld. We don't want to perturb // scheduler state, so we didn't gcstopm in findRunnable, but @@ -4024,7 +4180,6 @@ top: unlock(&sched.lock) } else { sched.disable.runnable.pushBack(gp) - sched.disable.n++ unlock(&sched.lock) goto top } @@ -4070,6 +4225,15 @@ func park_m(gp *g) { trace := traceAcquire() + // If g is in a synctest group, we don't want to let the group + // become idle until after the waitunlockf (if any) has confirmed + // that the park is happening. + // We need to record gp.bubble here, since waitunlockf can change it. + bubble := gp.bubble + if bubble != nil { + bubble.incActive() + } + if trace.ok() { // Trace the event before the transition. It may take a // stack trace, but we won't own the stack after the @@ -4092,6 +4256,9 @@ func park_m(gp *g) { if !ok { trace := traceAcquire() casgstatus(gp, _Gwaiting, _Grunnable) + if bubble != nil { + bubble.decActive() + } if trace.ok() { trace.GoUnpark(gp, 2) traceRelease(trace) @@ -4099,6 +4266,11 @@ func park_m(gp *g) { execute(gp, true) // Schedule it back, never returns. } } + + if bubble != nil { + bubble.decActive() + } + schedule() } @@ -4252,6 +4424,9 @@ func goyield_m(gp *g) { // Finishes execution of the current goroutine. func goexit1() { if raceenabled { + if gp := getg(); gp.bubble != nil { + racereleasemergeg(gp, gp.bubble.raceaddr()) + } racegoend() } trace := traceAcquire() @@ -4290,6 +4465,7 @@ func gdestroy(gp *g) { gp.param = nil gp.labels = nil gp.timer = nil + gp.bubble = nil if gcBlackenEnabled != 0 && gp.gcAssistBytes > 0 { // Flush assist credit to the global pool. This gives @@ -4310,6 +4486,9 @@ func gdestroy(gp *g) { if locked && mp.lockedInt != 0 { print("runtime: mp.lockedInt = ", mp.lockedInt, "\n") + if mp.isextra { + throw("runtime.Goexit called in a thread that was not created by the Go runtime") + } throw("exited a goroutine internally locked to the OS thread") } gfput(pp, gp) @@ -4353,7 +4532,6 @@ func save(pc, sp, bp uintptr) { gp.sched.pc = pc gp.sched.sp = sp gp.sched.lr = 0 - gp.sched.ret = 0 gp.sched.bp = bp // We need to ensure ctxt is zero, but can't have a write // barrier here. However, it should always already be zero. @@ -4415,7 +4593,13 @@ func reentersyscall(pc, sp, bp uintptr) { } if gp.syscallsp < gp.stack.lo || gp.stack.hi < gp.syscallsp { systemstack(func() { - print("entersyscall inconsistent ", hex(gp.syscallsp), " [", hex(gp.stack.lo), ",", hex(gp.stack.hi), "]\n") + print("entersyscall inconsistent sp ", hex(gp.syscallsp), " [", hex(gp.stack.lo), ",", hex(gp.stack.hi), "]\n") + throw("entersyscall") + }) + } + if gp.syscallbp != 0 && gp.syscallbp < gp.stack.lo || gp.stack.hi < gp.syscallbp { + systemstack(func() { + print("entersyscall inconsistent bp ", hex(gp.syscallbp), " [", hex(gp.stack.lo), ",", hex(gp.stack.hi), "]\n") throw("entersyscall") }) } @@ -4476,7 +4660,7 @@ func entersyscall() { // the stack. This results in exceeding the nosplit stack requirements // on some platforms. fp := getcallerfp() - reentersyscall(getcallerpc(), getcallersp(), fp) + reentersyscall(sys.GetCallerPC(), sys.GetCallerSP(), fp) } func entersyscall_sysmon() { @@ -4508,6 +4692,7 @@ func entersyscall_gcwait() { trace.ProcSteal(pp, true) traceRelease(trace) } + sched.nGsyscallNoP.Add(1) pp.gcStopTime = nanotime() pp.syscalltick++ if sched.stopwait--; sched.stopwait == 0 { @@ -4540,9 +4725,11 @@ func entersyscallblock() { gp.m.syscalltick = gp.m.p.ptr().syscalltick gp.m.p.ptr().syscalltick++ + sched.nGsyscallNoP.Add(1) + // Leave SP around for GC and traceback. - pc := getcallerpc() - sp := getcallersp() + pc := sys.GetCallerPC() + sp := sys.GetCallerSP() bp := getcallerfp() save(pc, sp, bp) gp.syscallsp = gp.sched.sp @@ -4553,14 +4740,20 @@ func entersyscallblock() { sp2 := gp.sched.sp sp3 := gp.syscallsp systemstack(func() { - print("entersyscallblock inconsistent ", hex(sp1), " ", hex(sp2), " ", hex(sp3), " [", hex(gp.stack.lo), ",", hex(gp.stack.hi), "]\n") + print("entersyscallblock inconsistent sp ", hex(sp1), " ", hex(sp2), " ", hex(sp3), " [", hex(gp.stack.lo), ",", hex(gp.stack.hi), "]\n") throw("entersyscallblock") }) } casgstatus(gp, _Grunning, _Gsyscall) if gp.syscallsp < gp.stack.lo || gp.stack.hi < gp.syscallsp { systemstack(func() { - print("entersyscallblock inconsistent ", hex(sp), " ", hex(gp.sched.sp), " ", hex(gp.syscallsp), " [", hex(gp.stack.lo), ",", hex(gp.stack.hi), "]\n") + print("entersyscallblock inconsistent sp ", hex(sp), " ", hex(gp.sched.sp), " ", hex(gp.syscallsp), " [", hex(gp.stack.lo), ",", hex(gp.stack.hi), "]\n") + throw("entersyscallblock") + }) + } + if gp.syscallbp != 0 && gp.syscallbp < gp.stack.lo || gp.stack.hi < gp.syscallbp { + systemstack(func() { + print("entersyscallblock inconsistent bp ", hex(bp), " ", hex(gp.sched.bp), " ", hex(gp.syscallbp), " [", hex(gp.stack.lo), ",", hex(gp.stack.hi), "]\n") throw("entersyscallblock") }) } @@ -4568,7 +4761,7 @@ func entersyscallblock() { systemstack(entersyscallblock_handoff) // Resave for traceback during blocked call. - save(getcallerpc(), getcallersp(), getcallerfp()) + save(sys.GetCallerPC(), sys.GetCallerSP(), getcallerfp()) gp.m.locks-- } @@ -4606,7 +4799,7 @@ func exitsyscall() { gp := getg() gp.m.locks++ // see comment in entersyscall - if getcallersp() > gp.syscallsp { + if sys.GetCallerSP() > gp.syscallsp { throw("exitsyscall: syscall frame is no longer valid") } @@ -4635,7 +4828,7 @@ func exitsyscall() { trace.GoSysExit(lostP) if lostP { // We lost the P at some point, even though we got it back here. - // Trace that we're starting again, because there was a traceGoSysBlock + // Trace that we're starting again, because there was a tracev2.GoSysBlock // call somewhere in exitsyscallfast (indicating that this goroutine // had blocked) and we're about to start running again. trace.GoStart() @@ -4732,7 +4925,7 @@ func exitsyscallfast_reacquired(trace traceLocker) { if gp.m.syscalltick != gp.m.p.ptr().syscalltick { if trace.ok() { // The p was retaken and then enter into syscall again (since gp.m.syscalltick has changed). - // traceGoSysBlock for this syscall was already emitted, + // tracev2.GoSysBlock for this syscall was already emitted, // but here we effectively retake the p from the new syscall running on the same p. systemstack(func() { // We're stealing the P. It's treated @@ -4754,6 +4947,7 @@ func exitsyscallfast_pidle() bool { } unlock(&sched.lock) if pp != nil { + sched.nGsyscallNoP.Add(-1) acquirep(pp) return true } @@ -4780,6 +4974,7 @@ func exitsyscall0(gp *g) { trace.GoSysExit(true) traceRelease(trace) } + sched.nGsyscallNoP.Add(-1) dropg() lock(&sched.lock) var pp *p @@ -4822,7 +5017,6 @@ func exitsyscall0(gp *g) { // syscall_runtime_BeforeFork is for package syscall, // but widely used packages access it using linkname. // Notable members of the hall of shame include: -// - github.com/containerd/containerd // - gvisor.dev/gvisor // // Do not remove or change the type signature. @@ -4852,7 +5046,6 @@ func syscall_runtime_BeforeFork() { // syscall_runtime_AfterFork is for package syscall, // but widely used packages access it using linkname. // Notable members of the hall of shame include: -// - github.com/containerd/containerd // - gvisor.dev/gvisor // // Do not remove or change the type signature. @@ -4886,7 +5079,6 @@ var inForkedChild bool // syscall_runtime_AfterForkInChild is for package syscall, // but widely used packages access it using linkname. // Notable members of the hall of shame include: -// - github.com/containerd/containerd // - gvisor.dev/gvisor // // Do not remove or change the type signature. @@ -4946,6 +5138,9 @@ func malg(stacksize int32) *g { stacksize = round2(stackSystem + stacksize) systemstack(func() { newg.stack = stackalloc(uint32(stacksize)) + if valgrindenabled { + newg.valgrindStackID = valgrindRegisterStack(unsafe.Pointer(newg.stack.lo), unsafe.Pointer(newg.stack.hi)) + } }) newg.stackguard0 = newg.stack.lo + stackGuard newg.stackguard1 = ^uintptr(0) @@ -4961,7 +5156,7 @@ func malg(stacksize int32) *g { // The compiler turns a go statement into a call to this. func newproc(fn *funcval) { gp := getg() - pc := getcallerpc() + pc := sys.GetCallerPC() systemstack(func() { newg := newproc1(fn, gp, pc, false, waitReasonZero) @@ -5021,10 +5216,12 @@ func newproc1(fn *funcval, callergp *g, callerpc uintptr, parked bool, waitreaso newg.gopc = callerpc newg.ancestors = saveAncestors(callergp) newg.startpc = fn.fn + newg.runningCleanups.Store(false) if isSystemGoroutine(newg, false) { sched.ngsys.Add(1) } else { - // Only user goroutines inherit pprof labels. + // Only user goroutines inherit synctest groups and pprof labels. + newg.bubble = callergp.bubble if mp.curg != nil { newg.labels = mp.curg.labels } @@ -5051,7 +5248,6 @@ func newproc1(fn *funcval, callergp *g, callerpc uintptr, parked bool, waitreaso status = _Gwaiting newg.waitreason = waitreason } - casgstatus(newg, _Gdead, status) if pp.goidcache == pp.goidcacheend { // Sched.goidgen is the last allocated id, // this batch must be [sched.goidgen+1, sched.goidgen+GoidCacheBatch]. @@ -5061,6 +5257,7 @@ func newproc1(fn *funcval, callergp *g, callerpc uintptr, parked bool, waitreaso pp.goidcacheend = pp.goidcache + _GoidCacheBatch } newg.goid = pp.goidcache + casgstatus(newg, _Gdead, status) pp.goidcache++ newg.trace.reset() if trace.ok() { @@ -5078,6 +5275,7 @@ func newproc1(fn *funcval, callergp *g, callerpc uintptr, parked bool, waitreaso racereleasemergeg(newg, unsafe.Pointer(&labelSync)) } } + pp.goroutinesCreated++ releasem(mp) return newg @@ -5132,30 +5330,29 @@ func gfput(pp *p, gp *g) { gp.stack.lo = 0 gp.stack.hi = 0 gp.stackguard0 = 0 + if valgrindenabled { + valgrindDeregisterStack(gp.valgrindStackID) + gp.valgrindStackID = 0 + } } pp.gFree.push(gp) - pp.gFree.n++ - if pp.gFree.n >= 64 { + if pp.gFree.size >= 64 { var ( - inc int32 stackQ gQueue noStackQ gQueue ) - for pp.gFree.n >= 32 { + for pp.gFree.size >= 32 { gp := pp.gFree.pop() - pp.gFree.n-- if gp.stack.lo == 0 { noStackQ.push(gp) } else { stackQ.push(gp) } - inc++ } lock(&sched.gFree.lock) sched.gFree.noStack.pushAll(noStackQ) sched.gFree.stack.pushAll(stackQ) - sched.gFree.n += inc unlock(&sched.gFree.lock) } } @@ -5167,7 +5364,7 @@ retry: if pp.gFree.empty() && (!sched.gFree.stack.empty() || !sched.gFree.noStack.empty()) { lock(&sched.gFree.lock) // Move a batch of free Gs to the P. - for pp.gFree.n < 32 { + for pp.gFree.size < 32 { // Prefer Gs with stacks. gp := sched.gFree.stack.pop() if gp == nil { @@ -5176,9 +5373,7 @@ retry: break } } - sched.gFree.n-- pp.gFree.push(gp) - pp.gFree.n++ } unlock(&sched.gFree.lock) goto retry @@ -5187,7 +5382,6 @@ retry: if gp == nil { return nil } - pp.gFree.n-- if gp.stack.lo != 0 && gp.stack.hi-gp.stack.lo != uintptr(startingStackSize) { // Deallocate old stack. We kept it in gfput because it was the // right size when the goroutine was put on the free list, but @@ -5197,12 +5391,19 @@ retry: gp.stack.lo = 0 gp.stack.hi = 0 gp.stackguard0 = 0 + if valgrindenabled { + valgrindDeregisterStack(gp.valgrindStackID) + gp.valgrindStackID = 0 + } }) } if gp.stack.lo == 0 { // Stack was deallocated in gfput or just above. Allocate a new one. systemstack(func() { gp.stack = stackalloc(startingStackSize) + if valgrindenabled { + gp.valgrindStackID = valgrindRegisterStack(unsafe.Pointer(gp.stack.lo), unsafe.Pointer(gp.stack.hi)) + } }) gp.stackguard0 = gp.stack.lo + stackGuard } else { @@ -5222,24 +5423,20 @@ retry: // Purge all cached G's from gfree list to the global list. func gfpurge(pp *p) { var ( - inc int32 stackQ gQueue noStackQ gQueue ) for !pp.gFree.empty() { gp := pp.gFree.pop() - pp.gFree.n-- if gp.stack.lo == 0 { noStackQ.push(gp) } else { stackQ.push(gp) } - inc++ } lock(&sched.gFree.lock) sched.gFree.noStack.pushAll(noStackQ) sched.gFree.stack.pushAll(stackQ) - sched.gFree.n += inc unlock(&sched.gFree.lock) } @@ -5354,10 +5551,13 @@ func badunlockosthread() { throw("runtime: internal error: misuse of lockOSThread/unlockOSThread") } -func gcount() int32 { - n := int32(atomic.Loaduintptr(&allglen)) - sched.gFree.n - sched.ngsys.Load() +func gcount(includeSys bool) int32 { + n := int32(atomic.Loaduintptr(&allglen)) - sched.gFree.stack.size - sched.gFree.noStack.size + if !includeSys { + n -= sched.ngsys.Load() + } for _, pp := range allp { - n -= pp.gFree.n + n -= pp.gFree.size } // All these variables can be changed concurrently, so the result can be inconsistent. @@ -5368,6 +5568,14 @@ func gcount() int32 { return n } +// goroutineleakcount returns the number of leaked goroutines last reported by +// the runtime. +// +//go:linkname goroutineleakcount runtime/pprof.runtime_goroutineleakcount +func goroutineleakcount() int { + return work.goroutineLeak.count +} + func mcount() int32 { return int32(sched.mnext - sched.nmfreed) } @@ -5552,6 +5760,7 @@ func setcpuprofilerate(hz int32) { // previously destroyed p, and transitions it to status _Pgcstop. func (pp *p) init(id int32) { pp.id = id + pp.gcw.id = id pp.status = _Pgcstop pp.sudogcache = pp.sudogbuf[:0] pp.deferpool = pp.deferpoolbuf[:0] @@ -5610,19 +5819,19 @@ func (pp *p) destroy() { // Move all timers to the local P. getg().m.p.ptr().timers.take(&pp.timers) - // Flush p's write barrier buffer. - if gcphase != _GCoff { - wbBufFlush1(pp) - pp.gcw.dispose() - } - for i := range pp.sudogbuf { - pp.sudogbuf[i] = nil + // No need to flush p's write barrier buffer or span queue, as Ps + // cannot be destroyed during the mark phase. + if phase := gcphase; phase != _GCoff { + println("runtime: p id", pp.id, "destroyed during GC phase", phase) + throw("P destroyed while GC is running") } + // We should free the queues though. + pp.gcw.spanq.destroy() + + clear(pp.sudogbuf[:]) pp.sudogcache = pp.sudogbuf[:0] pp.pinnerCache = nil - for j := range pp.deferpoolbuf { - pp.deferpoolbuf[j] = nil - } + clear(pp.deferpoolbuf[:]) pp.deferpool = pp.deferpoolbuf[:0] systemstack(func() { for i := 0; i < pp.mspancache.len; i++ { @@ -5657,6 +5866,11 @@ func (pp *p) destroy() { pp.raceprocctx = 0 } pp.gcAssistTime = 0 + gcCleanups.queued += pp.cleanupsQueued + pp.cleanupsQueued = 0 + sched.goroutinesCreated.Add(int64(pp.goroutinesCreated)) + pp.goroutinesCreated = 0 + pp.xRegs.free() pp.status = _Pdead } @@ -5689,8 +5903,6 @@ func procresize(nprocs int32) *p { } sched.procresizetime = now - maskWords := (nprocs + 31) / 32 - // Grow allp if necessary. if nprocs > int32(len(allp)) { // Synchronize with retake, which could be running @@ -5706,19 +5918,9 @@ func procresize(nprocs int32) *p { allp = nallp } - if maskWords <= int32(cap(idlepMask)) { - idlepMask = idlepMask[:maskWords] - timerpMask = timerpMask[:maskWords] - } else { - nidlepMask := make([]uint32, maskWords) - // No need to copy beyond len, old Ps are irrelevant. - copy(nidlepMask, idlepMask) - idlepMask = nidlepMask - - ntimerpMask := make([]uint32, maskWords) - copy(ntimerpMask, timerpMask) - timerpMask = ntimerpMask - } + idlepMask = idlepMask.resize(nprocs) + timerpMask = timerpMask.resize(nprocs) + work.spanqMask = work.spanqMask.resize(nprocs) unlock(&allpLock) } @@ -5781,8 +5983,9 @@ func procresize(nprocs int32) *p { if int32(len(allp)) != nprocs { lock(&allpLock) allp = allp[:nprocs] - idlepMask = idlepMask[:maskWords] - timerpMask = timerpMask[:maskWords] + idlepMask = idlepMask.resize(nprocs) + timerpMask = timerpMask.resize(nprocs) + work.spanqMask = work.spanqMask.resize(nprocs) unlock(&allpLock) } @@ -5913,7 +6116,9 @@ func checkdead() { // For -buildmode=c-shared or -buildmode=c-archive it's OK if // there are no running goroutines. The calling program is // assumed to be running. - if islibrary || isarchive { + // One exception is Wasm, which is single-threaded. If we are + // in Go and all goroutines are blocked, it deadlocks. + if (islibrary || isarchive) && GOARCH != "wasm" { return } @@ -6016,10 +6221,6 @@ func checkdead() { // This is a variable for testing purposes. It normally doesn't change. var forcegcperiod int64 = 2 * 60 * 1e9 -// needSysmonWorkaround is true if the workaround for -// golang.org/issue/42515 is needed on NetBSD. -var needSysmonWorkaround bool = false - // haveSysmon indicates whether there is sysmon thread support. // // No threads on wasm yet, so no sysmon. @@ -6034,6 +6235,7 @@ func sysmon() { checkdead() unlock(&sched.lock) + lastgomaxprocs := int64(0) lasttrace := int64(0) idle := 0 // how many cycles in succession we had not wokeup somebody delay := uint32(0) @@ -6127,25 +6329,10 @@ func sysmon() { netpollAdjustWaiters(delta) } } - if GOOS == "netbsd" && needSysmonWorkaround { - // netpoll is responsible for waiting for timer - // expiration, so we typically don't have to worry - // about starting an M to service timers. (Note that - // sleep for timeSleepUntil above simply ensures sysmon - // starts running again when that timer expiration may - // cause Go code to run again). - // - // However, netbsd has a kernel bug that sometimes - // misses netpollBreak wake-ups, which can lead to - // unbounded delays servicing timers. If we detect this - // overrun, then startm to get something to handle the - // timer. - // - // See issue 42515 and - // https://gnats.netbsd.org/cgi-bin/query-pr-single.pl?number=50094. - if next := timeSleepUntil(); next < now { - startm(nil, false, false) - } + // Check if we need to update GOMAXPROCS at most once per second. + if debug.updatemaxprocs != 0 && lastgomaxprocs+1e9 <= now { + sysmonUpdateGOMAXPROCS() + lastgomaxprocs = now } if scavenger.sysmonWake.Load() != 0 { // Kick the scavenger awake if someone requested it. @@ -6247,6 +6434,7 @@ func retake(now int64) uint32 { trace.ProcSteal(pp, false) traceRelease(trace) } + sched.nGsyscallNoP.Add(1) n++ pp.syscalltick++ handoffp(pp) @@ -6325,7 +6513,7 @@ func schedtrace(detailed bool) { } lock(&sched.lock) - print("SCHED ", (now-starttime)/1e6, "ms: gomaxprocs=", gomaxprocs, " idleprocs=", sched.npidle.Load(), " threads=", mcount(), " spinningthreads=", sched.nmspinning.Load(), " needspinning=", sched.needspinning.Load(), " idlethreads=", sched.nmidle, " runqueue=", sched.runqsize) + print("SCHED ", (now-starttime)/1e6, "ms: gomaxprocs=", gomaxprocs, " idleprocs=", sched.npidle.Load(), " threads=", mcount(), " spinningthreads=", sched.nmspinning.Load(), " needspinning=", sched.needspinning.Load(), " idlethreads=", sched.nmidle, " runqueue=", sched.runq.size) if detailed { print(" gcwaiting=", sched.gcwaiting.Load(), " nmidlelocked=", sched.nmidlelocked, " stopwait=", sched.stopwait, " sysmonwait=", sched.sysmonwait.Load(), "\n") } @@ -6333,31 +6521,41 @@ func schedtrace(detailed bool) { // Even if we hold schedlock, most data can be changed concurrently. // E.g. (p->m ? p->m->id : -1) can crash if p->m changes from non-nil to nil. for i, pp := range allp { - mp := pp.m.ptr() h := atomic.Load(&pp.runqhead) t := atomic.Load(&pp.runqtail) if detailed { print(" P", i, ": status=", pp.status, " schedtick=", pp.schedtick, " syscalltick=", pp.syscalltick, " m=") + mp := pp.m.ptr() if mp != nil { print(mp.id) } else { print("nil") } - print(" runqsize=", t-h, " gfreecnt=", pp.gFree.n, " timerslen=", len(pp.timers.heap), "\n") + print(" runqsize=", t-h, " gfreecnt=", pp.gFree.size, " timerslen=", len(pp.timers.heap), "\n") } else { // In non-detailed mode format lengths of per-P run queues as: - // [len1 len2 len3 len4] + // [ len1 len2 len3 len4 ] print(" ") if i == 0 { - print("[") + print("[ ") } print(t - h) if i == len(allp)-1 { - print("]\n") + print(" ]") } } } + if !detailed { + // Format per-P schedticks as: schedticks=[ tick1 tick2 tick3 tick4 ]. + print(" schedticks=[ ") + for _, pp := range allp { + print(pp.schedtick) + print(" ") + } + print("]\n") + } + if !detailed { unlock(&sched.lock) return @@ -6404,6 +6602,169 @@ func schedtrace(detailed bool) { unlock(&sched.lock) } +type updateMaxProcsGState struct { + lock mutex + g *g + idle atomic.Bool + + // Readable when idle == false, writable when idle == true. + procs int32 // new GOMAXPROCS value +} + +var ( + // GOMAXPROCS update godebug metric. Incremented if automatic + // GOMAXPROCS updates actually change the value of GOMAXPROCS. + updatemaxprocs = &godebugInc{name: "updatemaxprocs"} + + // Synchronization and state between updateMaxProcsGoroutine and + // sysmon. + updateMaxProcsG updateMaxProcsGState + + // Synchronization between GOMAXPROCS and sysmon. + // + // Setting GOMAXPROCS via a call to GOMAXPROCS disables automatic + // GOMAXPROCS updates. + // + // We want to make two guarantees to callers of GOMAXPROCS. After + // GOMAXPROCS returns: + // + // 1. The runtime will not make any automatic changes to GOMAXPROCS. + // + // 2. The runtime will not perform any of the system calls used to + // determine the appropriate value of GOMAXPROCS (i.e., it won't + // call defaultGOMAXPROCS). + // + // (1) is the baseline guarantee that everyone needs. The GOMAXPROCS + // API isn't useful to anyone if automatic updates may occur after it + // returns. This is easily achieved by double-checking the state under + // STW before committing an automatic GOMAXPROCS update. + // + // (2) doesn't matter to most users, as it is isn't observable as long + // as (1) holds. However, it can be important to users sandboxing Go. + // They want disable these system calls and need some way to know when + // they are guaranteed the calls will stop. + // + // This would be simple to achieve if we simply called + // defaultGOMAXPROCS under STW in updateMaxProcsGoroutine below. + // However, we would like to avoid scheduling this goroutine every + // second when it will almost never do anything. Instead, sysmon calls + // defaultGOMAXPROCS to decide whether to schedule + // updateMaxProcsGoroutine. Thus we need to synchronize between sysmon + // and GOMAXPROCS calls. + // + // GOMAXPROCS can't hold a runtime mutex across STW. It could hold a + // semaphore, but sysmon cannot take semaphores. Instead, we have a + // more complex scheme: + // + // * sysmon holds computeMaxProcsLock while calling defaultGOMAXPROCS. + // * sysmon skips the current update if sched.customGOMAXPROCS is + // set. + // * GOMAXPROCS sets sched.customGOMAXPROCS once it is committed to + // changing GOMAXPROCS. + // * GOMAXPROCS takes computeMaxProcsLock to wait for outstanding + // defaultGOMAXPROCS calls to complete. + // + // N.B. computeMaxProcsLock could simply be sched.lock, but we want to + // avoid holding that lock during the potentially slow + // defaultGOMAXPROCS. + computeMaxProcsLock mutex +) + +// Start GOMAXPROCS update helper goroutine. +// +// This is based on forcegchelper. +func defaultGOMAXPROCSUpdateEnable() { + if debug.updatemaxprocs == 0 { + // Unconditionally increment the metric when updates are disabled. + // + // It would be more descriptive if we did a dry run of the + // complete update, determining the appropriate value of + // GOMAXPROCS and the bailing out and just incrementing the + // metric if a change would occur. + // + // Not only is that a lot of ongoing work for a disabled + // feature, but some users need to be able to completely + // disable the update system calls (such as sandboxes). + // Currently, updatemaxprocs=0 serves that purpose. + updatemaxprocs.IncNonDefault() + return + } + + go updateMaxProcsGoroutine() +} + +func updateMaxProcsGoroutine() { + updateMaxProcsG.g = getg() + lockInit(&updateMaxProcsG.lock, lockRankUpdateMaxProcsG) + for { + lock(&updateMaxProcsG.lock) + if updateMaxProcsG.idle.Load() { + throw("updateMaxProcsGoroutine: phase error") + } + updateMaxProcsG.idle.Store(true) + goparkunlock(&updateMaxProcsG.lock, waitReasonUpdateGOMAXPROCSIdle, traceBlockSystemGoroutine, 1) + // This goroutine is explicitly resumed by sysmon. + + stw := stopTheWorldGC(stwGOMAXPROCS) + + // Still OK to update? + lock(&sched.lock) + custom := sched.customGOMAXPROCS + unlock(&sched.lock) + if custom { + startTheWorldGC(stw) + return + } + + // newprocs will be processed by startTheWorld + // + // TODO(prattmic): this could use a nicer API. Perhaps add it to the + // stw parameter? + newprocs = updateMaxProcsG.procs + lock(&sched.lock) + sched.customGOMAXPROCS = false + unlock(&sched.lock) + + startTheWorldGC(stw) + } +} + +func sysmonUpdateGOMAXPROCS() { + // Synchronize with GOMAXPROCS. See comment on computeMaxProcsLock. + lock(&computeMaxProcsLock) + + // No update if GOMAXPROCS was set manually. + lock(&sched.lock) + custom := sched.customGOMAXPROCS + curr := gomaxprocs + unlock(&sched.lock) + if custom { + unlock(&computeMaxProcsLock) + return + } + + // Don't hold sched.lock while we read the filesystem. + procs := defaultGOMAXPROCS(0) + unlock(&computeMaxProcsLock) + if procs == curr { + // Nothing to do. + return + } + + // Sysmon can't directly stop the world. Run the helper to do so on our + // behalf. If updateGOMAXPROCS.idle is false, then a previous update is + // still pending. + if updateMaxProcsG.idle.Load() { + lock(&updateMaxProcsG.lock) + updateMaxProcsG.procs = procs + updateMaxProcsG.idle.Store(false) + var list gList + list.push(updateMaxProcsG.g) + injectglist(&list) + unlock(&updateMaxProcsG.lock) + } +} + // schedEnableUser enables or disables the scheduling of user // goroutines. // @@ -6417,9 +6778,8 @@ func schedEnableUser(enable bool) { } sched.disable.user = !enable if enable { - n := sched.disable.n - sched.disable.n = 0 - globrunqputbatch(&sched.disable.runnable, n) + n := sched.disable.runnable.size + globrunqputbatch(&sched.disable.runnable) unlock(&sched.lock) for ; n != 0 && sched.npidle.Load() != 0; n-- { startm(nil, false, false) @@ -6481,7 +6841,6 @@ func globrunqput(gp *g) { assertLockHeld(&sched.lock) sched.runq.pushBack(gp) - sched.runqsize++ } // Put gp at the head of the global runnable queue. @@ -6493,7 +6852,6 @@ func globrunqputhead(gp *g) { assertLockHeld(&sched.lock) sched.runq.push(gp) - sched.runqsize++ } // Put a batch of runnable goroutines on the global runnable queue. @@ -6502,43 +6860,44 @@ func globrunqputhead(gp *g) { // May run during STW, so write barriers are not allowed. // //go:nowritebarrierrec -func globrunqputbatch(batch *gQueue, n int32) { +func globrunqputbatch(batch *gQueue) { assertLockHeld(&sched.lock) sched.runq.pushBackAll(*batch) - sched.runqsize += n *batch = gQueue{} } -// Try get a batch of G's from the global runnable queue. +// Try get a single G from the global runnable queue. // sched.lock must be held. -func globrunqget(pp *p, max int32) *g { +func globrunqget() *g { assertLockHeld(&sched.lock) - if sched.runqsize == 0 { + if sched.runq.size == 0 { return nil } - n := sched.runqsize/gomaxprocs + 1 - if n > sched.runqsize { - n = sched.runqsize - } - if max > 0 && n > max { - n = max - } - if n > int32(len(pp.runq))/2 { - n = int32(len(pp.runq)) / 2 + return sched.runq.pop() +} + +// Try get a batch of G's from the global runnable queue. +// sched.lock must be held. +func globrunqgetbatch(n int32) (gp *g, q gQueue) { + assertLockHeld(&sched.lock) + + if sched.runq.size == 0 { + return } - sched.runqsize -= n + n = min(n, sched.runq.size, sched.runq.size/gomaxprocs+1) - gp := sched.runq.pop() + gp = sched.runq.pop() n-- + for ; n > 0; n-- { gp1 := sched.runq.pop() - runqput(pp, gp1, false) + q.pushBack(gp1) } - return gp + return } // pMask is an atomic bitstring with one bit per P. @@ -6565,6 +6924,32 @@ func (p pMask) clear(id int32) { atomic.And(&p[word], ^mask) } +// any returns true if any bit in p is set. +func (p pMask) any() bool { + for i := range p { + if atomic.Load(&p[i]) != 0 { + return true + } + } + return false +} + +// resize resizes the pMask and returns a new one. +// +// The result may alias p, so callers are encouraged to +// discard p. Not safe for concurrent use. +func (p pMask) resize(nprocs int32) pMask { + maskWords := (nprocs + 31) / 32 + + if maskWords <= int32(cap(p)) { + return p[:maskWords] + } + newMask := make([]uint32, maskWords) + // No need to copy beyond len, old Ps are irrelevant. + copy(newMask, p) + return newMask +} + // pidleput puts p on the _Pidle list. now must be a relatively recent call // to nanotime or zero. Returns now or the current time if now was zero. // @@ -6755,22 +7140,23 @@ func runqputslow(pp *p, gp *g, h, t uint32) bool { for i := uint32(0); i < n; i++ { batch[i].schedlink.set(batch[i+1]) } - var q gQueue - q.head.set(batch[0]) - q.tail.set(batch[n]) + + q := gQueue{batch[0].guintptr(), batch[n].guintptr(), int32(n + 1)} // Now put the batch on global queue. lock(&sched.lock) - globrunqputbatch(&q, int32(n+1)) + globrunqputbatch(&q) unlock(&sched.lock) return true } // runqputbatch tries to put all the G's on q on the local runnable queue. -// If the queue is full, they are put on the global queue; in that case -// this will temporarily acquire the scheduler lock. +// If the local runq is full the input queue still contains unqueued Gs. // Executed only by the owner P. -func runqputbatch(pp *p, q *gQueue, qsize int) { +func runqputbatch(pp *p, q *gQueue) { + if q.empty() { + return + } h := atomic.LoadAcq(&pp.runqhead) t := pp.runqtail n := uint32(0) @@ -6780,7 +7166,6 @@ func runqputbatch(pp *p, q *gQueue, qsize int) { t++ n++ } - qsize -= int(n) if randomizeScheduler { off := func(o uint32) uint32 { @@ -6793,11 +7178,8 @@ func runqputbatch(pp *p, q *gQueue, qsize int) { } atomic.StoreRel(&pp.runqtail, t) - if !q.empty() { - lock(&sched.lock) - globrunqputbatch(q, int32(qsize)) - unlock(&sched.lock) - } + + return } // Get g from local runnable queue. @@ -6829,11 +7211,10 @@ func runqget(pp *p) (gp *g, inheritTime bool) { // runqdrain drains the local runnable queue of pp and returns all goroutines in it. // Executed only by the owner P. -func runqdrain(pp *p) (drainQ gQueue, n uint32) { +func runqdrain(pp *p) (drainQ gQueue) { oldNext := pp.runnext if oldNext != 0 && pp.runnext.cas(oldNext, 0) { drainQ.pushBack(oldNext.ptr()) - n++ } retry: @@ -6861,7 +7242,6 @@ retry: for i := uint32(0); i < qn; i++ { gp := pp.runq[(h+i)%uint32(len(pp.runq))].ptr() drainQ.pushBack(gp) - n++ } return } @@ -6949,6 +7329,7 @@ func runqsteal(pp, p2 *p, stealRunNextG bool) *g { type gQueue struct { head guintptr tail guintptr + size int32 } // empty reports whether q is empty. @@ -6963,6 +7344,7 @@ func (q *gQueue) push(gp *g) { if q.tail == 0 { q.tail.set(gp) } + q.size++ } // pushBack adds gp to the tail of q. @@ -6974,6 +7356,7 @@ func (q *gQueue) pushBack(gp *g) { q.head.set(gp) } q.tail.set(gp) + q.size++ } // pushBackAll adds all Gs in q2 to the tail of q. After this q2 must @@ -6989,6 +7372,7 @@ func (q *gQueue) pushBackAll(q2 gQueue) { q.head = q2.head } q.tail = q2.tail + q.size += q2.size } // pop removes and returns the head of queue q. It returns nil if @@ -7000,13 +7384,14 @@ func (q *gQueue) pop() *g { if q.head == 0 { q.tail = 0 } + q.size-- } return gp } // popList takes all Gs in q and returns them as a gList. func (q *gQueue) popList() gList { - stack := gList{q.head} + stack := gList{q.head, q.size} *q = gQueue{} return stack } @@ -7015,6 +7400,7 @@ func (q *gQueue) popList() gList { // on one gQueue or gList at a time. type gList struct { head guintptr + size int32 } // empty reports whether l is empty. @@ -7026,13 +7412,15 @@ func (l *gList) empty() bool { func (l *gList) push(gp *g) { gp.schedlink = l.head l.head.set(gp) + l.size++ } -// pushAll prepends all Gs in q to l. +// pushAll prepends all Gs in q to l. After this q must not be used. func (l *gList) pushAll(q gQueue) { if !q.empty() { q.tail.ptr().schedlink = l.head l.head = q.head + l.size += q.size } } @@ -7041,6 +7429,7 @@ func (l *gList) pop() *g { gp := l.head.ptr() if gp != nil { l.head = gp.schedlink + l.size-- } return gp } @@ -7120,6 +7509,31 @@ func sync_atomic_runtime_procUnpin() { procUnpin() } +// Active spinning for sync.Mutex. +// +//go:linkname internal_sync_runtime_canSpin internal/sync.runtime_canSpin +//go:nosplit +func internal_sync_runtime_canSpin(i int) bool { + // sync.Mutex is cooperative, so we are conservative with spinning. + // Spin only few times and only if running on a multicore machine and + // GOMAXPROCS>1 and there is at least one other running P and local runq is empty. + // As opposed to runtime mutex we don't do passive spinning here, + // because there can be work on global runq or on other Ps. + if i >= active_spin || numCPUStartup <= 1 || gomaxprocs <= sched.npidle.Load()+sched.nmspinning.Load()+1 { + return false + } + if p := getg().m.p.ptr(); !runqempty(p) { + return false + } + return true +} + +//go:linkname internal_sync_runtime_doSpin internal/sync.runtime_doSpin +//go:nosplit +func internal_sync_runtime_doSpin() { + procyield(active_spin_cnt) +} + // Active spinning for sync.Mutex. // // sync_runtime_canSpin should be an internal detail, @@ -7135,18 +7549,7 @@ func sync_atomic_runtime_procUnpin() { //go:linkname sync_runtime_canSpin sync.runtime_canSpin //go:nosplit func sync_runtime_canSpin(i int) bool { - // sync.Mutex is cooperative, so we are conservative with spinning. - // Spin only few times and only if running on a multicore machine and - // GOMAXPROCS>1 and there is at least one other running P and local runq is empty. - // As opposed to runtime mutex we don't do passive spinning here, - // because there can be work on global runq or on other Ps. - if i >= active_spin || ncpu <= 1 || gomaxprocs <= sched.npidle.Load()+sched.nmspinning.Load()+1 { - return false - } - if p := getg().m.p.ptr(); !runqempty(p) { - return false - } - return true + return internal_sync_runtime_canSpin(i) } // sync_runtime_doSpin should be an internal detail, @@ -7162,7 +7565,7 @@ func sync_runtime_canSpin(i int) bool { //go:linkname sync_runtime_doSpin sync.runtime_doSpin //go:nosplit func sync_runtime_doSpin() { - procyield(active_spin_cnt) + internal_sync_runtime_doSpin() } var stealOrder randomOrder diff --git a/src/runtime/proc_test.go b/src/runtime/proc_test.go index a930ea707f12cc..d10d4a1fc931cc 100644 --- a/src/runtime/proc_test.go +++ b/src/runtime/proc_test.go @@ -696,7 +696,6 @@ func BenchmarkCreateGoroutinesCapture(b *testing.B) { var wg sync.WaitGroup wg.Add(N) for i := 0; i < N; i++ { - i := i go func() { if i >= N { b.Logf("bad") // just to capture b @@ -1026,6 +1025,17 @@ func TestLockOSThreadTemplateThreadRace(t *testing.T) { } } +func TestLockOSThreadVgetrandom(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skipf("vgetrandom only relevant on Linux") + } + output := runTestProg(t, "testprog", "LockOSThreadVgetrandom") + want := "OK\n" + if output != want { + t.Errorf("want %q, got %q", want, output) + } +} + // fakeSyscall emulates a system call. // //go:nosplit diff --git a/src/runtime/proflabel.go b/src/runtime/proflabel.go index 1a5e7e5e2f14a0..f9b9dd16a50866 100644 --- a/src/runtime/proflabel.go +++ b/src/runtime/proflabel.go @@ -12,7 +12,6 @@ var labelSync uintptr // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/cloudwego/localsession -// - github.com/DataDog/datadog-agent // // Do not remove or change the type signature. // See go.dev/issue/67401. @@ -47,7 +46,6 @@ func runtime_setProfLabel(labels unsafe.Pointer) { // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/cloudwego/localsession -// - github.com/DataDog/datadog-agent // // Do not remove or change the type signature. // See go.dev/issue/67401. diff --git a/src/runtime/race.go b/src/runtime/race.go index 7d5cbce49ee20d..2cd4e3a9a24f23 100644 --- a/src/runtime/race.go +++ b/src/runtime/race.go @@ -14,16 +14,49 @@ import ( // Public race detection API, present iff build with -race. func RaceRead(addr unsafe.Pointer) + +//go:linkname race_Read internal/race.Read +//go:nosplit +func race_Read(addr unsafe.Pointer) { + RaceRead(addr) +} + func RaceWrite(addr unsafe.Pointer) + +//go:linkname race_Write internal/race.Write +//go:nosplit +func race_Write(addr unsafe.Pointer) { + RaceWrite(addr) +} + func RaceReadRange(addr unsafe.Pointer, len int) + +//go:linkname race_ReadRange internal/race.ReadRange +//go:nosplit +func race_ReadRange(addr unsafe.Pointer, len int) { + RaceReadRange(addr, len) +} + func RaceWriteRange(addr unsafe.Pointer, len int) +//go:linkname race_WriteRange internal/race.WriteRange +//go:nosplit +func race_WriteRange(addr unsafe.Pointer, len int) { + RaceWriteRange(addr, len) +} + func RaceErrors() int { var n uint64 racecall(&__tsan_report_count, uintptr(unsafe.Pointer(&n)), 0, 0, 0) return int(n) } +//go:linkname race_Errors internal/race.Errors +//go:nosplit +func race_Errors() int { + return RaceErrors() +} + // RaceAcquire/RaceRelease/RaceReleaseMerge establish happens-before relations // between goroutines. These inform the race detector about actual synchronization // that it can't see for some reason (e.g. synchronization within RaceDisable/RaceEnable @@ -38,6 +71,12 @@ func RaceAcquire(addr unsafe.Pointer) { raceacquire(addr) } +//go:linkname race_Acquire internal/race.Acquire +//go:nosplit +func race_Acquire(addr unsafe.Pointer) { + RaceAcquire(addr) +} + // RaceRelease performs a release operation on addr that // can synchronize with a later RaceAcquire on addr. // @@ -49,6 +88,12 @@ func RaceRelease(addr unsafe.Pointer) { racerelease(addr) } +//go:linkname race_Release internal/race.Release +//go:nosplit +func race_Release(addr unsafe.Pointer) { + RaceRelease(addr) +} + // RaceReleaseMerge is like RaceRelease, but also establishes a happens-before // relation with the preceding RaceRelease or RaceReleaseMerge on addr. // @@ -60,6 +105,12 @@ func RaceReleaseMerge(addr unsafe.Pointer) { racereleasemerge(addr) } +//go:linkname race_ReleaseMerge internal/race.ReleaseMerge +//go:nosplit +func race_ReleaseMerge(addr unsafe.Pointer) { + RaceReleaseMerge(addr) +} + // RaceDisable disables handling of race synchronization events in the current goroutine. // Handling is re-enabled with RaceEnable. RaceDisable/RaceEnable can be nested. // Non-synchronization events (memory accesses, function entry/exit) still affect @@ -74,6 +125,12 @@ func RaceDisable() { gp.raceignore++ } +//go:linkname race_Disable internal/race.Disable +//go:nosplit +func race_Disable() { + RaceDisable() +} + // RaceEnable re-enables handling of race events in the current goroutine. // //go:nosplit @@ -85,6 +142,12 @@ func RaceEnable() { } } +//go:linkname race_Enable internal/race.Enable +//go:nosplit +func race_Enable() { + RaceEnable() +} + // Private interface for the runtime. const raceenabled = true @@ -93,7 +156,7 @@ const raceenabled = true // callerpc is a return PC of the function that calls this function, // pc is start PC of the function that calls this function. func raceReadObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) { - kind := t.Kind_ & abi.KindMask + kind := t.Kind() if kind == abi.Array || kind == abi.Struct { // for composite objects we have to read every address // because a write might happen to any subobject. @@ -105,8 +168,13 @@ func raceReadObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) { } } +//go:linkname race_ReadObjectPC internal/race.ReadObjectPC +func race_ReadObjectPC(t *abi.Type, addr unsafe.Pointer, callerpc, pc uintptr) { + raceReadObjectPC(t, addr, callerpc, pc) +} + func raceWriteObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) { - kind := t.Kind_ & abi.KindMask + kind := t.Kind() if kind == abi.Array || kind == abi.Struct { // for composite objects we have to write every address // because a write might happen to any subobject. @@ -118,12 +186,27 @@ func raceWriteObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) { } } +//go:linkname race_WriteObjectPC internal/race.WriteObjectPC +func race_WriteObjectPC(t *abi.Type, addr unsafe.Pointer, callerpc, pc uintptr) { + raceWriteObjectPC(t, addr, callerpc, pc) +} + //go:noescape func racereadpc(addr unsafe.Pointer, callpc, pc uintptr) //go:noescape func racewritepc(addr unsafe.Pointer, callpc, pc uintptr) +//go:linkname race_ReadPC internal/race.ReadPC +func race_ReadPC(addr unsafe.Pointer, callerpc, pc uintptr) { + racereadpc(addr, callerpc, pc) +} + +//go:linkname race_WritePC internal/race.WritePC +func race_WritePC(addr unsafe.Pointer, callerpc, pc uintptr) { + racewritepc(addr, callerpc, pc) +} + type symbolizeCodeContext struct { pc uintptr fn *byte @@ -372,7 +455,6 @@ func raceinit() (gctx, pctx uintptr) { racecall(&__tsan_init, uintptr(unsafe.Pointer(&gctx)), uintptr(unsafe.Pointer(&pctx)), abi.FuncPCABI0(racecallbackthunk), 0) - // Round data segment to page boundaries, because it's used in mmap(). start := ^uintptr(0) end := uintptr(0) if start > firstmoduledata.noptrdata { @@ -399,10 +481,13 @@ func raceinit() (gctx, pctx uintptr) { if end < firstmoduledata.ebss { end = firstmoduledata.ebss } - size := alignUp(end-start, _PageSize) - racecall(&__tsan_map_shadow, start, size, 0, 0) + // Use exact bounds for boundary check in racecalladdr. See issue 73483. racedatastart = start - racedataend = start + size + racedataend = end + // Round data segment to page boundaries for race detector (TODO: still needed?) + start = alignDown(start, _PageSize) + end = alignUp(end, _PageSize) + racecall(&__tsan_map_shadow, start, end-start, 0, 0) return } @@ -481,6 +566,13 @@ func racegoend() { racecall(&__tsan_go_end, getg().racectx, 0, 0, 0) } +//go:nosplit +func racectxstart(pc, spawnctx uintptr) uintptr { + var racectx uintptr + racecall(&__tsan_go_start, spawnctx, uintptr(unsafe.Pointer(&racectx)), pc, 0) + return racectx +} + //go:nosplit func racectxend(racectx uintptr) { racecall(&__tsan_go_end, racectx, 0, 0, 0) diff --git a/src/runtime/race/README b/src/runtime/race/README index 47c51ca9c1c3fa..a65d463e10f5c6 100644 --- a/src/runtime/race/README +++ b/src/runtime/race/README @@ -13,5 +13,7 @@ internal/amd64v1/race_windows.syso built with LLVM 51bfeff0e4b0757ff773da6882f4d internal/amd64v3/race_linux.syso built with LLVM 51bfeff0e4b0757ff773da6882f4d538996c9b04 and Go e7d582b55dda36e76ce4d0ce770139ca0915b7c5. race_darwin_arm64.syso built with LLVM 51bfeff0e4b0757ff773da6882f4d538996c9b04 and Go e7d582b55dda36e76ce4d0ce770139ca0915b7c5. race_linux_arm64.syso built with LLVM 51bfeff0e4b0757ff773da6882f4d538996c9b04 and Go e7d582b55dda36e76ce4d0ce770139ca0915b7c5. +race_linux_loong64.syso built with LLVM 83fe85115da9dc25fa270d2ea8140113c8d49670 and Go 037112464b4439571b45536de9ebe4bc9e10ecb7. race_linux_ppc64le.syso built with LLVM 51bfeff0e4b0757ff773da6882f4d538996c9b04 and Go e7d582b55dda36e76ce4d0ce770139ca0915b7c5. +race_linux_riscv64.syso built with LLVM c3c24be13f7928460ca1e2fe613a1146c868854e and Go a21249436b6e1fd47356361d53dc053bbc074f90. race_linux_s390x.syso built with LLVM 51bfeff0e4b0757ff773da6882f4d538996c9b04 and Go e7d582b55dda36e76ce4d0ce770139ca0915b7c5. diff --git a/src/runtime/race/internal/amd64v1/race_darwin.syso b/src/runtime/race/internal/amd64v1/race_darwin.syso index e92f4ce74533f7..d3a9c200a500c8 100644 Binary files a/src/runtime/race/internal/amd64v1/race_darwin.syso and b/src/runtime/race/internal/amd64v1/race_darwin.syso differ diff --git a/src/runtime/race/output_test.go b/src/runtime/race/output_test.go index 0ee0f41334bea6..0d5c9096f0ccf1 100644 --- a/src/runtime/race/output_test.go +++ b/src/runtime/race/output_test.go @@ -476,5 +476,59 @@ Previous write at 0x[0-9,a-f]+ by main goroutine: main\.main\(\) .*/main.go:10 \+0x[0-9,a-f]+ +`}}, + {"non_inline_array_compare", "run", "", "atexit_sleep_ms=0", ` +package main + +import ( + "math/rand/v2" +) + +var x = [1024]byte{} + +var ch = make(chan bool) + +func main() { + started := make(chan struct{}) + go func() { + close(started) + var y = [len(x)]byte{} + eq := x == y + ch <- eq + }() + <-started + x[rand.IntN(len(x))]++ + println(<-ch) +} +`, []string{`================== +WARNING: DATA RACE +`}}, + {"non_inline_struct_compare", "run", "", "atexit_sleep_ms=0", ` +package main + +import "math/rand/v2" + +type S struct { + a [1024]byte +} + +var x = S{a: [1024]byte{}} + +var ch = make(chan bool) + +func main() { + started := make(chan struct{}) + go func() { + close(started) + var y = S{a: [len(x.a)]byte{}} + eq := x == y + ch <- eq + }() + <-started + x.a[rand.IntN(len(x.a))]++ + println(<-ch) +} +`, []string{`================== +WARNING: DATA RACE `}}, } diff --git a/src/runtime/race/race.go b/src/runtime/race/race.go index 9c508ebc2b3227..c2c5913966ead9 100644 --- a/src/runtime/race/race.go +++ b/src/runtime/race/race.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build race && ((linux && (amd64 || arm64 || ppc64le || s390x)) || ((freebsd || netbsd || openbsd || windows) && amd64)) +//go:build race && ((linux && (amd64 || arm64 || loong64 || ppc64le || riscv64 || s390x)) || ((freebsd || netbsd || openbsd || windows) && amd64)) package race diff --git a/src/runtime/race/race_darwin_amd64.go b/src/runtime/race/race_darwin_amd64.go index 02d73f8d388d3c..3f4e587e6bbead 100644 --- a/src/runtime/race/race_darwin_amd64.go +++ b/src/runtime/race/race_darwin_amd64.go @@ -28,9 +28,6 @@ package race //go:cgo_import_dynamic _dyld_get_shared_cache_uuid _dyld_get_shared_cache_uuid "" //go:cgo_import_dynamic _dyld_image_count _dyld_image_count "" //go:cgo_import_dynamic _exit _exit "" -//go:cgo_import_dynamic _sanitizer_internal_memcpy _sanitizer_internal_memcpy "" -//go:cgo_import_dynamic _sanitizer_internal_memmove _sanitizer_internal_memmove "" -//go:cgo_import_dynamic _sanitizer_internal_memset _sanitizer_internal_memset "" //go:cgo_import_dynamic abort abort "" //go:cgo_import_dynamic arc4random_buf arc4random_buf "" //go:cgo_import_dynamic close close "" @@ -57,7 +54,6 @@ package race //go:cgo_import_dynamic madvise madvise "" //go:cgo_import_dynamic malloc_num_zones malloc_num_zones "" //go:cgo_import_dynamic malloc_zones malloc_zones "" -//go:cgo_import_dynamic memcpy memcpy "" //go:cgo_import_dynamic memset_pattern16 memset_pattern16 "" //go:cgo_import_dynamic mkdir mkdir "" //go:cgo_import_dynamic mprotect mprotect "" @@ -103,6 +99,3 @@ package race //go:cgo_import_dynamic vm_region_recurse_64 vm_region_recurse_64 "" //go:cgo_import_dynamic waitpid waitpid "" //go:cgo_import_dynamic write write "" -//go:cgo_import_dynamic memcpy memcpy "" -//go:cgo_import_dynamic memmove memmove "" -//go:cgo_import_dynamic memset memset "" diff --git a/src/runtime/race/race_darwin_arm64.go b/src/runtime/race/race_darwin_arm64.go index cb703a6dedd3de..66e327efcc4105 100644 --- a/src/runtime/race/race_darwin_arm64.go +++ b/src/runtime/race/race_darwin_arm64.go @@ -27,9 +27,6 @@ package race //go:cgo_import_dynamic _dyld_get_shared_cache_uuid _dyld_get_shared_cache_uuid "" //go:cgo_import_dynamic _dyld_image_count _dyld_image_count "" //go:cgo_import_dynamic _exit _exit "" -//go:cgo_import_dynamic _sanitizer_internal_memcpy _sanitizer_internal_memcpy "" -//go:cgo_import_dynamic _sanitizer_internal_memmove _sanitizer_internal_memmove "" -//go:cgo_import_dynamic _sanitizer_internal_memset _sanitizer_internal_memset "" //go:cgo_import_dynamic abort abort "" //go:cgo_import_dynamic arc4random_buf arc4random_buf "" //go:cgo_import_dynamic bzero bzero "" @@ -57,7 +54,6 @@ package race //go:cgo_import_dynamic madvise madvise "" //go:cgo_import_dynamic malloc_num_zones malloc_num_zones "" //go:cgo_import_dynamic malloc_zones malloc_zones "" -//go:cgo_import_dynamic memcpy memcpy "" //go:cgo_import_dynamic memset_pattern16 memset_pattern16 "" //go:cgo_import_dynamic mkdir mkdir "" //go:cgo_import_dynamic mprotect mprotect "" @@ -103,6 +99,3 @@ package race //go:cgo_import_dynamic vm_region_recurse_64 vm_region_recurse_64 "" //go:cgo_import_dynamic waitpid waitpid "" //go:cgo_import_dynamic write write "" -//go:cgo_import_dynamic memcpy memcpy "" -//go:cgo_import_dynamic memmove memmove "" -//go:cgo_import_dynamic memset memset "" diff --git a/src/runtime/race/race_darwin_arm64.syso b/src/runtime/race/race_darwin_arm64.syso index 8d8c120717fe71..706951f4716dc1 100644 Binary files a/src/runtime/race/race_darwin_arm64.syso and b/src/runtime/race/race_darwin_arm64.syso differ diff --git a/src/runtime/race/race_linux_loong64.syso b/src/runtime/race/race_linux_loong64.syso new file mode 100644 index 00000000000000..0c8ecef75d13af Binary files /dev/null and b/src/runtime/race/race_linux_loong64.syso differ diff --git a/src/runtime/race/race_linux_riscv64.syso b/src/runtime/race/race_linux_riscv64.syso new file mode 100644 index 00000000000000..e5c5b88498a7b4 Binary files /dev/null and b/src/runtime/race/race_linux_riscv64.syso differ diff --git a/src/runtime/race/race_test.go b/src/runtime/race/race_test.go index 4fe61683eb02d7..9e651edd246bdd 100644 --- a/src/runtime/race/race_test.go +++ b/src/runtime/race/race_test.go @@ -14,6 +14,7 @@ package race_test import ( "bufio" "bytes" + "errors" "fmt" "internal/testenv" "io" @@ -107,13 +108,18 @@ func processLog(testName string, tsanLog []string) string { gotRace = true break } + if strings.Contains(s, "fatal error: concurrent map") { + // Detected by the runtime, not the race detector. + gotRace = true + break + } + if strings.Contains(s, "--- SKIP:") { + return fmt.Sprintf("%-*s SKIPPED", visibleLen, testName) + } } failing := strings.Contains(testName, "Failing") expRace := !strings.HasPrefix(testName, "No") - for len(testName) < visibleLen { - testName += " " - } if expRace == gotRace { passedTests++ totalTests++ @@ -121,7 +127,7 @@ func processLog(testName string, tsanLog []string) string { failed = true failingNeg++ } - return fmt.Sprintf("%s .", testName) + return fmt.Sprintf("%-*s .", visibleLen, testName) } pos := "" if expRace { @@ -136,7 +142,7 @@ func processLog(testName string, tsanLog []string) string { failed = true } totalTests++ - return fmt.Sprintf("%s %s%s", testName, "FAILED", pos) + return fmt.Sprintf("%-*s %s%s", visibleLen, testName, "FAILED", pos) } // runTests assures that the package and its dependencies is @@ -177,9 +183,21 @@ func runTests(t *testing.T) ([]byte, error) { ) // There are races: we expect tests to fail and the exit code to be non-zero. out, _ := cmd.CombinedOutput() - if bytes.Contains(out, []byte("fatal error:")) { - // But don't expect runtime to crash. - return out, fmt.Errorf("runtime fatal error") + fatals := bytes.Count(out, []byte("fatal error:")) + mapFatals := bytes.Count(out, []byte("fatal error: concurrent map")) + if fatals > mapFatals { + // But don't expect runtime to crash (other than + // in the map concurrent access detector). + return out, errors.New("runtime fatal error") + } + // A crash in the map concurrent access detector will cause other tests not to run. + // Perhaps we should run tests with concurrent map access separately to avoid this, + // but for the moment just skip the remaining tests. + if mapFatals != 0 { + return out, nil + } + if !bytes.Contains(out, []byte("ALL TESTS COMPLETE")) { + return out, errors.New("not all tests ran") } return out, nil } diff --git a/src/runtime/race/testdata/finalizer_test.go b/src/runtime/race/testdata/finalizer_test.go index 3ac33d2b590c85..ad6fe717c6dcf8 100644 --- a/src/runtime/race/testdata/finalizer_test.go +++ b/src/runtime/race/testdata/finalizer_test.go @@ -9,6 +9,7 @@ import ( "sync" "testing" "time" + "unsafe" ) func TestNoRaceFin(t *testing.T) { @@ -66,3 +67,43 @@ func TestRaceFin(t *testing.T) { time.Sleep(100 * time.Millisecond) y = 66 } + +func TestNoRaceCleanup(t *testing.T) { + c := make(chan bool) + go func() { + x := new(string) + y := new(string) + runtime.AddCleanup(x, func(y *string) { + *y = "foo" + }, y) + *y = "bar" + runtime.KeepAlive(x) + c <- true + }() + <-c + runtime.GC() + time.Sleep(100 * time.Millisecond) +} + +func TestRaceBetweenCleanups(t *testing.T) { + // Allocate struct with pointer to avoid hitting tinyalloc. + // Otherwise we can't be sure when the allocation will + // be freed. + type T struct { + v int + p unsafe.Pointer + } + sharedVar := new(int) + v0 := new(T) + v1 := new(T) + cleanup := func(x int) { + *sharedVar = x + } + runtime.AddCleanup(v0, cleanup, 0) + runtime.AddCleanup(v1, cleanup, 0) + v0 = nil + v1 = nil + + runtime.GC() + time.Sleep(100 * time.Millisecond) +} diff --git a/src/runtime/race/testdata/main_test.go b/src/runtime/race/testdata/main_test.go new file mode 100644 index 00000000000000..286589672bd622 --- /dev/null +++ b/src/runtime/race/testdata/main_test.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package race_test + +import ( + "fmt" + "testing" +) + +func TestMain(m *testing.M) { + m.Run() + fmt.Println("ALL TESTS COMPLETE") +} diff --git a/src/runtime/race/testdata/map_test.go b/src/runtime/race/testdata/map_test.go index 88e735ecd37783..83f59b751ece98 100644 --- a/src/runtime/race/testdata/map_test.go +++ b/src/runtime/race/testdata/map_test.go @@ -242,7 +242,7 @@ func TestRaceMapAssignMultipleReturn(t *testing.T) { } // BigKey and BigVal must be larger than 256 bytes, -// so that compiler sets KindGCProg for them. +// so that compiler stores them indirectly. type BigKey [1000]*int type BigVal struct { diff --git a/src/runtime/race/testdata/mop_test.go b/src/runtime/race/testdata/mop_test.go index 6b1069fcca2a81..0d7d879d834d36 100644 --- a/src/runtime/race/testdata/mop_test.go +++ b/src/runtime/race/testdata/mop_test.go @@ -612,6 +612,8 @@ func TestNoRaceEnoughRegisters(t *testing.T) { } // emptyFunc should not be inlined. +// +//go:noinline func emptyFunc(x int) { if false { fmt.Println(x) @@ -1176,7 +1178,7 @@ func TestNoRaceHeapReallocation(t *testing.T) { // others. const n = 2 done := make(chan bool, n) - empty := func(p *int) {} + empty := func(p *int) { _ = p } for i := 0; i < n; i++ { ms := i go func() { @@ -1417,7 +1419,7 @@ func TestRaceInterCall2(t *testing.T) { func TestRaceFuncCall(t *testing.T) { c := make(chan bool, 1) - f := func(x, y int) {} + f := func(x, y int) { _ = y } x, y := 0, 0 go func() { y = 42 @@ -1804,6 +1806,7 @@ func TestRaceAsFunc2(t *testing.T) { x := 0 go func() { func(x int) { + _ = x }(x) c <- true }() @@ -1817,6 +1820,7 @@ func TestRaceAsFunc3(t *testing.T) { x := 0 go func() { func(x int) { + _ = x mu.Lock() }(x) // Read of x must be outside of the mutex. mu.Unlock() diff --git a/src/runtime/race/testdata/rangefunc_test.go b/src/runtime/race/testdata/rangefunc_test.go index f2ff793df723e0..986395bfb94e5b 100644 --- a/src/runtime/race/testdata/rangefunc_test.go +++ b/src/runtime/race/testdata/rangefunc_test.go @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build goexperiment.rangefunc - package race_test import ( @@ -65,6 +63,7 @@ func foo(v int) int64 { // TestRaceRangeFuncIterator races because x%5 can be equal to 4, // therefore foo can early exit. func TestRaceRangeFuncIterator(t *testing.T) { + t.Skip("#72925: uncaught panic ends tests") x := foo(4) t.Logf("foo(4)=%d", x) } @@ -72,6 +71,7 @@ func TestRaceRangeFuncIterator(t *testing.T) { // TestNoRaceRangeFuncIterator does not race because x%5 is never 5, // therefore foo's loop will not exit early, and this it will not race. func TestNoRaceRangeFuncIterator(t *testing.T) { + t.Skip("#72925: unexpected data race") x := foo(5) t.Logf("foo(5)=%d", x) } diff --git a/src/runtime/race/testdata/synctest_test.go b/src/runtime/race/testdata/synctest_test.go new file mode 100644 index 00000000000000..dfbd5682cae3f0 --- /dev/null +++ b/src/runtime/race/testdata/synctest_test.go @@ -0,0 +1,97 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package race_test + +import ( + "internal/synctest" + "testing" + "time" +) + +func TestRaceSynctestGoroutinesExit(t *testing.T) { + synctest.Run(func() { + x := 0 + _ = x + f := func() { + x = 1 + } + go f() + go f() + }) +} + +func TestNoRaceSynctestGoroutinesExit(t *testing.T) { + synctest.Run(func() { + x := 0 + _ = x + f := func() { + x = 1 + } + go f() + synctest.Wait() + go f() + }) +} + +func TestRaceSynctestGoroutinesRecv(t *testing.T) { + synctest.Run(func() { + x := 0 + _ = x + ch := make(chan struct{}) + f := func() { + x = 1 + <-ch + } + go f() + go f() + close(ch) + }) +} + +func TestRaceSynctestGoroutinesUnblocked(t *testing.T) { + synctest.Run(func() { + x := 0 + _ = x + ch := make(chan struct{}) + f := func() { + <-ch + x = 1 + } + go f() + go f() + close(ch) + }) +} + +func TestRaceSynctestGoroutinesSleep(t *testing.T) { + synctest.Run(func() { + x := 0 + _ = x + go func() { + time.Sleep(1 * time.Second) + x = 1 + time.Sleep(2 * time.Second) + }() + go func() { + time.Sleep(2 * time.Second) + x = 1 + time.Sleep(1 * time.Second) + }() + time.Sleep(5 * time.Second) + }) +} + +func TestRaceSynctestTimers(t *testing.T) { + synctest.Run(func() { + x := 0 + _ = x + f := func() { + x = 1 + } + time.AfterFunc(1*time.Second, f) + time.AfterFunc(2*time.Second, f) + time.Sleep(5 * time.Second) + }) +} diff --git a/src/runtime/race0.go b/src/runtime/race0.go index f36d4387c74fe2..9d43909562174b 100644 --- a/src/runtime/race0.go +++ b/src/runtime/race0.go @@ -41,4 +41,5 @@ func racemalloc(p unsafe.Pointer, sz uintptr) { th func racefree(p unsafe.Pointer, sz uintptr) { throw("race") } func racegostart(pc uintptr) uintptr { throw("race"); return 0 } func racegoend() { throw("race") } +func racectxstart(spawnctx, racectx uintptr) uintptr { throw("race"); return 0 } func racectxend(racectx uintptr) { throw("race") } diff --git a/src/runtime/race_amd64.s b/src/runtime/race_amd64.s index c4a6d493162c35..e19118bd54e6ee 100644 --- a/src/runtime/race_amd64.s +++ b/src/runtime/race_amd64.s @@ -43,7 +43,7 @@ // func runtime·raceread(addr uintptr) // Called from instrumented code. // Defined as ABIInternal so as to avoid introducing a wrapper, -// which would render runtime.getcallerpc ineffective. +// which would render sys.GetCallerPC ineffective. TEXT runtime·raceread(SB), NOSPLIT, $0-8 MOVQ AX, RARG1 MOVQ (SP), RARG2 @@ -69,7 +69,7 @@ TEXT runtime·racereadpc(SB), NOSPLIT, $0-24 // func runtime·racewrite(addr uintptr) // Called from instrumented code. // Defined as ABIInternal so as to avoid introducing a wrapper, -// which would render runtime.getcallerpc ineffective. +// which would render sys.GetCallerPC ineffective. TEXT runtime·racewrite(SB), NOSPLIT, $0-8 MOVQ AX, RARG1 MOVQ (SP), RARG2 @@ -95,7 +95,7 @@ TEXT runtime·racewritepc(SB), NOSPLIT, $0-24 // func runtime·racereadrange(addr, size uintptr) // Called from instrumented code. // Defined as ABIInternal so as to avoid introducing a wrapper, -// which would render runtime.getcallerpc ineffective. +// which would render sys.GetCallerPC ineffective. TEXT runtime·racereadrange(SB), NOSPLIT, $0-16 MOVQ AX, RARG1 MOVQ BX, RARG2 @@ -122,7 +122,7 @@ TEXT runtime·racereadrangepc1(SB), NOSPLIT, $0-24 // func runtime·racewriterange(addr, size uintptr) // Called from instrumented code. // Defined as ABIInternal so as to avoid introducing a wrapper, -// which would render runtime.getcallerpc ineffective. +// which would render sys.GetCallerPC ineffective. TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 MOVQ AX, RARG1 MOVQ BX, RARG2 @@ -438,9 +438,16 @@ TEXT racecall<>(SB), NOSPLIT|NOFRAME, $0-0 MOVQ g_m(R14), R13 // Switch to g0 stack. MOVQ SP, R12 // callee-saved, preserved across the CALL + + // Switch to g0 stack if we aren't already on g0 or gsignal. + MOVQ m_gsignal(R13), R10 + CMPQ R10, R14 + JE call // already on gsignal + MOVQ m_g0(R13), R10 CMPQ R10, R14 JE call // already on g0 + MOVQ (g_sched+gobuf_sp)(R10), SP call: ANDQ $~15, SP // alignment for gcc ABI diff --git a/src/runtime/race_arm64.s b/src/runtime/race_arm64.s index c42a6c1377528e..5df650105bb4d5 100644 --- a/src/runtime/race_arm64.s +++ b/src/runtime/race_arm64.s @@ -163,7 +163,7 @@ data: BLT ret MOVD runtime·racedataend(SB), R10 CMP R10, R1 - BGT ret + BGE ret call: JMP racecall<>(SB) ret: @@ -463,9 +463,16 @@ TEXT racecall<>(SB), NOSPLIT|NOFRAME, $0-0 // Switch to g0 stack. MOVD RSP, R19 // callee-saved, preserved across the CALL MOVD R30, R20 // callee-saved, preserved across the CALL + + // Switch to g0 stack if we aren't already on g0 or gsignal. + MOVD m_gsignal(R10), R11 + CMP R11, g + BEQ call + MOVD m_g0(R10), R11 CMP R11, g - BEQ call // already on g0 + BEQ call + MOVD (g_sched+gobuf_sp)(R11), R12 MOVD R12, RSP call: diff --git a/src/runtime/race_loong64.s b/src/runtime/race_loong64.s new file mode 100644 index 00000000000000..d731871d4494fe --- /dev/null +++ b/src/runtime/race_loong64.s @@ -0,0 +1,530 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build race + +#include "go_asm.h" +#include "funcdata.h" +#include "textflag.h" +#include "cgo/abi_loong64.h" + +// The following thunks allow calling the gcc-compiled race runtime directly +// from Go code without going all the way through cgo. +// First, it's much faster (up to 50% speedup for real Go programs). +// Second, it eliminates race-related special cases from cgocall and scheduler. +// Third, in long-term it will allow to remove cyclic runtime/race dependency on cmd/go. + +// A brief recap of the loong64 calling convention. +// Arguments are passed in R4...R11, the rest is on stack. +// Callee-saved registers are: R23...R30. +// Temporary registers are: R12...R20 +// SP must be 16-byte aligned. + +// When calling racecalladdr, R20 is the call target address. + +// The race ctx, ThreadState *thr below, is passed in R4 and loaded in racecalladdr. + +// Load g from TLS. (See tls_loong64.s) +#define load_g \ + MOVV runtime·tls_g(SB), g + +#define RARG0 R4 +#define RARG1 R5 +#define RARG2 R6 +#define RARG3 R7 +#define RCALL R20 + +// func runtime·raceread(addr uintptr) +// Called from instrumented code. +// Defined as ABIInternal so as to avoid introducing a wrapper, +// which would make caller's PC ineffective. +TEXT runtime·raceread(SB), NOSPLIT, $0-8 + MOVV R4, RARG1 + MOVV R1, RARG2 + // void __tsan_read(ThreadState *thr, void *addr, void *pc); + MOVV $__tsan_read(SB), RCALL + JMP racecalladdr<>(SB) + +// func runtime·RaceRead(addr uintptr) +TEXT runtime·RaceRead(SB), NOSPLIT, $0-8 + // This needs to be a tail call, because raceread reads caller pc. + JMP runtime·raceread(SB) + +// func runtime·racereadpc(void *addr, void *callpc, void *pc) +TEXT runtime·racereadpc(SB), NOSPLIT, $0-24 + MOVV addr+0(FP), RARG1 + MOVV callpc+8(FP), RARG2 + MOVV pc+16(FP), RARG3 + // void __tsan_read_pc(ThreadState *thr, void *addr, void *callpc, void *pc); + MOVV $__tsan_read_pc(SB), RCALL + JMP racecalladdr<>(SB) + +// func runtime·racewrite(addr uintptr) +// Called from instrumented code. +// Defined as ABIInternal so as to avoid introducing a wrapper, +// which would make caller's PC ineffective. +TEXT runtime·racewrite(SB), NOSPLIT, $0-8 + MOVV R4, RARG1 + MOVV R1, RARG2 + // void __tsan_write(ThreadState *thr, void *addr, void *pc); + MOVV $__tsan_write(SB), RCALL + JMP racecalladdr<>(SB) + +// func runtime·RaceWrite(addr uintptr) +TEXT runtime·RaceWrite(SB), NOSPLIT, $0-8 + // This needs to be a tail call, because racewrite reads caller pc. + JMP runtime·racewrite(SB) + +// func runtime·racewritepc(void *addr, void *callpc, void *pc) +TEXT runtime·racewritepc(SB), NOSPLIT, $0-24 + MOVV addr+0(FP), RARG1 + MOVV callpc+8(FP), RARG2 + MOVV pc+16(FP), RARG3 + // void __tsan_write_pc(ThreadState *thr, void *addr, void *callpc, void *pc); + MOVV $__tsan_write_pc(SB), RCALL + JMP racecalladdr<>(SB) + +// func runtime·racereadrange(addr, size uintptr) +// Called from instrumented code. +// Defined as ABIInternal so as to avoid introducing a wrapper, +// which would make caller's PC ineffective. +TEXT runtime·racereadrange(SB), NOSPLIT, $0-16 + MOVV R5, RARG2 + MOVV R4, RARG1 + MOVV R1, RARG3 + // void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, void *pc); + MOVV $__tsan_read_range(SB), RCALL + JMP racecalladdr<>(SB) + +// func runtime·RaceReadRange(addr, size uintptr) +TEXT runtime·RaceReadRange(SB), NOSPLIT, $0-16 + // This needs to be a tail call, because racereadrange reads caller pc. + JMP runtime·racereadrange(SB) + +// func runtime·racereadrangepc1(void *addr, uintptr sz, void *pc) +TEXT runtime·racereadrangepc1(SB), NOSPLIT, $0-24 + MOVV addr+0(FP), RARG1 + MOVV size+8(FP), RARG2 + MOVV pc+16(FP), RARG3 + ADDV $4, RARG3 // pc is function start, tsan wants return address. + // void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, void *pc); + MOVV $__tsan_read_range(SB), RCALL + JMP racecalladdr<>(SB) + +// func runtime·racewriterange(addr, size uintptr) +// Called from instrumented code. +// Defined as ABIInternal so as to avoid introducing a wrapper, +// which would make caller's PC ineffective. +TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 + MOVV R5, RARG2 + MOVV R4, RARG1 + MOVV R1, RARG3 + // void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, void *pc); + MOVV $__tsan_write_range(SB), RCALL + JMP racecalladdr<>(SB) + +// func runtime·RaceWriteRange(addr, size uintptr) +TEXT runtime·RaceWriteRange(SB), NOSPLIT, $0-16 + // This needs to be a tail call, because racewriterange reads caller pc. + JMP runtime·racewriterange(SB) + +// func runtime·racewriterangepc1(void *addr, uintptr sz, void *pc) +TEXT runtime·racewriterangepc1(SB), NOSPLIT, $0-24 + MOVV addr+0(FP), RARG1 + MOVV size+8(FP), RARG2 + MOVV pc+16(FP), RARG3 + ADDV $4, RARG3 // pc is function start, tsan wants return address. + // void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, void *pc); + MOVV $__tsan_write_range(SB), RCALL + JMP racecalladdr<>(SB) + +// Call a __tsan function from Go code. +// +// RCALL = tsan function address +// RARG0 = *ThreadState a.k.a. g_racectx from g +// RARG1 = addr passed to __tsan function +// +// If addr (RARG1) is out of range, do nothing. Otherwise, setup goroutine +// context and invoke racecall. Other arguments already set. +TEXT racecalladdr<>(SB), NOSPLIT, $0-0 + // Check that addr is within [arenastart, arenaend) or within [racedatastart, racedataend). + MOVV runtime·racearenastart(SB), R12 + BLT RARG1, R12, data + MOVV runtime·racearenaend(SB), R12 + BLT RARG1, R12, call +data: + MOVV runtime·racedatastart(SB), R12 + BLT RARG1, R12, ret + MOVV runtime·racedataend(SB), R12 + BGE RARG1, R12, ret +call: + load_g + MOVV g_racectx(g), RARG0 + JMP racecall<>(SB) +ret: + RET + +// func runtime·racefuncenter(pc uintptr) +// Called from instrumented code. +TEXT runtime·racefuncenter(SB), NOSPLIT, $0-8 + MOVV R4, RCALL + JMP racefuncenter<>(SB) + +// Common code for racefuncenter +// RCALL = caller's return address +TEXT racefuncenter<>(SB), NOSPLIT, $0-0 + load_g + MOVV g_racectx(g), RARG0 // goroutine racectx + MOVV RCALL, RARG1 + // void __tsan_func_enter(ThreadState *thr, void *pc); + MOVV $__tsan_func_enter(SB), RCALL + JAL racecall<>(SB) + RET + +// func runtime·racefuncexit() +// Called from instrumented code. +TEXT runtime·racefuncexit(SB), NOSPLIT, $0-0 + load_g + MOVV g_racectx(g), RARG0 // race context + // void __tsan_func_exit(ThreadState *thr); + MOVV $__tsan_func_exit(SB), RCALL + JMP racecall<>(SB) + +// Atomic operations for sync/atomic package. +// R7 = addr of arguments passed to this function, it can +// be fetched at 24(R3) in racecallatomic after two times JAL +// RARG0, RARG1, RARG2 set in racecallatomic + +// Load +TEXT sync∕atomic·LoadInt32(SB), NOSPLIT, $0-12 + GO_ARGS + MOVV $__tsan_go_atomic32_load(SB), RCALL + JAL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·LoadInt64(SB), NOSPLIT, $0-16 + GO_ARGS + MOVV $__tsan_go_atomic64_load(SB), RCALL + JAL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·LoadUint32(SB), NOSPLIT, $0-12 + GO_ARGS + JMP sync∕atomic·LoadInt32(SB) + +TEXT sync∕atomic·LoadUint64(SB), NOSPLIT, $0-16 + GO_ARGS + JMP sync∕atomic·LoadInt64(SB) + +TEXT sync∕atomic·LoadUintptr(SB), NOSPLIT, $0-16 + GO_ARGS + JMP sync∕atomic·LoadInt64(SB) + +TEXT sync∕atomic·LoadPointer(SB), NOSPLIT, $0-16 + GO_ARGS + JMP sync∕atomic·LoadInt64(SB) + +// Store +TEXT sync∕atomic·StoreInt32(SB), NOSPLIT, $0-12 + GO_ARGS + MOVV $__tsan_go_atomic32_store(SB), RCALL + JAL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·StoreInt64(SB), NOSPLIT, $0-16 + GO_ARGS + MOVV $__tsan_go_atomic64_store(SB), RCALL + JAL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·StoreUint32(SB), NOSPLIT, $0-12 + GO_ARGS + JMP sync∕atomic·StoreInt32(SB) + +TEXT sync∕atomic·StoreUint64(SB), NOSPLIT, $0-16 + GO_ARGS + JMP sync∕atomic·StoreInt64(SB) + +TEXT sync∕atomic·StoreUintptr(SB), NOSPLIT, $0-16 + GO_ARGS + JMP sync∕atomic·StoreInt64(SB) + +// Swap +TEXT sync∕atomic·SwapInt32(SB), NOSPLIT, $0-20 + GO_ARGS + MOVV $__tsan_go_atomic32_exchange(SB), RCALL + JAL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·SwapInt64(SB), NOSPLIT, $0-24 + GO_ARGS + MOVV $__tsan_go_atomic64_exchange(SB), RCALL + JAL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·SwapUint32(SB), NOSPLIT, $0-20 + GO_ARGS + JMP sync∕atomic·SwapInt32(SB) + +TEXT sync∕atomic·SwapUint64(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·SwapInt64(SB) + +TEXT sync∕atomic·SwapUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·SwapInt64(SB) + +// Add +TEXT sync∕atomic·AddInt32(SB), NOSPLIT, $0-20 + GO_ARGS + MOVV $__tsan_go_atomic32_fetch_add(SB), RCALL + JAL racecallatomic<>(SB) + MOVW add+8(FP), RARG0 // convert fetch_add to add_fetch + MOVW ret+16(FP), RARG1 + ADD RARG0, RARG1, RARG0 + MOVW RARG0, ret+16(FP) + RET + +TEXT sync∕atomic·AddInt64(SB), NOSPLIT, $0-24 + GO_ARGS + MOVV $__tsan_go_atomic64_fetch_add(SB), RCALL + JAL racecallatomic<>(SB) + MOVV add+8(FP), RARG0 // convert fetch_add to add_fetch + MOVV ret+16(FP), RARG1 + ADDV RARG0, RARG1, RARG0 + MOVV RARG0, ret+16(FP) + RET + +TEXT sync∕atomic·AddUint32(SB), NOSPLIT, $0-20 + GO_ARGS + JMP sync∕atomic·AddInt32(SB) + +TEXT sync∕atomic·AddUint64(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·AddInt64(SB) + +TEXT sync∕atomic·AddUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·AddInt64(SB) + +// And +TEXT sync∕atomic·AndInt32(SB), NOSPLIT, $0-20 + GO_ARGS + MOVV $__tsan_go_atomic32_fetch_and(SB), RCALL + JAL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·AndInt64(SB), NOSPLIT, $0-24 + GO_ARGS + MOVV $__tsan_go_atomic64_fetch_and(SB), RCALL + JAL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·AndUint32(SB), NOSPLIT, $0-20 + GO_ARGS + JMP sync∕atomic·AndInt32(SB) + +TEXT sync∕atomic·AndUint64(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·AndInt64(SB) + +TEXT sync∕atomic·AndUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·AndInt64(SB) + +// Or +TEXT sync∕atomic·OrInt32(SB), NOSPLIT, $0-20 + GO_ARGS + MOVV $__tsan_go_atomic32_fetch_or(SB), RCALL + JAL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·OrInt64(SB), NOSPLIT, $0-24 + GO_ARGS + MOVV $__tsan_go_atomic64_fetch_or(SB), RCALL + JAL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·OrUint32(SB), NOSPLIT, $0-20 + GO_ARGS + JMP sync∕atomic·OrInt32(SB) + +TEXT sync∕atomic·OrUint64(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·OrInt64(SB) + +TEXT sync∕atomic·OrUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·OrInt64(SB) + +// CompareAndSwap +TEXT sync∕atomic·CompareAndSwapInt32(SB), NOSPLIT, $0-17 + GO_ARGS + MOVV $__tsan_go_atomic32_compare_exchange(SB), RCALL + JAL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·CompareAndSwapInt64(SB), NOSPLIT, $0-25 + GO_ARGS + MOVV $__tsan_go_atomic64_compare_exchange(SB), RCALL + JAL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·CompareAndSwapUint32(SB), NOSPLIT, $0-17 + GO_ARGS + JMP sync∕atomic·CompareAndSwapInt32(SB) + +TEXT sync∕atomic·CompareAndSwapUint64(SB), NOSPLIT, $0-25 + GO_ARGS + JMP sync∕atomic·CompareAndSwapInt64(SB) + +TEXT sync∕atomic·CompareAndSwapUintptr(SB), NOSPLIT, $0-25 + GO_ARGS + JMP sync∕atomic·CompareAndSwapInt64(SB) + +// Generic atomic operation implementation. +// RCALL = addr of target function +TEXT racecallatomic<>(SB), NOSPLIT, $0 + // Set up these registers + // RARG0 = *ThreadState + // RARG1 = caller pc + // RARG2 = pc + // RARG3 = addr of incoming arg list + + // Trigger SIGSEGV early. + MOVV 24(R3), RARG3 // 1st arg is addr. after two times JAL, get it at 24(R3) + MOVB (RARG3), R12 // segv here if addr is bad + + // Check that addr is within [arenastart, arenaend) or within [racedatastart, racedataend). + MOVV runtime·racearenastart(SB), R12 + BLT RARG3, R12, racecallatomic_data + MOVV runtime·racearenaend(SB), R12 + BLT RARG3, R12, racecallatomic_ok + +racecallatomic_data: + MOVV runtime·racedatastart(SB), R12 + BLT RARG3, R12, racecallatomic_ignore + MOVV runtime·racedataend(SB), R12 + BGE RARG3, R12, racecallatomic_ignore + +racecallatomic_ok: + // Addr is within the good range, call the atomic function. + load_g + MOVV g_racectx(g), RARG0 // goroutine context + MOVV 8(R3), RARG1 // caller pc + MOVV RCALL, RARG2 // pc + ADDV $24, R3, RARG3 + JAL racecall<>(SB) // does not return + RET + +racecallatomic_ignore: + // Addr is outside the good range. + // Call __tsan_go_ignore_sync_begin to ignore synchronization during the atomic op. + // An attempt to synchronize on the address would cause crash. + MOVV RCALL, R25 // remember the original function + MOVV $__tsan_go_ignore_sync_begin(SB), RCALL + load_g + MOVV g_racectx(g), RARG0 // goroutine context + JAL racecall<>(SB) + MOVV R25, RCALL // restore the original function + + // Call the atomic function. + // racecall will call LLVM race code which might clobber R22 (g) + load_g + MOVV g_racectx(g), RARG0 // goroutine context + MOVV 8(R3), RARG1 // caller pc + MOVV RCALL, RARG2 // pc + ADDV $24, R3, RARG3 // arguments + JAL racecall<>(SB) + + // Call __tsan_go_ignore_sync_end. + MOVV $__tsan_go_ignore_sync_end(SB), RCALL + MOVV g_racectx(g), RARG0 // goroutine context + JAL racecall<>(SB) + RET + +// func runtime·racecall(void(*f)(...), ...) +// Calls C function f from race runtime and passes up to 4 arguments to it. +// The arguments are never heap-object-preserving pointers, so we pretend there are no arguments. +TEXT runtime·racecall(SB), NOSPLIT, $0-0 + MOVV fn+0(FP), RCALL + MOVV arg0+8(FP), RARG0 + MOVV arg1+16(FP), RARG1 + MOVV arg2+24(FP), RARG2 + MOVV arg3+32(FP), RARG3 + JMP racecall<>(SB) + +// Switches SP to g0 stack and calls (RCALL). Arguments already set. +TEXT racecall<>(SB), NOSPLIT|NOFRAME, $0-0 + MOVV g_m(g), R12 + // Switch to g0 stack. + MOVV R3, R23 // callee-saved, preserved across the CALL + MOVV R1, R24 // callee-saved, preserved across the CALL + + // Switch to g0 stack if we aren't already on g0 or gsignal. + MOVV m_gsignal(R12), R13 + BEQ R13, g, call + + MOVV m_g0(R12), R13 + BEQ R13, g, call + MOVV (g_sched+gobuf_sp)(R13), R3 +call: + JAL (RCALL) + MOVV R23, R3 + JAL (R24) + RET + +// C->Go callback thunk that allows to call runtime·racesymbolize from C code. +// Direct Go->C race call has only switched SP, finish g->g0 switch by setting correct g. +// The overall effect of Go->C->Go call chain is similar to that of mcall. +// RARG0 contains command code. RARG1 contains command-specific context. +// See racecallback for command codes. +TEXT runtime·racecallbackthunk(SB), NOSPLIT|NOFRAME, $0 + // Handle command raceGetProcCmd (0) here. + // First, code below assumes that we are on curg, while raceGetProcCmd + // can be executed on g0. Second, it is called frequently, so will + // benefit from this fast path. + BNE RARG0, R0, rest + MOVV g, R15 + load_g + MOVV g_m(g), RARG0 + MOVV m_p(RARG0), RARG0 + MOVV p_raceprocctx(RARG0), RARG0 + MOVV RARG0, (RARG1) + MOVV R15, g + JMP (R1) +rest: + // Save callee-saved registers (Go code won't respect that). + // 8(R3) and 16(R3) are for args passed through racecallback + ADDV $-176, R3 + MOVV R1, 0(R3) + + SAVE_R22_TO_R31(8*3) + SAVE_F24_TO_F31(8*13) + // Set g = g0. + load_g + MOVV g_m(g), R15 + MOVV m_g0(R15), R14 + BEQ R14, g, noswitch // branch if already on g0 + MOVV R14, g + + JAL runtime·racecallback(SB) + // All registers are smashed after Go code, reload. + MOVV g_m(g), R15 + MOVV m_curg(R15), g // g = m->curg +ret: + // Restore callee-saved registers. + MOVV 0(R3), R1 + RESTORE_F24_TO_F31(8*13) + RESTORE_R22_TO_R31(8*3) + ADDV $176, R3 + JMP (R1) + +noswitch: + // already on g0 + JAL runtime·racecallback(SB) + JMP ret + +// tls_g, g value for each thread in TLS +GLOBL runtime·tls_g+0(SB), TLSBSS+DUPOK, $8 diff --git a/src/runtime/race_ppc64le.s b/src/runtime/race_ppc64le.s index 43829479bde868..b327e49a2f181e 100644 --- a/src/runtime/race_ppc64le.s +++ b/src/runtime/race_ppc64le.s @@ -153,7 +153,7 @@ data: BLT ret MOVD runtime·racedataend(SB), R9 CMP R4, R9 - BGT ret + BGE ret call: // Careful!! racecall will save LR on its // stack, which is OK as long as racecalladdr @@ -484,9 +484,16 @@ TEXT racecall<>(SB), NOSPLIT, $0-0 MOVD 0(R10), g MOVD g_m(g), R7 // m for g MOVD R1, R16 // callee-saved, preserved across C call - MOVD m_g0(R7), R10 // g0 for m - CMP R10, g // same g0? - BEQ call // already on g0 + + // Switch to g0 stack if we aren't already on g0 or gsignal. + MOVD m_gsignal(R7), R10 + CMP R10, g + BEQ call + + MOVD m_g0(R7), R10 + CMP R10, g + BEQ call + MOVD (g_sched+gobuf_sp)(R10), R1 // switch R1 call: // prepare frame for C ABI diff --git a/src/runtime/race_riscv64.s b/src/runtime/race_riscv64.s new file mode 100644 index 00000000000000..9992a519ebe2a0 --- /dev/null +++ b/src/runtime/race_riscv64.s @@ -0,0 +1,551 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build race + +#include "go_asm.h" +#include "funcdata.h" +#include "textflag.h" + +// The following thunks allow calling the gcc-compiled race runtime directly +// from Go code without going all the way through cgo. +// First, it's much faster (up to 50% speedup for real Go programs). +// Second, it eliminates race-related special cases from cgocall and scheduler. +// Third, in long-term it will allow to remove cyclic runtime/race dependency on cmd/go. + +// A brief recap of the riscv C calling convention. +// Arguments are passed in X10...X17 +// Callee-saved registers are: X8, X9, X18..X27 +// Temporary registers are: X5..X7, X28..X31 + +// When calling racecalladdr, X11 is the call target address. + +// The race ctx, ThreadState *thr below, is passed in X10 and loaded in racecalladdr. + +// func runtime·raceread(addr uintptr) +// Called from instrumented code. +TEXT runtime·raceread(SB), NOSPLIT, $0-8 + // void __tsan_read(ThreadState *thr, void *addr, void *pc); + MOV $__tsan_read(SB), X5 + MOV X10, X11 + MOV X1, X12 + JMP racecalladdr<>(SB) + +// func runtime·RaceRead(addr uintptr) +TEXT runtime·RaceRead(SB), NOSPLIT, $0-8 + // This needs to be a tail call, because raceread reads caller pc. + JMP runtime·raceread(SB) + +// func runtime·racereadpc(void *addr, void *callpc, void *pc) +TEXT runtime·racereadpc(SB), NOSPLIT, $0-24 + // void __tsan_read_pc(ThreadState *thr, void *addr, void *callpc, void *pc); + MOV $__tsan_read_pc(SB), X5 + MOV addr+0(FP), X11 + MOV callpc+8(FP), X12 + MOV pc+16(FP), X13 + JMP racecalladdr<>(SB) + +// func runtime·racewrite(addr uintptr) +// Called from instrumented code. +TEXT runtime·racewrite(SB), NOSPLIT, $0-8 + // void __tsan_write(ThreadState *thr, void *addr, void *pc); + MOV $__tsan_write(SB), X5 + MOV X10, X11 + MOV X1, X12 + JMP racecalladdr<>(SB) + +// func runtime·RaceWrite(addr uintptr) +TEXT runtime·RaceWrite(SB), NOSPLIT, $0-8 + // This needs to be a tail call, because racewrite reads caller pc. + JMP runtime·racewrite(SB) + +// func runtime·racewritepc(void *addr, void *callpc, void *pc) +TEXT runtime·racewritepc(SB), NOSPLIT, $0-24 + // void __tsan_write_pc(ThreadState *thr, void *addr, void *callpc, void *pc); + MOV $__tsan_write_pc(SB), X5 + MOV addr+0(FP), X11 + MOV callpc+8(FP), X12 + MOV pc+16(FP), X13 + JMP racecalladdr<>(SB) + +// func runtime·racereadrange(addr, size uintptr) +// Called from instrumented code. +TEXT runtime·racereadrange(SB), NOSPLIT, $0-16 + // void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, void *pc); + MOV $__tsan_read_range(SB), X5 + MOV X11, X12 + MOV X10, X11 + MOV X1, X13 + JMP racecalladdr<>(SB) + +// func runtime·RaceReadRange(addr, size uintptr) +TEXT runtime·RaceReadRange(SB), NOSPLIT, $0-16 + // This needs to be a tail call, because racereadrange reads caller pc. + JMP runtime·racereadrange(SB) + +// func runtime·racereadrangepc1(void *addr, uintptr sz, void *pc) +TEXT runtime·racereadrangepc1(SB), NOSPLIT, $0-24 + // void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, void *pc); + MOV $__tsan_read_range(SB), X5 + MOV addr+0(FP), X11 + MOV size+8(FP), X12 + MOV pc+16(FP), X13 + + // pc is an interceptor address, but TSan expects it to point to the + // middle of an interceptor (see LLVM's SCOPED_INTERCEPTOR_RAW). + ADD $4, X13 + JMP racecalladdr<>(SB) + +// func runtime·racewriterange(addr, size uintptr) +// Called from instrumented code. +TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 + // void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, void *pc); + MOV $__tsan_write_range(SB), X5 + MOV X11, X12 + MOV X10, X11 + MOV X1, X13 + JMP racecalladdr<>(SB) + +// func runtime·RaceWriteRange(addr, size uintptr) +TEXT runtime·RaceWriteRange(SB), NOSPLIT, $0-16 + // This needs to be a tail call, because racewriterange reads caller pc. + JMP runtime·racewriterange(SB) + +// func runtime·racewriterangepc1(void *addr, uintptr sz, void *pc) +TEXT runtime·racewriterangepc1(SB), NOSPLIT, $0-24 + // void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, void *pc); + MOV $__tsan_write_range(SB), X5 + MOV addr+0(FP), X11 + MOV size+8(FP), X12 + MOV pc+16(FP), X13 + // pc is an interceptor address, but TSan expects it to point to the + // middle of an interceptor (see LLVM's SCOPED_INTERCEPTOR_RAW). + ADD $4, X13 + JMP racecalladdr<>(SB) + +// If addr (X11) is out of range, do nothing. Otherwise, setup goroutine context and +// invoke racecall. Other arguments are already set. +TEXT racecalladdr<>(SB), NOSPLIT, $0-0 + MOV runtime·racearenastart(SB), X7 + BLT X11, X7, data // Before racearena start? + MOV runtime·racearenaend(SB), X7 + BLT X11, X7, call // Before racearena end? +data: + MOV runtime·racedatastart(SB), X7 + BLT X11, X7, ret // Before racedata start? + MOV runtime·racedataend(SB), X7 + BGE X11, X7, ret // At or after racedata end? +call: + MOV g_racectx(g), X10 + JMP racecall<>(SB) +ret: + RET + +// func runtime·racefuncenter(pc uintptr) +// Called from instrumented code. +TEXT runtime·racefuncenter(SB), NOSPLIT, $0-8 + MOV $__tsan_func_enter(SB), X5 + MOV X10, X11 + MOV g_racectx(g), X10 + JMP racecall<>(SB) + +// Common code for racefuncenter +// X1 = caller's return address +TEXT racefuncenter<>(SB), NOSPLIT, $0-0 + // void __tsan_func_enter(ThreadState *thr, void *pc); + MOV $__tsan_func_enter(SB), X5 + MOV g_racectx(g), X10 + MOV X1, X11 + JMP racecall<>(SB) + +// func runtime·racefuncexit() +// Called from instrumented code. +TEXT runtime·racefuncexit(SB), NOSPLIT, $0-0 + // void __tsan_func_exit(ThreadState *thr); + MOV $__tsan_func_exit(SB), X5 + MOV g_racectx(g), X10 + JMP racecall<>(SB) + +// Atomic operations for sync/atomic package. + +// Load + +TEXT sync∕atomic·LoadInt32(SB), NOSPLIT, $0-12 + GO_ARGS + MOV $__tsan_go_atomic32_load(SB), X5 + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·LoadInt64(SB), NOSPLIT, $0-16 + GO_ARGS + MOV $__tsan_go_atomic64_load(SB), X5 + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·LoadUint32(SB), NOSPLIT, $0-12 + GO_ARGS + JMP sync∕atomic·LoadInt32(SB) + +TEXT sync∕atomic·LoadUint64(SB), NOSPLIT, $0-16 + GO_ARGS + JMP sync∕atomic·LoadInt64(SB) + +TEXT sync∕atomic·LoadUintptr(SB), NOSPLIT, $0-16 + GO_ARGS + JMP sync∕atomic·LoadInt64(SB) + +TEXT sync∕atomic·LoadPointer(SB), NOSPLIT, $0-16 + GO_ARGS + JMP sync∕atomic·LoadInt64(SB) + +// Store + +TEXT sync∕atomic·StoreInt32(SB), NOSPLIT, $0-12 + GO_ARGS + MOV $__tsan_go_atomic32_store(SB), X5 + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·StoreInt64(SB), NOSPLIT, $0-16 + GO_ARGS + MOV $__tsan_go_atomic64_store(SB), X5 + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·StoreUint32(SB), NOSPLIT, $0-12 + GO_ARGS + JMP sync∕atomic·StoreInt32(SB) + +TEXT sync∕atomic·StoreUint64(SB), NOSPLIT, $0-16 + GO_ARGS + JMP sync∕atomic·StoreInt64(SB) + +TEXT sync∕atomic·StoreUintptr(SB), NOSPLIT, $0-16 + GO_ARGS + JMP sync∕atomic·StoreInt64(SB) + +// Swap + +TEXT sync∕atomic·SwapInt32(SB), NOSPLIT, $0-20 + GO_ARGS + MOV $__tsan_go_atomic32_exchange(SB), X5 + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·SwapInt64(SB), NOSPLIT, $0-24 + GO_ARGS + MOV $__tsan_go_atomic64_exchange(SB), X5 + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·SwapUint32(SB), NOSPLIT, $0-20 + GO_ARGS + JMP sync∕atomic·SwapInt32(SB) + +TEXT sync∕atomic·SwapUint64(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·SwapInt64(SB) + +TEXT sync∕atomic·SwapUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·SwapInt64(SB) + +// Add + +TEXT sync∕atomic·AddInt32(SB), NOSPLIT, $0-20 + GO_ARGS + MOV $__tsan_go_atomic32_fetch_add(SB), X5 + CALL racecallatomic<>(SB) + // TSan performed fetch_add, but Go needs add_fetch. + MOVW add+8(FP), X5 + MOVW ret+16(FP), X6 + ADD X5, X6, X5 + MOVW X5, ret+16(FP) + RET + +TEXT sync∕atomic·AddInt64(SB), NOSPLIT, $0-24 + GO_ARGS + MOV $__tsan_go_atomic64_fetch_add(SB), X5 + CALL racecallatomic<>(SB) + // TSan performed fetch_add, but Go needs add_fetch. + MOV add+8(FP), X5 + MOV ret+16(FP), X6 + ADD X5, X6, X5 + MOV X5, ret+16(FP) + RET + +TEXT sync∕atomic·AddUint32(SB), NOSPLIT, $0-20 + GO_ARGS + JMP sync∕atomic·AddInt32(SB) + +TEXT sync∕atomic·AddUint64(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·AddInt64(SB) + +TEXT sync∕atomic·AddUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·AddInt64(SB) + +// And +TEXT sync∕atomic·AndInt32(SB), NOSPLIT, $0-20 + GO_ARGS + MOV $__tsan_go_atomic32_fetch_and(SB), X5 + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·AndInt64(SB), NOSPLIT, $0-24 + GO_ARGS + MOV $__tsan_go_atomic64_fetch_and(SB), X5 + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·AndUint32(SB), NOSPLIT, $0-20 + GO_ARGS + JMP sync∕atomic·AndInt32(SB) + +TEXT sync∕atomic·AndUint64(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·AndInt64(SB) + +TEXT sync∕atomic·AndUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·AndInt64(SB) + +// Or +TEXT sync∕atomic·OrInt32(SB), NOSPLIT, $0-20 + GO_ARGS + MOV $__tsan_go_atomic32_fetch_or(SB), X5 + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·OrInt64(SB), NOSPLIT, $0-24 + GO_ARGS + MOV $__tsan_go_atomic64_fetch_or(SB), X5 + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·OrUint32(SB), NOSPLIT, $0-20 + GO_ARGS + JMP sync∕atomic·OrInt32(SB) + +TEXT sync∕atomic·OrUint64(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·OrInt64(SB) + +TEXT sync∕atomic·OrUintptr(SB), NOSPLIT, $0-24 + GO_ARGS + JMP sync∕atomic·OrInt64(SB) + +// CompareAndSwap + +TEXT sync∕atomic·CompareAndSwapInt32(SB), NOSPLIT, $0-17 + GO_ARGS + MOV $__tsan_go_atomic32_compare_exchange(SB), X5 + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·CompareAndSwapInt64(SB), NOSPLIT, $0-25 + GO_ARGS + MOV $__tsan_go_atomic64_compare_exchange(SB), X5 + CALL racecallatomic<>(SB) + RET + +TEXT sync∕atomic·CompareAndSwapUint32(SB), NOSPLIT, $0-17 + GO_ARGS + JMP sync∕atomic·CompareAndSwapInt32(SB) + +TEXT sync∕atomic·CompareAndSwapUint64(SB), NOSPLIT, $0-25 + GO_ARGS + JMP sync∕atomic·CompareAndSwapInt64(SB) + +TEXT sync∕atomic·CompareAndSwapUintptr(SB), NOSPLIT, $0-25 + GO_ARGS + JMP sync∕atomic·CompareAndSwapInt64(SB) + +// Generic atomic operation implementation. +// X5 = addr of target function +TEXT racecallatomic<>(SB), NOSPLIT, $0 + // Set up these registers + // X10 = *ThreadState + // X11 = caller pc + // X12 = pc + // X13 = addr of incoming arg list + + // Trigger SIGSEGV early. + MOV 24(X2), X6 // 1st arg is addr. after two times CALL, get it at 24(X2) + MOVB (X6), X0 // segv here if addr is bad + // Check that addr is within [arenastart, arenaend) or within [racedatastart, racedataend). + MOV runtime·racearenastart(SB), X7 + BLT X6, X7, racecallatomic_data + MOV runtime·racearenaend(SB), X7 + BLT X6, X7, racecallatomic_ok +racecallatomic_data: + MOV runtime·racedatastart(SB), X7 + BLT X6, X7, racecallatomic_ignore + MOV runtime·racedataend(SB), X7 + BGE X6, X7, racecallatomic_ignore +racecallatomic_ok: + // Addr is within the good range, call the atomic function. + MOV g_racectx(g), X10 // goroutine context + MOV 8(X2), X11 // caller pc + MOV X1, X12 // pc + ADD $24, X2, X13 + CALL racecall<>(SB) + RET +racecallatomic_ignore: + // Addr is outside the good range. + // Call __tsan_go_ignore_sync_begin to ignore synchronization during the atomic op. + // An attempt to synchronize on the address would cause crash. + MOV X1, X20 // save PC + MOV X5, X21 // save target function + MOV $__tsan_go_ignore_sync_begin(SB), X5 + MOV g_racectx(g), X10 // goroutine context + CALL racecall<>(SB) + MOV X21, X5 // restore the target function + // Call the atomic function. + MOV g_racectx(g), X10 // goroutine context + MOV 8(X2), X11 // caller pc + MOV X20, X12 // pc + ADD $24, X2, X13 // arguments + CALL racecall<>(SB) + // Call __tsan_go_ignore_sync_end. + MOV $__tsan_go_ignore_sync_end(SB), X5 + MOV g_racectx(g), X10 // goroutine context + CALL racecall<>(SB) + RET + +// func runtime·racecall(void(*f)(...), ...) +// Calls C function f from race runtime and passes up to 4 arguments to it. +// The arguments are never heap-object-preserving pointers, so we pretend there +// are no arguments. +TEXT runtime·racecall(SB), NOSPLIT, $0-0 + MOV fn+0(FP), X5 + MOV arg0+8(FP), X10 + MOV arg1+16(FP), X11 + MOV arg2+24(FP), X12 + MOV arg3+32(FP), X13 + JMP racecall<>(SB) + +// Switches SP to g0 stack and calls X5. Arguments are already set. +TEXT racecall<>(SB), NOSPLIT|NOFRAME, $0-0 + MOV X1, X18 // Save RA in callee save register + MOV X2, X19 // Save SP in callee save register + CALL runtime·save_g(SB) // Save g for callbacks + + MOV g_m(g), X6 + + // Switch to g0 stack if we aren't already on g0 or gsignal. + MOV m_gsignal(X6), X7 + BEQ X7, g, call + MOV m_g0(X6), X7 + BEQ X7, g, call + + MOV (g_sched+gobuf_sp)(X7), X2 // Switch to g0 stack +call: + JALR RA, (X5) // Call C function + MOV X19, X2 // Restore SP + JMP (X18) // Return to Go. + +// C->Go callback thunk that allows to call runtime·racesymbolize from C code. +// Direct Go->C race call has only switched SP, finish g->g0 switch by setting correct g. +// The overall effect of Go->C->Go call chain is similar to that of mcall. +// R0 contains command code. R1 contains command-specific context. +// See racecallback for command codes. +TEXT runtime·racecallbackthunk(SB), NOSPLIT|NOFRAME, $0 + // Handle command raceGetProcCmd (0) here. + // First, code below assumes that we are on curg, while raceGetProcCmd + // can be executed on g0. Second, it is called frequently, so will + // benefit from this fast path. + BNEZ X10, rest + MOV X1, X5 + MOV g, X6 + CALL runtime·load_g(SB) + MOV g_m(g), X7 + MOV m_p(X7), X7 + MOV p_raceprocctx(X7), X7 + MOV X7, (X11) + MOV X6, g + JMP (X5) +rest: + // Save callee-save registers (X8, X9, X18..X27, F8, F9, F18..F27), + // since Go code will not respect this. + // 8(X2) and 16(X2) are for args passed to racecallback + SUB $(27*8), X2 + MOV X1, (0*8)(X2) + MOV X8, (3*8)(X2) + MOV X9, (4*8)(X2) + MOV X18, (5*8)(X2) + MOV X19, (6*8)(X2) + MOV X20, (7*8)(X2) + MOV X21, (8*8)(X2) + MOV X22, (9*8)(X2) + MOV X23, (10*8)(X2) + MOV X24, (11*8)(X2) + MOV X25, (12*8)(X2) + MOV X26, (13*8)(X2) + MOV g, (14*8)(X2) + MOVD F8, (15*8)(X2) + MOVD F9, (16*8)(X2) + MOVD F18, (17*8)(X2) + MOVD F19, (18*8)(X2) + MOVD F20, (19*8)(X2) + MOVD F21, (20*8)(X2) + MOVD F22, (21*8)(X2) + MOVD F23, (22*8)(X2) + MOVD F24, (23*8)(X2) + MOVD F25, (24*8)(X2) + MOVD F26, (25*8)(X2) + MOVD F27, (26*8)(X2) + + // Set g = g0. + CALL runtime·load_g(SB) + MOV g_m(g), X5 + MOV m_g0(X5), X6 + BEQ X6, g, noswitch // branch if already on g0 + MOV X6, g + + MOV X10, 8(X2) // func arg + MOV X11, 16(X2) // func arg + CALL runtime·racecallback(SB) + + // All registers are smashed after Go code, reload. + MOV g_m(g), X5 + MOV m_curg(X5), g // g = m->curg +ret: + // Restore callee-save registers. + MOV (0*8)(X2), X1 + MOV (3*8)(X2), X8 + MOV (4*8)(X2), X9 + MOV (5*8)(X2), X18 + MOV (6*8)(X2), X19 + MOV (7*8)(X2), X20 + MOV (8*8)(X2), X21 + MOV (9*8)(X2), X22 + MOV (10*8)(X2), X23 + MOV (11*8)(X2), X24 + MOV (12*8)(X2), X25 + MOV (13*8)(X2), X26 + MOV (14*8)(X2), g + MOVD (15*8)(X2), F8 + MOVD (16*8)(X2), F9 + MOVD (17*8)(X2), F18 + MOVD (18*8)(X2), F19 + MOVD (19*8)(X2), F20 + MOVD (20*8)(X2), F21 + MOVD (21*8)(X2), F22 + MOVD (22*8)(X2), F23 + MOVD (23*8)(X2), F24 + MOVD (24*8)(X2), F25 + MOVD (25*8)(X2), F26 + MOVD (26*8)(X2), F27 + + ADD $(27*8), X2 + JMP (X1) + +noswitch: + // already on g0 + MOV X10, 8(X2) // func arg + MOV X11, 16(X2) // func arg + CALL runtime·racecallback(SB) + JMP ret diff --git a/src/runtime/race_s390x.s b/src/runtime/race_s390x.s index 8e6a5d576ab4ce..3dfda9e733b09d 100644 --- a/src/runtime/race_s390x.s +++ b/src/runtime/race_s390x.s @@ -410,9 +410,16 @@ TEXT racecall<>(SB), NOSPLIT, $0-0 BL runtime·save_g(SB) // Save g for callbacks. MOVD R15, R7 // Save SP. MOVD g_m(g), R8 // R8 = thread. - MOVD m_g0(R8), R8 // R8 = g0. - CMPBEQ R8, g, call // Already on g0? - MOVD (g_sched+gobuf_sp)(R8), R15 // Switch SP to g0. + + // Switch to g0 stack if we aren't already on g0 or gsignal. + MOVD m_gsignal(R8), R9 + CMPBEQ R9, g, call + + MOVD m_g0(R8), R9 + CMPBEQ R9, g, call + + MOVD (g_sched+gobuf_sp)(R9), R15 // Switch SP to g0. + call: SUB $160, R15 // Allocate C frame. BL R1 // Call C code. MOVD R7, R15 // Restore SP. diff --git a/src/runtime/rand.go b/src/runtime/rand.go index 11be6552aa3314..1739e9f8f5f4eb 100644 --- a/src/runtime/rand.go +++ b/src/runtime/rand.go @@ -7,6 +7,7 @@ package runtime import ( + "internal/byteorder" "internal/chacha8rand" "internal/goarch" "internal/runtime/math" @@ -41,14 +42,15 @@ func randinit() { } seed := &globalRand.seed - if startupRand != nil { + if len(startupRand) >= 16 && + // Check that at least the first two words of startupRand weren't + // cleared by any libc initialization. + !allZero(startupRand[:8]) && !allZero(startupRand[8:16]) { for i, c := range startupRand { seed[i%len(seed)] ^= c } - clear(startupRand) - startupRand = nil } else { - if readRandom(seed[:]) != len(seed) { + if readRandom(seed[:]) != len(seed) || allZero(seed[:]) { // readRandom should never fail, but if it does we'd rather // not make Go binaries completely unusable, so make up // some random data based on the current time. @@ -58,6 +60,25 @@ func randinit() { } globalRand.state.Init(*seed) clear(seed[:]) + + if startupRand != nil { + // Overwrite startupRand instead of clearing it, in case cgo programs + // access it after we used it. + for len(startupRand) > 0 { + buf := make([]byte, 8) + for { + if x, ok := globalRand.state.Next(); ok { + byteorder.BEPutUint64(buf, x) + break + } + globalRand.state.Refill() + } + n := copy(startupRand, buf) + startupRand = startupRand[n:] + } + startupRand = nil + } + globalRand.init = true unlock(&globalRand.lock) } @@ -88,6 +109,14 @@ func readTimeRandom(r []byte) { } } +func allZero(b []byte) bool { + var acc byte + for _, x := range b { + acc |= x + } + return acc == 0 +} + // bootstrapRand returns a random uint64 from the global random generator. func bootstrapRand() uint64 { lock(&globalRand.lock) @@ -122,6 +151,8 @@ func rand32() uint32 { } // rand returns a random uint64 from the per-m chacha8 state. +// This is called from compiler-generated code. +// // Do not change signature: used via linkname from other packages. // //go:nosplit @@ -148,6 +179,11 @@ func rand() uint64 { } } +//go:linkname maps_rand internal/runtime/maps.rand +func maps_rand() uint64 { + return rand() +} + // mrandinit initializes the random state of an m. func mrandinit(mp *m) { var seed [4]uint64 diff --git a/src/runtime/rt0_aix_ppc64.s b/src/runtime/rt0_aix_ppc64.s index 1670a809862a2b..518fcb3b88d670 100644 --- a/src/runtime/rt0_aix_ppc64.s +++ b/src/runtime/rt0_aix_ppc64.s @@ -41,150 +41,7 @@ TEXT _main(SB),NOSPLIT,$-8 MOVD R12, CTR BR (CTR) - -TEXT _rt0_ppc64_aix_lib(SB),NOSPLIT,$-8 - // Start with standard C stack frame layout and linkage. - MOVD LR, R0 - MOVD R0, 16(R1) // Save LR in caller's frame. - MOVW CR, R0 // Save CR in caller's frame - MOVD R0, 8(R1) - - MOVDU R1, -344(R1) // Allocate frame. - - // Preserve callee-save registers. - MOVD R14, 48(R1) - MOVD R15, 56(R1) - MOVD R16, 64(R1) - MOVD R17, 72(R1) - MOVD R18, 80(R1) - MOVD R19, 88(R1) - MOVD R20, 96(R1) - MOVD R21,104(R1) - MOVD R22, 112(R1) - MOVD R23, 120(R1) - MOVD R24, 128(R1) - MOVD R25, 136(R1) - MOVD R26, 144(R1) - MOVD R27, 152(R1) - MOVD R28, 160(R1) - MOVD R29, 168(R1) - MOVD g, 176(R1) // R30 - MOVD R31, 184(R1) - FMOVD F14, 192(R1) - FMOVD F15, 200(R1) - FMOVD F16, 208(R1) - FMOVD F17, 216(R1) - FMOVD F18, 224(R1) - FMOVD F19, 232(R1) - FMOVD F20, 240(R1) - FMOVD F21, 248(R1) - FMOVD F22, 256(R1) - FMOVD F23, 264(R1) - FMOVD F24, 272(R1) - FMOVD F25, 280(R1) - FMOVD F26, 288(R1) - FMOVD F27, 296(R1) - FMOVD F28, 304(R1) - FMOVD F29, 312(R1) - FMOVD F30, 320(R1) - FMOVD F31, 328(R1) - - // Synchronous initialization. - MOVD $runtime·reginit(SB), R12 - MOVD R12, CTR - BL (CTR) - - MOVBZ runtime·isarchive(SB), R3 // Check buildmode = c-archive - CMP $0, R3 - BEQ done - - MOVD R14, _rt0_ppc64_aix_lib_argc<>(SB) - MOVD R15, _rt0_ppc64_aix_lib_argv<>(SB) - - MOVD $runtime·libpreinit(SB), R12 - MOVD R12, CTR - BL (CTR) - - // Create a new thread to do the runtime initialization and return. - MOVD _cgo_sys_thread_create(SB), R12 - CMP $0, R12 - BEQ nocgo - MOVD $_rt0_ppc64_aix_lib_go(SB), R3 - MOVD $0, R4 - MOVD R2, 40(R1) - MOVD 8(R12), R2 - MOVD (R12), R12 - MOVD R12, CTR - BL (CTR) - MOVD 40(R1), R2 - BR done - -nocgo: - MOVD $0x800000, R12 // stacksize = 8192KB - MOVD R12, 8(R1) - MOVD $_rt0_ppc64_aix_lib_go(SB), R12 - MOVD R12, 16(R1) - MOVD $runtime·newosproc0(SB),R12 - MOVD R12, CTR - BL (CTR) - -done: - // Restore saved registers. - MOVD 48(R1), R14 - MOVD 56(R1), R15 - MOVD 64(R1), R16 - MOVD 72(R1), R17 - MOVD 80(R1), R18 - MOVD 88(R1), R19 - MOVD 96(R1), R20 - MOVD 104(R1), R21 - MOVD 112(R1), R22 - MOVD 120(R1), R23 - MOVD 128(R1), R24 - MOVD 136(R1), R25 - MOVD 144(R1), R26 - MOVD 152(R1), R27 - MOVD 160(R1), R28 - MOVD 168(R1), R29 - MOVD 176(R1), g // R30 - MOVD 184(R1), R31 - FMOVD 196(R1), F14 - FMOVD 200(R1), F15 - FMOVD 208(R1), F16 - FMOVD 216(R1), F17 - FMOVD 224(R1), F18 - FMOVD 232(R1), F19 - FMOVD 240(R1), F20 - FMOVD 248(R1), F21 - FMOVD 256(R1), F22 - FMOVD 264(R1), F23 - FMOVD 272(R1), F24 - FMOVD 280(R1), F25 - FMOVD 288(R1), F26 - FMOVD 296(R1), F27 - FMOVD 304(R1), F28 - FMOVD 312(R1), F29 - FMOVD 320(R1), F30 - FMOVD 328(R1), F31 - - ADD $344, R1 - - MOVD 8(R1), R0 - MOVFL R0, $0xff - MOVD 16(R1), R0 - MOVD R0, LR - RET - -DEFINE_PPC64X_FUNCDESC(_rt0_ppc64_aix_lib_go, __rt0_ppc64_aix_lib_go) - -TEXT __rt0_ppc64_aix_lib_go(SB),NOSPLIT,$0 - MOVD _rt0_ppc64_aix_lib_argc<>(SB), R3 - MOVD _rt0_ppc64_aix_lib_argv<>(SB), R4 - MOVD $runtime·rt0_go(SB), R12 - MOVD R12, CTR - BR (CTR) - -DATA _rt0_ppc64_aix_lib_argc<>(SB)/8, $0 -GLOBL _rt0_ppc64_aix_lib_argc<>(SB),NOPTR, $8 -DATA _rt0_ppc64_aix_lib_argv<>(SB)/8, $0 -GLOBL _rt0_ppc64_aix_lib_argv<>(SB),NOPTR, $8 +TEXT _rt0_ppc64_aix_lib(SB),NOSPLIT,$0 + MOVD R14, R3 // argc + MOVD R15, R4 // argv + JMP _rt0_ppc64x_lib(SB) diff --git a/src/runtime/rt0_android_arm64.s b/src/runtime/rt0_android_arm64.s index 4135bf07d5f15c..ff4b6912220141 100644 --- a/src/runtime/rt0_android_arm64.s +++ b/src/runtime/rt0_android_arm64.s @@ -4,17 +4,15 @@ #include "textflag.h" -TEXT _rt0_arm64_android(SB),NOSPLIT|NOFRAME,$0 - MOVD $_rt0_arm64_linux(SB), R4 - B (R4) +TEXT _rt0_arm64_android(SB),NOSPLIT,$0 + JMP _rt0_arm64(SB) // When building with -buildmode=c-shared, this symbol is called when the shared // library is loaded. -TEXT _rt0_arm64_android_lib(SB),NOSPLIT|NOFRAME,$0 +TEXT _rt0_arm64_android_lib(SB),NOSPLIT,$0 MOVW $1, R0 // argc MOVD $_rt0_arm64_android_argv(SB), R1 // **argv - MOVD $_rt0_arm64_linux_lib(SB), R4 - B (R4) + JMP _rt0_arm64_lib(SB) DATA _rt0_arm64_android_argv+0x00(SB)/8,$_rt0_arm64_android_argv0(SB) DATA _rt0_arm64_android_argv+0x08(SB)/8,$0 // end argv diff --git a/src/runtime/rt0_darwin_arm64.s b/src/runtime/rt0_darwin_arm64.s index 697104ac641e8e..e9cd5fc7f5e1fc 100644 --- a/src/runtime/rt0_darwin_arm64.s +++ b/src/runtime/rt0_darwin_arm64.s @@ -3,61 +3,13 @@ // license that can be found in the LICENSE file. #include "textflag.h" -#include "cgo/abi_arm64.h" -TEXT _rt0_arm64_darwin(SB),NOSPLIT|NOFRAME,$0 - MOVD $runtime·rt0_go(SB), R2 - BL (R2) -exit: - MOVD $0, R0 - MOVD $1, R16 // sys_exit - SVC $0x80 - B exit +TEXT _rt0_arm64_darwin(SB),NOSPLIT,$0 + // Darwin puts argc and argv in R0 and R1, + // so there is no need to go through _rt0_arm64. + JMP runtime·rt0_go(SB) // When linking with -buildmode=c-archive or -buildmode=c-shared, // this symbol is called from a global initialization function. -// -// Note that all currently shipping darwin/arm64 platforms require -// cgo and do not support c-shared. -TEXT _rt0_arm64_darwin_lib(SB),NOSPLIT,$152 - // Preserve callee-save registers. - SAVE_R19_TO_R28(8) - SAVE_F8_TO_F15(88) - - MOVD R0, _rt0_arm64_darwin_lib_argc<>(SB) - MOVD R1, _rt0_arm64_darwin_lib_argv<>(SB) - - MOVD $0, g // initialize g to nil - - // Synchronous initialization. - MOVD $runtime·libpreinit(SB), R4 - BL (R4) - - // Create a new thread to do the runtime initialization and return. - MOVD _cgo_sys_thread_create(SB), R4 - MOVD $_rt0_arm64_darwin_lib_go(SB), R0 - MOVD $0, R1 - SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved. - BL (R4) - ADD $16, RSP - - // Restore callee-save registers. - RESTORE_R19_TO_R28(8) - RESTORE_F8_TO_F15(88) - - RET - -TEXT _rt0_arm64_darwin_lib_go(SB),NOSPLIT,$0 - MOVD _rt0_arm64_darwin_lib_argc<>(SB), R0 - MOVD _rt0_arm64_darwin_lib_argv<>(SB), R1 - MOVD $runtime·rt0_go(SB), R4 - B (R4) - -DATA _rt0_arm64_darwin_lib_argc<>(SB)/8, $0 -GLOBL _rt0_arm64_darwin_lib_argc<>(SB),NOPTR, $8 -DATA _rt0_arm64_darwin_lib_argv<>(SB)/8, $0 -GLOBL _rt0_arm64_darwin_lib_argv<>(SB),NOPTR, $8 - -// external linking entry point. -TEXT main(SB),NOSPLIT|NOFRAME,$0 - JMP _rt0_arm64_darwin(SB) +TEXT _rt0_arm64_darwin_lib(SB),NOSPLIT,$0 + JMP _rt0_arm64_lib(SB) diff --git a/src/runtime/rt0_freebsd_arm64.s b/src/runtime/rt0_freebsd_arm64.s index e517ae059dc793..a7a952664e575b 100644 --- a/src/runtime/rt0_freebsd_arm64.s +++ b/src/runtime/rt0_freebsd_arm64.s @@ -3,72 +3,12 @@ // license that can be found in the LICENSE file. #include "textflag.h" -#include "cgo/abi_arm64.h" // On FreeBSD argc/argv are passed in R0, not RSP -TEXT _rt0_arm64_freebsd(SB),NOSPLIT|NOFRAME,$0 - ADD $8, R0, R1 // argv - MOVD 0(R0), R0 // argc - BL main(SB) +TEXT _rt0_arm64_freebsd(SB),NOSPLIT,$0 + JMP _rt0_arm64(SB) // When building with -buildmode=c-shared, this symbol is called when the shared // library is loaded. -TEXT _rt0_arm64_freebsd_lib(SB),NOSPLIT,$184 - // Preserve callee-save registers. - SAVE_R19_TO_R28(24) - SAVE_F8_TO_F15(104) - - // Initialize g as null in case of using g later e.g. sigaction in cgo_sigaction.go - MOVD ZR, g - - MOVD R0, _rt0_arm64_freebsd_lib_argc<>(SB) - MOVD R1, _rt0_arm64_freebsd_lib_argv<>(SB) - - // Synchronous initialization. - MOVD $runtime·libpreinit(SB), R4 - BL (R4) - - // Create a new thread to do the runtime initialization and return. - MOVD _cgo_sys_thread_create(SB), R4 - CBZ R4, nocgo - MOVD $_rt0_arm64_freebsd_lib_go(SB), R0 - MOVD $0, R1 - SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved. - BL (R4) - ADD $16, RSP - B restore - -nocgo: - MOVD $0x800000, R0 // stacksize = 8192KB - MOVD $_rt0_arm64_freebsd_lib_go(SB), R1 - MOVD R0, 8(RSP) - MOVD R1, 16(RSP) - MOVD $runtime·newosproc0(SB),R4 - BL (R4) - -restore: - // Restore callee-save registers. - RESTORE_R19_TO_R28(24) - RESTORE_F8_TO_F15(104) - RET - -TEXT _rt0_arm64_freebsd_lib_go(SB),NOSPLIT,$0 - MOVD _rt0_arm64_freebsd_lib_argc<>(SB), R0 - MOVD _rt0_arm64_freebsd_lib_argv<>(SB), R1 - MOVD $runtime·rt0_go(SB),R4 - B (R4) - -DATA _rt0_arm64_freebsd_lib_argc<>(SB)/8, $0 -GLOBL _rt0_arm64_freebsd_lib_argc<>(SB),NOPTR, $8 -DATA _rt0_arm64_freebsd_lib_argv<>(SB)/8, $0 -GLOBL _rt0_arm64_freebsd_lib_argv<>(SB),NOPTR, $8 - - -TEXT main(SB),NOSPLIT|NOFRAME,$0 - MOVD $runtime·rt0_go(SB), R2 - BL (R2) -exit: - MOVD $0, R0 - MOVD $1, R8 // SYS_exit - SVC - B exit +TEXT _rt0_arm64_freebsd_lib(SB),NOSPLIT,$0 + JMP _rt0_arm64_lib(SB) diff --git a/src/runtime/rt0_freebsd_riscv64.s b/src/runtime/rt0_freebsd_riscv64.s index dc46b7047669ad..f3fb4ffbe75ffd 100644 --- a/src/runtime/rt0_freebsd_riscv64.s +++ b/src/runtime/rt0_freebsd_riscv64.s @@ -12,100 +12,8 @@ TEXT _rt0_riscv64_freebsd(SB),NOSPLIT|NOFRAME,$0 // When building with -buildmode=c-shared, this symbol is called when the shared // library is loaded. -TEXT _rt0_riscv64_freebsd_lib(SB),NOSPLIT,$224 - // Preserve callee-save registers, along with X1 (LR). - MOV X1, (8*3)(X2) - MOV X8, (8*4)(X2) - MOV X9, (8*5)(X2) - MOV X18, (8*6)(X2) - MOV X19, (8*7)(X2) - MOV X20, (8*8)(X2) - MOV X21, (8*9)(X2) - MOV X22, (8*10)(X2) - MOV X23, (8*11)(X2) - MOV X24, (8*12)(X2) - MOV X25, (8*13)(X2) - MOV X26, (8*14)(X2) - MOV g, (8*15)(X2) - MOVD F8, (8*16)(X2) - MOVD F9, (8*17)(X2) - MOVD F18, (8*18)(X2) - MOVD F19, (8*19)(X2) - MOVD F20, (8*20)(X2) - MOVD F21, (8*21)(X2) - MOVD F22, (8*22)(X2) - MOVD F23, (8*23)(X2) - MOVD F24, (8*24)(X2) - MOVD F25, (8*25)(X2) - MOVD F26, (8*26)(X2) - MOVD F27, (8*27)(X2) - - // Initialize g as nil in case of using g later e.g. sigaction in cgo_sigaction.go - MOV X0, g - - MOV A0, _rt0_riscv64_freebsd_lib_argc<>(SB) - MOV A1, _rt0_riscv64_freebsd_lib_argv<>(SB) - - // Synchronous initialization. - MOV $runtime·libpreinit(SB), T0 - JALR RA, T0 - - // Create a new thread to do the runtime initialization and return. - MOV _cgo_sys_thread_create(SB), T0 - BEQZ T0, nocgo - MOV $_rt0_riscv64_freebsd_lib_go(SB), A0 - MOV $0, A1 - JALR RA, T0 - JMP restore - -nocgo: - MOV $0x800000, A0 // stacksize = 8192KB - MOV $_rt0_riscv64_freebsd_lib_go(SB), A1 - MOV A0, 8(X2) - MOV A1, 16(X2) - MOV $runtime·newosproc0(SB), T0 - JALR RA, T0 - -restore: - // Restore callee-save registers, along with X1 (LR). - MOV (8*3)(X2), X1 - MOV (8*4)(X2), X8 - MOV (8*5)(X2), X9 - MOV (8*6)(X2), X18 - MOV (8*7)(X2), X19 - MOV (8*8)(X2), X20 - MOV (8*9)(X2), X21 - MOV (8*10)(X2), X22 - MOV (8*11)(X2), X23 - MOV (8*12)(X2), X24 - MOV (8*13)(X2), X25 - MOV (8*14)(X2), X26 - MOV (8*15)(X2), g - MOVD (8*16)(X2), F8 - MOVD (8*17)(X2), F9 - MOVD (8*18)(X2), F18 - MOVD (8*19)(X2), F19 - MOVD (8*20)(X2), F20 - MOVD (8*21)(X2), F21 - MOVD (8*22)(X2), F22 - MOVD (8*23)(X2), F23 - MOVD (8*24)(X2), F24 - MOVD (8*25)(X2), F25 - MOVD (8*26)(X2), F26 - MOVD (8*27)(X2), F27 - - RET - -TEXT _rt0_riscv64_freebsd_lib_go(SB),NOSPLIT,$0 - MOV _rt0_riscv64_freebsd_lib_argc<>(SB), A0 - MOV _rt0_riscv64_freebsd_lib_argv<>(SB), A1 - MOV $runtime·rt0_go(SB), T0 - JALR ZERO, T0 - -DATA _rt0_riscv64_freebsd_lib_argc<>(SB)/8, $0 -GLOBL _rt0_riscv64_freebsd_lib_argc<>(SB),NOPTR, $8 -DATA _rt0_riscv64_freebsd_lib_argv<>(SB)/8, $0 -GLOBL _rt0_riscv64_freebsd_lib_argv<>(SB),NOPTR, $8 +TEXT _rt0_riscv64_freebsd_lib(SB),NOSPLIT,$0 + JMP _rt0_riscv64_lib(SB) TEXT main(SB),NOSPLIT|NOFRAME,$0 MOV $runtime·rt0_go(SB), T0 diff --git a/src/runtime/rt0_ios_arm64.s b/src/runtime/rt0_ios_arm64.s index dcc83656e2c713..1ab945b31c9597 100644 --- a/src/runtime/rt0_ios_arm64.s +++ b/src/runtime/rt0_ios_arm64.s @@ -11,4 +11,4 @@ TEXT _rt0_arm64_ios(SB),NOSPLIT|NOFRAME,$0 // library entry point. TEXT _rt0_arm64_ios_lib(SB),NOSPLIT|NOFRAME,$0 - JMP _rt0_arm64_darwin_lib(SB) + JMP _rt0_arm64_lib(SB) diff --git a/src/runtime/rt0_js_wasm.s b/src/runtime/rt0_js_wasm.s index 34a60474f7a9be..c7a0a2636dce0c 100644 --- a/src/runtime/rt0_js_wasm.s +++ b/src/runtime/rt0_js_wasm.s @@ -53,12 +53,6 @@ TEXT wasm_export_getsp(SB),NOSPLIT,$0 Get SP Return -TEXT runtime·pause(SB), NOSPLIT, $0-8 - MOVD newsp+0(FP), SP - I32Const $1 - Set PAUSE - RETUNWIND - TEXT runtime·exit(SB), NOSPLIT, $0-4 I32Const $0 Call runtime·wasmExit(SB) diff --git a/src/runtime/rt0_linux_arm64.s b/src/runtime/rt0_linux_arm64.s index 0eb8fc2f481a08..eed9c5f9201b1d 100644 --- a/src/runtime/rt0_linux_arm64.s +++ b/src/runtime/rt0_linux_arm64.s @@ -3,71 +3,11 @@ // license that can be found in the LICENSE file. #include "textflag.h" -#include "cgo/abi_arm64.h" -TEXT _rt0_arm64_linux(SB),NOSPLIT|NOFRAME,$0 - MOVD 0(RSP), R0 // argc - ADD $8, RSP, R1 // argv - BL main(SB) +TEXT _rt0_arm64_linux(SB),NOSPLIT,$0 + JMP _rt0_arm64(SB) // When building with -buildmode=c-shared, this symbol is called when the shared // library is loaded. -TEXT _rt0_arm64_linux_lib(SB),NOSPLIT,$184 - // Preserve callee-save registers. - SAVE_R19_TO_R28(24) - SAVE_F8_TO_F15(104) - - // Initialize g as null in case of using g later e.g. sigaction in cgo_sigaction.go - MOVD ZR, g - - MOVD R0, _rt0_arm64_linux_lib_argc<>(SB) - MOVD R1, _rt0_arm64_linux_lib_argv<>(SB) - - // Synchronous initialization. - MOVD $runtime·libpreinit(SB), R4 - BL (R4) - - // Create a new thread to do the runtime initialization and return. - MOVD _cgo_sys_thread_create(SB), R4 - CBZ R4, nocgo - MOVD $_rt0_arm64_linux_lib_go(SB), R0 - MOVD $0, R1 - SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved. - BL (R4) - ADD $16, RSP - B restore - -nocgo: - MOVD $0x800000, R0 // stacksize = 8192KB - MOVD $_rt0_arm64_linux_lib_go(SB), R1 - MOVD R0, 8(RSP) - MOVD R1, 16(RSP) - MOVD $runtime·newosproc0(SB),R4 - BL (R4) - -restore: - // Restore callee-save registers. - RESTORE_R19_TO_R28(24) - RESTORE_F8_TO_F15(104) - RET - -TEXT _rt0_arm64_linux_lib_go(SB),NOSPLIT,$0 - MOVD _rt0_arm64_linux_lib_argc<>(SB), R0 - MOVD _rt0_arm64_linux_lib_argv<>(SB), R1 - MOVD $runtime·rt0_go(SB),R4 - B (R4) - -DATA _rt0_arm64_linux_lib_argc<>(SB)/8, $0 -GLOBL _rt0_arm64_linux_lib_argc<>(SB),NOPTR, $8 -DATA _rt0_arm64_linux_lib_argv<>(SB)/8, $0 -GLOBL _rt0_arm64_linux_lib_argv<>(SB),NOPTR, $8 - - -TEXT main(SB),NOSPLIT|NOFRAME,$0 - MOVD $runtime·rt0_go(SB), R2 - BL (R2) -exit: - MOVD $0, R0 - MOVD $94, R8 // sys_exit - SVC - B exit +TEXT _rt0_arm64_linux_lib(SB),NOSPLIT,$0 + JMP _rt0_arm64_lib(SB) diff --git a/src/runtime/rt0_linux_loong64.s b/src/runtime/rt0_linux_loong64.s index b52f7d530a6a98..d8da4461a2f56d 100644 --- a/src/runtime/rt0_linux_loong64.s +++ b/src/runtime/rt0_linux_loong64.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. #include "textflag.h" -#include "cgo/abi_loong64.h" TEXT _rt0_loong64_linux(SB),NOSPLIT|NOFRAME,$0 // In a statically linked binary, the stack contains argc, @@ -16,53 +15,8 @@ TEXT _rt0_loong64_linux(SB),NOSPLIT|NOFRAME,$0 // When building with -buildmode=c-shared, this symbol is called when the shared // library is loaded. -TEXT _rt0_loong64_linux_lib(SB),NOSPLIT,$168 - // Preserve callee-save registers. - SAVE_R22_TO_R31(3*8) - SAVE_F24_TO_F31(13*8) - - // Initialize g as nil in case of using g later e.g. sigaction in cgo_sigaction.go - MOVV R0, g - - MOVV R4, _rt0_loong64_linux_lib_argc<>(SB) - MOVV R5, _rt0_loong64_linux_lib_argv<>(SB) - - // Synchronous initialization. - MOVV $runtime·libpreinit(SB), R19 - JAL (R19) - - // Create a new thread to do the runtime initialization and return. - MOVV _cgo_sys_thread_create(SB), R19 - BEQ R19, nocgo - MOVV $_rt0_loong64_linux_lib_go(SB), R4 - MOVV $0, R5 - JAL (R19) - JMP restore - -nocgo: - MOVV $0x800000, R4 // stacksize = 8192KB - MOVV $_rt0_loong64_linux_lib_go(SB), R5 - MOVV R4, 8(R3) - MOVV R5, 16(R3) - MOVV $runtime·newosproc0(SB), R19 - JAL (R19) - -restore: - // Restore callee-save registers. - RESTORE_R22_TO_R31(3*8) - RESTORE_F24_TO_F31(13*8) - RET - -TEXT _rt0_loong64_linux_lib_go(SB),NOSPLIT,$0 - MOVV _rt0_loong64_linux_lib_argc<>(SB), R4 - MOVV _rt0_loong64_linux_lib_argv<>(SB), R5 - MOVV $runtime·rt0_go(SB),R19 - JMP (R19) - -DATA _rt0_loong64_linux_lib_argc<>(SB)/8, $0 -GLOBL _rt0_loong64_linux_lib_argc<>(SB),NOPTR, $8 -DATA _rt0_loong64_linux_lib_argv<>(SB)/8, $0 -GLOBL _rt0_loong64_linux_lib_argv<>(SB),NOPTR, $8 +TEXT _rt0_loong64_linux_lib(SB),NOSPLIT,$0 + JMP _rt0_loong64_lib(SB) TEXT main(SB),NOSPLIT|NOFRAME,$0 // in external linking, glibc jumps to main with argc in R4 diff --git a/src/runtime/rt0_linux_ppc64le.s b/src/runtime/rt0_linux_ppc64le.s index 4b7d8e1b940a1e..3a6e8863b2da1d 100644 --- a/src/runtime/rt0_linux_ppc64le.s +++ b/src/runtime/rt0_linux_ppc64le.s @@ -5,60 +5,13 @@ #include "go_asm.h" #include "textflag.h" #include "asm_ppc64x.h" -#include "cgo/abi_ppc64x.h" TEXT _rt0_ppc64le_linux(SB),NOSPLIT,$0 XOR R0, R0 // Make sure R0 is zero before _main BR _main<>(SB) TEXT _rt0_ppc64le_linux_lib(SB),NOSPLIT|NOFRAME,$0 - // This is called with ELFv2 calling conventions. Convert to Go. - // Allocate argument storage for call to newosproc0. - STACK_AND_SAVE_HOST_TO_GO_ABI(16) - - MOVD R3, _rt0_ppc64le_linux_lib_argc<>(SB) - MOVD R4, _rt0_ppc64le_linux_lib_argv<>(SB) - - // Synchronous initialization. - MOVD $runtime·libpreinit(SB), R12 - MOVD R12, CTR - BL (CTR) - - // Create a new thread to do the runtime initialization and return. - MOVD _cgo_sys_thread_create(SB), R12 - CMP $0, R12 - BEQ nocgo - MOVD $_rt0_ppc64le_linux_lib_go(SB), R3 - MOVD $0, R4 - MOVD R12, CTR - BL (CTR) - BR done - -nocgo: - MOVD $0x800000, R12 // stacksize = 8192KB - MOVD R12, 8+FIXED_FRAME(R1) - MOVD $_rt0_ppc64le_linux_lib_go(SB), R12 - MOVD R12, 16+FIXED_FRAME(R1) - MOVD $runtime·newosproc0(SB),R12 - MOVD R12, CTR - BL (CTR) - -done: - // Restore and return to ELFv2 caller. - UNSTACK_AND_RESTORE_GO_TO_HOST_ABI(16) - RET - -TEXT _rt0_ppc64le_linux_lib_go(SB),NOSPLIT,$0 - MOVD _rt0_ppc64le_linux_lib_argc<>(SB), R3 - MOVD _rt0_ppc64le_linux_lib_argv<>(SB), R4 - MOVD $runtime·rt0_go(SB), R12 - MOVD R12, CTR - BR (CTR) - -DATA _rt0_ppc64le_linux_lib_argc<>(SB)/8, $0 -GLOBL _rt0_ppc64le_linux_lib_argc<>(SB),NOPTR, $8 -DATA _rt0_ppc64le_linux_lib_argv<>(SB)/8, $0 -GLOBL _rt0_ppc64le_linux_lib_argv<>(SB),NOPTR, $8 + JMP _rt0_ppc64x_lib(SB) TEXT _main<>(SB),NOSPLIT,$-8 // In a statically linked binary, the stack contains argc, diff --git a/src/runtime/rt0_linux_riscv64.s b/src/runtime/rt0_linux_riscv64.s index d6b8ac85dca646..4139c9bcecdc62 100644 --- a/src/runtime/rt0_linux_riscv64.s +++ b/src/runtime/rt0_linux_riscv64.s @@ -11,100 +11,8 @@ TEXT _rt0_riscv64_linux(SB),NOSPLIT|NOFRAME,$0 // When building with -buildmode=c-shared, this symbol is called when the shared // library is loaded. -TEXT _rt0_riscv64_linux_lib(SB),NOSPLIT,$224 - // Preserve callee-save registers, along with X1 (LR). - MOV X1, (8*3)(X2) - MOV X8, (8*4)(X2) - MOV X9, (8*5)(X2) - MOV X18, (8*6)(X2) - MOV X19, (8*7)(X2) - MOV X20, (8*8)(X2) - MOV X21, (8*9)(X2) - MOV X22, (8*10)(X2) - MOV X23, (8*11)(X2) - MOV X24, (8*12)(X2) - MOV X25, (8*13)(X2) - MOV X26, (8*14)(X2) - MOV g, (8*15)(X2) - MOVD F8, (8*16)(X2) - MOVD F9, (8*17)(X2) - MOVD F18, (8*18)(X2) - MOVD F19, (8*19)(X2) - MOVD F20, (8*20)(X2) - MOVD F21, (8*21)(X2) - MOVD F22, (8*22)(X2) - MOVD F23, (8*23)(X2) - MOVD F24, (8*24)(X2) - MOVD F25, (8*25)(X2) - MOVD F26, (8*26)(X2) - MOVD F27, (8*27)(X2) - - // Initialize g as nil in case of using g later e.g. sigaction in cgo_sigaction.go - MOV X0, g - - MOV A0, _rt0_riscv64_linux_lib_argc<>(SB) - MOV A1, _rt0_riscv64_linux_lib_argv<>(SB) - - // Synchronous initialization. - MOV $runtime·libpreinit(SB), T0 - JALR RA, T0 - - // Create a new thread to do the runtime initialization and return. - MOV _cgo_sys_thread_create(SB), T0 - BEQZ T0, nocgo - MOV $_rt0_riscv64_linux_lib_go(SB), A0 - MOV $0, A1 - JALR RA, T0 - JMP restore - -nocgo: - MOV $0x800000, A0 // stacksize = 8192KB - MOV $_rt0_riscv64_linux_lib_go(SB), A1 - MOV A0, 8(X2) - MOV A1, 16(X2) - MOV $runtime·newosproc0(SB), T0 - JALR RA, T0 - -restore: - // Restore callee-save registers, along with X1 (LR). - MOV (8*3)(X2), X1 - MOV (8*4)(X2), X8 - MOV (8*5)(X2), X9 - MOV (8*6)(X2), X18 - MOV (8*7)(X2), X19 - MOV (8*8)(X2), X20 - MOV (8*9)(X2), X21 - MOV (8*10)(X2), X22 - MOV (8*11)(X2), X23 - MOV (8*12)(X2), X24 - MOV (8*13)(X2), X25 - MOV (8*14)(X2), X26 - MOV (8*15)(X2), g - MOVD (8*16)(X2), F8 - MOVD (8*17)(X2), F9 - MOVD (8*18)(X2), F18 - MOVD (8*19)(X2), F19 - MOVD (8*20)(X2), F20 - MOVD (8*21)(X2), F21 - MOVD (8*22)(X2), F22 - MOVD (8*23)(X2), F23 - MOVD (8*24)(X2), F24 - MOVD (8*25)(X2), F25 - MOVD (8*26)(X2), F26 - MOVD (8*27)(X2), F27 - - RET - -TEXT _rt0_riscv64_linux_lib_go(SB),NOSPLIT,$0 - MOV _rt0_riscv64_linux_lib_argc<>(SB), A0 - MOV _rt0_riscv64_linux_lib_argv<>(SB), A1 - MOV $runtime·rt0_go(SB), T0 - JALR ZERO, T0 - -DATA _rt0_riscv64_linux_lib_argc<>(SB)/8, $0 -GLOBL _rt0_riscv64_linux_lib_argc<>(SB),NOPTR, $8 -DATA _rt0_riscv64_linux_lib_argv<>(SB)/8, $0 -GLOBL _rt0_riscv64_linux_lib_argv<>(SB),NOPTR, $8 +TEXT _rt0_riscv64_linux_lib(SB),NOSPLIT,$0 + JMP _rt0_riscv64_lib(SB) TEXT main(SB),NOSPLIT|NOFRAME,$0 diff --git a/src/runtime/rt0_netbsd_arm64.s b/src/runtime/rt0_netbsd_arm64.s index 691a8e4be77eed..80802209b78f12 100644 --- a/src/runtime/rt0_netbsd_arm64.s +++ b/src/runtime/rt0_netbsd_arm64.s @@ -3,69 +3,13 @@ // license that can be found in the LICENSE file. #include "textflag.h" -#include "cgo/abi_arm64.h" -TEXT _rt0_arm64_netbsd(SB),NOSPLIT|NOFRAME,$0 +TEXT _rt0_arm64_netbsd(SB),NOSPLIT,$0 MOVD 0(RSP), R0 // argc ADD $8, RSP, R1 // argv - BL main(SB) + JMP main(SB) // When building with -buildmode=c-shared, this symbol is called when the shared // library is loaded. -TEXT _rt0_arm64_netbsd_lib(SB),NOSPLIT,$184 - // Preserve callee-save registers. - SAVE_R19_TO_R28(24) - SAVE_F8_TO_F15(104) - - // Initialize g as null in case of using g later e.g. sigaction in cgo_sigaction.go - MOVD ZR, g - - MOVD R0, _rt0_arm64_netbsd_lib_argc<>(SB) - MOVD R1, _rt0_arm64_netbsd_lib_argv<>(SB) - - // Synchronous initialization. - MOVD $runtime·libpreinit(SB), R4 - BL (R4) - - // Create a new thread to do the runtime initialization and return. - MOVD _cgo_sys_thread_create(SB), R4 - CBZ R4, nocgo - MOVD $_rt0_arm64_netbsd_lib_go(SB), R0 - MOVD $0, R1 - SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved. - BL (R4) - ADD $16, RSP - B restore - -nocgo: - MOVD $0x800000, R0 // stacksize = 8192KB - MOVD $_rt0_arm64_netbsd_lib_go(SB), R1 - MOVD R0, 8(RSP) - MOVD R1, 16(RSP) - MOVD $runtime·newosproc0(SB),R4 - BL (R4) - -restore: - // Restore callee-save registers. - RESTORE_R19_TO_R28(24) - RESTORE_F8_TO_F15(104) - RET - -TEXT _rt0_arm64_netbsd_lib_go(SB),NOSPLIT,$0 - MOVD _rt0_arm64_netbsd_lib_argc<>(SB), R0 - MOVD _rt0_arm64_netbsd_lib_argv<>(SB), R1 - MOVD $runtime·rt0_go(SB),R4 - B (R4) - -DATA _rt0_arm64_netbsd_lib_argc<>(SB)/8, $0 -GLOBL _rt0_arm64_netbsd_lib_argc<>(SB),NOPTR, $8 -DATA _rt0_arm64_netbsd_lib_argv<>(SB)/8, $0 -GLOBL _rt0_arm64_netbsd_lib_argv<>(SB),NOPTR, $8 - - -TEXT main(SB),NOSPLIT|NOFRAME,$0 - MOVD $runtime·rt0_go(SB), R2 - BL (R2) -exit: - MOVD $0, R0 - SVC $1 // sys_exit +TEXT _rt0_arm64_netbsd_lib(SB),NOSPLIT,$0 + JMP _rt0_arm64_lib(SB) diff --git a/src/runtime/rt0_openbsd_arm64.s b/src/runtime/rt0_openbsd_arm64.s index 49d49b34ac4f81..2050f6a2007879 100644 --- a/src/runtime/rt0_openbsd_arm64.s +++ b/src/runtime/rt0_openbsd_arm64.s @@ -3,77 +3,11 @@ // license that can be found in the LICENSE file. #include "textflag.h" -#include "cgo/abi_arm64.h" -// See comment in runtime/sys_openbsd_arm64.s re this construction. -#define INVOKE_SYSCALL \ - SVC; \ - NOOP; \ - NOOP - -TEXT _rt0_arm64_openbsd(SB),NOSPLIT|NOFRAME,$0 - MOVD 0(RSP), R0 // argc - ADD $8, RSP, R1 // argv - BL main(SB) +TEXT _rt0_arm64_openbsd(SB),NOSPLIT,$0 + JMP _rt0_arm64(SB) // When building with -buildmode=c-shared, this symbol is called when the shared // library is loaded. TEXT _rt0_arm64_openbsd_lib(SB),NOSPLIT,$184 - // Preserve callee-save registers. - SAVE_R19_TO_R28(24) - SAVE_F8_TO_F15(104) - - // Initialize g as null in case of using g later e.g. sigaction in cgo_sigaction.go - MOVD ZR, g - - MOVD R0, _rt0_arm64_openbsd_lib_argc<>(SB) - MOVD R1, _rt0_arm64_openbsd_lib_argv<>(SB) - - // Synchronous initialization. - MOVD $runtime·libpreinit(SB), R4 - BL (R4) - - // Create a new thread to do the runtime initialization and return. - MOVD _cgo_sys_thread_create(SB), R4 - CBZ R4, nocgo - MOVD $_rt0_arm64_openbsd_lib_go(SB), R0 - MOVD $0, R1 - SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved. - BL (R4) - ADD $16, RSP - B restore - -nocgo: - MOVD $0x800000, R0 // stacksize = 8192KB - MOVD $_rt0_arm64_openbsd_lib_go(SB), R1 - MOVD R0, 8(RSP) - MOVD R1, 16(RSP) - MOVD $runtime·newosproc0(SB),R4 - BL (R4) - -restore: - // Restore callee-save registers. - RESTORE_R19_TO_R28(24) - RESTORE_F8_TO_F15(104) - RET - -TEXT _rt0_arm64_openbsd_lib_go(SB),NOSPLIT,$0 - MOVD _rt0_arm64_openbsd_lib_argc<>(SB), R0 - MOVD _rt0_arm64_openbsd_lib_argv<>(SB), R1 - MOVD $runtime·rt0_go(SB),R4 - B (R4) - -DATA _rt0_arm64_openbsd_lib_argc<>(SB)/8, $0 -GLOBL _rt0_arm64_openbsd_lib_argc<>(SB),NOPTR, $8 -DATA _rt0_arm64_openbsd_lib_argv<>(SB)/8, $0 -GLOBL _rt0_arm64_openbsd_lib_argv<>(SB),NOPTR, $8 - - -TEXT main(SB),NOSPLIT|NOFRAME,$0 - MOVD $runtime·rt0_go(SB), R2 - BL (R2) -exit: - MOVD $0, R0 - MOVD $1, R8 // sys_exit - INVOKE_SYSCALL - B exit + JMP _rt0_arm64_lib(SB) diff --git a/src/runtime/rt0_openbsd_mips64.s b/src/runtime/rt0_openbsd_mips64.s deleted file mode 100644 index 82a8dfaba65f0b..00000000000000 --- a/src/runtime/rt0_openbsd_mips64.s +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "textflag.h" - -TEXT _rt0_mips64_openbsd(SB),NOSPLIT,$0 - JMP _main<>(SB) - -TEXT _rt0_mips64le_openbsd(SB),NOSPLIT,$0 - JMP _main<>(SB) - -TEXT _main<>(SB),NOSPLIT|NOFRAME,$0 - // In a statically linked binary, the stack contains argc, - // argv as argc string pointers followed by a NULL, envv as a - // sequence of string pointers followed by a NULL, and auxv. - // There is no TLS base pointer. -#ifdef GOARCH_mips64 - MOVW 4(R29), R4 // argc, big-endian ABI places int32 at offset 4 -#else - MOVW 0(R29), R4 // argc -#endif - ADDV $8, R29, R5 // argv - JMP main(SB) - -TEXT main(SB),NOSPLIT|NOFRAME,$0 - // in external linking, glibc jumps to main with argc in R4 - // and argv in R5 - - // initialize REGSB = PC&0xffffffff00000000 - BGEZAL R0, 1(PC) - SRLV $32, R31, RSB - SLLV $32, RSB - - MOVV $runtime·rt0_go(SB), R1 - JMP (R1) diff --git a/src/runtime/rt0_wasip1_wasm.s b/src/runtime/rt0_wasip1_wasm.s index 6dc239306b6497..a60566fe066384 100644 --- a/src/runtime/rt0_wasip1_wasm.s +++ b/src/runtime/rt0_wasip1_wasm.s @@ -14,3 +14,7 @@ TEXT _rt0_wasm_wasip1(SB),NOSPLIT,$0 Call wasm_pc_f_loop(SB) Return + +TEXT _rt0_wasm_wasip1_lib(SB),NOSPLIT,$0 + Call _rt0_wasm_wasip1(SB) + Return diff --git a/src/runtime/rt0_windows_386.s b/src/runtime/rt0_windows_386.s index fa39edd787c3d5..bb92a99f6bac5b 100644 --- a/src/runtime/rt0_windows_386.s +++ b/src/runtime/rt0_windows_386.s @@ -12,33 +12,8 @@ TEXT _rt0_386_windows(SB),NOSPLIT,$0 // library is loaded. For static libraries it is called when the // final executable starts, during the C runtime initialization // phase. -TEXT _rt0_386_windows_lib(SB),NOSPLIT,$0x1C - MOVL BP, 0x08(SP) - MOVL BX, 0x0C(SP) - MOVL AX, 0x10(SP) - MOVL CX, 0x14(SP) - MOVL DX, 0x18(SP) - - // Create a new thread to do the runtime initialization and return. - MOVL _cgo_sys_thread_create(SB), AX - MOVL $_rt0_386_windows_lib_go(SB), 0x00(SP) - MOVL $0, 0x04(SP) - - // Top two items on the stack are passed to _cgo_sys_thread_create - // as parameters. This is the calling convention on 32-bit Windows. - CALL AX - - MOVL 0x08(SP), BP - MOVL 0x0C(SP), BX - MOVL 0x10(SP), AX - MOVL 0x14(SP), CX - MOVL 0x18(SP), DX - RET - -TEXT _rt0_386_windows_lib_go(SB),NOSPLIT,$0 - PUSHL $0 - PUSHL $0 - JMP runtime·rt0_go(SB) +TEXT _rt0_386_windows_lib(SB),NOSPLIT,$0 + JMP _rt0_386_lib(SB) TEXT _main(SB),NOSPLIT,$0 // Remove the return address from the stack. diff --git a/src/runtime/rt0_windows_amd64.s b/src/runtime/rt0_windows_amd64.s index bd18bdd311e6dd..09869c534cb9c4 100644 --- a/src/runtime/rt0_windows_amd64.s +++ b/src/runtime/rt0_windows_amd64.s @@ -6,31 +6,10 @@ #include "go_tls.h" #include "textflag.h" -TEXT _rt0_amd64_windows(SB),NOSPLIT|NOFRAME,$-8 +TEXT _rt0_amd64_windows(SB),NOSPLIT,$0 JMP _rt0_amd64(SB) // When building with -buildmode=(c-shared or c-archive), this -// symbol is called. For dynamic libraries it is called when the -// library is loaded. For static libraries it is called when the -// final executable starts, during the C runtime initialization -// phase. -// Leave space for four pointers on the stack as required -// by the Windows amd64 calling convention. -TEXT _rt0_amd64_windows_lib(SB),NOSPLIT|NOFRAME,$40 - // Create a new thread to do the runtime initialization and return. - MOVQ BX, 32(SP) // callee-saved, preserved across the CALL - MOVQ SP, BX - ANDQ $~15, SP // alignment as per Windows requirement - MOVQ _cgo_sys_thread_create(SB), AX - MOVQ $_rt0_amd64_windows_lib_go(SB), CX - MOVQ $0, DX - CALL AX - MOVQ BX, SP - MOVQ 32(SP), BX - RET - -TEXT _rt0_amd64_windows_lib_go(SB),NOSPLIT|NOFRAME,$0 - MOVQ $0, DI - MOVQ $0, SI - MOVQ $runtime·rt0_go(SB), AX - JMP AX +// symbol is called. +TEXT _rt0_amd64_windows_lib(SB),NOSPLIT,$0 + JMP _rt0_amd64_lib(SB) diff --git a/src/runtime/rt0_windows_arm.s b/src/runtime/rt0_windows_arm.s deleted file mode 100644 index c5787d0dee0034..00000000000000 --- a/src/runtime/rt0_windows_arm.s +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "go_asm.h" -#include "go_tls.h" -#include "textflag.h" - -// This is the entry point for the program from the -// kernel for an ordinary -buildmode=exe program. -TEXT _rt0_arm_windows(SB),NOSPLIT|NOFRAME,$0 - B ·rt0_go(SB) diff --git a/src/runtime/rt0_windows_arm64.s b/src/runtime/rt0_windows_arm64.s index 8802c2b82e0afd..dc68cff5f2798b 100644 --- a/src/runtime/rt0_windows_arm64.s +++ b/src/runtime/rt0_windows_arm64.s @@ -8,22 +8,15 @@ // This is the entry point for the program from the // kernel for an ordinary -buildmode=exe program. -TEXT _rt0_arm64_windows(SB),NOSPLIT|NOFRAME,$0 - B ·rt0_go(SB) +TEXT _rt0_arm64_windows(SB),NOSPLIT,$0 + // Windows doesn't use argc and argv, + // so there is no need to go through _rt0_arm64. + JMP runtime·rt0_go(SB) -TEXT _rt0_arm64_windows_lib(SB),NOSPLIT|NOFRAME,$0 - MOVD $_rt0_arm64_windows_lib_go(SB), R0 - MOVD $0, R1 - MOVD _cgo_sys_thread_create(SB), R2 - B (R2) - -TEXT _rt0_arm64_windows_lib_go(SB),NOSPLIT|NOFRAME,$0 +// When building with -buildmode=c-shared, this symbol is called when the shared +// library is loaded. +TEXT _rt0_arm64_windows_lib(SB),NOSPLIT,$0 + // We get the argc and argv parameters from Win32. MOVD $0, R0 MOVD $0, R1 - MOVD $runtime·rt0_go(SB), R2 - B (R2) - -TEXT main(SB),NOSPLIT|NOFRAME,$0 - MOVD $runtime·rt0_go(SB), R2 - B (R2) - + JMP _rt0_arm64_lib(SB) diff --git a/src/runtime/runtime-gdb.py b/src/runtime/runtime-gdb.py index 46f014fc76e50c..9d2446a1ebee3e 100644 --- a/src/runtime/runtime-gdb.py +++ b/src/runtime/runtime-gdb.py @@ -46,11 +46,13 @@ def read_runtime_const(varname, default): G_DEAD = read_runtime_const("'runtime._Gdead'", 6) G_ENQUEUE_UNUSED = read_runtime_const("'runtime._Genqueue_unused'", 7) G_COPYSTACK = read_runtime_const("'runtime._Gcopystack'", 8) +G_DEADEXTRA = read_runtime_const("'runtime._Gdeadextra'", 10) G_SCAN = read_runtime_const("'runtime._Gscan'", 0x1000) G_SCANRUNNABLE = G_SCAN+G_RUNNABLE G_SCANRUNNING = G_SCAN+G_RUNNING G_SCANSYSCALL = G_SCAN+G_SYSCALL G_SCANWAITING = G_SCAN+G_WAITING +G_SCANEXTRA = G_SCAN+G_DEADEXTRA sts = { G_IDLE: 'idle', @@ -62,11 +64,13 @@ def read_runtime_const(varname, default): G_DEAD: 'dead', G_ENQUEUE_UNUSED: 'enqueue', G_COPYSTACK: 'copystack', + G_DEADEXTRA: 'extra', G_SCAN: 'scan', G_SCANRUNNABLE: 'runnable+s', G_SCANRUNNING: 'running+s', G_SCANSYSCALL: 'syscall+s', G_SCANWAITING: 'waiting+s', + G_SCANEXTRA: 'extra+s', } @@ -160,37 +164,107 @@ def to_string(self): return str(self.val.type) def children(self): - MapBucketCount = 8 # see internal/abi.go:MapBucketCount - B = self.val['B'] - buckets = self.val['buckets'] - oldbuckets = self.val['oldbuckets'] - flags = self.val['flags'] - inttype = self.val['hash0'].type + MapGroupSlots = 8 # see internal/abi:MapGroupSlots + + cnt = 0 + # Yield keys and elements in group. + # group is a value of type *group[K,V] + def group_slots(group): + ctrl = group['ctrl'] + + for i in xrange(MapGroupSlots): + c = (ctrl >> (8*i)) & 0xff + if (c & 0x80) != 0: + # Empty or deleted + continue + + # Full + yield str(cnt), group['slots'][i]['key'] + yield str(cnt+1), group['slots'][i]['elem'] + + # The linker DWARF generation + # (cmd/link/internal/ld.(*dwctxt).synthesizemaptypes) records + # dirPtr as a **table[K,V], but it may actually be two different types: + # + # For "full size" maps (dirLen > 0), dirPtr is actually a pointer to + # variable length array *[dirLen]*table[K,V]. In other words, dirPtr + + # dirLen are a deconstructed slice []*table[K,V]. + # + # For "small" maps (dirLen <= 0), dirPtr is a pointer directly to a + # single group *group[K,V] containing the map slots. + # + # N.B. array() takes an _inclusive_ upper bound. + + # table[K,V] + table_type = self.val['dirPtr'].type.target().target() + + if self.val['dirLen'] <= 0: + # Small map + + # We need to find the group type we'll cast to. Since dirPtr isn't + # actually **table[K,V], we can't use the nice API of + # obj['field'].type, as that actually wants to dereference obj. + # Instead, search only via the type API. + ptr_group_type = None + for tf in table_type.fields(): + if tf.name != 'groups': + continue + groups_type = tf.type + for gf in groups_type.fields(): + if gf.name != 'data': + continue + # *group[K,V] + ptr_group_type = gf.type + + if ptr_group_type is None: + raise TypeError("unable to find table[K,V].groups.data") + + # group = (*group[K,V])(dirPtr) + group = self.val['dirPtr'].cast(ptr_group_type) + + yield from group_slots(group) + + return + + # Full size map. + + # *table[K,V] + ptr_table_type = table_type.pointer() + # [dirLen]*table[K,V] + array_ptr_table_type = ptr_table_type.array(self.val['dirLen']-1) + # *[dirLen]*table[K,V] + ptr_array_ptr_table_type = array_ptr_table_type.pointer() + # tables = (*[dirLen]*table[K,V])(dirPtr) + tables = self.val['dirPtr'].cast(ptr_array_ptr_table_type) + cnt = 0 - for bucket in xrange(2 ** int(B)): - bp = buckets + bucket - if oldbuckets: - oldbucket = bucket & (2 ** (B - 1) - 1) - oldbp = oldbuckets + oldbucket - oldb = oldbp.dereference() - if (oldb['overflow'].cast(inttype) & 1) == 0: # old bucket not evacuated yet - if bucket >= 2 ** (B - 1): - continue # already did old bucket - bp = oldbp - while bp: - b = bp.dereference() - for i in xrange(MapBucketCount): - if b['tophash'][i] != 0: - k = b['keys'][i] - v = b['values'][i] - if flags & 1: - k = k.dereference() - if flags & 2: - v = v.dereference() - yield str(cnt), k - yield str(cnt + 1), v - cnt += 2 - bp = b['overflow'] + for t in xrange(self.val['dirLen']): + table = tables[t] + table = table.dereference() + + groups = table['groups']['data'] + length = table['groups']['lengthMask'] + 1 + + # The linker DWARF generation + # (cmd/link/internal/ld.(*dwctxt).synthesizemaptypes) records + # groups.data as a *group[K,V], but it is actually a pointer to + # variable length array *[length]group[K,V]. + # + # N.B. array() takes an _inclusive_ upper bound. + + # group[K,V] + group_type = groups.type.target() + # [length]group[K,V] + array_group_type = group_type.array(length-1) + # *[length]group[K,V] + ptr_array_group_type = array_group_type.pointer() + # groups = (*[length]group[K,V])(groups.data) + groups = groups.cast(ptr_array_group_type) + groups = groups.dereference() + + for i in xrange(length): + group = groups[i] + yield from group_slots(group) class ChanTypePrinter: @@ -385,7 +459,7 @@ def ifacematcher(val): class GoLenFunc(gdb.Function): "Length of strings, slices, maps or channels" - how = ((StringTypePrinter, 'len'), (SliceTypePrinter, 'len'), (MapTypePrinter, 'count'), (ChanTypePrinter, 'qcount')) + how = ((StringTypePrinter, 'len'), (SliceTypePrinter, 'len'), (MapTypePrinter, 'used'), (ChanTypePrinter, 'qcount')) def __init__(self): gdb.Function.__init__(self, "len") @@ -394,6 +468,12 @@ def invoke(self, obj): typename = str(obj.type) for klass, fld in self.how: if klass.pattern.match(typename) or paramtypematch(obj.type, klass.pattern): + if klass == MapTypePrinter: + fields = [f.name for f in self.val.type.strip_typedefs().target().fields()] + if 'buckets' in fields: + # Old maps. + fld = 'count' + return obj[fld] @@ -448,7 +528,7 @@ def invoke(self, _arg, _from_tty): # args = gdb.string_to_argv(arg) vp = gdb.lookup_type('void').pointer() for ptr in SliceValue(gdb.parse_and_eval("'runtime.allgs'")): - if ptr['atomicstatus']['value'] == G_DEAD: + if ptr['atomicstatus']['value'] in [G_DEAD, G_DEADEXTRA]: continue s = ' ' if ptr['m']: @@ -473,7 +553,7 @@ def find_goroutine(goid): """ vp = gdb.lookup_type('void').pointer() for ptr in SliceValue(gdb.parse_and_eval("'runtime.allgs'")): - if ptr['atomicstatus']['value'] == G_DEAD: + if ptr['atomicstatus']['value'] in [G_DEAD, G_DEADEXTRA]: continue if ptr['goid'] == goid: break diff --git a/src/runtime/runtime-gdb_test.go b/src/runtime/runtime-gdb_test.go index 5defe2f615eaa4..47d125bd67ae5b 100644 --- a/src/runtime/runtime-gdb_test.go +++ b/src/runtime/runtime-gdb_test.go @@ -8,7 +8,6 @@ import ( "bytes" "flag" "fmt" - "internal/abi" "internal/testenv" "os" "os/exec" @@ -72,9 +71,14 @@ func checkGdbVersion(t *testing.T) { if err1 != nil || err2 != nil { t.Skipf("skipping: can't determine gdb version: %v, %v", err1, err2) } - if major < 7 || (major == 7 && minor < 7) { + // The Go toolchain now generates DWARF 5 by default, which needs + // a GDB version of 10 or above. + if major < 10 { t.Skipf("skipping: gdb version %d.%d too old", major, minor) } + if major < 12 || (major == 12 && minor < 1) { + t.Logf("gdb version <12.1 is known to crash due to a SIGWINCH received in non-interactive mode; if you see a crash, some test may be sending SIGWINCH to the whole process group. See go.dev/issue/58932.") + } t.Logf("gdb version %d.%d", major, minor) } @@ -111,26 +115,67 @@ func checkCleanBacktrace(t *testing.T, backtrace string) { // TODO(mundaym): check for unknown frames (e.g. "??"). } -// NOTE: the maps below are allocated larger than abi.MapBucketCount -// to ensure that they are not "optimized out". +// checkPtraceScope checks the value of the kernel parameter ptrace_scope, +// skips the test when gdb cannot attach to the target process via ptrace. +// See issue 69932 +// +// 0 - Default attach security permissions. +// 1 - Restricted attach. Only child processes plus normal permissions. +// 2 - Admin-only attach. Only executables with CAP_SYS_PTRACE. +// 3 - No attach. No process may call ptrace at all. Irrevocable. +func checkPtraceScope(t *testing.T) { + if runtime.GOOS != "linux" { + return + } + + // If the Linux kernel does not have the YAMA module enabled, + // there will be no ptrace_scope file, which does not affect the tests. + path := "/proc/sys/kernel/yama/ptrace_scope" + if _, err := os.Stat(path); os.IsNotExist(err) { + return + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read file: %v", err) + } + value, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil { + t.Fatalf("failed converting value to int: %v", err) + } + switch value { + case 3: + t.Skip("skipping ptrace: Operation not permitted") + case 2: + if os.Geteuid() != 0 { + t.Skip("skipping ptrace: Operation not permitted with non-root user") + } + } +} var helloSource = ` import "fmt" import "runtime" var gslice []string +// TODO(prattmic): Stack allocated maps initialized inline appear "optimized out" in GDB. +var smallmapvar map[string]string func main() { - mapvar := make(map[string]string, ` + strconv.FormatInt(abi.MapBucketCount+9, 10) + `) - slicemap := make(map[string][]string,` + strconv.FormatInt(abi.MapBucketCount+3, 10) + `) - chanint := make(chan int, 10) - chanstr := make(chan string, 10) - chanint <- 99 + smallmapvar = make(map[string]string) + // NOTE: the maps below are allocated large to ensure that they are not + // "optimized out". + mapvar := make(map[string]string, 10) + slicemap := make(map[string][]string, 10) + chanint := make(chan int, 10) + chanstr := make(chan string, 10) + chanint <- 99 chanint <- 11 - chanstr <- "spongepants" - chanstr <- "squarebob" + chanstr <- "spongepants" + chanstr <- "squarebob" + smallmapvar["abc"] = "def" mapvar["abc"] = "def" mapvar["ghi"] = "jkl" slicemap["a"] = []string{"b","c","d"} - slicemap["e"] = []string{"f","g","h"} + slicemap["e"] = []string{"f","g","h"} strvar := "abc" ptrvar := &strvar slicevar := make([]string, 0, 16) @@ -140,6 +185,7 @@ func main() { _ = ptrvar // set breakpoint here gslice = slicevar fmt.Printf("%v, %v, %v\n", slicemap, <-chanint, <-chanstr) + runtime.KeepAlive(smallmapvar) runtime.KeepAlive(mapvar) } // END_OF_PROGRAM ` @@ -193,6 +239,7 @@ func testGdbPython(t *testing.T, cgo bool) { t.Parallel() checkGdbVersion(t) checkGdbPython(t) + checkPtraceScope(t) dir := t.TempDir() @@ -254,6 +301,9 @@ func testGdbPython(t *testing.T, cgo bool) { "-ex", "echo BEGIN info goroutines\n", "-ex", "info goroutines", "-ex", "echo END\n", + "-ex", "echo BEGIN print smallmapvar\n", + "-ex", "print smallmapvar", + "-ex", "echo END\n", "-ex", "echo BEGIN print mapvar\n", "-ex", "print mapvar", "-ex", "echo END\n", @@ -306,6 +356,11 @@ func testGdbPython(t *testing.T, cgo bool) { t.Fatalf("info goroutines failed: %s", bl) } + printSmallMapvarRe := regexp.MustCompile(`^\$[0-9]+ = map\[string\]string = {\[(0x[0-9a-f]+\s+)?"abc"\] = (0x[0-9a-f]+\s+)?"def"}$`) + if bl := blocks["print smallmapvar"]; !printSmallMapvarRe.MatchString(bl) { + t.Fatalf("print smallmapvar failed: %s", bl) + } + printMapvarRe1 := regexp.MustCompile(`^\$[0-9]+ = map\[string\]string = {\[(0x[0-9a-f]+\s+)?"abc"\] = (0x[0-9a-f]+\s+)?"def", \[(0x[0-9a-f]+\s+)?"ghi"\] = (0x[0-9a-f]+\s+)?"jkl"}$`) printMapvarRe2 := regexp.MustCompile(`^\$[0-9]+ = map\[string\]string = {\[(0x[0-9a-f]+\s+)?"ghi"\] = (0x[0-9a-f]+\s+)?"jkl", \[(0x[0-9a-f]+\s+)?"abc"\] = (0x[0-9a-f]+\s+)?"def"}$`) if bl := blocks["print mapvar"]; !printMapvarRe1.MatchString(bl) && @@ -416,6 +471,7 @@ func TestGdbBacktrace(t *testing.T) { checkGdbEnvironment(t) t.Parallel() checkGdbVersion(t) + checkPtraceScope(t) dir := t.TempDir() @@ -472,11 +528,12 @@ func TestGdbBacktrace(t *testing.T) { got, err := cmd.CombinedOutput() t.Logf("gdb output:\n%s", got) if err != nil { + noProcessRE := regexp.MustCompile(`Couldn't get [a-zA-Z_ -]* ?registers: No such process\.`) switch { case bytes.Contains(got, []byte("internal-error: wait returned unexpected status 0x0")): // GDB bug: https://sourceware.org/bugzilla/show_bug.cgi?id=28551 testenv.SkipFlaky(t, 43068) - case bytes.Contains(got, []byte("Couldn't get registers: No such process.")), + case noProcessRE.Match(got), bytes.Contains(got, []byte("Unable to fetch general registers.: No such process.")), bytes.Contains(got, []byte("reading register pc (#64): No such process.")): // GDB bug: https://sourceware.org/bugzilla/show_bug.cgi?id=9086 @@ -530,6 +587,7 @@ func TestGdbAutotmpTypes(t *testing.T) { checkGdbEnvironment(t) t.Parallel() checkGdbVersion(t) + checkPtraceScope(t) if runtime.GOOS == "aix" && testing.Short() { t.Skip("TestGdbAutotmpTypes is too slow on aix/ppc64") @@ -575,15 +633,16 @@ func TestGdbAutotmpTypes(t *testing.T) { // Check that the backtrace matches the source code. types := []string{ - "[]main.astruct;", - "bucket;", - "hash;", - "main.astruct;", - "hash * map[string]main.astruct;", + "[]main.astruct", + "main.astruct", + "groupReference", + "table", + "map", + "map * map[string]main.astruct", } for _, name := range types { if !strings.Contains(sgot, name) { - t.Fatalf("could not find %s in 'info typrs astruct' output", name) + t.Fatalf("could not find %q in 'info typrs astruct' output", name) } } } @@ -604,6 +663,7 @@ func TestGdbConst(t *testing.T) { checkGdbEnvironment(t) t.Parallel() checkGdbVersion(t) + checkPtraceScope(t) dir := t.TempDir() @@ -668,6 +728,7 @@ func TestGdbPanic(t *testing.T) { checkGdbEnvironment(t) t.Parallel() checkGdbVersion(t) + checkPtraceScope(t) if runtime.GOOS == "windows" { t.Skip("no signals on windows") @@ -747,6 +808,7 @@ func TestGdbInfCallstack(t *testing.T) { t.Parallel() checkGdbVersion(t) + checkPtraceScope(t) dir := t.TempDir() diff --git a/src/runtime/runtime-seh_windows_test.go b/src/runtime/runtime-seh_windows_test.go index 42509532be547d..e1cd0601bbd3f2 100644 --- a/src/runtime/runtime-seh_windows_test.go +++ b/src/runtime/runtime-seh_windows_test.go @@ -6,6 +6,7 @@ package runtime_test import ( "internal/abi" + "internal/runtime/sys" "internal/syscall/windows" "runtime" "slices" @@ -47,40 +48,49 @@ func TestSehLookupFunctionEntry(t *testing.T) { {"func in prologue", sehf1pc + 1, true}, {"anonymous func with frame", abi.FuncPCABIInternal(fnwithframe), true}, {"anonymous func without frame", abi.FuncPCABIInternal(fnwithoutframe), false}, - {"pc at func body", runtime.NewContextStub().GetPC(), true}, + {"pc at func body", sys.GetCallerPC(), true}, } for _, tt := range tests { var base uintptr fn := windows.RtlLookupFunctionEntry(tt.pc, &base, nil) if !tt.hasframe { - if fn != 0 { + if fn != nil { t.Errorf("%s: unexpected frame", tt.name) } continue } - if fn == 0 { + if fn == nil { t.Errorf("%s: missing frame", tt.name) } } } +//go:noinline +func newCtx() *windows.Context { + var ctx windows.Context + ctx.SetPC(sys.GetCallerPC()) + ctx.SetSP(sys.GetCallerSP()) + ctx.SetFP(runtime.GetCallerFp()) + return &ctx +} + func sehCallers() []uintptr { // We don't need a real context, // RtlVirtualUnwind just needs a context with // valid a pc, sp and fp (aka bp). - ctx := runtime.NewContextStub() + ctx := newCtx() pcs := make([]uintptr, 15) var base, frame uintptr var n int for i := 0; i < len(pcs); i++ { - fn := windows.RtlLookupFunctionEntry(ctx.GetPC(), &base, nil) - if fn == 0 { + fn := windows.RtlLookupFunctionEntry(ctx.PC(), &base, nil) + if fn == nil { break } - pcs[i] = ctx.GetPC() + pcs[i] = ctx.PC() n++ - windows.RtlVirtualUnwind(0, base, ctx.GetPC(), fn, uintptr(unsafe.Pointer(ctx)), nil, &frame, nil) + windows.RtlVirtualUnwind(0, base, ctx.PC(), fn, unsafe.Pointer(ctx), nil, &frame, nil) } return pcs[:n] } @@ -116,6 +126,9 @@ func testSehCallersEqual(t *testing.T, pcs []uintptr, want []string) { // These functions are skipped as they appear inconsistently depending // whether inlining is on or off. continue + case "runtime_test.sehCallers": + // This is an artifact of the implementation of sehCallers. + continue } got = append(got, name) } @@ -129,15 +142,14 @@ func TestSehUnwind(t *testing.T) { t.Skip("skipping amd64-only test") } pcs := sehf3(false) - testSehCallersEqual(t, pcs, []string{"runtime_test.sehCallers", "runtime_test.sehf4", - "runtime_test.sehf3", "runtime_test.TestSehUnwind"}) + testSehCallersEqual(t, pcs, []string{"runtime_test.sehf4", "runtime_test.sehf3", "runtime_test.TestSehUnwind"}) } func TestSehUnwindPanic(t *testing.T) { if runtime.GOARCH != "amd64" { t.Skip("skipping amd64-only test") } - want := []string{"runtime_test.sehCallers", "runtime_test.TestSehUnwindPanic.func1", "runtime.gopanic", + want := []string{"runtime_test.TestSehUnwindPanic.func1", "runtime.gopanic", "runtime_test.sehf4", "runtime_test.sehf3", "runtime_test.TestSehUnwindPanic"} defer func() { if r := recover(); r == nil { @@ -153,7 +165,7 @@ func TestSehUnwindDoublePanic(t *testing.T) { if runtime.GOARCH != "amd64" { t.Skip("skipping amd64-only test") } - want := []string{"runtime_test.sehCallers", "runtime_test.TestSehUnwindDoublePanic.func1.1", "runtime.gopanic", + want := []string{"runtime_test.TestSehUnwindDoublePanic.func1.1", "runtime.gopanic", "runtime_test.TestSehUnwindDoublePanic.func1", "runtime.gopanic", "runtime_test.TestSehUnwindDoublePanic"} defer func() { defer func() { @@ -175,7 +187,7 @@ func TestSehUnwindNilPointerPanic(t *testing.T) { if runtime.GOARCH != "amd64" { t.Skip("skipping amd64-only test") } - want := []string{"runtime_test.sehCallers", "runtime_test.TestSehUnwindNilPointerPanic.func1", "runtime.gopanic", + want := []string{"runtime_test.TestSehUnwindNilPointerPanic.func1", "runtime.gopanic", "runtime.sigpanic", "runtime_test.TestSehUnwindNilPointerPanic"} defer func() { if r := recover(); r == nil { diff --git a/src/runtime/runtime.go b/src/runtime/runtime.go index e8e614815d236b..016cbdae58ccb6 100644 --- a/src/runtime/runtime.go +++ b/src/runtime/runtime.go @@ -151,6 +151,7 @@ func godebug_setNewIncNonDefault(newIncNonDefault func(string) func()) { p := new(func(string) func()) *p = newIncNonDefault godebugNewIncNonDefault.Store(p) + defaultGOMAXPROCSUpdateGODEBUG() } // A godebugInc provides access to internal/godebug's IncNonDefault function @@ -290,23 +291,15 @@ func setCrashFD(fd uintptr) uintptr { } // auxv is populated on relevant platforms but defined here for all platforms -// so x/sys/cpu can assume the getAuxv symbol exists without keeping its list -// of auxv-using GOOS build tags in sync. +// so x/sys/cpu and x/sys/unix can assume the getAuxv symbol exists without +// keeping its list of auxv-using GOOS build tags in sync. // // It contains an even number of elements, (tag, value) pairs. var auxv []uintptr -// golang.org/x/sys/cpu uses getAuxv via linkname. +// golang.org/x/sys/cpu and golang.org/x/sys/unix use getAuxv via linkname. // Do not remove or change the type signature. -// (See go.dev/issue/57336.) -// -// getAuxv should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cilium/ebpf -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. +// See go.dev/issue/57336 and go.dev/issue/67401. // //go:linkname getAuxv func getAuxv() []uintptr { return auxv } diff --git a/src/runtime/runtime1.go b/src/runtime/runtime1.go index 03ef74b8dc4b54..15b546783b5e53 100644 --- a/src/runtime/runtime1.go +++ b/src/runtime/runtime1.go @@ -8,6 +8,7 @@ import ( "internal/bytealg" "internal/goarch" "internal/runtime/atomic" + "internal/runtime/strconv" "unsafe" ) @@ -309,6 +310,8 @@ type dbgVar struct { var debug struct { cgocheck int32 clobberfree int32 + containermaxprocs int32 + decoratemappings int32 disablethp int32 dontfreezetheworld int32 efence int32 @@ -319,11 +322,11 @@ var debug struct { gctrace int32 invalidptr int32 madvdontneed int32 // for Linux; issue 28466 - runtimeContentionStacks atomic.Int32 scavtrace int32 scheddetail int32 schedtrace int32 tracebackancestors int32 + updatemaxprocs int32 asyncpreemptoff int32 harddecommit int32 adaptivestackstart int32 @@ -331,13 +334,15 @@ var debug struct { traceadvanceperiod int32 traceCheckStackOwnership int32 profstackdepth int32 + dataindependenttiming int32 // debug.malloc is used as a combined debug check // in the malloc function and should be set // if any of the below debug options is != 0. - malloc bool - inittrace int32 - sbrk int32 + malloc bool + inittrace int32 + sbrk int32 + checkfinalizers int32 // traceallocfree controls whether execution traces contain // detailed trace data about memory allocation. This value // affects debug.malloc only if it is != 0 and the execution @@ -367,8 +372,12 @@ var dbgvars = []*dbgVar{ {name: "asynctimerchan", atomic: &debug.asynctimerchan}, {name: "cgocheck", value: &debug.cgocheck}, {name: "clobberfree", value: &debug.clobberfree}, + {name: "containermaxprocs", value: &debug.containermaxprocs, def: 1}, + {name: "dataindependenttiming", value: &debug.dataindependenttiming}, + {name: "decoratemappings", value: &debug.decoratemappings, def: 1}, {name: "disablethp", value: &debug.disablethp}, {name: "dontfreezetheworld", value: &debug.dontfreezetheworld}, + {name: "checkfinalizers", value: &debug.checkfinalizers}, {name: "efence", value: &debug.efence}, {name: "gccheckmark", value: &debug.gccheckmark}, {name: "gcpacertrace", value: &debug.gcpacertrace}, @@ -381,7 +390,6 @@ var dbgvars = []*dbgVar{ {name: "madvdontneed", value: &debug.madvdontneed}, {name: "panicnil", atomic: &debug.panicnil}, {name: "profstackdepth", value: &debug.profstackdepth, def: 128}, - {name: "runtimecontentionstacks", atomic: &debug.runtimeContentionStacks}, {name: "sbrk", value: &debug.sbrk}, {name: "scavtrace", value: &debug.scavtrace}, {name: "scheddetail", value: &debug.scheddetail}, @@ -391,9 +399,10 @@ var dbgvars = []*dbgVar{ {name: "tracecheckstackownership", value: &debug.traceCheckStackOwnership}, {name: "tracebackancestors", value: &debug.tracebackancestors}, {name: "tracefpunwindoff", value: &debug.tracefpunwindoff}, + {name: "updatemaxprocs", value: &debug.updatemaxprocs, def: 1}, } -func parsedebugvars() { +func parseRuntimeDebugVars(godebug string) { // defaults debug.cgocheck = 1 debug.invalidptr = 1 @@ -411,12 +420,6 @@ func parsedebugvars() { } debug.traceadvanceperiod = defaultTraceAdvancePeriod - godebug := gogetenv("GODEBUG") - - p := new(string) - *p = godebug - godebugEnv.Store(p) - // apply runtime defaults, if any for _, v := range dbgvars { if v.def != 0 { @@ -428,16 +431,38 @@ func parsedebugvars() { } } } - // apply compile-time GODEBUG settings parsegodebug(godebugDefault, nil) // apply environment settings parsegodebug(godebug, nil) - debug.malloc = (debug.inittrace | debug.sbrk) != 0 + debug.malloc = (debug.inittrace | debug.sbrk | debug.checkfinalizers) != 0 debug.profstackdepth = min(debug.profstackdepth, maxProfStackDepth) + // Disable async preemption in checkmark mode. The following situation is + // problematic with checkmark mode: + // + // - The GC doesn't mark object A because it is truly dead. + // - The GC stops the world, asynchronously preempting G1 which has a reference + // to A in its top stack frame + // - During the stop the world, we run the second checkmark GC. It marks the roots + // and discovers A through G1. + // - Checkmark mode reports a failure since there's a discrepancy in mark metadata. + // + // We could disable just conservative scanning during the checkmark scan, which is + // safe but makes checkmark slightly less powerful, but that's a lot more invasive + // than just disabling async preemption altogether. + if debug.gccheckmark > 0 { + debug.asyncpreemptoff = 1 + } +} + +func finishDebugVarsSetup() { + p := new(string) + *p = gogetenv("GODEBUG") + godebugEnv.Store(p) + setTraceback(gogetenv("GOTRACEBACK")) traceback_env = traceback_cache } @@ -507,13 +532,13 @@ func parsegodebug(godebug string, seen map[string]bool) { // is int, not int32, and should only be updated // if specified in GODEBUG. if seen == nil && key == "memprofilerate" { - if n, ok := atoi(value); ok { + if n, ok := strconv.Atoi(value); ok { MemProfileRate = n } } else { for _, v := range dbgvars { if v.name == key { - if n, ok := atoi32(value); ok { + if n, ok := strconv.Atoi32(value); ok { if seen == nil && v.value != nil { *v.value = n } else if v.atomic != nil { @@ -553,7 +578,7 @@ func setTraceback(level string) { fallthrough default: t = tracebackAll - if n, ok := atoi(level); ok && n == int(uint32(n)) { + if n, ok := strconv.Atoi(level); ok && n == int(uint32(n)) { t |= uint32(n) << tracebackShift } } @@ -680,7 +705,6 @@ func reflect_resolveTypeOff(rtype unsafe.Pointer, off int32) unsafe.Pointer { // reflect_resolveTextOff is for package reflect, // but widely used packages access it using linkname. // Notable members of the hall of shame include: -// - github.com/cloudwego/frugal // - github.com/agiledragon/gomonkey/v2 // // Do not remove or change the type signature. @@ -725,3 +749,13 @@ func reflect_addReflectOff(ptr unsafe.Pointer) int32 { reflectOffsUnlock() return id } + +//go:linkname fips_getIndicator crypto/internal/fips140.getIndicator +func fips_getIndicator() uint8 { + return getg().fipsIndicator +} + +//go:linkname fips_setIndicator crypto/internal/fips140.setIndicator +func fips_setIndicator(indicator uint8) { + getg().fipsIndicator = indicator +} diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 62ed77aae56fed..eaf24fe908c5e4 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -87,6 +87,13 @@ const ( // ready()ing this G. _Gpreempted // 9 + // _Gleaked represents a leaked goroutine caught by the GC. + _Gleaked // 10 + + // _Gdeadextra is a _Gdead goroutine that's attached to an extra M + // used for cgo callbacks. + _Gdeadextra // 11 + // _Gscan combined with one of the above states other than // _Grunning indicates that GC is scanning the stack. The // goroutine is not executing user code and the stack is owned @@ -104,6 +111,8 @@ const ( _Gscansyscall = _Gscan + _Gsyscall // 0x1003 _Gscanwaiting = _Gscan + _Gwaiting // 0x1004 _Gscanpreempted = _Gscan + _Gpreempted // 0x1009 + _Gscanleaked = _Gscan + _Gleaked // 0x100a + _Gscandeadextra = _Gscan + _Gdeadextra // 0x100b ) const ( @@ -170,33 +179,6 @@ type mutex struct { key uintptr } -// sleep and wakeup on one-time events. -// before any calls to notesleep or notewakeup, -// must call noteclear to initialize the Note. -// then, exactly one thread can call notesleep -// and exactly one thread can call notewakeup (once). -// once notewakeup has been called, the notesleep -// will return. future notesleep will return immediately. -// subsequent noteclear must be called only after -// previous notesleep has returned, e.g. it's disallowed -// to call noteclear straight after notewakeup. -// -// notetsleep is like notesleep but wakes up after -// a given number of nanoseconds even if the event -// has not yet happened. if a goroutine uses notetsleep to -// wake up early, it must wait to call noteclear until it -// can be sure that no other goroutine is calling -// notewakeup. -// -// notesleep/notetsleep are generally called on g0, -// notetsleepg is similar to notetsleep but is called on user g. -type note struct { - // Futex-based impl treats it as uint32 key, - // while sema-based impl as M* waitm. - // Used to be a union, but unions break precise GC. - key uintptr -} - type funcval struct { fn uintptr // variable-size, fn-specific data here @@ -338,11 +320,82 @@ type gobuf struct { pc uintptr g guintptr ctxt unsafe.Pointer - ret uintptr lr uintptr bp uintptr // for framepointer-enabled architectures } +// maybeTraceablePtr is a special pointer that is conditionally trackable +// by the GC. It consists of an address as a uintptr (vu) and a pointer +// to a data element (vp). +// +// maybeTraceablePtr values can be in one of three states: +// 1. Unset: vu == 0 && vp == nil +// 2. Untracked: vu != 0 && vp == nil +// 3. Tracked: vu != 0 && vp != nil +// +// Do not set fields manually. Use methods instead. +// Extend this type with additional methods if needed. +type maybeTraceablePtr struct { + vp unsafe.Pointer // For liveness only. + vu uintptr // Source of truth. +} + +// untrack unsets the pointer but preserves the address. +// This is used to hide the pointer from the GC. +// +//go:nosplit +func (p *maybeTraceablePtr) setUntraceable() { + p.vp = nil +} + +// setTraceable resets the pointer to the stored address. +// This is used to make the pointer visible to the GC. +// +//go:nosplit +func (p *maybeTraceablePtr) setTraceable() { + p.vp = unsafe.Pointer(p.vu) +} + +// set sets the pointer to the data element and updates the address. +// +//go:nosplit +func (p *maybeTraceablePtr) set(v unsafe.Pointer) { + p.vp = v + p.vu = uintptr(v) +} + +// get retrieves the pointer to the data element. +// +//go:nosplit +func (p *maybeTraceablePtr) get() unsafe.Pointer { + return unsafe.Pointer(p.vu) +} + +// uintptr returns the uintptr address of the pointer. +// +//go:nosplit +func (p *maybeTraceablePtr) uintptr() uintptr { + return p.vu +} + +// maybeTraceableChan extends conditionally trackable pointers (maybeTraceablePtr) +// to track hchan pointers. +// +// Do not set fields manually. Use methods instead. +type maybeTraceableChan struct { + maybeTraceablePtr +} + +//go:nosplit +func (p *maybeTraceableChan) set(c *hchan) { + p.maybeTraceablePtr.set(unsafe.Pointer(c)) +} + +//go:nosplit +func (p *maybeTraceableChan) get() *hchan { + return (*hchan)(p.maybeTraceablePtr.get()) +} + // sudog (pseudo-g) represents a g in a wait list, such as for sending/receiving // on a channel. // @@ -362,7 +415,8 @@ type sudog struct { next *sudog prev *sudog - elem unsafe.Pointer // data element (may point to stack) + + elem maybeTraceablePtr // data element (may point to stack) // The following fields are never accessed concurrently. // For channels, waitlink is only accessed by g. @@ -390,10 +444,10 @@ type sudog struct { // in the second entry in the list.) waiters uint16 - parent *sudog // semaRoot binary tree - waitlink *sudog // g.waiting list or semaRoot - waittail *sudog // semaRoot - c *hchan // channel + parent *sudog // semaRoot binary tree + waitlink *sudog // g.waiting list or semaRoot + waittail *sudog // semaRoot + c maybeTraceableChan // channel } type libcall struct { @@ -486,35 +540,43 @@ type g struct { inMarkAssist bool coroexit bool // argument to coroswitch_m - raceignore int8 // ignore race detection events - nocgocallback bool // whether disable callback from C - tracking bool // whether we're tracking this G for sched latency statistics - trackingSeq uint8 // used to decide whether to track this G - trackingStamp int64 // timestamp of when the G last started being tracked - runnableTime int64 // the amount of time spent runnable, cleared when running, only used when tracking - lockedm muintptr - sig uint32 - writebuf []byte - sigcode0 uintptr - sigcode1 uintptr - sigpc uintptr - parentGoid uint64 // goid of goroutine that created this goroutine - gopc uintptr // pc of go statement that created this goroutine - ancestors *[]ancestorInfo // ancestor information goroutine(s) that created this goroutine (only used if debug.tracebackancestors) - startpc uintptr // pc of goroutine function - racectx uintptr - waiting *sudog // sudog structures this g is waiting on (that have a valid elem ptr); in lock order - cgoCtxt []uintptr // cgo traceback context - labels unsafe.Pointer // profiler labels - timer *timer // cached timer for time.Sleep - sleepWhen int64 // when to sleep until - selectDone atomic.Uint32 // are we participating in a select and did someone win the race? + raceignore int8 // ignore race detection events + nocgocallback bool // whether disable callback from C + tracking bool // whether we're tracking this G for sched latency statistics + trackingSeq uint8 // used to decide whether to track this G + trackingStamp int64 // timestamp of when the G last started being tracked + runnableTime int64 // the amount of time spent runnable, cleared when running, only used when tracking + lockedm muintptr + fipsIndicator uint8 + syncSafePoint bool // set if g is stopped at a synchronous safe point. + runningCleanups atomic.Bool + sig uint32 + writebuf []byte + sigcode0 uintptr + sigcode1 uintptr + sigpc uintptr + parentGoid uint64 // goid of goroutine that created this goroutine + gopc uintptr // pc of go statement that created this goroutine + ancestors *[]ancestorInfo // ancestor information goroutine(s) that created this goroutine (only used if debug.tracebackancestors) + startpc uintptr // pc of goroutine function + racectx uintptr + waiting *sudog // sudog structures this g is waiting on (that have a valid elem ptr); in lock order + cgoCtxt []uintptr // cgo traceback context + labels unsafe.Pointer // profiler labels + timer *timer // cached timer for time.Sleep + sleepWhen int64 // when to sleep until + selectDone atomic.Uint32 // are we participating in a select and did someone win the race? // goroutineProfiled indicates the status of this goroutine's stack for the // current in-progress goroutine profile goroutineProfiled goroutineProfileStateHolder coroarg *coro // argument during coroutine transfers + bubble *synctestBubble + + // xRegs stores the extended register state if this G has been + // asynchronously preempted. + xRegs xRegPerG // Per-G tracer state. trace gTraceState @@ -529,6 +591,10 @@ type g struct { // and check for debt in the malloc hot path. The assist ratio // determines how this corresponds to scan work debt. gcAssistBytes int64 + + // valgrindStackID is used to track what memory is used for stacks when a program is + // built with the "valgrind" build tag, otherwise it is unused. + valgrindStackID uintptr } // gTrackingPeriod is the number of transitions out of _Grunning between @@ -552,51 +618,52 @@ const ( type m struct { g0 *g // goroutine with scheduling stack morebuf gobuf // gobuf arg to morestack - divmod uint32 // div/mod denominator for arm - known to liblink - _ uint32 // align next field to 8 bytes + divmod uint32 // div/mod denominator for arm - known to liblink (cmd/internal/obj/arm/obj5.go) // Fields not known to debuggers. - procid uint64 // for debuggers, but offset not hard-coded - gsignal *g // signal-handling g - goSigStack gsignalStack // Go-allocated signal handling stack - sigmask sigset // storage for saved signal mask - tls [tlsSlots]uintptr // thread-local storage (for x86 extern register) - mstartfn func() - curg *g // current running goroutine - caughtsig guintptr // goroutine running during fatal signal - p puintptr // attached p for executing go code (nil if not executing go code) - nextp puintptr - oldp puintptr // the p that was attached before executing a syscall - id int64 - mallocing int32 - throwing throwType - preemptoff string // if != "", keep curg running on this m - locks int32 - dying int32 - profilehz int32 - spinning bool // m is out of work and is actively looking for work - blocked bool // m is blocked on a note - newSigstack bool // minit on C thread called sigaltstack - printlock int8 - incgo bool // m is executing a cgo call - isextra bool // m is an extra m - isExtraInC bool // m is an extra m that is not executing Go code - isExtraInSig bool // m is an extra m in a signal handler - freeWait atomic.Uint32 // Whether it is safe to free g0 and delete m (one of freeMRef, freeMStack, freeMWait) - needextram bool - traceback uint8 - ncgocall uint64 // number of cgo calls in total - ncgo int32 // number of cgo calls currently in progress - cgoCallersUse atomic.Uint32 // if non-zero, cgoCallers in use temporarily - cgoCallers *cgoCallers // cgo traceback if crashing in cgo call - park note - alllink *m // on allm - schedlink muintptr - lockedg guintptr - createstack [32]uintptr // stack that created this thread, it's used for StackRecord.Stack0, so it must align with it. - lockedExt uint32 // tracking for external LockOSThread - lockedInt uint32 // tracking for internal lockOSThread - nextwaitm muintptr // next m waiting for lock + procid uint64 // for debuggers, but offset not hard-coded + gsignal *g // signal-handling g + goSigStack gsignalStack // Go-allocated signal handling stack + sigmask sigset // storage for saved signal mask + tls [tlsSlots]uintptr // thread-local storage (for x86 extern register) + mstartfn func() + curg *g // current running goroutine + caughtsig guintptr // goroutine running during fatal signal + p puintptr // attached p for executing go code (nil if not executing go code) + nextp puintptr + oldp puintptr // the p that was attached before executing a syscall + id int64 + mallocing int32 + throwing throwType + preemptoff string // if != "", keep curg running on this m + locks int32 + dying int32 + profilehz int32 + spinning bool // m is out of work and is actively looking for work + blocked bool // m is blocked on a note + newSigstack bool // minit on C thread called sigaltstack + printlock int8 + incgo bool // m is executing a cgo call + isextra bool // m is an extra m + isExtraInC bool // m is an extra m that does not have any Go frames + isExtraInSig bool // m is an extra m in a signal handler + freeWait atomic.Uint32 // Whether it is safe to free g0 and delete m (one of freeMRef, freeMStack, freeMWait) + needextram bool + g0StackAccurate bool // whether the g0 stack has accurate bounds + traceback uint8 + allpSnapshot []*p // Snapshot of allp for use after dropping P in findRunnable, nil otherwise. + ncgocall uint64 // number of cgo calls in total + ncgo int32 // number of cgo calls currently in progress + cgoCallersUse atomic.Uint32 // if non-zero, cgoCallers in use temporarily + cgoCallers *cgoCallers // cgo traceback if crashing in cgo call + park note + alllink *m // on allm + schedlink muintptr + lockedg guintptr + createstack [32]uintptr // stack that created this thread, it's used for StackRecord.Stack0, so it must align with it. + lockedExt uint32 // tracking for external LockOSThread + lockedInt uint32 // tracking for internal lockOSThread + mWaitList mWaitList // list of runtime lock waiters mLockProfile mLockProfile // fields relating to runtime.lock contention profStack []uintptr // used for memory/block/mutex stack traces @@ -612,9 +679,7 @@ type m struct { freelink *m // on sched.freem trace mTraceState - // these are here because they are too large to be on the stack - // of low-level NOSPLIT functions. - libcall libcall + // These are here to avoid using the G stack so the stack can move during the call. libcallpc uintptr // for cpu profiler libcallsp uintptr libcallg guintptr @@ -646,6 +711,18 @@ type m struct { locksHeld [10]heldLockInfo } +const mRedZoneSize = (16 << 3) * asanenabledBit // redZoneSize(2048) + +type mPadded struct { + m + + // Size the runtime.m structure so it fits in the 2048-byte size class, and + // not in the next-smallest (1792-byte) size class. That leaves the 11 low + // bits of muintptr values available for flags, as required by + // lock_spinbit.go. + _ [(1 - goarch.IsWasm) * (2048 - mallocHeaderSize - mRedZoneSize - unsafe.Sizeof(m{}))]byte +} + type p struct { id int32 status uint32 // one of pidle/prunning/... @@ -684,10 +761,7 @@ type p struct { runnext guintptr // Available G's (status == Gdead) - gFree struct { - gList - n int32 - } + gFree gList sudogcache []*sudog sudogbuf [128]*sudog @@ -747,6 +821,10 @@ type p struct { // Timer heap. timers timers + // Cleanups. + cleanups *cleanupBlock + cleanupsQueued uint64 // monotonic count of cleanups queued by this P + // maxStackScanDelta accumulates the amount of stack space held by // live goroutines (i.e. those eligible for stack scanning). // Flushed to gcController.maxStackScan once maxStackScanSlack @@ -768,14 +846,23 @@ type p struct { // gcStopTime is the nanotime timestamp that this P last entered _Pgcstop. gcStopTime int64 + // goroutinesCreated is the total count of goroutines created by this P. + goroutinesCreated uint64 + + // xRegs is the per-P extended register state used by asynchronous + // preemption. This is an empty struct on platforms that don't use extended + // register state. + xRegs xRegPerP + // Padding is no longer needed. False sharing is now not a worry because p is large enough // that its size class is an integer multiple of the cache line size (for any of our architectures). } type schedt struct { - goidgen atomic.Uint64 - lastpoll atomic.Int64 // time of last network poll, 0 if currently polling - pollUntil atomic.Int64 // time to which current poll is sleeping + goidgen atomic.Uint64 + lastpoll atomic.Int64 // time of last network poll, 0 if currently polling + pollUntil atomic.Int64 // time to which current poll is sleeping + pollingNet atomic.Int32 // 1 if some P doing non-blocking network poll lock mutex @@ -790,7 +877,8 @@ type schedt struct { nmsys int32 // number of system m's not counted for deadlock nmfreed int64 // cumulative number of freed m's - ngsys atomic.Int32 // number of system goroutines + ngsys atomic.Int32 // number of system goroutines + nGsyscallNoP atomic.Int32 // number of goroutines in syscalls without a P pidle puintptr // idle p's npidle atomic.Int32 @@ -798,8 +886,7 @@ type schedt struct { needspinning atomic.Uint32 // See "Delicate dance" comment in proc.go. Boolean. Must hold sched.lock to set to 1. // Global runnable queue. - runq gQueue - runqsize int32 + runq gQueue // disable controls selective disabling of the scheduler. // @@ -810,7 +897,6 @@ type schedt struct { // user disables scheduling of user goroutines. user bool runnable gQueue // pending runnable Gs - n int32 // length of runnable } // Global cache of dead G's. @@ -818,7 +904,6 @@ type schedt struct { lock mutex stack gList // Gs with stacks noStack gList // Gs without stacks - n int32 } // Central cache of sudog structs. @@ -850,6 +935,8 @@ type schedt struct { procresizetime int64 // nanotime() of last change to gomaxprocs totaltime int64 // ∫gomaxprocs dt up to procresizetime + customGOMAXPROCS bool // GOMAXPROCS was manually set from the environment or runtime.GOMAXPROCS + // sysmonlock protects sysmon's actions on the runtime. // // Acquire and hold this mutex to block sysmon from interacting @@ -890,6 +977,10 @@ type schedt struct { // M, but waiting for locks within the runtime. This field stores the value // for Ms that have exited. totalRuntimeLockWaitTime atomic.Int64 + + // goroutinesCreated (plus the value of goroutinesCreated on each P in allp) + // is the sum of all goroutines created by the program. + goroutinesCreated atomic.Uint64 } // Values for the flags field of a sigTabT. @@ -1006,14 +1097,13 @@ type _defer struct { // // A _panic value must only ever live on the stack. // -// The argp and link fields are stack pointers, but don't need special +// The gopanicFP and link fields are stack pointers, but don't need special // handling during stack growth: because they are pointer-typed and // _panic values only live on the stack, regular stack pointer // adjustment takes care of them. type _panic struct { - argp unsafe.Pointer // pointer to arguments of deferred call run during panic; cannot move - known to liblink - arg any // argument to panic - link *_panic // link to earlier panic + arg any // argument to panic + link *_panic // link to earlier panic // startPC and startSP track where _panic.start was called. startPC uintptr @@ -1033,8 +1123,11 @@ type _panic struct { slotsPtr unsafe.Pointer recovered bool // whether this panic has been recovered + repanicked bool // whether this panic repanicked goexit bool deferreturn bool + + gopanicFP unsafe.Pointer // frame pointer of the gopanic frame } // savedOpenDeferState tracks the extra state from _panic that's @@ -1061,27 +1154,29 @@ const ( waitReasonZero waitReason = iota // "" waitReasonGCAssistMarking // "GC assist marking" waitReasonIOWait // "IO wait" - waitReasonChanReceiveNilChan // "chan receive (nil chan)" - waitReasonChanSendNilChan // "chan send (nil chan)" waitReasonDumpingHeap // "dumping heap" waitReasonGarbageCollection // "garbage collection" waitReasonGarbageCollectionScan // "garbage collection scan" waitReasonPanicWait // "panicwait" - waitReasonSelect // "select" - waitReasonSelectNoCases // "select (no cases)" waitReasonGCAssistWait // "GC assist wait" waitReasonGCSweepWait // "GC sweep wait" waitReasonGCScavengeWait // "GC scavenge wait" - waitReasonChanReceive // "chan receive" - waitReasonChanSend // "chan send" waitReasonFinalizerWait // "finalizer wait" waitReasonForceGCIdle // "force gc (idle)" + waitReasonUpdateGOMAXPROCSIdle // "GOMAXPROCS updater (idle)" waitReasonSemacquire // "semacquire" waitReasonSleep // "sleep" + waitReasonChanReceiveNilChan // "chan receive (nil chan)" + waitReasonChanSendNilChan // "chan send (nil chan)" + waitReasonSelectNoCases // "select (no cases)" + waitReasonSelect // "select" + waitReasonChanReceive // "chan receive" + waitReasonChanSend // "chan send" waitReasonSyncCondWait // "sync.Cond.Wait" waitReasonSyncMutexLock // "sync.Mutex.Lock" waitReasonSyncRWMutexRLock // "sync.RWMutex.RLock" waitReasonSyncRWMutexLock // "sync.RWMutex.Lock" + waitReasonSyncWaitGroupWait // "sync.WaitGroup.Wait" waitReasonTraceReaderBlocked // "trace reader (blocked)" waitReasonWaitForGCCycle // "wait for GC cycle" waitReasonGCWorkerIdle // "GC worker (idle)" @@ -1095,6 +1190,14 @@ const ( waitReasonTraceProcStatus // "trace proc status" waitReasonPageTraceFlush // "page trace flush" waitReasonCoroutine // "coroutine" + waitReasonGCWeakToStrongWait // "GC weak to strong wait" + waitReasonSynctestRun // "synctest.Run" + waitReasonSynctestWait // "synctest.Wait" + waitReasonSynctestChanReceive // "chan receive (durable)" + waitReasonSynctestChanSend // "chan send (durable)" + waitReasonSynctestSelect // "select (durable)" + waitReasonSynctestWaitGroupWait // "sync.WaitGroup.Wait (durable)" + waitReasonCleanupWait // "cleanup wait" ) var waitReasonStrings = [...]string{ @@ -1116,12 +1219,14 @@ var waitReasonStrings = [...]string{ waitReasonChanSend: "chan send", waitReasonFinalizerWait: "finalizer wait", waitReasonForceGCIdle: "force gc (idle)", + waitReasonUpdateGOMAXPROCSIdle: "GOMAXPROCS updater (idle)", waitReasonSemacquire: "semacquire", waitReasonSleep: "sleep", waitReasonSyncCondWait: "sync.Cond.Wait", waitReasonSyncMutexLock: "sync.Mutex.Lock", waitReasonSyncRWMutexRLock: "sync.RWMutex.RLock", waitReasonSyncRWMutexLock: "sync.RWMutex.Lock", + waitReasonSyncWaitGroupWait: "sync.WaitGroup.Wait", waitReasonTraceReaderBlocked: "trace reader (blocked)", waitReasonWaitForGCCycle: "wait for GC cycle", waitReasonGCWorkerIdle: "GC worker (idle)", @@ -1135,6 +1240,14 @@ var waitReasonStrings = [...]string{ waitReasonTraceProcStatus: "trace proc status", waitReasonPageTraceFlush: "page trace flush", waitReasonCoroutine: "coroutine", + waitReasonGCWeakToStrongWait: "GC weak to strong wait", + waitReasonSynctestRun: "synctest.Run", + waitReasonSynctestWait: "synctest.Wait", + waitReasonSynctestChanReceive: "chan receive (durable)", + waitReasonSynctestChanSend: "chan send (durable)", + waitReasonSynctestSelect: "select (durable)", + waitReasonSynctestWaitGroupWait: "sync.WaitGroup.Wait (durable)", + waitReasonCleanupWait: "cleanup wait", } func (w waitReason) String() string { @@ -1144,23 +1257,45 @@ func (w waitReason) String() string { return waitReasonStrings[w] } +// isMutexWait returns true if the goroutine is blocked because of +// sync.Mutex.Lock or sync.RWMutex.[R]Lock. +// +//go:nosplit func (w waitReason) isMutexWait() bool { return w == waitReasonSyncMutexLock || w == waitReasonSyncRWMutexRLock || w == waitReasonSyncRWMutexLock } -func (w waitReason) isWaitingForGC() bool { - return isWaitingForGC[w] +// isSyncWait returns true if the goroutine is blocked because of +// sync library primitive operations. +// +//go:nosplit +func (w waitReason) isSyncWait() bool { + return waitReasonSyncCondWait <= w && w <= waitReasonSyncWaitGroupWait } -// isWaitingForGC indicates that a goroutine is only entering _Gwaiting and -// setting a waitReason because it needs to be able to let the GC take ownership -// of its stack. The G is always actually executing on the system stack, in -// these cases. +// isChanWait is true if the goroutine is blocked because of non-nil +// channel operations or a select statement with at least one case. +// +//go:nosplit +func (w waitReason) isChanWait() bool { + return w == waitReasonSelect || + w == waitReasonChanReceive || + w == waitReasonChanSend +} + +func (w waitReason) isWaitingForSuspendG() bool { + return isWaitingForSuspendG[w] +} + +// isWaitingForSuspendG indicates that a goroutine is only entering _Gwaiting and +// setting a waitReason because it needs to be able to let the suspendG +// (used by the GC and the execution tracer) take ownership of its stack. +// The G is always actually executing on the system stack in these cases. // // TODO(mknyszek): Consider replacing this with a new dedicated G status. -var isWaitingForGC = [len(waitReasonStrings)]bool{ +var isWaitingForSuspendG = [len(waitReasonStrings)]bool{ waitReasonStoppingTheWorld: true, waitReasonGCMarkTermination: true, waitReasonGarbageCollection: true, @@ -1173,13 +1308,35 @@ var isWaitingForGC = [len(waitReasonStrings)]bool{ waitReasonFlushProcCaches: true, } +func (w waitReason) isIdleInSynctest() bool { + return isIdleInSynctest[w] +} + +// isIdleInSynctest indicates that a goroutine is considered idle by synctest.Wait. +var isIdleInSynctest = [len(waitReasonStrings)]bool{ + waitReasonChanReceiveNilChan: true, + waitReasonChanSendNilChan: true, + waitReasonSelectNoCases: true, + waitReasonSleep: true, + waitReasonSyncCondWait: true, + waitReasonSynctestWaitGroupWait: true, + waitReasonCoroutine: true, + waitReasonSynctestRun: true, + waitReasonSynctestWait: true, + waitReasonSynctestChanReceive: true, + waitReasonSynctestChanSend: true, + waitReasonSynctestSelect: true, +} + var ( - allm *m - gomaxprocs int32 - ncpu int32 - forcegc forcegcstate - sched schedt - newprocs int32 + // Linked-list of all Ms. Written under sched.lock, read atomically. + allm *m + + gomaxprocs int32 + numCPUStartup int32 + forcegc forcegcstate + sched schedt + newprocs int32 ) var ( diff --git a/src/runtime/runtime_boring.go b/src/runtime/runtime_boring.go index 5a98b20253181c..831ee67afc952d 100644 --- a/src/runtime/runtime_boring.go +++ b/src/runtime/runtime_boring.go @@ -14,6 +14,3 @@ func boring_runtime_arg0() string { } return argslice[0] } - -//go:linkname fipstls_runtime_arg0 crypto/internal/boring/fipstls.runtime_arg0 -func fipstls_runtime_arg0() string { return boring_runtime_arg0() } diff --git a/src/runtime/runtime_clearenv.go b/src/runtime/runtime_clearenv.go new file mode 100644 index 00000000000000..0cb14101c1de20 --- /dev/null +++ b/src/runtime/runtime_clearenv.go @@ -0,0 +1,29 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux + +package runtime + +import "unsafe" + +var _cgo_clearenv unsafe.Pointer // pointer to C function + +// Clear the C environment if cgo is loaded. +func clearenv_c() { + if _cgo_clearenv == nil { + return + } + asmcgocall(_cgo_clearenv, nil) +} + +//go:linkname syscall_runtimeClearenv syscall.runtimeClearenv +func syscall_runtimeClearenv(env map[string]int) { + clearenv_c() + // Did we just unset GODEBUG? + if _, ok := env["GODEBUG"]; ok { + godebugEnv.Store(nil) + godebugNotify(true) + } +} diff --git a/src/runtime/runtime_noclearenv.go b/src/runtime/runtime_noclearenv.go new file mode 100644 index 00000000000000..67e60cc9050377 --- /dev/null +++ b/src/runtime/runtime_noclearenv.go @@ -0,0 +1,18 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !linux + +package runtime + +import _ "unsafe" // for go:linkname + +//go:linkname syscall_runtimeClearenv syscall.runtimeClearenv +func syscall_runtimeClearenv(env map[string]int) { + // The system doesn't have clearenv(3) so emulate it by unsetting all of + // the variables manually. + for k := range env { + syscall_runtimeUnsetenv(k) + } +} diff --git a/src/runtime/runtime_test.go b/src/runtime/runtime_test.go index c1bf7f87dbd825..6c628f8903cb8b 100644 --- a/src/runtime/runtime_test.go +++ b/src/runtime/runtime_test.go @@ -7,7 +7,14 @@ package runtime_test import ( "flag" "fmt" + "internal/asan" + "internal/cpu" + "internal/msan" + "internal/race" + "internal/runtime/atomic" + "internal/testenv" "io" + "math/bits" . "runtime" "runtime/debug" "slices" @@ -304,7 +311,7 @@ func TestTrailingZero(t *testing.T) { } } -func TestAppendGrowth(t *testing.T) { +func TestAppendGrowthHeap(t *testing.T) { var x []int64 check := func(want int) { if cap(x) != want { @@ -321,6 +328,32 @@ func TestAppendGrowth(t *testing.T) { want = 2 * i } } + Escape(&x[0]) // suppress stack-allocated backing store +} + +func TestAppendGrowthStack(t *testing.T) { + if race.Enabled || asan.Enabled || msan.Enabled { + t.Skip("instrumentation breaks this optimization") + } + var x []int64 + check := func(want int) { + if cap(x) != want { + t.Errorf("len=%d, cap=%d, want cap=%d", len(x), cap(x), want) + } + } + + check(0) + want := 32 / 8 // 32 is the default for cmd/compile/internal/base.DebugFlags.VariableMakeThreshold + if testenv.OptimizationOff() { + want = 1 + } + for i := 1; i <= 100; i++ { + x = append(x, 1) + check(want) + if i&(i-1) == 0 { + want = max(want, 2*i) + } + } } var One = []int64{1} @@ -539,3 +572,280 @@ func TestTimediv(t *testing.T) { }) } } + +func BenchmarkProcYield(b *testing.B) { + benchN := func(n uint32) func(*testing.B) { + return func(b *testing.B) { + for i := 0; i < b.N; i++ { + ProcYield(n) + } + } + } + + b.Run("1", benchN(1)) + b.Run("10", benchN(10)) + b.Run("30", benchN(30)) // active_spin_cnt in lock_sema.go and lock_futex.go + b.Run("100", benchN(100)) + b.Run("1000", benchN(1000)) +} + +func BenchmarkOSYield(b *testing.B) { + for i := 0; i < b.N; i++ { + OSYield() + } +} + +func BenchmarkMutexContention(b *testing.B) { + // Measure throughput of a single mutex with all threads contending + // + // Share a single counter across all threads. Progress from any thread is + // progress for the benchmark as a whole. We don't measure or give points + // for fairness here, arbitrary delay to any given thread's progress is + // invisible and allowed. + // + // The cache line that holds the count value will need to move between + // processors, but not as often as the cache line that holds the mutex. The + // mutex protects access to the count value, which limits contention on that + // cache line. This is a simple design, but it helps to make the behavior of + // the benchmark clear. Most real uses of mutex will protect some number of + // cache lines anyway. + + var state struct { + _ cpu.CacheLinePad + lock Mutex + _ cpu.CacheLinePad + count atomic.Int64 + _ cpu.CacheLinePad + } + + procs := GOMAXPROCS(0) + var wg sync.WaitGroup + for range procs { + wg.Add(1) + go func() { + defer wg.Done() + for { + Lock(&state.lock) + ours := state.count.Add(1) + Unlock(&state.lock) + if ours >= int64(b.N) { + return + } + } + }() + } + wg.Wait() +} + +func BenchmarkMutexCapture(b *testing.B) { + + // Measure mutex fairness. + // + // Have several threads contend for a single mutex value. Measure how + // effectively a single thread is able to capture the lock and report the + // duration of those "streak" events. Measure how long other individual + // threads need to wait between their turns with the lock. Report the + // duration of those "starve" events. + // + // Report in terms of wall clock time (assuming a constant time per + // lock/unlock pair) rather than number of locks/unlocks. This keeps + // timekeeping overhead out of the critical path, and avoids giving an + // advantage to lock/unlock implementations that take less time per + // operation. + + var state struct { + _ cpu.CacheLinePad + lock Mutex + _ cpu.CacheLinePad + count atomic.Int64 + _ cpu.CacheLinePad + } + + procs := GOMAXPROCS(0) + var wg sync.WaitGroup + histograms := make(chan [2][65]int) + for range procs { + wg.Add(1) + go func() { + var ( + prev int64 + streak int64 + histogram [2][65]int + ) + for { + Lock(&state.lock) + ours := state.count.Add(1) + Unlock(&state.lock) + delta := ours - prev - 1 + prev = ours + if delta == 0 { + streak++ + } else { + histogram[0][bits.LeadingZeros64(uint64(streak))]++ + histogram[1][bits.LeadingZeros64(uint64(delta))]++ + streak = 1 + } + if ours >= int64(b.N) { + wg.Done() + if delta == 0 { + histogram[0][bits.LeadingZeros64(uint64(streak))]++ + histogram[1][bits.LeadingZeros64(uint64(delta))]++ + } + histograms <- histogram + return + } + } + }() + } + + wg.Wait() + b.StopTimer() + + var histogram [2][65]int + for range procs { + h := <-histograms + for i := range h { + for j := range h[i] { + histogram[i][j] += h[i][j] + } + } + } + + percentile := func(h [65]int, p float64) int { + sum := 0 + for i, v := range h { + bound := uint64(1<<63) >> i + sum += int(bound) * v + } + + // Imagine that the longest streak / starvation events were instead half + // as long but twice in number. (Note that we've pre-multiplied by the + // [lower] "bound" value.) Continue those splits until we meet the + // percentile target. + part := 0 + for i, v := range h { + bound := uint64(1<<63) >> i + part += int(bound) * v + // have we trimmed off enough at the head to dip below the percentile goal + if float64(sum-part) < float64(sum)*p { + return int(bound) + } + } + + return 0 + } + + perOp := float64(b.Elapsed().Nanoseconds()) / float64(b.N) + b.ReportMetric(perOp*float64(percentile(histogram[0], 1.0)), "ns/streak-p100") + b.ReportMetric(perOp*float64(percentile(histogram[0], 0.9)), "ns/streak-p90") + b.ReportMetric(perOp*float64(percentile(histogram[1], 1.0)), "ns/starve-p100") + b.ReportMetric(perOp*float64(percentile(histogram[1], 0.9)), "ns/starve-p90") +} + +func BenchmarkMutexHandoff(b *testing.B) { + testcase := func(delay func(l *Mutex)) func(b *testing.B) { + return func(b *testing.B) { + if workers := 2; GOMAXPROCS(0) < workers { + b.Skipf("requires GOMAXPROCS >= %d", workers) + } + + // Measure latency of mutex handoff between threads. + // + // Hand off a runtime.mutex between two threads, one running a + // "coordinator" goroutine and the other running a "worker" + // goroutine. We don't override the runtime's typical + // goroutine/thread mapping behavior. + // + // Measure the latency, starting when the coordinator enters a call + // to runtime.unlock and ending when the worker's call to + // runtime.lock returns. The benchmark can specify a "delay" + // function to simulate the length of the mutex-holder's critical + // section, including to arrange for the worker's thread to be in + // either the "spinning" or "sleeping" portions of the runtime.lock2 + // implementation. Measurement starts after any such "delay". + // + // The two threads' goroutines communicate their current position to + // each other in a non-blocking way via the "turn" state. + + var state struct { + _ cpu.CacheLinePad + lock Mutex + _ cpu.CacheLinePad + turn atomic.Int64 + _ cpu.CacheLinePad + } + + var delta atomic.Int64 + var wg sync.WaitGroup + + // coordinator: + // - acquire the mutex + // - set the turn to 2 mod 4, instructing the worker to begin its Lock call + // - wait until the mutex is contended + // - wait a bit more so the worker can commit to its sleep + // - release the mutex and wait for it to be our turn (0 mod 4) again + wg.Add(1) + go func() { + defer wg.Done() + var t int64 + for range b.N { + Lock(&state.lock) + state.turn.Add(2) + delay(&state.lock) + t -= Nanotime() // start the timer + Unlock(&state.lock) + for state.turn.Load()&0x2 != 0 { + } + } + state.turn.Add(1) + delta.Add(t) + }() + + // worker: + // - wait until its our turn (2 mod 4) + // - acquire and release the mutex + // - switch the turn counter back to the coordinator (0 mod 4) + wg.Add(1) + go func() { + defer wg.Done() + var t int64 + for { + switch state.turn.Load() & 0x3 { + case 0: + case 1, 3: + delta.Add(t) + return + case 2: + Lock(&state.lock) + t += Nanotime() // stop the timer + Unlock(&state.lock) + state.turn.Add(2) + } + } + }() + + wg.Wait() + b.ReportMetric(float64(delta.Load())/float64(b.N), "ns/op") + } + } + + b.Run("Solo", func(b *testing.B) { + var lock Mutex + for range b.N { + Lock(&lock) + Unlock(&lock) + } + }) + + b.Run("FastPingPong", testcase(func(l *Mutex) {})) + b.Run("SlowPingPong", testcase(func(l *Mutex) { + // Wait for the worker to stop spinning and prepare to sleep + for !MutexContended(l) { + } + // Wait a bit longer so the OS can finish committing the worker to its + // sleep. Balance consistency against getting enough iterations. + const extraNs = 10e3 + for t0 := Nanotime(); Nanotime()-t0 < extraNs; { + } + })) +} diff --git a/src/runtime/select.go b/src/runtime/select.go index 17c49d7484a3ec..553f6960eb1286 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -8,6 +8,7 @@ package runtime import ( "internal/abi" + "internal/runtime/sys" "unsafe" ) @@ -27,7 +28,7 @@ var ( ) func selectsetpc(pc *uintptr) { - *pc = getcallerpc() + *pc = sys.GetCallerPC() } func sellock(scases []scase, lockorder []uint16) { @@ -82,7 +83,7 @@ func selparkcommit(gp *g, _ unsafe.Pointer) bool { // channels in lock order. var lastc *hchan for sg := gp.waiting; sg != nil; sg = sg.waitlink { - if sg.c != lastc && lastc != nil { + if sg.c.get() != lastc && lastc != nil { // As soon as we unlock the channel, fields in // any sudog with that channel may change, // including c and waitlink. Since multiple @@ -91,7 +92,7 @@ func selparkcommit(gp *g, _ unsafe.Pointer) bool { // of a channel. unlock(&lastc.lock) } - lastc = sg.c + lastc = sg.c.get() } if lastc != nil { unlock(&lastc.lock) @@ -119,6 +120,7 @@ func block() { // Also, if the chosen scase was a receive operation, it reports whether // a value was received. func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, block bool) (int, bool) { + gp := getg() if debugSelect { print("select: cas0=", cas0, "\n") } @@ -164,6 +166,7 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, blo // generate permuted order norder := 0 + allSynctest := true for i := range scases { cas := &scases[i] @@ -173,8 +176,16 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, blo continue } + if cas.c.bubble != nil { + if getg().bubble != cas.c.bubble { + fatal("select on synctest channel from outside bubble") + } + } else { + allSynctest = false + } + if cas.c.timer != nil { - cas.c.timer.maybeRunChan() + cas.c.timer.maybeRunChan(cas.c) } j := cheaprandn(uint32(norder + 1)) @@ -185,6 +196,13 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, blo pollorder = pollorder[:norder] lockorder = lockorder[:norder] + waitReason := waitReasonSelect + if gp.bubble != nil && allSynctest { + // Every channel selected on is in a synctest bubble, + // so this goroutine will count as idle while selecting. + waitReason = waitReasonSynctestSelect + } + // sort the cases by Hchan address to get the locking order. // simple heap sort, to guarantee n log n time and constant stack footprint. for i := range lockorder { @@ -234,7 +252,6 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, blo sellock(scases, lockorder) var ( - gp *g sg *sudog c *hchan k *scase @@ -290,7 +307,6 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, blo } // pass 2 - enqueue on all chans - gp = getg() if gp.waiting != nil { throw("gp.waiting != nil") } @@ -304,12 +320,12 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, blo sg.isSelect = true // No stack splits between assigning elem and enqueuing // sg on gp.waiting where copystack can find it. - sg.elem = cas.elem + sg.elem.set(cas.elem) sg.releasetime = 0 if t0 != 0 { sg.releasetime = -1 } - sg.c = c + sg.c.set(c) // Construct waiting list in lock order. *nextp = sg nextp = &sg.waitlink @@ -332,7 +348,7 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, blo // changes and when we set gp.activeStackChans is not safe for // stack shrinking. gp.parkingOnChan.Store(true) - gopark(selparkcommit, nil, waitReasonSelect, traceBlockSelect, 1) + gopark(selparkcommit, nil, waitReason, traceBlockSelect, 1) gp.activeStackChans = false sellock(scases, lockorder) @@ -352,8 +368,8 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, blo // Clear all elem before unlinking from gp.waiting. for sg1 := gp.waiting; sg1 != nil; sg1 = sg1.waitlink { sg1.isSelect = false - sg1.elem = nil - sg1.c = nil + sg1.elem.set(nil) + sg1.c.set(nil) } gp.waiting = nil diff --git a/src/runtime/sema.go b/src/runtime/sema.go index f6b1b84f5f12fb..0fe0f2a2c20739 100644 --- a/src/runtime/sema.go +++ b/src/runtime/sema.go @@ -90,8 +90,8 @@ func sync_runtime_Semrelease(addr *uint32, handoff bool, skipframes int) { semrelease1(addr, handoff, skipframes) } -//go:linkname sync_runtime_SemacquireMutex sync.runtime_SemacquireMutex -func sync_runtime_SemacquireMutex(addr *uint32, lifo bool, skipframes int) { +//go:linkname internal_sync_runtime_SemacquireMutex internal/sync.runtime_SemacquireMutex +func internal_sync_runtime_SemacquireMutex(addr *uint32, lifo bool, skipframes int) { semacquire1(addr, lifo, semaBlockProfile|semaMutexProfile, skipframes, waitReasonSyncMutexLock) } @@ -105,11 +105,25 @@ func sync_runtime_SemacquireRWMutex(addr *uint32, lifo bool, skipframes int) { semacquire1(addr, lifo, semaBlockProfile|semaMutexProfile, skipframes, waitReasonSyncRWMutexLock) } +//go:linkname sync_runtime_SemacquireWaitGroup sync.runtime_SemacquireWaitGroup +func sync_runtime_SemacquireWaitGroup(addr *uint32, synctestDurable bool) { + reason := waitReasonSyncWaitGroupWait + if synctestDurable { + reason = waitReasonSynctestWaitGroupWait + } + semacquire1(addr, false, semaBlockProfile, 0, reason) +} + //go:linkname poll_runtime_Semrelease internal/poll.runtime_Semrelease func poll_runtime_Semrelease(addr *uint32) { semrelease(addr) } +//go:linkname internal_sync_runtime_Semrelease internal/sync.runtime_Semrelease +func internal_sync_runtime_Semrelease(addr *uint32, handoff bool, skipframes int) { + semrelease1(addr, handoff, skipframes) +} + func readyWithTime(s *sudog, traceskip int) { if s.releasetime != 0 { s.releasetime = cputicks() @@ -247,11 +261,13 @@ func semrelease1(addr *uint32, handoff bool, skipframes int) { s.ticket = 1 } readyWithTime(s, 5+skipframes) - if s.ticket == 1 && getg().m.locks == 0 { + if s.ticket == 1 && getg().m.locks == 0 && getg() != getg().m.g0 { // Direct G handoff + // // readyWithTime has added the waiter G as runnext in the // current P; we now call the scheduler so that we start running // the waiter G immediately. + // // Note that waiter inherits our time slice: this is desirable // to avoid having a highly contended semaphore hog the P // indefinitely. goyield is like Gosched, but it emits a @@ -261,9 +277,12 @@ func semrelease1(addr *uint32, handoff bool, skipframes int) { // the non-starving case it is possible for a different waiter // to acquire the semaphore while we are yielding/scheduling, // and this would be wasteful. We wait instead to enter starving - // regime, and then we start to do direct handoffs of ticket and - // P. + // regime, and then we start to do direct handoffs of ticket and P. + // // See issue 33747 for discussion. + // + // We don't handoff directly if we're holding locks or on the + // system stack, since it's not safe to enter the scheduler. goyield() } } @@ -284,7 +303,10 @@ func cansemacquire(addr *uint32) bool { // queue adds s to the blocked goroutines in semaRoot. func (root *semaRoot) queue(addr *uint32, s *sudog, lifo bool) { s.g = getg() - s.elem = unsafe.Pointer(addr) + s.elem.set(unsafe.Pointer(addr)) + // Storing this pointer so that we can trace the semaphore address + // from the blocked goroutine when checking for goroutine leaks. + s.g.waiting = s s.next = nil s.prev = nil s.waiters = 0 @@ -292,7 +314,7 @@ func (root *semaRoot) queue(addr *uint32, s *sudog, lifo bool) { var last *sudog pt := &root.treap for t := *pt; t != nil; t = *pt { - if t.elem == unsafe.Pointer(addr) { + if uintptr(unsafe.Pointer(addr)) == t.elem.uintptr() { // Already have addr in list. if lifo { // Substitute s in t's place in treap. @@ -338,7 +360,7 @@ func (root *semaRoot) queue(addr *uint32, s *sudog, lifo bool) { return } last = t - if uintptr(unsafe.Pointer(addr)) < uintptr(t.elem) { + if uintptr(unsafe.Pointer(addr)) < t.elem.uintptr() { pt = &t.prev } else { pt = &t.next @@ -383,11 +405,13 @@ func (root *semaRoot) queue(addr *uint32, s *sudog, lifo bool) { func (root *semaRoot) dequeue(addr *uint32) (found *sudog, now, tailtime int64) { ps := &root.treap s := *ps + for ; s != nil; s = *ps { - if s.elem == unsafe.Pointer(addr) { + if uintptr(unsafe.Pointer(addr)) == s.elem.uintptr() { goto Found } - if uintptr(unsafe.Pointer(addr)) < uintptr(s.elem) { + + if uintptr(unsafe.Pointer(addr)) < s.elem.uintptr() { ps = &s.prev } else { ps = &s.next @@ -451,8 +475,10 @@ Found: } tailtime = s.acquiretime } + // Goroutine is no longer blocked. Clear the waiting pointer. + s.g.waiting = nil s.parent = nil - s.elem = nil + s.elem.set(nil) s.next = nil s.prev = nil s.ticket = 0 @@ -571,6 +597,10 @@ func notifyListWait(l *notifyList, t uint32) { // Enqueue itself. s := acquireSudog() s.g = getg() + // Storing this pointer so that we can trace the condvar address + // from the blocked goroutine when checking for goroutine leaks. + s.elem.set(unsafe.Pointer(l)) + s.g.waiting = s s.ticket = t s.releasetime = 0 t0 := int64(0) @@ -588,6 +618,10 @@ func notifyListWait(l *notifyList, t uint32) { if t0 != 0 { blockevent(s.releasetime-t0, 2) } + // Goroutine is no longer blocked. Clear up its waiting pointer, + // and clean up the sudog before releasing it. + s.g.waiting = nil + s.elem.set(nil) releaseSudog(s) } @@ -619,6 +653,10 @@ func notifyListNotifyAll(l *notifyList) { for s != nil { next := s.next s.next = nil + if s.g.bubble != nil && getg().bubble != s.g.bubble { + println("semaphore wake of synctest goroutine", s.g.goid, "from outside bubble") + fatal("semaphore wake of synctest goroutine from outside bubble") + } readyWithTime(s, 4) s = next } @@ -672,6 +710,10 @@ func notifyListNotifyOne(l *notifyList) { } unlock(&l.lock) s.next = nil + if s.g.bubble != nil && getg().bubble != s.g.bubble { + println("semaphore wake of synctest goroutine", s.g.goid, "from outside bubble") + fatal("semaphore wake of synctest goroutine from outside bubble") + } readyWithTime(s, 4) return } @@ -687,7 +729,7 @@ func notifyListCheck(sz uintptr) { } } -//go:linkname sync_nanotime sync.runtime_nanotime -func sync_nanotime() int64 { +//go:linkname internal_sync_nanotime internal/sync.runtime_nanotime +func internal_sync_nanotime() int64 { return nanotime() } diff --git a/src/runtime/semasleep_test.go b/src/runtime/semasleep_test.go index 711d5df7352565..4f2eea9d928c38 100644 --- a/src/runtime/semasleep_test.go +++ b/src/runtime/semasleep_test.go @@ -72,14 +72,6 @@ func TestSpuriousWakeupsNeverHangSemasleep(t *testing.T) { close(doneCh) }() - // Wait for an arbitrary timeout longer than one second. The subprocess itself - // attempts to sleep for one second, but if the machine running the test is - // heavily loaded that subprocess may not schedule very quickly even if the - // bug remains fixed. (This is fine, because if the bug really is unfixed we - // can keep the process hung indefinitely, as long as we signal it often - // enough.) - timeout := 10 * time.Second - // The subprocess begins sleeping for 1s after it writes to stdout, so measure // the timeout from here (not from when we started creating the process). // That should reduce noise from process startup overhead. @@ -91,30 +83,63 @@ func TestSpuriousWakeupsNeverHangSemasleep(t *testing.T) { // pthread_cond_timedwait_relative_np. ticker := time.NewTicker(200 * time.Millisecond) defer ticker.Stop() + + checkDoneErr := func(err error) { + if err != nil { + t.Fatalf("The program returned but unfortunately with an error: %v", err) + } + if time.Since(beforeStart) < 1*time.Second { + // The program was supposed to sleep for a full (monotonic) second; + // it should not return before that has elapsed. + t.Fatalf("The program stopped too quickly.") + } + } + signaled := 0 for { select { - case now := <-ticker.C: - if now.Sub(ready) > timeout { - t.Error("Program failed to return on time and has to be killed, issue #27520 still exists") - // Send SIGQUIT to get a goroutine dump. - // Stop sending SIGIO so that the program can clean up and actually terminate. - cmd.Process.Signal(syscall.SIGQUIT) + // Wait for an arbitrary timeout longer than one second (we use the test binary's + // timeout). The subprocess itself attempts to sleep for one second, but if the + // machine running the test is heavily loaded that subprocess may not schedule + // very quickly even if the bug remains fixed. (This is fine, because if the bug + // really is unfixed we can keep the process hung indefinitely, as long as we + // signal it often enough.) + case <-t.Context().Done(): + // If we got paused for a long time for any reason (we're running in parallel + // with other tests after all) it could be that the subprocess did actually + // finish and not deadlock, we just got stuck as runnable or our wakeup was + // delayed. Double-check that we don't have anything from doneCh before + // declaring failure. + select { + case err := <-doneCh: + checkDoneErr(err) return + default: } + // Send SIGQUIT to get a goroutine dump. + // Stop sending SIGIO so that the program can clean up and actually terminate. + cmd.Process.Signal(syscall.SIGQUIT) + + // For failure, we require that we have waited at least 2 seconds. Otherwise + // if another test just ran for a long time and we just happened to push the + // overall binary to timeout, we will report a failure here that will just + // muddy the waters for whoever is trying to investigate the timeout. + const minTime = 2 * time.Second + if dur := time.Now().Sub(ready); dur > minTime { + t.Logf("signaled program %d times", signaled) + t.Logf("time waited: %v", dur) + t.Error("program failed to return on time and has to be killed, issue #27520 still exists") + } else { + t.Skipf("test context deadline ended, but test ran for %s<%s, cutting it short", dur, minTime) + } + return + case <-ticker.C: // Send the pesky signal that toggles spinning // indefinitely if #27520 is not fixed. cmd.Process.Signal(syscall.SIGIO) - + signaled++ case err := <-doneCh: - if err != nil { - t.Fatalf("The program returned but unfortunately with an error: %v", err) - } - if time.Since(beforeStart) < 1*time.Second { - // The program was supposed to sleep for a full (monotonic) second; - // it should not return before that has elapsed. - t.Fatalf("The program stopped too quickly.") - } + checkDoneErr(err) return } } diff --git a/src/runtime/set_vma_name_linux.go b/src/runtime/set_vma_name_linux.go new file mode 100644 index 00000000000000..03c7739c3465c9 --- /dev/null +++ b/src/runtime/set_vma_name_linux.go @@ -0,0 +1,36 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux + +package runtime + +import ( + "internal/runtime/atomic" + "internal/runtime/syscall/linux" + "unsafe" +) + +var prSetVMAUnsupported atomic.Bool + +func setVMANameSupported() bool { + return !prSetVMAUnsupported.Load() +} + +// setVMAName calls prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, start, len, name) +func setVMAName(start unsafe.Pointer, length uintptr, name string) { + if debug.decoratemappings == 0 || !setVMANameSupported() { + return + } + + var sysName [80]byte + n := copy(sysName[:], " Go: ") + copy(sysName[n:79], name) // leave final byte zero + + _, _, err := linux.Syscall6(linux.SYS_PRCTL, linux.PR_SET_VMA, linux.PR_SET_VMA_ANON_NAME, uintptr(start), length, uintptr(unsafe.Pointer(&sysName[0])), 0) + if err == _EINVAL { + prSetVMAUnsupported.Store(true) + } + // ignore other errors +} diff --git a/src/runtime/set_vma_name_stub.go b/src/runtime/set_vma_name_stub.go new file mode 100644 index 00000000000000..6cb01ebf50dcef --- /dev/null +++ b/src/runtime/set_vma_name_stub.go @@ -0,0 +1,14 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !linux + +package runtime + +import "unsafe" + +// setVMAName isn’t implemented +func setVMAName(start unsafe.Pointer, len uintptr, name string) {} + +func setVMANameSupported() bool { return false } diff --git a/src/runtime/sigaction.go b/src/runtime/sigaction.go index 05f44f65dbcd56..1a99f7f3ecf699 100644 --- a/src/runtime/sigaction.go +++ b/src/runtime/sigaction.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (linux && !amd64 && !arm64 && !ppc64le) || (freebsd && !amd64) +//go:build (linux && !386 && !amd64 && !arm64 && !loong64 && !ppc64le) || (freebsd && !amd64) package runtime diff --git a/src/runtime/signal_loong64.go b/src/runtime/signal_loong64.go index ac842c0c9494f8..970af01ceee976 100644 --- a/src/runtime/signal_loong64.go +++ b/src/runtime/signal_loong64.go @@ -53,9 +53,10 @@ func dumpregs(c *sigctxt) { //go:nowritebarrierrec func (c *sigctxt) sigpc() uintptr { return uintptr(c.pc()) } -func (c *sigctxt) sigsp() uintptr { return uintptr(c.sp()) } -func (c *sigctxt) siglr() uintptr { return uintptr(c.link()) } -func (c *sigctxt) fault() uintptr { return uintptr(c.sigaddr()) } +func (c *sigctxt) setsigpc(x uint64) { c.set_pc(x) } +func (c *sigctxt) sigsp() uintptr { return uintptr(c.sp()) } +func (c *sigctxt) siglr() uintptr { return uintptr(c.link()) } +func (c *sigctxt) fault() uintptr { return uintptr(c.sigaddr()) } // preparePanic sets up the stack to look like a call to sigpanic. func (c *sigctxt) preparePanic(sig uint32, gp *g) { diff --git a/src/runtime/signal_mips64x.go b/src/runtime/signal_mips64x.go index cee1bf7a1b23dc..eea2169408cd8d 100644 --- a/src/runtime/signal_mips64x.go +++ b/src/runtime/signal_mips64x.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (linux || openbsd) && (mips64 || mips64le) +//go:build linux && (mips64 || mips64le) package runtime diff --git a/src/runtime/signal_openbsd_mips64.go b/src/runtime/signal_openbsd_mips64.go deleted file mode 100644 index 54ed523c7b991f..00000000000000 --- a/src/runtime/signal_openbsd_mips64.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -import ( - "unsafe" -) - -type sigctxt struct { - info *siginfo - ctxt unsafe.Pointer -} - -//go:nosplit -//go:nowritebarrierrec -func (c *sigctxt) regs() *sigcontext { - return (*sigcontext)(c.ctxt) -} - -func (c *sigctxt) r0() uint64 { return c.regs().sc_regs[0] } -func (c *sigctxt) r1() uint64 { return c.regs().sc_regs[1] } -func (c *sigctxt) r2() uint64 { return c.regs().sc_regs[2] } -func (c *sigctxt) r3() uint64 { return c.regs().sc_regs[3] } -func (c *sigctxt) r4() uint64 { return c.regs().sc_regs[4] } -func (c *sigctxt) r5() uint64 { return c.regs().sc_regs[5] } -func (c *sigctxt) r6() uint64 { return c.regs().sc_regs[6] } -func (c *sigctxt) r7() uint64 { return c.regs().sc_regs[7] } -func (c *sigctxt) r8() uint64 { return c.regs().sc_regs[8] } -func (c *sigctxt) r9() uint64 { return c.regs().sc_regs[9] } -func (c *sigctxt) r10() uint64 { return c.regs().sc_regs[10] } -func (c *sigctxt) r11() uint64 { return c.regs().sc_regs[11] } -func (c *sigctxt) r12() uint64 { return c.regs().sc_regs[12] } -func (c *sigctxt) r13() uint64 { return c.regs().sc_regs[13] } -func (c *sigctxt) r14() uint64 { return c.regs().sc_regs[14] } -func (c *sigctxt) r15() uint64 { return c.regs().sc_regs[15] } -func (c *sigctxt) r16() uint64 { return c.regs().sc_regs[16] } -func (c *sigctxt) r17() uint64 { return c.regs().sc_regs[17] } -func (c *sigctxt) r18() uint64 { return c.regs().sc_regs[18] } -func (c *sigctxt) r19() uint64 { return c.regs().sc_regs[19] } -func (c *sigctxt) r20() uint64 { return c.regs().sc_regs[20] } -func (c *sigctxt) r21() uint64 { return c.regs().sc_regs[21] } -func (c *sigctxt) r22() uint64 { return c.regs().sc_regs[22] } -func (c *sigctxt) r23() uint64 { return c.regs().sc_regs[23] } -func (c *sigctxt) r24() uint64 { return c.regs().sc_regs[24] } -func (c *sigctxt) r25() uint64 { return c.regs().sc_regs[25] } -func (c *sigctxt) r26() uint64 { return c.regs().sc_regs[26] } -func (c *sigctxt) r27() uint64 { return c.regs().sc_regs[27] } -func (c *sigctxt) r28() uint64 { return c.regs().sc_regs[28] } -func (c *sigctxt) r29() uint64 { return c.regs().sc_regs[29] } -func (c *sigctxt) r30() uint64 { return c.regs().sc_regs[30] } -func (c *sigctxt) r31() uint64 { return c.regs().sc_regs[31] } -func (c *sigctxt) sp() uint64 { return c.regs().sc_regs[29] } - -//go:nosplit -//go:nowritebarrierrec -func (c *sigctxt) pc() uint64 { return c.regs().sc_pc } - -func (c *sigctxt) link() uint64 { return c.regs().sc_regs[31] } -func (c *sigctxt) lo() uint64 { return c.regs().mullo } -func (c *sigctxt) hi() uint64 { return c.regs().mulhi } - -func (c *sigctxt) sigcode() uint32 { return uint32(c.info.si_code) } -func (c *sigctxt) sigaddr() uint64 { - return *(*uint64)(add(unsafe.Pointer(c.info), 16)) -} - -func (c *sigctxt) set_r28(x uint64) { c.regs().sc_regs[28] = x } -func (c *sigctxt) set_r30(x uint64) { c.regs().sc_regs[30] = x } -func (c *sigctxt) set_pc(x uint64) { c.regs().sc_pc = x } -func (c *sigctxt) set_sp(x uint64) { c.regs().sc_regs[29] = x } -func (c *sigctxt) set_link(x uint64) { c.regs().sc_regs[31] = x } - -func (c *sigctxt) set_sigcode(x uint32) { c.info.si_code = int32(x) } -func (c *sigctxt) set_sigaddr(x uint64) { - *(*uint64)(add(unsafe.Pointer(c.info), 16)) = x -} diff --git a/src/runtime/signal_unix.go b/src/runtime/signal_unix.go index a42972bb35686d..96628d6baae910 100644 --- a/src/runtime/signal_unix.go +++ b/src/runtime/signal_unix.go @@ -405,7 +405,7 @@ func sigFetchG(c *sigctxt) *g { // bottom of the signal stack. Fetch from there. // TODO: in efence mode, stack is sysAlloc'd, so this wouldn't // work. - sp := getcallersp() + sp := sys.GetCallerSP() s := spanOf(sp) if s != nil && s.state.get() == mSpanManual && s.base() < sp && sp < s.limit { gp := *(**g)(unsafe.Pointer(s.base())) @@ -479,7 +479,7 @@ func sigtrampgo(sig uint32, info *siginfo, ctx unsafe.Pointer) { var gsignalStack gsignalStack setStack := adjustSignalStack(sig, gp.m, &gsignalStack) if setStack { - gp.m.gsignal.stktopsp = getcallersp() + gp.m.gsignal.stktopsp = sys.GetCallerSP() } if gp.stackguard0 == stackFork { @@ -584,15 +584,23 @@ func adjustSignalStack(sig uint32, mp *m, gsigStack *gsignalStack) bool { } // sp is not within gsignal stack, g0 stack, or sigaltstack. Bad. + // Call indirectly to avoid nosplit stack overflow on OpenBSD. + adjustSignalStack2Indirect(sig, sp, mp, st.ss_flags&_SS_DISABLE != 0) + return false +} + +var adjustSignalStack2Indirect = adjustSignalStack2 + +//go:nosplit +func adjustSignalStack2(sig uint32, sp uintptr, mp *m, ssDisable bool) { setg(nil) needm(true) - if st.ss_flags&_SS_DISABLE != 0 { + if ssDisable { noSignalStack(sig) } else { sigNotOnStack(sig, sp, mp) } dropm() - return false } // crashing is the number of m's we have waited for when implementing @@ -605,6 +613,19 @@ var crashing atomic.Int32 var testSigtrap func(info *siginfo, ctxt *sigctxt, gp *g) bool var testSigusr1 func(gp *g) bool +// sigsysIgnored is non-zero if we are currently ignoring SIGSYS. See issue #69065. +var sigsysIgnored uint32 + +//go:linkname ignoreSIGSYS os.ignoreSIGSYS +func ignoreSIGSYS() { + atomic.Store(&sigsysIgnored, 1) +} + +//go:linkname restoreSIGSYS os.restoreSIGSYS +func restoreSIGSYS() { + atomic.Store(&sigsysIgnored, 0) +} + // sighandler is invoked when a signal occurs. The global g will be // set to a gsignal goroutine and we will be running on the alternate // signal stack. The parameter gp will be the value of the global g @@ -715,6 +736,10 @@ func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) { return } + if sig == _SIGSYS && c.sigFromSeccomp() && atomic.Load(&sigsysIgnored) != 0 { + return + } + if flags&_SigKill != 0 { dieFromSignal(sig) } diff --git a/src/runtime/signal_windows.go b/src/runtime/signal_windows.go index b0c653ee46dd0b..40547b8113fffb 100644 --- a/src/runtime/signal_windows.go +++ b/src/runtime/signal_windows.go @@ -6,41 +6,33 @@ package runtime import ( "internal/abi" - "internal/runtime/sys" + "internal/runtime/syscall/windows" "unsafe" ) -const ( - _SEM_FAILCRITICALERRORS = 0x0001 - _SEM_NOGPFAULTERRORBOX = 0x0002 - _SEM_NOOPENFILEERRORBOX = 0x8000 - - _WER_FAULT_REPORTING_NO_UI = 0x0020 -) - func preventErrorDialogs() { - errormode := stdcall0(_GetErrorMode) - stdcall1(_SetErrorMode, errormode|_SEM_FAILCRITICALERRORS|_SEM_NOGPFAULTERRORBOX|_SEM_NOOPENFILEERRORBOX) + errormode := stdcall(_GetErrorMode) + stdcall(_SetErrorMode, errormode|windows.SEM_FAILCRITICALERRORS|windows.SEM_NOGPFAULTERRORBOX|windows.SEM_NOOPENFILEERRORBOX) // Disable WER fault reporting UI. // Do this even if WER is disabled as a whole, // as WER might be enabled later with setTraceback("wer") // and we still want the fault reporting UI to be disabled if this happens. var werflags uintptr - stdcall2(_WerGetFlags, currentProcess, uintptr(unsafe.Pointer(&werflags))) - stdcall1(_WerSetFlags, werflags|_WER_FAULT_REPORTING_NO_UI) + stdcall(_WerGetFlags, windows.CurrentProcess, uintptr(unsafe.Pointer(&werflags))) + stdcall(_WerSetFlags, werflags|windows.WER_FAULT_REPORTING_NO_UI) } // enableWER re-enables Windows error reporting without fault reporting UI. func enableWER() { // re-enable Windows Error Reporting - errormode := stdcall0(_GetErrorMode) - if errormode&_SEM_NOGPFAULTERRORBOX != 0 { - stdcall1(_SetErrorMode, errormode^_SEM_NOGPFAULTERRORBOX) + errormode := stdcall(_GetErrorMode) + if errormode&windows.SEM_NOGPFAULTERRORBOX != 0 { + stdcall(_SetErrorMode, errormode^windows.SEM_NOGPFAULTERRORBOX) } } -// in sys_windows_386.s, sys_windows_amd64.s, sys_windows_arm.s, and sys_windows_arm64.s +// in sys_windows_386.s, sys_windows_amd64.s, and sys_windows_arm64.s func exceptiontramp() func firstcontinuetramp() func lastcontinuetramp() @@ -48,14 +40,14 @@ func sehtramp() func sigresume() func initExceptionHandler() { - stdcall2(_AddVectoredExceptionHandler, 1, abi.FuncPCABI0(exceptiontramp)) + stdcall(_AddVectoredExceptionHandler, 1, abi.FuncPCABI0(exceptiontramp)) if GOARCH == "386" { // use SetUnhandledExceptionFilter for windows-386. // note: SetUnhandledExceptionFilter handler won't be called, if debugging. - stdcall1(_SetUnhandledExceptionFilter, abi.FuncPCABI0(lastcontinuetramp)) + stdcall(_SetUnhandledExceptionFilter, abi.FuncPCABI0(lastcontinuetramp)) } else { - stdcall2(_AddVectoredContinueHandler, 1, abi.FuncPCABI0(firstcontinuetramp)) - stdcall2(_AddVectoredContinueHandler, 0, abi.FuncPCABI0(lastcontinuetramp)) + stdcall(_AddVectoredContinueHandler, 1, abi.FuncPCABI0(firstcontinuetramp)) + stdcall(_AddVectoredContinueHandler, 0, abi.FuncPCABI0(lastcontinuetramp)) } } @@ -63,12 +55,11 @@ func initExceptionHandler() { // by calling runtime.abort function. // //go:nosplit -func isAbort(r *context) bool { - pc := r.ip() - if GOARCH == "386" || GOARCH == "amd64" || GOARCH == "arm" { +func isAbort(r *windows.Context) bool { + pc := r.PC() + if GOARCH == "386" || GOARCH == "amd64" { // In the case of an abort, the exception IP is one byte after - // the INT3 (this differs from UNIX OSes). Note that on ARM, - // this means that the exception IP is no longer aligned. + // the INT3 (this differs from UNIX OSes). pc-- } return isAbortPC(pc) @@ -81,29 +72,29 @@ func isAbort(r *context) bool { // because of a stack overflow. // //go:nosplit -func isgoexception(info *exceptionrecord, r *context) bool { +func isgoexception(info *windows.ExceptionRecord, r *windows.Context) bool { // Only handle exception if executing instructions in Go binary // (not Windows library code). // TODO(mwhudson): needs to loop to support shared libs - if r.ip() < firstmoduledata.text || firstmoduledata.etext < r.ip() { + if r.PC() < firstmoduledata.text || firstmoduledata.etext < r.PC() { return false } // Go will only handle some exceptions. - switch info.exceptioncode { + switch info.ExceptionCode { default: return false - case _EXCEPTION_ACCESS_VIOLATION: - case _EXCEPTION_IN_PAGE_ERROR: - case _EXCEPTION_INT_DIVIDE_BY_ZERO: - case _EXCEPTION_INT_OVERFLOW: - case _EXCEPTION_FLT_DENORMAL_OPERAND: - case _EXCEPTION_FLT_DIVIDE_BY_ZERO: - case _EXCEPTION_FLT_INEXACT_RESULT: - case _EXCEPTION_FLT_OVERFLOW: - case _EXCEPTION_FLT_UNDERFLOW: - case _EXCEPTION_BREAKPOINT: - case _EXCEPTION_ILLEGAL_INSTRUCTION: // breakpoint arrives this way on arm64 + case windows.EXCEPTION_ACCESS_VIOLATION: + case windows.EXCEPTION_IN_PAGE_ERROR: + case windows.EXCEPTION_INT_DIVIDE_BY_ZERO: + case windows.EXCEPTION_INT_OVERFLOW: + case windows.EXCEPTION_FLT_DENORMAL_OPERAND: + case windows.EXCEPTION_FLT_DIVIDE_BY_ZERO: + case windows.EXCEPTION_FLT_INEXACT_RESULT: + case windows.EXCEPTION_FLT_OVERFLOW: + case windows.EXCEPTION_FLT_UNDERFLOW: + case windows.EXCEPTION_BREAKPOINT: + case windows.EXCEPTION_ILLEGAL_INSTRUCTION: // breakpoint arrives this way on arm64 } return true } @@ -136,13 +127,13 @@ func sigFetchG() *g { // It is nosplit for the same reason as exceptionhandler. // //go:nosplit -func sigtrampgo(ep *exceptionpointers, kind int) int32 { +func sigtrampgo(ep *windows.ExceptionPointers, kind int) int32 { gp := sigFetchG() if gp == nil { - return _EXCEPTION_CONTINUE_SEARCH + return windows.EXCEPTION_CONTINUE_SEARCH } - var fn func(info *exceptionrecord, r *context, gp *g) int32 + var fn func(info *windows.ExceptionRecord, r *windows.Context, gp *g) int32 switch kind { case callbackVEH: fn = exceptionhandler @@ -171,12 +162,12 @@ func sigtrampgo(ep *exceptionpointers, kind int) int32 { var ret int32 if gp != gp.m.g0 { systemstack(func() { - ret = fn(ep.record, ep.context, gp) + ret = fn(ep.Record, ep.Context, gp) }) } else { - ret = fn(ep.record, ep.context, gp) + ret = fn(ep.Record, ep.Context, gp) } - if ret == _EXCEPTION_CONTINUE_SEARCH { + if ret == windows.EXCEPTION_CONTINUE_SEARCH { return ret } @@ -191,13 +182,13 @@ func sigtrampgo(ep *exceptionpointers, kind int) int32 { // will not actually return to the original frame, so the registers // are effectively dead. But this does mean we can't use the // same mechanism for async preemption. - if ep.context.ip() == abi.FuncPCABI0(sigresume) { + if ep.Context.PC() == abi.FuncPCABI0(sigresume) { // sigresume has already been set up by a previous exception. return ret } - prepareContextForSigResume(ep.context) - ep.context.set_sp(gp.m.g0.sched.sp) - ep.context.set_ip(abi.FuncPCABI0(sigresume)) + prepareContextForSigResume(ep.Context) + ep.Context.SetSP(gp.m.g0.sched.sp) + ep.Context.SetPC(abi.FuncPCABI0(sigresume)) return ret } @@ -209,9 +200,9 @@ func sigtrampgo(ep *exceptionpointers, kind int) int32 { // _EXCEPTION_BREAKPOINT, which is raised by abort() if we overflow the g0 stack. // //go:nosplit -func exceptionhandler(info *exceptionrecord, r *context, gp *g) int32 { +func exceptionhandler(info *windows.ExceptionRecord, r *windows.Context, gp *g) int32 { if !isgoexception(info, r) { - return _EXCEPTION_CONTINUE_SEARCH + return windows.EXCEPTION_CONTINUE_SEARCH } if gp.throwsplit || isAbort(r) { @@ -228,10 +219,10 @@ func exceptionhandler(info *exceptionrecord, r *context, gp *g) int32 { // Have to pass arguments out of band since // augmenting the stack frame would break // the unwinding code. - gp.sig = info.exceptioncode - gp.sigcode0 = info.exceptioninformation[0] - gp.sigcode1 = info.exceptioninformation[1] - gp.sigpc = r.ip() + gp.sig = info.ExceptionCode + gp.sigcode0 = info.ExceptionInformation[0] + gp.sigcode1 = info.ExceptionInformation[1] + gp.sigpc = r.PC() // Only push runtime·sigpanic if r.ip() != 0. // If r.ip() == 0, probably panicked because of a @@ -246,20 +237,13 @@ func exceptionhandler(info *exceptionrecord, r *context, gp *g) int32 { // The exception is not from asyncPreempt, so not to push a // sigpanic call to make it look like that. Instead, just // overwrite the PC. (See issue #35773) - if r.ip() != 0 && r.ip() != abi.FuncPCABI0(asyncPreempt) { - sp := unsafe.Pointer(r.sp()) - delta := uintptr(sys.StackAlign) - sp = add(sp, -delta) - r.set_sp(uintptr(sp)) - if usesLR { - *((*uintptr)(sp)) = r.lr() - r.set_lr(r.ip()) - } else { - *((*uintptr)(sp)) = r.ip() - } + if r.PC() != 0 && r.PC() != abi.FuncPCABI0(asyncPreempt) { + r.PushCall(abi.FuncPCABI0(sigpanic0), r.PC()) + } else { + // Not safe to push the call. Just clobber the frame. + r.SetPC(abi.FuncPCABI0(sigpanic0)) } - r.set_ip(abi.FuncPCABI0(sigpanic0)) - return _EXCEPTION_CONTINUE_EXECUTION + return windows.EXCEPTION_CONTINUE_EXECUTION } // sehhandler is reached as part of the SEH chain. @@ -267,11 +251,11 @@ func exceptionhandler(info *exceptionrecord, r *context, gp *g) int32 { // It is nosplit for the same reason as exceptionhandler. // //go:nosplit -func sehhandler(_ *exceptionrecord, _ uint64, _ *context, dctxt *_DISPATCHER_CONTEXT) int32 { +func sehhandler(_ *windows.ExceptionRecord, _ uint64, _ *windows.Context, dctxt *windows.DISPATCHER_CONTEXT) int32 { g0 := getg() if g0 == nil || g0.m.curg == nil { // No g available, nothing to do here. - return _EXCEPTION_CONTINUE_SEARCH_SEH + return windows.EXCEPTION_CONTINUE_SEARCH_SEH } // The Windows SEH machinery will unwind the stack until it finds // a frame with a handler for the exception or until the frame is @@ -284,19 +268,19 @@ func sehhandler(_ *exceptionrecord, _ uint64, _ *context, dctxt *_DISPATCHER_CON // To work around this, manually unwind the stack until the top of the goroutine // stack is reached, and then pass the control back to Windows. gp := g0.m.curg - ctxt := dctxt.ctx() + ctxt := dctxt.Ctx() var base, sp uintptr for { - entry := stdcall3(_RtlLookupFunctionEntry, ctxt.ip(), uintptr(unsafe.Pointer(&base)), 0) + entry := stdcall(_RtlLookupFunctionEntry, ctxt.PC(), uintptr(unsafe.Pointer(&base)), 0) if entry == 0 { break } - stdcall8(_RtlVirtualUnwind, 0, base, ctxt.ip(), entry, uintptr(unsafe.Pointer(ctxt)), 0, uintptr(unsafe.Pointer(&sp)), 0) + stdcall(_RtlVirtualUnwind, 0, base, ctxt.PC(), entry, uintptr(unsafe.Pointer(ctxt)), 0, uintptr(unsafe.Pointer(&sp)), 0) if sp < gp.stack.lo || gp.stack.hi <= sp { break } } - return _EXCEPTION_CONTINUE_SEARCH_SEH + return windows.EXCEPTION_CONTINUE_SEARCH_SEH } // It seems Windows searches ContinueHandler's list even @@ -307,11 +291,11 @@ func sehhandler(_ *exceptionrecord, _ uint64, _ *context, dctxt *_DISPATCHER_CON // It is nosplit for the same reason as exceptionhandler. // //go:nosplit -func firstcontinuehandler(info *exceptionrecord, r *context, gp *g) int32 { +func firstcontinuehandler(info *windows.ExceptionRecord, r *windows.Context, gp *g) int32 { if !isgoexception(info, r) { - return _EXCEPTION_CONTINUE_SEARCH + return windows.EXCEPTION_CONTINUE_SEARCH } - return _EXCEPTION_CONTINUE_EXECUTION + return windows.EXCEPTION_CONTINUE_EXECUTION } // lastcontinuehandler is reached, because runtime cannot handle @@ -320,12 +304,12 @@ func firstcontinuehandler(info *exceptionrecord, r *context, gp *g) int32 { // It is nosplit for the same reason as exceptionhandler. // //go:nosplit -func lastcontinuehandler(info *exceptionrecord, r *context, gp *g) int32 { +func lastcontinuehandler(info *windows.ExceptionRecord, r *windows.Context, gp *g) int32 { if islibrary || isarchive { // Go DLL/archive has been loaded in a non-go program. // If the exception does not originate from go, the go runtime // should not take responsibility of crashing the process. - return _EXCEPTION_CONTINUE_SEARCH + return windows.EXCEPTION_CONTINUE_SEARCH } // VEH is called before SEH, but arm64 MSVC DLLs use SEH to trap @@ -334,9 +318,9 @@ func lastcontinuehandler(info *exceptionrecord, r *context, gp *g) int32 { // arm64 and it's an illegal instruction and this is coming from // non-Go code, then assume it's this runtime probing happen, and // pass that onward to SEH. - if GOARCH == "arm64" && info.exceptioncode == _EXCEPTION_ILLEGAL_INSTRUCTION && - (r.ip() < firstmoduledata.text || firstmoduledata.etext < r.ip()) { - return _EXCEPTION_CONTINUE_SEARCH + if GOARCH == "arm64" && info.ExceptionCode == windows.EXCEPTION_ILLEGAL_INSTRUCTION && + (r.PC() < firstmoduledata.text || firstmoduledata.etext < r.PC()) { + return windows.EXCEPTION_CONTINUE_SEARCH } winthrow(info, r, gp) @@ -346,7 +330,7 @@ func lastcontinuehandler(info *exceptionrecord, r *context, gp *g) int32 { // Always called on g0. gp is the G where the exception occurred. // //go:nosplit -func winthrow(info *exceptionrecord, r *context, gp *g) { +func winthrow(info *windows.ExceptionRecord, r *windows.Context, gp *g) { g0 := getg() if panicking.Load() != 0 { // traceback already printed @@ -361,9 +345,9 @@ func winthrow(info *exceptionrecord, r *context, gp *g) { g0.stackguard0 = g0.stack.lo + stackGuard g0.stackguard1 = g0.stackguard0 - print("Exception ", hex(info.exceptioncode), " ", hex(info.exceptioninformation[0]), " ", hex(info.exceptioninformation[1]), " ", hex(r.ip()), "\n") + print("Exception ", hex(info.ExceptionCode), " ", hex(info.ExceptionInformation[0]), " ", hex(info.ExceptionInformation[1]), " ", hex(r.PC()), "\n") - print("PC=", hex(r.ip()), "\n") + print("PC=", hex(r.PC()), "\n") if g0.m.incgo && gp == g0.m.g0 && g0.m.curg != nil { if iscgo { print("signal arrived during external code execution\n") @@ -377,7 +361,7 @@ func winthrow(info *exceptionrecord, r *context, gp *g) { level, _, docrash := gotraceback() if level > 0 { - tracebacktrap(r.ip(), r.sp(), r.lr(), gp) + tracebacktrap(r.PC(), r.SP(), r.LR(), gp) tracebackothers(gp) dumpregs(r) } @@ -396,7 +380,7 @@ func sigpanic() { } switch gp.sig { - case _EXCEPTION_ACCESS_VIOLATION, _EXCEPTION_IN_PAGE_ERROR: + case windows.EXCEPTION_ACCESS_VIOLATION, windows.EXCEPTION_IN_PAGE_ERROR: if gp.sigcode1 < 0x1000 { panicmem() } @@ -412,15 +396,15 @@ func sigpanic() { print("unexpected fault address ", hex(gp.sigcode1), "\n") } throw("fault") - case _EXCEPTION_INT_DIVIDE_BY_ZERO: + case windows.EXCEPTION_INT_DIVIDE_BY_ZERO: panicdivide() - case _EXCEPTION_INT_OVERFLOW: + case windows.EXCEPTION_INT_OVERFLOW: panicoverflow() - case _EXCEPTION_FLT_DENORMAL_OPERAND, - _EXCEPTION_FLT_DIVIDE_BY_ZERO, - _EXCEPTION_FLT_INEXACT_RESULT, - _EXCEPTION_FLT_OVERFLOW, - _EXCEPTION_FLT_UNDERFLOW: + case windows.EXCEPTION_FLT_DENORMAL_OPERAND, + windows.EXCEPTION_FLT_DIVIDE_BY_ZERO, + windows.EXCEPTION_FLT_INEXACT_RESULT, + windows.EXCEPTION_FLT_OVERFLOW, + windows.EXCEPTION_FLT_UNDERFLOW: panicfloat() } throw("fault") @@ -453,29 +437,28 @@ func crash() { // This provides the expected exit status for the shell. // //go:nosplit -func dieFromException(info *exceptionrecord, r *context) { +func dieFromException(info *windows.ExceptionRecord, r *windows.Context) { if info == nil { gp := getg() if gp.sig != 0 { // Try to reconstruct an exception record from // the exception information stored in gp. - info = &exceptionrecord{ - exceptionaddress: gp.sigpc, - exceptioncode: gp.sig, - numberparameters: 2, + info = &windows.ExceptionRecord{ + ExceptionAddress: gp.sigpc, + ExceptionCode: gp.sig, + NumberParameters: 2, } - info.exceptioninformation[0] = gp.sigcode0 - info.exceptioninformation[1] = gp.sigcode1 + info.ExceptionInformation[0] = gp.sigcode0 + info.ExceptionInformation[1] = gp.sigcode1 } else { // By default, a failing Go application exits with exit code 2. // Use this value when gp does not contain exception info. - info = &exceptionrecord{ - exceptioncode: 2, + info = &windows.ExceptionRecord{ + ExceptionCode: 2, } } } - const FAIL_FAST_GENERATE_EXCEPTION_ADDRESS = 0x1 - stdcall3(_RaiseFailFastException, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(r)), FAIL_FAST_GENERATE_EXCEPTION_ADDRESS) + stdcall(_RaiseFailFastException, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(r)), windows.FAIL_FAST_GENERATE_EXCEPTION_ADDRESS) } // gsignalStack is unused on Windows. diff --git a/src/runtime/signal_windows_386.go b/src/runtime/signal_windows_386.go new file mode 100644 index 00000000000000..1c731290b6b8d3 --- /dev/null +++ b/src/runtime/signal_windows_386.go @@ -0,0 +1,28 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "internal/runtime/syscall/windows" + +func prepareContextForSigResume(c *windows.Context) { + c.Edx = c.Esp + c.Ecx = c.Eip +} + +func dumpregs(r *windows.Context) { + print("eax ", hex(r.Eax), "\n") + print("ebx ", hex(r.Ebx), "\n") + print("ecx ", hex(r.Ecx), "\n") + print("edx ", hex(r.Edx), "\n") + print("edi ", hex(r.Edi), "\n") + print("esi ", hex(r.Esi), "\n") + print("ebp ", hex(r.Ebp), "\n") + print("esp ", hex(r.Esp), "\n") + print("eip ", hex(r.Eip), "\n") + print("eflags ", hex(r.EFlags), "\n") + print("cs ", hex(r.SegCs), "\n") + print("fs ", hex(r.SegFs), "\n") + print("gs ", hex(r.SegGs), "\n") +} diff --git a/src/runtime/signal_windows_amd64.go b/src/runtime/signal_windows_amd64.go new file mode 100644 index 00000000000000..ecb7024548d6ca --- /dev/null +++ b/src/runtime/signal_windows_amd64.go @@ -0,0 +1,36 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "internal/runtime/syscall/windows" + +func prepareContextForSigResume(c *windows.Context) { + c.R8 = c.Rsp + c.R9 = c.Rip +} + +func dumpregs(r *windows.Context) { + print("rax ", hex(r.Rax), "\n") + print("rbx ", hex(r.Rbx), "\n") + print("rcx ", hex(r.Rcx), "\n") + print("rdx ", hex(r.Rdx), "\n") + print("rdi ", hex(r.Rdi), "\n") + print("rsi ", hex(r.Rsi), "\n") + print("rbp ", hex(r.Rbp), "\n") + print("rsp ", hex(r.Rsp), "\n") + print("r8 ", hex(r.R8), "\n") + print("r9 ", hex(r.R9), "\n") + print("r10 ", hex(r.R10), "\n") + print("r11 ", hex(r.R11), "\n") + print("r12 ", hex(r.R12), "\n") + print("r13 ", hex(r.R13), "\n") + print("r14 ", hex(r.R14), "\n") + print("r15 ", hex(r.R15), "\n") + print("rip ", hex(r.Rip), "\n") + print("rflags ", hex(r.EFlags), "\n") + print("cs ", hex(r.SegCs), "\n") + print("fs ", hex(r.SegFs), "\n") + print("gs ", hex(r.SegGs), "\n") +} diff --git a/src/runtime/signal_windows_arm64.go b/src/runtime/signal_windows_arm64.go new file mode 100644 index 00000000000000..78bddb9fb3ca03 --- /dev/null +++ b/src/runtime/signal_windows_arm64.go @@ -0,0 +1,49 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "internal/runtime/syscall/windows" + +func prepareContextForSigResume(c *windows.Context) { + c.X[0] = c.XSp + c.X[1] = c.Pc +} + +func dumpregs(r *windows.Context) { + print("r0 ", hex(r.X[0]), "\n") + print("r1 ", hex(r.X[1]), "\n") + print("r2 ", hex(r.X[2]), "\n") + print("r3 ", hex(r.X[3]), "\n") + print("r4 ", hex(r.X[4]), "\n") + print("r5 ", hex(r.X[5]), "\n") + print("r6 ", hex(r.X[6]), "\n") + print("r7 ", hex(r.X[7]), "\n") + print("r8 ", hex(r.X[8]), "\n") + print("r9 ", hex(r.X[9]), "\n") + print("r10 ", hex(r.X[10]), "\n") + print("r11 ", hex(r.X[11]), "\n") + print("r12 ", hex(r.X[12]), "\n") + print("r13 ", hex(r.X[13]), "\n") + print("r14 ", hex(r.X[14]), "\n") + print("r15 ", hex(r.X[15]), "\n") + print("r16 ", hex(r.X[16]), "\n") + print("r17 ", hex(r.X[17]), "\n") + print("r18 ", hex(r.X[18]), "\n") + print("r19 ", hex(r.X[19]), "\n") + print("r20 ", hex(r.X[20]), "\n") + print("r21 ", hex(r.X[21]), "\n") + print("r22 ", hex(r.X[22]), "\n") + print("r23 ", hex(r.X[23]), "\n") + print("r24 ", hex(r.X[24]), "\n") + print("r25 ", hex(r.X[25]), "\n") + print("r26 ", hex(r.X[26]), "\n") + print("r27 ", hex(r.X[27]), "\n") + print("r28 ", hex(r.X[28]), "\n") + print("r29 ", hex(r.X[29]), "\n") + print("lr ", hex(r.X[30]), "\n") + print("sp ", hex(r.XSp), "\n") + print("pc ", hex(r.Pc), "\n") + print("cpsr ", hex(r.Cpsr), "\n") +} diff --git a/src/runtime/signal_windows_test.go b/src/runtime/signal_windows_test.go index 9318ff9c007880..7a9afcce2239f4 100644 --- a/src/runtime/signal_windows_test.go +++ b/src/runtime/signal_windows_test.go @@ -79,12 +79,7 @@ func TestVectoredHandlerDontCrashOnLibrary(t *testing.T) { if *flagQuick { t.Skip("-quick") } - if runtime.GOARCH == "arm" { - //TODO: remove this skip and update testwinlib/main.c - // once windows/arm supports c-shared buildmode. - // See go.dev/issues/43800. - t.Skip("this test can't run on windows/arm") - } + testenv.MustHaveGoBuild(t) testenv.MustHaveCGO(t) testenv.MustHaveExecPath(t, "gcc") @@ -115,8 +110,8 @@ func TestVectoredHandlerDontCrashOnLibrary(t *testing.T) { t.Fatalf("failure while running executable: %s\n%s", err, out) } var expectedOutput string - if runtime.GOARCH == "arm64" || runtime.GOARCH == "arm" { - // TODO: remove when windows/arm64 and windows/arm support SEH stack unwinding. + if runtime.GOARCH == "arm64" { + // TODO: remove when windows/arm64 support SEH stack unwinding. expectedOutput = "exceptionCount: 1\ncontinueCount: 1\nunhandledCount: 0\n" } else { expectedOutput = "exceptionCount: 1\ncontinueCount: 1\nunhandledCount: 1\n" diff --git a/src/runtime/sizeclasses.go b/src/runtime/sizeclasses.go deleted file mode 100644 index bbcaa9e983fd04..00000000000000 --- a/src/runtime/sizeclasses.go +++ /dev/null @@ -1,99 +0,0 @@ -// Code generated by mksizeclasses.go; DO NOT EDIT. -//go:generate go run mksizeclasses.go - -package runtime - -// class bytes/obj bytes/span objects tail waste max waste min align -// 1 8 8192 1024 0 87.50% 8 -// 2 16 8192 512 0 43.75% 16 -// 3 24 8192 341 8 29.24% 8 -// 4 32 8192 256 0 21.88% 32 -// 5 48 8192 170 32 31.52% 16 -// 6 64 8192 128 0 23.44% 64 -// 7 80 8192 102 32 19.07% 16 -// 8 96 8192 85 32 15.95% 32 -// 9 112 8192 73 16 13.56% 16 -// 10 128 8192 64 0 11.72% 128 -// 11 144 8192 56 128 11.82% 16 -// 12 160 8192 51 32 9.73% 32 -// 13 176 8192 46 96 9.59% 16 -// 14 192 8192 42 128 9.25% 64 -// 15 208 8192 39 80 8.12% 16 -// 16 224 8192 36 128 8.15% 32 -// 17 240 8192 34 32 6.62% 16 -// 18 256 8192 32 0 5.86% 256 -// 19 288 8192 28 128 12.16% 32 -// 20 320 8192 25 192 11.80% 64 -// 21 352 8192 23 96 9.88% 32 -// 22 384 8192 21 128 9.51% 128 -// 23 416 8192 19 288 10.71% 32 -// 24 448 8192 18 128 8.37% 64 -// 25 480 8192 17 32 6.82% 32 -// 26 512 8192 16 0 6.05% 512 -// 27 576 8192 14 128 12.33% 64 -// 28 640 8192 12 512 15.48% 128 -// 29 704 8192 11 448 13.93% 64 -// 30 768 8192 10 512 13.94% 256 -// 31 896 8192 9 128 15.52% 128 -// 32 1024 8192 8 0 12.40% 1024 -// 33 1152 8192 7 128 12.41% 128 -// 34 1280 8192 6 512 15.55% 256 -// 35 1408 16384 11 896 14.00% 128 -// 36 1536 8192 5 512 14.00% 512 -// 37 1792 16384 9 256 15.57% 256 -// 38 2048 8192 4 0 12.45% 2048 -// 39 2304 16384 7 256 12.46% 256 -// 40 2688 8192 3 128 15.59% 128 -// 41 3072 24576 8 0 12.47% 1024 -// 42 3200 16384 5 384 6.22% 128 -// 43 3456 24576 7 384 8.83% 128 -// 44 4096 8192 2 0 15.60% 4096 -// 45 4864 24576 5 256 16.65% 256 -// 46 5376 16384 3 256 10.92% 256 -// 47 6144 24576 4 0 12.48% 2048 -// 48 6528 32768 5 128 6.23% 128 -// 49 6784 40960 6 256 4.36% 128 -// 50 6912 49152 7 768 3.37% 256 -// 51 8192 8192 1 0 15.61% 8192 -// 52 9472 57344 6 512 14.28% 256 -// 53 9728 49152 5 512 3.64% 512 -// 54 10240 40960 4 0 4.99% 2048 -// 55 10880 32768 3 128 6.24% 128 -// 56 12288 24576 2 0 11.45% 4096 -// 57 13568 40960 3 256 9.99% 256 -// 58 14336 57344 4 0 5.35% 2048 -// 59 16384 16384 1 0 12.49% 8192 -// 60 18432 73728 4 0 11.11% 2048 -// 61 19072 57344 3 128 3.57% 128 -// 62 20480 40960 2 0 6.87% 4096 -// 63 21760 65536 3 256 6.25% 256 -// 64 24576 24576 1 0 11.45% 8192 -// 65 27264 81920 3 128 10.00% 128 -// 66 28672 57344 2 0 4.91% 4096 -// 67 32768 32768 1 0 12.50% 8192 - -// alignment bits min obj size -// 8 3 8 -// 16 4 32 -// 32 5 256 -// 64 6 512 -// 128 7 768 -// 4096 12 28672 -// 8192 13 32768 - -const ( - minHeapAlign = 8 - _MaxSmallSize = 32768 - smallSizeDiv = 8 - smallSizeMax = 1024 - largeSizeDiv = 128 - _NumSizeClasses = 68 - _PageShift = 13 - maxObjsPerSpan = 1024 -) - -var class_to_size = [_NumSizeClasses]uint16{0, 8, 16, 24, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 896, 1024, 1152, 1280, 1408, 1536, 1792, 2048, 2304, 2688, 3072, 3200, 3456, 4096, 4864, 5376, 6144, 6528, 6784, 6912, 8192, 9472, 9728, 10240, 10880, 12288, 13568, 14336, 16384, 18432, 19072, 20480, 21760, 24576, 27264, 28672, 32768} -var class_to_allocnpages = [_NumSizeClasses]uint8{0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 3, 2, 3, 1, 3, 2, 3, 4, 5, 6, 1, 7, 6, 5, 4, 3, 5, 7, 2, 9, 7, 5, 8, 3, 10, 7, 4} -var class_to_divmagic = [_NumSizeClasses]uint32{0, ^uint32(0)/8 + 1, ^uint32(0)/16 + 1, ^uint32(0)/24 + 1, ^uint32(0)/32 + 1, ^uint32(0)/48 + 1, ^uint32(0)/64 + 1, ^uint32(0)/80 + 1, ^uint32(0)/96 + 1, ^uint32(0)/112 + 1, ^uint32(0)/128 + 1, ^uint32(0)/144 + 1, ^uint32(0)/160 + 1, ^uint32(0)/176 + 1, ^uint32(0)/192 + 1, ^uint32(0)/208 + 1, ^uint32(0)/224 + 1, ^uint32(0)/240 + 1, ^uint32(0)/256 + 1, ^uint32(0)/288 + 1, ^uint32(0)/320 + 1, ^uint32(0)/352 + 1, ^uint32(0)/384 + 1, ^uint32(0)/416 + 1, ^uint32(0)/448 + 1, ^uint32(0)/480 + 1, ^uint32(0)/512 + 1, ^uint32(0)/576 + 1, ^uint32(0)/640 + 1, ^uint32(0)/704 + 1, ^uint32(0)/768 + 1, ^uint32(0)/896 + 1, ^uint32(0)/1024 + 1, ^uint32(0)/1152 + 1, ^uint32(0)/1280 + 1, ^uint32(0)/1408 + 1, ^uint32(0)/1536 + 1, ^uint32(0)/1792 + 1, ^uint32(0)/2048 + 1, ^uint32(0)/2304 + 1, ^uint32(0)/2688 + 1, ^uint32(0)/3072 + 1, ^uint32(0)/3200 + 1, ^uint32(0)/3456 + 1, ^uint32(0)/4096 + 1, ^uint32(0)/4864 + 1, ^uint32(0)/5376 + 1, ^uint32(0)/6144 + 1, ^uint32(0)/6528 + 1, ^uint32(0)/6784 + 1, ^uint32(0)/6912 + 1, ^uint32(0)/8192 + 1, ^uint32(0)/9472 + 1, ^uint32(0)/9728 + 1, ^uint32(0)/10240 + 1, ^uint32(0)/10880 + 1, ^uint32(0)/12288 + 1, ^uint32(0)/13568 + 1, ^uint32(0)/14336 + 1, ^uint32(0)/16384 + 1, ^uint32(0)/18432 + 1, ^uint32(0)/19072 + 1, ^uint32(0)/20480 + 1, ^uint32(0)/21760 + 1, ^uint32(0)/24576 + 1, ^uint32(0)/27264 + 1, ^uint32(0)/28672 + 1, ^uint32(0)/32768 + 1} -var size_to_class8 = [smallSizeMax/smallSizeDiv + 1]uint8{0, 1, 2, 3, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 23, 23, 23, 23, 24, 24, 24, 24, 25, 25, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27, 27, 27, 28, 28, 28, 28, 28, 28, 28, 28, 29, 29, 29, 29, 29, 29, 29, 29, 30, 30, 30, 30, 30, 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32} -var size_to_class128 = [(_MaxSmallSize-smallSizeMax)/largeSizeDiv + 1]uint8{32, 33, 34, 35, 36, 37, 37, 38, 38, 39, 39, 40, 40, 40, 41, 41, 41, 42, 43, 43, 44, 44, 44, 44, 44, 45, 45, 45, 45, 45, 45, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 48, 48, 48, 49, 49, 50, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 53, 53, 54, 54, 54, 54, 55, 55, 55, 55, 55, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, 58, 58, 58, 58, 58, 58, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 60, 61, 61, 61, 61, 61, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67, 67} diff --git a/src/runtime/sizeof_test.go b/src/runtime/sizeof_test.go index 43aba98dcebcdb..5888177f0ea7a1 100644 --- a/src/runtime/sizeof_test.go +++ b/src/runtime/sizeof_test.go @@ -15,13 +15,18 @@ import ( func TestSizeof(t *testing.T) { const _64bit = unsafe.Sizeof(uintptr(0)) == 8 + const xreg = unsafe.Sizeof(runtime.XRegPerG{}) // Varies per architecture var tests = []struct { val any // type as a value _32bit uintptr // size on 32bit platforms _64bit uintptr // size on 64bit platforms }{ - {runtime.G{}, 272, 432}, // g, but exported for testing - {runtime.Sudog{}, 56, 88}, // sudog, but exported for testing + {runtime.G{}, 280 + xreg, 440 + xreg}, // g, but exported for testing + {runtime.Sudog{}, 64, 104}, // sudog, but exported for testing + } + + if xreg > runtime.PtrSize { + t.Errorf("unsafe.Sizeof(xRegPerG) = %d, want <= %d", xreg, runtime.PtrSize) } for _, tt := range tests { diff --git a/src/runtime/slice.go b/src/runtime/slice.go index ecc2e2921b3bbd..e31d5dccb24b4f 100644 --- a/src/runtime/slice.go +++ b/src/runtime/slice.go @@ -73,7 +73,7 @@ func makeslicecopy(et *_type, tolen int, fromlen int, from unsafe.Pointer) unsaf } if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() pc := abi.FuncPCABIInternal(makeslicecopy) racereadrangepc(from, copymem, callerpc, pc) } @@ -177,7 +177,7 @@ func makeslice64(et *_type, len64, cap64 int64) unsafe.Pointer { func growslice(oldPtr unsafe.Pointer, newLen, oldCap, num int, et *_type) slice { oldLen := newLen - num if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() racereadrangepc(oldPtr, uintptr(oldLen*int(et.Size_)), callerpc, abi.FuncPCABIInternal(growslice)) } if msanenabled { @@ -368,7 +368,7 @@ func slicecopy(toPtr unsafe.Pointer, toLen int, fromPtr unsafe.Pointer, fromLen size := uintptr(n) * width if raceenabled { - callerpc := getcallerpc() + callerpc := sys.GetCallerPC() pc := abi.FuncPCABIInternal(slicecopy) racereadrangepc(fromPtr, size, callerpc, pc) racewriterangepc(toPtr, size, callerpc, pc) @@ -397,5 +397,5 @@ func bytealg_MakeNoZero(len int) []byte { panicmakeslicelen() } cap := roundupsize(uintptr(len), true) - return unsafe.Slice((*byte)(mallocgc(uintptr(cap), nil, false)), cap)[:len] + return unsafe.Slice((*byte)(mallocgc(cap, nil, false)), cap)[:len] } diff --git a/src/runtime/stack.go b/src/runtime/stack.go index 76a14b2498318b..55e97e77afa957 100644 --- a/src/runtime/stack.go +++ b/src/runtime/stack.go @@ -10,6 +10,7 @@ import ( "internal/goarch" "internal/goos" "internal/runtime/atomic" + "internal/runtime/gc" "internal/runtime/sys" "unsafe" ) @@ -69,7 +70,7 @@ const ( // to each stack below the usual guard area for OS-specific // purposes like signal handling. Used on Windows, Plan 9, // and iOS because they do not use a separate stack. - stackSystem = goos.IsWindows*512*goarch.PtrSize + goos.IsPlan9*512 + goos.IsIos*goarch.IsArm64*1024 + stackSystem = goos.IsWindows*4096 + goos.IsPlan9*512 + goos.IsIos*goarch.IsArm64*1024 // The minimum size of stack used by Go code stackMin = 2048 @@ -161,11 +162,11 @@ type stackpoolItem struct { // Global pool of large stack spans. var stackLarge struct { lock mutex - free [heapAddrBits - pageShift]mSpanList // free lists by log_2(s.npages) + free [heapAddrBits - gc.PageShift]mSpanList // free lists by log_2(s.npages) } func stackinit() { - if _StackCacheSize&_PageMask != 0 { + if _StackCacheSize&pageMask != 0 { throw("cache size must be a multiple of page size") } for i := range stackpool { @@ -196,7 +197,7 @@ func stackpoolalloc(order uint8) gclinkptr { lockWithRankMayAcquire(&mheap_.lock, lockRankMheap) if s == nil { // no free stacks. Allocate another span worth. - s = mheap_.allocManual(_StackCacheSize>>_PageShift, spanAllocStack) + s = mheap_.allocManual(_StackCacheSize>>gc.PageShift, spanAllocStack) if s == nil { throw("out of memory") } @@ -210,6 +211,13 @@ func stackpoolalloc(order uint8) gclinkptr { s.elemsize = fixedStack << order for i := uintptr(0); i < _StackCacheSize; i += s.elemsize { x := gclinkptr(s.base() + i) + if valgrindenabled { + // The address of x.ptr() becomes the base of stacks. We need to + // mark it allocated here and in stackfree and stackpoolfree, and free'd in + // stackalloc in order to avoid overlapping allocations and + // uninitialized memory errors in valgrind. + valgrindMalloc(unsafe.Pointer(x.ptr()), unsafe.Sizeof(x.ptr())) + } x.ptr().next = s.manualFreeList s.manualFreeList = x } @@ -350,7 +358,7 @@ func stackalloc(n uint32) stack { if debug.efence != 0 || stackFromSystem != 0 { n = uint32(alignUp(uintptr(n), physPageSize)) - v := sysAlloc(uintptr(n), &memstats.stacks_sys) + v := sysAlloc(uintptr(n), &memstats.stacks_sys, "goroutine stack (system)") if v == nil { throw("out of memory (stackalloc)") } @@ -387,10 +395,16 @@ func stackalloc(n uint32) stack { c.stackcache[order].list = x.ptr().next c.stackcache[order].size -= uintptr(n) } + if valgrindenabled { + // We're about to allocate the stack region starting at x.ptr(). + // To prevent valgrind from complaining about overlapping allocations, + // we need to mark the (previously allocated) memory as free'd. + valgrindFree(unsafe.Pointer(x.ptr())) + } v = unsafe.Pointer(x) } else { var s *mspan - npage := uintptr(n) >> _PageShift + npage := uintptr(n) >> gc.PageShift log2npage := stacklog2(npage) // Try to get a stack from the large stack cache. @@ -431,6 +445,9 @@ func stackalloc(n uint32) stack { if asanenabled { asanunpoison(v, uintptr(n)) } + if valgrindenabled { + valgrindMalloc(v, uintptr(n)) + } if stackDebug >= 1 { print(" allocated ", v, "\n") } @@ -478,6 +495,9 @@ func stackfree(stk stack) { if asanenabled { asanpoison(v, n) } + if valgrindenabled { + valgrindFree(v) + } if n < fixedStack<<_NumStackOrders && n < _StackCacheSize { order := uint8(0) n2 := n @@ -488,6 +508,11 @@ func stackfree(stk stack) { x := gclinkptr(v) if stackNoCache != 0 || gp.m.p == 0 || gp.m.preemptoff != "" { lock(&stackpool[order].item.mu) + if valgrindenabled { + // x.ptr() is the head of the list of free stacks, and will be used + // when allocating a new stack, so it has to be marked allocated. + valgrindMalloc(unsafe.Pointer(x.ptr()), unsafe.Sizeof(x.ptr())) + } stackpoolfree(x, order) unlock(&stackpool[order].item.mu) } else { @@ -495,6 +520,12 @@ func stackfree(stk stack) { if c.stackcache[order].size >= _StackCacheSize { stackcacherelease(c, order) } + if valgrindenabled { + // x.ptr() is the head of the list of free stacks, and will + // be used when allocating a new stack, so it has to be + // marked allocated. + valgrindMalloc(unsafe.Pointer(x.ptr()), unsafe.Sizeof(x.ptr())) + } x.ptr().next = c.stackcache[order].list c.stackcache[order].list = x c.stackcache[order].size += n @@ -582,6 +613,16 @@ func adjustpointer(adjinfo *adjustinfo, vpp unsafe.Pointer) { if stackDebug >= 4 { print(" ", pp, ":", hex(p), "\n") } + if valgrindenabled { + // p is a pointer on a stack, it is inherently initialized, as + // everything on the stack is, but valgrind for _some unknown reason_ + // sometimes thinks it's uninitialized, and flags operations on p below + // as uninitialized. We just initialize it if valgrind thinks its + // uninitialized. + // + // See go.dev/issues/73801. + valgrindMakeMemDefined(unsafe.Pointer(&p), unsafe.Sizeof(&p)) + } if adjinfo.old.lo <= p && p < adjinfo.old.hi { *pp = p + adjinfo.delta if stackDebug >= 3 { @@ -722,22 +763,12 @@ func adjustframe(frame *stkframe, adjinfo *adjustinfo) { // we call into morestack.) continue } - ptrdata := obj.ptrdata() - gcdata := obj.gcdata() - var s *mspan - if obj.useGCProg() { - // See comments in mgcmark.go:scanstack - s = materializeGCProg(ptrdata, gcdata) - gcdata = (*byte)(unsafe.Pointer(s.startAddr)) - } - for i := uintptr(0); i < ptrdata; i += goarch.PtrSize { - if *addb(gcdata, i/(8*goarch.PtrSize))>>(i/goarch.PtrSize&7)&1 != 0 { + ptrBytes, gcData := obj.gcdata() + for i := uintptr(0); i < ptrBytes; i += goarch.PtrSize { + if *addb(gcData, i/(8*goarch.PtrSize))>>(i/goarch.PtrSize&7)&1 != 0 { adjustpointer(adjinfo, unsafe.Pointer(p+i)) } } - if s != nil { - dematerializeGCProg(s) - } } } } @@ -790,7 +821,8 @@ func adjustsudogs(gp *g, adjinfo *adjustinfo) { // the data elements pointed to by a SudoG structure // might be in the stack. for s := gp.waiting; s != nil; s = s.waitlink { - adjustpointer(adjinfo, unsafe.Pointer(&s.elem)) + adjustpointer(adjinfo, unsafe.Pointer(&s.elem.vu)) + adjustpointer(adjinfo, unsafe.Pointer(&s.elem.vp)) } } @@ -803,7 +835,7 @@ func fillstack(stk stack, b byte) { func findsghi(gp *g, stk stack) uintptr { var sghi uintptr for sg := gp.waiting; sg != nil; sg = sg.waitlink { - p := uintptr(sg.elem) + uintptr(sg.c.elemsize) + p := sg.elem.uintptr() + uintptr(sg.c.get().elemsize) if stk.lo <= p && p < stk.hi && p > sghi { sghi = p } @@ -822,7 +854,7 @@ func syncadjustsudogs(gp *g, used uintptr, adjinfo *adjustinfo) uintptr { // Lock channels to prevent concurrent send/receive. var lastc *hchan for sg := gp.waiting; sg != nil; sg = sg.waitlink { - if sg.c != lastc { + if sg.c.get() != lastc { // There is a ranking cycle here between gscan bit and // hchan locks. Normally, we only allow acquiring hchan // locks and then getting a gscan bit. In this case, we @@ -832,9 +864,9 @@ func syncadjustsudogs(gp *g, used uintptr, adjinfo *adjustinfo) uintptr { // suspended. So, we get a special hchan lock rank here // that is lower than gscan, but doesn't allow acquiring // any other locks other than hchan. - lockWithRank(&sg.c.lock, lockRankHchanLeaf) + lockWithRank(&sg.c.get().lock, lockRankHchanLeaf) } - lastc = sg.c + lastc = sg.c.get() } // Adjust sudogs. @@ -854,10 +886,10 @@ func syncadjustsudogs(gp *g, used uintptr, adjinfo *adjustinfo) uintptr { // Unlock channels. lastc = nil for sg := gp.waiting; sg != nil; sg = sg.waitlink { - if sg.c != lastc { - unlock(&sg.c.lock) + if sg.c.get() != lastc { + unlock(&sg.c.get().lock) } - lastc = sg.c + lastc = sg.c.get() } return sgsize @@ -945,6 +977,14 @@ func copystack(gp *g, newsize uintptr) { adjustframe(&u.frame, &adjinfo) } + if valgrindenabled { + if gp.valgrindStackID == 0 { + gp.valgrindStackID = valgrindRegisterStack(unsafe.Pointer(new.lo), unsafe.Pointer(new.hi)) + } else { + valgrindChangeStack(gp.valgrindStackID, unsafe.Pointer(new.lo), unsafe.Pointer(new.hi)) + } + } + // free old stack if stackPoisonCopy != 0 { fillstack(old, 0xfc) @@ -1076,6 +1116,9 @@ func newstack() { shrinkstack(gp) } + // Set a flag indicated that we've been synchronously preempted. + gp.syncSafePoint = true + if gp.preemptStop { preemptPark(gp) // never returns } @@ -1172,15 +1215,18 @@ func isShrinkStackSafe(gp *g) bool { if gp.parkingOnChan.Load() { return false } - // We also can't copy the stack while tracing is enabled, and - // gp is in _Gwaiting solely to make itself available to the GC. + // We also can't copy the stack while a gp is in _Gwaiting solely + // to make itself available to suspendG. + // // In these cases, the G is actually executing on the system - // stack, and the execution tracer may want to take a stack trace - // of the G's stack. Note: it's safe to access gp.waitreason here. - // We're only checking if this is true if we took ownership of the + // stack, and the execution tracer, mutex profiler, etc. may want + // to take a stack trace of the G's stack. + // + // Note: it's safe to access gp.waitreason here. + // We're only calling isShrinkStackSafe if we took ownership of the // G with the _Gscan bit. This prevents the goroutine from transitioning, // which prevents gp.waitreason from changing. - if traceEnabled() && readgstatus(gp)&^_Gscan == _Gwaiting && gp.waitreason.isWaitingForGC() { + if readgstatus(gp)&^_Gscan == _Gwaiting && gp.waitreason.isWaitingForSuspendG() { return false } return true @@ -1216,12 +1262,6 @@ func shrinkstack(gp *g) { if debug.gcshrinkstackoff > 0 { return } - f := findfunc(gp.startpc) - if f.valid() && f.funcID == abi.FuncID_gcBgMarkWorker { - // We're not allowed to shrink the gcBgMarkWorker - // stack (see gcBgMarkWorker for explanation). - return - } oldsize := gp.stack.hi - gp.stack.lo newsize := oldsize / 2 @@ -1288,24 +1328,14 @@ type stackObjectRecord struct { // if non-negative, offset from argp off int32 size int32 - _ptrdata int32 // ptrdata, or -ptrdata is GC prog is used + ptrBytes int32 gcdataoff uint32 // offset to gcdata from moduledata.rodata } -func (r *stackObjectRecord) useGCProg() bool { - return r._ptrdata < 0 -} - -func (r *stackObjectRecord) ptrdata() uintptr { - x := r._ptrdata - if x < 0 { - return uintptr(-x) - } - return uintptr(x) -} - -// gcdata returns pointer map or GC prog of the type. -func (r *stackObjectRecord) gcdata() *byte { +// gcdata returns the number of bytes that contain pointers, and +// a ptr/nonptr bitmask covering those bytes. +// Note that this bitmask might be larger than internal/abi.MaxPtrmaskBytes. +func (r *stackObjectRecord) gcdata() (uintptr, *byte) { ptr := uintptr(unsafe.Pointer(r)) var mod *moduledata for datap := &firstmoduledata; datap != nil; datap = datap.next { @@ -1318,7 +1348,7 @@ func (r *stackObjectRecord) gcdata() *byte { // you may have made a copy of a stackObjectRecord. // You must use the original pointer. res := mod.rodata + uintptr(r.gcdataoff) - return (*byte)(unsafe.Pointer(res)) + return uintptr(r.ptrBytes), (*byte)(unsafe.Pointer(res)) } // This is exported as ABI0 via linkname so obj can call it. @@ -1330,7 +1360,7 @@ func morestackc() { } // startingStackSize is the amount of stack that new goroutines start with. -// It is a power of 2, and between _FixedStack and maxstacksize, inclusive. +// It is a power of 2, and between fixedStack and maxstacksize, inclusive. // startingStackSize is updated every GC by tracking the average size of // stacks scanned during the GC. var startingStackSize uint32 = fixedStack diff --git a/src/runtime/stack_test.go b/src/runtime/stack_test.go index 600e80d8bfdba4..eb8de47e863635 100644 --- a/src/runtime/stack_test.go +++ b/src/runtime/stack_test.go @@ -6,6 +6,7 @@ package runtime_test import ( "fmt" + "internal/asan" "internal/testenv" "reflect" "regexp" @@ -289,7 +290,7 @@ func setBig(p *int, x int, b bigBuf) { func TestDeferPtrsPanic(t *testing.T) { for i := 0; i < 100; i++ { c := make(chan int, 1) - go testDeferPtrsGoexit(c, i) + go testDeferPtrsPanic(c, i) if n := <-c; n != 42 { t.Fatalf("defer's stack references were not adjusted appropriately (i=%d n=%d)", i, n) } @@ -932,6 +933,9 @@ func TestFramePointerAdjust(t *testing.T) { default: t.Skipf("frame pointer is not supported on %s", GOARCH) } + if asan.Enabled { + t.Skip("skipping test: ASAN forces heap allocation") + } output := runTestProg(t, "testprog", "FramePointerAdjust") if output != "" { t.Errorf("output:\n%s\n\nwant no output", output) diff --git a/src/runtime/start_line_amd64_test.go b/src/runtime/start_line_amd64_test.go index 305ed0b126621f..0b4807ad154c39 100644 --- a/src/runtime/start_line_amd64_test.go +++ b/src/runtime/start_line_amd64_test.go @@ -5,7 +5,7 @@ package runtime_test import ( - "runtime/internal/startlinetest" + "internal/runtime/startlinetest" "testing" ) diff --git a/src/runtime/stkframe.go b/src/runtime/stkframe.go index 2bab5a3a0ea8d0..819b7f6c7d9e7c 100644 --- a/src/runtime/stkframe.go +++ b/src/runtime/stkframe.go @@ -264,9 +264,6 @@ var methodValueCallFrameObjs [1]stackObjectRecord // initialized in stackobjecti func stkobjinit() { var abiRegArgsEface any = abi.RegArgs{} abiRegArgsType := efaceOf(&abiRegArgsEface)._type - if abiRegArgsType.Kind_&abi.KindGCProg != 0 { - throw("abiRegArgsType needs GC Prog, update methodValueCallFrameObjs") - } // Set methodValueCallFrameObjs[0].gcdataoff so that // stackObjectRecord.gcdata() will work correctly with it. ptr := uintptr(unsafe.Pointer(&methodValueCallFrameObjs[0])) @@ -283,7 +280,7 @@ func stkobjinit() { methodValueCallFrameObjs[0] = stackObjectRecord{ off: -int32(alignUp(abiRegArgsType.Size_, 8)), // It's always the highest address local. size: int32(abiRegArgsType.Size_), - _ptrdata: int32(abiRegArgsType.PtrBytes), - gcdataoff: uint32(uintptr(unsafe.Pointer(abiRegArgsType.GCData)) - mod.rodata), + ptrBytes: int32(abiRegArgsType.PtrBytes), + gcdataoff: uint32(uintptr(unsafe.Pointer(getGCMask(abiRegArgsType))) - mod.rodata), } } diff --git a/src/runtime/string.go b/src/runtime/string.go index 5bdb25b9db7cf4..3726d9235bfa4b 100644 --- a/src/runtime/string.go +++ b/src/runtime/string.go @@ -8,6 +8,9 @@ import ( "internal/abi" "internal/bytealg" "internal/goarch" + "internal/runtime/math" + "internal/runtime/strconv" + "internal/runtime/sys" "unsafe" ) @@ -50,12 +53,15 @@ func concatstrings(buf *tmpBuf, a []string) string { } s, b := rawstringtmp(buf, l) for _, x := range a { - copy(b, x) - b = b[len(x):] + n := copy(b, x) + b = b[n:] } return s } +// concatstring2 helps make the callsite smaller (compared to concatstrings), +// and we think this is currently more valuable than omitting one call in the +// chain, the same goes for concatstring{3,4,5}. func concatstring2(buf *tmpBuf, a0, a1 string) string { return concatstrings(buf, []string{a0, a1}) } @@ -72,22 +78,64 @@ func concatstring5(buf *tmpBuf, a0, a1, a2, a3, a4 string) string { return concatstrings(buf, []string{a0, a1, a2, a3, a4}) } +// concatbytes implements a Go string concatenation x+y+z+... returning a slice +// of bytes. +// The operands are passed in the slice a. +func concatbytes(buf *tmpBuf, a []string) []byte { + l := 0 + for _, x := range a { + n := len(x) + if l+n < l { + throw("string concatenation too long") + } + l += n + } + if l == 0 { + // This is to match the return type of the non-optimized concatenation. + return []byte{} + } + + var b []byte + if buf != nil && l <= len(buf) { + *buf = tmpBuf{} + b = buf[:l] + } else { + b = rawbyteslice(l) + } + offset := 0 + for _, x := range a { + copy(b[offset:], x) + offset += len(x) + } + + return b +} + +// concatbyte2 helps make the callsite smaller (compared to concatbytes), +// and we think this is currently more valuable than omitting one call in +// the chain, the same goes for concatbyte{3,4,5}. +func concatbyte2(buf *tmpBuf, a0, a1 string) []byte { + return concatbytes(buf, []string{a0, a1}) +} + +func concatbyte3(buf *tmpBuf, a0, a1, a2 string) []byte { + return concatbytes(buf, []string{a0, a1, a2}) +} + +func concatbyte4(buf *tmpBuf, a0, a1, a2, a3 string) []byte { + return concatbytes(buf, []string{a0, a1, a2, a3}) +} + +func concatbyte5(buf *tmpBuf, a0, a1, a2, a3, a4 string) []byte { + return concatbytes(buf, []string{a0, a1, a2, a3, a4}) +} + // slicebytetostring converts a byte slice to a string. // It is inserted by the compiler into generated code. // ptr is a pointer to the first element of the slice; // n is the length of the slice. // Buf is a fixed-size buffer for the result, // it is not nil if the result does not escape. -// -// slicebytetostring should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname slicebytetostring func slicebytetostring(buf *tmpBuf, ptr *byte, n int) string { if n == 0 { // Turns out to be a relatively common case. @@ -98,7 +146,7 @@ func slicebytetostring(buf *tmpBuf, ptr *byte, n int) string { if raceenabled { racereadrangepc(unsafe.Pointer(ptr), uintptr(n), - getcallerpc(), + sys.GetCallerPC(), abi.FuncPCABIInternal(slicebytetostring)) } if msanenabled { @@ -161,7 +209,7 @@ func slicebytetostringtmp(ptr *byte, n int) string { if raceenabled && n > 0 { racereadrangepc(unsafe.Pointer(ptr), uintptr(n), - getcallerpc(), + sys.GetCallerPC(), abi.FuncPCABIInternal(slicebytetostringtmp)) } if msanenabled && n > 0 { @@ -213,7 +261,7 @@ func slicerunetostring(buf *tmpBuf, a []rune) string { if raceenabled && len(a) > 0 { racereadrangepc(unsafe.Pointer(&a[0]), uintptr(len(a))*unsafe.Sizeof(a[0]), - getcallerpc(), + sys.GetCallerPC(), abi.FuncPCABIInternal(slicerunetostring)) } if msanenabled && len(a) > 0 { @@ -351,77 +399,6 @@ func gostringn(p *byte, l int) string { return s } -const ( - maxUint64 = ^uint64(0) - maxInt64 = int64(maxUint64 >> 1) -) - -// atoi64 parses an int64 from a string s. -// The bool result reports whether s is a number -// representable by a value of type int64. -func atoi64(s string) (int64, bool) { - if s == "" { - return 0, false - } - - neg := false - if s[0] == '-' { - neg = true - s = s[1:] - } - - un := uint64(0) - for i := 0; i < len(s); i++ { - c := s[i] - if c < '0' || c > '9' { - return 0, false - } - if un > maxUint64/10 { - // overflow - return 0, false - } - un *= 10 - un1 := un + uint64(c) - '0' - if un1 < un { - // overflow - return 0, false - } - un = un1 - } - - if !neg && un > uint64(maxInt64) { - return 0, false - } - if neg && un > uint64(maxInt64)+1 { - return 0, false - } - - n := int64(un) - if neg { - n = -n - } - - return n, true -} - -// atoi is like atoi64 but for integers -// that fit into an int. -func atoi(s string) (int, bool) { - if n, ok := atoi64(s); n == int64(int(n)) { - return int(n), ok - } - return 0, false -} - -// atoi32 is like atoi but for integers -// that fit into an int32. -func atoi32(s string) (int32, bool) { - if n, ok := atoi64(s); n == int64(int32(n)) { - return int32(n), ok - } - return 0, false -} - // parseByteCount parses a string that represents a count of bytes. // // s must match the following regular expression: @@ -443,7 +420,7 @@ func parseByteCount(s string) (int64, bool) { // Handle the easy non-suffix case. last := s[len(s)-1] if last >= '0' && last <= '9' { - n, ok := atoi64(s) + n, ok := strconv.Atoi64(s) if !ok || n < 0 { return 0, false } @@ -458,7 +435,7 @@ func parseByteCount(s string) (int64, bool) { // The one before that must always be a digit or 'i'. if c := s[len(s)-2]; c >= '0' && c <= '9' { // Trivial 'B' suffix. - n, ok := atoi64(s[:len(s)-1]) + n, ok := strconv.Atoi64(s[:len(s)-1]) if !ok || n < 0 { return 0, false } @@ -489,17 +466,17 @@ func parseByteCount(s string) (int64, bool) { for i := 0; i < power; i++ { m *= 1024 } - n, ok := atoi64(s[:len(s)-3]) + n, ok := strconv.Atoi64(s[:len(s)-3]) if !ok || n < 0 { return 0, false } un := uint64(n) - if un > maxUint64/m { + if un > math.MaxUint64/m { // Overflow. return 0, false } un *= m - if un > uint64(maxInt64) { + if un > uint64(math.MaxInt64) { // Overflow. return 0, false } diff --git a/src/runtime/string_test.go b/src/runtime/string_test.go index cfc0ad7cde8786..942e7ac04a7bf2 100644 --- a/src/runtime/string_test.go +++ b/src/runtime/string_test.go @@ -388,100 +388,6 @@ func TestString2Slice(t *testing.T) { } } -const intSize = 32 << (^uint(0) >> 63) - -type atoi64Test struct { - in string - out int64 - ok bool -} - -var atoi64tests = []atoi64Test{ - {"", 0, false}, - {"0", 0, true}, - {"-0", 0, true}, - {"1", 1, true}, - {"-1", -1, true}, - {"12345", 12345, true}, - {"-12345", -12345, true}, - {"012345", 12345, true}, - {"-012345", -12345, true}, - {"12345x", 0, false}, - {"-12345x", 0, false}, - {"98765432100", 98765432100, true}, - {"-98765432100", -98765432100, true}, - {"20496382327982653440", 0, false}, - {"-20496382327982653440", 0, false}, - {"9223372036854775807", 1<<63 - 1, true}, - {"-9223372036854775807", -(1<<63 - 1), true}, - {"9223372036854775808", 0, false}, - {"-9223372036854775808", -1 << 63, true}, - {"9223372036854775809", 0, false}, - {"-9223372036854775809", 0, false}, -} - -func TestAtoi(t *testing.T) { - switch intSize { - case 32: - for i := range atoi32tests { - test := &atoi32tests[i] - out, ok := runtime.Atoi(test.in) - if test.out != int32(out) || test.ok != ok { - t.Errorf("atoi(%q) = (%v, %v) want (%v, %v)", - test.in, out, ok, test.out, test.ok) - } - } - case 64: - for i := range atoi64tests { - test := &atoi64tests[i] - out, ok := runtime.Atoi(test.in) - if test.out != int64(out) || test.ok != ok { - t.Errorf("atoi(%q) = (%v, %v) want (%v, %v)", - test.in, out, ok, test.out, test.ok) - } - } - } -} - -type atoi32Test struct { - in string - out int32 - ok bool -} - -var atoi32tests = []atoi32Test{ - {"", 0, false}, - {"0", 0, true}, - {"-0", 0, true}, - {"1", 1, true}, - {"-1", -1, true}, - {"12345", 12345, true}, - {"-12345", -12345, true}, - {"012345", 12345, true}, - {"-012345", -12345, true}, - {"12345x", 0, false}, - {"-12345x", 0, false}, - {"987654321", 987654321, true}, - {"-987654321", -987654321, true}, - {"2147483647", 1<<31 - 1, true}, - {"-2147483647", -(1<<31 - 1), true}, - {"2147483648", 0, false}, - {"-2147483648", -1 << 31, true}, - {"2147483649", 0, false}, - {"-2147483649", 0, false}, -} - -func TestAtoi32(t *testing.T) { - for i := range atoi32tests { - test := &atoi32tests[i] - out, ok := runtime.Atoi32(test.in) - if test.out != out || test.ok != ok { - t.Errorf("atoi32(%q) = (%v, %v) want (%v, %v)", - test.in, out, ok, test.out, test.ok) - } - } -} - func TestParseByteCount(t *testing.T) { for _, test := range []struct { in string diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go index 2aeb4774b97218..d5a35d15b25141 100644 --- a/src/runtime/stubs.go +++ b/src/runtime/stubs.go @@ -97,7 +97,6 @@ func badsystemstack() { // Notable members of the hall of shame include: // - github.com/bytedance/sonic // - github.com/chenzhuoyu/iasm -// - github.com/cloudwego/frugal // - github.com/dgraph-io/ristretto // - github.com/outcaste-io/ristretto // @@ -132,7 +131,6 @@ func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) { // Notable members of the hall of shame include: // - github.com/bytedance/sonic // - github.com/cloudwego/dynamicgo -// - github.com/cloudwego/frugal // - github.com/ebitengine/purego // - github.com/tetratelabs/wazero // - github.com/ugorji/go/codec @@ -270,13 +268,22 @@ func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSi // Notable members of the hall of shame include: // - github.com/sagernet/sing-tun // - github.com/slackhq/nebula -// - github.com/tailscale/wireguard-go +// - golang.zx2c4.com/wireguard // // Do not remove or change the type signature. // See go.dev/issue/67401. // //go:linkname procyield -func procyield(cycles uint32) +//go:nosplit +func procyield(cycles uint32) { + if cycles == 0 { + return + } + procyieldAsm(cycles) +} + +// procyieldAsm is the assembly implementation of procyield. +func procyieldAsm(cycles uint32) type neverCallThisFunction struct{} @@ -309,57 +316,6 @@ func goexit(neverCallThisFunction) // data dependency ordering. func publicationBarrier() -// getcallerpc returns the program counter (PC) of its caller's caller. -// getcallersp returns the stack pointer (SP) of its caller's caller. -// The implementation may be a compiler intrinsic; there is not -// necessarily code implementing this on every platform. -// -// For example: -// -// func f(arg1, arg2, arg3 int) { -// pc := getcallerpc() -// sp := getcallersp() -// } -// -// These two lines find the PC and SP immediately following -// the call to f (where f will return). -// -// The call to getcallerpc and getcallersp must be done in the -// frame being asked about. -// -// The result of getcallersp is correct at the time of the return, -// but it may be invalidated by any subsequent call to a function -// that might relocate the stack in order to grow or shrink it. -// A general rule is that the result of getcallersp should be used -// immediately and can only be passed to nosplit functions. - -//go:noescape -func getcallerpc() uintptr - -//go:noescape -func getcallersp() uintptr // implemented as an intrinsic on all platforms - -// getclosureptr returns the pointer to the current closure. -// getclosureptr can only be used in an assignment statement -// at the entry of a function. Moreover, go:nosplit directive -// must be specified at the declaration of caller function, -// so that the function prolog does not clobber the closure register. -// for example: -// -// //go:nosplit -// func f(arg1, arg2, arg3 int) { -// dx := getclosureptr() -// } -// -// The compiler rewrites calls to this function into instructions that fetch the -// pointer from a well-known register (DX on x86 architecture, etc.) directly. -// -// WARNING: PGO-based devirtualization cannot detect that caller of -// getclosureptr require closure context, and thus must maintain a list of -// these functions, which is in -// cmd/compile/internal/devirtualize/pgo.maybeDevirtualizeFunctionCall. -func getclosureptr() uintptr - //go:noescape func asmcgocall(fn, arg unsafe.Pointer) int32 @@ -368,23 +324,17 @@ func morestack() // morestack_noctxt should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: -// - github.com/cloudwego/frugal +// - github.com/bytedance/sonic // // Do not remove or change the type signature. -// See go.dev/issue/67401. +// See go.dev/issues/67401. +// See go.dev/issues/71672. // //go:linkname morestack_noctxt func morestack_noctxt() func rt0_go() -// return0 is a stub used to return 0 from deferproc. -// It is called at the very end of deferproc to signal -// the calling Go function that it should not jump -// to deferreturn. -// in asm_*.s -func return0() - // in asm_*.s // not called directly; definitions here supply type information for traceback. // These must have the same signature (arg pointer map) as reflectcall. @@ -433,6 +383,8 @@ func alignDown(n, a uintptr) uintptr { } // divRoundUp returns ceil(n / a). +// +//go:nosplit func divRoundUp(n, a uintptr) uintptr { // a is generally a power of two. This will get inlined and // the compiler will optimize the division. @@ -465,7 +417,6 @@ func gcWriteBarrier1() // but widely used packages access it using linkname. // Notable members of the hall of shame include: // - github.com/bytedance/sonic -// - github.com/cloudwego/frugal // // Do not remove or change the type signature. // See go.dev/issue/67401. diff --git a/src/runtime/stubs_nonwasm.go b/src/runtime/stubs_nonwasm.go new file mode 100644 index 00000000000000..fa4058bcccb3de --- /dev/null +++ b/src/runtime/stubs_nonwasm.go @@ -0,0 +1,10 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !wasm + +package runtime + +// pause is only used on wasm. +func pause(newsp uintptr) { panic("unreachable") } diff --git a/src/runtime/stubs_wasm.go b/src/runtime/stubs_wasm.go new file mode 100644 index 00000000000000..fafc923b7668cd --- /dev/null +++ b/src/runtime/stubs_wasm.go @@ -0,0 +1,16 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +// pause sets SP to newsp and pauses the execution of Go's WebAssembly +// code until an event is triggered, or call back into Go. +// +// Note: the epilogue of pause pops 8 bytes from the stack, so when +// returning to the host, the SP is newsp+8. +// If we want to set the SP such that when it calls back into Go, the +// Go function appears to be called from pause's caller's caller, then +// call pause with newsp = internal/runtime/sys.GetCallerSP()-16 (another 8 is +// the return PC pushed to the stack). +func pause(newsp uintptr) diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index f889d669926695..62ad8d13611150 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -49,7 +49,8 @@ type Frame struct { // File and Line are the file name and line number of the // location in this frame. For non-leaf frames, this will be // the location of a call. These may be the empty string and - // zero, respectively, if not known. + // zero, respectively, if not known. The file name uses + // forward slashes, even on Windows. File string Line int @@ -107,7 +108,7 @@ func (ci *Frames) Next() (frame Frame, more bool) { } funcInfo := findfunc(pc) if !funcInfo.valid() { - if cgoSymbolizer != nil { + if cgoSymbolizerAvailable() { // Pre-expand cgo frames. We could do this // incrementally, too, but there's no way to // avoid allocation in this case anyway. @@ -117,11 +118,16 @@ func (ci *Frames) Next() (frame Frame, more bool) { } f := funcInfo._Func() entry := f.Entry() + // We store the pc of the start of the instruction following + // the instruction in question (the call or the inline mark). + // This is done for historical reasons, and to make FuncForPC + // work correctly for entries in the result of runtime.Callers. + // Decrement to get back to the instruction we care about. + // + // It is not possible to get pc == entry from runtime.Callers, + // but if the caller does provide one, provide best-effort + // results by avoiding backing out of the function entirely. if pc > entry { - // We store the pc of the start of the instruction following - // the instruction in question (the call or the inline mark). - // This is done for historical reasons, and to make FuncForPC - // work correctly for entries in the result of runtime.Callers. pc-- } // It's important that interpret pc non-strictly as cgoTraceback may @@ -289,6 +295,8 @@ func runtime_expandFinalInlineFrame(stk []uintptr) []uintptr { // expandCgoFrames expands frame information for pc, known to be // a non-Go function, using the cgoSymbolizer hook. expandCgoFrames // returns nil if pc could not be expanded. +// +// Preconditions: cgoSymbolizerAvailable returns true. func expandCgoFrames(pc uintptr) []Frame { arg := cgoSymbolizerArg{pc: pc} callCgoSymbolizer(&arg) @@ -462,15 +470,28 @@ type modulehash struct { // To make sure the map isn't collected, we keep a second reference here. var pinnedTypemaps []map[typeOff]*_type +// aixStaticDataBase (used only on AIX) holds the unrelocated address +// of the data section, set by the linker. +// +// On AIX, an R_ADDR relocation from an RODATA symbol to a DATA symbol +// does not work, as the dynamic loader can change the address of the +// data section, and it is not possible to apply a dynamic relocation +// to RODATA. In order to get the correct address, we need to apply +// the delta between unrelocated and relocated data section addresses. +// aixStaticDataBase is the unrelocated address, and moduledata.data is +// the relocated one. +var aixStaticDataBase uintptr // linker symbol + var firstmoduledata moduledata // linker symbol // lastmoduledatap should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: -// - github.com/cloudwego/frugal +// - github.com/bytedance/sonic // // Do not remove or change the type signature. -// See go.dev/issue/67401. +// See go.dev/issues/67401. +// See go.dev/issues/71672. // //go:linkname lastmoduledatap var lastmoduledatap *moduledata // linker symbol @@ -586,10 +607,11 @@ const debugPcln = false // moduledataverify1 should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: -// - github.com/cloudwego/frugal +// - github.com/bytedance/sonic // // Do not remove or change the type signature. -// See go.dev/issue/67401. +// See go.dev/issues/67401. +// See go.dev/issues/71672. // //go:linkname moduledataverify1 func moduledataverify1(datap *moduledata) { @@ -627,8 +649,15 @@ func moduledataverify1(datap *moduledata) { min := datap.textAddr(datap.ftab[0].entryoff) max := datap.textAddr(datap.ftab[nftab].entryoff) - if datap.minpc != min || datap.maxpc != max { - println("minpc=", hex(datap.minpc), "min=", hex(min), "maxpc=", hex(datap.maxpc), "max=", hex(max)) + minpc := datap.minpc + maxpc := datap.maxpc + if GOARCH == "wasm" { + // On Wasm, the func table contains the function index, whereas + // the "PC" is function index << 16 + block index. + maxpc = alignUp(maxpc, 1<<16) // round up for end PC + } + if minpc != min || maxpc != max { + println("minpc=", hex(minpc), "min=", hex(min), "maxpc=", hex(maxpc), "max=", hex(max)) throw("minpc or maxpc invalid") } @@ -674,6 +703,11 @@ func (md *moduledata) textAddr(off32 uint32) uintptr { throw("runtime: text offset out of range") } } + if GOARCH == "wasm" { + // On Wasm, a text offset (e.g. in the method table) is function index, whereas + // the "PC" is function index << 16 + block index. + res <<= 16 + } return res } @@ -684,8 +718,17 @@ func (md *moduledata) textAddr(off32 uint32) uintptr { // //go:nosplit func (md *moduledata) textOff(pc uintptr) (uint32, bool) { - res := uint32(pc - md.text) + off := pc - md.text + if GOARCH == "wasm" { + // On Wasm, the func table contains the function index, whereas + // the "PC" is function index << 16 + block index. + off >>= 16 + } + res := uint32(off) if len(md.textsectmap) > 1 { + if GOARCH == "wasm" { + fatal("unexpected multiple text sections on Wasm") + } for i, sect := range md.textsectmap { if sect.baseaddr > pc { // pc is not in any section. @@ -713,22 +756,24 @@ func (md *moduledata) funcName(nameOff int32) string { return gostringnocopy(&md.funcnametab[nameOff]) } -// FuncForPC returns a *[Func] describing the function that contains the -// given program counter address, or else nil. -// -// If pc represents multiple functions because of inlining, it returns -// the *Func describing the innermost function, but with an entry of -// the outermost function. -// -// For completely unclear reasons, even though they can import runtime, -// some widely used packages access this using linkname. +// Despite being an exported symbol, +// FuncForPC is linknamed by widely used packages. // Notable members of the hall of shame include: // - gitee.com/quant1x/gox // // Do not remove or change the type signature. // See go.dev/issue/67401. // +// Note that this comment is not part of the doc comment. +// //go:linkname FuncForPC + +// FuncForPC returns a *[Func] describing the function that contains the +// given program counter address, or else nil. +// +// If pc represents multiple functions because of inlining, it returns +// the *Func describing the innermost function, but with an entry of +// the outermost function. func FuncForPC(pc uintptr) *Func { f := findfunc(pc) if !f.valid() { @@ -862,7 +907,6 @@ func badFuncInfoEntry(funcInfo) uintptr // findfunc should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: -// - github.com/cloudwego/frugal // - github.com/phuslu/log // // Do not remove or change the type signature. @@ -883,6 +927,11 @@ func findfunc(pc uintptr) funcInfo { } x := uintptr(pcOff) + datap.text - datap.minpc // TODO: are datap.text and datap.minpc always equal? + if GOARCH == "wasm" { + // On Wasm, pcOff is the function index, whereas + // the "PC" is function index << 16 + block index. + x = uintptr(pcOff)<<16 + datap.text - datap.minpc + } b := x / abi.FuncTabBucketSize i := x % abi.FuncTabBucketSize / (abi.FuncTabBucketSize / nsub) @@ -960,6 +1009,9 @@ func pcvalue(f funcInfo, off uint32, targetpc uintptr, strict bool) (int32, uint // matches the cached contents. const debugCheckCache = false + // If true, skip checking the cache entirely. + const skipCache = false + if off == 0 { return -1, 0 } @@ -970,7 +1022,7 @@ func pcvalue(f funcInfo, off uint32, targetpc uintptr, strict bool) (int32, uint var checkVal int32 var checkPC uintptr ck := pcvalueCacheKey(targetpc) - { + if !skipCache { mp := acquirem() cache := &mp.pcvalueCache // The cache can be used by the signal handler on this M. Avoid @@ -1196,16 +1248,6 @@ func pcdatavalue1(f funcInfo, table uint32, targetpc uintptr, strict bool) int32 } // Like pcdatavalue, but also return the start PC of this PCData value. -// -// pcdatavalue2 should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname pcdatavalue2 func pcdatavalue2(f funcInfo, table uint32, targetpc uintptr) (int32, uintptr) { if table >= f.npcdata { return -1, 0 @@ -1234,16 +1276,6 @@ func funcdata(f funcInfo, i uint8) unsafe.Pointer { } // step advances to the next pc, value pair in the encoded table. -// -// step should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname step func step(p []byte, pc *uintptr, val *int32, first bool) (newp []byte, ok bool) { // For both uvdelta and pcdelta, the common case (~70%) // is that they are a single byte. If so, avoid calling readvarint. @@ -1289,15 +1321,6 @@ type stackmap struct { bytedata [1]byte // bitmaps, each starting on a byte boundary } -// stackmapdata should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname stackmapdata //go:nowritebarrier func stackmapdata(stkmap *stackmap, n int32) bitvector { // Check this invariant only when stackDebug is on at all. diff --git a/src/runtime/synctest.go b/src/runtime/synctest.go new file mode 100644 index 00000000000000..8fdf437acbbac1 --- /dev/null +++ b/src/runtime/synctest.go @@ -0,0 +1,454 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "internal/runtime/atomic" + "internal/runtime/sys" + "unsafe" +) + +// A synctestBubble is a set of goroutines started by synctest.Run. +type synctestBubble struct { + mu mutex + timers timers + id uint64 // unique id + now int64 // current fake time + root *g // caller of synctest.Run + waiter *g // caller of synctest.Wait + main *g // goroutine started by synctest.Run + waiting bool // true if a goroutine is calling synctest.Wait + done bool // true if main has exited + + // The bubble is active (not blocked) so long as running > 0 || active > 0. + // + // running is the number of goroutines which are not "durably blocked": + // Goroutines which are either running, runnable, or non-durably blocked + // (for example, blocked in a syscall). + // + // active is used to keep the bubble from becoming blocked, + // even if all goroutines in the bubble are blocked. + // For example, park_m can choose to immediately unpark a goroutine after parking it. + // It increments the active count to keep the bubble active until it has determined + // that the park operation has completed. + total int // total goroutines + running int // non-blocked goroutines + active int // other sources of activity +} + +// changegstatus is called when the non-lock status of a g changes. +// It is never called with a Gscanstatus. +func (bubble *synctestBubble) changegstatus(gp *g, oldval, newval uint32) { + // Determine whether this change in status affects the idleness of the bubble. + // If this isn't a goroutine starting, stopping, durably blocking, + // or waking up after durably blocking, then return immediately without + // locking bubble.mu. + // + // For example, stack growth (newstack) will changegstatus + // from _Grunning to _Gcopystack. This is uninteresting to synctest, + // but if stack growth occurs while bubble.mu is held, we must not recursively lock. + totalDelta := 0 + wasRunning := true + switch oldval { + case _Gdead, _Gdeadextra: + wasRunning = false + totalDelta++ + case _Gwaiting: + if gp.waitreason.isIdleInSynctest() { + wasRunning = false + } + } + isRunning := true + switch newval { + case _Gdead, _Gdeadextra: + isRunning = false + totalDelta-- + if gp == bubble.main { + bubble.done = true + } + case _Gwaiting: + if gp.waitreason.isIdleInSynctest() { + isRunning = false + } + } + // It's possible for wasRunning == isRunning while totalDelta != 0; + // for example, if a new goroutine is created in a non-running state. + if wasRunning == isRunning && totalDelta == 0 { + return + } + + lock(&bubble.mu) + bubble.total += totalDelta + if wasRunning != isRunning { + if isRunning { + bubble.running++ + } else { + bubble.running-- + if raceenabled && newval != _Gdead && newval != _Gdeadextra { + // Record that this goroutine parking happens before + // any subsequent Wait. + racereleasemergeg(gp, bubble.raceaddr()) + } + } + } + if bubble.total < 0 { + fatal("total < 0") + } + if bubble.running < 0 { + fatal("running < 0") + } + wake := bubble.maybeWakeLocked() + unlock(&bubble.mu) + if wake != nil { + goready(wake, 0) + } +} + +// incActive increments the active-count for the bubble. +// A bubble does not become durably blocked while the active-count is non-zero. +func (bubble *synctestBubble) incActive() { + lock(&bubble.mu) + bubble.active++ + unlock(&bubble.mu) +} + +// decActive decrements the active-count for the bubble. +func (bubble *synctestBubble) decActive() { + lock(&bubble.mu) + bubble.active-- + if bubble.active < 0 { + throw("active < 0") + } + wake := bubble.maybeWakeLocked() + unlock(&bubble.mu) + if wake != nil { + goready(wake, 0) + } +} + +// maybeWakeLocked returns a g to wake if the bubble is durably blocked. +func (bubble *synctestBubble) maybeWakeLocked() *g { + if bubble.running > 0 || bubble.active > 0 { + return nil + } + // Increment the bubble active count, since we've determined to wake something. + // The woken goroutine will decrement the count. + // We can't just call goready and let it increment bubble.running, + // since we can't call goready with bubble.mu held. + // + // Incrementing the active count here is only necessary if something has gone wrong, + // and a goroutine that we considered durably blocked wakes up unexpectedly. + // Two wakes happening at the same time leads to very confusing failure modes, + // so we take steps to avoid it happening. + bubble.active++ + next := bubble.timers.wakeTime() + if next > 0 && next <= bubble.now { + // A timer is scheduled to fire. Wake the root goroutine to handle it. + return bubble.root + } + if gp := bubble.waiter; gp != nil { + // A goroutine is blocked in Wait. Wake it. + return gp + } + // All goroutines in the bubble are durably blocked, and nothing has called Wait. + // Wake the root goroutine. + return bubble.root +} + +func (bubble *synctestBubble) raceaddr() unsafe.Pointer { + // Address used to record happens-before relationships created by the bubble. + // + // Wait creates a happens-before relationship between itself and + // the blocking operations which caused other goroutines in the bubble to park. + return unsafe.Pointer(bubble) +} + +var bubbleGen atomic.Uint64 // bubble ID counter + +//go:linkname synctestRun internal/synctest.Run +func synctestRun(f func()) { + if debug.asynctimerchan.Load() != 0 { + panic("synctest.Run not supported with asynctimerchan!=0") + } + + gp := getg() + if gp.bubble != nil { + panic("synctest.Run called from within a synctest bubble") + } + bubble := &synctestBubble{ + id: bubbleGen.Add(1), + total: 1, + running: 1, + root: gp, + } + const synctestBaseTime = 946684800000000000 // midnight UTC 2000-01-01 + bubble.now = synctestBaseTime + lockInit(&bubble.mu, lockRankSynctest) + lockInit(&bubble.timers.mu, lockRankTimers) + + gp.bubble = bubble + defer func() { + gp.bubble = nil + }() + + // This is newproc, but also records the new g in bubble.main. + pc := sys.GetCallerPC() + systemstack(func() { + fv := *(**funcval)(unsafe.Pointer(&f)) + bubble.main = newproc1(fv, gp, pc, false, waitReasonZero) + pp := getg().m.p.ptr() + runqput(pp, bubble.main, true) + wakep() + }) + + lock(&bubble.mu) + bubble.active++ + for { + unlock(&bubble.mu) + systemstack(func() { + // Clear gp.m.curg while running timers, + // so timer goroutines inherit their child race context from g0. + curg := gp.m.curg + gp.m.curg = nil + gp.bubble.timers.check(bubble.now, bubble) + gp.m.curg = curg + }) + gopark(synctestidle_c, nil, waitReasonSynctestRun, traceBlockSynctest, 0) + lock(&bubble.mu) + if bubble.active < 0 { + throw("active < 0") + } + next := bubble.timers.wakeTime() + if next == 0 { + break + } + if next < bubble.now { + throw("time went backwards") + } + if bubble.done { + // Time stops once the bubble's main goroutine has exited. + break + } + bubble.now = next + } + + total := bubble.total + unlock(&bubble.mu) + if raceenabled { + // Establish a happens-before relationship between bubbled goroutines exiting + // and Run returning. + raceacquireg(gp, gp.bubble.raceaddr()) + } + if total != 1 { + var reason string + if bubble.done { + reason = "deadlock: main bubble goroutine has exited but blocked goroutines remain" + } else { + reason = "deadlock: all goroutines in bubble are blocked" + } + panic(synctestDeadlockError{reason: reason, bubble: bubble}) + } + if gp.timer != nil && gp.timer.isFake { + // Verify that we haven't marked this goroutine's sleep timer as fake. + // This could happen if something in Run were to call timeSleep. + throw("synctest root goroutine has a fake timer") + } +} + +type synctestDeadlockError struct { + reason string + bubble *synctestBubble +} + +func (e synctestDeadlockError) Error() string { + return e.reason +} + +func synctestidle_c(gp *g, _ unsafe.Pointer) bool { + lock(&gp.bubble.mu) + canIdle := true + if gp.bubble.running == 0 && gp.bubble.active == 1 { + // All goroutines in the bubble have blocked or exited. + canIdle = false + } else { + gp.bubble.active-- + } + unlock(&gp.bubble.mu) + return canIdle +} + +//go:linkname synctestWait internal/synctest.Wait +func synctestWait() { + gp := getg() + if gp.bubble == nil { + panic("goroutine is not in a bubble") + } + lock(&gp.bubble.mu) + // We use a bubble.waiting bool to detect simultaneous calls to Wait rather than + // checking to see if bubble.waiter is non-nil. This avoids a race between unlocking + // bubble.mu and setting bubble.waiter while parking. + if gp.bubble.waiting { + unlock(&gp.bubble.mu) + panic("wait already in progress") + } + gp.bubble.waiting = true + unlock(&gp.bubble.mu) + gopark(synctestwait_c, nil, waitReasonSynctestWait, traceBlockSynctest, 0) + + lock(&gp.bubble.mu) + gp.bubble.active-- + if gp.bubble.active < 0 { + throw("active < 0") + } + gp.bubble.waiter = nil + gp.bubble.waiting = false + unlock(&gp.bubble.mu) + + // Establish a happens-before relationship on the activity of the now-blocked + // goroutines in the bubble. + if raceenabled { + raceacquireg(gp, gp.bubble.raceaddr()) + } +} + +func synctestwait_c(gp *g, _ unsafe.Pointer) bool { + lock(&gp.bubble.mu) + if gp.bubble.running == 0 && gp.bubble.active == 0 { + // This shouldn't be possible, since gopark increments active during unlockf. + throw("running == 0 && active == 0") + } + gp.bubble.waiter = gp + unlock(&gp.bubble.mu) + return true +} + +//go:linkname synctest_isInBubble internal/synctest.IsInBubble +func synctest_isInBubble() bool { + return getg().bubble != nil +} + +//go:linkname synctest_acquire internal/synctest.acquire +func synctest_acquire() any { + if bubble := getg().bubble; bubble != nil { + bubble.incActive() + return bubble + } + return nil +} + +//go:linkname synctest_release internal/synctest.release +func synctest_release(bubble any) { + bubble.(*synctestBubble).decActive() +} + +//go:linkname synctest_inBubble internal/synctest.inBubble +func synctest_inBubble(bubble any, f func()) { + gp := getg() + if gp.bubble != nil { + panic("goroutine is already bubbled") + } + gp.bubble = bubble.(*synctestBubble) + defer func() { + gp.bubble = nil + }() + f() +} + +// specialBubble is a special used to associate objects with bubbles. +type specialBubble struct { + _ sys.NotInHeap + special special + bubbleid uint64 +} + +// Keep these in sync with internal/synctest. +const ( + bubbleAssocUnbubbled = iota // not associated with any bubble + bubbleAssocCurrentBubble // associated with the current bubble + bubbleAssocOtherBubble // associated with a different bubble +) + +// getOrSetBubbleSpecial checks the special record for p's bubble membership. +// +// If add is true and p is not associated with any bubble, +// it adds a special record for p associating it with bubbleid. +// +// It returns ok==true if p is associated with bubbleid +// (including if a new association was added), +// and ok==false if not. +func getOrSetBubbleSpecial(p unsafe.Pointer, bubbleid uint64, add bool) (assoc int) { + span := spanOfHeap(uintptr(p)) + if span == nil { + // This is probably a package var. + // We can't attach a special to it, so always consider it unbubbled. + return bubbleAssocUnbubbled + } + + // Ensure that the span is swept. + // Sweeping accesses the specials list w/o locks, so we have + // to synchronize with it. And it's just much safer. + mp := acquirem() + span.ensureSwept() + + offset := uintptr(p) - span.base() + + lock(&span.speciallock) + + // Find splice point, check for existing record. + iter, exists := span.specialFindSplicePoint(offset, _KindSpecialBubble) + if exists { + // p is already associated with a bubble. + // Return true iff it's the same bubble. + s := (*specialBubble)((unsafe.Pointer)(*iter)) + if s.bubbleid == bubbleid { + assoc = bubbleAssocCurrentBubble + } else { + assoc = bubbleAssocOtherBubble + } + } else if add { + // p is not associated with a bubble, + // and we've been asked to add an association. + lock(&mheap_.speciallock) + s := (*specialBubble)(mheap_.specialBubbleAlloc.alloc()) + unlock(&mheap_.speciallock) + s.bubbleid = bubbleid + s.special.kind = _KindSpecialBubble + s.special.offset = offset + s.special.next = *iter + *iter = (*special)(unsafe.Pointer(s)) + spanHasSpecials(span) + assoc = bubbleAssocCurrentBubble + } else { + // p is not associated with a bubble. + assoc = bubbleAssocUnbubbled + } + + unlock(&span.speciallock) + releasem(mp) + + return assoc +} + +// synctest_associate associates p with the current bubble. +// It returns false if p is already associated with a different bubble. +// +//go:linkname synctest_associate internal/synctest.associate +func synctest_associate(p unsafe.Pointer) int { + return getOrSetBubbleSpecial(p, getg().bubble.id, true) +} + +// synctest_disassociate disassociates p from its bubble. +// +//go:linkname synctest_disassociate internal/synctest.disassociate +func synctest_disassociate(p unsafe.Pointer) { + removespecial(p, _KindSpecialBubble) +} + +// synctest_isAssociated reports whether p is associated with the current bubble. +// +//go:linkname synctest_isAssociated internal/synctest.isAssociated +func synctest_isAssociated(p unsafe.Pointer) bool { + return getOrSetBubbleSpecial(p, getg().bubble.id, false) == bubbleAssocCurrentBubble +} diff --git a/src/runtime/synctest_test.go b/src/runtime/synctest_test.go new file mode 100644 index 00000000000000..1059d629d3b6f7 --- /dev/null +++ b/src/runtime/synctest_test.go @@ -0,0 +1,29 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "internal/synctest" + "runtime" + "testing" +) + +func TestSynctest(t *testing.T) { + output := runTestProg(t, "testsynctest", "") + want := "success\n" + if output != want { + t.Fatalf("output:\n%s\n\nwanted:\n%s", output, want) + } +} + +// TestSynctestAssocConsts verifies that constants defined +// in both runtime and internal/synctest match. +func TestSynctestAssocConsts(t *testing.T) { + if runtime.BubbleAssocUnbubbled != synctest.Unbubbled || + runtime.BubbleAssocCurrentBubble != synctest.CurrentBubble || + runtime.BubbleAssocOtherBubble != synctest.OtherBubble { + t.Fatal("mismatch: runtime.BubbleAssoc? != synctest.*") + } +} diff --git a/src/runtime/sys_aix_ppc64.s b/src/runtime/sys_aix_ppc64.s index 66081977b14cae..a0ef7e111a05d0 100644 --- a/src/runtime/sys_aix_ppc64.s +++ b/src/runtime/sys_aix_ppc64.s @@ -130,15 +130,15 @@ TEXT sigtramp<>(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 // Save m->libcall. We need to do this because we // might get interrupted by a signal in runtime·asmcgocall. - MOVD (m_libcall+libcall_fn)(R6), R7 + MOVD (m_mOS+mOS_libcall+libcall_fn)(R6), R7 MOVD R7, 96(R1) - MOVD (m_libcall+libcall_args)(R6), R7 + MOVD (m_mOS+mOS_libcall+libcall_args)(R6), R7 MOVD R7, 104(R1) - MOVD (m_libcall+libcall_n)(R6), R7 + MOVD (m_mOS+mOS_libcall+libcall_n)(R6), R7 MOVD R7, 112(R1) - MOVD (m_libcall+libcall_r1)(R6), R7 + MOVD (m_mOS+mOS_libcall+libcall_r1)(R6), R7 MOVD R7, 120(R1) - MOVD (m_libcall+libcall_r2)(R6), R7 + MOVD (m_mOS+mOS_libcall+libcall_r2)(R6), R7 MOVD R7, 128(R1) // save errno, it might be EINTR; stuff we do here might reset it. @@ -162,15 +162,15 @@ sigtramp: // restore libcall MOVD 96(R1), R7 - MOVD R7, (m_libcall+libcall_fn)(R6) + MOVD R7, (m_mOS+mOS_libcall+libcall_fn)(R6) MOVD 104(R1), R7 - MOVD R7, (m_libcall+libcall_args)(R6) + MOVD R7, (m_mOS+mOS_libcall+libcall_args)(R6) MOVD 112(R1), R7 - MOVD R7, (m_libcall+libcall_n)(R6) + MOVD R7, (m_mOS+mOS_libcall+libcall_n)(R6) MOVD 120(R1), R7 - MOVD R7, (m_libcall+libcall_r1)(R6) + MOVD R7, (m_mOS+mOS_libcall+libcall_r1)(R6) MOVD 128(R1), R7 - MOVD R7, (m_libcall+libcall_r2)(R6) + MOVD R7, (m_mOS+mOS_libcall+libcall_r2)(R6) // restore errno MOVD (m_mOS+mOS_perrno)(R6), R7 diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go index 1e4b2ac79efdec..48ad5afd8a009d 100644 --- a/src/runtime/sys_darwin.go +++ b/src/runtime/sys_darwin.go @@ -10,124 +10,65 @@ import ( "unsafe" ) -// The X versions of syscall expect the libc call to return a 64-bit result. -// Otherwise (the non-X version) expects a 32-bit result. -// This distinction is required because an error is indicated by returning -1, -// and we need to know whether to check 32 or 64 bits of the result. -// (Some libc functions that return 32 bits put junk in the upper 32 bits of AX.) - -// golang.org/x/sys linknames syscall_syscall -// (in addition to standard package syscall). -// Do not remove or change the type signature. -// -//go:linkname syscall_syscall syscall.syscall -//go:nosplit -func syscall_syscall(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { - args := struct{ fn, a1, a2, a3, r1, r2, err uintptr }{fn, a1, a2, a3, r1, r2, err} - entersyscall() - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall)), unsafe.Pointer(&args)) - exitsyscall() - return args.r1, args.r2, args.err -} -func syscall() - -//go:linkname syscall_syscallX syscall.syscallX -//go:nosplit -func syscall_syscallX(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { - args := struct{ fn, a1, a2, a3, r1, r2, err uintptr }{fn, a1, a2, a3, r1, r2, err} - entersyscall() - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscallX)), unsafe.Pointer(&args)) - exitsyscall() - return args.r1, args.r2, args.err -} -func syscallX() - -// golang.org/x/sys linknames syscall.syscall6 -// (in addition to standard package syscall). -// Do not remove or change the type signature. -// -// syscall.syscall6 is meant for package syscall (and x/sys), -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/tetratelabs/wazero -// -// See go.dev/issue/67401. -// -//go:linkname syscall_syscall6 syscall.syscall6 -//go:nosplit -func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { - args := struct{ fn, a1, a2, a3, a4, a5, a6, r1, r2, err uintptr }{fn, a1, a2, a3, a4, a5, a6, r1, r2, err} - entersyscall() - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall6)), unsafe.Pointer(&args)) - exitsyscall() - return args.r1, args.r2, args.err -} -func syscall6() +func libc_error_trampoline() -// golang.org/x/sys linknames syscall.syscall9 -// (in addition to standard package syscall). -// Do not remove or change the type signature. +// libc_error_addr puts the libc error +// address into addr. // -//go:linkname syscall_syscall9 syscall.syscall9 //go:nosplit //go:cgo_unsafe_args -func syscall_syscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2, err uintptr) { - entersyscall() - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall9)), unsafe.Pointer(&fn)) - exitsyscall() - return +func libc_error_addr(addr **int32) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(libc_error_trampoline)), unsafe.Pointer(&addr)) } -func syscall9() -//go:linkname syscall_syscall6X syscall.syscall6X -//go:nosplit -func syscall_syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { - args := struct{ fn, a1, a2, a3, a4, a5, a6, r1, r2, err uintptr }{fn, a1, a2, a3, a4, a5, a6, r1, r2, err} - entersyscall() - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall6X)), unsafe.Pointer(&args)) - exitsyscall() - return args.r1, args.r2, args.err +// libcCallInfo is a structure used to pass parameters to the system call. +type libcCallInfo struct { + fn uintptr + n uintptr // number of parameters + args uintptr // parameters + r1, r2 uintptr // return values } -func syscall6X() -// golang.org/x/sys linknames syscall.syscallPtr -// (in addition to standard package syscall). -// Do not remove or change the type signature. +// syscall_syscalln is a wrapper around the libc call with variable arguments. // -//go:linkname syscall_syscallPtr syscall.syscallPtr +//go:linkname syscall_syscalln syscall.syscalln //go:nosplit -func syscall_syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { - args := struct{ fn, a1, a2, a3, r1, r2, err uintptr }{fn, a1, a2, a3, r1, r2, err} +//go:uintptrkeepalive +func syscall_syscalln(fn uintptr, args ...uintptr) (r1, r2, err uintptr) { entersyscall() - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscallPtr)), unsafe.Pointer(&args)) + r1, r2, err = syscall_rawsyscalln(fn, args...) exitsyscall() - return args.r1, args.r2, args.err + return r1, r2, err } -func syscallPtr() -// golang.org/x/sys linknames syscall_rawSyscall -// (in addition to standard package syscall). -// Do not remove or change the type signature. +// syscall_rawsyscalln is a wrapper around the libc call with variable arguments. +// The scheduler is not notified about the system call. +// The syscall is executed on the current goroutine thread rather than on a +// dedicated syscall thread. // -//go:linkname syscall_rawSyscall syscall.rawSyscall +//go:linkname syscall_rawsyscalln syscall.rawsyscalln //go:nosplit -func syscall_rawSyscall(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { - args := struct{ fn, a1, a2, a3, r1, r2, err uintptr }{fn, a1, a2, a3, r1, r2, err} - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall)), unsafe.Pointer(&args)) - return args.r1, args.r2, args.err +//go:uintptrkeepalive +func syscall_rawsyscalln(fn uintptr, args ...uintptr) (r1, r2, err uintptr) { + c := &libcCallInfo{ + fn: fn, + n: uintptr(len(args)), + } + if c.n != 0 { + c.args = uintptr(noescape(unsafe.Pointer(&args[0]))) + } + libcCall(unsafe.Pointer(abi.FuncPCABI0(syscallN_trampoline)), unsafe.Pointer(c)) + if gp := getg(); gp != nil && gp.m != nil && gp.m.errnoAddr != nil { + err = uintptr(*gp.m.errnoAddr) + } else { + var errnoAddr *int32 + libc_error_addr(&errnoAddr) + err = uintptr(*errnoAddr) + } + return c.r1, c.r2, err } -// golang.org/x/sys linknames syscall_rawSyscall6 -// (in addition to standard package syscall). -// Do not remove or change the type signature. -// -//go:linkname syscall_rawSyscall6 syscall.rawSyscall6 -//go:nosplit -func syscall_rawSyscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { - args := struct{ fn, a1, a2, a3, a4, a5, a6, r1, r2, err uintptr }{fn, a1, a2, a3, a4, a5, a6, r1, r2, err} - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall6)), unsafe.Pointer(&args)) - return args.r1, args.r2, args.err -} +func syscallN_trampoline() // crypto_x509_syscall is used in crypto/x509/internal/macos to call into Security.framework and CF. @@ -571,6 +512,15 @@ func pthread_cond_signal(c *pthreadcond) int32 { } func pthread_cond_signal_trampoline() +//go:nosplit +//go:cgo_unsafe_args +func arc4random_buf(p unsafe.Pointer, n int32) { + // arc4random_buf() never fails, per its man page, so it's safe to ignore the return value. + libcCall(unsafe.Pointer(abi.FuncPCABI0(arc4random_buf_trampoline)), unsafe.Pointer(&p)) + KeepAlive(p) +} +func arc4random_buf_trampoline() + // Not used on Darwin, but must be defined. func exitThread(wait *atomic.Uint32) { throw("exitThread") @@ -691,6 +641,7 @@ func proc_regionfilename_trampoline() //go:cgo_import_dynamic libc_pthread_cond_wait pthread_cond_wait "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_pthread_cond_timedwait_relative_np pthread_cond_timedwait_relative_np "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_pthread_cond_signal pthread_cond_signal "/usr/lib/libSystem.B.dylib" +//go:cgo_import_dynamic libc_arc4random_buf arc4random_buf "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_notify_is_valid_token notify_is_valid_token "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_xpc_date_create_from_current xpc_date_create_from_current "/usr/lib/libSystem.B.dylib" diff --git a/src/runtime/sys_darwin_amd64.s b/src/runtime/sys_darwin_amd64.s index 01992d59d434eb..99d67a9cfd2d6e 100644 --- a/src/runtime/sys_darwin_amd64.s +++ b/src/runtime/sys_darwin_amd64.s @@ -500,275 +500,79 @@ TEXT runtime·osinit_hack_trampoline(SB),NOSPLIT,$0 CALL libc_xpc_date_create_from_current(SB) RET -// syscall calls a function in libc on behalf of the syscall package. -// syscall takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// r1 uintptr -// r2 uintptr -// err uintptr -// } -// syscall must be called on the g0 stack with the -// C calling convention (use libcCall). -// -// syscall expects a 32-bit result and tests for 32-bit -1 -// to decide there was an error. -TEXT runtime·syscall(SB),NOSPLIT,$16 - MOVQ (0*8)(DI), CX // fn - MOVQ (2*8)(DI), SI // a2 - MOVQ (3*8)(DI), DX // a3 - MOVQ DI, (SP) - MOVQ (1*8)(DI), DI // a1 - XORL AX, AX // vararg: say "no float args" - - CALL CX - - MOVQ (SP), DI - MOVQ AX, (4*8)(DI) // r1 - MOVQ DX, (5*8)(DI) // r2 - - // Standard libc functions return -1 on error - // and set errno. - CMPL AX, $-1 // Note: high 32 bits are junk - JNE ok - - // Get error code from libc. - CALL libc_error(SB) - MOVLQSX (AX), AX - MOVQ (SP), DI - MOVQ AX, (6*8)(DI) // err - -ok: - XORL AX, AX // no error (it's ignored anyway) - RET - -// syscallX calls a function in libc on behalf of the syscall package. -// syscallX takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// r1 uintptr -// r2 uintptr -// err uintptr -// } -// syscallX must be called on the g0 stack with the -// C calling convention (use libcCall). -// -// syscallX is like syscall but expects a 64-bit result -// and tests for 64-bit -1 to decide there was an error. -TEXT runtime·syscallX(SB),NOSPLIT,$16 - MOVQ (0*8)(DI), CX // fn - MOVQ (2*8)(DI), SI // a2 - MOVQ (3*8)(DI), DX // a3 - MOVQ DI, (SP) - MOVQ (1*8)(DI), DI // a1 - XORL AX, AX // vararg: say "no float args" - - CALL CX - - MOVQ (SP), DI - MOVQ AX, (4*8)(DI) // r1 - MOVQ DX, (5*8)(DI) // r2 - - // Standard libc functions return -1 on error - // and set errno. - CMPQ AX, $-1 - JNE ok - - // Get error code from libc. - CALL libc_error(SB) - MOVLQSX (AX), AX - MOVQ (SP), DI - MOVQ AX, (6*8)(DI) // err - -ok: - XORL AX, AX // no error (it's ignored anyway) - RET - -// syscallPtr is like syscallX except that the libc function reports an -// error by returning NULL and setting errno. -TEXT runtime·syscallPtr(SB),NOSPLIT,$16 - MOVQ (0*8)(DI), CX // fn - MOVQ (2*8)(DI), SI // a2 - MOVQ (3*8)(DI), DX // a3 - MOVQ DI, (SP) - MOVQ (1*8)(DI), DI // a1 - XORL AX, AX // vararg: say "no float args" - - CALL CX - - MOVQ (SP), DI - MOVQ AX, (4*8)(DI) // r1 - MOVQ DX, (5*8)(DI) // r2 - - // syscallPtr libc functions return NULL on error - // and set errno. - TESTQ AX, AX - JNE ok - - // Get error code from libc. - CALL libc_error(SB) - MOVLQSX (AX), AX - MOVQ (SP), DI - MOVQ AX, (6*8)(DI) // err - -ok: - XORL AX, AX // no error (it's ignored anyway) - RET - -// syscall6 calls a function in libc on behalf of the syscall package. -// syscall6 takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// a4 uintptr -// a5 uintptr -// a6 uintptr -// r1 uintptr -// r2 uintptr -// err uintptr -// } -// syscall6 must be called on the g0 stack with the -// C calling convention (use libcCall). -// -// syscall6 expects a 32-bit result and tests for 32-bit -1 -// to decide there was an error. -TEXT runtime·syscall6(SB),NOSPLIT,$16 - MOVQ (0*8)(DI), R11// fn - MOVQ (2*8)(DI), SI // a2 - MOVQ (3*8)(DI), DX // a3 - MOVQ (4*8)(DI), CX // a4 - MOVQ (5*8)(DI), R8 // a5 - MOVQ (6*8)(DI), R9 // a6 - MOVQ DI, (SP) - MOVQ (1*8)(DI), DI // a1 - XORL AX, AX // vararg: say "no float args" - - CALL R11 - - MOVQ (SP), DI - MOVQ AX, (7*8)(DI) // r1 - MOVQ DX, (8*8)(DI) // r2 - - CMPL AX, $-1 - JNE ok - - CALL libc_error(SB) - MOVLQSX (AX), AX - MOVQ (SP), DI - MOVQ AX, (9*8)(DI) // err - -ok: - XORL AX, AX // no error (it's ignored anyway) - RET - -// syscall6X calls a function in libc on behalf of the syscall package. -// syscall6X takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// a4 uintptr -// a5 uintptr -// a6 uintptr -// r1 uintptr -// r2 uintptr -// err uintptr -// } -// syscall6X must be called on the g0 stack with the -// C calling convention (use libcCall). -// -// syscall6X is like syscall6 but expects a 64-bit result -// and tests for 64-bit -1 to decide there was an error. -TEXT runtime·syscall6X(SB),NOSPLIT,$16 - MOVQ (0*8)(DI), R11// fn - MOVQ (2*8)(DI), SI // a2 - MOVQ (3*8)(DI), DX // a3 - MOVQ (4*8)(DI), CX // a4 - MOVQ (5*8)(DI), R8 // a5 - MOVQ (6*8)(DI), R9 // a6 - MOVQ DI, (SP) - MOVQ (1*8)(DI), DI // a1 - XORL AX, AX // vararg: say "no float args" +TEXT runtime·arc4random_buf_trampoline(SB),NOSPLIT,$0 + MOVL 8(DI), SI // arg 2 nbytes + MOVQ 0(DI), DI // arg 1 buf + CALL libc_arc4random_buf(SB) + RET + + +TEXT runtime·syscallN_trampoline(SB),NOSPLIT,$0 + // store argument and original SP in a callee-saved register + MOVQ DI, R13 + MOVQ SP, R14 + + MOVQ libcCallInfo_fn(R13), R11 + MOVQ libcCallInfo_n(R13), CX + MOVQ libcCallInfo_args(R13), R10 + + // Fast version, do not store args on the stack. + CMPL CX, $0; JE _0args + CMPL CX, $1; JE _1args + CMPL CX, $2; JE _2args + CMPL CX, $3; JE _3args + CMPL CX, $4; JE _4args + CMPL CX, $5; JE _5args + CMPL CX, $6; JE _6args + + // Reserve stack space for remaining args + MOVQ CX, R12 + SUBQ $6, R12 + ADDQ $1, R12 // make even number of words for stack alignment + ANDQ $~1, R12 + SHLQ $3, R12 + SUBQ R12, SP + + // Copy args to the stack. + // CX: count of stack arguments (n-6) + // SI: &args[6] + // DI: copy of RSP + SUBQ $6, CX + MOVQ R10, SI + ADDQ $(8*6), SI + MOVQ SP, DI + CLD + REP; MOVSQ + +_6args: + MOVQ (5*8)(R10), R9 +_5args: + MOVQ (4*8)(R10), R8 +_4args: + MOVQ (3*8)(R10), CX +_3args: + MOVQ (2*8)(R10), DX +_2args: + MOVQ (1*8)(R10), SI +_1args: + MOVQ (0*8)(R10), DI +_0args: + + XORL AX, AX // vararg: say "no float args" CALL R11 - MOVQ (SP), DI - MOVQ AX, (7*8)(DI) // r1 - MOVQ DX, (8*8)(DI) // r2 - - CMPQ AX, $-1 - JNE ok + MOVQ R14, SP // free stack space - CALL libc_error(SB) - MOVLQSX (AX), AX - MOVQ (SP), DI - MOVQ AX, (9*8)(DI) // err - -ok: - XORL AX, AX // no error (it's ignored anyway) + // Return result. + MOVQ AX, libcCallInfo_r1(R13) + MOVQ DX, libcCallInfo_r2(R13) RET -// syscall9 calls a function in libc on behalf of the syscall package. -// syscall9 takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// a4 uintptr -// a5 uintptr -// a6 uintptr -// a7 uintptr -// a8 uintptr -// a9 uintptr -// r1 uintptr -// r2 uintptr -// err uintptr -// } -// syscall9 must be called on the g0 stack with the -// C calling convention (use libcCall). -// -// syscall9 expects a 32-bit result and tests for 32-bit -1 -// to decide there was an error. -TEXT runtime·syscall9(SB),NOSPLIT,$16 - MOVQ (0*8)(DI), R13// fn - MOVQ (2*8)(DI), SI // a2 - MOVQ (3*8)(DI), DX // a3 - MOVQ (4*8)(DI), CX // a4 - MOVQ (5*8)(DI), R8 // a5 - MOVQ (6*8)(DI), R9 // a6 - MOVQ (7*8)(DI), R10 // a7 - MOVQ (8*8)(DI), R11 // a8 - MOVQ (9*8)(DI), R12 // a9 - MOVQ DI, (SP) - MOVQ (1*8)(DI), DI // a1 - XORL AX, AX // vararg: say "no float args" - - CALL R13 - - MOVQ (SP), DI - MOVQ AX, (10*8)(DI) // r1 - MOVQ DX, (11*8)(DI) // r2 - - CMPL AX, $-1 - JNE ok - +TEXT runtime·libc_error_trampoline(SB),NOSPLIT,$0 + MOVQ 0(DI), R14 CALL libc_error(SB) - MOVLQSX (AX), AX - MOVQ (SP), DI - MOVQ AX, (12*8)(DI) // err - -ok: - XORL AX, AX // no error (it's ignored anyway) + MOVQ AX, (R14) RET // syscall_x509 is for crypto/x509. It is like syscall6 but does not check for errors, diff --git a/src/runtime/sys_darwin_arm64.s b/src/runtime/sys_darwin_arm64.s index 32d1f95d565f3b..7bbe965c1583f8 100644 --- a/src/runtime/sys_darwin_arm64.s +++ b/src/runtime/sys_darwin_arm64.s @@ -475,272 +475,104 @@ TEXT runtime·osinit_hack_trampoline(SB),NOSPLIT,$0 BL libc_xpc_date_create_from_current(SB) RET -// syscall calls a function in libc on behalf of the syscall package. -// syscall takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// r1 uintptr -// r2 uintptr -// err uintptr -// } -// syscall must be called on the g0 stack with the -// C calling convention (use libcCall). -TEXT runtime·syscall(SB),NOSPLIT,$0 - SUB $16, RSP // push structure pointer - MOVD R0, 8(RSP) - - MOVD 0(R0), R12 // fn - MOVD 16(R0), R1 // a2 - MOVD 24(R0), R2 // a3 - MOVD 8(R0), R0 // a1 +TEXT runtime·arc4random_buf_trampoline(SB),NOSPLIT,$0 + MOVW 8(R0), R1 // arg 2 nbytes + MOVD 0(R0), R0 // arg 1 buf + BL libc_arc4random_buf(SB) + RET + +TEXT runtime·syscallN_trampoline(SB),NOSPLIT,$16 + STP (R19, R20), 16(RSP) // save old R19, R20 + MOVD R0, R19 // save struct pointer + MOVD RSP, R20 // save stack pointer + SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved. + + MOVD libcCallInfo_args(R19), R12 + // Do we have more than 8 arguments? + MOVD libcCallInfo_n(R19), R0 + CMP $0, R0; BEQ _0args + CMP $1, R0; BEQ _1args + CMP $2, R0; BEQ _2args + CMP $3, R0; BEQ _3args + CMP $4, R0; BEQ _4args + CMP $5, R0; BEQ _5args + CMP $6, R0; BEQ _6args + CMP $7, R0; BEQ _7args + CMP $8, R0; BEQ _8args + + // Reserve stack space for remaining args + SUB $8, R0, R2 + ADD $1, R2, R3 // make even number of words for stack alignment + AND $~1, R3 + LSL $3, R3 + SUB R3, RSP + + // R4: size of stack arguments (n-8)*8 + // R5: &args[8] + // R6: loop counter, from 0 to (n-8)*8 + // R7: scratch + // R8: copy of RSP - (R2)(RSP) assembles as (R2)(ZR) + SUB $8, R0, R4 + LSL $3, R4 + ADD $(8*8), R12, R5 + MOVD $0, R6 + MOVD RSP, R8 +stackargs: + MOVD (R6)(R5), R7 + MOVD R7, (R6)(R8) + ADD $8, R6 + CMP R6, R4 + BNE stackargs + +_8args: + MOVD (7*8)(R12), R7 +_7args: + MOVD (6*8)(R12), R6 +_6args: + MOVD (5*8)(R12), R5 +_5args: + MOVD (4*8)(R12), R4 +_4args: + MOVD (3*8)(R12), R3 +_3args: + MOVD (2*8)(R12), R2 +_2args: + MOVD (1*8)(R12), R1 +_1args: + MOVD (0*8)(R12), R0 +_0args: // If fn is declared as vararg, we have to pass the vararg arguments on the stack. // (Because ios decided not to adhere to the standard arm64 calling convention, sigh...) - // The only libSystem calls we support that are vararg are open, fcntl, and ioctl, - // which are all of the form fn(x, y, ...). So we just need to put the 3rd arg - // on the stack as well. + // The only libSystem calls we support with vararg are open, fcntl, ioctl, + // which are all of the form fn(x, y, ...), and openat, which is of the form fn(x, y, z, ...). + // So we just need to put the 3rd and the 4th arg on the stack as well. + // Note that historically openat has been called with syscall6, so we need to handle that case too. // If we ever have other vararg libSystem calls, we might need to handle more cases. + MOVD libcCallInfo_n(R19), R12 + CMP $3, R12; BNE 2(PC); MOVD R2, (RSP) - - BL (R12) - - MOVD 8(RSP), R2 // pop structure pointer - ADD $16, RSP - MOVD R0, 32(R2) // save r1 - MOVD R1, 40(R2) // save r2 - CMPW $-1, R0 - BNE ok - SUB $16, RSP // push structure pointer - MOVD R2, 8(RSP) - BL libc_error(SB) - MOVW (R0), R0 - MOVD 8(RSP), R2 // pop structure pointer - ADD $16, RSP - MOVD R0, 48(R2) // save err -ok: - RET - -// syscallX calls a function in libc on behalf of the syscall package. -// syscallX takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// r1 uintptr -// r2 uintptr -// err uintptr -// } -// syscallX must be called on the g0 stack with the -// C calling convention (use libcCall). -TEXT runtime·syscallX(SB),NOSPLIT,$0 - SUB $16, RSP // push structure pointer - MOVD R0, (RSP) - - MOVD 0(R0), R12 // fn - MOVD 16(R0), R1 // a2 - MOVD 24(R0), R2 // a3 - MOVD 8(R0), R0 // a1 - BL (R12) - - MOVD (RSP), R2 // pop structure pointer - ADD $16, RSP - MOVD R0, 32(R2) // save r1 - MOVD R1, 40(R2) // save r2 - CMP $-1, R0 - BNE ok - SUB $16, RSP // push structure pointer - MOVD R2, (RSP) - BL libc_error(SB) - MOVW (R0), R0 - MOVD (RSP), R2 // pop structure pointer - ADD $16, RSP - MOVD R0, 48(R2) // save err -ok: - RET - -// syscallPtr is like syscallX except that the libc function reports an -// error by returning NULL and setting errno. -TEXT runtime·syscallPtr(SB),NOSPLIT,$0 - SUB $16, RSP // push structure pointer - MOVD R0, (RSP) - - MOVD 0(R0), R12 // fn - MOVD 16(R0), R1 // a2 - MOVD 24(R0), R2 // a3 - MOVD 8(R0), R0 // a1 - BL (R12) - - MOVD (RSP), R2 // pop structure pointer - ADD $16, RSP - MOVD R0, 32(R2) // save r1 - MOVD R1, 40(R2) // save r2 - CMP $0, R0 - BNE ok - SUB $16, RSP // push structure pointer - MOVD R2, (RSP) - BL libc_error(SB) - MOVW (R0), R0 - MOVD (RSP), R2 // pop structure pointer - ADD $16, RSP - MOVD R0, 48(R2) // save err -ok: - RET - -// syscall6 calls a function in libc on behalf of the syscall package. -// syscall6 takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// a4 uintptr -// a5 uintptr -// a6 uintptr -// r1 uintptr -// r2 uintptr -// err uintptr -// } -// syscall6 must be called on the g0 stack with the -// C calling convention (use libcCall). -TEXT runtime·syscall6(SB),NOSPLIT,$0 - SUB $16, RSP // push structure pointer - MOVD R0, 8(RSP) - - MOVD 0(R0), R12 // fn - MOVD 16(R0), R1 // a2 - MOVD 24(R0), R2 // a3 - MOVD 32(R0), R3 // a4 - MOVD 40(R0), R4 // a5 - MOVD 48(R0), R5 // a6 - MOVD 8(R0), R0 // a1 - - // If fn is declared as vararg, we have to pass the vararg arguments on the stack. - // See syscall above. The only function this applies to is openat, for which the 4th - // arg must be on the stack. + CMP $4, R12; BNE 2(PC); + MOVD R3, (RSP) + CMP $6, R12; BNE 2(PC); MOVD R3, (RSP) + MOVD libcCallInfo_fn(R19), R12 BL (R12) - MOVD 8(RSP), R2 // pop structure pointer - ADD $16, RSP - MOVD R0, 56(R2) // save r1 - MOVD R1, 64(R2) // save r2 - CMPW $-1, R0 - BNE ok - SUB $16, RSP // push structure pointer - MOVD R2, 8(RSP) - BL libc_error(SB) - MOVW (R0), R0 - MOVD 8(RSP), R2 // pop structure pointer - ADD $16, RSP - MOVD R0, 72(R2) // save err -ok: - RET + MOVD R20, RSP // free stack space -// syscall6X calls a function in libc on behalf of the syscall package. -// syscall6X takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// a4 uintptr -// a5 uintptr -// a6 uintptr -// r1 uintptr -// r2 uintptr -// err uintptr -// } -// syscall6X must be called on the g0 stack with the -// C calling convention (use libcCall). -TEXT runtime·syscall6X(SB),NOSPLIT,$0 - SUB $16, RSP // push structure pointer - MOVD R0, (RSP) + MOVD R0, libcCallInfo_r1(R19) + MOVD R1, libcCallInfo_r2(R19) - MOVD 0(R0), R12 // fn - MOVD 16(R0), R1 // a2 - MOVD 24(R0), R2 // a3 - MOVD 32(R0), R3 // a4 - MOVD 40(R0), R4 // a5 - MOVD 48(R0), R5 // a6 - MOVD 8(R0), R0 // a1 - BL (R12) + // Restore callee-saved registers. + LDP 16(RSP), (R19, R20) + RET - MOVD (RSP), R2 // pop structure pointer - ADD $16, RSP - MOVD R0, 56(R2) // save r1 - MOVD R1, 64(R2) // save r2 - CMP $-1, R0 - BNE ok - SUB $16, RSP // push structure pointer - MOVD R2, (RSP) - BL libc_error(SB) - MOVW (R0), R0 - MOVD (RSP), R2 // pop structure pointer - ADD $16, RSP - MOVD R0, 72(R2) // save err -ok: - RET - -// syscall9 calls a function in libc on behalf of the syscall package. -// syscall9 takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// a4 uintptr -// a5 uintptr -// a6 uintptr -// a7 uintptr -// a8 uintptr -// a9 uintptr -// r1 uintptr -// r2 uintptr -// err uintptr -// } -// syscall9 must be called on the g0 stack with the -// C calling convention (use libcCall). -TEXT runtime·syscall9(SB),NOSPLIT,$0 - SUB $16, RSP // push structure pointer - MOVD R0, 8(RSP) - - MOVD 0(R0), R12 // fn - MOVD 16(R0), R1 // a2 - MOVD 24(R0), R2 // a3 - MOVD 32(R0), R3 // a4 - MOVD 40(R0), R4 // a5 - MOVD 48(R0), R5 // a6 - MOVD 56(R0), R6 // a7 - MOVD 64(R0), R7 // a8 - MOVD 72(R0), R8 // a9 - MOVD 8(R0), R0 // a1 - - // If fn is declared as vararg, we have to pass the vararg arguments on the stack. - // See syscall above. The only function this applies to is openat, for which the 4th - // arg must be on the stack. - MOVD R3, (RSP) - - BL (R12) - - MOVD 8(RSP), R2 // pop structure pointer - ADD $16, RSP - MOVD R0, 80(R2) // save r1 - MOVD R1, 88(R2) // save r2 - CMPW $-1, R0 - BNE ok - SUB $16, RSP // push structure pointer - MOVD R2, 8(RSP) +TEXT runtime·libc_error_trampoline(SB),NOSPLIT,$0 + MOVD 0(R0), R20 BL libc_error(SB) - MOVW (R0), R0 - MOVD 8(RSP), R2 // pop structure pointer - ADD $16, RSP - MOVD R0, 96(R2) // save err -ok: + MOVD R0, (R20) RET // syscall_x509 is for crypto/x509. It is like syscall6 but does not check for errors, diff --git a/src/runtime/sys_libc.go b/src/runtime/sys_libc.go index 0c6f13ca9f6fc1..214e87931993af 100644 --- a/src/runtime/sys_libc.go +++ b/src/runtime/sys_libc.go @@ -2,11 +2,14 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build darwin || (openbsd && !mips64) +//go:build darwin || openbsd package runtime -import "unsafe" +import ( + "internal/runtime/sys" + "unsafe" +) // Call fn with arg as its argument. Return what fn returns. // fn is the raw pc value of the entry point of the desired function. @@ -23,10 +26,10 @@ func libcCall(fn, arg unsafe.Pointer) int32 { } if mp != nil && mp.libcallsp == 0 { mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() + mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() + mp.libcallsp = sys.GetCallerSP() } else { // Make sure we don't reset libcallsp. This makes // libcCall reentrant; We remember the g/pc/sp for the diff --git a/src/runtime/sys_linux_386.s b/src/runtime/sys_linux_386.s index d53be243fe7001..f664d8ace7f997 100644 --- a/src/runtime/sys_linux_386.s +++ b/src/runtime/sys_linux_386.s @@ -48,11 +48,13 @@ #define SYS_madvise 219 #define SYS_gettid 224 #define SYS_futex 240 +#define SYS_futex_time64 422 #define SYS_sched_getaffinity 242 #define SYS_set_thread_area 243 #define SYS_exit_group 252 #define SYS_timer_create 259 #define SYS_timer_settime 260 +#define SYS_timer_settime64 409 #define SYS_timer_delete 263 #define SYS_clock_gettime 265 #define SYS_tgkill 270 @@ -208,7 +210,8 @@ TEXT runtime·timer_create(SB),NOSPLIT,$0-16 MOVL AX, ret+12(FP) RET -TEXT runtime·timer_settime(SB),NOSPLIT,$0-20 +// Linux: kernel/time/posix-timer.c, requiring COMPAT_32BIT_TIME +TEXT runtime·timer_settime32(SB),NOSPLIT,$0-20 MOVL $SYS_timer_settime, AX MOVL timerid+0(FP), BX MOVL flags+4(FP), CX @@ -218,6 +221,16 @@ TEXT runtime·timer_settime(SB),NOSPLIT,$0-20 MOVL AX, ret+16(FP) RET +TEXT runtime·timer_settime64(SB),NOSPLIT,$0-20 + MOVL $SYS_timer_settime64, AX + MOVL timerid+0(FP), BX + MOVL flags+4(FP), CX + MOVL new+8(FP), DX + MOVL old+12(FP), SI + INVOKE_SYSCALL + MOVL AX, ret+16(FP) + RET + TEXT runtime·timer_delete(SB),NOSPLIT,$0-8 MOVL $SYS_timer_delete, AX MOVL timerid+0(FP), BX @@ -410,6 +423,25 @@ TEXT runtime·rt_sigaction(SB),NOSPLIT,$0 MOVL AX, ret+16(FP) RET +// Call the function stored in _cgo_sigaction using the GCC calling convention. +TEXT runtime·callCgoSigaction(SB),NOSPLIT,$0-16 + MOVL _cgo_sigaction(SB), AX + MOVL sig+0(FP), BX + MOVL new+4(FP), CX + MOVL old+8(FP), DX + MOVL SP, SI // align stack to call C function + SUBL $32, SP + ANDL $~15, SP + MOVL BX, 0(SP) + MOVL CX, 4(SP) + MOVL DX, 8(SP) + MOVL SI, 12(SP) + CALL AX + MOVL 12(SP), BX + MOVL BX, SP + MOVL AX, ret+12(FP) + RET + TEXT runtime·sigfwd(SB),NOSPLIT,$12-16 MOVL fn+0(FP), AX MOVL sig+4(FP), BX @@ -513,9 +545,10 @@ TEXT runtime·madvise(SB),NOSPLIT,$0 MOVL AX, ret+12(FP) RET +// Linux: kernel/futex/syscalls.c, requiring COMPAT_32BIT_TIME // int32 futex(int32 *uaddr, int32 op, int32 val, -// struct timespec *timeout, int32 *uaddr2, int32 val2); -TEXT runtime·futex(SB),NOSPLIT,$0 +// struct old_timespec32 *timeout, int32 *uaddr2, int32 val2); +TEXT runtime·futex_time32(SB),NOSPLIT,$0 MOVL $SYS_futex, AX MOVL addr+0(FP), BX MOVL op+4(FP), CX @@ -527,6 +560,21 @@ TEXT runtime·futex(SB),NOSPLIT,$0 MOVL AX, ret+24(FP) RET +// Linux: kernel/futex/syscalls.c +// int32 futex(int32 *uaddr, int32 op, int32 val, +// struct timespec *timeout, int32 *uaddr2, int32 val2); +TEXT runtime·futex_time64(SB),NOSPLIT,$0 + MOVL $SYS_futex_time64, AX + MOVL addr+0(FP), BX + MOVL op+4(FP), CX + MOVL val+8(FP), DX + MOVL ts+12(FP), SI + MOVL addr2+16(FP), DI + MOVL val3+20(FP), BP + INVOKE_SYSCALL + MOVL AX, ret+24(FP) + RET + // int32 clone(int32 flags, void *stack, M *mp, G *gp, void (*fn)(void)); TEXT runtime·clone(SB),NOSPLIT,$0 MOVL $SYS_clone, AX diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index b6c64dc0959c65..941f70b0e8e652 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -704,3 +704,36 @@ TEXT runtime·sbrk0(SB),NOSPLIT,$0-8 SYSCALL MOVQ AX, ret+0(FP) RET + +// func vgetrandom1(buf *byte, length uintptr, flags uint32, state uintptr, stateSize uintptr) int +TEXT runtime·vgetrandom1(SB),NOSPLIT,$16-48 + MOVQ SI, R8 // stateSize + MOVL CX, DX // flags + MOVQ DI, CX // state + MOVQ BX, SI // length + MOVQ AX, DI // buf + + MOVQ SP, R12 + + MOVQ runtime·vdsoGetrandomSym(SB), AX + MOVQ g_m(R14), BX + + MOVQ m_vdsoPC(BX), R9 + MOVQ R9, 0(SP) + MOVQ m_vdsoSP(BX), R9 + MOVQ R9, 8(SP) + LEAQ buf+0(FP), R9 + MOVQ R9, m_vdsoSP(BX) + MOVQ -8(R9), R9 + MOVQ R9, m_vdsoPC(BX) + + ANDQ $~15, SP + + CALL AX + + MOVQ R12, SP + MOVQ 8(SP), R9 + MOVQ R9, m_vdsoSP(BX) + MOVQ 0(SP), R9 + MOVQ R9, m_vdsoPC(BX) + RET diff --git a/src/runtime/sys_linux_arm.s b/src/runtime/sys_linux_arm.s index 992d32ab6c9b58..f19e293d0fc3e0 100644 --- a/src/runtime/sys_linux_arm.s +++ b/src/runtime/sys_linux_arm.s @@ -30,6 +30,7 @@ #define SYS_sigaltstack (SYS_BASE + 186) #define SYS_mmap2 (SYS_BASE + 192) #define SYS_futex (SYS_BASE + 240) +#define SYS_futex_time64 (SYS_BASE + 422) #define SYS_exit_group (SYS_BASE + 248) #define SYS_munmap (SYS_BASE + 91) #define SYS_madvise (SYS_BASE + 220) @@ -43,6 +44,7 @@ #define SYS_clock_gettime (SYS_BASE + 263) #define SYS_timer_create (SYS_BASE + 257) #define SYS_timer_settime (SYS_BASE + 258) +#define SYS_timer_settime64 (SYS_BASE + 409) #define SYS_timer_delete (SYS_BASE + 261) #define SYS_pipe2 (SYS_BASE + 359) #define SYS_access (SYS_BASE + 33) @@ -230,8 +232,8 @@ TEXT runtime·timer_create(SB),NOSPLIT,$0-16 SWI $0 MOVW R0, ret+12(FP) RET - -TEXT runtime·timer_settime(SB),NOSPLIT,$0-20 +// Linux: kernel/time/posix-timer.c, requiring COMPAT_32BIT_TIME. +TEXT runtime·timer_settime32(SB),NOSPLIT,$0-20 MOVW timerid+0(FP), R0 MOVW flags+4(FP), R1 MOVW new+8(FP), R2 @@ -241,6 +243,16 @@ TEXT runtime·timer_settime(SB),NOSPLIT,$0-20 MOVW R0, ret+16(FP) RET +TEXT runtime·timer_settime64(SB),NOSPLIT,$0-20 + MOVW timerid+0(FP), R0 + MOVW flags+4(FP), R1 + MOVW new+8(FP), R2 + MOVW old+12(FP), R3 + MOVW $SYS_timer_settime64, R7 + SWI $0 + MOVW R0, ret+16(FP) + RET + TEXT runtime·timer_delete(SB),NOSPLIT,$0-8 MOVW timerid+0(FP), R0 MOVW $SYS_timer_delete, R7 @@ -403,9 +415,10 @@ finish: RET +// Linux: kernel/futex/syscalls.c, requiring COMPAT_32BIT_TIME // int32 futex(int32 *uaddr, int32 op, int32 val, -// struct timespec *timeout, int32 *uaddr2, int32 val2); -TEXT runtime·futex(SB),NOSPLIT,$0 +// struct old_timespec32 *timeout, int32 *uaddr2, int32 val2); +TEXT runtime·futex_time32(SB),NOSPLIT,$0 MOVW addr+0(FP), R0 MOVW op+4(FP), R1 MOVW val+8(FP), R2 @@ -417,6 +430,21 @@ TEXT runtime·futex(SB),NOSPLIT,$0 MOVW R0, ret+24(FP) RET +// Linux: kernel/futex/syscalls.c +// int32 futex(int32 *uaddr, int32 op, int32 val, +// struct timespec *timeout, int32 *uaddr2, int32 val2); +TEXT runtime·futex_time64(SB),NOSPLIT,$0 + MOVW addr+0(FP), R0 + MOVW op+4(FP), R1 + MOVW val+8(FP), R2 + MOVW ts+12(FP), R3 + MOVW addr2+16(FP), R4 + MOVW val3+20(FP), R5 + MOVW $SYS_futex_time64, R7 + SWI $0 + MOVW R0, ret+24(FP) + RET + // int32 clone(int32 flags, void *stack, M *mp, G *gp, void (*fn)(void)); TEXT runtime·clone(SB),NOSPLIT,$0 MOVW flags+0(FP), R0 diff --git a/src/runtime/sys_linux_arm64.s b/src/runtime/sys_linux_arm64.s index 51c87bea05a24e..7a81d5479e3e28 100644 --- a/src/runtime/sys_linux_arm64.s +++ b/src/runtime/sys_linux_arm64.s @@ -785,3 +785,48 @@ TEXT runtime·sbrk0(SB),NOSPLIT,$0-8 SVC MOVD R0, ret+0(FP) RET + +// func vgetrandom1(buf *byte, length uintptr, flags uint32, state uintptr, stateSize uintptr) int +TEXT runtime·vgetrandom1(SB),NOSPLIT,$16-48 + MOVD RSP, R20 + + MOVD runtime·vdsoGetrandomSym(SB), R8 + MOVD g_m(g), R21 + + MOVD m_vdsoPC(R21), R9 + MOVD R9, 8(RSP) + MOVD m_vdsoSP(R21), R9 + MOVD R9, 16(RSP) + MOVD LR, m_vdsoPC(R21) + MOVD $buf-8(FP), R9 + MOVD R9, m_vdsoSP(R21) + + MOVD RSP, R9 + BIC $15, R9 + MOVD R9, RSP + + MOVBU runtime·iscgo(SB), R9 + CBNZ R9, nosaveg + MOVD m_gsignal(R21), R9 + CBZ R9, nosaveg + CMP g, R9 + BEQ nosaveg + MOVD (g_stack+stack_lo)(R9), R22 + MOVD g, (R22) + + BL (R8) + + MOVD ZR, (R22) + B restore + +nosaveg: + BL (R8) + +restore: + MOVD R20, RSP + MOVD 16(RSP), R1 + MOVD R1, m_vdsoSP(R21) + MOVD 8(RSP), R1 + MOVD R1, m_vdsoPC(R21) + NOP R0 // Satisfy go vet, since the return value comes from the vDSO function. + RET diff --git a/src/runtime/sys_linux_loong64.s b/src/runtime/sys_linux_loong64.s index eba8e1f24ce204..41e12c30f87cf0 100644 --- a/src/runtime/sys_linux_loong64.s +++ b/src/runtime/sys_linux_loong64.s @@ -47,68 +47,58 @@ #define SYS_timer_delete 111 // func exit(code int32) -TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 - MOVW code+0(FP), R4 +TEXT runtime·exit(SB),NOSPLIT,$0 MOVV $SYS_exit_group, R11 SYSCALL RET // func exitThread(wait *atomic.Uint32) -TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 - MOVV wait+0(FP), R19 +TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0 // We're done using the stack. - MOVW $0, R11 - DBAR - MOVW R11, (R19) - DBAR + DBAR $0x12 // StoreRelease barrier + MOVW R0, (R4) MOVW $0, R4 // exit code MOVV $SYS_exit, R11 SYSCALL JMP 0(PC) // func open(name *byte, mode, perm int32) int32 -TEXT runtime·open(SB),NOSPLIT|NOFRAME,$0-20 +TEXT runtime·open(SB),NOSPLIT,$0 + // before: + // R4: name, R5: mode, R6: perm + // after: + // R4: AT_FDCWD, R5: name, R6: mode, R7: perm + MOVW R6, R7 + MOVW R5, R6 + MOVV R4, R5 MOVW $AT_FDCWD, R4 // AT_FDCWD, so this acts like open - MOVV name+0(FP), R5 - MOVW mode+8(FP), R6 - MOVW perm+12(FP), R7 + MOVV $SYS_openat, R11 SYSCALL MOVW $-4096, R5 BGEU R5, R4, 2(PC) MOVW $-1, R4 - MOVW R4, ret+16(FP) RET // func closefd(fd int32) int32 -TEXT runtime·closefd(SB),NOSPLIT|NOFRAME,$0-12 - MOVW fd+0(FP), R4 +TEXT runtime·closefd(SB),NOSPLIT,$0 MOVV $SYS_close, R11 SYSCALL MOVW $-4096, R5 BGEU R5, R4, 2(PC) MOVW $-1, R4 - MOVW R4, ret+8(FP) RET // func write1(fd uintptr, p unsafe.Pointer, n int32) int32 -TEXT runtime·write1(SB),NOSPLIT|NOFRAME,$0-28 - MOVV fd+0(FP), R4 - MOVV p+8(FP), R5 - MOVW n+16(FP), R6 +TEXT runtime·write1(SB),NOSPLIT,$0 MOVV $SYS_write, R11 SYSCALL - MOVW R4, ret+24(FP) RET // func read(fd int32, p unsafe.Pointer, n int32) int32 -TEXT runtime·read(SB),NOSPLIT|NOFRAME,$0-28 - MOVW fd+0(FP), R4 - MOVV p+8(FP), R5 - MOVW n+16(FP), R6 +TEXT runtime·read(SB),NOSPLIT,$0 MOVV $SYS_read, R11 SYSCALL - MOVW R4, ret+24(FP) RET // func pipe2(flags int32) (r, w int32, errno int32) @@ -121,16 +111,15 @@ TEXT runtime·pipe2(SB),NOSPLIT|NOFRAME,$0-20 RET // func usleep(usec uint32) -TEXT runtime·usleep(SB),NOSPLIT,$16-4 - MOVWU usec+0(FP), R7 +TEXT runtime·usleep(SB),NOSPLIT,$16 MOVV $1000, R6 - MULVU R6, R7, R7 + MULVU R6, R4, R4 MOVV $1000000000, R6 - DIVVU R6, R7, R5 // ts->tv_sec - REMVU R6, R7, R4 // ts->tv_nsec + DIVVU R6, R4, R5 // ts->tv_sec + REMVU R6, R4, R8 // ts->tv_nsec MOVV R5, 8(R3) - MOVV R4, 16(R3) + MOVV R8, 16(R3) // nanosleep(&ts, 0) ADDV $8, R3, R4 @@ -140,14 +129,14 @@ TEXT runtime·usleep(SB),NOSPLIT,$16-4 RET // func gettid() uint32 -TEXT runtime·gettid(SB),NOSPLIT,$0-4 +TEXT runtime·gettid(SB),NOSPLIT,$0 MOVV $SYS_gettid, R11 SYSCALL - MOVW R4, ret+0(FP) RET // func raise(sig uint32) -TEXT runtime·raise(SB),NOSPLIT|NOFRAME,$0 +TEXT runtime·raise(SB),NOSPLIT,$0 + MOVW R4, R24 // backup sig MOVV $SYS_getpid, R11 SYSCALL MOVW R4, R23 @@ -155,87 +144,66 @@ TEXT runtime·raise(SB),NOSPLIT|NOFRAME,$0 SYSCALL MOVW R4, R5 // arg 2 tid MOVW R23, R4 // arg 1 pid - MOVW sig+0(FP), R6 // arg 3 + MOVW R24, R6 // arg 3 MOVV $SYS_tgkill, R11 SYSCALL RET // func raiseproc(sig uint32) -TEXT runtime·raiseproc(SB),NOSPLIT|NOFRAME,$0 +TEXT runtime·raiseproc(SB),NOSPLIT,$0 + MOVW R4, R24 // backup sig MOVV $SYS_getpid, R11 SYSCALL //MOVW R4, R4 // arg 1 pid - MOVW sig+0(FP), R5 // arg 2 + MOVW R24, R5 // arg 2 MOVV $SYS_kill, R11 SYSCALL RET // func getpid() int -TEXT ·getpid(SB),NOSPLIT|NOFRAME,$0-8 +TEXT ·getpid(SB),NOSPLIT,$0 MOVV $SYS_getpid, R11 SYSCALL - MOVV R4, ret+0(FP) RET // func tgkill(tgid, tid, sig int) -TEXT ·tgkill(SB),NOSPLIT|NOFRAME,$0-24 - MOVV tgid+0(FP), R4 - MOVV tid+8(FP), R5 - MOVV sig+16(FP), R6 +TEXT ·tgkill(SB),NOSPLIT,$0 MOVV $SYS_tgkill, R11 SYSCALL RET // func setitimer(mode int32, new, old *itimerval) -TEXT runtime·setitimer(SB),NOSPLIT|NOFRAME,$0-24 - MOVW mode+0(FP), R4 - MOVV new+8(FP), R5 - MOVV old+16(FP), R6 +TEXT runtime·setitimer(SB),NOSPLIT,$0 MOVV $SYS_setitimer, R11 SYSCALL RET // func timer_create(clockid int32, sevp *sigevent, timerid *int32) int32 -TEXT runtime·timer_create(SB),NOSPLIT,$0-28 - MOVW clockid+0(FP), R4 - MOVV sevp+8(FP), R5 - MOVV timerid+16(FP), R6 +TEXT runtime·timer_create(SB),NOSPLIT,$0 MOVV $SYS_timer_create, R11 SYSCALL - MOVW R4, ret+24(FP) RET // func timer_settime(timerid int32, flags int32, new, old *itimerspec) int32 -TEXT runtime·timer_settime(SB),NOSPLIT,$0-28 - MOVW timerid+0(FP), R4 - MOVW flags+4(FP), R5 - MOVV new+8(FP), R6 - MOVV old+16(FP), R7 +TEXT runtime·timer_settime(SB),NOSPLIT,$0 MOVV $SYS_timer_settime, R11 SYSCALL - MOVW R4, ret+24(FP) RET // func timer_delete(timerid int32) int32 -TEXT runtime·timer_delete(SB),NOSPLIT,$0-12 - MOVW timerid+0(FP), R4 +TEXT runtime·timer_delete(SB),NOSPLIT,$0 MOVV $SYS_timer_delete, R11 SYSCALL - MOVW R4, ret+8(FP) RET // func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32 -TEXT runtime·mincore(SB),NOSPLIT|NOFRAME,$0-28 - MOVV addr+0(FP), R4 - MOVV n+8(FP), R5 - MOVV dst+16(FP), R6 +TEXT runtime·mincore(SB),NOSPLIT,$0 MOVV $SYS_mincore, R11 SYSCALL - MOVW R4, ret+24(FP) RET // func walltime() (sec int64, nsec int32) -TEXT runtime·walltime(SB),NOSPLIT,$24-12 +TEXT runtime·walltime(SB),NOSPLIT,$24 MOVV R3, R23 // R23 is unchanged by C code MOVV R3, R25 @@ -291,7 +259,7 @@ nosaveg: JAL (R20) finish: - MOVV 0(R3), R7 // sec + MOVV 0(R3), R4 // sec MOVV 8(R3), R5 // nsec MOVV R23, R3 // restore SP @@ -305,8 +273,6 @@ finish: MOVV 8(R3), R25 MOVV R25, m_vdsoPC(R24) - MOVV R7, sec+0(FP) - MOVW R5, nsec+8(FP) RET fallback: @@ -315,7 +281,7 @@ fallback: JMP finish // func nanotime1() int64 -TEXT runtime·nanotime1(SB),NOSPLIT,$16-8 +TEXT runtime·nanotime1(SB),NOSPLIT,$24 MOVV R3, R23 // R23 is unchanged by C code MOVV R3, R25 @@ -389,8 +355,7 @@ finish: // return nsec in R7 MOVV $1000000000, R4 MULVU R4, R7, R7 - ADDVU R5, R7 - MOVV R7, ret+0(FP) + ADDVU R5, R7, R4 RET fallback: @@ -399,11 +364,7 @@ fallback: JMP finish // func rtsigprocmask(how int32, new, old *sigset, size int32) -TEXT runtime·rtsigprocmask(SB),NOSPLIT|NOFRAME,$0-28 - MOVW how+0(FP), R4 - MOVV new+8(FP), R5 - MOVV old+16(FP), R6 - MOVW size+24(FP), R7 +TEXT runtime·rtsigprocmask(SB),NOSPLIT,$0 MOVV $SYS_rt_sigprocmask, R11 SYSCALL MOVW $-4096, R5 @@ -412,31 +373,37 @@ TEXT runtime·rtsigprocmask(SB),NOSPLIT|NOFRAME,$0-28 RET // func rt_sigaction(sig uintptr, new, old *sigactiont, size uintptr) int32 -TEXT runtime·rt_sigaction(SB),NOSPLIT|NOFRAME,$0-36 - MOVV sig+0(FP), R4 - MOVV new+8(FP), R5 - MOVV old+16(FP), R6 - MOVV size+24(FP), R7 +TEXT runtime·rt_sigaction(SB),NOSPLIT,$0 MOVV $SYS_rt_sigaction, R11 SYSCALL - MOVW R4, ret+32(FP) + RET + +// Call the function stored in _cgo_sigaction using the GCC calling convention. +TEXT runtime·callCgoSigaction(SB),NOSPLIT,$0 + // R4: sig, R5: new, R6: old + MOVV _cgo_sigaction(SB), R7 + SUBV $16, R3 // reserve 16 bytes for sp-8 where fp may be saved. + JAL (R7) + ADDV $16, R3 + MOVW R4, R4 RET // func sigfwd(fn uintptr, sig uint32, info *siginfo, ctx unsafe.Pointer) -TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 - MOVW sig+8(FP), R4 - MOVV info+16(FP), R5 - MOVV ctx+24(FP), R6 - MOVV fn+0(FP), R20 +TEXT runtime·sigfwd(SB),NOSPLIT,$0 + // before: + // R4: fn, R5: sig, R6: info, R7: ctx + // after: + // R20: fn, R4: sig, R5: info, R6: ctx + MOVV R4, R20 + MOVV R5, R4 + MOVV R6, R5 + MOVV R7, R6 JAL (R20) RET +// Called from c-abi, R4: sig, R5: info, R6: cxt // func sigtramp(signo, ureg, ctxt unsafe.Pointer) TEXT runtime·sigtramp(SB),NOSPLIT|TOPFRAME,$168 - MOVW R4, (1*8)(R3) - MOVV R5, (2*8)(R3) - MOVV R6, (3*8)(R3) - // Save callee-save registers in the case of signal forwarding. // Please refer to https://golang.org/issue/31827 . SAVE_R22_TO_R31((4*8)) @@ -444,12 +411,13 @@ TEXT runtime·sigtramp(SB),NOSPLIT|TOPFRAME,$168 // this might be called in external code context, // where g is not set. - MOVB runtime·iscgo(SB), R4 - BEQ R4, 2(PC) + MOVB runtime·iscgo(SB), R7 + BEQ R7, 2(PC) JAL runtime·load_g(SB) - MOVV $runtime·sigtrampgo(SB), R4 - JAL (R4) + // R5 and R6 already contain info and ctx, respectively. + MOVV $runtime·sigtrampgo(SB), R7 + JAL (R7) // Restore callee-save registers. RESTORE_R22_TO_R31((4*8)) @@ -457,53 +425,136 @@ TEXT runtime·sigtramp(SB),NOSPLIT|TOPFRAME,$168 RET -// func cgoSigtramp() -TEXT runtime·cgoSigtramp(SB),NOSPLIT,$0 +// Called from c-abi, R4: sig, R5: info, R6: cxt +TEXT runtime·sigprofNonGoWrapper<>(SB),NOSPLIT,$168 + // Save callee-save registers because it's a callback from c code. + SAVE_R22_TO_R31((4*8)) + SAVE_F24_TO_F31((14*8)) + + // R4, R5 and R6 already contain sig, info and ctx, respectively. + CALL runtime·sigprofNonGo(SB) + + // Restore callee-save registers. + RESTORE_R22_TO_R31((4*8)) + RESTORE_F24_TO_F31((14*8)) + RET + +// Called from c-abi, R4: sig, R5: info, R6: cxt +TEXT runtime·cgoSigtramp(SB),NOSPLIT|NOFRAME,$0 + // The stack unwinder, presumably written in C, may not be able to + // handle Go frame correctly. So, this function is NOFRAME, and we + // save/restore LR manually. + MOVV R1, R12 + // Save R30, g because they will be clobbered, + // we need to restore them before jump to sigtramp. + MOVV R30, R13 + MOVV g, R14 + + // If no traceback function, do usual sigtramp. + MOVV runtime·cgoTraceback(SB), R15 + BEQ R15, sigtramp + + // If no traceback support function, which means that + // runtime/cgo was not linked in, do usual sigtramp. + MOVV _cgo_callers(SB), R15 + BEQ R15, sigtramp + + // Figure out if we are currently in a cgo call. + // If not, just do usual sigtramp. + CALL runtime·load_g(SB) + BEQ g, sigtrampnog // g == nil + + MOVV g_m(g), R15 + BEQ R15, sigtramp // g.m == nil + MOVW m_ncgo(R15), R16 + BEQ R16, sigtramp // g.m.ncgo = 0 + MOVV m_curg(R15), R16 + BEQ R16, sigtramp // g.m.curg == nil + MOVV g_syscallsp(R16), R17 + BEQ R17, sigtramp // g.m.curg.syscallsp == 0 + MOVV m_cgoCallers(R15), R8 // R8 is the fifth arg in C calling convention. + BEQ R8, sigtramp // g.m.cgoCallers == nil + MOVW m_cgoCallersUse(R15), R16 + BNE R16, sigtramp // g.m.cgoCallersUse != 0 + + // Jump to a function in runtime/cgo. + // That function, written in C, will call the user's traceback + // function with proper unwind info, and will then call back here. + // The first three arguments, and the fifth, are already in registers. + // Set the two remaining arguments now. + MOVV runtime·cgoTraceback(SB), R7 + MOVV $runtime·sigtramp(SB), R9 + MOVV _cgo_callers(SB), R15 + MOVV R12, R1 // restore + MOVV R13, R30 + MOVV R14, g + JMP (R15) + +sigtramp: + MOVV R12, R1 // restore + MOVV R13, R30 + MOVV R14, g JMP runtime·sigtramp(SB) -// func sysMmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (p unsafe.Pointer, err int) -TEXT runtime·sysMmap(SB),NOSPLIT|NOFRAME,$0 - MOVV addr+0(FP), R4 - MOVV n+8(FP), R5 - MOVW prot+16(FP), R6 - MOVW flags+20(FP), R7 - MOVW fd+24(FP), R8 - MOVW off+28(FP), R9 +sigtrampnog: + // Signal arrived on a non-Go thread. If this is SIGPROF, get a + // stack trace. + MOVW $27, R15 // 27 == SIGPROF + BNE R4, R15, sigtramp + + MOVV $runtime·sigprofCallersUse(SB), R16 + DBAR $0x14 +cas_again: + MOVV $1, R15 + LL (R16), R17 + BNE R17, fail + SC R15, (R16) + BEQ R15, cas_again + DBAR $0x14 + + // Jump to the traceback function in runtime/cgo. + // It will call back to sigprofNonGo, which will ignore the + // arguments passed in registers. + // First three arguments to traceback function are in registers already. + MOVV runtime·cgoTraceback(SB), R7 + MOVV $runtime·sigprofCallers(SB), R8 + MOVV $runtime·sigprofNonGoWrapper<>(SB), R9 + MOVV _cgo_callers(SB), R15 + MOVV R12, R1 // restore + MOVV R13, R30 + MOVV R14, g + JMP (R15) + +fail: + DBAR $0x14 + JMP sigtramp +// func sysMmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (p unsafe.Pointer, err int) +TEXT runtime·sysMmap(SB),NOSPLIT,$0 MOVV $SYS_mmap, R11 SYSCALL MOVW $-4096, R5 BGEU R5, R4, ok - MOVV $0, p+32(FP) - SUBVU R4, R0, R4 - MOVV R4, err+40(FP) + SUBVU R4, R0, R5 + MOVV $0, R4 RET ok: - MOVV R4, p+32(FP) - MOVV $0, err+40(FP) + MOVV $0, R5 RET // Call the function stored in _cgo_mmap using the GCC calling convention. // This must be called on the system stack. // func callCgoMmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) uintptr -TEXT runtime·callCgoMmap(SB),NOSPLIT,$0 - MOVV addr+0(FP), R4 - MOVV n+8(FP), R5 - MOVW prot+16(FP), R6 - MOVW flags+20(FP), R7 - MOVW fd+24(FP), R8 - MOVW off+28(FP), R9 +TEXT runtime·callCgoMmap(SB),NOSPLIT,$0 MOVV _cgo_mmap(SB), R13 SUBV $16, R3 // reserve 16 bytes for sp-8 where fp may be saved. JAL (R13) ADDV $16, R3 - MOVV R4, ret+32(FP) + MOVV R4, R4 RET // func sysMunmap(addr unsafe.Pointer, n uintptr) -TEXT runtime·sysMunmap(SB),NOSPLIT|NOFRAME,$0 - MOVV addr+0(FP), R4 - MOVV n+8(FP), R5 +TEXT runtime·sysMunmap(SB),NOSPLIT,$0 MOVV $SYS_munmap, R11 SYSCALL MOVW $-4096, R5 @@ -514,9 +565,7 @@ TEXT runtime·sysMunmap(SB),NOSPLIT|NOFRAME,$0 // Call the function stored in _cgo_munmap using the GCC calling convention. // This must be called on the system stack. // func callCgoMunmap(addr unsafe.Pointer, n uintptr) -TEXT runtime·callCgoMunmap(SB),NOSPLIT,$0 - MOVV addr+0(FP), R4 - MOVV n+8(FP), R5 +TEXT runtime·callCgoMunmap(SB),NOSPLIT,$0 MOVV _cgo_munmap(SB), R13 SUBV $16, R3 // reserve 16 bytes for sp-8 where fp may be saved. JAL (R13) @@ -524,38 +573,24 @@ TEXT runtime·callCgoMunmap(SB),NOSPLIT,$0 RET // func madvise(addr unsafe.Pointer, n uintptr, flags int32) -TEXT runtime·madvise(SB),NOSPLIT|NOFRAME,$0 - MOVV addr+0(FP), R4 - MOVV n+8(FP), R5 - MOVW flags+16(FP), R6 +TEXT runtime·madvise(SB),NOSPLIT,$0 MOVV $SYS_madvise, R11 SYSCALL - MOVW R4, ret+24(FP) RET // func futex(addr unsafe.Pointer, op int32, val uint32, ts, addr2 unsafe.Pointer, val3 uint32) int32 -TEXT runtime·futex(SB),NOSPLIT|NOFRAME,$0 - MOVV addr+0(FP), R4 - MOVW op+8(FP), R5 - MOVW val+12(FP), R6 - MOVV ts+16(FP), R7 - MOVV addr2+24(FP), R8 - MOVW val3+32(FP), R9 +TEXT runtime·futex(SB),NOSPLIT,$0 MOVV $SYS_futex, R11 SYSCALL - MOVW R4, ret+40(FP) RET // int64 clone(int32 flags, void *stk, M *mp, G *gp, void (*fn)(void)); -TEXT runtime·clone(SB),NOSPLIT|NOFRAME,$0 - MOVW flags+0(FP), R4 - MOVV stk+8(FP), R5 - +TEXT runtime·clone(SB),NOSPLIT,$0 // Copy mp, gp, fn off parent stack for use by child. // Careful: Linux system call clobbers ???. - MOVV mp+16(FP), R23 - MOVV gp+24(FP), R24 - MOVV fn+32(FP), R25 + MOVV R6, R23 + MOVV R7, R24 + MOVV R8, R25 MOVV R23, -8(R5) MOVV R24, -16(R5) @@ -567,8 +602,7 @@ TEXT runtime·clone(SB),NOSPLIT|NOFRAME,$0 SYSCALL // In parent, return. - BEQ R4, 3(PC) - MOVW R4, ret+40(FP) + BEQ R4, 2(PC) RET // In child, on new stack. @@ -608,9 +642,7 @@ nog: JMP -3(PC) // keep exiting // func sigaltstack(new, old *stackt) -TEXT runtime·sigaltstack(SB),NOSPLIT|NOFRAME,$0 - MOVV new+0(FP), R4 - MOVV old+8(FP), R5 +TEXT runtime·sigaltstack(SB),NOSPLIT,$0 MOVV $SYS_sigaltstack, R11 SYSCALL MOVW $-4096, R5 @@ -619,41 +651,82 @@ TEXT runtime·sigaltstack(SB),NOSPLIT|NOFRAME,$0 RET // func osyield() -TEXT runtime·osyield(SB),NOSPLIT|NOFRAME,$0 +TEXT runtime·osyield(SB),NOSPLIT,$0 MOVV $SYS_sched_yield, R11 SYSCALL RET // func sched_getaffinity(pid, len uintptr, buf *uintptr) int32 -TEXT runtime·sched_getaffinity(SB),NOSPLIT|NOFRAME,$0 - MOVV pid+0(FP), R4 - MOVV len+8(FP), R5 - MOVV buf+16(FP), R6 +TEXT runtime·sched_getaffinity(SB),NOSPLIT,$0 MOVV $SYS_sched_getaffinity, R11 SYSCALL - MOVW R4, ret+24(FP) RET // func sbrk0() uintptr -TEXT runtime·sbrk0(SB),NOSPLIT|NOFRAME,$0-8 +TEXT runtime·sbrk0(SB),NOSPLIT,$0 // Implemented as brk(NULL). MOVV $0, R4 MOVV $SYS_brk, R11 SYSCALL - MOVV R4, ret+0(FP) RET +// unimplemented, only needed for android; declared in stubs_linux.go TEXT runtime·access(SB),$0-20 - MOVV R0, 2(R0) // unimplemented, only needed for android; declared in stubs_linux.go + MOVV R0, 2(R0) MOVW R0, ret+16(FP) // for vet RET +// unimplemented, only needed for android; declared in stubs_linux.go TEXT runtime·connect(SB),$0-28 - MOVV R0, 2(R0) // unimplemented, only needed for android; declared in stubs_linux.go + MOVV R0, 2(R0) MOVW R0, ret+24(FP) // for vet RET +// unimplemented, only needed for android; declared in stubs_linux.go TEXT runtime·socket(SB),$0-20 - MOVV R0, 2(R0) // unimplemented, only needed for android; declared in stubs_linux.go + MOVV R0, 2(R0) MOVW R0, ret+16(FP) // for vet RET + +// func vgetrandom1(buf *byte, length uintptr, flags uint32, state uintptr, stateSize uintptr) int +TEXT runtime·vgetrandom1(SB),NOSPLIT,$16 + MOVV R3, R23 + + MOVV runtime·vdsoGetrandomSym(SB), R12 + + MOVV g_m(g), R24 + + MOVV m_vdsoPC(R24), R13 + MOVV R13, 8(R3) + MOVV m_vdsoSP(R24), R13 + MOVV R13, 16(R3) + MOVV R1, m_vdsoPC(R24) + MOVV $buf-8(FP), R13 + MOVV R13, m_vdsoSP(R24) + + AND $~15, R3 + + MOVBU runtime·iscgo(SB), R13 + BNE R13, nosaveg + MOVV m_gsignal(R24), R13 + BEQ R13, nosaveg + BEQ g, R13, nosaveg + MOVV (g_stack+stack_lo)(R13), R25 + MOVV g, (R25) + + JAL (R12) + + MOVV R0, (R25) + JMP restore + +nosaveg: + JAL (R12) + +restore: + MOVV R23, R3 + MOVV 16(R3), R25 + MOVV R25, m_vdsoSP(R24) + MOVV 8(R3), R25 + MOVV R25, m_vdsoPC(R24) + NOP R4 // Satisfy go vet, since the return value comes from the vDSO function. + RET diff --git a/src/runtime/sys_linux_mipsx.s b/src/runtime/sys_linux_mipsx.s index 5e6b6c150418d2..bc3f84fbb9591e 100644 --- a/src/runtime/sys_linux_mipsx.s +++ b/src/runtime/sys_linux_mipsx.s @@ -34,10 +34,12 @@ #define SYS_mincore 4217 #define SYS_gettid 4222 #define SYS_futex 4238 +#define SYS_futex_time64 4422 #define SYS_sched_getaffinity 4240 #define SYS_exit_group 4246 #define SYS_timer_create 4257 #define SYS_timer_settime 4258 +#define SYS_timer_settime64 4409 #define SYS_timer_delete 4261 #define SYS_clock_gettime 4263 #define SYS_tgkill 4266 @@ -196,7 +198,8 @@ TEXT runtime·timer_create(SB),NOSPLIT,$0-16 MOVW R2, ret+12(FP) RET -TEXT runtime·timer_settime(SB),NOSPLIT,$0-20 +// Linux: kernel/time/posix-timer.c, requiring COMPAT_32BIT_TIME +TEXT runtime·timer_settime32(SB),NOSPLIT,$0-20 MOVW timerid+0(FP), R4 MOVW flags+4(FP), R5 MOVW new+8(FP), R6 @@ -206,6 +209,16 @@ TEXT runtime·timer_settime(SB),NOSPLIT,$0-20 MOVW R2, ret+16(FP) RET +TEXT runtime·timer_settime64(SB),NOSPLIT,$0-20 + MOVW timerid+0(FP), R4 + MOVW flags+4(FP), R5 + MOVW new+8(FP), R6 + MOVW old+12(FP), R7 + MOVW $SYS_timer_settime64, R2 + SYSCALL + MOVW R2, ret+16(FP) + RET + TEXT runtime·timer_delete(SB),NOSPLIT,$0-8 MOVW timerid+0(FP), R4 MOVW $SYS_timer_delete, R2 @@ -362,8 +375,10 @@ TEXT runtime·madvise(SB),NOSPLIT,$0-16 MOVW R2, ret+12(FP) RET -// int32 futex(int32 *uaddr, int32 op, int32 val, struct timespec *timeout, int32 *uaddr2, int32 val2); -TEXT runtime·futex(SB),NOSPLIT,$20-28 +// Linux: kernel/futex/syscalls.c, requiring COMPAT_32BIT_TIME +// int32 futex(int32 *uaddr, int32 op, int32 val, +// struct old_timespec32 *timeout, int32 *uaddr2, int32 val2); +TEXT runtime·futex_time32(SB),NOSPLIT,$20-28 MOVW addr+0(FP), R4 MOVW op+4(FP), R5 MOVW val+8(FP), R6 @@ -382,6 +397,27 @@ TEXT runtime·futex(SB),NOSPLIT,$20-28 MOVW R2, ret+24(FP) RET +// Linux: kernel/futex/syscalls.c +// int32 futex(int32 *uaddr, int32 op, int32 val, +// struct timespec *timeout, int32 *uaddr2, int32 val2); +TEXT runtime·futex_time64(SB),NOSPLIT,$20-28 + MOVW addr+0(FP), R4 + MOVW op+4(FP), R5 + MOVW val+8(FP), R6 + MOVW ts+12(FP), R7 + + MOVW addr2+16(FP), R8 + MOVW val3+20(FP), R9 + + MOVW R8, 16(R29) + MOVW R9, 20(R29) + + MOVW $SYS_futex_time64, R2 + SYSCALL + BEQ R7, 2(PC) + SUBU R2, R0, R2 // caller expects negative errno + MOVW R2, ret+24(FP) + RET // int32 clone(int32 flags, void *stk, M *mp, G *gp, void (*fn)(void)); TEXT runtime·clone(SB),NOSPLIT|NOFRAME,$0-24 diff --git a/src/runtime/sys_linux_ppc64x.s b/src/runtime/sys_linux_ppc64x.s index ba4988b7233226..8735b932ed76d9 100644 --- a/src/runtime/sys_linux_ppc64x.s +++ b/src/runtime/sys_linux_ppc64x.s @@ -757,3 +757,53 @@ TEXT runtime·socket(SB),$0-20 MOVD R0, 0(R0) // unimplemented, only needed for android; declared in stubs_linux.go MOVW R0, ret+16(FP) // for vet RET + +// func vgetrandom1(buf *byte, length uintptr, flags uint32, state uintptr, stateSize uintptr) int +TEXT runtime·vgetrandom1(SB),NOSPLIT,$16-48 + MOVD R1, R15 + + MOVD runtime·vdsoGetrandomSym(SB), R12 + MOVD R12, CTR + MOVD g_m(g), R21 + + MOVD m_vdsoPC(R21), R22 + MOVD R22, 32(R1) + MOVD m_vdsoSP(R21), R22 + MOVD R22, 40(R1) + MOVD LR, m_vdsoPC(R21) + MOVD $buf-FIXED_FRAME(FP), R22 + MOVD R22, m_vdsoSP(R21) + + RLDICR $0, R1, $59, R1 + + MOVBZ runtime·iscgo(SB), R22 + CMP R22, $0 + BNE nosaveg + MOVD m_gsignal(R21), R22 + CMP R22, $0 + BEQ nosaveg + CMP R22, g + BEQ nosaveg + MOVD (g_stack+stack_lo)(R22), R22 + MOVD g, (R22) + + BL (CTR) + + MOVD $0, (R22) + JMP restore + +nosaveg: + BL (CTR) + +restore: + MOVD $0, R0 + MOVD R15, R1 + MOVD 40(R1), R22 + MOVD R22, m_vdsoSP(R21) + MOVD 32(R1), R22 + MOVD R22, m_vdsoPC(R21) + + BVC out + NEG R3, R3 +out: + RET diff --git a/src/runtime/sys_linux_riscv64.s b/src/runtime/sys_linux_riscv64.s index ffec2b5b759e92..0d4fb3b5d9a66e 100644 --- a/src/runtime/sys_linux_riscv64.s +++ b/src/runtime/sys_linux_riscv64.s @@ -51,8 +51,7 @@ #define SYS_write 64 // func exit(code int32) -TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 - MOVW code+0(FP), A0 +TEXT runtime·exit(SB),NOSPLIT,$0 MOV $SYS_exit_group, A7 ECALL RET @@ -95,23 +94,15 @@ TEXT runtime·closefd(SB),NOSPLIT|NOFRAME,$0-12 RET // func write1(fd uintptr, p unsafe.Pointer, n int32) int32 -TEXT runtime·write1(SB),NOSPLIT|NOFRAME,$0-28 - MOV fd+0(FP), A0 - MOV p+8(FP), A1 - MOVW n+16(FP), A2 +TEXT runtime·write1(SB),NOSPLIT,$0 MOV $SYS_write, A7 ECALL - MOVW A0, ret+24(FP) RET // func read(fd int32, p unsafe.Pointer, n int32) int32 -TEXT runtime·read(SB),NOSPLIT|NOFRAME,$0-28 - MOVW fd+0(FP), A0 - MOV p+8(FP), A1 - MOVW n+16(FP), A2 +TEXT runtime·read(SB),NOSPLIT,$0 MOV $SYS_read, A7 ECALL - MOVW A0, ret+24(FP) RET // func pipe2(flags int32) (r, w int32, errno int32) @@ -140,10 +131,9 @@ TEXT runtime·usleep(SB),NOSPLIT,$24-4 RET // func gettid() uint32 -TEXT runtime·gettid(SB),NOSPLIT,$0-4 +TEXT runtime·gettid(SB),NOSPLIT,$0 MOV $SYS_gettid, A7 ECALL - MOVW A0, ret+0(FP) RET // func raise(sig uint32) @@ -167,67 +157,45 @@ TEXT runtime·raiseproc(SB),NOSPLIT|NOFRAME,$0 RET // func getpid() int -TEXT ·getpid(SB),NOSPLIT|NOFRAME,$0-8 +TEXT ·getpid(SB),NOSPLIT,$0 MOV $SYS_getpid, A7 ECALL - MOV A0, ret+0(FP) RET // func tgkill(tgid, tid, sig int) -TEXT ·tgkill(SB),NOSPLIT|NOFRAME,$0-24 - MOV tgid+0(FP), A0 - MOV tid+8(FP), A1 - MOV sig+16(FP), A2 +TEXT ·tgkill(SB),NOSPLIT,$0 MOV $SYS_tgkill, A7 ECALL RET // func setitimer(mode int32, new, old *itimerval) -TEXT runtime·setitimer(SB),NOSPLIT|NOFRAME,$0-24 - MOVW mode+0(FP), A0 - MOV new+8(FP), A1 - MOV old+16(FP), A2 +TEXT runtime·setitimer(SB),NOSPLIT,$0 MOV $SYS_setitimer, A7 ECALL RET // func timer_create(clockid int32, sevp *sigevent, timerid *int32) int32 -TEXT runtime·timer_create(SB),NOSPLIT,$0-28 - MOVW clockid+0(FP), A0 - MOV sevp+8(FP), A1 - MOV timerid+16(FP), A2 +TEXT runtime·timer_create(SB),NOSPLIT,$0 MOV $SYS_timer_create, A7 ECALL - MOVW A0, ret+24(FP) RET // func timer_settime(timerid int32, flags int32, new, old *itimerspec) int32 -TEXT runtime·timer_settime(SB),NOSPLIT,$0-28 - MOVW timerid+0(FP), A0 - MOVW flags+4(FP), A1 - MOV new+8(FP), A2 - MOV old+16(FP), A3 +TEXT runtime·timer_settime(SB),NOSPLIT,$0 MOV $SYS_timer_settime, A7 ECALL - MOVW A0, ret+24(FP) RET // func timer_delete(timerid int32) int32 -TEXT runtime·timer_delete(SB),NOSPLIT,$0-12 - MOVW timerid+0(FP), A0 +TEXT runtime·timer_delete(SB),NOSPLIT,$0 MOV $SYS_timer_delete, A7 ECALL - MOVW A0, ret+8(FP) RET // func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32 -TEXT runtime·mincore(SB),NOSPLIT|NOFRAME,$0-28 - MOV addr+0(FP), A0 - MOV n+8(FP), A1 - MOV dst+16(FP), A2 +TEXT runtime·mincore(SB),NOSPLIT,$0 MOV $SYS_mincore, A7 ECALL - MOVW A0, ret+24(FP) RET // func walltime() (sec int64, nsec int32) @@ -393,14 +361,9 @@ TEXT runtime·rtsigprocmask(SB),NOSPLIT|NOFRAME,$0-28 RET // func rt_sigaction(sig uintptr, new, old *sigactiont, size uintptr) int32 -TEXT runtime·rt_sigaction(SB),NOSPLIT|NOFRAME,$0-36 - MOV sig+0(FP), A0 - MOV new+8(FP), A1 - MOV old+16(FP), A2 - MOV size+24(FP), A3 +TEXT runtime·rt_sigaction(SB),NOSPLIT,$0 MOV $SYS_rt_sigaction, A7 ECALL - MOVW A0, ret+32(FP) RET // func sigfwd(fn uintptr, sig uint32, info *siginfo, ctx unsafe.Pointer) @@ -466,26 +429,15 @@ TEXT runtime·munmap(SB),NOSPLIT|NOFRAME,$0 RET // func madvise(addr unsafe.Pointer, n uintptr, flags int32) -TEXT runtime·madvise(SB),NOSPLIT|NOFRAME,$0 - MOV addr+0(FP), A0 - MOV n+8(FP), A1 - MOVW flags+16(FP), A2 +TEXT runtime·madvise(SB),NOSPLIT,$0 MOV $SYS_madvise, A7 ECALL - MOVW A0, ret+24(FP) RET // func futex(addr unsafe.Pointer, op int32, val uint32, ts, addr2 unsafe.Pointer, val3 uint32) int32 -TEXT runtime·futex(SB),NOSPLIT|NOFRAME,$0 - MOV addr+0(FP), A0 - MOVW op+8(FP), A1 - MOVW val+12(FP), A2 - MOV ts+16(FP), A3 - MOV addr2+24(FP), A4 - MOVW val3+32(FP), A5 +TEXT runtime·futex(SB),NOSPLIT,$0 MOV $SYS_futex, A7 ECALL - MOVW A0, ret+40(FP) RET // func clone(flags int32, stk, mp, gp, fn unsafe.Pointer) int32 @@ -559,26 +511,21 @@ TEXT runtime·sigaltstack(SB),NOSPLIT|NOFRAME,$0 RET // func osyield() -TEXT runtime·osyield(SB),NOSPLIT|NOFRAME,$0 +TEXT runtime·osyield(SB),NOSPLIT,$0 MOV $SYS_sched_yield, A7 ECALL RET // func sched_getaffinity(pid, len uintptr, buf *uintptr) int32 -TEXT runtime·sched_getaffinity(SB),NOSPLIT|NOFRAME,$0 - MOV pid+0(FP), A0 - MOV len+8(FP), A1 - MOV buf+16(FP), A2 +TEXT runtime·sched_getaffinity(SB),NOSPLIT,$0 MOV $SYS_sched_getaffinity, A7 ECALL - MOV A0, ret+24(FP) RET // func sbrk0() uintptr -TEXT runtime·sbrk0(SB),NOSPLIT,$0-8 +TEXT runtime·sbrk0(SB),NOSPLIT,$0 // Implemented as brk(NULL). - MOV $0, A0 + MOV ZERO, A0 MOV $SYS_brk, A7 ECALL - MOVW A0, ret+0(FP) RET diff --git a/src/runtime/sys_linux_s390x.s b/src/runtime/sys_linux_s390x.s index adf5612c3cfbee..a3472a4508720b 100644 --- a/src/runtime/sys_linux_s390x.s +++ b/src/runtime/sys_linux_s390x.s @@ -112,9 +112,10 @@ TEXT runtime·usleep(SB),NOSPLIT,$16-4 MOVW $1000000, R3 DIVD R3, R2 MOVD R2, 8(R15) - MOVW $1000, R3 - MULLD R2, R3 + MULLD R2, R3 // Convert sec to usec and subtract SUB R3, R4 + MOVW $1000, R3 + MULLD R3, R4 // Convert remaining usec into nsec. MOVD R4, 16(R15) // nanosleep(&ts, 0) @@ -225,7 +226,7 @@ TEXT runtime·walltime(SB),NOSPLIT,$32-12 MOVD R4, 24(R15) MOVD R14, R8 // Backup return address - MOVD $sec+0(FP), R4 // return parameter caller + MOVD $ret-8(FP), R4 // caller's SP MOVD R8, m_vdsoPC(R6) MOVD R4, m_vdsoSP(R6) @@ -311,7 +312,7 @@ TEXT runtime·nanotime1(SB),NOSPLIT,$32-8 MOVD R4, 24(R15) MOVD R14, R8 // Backup return address - MOVD $ret+0(FP), R4 // caller's SP + MOVD $ret-8(FP), R4 // caller's SP MOVD R8, m_vdsoPC(R6) MOVD R4, m_vdsoSP(R6) @@ -604,3 +605,53 @@ TEXT runtime·socket(SB),$0-20 MOVD $0, 2(R0) // unimplemented, only needed for android; declared in stubs_linux.go MOVW R0, ret+16(FP) RET + +// func vgetrandom1(buf *byte, length uintptr, flags uint32, state uintptr, stateSize uintptr) int +TEXT runtime·vgetrandom1(SB),NOSPLIT,$16-48 + MOVD buf+0(FP), R2 + MOVD length+8(FP), R3 + MOVW flags+16(FP), R4 + MOVD state+24(FP), R5 + MOVD stateSize+32(FP), R6 + + MOVD R15, R7 + + MOVD runtime·vdsoGetrandomSym(SB), R1 + MOVD g_m(g), R9 + + MOVD m_vdsoPC(R9), R12 + MOVD R12, 8(R15) + MOVD m_vdsoSP(R9), R12 + MOVD R12, 16(R15) + MOVD R14, m_vdsoPC(R9) + MOVD $buf+0(FP), R12 + MOVD R12, m_vdsoSP(R9) + + SUB $160, R15 + MOVD $~7, R12 + AND R12, R15 + + MOVB runtime·iscgo(SB), R12 + CMPBNE R12, $0, nosaveg + MOVD m_gsignal(R9), R12 + CMPBEQ R12, $0, nosaveg + CMPBEQ g, R12, nosaveg + MOVD (g_stack+stack_lo)(R12), R12 + MOVD g, (R12) + + BL R1 + + MOVD $0, (R12) + JMP restore + +nosaveg: + BL R1 + +restore: + MOVD R7, R15 + MOVD 16(R15), R12 + MOVD R12, m_vdsoSP(R9) + MOVD 8(R15), R12 + MOVD R12, m_vdsoPC(R9) + MOVD R2, ret+40(FP) + RET diff --git a/src/runtime/sys_openbsd.go b/src/runtime/sys_openbsd.go index c4b8489612a5db..df503d24c67fee 100644 --- a/src/runtime/sys_openbsd.go +++ b/src/runtime/sys_openbsd.go @@ -2,12 +2,11 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build openbsd && !mips64 - package runtime import ( "internal/abi" + "internal/runtime/atomic" "unsafe" ) @@ -61,6 +60,412 @@ func pthread_create(attr *pthreadattr, start uintptr, arg unsafe.Pointer) int32 } func pthread_create_trampoline() +//go:nosplit +//go:cgo_unsafe_args +func thrsleep(ident uintptr, clock_id int32, tsp *timespec, lock uintptr, abort *uint32) int32 { + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(thrsleep_trampoline)), unsafe.Pointer(&ident)) + KeepAlive(tsp) + KeepAlive(abort) + return ret +} +func thrsleep_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func thrwakeup(ident uintptr, n int32) int32 { + return libcCall(unsafe.Pointer(abi.FuncPCABI0(thrwakeup_trampoline)), unsafe.Pointer(&ident)) +} +func thrwakeup_trampoline() + +//go:nosplit +func osyield() { + libcCall(unsafe.Pointer(abi.FuncPCABI0(sched_yield_trampoline)), unsafe.Pointer(nil)) +} +func sched_yield_trampoline() + +//go:nosplit +func osyield_no_g() { + asmcgocall_no_g(unsafe.Pointer(abi.FuncPCABI0(sched_yield_trampoline)), unsafe.Pointer(nil)) +} + +// This is exported via linkname to assembly in runtime/cgo. +// +//go:linkname exit +//go:nosplit +//go:cgo_unsafe_args +func exit(code int32) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(exit_trampoline)), unsafe.Pointer(&code)) +} +func exit_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func getthrid() (tid int32) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(getthrid_trampoline)), unsafe.Pointer(&tid)) + return +} +func getthrid_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func raiseproc(sig uint32) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(raiseproc_trampoline)), unsafe.Pointer(&sig)) +} +func raiseproc_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func thrkill(tid int32, sig int) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(thrkill_trampoline)), unsafe.Pointer(&tid)) +} +func thrkill_trampoline() + +// mmap is used to do low-level memory allocation via mmap. Don't allow stack +// splits, since this function (used by sysAlloc) is called in a lot of low-level +// parts of the runtime and callers often assume it won't acquire any locks. +// +//go:nosplit +func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (unsafe.Pointer, int) { + args := struct { + addr unsafe.Pointer + n uintptr + prot, flags, fd int32 + off uint32 + ret1 unsafe.Pointer + ret2 int + }{addr, n, prot, flags, fd, off, nil, 0} + libcCall(unsafe.Pointer(abi.FuncPCABI0(mmap_trampoline)), unsafe.Pointer(&args)) + KeepAlive(addr) // Just for consistency. Hopefully addr is not a Go address. + return args.ret1, args.ret2 +} +func mmap_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func munmap(addr unsafe.Pointer, n uintptr) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(munmap_trampoline)), unsafe.Pointer(&addr)) + KeepAlive(addr) // Just for consistency. Hopefully addr is not a Go address. +} +func munmap_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func madvise(addr unsafe.Pointer, n uintptr, flags int32) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(madvise_trampoline)), unsafe.Pointer(&addr)) + KeepAlive(addr) // Just for consistency. Hopefully addr is not a Go address. +} +func madvise_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func open(name *byte, mode, perm int32) (ret int32) { + ret = libcCall(unsafe.Pointer(abi.FuncPCABI0(open_trampoline)), unsafe.Pointer(&name)) + KeepAlive(name) + return +} +func open_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func closefd(fd int32) int32 { + return libcCall(unsafe.Pointer(abi.FuncPCABI0(close_trampoline)), unsafe.Pointer(&fd)) +} +func close_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func read(fd int32, p unsafe.Pointer, n int32) int32 { + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(read_trampoline)), unsafe.Pointer(&fd)) + KeepAlive(p) + return ret +} +func read_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func write1(fd uintptr, p unsafe.Pointer, n int32) int32 { + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(write_trampoline)), unsafe.Pointer(&fd)) + KeepAlive(p) + return ret +} +func write_trampoline() + +func pipe2(flags int32) (r, w int32, errno int32) { + var p [2]int32 + args := struct { + p unsafe.Pointer + flags int32 + }{noescape(unsafe.Pointer(&p)), flags} + errno = libcCall(unsafe.Pointer(abi.FuncPCABI0(pipe2_trampoline)), unsafe.Pointer(&args)) + return p[0], p[1], errno +} +func pipe2_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func setitimer(mode int32, new, old *itimerval) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(setitimer_trampoline)), unsafe.Pointer(&mode)) + KeepAlive(new) + KeepAlive(old) +} +func setitimer_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func usleep(usec uint32) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(usleep_trampoline)), unsafe.Pointer(&usec)) +} +func usleep_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func usleep_no_g(usec uint32) { + asmcgocall_no_g(unsafe.Pointer(abi.FuncPCABI0(usleep_trampoline)), unsafe.Pointer(&usec)) +} + +//go:nosplit +//go:cgo_unsafe_args +func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 { + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(sysctl_trampoline)), unsafe.Pointer(&mib)) + KeepAlive(mib) + KeepAlive(out) + KeepAlive(size) + KeepAlive(dst) + return ret +} +func sysctl_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func fcntl(fd, cmd, arg int32) (ret int32, errno int32) { + args := struct { + fd, cmd, arg int32 + ret, errno int32 + }{fd, cmd, arg, 0, 0} + libcCall(unsafe.Pointer(abi.FuncPCABI0(fcntl_trampoline)), unsafe.Pointer(&args)) + return args.ret, args.errno +} +func fcntl_trampoline() + +//go:nosplit +func nanotime1() int64 { + var ts timespec + args := struct { + clock_id int32 + tp unsafe.Pointer + }{_CLOCK_MONOTONIC, unsafe.Pointer(&ts)} + if errno := libcCall(unsafe.Pointer(abi.FuncPCABI0(clock_gettime_trampoline)), unsafe.Pointer(&args)); errno < 0 { + // Avoid growing the nosplit stack. + systemstack(func() { + println("runtime: errno", -errno) + throw("clock_gettime failed") + }) + } + return ts.tv_sec*1e9 + int64(ts.tv_nsec) +} +func clock_gettime_trampoline() + +//go:nosplit +func walltime() (int64, int32) { + var ts timespec + args := struct { + clock_id int32 + tp unsafe.Pointer + }{_CLOCK_REALTIME, unsafe.Pointer(&ts)} + if errno := libcCall(unsafe.Pointer(abi.FuncPCABI0(clock_gettime_trampoline)), unsafe.Pointer(&args)); errno < 0 { + // Avoid growing the nosplit stack. + systemstack(func() { + println("runtime: errno", -errno) + throw("clock_gettime failed") + }) + } + return ts.tv_sec, int32(ts.tv_nsec) +} + +//go:nosplit +//go:cgo_unsafe_args +func kqueue() int32 { + return libcCall(unsafe.Pointer(abi.FuncPCABI0(kqueue_trampoline)), nil) +} +func kqueue_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func kevent(kq int32, ch *keventt, nch int32, ev *keventt, nev int32, ts *timespec) int32 { + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(kevent_trampoline)), unsafe.Pointer(&kq)) + KeepAlive(ch) + KeepAlive(ev) + KeepAlive(ts) + return ret +} +func kevent_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func sigaction(sig uint32, new *sigactiont, old *sigactiont) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(sigaction_trampoline)), unsafe.Pointer(&sig)) + KeepAlive(new) + KeepAlive(old) +} +func sigaction_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func sigprocmask(how uint32, new *sigset, old *sigset) { + // sigprocmask is called from sigsave, which is called from needm. + // As such, we have to be able to run with no g here. + asmcgocall_no_g(unsafe.Pointer(abi.FuncPCABI0(sigprocmask_trampoline)), unsafe.Pointer(&how)) + KeepAlive(new) + KeepAlive(old) +} +func sigprocmask_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func sigaltstack(new *stackt, old *stackt) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(sigaltstack_trampoline)), unsafe.Pointer(&new)) + KeepAlive(new) + KeepAlive(old) +} +func sigaltstack_trampoline() + +// Not used on OpenBSD, but must be defined. +func exitThread(wait *atomic.Uint32) { + throw("exitThread") +} + +//go:nosplit +//go:cgo_unsafe_args +func issetugid() (ret int32) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(issetugid_trampoline)), unsafe.Pointer(&ret)) + return +} +func issetugid_trampoline() + +// The X versions of syscall expect the libc call to return a 64-bit result. +// Otherwise (the non-X version) expects a 32-bit result. +// This distinction is required because an error is indicated by returning -1, +// and we need to know whether to check 32 or 64 bits of the result. +// (Some libc functions that return 32 bits put junk in the upper 32 bits of AX.) + +// golang.org/x/sys linknames syscall_syscall +// (in addition to standard package syscall). +// Do not remove or change the type signature. +// +//go:linkname syscall_syscall syscall.syscall +//go:nosplit +//go:cgo_unsafe_args +func syscall_syscall(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { + entersyscall() + libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall)), unsafe.Pointer(&fn)) + exitsyscall() + return +} +func syscall() + +//go:linkname syscall_syscallX syscall.syscallX +//go:nosplit +//go:cgo_unsafe_args +func syscall_syscallX(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { + entersyscall() + libcCall(unsafe.Pointer(abi.FuncPCABI0(syscallX)), unsafe.Pointer(&fn)) + exitsyscall() + return +} +func syscallX() + +// golang.org/x/sys linknames syscall.syscall6 +// (in addition to standard package syscall). +// Do not remove or change the type signature. +// +//go:linkname syscall_syscall6 syscall.syscall6 +//go:nosplit +//go:cgo_unsafe_args +func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { + entersyscall() + libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall6)), unsafe.Pointer(&fn)) + exitsyscall() + return +} +func syscall6() + +//go:linkname syscall_syscall6X syscall.syscall6X +//go:nosplit +//go:cgo_unsafe_args +func syscall_syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { + entersyscall() + libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall6X)), unsafe.Pointer(&fn)) + exitsyscall() + return +} +func syscall6X() + +// golang.org/x/sys linknames syscall.syscall10 +// (in addition to standard package syscall). +// Do not remove or change the type signature. +// +//go:linkname syscall_syscall10 syscall.syscall10 +//go:nosplit +//go:cgo_unsafe_args +func syscall_syscall10(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10 uintptr) (r1, r2, err uintptr) { + entersyscall() + libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall10)), unsafe.Pointer(&fn)) + exitsyscall() + return +} +func syscall10() + +//go:linkname syscall_syscall10X syscall.syscall10X +//go:nosplit +//go:cgo_unsafe_args +func syscall_syscall10X(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10 uintptr) (r1, r2, err uintptr) { + entersyscall() + libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall10X)), unsafe.Pointer(&fn)) + exitsyscall() + return +} +func syscall10X() + +// golang.org/x/sys linknames syscall_rawSyscall +// (in addition to standard package syscall). +// Do not remove or change the type signature. +// +//go:linkname syscall_rawSyscall syscall.rawSyscall +//go:nosplit +//go:cgo_unsafe_args +func syscall_rawSyscall(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall)), unsafe.Pointer(&fn)) + return +} + +// golang.org/x/sys linknames syscall_rawSyscall6 +// (in addition to standard package syscall). +// Do not remove or change the type signature. +// +//go:linkname syscall_rawSyscall6 syscall.rawSyscall6 +//go:nosplit +//go:cgo_unsafe_args +func syscall_rawSyscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall6)), unsafe.Pointer(&fn)) + return +} + +//go:linkname syscall_rawSyscall6X syscall.rawSyscall6X +//go:nosplit +//go:cgo_unsafe_args +func syscall_rawSyscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall6X)), unsafe.Pointer(&fn)) + return +} + +//go:linkname syscall_rawSyscall10X syscall.rawSyscall10X +//go:nosplit +//go:cgo_unsafe_args +func syscall_rawSyscall10X(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10 uintptr) (r1, r2, err uintptr) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall10X)), unsafe.Pointer(&fn)) + return +} + // Tell the linker that the libc_* functions are to be found // in a system library, with the libc_ prefix missing. @@ -71,5 +476,40 @@ func pthread_create_trampoline() //go:cgo_import_dynamic libc_pthread_create pthread_create "libpthread.so" //go:cgo_import_dynamic libc_pthread_sigmask pthread_sigmask "libpthread.so" +//go:cgo_import_dynamic libc_thrsleep __thrsleep "libc.so" +//go:cgo_import_dynamic libc_thrwakeup __thrwakeup "libc.so" +//go:cgo_import_dynamic libc_sched_yield sched_yield "libc.so" + +//go:cgo_import_dynamic libc_errno __errno "libc.so" +//go:cgo_import_dynamic libc_exit exit "libc.so" +//go:cgo_import_dynamic libc_getthrid getthrid "libc.so" +//go:cgo_import_dynamic libc_sched_yield sched_yield "libc.so" +//go:cgo_import_dynamic libc_thrkill thrkill "libc.so" + +//go:cgo_import_dynamic libc_mmap mmap "libc.so" +//go:cgo_import_dynamic libc_munmap munmap "libc.so" +//go:cgo_import_dynamic libc_madvise madvise "libc.so" + +//go:cgo_import_dynamic libc_open open "libc.so" +//go:cgo_import_dynamic libc_close close "libc.so" +//go:cgo_import_dynamic libc_read read "libc.so" +//go:cgo_import_dynamic libc_write write "libc.so" +//go:cgo_import_dynamic libc_pipe2 pipe2 "libc.so" + +//go:cgo_import_dynamic libc_clock_gettime clock_gettime "libc.so" +//go:cgo_import_dynamic libc_setitimer setitimer "libc.so" +//go:cgo_import_dynamic libc_usleep usleep "libc.so" +//go:cgo_import_dynamic libc_sysctl sysctl "libc.so" +//go:cgo_import_dynamic libc_fcntl fcntl "libc.so" +//go:cgo_import_dynamic libc_getpid getpid "libc.so" +//go:cgo_import_dynamic libc_kill kill "libc.so" +//go:cgo_import_dynamic libc_kqueue kqueue "libc.so" +//go:cgo_import_dynamic libc_kevent kevent "libc.so" + +//go:cgo_import_dynamic libc_sigaction sigaction "libc.so" +//go:cgo_import_dynamic libc_sigaltstack sigaltstack "libc.so" + +//go:cgo_import_dynamic libc_issetugid issetugid "libc.so" + //go:cgo_import_dynamic _ _ "libpthread.so" //go:cgo_import_dynamic _ _ "libc.so" diff --git a/src/runtime/sys_openbsd1.go b/src/runtime/sys_openbsd1.go deleted file mode 100644 index d852e3c58ae0d5..00000000000000 --- a/src/runtime/sys_openbsd1.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build openbsd && !mips64 - -package runtime - -import ( - "internal/abi" - "unsafe" -) - -//go:nosplit -//go:cgo_unsafe_args -func thrsleep(ident uintptr, clock_id int32, tsp *timespec, lock uintptr, abort *uint32) int32 { - ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(thrsleep_trampoline)), unsafe.Pointer(&ident)) - KeepAlive(tsp) - KeepAlive(abort) - return ret -} -func thrsleep_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func thrwakeup(ident uintptr, n int32) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(thrwakeup_trampoline)), unsafe.Pointer(&ident)) -} -func thrwakeup_trampoline() - -//go:nosplit -func osyield() { - libcCall(unsafe.Pointer(abi.FuncPCABI0(sched_yield_trampoline)), unsafe.Pointer(nil)) -} -func sched_yield_trampoline() - -//go:nosplit -func osyield_no_g() { - asmcgocall_no_g(unsafe.Pointer(abi.FuncPCABI0(sched_yield_trampoline)), unsafe.Pointer(nil)) -} - -//go:cgo_import_dynamic libc_thrsleep __thrsleep "libc.so" -//go:cgo_import_dynamic libc_thrwakeup __thrwakeup "libc.so" -//go:cgo_import_dynamic libc_sched_yield sched_yield "libc.so" - -//go:cgo_import_dynamic _ _ "libc.so" diff --git a/src/runtime/sys_openbsd2.go b/src/runtime/sys_openbsd2.go deleted file mode 100644 index 8f5242018dd08a..00000000000000 --- a/src/runtime/sys_openbsd2.go +++ /dev/null @@ -1,303 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build openbsd && !mips64 - -package runtime - -import ( - "internal/abi" - "internal/runtime/atomic" - "unsafe" -) - -// This is exported via linkname to assembly in runtime/cgo. -// -//go:linkname exit -//go:nosplit -//go:cgo_unsafe_args -func exit(code int32) { - libcCall(unsafe.Pointer(abi.FuncPCABI0(exit_trampoline)), unsafe.Pointer(&code)) -} -func exit_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func getthrid() (tid int32) { - libcCall(unsafe.Pointer(abi.FuncPCABI0(getthrid_trampoline)), unsafe.Pointer(&tid)) - return -} -func getthrid_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func raiseproc(sig uint32) { - libcCall(unsafe.Pointer(abi.FuncPCABI0(raiseproc_trampoline)), unsafe.Pointer(&sig)) -} -func raiseproc_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func thrkill(tid int32, sig int) { - libcCall(unsafe.Pointer(abi.FuncPCABI0(thrkill_trampoline)), unsafe.Pointer(&tid)) -} -func thrkill_trampoline() - -// mmap is used to do low-level memory allocation via mmap. Don't allow stack -// splits, since this function (used by sysAlloc) is called in a lot of low-level -// parts of the runtime and callers often assume it won't acquire any locks. -// -//go:nosplit -func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (unsafe.Pointer, int) { - args := struct { - addr unsafe.Pointer - n uintptr - prot, flags, fd int32 - off uint32 - ret1 unsafe.Pointer - ret2 int - }{addr, n, prot, flags, fd, off, nil, 0} - libcCall(unsafe.Pointer(abi.FuncPCABI0(mmap_trampoline)), unsafe.Pointer(&args)) - KeepAlive(addr) // Just for consistency. Hopefully addr is not a Go address. - return args.ret1, args.ret2 -} -func mmap_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func munmap(addr unsafe.Pointer, n uintptr) { - libcCall(unsafe.Pointer(abi.FuncPCABI0(munmap_trampoline)), unsafe.Pointer(&addr)) - KeepAlive(addr) // Just for consistency. Hopefully addr is not a Go address. -} -func munmap_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func madvise(addr unsafe.Pointer, n uintptr, flags int32) { - libcCall(unsafe.Pointer(abi.FuncPCABI0(madvise_trampoline)), unsafe.Pointer(&addr)) - KeepAlive(addr) // Just for consistency. Hopefully addr is not a Go address. -} -func madvise_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func open(name *byte, mode, perm int32) (ret int32) { - ret = libcCall(unsafe.Pointer(abi.FuncPCABI0(open_trampoline)), unsafe.Pointer(&name)) - KeepAlive(name) - return -} -func open_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func closefd(fd int32) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(close_trampoline)), unsafe.Pointer(&fd)) -} -func close_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func read(fd int32, p unsafe.Pointer, n int32) int32 { - ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(read_trampoline)), unsafe.Pointer(&fd)) - KeepAlive(p) - return ret -} -func read_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func write1(fd uintptr, p unsafe.Pointer, n int32) int32 { - ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(write_trampoline)), unsafe.Pointer(&fd)) - KeepAlive(p) - return ret -} -func write_trampoline() - -func pipe2(flags int32) (r, w int32, errno int32) { - var p [2]int32 - args := struct { - p unsafe.Pointer - flags int32 - }{noescape(unsafe.Pointer(&p)), flags} - errno = libcCall(unsafe.Pointer(abi.FuncPCABI0(pipe2_trampoline)), unsafe.Pointer(&args)) - return p[0], p[1], errno -} -func pipe2_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func setitimer(mode int32, new, old *itimerval) { - libcCall(unsafe.Pointer(abi.FuncPCABI0(setitimer_trampoline)), unsafe.Pointer(&mode)) - KeepAlive(new) - KeepAlive(old) -} -func setitimer_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func usleep(usec uint32) { - libcCall(unsafe.Pointer(abi.FuncPCABI0(usleep_trampoline)), unsafe.Pointer(&usec)) -} -func usleep_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func usleep_no_g(usec uint32) { - asmcgocall_no_g(unsafe.Pointer(abi.FuncPCABI0(usleep_trampoline)), unsafe.Pointer(&usec)) -} - -//go:nosplit -//go:cgo_unsafe_args -func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 { - ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(sysctl_trampoline)), unsafe.Pointer(&mib)) - KeepAlive(mib) - KeepAlive(out) - KeepAlive(size) - KeepAlive(dst) - return ret -} -func sysctl_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func fcntl(fd, cmd, arg int32) (ret int32, errno int32) { - args := struct { - fd, cmd, arg int32 - ret, errno int32 - }{fd, cmd, arg, 0, 0} - libcCall(unsafe.Pointer(abi.FuncPCABI0(fcntl_trampoline)), unsafe.Pointer(&args)) - return args.ret, args.errno -} -func fcntl_trampoline() - -//go:nosplit -func nanotime1() int64 { - var ts timespec - args := struct { - clock_id int32 - tp unsafe.Pointer - }{_CLOCK_MONOTONIC, unsafe.Pointer(&ts)} - if errno := libcCall(unsafe.Pointer(abi.FuncPCABI0(clock_gettime_trampoline)), unsafe.Pointer(&args)); errno < 0 { - // Avoid growing the nosplit stack. - systemstack(func() { - println("runtime: errno", -errno) - throw("clock_gettime failed") - }) - } - return ts.tv_sec*1e9 + int64(ts.tv_nsec) -} -func clock_gettime_trampoline() - -//go:nosplit -func walltime() (int64, int32) { - var ts timespec - args := struct { - clock_id int32 - tp unsafe.Pointer - }{_CLOCK_REALTIME, unsafe.Pointer(&ts)} - if errno := libcCall(unsafe.Pointer(abi.FuncPCABI0(clock_gettime_trampoline)), unsafe.Pointer(&args)); errno < 0 { - // Avoid growing the nosplit stack. - systemstack(func() { - println("runtime: errno", -errno) - throw("clock_gettime failed") - }) - } - return ts.tv_sec, int32(ts.tv_nsec) -} - -//go:nosplit -//go:cgo_unsafe_args -func kqueue() int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(kqueue_trampoline)), nil) -} -func kqueue_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func kevent(kq int32, ch *keventt, nch int32, ev *keventt, nev int32, ts *timespec) int32 { - ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(kevent_trampoline)), unsafe.Pointer(&kq)) - KeepAlive(ch) - KeepAlive(ev) - KeepAlive(ts) - return ret -} -func kevent_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func sigaction(sig uint32, new *sigactiont, old *sigactiont) { - libcCall(unsafe.Pointer(abi.FuncPCABI0(sigaction_trampoline)), unsafe.Pointer(&sig)) - KeepAlive(new) - KeepAlive(old) -} -func sigaction_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func sigprocmask(how uint32, new *sigset, old *sigset) { - // sigprocmask is called from sigsave, which is called from needm. - // As such, we have to be able to run with no g here. - asmcgocall_no_g(unsafe.Pointer(abi.FuncPCABI0(sigprocmask_trampoline)), unsafe.Pointer(&how)) - KeepAlive(new) - KeepAlive(old) -} -func sigprocmask_trampoline() - -//go:nosplit -//go:cgo_unsafe_args -func sigaltstack(new *stackt, old *stackt) { - libcCall(unsafe.Pointer(abi.FuncPCABI0(sigaltstack_trampoline)), unsafe.Pointer(&new)) - KeepAlive(new) - KeepAlive(old) -} -func sigaltstack_trampoline() - -// Not used on OpenBSD, but must be defined. -func exitThread(wait *atomic.Uint32) { - throw("exitThread") -} - -//go:nosplit -//go:cgo_unsafe_args -func issetugid() (ret int32) { - libcCall(unsafe.Pointer(abi.FuncPCABI0(issetugid_trampoline)), unsafe.Pointer(&ret)) - return -} -func issetugid_trampoline() - -// Tell the linker that the libc_* functions are to be found -// in a system library, with the libc_ prefix missing. - -//go:cgo_import_dynamic libc_errno __errno "libc.so" -//go:cgo_import_dynamic libc_exit exit "libc.so" -//go:cgo_import_dynamic libc_getthrid getthrid "libc.so" -//go:cgo_import_dynamic libc_sched_yield sched_yield "libc.so" -//go:cgo_import_dynamic libc_thrkill thrkill "libc.so" - -//go:cgo_import_dynamic libc_mmap mmap "libc.so" -//go:cgo_import_dynamic libc_munmap munmap "libc.so" -//go:cgo_import_dynamic libc_madvise madvise "libc.so" - -//go:cgo_import_dynamic libc_open open "libc.so" -//go:cgo_import_dynamic libc_close close "libc.so" -//go:cgo_import_dynamic libc_read read "libc.so" -//go:cgo_import_dynamic libc_write write "libc.so" -//go:cgo_import_dynamic libc_pipe2 pipe2 "libc.so" - -//go:cgo_import_dynamic libc_clock_gettime clock_gettime "libc.so" -//go:cgo_import_dynamic libc_setitimer setitimer "libc.so" -//go:cgo_import_dynamic libc_usleep usleep "libc.so" -//go:cgo_import_dynamic libc_sysctl sysctl "libc.so" -//go:cgo_import_dynamic libc_fcntl fcntl "libc.so" -//go:cgo_import_dynamic libc_getpid getpid "libc.so" -//go:cgo_import_dynamic libc_kill kill "libc.so" -//go:cgo_import_dynamic libc_kqueue kqueue "libc.so" -//go:cgo_import_dynamic libc_kevent kevent "libc.so" - -//go:cgo_import_dynamic libc_sigaction sigaction "libc.so" -//go:cgo_import_dynamic libc_sigaltstack sigaltstack "libc.so" - -//go:cgo_import_dynamic libc_issetugid issetugid "libc.so" - -//go:cgo_import_dynamic _ _ "libc.so" diff --git a/src/runtime/sys_openbsd3.go b/src/runtime/sys_openbsd3.go deleted file mode 100644 index de09ec5e25818a..00000000000000 --- a/src/runtime/sys_openbsd3.go +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build openbsd && !mips64 - -package runtime - -import ( - "internal/abi" - "unsafe" -) - -// The X versions of syscall expect the libc call to return a 64-bit result. -// Otherwise (the non-X version) expects a 32-bit result. -// This distinction is required because an error is indicated by returning -1, -// and we need to know whether to check 32 or 64 bits of the result. -// (Some libc functions that return 32 bits put junk in the upper 32 bits of AX.) - -// golang.org/x/sys linknames syscall_syscall -// (in addition to standard package syscall). -// Do not remove or change the type signature. -// -//go:linkname syscall_syscall syscall.syscall -//go:nosplit -//go:cgo_unsafe_args -func syscall_syscall(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { - entersyscall() - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall)), unsafe.Pointer(&fn)) - exitsyscall() - return -} -func syscall() - -//go:linkname syscall_syscallX syscall.syscallX -//go:nosplit -//go:cgo_unsafe_args -func syscall_syscallX(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { - entersyscall() - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscallX)), unsafe.Pointer(&fn)) - exitsyscall() - return -} -func syscallX() - -// golang.org/x/sys linknames syscall.syscall6 -// (in addition to standard package syscall). -// Do not remove or change the type signature. -// -//go:linkname syscall_syscall6 syscall.syscall6 -//go:nosplit -//go:cgo_unsafe_args -func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { - entersyscall() - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall6)), unsafe.Pointer(&fn)) - exitsyscall() - return -} -func syscall6() - -//go:linkname syscall_syscall6X syscall.syscall6X -//go:nosplit -//go:cgo_unsafe_args -func syscall_syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { - entersyscall() - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall6X)), unsafe.Pointer(&fn)) - exitsyscall() - return -} -func syscall6X() - -// golang.org/x/sys linknames syscall.syscall10 -// (in addition to standard package syscall). -// Do not remove or change the type signature. -// -//go:linkname syscall_syscall10 syscall.syscall10 -//go:nosplit -//go:cgo_unsafe_args -func syscall_syscall10(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10 uintptr) (r1, r2, err uintptr) { - entersyscall() - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall10)), unsafe.Pointer(&fn)) - exitsyscall() - return -} -func syscall10() - -//go:linkname syscall_syscall10X syscall.syscall10X -//go:nosplit -//go:cgo_unsafe_args -func syscall_syscall10X(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10 uintptr) (r1, r2, err uintptr) { - entersyscall() - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall10X)), unsafe.Pointer(&fn)) - exitsyscall() - return -} -func syscall10X() - -// golang.org/x/sys linknames syscall_rawSyscall -// (in addition to standard package syscall). -// Do not remove or change the type signature. -// -//go:linkname syscall_rawSyscall syscall.rawSyscall -//go:nosplit -//go:cgo_unsafe_args -func syscall_rawSyscall(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall)), unsafe.Pointer(&fn)) - return -} - -// golang.org/x/sys linknames syscall_rawSyscall6 -// (in addition to standard package syscall). -// Do not remove or change the type signature. -// -//go:linkname syscall_rawSyscall6 syscall.rawSyscall6 -//go:nosplit -//go:cgo_unsafe_args -func syscall_rawSyscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall6)), unsafe.Pointer(&fn)) - return -} - -//go:linkname syscall_rawSyscall6X syscall.rawSyscall6X -//go:nosplit -//go:cgo_unsafe_args -func syscall_rawSyscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall6X)), unsafe.Pointer(&fn)) - return -} - -//go:linkname syscall_rawSyscall10X syscall.rawSyscall10X -//go:nosplit -//go:cgo_unsafe_args -func syscall_rawSyscall10X(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10 uintptr) (r1, r2, err uintptr) { - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall10X)), unsafe.Pointer(&fn)) - return -} diff --git a/src/runtime/sys_openbsd_mips64.s b/src/runtime/sys_openbsd_mips64.s deleted file mode 100644 index 7ac0db048078a1..00000000000000 --- a/src/runtime/sys_openbsd_mips64.s +++ /dev/null @@ -1,388 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// -// System calls and other sys.stuff for mips64, OpenBSD -// /usr/src/sys/kern/syscalls.master for syscall numbers. -// - -#include "go_asm.h" -#include "go_tls.h" -#include "textflag.h" - -#define CLOCK_REALTIME $0 -#define CLOCK_MONOTONIC $3 - -// Exit the entire program (like C exit) -TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0 - MOVW code+0(FP), R4 // arg 1 - status - MOVV $1, R2 // sys_exit - SYSCALL - BEQ R7, 3(PC) - MOVV $0, R2 // crash on syscall failure - MOVV R2, (R2) - RET - -// func exitThread(wait *atomic.Uint32) -TEXT runtime·exitThread(SB),NOSPLIT,$0 - MOVV wait+0(FP), R4 // arg 1 - notdead - MOVV $302, R2 // sys___threxit - SYSCALL - MOVV $0, R2 // crash on syscall failure - MOVV R2, (R2) - JMP 0(PC) - -TEXT runtime·open(SB),NOSPLIT|NOFRAME,$0 - MOVV name+0(FP), R4 // arg 1 - path - MOVW mode+8(FP), R5 // arg 2 - mode - MOVW perm+12(FP), R6 // arg 3 - perm - MOVV $5, R2 // sys_open - SYSCALL - BEQ R7, 2(PC) - MOVW $-1, R2 - MOVW R2, ret+16(FP) - RET - -TEXT runtime·closefd(SB),NOSPLIT|NOFRAME,$0 - MOVW fd+0(FP), R4 // arg 1 - fd - MOVV $6, R2 // sys_close - SYSCALL - BEQ R7, 2(PC) - MOVW $-1, R2 - MOVW R2, ret+8(FP) - RET - -TEXT runtime·read(SB),NOSPLIT|NOFRAME,$0 - MOVW fd+0(FP), R4 // arg 1 - fd - MOVV p+8(FP), R5 // arg 2 - buf - MOVW n+16(FP), R6 // arg 3 - nbyte - MOVV $3, R2 // sys_read - SYSCALL - BEQ R7, 2(PC) - SUBVU R2, R0, R2 // caller expects negative errno - MOVW R2, ret+24(FP) - RET - -// func pipe2(flags int32) (r, w int32, errno int32) -TEXT runtime·pipe2(SB),NOSPLIT|NOFRAME,$0-20 - MOVV $r+8(FP), R4 - MOVW flags+0(FP), R5 - MOVV $101, R2 // sys_pipe2 - SYSCALL - BEQ R7, 2(PC) - SUBVU R2, R0, R2 // caller expects negative errno - MOVW R2, errno+16(FP) - RET - -TEXT runtime·write1(SB),NOSPLIT|NOFRAME,$0 - MOVV fd+0(FP), R4 // arg 1 - fd - MOVV p+8(FP), R5 // arg 2 - buf - MOVW n+16(FP), R6 // arg 3 - nbyte - MOVV $4, R2 // sys_write - SYSCALL - BEQ R7, 2(PC) - SUBVU R2, R0, R2 // caller expects negative errno - MOVW R2, ret+24(FP) - RET - -TEXT runtime·usleep(SB),NOSPLIT,$24-4 - MOVWU usec+0(FP), R3 - MOVV R3, R5 - MOVW $1000000, R4 - DIVVU R4, R3 - MOVV LO, R3 - MOVV R3, 8(R29) // tv_sec - MOVW $1000, R4 - MULVU R3, R4 - MOVV LO, R4 - SUBVU R4, R5 - MOVV R5, 16(R29) // tv_nsec - - ADDV $8, R29, R4 // arg 1 - rqtp - MOVV $0, R5 // arg 2 - rmtp - MOVV $91, R2 // sys_nanosleep - SYSCALL - RET - -TEXT runtime·getthrid(SB),NOSPLIT,$0-4 - MOVV $299, R2 // sys_getthrid - SYSCALL - MOVW R2, ret+0(FP) - RET - -TEXT runtime·thrkill(SB),NOSPLIT,$0-16 - MOVW tid+0(FP), R4 // arg 1 - tid - MOVV sig+8(FP), R5 // arg 2 - signum - MOVW $0, R6 // arg 3 - tcb - MOVV $119, R2 // sys_thrkill - SYSCALL - RET - -TEXT runtime·raiseproc(SB),NOSPLIT,$0 - MOVV $20, R4 // sys_getpid - SYSCALL - MOVV R2, R4 // arg 1 - pid - MOVW sig+0(FP), R5 // arg 2 - signum - MOVV $122, R2 // sys_kill - SYSCALL - RET - -TEXT runtime·mmap(SB),NOSPLIT,$0 - MOVV addr+0(FP), R4 // arg 1 - addr - MOVV n+8(FP), R5 // arg 2 - len - MOVW prot+16(FP), R6 // arg 3 - prot - MOVW flags+20(FP), R7 // arg 4 - flags - MOVW fd+24(FP), R8 // arg 5 - fd - MOVW $0, R9 // arg 6 - pad - MOVW off+28(FP), R10 // arg 7 - offset - MOVV $197, R2 // sys_mmap - SYSCALL - MOVV $0, R4 - BEQ R7, 3(PC) - MOVV R2, R4 // if error, move to R4 - MOVV $0, R2 - MOVV R2, p+32(FP) - MOVV R4, err+40(FP) - RET - -TEXT runtime·munmap(SB),NOSPLIT,$0 - MOVV addr+0(FP), R4 // arg 1 - addr - MOVV n+8(FP), R5 // arg 2 - len - MOVV $73, R2 // sys_munmap - SYSCALL - BEQ R7, 3(PC) - MOVV $0, R2 // crash on syscall failure - MOVV R2, (R2) - RET - -TEXT runtime·madvise(SB),NOSPLIT,$0 - MOVV addr+0(FP), R4 // arg 1 - addr - MOVV n+8(FP), R5 // arg 2 - len - MOVW flags+16(FP), R6 // arg 2 - flags - MOVV $75, R2 // sys_madvise - SYSCALL - BEQ R7, 2(PC) - MOVW $-1, R2 - MOVW R2, ret+24(FP) - RET - -TEXT runtime·setitimer(SB),NOSPLIT,$0 - MOVW mode+0(FP), R4 // arg 1 - mode - MOVV new+8(FP), R5 // arg 2 - new value - MOVV old+16(FP), R6 // arg 3 - old value - MOVV $69, R2 // sys_setitimer - SYSCALL - RET - -// func walltime() (sec int64, nsec int32) -TEXT runtime·walltime(SB), NOSPLIT, $32 - MOVW CLOCK_REALTIME, R4 // arg 1 - clock_id - MOVV $8(R29), R5 // arg 2 - tp - MOVV $87, R2 // sys_clock_gettime - SYSCALL - - MOVV 8(R29), R4 // sec - MOVV 16(R29), R5 // nsec - MOVV R4, sec+0(FP) - MOVW R5, nsec+8(FP) - - RET - -// int64 nanotime1(void) so really -// void nanotime1(int64 *nsec) -TEXT runtime·nanotime1(SB),NOSPLIT,$32 - MOVW CLOCK_MONOTONIC, R4 // arg 1 - clock_id - MOVV $8(R29), R5 // arg 2 - tp - MOVV $87, R2 // sys_clock_gettime - SYSCALL - - MOVV 8(R29), R3 // sec - MOVV 16(R29), R5 // nsec - - MOVV $1000000000, R4 - MULVU R4, R3 - MOVV LO, R3 - ADDVU R5, R3 - MOVV R3, ret+0(FP) - RET - -TEXT runtime·sigaction(SB),NOSPLIT,$0 - MOVW sig+0(FP), R4 // arg 1 - signum - MOVV new+8(FP), R5 // arg 2 - new sigaction - MOVV old+16(FP), R6 // arg 3 - old sigaction - MOVV $46, R2 // sys_sigaction - SYSCALL - BEQ R7, 3(PC) - MOVV $3, R2 // crash on syscall failure - MOVV R2, (R2) - RET - -TEXT runtime·obsdsigprocmask(SB),NOSPLIT,$0 - MOVW how+0(FP), R4 // arg 1 - mode - MOVW new+4(FP), R5 // arg 2 - new - MOVV $48, R2 // sys_sigprocmask - SYSCALL - BEQ R7, 3(PC) - MOVV $3, R2 // crash on syscall failure - MOVV R2, (R2) - MOVW R2, ret+8(FP) - RET - -TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 - MOVW sig+8(FP), R4 - MOVV info+16(FP), R5 - MOVV ctx+24(FP), R6 - MOVV fn+0(FP), R25 // Must use R25, needed for PIC code. - CALL (R25) - RET - -TEXT runtime·sigtramp(SB),NOSPLIT|TOPFRAME,$192 - // initialize REGSB = PC&0xffffffff00000000 - BGEZAL R0, 1(PC) - SRLV $32, R31, RSB - SLLV $32, RSB - - // this might be called in external code context, - // where g is not set. - MOVB runtime·iscgo(SB), R1 - BEQ R1, 2(PC) - JAL runtime·load_g(SB) - - MOVW R4, 8(R29) - MOVV R5, 16(R29) - MOVV R6, 24(R29) - MOVV $runtime·sigtrampgo(SB), R1 - JAL (R1) - RET - -// int32 tfork(void *param, uintptr psize, M *mp, G *gp, void (*fn)(void)); -TEXT runtime·tfork(SB),NOSPLIT,$0 - - // Copy mp, gp and fn off parent stack for use by child. - MOVV mm+16(FP), R16 - MOVV gg+24(FP), R17 - MOVV fn+32(FP), R18 - - MOVV param+0(FP), R4 // arg 1 - param - MOVV psize+8(FP), R5 // arg 2 - psize - MOVV $8, R2 // sys___tfork - SYSCALL - - // Return if syscall failed. - BEQ R7, 4(PC) - SUBVU R2, R0, R2 // caller expects negative errno - MOVW R2, ret+40(FP) - RET - - // In parent, return. - BEQ R2, 3(PC) - MOVW $0, ret+40(FP) - RET - - // Initialise m, g. - MOVV R17, g - MOVV R16, g_m(g) - - // Call fn. - CALL (R18) - - // fn should never return. - MOVV $2, R8 // crash if reached - MOVV R8, (R8) - RET - -TEXT runtime·sigaltstack(SB),NOSPLIT,$0 - MOVV new+0(FP), R4 // arg 1 - new sigaltstack - MOVV old+8(FP), R5 // arg 2 - old sigaltstack - MOVV $288, R2 // sys_sigaltstack - SYSCALL - BEQ R7, 3(PC) - MOVV $0, R8 // crash on syscall failure - MOVV R8, (R8) - RET - -TEXT runtime·osyield(SB),NOSPLIT,$0 - MOVV $298, R2 // sys_sched_yield - SYSCALL - RET - -TEXT runtime·thrsleep(SB),NOSPLIT,$0 - MOVV ident+0(FP), R4 // arg 1 - ident - MOVW clock_id+8(FP), R5 // arg 2 - clock_id - MOVV tsp+16(FP), R6 // arg 3 - tsp - MOVV lock+24(FP), R7 // arg 4 - lock - MOVV abort+32(FP), R8 // arg 5 - abort - MOVV $94, R2 // sys___thrsleep - SYSCALL - MOVW R2, ret+40(FP) - RET - -TEXT runtime·thrwakeup(SB),NOSPLIT,$0 - MOVV ident+0(FP), R4 // arg 1 - ident - MOVW n+8(FP), R5 // arg 2 - n - MOVV $301, R2 // sys___thrwakeup - SYSCALL - MOVW R2, ret+16(FP) - RET - -TEXT runtime·sysctl(SB),NOSPLIT,$0 - MOVV mib+0(FP), R4 // arg 1 - mib - MOVW miblen+8(FP), R5 // arg 2 - miblen - MOVV out+16(FP), R6 // arg 3 - out - MOVV size+24(FP), R7 // arg 4 - size - MOVV dst+32(FP), R8 // arg 5 - dest - MOVV ndst+40(FP), R9 // arg 6 - newlen - MOVV $202, R2 // sys___sysctl - SYSCALL - BEQ R7, 2(PC) - SUBVU R2, R0, R2 // caller expects negative errno - MOVW R2, ret+48(FP) - RET - -// int32 runtime·kqueue(void); -TEXT runtime·kqueue(SB),NOSPLIT,$0 - MOVV $269, R2 // sys_kqueue - SYSCALL - BEQ R7, 2(PC) - SUBVU R2, R0, R2 // caller expects negative errno - MOVW R2, ret+0(FP) - RET - -// int32 runtime·kevent(int kq, Kevent *changelist, int nchanges, Kevent *eventlist, int nevents, Timespec *timeout); -TEXT runtime·kevent(SB),NOSPLIT,$0 - MOVW kq+0(FP), R4 // arg 1 - kq - MOVV ch+8(FP), R5 // arg 2 - changelist - MOVW nch+16(FP), R6 // arg 3 - nchanges - MOVV ev+24(FP), R7 // arg 4 - eventlist - MOVW nev+32(FP), R8 // arg 5 - nevents - MOVV ts+40(FP), R9 // arg 6 - timeout - MOVV $72, R2 // sys_kevent - SYSCALL - BEQ R7, 2(PC) - SUBVU R2, R0, R2 // caller expects negative errno - MOVW R2, ret+48(FP) - RET - -// func fcntl(fd, cmd, arg int32) (int32, int32) -TEXT runtime·fcntl(SB),NOSPLIT,$0 - MOVW fd+0(FP), R4 // fd - MOVW cmd+4(FP), R5 // cmd - MOVW arg+8(FP), R6 // arg - MOVV $92, R2 // sys_fcntl - SYSCALL - MOVV $0, R4 - BEQ R7, noerr - MOVV R2, R4 - MOVW $-1, R2 -noerr: - MOVW R2, ret+16(FP) - MOVW R4, errno+20(FP) - RET - -// func issetugid() int32 -TEXT runtime·issetugid(SB),NOSPLIT,$0 - MOVV $253, R2 // sys_issetugid - SYSCALL - MOVW R2, ret+0(FP) - RET diff --git a/src/runtime/sys_plan9_386.s b/src/runtime/sys_plan9_386.s index bdcb98e19e0bbf..4eefeaf80a7034 100644 --- a/src/runtime/sys_plan9_386.s +++ b/src/runtime/sys_plan9_386.s @@ -58,6 +58,12 @@ TEXT runtime·closefd(SB),NOSPLIT,$0 MOVL AX, ret+4(FP) RET +TEXT runtime·dupfd(SB),NOSPLIT,$0 + MOVL $5, AX + INT $64 + MOVL AX, ret+8(FP) + RET + TEXT runtime·exits(SB),NOSPLIT,$0 MOVL $8, AX INT $64 @@ -87,32 +93,15 @@ TEXT runtime·plan9_tsemacquire(SB),NOSPLIT,$0 MOVL AX, ret+8(FP) RET -TEXT nsec<>(SB),NOSPLIT,$0 - MOVL $53, AX - INT $64 - RET - -TEXT runtime·nsec(SB),NOSPLIT,$8 - LEAL ret+4(FP), AX - MOVL AX, 0(SP) - CALL nsec<>(SB) - CMPL AX, $0 - JGE 3(PC) - MOVL $-1, ret_lo+4(FP) - MOVL $-1, ret_hi+8(FP) - RET - -// func walltime() (sec int64, nsec int32) -TEXT runtime·walltime(SB),NOSPLIT,$8-12 - CALL runtime·nanotime1(SB) - MOVL 0(SP), AX - MOVL 4(SP), DX - +// func timesplit(u uint64) (sec int64, nsec int32) +TEXT runtime·timesplit(SB),NOSPLIT,$0 + MOVL u_lo+0(FP), AX + MOVL u_hi+4(FP), DX MOVL $1000000000, CX DIVL CX - MOVL AX, sec_lo+0(FP) - MOVL $0, sec_hi+4(FP) - MOVL DX, nsec+8(FP) + MOVL AX, sec_lo+8(FP) + MOVL $0, sec_hi+12(FP) + MOVL DX, nsec+16(FP) RET TEXT runtime·notify(SB),NOSPLIT,$0 diff --git a/src/runtime/sys_plan9_amd64.s b/src/runtime/sys_plan9_amd64.s index a53f9201f47cbf..67cff8250514c3 100644 --- a/src/runtime/sys_plan9_amd64.s +++ b/src/runtime/sys_plan9_amd64.s @@ -53,6 +53,17 @@ TEXT runtime·closefd(SB),NOSPLIT,$0 MOVL AX, ret+8(FP) RET +TEXT runtime·dupfd(SB),NOSPLIT,$0 + MOVQ $5, BP + // Kernel expects each int32 arg to be 64-bit-aligned. + // The return value slot is where the kernel + // expects to find the second argument, so copy it there. + MOVL new+4(FP), AX + MOVL AX, ret+8(FP) + SYSCALL + MOVL AX, ret+8(FP) + RET + TEXT runtime·exits(SB),NOSPLIT,$0 MOVQ $8, BP SYSCALL @@ -82,17 +93,9 @@ TEXT runtime·plan9_tsemacquire(SB),NOSPLIT,$0 MOVL AX, ret+16(FP) RET -TEXT runtime·nsec(SB),NOSPLIT,$0 - MOVQ $53, BP - SYSCALL - MOVQ AX, ret+8(FP) - RET - -// func walltime() (sec int64, nsec int32) -TEXT runtime·walltime(SB),NOSPLIT,$8-12 - CALL runtime·nanotime1(SB) - MOVQ 0(SP), AX - +// func timesplit(u uint64) (sec int64, nsec int32) +TEXT runtime·timesplit(SB),NOSPLIT,$0 + MOVQ u+0(FP), AX // generated code for // func f(x uint64) (uint64, uint64) { return x/1000000000, x%1000000000 } // adapted to reduce duplication @@ -102,10 +105,10 @@ TEXT runtime·walltime(SB),NOSPLIT,$8-12 ADDQ CX, DX RCRQ $1, DX SHRQ $29, DX - MOVQ DX, sec+0(FP) + MOVQ DX, sec+8(FP) IMULQ $1000000000, DX SUBQ DX, CX - MOVL CX, nsec+8(FP) + MOVL CX, nsec+16(FP) RET TEXT runtime·notify(SB),NOSPLIT,$0 diff --git a/src/runtime/sys_plan9_arm.s b/src/runtime/sys_plan9_arm.s index 53430857437793..e1faacecf50928 100644 --- a/src/runtime/sys_plan9_arm.s +++ b/src/runtime/sys_plan9_arm.s @@ -93,6 +93,13 @@ TEXT runtime·closefd(SB),NOSPLIT,$0-8 MOVW R0, ret+4(FP) RET +//func dupfd(old, new int32) int32 +TEXT runtime·dupfd(SB),NOSPLIT,$0-12 + MOVW $SYS_DUP, R0 + SWI $0 + MOVW R0, ret+8(FP) + RET + //func exits(msg *byte) TEXT runtime·exits(SB),NOSPLIT,$0-4 MOVW $SYS_EXITS, R0 @@ -127,26 +134,10 @@ TEXT runtime·plan9_tsemacquire(SB),NOSPLIT,$0-12 MOVW R0, ret+8(FP) RET -//func nsec(*int64) int64 -TEXT runtime·nsec(SB),NOSPLIT|NOFRAME,$0-12 - MOVW $SYS_NSEC, R0 - SWI $0 - MOVW arg+0(FP), R1 - MOVW 0(R1), R0 - MOVW R0, ret_lo+4(FP) - MOVW 4(R1), R0 - MOVW R0, ret_hi+8(FP) - RET - -// func walltime() (sec int64, nsec int32) -TEXT runtime·walltime(SB),NOSPLIT,$12-12 - // use nsec system call to get current time in nanoseconds - MOVW $sysnsec_lo-8(SP), R0 // destination addr - MOVW R0,res-12(SP) - MOVW $SYS_NSEC, R0 - SWI $0 - MOVW sysnsec_lo-8(SP), R1 // R1:R2 = nsec - MOVW sysnsec_hi-4(SP), R2 +// func timesplit(u uint64) (sec int64, nsec int32) +TEXT runtime·timesplit(SB),NOSPLIT,$0 + MOVW u_lo+0(FP), R1 // R1:R2 = nsec + MOVW u_hi+4(FP), R2 // multiply nanoseconds by reciprocal of 10**9 (scaled by 2**61) // to get seconds (96 bit scaled result) @@ -173,9 +164,9 @@ TEXT runtime·walltime(SB),NOSPLIT,$12-12 SUB.HS R5,R1 // remainder -= 10**9 ADD.HS $1,R6 // sec += 1 - MOVW R6,sec_lo+0(FP) - MOVW R7,sec_hi+4(FP) - MOVW R1,nsec+8(FP) + MOVW R6,sec_lo+8(FP) + MOVW R7,sec_hi+12(FP) + MOVW R1,nsec+16(FP) RET //func notify(fn unsafe.Pointer) int32 diff --git a/src/runtime/sys_solaris_amd64.s b/src/runtime/sys_solaris_amd64.s index 7a80020ba34712..9235cad391dfd2 100644 --- a/src/runtime/sys_solaris_amd64.s +++ b/src/runtime/sys_solaris_amd64.s @@ -155,7 +155,7 @@ allgood: // save m->libcall MOVQ g_m(R10), BP - LEAQ m_libcall(BP), R11 + LEAQ (m_mOS+mOS_libcall)(BP), R11 MOVQ libcall_fn(R11), R10 MOVQ R10, 72(SP) MOVQ libcall_args(R11), R10 @@ -197,7 +197,7 @@ allgood: MOVQ g(BX), BP MOVQ g_m(BP), BP // restore libcall - LEAQ m_libcall(BP), R11 + LEAQ (m_mOS+mOS_libcall)(BP), R11 MOVQ 72(SP), R10 MOVQ R10, libcall_fn(R11) MOVQ 80(SP), R10 diff --git a/src/runtime/sys_wasm.go b/src/runtime/sys_wasm.go index f88b992e9c39a3..6b40a8d3e94c96 100644 --- a/src/runtime/sys_wasm.go +++ b/src/runtime/sys_wasm.go @@ -34,3 +34,17 @@ func gostartcall(buf *gobuf, fn, ctxt unsafe.Pointer) { buf.pc = uintptr(fn) buf.ctxt = ctxt } + +func notInitialized() // defined in assembly, call notInitialized1 + +// Called if a wasmexport function is called before runtime initialization +// +//go:nosplit +func notInitialized1() { + writeErrStr("runtime: wasmexport function called before runtime initialization\n") + if isarchive || islibrary { + writeErrStr("\tcall _initialize first\n") + } else { + writeErrStr("\tcall _start first\n") + } +} diff --git a/src/runtime/sys_wasm.s b/src/runtime/sys_wasm.s index 1e73adadd56aa7..95c162eb857ac4 100644 --- a/src/runtime/sys_wasm.s +++ b/src/runtime/sys_wasm.s @@ -22,70 +22,18 @@ TEXT runtime·wasmDiv(SB), NOSPLIT, $0-0 I64DivS Return -TEXT runtime·wasmTruncS(SB), NOSPLIT, $0-0 - Get R0 - Get R0 - F64Ne // NaN - If - I64Const $0x8000000000000000 - Return - End - - Get R0 - F64Const $0x7ffffffffffffc00p0 // Maximum truncated representation of 0x7fffffffffffffff - F64Gt - If - I64Const $0x8000000000000000 - Return - End - - Get R0 - F64Const $-0x7ffffffffffffc00p0 // Minimum truncated representation of -0x8000000000000000 - F64Lt - If - I64Const $0x8000000000000000 - Return - End - - Get R0 - I64TruncF64S - Return - -TEXT runtime·wasmTruncU(SB), NOSPLIT, $0-0 - Get R0 - Get R0 - F64Ne // NaN - If - I64Const $0x8000000000000000 - Return - End - - Get R0 - F64Const $0xfffffffffffff800p0 // Maximum truncated representation of 0xffffffffffffffff - F64Gt - If - I64Const $0x8000000000000000 - Return - End - - Get R0 - F64Const $0. - F64Lt - If - I64Const $0x8000000000000000 - Return - End - - Get R0 - I64TruncF64U - Return - TEXT runtime·exitThread(SB), NOSPLIT, $0-0 UNDEF TEXT runtime·osyield(SB), NOSPLIT, $0-0 UNDEF +TEXT runtime·currentMemory(SB), NOSPLIT, $0 + Get SP + CurrentMemory + I32Store ret+0(FP) + RET + TEXT runtime·growMemory(SB), NOSPLIT, $0 Get SP I32Load pages+0(FP) diff --git a/src/runtime/sys_windows_386.s b/src/runtime/sys_windows_386.s index e71fda78aee353..4030e4c38b3ce4 100644 --- a/src/runtime/sys_windows_386.s +++ b/src/runtime/sys_windows_386.s @@ -11,49 +11,6 @@ #define TEB_TlsSlots 0xE10 #define TEB_ArbitraryPtr 0x14 -TEXT runtime·asmstdcall_trampoline(SB),NOSPLIT,$0 - JMP runtime·asmstdcall(SB) - -// void runtime·asmstdcall(void *c); -TEXT runtime·asmstdcall(SB),NOSPLIT,$0 - MOVL fn+0(FP), BX - MOVL SP, BP // save stack pointer - - // SetLastError(0). - MOVL $0, 0x34(FS) - - MOVL libcall_n(BX), CX - - // Fast version, do not store args on the stack. - CMPL CX, $0 - JE docall - - // Copy args to the stack. - MOVL CX, AX - SALL $2, AX - SUBL AX, SP // room for args - MOVL SP, DI - MOVL libcall_args(BX), SI - CLD - REP; MOVSL - -docall: - // Call stdcall or cdecl function. - // DI SI BP BX are preserved, SP is not - CALL libcall_fn(BX) - MOVL BP, SP - - // Return result. - MOVL fn+0(FP), BX - MOVL AX, libcall_r1(BX) - MOVL DX, libcall_r2(BX) - - // GetLastError(). - MOVL 0x34(FS), AX - MOVL AX, libcall_err(BX) - - RET - // faster get/set last error TEXT runtime·getlasterror(SB),NOSPLIT,$0 MOVL 0x34(FS), AX diff --git a/src/runtime/sys_windows_amd64.s b/src/runtime/sys_windows_amd64.s index 56a2dc0bcf24f2..e438599910f4ee 100644 --- a/src/runtime/sys_windows_amd64.s +++ b/src/runtime/sys_windows_amd64.s @@ -12,85 +12,6 @@ #define TEB_TlsSlots 0x1480 #define TEB_ArbitraryPtr 0x28 -TEXT runtime·asmstdcall_trampoline(SB),NOSPLIT,$0 - MOVQ AX, CX - JMP runtime·asmstdcall(SB) - -// void runtime·asmstdcall(void *c); -TEXT runtime·asmstdcall(SB),NOSPLIT,$16 - MOVQ SP, AX - ANDQ $~15, SP // alignment as per Windows requirement - MOVQ AX, 8(SP) - MOVQ CX, 0(SP) // asmcgocall will put first argument into CX. - - MOVQ libcall_fn(CX), AX - MOVQ libcall_args(CX), SI - MOVQ libcall_n(CX), CX - - // SetLastError(0). - MOVQ 0x30(GS), DI - MOVL $0, 0x68(DI) - - SUBQ $(const_maxArgs*8), SP // room for args - - // Fast version, do not store args on the stack. - CMPL CX, $0; JE _0args - CMPL CX, $1; JE _1args - CMPL CX, $2; JE _2args - CMPL CX, $3; JE _3args - CMPL CX, $4; JE _4args - - // Check we have enough room for args. - CMPL CX, $const_maxArgs - JLE 2(PC) - INT $3 // not enough room -> crash - - // Copy args to the stack. - MOVQ SP, DI - CLD - REP; MOVSQ - MOVQ SP, SI - - // Load first 4 args into correspondent registers. - // Floating point arguments are passed in the XMM - // registers. Set them here in case any of the arguments - // are floating point values. For details see - // https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention?view=msvc-170 -_4args: - MOVQ 24(SI), R9 - MOVQ R9, X3 -_3args: - MOVQ 16(SI), R8 - MOVQ R8, X2 -_2args: - MOVQ 8(SI), DX - MOVQ DX, X1 -_1args: - MOVQ 0(SI), CX - MOVQ CX, X0 -_0args: - - // Call stdcall function. - CALL AX - - ADDQ $(const_maxArgs*8), SP - - // Return result. - MOVQ 0(SP), CX - MOVQ 8(SP), SP - MOVQ AX, libcall_r1(CX) - // Floating point return values are returned in XMM0. Setting r2 to this - // value in case this call returned a floating point value. For details, - // see https://docs.microsoft.com/en-us/cpp/build/x64-calling-convention - MOVQ X0, libcall_r2(CX) - - // GetLastError(). - MOVQ 0x30(GS), DI - MOVL 0x68(DI), AX - MOVQ AX, libcall_err(CX) - - RET - // faster get/set last error TEXT runtime·getlasterror(SB),NOSPLIT,$0 MOVQ 0x30(GS), AX diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s deleted file mode 100644 index d194899d50898d..00000000000000 --- a/src/runtime/sys_windows_arm.s +++ /dev/null @@ -1,287 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "go_asm.h" -#include "go_tls.h" -#include "textflag.h" -#include "time_windows.h" - -// Note: For system ABI, R0-R3 are args, R4-R11 are callee-save. - -TEXT runtime·asmstdcall_trampoline(SB),NOSPLIT,$0 - B runtime·asmstdcall(SB) - -// void runtime·asmstdcall(void *c); -TEXT runtime·asmstdcall(SB),NOSPLIT|NOFRAME,$0 - MOVM.DB.W [R4, R5, R14], (R13) // push {r4, r5, lr} - MOVW R0, R4 // put libcall * in r4 - MOVW R13, R5 // save stack pointer in r5 - - // SetLastError(0) - MOVW $0, R0 - MRC 15, 0, R1, C13, C0, 2 - MOVW R0, 0x34(R1) - - MOVW 8(R4), R12 // libcall->args - - // Do we have more than 4 arguments? - MOVW 4(R4), R0 // libcall->n - SUB.S $4, R0, R2 - BLE loadregs - - // Reserve stack space for remaining args - SUB R2<<2, R13 - BIC $0x7, R13 // alignment for ABI - - // R0: count of arguments - // R1: - // R2: loop counter, from 0 to (n-4) - // R3: scratch - // R4: pointer to libcall struct - // R12: libcall->args - MOVW $0, R2 -stackargs: - ADD $4, R2, R3 // r3 = args[4 + i] - MOVW R3<<2(R12), R3 - MOVW R3, R2<<2(R13) // stack[i] = r3 - - ADD $1, R2 // i++ - SUB $4, R0, R3 // while (i < (n - 4)) - CMP R3, R2 - BLT stackargs - -loadregs: - CMP $3, R0 - MOVW.GT 12(R12), R3 - - CMP $2, R0 - MOVW.GT 8(R12), R2 - - CMP $1, R0 - MOVW.GT 4(R12), R1 - - CMP $0, R0 - MOVW.GT 0(R12), R0 - - BIC $0x7, R13 // alignment for ABI - MOVW 0(R4), R12 // branch to libcall->fn - BL (R12) - - MOVW R5, R13 // free stack space - MOVW R0, 12(R4) // save return value to libcall->r1 - MOVW R1, 16(R4) - - // GetLastError - MRC 15, 0, R1, C13, C0, 2 - MOVW 0x34(R1), R0 - MOVW R0, 20(R4) // store in libcall->err - - MOVM.IA.W (R13), [R4, R5, R15] - -TEXT runtime·getlasterror(SB),NOSPLIT,$0 - MRC 15, 0, R0, C13, C0, 2 - MOVW 0x34(R0), R0 - MOVW R0, ret+0(FP) - RET - -// Called by Windows as a Vectored Exception Handler (VEH). -// R0 is pointer to struct containing -// exception record and context pointers. -// R1 is the kind of sigtramp function. -// Return value of sigtrampgo is stored in R0. -TEXT sigtramp<>(SB),NOSPLIT|NOFRAME,$0 - MOVM.DB.W [R4-R11, R14], (R13) // push {r4-r11, lr} (SP-=40) - SUB $(16), R13 // reserve space for parameters/retval to go call - - MOVW R0, R6 // Save param0 - MOVW R1, R7 // Save param1 - BL runtime·load_g(SB) // Clobbers R0 - - MOVW $0, R4 - MOVW R4, 0(R13) // No saved link register. - MOVW R6, 4(R13) // Move arg0 into position - MOVW R7, 8(R13) // Move arg1 into position - BL runtime·sigtrampgo(SB) - MOVW 12(R13), R0 // Fetch return value from stack - - ADD $(16), R13 // free locals - MOVM.IA.W (R13), [R4-R11, R14] // pop {r4-r11, lr} - - B (R14) // return - -// Trampoline to resume execution from exception handler. -// This is part of the control flow guard workaround. -// It switches stacks and jumps to the continuation address. -// R0 and R1 are set above at the end of sigtrampgo -// in the context that starts executing at sigresume. -TEXT runtime·sigresume(SB),NOSPLIT|NOFRAME,$0 - // Important: do not smash LR, - // which is set to a live value when handling - // a signal by pushing a call to sigpanic onto the stack. - MOVW R0, R13 - B (R1) - -TEXT runtime·exceptiontramp(SB),NOSPLIT|NOFRAME,$0 - MOVW $const_callbackVEH, R1 - B sigtramp<>(SB) - -TEXT runtime·firstcontinuetramp(SB),NOSPLIT|NOFRAME,$0 - MOVW $const_callbackFirstVCH, R1 - B sigtramp<>(SB) - -TEXT runtime·lastcontinuetramp(SB),NOSPLIT|NOFRAME,$0 - MOVW $const_callbackLastVCH, R1 - B sigtramp<>(SB) - -TEXT runtime·callbackasm1(SB),NOSPLIT|NOFRAME,$0 - // On entry, the trampoline in zcallback_windows_arm.s left - // the callback index in R12 (which is volatile in the C ABI). - - // Push callback register arguments r0-r3. We do this first so - // they're contiguous with stack arguments. - MOVM.DB.W [R0-R3], (R13) - // Push C callee-save registers r4-r11 and lr. - MOVM.DB.W [R4-R11, R14], (R13) - SUB $(16 + callbackArgs__size), R13 // space for locals - - // Create a struct callbackArgs on our stack. - MOVW R12, (16+callbackArgs_index)(R13) // callback index - MOVW $(16+callbackArgs__size+4*9)(R13), R0 - MOVW R0, (16+callbackArgs_args)(R13) // address of args vector - MOVW $0, R0 - MOVW R0, (16+callbackArgs_result)(R13) // result - - // Prepare for entry to Go. - BL runtime·load_g(SB) - - // Call cgocallback, which will call callbackWrap(frame). - MOVW $0, R0 - MOVW R0, 12(R13) // context - MOVW $16(R13), R1 // R1 = &callbackArgs{...} - MOVW R1, 8(R13) // frame (address of callbackArgs) - MOVW $·callbackWrap(SB), R1 - MOVW R1, 4(R13) // PC of function to call - BL runtime·cgocallback(SB) - - // Get callback result. - MOVW (16+callbackArgs_result)(R13), R0 - - ADD $(16 + callbackArgs__size), R13 // free locals - MOVM.IA.W (R13), [R4-R11, R12] // pop {r4-r11, lr=>r12} - ADD $(4*4), R13 // skip r0-r3 - B (R12) // return - -// uint32 tstart_stdcall(M *newm); -TEXT runtime·tstart_stdcall(SB),NOSPLIT|NOFRAME,$0 - MOVM.DB.W [R4-R11, R14], (R13) // push {r4-r11, lr} - - MOVW m_g0(R0), g - MOVW R0, g_m(g) - BL runtime·save_g(SB) - - // Layout new m scheduler stack on os stack. - MOVW R13, R0 - MOVW R0, g_stack+stack_hi(g) - SUB $(64*1024), R0 - MOVW R0, (g_stack+stack_lo)(g) - MOVW R0, g_stackguard0(g) - MOVW R0, g_stackguard1(g) - - BL runtime·emptyfunc(SB) // fault if stack check is wrong - BL runtime·mstart(SB) - - // Exit the thread. - MOVW $0, R0 - MOVM.IA.W (R13), [R4-R11, R15] // pop {r4-r11, pc} - -TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0 - B runtime·armPublicationBarrier(SB) - -// never called (this is a GOARM=7 platform) -TEXT runtime·read_tls_fallback(SB),NOSPLIT,$0 - MOVW $0xabcd, R0 - MOVW R0, (R0) - RET - -TEXT runtime·nanotime1(SB),NOSPLIT,$0-8 -loop: - MOVW time_hi1(R3), R1 - DMB MB_ISH - MOVW time_lo(R3), R0 - DMB MB_ISH - MOVW time_hi2(R3), R2 - CMP R1, R2 - BNE loop - - // wintime = R1:R0, multiply by 100 - MOVW $100, R2 - MULLU R0, R2, (R4, R3) // R4:R3 = R1:R0 * R2 - MULA R1, R2, R4, R4 - - // wintime*100 = R4:R3 - MOVW R3, ret_lo+0(FP) - MOVW R4, ret_hi+4(FP) - RET - -// save_g saves the g register (R10) into thread local memory -// so that we can call externally compiled -// ARM code that will overwrite those registers. -// NOTE: runtime.gogo assumes that R1 is preserved by this function. -// runtime.mcall assumes this function only clobbers R0 and R11. -// Returns with g in R0. -// Save the value in the _TEB->TlsSlots array. -// Effectively implements TlsSetValue(). -// tls_g stores the TLS slot allocated TlsAlloc(). -TEXT runtime·save_g(SB),NOSPLIT,$0 - MRC 15, 0, R0, C13, C0, 2 - ADD $0xe10, R0 - MOVW $runtime·tls_g(SB), R11 - MOVW (R11), R11 - MOVW g, R11<<2(R0) - MOVW g, R0 // preserve R0 across call to setg<> - RET - -// load_g loads the g register from thread-local memory, -// for use after calling externally compiled -// ARM code that overwrote those registers. -// Get the value from the _TEB->TlsSlots array. -// Effectively implements TlsGetValue(). -TEXT runtime·load_g(SB),NOSPLIT,$0 - MRC 15, 0, R0, C13, C0, 2 - ADD $0xe10, R0 - MOVW $runtime·tls_g(SB), g - MOVW (g), g - MOVW g<<2(R0), g - RET - -// This is called from rt0_go, which runs on the system stack -// using the initial stack allocated by the OS. -// It calls back into standard C using the BL below. -// To do that, the stack pointer must be 8-byte-aligned. -TEXT runtime·_initcgo(SB),NOSPLIT|NOFRAME,$0 - MOVM.DB.W [R4, R14], (R13) // push {r4, lr} - - // Ensure stack is 8-byte aligned before calling C code - MOVW R13, R4 - BIC $0x7, R13 - - // Allocate a TLS slot to hold g across calls to external code - MOVW $runtime·_TlsAlloc(SB), R0 - MOVW (R0), R0 - BL (R0) - - // Assert that slot is less than 64 so we can use _TEB->TlsSlots - CMP $64, R0 - MOVW $runtime·abort(SB), R1 - BL.GE (R1) - - // Save Slot into tls_g - MOVW $runtime·tls_g(SB), R1 - MOVW R0, (R1) - - MOVW R4, R13 - MOVM.IA.W (R13), [R4, R15] // pop {r4, pc} - -// Holds the TLS Slot, which was allocated by TlsAlloc() -GLOBL runtime·tls_g+0(SB), NOPTR, $4 diff --git a/src/runtime/sys_windows_arm64.s b/src/runtime/sys_windows_arm64.s index 1f6d411b07c363..da3cb7e5465089 100644 --- a/src/runtime/sys_windows_arm64.s +++ b/src/runtime/sys_windows_arm64.s @@ -19,88 +19,6 @@ // // load_g and save_g (in tls_arm64.s) clobber R27 (REGTMP) and R0. -TEXT runtime·asmstdcall_trampoline(SB),NOSPLIT,$0 - B runtime·asmstdcall(SB) - -// void runtime·asmstdcall(void *c); -TEXT runtime·asmstdcall(SB),NOSPLIT,$16 - STP (R19, R20), 16(RSP) // save old R19, R20 - MOVD R0, R19 // save libcall pointer - MOVD RSP, R20 // save stack pointer - - // SetLastError(0) - MOVD $0, TEB_error(R18_PLATFORM) - MOVD libcall_args(R19), R12 // libcall->args - - // Do we have more than 8 arguments? - MOVD libcall_n(R19), R0 - CMP $0, R0; BEQ _0args - CMP $1, R0; BEQ _1args - CMP $2, R0; BEQ _2args - CMP $3, R0; BEQ _3args - CMP $4, R0; BEQ _4args - CMP $5, R0; BEQ _5args - CMP $6, R0; BEQ _6args - CMP $7, R0; BEQ _7args - CMP $8, R0; BEQ _8args - - // Reserve stack space for remaining args - SUB $8, R0, R2 - ADD $1, R2, R3 // make even number of words for stack alignment - AND $~1, R3 - LSL $3, R3 - SUB R3, RSP - - // R4: size of stack arguments (n-8)*8 - // R5: &args[8] - // R6: loop counter, from 0 to (n-8)*8 - // R7: scratch - // R8: copy of RSP - (R2)(RSP) assembles as (R2)(ZR) - SUB $8, R0, R4 - LSL $3, R4 - ADD $(8*8), R12, R5 - MOVD $0, R6 - MOVD RSP, R8 -stackargs: - MOVD (R6)(R5), R7 - MOVD R7, (R6)(R8) - ADD $8, R6 - CMP R6, R4 - BNE stackargs - -_8args: - MOVD (7*8)(R12), R7 -_7args: - MOVD (6*8)(R12), R6 -_6args: - MOVD (5*8)(R12), R5 -_5args: - MOVD (4*8)(R12), R4 -_4args: - MOVD (3*8)(R12), R3 -_3args: - MOVD (2*8)(R12), R2 -_2args: - MOVD (1*8)(R12), R1 -_1args: - MOVD (0*8)(R12), R0 -_0args: - - MOVD libcall_fn(R19), R12 // branch to libcall->fn - BL (R12) - - MOVD R20, RSP // free stack space - MOVD R0, libcall_r1(R19) // save return value to libcall->r1 - // TODO(rsc) floating point like amd64 in libcall->r2? - - // GetLastError - MOVD TEB_error(R18_PLATFORM), R0 - MOVD R0, libcall_err(R19) - - // Restore callee-saved registers. - LDP 16(RSP), (R19, R20) - RET - TEXT runtime·getlasterror(SB),NOSPLIT,$0 MOVD TEB_error(R18_PLATFORM), R0 MOVD R0, ret+0(FP) diff --git a/src/runtime/syscall_test.go b/src/runtime/syscall_test.go new file mode 100644 index 00000000000000..18f3e8e3151f45 --- /dev/null +++ b/src/runtime/syscall_test.go @@ -0,0 +1,28 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "internal/testenv" + "runtime" + "testing" +) + +func TestSyscallArgs(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skipf("skipping test: GOARCH=%s", runtime.GOARCH) + } + testenv.MustHaveCGO(t) + + exe, err := buildTestProg(t, "testsyscall") + if err != nil { + t.Fatal(err) + } + + cmd := testenv.Command(t, exe) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("test program failed: %v\n%s", err, out) + } +} diff --git a/src/runtime/syscall_windows.go b/src/runtime/syscall_windows.go index 69d720a395c48d..9ad21c8649c1a2 100644 --- a/src/runtime/syscall_windows.go +++ b/src/runtime/syscall_windows.go @@ -7,6 +7,7 @@ package runtime import ( "internal/abi" "internal/goarch" + "internal/runtime/syscall/windows" "unsafe" ) @@ -97,19 +98,14 @@ func (p *abiDesc) assignArg(t *_type) { // passed as two words (little endian); and // structs are pushed on the stack. In // fastcall, arguments larger than the word - // size are passed by reference. On arm, - // 8-byte aligned arguments round up to the - // next even register and can be split across - // registers and the stack. + // size are passed by reference. panic("compileCallback: argument size is larger than uintptr") } - if k := t.Kind_ & abi.KindMask; GOARCH != "386" && (k == abi.Float32 || k == abi.Float64) { + if k := t.Kind(); GOARCH != "386" && (k == abi.Float32 || k == abi.Float64) { // In fastcall, floating-point arguments in // the first four positions are passed in // floating-point registers, which we don't - // currently spill. arm passes floating-point - // arguments in VFP registers, which we also - // don't support. + // currently spill. // So basically we only support 386. panic("compileCallback: float arguments not supported") } @@ -126,7 +122,7 @@ func (p *abiDesc) assignArg(t *_type) { // argument word and all supported Windows // architectures are little endian, so srcStackOffset // is already pointing to the right place for smaller - // arguments. The same is true on arm. + // arguments. oldParts := p.parts if p.tryRegAssignArg(t, 0) { @@ -162,8 +158,8 @@ func (p *abiDesc) assignArg(t *_type) { p.dstStackSize += t.Size_ } - // cdecl, stdcall, fastcall, and arm pad arguments to word size. - // TODO(rsc): On arm and arm64 do we need to skip the caller's saved LR? + // cdecl, stdcall, and fastcall pad arguments to word size. + // TODO(rsc): On arm64 do we need to skip the caller's saved LR? p.srcStackSize += goarch.PtrSize } @@ -174,7 +170,7 @@ func (p *abiDesc) assignArg(t *_type) { // // Returns whether the assignment succeeded. func (p *abiDesc) tryRegAssignArg(t *_type, offset uintptr) bool { - switch k := t.Kind_ & abi.KindMask; k { + switch k := t.Kind(); k { case abi.Bool, abi.Int, abi.Int8, abi.Int16, abi.Int32, abi.Uint, abi.Uint8, abi.Uint16, abi.Uint32, abi.Uintptr, abi.Pointer, abi.UnsafePointer: // Assign a register for all these types. return p.assignReg(t.Size_, offset) @@ -235,7 +231,7 @@ func callbackasm() // and we want callback to arrive at // correspondent call instruction instead of start of // runtime.callbackasm. -// On ARM, runtime.callbackasm is a series of mov and branch instructions. +// On ARM64, runtime.callbackasm is a series of mov and branch instructions. // R12 is loaded with the callback index. Each entry is two instructions, // hence 8 bytes. func callbackasmAddr(i int) uintptr { @@ -245,8 +241,8 @@ func callbackasmAddr(i int) uintptr { panic("unsupported architecture") case "386", "amd64": entrySize = 5 - case "arm", "arm64": - // On ARM and ARM64, each entry is a MOV instruction + case "arm64": + // On ARM64, each entry is a MOV instruction // followed by a branch instruction entrySize = 8 } @@ -260,7 +256,7 @@ const callbackMaxFrame = 64 * goarch.PtrSize // // On 386, if cdecl is true, the returned C function will use the // cdecl calling convention; otherwise, it will use stdcall. On amd64, -// it always uses fastcall. On arm, it always uses the ARM convention. +// it always uses fastcall. // //go:linkname compileCallback syscall.compileCallback func compileCallback(fn eface, cdecl bool) (code uintptr) { @@ -269,7 +265,7 @@ func compileCallback(fn eface, cdecl bool) (code uintptr) { cdecl = false } - if fn._type == nil || (fn._type.Kind_&abi.KindMask) != abi.Func { + if fn._type == nil || fn._type.Kind() != abi.Func { panic("compileCallback: expected function with one uintptr-sized result") } ft := (*functype)(unsafe.Pointer(fn._type)) @@ -290,7 +286,7 @@ func compileCallback(fn eface, cdecl bool) (code uintptr) { if ft.OutSlice()[0].Size_ != goarch.PtrSize { panic("compileCallback: expected function with one uintptr-sized result") } - if k := ft.OutSlice()[0].Kind_ & abi.KindMask; k == abi.Float32 || k == abi.Float64 { + if k := ft.OutSlice()[0].Kind(); k == abi.Float32 || k == abi.Float64 { // In cdecl and stdcall, float results are returned in // ST(0). In fastcall, they're returned in XMM0. // Either way, it's not AX. @@ -355,10 +351,6 @@ type callbackArgs struct { // For fastcall, the trampoline spills register arguments to // the reserved spill slots below the stack arguments, // resulting in a layout equivalent to stdcall. - // - // For arm, the trampoline stores the register arguments just - // below the stack arguments, so again we can treat it as one - // big stack arguments frame. args unsafe.Pointer // Below are out-args from callbackWrap result uintptr @@ -411,99 +403,19 @@ func callbackWrap(a *callbackArgs) { } } -const _LOAD_LIBRARY_SEARCH_SYSTEM32 = 0x00000800 - -//go:linkname syscall_loadsystemlibrary syscall.loadsystemlibrary -func syscall_loadsystemlibrary(filename *uint16) (handle, err uintptr) { - handle, _, err = syscall_SyscallN(uintptr(unsafe.Pointer(_LoadLibraryExW)), uintptr(unsafe.Pointer(filename)), 0, _LOAD_LIBRARY_SEARCH_SYSTEM32) - KeepAlive(filename) - if handle != 0 { - err = 0 - } - return -} - -// golang.org/x/sys linknames syscall.loadlibrary -// (in addition to standard package syscall). -// Do not remove or change the type signature. -// -//go:linkname syscall_loadlibrary syscall.loadlibrary -func syscall_loadlibrary(filename *uint16) (handle, err uintptr) { - handle, _, err = syscall_SyscallN(uintptr(unsafe.Pointer(_LoadLibraryW)), uintptr(unsafe.Pointer(filename))) - KeepAlive(filename) - if handle != 0 { - err = 0 - } - return -} - -// golang.org/x/sys linknames syscall.getprocaddress -// (in addition to standard package syscall). -// Do not remove or change the type signature. +// syscall_syscalln calls fn with args[:n]. +// It is used to implement [syscall.SyscallN]. +// It shouldn't be used in the runtime package, +// use [stdcall] instead. // -//go:linkname syscall_getprocaddress syscall.getprocaddress -func syscall_getprocaddress(handle uintptr, procname *byte) (outhandle, err uintptr) { - outhandle, _, err = syscall_SyscallN(uintptr(unsafe.Pointer(_GetProcAddress)), handle, uintptr(unsafe.Pointer(procname))) - KeepAlive(procname) - if outhandle != 0 { - err = 0 - } - return -} - -//go:linkname syscall_Syscall syscall.Syscall -//go:nosplit -func syscall_Syscall(fn, nargs, a1, a2, a3 uintptr) (r1, r2, err uintptr) { - args := [...]uintptr{a1, a2, a3} - return syscall_SyscallN(fn, args[:nargs]...) -} - -//go:linkname syscall_Syscall6 syscall.Syscall6 +//go:linkname syscall_syscalln syscall.syscalln //go:nosplit -func syscall_Syscall6(fn, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { - args := [...]uintptr{a1, a2, a3, a4, a5, a6} - return syscall_SyscallN(fn, args[:nargs]...) -} - -//go:linkname syscall_Syscall9 syscall.Syscall9 -//go:nosplit -func syscall_Syscall9(fn, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2, err uintptr) { - args := [...]uintptr{a1, a2, a3, a4, a5, a6, a7, a8, a9} - return syscall_SyscallN(fn, args[:nargs]...) -} - -//go:linkname syscall_Syscall12 syscall.Syscall12 -//go:nosplit -func syscall_Syscall12(fn, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12 uintptr) (r1, r2, err uintptr) { - args := [...]uintptr{a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12} - return syscall_SyscallN(fn, args[:nargs]...) -} - -//go:linkname syscall_Syscall15 syscall.Syscall15 -//go:nosplit -func syscall_Syscall15(fn, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15 uintptr) (r1, r2, err uintptr) { - args := [...]uintptr{a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15} - return syscall_SyscallN(fn, args[:nargs]...) -} - -//go:linkname syscall_Syscall18 syscall.Syscall18 -//go:nosplit -func syscall_Syscall18(fn, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18 uintptr) (r1, r2, err uintptr) { - args := [...]uintptr{a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18} - return syscall_SyscallN(fn, args[:nargs]...) -} - -// maxArgs should be divisible by 2, as Windows stack -// must be kept 16-byte aligned on syscall entry. -// -// Although it only permits maximum 42 parameters, it -// is arguably large enough. -const maxArgs = 42 - -//go:linkname syscall_SyscallN syscall.SyscallN -//go:nosplit -func syscall_SyscallN(fn uintptr, args ...uintptr) (r1, r2, err uintptr) { - if len(args) > maxArgs { +//go:uintptrkeepalive +func syscall_syscalln(fn, n uintptr, args ...uintptr) (r1, r2, err uintptr) { + if n > uintptr(len(args)) { + panic("syscall: n > len(args)") // should not be reachable from user code + } + if n > windows.MaxArgs { panic("runtime: SyscallN has too many arguments") } @@ -511,15 +423,15 @@ func syscall_SyscallN(fn uintptr, args ...uintptr) (r1, r2, err uintptr) { // the stack because the stack can move during fn if it // calls back into Go. c := &getg().m.winsyscall - c.fn = fn - c.n = uintptr(len(args)) - if c.n != 0 { - c.args = uintptr(noescape(unsafe.Pointer(&args[0]))) + c.Fn = fn + c.N = n + if c.N != 0 { + c.Args = uintptr(noescape(unsafe.Pointer(&args[0]))) } cgocall(asmstdcallAddr, unsafe.Pointer(c)) // cgocall may reschedule us on to a different M, // but it copies the return values into the new M's // so we can read them from there. c = &getg().m.winsyscall - return c.r1, c.r2, c.err + return c.R1, c.R2, c.Err } diff --git a/src/runtime/syscall_windows_test.go b/src/runtime/syscall_windows_test.go index 6a056c8d2b190c..6a9b165d62422e 100644 --- a/src/runtime/syscall_windows_test.go +++ b/src/runtime/syscall_windows_test.go @@ -7,6 +7,8 @@ package runtime_test import ( "fmt" "internal/abi" + "internal/race" + "internal/runtime/syscall/windows" "internal/syscall/windows/sysdll" "internal/testenv" "io" @@ -657,7 +659,7 @@ func TestWERDialogue(t *testing.T) { if err != nil { t.Fatal(err) } - cmd := testenv.CleanCmdEnv(testenv.Command(t, exe, "-test.run=TestWERDialogue")) + cmd := testenv.CleanCmdEnv(testenv.Command(t, exe, "-test.run=^TestWERDialogue$")) cmd.Env = append(cmd.Env, "TEST_WER_DIALOGUE=1", "GOTRACEBACK=wer") // Child process should not open WER dialogue, but return immediately instead. // The exit code can't be reliably tested here because Windows can change it. @@ -668,12 +670,15 @@ func TestWERDialogue(t *testing.T) { } func TestWindowsStackMemory(t *testing.T) { + if race.Enabled { + t.Skip("skipping test: race mode uses more stack memory") + } o := runTestProg(t, "testprog", "StackMemory") stackUsage, err := strconv.Atoi(o) if err != nil { t.Fatalf("Failed to read stack usage: %v", err) } - if expected, got := 100<<10, stackUsage; got > expected { + if expected, got := 128<<10, stackUsage; got > expected { t.Fatalf("expected < %d bytes of memory per thread, got %d", expected, got) } } @@ -772,7 +777,7 @@ func TestSyscallN(t *testing.T) { t.Skipf("skipping test: GOARCH=%s", runtime.GOARCH) } - for arglen := 0; arglen <= runtime.MaxArgs; arglen++ { + for arglen := 0; arglen <= windows.MaxArgs; arglen++ { arglen := arglen t.Run(fmt.Sprintf("arg-%d", arglen), func(t *testing.T) { t.Parallel() @@ -1043,7 +1048,7 @@ func TestNumCPU(t *testing.T) { _GetProcessAffinityMask := kernel32.MustFindProc("GetProcessAffinityMask") _SetProcessAffinityMask := kernel32.MustFindProc("SetProcessAffinityMask") - cmd := exec.Command(os.Args[0], "-test.run=TestNumCPU") + cmd := exec.Command(testenv.Executable(t), "-test.run=^TestNumCPU$") cmd.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=1") var buf strings.Builder cmd.Stdout = &buf @@ -1111,12 +1116,6 @@ func TestDLLPreloadMitigation(t *testing.T) { tmpdir := t.TempDir() - dir0, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - defer os.Chdir(dir0) - const src = ` #include #include @@ -1127,7 +1126,7 @@ uintptr_t cfunc(void) { } ` srcname := "nojack.c" - err = os.WriteFile(filepath.Join(tmpdir, srcname), []byte(src), 0) + err := os.WriteFile(filepath.Join(tmpdir, srcname), []byte(src), 0) if err != nil { t.Fatal(err) } @@ -1148,7 +1147,7 @@ uintptr_t cfunc(void) { // ("nojack.dll") Think of this as the user double-clicking an // installer from their Downloads directory where a browser // silently downloaded some malicious DLLs. - os.Chdir(tmpdir) + t.Chdir(tmpdir) // First before we can load a DLL from the current directory, // loading it only as "nojack.dll", without an absolute path. @@ -1212,6 +1211,13 @@ func TestBigStackCallbackSyscall(t *testing.T) { } } +func TestSyscallStackUsage(t *testing.T) { + // Test that the stack usage of a syscall doesn't exceed the limit. + // See https://go.dev/issue/69813. + syscall.Syscall15(procSetEvent.Addr(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + syscall.Syscall18(procSetEvent.Addr(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) +} + var ( modwinmm = syscall.NewLazyDLL("winmm.dll") modkernel32 = syscall.NewLazyDLL("kernel32.dll") diff --git a/src/runtime/tagptr.go b/src/runtime/tagptr.go index 0e17a1598f5613..0e8e6cbce2e65f 100644 --- a/src/runtime/tagptr.go +++ b/src/runtime/tagptr.go @@ -6,9 +6,18 @@ package runtime // taggedPointer is a pointer with a numeric tag. // The size of the numeric tag is GOARCH-dependent, -// currently at least 10 bits. +// currently at least 16 bits. // This should only be used with pointers allocated outside the Go heap. type taggedPointer uint64 // minTagBits is the minimum number of tag bits that we expect. -const minTagBits = 10 +const minTagBits = 16 + +// # of bits we can steal from the bottom. We enforce that all pointers +// that we tag are aligned to at least this many bits. +// Currently the long pole in this tent is pollDesc at 280 bytes. Setting +// 9 here rounds those structs up to 512 bytes. +// gcBgMarkWorkerNode is also small, but we don't make many of those +// so it is ok to waste space on them. +const tagAlignBits = 9 +const tagAlign = 1 << tagAlignBits diff --git a/src/runtime/tagptr_32bit.go b/src/runtime/tagptr_32bit.go index f79e1821a1e7aa..b2175df9130ad0 100644 --- a/src/runtime/tagptr_32bit.go +++ b/src/runtime/tagptr_32bit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build 386 || arm || mips || mipsle +//go:build 386 || arm || mips || mipsle || (gccgo && (ppc || s390)) package runtime @@ -11,6 +11,9 @@ import "unsafe" // The number of bits stored in the numeric tag of a taggedPointer const taggedPointerBits = 32 +// The number of bits allowed in a tag. +const tagBits = 32 + // On 32-bit systems, taggedPointer has a 32-bit pointer and 32-bit count. // taggedPointerPack created a taggedPointer from a pointer and a tag. diff --git a/src/runtime/tagptr_64bit.go b/src/runtime/tagptr_64bit.go index 9ff11ccd16211c..76733cc1d64630 100644 --- a/src/runtime/tagptr_64bit.go +++ b/src/runtime/tagptr_64bit.go @@ -17,21 +17,22 @@ const ( // // See heapAddrBits for a table of address space sizes on // various architectures. 48 bits is enough for all - // architectures except s390x. + // arch/os combos except s390x, aix, and riscv64. // - // On AMD64, virtual addresses are 48-bit (or 57-bit) numbers sign extended to 64. - // We shift the address left 16 to eliminate the sign extended part and make - // room in the bottom for the count. + // On AMD64, virtual addresses are 48-bit (or 57-bit) sign-extended. + // Other archs are 48-bit zero-extended. + // + // We use one extra bit to placate systems which simulate amd64 binaries on + // an arm64 host. Allocated arm64 addresses could be as high as 1<<48-1, + // which would be invalid if we assumed 48-bit sign-extended addresses. + // See issue 69255. + // (Note that this does not help the other way around, simluating arm64 + // on amd64, but we don't have that problem at the moment.) // // On s390x, virtual addresses are 64-bit. There's not much we // can do about this, so we just hope that the kernel doesn't // get to really high addresses and panic if it does. - addrBits = 48 - - // In addition to the 16 bits taken from the top, we can take 3 from the - // bottom, because node must be pointer-aligned, giving a total of 19 bits - // of count. - tagBits = 64 - addrBits + 3 + defaultAddrBits = 48 + 1 // On AIX, 64-bit addresses are split into 36-bit segment number and 28-bit // offset in segment. Segment numbers in the range 0x0A0000000-0x0AFFFFFFF(LSA) @@ -39,32 +40,33 @@ const ( // We assume all tagged addresses are from memory allocated with mmap. // We use one bit to distinguish between the two ranges. aixAddrBits = 57 - aixTagBits = 64 - aixAddrBits + 3 + + // Later versions of FreeBSD enable amd64's la57 by default. + freebsdAmd64AddrBits = 57 // riscv64 SV57 mode gives 56 bits of userspace VA. // tagged pointer code supports it, // but broader support for SV57 mode is incomplete, // and there may be other issues (see #54104). riscv64AddrBits = 56 - riscv64TagBits = 64 - riscv64AddrBits + 3 -) -// The number of bits stored in the numeric tag of a taggedPointer -const taggedPointerBits = (goos.IsAix * aixTagBits) + (goarch.IsRiscv64 * riscv64TagBits) + ((1 - goos.IsAix) * (1 - goarch.IsRiscv64) * tagBits) + addrBits = goos.IsAix*aixAddrBits + goarch.IsRiscv64*riscv64AddrBits + goos.IsFreebsd*goarch.IsAmd64*freebsdAmd64AddrBits + (1-goos.IsAix)*(1-goarch.IsRiscv64)*(1-goos.IsFreebsd*goarch.IsAmd64)*defaultAddrBits + + // In addition to the 16 bits (or other, depending on arch/os) taken from the top, + // we can take 9 from the bottom, because we require pointers to be well-aligned + // (see tagptr.go:tagAlignBits). That gives us a total of 25 bits for the tag. + tagBits = 64 - addrBits + tagAlignBits +) // taggedPointerPack created a taggedPointer from a pointer and a tag. // Tag bits that don't fit in the result are discarded. func taggedPointerPack(ptr unsafe.Pointer, tag uintptr) taggedPointer { - if GOOS == "aix" { - if GOARCH != "ppc64" { - throw("check this code for aix on non-ppc64") - } - return taggedPointer(uint64(uintptr(ptr))<<(64-aixAddrBits) | uint64(tag&(1< ptr=", t.pointer(), " tag=", hex(t.tag()), "\n") + throw("taggedPointerPack") } - if GOARCH == "riscv64" { - return taggedPointer(uint64(uintptr(ptr))<<(64-riscv64AddrBits) | uint64(tag&(1<> tagBits << 3)) + return unsafe.Pointer(uintptr(int64(tp) >> tagBits << tagAlignBits)) } if GOOS == "aix" { - return unsafe.Pointer(uintptr((tp >> aixTagBits << 3) | 0xa<<56)) - } - if GOARCH == "riscv64" { - return unsafe.Pointer(uintptr(tp >> riscv64TagBits << 3)) + return unsafe.Pointer(uintptr((tp >> tagBits << tagAlignBits) | 0xa<<56)) } - return unsafe.Pointer(uintptr(tp >> tagBits << 3)) + return unsafe.Pointer(uintptr(tp >> tagBits << tagAlignBits)) } // Tag returns the tag from a taggedPointer. func (tp taggedPointer) tag() uintptr { - return uintptr(tp & (1<>> +s.stop.Wait() . +----------------------------------------------G1,G2 leak----------------------------------------------- +``` + +## Cockroach/16167 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[cockroach#16167]|[pull request]|[patch]| Resource | Double Locking | + +[cockroach#16167]:(cockroach16167_test.go) +[patch]:https://github.com/cockroachdb/cockroach/pull/16167/files +[pull request]:https://github.com/cockroachdb/cockroach/pull/16167 + +### Description + +This is another example of goroutine leaks caused by recursively +acquiring `RWLock`. +There are two lock variables (`systemConfigCond` and `systemConfigMu`) +which refer to the same underlying lock. The leak invovlves two goroutines. +The first acquires `systemConfigMu.Lock()`, then tries to acquire `systemConfigMu.RLock()`. +The second acquires `systemConfigMu.Lock()`. +If the second goroutine interleaves in between the two lock operations of the +first goroutine, both goroutines will leak. + +### Example execution + +```go +G1 G2 +--------------------------------------------------------------- +. e.Start() +. e.updateSystemConfig() +e.execParsed() . +e.systemConfigCond.L.Lock() [L1] . +. e.systemConfigMu.Lock() [L1] +e.systemConfigMu.RLock() [L1] . +------------------------G1,G2 leak----------------------------- +``` + +## Cockroach/18101 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[cockroach#18101]|[pull request]|[patch]| Resource | Double Locking | + +[cockroach#18101]:(cockroach18101_test.go) +[patch]:https://github.com/cockroachdb/cockroach/pull/18101/files +[pull request]:https://github.com/cockroachdb/cockroach/pull/18101 + +### Description + +The `context.Done()` signal short-circuits the reader goroutine, but not +the senders, leading them to leak. + +### Example execution + +```go +G1 G2 helper goroutine +-------------------------------------------------------------- +restore() +. splitAndScatter() +<-readyForImportCh . +<-readyForImportCh <==> readyForImportCh<- +... +. . cancel() +<> . <> + readyForImportCh<- +-----------------------G2 leaks-------------------------------- +``` + +## Cockroach/2448 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[cockroach#2448]|[pull request]|[patch]| Communication | Channel | + +[cockroach#2448]:(cockroach2448_test.go) +[patch]:https://github.com/cockroachdb/cockroach/pull/2448/files +[pull request]:https://github.com/cockroachdb/cockroach/pull/2448 + +### Description + +This bug is caused by two goroutines waiting for each other +to unblock their channels: + +1) `MultiRaft` sends the commit event for the Membership change +2) `store.processRaft` takes it and begins processing +3) another command commits and triggers another `sendEvent`, but + this blocks since `store.processRaft` isn't ready for another + `select`. Consequently the main `MultiRaft` loop is waiting for + that as well. +4) the `Membership` change was applied to the range, and the store + now tries to execute the callback +5) the callback tries to write to `callbackChan`, but that is + consumed by the `MultiRaft` loop, which is currently waiting + for `store.processRaft` to consume from the events channel, + which it will only do after the callback has completed. + +### Example execution + +```go +G1 G2 +-------------------------------------------------------------------------- +s.processRaft() st.start() +select . +. select [default] +. s.handleWriteResponse() +. s.sendEvent() +. select +<-s.multiraft.Events <----> m.Events <- event +. select [default] +. s.handleWriteResponse() +. s.sendEvent() +. select [m.Events<-, <-s.stopper.ShouldStop()] +callback() . +select [ + m.callbackChan<-, + <-s.stopper.ShouldStop() +] . +------------------------------G1,G2 leak---------------------------------- +``` + +## Cockroach/24808 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[cockroach#24808]|[pull request]|[patch]| Communication | Channel | + +[cockroach#24808]:(cockroach24808_test.go) +[patch]:https://github.com/cockroachdb/cockroach/pull/24808/files +[pull request]:https://github.com/cockroachdb/cockroach/pull/24808 + +### Description + +When we `Start` the `Compactor`, it may already have received +`Suggestions`, leaking the previously blocking write to a full channel. + +### Example execution + +```go +G1 +------------------------------------------------ +... +compactor.ch <- +compactor.Start() +compactor.ch <- +--------------------G1 leaks-------------------- +``` + +## Cockroach/25456 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[cockroach#25456]|[pull request]|[patch]| Communication | Channel | + +[cockroach#25456]:(cockroach25456_test.go) +[patch]:https://github.com/cockroachdb/cockroach/pull/25456/files +[pull request]:https://github.com/cockroachdb/cockroach/pull/25456 + +### Description + +When `CheckConsistency` (in the complete code) returns an error, the queue +checks whether the store is draining to decide whether the error is worth +logging. This check was incorrect and would block until the store actually +started draining. + +### Example execution + +```go +G1 +--------------------------------------- +... +<-repl.store.Stopper().ShouldQuiesce() +---------------G1 leaks---------------- +``` + +## Cockroach/35073 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[cockroach#35073]|[pull request]|[patch]| Communication | Channel | + +[cockroach#35073]:(cockroach35073_test.go) +[patch]:https://github.com/cockroachdb/cockroach/pull/35073/files +[pull request]:https://github.com/cockroachdb/cockroach/pull/35073 + +### Description + +Previously, the outbox could fail during startup without closing its +`RowChannel`. This could lead to goroutine leaks in rare cases due +to channel communication mismatch. + +### Example execution + +## Cockroach/35931 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[cockroach#35931]|[pull request]|[patch]| Communication | Channel | + +[cockroach#35931]:(cockroach35931_test.go) +[patch]:https://github.com/cockroachdb/cockroach/pull/35931/files +[pull request]:https://github.com/cockroachdb/cockroach/pull/35931 + +### Description + +Previously, if a processor that reads from multiple inputs was waiting +on one input to provide more data, and the other input was full, and +both inputs were connected to inbound streams, it was possible to +cause goroutine leaks during flow cancellation when trying to propagate +the cancellation metadata messages into the flow. The cancellation method +wrote metadata messages to each inbound stream one at a time, so if the +first one was full, the canceller would block and never send a cancellation +message to the second stream, which was the one actually being read from. + +### Example execution + +## Cockroach/3710 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[cockroach#3710]|[pull request]|[patch]| Resource | RWR Deadlock | + +[cockroach#3710]:(cockroach3710_test.go) +[patch]:https://github.com/cockroachdb/cockroach/pull/3710/files +[pull request]:https://github.com/cockroachdb/cockroach/pull/3710 + +### Description + +The goroutine leak is caused by acquiring a RLock twice in a call chain. +`ForceRaftLogScanAndProcess(acquire s.mu.RLock())` +`-> MaybeAdd()` +`-> shouldQueue()` +`-> getTruncatableIndexes()` +`->RaftStatus(acquire s.mu.Rlock())` + +### Example execution + +```go +G1 G2 +------------------------------------------------------------ +store.ForceRaftLogScanAndProcess() +s.mu.RLock() +s.raftLogQueue.MaybeAdd() +bq.impl.shouldQueue() +getTruncatableIndexes() +r.store.RaftStatus() +. store.processRaft() +. s.mu.Lock() +s.mu.RLock() +----------------------G1,G2 leak----------------------------- +``` + +## Cockroach/584 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[cockroach#584]|[pull request]|[patch]| Resource | Double Locking | + +[cockroach#584]:(cockroach584_test.go) +[patch]:https://github.com/cockroachdb/cockroach/pull/584/files +[pull request]:https://github.com/cockroachdb/cockroach/pull/584 + +### Description + +Missing call to `mu.Unlock()` before the `break` in the loop. + +### Example execution + +```go +G1 +--------------------------- +g.bootstrap() +g.mu.Lock() [L1] +if g.closed { ==> break +g.manage() +g.mu.Lock() [L1] +----------G1 leaks--------- +``` + +## Cockroach/6181 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[cockroach#6181]|[pull request]|[patch]| Resource | RWR Deadlock | + +[cockroach#6181]:(cockroach6181_test.go) +[patch]:https://github.com/cockroachdb/cockroach/pull/6181/files +[pull request]:https://github.com/cockroachdb/cockroach/pull/6181 + +### Description + +The same `RWMutex` may be recursively acquired for both reading and writing. + +### Example execution + +```go +G1 G2 G3 ... +----------------------------------------------------------------------------------------------- +testRangeCacheCoalescedRquests() +initTestDescriptorDB() +pauseLookupResumeAndAssert() +return +. doLookupWithToken() +. . doLookupWithToken() +. rc.LookupRangeDescriptor() . +. . rc.LookupRangeDescriptor() +. rdc.rangeCacheMu.RLock() . +. rdc.String() . +. . rdc.rangeCacheMu.RLock() +. . fmt.Printf() +. . rdc.rangeCacheMu.RUnlock() +. . rdc.rangeCacheMu.Lock() +. rdc.rangeCacheMu.RLock() . +-----------------------------------G2,G3,... leak---------------------------------------------- +``` + +## Cockroach/7504 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[cockroach#7504]|[pull request]|[patch]| Resource | AB-BA Deadlock | + +[cockroach#7504]:(cockroach7504_test.go) +[patch]:https://github.com/cockroachdb/cockroach/pull/7504/files +[pull request]:https://github.com/cockroachdb/cockroach/pull/7504 + +### Description + +The locks are acquired as `leaseState` and `tableNameCache` in `Release()`, but +as `tableNameCache` and `leaseState` in `AcquireByName`, leading to an AB-BA deadlock. + +### Example execution + +```go +G1 G2 +----------------------------------------------------- +mgr.AcquireByName() mgr.Release() +m.tableNames.get(id) . +c.mu.Lock() [L2] . +. t.release(lease) +. t.mu.Lock() [L3] +. s.mu.Lock() [L1] +lease.mu.Lock() [L1] . +. t.removeLease(s) +. t.tableNameCache.remove() +. c.mu.Lock() [L2] +---------------------G1, G2 leak--------------------- +``` + +## Cockroach/9935 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[cockroach#9935]|[pull request]|[patch]| Resource | Double Locking | + +[cockroach#9935]:(cockroach9935_test.go) +[patch]:https://github.com/cockroachdb/cockroach/pull/9935/files +[pull request]:https://github.com/cockroachdb/cockroach/pull/9935 + +### Description + +This bug is caused by acquiring `l.mu.Lock()` twice. + +### Example execution + +## Etcd/10492 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[etcd#10492]|[pull request]|[patch]| Resource | Double locking | + +[etcd#10492]:(etcd10492_test.go) +[patch]:https://github.com/etcd-io/etcd/pull/10492/files +[pull request]:https://github.com/etcd-io/etcd/pull/10492 + +### Description + +A simple double locking case for lines 19, 31. + +## Etcd/5509 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[etcd#5509]|[pull request]|[patch]| Resource | Double locking | + +[etcd#5509]:(etcd5509_test.go) +[patch]:https://github.com/etcd-io/etcd/pull/5509/files +[pull request]:https://github.com/etcd-io/etcd/pull/5509 + +### Description + +`r.acquire()` returns holding `r.client.mu.RLock()` on a failure path (line 42). +This causes any call to `client.Close()` to leak goroutines. + +## Etcd/6708 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[etcd#6708]|[pull request]|[patch]| Resource | Double locking | + +[etcd#6708]:(etcd6708_test.go) +[patch]:https://github.com/etcd-io/etcd/pull/6708/files +[pull request]:https://github.com/etcd-io/etcd/pull/6708 + +### Description + +Line 54, 49 double locking + +## Etcd/6857 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[etcd#6857]|[pull request]|[patch]| Communication | Channel | + +[etcd#6857]:(etcd6857_test.go) +[patch]:https://github.com/etcd-io/etcd/pull/6857/files +[pull request]:https://github.com/etcd-io/etcd/pull/6857 + +### Description + +Choosing a different case in a `select` statement (`n.stop`) will +lead to goroutine leaks when sending over `n.status`. + +### Example execution + +```go +G1 G2 G3 +------------------------------------------- +n.run() . . +. . n.Stop() +. . n.stop<- +<-n.stop . . +. . <-n.done +close(n.done) . . +return . . +. . return +. n.Status() +. n.status<- +----------------G2 leaks------------------- +``` + +## Etcd/6873 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[etcd#6873]|[pull request]|[patch]| Mixed | Channel & Lock | + +[etcd#6873]:(etcd6873_test.go) +[patch]:https://github.com/etcd-io/etcd/pull/6873/files +[pull request]:https://github.com/etcd-io/etcd/pull/6873 + +### Description + +This goroutine leak involves a goroutine acquiring a lock and being +blocked over a channel operation with no partner, while another tries +to acquire the same lock. + +### Example execution + +```go +G1 G2 G3 +-------------------------------------------------------------- +newWatchBroadcasts() +wbs.update() +wbs.updatec <- +return +. <-wbs.updatec . +. wbs.coalesce() . +. . wbs.stop() +. . wbs.mu.Lock() +. . close(wbs.updatec) +. . <-wbs.donec +. wbs.mu.Lock() . +---------------------G2,G3 leak-------------------------------- +``` + +## Etcd/7492 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[etcd#7492]|[pull request]|[patch]| Mixed | Channel & Lock | + +[etcd#7492]:(etcd7492_test.go) +[patch]:https://github.com/etcd-io/etcd/pull/7492/files +[pull request]:https://github.com/etcd-io/etcd/pull/7492 + +### Description + +This goroutine leak involves a goroutine acquiring a lock and being +blocked over a channel operation with no partner, while another tries +to acquire the same lock. + +### Example execution + +```go +G2 G1 +--------------------------------------------------------------- +. stk.run() +ts.assignSimpleTokenToUser() . +t.simpleTokensMu.Lock() . +t.simpleTokenKeeper.addSimpleToken() . +tm.addSimpleTokenCh <- true . +. <-tm.addSimpleTokenCh +t.simpleTokensMu.Unlock() . +ts.assignSimpleTokenToUser() . +... +t.simpleTokensMu.Lock() +. <-tokenTicker.C +tm.addSimpleTokenCh <- true . +. tm.deleteTokenFunc() +. t.simpleTokensMu.Lock() +---------------------------G1,G2 leak-------------------------- +``` + +## Etcd/7902 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[etcd#7902]|[pull request]|[patch]| Mixed | Channel & Lock | + +[etcd#7902]:(etcd7902_test.go) +[patch]:https://github.com/etcd-io/etcd/pull/7902/files +[pull request]:https://github.com/etcd-io/etcd/pull/7902 + +### Description + +If the follower gooroutine acquires `mu.Lock()` first and calls +`rc.release()`, it will be blocked sending over `rcNextc`. +Only the leader can `close(nextc)` to unblock the follower. +However, in order to invoke `rc.release()`, the leader needs +to acquires `mu.Lock()`. +The fix is to remove the lock and unlock around `rc.release()`. + +### Example execution + +```go +G1 G2 (leader) G3 (follower) +--------------------------------------------------------------------- +runElectionFunc() +doRounds() +wg.Wait() +. ... +. mu.Lock() +. rc.validate() +. rcNextc = nextc +. mu.Unlock() ... +. . mu.Lock() +. . rc.validate() +. . mu.Unlock() +. . mu.Lock() +. . rc.release() +. . <-rcNextc +. mu.Lock() +-------------------------G1,G2,G3 leak-------------------------- +``` + +## Grpc/1275 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[grpc#1275]|[pull request]|[patch]| Communication | Channel | + +[grpc#1275]:(grpc1275_test.go) +[patch]:https://github.com/grpc/grpc-go/pull/1275/files +[pull request]:https://github.com/grpc/grpc-go/pull/1275 + +### Description + +Two goroutines are involved in this leak. The main goroutine +is blocked at `case <- donec`, and is waiting for the second goroutine +to close the channel. +The second goroutine is created by the main goroutine. It is blocked +when calling `stream.Read()`, which invokes `recvBufferRead.Read()`. +The second goroutine is blocked at case `i := r.recv.get()`, and it is +waiting for someone to send a message to this channel. +It is the `client.CloseSream()` method called by the main goroutine that +should send the message, but it is not. The patch is to send out this message. + +### Example execution + +```go +G1 G2 +----------------------------------------------------- +testInflightStreamClosing() +. stream.Read() +. io.ReadFull() +. <-r.recv.get() +CloseStream() +<-donec +---------------------G1, G2 leak--------------------- +``` + +## Grpc/1424 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[grpc#1424]|[pull request]|[patch]| Communication | Channel | + +[grpc#1424]:(grpc1424_test.go) +[patch]:https://github.com/grpc/grpc-go/pull/1424/files +[pull request]:https://github.com/grpc/grpc-go/pull/1424 + +### Description + +The goroutine running `cc.lbWatcher` returns without +draining the `done` channel. + +### Example execution + +```go +G1 G2 G3 +----------------------------------------------------------------- +DialContext() . . +. cc.dopts.balancer.Notify() . +. . cc.lbWatcher() +. <-doneChan +close() +---------------------------G2 leaks------------------------------- +``` + +## Grpc/1460 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[grpc#1460]|[pull request]|[patch]| Mixed | Channel & Lock | + +[grpc#1460]:(grpc1460_test.go) +[patch]:https://github.com/grpc/grpc-go/pull/1460/files +[pull request]:https://github.com/grpc/grpc-go/pull/1460 + +### Description + +When gRPC keepalives are enabled (which isn't the case +by default at this time) and PermitWithoutStream is false +(the default), the client can leak goroutines when transitioning +between having no active stream and having one active +stream.The keepalive() goroutine is stuck at “<-t.awakenKeepalive”, +while the main goroutine is stuck in NewStream() on t.mu.Lock(). + +### Example execution + +```go +G1 G2 +-------------------------------------------- +client.keepalive() +. client.NewStream() +t.mu.Lock() +<-t.awakenKeepalive +. t.mu.Lock() +---------------G1,G2 leak------------------- +``` + +## Grpc/3017 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[grpc#3017]|[pull request]|[patch]| Resource | Missing unlock | + +[grpc#3017]:(grpc3017_test.go) +[patch]:https://github.com/grpc/grpc-go/pull/3017/files +[pull request]:https://github.com/grpc/grpc-go/pull/3017 + +### Description + +Line 65 is an execution path with a missing unlock. + +### Example execution + +```go +G1 G2 G3 +------------------------------------------------------------------------------------------------ +NewSubConn([1]) +ccc.mu.Lock() [L1] +sc = 1 +ccc.subConnToAddr[1] = 1 +go func() [G2] +<-done . +. ccc.RemoveSubConn(1) +. ccc.mu.Lock() +. addr = 1 +. entry = &subConnCacheEntry_grpc3017{} +. cc.subConnCache[1] = entry +. timer = time.AfterFunc() [G3] +. entry.cancel = func() +. sc = ccc.NewSubConn([1]) +. ccc.mu.Lock() [L1] +. entry.cancel() +. !timer.Stop() [true] +. entry.abortDeleting = true +. . ccc.mu.Lock() +. . <<>> +. ccc.RemoveSubConn(1) +. ccc.mu.Lock() [L1] +-------------------------------------------G1, G2 leak----------------------------------------- +``` + +## Grpc/660 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[grpc#660]|[pull request]|[patch]| Communication | Channel | + +[grpc#660]:(grpc660_test.go) +[patch]:https://github.com/grpc/grpc-go/pull/660/files +[pull request]:https://github.com/grpc/grpc-go/pull/660 + +### Description + +The parent function could return without draining the done channel. + +### Example execution + +```go +G1 G2 helper goroutine +------------------------------------------------------------- +doCloseLoopUnary() +. bc.stop <- true +<-bc.stop +return +. done <- +----------------------G2 leak-------------------------------- +``` + +## Grpc/795 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[grpc#795]|[pull request]|[patch]| Resource | Double locking | + +[grpc#795]:(grpc795_test.go) +[patch]:https://github.com/grpc/grpc-go/pull/795/files +[pull request]:https://github.com/grpc/grpc-go/pull/795 + +### Description + +Line 20 is an execution path with a missing unlock. + +## Grpc/862 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[grpc#862]|[pull request]|[patch]| Communication | Channel & Context | + +[grpc#862]:(grpc862_test.go) +[patch]:https://github.com/grpc/grpc-go/pull/862/files +[pull request]:https://github.com/grpc/grpc-go/pull/862 + +### Description + +When return value `conn` is `nil`, `cc(ClientConn)` is not closed. +The goroutine executing resetAddrConn is leaked. The patch is to +close `ClientConn` in `defer func()`. + +### Example execution + +```go +G1 G2 +--------------------------------------- +DialContext() +. cc.resetAddrConn() +. resetTransport() +. <-ac.ctx.Done() +--------------G2 leak------------------ +``` + +## Hugo/3251 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[hugo#3251]|[pull request]|[patch]| Resource | RWR deadlock | + +[hugo#3251]:(hugo3251_test.go) +[patch]:https://github.com/gohugoio/hugo/pull/3251/files +[pull request]:https://github.com/gohugoio/hugo/pull/3251 + +### Description + +A goroutine can hold `Lock()` at line 20 then acquire `RLock()` at +line 29. `RLock()` at line 29 will never be acquired because `Lock()` +at line 20 will never be released. + +### Example execution + +```go +G1 G2 G3 +------------------------------------------------------------------------------------------ +wg.Add(1) [W1: 1] +go func() [G2] +go func() [G3] +. resGetRemote() +. remoteURLLock.URLLock(url) +. l.Lock() [L1] +. l.m[url] = &sync.Mutex{} [L2] +. l.m[url].Lock() [L2] +. l.Unlock() [L1] +. . resGetRemote() +. . remoteURLLock.URLLock(url) +. . l.Lock() [L1] +. . l.m[url].Lock() [L2] +. remoteURLLock.URLUnlock(url) +. l.RLock() [L1] +... +wg.Wait() [W1] +----------------------------------------G1,G2,G3 leak-------------------------------------- +``` + +## Hugo/5379 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[hugo#5379]|[pull request]|[patch]| Resource | Double locking | + +[hugo#5379]:(hugo5379_test.go) +[patch]:https://github.com/gohugoio/hugo/pull/5379/files +[pull request]:https://github.com/gohugoio/hugo/pull/5379 + +### Description + +A goroutine first acquire `contentInitMu` at line 99 then +acquire the same `Mutex` at line 66 + +## Istio/16224 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[istio#16224]|[pull request]|[patch]| Mixed | Channel & Lock | + +[istio#16224]:(istio16224_test.go) +[patch]:https://github.com/istio/istio/pull/16224/files +[pull request]:https://github.com/istio/istio/pull/16224 + +### Description + +A goroutine holds a `Mutex` at line 91 and is then blocked at line 93. +Another goroutine attempts to acquire the same `Mutex` at line 101 to +further drains the same channel at 103. + +## Istio/17860 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[istio#17860]|[pull request]|[patch]| Communication | Channel | + +[istio#17860]:(istio17860_test.go) +[patch]:https://github.com/istio/istio/pull/17860/files +[pull request]:https://github.com/istio/istio/pull/17860 + +### Description + +`a.statusCh` can't be drained at line 70. + +## Istio/18454 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[istio#18454]|[pull request]|[patch]| Communication | Channel & Context | + +[istio#18454]:(istio18454_test.go) +[patch]:https://github.com/istio/istio/pull/18454/files +[pull request]:https://github.com/istio/istio/pull/18454 + +### Description + +`s.timer.Stop()` at line 56 and 61 can be called concurrency +(i.e. from their entry point at line 104 and line 66). +See [Timer](https://golang.org/pkg/time/#Timer). + +## Kubernetes/10182 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[kubernetes#10182]|[pull request]|[patch]| Mixed | Channel & Lock | + +[kubernetes#10182]:(kubernetes10182_test.go) +[patch]:https://github.com/kubernetes/kubernetes/pull/10182/files +[pull request]:https://github.com/kubernetes/kubernetes/pull/10182 + +### Description + +Goroutine 1 is blocked on a lock held by goroutine 3, +while goroutine 3 is blocked on sending message to `ch`, +which is read by goroutine 1. + +### Example execution + +```go +G1 G2 G3 +------------------------------------------------------------------------------- +s.Start() +s.syncBatch() +. s.SetPodStatus() +. s.podStatusesLock.Lock() +<-s.podStatusChannel <===> s.podStatusChannel <- true +. s.podStatusesLock.Unlock() +. return +s.DeletePodStatus() . +. . s.podStatusesLock.Lock() +. . s.podStatusChannel <- true +s.podStatusesLock.Lock() +-----------------------------G1,G3 leak----------------------------------------- +``` + +## Kubernetes/11298 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[kubernetes#11298]|[pull request]|[patch]| Communication | Channel & Condition Variable | + +[kubernetes#11298]:(kubernetes11298_test.go) +[patch]:https://github.com/kubernetes/kubernetes/pull/11298/files +[pull request]:https://github.com/kubernetes/kubernetes/pull/11298 + +### Description + +`n.node` used the `n.lock` as underlaying locker. The service loop initially +locked it, the `Notify` function tried to lock it before calling `n.node.Signal()`, +leading to a goroutine leak. `n.cond.Signal()` at line 59 and line 81 are not +guaranteed to unblock the `n.cond.Wait` at line 56. + +## Kubernetes/13135 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[kubernetes#13135]|[pull request]|[patch]| Resource | AB-BA deadlock | + +[kubernetes#13135]:(kubernetes13135_test.go) +[patch]:https://github.com/kubernetes/kubernetes/pull/13135/files +[pull request]:https://github.com/kubernetes/kubernetes/pull/13135 + +### Description + +```go +G1 G2 G3 +---------------------------------------------------------------------------------- +NewCacher() +watchCache.SetOnReplace() +watchCache.SetOnEvent() +. cacher.startCaching() +. c.Lock() +. c.reflector.ListAndWatch() +. r.syncWith() +. r.store.Replace() +. w.Lock() +. w.onReplace() +. cacher.initOnce.Do() +. cacher.Unlock() +return cacher . +. . c.watchCache.Add() +. . w.processEvent() +. . w.Lock() +. cacher.startCaching() . +. c.Lock() . +... +. c.Lock() +. w.Lock() +--------------------------------G2,G3 leak----------------------------------------- +``` + +## Kubernetes/1321 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[kubernetes#1321]|[pull request]|[patch]| Mixed | Channel & Lock | + +[kubernetes#1321]:(kubernetes1321_test.go) +[patch]:https://github.com/kubernetes/kubernetes/pull/1321/files +[pull request]:https://github.com/kubernetes/kubernetes/pull/1321 + +### Description + +This is a lock-channel bug. The first goroutine invokes +`distribute()`, which holds `m.lock.Lock()`, while blocking +at sending message to `w.result`. The second goroutine +invokes `stopWatching()` function, which can unblock the first +goroutine by closing `w.result`. However, in order to close `w.result`, +`stopWatching()` function needs to acquire `m.lock.Lock()`. + +The fix is to introduce another channel and put receive message +from the second channel in the same `select` statement as the +`w.result`. Close the second channel can unblock the first +goroutine, while no need to hold `m.lock.Lock()`. + +### Example execution + +```go +G1 G2 +---------------------------------------------- +testMuxWatcherClose() +NewMux() +. m.loop() +. m.distribute() +. m.lock.Lock() +. w.result <- true +w := m.Watch() +w.Stop() +mw.m.stopWatching() +m.lock.Lock() +---------------G1,G2 leak--------------------- +``` + +## Kubernetes/25331 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[kubernetes#25331]|[pull request]|[patch]| Communication | Channel & Context | + +[kubernetes#25331]:(kubernetes25331_test.go) +[patch]:https://github.com/kubernetes/kubernetes/pull/25331/files +[pull request]:https://github.com/kubernetes/kubernetes/pull/25331 + +### Description + +A potential goroutine leak occurs when an error has happened, +blocking `resultChan`, while cancelling context in `Stop()`. + +### Example execution + +```go +G1 G2 +------------------------------------ +wc.run() +. wc.Stop() +. wc.errChan <- +. wc.cancel() +<-wc.errChan +wc.cancel() +wc.resultChan <- +-------------G1 leak---------------- +``` + +## Kubernetes/26980 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[kubernetes#26980]|[pull request]|[patch]| Mixed | Channel & Lock | + +[kubernetes#26980]:(kubernetes26980_test.go) +[patch]:https://github.com/kubernetes/kubernetes/pull/26980/files +[pull request]:https://github.com/kubernetes/kubernetes/pull/26980 + +### Description + +A goroutine holds a `Mutex` at line 24 and blocked at line 35. +Another goroutine blocked at line 58 by acquiring the same `Mutex`. + +## Kubernetes/30872 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[kubernetes#30872]|[pull request]|[patch]| Resource | AB-BA deadlock | + +[kubernetes#30872]:(kubernetes30872_test.go) +[patch]:https://github.com/kubernetes/kubernetes/pull/30872/files +[pull request]:https://github.com/kubernetes/kubernetes/pull/30872 + +### Description + +The lock is acquired both at lines 92 and 157. + +## Kubernetes/38669 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[kubernetes#38669]|[pull request]|[patch]| Communication | Channel | + +[kubernetes#38669]:(kubernetes38669_test.go) +[patch]:https://github.com/kubernetes/kubernetes/pull/38669/files +[pull request]:https://github.com/kubernetes/kubernetes/pull/38669 + +### Description + +No sender for line 33. + +## Kubernetes/5316 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[kubernetes#5316]|[pull request]|[patch]| Communication | Channel | + +[kubernetes#5316]:(kubernetes5316_test.go) +[patch]:https://github.com/kubernetes/kubernetes/pull/5316/files +[pull request]:https://github.com/kubernetes/kubernetes/pull/5316 + +### Description + +If the main goroutine selects a case that doesn’t consumes +the channels, the anonymous goroutine will be blocked on sending +to channel. + +### Example execution + +```go +G1 G2 +-------------------------------------- +finishRequest() +. fn() +time.After() +. errCh<-/ch<- +--------------G2 leaks---------------- +``` + +## Kubernetes/58107 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[kubernetes#58107]|[pull request]|[patch]| Resource | RWR deadlock | + +[kubernetes#58107]:(kubernetes58107_test.go) +[patch]:https://github.com/kubernetes/kubernetes/pull/58107/files +[pull request]:https://github.com/kubernetes/kubernetes/pull/58107 + +### Description + +The rules for read and write lock: allows concurrent read lock; +write lock has higher priority than read lock. + +There are two queues (queue 1 and queue 2) involved in this bug, +and the two queues are protected by the same read-write lock +(`rq.workerLock.RLock()`). Before getting an element from queue 1 or +queue 2, `rq.workerLock.RLock()` is acquired. If the queue is empty, +`cond.Wait()` will be invoked. There is another goroutine (goroutine D), +which will periodically invoke `rq.workerLock.Lock()`. Under the following +situation, deadlock will happen. Queue 1 is empty, so that some goroutines +hold `rq.workerLock.RLock()`, and block at `cond.Wait()`. Goroutine D is +blocked when acquiring `rq.workerLock.Lock()`. Some goroutines try to process +jobs in queue 2, but they are blocked when acquiring `rq.workerLock.RLock()`, +since write lock has a higher priority. + +The fix is to not acquire `rq.workerLock.RLock()`, while pulling data +from any queue. Therefore, when a goroutine is blocked at `cond.Wait()`, +`rq.workLock.RLock()` is not held. + +### Example execution + +```go +G3 G4 G5 +-------------------------------------------------------------------- +. . Sync() +rq.workerLock.RLock() . . +q.cond.Wait() . . +. . rq.workerLock.Lock() +. rq.workerLock.RLock() +. q.cond.L.Lock() +-----------------------------G3,G4,G5 leak----------------------------- +``` + +## Kubernetes/62464 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[kubernetes#62464]|[pull request]|[patch]| Resource | RWR deadlock | + +[kubernetes#62464]:(kubernetes62464_test.go) +[patch]:https://github.com/kubernetes/kubernetes/pull/62464/files +[pull request]:https://github.com/kubernetes/kubernetes/pull/62464 + +### Description + +This is another example for recursive read lock bug. It has +been noticed by the go developers that RLock should not be +recursively used in the same thread. + +### Example execution + +```go +G1 G2 +-------------------------------------------------------- +m.reconcileState() +m.state.GetCPUSetOrDefault() +s.RLock() +s.GetCPUSet() +. p.RemoveContainer() +. s.GetDefaultCPUSet() +. s.SetDefaultCPUSet() +. s.Lock() +s.RLock() +---------------------G1,G2 leak-------------------------- +``` + +## Kubernetes/6632 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[kubernetes#6632]|[pull request]|[patch]| Mixed | Channel & Lock | + +[kubernetes#6632]:(kubernetes6632_test.go) +[patch]:https://github.com/kubernetes/kubernetes/pull/6632/files +[pull request]:https://github.com/kubernetes/kubernetes/pull/6632 + +### Description + +When `resetChan` is full, `WriteFrame` holds the lock and blocks +on the channel. Then `monitor()` fails to close the `resetChan` +because the lock is already held by `WriteFrame`. + + +### Example execution + +```go +G1 G2 helper goroutine +---------------------------------------------------------------- +i.monitor() +<-i.conn.closeChan +. i.WriteFrame() +. i.writeLock.Lock() +. i.resetChan <- +. . i.conn.closeChan<- +i.writeLock.Lock() +----------------------G1,G2 leak-------------------------------- +``` + +## Kubernetes/70277 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[kubernetes#70277]|[pull request]|[patch]| Communication | Channel | + +[kubernetes#70277]:kubernetes70277_test.go +[patch]:https://github.com/kubernetes/kubernetes/pull/70277/files +[pull request]:https://github.com/kubernetes/kubernetes/pull/70277 + +### Description + +`wait.poller()` returns a function with type `WaitFunc`. +the function creates a goroutine and the goroutine only +quits when after or done closed. + +The `doneCh` defined at line 70 is never closed. + +## Moby/17176 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[moby#17176]|[pull request]|[patch]| Resource | Double locking | + +[moby#17176]:(moby17176_test.go) +[patch]:https://github.com/moby/moby/pull/17176/files +[pull request]:https://github.com/moby/moby/pull/17176 + +### Description + +`devices.nrDeletedDevices` takes `devices.Lock()` but does +not release it (line 36) if there are no deleted devices. This will block +other goroutines trying to acquire `devices.Lock()`. + +## Moby/21233 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[moby#21233]|[pull request]|[patch]| Communication | Channel | + +[moby#21233]:(moby21233_test.go) +[patch]:https://github.com/moby/moby/pull/21233/files +[pull request]:https://github.com/moby/moby/pull/21233 + +### Description + +This test was checking that it received every progress update that was +produced. But delivery of these intermediate progress updates is not +guaranteed. A new update can overwrite the previous one if the previous +one hasn't been sent to the channel yet. + +The call to `t.Fatalf` terminated the current goroutine which was consuming +the channel, which caused a deadlock and eventual test timeout rather +than a proper failure message. + +### Example execution + +```go +G1 G2 G3 +---------------------------------------------------------- +testTransfer() . . +tm.Transfer() . . +t.Watch() . . +. WriteProgress() . +. ProgressChan<- . +. . <-progressChan +. ... ... +. return . +. <-progressChan +<-watcher.running +----------------------G1,G3 leak-------------------------- +``` + +## Moby/25384 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[moby#25384]|[pull request]|[patch]| Mixed | Misuse WaitGroup | + +[moby#25384]:(moby25384_test.go) +[patch]:https://github.com/moby/moby/pull/25384/files +[pull request]:https://github.com/moby/moby/pull/25384 + +### Description + +When `n=1` (where `n` is `len(pm.plugins)`), the location of `group.Wait()` doesn’t matter. +When `n > 1`, `group.Wait()` is invoked in each iteration. Whenever +`group.Wait()` is invoked, it waits for `group.Done()` to be executed `n` times. +However, `group.Done()` is only executed once in one iteration. + +Misuse of sync.WaitGroup + +## Moby/27782 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[moby#27782]|[pull request]|[patch]| Communication | Channel & Condition Variable | + +[moby#27782]:(moby27782_test.go) +[patch]:https://github.com/moby/moby/pull/27782/files +[pull request]:https://github.com/moby/moby/pull/27782 + +### Description + +### Example execution + +```go +G1 G2 G3 +----------------------------------------------------------------------- +InitializeStdio() +startLogging() +l.ReadLogs() +NewLogWatcher() +. l.readLogs() +container.Reset() . +LogDriver.Close() . +r.Close() . +close(w.closeNotifier) . +. followLogs(logWatcher) +. watchFile() +. New() +. NewEventWatcher() +. NewWatcher() +. . w.readEvents() +. . event.ignoreLinux() +. . return false +. <-logWatcher.WatchClose() . +. fileWatcher.Remove() . +. w.cv.Wait() . +. . w.Events <- event +------------------------------G2,G3 leak------------------------------- +``` + +## Moby/28462 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[moby#28462]|[pull request]|[patch]| Mixed | Channel & Lock | + +[moby#28462]:(moby28462_test.go) +[patch]:https://github.com/moby/moby/pull/28462/files +[pull request]:https://github.com/moby/moby/pull/28462 + +### Description + +One goroutine may acquire a lock and try to send a message over channel `stop`, +while the other will try to acquire the same lock. With the wrong ordering, +both goroutines will leak. + +### Example execution + +```go +G1 G2 +-------------------------------------------------------------- +monitor() +handleProbeResult() +. d.StateChanged() +. c.Lock() +. d.updateHealthMonitorElseBranch() +. h.CloseMonitorChannel() +. s.stop <- struct{}{} +c.Lock() +----------------------G1,G2 leak------------------------------ +``` + +## Moby/30408 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[moby#30408]|[pull request]|[patch]| Communication | Condition Variable | + +[moby#30408]:(moby30408_test.go) +[patch]:https://github.com/moby/moby/pull/30408/files +[pull request]:https://github.com/moby/moby/pull/30408 + +### Description + +`Wait()` at line 22 has no corresponding `Signal()` or `Broadcast()`. + +### Example execution + +```go +G1 G2 +------------------------------------------ +testActive() +. p.waitActive() +. p.activateWait.L.Lock() +. p.activateWait.Wait() +<-done +-----------------G1,G2 leak--------------- +``` + +## Moby/33781 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[moby#33781]|[pull request]|[patch]| Communication | Channel & Context | + +[moby#33781]:(moby33781_test.go) +[patch]:https://github.com/moby/moby/pull/33781/files +[pull request]:https://github.com/moby/moby/pull/33781 + +### Description + +The goroutine created using an anonymous function is blocked +sending a message over an unbuffered channel. However there +exists a path in the parent goroutine where the parent function +will return without draining the channel. + +### Example execution + +```go +G1 G2 G3 +---------------------------------------- +monitor() . +<-time.After() . +. . +<-stop stop<- +. +cancelProbe() +return +. result<- +----------------G3 leak------------------ +``` + +## Moby/36114 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[moby#36114]|[pull request]|[patch]| Resource | Double locking | + +[moby#36114]:(moby36114_test.go) +[patch]:https://github.com/moby/moby/pull/36114/files +[pull request]:https://github.com/moby/moby/pull/36114 + +### Description + +The the lock for the struct svm has already been locked when calling +`svm.hotRemoveVHDsAtStart()`. + +## Moby/4951 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[moby#4951]|[pull request]|[patch]| Resource | AB-BA deadlock | + +[moby#4951]:(moby4951_test.go) +[patch]:https://github.com/moby/moby/pull/4951/files +[pull request]:https://github.com/moby/moby/pull/4951 + +### Description + +The root cause and patch is clearly explained in the commit +description. The global lock is `devices.Lock()`, and the device +lock is `baseInfo.lock.Lock()`. It is very likely that this bug +can be reproduced. + +## Moby/7559 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[moby#7559]|[pull request]|[patch]| Resource | Double locking | + +[moby#7559]:(moby7559_test.go) +[patch]:https://github.com/moby/moby/pull/7559/files +[pull request]:https://github.com/moby/moby/pull/7559 + +### Description + +Line 25 is missing a call to `.Unlock`. + +### Example execution + +```go +G1 +--------------------------- +proxy.connTrackLock.Lock() +if err != nil { continue } +proxy.connTrackLock.Lock() +-----------G1 leaks-------- +``` + +## Serving/2137 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[serving#2137]|[pull request]|[patch]| Mixed | Channel & Lock | + +[serving#2137]:(serving2137_test.go) +[patch]:https://github.com/ knative/serving/pull/2137/files +[pull request]:https://github.com/ knative/serving/pull/2137 + +### Description + +### Example execution + +```go +G1 G2 G3 +---------------------------------------------------------------------------------- +b.concurrentRequests(2) . . +b.concurrentRequest() . . +r.lock.Lock() . . +. start.Done() . +start.Wait() . . +b.concurrentRequest() . . +r.lock.Lock() . . +. . start.Done() +start.Wait() . . +unlockAll(locks) . . +unlock(lc) . . +req.lock.Unlock() . . +ok := <-req.accepted . . +. b.Maybe() . +. b.activeRequests <- t . +. thunk() . +. r.lock.Lock() . +. . b.Maybe() +. . b.activeRequests <- t +----------------------------G1,G2,G3 leak----------------------------------------- +``` + +## Syncthing/4829 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[syncthing#4829]|[pull request]|[patch]| Resource | Double locking | + +[syncthing#4829]:(syncthing4829_test.go) +[patch]:https://github.com/syncthing/syncthing/pull/4829/files +[pull request]:https://github.com/syncthing/syncthing/pull/4829 + +### Description + +Double locking at line 17 and line 30. + +### Example execution + +```go +G1 +--------------------------- +mapping.clearAddresses() +m.mut.Lock() [L2] +m.notify(...) +m.mut.RLock() [L2] +----------G1 leaks--------- +``` + +## Syncthing/5795 + +| Bug ID | Ref | Patch | Type | Sub-type | +| ---- | ---- | ---- | ---- | ---- | +|[syncthing#5795]|[pull request]|[patch]| Communication | Channel | + +[syncthing#5795]:(syncthing5795_test.go) +[patch]:https://github.com/syncthing/syncthing/pull/5795/files +[pull request]:https://github.com/syncthing/syncthing/pull/5795 + +### Description + +`<-c.dispatcherLoopStopped` at line 82 is blocking forever because +`dispatcherLoop()` is blocking at line 72. + +### Example execution + +```go +G1 G2 +-------------------------------------------------------------- +c.Start() +go c.dispatcherLoop() [G3] +. select [<-c.inbox, <-c.closed] +c.inbox <- <================> [<-c.inbox] +<-c.dispatcherLoopStopped . +. default +. c.ccFn()/c.Close() +. close(c.closed) +. <-c.dispatcherLoopStopped +---------------------G1,G2 leak------------------------------- +``` diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach10214.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach10214.go new file mode 100644 index 00000000000000..4f5ef3b0fc32fc --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach10214.go @@ -0,0 +1,145 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: cockroach + * Issue or PR : https://github.com/cockroachdb/cockroach/pull/10214 + * Buggy version: 7207111aa3a43df0552509365fdec741a53f873f + * fix commit-id: 27e863d90ab0660494778f1c35966cc5ddc38e32 + * Flaky: 3/100 + * Description: This goroutine leak is caused by different order when acquiring + * coalescedMu.Lock() and raftMu.Lock(). The fix is to refactor sendQueuedHeartbeats() + * so that cockroachdb can unlock coalescedMu before locking raftMu. + */ +package main + +import ( + "os" + "runtime/pprof" + "sync" + "time" + "unsafe" +) + +func init() { + register("Cockroach10214", Cockroach10214) +} + +type Store_cockroach10214 struct { + coalescedMu struct { + sync.Mutex // L1 + heartbeatResponses []int + } + mu struct { + replicas map[int]*Replica_cockroach10214 + } +} + +func (s *Store_cockroach10214) sendQueuedHeartbeats() { + s.coalescedMu.Lock() // L1 acquire + defer s.coalescedMu.Unlock() // L2 release + for i := 0; i < len(s.coalescedMu.heartbeatResponses); i++ { + s.sendQueuedHeartbeatsToNode() // L2 + } +} + +func (s *Store_cockroach10214) sendQueuedHeartbeatsToNode() { + for i := 0; i < len(s.mu.replicas); i++ { + r := s.mu.replicas[i] + r.reportUnreachable() // L2 + } +} + +type Replica_cockroach10214 struct { + raftMu sync.Mutex // L2 + mu sync.Mutex // L3 + store *Store_cockroach10214 +} + +func (r *Replica_cockroach10214) reportUnreachable() { + r.raftMu.Lock() // L2 acquire + time.Sleep(time.Millisecond) + defer r.raftMu.Unlock() // L2 release +} + +func (r *Replica_cockroach10214) tick() { + r.raftMu.Lock() // L2 acquire + defer r.raftMu.Unlock() // L2 release + r.tickRaftMuLocked() +} + +func (r *Replica_cockroach10214) tickRaftMuLocked() { + r.mu.Lock() // L3 acquire + defer r.mu.Unlock() // L3 release + if r.maybeQuiesceLocked() { + return + } +} + +func (r *Replica_cockroach10214) maybeQuiesceLocked() bool { + for i := 0; i < 2; i++ { + if !r.maybeCoalesceHeartbeat() { + return true + } + } + return false +} + +func (r *Replica_cockroach10214) maybeCoalesceHeartbeat() bool { + msgtype := uintptr(unsafe.Pointer(r)) % 3 + switch msgtype { + case 0, 1, 2: + r.store.coalescedMu.Lock() // L1 acquire + default: + return false + } + r.store.coalescedMu.Unlock() // L1 release + return true +} + +func Cockroach10214() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + for i := 0; i < 1000; i++ { + go func() { + store := &Store_cockroach10214{} + responses := &store.coalescedMu.heartbeatResponses + *responses = append(*responses, 1, 2) + store.mu.replicas = make(map[int]*Replica_cockroach10214) + + rp1 := &Replica_cockroach10214{ // L2,3[0] + store: store, + } + rp2 := &Replica_cockroach10214{ // L2,3[1] + store: store, + } + store.mu.replicas[0] = rp1 + store.mu.replicas[1] = rp2 + + go store.sendQueuedHeartbeats() // G1 + go rp1.tick() // G2 + }() + } +} + +// Example of goroutine leak trace: +// +// G1 G2 +//------------------------------------------------------------------------------------ +// s.sendQueuedHeartbeats() . +// s.coalescedMu.Lock() [L1] . +// s.sendQueuedHeartbeatsToNode() . +// s.mu.replicas[0].reportUnreachable() . +// s.mu.replicas[0].raftMu.Lock() [L2] . +// . s.mu.replicas[0].tick() +// . s.mu.replicas[0].raftMu.Lock() [L2] +// . s.mu.replicas[0].tickRaftMuLocked() +// . s.mu.replicas[0].mu.Lock() [L3] +// . s.mu.replicas[0].maybeQuiesceLocked() +// . s.mu.replicas[0].maybeCoalesceHeartbeat() +// . s.coalescedMu.Lock() [L1] +//--------------------------------G1,G2 leak------------------------------------------ \ No newline at end of file diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach1055.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach1055.go new file mode 100644 index 00000000000000..87cf157996800e --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach1055.go @@ -0,0 +1,115 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime/pprof" + "sync" + "sync/atomic" + "time" +) + +func init() { + register("Cockroach1055", Cockroach1055) +} + +type Stopper_cockroach1055 struct { + stopper chan struct{} + stop sync.WaitGroup + mu sync.Mutex + draining int32 + drain sync.WaitGroup +} + +func (s *Stopper_cockroach1055) AddWorker() { + s.stop.Add(1) +} + +func (s *Stopper_cockroach1055) ShouldStop() <-chan struct{} { + if s == nil { + return nil + } + return s.stopper +} + +func (s *Stopper_cockroach1055) SetStopped() { + if s != nil { + s.stop.Done() + } +} + +func (s *Stopper_cockroach1055) Quiesce() { + s.mu.Lock() + defer s.mu.Unlock() + atomic.StoreInt32(&s.draining, 1) + s.drain.Wait() + atomic.StoreInt32(&s.draining, 0) +} + +func (s *Stopper_cockroach1055) Stop() { + s.mu.Lock() // L1 + defer s.mu.Unlock() + atomic.StoreInt32(&s.draining, 1) + s.drain.Wait() + close(s.stopper) + s.stop.Wait() +} + +func (s *Stopper_cockroach1055) StartTask() bool { + if atomic.LoadInt32(&s.draining) == 0 { + s.mu.Lock() + defer s.mu.Unlock() + s.drain.Add(1) + return true + } + return false +} + +func NewStopper_cockroach1055() *Stopper_cockroach1055 { + return &Stopper_cockroach1055{ + stopper: make(chan struct{}), + } +} + +func Cockroach1055() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i <= 1000; i++ { + go func() { // G1 + var stoppers []*Stopper_cockroach1055 + for i := 0; i < 2; i++ { + stoppers = append(stoppers, NewStopper_cockroach1055()) + } + + for i := range stoppers { + s := stoppers[i] + s.AddWorker() + go func() { // G2 + s.StartTask() + <-s.ShouldStop() + s.SetStopped() + }() + } + + done := make(chan struct{}) + go func() { // G3 + for _, s := range stoppers { + s.Quiesce() + } + for _, s := range stoppers { + s.Stop() + } + close(done) + }() + + <-done + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach10790.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach10790.go new file mode 100644 index 00000000000000..636f45b3e0e765 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach10790.go @@ -0,0 +1,98 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: cockroach + * Issue or PR : https://github.com/cockroachdb/cockroach/pull/10790 + * Buggy version: 96b5452557ebe26bd9d85fe7905155009204d893 + * fix commit-id: f1a5c19125c65129b966fbdc0e6408e8df214aba + * Flaky: 28/100 + * Description: + * It is possible that a message from ctxDone will make the function beginCmds + * returns without draining the channel ch, so that goroutines created by anonymous + * function will leak. + */ + +package main + +import ( + "context" + "os" + "runtime/pprof" + "time" +) + +func init() { + register("Cockroach10790", Cockroach10790) +} + +type Replica_cockroach10790 struct { + chans []chan bool +} + +func (r *Replica_cockroach10790) beginCmds(ctx context.Context) { + ctxDone := ctx.Done() + for _, ch := range r.chans { + select { + case <-ch: + case <-ctxDone: + go func() { // G3 + for _, ch := range r.chans { + <-ch + } + }() + } + } +} + +func (r *Replica_cockroach10790) sendChans(ctx context.Context) { + for _, ch := range r.chans { + select { + case ch <- true: + case <-ctx.Done(): + return + } + } +} + +func NewReplica_cockroach10790() *Replica_cockroach10790 { + r := &Replica_cockroach10790{} + r.chans = append(r.chans, make(chan bool), make(chan bool)) + return r +} + +// Example of goroutine leak trace: +// +// G1 G2 G3 helper goroutine +//-------------------------------------------------------------------------------------- +// . . r.sendChans() +// r.beginCmds() . . +// . . ch1 <- +// <-ch1 <================================================> ch1 <- +// . . select [ch2<-, <-ctx.Done()] +// . cancel() . +// . <> [<-ctx.Done()] ==> return +// . <> +// go func() [G3] . +// . <-ch1 +// ------------------------------G3 leaks---------------------------------------------- +// + +func Cockroach10790() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 100; i++ { + go func() { + r := NewReplica_cockroach10790() + ctx, cancel := context.WithCancel(context.Background()) + go r.sendChans(ctx) // helper goroutine + go r.beginCmds(ctx) // G1 + go cancel() // G2 + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach13197.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach13197.go new file mode 100644 index 00000000000000..a0a9a792676692 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach13197.go @@ -0,0 +1,82 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: cockroach + * Issue or PR : https://github.com/cockroachdb/cockroach/pull/13197 + * Buggy version: fff27aedabafe20cef57f75905fe340cab48c2a4 + * fix commit-id: 9bf770cd8f6eaff5441b80d3aec1a5614e8747e1 + * Flaky: 100/100 + * Description: One goroutine executing (*Tx).awaitDone() blocks + * waiting for a signal over context.Done() that never comes. + */ +package main + +import ( + "context" + "os" + "runtime" + "runtime/pprof" +) + +func init() { + register("Cockroach13197", Cockroach13197) +} + +type DB_cockroach13197 struct{} + +func (db *DB_cockroach13197) begin(ctx context.Context) *Tx_cockroach13197 { + ctx, cancel := context.WithCancel(ctx) + tx := &Tx_cockroach13197{ + cancel: cancel, + ctx: ctx, + } + go tx.awaitDone() // G2 + return tx +} + +type Tx_cockroach13197 struct { + cancel context.CancelFunc + ctx context.Context +} + +func (tx *Tx_cockroach13197) awaitDone() { + <-tx.ctx.Done() +} + +func (tx *Tx_cockroach13197) Rollback() { + tx.rollback() +} + +func (tx *Tx_cockroach13197) rollback() { + tx.close() +} + +func (tx *Tx_cockroach13197) close() { + tx.cancel() +} + +// Example of goroutine leak trace: +// +// G1 G2 +//-------------------------------- +// begin() +// . awaitDone() +// <> . +// <-tx.ctx.Done() +//------------G2 leak------------- + +func Cockroach13197() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + + db := &DB_cockroach13197{} + db.begin(context.Background()) // G1 +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach13755.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach13755.go new file mode 100644 index 00000000000000..5ef6fa1e28a06d --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach13755.go @@ -0,0 +1,66 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: cockroach + * Issue or PR : https://github.com/cockroachdb/cockroach/pull/13755 + * Buggy version: 7acb881bbb8f23e87b69fce9568d9a3316b5259c + * fix commit-id: ef906076adc1d0e3721944829cfedfed51810088 + * Flaky: 100/100 + */ + +package main + +import ( + "context" + "os" + "runtime" + "runtime/pprof" +) + +func init() { + register("Cockroach13755", Cockroach13755) +} + +type Rows_cockroach13755 struct { + cancel context.CancelFunc +} + +func (rs *Rows_cockroach13755) initContextClose(ctx context.Context) { + ctx, rs.cancel = context.WithCancel(ctx) + go rs.awaitDone(ctx) +} + +func (rs *Rows_cockroach13755) awaitDone(ctx context.Context) { + <-ctx.Done() + rs.close(ctx.Err()) +} + +func (rs *Rows_cockroach13755) close(err error) { + rs.cancel() +} + +// Example of goroutine leak trace: +// +// G1 G2 +//---------------------------------------- +// initContextClose() +// . awaitDone() +// <> . +// <-tx.ctx.Done() +//----------------G2 leak----------------- + +func Cockroach13755() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + + rs := &Rows_cockroach13755{} + rs.initContextClose(context.Background()) +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach1462.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach1462.go new file mode 100644 index 00000000000000..108d7884a3d82b --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach1462.go @@ -0,0 +1,167 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Cockroach1462", Cockroach1462) +} + +type Stopper_cockroach1462 struct { + stopper chan struct{} + stopped chan struct{} + stop sync.WaitGroup + mu sync.Mutex + drain *sync.Cond + draining bool + numTasks int +} + +func NewStopper_cockroach1462() *Stopper_cockroach1462 { + s := &Stopper_cockroach1462{ + stopper: make(chan struct{}), + stopped: make(chan struct{}), + } + s.drain = sync.NewCond(&s.mu) + return s +} + +func (s *Stopper_cockroach1462) RunWorker(f func()) { + s.AddWorker() + go func() { // G2, G3 + defer s.SetStopped() + f() + }() +} + +func (s *Stopper_cockroach1462) AddWorker() { + s.stop.Add(1) +} +func (s *Stopper_cockroach1462) StartTask() bool { + s.mu.Lock() + runtime.Gosched() + defer s.mu.Unlock() + if s.draining { + return false + } + s.numTasks++ + return true +} + +func (s *Stopper_cockroach1462) FinishTask() { + s.mu.Lock() + runtime.Gosched() + defer s.mu.Unlock() + s.numTasks-- + s.drain.Broadcast() +} +func (s *Stopper_cockroach1462) SetStopped() { + if s != nil { + s.stop.Done() + } +} +func (s *Stopper_cockroach1462) ShouldStop() <-chan struct{} { + if s == nil { + return nil + } + return s.stopper +} + +func (s *Stopper_cockroach1462) Quiesce() { + s.mu.Lock() + runtime.Gosched() + defer s.mu.Unlock() + s.draining = true + for s.numTasks > 0 { + // Unlock s.mu, wait for the signal, and lock s.mu. + s.drain.Wait() + } +} + +func (s *Stopper_cockroach1462) Stop() { + s.Quiesce() + close(s.stopper) + s.stop.Wait() + s.mu.Lock() + runtime.Gosched() + defer s.mu.Unlock() + close(s.stopped) +} + +type interceptMessage_cockroach1462 int + +type localInterceptableTransport_cockroach1462 struct { + mu sync.Mutex + Events chan interceptMessage_cockroach1462 + stopper *Stopper_cockroach1462 +} + +func (lt *localInterceptableTransport_cockroach1462) Close() {} + +type Transport_cockroach1462 interface { + Close() +} + +func NewLocalInterceptableTransport_cockroach1462(stopper *Stopper_cockroach1462) Transport_cockroach1462 { + lt := &localInterceptableTransport_cockroach1462{ + Events: make(chan interceptMessage_cockroach1462), + stopper: stopper, + } + lt.start() + return lt +} + +func (lt *localInterceptableTransport_cockroach1462) start() { + lt.stopper.RunWorker(func() { + for { + select { + case <-lt.stopper.ShouldStop(): + return + default: + lt.Events <- interceptMessage_cockroach1462(0) + } + } + }) +} + +func processEventsUntil_cockroach1462(ch <-chan interceptMessage_cockroach1462, stopper *Stopper_cockroach1462) { + for { + select { + case _, ok := <-ch: + runtime.Gosched() + if !ok { + return + } + case <-stopper.ShouldStop(): + return + } + } +} + +func Cockroach1462() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(2000 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + for i := 0; i <= 1000; i++ { + go func() { // G1 + stopper := NewStopper_cockroach1462() + transport := NewLocalInterceptableTransport_cockroach1462(stopper).(*localInterceptableTransport_cockroach1462) + stopper.RunWorker(func() { + processEventsUntil_cockroach1462(transport.Events, stopper) + }) + stopper.Stop() + }() + } +} + diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach16167.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach16167.go new file mode 100644 index 00000000000000..4cd14c7a5b3ac5 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach16167.go @@ -0,0 +1,108 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: cockroach + * Issue or PR : https://github.com/cockroachdb/cockroach/pull/16167 + * Buggy version: 36fa784aa846b46c29e077634c4e362635f6e74a + * fix commit-id: d064942b067ab84628f79cbfda001fa3138d8d6e + * Flaky: 1/100 + */ + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Cockroach16167", Cockroach16167) +} + +type PreparedStatements_cockroach16167 struct { + session *Session_cockroach16167 +} + +func (ps PreparedStatements_cockroach16167) New(e *Executor_cockroach16167) { + e.Prepare(ps.session) +} + +type Session_cockroach16167 struct { + PreparedStatements PreparedStatements_cockroach16167 +} + +func (s *Session_cockroach16167) resetForBatch(e *Executor_cockroach16167) { + e.getDatabaseCache() +} + +type Executor_cockroach16167 struct { + systemConfigCond *sync.Cond + systemConfigMu sync.RWMutex // L1 +} + +func (e *Executor_cockroach16167) Start() { + e.updateSystemConfig() +} + +func (e *Executor_cockroach16167) execParsed(session *Session_cockroach16167) { + e.systemConfigCond.L.Lock() // Same as e.systemConfigMu.RLock() + runtime.Gosched() + defer e.systemConfigCond.L.Unlock() + runTxnAttempt_cockroach16167(e, session) +} + +func (e *Executor_cockroach16167) execStmtsInCurrentTxn(session *Session_cockroach16167) { + e.execStmtInOpenTxn(session) +} + +func (e *Executor_cockroach16167) execStmtInOpenTxn(session *Session_cockroach16167) { + session.PreparedStatements.New(e) +} + +func (e *Executor_cockroach16167) Prepare(session *Session_cockroach16167) { + session.resetForBatch(e) +} + +func (e *Executor_cockroach16167) getDatabaseCache() { + e.systemConfigMu.RLock() + defer e.systemConfigMu.RUnlock() +} + +func (e *Executor_cockroach16167) updateSystemConfig() { + e.systemConfigMu.Lock() + runtime.Gosched() + defer e.systemConfigMu.Unlock() +} + +func runTxnAttempt_cockroach16167(e *Executor_cockroach16167, session *Session_cockroach16167) { + e.execStmtsInCurrentTxn(session) +} + +func NewExectorAndSession_cockroach16167() (*Executor_cockroach16167, *Session_cockroach16167) { + session := &Session_cockroach16167{} + session.PreparedStatements = PreparedStatements_cockroach16167{session} + e := &Executor_cockroach16167{} + return e, session +} + +func Cockroach16167() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 100; i++ { + go func() { // G1 + e, s := NewExectorAndSession_cockroach16167() + e.systemConfigCond = sync.NewCond(e.systemConfigMu.RLocker()) + go e.Start() // G2 + e.execParsed(s) + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach18101.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach18101.go new file mode 100644 index 00000000000000..17b03203304ac7 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach18101.go @@ -0,0 +1,60 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: cockroach + * Issue or PR : https://github.com/cockroachdb/cockroach/pull/18101 + * Buggy version: f7a8e2f57b6bcf00b9abaf3da00598e4acd3a57f + * fix commit-id: 822bd176cc725c6b50905ea615023200b395e14f + * Flaky: 100/100 + */ + +package main + +import ( + "context" + "os" + "runtime/pprof" + "time" +) + +func init() { + register("Cockroach18101", Cockroach18101) +} + +const chanSize_cockroach18101 = 6 + +func restore_cockroach18101(ctx context.Context) bool { + readyForImportCh := make(chan bool, chanSize_cockroach18101) + go func() { // G2 + defer close(readyForImportCh) + splitAndScatter_cockroach18101(ctx, readyForImportCh) + }() + for readyForImportSpan := range readyForImportCh { + select { + case <-ctx.Done(): + return readyForImportSpan + } + } + return true +} + +func splitAndScatter_cockroach18101(ctx context.Context, readyForImportCh chan bool) { + for i := 0; i < chanSize_cockroach18101+2; i++ { + readyForImportCh <- (false || i != 0) + } +} + +func Cockroach18101() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + for i := 0; i < 100; i++ { + ctx, cancel := context.WithCancel(context.Background()) + go restore_cockroach18101(ctx) // G1 + go cancel() // helper goroutine + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach2448.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach2448.go new file mode 100644 index 00000000000000..a7544bc8a46a18 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach2448.go @@ -0,0 +1,125 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "time" +) + +func init() { + register("Cockroach2448", Cockroach2448) +} + +type Stopper_cockroach2448 struct { + Done chan bool +} + +func (s *Stopper_cockroach2448) ShouldStop() <-chan bool { + return s.Done +} + +type EventMembershipChangeCommitted_cockroach2448 struct { + Callback func() +} + +type MultiRaft_cockroach2448 struct { + stopper *Stopper_cockroach2448 + Events chan interface{} + callbackChan chan func() +} + +// sendEvent can be invoked many times +func (m *MultiRaft_cockroach2448) sendEvent(event interface{}) { + select { + case m.Events <- event: // Waiting for events consumption + case <-m.stopper.ShouldStop(): + } +} + +type state_cockroach2448 struct { + *MultiRaft_cockroach2448 +} + +func (s *state_cockroach2448) start() { + for { + select { + case <-s.stopper.ShouldStop(): + return + case cb := <-s.callbackChan: + cb() + default: + s.handleWriteResponse() + time.Sleep(100 * time.Microsecond) + } + } +} + +func (s *state_cockroach2448) handleWriteResponse() { + s.sendEvent(&EventMembershipChangeCommitted_cockroach2448{ + Callback: func() { + select { + case s.callbackChan <- func() { // Waiting for callbackChan consumption + time.Sleep(time.Nanosecond) + }: + case <-s.stopper.ShouldStop(): + } + }, + }) +} + +type Store_cockroach2448 struct { + multiraft *MultiRaft_cockroach2448 +} + +func (s *Store_cockroach2448) processRaft() { + for { + select { + case e := <-s.multiraft.Events: + switch e := e.(type) { + case *EventMembershipChangeCommitted_cockroach2448: + callback := e.Callback + runtime.Gosched() + if callback != nil { + callback() // Waiting for callbackChan consumption + } + } + case <-s.multiraft.stopper.ShouldStop(): + return + } + } +} + +func NewStoreAndState_cockroach2448() (*Store_cockroach2448, *state_cockroach2448) { + stopper := &Stopper_cockroach2448{ + Done: make(chan bool), + } + mltrft := &MultiRaft_cockroach2448{ + stopper: stopper, + Events: make(chan interface{}), + callbackChan: make(chan func()), + } + st := &state_cockroach2448{mltrft} + s := &Store_cockroach2448{mltrft} + return s, st +} + +func Cockroach2448() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + for i := 0; i < 1000; i++ { + go func() { + s, st := NewStoreAndState_cockroach2448() + go s.processRaft() // G1 + go st.start() // G2 + }() + } +} + diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach24808.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach24808.go new file mode 100644 index 00000000000000..a916d3c928ee2e --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach24808.go @@ -0,0 +1,78 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "context" + "os" + "runtime" + "runtime/pprof" + "sync" +) + +func init() { + register("Cockroach24808", Cockroach24808) +} + +type Compactor_cockroach24808 struct { + ch chan struct{} +} + +type Stopper_cockroach24808 struct { + stop sync.WaitGroup + stopper chan struct{} +} + +func (s *Stopper_cockroach24808) RunWorker(ctx context.Context, f func(context.Context)) { + s.stop.Add(1) + go func() { + defer s.stop.Done() + f(ctx) + }() +} + +func (s *Stopper_cockroach24808) ShouldStop() <-chan struct{} { + if s == nil { + return nil + } + return s.stopper +} + +func (s *Stopper_cockroach24808) Stop() { + close(s.stopper) +} + +func (c *Compactor_cockroach24808) Start(ctx context.Context, stopper *Stopper_cockroach24808) { + c.ch <- struct{}{} + stopper.RunWorker(ctx, func(ctx context.Context) { + for { + select { + case <-stopper.ShouldStop(): + return + case <-c.ch: + } + } + }) +} + +func Cockroach24808() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + go func() { // G1 + stopper := &Stopper_cockroach24808{stopper: make(chan struct{})} + defer stopper.Stop() + + compactor := &Compactor_cockroach24808{ch: make(chan struct{}, 1)} + compactor.ch <- struct{}{} + + compactor.Start(context.Background(), stopper) + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach25456.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach25456.go new file mode 100644 index 00000000000000..b9259c9f918293 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach25456.go @@ -0,0 +1,92 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "runtime/pprof" +) + +func init() { + register("Cockroach25456", Cockroach25456) +} + +type Stopper_cockroach25456 struct { + quiescer chan struct{} +} + +func (s *Stopper_cockroach25456) ShouldQuiesce() <-chan struct{} { + if s == nil { + return nil + } + return s.quiescer +} + +func NewStopper_cockroach25456() *Stopper_cockroach25456 { + return &Stopper_cockroach25456{quiescer: make(chan struct{})} +} + +type Store_cockroach25456 struct { + stopper *Stopper_cockroach25456 + consistencyQueue *consistencyQueue_cockroach25456 +} + +func (s *Store_cockroach25456) Stopper() *Stopper_cockroach25456 { + return s.stopper +} + +type Replica_cockroach25456 struct { + store *Store_cockroach25456 +} + +func NewReplica_cockroach25456(store *Store_cockroach25456) *Replica_cockroach25456 { + return &Replica_cockroach25456{store: store} +} + +type consistencyQueue_cockroach25456 struct{} + +func (q *consistencyQueue_cockroach25456) process(repl *Replica_cockroach25456) { + <-repl.store.Stopper().ShouldQuiesce() +} + +func newConsistencyQueue_cockroach25456() *consistencyQueue_cockroach25456 { + return &consistencyQueue_cockroach25456{} +} + +type testContext_cockroach25456 struct { + store *Store_cockroach25456 + repl *Replica_cockroach25456 +} + +func (tc *testContext_cockroach25456) StartWithStoreConfig(stopper *Stopper_cockroach25456) { + if tc.store == nil { + tc.store = &Store_cockroach25456{ + consistencyQueue: newConsistencyQueue_cockroach25456(), + } + } + tc.store.stopper = stopper + tc.repl = NewReplica_cockroach25456(tc.store) +} + +func Cockroach25456() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + go func() { // G1 + stopper := NewStopper_cockroach25456() + tc := testContext_cockroach25456{} + tc.StartWithStoreConfig(stopper) + + for i := 0; i < 2; i++ { + tc.store.consistencyQueue.process(tc.repl) + } + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach35073.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach35073.go new file mode 100644 index 00000000000000..f00a7bd46259eb --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach35073.go @@ -0,0 +1,124 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "sync/atomic" +) + +func init() { + register("Cockroach35073", Cockroach35073) +} + +type ConsumerStatus_cockroach35073 uint32 + +const ( + NeedMoreRows_cockroach35073 ConsumerStatus_cockroach35073 = iota + DrainRequested_cockroach35073 + ConsumerClosed_cockroach35073 +) + +const rowChannelBufSize_cockroach35073 = 16 +const outboxBufRows_cockroach35073 = 16 + +type rowSourceBase_cockroach35073 struct { + consumerStatus ConsumerStatus_cockroach35073 +} + +func (rb *rowSourceBase_cockroach35073) consumerClosed() { + atomic.StoreUint32((*uint32)(&rb.consumerStatus), uint32(ConsumerClosed_cockroach35073)) +} + +type RowChannelMsg_cockroach35073 int + +type RowChannel_cockroach35073 struct { + rowSourceBase_cockroach35073 + dataChan chan RowChannelMsg_cockroach35073 +} + +func (rc *RowChannel_cockroach35073) ConsumerClosed() { + rc.consumerClosed() + select { + case <-rc.dataChan: + default: + } +} + +func (rc *RowChannel_cockroach35073) Push() ConsumerStatus_cockroach35073 { + consumerStatus := ConsumerStatus_cockroach35073( + atomic.LoadUint32((*uint32)(&rc.consumerStatus))) + switch consumerStatus { + case NeedMoreRows_cockroach35073: + rc.dataChan <- RowChannelMsg_cockroach35073(0) + case DrainRequested_cockroach35073: + case ConsumerClosed_cockroach35073: + } + return consumerStatus +} + +func (rc *RowChannel_cockroach35073) InitWithNumSenders() { + rc.initWithBufSizeAndNumSenders(rowChannelBufSize_cockroach35073) +} + +func (rc *RowChannel_cockroach35073) initWithBufSizeAndNumSenders(chanBufSize int) { + rc.dataChan = make(chan RowChannelMsg_cockroach35073, chanBufSize) +} + +type outbox_cockroach35073 struct { + RowChannel_cockroach35073 +} + +func (m *outbox_cockroach35073) init() { + m.RowChannel_cockroach35073.InitWithNumSenders() +} + +func (m *outbox_cockroach35073) start(wg *sync.WaitGroup) { + if wg != nil { + wg.Add(1) + } + go m.run(wg) +} + +func (m *outbox_cockroach35073) run(wg *sync.WaitGroup) { + if wg != nil { + wg.Done() + } +} + +func Cockroach35073() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + go func() { + outbox := &outbox_cockroach35073{} + outbox.init() + + var wg sync.WaitGroup + for i := 0; i < outboxBufRows_cockroach35073; i++ { + outbox.Push() + } + + var blockedPusherWg sync.WaitGroup + blockedPusherWg.Add(1) + go func() { + outbox.Push() + blockedPusherWg.Done() + }() + + outbox.start(&wg) + + wg.Wait() + outbox.RowChannel_cockroach35073.Push() + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach35931.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach35931.go new file mode 100644 index 00000000000000..9ddcda1b6242bf --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach35931.go @@ -0,0 +1,135 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" +) + +func init() { + register("Cockroach35931", Cockroach35931) +} + +type RowReceiver_cockroach35931 interface { + Push() +} + +type inboundStreamInfo_cockroach35931 struct { + receiver RowReceiver_cockroach35931 +} + +type RowChannel_cockroach35931 struct { + dataChan chan struct{} +} + +func (rc *RowChannel_cockroach35931) Push() { + // The buffer size can be either 0 or 1 when this function is entered. + // We need context sensitivity or a path-condition on the buffer size + // to find this bug. + rc.dataChan <- struct{}{} +} + +func (rc *RowChannel_cockroach35931) initWithBufSizeAndNumSenders(chanBufSize int) { + rc.dataChan = make(chan struct{}, chanBufSize) +} + +type flowEntry_cockroach35931 struct { + flow *Flow_cockroach35931 + inboundStreams map[int]*inboundStreamInfo_cockroach35931 +} + +type flowRegistry_cockroach35931 struct { + sync.Mutex + flows map[int]*flowEntry_cockroach35931 +} + +func (fr *flowRegistry_cockroach35931) getEntryLocked(id int) *flowEntry_cockroach35931 { + entry, ok := fr.flows[id] + if !ok { + entry = &flowEntry_cockroach35931{} + fr.flows[id] = entry + } + return entry +} + +func (fr *flowRegistry_cockroach35931) cancelPendingStreamsLocked(id int) []RowReceiver_cockroach35931 { + entry := fr.flows[id] + pendingReceivers := make([]RowReceiver_cockroach35931, 0) + for _, is := range entry.inboundStreams { + pendingReceivers = append(pendingReceivers, is.receiver) + } + return pendingReceivers +} + +type Flow_cockroach35931 struct { + id int + flowRegistry *flowRegistry_cockroach35931 + inboundStreams map[int]*inboundStreamInfo_cockroach35931 +} + +func (f *Flow_cockroach35931) cancel() { + f.flowRegistry.Lock() + timedOutReceivers := f.flowRegistry.cancelPendingStreamsLocked(f.id) + f.flowRegistry.Unlock() + + for _, receiver := range timedOutReceivers { + receiver.Push() + } +} + +func (fr *flowRegistry_cockroach35931) RegisterFlow(f *Flow_cockroach35931, inboundStreams map[int]*inboundStreamInfo_cockroach35931) { + entry := fr.getEntryLocked(f.id) + entry.flow = f + entry.inboundStreams = inboundStreams +} + +func makeFlowRegistry_cockroach35931() *flowRegistry_cockroach35931 { + return &flowRegistry_cockroach35931{ + flows: make(map[int]*flowEntry_cockroach35931), + } +} + +func Cockroach35931() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + go func() { + fr := makeFlowRegistry_cockroach35931() + + left := &RowChannel_cockroach35931{} + left.initWithBufSizeAndNumSenders(1) + right := &RowChannel_cockroach35931{} + right.initWithBufSizeAndNumSenders(1) + + inboundStreams := map[int]*inboundStreamInfo_cockroach35931{ + 0: { + receiver: left, + }, + 1: { + receiver: right, + }, + } + + left.Push() + + flow := &Flow_cockroach35931{ + id: 0, + flowRegistry: fr, + inboundStreams: inboundStreams, + } + + fr.RegisterFlow(flow, inboundStreams) + + flow.cancel() + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach3710.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach3710.go new file mode 100644 index 00000000000000..e419cd2fc32019 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach3710.go @@ -0,0 +1,122 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: cockroach + * Issue or PR : https://github.com/cockroachdb/cockroach/pull/3710 + * Buggy version: 4afdd4860fd7c3bd9e92489f84a95e5cc7d11a0d + * fix commit-id: cb65190f9caaf464723e7d072b1f1b69a044ef7b + * Flaky: 2/100 + */ + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "time" + "unsafe" +) + +func init() { + register("Cockroach3710", Cockroach3710) +} + +type Store_cockroach3710 struct { + raftLogQueue *baseQueue + replicas map[int]*Replica_cockroach3710 + + mu struct { + sync.RWMutex + } +} + +func (s *Store_cockroach3710) ForceRaftLogScanAndProcess() { + s.mu.RLock() + runtime.Gosched() + for _, r := range s.replicas { + s.raftLogQueue.MaybeAdd(r) + } + s.mu.RUnlock() +} + +func (s *Store_cockroach3710) RaftStatus() { + s.mu.RLock() + defer s.mu.RUnlock() +} + +func (s *Store_cockroach3710) processRaft() { + go func() { + for { + var replicas []*Replica_cockroach3710 + s.mu.Lock() + for _, r := range s.replicas { + replicas = append(replicas, r) + } + s.mu.Unlock() + break + } + }() +} + +type Replica_cockroach3710 struct { + store *Store_cockroach3710 +} + +type baseQueue struct { + sync.Mutex + impl *raftLogQueue +} + +func (bq *baseQueue) MaybeAdd(repl *Replica_cockroach3710) { + bq.Lock() + defer bq.Unlock() + bq.impl.shouldQueue(repl) +} + +type raftLogQueue struct{} + +func (*raftLogQueue) shouldQueue(r *Replica_cockroach3710) { + getTruncatableIndexes(r) +} + +func getTruncatableIndexes(r *Replica_cockroach3710) { + r.store.RaftStatus() +} + +func NewStore_cockroach3710() *Store_cockroach3710 { + rlq := &raftLogQueue{} + bq := &baseQueue{impl: rlq} + store := &Store_cockroach3710{ + raftLogQueue: bq, + replicas: make(map[int]*Replica_cockroach3710), + } + r1 := &Replica_cockroach3710{store} + r2 := &Replica_cockroach3710{store} + + makeKey := func(r *Replica_cockroach3710) int { + return int((uintptr(unsafe.Pointer(r)) >> 1) % 7) + } + store.replicas[makeKey(r1)] = r1 + store.replicas[makeKey(r2)] = r2 + + return store +} + +func Cockroach3710() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + for i := 0; i < 10000; i++ { + go func() { + store := NewStore_cockroach3710() + go store.ForceRaftLogScanAndProcess() // G1 + go store.processRaft() // G2 + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach584.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach584.go new file mode 100644 index 00000000000000..33f7ba7a45ec9a --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach584.go @@ -0,0 +1,62 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" +) + +func init() { + register("Cockroach584", Cockroach584) +} + +type gossip_cockroach584 struct { + mu sync.Mutex // L1 + closed bool +} + +func (g *gossip_cockroach584) bootstrap() { + for { + g.mu.Lock() + if g.closed { + // Missing g.mu.Unlock + break + } + g.mu.Unlock() + } +} + +func (g *gossip_cockroach584) manage() { + for { + g.mu.Lock() + if g.closed { + // Missing g.mu.Unlock + break + } + g.mu.Unlock() + } +} + +func Cockroach584() { + prof := pprof.Lookup("goroutineleak") + defer func() { + for i := 0; i < yieldCount; i++ { + // Yield several times to allow the child goroutine to run. + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + + g := &gossip_cockroach584{ + closed: true, + } + go func() { // G1 + g.bootstrap() + g.manage() + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach6181.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach6181.go new file mode 100644 index 00000000000000..80f1dd504daae5 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach6181.go @@ -0,0 +1,87 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: cockroach + * Issue or PR : https://github.com/cockroachdb/cockroach/pull/6181 + * Buggy version: c0a232b5521565904b851699853bdbd0c670cf1e + * fix commit-id: d5814e4886a776bf7789b3c51b31f5206480d184 + * Flaky: 57/100 + */ +package main + +import ( + "io" + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Cockroach6181", Cockroach6181) +} + +type testDescriptorDB_cockroach6181 struct { + cache *rangeDescriptorCache_cockroach6181 +} + +func initTestDescriptorDB_cockroach6181() *testDescriptorDB_cockroach6181 { + return &testDescriptorDB_cockroach6181{&rangeDescriptorCache_cockroach6181{}} +} + +type rangeDescriptorCache_cockroach6181 struct { + rangeCacheMu sync.RWMutex +} + +func (rdc *rangeDescriptorCache_cockroach6181) LookupRangeDescriptor() { + rdc.rangeCacheMu.RLock() + runtime.Gosched() + io.Discard.Write([]byte(rdc.String())) + rdc.rangeCacheMu.RUnlock() + rdc.rangeCacheMu.Lock() + rdc.rangeCacheMu.Unlock() +} + +func (rdc *rangeDescriptorCache_cockroach6181) String() string { + rdc.rangeCacheMu.RLock() + defer rdc.rangeCacheMu.RUnlock() + return rdc.stringLocked() +} + +func (rdc *rangeDescriptorCache_cockroach6181) stringLocked() string { + return "something here" +} + +func doLookupWithToken_cockroach6181(rc *rangeDescriptorCache_cockroach6181) { + rc.LookupRangeDescriptor() +} + +func testRangeCacheCoalescedRequests_cockroach6181() { + db := initTestDescriptorDB_cockroach6181() + pauseLookupResumeAndAssert := func() { + var wg sync.WaitGroup + for i := 0; i < 3; i++ { + wg.Add(1) + go func() { // G2,G3,... + doLookupWithToken_cockroach6181(db.cache) + wg.Done() + }() + } + wg.Wait() + } + pauseLookupResumeAndAssert() +} + +func Cockroach6181() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + for i := 0; i < 100; i++ { + go testRangeCacheCoalescedRequests_cockroach6181() // G1 + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach7504.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach7504.go new file mode 100644 index 00000000000000..945308a76f92b1 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach7504.go @@ -0,0 +1,183 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: cockroach + * Issue or PR : https://github.com/cockroachdb/cockroach/pull/7504 + * Buggy version: bc963b438cdc3e0ad058a5282358e5aee0595e17 + * fix commit-id: cab761b9f5ee5dee1448bc5d6b1d9f5a0ff0bad5 + * Flaky: 1/100 + */ +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Cockroach7504", Cockroach7504) +} + +func MakeCacheKey_cockroach7504(lease *LeaseState_cockroach7504) int { + return lease.id +} + +type LeaseState_cockroach7504 struct { + mu sync.Mutex // L1 + id int +} +type LeaseSet_cockroach7504 struct { + data []*LeaseState_cockroach7504 +} + +func (l *LeaseSet_cockroach7504) find(id int) *LeaseState_cockroach7504 { + return l.data[id] +} + +func (l *LeaseSet_cockroach7504) remove(s *LeaseState_cockroach7504) { + for i := 0; i < len(l.data); i++ { + if s == l.data[i] { + l.data = append(l.data[:i], l.data[i+1:]...) + break + } + } +} + +type tableState_cockroach7504 struct { + tableNameCache *tableNameCache_cockroach7504 + mu sync.Mutex // L3 + active *LeaseSet_cockroach7504 +} + +func (t *tableState_cockroach7504) release(lease *LeaseState_cockroach7504) { + t.mu.Lock() // L3 + defer t.mu.Unlock() // L3 + + s := t.active.find(MakeCacheKey_cockroach7504(lease)) + s.mu.Lock() // L1 + runtime.Gosched() + defer s.mu.Unlock() // L1 + + t.removeLease(s) +} +func (t *tableState_cockroach7504) removeLease(lease *LeaseState_cockroach7504) { + t.active.remove(lease) + t.tableNameCache.remove(lease) // L1 acquire/release +} + +type tableNameCache_cockroach7504 struct { + mu sync.Mutex // L2 + tables map[int]*LeaseState_cockroach7504 +} + +func (c *tableNameCache_cockroach7504) get(id int) { + c.mu.Lock() // L2 + defer c.mu.Unlock() // L2 + lease, ok := c.tables[id] + if !ok { + return + } + if lease == nil { + panic("nil lease in name cache") + } + lease.mu.Lock() // L1 + defer lease.mu.Unlock() // L1 +} + +func (c *tableNameCache_cockroach7504) remove(lease *LeaseState_cockroach7504) { + c.mu.Lock() // L2 + runtime.Gosched() + defer c.mu.Unlock() // L2 + key := MakeCacheKey_cockroach7504(lease) + existing, ok := c.tables[key] + if !ok { + return + } + if existing == lease { + delete(c.tables, key) + } +} + +type LeaseManager_cockroach7504 struct { + _ [64]byte + tableNames *tableNameCache_cockroach7504 + tables map[int]*tableState_cockroach7504 +} + +func (m *LeaseManager_cockroach7504) AcquireByName(id int) { + m.tableNames.get(id) +} + +func (m *LeaseManager_cockroach7504) findTableState(lease *LeaseState_cockroach7504) *tableState_cockroach7504 { + existing, ok := m.tables[lease.id] + if !ok { + return nil + } + return existing +} + +func (m *LeaseManager_cockroach7504) Release(lease *LeaseState_cockroach7504) { + t := m.findTableState(lease) + t.release(lease) +} +func NewLeaseManager_cockroach7504(tname *tableNameCache_cockroach7504, ts *tableState_cockroach7504) *LeaseManager_cockroach7504 { + mgr := &LeaseManager_cockroach7504{ + tableNames: tname, + tables: make(map[int]*tableState_cockroach7504), + } + mgr.tables[0] = ts + return mgr +} +func NewLeaseSet_cockroach7504(n int) *LeaseSet_cockroach7504 { + lset := &LeaseSet_cockroach7504{} + for i := 0; i < n; i++ { + lease := new(LeaseState_cockroach7504) + lset.data = append(lset.data, lease) + } + return lset +} + +func Cockroach7504() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + for i := 0; i < 100; i++ { + go func() { + leaseNum := 2 + lset := NewLeaseSet_cockroach7504(leaseNum) + + nc := &tableNameCache_cockroach7504{ + tables: make(map[int]*LeaseState_cockroach7504), + } + for i := 0; i < leaseNum; i++ { + nc.tables[i] = lset.find(i) + } + + ts := &tableState_cockroach7504{ + tableNameCache: nc, + active: lset, + } + + mgr := NewLeaseManager_cockroach7504(nc, ts) + + // G1 + go func() { + // lock L2-L1 + mgr.AcquireByName(0) + }() + + // G2 + go func() { + // lock L1-L2 + mgr.Release(lset.find(0)) + }() + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach9935.go b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach9935.go new file mode 100644 index 00000000000000..e143a6670d8ff2 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/cockroach9935.go @@ -0,0 +1,77 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: cockroach + * Issue or PR : https://github.com/cockroachdb/cockroach/pull/9935 + * Buggy version: 4df302cc3f03328395dc3fefbfba58b7718e4f2f + * fix commit-id: ed6a100ba38dd51b0888b9a3d3ac6bdbb26c528c + * Flaky: 100/100 + * Description: This leak is caused by acquiring l.mu.Lock() twice. The fix is + * to release l.mu.Lock() before acquiring l.mu.Lock for the second time. + */ +package main + +import ( + "errors" + "math/rand" + "os" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Cockroach9935", Cockroach9935) +} + +type loggingT_cockroach9935 struct { + mu sync.Mutex +} + +func (l *loggingT_cockroach9935) outputLogEntry() { + l.mu.Lock() + if err := l.createFile(); err != nil { + l.exit(err) + } + l.mu.Unlock() +} + +func (l *loggingT_cockroach9935) createFile() error { + if rand.Intn(8)%4 > 0 { + return errors.New("") + } + return nil +} + +func (l *loggingT_cockroach9935) exit(err error) { + l.mu.Lock() // Blocked forever + defer l.mu.Unlock() +} + +// Example of goroutine leak trace: +// +// G1 +//---------------------------- +// l.outputLogEntry() +// l.mu.Lock() +// l.createFile() +// l.exit() +// l.mu.Lock() +//-----------G1 leaks--------- + +func Cockroach9935() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 100; i++ { + go func() { + l := &loggingT_cockroach9935{} + go l.outputLogEntry() // G1 + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/etcd10492.go b/src/runtime/testdata/testgoroutineleakprofile/goker/etcd10492.go new file mode 100644 index 00000000000000..7d56642d5e9ed9 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/etcd10492.go @@ -0,0 +1,72 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "context" + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Etcd10492", Etcd10492) +} + +type Checkpointer_etcd10492 func(ctx context.Context) + +type lessor_etcd10492 struct { + mu sync.RWMutex + cp Checkpointer_etcd10492 + checkpointInterval time.Duration +} + +func (le *lessor_etcd10492) Checkpoint() { + le.mu.Lock() // Lock acquired twice here + defer le.mu.Unlock() +} + +func (le *lessor_etcd10492) SetCheckpointer(cp Checkpointer_etcd10492) { + le.mu.Lock() + defer le.mu.Unlock() + + le.cp = cp +} + +func (le *lessor_etcd10492) Renew() { + le.mu.Lock() + unlock := func() { le.mu.Unlock() } + defer func() { unlock() }() + + if le.cp != nil { + le.cp(context.Background()) + } +} + +func Etcd10492() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + + go func() { // G1 + le := &lessor_etcd10492{ + checkpointInterval: 0, + } + fakerCheckerpointer_etcd10492 := func(ctx context.Context) { + le.Checkpoint() + } + le.SetCheckpointer(fakerCheckerpointer_etcd10492) + le.mu.Lock() + le.mu.Unlock() + le.Renew() + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/etcd5509.go b/src/runtime/testdata/testgoroutineleakprofile/goker/etcd5509.go new file mode 100644 index 00000000000000..868e926e66949a --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/etcd5509.go @@ -0,0 +1,126 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "context" + "io" + "os" + "runtime" + "runtime/pprof" + "sync" +) + +func init() { + register("Etcd5509", Etcd5509) +} + +var ErrConnClosed_etcd5509 error + +type Client_etcd5509 struct { + mu sync.RWMutex + ctx context.Context + cancel context.CancelFunc +} + +func (c *Client_etcd5509) Close() { + c.mu.Lock() + defer c.mu.Unlock() + if c.cancel == nil { + return + } + c.cancel() + c.cancel = nil + c.mu.Unlock() + c.mu.Lock() +} + +type remoteClient_etcd5509 struct { + client *Client_etcd5509 + mu sync.Mutex +} + +func (r *remoteClient_etcd5509) acquire(ctx context.Context) error { + for { + r.client.mu.RLock() + closed := r.client.cancel == nil + r.mu.Lock() + r.mu.Unlock() + if closed { + return ErrConnClosed_etcd5509 // Missing RUnlock before return + } + r.client.mu.RUnlock() + } +} + +type kv_etcd5509 struct { + rc *remoteClient_etcd5509 +} + +func (kv *kv_etcd5509) Get(ctx context.Context) error { + return kv.Do(ctx) +} + +func (kv *kv_etcd5509) Do(ctx context.Context) error { + for { + err := kv.do(ctx) + if err == nil { + return nil + } + return err + } +} + +func (kv *kv_etcd5509) do(ctx context.Context) error { + err := kv.getRemote(ctx) + return err +} + +func (kv *kv_etcd5509) getRemote(ctx context.Context) error { + return kv.rc.acquire(ctx) +} + +type KV interface { + Get(ctx context.Context) error + Do(ctx context.Context) error +} + +func NewKV_etcd5509(c *Client_etcd5509) KV { + return &kv_etcd5509{rc: &remoteClient_etcd5509{ + client: c, + }} +} + +func Etcd5509() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + + go func() { + ctx, _ := context.WithCancel(context.TODO()) + cli := &Client_etcd5509{ + ctx: ctx, + } + kv := NewKV_etcd5509(cli) + donec := make(chan struct{}) + go func() { + defer close(donec) + err := kv.Get(context.TODO()) + if err != nil && err != ErrConnClosed_etcd5509 { + io.Discard.Write([]byte("Expect ErrConnClosed")) + } + }() + + runtime.Gosched() + cli.Close() + + <-donec + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/etcd6708.go b/src/runtime/testdata/testgoroutineleakprofile/goker/etcd6708.go new file mode 100644 index 00000000000000..afbbe35104bbb8 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/etcd6708.go @@ -0,0 +1,100 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "context" + "os" + "runtime" + "runtime/pprof" + "sync" +) + +func init() { + register("Etcd6708", Etcd6708) +} + +type EndpointSelectionMode_etcd6708 int + +const ( + EndpointSelectionRandom_etcd6708 EndpointSelectionMode_etcd6708 = iota + EndpointSelectionPrioritizeLeader_etcd6708 +) + +type MembersAPI_etcd6708 interface { + Leader(ctx context.Context) +} + +type Client_etcd6708 interface { + Sync(ctx context.Context) + SetEndpoints() + httpClient_etcd6708 +} + +type httpClient_etcd6708 interface { + Do(context.Context) +} + +type httpClusterClient_etcd6708 struct { + sync.RWMutex + selectionMode EndpointSelectionMode_etcd6708 +} + +func (c *httpClusterClient_etcd6708) getLeaderEndpoint() { + mAPI := NewMembersAPI_etcd6708(c) + mAPI.Leader(context.Background()) +} + +func (c *httpClusterClient_etcd6708) SetEndpoints() { + switch c.selectionMode { + case EndpointSelectionRandom_etcd6708: + case EndpointSelectionPrioritizeLeader_etcd6708: + c.getLeaderEndpoint() + } +} + +func (c *httpClusterClient_etcd6708) Do(ctx context.Context) { + c.RLock() + c.RUnlock() +} + +func (c *httpClusterClient_etcd6708) Sync(ctx context.Context) { + c.Lock() + defer c.Unlock() + + c.SetEndpoints() +} + +type httpMembersAPI_etcd6708 struct { + client httpClient_etcd6708 +} + +func (m *httpMembersAPI_etcd6708) Leader(ctx context.Context) { + m.client.Do(ctx) +} + +func NewMembersAPI_etcd6708(c Client_etcd6708) MembersAPI_etcd6708 { + return &httpMembersAPI_etcd6708{ + client: c, + } +} + +func Etcd6708() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + + go func() { + hc := &httpClusterClient_etcd6708{ + selectionMode: EndpointSelectionPrioritizeLeader_etcd6708, + } + hc.Sync(context.Background()) + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/etcd6857.go b/src/runtime/testdata/testgoroutineleakprofile/goker/etcd6857.go new file mode 100644 index 00000000000000..0798ab23d3b588 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/etcd6857.go @@ -0,0 +1,81 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: etcd + * Issue or PR : https://github.com/etcd-io/etcd/pull/6857 + * Buggy version: 7c8f13aed7fe251e7066ed6fc1a090699c2cae0e + * fix commit-id: 7afc490c95789c408fbc256d8e790273d331c984 + * Flaky: 19/100 + */ +package main + +import ( + "os" + "runtime/pprof" + "time" +) + +func init() { + register("Etcd6857", Etcd6857) +} + +type Status_etcd6857 struct{} + +type node_etcd6857 struct { + status chan chan Status_etcd6857 + stop chan struct{} + done chan struct{} +} + +func (n *node_etcd6857) Status() Status_etcd6857 { + c := make(chan Status_etcd6857) + n.status <- c + return <-c +} + +func (n *node_etcd6857) run() { + for { + select { + case c := <-n.status: + c <- Status_etcd6857{} + case <-n.stop: + close(n.done) + return + } + } +} + +func (n *node_etcd6857) Stop() { + select { + case n.stop <- struct{}{}: + case <-n.done: + return + } + <-n.done +} + +func NewNode_etcd6857() *node_etcd6857 { + return &node_etcd6857{ + status: make(chan chan Status_etcd6857), + stop: make(chan struct{}), + done: make(chan struct{}), + } +} + +func Etcd6857() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + for i := 0; i <= 100; i++ { + go func() { + n := NewNode_etcd6857() + go n.run() // G1 + go n.Status() // G2 + go n.Stop() // G3 + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/etcd6873.go b/src/runtime/testdata/testgoroutineleakprofile/goker/etcd6873.go new file mode 100644 index 00000000000000..1846d0f260c2f6 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/etcd6873.go @@ -0,0 +1,98 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: etcd + * Issue or PR : https://github.com/etcd-io/etcd/commit/7618fdd1d642e47cac70c03f637b0fd798a53a6e + * Buggy version: 377f19b0031f9c0aafe2aec28b6f9019311f52f9 + * fix commit-id: 7618fdd1d642e47cac70c03f637b0fd798a53a6e + * Flaky: 9/100 + */ +package main + +import ( + "os" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Etcd6873", Etcd6873) +} + +type watchBroadcast_etcd6873 struct{} + +type watchBroadcasts_etcd6873 struct { + mu sync.Mutex + updatec chan *watchBroadcast_etcd6873 + donec chan struct{} +} + +func newWatchBroadcasts_etcd6873() *watchBroadcasts_etcd6873 { + wbs := &watchBroadcasts_etcd6873{ + updatec: make(chan *watchBroadcast_etcd6873, 1), + donec: make(chan struct{}), + } + go func() { // G2 + defer close(wbs.donec) + for wb := range wbs.updatec { + wbs.coalesce(wb) + } + }() + return wbs +} + +func (wbs *watchBroadcasts_etcd6873) coalesce(wb *watchBroadcast_etcd6873) { + wbs.mu.Lock() + wbs.mu.Unlock() +} + +func (wbs *watchBroadcasts_etcd6873) stop() { + wbs.mu.Lock() + defer wbs.mu.Unlock() + close(wbs.updatec) + <-wbs.donec +} + +func (wbs *watchBroadcasts_etcd6873) update(wb *watchBroadcast_etcd6873) { + select { + case wbs.updatec <- wb: + default: + } +} + +// Example of goroutine leak trace: +// +// G1 G2 G3 +//--------------------------------------------------------- +// newWatchBroadcasts() +// wbs.update() +// wbs.updatec <- +// return +// <-wbs.updatec +// wbs.coalesce() +// wbs.stop() +// wbs.mu.Lock() +// close(wbs.updatec) +// <-wbs.donec +// wbs.mu.Lock() +//---------------------G2,G3 leak------------------------- +// + +func Etcd6873() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 100; i++ { + go func() { + wbs := newWatchBroadcasts_etcd6873() // G1 + wbs.update(&watchBroadcast_etcd6873{}) + go wbs.stop() // G3 + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/etcd7492.go b/src/runtime/testdata/testgoroutineleakprofile/goker/etcd7492.go new file mode 100644 index 00000000000000..3c8d58a221cac5 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/etcd7492.go @@ -0,0 +1,163 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: etcd + * Issue or PR : https://github.com/etcd-io/etcd/pull/7492 + * Buggy version: 51939650057d602bb5ab090633138fffe36854dc + * fix commit-id: 1b1fabef8ffec606909f01c3983300fff539f214 + * Flaky: 40/100 + */ +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Etcd7492", Etcd7492) +} + +type TokenProvider_etcd7492 interface { + assign() + enable() + disable() +} + +type simpleTokenTTLKeeper_etcd7492 struct { + tokens map[string]time.Time + addSimpleTokenCh chan struct{} + stopCh chan chan struct{} + deleteTokenFunc func(string) +} + +type authStore_etcd7492 struct { + tokenProvider TokenProvider_etcd7492 +} + +func (as *authStore_etcd7492) Authenticate() { + as.tokenProvider.assign() +} + +func NewSimpleTokenTTLKeeper_etcd7492(deletefunc func(string)) *simpleTokenTTLKeeper_etcd7492 { + stk := &simpleTokenTTLKeeper_etcd7492{ + tokens: make(map[string]time.Time), + addSimpleTokenCh: make(chan struct{}, 1), + stopCh: make(chan chan struct{}), + deleteTokenFunc: deletefunc, + } + go stk.run() // G1 + return stk +} + +func (tm *simpleTokenTTLKeeper_etcd7492) run() { + tokenTicker := time.NewTicker(time.Nanosecond) + defer tokenTicker.Stop() + for { + select { + case <-tm.addSimpleTokenCh: + runtime.Gosched() + /// Make tm.tokens not empty is enough + tm.tokens["1"] = time.Now() + case <-tokenTicker.C: + runtime.Gosched() + for t, _ := range tm.tokens { + tm.deleteTokenFunc(t) + delete(tm.tokens, t) + } + case waitCh := <-tm.stopCh: + waitCh <- struct{}{} + return + } + } +} + +func (tm *simpleTokenTTLKeeper_etcd7492) addSimpleToken() { + tm.addSimpleTokenCh <- struct{}{} + runtime.Gosched() +} + +func (tm *simpleTokenTTLKeeper_etcd7492) stop() { + waitCh := make(chan struct{}) + tm.stopCh <- waitCh + <-waitCh + close(tm.stopCh) +} + +type tokenSimple_etcd7492 struct { + simpleTokenKeeper *simpleTokenTTLKeeper_etcd7492 + simpleTokensMu sync.RWMutex +} + +func (t *tokenSimple_etcd7492) assign() { + t.assignSimpleTokenToUser() +} + +func (t *tokenSimple_etcd7492) assignSimpleTokenToUser() { + t.simpleTokensMu.Lock() + runtime.Gosched() + t.simpleTokenKeeper.addSimpleToken() + t.simpleTokensMu.Unlock() +} +func newDeleterFunc(t *tokenSimple_etcd7492) func(string) { + return func(tk string) { + t.simpleTokensMu.Lock() + defer t.simpleTokensMu.Unlock() + } +} + +func (t *tokenSimple_etcd7492) enable() { + t.simpleTokenKeeper = NewSimpleTokenTTLKeeper_etcd7492(newDeleterFunc(t)) +} + +func (t *tokenSimple_etcd7492) disable() { + if t.simpleTokenKeeper != nil { + t.simpleTokenKeeper.stop() + t.simpleTokenKeeper = nil + } + t.simpleTokensMu.Lock() + t.simpleTokensMu.Unlock() +} + +func newTokenProviderSimple_etcd7492() *tokenSimple_etcd7492 { + return &tokenSimple_etcd7492{} +} + +func setupAuthStore_etcd7492() (store *authStore_etcd7492, teardownfunc func()) { + as := &authStore_etcd7492{ + tokenProvider: newTokenProviderSimple_etcd7492(), + } + as.tokenProvider.enable() + tearDown := func() { + as.tokenProvider.disable() + } + return as, tearDown +} + +func Etcd7492() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + for i := 0; i < 100; i++ { + go func() { + as, tearDown := setupAuthStore_etcd7492() + defer tearDown() + var wg sync.WaitGroup + wg.Add(3) + for i := 0; i < 3; i++ { + go func() { // G2 + as.Authenticate() + defer wg.Done() + }() + } + wg.Wait() + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/etcd7902.go b/src/runtime/testdata/testgoroutineleakprofile/goker/etcd7902.go new file mode 100644 index 00000000000000..0a96d7f0472d9e --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/etcd7902.go @@ -0,0 +1,87 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: etcd + * Issue or PR : https://github.com/coreos/etcd/pull/7902 + * Buggy version: dfdaf082c51ba14861267f632f6af795a27eb4ef + * fix commit-id: 87d99fe0387ee1df1cf1811d88d37331939ef4ae + * Flaky: 100/100 + */ +package main + +import ( + "os" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Etcd7902", Etcd7902) +} + +type roundClient_etcd7902 struct { + progress int + acquire func() + validate func() + release func() +} + +func runElectionFunc_etcd7902() { + rcs := make([]roundClient_etcd7902, 3) + nextc := make(chan bool) + for i := range rcs { + var rcNextc chan bool + setRcNextc := func() { + rcNextc = nextc + } + rcs[i].acquire = func() {} + rcs[i].validate = func() { + setRcNextc() + } + rcs[i].release = func() { + if i == 0 { // Assume the first roundClient is the leader + close(nextc) + nextc = make(chan bool) + } + <-rcNextc // Follower is blocking here + } + } + doRounds_etcd7902(rcs, 100) +} + +func doRounds_etcd7902(rcs []roundClient_etcd7902, rounds int) { + var mu sync.Mutex + var wg sync.WaitGroup + wg.Add(len(rcs)) + for i := range rcs { + go func(rc *roundClient_etcd7902) { // G2,G3 + defer wg.Done() + for rc.progress < rounds || rounds <= 0 { + rc.acquire() + mu.Lock() + rc.validate() + mu.Unlock() + time.Sleep(10 * time.Millisecond) + rc.progress++ + mu.Lock() + rc.release() + mu.Unlock() + } + }(&rcs[i]) + } + wg.Wait() +} + +func Etcd7902() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + for i := 0; i < 100; i++ { + go runElectionFunc_etcd7902() // G1 + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/grpc1275.go b/src/runtime/testdata/testgoroutineleakprofile/goker/grpc1275.go new file mode 100644 index 00000000000000..ec5491e438f74a --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/grpc1275.go @@ -0,0 +1,111 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: grpc-go + * Issue or PR : https://github.com/grpc/grpc-go/pull/1275 + * Buggy version: (missing) + * fix commit-id: 0669f3f89e0330e94bb13fa1ce8cc704aab50c9c + * Flaky: 100/100 + */ +package main + +import ( + "io" + "os" + "runtime/pprof" + "time" +) + +func init() { + register("Grpc1275", Grpc1275) +} + +type recvBuffer_grpc1275 struct { + c chan bool +} + +func (b *recvBuffer_grpc1275) get() <-chan bool { + return b.c +} + +type recvBufferReader_grpc1275 struct { + recv *recvBuffer_grpc1275 +} + +func (r *recvBufferReader_grpc1275) Read(p []byte) (int, error) { + select { + case <-r.recv.get(): + } + return 0, nil +} + +type Stream_grpc1275 struct { + trReader io.Reader +} + +func (s *Stream_grpc1275) Read(p []byte) (int, error) { + return io.ReadFull(s.trReader, p) +} + +type http2Client_grpc1275 struct{} + +func (t *http2Client_grpc1275) CloseStream(s *Stream_grpc1275) { + // It is the client.CloseSream() method called by the + // main goroutine that should send the message, but it + // is not. The patch is to send out this message. +} + +func (t *http2Client_grpc1275) NewStream() *Stream_grpc1275 { + return &Stream_grpc1275{ + trReader: &recvBufferReader_grpc1275{ + recv: &recvBuffer_grpc1275{ + c: make(chan bool), + }, + }, + } +} + +func testInflightStreamClosing_grpc1275() { + client := &http2Client_grpc1275{} + stream := client.NewStream() + donec := make(chan bool) + go func() { // G2 + defer close(donec) + stream.Read([]byte{1}) + }() + + client.CloseStream(stream) + + timeout := time.NewTimer(300 * time.Nanosecond) + select { + case <-donec: + if !timeout.Stop() { + <-timeout.C + } + case <-timeout.C: + } +} + +/// +/// G1 G2 +/// testInflightStreamClosing() +/// stream.Read() +/// io.ReadFull() +/// <- r.recv.get() +/// CloseStream() +/// <- donec +/// ------------G1 timeout, G2 leak--------------------- +/// + +func Grpc1275() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + go func() { + testInflightStreamClosing_grpc1275() // G1 + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/grpc1424.go b/src/runtime/testdata/testgoroutineleakprofile/goker/grpc1424.go new file mode 100644 index 00000000000000..777534a788163c --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/grpc1424.go @@ -0,0 +1,105 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: grpc-go + * Issue or PR : https://github.com/grpc/grpc-go/pull/1424 + * Buggy version: 39c8c3866d926d95e11c03508bf83d00f2963f91 + * fix commit-id: 64bd0b04a7bb1982078bae6a2ab34c226125fbc1 + * Flaky: 100/100 + */ +package main + +import ( + "os" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Grpc1424", Grpc1424) +} + +type Balancer_grpc1424 interface { + Notify() <-chan bool +} + +type roundRobin_grpc1424 struct { + mu sync.Mutex + addrCh chan bool +} + +func (rr *roundRobin_grpc1424) Notify() <-chan bool { + return rr.addrCh +} + +type addrConn_grpc1424 struct { + mu sync.Mutex +} + +func (ac *addrConn_grpc1424) tearDown() { + ac.mu.Lock() + defer ac.mu.Unlock() +} + +type dialOption_grpc1424 struct { + balancer Balancer_grpc1424 +} + +type ClientConn_grpc1424 struct { + dopts dialOption_grpc1424 + conns []*addrConn_grpc1424 +} + +func (cc *ClientConn_grpc1424) lbWatcher(doneChan chan bool) { + for addr := range cc.dopts.balancer.Notify() { + if addr { + // nop, make compiler happy + } + var ( + del []*addrConn_grpc1424 + ) + for _, a := range cc.conns { + del = append(del, a) + } + for _, c := range del { + c.tearDown() + } + } +} + +func NewClientConn_grpc1424() *ClientConn_grpc1424 { + cc := &ClientConn_grpc1424{ + dopts: dialOption_grpc1424{ + &roundRobin_grpc1424{addrCh: make(chan bool)}, + }, + } + return cc +} + +func DialContext_grpc1424() { + cc := NewClientConn_grpc1424() + waitC := make(chan error, 1) + go func() { // G2 + defer close(waitC) + ch := cc.dopts.balancer.Notify() + if ch != nil { + doneChan := make(chan bool) + go cc.lbWatcher(doneChan) // G3 + <-doneChan + } + }() + /// close addrCh + close(cc.dopts.balancer.(*roundRobin_grpc1424).addrCh) +} + +func Grpc1424() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + go DialContext_grpc1424() // G1 +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/grpc1460.go b/src/runtime/testdata/testgoroutineleakprofile/goker/grpc1460.go new file mode 100644 index 00000000000000..bc658b408d8c6b --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/grpc1460.go @@ -0,0 +1,84 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: grpc + * Issue or PR : https://github.com/grpc/grpc-go/pull/1460 + * Buggy version: 7db1564ba1229bc42919bb1f6d9c4186f3aa8678 + * fix commit-id: e605a1ecf24b634f94f4eefdab10a9ada98b70dd + * Flaky: 100/100 + */ +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Grpc1460", Grpc1460) +} + +type Stream_grpc1460 struct{} + +type http2Client_grpc1460 struct { + mu sync.Mutex + awakenKeepalive chan struct{} + activeStream []*Stream_grpc1460 +} + +func (t *http2Client_grpc1460) keepalive() { + t.mu.Lock() + if len(t.activeStream) < 1 { + <-t.awakenKeepalive + runtime.Gosched() + t.mu.Unlock() + } else { + t.mu.Unlock() + } +} + +func (t *http2Client_grpc1460) NewStream() { + t.mu.Lock() + runtime.Gosched() + t.activeStream = append(t.activeStream, &Stream_grpc1460{}) + if len(t.activeStream) == 1 { + select { + case t.awakenKeepalive <- struct{}{}: + default: + } + } + t.mu.Unlock() +} + +/// +/// G1 G2 +/// client.keepalive() +/// client.NewStream() +/// t.mu.Lock() +/// <-t.awakenKeepalive +/// t.mu.Lock() +/// ---------------G1, G2 deadlock-------------- +/// + +func Grpc1460() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 1000; i++ { + go func() { + client := &http2Client_grpc1460{ + awakenKeepalive: make(chan struct{}), + } + go client.keepalive() //G1 + go client.NewStream() //G2 + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/grpc3017.go b/src/runtime/testdata/testgoroutineleakprofile/goker/grpc3017.go new file mode 100644 index 00000000000000..0523b9509fdcbe --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/grpc3017.go @@ -0,0 +1,123 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +// This test case is a reproduction of grpc/3017. +// +// It is a goroutine leak that also simultaneously engages many GC assists. +// Testing runtime behaviour when pivoting between regular and goroutine leak detection modes. + +func init() { + register("Grpc3017", Grpc3017) +} + +type Address_grpc3017 int +type SubConn_grpc3017 int + +type subConnCacheEntry_grpc3017 struct { + sc SubConn_grpc3017 + cancel func() + abortDeleting bool +} + +type lbCacheClientConn_grpc3017 struct { + mu sync.Mutex // L1 + timeout time.Duration + subConnCache map[Address_grpc3017]*subConnCacheEntry_grpc3017 + subConnToAddr map[SubConn_grpc3017]Address_grpc3017 +} + +func (ccc *lbCacheClientConn_grpc3017) NewSubConn(addrs []Address_grpc3017) SubConn_grpc3017 { + if len(addrs) != 1 { + return SubConn_grpc3017(1) + } + addrWithoutMD := addrs[0] + ccc.mu.Lock() // L1 + defer ccc.mu.Unlock() + if entry, ok := ccc.subConnCache[addrWithoutMD]; ok { + entry.cancel() + delete(ccc.subConnCache, addrWithoutMD) + return entry.sc + } + scNew := SubConn_grpc3017(1) + ccc.subConnToAddr[scNew] = addrWithoutMD + return scNew +} + +func (ccc *lbCacheClientConn_grpc3017) RemoveSubConn(sc SubConn_grpc3017) { + ccc.mu.Lock() // L1 + defer ccc.mu.Unlock() + addr, ok := ccc.subConnToAddr[sc] + if !ok { + return + } + + if entry, ok := ccc.subConnCache[addr]; ok { + if entry.sc != sc { + delete(ccc.subConnToAddr, sc) + } + return + } + + entry := &subConnCacheEntry_grpc3017{ + sc: sc, + } + ccc.subConnCache[addr] = entry + + timer := time.AfterFunc(ccc.timeout, func() { // G3 + runtime.Gosched() + ccc.mu.Lock() // L1 + if entry.abortDeleting { + return // Missing unlock + } + delete(ccc.subConnToAddr, sc) + delete(ccc.subConnCache, addr) + ccc.mu.Unlock() + }) + + entry.cancel = func() { + if !timer.Stop() { + entry.abortDeleting = true + } + } +} + +func Grpc3017() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 100; i++ { + go func() { //G1 + done := make(chan struct{}) + + ccc := &lbCacheClientConn_grpc3017{ + timeout: time.Nanosecond, + subConnCache: make(map[Address_grpc3017]*subConnCacheEntry_grpc3017), + subConnToAddr: make(map[SubConn_grpc3017]Address_grpc3017), + } + + sc := ccc.NewSubConn([]Address_grpc3017{Address_grpc3017(1)}) + go func() { // G2 + for i := 0; i < 10000; i++ { + ccc.RemoveSubConn(sc) + sc = ccc.NewSubConn([]Address_grpc3017{Address_grpc3017(1)}) + } + close(done) + }() + <-done + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/grpc660.go b/src/runtime/testdata/testgoroutineleakprofile/goker/grpc660.go new file mode 100644 index 00000000000000..5f6201ec8062d9 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/grpc660.go @@ -0,0 +1,65 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: grpc-go + * Issue or PR : https://github.com/grpc/grpc-go/pull/660 + * Buggy version: db85417dd0de6cc6f583672c6175a7237e5b5dd2 + * fix commit-id: ceacfbcbc1514e4e677932fd55938ac455d182fb + * Flaky: 100/100 + */ +package main + +import ( + "math/rand" + "os" + "runtime" + "runtime/pprof" +) + +func init() { + register("Grpc660", Grpc660) +} + +type benchmarkClient_grpc660 struct { + stop chan bool +} + +func (bc *benchmarkClient_grpc660) doCloseLoopUnary() { + for { + done := make(chan bool) + go func() { // G2 + if rand.Intn(10) > 7 { + done <- false + return + } + done <- true + }() + select { + case <-bc.stop: + return + case <-done: + } + } +} + +func Grpc660() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + go func() { + bc := &benchmarkClient_grpc660{ + stop: make(chan bool), + } + go bc.doCloseLoopUnary() // G1 + go func() { // helper goroutine + bc.stop <- true + }() + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/grpc795.go b/src/runtime/testdata/testgoroutineleakprofile/goker/grpc795.go new file mode 100644 index 00000000000000..72005cc844e225 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/grpc795.go @@ -0,0 +1,74 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Grpc795", Grpc795) +} + +type Server_grpc795 struct { + mu sync.Mutex + drain bool +} + +func (s *Server_grpc795) GracefulStop() { + s.mu.Lock() + if s.drain { + s.mu.Lock() + return + } + s.drain = true + s.mu.Unlock() +} +func (s *Server_grpc795) Serve() { + s.mu.Lock() + s.mu.Unlock() +} + +func NewServer_grpc795() *Server_grpc795 { + return &Server_grpc795{} +} + +type test_grpc795 struct { + srv *Server_grpc795 +} + +func (te *test_grpc795) startServer() { + s := NewServer_grpc795() + te.srv = s + go s.Serve() +} + +func newTest_grpc795() *test_grpc795 { + return &test_grpc795{} +} + +func testServerGracefulStopIdempotent_grpc795() { + te := newTest_grpc795() + + te.startServer() + + for i := 0; i < 3; i++ { + te.srv.GracefulStop() + } +} + +func Grpc795() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + for i := 0; i < 100; i++ { + go testServerGracefulStopIdempotent_grpc795() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/grpc862.go b/src/runtime/testdata/testgoroutineleakprofile/goker/grpc862.go new file mode 100644 index 00000000000000..188b3b88ba5f8a --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/grpc862.go @@ -0,0 +1,105 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: grpc-go + * Issue or PR : https://github.com/grpc/grpc-go/pull/862 + * Buggy version: d8f4ebe77f6b7b6403d7f98626de8a534f9b93a7 + * fix commit-id: dd5645bebff44f6b88780bb949022a09eadd7dae + * Flaky: 100/100 + */ +package main + +import ( + "context" + "os" + "runtime" + "runtime/pprof" + "time" +) + +func init() { + register("Grpc862", Grpc862) +} + +type ClientConn_grpc862 struct { + ctx context.Context + cancel context.CancelFunc + conns []*addrConn_grpc862 +} + +func (cc *ClientConn_grpc862) Close() { + cc.cancel() + conns := cc.conns + cc.conns = nil + for _, ac := range conns { + ac.tearDown() + } +} + +func (cc *ClientConn_grpc862) resetAddrConn() { + ac := &addrConn_grpc862{ + cc: cc, + } + cc.conns = append(cc.conns, ac) + ac.ctx, ac.cancel = context.WithCancel(cc.ctx) + ac.resetTransport() +} + +type addrConn_grpc862 struct { + cc *ClientConn_grpc862 + ctx context.Context + cancel context.CancelFunc +} + +func (ac *addrConn_grpc862) resetTransport() { + for retries := 1; ; retries++ { + _ = 2 * time.Nanosecond * time.Duration(retries) + timeout := 10 * time.Nanosecond + _, cancel := context.WithTimeout(ac.ctx, timeout) + _ = time.Now() + cancel() + <-ac.ctx.Done() + return + } +} + +func (ac *addrConn_grpc862) tearDown() { + ac.cancel() +} + +func DialContext_grpc862(ctx context.Context) (conn *ClientConn_grpc862) { + cc := &ClientConn_grpc862{} + cc.ctx, cc.cancel = context.WithCancel(context.Background()) + defer func() { + select { + case <-ctx.Done(): + if conn != nil { + conn.Close() + } + conn = nil + default: + } + }() + go func() { // G2 + cc.resetAddrConn() + }() + return conn +} + +func Grpc862() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + go func() { + ctx, cancel := context.WithCancel(context.Background()) + go DialContext_grpc862(ctx) // G1 + go cancel() // helper goroutine + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/hugo3251.go b/src/runtime/testdata/testgoroutineleakprofile/goker/hugo3251.go new file mode 100644 index 00000000000000..3804692a8bf6bd --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/hugo3251.go @@ -0,0 +1,81 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Hugo3251", Hugo3251) +} + +type remoteLock_hugo3251 struct { + sync.RWMutex // L1 + m map[string]*sync.Mutex // L2 +} + +func (l *remoteLock_hugo3251) URLLock(url string) { + l.Lock() // L1 + if _, ok := l.m[url]; !ok { + l.m[url] = &sync.Mutex{} + } + l.m[url].Lock() // L2 + runtime.Gosched() + l.Unlock() // L1 + // runtime.Gosched() +} + +func (l *remoteLock_hugo3251) URLUnlock(url string) { + l.RLock() // L1 + defer l.RUnlock() // L1 + if um, ok := l.m[url]; ok { + um.Unlock() // L2 + } +} + +func resGetRemote_hugo3251(remoteURLLock *remoteLock_hugo3251, url string) error { + remoteURLLock.URLLock(url) + defer func() { remoteURLLock.URLUnlock(url) }() + + return nil +} + +func Hugo3251() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(time.Second) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 11; i++ { + go func() { // G1 + url := "http://foo.bar/foo_Bar-Foo" + remoteURLLock := &remoteLock_hugo3251{m: make(map[string]*sync.Mutex)} + for range []bool{false, true} { + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func(gor int) { // G2 + defer wg.Done() + for j := 0; j < 200; j++ { + err := resGetRemote_hugo3251(remoteURLLock, url) + if err != nil { + fmt.Errorf("Error getting resource content: %s", err) + } + time.Sleep(300 * time.Nanosecond) + } + }(i) + } + wg.Wait() + } + }() + } +} \ No newline at end of file diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/hugo5379.go b/src/runtime/testdata/testgoroutineleakprofile/goker/hugo5379.go new file mode 100644 index 00000000000000..6a1bbe9a3f7767 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/hugo5379.go @@ -0,0 +1,317 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "context" + "log" + "os" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Hugo5379", Hugo5379) +} + +type shortcodeHandler_hugo5379 struct { + p *PageWithoutContent_hugo5379 + contentShortcodes map[int]func() error + contentShortcodesDelta map[int]func() error + init sync.Once // O1 +} + +func (s *shortcodeHandler_hugo5379) executeShortcodesForDelta(p *PageWithoutContent_hugo5379) error { + for k, _ := range s.contentShortcodesDelta { + render := s.contentShortcodesDelta[k] + if err := render(); err != nil { + continue + } + } + return nil +} + +func (s *shortcodeHandler_hugo5379) updateDelta() { + s.init.Do(func() { + s.contentShortcodes = createShortcodeRenderers_hugo5379(s.p.withoutContent()) + }) + + delta := make(map[int]func() error) + + for k, v := range s.contentShortcodes { + if _, ok := delta[k]; !ok { + delta[k] = v + } + } + + s.contentShortcodesDelta = delta +} + +type Page_hugo5379 struct { + *pageInit_hugo5379 + *pageContentInit_hugo5379 + pageWithoutContent *PageWithoutContent_hugo5379 + contentInit sync.Once // O2 + contentInitMu sync.Mutex // L1 + shortcodeState *shortcodeHandler_hugo5379 +} + +func (p *Page_hugo5379) WordCount() { + p.initContentPlainAndMeta() +} + +func (p *Page_hugo5379) initContentPlainAndMeta() { + p.initContent() + p.initPlain(true) +} + +func (p *Page_hugo5379) initPlain(lock bool) { + p.plainInit.Do(func() { + if lock { + /// Double locking here. + p.contentInitMu.Lock() + defer p.contentInitMu.Unlock() + } + }) +} + +func (p *Page_hugo5379) withoutContent() *PageWithoutContent_hugo5379 { + p.pageInit_hugo5379.withoutContentInit.Do(func() { + p.pageWithoutContent = &PageWithoutContent_hugo5379{Page_hugo5379: p} + }) + return p.pageWithoutContent +} + +func (p *Page_hugo5379) prepareForRender() error { + var err error + if err = handleShortcodes_hugo5379(p.withoutContent()); err != nil { + return err + } + return nil +} + +func (p *Page_hugo5379) setContentInit() { + p.shortcodeState.updateDelta() +} + +func (p *Page_hugo5379) initContent() { + p.contentInit.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) + defer cancel() + c := make(chan error, 1) + + go func() { // G2 + var err error + p.contentInitMu.Lock() // first lock here + defer p.contentInitMu.Unlock() + + err = p.prepareForRender() + if err != nil { + c <- err + return + } + c <- err + }() + + select { + case <-ctx.Done(): + case <-c: + } + }) +} + +type PageWithoutContent_hugo5379 struct { + *Page_hugo5379 +} + +type pageInit_hugo5379 struct { + withoutContentInit sync.Once +} + +type pageContentInit_hugo5379 struct { + contentInit sync.Once // O3 + plainInit sync.Once // O4 +} + +type HugoSites_hugo5379 struct { + Sites []*Site_hugo5379 +} + +func (h *HugoSites_hugo5379) render() { + for _, s := range h.Sites { + for _, s2 := range h.Sites { + s2.preparePagesForRender() + } + s.renderPages() + } +} + +func (h *HugoSites_hugo5379) Build() { + h.render() +} + +type Pages_hugo5379 []*Page_hugo5379 + +type PageCollections_hugo5379 struct { + Pages Pages_hugo5379 +} + +type Site_hugo5379 struct { + *PageCollections_hugo5379 +} + +func (s *Site_hugo5379) preparePagesForRender() { + for _, p := range s.Pages { + p.setContentInit() + } +} + +func (s *Site_hugo5379) renderForLayouts() { + /// Omit reflections + for _, p := range s.Pages { + p.WordCount() + } +} + +func (s *Site_hugo5379) renderAndWritePage() { + s.renderForLayouts() +} + +func (s *Site_hugo5379) renderPages() { + numWorkers := 2 + wg := &sync.WaitGroup{} + + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go pageRenderer_hugo5379(s, wg) // G3 + } + + wg.Wait() +} + +type sitesBuilder_hugo5379 struct { + H *HugoSites_hugo5379 +} + +func (s *sitesBuilder_hugo5379) Build() *sitesBuilder_hugo5379 { + return s.build() +} + +func (s *sitesBuilder_hugo5379) build() *sitesBuilder_hugo5379 { + s.H.Build() + return s +} + +func (s *sitesBuilder_hugo5379) CreateSitesE() error { + sites, err := NewHugoSites_hugo5379() + if err != nil { + return err + } + s.H = sites + return nil +} + +func (s *sitesBuilder_hugo5379) CreateSites() *sitesBuilder_hugo5379 { + if err := s.CreateSitesE(); err != nil { + log.Fatalf("Failed to create sites: %s", err) + } + return s +} + +func newHugoSites_hugo5379(sites ...*Site_hugo5379) (*HugoSites_hugo5379, error) { + h := &HugoSites_hugo5379{Sites: sites} + return h, nil +} + +func newSite_hugo5379() *Site_hugo5379 { + c := &PageCollections_hugo5379{} + s := &Site_hugo5379{ + PageCollections_hugo5379: c, + } + return s +} + +func createSitesFromConfig_hugo5379() []*Site_hugo5379 { + var ( + sites []*Site_hugo5379 + ) + + var s *Site_hugo5379 = newSite_hugo5379() + sites = append(sites, s) + return sites +} + +func NewHugoSites_hugo5379() (*HugoSites_hugo5379, error) { + sites := createSitesFromConfig_hugo5379() + return newHugoSites_hugo5379(sites...) +} + +func prepareShortcodeForPage_hugo5379(p *PageWithoutContent_hugo5379) map[int]func() error { + m := make(map[int]func() error) + m[0] = func() error { + return renderShortcode_hugo5379(p) + } + return m +} + +func renderShortcode_hugo5379(p *PageWithoutContent_hugo5379) error { + return renderShortcodeWithPage_hugo5379(p) +} + +func renderShortcodeWithPage_hugo5379(p *PageWithoutContent_hugo5379) error { + /// Omit reflections + p.WordCount() + return nil +} + +func createShortcodeRenderers_hugo5379(p *PageWithoutContent_hugo5379) map[int]func() error { + return prepareShortcodeForPage_hugo5379(p) +} + +func newShortcodeHandler_hugo5379(p *Page_hugo5379) *shortcodeHandler_hugo5379 { + return &shortcodeHandler_hugo5379{ + p: p.withoutContent(), + contentShortcodes: make(map[int]func() error), + contentShortcodesDelta: make(map[int]func() error), + } +} + +func handleShortcodes_hugo5379(p *PageWithoutContent_hugo5379) error { + return p.shortcodeState.executeShortcodesForDelta(p) +} + +func pageRenderer_hugo5379(s *Site_hugo5379, wg *sync.WaitGroup) { + defer wg.Done() + s.renderAndWritePage() +} + +func Hugo5379() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 100; i++ { + go func() { // G1 + b := &sitesBuilder_hugo5379{} + s := b.CreateSites() + for _, site := range s.H.Sites { + p := &Page_hugo5379{ + pageInit_hugo5379: &pageInit_hugo5379{}, + pageContentInit_hugo5379: &pageContentInit_hugo5379{}, + pageWithoutContent: &PageWithoutContent_hugo5379{}, + contentInit: sync.Once{}, + contentInitMu: sync.Mutex{}, + shortcodeState: nil, + } + p.shortcodeState = newShortcodeHandler_hugo5379(p) + site.Pages = append(site.Pages, p) + } + s.Build() + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/istio16224.go b/src/runtime/testdata/testgoroutineleakprofile/goker/istio16224.go new file mode 100644 index 00000000000000..839051cc64a18d --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/istio16224.go @@ -0,0 +1,129 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Istio16224", Istio16224) +} + +type ConfigStoreCache_istio16224 interface { + RegisterEventHandler(handler func()) + Run() +} + +type Event_istio16224 int + +type Handler_istio16224 func(Event_istio16224) + +type configstoreMonitor_istio16224 struct { + handlers []Handler_istio16224 + eventCh chan Event_istio16224 +} + +func (m *configstoreMonitor_istio16224) Run(stop <-chan struct{}) { + for { + select { + case <-stop: + // This bug is not descibed, but is a true positive (in our eyes) + // In a real run main exits when the goro is blocked here. + if _, ok := <-m.eventCh; ok { + close(m.eventCh) + } + return + case ce, ok := <-m.eventCh: + if ok { + m.processConfigEvent(ce) + } + } + } +} + +func (m *configstoreMonitor_istio16224) processConfigEvent(ce Event_istio16224) { + m.applyHandlers(ce) +} + +func (m *configstoreMonitor_istio16224) AppendEventHandler(h Handler_istio16224) { + m.handlers = append(m.handlers, h) +} + +func (m *configstoreMonitor_istio16224) applyHandlers(e Event_istio16224) { + for _, f := range m.handlers { + f(e) + } +} +func (m *configstoreMonitor_istio16224) ScheduleProcessEvent(configEvent Event_istio16224) { + m.eventCh <- configEvent +} + +type Monitor_istio16224 interface { + Run(<-chan struct{}) + AppendEventHandler(Handler_istio16224) + ScheduleProcessEvent(Event_istio16224) +} + +type controller_istio16224 struct { + monitor Monitor_istio16224 +} + +func (c *controller_istio16224) RegisterEventHandler(f func(Event_istio16224)) { + c.monitor.AppendEventHandler(f) +} + +func (c *controller_istio16224) Run(stop <-chan struct{}) { + c.monitor.Run(stop) +} + +func (c *controller_istio16224) Create() { + c.monitor.ScheduleProcessEvent(Event_istio16224(0)) +} + +func NewMonitor_istio16224() Monitor_istio16224 { + return NewBufferedMonitor_istio16224() +} + +func NewBufferedMonitor_istio16224() Monitor_istio16224 { + return &configstoreMonitor_istio16224{ + eventCh: make(chan Event_istio16224), + } +} + +func Istio16224() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 100; i++ { + go func() { + controller := &controller_istio16224{monitor: NewMonitor_istio16224()} + done := make(chan bool) + lock := sync.Mutex{} + controller.RegisterEventHandler(func(event Event_istio16224) { + lock.Lock() + defer lock.Unlock() + done <- true + }) + + stop := make(chan struct{}) + go controller.Run(stop) + + controller.Create() + + lock.Lock() // blocks + lock.Unlock() + <-done + + close(stop) + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/istio17860.go b/src/runtime/testdata/testgoroutineleakprofile/goker/istio17860.go new file mode 100644 index 00000000000000..aa8317c6d5f41b --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/istio17860.go @@ -0,0 +1,144 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "context" + "os" + "runtime/pprof" + + "sync" + "time" +) + +func init() { + register("Istio17860", Istio17860) +} + +type Proxy_istio17860 interface { + IsLive() bool +} + +type TestProxy_istio17860 struct { + live func() bool +} + +func (tp TestProxy_istio17860) IsLive() bool { + if tp.live == nil { + return true + } + return tp.live() +} + +type Agent_istio17860 interface { + Run(ctx context.Context) + Restart() +} + +type exitStatus_istio17860 int + +type agent_istio17860 struct { + proxy Proxy_istio17860 + mu *sync.Mutex + statusCh chan exitStatus_istio17860 + currentEpoch int + activeEpochs map[int]struct{} +} + +func (a *agent_istio17860) Run(ctx context.Context) { + for { + select { + case status := <-a.statusCh: + a.mu.Lock() + delete(a.activeEpochs, int(status)) + active := len(a.activeEpochs) + a.mu.Unlock() + if active == 0 { + return + } + case <-ctx.Done(): + return + } + } +} + +func (a *agent_istio17860) Restart() { + a.mu.Lock() + defer a.mu.Unlock() + + a.waitUntilLive() + a.currentEpoch++ + a.activeEpochs[a.currentEpoch] = struct{}{} + + go a.runWait(a.currentEpoch) +} + +func (a *agent_istio17860) runWait(epoch int) { + a.statusCh <- exitStatus_istio17860(epoch) +} + +func (a *agent_istio17860) waitUntilLive() { + if len(a.activeEpochs) == 0 { + return + } + + interval := time.NewTicker(30 * time.Nanosecond) + timer := time.NewTimer(100 * time.Nanosecond) + defer func() { + interval.Stop() + timer.Stop() + }() + + if a.proxy.IsLive() { + return + } + + for { + select { + case <-timer.C: + return + case <-interval.C: + if a.proxy.IsLive() { + return + } + } + } +} + +func NewAgent_istio17860(proxy Proxy_istio17860) Agent_istio17860 { + return &agent_istio17860{ + proxy: proxy, + mu: &sync.Mutex{}, + statusCh: make(chan exitStatus_istio17860), + activeEpochs: make(map[int]struct{}), + } +} + +func Istio17860() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 100; i++ { + go func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + neverLive := func() bool { + return false + } + + a := NewAgent_istio17860(TestProxy_istio17860{live: neverLive}) + go func() { a.Run(ctx) }() + + a.Restart() + go a.Restart() + + time.Sleep(200 * time.Nanosecond) + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/istio18454.go b/src/runtime/testdata/testgoroutineleakprofile/goker/istio18454.go new file mode 100644 index 00000000000000..b410c490322440 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/istio18454.go @@ -0,0 +1,154 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "context" + "os" + "runtime/pprof" + + "sync" + "time" +) + +func init() { + register("Istio18454", Istio18454) +} + +const eventChCap_istio18454 = 1024 + +type Worker_istio18454 struct { + ctx context.Context + ctxCancel context.CancelFunc +} + +func (w *Worker_istio18454) Start(setupFn func(), runFn func(c context.Context)) { + if setupFn != nil { + setupFn() + } + go func() { + runFn(w.ctx) + }() +} + +func (w *Worker_istio18454) Stop() { + w.ctxCancel() +} + +type Strategy_istio18454 struct { + timer *time.Timer + timerFrequency time.Duration + stateLock sync.Mutex + resetChan chan struct{} + worker *Worker_istio18454 + startTimerFn func() +} + +func (s *Strategy_istio18454) OnChange() { + s.stateLock.Lock() + if s.timer != nil { + s.stateLock.Unlock() + s.resetChan <- struct{}{} + return + } + s.startTimerFn() + s.stateLock.Unlock() +} + +func (s *Strategy_istio18454) startTimer() { + s.timer = time.NewTimer(s.timerFrequency) + eventLoop := func(ctx context.Context) { + for { + select { + case <-s.timer.C: + case <-s.resetChan: + if !s.timer.Stop() { + <-s.timer.C + } + s.timer.Reset(s.timerFrequency) + case <-ctx.Done(): + s.timer.Stop() + return + } + } + } + s.worker.Start(nil, eventLoop) +} + +func (s *Strategy_istio18454) Close() { + s.worker.Stop() +} + +type Event_istio18454 int + +type Processor_istio18454 struct { + stateStrategy *Strategy_istio18454 + worker *Worker_istio18454 + eventCh chan Event_istio18454 +} + +func (p *Processor_istio18454) processEvent() { + p.stateStrategy.OnChange() +} + +func (p *Processor_istio18454) Start() { + setupFn := func() { + for i := 0; i < eventChCap_istio18454; i++ { + p.eventCh <- Event_istio18454(0) + } + } + runFn := func(ctx context.Context) { + defer func() { + p.stateStrategy.Close() + }() + for { + select { + case <-ctx.Done(): + return + case <-p.eventCh: + p.processEvent() + } + } + } + p.worker.Start(setupFn, runFn) +} + +func (p *Processor_istio18454) Stop() { + p.worker.Stop() +} + +func NewWorker_istio18454() *Worker_istio18454 { + worker := &Worker_istio18454{} + worker.ctx, worker.ctxCancel = context.WithCancel(context.Background()) + return worker +} + +func Istio18454() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 100; i++ { + go func() { + stateStrategy := &Strategy_istio18454{ + timerFrequency: time.Nanosecond, + resetChan: make(chan struct{}, 1), + worker: NewWorker_istio18454(), + } + stateStrategy.startTimerFn = stateStrategy.startTimer + + p := &Processor_istio18454{ + stateStrategy: stateStrategy, + worker: NewWorker_istio18454(), + eventCh: make(chan Event_istio18454, eventChCap_istio18454), + } + + p.Start() + defer p.Stop() + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes10182.go b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes10182.go new file mode 100644 index 00000000000000..0eca5f41fbfab4 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes10182.go @@ -0,0 +1,95 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: kubernetes + * Issue or PR : https://github.com/kubernetes/kubernetes/pull/10182 + * Buggy version: 4b990d128a17eea9058d28a3b3688ab8abafbd94 + * fix commit-id: 64ad3e17ad15cd0f9a4fd86706eec1c572033254 + * Flaky: 15/100 + */ +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Kubernetes10182", Kubernetes10182) +} + +type statusManager_kubernetes10182 struct { + podStatusesLock sync.RWMutex + podStatusChannel chan bool +} + +func (s *statusManager_kubernetes10182) Start() { + go func() { + for i := 0; i < 2; i++ { + s.syncBatch() + } + }() +} + +func (s *statusManager_kubernetes10182) syncBatch() { + runtime.Gosched() + <-s.podStatusChannel + s.DeletePodStatus() +} + +func (s *statusManager_kubernetes10182) DeletePodStatus() { + s.podStatusesLock.Lock() + defer s.podStatusesLock.Unlock() +} + +func (s *statusManager_kubernetes10182) SetPodStatus() { + s.podStatusesLock.Lock() + defer s.podStatusesLock.Unlock() + s.podStatusChannel <- true +} + +func NewStatusManager_kubernetes10182() *statusManager_kubernetes10182 { + return &statusManager_kubernetes10182{ + podStatusChannel: make(chan bool), + } +} + +// Example of deadlock trace: +// +// G1 G2 G3 +// -------------------------------------------------------------------------------- +// s.Start() +// s.syncBatch() +// s.SetPodStatus() +// <-s.podStatusChannel +// s.podStatusesLock.Lock() +// s.podStatusChannel <- true +// s.podStatusesLock.Unlock() +// return +// s.DeletePodStatus() +// s.podStatusesLock.Lock() +// s.podStatusChannel <- true +// s.podStatusesLock.Lock() +// -----------------------------------G1,G3 leak------------------------------------- + +func Kubernetes10182() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 1000; i++ { + go func() { + s := NewStatusManager_kubernetes10182() + go s.Start() + go s.SetPodStatus() + go s.SetPodStatus() + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes11298.go b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes11298.go new file mode 100644 index 00000000000000..36405f240a6612 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes11298.go @@ -0,0 +1,118 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Kubernetes11298", Kubernetes11298) +} + +type Signal_kubernetes11298 <-chan struct{} + +func After_kubernetes11298(f func()) Signal_kubernetes11298 { + ch := make(chan struct{}) + go func() { + defer close(ch) + if f != nil { + f() + } + }() + return Signal_kubernetes11298(ch) +} + +func Until_kubernetes11298(f func(), period time.Duration, stopCh <-chan struct{}) { + if f == nil { + return + } + for { + select { + case <-stopCh: + return + default: + } + f() + select { + case <-stopCh: + case <-time.After(period): + } + } + +} + +type notifier_kubernetes11298 struct { + lock sync.Mutex + cond *sync.Cond +} + +// abort will be closed no matter what +func (n *notifier_kubernetes11298) serviceLoop(abort <-chan struct{}) { + n.lock.Lock() + defer n.lock.Unlock() + for { + select { + case <-abort: + return + default: + ch := After_kubernetes11298(func() { + n.cond.Wait() + }) + select { + case <-abort: + n.cond.Signal() + <-ch + return + case <-ch: + } + } + } +} + +// abort will be closed no matter what +func Notify_kubernetes11298(abort <-chan struct{}) { + n := ¬ifier_kubernetes11298{} + n.cond = sync.NewCond(&n.lock) + finished := After_kubernetes11298(func() { + Until_kubernetes11298(func() { + for { + select { + case <-abort: + return + default: + func() { + n.lock.Lock() + defer n.lock.Unlock() + n.cond.Signal() + }() + } + } + }, 0, abort) + }) + Until_kubernetes11298(func() { n.serviceLoop(finished) }, 0, abort) +} +func Kubernetes11298() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 1000; i++ { + go func() { + done := make(chan struct{}) + notifyDone := After_kubernetes11298(func() { Notify_kubernetes11298(done) }) + go func() { + defer close(done) + time.Sleep(300 * time.Nanosecond) + }() + <-notifyDone + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes13135.go b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes13135.go new file mode 100644 index 00000000000000..f6aa8b9ddfe178 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes13135.go @@ -0,0 +1,166 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: kubernetes + * Issue or PR : https://github.com/kubernetes/kubernetes/pull/13135 + * Buggy version: 6ced66249d4fd2a81e86b4a71d8df0139fe5ceae + * fix commit-id: a12b7edc42c5c06a2e7d9f381975658692951d5a + * Flaky: 93/100 + */ +package main + +import ( + "os" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Kubernetes13135", Kubernetes13135) +} + +var ( + StopChannel_kubernetes13135 chan struct{} +) + +func Util_kubernetes13135(f func(), period time.Duration, stopCh <-chan struct{}) { + for { + select { + case <-stopCh: + return + default: + } + func() { + f() + }() + time.Sleep(period) + } +} + +type Store_kubernetes13135 interface { + Add(obj interface{}) + Replace(obj interface{}) +} + +type Reflector_kubernetes13135 struct { + store Store_kubernetes13135 +} + +func (r *Reflector_kubernetes13135) ListAndWatch(stopCh <-chan struct{}) error { + r.syncWith() + return nil +} + +func NewReflector_kubernetes13135(store Store_kubernetes13135) *Reflector_kubernetes13135 { + return &Reflector_kubernetes13135{ + store: store, + } +} + +func (r *Reflector_kubernetes13135) syncWith() { + r.store.Replace(nil) +} + +type Cacher_kubernetes13135 struct { + sync.Mutex + initialized sync.WaitGroup + initOnce sync.Once + watchCache *WatchCache_kubernetes13135 + reflector *Reflector_kubernetes13135 +} + +func (c *Cacher_kubernetes13135) processEvent() { + c.Lock() + defer c.Unlock() +} + +func (c *Cacher_kubernetes13135) startCaching(stopChannel <-chan struct{}) { + c.Lock() + for { + err := c.reflector.ListAndWatch(stopChannel) + if err == nil { + break + } + } +} + +type WatchCache_kubernetes13135 struct { + sync.RWMutex + onReplace func() + onEvent func() +} + +func (w *WatchCache_kubernetes13135) SetOnEvent(onEvent func()) { + w.Lock() + defer w.Unlock() + w.onEvent = onEvent +} + +func (w *WatchCache_kubernetes13135) SetOnReplace(onReplace func()) { + w.Lock() + defer w.Unlock() + w.onReplace = onReplace +} + +func (w *WatchCache_kubernetes13135) processEvent() { + w.Lock() + defer w.Unlock() + if w.onEvent != nil { + w.onEvent() + } +} + +func (w *WatchCache_kubernetes13135) Add(obj interface{}) { + w.processEvent() +} + +func (w *WatchCache_kubernetes13135) Replace(obj interface{}) { + w.Lock() + defer w.Unlock() + if w.onReplace != nil { + w.onReplace() + } +} + +func NewCacher_kubernetes13135(stopCh <-chan struct{}) *Cacher_kubernetes13135 { + watchCache := &WatchCache_kubernetes13135{} + cacher := &Cacher_kubernetes13135{ + initialized: sync.WaitGroup{}, + watchCache: watchCache, + reflector: NewReflector_kubernetes13135(watchCache), + } + cacher.initialized.Add(1) + watchCache.SetOnReplace(func() { + cacher.initOnce.Do(func() { cacher.initialized.Done() }) + cacher.Unlock() + }) + watchCache.SetOnEvent(cacher.processEvent) + go Util_kubernetes13135(func() { cacher.startCaching(stopCh) }, 0, stopCh) // G2 + cacher.initialized.Wait() + return cacher +} + +func Kubernetes13135() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + StopChannel_kubernetes13135 = make(chan struct{}) + for i := 0; i < 50; i++ { + go func() { + // Should create a local channel. Using a single global channel + // concurrently will cause a deadlock which does not actually exist + // in the original microbenchmark. + StopChannel_kubernetes13135 := make(chan struct{}) + + c := NewCacher_kubernetes13135(StopChannel_kubernetes13135) // G1 + go c.watchCache.Add(nil) // G3 + go close(StopChannel_kubernetes13135) + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes1321.go b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes1321.go new file mode 100644 index 00000000000000..6c0139a9d24fcb --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes1321.go @@ -0,0 +1,100 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: kubernetes + * Issue or PR : https://github.com/kubernetes/kubernetes/pull/1321 + * Buggy version: 9cd0fc70f1ca852c903b18b0933991036b3b2fa1 + * fix commit-id: 435e0b73bb99862f9dedf56a50260ff3dfef14ff + * Flaky: 1/100 + */ +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Kubernetes1321", Kubernetes1321) +} + +type muxWatcher_kubernetes1321 struct { + result chan struct{} + m *Mux_kubernetes1321 + id int64 +} + +func (mw *muxWatcher_kubernetes1321) Stop() { + mw.m.stopWatching(mw.id) +} + +type Mux_kubernetes1321 struct { + lock sync.Mutex + watchers map[int64]*muxWatcher_kubernetes1321 +} + +func NewMux_kubernetes1321() *Mux_kubernetes1321 { + m := &Mux_kubernetes1321{ + watchers: map[int64]*muxWatcher_kubernetes1321{}, + } + go m.loop() // G2 + return m +} + +func (m *Mux_kubernetes1321) Watch() *muxWatcher_kubernetes1321 { + mw := &muxWatcher_kubernetes1321{ + result: make(chan struct{}), + m: m, + id: int64(len(m.watchers)), + } + m.watchers[mw.id] = mw + runtime.Gosched() + return mw +} + +func (m *Mux_kubernetes1321) loop() { + for i := 0; i < 100; i++ { + m.distribute() + } +} + +func (m *Mux_kubernetes1321) distribute() { + m.lock.Lock() + defer m.lock.Unlock() + for _, w := range m.watchers { + w.result <- struct{}{} + runtime.Gosched() + } +} + +func (m *Mux_kubernetes1321) stopWatching(id int64) { + m.lock.Lock() + defer m.lock.Unlock() + w, ok := m.watchers[id] + if !ok { + return + } + delete(m.watchers, id) + close(w.result) +} + +func testMuxWatcherClose_kubernetes1321() { + m := NewMux_kubernetes1321() + m.watchers[m.Watch().id].Stop() +} + +func Kubernetes1321() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + for i := 0; i < 1000; i++ { + go testMuxWatcherClose_kubernetes1321() // G1 + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes25331.go b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes25331.go new file mode 100644 index 00000000000000..323cb236c0687a --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes25331.go @@ -0,0 +1,72 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: kubernetes + * Issue or PR : https://github.com/kubernetes/kubernetes/pull/25331 + * Buggy version: 5dd087040bb13434f1ddf2f0693d0203c30f28cb + * fix commit-id: 97f4647dc3d8cf46c2b66b89a31c758a6edfb57c + * Flaky: 100/100 + */ +package main + +import ( + "context" + "errors" + "os" + "runtime" + "runtime/pprof" +) + +func init() { + register("Kubernetes25331", Kubernetes25331) +} + +type watchChan_kubernetes25331 struct { + ctx context.Context + cancel context.CancelFunc + resultChan chan bool + errChan chan error +} + +func (wc *watchChan_kubernetes25331) Stop() { + wc.errChan <- errors.New("Error") + wc.cancel() +} + +func (wc *watchChan_kubernetes25331) run() { + select { + case err := <-wc.errChan: + errResult := len(err.Error()) != 0 + wc.cancel() // Removed in fix + wc.resultChan <- errResult + case <-wc.ctx.Done(): + } +} + +func NewWatchChan_kubernetes25331() *watchChan_kubernetes25331 { + ctx, cancel := context.WithCancel(context.Background()) + return &watchChan_kubernetes25331{ + ctx: ctx, + cancel: cancel, + resultChan: make(chan bool), + errChan: make(chan error), + } +} + +func Kubernetes25331() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + go func() { + wc := NewWatchChan_kubernetes25331() + go wc.run() // G1 + go wc.Stop() // G2 + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes26980.go b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes26980.go new file mode 100644 index 00000000000000..38e53cf4ad7218 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes26980.go @@ -0,0 +1,87 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Kubernetes26980", Kubernetes26980) +} + +type processorListener_kubernetes26980 struct { + lock sync.RWMutex + cond sync.Cond + + pendingNotifications []interface{} +} + +func (p *processorListener_kubernetes26980) add(notification interface{}) { + p.lock.Lock() + defer p.lock.Unlock() + + p.pendingNotifications = append(p.pendingNotifications, notification) + p.cond.Broadcast() +} + +func (p *processorListener_kubernetes26980) pop(stopCh <-chan struct{}) { + p.lock.Lock() + runtime.Gosched() + defer p.lock.Unlock() + for { + for len(p.pendingNotifications) == 0 { + select { + case <-stopCh: + return + default: + } + p.cond.Wait() + } + select { + case <-stopCh: + return + } + } +} + +func newProcessListener_kubernetes26980() *processorListener_kubernetes26980 { + ret := &processorListener_kubernetes26980{ + pendingNotifications: []interface{}{}, + } + ret.cond.L = &ret.lock + return ret +} +func Kubernetes26980() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 3000; i++ { + go func() { + pl := newProcessListener_kubernetes26980() + stopCh := make(chan struct{}) + defer close(stopCh) + pl.add(1) + runtime.Gosched() + go pl.pop(stopCh) + + resultCh := make(chan struct{}) + go func() { + pl.lock.Lock() + close(resultCh) + }() + runtime.Gosched() + <-resultCh + pl.lock.Unlock() + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes30872.go b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes30872.go new file mode 100644 index 00000000000000..00cdcf2b678692 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes30872.go @@ -0,0 +1,223 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Kubernetes30872", Kubernetes30872) +} + +type PopProcessFunc_kubernetes30872 func() + +type ProcessFunc_kubernetes30872 func() + +func Util_kubernetes30872(f func(), stopCh <-chan struct{}) { + JitterUntil_kubernetes30872(f, stopCh) +} + +func JitterUntil_kubernetes30872(f func(), stopCh <-chan struct{}) { + for { + select { + case <-stopCh: + return + default: + } + func() { + f() + }() + } +} + +type Queue_kubernetes30872 interface { + HasSynced() + Pop(PopProcessFunc_kubernetes30872) +} + +type Config_kubernetes30872 struct { + Queue Queue_kubernetes30872 + Process ProcessFunc_kubernetes30872 +} + +type Controller_kubernetes30872 struct { + config Config_kubernetes30872 +} + +func (c *Controller_kubernetes30872) Run(stopCh <-chan struct{}) { + Util_kubernetes30872(c.processLoop, stopCh) +} + +func (c *Controller_kubernetes30872) HasSynced() { + c.config.Queue.HasSynced() +} + +func (c *Controller_kubernetes30872) processLoop() { + c.config.Queue.Pop(PopProcessFunc_kubernetes30872(c.config.Process)) +} + +type ControllerInterface_kubernetes30872 interface { + Run(<-chan struct{}) + HasSynced() +} + +type ResourceEventHandler_kubernetes30872 interface { + OnAdd() +} + +type ResourceEventHandlerFuncs_kubernetes30872 struct { + AddFunc func() +} + +func (r ResourceEventHandlerFuncs_kubernetes30872) OnAdd() { + if r.AddFunc != nil { + r.AddFunc() + } +} + +type informer_kubernetes30872 struct { + controller ControllerInterface_kubernetes30872 + + stopChan chan struct{} +} + +type federatedInformerImpl_kubernetes30872 struct { + sync.Mutex + clusterInformer informer_kubernetes30872 +} + +func (f *federatedInformerImpl_kubernetes30872) ClustersSynced() { + f.Lock() // L1 + defer f.Unlock() + f.clusterInformer.controller.HasSynced() +} + +func (f *federatedInformerImpl_kubernetes30872) addCluster() { + f.Lock() // L1 + defer f.Unlock() +} + +func (f *federatedInformerImpl_kubernetes30872) Start() { + f.Lock() // L1 + defer f.Unlock() + + f.clusterInformer.stopChan = make(chan struct{}) + go f.clusterInformer.controller.Run(f.clusterInformer.stopChan) // G2 + runtime.Gosched() +} + +func (f *federatedInformerImpl_kubernetes30872) Stop() { + f.Lock() // L1 + defer f.Unlock() + close(f.clusterInformer.stopChan) +} + +type DelayingDeliverer_kubernetes30872 struct{} + +func (d *DelayingDeliverer_kubernetes30872) StartWithHandler(handler func()) { + go func() { // G4 + handler() + }() +} + +type FederationView_kubernetes30872 interface { + ClustersSynced() +} + +type FederatedInformer_kubernetes30872 interface { + FederationView_kubernetes30872 + Start() + Stop() +} + +type NamespaceController_kubernetes30872 struct { + namespaceDeliverer *DelayingDeliverer_kubernetes30872 + namespaceFederatedInformer FederatedInformer_kubernetes30872 +} + +func (nc *NamespaceController_kubernetes30872) isSynced() { + nc.namespaceFederatedInformer.ClustersSynced() +} + +func (nc *NamespaceController_kubernetes30872) reconcileNamespace() { + nc.isSynced() +} + +func (nc *NamespaceController_kubernetes30872) Run(stopChan <-chan struct{}) { + nc.namespaceFederatedInformer.Start() + go func() { // G3 + <-stopChan + nc.namespaceFederatedInformer.Stop() + }() + nc.namespaceDeliverer.StartWithHandler(func() { + nc.reconcileNamespace() + }) +} + +type DeltaFIFO_kubernetes30872 struct { + lock sync.RWMutex +} + +func (f *DeltaFIFO_kubernetes30872) HasSynced() { + f.lock.Lock() // L2 + defer f.lock.Unlock() +} + +func (f *DeltaFIFO_kubernetes30872) Pop(process PopProcessFunc_kubernetes30872) { + f.lock.Lock() // L2 + defer f.lock.Unlock() + process() +} + +func NewFederatedInformer_kubernetes30872() FederatedInformer_kubernetes30872 { + federatedInformer := &federatedInformerImpl_kubernetes30872{} + federatedInformer.clusterInformer.controller = NewInformer_kubernetes30872( + ResourceEventHandlerFuncs_kubernetes30872{ + AddFunc: func() { + federatedInformer.addCluster() + }, + }) + return federatedInformer +} + +func NewInformer_kubernetes30872(h ResourceEventHandler_kubernetes30872) *Controller_kubernetes30872 { + fifo := &DeltaFIFO_kubernetes30872{} + cfg := &Config_kubernetes30872{ + Queue: fifo, + Process: func() { + h.OnAdd() + }, + } + return &Controller_kubernetes30872{config: *cfg} +} + +func NewNamespaceController_kubernetes30872() *NamespaceController_kubernetes30872 { + nc := &NamespaceController_kubernetes30872{} + nc.namespaceDeliverer = &DelayingDeliverer_kubernetes30872{} + nc.namespaceFederatedInformer = NewFederatedInformer_kubernetes30872() + return nc +} + +func Kubernetes30872() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 100; i++ { + go func() { // G1 + namespaceController := NewNamespaceController_kubernetes30872() + stop := make(chan struct{}) + namespaceController.Run(stop) + close(stop) + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes38669.go b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes38669.go new file mode 100644 index 00000000000000..27020d580493bf --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes38669.go @@ -0,0 +1,83 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" +) + +func init() { + register("Kubernetes38669", Kubernetes38669) +} + +type Event_kubernetes38669 int +type watchCacheEvent_kubernetes38669 int + +type cacheWatcher_kubernetes38669 struct { + sync.Mutex + input chan watchCacheEvent_kubernetes38669 + result chan Event_kubernetes38669 + stopped bool +} + +func (c *cacheWatcher_kubernetes38669) process(initEvents []watchCacheEvent_kubernetes38669) { + for _, event := range initEvents { + c.sendWatchCacheEvent(&event) + } + defer close(c.result) + defer c.Stop() + for { + _, ok := <-c.input + if !ok { + return + } + } +} + +func (c *cacheWatcher_kubernetes38669) sendWatchCacheEvent(event *watchCacheEvent_kubernetes38669) { + c.result <- Event_kubernetes38669(*event) +} + +func (c *cacheWatcher_kubernetes38669) Stop() { + c.stop() +} + +func (c *cacheWatcher_kubernetes38669) stop() { + c.Lock() + defer c.Unlock() + if !c.stopped { + c.stopped = true + close(c.input) + } +} + +func newCacheWatcher_kubernetes38669(chanSize int, initEvents []watchCacheEvent_kubernetes38669) *cacheWatcher_kubernetes38669 { + watcher := &cacheWatcher_kubernetes38669{ + input: make(chan watchCacheEvent_kubernetes38669, chanSize), + result: make(chan Event_kubernetes38669, chanSize), + stopped: false, + } + go watcher.process(initEvents) + return watcher +} + +func Kubernetes38669() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + go func() { + initEvents := []watchCacheEvent_kubernetes38669{1, 2} + w := newCacheWatcher_kubernetes38669(0, initEvents) + w.Stop() + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes5316.go b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes5316.go new file mode 100644 index 00000000000000..fd51484a0f773b --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes5316.go @@ -0,0 +1,68 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: kubernetes + * Issue or PR : https://github.com/kubernetes/kubernetes/pull/5316 + * Buggy version: c868b0bbf09128960bc7c4ada1a77347a464d876 + * fix commit-id: cc3a433a7abc89d2f766d4c87eaae9448e3dc091 + * Flaky: 100/100 + */ + +package main + +import ( + "errors" + "math/rand" + "os" + "runtime" + "runtime/pprof" + "time" +) + +func init() { + register("Kubernetes5316", Kubernetes5316) +} + +func finishRequest_kubernetes5316(timeout time.Duration, fn func() error) { + ch := make(chan bool) + errCh := make(chan error) + go func() { // G2 + if err := fn(); err != nil { + errCh <- err + } else { + ch <- true + } + }() + + select { + case <-ch: + case <-errCh: + case <-time.After(timeout): + } +} + +func Kubernetes5316() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Wait a bit because the child goroutine relies on timed operations. + time.Sleep(100 * time.Millisecond) + + // Yield several times to allow the child goroutine to run + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + go func() { + fn := func() error { + time.Sleep(2 * time.Millisecond) + if rand.Intn(10) > 5 { + return errors.New("Error") + } + return nil + } + go finishRequest_kubernetes5316(time.Microsecond, fn) // G1 + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes58107.go b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes58107.go new file mode 100644 index 00000000000000..0ca707e981693a --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes58107.go @@ -0,0 +1,108 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: kubernetes + * Tag: Reproduce misbehavior + * Issue or PR : https://github.com/kubernetes/kubernetes/pull/58107 + * Buggy version: 2f17d782eb2772d6401da7ddced9ac90656a7a79 + * fix commit-id: 010a127314a935d8d038f8dd4559fc5b249813e4 + * Flaky: 53/100 + */ + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Kubernetes58107", Kubernetes58107) +} + +type RateLimitingInterface_kubernetes58107 interface { + Get() + Put() +} + +type Type_kubernetes58107 struct { + cond *sync.Cond +} + +func (q *Type_kubernetes58107) Get() { + q.cond.L.Lock() + defer q.cond.L.Unlock() + q.cond.Wait() +} + +func (q *Type_kubernetes58107) Put() { + q.cond.Signal() +} + +type ResourceQuotaController_kubernetes58107 struct { + workerLock sync.RWMutex + queue RateLimitingInterface_kubernetes58107 + missingUsageQueue RateLimitingInterface_kubernetes58107 +} + +func (rq *ResourceQuotaController_kubernetes58107) worker(queue RateLimitingInterface_kubernetes58107, _ string) { + workFunc := func() bool { + rq.workerLock.RLock() + defer rq.workerLock.RUnlock() + queue.Get() + return true + } + for { + if quit := workFunc(); quit { + return + } + } +} + +func (rq *ResourceQuotaController_kubernetes58107) Run() { + go rq.worker(rq.queue, "G1") // G3 + go rq.worker(rq.missingUsageQueue, "G2") // G4 +} + +func (rq *ResourceQuotaController_kubernetes58107) Sync() { + for i := 0; i < 100000; i++ { + rq.workerLock.Lock() + runtime.Gosched() + rq.workerLock.Unlock() + } +} + +func (rq *ResourceQuotaController_kubernetes58107) HelperSignals() { + for i := 0; i < 100000; i++ { + rq.queue.Put() + rq.missingUsageQueue.Put() + } +} + +func startResourceQuotaController_kubernetes58107() { + resourceQuotaController := &ResourceQuotaController_kubernetes58107{ + queue: &Type_kubernetes58107{sync.NewCond(&sync.Mutex{})}, + missingUsageQueue: &Type_kubernetes58107{sync.NewCond(&sync.Mutex{})}, + } + + go resourceQuotaController.Run() // G2 + go resourceQuotaController.Sync() // G5 + resourceQuotaController.HelperSignals() +} + +func Kubernetes58107() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(1000 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 1000; i++ { + go startResourceQuotaController_kubernetes58107() // G1 + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes62464.go b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes62464.go new file mode 100644 index 00000000000000..0d07ebc4a9c451 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes62464.go @@ -0,0 +1,120 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: kubernetes + * Issue or PR : https://github.com/kubernetes/kubernetes/pull/62464 + * Buggy version: a048ca888ad27367b1a7b7377c67658920adbf5d + * fix commit-id: c1b19fce903675b82e9fdd1befcc5f5d658bfe78 + * Flaky: 8/100 + */ + +package main + +import ( + "math/rand" + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Kubernetes62464", Kubernetes62464) +} + +type State_kubernetes62464 interface { + GetCPUSetOrDefault() + GetCPUSet() bool + GetDefaultCPUSet() + SetDefaultCPUSet() +} + +type stateMemory_kubernetes62464 struct { + sync.RWMutex +} + +func (s *stateMemory_kubernetes62464) GetCPUSetOrDefault() { + s.RLock() + defer s.RUnlock() + if ok := s.GetCPUSet(); ok { + return + } + s.GetDefaultCPUSet() +} + +func (s *stateMemory_kubernetes62464) GetCPUSet() bool { + runtime.Gosched() + s.RLock() + defer s.RUnlock() + + if rand.Intn(10) > 5 { + return true + } + return false +} + +func (s *stateMemory_kubernetes62464) GetDefaultCPUSet() { + s.RLock() + defer s.RUnlock() +} + +func (s *stateMemory_kubernetes62464) SetDefaultCPUSet() { + s.Lock() + runtime.Gosched() + defer s.Unlock() +} + +type staticPolicy_kubernetes62464 struct{} + +func (p *staticPolicy_kubernetes62464) RemoveContainer(s State_kubernetes62464) { + s.GetDefaultCPUSet() + s.SetDefaultCPUSet() +} + +type manager_kubernetes62464 struct { + state *stateMemory_kubernetes62464 +} + +func (m *manager_kubernetes62464) reconcileState() { + m.state.GetCPUSetOrDefault() +} + +func NewPolicyAndManager_kubernetes62464() (*staticPolicy_kubernetes62464, *manager_kubernetes62464) { + s := &stateMemory_kubernetes62464{} + m := &manager_kubernetes62464{s} + p := &staticPolicy_kubernetes62464{} + return p, m +} + +/// +/// G1 G2 +/// m.reconcileState() +/// m.state.GetCPUSetOrDefault() +/// s.RLock() +/// s.GetCPUSet() +/// p.RemoveContainer() +/// s.GetDefaultCPUSet() +/// s.SetDefaultCPUSet() +/// s.Lock() +/// s.RLock() +/// ---------------------G1,G2 deadlock--------------------- +/// + +func Kubernetes62464() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 1000; i++ { + go func() { + p, m := NewPolicyAndManager_kubernetes62464() + go m.reconcileState() + go p.RemoveContainer(m.state) + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes6632.go b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes6632.go new file mode 100644 index 00000000000000..a3af3b24ae81ac --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes6632.go @@ -0,0 +1,86 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: kubernetes + * Issue or PR : https://github.com/kubernetes/kubernetes/pull/6632 + * Buggy version: e597b41d939573502c8dda1dde7bf3439325fb5d + * fix commit-id: 82afb7ab1fe12cf2efceede2322d082eaf5d5adc + * Flaky: 4/100 + */ +package main + +import ( + "os" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Kubernetes6632", Kubernetes6632) +} + +type Connection_kubernetes6632 struct { + closeChan chan bool +} + +type idleAwareFramer_kubernetes6632 struct { + resetChan chan bool + writeLock sync.Mutex + conn *Connection_kubernetes6632 +} + +func (i *idleAwareFramer_kubernetes6632) monitor() { + var resetChan = i.resetChan +Loop: + for { + select { + case <-i.conn.closeChan: + i.writeLock.Lock() + close(resetChan) + i.resetChan = nil + i.writeLock.Unlock() + break Loop + } + } +} + +func (i *idleAwareFramer_kubernetes6632) WriteFrame() { + i.writeLock.Lock() + defer i.writeLock.Unlock() + if i.resetChan == nil { + return + } + i.resetChan <- true +} + +func NewIdleAwareFramer_kubernetes6632() *idleAwareFramer_kubernetes6632 { + return &idleAwareFramer_kubernetes6632{ + resetChan: make(chan bool), + conn: &Connection_kubernetes6632{ + closeChan: make(chan bool), + }, + } +} + +func Kubernetes6632() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 100; i++ { + go func() { + i := NewIdleAwareFramer_kubernetes6632() + + go func() { // helper goroutine + i.conn.closeChan <- true + }() + go i.monitor() // G1 + go i.WriteFrame() // G2 + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes70277.go b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes70277.go new file mode 100644 index 00000000000000..ae02ec83040b99 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/kubernetes70277.go @@ -0,0 +1,97 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime/pprof" + "time" +) + +func init() { + register("Kubernetes70277", Kubernetes70277) +} + +type WaitFunc_kubernetes70277 func(done <-chan struct{}) <-chan struct{} + +type ConditionFunc_kubernetes70277 func() (done bool, err error) + +func WaitFor_kubernetes70277(wait WaitFunc_kubernetes70277, fn ConditionFunc_kubernetes70277, done <-chan struct{}) error { + c := wait(done) + for { + _, open := <-c + ok, err := fn() + if err != nil { + return err + } + if ok { + return nil + } + if !open { + break + } + } + return nil +} + +func poller_kubernetes70277(interval, timeout time.Duration) WaitFunc_kubernetes70277 { + return WaitFunc_kubernetes70277(func(done <-chan struct{}) <-chan struct{} { + ch := make(chan struct{}) + go func() { + defer close(ch) + + tick := time.NewTicker(interval) + defer tick.Stop() + + var after <-chan time.Time + if timeout != 0 { + timer := time.NewTimer(timeout) + after = timer.C + defer timer.Stop() + } + for { + select { + case <-tick.C: + select { + case ch <- struct{}{}: + default: + } + case <-after: + return + case <-done: + return + } + } + }() + + return ch + }) +} + +func Kubernetes70277() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + for i := 0; i < 1000; i++ { + go func() { + stopCh := make(chan struct{}) + defer close(stopCh) + waitFunc := poller_kubernetes70277(time.Millisecond, 80*time.Millisecond) + var doneCh <-chan struct{} + + WaitFor_kubernetes70277(func(done <-chan struct{}) <-chan struct{} { + doneCh = done + return waitFunc(done) + }, func() (bool, error) { + time.Sleep(10 * time.Millisecond) + return true, nil + }, stopCh) + + <-doneCh // block here + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/main.go b/src/runtime/testdata/testgoroutineleakprofile/goker/main.go new file mode 100644 index 00000000000000..5787c1e2b2bb90 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/main.go @@ -0,0 +1,39 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "os" + +// The number of times the main (profiling) goroutine should yield +// in order to allow the leaking goroutines to get stuck. +const yieldCount = 10 + +var cmds = map[string]func(){} + +func register(name string, f func()) { + if cmds[name] != nil { + panic("duplicate registration: " + name) + } + cmds[name] = f +} + +func registerInit(name string, f func()) { + if len(os.Args) >= 2 && os.Args[1] == name { + f() + } +} + +func main() { + if len(os.Args) < 2 { + println("usage: " + os.Args[0] + " name-of-test") + return + } + f := cmds[os.Args[1]] + if f == nil { + println("unknown function: " + os.Args[1]) + return + } + f() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/moby17176.go b/src/runtime/testdata/testgoroutineleakprofile/goker/moby17176.go new file mode 100644 index 00000000000000..884d077550f6b8 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/moby17176.go @@ -0,0 +1,68 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: moby + * Issue or PR : https://github.com/moby/moby/pull/17176 + * Buggy version: d295dc66521e2734390473ec1f1da8a73ad3288a + * fix commit-id: 2f16895ee94848e2d8ad72bc01968b4c88d84cb8 + * Flaky: 100/100 + */ +package main + +import ( + "errors" + "os" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Moby17176", Moby17176) +} + +type DeviceSet_moby17176 struct { + sync.Mutex + nrDeletedDevices int +} + +func (devices *DeviceSet_moby17176) cleanupDeletedDevices() error { + devices.Lock() + if devices.nrDeletedDevices == 0 { + /// Missing devices.Unlock() + return nil + } + devices.Unlock() + return errors.New("Error") +} + +func testDevmapperLockReleasedDeviceDeletion_moby17176() { + ds := &DeviceSet_moby17176{ + nrDeletedDevices: 0, + } + ds.cleanupDeletedDevices() + doneChan := make(chan bool) + go func() { + ds.Lock() + defer ds.Unlock() + doneChan <- true + }() + + select { + case <-time.After(time.Millisecond): + case <-doneChan: + } +} +func Moby17176() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 100; i++ { + go testDevmapperLockReleasedDeviceDeletion_moby17176() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/moby21233.go b/src/runtime/testdata/testgoroutineleakprofile/goker/moby21233.go new file mode 100644 index 00000000000000..36017a9488cc06 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/moby21233.go @@ -0,0 +1,146 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: moby + * Issue or PR : https://github.com/moby/moby/pull/21233 + * Buggy version: cc12d2bfaae135e63b1f962ad80e6943dd995337 + * fix commit-id: 2f4aa9658408ac72a598363c6e22eadf93dbb8a7 + * Flaky:100/100 + */ +package main + +import ( + "math/rand" + "os" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Moby21233", Moby21233) +} + +type Progress_moby21233 struct{} + +type Output_moby21233 interface { + WriteProgress(Progress_moby21233) error +} + +type chanOutput_moby21233 chan<- Progress_moby21233 + +type TransferManager_moby21233 struct { + mu sync.Mutex +} + +type Transfer_moby21233 struct { + mu sync.Mutex +} + +type Watcher_moby21233 struct { + signalChan chan struct{} + releaseChan chan struct{} + running chan struct{} +} + +func ChanOutput_moby21233(progressChan chan<- Progress_moby21233) Output_moby21233 { + return chanOutput_moby21233(progressChan) +} +func (out chanOutput_moby21233) WriteProgress(p Progress_moby21233) error { + out <- p + return nil +} +func NewTransferManager_moby21233() *TransferManager_moby21233 { + return &TransferManager_moby21233{} +} +func NewTransfer_moby21233() *Transfer_moby21233 { + return &Transfer_moby21233{} +} +func (t *Transfer_moby21233) Release(watcher *Watcher_moby21233) { + t.mu.Lock() + t.mu.Unlock() + close(watcher.releaseChan) + <-watcher.running +} +func (t *Transfer_moby21233) Watch(progressOutput Output_moby21233) *Watcher_moby21233 { + t.mu.Lock() + defer t.mu.Unlock() + lastProgress := Progress_moby21233{} + w := &Watcher_moby21233{ + releaseChan: make(chan struct{}), + signalChan: make(chan struct{}), + running: make(chan struct{}), + } + go func() { // G2 + defer func() { + close(w.running) + }() + done := false + for { + t.mu.Lock() + t.mu.Unlock() + if rand.Int31n(2) >= 1 { + progressOutput.WriteProgress(lastProgress) + } + if done { + return + } + select { + case <-w.signalChan: + case <-w.releaseChan: + done = true + } + } + }() + return w +} +func (tm *TransferManager_moby21233) Transfer(progressOutput Output_moby21233) (*Transfer_moby21233, *Watcher_moby21233) { + tm.mu.Lock() + defer tm.mu.Unlock() + t := NewTransfer_moby21233() + return t, t.Watch(progressOutput) +} + +func testTransfer_moby21233() { // G1 + tm := NewTransferManager_moby21233() + progressChan := make(chan Progress_moby21233) + progressDone := make(chan struct{}) + go func() { // G3 + time.Sleep(1 * time.Millisecond) + for p := range progressChan { /// Chan consumer + if rand.Int31n(2) >= 1 { + return + } + _ = p + } + close(progressDone) + }() + time.Sleep(1 * time.Millisecond) + ids := []string{"id1", "id2", "id3"} + xrefs := make([]*Transfer_moby21233, len(ids)) + watchers := make([]*Watcher_moby21233, len(ids)) + for i := range ids { + xrefs[i], watchers[i] = tm.Transfer(ChanOutput_moby21233(progressChan)) /// Chan producer + time.Sleep(2 * time.Millisecond) + } + + for i := range xrefs { + xrefs[i].Release(watchers[i]) + } + + close(progressChan) + <-progressDone +} + +func Moby21233() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + for i := 0; i < 100; i++ { + go testTransfer_moby21233() // G1 + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/moby25384.go b/src/runtime/testdata/testgoroutineleakprofile/goker/moby25384.go new file mode 100644 index 00000000000000..d653731f6caea8 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/moby25384.go @@ -0,0 +1,59 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: moby + * Issue or PR : https://github.com/moby/moby/pull/25384 + * Buggy version: 58befe3081726ef74ea09198cd9488fb42c51f51 + * fix commit-id: 42360d164b9f25fb4b150ef066fcf57fa39559a7 + * Flaky: 100/100 + */ +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" +) + +func init() { + register("Moby25348", Moby25348) +} + +type plugin_moby25348 struct{} + +type Manager_moby25348 struct { + plugins []*plugin_moby25348 +} + +func (pm *Manager_moby25348) init() { + var group sync.WaitGroup + group.Add(len(pm.plugins)) + for _, p := range pm.plugins { + go func(p *plugin_moby25348) { + defer group.Done() + }(p) + group.Wait() // Block here + } +} + +func Moby25348() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + go func() { + p1 := &plugin_moby25348{} + p2 := &plugin_moby25348{} + pm := &Manager_moby25348{ + plugins: []*plugin_moby25348{p1, p2}, + } + go pm.init() + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/moby27782.go b/src/runtime/testdata/testgoroutineleakprofile/goker/moby27782.go new file mode 100644 index 00000000000000..9b53d9035ca1e1 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/moby27782.go @@ -0,0 +1,249 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: moby + * Issue or PR : https://github.com/moby/moby/pull/27782 + * Buggy version: 18768fdc2e76ec6c600c8ab57d2d487ee7877794 + * fix commit-id: a69a59ffc7e3d028a72d1195c2c1535f447eaa84 + * Flaky: 2/100 + */ +package main + +import ( + "errors" + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Moby27782", Moby27782) +} + +type Event_moby27782 struct { + Op Op_moby27782 +} + +type Op_moby27782 uint32 + +const ( + Create_moby27782 Op_moby27782 = 1 << iota + Write_moby27782 + Remove_moby27782 + Rename_moby27782 + Chmod_moby27782 +) + +func newEvent(op Op_moby27782) Event_moby27782 { + return Event_moby27782{op} +} + +func (e *Event_moby27782) ignoreLinux(w *Watcher_moby27782) bool { + if e.Op != Write_moby27782 { + w.mu.Lock() + defer w.mu.Unlock() + w.cv.Broadcast() + return true + } + runtime.Gosched() + return false +} + +type Watcher_moby27782 struct { + Events chan Event_moby27782 + mu sync.Mutex // L1 + cv *sync.Cond // C1 + done chan struct{} +} + +func NewWatcher_moby27782() *Watcher_moby27782 { + w := &Watcher_moby27782{ + Events: make(chan Event_moby27782), + done: make(chan struct{}), + } + w.cv = sync.NewCond(&w.mu) + go w.readEvents() // G3 + return w +} + +func (w *Watcher_moby27782) readEvents() { + defer close(w.Events) + for { + if w.isClosed() { + return + } + event := newEvent(Write_moby27782) // MODIFY event + if !event.ignoreLinux(w) { + runtime.Gosched() + select { + case w.Events <- event: + case <-w.done: + return + } + } + } +} + +func (w *Watcher_moby27782) isClosed() bool { + select { + case <-w.done: + return true + default: + return false + } +} + +func (w *Watcher_moby27782) Close() { + if w.isClosed() { + return + } + close(w.done) +} + +func (w *Watcher_moby27782) Remove() { + w.mu.Lock() + defer w.mu.Unlock() + exists := true + for exists { + w.cv.Wait() + runtime.Gosched() + } +} + +type FileWatcher_moby27782 interface { + Events() <-chan Event_moby27782 + Remove() + Close() +} + +func New_moby27782() FileWatcher_moby27782 { + return NewEventWatcher_moby27782() +} + +func NewEventWatcher_moby27782() FileWatcher_moby27782 { + return &fsNotifyWatcher_moby27782{NewWatcher_moby27782()} +} + +type fsNotifyWatcher_moby27782 struct { + *Watcher_moby27782 +} + +func (w *fsNotifyWatcher_moby27782) Events() <-chan Event_moby27782 { + return w.Watcher_moby27782.Events +} + +func watchFile_moby27782() FileWatcher_moby27782 { + fileWatcher := New_moby27782() + return fileWatcher +} + +type LogWatcher_moby27782 struct { + closeOnce sync.Once + closeNotifier chan struct{} +} + +func (w *LogWatcher_moby27782) Close() { + w.closeOnce.Do(func() { + close(w.closeNotifier) + }) +} + +func (w *LogWatcher_moby27782) WatchClose() <-chan struct{} { + return w.closeNotifier +} + +func NewLogWatcher_moby27782() *LogWatcher_moby27782 { + return &LogWatcher_moby27782{ + closeNotifier: make(chan struct{}), + } +} + +func followLogs_moby27782(logWatcher *LogWatcher_moby27782) { + fileWatcher := watchFile_moby27782() + defer func() { + fileWatcher.Close() + }() + waitRead := func() { + runtime.Gosched() + select { + case <-fileWatcher.Events(): + case <-logWatcher.WatchClose(): + fileWatcher.Remove() + return + } + } + handleDecodeErr := func() { + waitRead() + } + handleDecodeErr() +} + +type Container_moby27782 struct { + LogDriver *JSONFileLogger_moby27782 +} + +func (container *Container_moby27782) InitializeStdio() { + if err := container.startLogging(); err != nil { + container.Reset() + } +} + +func (container *Container_moby27782) startLogging() error { + l := &JSONFileLogger_moby27782{ + readers: make(map[*LogWatcher_moby27782]struct{}), + } + container.LogDriver = l + l.ReadLogs() + return errors.New("Some error") +} + +func (container *Container_moby27782) Reset() { + if container.LogDriver != nil { + container.LogDriver.Close() + } +} + +type JSONFileLogger_moby27782 struct { + mu sync.Mutex + readers map[*LogWatcher_moby27782]struct{} +} + +func (l *JSONFileLogger_moby27782) ReadLogs() *LogWatcher_moby27782 { + logWatcher := NewLogWatcher_moby27782() + go l.readLogs(logWatcher) // G2 + return logWatcher +} + +func (l *JSONFileLogger_moby27782) readLogs(logWatcher *LogWatcher_moby27782) { + l.mu.Lock() + defer l.mu.Unlock() + + l.readers[logWatcher] = struct{}{} + followLogs_moby27782(logWatcher) +} + +func (l *JSONFileLogger_moby27782) Close() { + l.mu.Lock() + defer l.mu.Unlock() + + for r := range l.readers { + r.Close() + delete(l.readers, r) + } +} + +func Moby27782() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 10000; i++ { + go (&Container_moby27782{}).InitializeStdio() // G1 + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/moby28462.go b/src/runtime/testdata/testgoroutineleakprofile/goker/moby28462.go new file mode 100644 index 00000000000000..56467e0b56f7cc --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/moby28462.go @@ -0,0 +1,125 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: moby + * Issue or PR : https://github.com/moby/moby/pull/28462 + * Buggy version: b184bdabf7a01c4b802304ac64ac133743c484be + * fix commit-id: 89b123473774248fc3a0356dd3ce5b116cc69b29 + * Flaky: 69/100 + */ +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Moby28462", Moby28462) +} + +type State_moby28462 struct { + Health *Health_moby28462 +} + +type Container_moby28462 struct { + sync.Mutex + State *State_moby28462 +} + +func (ctr *Container_moby28462) start() { + go ctr.waitExit() +} +func (ctr *Container_moby28462) waitExit() { + +} + +type Store_moby28462 struct { + ctr *Container_moby28462 +} + +func (s *Store_moby28462) Get() *Container_moby28462 { + return s.ctr +} + +type Daemon_moby28462 struct { + containers Store_moby28462 +} + +func (d *Daemon_moby28462) StateChanged() { + c := d.containers.Get() + c.Lock() + d.updateHealthMonitorElseBranch(c) + defer c.Unlock() +} + +func (d *Daemon_moby28462) updateHealthMonitorIfBranch(c *Container_moby28462) { + h := c.State.Health + if stop := h.OpenMonitorChannel(); stop != nil { + go monitor_moby28462(c, stop) + } +} +func (d *Daemon_moby28462) updateHealthMonitorElseBranch(c *Container_moby28462) { + h := c.State.Health + h.CloseMonitorChannel() +} + +type Health_moby28462 struct { + stop chan struct{} +} + +func (s *Health_moby28462) OpenMonitorChannel() chan struct{} { + return s.stop +} + +func (s *Health_moby28462) CloseMonitorChannel() { + if s.stop != nil { + s.stop <- struct{}{} + } +} + +func monitor_moby28462(c *Container_moby28462, stop chan struct{}) { + for { + select { + case <-stop: + return + default: + handleProbeResult_moby28462(c) + } + } +} + +func handleProbeResult_moby28462(c *Container_moby28462) { + runtime.Gosched() + c.Lock() + defer c.Unlock() +} + +func NewDaemonAndContainer_moby28462() (*Daemon_moby28462, *Container_moby28462) { + c := &Container_moby28462{ + State: &State_moby28462{&Health_moby28462{make(chan struct{})}}, + } + d := &Daemon_moby28462{Store_moby28462{c}} + return d, c +} + +func Moby28462() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 100; i++ { + go func() { + d, c := NewDaemonAndContainer_moby28462() + go monitor_moby28462(c, c.State.Health.OpenMonitorChannel()) // G1 + go d.StateChanged() // G2 + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/moby30408.go b/src/runtime/testdata/testgoroutineleakprofile/goker/moby30408.go new file mode 100644 index 00000000000000..561e2faf576ab0 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/moby30408.go @@ -0,0 +1,67 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "errors" + "os" + "runtime" + "runtime/pprof" + "sync" +) + +func init() { + register("Moby30408", Moby30408) +} + +type Manifest_moby30408 struct { + Implements []string +} + +type Plugin_moby30408 struct { + activateWait *sync.Cond + activateErr error + Manifest *Manifest_moby30408 +} + +func (p *Plugin_moby30408) waitActive() error { + p.activateWait.L.Lock() + for !p.activated() { + p.activateWait.Wait() + } + p.activateWait.L.Unlock() + return p.activateErr +} + +func (p *Plugin_moby30408) activated() bool { + return p.Manifest != nil +} + +func testActive_moby30408(p *Plugin_moby30408) { + done := make(chan struct{}) + go func() { // G2 + p.waitActive() + close(done) + }() + <-done +} + +func Moby30408() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + + go func() { // G1 + p := &Plugin_moby30408{activateWait: sync.NewCond(&sync.Mutex{})} + p.activateErr = errors.New("some junk happened") + + testActive_moby30408(p) + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/moby33781.go b/src/runtime/testdata/testgoroutineleakprofile/goker/moby33781.go new file mode 100644 index 00000000000000..4be50546e9be2c --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/moby33781.go @@ -0,0 +1,71 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: moby + * Issue or PR : https://github.com/moby/moby/pull/33781 + * Buggy version: 33fd3817b0f5ca4b87f0a75c2bd583b4425d392b + * fix commit-id: 67297ba0051d39be544009ba76abea14bc0be8a4 + * Flaky: 25/100 + */ + +package main + +import ( + "context" + "os" + "runtime/pprof" + "time" +) + +func init() { + register("Moby33781", Moby33781) +} + +func monitor_moby33781(stop chan bool) { + probeInterval := time.Millisecond + probeTimeout := time.Millisecond + for { + select { + case <-stop: + return + case <-time.After(probeInterval): + results := make(chan bool) + ctx, cancelProbe := context.WithTimeout(context.Background(), probeTimeout) + go func() { // G3 + results <- true + close(results) + }() + select { + case <-stop: + // results should be drained here + cancelProbe() + return + case <-results: + cancelProbe() + case <-ctx.Done(): + cancelProbe() + <-results + } + } + } +} + +func Moby33781() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + for i := 0; i < 100; i++ { + go func(i int) { + stop := make(chan bool) + go monitor_moby33781(stop) // G1 + go func() { // G2 + time.Sleep(time.Duration(i) * time.Millisecond) + stop <- true + }() + }(i) + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/moby36114.go b/src/runtime/testdata/testgoroutineleakprofile/goker/moby36114.go new file mode 100644 index 00000000000000..577c81651a0668 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/moby36114.go @@ -0,0 +1,53 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: moby + * Issue or PR : https://github.com/moby/moby/pull/36114 + * Buggy version: 6d4d3c52ae7c3f910bfc7552a2a673a8338e5b9f + * fix commit-id: a44fcd3d27c06aaa60d8d1cbce169f0d982e74b1 + * Flaky: 100/100 + */ +package main + +import ( + "os" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Moby36114", Moby36114) +} + +type serviceVM_moby36114 struct { + sync.Mutex +} + +func (svm *serviceVM_moby36114) hotAddVHDsAtStart() { + svm.Lock() + defer svm.Unlock() + svm.hotRemoveVHDsAtStart() +} + +func (svm *serviceVM_moby36114) hotRemoveVHDsAtStart() { + svm.Lock() + defer svm.Unlock() +} + +func Moby36114() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 100; i++ { + go func() { + s := &serviceVM_moby36114{} + go s.hotAddVHDsAtStart() + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/moby4951.go b/src/runtime/testdata/testgoroutineleakprofile/goker/moby4951.go new file mode 100644 index 00000000000000..7f0497648cf606 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/moby4951.go @@ -0,0 +1,101 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: moby + * Issue or PR : https://github.com/moby/moby/pull/4951 + * Buggy version: 81f148be566ab2b17810ad4be61a5d8beac8330f + * fix commit-id: 2ffef1b7eb618162673c6ffabccb9ca57c7dfce3 + * Flaky: 100/100 + */ +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Moby4951", Moby4951) +} + +type DeviceSet_moby4951 struct { + sync.Mutex + infos map[string]*DevInfo_moby4951 + nrDeletedDevices int +} + +func (devices *DeviceSet_moby4951) DeleteDevice(hash string) { + devices.Lock() + defer devices.Unlock() + + info := devices.lookupDevice(hash) + + info.lock.Lock() + runtime.Gosched() + defer info.lock.Unlock() + + devices.deleteDevice(info) +} + +func (devices *DeviceSet_moby4951) lookupDevice(hash string) *DevInfo_moby4951 { + existing, ok := devices.infos[hash] + if !ok { + return nil + } + return existing +} + +func (devices *DeviceSet_moby4951) deleteDevice(info *DevInfo_moby4951) { + devices.removeDeviceAndWait(info.Name()) +} + +func (devices *DeviceSet_moby4951) removeDeviceAndWait(devname string) { + /// remove devices by devname + devices.Unlock() + time.Sleep(300 * time.Nanosecond) + devices.Lock() +} + +type DevInfo_moby4951 struct { + lock sync.Mutex + name string +} + +func (info *DevInfo_moby4951) Name() string { + return info.name +} + +func NewDeviceSet_moby4951() *DeviceSet_moby4951 { + devices := &DeviceSet_moby4951{ + infos: make(map[string]*DevInfo_moby4951), + } + info1 := &DevInfo_moby4951{ + name: "info1", + } + info2 := &DevInfo_moby4951{ + name: "info2", + } + devices.infos[info1.name] = info1 + devices.infos[info2.name] = info2 + return devices +} + +func Moby4951() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + go func() { + ds := NewDeviceSet_moby4951() + /// Delete devices by the same info + go ds.DeleteDevice("info1") + go ds.DeleteDevice("info1") + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/moby7559.go b/src/runtime/testdata/testgoroutineleakprofile/goker/moby7559.go new file mode 100644 index 00000000000000..212f65b1f3f2c6 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/moby7559.go @@ -0,0 +1,55 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +/* + * Project: moby + * Issue or PR : https://github.com/moby/moby/pull/7559 + * Buggy version: 64579f51fcb439c36377c0068ccc9a007b368b5a + * fix commit-id: 6cbb8e070d6c3a66bf48fbe5cbf689557eee23db + * Flaky: 100/100 + */ +package main + +import ( + "net" + "os" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Moby7559", Moby7559) +} + +type UDPProxy_moby7559 struct { + connTrackLock sync.Mutex +} + +func (proxy *UDPProxy_moby7559) Run() { + for i := 0; i < 2; i++ { + proxy.connTrackLock.Lock() + _, err := net.DialUDP("udp", nil, nil) + if err != nil { + continue + /// Missing unlock here + } + if i == 0 { + break + } + } + proxy.connTrackLock.Unlock() +} + +func Moby7559() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 20; i++ { + go (&UDPProxy_moby7559{}).Run() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/serving2137.go b/src/runtime/testdata/testgoroutineleakprofile/goker/serving2137.go new file mode 100644 index 00000000000000..49905315a01b4c --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/serving2137.go @@ -0,0 +1,122 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +func init() { + register("Serving2137", Serving2137) +} + +type token_serving2137 struct{} + +type request_serving2137 struct { + lock *sync.Mutex + accepted chan bool +} + +type Breaker_serving2137 struct { + pendingRequests chan token_serving2137 + activeRequests chan token_serving2137 +} + +func (b *Breaker_serving2137) Maybe(thunk func()) bool { + var t token_serving2137 + select { + default: + // Pending request queue is full. Report failure. + return false + case b.pendingRequests <- t: + // Pending request has capacity. + // Wait for capacity in the active queue. + b.activeRequests <- t + // Defer releasing capacity in the active and pending request queue. + defer func() { + <-b.activeRequests + runtime.Gosched() + <-b.pendingRequests + }() + // Do the thing. + thunk() + // Report success + return true + } +} + +func (b *Breaker_serving2137) concurrentRequest() request_serving2137 { + r := request_serving2137{lock: &sync.Mutex{}, accepted: make(chan bool, 1)} + r.lock.Lock() + var start sync.WaitGroup + start.Add(1) + go func() { // G2, G3 + start.Done() + runtime.Gosched() + ok := b.Maybe(func() { + // Will block on locked mutex. + r.lock.Lock() + runtime.Gosched() + r.lock.Unlock() + }) + r.accepted <- ok + }() + start.Wait() // Ensure that the go func has had a chance to execute. + return r +} + +// Perform n requests against the breaker, returning mutexes for each +// request which succeeded, and a slice of bools for all requests. +func (b *Breaker_serving2137) concurrentRequests(n int) []request_serving2137 { + requests := make([]request_serving2137, n) + for i := range requests { + requests[i] = b.concurrentRequest() + } + return requests +} + +func NewBreaker_serving2137(queueDepth, maxConcurrency int32) *Breaker_serving2137 { + return &Breaker_serving2137{ + pendingRequests: make(chan token_serving2137, queueDepth+maxConcurrency), + activeRequests: make(chan token_serving2137, maxConcurrency), + } +} + +func unlock_serving2137(req request_serving2137) { + req.lock.Unlock() + runtime.Gosched() + // Verify that function has completed + ok := <-req.accepted + runtime.Gosched() + // Requeue for next usage + req.accepted <- ok +} + +func unlockAll_serving2137(requests []request_serving2137) { + for _, lc := range requests { + unlock_serving2137(lc) + } +} + +func Serving2137() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(100 * time.Millisecond) + prof.WriteTo(os.Stdout, 2) + }() + + for i := 0; i < 1000; i++ { + go func() { + b := NewBreaker_serving2137(1, 1) + + locks := b.concurrentRequests(2) // G1 + unlockAll_serving2137(locks) + }() + } +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/syncthing4829.go b/src/runtime/testdata/testgoroutineleakprofile/goker/syncthing4829.go new file mode 100644 index 00000000000000..7967db7cfe083f --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/syncthing4829.go @@ -0,0 +1,87 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" +) + +func init() { + register("Syncthing4829", Syncthing4829) +} + +type Address_syncthing4829 int + +type Mapping_syncthing4829 struct { + mut sync.RWMutex // L2 + + extAddresses map[string]Address_syncthing4829 +} + +func (m *Mapping_syncthing4829) clearAddresses() { + m.mut.Lock() // L2 + var removed []Address_syncthing4829 + for id, addr := range m.extAddresses { + removed = append(removed, addr) + delete(m.extAddresses, id) + } + if len(removed) > 0 { + m.notify(nil, removed) + } + m.mut.Unlock() // L2 +} + +func (m *Mapping_syncthing4829) notify(added, remove []Address_syncthing4829) { + m.mut.RLock() // L2 + m.mut.RUnlock() // L2 +} + +type Service_syncthing4829 struct { + mut sync.RWMutex // L1 + + mappings []*Mapping_syncthing4829 +} + +func (s *Service_syncthing4829) NewMapping() *Mapping_syncthing4829 { + mapping := &Mapping_syncthing4829{ + extAddresses: make(map[string]Address_syncthing4829), + } + s.mut.Lock() // L1 + s.mappings = append(s.mappings, mapping) + s.mut.Unlock() // L1 + return mapping +} + +func (s *Service_syncthing4829) RemoveMapping(mapping *Mapping_syncthing4829) { + s.mut.Lock() // L1 + defer s.mut.Unlock() // L1 + for _, existing := range s.mappings { + if existing == mapping { + mapping.clearAddresses() + } + } +} + +func Syncthing4829() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + + go func() { // G1 + natSvc := &Service_syncthing4829{} + m := natSvc.NewMapping() + m.extAddresses["test"] = 0 + + natSvc.RemoveMapping(m) + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/goker/syncthing5795.go b/src/runtime/testdata/testgoroutineleakprofile/goker/syncthing5795.go new file mode 100644 index 00000000000000..e25494a688dd5c --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/goker/syncthing5795.go @@ -0,0 +1,103 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" +) + +func init() { + register("Syncthing5795", Syncthing5795) +} + +type message_syncthing5795 interface{} + +type ClusterConfig_syncthing5795 struct{} + +type Model_syncthing5795 interface { + ClusterConfig(message_syncthing5795) +} + +type TestModel_syncthing5795 struct { + ccFn func() +} + +func (t *TestModel_syncthing5795) ClusterConfig(msg message_syncthing5795) { + if t.ccFn != nil { + t.ccFn() + } +} + +type Connection_syncthing5795 interface { + Start() + Close() +} + +type rawConnection_syncthing5795 struct { + receiver Model_syncthing5795 + + inbox chan message_syncthing5795 + dispatcherLoopStopped chan struct{} + closed chan struct{} + closeOnce sync.Once +} + +func (c *rawConnection_syncthing5795) Start() { + go c.dispatcherLoop() // G2 +} + +func (c *rawConnection_syncthing5795) dispatcherLoop() { + defer close(c.dispatcherLoopStopped) + var msg message_syncthing5795 + for { + select { + case msg = <-c.inbox: + case <-c.closed: + return + } + switch msg := msg.(type) { + case *ClusterConfig_syncthing5795: + c.receiver.ClusterConfig(msg) + default: + return + } + } +} + +func (c *rawConnection_syncthing5795) Close() { + c.closeOnce.Do(func() { + close(c.closed) + <-c.dispatcherLoopStopped + }) +} + +func Syncthing5795() { + prof := pprof.Lookup("goroutineleak") + defer func() { + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) + }() + go func() { // G1 + m := &TestModel_syncthing5795{} + c := &rawConnection_syncthing5795{ + dispatcherLoopStopped: make(chan struct{}), + closed: make(chan struct{}), + inbox: make(chan message_syncthing5795), + receiver: m, + } + m.ccFn = c.Close + + c.Start() + c.inbox <- &ClusterConfig_syncthing5795{} + + <-c.dispatcherLoopStopped + }() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/main.go b/src/runtime/testdata/testgoroutineleakprofile/main.go new file mode 100644 index 00000000000000..5787c1e2b2bb90 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/main.go @@ -0,0 +1,39 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "os" + +// The number of times the main (profiling) goroutine should yield +// in order to allow the leaking goroutines to get stuck. +const yieldCount = 10 + +var cmds = map[string]func(){} + +func register(name string, f func()) { + if cmds[name] != nil { + panic("duplicate registration: " + name) + } + cmds[name] = f +} + +func registerInit(name string, f func()) { + if len(os.Args) >= 2 && os.Args[1] == name { + f() + } +} + +func main() { + if len(os.Args) < 2 { + println("usage: " + os.Args[0] + " name-of-test") + return + } + f := cmds[os.Args[1]] + if f == nil { + println("unknown function: " + os.Args[1]) + return + } + f() +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/simple.go b/src/runtime/testdata/testgoroutineleakprofile/simple.go new file mode 100644 index 00000000000000..b8172cd6df8aa7 --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/simple.go @@ -0,0 +1,253 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "sync" +) + +// This is a set of micro-tests with obvious goroutine leaks that +// ensures goroutine leak detection works. +// +// Tests in this file are not flaky iff. run with GOMAXPROCS=1. +// The main goroutine forcefully yields via `runtime.Gosched()` before +// running the profiler. This moves them to the back of the run queue, +// allowing the leaky goroutines to be scheduled beforehand and get stuck. + +func init() { + register("NilRecv", NilRecv) + register("NilSend", NilSend) + register("SelectNoCases", SelectNoCases) + register("ChanRecv", ChanRecv) + register("ChanSend", ChanSend) + register("Select", Select) + register("WaitGroup", WaitGroup) + register("MutexStack", MutexStack) + register("MutexHeap", MutexHeap) + register("RWMutexRLock", RWMutexRLock) + register("RWMutexLock", RWMutexLock) + register("Cond", Cond) + register("Mixed", Mixed) + register("NoLeakGlobal", NoLeakGlobal) +} + +func NilRecv() { + prof := pprof.Lookup("goroutineleak") + go func() { + var c chan int + <-c + panic("should not be reached") + }() + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) +} + +func NilSend() { + prof := pprof.Lookup("goroutineleak") + go func() { + var c chan int + c <- 0 + panic("should not be reached") + }() + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) +} + +func ChanRecv() { + prof := pprof.Lookup("goroutineleak") + go func() { + <-make(chan int) + panic("should not be reached") + }() + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) +} + +func SelectNoCases() { + prof := pprof.Lookup("goroutineleak") + go func() { + select {} + panic("should not be reached") + }() + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) +} + +func ChanSend() { + prof := pprof.Lookup("goroutineleak") + go func() { + make(chan int) <- 0 + panic("should not be reached") + }() + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) +} + +func Select() { + prof := pprof.Lookup("goroutineleak") + go func() { + select { + case make(chan int) <- 0: + case <-make(chan int): + } + panic("should not be reached") + }() + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) +} + +func WaitGroup() { + prof := pprof.Lookup("goroutineleak") + go func() { + var wg sync.WaitGroup + wg.Add(1) + wg.Wait() + panic("should not be reached") + }() + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) +} + +func MutexStack() { + prof := pprof.Lookup("goroutineleak") + for i := 0; i < 1000; i++ { + go func() { + var mu sync.Mutex + mu.Lock() + mu.Lock() + panic("should not be reached") + }() + } + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) +} + +func MutexHeap() { + prof := pprof.Lookup("goroutineleak") + for i := 0; i < 1000; i++ { + go func() { + mu := &sync.Mutex{} + go func() { + mu.Lock() + mu.Lock() + panic("should not be reached") + }() + }() + } + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) +} + +func RWMutexRLock() { + prof := pprof.Lookup("goroutineleak") + go func() { + mu := &sync.RWMutex{} + mu.Lock() + mu.RLock() + panic("should not be reached") + }() + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) +} + +func RWMutexLock() { + prof := pprof.Lookup("goroutineleak") + go func() { + mu := &sync.RWMutex{} + mu.Lock() + mu.Lock() + panic("should not be reached") + }() + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) +} + +func Cond() { + prof := pprof.Lookup("goroutineleak") + go func() { + cond := sync.NewCond(&sync.Mutex{}) + cond.L.Lock() + cond.Wait() + panic("should not be reached") + }() + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) +} + +func Mixed() { + prof := pprof.Lookup("goroutineleak") + go func() { + ch := make(chan int) + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + ch <- 0 + wg.Done() + panic("should not be reached") + }() + wg.Wait() + <-ch + panic("should not be reached") + }() + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) +} + +var ch = make(chan int) + +// No leak should be reported by this test +func NoLeakGlobal() { + prof := pprof.Lookup("goroutineleak") + go func() { + <-ch + }() + // Yield several times to allow the child goroutine to run. + for i := 0; i < yieldCount; i++ { + runtime.Gosched() + } + prof.WriteTo(os.Stdout, 2) +} diff --git a/src/runtime/testdata/testgoroutineleakprofile/stresstests.go b/src/runtime/testdata/testgoroutineleakprofile/stresstests.go new file mode 100644 index 00000000000000..64b535f51c87ef --- /dev/null +++ b/src/runtime/testdata/testgoroutineleakprofile/stresstests.go @@ -0,0 +1,89 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "io" + "os" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +const spawnGCMaxDepth = 5 + +func init() { + register("SpawnGC", SpawnGC) + register("DaisyChain", DaisyChain) +} + +func spawnGC(i int) { + prof := pprof.Lookup("goroutineleak") + if i == 0 { + return + } + wg := &sync.WaitGroup{} + wg.Add(i + 1) + go func() { + wg.Done() + <-make(chan int) + }() + for j := 0; j < i; j++ { + go func() { + wg.Done() + spawnGC(i - 1) + }() + } + wg.Wait() + runtime.Gosched() + if i == spawnGCMaxDepth { + prof.WriteTo(os.Stdout, 2) + } else { + // We want to concurrently trigger the profile in order to concurrently run + // the GC, but we don't want to stream all the profiles to standard output. + // + // Only output the profile for the root call to spawnGC, and otherwise stream + // the profile outputs to /dev/null to avoid jumbling. + prof.WriteTo(io.Discard, 2) + } +} + +// SpawnGC spawns a tree of goroutine leaks and calls the goroutine leak profiler +// for each node in the tree. It is supposed to stress the goroutine leak profiler +// under a heavily concurrent workload. +func SpawnGC() { + spawnGC(spawnGCMaxDepth) +} + +// DaisyChain spawns a daisy-chain of runnable goroutines. +// +// Each goroutine in the chain creates a new channel and goroutine. +// +// This illustrates a pathological worstcase for the goroutine leak GC complexity, +// as opposed to the regular GC, which is not negatively affected by this pattern. +func DaisyChain() { + prof := pprof.Lookup("goroutineleak") + defer func() { + time.Sleep(time.Second) + prof.WriteTo(os.Stdout, 2) + }() + var chain func(i int, ch chan struct{}) + chain = func(i int, ch chan struct{}) { + if i <= 0 { + go func() { + time.Sleep(time.Hour) + ch <- struct{}{} + }() + return + } + ch2 := make(chan struct{}) + go chain(i-1, ch2) + <-ch2 + ch <- struct{}{} + } + // The channel buffer avoids goroutine leaks. + go chain(1000, make(chan struct{}, 1)) +} diff --git a/src/runtime/testdata/testprog/badtraceback.go b/src/runtime/testdata/testprog/badtraceback.go index 09aa2b877ecf5a..455118a54371d7 100644 --- a/src/runtime/testdata/testprog/badtraceback.go +++ b/src/runtime/testdata/testprog/badtraceback.go @@ -44,6 +44,8 @@ func badLR2(arg int) { lrPtr := (*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(&arg)) - lrOff)) *lrPtr = 0xbad + runtime.KeepAlive(lrPtr) // prevent dead store elimination + // Print a backtrace. This should include diagnostics for the // bad return PC and a hex dump. panic("backtrace") diff --git a/src/runtime/testdata/testprog/checkfinalizers.go b/src/runtime/testdata/testprog/checkfinalizers.go new file mode 100644 index 00000000000000..ea352a4e3e92f1 --- /dev/null +++ b/src/runtime/testdata/testprog/checkfinalizers.go @@ -0,0 +1,79 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "internal/asan" + "internal/race" + "runtime" + "runtime/debug" + "unsafe" +) + +func init() { + register("DetectFinalizerAndCleanupLeaks", DetectFinalizerAndCleanupLeaks) +} + +type tiny uint8 + +var tinySink *tiny + +// Intended to be run only with `GODEBUG=checkfinalizers=1`. +func DetectFinalizerAndCleanupLeaks() { + type T *int + + defer debug.SetGCPercent(debug.SetGCPercent(-1)) + + // Leak a cleanup. + cLeak := new(T) + runtime.AddCleanup(cLeak, func(x int) { + **cLeak = x + }, int(0)) + + // Have a regular cleanup to make sure it doesn't trip the detector. + cNoLeak := new(T) + runtime.AddCleanup(cNoLeak, func(_ int) {}, int(0)) + + // Add a cleanup that only temporarily leaks cNoLeak. + runtime.AddCleanup(cNoLeak, func(x int) { + **cNoLeak = x + }, int(0)).Stop() + + if !asan.Enabled && !race.Enabled { + // Ensure we create an allocation into a tiny block that shares space among several values. + // + // Don't do this with ASAN and in race mode, where the tiny allocator is disabled. + // We might just loop forever here in that case. + var ctLeak *tiny + for { + tinySink = ctLeak + ctLeak = new(tiny) + *ctLeak = tiny(55) + // Make sure the address is an odd value. This is sufficient to + // be certain that we're sharing a block with another value and + // trip the detector. + if uintptr(unsafe.Pointer(ctLeak))%2 != 0 { + break + } + } + runtime.AddCleanup(ctLeak, func(_ struct{}) {}, struct{}{}) + } + + // Leak a finalizer. + fLeak := new(T) + runtime.SetFinalizer(fLeak, func(_ *T) { + **fLeak = 12 + }) + + // Have a regular finalizer to make sure it doesn't trip the detector. + fNoLeak := new(T) + runtime.SetFinalizer(fNoLeak, func(x *T) { + **x = 51 + }) + + // runtime.GC here should crash. + runtime.GC() + println("OK") +} diff --git a/src/runtime/testdata/testprog/checkptr.go b/src/runtime/testdata/testprog/checkptr.go index 60e71e66d7f4f5..ff99fa8c7b702f 100644 --- a/src/runtime/testdata/testprog/checkptr.go +++ b/src/runtime/testdata/testprog/checkptr.go @@ -16,6 +16,7 @@ func init() { register("CheckPtrAlignmentNilPtr", CheckPtrAlignmentNilPtr) register("CheckPtrArithmetic", CheckPtrArithmetic) register("CheckPtrArithmetic2", CheckPtrArithmetic2) + register("CheckPtrArithmeticUnsafeAdd", CheckPtrArithmeticUnsafeAdd) register("CheckPtrSize", CheckPtrSize) register("CheckPtrSmall", CheckPtrSmall) register("CheckPtrSliceOK", CheckPtrSliceOK) @@ -79,6 +80,11 @@ func CheckPtrArithmetic2() { sink2 = unsafe.Pointer(uintptr(p) & ^one) } +func CheckPtrArithmeticUnsafeAdd() { + data := make([]byte, 128) + sink2 = (*byte)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(data)), len(data))) +} + func CheckPtrSize() { p := new(int64) sink2 = p diff --git a/src/runtime/testdata/testprog/coro.go b/src/runtime/testdata/testprog/coro.go index 032215b8012820..5f3d30298789a8 100644 --- a/src/runtime/testdata/testprog/coro.go +++ b/src/runtime/testdata/testprog/coro.go @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build goexperiment.rangefunc - package main import ( diff --git a/src/runtime/testdata/testprog/crash.go b/src/runtime/testdata/testprog/crash.go index bdc395f652edca..556215a71ea08f 100644 --- a/src/runtime/testdata/testprog/crash.go +++ b/src/runtime/testdata/testprog/crash.go @@ -19,6 +19,9 @@ func init() { register("StringPanic", StringPanic) register("NilPanic", NilPanic) register("CircularPanic", CircularPanic) + register("RepanickedPanic", RepanickedPanic) + register("RepanickedMiddlePanic", RepanickedMiddlePanic) + register("RepanickedPanicSandwich", RepanickedPanicSandwich) } func test(name string) { @@ -137,3 +140,52 @@ func (e exampleCircleEndError) Error() string { func CircularPanic() { panic(exampleCircleStartError{}) } + +func RepanickedPanic() { + defer func() { + panic(recover()) + }() + panic("message") +} + +func RepanickedMiddlePanic() { + defer func() { + recover() + panic("outer") + }() + func() { + defer func() { + panic(recover()) + }() + func() { + defer func() { + recover() + panic("middle") + }() + panic("inner") + }() + }() +} + +// Panic sandwich: +// +// panic("outer") => +// recovered, panic("inner") => +// panic(recovered outer panic value) +// +// Exercises the edge case where we repanic a panic value, +// but with another panic in the middle. +func RepanickedPanicSandwich() { + var outer any + defer func() { + recover() + panic(outer) + }() + func() { + defer func() { + outer = recover() + panic("inner") + }() + panic("outer") + }() +} diff --git a/src/runtime/testdata/testprog/finalizer_deadlock.go b/src/runtime/testdata/testprog/finalizer_deadlock.go new file mode 100644 index 00000000000000..e3131541aa1a8e --- /dev/null +++ b/src/runtime/testdata/testprog/finalizer_deadlock.go @@ -0,0 +1,80 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "fmt" + "os" + "runtime" + "runtime/pprof" +) + +var finalizerDeadlockMode = flag.String("finalizer-deadlock-mode", "panic", "Trigger mode of FinalizerDeadlock") + +func init() { + register("FinalizerDeadlock", func() { FinalizerOrCleanupDeadlock(false) }) + register("CleanupDeadlock", func() { FinalizerOrCleanupDeadlock(true) }) +} + +func FinalizerOrCleanupDeadlock(useCleanup bool) { + flag.Parse() + + started := make(chan struct{}) + fn := func() { + started <- struct{}{} + select {} + } + b := new([16]byte) + if useCleanup { + runtime.AddCleanup(b, func(struct{}) { fn() }, struct{}{}) + } else { + runtime.SetFinalizer(b, func(*[16]byte) { fn() }) + } + b = nil + + runtime.GC() + + <-started + // We know the finalizer has started running. The goroutine might still + // be running or it may now be blocked. Either is fine, the goroutine + // should appear in stacks either way. + + mode := os.Getenv("GO_TEST_FINALIZER_DEADLOCK") + switch mode { + case "panic": + panic("panic") + case "stack": + buf := make([]byte, 4096) + for { + n := runtime.Stack(buf, true) + if n >= len(buf) { + buf = make([]byte, 2*len(buf)) + continue + } + buf = buf[:n] + break + } + fmt.Printf("%s\n", string(buf)) + case "pprof_proto": + if err := pprof.Lookup("goroutine").WriteTo(os.Stdout, 0); err != nil { + fmt.Fprintf(os.Stderr, "Error writing profile: %v\n", err) + os.Exit(1) + } + case "pprof_debug1": + if err := pprof.Lookup("goroutine").WriteTo(os.Stdout, 1); err != nil { + fmt.Fprintf(os.Stderr, "Error writing profile: %v\n", err) + os.Exit(1) + } + case "pprof_debug2": + if err := pprof.Lookup("goroutine").WriteTo(os.Stdout, 2); err != nil { + fmt.Fprintf(os.Stderr, "Error writing profile: %v\n", err) + os.Exit(1) + } + default: + fmt.Fprintf(os.Stderr, "Unknown mode %q. GO_TEST_FINALIZER_DEADLOCK must be one of panic, stack, pprof_proto, pprof_debug1, pprof_debug2\n", mode) + os.Exit(1) + } +} diff --git a/src/runtime/testdata/testprog/gc.go b/src/runtime/testdata/testprog/gc.go index 5dc85fbb6247c5..bbe1453401325f 100644 --- a/src/runtime/testdata/testprog/gc.go +++ b/src/runtime/testdata/testprog/gc.go @@ -395,6 +395,9 @@ func gcMemoryLimit(gcPercent int) { // somewhat heavily here) this bound is kept loose. In practice the Go runtime // should do considerably better than this bound. bound := int64(myLimit + 16<<20) + if runtime.GOOS == "darwin" { + bound += 16 << 20 // Be more lax on Darwin, see issue 73136. + } start := time.Now() for time.Since(start) < 200*time.Millisecond { metrics.Read(m[:]) diff --git a/src/runtime/testdata/testprog/gomaxprocs.go b/src/runtime/testdata/testprog/gomaxprocs.go new file mode 100644 index 00000000000000..99bc9f1dbb350a --- /dev/null +++ b/src/runtime/testdata/testprog/gomaxprocs.go @@ -0,0 +1,166 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "os" + "runtime" + "strconv" + "time" +) + +func init() { + register("PrintGOMAXPROCS", PrintGOMAXPROCS) + register("SetLimitThenDefaultGOMAXPROCS", SetLimitThenDefaultGOMAXPROCS) + register("UpdateGOMAXPROCS", UpdateGOMAXPROCS) + register("DontUpdateGOMAXPROCS", DontUpdateGOMAXPROCS) +} + +func PrintGOMAXPROCS() { + println(runtime.GOMAXPROCS(0)) +} + +func mustSetCPUMax(path string, quota int64) { + q := "max" + if quota >= 0 { + q = strconv.FormatInt(quota, 10) + } + buf := fmt.Sprintf("%s 100000", q) + if err := os.WriteFile(path, []byte(buf), 0); err != nil { + panic(fmt.Sprintf("error setting cpu.max: %v", err)) + } +} + +func mustParseInt64(s string) int64 { + v, err := strconv.ParseInt(s, 10, 64) + if err != nil { + panic(err) + } + return v +} + +// Inputs: +// GO_TEST_CPU_MAX_PATH: Path to cgroup v2 cpu.max file. +// GO_TEST_CPU_MAX_QUOTA: CPU quota to set. +func SetLimitThenDefaultGOMAXPROCS() { + path := os.Getenv("GO_TEST_CPU_MAX_PATH") + quota := mustParseInt64(os.Getenv("GO_TEST_CPU_MAX_QUOTA")) + + mustSetCPUMax(path, quota) + + runtime.SetDefaultGOMAXPROCS() + println(runtime.GOMAXPROCS(0)) +} + +// Wait for GOMAXPROCS to change from from to to. Times out after 10s. +func waitForMaxProcsChange(from, to int) { + start := time.Now() + for { + if time.Since(start) > 10*time.Second { + panic("no update for >10s") + } + + procs := runtime.GOMAXPROCS(0) + println("GOMAXPROCS:", procs) + if procs == to { + return + } + if procs != from { + panic(fmt.Sprintf("GOMAXPROCS change got %d want %d", procs, to)) + } + + time.Sleep(100*time.Millisecond) + } +} + +// Make sure that GOMAXPROCS does not change from curr. +// +// It is impossible to assert that it never changes, so this just makes sure it +// stays for 5s. +func mustNotChangeMaxProcs(curr int) { + start := time.Now() + for { + if time.Since(start) > 5*time.Second { + return + } + + procs := runtime.GOMAXPROCS(0) + println("GOMAXPROCS:", procs) + if procs != curr { + panic(fmt.Sprintf("GOMAXPROCS change got %d want %d", procs, curr)) + } + + time.Sleep(100*time.Millisecond) + } +} + +// Inputs: +// GO_TEST_CPU_MAX_PATH: Path to cgroup v2 cpu.max file. +func UpdateGOMAXPROCS() { + // We start with no limit. + + ncpu := runtime.NumCPU() + + procs := runtime.GOMAXPROCS(0) + println("GOMAXPROCS:", procs) + if procs != ncpu { + panic(fmt.Sprintf("GOMAXPROCS got %d want %d", procs, ncpu)) + } + + path := os.Getenv("GO_TEST_CPU_MAX_PATH") + + // Drop down to 3 CPU. + mustSetCPUMax(path, 300000) + waitForMaxProcsChange(ncpu, 3) + + // Drop even further. Now we hit the minimum GOMAXPROCS=2. + mustSetCPUMax(path, 100000) + waitForMaxProcsChange(3, 2) + + // Increase back up. + mustSetCPUMax(path, 300000) + waitForMaxProcsChange(2, 3) + + // Remove limit entirely. + mustSetCPUMax(path, -1) + waitForMaxProcsChange(3, ncpu) + + // Setting GOMAXPROCS explicitly disables updates. + runtime.GOMAXPROCS(3) + mustSetCPUMax(path, 200000) + mustNotChangeMaxProcs(3) + + // Re-enable updates. Change is immediately visible. + runtime.SetDefaultGOMAXPROCS() + procs = runtime.GOMAXPROCS(0) + println("GOMAXPROCS:", procs) + if procs != 2 { + panic(fmt.Sprintf("GOMAXPROCS got %d want %d", procs, 2)) + } + + // Setting GOMAXPROCS to itself also disables updates, despite not + // changing the value itself. + runtime.GOMAXPROCS(runtime.GOMAXPROCS(0)) + mustSetCPUMax(path, 300000) + mustNotChangeMaxProcs(2) + + println("OK") +} + +// Inputs: +// GO_TEST_CPU_MAX_PATH: Path to cgroup v2 cpu.max file. +func DontUpdateGOMAXPROCS() { + // The caller has disabled updates. Make sure they don't happen. + + curr := runtime.GOMAXPROCS(0) + println("GOMAXPROCS:", curr) + + path := os.Getenv("GO_TEST_CPU_MAX_PATH") + mustSetCPUMax(path, 300000) + mustNotChangeMaxProcs(curr) + + println("OK") +} diff --git a/src/runtime/testdata/testprog/gomaxprocs_windows.go b/src/runtime/testdata/testprog/gomaxprocs_windows.go new file mode 100644 index 00000000000000..cae9dba8b3afd8 --- /dev/null +++ b/src/runtime/testdata/testprog/gomaxprocs_windows.go @@ -0,0 +1,63 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "syscall" + "unsafe" +) + +func init() { + register("WindowsUpdateGOMAXPROCS", WindowsUpdateGOMAXPROCS) + register("WindowsDontUpdateGOMAXPROCS", WindowsDontUpdateGOMAXPROCS) +} + +// Set CPU affinity mask to only two CPUs. +// +// Skips the test if CPUs 0 and 1 are not available. +func setAffinity2() { + kernel32 := syscall.MustLoadDLL("kernel32.dll") + _GetProcessAffinityMask := kernel32.MustFindProc("GetProcessAffinityMask") + _SetProcessAffinityMask := kernel32.MustFindProc("SetProcessAffinityMask") + + h, err := syscall.GetCurrentProcess() + if err != nil { + panic(err) + } + + var mask, sysmask uintptr + ret, _, err := _GetProcessAffinityMask.Call(uintptr(h), uintptr(unsafe.Pointer(&mask)), uintptr(unsafe.Pointer(&sysmask))) + if ret == 0 { + panic(err) + } + + // We're going to restrict to CPUs 0 and 1. Make sure those are already available. + if mask & 0b11 != 0b11 { + println("SKIP: CPUs 0 and 1 not available") + os.Exit(0) + } + + mask = 0b11 + ret, _, err = _SetProcessAffinityMask.Call(uintptr(h), mask) + if ret == 0 { + panic(err) + } +} + +func WindowsUpdateGOMAXPROCS() { + ncpu := runtime.NumCPU() + setAffinity2() + waitForMaxProcsChange(ncpu, 2) + println("OK") +} + +func WindowsDontUpdateGOMAXPROCS() { + procs := runtime.GOMAXPROCS(0) + setAffinity2() + mustNotChangeMaxProcs(procs) + println("OK") +} diff --git a/src/runtime/testdata/testprog/lockosthread_linux.go b/src/runtime/testdata/testprog/lockosthread_linux.go new file mode 100644 index 00000000000000..9a5e2664274fc8 --- /dev/null +++ b/src/runtime/testdata/testprog/lockosthread_linux.go @@ -0,0 +1,72 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "internal/syscall/unix" + "runtime" +) + +func init() { + register("LockOSThreadVgetrandom", LockOSThreadVgetrandom) +} + +var sinkInt = 1 + +func LockOSThreadVgetrandom() { + // This is a regression test for https://go.dev/issue/73141. When that + // reproduces, this crashes with SIGSEGV with no output or stack trace, + // and detail only available in a core file. + // + // Thread exit via mexit cleans up vgetrandom state. Stress test thread + // exit + vgetrandom to look for issues by creating lots of threads + // that use GetRandom and then exit. + + // Launch at most 100 threads at a time. + const parallelism = 100 + ch := make(chan struct{}, parallelism) + for range 100 { + ch <- struct{}{} + } + + // Create at most 1000 threads to avoid completely exhausting the + // system. This test generally reproduces https://go.dev/issue/73141 in + // less than 500 iterations. + const iterations = 1000 + for range iterations { + <-ch + go func() { + defer func() { + ch <- struct{}{} + }() + + // Exit with LockOSThread held. + runtime.LockOSThread() + + // Be sure to use GetRandom to initialize vgetrandom state. + b := make([]byte, 1) + _, err := unix.GetRandom(b, 0) + if err != nil { + panic(err) + } + + // Do some busy-work. It is unclear why this is + // necessary to reproduce. Perhaps to introduce + // interesting scheduling where threads get descheduled + // in the middle of getting or putting vgetrandom + // state. + i := 0 + for range 10 * 1000 * 1000 { + i += sinkInt + } + }() + } + + // Wait for all threads to finish. + for range parallelism { + <-ch + } + println("OK") +} diff --git a/src/runtime/testdata/testprog/numcpu_freebsd.go b/src/runtime/testdata/testprog/numcpu_freebsd.go index 310c21244cfa54..a51c04a4835316 100644 --- a/src/runtime/testdata/testprog/numcpu_freebsd.go +++ b/src/runtime/testdata/testprog/numcpu_freebsd.go @@ -45,7 +45,7 @@ func FreeBSDNumCPU() { cmd := exec.Command("sysctl", "-n", "kern.smp.active") output, err := cmd.CombinedOutput() if err != nil { - fmt.Printf("fail to launch '%s', error: %s, output: %s\n", strings.Join(cmd.Args, " "), err, output) + fmt.Printf("fail to launch %#q, error: %s, output: %s\n", cmd, err, output) return } if !bytes.Equal(output, []byte("1\n")) { @@ -80,19 +80,18 @@ func getList() ([]string, error) { // Launch cpuset to print a list of available CPUs: pid mask: 0, 1, 2, 3. cmd := exec.Command("cpuset", "-g", "-p", strconv.Itoa(pid)) - cmdline := strings.Join(cmd.Args, " ") output, err := cmd.CombinedOutput() if err != nil { - return nil, fmt.Errorf("fail to execute '%s': %s", cmdline, err) + return nil, fmt.Errorf("fail to execute %#q: %s", cmd, err) } output, _, ok := bytes.Cut(output, []byte("\n")) if !ok { - return nil, fmt.Errorf("invalid output from '%s', '\\n' not found: %s", cmdline, output) + return nil, fmt.Errorf("invalid output from %#q, '\\n' not found: %s", cmd, output) } _, cpus, ok := bytes.Cut(output, []byte(":")) if !ok { - return nil, fmt.Errorf("invalid output from '%s', ':' not found: %s", cmdline, output) + return nil, fmt.Errorf("invalid output from %#q, ':' not found: %s", cmd, output) } var list []string @@ -104,7 +103,7 @@ func getList() ([]string, error) { list = append(list, index) } if len(list) == 0 { - return nil, fmt.Errorf("empty CPU list from '%s': %s", cmdline, output) + return nil, fmt.Errorf("empty CPU list from %#q: %s", cmd, output) } return list, nil } @@ -121,17 +120,16 @@ func checkNCPU(list []string) error { } // Launch FreeBSDNumCPUHelper() with specified CPUs list. cmd := exec.Command("cpuset", "-l", cListString, os.Args[0], "FreeBSDNumCPUHelper") - cmdline := strings.Join(cmd.Args, " ") output, err := cmd.CombinedOutput() if err != nil { - return fmt.Errorf("fail to launch child '%s', error: %s, output: %s", cmdline, err, output) + return fmt.Errorf("fail to launch child %#q, error: %s, output: %s", cmd, err, output) } // NumCPU from FreeBSDNumCPUHelper come with '\n'. output = bytes.TrimSpace(output) n, err := strconv.Atoi(string(output)) if err != nil { - return fmt.Errorf("fail to parse output from child '%s', error: %s, output: %s", cmdline, err, output) + return fmt.Errorf("fail to parse output from child %#q, error: %s, output: %s", cmd, err, output) } if n != len(list) { return fmt.Errorf("runtime.NumCPU() expected to %d, got %d when run with CPU list %s", len(list), n, cListString) diff --git a/src/runtime/testdata/testprog/pipe_unix.go b/src/runtime/testdata/testprog/pipe_unix.go new file mode 100644 index 00000000000000..cee4da65f6e6f3 --- /dev/null +++ b/src/runtime/testdata/testprog/pipe_unix.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !windows + +package main + +import "syscall" + +func pipe() (r, w int, err error) { + var p [2]int + err = syscall.Pipe(p[:]) + return p[0], p[1], err +} diff --git a/src/runtime/testdata/testprog/pipe_windows.go b/src/runtime/testdata/testprog/pipe_windows.go new file mode 100644 index 00000000000000..597601a1790d24 --- /dev/null +++ b/src/runtime/testdata/testprog/pipe_windows.go @@ -0,0 +1,13 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "syscall" + +func pipe() (r, w syscall.Handle, err error) { + var p [2]syscall.Handle + err = syscall.Pipe(p[:]) + return p[0], p[1], err +} diff --git a/src/runtime/testdata/testprog/schedmetrics.go b/src/runtime/testdata/testprog/schedmetrics.go new file mode 100644 index 00000000000000..bc0906330f1a4f --- /dev/null +++ b/src/runtime/testdata/testprog/schedmetrics.go @@ -0,0 +1,272 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "fmt" + "internal/testenv" + "log" + "os" + "runtime" + "runtime/debug" + "runtime/metrics" + "strings" + "sync/atomic" + "syscall" + "time" +) + +func init() { + register("SchedMetrics", SchedMetrics) +} + +// Tests runtime/metrics.Read for various scheduler metrics. +// +// Implemented in testprog to prevent other tests from polluting +// the metrics. +func SchedMetrics() { + const ( + notInGo = iota + runnable + running + waiting + created + threads + numSamples + ) + var s [numSamples]metrics.Sample + s[notInGo].Name = "/sched/goroutines/not-in-go:goroutines" + s[runnable].Name = "/sched/goroutines/runnable:goroutines" + s[running].Name = "/sched/goroutines/running:goroutines" + s[waiting].Name = "/sched/goroutines/waiting:goroutines" + s[created].Name = "/sched/goroutines-created:goroutines" + s[threads].Name = "/sched/threads/total:threads" + + var failed bool + var out bytes.Buffer + logger := log.New(&out, "", 0) + indent := 0 + logf := func(s string, a ...any) { + var prefix strings.Builder + for range indent { + prefix.WriteString("\t") + } + logger.Printf(prefix.String()+s, a...) + } + errorf := func(s string, a ...any) { + logf(s, a...) + failed = true + } + run := func(name string, f func()) { + logf("=== Checking %q", name) + indent++ + f() + indent-- + } + logMetrics := func(s []metrics.Sample) { + for i := range s { + logf("%s: %d", s[i].Name, s[i].Value.Uint64()) + } + } + + // generalSlack is the amount of goroutines we allow ourselves to be + // off by in any given category, either due to background system + // goroutines. This excludes GC goroutines. + generalSlack := uint64(4) + + // waitingSlack is the max number of blocked goroutines controlled + // by the runtime that we'll allow for. This includes GC goroutines + // as well as finalizer and cleanup goroutines. + waitingSlack := generalSlack + uint64(2*runtime.GOMAXPROCS(-1)) + + // threadsSlack is the maximum number of threads left over + // from the runtime (sysmon, the template thread, etc.) + // Certain build modes may also cause the creation of additional + // threads through frequent scheduling, like mayMoreStackPreempt. + // A slack of 5 is arbitrary but appears to be enough to cover + // the leftovers plus any inflation from scheduling-heavy build + // modes. + const threadsSlack = 5 + + // Make sure GC isn't running, since GC workers interfere with + // expected counts. + defer debug.SetGCPercent(debug.SetGCPercent(-1)) + runtime.GC() + + check := func(s *metrics.Sample, min, max uint64) { + val := s.Value.Uint64() + if val < min { + errorf("%s too low; %d < %d", s.Name, val, min) + } + if val > max { + errorf("%s too high; %d > %d", s.Name, val, max) + } + } + checkEq := func(s *metrics.Sample, value uint64) { + check(s, value, value) + } + spinUntil := func(f func() bool) bool { + for { + if f() { + return true + } + time.Sleep(50 * time.Millisecond) + } + } + + // Check base values. + run("base", func() { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1)) + metrics.Read(s[:]) + logMetrics(s[:]) + check(&s[notInGo], 0, generalSlack) + check(&s[runnable], 0, generalSlack) + checkEq(&s[running], 1) + check(&s[waiting], 0, waitingSlack) + }) + + metrics.Read(s[:]) + createdAfterBase := s[created].Value.Uint64() + + // Force Running count to be high. We'll use these goroutines + // for Runnable, too. + const count = 10 + var ready, exit atomic.Uint32 + for range count { + go func() { + ready.Add(1) + for exit.Load() == 0 { + // Spin to get us and keep us running, but check + // the exit condition so we exit out early if we're + // done. + start := time.Now() + for time.Since(start) < 10*time.Millisecond && exit.Load() == 0 { + } + runtime.Gosched() + } + }() + } + for ready.Load() < count { + runtime.Gosched() + } + + // Be careful. We've entered a dangerous state for platforms + // that do not return back to the underlying system unless all + // goroutines are blocked, like js/wasm, since we have a bunch + // of runnable goroutines all spinning. We cannot write anything + // out. + if testenv.HasParallelism() { + run("created", func() { + metrics.Read(s[:]) + logMetrics(s[:]) + checkEq(&s[created], createdAfterBase+count) + }) + run("running", func() { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(count + 4)) + // It can take a little bit for the scheduler to + // distribute the goroutines to Ps, so retry until + // we see the count we expect or the test times out. + spinUntil(func() bool { + metrics.Read(s[:]) + return s[running].Value.Uint64() >= count + }) + logMetrics(s[:]) + check(&s[running], count, count+4) + check(&s[threads], count, count+4+threadsSlack) + }) + + // Force runnable count to be high. + run("runnable", func() { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1)) + metrics.Read(s[:]) + logMetrics(s[:]) + checkEq(&s[running], 1) + check(&s[runnable], count-1, count+generalSlack) + }) + + // Done with the running/runnable goroutines. + exit.Store(1) + } else { + // Read metrics and then exit all the other goroutines, + // so that system calls may proceed. + metrics.Read(s[:]) + + // Done with the running/runnable goroutines. + exit.Store(1) + + // Now we can check our invariants. + run("created", func() { + // Look for count-1 goroutines because we read metrics + // *before* run goroutine was created for this sub-test. + checkEq(&s[created], createdAfterBase+count-1) + }) + run("running", func() { + logMetrics(s[:]) + checkEq(&s[running], 1) + checkEq(&s[threads], 1) + }) + run("runnable", func() { + logMetrics(s[:]) + check(&s[runnable], count-1, count+generalSlack) + }) + } + + // Force not-in-go count to be high. This is a little tricky since + // we try really hard not to let things block in system calls. + // We have to drop to the syscall package to do this reliably. + run("not-in-go", func() { + // Block a bunch of goroutines on an OS pipe. + pr, pw, err := pipe() + if err != nil { + switch runtime.GOOS { + case "js", "wasip1": + logf("creating pipe: %v", err) + return + } + panic(fmt.Sprintf("creating pipe: %v", err)) + } + for i := 0; i < count; i++ { + go syscall.Read(pr, make([]byte, 1)) + } + + // Let the goroutines block. + spinUntil(func() bool { + metrics.Read(s[:]) + return s[notInGo].Value.Uint64() >= count + }) + logMetrics(s[:]) + check(&s[notInGo], count, count+generalSlack) + + syscall.Close(pw) + syscall.Close(pr) + }) + + run("waiting", func() { + // Force waiting count to be high. + const waitingCount = 1000 + stop := make(chan bool) + for i := 0; i < waitingCount; i++ { + go func() { <-stop }() + } + + // Let the goroutines block. + spinUntil(func() bool { + metrics.Read(s[:]) + return s[waiting].Value.Uint64() >= waitingCount + }) + logMetrics(s[:]) + check(&s[waiting], waitingCount, waitingCount+waitingSlack) + + close(stop) + }) + + if failed { + fmt.Fprintln(os.Stderr, out.String()) + os.Exit(1) + } else { + fmt.Fprintln(os.Stderr, "OK") + } +} diff --git a/src/runtime/testdata/testprog/setcgotraceback.go b/src/runtime/testdata/testprog/setcgotraceback.go new file mode 100644 index 00000000000000..de005027ec75ae --- /dev/null +++ b/src/runtime/testdata/testprog/setcgotraceback.go @@ -0,0 +1,45 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "internal/abi" + "runtime" + "unsafe" +) + +func init() { + register("SetCgoTracebackNoCgo", SetCgoTracebackNoCgo) +} + +func cgoTraceback() { + panic("unexpectedly reached cgo traceback function") +} + +func cgoContext() { + panic("unexpectedly reached cgo context function") +} + +func cgoSymbolizer() { + panic("unexpectedly reached cgo symbolizer function") +} + +// SetCgoTraceback is a no-op in non-cgo binaries. +func SetCgoTracebackNoCgo() { + traceback := unsafe.Pointer(abi.FuncPCABIInternal(cgoTraceback)) + context := unsafe.Pointer(abi.FuncPCABIInternal(cgoContext)) + symbolizer := unsafe.Pointer(abi.FuncPCABIInternal(cgoSymbolizer)) + runtime.SetCgoTraceback(0, traceback, context, symbolizer) + + // In a cgo binary, runtime.(*Frames).Next calls the cgo symbolizer for + // any non-Go frames. Pass in a bogus frame to verify that Next does + // not attempt to call the cgo symbolizer, which would crash in a + // non-cgo binary like this one. + frames := runtime.CallersFrames([]uintptr{0x12345678}) + frames.Next() + + fmt.Println("OK") +} diff --git a/src/runtime/testdata/testprog/synctest.go b/src/runtime/testdata/testprog/synctest.go new file mode 100644 index 00000000000000..dd3a6df8a0d0d7 --- /dev/null +++ b/src/runtime/testdata/testprog/synctest.go @@ -0,0 +1,58 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "internal/synctest" + "sync" +) + +func init() { + register("SynctestCond/signal/no_bubble", func() { + synctestCond(func(cond *sync.Cond) { + cond.Signal() + }) + }) + register("SynctestCond/broadcast/no_bubble", func() { + synctestCond(func(cond *sync.Cond) { + cond.Broadcast() + }) + }) + register("SynctestCond/signal/other_bubble", func() { + synctestCond(func(cond *sync.Cond) { + synctest.Run(cond.Signal) + }) + }) + register("SynctestCond/broadcast/other_bubble", func() { + synctestCond(func(cond *sync.Cond) { + synctest.Run(cond.Broadcast) + }) + }) +} + +func synctestCond(f func(*sync.Cond)) { + var ( + mu sync.Mutex + cond = sync.NewCond(&mu) + readyc = make(chan struct{}) + wg sync.WaitGroup + ) + defer wg.Wait() + wg.Go(func() { + synctest.Run(func() { + go func() { + mu.Lock() + defer mu.Unlock() + cond.Wait() + }() + synctest.Wait() + <-readyc // #1: signal that cond.Wait is waiting + <-readyc // #2: wait to continue + cond.Signal() + }) + }) + readyc <- struct{}{} + f(cond) +} diff --git a/src/runtime/testdata/testprogcgo/callback.go b/src/runtime/testdata/testprogcgo/callback.go index 319572fe109034..39993f13a678a5 100644 --- a/src/runtime/testdata/testprogcgo/callback.go +++ b/src/runtime/testdata/testprogcgo/callback.go @@ -38,10 +38,21 @@ import ( func init() { register("CgoCallbackGC", CgoCallbackGC) + register("CgoToGoCallGoexit", CgoToGoCallGoexit) } +func CgoToGoCallGoexit() { + goexit = true + C.foo() +} + +var goexit = false + //export go_callback func go_callback() { + if goexit { + runtime.Goexit() + } if e := extraMInUse.Load(); e == 0 { fmt.Printf("in callback extraMInUse got %d want >0\n", e) os.Exit(1) diff --git a/src/runtime/testdata/testprogcgo/callback_pprof.go b/src/runtime/testdata/testprogcgo/callback_pprof.go new file mode 100644 index 00000000000000..cd235d03419d11 --- /dev/null +++ b/src/runtime/testdata/testprogcgo/callback_pprof.go @@ -0,0 +1,138 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !plan9 && !windows + +package main + +// Regression test for https://go.dev/issue/72870. Go code called from C should +// never be reported as external code. + +/* +#include + +void go_callback1(); +void go_callback2(); + +static void *callback_pprof_thread(void *arg) { + go_callback1(); + return 0; +} + +static void c_callback(void) { + go_callback2(); +} + +static void start_callback_pprof_thread() { + pthread_t th; + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_create(&th, &attr, callback_pprof_thread, 0); + // Don't join, caller will watch pprof. +} +*/ +import "C" + +import ( + "bytes" + "fmt" + "internal/profile" + "os" + "runtime/pprof" + "time" +) + +func init() { + register("CgoCallbackPprof", CgoCallbackPprof) +} + +func CgoCallbackPprof() { + C.start_callback_pprof_thread() + + var buf bytes.Buffer + if err := pprof.StartCPUProfile(&buf); err != nil { + fmt.Printf("Error starting CPU profile: %v\n", err) + os.Exit(1) + } + time.Sleep(1 * time.Second) + pprof.StopCPUProfile() + + p, err := profile.Parse(&buf) + if err != nil { + fmt.Printf("Error parsing profile: %v\n", err) + os.Exit(1) + } + + foundCallee := false + for _, s := range p.Sample { + funcs := flattenFrames(s) + if len(funcs) == 0 { + continue + } + + leaf := funcs[0] + if leaf.Name != "main.go_callback1_callee" { + continue + } + foundCallee = true + + if len(funcs) < 2 { + fmt.Printf("Profile: %s\n", p) + frames := make([]string, len(funcs)) + for i := range funcs { + frames[i] = funcs[i].Name + } + fmt.Printf("FAIL: main.go_callback1_callee sample missing caller in frames %v\n", frames) + os.Exit(1) + } + + if funcs[1].Name != "main.go_callback1" { + // In https://go.dev/issue/72870, this will be runtime._ExternalCode. + fmt.Printf("Profile: %s\n", p) + frames := make([]string, len(funcs)) + for i := range funcs { + frames[i] = funcs[i].Name + } + fmt.Printf("FAIL: main.go_callback1_callee sample caller got %s want main.go_callback1 in frames %v\n", funcs[1].Name, frames) + os.Exit(1) + } + } + + if !foundCallee { + fmt.Printf("Missing main.go_callback1_callee sample in profile %s\n", p) + os.Exit(1) + } + + fmt.Printf("OK\n") +} + +// Return the frame functions in s, regardless of inlining. +func flattenFrames(s *profile.Sample) []*profile.Function { + ret := make([]*profile.Function, 0, len(s.Location)) + for _, loc := range s.Location { + for _, line := range loc.Line { + ret = append(ret, line.Function) + } + } + return ret +} + +//export go_callback1 +func go_callback1() { + // This is a separate function just to ensure we have another Go + // function as the caller in the profile. + go_callback1_callee() +} + +func go_callback1_callee() { + C.c_callback() + + // Spin for CPU samples. + for { + } +} + +//export go_callback2 +func go_callback2() { +} diff --git a/src/runtime/testdata/testprogcgo/cgonocallback.go b/src/runtime/testdata/testprogcgo/cgonocallback.go index c13bf271a4aa33..8cbbfd1957d668 100644 --- a/src/runtime/testdata/testprogcgo/cgonocallback.go +++ b/src/runtime/testdata/testprogcgo/cgonocallback.go @@ -8,7 +8,8 @@ package main // But it do callback to go in this test, Go should crash here. /* -// TODO(#56378): #cgo nocallback runCShouldNotCallback +#cgo nocallback runCShouldNotCallback + extern void runCShouldNotCallback(); */ import "C" diff --git a/src/runtime/testdata/testprogcgo/cgonoescape.go b/src/runtime/testdata/testprogcgo/cgonoescape.go index f5eebac677ad1c..2b7c7a9c551f46 100644 --- a/src/runtime/testdata/testprogcgo/cgonoescape.go +++ b/src/runtime/testdata/testprogcgo/cgonoescape.go @@ -13,7 +13,8 @@ package main // 2. less than 100 new allocated heap objects after invoking withoutNoEscape 100 times. /* -// TODO(#56378): #cgo noescape runCWithNoEscape +#cgo noescape runCWithNoEscape +#cgo nocallback runCWithNoEscape void runCWithNoEscape(void *p) { } diff --git a/src/runtime/testdata/testprogcgo/coro.go b/src/runtime/testdata/testprogcgo/coro.go index e0cb9451121421..93be92cb7a046a 100644 --- a/src/runtime/testdata/testprogcgo/coro.go +++ b/src/runtime/testdata/testprogcgo/coro.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build goexperiment.rangefunc && !windows +//go:build !windows package main diff --git a/src/runtime/testdata/testprogcgo/issue63739.go b/src/runtime/testdata/testprogcgo/issue63739.go new file mode 100644 index 00000000000000..dbe37b6d0e732c --- /dev/null +++ b/src/runtime/testdata/testprogcgo/issue63739.go @@ -0,0 +1,59 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +// This is for issue #63739. +// Ensure that parameters are kept alive until the end of the C call. If not, +// then a stack copy at just the right time while calling into C might think +// that any stack pointers are not alive and fail to update them, causing the C +// function to see the old, no longer correct, pointer values. + +/* +int add_from_multiple_pointers(int *a, int *b, int *c) { + *a = *a + 1; + *b = *b + 1; + *c = *c + 1; + return *a + *b + *c; +} +#cgo noescape add_from_multiple_pointers +#cgo nocallback add_from_multiple_pointers +*/ +import "C" + +import ( + "fmt" +) + +const ( + maxStack = 1024 +) + +func init() { + register("CgoEscapeWithMultiplePointers", CgoEscapeWithMultiplePointers) +} + +func CgoEscapeWithMultiplePointers() { + stackGrow(maxStack) + fmt.Println("OK") +} + +//go:noinline +func testCWithMultiplePointers() { + var a C.int = 1 + var b C.int = 2 + var c C.int = 3 + v := C.add_from_multiple_pointers(&a, &b, &c) + if v != 9 || a != 2 || b != 3 || c != 4 { + fmt.Printf("%d + %d + %d != %d\n", a, b, c, v) + } +} + +func stackGrow(n int) { + if n == 0 { + return + } + testCWithMultiplePointers() + stackGrow(n - 1) +} diff --git a/src/runtime/testdata/testprogcgo/needmdeadlock.go b/src/runtime/testdata/testprogcgo/needmdeadlock.go index b95ec7746895b2..f4710488c9e163 100644 --- a/src/runtime/testdata/testprogcgo/needmdeadlock.go +++ b/src/runtime/testdata/testprogcgo/needmdeadlock.go @@ -70,8 +70,6 @@ import "C" import ( "fmt" - "os" - "time" ) func init() { @@ -84,12 +82,8 @@ func GoNeedM() { func NeedmDeadlock() { // The failure symptom is that the program hangs because of a - // deadlock in needm, so set an alarm. - go func() { - time.Sleep(5 * time.Second) - fmt.Println("Hung for 5 seconds") - os.Exit(1) - }() + // deadlock in needm. Instead of using an arbitrary timeout, + // we let the test deadline expire if it deadlocks. C.runNeedmSignalThread() fmt.Println("OK") diff --git a/src/runtime/testdata/testprogcgo/threadprof.go b/src/runtime/testdata/testprogcgo/threadprof.go index 00b511d23be609..2d8c0f6d90002e 100644 --- a/src/runtime/testdata/testprogcgo/threadprof.go +++ b/src/runtime/testdata/testprogcgo/threadprof.go @@ -36,10 +36,10 @@ __attribute__((constructor)) void issue9456() { } } -void **nullptr; +void **nullpointer; void *crash(void *p) { - *nullptr = p; + *nullpointer = p; return 0; } diff --git a/src/runtime/testdata/testprogcgo/tracebackctxt.go b/src/runtime/testdata/testprogcgo/tracebackctxt.go index 62ff8eccd671a2..5b622c38a60503 100644 --- a/src/runtime/testdata/testprogcgo/tracebackctxt.go +++ b/src/runtime/testdata/testprogcgo/tracebackctxt.go @@ -17,19 +17,24 @@ extern void tcTraceback(void*); extern void tcSymbolizer(void*); extern int getContextCount(void); extern void TracebackContextPreemptionCallGo(int); +extern void TracebackContextProfileCallGo(void); */ import "C" import ( "fmt" + "io" "runtime" + "runtime/pprof" "sync" + "sync/atomic" "unsafe" ) func init() { register("TracebackContext", TracebackContext) register("TracebackContextPreemption", TracebackContextPreemption) + register("TracebackContextProfile", TracebackContextProfile) } var tracebackOK bool @@ -134,3 +139,41 @@ func TracebackContextPreemptionGoFunction(i C.int) { // Do some busy work. fmt.Sprintf("%d\n", i) } + +// Regression test for issue 71395. +// +// The SIGPROF handler can call the SetCgoTraceback traceback function if the +// context function is also provided. Ensure that call is safe. +func TracebackContextProfile() { + runtime.SetCgoTraceback(0, unsafe.Pointer(C.tcTraceback), unsafe.Pointer(C.tcContextSimple), unsafe.Pointer(C.tcSymbolizer)) + + if err := pprof.StartCPUProfile(io.Discard); err != nil { + panic(fmt.Sprintf("error starting CPU profile: %v", err)) + } + defer pprof.StopCPUProfile() + + const calls = 1e5 + var wg sync.WaitGroup + for i := 0; i < runtime.GOMAXPROCS(0); i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < calls; j++ { + C.TracebackContextProfileCallGo() + } + }() + } + wg.Wait() + + fmt.Println("OK") +} + +var sink atomic.Pointer[byte] + +//export TracebackContextProfileGoFunction +func TracebackContextProfileGoFunction() { + // Issue 71395 occurs when SIGPROF lands on code running on the system + // stack in a cgo callback. The allocator uses the system stack. + b := make([]byte, 128) + sink.Store(&b[0]) +} diff --git a/src/runtime/testdata/testprogcgo/tracebackctxt_c.c b/src/runtime/testdata/testprogcgo/tracebackctxt_c.c index 910cb7b8997a67..98b43038c58e30 100644 --- a/src/runtime/testdata/testprogcgo/tracebackctxt_c.c +++ b/src/runtime/testdata/testprogcgo/tracebackctxt_c.c @@ -12,6 +12,7 @@ extern void G1(void); extern void G2(void); extern void TracebackContextPreemptionGoFunction(int); +extern void TracebackContextProfileGoFunction(void); void C1() { G1(); @@ -101,3 +102,7 @@ void tcSymbolizer(void *parg) { void TracebackContextPreemptionCallGo(int i) { TracebackContextPreemptionGoFunction(i); } + +void TracebackContextProfileCallGo(void) { + TracebackContextProfileGoFunction(); +} diff --git a/src/runtime/testdata/testprognet/signalexec.go b/src/runtime/testdata/testprognet/signalexec.go index 62ebce7176f220..7e7591e8f79a0a 100644 --- a/src/runtime/testdata/testprognet/signalexec.go +++ b/src/runtime/testdata/testprognet/signalexec.go @@ -13,9 +13,11 @@ package main import ( "fmt" + "io" "os" "os/exec" "os/signal" + "runtime" "sync" "syscall" "time" @@ -23,10 +25,51 @@ import ( func init() { register("SignalDuringExec", SignalDuringExec) + register("SignalDuringExecPgrp", SignalDuringExecPgrp) register("Nop", Nop) } func SignalDuringExec() { + // Re-launch ourselves in a new process group. + cmd := exec.Command(os.Args[0], "SignalDuringExecPgrp") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, + } + + // Start the new process with an extra pipe. It will + // exit if the pipe is closed. + rp, wp, err := os.Pipe() + if err != nil { + fmt.Printf("Failed to create pipe: %v", err) + return + } + cmd.ExtraFiles = []*os.File{rp} + + // Run the command. + if err := cmd.Run(); err != nil { + fmt.Printf("Run failed: %v", err) + } + + // We don't actually need to write to the pipe, it just + // needs to get closed, which will happen on process + // exit. + runtime.KeepAlive(wp) +} + +func SignalDuringExecPgrp() { + // Grab fd 3 which is a pipe we need to read on. + f := os.NewFile(3, "pipe") + go func() { + // Nothing will ever get written to the pipe, so we'll + // just block on it. If it closes, ReadAll will return + // one way or another, at which point we'll exit. + io.ReadAll(f) + os.Exit(1) + }() + + // This is just for SignalDuringExec. pgrp := syscall.Getpgrp() const tries = 10 diff --git a/src/runtime/testdata/testsynctest/main.go b/src/runtime/testdata/testsynctest/main.go new file mode 100644 index 00000000000000..973d3eac0273fd --- /dev/null +++ b/src/runtime/testdata/testsynctest/main.go @@ -0,0 +1,92 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "internal/synctest" + "runtime" + "runtime/metrics" + "sync/atomic" + "unsafe" +) + +// This program ensures system goroutines (GC workers, finalizer goroutine) +// started from within a synctest bubble do not participate in that bubble. +// +// To ensure none of these goroutines start before synctest.Run, +// it must have no dependencies on packages which may start system goroutines. +// This includes the os package, which creates finalizers at init time. + +func numGCCycles() uint64 { + samples := []metrics.Sample{{Name: "/gc/cycles/total:gc-cycles"}} + metrics.Read(samples) + if samples[0].Value.Kind() == metrics.KindBad { + panic("metric not supported") + } + return samples[0].Value.Uint64() +} + +type T struct { + v int + p unsafe.Pointer +} + +func main() { + // Channels created by a finalizer and cleanup func registered within the bubble. + var ( + finalizerCh atomic.Pointer[chan struct{}] + cleanupCh atomic.Pointer[chan struct{}] + ) + synctest.Run(func() { + // Start the finalizer and cleanup goroutines. + { + p := new(T) + runtime.SetFinalizer(p, func(*T) { + ch := make(chan struct{}) + finalizerCh.Store(&ch) + }) + runtime.AddCleanup(p, func(struct{}) { + ch := make(chan struct{}) + cleanupCh.Store(&ch) + }, struct{}{}) + } + startingCycles := numGCCycles() + ch1 := make(chan *T) + ch2 := make(chan *T) + defer close(ch1) + go func() { + for range ch1 { + ch2 <- &T{} + } + }() + const iterations = 1000 + for range iterations { + // Make a lot of short-lived allocations to get the GC working. + for range 1000 { + v := new(T) + // Set finalizers on these values, just for added stress. + runtime.SetFinalizer(v, func(*T) {}) + ch1 <- v + <-ch2 + } + + // If we've improperly put a GC goroutine into the synctest group, + // this Wait is going to hang. + synctest.Wait() + + // End the test after a couple of GC cycles have passed. + if numGCCycles()-startingCycles > 1 && finalizerCh.Load() != nil && cleanupCh.Load() != nil { + return + } + } + println("finalizers/cleanups failed to run after", iterations, "cycles") + }) + // Close the channels created by the finalizer and cleanup func. + // If the funcs improperly ran inside the bubble, these channels are bubbled + // and trying to close them will panic. + close(*finalizerCh.Load()) + close(*cleanupCh.Load()) + println("success") +} diff --git a/src/runtime/testdata/testsyscall/testsyscall.go b/src/runtime/testdata/testsyscall/testsyscall.go new file mode 100644 index 00000000000000..23cca16494d530 --- /dev/null +++ b/src/runtime/testdata/testsyscall/testsyscall.go @@ -0,0 +1,65 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + _ "runtime/testdata/testsyscall/testsyscallc" // unfortunately, we can't put C and assembly in the package + _ "unsafe" // for go:linkname +) + +//go:linkname syscall_syscall syscall.syscall +func syscall_syscall(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) + +//go:linkname syscall_syscall6 syscall.syscall6 +func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) + +//go:linkname syscall_syscall9 syscall.syscall9 +func syscall_syscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2, err uintptr) + +var ( + syscall_check0_trampoline_addr uintptr + syscall_check1_trampoline_addr uintptr + syscall_check2_trampoline_addr uintptr + syscall_check3_trampoline_addr uintptr + syscall_check4_trampoline_addr uintptr + syscall_check5_trampoline_addr uintptr + syscall_check6_trampoline_addr uintptr + syscall_check7_trampoline_addr uintptr + syscall_check8_trampoline_addr uintptr + syscall_check9_trampoline_addr uintptr +) + +func main() { + if ret, _, _ := syscall_syscall(syscall_check0_trampoline_addr, 0, 0, 0); ret != 1 { + panic("hello0") + } + if ret, _, _ := syscall_syscall(syscall_check1_trampoline_addr, 1, 0, 0); ret != 1 { + panic("hello1") + } + if ret, _, _ := syscall_syscall(syscall_check2_trampoline_addr, 1, 2, 0); ret != 1 { + panic("hello2") + } + if ret, _, _ := syscall_syscall(syscall_check3_trampoline_addr, 1, 2, 3); ret != 1 { + panic("hello3") + } + if ret, _, _ := syscall_syscall6(syscall_check4_trampoline_addr, 1, 2, 3, 4, 0, 0); ret != 1 { + panic("hello4") + } + if ret, _, _ := syscall_syscall6(syscall_check5_trampoline_addr, 1, 2, 3, 4, 5, 0); ret != 1 { + panic("hello5") + } + if ret, _, _ := syscall_syscall6(syscall_check6_trampoline_addr, 1, 2, 3, 4, 5, 6); ret != 1 { + panic("hello6") + } + if ret, _, _ := syscall_syscall9(syscall_check7_trampoline_addr, 1, 2, 3, 4, 5, 6, 7, 0, 0); ret != 1 { + panic("hello7") + } + if ret, _, _ := syscall_syscall9(syscall_check8_trampoline_addr, 1, 2, 3, 4, 5, 6, 7, 8, 0); ret != 1 { + panic("hello8") + } + if ret, _, _ := syscall_syscall9(syscall_check9_trampoline_addr, 1, 2, 3, 4, 5, 6, 7, 8, 9); ret != 1 { + panic("hello9") + } +} diff --git a/src/runtime/testdata/testsyscall/testsyscall.s b/src/runtime/testdata/testsyscall/testsyscall.s new file mode 100644 index 00000000000000..c8d556dfd9a4b1 --- /dev/null +++ b/src/runtime/testdata/testsyscall/testsyscall.s @@ -0,0 +1,55 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +TEXT syscall_check0_trampoline<>(SB),NOSPLIT,$0-0 + JMP syscall_check0(SB) +GLOBL ·syscall_check0_trampoline_addr(SB), RODATA, $8 +DATA ·syscall_check0_trampoline_addr(SB)/8, $syscall_check0_trampoline<>(SB) + +TEXT syscall_check1_trampoline<>(SB),NOSPLIT,$0-0 + JMP syscall_check1(SB) +GLOBL ·syscall_check1_trampoline_addr(SB), RODATA, $8 +DATA ·syscall_check1_trampoline_addr(SB)/8, $syscall_check1_trampoline<>(SB) + +TEXT syscall_check2_trampoline<>(SB),NOSPLIT,$0-0 + JMP syscall_check2(SB) +GLOBL ·syscall_check2_trampoline_addr(SB), RODATA, $8 +DATA ·syscall_check2_trampoline_addr(SB)/8, $syscall_check2_trampoline<>(SB) + +TEXT syscall_check3_trampoline<>(SB),NOSPLIT,$0-0 + JMP syscall_check3(SB) +GLOBL ·syscall_check3_trampoline_addr(SB), RODATA, $8 +DATA ·syscall_check3_trampoline_addr(SB)/8, $syscall_check3_trampoline<>(SB) + +TEXT syscall_check4_trampoline<>(SB),NOSPLIT,$0-0 + JMP syscall_check4(SB) +GLOBL ·syscall_check4_trampoline_addr(SB), RODATA, $8 +DATA ·syscall_check4_trampoline_addr(SB)/8, $syscall_check4_trampoline<>(SB) + +TEXT syscall_check5_trampoline<>(SB),NOSPLIT,$0-0 + JMP syscall_check5(SB) +GLOBL ·syscall_check5_trampoline_addr(SB), RODATA, $8 +DATA ·syscall_check5_trampoline_addr(SB)/8, $syscall_check5_trampoline<>(SB) + +TEXT syscall_check6_trampoline<>(SB),NOSPLIT,$0-0 + JMP syscall_check6(SB) +GLOBL ·syscall_check6_trampoline_addr(SB), RODATA, $8 +DATA ·syscall_check6_trampoline_addr(SB)/8, $syscall_check6_trampoline<>(SB) + +TEXT syscall_check7_trampoline<>(SB),NOSPLIT,$0-0 + JMP syscall_check7(SB) +GLOBL ·syscall_check7_trampoline_addr(SB), RODATA, $8 +DATA ·syscall_check7_trampoline_addr(SB)/8, $syscall_check7_trampoline<>(SB) + +TEXT syscall_check8_trampoline<>(SB),NOSPLIT,$0-0 + JMP syscall_check8(SB) +GLOBL ·syscall_check8_trampoline_addr(SB), RODATA, $8 +DATA ·syscall_check8_trampoline_addr(SB)/8, $syscall_check8_trampoline<>(SB) + +TEXT syscall_check9_trampoline<>(SB),NOSPLIT,$0-0 + JMP syscall_check9(SB) +GLOBL ·syscall_check9_trampoline_addr(SB), RODATA, $8 +DATA ·syscall_check9_trampoline_addr(SB)/8, $syscall_check9_trampoline<>(SB) diff --git a/src/runtime/testdata/testsyscall/testsyscallc/testsyscallc.go b/src/runtime/testdata/testsyscall/testsyscallc/testsyscallc.go new file mode 100644 index 00000000000000..0b2a220b590563 --- /dev/null +++ b/src/runtime/testdata/testsyscall/testsyscallc/testsyscallc.go @@ -0,0 +1,48 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testsyscallc + +/* +int syscall_check0(void) { + return 1; +} + +int syscall_check1(int a1) { + return a1 == 1; +} + +int syscall_check2(int a1, int a2) { + return a1 == 1 && a2 == 2; +} + +int syscall_check3(int a1, int a2, int a3) { + return a1 == 1 && a2 == 2 && a3 == 3; +} + +int syscall_check4(int a1, int a2, int a3, int a4) { + return a1 == 1 && a2 == 2 && a3 == 3 && a4 == 4; +} + +int syscall_check5(int a1, int a2, int a3, int a4, int a5) { + return a1 == 1 && a2 == 2 && a3 == 3 && a4 == 4 && a5 == 5; +} + +int syscall_check6(int a1, int a2, int a3, int a4, int a5, int a6) { + return a1 == 1 && a2 == 2 && a3 == 3 && a4 == 4 && a5 == 5 && a6 == 6; +} + +int syscall_check7(int a1, int a2, int a3, int a4, int a5, int a6, int a7) { + return a1 == 1 && a2 == 2 && a3 == 3 && a4 == 4 && a5 == 5 && a6 == 6 && a7 == 7; +} + +int syscall_check8(int a1, int a2, int a3, int a4, int a5, int a6, int a7, int a8) { + return a1 == 1 && a2 == 2 && a3 == 3 && a4 == 4 && a5 == 5 && a6 == 6 && a7 == 7 && a8 == 8; +} + +int syscall_check9(int a1, int a2, int a3, int a4, int a5, int a6, int a7, int a8, int a9) { + return a1 == 1 && a2 == 2 && a3 == 3 && a4 == 4 && a5 == 5 && a6 == 6 && a7 == 7 && a8 == 8 && a9 == 9; +} +*/ +import "C" diff --git a/src/runtime/time.go b/src/runtime/time.go index 79f0514c6eec0a..e9d1f0b6c9a10f 100644 --- a/src/runtime/time.go +++ b/src/runtime/time.go @@ -13,6 +13,36 @@ import ( "unsafe" ) +//go:linkname time_runtimeNow time.runtimeNow +func time_runtimeNow() (sec int64, nsec int32, mono int64) { + if bubble := getg().bubble; bubble != nil { + sec = bubble.now / (1000 * 1000 * 1000) + nsec = int32(bubble.now % (1000 * 1000 * 1000)) + // Don't return a monotonic time inside a synctest bubble. + // If we return a monotonic time based on the fake clock, + // arithmetic on times created inside/outside bubbles is confusing. + // If we return a monotonic time based on the real monotonic clock, + // arithmetic on times created in the same bubble is confusing. + // Simplest is to omit the monotonic time within a bubble. + return sec, nsec, 0 + } + return time_now() +} + +//go:linkname time_runtimeNano time.runtimeNano +func time_runtimeNano() int64 { + gp := getg() + if gp.bubble != nil { + return gp.bubble.now + } + return nanotime() +} + +//go:linkname time_runtimeIsBubbled time.runtimeIsBubbled +func time_runtimeIsBubbled() bool { + return getg().bubble != nil +} + // A timer is a potentially repeating trigger for calling t.f(t.arg, t.seq). // Timers are allocated by client code, often as part of other data structures. // Each P has a heap of pointers to timers that it manages. @@ -26,10 +56,13 @@ type timer struct { // mu protects reads and writes to all fields, with exceptions noted below. mu mutex - astate atomic.Uint8 // atomic copy of state bits at last unlock - state uint8 // state bits - isChan bool // timer has a channel; immutable; can be read without lock - blocked uint32 // number of goroutines blocked on timer's channel + astate atomic.Uint8 // atomic copy of state bits at last unlock + state uint8 // state bits + isChan bool // timer has a channel; immutable; can be read without lock + isFake bool // timer is using fake time; immutable; can be read without lock + + blocked uint32 // number of goroutines blocked on timer's channel + rand uint32 // randomizes order of timers at same instant; only set when isFake // Timer wakes up at when, and then at when+period, ... (period > 0 only) // each time calling f(arg, seq, delay) in the timer goroutine, so f must be @@ -38,7 +71,7 @@ type timer struct { // The arg and seq are client-specified opaque arguments passed back to f. // When used from netpoll, arg and seq have meanings defined by netpoll // and are completely opaque to this code; in that context, seq is a sequence - // number to recognize and squech stale function invocations. + // number to recognize and squelch stale function invocations. // When used from package time, arg is a channel (for After, NewTicker) // or the function to call (for AfterFunc) and seq is unused (0). // @@ -68,6 +101,20 @@ type timer struct { // sendLock protects sends on the timer's channel. // Not used for async (pre-Go 1.23) behavior when debug.asynctimerchan.Load() != 0. sendLock mutex + + // isSending is used to handle races between running a + // channel timer and stopping or resetting the timer. + // It is used only for channel timers (t.isChan == true). + // It is not used for tickers. + // The value is incremented when about to send a value on the channel, + // and decremented after sending the value. + // The stop/reset code uses this to detect whether it + // stopped the channel send. + // + // isSending is incremented only when t.mu is held. + // isSending is decremented only when t.sendLock is held. + // isSending is read only when both t.mu and t.sendLock are held. + isSending atomic.Int32 } // init initializes a newly allocated timer t. @@ -117,6 +164,21 @@ type timerWhen struct { when int64 } +// less reports whether tw is less than other. +func (tw timerWhen) less(other timerWhen) bool { + switch { + case tw.when < other.when: + return true + case tw.when > other.when: + return false + default: + // When timers wake at the same time, use a per-timer random value to order them. + // We only set the random value for timers using fake time, since there's + // no practical way to schedule real-time timers for the same instant. + return tw.timer.rand < other.timer.rand + } +} + func (ts *timers) lock() { lock(&ts.mu) } @@ -275,14 +337,31 @@ func timeSleep(ns int64) { if t == nil { t = new(timer) t.init(goroutineReady, gp) + if gp.bubble != nil { + t.isFake = true + } gp.timer = t } - when := nanotime() + ns + var now int64 + if bubble := gp.bubble; bubble != nil { + now = bubble.now + } else { + now = nanotime() + } + when := now + ns if when < 0 { // check for overflow. when = maxWhen } gp.sleepWhen = when - gopark(resetForSleep, nil, waitReasonSleep, traceBlockSleep, 1) + if t.isFake { + // Call timer.reset in this goroutine, since it's the one in a bubble. + // We don't need to worry about the timer function running before the goroutine + // is parked, because time won't advance until we park. + resetForSleep(gp, nil) + gopark(nil, nil, waitReasonSleep, traceBlockSleep, 1) + } else { + gopark(resetForSleep, nil, waitReasonSleep, traceBlockSleep, 1) + } } // resetForSleep is called after the goroutine is parked for timeSleep. @@ -322,6 +401,9 @@ func newTimer(when, period int64, f func(arg any, seq uintptr, delay int64), arg throw("invalid timer channel: no capacity") } } + if bubble := getg().bubble; bubble != nil { + t.isFake = true + } t.modify(when, period, f, arg, 0) t.init = true return t @@ -332,6 +414,9 @@ func newTimer(when, period int64, f func(arg any, seq uintptr, delay int64), arg // //go:linkname stopTimer time.stopTimer func stopTimer(t *timeTimer) bool { + if t.isFake && getg().bubble == nil { + fatal("stop of synctest timer from outside bubble") + } return t.stop() } @@ -344,6 +429,9 @@ func resetTimer(t *timeTimer, when, period int64) bool { if raceenabled { racerelease(unsafe.Pointer(&t.timer)) } + if t.isFake && getg().bubble == nil { + fatal("reset of synctest timer from outside bubble") + } return t.reset(when, period) } @@ -395,7 +483,7 @@ func (t *timer) maybeRunAsync() { // timer ourselves now is fine.) if now := nanotime(); t.when <= now { systemstack(func() { - t.unlockAndRun(now) // resets t.when + t.unlockAndRun(now, nil) // resets t.when }) t.lock() } @@ -431,6 +519,15 @@ func (t *timer) stop() bool { // Stop any future sends with stale values. // See timer.unlockAndRun. t.seq++ + + // If there is currently a send in progress, + // incrementing seq is going to prevent that + // send from actually happening. That means + // that we should return true: the timer was + // stopped, even though t.when may be zero. + if t.period == 0 && t.isSending.Load() > 0 { + pending = true + } } t.unlock() if !async && t.isChan { @@ -490,6 +587,7 @@ func (t *timer) modify(when, period int64, f func(arg any, seq uintptr, delay in t.maybeRunAsync() } t.trace("modify") + oldPeriod := t.period t.period = period if f != nil { t.f = f @@ -521,10 +619,42 @@ func (t *timer) modify(when, period int64, f func(arg any, seq uintptr, delay in add := t.needsAdd() + if add && t.isFake { + // If this is a bubbled timer scheduled to fire immediately, + // run it now rather than waiting for the bubble's timer scheduler. + // This avoids deferring timer execution until after the bubble + // becomes durably blocked. + // + // Don't do this for non-bubbled timers: It isn't necessary, + // and there may be cases where the runtime executes timers with + // the expectation the timer func will not run in the current goroutine. + // Bubbled timers are always created by the time package, and are + // safe to run in the current goroutine. + bubble := getg().bubble + if bubble == nil { + throw("fake timer executing with no bubble") + } + if t.state&timerHeaped == 0 && when <= bubble.now { + systemstack(func() { + t.unlockAndRun(bubble.now, bubble) + }) + return pending + } + } + if !async && t.isChan { // Stop any future sends with stale values. // See timer.unlockAndRun. t.seq++ + + // If there is currently a send in progress, + // incrementing seq is going to prevent that + // send from actually happening. That means + // that we should return true: the timer was + // stopped, even though t.when may be zero. + if oldPeriod == 0 && t.isSending.Load() > 0 { + pending = true + } } t.unlock() if !async && t.isChan { @@ -586,7 +716,16 @@ func (t *timer) maybeAdd() { // Calling acquirem instead of using getg().m makes sure that // we end up locking and inserting into the current P's timers. mp := acquirem() - ts := &mp.p.ptr().timers + var ts *timers + if t.isFake { + bubble := getg().bubble + if bubble == nil { + throw("invalid timer: fake time but no syncgroup") + } + ts = &bubble.timers + } else { + ts = &mp.p.ptr().timers + } ts.lock() ts.cleanHead() t.lock() @@ -594,6 +733,12 @@ func (t *timer) maybeAdd() { when := int64(0) wake := false if t.needsAdd() { + if t.isFake { + // Re-randomize timer order. + // We could do this for all timers, but unbubbled timers are highly + // unlikely to have the same when. + t.rand = cheaprand() + } t.state |= timerHeaped when = t.when wakeTime := ts.wakeTime() @@ -858,7 +1003,7 @@ func (ts *timers) wakeTime() int64 { // We pass now in and out to avoid extra calls of nanotime. // //go:yeswritebarrierrec -func (ts *timers) check(now int64) (rnow, pollUntil int64, ran bool) { +func (ts *timers) check(now int64, bubble *synctestBubble) (rnow, pollUntil int64, ran bool) { ts.trace("check") // If it's not yet time for the first timer, or the first adjusted // timer, then there is nothing to do. @@ -891,7 +1036,7 @@ func (ts *timers) check(now int64) (rnow, pollUntil int64, ran bool) { ts.adjust(now, false) for len(ts.heap) > 0 { // Note that runtimer may temporarily unlock ts. - if tw := ts.run(now); tw != 0 { + if tw := ts.run(now, bubble); tw != 0 { if tw > 0 { pollUntil = tw } @@ -923,7 +1068,7 @@ func (ts *timers) check(now int64) (rnow, pollUntil int64, ran bool) { // If a timer is run, this will temporarily unlock ts. // //go:systemstack -func (ts *timers) run(now int64) int64 { +func (ts *timers) run(now int64, bubble *synctestBubble) int64 { ts.trace("run") assertLockHeld(&ts.mu) Redo: @@ -957,7 +1102,7 @@ Redo: return t.when } - t.unlockAndRun(now) + t.unlockAndRun(now, bubble) assertLockHeld(&ts.mu) // t is unlocked now, but not ts return 0 } @@ -968,7 +1113,7 @@ Redo: // unlockAndRun returns with t unlocked and t.ts (re-)locked. // //go:systemstack -func (t *timer) unlockAndRun(now int64) { +func (t *timer) unlockAndRun(now int64, bubble *synctestBubble) { t.trace("unlockAndRun") assertLockHeld(&t.mu) if t.ts != nil { @@ -978,7 +1123,13 @@ func (t *timer) unlockAndRun(now int64) { // Note that we are running on a system stack, // so there is no chance of getg().m being reassigned // out from under us while this function executes. - tsLocal := &getg().m.p.ptr().timers + gp := getg() + var tsLocal *timers + if bubble == nil { + tsLocal = &gp.m.p.ptr().timers + } else { + tsLocal = &bubble.timers + } if tsLocal.raceCtx == 0 { tsLocal.raceCtx = racegostart(abi.FuncPCABIInternal((*timers).run) + sys.PCQuantum) } @@ -1013,6 +1164,15 @@ func (t *timer) unlockAndRun(now int64) { } t.updateHeap() } + + async := debug.asynctimerchan.Load() != 0 + if !async && t.isChan && t.period == 0 { + // Tell Stop/Reset that we are sending a value. + if t.isSending.Add(1) < 0 { + throw("too many concurrent timer firings") + } + } + t.unlock() if raceenabled { @@ -1021,14 +1181,27 @@ func (t *timer) unlockAndRun(now int64) { if gp.racectx != 0 { throw("unexpected racectx") } - gp.racectx = gp.m.p.ptr().timers.raceCtx + if bubble == nil { + gp.racectx = gp.m.p.ptr().timers.raceCtx + } else { + gp.racectx = bubble.timers.raceCtx + } } if ts != nil { ts.unlock() } - async := debug.asynctimerchan.Load() != 0 + if bubble != nil { + // Temporarily use the timer's synctest group for the G running this timer. + gp := getg() + if gp.bubble != nil { + throw("unexpected syncgroup set") + } + gp.bubble = bubble + bubble.changegstatus(gp, _Gdead, _Grunning) + } + if !async && t.isChan { // For a timer channel, we want to make sure that no stale sends // happen after a t.stop or t.modify, but we cannot hold t.mu @@ -1044,7 +1217,21 @@ func (t *timer) unlockAndRun(now int64) { // and double-check that t.seq is still the seq value we saw above. // If not, the timer has been updated and we should skip the send. // We skip the send by reassigning f to a no-op function. + // + // The isSending field tells t.stop or t.modify that we have + // started to send the value. That lets them correctly return + // true meaning that no value was sent. lock(&t.sendLock) + + if t.period == 0 { + // We are committed to possibly sending a value + // based on seq, so no need to keep telling + // stop/modify that we are sending. + if t.isSending.Add(-1) < 0 { + throw("mismatched isSending updates") + } + } + if t.seq != seq { f = func(any, uintptr, int64) {} } @@ -1056,6 +1243,17 @@ func (t *timer) unlockAndRun(now int64) { unlock(&t.sendLock) } + if bubble != nil { + gp := getg() + bubble.changegstatus(gp, _Grunning, _Gdead) + if raceenabled { + // Establish a happens-before between this timer event and + // the next synctest.Wait call. + racereleasemergeg(gp, bubble.raceaddr()) + } + gp.bubble = nil + } + if ts != nil { ts.lock() } @@ -1079,7 +1277,7 @@ func (ts *timers) verify() { // The heap is timerHeapN-ary. See siftupTimer and siftdownTimer. p := int(uint(i-1) / timerHeapN) - if tw.when < ts.heap[p].when { + if tw.less(ts.heap[p]) { print("bad timer heap at ", i, ": ", p, ": ", ts.heap[p].when, ", ", i, ": ", tw.when, "\n") throw("bad timer heap") } @@ -1157,13 +1355,12 @@ func (ts *timers) siftUp(i int) { badTimer() } tw := heap[i] - when := tw.when - if when <= 0 { + if tw.when <= 0 { badTimer() } for i > 0 { p := int(uint(i-1) / timerHeapN) // parent - if when >= heap[p].when { + if !tw.less(heap[p]) { break } heap[i] = heap[p] @@ -1186,8 +1383,7 @@ func (ts *timers) siftDown(i int) { return } tw := heap[i] - when := tw.when - if when <= 0 { + if tw.when <= 0 { badTimer() } for { @@ -1195,11 +1391,11 @@ func (ts *timers) siftDown(i int) { if leftChild >= n { break } - w := when + w := tw c := -1 for j, tw := range heap[leftChild:min(leftChild+timerHeapN, n)] { - if tw.when < w { - w = tw.when + if tw.less(w) { + w = tw c = leftChild + j } } @@ -1240,7 +1436,11 @@ func badTimer() { // maybeRunChan checks whether the timer needs to run // to send a value to its associated channel. If so, it does. // The timer must not be locked. -func (t *timer) maybeRunChan() { +func (t *timer) maybeRunChan(c *hchan) { + if t.isFake && getg().bubble != c.bubble { + // This should have been checked by the caller, but check just in case. + fatal("synctest timer accessed from outside bubble") + } if t.astate.Load()&timerHeaped != 0 { // If the timer is in the heap, the ordinary timer code // is in charge of sending when appropriate. @@ -1249,6 +1449,9 @@ func (t *timer) maybeRunChan() { t.lock() now := nanotime() + if t.isFake { + now = getg().bubble.now + } if t.state&timerHeaped != 0 || t.when == 0 || t.when > now { t.trace("maybeRunChan-") // Timer in the heap, or not running at all, or not triggered. @@ -1257,7 +1460,7 @@ func (t *timer) maybeRunChan() { } t.trace("maybeRunChan+") systemstack(func() { - t.unlockAndRun(now) + t.unlockAndRun(now, c.bubble) }) } @@ -1267,6 +1470,11 @@ func (t *timer) maybeRunChan() { // adding it if needed. func blockTimerChan(c *hchan) { t := c.timer + if t.isFake && c.bubble != getg().bubble { + // This should have been checked by the caller, but check just in case. + fatal("synctest timer accessed from outside bubble") + } + t.lock() t.trace("blockTimerChan") if !t.isChan { diff --git a/src/runtime/time_linux_amd64.s b/src/runtime/time_linux_amd64.s index 1416d232304f39..fa9561b25b5696 100644 --- a/src/runtime/time_linux_amd64.s +++ b/src/runtime/time_linux_amd64.s @@ -10,7 +10,7 @@ #define SYS_clock_gettime 228 -// func time.now() (sec int64, nsec int32, mono int64) +// func now() (sec int64, nsec int32, mono int64) TEXT time·now(SB),NOSPLIT,$16-24 MOVQ SP, R12 // Save old SP; R12 unchanged by C code. diff --git a/src/runtime/time_plan9.go b/src/runtime/time_plan9.go new file mode 100644 index 00000000000000..0bd165d07c097f --- /dev/null +++ b/src/runtime/time_plan9.go @@ -0,0 +1,24 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !faketime + +package runtime + +import _ "unsafe" // for go:linkname + +// Do not remove or change the type signature. +// See comment in timestub.go. +// +//go:linkname time_now time.now +func time_now() (sec int64, nsec int32, mono int64) { + var t [4]uint64 + if readtime(&t[0], 1, 4) == 4 { + mono = int64(frombe(t[3])) // new kernel, use monotonic time + } else { + mono = int64(frombe(t[0])) // old kernel, fall back to unix time + } + sec, nsec = timesplit(frombe(t[0])) + return +} diff --git a/src/runtime/time_test.go b/src/runtime/time_test.go index 7ac86998c68a0c..92ffe7f8f4eeea 100644 --- a/src/runtime/time_test.go +++ b/src/runtime/time_test.go @@ -23,7 +23,7 @@ func TestFakeTime(t *testing.T) { // Faketime is advanced in checkdead. External linking brings in cgo, // causing checkdead not working. - testenv.MustInternalLink(t, false) + testenv.MustInternalLink(t, deadlockBuildTypes) t.Parallel() diff --git a/src/runtime/time_windows_arm.s b/src/runtime/time_windows_arm.s deleted file mode 100644 index ff5686d9c41139..00000000000000 --- a/src/runtime/time_windows_arm.s +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !faketime - -#include "go_asm.h" -#include "textflag.h" -#include "time_windows.h" - -TEXT time·now(SB),NOSPLIT,$0-20 - MOVW $_INTERRUPT_TIME, R3 -loop: - MOVW time_hi1(R3), R1 - DMB MB_ISH - MOVW time_lo(R3), R0 - DMB MB_ISH - MOVW time_hi2(R3), R2 - CMP R1, R2 - BNE loop - - // wintime = R1:R0, multiply by 100 - MOVW $100, R2 - MULLU R0, R2, (R4, R3) // R4:R3 = R1:R0 * R2 - MULA R1, R2, R4, R4 - - // wintime*100 = R4:R3 - MOVW R3, mono+12(FP) - MOVW R4, mono+16(FP) - - MOVW $_SYSTEM_TIME, R3 -wall: - MOVW time_hi1(R3), R1 - DMB MB_ISH - MOVW time_lo(R3), R0 - DMB MB_ISH - MOVW time_hi2(R3), R2 - CMP R1, R2 - BNE wall - - // w = R1:R0 in 100ns untis - // convert to Unix epoch (but still 100ns units) - #define delta 116444736000000000 - SUB.S $(delta & 0xFFFFFFFF), R0 - SBC $(delta >> 32), R1 - - // Convert to nSec - MOVW $100, R2 - MULLU R0, R2, (R4, R3) // R4:R3 = R1:R0 * R2 - MULA R1, R2, R4, R4 - // w = R2:R1 in nSec - MOVW R3, R1 // R4:R3 -> R2:R1 - MOVW R4, R2 - - // multiply nanoseconds by reciprocal of 10**9 (scaled by 2**61) - // to get seconds (96 bit scaled result) - MOVW $0x89705f41, R3 // 2**61 * 10**-9 - MULLU R1,R3,(R6,R5) // R7:R6:R5 = R2:R1 * R3 - MOVW $0,R7 - MULALU R2,R3,(R7,R6) - - // unscale by discarding low 32 bits, shifting the rest by 29 - MOVW R6>>29,R6 // R7:R6 = (R7:R6:R5 >> 61) - ORR R7<<3,R6 - MOVW R7>>29,R7 - - // subtract (10**9 * sec) from nsec to get nanosecond remainder - MOVW $1000000000, R5 // 10**9 - MULLU R6,R5,(R9,R8) // R9:R8 = R7:R6 * R5 - MULA R7,R5,R9,R9 - SUB.S R8,R1 // R2:R1 -= R9:R8 - SBC R9,R2 - - // because reciprocal was a truncated repeating fraction, quotient - // may be slightly too small -- adjust to make remainder < 10**9 - CMP R5,R1 // if remainder > 10**9 - SUB.HS R5,R1 // remainder -= 10**9 - ADD.HS $1,R6 // sec += 1 - - MOVW R6,sec_lo+0(FP) - MOVW R7,sec_hi+4(FP) - MOVW R1,nsec+8(FP) - RET - diff --git a/src/runtime/timestub.go b/src/runtime/timestub.go index da8699b5ee60da..eb91c022afa8c7 100644 --- a/src/runtime/timestub.go +++ b/src/runtime/timestub.go @@ -5,7 +5,7 @@ // Declarations for operating systems implementing time.now // indirectly, in terms of walltime and nanotime assembly. -//go:build !faketime && !windows && !(linux && amd64) +//go:build !faketime && !windows && !(linux && amd64) && !plan9 package runtime diff --git a/src/runtime/timestub2.go b/src/runtime/timestub2.go index 49bfeb60c88de2..336bac4b98e4ce 100644 --- a/src/runtime/timestub2.go +++ b/src/runtime/timestub2.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !aix && !darwin && !freebsd && !openbsd && !solaris && !wasip1 && !windows && !(linux && amd64) +//go:build !aix && !darwin && !freebsd && !openbsd && !solaris && !wasip1 && !windows && !(linux && amd64) && !plan9 package runtime diff --git a/src/runtime/trace.go b/src/runtime/trace.go index bc2978bb4bf6ef..2c712469ea66bc 100644 --- a/src/runtime/trace.go +++ b/src/runtime/trace.go @@ -21,6 +21,7 @@ package runtime import ( "internal/runtime/atomic" + "internal/trace/tracev2" "unsafe" ) @@ -51,9 +52,10 @@ var trace struct { // State for the trace reader goroutine. // // Protected by trace.lock. - readerGen atomic.Uintptr // the generation the reader is currently reading for - flushedGen atomic.Uintptr // the last completed generation - headerWritten bool // whether ReadTrace has emitted trace header + readerGen atomic.Uintptr // the generation the reader is currently reading for + flushedGen atomic.Uintptr // the last completed generation + headerWritten bool // whether ReadTrace has emitted trace header + endOfGenerationWritten bool // whether readTrace has emitted the end of the generation signal // doneSema is used to synchronize the reader and traceAdvance. Specifically, // it notifies traceAdvance that the reader is done with a generation. @@ -184,6 +186,10 @@ func StartTrace() error { // Register some basic strings in the string tables. traceRegisterLabelsAndReasons(firstGen) + // N.B. This may block for quite a while to get a frequency estimate. Do it + // here to minimize the time that the world is stopped. + frequency := traceClockUnitsPerSecond() + // Stop the world. // // The purpose of stopping the world is to make sure that no goroutine is in a @@ -280,8 +286,9 @@ func StartTrace() error { // // N.B. This will also emit a status event for this goroutine. tl := traceAcquire() - tl.Gomaxprocs(gomaxprocs) // Get this as early in the trace as possible. See comment in traceAdvance. - tl.STWStart(stwStartTrace) // We didn't trace this above, so trace it now. + traceSyncBatch(firstGen, frequency) // Get this as early in the trace as possible. See comment in traceAdvance. + tl.Gomaxprocs(gomaxprocs) // Get this as early in the trace as possible. See comment in traceAdvance. + tl.STWStart(stwStartTrace) // We didn't trace this above, so trace it now. // Record the fact that a GC is active, if applicable. if gcphase == _GCmark || gcphase == _GCmarktermination { @@ -324,7 +331,7 @@ func StopTrace() { // // traceAdvanceSema must not be held. // -// traceAdvance is called by golang.org/x/exp/trace using linkname. +// traceAdvance is called by runtime/trace and golang.org/x/exp/trace using linkname. // //go:linkname traceAdvance func traceAdvance(stopTrace bool) { @@ -340,12 +347,6 @@ func traceAdvance(stopTrace bool) { return } - // Write an EvFrequency event for this generation. - // - // N.B. This may block for quite a while to get a good frequency estimate, so make sure we do - // this here and not e.g. on the trace reader. - traceFrequency(gen) - // Collect all the untraced Gs. type untracedG struct { gp *g @@ -375,7 +376,7 @@ func traceAdvance(stopTrace bool) { me := getg().m.curg // We don't have to handle this G status transition because we // already eliminated ourselves from consideration above. - casGToWaitingForGC(me, _Grunning, waitReasonTraceGoroutineStatus) + casGToWaitingForSuspendG(me, _Grunning, waitReasonTraceGoroutineStatus) // We need to suspend and take ownership of the G to safely read its // goid. Note that we can't actually emit the event at this point // because we might stop the G in a window where it's unsafe to write @@ -395,7 +396,7 @@ func traceAdvance(stopTrace bool) { ug.status = readgstatus(s.g) &^ _Gscan ug.waitreason = s.g.waitreason ug.inMarkAssist = s.g.inMarkAssist - ug.stackID = traceStack(0, gp, gen) + ug.stackID = traceStack(0, gp, &trace.stackTab[gen%2]) } resumeG(s) casgstatus(me, _Gwaiting, _Grunning) @@ -410,6 +411,10 @@ func traceAdvance(stopTrace bool) { traceRegisterLabelsAndReasons(traceNextGen(gen)) } + // N.B. This may block for quite a while to get a frequency estimate. Do it + // here to minimize the time that we prevent the world from stopping. + frequency := traceClockUnitsPerSecond() + // Now that we've done some of the heavy stuff, prevent the world from stopping. // This is necessary to ensure the consistency of the STW events. If we're feeling // adventurous we could lift this restriction and add a STWActive event, but the @@ -441,14 +446,16 @@ func traceAdvance(stopTrace bool) { trace.gen.Store(traceNextGen(gen)) } - // Emit a ProcsChange event so we have one on record for each generation. - // Let's emit it as soon as possible so that downstream tools can rely on the value - // being there fairly soon in a generation. + // Emit a sync batch which contains a ClockSnapshot. Also emit a ProcsChange + // event so we have one on record for each generation. Let's emit it as soon + // as possible so that downstream tools can rely on the value being there + // fairly soon in a generation. // // It's important that we do this before allowing stop-the-worlds again, // because the procs count could change. if !stopTrace { tl := traceAcquire() + traceSyncBatch(tl.gen, frequency) tl.Gomaxprocs(gomaxprocs) traceRelease(tl) } @@ -747,9 +754,8 @@ func traceRegisterLabelsAndReasons(gen uintptr) { // was on has been returned, ReadTrace returns nil. The caller must copy the // returned data before calling ReadTrace again. // ReadTrace must be called from one goroutine at a time. -func ReadTrace() []byte { +func ReadTrace() (buf []byte) { top: - var buf []byte var park bool systemstack(func() { buf, park = readTrace0() @@ -777,7 +783,6 @@ top: }, nil, waitReasonTraceReaderBlocked, traceBlockSystemGoroutine, 2) goto top } - return buf } @@ -820,7 +825,7 @@ func readTrace0() (buf []byte, park bool) { if !trace.headerWritten { trace.headerWritten = true unlock(&trace.lock) - return []byte("go 1.23 trace\x00\x00\x00"), false + return []byte("go 1.26 trace\x00\x00\x00"), false } // Read the next buffer. @@ -844,6 +849,17 @@ func readTrace0() (buf []byte, park bool) { // is waiting on the reader to finish flushing the last generation so that it // can continue to advance. if trace.flushedGen.Load() == gen { + // Write out the internal in-band end-of-generation signal. + if !trace.endOfGenerationWritten { + trace.endOfGenerationWritten = true + unlock(&trace.lock) + return []byte{byte(tracev2.EvEndOfGeneration)}, false + } + + // Reset the flag. + trace.endOfGenerationWritten = false + + // Handle shutdown. if trace.shutdown.Load() { unlock(&trace.lock) @@ -863,6 +879,8 @@ func readTrace0() (buf []byte, park bool) { // read. We're done. return nil, false } + // Handle advancing to the next generation. + // The previous gen has had all of its buffers flushed, and // there's nothing else for us to read. Advance the generation // we're reading from and try again. @@ -921,7 +939,7 @@ func traceReader() *g { // scheduled and should be. Callers should first check that // (traceEnabled() || traceShuttingDown()) is true. func traceReaderAvailable() *g { - // There are three conditions under which we definitely want to schedule + // There are two conditions under which we definitely want to schedule // the reader: // - The reader is lagging behind in finishing off the last generation. // In this case, trace buffers could even be empty, but the trace @@ -930,12 +948,10 @@ func traceReaderAvailable() *g { // - The reader has pending work to process for it's reader generation // (assuming readerGen is not lagging behind). Note that we also want // to be careful *not* to schedule the reader if there's no work to do. - // - The trace is shutting down. The trace stopper blocks on the reader - // to finish, much like trace advancement. // // We also want to be careful not to schedule the reader if there's no // reason to. - if trace.flushedGen.Load() == trace.readerGen.Load() || trace.workAvailable.Load() || trace.shutdown.Load() { + if trace.flushedGen.Load() == trace.readerGen.Load() || trace.workAvailable.Load() { return trace.reader.Load() } return nil diff --git a/src/runtime/trace/batch.go b/src/runtime/trace/batch.go new file mode 100644 index 00000000000000..f8b0a96b3f06d9 --- /dev/null +++ b/src/runtime/trace/batch.go @@ -0,0 +1,88 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package trace + +import ( + "fmt" + "internal/trace/tracev2" +) + +// timestamp is an unprocessed timestamp. +type timestamp uint64 + +type batch struct { + time timestamp + gen uint64 + data []byte +} + +// readBatch copies b and parses the trace batch header inside. +// Returns the batch, bytes read, and an error. +func readBatch(b []byte) (batch, uint64, error) { + if len(b) == 0 { + return batch{}, 0, fmt.Errorf("batch is empty") + } + data := make([]byte, len(b)) + copy(data, b) + + // Read batch header byte. + if typ := tracev2.EventType(b[0]); typ == tracev2.EvEndOfGeneration { + if len(b) != 1 { + return batch{}, 1, fmt.Errorf("unexpected end of generation in batch of size >1") + } + return batch{data: data}, 1, nil + } + if typ := tracev2.EventType(b[0]); typ != tracev2.EvEventBatch && typ != tracev2.EvExperimentalBatch { + return batch{}, 1, fmt.Errorf("expected batch event, got event %d", typ) + } + total := 1 + b = b[1:] + + // Read the generation + gen, n, err := readUvarint(b) + if err != nil { + return batch{}, uint64(total + n), fmt.Errorf("error reading batch gen: %w", err) + } + total += n + b = b[n:] + + // Read the M (discard it). + _, n, err = readUvarint(b) + if err != nil { + return batch{}, uint64(total + n), fmt.Errorf("error reading batch M ID: %w", err) + } + total += n + b = b[n:] + + // Read the timestamp. + ts, n, err := readUvarint(b) + if err != nil { + return batch{}, uint64(total + n), fmt.Errorf("error reading batch timestamp: %w", err) + } + total += n + b = b[n:] + + // Read the size of the batch to follow. + size, n, err := readUvarint(b) + if err != nil { + return batch{}, uint64(total + n), fmt.Errorf("error reading batch size: %w", err) + } + if size > tracev2.MaxBatchSize { + return batch{}, uint64(total + n), fmt.Errorf("invalid batch size %d, maximum is %d", size, tracev2.MaxBatchSize) + } + total += n + total += int(size) + if total != len(data) { + return batch{}, uint64(total), fmt.Errorf("expected complete batch") + } + data = data[:total] + + // Return the batch. + return batch{ + gen: gen, + time: timestamp(ts), + data: data, + }, uint64(total), nil +} diff --git a/src/runtime/trace/encoding.go b/src/runtime/trace/encoding.go new file mode 100644 index 00000000000000..46990c658c5b47 --- /dev/null +++ b/src/runtime/trace/encoding.go @@ -0,0 +1,50 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package trace + +import ( + "errors" +) + +// maxVarintLenN is the maximum length of a varint-encoded N-bit integer. +const maxVarintLen64 = 10 + +var ( + errOverflow = errors.New("binary: varint overflows a 64-bit integer") + errEOB = errors.New("binary: end of buffer") +) + +// TODO deduplicate this function. +func readUvarint(b []byte) (uint64, int, error) { + var x uint64 + var s uint + var byt byte + for i := 0; i < maxVarintLen64 && i < len(b); i++ { + byt = b[i] + if byt < 0x80 { + if i == maxVarintLen64-1 && byt > 1 { + return x, i, errOverflow + } + return x | uint64(byt)<= 0x80 { + buf[i] = byte(x) | 0x80 + x >>= 7 + i++ + } + buf[i] = byte(x) + return i + 1 +} diff --git a/src/runtime/trace/flightrecorder.go b/src/runtime/trace/flightrecorder.go new file mode 100644 index 00000000000000..99ee4d060dcdf3 --- /dev/null +++ b/src/runtime/trace/flightrecorder.go @@ -0,0 +1,182 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package trace + +import ( + "fmt" + "io" + "sync" + "time" + _ "unsafe" // added for go linkname usage +) + +// FlightRecorder represents a single consumer of a Go execution +// trace. +// It tracks a moving window over the execution trace produced by +// the runtime, always containing the most recent trace data. +// +// At most one flight recorder may be active at any given time, +// though flight recording is allowed to be concurrently active +// with a trace consumer using trace.Start. +// This restriction of only a single flight recorder may be removed +// in the future. +type FlightRecorder struct { + err error + + // State specific to the recorder. + header [16]byte + active rawGeneration + ringMu sync.Mutex + ring []rawGeneration + freq frequency // timestamp conversion factor, from the runtime + + // Externally-set options. + targetSize uint64 + targetPeriod time.Duration + + enabled bool // whether the flight recorder is enabled. + writing sync.Mutex // protects concurrent calls to WriteTo + + // The values of targetSize and targetPeriod we've committed to since the last Start. + wantSize uint64 + wantDur time.Duration +} + +// NewFlightRecorder creates a new flight recorder from the provided configuration. +func NewFlightRecorder(cfg FlightRecorderConfig) *FlightRecorder { + fr := new(FlightRecorder) + if cfg.MaxBytes != 0 { + fr.targetSize = cfg.MaxBytes + } else { + fr.targetSize = 10 << 20 // 10 MiB. + } + + if cfg.MinAge != 0 { + fr.targetPeriod = cfg.MinAge + } else { + fr.targetPeriod = 10 * time.Second + } + return fr +} + +// Start activates the flight recorder and begins recording trace data. +// Only one call to trace.Start may be active at any given time. +// In addition, currently only one flight recorder may be active in the program. +// Returns an error if the flight recorder cannot be started or is already started. +func (fr *FlightRecorder) Start() error { + if fr.enabled { + return fmt.Errorf("cannot enable a enabled flight recorder") + } + fr.wantSize = fr.targetSize + fr.wantDur = fr.targetPeriod + fr.err = nil + fr.freq = frequency(1.0 / (float64(runtime_traceClockUnitsPerSecond()) / 1e9)) + + // Start tracing, data is sent to a recorder which forwards it to our own + // storage. + if err := tracing.subscribeFlightRecorder(&recorder{r: fr}); err != nil { + return err + } + + fr.enabled = true + return nil +} + +// Stop ends recording of trace data. It blocks until any concurrent WriteTo calls complete. +func (fr *FlightRecorder) Stop() { + if !fr.enabled { + return + } + fr.enabled = false + tracing.unsubscribeFlightRecorder() + + // Reset all state. No need to lock because the reader has already exited. + fr.active = rawGeneration{} + fr.ring = nil +} + +// Enabled returns true if the flight recorder is active. +// Specifically, it will return true if Start did not return an error, and Stop has not yet been called. +// It is safe to call from multiple goroutines simultaneously. +func (fr *FlightRecorder) Enabled() bool { return fr.enabled } + +// WriteTo snapshots the moving window tracked by the flight recorder. +// The snapshot is expected to contain data that is up-to-date as of when WriteTo is called, +// though this is not a hard guarantee. +// Only one goroutine may execute WriteTo at a time. +// An error is returned upon failure to write to w, if another WriteTo call is already in-progress, +// or if the flight recorder is inactive. +func (fr *FlightRecorder) WriteTo(w io.Writer) (n int64, err error) { + if !fr.enabled { + return 0, fmt.Errorf("cannot snapshot a disabled flight recorder") + } + if !fr.writing.TryLock() { + // Indicates that a call to WriteTo was made while one was already in progress. + // If the caller of WriteTo sees this error, they should use the result from the other call to WriteTo. + return 0, fmt.Errorf("call to WriteTo for trace.FlightRecorder already in progress") + } + defer fr.writing.Unlock() + + // Force a global buffer flush. + runtime_traceAdvance(false) + + // Now that everything has been flushed and written, grab whatever we have. + // + // N.B. traceAdvance blocks until the tracer goroutine has actually written everything + // out, which means the generation we just flushed must have been already been observed + // by the recorder goroutine. Because we flushed twice, the first flush is guaranteed to + // have been both completed *and* processed by the recorder goroutine. + fr.ringMu.Lock() + gens := fr.ring + fr.ringMu.Unlock() + + // Write the header. + nw, err := w.Write(fr.header[:]) + if err != nil { + return int64(nw), err + } + n += int64(nw) + + // Write all the data. + for _, gen := range gens { + for _, data := range gen.batches { + // Write batch data. + nw, err = w.Write(data) + n += int64(nw) + if err != nil { + return n, err + } + } + } + return n, nil +} + +type FlightRecorderConfig struct { + // MinAge is a lower bound on the age of an event in the flight recorder's window. + // + // The flight recorder will strive to promptly discard events older than the minimum age, + // but older events may appear in the window snapshot. The age setting will always be + // overridden by MaxBytes. + // + // If this is 0, the minimum age is implementation defined, but can be assumed to be on the order + // of seconds. + MinAge time.Duration + + // MaxBytes is an upper bound on the size of the window in bytes. + // + // This setting takes precedence over MinAge. + // However, it does not make any guarantees on the size of the data WriteTo will write, + // nor does it guarantee memory overheads will always stay below MaxBytes. Treat it + // as a hint. + // + // If this is 0, the maximum size is implementation defined. + MaxBytes uint64 +} + +//go:linkname runtime_traceClockUnitsPerSecond +func runtime_traceClockUnitsPerSecond() uint64 + +//go:linkname runtime_traceAdvance runtime.traceAdvance +func runtime_traceAdvance(stopTrace bool) diff --git a/src/runtime/trace/flightrecorder_test.go b/src/runtime/trace/flightrecorder_test.go new file mode 100644 index 00000000000000..221f9c6b673813 --- /dev/null +++ b/src/runtime/trace/flightrecorder_test.go @@ -0,0 +1,316 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package trace_test + +import ( + "bytes" + "context" + inttrace "internal/trace" + "internal/trace/testtrace" + "io" + "runtime/trace" + "slices" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestFlightRecorderDoubleStart(t *testing.T) { + fr := trace.NewFlightRecorder(trace.FlightRecorderConfig{}) + if err := fr.Start(); err != nil { + t.Fatalf("unexpected error on Start: %v", err) + } + if err := fr.Start(); err == nil { + t.Fatalf("expected error from double Start: %v", err) + } + fr.Stop() +} + +func TestFlightRecorderEnabled(t *testing.T) { + fr := trace.NewFlightRecorder(trace.FlightRecorderConfig{}) + + if fr.Enabled() { + t.Fatal("flight recorder is enabled, but never started") + } + if err := fr.Start(); err != nil { + t.Fatalf("unexpected error on Start: %v", err) + } + if !fr.Enabled() { + t.Fatal("flight recorder is not enabled, but started") + } + fr.Stop() + if fr.Enabled() { + t.Fatal("flight recorder is enabled, but stopped") + } +} + +func TestFlightRecorderWriteToDisabled(t *testing.T) { + var buf bytes.Buffer + + fr := trace.NewFlightRecorder(trace.FlightRecorderConfig{}) + if n, err := fr.WriteTo(&buf); err == nil { + t.Fatalf("successfully wrote %d bytes from disabled flight recorder", n) + } + if err := fr.Start(); err != nil { + t.Fatalf("unexpected error on Start: %v", err) + } + fr.Stop() + if n, err := fr.WriteTo(&buf); err == nil { + t.Fatalf("successfully wrote %d bytes from disabled flight recorder", n) + } +} + +func TestFlightRecorderConcurrentWriteTo(t *testing.T) { + fr := trace.NewFlightRecorder(trace.FlightRecorderConfig{}) + if err := fr.Start(); err != nil { + t.Fatalf("unexpected error on Start: %v", err) + } + + // Start two goroutines to write snapshots. + // + // Most of the time one will fail and one will succeed, but we don't require this. + // Due to a variety of factors, it's definitely possible for them both to succeed. + // However, at least one should succeed. + var bufs [2]bytes.Buffer + var wg sync.WaitGroup + var successes atomic.Uint32 + for i := range bufs { + wg.Add(1) + go func() { + defer wg.Done() + + n, err := fr.WriteTo(&bufs[i]) + // TODO(go.dev/issue/63185) was an exported error. Consider refactoring. + if err != nil && err.Error() == "call to WriteTo for trace.FlightRecorder already in progress" { + if n != 0 { + t.Errorf("(goroutine %d) WriteTo bytes written is non-zero for early bail out: %d", i, n) + } + return + } + if err != nil { + t.Errorf("(goroutine %d) failed to write snapshot for unexpected reason: %v", i, err) + } + successes.Add(1) + + if n == 0 { + t.Errorf("(goroutine %d) wrote invalid trace of zero bytes in size", i) + } + if n != int64(bufs[i].Len()) { + t.Errorf("(goroutine %d) trace length doesn't match WriteTo result: got %d, want %d", i, n, int64(bufs[i].Len())) + } + }() + } + wg.Wait() + + // Stop tracing. + fr.Stop() + + // Make sure at least one succeeded to write. + if successes.Load() == 0 { + t.Fatal("expected at least one success to write a snapshot, got zero") + } + + // Validate the traces that came out. + for i := range bufs { + buf := &bufs[i] + if buf.Len() == 0 { + continue + } + testReader(t, buf.Bytes(), testtrace.ExpectSuccess()) + } +} + +func TestFlightRecorder(t *testing.T) { + testFlightRecorder(t, trace.NewFlightRecorder(trace.FlightRecorderConfig{}), func(snapshot func()) { + snapshot() + }) +} + +func TestFlightRecorderStartStop(t *testing.T) { + fr := trace.NewFlightRecorder(trace.FlightRecorderConfig{}) + for i := 0; i < 5; i++ { + testFlightRecorder(t, fr, func(snapshot func()) { + snapshot() + }) + } +} + +func TestFlightRecorderLog(t *testing.T) { + tr := testFlightRecorder(t, trace.NewFlightRecorder(trace.FlightRecorderConfig{}), func(snapshot func()) { + trace.Log(context.Background(), "message", "hello") + snapshot() + }) + + // Prepare to read the trace snapshot. + r, err := inttrace.NewReader(bytes.NewReader(tr)) + if err != nil { + t.Fatalf("unexpected error creating trace reader: %v", err) + } + + // Find the log message in the trace. + found := false + for { + ev, err := r.ReadEvent() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("unexpected error reading trace: %v", err) + } + if !found && ev.Kind() == inttrace.EventLog { + log := ev.Log() + found = log.Category == "message" && log.Message == "hello" + } + } + if !found { + t.Errorf("failed to find expected log message (%q, %q) in snapshot", "message", "hello") + } +} + +func TestFlightRecorderGenerationCount(t *testing.T) { + test := func(t *testing.T, fr *trace.FlightRecorder) { + tr := testFlightRecorder(t, fr, func(snapshot func()) { + // Sleep to let a few generations pass. + time.Sleep(3 * time.Second) + snapshot() + }) + + // Prepare to read the trace snapshot. + r, err := inttrace.NewReader(bytes.NewReader(tr)) + if err != nil { + t.Fatalf("unexpected error creating trace reader: %v", err) + } + + // Make sure there are Sync events: at the start and end. + var syncs []int + evs := 0 + for { + ev, err := r.ReadEvent() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("unexpected error reading trace: %v", err) + } + if ev.Kind() == inttrace.EventSync { + syncs = append(syncs, evs) + } + evs++ + } + const wantMaxSyncs = 3 + if len(syncs) > wantMaxSyncs { + t.Errorf("expected at most %d sync events, found %d at %d", + wantMaxSyncs, len(syncs), syncs) + } + ends := []int{syncs[0], syncs[len(syncs)-1]} + if wantEnds := []int{0, evs - 1}; !slices.Equal(wantEnds, ends) { + t.Errorf("expected a sync event at each end of the trace, found sync events at %d instead of %d", + ends, wantEnds) + } + } + t.Run("MinAge", func(t *testing.T) { + fr := trace.NewFlightRecorder(trace.FlightRecorderConfig{MinAge: time.Millisecond}) + test(t, fr) + }) + t.Run("MaxBytes", func(t *testing.T) { + fr := trace.NewFlightRecorder(trace.FlightRecorderConfig{MaxBytes: 16}) + test(t, fr) + }) +} + +type flightRecorderTestFunc func(snapshot func()) + +func testFlightRecorder(t *testing.T, fr *trace.FlightRecorder, f flightRecorderTestFunc) []byte { + if trace.IsEnabled() { + t.Skip("cannot run flight recorder tests when tracing is enabled") + } + + // Start the flight recorder. + if err := fr.Start(); err != nil { + t.Fatalf("unexpected error on Start: %v", err) + } + + // Set up snapshot callback. + var buf bytes.Buffer + callback := func() { + n, err := fr.WriteTo(&buf) + if err != nil { + t.Errorf("unexpected failure during flight recording: %v", err) + return + } + if n < 16 { + t.Errorf("expected a trace size of at least 16 bytes, got %d", n) + } + if n != int64(buf.Len()) { + t.Errorf("WriteTo result doesn't match trace size: got %d, want %d", n, int64(buf.Len())) + } + } + + // Call the test function. + f(callback) + + // Stop the flight recorder. + fr.Stop() + + // Get the trace bytes; we don't want to use the Buffer as a Reader directly + // since we may want to consume this data more than once. + traceBytes := buf.Bytes() + + // Parse the trace to make sure it's not broken. + testReader(t, traceBytes, testtrace.ExpectSuccess()) + return traceBytes +} + +func testReader(t *testing.T, tb []byte, exp *testtrace.Expectation) { + r, err := inttrace.NewReader(bytes.NewReader(tb)) + if err != nil { + if err := exp.Check(err); err != nil { + t.Error(err) + } + return + } + v := testtrace.NewValidator() + v.SkipClockSnapshotChecks() + for { + ev, err := r.ReadEvent() + if err == io.EOF { + break + } + if err != nil { + if err := exp.Check(err); err != nil { + t.Error(err) + } + return + } + if err := v.Event(ev); err != nil { + t.Error(err) + } + } + if err := exp.Check(nil); err != nil { + t.Error(err) + } + if t.Failed() || *dumpTraces { + testtrace.Dump(t, "trace", tb, *dumpTraces) + } +} + +func TestTraceAndFlightRecorder(t *testing.T) { + var tBuf, frBuf bytes.Buffer + if err := trace.Start(&tBuf); err != nil { + t.Errorf("unable to start execution tracer: %s", err) + } + fr := trace.NewFlightRecorder(trace.FlightRecorderConfig{MaxBytes: 16}) + fr.Start() + fr.WriteTo(&frBuf) + fr.Stop() + trace.Stop() + if tBuf.Len() == 0 || frBuf.Len() == 0 { + t.Errorf("None of these should be equal to zero: %d %d", tBuf.Len(), frBuf.Len()) + } + if tBuf.Len() <= frBuf.Len() { + t.Errorf("trace should be longer than the flight recorder: trace=%d flight record=%d", tBuf.Len(), frBuf.Len()) + } +} diff --git a/src/runtime/trace/recorder.go b/src/runtime/trace/recorder.go new file mode 100644 index 00000000000000..4f2d3aa92a4124 --- /dev/null +++ b/src/runtime/trace/recorder.go @@ -0,0 +1,144 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package trace + +import ( + "fmt" + "slices" + "time" + _ "unsafe" // added for go linkname usage +) + +// A recorder receives bytes from the runtime tracer, processes it. +type recorder struct { + r *FlightRecorder + + headerReceived bool +} + +func (w *recorder) Write(b []byte) (n int, err error) { + r := w.r + + defer func() { + if err != nil { + // Propagate errors to the flightrecorder. + if r.err == nil { + r.err = err + } + } + }() + + if !w.headerReceived { + if len(b) < len(r.header) { + return 0, fmt.Errorf("expected at least %d bytes in the first write", len(r.header)) + } + r.header = ([16]byte)(b[:16]) + n += 16 + w.headerReceived = true + } + if len(b) == n { + return 0, nil + } + ba, nb, err := readBatch(b[n:]) // Every write from the runtime is guaranteed to be a complete batch. + if err != nil { + return len(b) - int(nb) - n, err + } + n += int(nb) + + // Append the batch to the current generation. + if ba.gen != 0 && r.active.gen == 0 { + r.active.gen = ba.gen + } + if ba.time != 0 && (r.active.minTime == 0 || r.active.minTime > r.freq.mul(ba.time)) { + r.active.minTime = r.freq.mul(ba.time) + } + r.active.size += len(ba.data) + r.active.batches = append(r.active.batches, ba.data) + + return len(b), nil +} + +func (w *recorder) endGeneration() { + r := w.r + + // Check if we're entering a new generation. + r.ringMu.Lock() + + // Get the current trace clock time. + now := traceTimeNow(r.freq) + + // Add the current generation to the ring. Make sure we always have at least one + // complete generation by putting the active generation onto the new list, regardless + // of whatever our settings are. + // + // N.B. Let's completely replace the ring here, so that WriteTo can just make a copy + // and not worry about aliasing. This creates allocations, but at a very low rate. + newRing := []rawGeneration{r.active} + size := r.active.size + for i := len(r.ring) - 1; i >= 0; i-- { + // Stop adding older generations if the new ring already exceeds the thresholds. + // This ensures we keep generations that cross a threshold, but not any that lie + // entirely outside it. + if uint64(size) > r.wantSize || now.Sub(newRing[len(newRing)-1].minTime) > r.wantDur { + break + } + size += r.ring[i].size + newRing = append(newRing, r.ring[i]) + } + slices.Reverse(newRing) + r.ring = newRing + r.ringMu.Unlock() + + // Start a new active generation. + r.active = rawGeneration{} +} + +type rawGeneration struct { + gen uint64 + size int + minTime eventTime + batches [][]byte +} + +func traceTimeNow(freq frequency) eventTime { + return freq.mul(timestamp(runtime_traceClockNow())) +} + +//go:linkname runtime_traceClockNow runtime.traceClockNow +func runtime_traceClockNow() uint64 + +// frequency is nanoseconds per timestamp unit. +type frequency float64 + +// mul multiplies an unprocessed to produce a time in nanoseconds. +func (f frequency) mul(t timestamp) eventTime { + return eventTime(float64(t) * float64(f)) +} + +// eventTime is a timestamp in nanoseconds. +// +// It corresponds to the monotonic clock on the platform that the +// trace was taken, and so is possible to correlate with timestamps +// for other traces taken on the same machine using the same clock +// (i.e. no reboots in between). +// +// The actual absolute value of the timestamp is only meaningful in +// relation to other timestamps from the same clock. +// +// BUG: Timestamps coming from traces on Windows platforms are +// only comparable with timestamps from the same trace. Timestamps +// across traces cannot be compared, because the system clock is +// not used as of Go 1.22. +// +// BUG: Traces produced by Go versions 1.21 and earlier cannot be +// compared with timestamps from other traces taken on the same +// machine. This is because the system clock was not used at all +// to collect those timestamps. +type eventTime int64 + +// Sub subtracts t0 from t, returning the duration in nanoseconds. +func (t eventTime) Sub(t0 eventTime) time.Duration { + return time.Duration(int64(t) - int64(t0)) +} diff --git a/src/runtime/trace/subscribe.go b/src/runtime/trace/subscribe.go new file mode 100644 index 00000000000000..a4d653dcaece92 --- /dev/null +++ b/src/runtime/trace/subscribe.go @@ -0,0 +1,204 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package trace + +import ( + "fmt" + "internal/trace/tracev2" + "io" + "runtime" + "sync" + "sync/atomic" + _ "unsafe" +) + +var tracing traceMultiplexer + +type traceMultiplexer struct { + sync.Mutex + enabled atomic.Bool + subscribers int + + subscribersMu sync.Mutex + traceStartWriter io.Writer + flightRecorder *recorder +} + +func (t *traceMultiplexer) subscribeFlightRecorder(r *recorder) error { + t.Lock() + defer t.Unlock() + + t.subscribersMu.Lock() + if t.flightRecorder != nil { + t.subscribersMu.Unlock() + return fmt.Errorf("flight recorder already enabled") + } + t.flightRecorder = r + t.subscribersMu.Unlock() + + if err := t.addedSubscriber(); err != nil { + t.subscribersMu.Lock() + t.flightRecorder = nil + t.subscribersMu.Unlock() + return err + } + return nil +} + +func (t *traceMultiplexer) unsubscribeFlightRecorder() error { + t.Lock() + defer t.Unlock() + + t.removingSubscriber() + + t.subscribersMu.Lock() + if t.flightRecorder == nil { + t.subscribersMu.Unlock() + return fmt.Errorf("attempt to unsubscribe missing flight recorder") + } + t.flightRecorder = nil + t.subscribersMu.Unlock() + + t.removedSubscriber() + return nil +} + +func (t *traceMultiplexer) subscribeTraceStartWriter(w io.Writer) error { + t.Lock() + defer t.Unlock() + + t.subscribersMu.Lock() + if t.traceStartWriter != nil { + t.subscribersMu.Unlock() + return fmt.Errorf("execution tracer already enabled") + } + t.traceStartWriter = w + t.subscribersMu.Unlock() + + if err := t.addedSubscriber(); err != nil { + t.subscribersMu.Lock() + t.traceStartWriter = nil + t.subscribersMu.Unlock() + return err + } + return nil +} + +func (t *traceMultiplexer) unsubscribeTraceStartWriter() { + t.Lock() + defer t.Unlock() + + t.removingSubscriber() + + t.subscribersMu.Lock() + if t.traceStartWriter == nil { + t.subscribersMu.Unlock() + return + } + t.traceStartWriter = nil + t.subscribersMu.Unlock() + + t.removedSubscriber() + return +} + +func (t *traceMultiplexer) addedSubscriber() error { + if t.enabled.Load() { + // This is necessary for the trace reader goroutine to pick up on the new subscriber. + runtime_traceAdvance(false) + } else { + if err := t.startLocked(); err != nil { + return err + } + } + t.subscribers++ + return nil +} + +func (t *traceMultiplexer) removingSubscriber() { + if t.subscribers == 0 { + return + } + t.subscribers-- + if t.subscribers == 0 { + runtime.StopTrace() + t.enabled.Store(false) + } else { + // This is necessary to avoid missing trace data when the system is under high load. + runtime_traceAdvance(false) + } +} + +func (t *traceMultiplexer) removedSubscriber() { + if t.subscribers > 0 { + // This is necessary for the trace reader goroutine to pick up on the new subscriber. + runtime_traceAdvance(false) + } +} + +func (t *traceMultiplexer) startLocked() error { + if err := runtime.StartTrace(); err != nil { + return err + } + + // Grab the trace reader goroutine's subscribers. + // + // We only update our subscribers if we see an end-of-generation + // signal from the runtime after this, so any new subscriptions + // or unsubscriptions must call traceAdvance to ensure the reader + // goroutine sees an end-of-generation signal. + t.subscribersMu.Lock() + flightRecorder := t.flightRecorder + traceStartWriter := t.traceStartWriter + t.subscribersMu.Unlock() + + go func() { + header := runtime.ReadTrace() + if traceStartWriter != nil { + traceStartWriter.Write(header) + } + if flightRecorder != nil { + flightRecorder.Write(header) + } + + for { + data := runtime.ReadTrace() + if data == nil { + break + } + if traceStartWriter != nil { + traceStartWriter.Write(data) + } + if flightRecorder != nil { + flightRecorder.Write(data) + } + if len(data) == 1 && tracev2.EventType(data[0]) == tracev2.EvEndOfGeneration { + if flightRecorder != nil { + flightRecorder.endGeneration() + } + + // Pick up any changes. + t.subscribersMu.Lock() + frIsNew := flightRecorder != t.flightRecorder && t.flightRecorder != nil + trIsNew := traceStartWriter != t.traceStartWriter && t.traceStartWriter != nil + flightRecorder = t.flightRecorder + traceStartWriter = t.traceStartWriter + t.subscribersMu.Unlock() + + if trIsNew { + traceStartWriter.Write(header) + } + if frIsNew { + flightRecorder.Write(header) + } + } + } + }() + t.enabled.Store(true) + return nil +} + +//go:linkname runtime_readTrace +func runtime_readTrace() (buf []byte) diff --git a/src/runtime/trace/subscribe_test.go b/src/runtime/trace/subscribe_test.go new file mode 100644 index 00000000000000..0e6c57cbc6d352 --- /dev/null +++ b/src/runtime/trace/subscribe_test.go @@ -0,0 +1,153 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package trace_test + +import ( + "bytes" + inttrace "internal/trace" + "internal/trace/testtrace" + "io" + "runtime" + "runtime/trace" + "slices" + "testing" +) + +func TestSubscribers(t *testing.T) { + validate := func(t *testing.T, source string, tr io.Reader) { + // Prepare to read the trace snapshot. + r, err := inttrace.NewReader(tr) + if err != nil { + t.Fatalf("unexpected error creating trace reader for %s: %v", source, err) + return + } + + v := testtrace.NewValidator() + // These platforms can't guarantee a monotonically increasing clock reading in a short trace. + if runtime.GOOS == "windows" || runtime.GOARCH == "wasm" { + v.SkipClockSnapshotChecks() + } + // Make sure there are Sync events: at the start and end. + var syncs []int + evs := 0 + for { + ev, err := r.ReadEvent() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("unexpected error reading trace for %s: %v", source, err) + } + if err := v.Event(ev); err != nil { + t.Fatalf("event validation failed: %s", err) + } + if ev.Kind() == inttrace.EventSync { + syncs = append(syncs, evs) + } + evs++ + } + ends := []int{syncs[0], syncs[len(syncs)-1]} + if wantEnds := []int{0, evs - 1}; !slices.Equal(wantEnds, ends) { + t.Errorf("expected a sync event at each end of the trace, found sync events at %d instead of %d for %s", + ends, wantEnds, source) + } + } + + validateTraces := func(t *testing.T, tReader, frReader io.Reader) { + validate(t, "tracer", tReader) + validate(t, "flightRecorder", frReader) + } + startFlightRecorder := func(t *testing.T) *trace.FlightRecorder { + fr := trace.NewFlightRecorder(trace.FlightRecorderConfig{}) + if err := fr.Start(); err != nil { + t.Fatalf("unexpected error creating flight recorder: %v", err) + } + return fr + } + startTrace := func(t *testing.T, w io.Writer) { + if err := trace.Start(w); err != nil { + t.Fatalf("unexpected error starting flight recorder: %v", err) + } + } + stopFlightRecorder := func(t *testing.T, fr *trace.FlightRecorder, w io.Writer) { + if _, err := fr.WriteTo(w); err != nil { + t.Fatalf("unexpected error writing trace from flight recorder: %v", err) + } + fr.Stop() + } + stopTrace := func() { + trace.Stop() + } + t.Run("start(flight)_start(trace)_stop(trace)_stop(flight)", func(t *testing.T) { + if trace.IsEnabled() { + t.Skip("skipping because trace is already enabled") + } + frBuf := new(bytes.Buffer) + tBuf := new(bytes.Buffer) + fr := startFlightRecorder(t) + defer fr.Stop() + startTrace(t, tBuf) + defer trace.Stop() + stopTrace() + stopFlightRecorder(t, fr, frBuf) + validateTraces(t, tBuf, frBuf) + }) + t.Run("start(trace)_start(flight)_stop(trace)_stop(flight)", func(t *testing.T) { + if trace.IsEnabled() { + t.Skip("skipping because trace is already enabled") + } + frBuf := new(bytes.Buffer) + tBuf := new(bytes.Buffer) + startTrace(t, tBuf) + defer trace.Stop() + fr := startFlightRecorder(t) + defer fr.Stop() + stopTrace() + stopFlightRecorder(t, fr, frBuf) + validateTraces(t, tBuf, frBuf) + }) + t.Run("start(flight)_stop(flight)_start(trace)_stop(trace)", func(t *testing.T) { + if trace.IsEnabled() { + t.Skip("skipping because trace is already enabled") + } + frBuf := new(bytes.Buffer) + tBuf := new(bytes.Buffer) + fr := startFlightRecorder(t) + defer fr.Stop() + stopFlightRecorder(t, fr, frBuf) + startTrace(t, tBuf) + defer trace.Stop() + stopTrace() + validateTraces(t, tBuf, frBuf) + }) + t.Run("start(flight)_stop(flight)_start(trace)_stop(trace)", func(t *testing.T) { + if trace.IsEnabled() { + t.Skip("skipping because trace is already enabled") + } + frBuf := new(bytes.Buffer) + tBuf := new(bytes.Buffer) + fr := startFlightRecorder(t) + defer fr.Stop() + stopFlightRecorder(t, fr, frBuf) + startTrace(t, tBuf) + defer trace.Stop() + stopTrace() + validateTraces(t, tBuf, frBuf) + }) + t.Run("start(flight)_start(trace)_stop(flight)_stop(trace)", func(t *testing.T) { + if trace.IsEnabled() { + t.Skip("skipping because trace is already enabled") + } + frBuf := new(bytes.Buffer) + tBuf := new(bytes.Buffer) + fr := startFlightRecorder(t) + defer fr.Stop() + startTrace(t, tBuf) + defer trace.Stop() + stopFlightRecorder(t, fr, frBuf) + stopTrace() + validateTraces(t, tBuf, frBuf) + }) +} diff --git a/src/runtime/trace/trace.go b/src/runtime/trace/trace.go index 935d222f02ba01..a858d1b1010b68 100644 --- a/src/runtime/trace/trace.go +++ b/src/runtime/trace/trace.go @@ -110,45 +110,17 @@ package trace import ( "io" - "runtime" - "sync" - "sync/atomic" ) // Start enables tracing for the current program. // While tracing, the trace will be buffered and written to w. // Start returns an error if tracing is already enabled. func Start(w io.Writer) error { - tracing.Lock() - defer tracing.Unlock() - - if err := runtime.StartTrace(); err != nil { - return err - } - go func() { - for { - data := runtime.ReadTrace() - if data == nil { - break - } - w.Write(data) - } - }() - tracing.enabled.Store(true) - return nil + return tracing.subscribeTraceStartWriter(w) } // Stop stops the current tracing, if any. // Stop only returns after all the writes for the trace have completed. func Stop() { - tracing.Lock() - defer tracing.Unlock() - tracing.enabled.Store(false) - - runtime.StopTrace() -} - -var tracing struct { - sync.Mutex // gate mutators (Start, Stop) - enabled atomic.Bool + tracing.unsubscribeTraceStartWriter() } diff --git a/src/runtime/trace/trace_test.go b/src/runtime/trace/trace_test.go index b891c8c8f9ff42..2174be061de9df 100644 --- a/src/runtime/trace/trace_test.go +++ b/src/runtime/trace/trace_test.go @@ -7,13 +7,16 @@ package trace_test import ( "bytes" "flag" - "os" . "runtime/trace" "testing" "time" ) -var saveTraces = flag.Bool("savetraces", false, "save traces collected by tests") +var dumpTraces = flag.Bool("dump-traces", false, "dump traces to a file, even on success") + +// This file just contains smoke tests and tests of runtime/trace logic only. +// It doesn't validate the resulting traces. See the internal/trace package for +// more comprehensive end-to-end tests. func TestTraceStartStop(t *testing.T) { if IsEnabled() { @@ -32,7 +35,6 @@ func TestTraceStartStop(t *testing.T) { if size != buf.Len() { t.Fatalf("trace writes after stop: %v -> %v", size, buf.Len()) } - saveTrace(t, buf, "TestTraceStartStop") } func TestTraceDoubleStart(t *testing.T) { @@ -50,12 +52,3 @@ func TestTraceDoubleStart(t *testing.T) { Stop() Stop() } - -func saveTrace(t *testing.T, buf *bytes.Buffer, name string) { - if !*saveTraces { - return - } - if err := os.WriteFile(name+".trace", buf.Bytes(), 0600); err != nil { - t.Errorf("failed to write trace file: %s", err) - } -} diff --git a/src/runtime/trace_cgo_test.go b/src/runtime/trace_cgo_test.go index f0db3b7ffb7624..2d078407825553 100644 --- a/src/runtime/trace_cgo_test.go +++ b/src/runtime/trace_cgo_test.go @@ -9,6 +9,7 @@ package runtime_test import ( "bytes" "fmt" + "internal/race" "internal/testenv" "internal/trace" "io" @@ -26,6 +27,9 @@ func TestTraceUnwindCGO(t *testing.T) { t.Skip("-quick") } testenv.MustHaveGoBuild(t) + if runtime.GOOS == "freebsd" && race.Enabled { + t.Skipf("race + cgo freebsd not supported. See https://go.dev/issue/73788.") + } t.Parallel() exe, err := buildTestProg(t, "testprogcgo") @@ -94,10 +98,9 @@ func mustFindLogV2(t *testing.T, trc io.Reader, category string) trace.Event { // dumpStack returns e.Stack() as a string. func dumpStackV2(e *trace.Event) string { var buf bytes.Buffer - e.Stack().Frames(func(f trace.StackFrame) bool { + for f := range e.Stack().Frames() { file := strings.TrimPrefix(f.File, runtime.GOROOT()) fmt.Fprintf(&buf, "%s\n\t%s:%d\n", f.Func, file, f.Line) - return true - }) + } return buf.String() } diff --git a/src/runtime/traceallocfree.go b/src/runtime/traceallocfree.go index 84188a55c45bad..b1b6c63462f4ab 100644 --- a/src/runtime/traceallocfree.go +++ b/src/runtime/traceallocfree.go @@ -8,7 +8,9 @@ package runtime import ( "internal/abi" + "internal/runtime/gc" "internal/runtime/sys" + "internal/trace/tracev2" ) // Batch type values for the alloc/free experiment. @@ -27,7 +29,7 @@ func traceSnapshotMemory(gen uintptr) { // Write a batch containing information that'll be necessary to // interpret the events. var flushed bool - w := unsafeTraceExpWriter(gen, nil, traceExperimentAllocFree) + w := unsafeTraceExpWriter(gen, nil, tracev2.AllocFree) w, flushed = w.ensure(1 + 4*traceBytesPerNumber) if flushed { // Annotate the batch as containing additional info. @@ -35,9 +37,9 @@ func traceSnapshotMemory(gen uintptr) { } // Emit info. - w.varint(uint64(trace.minPageHeapAddr)) + w.varint(trace.minPageHeapAddr) w.varint(uint64(pageSize)) - w.varint(uint64(minHeapAlign)) + w.varint(uint64(gc.MinHeapAlign)) w.varint(uint64(fixedStack)) // Finish writing the batch. @@ -89,17 +91,17 @@ func traceSpanTypeAndClass(s *mspan) traceArg { // SpanExists records an event indicating that the span exists. func (tl traceLocker) SpanExists(s *mspan) { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvSpan, traceSpanID(s), traceArg(s.npages), traceSpanTypeAndClass(s)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvSpan, traceSpanID(s), traceArg(s.npages), traceSpanTypeAndClass(s)) } // SpanAlloc records an event indicating that the span has just been allocated. func (tl traceLocker) SpanAlloc(s *mspan) { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvSpanAlloc, traceSpanID(s), traceArg(s.npages), traceSpanTypeAndClass(s)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvSpanAlloc, traceSpanID(s), traceArg(s.npages), traceSpanTypeAndClass(s)) } // SpanFree records an event indicating that the span is about to be freed. func (tl traceLocker) SpanFree(s *mspan) { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvSpanFree, traceSpanID(s)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvSpanFree, traceSpanID(s)) } // traceSpanID creates a trace ID for the span s for the trace. @@ -111,41 +113,41 @@ func traceSpanID(s *mspan) traceArg { // The type is optional, and the size of the slot occupied the object is inferred from the // span containing it. func (tl traceLocker) HeapObjectExists(addr uintptr, typ *abi.Type) { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvHeapObject, traceHeapObjectID(addr), tl.rtype(typ)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvHeapObject, traceHeapObjectID(addr), tl.rtype(typ)) } // HeapObjectAlloc records that an object was newly allocated at addr with the provided type. // The type is optional, and the size of the slot occupied the object is inferred from the // span containing it. func (tl traceLocker) HeapObjectAlloc(addr uintptr, typ *abi.Type) { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvHeapObjectAlloc, traceHeapObjectID(addr), tl.rtype(typ)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvHeapObjectAlloc, traceHeapObjectID(addr), tl.rtype(typ)) } // HeapObjectFree records that an object at addr is about to be freed. func (tl traceLocker) HeapObjectFree(addr uintptr) { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvHeapObjectFree, traceHeapObjectID(addr)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvHeapObjectFree, traceHeapObjectID(addr)) } // traceHeapObjectID creates a trace ID for a heap object at address addr. func traceHeapObjectID(addr uintptr) traceArg { - return traceArg(uint64(addr)-trace.minPageHeapAddr) / minHeapAlign + return traceArg(uint64(addr)-trace.minPageHeapAddr) / gc.MinHeapAlign } // GoroutineStackExists records that a goroutine stack already exists at address base with the provided size. func (tl traceLocker) GoroutineStackExists(base, size uintptr) { order := traceCompressStackSize(size) - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoroutineStack, traceGoroutineStackID(base), order) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoroutineStack, traceGoroutineStackID(base), order) } // GoroutineStackAlloc records that a goroutine stack was newly allocated at address base with the provided size.. func (tl traceLocker) GoroutineStackAlloc(base, size uintptr) { order := traceCompressStackSize(size) - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoroutineStackAlloc, traceGoroutineStackID(base), order) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoroutineStackAlloc, traceGoroutineStackID(base), order) } // GoroutineStackFree records that a goroutine stack at address base is about to be freed. func (tl traceLocker) GoroutineStackFree(base uintptr) { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoroutineStackFree, traceGoroutineStackID(base)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoroutineStackFree, traceGoroutineStackID(base)) } // traceGoroutineStackID creates a trace ID for the goroutine stack from its base address. diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 8946ec2528424b..de62762ba76906 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -143,7 +143,7 @@ func (u *unwinder) initAt(pc0, sp0, lr0 uintptr, gp *g, flags unwindFlags) { // on another stack. That could confuse callers quite a bit. // Instead, we require that initAt and any other function that // accepts an sp for the current goroutine (typically obtained by - // calling getcallersp) must not run on that goroutine's stack but + // calling GetCallerSP) must not run on that goroutine's stack but // instead on the g0 stack. throw("cannot trace user goroutine on its own stack") } @@ -429,7 +429,7 @@ func (u *unwinder) resolveInternal(innermost, isSyscall bool) { // gp._defer for a defer corresponding to this function, but that // is hard to do with defer records on the stack during a stack copy.) // Note: the +1 is to offset the -1 that - // stack.go:getStackMap does to back up a return + // (*stkframe).getStackMap does to back up a return // address make sure the pc is in the CALL instruction. } else { frame.continpc = 0 @@ -591,7 +591,7 @@ func (u *unwinder) symPC() uintptr { // If the current frame is not a cgo frame or if there's no registered cgo // unwinder, it returns 0. func (u *unwinder) cgoCallers(pcBuf []uintptr) int { - if cgoTraceback == nil || u.frame.fn.funcID != abi.FuncID_cgocallback || u.cgoCtxt < 0 { + if !cgoTracebackAvailable() || u.frame.fn.funcID != abi.FuncID_cgocallback || u.cgoCtxt < 0 { // We don't have a cgo unwinder (typical case), or we do but we're not // in a cgo frame or we're out of cgo context. return 0 @@ -804,7 +804,7 @@ func traceback(pc, sp, lr uintptr, gp *g) { } // tracebacktrap is like traceback but expects that the PC and SP were obtained -// from a trap, not from gp->sched or gp->syscallpc/gp->syscallsp or getcallerpc/getcallersp. +// from a trap, not from gp->sched or gp->syscallpc/gp->syscallsp or GetCallerPC/GetCallerSP. // Because they are from a trap instead of from a saved pair, // the initial PC must not be rewound to the previous instruction. // (All the saved pairs record a PC that is a return address, so we @@ -1014,7 +1014,7 @@ func traceback2(u *unwinder, showRuntime bool, skip, max int) (n, lastN int) { anySymbolized := false stop := false for _, pc := range cgoBuf[:cgoN] { - if cgoSymbolizer == nil { + if !cgoSymbolizerAvailable() { if pr, stop := commitFrame(); stop { break } else if pr { @@ -1090,8 +1090,8 @@ func printAncestorTracebackFuncInfo(f funcInfo, pc uintptr) { // //go:linkname callers func callers(skip int, pcbuf []uintptr) int { - sp := getcallersp() - pc := getcallerpc() + sp := sys.GetCallerSP() + pc := sys.GetCallerPC() gp := getg() var n int systemstack(func() { @@ -1131,6 +1131,22 @@ func showfuncinfo(sf srcFunc, firstFrame bool, calleeID abi.FuncID) bool { return false } + // Always show runtime.runFinalizers and runtime.runCleanups as + // context that this goroutine is running finalizers or cleanups, + // otherwise there is no obvious indicator. + // + // TODO(prattmic): A more general approach would be to always show the + // outermost frame (besides runtime.goexit), even if it is a runtime. + // Hiding the outermost frame allows the apparent outermost frame to + // change across different traces, which seems impossible. + // + // Unfortunately, implementing this requires looking ahead at the next + // frame, which goes against traceback's incremental approach (see big + // comment in traceback1). + if sf.funcID == abi.FuncID_runFinalizers || sf.funcID == abi.FuncID_runCleanups { + return true + } + name := sf.name() // Special case: always show runtime.gopanic frame @@ -1149,11 +1165,10 @@ func showfuncinfo(sf srcFunc, firstFrame bool, calleeID abi.FuncID) bool { // It is only for runtime functions, so ASCII A-Z is fine. func isExportedRuntime(name string) bool { // Check and remove package qualifier. - n := len("runtime.") - if len(name) <= n || name[:n] != "runtime." { + name, found := stringslite.CutPrefix(name, "runtime.") + if !found { return false } - name = name[n:] rcvr := "" // Extract receiver type, if any. @@ -1191,7 +1206,9 @@ var gStatusStrings = [...]string{ _Gwaiting: "waiting", _Gdead: "dead", _Gcopystack: "copystack", + _Gleaked: "leaked", _Gpreempted: "preempted", + _Gdeadextra: "waiting for cgo callback", } func goroutineheader(gp *g) { @@ -1211,7 +1228,7 @@ func goroutineheader(gp *g) { } // Override. - if gpstatus == _Gwaiting && gp.waitreason != waitReasonZero { + if (gpstatus == _Gwaiting || gpstatus == _Gleaked) && gp.waitreason != waitReasonZero { status = gp.waitreason.String() } @@ -1230,19 +1247,37 @@ func goroutineheader(gp *g) { } } print(" [", status) + if gpstatus == _Gleaked { + print(" (leaked)") + } if isScan { print(" (scan)") } + if bubble := gp.bubble; bubble != nil && + gpstatus == _Gwaiting && + gp.waitreason.isIdleInSynctest() && + !stringslite.HasSuffix(status, "(durable)") { + // If this isn't a status where the name includes a (durable) + // suffix to distinguish it from the non-durable form, add it here. + print(" (durable)") + } if waitfor >= 1 { print(", ", waitfor, " minutes") } if gp.lockedm != 0 { print(", locked to thread") } + if bubble := gp.bubble; bubble != nil { + print(", synctest bubble ", bubble.id) + } print("]:\n") } func tracebackothers(me *g) { + tracebacksomeothers(me, func(*g) bool { return true }) +} + +func tracebacksomeothers(me *g, showf func(*g) bool) { level, _, _ := gotraceback() // Show the current goroutine first, if we haven't already. @@ -1261,7 +1296,16 @@ func tracebackothers(me *g) { // against concurrent creation of new Gs, but even with allglock we may // miss Gs created after this loop. forEachGRace(func(gp *g) { - if gp == me || gp == curgp || readgstatus(gp) == _Gdead || isSystemGoroutine(gp, false) && level < 2 { + if gp == me || gp == curgp { + return + } + if status := readgstatus(gp); status == _Gdead || status == _Gdeadextra { + return + } + if !showf(gp) { + return + } + if isSystemGoroutine(gp, false) && level < 2 { return } print("\n") @@ -1329,7 +1373,8 @@ func tracebackHexdump(stk stack, frame *stkframe, bad uintptr) { // isSystemGoroutine reports whether the goroutine g must be omitted // in stack dumps and deadlock detector. This is any goroutine that // starts at a runtime.* entry point, except for runtime.main, -// runtime.handleAsyncEvent (wasm only) and sometimes runtime.runfinq. +// runtime.handleAsyncEvent (wasm only) and sometimes +// runtime.runFinalizers/runtime.runCleanups. // // If fixed is true, any goroutine that can vary between user and // system (that is, the finalizer goroutine) is considered a user @@ -1343,7 +1388,7 @@ func isSystemGoroutine(gp *g, fixed bool) bool { if f.funcID == abi.FuncID_runtime_main || f.funcID == abi.FuncID_corostart || f.funcID == abi.FuncID_handleAsyncEvent { return false } - if f.funcID == abi.FuncID_runfinq { + if f.funcID == abi.FuncID_runFinalizers { // We include the finalizer goroutine if it's calling // back into user code. if fixed { @@ -1353,6 +1398,16 @@ func isSystemGoroutine(gp *g, fixed bool) bool { } return fingStatus.Load()&fingRunningFinalizer == 0 } + if f.funcID == abi.FuncID_runCleanups { + // We include the cleanup goroutines if they're calling + // back into user code. + if fixed { + // This goroutine can vary. In fixed mode, + // always consider it a user goroutine. + return false + } + return !gp.runningCleanups.Load() + } return stringslite.HasPrefix(funcname(f), "runtime.") } @@ -1532,10 +1587,18 @@ func SetCgoTraceback(version int, traceback, context, symbolizer unsafe.Pointer) cgoContext = context cgoSymbolizer = symbolizer - // The context function is called when a C function calls a Go - // function. As such it is only called by C code in runtime/cgo. - if _cgo_set_context_function != nil { - cgocall(_cgo_set_context_function, context) + if _cgo_set_traceback_functions != nil { + type cgoSetTracebackFunctionsArg struct { + traceback unsafe.Pointer + context unsafe.Pointer + symbolizer unsafe.Pointer + } + arg := cgoSetTracebackFunctionsArg{ + traceback: traceback, + context: context, + symbolizer: symbolizer, + } + cgocall(_cgo_set_traceback_functions, noescape(unsafe.Pointer(&arg))) } } @@ -1543,6 +1606,18 @@ var cgoTraceback unsafe.Pointer var cgoContext unsafe.Pointer var cgoSymbolizer unsafe.Pointer +func cgoTracebackAvailable() bool { + // - The traceback function must be registered via SetCgoTraceback. + // - This must be a cgo binary (providing _cgo_call_traceback_function). + return cgoTraceback != nil && _cgo_call_traceback_function != nil +} + +func cgoSymbolizerAvailable() bool { + // - The symbolizer function must be registered via SetCgoTraceback. + // - This must be a cgo binary (providing _cgo_call_symbolizer_function). + return cgoSymbolizer != nil && _cgo_call_symbolizer_function != nil +} + // cgoTracebackArg is the type passed to cgoTraceback. type cgoTracebackArg struct { context uintptr @@ -1569,7 +1644,7 @@ type cgoSymbolizerArg struct { // printCgoTraceback prints a traceback of callers. func printCgoTraceback(callers *cgoCallers) { - if cgoSymbolizer == nil { + if !cgoSymbolizerAvailable() { for _, c := range callers { if c == 0 { break @@ -1594,6 +1669,8 @@ func printCgoTraceback(callers *cgoCallers) { // printOneCgoTraceback prints the traceback of a single cgo caller. // This can print more than one line because of inlining. // It returns the "stop" result of commitFrame. +// +// Preconditions: cgoSymbolizerAvailable returns true. func printOneCgoTraceback(pc uintptr, commitFrame func() (pr, stop bool), arg *cgoSymbolizerArg) bool { arg.pc = pc for { @@ -1624,6 +1701,8 @@ func printOneCgoTraceback(pc uintptr, commitFrame func() (pr, stop bool), arg *c } // callCgoSymbolizer calls the cgoSymbolizer function. +// +// Preconditions: cgoSymbolizerAvailable returns true. func callCgoSymbolizer(arg *cgoSymbolizerArg) { call := cgocall if panicking.Load() > 0 || getg().m.curg != getg() { @@ -1637,14 +1716,13 @@ func callCgoSymbolizer(arg *cgoSymbolizerArg) { if asanenabled { asanwrite(unsafe.Pointer(arg), unsafe.Sizeof(cgoSymbolizerArg{})) } - call(cgoSymbolizer, noescape(unsafe.Pointer(arg))) + call(_cgo_call_symbolizer_function, noescape(unsafe.Pointer(arg))) } // cgoContextPCs gets the PC values from a cgo traceback. +// +// Preconditions: cgoTracebackAvailable returns true. func cgoContextPCs(ctxt uintptr, buf []uintptr) { - if cgoTraceback == nil { - return - } call := cgocall if panicking.Load() > 0 || getg().m.curg != getg() { // We do not want to call into the scheduler when panicking @@ -1662,5 +1740,5 @@ func cgoContextPCs(ctxt uintptr, buf []uintptr) { if asanenabled { asanwrite(unsafe.Pointer(&arg), unsafe.Sizeof(arg)) } - call(cgoTraceback, noescape(unsafe.Pointer(&arg))) + call(_cgo_call_traceback_function, noescape(unsafe.Pointer(&arg))) } diff --git a/src/runtime/traceback_system_test.go b/src/runtime/traceback_system_test.go index ece58e806d51c1..3a3f33bbd41005 100644 --- a/src/runtime/traceback_system_test.go +++ b/src/runtime/traceback_system_test.go @@ -23,8 +23,8 @@ import ( ) // This is the entrypoint of the child process used by -// TestTracebackSystem. It prints a crash report to stdout. -func crash() { +// TestTracebackSystem/panic. It prints a crash report to stdout. +func crashViaPanic() { // Ensure that we get pc=0x%x values in the traceback. debug.SetTraceback("system") writeSentinel(os.Stdout) @@ -37,6 +37,21 @@ func crash() { select {} } +// This is the entrypoint of the child process used by +// TestTracebackSystem/trap. It prints a crash report to stdout. +func crashViaTrap() { + // Ensure that we get pc=0x%x values in the traceback. + debug.SetTraceback("system") + writeSentinel(os.Stdout) + debug.SetCrashOutput(os.Stdout, debug.CrashOptions{}) + + go func() { + // This call is typically inlined. + trap1() + }() + select {} +} + func child1() { child2() } @@ -85,6 +100,20 @@ func child7() { panic("oops") } +func trap1() { + trap2() +} + +var sinkPtr *int + +func trap2() { + trap3(sinkPtr) +} + +func trap3(i *int) { + *i = 42 +} + // TestTracebackSystem tests that the syntax of crash reports produced // by GOTRACEBACK=system (see traceback2) contains a complete, // parseable list of program counters for the running goroutine that @@ -100,46 +129,75 @@ func TestTracebackSystem(t *testing.T) { t.Skip("Can't read source code for this file on Android") } - // Fork+exec the crashing process. - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - cmd := testenv.Command(t, exe) - cmd.Env = append(cmd.Environ(), entrypointVar+"=crash") - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - cmd.Run() // expected to crash - t.Logf("stderr:\n%s\nstdout: %s\n", stderr.Bytes(), stdout.Bytes()) - crash := stdout.String() - - // If the only line is the sentinel, it wasn't a crash. - if strings.Count(crash, "\n") < 2 { - t.Fatalf("child process did not produce a crash report") + tests := []struct { + name string + want string + }{ + { + name: "panic", + want: `redacted.go:0: runtime.gopanic +traceback_system_test.go:100: runtime_test.child7: panic("oops") +traceback_system_test.go:83: runtime_test.child6: child7() // appears in stack trace +traceback_system_test.go:74: runtime_test.child5: child6() // appears in stack trace +traceback_system_test.go:68: runtime_test.child4: child5() +traceback_system_test.go:64: runtime_test.child3: child4() +traceback_system_test.go:60: runtime_test.child2: child3() +traceback_system_test.go:56: runtime_test.child1: child2() +traceback_system_test.go:35: runtime_test.crashViaPanic.func1: child1() +redacted.go:0: runtime.goexit +`, + }, + { + // Test panic via trap. x/telemetry is aware that trap + // PCs follow runtime.sigpanic and need to be + // incremented to offset the decrement done by + // CallersFrames. + name: "trap", + want: `redacted.go:0: runtime.gopanic +redacted.go:0: runtime.panicmem +redacted.go:0: runtime.sigpanic +traceback_system_test.go:114: runtime_test.trap3: *i = 42 +traceback_system_test.go:110: runtime_test.trap2: trap3(sinkPtr) +traceback_system_test.go:104: runtime_test.trap1: trap2() +traceback_system_test.go:50: runtime_test.crashViaTrap.func1: trap1() +redacted.go:0: runtime.goexit +`, + }, } - // Parse the PCs out of the child's crash report. - pcs, err := parseStackPCs(crash) - if err != nil { - t.Fatal(err) - } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Fork+exec the crashing process. + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + cmd := testenv.Command(t, exe) + cmd.Env = append(cmd.Environ(), entrypointVar+"="+tc.name) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + cmd.Run() // expected to crash + t.Logf("stderr:\n%s\nstdout: %s\n", stderr.Bytes(), stdout.Bytes()) + crash := stdout.String() + + // If the only line is the sentinel, it wasn't a crash. + if strings.Count(crash, "\n") < 2 { + t.Fatalf("child process did not produce a crash report") + } - // Unwind the stack using this executable's symbol table. - got := formatStack(pcs) - want := `redacted.go:0: runtime.gopanic -traceback_system_test.go:85: runtime_test.child7: panic("oops") -traceback_system_test.go:68: runtime_test.child6: child7() // appears in stack trace -traceback_system_test.go:59: runtime_test.child5: child6() // appears in stack trace -traceback_system_test.go:53: runtime_test.child4: child5() -traceback_system_test.go:49: runtime_test.child3: child4() -traceback_system_test.go:45: runtime_test.child2: child3() -traceback_system_test.go:41: runtime_test.child1: child2() -traceback_system_test.go:35: runtime_test.crash.func1: child1() -redacted.go:0: runtime.goexit -` - if strings.TrimSpace(got) != strings.TrimSpace(want) { - t.Errorf("got:\n%swant:\n%s", got, want) + // Parse the PCs out of the child's crash report. + pcs, err := parseStackPCs(crash) + if err != nil { + t.Fatal(err) + } + + // Unwind the stack using this executable's symbol table. + got := formatStack(pcs) + if strings.TrimSpace(got) != strings.TrimSpace(tc.want) { + t.Errorf("got:\n%swant:\n%s", got, tc.want) + } + }) } } @@ -154,6 +212,35 @@ redacted.go:0: runtime.goexit // // (Copied from golang.org/x/telemetry/crashmonitor.parseStackPCs.) func parseStackPCs(crash string) ([]uintptr, error) { + // getSymbol parses the symbol name out of a line of the form: + // SYMBOL(ARGS) + // + // Note: SYMBOL may contain parens "pkg.(*T).method". However, type + // parameters are always replaced with ..., so they cannot introduce + // more parens. e.g., "pkg.(*T[...]).method". + // + // ARGS can contain parens. We want the first paren that is not + // immediately preceded by a ".". + // + // TODO(prattmic): This is mildly complicated and is only used to find + // runtime.sigpanic, so perhaps simplify this by checking explicitly + // for sigpanic. + getSymbol := func(line string) (string, error) { + var prev rune + for i, c := range line { + if line[i] != '(' { + prev = c + continue + } + if prev == '.' { + prev = c + continue + } + return line[:i], nil + } + return "", fmt.Errorf("no symbol for stack frame: %s", line) + } + // getPC parses the PC out of a line of the form: // \tFILE:LINE +0xRELPC sp=... fp=... pc=... getPC := func(line string) (uint64, error) { @@ -170,6 +257,9 @@ func parseStackPCs(crash string) ([]uintptr, error) { childSentinel = sentinel() on = false // are we in the first running goroutine? lines = strings.Split(crash, "\n") + symLine = true // within a goroutine, every other line is a symbol or file/line/pc location, starting with symbol. + currSymbol string + prevSymbol string // symbol of the most recent previous frame with a PC. ) for i := 0; i < len(lines); i++ { line := lines[i] @@ -212,21 +302,76 @@ func parseStackPCs(crash string) ([]uintptr, error) { // Note: SYMBOL may contain parens "pkg.(*T).method" // The RELPC is sometimes missing. - // Skip the symbol(args) line. - i++ - if i == len(lines) { - break - } - line = lines[i] + if symLine { + var err error + currSymbol, err = getSymbol(line) + if err != nil { + return nil, fmt.Errorf("error extracting symbol: %v", err) + } - // Parse the PC, and correct for the parent and child's - // different mappings of the text section. - pc, err := getPC(line) - if err != nil { - // Inlined frame, perhaps; skip it. - continue + symLine = false // Next line is FILE:LINE. + } else { + // Parse the PC, and correct for the parent and child's + // different mappings of the text section. + pc, err := getPC(line) + if err != nil { + // Inlined frame, perhaps; skip it. + + // Done with this frame. Next line is a new frame. + // + // Don't update prevSymbol; we only want to + // track frames with a PC. + currSymbol = "" + symLine = true + continue + } + + pc = pc - parentSentinel + childSentinel + + // If the previous frame was sigpanic, then this frame + // was a trap (e.g., SIGSEGV). + // + // Typically all middle frames are calls, and report + // the "return PC". That is, the instruction following + // the CALL where the callee will eventually return to. + // + // runtime.CallersFrames is aware of this property and + // will decrement each PC by 1 to "back up" to the + // location of the CALL, which is the actual line + // number the user expects. + // + // This does not work for traps, as a trap is not a + // call, so the reported PC is not the return PC, but + // the actual PC of the trap. + // + // runtime.Callers is aware of this and will + // intentionally increment trap PCs in order to correct + // for the decrement performed by + // runtime.CallersFrames. See runtime.tracebackPCs and + // runtume.(*unwinder).symPC. + // + // We must emulate the same behavior, otherwise we will + // report the location of the instruction immediately + // prior to the trap, which may be on a different line, + // or even a different inlined functions. + // + // TODO(prattmic): The runtime applies the same trap + // behavior for other "injected calls", see injectCall + // in runtime.(*unwinder).next. Do we want to handle + // those as well? I don't believe we'd ever see + // runtime.asyncPreempt or runtime.debugCallV2 in a + // typical crash. + if prevSymbol == "runtime.sigpanic" { + pc++ + } + + pcs = append(pcs, uintptr(pc)) + + // Done with this frame. Next line is a new frame. + prevSymbol = currSymbol + currSymbol = "" + symLine = true } - pcs = append(pcs, uintptr(pc-parentSentinel+childSentinel)) } return pcs, nil } diff --git a/src/runtime/traceback_test.go b/src/runtime/traceback_test.go index 8cbccac673ac6c..1dac91311ca9a3 100644 --- a/src/runtime/traceback_test.go +++ b/src/runtime/traceback_test.go @@ -8,6 +8,9 @@ import ( "bytes" "fmt" "internal/abi" + "internal/asan" + "internal/msan" + "internal/race" "internal/testenv" "regexp" "runtime" @@ -867,3 +870,15 @@ func TestTracebackGeneric(t *testing.T) { } } } + +func TestSetCgoTracebackNoCgo(t *testing.T) { + if asan.Enabled || msan.Enabled || race.Enabled { + t.Skip("skipped test: sanitizer builds use cgo") + } + + output := runTestProg(t, "testprog", "SetCgoTracebackNoCgo") + want := "OK\n" + if output != want { + t.Fatalf("want %s, got %s\n", want, output) + } +} diff --git a/src/runtime/tracebuf.go b/src/runtime/tracebuf.go index 0849a57809e2ca..5adaede424c4db 100644 --- a/src/runtime/tracebuf.go +++ b/src/runtime/tracebuf.go @@ -8,6 +8,7 @@ package runtime import ( "internal/runtime/sys" + "internal/trace/tracev2" "unsafe" ) @@ -24,7 +25,7 @@ const traceBytesPerNumber = 10 // we can change it if it's deemed too error-prone. type traceWriter struct { traceLocker - exp traceExperiment + exp tracev2.Experiment *traceBuf } @@ -48,7 +49,7 @@ func (tl traceLocker) writer() traceWriter { gp.throwsplit = true } } - return traceWriter{traceLocker: tl, traceBuf: tl.mp.trace.buf[tl.gen%2][traceNoExperiment]} + return traceWriter{traceLocker: tl, traceBuf: tl.mp.trace.buf[tl.gen%2][tracev2.NoExperiment]} } // unsafeTraceWriter produces a traceWriter that doesn't lock the trace. @@ -70,7 +71,7 @@ func unsafeTraceWriter(gen uintptr, buf *traceBuf) traceWriter { // have any stack growth. // //go:nosplit -func (w traceWriter) event(ev traceEv, args ...traceArg) traceWriter { +func (w traceWriter) event(ev tracev2.EventType, args ...traceArg) traceWriter { // N.B. Everything in this call must be nosplit to maintain // the stack growth related invariants for writing events. @@ -164,7 +165,7 @@ func (w traceWriter) refill() traceWriter { unlock(&trace.lock) } else { unlock(&trace.lock) - w.traceBuf = (*traceBuf)(sysAlloc(unsafe.Sizeof(traceBuf{}), &memstats.other_sys)) + w.traceBuf = (*traceBuf)(sysAlloc(unsafe.Sizeof(traceBuf{}), &memstats.other_sys, "trace buffer")) if w.traceBuf == nil { throw("trace: out of memory") } @@ -182,23 +183,44 @@ func (w traceWriter) refill() traceWriter { // Tolerate a nil mp. mID := ^uint64(0) if w.mp != nil { - mID = uint64(w.mp.procid) + mID = w.mp.procid } // Write the buffer's header. - if w.exp == traceNoExperiment { - w.byte(byte(traceEvEventBatch)) + if w.exp == tracev2.NoExperiment { + w.byte(byte(tracev2.EvEventBatch)) } else { - w.byte(byte(traceEvExperimentalBatch)) + w.byte(byte(tracev2.EvExperimentalBatch)) w.byte(byte(w.exp)) } w.varint(uint64(w.gen)) - w.varint(uint64(mID)) + w.varint(mID) w.varint(uint64(ts)) w.traceBuf.lenPos = w.varintReserve() return w } +// expWriter returns a traceWriter that writes into the current M's stream for +// the given experiment. +func (tl traceLocker) expWriter(exp tracev2.Experiment) traceWriter { + return traceWriter{traceLocker: tl, traceBuf: tl.mp.trace.buf[tl.gen%2][exp], exp: exp} +} + +// unsafeTraceExpWriter produces a traceWriter for experimental trace batches +// that doesn't lock the trace. Data written to experimental batches need not +// conform to the standard trace format. +// +// It should only be used in contexts where either: +// - Another traceLocker is held. +// - trace.gen is prevented from advancing. +// +// This does not have the same stack growth restrictions as traceLocker.writer. +// +// buf may be nil. +func unsafeTraceExpWriter(gen uintptr, buf *traceBuf, exp tracev2.Experiment) traceWriter { + return traceWriter{traceLocker: traceLocker{gen: gen}, traceBuf: buf, exp: exp} +} + // traceBufQueue is a FIFO of traceBufs. type traceBufQueue struct { head, tail *traceBuf @@ -247,7 +269,7 @@ type traceBufHeader struct { type traceBuf struct { _ sys.NotInHeap traceBufHeader - arr [64<<10 - unsafe.Sizeof(traceBufHeader{})]byte // underlying buffer for traceBufHeader.buf + arr [tracev2.MaxBatchSize - unsafe.Sizeof(traceBufHeader{})]byte // underlying buffer for traceBufHeader.buf } // byte appends v to buf. diff --git a/src/runtime/tracecpu.go b/src/runtime/tracecpu.go index c8a6f56ff2f684..e64ca32cdf155e 100644 --- a/src/runtime/tracecpu.go +++ b/src/runtime/tracecpu.go @@ -6,6 +6,8 @@ package runtime +import "internal/trace/tracev2" + // traceInitReadCPU initializes CPU profile -> tracer state for tracing. // // Returns a profBuf for reading from. @@ -114,7 +116,7 @@ func traceStopReadCPU() { // Must not run on the system stack because profBuf.read performs race // operations. func traceReadCPU(gen uintptr) bool { - var pcBuf [traceStackSize]uintptr + var pcBuf [tracev2.MaxFramesPerStack]uintptr data, tags, eof := trace.cpuLogRead[gen%2].read(profBufNonBlocking) for len(data) > 0 { @@ -169,17 +171,17 @@ func traceReadCPU(gen uintptr) bool { // Ensure we have a place to write to. var flushed bool - w, flushed = w.ensure(2 + 5*traceBytesPerNumber /* traceEvCPUSamples + traceEvCPUSample + timestamp + g + m + p + stack ID */) + w, flushed = w.ensure(2 + 5*traceBytesPerNumber /* tracev2.EvCPUSamples + tracev2.EvCPUSample + timestamp + g + m + p + stack ID */) if flushed { // Annotate the batch as containing strings. - w.byte(byte(traceEvCPUSamples)) + w.byte(byte(tracev2.EvCPUSamples)) } // Add the stack to the table. stackID := trace.stackTab[gen%2].put(pcBuf[:nstk]) // Write out the CPU sample. - w.byte(byte(traceEvCPUSample)) + w.byte(byte(tracev2.EvCPUSample)) w.varint(timestamp) w.varint(mpid) w.varint(ppid) @@ -256,7 +258,7 @@ func traceCPUSample(gp *g, mp *m, pp *p, stk []uintptr) { if gp != nil { hdr[1] = gp.goid } - hdr[2] = uint64(mp.procid) + hdr[2] = mp.procid // Allow only one writer at a time for !trace.signalLock.CompareAndSwap(0, 1) { diff --git a/src/runtime/traceevent.go b/src/runtime/traceevent.go index 51d23688425ff3..b0bc4c017da266 100644 --- a/src/runtime/traceevent.go +++ b/src/runtime/traceevent.go @@ -9,88 +9,7 @@ package runtime import ( "internal/abi" "internal/runtime/sys" -) - -// Event types in the trace, args are given in square brackets. -// -// Naming scheme: -// - Time range event pairs have suffixes "Begin" and "End". -// - "Start", "Stop", "Create", "Destroy", "Block", "Unblock" -// are suffixes reserved for scheduling resources. -// -// NOTE: If you add an event type, make sure you also update all -// tables in this file! -type traceEv uint8 - -const ( - traceEvNone traceEv = iota // unused - - // Structural events. - traceEvEventBatch // start of per-M batch of events [generation, M ID, timestamp, batch length] - traceEvStacks // start of a section of the stack table [...traceEvStack] - traceEvStack // stack table entry [ID, ...{PC, func string ID, file string ID, line #}] - traceEvStrings // start of a section of the string dictionary [...traceEvString] - traceEvString // string dictionary entry [ID, length, string] - traceEvCPUSamples // start of a section of CPU samples [...traceEvCPUSample] - traceEvCPUSample // CPU profiling sample [timestamp, M ID, P ID, goroutine ID, stack ID] - traceEvFrequency // timestamp units per sec [freq] - - // Procs. - traceEvProcsChange // current value of GOMAXPROCS [timestamp, GOMAXPROCS, stack ID] - traceEvProcStart // start of P [timestamp, P ID, P seq] - traceEvProcStop // stop of P [timestamp] - traceEvProcSteal // P was stolen [timestamp, P ID, P seq, M ID] - traceEvProcStatus // P status at the start of a generation [timestamp, P ID, status] - - // Goroutines. - traceEvGoCreate // goroutine creation [timestamp, new goroutine ID, new stack ID, stack ID] - traceEvGoCreateSyscall // goroutine appears in syscall (cgo callback) [timestamp, new goroutine ID] - traceEvGoStart // goroutine starts running [timestamp, goroutine ID, goroutine seq] - traceEvGoDestroy // goroutine ends [timestamp] - traceEvGoDestroySyscall // goroutine ends in syscall (cgo callback) [timestamp] - traceEvGoStop // goroutine yields its time, but is runnable [timestamp, reason, stack ID] - traceEvGoBlock // goroutine blocks [timestamp, reason, stack ID] - traceEvGoUnblock // goroutine is unblocked [timestamp, goroutine ID, goroutine seq, stack ID] - traceEvGoSyscallBegin // syscall enter [timestamp, P seq, stack ID] - traceEvGoSyscallEnd // syscall exit [timestamp] - traceEvGoSyscallEndBlocked // syscall exit and it blocked at some point [timestamp] - traceEvGoStatus // goroutine status at the start of a generation [timestamp, goroutine ID, M ID, status] - - // STW. - traceEvSTWBegin // STW start [timestamp, kind] - traceEvSTWEnd // STW done [timestamp] - - // GC events. - traceEvGCActive // GC active [timestamp, seq] - traceEvGCBegin // GC start [timestamp, seq, stack ID] - traceEvGCEnd // GC done [timestamp, seq] - traceEvGCSweepActive // GC sweep active [timestamp, P ID] - traceEvGCSweepBegin // GC sweep start [timestamp, stack ID] - traceEvGCSweepEnd // GC sweep done [timestamp, swept bytes, reclaimed bytes] - traceEvGCMarkAssistActive // GC mark assist active [timestamp, goroutine ID] - traceEvGCMarkAssistBegin // GC mark assist start [timestamp, stack ID] - traceEvGCMarkAssistEnd // GC mark assist done [timestamp] - traceEvHeapAlloc // gcController.heapLive change [timestamp, heap alloc in bytes] - traceEvHeapGoal // gcController.heapGoal() change [timestamp, heap goal in bytes] - - // Annotations. - traceEvGoLabel // apply string label to current running goroutine [timestamp, label string ID] - traceEvUserTaskBegin // trace.NewTask [timestamp, internal task ID, internal parent task ID, name string ID, stack ID] - traceEvUserTaskEnd // end of a task [timestamp, internal task ID, stack ID] - traceEvUserRegionBegin // trace.{Start,With}Region [timestamp, internal task ID, name string ID, stack ID] - traceEvUserRegionEnd // trace.{End,With}Region [timestamp, internal task ID, name string ID, stack ID] - traceEvUserLog // trace.Log [timestamp, internal task ID, key string ID, stack, value string ID] - - // Coroutines. - traceEvGoSwitch // goroutine switch (coroswitch) [timestamp, goroutine ID, goroutine seq] - traceEvGoSwitchDestroy // goroutine switch and destroy [timestamp, goroutine ID, goroutine seq] - traceEvGoCreateBlocked // goroutine creation (starts blocked) [timestamp, new goroutine ID, new stack ID, stack ID] - - // GoStatus with stack. - traceEvGoStatusStack // goroutine status at the start of a generation, with a stack [timestamp, goroutine ID, M ID, status, stack ID] - - // Batch event for an experimental batch with a custom format. - traceEvExperimentalBatch // start of extra data [experiment ID, generation, M ID, timestamp, batch length, batch data...] + "internal/trace/tracev2" ) // traceArg is a simple wrapper type to help ensure that arguments passed @@ -117,19 +36,19 @@ type traceEventWriter struct { // been Runnable before a GoStart). Otherwise, callers can query the status of either the goroutine // or P and pass the appropriate status. // -// In this case, the default status should be traceGoBad or traceProcBad to help identify bugs sooner. -func (tl traceLocker) eventWriter(goStatus traceGoStatus, procStatus traceProcStatus) traceEventWriter { +// In this case, the default status should be tracev2.GoBad or tracev2.ProcBad to help identify bugs sooner. +func (tl traceLocker) eventWriter(goStatus tracev2.GoStatus, procStatus tracev2.ProcStatus) traceEventWriter { if pp := tl.mp.p.ptr(); pp != nil && !pp.trace.statusWasTraced(tl.gen) && pp.trace.acquireStatus(tl.gen) { tl.writer().writeProcStatus(uint64(pp.id), procStatus, pp.trace.inSweep).end() } if gp := tl.mp.curg; gp != nil && !gp.trace.statusWasTraced(tl.gen) && gp.trace.acquireStatus(tl.gen) { - tl.writer().writeGoStatus(uint64(gp.goid), int64(tl.mp.procid), goStatus, gp.inMarkAssist, 0 /* no stack */).end() + tl.writer().writeGoStatus(gp.goid, int64(tl.mp.procid), goStatus, gp.inMarkAssist, 0 /* no stack */).end() } return traceEventWriter{tl} } // event writes out a trace event. -func (e traceEventWriter) event(ev traceEv, args ...traceArg) { +func (e traceEventWriter) event(ev tracev2.EventType, args ...traceArg) { e.tl.writer().event(ev, args...).end() } @@ -137,7 +56,7 @@ func (e traceEventWriter) event(ev traceEv, args ...traceArg) { // It then returns a traceArg representing that stack which may be // passed to write. func (tl traceLocker) stack(skip int) traceArg { - return traceArg(traceStack(skip, nil, tl.gen)) + return traceArg(traceStack(skip, nil, &trace.stackTab[tl.gen%2])) } // startPC takes a start PC for a goroutine and produces a unique diff --git a/src/runtime/traceexp.go b/src/runtime/traceexp.go deleted file mode 100644 index 13eec0c0b66706..00000000000000 --- a/src/runtime/traceexp.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -// expWriter returns a traceWriter that writes into the current M's stream for -// the given experiment. -func (tl traceLocker) expWriter(exp traceExperiment) traceWriter { - return traceWriter{traceLocker: tl, traceBuf: tl.mp.trace.buf[tl.gen%2][exp], exp: exp} -} - -// unsafeTraceExpWriter produces a traceWriter for experimental trace batches -// that doesn't lock the trace. Data written to experimental batches need not -// conform to the standard trace format. -// -// It should only be used in contexts where either: -// - Another traceLocker is held. -// - trace.gen is prevented from advancing. -// -// This does not have the same stack growth restrictions as traceLocker.writer. -// -// buf may be nil. -func unsafeTraceExpWriter(gen uintptr, buf *traceBuf, exp traceExperiment) traceWriter { - return traceWriter{traceLocker: traceLocker{gen: gen}, traceBuf: buf, exp: exp} -} - -// traceExperiment is an enumeration of the different kinds of experiments supported for tracing. -type traceExperiment uint8 - -const ( - // traceNoExperiment indicates no experiment. - traceNoExperiment traceExperiment = iota - - // traceExperimentAllocFree is an experiment to add alloc/free events to the trace. - traceExperimentAllocFree - - // traceNumExperiments is the number of trace experiments (and 1 higher than - // the highest numbered experiment). - traceNumExperiments -) - -// Experimental events. -const ( - _ traceEv = 127 + iota - - // Experimental events for ExperimentAllocFree. - - // Experimental heap span events. IDs map reversibly to base addresses. - traceEvSpan // heap span exists [timestamp, id, npages, type/class] - traceEvSpanAlloc // heap span alloc [timestamp, id, npages, type/class] - traceEvSpanFree // heap span free [timestamp, id] - - // Experimental heap object events. IDs map reversibly to addresses. - traceEvHeapObject // heap object exists [timestamp, id, type] - traceEvHeapObjectAlloc // heap object alloc [timestamp, id, type] - traceEvHeapObjectFree // heap object free [timestamp, id] - - // Experimental goroutine stack events. IDs map reversibly to addresses. - traceEvGoroutineStack // stack exists [timestamp, id, order] - traceEvGoroutineStackAlloc // stack alloc [timestamp, id, order] - traceEvGoroutineStackFree // stack free [timestamp, id] -) diff --git a/src/runtime/traceregion.go b/src/runtime/traceregion.go index b45093ec862b8a..eb19294f1b16fa 100644 --- a/src/runtime/traceregion.go +++ b/src/runtime/traceregion.go @@ -59,37 +59,43 @@ func (a *traceRegionAlloc) alloc(n uintptr) *notInHeap { } // Try to install a new block. - lock(&a.lock) - - // Check block again under the lock. Someone may - // have gotten here first. - block = (*traceRegionAllocBlock)(a.current.Load()) - if block != nil { - r := block.off.Add(n) - if r <= uintptr(len(block.data)) { - unlock(&a.lock) - return (*notInHeap)(unsafe.Pointer(&block.data[r-n])) + var x *notInHeap + systemstack(func() { + // Acquire a.lock on the systemstack to avoid stack growth + // and accidentally entering the tracer again. + lock(&a.lock) + + // Check block again under the lock. Someone may + // have gotten here first. + block = (*traceRegionAllocBlock)(a.current.Load()) + if block != nil { + r := block.off.Add(n) + if r <= uintptr(len(block.data)) { + unlock(&a.lock) + x = (*notInHeap)(unsafe.Pointer(&block.data[r-n])) + return + } + + // Add the existing block to the full list. + block.next = a.full + a.full = block } - // Add the existing block to the full list. - block.next = a.full - a.full = block - } - - // Allocate a new block. - block = (*traceRegionAllocBlock)(sysAlloc(unsafe.Sizeof(traceRegionAllocBlock{}), &memstats.other_sys)) - if block == nil { - throw("traceRegion: out of memory") - } + // Allocate a new block. + block = (*traceRegionAllocBlock)(sysAlloc(unsafe.Sizeof(traceRegionAllocBlock{}), &memstats.other_sys, "trace arena alloc")) + if block == nil { + throw("traceRegion: out of memory") + } - // Allocate space for our current request, so we always make - // progress. - block.off.Store(n) - x := (*notInHeap)(unsafe.Pointer(&block.data[0])) + // Allocate space for our current request, so we always make + // progress. + block.off.Store(n) + x = (*notInHeap)(unsafe.Pointer(&block.data[0])) - // Publish the new block. - a.current.Store(unsafe.Pointer(block)) - unlock(&a.lock) + // Publish the new block. + a.current.Store(unsafe.Pointer(block)) + unlock(&a.lock) + }) return x } diff --git a/src/runtime/traceruntime.go b/src/runtime/traceruntime.go index dfbf183de5ba7c..7df6f68b706164 100644 --- a/src/runtime/traceruntime.go +++ b/src/runtime/traceruntime.go @@ -8,6 +8,7 @@ package runtime import ( "internal/runtime/atomic" + "internal/trace/tracev2" _ "unsafe" // for go:linkname ) @@ -24,11 +25,12 @@ func (s *gTraceState) reset() { // mTraceState is per-M state for the tracer. type mTraceState struct { - seqlock atomic.Uintptr // seqlock indicating that this M is writing to a trace buffer. - buf [2][traceNumExperiments]*traceBuf // Per-M traceBuf for writing. Indexed by trace.gen%2. - link *m // Snapshot of alllink or freelink. - reentered uint32 // Whether we've reentered tracing from within tracing. - oldthrowsplit bool // gp.throwsplit upon calling traceLocker.writer. For debugging. + seqlock atomic.Uintptr // seqlock indicating that this M is writing to a trace buffer. + buf [2][tracev2.NumExperiments]*traceBuf // Per-M traceBuf for writing. Indexed by trace.gen%2. + link *m // Snapshot of alllink or freelink. + reentered uint32 // Whether we've reentered tracing from within tracing. + entryGen uintptr // The generation value on first entry. + oldthrowsplit bool // gp.throwsplit upon calling traceLocker.writer. For debugging. } // pTraceState is per-P state for the tracer. @@ -101,24 +103,28 @@ const ( traceBlockDebugCall traceBlockUntilGCEnds traceBlockSleep + traceBlockGCWeakToStrongWait + traceBlockSynctest ) var traceBlockReasonStrings = [...]string{ - traceBlockGeneric: "unspecified", - traceBlockForever: "forever", - traceBlockNet: "network", - traceBlockSelect: "select", - traceBlockCondWait: "sync.(*Cond).Wait", - traceBlockSync: "sync", - traceBlockChanSend: "chan send", - traceBlockChanRecv: "chan receive", - traceBlockGCMarkAssist: "GC mark assist wait for work", - traceBlockGCSweep: "GC background sweeper wait", - traceBlockSystemGoroutine: "system goroutine wait", - traceBlockPreempted: "preempted", - traceBlockDebugCall: "wait for debug call", - traceBlockUntilGCEnds: "wait until GC ends", - traceBlockSleep: "sleep", + traceBlockGeneric: "unspecified", + traceBlockForever: "forever", + traceBlockNet: "network", + traceBlockSelect: "select", + traceBlockCondWait: "sync.(*Cond).Wait", + traceBlockSync: "sync", + traceBlockChanSend: "chan send", + traceBlockChanRecv: "chan receive", + traceBlockGCMarkAssist: "GC mark assist wait for work", + traceBlockGCSweep: "GC background sweeper wait", + traceBlockSystemGoroutine: "system goroutine wait", + traceBlockPreempted: "preempted", + traceBlockDebugCall: "wait for debug call", + traceBlockUntilGCEnds: "wait until GC ends", + traceBlockSleep: "sleep", + traceBlockGCWeakToStrongWait: "GC weak to strong wait", + traceBlockSynctest: "synctest", } // traceGoStopReason is an enumeration of reasons a goroutine might yield. @@ -207,7 +213,7 @@ func traceAcquireEnabled() traceLocker { // that it is. if mp.trace.seqlock.Load()%2 == 1 { mp.trace.reentered++ - return traceLocker{mp, trace.gen.Load()} + return traceLocker{mp, mp.trace.entryGen} } // Acquire the trace seqlock. This prevents traceAdvance from moving forward @@ -235,6 +241,7 @@ func traceAcquireEnabled() traceLocker { releasem(mp) return traceLocker{} } + mp.trace.entryGen = gen return traceLocker{mp, gen} } @@ -279,7 +286,7 @@ func traceExitedSyscall() { // Gomaxprocs emits a ProcsChange event. func (tl traceLocker) Gomaxprocs(procs int32) { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvProcsChange, traceArg(procs), tl.stack(1)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvProcsChange, traceArg(procs), tl.stack(1)) } // ProcStart traces a ProcStart event. @@ -290,14 +297,14 @@ func (tl traceLocker) ProcStart() { // Procs are typically started within the scheduler when there is no user goroutine. If there is a user goroutine, // it must be in _Gsyscall because the only time a goroutine is allowed to have its Proc moved around from under it // is during a syscall. - tl.eventWriter(traceGoSyscall, traceProcIdle).event(traceEvProcStart, traceArg(pp.id), pp.trace.nextSeq(tl.gen)) + tl.eventWriter(tracev2.GoSyscall, tracev2.ProcIdle).event(tracev2.EvProcStart, traceArg(pp.id), pp.trace.nextSeq(tl.gen)) } // ProcStop traces a ProcStop event. func (tl traceLocker) ProcStop(pp *p) { // The only time a goroutine is allowed to have its Proc moved around // from under it is during a syscall. - tl.eventWriter(traceGoSyscall, traceProcRunning).event(traceEvProcStop) + tl.eventWriter(tracev2.GoSyscall, tracev2.ProcRunning).event(tracev2.EvProcStop) } // GCActive traces a GCActive event. @@ -305,7 +312,7 @@ func (tl traceLocker) ProcStop(pp *p) { // Must be emitted by an actively running goroutine on an active P. This restriction can be changed // easily and only depends on where it's currently called. func (tl traceLocker) GCActive() { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGCActive, traceArg(trace.seqGC)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCActive, traceArg(trace.seqGC)) // N.B. Only one GC can be running at a time, so this is naturally // serialized by the caller. trace.seqGC++ @@ -316,7 +323,7 @@ func (tl traceLocker) GCActive() { // Must be emitted by an actively running goroutine on an active P. This restriction can be changed // easily and only depends on where it's currently called. func (tl traceLocker) GCStart() { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGCBegin, traceArg(trace.seqGC), tl.stack(3)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCBegin, traceArg(trace.seqGC), tl.stack(3)) // N.B. Only one GC can be running at a time, so this is naturally // serialized by the caller. trace.seqGC++ @@ -327,7 +334,7 @@ func (tl traceLocker) GCStart() { // Must be emitted by an actively running goroutine on an active P. This restriction can be changed // easily and only depends on where it's currently called. func (tl traceLocker) GCDone() { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGCEnd, traceArg(trace.seqGC)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCEnd, traceArg(trace.seqGC)) // N.B. Only one GC can be running at a time, so this is naturally // serialized by the caller. trace.seqGC++ @@ -337,14 +344,14 @@ func (tl traceLocker) GCDone() { func (tl traceLocker) STWStart(reason stwReason) { // Although the current P may be in _Pgcstop here, we model the P as running during the STW. This deviates from the // runtime's state tracking, but it's more accurate and doesn't result in any loss of information. - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvSTWBegin, tl.string(reason.String()), tl.stack(2)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvSTWBegin, tl.string(reason.String()), tl.stack(2)) } // STWDone traces a STWEnd event. func (tl traceLocker) STWDone() { // Although the current P may be in _Pgcstop here, we model the P as running during the STW. This deviates from the // runtime's state tracking, but it's more accurate and doesn't result in any loss of information. - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvSTWEnd) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvSTWEnd) } // GCSweepStart prepares to trace a sweep loop. This does not @@ -376,7 +383,7 @@ func (tl traceLocker) GCSweepSpan(bytesSwept uintptr) { pp := tl.mp.p.ptr() if pp.trace.maySweep { if pp.trace.swept == 0 { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGCSweepBegin, tl.stack(1)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCSweepBegin, tl.stack(1)) pp.trace.inSweep = true } pp.trace.swept += bytesSwept @@ -394,7 +401,7 @@ func (tl traceLocker) GCSweepDone() { throw("missing traceGCSweepStart") } if pp.trace.inSweep { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGCSweepEnd, traceArg(pp.trace.swept), traceArg(pp.trace.reclaimed)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCSweepEnd, traceArg(pp.trace.swept), traceArg(pp.trace.reclaimed)) pp.trace.inSweep = false } pp.trace.maySweep = false @@ -402,22 +409,22 @@ func (tl traceLocker) GCSweepDone() { // GCMarkAssistStart emits a MarkAssistBegin event. func (tl traceLocker) GCMarkAssistStart() { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGCMarkAssistBegin, tl.stack(1)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCMarkAssistBegin, tl.stack(1)) } // GCMarkAssistDone emits a MarkAssistEnd event. func (tl traceLocker) GCMarkAssistDone() { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGCMarkAssistEnd) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCMarkAssistEnd) } // GoCreate emits a GoCreate event. func (tl traceLocker) GoCreate(newg *g, pc uintptr, blocked bool) { newg.trace.setStatusTraced(tl.gen) - ev := traceEvGoCreate + ev := tracev2.EvGoCreate if blocked { - ev = traceEvGoCreateBlocked + ev = tracev2.EvGoCreateBlocked } - tl.eventWriter(traceGoRunning, traceProcRunning).event(ev, traceArg(newg.goid), tl.startPC(pc), tl.stack(2)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(ev, traceArg(newg.goid), tl.startPC(pc), tl.stack(2)) } // GoStart emits a GoStart event. @@ -426,10 +433,10 @@ func (tl traceLocker) GoCreate(newg *g, pc uintptr, blocked bool) { func (tl traceLocker) GoStart() { gp := getg().m.curg pp := gp.m.p - w := tl.eventWriter(traceGoRunnable, traceProcRunning) - w.event(traceEvGoStart, traceArg(gp.goid), gp.trace.nextSeq(tl.gen)) + w := tl.eventWriter(tracev2.GoRunnable, tracev2.ProcRunning) + w.event(tracev2.EvGoStart, traceArg(gp.goid), gp.trace.nextSeq(tl.gen)) if pp.ptr().gcMarkWorkerMode != gcMarkWorkerNotWorker { - w.event(traceEvGoLabel, trace.markWorkerLabels[tl.gen%2][pp.ptr().gcMarkWorkerMode]) + w.event(tracev2.EvGoLabel, trace.markWorkerLabels[tl.gen%2][pp.ptr().gcMarkWorkerMode]) } } @@ -437,7 +444,7 @@ func (tl traceLocker) GoStart() { // // TODO(mknyszek): Rename this to GoDestroy. func (tl traceLocker) GoEnd() { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoDestroy) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoDestroy) } // GoSched emits a GoStop event with a GoSched reason. @@ -452,7 +459,7 @@ func (tl traceLocker) GoPreempt() { // GoStop emits a GoStop event with the provided reason. func (tl traceLocker) GoStop(reason traceGoStopReason) { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoStop, traceArg(trace.goStopReasons[tl.gen%2][reason]), tl.stack(1)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoStop, trace.goStopReasons[tl.gen%2][reason], tl.stack(0)) } // GoPark emits a GoBlock event with the provided reason. @@ -460,25 +467,25 @@ func (tl traceLocker) GoStop(reason traceGoStopReason) { // TODO(mknyszek): Replace traceBlockReason with waitReason. It's silly // that we have both, and waitReason is way more descriptive. func (tl traceLocker) GoPark(reason traceBlockReason, skip int) { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoBlock, traceArg(trace.goBlockReasons[tl.gen%2][reason]), tl.stack(skip)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoBlock, trace.goBlockReasons[tl.gen%2][reason], tl.stack(skip)) } // GoUnpark emits a GoUnblock event. func (tl traceLocker) GoUnpark(gp *g, skip int) { // Emit a GoWaiting status if necessary for the unblocked goroutine. tl.emitUnblockStatus(gp, tl.gen) - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoUnblock, traceArg(gp.goid), gp.trace.nextSeq(tl.gen), tl.stack(skip)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoUnblock, traceArg(gp.goid), gp.trace.nextSeq(tl.gen), tl.stack(skip)) } -// GoCoroswitch emits a GoSwitch event. If destroy is true, the calling goroutine +// GoSwitch emits a GoSwitch event. If destroy is true, the calling goroutine // is simultaneously being destroyed. func (tl traceLocker) GoSwitch(nextg *g, destroy bool) { // Emit a GoWaiting status if necessary for the unblocked goroutine. tl.emitUnblockStatus(nextg, tl.gen) - w := tl.eventWriter(traceGoRunning, traceProcRunning) - ev := traceEvGoSwitch + w := tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning) + ev := tracev2.EvGoSwitch if destroy { - ev = traceEvGoSwitchDestroy + ev = tracev2.EvGoSwitchDestroy } w.event(ev, traceArg(nextg.goid), nextg.trace.nextSeq(tl.gen)) } @@ -490,7 +497,7 @@ func (tl traceLocker) emitUnblockStatus(gp *g, gen uintptr) { // TODO(go.dev/issue/65634): Although it would be nice to add a stack trace here of gp, // we cannot safely do so. gp is in _Gwaiting and so we don't have ownership of its stack. // We can fix this by acquiring the goroutine's scan bit. - tl.writer().writeGoStatus(gp.goid, -1, traceGoWaiting, gp.inMarkAssist, 0).end() + tl.writer().writeGoStatus(gp.goid, -1, tracev2.GoWaiting, gp.inMarkAssist, 0).end() } } @@ -501,7 +508,7 @@ func (tl traceLocker) GoSysCall() { // Scribble down the M that the P is currently attached to. pp := tl.mp.p.ptr() pp.trace.mSyscallID = int64(tl.mp.procid) - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvGoSyscallBegin, pp.trace.nextSeq(tl.gen), tl.stack(1)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoSyscallBegin, pp.trace.nextSeq(tl.gen), tl.stack(1)) } // GoSysExit emits a GoSyscallEnd event, possibly along with a GoSyscallBlocked event @@ -514,15 +521,15 @@ func (tl traceLocker) GoSysCall() { // - The goroutine lost its P and was unable to reacquire it, and is now running without a P. // - The goroutine lost its P and acquired a different one, and is now running with that P. func (tl traceLocker) GoSysExit(lostP bool) { - ev := traceEvGoSyscallEnd - procStatus := traceProcSyscall // Procs implicitly enter traceProcSyscall on GoSyscallBegin. + ev := tracev2.EvGoSyscallEnd + procStatus := tracev2.ProcSyscall // Procs implicitly enter tracev2.ProcSyscall on GoSyscallBegin. if lostP { - ev = traceEvGoSyscallEndBlocked - procStatus = traceProcRunning // If a G has a P when emitting this event, it reacquired a P and is indeed running. + ev = tracev2.EvGoSyscallEndBlocked + procStatus = tracev2.ProcRunning // If a G has a P when emitting this event, it reacquired a P and is indeed running. } else { tl.mp.p.ptr().trace.mSyscallID = -1 } - tl.eventWriter(traceGoSyscall, procStatus).event(ev) + tl.eventWriter(tracev2.GoSyscall, procStatus).event(ev) } // ProcSteal indicates that our current M stole a P from another M. @@ -543,7 +550,7 @@ func (tl traceLocker) ProcSteal(pp *p, inSyscall bool) { if !pp.trace.statusWasTraced(tl.gen) && pp.trace.acquireStatus(tl.gen) { // Careful: don't use the event writer. We never want status or in-progress events // to trigger more in-progress events. - tl.writer().writeProcStatus(uint64(pp.id), traceProcSyscallAbandoned, pp.trace.inSweep).end() + tl.writer().writeProcStatus(uint64(pp.id), tracev2.ProcSyscallAbandoned, pp.trace.inSweep).end() } // The status of the proc and goroutine, if we need to emit one here, is not evident from the @@ -552,28 +559,30 @@ func (tl traceLocker) ProcSteal(pp *p, inSyscall bool) { // ourselves specifically to keep running. The two contexts look different, but can be summarized // fairly succinctly. In the former, we're a regular running goroutine and proc, if we have either. // In the latter, we're a goroutine in a syscall. - goStatus := traceGoRunning - procStatus := traceProcRunning + goStatus := tracev2.GoRunning + procStatus := tracev2.ProcRunning if inSyscall { - goStatus = traceGoSyscall - procStatus = traceProcSyscallAbandoned + goStatus = tracev2.GoSyscall + procStatus = tracev2.ProcSyscallAbandoned } - tl.eventWriter(goStatus, procStatus).event(traceEvProcSteal, traceArg(pp.id), pp.trace.nextSeq(tl.gen), traceArg(mStolenFrom)) + tl.eventWriter(goStatus, procStatus).event(tracev2.EvProcSteal, traceArg(pp.id), pp.trace.nextSeq(tl.gen), traceArg(mStolenFrom)) } // HeapAlloc emits a HeapAlloc event. func (tl traceLocker) HeapAlloc(live uint64) { - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvHeapAlloc, traceArg(live)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvHeapAlloc, traceArg(live)) } // HeapGoal reads the current heap goal and emits a HeapGoal event. func (tl traceLocker) HeapGoal() { heapGoal := gcController.heapGoal() - if heapGoal == ^uint64(0) { + // The heapGoal calculations will result in strange numbers if the GC if off. See go.dev/issue/63864. + // Check gcPercent before using the heapGoal in the trace. + if heapGoal == ^uint64(0) || gcController.gcPercent.Load() < 0 { // Heap-based triggering is disabled. heapGoal = 0 } - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvHeapGoal, traceArg(heapGoal)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvHeapGoal, traceArg(heapGoal)) } // GoCreateSyscall indicates that a goroutine has transitioned from dead to GoSyscall. @@ -586,7 +595,7 @@ func (tl traceLocker) GoCreateSyscall(gp *g) { // N.B. We should never trace a status for this goroutine (which we're currently running on), // since we want this to appear like goroutine creation. gp.trace.setStatusTraced(tl.gen) - tl.eventWriter(traceGoBad, traceProcBad).event(traceEvGoCreateSyscall, traceArg(gp.goid)) + tl.eventWriter(tracev2.GoBad, tracev2.ProcBad).event(tracev2.EvGoCreateSyscall, traceArg(gp.goid)) } // GoDestroySyscall indicates that a goroutine has transitioned from GoSyscall to dead. @@ -598,7 +607,7 @@ func (tl traceLocker) GoCreateSyscall(gp *g) { func (tl traceLocker) GoDestroySyscall() { // N.B. If we trace a status here, we must never have a P, and we must be on a goroutine // that is in the syscall state. - tl.eventWriter(traceGoSyscall, traceProcBad).event(traceEvGoDestroySyscall) + tl.eventWriter(tracev2.GoSyscall, tracev2.ProcBad).event(tracev2.EvGoDestroySyscall) } // To access runtime functions from runtime/trace. @@ -613,7 +622,7 @@ func trace_userTaskCreate(id, parentID uint64, taskType string) { // Need to do this check because the caller won't have it. return } - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvUserTaskBegin, traceArg(id), traceArg(parentID), tl.string(taskType), tl.stack(3)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvUserTaskBegin, traceArg(id), traceArg(parentID), tl.string(taskType), tl.stack(3)) traceRelease(tl) } @@ -626,11 +635,11 @@ func trace_userTaskEnd(id uint64) { // Need to do this check because the caller won't have it. return } - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvUserTaskEnd, traceArg(id), tl.stack(2)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvUserTaskEnd, traceArg(id), tl.stack(2)) traceRelease(tl) } -// trace_userTaskEnd emits a UserRegionBegin or UserRegionEnd event, +// trace_userRegion emits a UserRegionBegin or UserRegionEnd event, // depending on mode (0 == Begin, 1 == End). // // TODO(mknyszek): Just make this two functions. @@ -642,20 +651,20 @@ func trace_userRegion(id, mode uint64, name string) { // Need to do this check because the caller won't have it. return } - var ev traceEv + var ev tracev2.EventType switch mode { case 0: - ev = traceEvUserRegionBegin + ev = tracev2.EvUserRegionBegin case 1: - ev = traceEvUserRegionEnd + ev = tracev2.EvUserRegionEnd default: return } - tl.eventWriter(traceGoRunning, traceProcRunning).event(ev, traceArg(id), tl.string(name), tl.stack(3)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(ev, traceArg(id), tl.string(name), tl.stack(3)) traceRelease(tl) } -// trace_userTaskEnd emits a UserRegionBegin or UserRegionEnd event. +// trace_userLog emits a UserRegionBegin or UserRegionEnd event. // //go:linkname trace_userLog runtime/trace.userLog func trace_userLog(id uint64, category, message string) { @@ -664,7 +673,7 @@ func trace_userLog(id uint64, category, message string) { // Need to do this check because the caller won't have it. return } - tl.eventWriter(traceGoRunning, traceProcRunning).event(traceEvUserLog, traceArg(id), tl.string(category), tl.uniqueString(message), tl.stack(3)) + tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvUserLog, traceArg(id), tl.string(category), tl.uniqueString(message), tl.stack(3)) traceRelease(tl) } diff --git a/src/runtime/tracestack.go b/src/runtime/tracestack.go index 225566d1020fc3..51f3c29445c5a7 100644 --- a/src/runtime/tracestack.go +++ b/src/runtime/tracestack.go @@ -9,15 +9,11 @@ package runtime import ( "internal/abi" "internal/goarch" + "internal/trace/tracev2" "unsafe" ) const ( - // Maximum number of PCs in a single stack trace. - // Since events contain only stack id rather than whole stack trace, - // we can allow quite large values here. - traceStackSize = 128 - // logicalStackSentinel is a sentinel value at pcBuf[0] signifying that // pcBuf[1:] holds a logical stack requiring no further processing. Any other // value at pcBuf[0] represents a skip value to apply to the physical stack in @@ -32,11 +28,9 @@ const ( // skip controls the number of leaf frames to omit in order to hide tracer internals // from stack traces, see CL 5523. // -// Avoid calling this function directly. gen needs to be the current generation -// that this stack trace is being written out for, which needs to be synchronized with -// generations moving forward. Prefer traceEventWriter.stack. -func traceStack(skip int, gp *g, gen uintptr) uint64 { - var pcBuf [traceStackSize]uintptr +// Avoid calling this function directly. Prefer traceEventWriter.stack. +func traceStack(skip int, gp *g, tab *traceStackTable) uint64 { + var pcBuf [tracev2.MaxFramesPerStack]uintptr // Figure out gp and mp for the backtrace. var mp *m @@ -55,7 +49,7 @@ func traceStack(skip int, gp *g, gen uintptr) uint64 { // are totally fine for taking a stack trace. They're captured // correctly in goStatusToTraceGoStatus. switch goStatusToTraceGoStatus(status, gp.waitreason) { - case traceGoRunning, traceGoSyscall: + case tracev2.GoRunning, tracev2.GoSyscall: if getg() == gp || mp.curg == gp { break } @@ -113,7 +107,22 @@ func traceStack(skip int, gp *g, gen uintptr) uint64 { nstk += 1 + fpTracebackPCs(unsafe.Pointer(gp.syscallbp), pcBuf[2:]) } else { pcBuf[1] = gp.sched.pc - nstk += 1 + fpTracebackPCs(unsafe.Pointer(gp.sched.bp), pcBuf[2:]) + if gp.syncSafePoint { + // We're stopped in morestack, which is an odd state because gp.sched.bp + // refers to our parent frame, since we haven't had the chance to push our + // frame pointer to the stack yet. If we just start walking from gp.sched.bp, + // we'll skip a frame as a result. Luckily, we can find the PC we want right + // at gp.sched.sp on non-LR platforms, and we have it directly on LR platforms. + // See issue go.dev/issue/68090. + if usesLR { + pcBuf[2] = gp.sched.lr + } else { + pcBuf[2] = *(*uintptr)(unsafe.Pointer(gp.sched.sp)) + } + nstk += 2 + fpTracebackPCs(unsafe.Pointer(gp.sched.bp), pcBuf[3:]) + } else { + nstk += 1 + fpTracebackPCs(unsafe.Pointer(gp.sched.bp), pcBuf[2:]) + } } } } @@ -123,7 +132,7 @@ func traceStack(skip int, gp *g, gen uintptr) uint64 { if nstk > 0 && gp.goid == 1 { nstk-- // skip runtime.main } - id := trace.stackTab[gen%2].put(pcBuf[:nstk]) + id := tab.put(pcBuf[:nstk]) return id } @@ -147,7 +156,7 @@ func (t *traceStackTable) put(pcs []uintptr) uint64 { // releases all memory and resets state. It must only be called once the caller // can guarantee that there are no more writers to the table. func (t *traceStackTable) dump(gen uintptr) { - stackBuf := make([]uintptr, traceStackSize) + stackBuf := make([]uintptr, tracev2.MaxFramesPerStack) w := unsafeTraceWriter(gen, nil) if root := (*traceMapNode)(t.tab.root.Load()); root != nil { w = dumpStacksRec(root, w, stackBuf) @@ -172,16 +181,16 @@ func dumpStacksRec(node *traceMapNode, w traceWriter, stackBuf []uintptr) traceW // bound is pretty loose, but avoids counting // lots of varint sizes. // - // Add 1 because we might also write traceEvStacks. + // Add 1 because we might also write tracev2.EvStacks. var flushed bool w, flushed = w.ensure(1 + maxBytes) if flushed { - w.byte(byte(traceEvStacks)) + w.byte(byte(tracev2.EvStacks)) } // Emit stack event. - w.byte(byte(traceEvStack)) - w.varint(uint64(node.id)) + w.byte(byte(tracev2.EvStack)) + w.varint(node.id) w.varint(uint64(len(frames))) for _, frame := range frames { w.varint(uint64(frame.PC)) diff --git a/src/runtime/tracestack_test.go b/src/runtime/tracestack_test.go new file mode 100644 index 00000000000000..eaf4d906e35fd3 --- /dev/null +++ b/src/runtime/tracestack_test.go @@ -0,0 +1,46 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "runtime" + "strconv" + "testing" +) + +func BenchmarkTraceStack(b *testing.B) { + for _, stackDepth := range []int{1, 10, 100} { + b.Run("stackDepth="+strconv.Itoa(stackDepth), func(b *testing.B) { + benchmarkTraceStack(b, stackDepth) + }) + } +} + +func benchmarkTraceStack(b *testing.B, stackDepth int) { + var tab runtime.TraceStackTable + defer tab.Reset() + + wait := make(chan struct{}) + ready := make(chan struct{}) + done := make(chan struct{}) + var gp *runtime.G + go func() { + gp = runtime.Getg() + useStackAndCall(stackDepth, func() { + ready <- struct{}{} + <-wait + }) + done <- struct{}{} + }() + <-ready + + for b.Loop() { + runtime.TraceStack(gp, &tab) + } + + // Clean up. + wait <- struct{}{} + <-done +} diff --git a/src/runtime/tracestatus.go b/src/runtime/tracestatus.go index 425ac37ba04ee0..f09bdfd3554811 100644 --- a/src/runtime/tracestatus.go +++ b/src/runtime/tracestatus.go @@ -6,43 +6,9 @@ package runtime -import "internal/runtime/atomic" - -// traceGoStatus is the status of a goroutine. -// -// They correspond directly to the various goroutine -// statuses. -type traceGoStatus uint8 - -const ( - traceGoBad traceGoStatus = iota - traceGoRunnable - traceGoRunning - traceGoSyscall - traceGoWaiting -) - -// traceProcStatus is the status of a P. -// -// They mostly correspond to the various P statuses. -type traceProcStatus uint8 - -const ( - traceProcBad traceProcStatus = iota - traceProcRunning - traceProcIdle - traceProcSyscall - - // traceProcSyscallAbandoned is a special case of - // traceProcSyscall. It's used in the very specific case - // where the first a P is mentioned in a generation is - // part of a ProcSteal event. If that's the first time - // it's mentioned, then there's no GoSyscallBegin to - // connect the P stealing back to at that point. This - // special state indicates this to the parser, so it - // doesn't try to find a GoSyscallEndBlocked that - // corresponds with the ProcSteal. - traceProcSyscallAbandoned +import ( + "internal/runtime/atomic" + "internal/trace/tracev2" ) // writeGoStatus emits a GoStatus event as well as any active ranges on the goroutine. @@ -51,23 +17,23 @@ const ( // have any stack growth. // //go:nosplit -func (w traceWriter) writeGoStatus(goid uint64, mid int64, status traceGoStatus, markAssist bool, stackID uint64) traceWriter { +func (w traceWriter) writeGoStatus(goid uint64, mid int64, status tracev2.GoStatus, markAssist bool, stackID uint64) traceWriter { // The status should never be bad. Some invariant must have been violated. - if status == traceGoBad { + if status == tracev2.GoBad { print("runtime: goid=", goid, "\n") throw("attempted to trace a bad status for a goroutine") } // Trace the status. if stackID == 0 { - w = w.event(traceEvGoStatus, traceArg(goid), traceArg(uint64(mid)), traceArg(status)) + w = w.event(tracev2.EvGoStatus, traceArg(goid), traceArg(uint64(mid)), traceArg(status)) } else { - w = w.event(traceEvGoStatusStack, traceArg(goid), traceArg(uint64(mid)), traceArg(status), traceArg(stackID)) + w = w.event(tracev2.EvGoStatusStack, traceArg(goid), traceArg(uint64(mid)), traceArg(status), traceArg(stackID)) } // Trace any special ranges that are in-progress. if markAssist { - w = w.event(traceEvGCMarkAssistActive, traceArg(goid)) + w = w.event(tracev2.EvGCMarkAssistActive, traceArg(goid)) } return w } @@ -85,26 +51,26 @@ func (w traceWriter) writeProcStatusForP(pp *p, inSTW bool) traceWriter { if !pp.trace.acquireStatus(w.gen) { return w } - var status traceProcStatus + var status tracev2.ProcStatus switch pp.status { case _Pidle, _Pgcstop: - status = traceProcIdle + status = tracev2.ProcIdle if pp.status == _Pgcstop && inSTW { // N.B. a P that is running and currently has the world stopped will be // in _Pgcstop, but we model it as running in the tracer. - status = traceProcRunning + status = tracev2.ProcRunning } case _Prunning: - status = traceProcRunning + status = tracev2.ProcRunning // There's a short window wherein the goroutine may have entered _Gsyscall // but it still owns the P (it's not in _Psyscall yet). The goroutine entering // _Gsyscall is the tracer's signal that the P its bound to is also in a syscall, // so we need to emit a status that matches. See #64318. if w.mp.p.ptr() == pp && w.mp.curg != nil && readgstatus(w.mp.curg)&^_Gscan == _Gsyscall { - status = traceProcSyscall + status = tracev2.ProcSyscall } case _Psyscall: - status = traceProcSyscall + status = tracev2.ProcSyscall default: throw("attempt to trace invalid or unsupported P status") } @@ -121,19 +87,19 @@ func (w traceWriter) writeProcStatusForP(pp *p, inSTW bool) traceWriter { // have any stack growth. // //go:nosplit -func (w traceWriter) writeProcStatus(pid uint64, status traceProcStatus, inSweep bool) traceWriter { +func (w traceWriter) writeProcStatus(pid uint64, status tracev2.ProcStatus, inSweep bool) traceWriter { // The status should never be bad. Some invariant must have been violated. - if status == traceProcBad { + if status == tracev2.ProcBad { print("runtime: pid=", pid, "\n") throw("attempted to trace a bad status for a proc") } // Trace the status. - w = w.event(traceEvProcStatus, traceArg(pid), traceArg(status)) + w = w.event(tracev2.EvProcStatus, traceArg(pid), traceArg(status)) // Trace any special ranges that are in-progress. if inSweep { - w = w.event(traceEvGCSweepActive, traceArg(pid)) + w = w.event(tracev2.EvGCSweepActive, traceArg(pid)) } return w } @@ -146,28 +112,29 @@ func (w traceWriter) writeProcStatus(pid uint64, status traceProcStatus, inSweep // have any stack growth. // //go:nosplit -func goStatusToTraceGoStatus(status uint32, wr waitReason) traceGoStatus { +func goStatusToTraceGoStatus(status uint32, wr waitReason) tracev2.GoStatus { // N.B. Ignore the _Gscan bit. We don't model it in the tracer. - var tgs traceGoStatus + var tgs tracev2.GoStatus switch status &^ _Gscan { case _Grunnable: - tgs = traceGoRunnable + tgs = tracev2.GoRunnable case _Grunning, _Gcopystack: - tgs = traceGoRunning + tgs = tracev2.GoRunning case _Gsyscall: - tgs = traceGoSyscall - case _Gwaiting, _Gpreempted: + tgs = tracev2.GoSyscall + case _Gwaiting, _Gpreempted, _Gleaked: // There are a number of cases where a G might end up in // _Gwaiting but it's actually running in a non-preemptive // state but needs to present itself as preempted to the - // garbage collector. In these cases, we're not going to - // emit an event, and we want these goroutines to appear in - // the final trace as if they're running, not blocked. - tgs = traceGoWaiting - if status == _Gwaiting && wr.isWaitingForGC() { - tgs = traceGoRunning + // garbage collector and traceAdvance (via suspendG). In + // these cases, we're not going to emit an event, and we + // want these goroutines to appear in the final trace as + // if they're running, not blocked. + tgs = tracev2.GoWaiting + if status == _Gwaiting && wr.isWaitingForSuspendG() { + tgs = tracev2.GoRunning } - case _Gdead: + case _Gdead, _Gdeadextra: throw("tried to trace dead goroutine") default: throw("tried to trace goroutine with invalid or unsupported status") diff --git a/src/runtime/tracestring.go b/src/runtime/tracestring.go index 2585c69cc0b7ff..d486f9efbdf8b5 100644 --- a/src/runtime/tracestring.go +++ b/src/runtime/tracestring.go @@ -6,9 +6,9 @@ package runtime -// Trace strings. +import "internal/trace/tracev2" -const maxTraceStringLen = 1024 +// Trace strings. // traceStringTable is map of string -> unique ID that also manages // writing strings out into the trace. @@ -52,8 +52,8 @@ func (t *traceStringTable) emit(gen uintptr, s string) uint64 { //go:systemstack func (t *traceStringTable) writeString(gen uintptr, id uint64, s string) { // Truncate the string if necessary. - if len(s) > maxTraceStringLen { - s = s[:maxTraceStringLen] + if len(s) > tracev2.MaxEventTrailerDataSize { + s = s[:tracev2.MaxEventTrailerDataSize] } lock(&t.lock) @@ -61,14 +61,14 @@ func (t *traceStringTable) writeString(gen uintptr, id uint64, s string) { // Ensure we have a place to write to. var flushed bool - w, flushed = w.ensure(2 + 2*traceBytesPerNumber + len(s) /* traceEvStrings + traceEvString + ID + len + string data */) + w, flushed = w.ensure(2 + 2*traceBytesPerNumber + len(s) /* tracev2.EvStrings + tracev2.EvString + ID + len + string data */) if flushed { // Annotate the batch as containing strings. - w.byte(byte(traceEvStrings)) + w.byte(byte(tracev2.EvStrings)) } // Write out the string. - w.byte(byte(traceEvString)) + w.byte(byte(tracev2.EvString)) w.varint(id) w.varint(uint64(len(s))) w.stringData(s) diff --git a/src/runtime/tracetime.go b/src/runtime/tracetime.go index d5ee2b078fc022..8be5c3d1306201 100644 --- a/src/runtime/tracetime.go +++ b/src/runtime/tracetime.go @@ -8,6 +8,7 @@ package runtime import ( "internal/goarch" + "internal/trace/tracev2" _ "unsafe" ) @@ -50,7 +51,7 @@ type traceTime uint64 // nosplit because it's called from exitsyscall and various trace writing functions, // which are nosplit. // -// traceClockNow is called by golang.org/x/exp/trace using linkname. +// traceClockNow is called by runtime/trace and golang.org/x/exp/trace using linkname. // //go:linkname traceClockNow //go:nosplit @@ -63,6 +64,8 @@ func traceClockNow() traceTime { // traceClockUnitsPerSecond estimates the number of trace clock units per // second that elapse. +// +//go:linkname traceClockUnitsPerSecond runtime/trace.runtime_traceClockUnitsPerSecond func traceClockUnitsPerSecond() uint64 { if osHasLowResClock { // We're using cputicks as our clock, so we need a real estimate. @@ -73,18 +76,32 @@ func traceClockUnitsPerSecond() uint64 { return uint64(1.0 / float64(traceTimeDiv) * 1e9) } -// traceFrequency writes a batch with a single EvFrequency event. -// -// freq is the number of trace clock units per second. -func traceFrequency(gen uintptr) { +func traceSyncBatch(gen uintptr, frequency uint64) { w := unsafeTraceWriter(gen, nil) // Ensure we have a place to write to. - w, _ = w.ensure(1 + traceBytesPerNumber /* traceEvFrequency + frequency */) + w, _ = w.ensure(3 /* EvSync + EvFrequency + EvClockSnapshot */ + 5*traceBytesPerNumber /* frequency, timestamp, mono, sec, nsec */) + + // Write out the sync batch event. + w.byte(byte(tracev2.EvSync)) - // Write out the string. - w.byte(byte(traceEvFrequency)) - w.varint(traceClockUnitsPerSecond()) + // Write out the frequency event. + w.byte(byte(tracev2.EvFrequency)) + w.varint(frequency) + + // Write out the clock snapshot event. + sec, nsec, mono := time_now() + ts := traceClockNow() + if ts <= w.traceBuf.lastTime { + ts = w.traceBuf.lastTime + 1 + } + tsDiff := uint64(ts - w.traceBuf.lastTime) + w.traceBuf.lastTime = ts + w.byte(byte(tracev2.EvClockSnapshot)) + w.varint(tsDiff) + w.varint(uint64(mono)) + w.varint(uint64(sec)) + w.varint(uint64(nsec)) // Immediately flush the buffer. systemstack(func() { diff --git a/src/runtime/tracetype.go b/src/runtime/tracetype.go index d9e340f64a4cb1..5fe16d5e6b0c61 100644 --- a/src/runtime/tracetype.go +++ b/src/runtime/tracetype.go @@ -2,17 +2,18 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Trace stack table and acquisition. +// Trace type table. package runtime import ( "internal/abi" "internal/goarch" + "internal/trace/tracev2" "unsafe" ) -// traceTypeTable maps stack traces (arrays of PC's) to unique uint32 ids. +// traceTypeTable maps types to unique uint32 ids. // It is lock-free for reading. type traceTypeTable struct { tab traceMap @@ -35,7 +36,7 @@ func (t *traceTypeTable) put(typ *abi.Type) uint64 { // releases all memory and resets state. It must only be called once the caller // can guarantee that there are no more writers to the table. func (t *traceTypeTable) dump(gen uintptr) { - w := unsafeTraceExpWriter(gen, nil, traceExperimentAllocFree) + w := unsafeTraceExpWriter(gen, nil, tracev2.AllocFree) if root := (*traceMapNode)(t.tab.root.Load()); root != nil { w = dumpTypesRec(root, w) } @@ -63,7 +64,7 @@ func dumpTypesRec(node *traceMapNode, w traceWriter) traceWriter { } // Emit type. - w.varint(uint64(node.id)) + w.varint(node.id) w.varint(uint64(uintptr(unsafe.Pointer(typ)))) w.varint(uint64(typ.Size())) w.varint(uint64(typ.PtrBytes)) diff --git a/src/runtime/type.go b/src/runtime/type.go index 201340752b810c..9009119464c760 100644 --- a/src/runtime/type.go +++ b/src/runtime/type.go @@ -8,9 +8,16 @@ package runtime import ( "internal/abi" + "internal/goarch" + "internal/runtime/atomic" "unsafe" ) +//go:linkname maps_typeString internal/runtime/maps.typeString +func maps_typeString(typ *abi.Type) string { + return toRType(typ).string() +} + type nameOff = abi.NameOff type typeOff = abi.TypeOff type textOff = abi.TextOff @@ -61,7 +68,7 @@ func (t rtype) pkgpath() string { if u := t.uncommon(); u != nil { return t.nameOff(u.PkgPath).Name() } - switch t.Kind_ & abi.KindMask { + switch t.Kind() { case abi.Struct: st := (*structtype)(unsafe.Pointer(t.Type)) return st.PkgPath.Name() @@ -72,6 +79,184 @@ func (t rtype) pkgpath() string { return "" } +// getGCMask returns the pointer/nonpointer bitmask for type t. +// +// nosplit because it is used during write barriers and must not be preempted. +// +//go:nosplit +func getGCMask(t *_type) *byte { + if t.TFlag&abi.TFlagGCMaskOnDemand != 0 { + // Split the rest into getGCMaskOnDemand so getGCMask itself is inlineable. + return getGCMaskOnDemand(t) + } + return t.GCData +} + +// inProgress is a byte whose address is a sentinel indicating that +// some thread is currently building the GC bitmask for a type. +var inProgress byte + +// nosplit because it is used during write barriers and must not be preempted. +// +//go:nosplit +func getGCMaskOnDemand(t *_type) *byte { + // For large types, GCData doesn't point directly to a bitmask. + // Instead it points to a pointer to a bitmask, and the runtime + // is responsible for (on first use) creating the bitmask and + // storing a pointer to it in that slot. + // TODO: we could use &t.GCData as the slot, but types are + // in read-only memory currently. + addr := unsafe.Pointer(t.GCData) + + if GOOS == "aix" { + addr = add(addr, firstmoduledata.data-aixStaticDataBase) + } + + for { + p := (*byte)(atomic.Loadp(addr)) + switch p { + default: // Already built. + return p + case &inProgress: // Someone else is currently building it. + // Just wait until the builder is done. + // We can't block here, so spinning while having + // the OS thread yield is about the best we can do. + osyield() + continue + case nil: // Not built yet. + // Attempt to get exclusive access to build it. + if !atomic.Casp1((*unsafe.Pointer)(addr), nil, unsafe.Pointer(&inProgress)) { + continue + } + + // Build gcmask for this type. + bytes := goarch.PtrSize * divRoundUp(t.PtrBytes/goarch.PtrSize, 8*goarch.PtrSize) + p = (*byte)(persistentalloc(bytes, goarch.PtrSize, &memstats.other_sys)) + systemstack(func() { + buildGCMask(t, bitCursor{ptr: p, n: 0}) + }) + + // Store the newly-built gcmask for future callers. + atomic.StorepNoWB(addr, unsafe.Pointer(p)) + return p + } + } +} + +// A bitCursor is a simple cursor to memory to which we +// can write a set of bits. +type bitCursor struct { + ptr *byte // base of region + n uintptr // cursor points to bit n of region +} + +// Write to b cnt bits starting at bit 0 of data. +// Requires cnt>0. +func (b bitCursor) write(data *byte, cnt uintptr) { + // Starting byte for writing. + p := addb(b.ptr, b.n/8) + + // Note: if we're starting halfway through a byte, we load the + // existing lower bits so we don't clobber them. + n := b.n % 8 // # of valid bits in buf + buf := uintptr(*p) & (1< 8 { + // Read 8 more bits, now buf has 8-15 valid bits in it. + buf |= uintptr(*data) << n + n += 8 + data = addb(data, 1) + cnt -= 8 + // Write 8 of the buffered bits out. + *p = byte(buf) + buf >>= 8 + n -= 8 + p = addb(p, 1) + } + // Read remaining bits. + buf |= (uintptr(*data) & (1< 8 { + *p = byte(buf) + buf >>= 8 + n -= 8 + p = addb(p, 1) + } + *p &^= 1< t.Size_/2 { + // Avoid recursive call for field type that + // is larger than half of the parent type. + // There can be only one. + bigField = f + continue + } + buildGCMask(ft, dst.offset(f.Offset/goarch.PtrSize)) + } + if bigField.Typ != nil { + // Note: this case causes bits to be written out of order. + t = bigField.Typ + dst = dst.offset(bigField.Offset / goarch.PtrSize) + goto top + } + default: + throw("unexpected kind") + } +} + // reflectOffs holds type offsets defined at run time by the reflect package. // // When a type is defined at run time, its *rtype data lives on the heap. @@ -106,15 +291,6 @@ func reflectOffsUnlock() { unlock(&reflectOffs.lock) } -// resolveNameOff should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname resolveNameOff func resolveNameOff(ptrInModule unsafe.Pointer, off nameOff) name { if off == 0 { return name{} @@ -149,15 +325,6 @@ func (t rtype) nameOff(off nameOff) name { return resolveNameOff(unsafe.Pointer(t.Type), off) } -// resolveTypeOff should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/cloudwego/frugal -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname resolveTypeOff func resolveTypeOff(ptrInModule unsafe.Pointer, off typeOff) *_type { if off == 0 || off == -1 { // -1 is the sentinel value for unreachable code. @@ -235,8 +402,6 @@ type uncommontype = abi.UncommonType type interfacetype = abi.InterfaceType -type maptype = abi.MapType - type arraytype = abi.ArrayType type chantype = abi.ChanType @@ -356,8 +521,8 @@ func typesEqual(t, v *_type, seen map[_typePair]struct{}) bool { if t == v { return true } - kind := t.Kind_ & abi.KindMask - if kind != v.Kind_&abi.KindMask { + kind := t.Kind() + if kind != v.Kind() { return false } rt, rv := toRType(t), toRType(v) @@ -439,8 +604,8 @@ func typesEqual(t, v *_type, seen map[_typePair]struct{}) bool { } return true case abi.Map: - mt := (*maptype)(unsafe.Pointer(t)) - mv := (*maptype)(unsafe.Pointer(v)) + mt := (*abi.MapType)(unsafe.Pointer(t)) + mv := (*abi.MapType)(unsafe.Pointer(v)) return typesEqual(mt.Key, mv.Key, seen) && typesEqual(mt.Elem, mv.Elem, seen) case abi.Pointer: pt := (*ptrtype)(unsafe.Pointer(t)) diff --git a/src/runtime/typekind.go b/src/runtime/typekind.go deleted file mode 100644 index 4920a7cf146c59..00000000000000 --- a/src/runtime/typekind.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -import "internal/abi" - -// isDirectIface reports whether t is stored directly in an interface value. -func isDirectIface(t *_type) bool { - return t.Kind_&abi.KindDirectIface != 0 -} diff --git a/src/runtime/unsafe.go b/src/runtime/unsafe.go index 85fc8b61ebc0ef..185eae0f095ad2 100644 --- a/src/runtime/unsafe.go +++ b/src/runtime/unsafe.go @@ -6,6 +6,7 @@ package runtime import ( "internal/runtime/math" + "internal/runtime/sys" "unsafe" ) @@ -52,21 +53,21 @@ func panicunsafestringnilptr() { // Keep this code in sync with cmd/compile/internal/walk/builtin.go:walkUnsafeSlice func unsafeslice(et *_type, ptr unsafe.Pointer, len int) { if len < 0 { - panicunsafeslicelen1(getcallerpc()) + panicunsafeslicelen1(sys.GetCallerPC()) } if et.Size_ == 0 { if ptr == nil && len > 0 { - panicunsafeslicenilptr1(getcallerpc()) + panicunsafeslicenilptr1(sys.GetCallerPC()) } } mem, overflow := math.MulUintptr(et.Size_, uintptr(len)) if overflow || mem > -uintptr(ptr) { if ptr == nil { - panicunsafeslicenilptr1(getcallerpc()) + panicunsafeslicenilptr1(sys.GetCallerPC()) } - panicunsafeslicelen1(getcallerpc()) + panicunsafeslicelen1(sys.GetCallerPC()) } } @@ -74,7 +75,7 @@ func unsafeslice(et *_type, ptr unsafe.Pointer, len int) { func unsafeslice64(et *_type, ptr unsafe.Pointer, len64 int64) { len := int(len64) if int64(len) != len64 { - panicunsafeslicelen1(getcallerpc()) + panicunsafeslicelen1(sys.GetCallerPC()) } unsafeslice(et, ptr, len) } @@ -92,7 +93,7 @@ func unsafeslicecheckptr(et *_type, ptr unsafe.Pointer, len64 int64) { func panicunsafeslicelen() { // This is called only from compiler-generated code, so we can get the // source of the panic. - panicunsafeslicelen1(getcallerpc()) + panicunsafeslicelen1(sys.GetCallerPC()) } //go:yeswritebarrierrec @@ -104,7 +105,7 @@ func panicunsafeslicelen1(pc uintptr) { func panicunsafeslicenilptr() { // This is called only from compiler-generated code, so we can get the // source of the panic. - panicunsafeslicenilptr1(getcallerpc()) + panicunsafeslicenilptr1(sys.GetCallerPC()) } //go:yeswritebarrierrec diff --git a/src/runtime/unsafepoint_test.go b/src/runtime/unsafepoint_test.go index 2c97adead8929e..79f0171854191f 100644 --- a/src/runtime/unsafepoint_test.go +++ b/src/runtime/unsafepoint_test.go @@ -43,7 +43,7 @@ func TestUnsafePoint(t *testing.T) { cmd := exec.Command(testenv.GoToolPath(t), "tool", "objdump", "-s", "setGlobalPointer", os.Args[0]) out, err := cmd.CombinedOutput() if err != nil { - t.Fatalf("can't objdump %v", err) + t.Fatalf("can't objdump %v:\n%s", err, out) } lines := strings.Split(string(out), "\n")[1:] diff --git a/src/runtime/valgrind.go b/src/runtime/valgrind.go new file mode 100644 index 00000000000000..3933d63e6bc947 --- /dev/null +++ b/src/runtime/valgrind.go @@ -0,0 +1,138 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build valgrind && linux && (arm64 || amd64) + +package runtime + +import "unsafe" + +const valgrindenabled = true + +// Valgrind provides a mechanism to allow programs under test to modify +// Valgrinds behavior in certain ways, referred to as client requests [0]. These +// requests are triggered putting the address of a series of uints in a specific +// register and emitting a very specific sequence of assembly instructions. The +// result of the request (if there is one) is then put in another register for +// the program to retrieve. Each request is identified by a unique uint, which +// is passed as the first "argument". +// +// Valgrind provides headers (valgrind/valgrind.h, valgrind/memcheck.h) with +// macros that emit the correct assembly for these requests. Instead of copying +// these headers into the tree and using cgo to call the macros, we implement +// the client request assembly ourselves. Since both the magic instruction +// sequences, and the request uint's are stable, it is safe for us to implement. +// +// The client requests we add are used to describe our memory allocator to +// Valgrind, per [1]. We describe the allocator using the two-level mempool +// structure a We also add annotations which allow Valgrind to track which +// memory we use for stacks, which seems necessary to let it properly function. +// +// We describe the memory model to Valgrind as follows: we treat heap arenas as +// "pools" created with VALGRIND_CREATE_MEMPOOL_EXT (so that we can use +// VALGRIND_MEMPOOL_METAPOOL and VALGRIND_MEMPOOL_AUTO_FREE). Within the pool we +// treat spans as "superblocks", annotated with VALGRIND_MEMPOOL_ALLOC. We then +// allocate individual objects within spans with VALGRIND_MALLOCLIKE_BLOCK. +// +// [0] https://valgrind.org/docs/manual/manual-core-adv.html#manual-core-adv.clientreq +// [1] https://valgrind.org/docs/manual/mc-manual.html#mc-manual.mempools + +const ( + // Valgrind request IDs, copied from valgrind/valgrind.h. + vg_userreq__malloclike_block = 0x1301 + vg_userreq__freelike_block = 0x1302 + vg_userreq__create_mempool = 0x1303 + vg_userreq__mempool_alloc = 0x1305 + vg_userreq__mempool_free = 0x1306 + vg_userreq__stack_register = 0x1501 + vg_userreq__stack_deregister = 0x1502 + vg_userreq__stack_change = 0x1503 +) + +const ( + // Memcheck request IDs are offset from ('M'&0xff) << 24 | ('C'&0xff) << 16, or 0x4d430000, + // copied from valgrind/memcheck.h. + vg_userreq__make_mem_noaccess = iota + ('M'&0xff)<<24 | ('C'&0xff)<<16 + vg_userreq__make_mem_undefined + vg_userreq__make_mem_defined +) + +const ( + // VALGRIND_CREATE_MEMPOOL_EXT flags, copied from valgrind/valgrind.h. + valgrind_mempool_auto_free = 1 + valgrind_mempool_metapool = 2 +) + +// + +//go:noescape +func valgrindClientRequest(uintptr, uintptr, uintptr, uintptr, uintptr, uintptr) uintptr + +//go:nosplit +func valgrindRegisterStack(start, end unsafe.Pointer) uintptr { + // VALGRIND_STACK_REGISTER + return valgrindClientRequest(vg_userreq__stack_register, uintptr(start), uintptr(end), 0, 0, 0) +} + +//go:nosplit +func valgrindDeregisterStack(id uintptr) { + // VALGRIND_STACK_DEREGISTER + valgrindClientRequest(vg_userreq__stack_deregister, id, 0, 0, 0, 0) +} + +//go:nosplit +func valgrindChangeStack(id uintptr, start, end unsafe.Pointer) { + // VALGRIND_STACK_CHANGE + valgrindClientRequest(vg_userreq__stack_change, id, uintptr(start), uintptr(end), 0, 0) +} + +//go:nosplit +func valgrindMalloc(addr unsafe.Pointer, size uintptr) { + // VALGRIND_MALLOCLIKE_BLOCK + valgrindClientRequest(vg_userreq__malloclike_block, uintptr(addr), size, 0, 1, 0) +} + +//go:nosplit +func valgrindFree(addr unsafe.Pointer) { + // VALGRIND_FREELIKE_BLOCK + valgrindClientRequest(vg_userreq__freelike_block, uintptr(addr), 0, 0, 0, 0) +} + +//go:nosplit +func valgrindCreateMempool(addr unsafe.Pointer) { + // VALGRIND_CREATE_MEMPOOL_EXT + valgrindClientRequest(vg_userreq__create_mempool, uintptr(addr), 0, 1, valgrind_mempool_auto_free|valgrind_mempool_metapool, 0) +} + +//go:nosplit +func valgrindMempoolMalloc(pool, addr unsafe.Pointer, size uintptr) { + // VALGRIND_MEMPOOL_ALLOC + valgrindClientRequest(vg_userreq__mempool_alloc, uintptr(pool), uintptr(addr), size, 0, 0) +} + +//go:nosplit +func valgrindMempoolFree(pool, addr unsafe.Pointer) { + // VALGRIND_MEMPOOL_FREE + valgrindClientRequest(vg_userreq__mempool_free, uintptr(pool), uintptr(addr), 0, 0, 0) +} + +// Memcheck client requests, copied from valgrind/memcheck.h + +//go:nosplit +func valgrindMakeMemUndefined(addr unsafe.Pointer, size uintptr) { + // VALGRIND_MAKE_MEM_UNDEFINED + valgrindClientRequest(vg_userreq__make_mem_undefined, uintptr(addr), size, 0, 0, 0) +} + +//go:nosplit +func valgrindMakeMemDefined(addr unsafe.Pointer, size uintptr) { + // VALGRIND_MAKE_MEM_DEFINED + valgrindClientRequest(vg_userreq__make_mem_defined, uintptr(addr), size, 0, 0, 0) +} + +//go:nosplit +func valgrindMakeMemNoAccess(addr unsafe.Pointer, size uintptr) { + // VALGRIND_MAKE_MEM_NOACCESS + valgrindClientRequest(vg_userreq__make_mem_noaccess, uintptr(addr), size, 0, 0, 0) +} diff --git a/src/runtime/valgrind0.go b/src/runtime/valgrind0.go new file mode 100644 index 00000000000000..adafa3001a1b21 --- /dev/null +++ b/src/runtime/valgrind0.go @@ -0,0 +1,25 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Valgrind instrumentation is only available on linux amd64 and arm64. + +//go:build !valgrind || !linux || (!amd64 && !arm64) + +package runtime + +import "unsafe" + +const valgrindenabled = false + +func valgrindRegisterStack(start, end unsafe.Pointer) uintptr { return 0 } +func valgrindDeregisterStack(id uintptr) {} +func valgrindChangeStack(id uintptr, start, end unsafe.Pointer) {} +func valgrindMalloc(addr unsafe.Pointer, size uintptr) {} +func valgrindFree(addr unsafe.Pointer) {} +func valgrindCreateMempool(addr unsafe.Pointer) {} +func valgrindMempoolMalloc(pool, addr unsafe.Pointer, size uintptr) {} +func valgrindMempoolFree(pool, addr unsafe.Pointer) {} +func valgrindMakeMemUndefined(addr unsafe.Pointer, size uintptr) {} +func valgrindMakeMemDefined(addr unsafe.Pointer, size uintptr) {} +func valgrindMakeMemNoAccess(addr unsafe.Pointer, size uintptr) {} diff --git a/src/runtime/valgrind_amd64.s b/src/runtime/valgrind_amd64.s new file mode 100644 index 00000000000000..1c53d4f4e5d631 --- /dev/null +++ b/src/runtime/valgrind_amd64.s @@ -0,0 +1,37 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build valgrind && linux + +#include "textflag.h" + +// Instead of using cgo and using the Valgrind macros, we just emit the special client request +// assembly ourselves. The client request mechanism is basically the same across all architectures, +// with the notable difference being the special preamble that lets Valgrind know we want to do +// a client request. +// +// The form of the VALGRIND_DO_CLIENT_REQUEST macro assembly can be found in the valgrind/valgrind.h +// header file [0]. +// +// [0] https://sourceware.org/git/?p=valgrind.git;a=blob;f=include/valgrind.h.in;h=f1710924aa7372e7b7e2abfbf7366a2286e33d2d;hb=HEAD + +// func valgrindClientRequest(uintptr, uintptr, uintptr, uintptr, uintptr, uintptr) (ret uintptr) +TEXT runtime·valgrindClientRequest(SB), NOSPLIT, $0-56 + // Load the address of the first of the (contiguous) arguments into AX. + LEAQ args+0(FP), AX + + // Zero DX, since some requests may not populate it. + XORL DX, DX + + // Emit the special preabmle. + ROLQ $3, DI; ROLQ $13, DI + ROLQ $61, DI; ROLQ $51, DI + + // "Execute" the client request. + XCHGQ BX, BX + + // Copy the result out of DX. + MOVQ DX, ret+48(FP) + + RET diff --git a/src/runtime/valgrind_arm64.s b/src/runtime/valgrind_arm64.s new file mode 100644 index 00000000000000..a46c3d4b9be99c --- /dev/null +++ b/src/runtime/valgrind_arm64.s @@ -0,0 +1,29 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build valgrind && linux + +#include "textflag.h" + +// See valgrind_amd64.s for notes about this assembly. + +// func valgrindClientRequest(uintptr, uintptr, uintptr, uintptr, uintptr, uintptr) (ret uintptr) +TEXT runtime·valgrindClientRequest(SB), NOSPLIT, $0-56 + // Load the address of the first of the (contiguous) arguments into x4. + MOVD $args+0(FP), R4 + + // Zero x3, since some requests may not populate it. + MOVD ZR, R3 + + // Emit the special preamble. + ROR $3, R12; ROR $13, R12 + ROR $51, R12; ROR $61, R12 + + // "Execute" the client request. + ORR R10, R10 + + // Copy the result out of x3. + MOVD R3, ret+48(FP) + + RET diff --git a/src/runtime/vdso_linux.go b/src/runtime/vdso_linux.go index 45236157116149..c068eede777918 100644 --- a/src/runtime/vdso_linux.go +++ b/src/runtime/vdso_linux.go @@ -97,6 +97,8 @@ type vdsoInfo struct { verdef *elfVerdef } +var vdsoLoadStart, vdsoLoadEnd uintptr + // see vdso_linux_*.go for vdsoSymbolKeys[] and vdso*Sym vars func vdsoInitFromSysinfoEhdr(info *vdsoInfo, hdr *elfEhdr) { @@ -116,6 +118,8 @@ func vdsoInitFromSysinfoEhdr(info *vdsoInfo, hdr *elfEhdr) { if !foundVaddr { foundVaddr = true info.loadOffset = info.loadAddr + uintptr(pt.p_offset-pt.p_vaddr) + vdsoLoadStart = info.loadOffset + vdsoLoadEnd = info.loadOffset + uintptr(pt.p_memsz) } case _PT_DYNAMIC: @@ -281,15 +285,9 @@ func vdsoauxv(tag, val uintptr) { } } -// vdsoMarker reports whether PC is on the VDSO page. +// inVDSOPage reports whether PC is on the VDSO page. // //go:nosplit func inVDSOPage(pc uintptr) bool { - for _, k := range vdsoSymbolKeys { - if *k.ptr != 0 { - page := *k.ptr &^ (physPageSize - 1) - return pc >= page && pc < page+physPageSize - } - } - return false + return pc >= vdsoLoadStart && pc < vdsoLoadEnd } diff --git a/src/runtime/vdso_linux_amd64.go b/src/runtime/vdso_linux_amd64.go index 9c564091377e01..8a89771ca4b473 100644 --- a/src/runtime/vdso_linux_amd64.go +++ b/src/runtime/vdso_linux_amd64.go @@ -17,11 +17,13 @@ var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6", 0x3ae75f6} var vdsoSymbolKeys = []vdsoSymbolKey{ {"__vdso_gettimeofday", 0x315ca59, 0xb01bca00, &vdsoGettimeofdaySym}, {"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym}, + {"__vdso_getrandom", 0x25425d, 0x84a559bf, &vdsoGetrandomSym}, } var ( vdsoGettimeofdaySym uintptr vdsoClockgettimeSym uintptr + vdsoGetrandomSym uintptr ) // vdsoGettimeofdaySym is accessed from the syscall package. diff --git a/src/runtime/vdso_linux_arm64.go b/src/runtime/vdso_linux_arm64.go index f5959525afd507..21f875d0df68f0 100644 --- a/src/runtime/vdso_linux_arm64.go +++ b/src/runtime/vdso_linux_arm64.go @@ -15,7 +15,10 @@ var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6.39", 0x75fcb89} var vdsoSymbolKeys = []vdsoSymbolKey{ {"__kernel_clock_gettime", 0xb0cd725, 0xdfa941fd, &vdsoClockgettimeSym}, + {"__kernel_getrandom", 0x9800c0d, 0x540d4e24, &vdsoGetrandomSym}, } -// initialize to fall back to syscall -var vdsoClockgettimeSym uintptr = 0 +var ( + vdsoClockgettimeSym uintptr + vdsoGetrandomSym uintptr +) diff --git a/src/runtime/vdso_linux_loong64.go b/src/runtime/vdso_linux_loong64.go index e00ef952b376d8..37ee02dcb01607 100644 --- a/src/runtime/vdso_linux_loong64.go +++ b/src/runtime/vdso_linux_loong64.go @@ -19,9 +19,10 @@ var vdsoLinuxVersion = vdsoVersionKey{"LINUX_5.10", 0xae78f70} var vdsoSymbolKeys = []vdsoSymbolKey{ {"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &vdsoClockgettimeSym}, + {"__vdso_getrandom", 0x25425d, 0x84a559bf, &vdsoGetrandomSym}, } -// initialize to fall back to syscall var ( - vdsoClockgettimeSym uintptr = 0 + vdsoClockgettimeSym uintptr + vdsoGetrandomSym uintptr ) diff --git a/src/runtime/vdso_linux_ppc64x.go b/src/runtime/vdso_linux_ppc64x.go index 09c8d9d20ed8a8..939da7b54c5d1a 100644 --- a/src/runtime/vdso_linux_ppc64x.go +++ b/src/runtime/vdso_linux_ppc64x.go @@ -16,9 +16,10 @@ var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6.15", 0x75fcba5} var vdsoSymbolKeys = []vdsoSymbolKey{ {"__kernel_clock_gettime", 0xb0cd725, 0xdfa941fd, &vdsoClockgettimeSym}, + {"__kernel_getrandom", 0x9800c0d, 0x540d4e24, &vdsoGetrandomSym}, } -// initialize with vsyscall fallbacks var ( - vdsoClockgettimeSym uintptr = 0 + vdsoClockgettimeSym uintptr + vdsoGetrandomSym uintptr ) diff --git a/src/runtime/vdso_linux_s390x.go b/src/runtime/vdso_linux_s390x.go index 970ecd3ce63076..113152ff0ac171 100644 --- a/src/runtime/vdso_linux_s390x.go +++ b/src/runtime/vdso_linux_s390x.go @@ -16,9 +16,10 @@ var vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6.29", 0x75fcbb9} var vdsoSymbolKeys = []vdsoSymbolKey{ {"__kernel_clock_gettime", 0xb0cd725, 0xdfa941fd, &vdsoClockgettimeSym}, + {"__kernel_getrandom", 0x9800c0d, 0x540d4e24, &vdsoGetrandomSym}, } -// initialize with vsyscall fallbacks var ( - vdsoClockgettimeSym uintptr = 0 + vdsoClockgettimeSym uintptr + vdsoGetrandomSym uintptr ) diff --git a/src/runtime/vdso_linux_test.go b/src/runtime/vdso_linux_test.go new file mode 100644 index 00000000000000..313dd6e7185a6a --- /dev/null +++ b/src/runtime/vdso_linux_test.go @@ -0,0 +1,52 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (386 || amd64 || arm || arm64 || loong64 || mips64 || mips64le || ppc64 || ppc64le || riscv64 || s390x) + +package runtime_test + +import ( + "runtime" + "testing" +) + +// DT_GNU_HASH hash function. +func gnuHash(s string) uint32 { + h := uint32(5381) + for _, r := range s { + h = (h << 5) + h + uint32(r) + } + return h +} + +// DT_HASH hash function. +func symHash(s string) uint32 { + var h, g uint32 + for _, r := range s { + h = (h << 4) + uint32(r) + g = h & 0xf0000000 + if g != 0 { + h ^= g >> 24 + } + h &^= g + } + return h +} + +func TestVDSOHash(t *testing.T) { + for _, sym := range runtime.VDSOSymbolKeys() { + name := sym.Name() + t.Run(name, func(t *testing.T) { + want := symHash(name) + if sym.SymHash() != want { + t.Errorf("SymHash got %#x want %#x", sym.SymHash(), want) + } + + want = gnuHash(name) + if sym.GNUHash() != want { + t.Errorf("GNUHash got %#x want %#x", sym.GNUHash(), want) + } + }) + } +} diff --git a/src/runtime/vdso_test.go b/src/runtime/vdso_test.go index d025ba50c212ac..cb70a040d65fb4 100644 --- a/src/runtime/vdso_test.go +++ b/src/runtime/vdso_test.go @@ -8,6 +8,7 @@ package runtime_test import ( "bytes" + "internal/asan" "internal/testenv" "os" "os/exec" @@ -20,6 +21,10 @@ import ( // TestUsingVDSO tests that we are actually using the VDSO to fetch // the time. func TestUsingVDSO(t *testing.T) { + if asan.Enabled { + t.Skip("test fails with ASAN beause the ASAN leak checker won't run under strace") + } + const calls = 100 if os.Getenv("GO_WANT_HELPER_PROCESS") == "1" { @@ -57,7 +62,7 @@ func TestUsingVDSO(t *testing.T) { t.Logf("%s", out) } if err != nil { - if err := err.(*exec.ExitError); err != nil && err.Sys().(syscall.WaitStatus).Signaled() { + if err, ok := err.(*exec.ExitError); ok && err.Sys().(syscall.WaitStatus).Signaled() { if !bytes.Contains(out, []byte("+++ killed by")) { // strace itself occasionally crashes. // Here, it exited with a signal, but diff --git a/src/runtime/vgetrandom_linux.go b/src/runtime/vgetrandom_linux.go new file mode 100644 index 00000000000000..225f7029be1b97 --- /dev/null +++ b/src/runtime/vgetrandom_linux.go @@ -0,0 +1,120 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (amd64 || arm64 || arm64be || ppc64 || ppc64le || loong64 || s390x) + +package runtime + +import ( + "internal/cpu" + "unsafe" +) + +//go:noescape +func vgetrandom1(buf *byte, length uintptr, flags uint32, state uintptr, stateSize uintptr) int + +var vgetrandomAlloc struct { + states []uintptr + statesLock mutex + stateSize uintptr + mmapProt int32 + mmapFlags int32 +} + +func vgetrandomInit() { + if vdsoGetrandomSym == 0 { + return + } + + var params struct { + SizeOfOpaqueState uint32 + MmapProt uint32 + MmapFlags uint32 + reserved [13]uint32 + } + if vgetrandom1(nil, 0, 0, uintptr(unsafe.Pointer(¶ms)), ^uintptr(0)) != 0 { + return + } + vgetrandomAlloc.stateSize = uintptr(params.SizeOfOpaqueState) + vgetrandomAlloc.mmapProt = int32(params.MmapProt) + vgetrandomAlloc.mmapFlags = int32(params.MmapFlags) + + lockInit(&vgetrandomAlloc.statesLock, lockRankVgetrandom) +} + +func vgetrandomGetState() uintptr { + lock(&vgetrandomAlloc.statesLock) + if len(vgetrandomAlloc.states) == 0 { + num := uintptr(numCPUStartup) // Just a reasonable size hint to start. + stateSizeCacheAligned := (vgetrandomAlloc.stateSize + cpu.CacheLineSize - 1) &^ (cpu.CacheLineSize - 1) + allocSize := (num*stateSizeCacheAligned + physPageSize - 1) &^ (physPageSize - 1) + num = (physPageSize / stateSizeCacheAligned) * (allocSize / physPageSize) + p, err := mmap(nil, allocSize, vgetrandomAlloc.mmapProt, vgetrandomAlloc.mmapFlags, -1, 0) + if err != 0 { + unlock(&vgetrandomAlloc.statesLock) + return 0 + } + setVMAName(p, allocSize, "getrandom states") + newBlock := uintptr(p) + if vgetrandomAlloc.states == nil { + vgetrandomAlloc.states = make([]uintptr, 0, num) + } + for i := uintptr(0); i < num; i++ { + if (newBlock&(physPageSize-1))+vgetrandomAlloc.stateSize > physPageSize { + newBlock = (newBlock + physPageSize - 1) &^ (physPageSize - 1) + } + vgetrandomAlloc.states = append(vgetrandomAlloc.states, newBlock) + newBlock += stateSizeCacheAligned + } + } + state := vgetrandomAlloc.states[len(vgetrandomAlloc.states)-1] + vgetrandomAlloc.states = vgetrandomAlloc.states[:len(vgetrandomAlloc.states)-1] + unlock(&vgetrandomAlloc.statesLock) + return state +} + +// Free vgetrandom state from the M (if any) prior to destroying the M. +// +// This may allocate, so it must have a P. +func vgetrandomDestroy(mp *m) { + if mp.vgetrandomState == 0 { + return + } + + lock(&vgetrandomAlloc.statesLock) + vgetrandomAlloc.states = append(vgetrandomAlloc.states, mp.vgetrandomState) + unlock(&vgetrandomAlloc.statesLock) +} + +// This is exported for use in internal/syscall/unix as well as x/sys/unix. +// +//go:linkname vgetrandom +func vgetrandom(p []byte, flags uint32) (ret int, supported bool) { + if vgetrandomAlloc.stateSize == 0 { + return -1, false + } + + // We use getg().m instead of acquirem() here, because always taking + // the lock is slightly more expensive than not always taking the lock. + // However, we *do* require that m doesn't migrate elsewhere during the + // execution of the vDSO. So, we exploit two details: + // 1) Asynchronous preemption is aborted when PC is in the runtime. + // 2) Most of the time, this function only calls vgetrandom1(), which + // does not have a preamble that synchronously preempts. + // We do need to take the lock when getting a new state for m, but this + // is very much the slow path, in the sense that it only ever happens + // once over the entire lifetime of an m. So, a simple getg().m suffices. + mp := getg().m + + if mp.vgetrandomState == 0 { + mp.locks++ + state := vgetrandomGetState() + mp.locks-- + if state == 0 { + return -1, false + } + mp.vgetrandomState = state + } + return vgetrandom1(unsafe.SliceData(p), uintptr(len(p)), flags, mp.vgetrandomState, vgetrandomAlloc.stateSize), true +} diff --git a/src/runtime/vgetrandom_unsupported.go b/src/runtime/vgetrandom_unsupported.go new file mode 100644 index 00000000000000..43c53e1198843c --- /dev/null +++ b/src/runtime/vgetrandom_unsupported.go @@ -0,0 +1,18 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !(linux && (amd64 || arm64 || arm64be || ppc64 || ppc64le || loong64 || s390x)) + +package runtime + +import _ "unsafe" + +//go:linkname vgetrandom +func vgetrandom(p []byte, flags uint32) (ret int, supported bool) { + return -1, false +} + +func vgetrandomDestroy(mp *m) {} + +func vgetrandomInit() {} diff --git a/src/runtime/wincallback.go b/src/runtime/wincallback.go index 14847db3fdbc78..7f0ac70bfd1ef3 100644 --- a/src/runtime/wincallback.go +++ b/src/runtime/wincallback.go @@ -47,34 +47,6 @@ TEXT runtime·callbackasm(SB),NOSPLIT|NOFRAME,$0 } } -func genasmArm() { - var buf bytes.Buffer - - buf.WriteString(`// Code generated by wincallback.go using 'go generate'. DO NOT EDIT. - -// External code calls into callbackasm at an offset corresponding -// to the callback index. Callbackasm is a table of MOV and B instructions. -// The MOV instruction loads R12 with the callback index, and the -// B instruction branches to callbackasm1. -// callbackasm1 takes the callback index from R12 and -// indexes into an array that stores information about each callback. -// It then calls the Go implementation for that callback. -#include "textflag.h" - -TEXT runtime·callbackasm(SB),NOSPLIT|NOFRAME,$0 -`) - for i := 0; i < maxCallback; i++ { - fmt.Fprintf(&buf, "\tMOVW\t$%d, R12\n", i) - buf.WriteString("\tB\truntime·callbackasm1(SB)\n") - } - - err := os.WriteFile("zcallback_windows_arm.s", buf.Bytes(), 0666) - if err != nil { - fmt.Fprintf(os.Stderr, "wincallback: %s\n", err) - os.Exit(2) - } -} - func genasmArm64() { var buf bytes.Buffer @@ -121,7 +93,6 @@ const cb_max = %d // maximum number of windows callbacks allowed func main() { genasm386Amd64() - genasmArm() genasmArm64() gengo() } diff --git a/src/runtime/write_err_android.go b/src/runtime/write_err_android.go index 34de106b50924a..bcc934e54c0461 100644 --- a/src/runtime/write_err_android.go +++ b/src/runtime/write_err_android.go @@ -4,7 +4,10 @@ package runtime -import "unsafe" +import ( + "internal/byteorder" + "unsafe" +) var ( writeHeader = []byte{6 /* ANDROID_LOG_ERROR */, 'G', 'o', 0} @@ -148,18 +151,10 @@ func writeLogdHeader() int { // hdr[7:11] nsec unsigned uint32, little endian. hdr[0] = 0 // LOG_ID_MAIN sec, nsec, _ := time_now() - packUint32(hdr[3:7], uint32(sec)) - packUint32(hdr[7:11], uint32(nsec)) + byteorder.LEPutUint32(hdr[3:7], uint32(sec)) + byteorder.LEPutUint32(hdr[7:11], uint32(nsec)) // TODO(hakim): hdr[1:2] = gettid? return 11 + len(writeHeader) } - -func packUint32(b []byte, v uint32) { - // little-endian. - b[0] = byte(v) - b[1] = byte(v >> 8) - b[2] = byte(v >> 16) - b[3] = byte(v >> 24) -} diff --git a/src/runtime/zcallback_windows_arm.s b/src/runtime/zcallback_windows_arm.s deleted file mode 100644 index f943d84cbfe5fd..00000000000000 --- a/src/runtime/zcallback_windows_arm.s +++ /dev/null @@ -1,4012 +0,0 @@ -// Code generated by wincallback.go using 'go generate'. DO NOT EDIT. - -// External code calls into callbackasm at an offset corresponding -// to the callback index. Callbackasm is a table of MOV and B instructions. -// The MOV instruction loads R12 with the callback index, and the -// B instruction branches to callbackasm1. -// callbackasm1 takes the callback index from R12 and -// indexes into an array that stores information about each callback. -// It then calls the Go implementation for that callback. -#include "textflag.h" - -TEXT runtime·callbackasm(SB),NOSPLIT|NOFRAME,$0 - MOVW $0, R12 - B runtime·callbackasm1(SB) - MOVW $1, R12 - B runtime·callbackasm1(SB) - MOVW $2, R12 - B runtime·callbackasm1(SB) - MOVW $3, R12 - B runtime·callbackasm1(SB) - MOVW $4, R12 - B runtime·callbackasm1(SB) - MOVW $5, R12 - B runtime·callbackasm1(SB) - MOVW $6, R12 - B runtime·callbackasm1(SB) - MOVW $7, R12 - B runtime·callbackasm1(SB) - MOVW $8, R12 - B runtime·callbackasm1(SB) - MOVW $9, R12 - B runtime·callbackasm1(SB) - MOVW $10, R12 - B runtime·callbackasm1(SB) - MOVW $11, R12 - B runtime·callbackasm1(SB) - MOVW $12, R12 - B runtime·callbackasm1(SB) - MOVW $13, R12 - B runtime·callbackasm1(SB) - MOVW $14, R12 - B runtime·callbackasm1(SB) - MOVW $15, R12 - B runtime·callbackasm1(SB) - MOVW $16, R12 - B runtime·callbackasm1(SB) - MOVW $17, R12 - B runtime·callbackasm1(SB) - MOVW $18, R12 - B runtime·callbackasm1(SB) - MOVW $19, R12 - B runtime·callbackasm1(SB) - MOVW $20, R12 - B runtime·callbackasm1(SB) - MOVW $21, R12 - B runtime·callbackasm1(SB) - MOVW $22, R12 - B runtime·callbackasm1(SB) - MOVW $23, R12 - B runtime·callbackasm1(SB) - MOVW $24, R12 - B runtime·callbackasm1(SB) - MOVW $25, R12 - B runtime·callbackasm1(SB) - MOVW $26, R12 - B runtime·callbackasm1(SB) - MOVW $27, R12 - B runtime·callbackasm1(SB) - MOVW $28, R12 - B runtime·callbackasm1(SB) - MOVW $29, R12 - B runtime·callbackasm1(SB) - MOVW $30, R12 - B runtime·callbackasm1(SB) - MOVW $31, R12 - B runtime·callbackasm1(SB) - MOVW $32, R12 - B runtime·callbackasm1(SB) - MOVW $33, R12 - B runtime·callbackasm1(SB) - MOVW $34, R12 - B runtime·callbackasm1(SB) - MOVW $35, R12 - B runtime·callbackasm1(SB) - MOVW $36, R12 - B runtime·callbackasm1(SB) - MOVW $37, R12 - B runtime·callbackasm1(SB) - MOVW $38, R12 - B runtime·callbackasm1(SB) - MOVW $39, R12 - B runtime·callbackasm1(SB) - MOVW $40, R12 - B runtime·callbackasm1(SB) - MOVW $41, R12 - B runtime·callbackasm1(SB) - MOVW $42, R12 - B runtime·callbackasm1(SB) - MOVW $43, R12 - B runtime·callbackasm1(SB) - MOVW $44, R12 - B runtime·callbackasm1(SB) - MOVW $45, R12 - B runtime·callbackasm1(SB) - MOVW $46, R12 - B runtime·callbackasm1(SB) - MOVW $47, R12 - B runtime·callbackasm1(SB) - MOVW $48, R12 - B runtime·callbackasm1(SB) - MOVW $49, R12 - B runtime·callbackasm1(SB) - MOVW $50, R12 - B runtime·callbackasm1(SB) - MOVW $51, R12 - B runtime·callbackasm1(SB) - MOVW $52, R12 - B runtime·callbackasm1(SB) - MOVW $53, R12 - B runtime·callbackasm1(SB) - MOVW $54, R12 - B runtime·callbackasm1(SB) - MOVW $55, R12 - B runtime·callbackasm1(SB) - MOVW $56, R12 - B runtime·callbackasm1(SB) - MOVW $57, R12 - B runtime·callbackasm1(SB) - MOVW $58, R12 - B runtime·callbackasm1(SB) - MOVW $59, R12 - B runtime·callbackasm1(SB) - MOVW $60, R12 - B runtime·callbackasm1(SB) - MOVW $61, R12 - B runtime·callbackasm1(SB) - MOVW $62, R12 - B runtime·callbackasm1(SB) - MOVW $63, R12 - B runtime·callbackasm1(SB) - MOVW $64, R12 - B runtime·callbackasm1(SB) - MOVW $65, R12 - B runtime·callbackasm1(SB) - MOVW $66, R12 - B runtime·callbackasm1(SB) - MOVW $67, R12 - B runtime·callbackasm1(SB) - MOVW $68, R12 - B runtime·callbackasm1(SB) - MOVW $69, R12 - B runtime·callbackasm1(SB) - MOVW $70, R12 - B runtime·callbackasm1(SB) - MOVW $71, R12 - B runtime·callbackasm1(SB) - MOVW $72, R12 - B runtime·callbackasm1(SB) - MOVW $73, R12 - B runtime·callbackasm1(SB) - MOVW $74, R12 - B runtime·callbackasm1(SB) - MOVW $75, R12 - B runtime·callbackasm1(SB) - MOVW $76, R12 - B runtime·callbackasm1(SB) - MOVW $77, R12 - B runtime·callbackasm1(SB) - MOVW $78, R12 - B runtime·callbackasm1(SB) - MOVW $79, R12 - B runtime·callbackasm1(SB) - MOVW $80, R12 - B runtime·callbackasm1(SB) - MOVW $81, R12 - B runtime·callbackasm1(SB) - MOVW $82, R12 - B runtime·callbackasm1(SB) - MOVW $83, R12 - B runtime·callbackasm1(SB) - MOVW $84, R12 - B runtime·callbackasm1(SB) - MOVW $85, R12 - B runtime·callbackasm1(SB) - MOVW $86, R12 - B runtime·callbackasm1(SB) - MOVW $87, R12 - B runtime·callbackasm1(SB) - MOVW $88, R12 - B runtime·callbackasm1(SB) - MOVW $89, R12 - B runtime·callbackasm1(SB) - MOVW $90, R12 - B runtime·callbackasm1(SB) - MOVW $91, R12 - B runtime·callbackasm1(SB) - MOVW $92, R12 - B runtime·callbackasm1(SB) - MOVW $93, R12 - B runtime·callbackasm1(SB) - MOVW $94, R12 - B runtime·callbackasm1(SB) - MOVW $95, R12 - B runtime·callbackasm1(SB) - MOVW $96, R12 - B runtime·callbackasm1(SB) - MOVW $97, R12 - B runtime·callbackasm1(SB) - MOVW $98, R12 - B runtime·callbackasm1(SB) - MOVW $99, R12 - B runtime·callbackasm1(SB) - MOVW $100, R12 - B runtime·callbackasm1(SB) - MOVW $101, R12 - B runtime·callbackasm1(SB) - MOVW $102, R12 - B runtime·callbackasm1(SB) - MOVW $103, R12 - B runtime·callbackasm1(SB) - MOVW $104, R12 - B runtime·callbackasm1(SB) - MOVW $105, R12 - B runtime·callbackasm1(SB) - MOVW $106, R12 - B runtime·callbackasm1(SB) - MOVW $107, R12 - B runtime·callbackasm1(SB) - MOVW $108, R12 - B runtime·callbackasm1(SB) - MOVW $109, R12 - B runtime·callbackasm1(SB) - MOVW $110, R12 - B runtime·callbackasm1(SB) - MOVW $111, R12 - B runtime·callbackasm1(SB) - MOVW $112, R12 - B runtime·callbackasm1(SB) - MOVW $113, R12 - B runtime·callbackasm1(SB) - MOVW $114, R12 - B runtime·callbackasm1(SB) - MOVW $115, R12 - B runtime·callbackasm1(SB) - MOVW $116, R12 - B runtime·callbackasm1(SB) - MOVW $117, R12 - B runtime·callbackasm1(SB) - MOVW $118, R12 - B runtime·callbackasm1(SB) - MOVW $119, R12 - B runtime·callbackasm1(SB) - MOVW $120, R12 - B runtime·callbackasm1(SB) - MOVW $121, R12 - B runtime·callbackasm1(SB) - MOVW $122, R12 - B runtime·callbackasm1(SB) - MOVW $123, R12 - B runtime·callbackasm1(SB) - MOVW $124, R12 - B runtime·callbackasm1(SB) - MOVW $125, R12 - B runtime·callbackasm1(SB) - MOVW $126, R12 - B runtime·callbackasm1(SB) - MOVW $127, R12 - B runtime·callbackasm1(SB) - MOVW $128, R12 - B runtime·callbackasm1(SB) - MOVW $129, R12 - B runtime·callbackasm1(SB) - MOVW $130, R12 - B runtime·callbackasm1(SB) - MOVW $131, R12 - B runtime·callbackasm1(SB) - MOVW $132, R12 - B runtime·callbackasm1(SB) - MOVW $133, R12 - B runtime·callbackasm1(SB) - MOVW $134, R12 - B runtime·callbackasm1(SB) - MOVW $135, R12 - B runtime·callbackasm1(SB) - MOVW $136, R12 - B runtime·callbackasm1(SB) - MOVW $137, R12 - B runtime·callbackasm1(SB) - MOVW $138, R12 - B runtime·callbackasm1(SB) - MOVW $139, R12 - B runtime·callbackasm1(SB) - MOVW $140, R12 - B runtime·callbackasm1(SB) - MOVW $141, R12 - B runtime·callbackasm1(SB) - MOVW $142, R12 - B runtime·callbackasm1(SB) - MOVW $143, R12 - B runtime·callbackasm1(SB) - MOVW $144, R12 - B runtime·callbackasm1(SB) - MOVW $145, R12 - B runtime·callbackasm1(SB) - MOVW $146, R12 - B runtime·callbackasm1(SB) - MOVW $147, R12 - B runtime·callbackasm1(SB) - MOVW $148, R12 - B runtime·callbackasm1(SB) - MOVW $149, R12 - B runtime·callbackasm1(SB) - MOVW $150, R12 - B runtime·callbackasm1(SB) - MOVW $151, R12 - B runtime·callbackasm1(SB) - MOVW $152, R12 - B runtime·callbackasm1(SB) - MOVW $153, R12 - B runtime·callbackasm1(SB) - MOVW $154, R12 - B runtime·callbackasm1(SB) - MOVW $155, R12 - B runtime·callbackasm1(SB) - MOVW $156, R12 - B runtime·callbackasm1(SB) - MOVW $157, R12 - B runtime·callbackasm1(SB) - MOVW $158, R12 - B runtime·callbackasm1(SB) - MOVW $159, R12 - B runtime·callbackasm1(SB) - MOVW $160, R12 - B runtime·callbackasm1(SB) - MOVW $161, R12 - B runtime·callbackasm1(SB) - MOVW $162, R12 - B runtime·callbackasm1(SB) - MOVW $163, R12 - B runtime·callbackasm1(SB) - MOVW $164, R12 - B runtime·callbackasm1(SB) - MOVW $165, R12 - B runtime·callbackasm1(SB) - MOVW $166, R12 - B runtime·callbackasm1(SB) - MOVW $167, R12 - B runtime·callbackasm1(SB) - MOVW $168, R12 - B runtime·callbackasm1(SB) - MOVW $169, R12 - B runtime·callbackasm1(SB) - MOVW $170, R12 - B runtime·callbackasm1(SB) - MOVW $171, R12 - B runtime·callbackasm1(SB) - MOVW $172, R12 - B runtime·callbackasm1(SB) - MOVW $173, R12 - B runtime·callbackasm1(SB) - MOVW $174, R12 - B runtime·callbackasm1(SB) - MOVW $175, R12 - B runtime·callbackasm1(SB) - MOVW $176, R12 - B runtime·callbackasm1(SB) - MOVW $177, R12 - B runtime·callbackasm1(SB) - MOVW $178, R12 - B runtime·callbackasm1(SB) - MOVW $179, R12 - B runtime·callbackasm1(SB) - MOVW $180, R12 - B runtime·callbackasm1(SB) - MOVW $181, R12 - B runtime·callbackasm1(SB) - MOVW $182, R12 - B runtime·callbackasm1(SB) - MOVW $183, R12 - B runtime·callbackasm1(SB) - MOVW $184, R12 - B runtime·callbackasm1(SB) - MOVW $185, R12 - B runtime·callbackasm1(SB) - MOVW $186, R12 - B runtime·callbackasm1(SB) - MOVW $187, R12 - B runtime·callbackasm1(SB) - MOVW $188, R12 - B runtime·callbackasm1(SB) - MOVW $189, R12 - B runtime·callbackasm1(SB) - MOVW $190, R12 - B runtime·callbackasm1(SB) - MOVW $191, R12 - B runtime·callbackasm1(SB) - MOVW $192, R12 - B runtime·callbackasm1(SB) - MOVW $193, R12 - B runtime·callbackasm1(SB) - MOVW $194, R12 - B runtime·callbackasm1(SB) - MOVW $195, R12 - B runtime·callbackasm1(SB) - MOVW $196, R12 - B runtime·callbackasm1(SB) - MOVW $197, R12 - B runtime·callbackasm1(SB) - MOVW $198, R12 - B runtime·callbackasm1(SB) - MOVW $199, R12 - B runtime·callbackasm1(SB) - MOVW $200, R12 - B runtime·callbackasm1(SB) - MOVW $201, R12 - B runtime·callbackasm1(SB) - MOVW $202, R12 - B runtime·callbackasm1(SB) - MOVW $203, R12 - B runtime·callbackasm1(SB) - MOVW $204, R12 - B runtime·callbackasm1(SB) - MOVW $205, R12 - B runtime·callbackasm1(SB) - MOVW $206, R12 - B runtime·callbackasm1(SB) - MOVW $207, R12 - B runtime·callbackasm1(SB) - MOVW $208, R12 - B runtime·callbackasm1(SB) - MOVW $209, R12 - B runtime·callbackasm1(SB) - MOVW $210, R12 - B runtime·callbackasm1(SB) - MOVW $211, R12 - B runtime·callbackasm1(SB) - MOVW $212, R12 - B runtime·callbackasm1(SB) - MOVW $213, R12 - B runtime·callbackasm1(SB) - MOVW $214, R12 - B runtime·callbackasm1(SB) - MOVW $215, R12 - B runtime·callbackasm1(SB) - MOVW $216, R12 - B runtime·callbackasm1(SB) - MOVW $217, R12 - B runtime·callbackasm1(SB) - MOVW $218, R12 - B runtime·callbackasm1(SB) - MOVW $219, R12 - B runtime·callbackasm1(SB) - MOVW $220, R12 - B runtime·callbackasm1(SB) - MOVW $221, R12 - B runtime·callbackasm1(SB) - MOVW $222, R12 - B runtime·callbackasm1(SB) - MOVW $223, R12 - B runtime·callbackasm1(SB) - MOVW $224, R12 - B runtime·callbackasm1(SB) - MOVW $225, R12 - B runtime·callbackasm1(SB) - MOVW $226, R12 - B runtime·callbackasm1(SB) - MOVW $227, R12 - B runtime·callbackasm1(SB) - MOVW $228, R12 - B runtime·callbackasm1(SB) - MOVW $229, R12 - B runtime·callbackasm1(SB) - MOVW $230, R12 - B runtime·callbackasm1(SB) - MOVW $231, R12 - B runtime·callbackasm1(SB) - MOVW $232, R12 - B runtime·callbackasm1(SB) - MOVW $233, R12 - B runtime·callbackasm1(SB) - MOVW $234, R12 - B runtime·callbackasm1(SB) - MOVW $235, R12 - B runtime·callbackasm1(SB) - MOVW $236, R12 - B runtime·callbackasm1(SB) - MOVW $237, R12 - B runtime·callbackasm1(SB) - MOVW $238, R12 - B runtime·callbackasm1(SB) - MOVW $239, R12 - B runtime·callbackasm1(SB) - MOVW $240, R12 - B runtime·callbackasm1(SB) - MOVW $241, R12 - B runtime·callbackasm1(SB) - MOVW $242, R12 - B runtime·callbackasm1(SB) - MOVW $243, R12 - B runtime·callbackasm1(SB) - MOVW $244, R12 - B runtime·callbackasm1(SB) - MOVW $245, R12 - B runtime·callbackasm1(SB) - MOVW $246, R12 - B runtime·callbackasm1(SB) - MOVW $247, R12 - B runtime·callbackasm1(SB) - MOVW $248, R12 - B runtime·callbackasm1(SB) - MOVW $249, R12 - B runtime·callbackasm1(SB) - MOVW $250, R12 - B runtime·callbackasm1(SB) - MOVW $251, R12 - B runtime·callbackasm1(SB) - MOVW $252, R12 - B runtime·callbackasm1(SB) - MOVW $253, R12 - B runtime·callbackasm1(SB) - MOVW $254, R12 - B runtime·callbackasm1(SB) - MOVW $255, R12 - B runtime·callbackasm1(SB) - MOVW $256, R12 - B runtime·callbackasm1(SB) - MOVW $257, R12 - B runtime·callbackasm1(SB) - MOVW $258, R12 - B runtime·callbackasm1(SB) - MOVW $259, R12 - B runtime·callbackasm1(SB) - MOVW $260, R12 - B runtime·callbackasm1(SB) - MOVW $261, R12 - B runtime·callbackasm1(SB) - MOVW $262, R12 - B runtime·callbackasm1(SB) - MOVW $263, R12 - B runtime·callbackasm1(SB) - MOVW $264, R12 - B runtime·callbackasm1(SB) - MOVW $265, R12 - B runtime·callbackasm1(SB) - MOVW $266, R12 - B runtime·callbackasm1(SB) - MOVW $267, R12 - B runtime·callbackasm1(SB) - MOVW $268, R12 - B runtime·callbackasm1(SB) - MOVW $269, R12 - B runtime·callbackasm1(SB) - MOVW $270, R12 - B runtime·callbackasm1(SB) - MOVW $271, R12 - B runtime·callbackasm1(SB) - MOVW $272, R12 - B runtime·callbackasm1(SB) - MOVW $273, R12 - B runtime·callbackasm1(SB) - MOVW $274, R12 - B runtime·callbackasm1(SB) - MOVW $275, R12 - B runtime·callbackasm1(SB) - MOVW $276, R12 - B runtime·callbackasm1(SB) - MOVW $277, R12 - B runtime·callbackasm1(SB) - MOVW $278, R12 - B runtime·callbackasm1(SB) - MOVW $279, R12 - B runtime·callbackasm1(SB) - MOVW $280, R12 - B runtime·callbackasm1(SB) - MOVW $281, R12 - B runtime·callbackasm1(SB) - MOVW $282, R12 - B runtime·callbackasm1(SB) - MOVW $283, R12 - B runtime·callbackasm1(SB) - MOVW $284, R12 - B runtime·callbackasm1(SB) - MOVW $285, R12 - B runtime·callbackasm1(SB) - MOVW $286, R12 - B runtime·callbackasm1(SB) - MOVW $287, R12 - B runtime·callbackasm1(SB) - MOVW $288, R12 - B runtime·callbackasm1(SB) - MOVW $289, R12 - B runtime·callbackasm1(SB) - MOVW $290, R12 - B runtime·callbackasm1(SB) - MOVW $291, R12 - B runtime·callbackasm1(SB) - MOVW $292, R12 - B runtime·callbackasm1(SB) - MOVW $293, R12 - B runtime·callbackasm1(SB) - MOVW $294, R12 - B runtime·callbackasm1(SB) - MOVW $295, R12 - B runtime·callbackasm1(SB) - MOVW $296, R12 - B runtime·callbackasm1(SB) - MOVW $297, R12 - B runtime·callbackasm1(SB) - MOVW $298, R12 - B runtime·callbackasm1(SB) - MOVW $299, R12 - B runtime·callbackasm1(SB) - MOVW $300, R12 - B runtime·callbackasm1(SB) - MOVW $301, R12 - B runtime·callbackasm1(SB) - MOVW $302, R12 - B runtime·callbackasm1(SB) - MOVW $303, R12 - B runtime·callbackasm1(SB) - MOVW $304, R12 - B runtime·callbackasm1(SB) - MOVW $305, R12 - B runtime·callbackasm1(SB) - MOVW $306, R12 - B runtime·callbackasm1(SB) - MOVW $307, R12 - B runtime·callbackasm1(SB) - MOVW $308, R12 - B runtime·callbackasm1(SB) - MOVW $309, R12 - B runtime·callbackasm1(SB) - MOVW $310, R12 - B runtime·callbackasm1(SB) - MOVW $311, R12 - B runtime·callbackasm1(SB) - MOVW $312, R12 - B runtime·callbackasm1(SB) - MOVW $313, R12 - B runtime·callbackasm1(SB) - MOVW $314, R12 - B runtime·callbackasm1(SB) - MOVW $315, R12 - B runtime·callbackasm1(SB) - MOVW $316, R12 - B runtime·callbackasm1(SB) - MOVW $317, R12 - B runtime·callbackasm1(SB) - MOVW $318, R12 - B runtime·callbackasm1(SB) - MOVW $319, R12 - B runtime·callbackasm1(SB) - MOVW $320, R12 - B runtime·callbackasm1(SB) - MOVW $321, R12 - B runtime·callbackasm1(SB) - MOVW $322, R12 - B runtime·callbackasm1(SB) - MOVW $323, R12 - B runtime·callbackasm1(SB) - MOVW $324, R12 - B runtime·callbackasm1(SB) - MOVW $325, R12 - B runtime·callbackasm1(SB) - MOVW $326, R12 - B runtime·callbackasm1(SB) - MOVW $327, R12 - B runtime·callbackasm1(SB) - MOVW $328, R12 - B runtime·callbackasm1(SB) - MOVW $329, R12 - B runtime·callbackasm1(SB) - MOVW $330, R12 - B runtime·callbackasm1(SB) - MOVW $331, R12 - B runtime·callbackasm1(SB) - MOVW $332, R12 - B runtime·callbackasm1(SB) - MOVW $333, R12 - B runtime·callbackasm1(SB) - MOVW $334, R12 - B runtime·callbackasm1(SB) - MOVW $335, R12 - B runtime·callbackasm1(SB) - MOVW $336, R12 - B runtime·callbackasm1(SB) - MOVW $337, R12 - B runtime·callbackasm1(SB) - MOVW $338, R12 - B runtime·callbackasm1(SB) - MOVW $339, R12 - B runtime·callbackasm1(SB) - MOVW $340, R12 - B runtime·callbackasm1(SB) - MOVW $341, R12 - B runtime·callbackasm1(SB) - MOVW $342, R12 - B runtime·callbackasm1(SB) - MOVW $343, R12 - B runtime·callbackasm1(SB) - MOVW $344, R12 - B runtime·callbackasm1(SB) - MOVW $345, R12 - B runtime·callbackasm1(SB) - MOVW $346, R12 - B runtime·callbackasm1(SB) - MOVW $347, R12 - B runtime·callbackasm1(SB) - MOVW $348, R12 - B runtime·callbackasm1(SB) - MOVW $349, R12 - B runtime·callbackasm1(SB) - MOVW $350, R12 - B runtime·callbackasm1(SB) - MOVW $351, R12 - B runtime·callbackasm1(SB) - MOVW $352, R12 - B runtime·callbackasm1(SB) - MOVW $353, R12 - B runtime·callbackasm1(SB) - MOVW $354, R12 - B runtime·callbackasm1(SB) - MOVW $355, R12 - B runtime·callbackasm1(SB) - MOVW $356, R12 - B runtime·callbackasm1(SB) - MOVW $357, R12 - B runtime·callbackasm1(SB) - MOVW $358, R12 - B runtime·callbackasm1(SB) - MOVW $359, R12 - B runtime·callbackasm1(SB) - MOVW $360, R12 - B runtime·callbackasm1(SB) - MOVW $361, R12 - B runtime·callbackasm1(SB) - MOVW $362, R12 - B runtime·callbackasm1(SB) - MOVW $363, R12 - B runtime·callbackasm1(SB) - MOVW $364, R12 - B runtime·callbackasm1(SB) - MOVW $365, R12 - B runtime·callbackasm1(SB) - MOVW $366, R12 - B runtime·callbackasm1(SB) - MOVW $367, R12 - B runtime·callbackasm1(SB) - MOVW $368, R12 - B runtime·callbackasm1(SB) - MOVW $369, R12 - B runtime·callbackasm1(SB) - MOVW $370, R12 - B runtime·callbackasm1(SB) - MOVW $371, R12 - B runtime·callbackasm1(SB) - MOVW $372, R12 - B runtime·callbackasm1(SB) - MOVW $373, R12 - B runtime·callbackasm1(SB) - MOVW $374, R12 - B runtime·callbackasm1(SB) - MOVW $375, R12 - B runtime·callbackasm1(SB) - MOVW $376, R12 - B runtime·callbackasm1(SB) - MOVW $377, R12 - B runtime·callbackasm1(SB) - MOVW $378, R12 - B runtime·callbackasm1(SB) - MOVW $379, R12 - B runtime·callbackasm1(SB) - MOVW $380, R12 - B runtime·callbackasm1(SB) - MOVW $381, R12 - B runtime·callbackasm1(SB) - MOVW $382, R12 - B runtime·callbackasm1(SB) - MOVW $383, R12 - B runtime·callbackasm1(SB) - MOVW $384, R12 - B runtime·callbackasm1(SB) - MOVW $385, R12 - B runtime·callbackasm1(SB) - MOVW $386, R12 - B runtime·callbackasm1(SB) - MOVW $387, R12 - B runtime·callbackasm1(SB) - MOVW $388, R12 - B runtime·callbackasm1(SB) - MOVW $389, R12 - B runtime·callbackasm1(SB) - MOVW $390, R12 - B runtime·callbackasm1(SB) - MOVW $391, R12 - B runtime·callbackasm1(SB) - MOVW $392, R12 - B runtime·callbackasm1(SB) - MOVW $393, R12 - B runtime·callbackasm1(SB) - MOVW $394, R12 - B runtime·callbackasm1(SB) - MOVW $395, R12 - B runtime·callbackasm1(SB) - MOVW $396, R12 - B runtime·callbackasm1(SB) - MOVW $397, R12 - B runtime·callbackasm1(SB) - MOVW $398, R12 - B runtime·callbackasm1(SB) - MOVW $399, R12 - B runtime·callbackasm1(SB) - MOVW $400, R12 - B runtime·callbackasm1(SB) - MOVW $401, R12 - B runtime·callbackasm1(SB) - MOVW $402, R12 - B runtime·callbackasm1(SB) - MOVW $403, R12 - B runtime·callbackasm1(SB) - MOVW $404, R12 - B runtime·callbackasm1(SB) - MOVW $405, R12 - B runtime·callbackasm1(SB) - MOVW $406, R12 - B runtime·callbackasm1(SB) - MOVW $407, R12 - B runtime·callbackasm1(SB) - MOVW $408, R12 - B runtime·callbackasm1(SB) - MOVW $409, R12 - B runtime·callbackasm1(SB) - MOVW $410, R12 - B runtime·callbackasm1(SB) - MOVW $411, R12 - B runtime·callbackasm1(SB) - MOVW $412, R12 - B runtime·callbackasm1(SB) - MOVW $413, R12 - B runtime·callbackasm1(SB) - MOVW $414, R12 - B runtime·callbackasm1(SB) - MOVW $415, R12 - B runtime·callbackasm1(SB) - MOVW $416, R12 - B runtime·callbackasm1(SB) - MOVW $417, R12 - B runtime·callbackasm1(SB) - MOVW $418, R12 - B runtime·callbackasm1(SB) - MOVW $419, R12 - B runtime·callbackasm1(SB) - MOVW $420, R12 - B runtime·callbackasm1(SB) - MOVW $421, R12 - B runtime·callbackasm1(SB) - MOVW $422, R12 - B runtime·callbackasm1(SB) - MOVW $423, R12 - B runtime·callbackasm1(SB) - MOVW $424, R12 - B runtime·callbackasm1(SB) - MOVW $425, R12 - B runtime·callbackasm1(SB) - MOVW $426, R12 - B runtime·callbackasm1(SB) - MOVW $427, R12 - B runtime·callbackasm1(SB) - MOVW $428, R12 - B runtime·callbackasm1(SB) - MOVW $429, R12 - B runtime·callbackasm1(SB) - MOVW $430, R12 - B runtime·callbackasm1(SB) - MOVW $431, R12 - B runtime·callbackasm1(SB) - MOVW $432, R12 - B runtime·callbackasm1(SB) - MOVW $433, R12 - B runtime·callbackasm1(SB) - MOVW $434, R12 - B runtime·callbackasm1(SB) - MOVW $435, R12 - B runtime·callbackasm1(SB) - MOVW $436, R12 - B runtime·callbackasm1(SB) - MOVW $437, R12 - B runtime·callbackasm1(SB) - MOVW $438, R12 - B runtime·callbackasm1(SB) - MOVW $439, R12 - B runtime·callbackasm1(SB) - MOVW $440, R12 - B runtime·callbackasm1(SB) - MOVW $441, R12 - B runtime·callbackasm1(SB) - MOVW $442, R12 - B runtime·callbackasm1(SB) - MOVW $443, R12 - B runtime·callbackasm1(SB) - MOVW $444, R12 - B runtime·callbackasm1(SB) - MOVW $445, R12 - B runtime·callbackasm1(SB) - MOVW $446, R12 - B runtime·callbackasm1(SB) - MOVW $447, R12 - B runtime·callbackasm1(SB) - MOVW $448, R12 - B runtime·callbackasm1(SB) - MOVW $449, R12 - B runtime·callbackasm1(SB) - MOVW $450, R12 - B runtime·callbackasm1(SB) - MOVW $451, R12 - B runtime·callbackasm1(SB) - MOVW $452, R12 - B runtime·callbackasm1(SB) - MOVW $453, R12 - B runtime·callbackasm1(SB) - MOVW $454, R12 - B runtime·callbackasm1(SB) - MOVW $455, R12 - B runtime·callbackasm1(SB) - MOVW $456, R12 - B runtime·callbackasm1(SB) - MOVW $457, R12 - B runtime·callbackasm1(SB) - MOVW $458, R12 - B runtime·callbackasm1(SB) - MOVW $459, R12 - B runtime·callbackasm1(SB) - MOVW $460, R12 - B runtime·callbackasm1(SB) - MOVW $461, R12 - B runtime·callbackasm1(SB) - MOVW $462, R12 - B runtime·callbackasm1(SB) - MOVW $463, R12 - B runtime·callbackasm1(SB) - MOVW $464, R12 - B runtime·callbackasm1(SB) - MOVW $465, R12 - B runtime·callbackasm1(SB) - MOVW $466, R12 - B runtime·callbackasm1(SB) - MOVW $467, R12 - B runtime·callbackasm1(SB) - MOVW $468, R12 - B runtime·callbackasm1(SB) - MOVW $469, R12 - B runtime·callbackasm1(SB) - MOVW $470, R12 - B runtime·callbackasm1(SB) - MOVW $471, R12 - B runtime·callbackasm1(SB) - MOVW $472, R12 - B runtime·callbackasm1(SB) - MOVW $473, R12 - B runtime·callbackasm1(SB) - MOVW $474, R12 - B runtime·callbackasm1(SB) - MOVW $475, R12 - B runtime·callbackasm1(SB) - MOVW $476, R12 - B runtime·callbackasm1(SB) - MOVW $477, R12 - B runtime·callbackasm1(SB) - MOVW $478, R12 - B runtime·callbackasm1(SB) - MOVW $479, R12 - B runtime·callbackasm1(SB) - MOVW $480, R12 - B runtime·callbackasm1(SB) - MOVW $481, R12 - B runtime·callbackasm1(SB) - MOVW $482, R12 - B runtime·callbackasm1(SB) - MOVW $483, R12 - B runtime·callbackasm1(SB) - MOVW $484, R12 - B runtime·callbackasm1(SB) - MOVW $485, R12 - B runtime·callbackasm1(SB) - MOVW $486, R12 - B runtime·callbackasm1(SB) - MOVW $487, R12 - B runtime·callbackasm1(SB) - MOVW $488, R12 - B runtime·callbackasm1(SB) - MOVW $489, R12 - B runtime·callbackasm1(SB) - MOVW $490, R12 - B runtime·callbackasm1(SB) - MOVW $491, R12 - B runtime·callbackasm1(SB) - MOVW $492, R12 - B runtime·callbackasm1(SB) - MOVW $493, R12 - B runtime·callbackasm1(SB) - MOVW $494, R12 - B runtime·callbackasm1(SB) - MOVW $495, R12 - B runtime·callbackasm1(SB) - MOVW $496, R12 - B runtime·callbackasm1(SB) - MOVW $497, R12 - B runtime·callbackasm1(SB) - MOVW $498, R12 - B runtime·callbackasm1(SB) - MOVW $499, R12 - B runtime·callbackasm1(SB) - MOVW $500, R12 - B runtime·callbackasm1(SB) - MOVW $501, R12 - B runtime·callbackasm1(SB) - MOVW $502, R12 - B runtime·callbackasm1(SB) - MOVW $503, R12 - B runtime·callbackasm1(SB) - MOVW $504, R12 - B runtime·callbackasm1(SB) - MOVW $505, R12 - B runtime·callbackasm1(SB) - MOVW $506, R12 - B runtime·callbackasm1(SB) - MOVW $507, R12 - B runtime·callbackasm1(SB) - MOVW $508, R12 - B runtime·callbackasm1(SB) - MOVW $509, R12 - B runtime·callbackasm1(SB) - MOVW $510, R12 - B runtime·callbackasm1(SB) - MOVW $511, R12 - B runtime·callbackasm1(SB) - MOVW $512, R12 - B runtime·callbackasm1(SB) - MOVW $513, R12 - B runtime·callbackasm1(SB) - MOVW $514, R12 - B runtime·callbackasm1(SB) - MOVW $515, R12 - B runtime·callbackasm1(SB) - MOVW $516, R12 - B runtime·callbackasm1(SB) - MOVW $517, R12 - B runtime·callbackasm1(SB) - MOVW $518, R12 - B runtime·callbackasm1(SB) - MOVW $519, R12 - B runtime·callbackasm1(SB) - MOVW $520, R12 - B runtime·callbackasm1(SB) - MOVW $521, R12 - B runtime·callbackasm1(SB) - MOVW $522, R12 - B runtime·callbackasm1(SB) - MOVW $523, R12 - B runtime·callbackasm1(SB) - MOVW $524, R12 - B runtime·callbackasm1(SB) - MOVW $525, R12 - B runtime·callbackasm1(SB) - MOVW $526, R12 - B runtime·callbackasm1(SB) - MOVW $527, R12 - B runtime·callbackasm1(SB) - MOVW $528, R12 - B runtime·callbackasm1(SB) - MOVW $529, R12 - B runtime·callbackasm1(SB) - MOVW $530, R12 - B runtime·callbackasm1(SB) - MOVW $531, R12 - B runtime·callbackasm1(SB) - MOVW $532, R12 - B runtime·callbackasm1(SB) - MOVW $533, R12 - B runtime·callbackasm1(SB) - MOVW $534, R12 - B runtime·callbackasm1(SB) - MOVW $535, R12 - B runtime·callbackasm1(SB) - MOVW $536, R12 - B runtime·callbackasm1(SB) - MOVW $537, R12 - B runtime·callbackasm1(SB) - MOVW $538, R12 - B runtime·callbackasm1(SB) - MOVW $539, R12 - B runtime·callbackasm1(SB) - MOVW $540, R12 - B runtime·callbackasm1(SB) - MOVW $541, R12 - B runtime·callbackasm1(SB) - MOVW $542, R12 - B runtime·callbackasm1(SB) - MOVW $543, R12 - B runtime·callbackasm1(SB) - MOVW $544, R12 - B runtime·callbackasm1(SB) - MOVW $545, R12 - B runtime·callbackasm1(SB) - MOVW $546, R12 - B runtime·callbackasm1(SB) - MOVW $547, R12 - B runtime·callbackasm1(SB) - MOVW $548, R12 - B runtime·callbackasm1(SB) - MOVW $549, R12 - B runtime·callbackasm1(SB) - MOVW $550, R12 - B runtime·callbackasm1(SB) - MOVW $551, R12 - B runtime·callbackasm1(SB) - MOVW $552, R12 - B runtime·callbackasm1(SB) - MOVW $553, R12 - B runtime·callbackasm1(SB) - MOVW $554, R12 - B runtime·callbackasm1(SB) - MOVW $555, R12 - B runtime·callbackasm1(SB) - MOVW $556, R12 - B runtime·callbackasm1(SB) - MOVW $557, R12 - B runtime·callbackasm1(SB) - MOVW $558, R12 - B runtime·callbackasm1(SB) - MOVW $559, R12 - B runtime·callbackasm1(SB) - MOVW $560, R12 - B runtime·callbackasm1(SB) - MOVW $561, R12 - B runtime·callbackasm1(SB) - MOVW $562, R12 - B runtime·callbackasm1(SB) - MOVW $563, R12 - B runtime·callbackasm1(SB) - MOVW $564, R12 - B runtime·callbackasm1(SB) - MOVW $565, R12 - B runtime·callbackasm1(SB) - MOVW $566, R12 - B runtime·callbackasm1(SB) - MOVW $567, R12 - B runtime·callbackasm1(SB) - MOVW $568, R12 - B runtime·callbackasm1(SB) - MOVW $569, R12 - B runtime·callbackasm1(SB) - MOVW $570, R12 - B runtime·callbackasm1(SB) - MOVW $571, R12 - B runtime·callbackasm1(SB) - MOVW $572, R12 - B runtime·callbackasm1(SB) - MOVW $573, R12 - B runtime·callbackasm1(SB) - MOVW $574, R12 - B runtime·callbackasm1(SB) - MOVW $575, R12 - B runtime·callbackasm1(SB) - MOVW $576, R12 - B runtime·callbackasm1(SB) - MOVW $577, R12 - B runtime·callbackasm1(SB) - MOVW $578, R12 - B runtime·callbackasm1(SB) - MOVW $579, R12 - B runtime·callbackasm1(SB) - MOVW $580, R12 - B runtime·callbackasm1(SB) - MOVW $581, R12 - B runtime·callbackasm1(SB) - MOVW $582, R12 - B runtime·callbackasm1(SB) - MOVW $583, R12 - B runtime·callbackasm1(SB) - MOVW $584, R12 - B runtime·callbackasm1(SB) - MOVW $585, R12 - B runtime·callbackasm1(SB) - MOVW $586, R12 - B runtime·callbackasm1(SB) - MOVW $587, R12 - B runtime·callbackasm1(SB) - MOVW $588, R12 - B runtime·callbackasm1(SB) - MOVW $589, R12 - B runtime·callbackasm1(SB) - MOVW $590, R12 - B runtime·callbackasm1(SB) - MOVW $591, R12 - B runtime·callbackasm1(SB) - MOVW $592, R12 - B runtime·callbackasm1(SB) - MOVW $593, R12 - B runtime·callbackasm1(SB) - MOVW $594, R12 - B runtime·callbackasm1(SB) - MOVW $595, R12 - B runtime·callbackasm1(SB) - MOVW $596, R12 - B runtime·callbackasm1(SB) - MOVW $597, R12 - B runtime·callbackasm1(SB) - MOVW $598, R12 - B runtime·callbackasm1(SB) - MOVW $599, R12 - B runtime·callbackasm1(SB) - MOVW $600, R12 - B runtime·callbackasm1(SB) - MOVW $601, R12 - B runtime·callbackasm1(SB) - MOVW $602, R12 - B runtime·callbackasm1(SB) - MOVW $603, R12 - B runtime·callbackasm1(SB) - MOVW $604, R12 - B runtime·callbackasm1(SB) - MOVW $605, R12 - B runtime·callbackasm1(SB) - MOVW $606, R12 - B runtime·callbackasm1(SB) - MOVW $607, R12 - B runtime·callbackasm1(SB) - MOVW $608, R12 - B runtime·callbackasm1(SB) - MOVW $609, R12 - B runtime·callbackasm1(SB) - MOVW $610, R12 - B runtime·callbackasm1(SB) - MOVW $611, R12 - B runtime·callbackasm1(SB) - MOVW $612, R12 - B runtime·callbackasm1(SB) - MOVW $613, R12 - B runtime·callbackasm1(SB) - MOVW $614, R12 - B runtime·callbackasm1(SB) - MOVW $615, R12 - B runtime·callbackasm1(SB) - MOVW $616, R12 - B runtime·callbackasm1(SB) - MOVW $617, R12 - B runtime·callbackasm1(SB) - MOVW $618, R12 - B runtime·callbackasm1(SB) - MOVW $619, R12 - B runtime·callbackasm1(SB) - MOVW $620, R12 - B runtime·callbackasm1(SB) - MOVW $621, R12 - B runtime·callbackasm1(SB) - MOVW $622, R12 - B runtime·callbackasm1(SB) - MOVW $623, R12 - B runtime·callbackasm1(SB) - MOVW $624, R12 - B runtime·callbackasm1(SB) - MOVW $625, R12 - B runtime·callbackasm1(SB) - MOVW $626, R12 - B runtime·callbackasm1(SB) - MOVW $627, R12 - B runtime·callbackasm1(SB) - MOVW $628, R12 - B runtime·callbackasm1(SB) - MOVW $629, R12 - B runtime·callbackasm1(SB) - MOVW $630, R12 - B runtime·callbackasm1(SB) - MOVW $631, R12 - B runtime·callbackasm1(SB) - MOVW $632, R12 - B runtime·callbackasm1(SB) - MOVW $633, R12 - B runtime·callbackasm1(SB) - MOVW $634, R12 - B runtime·callbackasm1(SB) - MOVW $635, R12 - B runtime·callbackasm1(SB) - MOVW $636, R12 - B runtime·callbackasm1(SB) - MOVW $637, R12 - B runtime·callbackasm1(SB) - MOVW $638, R12 - B runtime·callbackasm1(SB) - MOVW $639, R12 - B runtime·callbackasm1(SB) - MOVW $640, R12 - B runtime·callbackasm1(SB) - MOVW $641, R12 - B runtime·callbackasm1(SB) - MOVW $642, R12 - B runtime·callbackasm1(SB) - MOVW $643, R12 - B runtime·callbackasm1(SB) - MOVW $644, R12 - B runtime·callbackasm1(SB) - MOVW $645, R12 - B runtime·callbackasm1(SB) - MOVW $646, R12 - B runtime·callbackasm1(SB) - MOVW $647, R12 - B runtime·callbackasm1(SB) - MOVW $648, R12 - B runtime·callbackasm1(SB) - MOVW $649, R12 - B runtime·callbackasm1(SB) - MOVW $650, R12 - B runtime·callbackasm1(SB) - MOVW $651, R12 - B runtime·callbackasm1(SB) - MOVW $652, R12 - B runtime·callbackasm1(SB) - MOVW $653, R12 - B runtime·callbackasm1(SB) - MOVW $654, R12 - B runtime·callbackasm1(SB) - MOVW $655, R12 - B runtime·callbackasm1(SB) - MOVW $656, R12 - B runtime·callbackasm1(SB) - MOVW $657, R12 - B runtime·callbackasm1(SB) - MOVW $658, R12 - B runtime·callbackasm1(SB) - MOVW $659, R12 - B runtime·callbackasm1(SB) - MOVW $660, R12 - B runtime·callbackasm1(SB) - MOVW $661, R12 - B runtime·callbackasm1(SB) - MOVW $662, R12 - B runtime·callbackasm1(SB) - MOVW $663, R12 - B runtime·callbackasm1(SB) - MOVW $664, R12 - B runtime·callbackasm1(SB) - MOVW $665, R12 - B runtime·callbackasm1(SB) - MOVW $666, R12 - B runtime·callbackasm1(SB) - MOVW $667, R12 - B runtime·callbackasm1(SB) - MOVW $668, R12 - B runtime·callbackasm1(SB) - MOVW $669, R12 - B runtime·callbackasm1(SB) - MOVW $670, R12 - B runtime·callbackasm1(SB) - MOVW $671, R12 - B runtime·callbackasm1(SB) - MOVW $672, R12 - B runtime·callbackasm1(SB) - MOVW $673, R12 - B runtime·callbackasm1(SB) - MOVW $674, R12 - B runtime·callbackasm1(SB) - MOVW $675, R12 - B runtime·callbackasm1(SB) - MOVW $676, R12 - B runtime·callbackasm1(SB) - MOVW $677, R12 - B runtime·callbackasm1(SB) - MOVW $678, R12 - B runtime·callbackasm1(SB) - MOVW $679, R12 - B runtime·callbackasm1(SB) - MOVW $680, R12 - B runtime·callbackasm1(SB) - MOVW $681, R12 - B runtime·callbackasm1(SB) - MOVW $682, R12 - B runtime·callbackasm1(SB) - MOVW $683, R12 - B runtime·callbackasm1(SB) - MOVW $684, R12 - B runtime·callbackasm1(SB) - MOVW $685, R12 - B runtime·callbackasm1(SB) - MOVW $686, R12 - B runtime·callbackasm1(SB) - MOVW $687, R12 - B runtime·callbackasm1(SB) - MOVW $688, R12 - B runtime·callbackasm1(SB) - MOVW $689, R12 - B runtime·callbackasm1(SB) - MOVW $690, R12 - B runtime·callbackasm1(SB) - MOVW $691, R12 - B runtime·callbackasm1(SB) - MOVW $692, R12 - B runtime·callbackasm1(SB) - MOVW $693, R12 - B runtime·callbackasm1(SB) - MOVW $694, R12 - B runtime·callbackasm1(SB) - MOVW $695, R12 - B runtime·callbackasm1(SB) - MOVW $696, R12 - B runtime·callbackasm1(SB) - MOVW $697, R12 - B runtime·callbackasm1(SB) - MOVW $698, R12 - B runtime·callbackasm1(SB) - MOVW $699, R12 - B runtime·callbackasm1(SB) - MOVW $700, R12 - B runtime·callbackasm1(SB) - MOVW $701, R12 - B runtime·callbackasm1(SB) - MOVW $702, R12 - B runtime·callbackasm1(SB) - MOVW $703, R12 - B runtime·callbackasm1(SB) - MOVW $704, R12 - B runtime·callbackasm1(SB) - MOVW $705, R12 - B runtime·callbackasm1(SB) - MOVW $706, R12 - B runtime·callbackasm1(SB) - MOVW $707, R12 - B runtime·callbackasm1(SB) - MOVW $708, R12 - B runtime·callbackasm1(SB) - MOVW $709, R12 - B runtime·callbackasm1(SB) - MOVW $710, R12 - B runtime·callbackasm1(SB) - MOVW $711, R12 - B runtime·callbackasm1(SB) - MOVW $712, R12 - B runtime·callbackasm1(SB) - MOVW $713, R12 - B runtime·callbackasm1(SB) - MOVW $714, R12 - B runtime·callbackasm1(SB) - MOVW $715, R12 - B runtime·callbackasm1(SB) - MOVW $716, R12 - B runtime·callbackasm1(SB) - MOVW $717, R12 - B runtime·callbackasm1(SB) - MOVW $718, R12 - B runtime·callbackasm1(SB) - MOVW $719, R12 - B runtime·callbackasm1(SB) - MOVW $720, R12 - B runtime·callbackasm1(SB) - MOVW $721, R12 - B runtime·callbackasm1(SB) - MOVW $722, R12 - B runtime·callbackasm1(SB) - MOVW $723, R12 - B runtime·callbackasm1(SB) - MOVW $724, R12 - B runtime·callbackasm1(SB) - MOVW $725, R12 - B runtime·callbackasm1(SB) - MOVW $726, R12 - B runtime·callbackasm1(SB) - MOVW $727, R12 - B runtime·callbackasm1(SB) - MOVW $728, R12 - B runtime·callbackasm1(SB) - MOVW $729, R12 - B runtime·callbackasm1(SB) - MOVW $730, R12 - B runtime·callbackasm1(SB) - MOVW $731, R12 - B runtime·callbackasm1(SB) - MOVW $732, R12 - B runtime·callbackasm1(SB) - MOVW $733, R12 - B runtime·callbackasm1(SB) - MOVW $734, R12 - B runtime·callbackasm1(SB) - MOVW $735, R12 - B runtime·callbackasm1(SB) - MOVW $736, R12 - B runtime·callbackasm1(SB) - MOVW $737, R12 - B runtime·callbackasm1(SB) - MOVW $738, R12 - B runtime·callbackasm1(SB) - MOVW $739, R12 - B runtime·callbackasm1(SB) - MOVW $740, R12 - B runtime·callbackasm1(SB) - MOVW $741, R12 - B runtime·callbackasm1(SB) - MOVW $742, R12 - B runtime·callbackasm1(SB) - MOVW $743, R12 - B runtime·callbackasm1(SB) - MOVW $744, R12 - B runtime·callbackasm1(SB) - MOVW $745, R12 - B runtime·callbackasm1(SB) - MOVW $746, R12 - B runtime·callbackasm1(SB) - MOVW $747, R12 - B runtime·callbackasm1(SB) - MOVW $748, R12 - B runtime·callbackasm1(SB) - MOVW $749, R12 - B runtime·callbackasm1(SB) - MOVW $750, R12 - B runtime·callbackasm1(SB) - MOVW $751, R12 - B runtime·callbackasm1(SB) - MOVW $752, R12 - B runtime·callbackasm1(SB) - MOVW $753, R12 - B runtime·callbackasm1(SB) - MOVW $754, R12 - B runtime·callbackasm1(SB) - MOVW $755, R12 - B runtime·callbackasm1(SB) - MOVW $756, R12 - B runtime·callbackasm1(SB) - MOVW $757, R12 - B runtime·callbackasm1(SB) - MOVW $758, R12 - B runtime·callbackasm1(SB) - MOVW $759, R12 - B runtime·callbackasm1(SB) - MOVW $760, R12 - B runtime·callbackasm1(SB) - MOVW $761, R12 - B runtime·callbackasm1(SB) - MOVW $762, R12 - B runtime·callbackasm1(SB) - MOVW $763, R12 - B runtime·callbackasm1(SB) - MOVW $764, R12 - B runtime·callbackasm1(SB) - MOVW $765, R12 - B runtime·callbackasm1(SB) - MOVW $766, R12 - B runtime·callbackasm1(SB) - MOVW $767, R12 - B runtime·callbackasm1(SB) - MOVW $768, R12 - B runtime·callbackasm1(SB) - MOVW $769, R12 - B runtime·callbackasm1(SB) - MOVW $770, R12 - B runtime·callbackasm1(SB) - MOVW $771, R12 - B runtime·callbackasm1(SB) - MOVW $772, R12 - B runtime·callbackasm1(SB) - MOVW $773, R12 - B runtime·callbackasm1(SB) - MOVW $774, R12 - B runtime·callbackasm1(SB) - MOVW $775, R12 - B runtime·callbackasm1(SB) - MOVW $776, R12 - B runtime·callbackasm1(SB) - MOVW $777, R12 - B runtime·callbackasm1(SB) - MOVW $778, R12 - B runtime·callbackasm1(SB) - MOVW $779, R12 - B runtime·callbackasm1(SB) - MOVW $780, R12 - B runtime·callbackasm1(SB) - MOVW $781, R12 - B runtime·callbackasm1(SB) - MOVW $782, R12 - B runtime·callbackasm1(SB) - MOVW $783, R12 - B runtime·callbackasm1(SB) - MOVW $784, R12 - B runtime·callbackasm1(SB) - MOVW $785, R12 - B runtime·callbackasm1(SB) - MOVW $786, R12 - B runtime·callbackasm1(SB) - MOVW $787, R12 - B runtime·callbackasm1(SB) - MOVW $788, R12 - B runtime·callbackasm1(SB) - MOVW $789, R12 - B runtime·callbackasm1(SB) - MOVW $790, R12 - B runtime·callbackasm1(SB) - MOVW $791, R12 - B runtime·callbackasm1(SB) - MOVW $792, R12 - B runtime·callbackasm1(SB) - MOVW $793, R12 - B runtime·callbackasm1(SB) - MOVW $794, R12 - B runtime·callbackasm1(SB) - MOVW $795, R12 - B runtime·callbackasm1(SB) - MOVW $796, R12 - B runtime·callbackasm1(SB) - MOVW $797, R12 - B runtime·callbackasm1(SB) - MOVW $798, R12 - B runtime·callbackasm1(SB) - MOVW $799, R12 - B runtime·callbackasm1(SB) - MOVW $800, R12 - B runtime·callbackasm1(SB) - MOVW $801, R12 - B runtime·callbackasm1(SB) - MOVW $802, R12 - B runtime·callbackasm1(SB) - MOVW $803, R12 - B runtime·callbackasm1(SB) - MOVW $804, R12 - B runtime·callbackasm1(SB) - MOVW $805, R12 - B runtime·callbackasm1(SB) - MOVW $806, R12 - B runtime·callbackasm1(SB) - MOVW $807, R12 - B runtime·callbackasm1(SB) - MOVW $808, R12 - B runtime·callbackasm1(SB) - MOVW $809, R12 - B runtime·callbackasm1(SB) - MOVW $810, R12 - B runtime·callbackasm1(SB) - MOVW $811, R12 - B runtime·callbackasm1(SB) - MOVW $812, R12 - B runtime·callbackasm1(SB) - MOVW $813, R12 - B runtime·callbackasm1(SB) - MOVW $814, R12 - B runtime·callbackasm1(SB) - MOVW $815, R12 - B runtime·callbackasm1(SB) - MOVW $816, R12 - B runtime·callbackasm1(SB) - MOVW $817, R12 - B runtime·callbackasm1(SB) - MOVW $818, R12 - B runtime·callbackasm1(SB) - MOVW $819, R12 - B runtime·callbackasm1(SB) - MOVW $820, R12 - B runtime·callbackasm1(SB) - MOVW $821, R12 - B runtime·callbackasm1(SB) - MOVW $822, R12 - B runtime·callbackasm1(SB) - MOVW $823, R12 - B runtime·callbackasm1(SB) - MOVW $824, R12 - B runtime·callbackasm1(SB) - MOVW $825, R12 - B runtime·callbackasm1(SB) - MOVW $826, R12 - B runtime·callbackasm1(SB) - MOVW $827, R12 - B runtime·callbackasm1(SB) - MOVW $828, R12 - B runtime·callbackasm1(SB) - MOVW $829, R12 - B runtime·callbackasm1(SB) - MOVW $830, R12 - B runtime·callbackasm1(SB) - MOVW $831, R12 - B runtime·callbackasm1(SB) - MOVW $832, R12 - B runtime·callbackasm1(SB) - MOVW $833, R12 - B runtime·callbackasm1(SB) - MOVW $834, R12 - B runtime·callbackasm1(SB) - MOVW $835, R12 - B runtime·callbackasm1(SB) - MOVW $836, R12 - B runtime·callbackasm1(SB) - MOVW $837, R12 - B runtime·callbackasm1(SB) - MOVW $838, R12 - B runtime·callbackasm1(SB) - MOVW $839, R12 - B runtime·callbackasm1(SB) - MOVW $840, R12 - B runtime·callbackasm1(SB) - MOVW $841, R12 - B runtime·callbackasm1(SB) - MOVW $842, R12 - B runtime·callbackasm1(SB) - MOVW $843, R12 - B runtime·callbackasm1(SB) - MOVW $844, R12 - B runtime·callbackasm1(SB) - MOVW $845, R12 - B runtime·callbackasm1(SB) - MOVW $846, R12 - B runtime·callbackasm1(SB) - MOVW $847, R12 - B runtime·callbackasm1(SB) - MOVW $848, R12 - B runtime·callbackasm1(SB) - MOVW $849, R12 - B runtime·callbackasm1(SB) - MOVW $850, R12 - B runtime·callbackasm1(SB) - MOVW $851, R12 - B runtime·callbackasm1(SB) - MOVW $852, R12 - B runtime·callbackasm1(SB) - MOVW $853, R12 - B runtime·callbackasm1(SB) - MOVW $854, R12 - B runtime·callbackasm1(SB) - MOVW $855, R12 - B runtime·callbackasm1(SB) - MOVW $856, R12 - B runtime·callbackasm1(SB) - MOVW $857, R12 - B runtime·callbackasm1(SB) - MOVW $858, R12 - B runtime·callbackasm1(SB) - MOVW $859, R12 - B runtime·callbackasm1(SB) - MOVW $860, R12 - B runtime·callbackasm1(SB) - MOVW $861, R12 - B runtime·callbackasm1(SB) - MOVW $862, R12 - B runtime·callbackasm1(SB) - MOVW $863, R12 - B runtime·callbackasm1(SB) - MOVW $864, R12 - B runtime·callbackasm1(SB) - MOVW $865, R12 - B runtime·callbackasm1(SB) - MOVW $866, R12 - B runtime·callbackasm1(SB) - MOVW $867, R12 - B runtime·callbackasm1(SB) - MOVW $868, R12 - B runtime·callbackasm1(SB) - MOVW $869, R12 - B runtime·callbackasm1(SB) - MOVW $870, R12 - B runtime·callbackasm1(SB) - MOVW $871, R12 - B runtime·callbackasm1(SB) - MOVW $872, R12 - B runtime·callbackasm1(SB) - MOVW $873, R12 - B runtime·callbackasm1(SB) - MOVW $874, R12 - B runtime·callbackasm1(SB) - MOVW $875, R12 - B runtime·callbackasm1(SB) - MOVW $876, R12 - B runtime·callbackasm1(SB) - MOVW $877, R12 - B runtime·callbackasm1(SB) - MOVW $878, R12 - B runtime·callbackasm1(SB) - MOVW $879, R12 - B runtime·callbackasm1(SB) - MOVW $880, R12 - B runtime·callbackasm1(SB) - MOVW $881, R12 - B runtime·callbackasm1(SB) - MOVW $882, R12 - B runtime·callbackasm1(SB) - MOVW $883, R12 - B runtime·callbackasm1(SB) - MOVW $884, R12 - B runtime·callbackasm1(SB) - MOVW $885, R12 - B runtime·callbackasm1(SB) - MOVW $886, R12 - B runtime·callbackasm1(SB) - MOVW $887, R12 - B runtime·callbackasm1(SB) - MOVW $888, R12 - B runtime·callbackasm1(SB) - MOVW $889, R12 - B runtime·callbackasm1(SB) - MOVW $890, R12 - B runtime·callbackasm1(SB) - MOVW $891, R12 - B runtime·callbackasm1(SB) - MOVW $892, R12 - B runtime·callbackasm1(SB) - MOVW $893, R12 - B runtime·callbackasm1(SB) - MOVW $894, R12 - B runtime·callbackasm1(SB) - MOVW $895, R12 - B runtime·callbackasm1(SB) - MOVW $896, R12 - B runtime·callbackasm1(SB) - MOVW $897, R12 - B runtime·callbackasm1(SB) - MOVW $898, R12 - B runtime·callbackasm1(SB) - MOVW $899, R12 - B runtime·callbackasm1(SB) - MOVW $900, R12 - B runtime·callbackasm1(SB) - MOVW $901, R12 - B runtime·callbackasm1(SB) - MOVW $902, R12 - B runtime·callbackasm1(SB) - MOVW $903, R12 - B runtime·callbackasm1(SB) - MOVW $904, R12 - B runtime·callbackasm1(SB) - MOVW $905, R12 - B runtime·callbackasm1(SB) - MOVW $906, R12 - B runtime·callbackasm1(SB) - MOVW $907, R12 - B runtime·callbackasm1(SB) - MOVW $908, R12 - B runtime·callbackasm1(SB) - MOVW $909, R12 - B runtime·callbackasm1(SB) - MOVW $910, R12 - B runtime·callbackasm1(SB) - MOVW $911, R12 - B runtime·callbackasm1(SB) - MOVW $912, R12 - B runtime·callbackasm1(SB) - MOVW $913, R12 - B runtime·callbackasm1(SB) - MOVW $914, R12 - B runtime·callbackasm1(SB) - MOVW $915, R12 - B runtime·callbackasm1(SB) - MOVW $916, R12 - B runtime·callbackasm1(SB) - MOVW $917, R12 - B runtime·callbackasm1(SB) - MOVW $918, R12 - B runtime·callbackasm1(SB) - MOVW $919, R12 - B runtime·callbackasm1(SB) - MOVW $920, R12 - B runtime·callbackasm1(SB) - MOVW $921, R12 - B runtime·callbackasm1(SB) - MOVW $922, R12 - B runtime·callbackasm1(SB) - MOVW $923, R12 - B runtime·callbackasm1(SB) - MOVW $924, R12 - B runtime·callbackasm1(SB) - MOVW $925, R12 - B runtime·callbackasm1(SB) - MOVW $926, R12 - B runtime·callbackasm1(SB) - MOVW $927, R12 - B runtime·callbackasm1(SB) - MOVW $928, R12 - B runtime·callbackasm1(SB) - MOVW $929, R12 - B runtime·callbackasm1(SB) - MOVW $930, R12 - B runtime·callbackasm1(SB) - MOVW $931, R12 - B runtime·callbackasm1(SB) - MOVW $932, R12 - B runtime·callbackasm1(SB) - MOVW $933, R12 - B runtime·callbackasm1(SB) - MOVW $934, R12 - B runtime·callbackasm1(SB) - MOVW $935, R12 - B runtime·callbackasm1(SB) - MOVW $936, R12 - B runtime·callbackasm1(SB) - MOVW $937, R12 - B runtime·callbackasm1(SB) - MOVW $938, R12 - B runtime·callbackasm1(SB) - MOVW $939, R12 - B runtime·callbackasm1(SB) - MOVW $940, R12 - B runtime·callbackasm1(SB) - MOVW $941, R12 - B runtime·callbackasm1(SB) - MOVW $942, R12 - B runtime·callbackasm1(SB) - MOVW $943, R12 - B runtime·callbackasm1(SB) - MOVW $944, R12 - B runtime·callbackasm1(SB) - MOVW $945, R12 - B runtime·callbackasm1(SB) - MOVW $946, R12 - B runtime·callbackasm1(SB) - MOVW $947, R12 - B runtime·callbackasm1(SB) - MOVW $948, R12 - B runtime·callbackasm1(SB) - MOVW $949, R12 - B runtime·callbackasm1(SB) - MOVW $950, R12 - B runtime·callbackasm1(SB) - MOVW $951, R12 - B runtime·callbackasm1(SB) - MOVW $952, R12 - B runtime·callbackasm1(SB) - MOVW $953, R12 - B runtime·callbackasm1(SB) - MOVW $954, R12 - B runtime·callbackasm1(SB) - MOVW $955, R12 - B runtime·callbackasm1(SB) - MOVW $956, R12 - B runtime·callbackasm1(SB) - MOVW $957, R12 - B runtime·callbackasm1(SB) - MOVW $958, R12 - B runtime·callbackasm1(SB) - MOVW $959, R12 - B runtime·callbackasm1(SB) - MOVW $960, R12 - B runtime·callbackasm1(SB) - MOVW $961, R12 - B runtime·callbackasm1(SB) - MOVW $962, R12 - B runtime·callbackasm1(SB) - MOVW $963, R12 - B runtime·callbackasm1(SB) - MOVW $964, R12 - B runtime·callbackasm1(SB) - MOVW $965, R12 - B runtime·callbackasm1(SB) - MOVW $966, R12 - B runtime·callbackasm1(SB) - MOVW $967, R12 - B runtime·callbackasm1(SB) - MOVW $968, R12 - B runtime·callbackasm1(SB) - MOVW $969, R12 - B runtime·callbackasm1(SB) - MOVW $970, R12 - B runtime·callbackasm1(SB) - MOVW $971, R12 - B runtime·callbackasm1(SB) - MOVW $972, R12 - B runtime·callbackasm1(SB) - MOVW $973, R12 - B runtime·callbackasm1(SB) - MOVW $974, R12 - B runtime·callbackasm1(SB) - MOVW $975, R12 - B runtime·callbackasm1(SB) - MOVW $976, R12 - B runtime·callbackasm1(SB) - MOVW $977, R12 - B runtime·callbackasm1(SB) - MOVW $978, R12 - B runtime·callbackasm1(SB) - MOVW $979, R12 - B runtime·callbackasm1(SB) - MOVW $980, R12 - B runtime·callbackasm1(SB) - MOVW $981, R12 - B runtime·callbackasm1(SB) - MOVW $982, R12 - B runtime·callbackasm1(SB) - MOVW $983, R12 - B runtime·callbackasm1(SB) - MOVW $984, R12 - B runtime·callbackasm1(SB) - MOVW $985, R12 - B runtime·callbackasm1(SB) - MOVW $986, R12 - B runtime·callbackasm1(SB) - MOVW $987, R12 - B runtime·callbackasm1(SB) - MOVW $988, R12 - B runtime·callbackasm1(SB) - MOVW $989, R12 - B runtime·callbackasm1(SB) - MOVW $990, R12 - B runtime·callbackasm1(SB) - MOVW $991, R12 - B runtime·callbackasm1(SB) - MOVW $992, R12 - B runtime·callbackasm1(SB) - MOVW $993, R12 - B runtime·callbackasm1(SB) - MOVW $994, R12 - B runtime·callbackasm1(SB) - MOVW $995, R12 - B runtime·callbackasm1(SB) - MOVW $996, R12 - B runtime·callbackasm1(SB) - MOVW $997, R12 - B runtime·callbackasm1(SB) - MOVW $998, R12 - B runtime·callbackasm1(SB) - MOVW $999, R12 - B runtime·callbackasm1(SB) - MOVW $1000, R12 - B runtime·callbackasm1(SB) - MOVW $1001, R12 - B runtime·callbackasm1(SB) - MOVW $1002, R12 - B runtime·callbackasm1(SB) - MOVW $1003, R12 - B runtime·callbackasm1(SB) - MOVW $1004, R12 - B runtime·callbackasm1(SB) - MOVW $1005, R12 - B runtime·callbackasm1(SB) - MOVW $1006, R12 - B runtime·callbackasm1(SB) - MOVW $1007, R12 - B runtime·callbackasm1(SB) - MOVW $1008, R12 - B runtime·callbackasm1(SB) - MOVW $1009, R12 - B runtime·callbackasm1(SB) - MOVW $1010, R12 - B runtime·callbackasm1(SB) - MOVW $1011, R12 - B runtime·callbackasm1(SB) - MOVW $1012, R12 - B runtime·callbackasm1(SB) - MOVW $1013, R12 - B runtime·callbackasm1(SB) - MOVW $1014, R12 - B runtime·callbackasm1(SB) - MOVW $1015, R12 - B runtime·callbackasm1(SB) - MOVW $1016, R12 - B runtime·callbackasm1(SB) - MOVW $1017, R12 - B runtime·callbackasm1(SB) - MOVW $1018, R12 - B runtime·callbackasm1(SB) - MOVW $1019, R12 - B runtime·callbackasm1(SB) - MOVW $1020, R12 - B runtime·callbackasm1(SB) - MOVW $1021, R12 - B runtime·callbackasm1(SB) - MOVW $1022, R12 - B runtime·callbackasm1(SB) - MOVW $1023, R12 - B runtime·callbackasm1(SB) - MOVW $1024, R12 - B runtime·callbackasm1(SB) - MOVW $1025, R12 - B runtime·callbackasm1(SB) - MOVW $1026, R12 - B runtime·callbackasm1(SB) - MOVW $1027, R12 - B runtime·callbackasm1(SB) - MOVW $1028, R12 - B runtime·callbackasm1(SB) - MOVW $1029, R12 - B runtime·callbackasm1(SB) - MOVW $1030, R12 - B runtime·callbackasm1(SB) - MOVW $1031, R12 - B runtime·callbackasm1(SB) - MOVW $1032, R12 - B runtime·callbackasm1(SB) - MOVW $1033, R12 - B runtime·callbackasm1(SB) - MOVW $1034, R12 - B runtime·callbackasm1(SB) - MOVW $1035, R12 - B runtime·callbackasm1(SB) - MOVW $1036, R12 - B runtime·callbackasm1(SB) - MOVW $1037, R12 - B runtime·callbackasm1(SB) - MOVW $1038, R12 - B runtime·callbackasm1(SB) - MOVW $1039, R12 - B runtime·callbackasm1(SB) - MOVW $1040, R12 - B runtime·callbackasm1(SB) - MOVW $1041, R12 - B runtime·callbackasm1(SB) - MOVW $1042, R12 - B runtime·callbackasm1(SB) - MOVW $1043, R12 - B runtime·callbackasm1(SB) - MOVW $1044, R12 - B runtime·callbackasm1(SB) - MOVW $1045, R12 - B runtime·callbackasm1(SB) - MOVW $1046, R12 - B runtime·callbackasm1(SB) - MOVW $1047, R12 - B runtime·callbackasm1(SB) - MOVW $1048, R12 - B runtime·callbackasm1(SB) - MOVW $1049, R12 - B runtime·callbackasm1(SB) - MOVW $1050, R12 - B runtime·callbackasm1(SB) - MOVW $1051, R12 - B runtime·callbackasm1(SB) - MOVW $1052, R12 - B runtime·callbackasm1(SB) - MOVW $1053, R12 - B runtime·callbackasm1(SB) - MOVW $1054, R12 - B runtime·callbackasm1(SB) - MOVW $1055, R12 - B runtime·callbackasm1(SB) - MOVW $1056, R12 - B runtime·callbackasm1(SB) - MOVW $1057, R12 - B runtime·callbackasm1(SB) - MOVW $1058, R12 - B runtime·callbackasm1(SB) - MOVW $1059, R12 - B runtime·callbackasm1(SB) - MOVW $1060, R12 - B runtime·callbackasm1(SB) - MOVW $1061, R12 - B runtime·callbackasm1(SB) - MOVW $1062, R12 - B runtime·callbackasm1(SB) - MOVW $1063, R12 - B runtime·callbackasm1(SB) - MOVW $1064, R12 - B runtime·callbackasm1(SB) - MOVW $1065, R12 - B runtime·callbackasm1(SB) - MOVW $1066, R12 - B runtime·callbackasm1(SB) - MOVW $1067, R12 - B runtime·callbackasm1(SB) - MOVW $1068, R12 - B runtime·callbackasm1(SB) - MOVW $1069, R12 - B runtime·callbackasm1(SB) - MOVW $1070, R12 - B runtime·callbackasm1(SB) - MOVW $1071, R12 - B runtime·callbackasm1(SB) - MOVW $1072, R12 - B runtime·callbackasm1(SB) - MOVW $1073, R12 - B runtime·callbackasm1(SB) - MOVW $1074, R12 - B runtime·callbackasm1(SB) - MOVW $1075, R12 - B runtime·callbackasm1(SB) - MOVW $1076, R12 - B runtime·callbackasm1(SB) - MOVW $1077, R12 - B runtime·callbackasm1(SB) - MOVW $1078, R12 - B runtime·callbackasm1(SB) - MOVW $1079, R12 - B runtime·callbackasm1(SB) - MOVW $1080, R12 - B runtime·callbackasm1(SB) - MOVW $1081, R12 - B runtime·callbackasm1(SB) - MOVW $1082, R12 - B runtime·callbackasm1(SB) - MOVW $1083, R12 - B runtime·callbackasm1(SB) - MOVW $1084, R12 - B runtime·callbackasm1(SB) - MOVW $1085, R12 - B runtime·callbackasm1(SB) - MOVW $1086, R12 - B runtime·callbackasm1(SB) - MOVW $1087, R12 - B runtime·callbackasm1(SB) - MOVW $1088, R12 - B runtime·callbackasm1(SB) - MOVW $1089, R12 - B runtime·callbackasm1(SB) - MOVW $1090, R12 - B runtime·callbackasm1(SB) - MOVW $1091, R12 - B runtime·callbackasm1(SB) - MOVW $1092, R12 - B runtime·callbackasm1(SB) - MOVW $1093, R12 - B runtime·callbackasm1(SB) - MOVW $1094, R12 - B runtime·callbackasm1(SB) - MOVW $1095, R12 - B runtime·callbackasm1(SB) - MOVW $1096, R12 - B runtime·callbackasm1(SB) - MOVW $1097, R12 - B runtime·callbackasm1(SB) - MOVW $1098, R12 - B runtime·callbackasm1(SB) - MOVW $1099, R12 - B runtime·callbackasm1(SB) - MOVW $1100, R12 - B runtime·callbackasm1(SB) - MOVW $1101, R12 - B runtime·callbackasm1(SB) - MOVW $1102, R12 - B runtime·callbackasm1(SB) - MOVW $1103, R12 - B runtime·callbackasm1(SB) - MOVW $1104, R12 - B runtime·callbackasm1(SB) - MOVW $1105, R12 - B runtime·callbackasm1(SB) - MOVW $1106, R12 - B runtime·callbackasm1(SB) - MOVW $1107, R12 - B runtime·callbackasm1(SB) - MOVW $1108, R12 - B runtime·callbackasm1(SB) - MOVW $1109, R12 - B runtime·callbackasm1(SB) - MOVW $1110, R12 - B runtime·callbackasm1(SB) - MOVW $1111, R12 - B runtime·callbackasm1(SB) - MOVW $1112, R12 - B runtime·callbackasm1(SB) - MOVW $1113, R12 - B runtime·callbackasm1(SB) - MOVW $1114, R12 - B runtime·callbackasm1(SB) - MOVW $1115, R12 - B runtime·callbackasm1(SB) - MOVW $1116, R12 - B runtime·callbackasm1(SB) - MOVW $1117, R12 - B runtime·callbackasm1(SB) - MOVW $1118, R12 - B runtime·callbackasm1(SB) - MOVW $1119, R12 - B runtime·callbackasm1(SB) - MOVW $1120, R12 - B runtime·callbackasm1(SB) - MOVW $1121, R12 - B runtime·callbackasm1(SB) - MOVW $1122, R12 - B runtime·callbackasm1(SB) - MOVW $1123, R12 - B runtime·callbackasm1(SB) - MOVW $1124, R12 - B runtime·callbackasm1(SB) - MOVW $1125, R12 - B runtime·callbackasm1(SB) - MOVW $1126, R12 - B runtime·callbackasm1(SB) - MOVW $1127, R12 - B runtime·callbackasm1(SB) - MOVW $1128, R12 - B runtime·callbackasm1(SB) - MOVW $1129, R12 - B runtime·callbackasm1(SB) - MOVW $1130, R12 - B runtime·callbackasm1(SB) - MOVW $1131, R12 - B runtime·callbackasm1(SB) - MOVW $1132, R12 - B runtime·callbackasm1(SB) - MOVW $1133, R12 - B runtime·callbackasm1(SB) - MOVW $1134, R12 - B runtime·callbackasm1(SB) - MOVW $1135, R12 - B runtime·callbackasm1(SB) - MOVW $1136, R12 - B runtime·callbackasm1(SB) - MOVW $1137, R12 - B runtime·callbackasm1(SB) - MOVW $1138, R12 - B runtime·callbackasm1(SB) - MOVW $1139, R12 - B runtime·callbackasm1(SB) - MOVW $1140, R12 - B runtime·callbackasm1(SB) - MOVW $1141, R12 - B runtime·callbackasm1(SB) - MOVW $1142, R12 - B runtime·callbackasm1(SB) - MOVW $1143, R12 - B runtime·callbackasm1(SB) - MOVW $1144, R12 - B runtime·callbackasm1(SB) - MOVW $1145, R12 - B runtime·callbackasm1(SB) - MOVW $1146, R12 - B runtime·callbackasm1(SB) - MOVW $1147, R12 - B runtime·callbackasm1(SB) - MOVW $1148, R12 - B runtime·callbackasm1(SB) - MOVW $1149, R12 - B runtime·callbackasm1(SB) - MOVW $1150, R12 - B runtime·callbackasm1(SB) - MOVW $1151, R12 - B runtime·callbackasm1(SB) - MOVW $1152, R12 - B runtime·callbackasm1(SB) - MOVW $1153, R12 - B runtime·callbackasm1(SB) - MOVW $1154, R12 - B runtime·callbackasm1(SB) - MOVW $1155, R12 - B runtime·callbackasm1(SB) - MOVW $1156, R12 - B runtime·callbackasm1(SB) - MOVW $1157, R12 - B runtime·callbackasm1(SB) - MOVW $1158, R12 - B runtime·callbackasm1(SB) - MOVW $1159, R12 - B runtime·callbackasm1(SB) - MOVW $1160, R12 - B runtime·callbackasm1(SB) - MOVW $1161, R12 - B runtime·callbackasm1(SB) - MOVW $1162, R12 - B runtime·callbackasm1(SB) - MOVW $1163, R12 - B runtime·callbackasm1(SB) - MOVW $1164, R12 - B runtime·callbackasm1(SB) - MOVW $1165, R12 - B runtime·callbackasm1(SB) - MOVW $1166, R12 - B runtime·callbackasm1(SB) - MOVW $1167, R12 - B runtime·callbackasm1(SB) - MOVW $1168, R12 - B runtime·callbackasm1(SB) - MOVW $1169, R12 - B runtime·callbackasm1(SB) - MOVW $1170, R12 - B runtime·callbackasm1(SB) - MOVW $1171, R12 - B runtime·callbackasm1(SB) - MOVW $1172, R12 - B runtime·callbackasm1(SB) - MOVW $1173, R12 - B runtime·callbackasm1(SB) - MOVW $1174, R12 - B runtime·callbackasm1(SB) - MOVW $1175, R12 - B runtime·callbackasm1(SB) - MOVW $1176, R12 - B runtime·callbackasm1(SB) - MOVW $1177, R12 - B runtime·callbackasm1(SB) - MOVW $1178, R12 - B runtime·callbackasm1(SB) - MOVW $1179, R12 - B runtime·callbackasm1(SB) - MOVW $1180, R12 - B runtime·callbackasm1(SB) - MOVW $1181, R12 - B runtime·callbackasm1(SB) - MOVW $1182, R12 - B runtime·callbackasm1(SB) - MOVW $1183, R12 - B runtime·callbackasm1(SB) - MOVW $1184, R12 - B runtime·callbackasm1(SB) - MOVW $1185, R12 - B runtime·callbackasm1(SB) - MOVW $1186, R12 - B runtime·callbackasm1(SB) - MOVW $1187, R12 - B runtime·callbackasm1(SB) - MOVW $1188, R12 - B runtime·callbackasm1(SB) - MOVW $1189, R12 - B runtime·callbackasm1(SB) - MOVW $1190, R12 - B runtime·callbackasm1(SB) - MOVW $1191, R12 - B runtime·callbackasm1(SB) - MOVW $1192, R12 - B runtime·callbackasm1(SB) - MOVW $1193, R12 - B runtime·callbackasm1(SB) - MOVW $1194, R12 - B runtime·callbackasm1(SB) - MOVW $1195, R12 - B runtime·callbackasm1(SB) - MOVW $1196, R12 - B runtime·callbackasm1(SB) - MOVW $1197, R12 - B runtime·callbackasm1(SB) - MOVW $1198, R12 - B runtime·callbackasm1(SB) - MOVW $1199, R12 - B runtime·callbackasm1(SB) - MOVW $1200, R12 - B runtime·callbackasm1(SB) - MOVW $1201, R12 - B runtime·callbackasm1(SB) - MOVW $1202, R12 - B runtime·callbackasm1(SB) - MOVW $1203, R12 - B runtime·callbackasm1(SB) - MOVW $1204, R12 - B runtime·callbackasm1(SB) - MOVW $1205, R12 - B runtime·callbackasm1(SB) - MOVW $1206, R12 - B runtime·callbackasm1(SB) - MOVW $1207, R12 - B runtime·callbackasm1(SB) - MOVW $1208, R12 - B runtime·callbackasm1(SB) - MOVW $1209, R12 - B runtime·callbackasm1(SB) - MOVW $1210, R12 - B runtime·callbackasm1(SB) - MOVW $1211, R12 - B runtime·callbackasm1(SB) - MOVW $1212, R12 - B runtime·callbackasm1(SB) - MOVW $1213, R12 - B runtime·callbackasm1(SB) - MOVW $1214, R12 - B runtime·callbackasm1(SB) - MOVW $1215, R12 - B runtime·callbackasm1(SB) - MOVW $1216, R12 - B runtime·callbackasm1(SB) - MOVW $1217, R12 - B runtime·callbackasm1(SB) - MOVW $1218, R12 - B runtime·callbackasm1(SB) - MOVW $1219, R12 - B runtime·callbackasm1(SB) - MOVW $1220, R12 - B runtime·callbackasm1(SB) - MOVW $1221, R12 - B runtime·callbackasm1(SB) - MOVW $1222, R12 - B runtime·callbackasm1(SB) - MOVW $1223, R12 - B runtime·callbackasm1(SB) - MOVW $1224, R12 - B runtime·callbackasm1(SB) - MOVW $1225, R12 - B runtime·callbackasm1(SB) - MOVW $1226, R12 - B runtime·callbackasm1(SB) - MOVW $1227, R12 - B runtime·callbackasm1(SB) - MOVW $1228, R12 - B runtime·callbackasm1(SB) - MOVW $1229, R12 - B runtime·callbackasm1(SB) - MOVW $1230, R12 - B runtime·callbackasm1(SB) - MOVW $1231, R12 - B runtime·callbackasm1(SB) - MOVW $1232, R12 - B runtime·callbackasm1(SB) - MOVW $1233, R12 - B runtime·callbackasm1(SB) - MOVW $1234, R12 - B runtime·callbackasm1(SB) - MOVW $1235, R12 - B runtime·callbackasm1(SB) - MOVW $1236, R12 - B runtime·callbackasm1(SB) - MOVW $1237, R12 - B runtime·callbackasm1(SB) - MOVW $1238, R12 - B runtime·callbackasm1(SB) - MOVW $1239, R12 - B runtime·callbackasm1(SB) - MOVW $1240, R12 - B runtime·callbackasm1(SB) - MOVW $1241, R12 - B runtime·callbackasm1(SB) - MOVW $1242, R12 - B runtime·callbackasm1(SB) - MOVW $1243, R12 - B runtime·callbackasm1(SB) - MOVW $1244, R12 - B runtime·callbackasm1(SB) - MOVW $1245, R12 - B runtime·callbackasm1(SB) - MOVW $1246, R12 - B runtime·callbackasm1(SB) - MOVW $1247, R12 - B runtime·callbackasm1(SB) - MOVW $1248, R12 - B runtime·callbackasm1(SB) - MOVW $1249, R12 - B runtime·callbackasm1(SB) - MOVW $1250, R12 - B runtime·callbackasm1(SB) - MOVW $1251, R12 - B runtime·callbackasm1(SB) - MOVW $1252, R12 - B runtime·callbackasm1(SB) - MOVW $1253, R12 - B runtime·callbackasm1(SB) - MOVW $1254, R12 - B runtime·callbackasm1(SB) - MOVW $1255, R12 - B runtime·callbackasm1(SB) - MOVW $1256, R12 - B runtime·callbackasm1(SB) - MOVW $1257, R12 - B runtime·callbackasm1(SB) - MOVW $1258, R12 - B runtime·callbackasm1(SB) - MOVW $1259, R12 - B runtime·callbackasm1(SB) - MOVW $1260, R12 - B runtime·callbackasm1(SB) - MOVW $1261, R12 - B runtime·callbackasm1(SB) - MOVW $1262, R12 - B runtime·callbackasm1(SB) - MOVW $1263, R12 - B runtime·callbackasm1(SB) - MOVW $1264, R12 - B runtime·callbackasm1(SB) - MOVW $1265, R12 - B runtime·callbackasm1(SB) - MOVW $1266, R12 - B runtime·callbackasm1(SB) - MOVW $1267, R12 - B runtime·callbackasm1(SB) - MOVW $1268, R12 - B runtime·callbackasm1(SB) - MOVW $1269, R12 - B runtime·callbackasm1(SB) - MOVW $1270, R12 - B runtime·callbackasm1(SB) - MOVW $1271, R12 - B runtime·callbackasm1(SB) - MOVW $1272, R12 - B runtime·callbackasm1(SB) - MOVW $1273, R12 - B runtime·callbackasm1(SB) - MOVW $1274, R12 - B runtime·callbackasm1(SB) - MOVW $1275, R12 - B runtime·callbackasm1(SB) - MOVW $1276, R12 - B runtime·callbackasm1(SB) - MOVW $1277, R12 - B runtime·callbackasm1(SB) - MOVW $1278, R12 - B runtime·callbackasm1(SB) - MOVW $1279, R12 - B runtime·callbackasm1(SB) - MOVW $1280, R12 - B runtime·callbackasm1(SB) - MOVW $1281, R12 - B runtime·callbackasm1(SB) - MOVW $1282, R12 - B runtime·callbackasm1(SB) - MOVW $1283, R12 - B runtime·callbackasm1(SB) - MOVW $1284, R12 - B runtime·callbackasm1(SB) - MOVW $1285, R12 - B runtime·callbackasm1(SB) - MOVW $1286, R12 - B runtime·callbackasm1(SB) - MOVW $1287, R12 - B runtime·callbackasm1(SB) - MOVW $1288, R12 - B runtime·callbackasm1(SB) - MOVW $1289, R12 - B runtime·callbackasm1(SB) - MOVW $1290, R12 - B runtime·callbackasm1(SB) - MOVW $1291, R12 - B runtime·callbackasm1(SB) - MOVW $1292, R12 - B runtime·callbackasm1(SB) - MOVW $1293, R12 - B runtime·callbackasm1(SB) - MOVW $1294, R12 - B runtime·callbackasm1(SB) - MOVW $1295, R12 - B runtime·callbackasm1(SB) - MOVW $1296, R12 - B runtime·callbackasm1(SB) - MOVW $1297, R12 - B runtime·callbackasm1(SB) - MOVW $1298, R12 - B runtime·callbackasm1(SB) - MOVW $1299, R12 - B runtime·callbackasm1(SB) - MOVW $1300, R12 - B runtime·callbackasm1(SB) - MOVW $1301, R12 - B runtime·callbackasm1(SB) - MOVW $1302, R12 - B runtime·callbackasm1(SB) - MOVW $1303, R12 - B runtime·callbackasm1(SB) - MOVW $1304, R12 - B runtime·callbackasm1(SB) - MOVW $1305, R12 - B runtime·callbackasm1(SB) - MOVW $1306, R12 - B runtime·callbackasm1(SB) - MOVW $1307, R12 - B runtime·callbackasm1(SB) - MOVW $1308, R12 - B runtime·callbackasm1(SB) - MOVW $1309, R12 - B runtime·callbackasm1(SB) - MOVW $1310, R12 - B runtime·callbackasm1(SB) - MOVW $1311, R12 - B runtime·callbackasm1(SB) - MOVW $1312, R12 - B runtime·callbackasm1(SB) - MOVW $1313, R12 - B runtime·callbackasm1(SB) - MOVW $1314, R12 - B runtime·callbackasm1(SB) - MOVW $1315, R12 - B runtime·callbackasm1(SB) - MOVW $1316, R12 - B runtime·callbackasm1(SB) - MOVW $1317, R12 - B runtime·callbackasm1(SB) - MOVW $1318, R12 - B runtime·callbackasm1(SB) - MOVW $1319, R12 - B runtime·callbackasm1(SB) - MOVW $1320, R12 - B runtime·callbackasm1(SB) - MOVW $1321, R12 - B runtime·callbackasm1(SB) - MOVW $1322, R12 - B runtime·callbackasm1(SB) - MOVW $1323, R12 - B runtime·callbackasm1(SB) - MOVW $1324, R12 - B runtime·callbackasm1(SB) - MOVW $1325, R12 - B runtime·callbackasm1(SB) - MOVW $1326, R12 - B runtime·callbackasm1(SB) - MOVW $1327, R12 - B runtime·callbackasm1(SB) - MOVW $1328, R12 - B runtime·callbackasm1(SB) - MOVW $1329, R12 - B runtime·callbackasm1(SB) - MOVW $1330, R12 - B runtime·callbackasm1(SB) - MOVW $1331, R12 - B runtime·callbackasm1(SB) - MOVW $1332, R12 - B runtime·callbackasm1(SB) - MOVW $1333, R12 - B runtime·callbackasm1(SB) - MOVW $1334, R12 - B runtime·callbackasm1(SB) - MOVW $1335, R12 - B runtime·callbackasm1(SB) - MOVW $1336, R12 - B runtime·callbackasm1(SB) - MOVW $1337, R12 - B runtime·callbackasm1(SB) - MOVW $1338, R12 - B runtime·callbackasm1(SB) - MOVW $1339, R12 - B runtime·callbackasm1(SB) - MOVW $1340, R12 - B runtime·callbackasm1(SB) - MOVW $1341, R12 - B runtime·callbackasm1(SB) - MOVW $1342, R12 - B runtime·callbackasm1(SB) - MOVW $1343, R12 - B runtime·callbackasm1(SB) - MOVW $1344, R12 - B runtime·callbackasm1(SB) - MOVW $1345, R12 - B runtime·callbackasm1(SB) - MOVW $1346, R12 - B runtime·callbackasm1(SB) - MOVW $1347, R12 - B runtime·callbackasm1(SB) - MOVW $1348, R12 - B runtime·callbackasm1(SB) - MOVW $1349, R12 - B runtime·callbackasm1(SB) - MOVW $1350, R12 - B runtime·callbackasm1(SB) - MOVW $1351, R12 - B runtime·callbackasm1(SB) - MOVW $1352, R12 - B runtime·callbackasm1(SB) - MOVW $1353, R12 - B runtime·callbackasm1(SB) - MOVW $1354, R12 - B runtime·callbackasm1(SB) - MOVW $1355, R12 - B runtime·callbackasm1(SB) - MOVW $1356, R12 - B runtime·callbackasm1(SB) - MOVW $1357, R12 - B runtime·callbackasm1(SB) - MOVW $1358, R12 - B runtime·callbackasm1(SB) - MOVW $1359, R12 - B runtime·callbackasm1(SB) - MOVW $1360, R12 - B runtime·callbackasm1(SB) - MOVW $1361, R12 - B runtime·callbackasm1(SB) - MOVW $1362, R12 - B runtime·callbackasm1(SB) - MOVW $1363, R12 - B runtime·callbackasm1(SB) - MOVW $1364, R12 - B runtime·callbackasm1(SB) - MOVW $1365, R12 - B runtime·callbackasm1(SB) - MOVW $1366, R12 - B runtime·callbackasm1(SB) - MOVW $1367, R12 - B runtime·callbackasm1(SB) - MOVW $1368, R12 - B runtime·callbackasm1(SB) - MOVW $1369, R12 - B runtime·callbackasm1(SB) - MOVW $1370, R12 - B runtime·callbackasm1(SB) - MOVW $1371, R12 - B runtime·callbackasm1(SB) - MOVW $1372, R12 - B runtime·callbackasm1(SB) - MOVW $1373, R12 - B runtime·callbackasm1(SB) - MOVW $1374, R12 - B runtime·callbackasm1(SB) - MOVW $1375, R12 - B runtime·callbackasm1(SB) - MOVW $1376, R12 - B runtime·callbackasm1(SB) - MOVW $1377, R12 - B runtime·callbackasm1(SB) - MOVW $1378, R12 - B runtime·callbackasm1(SB) - MOVW $1379, R12 - B runtime·callbackasm1(SB) - MOVW $1380, R12 - B runtime·callbackasm1(SB) - MOVW $1381, R12 - B runtime·callbackasm1(SB) - MOVW $1382, R12 - B runtime·callbackasm1(SB) - MOVW $1383, R12 - B runtime·callbackasm1(SB) - MOVW $1384, R12 - B runtime·callbackasm1(SB) - MOVW $1385, R12 - B runtime·callbackasm1(SB) - MOVW $1386, R12 - B runtime·callbackasm1(SB) - MOVW $1387, R12 - B runtime·callbackasm1(SB) - MOVW $1388, R12 - B runtime·callbackasm1(SB) - MOVW $1389, R12 - B runtime·callbackasm1(SB) - MOVW $1390, R12 - B runtime·callbackasm1(SB) - MOVW $1391, R12 - B runtime·callbackasm1(SB) - MOVW $1392, R12 - B runtime·callbackasm1(SB) - MOVW $1393, R12 - B runtime·callbackasm1(SB) - MOVW $1394, R12 - B runtime·callbackasm1(SB) - MOVW $1395, R12 - B runtime·callbackasm1(SB) - MOVW $1396, R12 - B runtime·callbackasm1(SB) - MOVW $1397, R12 - B runtime·callbackasm1(SB) - MOVW $1398, R12 - B runtime·callbackasm1(SB) - MOVW $1399, R12 - B runtime·callbackasm1(SB) - MOVW $1400, R12 - B runtime·callbackasm1(SB) - MOVW $1401, R12 - B runtime·callbackasm1(SB) - MOVW $1402, R12 - B runtime·callbackasm1(SB) - MOVW $1403, R12 - B runtime·callbackasm1(SB) - MOVW $1404, R12 - B runtime·callbackasm1(SB) - MOVW $1405, R12 - B runtime·callbackasm1(SB) - MOVW $1406, R12 - B runtime·callbackasm1(SB) - MOVW $1407, R12 - B runtime·callbackasm1(SB) - MOVW $1408, R12 - B runtime·callbackasm1(SB) - MOVW $1409, R12 - B runtime·callbackasm1(SB) - MOVW $1410, R12 - B runtime·callbackasm1(SB) - MOVW $1411, R12 - B runtime·callbackasm1(SB) - MOVW $1412, R12 - B runtime·callbackasm1(SB) - MOVW $1413, R12 - B runtime·callbackasm1(SB) - MOVW $1414, R12 - B runtime·callbackasm1(SB) - MOVW $1415, R12 - B runtime·callbackasm1(SB) - MOVW $1416, R12 - B runtime·callbackasm1(SB) - MOVW $1417, R12 - B runtime·callbackasm1(SB) - MOVW $1418, R12 - B runtime·callbackasm1(SB) - MOVW $1419, R12 - B runtime·callbackasm1(SB) - MOVW $1420, R12 - B runtime·callbackasm1(SB) - MOVW $1421, R12 - B runtime·callbackasm1(SB) - MOVW $1422, R12 - B runtime·callbackasm1(SB) - MOVW $1423, R12 - B runtime·callbackasm1(SB) - MOVW $1424, R12 - B runtime·callbackasm1(SB) - MOVW $1425, R12 - B runtime·callbackasm1(SB) - MOVW $1426, R12 - B runtime·callbackasm1(SB) - MOVW $1427, R12 - B runtime·callbackasm1(SB) - MOVW $1428, R12 - B runtime·callbackasm1(SB) - MOVW $1429, R12 - B runtime·callbackasm1(SB) - MOVW $1430, R12 - B runtime·callbackasm1(SB) - MOVW $1431, R12 - B runtime·callbackasm1(SB) - MOVW $1432, R12 - B runtime·callbackasm1(SB) - MOVW $1433, R12 - B runtime·callbackasm1(SB) - MOVW $1434, R12 - B runtime·callbackasm1(SB) - MOVW $1435, R12 - B runtime·callbackasm1(SB) - MOVW $1436, R12 - B runtime·callbackasm1(SB) - MOVW $1437, R12 - B runtime·callbackasm1(SB) - MOVW $1438, R12 - B runtime·callbackasm1(SB) - MOVW $1439, R12 - B runtime·callbackasm1(SB) - MOVW $1440, R12 - B runtime·callbackasm1(SB) - MOVW $1441, R12 - B runtime·callbackasm1(SB) - MOVW $1442, R12 - B runtime·callbackasm1(SB) - MOVW $1443, R12 - B runtime·callbackasm1(SB) - MOVW $1444, R12 - B runtime·callbackasm1(SB) - MOVW $1445, R12 - B runtime·callbackasm1(SB) - MOVW $1446, R12 - B runtime·callbackasm1(SB) - MOVW $1447, R12 - B runtime·callbackasm1(SB) - MOVW $1448, R12 - B runtime·callbackasm1(SB) - MOVW $1449, R12 - B runtime·callbackasm1(SB) - MOVW $1450, R12 - B runtime·callbackasm1(SB) - MOVW $1451, R12 - B runtime·callbackasm1(SB) - MOVW $1452, R12 - B runtime·callbackasm1(SB) - MOVW $1453, R12 - B runtime·callbackasm1(SB) - MOVW $1454, R12 - B runtime·callbackasm1(SB) - MOVW $1455, R12 - B runtime·callbackasm1(SB) - MOVW $1456, R12 - B runtime·callbackasm1(SB) - MOVW $1457, R12 - B runtime·callbackasm1(SB) - MOVW $1458, R12 - B runtime·callbackasm1(SB) - MOVW $1459, R12 - B runtime·callbackasm1(SB) - MOVW $1460, R12 - B runtime·callbackasm1(SB) - MOVW $1461, R12 - B runtime·callbackasm1(SB) - MOVW $1462, R12 - B runtime·callbackasm1(SB) - MOVW $1463, R12 - B runtime·callbackasm1(SB) - MOVW $1464, R12 - B runtime·callbackasm1(SB) - MOVW $1465, R12 - B runtime·callbackasm1(SB) - MOVW $1466, R12 - B runtime·callbackasm1(SB) - MOVW $1467, R12 - B runtime·callbackasm1(SB) - MOVW $1468, R12 - B runtime·callbackasm1(SB) - MOVW $1469, R12 - B runtime·callbackasm1(SB) - MOVW $1470, R12 - B runtime·callbackasm1(SB) - MOVW $1471, R12 - B runtime·callbackasm1(SB) - MOVW $1472, R12 - B runtime·callbackasm1(SB) - MOVW $1473, R12 - B runtime·callbackasm1(SB) - MOVW $1474, R12 - B runtime·callbackasm1(SB) - MOVW $1475, R12 - B runtime·callbackasm1(SB) - MOVW $1476, R12 - B runtime·callbackasm1(SB) - MOVW $1477, R12 - B runtime·callbackasm1(SB) - MOVW $1478, R12 - B runtime·callbackasm1(SB) - MOVW $1479, R12 - B runtime·callbackasm1(SB) - MOVW $1480, R12 - B runtime·callbackasm1(SB) - MOVW $1481, R12 - B runtime·callbackasm1(SB) - MOVW $1482, R12 - B runtime·callbackasm1(SB) - MOVW $1483, R12 - B runtime·callbackasm1(SB) - MOVW $1484, R12 - B runtime·callbackasm1(SB) - MOVW $1485, R12 - B runtime·callbackasm1(SB) - MOVW $1486, R12 - B runtime·callbackasm1(SB) - MOVW $1487, R12 - B runtime·callbackasm1(SB) - MOVW $1488, R12 - B runtime·callbackasm1(SB) - MOVW $1489, R12 - B runtime·callbackasm1(SB) - MOVW $1490, R12 - B runtime·callbackasm1(SB) - MOVW $1491, R12 - B runtime·callbackasm1(SB) - MOVW $1492, R12 - B runtime·callbackasm1(SB) - MOVW $1493, R12 - B runtime·callbackasm1(SB) - MOVW $1494, R12 - B runtime·callbackasm1(SB) - MOVW $1495, R12 - B runtime·callbackasm1(SB) - MOVW $1496, R12 - B runtime·callbackasm1(SB) - MOVW $1497, R12 - B runtime·callbackasm1(SB) - MOVW $1498, R12 - B runtime·callbackasm1(SB) - MOVW $1499, R12 - B runtime·callbackasm1(SB) - MOVW $1500, R12 - B runtime·callbackasm1(SB) - MOVW $1501, R12 - B runtime·callbackasm1(SB) - MOVW $1502, R12 - B runtime·callbackasm1(SB) - MOVW $1503, R12 - B runtime·callbackasm1(SB) - MOVW $1504, R12 - B runtime·callbackasm1(SB) - MOVW $1505, R12 - B runtime·callbackasm1(SB) - MOVW $1506, R12 - B runtime·callbackasm1(SB) - MOVW $1507, R12 - B runtime·callbackasm1(SB) - MOVW $1508, R12 - B runtime·callbackasm1(SB) - MOVW $1509, R12 - B runtime·callbackasm1(SB) - MOVW $1510, R12 - B runtime·callbackasm1(SB) - MOVW $1511, R12 - B runtime·callbackasm1(SB) - MOVW $1512, R12 - B runtime·callbackasm1(SB) - MOVW $1513, R12 - B runtime·callbackasm1(SB) - MOVW $1514, R12 - B runtime·callbackasm1(SB) - MOVW $1515, R12 - B runtime·callbackasm1(SB) - MOVW $1516, R12 - B runtime·callbackasm1(SB) - MOVW $1517, R12 - B runtime·callbackasm1(SB) - MOVW $1518, R12 - B runtime·callbackasm1(SB) - MOVW $1519, R12 - B runtime·callbackasm1(SB) - MOVW $1520, R12 - B runtime·callbackasm1(SB) - MOVW $1521, R12 - B runtime·callbackasm1(SB) - MOVW $1522, R12 - B runtime·callbackasm1(SB) - MOVW $1523, R12 - B runtime·callbackasm1(SB) - MOVW $1524, R12 - B runtime·callbackasm1(SB) - MOVW $1525, R12 - B runtime·callbackasm1(SB) - MOVW $1526, R12 - B runtime·callbackasm1(SB) - MOVW $1527, R12 - B runtime·callbackasm1(SB) - MOVW $1528, R12 - B runtime·callbackasm1(SB) - MOVW $1529, R12 - B runtime·callbackasm1(SB) - MOVW $1530, R12 - B runtime·callbackasm1(SB) - MOVW $1531, R12 - B runtime·callbackasm1(SB) - MOVW $1532, R12 - B runtime·callbackasm1(SB) - MOVW $1533, R12 - B runtime·callbackasm1(SB) - MOVW $1534, R12 - B runtime·callbackasm1(SB) - MOVW $1535, R12 - B runtime·callbackasm1(SB) - MOVW $1536, R12 - B runtime·callbackasm1(SB) - MOVW $1537, R12 - B runtime·callbackasm1(SB) - MOVW $1538, R12 - B runtime·callbackasm1(SB) - MOVW $1539, R12 - B runtime·callbackasm1(SB) - MOVW $1540, R12 - B runtime·callbackasm1(SB) - MOVW $1541, R12 - B runtime·callbackasm1(SB) - MOVW $1542, R12 - B runtime·callbackasm1(SB) - MOVW $1543, R12 - B runtime·callbackasm1(SB) - MOVW $1544, R12 - B runtime·callbackasm1(SB) - MOVW $1545, R12 - B runtime·callbackasm1(SB) - MOVW $1546, R12 - B runtime·callbackasm1(SB) - MOVW $1547, R12 - B runtime·callbackasm1(SB) - MOVW $1548, R12 - B runtime·callbackasm1(SB) - MOVW $1549, R12 - B runtime·callbackasm1(SB) - MOVW $1550, R12 - B runtime·callbackasm1(SB) - MOVW $1551, R12 - B runtime·callbackasm1(SB) - MOVW $1552, R12 - B runtime·callbackasm1(SB) - MOVW $1553, R12 - B runtime·callbackasm1(SB) - MOVW $1554, R12 - B runtime·callbackasm1(SB) - MOVW $1555, R12 - B runtime·callbackasm1(SB) - MOVW $1556, R12 - B runtime·callbackasm1(SB) - MOVW $1557, R12 - B runtime·callbackasm1(SB) - MOVW $1558, R12 - B runtime·callbackasm1(SB) - MOVW $1559, R12 - B runtime·callbackasm1(SB) - MOVW $1560, R12 - B runtime·callbackasm1(SB) - MOVW $1561, R12 - B runtime·callbackasm1(SB) - MOVW $1562, R12 - B runtime·callbackasm1(SB) - MOVW $1563, R12 - B runtime·callbackasm1(SB) - MOVW $1564, R12 - B runtime·callbackasm1(SB) - MOVW $1565, R12 - B runtime·callbackasm1(SB) - MOVW $1566, R12 - B runtime·callbackasm1(SB) - MOVW $1567, R12 - B runtime·callbackasm1(SB) - MOVW $1568, R12 - B runtime·callbackasm1(SB) - MOVW $1569, R12 - B runtime·callbackasm1(SB) - MOVW $1570, R12 - B runtime·callbackasm1(SB) - MOVW $1571, R12 - B runtime·callbackasm1(SB) - MOVW $1572, R12 - B runtime·callbackasm1(SB) - MOVW $1573, R12 - B runtime·callbackasm1(SB) - MOVW $1574, R12 - B runtime·callbackasm1(SB) - MOVW $1575, R12 - B runtime·callbackasm1(SB) - MOVW $1576, R12 - B runtime·callbackasm1(SB) - MOVW $1577, R12 - B runtime·callbackasm1(SB) - MOVW $1578, R12 - B runtime·callbackasm1(SB) - MOVW $1579, R12 - B runtime·callbackasm1(SB) - MOVW $1580, R12 - B runtime·callbackasm1(SB) - MOVW $1581, R12 - B runtime·callbackasm1(SB) - MOVW $1582, R12 - B runtime·callbackasm1(SB) - MOVW $1583, R12 - B runtime·callbackasm1(SB) - MOVW $1584, R12 - B runtime·callbackasm1(SB) - MOVW $1585, R12 - B runtime·callbackasm1(SB) - MOVW $1586, R12 - B runtime·callbackasm1(SB) - MOVW $1587, R12 - B runtime·callbackasm1(SB) - MOVW $1588, R12 - B runtime·callbackasm1(SB) - MOVW $1589, R12 - B runtime·callbackasm1(SB) - MOVW $1590, R12 - B runtime·callbackasm1(SB) - MOVW $1591, R12 - B runtime·callbackasm1(SB) - MOVW $1592, R12 - B runtime·callbackasm1(SB) - MOVW $1593, R12 - B runtime·callbackasm1(SB) - MOVW $1594, R12 - B runtime·callbackasm1(SB) - MOVW $1595, R12 - B runtime·callbackasm1(SB) - MOVW $1596, R12 - B runtime·callbackasm1(SB) - MOVW $1597, R12 - B runtime·callbackasm1(SB) - MOVW $1598, R12 - B runtime·callbackasm1(SB) - MOVW $1599, R12 - B runtime·callbackasm1(SB) - MOVW $1600, R12 - B runtime·callbackasm1(SB) - MOVW $1601, R12 - B runtime·callbackasm1(SB) - MOVW $1602, R12 - B runtime·callbackasm1(SB) - MOVW $1603, R12 - B runtime·callbackasm1(SB) - MOVW $1604, R12 - B runtime·callbackasm1(SB) - MOVW $1605, R12 - B runtime·callbackasm1(SB) - MOVW $1606, R12 - B runtime·callbackasm1(SB) - MOVW $1607, R12 - B runtime·callbackasm1(SB) - MOVW $1608, R12 - B runtime·callbackasm1(SB) - MOVW $1609, R12 - B runtime·callbackasm1(SB) - MOVW $1610, R12 - B runtime·callbackasm1(SB) - MOVW $1611, R12 - B runtime·callbackasm1(SB) - MOVW $1612, R12 - B runtime·callbackasm1(SB) - MOVW $1613, R12 - B runtime·callbackasm1(SB) - MOVW $1614, R12 - B runtime·callbackasm1(SB) - MOVW $1615, R12 - B runtime·callbackasm1(SB) - MOVW $1616, R12 - B runtime·callbackasm1(SB) - MOVW $1617, R12 - B runtime·callbackasm1(SB) - MOVW $1618, R12 - B runtime·callbackasm1(SB) - MOVW $1619, R12 - B runtime·callbackasm1(SB) - MOVW $1620, R12 - B runtime·callbackasm1(SB) - MOVW $1621, R12 - B runtime·callbackasm1(SB) - MOVW $1622, R12 - B runtime·callbackasm1(SB) - MOVW $1623, R12 - B runtime·callbackasm1(SB) - MOVW $1624, R12 - B runtime·callbackasm1(SB) - MOVW $1625, R12 - B runtime·callbackasm1(SB) - MOVW $1626, R12 - B runtime·callbackasm1(SB) - MOVW $1627, R12 - B runtime·callbackasm1(SB) - MOVW $1628, R12 - B runtime·callbackasm1(SB) - MOVW $1629, R12 - B runtime·callbackasm1(SB) - MOVW $1630, R12 - B runtime·callbackasm1(SB) - MOVW $1631, R12 - B runtime·callbackasm1(SB) - MOVW $1632, R12 - B runtime·callbackasm1(SB) - MOVW $1633, R12 - B runtime·callbackasm1(SB) - MOVW $1634, R12 - B runtime·callbackasm1(SB) - MOVW $1635, R12 - B runtime·callbackasm1(SB) - MOVW $1636, R12 - B runtime·callbackasm1(SB) - MOVW $1637, R12 - B runtime·callbackasm1(SB) - MOVW $1638, R12 - B runtime·callbackasm1(SB) - MOVW $1639, R12 - B runtime·callbackasm1(SB) - MOVW $1640, R12 - B runtime·callbackasm1(SB) - MOVW $1641, R12 - B runtime·callbackasm1(SB) - MOVW $1642, R12 - B runtime·callbackasm1(SB) - MOVW $1643, R12 - B runtime·callbackasm1(SB) - MOVW $1644, R12 - B runtime·callbackasm1(SB) - MOVW $1645, R12 - B runtime·callbackasm1(SB) - MOVW $1646, R12 - B runtime·callbackasm1(SB) - MOVW $1647, R12 - B runtime·callbackasm1(SB) - MOVW $1648, R12 - B runtime·callbackasm1(SB) - MOVW $1649, R12 - B runtime·callbackasm1(SB) - MOVW $1650, R12 - B runtime·callbackasm1(SB) - MOVW $1651, R12 - B runtime·callbackasm1(SB) - MOVW $1652, R12 - B runtime·callbackasm1(SB) - MOVW $1653, R12 - B runtime·callbackasm1(SB) - MOVW $1654, R12 - B runtime·callbackasm1(SB) - MOVW $1655, R12 - B runtime·callbackasm1(SB) - MOVW $1656, R12 - B runtime·callbackasm1(SB) - MOVW $1657, R12 - B runtime·callbackasm1(SB) - MOVW $1658, R12 - B runtime·callbackasm1(SB) - MOVW $1659, R12 - B runtime·callbackasm1(SB) - MOVW $1660, R12 - B runtime·callbackasm1(SB) - MOVW $1661, R12 - B runtime·callbackasm1(SB) - MOVW $1662, R12 - B runtime·callbackasm1(SB) - MOVW $1663, R12 - B runtime·callbackasm1(SB) - MOVW $1664, R12 - B runtime·callbackasm1(SB) - MOVW $1665, R12 - B runtime·callbackasm1(SB) - MOVW $1666, R12 - B runtime·callbackasm1(SB) - MOVW $1667, R12 - B runtime·callbackasm1(SB) - MOVW $1668, R12 - B runtime·callbackasm1(SB) - MOVW $1669, R12 - B runtime·callbackasm1(SB) - MOVW $1670, R12 - B runtime·callbackasm1(SB) - MOVW $1671, R12 - B runtime·callbackasm1(SB) - MOVW $1672, R12 - B runtime·callbackasm1(SB) - MOVW $1673, R12 - B runtime·callbackasm1(SB) - MOVW $1674, R12 - B runtime·callbackasm1(SB) - MOVW $1675, R12 - B runtime·callbackasm1(SB) - MOVW $1676, R12 - B runtime·callbackasm1(SB) - MOVW $1677, R12 - B runtime·callbackasm1(SB) - MOVW $1678, R12 - B runtime·callbackasm1(SB) - MOVW $1679, R12 - B runtime·callbackasm1(SB) - MOVW $1680, R12 - B runtime·callbackasm1(SB) - MOVW $1681, R12 - B runtime·callbackasm1(SB) - MOVW $1682, R12 - B runtime·callbackasm1(SB) - MOVW $1683, R12 - B runtime·callbackasm1(SB) - MOVW $1684, R12 - B runtime·callbackasm1(SB) - MOVW $1685, R12 - B runtime·callbackasm1(SB) - MOVW $1686, R12 - B runtime·callbackasm1(SB) - MOVW $1687, R12 - B runtime·callbackasm1(SB) - MOVW $1688, R12 - B runtime·callbackasm1(SB) - MOVW $1689, R12 - B runtime·callbackasm1(SB) - MOVW $1690, R12 - B runtime·callbackasm1(SB) - MOVW $1691, R12 - B runtime·callbackasm1(SB) - MOVW $1692, R12 - B runtime·callbackasm1(SB) - MOVW $1693, R12 - B runtime·callbackasm1(SB) - MOVW $1694, R12 - B runtime·callbackasm1(SB) - MOVW $1695, R12 - B runtime·callbackasm1(SB) - MOVW $1696, R12 - B runtime·callbackasm1(SB) - MOVW $1697, R12 - B runtime·callbackasm1(SB) - MOVW $1698, R12 - B runtime·callbackasm1(SB) - MOVW $1699, R12 - B runtime·callbackasm1(SB) - MOVW $1700, R12 - B runtime·callbackasm1(SB) - MOVW $1701, R12 - B runtime·callbackasm1(SB) - MOVW $1702, R12 - B runtime·callbackasm1(SB) - MOVW $1703, R12 - B runtime·callbackasm1(SB) - MOVW $1704, R12 - B runtime·callbackasm1(SB) - MOVW $1705, R12 - B runtime·callbackasm1(SB) - MOVW $1706, R12 - B runtime·callbackasm1(SB) - MOVW $1707, R12 - B runtime·callbackasm1(SB) - MOVW $1708, R12 - B runtime·callbackasm1(SB) - MOVW $1709, R12 - B runtime·callbackasm1(SB) - MOVW $1710, R12 - B runtime·callbackasm1(SB) - MOVW $1711, R12 - B runtime·callbackasm1(SB) - MOVW $1712, R12 - B runtime·callbackasm1(SB) - MOVW $1713, R12 - B runtime·callbackasm1(SB) - MOVW $1714, R12 - B runtime·callbackasm1(SB) - MOVW $1715, R12 - B runtime·callbackasm1(SB) - MOVW $1716, R12 - B runtime·callbackasm1(SB) - MOVW $1717, R12 - B runtime·callbackasm1(SB) - MOVW $1718, R12 - B runtime·callbackasm1(SB) - MOVW $1719, R12 - B runtime·callbackasm1(SB) - MOVW $1720, R12 - B runtime·callbackasm1(SB) - MOVW $1721, R12 - B runtime·callbackasm1(SB) - MOVW $1722, R12 - B runtime·callbackasm1(SB) - MOVW $1723, R12 - B runtime·callbackasm1(SB) - MOVW $1724, R12 - B runtime·callbackasm1(SB) - MOVW $1725, R12 - B runtime·callbackasm1(SB) - MOVW $1726, R12 - B runtime·callbackasm1(SB) - MOVW $1727, R12 - B runtime·callbackasm1(SB) - MOVW $1728, R12 - B runtime·callbackasm1(SB) - MOVW $1729, R12 - B runtime·callbackasm1(SB) - MOVW $1730, R12 - B runtime·callbackasm1(SB) - MOVW $1731, R12 - B runtime·callbackasm1(SB) - MOVW $1732, R12 - B runtime·callbackasm1(SB) - MOVW $1733, R12 - B runtime·callbackasm1(SB) - MOVW $1734, R12 - B runtime·callbackasm1(SB) - MOVW $1735, R12 - B runtime·callbackasm1(SB) - MOVW $1736, R12 - B runtime·callbackasm1(SB) - MOVW $1737, R12 - B runtime·callbackasm1(SB) - MOVW $1738, R12 - B runtime·callbackasm1(SB) - MOVW $1739, R12 - B runtime·callbackasm1(SB) - MOVW $1740, R12 - B runtime·callbackasm1(SB) - MOVW $1741, R12 - B runtime·callbackasm1(SB) - MOVW $1742, R12 - B runtime·callbackasm1(SB) - MOVW $1743, R12 - B runtime·callbackasm1(SB) - MOVW $1744, R12 - B runtime·callbackasm1(SB) - MOVW $1745, R12 - B runtime·callbackasm1(SB) - MOVW $1746, R12 - B runtime·callbackasm1(SB) - MOVW $1747, R12 - B runtime·callbackasm1(SB) - MOVW $1748, R12 - B runtime·callbackasm1(SB) - MOVW $1749, R12 - B runtime·callbackasm1(SB) - MOVW $1750, R12 - B runtime·callbackasm1(SB) - MOVW $1751, R12 - B runtime·callbackasm1(SB) - MOVW $1752, R12 - B runtime·callbackasm1(SB) - MOVW $1753, R12 - B runtime·callbackasm1(SB) - MOVW $1754, R12 - B runtime·callbackasm1(SB) - MOVW $1755, R12 - B runtime·callbackasm1(SB) - MOVW $1756, R12 - B runtime·callbackasm1(SB) - MOVW $1757, R12 - B runtime·callbackasm1(SB) - MOVW $1758, R12 - B runtime·callbackasm1(SB) - MOVW $1759, R12 - B runtime·callbackasm1(SB) - MOVW $1760, R12 - B runtime·callbackasm1(SB) - MOVW $1761, R12 - B runtime·callbackasm1(SB) - MOVW $1762, R12 - B runtime·callbackasm1(SB) - MOVW $1763, R12 - B runtime·callbackasm1(SB) - MOVW $1764, R12 - B runtime·callbackasm1(SB) - MOVW $1765, R12 - B runtime·callbackasm1(SB) - MOVW $1766, R12 - B runtime·callbackasm1(SB) - MOVW $1767, R12 - B runtime·callbackasm1(SB) - MOVW $1768, R12 - B runtime·callbackasm1(SB) - MOVW $1769, R12 - B runtime·callbackasm1(SB) - MOVW $1770, R12 - B runtime·callbackasm1(SB) - MOVW $1771, R12 - B runtime·callbackasm1(SB) - MOVW $1772, R12 - B runtime·callbackasm1(SB) - MOVW $1773, R12 - B runtime·callbackasm1(SB) - MOVW $1774, R12 - B runtime·callbackasm1(SB) - MOVW $1775, R12 - B runtime·callbackasm1(SB) - MOVW $1776, R12 - B runtime·callbackasm1(SB) - MOVW $1777, R12 - B runtime·callbackasm1(SB) - MOVW $1778, R12 - B runtime·callbackasm1(SB) - MOVW $1779, R12 - B runtime·callbackasm1(SB) - MOVW $1780, R12 - B runtime·callbackasm1(SB) - MOVW $1781, R12 - B runtime·callbackasm1(SB) - MOVW $1782, R12 - B runtime·callbackasm1(SB) - MOVW $1783, R12 - B runtime·callbackasm1(SB) - MOVW $1784, R12 - B runtime·callbackasm1(SB) - MOVW $1785, R12 - B runtime·callbackasm1(SB) - MOVW $1786, R12 - B runtime·callbackasm1(SB) - MOVW $1787, R12 - B runtime·callbackasm1(SB) - MOVW $1788, R12 - B runtime·callbackasm1(SB) - MOVW $1789, R12 - B runtime·callbackasm1(SB) - MOVW $1790, R12 - B runtime·callbackasm1(SB) - MOVW $1791, R12 - B runtime·callbackasm1(SB) - MOVW $1792, R12 - B runtime·callbackasm1(SB) - MOVW $1793, R12 - B runtime·callbackasm1(SB) - MOVW $1794, R12 - B runtime·callbackasm1(SB) - MOVW $1795, R12 - B runtime·callbackasm1(SB) - MOVW $1796, R12 - B runtime·callbackasm1(SB) - MOVW $1797, R12 - B runtime·callbackasm1(SB) - MOVW $1798, R12 - B runtime·callbackasm1(SB) - MOVW $1799, R12 - B runtime·callbackasm1(SB) - MOVW $1800, R12 - B runtime·callbackasm1(SB) - MOVW $1801, R12 - B runtime·callbackasm1(SB) - MOVW $1802, R12 - B runtime·callbackasm1(SB) - MOVW $1803, R12 - B runtime·callbackasm1(SB) - MOVW $1804, R12 - B runtime·callbackasm1(SB) - MOVW $1805, R12 - B runtime·callbackasm1(SB) - MOVW $1806, R12 - B runtime·callbackasm1(SB) - MOVW $1807, R12 - B runtime·callbackasm1(SB) - MOVW $1808, R12 - B runtime·callbackasm1(SB) - MOVW $1809, R12 - B runtime·callbackasm1(SB) - MOVW $1810, R12 - B runtime·callbackasm1(SB) - MOVW $1811, R12 - B runtime·callbackasm1(SB) - MOVW $1812, R12 - B runtime·callbackasm1(SB) - MOVW $1813, R12 - B runtime·callbackasm1(SB) - MOVW $1814, R12 - B runtime·callbackasm1(SB) - MOVW $1815, R12 - B runtime·callbackasm1(SB) - MOVW $1816, R12 - B runtime·callbackasm1(SB) - MOVW $1817, R12 - B runtime·callbackasm1(SB) - MOVW $1818, R12 - B runtime·callbackasm1(SB) - MOVW $1819, R12 - B runtime·callbackasm1(SB) - MOVW $1820, R12 - B runtime·callbackasm1(SB) - MOVW $1821, R12 - B runtime·callbackasm1(SB) - MOVW $1822, R12 - B runtime·callbackasm1(SB) - MOVW $1823, R12 - B runtime·callbackasm1(SB) - MOVW $1824, R12 - B runtime·callbackasm1(SB) - MOVW $1825, R12 - B runtime·callbackasm1(SB) - MOVW $1826, R12 - B runtime·callbackasm1(SB) - MOVW $1827, R12 - B runtime·callbackasm1(SB) - MOVW $1828, R12 - B runtime·callbackasm1(SB) - MOVW $1829, R12 - B runtime·callbackasm1(SB) - MOVW $1830, R12 - B runtime·callbackasm1(SB) - MOVW $1831, R12 - B runtime·callbackasm1(SB) - MOVW $1832, R12 - B runtime·callbackasm1(SB) - MOVW $1833, R12 - B runtime·callbackasm1(SB) - MOVW $1834, R12 - B runtime·callbackasm1(SB) - MOVW $1835, R12 - B runtime·callbackasm1(SB) - MOVW $1836, R12 - B runtime·callbackasm1(SB) - MOVW $1837, R12 - B runtime·callbackasm1(SB) - MOVW $1838, R12 - B runtime·callbackasm1(SB) - MOVW $1839, R12 - B runtime·callbackasm1(SB) - MOVW $1840, R12 - B runtime·callbackasm1(SB) - MOVW $1841, R12 - B runtime·callbackasm1(SB) - MOVW $1842, R12 - B runtime·callbackasm1(SB) - MOVW $1843, R12 - B runtime·callbackasm1(SB) - MOVW $1844, R12 - B runtime·callbackasm1(SB) - MOVW $1845, R12 - B runtime·callbackasm1(SB) - MOVW $1846, R12 - B runtime·callbackasm1(SB) - MOVW $1847, R12 - B runtime·callbackasm1(SB) - MOVW $1848, R12 - B runtime·callbackasm1(SB) - MOVW $1849, R12 - B runtime·callbackasm1(SB) - MOVW $1850, R12 - B runtime·callbackasm1(SB) - MOVW $1851, R12 - B runtime·callbackasm1(SB) - MOVW $1852, R12 - B runtime·callbackasm1(SB) - MOVW $1853, R12 - B runtime·callbackasm1(SB) - MOVW $1854, R12 - B runtime·callbackasm1(SB) - MOVW $1855, R12 - B runtime·callbackasm1(SB) - MOVW $1856, R12 - B runtime·callbackasm1(SB) - MOVW $1857, R12 - B runtime·callbackasm1(SB) - MOVW $1858, R12 - B runtime·callbackasm1(SB) - MOVW $1859, R12 - B runtime·callbackasm1(SB) - MOVW $1860, R12 - B runtime·callbackasm1(SB) - MOVW $1861, R12 - B runtime·callbackasm1(SB) - MOVW $1862, R12 - B runtime·callbackasm1(SB) - MOVW $1863, R12 - B runtime·callbackasm1(SB) - MOVW $1864, R12 - B runtime·callbackasm1(SB) - MOVW $1865, R12 - B runtime·callbackasm1(SB) - MOVW $1866, R12 - B runtime·callbackasm1(SB) - MOVW $1867, R12 - B runtime·callbackasm1(SB) - MOVW $1868, R12 - B runtime·callbackasm1(SB) - MOVW $1869, R12 - B runtime·callbackasm1(SB) - MOVW $1870, R12 - B runtime·callbackasm1(SB) - MOVW $1871, R12 - B runtime·callbackasm1(SB) - MOVW $1872, R12 - B runtime·callbackasm1(SB) - MOVW $1873, R12 - B runtime·callbackasm1(SB) - MOVW $1874, R12 - B runtime·callbackasm1(SB) - MOVW $1875, R12 - B runtime·callbackasm1(SB) - MOVW $1876, R12 - B runtime·callbackasm1(SB) - MOVW $1877, R12 - B runtime·callbackasm1(SB) - MOVW $1878, R12 - B runtime·callbackasm1(SB) - MOVW $1879, R12 - B runtime·callbackasm1(SB) - MOVW $1880, R12 - B runtime·callbackasm1(SB) - MOVW $1881, R12 - B runtime·callbackasm1(SB) - MOVW $1882, R12 - B runtime·callbackasm1(SB) - MOVW $1883, R12 - B runtime·callbackasm1(SB) - MOVW $1884, R12 - B runtime·callbackasm1(SB) - MOVW $1885, R12 - B runtime·callbackasm1(SB) - MOVW $1886, R12 - B runtime·callbackasm1(SB) - MOVW $1887, R12 - B runtime·callbackasm1(SB) - MOVW $1888, R12 - B runtime·callbackasm1(SB) - MOVW $1889, R12 - B runtime·callbackasm1(SB) - MOVW $1890, R12 - B runtime·callbackasm1(SB) - MOVW $1891, R12 - B runtime·callbackasm1(SB) - MOVW $1892, R12 - B runtime·callbackasm1(SB) - MOVW $1893, R12 - B runtime·callbackasm1(SB) - MOVW $1894, R12 - B runtime·callbackasm1(SB) - MOVW $1895, R12 - B runtime·callbackasm1(SB) - MOVW $1896, R12 - B runtime·callbackasm1(SB) - MOVW $1897, R12 - B runtime·callbackasm1(SB) - MOVW $1898, R12 - B runtime·callbackasm1(SB) - MOVW $1899, R12 - B runtime·callbackasm1(SB) - MOVW $1900, R12 - B runtime·callbackasm1(SB) - MOVW $1901, R12 - B runtime·callbackasm1(SB) - MOVW $1902, R12 - B runtime·callbackasm1(SB) - MOVW $1903, R12 - B runtime·callbackasm1(SB) - MOVW $1904, R12 - B runtime·callbackasm1(SB) - MOVW $1905, R12 - B runtime·callbackasm1(SB) - MOVW $1906, R12 - B runtime·callbackasm1(SB) - MOVW $1907, R12 - B runtime·callbackasm1(SB) - MOVW $1908, R12 - B runtime·callbackasm1(SB) - MOVW $1909, R12 - B runtime·callbackasm1(SB) - MOVW $1910, R12 - B runtime·callbackasm1(SB) - MOVW $1911, R12 - B runtime·callbackasm1(SB) - MOVW $1912, R12 - B runtime·callbackasm1(SB) - MOVW $1913, R12 - B runtime·callbackasm1(SB) - MOVW $1914, R12 - B runtime·callbackasm1(SB) - MOVW $1915, R12 - B runtime·callbackasm1(SB) - MOVW $1916, R12 - B runtime·callbackasm1(SB) - MOVW $1917, R12 - B runtime·callbackasm1(SB) - MOVW $1918, R12 - B runtime·callbackasm1(SB) - MOVW $1919, R12 - B runtime·callbackasm1(SB) - MOVW $1920, R12 - B runtime·callbackasm1(SB) - MOVW $1921, R12 - B runtime·callbackasm1(SB) - MOVW $1922, R12 - B runtime·callbackasm1(SB) - MOVW $1923, R12 - B runtime·callbackasm1(SB) - MOVW $1924, R12 - B runtime·callbackasm1(SB) - MOVW $1925, R12 - B runtime·callbackasm1(SB) - MOVW $1926, R12 - B runtime·callbackasm1(SB) - MOVW $1927, R12 - B runtime·callbackasm1(SB) - MOVW $1928, R12 - B runtime·callbackasm1(SB) - MOVW $1929, R12 - B runtime·callbackasm1(SB) - MOVW $1930, R12 - B runtime·callbackasm1(SB) - MOVW $1931, R12 - B runtime·callbackasm1(SB) - MOVW $1932, R12 - B runtime·callbackasm1(SB) - MOVW $1933, R12 - B runtime·callbackasm1(SB) - MOVW $1934, R12 - B runtime·callbackasm1(SB) - MOVW $1935, R12 - B runtime·callbackasm1(SB) - MOVW $1936, R12 - B runtime·callbackasm1(SB) - MOVW $1937, R12 - B runtime·callbackasm1(SB) - MOVW $1938, R12 - B runtime·callbackasm1(SB) - MOVW $1939, R12 - B runtime·callbackasm1(SB) - MOVW $1940, R12 - B runtime·callbackasm1(SB) - MOVW $1941, R12 - B runtime·callbackasm1(SB) - MOVW $1942, R12 - B runtime·callbackasm1(SB) - MOVW $1943, R12 - B runtime·callbackasm1(SB) - MOVW $1944, R12 - B runtime·callbackasm1(SB) - MOVW $1945, R12 - B runtime·callbackasm1(SB) - MOVW $1946, R12 - B runtime·callbackasm1(SB) - MOVW $1947, R12 - B runtime·callbackasm1(SB) - MOVW $1948, R12 - B runtime·callbackasm1(SB) - MOVW $1949, R12 - B runtime·callbackasm1(SB) - MOVW $1950, R12 - B runtime·callbackasm1(SB) - MOVW $1951, R12 - B runtime·callbackasm1(SB) - MOVW $1952, R12 - B runtime·callbackasm1(SB) - MOVW $1953, R12 - B runtime·callbackasm1(SB) - MOVW $1954, R12 - B runtime·callbackasm1(SB) - MOVW $1955, R12 - B runtime·callbackasm1(SB) - MOVW $1956, R12 - B runtime·callbackasm1(SB) - MOVW $1957, R12 - B runtime·callbackasm1(SB) - MOVW $1958, R12 - B runtime·callbackasm1(SB) - MOVW $1959, R12 - B runtime·callbackasm1(SB) - MOVW $1960, R12 - B runtime·callbackasm1(SB) - MOVW $1961, R12 - B runtime·callbackasm1(SB) - MOVW $1962, R12 - B runtime·callbackasm1(SB) - MOVW $1963, R12 - B runtime·callbackasm1(SB) - MOVW $1964, R12 - B runtime·callbackasm1(SB) - MOVW $1965, R12 - B runtime·callbackasm1(SB) - MOVW $1966, R12 - B runtime·callbackasm1(SB) - MOVW $1967, R12 - B runtime·callbackasm1(SB) - MOVW $1968, R12 - B runtime·callbackasm1(SB) - MOVW $1969, R12 - B runtime·callbackasm1(SB) - MOVW $1970, R12 - B runtime·callbackasm1(SB) - MOVW $1971, R12 - B runtime·callbackasm1(SB) - MOVW $1972, R12 - B runtime·callbackasm1(SB) - MOVW $1973, R12 - B runtime·callbackasm1(SB) - MOVW $1974, R12 - B runtime·callbackasm1(SB) - MOVW $1975, R12 - B runtime·callbackasm1(SB) - MOVW $1976, R12 - B runtime·callbackasm1(SB) - MOVW $1977, R12 - B runtime·callbackasm1(SB) - MOVW $1978, R12 - B runtime·callbackasm1(SB) - MOVW $1979, R12 - B runtime·callbackasm1(SB) - MOVW $1980, R12 - B runtime·callbackasm1(SB) - MOVW $1981, R12 - B runtime·callbackasm1(SB) - MOVW $1982, R12 - B runtime·callbackasm1(SB) - MOVW $1983, R12 - B runtime·callbackasm1(SB) - MOVW $1984, R12 - B runtime·callbackasm1(SB) - MOVW $1985, R12 - B runtime·callbackasm1(SB) - MOVW $1986, R12 - B runtime·callbackasm1(SB) - MOVW $1987, R12 - B runtime·callbackasm1(SB) - MOVW $1988, R12 - B runtime·callbackasm1(SB) - MOVW $1989, R12 - B runtime·callbackasm1(SB) - MOVW $1990, R12 - B runtime·callbackasm1(SB) - MOVW $1991, R12 - B runtime·callbackasm1(SB) - MOVW $1992, R12 - B runtime·callbackasm1(SB) - MOVW $1993, R12 - B runtime·callbackasm1(SB) - MOVW $1994, R12 - B runtime·callbackasm1(SB) - MOVW $1995, R12 - B runtime·callbackasm1(SB) - MOVW $1996, R12 - B runtime·callbackasm1(SB) - MOVW $1997, R12 - B runtime·callbackasm1(SB) - MOVW $1998, R12 - B runtime·callbackasm1(SB) - MOVW $1999, R12 - B runtime·callbackasm1(SB) diff --git a/src/slices/example_test.go b/src/slices/example_test.go index cb601ada0ade65..f775b88f6b76ca 100644 --- a/src/slices/example_test.go +++ b/src/slices/example_test.go @@ -310,7 +310,7 @@ func ExampleSortStableFunc() { {"Bob", 24}, {"Alice", 55}, } - // Stable sort by name, keeping age ordering of Alices intact + // Stable sort by name, keeping age ordering of Alice intact slices.SortStableFunc(people, func(a, b Person) int { return strings.Compare(a.Name, b.Name) }) @@ -325,9 +325,11 @@ func ExampleClone() { fmt.Println(clone) clone[2] = 10 fmt.Println(numbers) + fmt.Println(clone) // Output: // [0 42 -10 8] // [0 42 -10 8] + // [0 42 10 8] } func ExampleGrow() { @@ -385,6 +387,133 @@ func ExampleRepeat() { // [0 1 2 3 0 1 2 3] } +func ExampleAll() { + names := []string{"Alice", "Bob", "Vera"} + for i, v := range slices.All(names) { + fmt.Println(i, ":", v) + } + // Output: + // 0 : Alice + // 1 : Bob + // 2 : Vera +} + +func ExampleBackward() { + names := []string{"Alice", "Bob", "Vera"} + for i, v := range slices.Backward(names) { + fmt.Println(i, ":", v) + } + // Output: + // 2 : Vera + // 1 : Bob + // 0 : Alice +} + +func ExampleValues() { + names := []string{"Alice", "Bob", "Vera"} + for v := range slices.Values(names) { + fmt.Println(v) + } + // Output: + // Alice + // Bob + // Vera +} + +func ExampleAppendSeq() { + seq := func(yield func(int) bool) { + for i := 0; i < 10; i += 2 { + if !yield(i) { + return + } + } + } + + s := slices.AppendSeq([]int{1, 2}, seq) + fmt.Println(s) + // Output: + // [1 2 0 2 4 6 8] +} + +func ExampleCollect() { + seq := func(yield func(int) bool) { + for i := 0; i < 10; i += 2 { + if !yield(i) { + return + } + } + } + + s := slices.Collect(seq) + fmt.Println(s) + // Output: + // [0 2 4 6 8] +} + +func ExampleSorted() { + seq := func(yield func(int) bool) { + flag := -1 + for i := 0; i < 10; i += 2 { + flag = -flag + if !yield(i * flag) { + return + } + } + } + + s := slices.Sorted(seq) + fmt.Println(s) + fmt.Println(slices.IsSorted(s)) + // Output: + // [-6 -2 0 4 8] + // true +} + +func ExampleSortedFunc() { + seq := func(yield func(int) bool) { + flag := -1 + for i := 0; i < 10; i += 2 { + flag = -flag + if !yield(i * flag) { + return + } + } + } + + sortFunc := func(a, b int) int { + return cmp.Compare(b, a) // the comparison is being done in reverse + } + + s := slices.SortedFunc(seq, sortFunc) + fmt.Println(s) + // Output: + // [8 4 0 -2 -6] +} + +func ExampleSortedStableFunc() { + type Person struct { + Name string + Age int + } + + people := []Person{ + {"Gopher", 13}, + {"Alice", 20}, + {"Bob", 5}, + {"Vera", 24}, + {"Zac", 20}, + } + + sortFunc := func(x, y Person) int { + return cmp.Compare(x.Age, y.Age) + } + + s := slices.SortedStableFunc(slices.Values(people), sortFunc) + fmt.Println(s) + // Output: + // [{Bob 5} {Gopher 13} {Alice 20} {Zac 20} {Vera 24}] +} + func ExampleChunk() { type Person struct { Name string diff --git a/src/slices/iter.go b/src/slices/iter.go index cd8f308ca08ece..bbea6134d9478e 100644 --- a/src/slices/iter.go +++ b/src/slices/iter.go @@ -46,6 +46,7 @@ func Values[Slice ~[]E, E any](s Slice) iter.Seq[E] { // AppendSeq appends the values from seq to the slice and // returns the extended slice. +// If seq is empty, the result preserves the nilness of s. func AppendSeq[Slice ~[]E, E any](s Slice, seq iter.Seq[E]) Slice { for v := range seq { s = append(s, v) @@ -54,12 +55,14 @@ func AppendSeq[Slice ~[]E, E any](s Slice, seq iter.Seq[E]) Slice { } // Collect collects values from seq into a new slice and returns it. +// If seq is empty, the result is nil. func Collect[E any](seq iter.Seq[E]) []E { return AppendSeq([]E(nil), seq) } // Sorted collects values from seq into a new slice, sorts the slice, // and returns it. +// If seq is empty, the result is nil. func Sorted[E cmp.Ordered](seq iter.Seq[E]) []E { s := Collect(seq) Sort(s) @@ -68,6 +71,7 @@ func Sorted[E cmp.Ordered](seq iter.Seq[E]) []E { // SortedFunc collects values from seq into a new slice, sorts the slice // using the comparison function, and returns it. +// If seq is empty, the result is nil. func SortedFunc[E any](seq iter.Seq[E], cmp func(E, E) int) []E { s := Collect(seq) SortFunc(s, cmp) @@ -78,6 +82,7 @@ func SortedFunc[E any](seq iter.Seq[E], cmp func(E, E) int) []E { // It then sorts the slice while keeping the original order of equal elements, // using the comparison function to compare elements. // It returns the new slice. +// If seq is empty, the result is nil. func SortedStableFunc[E any](seq iter.Seq[E], cmp func(E, E) int) []E { s := Collect(seq) SortStableFunc(s, cmp) diff --git a/src/slices/slices.go b/src/slices/slices.go index b3cd4e2c054001..30595793c962de 100644 --- a/src/slices/slices.go +++ b/src/slices/slices.go @@ -128,9 +128,10 @@ func ContainsFunc[S ~[]E, E any](s S, f func(E) bool) bool { // returning the modified slice. // The elements at s[i:] are shifted up to make room. // In the returned slice r, r[i] == v[0], -// and r[i+len(v)] == value originally at r[i]. -// Insert panics if i is out of range. +// and, if i < len(s), r[i+len(v)] == value originally at r[i]. +// Insert panics if i > len(s). // This function is O(len(s) + len(v)). +// If the result is empty, it has the same nilness as s. func Insert[S ~[]E, E any](s S, i int, v ...E) S { _ = s[i:] // bounds check @@ -217,6 +218,7 @@ func Insert[S ~[]E, E any](s S, i int, v ...E) S { // Delete is O(len(s)-i), so if many items must be deleted, it is better to // make a single call deleting them all together than to delete one at a time. // Delete zeroes the elements s[len(s)-(j-i):len(s)]. +// If the result is empty, it has the same nilness as s. func Delete[S ~[]E, E any](s S, i, j int) S { _ = s[i:j:len(s)] // bounds check @@ -233,6 +235,7 @@ func Delete[S ~[]E, E any](s S, i, j int) S { // DeleteFunc removes any elements from s for which del returns true, // returning the modified slice. // DeleteFunc zeroes the elements between the new length and the original length. +// If the result is empty, it has the same nilness as s. func DeleteFunc[S ~[]E, E any](s S, del func(E) bool) S { i := IndexFunc(s, del) if i == -1 { @@ -253,6 +256,7 @@ func DeleteFunc[S ~[]E, E any](s S, del func(E) bool) S { // modified slice. // Replace panics if j > len(s) or s[i:j] is not a valid slice of s. // When len(v) < (j-i), Replace zeroes the elements between the new length and the original length. +// If the result is empty, it has the same nilness as s. func Replace[S ~[]E, E any](s S, i, j int, v ...E) S { _ = s[i:j] // bounds check @@ -345,9 +349,15 @@ func Replace[S ~[]E, E any](s S, i, j int, v ...E) S { // Clone returns a copy of the slice. // The elements are copied using assignment, so this is a shallow clone. // The result may have additional unused capacity. +// The result preserves the nilness of s. func Clone[S ~[]E, E any](s S) S { - // The s[:0:0] preserves nil in case it matters. - return append(s[:0:0], s...) + // Preserve nilness in case it matters. + if s == nil { + return nil + } + // Avoid s[:0:0] as it leads to unwanted liveness when cloning a + // zero-length slice of a large array; see https://go.dev/issue/68488. + return append(S{}, s...) } // Compact replaces consecutive runs of equal elements with a single copy. @@ -355,6 +365,7 @@ func Clone[S ~[]E, E any](s S) S { // Compact modifies the contents of the slice s and returns the modified slice, // which may have a smaller length. // Compact zeroes the elements between the new length and the original length. +// The result preserves the nilness of s. func Compact[S ~[]E, E comparable](s S) S { if len(s) < 2 { return s @@ -379,6 +390,7 @@ func Compact[S ~[]E, E comparable](s S) S { // CompactFunc is like [Compact] but uses an equality function to compare elements. // For runs of elements that compare equal, CompactFunc keeps the first one. // CompactFunc zeroes the elements between the new length and the original length. +// The result preserves the nilness of s. func CompactFunc[S ~[]E, E any](s S, eq func(E, E) bool) S { if len(s) < 2 { return s @@ -404,17 +416,20 @@ func CompactFunc[S ~[]E, E any](s S, eq func(E, E) bool) S { // another n elements. After Grow(n), at least n elements can be appended // to the slice without another allocation. If n is negative or too large to // allocate the memory, Grow panics. +// The result preserves the nilness of s. func Grow[S ~[]E, E any](s S, n int) S { if n < 0 { panic("cannot be negative") } if n -= cap(s) - len(s); n > 0 { + // This expression allocates only once (see test). s = append(s[:cap(s)], make([]E, n)...)[:len(s)] } return s } // Clip removes unused capacity from the slice, returning s[:len(s):len(s)]. +// The result preserves the nilness of s. func Clip[S ~[]E, E any](s S) S { return s[:len(s):len(s)] } @@ -434,7 +449,7 @@ func rotateRight[E any](s []E, r int) { rotateLeft(s, len(s)-r) } -// overlaps reports whether the memory ranges a[0:len(a)] and b[0:len(b)] overlap. +// overlaps reports whether the memory ranges a[:len(a)] and b[:len(b)] overlap. func overlaps[E any](a, b []E) bool { if len(a) == 0 || len(b) == 0 { return false @@ -444,7 +459,7 @@ func overlaps[E any](a, b []E) bool { return false } // TODO: use a runtime/unsafe facility once one becomes available. See issue 12445. - // Also see crypto/internal/alias/alias.go:AnyOverlap + // Also see crypto/internal/fips140/alias/alias.go:AnyOverlap return uintptr(unsafe.Pointer(&a[0])) <= uintptr(unsafe.Pointer(&b[len(b)-1]))+(elemSize-1) && uintptr(unsafe.Pointer(&b[0])) <= uintptr(unsafe.Pointer(&a[len(a)-1]))+(elemSize-1) } @@ -470,6 +485,7 @@ func Reverse[S ~[]E, E any](s S) { } // Concat returns a new slice concatenating the passed in slices. +// If the concatenation is empty, the result is nil. func Concat[S ~[]E, E any](slices ...S) S { size := 0 for _, s := range slices { @@ -478,6 +494,9 @@ func Concat[S ~[]E, E any](slices ...S) S { panic("len out of range") } } + // Use Grow, not make, to round up to the size class: + // the extra space is otherwise unused and helps + // callers that append a few elements to the result. newslice := Grow[S](nil, size) for _, s := range slices { newslice = append(newslice, s...) @@ -496,11 +515,12 @@ func Repeat[S ~[]E, E any](x S, count int) S { } const maxInt = ^uint(0) >> 1 - if hi, lo := bits.Mul(uint(len(x)), uint(count)); hi > 0 || lo > maxInt { + hi, lo := bits.Mul(uint(len(x)), uint(count)) + if hi > 0 || lo > maxInt { panic("the result of (len(x) * count) overflows") } - newslice := make(S, len(x)*count) + newslice := make(S, int(lo)) // lo = len(x) * count n := copy(newslice, x) for n < len(newslice) { n += copy(newslice[n:], newslice[:n]) diff --git a/src/slices/slices_test.go b/src/slices/slices_test.go index 68c8a3adc252f0..19a3e9b0ddbdb7 100644 --- a/src/slices/slices_test.go +++ b/src/slices/slices_test.go @@ -6,12 +6,15 @@ package slices_test import ( "cmp" + "internal/asan" + "internal/msan" "internal/race" "internal/testenv" "math" . "slices" "strings" "testing" + "unsafe" ) var equalIntTests = []struct { @@ -496,7 +499,7 @@ func TestInsert(t *testing.T) { } } - if !testenv.OptimizationOff() && !race.Enabled { + if !testenv.OptimizationOff() && !race.Enabled && !asan.Enabled && !msan.Enabled { // Allocations should be amortized. const count = 50 n := testing.AllocsPerRun(10, func() { @@ -952,7 +955,7 @@ func TestGrow(t *testing.T) { } if n := testing.AllocsPerRun(100, func() { _ = Grow(s2, cap(s2)-len(s2)+1) }); n != 1 { errorf := t.Errorf - if race.Enabled || testenv.OptimizationOff() { + if race.Enabled || msan.Enabled || asan.Enabled || testenv.OptimizationOff() { errorf = t.Logf // this allocates multiple times in race detector mode } errorf("Grow should allocate once when given insufficient capacity; allocated %v times", n) @@ -1009,7 +1012,7 @@ func TestReverse(t *testing.T) { singleton := []string{"one"} Reverse(singleton) if want := []string{"one"}; !Equal(singleton, want) { - t.Errorf("Reverse(singeleton) = %v, want %v", singleton, want) + t.Errorf("Reverse(singleton) = %v, want %v", singleton, want) } Reverse[[]string](nil) @@ -1313,7 +1316,7 @@ func TestConcat(t *testing.T) { _ = sink if allocs > 1 { errorf := t.Errorf - if testenv.OptimizationOff() || race.Enabled { + if testenv.OptimizationOff() || race.Enabled || asan.Enabled || msan.Enabled { errorf = t.Logf } errorf("Concat(%v) allocated %v times; want 1", tc.s, allocs) @@ -1450,3 +1453,98 @@ func TestRepeatPanics(t *testing.T) { } } } + +var leak *int + +func TestIssue68488(t *testing.T) { + s := make([]int, 3) + clone := Clone(s[1:1]) + switch unsafe.SliceData(clone) { + case &s[0], &s[1], &s[2]: + t.Error("clone keeps alive s due to array overlap") + } + leak = &s[1] // see go.dev/issue/74387 +} + +// This test asserts the behavior when the primary slice operand is nil. +// +// Some operations preserve the nilness of their operand while others +// do not, but in all cases the behavior is documented. +func TestNilness(t *testing.T) { + var ( + emptySlice = []int{} + nilSlice = []int(nil) + emptySeq = func(yield func(int) bool) {} + truth = func(int) bool { return true } + equal = func(x, y int) bool { panic("unreachable") } + ) + + wantNil := func(slice []int, cond string) { + if slice != nil { + t.Errorf("%s != nil", cond) + } + } + wantNonNil := func(slice []int, cond string) { + if slice == nil { + t.Errorf("%s == nil", cond) + } + } + + // The update functions + // Insert, AppendSeq, Delete, DeleteFunc, Clone, Compact, CompactFunc + // preserve nilness, like s[i:j]. + wantNil(AppendSeq(nilSlice, emptySeq), "AppendSeq(nil, empty)") + wantNonNil(AppendSeq(emptySlice, emptySeq), "AppendSeq(nil, empty)") + + wantNil(Insert(nilSlice, 0), "Insert(nil, 0)") + wantNonNil(Insert(emptySlice, 0), "Insert(empty, 0)") + + wantNil(Delete(nilSlice, 0, 0), "Delete(nil, 0, 0)") + wantNonNil(Delete(emptySlice, 0, 0), "Delete(empty, 0, 0)") + wantNonNil(Delete([]int{1}, 0, 1), "Delete([]int{1}, 0, 1)") + + wantNil(DeleteFunc(nilSlice, truth), "DeleteFunc(nil, f)") + wantNonNil(DeleteFunc(emptySlice, truth), "DeleteFunc(empty, f)") + wantNonNil(DeleteFunc([]int{1}, truth), "DeleteFunc([]int{1}, truth)") + + wantNil(Replace(nilSlice, 0, 0), "Replace(nil, 0, 0)") + wantNonNil(Replace(emptySlice, 0, 0), "Replace(empty, 0, 0)") + wantNonNil(Replace([]int{1}, 0, 1), "Replace([]int{1}, 0, 1)") + + wantNil(Clone(nilSlice), "Clone(nil)") + wantNonNil(Clone(emptySlice), "Clone(empty)") + + wantNil(Compact(nilSlice), "Compact(nil)") + wantNonNil(Compact(emptySlice), "Compact(empty)") + + wantNil(CompactFunc(nilSlice, equal), "CompactFunc(nil)") + wantNonNil(CompactFunc(emptySlice, equal), "CompactFunc(empty)") + + wantNil(Grow(nilSlice, 0), "Grow(nil, 0)") + wantNonNil(Grow(emptySlice, 0), "Grow(empty, 0)") + + wantNil(Clip(nilSlice), "Clip(nil)") + wantNonNil(Clip(emptySlice), "Clip(empty)") + wantNonNil(Clip([]int{1}[:0:0]), "Clip([]int{1}[:0:0])") + + // Concat returns nil iff the result is empty. + // This is an unfortunate irregularity. + wantNil(Concat(nilSlice, emptySlice, nilSlice, emptySlice), "Concat(nil, ...empty...)") + wantNil(Concat(emptySlice, emptySlice, nilSlice, emptySlice), "Concat(empty, ...empty...)") + wantNil(Concat[[]int](), "Concat()") + + // Repeat never returns nil. Another irregularity. + wantNonNil(Repeat(nilSlice, 0), "Repeat(nil, 0)") + wantNonNil(Repeat(emptySlice, 0), "Repeat(empty, 0)") + wantNonNil(Repeat(nilSlice, 2), "Repeat(nil, 2)") + wantNonNil(Repeat(emptySlice, 2), "Repeat(empty, 2)") + + // The collection functions + // Collect, Sorted, SortedFunc, SortedStableFunc + // return nil given an empty sequence. + wantNil(Collect(emptySeq), "Collect(empty)") + + wantNil(Sorted(emptySeq), "Sorted(empty)") + wantNil(SortedFunc(emptySeq, cmp.Compare), "SortedFunc(empty)") + wantNil(SortedStableFunc(emptySeq, cmp.Compare), "SortedStableFunc(empty)") +} diff --git a/src/slices/sort.go b/src/slices/sort.go index f713ffe09473a7..4f66e7bb3952f2 100644 --- a/src/slices/sort.go +++ b/src/slices/sort.go @@ -180,8 +180,8 @@ type xorshift uint64 func (r *xorshift) Next() uint64 { *r ^= *r << 13 - *r ^= *r >> 17 - *r ^= *r << 5 + *r ^= *r >> 7 + *r ^= *r << 17 return uint64(*r) } diff --git a/src/slices/sort_benchmark_test.go b/src/slices/sort_benchmark_test.go index 1dde26ef1c1bc4..cafb1a46187af4 100644 --- a/src/slices/sort_benchmark_test.go +++ b/src/slices/sort_benchmark_test.go @@ -23,7 +23,7 @@ func BenchmarkBinarySearchFloats(b *testing.B) { needle := (floats[midpoint] + floats[midpoint+1]) / 2 b.ResetTimer() for i := 0; i < b.N; i++ { - slices.BinarySearch(floats, needle) + _, _ = slices.BinarySearch(floats, needle) } }) } @@ -46,7 +46,7 @@ func BenchmarkBinarySearchFuncStruct(b *testing.B) { cmpFunc := func(a, b *myStruct) int { return a.n - b.n } b.ResetTimer() for i := 0; i < b.N; i++ { - slices.BinarySearchFunc(structs, needle, cmpFunc) + _, _ = slices.BinarySearchFunc(structs, needle, cmpFunc) } }) } diff --git a/src/slices/sort_test.go b/src/slices/sort_test.go index 2e045e2af88939..855f861c514bd5 100644 --- a/src/slices/sort_test.go +++ b/src/slices/sort_test.go @@ -264,19 +264,19 @@ func TestMinMaxPanics(t *testing.T) { intCmp := func(a, b int) int { return a - b } emptySlice := []int{} - if !panics(func() { Min(emptySlice) }) { + if !panics(func() { _ = Min(emptySlice) }) { t.Errorf("Min([]): got no panic, want panic") } - if !panics(func() { Max(emptySlice) }) { + if !panics(func() { _ = Max(emptySlice) }) { t.Errorf("Max([]): got no panic, want panic") } - if !panics(func() { MinFunc(emptySlice, intCmp) }) { + if !panics(func() { _ = MinFunc(emptySlice, intCmp) }) { t.Errorf("MinFunc([]): got no panic, want panic") } - if !panics(func() { MaxFunc(emptySlice, intCmp) }) { + if !panics(func() { _ = MaxFunc(emptySlice, intCmp) }) { t.Errorf("MaxFunc([]): got no panic, want panic") } } diff --git a/src/sort/example_keys_test.go b/src/sort/example_keys_test.go index 648f919e68dbd9..5e25f0d3eed940 100644 --- a/src/sort/example_keys_test.go +++ b/src/sort/example_keys_test.go @@ -60,7 +60,7 @@ var planets = []Planet{ {"Mars", 0.107, 1.5}, } -// ExampleSortKeys demonstrates a technique for sorting a struct type using programmable sort criteria. +// Example_sortKeys demonstrates a technique for sorting a struct type using programmable sort criteria. func Example_sortKeys() { // Closures that order the Planet structure. name := func(p1, p2 *Planet) bool { diff --git a/src/sort/example_multi_test.go b/src/sort/example_multi_test.go index 93f2d3ec5754a5..c21050b725d5ed 100644 --- a/src/sort/example_multi_test.go +++ b/src/sort/example_multi_test.go @@ -87,7 +87,7 @@ var changes = []Change{ {"gri", "Smalltalk", 80}, } -// ExampleMultiKeys demonstrates a technique for sorting a struct type using different +// Example_sortMultiKeys demonstrates a technique for sorting a struct type using different // sets of multiple fields in the comparison. We chain together "Less" functions, each of // which compares a single field. func Example_sortMultiKeys() { diff --git a/src/sort/example_search_test.go b/src/sort/example_search_test.go index eadac9a7adcb97..f621dfb46f80fc 100644 --- a/src/sort/example_search_test.go +++ b/src/sort/example_search_test.go @@ -93,3 +93,20 @@ func ExampleSearchInts() { // found 2 at index 1 in [1 2 3 4 6 7 8] // 5 not found, can be inserted at index 4 in [1 2 3 4 6 7 8] } + +// This example demonstrates searching for string in a list sorted in ascending order. +func ExampleSearchStrings() { + a := []string{"apple", "banana", "cherry", "date", "fig", "grape"} + + x := "banana" + i := sort.SearchStrings(a, x) + fmt.Printf("found %s at index %d in %v\n", x, i, a) + + x = "coconut" + i = sort.SearchStrings(a, x) + fmt.Printf("%s not found, can be inserted at index %d in %v\n", x, i, a) + + // Output: + // found banana at index 1 in [apple banana cherry date fig grape] + // coconut not found, can be inserted at index 3 in [apple banana cherry date fig grape] +} diff --git a/src/sort/example_test.go b/src/sort/example_test.go index 1f85dbcbfba231..32eb73c890ded0 100644 --- a/src/sort/example_test.go +++ b/src/sort/example_test.go @@ -86,6 +86,34 @@ func ExampleSlice() { // By age: [{Gopher 7} {Vera 24} {Alice 55} {Bob 75}] } +func ExampleSliceIsSorted() { + numbers := []int{1, 2, 3, 4, 5, 6} + + isSortedAsc := sort.SliceIsSorted(numbers, func(i, j int) bool { + return numbers[i] < numbers[j] + }) + fmt.Printf("%v sorted ascending: %t\n", numbers, isSortedAsc) + + numbersDesc := []int{6, 5, 4, 3, 2, 1} + + isSortedDesc := sort.SliceIsSorted(numbersDesc, func(i, j int) bool { + return numbersDesc[i] > numbersDesc[j] + }) + fmt.Printf("%v sorted descending: %t\n", numbers, isSortedDesc) + + unsortedNumbers := []int{1, 3, 2, 4, 5} + + isSortedUnsorted := sort.SliceIsSorted(unsortedNumbers, func(i, j int) bool { + return unsortedNumbers[i] < unsortedNumbers[j] + }) + fmt.Printf("%v unsorted slice sorted: %t\n", unsortedNumbers, isSortedUnsorted) + + // Output: + // [1 2 3 4 5 6] sorted ascending: true + // [1 2 3 4 5 6] sorted descending: true + // [1 3 2 4 5] unsorted slice sorted: false +} + func ExampleSliceStable() { people := []struct { diff --git a/src/sort/sort.go b/src/sort/sort.go index 6db161f0c08885..fd3cd6dabf7d50 100644 --- a/src/sort/sort.go +++ b/src/sort/sort.go @@ -7,7 +7,10 @@ // Package sort provides primitives for sorting slices and user-defined collections. package sort -import "math/bits" +import ( + "math/bits" + "slices" +) // An implementation of Interface can be sorted by the routines in this package. // The methods refer to elements of the underlying collection by integer index. @@ -23,13 +26,15 @@ type Interface interface { // Sort may place equal elements in any order in the final result, // while Stable preserves the original input order of equal elements. // - // Less must describe a transitive ordering: + // Less must describe a [Strict Weak Ordering]. For example: // - if both Less(i, j) and Less(j, k) are true, then Less(i, k) must be true as well. // - if both Less(i, j) and Less(j, k) are false, then Less(i, k) must be false as well. // // Note that floating-point comparison (the < operator on float32 or float64 values) - // is not a transitive ordering when not-a-number (NaN) values are involved. + // is not a strict weak ordering when not-a-number (NaN) values are involved. // See Float64Slice.Less for a correct implementation for floating-point values. + // + // [Strict Weak Ordering]: https://en.wikipedia.org/wiki/Weak_ordering#Strict_weak_orderings Less(i, j int) bool // Swap swaps the elements with indexes i and j. @@ -64,8 +69,8 @@ type xorshift uint64 func (r *xorshift) Next() uint64 { *r ^= *r << 13 - *r ^= *r >> 17 - *r ^= *r << 5 + *r ^= *r >> 7 + *r ^= *r << 17 return uint64(*r) } @@ -162,34 +167,34 @@ func (x StringSlice) Sort() { Sort(x) } // Ints sorts a slice of ints in increasing order. // // Note: as of Go 1.22, this function simply calls [slices.Sort]. -func Ints(x []int) { intsImpl(x) } +func Ints(x []int) { slices.Sort(x) } // Float64s sorts a slice of float64s in increasing order. // Not-a-number (NaN) values are ordered before other values. // // Note: as of Go 1.22, this function simply calls [slices.Sort]. -func Float64s(x []float64) { float64sImpl(x) } +func Float64s(x []float64) { slices.Sort(x) } // Strings sorts a slice of strings in increasing order. // // Note: as of Go 1.22, this function simply calls [slices.Sort]. -func Strings(x []string) { stringsImpl(x) } +func Strings(x []string) { slices.Sort(x) } // IntsAreSorted reports whether the slice x is sorted in increasing order. // // Note: as of Go 1.22, this function simply calls [slices.IsSorted]. -func IntsAreSorted(x []int) bool { return intsAreSortedImpl(x) } +func IntsAreSorted(x []int) bool { return slices.IsSorted(x) } // Float64sAreSorted reports whether the slice x is sorted in increasing order, // with not-a-number (NaN) values before any other values. // // Note: as of Go 1.22, this function simply calls [slices.IsSorted]. -func Float64sAreSorted(x []float64) bool { return float64sAreSortedImpl(x) } +func Float64sAreSorted(x []float64) bool { return slices.IsSorted(x) } // StringsAreSorted reports whether the slice x is sorted in increasing order. // // Note: as of Go 1.22, this function simply calls [slices.IsSorted]. -func StringsAreSorted(x []string) bool { return stringsAreSortedImpl(x) } +func StringsAreSorted(x []string) bool { return slices.IsSorted(x) } // Notes on stable sorting: // The used algorithms are simple and provable correct on all input and use diff --git a/src/sort/sort_impl_120.go b/src/sort/sort_impl_120.go deleted file mode 100644 index 5980da67e7802c..00000000000000 --- a/src/sort/sort_impl_120.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !go1.21 - -package sort - -func intsImpl(x []int) { Sort(IntSlice(x)) } -func float64sImpl(x []float64) { Sort(Float64Slice(x)) } -func stringsImpl(x []string) { Sort(StringSlice(x)) } - -func intsAreSortedImpl(x []int) bool { return IsSorted(IntSlice(x)) } -func float64sAreSortedImpl(x []float64) bool { return IsSorted(Float64Slice(x)) } -func stringsAreSortedImpl(x []string) bool { return IsSorted(StringSlice(x)) } diff --git a/src/sort/sort_impl_go121.go b/src/sort/sort_impl_go121.go deleted file mode 100644 index 0a6a6a62e76910..00000000000000 --- a/src/sort/sort_impl_go121.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.21 - -// Starting with Go 1.21, we can leverage the new generic functions from the -// slices package to implement some `sort` functions faster. However, until -// the bootstrap compiler uses Go 1.21 or later, we keep a fallback version -// in sort_impl_120.go that retains the old implementation. - -package sort - -import "slices" - -func intsImpl(x []int) { slices.Sort(x) } -func float64sImpl(x []float64) { slices.Sort(x) } -func stringsImpl(x []string) { slices.Sort(x) } - -func intsAreSortedImpl(x []int) bool { return slices.IsSorted(x) } -func float64sAreSortedImpl(x []float64) bool { return slices.IsSorted(x) } -func stringsAreSortedImpl(x []string) bool { return slices.IsSorted(x) } diff --git a/src/sort/sort_slices_benchmark_test.go b/src/sort/sort_slices_benchmark_test.go index 069536df03b33c..b261eeb7db69cb 100644 --- a/src/sort/sort_slices_benchmark_test.go +++ b/src/sort/sort_slices_benchmark_test.go @@ -34,14 +34,6 @@ func makeSortedInts(n int) []int { return ints } -func makeReversedInts(n int) []int { - ints := make([]int, n) - for i := 0; i < n; i++ { - ints[i] = n - i - } - return ints -} - func makeSortedStrings(n int) []string { x := make([]string, n) for i := 0; i < n; i++ { @@ -85,7 +77,7 @@ func BenchmarkSlicesIsSorted(b *testing.B) { b.StopTimer() ints := makeSortedInts(N) b.StartTimer() - slices.IsSorted(ints) + _ = slices.IsSorted(ints) } } diff --git a/src/strconv/atof.go b/src/strconv/atof.go index 8fc90425f69801..c05d2982571451 100644 --- a/src/strconv/atof.go +++ b/src/strconv/atof.go @@ -18,10 +18,7 @@ var optimize = true // set to false to force slow-path conversions for testing // prefix of s and prefix, with the character case of s ignored. // The prefix argument must be all lower-case. func commonPrefixLenIgnoreCase(s, prefix string) int { - n := len(prefix) - if n > len(s) { - n = len(s) - } + n := min(len(prefix), len(s)) for i := 0; i < n; i++ { c := s[i] if 'A' <= c && c <= 'Z' { @@ -80,12 +77,12 @@ func (b *decimal) set(s string) (ok bool) { if i >= len(s) { return } - switch { - case s[i] == '+': + switch s[i] { + case '+': i++ - case s[i] == '-': - b.neg = true + case '-': i++ + b.neg = true } // digits @@ -138,9 +135,10 @@ func (b *decimal) set(s string) (ok bool) { return } esign := 1 - if s[i] == '+' { + switch s[i] { + case '+': i++ - } else if s[i] == '-' { + case '-': i++ esign = -1 } @@ -179,12 +177,12 @@ func readFloat(s string) (mantissa uint64, exp int, neg, trunc, hex bool, i int, if i >= len(s) { return } - switch { - case s[i] == '+': + switch s[i] { + case '+': i++ - case s[i] == '-': - neg = true + case '-': i++ + neg = true } // digits @@ -271,9 +269,10 @@ loop: return } esign := 1 - if s[i] == '+' { + switch s[i] { + case '+': i++ - } else if s[i] == '-' { + case '-': i++ esign = -1 } @@ -461,7 +460,7 @@ func atof64exact(mantissa uint64, exp int, neg bool) (f float64, ok bool) { // If possible to compute mantissa*10^exp to 32-bit float f exactly, // entirely in floating-point math, do so, avoiding the machinery above. func atof32exact(mantissa uint64, exp int, neg bool) (f float32, ok bool) { - if mantissa>>float32info.mantbits != 0 { + if mantissa>>float32MantBits != 0 { return } f = float32(mantissa) diff --git a/src/strconv/atoi.go b/src/strconv/atoi.go index 599ad9b8950833..83e931fe241ca5 100644 --- a/src/strconv/atoi.go +++ b/src/strconv/atoi.go @@ -204,11 +204,12 @@ func ParseInt(s string, base int, bitSize int) (i int64, err error) { // Pick off leading sign. s0 := s neg := false - if s[0] == '+' { + switch s[0] { + case '+': s = s[1:] - } else if s[0] == '-' { - neg = true + case '-': s = s[1:] + neg = true } // Convert unsigned and check range. diff --git a/src/strconv/atoi_test.go b/src/strconv/atoi_test.go index d7f8f25a295b37..ada175200c7b56 100644 --- a/src/strconv/atoi_test.go +++ b/src/strconv/atoi_test.go @@ -501,11 +501,11 @@ func TestAtoi(t *testing.T) { } func bitSizeErrStub(name string, bitSize int) error { - return BitSizeError(name, "0", bitSize) + return bitSizeError(name, "0", bitSize) } func baseErrStub(name string, base int) error { - return BaseError(name, "0", base) + return baseError(name, "0", base) } func noErrStub(name string, arg int) error { diff --git a/src/strconv/eisel_lemire.go b/src/strconv/eisel_lemire.go index 03842e50797ca0..7bc9c71a40516c 100644 --- a/src/strconv/eisel_lemire.go +++ b/src/strconv/eisel_lemire.go @@ -33,22 +33,22 @@ func eiselLemire64(man uint64, exp10 int, neg bool) (f float64, ok bool) { } return f, true } - if exp10 < detailedPowersOfTenMinExp10 || detailedPowersOfTenMaxExp10 < exp10 { + pow, exp2, ok := pow10(exp10) + if !ok { return 0, false } // Normalization. clz := bits.LeadingZeros64(man) man <<= uint(clz) - const float64ExponentBias = 1023 - retExp2 := uint64(217706*exp10>>16+64+float64ExponentBias) - uint64(clz) + retExp2 := uint64(exp2+64-float64Bias) - uint64(clz) // Multiplication. - xHi, xLo := bits.Mul64(man, detailedPowersOfTen[exp10-detailedPowersOfTenMinExp10][1]) + xHi, xLo := bits.Mul64(man, pow.Hi) // Wider Approximation. if xHi&0x1FF == 0x1FF && xLo+man < man { - yHi, yLo := bits.Mul64(man, detailedPowersOfTen[exp10-detailedPowersOfTenMinExp10][0]) + yHi, yLo := bits.Mul64(man, pow.Lo) mergedHi, mergedLo := xHi, xLo+yHi if mergedLo < xLo { mergedHi++ @@ -84,7 +84,7 @@ func eiselLemire64(man uint64, exp10 int, neg bool) (f float64, ok bool) { if retExp2-1 >= 0x7FF-1 { return 0, false } - retBits := retExp2<<52 | retMantissa&0x000FFFFFFFFFFFFF + retBits := retExp2<>16+64+float32ExponentBias) - uint64(clz) + retExp2 := uint64(exp2+64-float32Bias) - uint64(clz) // Multiplication. - xHi, xLo := bits.Mul64(man, detailedPowersOfTen[exp10-detailedPowersOfTenMinExp10][1]) + xHi, xLo := bits.Mul64(man, pow.Hi) // Wider Approximation. if xHi&0x3FFFFFFFFF == 0x3FFFFFFFFF && xLo+man < man { - yHi, yLo := bits.Mul64(man, detailedPowersOfTen[exp10-detailedPowersOfTenMinExp10][0]) + yHi, yLo := bits.Mul64(man, pow.Lo) mergedHi, mergedLo := xHi, xLo+yHi if mergedLo < xLo { mergedHi++ @@ -159,726 +159,9 @@ func eiselLemire32(man uint64, exp10 int, neg bool) (f float32, ok bool) { if retExp2-1 >= 0xFF-1 { return 0, false } - retBits := retExp2<<23 | retMantissa&0x007FFFFF + retBits := retExp2<= 0 { dst = append(dst, '+') } - dst, _ = formatBits(dst, uint64(exp), 10, exp < 0, true) + dst = AppendInt(dst, int64(exp), 10) return dst } diff --git a/src/strconv/ftoa_test.go b/src/strconv/ftoa_test.go index 3512ccf58070ec..40faa433a64cf8 100644 --- a/src/strconv/ftoa_test.go +++ b/src/strconv/ftoa_test.go @@ -172,6 +172,11 @@ var ftoatests = []ftoaTest{ {3.999969482421875, 'x', 2, "0x1.00p+02"}, {3.999969482421875, 'x', 1, "0x1.0p+02"}, {3.999969482421875, 'x', 0, "0x1p+02"}, + + // Cases that Java once mishandled, from David Chase. + {1.801439850948199e+16, 'g', -1, "1.801439850948199e+16"}, + {5.960464477539063e-08, 'g', -1, "5.960464477539063e-08"}, + {1.012e-320, 'g', -1, "1.012e-320"}, } func TestFtoa(t *testing.T) { @@ -186,13 +191,20 @@ func TestFtoa(t *testing.T) { t.Error("AppendFloat testN=64", test.f, string(test.fmt), test.prec, "want", "abc"+test.s, "got", string(x)) } if float64(float32(test.f)) == test.f && test.fmt != 'b' { + test_s := test.s + if test.f == 5.960464477539063e-08 { + // This test is an exact float32 but asking for float64 precision in the string. + // (All our other float64-only tests fail to exactness check above.) + test_s = "5.9604645e-08" + continue + } s := FormatFloat(test.f, test.fmt, test.prec, 32) if s != test.s { - t.Error("testN=32", test.f, string(test.fmt), test.prec, "want", test.s, "got", s) + t.Error("testN=32", test.f, string(test.fmt), test.prec, "want", test_s, "got", s) } x := AppendFloat([]byte("abc"), test.f, test.fmt, test.prec, 32) - if string(x) != "abc"+test.s { - t.Error("AppendFloat testN=32", test.f, string(test.fmt), test.prec, "want", "abc"+test.s, "got", string(x)) + if string(x) != "abc"+test_s { + t.Error("AppendFloat testN=32", test.f, string(test.fmt), test.prec, "want", "abc"+test_s, "got", string(x)) } } } diff --git a/src/strconv/ftoaryu.go b/src/strconv/ftoaryu.go index 2e7bf71df0b66a..9349df955f3c76 100644 --- a/src/strconv/ftoaryu.go +++ b/src/strconv/ftoaryu.go @@ -40,8 +40,8 @@ func ryuFtoaFixed32(d *decimalSlice, mant uint32, exp int, prec int) { // mant*(2^e2)*(10^q) >= 10^(prec-1) // Because mant >= 2^24, it is enough to choose: // 2^(e2+24) >= 10^(-q+prec-1) - // or q = -mulByLog2Log10(e2+24) + prec - 1 - q := -mulByLog2Log10(e2+24) + prec - 1 + // or q = -mulLog10_2(e2+24) + prec - 1 + q := -mulLog10_2(e2+24) + prec - 1 // Now compute mant*(2^e2)*(10^q). // Is it an exact computation? @@ -107,11 +107,11 @@ func ryuFtoaFixed64(d *decimalSlice, mant uint64, exp int, prec int) { // mant*(2^e2)*(10^q) >= 10^(prec-1) // Because mant >= 2^54, it is enough to choose: // 2^(e2+54) >= 10^(-q+prec-1) - // or q = -mulByLog2Log10(e2+54) + prec - 1 + // or q = -mulLog10_2(e2+54) + prec - 1 // - // The minimal required exponent is -mulByLog2Log10(1025)+18 = -291 - // The maximal required exponent is mulByLog2Log10(1074)+18 = 342 - q := -mulByLog2Log10(e2+54) + prec - 1 + // The minimal required exponent is -mulLog10_2(1025)+18 = -291 + // The maximal required exponent is mulLog10_2(1074)+18 = 342 + q := -mulLog10_2(e2+54) + prec - 1 // Now compute mant*(2^e2)*(10^q). // Is it an exact computation? @@ -192,30 +192,9 @@ func formatDecimal(d *decimalSlice, m uint64, trunc bool, roundUp bool, prec int m /= 10 trimmed++ } - // render digits (similar to formatBits) - n := uint(prec) + // render digits + formatBase10(d.d[:prec], m) d.nd = prec - v := m - for v >= 100 { - var v1, v2 uint64 - if v>>32 == 0 { - v1, v2 = uint64(uint32(v)/100), uint64(uint32(v)%100) - } else { - v1, v2 = v/100, v%100 - } - n -= 2 - d.d[n+1] = smallsString[2*v2+1] - d.d[n+0] = smallsString[2*v2+0] - v = v1 - } - if v > 0 { - n-- - d.d[n] = smallsString[2*v+1] - } - if v >= 10 { - n-- - d.d[n] = smallsString[2*v] - } for d.d[d.nd-1] == '0' { d.nd-- trimmed++ @@ -242,7 +221,7 @@ func ryuFtoaShortest(d *decimalSlice, mant uint64, exp int, flt *floatInfo) { return } // Find 10^q *larger* than 2^-e2 - q := mulByLog2Log10(-e2) + 1 + q := mulLog10_2(-e2) + 1 // We are going to multiply by 10^q using 128-bit arithmetic. // The exponent is the same for all 3 numbers. @@ -326,26 +305,6 @@ func ryuFtoaShortest(d *decimalSlice, mant uint64, exp int, flt *floatInfo) { d.dp -= q } -// mulByLog2Log10 returns math.Floor(x * log(2)/log(10)) for an integer x in -// the range -1600 <= x && x <= +1600. -// -// The range restriction lets us work in faster integer arithmetic instead of -// slower floating point arithmetic. Correctness is verified by unit tests. -func mulByLog2Log10(x int) int { - // log(2)/log(10) ≈ 0.30102999566 ≈ 78913 / 2^18 - return (x * 78913) >> 18 -} - -// mulByLog10Log2 returns math.Floor(x * log(10)/log(2)) for an integer x in -// the range -500 <= x && x <= +500. -// -// The range restriction lets us work in faster integer arithmetic instead of -// slower floating point arithmetic. Correctness is verified by unit tests. -func mulByLog10Log2(x int) int { - // log(10)/log(2) ≈ 3.32192809489 ≈ 108853 / 2^15 - return (x * 108853) >> 15 -} - // computeBounds returns a floating-point vector (l, c, u)×2^e2 // where the mantissas are 55-bit (or 26-bit) integers, describing the interval // represented by the input float64 or float32. @@ -468,8 +427,8 @@ func ryuDigits32(d *decimalSlice, lower, central, upper uint32, n := endindex for n > d.nd { v1, v2 := v/100, v%100 - d.d[n] = smallsString[2*v2+1] - d.d[n-1] = smallsString[2*v2+0] + d.d[n] = smalls[2*v2+1] + d.d[n-1] = smalls[2*v2+0] n -= 2 v = v1 } @@ -482,7 +441,7 @@ func ryuDigits32(d *decimalSlice, lower, central, upper uint32, // mult64bitPow10 takes a floating-point input with a 25-bit // mantissa and multiplies it with 10^q. The resulting mantissa -// is m*P >> 57 where P is a 64-bit element of the detailedPowersOfTen tables. +// is m*P >> 57 where P is a 64-bit truncated power of 10. // It is typically 31 or 32-bit wide. // The returned boolean is true if all trimmed bits were zero. // @@ -495,23 +454,23 @@ func mult64bitPow10(m uint32, e2, q int) (resM uint32, resE int, exact bool) { // P == 1<<63 return m << 6, e2 - 6, true } - if q < detailedPowersOfTenMinExp10 || detailedPowersOfTenMaxExp10 < q { + pow, exp2, ok := pow10(q) + if !ok { // This never happens due to the range of float32/float64 exponent panic("mult64bitPow10: power of 10 is out of range") } - pow := detailedPowersOfTen[q-detailedPowersOfTenMinExp10][1] if q < 0 { // Inverse powers of ten must be rounded up. - pow += 1 + pow.Hi++ } - hi, lo := bits.Mul64(uint64(m), pow) - e2 += mulByLog10Log2(q) - 63 + 57 + hi, lo := bits.Mul64(uint64(m), pow.Hi) + e2 += exp2 - 63 + 57 return uint32(hi<<7 | lo>>57), e2, lo<<7 == 0 } // mult128bitPow10 takes a floating-point input with a 55-bit // mantissa and multiplies it with 10^q. The resulting mantissa -// is m*P >> 119 where P is a 128-bit element of the detailedPowersOfTen tables. +// is m*P >> 119 where P is a 128-bit truncated power of 10. // It is typically 63 or 64-bit wide. // The returned boolean is true is all trimmed bits were zero. // @@ -524,23 +483,19 @@ func mult128bitPow10(m uint64, e2, q int) (resM uint64, resE int, exact bool) { // P == 1<<127 return m << 8, e2 - 8, true } - if q < detailedPowersOfTenMinExp10 || detailedPowersOfTenMaxExp10 < q { + pow, exp2, ok := pow10(q) + if !ok { // This never happens due to the range of float32/float64 exponent panic("mult128bitPow10: power of 10 is out of range") } - pow := detailedPowersOfTen[q-detailedPowersOfTenMinExp10] if q < 0 { // Inverse powers of ten must be rounded up. - pow[0] += 1 + pow.Lo++ } - e2 += mulByLog10Log2(q) - 127 + 119 + e2 += exp2 - 127 + 119 - // long multiplication - l1, l0 := bits.Mul64(m, pow[0]) - h1, h0 := bits.Mul64(m, pow[1]) - mid, carry := bits.Add64(l1, h0, 0) - h1 += carry - return h1<<9 | mid>>55, e2, mid<<9 == 0 && l0 == 0 + hi, mid, lo := umul192(m, pow) + return hi<<9 | mid>>55, e2, mid<<9 == 0 && lo == 0 } func divisibleByPower5(m uint64, k int) bool { @@ -559,7 +514,7 @@ func divisibleByPower5(m uint64, k int) bool { // divmod1e9 computes quotient and remainder of division by 1e9, // avoiding runtime uint64 division on 32-bit platforms. func divmod1e9(x uint64) (uint32, uint32) { - if !host32bit { + if host64bit { return uint32(x / 1e9), uint32(x % 1e9) } // Use the same sequence of operations as the amd64 compiler. diff --git a/src/strconv/ftoaryu_test.go b/src/strconv/ftoaryu_test.go deleted file mode 100644 index 9758619ad1d5b3..00000000000000 --- a/src/strconv/ftoaryu_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package strconv_test - -import ( - "math" - . "strconv" - "testing" -) - -func TestMulByLog2Log10(t *testing.T) { - for x := -1600; x <= +1600; x++ { - iMath := MulByLog2Log10(x) - fMath := int(math.Floor(float64(x) * math.Ln2 / math.Ln10)) - if iMath != fMath { - t.Errorf("mulByLog2Log10(%d) failed: %d vs %d\n", x, iMath, fMath) - } - } -} - -func TestMulByLog10Log2(t *testing.T) { - for x := -500; x <= +500; x++ { - iMath := MulByLog10Log2(x) - fMath := int(math.Floor(float64(x) * math.Ln10 / math.Ln2)) - if iMath != fMath { - t.Errorf("mulByLog10Log2(%d) failed: %d vs %d\n", x, iMath, fMath) - } - } -} diff --git a/src/strconv/import_test.go b/src/strconv/import_test.go new file mode 100644 index 00000000000000..05d957a6238b6b --- /dev/null +++ b/src/strconv/import_test.go @@ -0,0 +1,20 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package strconv_test + +import . "strconv" + +type uint128 = Uint128 + +var ( + baseError = BaseError + bitSizeError = BitSizeError + mulLog10_2 = MulLog10_2 + mulLog2_10 = MulLog2_10 + parseFloatPrefix = ParseFloatPrefix + pow10 = Pow10 + umul128 = Umul128 + umul192 = Umul192 +) diff --git a/src/strconv/internal_test.go b/src/strconv/internal_test.go deleted file mode 100644 index f2cceff20eb261..00000000000000 --- a/src/strconv/internal_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// export access to strconv internals for tests - -package strconv - -func NewDecimal(i uint64) *decimal { - d := new(decimal) - d.Assign(i) - return d -} - -func SetOptimize(b bool) bool { - old := optimize - optimize = b - return old -} - -func ParseFloatPrefix(s string, bitSize int) (float64, int, error) { - return parseFloatPrefix(s, bitSize) -} - -func MulByLog2Log10(x int) int { - return mulByLog2Log10(x) -} - -func MulByLog10Log2(x int) int { - return mulByLog10Log2(x) -} diff --git a/src/strconv/itoa.go b/src/strconv/itoa.go index 29fec41fe2e4f8..7884e8e9878996 100644 --- a/src/strconv/itoa.go +++ b/src/strconv/itoa.go @@ -4,16 +4,22 @@ package strconv -import "math/bits" - -const fastSmalls = true // enable fast path for small integers +import ( + "math/bits" + "runtime" +) // FormatUint returns the string representation of i in the given base, // for 2 <= base <= 36. The result uses the lower-case letters 'a' to 'z' // for digit values >= 10. func FormatUint(i uint64, base int) string { - if fastSmalls && i < nSmalls && base == 10 { - return small(int(i)) + if base == 10 { + if i < nSmalls { + return small(int(i)) + } + var a [24]byte + j := formatBase10(a[:], i) + return string(a[j:]) } _, s := formatBits(nil, i, base, false, false) return s @@ -23,8 +29,21 @@ func FormatUint(i uint64, base int) string { // for 2 <= base <= 36. The result uses the lower-case letters 'a' to 'z' // for digit values >= 10. func FormatInt(i int64, base int) string { - if fastSmalls && 0 <= i && i < nSmalls && base == 10 { - return small(int(i)) + if base == 10 { + if 0 <= i && i < nSmalls { + return small(int(i)) + } + var a [24]byte + u := uint64(i) + if i < 0 { + u = -u + } + j := formatBase10(a[:], u) + if i < 0 { + j-- + a[j] = '-' + } + return string(a[j:]) } _, s := formatBits(nil, uint64(i), base, i < 0, false) return s @@ -38,46 +57,29 @@ func Itoa(i int) string { // AppendInt appends the string form of the integer i, // as generated by [FormatInt], to dst and returns the extended buffer. func AppendInt(dst []byte, i int64, base int) []byte { - if fastSmalls && 0 <= i && i < nSmalls && base == 10 { - return append(dst, small(int(i))...) + u := uint64(i) + if i < 0 { + dst = append(dst, '-') + u = -u } - dst, _ = formatBits(dst, uint64(i), base, i < 0, true) - return dst + return AppendUint(dst, u, base) } // AppendUint appends the string form of the unsigned integer i, // as generated by [FormatUint], to dst and returns the extended buffer. func AppendUint(dst []byte, i uint64, base int) []byte { - if fastSmalls && i < nSmalls && base == 10 { - return append(dst, small(int(i))...) + if base == 10 { + if i < nSmalls { + return append(dst, small(int(i))...) + } + var a [24]byte + j := formatBase10(a[:], i) + return append(dst, a[j:]...) } dst, _ = formatBits(dst, i, base, false, true) return dst } -// small returns the string for an i with 0 <= i < nSmalls. -func small(i int) string { - if i < 10 { - return digits[i : i+1] - } - return smallsString[i*2 : i*2+2] -} - -const nSmalls = 100 - -const smallsString = "00010203040506070809" + - "10111213141516171819" + - "20212223242526272829" + - "30313233343536373839" + - "40414243444546474849" + - "50515253545556575859" + - "60616263646566676869" + - "70717273747576777879" + - "80818283848586878889" + - "90919293949596979899" - -const host32bit = ^uint(0)>>32 == 0 - const digits = "0123456789abcdefghijklmnopqrstuvwxyz" // formatBits computes the string representation of u in the given base. @@ -85,15 +87,15 @@ const digits = "0123456789abcdefghijklmnopqrstuvwxyz" // set, the string is appended to dst and the resulting byte slice is // returned as the first result value; otherwise the string is returned // as the second result value. +// The caller is expected to have handled base 10 separately for speed. func formatBits(dst []byte, u uint64, base int, neg, append_ bool) (d []byte, s string) { - if base < 2 || base > len(digits) { + if base < 2 || base == 10 || base > len(digits) { panic("strconv: illegal AppendInt/FormatInt base") } // 2 <= base && base <= len(digits) var a [64 + 1]byte // +1 for sign of 64bit value in base 2 i := len(a) - if neg { u = -u } @@ -101,64 +103,9 @@ func formatBits(dst []byte, u uint64, base int, neg, append_ bool) (d []byte, s // convert bits // We use uint values where we can because those will // fit into a single register even on a 32bit machine. - if base == 10 { - // common case: use constants for / because - // the compiler can optimize it into a multiply+shift - - if host32bit { - // convert the lower digits using 32bit operations - for u >= 1e9 { - // Avoid using r = a%b in addition to q = a/b - // since 64bit division and modulo operations - // are calculated by runtime functions on 32bit machines. - q := u / 1e9 - us := uint(u - q*1e9) // u % 1e9 fits into a uint - for j := 4; j > 0; j-- { - is := us % 100 * 2 - us /= 100 - i -= 2 - a[i+1] = smallsString[is+1] - a[i+0] = smallsString[is+0] - } - - // us < 10, since it contains the last digit - // from the initial 9-digit us. - i-- - a[i] = smallsString[us*2+1] - - u = q - } - // u < 1e9 - } - - // u guaranteed to fit into a uint - us := uint(u) - for us >= 100 { - is := us % 100 * 2 - us /= 100 - i -= 2 - a[i+1] = smallsString[is+1] - a[i+0] = smallsString[is+0] - } - - // us < 100 - is := us * 2 - i-- - a[i] = smallsString[is+1] - if us >= 10 { - i-- - a[i] = smallsString[is] - } - - } else if isPowerOfTwo(base) { + if isPowerOfTwo(base) { // Use shifts and masks instead of / and %. - // Base is a power of 2 and 2 <= base <= len(digits) where len(digits) is 36. - // The largest power of 2 below or equal to 36 is 32, which is 1 << 5; - // i.e., the largest possible shift count is 5. By &-ind that value with - // the constant 7 we tell the compiler that the shift count is always - // less than 8 which is smaller than any register width. This allows - // the compiler to generate better code for the shift operation. - shift := uint(bits.TrailingZeros(uint(base))) & 7 + shift := uint(bits.TrailingZeros(uint(base))) b := uint64(base) m := uint(base) - 1 // == 1<= b { @@ -203,3 +150,159 @@ func formatBits(dst []byte, u uint64, base int, neg, append_ bool) (d []byte, s func isPowerOfTwo(x int) bool { return x&(x-1) == 0 } + +const nSmalls = 100 + +// smalls is the formatting of 00..99 concatenated. +// It is then padded out with 56 x's to 256 bytes, +// so that smalls[x&0xFF] has no bounds check. +// +// TODO(rsc): Once the compiler does a better job +// at tracking mod bounds, the &0xFF should not be needed: +// go.dev/issue/75954 and go.dev/issue/63110. +const smalls = "00010203040506070809" + + "10111213141516171819" + + "20212223242526272829" + + "30313233343536373839" + + "40414243444546474849" + + "50515253545556575859" + + "60616263646566676869" + + "70717273747576777879" + + "80818283848586878889" + + "90919293949596979899" + + "xxxxxxxxxxxxxxxxxxxx" + + "xxxxxxxxxxxxxxxxxxxx" + + "xxxxxxxxxxxxxxxxxxxx" + + "xxxxxxxxxxxxxxxxxxxx" + + "xxxxxxxxxxxxxxxxxxxx" + + "xxxxxx" + +const host64bit = ^uint(0)>>32 != 0 + +// small returns the string for an i with 0 <= i < nSmalls. +func small(i int) string { + if i < 10 { + return digits[i : i+1] + } + return smalls[i*2 : i*2+2] +} + +// formatBase10 formats the decimal representation of u into the tail of a +// and returns the offset of the first byte written to a. That is, after +// +// i := formatBase10(a, u) +// +// the decimal representation is in a[i:]. +func formatBase10(a []byte, u uint64) int { + // Decide implementation strategy based on architecture. + const ( + // 64-bit systems can work in 64-bit math the whole time + // or can split the uint64 into uint32-sized chunks. + // On most systems, the uint32 math is faster, but not all. + // The decision here is based on benchmarking. + itoaPure64 = host64bit && runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64" && runtime.GOARCH != "s390x" + + // 64-bit systems can all use 64-bit div and mod by a constant, + // which the compiler rewrites to use 64x64→128-bit multiplies. + itoaDivMod64 = host64bit // can use 64-bit div/mod by constant + ) + + if itoaPure64 { + // Convert 2 digits at a time, using 64-bit math. + i := len(a) + u := uint(u) + for u >= 100 { + var dd uint + u, dd = u/100, (u%100)*2 + i -= 2 + a[i+0], a[i+1] = smalls[(dd+0)&0xFF], smalls[(dd+1)&0xFF] + } + + dd := u * 2 + i-- + a[i] = smalls[(dd+1)&0xFF] + if u >= 10 { + i-- + a[i] = smalls[(dd+0)&0xFF] + } + return i + } + + // Convert 9-digit chunks using 32-bit math. + // Most numbers are small, so the comparison u >= 1e9 is usually pure overhead, + // so we approximate it by u>>29 != 0, which is usually faster and good enough. + i := len(a) + for (host64bit && u>>29 != 0) || (!host64bit && (u>>32 != 0 || uint32(u)>>29 != 0)) { + var lo uint32 + if itoaDivMod64 { + u, lo = u/1e9, uint32(u%1e9) + } else { + // On 64-bit systems the compiler rewrites the div and mod above + // into a 64x64→128-bit multiply (https://godbolt.org/z/EPnK8zvMK): + // hi, _ := bits.Mul64(u>>1, 0x89705f4136b4a598) + // q := hi >> 28 + // lo = uint32(u - q*1e9) + // u = q + // On 32-bit systems, the compiler invokes a uint64 software divide, + // which is quite slow. We could write the bits.Mul64 code above + // but even that is slower than we'd like, since it calls a software mul64 + // instead of having a hardware instruction to use. + // Instead we inline bits.Mul64 here and change y0/y1 to constants. + // The compiler does use direct 32x32→64-bit multiplies for this code. + // + // For lots more about division by multiplication see Warren, _Hacker's Delight_. + // For a concise overview, see the first two sections of + // https://ridiculousfish.com/blog/posts/labor-of-division-episode-iii.html. + const mask32 = 1<<32 - 1 + x0 := ((u >> 1) & mask32) + x1 := (u >> 1) >> 32 + const y0 = 0x36b4a598 + const y1 = 0x89705f41 + w0 := x0 * y0 + t := x1*y0 + w0>>32 + w1 := t & mask32 + w2 := t >> 32 + w1 += x0 * y1 + hi := x1*y1 + w2 + w1>>32 + q := hi >> 28 + + lo = uint32(u) - uint32(q)*1e9 // uint32(u - q*1e9) but faster + u = q + } + + // Convert 9 digits. + for range 4 { + var dd uint32 + lo, dd = lo/100, (lo%100)*2 + i -= 2 + a[i+0], a[i+1] = smalls[(dd+0)&0xFF], smalls[(dd+1)&0xFF] + } + i-- + a[i] = smalls[(lo*2+1)&0xFF] + + // If we'd been using u >= 1e9 then we would be guaranteed that u/1e9 > 0, + // but since we used u>>29 != 0, u/1e9 might be 0, so we might be done. + // (If u is now 0, then at the start we had 2²⁹ ≤ u < 10⁹, so it was still correct + // to write 9 digits; we have not accidentally written any leading zeros.) + if u == 0 { + return i + } + } + + // Convert final chunk, at most 8 digits. + lo := uint32(u) + for lo >= 100 { + var dd uint32 + lo, dd = lo/100, (lo%100)*2 + i -= 2 + a[i+0], a[i+1] = smalls[(dd+0)&0xFF], smalls[(dd+1)&0xFF] + } + i-- + dd := lo * 2 + a[i] = smalls[(dd+1)&0xFF] + if lo >= 10 { + i-- + a[i] = smalls[(dd+0)&0xFF] + } + return i +} diff --git a/src/strconv/itoa_test.go b/src/strconv/itoa_test.go index b8bc52490a9723..9dbf812d4be279 100644 --- a/src/strconv/itoa_test.go +++ b/src/strconv/itoa_test.go @@ -5,6 +5,7 @@ package strconv_test import ( + "fmt" . "strconv" "testing" ) @@ -230,7 +231,7 @@ func BenchmarkAppendIntSmall(b *testing.B) { func BenchmarkAppendUintVarlen(b *testing.B) { for _, test := range varlenUints { - b.Run(test.out, func(b *testing.B) { + b.Run(fmt.Sprint("digits=", len(test.out)), func(b *testing.B) { dst := make([]byte, 0, 30) for j := 0; j < b.N; j++ { dst = AppendUint(dst[:0], test.in, 10) diff --git a/src/strconv/math.go b/src/strconv/math.go new file mode 100644 index 00000000000000..f0f3d5fe540b23 --- /dev/null +++ b/src/strconv/math.go @@ -0,0 +1,57 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package strconv + +import "math/bits" + +// A uint128 is a 128-bit uint. +// The fields are exported to make them visible to package strconv_test. +type uint128 struct { + Hi uint64 + Lo uint64 +} + +// umul128 returns the 128-bit product x*y. +func umul128(x, y uint64) uint128 { + hi, lo := bits.Mul64(x, y) + return uint128{hi, lo} +} + +// umul192 returns the 192-bit product x*y in three uint64s. +func umul192(x uint64, y uint128) (hi, mid, lo uint64) { + mid1, lo := bits.Mul64(x, y.Lo) + hi, mid2 := bits.Mul64(x, y.Hi) + mid, carry := bits.Add64(mid1, mid2, 0) + return hi + carry, mid, lo +} + +// pow10 returns the 128-bit mantissa and binary exponent of 10**e +// If e is out of range, pow10 returns ok=false. +func pow10(e int) (mant uint128, exp int, ok bool) { + if e < pow10Min || e > pow10Max { + return + } + return pow10Tab[e-pow10Min], mulLog2_10(e), true +} + +// mulLog10_2 returns math.Floor(x * log(2)/log(10)) for an integer x in +// the range -1600 <= x && x <= +1600. +// +// The range restriction lets us work in faster integer arithmetic instead of +// slower floating point arithmetic. Correctness is verified by unit tests. +func mulLog10_2(x int) int { + // log(2)/log(10) ≈ 0.30102999566 ≈ 78913 / 2^18 + return (x * 78913) >> 18 +} + +// mulLog2_10 returns math.Floor(x * log(10)/log(2)) for an integer x in +// the range -500 <= x && x <= +500. +// +// The range restriction lets us work in faster integer arithmetic instead of +// slower floating point arithmetic. Correctness is verified by unit tests. +func mulLog2_10(x int) int { + // log(10)/log(2) ≈ 3.32192809489 ≈ 108853 / 2^15 + return (x * 108853) >> 15 +} diff --git a/src/strconv/math_test.go b/src/strconv/math_test.go new file mode 100644 index 00000000000000..19b7ea6c751963 --- /dev/null +++ b/src/strconv/math_test.go @@ -0,0 +1,80 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package strconv_test + +import ( + "math" + . "strconv" + "testing" +) + +var pow10Tests = []struct { + exp10 int + mant uint128 + exp2 int + ok bool +}{ + {-349, uint128{0, 0}, 0, false}, + {-348, uint128{0xFA8FD5A0081C0288, 0x1732C869CD60E453}, -1157, true}, + {0, uint128{0x8000000000000000, 0x0000000000000000}, 0, true}, + {347, uint128{0xD13EB46469447567, 0x4B7195F2D2D1A9FB}, 1152, true}, + {348, uint128{0, 0}, 0, false}, +} + +func TestPow10(t *testing.T) { + for _, tt := range pow10Tests { + mant, exp2, ok := Pow10(tt.exp10) + if mant != tt.mant || exp2 != tt.exp2 { + t.Errorf("pow10(%d) = %#016x, %#016x, %d, %v want %#016x,%#016x, %d, %v", + tt.exp10, mant.Hi, mant.Lo, exp2, ok, + tt.mant.Hi, tt.mant.Lo, tt.exp2, tt.ok) + } + } +} + +func u128(hi, lo uint64) uint128 { + return uint128{Hi: hi, Lo: lo} +} + +var umul192Tests = []struct { + x uint64 + y uint128 + hi uint64 + mid uint64 + lo uint64 +}{ + {0, u128(0, 0), 0, 0, 0}, + {^uint64(0), u128(^uint64(0), ^uint64(0)), ^uint64(1), ^uint64(0), 1}, +} + +func TestUmul192(t *testing.T) { + for _, tt := range umul192Tests { + hi, mid, lo := Umul192(tt.x, tt.y) + if hi != tt.hi || mid != tt.mid || lo != tt.lo { + t.Errorf("umul192(%#x, {%#x,%#x}) = %#x, %#x, %#x, want %#x, %#x, %#x", + tt.x, tt.y.Hi, tt.y.Lo, hi, mid, lo, tt.hi, tt.mid, tt.lo) + } + } +} + +func TestMulLog10_2(t *testing.T) { + for x := -1600; x <= +1600; x++ { + iMath := mulLog10_2(x) + fMath := int(math.Floor(float64(x) * math.Ln2 / math.Ln10)) + if iMath != fMath { + t.Errorf("mulLog10_2(%d) failed: %d vs %d\n", x, iMath, fMath) + } + } +} + +func TestMulLog2_10(t *testing.T) { + for x := -500; x <= +500; x++ { + iMath := mulLog2_10(x) + fMath := int(math.Floor(float64(x) * math.Ln10 / math.Ln2)) + if iMath != fMath { + t.Errorf("mulLog2_10(%d) failed: %d vs %d\n", x, iMath, fMath) + } + } +} diff --git a/src/strconv/pow10gen.go b/src/strconv/pow10gen.go new file mode 100644 index 00000000000000..2d428fe088e3c1 --- /dev/null +++ b/src/strconv/pow10gen.go @@ -0,0 +1,91 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ignore + +package main + +import ( + "bytes" + "fmt" + "go/format" + "log" + "math/big" + "os" +) + +const ( + minExp = -348 + maxExp = 347 +) + +func main() { + log.SetPrefix("pow10gen: ") + log.SetFlags(0) + + var ( + one = big.NewInt(1) + ten = big.NewInt(10) + + b1p64 = new(big.Int).Lsh(one, 64) + b1p128 = new(big.Int).Lsh(one, 128) + + r2 = big.NewRat(2, 1) + r1p128 = new(big.Rat).SetInt(b1p128) + ) + + var out bytes.Buffer + fmt.Fprintf(&out, top, minExp, maxExp) + for e := int64(minExp); e <= maxExp; e++ { + var r *big.Rat + if e >= 0 { + r = new(big.Rat).SetInt(new(big.Int).Exp(ten, big.NewInt(e), nil)) + } else { + r = new(big.Rat).SetFrac(one, new(big.Int).Exp(ten, big.NewInt(-e), nil)) + } + be := 0 + for r.Cmp(r1p128) < 0 { + r.Mul(r, r2) + be++ + } + for r.Cmp(r1p128) >= 0 { + r.Quo(r, r2) + be-- + } + d := new(big.Int).Div(r.Num(), r.Denom()) + hi, lo := new(big.Int).DivMod(d, b1p64, new(big.Int)) + fmt.Fprintf(&out, "\t{%#016x, %#016x}, // 1e%d * 2**%d\n", hi.Uint64(), lo.Uint64(), e, be) + } + fmt.Fprintf(&out, "}\n") + + src, err := format.Source(out.Bytes()) + if err != nil { + log.Fatal(err) + } + + if err := os.WriteFile("pow10tab.go", src, 0666); err != nil { + log.Fatal(err) + } +} + +var top = `// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated by: go run pow10gen.go. DO NOT EDIT. +// +//go:generate go run pow10gen.go + +package strconv + +const ( + pow10Min = %d + pow10Max = %d +) + + +// pow10Tab holds 128-bit mantissas of powers of 10. +// The values are scaled so the high bit is always set; there is no "implicit leading 1 bit". +var pow10Tab = [...]uint128{ +` diff --git a/src/strconv/pow10tab.go b/src/strconv/pow10tab.go new file mode 100644 index 00000000000000..029ae02b6603f0 --- /dev/null +++ b/src/strconv/pow10tab.go @@ -0,0 +1,715 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated by: go run pow10gen.go. DO NOT EDIT. +// +//go:generate go run pow10gen.go + +package strconv + +const ( + pow10Min = -348 + pow10Max = 347 +) + +// pow10Tab holds 128-bit mantissas of powers of 10. +// The values are scaled so the high bit is always set; there is no "implicit leading 1 bit". +var pow10Tab = [...]uint128{ + {0xfa8fd5a0081c0288, 0x1732c869cd60e453}, // 1e-348 * 2**1284 + {0x9c99e58405118195, 0x0e7fbd42205c8eb4}, // 1e-347 * 2**1280 + {0xc3c05ee50655e1fa, 0x521fac92a873b261}, // 1e-346 * 2**1277 + {0xf4b0769e47eb5a78, 0xe6a797b752909ef9}, // 1e-345 * 2**1274 + {0x98ee4a22ecf3188b, 0x9028bed2939a635c}, // 1e-344 * 2**1270 + {0xbf29dcaba82fdeae, 0x7432ee873880fc33}, // 1e-343 * 2**1267 + {0xeef453d6923bd65a, 0x113faa2906a13b3f}, // 1e-342 * 2**1264 + {0x9558b4661b6565f8, 0x4ac7ca59a424c507}, // 1e-341 * 2**1260 + {0xbaaee17fa23ebf76, 0x5d79bcf00d2df649}, // 1e-340 * 2**1257 + {0xe95a99df8ace6f53, 0xf4d82c2c107973dc}, // 1e-339 * 2**1254 + {0x91d8a02bb6c10594, 0x79071b9b8a4be869}, // 1e-338 * 2**1250 + {0xb64ec836a47146f9, 0x9748e2826cdee284}, // 1e-337 * 2**1247 + {0xe3e27a444d8d98b7, 0xfd1b1b2308169b25}, // 1e-336 * 2**1244 + {0x8e6d8c6ab0787f72, 0xfe30f0f5e50e20f7}, // 1e-335 * 2**1240 + {0xb208ef855c969f4f, 0xbdbd2d335e51a935}, // 1e-334 * 2**1237 + {0xde8b2b66b3bc4723, 0xad2c788035e61382}, // 1e-333 * 2**1234 + {0x8b16fb203055ac76, 0x4c3bcb5021afcc31}, // 1e-332 * 2**1230 + {0xaddcb9e83c6b1793, 0xdf4abe242a1bbf3d}, // 1e-331 * 2**1227 + {0xd953e8624b85dd78, 0xd71d6dad34a2af0d}, // 1e-330 * 2**1224 + {0x87d4713d6f33aa6b, 0x8672648c40e5ad68}, // 1e-329 * 2**1220 + {0xa9c98d8ccb009506, 0x680efdaf511f18c2}, // 1e-328 * 2**1217 + {0xd43bf0effdc0ba48, 0x0212bd1b2566def2}, // 1e-327 * 2**1214 + {0x84a57695fe98746d, 0x014bb630f7604b57}, // 1e-326 * 2**1210 + {0xa5ced43b7e3e9188, 0x419ea3bd35385e2d}, // 1e-325 * 2**1207 + {0xcf42894a5dce35ea, 0x52064cac828675b9}, // 1e-324 * 2**1204 + {0x818995ce7aa0e1b2, 0x7343efebd1940993}, // 1e-323 * 2**1200 + {0xa1ebfb4219491a1f, 0x1014ebe6c5f90bf8}, // 1e-322 * 2**1197 + {0xca66fa129f9b60a6, 0xd41a26e077774ef6}, // 1e-321 * 2**1194 + {0xfd00b897478238d0, 0x8920b098955522b4}, // 1e-320 * 2**1191 + {0x9e20735e8cb16382, 0x55b46e5f5d5535b0}, // 1e-319 * 2**1187 + {0xc5a890362fddbc62, 0xeb2189f734aa831d}, // 1e-318 * 2**1184 + {0xf712b443bbd52b7b, 0xa5e9ec7501d523e4}, // 1e-317 * 2**1181 + {0x9a6bb0aa55653b2d, 0x47b233c92125366e}, // 1e-316 * 2**1177 + {0xc1069cd4eabe89f8, 0x999ec0bb696e840a}, // 1e-315 * 2**1174 + {0xf148440a256e2c76, 0xc00670ea43ca250d}, // 1e-314 * 2**1171 + {0x96cd2a865764dbca, 0x380406926a5e5728}, // 1e-313 * 2**1167 + {0xbc807527ed3e12bc, 0xc605083704f5ecf2}, // 1e-312 * 2**1164 + {0xeba09271e88d976b, 0xf7864a44c633682e}, // 1e-311 * 2**1161 + {0x93445b8731587ea3, 0x7ab3ee6afbe0211d}, // 1e-310 * 2**1157 + {0xb8157268fdae9e4c, 0x5960ea05bad82964}, // 1e-309 * 2**1154 + {0xe61acf033d1a45df, 0x6fb92487298e33bd}, // 1e-308 * 2**1151 + {0x8fd0c16206306bab, 0xa5d3b6d479f8e056}, // 1e-307 * 2**1147 + {0xb3c4f1ba87bc8696, 0x8f48a4899877186c}, // 1e-306 * 2**1144 + {0xe0b62e2929aba83c, 0x331acdabfe94de87}, // 1e-305 * 2**1141 + {0x8c71dcd9ba0b4925, 0x9ff0c08b7f1d0b14}, // 1e-304 * 2**1137 + {0xaf8e5410288e1b6f, 0x07ecf0ae5ee44dd9}, // 1e-303 * 2**1134 + {0xdb71e91432b1a24a, 0xc9e82cd9f69d6150}, // 1e-302 * 2**1131 + {0x892731ac9faf056e, 0xbe311c083a225cd2}, // 1e-301 * 2**1127 + {0xab70fe17c79ac6ca, 0x6dbd630a48aaf406}, // 1e-300 * 2**1124 + {0xd64d3d9db981787d, 0x092cbbccdad5b108}, // 1e-299 * 2**1121 + {0x85f0468293f0eb4e, 0x25bbf56008c58ea5}, // 1e-298 * 2**1117 + {0xa76c582338ed2621, 0xaf2af2b80af6f24e}, // 1e-297 * 2**1114 + {0xd1476e2c07286faa, 0x1af5af660db4aee1}, // 1e-296 * 2**1111 + {0x82cca4db847945ca, 0x50d98d9fc890ed4d}, // 1e-295 * 2**1107 + {0xa37fce126597973c, 0xe50ff107bab528a0}, // 1e-294 * 2**1104 + {0xcc5fc196fefd7d0c, 0x1e53ed49a96272c8}, // 1e-293 * 2**1101 + {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7a}, // 1e-292 * 2**1098 + {0x9faacf3df73609b1, 0x77b191618c54e9ac}, // 1e-291 * 2**1094 + {0xc795830d75038c1d, 0xd59df5b9ef6a2417}, // 1e-290 * 2**1091 + {0xf97ae3d0d2446f25, 0x4b0573286b44ad1d}, // 1e-289 * 2**1088 + {0x9becce62836ac577, 0x4ee367f9430aec32}, // 1e-288 * 2**1084 + {0xc2e801fb244576d5, 0x229c41f793cda73f}, // 1e-287 * 2**1081 + {0xf3a20279ed56d48a, 0x6b43527578c1110f}, // 1e-286 * 2**1078 + {0x9845418c345644d6, 0x830a13896b78aaa9}, // 1e-285 * 2**1074 + {0xbe5691ef416bd60c, 0x23cc986bc656d553}, // 1e-284 * 2**1071 + {0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa8}, // 1e-283 * 2**1068 + {0x94b3a202eb1c3f39, 0x7bf7d71432f3d6a9}, // 1e-282 * 2**1064 + {0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc53}, // 1e-281 * 2**1061 + {0xe858ad248f5c22c9, 0xd1b3400f8f9cff68}, // 1e-280 * 2**1058 + {0x91376c36d99995be, 0x23100809b9c21fa1}, // 1e-279 * 2**1054 + {0xb58547448ffffb2d, 0xabd40a0c2832a78a}, // 1e-278 * 2**1051 + {0xe2e69915b3fff9f9, 0x16c90c8f323f516c}, // 1e-277 * 2**1048 + {0x8dd01fad907ffc3b, 0xae3da7d97f6792e3}, // 1e-276 * 2**1044 + {0xb1442798f49ffb4a, 0x99cd11cfdf41779c}, // 1e-275 * 2**1041 + {0xdd95317f31c7fa1d, 0x40405643d711d583}, // 1e-274 * 2**1038 + {0x8a7d3eef7f1cfc52, 0x482835ea666b2572}, // 1e-273 * 2**1034 + {0xad1c8eab5ee43b66, 0xda3243650005eecf}, // 1e-272 * 2**1031 + {0xd863b256369d4a40, 0x90bed43e40076a82}, // 1e-271 * 2**1028 + {0x873e4f75e2224e68, 0x5a7744a6e804a291}, // 1e-270 * 2**1024 + {0xa90de3535aaae202, 0x711515d0a205cb36}, // 1e-269 * 2**1021 + {0xd3515c2831559a83, 0x0d5a5b44ca873e03}, // 1e-268 * 2**1018 + {0x8412d9991ed58091, 0xe858790afe9486c2}, // 1e-267 * 2**1014 + {0xa5178fff668ae0b6, 0x626e974dbe39a872}, // 1e-266 * 2**1011 + {0xce5d73ff402d98e3, 0xfb0a3d212dc8128f}, // 1e-265 * 2**1008 + {0x80fa687f881c7f8e, 0x7ce66634bc9d0b99}, // 1e-264 * 2**1004 + {0xa139029f6a239f72, 0x1c1fffc1ebc44e80}, // 1e-263 * 2**1001 + {0xc987434744ac874e, 0xa327ffb266b56220}, // 1e-262 * 2**998 + {0xfbe9141915d7a922, 0x4bf1ff9f0062baa8}, // 1e-261 * 2**995 + {0x9d71ac8fada6c9b5, 0x6f773fc3603db4a9}, // 1e-260 * 2**991 + {0xc4ce17b399107c22, 0xcb550fb4384d21d3}, // 1e-259 * 2**988 + {0xf6019da07f549b2b, 0x7e2a53a146606a48}, // 1e-258 * 2**985 + {0x99c102844f94e0fb, 0x2eda7444cbfc426d}, // 1e-257 * 2**981 + {0xc0314325637a1939, 0xfa911155fefb5308}, // 1e-256 * 2**978 + {0xf03d93eebc589f88, 0x793555ab7eba27ca}, // 1e-255 * 2**975 + {0x96267c7535b763b5, 0x4bc1558b2f3458de}, // 1e-254 * 2**971 + {0xbbb01b9283253ca2, 0x9eb1aaedfb016f16}, // 1e-253 * 2**968 + {0xea9c227723ee8bcb, 0x465e15a979c1cadc}, // 1e-252 * 2**965 + {0x92a1958a7675175f, 0x0bfacd89ec191ec9}, // 1e-251 * 2**961 + {0xb749faed14125d36, 0xcef980ec671f667b}, // 1e-250 * 2**958 + {0xe51c79a85916f484, 0x82b7e12780e7401a}, // 1e-249 * 2**955 + {0x8f31cc0937ae58d2, 0xd1b2ecb8b0908810}, // 1e-248 * 2**951 + {0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa15}, // 1e-247 * 2**948 + {0xdfbdcece67006ac9, 0x67a791e093e1d49a}, // 1e-246 * 2**945 + {0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e0}, // 1e-245 * 2**941 + {0xaecc49914078536d, 0x58fae9f773886e18}, // 1e-244 * 2**938 + {0xda7f5bf590966848, 0xaf39a475506a899e}, // 1e-243 * 2**935 + {0x888f99797a5e012d, 0x6d8406c952429603}, // 1e-242 * 2**931 + {0xaab37fd7d8f58178, 0xc8e5087ba6d33b83}, // 1e-241 * 2**928 + {0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a64}, // 1e-240 * 2**925 + {0x855c3be0a17fcd26, 0x5cf2eea09a55067f}, // 1e-239 * 2**921 + {0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481e}, // 1e-238 * 2**918 + {0xd0601d8efc57b08b, 0xf13b94daf124da26}, // 1e-237 * 2**915 + {0x823c12795db6ce57, 0x76c53d08d6b70858}, // 1e-236 * 2**911 + {0xa2cb1717b52481ed, 0x54768c4b0c64ca6e}, // 1e-235 * 2**908 + {0xcb7ddcdda26da268, 0xa9942f5dcf7dfd09}, // 1e-234 * 2**905 + {0xfe5d54150b090b02, 0xd3f93b35435d7c4c}, // 1e-233 * 2**902 + {0x9efa548d26e5a6e1, 0xc47bc5014a1a6daf}, // 1e-232 * 2**898 + {0xc6b8e9b0709f109a, 0x359ab6419ca1091b}, // 1e-231 * 2**895 + {0xf867241c8cc6d4c0, 0xc30163d203c94b62}, // 1e-230 * 2**892 + {0x9b407691d7fc44f8, 0x79e0de63425dcf1d}, // 1e-229 * 2**888 + {0xc21094364dfb5636, 0x985915fc12f542e4}, // 1e-228 * 2**885 + {0xf294b943e17a2bc4, 0x3e6f5b7b17b2939d}, // 1e-227 * 2**882 + {0x979cf3ca6cec5b5a, 0xa705992ceecf9c42}, // 1e-226 * 2**878 + {0xbd8430bd08277231, 0x50c6ff782a838353}, // 1e-225 * 2**875 + {0xece53cec4a314ebd, 0xa4f8bf5635246428}, // 1e-224 * 2**872 + {0x940f4613ae5ed136, 0x871b7795e136be99}, // 1e-223 * 2**868 + {0xb913179899f68584, 0x28e2557b59846e3f}, // 1e-222 * 2**865 + {0xe757dd7ec07426e5, 0x331aeada2fe589cf}, // 1e-221 * 2**862 + {0x9096ea6f3848984f, 0x3ff0d2c85def7621}, // 1e-220 * 2**858 + {0xb4bca50b065abe63, 0x0fed077a756b53a9}, // 1e-219 * 2**855 + {0xe1ebce4dc7f16dfb, 0xd3e8495912c62894}, // 1e-218 * 2**852 + {0x8d3360f09cf6e4bd, 0x64712dd7abbbd95c}, // 1e-217 * 2**848 + {0xb080392cc4349dec, 0xbd8d794d96aacfb3}, // 1e-216 * 2**845 + {0xdca04777f541c567, 0xecf0d7a0fc5583a0}, // 1e-215 * 2**842 + {0x89e42caaf9491b60, 0xf41686c49db57244}, // 1e-214 * 2**838 + {0xac5d37d5b79b6239, 0x311c2875c522ced5}, // 1e-213 * 2**835 + {0xd77485cb25823ac7, 0x7d633293366b828b}, // 1e-212 * 2**832 + {0x86a8d39ef77164bc, 0xae5dff9c02033197}, // 1e-211 * 2**828 + {0xa8530886b54dbdeb, 0xd9f57f830283fdfc}, // 1e-210 * 2**825 + {0xd267caa862a12d66, 0xd072df63c324fd7b}, // 1e-209 * 2**822 + {0x8380dea93da4bc60, 0x4247cb9e59f71e6d}, // 1e-208 * 2**818 + {0xa46116538d0deb78, 0x52d9be85f074e608}, // 1e-207 * 2**815 + {0xcd795be870516656, 0x67902e276c921f8b}, // 1e-206 * 2**812 + {0x806bd9714632dff6, 0x00ba1cd8a3db53b6}, // 1e-205 * 2**808 + {0xa086cfcd97bf97f3, 0x80e8a40eccd228a4}, // 1e-204 * 2**805 + {0xc8a883c0fdaf7df0, 0x6122cd128006b2cd}, // 1e-203 * 2**802 + {0xfad2a4b13d1b5d6c, 0x796b805720085f81}, // 1e-202 * 2**799 + {0x9cc3a6eec6311a63, 0xcbe3303674053bb0}, // 1e-201 * 2**795 + {0xc3f490aa77bd60fc, 0xbedbfc4411068a9c}, // 1e-200 * 2**792 + {0xf4f1b4d515acb93b, 0xee92fb5515482d44}, // 1e-199 * 2**789 + {0x991711052d8bf3c5, 0x751bdd152d4d1c4a}, // 1e-198 * 2**785 + {0xbf5cd54678eef0b6, 0xd262d45a78a0635d}, // 1e-197 * 2**782 + {0xef340a98172aace4, 0x86fb897116c87c34}, // 1e-196 * 2**779 + {0x9580869f0e7aac0e, 0xd45d35e6ae3d4da0}, // 1e-195 * 2**775 + {0xbae0a846d2195712, 0x8974836059cca109}, // 1e-194 * 2**772 + {0xe998d258869facd7, 0x2bd1a438703fc94b}, // 1e-193 * 2**769 + {0x91ff83775423cc06, 0x7b6306a34627ddcf}, // 1e-192 * 2**765 + {0xb67f6455292cbf08, 0x1a3bc84c17b1d542}, // 1e-191 * 2**762 + {0xe41f3d6a7377eeca, 0x20caba5f1d9e4a93}, // 1e-190 * 2**759 + {0x8e938662882af53e, 0x547eb47b7282ee9c}, // 1e-189 * 2**755 + {0xb23867fb2a35b28d, 0xe99e619a4f23aa43}, // 1e-188 * 2**752 + {0xdec681f9f4c31f31, 0x6405fa00e2ec94d4}, // 1e-187 * 2**749 + {0x8b3c113c38f9f37e, 0xde83bc408dd3dd04}, // 1e-186 * 2**745 + {0xae0b158b4738705e, 0x9624ab50b148d445}, // 1e-185 * 2**742 + {0xd98ddaee19068c76, 0x3badd624dd9b0957}, // 1e-184 * 2**739 + {0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d6}, // 1e-183 * 2**735 + {0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4c}, // 1e-182 * 2**732 + {0xd47487cc8470652b, 0x7647c3200069671f}, // 1e-181 * 2**729 + {0x84c8d4dfd2c63f3b, 0x29ecd9f40041e073}, // 1e-180 * 2**725 + {0xa5fb0a17c777cf09, 0xf468107100525890}, // 1e-179 * 2**722 + {0xcf79cc9db955c2cc, 0x7182148d4066eeb4}, // 1e-178 * 2**719 + {0x81ac1fe293d599bf, 0xc6f14cd848405530}, // 1e-177 * 2**715 + {0xa21727db38cb002f, 0xb8ada00e5a506a7c}, // 1e-176 * 2**712 + {0xca9cf1d206fdc03b, 0xa6d90811f0e4851c}, // 1e-175 * 2**709 + {0xfd442e4688bd304a, 0x908f4a166d1da663}, // 1e-174 * 2**706 + {0x9e4a9cec15763e2e, 0x9a598e4e043287fe}, // 1e-173 * 2**702 + {0xc5dd44271ad3cdba, 0x40eff1e1853f29fd}, // 1e-172 * 2**699 + {0xf7549530e188c128, 0xd12bee59e68ef47c}, // 1e-171 * 2**696 + {0x9a94dd3e8cf578b9, 0x82bb74f8301958ce}, // 1e-170 * 2**692 + {0xc13a148e3032d6e7, 0xe36a52363c1faf01}, // 1e-169 * 2**689 + {0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac1}, // 1e-168 * 2**686 + {0x96f5600f15a7b7e5, 0x29ab103a5ef8c0b9}, // 1e-167 * 2**682 + {0xbcb2b812db11a5de, 0x7415d448f6b6f0e7}, // 1e-166 * 2**679 + {0xebdf661791d60f56, 0x111b495b3464ad21}, // 1e-165 * 2**676 + {0x936b9fcebb25c995, 0xcab10dd900beec34}, // 1e-164 * 2**672 + {0xb84687c269ef3bfb, 0x3d5d514f40eea742}, // 1e-163 * 2**669 + {0xe65829b3046b0afa, 0x0cb4a5a3112a5112}, // 1e-162 * 2**666 + {0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ab}, // 1e-161 * 2**662 + {0xb3f4e093db73a093, 0x59ed216765690f56}, // 1e-160 * 2**659 + {0xe0f218b8d25088b8, 0x306869c13ec3532c}, // 1e-159 * 2**656 + {0x8c974f7383725573, 0x1e414218c73a13fb}, // 1e-158 * 2**652 + {0xafbd2350644eeacf, 0xe5d1929ef90898fa}, // 1e-157 * 2**649 + {0xdbac6c247d62a583, 0xdf45f746b74abf39}, // 1e-156 * 2**646 + {0x894bc396ce5da772, 0x6b8bba8c328eb783}, // 1e-155 * 2**642 + {0xab9eb47c81f5114f, 0x066ea92f3f326564}, // 1e-154 * 2**639 + {0xd686619ba27255a2, 0xc80a537b0efefebd}, // 1e-153 * 2**636 + {0x8613fd0145877585, 0xbd06742ce95f5f36}, // 1e-152 * 2**632 + {0xa798fc4196e952e7, 0x2c48113823b73704}, // 1e-151 * 2**629 + {0xd17f3b51fca3a7a0, 0xf75a15862ca504c5}, // 1e-150 * 2**626 + {0x82ef85133de648c4, 0x9a984d73dbe722fb}, // 1e-149 * 2**622 + {0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebba}, // 1e-148 * 2**619 + {0xcc963fee10b7d1b3, 0x318df905079926a8}, // 1e-147 * 2**616 + {0xffbbcfe994e5c61f, 0xfdf17746497f7052}, // 1e-146 * 2**613 + {0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa633}, // 1e-145 * 2**609 + {0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc0}, // 1e-144 * 2**606 + {0xf9bd690a1b68637b, 0x3dfdce7aa3c673b0}, // 1e-143 * 2**603 + {0x9c1661a651213e2d, 0x06bea10ca65c084e}, // 1e-142 * 2**599 + {0xc31bfa0fe5698db8, 0x486e494fcff30a62}, // 1e-141 * 2**596 + {0xf3e2f893dec3f126, 0x5a89dba3c3efccfa}, // 1e-140 * 2**593 + {0x986ddb5c6b3a76b7, 0xf89629465a75e01c}, // 1e-139 * 2**589 + {0xbe89523386091465, 0xf6bbb397f1135823}, // 1e-138 * 2**586 + {0xee2ba6c0678b597f, 0x746aa07ded582e2c}, // 1e-137 * 2**583 + {0x94db483840b717ef, 0xa8c2a44eb4571cdc}, // 1e-136 * 2**579 + {0xba121a4650e4ddeb, 0x92f34d62616ce413}, // 1e-135 * 2**576 + {0xe896a0d7e51e1566, 0x77b020baf9c81d17}, // 1e-134 * 2**573 + {0x915e2486ef32cd60, 0x0ace1474dc1d122e}, // 1e-133 * 2**569 + {0xb5b5ada8aaff80b8, 0x0d819992132456ba}, // 1e-132 * 2**566 + {0xe3231912d5bf60e6, 0x10e1fff697ed6c69}, // 1e-131 * 2**563 + {0x8df5efabc5979c8f, 0xca8d3ffa1ef463c1}, // 1e-130 * 2**559 + {0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb2}, // 1e-129 * 2**556 + {0xddd0467c64bce4a0, 0xac7cb3f6d05ddbde}, // 1e-128 * 2**553 + {0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96b}, // 1e-127 * 2**549 + {0xad4ab7112eb3929d, 0x86c16c98d2c953c6}, // 1e-126 * 2**546 + {0xd89d64d57a607744, 0xe871c7bf077ba8b7}, // 1e-125 * 2**543 + {0x87625f056c7c4a8b, 0x11471cd764ad4972}, // 1e-124 * 2**539 + {0xa93af6c6c79b5d2d, 0xd598e40d3dd89bcf}, // 1e-123 * 2**536 + {0xd389b47879823479, 0x4aff1d108d4ec2c3}, // 1e-122 * 2**533 + {0x843610cb4bf160cb, 0xcedf722a585139ba}, // 1e-121 * 2**529 + {0xa54394fe1eedb8fe, 0xc2974eb4ee658828}, // 1e-120 * 2**526 + {0xce947a3da6a9273e, 0x733d226229feea32}, // 1e-119 * 2**523 + {0x811ccc668829b887, 0x0806357d5a3f525f}, // 1e-118 * 2**519 + {0xa163ff802a3426a8, 0xca07c2dcb0cf26f7}, // 1e-117 * 2**516 + {0xc9bcff6034c13052, 0xfc89b393dd02f0b5}, // 1e-116 * 2**513 + {0xfc2c3f3841f17c67, 0xbbac2078d443ace2}, // 1e-115 * 2**510 + {0x9d9ba7832936edc0, 0xd54b944b84aa4c0d}, // 1e-114 * 2**506 + {0xc5029163f384a931, 0x0a9e795e65d4df11}, // 1e-113 * 2**503 + {0xf64335bcf065d37d, 0x4d4617b5ff4a16d5}, // 1e-112 * 2**500 + {0x99ea0196163fa42e, 0x504bced1bf8e4e45}, // 1e-111 * 2**496 + {0xc06481fb9bcf8d39, 0xe45ec2862f71e1d6}, // 1e-110 * 2**493 + {0xf07da27a82c37088, 0x5d767327bb4e5a4c}, // 1e-109 * 2**490 + {0x964e858c91ba2655, 0x3a6a07f8d510f86f}, // 1e-108 * 2**486 + {0xbbe226efb628afea, 0x890489f70a55368b}, // 1e-107 * 2**483 + {0xeadab0aba3b2dbe5, 0x2b45ac74ccea842e}, // 1e-106 * 2**480 + {0x92c8ae6b464fc96f, 0x3b0b8bc90012929d}, // 1e-105 * 2**476 + {0xb77ada0617e3bbcb, 0x09ce6ebb40173744}, // 1e-104 * 2**473 + {0xe55990879ddcaabd, 0xcc420a6a101d0515}, // 1e-103 * 2**470 + {0x8f57fa54c2a9eab6, 0x9fa946824a12232d}, // 1e-102 * 2**466 + {0xb32df8e9f3546564, 0x47939822dc96abf9}, // 1e-101 * 2**463 + {0xdff9772470297ebd, 0x59787e2b93bc56f7}, // 1e-100 * 2**460 + {0x8bfbea76c619ef36, 0x57eb4edb3c55b65a}, // 1e-99 * 2**456 + {0xaefae51477a06b03, 0xede622920b6b23f1}, // 1e-98 * 2**453 + {0xdab99e59958885c4, 0xe95fab368e45eced}, // 1e-97 * 2**450 + {0x88b402f7fd75539b, 0x11dbcb0218ebb414}, // 1e-96 * 2**446 + {0xaae103b5fcd2a881, 0xd652bdc29f26a119}, // 1e-95 * 2**443 + {0xd59944a37c0752a2, 0x4be76d3346f0495f}, // 1e-94 * 2**440 + {0x857fcae62d8493a5, 0x6f70a4400c562ddb}, // 1e-93 * 2**436 + {0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb952}, // 1e-92 * 2**433 + {0xd097ad07a71f26b2, 0x7e2000a41346a7a7}, // 1e-91 * 2**430 + {0x825ecc24c873782f, 0x8ed400668c0c28c8}, // 1e-90 * 2**426 + {0xa2f67f2dfa90563b, 0x728900802f0f32fa}, // 1e-89 * 2**423 + {0xcbb41ef979346bca, 0x4f2b40a03ad2ffb9}, // 1e-88 * 2**420 + {0xfea126b7d78186bc, 0xe2f610c84987bfa8}, // 1e-87 * 2**417 + {0x9f24b832e6b0f436, 0x0dd9ca7d2df4d7c9}, // 1e-86 * 2**413 + {0xc6ede63fa05d3143, 0x91503d1c79720dbb}, // 1e-85 * 2**410 + {0xf8a95fcf88747d94, 0x75a44c6397ce912a}, // 1e-84 * 2**407 + {0x9b69dbe1b548ce7c, 0xc986afbe3ee11aba}, // 1e-83 * 2**403 + {0xc24452da229b021b, 0xfbe85badce996168}, // 1e-82 * 2**400 + {0xf2d56790ab41c2a2, 0xfae27299423fb9c3}, // 1e-81 * 2**397 + {0x97c560ba6b0919a5, 0xdccd879fc967d41a}, // 1e-80 * 2**393 + {0xbdb6b8e905cb600f, 0x5400e987bbc1c920}, // 1e-79 * 2**390 + {0xed246723473e3813, 0x290123e9aab23b68}, // 1e-78 * 2**387 + {0x9436c0760c86e30b, 0xf9a0b6720aaf6521}, // 1e-77 * 2**383 + {0xb94470938fa89bce, 0xf808e40e8d5b3e69}, // 1e-76 * 2**380 + {0xe7958cb87392c2c2, 0xb60b1d1230b20e04}, // 1e-75 * 2**377 + {0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c2}, // 1e-74 * 2**373 + {0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af3}, // 1e-73 * 2**370 + {0xe2280b6c20dd5232, 0x25c6da63c38de1b0}, // 1e-72 * 2**367 + {0x8d590723948a535f, 0x579c487e5a38ad0e}, // 1e-71 * 2**363 + {0xb0af48ec79ace837, 0x2d835a9df0c6d851}, // 1e-70 * 2**360 + {0xdcdb1b2798182244, 0xf8e431456cf88e65}, // 1e-69 * 2**357 + {0x8a08f0f8bf0f156b, 0x1b8e9ecb641b58ff}, // 1e-68 * 2**353 + {0xac8b2d36eed2dac5, 0xe272467e3d222f3f}, // 1e-67 * 2**350 + {0xd7adf884aa879177, 0x5b0ed81dcc6abb0f}, // 1e-66 * 2**347 + {0x86ccbb52ea94baea, 0x98e947129fc2b4e9}, // 1e-65 * 2**343 + {0xa87fea27a539e9a5, 0x3f2398d747b36224}, // 1e-64 * 2**340 + {0xd29fe4b18e88640e, 0x8eec7f0d19a03aad}, // 1e-63 * 2**337 + {0x83a3eeeef9153e89, 0x1953cf68300424ac}, // 1e-62 * 2**333 + {0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd7}, // 1e-61 * 2**330 + {0xcdb02555653131b6, 0x3792f412cb06794d}, // 1e-60 * 2**327 + {0x808e17555f3ebf11, 0xe2bbd88bbee40bd0}, // 1e-59 * 2**323 + {0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec4}, // 1e-58 * 2**320 + {0xc8de047564d20a8b, 0xf245825a5a445275}, // 1e-57 * 2**317 + {0xfb158592be068d2e, 0xeed6e2f0f0d56712}, // 1e-56 * 2**314 + {0x9ced737bb6c4183d, 0x55464dd69685606b}, // 1e-55 * 2**310 + {0xc428d05aa4751e4c, 0xaa97e14c3c26b886}, // 1e-54 * 2**307 + {0xf53304714d9265df, 0xd53dd99f4b3066a8}, // 1e-53 * 2**304 + {0x993fe2c6d07b7fab, 0xe546a8038efe4029}, // 1e-52 * 2**300 + {0xbf8fdb78849a5f96, 0xde98520472bdd033}, // 1e-51 * 2**297 + {0xef73d256a5c0f77c, 0x963e66858f6d4440}, // 1e-50 * 2**294 + {0x95a8637627989aad, 0xdde7001379a44aa8}, // 1e-49 * 2**290 + {0xbb127c53b17ec159, 0x5560c018580d5d52}, // 1e-48 * 2**287 + {0xe9d71b689dde71af, 0xaab8f01e6e10b4a6}, // 1e-47 * 2**284 + {0x9226712162ab070d, 0xcab3961304ca70e8}, // 1e-46 * 2**280 + {0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d22}, // 1e-45 * 2**277 + {0xe45c10c42a2b3b05, 0x8cb89a7db77c506a}, // 1e-44 * 2**274 + {0x8eb98a7a9a5b04e3, 0x77f3608e92adb242}, // 1e-43 * 2**270 + {0xb267ed1940f1c61c, 0x55f038b237591ed3}, // 1e-42 * 2**267 + {0xdf01e85f912e37a3, 0x6b6c46dec52f6688}, // 1e-41 * 2**264 + {0x8b61313bbabce2c6, 0x2323ac4b3b3da015}, // 1e-40 * 2**260 + {0xae397d8aa96c1b77, 0xabec975e0a0d081a}, // 1e-39 * 2**257 + {0xd9c7dced53c72255, 0x96e7bd358c904a21}, // 1e-38 * 2**254 + {0x881cea14545c7575, 0x7e50d64177da2e54}, // 1e-37 * 2**250 + {0xaa242499697392d2, 0xdde50bd1d5d0b9e9}, // 1e-36 * 2**247 + {0xd4ad2dbfc3d07787, 0x955e4ec64b44e864}, // 1e-35 * 2**244 + {0x84ec3c97da624ab4, 0xbd5af13bef0b113e}, // 1e-34 * 2**240 + {0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58e}, // 1e-33 * 2**237 + {0xcfb11ead453994ba, 0x67de18eda5814af2}, // 1e-32 * 2**234 + {0x81ceb32c4b43fcf4, 0x80eacf948770ced7}, // 1e-31 * 2**230 + {0xa2425ff75e14fc31, 0xa1258379a94d028d}, // 1e-30 * 2**227 + {0xcad2f7f5359a3b3e, 0x096ee45813a04330}, // 1e-29 * 2**224 + {0xfd87b5f28300ca0d, 0x8bca9d6e188853fc}, // 1e-28 * 2**221 + {0x9e74d1b791e07e48, 0x775ea264cf55347d}, // 1e-27 * 2**217 + {0xc612062576589dda, 0x95364afe032a819d}, // 1e-26 * 2**214 + {0xf79687aed3eec551, 0x3a83ddbd83f52204}, // 1e-25 * 2**211 + {0x9abe14cd44753b52, 0xc4926a9672793542}, // 1e-24 * 2**207 + {0xc16d9a0095928a27, 0x75b7053c0f178293}, // 1e-23 * 2**204 + {0xf1c90080baf72cb1, 0x5324c68b12dd6338}, // 1e-22 * 2**201 + {0x971da05074da7bee, 0xd3f6fc16ebca5e03}, // 1e-21 * 2**197 + {0xbce5086492111aea, 0x88f4bb1ca6bcf584}, // 1e-20 * 2**194 + {0xec1e4a7db69561a5, 0x2b31e9e3d06c32e5}, // 1e-19 * 2**191 + {0x9392ee8e921d5d07, 0x3aff322e62439fcf}, // 1e-18 * 2**187 + {0xb877aa3236a4b449, 0x09befeb9fad487c2}, // 1e-17 * 2**184 + {0xe69594bec44de15b, 0x4c2ebe687989a9b3}, // 1e-16 * 2**181 + {0x901d7cf73ab0acd9, 0x0f9d37014bf60a10}, // 1e-15 * 2**177 + {0xb424dc35095cd80f, 0x538484c19ef38c94}, // 1e-14 * 2**174 + {0xe12e13424bb40e13, 0x2865a5f206b06fb9}, // 1e-13 * 2**171 + {0x8cbccc096f5088cb, 0xf93f87b7442e45d3}, // 1e-12 * 2**167 + {0xafebff0bcb24aafe, 0xf78f69a51539d748}, // 1e-11 * 2**164 + {0xdbe6fecebdedd5be, 0xb573440e5a884d1b}, // 1e-10 * 2**161 + {0x89705f4136b4a597, 0x31680a88f8953030}, // 1e-9 * 2**157 + {0xabcc77118461cefc, 0xfdc20d2b36ba7c3d}, // 1e-8 * 2**154 + {0xd6bf94d5e57a42bc, 0x3d32907604691b4c}, // 1e-7 * 2**151 + {0x8637bd05af6c69b5, 0xa63f9a49c2c1b10f}, // 1e-6 * 2**147 + {0xa7c5ac471b478423, 0x0fcf80dc33721d53}, // 1e-5 * 2**144 + {0xd1b71758e219652b, 0xd3c36113404ea4a8}, // 1e-4 * 2**141 + {0x83126e978d4fdf3b, 0x645a1cac083126e9}, // 1e-3 * 2**137 + {0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a3}, // 1e-2 * 2**134 + {0xcccccccccccccccc, 0xcccccccccccccccc}, // 1e-1 * 2**131 + {0x8000000000000000, 0x0000000000000000}, // 1e0 * 2**127 + {0xa000000000000000, 0x0000000000000000}, // 1e1 * 2**124 + {0xc800000000000000, 0x0000000000000000}, // 1e2 * 2**121 + {0xfa00000000000000, 0x0000000000000000}, // 1e3 * 2**118 + {0x9c40000000000000, 0x0000000000000000}, // 1e4 * 2**114 + {0xc350000000000000, 0x0000000000000000}, // 1e5 * 2**111 + {0xf424000000000000, 0x0000000000000000}, // 1e6 * 2**108 + {0x9896800000000000, 0x0000000000000000}, // 1e7 * 2**104 + {0xbebc200000000000, 0x0000000000000000}, // 1e8 * 2**101 + {0xee6b280000000000, 0x0000000000000000}, // 1e9 * 2**98 + {0x9502f90000000000, 0x0000000000000000}, // 1e10 * 2**94 + {0xba43b74000000000, 0x0000000000000000}, // 1e11 * 2**91 + {0xe8d4a51000000000, 0x0000000000000000}, // 1e12 * 2**88 + {0x9184e72a00000000, 0x0000000000000000}, // 1e13 * 2**84 + {0xb5e620f480000000, 0x0000000000000000}, // 1e14 * 2**81 + {0xe35fa931a0000000, 0x0000000000000000}, // 1e15 * 2**78 + {0x8e1bc9bf04000000, 0x0000000000000000}, // 1e16 * 2**74 + {0xb1a2bc2ec5000000, 0x0000000000000000}, // 1e17 * 2**71 + {0xde0b6b3a76400000, 0x0000000000000000}, // 1e18 * 2**68 + {0x8ac7230489e80000, 0x0000000000000000}, // 1e19 * 2**64 + {0xad78ebc5ac620000, 0x0000000000000000}, // 1e20 * 2**61 + {0xd8d726b7177a8000, 0x0000000000000000}, // 1e21 * 2**58 + {0x878678326eac9000, 0x0000000000000000}, // 1e22 * 2**54 + {0xa968163f0a57b400, 0x0000000000000000}, // 1e23 * 2**51 + {0xd3c21bcecceda100, 0x0000000000000000}, // 1e24 * 2**48 + {0x84595161401484a0, 0x0000000000000000}, // 1e25 * 2**44 + {0xa56fa5b99019a5c8, 0x0000000000000000}, // 1e26 * 2**41 + {0xcecb8f27f4200f3a, 0x0000000000000000}, // 1e27 * 2**38 + {0x813f3978f8940984, 0x4000000000000000}, // 1e28 * 2**34 + {0xa18f07d736b90be5, 0x5000000000000000}, // 1e29 * 2**31 + {0xc9f2c9cd04674ede, 0xa400000000000000}, // 1e30 * 2**28 + {0xfc6f7c4045812296, 0x4d00000000000000}, // 1e31 * 2**25 + {0x9dc5ada82b70b59d, 0xf020000000000000}, // 1e32 * 2**21 + {0xc5371912364ce305, 0x6c28000000000000}, // 1e33 * 2**18 + {0xf684df56c3e01bc6, 0xc732000000000000}, // 1e34 * 2**15 + {0x9a130b963a6c115c, 0x3c7f400000000000}, // 1e35 * 2**11 + {0xc097ce7bc90715b3, 0x4b9f100000000000}, // 1e36 * 2**8 + {0xf0bdc21abb48db20, 0x1e86d40000000000}, // 1e37 * 2**5 + {0x96769950b50d88f4, 0x1314448000000000}, // 1e38 * 2**1 + {0xbc143fa4e250eb31, 0x17d955a000000000}, // 1e39 * 2**-2 + {0xeb194f8e1ae525fd, 0x5dcfab0800000000}, // 1e40 * 2**-5 + {0x92efd1b8d0cf37be, 0x5aa1cae500000000}, // 1e41 * 2**-9 + {0xb7abc627050305ad, 0xf14a3d9e40000000}, // 1e42 * 2**-12 + {0xe596b7b0c643c719, 0x6d9ccd05d0000000}, // 1e43 * 2**-15 + {0x8f7e32ce7bea5c6f, 0xe4820023a2000000}, // 1e44 * 2**-19 + {0xb35dbf821ae4f38b, 0xdda2802c8a800000}, // 1e45 * 2**-22 + {0xe0352f62a19e306e, 0xd50b2037ad200000}, // 1e46 * 2**-25 + {0x8c213d9da502de45, 0x4526f422cc340000}, // 1e47 * 2**-29 + {0xaf298d050e4395d6, 0x9670b12b7f410000}, // 1e48 * 2**-32 + {0xdaf3f04651d47b4c, 0x3c0cdd765f114000}, // 1e49 * 2**-35 + {0x88d8762bf324cd0f, 0xa5880a69fb6ac800}, // 1e50 * 2**-39 + {0xab0e93b6efee0053, 0x8eea0d047a457a00}, // 1e51 * 2**-42 + {0xd5d238a4abe98068, 0x72a4904598d6d880}, // 1e52 * 2**-45 + {0x85a36366eb71f041, 0x47a6da2b7f864750}, // 1e53 * 2**-49 + {0xa70c3c40a64e6c51, 0x999090b65f67d924}, // 1e54 * 2**-52 + {0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d}, // 1e55 * 2**-55 + {0x82818f1281ed449f, 0xbff8f10e7a8921a4}, // 1e56 * 2**-59 + {0xa321f2d7226895c7, 0xaff72d52192b6a0d}, // 1e57 * 2**-62 + {0xcbea6f8ceb02bb39, 0x9bf4f8a69f764490}, // 1e58 * 2**-65 + {0xfee50b7025c36a08, 0x02f236d04753d5b4}, // 1e59 * 2**-68 + {0x9f4f2726179a2245, 0x01d762422c946590}, // 1e60 * 2**-72 + {0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef5}, // 1e61 * 2**-75 + {0xf8ebad2b84e0d58b, 0xd2e0898765a7deb2}, // 1e62 * 2**-78 + {0x9b934c3b330c8577, 0x63cc55f49f88eb2f}, // 1e63 * 2**-82 + {0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fb}, // 1e64 * 2**-85 + {0xf316271c7fc3908a, 0x8bef464e3945ef7a}, // 1e65 * 2**-88 + {0x97edd871cfda3a56, 0x97758bf0e3cbb5ac}, // 1e66 * 2**-92 + {0xbde94e8e43d0c8ec, 0x3d52eeed1cbea317}, // 1e67 * 2**-95 + {0xed63a231d4c4fb27, 0x4ca7aaa863ee4bdd}, // 1e68 * 2**-98 + {0x945e455f24fb1cf8, 0x8fe8caa93e74ef6a}, // 1e69 * 2**-102 + {0xb975d6b6ee39e436, 0xb3e2fd538e122b44}, // 1e70 * 2**-105 + {0xe7d34c64a9c85d44, 0x60dbbca87196b616}, // 1e71 * 2**-108 + {0x90e40fbeea1d3a4a, 0xbc8955e946fe31cd}, // 1e72 * 2**-112 + {0xb51d13aea4a488dd, 0x6babab6398bdbe41}, // 1e73 * 2**-115 + {0xe264589a4dcdab14, 0xc696963c7eed2dd1}, // 1e74 * 2**-118 + {0x8d7eb76070a08aec, 0xfc1e1de5cf543ca2}, // 1e75 * 2**-122 + {0xb0de65388cc8ada8, 0x3b25a55f43294bcb}, // 1e76 * 2**-125 + {0xdd15fe86affad912, 0x49ef0eb713f39ebe}, // 1e77 * 2**-128 + {0x8a2dbf142dfcc7ab, 0x6e3569326c784337}, // 1e78 * 2**-132 + {0xacb92ed9397bf996, 0x49c2c37f07965404}, // 1e79 * 2**-135 + {0xd7e77a8f87daf7fb, 0xdc33745ec97be906}, // 1e80 * 2**-138 + {0x86f0ac99b4e8dafd, 0x69a028bb3ded71a3}, // 1e81 * 2**-142 + {0xa8acd7c0222311bc, 0xc40832ea0d68ce0c}, // 1e82 * 2**-145 + {0xd2d80db02aabd62b, 0xf50a3fa490c30190}, // 1e83 * 2**-148 + {0x83c7088e1aab65db, 0x792667c6da79e0fa}, // 1e84 * 2**-152 + {0xa4b8cab1a1563f52, 0x577001b891185938}, // 1e85 * 2**-155 + {0xcde6fd5e09abcf26, 0xed4c0226b55e6f86}, // 1e86 * 2**-158 + {0x80b05e5ac60b6178, 0x544f8158315b05b4}, // 1e87 * 2**-162 + {0xa0dc75f1778e39d6, 0x696361ae3db1c721}, // 1e88 * 2**-165 + {0xc913936dd571c84c, 0x03bc3a19cd1e38e9}, // 1e89 * 2**-168 + {0xfb5878494ace3a5f, 0x04ab48a04065c723}, // 1e90 * 2**-171 + {0x9d174b2dcec0e47b, 0x62eb0d64283f9c76}, // 1e91 * 2**-175 + {0xc45d1df942711d9a, 0x3ba5d0bd324f8394}, // 1e92 * 2**-178 + {0xf5746577930d6500, 0xca8f44ec7ee36479}, // 1e93 * 2**-181 + {0x9968bf6abbe85f20, 0x7e998b13cf4e1ecb}, // 1e94 * 2**-185 + {0xbfc2ef456ae276e8, 0x9e3fedd8c321a67e}, // 1e95 * 2**-188 + {0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101e}, // 1e96 * 2**-191 + {0x95d04aee3b80ece5, 0xbba1f1d158724a12}, // 1e97 * 2**-195 + {0xbb445da9ca61281f, 0x2a8a6e45ae8edc97}, // 1e98 * 2**-198 + {0xea1575143cf97226, 0xf52d09d71a3293bd}, // 1e99 * 2**-201 + {0x924d692ca61be758, 0x593c2626705f9c56}, // 1e100 * 2**-205 + {0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836c}, // 1e101 * 2**-208 + {0xe498f455c38b997a, 0x0b6dfb9c0f956447}, // 1e102 * 2**-211 + {0x8edf98b59a373fec, 0x4724bd4189bd5eac}, // 1e103 * 2**-215 + {0xb2977ee300c50fe7, 0x58edec91ec2cb657}, // 1e104 * 2**-218 + {0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ed}, // 1e105 * 2**-221 + {0x8b865b215899f46c, 0xbd79e0d20082ee74}, // 1e106 * 2**-225 + {0xae67f1e9aec07187, 0xecd8590680a3aa11}, // 1e107 * 2**-228 + {0xda01ee641a708de9, 0xe80e6f4820cc9495}, // 1e108 * 2**-231 + {0x884134fe908658b2, 0x3109058d147fdcdd}, // 1e109 * 2**-235 + {0xaa51823e34a7eede, 0xbd4b46f0599fd415}, // 1e110 * 2**-238 + {0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91a}, // 1e111 * 2**-241 + {0x850fadc09923329e, 0x03e2cf6bc604ddb0}, // 1e112 * 2**-245 + {0xa6539930bf6bff45, 0x84db8346b786151c}, // 1e113 * 2**-248 + {0xcfe87f7cef46ff16, 0xe612641865679a63}, // 1e114 * 2**-251 + {0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07e}, // 1e115 * 2**-255 + {0xa26da3999aef7749, 0xe3be5e330f38f09d}, // 1e116 * 2**-258 + {0xcb090c8001ab551c, 0x5cadf5bfd3072cc5}, // 1e117 * 2**-261 + {0xfdcb4fa002162a63, 0x73d9732fc7c8f7f6}, // 1e118 * 2**-264 + {0x9e9f11c4014dda7e, 0x2867e7fddcdd9afa}, // 1e119 * 2**-268 + {0xc646d63501a1511d, 0xb281e1fd541501b8}, // 1e120 * 2**-271 + {0xf7d88bc24209a565, 0x1f225a7ca91a4226}, // 1e121 * 2**-274 + {0x9ae757596946075f, 0x3375788de9b06958}, // 1e122 * 2**-278 + {0xc1a12d2fc3978937, 0x0052d6b1641c83ae}, // 1e123 * 2**-281 + {0xf209787bb47d6b84, 0xc0678c5dbd23a49a}, // 1e124 * 2**-284 + {0x9745eb4d50ce6332, 0xf840b7ba963646e0}, // 1e125 * 2**-288 + {0xbd176620a501fbff, 0xb650e5a93bc3d898}, // 1e126 * 2**-291 + {0xec5d3fa8ce427aff, 0xa3e51f138ab4cebe}, // 1e127 * 2**-294 + {0x93ba47c980e98cdf, 0xc66f336c36b10137}, // 1e128 * 2**-298 + {0xb8a8d9bbe123f017, 0xb80b0047445d4184}, // 1e129 * 2**-301 + {0xe6d3102ad96cec1d, 0xa60dc059157491e5}, // 1e130 * 2**-304 + {0x9043ea1ac7e41392, 0x87c89837ad68db2f}, // 1e131 * 2**-308 + {0xb454e4a179dd1877, 0x29babe4598c311fb}, // 1e132 * 2**-311 + {0xe16a1dc9d8545e94, 0xf4296dd6fef3d67a}, // 1e133 * 2**-314 + {0x8ce2529e2734bb1d, 0x1899e4a65f58660c}, // 1e134 * 2**-318 + {0xb01ae745b101e9e4, 0x5ec05dcff72e7f8f}, // 1e135 * 2**-321 + {0xdc21a1171d42645d, 0x76707543f4fa1f73}, // 1e136 * 2**-324 + {0x899504ae72497eba, 0x6a06494a791c53a8}, // 1e137 * 2**-328 + {0xabfa45da0edbde69, 0x0487db9d17636892}, // 1e138 * 2**-331 + {0xd6f8d7509292d603, 0x45a9d2845d3c42b6}, // 1e139 * 2**-334 + {0x865b86925b9bc5c2, 0x0b8a2392ba45a9b2}, // 1e140 * 2**-338 + {0xa7f26836f282b732, 0x8e6cac7768d7141e}, // 1e141 * 2**-341 + {0xd1ef0244af2364ff, 0x3207d795430cd926}, // 1e142 * 2**-344 + {0x8335616aed761f1f, 0x7f44e6bd49e807b8}, // 1e143 * 2**-348 + {0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a6}, // 1e144 * 2**-351 + {0xcd036837130890a1, 0x36dba887c37a8c0f}, // 1e145 * 2**-354 + {0x802221226be55a64, 0xc2494954da2c9789}, // 1e146 * 2**-358 + {0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6c}, // 1e147 * 2**-361 + {0xc83553c5c8965d3d, 0x6f92829494e5acc7}, // 1e148 * 2**-364 + {0xfa42a8b73abbf48c, 0xcb772339ba1f17f9}, // 1e149 * 2**-367 + {0x9c69a97284b578d7, 0xff2a760414536efb}, // 1e150 * 2**-371 + {0xc38413cf25e2d70d, 0xfef5138519684aba}, // 1e151 * 2**-374 + {0xf46518c2ef5b8cd1, 0x7eb258665fc25d69}, // 1e152 * 2**-377 + {0x98bf2f79d5993802, 0xef2f773ffbd97a61}, // 1e153 * 2**-381 + {0xbeeefb584aff8603, 0xaafb550ffacfd8fa}, // 1e154 * 2**-384 + {0xeeaaba2e5dbf6784, 0x95ba2a53f983cf38}, // 1e155 * 2**-387 + {0x952ab45cfa97a0b2, 0xdd945a747bf26183}, // 1e156 * 2**-391 + {0xba756174393d88df, 0x94f971119aeef9e4}, // 1e157 * 2**-394 + {0xe912b9d1478ceb17, 0x7a37cd5601aab85d}, // 1e158 * 2**-397 + {0x91abb422ccb812ee, 0xac62e055c10ab33a}, // 1e159 * 2**-401 + {0xb616a12b7fe617aa, 0x577b986b314d6009}, // 1e160 * 2**-404 + {0xe39c49765fdf9d94, 0xed5a7e85fda0b80b}, // 1e161 * 2**-407 + {0x8e41ade9fbebc27d, 0x14588f13be847307}, // 1e162 * 2**-411 + {0xb1d219647ae6b31c, 0x596eb2d8ae258fc8}, // 1e163 * 2**-414 + {0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bb}, // 1e164 * 2**-417 + {0x8aec23d680043bee, 0x25de7bb9480d5854}, // 1e165 * 2**-421 + {0xada72ccc20054ae9, 0xaf561aa79a10ae6a}, // 1e166 * 2**-424 + {0xd910f7ff28069da4, 0x1b2ba1518094da04}, // 1e167 * 2**-427 + {0x87aa9aff79042286, 0x90fb44d2f05d0842}, // 1e168 * 2**-431 + {0xa99541bf57452b28, 0x353a1607ac744a53}, // 1e169 * 2**-434 + {0xd3fa922f2d1675f2, 0x42889b8997915ce8}, // 1e170 * 2**-437 + {0x847c9b5d7c2e09b7, 0x69956135febada11}, // 1e171 * 2**-441 + {0xa59bc234db398c25, 0x43fab9837e699095}, // 1e172 * 2**-444 + {0xcf02b2c21207ef2e, 0x94f967e45e03f4bb}, // 1e173 * 2**-447 + {0x8161afb94b44f57d, 0x1d1be0eebac278f5}, // 1e174 * 2**-451 + {0xa1ba1ba79e1632dc, 0x6462d92a69731732}, // 1e175 * 2**-454 + {0xca28a291859bbf93, 0x7d7b8f7503cfdcfe}, // 1e176 * 2**-457 + {0xfcb2cb35e702af78, 0x5cda735244c3d43e}, // 1e177 * 2**-460 + {0x9defbf01b061adab, 0x3a0888136afa64a7}, // 1e178 * 2**-464 + {0xc56baec21c7a1916, 0x088aaa1845b8fdd0}, // 1e179 * 2**-467 + {0xf6c69a72a3989f5b, 0x8aad549e57273d45}, // 1e180 * 2**-470 + {0x9a3c2087a63f6399, 0x36ac54e2f678864b}, // 1e181 * 2**-474 + {0xc0cb28a98fcf3c7f, 0x84576a1bb416a7dd}, // 1e182 * 2**-477 + {0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d5}, // 1e183 * 2**-480 + {0x969eb7c47859e743, 0x9f644ae5a4b1b325}, // 1e184 * 2**-484 + {0xbc4665b596706114, 0x873d5d9f0dde1fee}, // 1e185 * 2**-487 + {0xeb57ff22fc0c7959, 0xa90cb506d155a7ea}, // 1e186 * 2**-490 + {0x9316ff75dd87cbd8, 0x09a7f12442d588f2}, // 1e187 * 2**-494 + {0xb7dcbf5354e9bece, 0x0c11ed6d538aeb2f}, // 1e188 * 2**-497 + {0xe5d3ef282a242e81, 0x8f1668c8a86da5fa}, // 1e189 * 2**-500 + {0x8fa475791a569d10, 0xf96e017d694487bc}, // 1e190 * 2**-504 + {0xb38d92d760ec4455, 0x37c981dcc395a9ac}, // 1e191 * 2**-507 + {0xe070f78d3927556a, 0x85bbe253f47b1417}, // 1e192 * 2**-510 + {0x8c469ab843b89562, 0x93956d7478ccec8e}, // 1e193 * 2**-514 + {0xaf58416654a6babb, 0x387ac8d1970027b2}, // 1e194 * 2**-517 + {0xdb2e51bfe9d0696a, 0x06997b05fcc0319e}, // 1e195 * 2**-520 + {0x88fcf317f22241e2, 0x441fece3bdf81f03}, // 1e196 * 2**-524 + {0xab3c2fddeeaad25a, 0xd527e81cad7626c3}, // 1e197 * 2**-527 + {0xd60b3bd56a5586f1, 0x8a71e223d8d3b074}, // 1e198 * 2**-530 + {0x85c7056562757456, 0xf6872d5667844e49}, // 1e199 * 2**-534 + {0xa738c6bebb12d16c, 0xb428f8ac016561db}, // 1e200 * 2**-537 + {0xd106f86e69d785c7, 0xe13336d701beba52}, // 1e201 * 2**-540 + {0x82a45b450226b39c, 0xecc0024661173473}, // 1e202 * 2**-544 + {0xa34d721642b06084, 0x27f002d7f95d0190}, // 1e203 * 2**-547 + {0xcc20ce9bd35c78a5, 0x31ec038df7b441f4}, // 1e204 * 2**-550 + {0xff290242c83396ce, 0x7e67047175a15271}, // 1e205 * 2**-553 + {0x9f79a169bd203e41, 0x0f0062c6e984d386}, // 1e206 * 2**-557 + {0xc75809c42c684dd1, 0x52c07b78a3e60868}, // 1e207 * 2**-560 + {0xf92e0c3537826145, 0xa7709a56ccdf8a82}, // 1e208 * 2**-563 + {0x9bbcc7a142b17ccb, 0x88a66076400bb691}, // 1e209 * 2**-567 + {0xc2abf989935ddbfe, 0x6acff893d00ea435}, // 1e210 * 2**-570 + {0xf356f7ebf83552fe, 0x0583f6b8c4124d43}, // 1e211 * 2**-573 + {0x98165af37b2153de, 0xc3727a337a8b704a}, // 1e212 * 2**-577 + {0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5c}, // 1e213 * 2**-580 + {0xeda2ee1c7064130c, 0x1162def06f79df73}, // 1e214 * 2**-583 + {0x9485d4d1c63e8be7, 0x8addcb5645ac2ba8}, // 1e215 * 2**-587 + {0xb9a74a0637ce2ee1, 0x6d953e2bd7173692}, // 1e216 * 2**-590 + {0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0437}, // 1e217 * 2**-593 + {0x910ab1d4db9914a0, 0x1d9c9892400a22a2}, // 1e218 * 2**-597 + {0xb54d5e4a127f59c8, 0x2503beb6d00cab4b}, // 1e219 * 2**-600 + {0xe2a0b5dc971f303a, 0x2e44ae64840fd61d}, // 1e220 * 2**-603 + {0x8da471a9de737e24, 0x5ceaecfed289e5d2}, // 1e221 * 2**-607 + {0xb10d8e1456105dad, 0x7425a83e872c5f47}, // 1e222 * 2**-610 + {0xdd50f1996b947518, 0xd12f124e28f77719}, // 1e223 * 2**-613 + {0x8a5296ffe33cc92f, 0x82bd6b70d99aaa6f}, // 1e224 * 2**-617 + {0xace73cbfdc0bfb7b, 0x636cc64d1001550b}, // 1e225 * 2**-620 + {0xd8210befd30efa5a, 0x3c47f7e05401aa4e}, // 1e226 * 2**-623 + {0x8714a775e3e95c78, 0x65acfaec34810a71}, // 1e227 * 2**-627 + {0xa8d9d1535ce3b396, 0x7f1839a741a14d0d}, // 1e228 * 2**-630 + {0xd31045a8341ca07c, 0x1ede48111209a050}, // 1e229 * 2**-633 + {0x83ea2b892091e44d, 0x934aed0aab460432}, // 1e230 * 2**-637 + {0xa4e4b66b68b65d60, 0xf81da84d5617853f}, // 1e231 * 2**-640 + {0xce1de40642e3f4b9, 0x36251260ab9d668e}, // 1e232 * 2**-643 + {0x80d2ae83e9ce78f3, 0xc1d72b7c6b426019}, // 1e233 * 2**-647 + {0xa1075a24e4421730, 0xb24cf65b8612f81f}, // 1e234 * 2**-650 + {0xc94930ae1d529cfc, 0xdee033f26797b627}, // 1e235 * 2**-653 + {0xfb9b7cd9a4a7443c, 0x169840ef017da3b1}, // 1e236 * 2**-656 + {0x9d412e0806e88aa5, 0x8e1f289560ee864e}, // 1e237 * 2**-660 + {0xc491798a08a2ad4e, 0xf1a6f2bab92a27e2}, // 1e238 * 2**-663 + {0xf5b5d7ec8acb58a2, 0xae10af696774b1db}, // 1e239 * 2**-666 + {0x9991a6f3d6bf1765, 0xacca6da1e0a8ef29}, // 1e240 * 2**-670 + {0xbff610b0cc6edd3f, 0x17fd090a58d32af3}, // 1e241 * 2**-673 + {0xeff394dcff8a948e, 0xddfc4b4cef07f5b0}, // 1e242 * 2**-676 + {0x95f83d0a1fb69cd9, 0x4abdaf101564f98e}, // 1e243 * 2**-680 + {0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f1}, // 1e244 * 2**-683 + {0xea53df5fd18d5513, 0x84c86189216dc5ed}, // 1e245 * 2**-686 + {0x92746b9be2f8552c, 0x32fd3cf5b4e49bb4}, // 1e246 * 2**-690 + {0xb7118682dbb66a77, 0x3fbc8c33221dc2a1}, // 1e247 * 2**-693 + {0xe4d5e82392a40515, 0x0fabaf3feaa5334a}, // 1e248 * 2**-696 + {0x8f05b1163ba6832d, 0x29cb4d87f2a7400e}, // 1e249 * 2**-700 + {0xb2c71d5bca9023f8, 0x743e20e9ef511012}, // 1e250 * 2**-703 + {0xdf78e4b2bd342cf6, 0x914da9246b255416}, // 1e251 * 2**-706 + {0x8bab8eefb6409c1a, 0x1ad089b6c2f7548e}, // 1e252 * 2**-710 + {0xae9672aba3d0c320, 0xa184ac2473b529b1}, // 1e253 * 2**-713 + {0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741e}, // 1e254 * 2**-716 + {0x8865899617fb1871, 0x7e2fa67c7a658892}, // 1e255 * 2**-720 + {0xaa7eebfb9df9de8d, 0xddbb901b98feeab7}, // 1e256 * 2**-723 + {0xd51ea6fa85785631, 0x552a74227f3ea565}, // 1e257 * 2**-726 + {0x8533285c936b35de, 0xd53a88958f87275f}, // 1e258 * 2**-730 + {0xa67ff273b8460356, 0x8a892abaf368f137}, // 1e259 * 2**-733 + {0xd01fef10a657842c, 0x2d2b7569b0432d85}, // 1e260 * 2**-736 + {0x8213f56a67f6b29b, 0x9c3b29620e29fc73}, // 1e261 * 2**-740 + {0xa298f2c501f45f42, 0x8349f3ba91b47b8f}, // 1e262 * 2**-743 + {0xcb3f2f7642717713, 0x241c70a936219a73}, // 1e263 * 2**-746 + {0xfe0efb53d30dd4d7, 0xed238cd383aa0110}, // 1e264 * 2**-749 + {0x9ec95d1463e8a506, 0xf4363804324a40aa}, // 1e265 * 2**-753 + {0xc67bb4597ce2ce48, 0xb143c6053edcd0d5}, // 1e266 * 2**-756 + {0xf81aa16fdc1b81da, 0xdd94b7868e94050a}, // 1e267 * 2**-759 + {0x9b10a4e5e9913128, 0xca7cf2b4191c8326}, // 1e268 * 2**-763 + {0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f0}, // 1e269 * 2**-766 + {0xf24a01a73cf2dccf, 0xbc633b39673c8cec}, // 1e270 * 2**-769 + {0x976e41088617ca01, 0xd5be0503e085d813}, // 1e271 * 2**-773 + {0xbd49d14aa79dbc82, 0x4b2d8644d8a74e18}, // 1e272 * 2**-776 + {0xec9c459d51852ba2, 0xddf8e7d60ed1219e}, // 1e273 * 2**-779 + {0x93e1ab8252f33b45, 0xcabb90e5c942b503}, // 1e274 * 2**-783 + {0xb8da1662e7b00a17, 0x3d6a751f3b936243}, // 1e275 * 2**-786 + {0xe7109bfba19c0c9d, 0x0cc512670a783ad4}, // 1e276 * 2**-789 + {0x906a617d450187e2, 0x27fb2b80668b24c5}, // 1e277 * 2**-793 + {0xb484f9dc9641e9da, 0xb1f9f660802dedf6}, // 1e278 * 2**-796 + {0xe1a63853bbd26451, 0x5e7873f8a0396973}, // 1e279 * 2**-799 + {0x8d07e33455637eb2, 0xdb0b487b6423e1e8}, // 1e280 * 2**-803 + {0xb049dc016abc5e5f, 0x91ce1a9a3d2cda62}, // 1e281 * 2**-806 + {0xdc5c5301c56b75f7, 0x7641a140cc7810fb}, // 1e282 * 2**-809 + {0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9d}, // 1e283 * 2**-813 + {0xac2820d9623bf429, 0x546345fa9fbdcd44}, // 1e284 * 2**-816 + {0xd732290fbacaf133, 0xa97c177947ad4095}, // 1e285 * 2**-819 + {0x867f59a9d4bed6c0, 0x49ed8eabcccc485d}, // 1e286 * 2**-823 + {0xa81f301449ee8c70, 0x5c68f256bfff5a74}, // 1e287 * 2**-826 + {0xd226fc195c6a2f8c, 0x73832eec6fff3111}, // 1e288 * 2**-829 + {0x83585d8fd9c25db7, 0xc831fd53c5ff7eab}, // 1e289 * 2**-833 + {0xa42e74f3d032f525, 0xba3e7ca8b77f5e55}, // 1e290 * 2**-836 + {0xcd3a1230c43fb26f, 0x28ce1bd2e55f35eb}, // 1e291 * 2**-839 + {0x80444b5e7aa7cf85, 0x7980d163cf5b81b3}, // 1e292 * 2**-843 + {0xa0555e361951c366, 0xd7e105bcc332621f}, // 1e293 * 2**-846 + {0xc86ab5c39fa63440, 0x8dd9472bf3fefaa7}, // 1e294 * 2**-849 + {0xfa856334878fc150, 0xb14f98f6f0feb951}, // 1e295 * 2**-852 + {0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d3}, // 1e296 * 2**-856 + {0xc3b8358109e84f07, 0x0a862f80ec4700c8}, // 1e297 * 2**-859 + {0xf4a642e14c6262c8, 0xcd27bb612758c0fa}, // 1e298 * 2**-862 + {0x98e7e9cccfbd7dbd, 0x8038d51cb897789c}, // 1e299 * 2**-866 + {0xbf21e44003acdd2c, 0xe0470a63e6bd56c3}, // 1e300 * 2**-869 + {0xeeea5d5004981478, 0x1858ccfce06cac74}, // 1e301 * 2**-872 + {0x95527a5202df0ccb, 0x0f37801e0c43ebc8}, // 1e302 * 2**-876 + {0xbaa718e68396cffd, 0xd30560258f54e6ba}, // 1e303 * 2**-879 + {0xe950df20247c83fd, 0x47c6b82ef32a2069}, // 1e304 * 2**-882 + {0x91d28b7416cdd27e, 0x4cdc331d57fa5441}, // 1e305 * 2**-886 + {0xb6472e511c81471d, 0xe0133fe4adf8e952}, // 1e306 * 2**-889 + {0xe3d8f9e563a198e5, 0x58180fddd97723a6}, // 1e307 * 2**-892 + {0x8e679c2f5e44ff8f, 0x570f09eaa7ea7648}, // 1e308 * 2**-896 + {0xb201833b35d63f73, 0x2cd2cc6551e513da}, // 1e309 * 2**-899 + {0xde81e40a034bcf4f, 0xf8077f7ea65e58d1}, // 1e310 * 2**-902 + {0x8b112e86420f6191, 0xfb04afaf27faf782}, // 1e311 * 2**-906 + {0xadd57a27d29339f6, 0x79c5db9af1f9b563}, // 1e312 * 2**-909 + {0xd94ad8b1c7380874, 0x18375281ae7822bc}, // 1e313 * 2**-912 + {0x87cec76f1c830548, 0x8f2293910d0b15b5}, // 1e314 * 2**-916 + {0xa9c2794ae3a3c69a, 0xb2eb3875504ddb22}, // 1e315 * 2**-919 + {0xd433179d9c8cb841, 0x5fa60692a46151eb}, // 1e316 * 2**-922 + {0x849feec281d7f328, 0xdbc7c41ba6bcd333}, // 1e317 * 2**-926 + {0xa5c7ea73224deff3, 0x12b9b522906c0800}, // 1e318 * 2**-929 + {0xcf39e50feae16bef, 0xd768226b34870a00}, // 1e319 * 2**-932 + {0x81842f29f2cce375, 0xe6a1158300d46640}, // 1e320 * 2**-936 + {0xa1e53af46f801c53, 0x60495ae3c1097fd0}, // 1e321 * 2**-939 + {0xca5e89b18b602368, 0x385bb19cb14bdfc4}, // 1e322 * 2**-942 + {0xfcf62c1dee382c42, 0x46729e03dd9ed7b5}, // 1e323 * 2**-945 + {0x9e19db92b4e31ba9, 0x6c07a2c26a8346d1}, // 1e324 * 2**-949 + {0xc5a05277621be293, 0xc7098b7305241885}, // 1e325 * 2**-952 + {0xf70867153aa2db38, 0xb8cbee4fc66d1ea7}, // 1e326 * 2**-955 + {0x9a65406d44a5c903, 0x737f74f1dc043328}, // 1e327 * 2**-959 + {0xc0fe908895cf3b44, 0x505f522e53053ff2}, // 1e328 * 2**-962 + {0xf13e34aabb430a15, 0x647726b9e7c68fef}, // 1e329 * 2**-965 + {0x96c6e0eab509e64d, 0x5eca783430dc19f5}, // 1e330 * 2**-969 + {0xbc789925624c5fe0, 0xb67d16413d132072}, // 1e331 * 2**-972 + {0xeb96bf6ebadf77d8, 0xe41c5bd18c57e88f}, // 1e332 * 2**-975 + {0x933e37a534cbaae7, 0x8e91b962f7b6f159}, // 1e333 * 2**-979 + {0xb80dc58e81fe95a1, 0x723627bbb5a4adb0}, // 1e334 * 2**-982 + {0xe61136f2227e3b09, 0xcec3b1aaa30dd91c}, // 1e335 * 2**-985 + {0x8fcac257558ee4e6, 0x213a4f0aa5e8a7b1}, // 1e336 * 2**-989 + {0xb3bd72ed2af29e1f, 0xa988e2cd4f62d19d}, // 1e337 * 2**-992 + {0xe0accfa875af45a7, 0x93eb1b80a33b8605}, // 1e338 * 2**-995 + {0x8c6c01c9498d8b88, 0xbc72f130660533c3}, // 1e339 * 2**-999 + {0xaf87023b9bf0ee6a, 0xeb8fad7c7f8680b4}, // 1e340 * 2**-1002 + {0xdb68c2ca82ed2a05, 0xa67398db9f6820e1}, // 1e341 * 2**-1005 + {0x892179be91d43a43, 0x88083f8943a1148c}, // 1e342 * 2**-1009 + {0xab69d82e364948d4, 0x6a0a4f6b948959b0}, // 1e343 * 2**-1012 + {0xd6444e39c3db9b09, 0x848ce34679abb01c}, // 1e344 * 2**-1015 + {0x85eab0e41a6940e5, 0xf2d80e0c0c0b4e11}, // 1e345 * 2**-1019 + {0xa7655d1d2103911f, 0x6f8e118f0f0e2195}, // 1e346 * 2**-1022 + {0xd13eb46469447567, 0x4b7195f2d2d1a9fb}, // 1e347 * 2**-1025 +} diff --git a/src/strconv/quote.go b/src/strconv/quote.go index 1f4929a952c6bf..da2325647d3817 100644 --- a/src/strconv/quote.go +++ b/src/strconv/quote.go @@ -37,12 +37,8 @@ func appendQuotedWith(buf []byte, s string, quote byte, ASCIIonly, graphicOnly b buf = nBuf } buf = append(buf, quote) - for width := 0; len(s) > 0; s = s[width:] { - r := rune(s[0]) - width = 1 - if r >= utf8.RuneSelf { - r, width = utf8.DecodeRuneInString(s) - } + for r, width := rune(0), 0; len(s) > 0; s = s[width:] { + r, width = utf8.DecodeRuneInString(s) if width == 1 && r == utf8.RuneError { buf = append(buf, `\x`...) buf = append(buf, lowerhex[s[0]>>4]) @@ -378,7 +374,8 @@ func QuotedPrefix(s string) (string, error) { // or backquoted Go string literal, returning the string value // that s quotes. (If s is single-quoted, it would be a Go // character literal; Unquote returns the corresponding -// one-character string. For '' Unquote returns the empty string.) +// one-character string. For an empty character literal +// Unquote returns the empty string.) func Unquote(s string) (string, error) { out, rem, err := unquote(s, true) if len(rem) > 0 { diff --git a/src/strings/builder.go b/src/strings/builder.go index e6df08c6f479ad..7ecef3176b2ce5 100644 --- a/src/strings/builder.go +++ b/src/strings/builder.go @@ -23,6 +23,12 @@ type Builder struct { buf []byte } +// copyCheck implements a dynamic check to prevent modification after +// copying a non-zero Builder, which would be unsafe (see #25907, #47276). +// +// We cannot add a noCopy field to Builder, to cause vet's copylocks +// check to report copying, because copylocks cannot reliably +// discriminate the zero and nonzero cases. func (b *Builder) copyCheck() { if b.addr == nil { // This hack works around a failing of Go's escape analysis diff --git a/src/strings/builder_test.go b/src/strings/builder_test.go index 36fd7a77e3abaa..06cd3e3b7ab77f 100644 --- a/src/strings/builder_test.go +++ b/src/strings/builder_test.go @@ -6,6 +6,7 @@ package strings_test import ( "bytes" + "internal/asan" . "strings" "testing" "unicode/utf8" @@ -89,6 +90,10 @@ func TestBuilderReset(t *testing.T) { func TestBuilderGrow(t *testing.T) { for _, growLen := range []int{0, 100, 1000, 10000, 100000} { + if asan.Enabled { + t.Logf("skipping allocs check for growLen %d: extra allocs with -asan; see #70079", growLen) + continue + } p := bytes.Repeat([]byte{'a'}, growLen) allocs := testing.AllocsPerRun(100, func() { var b Builder @@ -188,6 +193,9 @@ func TestBuilderWriteByte(t *testing.T) { } func TestBuilderAllocs(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } // Issue 23382; verify that copyCheck doesn't force the // Builder to escape and be heap allocated. n := testing.AllocsPerRun(10000, func() { @@ -387,6 +395,9 @@ func BenchmarkBuildString_ByteBuffer(b *testing.B) { } func TestBuilderGrowSizeclasses(t *testing.T) { + if asan.Enabled { + t.Skip("test allocates more with -asan; see #70079") + } s := Repeat("a", 19) allocs := testing.AllocsPerRun(100, func() { var b Builder diff --git a/src/strings/compare_test.go b/src/strings/compare_test.go index a43578423da013..2fd3df4435d8ae 100644 --- a/src/strings/compare_test.go +++ b/src/strings/compare_test.go @@ -36,9 +36,13 @@ var compareTests = []struct { func TestCompare(t *testing.T) { for _, tt := range compareTests { - cmp := Compare(tt.a, tt.b) - if cmp != tt.i { - t.Errorf(`Compare(%q, %q) = %v`, tt.a, tt.b, cmp) + numShifts := 16 + for offset := 0; offset <= numShifts; offset++ { + shiftedB := (Repeat("*", offset) + tt.b)[offset:] + cmp := Compare(tt.a, shiftedB) + if cmp != tt.i { + t.Errorf(`Compare(%q, %q), offset %d = %v; want %v`, tt.a, tt.b, offset, cmp, tt.i) + } } } } diff --git a/src/strings/example_test.go b/src/strings/example_test.go index bdab7ae8dea9d6..72adbae5f26a31 100644 --- a/src/strings/example_test.go +++ b/src/strings/example_test.go @@ -116,9 +116,9 @@ func ExampleCut() { } func ExampleCutPrefix() { - show := func(s, sep string) { - after, found := strings.CutPrefix(s, sep) - fmt.Printf("CutPrefix(%q, %q) = %q, %v\n", s, sep, after, found) + show := func(s, prefix string) { + after, found := strings.CutPrefix(s, prefix) + fmt.Printf("CutPrefix(%q, %q) = %q, %v\n", s, prefix, after, found) } show("Gopher", "Go") show("Gopher", "ph") @@ -128,9 +128,9 @@ func ExampleCutPrefix() { } func ExampleCutSuffix() { - show := func(s, sep string) { - before, found := strings.CutSuffix(s, sep) - fmt.Printf("CutSuffix(%q, %q) = %q, %v\n", s, sep, before, found) + show := func(s, suffix string) { + before, found := strings.CutSuffix(s, suffix) + fmt.Printf("CutSuffix(%q, %q) = %q, %v\n", s, suffix, before, found) } show("Gopher", "Go") show("Gopher", "er") @@ -328,22 +328,22 @@ func ExampleTitle() { // Compare this example to the ToTitle example. fmt.Println(strings.Title("her royal highness")) fmt.Println(strings.Title("loud noises")) - fmt.Println(strings.Title("хлеб")) + fmt.Println(strings.Title("брат")) // Output: // Her Royal Highness // Loud Noises - // Хлеб + // Брат } func ExampleToTitle() { // Compare this example to the Title example. fmt.Println(strings.ToTitle("her royal highness")) fmt.Println(strings.ToTitle("loud noises")) - fmt.Println(strings.ToTitle("хлеб")) + fmt.Println(strings.ToTitle("брат")) // Output: // HER ROYAL HIGHNESS // LOUD NOISES - // ХЛЕБ + // БРАТ } func ExampleToTitleSpecial() { @@ -388,8 +388,8 @@ func ExampleToLower() { } func ExampleToLowerSpecial() { - fmt.Println(strings.ToLowerSpecial(unicode.TurkishCase, "Önnek İş")) - // Output: önnek iş + fmt.Println(strings.ToLowerSpecial(unicode.TurkishCase, "Örnek İş")) + // Output: örnek iş } func ExampleTrim() { @@ -458,3 +458,93 @@ func ExampleToValidUTF8() { // abc // abc } + +func ExampleLines() { + text := "Hello\nWorld\nGo Programming\n" + for line := range strings.Lines(text) { + fmt.Printf("%q\n", line) + } + + // Output: + // "Hello\n" + // "World\n" + // "Go Programming\n" +} + +func ExampleSplitSeq() { + s := "a,b,c,d" + for part := range strings.SplitSeq(s, ",") { + fmt.Printf("%q\n", part) + } + + // Output: + // "a" + // "b" + // "c" + // "d" +} + +func ExampleSplitAfterSeq() { + s := "a,b,c,d" + for part := range strings.SplitAfterSeq(s, ",") { + fmt.Printf("%q\n", part) + } + + // Output: + // "a," + // "b," + // "c," + // "d" +} + +func ExampleFieldsSeq() { + text := "The quick brown fox" + fmt.Println("Split string into fields:") + for word := range strings.FieldsSeq(text) { + fmt.Printf("%q\n", word) + } + + textWithSpaces := " lots of spaces " + fmt.Println("\nSplit string with multiple spaces:") + for word := range strings.FieldsSeq(textWithSpaces) { + fmt.Printf("%q\n", word) + } + + // Output: + // Split string into fields: + // "The" + // "quick" + // "brown" + // "fox" + // + // Split string with multiple spaces: + // "lots" + // "of" + // "spaces" +} + +func ExampleFieldsFuncSeq() { + text := "The quick brown fox" + fmt.Println("Split on whitespace(similar to FieldsSeq):") + for word := range strings.FieldsFuncSeq(text, unicode.IsSpace) { + fmt.Printf("%q\n", word) + } + + mixedText := "abc123def456ghi" + fmt.Println("\nSplit on digits:") + for word := range strings.FieldsFuncSeq(mixedText, unicode.IsDigit) { + fmt.Printf("%q\n", word) + } + + // Output: + // Split on whitespace(similar to FieldsSeq): + // "The" + // "quick" + // "brown" + // "fox" + // + // Split on digits: + // "abc" + // "def" + // "ghi" +} diff --git a/src/strings/iter.go b/src/strings/iter.go new file mode 100644 index 00000000000000..84e763a8343df4 --- /dev/null +++ b/src/strings/iter.go @@ -0,0 +1,137 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package strings + +import ( + "iter" + "unicode" + "unicode/utf8" +) + +// Lines returns an iterator over the newline-terminated lines in the string s. +// The lines yielded by the iterator include their terminating newlines. +// If s is empty, the iterator yields no lines at all. +// If s does not end in a newline, the final yielded line will not end in a newline. +// It returns a single-use iterator. +func Lines(s string) iter.Seq[string] { + return func(yield func(string) bool) { + for len(s) > 0 { + var line string + if i := IndexByte(s, '\n'); i >= 0 { + line, s = s[:i+1], s[i+1:] + } else { + line, s = s, "" + } + if !yield(line) { + return + } + } + } +} + +// splitSeq is SplitSeq or SplitAfterSeq, configured by how many +// bytes of sep to include in the results (none or all). +func splitSeq(s, sep string, sepSave int) iter.Seq[string] { + return func(yield func(string) bool) { + if len(sep) == 0 { + for len(s) > 0 { + _, size := utf8.DecodeRuneInString(s) + if !yield(s[:size]) { + return + } + s = s[size:] + } + return + } + for { + i := Index(s, sep) + if i < 0 { + break + } + frag := s[:i+sepSave] + if !yield(frag) { + return + } + s = s[i+len(sep):] + } + yield(s) + } +} + +// SplitSeq returns an iterator over all substrings of s separated by sep. +// The iterator yields the same strings that would be returned by [Split](s, sep), +// but without constructing the slice. +// It returns a single-use iterator. +func SplitSeq(s, sep string) iter.Seq[string] { + return splitSeq(s, sep, 0) +} + +// SplitAfterSeq returns an iterator over substrings of s split after each instance of sep. +// The iterator yields the same strings that would be returned by [SplitAfter](s, sep), +// but without constructing the slice. +// It returns a single-use iterator. +func SplitAfterSeq(s, sep string) iter.Seq[string] { + return splitSeq(s, sep, len(sep)) +} + +// FieldsSeq returns an iterator over substrings of s split around runs of +// whitespace characters, as defined by [unicode.IsSpace]. +// The iterator yields the same strings that would be returned by [Fields](s), +// but without constructing the slice. +func FieldsSeq(s string) iter.Seq[string] { + return func(yield func(string) bool) { + start := -1 + for i := 0; i < len(s); { + size := 1 + r := rune(s[i]) + isSpace := asciiSpace[s[i]] != 0 + if r >= utf8.RuneSelf { + r, size = utf8.DecodeRuneInString(s[i:]) + isSpace = unicode.IsSpace(r) + } + if isSpace { + if start >= 0 { + if !yield(s[start:i]) { + return + } + start = -1 + } + } else if start < 0 { + start = i + } + i += size + } + if start >= 0 { + yield(s[start:]) + } + } +} + +// FieldsFuncSeq returns an iterator over substrings of s split around runs of +// Unicode code points satisfying f(c). +// The iterator yields the same strings that would be returned by [FieldsFunc](s), +// but without constructing the slice. +func FieldsFuncSeq(s string, f func(rune) bool) iter.Seq[string] { + return func(yield func(string) bool) { + start := -1 + for i := 0; i < len(s); { + r, size := utf8.DecodeRuneInString(s[i:]) + if f(r) { + if start >= 0 { + if !yield(s[start:i]) { + return + } + start = -1 + } + } else if start < 0 { + start = i + } + i += size + } + if start >= 0 { + yield(s[start:]) + } + } +} diff --git a/src/strings/iter_test.go b/src/strings/iter_test.go new file mode 100644 index 00000000000000..963349b1f41441 --- /dev/null +++ b/src/strings/iter_test.go @@ -0,0 +1,105 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package strings_test + +import ( + . "strings" + "testing" +) + +func BenchmarkSplitSeqEmptySeparator(b *testing.B) { + for range b.N { + for range SplitSeq(benchInputHard, "") { + } + } +} + +func BenchmarkSplitSeqSingleByteSeparator(b *testing.B) { + for range b.N { + for range SplitSeq(benchInputHard, "/") { + } + } +} + +func BenchmarkSplitSeqMultiByteSeparator(b *testing.B) { + for range b.N { + for range SplitSeq(benchInputHard, "hello") { + } + } +} + +func BenchmarkSplitAfterSeqEmptySeparator(b *testing.B) { + for range b.N { + for range SplitAfterSeq(benchInputHard, "") { + } + } +} + +func BenchmarkSplitAfterSeqSingleByteSeparator(b *testing.B) { + for range b.N { + for range SplitAfterSeq(benchInputHard, "/") { + } + } +} + +func BenchmarkSplitAfterSeqMultiByteSeparator(b *testing.B) { + for range b.N { + for range SplitAfterSeq(benchInputHard, "hello") { + } + } +} + +func findKvBySplit(s string, k string) string { + for _, kv := range Split(s, ",") { + if HasPrefix(kv, k) { + return kv + } + } + return "" +} + +func findKvBySplitSeq(s string, k string) string { + for kv := range SplitSeq(s, ",") { + if HasPrefix(kv, k) { + return kv + } + } + return "" +} + +func BenchmarkSplitAndSplitSeq(b *testing.B) { + testSplitString := "k1=v1,k2=v2,k3=v3,k4=v4" + testCases := []struct { + name string + input string + }{ + { + name: "Key found", + input: "k3", + }, + { + name: "Key not found", + input: "k100", + }, + } + + for _, testCase := range testCases { + b.Run("bySplit "+testCase.name, func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + for b.Loop() { + findKvBySplit(testSplitString, testCase.input) + } + }) + + b.Run("bySplitSeq "+testCase.name, func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + for b.Loop() { + findKvBySplitSeq(testSplitString, testCase.input) + } + }) + } +} diff --git a/src/strings/reader.go b/src/strings/reader.go index 497ffb7a39c635..f12c9b18b36d43 100644 --- a/src/strings/reader.go +++ b/src/strings/reader.go @@ -90,10 +90,6 @@ func (r *Reader) ReadRune() (ch rune, size int, err error) { return 0, 0, io.EOF } r.prevRune = int(r.i) - if c := r.s[r.i]; c < utf8.RuneSelf { - r.i++ - return rune(c), 1, nil - } ch, size = utf8.DecodeRuneInString(r.s[r.i:]) r.i += int64(size) return diff --git a/src/strings/strings.go b/src/strings/strings.go index 0bd3c1c2337801..80cea67d626d2e 100644 --- a/src/strings/strings.go +++ b/src/strings/strings.go @@ -10,6 +10,7 @@ package strings import ( "internal/bytealg" "internal/stringslite" + "math/bits" "unicode" "unicode/utf8" ) @@ -124,6 +125,7 @@ func IndexByte(s string, c byte) int { // If r is [utf8.RuneError], it returns the first instance of any // invalid UTF-8 byte sequence. func IndexRune(s string, r rune) int { + const haveFastIndex = bytealg.MaxBruteForce > 0 switch { case 0 <= r && r < utf8.RuneSelf: return IndexByte(s, byte(r)) @@ -137,7 +139,60 @@ func IndexRune(s string, r rune) int { case !utf8.ValidRune(r): return -1 default: - return Index(s, string(r)) + // Search for rune r using the last byte of its UTF-8 encoded form. + // The distribution of the last byte is more uniform compared to the + // first byte which has a 78% chance of being [240, 243, 244]. + rs := string(r) + last := len(rs) - 1 + i := last + fails := 0 + for i < len(s) { + if s[i] != rs[last] { + o := IndexByte(s[i+1:], rs[last]) + if o < 0 { + return -1 + } + i += o + 1 + } + // Step backwards comparing bytes. + for j := 1; j < len(rs); j++ { + if s[i-j] != rs[last-j] { + goto next + } + } + return i - last + next: + fails++ + i++ + if (haveFastIndex && fails > bytealg.Cutover(i)) && i < len(s) || + (!haveFastIndex && fails >= 4+i>>4 && i < len(s)) { + goto fallback + } + } + return -1 + + fallback: + // see comment in ../bytes/bytes.go + if haveFastIndex { + if j := bytealg.IndexString(s[i-last:], string(r)); j >= 0 { + return i + j - last + } + } else { + c0 := rs[last] + c1 := rs[last-1] + loop: + for ; i < len(s); i++ { + if s[i] == c0 && s[i-1] == c1 { + for k := 2; k < len(rs); k++ { + if s[i-k] != rs[last-k] { + continue loop + } + } + return i - last + } + } + } + return -1 } } @@ -323,7 +378,9 @@ var asciiSpace = [256]uint8{'\t': 1, '\n': 1, '\v': 1, '\f': 1, '\r': 1, ' ': 1} // Fields splits the string s around each instance of one or more consecutive white space // characters, as defined by [unicode.IsSpace], returning a slice of substrings of s or an -// empty slice if s contains only white space. +// empty slice if s contains only white space. Every element of the returned slice is +// non-empty. Unlike [Split], leading and trailing runs of white space characters +// are discarded. func Fields(s string) []string { // First count the fields. // This is an exact count if s is ASCII, otherwise it is an approximation. @@ -375,7 +432,9 @@ func Fields(s string) []string { // FieldsFunc splits the string s at each run of Unicode code points c satisfying f(c) // and returns an array of slices of s. If all code points in s satisfy f(c) or the -// string is empty, an empty slice is returned. +// string is empty, an empty slice is returned. Every element of the returned slice is +// non-empty. Unlike [Split], leading and trailing runs of code points satisfying f(c) +// are discarded. // // FieldsFunc makes no guarantees about the order in which it calls f(c) // and assumes that f always returns the same value for a given c. @@ -568,10 +627,11 @@ func Repeat(s string, count int) string { if count < 0 { panic("strings: negative Repeat count") } - if len(s) > maxInt/count { + hi, lo := bits.Mul(uint(len(s)), uint(count)) + if hi > 0 || lo > uint(maxInt) { panic("strings: Repeat output length overflow") } - n := len(s) * count + n := int(lo) // lo = len(s) * count if len(s) == 0 { return "" @@ -617,13 +677,7 @@ func Repeat(s string, count int) string { b.Grow(n) b.WriteString(s) for b.Len() < n { - chunk := n - b.Len() - if chunk > b.Len() { - chunk = b.Len() - } - if chunk > chunkMax { - chunk = chunkMax - } + chunk := min(n-b.Len(), b.Len(), chunkMax) b.WriteString(b.String()[:chunk]) } return b.String() @@ -842,7 +896,7 @@ func TrimLeftFunc(s string, f func(rune) bool) string { // Unicode code points c satisfying f(c) removed. func TrimRightFunc(s string, f func(rune) bool) string { i := lastIndexFunc(s, f, false) - if i >= 0 && s[i] >= utf8.RuneSelf { + if i >= 0 { _, wid := utf8.DecodeRuneInString(s[i:]) i += wid } else { @@ -974,10 +1028,7 @@ func trimLeftASCII(s string, as *asciiSet) string { func trimLeftUnicode(s, cutset string) string { for len(s) > 0 { - r, n := rune(s[0]), 1 - if r >= utf8.RuneSelf { - r, n = utf8.DecodeRuneInString(s) - } + r, n := utf8.DecodeRuneInString(s) if !ContainsRune(cutset, r) { break } @@ -1034,40 +1085,36 @@ func trimRightUnicode(s, cutset string) string { return s } -// TrimSpace returns a slice of the string s, with all leading -// and trailing white space removed, as defined by Unicode. +// TrimSpace returns a slice (substring) of the string s, +// with all leading and trailing white space removed, +// as defined by Unicode. func TrimSpace(s string) string { - // Fast path for ASCII: look for the first ASCII non-space byte - start := 0 - for ; start < len(s); start++ { - c := s[start] + // Fast path for ASCII: look for the first ASCII non-space byte. + for lo, c := range []byte(s) { if c >= utf8.RuneSelf { // If we run into a non-ASCII byte, fall back to the - // slower unicode-aware method on the remaining bytes - return TrimFunc(s[start:], unicode.IsSpace) + // slower unicode-aware method on the remaining bytes. + return TrimFunc(s[lo:], unicode.IsSpace) } - if asciiSpace[c] == 0 { - break - } - } - - // Now look for the first ASCII non-space byte from the end - stop := len(s) - for ; stop > start; stop-- { - c := s[stop-1] - if c >= utf8.RuneSelf { - // start has been already trimmed above, should trim end only - return TrimRightFunc(s[start:stop], unicode.IsSpace) + if asciiSpace[c] != 0 { + continue } - if asciiSpace[c] == 0 { - break + s = s[lo:] + // Now look for the first ASCII non-space byte from the end. + for hi := len(s) - 1; hi >= 0; hi-- { + c := s[hi] + if c >= utf8.RuneSelf { + return TrimRightFunc(s[:hi+1], unicode.IsSpace) + } + if asciiSpace[c] == 0 { + // At this point, s[:hi+1] starts and ends with ASCII + // non-space bytes, so we're done. Non-ASCII cases have + // already been handled above. + return s[:hi+1] + } } } - - // At this point s[start:stop] starts and ends with an ASCII - // non-space bytes, so we're done. Non-ASCII cases have already - // been handled above. - return s[start:stop] + return "" } // TrimPrefix returns s without the provided leading prefix string. @@ -1104,19 +1151,22 @@ func Replace(s, old, new string, n int) string { var b Builder b.Grow(len(s) + n*(len(new)-len(old))) start := 0 - for i := 0; i < n; i++ { - j := start - if len(old) == 0 { - if i > 0 { - _, wid := utf8.DecodeRuneInString(s[start:]) - j += wid - } - } else { - j += Index(s[start:], old) - } - b.WriteString(s[start:j]) + if len(old) > 0 { + for range n { + j := start + Index(s[start:], old) + b.WriteString(s[start:j]) + b.WriteString(new) + start = j + len(old) + } + } else { // len(old) == 0 b.WriteString(new) - start = j + len(old) + for range n - 1 { + _, wid := utf8.DecodeRuneInString(s[start:]) + j := start + wid + b.WriteString(s[start:j]) + b.WriteString(new) + start = j + } } b.WriteString(s[start:]) return b.String() @@ -1137,7 +1187,7 @@ func ReplaceAll(s, old, new string) string { func EqualFold(s, t string) bool { // ASCII fast path i := 0 - for ; i < len(s) && i < len(t); i++ { + for n := min(len(s), len(t)); i < n; i++ { sr := s[i] tr := t[i] if sr|tr >= utf8.RuneSelf { @@ -1172,13 +1222,8 @@ hasUnicode: } // Extract first rune from second string. - var tr rune - if t[0] < utf8.RuneSelf { - tr, t = rune(t[0]), t[1:] - } else { - r, size := utf8.DecodeRuneInString(t) - tr, t = r, t[size:] - } + tr, size := utf8.DecodeRuneInString(t) + t = t[size:] // If they match, keep going; if not, return false. diff --git a/src/strings/strings_test.go b/src/strings/strings_test.go index 4c8c25ee132a28..b10b5f05ccae53 100644 --- a/src/strings/strings_test.go +++ b/src/strings/strings_test.go @@ -7,7 +7,9 @@ package strings_test import ( "bytes" "fmt" + "internal/asan" "io" + "iter" "math" "math/rand" "slices" @@ -19,16 +21,35 @@ import ( "unsafe" ) -func eq(a, b []string) bool { - if len(a) != len(b) { - return false +func collect(t *testing.T, seq iter.Seq[string]) []string { + out := slices.Collect(seq) + out1 := slices.Collect(seq) + if !slices.Equal(out, out1) { + t.Fatalf("inconsistent seq:\n%s\n%s", out, out1) } - for i := 0; i < len(a); i++ { - if a[i] != b[i] { - return false + return out +} + +type LinesTest struct { + a string + b []string +} + +var linesTests = []LinesTest{ + {a: "abc\nabc\n", b: []string{"abc\n", "abc\n"}}, + {a: "abc\r\nabc", b: []string{"abc\r\n", "abc"}}, + {a: "abc\r\n", b: []string{"abc\r\n"}}, + {a: "\nabc", b: []string{"\n", "abc"}}, + {a: "\nabc\n\n", b: []string{"\n", "abc\n", "\n"}}, +} + +func TestLines(t *testing.T) { + for _, s := range linesTests { + result := slices.Collect(Lines(s.a)) + if !slices.Equal(result, s.b) { + t.Errorf(`slices.Collect(Lines(%q)) = %q; want %q`, s.a, result, s.b) } } - return true } var abcd = "abcd" @@ -135,6 +156,11 @@ var indexTests = []IndexTest{ // test fallback to Rabin-Karp. {"oxoxoxoxoxoxoxoxoxoxoxoy", "oy", 22}, {"oxoxoxoxoxoxoxoxoxoxoxox", "oy", -1}, + // test fallback to IndexRune + {"oxoxoxoxoxoxoxoxoxoxox☺", "☺", 22}, + // invalid UTF-8 byte sequence (must be longer than bytealg.MaxBruteForce to + // test that we don't use IndexRune) + {"xx0123456789012345678901234567890123456789012345678901234567890120123456789012345678901234567890123456xxx\xed\x9f\xc0", "\xed\x9f\xc0", 105}, } var lastIndexTests = []IndexTest{ @@ -306,6 +332,37 @@ func TestIndexRune(t *testing.T) { {"a☺b☻c☹d\xe2\x98�\xff�\xed\xa0\x80", -1, -1}, {"a☺b☻c☹d\xe2\x98�\xff�\xed\xa0\x80", 0xD800, -1}, // Surrogate pair {"a☺b☻c☹d\xe2\x98�\xff�\xed\xa0\x80", utf8.MaxRune + 1, -1}, + + // 2 bytes + {"ӆ", 'ӆ', 0}, + {"a", 'ӆ', -1}, + {" ӆ", 'ӆ', 2}, + {" a", 'ӆ', -1}, + {Repeat("ц", 64) + "ӆ", 'ӆ', 128}, // test cutover + {Repeat("Ꙁ", 64) + "Ꚁ", '䚀', -1}, // 'Ꚁ' and '䚀' share the same last two bytes + + // 3 bytes + {"Ꚁ", 'Ꚁ', 0}, + {"a", 'Ꚁ', -1}, + {" Ꚁ", 'Ꚁ', 2}, + {" a", 'Ꚁ', -1}, + {Repeat("Ꙁ", 64) + "Ꚁ", 'Ꚁ', 192}, // test cutover + {Repeat("𡋀", 64) + "𡌀", '𣌀', -1}, // '𡌀' and '𣌀' share the same last two bytes + + // 4 bytes + {"𡌀", '𡌀', 0}, + {"a", '𡌀', -1}, + {" 𡌀", '𡌀', 2}, + {" a", '𡌀', -1}, + {Repeat("𡋀", 64) + "𡌀", '𡌀', 256}, // test cutover + {Repeat("𡋀", 64), '𡌀', -1}, + + // Test the cutover to bytealg.IndexString when it is triggered in + // the middle of rune that contains consecutive runs of equal bytes. + {"aaaaaKKKK\U000bc104", '\U000bc104', 17}, // cutover: (n + 16) / 8 + {"aaaaaKKKK鄄", '鄄', 17}, + {"aaKKKKKa\U000bc104", '\U000bc104', 18}, // cutover: 4 + n>>4 + {"aaKKKKKa鄄", '鄄', 18}, } for _, tt := range tests { if got := IndexRune(tt.in, tt.rune); got != tt.want { @@ -313,13 +370,14 @@ func TestIndexRune(t *testing.T) { } } - haystack := "test世界" + // Make sure we trigger the cutover and string(rune) conversion. + haystack := "test" + Repeat("𡋀", 32) + "𡌀" allocs := testing.AllocsPerRun(1000, func() { if i := IndexRune(haystack, 's'); i != 2 { t.Fatalf("'s' at %d; want 2", i) } - if i := IndexRune(haystack, '世'); i != 4 { - t.Fatalf("'世' at %d; want 4", i) + if i := IndexRune(haystack, '𡌀'); i != 132 { + t.Fatalf("'𡌀' at %d; want 4", i) } }) if allocs != 0 && testing.CoverMode() == "" { @@ -418,10 +476,16 @@ var splittests = []SplitTest{ func TestSplit(t *testing.T) { for _, tt := range splittests { a := SplitN(tt.s, tt.sep, tt.n) - if !eq(a, tt.a) { + if !slices.Equal(a, tt.a) { t.Errorf("Split(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, a, tt.a) continue } + if tt.n < 0 { + a2 := slices.Collect(SplitSeq(tt.s, tt.sep)) + if !slices.Equal(a2, tt.a) { + t.Errorf(`collect(SplitSeq(%q, %q)) = %v; want %v`, tt.s, tt.sep, a2, tt.a) + } + } if tt.n == 0 { continue } @@ -457,10 +521,16 @@ var splitaftertests = []SplitTest{ func TestSplitAfter(t *testing.T) { for _, tt := range splitaftertests { a := SplitAfterN(tt.s, tt.sep, tt.n) - if !eq(a, tt.a) { + if !slices.Equal(a, tt.a) { t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, a, tt.a) continue } + if tt.n < 0 { + a2 := slices.Collect(SplitAfterSeq(tt.s, tt.sep)) + if !slices.Equal(a2, tt.a) { + t.Errorf(`collect(SplitAfterSeq(%q, %q)) = %v; want %v`, tt.s, tt.sep, a2, tt.a) + } + } s := Join(a, "") if s != tt.s { t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s) @@ -500,10 +570,14 @@ var fieldstests = []FieldsTest{ func TestFields(t *testing.T) { for _, tt := range fieldstests { a := Fields(tt.s) - if !eq(a, tt.a) { + if !slices.Equal(a, tt.a) { t.Errorf("Fields(%q) = %v; want %v", tt.s, a, tt.a) continue } + a2 := collect(t, FieldsSeq(tt.s)) + if !slices.Equal(a2, tt.a) { + t.Errorf(`collect(FieldsSeq(%q)) = %v; want %v`, tt.s, a2, tt.a) + } } } @@ -517,7 +591,7 @@ var FieldsFuncTests = []FieldsTest{ func TestFieldsFunc(t *testing.T) { for _, tt := range fieldstests { a := FieldsFunc(tt.s, unicode.IsSpace) - if !eq(a, tt.a) { + if !slices.Equal(a, tt.a) { t.Errorf("FieldsFunc(%q, unicode.IsSpace) = %v; want %v", tt.s, a, tt.a) continue } @@ -525,9 +599,13 @@ func TestFieldsFunc(t *testing.T) { pred := func(c rune) bool { return c == 'X' } for _, tt := range FieldsFuncTests { a := FieldsFunc(tt.s, pred) - if !eq(a, tt.a) { + if !slices.Equal(a, tt.a) { t.Errorf("FieldsFunc(%q) = %v, want %v", tt.s, a, tt.a) } + a2 := collect(t, FieldsFuncSeq(tt.s, pred)) + if !slices.Equal(a2, tt.a) { + t.Errorf(`collect(FieldsFuncSeq(%q)) = %v; want %v`, tt.s, a2, tt.a) + } } } @@ -1396,6 +1474,12 @@ var ReplaceTests = []struct { func TestReplace(t *testing.T) { for _, tt := range ReplaceTests { + if !asan.Enabled { // See issue #72973. + allocs := testing.AllocsPerRun(10, func() { Replace(tt.in, tt.old, tt.new, tt.n) }) + if allocs > 1 { + t.Errorf("Replace(%q, %q, %q, %d) allocates %.2f objects", tt.in, tt.old, tt.new, tt.n, allocs) + } + } if s := Replace(tt.in, tt.old, tt.new, tt.n); s != tt.out { t.Errorf("Replace(%q, %q, %q, %d) = %q, want %q", tt.in, tt.old, tt.new, tt.n, s, tt.out) } @@ -1408,6 +1492,64 @@ func TestReplace(t *testing.T) { } } +func FuzzReplace(f *testing.F) { + for _, tt := range ReplaceTests { + f.Add(tt.in, tt.old, tt.new, tt.n) + } + f.Fuzz(func(t *testing.T, in, old, new string, n int) { + differentImpl := func(in, old, new string, n int) string { + var out Builder + if n < 0 { + n = math.MaxInt + } + for i := 0; i < len(in); { + if n == 0 { + out.WriteString(in[i:]) + break + } + if HasPrefix(in[i:], old) { + out.WriteString(new) + i += len(old) + n-- + if len(old) != 0 { + continue + } + if i == len(in) { + break + } + } + if len(old) == 0 { + _, length := utf8.DecodeRuneInString(in[i:]) + out.WriteString(in[i : i+length]) + i += length + } else { + out.WriteByte(in[i]) + i++ + } + } + if len(old) == 0 && n != 0 { + out.WriteString(new) + } + return out.String() + } + if simple, replace := differentImpl(in, old, new, n), Replace(in, old, new, n); simple != replace { + t.Errorf("The two implementations do not match %q != %q for Replace(%q, %q, %q, %d)", simple, replace, in, old, new, n) + } + }) +} + +func BenchmarkReplace(b *testing.B) { + for _, tt := range ReplaceTests { + desc := fmt.Sprintf("%q %q %q %d", tt.in, tt.old, tt.new, tt.n) + b.Run(desc, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + Replace(tt.in, tt.old, tt.new, tt.n) + } + }) + } +} + var TitleTests = []struct { in, out string }{ @@ -1733,8 +1875,9 @@ func makeBenchInputHard() string { "hello", "world", } x := make([]byte, 0, 1<<20) + r := rand.New(rand.NewSource(99)) for { - i := rand.Intn(len(tokens)) + i := r.Intn(len(tokens)) if len(x)+len(tokens[i]) >= 1<<20 { break } @@ -1822,8 +1965,9 @@ func BenchmarkCountByte(b *testing.B) { var makeFieldsInput = func() string { x := make([]byte, 1<<20) // Input is ~10% space, ~10% 2-byte UTF-8, rest ASCII non-space. + r := rand.New(rand.NewSource(99)) for i := range x { - switch rand.Intn(10) { + switch r.Intn(10) { case 0: x[i] = ' ' case 1: @@ -1842,8 +1986,9 @@ var makeFieldsInput = func() string { var makeFieldsInputASCII = func() string { x := make([]byte, 1<<20) // Input is ~10% space, rest ASCII non-space. + r := rand.New(rand.NewSource(99)) for i := range x { - if rand.Intn(10) == 0 { + if r.Intn(10) == 0 { x[i] = ' ' } else { x[i] = 'x' diff --git a/src/sync/atomic/doc.go b/src/sync/atomic/doc.go index 7f9d64b74e624c..4e93404757146a 100644 --- a/src/sync/atomic/doc.go +++ b/src/sync/atomic/doc.go @@ -60,29 +60,26 @@ import ( // for 64-bit alignment of 64-bit words accessed atomically via the primitive // atomic functions (types [Int64] and [Uint64] are automatically aligned). // The first word in an allocated struct, array, or slice; in a global -// variable; or in a local variable (because the subject of all atomic operations -// will escape to the heap) can be relied upon to be 64-bit aligned. +// variable; or in a local variable (because on 32-bit architectures, the +// subject of 64-bit atomic operations will escape to the heap) can be +// relied upon to be 64-bit aligned. // SwapInt32 atomically stores new into *addr and returns the previous *addr value. // Consider using the more ergonomic and less error-prone [Int32.Swap] instead. +// +//go:noescape func SwapInt32(addr *int32, new int32) (old int32) -// SwapInt64 atomically stores new into *addr and returns the previous *addr value. -// Consider using the more ergonomic and less error-prone [Int64.Swap] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func SwapInt64(addr *int64, new int64) (old int64) - // SwapUint32 atomically stores new into *addr and returns the previous *addr value. // Consider using the more ergonomic and less error-prone [Uint32.Swap] instead. +// +//go:noescape func SwapUint32(addr *uint32, new uint32) (old uint32) -// SwapUint64 atomically stores new into *addr and returns the previous *addr value. -// Consider using the more ergonomic and less error-prone [Uint64.Swap] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func SwapUint64(addr *uint64, new uint64) (old uint64) - // SwapUintptr atomically stores new into *addr and returns the previous *addr value. // Consider using the more ergonomic and less error-prone [Uintptr.Swap] instead. +// +//go:noescape func SwapUintptr(addr *uintptr, new uintptr) (old uintptr) // SwapPointer atomically stores new into *addr and returns the previous *addr value. @@ -91,24 +88,20 @@ func SwapPointer(addr *unsafe.Pointer, new unsafe.Pointer) (old unsafe.Pointer) // CompareAndSwapInt32 executes the compare-and-swap operation for an int32 value. // Consider using the more ergonomic and less error-prone [Int32.CompareAndSwap] instead. +// +//go:noescape func CompareAndSwapInt32(addr *int32, old, new int32) (swapped bool) -// CompareAndSwapInt64 executes the compare-and-swap operation for an int64 value. -// Consider using the more ergonomic and less error-prone [Int64.CompareAndSwap] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func CompareAndSwapInt64(addr *int64, old, new int64) (swapped bool) - // CompareAndSwapUint32 executes the compare-and-swap operation for a uint32 value. // Consider using the more ergonomic and less error-prone [Uint32.CompareAndSwap] instead. +// +//go:noescape func CompareAndSwapUint32(addr *uint32, old, new uint32) (swapped bool) -// CompareAndSwapUint64 executes the compare-and-swap operation for a uint64 value. -// Consider using the more ergonomic and less error-prone [Uint64.CompareAndSwap] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func CompareAndSwapUint64(addr *uint64, old, new uint64) (swapped bool) - // CompareAndSwapUintptr executes the compare-and-swap operation for a uintptr value. // Consider using the more ergonomic and less error-prone [Uintptr.CompareAndSwap] instead. +// +//go:noescape func CompareAndSwapUintptr(addr *uintptr, old, new uintptr) (swapped bool) // CompareAndSwapPointer executes the compare-and-swap operation for a unsafe.Pointer value. @@ -117,100 +110,82 @@ func CompareAndSwapPointer(addr *unsafe.Pointer, old, new unsafe.Pointer) (swapp // AddInt32 atomically adds delta to *addr and returns the new value. // Consider using the more ergonomic and less error-prone [Int32.Add] instead. +// +//go:noescape func AddInt32(addr *int32, delta int32) (new int32) // AddUint32 atomically adds delta to *addr and returns the new value. // To subtract a signed positive constant value c from x, do AddUint32(&x, ^uint32(c-1)). // In particular, to decrement x, do AddUint32(&x, ^uint32(0)). // Consider using the more ergonomic and less error-prone [Uint32.Add] instead. +// +//go:noescape func AddUint32(addr *uint32, delta uint32) (new uint32) -// AddInt64 atomically adds delta to *addr and returns the new value. -// Consider using the more ergonomic and less error-prone [Int64.Add] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func AddInt64(addr *int64, delta int64) (new int64) - -// AddUint64 atomically adds delta to *addr and returns the new value. -// To subtract a signed positive constant value c from x, do AddUint64(&x, ^uint64(c-1)). -// In particular, to decrement x, do AddUint64(&x, ^uint64(0)). -// Consider using the more ergonomic and less error-prone [Uint64.Add] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func AddUint64(addr *uint64, delta uint64) (new uint64) - // AddUintptr atomically adds delta to *addr and returns the new value. // Consider using the more ergonomic and less error-prone [Uintptr.Add] instead. +// +//go:noescape func AddUintptr(addr *uintptr, delta uintptr) (new uintptr) // AndInt32 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask // and returns the old value. // Consider using the more ergonomic and less error-prone [Int32.And] instead. +// +//go:noescape func AndInt32(addr *int32, mask int32) (old int32) // AndUint32 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask // and returns the old value. // Consider using the more ergonomic and less error-prone [Uint32.And] instead. +// +//go:noescape func AndUint32(addr *uint32, mask uint32) (old uint32) -// AndInt64 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask -// and returns the old value. -// Consider using the more ergonomic and less error-prone [Int64.And] instead. -func AndInt64(addr *int64, mask int64) (old int64) - -// AndUint64 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask -// and returns the old. -// Consider using the more ergonomic and less error-prone [Uint64.And] instead. -func AndUint64(addr *uint64, mask uint64) (old uint64) - // AndUintptr atomically performs a bitwise AND operation on *addr using the bitmask provided as mask // and returns the old value. // Consider using the more ergonomic and less error-prone [Uintptr.And] instead. +// +//go:noescape func AndUintptr(addr *uintptr, mask uintptr) (old uintptr) // OrInt32 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask // and returns the old value. // Consider using the more ergonomic and less error-prone [Int32.Or] instead. +// +//go:noescape func OrInt32(addr *int32, mask int32) (old int32) // OrUint32 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask // and returns the old value. // Consider using the more ergonomic and less error-prone [Uint32.Or] instead. +// +//go:noescape func OrUint32(addr *uint32, mask uint32) (old uint32) -// OrInt64 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask -// and returns the old value. -// Consider using the more ergonomic and less error-prone [Int64.Or] instead. -func OrInt64(addr *int64, mask int64) (old int64) - -// OrUint64 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask -// and returns the old value. -// Consider using the more ergonomic and less error-prone [Uint64.Or] instead. -func OrUint64(addr *uint64, mask uint64) (old uint64) - // OrUintptr atomically performs a bitwise OR operation on *addr using the bitmask provided as mask // and returns the old value. // Consider using the more ergonomic and less error-prone [Uintptr.Or] instead. +// +//go:noescape func OrUintptr(addr *uintptr, mask uintptr) (old uintptr) // LoadInt32 atomically loads *addr. // Consider using the more ergonomic and less error-prone [Int32.Load] instead. +// +//go:noescape func LoadInt32(addr *int32) (val int32) -// LoadInt64 atomically loads *addr. -// Consider using the more ergonomic and less error-prone [Int64.Load] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func LoadInt64(addr *int64) (val int64) - // LoadUint32 atomically loads *addr. // Consider using the more ergonomic and less error-prone [Uint32.Load] instead. +// +//go:noescape func LoadUint32(addr *uint32) (val uint32) -// LoadUint64 atomically loads *addr. -// Consider using the more ergonomic and less error-prone [Uint64.Load] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func LoadUint64(addr *uint64) (val uint64) - // LoadUintptr atomically loads *addr. // Consider using the more ergonomic and less error-prone [Uintptr.Load] instead. +// +//go:noescape func LoadUintptr(addr *uintptr) (val uintptr) // LoadPointer atomically loads *addr. @@ -219,24 +194,20 @@ func LoadPointer(addr *unsafe.Pointer) (val unsafe.Pointer) // StoreInt32 atomically stores val into *addr. // Consider using the more ergonomic and less error-prone [Int32.Store] instead. +// +//go:noescape func StoreInt32(addr *int32, val int32) -// StoreInt64 atomically stores val into *addr. -// Consider using the more ergonomic and less error-prone [Int64.Store] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func StoreInt64(addr *int64, val int64) - // StoreUint32 atomically stores val into *addr. // Consider using the more ergonomic and less error-prone [Uint32.Store] instead. +// +//go:noescape func StoreUint32(addr *uint32, val uint32) -// StoreUint64 atomically stores val into *addr. -// Consider using the more ergonomic and less error-prone [Uint64.Store] instead -// (particularly if you target 32-bit platforms; see the bugs section). -func StoreUint64(addr *uint64, val uint64) - // StoreUintptr atomically stores val into *addr. // Consider using the more ergonomic and less error-prone [Uintptr.Store] instead. +// +//go:noescape func StoreUintptr(addr *uintptr, val uintptr) // StorePointer atomically stores val into *addr. diff --git a/src/sync/atomic/doc_32.go b/src/sync/atomic/doc_32.go new file mode 100644 index 00000000000000..9d644f25ec7b8d --- /dev/null +++ b/src/sync/atomic/doc_32.go @@ -0,0 +1,79 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 || arm || mips || mipsle + +package atomic + +// SwapInt64 atomically stores new into *addr and returns the previous *addr value. +// Consider using the more ergonomic and less error-prone [Int64.Swap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func SwapInt64(addr *int64, new int64) (old int64) + +// SwapUint64 atomically stores new into *addr and returns the previous *addr value. +// Consider using the more ergonomic and less error-prone [Uint64.Swap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func SwapUint64(addr *uint64, new uint64) (old uint64) + +// CompareAndSwapInt64 executes the compare-and-swap operation for an int64 value. +// Consider using the more ergonomic and less error-prone [Int64.CompareAndSwap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func CompareAndSwapInt64(addr *int64, old, new int64) (swapped bool) + +// CompareAndSwapUint64 executes the compare-and-swap operation for a uint64 value. +// Consider using the more ergonomic and less error-prone [Uint64.CompareAndSwap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func CompareAndSwapUint64(addr *uint64, old, new uint64) (swapped bool) + +// AddInt64 atomically adds delta to *addr and returns the new value. +// Consider using the more ergonomic and less error-prone [Int64.Add] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func AddInt64(addr *int64, delta int64) (new int64) + +// AddUint64 atomically adds delta to *addr and returns the new value. +// To subtract a signed positive constant value c from x, do AddUint64(&x, ^uint64(c-1)). +// In particular, to decrement x, do AddUint64(&x, ^uint64(0)). +// Consider using the more ergonomic and less error-prone [Uint64.Add] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func AddUint64(addr *uint64, delta uint64) (new uint64) + +// AndInt64 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask +// and returns the old value. +// Consider using the more ergonomic and less error-prone [Int64.And] instead. +func AndInt64(addr *int64, mask int64) (old int64) + +// AndUint64 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask +// and returns the old. +// Consider using the more ergonomic and less error-prone [Uint64.And] instead. +func AndUint64(addr *uint64, mask uint64) (old uint64) + +// OrInt64 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask +// and returns the old value. +// Consider using the more ergonomic and less error-prone [Int64.Or] instead. +func OrInt64(addr *int64, mask int64) (old int64) + +// OrUint64 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask +// and returns the old value. +// Consider using the more ergonomic and less error-prone [Uint64.Or] instead. +func OrUint64(addr *uint64, mask uint64) (old uint64) + +// LoadInt64 atomically loads *addr. +// Consider using the more ergonomic and less error-prone [Int64.Load] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func LoadInt64(addr *int64) (val int64) + +// LoadUint64 atomically loads *addr. +// Consider using the more ergonomic and less error-prone [Uint64.Load] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func LoadUint64(addr *uint64) (val uint64) + +// StoreInt64 atomically stores val into *addr. +// Consider using the more ergonomic and less error-prone [Int64.Store] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func StoreInt64(addr *int64, val int64) + +// StoreUint64 atomically stores val into *addr. +// Consider using the more ergonomic and less error-prone [Uint64.Store] instead +// (particularly if you target 32-bit platforms; see the bugs section). +func StoreUint64(addr *uint64, val uint64) diff --git a/src/sync/atomic/doc_64.go b/src/sync/atomic/doc_64.go new file mode 100644 index 00000000000000..5fec3f4e8f1d55 --- /dev/null +++ b/src/sync/atomic/doc_64.go @@ -0,0 +1,107 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !(386 || arm || mips || mipsle) + +package atomic + +// SwapInt64 atomically stores new into *addr and returns the previous *addr value. +// Consider using the more ergonomic and less error-prone [Int64.Swap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func SwapInt64(addr *int64, new int64) (old int64) + +// SwapUint64 atomically stores new into *addr and returns the previous *addr value. +// Consider using the more ergonomic and less error-prone [Uint64.Swap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func SwapUint64(addr *uint64, new uint64) (old uint64) + +// CompareAndSwapInt64 executes the compare-and-swap operation for an int64 value. +// Consider using the more ergonomic and less error-prone [Int64.CompareAndSwap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func CompareAndSwapInt64(addr *int64, old, new int64) (swapped bool) + +// CompareAndSwapUint64 executes the compare-and-swap operation for a uint64 value. +// Consider using the more ergonomic and less error-prone [Uint64.CompareAndSwap] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func CompareAndSwapUint64(addr *uint64, old, new uint64) (swapped bool) + +// AddInt64 atomically adds delta to *addr and returns the new value. +// Consider using the more ergonomic and less error-prone [Int64.Add] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func AddInt64(addr *int64, delta int64) (new int64) + +// AddUint64 atomically adds delta to *addr and returns the new value. +// To subtract a signed positive constant value c from x, do AddUint64(&x, ^uint64(c-1)). +// In particular, to decrement x, do AddUint64(&x, ^uint64(0)). +// Consider using the more ergonomic and less error-prone [Uint64.Add] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func AddUint64(addr *uint64, delta uint64) (new uint64) + +// AndInt64 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask +// and returns the old value. +// Consider using the more ergonomic and less error-prone [Int64.And] instead. +// +//go:noescape +func AndInt64(addr *int64, mask int64) (old int64) + +// AndUint64 atomically performs a bitwise AND operation on *addr using the bitmask provided as mask +// and returns the old. +// Consider using the more ergonomic and less error-prone [Uint64.And] instead. +// +//go:noescape +func AndUint64(addr *uint64, mask uint64) (old uint64) + +// OrInt64 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask +// and returns the old value. +// Consider using the more ergonomic and less error-prone [Int64.Or] instead. +// +//go:noescape +func OrInt64(addr *int64, mask int64) (old int64) + +// OrUint64 atomically performs a bitwise OR operation on *addr using the bitmask provided as mask +// and returns the old value. +// Consider using the more ergonomic and less error-prone [Uint64.Or] instead. +// +//go:noescape +func OrUint64(addr *uint64, mask uint64) (old uint64) + +// LoadInt64 atomically loads *addr. +// Consider using the more ergonomic and less error-prone [Int64.Load] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func LoadInt64(addr *int64) (val int64) + +// LoadUint64 atomically loads *addr. +// Consider using the more ergonomic and less error-prone [Uint64.Load] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func LoadUint64(addr *uint64) (val uint64) + +// StoreInt64 atomically stores val into *addr. +// Consider using the more ergonomic and less error-prone [Int64.Store] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func StoreInt64(addr *int64, val int64) + +// StoreUint64 atomically stores val into *addr. +// Consider using the more ergonomic and less error-prone [Uint64.Store] instead +// (particularly if you target 32-bit platforms; see the bugs section). +// +//go:noescape +func StoreUint64(addr *uint64, val uint64) diff --git a/src/sync/atomic/type.go b/src/sync/atomic/type.go index f487cb9c5f7eaa..4a74150b41bace 100644 --- a/src/sync/atomic/type.go +++ b/src/sync/atomic/type.go @@ -8,6 +8,8 @@ import "unsafe" // A Bool is an atomic boolean value. // The zero value is false. +// +// Bool must not be copied after first use. type Bool struct { _ noCopy v uint32 @@ -40,6 +42,8 @@ func b32(b bool) uint32 { var _ = &Pointer[int]{} // A Pointer is an atomic pointer of type *T. The zero value is a nil *T. +// +// Pointer must not be copied after first use. type Pointer[T any] struct { // Mention *T in a field to disallow conversion between Pointer types. // See go.dev/issue/56603 for more details. @@ -65,6 +69,8 @@ func (x *Pointer[T]) CompareAndSwap(old, new *T) (swapped bool) { } // An Int32 is an atomic int32. The zero value is zero. +// +// Int32 must not be copied after first use. type Int32 struct { _ noCopy v int32 @@ -96,6 +102,8 @@ func (x *Int32) And(mask int32) (old int32) { return AndInt32(&x.v, mask) } func (x *Int32) Or(mask int32) (old int32) { return OrInt32(&x.v, mask) } // An Int64 is an atomic int64. The zero value is zero. +// +// Int64 must not be copied after first use. type Int64 struct { _ noCopy _ align64 @@ -128,6 +136,8 @@ func (x *Int64) And(mask int64) (old int64) { return AndInt64(&x.v, mask) } func (x *Int64) Or(mask int64) (old int64) { return OrInt64(&x.v, mask) } // A Uint32 is an atomic uint32. The zero value is zero. +// +// Uint32 must not be copied after first use. type Uint32 struct { _ noCopy v uint32 @@ -159,6 +169,8 @@ func (x *Uint32) And(mask uint32) (old uint32) { return AndUint32(&x.v, mask) } func (x *Uint32) Or(mask uint32) (old uint32) { return OrUint32(&x.v, mask) } // A Uint64 is an atomic uint64. The zero value is zero. +// +// Uint64 must not be copied after first use. type Uint64 struct { _ noCopy _ align64 @@ -191,6 +203,8 @@ func (x *Uint64) And(mask uint64) (old uint64) { return AndUint64(&x.v, mask) } func (x *Uint64) Or(mask uint64) (old uint64) { return OrUint64(&x.v, mask) } // A Uintptr is an atomic uintptr. The zero value is zero. +// +// Uintptr must not be copied after first use. type Uintptr struct { _ noCopy v uintptr @@ -218,7 +232,7 @@ func (x *Uintptr) Add(delta uintptr) (new uintptr) { return AddUintptr(&x.v, del func (x *Uintptr) And(mask uintptr) (old uintptr) { return AndUintptr(&x.v, mask) } // Or atomically performs a bitwise OR operation on x using the bitmask -// provided as mask and returns the updated value after the OR operation. +// provided as mask and returns the old value. func (x *Uintptr) Or(mask uintptr) (old uintptr) { return OrUintptr(&x.v, mask) } // noCopy may be added to structs which must not be copied diff --git a/src/sync/atomic/value_test.go b/src/sync/atomic/value_test.go index 721da965e35800..b8bc8b8851b656 100644 --- a/src/sync/atomic/value_test.go +++ b/src/sync/atomic/value_test.go @@ -185,7 +185,6 @@ func TestValueSwapConcurrent(t *testing.T) { n = 1000 } for i := uint64(0); i < m*n; i += n { - i := i g.Add(1) go func() { var c uint64 @@ -256,7 +255,6 @@ func TestValueCompareAndSwapConcurrent(t *testing.T) { n = 100 } for i := 0; i < m; i++ { - i := i w.Add(1) go func() { for j := i; j < m*n; runtime.Gosched() { diff --git a/src/sync/example_test.go b/src/sync/example_test.go index ed240e57ae0eb5..a019beebc5d26c 100644 --- a/src/sync/example_test.go +++ b/src/sync/example_test.go @@ -19,6 +19,26 @@ var http httpPkg // This example fetches several URLs concurrently, // using a WaitGroup to block until all the fetches are complete. func ExampleWaitGroup() { + var wg sync.WaitGroup + var urls = []string{ + "http://www.golang.org/", + "http://www.google.com/", + "http://www.example.com/", + } + for _, url := range urls { + // Launch a goroutine to fetch the URL. + wg.Go(func() { + // Fetch the URL. + http.Get(url) + }) + } + // Wait for all HTTP fetches to complete. + wg.Wait() +} + +// This example is equivalent to the main example, but uses Add/Done +// instead of Go. +func ExampleWaitGroup_addAndDone() { var wg sync.WaitGroup var urls = []string{ "http://www.golang.org/", diff --git a/src/sync/map.go b/src/sync/map.go index 33bc8141abd4c8..934a651117fd15 100644 --- a/src/sync/map.go +++ b/src/sync/map.go @@ -1,11 +1,11 @@ -// Copyright 2016 The Go Authors. All rights reserved. +// Copyright 2024 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package sync import ( - "sync/atomic" + isync "internal/sync" ) // Map is like a Go map[any]any but is safe for concurrent use @@ -36,390 +36,58 @@ import ( // // [the Go memory model]: https://go.dev/ref/mem type Map struct { - mu Mutex + _ noCopy - // read contains the portion of the map's contents that are safe for - // concurrent access (with or without mu held). - // - // The read field itself is always safe to load, but must only be stored with - // mu held. - // - // Entries stored in read may be updated concurrently without mu, but updating - // a previously-expunged entry requires that the entry be copied to the dirty - // map and unexpunged with mu held. - read atomic.Pointer[readOnly] - - // dirty contains the portion of the map's contents that require mu to be - // held. To ensure that the dirty map can be promoted to the read map quickly, - // it also includes all of the non-expunged entries in the read map. - // - // Expunged entries are not stored in the dirty map. An expunged entry in the - // clean map must be unexpunged and added to the dirty map before a new value - // can be stored to it. - // - // If the dirty map is nil, the next write to the map will initialize it by - // making a shallow copy of the clean map, omitting stale entries. - dirty map[any]*entry - - // misses counts the number of loads since the read map was last updated that - // needed to lock mu to determine whether the key was present. - // - // Once enough misses have occurred to cover the cost of copying the dirty - // map, the dirty map will be promoted to the read map (in the unamended - // state) and the next store to the map will make a new dirty copy. - misses int -} - -// readOnly is an immutable struct stored atomically in the Map.read field. -type readOnly struct { - m map[any]*entry - amended bool // true if the dirty map contains some key not in m. -} - -// expunged is an arbitrary pointer that marks entries which have been deleted -// from the dirty map. -var expunged = new(any) - -// An entry is a slot in the map corresponding to a particular key. -type entry struct { - // p points to the interface{} value stored for the entry. - // - // If p == nil, the entry has been deleted, and either m.dirty == nil or - // m.dirty[key] is e. - // - // If p == expunged, the entry has been deleted, m.dirty != nil, and the entry - // is missing from m.dirty. - // - // Otherwise, the entry is valid and recorded in m.read.m[key] and, if m.dirty - // != nil, in m.dirty[key]. - // - // An entry can be deleted by atomic replacement with nil: when m.dirty is - // next created, it will atomically replace nil with expunged and leave - // m.dirty[key] unset. - // - // An entry's associated value can be updated by atomic replacement, provided - // p != expunged. If p == expunged, an entry's associated value can be updated - // only after first setting m.dirty[key] = e so that lookups using the dirty - // map find the entry. - p atomic.Pointer[any] -} - -func newEntry(i any) *entry { - e := &entry{} - e.p.Store(&i) - return e -} - -func (m *Map) loadReadOnly() readOnly { - if p := m.read.Load(); p != nil { - return *p - } - return readOnly{} + m isync.HashTrieMap[any, any] } // Load returns the value stored in the map for a key, or nil if no // value is present. // The ok result indicates whether value was found in the map. func (m *Map) Load(key any) (value any, ok bool) { - read := m.loadReadOnly() - e, ok := read.m[key] - if !ok && read.amended { - m.mu.Lock() - // Avoid reporting a spurious miss if m.dirty got promoted while we were - // blocked on m.mu. (If further loads of the same key will not miss, it's - // not worth copying the dirty map for this key.) - read = m.loadReadOnly() - e, ok = read.m[key] - if !ok && read.amended { - e, ok = m.dirty[key] - // Regardless of whether the entry was present, record a miss: this key - // will take the slow path until the dirty map is promoted to the read - // map. - m.missLocked() - } - m.mu.Unlock() - } - if !ok { - return nil, false - } - return e.load() -} - -func (e *entry) load() (value any, ok bool) { - p := e.p.Load() - if p == nil || p == expunged { - return nil, false - } - return *p, true + return m.m.Load(key) } // Store sets the value for a key. func (m *Map) Store(key, value any) { - _, _ = m.Swap(key, value) + m.m.Store(key, value) } // Clear deletes all the entries, resulting in an empty Map. func (m *Map) Clear() { - read := m.loadReadOnly() - if len(read.m) == 0 && !read.amended { - // Avoid allocating a new readOnly when the map is already clear. - return - } - - m.mu.Lock() - defer m.mu.Unlock() - - read = m.loadReadOnly() - if len(read.m) > 0 || read.amended { - m.read.Store(&readOnly{}) - } - - clear(m.dirty) - // Don't immediately promote the newly-cleared dirty map on the next operation. - m.misses = 0 -} - -// tryCompareAndSwap compare the entry with the given old value and swaps -// it with a new value if the entry is equal to the old value, and the entry -// has not been expunged. -// -// If the entry is expunged, tryCompareAndSwap returns false and leaves -// the entry unchanged. -func (e *entry) tryCompareAndSwap(old, new any) bool { - p := e.p.Load() - if p == nil || p == expunged || *p != old { - return false - } - - // Copy the interface after the first load to make this method more amenable - // to escape analysis: if the comparison fails from the start, we shouldn't - // bother heap-allocating an interface value to store. - nc := new - for { - if e.p.CompareAndSwap(p, &nc) { - return true - } - p = e.p.Load() - if p == nil || p == expunged || *p != old { - return false - } - } -} - -// unexpungeLocked ensures that the entry is not marked as expunged. -// -// If the entry was previously expunged, it must be added to the dirty map -// before m.mu is unlocked. -func (e *entry) unexpungeLocked() (wasExpunged bool) { - return e.p.CompareAndSwap(expunged, nil) -} - -// swapLocked unconditionally swaps a value into the entry. -// -// The entry must be known not to be expunged. -func (e *entry) swapLocked(i *any) *any { - return e.p.Swap(i) + m.m.Clear() } // LoadOrStore returns the existing value for the key if present. // Otherwise, it stores and returns the given value. // The loaded result is true if the value was loaded, false if stored. func (m *Map) LoadOrStore(key, value any) (actual any, loaded bool) { - // Avoid locking if it's a clean hit. - read := m.loadReadOnly() - if e, ok := read.m[key]; ok { - actual, loaded, ok := e.tryLoadOrStore(value) - if ok { - return actual, loaded - } - } - - m.mu.Lock() - read = m.loadReadOnly() - if e, ok := read.m[key]; ok { - if e.unexpungeLocked() { - m.dirty[key] = e - } - actual, loaded, _ = e.tryLoadOrStore(value) - } else if e, ok := m.dirty[key]; ok { - actual, loaded, _ = e.tryLoadOrStore(value) - m.missLocked() - } else { - if !read.amended { - // We're adding the first new key to the dirty map. - // Make sure it is allocated and mark the read-only map as incomplete. - m.dirtyLocked() - m.read.Store(&readOnly{m: read.m, amended: true}) - } - m.dirty[key] = newEntry(value) - actual, loaded = value, false - } - m.mu.Unlock() - - return actual, loaded -} - -// tryLoadOrStore atomically loads or stores a value if the entry is not -// expunged. -// -// If the entry is expunged, tryLoadOrStore leaves the entry unchanged and -// returns with ok==false. -func (e *entry) tryLoadOrStore(i any) (actual any, loaded, ok bool) { - p := e.p.Load() - if p == expunged { - return nil, false, false - } - if p != nil { - return *p, true, true - } - - // Copy the interface after the first load to make this method more amenable - // to escape analysis: if we hit the "load" path or the entry is expunged, we - // shouldn't bother heap-allocating. - ic := i - for { - if e.p.CompareAndSwap(nil, &ic) { - return i, false, true - } - p = e.p.Load() - if p == expunged { - return nil, false, false - } - if p != nil { - return *p, true, true - } - } + return m.m.LoadOrStore(key, value) } // LoadAndDelete deletes the value for a key, returning the previous value if any. // The loaded result reports whether the key was present. func (m *Map) LoadAndDelete(key any) (value any, loaded bool) { - read := m.loadReadOnly() - e, ok := read.m[key] - if !ok && read.amended { - m.mu.Lock() - read = m.loadReadOnly() - e, ok = read.m[key] - if !ok && read.amended { - e, ok = m.dirty[key] - delete(m.dirty, key) - // Regardless of whether the entry was present, record a miss: this key - // will take the slow path until the dirty map is promoted to the read - // map. - m.missLocked() - } - m.mu.Unlock() - } - if ok { - return e.delete() - } - return nil, false + return m.m.LoadAndDelete(key) } // Delete deletes the value for a key. +// If the key is not in the map, Delete does nothing. func (m *Map) Delete(key any) { - m.LoadAndDelete(key) -} - -func (e *entry) delete() (value any, ok bool) { - for { - p := e.p.Load() - if p == nil || p == expunged { - return nil, false - } - if e.p.CompareAndSwap(p, nil) { - return *p, true - } - } -} - -// trySwap swaps a value if the entry has not been expunged. -// -// If the entry is expunged, trySwap returns false and leaves the entry -// unchanged. -func (e *entry) trySwap(i *any) (*any, bool) { - for { - p := e.p.Load() - if p == expunged { - return nil, false - } - if e.p.CompareAndSwap(p, i) { - return p, true - } - } + m.m.Delete(key) } // Swap swaps the value for a key and returns the previous value if any. // The loaded result reports whether the key was present. func (m *Map) Swap(key, value any) (previous any, loaded bool) { - read := m.loadReadOnly() - if e, ok := read.m[key]; ok { - if v, ok := e.trySwap(&value); ok { - if v == nil { - return nil, false - } - return *v, true - } - } - - m.mu.Lock() - read = m.loadReadOnly() - if e, ok := read.m[key]; ok { - if e.unexpungeLocked() { - // The entry was previously expunged, which implies that there is a - // non-nil dirty map and this entry is not in it. - m.dirty[key] = e - } - if v := e.swapLocked(&value); v != nil { - loaded = true - previous = *v - } - } else if e, ok := m.dirty[key]; ok { - if v := e.swapLocked(&value); v != nil { - loaded = true - previous = *v - } - } else { - if !read.amended { - // We're adding the first new key to the dirty map. - // Make sure it is allocated and mark the read-only map as incomplete. - m.dirtyLocked() - m.read.Store(&readOnly{m: read.m, amended: true}) - } - m.dirty[key] = newEntry(value) - } - m.mu.Unlock() - return previous, loaded + return m.m.Swap(key, value) } // CompareAndSwap swaps the old and new values for key // if the value stored in the map is equal to old. // The old value must be of a comparable type. func (m *Map) CompareAndSwap(key, old, new any) (swapped bool) { - read := m.loadReadOnly() - if e, ok := read.m[key]; ok { - return e.tryCompareAndSwap(old, new) - } else if !read.amended { - return false // No existing value for key. - } - - m.mu.Lock() - defer m.mu.Unlock() - read = m.loadReadOnly() - swapped = false - if e, ok := read.m[key]; ok { - swapped = e.tryCompareAndSwap(old, new) - } else if e, ok := m.dirty[key]; ok { - swapped = e.tryCompareAndSwap(old, new) - // We needed to lock mu in order to load the entry for key, - // and the operation didn't change the set of keys in the map - // (so it would be made more efficient by promoting the dirty - // map to read-only). - // Count it as a miss so that we will eventually switch to the - // more efficient steady state. - m.missLocked() - } - return swapped + return m.m.CompareAndSwap(key, old, new) } // CompareAndDelete deletes the entry for key if its value is equal to old. @@ -428,35 +96,7 @@ func (m *Map) CompareAndSwap(key, old, new any) (swapped bool) { // If there is no current value for key in the map, CompareAndDelete // returns false (even if the old value is the nil interface value). func (m *Map) CompareAndDelete(key, old any) (deleted bool) { - read := m.loadReadOnly() - e, ok := read.m[key] - if !ok && read.amended { - m.mu.Lock() - read = m.loadReadOnly() - e, ok = read.m[key] - if !ok && read.amended { - e, ok = m.dirty[key] - // Don't delete key from m.dirty: we still need to do the “compare” part - // of the operation. The entry will eventually be expunged when the - // dirty map is promoted to the read map. - // - // Regardless of whether the entry was present, record a miss: this key - // will take the slow path until the dirty map is promoted to the read - // map. - m.missLocked() - } - m.mu.Unlock() - } - for ok { - p := e.p.Load() - if p == nil || p == expunged || *p != old { - return false - } - if e.p.CompareAndSwap(p, nil) { - return true - } - } - return false + return m.m.CompareAndDelete(key, old) } // Range calls f sequentially for each key and value present in the map. @@ -471,70 +111,5 @@ func (m *Map) CompareAndDelete(key, old any) (deleted bool) { // Range may be O(N) with the number of elements in the map even if f returns // false after a constant number of calls. func (m *Map) Range(f func(key, value any) bool) { - // We need to be able to iterate over all of the keys that were already - // present at the start of the call to Range. - // If read.amended is false, then read.m satisfies that property without - // requiring us to hold m.mu for a long time. - read := m.loadReadOnly() - if read.amended { - // m.dirty contains keys not in read.m. Fortunately, Range is already O(N) - // (assuming the caller does not break out early), so a call to Range - // amortizes an entire copy of the map: we can promote the dirty copy - // immediately! - m.mu.Lock() - read = m.loadReadOnly() - if read.amended { - read = readOnly{m: m.dirty} - copyRead := read - m.read.Store(©Read) - m.dirty = nil - m.misses = 0 - } - m.mu.Unlock() - } - - for k, e := range read.m { - v, ok := e.load() - if !ok { - continue - } - if !f(k, v) { - break - } - } -} - -func (m *Map) missLocked() { - m.misses++ - if m.misses < len(m.dirty) { - return - } - m.read.Store(&readOnly{m: m.dirty}) - m.dirty = nil - m.misses = 0 -} - -func (m *Map) dirtyLocked() { - if m.dirty != nil { - return - } - - read := m.loadReadOnly() - m.dirty = make(map[any]*entry, len(read.m)) - for k, e := range read.m { - if !e.tryExpungeLocked() { - m.dirty[k] = e - } - } -} - -func (e *entry) tryExpungeLocked() (isExpunged bool) { - p := e.p.Load() - for p == nil { - if e.p.CompareAndSwap(nil, expunged) { - return true - } - p = e.p.Load() - } - return p == expunged + m.m.Range(f) } diff --git a/src/sync/map_bench_test.go b/src/sync/map_bench_test.go index fb9eb25432d4b8..f7469aedbe3e7f 100644 --- a/src/sync/map_bench_test.go +++ b/src/sync/map_bench_test.go @@ -6,6 +6,7 @@ package sync_test import ( "fmt" + isync "internal/sync" "reflect" "sync" "sync/atomic" @@ -18,13 +19,14 @@ type bench struct { } func benchMap(b *testing.B, bench bench) { - for _, m := range [...]mapInterface{&DeepCopyMap{}, &RWMutexMap{}, &sync.Map{}} { + for _, m := range [...]mapInterface{&DeepCopyMap{}, &RWMutexMap{}, &isync.HashTrieMap[any, any]{}, &sync.Map{}} { b.Run(fmt.Sprintf("%T", m), func(b *testing.B) { m = reflect.New(reflect.TypeOf(m).Elem()).Interface().(mapInterface) if bench.setup != nil { bench.setup(b, m) } + b.ReportAllocs() b.ResetTimer() var i int64 @@ -36,7 +38,7 @@ func benchMap(b *testing.B, bench bench) { } } -func BenchmarkLoadMostlyHits(b *testing.B) { +func BenchmarkMapLoadMostlyHits(b *testing.B) { const hits, misses = 1023, 1 benchMap(b, bench{ @@ -58,7 +60,7 @@ func BenchmarkLoadMostlyHits(b *testing.B) { }) } -func BenchmarkLoadMostlyMisses(b *testing.B) { +func BenchmarkMapLoadMostlyMisses(b *testing.B) { const hits, misses = 1, 1023 benchMap(b, bench{ @@ -80,7 +82,7 @@ func BenchmarkLoadMostlyMisses(b *testing.B) { }) } -func BenchmarkLoadOrStoreBalanced(b *testing.B) { +func BenchmarkMapLoadOrStoreBalanced(b *testing.B) { const hits, misses = 128, 128 benchMap(b, bench{ @@ -114,7 +116,7 @@ func BenchmarkLoadOrStoreBalanced(b *testing.B) { }) } -func BenchmarkLoadOrStoreUnique(b *testing.B) { +func BenchmarkMapLoadOrStoreUnique(b *testing.B) { benchMap(b, bench{ setup: func(b *testing.B, m mapInterface) { if _, ok := m.(*DeepCopyMap); ok { @@ -130,7 +132,7 @@ func BenchmarkLoadOrStoreUnique(b *testing.B) { }) } -func BenchmarkLoadOrStoreCollision(b *testing.B) { +func BenchmarkMapLoadOrStoreCollision(b *testing.B) { benchMap(b, bench{ setup: func(_ *testing.B, m mapInterface) { m.LoadOrStore(0, 0) @@ -144,7 +146,7 @@ func BenchmarkLoadOrStoreCollision(b *testing.B) { }) } -func BenchmarkLoadAndDeleteBalanced(b *testing.B) { +func BenchmarkMapLoadAndDeleteBalanced(b *testing.B) { const hits, misses = 128, 128 benchMap(b, bench{ @@ -174,7 +176,7 @@ func BenchmarkLoadAndDeleteBalanced(b *testing.B) { }) } -func BenchmarkLoadAndDeleteUnique(b *testing.B) { +func BenchmarkMapLoadAndDeleteUnique(b *testing.B) { benchMap(b, bench{ setup: func(b *testing.B, m mapInterface) { if _, ok := m.(*DeepCopyMap); ok { @@ -190,7 +192,7 @@ func BenchmarkLoadAndDeleteUnique(b *testing.B) { }) } -func BenchmarkLoadAndDeleteCollision(b *testing.B) { +func BenchmarkMapLoadAndDeleteCollision(b *testing.B) { benchMap(b, bench{ setup: func(_ *testing.B, m mapInterface) { m.LoadOrStore(0, 0) @@ -206,7 +208,7 @@ func BenchmarkLoadAndDeleteCollision(b *testing.B) { }) } -func BenchmarkRange(b *testing.B) { +func BenchmarkMapRange(b *testing.B) { const mapSize = 1 << 10 benchMap(b, bench{ @@ -224,12 +226,12 @@ func BenchmarkRange(b *testing.B) { }) } -// BenchmarkAdversarialAlloc tests performance when we store a new value +// BenchmarkMapAdversarialAlloc tests performance when we store a new value // immediately whenever the map is promoted to clean and otherwise load a // unique, missing key. // // This forces the Load calls to always acquire the map's mutex. -func BenchmarkAdversarialAlloc(b *testing.B) { +func BenchmarkMapAdversarialAlloc(b *testing.B) { benchMap(b, bench{ perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface) { var stores, loadsSinceStore int64 @@ -245,12 +247,12 @@ func BenchmarkAdversarialAlloc(b *testing.B) { }) } -// BenchmarkAdversarialDelete tests performance when we periodically delete +// BenchmarkMapAdversarialDelete tests performance when we periodically delete // one key and add a different one in a large map. // // This forces the Load calls to always acquire the map's mutex and periodically // makes a full copy of the map despite changing only one entry. -func BenchmarkAdversarialDelete(b *testing.B) { +func BenchmarkMapAdversarialDelete(b *testing.B) { const mapSize = 1 << 10 benchMap(b, bench{ @@ -276,7 +278,7 @@ func BenchmarkAdversarialDelete(b *testing.B) { }) } -func BenchmarkDeleteCollision(b *testing.B) { +func BenchmarkMapDeleteCollision(b *testing.B) { benchMap(b, bench{ setup: func(_ *testing.B, m mapInterface) { m.LoadOrStore(0, 0) @@ -290,7 +292,7 @@ func BenchmarkDeleteCollision(b *testing.B) { }) } -func BenchmarkSwapCollision(b *testing.B) { +func BenchmarkMapSwapCollision(b *testing.B) { benchMap(b, bench{ setup: func(_ *testing.B, m mapInterface) { m.LoadOrStore(0, 0) @@ -304,7 +306,7 @@ func BenchmarkSwapCollision(b *testing.B) { }) } -func BenchmarkSwapMostlyHits(b *testing.B) { +func BenchmarkMapSwapMostlyHits(b *testing.B) { const hits, misses = 1023, 1 benchMap(b, bench{ @@ -332,7 +334,7 @@ func BenchmarkSwapMostlyHits(b *testing.B) { }) } -func BenchmarkSwapMostlyMisses(b *testing.B) { +func BenchmarkMapSwapMostlyMisses(b *testing.B) { const hits, misses = 1, 1023 benchMap(b, bench{ @@ -360,7 +362,7 @@ func BenchmarkSwapMostlyMisses(b *testing.B) { }) } -func BenchmarkCompareAndSwapCollision(b *testing.B) { +func BenchmarkMapCompareAndSwapCollision(b *testing.B) { benchMap(b, bench{ setup: func(_ *testing.B, m mapInterface) { m.LoadOrStore(0, 0) @@ -376,7 +378,7 @@ func BenchmarkCompareAndSwapCollision(b *testing.B) { }) } -func BenchmarkCompareAndSwapNoExistingKey(b *testing.B) { +func BenchmarkMapCompareAndSwapNoExistingKey(b *testing.B) { benchMap(b, bench{ perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface) { for ; pb.Next(); i++ { @@ -388,7 +390,7 @@ func BenchmarkCompareAndSwapNoExistingKey(b *testing.B) { }) } -func BenchmarkCompareAndSwapValueNotEqual(b *testing.B) { +func BenchmarkMapCompareAndSwapValueNotEqual(b *testing.B) { benchMap(b, bench{ setup: func(_ *testing.B, m mapInterface) { m.Store(0, 0) @@ -402,7 +404,7 @@ func BenchmarkCompareAndSwapValueNotEqual(b *testing.B) { }) } -func BenchmarkCompareAndSwapMostlyHits(b *testing.B) { +func BenchmarkMapCompareAndSwapMostlyHits(b *testing.B) { const hits, misses = 1023, 1 benchMap(b, bench{ @@ -432,7 +434,7 @@ func BenchmarkCompareAndSwapMostlyHits(b *testing.B) { }) } -func BenchmarkCompareAndSwapMostlyMisses(b *testing.B) { +func BenchmarkMapCompareAndSwapMostlyMisses(b *testing.B) { const hits, misses = 1, 1023 benchMap(b, bench{ @@ -458,7 +460,7 @@ func BenchmarkCompareAndSwapMostlyMisses(b *testing.B) { }) } -func BenchmarkCompareAndDeleteCollision(b *testing.B) { +func BenchmarkMapCompareAndDeleteCollision(b *testing.B) { benchMap(b, bench{ setup: func(_ *testing.B, m mapInterface) { m.LoadOrStore(0, 0) @@ -474,7 +476,7 @@ func BenchmarkCompareAndDeleteCollision(b *testing.B) { }) } -func BenchmarkCompareAndDeleteMostlyHits(b *testing.B) { +func BenchmarkMapCompareAndDeleteMostlyHits(b *testing.B) { const hits, misses = 1023, 1 benchMap(b, bench{ @@ -506,7 +508,7 @@ func BenchmarkCompareAndDeleteMostlyHits(b *testing.B) { }) } -func BenchmarkCompareAndDeleteMostlyMisses(b *testing.B) { +func BenchmarkMapCompareAndDeleteMostlyMisses(b *testing.B) { const hits, misses = 1, 1023 benchMap(b, bench{ @@ -534,7 +536,7 @@ func BenchmarkCompareAndDeleteMostlyMisses(b *testing.B) { }) } -func BenchmarkClear(b *testing.B) { +func BenchmarkMapClear(b *testing.B) { benchMap(b, bench{ perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface) { for ; pb.Next(); i++ { diff --git a/src/sync/map_reference_test.go b/src/sync/map_reference_test.go index 283da0f3a95fde..f98bb98b33fb88 100644 --- a/src/sync/map_reference_test.go +++ b/src/sync/map_reference_test.go @@ -5,6 +5,7 @@ package sync_test import ( + isync "internal/sync" "sync" "sync/atomic" ) @@ -28,6 +29,7 @@ type mapInterface interface { var ( _ mapInterface = &RWMutexMap{} _ mapInterface = &DeepCopyMap{} + _ mapInterface = &isync.HashTrieMap[any, any]{} ) // RWMutexMap is an implementation of mapInterface using a sync.RWMutex. diff --git a/src/sync/map_test.go b/src/sync/map_test.go index e1d0380765abc9..0d6690c74672a8 100644 --- a/src/sync/map_test.go +++ b/src/sync/map_test.go @@ -5,6 +5,7 @@ package sync_test import ( + isync "internal/sync" "internal/testenv" "math/rand" "reflect" @@ -133,6 +134,10 @@ func applyDeepCopyMap(calls []mapCall) ([]mapResult, map[any]any) { return applyCalls(new(DeepCopyMap), calls) } +func applyHashTrieMap(calls []mapCall) ([]mapResult, map[any]any) { + return applyCalls(new(isync.HashTrieMap[any, any]), calls) +} + func TestMapMatchesRWMutex(t *testing.T) { if err := quick.CheckEqual(applyMap, applyRWMutexMap, nil); err != nil { t.Error(err) @@ -145,12 +150,18 @@ func TestMapMatchesDeepCopy(t *testing.T) { } } +func TestMapMatchesHashTrieMap(t *testing.T) { + if err := quick.CheckEqual(applyMap, applyHashTrieMap, nil); err != nil { + t.Error(err) + } +} + func TestConcurrentRange(t *testing.T) { const mapSize = 1 << 10 m := new(sync.Map) for n := int64(1); n <= mapSize; n++ { - m.Store(n, int64(n)) + m.Store(n, n) } done := make(chan struct{}) @@ -214,15 +225,13 @@ func TestIssue40999(t *testing.T) { // add an initial entry to bias len(m.dirty) above the miss count. m.Store(nil, struct{}{}) - var finalized uint32 + var cleanedUp uint32 - // Set finalizers that count for collected keys. A non-zero count + // Add cleanups that count for collected keys. A non-zero count // indicates that keys have not been leaked. - for atomic.LoadUint32(&finalized) == 0 { + for atomic.LoadUint32(&cleanedUp) == 0 { p := new(int) - runtime.SetFinalizer(p, func(*int) { - atomic.AddUint32(&finalized, 1) - }) + runtime.AddCleanup(p, func(c *uint32) { atomic.AddUint32(c, 1) }, &cleanedUp) m.Store(p, struct{}{}) m.Delete(p) runtime.GC() @@ -347,13 +356,13 @@ func TestConcurrentClear(t *testing.T) { }) } -func TestMapClearNoAllocations(t *testing.T) { +func TestMapClearOneAllocation(t *testing.T) { testenv.SkipIfOptimizationOff(t) var m sync.Map allocs := testing.AllocsPerRun(10, func() { m.Clear() }) - if allocs > 0 { - t.Errorf("AllocsPerRun of m.Clear = %v; want 0", allocs) + if allocs > 1 { + t.Errorf("AllocsPerRun of m.Clear = %v; want 1", allocs) } } diff --git a/src/sync/mutex.go b/src/sync/mutex.go index e4ed47c75c2a3c..133c9530fd763c 100644 --- a/src/sync/mutex.go +++ b/src/sync/mutex.go @@ -11,15 +11,9 @@ package sync import ( - "internal/race" - "sync/atomic" - "unsafe" + isync "internal/sync" ) -// Provided by runtime via linkname. -func throw(string) -func fatal(string) - // A Mutex is a mutual exclusion lock. // The zero value for a Mutex is an unlocked mutex. // @@ -34,8 +28,9 @@ func fatal(string) // // [the Go memory model]: https://go.dev/ref/mem type Mutex struct { - state int32 - sema uint32 + _ noCopy + + mu isync.Mutex } // A Locker represents an object that can be locked and unlocked. @@ -44,52 +39,11 @@ type Locker interface { Unlock() } -const ( - mutexLocked = 1 << iota // mutex is locked - mutexWoken - mutexStarving - mutexWaiterShift = iota - - // Mutex fairness. - // - // Mutex can be in 2 modes of operations: normal and starvation. - // In normal mode waiters are queued in FIFO order, but a woken up waiter - // does not own the mutex and competes with new arriving goroutines over - // the ownership. New arriving goroutines have an advantage -- they are - // already running on CPU and there can be lots of them, so a woken up - // waiter has good chances of losing. In such case it is queued at front - // of the wait queue. If a waiter fails to acquire the mutex for more than 1ms, - // it switches mutex to the starvation mode. - // - // In starvation mode ownership of the mutex is directly handed off from - // the unlocking goroutine to the waiter at the front of the queue. - // New arriving goroutines don't try to acquire the mutex even if it appears - // to be unlocked, and don't try to spin. Instead they queue themselves at - // the tail of the wait queue. - // - // If a waiter receives ownership of the mutex and sees that either - // (1) it is the last waiter in the queue, or (2) it waited for less than 1 ms, - // it switches mutex back to normal operation mode. - // - // Normal mode has considerably better performance as a goroutine can acquire - // a mutex several times in a row even if there are blocked waiters. - // Starvation mode is important to prevent pathological cases of tail latency. - starvationThresholdNs = 1e6 -) - // Lock locks m. // If the lock is already in use, the calling goroutine // blocks until the mutex is available. func (m *Mutex) Lock() { - // Fast path: grab unlocked mutex. - if atomic.CompareAndSwapInt32(&m.state, 0, mutexLocked) { - if race.Enabled { - race.Acquire(unsafe.Pointer(m)) - } - return - } - // Slow path (outlined so that the fast path can be inlined) - m.lockSlow() + m.mu.Lock() } // TryLock tries to lock m and reports whether it succeeded. @@ -98,111 +52,7 @@ func (m *Mutex) Lock() { // and use of TryLock is often a sign of a deeper problem // in a particular use of mutexes. func (m *Mutex) TryLock() bool { - old := m.state - if old&(mutexLocked|mutexStarving) != 0 { - return false - } - - // There may be a goroutine waiting for the mutex, but we are - // running now and can try to grab the mutex before that - // goroutine wakes up. - if !atomic.CompareAndSwapInt32(&m.state, old, old|mutexLocked) { - return false - } - - if race.Enabled { - race.Acquire(unsafe.Pointer(m)) - } - return true -} - -func (m *Mutex) lockSlow() { - var waitStartTime int64 - starving := false - awoke := false - iter := 0 - old := m.state - for { - // Don't spin in starvation mode, ownership is handed off to waiters - // so we won't be able to acquire the mutex anyway. - if old&(mutexLocked|mutexStarving) == mutexLocked && runtime_canSpin(iter) { - // Active spinning makes sense. - // Try to set mutexWoken flag to inform Unlock - // to not wake other blocked goroutines. - if !awoke && old&mutexWoken == 0 && old>>mutexWaiterShift != 0 && - atomic.CompareAndSwapInt32(&m.state, old, old|mutexWoken) { - awoke = true - } - runtime_doSpin() - iter++ - old = m.state - continue - } - new := old - // Don't try to acquire starving mutex, new arriving goroutines must queue. - if old&mutexStarving == 0 { - new |= mutexLocked - } - if old&(mutexLocked|mutexStarving) != 0 { - new += 1 << mutexWaiterShift - } - // The current goroutine switches mutex to starvation mode. - // But if the mutex is currently unlocked, don't do the switch. - // Unlock expects that starving mutex has waiters, which will not - // be true in this case. - if starving && old&mutexLocked != 0 { - new |= mutexStarving - } - if awoke { - // The goroutine has been woken from sleep, - // so we need to reset the flag in either case. - if new&mutexWoken == 0 { - throw("sync: inconsistent mutex state") - } - new &^= mutexWoken - } - if atomic.CompareAndSwapInt32(&m.state, old, new) { - if old&(mutexLocked|mutexStarving) == 0 { - break // locked the mutex with CAS - } - // If we were already waiting before, queue at the front of the queue. - queueLifo := waitStartTime != 0 - if waitStartTime == 0 { - waitStartTime = runtime_nanotime() - } - runtime_SemacquireMutex(&m.sema, queueLifo, 1) - starving = starving || runtime_nanotime()-waitStartTime > starvationThresholdNs - old = m.state - if old&mutexStarving != 0 { - // If this goroutine was woken and mutex is in starvation mode, - // ownership was handed off to us but mutex is in somewhat - // inconsistent state: mutexLocked is not set and we are still - // accounted as waiter. Fix that. - if old&(mutexLocked|mutexWoken) != 0 || old>>mutexWaiterShift == 0 { - throw("sync: inconsistent mutex state") - } - delta := int32(mutexLocked - 1<>mutexWaiterShift == 1 { - // Exit starvation mode. - // Critical to do it here and consider wait time. - // Starvation mode is so inefficient, that two goroutines - // can go lock-step infinitely once they switch mutex - // to starvation mode. - delta -= mutexStarving - } - atomic.AddInt32(&m.state, delta) - break - } - awoke = true - iter = 0 - } else { - old = m.state - } - } - - if race.Enabled { - race.Acquire(unsafe.Pointer(m)) - } + return m.mu.TryLock() } // Unlock unlocks m. @@ -212,50 +62,5 @@ func (m *Mutex) lockSlow() { // It is allowed for one goroutine to lock a Mutex and then // arrange for another goroutine to unlock it. func (m *Mutex) Unlock() { - if race.Enabled { - _ = m.state - race.Release(unsafe.Pointer(m)) - } - - // Fast path: drop lock bit. - new := atomic.AddInt32(&m.state, -mutexLocked) - if new != 0 { - // Outlined slow path to allow inlining the fast path. - // To hide unlockSlow during tracing we skip one extra frame when tracing GoUnblock. - m.unlockSlow(new) - } -} - -func (m *Mutex) unlockSlow(new int32) { - if (new+mutexLocked)&mutexLocked == 0 { - fatal("sync: unlock of unlocked mutex") - } - if new&mutexStarving == 0 { - old := new - for { - // If there are no waiters or a goroutine has already - // been woken or grabbed the lock, no need to wake anyone. - // In starvation mode ownership is directly handed off from unlocking - // goroutine to the next waiter. We are not part of this chain, - // since we did not observe mutexStarving when we unlocked the mutex above. - // So get off the way. - if old>>mutexWaiterShift == 0 || old&(mutexLocked|mutexWoken|mutexStarving) != 0 { - return - } - // Grab the right to wake someone. - new = (old - 1< 2 { + t.Errorf("want at most 2 allocations per call to OnceFunc, got %v", allocs) } } func TestOnceValue(t *testing.T) { calls := 0 - f := sync.OnceValue(func() int { + of := func() int { calls++ return calls - }) + } + f := sync.OnceValue(of) allocs := testing.AllocsPerRun(10, func() { f() }) value := f() if calls != 1 { @@ -44,16 +52,23 @@ func TestOnceValue(t *testing.T) { t.Errorf("want value==1, got %d", value) } if allocs != 0 { - t.Errorf("want 0 allocations per call, got %v", allocs) + t.Errorf("want 0 allocations per call to f, got %v", allocs) + } + allocs = testing.AllocsPerRun(10, func() { + f = sync.OnceValue(of) + }) + if allocs > 2 { + t.Errorf("want at most 2 allocations per call to OnceValue, got %v", allocs) } } func TestOnceValues(t *testing.T) { calls := 0 - f := sync.OnceValues(func() (int, int) { + of := func() (int, int) { calls++ return calls, calls + 1 - }) + } + f := sync.OnceValues(of) allocs := testing.AllocsPerRun(10, func() { f() }) v1, v2 := f() if calls != 1 { @@ -63,7 +78,13 @@ func TestOnceValues(t *testing.T) { t.Errorf("want v1==1 and v2==2, got %d and %d", v1, v2) } if allocs != 0 { - t.Errorf("want 0 allocations per call, got %v", allocs) + t.Errorf("want 0 allocations per call to f, got %v", allocs) + } + allocs = testing.AllocsPerRun(10, func() { + f = sync.OnceValues(of) + }) + if allocs > 2 { + t.Errorf("want at most 2 allocations per call to OnceValues, got %v", allocs) } } @@ -198,44 +219,61 @@ func TestOnceXGC(t *testing.T) { f := sync.OnceValues(func() (any, any) { buf[0] = 1; return nil, nil }) return func() { f() } }, + "OnceFunc panic": func(buf []byte) func() { + return sync.OnceFunc(func() { buf[0] = 1; panic("test panic") }) + }, + "OnceValue panic": func(buf []byte) func() { + f := sync.OnceValue(func() any { buf[0] = 1; panic("test panic") }) + return func() { f() } + }, + "OnceValues panic": func(buf []byte) func() { + f := sync.OnceValues(func() (any, any) { buf[0] = 1; panic("test panic") }) + return func() { f() } + }, } for n, fn := range fns { t.Run(n, func(t *testing.T) { buf := make([]byte, 1024) var gc atomic.Bool - runtime.SetFinalizer(&buf[0], func(_ *byte) { - gc.Store(true) - }) + runtime.AddCleanup(&buf[0], func(g *atomic.Bool) { g.Store(true) }, &gc) f := fn(buf) - gcwaitfin() + runCleanups() if gc.Load() != false { t.Fatal("wrapped function garbage collected too early") } - f() - gcwaitfin() + func() { + defer func() { recover() }() + f() + }() + runCleanups() if gc.Load() != true { // Even if f is still alive, the function passed to Once(Func|Value|Values) // is not kept alive after the first call to f. t.Fatal("wrapped function should be garbage collected, but still live") } - f() + func() { + defer func() { recover() }() + f() + }() }) } } -// gcwaitfin performs garbage collection and waits for all finalizers to run. -func gcwaitfin() { +// runCleanups performs garbage collection and waits for all cleanups to run. +func runCleanups() { runtime.GC() - runtime_blockUntilEmptyFinalizerQueue(math.MaxInt64) + runtime_blockUntilEmptyCleanupQueue(math.MaxInt64) } -//go:linkname runtime_blockUntilEmptyFinalizerQueue runtime.blockUntilEmptyFinalizerQueue -func runtime_blockUntilEmptyFinalizerQueue(int64) bool +//go:linkname runtime_blockUntilEmptyCleanupQueue +func runtime_blockUntilEmptyCleanupQueue(int64) bool var ( onceFunc = sync.OnceFunc(func() {}) onceFuncOnce sync.Once + + onceFuncFunc func() ) func doOnceFunc() { @@ -269,6 +307,12 @@ func BenchmarkOnceFunc(b *testing.B) { f() } }) + b.Run("v=Make", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + onceFuncFunc = sync.OnceFunc(func() {}) + } + }) } var ( @@ -276,6 +320,8 @@ var ( onceValueOnce sync.Once onceValueValue int + + onceValueFunc func() int ) func doOnceValue() int { @@ -312,4 +358,80 @@ func BenchmarkOnceValue(b *testing.B) { } } }) + b.Run("v=Make", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + onceValueFunc = sync.OnceValue(func() int { return 42 }) + } + }) +} + +const ( + onceValuesWant1 = 42 + onceValuesWant2 = true +) + +var ( + onceValues = sync.OnceValues(func() (int, bool) { + return onceValuesWant1, onceValuesWant2 + }) + + onceValuesOnce sync.Once + onceValuesValue1 int + onceValuesValue2 bool + + onceValuesFunc func() (int, bool) +) + +func doOnceValues() (int, bool) { + onceValuesOnce.Do(func() { + onceValuesValue1 = onceValuesWant1 + onceValuesValue2 = onceValuesWant2 + }) + return onceValuesValue1, onceValuesValue2 +} + +func BenchmarkOnceValues(b *testing.B) { + // See BenchmarkOnceFunc + b.Run("v=Once", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if got1, got2 := doOnceValues(); got1 != onceValuesWant1 { + b.Fatalf("value 1: got %d, want %d", got1, onceValuesWant1) + } else if got2 != onceValuesWant2 { + b.Fatalf("value 2: got %v, want %v", got2, onceValuesWant2) + } + } + }) + b.Run("v=Global", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if got1, got2 := onceValues(); got1 != onceValuesWant1 { + b.Fatalf("value 1: got %d, want %d", got1, onceValuesWant1) + } else if got2 != onceValuesWant2 { + b.Fatalf("value 2: got %v, want %v", got2, onceValuesWant2) + } + } + }) + b.Run("v=Local", func(b *testing.B) { + b.ReportAllocs() + onceValues := sync.OnceValues(func() (int, bool) { + return onceValuesWant1, onceValuesWant2 + }) + for i := 0; i < b.N; i++ { + if got1, got2 := onceValues(); got1 != onceValuesWant1 { + b.Fatalf("value 1: got %d, want %d", got1, onceValuesWant1) + } else if got2 != onceValuesWant2 { + b.Fatalf("value 2: got %v, want %v", got2, onceValuesWant2) + } + } + }) + b.Run("v=Make", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + onceValuesFunc = sync.OnceValues(func() (int, bool) { + return onceValuesWant1, onceValuesWant2 + }) + } + }) } diff --git a/src/sync/pool_test.go b/src/sync/pool_test.go index b6ee983c2989b2..f0f0d3d40a30af 100644 --- a/src/sync/pool_test.go +++ b/src/sync/pool_test.go @@ -102,35 +102,47 @@ func TestPoolRelease(t *testing.T) { } func testPool(t *testing.T, drain bool) { + if drain { + // Run with GOMAXPROCS=1 if drain is set. The code below implicitly + // assumes it can remove all the pool-cached values with cleanups + // with Get, but this isn't necessarily true if a value gets stuck + // in the private slot for some P. This is especially likely when + // running with mayMoreStackPreempt. We can make this exact, however, + // by setting GOMAXPROCS to 1, so there's only 1 P. This is fine for + // this test, since we're not trying to check any concurrent properties + // of Pool anyway. + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1)) + } + var p Pool const N = 100 -loop: for try := 0; try < 3; try++ { if try == 1 && testing.Short() { break } - var fin, fin1 uint32 + var cln, cln1 uint32 for i := 0; i < N; i++ { v := new(string) - runtime.SetFinalizer(v, func(vv *string) { - atomic.AddUint32(&fin, 1) - }) + runtime.AddCleanup(v, func(f *uint32) { atomic.AddUint32(f, 1) }, &cln) p.Put(v) } if drain { for i := 0; i < N; i++ { p.Get() } - } - for i := 0; i < 5; i++ { + } else { + // Run an extra GC cycles to drop items from the pool. runtime.GC() - time.Sleep(time.Duration(i*100+10) * time.Millisecond) - // 1 pointer can remain on stack or elsewhere - if fin1 = atomic.LoadUint32(&fin); fin1 >= N-1 { - continue loop - } } - t.Fatalf("only %v out of %v resources are finalized on try %v", fin1, N, try) + + // Run a GC and wait for all the cleanups to run. + runtime.GC() + runtime_blockUntilEmptyCleanupQueue(int64(5 * time.Second)) + + // 1 pointer can remain on stack or elsewhere + if cln1 = atomic.LoadUint32(&cln); cln1 < N-1 { + t.Fatalf("only %v out of %v resources are cleaned up on try %v", cln1, N, try) + } } } diff --git a/src/sync/runtime.go b/src/sync/runtime.go index 5a9081358545d5..ae3368e58d0248 100644 --- a/src/sync/runtime.go +++ b/src/sync/runtime.go @@ -13,6 +13,9 @@ import "unsafe" // library and should not be used directly. func runtime_Semacquire(s *uint32) +// SemacquireWaitGroup is like Semacquire, but for WaitGroup.Wait. +func runtime_SemacquireWaitGroup(s *uint32, synctestDurable bool) + // Semacquire(RW)Mutex(R) is like Semacquire, but for profiling contended // Mutexes and RWMutexes. // If lifo is true, queue waiter at the head of wait queue. @@ -21,7 +24,6 @@ func runtime_Semacquire(s *uint32) // The different forms of this function just tell the runtime how to present // the reason for waiting in a backtrace, and is used to compute some metrics. // Otherwise they're functionally identical. -func runtime_SemacquireMutex(s *uint32, lifo bool, skipframes int) func runtime_SemacquireRWMutexR(s *uint32, lifo bool, skipframes int) func runtime_SemacquireRWMutex(s *uint32, lifo bool, skipframes int) @@ -53,11 +55,5 @@ func init() { runtime_notifyListCheck(unsafe.Sizeof(n)) } -// Active spinning runtime support. -// runtime_canSpin reports whether spinning makes sense at the moment. -func runtime_canSpin(i int) bool - -// runtime_doSpin does active spinning. -func runtime_doSpin() - -func runtime_nanotime() int64 +func throw(string) +func fatal(string) diff --git a/src/sync/rwmutex.go b/src/sync/rwmutex.go index 1d5b8fde4a8c62..3fbfd28d7f128a 100644 --- a/src/sync/rwmutex.go +++ b/src/sync/rwmutex.go @@ -24,6 +24,8 @@ import ( // the writer has acquired (and released) the lock, to ensure that // the lock eventually becomes available to the writer. // Note that this prohibits recursive read-locking. +// A [RWMutex.RLock] cannot be upgraded into a [RWMutex.Lock], +// nor can a [RWMutex.Lock] be downgraded into a [RWMutex.RLock]. // // In the terminology of [the Go memory model], // the n'th call to [RWMutex.Unlock] “synchronizes before” the m'th call to Lock @@ -64,7 +66,7 @@ const rwmutexMaxReaders = 1 << 30 // documentation on the [RWMutex] type. func (rw *RWMutex) RLock() { if race.Enabled { - _ = rw.w.state + race.Read(unsafe.Pointer(&rw.w)) race.Disable() } if rw.readerCount.Add(1) < 0 { @@ -84,7 +86,7 @@ func (rw *RWMutex) RLock() { // in a particular use of mutexes. func (rw *RWMutex) TryRLock() bool { if race.Enabled { - _ = rw.w.state + race.Read(unsafe.Pointer(&rw.w)) race.Disable() } for { @@ -111,7 +113,7 @@ func (rw *RWMutex) TryRLock() bool { // on entry to RUnlock. func (rw *RWMutex) RUnlock() { if race.Enabled { - _ = rw.w.state + race.Read(unsafe.Pointer(&rw.w)) race.ReleaseMerge(unsafe.Pointer(&rw.writerSem)) race.Disable() } @@ -141,7 +143,7 @@ func (rw *RWMutex) rUnlockSlow(r int32) { // Lock blocks until the lock is available. func (rw *RWMutex) Lock() { if race.Enabled { - _ = rw.w.state + race.Read(unsafe.Pointer(&rw.w)) race.Disable() } // First, resolve competition with other writers. @@ -166,7 +168,7 @@ func (rw *RWMutex) Lock() { // in a particular use of mutexes. func (rw *RWMutex) TryLock() bool { if race.Enabled { - _ = rw.w.state + race.Read(unsafe.Pointer(&rw.w)) race.Disable() } if !rw.w.TryLock() { @@ -198,7 +200,7 @@ func (rw *RWMutex) TryLock() bool { // arrange for another goroutine to [RWMutex.RUnlock] ([RWMutex.Unlock]) it. func (rw *RWMutex) Unlock() { if race.Enabled { - _ = rw.w.state + race.Read(unsafe.Pointer(&rw.w)) race.Release(unsafe.Pointer(&rw.readerSem)) race.Disable() } diff --git a/src/sync/waitgroup.go b/src/sync/waitgroup.go index 872d6d87c0dded..195f839da417bd 100644 --- a/src/sync/waitgroup.go +++ b/src/sync/waitgroup.go @@ -6,33 +6,65 @@ package sync import ( "internal/race" + "internal/synctest" "sync/atomic" "unsafe" ) -// A WaitGroup waits for a collection of goroutines to finish. -// The main goroutine calls [WaitGroup.Add] to set the number of -// goroutines to wait for. Then each of the goroutines -// runs and calls [WaitGroup.Done] when finished. At the same time, -// [WaitGroup.Wait] can be used to block until all goroutines have finished. +// A WaitGroup is a counting semaphore typically used to wait +// for a group of goroutines or tasks to finish. // -// A WaitGroup must not be copied after first use. +// Typically, a main goroutine will start tasks, each in a new +// goroutine, by calling [WaitGroup.Go] and then wait for all tasks to +// complete by calling [WaitGroup.Wait]. For example: // -// In the terminology of [the Go memory model], a call to [WaitGroup.Done] -// “synchronizes before” the return of any Wait call that it unblocks. +// var wg sync.WaitGroup +// wg.Go(task1) +// wg.Go(task2) +// wg.Wait() // -// [the Go memory model]: https://go.dev/ref/mem +// A WaitGroup may also be used for tracking tasks without using Go to +// start new goroutines by using [WaitGroup.Add] and [WaitGroup.Done]. +// +// The previous example can be rewritten using explicitly created +// goroutines along with Add and Done: +// +// var wg sync.WaitGroup +// wg.Add(1) +// go func() { +// defer wg.Done() +// task1() +// }() +// wg.Add(1) +// go func() { +// defer wg.Done() +// task2() +// }() +// wg.Wait() +// +// This pattern is common in code that predates [WaitGroup.Go]. +// +// A WaitGroup must not be copied after first use. type WaitGroup struct { noCopy noCopy - state atomic.Uint64 // high 32 bits are counter, low 32 bits are waiter count. + // Bits (high to low): + // bits[0:32] counter + // bits[32] flag: synctest bubble membership + // bits[33:64] wait count + state atomic.Uint64 sema uint32 } -// Add adds delta, which may be negative, to the [WaitGroup] counter. +// waitGroupBubbleFlag indicates that a WaitGroup is associated with a synctest bubble. +const waitGroupBubbleFlag = 0x8000_0000 + +// Add adds delta, which may be negative, to the [WaitGroup] task counter. // If the counter becomes zero, all goroutines blocked on [WaitGroup.Wait] are released. // If the counter goes negative, Add panics. // +// Callers should prefer [WaitGroup.Go]. +// // Note that calls with a positive delta that occur when the counter is zero // must happen before a Wait. Calls with a negative delta, or calls with a // positive delta that start when the counter is greater than zero, may happen @@ -51,9 +83,31 @@ func (wg *WaitGroup) Add(delta int) { race.Disable() defer race.Enable() } + bubbled := false + if synctest.IsInBubble() { + // If Add is called from within a bubble, then all Add calls must be made + // from the same bubble. + switch synctest.Associate(wg) { + case synctest.Unbubbled: + case synctest.OtherBubble: + // wg is already associated with a different bubble. + fatal("sync: WaitGroup.Add called from multiple synctest bubbles") + case synctest.CurrentBubble: + bubbled = true + state := wg.state.Or(waitGroupBubbleFlag) + if state != 0 && state&waitGroupBubbleFlag == 0 { + // Add has been called from outside this bubble. + fatal("sync: WaitGroup.Add called from inside and outside synctest bubble") + } + } + } state := wg.state.Add(uint64(delta) << 32) + if state&waitGroupBubbleFlag != 0 && !bubbled { + // Add has been called from within a synctest bubble (and we aren't in one). + fatal("sync: WaitGroup.Add called from inside and outside synctest bubble") + } v := int32(state >> 32) - w := uint32(state) + w := uint32(state & 0x7fffffff) if race.Enabled && delta > 0 && v == int32(delta) { // The first increment must be synchronized with Wait. // Need to model this as a read, because there can be @@ -79,17 +133,30 @@ func (wg *WaitGroup) Add(delta int) { } // Reset waiters count to 0. wg.state.Store(0) + if bubbled { + // Adds must not happen concurrently with wait when counter is 0, + // so we can safely disassociate wg from its current bubble. + synctest.Disassociate(wg) + } for ; w != 0; w-- { runtime_Semrelease(&wg.sema, false, 0) } } -// Done decrements the [WaitGroup] counter by one. +// Done decrements the [WaitGroup] task counter by one. +// It is equivalent to Add(-1). +// +// Callers should prefer [WaitGroup.Go]. +// +// In the terminology of [the Go memory model], a call to Done +// "synchronizes before" the return of any Wait call that it unblocks. +// +// [the Go memory model]: https://go.dev/ref/mem func (wg *WaitGroup) Done() { wg.Add(-1) } -// Wait blocks until the [WaitGroup] counter is zero. +// Wait blocks until the [WaitGroup] task counter is zero. func (wg *WaitGroup) Wait() { if race.Enabled { race.Disable() @@ -97,13 +164,20 @@ func (wg *WaitGroup) Wait() { for { state := wg.state.Load() v := int32(state >> 32) - w := uint32(state) + w := uint32(state & 0x7fffffff) if v == 0 { // Counter is 0, no need to wait. if race.Enabled { race.Enable() race.Acquire(unsafe.Pointer(wg)) } + if w == 0 && state&waitGroupBubbleFlag != 0 && synctest.IsAssociated(wg) { + // Adds must not happen concurrently with wait when counter is 0, + // so we can disassociate wg from its current bubble. + if wg.state.CompareAndSwap(state, 0) { + synctest.Disassociate(wg) + } + } return } // Increment waiters count. @@ -115,15 +189,54 @@ func (wg *WaitGroup) Wait() { // otherwise concurrent Waits will race with each other. race.Write(unsafe.Pointer(&wg.sema)) } - runtime_Semacquire(&wg.sema) - if wg.state.Load() != 0 { - panic("sync: WaitGroup is reused before previous Wait has returned") + synctestDurable := false + if state&waitGroupBubbleFlag != 0 && synctest.IsInBubble() { + if race.Enabled { + race.Enable() + } + if synctest.IsAssociated(wg) { + // Add was called within the current bubble, + // so this Wait is durably blocking. + synctestDurable = true + } + if race.Enabled { + race.Disable() + } } + runtime_SemacquireWaitGroup(&wg.sema, synctestDurable) + isReset := wg.state.Load() != 0 if race.Enabled { race.Enable() race.Acquire(unsafe.Pointer(wg)) } + if isReset { + panic("sync: WaitGroup is reused before previous Wait has returned") + } return } } } + +// Go calls f in a new goroutine and adds that task to the [WaitGroup]. +// When f returns, the task is removed from the WaitGroup. +// +// The function f must not panic. +// +// If the WaitGroup is empty, Go must happen before a [WaitGroup.Wait]. +// Typically, this simply means Go is called to start tasks before Wait is called. +// If the WaitGroup is not empty, Go may happen at any time. +// This means a goroutine started by Go may itself call Go. +// If a WaitGroup is reused to wait for several independent sets of tasks, +// new Go calls must happen after all previous Wait calls have returned. +// +// In the terminology of [the Go memory model], the return from f +// "synchronizes before" the return of any Wait call that it unblocks. +// +// [the Go memory model]: https://go.dev/ref/mem +func (wg *WaitGroup) Go(f func()) { + wg.Add(1) + go func() { + defer wg.Done() + f() + }() +} diff --git a/src/sync/waitgroup_test.go b/src/sync/waitgroup_test.go index 4ded218d2d8d0c..8a948f8972c8a7 100644 --- a/src/sync/waitgroup_test.go +++ b/src/sync/waitgroup_test.go @@ -98,6 +98,18 @@ func TestWaitGroupAlign(t *testing.T) { x.wg.Wait() } +func TestWaitGroupGo(t *testing.T) { + wg := &WaitGroup{} + var i int + wg.Go(func() { + i++ + }) + wg.Wait() + if i != 1 { + t.Fatalf("got %d, want 1", i) + } +} + func BenchmarkWaitGroupUncontended(b *testing.B) { type PaddedWaitGroup struct { WaitGroup diff --git a/src/syscall/badlinkname_unix.go b/src/syscall/badlinkname_unix.go index 4964a830b003d6..70ba073b37b99e 100644 --- a/src/syscall/badlinkname_unix.go +++ b/src/syscall/badlinkname_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris +//go:build unix package syscall diff --git a/src/syscall/dir_plan9.go b/src/syscall/dir_plan9.go index 34869b62091586..835c43e4b10346 100644 --- a/src/syscall/dir_plan9.go +++ b/src/syscall/dir_plan9.go @@ -146,19 +146,19 @@ func pbit8(b []byte, v uint8) []byte { // pbit16 copies the 16-bit number v to b in little-endian order and returns the remaining slice of b. func pbit16(b []byte, v uint16) []byte { - byteorder.LePutUint16(b, v) + byteorder.LEPutUint16(b, v) return b[2:] } // pbit32 copies the 32-bit number v to b in little-endian order and returns the remaining slice of b. func pbit32(b []byte, v uint32) []byte { - byteorder.LePutUint32(b, v) + byteorder.LEPutUint32(b, v) return b[4:] } // pbit64 copies the 64-bit number v to b in little-endian order and returns the remaining slice of b. func pbit64(b []byte, v uint64) []byte { - byteorder.LePutUint64(b, v) + byteorder.LEPutUint64(b, v) return b[8:] } @@ -179,17 +179,17 @@ func gbit8(b []byte) (uint8, []byte) { // //go:nosplit func gbit16(b []byte) (uint16, []byte) { - return byteorder.LeUint16(b), b[2:] + return byteorder.LEUint16(b), b[2:] } // gbit32 reads a 32-bit number in little-endian order from b and returns it with the remaining slice of b. func gbit32(b []byte) (uint32, []byte) { - return byteorder.LeUint32(b), b[4:] + return byteorder.LEUint32(b), b[4:] } // gbit64 reads a 64-bit number in little-endian order from b and returns it with the remaining slice of b. func gbit64(b []byte) (uint64, []byte) { - return byteorder.LeUint64(b), b[8:] + return byteorder.LEUint64(b), b[8:] } // gstring reads a string from b, prefixed with a 16-bit length in little-endian order. diff --git a/src/syscall/dirent.go b/src/syscall/dirent.go index e4f36a191bfadd..c12b1193356d21 100644 --- a/src/syscall/dirent.go +++ b/src/syscall/dirent.go @@ -29,11 +29,11 @@ func readIntBE(b []byte, size uintptr) uint64 { case 1: return uint64(b[0]) case 2: - return uint64(byteorder.BeUint16(b)) + return uint64(byteorder.BEUint16(b)) case 4: - return uint64(byteorder.BeUint32(b)) + return uint64(byteorder.BEUint32(b)) case 8: - return uint64(byteorder.BeUint64(b)) + return byteorder.BEUint64(b) default: panic("syscall: readInt with unsupported size") } @@ -44,11 +44,11 @@ func readIntLE(b []byte, size uintptr) uint64 { case 1: return uint64(b[0]) case 2: - return uint64(byteorder.LeUint16(b)) + return uint64(byteorder.LEUint16(b)) case 4: - return uint64(byteorder.LeUint32(b)) + return uint64(byteorder.LEUint32(b)) case 8: - return uint64(byteorder.LeUint64(b)) + return byteorder.LEUint64(b) default: panic("syscall: readInt with unsupported size") } diff --git a/src/syscall/dirent_test.go b/src/syscall/dirent_test.go index cfa5478feb1bc8..173ccc3ed28ef3 100644 --- a/src/syscall/dirent_test.go +++ b/src/syscall/dirent_test.go @@ -140,7 +140,7 @@ func TestDirentRepeat(t *testing.T) { // Check results slices.Sort(files) slices.Sort(files2) - if strings.Join(files, "|") != strings.Join(files2, "|") { + if !slices.Equal(files, files2) { t.Errorf("bad file list: want\n%q\ngot\n%q", files, files2) } } diff --git a/src/syscall/dll_windows.go b/src/syscall/dll_windows.go index 81134cb0bd27ff..9d779869531788 100644 --- a/src/syscall/dll_windows.go +++ b/src/syscall/dll_windows.go @@ -11,6 +11,15 @@ import ( "unsafe" ) +// Use double underscore to avoid name collision autogenerated functions. +//go:cgo_import_dynamic syscall.__LoadLibraryExW LoadLibraryExW%3 "kernel32.dll" +//go:cgo_import_dynamic syscall.__GetProcAddress GetProcAddress%2 "kernel32.dll" + +var ( + __LoadLibraryExW unsafe.Pointer + __GetProcAddress unsafe.Pointer +) + // DLLError describes reasons for DLL load failures. type DLLError struct { Err error @@ -22,30 +31,111 @@ func (e *DLLError) Error() string { return e.Msg } func (e *DLLError) Unwrap() error { return e.Err } -// Implemented in ../runtime/syscall_windows.go. +// N.B. For the Syscall functions below: +// +// //go:uintptrkeepalive because the uintptr argument may be converted pointers +// that need to be kept alive in the caller. +// +// //go:nosplit because stack copying does not account for uintptrkeepalive, so +// the stack must not grow. Stack copying cannot blindly assume that all +// uintptr arguments are pointers, because some values may look like pointers, +// but not really be pointers, and adjusting their value would break the call. // Deprecated: Use [SyscallN] instead. -func Syscall(trap, nargs, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) +// +//go:nosplit +//go:uintptrkeepalive +func Syscall(trap, nargs, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) { + return syscalln(trap, nargs, a1, a2, a3) +} // Deprecated: Use [SyscallN] instead. -func Syscall6(trap, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) +// +//go:nosplit +//go:uintptrkeepalive +func Syscall6(trap, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) { + return syscalln(trap, nargs, a1, a2, a3, a4, a5, a6) +} // Deprecated: Use [SyscallN] instead. -func Syscall9(trap, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err Errno) +// +//go:nosplit +//go:uintptrkeepalive +func Syscall9(trap, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err Errno) { + return syscalln(trap, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9) +} // Deprecated: Use [SyscallN] instead. -func Syscall12(trap, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12 uintptr) (r1, r2 uintptr, err Errno) +// +//go:nosplit +//go:uintptrkeepalive +func Syscall12(trap, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12 uintptr) (r1, r2 uintptr, err Errno) { + return syscalln(trap, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12) +} // Deprecated: Use [SyscallN] instead. -func Syscall15(trap, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15 uintptr) (r1, r2 uintptr, err Errno) +// +//go:nosplit +//go:uintptrkeepalive +func Syscall15(trap, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15 uintptr) (r1, r2 uintptr, err Errno) { + return syscalln(trap, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15) +} // Deprecated: Use [SyscallN] instead. -func Syscall18(trap, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18 uintptr) (r1, r2 uintptr, err Errno) +// +//go:nosplit +//go:uintptrkeepalive +func Syscall18(trap, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18 uintptr) (r1, r2 uintptr, err Errno) { + return syscalln(trap, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18) +} + +// SyscallN executes procedure p with arguments args. +// +// See [Proc.Call] for more information. +// +//go:nosplit +//go:uintptrkeepalive +func SyscallN(p uintptr, args ...uintptr) (r1, r2 uintptr, err Errno) { + return syscalln(p, uintptr(len(args)), args...) +} + +// syscalln is implemented in runtime/syscall_windows.go. +// +//go:noescape +func syscalln(fn, n uintptr, args ...uintptr) (r1, r2 uintptr, err Errno) + +// N.B. For the loadlibrary, loadlibrary, and getprocaddress functions below: +// +// //go:linkname to act as an allowlist for linker's -checklinkname, as +// golang.org/x/sys/windows linknames these functions. + +//go:linkname loadlibrary +func loadlibrary(filename *uint16) (uintptr, Errno) { + handle, _, err := SyscallN(uintptr(__LoadLibraryExW), uintptr(unsafe.Pointer(filename)), 0, 0) + if handle != 0 { + err = 0 + } + return handle, err +} + +//go:linkname loadsystemlibrary +func loadsystemlibrary(filename *uint16) (uintptr, Errno) { + const _LOAD_LIBRARY_SEARCH_SYSTEM32 = 0x00000800 + handle, _, err := SyscallN(uintptr(__LoadLibraryExW), uintptr(unsafe.Pointer(filename)), 0, _LOAD_LIBRARY_SEARCH_SYSTEM32) + if handle != 0 { + err = 0 + } + return handle, err +} -func SyscallN(trap uintptr, args ...uintptr) (r1, r2 uintptr, err Errno) -func loadlibrary(filename *uint16) (handle uintptr, err Errno) -func loadsystemlibrary(filename *uint16) (handle uintptr, err Errno) -func getprocaddress(handle uintptr, procname *uint8) (proc uintptr, err Errno) +//go:linkname getprocaddress +func getprocaddress(handle uintptr, procname *uint8) (uintptr, Errno) { + proc, _, err := SyscallN(uintptr(__GetProcAddress), handle, uintptr(unsafe.Pointer(procname))) + if proc != 0 { + err = 0 + } + return proc, err +} // A DLL implements access to a single DLL. type DLL struct { diff --git a/src/syscall/env_unix.go b/src/syscall/env_unix.go index 8e87e018e82de3..e1d7e213f7608e 100644 --- a/src/syscall/env_unix.go +++ b/src/syscall/env_unix.go @@ -14,9 +14,6 @@ import ( ) var ( - // envOnce guards initialization by copyenv, which populates env. - envOnce sync.Once - // envLock guards env and envs. envLock sync.RWMutex @@ -31,7 +28,7 @@ var ( func runtime_envs() []string // in package runtime -func copyenv() { +var copyenv = sync.OnceFunc(func() { env = make(map[string]int) for i, s := range envs { for j := 0; j < len(s); j++ { @@ -50,10 +47,10 @@ func copyenv() { } } } -} +}) func Unsetenv(key string) error { - envOnce.Do(copyenv) + copyenv() envLock.Lock() defer envLock.Unlock() @@ -67,7 +64,7 @@ func Unsetenv(key string) error { } func Getenv(key string) (value string, found bool) { - envOnce.Do(copyenv) + copyenv() if len(key) == 0 { return "", false } @@ -89,7 +86,7 @@ func Getenv(key string) (value string, found bool) { } func Setenv(key, value string) error { - envOnce.Do(copyenv) + copyenv() if len(key) == 0 { return EINVAL } @@ -124,20 +121,19 @@ func Setenv(key, value string) error { } func Clearenv() { - envOnce.Do(copyenv) // prevent copyenv in Getenv/Setenv + copyenv() envLock.Lock() defer envLock.Unlock() - for k := range env { - runtimeUnsetenv(k) - } + runtimeClearenv(env) + env = make(map[string]int) envs = []string{} } func Environ() []string { - envOnce.Do(copyenv) + copyenv() envLock.RLock() defer envLock.RUnlock() a := make([]string, 0, len(envs)) diff --git a/src/syscall/env_unix_test.go b/src/syscall/env_unix_test.go new file mode 100644 index 00000000000000..a0050606099c01 --- /dev/null +++ b/src/syscall/env_unix_test.go @@ -0,0 +1,144 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix || (js && wasm) || plan9 || wasip1 + +package syscall_test + +import ( + "fmt" + "strconv" + "strings" + "syscall" + "testing" +) + +type env struct { + name, val string +} + +func genDummyEnv(tb testing.TB, size int) []env { + tb.Helper() + envList := make([]env, size) + for idx := range size { + envList[idx] = env{ + name: fmt.Sprintf("DUMMY_VAR_%d", idx), + val: fmt.Sprintf("val-%d", idx*100), + } + } + return envList +} + +func setDummyEnv(tb testing.TB, envList []env) { + tb.Helper() + for _, env := range envList { + if err := syscall.Setenv(env.name, env.val); err != nil { + tb.Fatalf("setenv %s=%q failed: %v", env.name, env.val, err) + } + } +} + +func setupEnvCleanup(tb testing.TB) { + tb.Helper() + originalEnv := map[string]string{} + for _, env := range syscall.Environ() { + fields := strings.SplitN(env, "=", 2) + name, val := fields[0], fields[1] + originalEnv[name] = val + } + tb.Cleanup(func() { + syscall.Clearenv() + for name, val := range originalEnv { + if err := syscall.Setenv(name, val); err != nil { + tb.Fatalf("could not reset env %s=%q: %v", name, val, err) + } + } + }) +} + +func TestClearenv(t *testing.T) { + setupEnvCleanup(t) + + t.Run("DummyVars-4096", func(t *testing.T) { + envList := genDummyEnv(t, 4096) + setDummyEnv(t, envList) + + if env := syscall.Environ(); len(env) < 4096 { + t.Fatalf("env is missing dummy variables: %v", env) + } + for idx := range 4096 { + name := fmt.Sprintf("DUMMY_VAR_%d", idx) + if _, ok := syscall.Getenv(name); !ok { + t.Fatalf("env is missing dummy variable %s", name) + } + } + + syscall.Clearenv() + + if env := syscall.Environ(); len(env) != 0 { + t.Fatalf("clearenv should've cleared all variables: %v still set", env) + } + for idx := range 4096 { + name := fmt.Sprintf("DUMMY_VAR_%d", idx) + if val, ok := syscall.Getenv(name); ok { + t.Fatalf("clearenv should've cleared all variables: %s=%q still set", name, val) + } + } + }) + + // Test that GODEBUG getting cleared by Clearenv also resets the behaviour. + t.Run("GODEBUG", func(t *testing.T) { + envList := genDummyEnv(t, 100) + setDummyEnv(t, envList) + + doNilPanic := func() (ret any) { + defer func() { + ret = recover() + }() + panic(nil) + } + + // Allow panic(nil). + if err := syscall.Setenv("GODEBUG", "panicnil=1"); err != nil { + t.Fatalf("setenv GODEBUG=panicnil=1 failed: %v", err) + } + + got := doNilPanic() + if got != nil { + t.Fatalf("GODEBUG=panicnil=1 did not allow for nil panic: got %#v", got) + } + + // Disallow panic(nil). + syscall.Clearenv() + + if env := syscall.Environ(); len(env) != 0 { + t.Fatalf("clearenv should've cleared all variables: %v still set", env) + } + + got = doNilPanic() + if got == nil { + t.Fatalf("GODEBUG=panicnil=1 being unset didn't reset panicnil behaviour") + } + if godebug, ok := syscall.Getenv("GODEBUG"); ok { + t.Fatalf("GODEBUG still exists in environment despite being unset: GODEBUG=%q", godebug) + } + }) +} + +func BenchmarkClearenv(b *testing.B) { + setupEnvCleanup(b) + b.ResetTimer() + for _, size := range []int{100, 1000, 10000} { + b.Run(strconv.Itoa(size), func(b *testing.B) { + envList := genDummyEnv(b, size) + for b.Loop() { + // Ideally we would use b.StopTimer() for the setDummyEnv + // portion, but this causes the benchmark time to get confused + // and take forever. See . + setDummyEnv(b, envList) + syscall.Clearenv() + } + }) + } +} diff --git a/src/syscall/exec_bsd.go b/src/syscall/exec_bsd.go index 149cc2f11c128c..bbdab46de48c03 100644 --- a/src/syscall/exec_bsd.go +++ b/src/syscall/exec_bsd.go @@ -293,3 +293,8 @@ childerror: RawSyscall(SYS_EXIT, 253, 0, 0) } } + +// forkAndExecFailureCleanup cleans up after an exec failure. +func forkAndExecFailureCleanup(attr *ProcAttr, sys *SysProcAttr) { + // Nothing to do. +} diff --git a/src/syscall/exec_freebsd.go b/src/syscall/exec_freebsd.go index 3226cb88cd999a..e2896d2cfea2b9 100644 --- a/src/syscall/exec_freebsd.go +++ b/src/syscall/exec_freebsd.go @@ -68,13 +68,15 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr pgrp _C_int cred *Credential ngroups, groups uintptr - upid uintptr + upid, ppid uintptr ) rlim := origRlimitNofile.Load() // Record parent PID so child can test if it has died. - ppid, _, _ := RawSyscall(SYS_GETPID, 0, 0, 0) + if sys.Pdeathsig != 0 { + ppid, _, _ = RawSyscall(SYS_GETPID, 0, 0, 0) + } // guard against side effects of shuffling fds below. // Make sure that nextfd is beyond any currently open files so @@ -317,3 +319,8 @@ childerror: RawSyscall(SYS_EXIT, 253, 0, 0) } } + +// forkAndExecFailureCleanup cleans up after an exec failure. +func forkAndExecFailureCleanup(attr *ProcAttr, sys *SysProcAttr) { + // Nothing to do. +} diff --git a/src/syscall/exec_freebsd_test.go b/src/syscall/exec_freebsd_test.go index 2e9513f0980dc8..34c4b9e2bbb6bc 100644 --- a/src/syscall/exec_freebsd_test.go +++ b/src/syscall/exec_freebsd_test.go @@ -88,7 +88,7 @@ func TestJailAttach(t *testing.T) { t.Fatalf("Build of syscall in jail root failed, output %v, err %v", o, err) } - cmd = exec.Command("/syscall.test", "-test.run=TestJailAttach", "/") + cmd = exec.Command("/syscall.test", "-test.run=^TestJailAttach$", "/") cmd.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=1") cmd.SysProcAttr = &syscall.SysProcAttr{Jail: jid} out, err := cmd.CombinedOutput() diff --git a/src/syscall/exec_libc.go b/src/syscall/exec_libc.go index 768e8c131c1323..2e7e33d9ef2ebd 100644 --- a/src/syscall/exec_libc.go +++ b/src/syscall/exec_libc.go @@ -61,7 +61,13 @@ func write1(fd uintptr, buf uintptr, nbyte uintptr) (n uintptr, err Errno) // syscall defines this global on our behalf to avoid a build dependency on other platforms func init() { - execveLibc = execve + execveLibc = execveLibcWrapper +} + +func execveLibcWrapper(path *byte, argv **byte, envp **byte) error { + return execve(uintptr(unsafe.Pointer(path)), + uintptr(unsafe.Pointer(argv)), + uintptr(unsafe.Pointer(envp))) } // Fork, dup fd onto 0..len(fd), and exec(argv0, argvv, envv) in child. @@ -314,6 +320,11 @@ childerror: } } +// forkAndExecFailureCleanup cleans up after an exec failure. +func forkAndExecFailureCleanup(attr *ProcAttr, sys *SysProcAttr) { + // Nothing to do. +} + func ioctlPtr(fd, req uintptr, arg unsafe.Pointer) (err Errno) { return ioctl(fd, req, uintptr(arg)) } diff --git a/src/syscall/exec_libc2.go b/src/syscall/exec_libc2.go index 7a6750084486cf..5de09dfe998070 100644 --- a/src/syscall/exec_libc2.go +++ b/src/syscall/exec_libc2.go @@ -59,7 +59,6 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr r1 uintptr nextfd int i int - err error pgrp _C_int cred *Credential ngroups, groups uintptr @@ -99,8 +98,12 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr // Enable tracing if requested. if sys.Ptrace { - if err = ptrace(PTRACE_TRACEME, 0, 0, 0); err != nil { - err1 = err.(Errno) + if runtime.GOOS == "ios" { + err1 = ENOSYS + goto childerror + } + _, _, err1 = rawSyscall6(abi.FuncPCABI0(libc_ptrace_trampoline), PTRACE_TRACEME, 0, 0, 0, 0, 0) + if err1 != 0 { goto childerror } } @@ -289,3 +292,8 @@ childerror: rawSyscall(abi.FuncPCABI0(libc_exit_trampoline), 253, 0, 0) } } + +// forkAndExecFailureCleanup cleans up after an exec failure. +func forkAndExecFailureCleanup(attr *ProcAttr, sys *SysProcAttr) { + // Nothing to do. +} diff --git a/src/syscall/exec_linux.go b/src/syscall/exec_linux.go index e4b9ce1bf47da3..14c13e273a4983 100644 --- a/src/syscall/exec_linux.go +++ b/src/syscall/exec_linux.go @@ -7,6 +7,7 @@ package syscall import ( + errpkg "errors" "internal/itoa" "runtime" "unsafe" @@ -243,7 +244,7 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att nextfd int i int caps caps - fd1, flags uintptr + fd1, flags, ppid uintptr puid, psetgroups, pgid []byte uidmap, setgroups, gidmap []byte clone3 *cloneArgs @@ -252,10 +253,12 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att cred *Credential ngroups, groups uintptr c uintptr + rlim *Rlimit + lim Rlimit ) pidfd = -1 - rlim := origRlimitNofile.Load() + rlim = origRlimitNofile.Load() if sys.UidMappings != nil { puid = []byte("/proc/self/uid_map\000") @@ -275,7 +278,9 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att } // Record parent PID so child can test if it has died. - ppid, _ := rawSyscallNoError(SYS_GETPID, 0, 0, 0) + if sys.Pdeathsig != 0 { + ppid, _ = rawSyscallNoError(SYS_GETPID, 0, 0, 0) + } // Guard against side effects of shuffling fds below. // Make sure that nextfd is beyond any currently open files so @@ -328,6 +333,7 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att if clone3 != nil { pid, err1 = rawVforkSyscall(_SYS_clone3, uintptr(unsafe.Pointer(clone3)), unsafe.Sizeof(*clone3), 0) } else { + // N.B. Keep in sync with doCheckClonePidfd. flags |= uintptr(SIGCHLD) if runtime.GOARCH == "s390x" { // On Linux/s390, the first two arguments of clone(2) are swapped. @@ -361,11 +367,11 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att if _, _, err1 = RawSyscall(SYS_CLOSE, uintptr(mapPipe[1]), 0, 0); err1 != 0 { goto childerror } - pid, _, err1 = RawSyscall(SYS_READ, uintptr(mapPipe[0]), uintptr(unsafe.Pointer(&err2)), unsafe.Sizeof(err2)) + c, _, err1 = RawSyscall(SYS_READ, uintptr(mapPipe[0]), uintptr(unsafe.Pointer(&err2)), unsafe.Sizeof(err2)) if err1 != 0 { goto childerror } - if pid != unsafe.Sizeof(err2) { + if c != unsafe.Sizeof(err2) { err1 = EINVAL goto childerror } @@ -423,7 +429,7 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att if fd1, _, err1 = RawSyscall6(SYS_OPENAT, uintptr(dirfd), uintptr(unsafe.Pointer(&psetgroups[0])), uintptr(O_WRONLY), 0, 0, 0); err1 != 0 { goto childerror } - pid, _, err1 = RawSyscall(SYS_WRITE, fd1, uintptr(unsafe.Pointer(&setgroups[0])), uintptr(len(setgroups))) + _, _, err1 = RawSyscall(SYS_WRITE, fd1, uintptr(unsafe.Pointer(&setgroups[0])), uintptr(len(setgroups))) if err1 != 0 { goto childerror } @@ -434,7 +440,7 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att if fd1, _, err1 = RawSyscall6(SYS_OPENAT, uintptr(dirfd), uintptr(unsafe.Pointer(&pgid[0])), uintptr(O_WRONLY), 0, 0, 0); err1 != 0 { goto childerror } - pid, _, err1 = RawSyscall(SYS_WRITE, fd1, uintptr(unsafe.Pointer(&gidmap[0])), uintptr(len(gidmap))) + _, _, err1 = RawSyscall(SYS_WRITE, fd1, uintptr(unsafe.Pointer(&gidmap[0])), uintptr(len(gidmap))) if err1 != 0 { goto childerror } @@ -448,7 +454,7 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att if fd1, _, err1 = RawSyscall6(SYS_OPENAT, uintptr(dirfd), uintptr(unsafe.Pointer(&puid[0])), uintptr(O_WRONLY), 0, 0, 0); err1 != 0 { goto childerror } - pid, _, err1 = RawSyscall(SYS_WRITE, fd1, uintptr(unsafe.Pointer(&uidmap[0])), uintptr(len(uidmap))) + _, _, err1 = RawSyscall(SYS_WRITE, fd1, uintptr(unsafe.Pointer(&uidmap[0])), uintptr(len(uidmap))) if err1 != 0 { goto childerror } @@ -632,7 +638,20 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att // Restore original rlimit. if rlim != nil { - rawSetrlimit(RLIMIT_NOFILE, rlim) + // Some other process may have changed our rlimit by + // calling prlimit. We can check for that case because + // our current rlimit will not be the value we set when + // caching the rlimit in the init function in rlimit.go. + // + // Note that this test is imperfect, since it won't catch + // the case in which some other process used prlimit to + // set our rlimits to max-1/max. In that case we will fall + // back to the original cur/max when starting the child. + // We hope that setting to max-1/max is unlikely. + _, _, err1 = RawSyscall6(SYS_PRLIMIT64, 0, RLIMIT_NOFILE, 0, uintptr(unsafe.Pointer(&lim)), 0, 0) + if err1 != 0 || (lim.Cur == rlim.Max-1 && lim.Max == rlim.Max) { + RawSyscall6(SYS_PRLIMIT64, 0, RLIMIT_NOFILE, uintptr(unsafe.Pointer(rlim)), 0, 0, 0) + } } // Enable tracing if requested. @@ -735,3 +754,120 @@ func writeUidGidMappings(pid int, sys *SysProcAttr) error { return nil } + +// forkAndExecFailureCleanup cleans up after an exec failure. +func forkAndExecFailureCleanup(attr *ProcAttr, sys *SysProcAttr) { + if sys.PidFD != nil && *sys.PidFD != -1 { + Close(*sys.PidFD) + *sys.PidFD = -1 + } +} + +// checkClonePidfd verifies that clone(CLONE_PIDFD) works by actually doing a +// clone. +// +//go:linkname os_checkClonePidfd os.checkClonePidfd +func os_checkClonePidfd() error { + pidfd := int32(-1) + pid, errno := doCheckClonePidfd(&pidfd) + if errno != 0 { + return errno + } + + if pidfd == -1 { + // Bad: CLONE_PIDFD failed to provide a pidfd. Reap the process + // before returning. + + var err error + for { + var status WaitStatus + // WCLONE is an untyped constant that sets bit 31, so + // it cannot convert directly to int on 32-bit + // GOARCHes. We must convert through another type + // first. + flags := uint(WCLONE) + _, err = Wait4(int(pid), &status, int(flags), nil) + if err != EINTR { + break + } + } + if err != nil { + return err + } + + return errpkg.New("clone(CLONE_PIDFD) failed to return pidfd") + } + + // Good: CLONE_PIDFD provided a pidfd. Reap the process and close the + // pidfd. + defer Close(int(pidfd)) + + // TODO(roland): this is necessary to prevent valgrind from complaining + // about passing 0x0 to waitid, which is doesn't like. This is clearly not + // ideal. The structures are copied (mostly) verbatim from syscall/unix, + // which we obviously cannot import because of an import loop. + + const is64bit = ^uint(0) >> 63 // 0 for 32-bit hosts, 1 for 64-bit ones. + type sigInfo struct { + Signo int32 + _ struct { + Errno int32 + Code int32 + } // Two int32 fields, swapped on MIPS. + _ [is64bit]int32 // Extra padding for 64-bit hosts only. + + // End of common part. Beginning of signal-specific part. + + Pid int32 + Uid uint32 + Status int32 + + // Pad to 128 bytes. + _ [128 - (6+is64bit)*4]byte + } + + for { + const _P_PIDFD = 3 + var info sigInfo + _, _, errno = Syscall6(SYS_WAITID, _P_PIDFD, uintptr(pidfd), uintptr(unsafe.Pointer(&info)), WEXITED|WCLONE, 0, 0) + if errno != EINTR { + break + } + } + if errno != 0 { + return errno + } + + return nil +} + +// doCheckClonePidfd implements the actual clone call of os_checkClonePidfd and +// child execution. This is a separate function so we can separate the child's +// and parent's stack frames if we're using vfork. +// +// This is go:noinline because the point is to keep the stack frames of this +// and os_checkClonePidfd separate. +// +//go:noinline +func doCheckClonePidfd(pidfd *int32) (pid uintptr, errno Errno) { + flags := uintptr(CLONE_VFORK | CLONE_VM | CLONE_PIDFD) + if runtime.GOARCH == "s390x" { + // On Linux/s390, the first two arguments of clone(2) are swapped. + pid, errno = rawVforkSyscall(SYS_CLONE, 0, flags, uintptr(unsafe.Pointer(pidfd))) + } else { + pid, errno = rawVforkSyscall(SYS_CLONE, flags, 0, uintptr(unsafe.Pointer(pidfd))) + } + if errno != 0 || pid != 0 { + // If we're in the parent, we must return immediately + // so we're not in the same stack frame as the child. + // This can at most use the return PC, which the child + // will not modify, and the results of + // rawVforkSyscall, which must have been written after + // the child was replaced. + return + } + + for { + RawSyscall(SYS_EXIT_GROUP, 0, 0, 0) + } +} diff --git a/src/syscall/exec_linux_test.go b/src/syscall/exec_linux_test.go index 079220eab16ce7..ac04e51b144cb3 100644 --- a/src/syscall/exec_linux_test.go +++ b/src/syscall/exec_linux_test.go @@ -11,6 +11,7 @@ import ( "errors" "flag" "fmt" + "internal/asan" "internal/platform" "internal/syscall/unix" "internal/testenv" @@ -50,7 +51,6 @@ func whoamiNEWUSER(t *testing.T, uid, gid int, setgroups bool) *exec.Cmd { func TestCloneNEWUSERAndRemap(t *testing.T) { for _, setgroups := range []bool{false, true} { - setgroups := setgroups t.Run(fmt.Sprintf("setgroups=%v", setgroups), func(t *testing.T) { uid := os.Getuid() gid := os.Getgid() @@ -232,12 +232,7 @@ func TestUnshareMountNameSpace(t *testing.T) { os.Exit(0) } - testenv.MustHaveExec(t) - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - + exe := testenv.Executable(t) d := t.TempDir() t.Cleanup(func() { // If the subprocess fails to unshare the parent directory, force-unmount it @@ -339,6 +334,10 @@ func TestUnshareMountNameSpaceChroot(t *testing.T) { // Test for Issue 29789: unshare fails when uid/gid mapping is specified func TestUnshareUidGidMapping(t *testing.T) { + if asan.Enabled { + t.Skip("test fails with ASAN beause the ASAN leak checker fails finding memory regions") + } + if os.Getenv("GO_WANT_HELPER_PROCESS") == "1" { defer os.Exit(0) if err := syscall.Chroot(os.TempDir()); err != nil { @@ -351,12 +350,7 @@ func TestUnshareUidGidMapping(t *testing.T) { t.Skip("test exercises unprivileged user namespace, fails with privileges") } - testenv.MustHaveExec(t) - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - + exe := testenv.Executable(t) cmd := testenv.Command(t, exe, "-test.run=^TestUnshareUidGidMapping$") cmd.Env = append(cmd.Environ(), "GO_WANT_HELPER_PROCESS=1") cmd.SysProcAttr = &syscall.SysProcAttr{ @@ -434,8 +428,6 @@ func prepareCgroupFD(t *testing.T) (int, string) { } func TestUseCgroupFD(t *testing.T) { - testenv.MustHaveExec(t) - if os.Getenv("GO_WANT_HELPER_PROCESS") == "1" { // Read and print own cgroup path. selfCg, err := os.ReadFile("/proc/self/cgroup") @@ -447,11 +439,7 @@ func TestUseCgroupFD(t *testing.T) { os.Exit(0) } - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - + exe := testenv.Executable(t) fd, suffix := prepareCgroupFD(t) cmd := testenv.Command(t, exe, "-test.run=^TestUseCgroupFD$") @@ -478,8 +466,6 @@ func TestUseCgroupFD(t *testing.T) { } func TestCloneTimeNamespace(t *testing.T) { - testenv.MustHaveExec(t) - if os.Getenv("GO_WANT_HELPER_PROCESS") == "1" { timens, err := os.Readlink("/proc/self/ns/time") if err != nil { @@ -490,11 +476,7 @@ func TestCloneTimeNamespace(t *testing.T) { os.Exit(0) } - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - + exe := testenv.Executable(t) cmd := testenv.Command(t, exe, "-test.run=^TestCloneTimeNamespace$") cmd.Env = append(cmd.Environ(), "GO_WANT_HELPER_PROCESS=1") cmd.SysProcAttr = &syscall.SysProcAttr{ @@ -524,18 +506,12 @@ func TestCloneTimeNamespace(t *testing.T) { } func testPidFD(t *testing.T, userns bool) error { - testenv.MustHaveExec(t) - if os.Getenv("GO_WANT_HELPER_PROCESS") == "1" { // Child: wait for a signal. time.Sleep(time.Hour) } - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - + exe := testenv.Executable(t) var pidfd int cmd := testenv.Command(t, exe, "-test.run=^TestPidFD$") cmd.Env = append(cmd.Environ(), "GO_WANT_HELPER_PROCESS=1") @@ -568,7 +544,7 @@ func testPidFD(t *testing.T, userns bool) error { t.Fatal("pidfd_send_signal syscall failed:", err) } // Check if the child received our signal. - err = cmd.Wait() + err := cmd.Wait() if cmd.ProcessState == nil || cmd.ProcessState.Sys().(syscall.WaitStatus).Signal() != sig { t.Fatal("unexpected child error:", err) } @@ -674,7 +650,7 @@ func testAmbientCaps(t *testing.T, userns bool) { u, err := user.Lookup("nobody") if err != nil { - t.Fatal(err) + t.Skip("skipping: the nobody user does not exist; see Issue 71644") } uid, err := strconv.ParseInt(u.Uid, 0, 32) if err != nil { @@ -695,12 +671,7 @@ func testAmbientCaps(t *testing.T, userns bool) { os.Remove(f.Name()) }) - testenv.MustHaveExec(t) - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - + exe := testenv.Executable(t) e, err := os.Open(exe) if err != nil { t.Fatal(err) diff --git a/src/syscall/exec_pdeathsig_test.go b/src/syscall/exec_pdeathsig_test.go index a907afd900a002..22bb2b44749956 100644 --- a/src/syscall/exec_pdeathsig_test.go +++ b/src/syscall/exec_pdeathsig_test.go @@ -34,11 +34,7 @@ func TestDeathSignalSetuid(t *testing.T) { // // TODO(bcmills): Why do we believe that another users will be able to // execute a binary in this directory? (It could be mounted noexec.) - tempDir, err := os.MkdirTemp("", "TestDeathSignal") - if err != nil { - t.Fatalf("cannot create temporary directory: %v", err) - } - defer os.RemoveAll(tempDir) + tempDir := t.TempDir() os.Chmod(tempDir, 0755) tmpBinary := filepath.Join(tempDir, filepath.Base(os.Args[0])) diff --git a/src/syscall/exec_unix.go b/src/syscall/exec_unix.go index 1b90aa7e72e0ed..b830057dff80a3 100644 --- a/src/syscall/exec_unix.go +++ b/src/syscall/exec_unix.go @@ -237,6 +237,10 @@ func forkExec(argv0 string, argv []string, attr *ProcAttr) (pid int, err error) for err1 == EINTR { _, err1 = Wait4(pid, &wstatus, 0, nil) } + + // OS-specific cleanup on failure. + forkAndExecFailureCleanup(attr, sys) + return 0, err } @@ -261,9 +265,7 @@ func runtime_AfterExec() // execveLibc is non-nil on OS using libc syscall, set to execve in exec_libc.go; this // avoids a build dependency for other platforms. -var execveLibc func(path uintptr, argv uintptr, envp uintptr) Errno -var execveDarwin func(path *byte, argv **byte, envp **byte) error -var execveOpenBSD func(path *byte, argv **byte, envp **byte) error +var execveLibc func(path *byte, argv **byte, envp **byte) error // Exec invokes the execve(2) system call. func Exec(argv0 string, argv []string, envv []string) (err error) { @@ -287,19 +289,12 @@ func Exec(argv0 string, argv []string, envv []string) (err error) { } var err1 error - if runtime.GOOS == "solaris" || runtime.GOOS == "illumos" || runtime.GOOS == "aix" { - // RawSyscall should never be used on Solaris, illumos, or AIX. - err1 = execveLibc( - uintptr(unsafe.Pointer(argv0p)), - uintptr(unsafe.Pointer(&argvp[0])), - uintptr(unsafe.Pointer(&envvp[0]))) - } else if runtime.GOOS == "darwin" || runtime.GOOS == "ios" { - // Similarly on Darwin. - err1 = execveDarwin(argv0p, &argvp[0], &envvp[0]) - } else if runtime.GOOS == "openbsd" && runtime.GOARCH != "mips64" { - // Similarly on OpenBSD. - err1 = execveOpenBSD(argv0p, &argvp[0], &envvp[0]) - } else { + switch runtime.GOOS { + case "aix", "darwin", "illumos", "ios", "openbsd", "solaris": + // RawSyscall should never be used on these platforms. + err1 = execveLibc(argv0p, &argvp[0], &envvp[0]) + + default: _, _, err1 = RawSyscall(SYS_EXECVE, uintptr(unsafe.Pointer(argv0p)), uintptr(unsafe.Pointer(&argvp[0])), diff --git a/src/syscall/exec_unix_test.go b/src/syscall/exec_unix_test.go index bb0b3e703714a6..3a95356c1e5ece 100644 --- a/src/syscall/exec_unix_test.go +++ b/src/syscall/exec_unix_test.go @@ -302,8 +302,7 @@ func TestInvalidExec(t *testing.T) { // TestExec is for issue #41702. func TestExec(t *testing.T) { - testenv.MustHaveExec(t) - cmd := exec.Command(os.Args[0], "-test.run=^TestExecHelper$") + cmd := exec.Command(testenv.Executable(t), "-test.run=^TestExecHelper$") cmd.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=2") o, err := cmd.CombinedOutput() if err != nil { @@ -354,12 +353,8 @@ func TestRlimitRestored(t *testing.T) { t.Skip("skipping test because rlimit not adjusted at startup") } - executable, err := os.Executable() - if err != nil { - executable = os.Args[0] - } - - cmd := testenv.Command(t, executable, "-test.run=^TestRlimitRestored$") + exe := testenv.Executable(t) + cmd := testenv.Command(t, exe, "-test.run=^TestRlimitRestored$") cmd = testenv.CleanCmdEnv(cmd) cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=1") diff --git a/src/syscall/exec_windows.go b/src/syscall/exec_windows.go index 1220de4cdf1e36..96089fcd93a634 100644 --- a/src/syscall/exec_windows.go +++ b/src/syscall/exec_windows.go @@ -9,6 +9,7 @@ package syscall import ( "internal/bytealg" "runtime" + "slices" "sync" "unicode/utf16" "unsafe" @@ -113,6 +114,49 @@ func makeCmdLine(args []string) string { return string(b) } +func envSorted(envv []string) []string { + if len(envv) < 2 { + return envv + } + + lowerKeyCache := map[string][]byte{} // lowercased keys to avoid recomputing them in sort + lowerKey := func(kv string) []byte { + eq := bytealg.IndexByteString(kv, '=') + if eq < 0 { + return nil + } + k := kv[:eq] + v, ok := lowerKeyCache[k] + if !ok { + v = []byte(k) + for i, b := range v { + // We only normalize ASCII for now. + // In practice, all environment variables are ASCII, and the + // syscall package can't import "unicode" anyway. + // Also, per https://nullprogram.com/blog/2023/08/23/ the + // sorting of environment variables doesn't really matter. + // TODO(bradfitz): use RtlCompareUnicodeString instead, + // per that blog post? For now, ASCII is good enough. + if 'a' <= b && b <= 'z' { + v[i] -= 'a' - 'A' + } + } + lowerKeyCache[k] = v + } + return v + } + + cmpEnv := func(a, b string) int { + return bytealg.Compare(lowerKey(a), lowerKey(b)) + } + + if !slices.IsSortedFunc(envv, cmpEnv) { + envv = slices.Clone(envv) + slices.SortFunc(envv, cmpEnv) + } + return envv +} + // createEnvBlock converts an array of environment strings into // the representation required by CreateProcess: a sequence of NUL // terminated strings followed by a nil. @@ -122,6 +166,12 @@ func createEnvBlock(envv []string) ([]uint16, error) { if len(envv) == 0 { return utf16.Encode([]rune("\x00\x00")), nil } + + // https://learn.microsoft.com/en-us/windows/win32/procthread/changing-environment-variables + // says that: "All strings in the environment block must be sorted + // alphabetically by name." + envv = envSorted(envv) + var length int for _, s := range envv { if bytealg.IndexByteString(s, 0) != -1 { @@ -332,12 +382,12 @@ func StartProcess(argv0 string, argv []string, attr *ProcAttr) (pid int, handle defer DuplicateHandle(parentProcess, fd[i], 0, nil, 0, false, DUPLICATE_CLOSE_SOURCE) } } - si := new(_STARTUPINFOEXW) - si.ProcThreadAttributeList, err = newProcThreadAttributeList(2) + procAttrList, err := newProcThreadAttributeList(2) if err != nil { return 0, 0, err } - defer deleteProcThreadAttributeList(si.ProcThreadAttributeList) + defer procAttrList.delete() + si := new(_STARTUPINFOEXW) si.Cb = uint32(unsafe.Sizeof(*si)) si.Flags = STARTF_USESTDHANDLES if sys.HideWindow { @@ -345,7 +395,7 @@ func StartProcess(argv0 string, argv []string, attr *ProcAttr) (pid int, handle si.ShowWindow = SW_HIDE } if sys.ParentProcess != 0 { - err = updateProcThreadAttribute(si.ProcThreadAttributeList, 0, _PROC_THREAD_ATTRIBUTE_PARENT_PROCESS, unsafe.Pointer(&sys.ParentProcess), unsafe.Sizeof(sys.ParentProcess), nil, nil) + err = procAttrList.update(_PROC_THREAD_ATTRIBUTE_PARENT_PROCESS, unsafe.Pointer(&sys.ParentProcess), unsafe.Sizeof(sys.ParentProcess)) if err != nil { return 0, 0, err } @@ -371,7 +421,7 @@ func StartProcess(argv0 string, argv []string, attr *ProcAttr) (pid int, handle // Do not accidentally inherit more than these handles. if willInheritHandles { - err = updateProcThreadAttribute(si.ProcThreadAttributeList, 0, _PROC_THREAD_ATTRIBUTE_HANDLE_LIST, unsafe.Pointer(&fd[0]), uintptr(len(fd))*unsafe.Sizeof(fd[0]), nil, nil) + err = procAttrList.update(_PROC_THREAD_ATTRIBUTE_HANDLE_LIST, unsafe.Pointer(&fd[0]), uintptr(len(fd))*unsafe.Sizeof(fd[0])) if err != nil { return 0, 0, err } @@ -382,6 +432,7 @@ func StartProcess(argv0 string, argv []string, attr *ProcAttr) (pid int, handle return 0, 0, err } + si.ProcThreadAttributeList = procAttrList.list() pi := new(ProcessInformation) flags := sys.CreationFlags | CREATE_UNICODE_ENVIRONMENT | _EXTENDED_STARTUPINFO_PRESENT if sys.Token != 0 { diff --git a/src/syscall/exec_windows_test.go b/src/syscall/exec_windows_test.go index 5cacf42b6b92d0..60a995e4e74eff 100644 --- a/src/syscall/exec_windows_test.go +++ b/src/syscall/exec_windows_test.go @@ -6,9 +6,11 @@ package syscall_test import ( "fmt" + "internal/testenv" "os" "os/exec" "path/filepath" + "slices" "syscall" "testing" "time" @@ -47,6 +49,37 @@ func TestEscapeArg(t *testing.T) { } } +func TestEnvBlockSorted(t *testing.T) { + tests := []struct { + env []string + want []string + }{ + {}, + { + env: []string{"A=1"}, + want: []string{"A=1"}, + }, + { + env: []string{"A=1", "B=2", "C=3"}, + want: []string{"A=1", "B=2", "C=3"}, + }, + { + env: []string{"C=3", "B=2", "A=1"}, + want: []string{"A=1", "B=2", "C=3"}, + }, + { + env: []string{"c=3", "B=2", "a=1"}, + want: []string{"a=1", "B=2", "c=3"}, + }, + } + for _, tt := range tests { + got := syscall.EnvSorted(tt.env) + if !slices.Equal(got, tt.want) { + t.Errorf("EnvSorted(%q) = %q, want %q", tt.env, got, tt.want) + } + } +} + func TestChangingProcessParent(t *testing.T) { if os.Getenv("GO_WANT_HELPER_PROCESS") == "parent" { // in parent process @@ -73,7 +106,7 @@ func TestChangingProcessParent(t *testing.T) { // run parent process - parent := exec.Command(os.Args[0], "-test.run=^TestChangingProcessParent$") + parent := exec.Command(testenv.Executable(t), "-test.run=^TestChangingProcessParent$") parent.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=parent") err := parent.Start() if err != nil { @@ -96,7 +129,7 @@ func TestChangingProcessParent(t *testing.T) { } defer syscall.CloseHandle(ph) - child := exec.Command(os.Args[0], "-test.run=^TestChangingProcessParent$") + child := exec.Command(testenv.Executable(t), "-test.run=^TestChangingProcessParent$") child.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=child", "GO_WANT_HELPER_PROCESS_FILE="+childDumpPath) diff --git a/src/syscall/export_windows_test.go b/src/syscall/export_windows_test.go index eccf1bccace4d9..a816d7760b74a8 100644 --- a/src/syscall/export_windows_test.go +++ b/src/syscall/export_windows_test.go @@ -12,3 +12,5 @@ const PROC_THREAD_ATTRIBUTE_HANDLE_LIST = _PROC_THREAD_ATTRIBUTE_HANDLE_LIST var EncodeWTF16 = encodeWTF16 var DecodeWTF16 = decodeWTF16 + +var EnvSorted = envSorted diff --git a/src/syscall/fs_js.go b/src/syscall/fs_js.go index 793b9a2d41c38e..7ef3cdee14a8b6 100644 --- a/src/syscall/fs_js.go +++ b/src/syscall/fs_js.go @@ -16,6 +16,7 @@ import ( func now() (sec int64, nsec int32) var jsProcess = js.Global().Get("process") +var jsPath = js.Global().Get("path") var jsFS = js.Global().Get("fs") var constants = jsFS.Get("constants") @@ -28,8 +29,20 @@ var ( nodeTRUNC = constants.Get("O_TRUNC").Int() nodeAPPEND = constants.Get("O_APPEND").Int() nodeEXCL = constants.Get("O_EXCL").Int() + + // NodeJS on Windows does not support O_DIRECTORY, so we default + // to -1 and assign it in init if available. + // See https://nodejs.org/docs/latest/api/fs.html#file-open-constants. + nodeDIRECTORY = -1 ) +func init() { + oDir := constants.Get("O_DIRECTORY") + if !oDir.IsUndefined() { + nodeDIRECTORY = oDir.Int() + } +} + type jsFile struct { path string entries []string @@ -82,6 +95,13 @@ func Open(path string, openmode int, perm uint32) (int, error) { if openmode&O_SYNC != 0 { return 0, errors.New("syscall.Open: O_SYNC is not supported by js/wasm") } + if openmode&O_DIRECTORY != 0 { + if nodeDIRECTORY != -1 { + flags |= nodeDIRECTORY + } else { + return 0, errors.New("syscall.Open: O_DIRECTORY is not supported on Windows") + } + } jsFD, err := fsCall("open", path, flags, perm) if err != nil { @@ -101,10 +121,8 @@ func Open(path string, openmode int, perm uint32) (int, error) { } } - if path[0] != '/' { - cwd := jsProcess.Call("cwd").String() - path = cwd + "/" + path - } + path = jsPath.Call("resolve", path).String() + f := &jsFile{ path: path, entries: entries, diff --git a/src/syscall/fs_wasip1.go b/src/syscall/fs_wasip1.go index fc361ee898978f..0eff07e95a1858 100644 --- a/src/syscall/fs_wasip1.go +++ b/src/syscall/fs_wasip1.go @@ -9,6 +9,7 @@ package syscall import ( "internal/stringslite" "runtime" + "structs" "unsafe" ) @@ -40,6 +41,7 @@ type filedelta = int64 type fstflags = uint32 type iovec struct { + _ structs.HostLayout buf uintptr32 bufLen size } @@ -190,23 +192,23 @@ func fd_filestat_set_size(fd int32, set_size filesize) Errno // //go:wasmimport wasi_snapshot_preview1 fd_pread //go:noescape -func fd_pread(fd int32, iovs unsafe.Pointer, iovsLen size, offset filesize, nread unsafe.Pointer) Errno +func fd_pread(fd int32, iovs *iovec, iovsLen size, offset filesize, nread *size) Errno //go:wasmimport wasi_snapshot_preview1 fd_pwrite //go:noescape -func fd_pwrite(fd int32, iovs unsafe.Pointer, iovsLen size, offset filesize, nwritten unsafe.Pointer) Errno +func fd_pwrite(fd int32, iovs *iovec, iovsLen size, offset filesize, nwritten *size) Errno //go:wasmimport wasi_snapshot_preview1 fd_read //go:noescape -func fd_read(fd int32, iovs unsafe.Pointer, iovsLen size, nread unsafe.Pointer) Errno +func fd_read(fd int32, iovs *iovec, iovsLen size, nread *size) Errno //go:wasmimport wasi_snapshot_preview1 fd_readdir //go:noescape -func fd_readdir(fd int32, buf unsafe.Pointer, bufLen size, cookie dircookie, nwritten unsafe.Pointer) Errno +func fd_readdir(fd int32, buf *byte, bufLen size, cookie dircookie, nwritten *size) Errno //go:wasmimport wasi_snapshot_preview1 fd_seek //go:noescape -func fd_seek(fd int32, offset filedelta, whence uint32, newoffset unsafe.Pointer) Errno +func fd_seek(fd int32, offset filedelta, whence uint32, newoffset *filesize) Errno // https://github.com/WebAssembly/WASI/blob/a2b96e81c0586125cc4dc79a5be0b78d9a059925/legacy/preview1/docs.md#-fd_fdstat_set_rightsfd-fd-fs_rights_base-rights-fs_rights_inheriting-rights---result-errno // @@ -220,7 +222,7 @@ func fd_filestat_get(fd int32, buf unsafe.Pointer) Errno //go:wasmimport wasi_snapshot_preview1 fd_write //go:noescape -func fd_write(fd int32, iovs unsafe.Pointer, iovsLen size, nwritten unsafe.Pointer) Errno +func fd_write(fd int32, iovs *iovec, iovsLen size, nwritten *size) Errno //go:wasmimport wasi_snapshot_preview1 fd_sync //go:noescape @@ -228,52 +230,53 @@ func fd_sync(fd int32) Errno //go:wasmimport wasi_snapshot_preview1 path_create_directory //go:noescape -func path_create_directory(fd int32, path unsafe.Pointer, pathLen size) Errno +func path_create_directory(fd int32, path *byte, pathLen size) Errno //go:wasmimport wasi_snapshot_preview1 path_filestat_get //go:noescape -func path_filestat_get(fd int32, flags lookupflags, path unsafe.Pointer, pathLen size, buf unsafe.Pointer) Errno +func path_filestat_get(fd int32, flags lookupflags, path *byte, pathLen size, buf unsafe.Pointer) Errno //go:wasmimport wasi_snapshot_preview1 path_filestat_set_times //go:noescape -func path_filestat_set_times(fd int32, flags lookupflags, path unsafe.Pointer, pathLen size, atim timestamp, mtim timestamp, fstflags fstflags) Errno +func path_filestat_set_times(fd int32, flags lookupflags, path *byte, pathLen size, atim timestamp, mtim timestamp, fstflags fstflags) Errno //go:wasmimport wasi_snapshot_preview1 path_link //go:noescape -func path_link(oldFd int32, oldFlags lookupflags, oldPath unsafe.Pointer, oldPathLen size, newFd int32, newPath unsafe.Pointer, newPathLen size) Errno +func path_link(oldFd int32, oldFlags lookupflags, oldPath *byte, oldPathLen size, newFd int32, newPath *byte, newPathLen size) Errno //go:wasmimport wasi_snapshot_preview1 path_readlink //go:noescape -func path_readlink(fd int32, path unsafe.Pointer, pathLen size, buf unsafe.Pointer, bufLen size, nwritten unsafe.Pointer) Errno +func path_readlink(fd int32, path *byte, pathLen size, buf *byte, bufLen size, nwritten *size) Errno //go:wasmimport wasi_snapshot_preview1 path_remove_directory //go:noescape -func path_remove_directory(fd int32, path unsafe.Pointer, pathLen size) Errno +func path_remove_directory(fd int32, path *byte, pathLen size) Errno //go:wasmimport wasi_snapshot_preview1 path_rename //go:noescape -func path_rename(oldFd int32, oldPath unsafe.Pointer, oldPathLen size, newFd int32, newPath unsafe.Pointer, newPathLen size) Errno +func path_rename(oldFd int32, oldPath *byte, oldPathLen size, newFd int32, newPath *byte, newPathLen size) Errno //go:wasmimport wasi_snapshot_preview1 path_symlink //go:noescape -func path_symlink(oldPath unsafe.Pointer, oldPathLen size, fd int32, newPath unsafe.Pointer, newPathLen size) Errno +func path_symlink(oldPath *byte, oldPathLen size, fd int32, newPath *byte, newPathLen size) Errno //go:wasmimport wasi_snapshot_preview1 path_unlink_file //go:noescape -func path_unlink_file(fd int32, path unsafe.Pointer, pathLen size) Errno +func path_unlink_file(fd int32, path *byte, pathLen size) Errno //go:wasmimport wasi_snapshot_preview1 path_open //go:noescape -func path_open(rootFD int32, dirflags lookupflags, path unsafe.Pointer, pathLen size, oflags oflags, fsRightsBase rights, fsRightsInheriting rights, fsFlags fdflags, fd unsafe.Pointer) Errno +func path_open(rootFD int32, dirflags lookupflags, path *byte, pathLen size, oflags oflags, fsRightsBase rights, fsRightsInheriting rights, fsFlags fdflags, fd *int32) Errno //go:wasmimport wasi_snapshot_preview1 random_get //go:noescape -func random_get(buf unsafe.Pointer, bufLen size) Errno +func random_get(buf *byte, bufLen size) Errno // https://github.com/WebAssembly/WASI/blob/a2b96e81c0586125cc4dc79a5be0b78d9a059925/legacy/preview1/docs.md#-fdstat-record // fdflags must be at offset 2, hence the uint16 type rather than the // fdflags (uint32) type. type fdstat struct { + _ structs.HostLayout filetype filetype fdflags uint16 rightsBase rights @@ -282,7 +285,7 @@ type fdstat struct { //go:wasmimport wasi_snapshot_preview1 fd_fdstat_get //go:noescape -func fd_fdstat_get(fd int32, buf unsafe.Pointer) Errno +func fd_fdstat_get(fd int32, buf *fdstat) Errno //go:wasmimport wasi_snapshot_preview1 fd_fdstat_set_flags //go:noescape @@ -293,7 +296,7 @@ func fd_fdstat_set_flags(fd int32, flags fdflags) Errno func fd_fdstat_get_flags(fd int) (uint32, error) { var stat fdstat - errno := fd_fdstat_get(int32(fd), unsafe.Pointer(&stat)) + errno := fd_fdstat_get(int32(fd), &stat) return uint32(stat.fdflags), errnoErr(errno) } @@ -302,7 +305,7 @@ func fd_fdstat_get_flags(fd int) (uint32, error) { func fd_fdstat_get_type(fd int) (uint8, error) { var stat fdstat - errno := fd_fdstat_get(int32(fd), unsafe.Pointer(&stat)) + errno := fd_fdstat_get(int32(fd), &stat) return stat.filetype, errnoErr(errno) } @@ -313,21 +316,23 @@ const ( ) type prestatDir struct { + _ structs.HostLayout prNameLen size } type prestat struct { + _ structs.HostLayout typ preopentype dir prestatDir } //go:wasmimport wasi_snapshot_preview1 fd_prestat_get //go:noescape -func fd_prestat_get(fd int32, prestat unsafe.Pointer) Errno +func fd_prestat_get(fd int32, prestat *prestat) Errno //go:wasmimport wasi_snapshot_preview1 fd_prestat_dir_name //go:noescape -func fd_prestat_dir_name(fd int32, path unsafe.Pointer, pathLen size) Errno +func fd_prestat_dir_name(fd int32, path *byte, pathLen size) Errno type opendir struct { fd int32 @@ -354,7 +359,7 @@ func init() { for preopenFd := int32(3); ; preopenFd++ { var prestat prestat - errno := fd_prestat_get(preopenFd, unsafe.Pointer(&prestat)) + errno := fd_prestat_get(preopenFd, &prestat) if errno == EBADF { break } @@ -368,7 +373,7 @@ func init() { dirNameBuf = make([]byte, prestat.dir.prNameLen) } - errno = fd_prestat_dir_name(preopenFd, unsafe.Pointer(&dirNameBuf[0]), prestat.dir.prNameLen) + errno = fd_prestat_dir_name(preopenFd, &dirNameBuf[0], prestat.dir.prNameLen) if errno != 0 { panic("fd_prestat_dir_name: " + errno.Error()) } @@ -488,7 +493,7 @@ func isDir(path string) bool { // // If the path argument is not absolute, it is first appended to the current // working directory before resolution. -func preparePath(path string) (int32, unsafe.Pointer, size) { +func preparePath(path string) (int32, *byte, size) { var dirFd = int32(-1) var dirName string @@ -512,7 +517,7 @@ func preparePath(path string) (int32, unsafe.Pointer, size) { path = "." } - return dirFd, stringPointer(path), size(len(path)) + return dirFd, unsafe.StringData(path), size(len(path)) } func Open(path string, openmode int, perm uint32) (int, error) { @@ -520,7 +525,14 @@ func Open(path string, openmode int, perm uint32) (int, error) { return -1, EINVAL } dirFd, pathPtr, pathLen := preparePath(path) + return openat(dirFd, pathPtr, pathLen, openmode, perm) +} + +func Openat(dirFd int, path string, openmode int, perm uint32) (int, error) { + return openat(int32(dirFd), unsafe.StringData(path), size(len(path)), openmode, perm) +} +func openat(dirFd int32, pathPtr *byte, pathLen size, openmode int, perm uint32) (int, error) { var oflags oflags if (openmode & O_CREATE) != 0 { oflags |= OFLAG_CREATE @@ -542,6 +554,14 @@ func Open(path string, openmode int, perm uint32) (int, error) { rights = fileRights } + if (openmode & O_DIRECTORY) != 0 { + if openmode&(O_WRONLY|O_RDWR) != 0 { + return -1, EISDIR + } + oflags |= OFLAG_DIRECTORY + rights &= dirRights + } + var fdflags fdflags if (openmode & O_APPEND) != 0 { fdflags |= FDFLAG_APPEND @@ -550,17 +570,22 @@ func Open(path string, openmode int, perm uint32) (int, error) { fdflags |= FDFLAG_SYNC } + var lflags lookupflags + if openmode&O_NOFOLLOW == 0 { + lflags = LOOKUP_SYMLINK_FOLLOW + } + var fd int32 errno := path_open( dirFd, - LOOKUP_SYMLINK_FOLLOW, + lflags, pathPtr, pathLen, oflags, rights, fileRights, fdflags, - unsafe.Pointer(&fd), + &fd, ) if errno == EISDIR && oflags == 0 && fdflags == 0 && ((rights & writeRights) == 0) { // wasmtime and wasmedge will error if attempting to open a directory @@ -581,7 +606,7 @@ func Open(path string, openmode int, perm uint32) (int, error) { rights&dirRights, fileRights, fdflags, - unsafe.Pointer(&fd), + &fd, ) } return int(fd), errnoErr(errno) @@ -607,7 +632,7 @@ func Mkdir(path string, perm uint32) error { func ReadDir(fd int, buf []byte, cookie dircookie) (int, error) { var nwritten size - errno := fd_readdir(int32(fd), unsafe.Pointer(&buf[0]), size(len(buf)), cookie, unsafe.Pointer(&nwritten)) + errno := fd_readdir(int32(fd), &buf[0], size(len(buf)), cookie, &nwritten) return int(nwritten), errnoErr(errno) } @@ -815,9 +840,9 @@ func Readlink(path string, buf []byte) (n int, err error) { dirFd, pathPtr, pathLen, - unsafe.Pointer(&buf[0]), + &buf[0], size(len(buf)), - unsafe.Pointer(&nwritten), + &nwritten, ) // For some reason wasmtime returns ERANGE when the output buffer is // shorter than the symbolic link value. os.Readlink expects a nil @@ -851,7 +876,7 @@ func Symlink(path, link string) error { } dirFd, pathPtr, pathlen := preparePath(link) errno := path_symlink( - stringPointer(path), + unsafe.StringData(path), size(len(path)), dirFd, pathPtr, @@ -865,52 +890,44 @@ func Fsync(fd int) error { return errnoErr(errno) } -func bytesPointer(b []byte) unsafe.Pointer { - return unsafe.Pointer(unsafe.SliceData(b)) -} - -func stringPointer(s string) unsafe.Pointer { - return unsafe.Pointer(unsafe.StringData(s)) -} - -func makeIOVec(b []byte) unsafe.Pointer { - return unsafe.Pointer(&iovec{ - buf: uintptr32(uintptr(bytesPointer(b))), +func makeIOVec(b []byte) *iovec { + return &iovec{ + buf: uintptr32(uintptr(unsafe.Pointer(unsafe.SliceData(b)))), bufLen: size(len(b)), - }) + } } func Read(fd int, b []byte) (int, error) { var nread size - errno := fd_read(int32(fd), makeIOVec(b), 1, unsafe.Pointer(&nread)) + errno := fd_read(int32(fd), makeIOVec(b), 1, &nread) runtime.KeepAlive(b) return int(nread), errnoErr(errno) } func Write(fd int, b []byte) (int, error) { var nwritten size - errno := fd_write(int32(fd), makeIOVec(b), 1, unsafe.Pointer(&nwritten)) + errno := fd_write(int32(fd), makeIOVec(b), 1, &nwritten) runtime.KeepAlive(b) return int(nwritten), errnoErr(errno) } func Pread(fd int, b []byte, offset int64) (int, error) { var nread size - errno := fd_pread(int32(fd), makeIOVec(b), 1, filesize(offset), unsafe.Pointer(&nread)) + errno := fd_pread(int32(fd), makeIOVec(b), 1, filesize(offset), &nread) runtime.KeepAlive(b) return int(nread), errnoErr(errno) } func Pwrite(fd int, b []byte, offset int64) (int, error) { var nwritten size - errno := fd_pwrite(int32(fd), makeIOVec(b), 1, filesize(offset), unsafe.Pointer(&nwritten)) + errno := fd_pwrite(int32(fd), makeIOVec(b), 1, filesize(offset), &nwritten) runtime.KeepAlive(b) return int(nwritten), errnoErr(errno) } func Seek(fd int, offset int64, whence int) (int64, error) { var newoffset filesize - errno := fd_seek(int32(fd), filedelta(offset), uint32(whence), unsafe.Pointer(&newoffset)) + errno := fd_seek(int32(fd), filedelta(offset), uint32(whence), &newoffset) return int64(newoffset), errnoErr(errno) } @@ -927,6 +944,6 @@ func Pipe(fd []int) error { } func RandomGet(b []byte) error { - errno := random_get(bytesPointer(b), size(len(b))) + errno := random_get(unsafe.SliceData(b), size(len(b))) return errnoErr(errno) } diff --git a/src/syscall/getdirentries_test.go b/src/syscall/getdirentries_test.go index 5d401d8dd6fa2c..f879e2a5390c61 100644 --- a/src/syscall/getdirentries_test.go +++ b/src/syscall/getdirentries_test.go @@ -8,10 +8,10 @@ package syscall_test import ( "fmt" + "internal/testenv" "os" "path/filepath" "slices" - "strings" "syscall" "testing" "unsafe" @@ -25,7 +25,7 @@ func TestGetdirentries(t *testing.T) { } } func testGetdirentries(t *testing.T, count int) { - if count > 100 && testing.Short() && os.Getenv("GO_BUILDER_NAME") == "" { + if count > 100 && testing.Short() && testenv.Builder() == "" { t.Skip("skipping in -short mode") } d := t.TempDir() @@ -78,7 +78,7 @@ func testGetdirentries(t *testing.T, count int) { names = append(names, ".", "..") // Getdirentries returns these also slices.Sort(names) slices.Sort(names2) - if strings.Join(names, ":") != strings.Join(names2, ":") { + if !slices.Equal(names, names2) { t.Errorf("names don't match\n names: %q\nnames2: %q", names, names2) } } diff --git a/src/syscall/js/func.go b/src/syscall/js/func.go index 53a4d79a95e322..23530170ffcd4c 100644 --- a/src/syscall/js/func.go +++ b/src/syscall/js/func.go @@ -6,7 +6,10 @@ package js -import "sync" +import ( + "internal/synctest" + "sync" +) var ( funcsMu sync.Mutex @@ -16,8 +19,9 @@ var ( // Func is a wrapped Go function to be called by JavaScript. type Func struct { - Value // the JavaScript function that invokes the Go function - id uint32 + Value // the JavaScript function that invokes the Go function + bubble *synctest.Bubble + id uint32 } // FuncOf returns a function to be used by JavaScript. @@ -42,11 +46,23 @@ func FuncOf(fn func(this Value, args []Value) any) Func { funcsMu.Lock() id := nextFuncID nextFuncID++ + bubble := synctest.Acquire() + if bubble != nil { + origFn := fn + fn = func(this Value, args []Value) any { + var r any + bubble.Run(func() { + r = origFn(this, args) + }) + return r + } + } funcs[id] = fn funcsMu.Unlock() return Func{ - id: id, - Value: jsGo.Call("_makeFuncWrapper", id), + id: id, + bubble: bubble, + Value: jsGo.Call("_makeFuncWrapper", id), } } @@ -54,6 +70,7 @@ func FuncOf(fn func(this Value, args []Value) any) Func { // The function must not be invoked after calling Release. // It is allowed to call Release while the function is still running. func (c Func) Release() { + c.bubble.Release() funcsMu.Lock() delete(funcs, c.id) funcsMu.Unlock() diff --git a/src/syscall/js/js.go b/src/syscall/js/js.go index 74c02cdbe64ada..bbf3de199b4b82 100644 --- a/src/syscall/js/js.go +++ b/src/syscall/js/js.go @@ -212,8 +212,8 @@ func ValueOf(x any) Value { // stringVal copies string x to Javascript and returns a ref. // -// (noescape): This is safe because no references are maintained to the -// Go string x after the syscall returns. +// Using go:noescape is safe because no references are maintained to the +// Go string x after the syscall returns. // //go:wasmimport gojs syscall/js.stringVal //go:noescape @@ -302,8 +302,8 @@ func (v Value) Get(p string) Value { // valueGet returns a ref to JavaScript property p of ref v. // -// (noescape): This is safe because no references are maintained to the -// Go string p after the syscall returns. +// Using go:noescape is safe because no references are maintained to the +// Go string p after the syscall returns. // //go:wasmimport gojs syscall/js.valueGet //go:noescape @@ -323,8 +323,8 @@ func (v Value) Set(p string, x any) { // valueSet sets property p of ref v to ref x. // -// (noescape): This is safe because no references are maintained to the -// Go string p after the syscall returns. +// Using go:noescape is safe because no references are maintained to the +// Go string p after the syscall returns. // //go:wasmimport gojs syscall/js.valueSet //go:noescape @@ -342,8 +342,8 @@ func (v Value) Delete(p string) { // valueDelete deletes the JavaScript property p of ref v. // -// (noescape): This is safe because no references are maintained to the -// Go string p after the syscall returns. +// Using go:noescape is safe because no references are maintained to the +// Go string p after the syscall returns. // //go:wasmimport gojs syscall/js.valueDelete //go:noescape @@ -447,10 +447,10 @@ func (v Value) Call(m string, args ...any) Value { // valueCall does a JavaScript call to the method name m of ref v with the given arguments. // -// (noescape): This is safe because no references are maintained to the -// Go string m after the syscall returns. Additionally, the args slice -// is only used temporarily to collect the JavaScript objects for -// the JavaScript method invocation. +// Using go:noescape is safe because no references are maintained to the +// Go string m after the syscall returns. Additionally, the args slice +// is only used temporarily to collect the JavaScript objects for +// the JavaScript method invocation. // //go:wasmimport gojs syscall/js.valueCall //go:nosplit @@ -477,9 +477,9 @@ func (v Value) Invoke(args ...any) Value { // valueInvoke does a JavaScript call to value v with the given arguments. // -// (noescape): This is safe because the args slice is only used temporarily -// to collect the JavaScript objects for the JavaScript method -// invocation. +// Using go:noescape is safe because the args slice is only used temporarily +// to collect the JavaScript objects for the JavaScript method +// invocation. // //go:wasmimport gojs syscall/js.valueInvoke //go:noescape @@ -505,8 +505,8 @@ func (v Value) New(args ...any) Value { // valueNew uses JavaScript's "new" operator with value v as a constructor and the given arguments. // -// (noescape): This is safe because the args slice is only used temporarily -// to collect the JavaScript objects for the constructor execution. +// Using go:noescape is safe because the args slice is only used temporarily +// to collect the JavaScript objects for the constructor execution. // //go:wasmimport gojs syscall/js.valueNew //go:noescape @@ -614,8 +614,8 @@ func valuePrepareString(v ref) (ref, int) // valueLoadString loads string data located at ref v into byte slice b. // -// (noescape): This is safe because the byte slice is only used as a destination -// for storing the string data and references to it are not maintained. +// Using go:noescape is safe because the byte slice is only used as a destination +// for storing the string data and references to it are not maintained. // //go:wasmimport gojs syscall/js.valueLoadString //go:noescape @@ -658,8 +658,8 @@ func CopyBytesToGo(dst []byte, src Value) int { // copyBytesToGo copies bytes from src to dst. // -// (noescape): This is safe because the dst byte slice is only used as a dst -// copy buffer and no references to it are maintained. +// Using go:noescape is safe because the dst byte slice is only used as a dst +// copy buffer and no references to it are maintained. // //go:wasmimport gojs syscall/js.copyBytesToGo //go:noescape @@ -677,10 +677,10 @@ func CopyBytesToJS(dst Value, src []byte) int { return n } -// copyBytesToJs copies bytes from src to dst. +// copyBytesToJS copies bytes from src to dst. // -// (noescape): This is safe because the src byte slice is only used as a src -// copy buffer and no references to it are maintained. +// Using go:noescape is safe because the src byte slice is only used as a src +// copy buffer and no references to it are maintained. // //go:wasmimport gojs syscall/js.copyBytesToJS //go:noescape diff --git a/src/syscall/js/js_test.go b/src/syscall/js/js_test.go index cc809ac107e16c..9ab913f2abf714 100644 --- a/src/syscall/js/js_test.go +++ b/src/syscall/js/js_test.go @@ -7,7 +7,7 @@ // To run these tests: // // - Install Node -// - Add /path/to/go/misc/wasm to your $PATH (so that "go test" can find +// - Add /path/to/go/lib/wasm to your $PATH (so that "go test" can find // "go_js_wasm_exec"). // - GOOS=js GOARCH=wasm go test // @@ -56,6 +56,51 @@ func TestWasmImport(t *testing.T) { } } +// testCallExport is imported from host (wasm_exec.js), which calls testExport. +// +//go:wasmimport _gotest callExport +func testCallExport(a int32, b int64) int64 + +//go:wasmexport testExport +func testExport(a int32, b int64) int64 { + testExportCalled = true + // test stack growth + growStack(1000) + // force a goroutine switch + ch := make(chan int64) + go func() { + ch <- int64(a) + ch <- b + }() + return <-ch + <-ch +} + +//go:wasmexport testExport0 +func testExport0() { // no arg or result (see issue 69584) + runtime.GC() +} + +var testExportCalled bool + +func growStack(n int64) { + if n > 0 { + growStack(n - 1) + } +} + +func TestWasmExport(t *testing.T) { + testExportCalled = false + a := int32(123) + b := int64(456) + want := int64(a) + b + if got := testCallExport(a, b); got != want { + t.Errorf("got %v, want %v", got, want) + } + if !testExportCalled { + t.Error("testExport not called") + } +} + func TestBool(t *testing.T) { want := true o := dummys.Get("someBool") @@ -587,16 +632,16 @@ func TestGarbageCollection(t *testing.T) { // Note: All JavaScript functions return a JavaScript array, which will cause // one allocation to be created to track the Value.gcPtr for the Value finalizer. var allocTests = []struct { - argLen int // The number of arguments to use for the syscall + argLen int // The number of arguments to use for the syscall expected int // The expected number of allocations }{ - // For less than or equal to 16 arguments, we expect 1 alloction: + // For less than or equal to 16 arguments, we expect 1 allocation: // - makeValue new(ref) - {0, 1}, - {2, 1}, + {0, 1}, + {2, 1}, {15, 1}, {16, 1}, - // For greater than 16 arguments, we expect 3 alloction: + // For greater than 16 arguments, we expect 3 allocation: // - makeValue: new(ref) // - makeArgSlices: argVals = make([]Value, size) // - makeArgSlices: argRefs = make([]ref, size) @@ -613,7 +658,7 @@ func TestCallAllocations(t *testing.T) { tmpArray := js.Global().Get("Array").New(0) numAllocs := testing.AllocsPerRun(100, func() { tmpArray.Call("concat", args...) - }); + }) if numAllocs != float64(test.expected) { t.Errorf("got numAllocs %#v, want %#v", numAllocs, test.expected) @@ -630,7 +675,7 @@ func TestInvokeAllocations(t *testing.T) { concatFunc := tmpArray.Get("concat").Call("bind", tmpArray) numAllocs := testing.AllocsPerRun(100, func() { concatFunc.Invoke(args...) - }); + }) if numAllocs != float64(test.expected) { t.Errorf("got numAllocs %#v, want %#v", numAllocs, test.expected) @@ -647,7 +692,7 @@ func TestNewAllocations(t *testing.T) { numAllocs := testing.AllocsPerRun(100, func() { arrayConstructor.New(args...) - }); + }) if numAllocs != float64(test.expected) { t.Errorf("got numAllocs %#v, want %#v", numAllocs, test.expected) diff --git a/src/syscall/linkname_openbsd.go b/src/syscall/linkname_openbsd.go index 5f5c517ab58599..bbb56c10eca378 100644 --- a/src/syscall/linkname_openbsd.go +++ b/src/syscall/linkname_openbsd.go @@ -12,4 +12,3 @@ import _ "unsafe" //go:linkname unlinkat //go:linkname openat //go:linkname fstatat -//go:linkname getentropy diff --git a/src/syscall/mkall.sh b/src/syscall/mkall.sh index a3bc767628f209..b9a0ed3d4f57c3 100755 --- a/src/syscall/mkall.sh +++ b/src/syscall/mkall.sh @@ -190,7 +190,7 @@ linux_amd64) mktypes="GOARCH=$GOARCH go tool cgo -godefs" ;; linux_arm) - GOOSARCH_in="syscall_linux_arm.go syscall_linux_accept.go" + GOOSARCH_in="syscall_linux_arm.go" mkerrors="$mkerrors" mksyscall="./mksyscall.pl -l32 -arm" mksysnum="curl -s 'http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/plain/arch/arm/include/uapi/asm/unistd.h' | ./mksysnum_linux.pl -" diff --git a/src/syscall/mksyscall.pl b/src/syscall/mksyscall.pl index b46a3f9438bd87..6be94c9c44f065 100755 --- a/src/syscall/mksyscall.pl +++ b/src/syscall/mksyscall.pl @@ -141,12 +141,6 @@ ($) # without reading the header. $text .= "// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\n"; - if ((($darwin || ($openbsd && $libc)) && $func =~ /^ptrace(Ptr)?$/)) { - # The ptrace function is called from forkAndExecInChild where stack - # growth is forbidden. - $text .= "//go:nosplit\n" - } - # Go function header. my $out_decl = @out ? sprintf(" (%s)", join(', ', @out)) : ""; $text .= sprintf "func %s(%s)%s {\n", $func, join(', ', @in), $out_decl; diff --git a/src/syscall/net_wasip1.go b/src/syscall/net_wasip1.go index 3918840a7e05e1..fefd939a0a973a 100644 --- a/src/syscall/net_wasip1.go +++ b/src/syscall/net_wasip1.go @@ -6,8 +6,6 @@ package syscall -import "unsafe" - const ( SHUT_RD = 0x1 SHUT_WR = 0x2 @@ -18,7 +16,7 @@ type sdflags = uint32 //go:wasmimport wasi_snapshot_preview1 sock_accept //go:noescape -func sock_accept(fd int32, flags fdflags, newfd unsafe.Pointer) Errno +func sock_accept(fd int32, flags fdflags, newfd *int32) Errno //go:wasmimport wasi_snapshot_preview1 sock_shutdown //go:noescape @@ -42,7 +40,7 @@ func Listen(fd int, backlog int) error { func Accept(fd int) (int, Sockaddr, error) { var newfd int32 - errno := sock_accept(int32(fd), 0, unsafe.Pointer(&newfd)) + errno := sock_accept(int32(fd), 0, &newfd) return int(newfd), nil, errnoErr(errno) } diff --git a/src/syscall/rlimit.go b/src/syscall/rlimit.go index 8184f17ab68f29..3812303febb791 100644 --- a/src/syscall/rlimit.go +++ b/src/syscall/rlimit.go @@ -29,10 +29,18 @@ var origRlimitNofile atomic.Pointer[Rlimit] // which Go of course has no choice but to respect. func init() { var lim Rlimit - if err := Getrlimit(RLIMIT_NOFILE, &lim); err == nil && lim.Cur != lim.Max { + if err := Getrlimit(RLIMIT_NOFILE, &lim); err == nil && lim.Max > 0 && lim.Cur < lim.Max-1 { origRlimitNofile.Store(&lim) nlim := lim - nlim.Cur = nlim.Max + + // We set Cur to Max - 1 so that we are more likely to + // detect cases where another process uses prlimit + // to change our resource limits. The theory is that + // using prlimit to change to Cur == Max is more likely + // than using prlimit to change to Cur == Max - 1. + // The place we check for this is in exec_linux.go. + nlim.Cur = nlim.Max - 1 + adjustFileLimit(&nlim) setrlimit(RLIMIT_NOFILE, &nlim) } diff --git a/src/syscall/syscall.go b/src/syscall/syscall.go index a46f22ddb53c6a..d70f4fbd15a53d 100644 --- a/src/syscall/syscall.go +++ b/src/syscall/syscall.go @@ -104,3 +104,7 @@ func Exit(code int) // runtimeSetenv and runtimeUnsetenv are provided by the runtime. func runtimeSetenv(k, v string) func runtimeUnsetenv(k string) + +// runtimeClearenv is provided by the runtime (on platforms without +// clearenv(3), it is just a wrapper around runtimeUnsetenv). +func runtimeClearenv(env map[string]int) diff --git a/src/syscall/syscall_aix.go b/src/syscall/syscall_aix.go index a9bd7a37336dde..36dfd90678804f 100644 --- a/src/syscall/syscall_aix.go +++ b/src/syscall/syscall_aix.go @@ -119,11 +119,11 @@ func Getwd() (ret string, err error) { b := make([]byte, len) err := getcwd(&b[0], len) if err == nil { - i := 0 - for b[i] != 0 { - i++ + n := clen(b[:]) + if n < 1 { + return "", EINVAL } - return string(b[0:i]), nil + return string(b[:n]), nil } if err != ERANGE { return "", err diff --git a/src/syscall/syscall_darwin.go b/src/syscall/syscall_darwin.go index 5b38aeae31a6fc..ca76cc2962b098 100644 --- a/src/syscall/syscall_darwin.go +++ b/src/syscall/syscall_darwin.go @@ -228,7 +228,7 @@ func Kill(pid int, signum Signal) (err error) { return kill(pid, int(signum), 1) //sys getcwd(buf []byte) (n int, err error) func init() { - execveDarwin = execve + execveLibc = execve } func fdopendir(fd int) (dir uintptr, err error) { @@ -323,10 +323,126 @@ func Getdirentries(fd int, buf []byte, basep *uintptr) (n int, err error) { return n, nil } +// errno return e if int32(r) is -1, else it returns 0. +// +//go:nosplit +func errno(r uintptr, e Errno) Errno { + if int32(r) == -1 { + return e + } + return 0 +} + +// errnoX return e if r is -1, else it returns 0. +// +//go:nosplit +func errnoX(r uintptr, e Errno) Errno { + if r == ^uintptr(0) { + return e + } + return 0 +} + +// errnoPtr return e if r is 0, else it returns 0. +// +//go:nosplit +func errnoPtr(r uintptr, e Errno) Errno { + if r == 0 { + return e + } + return 0 +} + +//go:cgo_import_dynamic libc_error __error "/usr/lib/libSystem.B.dylib" + +// golang.org/x/sys linknames the following syscalls. +// Do not remove or change the type signature. + +//go:linkname syscall +//go:nosplit +//go:uintptrkeepalive +func syscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = syscalln(fn, a1, a2, a3) + return r1, r2, errno(r1, err) +} + +//go:linkname syscallX +//go:nosplit +//go:uintptrkeepalive +func syscallX(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = syscalln(fn, a1, a2, a3) + return r1, r2, errnoX(r1, err) +} + +// syscall.syscall6 is meant for package syscall (and x/sys), +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/tetratelabs/wazero +// +// See go.dev/issue/67401. +// +//go:linkname syscall6 +//go:nosplit +//go:uintptrkeepalive +func syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = syscalln(fn, a1, a2, a3, a4, a5, a6) + return r1, r2, errno(r1, err) +} + +//go:linkname syscall6X +//go:nosplit +//go:uintptrkeepalive +func syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = syscalln(fn, a1, a2, a3, a4, a5, a6) + return r1, r2, errnoX(r1, err) +} + +// syscall9 is used in [internal/syscall/unix]. +// +//go:linkname syscall9 +//go:nosplit +//go:uintptrkeepalive +func syscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = syscalln(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9) + return r1, r2, errno(r1, err) +} + +//go:linkname rawSyscall +//go:nosplit +//go:uintptrkeepalive +func rawSyscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = rawsyscalln(fn, a1, a2, a3) + return r1, r2, errno(r1, err) +} + +//go:linkname rawSyscall6 +//go:nosplit +//go:uintptrkeepalive +func rawSyscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = rawsyscalln(fn, a1, a2, a3, a4, a5, a6) + return r1, r2, errno(r1, err) +} + +//go:linkname rawSyscall9 +//go:nosplit +//go:uintptrkeepalive +func rawSyscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = rawsyscalln(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9) + return r1, r2, errno(r1, err) +} + +//go:linkname syscallPtr +//go:nosplit +//go:uintptrkeepalive +func syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, e1 := syscalln(fn, a1, a2, a3) + return r1, r2, errnoPtr(r1, e1) +} + // Implemented in the runtime package (runtime/sys_darwin.go) -func syscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) -func syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) -func syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) -func rawSyscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) -func rawSyscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) -func syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) + +//go:noescape +func syscalln(fn uintptr, args ...uintptr) (r1, r2 uintptr, err Errno) + +//go:noescape +func rawsyscalln(fn uintptr, args ...uintptr) (r1, r2 uintptr, err Errno) diff --git a/src/syscall/syscall_darwin_amd64.go b/src/syscall/syscall_darwin_amd64.go index 64e54ad73081af..d4de4c25a43aa8 100644 --- a/src/syscall/syscall_darwin_amd64.go +++ b/src/syscall/syscall_darwin_amd64.go @@ -61,7 +61,4 @@ func libc_sendfile_trampoline() //go:cgo_import_dynamic libc_sendfile sendfile "/usr/lib/libSystem.B.dylib" -// Implemented in the runtime package (runtime/sys_darwin_64.go) -func syscallX(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) - func Syscall9(trap, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err Errno) diff --git a/src/syscall/syscall_darwin_arm64.go b/src/syscall/syscall_darwin_arm64.go index 913c7483746864..4a3dce8088245a 100644 --- a/src/syscall/syscall_darwin_arm64.go +++ b/src/syscall/syscall_darwin_arm64.go @@ -61,7 +61,4 @@ func libc_sendfile_trampoline() //go:cgo_import_dynamic libc_sendfile sendfile "/usr/lib/libSystem.B.dylib" -// Implemented in the runtime package (runtime/sys_darwin_64.go) -func syscallX(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) - func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err Errno) // sic diff --git a/src/syscall/syscall_freebsd_386.go b/src/syscall/syscall_freebsd_386.go index 60359e38f6b50d..a217dc758b904f 100644 --- a/src/syscall/syscall_freebsd_386.go +++ b/src/syscall/syscall_freebsd_386.go @@ -36,7 +36,13 @@ func sendfile(outfd int, infd int, offset *int64, count int) (written int, err e var writtenOut uint64 = 0 _, _, e1 := Syscall9(SYS_SENDFILE, uintptr(infd), uintptr(outfd), uintptr(*offset), uintptr((*offset)>>32), uintptr(count), 0, uintptr(unsafe.Pointer(&writtenOut)), 0, 0) - written = int(writtenOut) + // For some reason on the freebsd-386 builder writtenOut + // is modified when the system call returns EINVAL. + // The man page says that the value is only written for + // success, EINTR, or EAGAIN, so only use those cases. + if e1 == 0 || e1 == EINTR || e1 == EAGAIN { + written = int(writtenOut) + } if e1 != 0 { err = e1 diff --git a/src/syscall/syscall_js.go b/src/syscall/syscall_js.go index 0e529e0343738a..c320e34f2601cb 100644 --- a/src/syscall/syscall_js.go +++ b/src/syscall/syscall_js.go @@ -128,12 +128,13 @@ const ( O_WRONLY = 1 O_RDWR = 2 - O_CREAT = 0100 - O_CREATE = O_CREAT - O_TRUNC = 01000 - O_APPEND = 02000 - O_EXCL = 0200 - O_SYNC = 010000 + O_CREAT = 0100 + O_CREATE = O_CREAT + O_TRUNC = 01000 + O_APPEND = 02000 + O_EXCL = 0200 + O_SYNC = 010000 + O_DIRECTORY = 020000 O_CLOEXEC = 0 ) diff --git a/src/syscall/syscall_linux.go b/src/syscall/syscall_linux.go index 2706973596657b..ec9f771daad93e 100644 --- a/src/syscall/syscall_linux.go +++ b/src/syscall/syscall_linux.go @@ -13,8 +13,9 @@ package syscall import ( "internal/itoa" - runtimesyscall "internal/runtime/syscall" + "internal/runtime/syscall/linux" "runtime" + "slices" "unsafe" ) @@ -61,7 +62,7 @@ func RawSyscall(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) { //go:linkname RawSyscall6 func RawSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) { var errno uintptr - r1, r2, errno = runtimesyscall.Syscall6(trap, a1, a2, a3, a4, a5, a6) + r1, r2, errno = linux.Syscall6(trap, a1, a2, a3, a4, a5, a6) err = Errno(errno) return } @@ -134,12 +135,7 @@ func isGroupMember(gid int) bool { return false } - for _, g := range groups { - if g == gid { - return true - } - } - return false + return slices.Contains(groups, gid) } func isCapDacOverrideSet() bool { @@ -554,23 +550,29 @@ func (sa *SockaddrUnix) sockaddr() (unsafe.Pointer, _Socklen, error) { if n > len(sa.raw.Path) { return nil, 0, EINVAL } - if n == len(sa.raw.Path) && name[0] != '@' { + // Abstract addresses start with NUL. + // '@' is also a valid way to specify abstract addresses. + isAbstract := n > 0 && (name[0] == '@' || name[0] == '\x00') + + // Non-abstract named addresses are NUL terminated. + // The length can't use the full capacity as we need to add NUL. + if n == len(sa.raw.Path) && !isAbstract { return nil, 0, EINVAL } sa.raw.Family = AF_UNIX for i := 0; i < n; i++ { sa.raw.Path[i] = int8(name[i]) } - // length is family (uint16), name, NUL. - sl := _Socklen(2) - if n > 0 { - sl += _Socklen(n) + 1 - } - if sa.raw.Path[0] == '@' || (sa.raw.Path[0] == 0 && sl > 3) { - // Check sl > 3 so we don't change unnamed socket behavior. + // Length is family + name (+ NUL if non-abstract). + // Family is of type uint16 (2 bytes). + sl := _Socklen(2 + n) + if isAbstract { + // Abstract addresses are not NUL terminated. + // We rewrite '@' prefix to NUL here. sa.raw.Path[0] = 0 - // Don't count trailing NUL for abstract address. - sl-- + } else if n > 0 { + // Add NUL for non-abstract named addresses. + sl++ } return unsafe.Pointer(&sa.raw), sl, nil @@ -682,6 +684,10 @@ func anyToSockaddr(rsa *RawSockaddrAny) (Sockaddr, error) { return nil, EAFNOSUPPORT } +func Accept(fd int) (nfd int, sa Sockaddr, err error) { + return Accept4(fd, 0) +} + func Accept4(fd int, flags int) (nfd int, sa Sockaddr, err error) { var rsa RawSockaddrAny var len _Socklen = SizeofSockaddrAny @@ -1284,6 +1290,17 @@ func Munmap(b []byte) (err error) { //sys Mlockall(flags int) (err error) //sys Munlockall() (err error) +func Getrlimit(resource int, rlim *Rlimit) (err error) { + // prlimit1 is the same as prlimit when newlimit == nil + return prlimit1(0, resource, nil, rlim) +} + +// setrlimit sets a resource limit. +// The Setrlimit function is in rlimit.go, and calls this one. +func setrlimit(resource int, rlim *Rlimit) (err error) { + return prlimit1(0, resource, rlim, nil) +} + // prlimit changes a resource limit. We use a single definition so that // we can tell StartProcess to not restore the original NOFILE limit. // diff --git a/src/syscall/syscall_linux_386.go b/src/syscall/syscall_linux_386.go index a559f7e288ac5c..a90565a001ad77 100644 --- a/src/syscall/syscall_linux_386.go +++ b/src/syscall/syscall_linux_386.go @@ -72,96 +72,6 @@ func mmap(addr uintptr, length uintptr, prot int, flags int, fd int, offset int6 return mmap2(addr, length, prot, flags, fd, page) } -type rlimit32 struct { - Cur uint32 - Max uint32 -} - -//sysnb getrlimit(resource int, rlim *rlimit32) (err error) = SYS_GETRLIMIT - -const rlimInf32 = ^uint32(0) -const rlimInf64 = ^uint64(0) - -func Getrlimit(resource int, rlim *Rlimit) (err error) { - err = prlimit(0, resource, nil, rlim) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - err = getrlimit(resource, &rl) - if err != nil { - return - } - - if rl.Cur == rlimInf32 { - rlim.Cur = rlimInf64 - } else { - rlim.Cur = uint64(rl.Cur) - } - - if rl.Max == rlimInf32 { - rlim.Max = rlimInf64 - } else { - rlim.Max = uint64(rl.Max) - } - return -} - -//sysnb setrlimit1(resource int, rlim *rlimit32) (err error) = SYS_SETRLIMIT - -func setrlimit(resource int, rlim *Rlimit) (err error) { - err = prlimit(0, resource, rlim, nil) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - return setrlimit1(resource, &rl) -} - -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall6(SYS_PRLIMIT64, 0, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0, 0, 0) - if errno != ENOSYS { - return errno - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - _, _, errno = RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - // Underlying system call writes to newoffset via pointer. // Implemented in assembly to avoid allocation. func seek(fd int, offset int64, whence int) (newoffset int64, err Errno) diff --git a/src/syscall/syscall_linux_accept.go b/src/syscall/syscall_linux_accept.go deleted file mode 100644 index 66c0f84cb8e5d1..00000000000000 --- a/src/syscall/syscall_linux_accept.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// We require Linux kernel version 2.6.32. The accept4 system call was -// added in version 2.6.28, so in general we can use accept4. -// Unfortunately, for ARM only, accept4 was added in version 2.6.36. -// Handle that case here, by using a copy of the Accept function that -// we used in Go 1.17. - -//go:build linux && arm - -package syscall - -//sys accept(s int, rsa *RawSockaddrAny, addrlen *_Socklen) (fd int, err error) - -func Accept(fd int) (nfd int, sa Sockaddr, err error) { - var rsa RawSockaddrAny - var len _Socklen = SizeofSockaddrAny - // Try accept4 first for Android and newer kernels. - nfd, err = accept4(fd, &rsa, &len, 0) - if err == ENOSYS { - nfd, err = accept(fd, &rsa, &len) - } - if err != nil { - return - } - sa, err = anyToSockaddr(&rsa) - if err != nil { - Close(nfd) - nfd = 0 - } - return -} diff --git a/src/syscall/syscall_linux_accept4.go b/src/syscall/syscall_linux_accept4.go deleted file mode 100644 index 74898672c0466a..00000000000000 --- a/src/syscall/syscall_linux_accept4.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file provides the Accept function used on all systems -// other than arm. See syscall_linux_accept.go for why. - -//go:build linux && !arm - -package syscall - -func Accept(fd int) (nfd int, sa Sockaddr, err error) { - var rsa RawSockaddrAny - var len _Socklen = SizeofSockaddrAny - nfd, err = accept4(fd, &rsa, &len, 0) - if err != nil { - return - } - sa, err = anyToSockaddr(&rsa) - if err != nil { - Close(nfd) - nfd = 0 - } - return -} diff --git a/src/syscall/syscall_linux_amd64.go b/src/syscall/syscall_linux_amd64.go index ec52f8a4bd9ec4..725ebdba22f95a 100644 --- a/src/syscall/syscall_linux_amd64.go +++ b/src/syscall/syscall_linux_amd64.go @@ -4,10 +4,6 @@ package syscall -import ( - "unsafe" -) - const ( _SYS_setgroups = SYS_SETGROUPS _SYS_clone3 = 435 @@ -23,7 +19,6 @@ const ( //sysnb Getegid() (egid int) //sysnb Geteuid() (euid int) //sysnb Getgid() (gid int) -//sysnb Getrlimit(resource int, rlim *Rlimit) (err error) //sysnb Getuid() (uid int) //sysnb InotifyInit() (fd int, err error) //sys Ioperm(from int, num int, on int) (err error) @@ -38,7 +33,6 @@ const ( //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys Setfsgid(gid int) (err error) //sys Setfsuid(uid int) (err error) -//sysnb setrlimit(resource int, rlim *Rlimit) (err error) = SYS_SETRLIMIT //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) //sys Statfs(path string, buf *Statfs_t) (err error) @@ -103,12 +97,6 @@ func Time(t *Time_t) (tt Time_t, err error) { //sys Utime(path string, buf *Utimbuf) (err error) //sys utimes(path string, times *[2]Timeval) (err error) -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func setTimespec(sec, nsec int64) Timespec { return Timespec{Sec: sec, Nsec: nsec} } diff --git a/src/syscall/syscall_linux_arm.go b/src/syscall/syscall_linux_arm.go index a6d92cea1317e8..2019b15169d56b 100644 --- a/src/syscall/syscall_linux_arm.go +++ b/src/syscall/syscall_linux_arm.go @@ -124,96 +124,6 @@ func mmap(addr uintptr, length uintptr, prot int, flags int, fd int, offset int6 return mmap2(addr, length, prot, flags, fd, page) } -type rlimit32 struct { - Cur uint32 - Max uint32 -} - -//sysnb getrlimit(resource int, rlim *rlimit32) (err error) = SYS_GETRLIMIT - -const rlimInf32 = ^uint32(0) -const rlimInf64 = ^uint64(0) - -func Getrlimit(resource int, rlim *Rlimit) (err error) { - err = prlimit(0, resource, nil, rlim) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - err = getrlimit(resource, &rl) - if err != nil { - return - } - - if rl.Cur == rlimInf32 { - rlim.Cur = rlimInf64 - } else { - rlim.Cur = uint64(rl.Cur) - } - - if rl.Max == rlimInf32 { - rlim.Max = rlimInf64 - } else { - rlim.Max = uint64(rl.Max) - } - return -} - -//sysnb setrlimit1(resource int, rlim *rlimit32) (err error) = SYS_SETRLIMIT - -func setrlimit(resource int, rlim *Rlimit) (err error) { - err = prlimit(0, resource, rlim, nil) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - return setrlimit1(resource, &rl) -} - -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall6(SYS_PRLIMIT64, 0, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0, 0, 0) - if errno != ENOSYS { - return errno - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - _, _, errno = RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func (r *PtraceRegs) PC() uint64 { return uint64(r.Uregs[15]) } func (r *PtraceRegs) SetPC(pc uint64) { r.Uregs[15] = uint32(pc) } diff --git a/src/syscall/syscall_linux_arm64.go b/src/syscall/syscall_linux_arm64.go index b87b51c0c04e44..fb0ecd77b7ff5a 100644 --- a/src/syscall/syscall_linux_arm64.go +++ b/src/syscall/syscall_linux_arm64.go @@ -27,7 +27,6 @@ func Fstatat(fd int, path string, stat *Stat_t, flags int) error { //sysnb Getegid() (egid int) //sysnb Geteuid() (euid int) //sysnb Getgid() (gid int) -//sysnb getrlimit(resource int, rlim *Rlimit) (err error) //sysnb Getuid() (uid int) //sys Listen(s int, n int) (err error) //sys pread(fd int, p []byte, offset int64) (n int, err error) = SYS_PREAD64 @@ -37,7 +36,6 @@ func Fstatat(fd int, path string, stat *Stat_t, flags int) error { //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys Setfsgid(gid int) (err error) //sys Setfsuid(uid int) (err error) -//sysnb setrlimit1(resource int, rlim *Rlimit) (err error) = SYS_SETRLIMIT //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) @@ -140,34 +138,6 @@ func utimes(path string, tv *[2]Timeval) (err error) { return utimensat(_AT_FDCWD, path, (*[2]Timespec)(unsafe.Pointer(&ts[0])), 0) } -// Getrlimit prefers the prlimit64 system call. See issue 38604. -func Getrlimit(resource int, rlim *Rlimit) error { - err := prlimit(0, resource, nil, rlim) - if err != ENOSYS { - return err - } - return getrlimit(resource, rlim) -} - -// setrlimit prefers the prlimit64 system call. See issue 38604. -func setrlimit(resource int, rlim *Rlimit) error { - err := prlimit(0, resource, rlim, nil) - if err != ENOSYS { - return err - } - return setrlimit1(resource, rlim) -} - -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall6(SYS_PRLIMIT64, 0, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0, 0, 0) - if errno != ENOSYS { - return errno - } - _, _, errno = RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func (r *PtraceRegs) PC() uint64 { return r.Pc } func (r *PtraceRegs) SetPC(pc uint64) { r.Pc = pc } diff --git a/src/syscall/syscall_linux_loong64.go b/src/syscall/syscall_linux_loong64.go index 634cf30cf26902..360590d2d706ce 100644 --- a/src/syscall/syscall_linux_loong64.go +++ b/src/syscall/syscall_linux_loong64.go @@ -183,22 +183,6 @@ func utimes(path string, tv *[2]Timeval) (err error) { return utimensat(_AT_FDCWD, path, (*[2]Timespec)(unsafe.Pointer(&ts[0])), 0) } -// Getrlimit prefers the prlimit64 system call. -func Getrlimit(resource int, rlim *Rlimit) error { - return prlimit(0, resource, nil, rlim) -} - -// setrlimit prefers the prlimit64 system call. -func setrlimit(resource int, rlim *Rlimit) error { - return prlimit(0, resource, rlim, nil) -} - -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall6(SYS_PRLIMIT64, 0, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0, 0, 0) - return errno -} - func (r *PtraceRegs) GetEra() uint64 { return r.Era } func (r *PtraceRegs) SetEra(era uint64) { r.Era = era } diff --git a/src/syscall/syscall_linux_mips64x.go b/src/syscall/syscall_linux_mips64x.go index 41106ed81ff1ed..e826e08615e6d1 100644 --- a/src/syscall/syscall_linux_mips64x.go +++ b/src/syscall/syscall_linux_mips64x.go @@ -6,10 +6,6 @@ package syscall -import ( - "unsafe" -) - const ( _SYS_setgroups = SYS_SETGROUPS _SYS_clone3 = 5435 @@ -20,12 +16,10 @@ const ( //sys Dup2(oldfd int, newfd int) (err error) //sys Fchown(fd int, uid int, gid int) (err error) //sys Fstatfs(fd int, buf *Statfs_t) (err error) -//sys fstatat(dirfd int, path string, stat *Stat_t, flags int) (err error) = SYS_NEWFSTATAT //sys Ftruncate(fd int, length int64) (err error) //sysnb Getegid() (egid int) //sysnb Geteuid() (euid int) //sysnb Getgid() (gid int) -//sysnb Getrlimit(resource int, rlim *Rlimit) (err error) //sysnb Getuid() (uid int) //sysnb InotifyInit() (fd int, err error) //sys Lchown(path string, uid int, gid int) (err error) @@ -38,7 +32,6 @@ const ( //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys Setfsgid(gid int) (err error) //sys Setfsuid(uid int) (err error) -//sysnb setrlimit(resource int, rlim *Rlimit) (err error) = SYS_SETRLIMIT //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) //sys Statfs(path string, buf *Statfs_t) (err error) @@ -94,12 +87,6 @@ func Time(t *Time_t) (tt Time_t, err error) { //sys Utime(path string, buf *Utimbuf) (err error) //sys utimes(path string, times *[2]Timeval) (err error) -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func setTimespec(sec, nsec int64) Timespec { return Timespec{Sec: sec, Nsec: nsec} } @@ -138,10 +125,22 @@ type stat_t struct { Blocks int64 } +//sys fstatatInternal(dirfd int, path string, stat *stat_t, flags int) (err error) = SYS_NEWFSTATAT //sys fstat(fd int, st *stat_t) (err error) //sys lstat(path string, st *stat_t) (err error) //sys stat(path string, st *stat_t) (err error) +func fstatat(fd int, path string, s *Stat_t, flags int) (err error) { + st := &stat_t{} + err = fstatatInternal(fd, path, st, flags) + fillStat_t(s, st) + return +} + +func Fstatat(fd int, path string, s *Stat_t, flags int) (err error) { + return fstatat(fd, path, s, flags) +} + func Fstat(fd int, s *Stat_t) (err error) { st := &stat_t{} err = fstat(fd, st) diff --git a/src/syscall/syscall_linux_mipsx.go b/src/syscall/syscall_linux_mipsx.go index 7d4f8f2264df70..aa08792606b102 100644 --- a/src/syscall/syscall_linux_mipsx.go +++ b/src/syscall/syscall_linux_mipsx.go @@ -117,96 +117,6 @@ func mmap(addr uintptr, length uintptr, prot int, flags int, fd int, offset int6 return mmap2(addr, length, prot, flags, fd, page) } -const rlimInf32 = ^uint32(0) -const rlimInf64 = ^uint64(0) - -type rlimit32 struct { - Cur uint32 - Max uint32 -} - -//sysnb getrlimit(resource int, rlim *rlimit32) (err error) = SYS_GETRLIMIT - -func Getrlimit(resource int, rlim *Rlimit) (err error) { - err = prlimit(0, resource, nil, rlim) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - err = getrlimit(resource, &rl) - if err != nil { - return - } - - if rl.Cur == rlimInf32 { - rlim.Cur = rlimInf64 - } else { - rlim.Cur = uint64(rl.Cur) - } - - if rl.Max == rlimInf32 { - rlim.Max = rlimInf64 - } else { - rlim.Max = uint64(rl.Max) - } - return -} - -//sysnb setrlimit1(resource int, rlim *rlimit32) (err error) = SYS_SETRLIMIT - -func setrlimit(resource int, rlim *Rlimit) (err error) { - err = prlimit(0, resource, rlim, nil) - if err != ENOSYS { - return err - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - return setrlimit1(resource, &rl) -} - -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall6(SYS_PRLIMIT64, 0, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0, 0, 0) - if errno != ENOSYS { - return errno - } - - rl := rlimit32{} - if rlim.Cur == rlimInf64 { - rl.Cur = rlimInf32 - } else if rlim.Cur < uint64(rlimInf32) { - rl.Cur = uint32(rlim.Cur) - } else { - return EINVAL - } - if rlim.Max == rlimInf64 { - rl.Max = rlimInf32 - } else if rlim.Max < uint64(rlimInf32) { - rl.Max = uint32(rlim.Max) - } else { - return EINVAL - } - - _, _, errno = RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func (r *PtraceRegs) PC() uint64 { return uint64(r.Regs[64]) } func (r *PtraceRegs) SetPC(pc uint64) { r.Regs[64] = uint32(pc) } diff --git a/src/syscall/syscall_linux_ppc64x.go b/src/syscall/syscall_linux_ppc64x.go index 13c184c44fec98..124240231e4a96 100644 --- a/src/syscall/syscall_linux_ppc64x.go +++ b/src/syscall/syscall_linux_ppc64x.go @@ -6,10 +6,6 @@ package syscall -import ( - "unsafe" -) - const ( _SYS_setgroups = SYS_SETGROUPS _SYS_clone3 = 435 @@ -27,7 +23,6 @@ const ( //sysnb Getegid() (egid int) //sysnb Geteuid() (euid int) //sysnb Getgid() (gid int) -//sysnb Getrlimit(resource int, rlim *Rlimit) (err error) = SYS_UGETRLIMIT //sysnb Getuid() (uid int) //sysnb InotifyInit() (fd int, err error) //sys Ioperm(from int, num int, on int) (err error) @@ -44,7 +39,6 @@ const ( //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys Setfsgid(gid int) (err error) //sys Setfsuid(uid int) (err error) -//sysnb setrlimit(resource int, rlim *Rlimit) (err error) = SYS_SETRLIMIT //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) //sys Stat(path string, stat *Stat_t) (err error) @@ -73,12 +67,6 @@ const ( //sys Utime(path string, buf *Utimbuf) (err error) //sys utimes(path string, times *[2]Timeval) (err error) -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func setTimespec(sec, nsec int64) Timespec { return Timespec{Sec: sec, Nsec: nsec} } diff --git a/src/syscall/syscall_linux_riscv64.go b/src/syscall/syscall_linux_riscv64.go index 00872a74fba96d..c9159762d89c5d 100644 --- a/src/syscall/syscall_linux_riscv64.go +++ b/src/syscall/syscall_linux_riscv64.go @@ -27,7 +27,6 @@ func Fstatat(fd int, path string, stat *Stat_t, flags int) error { //sysnb Getegid() (egid int) //sysnb Geteuid() (euid int) //sysnb Getgid() (gid int) -//sysnb Getrlimit(resource int, rlim *Rlimit) (err error) //sysnb Getuid() (uid int) //sys Listen(s int, n int) (err error) //sys pread(fd int, p []byte, offset int64) (n int, err error) = SYS_PREAD64 @@ -37,7 +36,6 @@ func Fstatat(fd int, path string, stat *Stat_t, flags int) error { //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys Setfsgid(gid int) (err error) //sys Setfsuid(uid int) (err error) -//sysnb setrlimit(resource int, rlim *Rlimit) (err error) = SYS_SETRLIMIT //sys Shutdown(fd int, how int) (err error) //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) @@ -144,12 +142,6 @@ func utimes(path string, tv *[2]Timeval) (err error) { return utimensat(_AT_FDCWD, path, (*[2]Timespec)(unsafe.Pointer(&ts[0])), 0) } -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func (r *PtraceRegs) PC() uint64 { return r.Pc } func (r *PtraceRegs) SetPC(pc uint64) { r.Pc = pc } diff --git a/src/syscall/syscall_linux_s390x.go b/src/syscall/syscall_linux_s390x.go index ea667ec1da5411..65c86140fc5a82 100644 --- a/src/syscall/syscall_linux_s390x.go +++ b/src/syscall/syscall_linux_s390x.go @@ -23,7 +23,6 @@ const ( //sysnb Getegid() (egid int) //sysnb Geteuid() (euid int) //sysnb Getgid() (gid int) -//sysnb Getrlimit(resource int, rlim *Rlimit) (err error) = SYS_GETRLIMIT //sysnb Getuid() (uid int) //sysnb InotifyInit() (fd int, err error) //sys Lchown(path string, uid int, gid int) (err error) @@ -37,7 +36,6 @@ const ( //sys sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) //sys Setfsgid(gid int) (err error) //sys Setfsuid(uid int) (err error) -//sysnb setrlimit(resource int, rlim *Rlimit) (err error) = SYS_SETRLIMIT //sys Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) //sys Stat(path string, stat *Stat_t) (err error) //sys Statfs(path string, buf *Statfs_t) (err error) @@ -244,12 +242,6 @@ func Shutdown(s, how int) (err error) { return } -//go:nosplit -func rawSetrlimit(resource int, rlim *Rlimit) Errno { - _, _, errno := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - return errno -} - func (r *PtraceRegs) PC() uint64 { return r.Psw.Addr } func (r *PtraceRegs) SetPC(pc uint64) { r.Psw.Addr = pc } diff --git a/src/syscall/syscall_linux_test.go b/src/syscall/syscall_linux_test.go index d7543ceb4b1644..df096a0b47b81f 100644 --- a/src/syscall/syscall_linux_test.go +++ b/src/syscall/syscall_linux_test.go @@ -22,28 +22,6 @@ import ( "unsafe" ) -// chtmpdir changes the working directory to a new temporary directory and -// provides a cleanup function. Used when PWD is read-only. -func chtmpdir(t *testing.T) func() { - oldwd, err := os.Getwd() - if err != nil { - t.Fatalf("chtmpdir: %v", err) - } - d, err := os.MkdirTemp("", "test") - if err != nil { - t.Fatalf("chtmpdir: %v", err) - } - if err := os.Chdir(d); err != nil { - t.Fatalf("chtmpdir: %v", err) - } - return func() { - if err := os.Chdir(oldwd); err != nil { - t.Fatalf("chtmpdir: %v", err) - } - os.RemoveAll(d) - } -} - func touch(t *testing.T, name string) { f, err := os.Create(name) if err != nil { @@ -63,7 +41,7 @@ const ( ) func TestFaccessat(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) touch(t, "file1") err := syscall.Faccessat(_AT_FDCWD, "file1", _R_OK, 0) @@ -115,7 +93,7 @@ func TestFaccessat(t *testing.T) { } func TestFchmodat(t *testing.T) { - defer chtmpdir(t)() + t.Chdir(t.TempDir()) touch(t, "file1") os.Symlink("file1", "symlink1") @@ -212,12 +190,8 @@ func TestSyscallNoError(t *testing.T) { } // Copy the test binary to a location that a non-root user can read/execute - // after we drop privileges - tempDir, err := os.MkdirTemp("", "TestSyscallNoError") - if err != nil { - t.Fatalf("cannot create temporary directory: %v", err) - } - defer os.RemoveAll(tempDir) + // after we drop privileges. + tempDir := t.TempDir() os.Chmod(tempDir, 0755) tmpBinary := filepath.Join(tempDir, filepath.Base(os.Args[0])) @@ -524,6 +498,9 @@ func TestSetuidEtc(t *testing.T) { if syscall.Getuid() != 0 { t.Skip("skipping root only test") } + if syscall.Getgid() != 0 { + t.Skip("skipping the test when root's gid is not default value 0") + } if testing.Short() && testenv.Builder() != "" && os.Getenv("USER") == "swarming" { // The Go build system's swarming user is known not to be root. // Unfortunately, it sometimes appears as root due the current @@ -720,3 +697,191 @@ func TestPrlimitOtherProcess(t *testing.T) { t.Fatalf("origRlimitNofile got=%v, want=%v", rlimLater, rlimOrig) } } + +const magicRlimitValue = 42 + +// TestPrlimitFileLimit tests that we can start a Go program, use +// prlimit to change its NOFILE limit, and have that updated limit be +// seen by children. See issue #66797. +func TestPrlimitFileLimit(t *testing.T) { + switch os.Getenv("GO_WANT_HELPER_PROCESS") { + case "prlimit1": + testPrlimitFileLimitHelper1(t) + return + case "prlimit2": + testPrlimitFileLimitHelper2(t) + return + } + + origRlimitNofile := syscall.GetInternalOrigRlimitNofile() + defer origRlimitNofile.Store(origRlimitNofile.Load()) + + // Set our rlimit to magic+1/max. + // That will also become the rlimit of the child. + + var lim syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil { + t.Fatal(err) + } + max := lim.Max + + lim = syscall.Rlimit{ + Cur: magicRlimitValue + 1, + Max: max, + } + if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil { + t.Fatal(err) + } + + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + + r1, w1, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer r1.Close() + defer w1.Close() + + r2, w2, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer r2.Close() + defer w2.Close() + + var output strings.Builder + + const arg = "-test.run=^TestPrlimitFileLimit$" + cmd := testenv.CommandContext(t, t.Context(), exe, arg, "-test.v") + cmd = testenv.CleanCmdEnv(cmd) + cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=prlimit1") + cmd.ExtraFiles = []*os.File{r1, w2} + cmd.Stdout = &output + cmd.Stderr = &output + + t.Logf("running %s %s", exe, arg) + + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + + // Wait for the child to start. + b := make([]byte, 1) + if n, err := r2.Read(b); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatalf("read %d bytes, want 1", n) + } + + // Set the child's prlimit. + lim = syscall.Rlimit{ + Cur: magicRlimitValue, + Max: max, + } + if err := syscall.Prlimit(cmd.Process.Pid, syscall.RLIMIT_NOFILE, &lim, nil); err != nil { + t.Fatalf("Prlimit failed: %v", err) + } + + // Tell the child to continue. + if n, err := w1.Write(b); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatalf("wrote %d bytes, want 1", n) + } + + err = cmd.Wait() + if output.Len() > 0 { + t.Logf("%s", output.String()) + } + + if err != nil { + t.Errorf("child failed: %v", err) + } +} + +// testPrlimitFileLimitHelper1 is run by TestPrlimitFileLimit. +func testPrlimitFileLimitHelper1(t *testing.T) { + var lim syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil { + t.Fatal(err) + } + t.Logf("helper1 rlimit is %v", lim) + t.Logf("helper1 cached rlimit is %v", syscall.OrigRlimitNofile()) + + // Tell the parent that we are ready. + b := []byte{0} + if n, err := syscall.Write(4, b); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatalf("wrote %d bytes, want 1", n) + } + + // Wait for the parent to tell us that prlimit was used. + if n, err := syscall.Read(3, b); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatalf("read %d bytes, want 1", n) + } + + if err := syscall.Close(3); err != nil { + t.Errorf("Close(3): %v", err) + } + if err := syscall.Close(4); err != nil { + t.Errorf("Close(4): %v", err) + } + + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil { + t.Fatal(err) + } + t.Logf("after prlimit helper1 rlimit is %v", lim) + t.Logf("after prlimit helper1 cached rlimit is %v", syscall.OrigRlimitNofile()) + + // Start the grandchild, which should see the rlimit + // set by the prlimit called by the parent. + + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + + const arg = "-test.run=^TestPrlimitFileLimit$" + cmd := testenv.CommandContext(t, t.Context(), exe, arg, "-test.v") + cmd = testenv.CleanCmdEnv(cmd) + cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=prlimit2") + t.Logf("running %s %s", exe, arg) + out, err := cmd.CombinedOutput() + if len(out) > 0 { + t.Logf("%s", out) + } + if err != nil { + t.Errorf("grandchild failed: %v", err) + } else { + fmt.Println("OK") + } +} + +// testPrlimitFileLimitHelper2 is run by testPrlimitFileLimit1. +func testPrlimitFileLimitHelper2(t *testing.T) { + var lim syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil { + t.Fatal(err) + } + + t.Logf("helper2 rlimit is %v", lim) + cached := syscall.OrigRlimitNofile() + t.Logf("helper2 cached rlimit is %v", cached) + + // The value return by Getrlimit will have been adjusted. + // We should have cached the value set by prlimit called by the parent. + + if cached == nil { + t.Fatal("no cached rlimit") + } else if cached.Cur != magicRlimitValue { + t.Fatalf("cached rlimit is %d, want %d", cached.Cur, magicRlimitValue) + } + + fmt.Println("OK") +} diff --git a/src/syscall/syscall_openbsd_libc.go b/src/syscall/syscall_openbsd_libc.go index 5dea268c3e4232..eca1ad1f98c721 100644 --- a/src/syscall/syscall_openbsd_libc.go +++ b/src/syscall/syscall_openbsd_libc.go @@ -13,7 +13,7 @@ import ( var dupTrampoline = abi.FuncPCABI0(libc_dup3_trampoline) func init() { - execveOpenBSD = execve + execveLibc = execve } func syscallInternal(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) { @@ -79,7 +79,6 @@ func syscall9X(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, //sysnb execve(path *byte, argv **byte, envp **byte) (err error) //sysnb exit(res int) (err error) //sys ptrace(request int, pid int, addr uintptr, data uintptr) (err error) -//sysnb getentropy(p []byte) (err error) //sys fstatat(fd int, path string, stat *Stat_t, flags int) (err error) //sys unlinkat(fd int, path string, flags int) (err error) //sys openat(fd int, path string, flags int, perm uint32) (fdret int, err error) diff --git a/src/syscall/syscall_unix.go b/src/syscall/syscall_unix.go index ecd5952975a73d..7de2272b591b95 100644 --- a/src/syscall/syscall_unix.go +++ b/src/syscall/syscall_unix.go @@ -410,17 +410,25 @@ func SendmsgN(fd int, p, oob []byte, to Sockaddr, flags int) (n int, err error) } func sendmsgNInet4(fd int, p, oob []byte, to *SockaddrInet4, flags int) (n int, err error) { - ptr, salen, err := to.sockaddr() - if err != nil { - return 0, err + var ptr unsafe.Pointer + var salen _Socklen + if to != nil { + ptr, salen, err = to.sockaddr() + if err != nil { + return 0, err + } } return sendmsgN(fd, p, oob, ptr, salen, flags) } func sendmsgNInet6(fd int, p, oob []byte, to *SockaddrInet6, flags int) (n int, err error) { - ptr, salen, err := to.sockaddr() - if err != nil { - return 0, err + var ptr unsafe.Pointer + var salen _Socklen + if to != nil { + ptr, salen, err = to.sockaddr() + if err != nil { + return 0, err + } } return sendmsgN(fd, p, oob, ptr, salen, flags) } diff --git a/src/syscall/syscall_unix_test.go b/src/syscall/syscall_unix_test.go index 56e771e08603d0..8c6de43231e032 100644 --- a/src/syscall/syscall_unix_test.go +++ b/src/syscall/syscall_unix_test.go @@ -99,7 +99,7 @@ func TestFcntlFlock(t *testing.T) { t.Fatalf("FcntlFlock(F_SETLK) failed: %v", err) } - cmd := exec.Command(os.Args[0], "-test.run=^TestFcntlFlock$") + cmd := exec.Command(testenv.Executable(t), "-test.run=^TestFcntlFlock$") cmd.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=1") cmd.ExtraFiles = []*os.File{f} out, err := cmd.CombinedOutput() @@ -171,7 +171,7 @@ func TestPassFD(t *testing.T) { defer writeFile.Close() defer readFile.Close() - cmd := exec.Command(os.Args[0], "-test.run=^TestPassFD$", "--", tempDir) + cmd := exec.Command(testenv.Executable(t), "-test.run=^TestPassFD$", "--", tempDir) cmd.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=1") cmd.ExtraFiles = []*os.File{writeFile} diff --git a/src/syscall/syscall_wasip1.go b/src/syscall/syscall_wasip1.go index 84c6bddc0897df..c9225293a0cc31 100644 --- a/src/syscall/syscall_wasip1.go +++ b/src/syscall/syscall_wasip1.go @@ -216,12 +216,14 @@ const ( O_WRONLY = 1 O_RDWR = 2 - O_CREAT = 0100 - O_CREATE = O_CREAT - O_TRUNC = 01000 - O_APPEND = 02000 - O_EXCL = 0200 - O_SYNC = 010000 + O_CREAT = 0100 + O_CREATE = O_CREAT + O_TRUNC = 01000 + O_APPEND = 02000 + O_EXCL = 0200 + O_SYNC = 010000 + O_DIRECTORY = 020000 + O_NOFOLLOW = 0400 O_CLOEXEC = 0 ) @@ -307,7 +309,7 @@ func (w WaitStatus) TrapCause() int { return 0 } // Rusage is a placeholder to allow compilation of the [os/exec] package // because we need Go programs to be portable across platforms. WASI does -// not have a mechanism to to spawn processes so there is no reason for an +// not have a mechanism to spawn processes so there is no reason for an // application to take a dependency on this type. type Rusage struct { Utime Timeval @@ -316,7 +318,7 @@ type Rusage struct { // ProcAttr is a placeholder to allow compilation of the [os/exec] package // because we need Go programs to be portable across platforms. WASI does -// not have a mechanism to to spawn processes so there is no reason for an +// not have a mechanism to spawn processes so there is no reason for an // application to take a dependency on this type. type ProcAttr struct { Dir string @@ -381,7 +383,7 @@ func Getppid() int { func Gettimeofday(tv *Timeval) error { var time timestamp - if errno := clock_time_get(clockRealtime, 1e3, unsafe.Pointer(&time)); errno != 0 { + if errno := clock_time_get(clockRealtime, 1e3, &time); errno != 0 { return errno } tv.setTimestamp(time) @@ -463,7 +465,7 @@ const ( //go:wasmimport wasi_snapshot_preview1 clock_time_get //go:noescape -func clock_time_get(id clockid, precision timestamp, time unsafe.Pointer) Errno +func clock_time_get(id clockid, precision timestamp, time *timestamp) Errno func SetNonblock(fd int, nonblocking bool) error { flags, err := fd_fdstat_get_flags(fd) diff --git a/src/syscall/syscall_windows.go b/src/syscall/syscall_windows.go index d49ee522c4fe88..f86f03e20f0a0f 100644 --- a/src/syscall/syscall_windows.go +++ b/src/syscall/syscall_windows.go @@ -14,7 +14,6 @@ import ( "internal/msan" "internal/oserror" "internal/race" - "runtime" "sync" "unsafe" ) @@ -138,12 +137,32 @@ func FormatMessage(flags uint32, msgsrc uint32, msgid uint32, langid uint32, buf return formatMessage(flags, uintptr(msgsrc), msgid, langid, buf, args) } +var errnoErrorCache sync.Map + func (e Errno) Error() string { // deal with special go errors idx := int(e - APPLICATION_ERROR) if 0 <= idx && idx < len(errors) { return errors[idx] } + + cache := false + switch e { + case ERROR_FILE_NOT_FOUND, ERROR_PATH_NOT_FOUND: + if cached, ok := errnoErrorCache.Load(e); ok { + return cached.(string) + } + cache = true + } + + result := e.error() + if cache { + errnoErrorCache.Store(e, result) + } + return result +} + +func (e Errno) error() string { // ask windows for the remaining errors var flags uint32 = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ARGUMENT_ARRAY | FORMAT_MESSAGE_IGNORE_INSERTS b := make([]uint16, 300) @@ -235,7 +254,7 @@ func NewCallbackCDecl(fn any) uintptr { //sys GetVersion() (ver uint32, err error) //sys formatMessage(flags uint32, msgsrc uintptr, msgid uint32, langid uint32, buf []uint16, args *byte) (n uint32, err error) = FormatMessageW //sys ExitProcess(exitcode uint32) -//sys CreateFile(name *uint16, access uint32, mode uint32, sa *SecurityAttributes, createmode uint32, attrs uint32, templatefile int32) (handle Handle, err error) [failretval==InvalidHandle] = CreateFileW +//sys createFile(name *uint16, access uint32, mode uint32, sa *SecurityAttributes, createmode uint32, attrs uint32, templatefile int32) (handle Handle, err error) [failretval == InvalidHandle || e1 == ERROR_ALREADY_EXISTS ] = CreateFileW //sys readFile(handle Handle, buf []byte, done *uint32, overlapped *Overlapped) (err error) = ReadFile //sys writeFile(handle Handle, buf []byte, done *uint32, overlapped *Overlapped) (err error) = WriteFile //sys SetFilePointer(handle Handle, lowoffset int32, highoffsetptr *int32, whence uint32) (newlowoffset uint32, err error) [failretval==0xffffffff] @@ -287,6 +306,7 @@ func NewCallbackCDecl(fn any) uintptr { //sys GetCommandLine() (cmd *uint16) = kernel32.GetCommandLineW //sys CommandLineToArgv(cmd *uint16, argc *int32) (argv *[8192]*[8192]uint16, err error) [failretval==nil] = shell32.CommandLineToArgvW //sys LocalFree(hmem Handle) (handle Handle, err error) [failretval!=0] +//sys localAlloc(flags uint32, length uint32) (ptr uintptr, err error) = kernel32.LocalAlloc //sys SetHandleInformation(handle Handle, mask uint32, flags uint32) (err error) //sys FlushFileBuffers(handle Handle) (err error) //sys GetFullPathName(path *uint16, buflen uint32, buf *uint16, fname **uint16) (n uint32, err error) = kernel32.GetFullPathNameW @@ -323,6 +343,7 @@ func NewCallbackCDecl(fn any) uintptr { //sys Process32First(snapshot Handle, procEntry *ProcessEntry32) (err error) = kernel32.Process32FirstW //sys Process32Next(snapshot Handle, procEntry *ProcessEntry32) (err error) = kernel32.Process32NextW //sys DeviceIoControl(handle Handle, ioControlCode uint32, inBuffer *byte, inBufferSize uint32, outBuffer *byte, outBufferSize uint32, bytesReturned *uint32, overlapped *Overlapped) (err error) +//sys setFileInformationByHandle(handle Handle, fileInformationClass uint32, buf unsafe.Pointer, bufsize uint32) (err error) = kernel32.SetFileInformationByHandle // This function returns 1 byte BOOLEAN rather than the 4 byte BOOL. //sys CreateSymbolicLink(symlinkfilename *uint16, targetfilename *uint16, flags uint32) (err error) [failretval&0xff==0] = CreateSymbolicLinkW //sys CreateHardLink(filename *uint16, existingfilename *uint16, reserved uintptr) (err error) [failretval&0xff==0] = CreateHardLinkW @@ -340,16 +361,17 @@ func makeInheritSa() *SecurityAttributes { return &sa } -func Open(path string, mode int, perm uint32) (fd Handle, err error) { - if len(path) == 0 { +func Open(name string, flag int, perm uint32) (fd Handle, err error) { + if len(name) == 0 { return InvalidHandle, ERROR_FILE_NOT_FOUND } - pathp, err := UTF16PtrFromString(path) + namep, err := UTF16PtrFromString(name) if err != nil { return InvalidHandle, err } + accessFlags := flag & (O_RDONLY | O_WRONLY | O_RDWR) var access uint32 - switch mode & (O_RDONLY | O_WRONLY | O_RDWR) { + switch accessFlags { case O_RDONLY: access = GENERIC_READ case O_WRONLY: @@ -357,64 +379,101 @@ func Open(path string, mode int, perm uint32) (fd Handle, err error) { case O_RDWR: access = GENERIC_READ | GENERIC_WRITE } - if mode&O_CREAT != 0 { + if flag&O_CREAT != 0 { access |= GENERIC_WRITE } - if mode&O_APPEND != 0 { - access &^= GENERIC_WRITE - access |= FILE_APPEND_DATA + if flag&O_APPEND != 0 { + // Remove GENERIC_WRITE unless O_TRUNC is set, in which case we need it to truncate the file. + // We can't just remove FILE_WRITE_DATA because GENERIC_WRITE without FILE_WRITE_DATA + // starts appending at the beginning of the file rather than at the end. + if flag&O_TRUNC == 0 { + access &^= GENERIC_WRITE + } + // Set all access rights granted by GENERIC_WRITE except for FILE_WRITE_DATA. + access |= FILE_APPEND_DATA | FILE_WRITE_ATTRIBUTES | _FILE_WRITE_EA | STANDARD_RIGHTS_WRITE | SYNCHRONIZE } sharemode := uint32(FILE_SHARE_READ | FILE_SHARE_WRITE) var sa *SecurityAttributes - if mode&O_CLOEXEC == 0 { + if flag&O_CLOEXEC == 0 { sa = makeInheritSa() } + var attrs uint32 = FILE_ATTRIBUTE_NORMAL + if perm&S_IWRITE == 0 { + attrs = FILE_ATTRIBUTE_READONLY + } + // fileFlags contains the high 12 bits of flag. + // This bit range can be used by the caller to specify the file flags + // passed to CreateFile. It is an error to use if the bits can't be + // mapped to the supported FILE_FLAG_* constants. + if fileFlags := uint32(flag) & fileFlagsMask; fileFlags&^validFileFlagsMask == 0 { + attrs |= fileFlags + } else { + return InvalidHandle, oserror.ErrInvalid + } + + switch accessFlags { + case O_WRONLY, O_RDWR: + // Unix doesn't allow opening a directory with O_WRONLY + // or O_RDWR, so we don't set the flag in that case, + // which will make CreateFile fail with ERROR_ACCESS_DENIED. + // We will map that to EISDIR if the file is a directory. + default: + // We might be opening a directory for reading, + // and CreateFile requires FILE_FLAG_BACKUP_SEMANTICS + // to work with directories. + attrs |= FILE_FLAG_BACKUP_SEMANTICS + } + if flag&O_SYNC != 0 { + attrs |= _FILE_FLAG_WRITE_THROUGH + } + // We don't use CREATE_ALWAYS, because when opening a file with + // FILE_ATTRIBUTE_READONLY these will replace an existing file + // with a new, read-only one. See https://go.dev/issue/38225. + // + // Instead, we ftruncate the file after opening when O_TRUNC is set. var createmode uint32 switch { - case mode&(O_CREAT|O_EXCL) == (O_CREAT | O_EXCL): + case flag&(O_CREAT|O_EXCL) == (O_CREAT | O_EXCL): createmode = CREATE_NEW - case mode&(O_CREAT|O_TRUNC) == (O_CREAT | O_TRUNC): - createmode = CREATE_ALWAYS - case mode&O_CREAT == O_CREAT: + attrs |= FILE_FLAG_OPEN_REPARSE_POINT // don't follow symlinks + case flag&O_CREAT == O_CREAT: createmode = OPEN_ALWAYS - case mode&O_TRUNC == O_TRUNC: - createmode = TRUNCATE_EXISTING default: createmode = OPEN_EXISTING } - var attrs uint32 = FILE_ATTRIBUTE_NORMAL - if perm&S_IWRITE == 0 { - attrs = FILE_ATTRIBUTE_READONLY - if createmode == CREATE_ALWAYS { - // We have been asked to create a read-only file. - // If the file already exists, the semantics of - // the Unix open system call is to preserve the - // existing permissions. If we pass CREATE_ALWAYS - // and FILE_ATTRIBUTE_READONLY to CreateFile, - // and the file already exists, CreateFile will - // change the file permissions. - // Avoid that to preserve the Unix semantics. - h, e := CreateFile(pathp, access, sharemode, sa, TRUNCATE_EXISTING, FILE_ATTRIBUTE_NORMAL, 0) - switch e { - case ERROR_FILE_NOT_FOUND, _ERROR_BAD_NETPATH, ERROR_PATH_NOT_FOUND: - // File does not exist. These are the same - // errors as Errno.Is checks for ErrNotExist. - // Carry on to create the file. - default: - // Success or some different error. - return h, e + h, err := createFile(namep, access, sharemode, sa, createmode, attrs, 0) + if h == InvalidHandle { + if err == ERROR_ACCESS_DENIED && (attrs&FILE_FLAG_BACKUP_SEMANTICS == 0) { + // We should return EISDIR when we are trying to open a directory with write access. + fa, e1 := GetFileAttributes(namep) + if e1 == nil && fa&FILE_ATTRIBUTE_DIRECTORY != 0 { + err = EISDIR } } + return h, err + } + if flag&o_DIRECTORY != 0 { + // Check if the file is a directory, else return ENOTDIR. + var fi ByHandleFileInformation + if err := GetFileInformationByHandle(h, &fi); err != nil { + CloseHandle(h) + return InvalidHandle, err + } + if fi.FileAttributes&FILE_ATTRIBUTE_DIRECTORY == 0 { + CloseHandle(h) + return InvalidHandle, ENOTDIR + } } - if createmode == OPEN_EXISTING && access == GENERIC_READ { - // Necessary for opening directory handles. - attrs |= FILE_FLAG_BACKUP_SEMANTICS - } - if mode&O_SYNC != 0 { - const _FILE_FLAG_WRITE_THROUGH = 0x80000000 - attrs |= _FILE_FLAG_WRITE_THROUGH + // Ignore O_TRUNC if the file has just been created. + if flag&O_TRUNC == O_TRUNC && + (createmode == OPEN_EXISTING || (createmode == OPEN_ALWAYS && err == ERROR_ALREADY_EXISTS)) { + err = Ftruncate(h, 0) + if err != nil { + CloseHandle(h) + return InvalidHandle, err + } } - return CreateFile(pathp, access, sharemode, sa, createmode, attrs, 0) + return h, nil } func Read(fd Handle, p []byte) (n int, err error) { @@ -486,18 +545,8 @@ func setFilePointerEx(handle Handle, distToMove int64, newFilePointer *int64, wh if unsafe.Sizeof(uintptr(0)) == 8 { _, _, e1 = Syscall6(procSetFilePointerEx.Addr(), 4, uintptr(handle), uintptr(distToMove), uintptr(unsafe.Pointer(newFilePointer)), uintptr(whence), 0, 0) } else { - // Different 32-bit systems disgaree about whether distToMove starts 8-byte aligned. - switch runtime.GOARCH { - default: - panic("unsupported 32-bit architecture") - case "386": - // distToMove is a LARGE_INTEGER, which is 64 bits. - _, _, e1 = Syscall6(procSetFilePointerEx.Addr(), 5, uintptr(handle), uintptr(distToMove), uintptr(distToMove>>32), uintptr(unsafe.Pointer(newFilePointer)), uintptr(whence), 0) - case "arm": - // distToMove must be 8-byte aligned per ARM calling convention - // https://docs.microsoft.com/en-us/cpp/build/overview-of-arm-abi-conventions#stage-c-assignment-of-arguments-to-registers-and-stack - _, _, e1 = Syscall6(procSetFilePointerEx.Addr(), 6, uintptr(handle), 0, uintptr(distToMove), uintptr(distToMove>>32), uintptr(unsafe.Pointer(newFilePointer)), uintptr(whence)) - } + // distToMove is a LARGE_INTEGER, which is 64 bits. + _, _, e1 = Syscall6(procSetFilePointerEx.Addr(), 5, uintptr(handle), uintptr(distToMove), uintptr(distToMove>>32), uintptr(unsafe.Pointer(newFilePointer)), uintptr(whence), 0) } if e1 != 0 { return errnoErr(e1) @@ -610,20 +659,13 @@ func ComputerName() (name string, err error) { } func Ftruncate(fd Handle, length int64) (err error) { - curoffset, e := Seek(fd, 0, 1) - if e != nil { - return e + type _FILE_END_OF_FILE_INFO struct { + EndOfFile int64 } - defer Seek(fd, curoffset, 0) - _, e = Seek(fd, length, 0) - if e != nil { - return e - } - e = SetEndOfFile(fd) - if e != nil { - return e - } - return nil + const FileEndOfFileInfo = 6 + var info _FILE_END_OF_FILE_INFO + info.EndOfFile = length + return setFileInformationByHandle(fd, FileEndOfFileInfo, unsafe.Pointer(&info), uint32(unsafe.Sizeof(info))) } func Gettimeofday(tv *Timeval) (err error) { @@ -855,23 +897,29 @@ func (sa *SockaddrUnix) sockaddr() (unsafe.Pointer, int32, error) { if n > len(sa.raw.Path) { return nil, 0, EINVAL } - if n == len(sa.raw.Path) && name[0] != '@' { + // Abstract addresses start with NUL. + // '@' is also a valid way to specify abstract addresses. + isAbstract := n > 0 && (name[0] == '@' || name[0] == '\x00') + + // Non-abstract named addresses are NUL terminated. + // The length can't use the full capacity as we need to add NUL. + if n == len(sa.raw.Path) && !isAbstract { return nil, 0, EINVAL } sa.raw.Family = AF_UNIX for i := 0; i < n; i++ { sa.raw.Path[i] = int8(name[i]) } - // length is family (uint16), name, NUL. - sl := int32(2) - if n > 0 { - sl += int32(n) + 1 - } - if sa.raw.Path[0] == '@' || (sa.raw.Path[0] == 0 && sl > 3) { - // Check sl > 3 so we don't change unnamed socket behavior. + // Length is family + name (+ NUL if non-abstract). + // Family is of type uint16 (2 bytes). + sl := int32(2 + n) + if isAbstract { + // Abstract addresses are not NUL terminated. + // We rewrite '@' prefix to NUL here. sa.raw.Path[0] = 0 - // Don't count trailing NUL for abstract address. - sl-- + } else if n > 0 { + // Add NUL for non-abstract named addresses. + sl++ } return unsafe.Pointer(&sa.raw), sl, nil @@ -1182,7 +1230,9 @@ func SetsockoptInet4Addr(fd Handle, level, opt int, value [4]byte) (err error) { func SetsockoptIPMreq(fd Handle, level, opt int, mreq *IPMreq) (err error) { return Setsockopt(fd, int32(level), int32(opt), (*byte)(unsafe.Pointer(mreq)), int32(unsafe.Sizeof(*mreq))) } -func SetsockoptIPv6Mreq(fd Handle, level, opt int, mreq *IPv6Mreq) (err error) { return EWINDOWS } +func SetsockoptIPv6Mreq(fd Handle, level, opt int, mreq *IPv6Mreq) (err error) { + return Setsockopt(fd, int32(level), int32(opt), (*byte)(unsafe.Pointer(mreq)), int32(unsafe.Sizeof(*mreq))) +} func Getpid() (pid int) { return int(getCurrentProcessId()) } @@ -1400,10 +1450,8 @@ func PostQueuedCompletionStatus(cphandle Handle, qty uint32, key uint32, overlap return postQueuedCompletionStatus(cphandle, qty, uintptr(key), overlapped) } -// newProcThreadAttributeList allocates new PROC_THREAD_ATTRIBUTE_LIST, with -// the requested maximum number of attributes, which must be cleaned up by -// deleteProcThreadAttributeList. -func newProcThreadAttributeList(maxAttrCount uint32) (*_PROC_THREAD_ATTRIBUTE_LIST, error) { +// newProcThreadAttributeList allocates a new [procThreadAttributeListContainer], with the requested maximum number of attributes. +func newProcThreadAttributeList(maxAttrCount uint32) (*procThreadAttributeListContainer, error) { var size uintptr err := initializeProcThreadAttributeList(nil, maxAttrCount, 0, &size) if err != ERROR_INSUFFICIENT_BUFFER { @@ -1412,13 +1460,38 @@ func newProcThreadAttributeList(maxAttrCount uint32) (*_PROC_THREAD_ATTRIBUTE_LI } return nil, err } - // size is guaranteed to be ≥1 by initializeProcThreadAttributeList. - al := (*_PROC_THREAD_ATTRIBUTE_LIST)(unsafe.Pointer(&make([]byte, size)[0])) - err = initializeProcThreadAttributeList(al, maxAttrCount, 0, &size) + const LMEM_FIXED = 0 + alloc, err := localAlloc(LMEM_FIXED, uint32(size)) if err != nil { return nil, err } - return al, nil + // size is guaranteed to be ≥1 by InitializeProcThreadAttributeList. + al := &procThreadAttributeListContainer{data: (*_PROC_THREAD_ATTRIBUTE_LIST)(unsafe.Pointer(alloc))} + err = initializeProcThreadAttributeList(al.data, maxAttrCount, 0, &size) + if err != nil { + return nil, err + } + al.pointers = make([]unsafe.Pointer, 0, maxAttrCount) + return al, err +} + +// Update modifies the ProcThreadAttributeList using UpdateProcThreadAttribute. +func (al *procThreadAttributeListContainer) update(attribute uintptr, value unsafe.Pointer, size uintptr) error { + al.pointers = append(al.pointers, value) + return updateProcThreadAttribute(al.data, 0, attribute, value, size, nil, nil) +} + +// Delete frees ProcThreadAttributeList's resources. +func (al *procThreadAttributeListContainer) delete() { + deleteProcThreadAttributeList(al.data) + LocalFree(Handle(unsafe.Pointer(al.data))) + al.data = nil + al.pointers = nil +} + +// List returns the actual ProcThreadAttributeList to be passed to StartupInfoEx. +func (al *procThreadAttributeListContainer) list() *_PROC_THREAD_ATTRIBUTE_LIST { + return al.data } // RegEnumKeyEx enumerates the subkeys of an open registry key. @@ -1453,3 +1526,13 @@ func GetStartupInfo(startupInfo *StartupInfo) error { getStartupInfo(startupInfo) return nil } + +func CreateFile(name *uint16, access uint32, mode uint32, sa *SecurityAttributes, createmode uint32, attrs uint32, templatefile int32) (handle Handle, err error) { + handle, err = createFile(name, access, mode, sa, createmode, attrs, templatefile) + if handle != InvalidHandle { + // CreateFileW can return ERROR_ALREADY_EXISTS with a valid handle. + // We only want to return an error if the handle is invalid. + err = nil + } + return handle, err +} diff --git a/src/syscall/syscall_windows_test.go b/src/syscall/syscall_windows_test.go index f67e8991591601..30dcddfd28ce6e 100644 --- a/src/syscall/syscall_windows_test.go +++ b/src/syscall/syscall_windows_test.go @@ -15,25 +15,41 @@ import ( "testing" ) -func TestOpen_Dir(t *testing.T) { - dir := t.TempDir() +func TestOpen(t *testing.T) { + t.Parallel() - h, err := syscall.Open(dir, syscall.O_RDONLY, 0) + dir := t.TempDir() + file := filepath.Join(dir, "a") + f, err := os.Create(file) if err != nil { - t.Fatalf("Open failed: %v", err) + t.Fatal(err) } - syscall.CloseHandle(h) - h, err = syscall.Open(dir, syscall.O_RDONLY|syscall.O_TRUNC, 0) - if err == nil { - t.Error("Open should have failed") - } else { - syscall.CloseHandle(h) + f.Close() + + tests := []struct { + path string + flag int + err error + }{ + {dir, syscall.O_RDONLY, nil}, + {dir, syscall.O_CREAT, nil}, + {dir, syscall.O_RDONLY | syscall.O_CREAT, nil}, + {file, syscall.O_APPEND | syscall.O_WRONLY | os.O_CREATE, nil}, + {file, syscall.O_APPEND | syscall.O_WRONLY | os.O_CREATE | os.O_TRUNC, nil}, + {file, syscall.O_WRONLY | syscall.O_RDWR, nil}, + {dir, syscall.O_WRONLY | syscall.O_RDWR, nil}, + {dir, syscall.O_RDONLY | syscall.O_TRUNC, syscall.ERROR_ACCESS_DENIED}, + {dir, syscall.O_WRONLY, syscall.EISDIR}, + {dir, syscall.O_RDWR, syscall.EISDIR}, } - h, err = syscall.Open(dir, syscall.O_RDONLY|syscall.O_CREAT, 0) - if err == nil { - t.Error("Open should have failed") - } else { - syscall.CloseHandle(h) + for i, tt := range tests { + h, err := syscall.Open(tt.path, tt.flag, 0o660) + if err == nil { + syscall.CloseHandle(h) + } + if err != tt.err { + t.Errorf("%d: Open got %q, want %q", i, err, tt.err) + } } } @@ -182,12 +198,14 @@ int main(int argc, char *argv[]) func TestGetwd_DoesNotPanicWhenPathIsLong(t *testing.T) { // Regression test for https://github.com/golang/go/issues/60051. + tmp := t.TempDir() + t.Chdir(tmp) // The length of a filename is also limited, so we can't reproduce the // crash by creating a single directory with a very long name; we need two // layers. a200 := strings.Repeat("a", 200) - dirname := filepath.Join(t.TempDir(), a200, a200) + dirname := filepath.Join(tmp, a200, a200) err := os.MkdirAll(dirname, 0o700) if err != nil { @@ -197,9 +215,6 @@ func TestGetwd_DoesNotPanicWhenPathIsLong(t *testing.T) { if err != nil { t.Skipf("Chdir failed: %v", err) } - // Change out of the temporary directory so that we don't inhibit its - // removal during test cleanup. - defer os.Chdir(`\`) syscall.Getwd() } @@ -213,6 +228,51 @@ func TestGetStartupInfo(t *testing.T) { } } +func TestSyscallAllocations(t *testing.T) { + testenv.SkipIfOptimizationOff(t) + + // Test that syscall.SyscallN arguments do not escape. + // The function used (in this case GetVersion) doesn't matter + // as long as it is always available and doesn't panic. + h, err := syscall.LoadLibrary("kernel32.dll") + if err != nil { + t.Fatal(err) + } + defer syscall.FreeLibrary(h) + proc, err := syscall.GetProcAddress(h, "GetVersion") + if err != nil { + t.Fatal(err) + } + + testAllocs := func(t *testing.T, name string, fn func() error) { + t.Run(name, func(t *testing.T) { + n := int(testing.AllocsPerRun(10, func() { + if err := fn(); err != nil { + t.Fatalf("%s: %v", name, err) + } + })) + if n > 0 { + t.Errorf("allocs = %d, want 0", n) + } + }) + } + + testAllocs(t, "SyscallN", func() error { + r0, _, e1 := syscall.SyscallN(proc, 0, 0, 0) + if r0 == 0 { + return syscall.Errno(e1) + } + return nil + }) + testAllocs(t, "Syscall", func() error { + r0, _, e1 := syscall.Syscall(proc, 3, 0, 0, 0) + if r0 == 0 { + return syscall.Errno(e1) + } + return nil + }) +} + func FuzzUTF16FromString(f *testing.F) { f.Add("hi") // ASCII f.Add("â") // latin1 @@ -240,3 +300,10 @@ func FuzzUTF16FromString(f *testing.F) { } }) } + +func BenchmarkErrnoString(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = syscall.Errno(2).Error() + } +} diff --git a/src/syscall/tables_wasip1.go b/src/syscall/tables_wasip1.go index 973a56e2744e95..be178c1cca4943 100644 --- a/src/syscall/tables_wasip1.go +++ b/src/syscall/tables_wasip1.go @@ -86,6 +86,7 @@ const ( ETXTBSY Errno = 74 EXDEV Errno = 75 ENOTCAPABLE Errno = 76 + EBADFD Errno = 77 // needed by src/net/error_unix_test.go EOPNOTSUPP = ENOTSUP ) @@ -100,6 +101,7 @@ var errorstr = [...]string{ EAGAIN: "Try again", EALREADY: "Socket already connected", EBADF: "Bad file number", + EBADFD: "file descriptor in bad state", EBADMSG: "Trying to read unreadable message", EBUSY: "Device or resource busy", ECANCELED: "Operation canceled.", diff --git a/src/syscall/types_windows.go b/src/syscall/types_windows.go index 6743675b95909b..b40b455e7de9b2 100644 --- a/src/syscall/types_windows.go +++ b/src/syscall/types_windows.go @@ -4,6 +4,8 @@ package syscall +import "unsafe" + const ( // Windows errors. ERROR_FILE_NOT_FOUND Errno = 2 @@ -34,18 +36,22 @@ const ( const ( // Invented values to support what package os expects. - O_RDONLY = 0x00000 - O_WRONLY = 0x00001 - O_RDWR = 0x00002 - O_CREAT = 0x00040 - O_EXCL = 0x00080 - O_NOCTTY = 0x00100 - O_TRUNC = 0x00200 - O_NONBLOCK = 0x00800 - O_APPEND = 0x00400 - O_SYNC = 0x01000 - O_ASYNC = 0x02000 - O_CLOEXEC = 0x80000 + O_RDONLY = 0x00000 + O_WRONLY = 0x00001 + O_RDWR = 0x00002 + O_CREAT = 0x00040 + O_EXCL = 0x00080 + O_NOCTTY = 0x00100 + O_TRUNC = 0x00200 + O_NONBLOCK = 0x00800 + O_APPEND = 0x00400 + O_SYNC = 0x01000 + O_ASYNC = 0x02000 + o_DIRECTORY = 0x04000 + O_CLOEXEC = 0x80000 + o_NOFOLLOW_ANY = 0x200000000 // used by internal/syscall/windows + o_OPEN_REPARSE = 0x400000000 // used by internal/syscall/windows + o_WRITE_ATTRS = 0x800000000 // used by internal/syscall/windows ) const ( @@ -83,6 +89,20 @@ var signals = [...]string{ 15: "terminated", } +const fileFlagsMask = 0xFFF00000 + +const validFileFlagsMask = FILE_FLAG_OPEN_REPARSE_POINT | + FILE_FLAG_BACKUP_SEMANTICS | + FILE_FLAG_OVERLAPPED | + _FILE_FLAG_OPEN_NO_RECALL | + _FILE_FLAG_SESSION_AWARE | + _FILE_FLAG_POSIX_SEMANTICS | + _FILE_FLAG_DELETE_ON_CLOSE | + _FILE_FLAG_SEQUENTIAL_SCAN | + _FILE_FLAG_NO_BUFFERING | + _FILE_FLAG_RANDOM_ACCESS | + _FILE_FLAG_WRITE_THROUGH + const ( GENERIC_READ = 0x80000000 GENERIC_WRITE = 0x40000000 @@ -91,6 +111,7 @@ const ( FILE_LIST_DIRECTORY = 0x00000001 FILE_APPEND_DATA = 0x00000004 + _FILE_WRITE_EA = 0x00000010 FILE_WRITE_ATTRIBUTES = 0x00000100 FILE_SHARE_READ = 0x00000001 @@ -112,9 +133,19 @@ const ( OPEN_ALWAYS = 4 TRUNCATE_EXISTING = 5 + // The following flags are supported by [Open] + // and exported in [golang.org/x/sys/windows]. + _FILE_FLAG_OPEN_NO_RECALL = 0x00100000 FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000 + _FILE_FLAG_SESSION_AWARE = 0x00800000 + _FILE_FLAG_POSIX_SEMANTICS = 0x01000000 FILE_FLAG_BACKUP_SEMANTICS = 0x02000000 + _FILE_FLAG_DELETE_ON_CLOSE = 0x04000000 + _FILE_FLAG_SEQUENTIAL_SCAN = 0x08000000 + _FILE_FLAG_RANDOM_ACCESS = 0x10000000 + _FILE_FLAG_NO_BUFFERING = 0x20000000 FILE_FLAG_OVERLAPPED = 0x40000000 + _FILE_FLAG_WRITE_THROUGH = 0x80000000 HANDLE_FLAG_INHERIT = 0x00000001 STARTF_USESTDHANDLES = 0x00000100 @@ -491,8 +522,15 @@ type StartupInfo struct { StdErr Handle } -type _PROC_THREAD_ATTRIBUTE_LIST struct { - _ [1]byte +// _PROC_THREAD_ATTRIBUTE_LIST is a placeholder type to represent a the opaque PROC_THREAD_ATTRIBUTE_LIST. +// +// Manipulate this type only through [procThreadAttributeListContainer] to ensure proper handling of the +// underlying memory. See https://g.dev/issue/73170. +type _PROC_THREAD_ATTRIBUTE_LIST struct{} + +type procThreadAttributeListContainer struct { + data *_PROC_THREAD_ATTRIBUTE_LIST + pointers []unsafe.Pointer } const ( diff --git a/src/syscall/types_windows_arm.go b/src/syscall/types_windows_arm.go deleted file mode 100644 index e72e9f5ced2bd0..00000000000000 --- a/src/syscall/types_windows_arm.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package syscall - -type WSAData struct { - Version uint16 - HighVersion uint16 - Description [WSADESCRIPTION_LEN + 1]byte - SystemStatus [WSASYS_STATUS_LEN + 1]byte - MaxSockets uint16 - MaxUdpDg uint16 - VendorInfo *byte -} - -type Servent struct { - Name *byte - Aliases **byte - Port uint16 - Proto *byte -} diff --git a/src/syscall/zerrors_darwin_amd64.go b/src/syscall/zerrors_darwin_amd64.go index ecbe89c5473a7c..bad1f889b6365e 100644 --- a/src/syscall/zerrors_darwin_amd64.go +++ b/src/syscall/zerrors_darwin_amd64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build amd64 && darwin - package syscall const ( diff --git a/src/syscall/zerrors_darwin_arm64.go b/src/syscall/zerrors_darwin_arm64.go index fa7cb845c31689..53236ca4608147 100644 --- a/src/syscall/zerrors_darwin_arm64.go +++ b/src/syscall/zerrors_darwin_arm64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build arm64 && darwin - package syscall const ( diff --git a/src/syscall/zerrors_dragonfly_amd64.go b/src/syscall/zerrors_dragonfly_amd64.go index bca2f50c9695da..d87ab132fe76ad 100644 --- a/src/syscall/zerrors_dragonfly_amd64.go +++ b/src/syscall/zerrors_dragonfly_amd64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build amd64 && dragonfly - package syscall const ( diff --git a/src/syscall/zerrors_freebsd_386.go b/src/syscall/zerrors_freebsd_386.go index b1441e76a3c77d..c59b7dae6c5e42 100644 --- a/src/syscall/zerrors_freebsd_386.go +++ b/src/syscall/zerrors_freebsd_386.go @@ -1,11 +1,9 @@ // mkerrors.sh -m32 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m32 _const.go -//go:build 386 && freebsd - package syscall const ( diff --git a/src/syscall/zerrors_freebsd_amd64.go b/src/syscall/zerrors_freebsd_amd64.go index 3aed004986d0be..305b8c66ffe35b 100644 --- a/src/syscall/zerrors_freebsd_amd64.go +++ b/src/syscall/zerrors_freebsd_amd64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build amd64 && freebsd - package syscall const ( diff --git a/src/syscall/zerrors_freebsd_arm.go b/src/syscall/zerrors_freebsd_arm.go index e1f91ff86cb7ec..9e082e308f13a9 100644 --- a/src/syscall/zerrors_freebsd_arm.go +++ b/src/syscall/zerrors_freebsd_arm.go @@ -1,11 +1,9 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- _const.go -//go:build arm && freebsd - package syscall const ( diff --git a/src/syscall/zerrors_freebsd_arm64.go b/src/syscall/zerrors_freebsd_arm64.go index d0cb6c8ac751ba..305b8c66ffe35b 100644 --- a/src/syscall/zerrors_freebsd_arm64.go +++ b/src/syscall/zerrors_freebsd_arm64.go @@ -1,8 +1,6 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -//go:build freebsd && arm64 - // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go diff --git a/src/syscall/zerrors_freebsd_riscv64.go b/src/syscall/zerrors_freebsd_riscv64.go index 7aa9aa985a3f58..305b8c66ffe35b 100644 --- a/src/syscall/zerrors_freebsd_riscv64.go +++ b/src/syscall/zerrors_freebsd_riscv64.go @@ -1,8 +1,6 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -//go:build freebsd && riscv64 - // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go diff --git a/src/syscall/zerrors_linux_386.go b/src/syscall/zerrors_linux_386.go index 045a4166b0418b..05b621a2cc51d7 100644 --- a/src/syscall/zerrors_linux_386.go +++ b/src/syscall/zerrors_linux_386.go @@ -1,11 +1,9 @@ // mkerrors.sh -m32 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m32 _const.go -//go:build 386 && linux - package syscall const ( @@ -1469,7 +1467,7 @@ var errors = [...]string{ 113: "no route to host", 114: "operation already in progress", 115: "operation now in progress", - 116: "stale NFS file handle", + 116: "stale file handle", 117: "structure needs cleaning", 118: "not a XENIX named type file", 119: "no XENIX semaphores available", diff --git a/src/syscall/zerrors_linux_amd64.go b/src/syscall/zerrors_linux_amd64.go index 4eb44746487634..8bc322a707e8a2 100644 --- a/src/syscall/zerrors_linux_amd64.go +++ b/src/syscall/zerrors_linux_amd64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build amd64 && linux - package syscall const ( @@ -1470,7 +1468,7 @@ var errors = [...]string{ 113: "no route to host", 114: "operation already in progress", 115: "operation now in progress", - 116: "stale NFS file handle", + 116: "stale file handle", 117: "structure needs cleaning", 118: "not a XENIX named type file", 119: "no XENIX semaphores available", diff --git a/src/syscall/zerrors_linux_arm.go b/src/syscall/zerrors_linux_arm.go index a5f925ec6dde65..ccbea5f287d6be 100644 --- a/src/syscall/zerrors_linux_arm.go +++ b/src/syscall/zerrors_linux_arm.go @@ -1,11 +1,9 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- _const.go -//go:build arm && linux - package syscall const ( @@ -1482,7 +1480,7 @@ var errors = [...]string{ 113: "no route to host", 114: "operation already in progress", 115: "operation now in progress", - 116: "stale NFS file handle", + 116: "stale file handle", 117: "structure needs cleaning", 118: "not a XENIX named type file", 119: "no XENIX semaphores available", diff --git a/src/syscall/zerrors_linux_arm64.go b/src/syscall/zerrors_linux_arm64.go index ec8ac0708e0e49..d32a290ad35ab3 100644 --- a/src/syscall/zerrors_linux_arm64.go +++ b/src/syscall/zerrors_linux_arm64.go @@ -1,11 +1,9 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- _const.go -//go:build arm64 && linux - package syscall const ( diff --git a/src/syscall/zerrors_linux_mips.go b/src/syscall/zerrors_linux_mips.go index 3fe5c007576783..81657d1c33a8a2 100644 --- a/src/syscall/zerrors_linux_mips.go +++ b/src/syscall/zerrors_linux_mips.go @@ -1,7 +1,7 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- _const.go package syscall diff --git a/src/syscall/zerrors_linux_mips64.go b/src/syscall/zerrors_linux_mips64.go index 74a1843ec2975d..7d505c7bb05618 100644 --- a/src/syscall/zerrors_linux_mips64.go +++ b/src/syscall/zerrors_linux_mips64.go @@ -1,7 +1,7 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs _const.go package syscall @@ -1744,7 +1744,7 @@ var errors = [...]string{ 148: "no route to host", 149: "operation already in progress", 150: "operation now in progress", - 151: "stale NFS file handle", + 151: "stale file handle", 158: "operation canceled", 159: "no medium found", 160: "wrong medium type", diff --git a/src/syscall/zerrors_linux_mips64le.go b/src/syscall/zerrors_linux_mips64le.go index 74a1843ec2975d..7d505c7bb05618 100644 --- a/src/syscall/zerrors_linux_mips64le.go +++ b/src/syscall/zerrors_linux_mips64le.go @@ -1,7 +1,7 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs _const.go package syscall @@ -1744,7 +1744,7 @@ var errors = [...]string{ 148: "no route to host", 149: "operation already in progress", 150: "operation now in progress", - 151: "stale NFS file handle", + 151: "stale file handle", 158: "operation canceled", 159: "no medium found", 160: "wrong medium type", diff --git a/src/syscall/zerrors_linux_mipsle.go b/src/syscall/zerrors_linux_mipsle.go index 3fe5c007576783..81657d1c33a8a2 100644 --- a/src/syscall/zerrors_linux_mipsle.go +++ b/src/syscall/zerrors_linux_mipsle.go @@ -1,7 +1,7 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- _const.go package syscall diff --git a/src/syscall/zerrors_linux_ppc64.go b/src/syscall/zerrors_linux_ppc64.go index b63daea853b2c8..fba5353b3e4b16 100644 --- a/src/syscall/zerrors_linux_ppc64.go +++ b/src/syscall/zerrors_linux_ppc64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build ppc64 && linux - package syscall const ( @@ -1802,7 +1800,7 @@ var errors = [...]string{ 113: "no route to host", 114: "operation already in progress", 115: "operation now in progress", - 116: "stale NFS file handle", + 116: "stale file handle", 117: "structure needs cleaning", 118: "not a XENIX named type file", 119: "no XENIX semaphores available", diff --git a/src/syscall/zerrors_linux_ppc64le.go b/src/syscall/zerrors_linux_ppc64le.go index 01f8adb0275280..73849d49880fa1 100644 --- a/src/syscall/zerrors_linux_ppc64le.go +++ b/src/syscall/zerrors_linux_ppc64le.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build ppc64le && linux - package syscall const ( diff --git a/src/syscall/zerrors_linux_riscv64.go b/src/syscall/zerrors_linux_riscv64.go index f4b1d9ae65f21f..760d87429a4a45 100644 --- a/src/syscall/zerrors_linux_riscv64.go +++ b/src/syscall/zerrors_linux_riscv64.go @@ -1,7 +1,7 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- _const.go package syscall diff --git a/src/syscall/zerrors_linux_s390x.go b/src/syscall/zerrors_linux_s390x.go index 8b99a60995e26e..9abad9ed704f8b 100644 --- a/src/syscall/zerrors_linux_s390x.go +++ b/src/syscall/zerrors_linux_s390x.go @@ -1,7 +1,7 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go package syscall diff --git a/src/syscall/zerrors_netbsd_386.go b/src/syscall/zerrors_netbsd_386.go index b2fcb65782e98c..6f21b562781794 100644 --- a/src/syscall/zerrors_netbsd_386.go +++ b/src/syscall/zerrors_netbsd_386.go @@ -1,11 +1,9 @@ // mkerrors.sh -m32 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m32 _const.go -//go:build 386 && netbsd - package syscall const ( diff --git a/src/syscall/zerrors_netbsd_amd64.go b/src/syscall/zerrors_netbsd_amd64.go index dc52c3ca573ec8..a404ac213d2018 100644 --- a/src/syscall/zerrors_netbsd_amd64.go +++ b/src/syscall/zerrors_netbsd_amd64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build amd64 && netbsd - package syscall const ( diff --git a/src/syscall/zerrors_netbsd_arm.go b/src/syscall/zerrors_netbsd_arm.go index 3137e18a2471f5..fdf0ca0cfda46d 100644 --- a/src/syscall/zerrors_netbsd_arm.go +++ b/src/syscall/zerrors_netbsd_arm.go @@ -1,11 +1,9 @@ // mkerrors.sh -marm // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -marm _const.go -//go:build arm && netbsd - package syscall const ( diff --git a/src/syscall/zerrors_netbsd_arm64.go b/src/syscall/zerrors_netbsd_arm64.go index cc1b0088c9ab95..a404ac213d2018 100644 --- a/src/syscall/zerrors_netbsd_arm64.go +++ b/src/syscall/zerrors_netbsd_arm64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build arm64 && netbsd - package syscall const ( diff --git a/src/syscall/zerrors_openbsd_386.go b/src/syscall/zerrors_openbsd_386.go index a37d6d16346d4f..4bdae34bbbaf46 100644 --- a/src/syscall/zerrors_openbsd_386.go +++ b/src/syscall/zerrors_openbsd_386.go @@ -1,11 +1,9 @@ // mkerrors.sh -m32 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m32 _const.go -//go:build 386 && openbsd - package syscall const ( diff --git a/src/syscall/zerrors_openbsd_amd64.go b/src/syscall/zerrors_openbsd_amd64.go index 812fd950a7fc89..69c68502bee64d 100644 --- a/src/syscall/zerrors_openbsd_amd64.go +++ b/src/syscall/zerrors_openbsd_amd64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build amd64 && openbsd - package syscall const ( diff --git a/src/syscall/zerrors_openbsd_arm.go b/src/syscall/zerrors_openbsd_arm.go index 2e19672b05a226..dc6379b3c05d7a 100644 --- a/src/syscall/zerrors_openbsd_arm.go +++ b/src/syscall/zerrors_openbsd_arm.go @@ -1,11 +1,9 @@ // mkerrors.sh // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- _const.go -//go:build arm && openbsd - package syscall const ( diff --git a/src/syscall/zerrors_solaris_amd64.go b/src/syscall/zerrors_solaris_amd64.go index b2c81d9a51f97a..094a78dcc515ad 100644 --- a/src/syscall/zerrors_solaris_amd64.go +++ b/src/syscall/zerrors_solaris_amd64.go @@ -1,11 +1,9 @@ // mkerrors.sh -m64 // Code generated by the command above; DO NOT EDIT. -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -m64 _const.go -//go:build amd64 && solaris - package syscall const ( diff --git a/src/syscall/zsyscall_darwin_amd64.go b/src/syscall/zsyscall_darwin_amd64.go index 8812fb12cd19c6..fe4a264ed25c66 100644 --- a/src/syscall/zsyscall_darwin_amd64.go +++ b/src/syscall/zsyscall_darwin_amd64.go @@ -2011,7 +2011,6 @@ func libc_fstatat64_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -//go:nosplit func ptrace(request int, pid int, addr uintptr, data uintptr) (err error) { if runtime.GOOS == "ios" { panic("unimplemented") diff --git a/src/syscall/zsyscall_darwin_arm64.go b/src/syscall/zsyscall_darwin_arm64.go index 22b096349d47b9..8fd7392d5b50f2 100644 --- a/src/syscall/zsyscall_darwin_arm64.go +++ b/src/syscall/zsyscall_darwin_arm64.go @@ -2011,7 +2011,6 @@ func libc_fstatat_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -//go:nosplit func ptrace(request int, pid int, addr uintptr, data uintptr) (err error) { if runtime.GOOS == "ios" { panic("unimplemented") diff --git a/src/syscall/zsyscall_linux_386.go b/src/syscall/zsyscall_linux_386.go index 661cfe7bed6f8d..762fb0078b254a 100644 --- a/src/syscall/zsyscall_linux_386.go +++ b/src/syscall/zsyscall_linux_386.go @@ -1395,26 +1395,6 @@ func EpollWait(epfd int, events []EpollEvent, msec int) (n int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func setrlimit1(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func futimesat(dirfd int, path string, times *[2]Timeval) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/src/syscall/zsyscall_linux_amd64.go b/src/syscall/zsyscall_linux_amd64.go index 9d05781893c255..1249ff66809abe 100644 --- a/src/syscall/zsyscall_linux_amd64.go +++ b/src/syscall/zsyscall_linux_amd64.go @@ -1160,16 +1160,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1336,16 +1326,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/src/syscall/zsyscall_linux_arm.go b/src/syscall/zsyscall_linux_arm.go index a386120f7833e7..c359fe531eec67 100644 --- a/src/syscall/zsyscall_linux_arm.go +++ b/src/syscall/zsyscall_linux_arm.go @@ -1,4 +1,4 @@ -// mksyscall.pl -l32 -arm -tags linux,arm syscall_linux.go syscall_linux_arm.go syscall_linux_accept.go +// mksyscall.pl -l32 -arm -tags linux,arm syscall_linux.go syscall_linux_arm.go // Code generated by the command above; DO NOT EDIT. //go:build linux && arm @@ -1595,34 +1595,3 @@ func EpollWait(epfd int, events []EpollEvent, msec int) (n int, err error) { } return } - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func getrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func setrlimit1(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func accept(s int, rsa *RawSockaddrAny, addrlen *_Socklen) (fd int, err error) { - r0, _, e1 := Syscall(SYS_ACCEPT, uintptr(s), uintptr(unsafe.Pointer(rsa)), uintptr(unsafe.Pointer(addrlen))) - fd = int(r0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/src/syscall/zsyscall_linux_arm64.go b/src/syscall/zsyscall_linux_arm64.go index a1c91071f1253d..6625cddb4bb245 100644 --- a/src/syscall/zsyscall_linux_arm64.go +++ b/src/syscall/zsyscall_linux_arm64.go @@ -1182,16 +1182,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1306,16 +1296,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit1(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/src/syscall/zsyscall_linux_mips.go b/src/syscall/zsyscall_linux_mips.go index 7e216b04ea6f1e..0bd4a806f569c0 100644 --- a/src/syscall/zsyscall_linux_mips.go +++ b/src/syscall/zsyscall_linux_mips.go @@ -1670,23 +1670,3 @@ func mmap2(addr uintptr, length uintptr, prot int, flags int, fd int, pageOffset } return } - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func getrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func setrlimit1(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/src/syscall/zsyscall_linux_mips64.go b/src/syscall/zsyscall_linux_mips64.go index 8c894b1088cb48..449088c815504a 100644 --- a/src/syscall/zsyscall_linux_mips64.go +++ b/src/syscall/zsyscall_linux_mips64.go @@ -1116,21 +1116,6 @@ func Fstatfs(fd int, buf *Statfs_t) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func fstatat(dirfd int, path string, stat *Stat_t, flags int) (err error) { - var _p0 *byte - _p0, err = BytePtrFromString(path) - if err != nil { - return - } - _, _, e1 := Syscall6(SYS_NEWFSTATAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), uintptr(flags), 0, 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Ftruncate(fd int, length int64) (err error) { _, _, e1 := Syscall(SYS_FTRUNCATE, uintptr(fd), uintptr(length), 0) if e1 != 0 { @@ -1165,16 +1150,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1325,16 +1300,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { @@ -1658,6 +1623,21 @@ func utimes(path string, times *[2]Timeval) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func fstatatInternal(dirfd int, path string, stat *stat_t, flags int) (err error) { + var _p0 *byte + _p0, err = BytePtrFromString(path) + if err != nil { + return + } + _, _, e1 := Syscall6(SYS_NEWFSTATAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), uintptr(flags), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func fstat(fd int, st *stat_t) (err error) { _, _, e1 := Syscall(SYS_FSTAT, uintptr(fd), uintptr(unsafe.Pointer(st)), 0) if e1 != 0 { diff --git a/src/syscall/zsyscall_linux_mips64le.go b/src/syscall/zsyscall_linux_mips64le.go index 812a6ba8e69af1..048298a39c910d 100644 --- a/src/syscall/zsyscall_linux_mips64le.go +++ b/src/syscall/zsyscall_linux_mips64le.go @@ -1116,21 +1116,6 @@ func Fstatfs(fd int, buf *Statfs_t) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func fstatat(dirfd int, path string, stat *Stat_t, flags int) (err error) { - var _p0 *byte - _p0, err = BytePtrFromString(path) - if err != nil { - return - } - _, _, e1 := Syscall6(SYS_NEWFSTATAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), uintptr(flags), 0, 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Ftruncate(fd int, length int64) (err error) { _, _, e1 := Syscall(SYS_FTRUNCATE, uintptr(fd), uintptr(length), 0) if e1 != 0 { @@ -1165,16 +1150,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1325,16 +1300,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { @@ -1658,6 +1623,21 @@ func utimes(path string, times *[2]Timeval) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func fstatatInternal(dirfd int, path string, stat *stat_t, flags int) (err error) { + var _p0 *byte + _p0, err = BytePtrFromString(path) + if err != nil { + return + } + _, _, e1 := Syscall6(SYS_NEWFSTATAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), uintptr(flags), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func fstat(fd int, st *stat_t) (err error) { _, _, e1 := Syscall(SYS_FSTAT, uintptr(fd), uintptr(unsafe.Pointer(st)), 0) if e1 != 0 { diff --git a/src/syscall/zsyscall_linux_mipsle.go b/src/syscall/zsyscall_linux_mipsle.go index d32a8afa4f6a33..7f880e53d59343 100644 --- a/src/syscall/zsyscall_linux_mipsle.go +++ b/src/syscall/zsyscall_linux_mipsle.go @@ -1670,23 +1670,3 @@ func mmap2(addr uintptr, length uintptr, prot int, flags int, fd int, pageOffset } return } - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func getrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func setrlimit1(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/src/syscall/zsyscall_linux_ppc64.go b/src/syscall/zsyscall_linux_ppc64.go index c321267b9ba431..c4ca96f2f9860d 100644 --- a/src/syscall/zsyscall_linux_ppc64.go +++ b/src/syscall/zsyscall_linux_ppc64.go @@ -1192,16 +1192,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_UGETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1398,16 +1388,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/src/syscall/zsyscall_linux_ppc64le.go b/src/syscall/zsyscall_linux_ppc64le.go index 40475d76bfe85b..a08471b4e0ea85 100644 --- a/src/syscall/zsyscall_linux_ppc64le.go +++ b/src/syscall/zsyscall_linux_ppc64le.go @@ -1192,16 +1192,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_UGETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1398,16 +1388,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/src/syscall/zsyscall_linux_riscv64.go b/src/syscall/zsyscall_linux_riscv64.go index dc74acfc880d43..4098b790042d52 100644 --- a/src/syscall/zsyscall_linux_riscv64.go +++ b/src/syscall/zsyscall_linux_riscv64.go @@ -1182,16 +1182,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1306,16 +1296,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/src/syscall/zsyscall_linux_s390x.go b/src/syscall/zsyscall_linux_s390x.go index cc189d9ea7748e..81979169ffce7b 100644 --- a/src/syscall/zsyscall_linux_s390x.go +++ b/src/syscall/zsyscall_linux_s390x.go @@ -1192,16 +1192,6 @@ func Getgid() (gid int) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Getrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Getuid() (uid int) { r0, _ := rawSyscallNoError(SYS_GETUID, 0, 0, 0) uid = int(r0) @@ -1368,16 +1358,6 @@ func Setfsuid(uid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) { r0, _, e1 := Syscall6(SYS_SPLICE, uintptr(rfd), uintptr(unsafe.Pointer(roff)), uintptr(wfd), uintptr(unsafe.Pointer(woff)), uintptr(len), uintptr(flags)) n = int64(r0) diff --git a/src/syscall/zsyscall_openbsd_386.go b/src/syscall/zsyscall_openbsd_386.go index d2bd3ea0125a12..d914e19da0d8d3 100644 --- a/src/syscall/zsyscall_openbsd_386.go +++ b/src/syscall/zsyscall_openbsd_386.go @@ -1839,7 +1839,6 @@ func libc_exit_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -//go:nosplit func ptrace(request int, pid int, addr uintptr, data uintptr) (err error) { _, _, e1 := syscall6(abi.FuncPCABI0(libc_ptrace_trampoline), uintptr(request), uintptr(pid), uintptr(addr), uintptr(data), 0, 0) if e1 != 0 { @@ -1854,26 +1853,6 @@ func libc_ptrace_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getentropy(p []byte) (err error) { - var _p0 unsafe.Pointer - if len(p) > 0 { - _p0 = unsafe.Pointer(&p[0]) - } else { - _p0 = unsafe.Pointer(&_zero) - } - _, _, e1 := rawSyscall(abi.FuncPCABI0(libc_getentropy_trampoline), uintptr(_p0), uintptr(len(p)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -func libc_getentropy_trampoline() - -//go:cgo_import_dynamic libc_getentropy getentropy "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func fstatat(fd int, path string, stat *Stat_t, flags int) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/src/syscall/zsyscall_openbsd_386.s b/src/syscall/zsyscall_openbsd_386.s index 9a820e9f3eb911..f86ac2c0779e1f 100644 --- a/src/syscall/zsyscall_openbsd_386.s +++ b/src/syscall/zsyscall_openbsd_386.s @@ -227,8 +227,6 @@ TEXT ·libc_exit_trampoline(SB),NOSPLIT,$0-0 JMP libc_exit(SB) TEXT ·libc_ptrace_trampoline(SB),NOSPLIT,$0-0 JMP libc_ptrace(SB) -TEXT ·libc_getentropy_trampoline(SB),NOSPLIT,$0-0 - JMP libc_getentropy(SB) TEXT ·libc_fstatat_trampoline(SB),NOSPLIT,$0-0 JMP libc_fstatat(SB) TEXT ·libc_unlinkat_trampoline(SB),NOSPLIT,$0-0 diff --git a/src/syscall/zsyscall_openbsd_amd64.go b/src/syscall/zsyscall_openbsd_amd64.go index 170a74b457ba0a..5efe4cd8686f22 100644 --- a/src/syscall/zsyscall_openbsd_amd64.go +++ b/src/syscall/zsyscall_openbsd_amd64.go @@ -1839,7 +1839,6 @@ func libc_exit_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -//go:nosplit func ptrace(request int, pid int, addr uintptr, data uintptr) (err error) { _, _, e1 := syscall6(abi.FuncPCABI0(libc_ptrace_trampoline), uintptr(request), uintptr(pid), uintptr(addr), uintptr(data), 0, 0) if e1 != 0 { @@ -1854,26 +1853,6 @@ func libc_ptrace_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getentropy(p []byte) (err error) { - var _p0 unsafe.Pointer - if len(p) > 0 { - _p0 = unsafe.Pointer(&p[0]) - } else { - _p0 = unsafe.Pointer(&_zero) - } - _, _, e1 := rawSyscall(abi.FuncPCABI0(libc_getentropy_trampoline), uintptr(_p0), uintptr(len(p)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -func libc_getentropy_trampoline() - -//go:cgo_import_dynamic libc_getentropy getentropy "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func fstatat(fd int, path string, stat *Stat_t, flags int) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/src/syscall/zsyscall_openbsd_amd64.s b/src/syscall/zsyscall_openbsd_amd64.s index 9b70dc096e12d6..4ec62202814c3e 100644 --- a/src/syscall/zsyscall_openbsd_amd64.s +++ b/src/syscall/zsyscall_openbsd_amd64.s @@ -227,8 +227,6 @@ TEXT ·libc_exit_trampoline(SB),NOSPLIT,$0-0 JMP libc_exit(SB) TEXT ·libc_ptrace_trampoline(SB),NOSPLIT,$0-0 JMP libc_ptrace(SB) -TEXT ·libc_getentropy_trampoline(SB),NOSPLIT,$0-0 - JMP libc_getentropy(SB) TEXT ·libc_fstatat_trampoline(SB),NOSPLIT,$0-0 JMP libc_fstatat(SB) TEXT ·libc_unlinkat_trampoline(SB),NOSPLIT,$0-0 diff --git a/src/syscall/zsyscall_openbsd_arm.go b/src/syscall/zsyscall_openbsd_arm.go index e75bd0b443b902..db8ea482ef5983 100644 --- a/src/syscall/zsyscall_openbsd_arm.go +++ b/src/syscall/zsyscall_openbsd_arm.go @@ -1839,7 +1839,6 @@ func libc_exit_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -//go:nosplit func ptrace(request int, pid int, addr uintptr, data uintptr) (err error) { _, _, e1 := syscall6(abi.FuncPCABI0(libc_ptrace_trampoline), uintptr(request), uintptr(pid), uintptr(addr), uintptr(data), 0, 0) if e1 != 0 { @@ -1854,26 +1853,6 @@ func libc_ptrace_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getentropy(p []byte) (err error) { - var _p0 unsafe.Pointer - if len(p) > 0 { - _p0 = unsafe.Pointer(&p[0]) - } else { - _p0 = unsafe.Pointer(&_zero) - } - _, _, e1 := rawSyscall(abi.FuncPCABI0(libc_getentropy_trampoline), uintptr(_p0), uintptr(len(p)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -func libc_getentropy_trampoline() - -//go:cgo_import_dynamic libc_getentropy getentropy "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func fstatat(fd int, path string, stat *Stat_t, flags int) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/src/syscall/zsyscall_openbsd_arm.s b/src/syscall/zsyscall_openbsd_arm.s index 0333377b8b7155..75251d0305a43e 100644 --- a/src/syscall/zsyscall_openbsd_arm.s +++ b/src/syscall/zsyscall_openbsd_arm.s @@ -227,8 +227,6 @@ TEXT ·libc_exit_trampoline(SB),NOSPLIT,$0-0 JMP libc_exit(SB) TEXT ·libc_ptrace_trampoline(SB),NOSPLIT,$0-0 JMP libc_ptrace(SB) -TEXT ·libc_getentropy_trampoline(SB),NOSPLIT,$0-0 - JMP libc_getentropy(SB) TEXT ·libc_fstatat_trampoline(SB),NOSPLIT,$0-0 JMP libc_fstatat(SB) TEXT ·libc_unlinkat_trampoline(SB),NOSPLIT,$0-0 diff --git a/src/syscall/zsyscall_openbsd_arm64.go b/src/syscall/zsyscall_openbsd_arm64.go index bc027b44755f22..673791c8241471 100644 --- a/src/syscall/zsyscall_openbsd_arm64.go +++ b/src/syscall/zsyscall_openbsd_arm64.go @@ -1839,7 +1839,6 @@ func libc_exit_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -//go:nosplit func ptrace(request int, pid int, addr uintptr, data uintptr) (err error) { _, _, e1 := syscall6(abi.FuncPCABI0(libc_ptrace_trampoline), uintptr(request), uintptr(pid), uintptr(addr), uintptr(data), 0, 0) if e1 != 0 { @@ -1854,26 +1853,6 @@ func libc_ptrace_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getentropy(p []byte) (err error) { - var _p0 unsafe.Pointer - if len(p) > 0 { - _p0 = unsafe.Pointer(&p[0]) - } else { - _p0 = unsafe.Pointer(&_zero) - } - _, _, e1 := rawSyscall(abi.FuncPCABI0(libc_getentropy_trampoline), uintptr(_p0), uintptr(len(p)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -func libc_getentropy_trampoline() - -//go:cgo_import_dynamic libc_getentropy getentropy "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func fstatat(fd int, path string, stat *Stat_t, flags int) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/src/syscall/zsyscall_openbsd_arm64.s b/src/syscall/zsyscall_openbsd_arm64.s index 654e6c69a35316..deea88ec7c4149 100644 --- a/src/syscall/zsyscall_openbsd_arm64.s +++ b/src/syscall/zsyscall_openbsd_arm64.s @@ -227,8 +227,6 @@ TEXT ·libc_exit_trampoline(SB),NOSPLIT,$0-0 JMP libc_exit(SB) TEXT ·libc_ptrace_trampoline(SB),NOSPLIT,$0-0 JMP libc_ptrace(SB) -TEXT ·libc_getentropy_trampoline(SB),NOSPLIT,$0-0 - JMP libc_getentropy(SB) TEXT ·libc_fstatat_trampoline(SB),NOSPLIT,$0-0 JMP libc_fstatat(SB) TEXT ·libc_unlinkat_trampoline(SB),NOSPLIT,$0-0 diff --git a/src/syscall/zsyscall_openbsd_ppc64.go b/src/syscall/zsyscall_openbsd_ppc64.go index 6808092a5a27b0..7e0dc88a4c155f 100644 --- a/src/syscall/zsyscall_openbsd_ppc64.go +++ b/src/syscall/zsyscall_openbsd_ppc64.go @@ -1839,7 +1839,6 @@ func libc_exit_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -//go:nosplit func ptrace(request int, pid int, addr uintptr, data uintptr) (err error) { _, _, e1 := syscall6(abi.FuncPCABI0(libc_ptrace_trampoline), uintptr(request), uintptr(pid), uintptr(addr), uintptr(data), 0, 0) if e1 != 0 { @@ -1854,26 +1853,6 @@ func libc_ptrace_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getentropy(p []byte) (err error) { - var _p0 unsafe.Pointer - if len(p) > 0 { - _p0 = unsafe.Pointer(&p[0]) - } else { - _p0 = unsafe.Pointer(&_zero) - } - _, _, e1 := rawSyscall(abi.FuncPCABI0(libc_getentropy_trampoline), uintptr(_p0), uintptr(len(p)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -func libc_getentropy_trampoline() - -//go:cgo_import_dynamic libc_getentropy getentropy "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func fstatat(fd int, path string, stat *Stat_t, flags int) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/src/syscall/zsyscall_openbsd_ppc64.s b/src/syscall/zsyscall_openbsd_ppc64.s index 86a5745c0ae63e..cc4eb952eb62c1 100644 --- a/src/syscall/zsyscall_openbsd_ppc64.s +++ b/src/syscall/zsyscall_openbsd_ppc64.s @@ -340,9 +340,6 @@ TEXT ·libc_exit_trampoline(SB),NOSPLIT,$0-0 TEXT ·libc_ptrace_trampoline(SB),NOSPLIT,$0-0 CALL libc_ptrace(SB) RET -TEXT ·libc_getentropy_trampoline(SB),NOSPLIT,$0-0 - CALL libc_getentropy(SB) - RET TEXT ·libc_fstatat_trampoline(SB),NOSPLIT,$0-0 CALL libc_fstatat(SB) RET diff --git a/src/syscall/zsyscall_openbsd_riscv64.go b/src/syscall/zsyscall_openbsd_riscv64.go index 2979ff78c25b70..7b5a89ceac150d 100644 --- a/src/syscall/zsyscall_openbsd_riscv64.go +++ b/src/syscall/zsyscall_openbsd_riscv64.go @@ -1839,7 +1839,6 @@ func libc_exit_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -//go:nosplit func ptrace(request int, pid int, addr uintptr, data uintptr) (err error) { _, _, e1 := syscall6(abi.FuncPCABI0(libc_ptrace_trampoline), uintptr(request), uintptr(pid), uintptr(addr), uintptr(data), 0, 0) if e1 != 0 { @@ -1854,26 +1853,6 @@ func libc_ptrace_trampoline() // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func getentropy(p []byte) (err error) { - var _p0 unsafe.Pointer - if len(p) > 0 { - _p0 = unsafe.Pointer(&p[0]) - } else { - _p0 = unsafe.Pointer(&_zero) - } - _, _, e1 := rawSyscall(abi.FuncPCABI0(libc_getentropy_trampoline), uintptr(_p0), uintptr(len(p)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -func libc_getentropy_trampoline() - -//go:cgo_import_dynamic libc_getentropy getentropy "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func fstatat(fd int, path string, stat *Stat_t, flags int) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/src/syscall/zsyscall_openbsd_riscv64.s b/src/syscall/zsyscall_openbsd_riscv64.s index c8728190e5554c..14b7b23222df08 100644 --- a/src/syscall/zsyscall_openbsd_riscv64.s +++ b/src/syscall/zsyscall_openbsd_riscv64.s @@ -227,8 +227,6 @@ TEXT ·libc_exit_trampoline(SB),NOSPLIT,$0-0 JMP libc_exit(SB) TEXT ·libc_ptrace_trampoline(SB),NOSPLIT,$0-0 JMP libc_ptrace(SB) -TEXT ·libc_getentropy_trampoline(SB),NOSPLIT,$0-0 - JMP libc_getentropy(SB) TEXT ·libc_fstatat_trampoline(SB),NOSPLIT,$0-0 JMP libc_fstatat(SB) TEXT ·libc_unlinkat_trampoline(SB),NOSPLIT,$0-0 diff --git a/src/syscall/zsyscall_windows.go b/src/syscall/zsyscall_windows.go index d8d8594a556111..70ac19a9d6dc08 100644 --- a/src/syscall/zsyscall_windows.go +++ b/src/syscall/zsyscall_windows.go @@ -134,6 +134,7 @@ var ( procGetVersion = modkernel32.NewProc("GetVersion") procInitializeProcThreadAttributeList = modkernel32.NewProc("InitializeProcThreadAttributeList") procLoadLibraryW = modkernel32.NewProc("LoadLibraryW") + procLocalAlloc = modkernel32.NewProc("LocalAlloc") procLocalFree = modkernel32.NewProc("LocalFree") procMapViewOfFile = modkernel32.NewProc("MapViewOfFile") procMoveFileW = modkernel32.NewProc("MoveFileW") @@ -150,6 +151,7 @@ var ( procSetEnvironmentVariableW = modkernel32.NewProc("SetEnvironmentVariableW") procSetFileAttributesW = modkernel32.NewProc("SetFileAttributesW") procSetFileCompletionNotificationModes = modkernel32.NewProc("SetFileCompletionNotificationModes") + procSetFileInformationByHandle = modkernel32.NewProc("SetFileInformationByHandle") procSetFilePointer = modkernel32.NewProc("SetFilePointer") procSetFileTime = modkernel32.NewProc("SetFileTime") procSetHandleInformation = modkernel32.NewProc("SetHandleInformation") @@ -198,7 +200,7 @@ var ( ) func ConvertSidToStringSid(sid *SID, stringSid **uint16) (err error) { - r1, _, e1 := Syscall(procConvertSidToStringSidW.Addr(), 2, uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(stringSid)), 0) + r1, _, e1 := SyscallN(procConvertSidToStringSidW.Addr(), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(stringSid))) if r1 == 0 { err = errnoErr(e1) } @@ -206,7 +208,7 @@ func ConvertSidToStringSid(sid *SID, stringSid **uint16) (err error) { } func ConvertStringSidToSid(stringSid *uint16, sid **SID) (err error) { - r1, _, e1 := Syscall(procConvertStringSidToSidW.Addr(), 2, uintptr(unsafe.Pointer(stringSid)), uintptr(unsafe.Pointer(sid)), 0) + r1, _, e1 := SyscallN(procConvertStringSidToSidW.Addr(), uintptr(unsafe.Pointer(stringSid)), uintptr(unsafe.Pointer(sid))) if r1 == 0 { err = errnoErr(e1) } @@ -214,7 +216,7 @@ func ConvertStringSidToSid(stringSid *uint16, sid **SID) (err error) { } func CopySid(destSidLen uint32, destSid *SID, srcSid *SID) (err error) { - r1, _, e1 := Syscall(procCopySid.Addr(), 3, uintptr(destSidLen), uintptr(unsafe.Pointer(destSid)), uintptr(unsafe.Pointer(srcSid))) + r1, _, e1 := SyscallN(procCopySid.Addr(), uintptr(destSidLen), uintptr(unsafe.Pointer(destSid)), uintptr(unsafe.Pointer(srcSid))) if r1 == 0 { err = errnoErr(e1) } @@ -226,7 +228,7 @@ func CreateProcessAsUser(token Token, appName *uint16, commandLine *uint16, proc if inheritHandles { _p0 = 1 } - r1, _, e1 := Syscall12(procCreateProcessAsUserW.Addr(), 11, uintptr(token), uintptr(unsafe.Pointer(appName)), uintptr(unsafe.Pointer(commandLine)), uintptr(unsafe.Pointer(procSecurity)), uintptr(unsafe.Pointer(threadSecurity)), uintptr(_p0), uintptr(creationFlags), uintptr(unsafe.Pointer(env)), uintptr(unsafe.Pointer(currentDir)), uintptr(unsafe.Pointer(startupInfo)), uintptr(unsafe.Pointer(outProcInfo)), 0) + r1, _, e1 := SyscallN(procCreateProcessAsUserW.Addr(), uintptr(token), uintptr(unsafe.Pointer(appName)), uintptr(unsafe.Pointer(commandLine)), uintptr(unsafe.Pointer(procSecurity)), uintptr(unsafe.Pointer(threadSecurity)), uintptr(_p0), uintptr(creationFlags), uintptr(unsafe.Pointer(env)), uintptr(unsafe.Pointer(currentDir)), uintptr(unsafe.Pointer(startupInfo)), uintptr(unsafe.Pointer(outProcInfo))) if r1 == 0 { err = errnoErr(e1) } @@ -234,7 +236,7 @@ func CreateProcessAsUser(token Token, appName *uint16, commandLine *uint16, proc } func CryptAcquireContext(provhandle *Handle, container *uint16, provider *uint16, provtype uint32, flags uint32) (err error) { - r1, _, e1 := Syscall6(procCryptAcquireContextW.Addr(), 5, uintptr(unsafe.Pointer(provhandle)), uintptr(unsafe.Pointer(container)), uintptr(unsafe.Pointer(provider)), uintptr(provtype), uintptr(flags), 0) + r1, _, e1 := SyscallN(procCryptAcquireContextW.Addr(), uintptr(unsafe.Pointer(provhandle)), uintptr(unsafe.Pointer(container)), uintptr(unsafe.Pointer(provider)), uintptr(provtype), uintptr(flags)) if r1 == 0 { err = errnoErr(e1) } @@ -242,7 +244,7 @@ func CryptAcquireContext(provhandle *Handle, container *uint16, provider *uint16 } func CryptGenRandom(provhandle Handle, buflen uint32, buf *byte) (err error) { - r1, _, e1 := Syscall(procCryptGenRandom.Addr(), 3, uintptr(provhandle), uintptr(buflen), uintptr(unsafe.Pointer(buf))) + r1, _, e1 := SyscallN(procCryptGenRandom.Addr(), uintptr(provhandle), uintptr(buflen), uintptr(unsafe.Pointer(buf))) if r1 == 0 { err = errnoErr(e1) } @@ -250,7 +252,7 @@ func CryptGenRandom(provhandle Handle, buflen uint32, buf *byte) (err error) { } func CryptReleaseContext(provhandle Handle, flags uint32) (err error) { - r1, _, e1 := Syscall(procCryptReleaseContext.Addr(), 2, uintptr(provhandle), uintptr(flags), 0) + r1, _, e1 := SyscallN(procCryptReleaseContext.Addr(), uintptr(provhandle), uintptr(flags)) if r1 == 0 { err = errnoErr(e1) } @@ -258,13 +260,13 @@ func CryptReleaseContext(provhandle Handle, flags uint32) (err error) { } func GetLengthSid(sid *SID) (len uint32) { - r0, _, _ := Syscall(procGetLengthSid.Addr(), 1, uintptr(unsafe.Pointer(sid)), 0, 0) + r0, _, _ := SyscallN(procGetLengthSid.Addr(), uintptr(unsafe.Pointer(sid))) len = uint32(r0) return } func GetTokenInformation(t Token, infoClass uint32, info *byte, infoLen uint32, returnedLen *uint32) (err error) { - r1, _, e1 := Syscall6(procGetTokenInformation.Addr(), 5, uintptr(t), uintptr(infoClass), uintptr(unsafe.Pointer(info)), uintptr(infoLen), uintptr(unsafe.Pointer(returnedLen)), 0) + r1, _, e1 := SyscallN(procGetTokenInformation.Addr(), uintptr(t), uintptr(infoClass), uintptr(unsafe.Pointer(info)), uintptr(infoLen), uintptr(unsafe.Pointer(returnedLen))) if r1 == 0 { err = errnoErr(e1) } @@ -272,7 +274,7 @@ func GetTokenInformation(t Token, infoClass uint32, info *byte, infoLen uint32, } func LookupAccountName(systemName *uint16, accountName *uint16, sid *SID, sidLen *uint32, refdDomainName *uint16, refdDomainNameLen *uint32, use *uint32) (err error) { - r1, _, e1 := Syscall9(procLookupAccountNameW.Addr(), 7, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(accountName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(sidLen)), uintptr(unsafe.Pointer(refdDomainName)), uintptr(unsafe.Pointer(refdDomainNameLen)), uintptr(unsafe.Pointer(use)), 0, 0) + r1, _, e1 := SyscallN(procLookupAccountNameW.Addr(), uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(accountName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(sidLen)), uintptr(unsafe.Pointer(refdDomainName)), uintptr(unsafe.Pointer(refdDomainNameLen)), uintptr(unsafe.Pointer(use))) if r1 == 0 { err = errnoErr(e1) } @@ -280,7 +282,7 @@ func LookupAccountName(systemName *uint16, accountName *uint16, sid *SID, sidLen } func LookupAccountSid(systemName *uint16, sid *SID, name *uint16, nameLen *uint32, refdDomainName *uint16, refdDomainNameLen *uint32, use *uint32) (err error) { - r1, _, e1 := Syscall9(procLookupAccountSidW.Addr(), 7, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(nameLen)), uintptr(unsafe.Pointer(refdDomainName)), uintptr(unsafe.Pointer(refdDomainNameLen)), uintptr(unsafe.Pointer(use)), 0, 0) + r1, _, e1 := SyscallN(procLookupAccountSidW.Addr(), uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(nameLen)), uintptr(unsafe.Pointer(refdDomainName)), uintptr(unsafe.Pointer(refdDomainNameLen)), uintptr(unsafe.Pointer(use))) if r1 == 0 { err = errnoErr(e1) } @@ -288,7 +290,7 @@ func LookupAccountSid(systemName *uint16, sid *SID, name *uint16, nameLen *uint3 } func OpenProcessToken(h Handle, access uint32, token *Token) (err error) { - r1, _, e1 := Syscall(procOpenProcessToken.Addr(), 3, uintptr(h), uintptr(access), uintptr(unsafe.Pointer(token))) + r1, _, e1 := SyscallN(procOpenProcessToken.Addr(), uintptr(h), uintptr(access), uintptr(unsafe.Pointer(token))) if r1 == 0 { err = errnoErr(e1) } @@ -296,7 +298,7 @@ func OpenProcessToken(h Handle, access uint32, token *Token) (err error) { } func RegCloseKey(key Handle) (regerrno error) { - r0, _, _ := Syscall(procRegCloseKey.Addr(), 1, uintptr(key), 0, 0) + r0, _, _ := SyscallN(procRegCloseKey.Addr(), uintptr(key)) if r0 != 0 { regerrno = Errno(r0) } @@ -304,7 +306,7 @@ func RegCloseKey(key Handle) (regerrno error) { } func regEnumKeyEx(key Handle, index uint32, name *uint16, nameLen *uint32, reserved *uint32, class *uint16, classLen *uint32, lastWriteTime *Filetime) (regerrno error) { - r0, _, _ := Syscall9(procRegEnumKeyExW.Addr(), 8, uintptr(key), uintptr(index), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(nameLen)), uintptr(unsafe.Pointer(reserved)), uintptr(unsafe.Pointer(class)), uintptr(unsafe.Pointer(classLen)), uintptr(unsafe.Pointer(lastWriteTime)), 0) + r0, _, _ := SyscallN(procRegEnumKeyExW.Addr(), uintptr(key), uintptr(index), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(nameLen)), uintptr(unsafe.Pointer(reserved)), uintptr(unsafe.Pointer(class)), uintptr(unsafe.Pointer(classLen)), uintptr(unsafe.Pointer(lastWriteTime))) if r0 != 0 { regerrno = Errno(r0) } @@ -312,7 +314,7 @@ func regEnumKeyEx(key Handle, index uint32, name *uint16, nameLen *uint32, reser } func RegOpenKeyEx(key Handle, subkey *uint16, options uint32, desiredAccess uint32, result *Handle) (regerrno error) { - r0, _, _ := Syscall6(procRegOpenKeyExW.Addr(), 5, uintptr(key), uintptr(unsafe.Pointer(subkey)), uintptr(options), uintptr(desiredAccess), uintptr(unsafe.Pointer(result)), 0) + r0, _, _ := SyscallN(procRegOpenKeyExW.Addr(), uintptr(key), uintptr(unsafe.Pointer(subkey)), uintptr(options), uintptr(desiredAccess), uintptr(unsafe.Pointer(result))) if r0 != 0 { regerrno = Errno(r0) } @@ -320,7 +322,7 @@ func RegOpenKeyEx(key Handle, subkey *uint16, options uint32, desiredAccess uint } func RegQueryInfoKey(key Handle, class *uint16, classLen *uint32, reserved *uint32, subkeysLen *uint32, maxSubkeyLen *uint32, maxClassLen *uint32, valuesLen *uint32, maxValueNameLen *uint32, maxValueLen *uint32, saLen *uint32, lastWriteTime *Filetime) (regerrno error) { - r0, _, _ := Syscall12(procRegQueryInfoKeyW.Addr(), 12, uintptr(key), uintptr(unsafe.Pointer(class)), uintptr(unsafe.Pointer(classLen)), uintptr(unsafe.Pointer(reserved)), uintptr(unsafe.Pointer(subkeysLen)), uintptr(unsafe.Pointer(maxSubkeyLen)), uintptr(unsafe.Pointer(maxClassLen)), uintptr(unsafe.Pointer(valuesLen)), uintptr(unsafe.Pointer(maxValueNameLen)), uintptr(unsafe.Pointer(maxValueLen)), uintptr(unsafe.Pointer(saLen)), uintptr(unsafe.Pointer(lastWriteTime))) + r0, _, _ := SyscallN(procRegQueryInfoKeyW.Addr(), uintptr(key), uintptr(unsafe.Pointer(class)), uintptr(unsafe.Pointer(classLen)), uintptr(unsafe.Pointer(reserved)), uintptr(unsafe.Pointer(subkeysLen)), uintptr(unsafe.Pointer(maxSubkeyLen)), uintptr(unsafe.Pointer(maxClassLen)), uintptr(unsafe.Pointer(valuesLen)), uintptr(unsafe.Pointer(maxValueNameLen)), uintptr(unsafe.Pointer(maxValueLen)), uintptr(unsafe.Pointer(saLen)), uintptr(unsafe.Pointer(lastWriteTime))) if r0 != 0 { regerrno = Errno(r0) } @@ -328,7 +330,7 @@ func RegQueryInfoKey(key Handle, class *uint16, classLen *uint32, reserved *uint } func RegQueryValueEx(key Handle, name *uint16, reserved *uint32, valtype *uint32, buf *byte, buflen *uint32) (regerrno error) { - r0, _, _ := Syscall6(procRegQueryValueExW.Addr(), 6, uintptr(key), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(reserved)), uintptr(unsafe.Pointer(valtype)), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(buflen))) + r0, _, _ := SyscallN(procRegQueryValueExW.Addr(), uintptr(key), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(reserved)), uintptr(unsafe.Pointer(valtype)), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(buflen))) if r0 != 0 { regerrno = Errno(r0) } @@ -336,7 +338,7 @@ func RegQueryValueEx(key Handle, name *uint16, reserved *uint32, valtype *uint32 } func CertAddCertificateContextToStore(store Handle, certContext *CertContext, addDisposition uint32, storeContext **CertContext) (err error) { - r1, _, e1 := Syscall6(procCertAddCertificateContextToStore.Addr(), 4, uintptr(store), uintptr(unsafe.Pointer(certContext)), uintptr(addDisposition), uintptr(unsafe.Pointer(storeContext)), 0, 0) + r1, _, e1 := SyscallN(procCertAddCertificateContextToStore.Addr(), uintptr(store), uintptr(unsafe.Pointer(certContext)), uintptr(addDisposition), uintptr(unsafe.Pointer(storeContext))) if r1 == 0 { err = errnoErr(e1) } @@ -344,7 +346,7 @@ func CertAddCertificateContextToStore(store Handle, certContext *CertContext, ad } func CertCloseStore(store Handle, flags uint32) (err error) { - r1, _, e1 := Syscall(procCertCloseStore.Addr(), 2, uintptr(store), uintptr(flags), 0) + r1, _, e1 := SyscallN(procCertCloseStore.Addr(), uintptr(store), uintptr(flags)) if r1 == 0 { err = errnoErr(e1) } @@ -352,7 +354,7 @@ func CertCloseStore(store Handle, flags uint32) (err error) { } func CertCreateCertificateContext(certEncodingType uint32, certEncoded *byte, encodedLen uint32) (context *CertContext, err error) { - r0, _, e1 := Syscall(procCertCreateCertificateContext.Addr(), 3, uintptr(certEncodingType), uintptr(unsafe.Pointer(certEncoded)), uintptr(encodedLen)) + r0, _, e1 := SyscallN(procCertCreateCertificateContext.Addr(), uintptr(certEncodingType), uintptr(unsafe.Pointer(certEncoded)), uintptr(encodedLen)) context = (*CertContext)(unsafe.Pointer(r0)) if context == nil { err = errnoErr(e1) @@ -361,7 +363,7 @@ func CertCreateCertificateContext(certEncodingType uint32, certEncoded *byte, en } func CertEnumCertificatesInStore(store Handle, prevContext *CertContext) (context *CertContext, err error) { - r0, _, e1 := Syscall(procCertEnumCertificatesInStore.Addr(), 2, uintptr(store), uintptr(unsafe.Pointer(prevContext)), 0) + r0, _, e1 := SyscallN(procCertEnumCertificatesInStore.Addr(), uintptr(store), uintptr(unsafe.Pointer(prevContext))) context = (*CertContext)(unsafe.Pointer(r0)) if context == nil { err = errnoErr(e1) @@ -370,12 +372,12 @@ func CertEnumCertificatesInStore(store Handle, prevContext *CertContext) (contex } func CertFreeCertificateChain(ctx *CertChainContext) { - Syscall(procCertFreeCertificateChain.Addr(), 1, uintptr(unsafe.Pointer(ctx)), 0, 0) + SyscallN(procCertFreeCertificateChain.Addr(), uintptr(unsafe.Pointer(ctx))) return } func CertFreeCertificateContext(ctx *CertContext) (err error) { - r1, _, e1 := Syscall(procCertFreeCertificateContext.Addr(), 1, uintptr(unsafe.Pointer(ctx)), 0, 0) + r1, _, e1 := SyscallN(procCertFreeCertificateContext.Addr(), uintptr(unsafe.Pointer(ctx))) if r1 == 0 { err = errnoErr(e1) } @@ -383,7 +385,7 @@ func CertFreeCertificateContext(ctx *CertContext) (err error) { } func CertGetCertificateChain(engine Handle, leaf *CertContext, time *Filetime, additionalStore Handle, para *CertChainPara, flags uint32, reserved uintptr, chainCtx **CertChainContext) (err error) { - r1, _, e1 := Syscall9(procCertGetCertificateChain.Addr(), 8, uintptr(engine), uintptr(unsafe.Pointer(leaf)), uintptr(unsafe.Pointer(time)), uintptr(additionalStore), uintptr(unsafe.Pointer(para)), uintptr(flags), uintptr(reserved), uintptr(unsafe.Pointer(chainCtx)), 0) + r1, _, e1 := SyscallN(procCertGetCertificateChain.Addr(), uintptr(engine), uintptr(unsafe.Pointer(leaf)), uintptr(unsafe.Pointer(time)), uintptr(additionalStore), uintptr(unsafe.Pointer(para)), uintptr(flags), uintptr(reserved), uintptr(unsafe.Pointer(chainCtx))) if r1 == 0 { err = errnoErr(e1) } @@ -391,7 +393,7 @@ func CertGetCertificateChain(engine Handle, leaf *CertContext, time *Filetime, a } func CertOpenStore(storeProvider uintptr, msgAndCertEncodingType uint32, cryptProv uintptr, flags uint32, para uintptr) (handle Handle, err error) { - r0, _, e1 := Syscall6(procCertOpenStore.Addr(), 5, uintptr(storeProvider), uintptr(msgAndCertEncodingType), uintptr(cryptProv), uintptr(flags), uintptr(para), 0) + r0, _, e1 := SyscallN(procCertOpenStore.Addr(), uintptr(storeProvider), uintptr(msgAndCertEncodingType), uintptr(cryptProv), uintptr(flags), uintptr(para)) handle = Handle(r0) if handle == 0 { err = errnoErr(e1) @@ -400,7 +402,7 @@ func CertOpenStore(storeProvider uintptr, msgAndCertEncodingType uint32, cryptPr } func CertOpenSystemStore(hprov Handle, name *uint16) (store Handle, err error) { - r0, _, e1 := Syscall(procCertOpenSystemStoreW.Addr(), 2, uintptr(hprov), uintptr(unsafe.Pointer(name)), 0) + r0, _, e1 := SyscallN(procCertOpenSystemStoreW.Addr(), uintptr(hprov), uintptr(unsafe.Pointer(name))) store = Handle(r0) if store == 0 { err = errnoErr(e1) @@ -409,7 +411,7 @@ func CertOpenSystemStore(hprov Handle, name *uint16) (store Handle, err error) { } func CertVerifyCertificateChainPolicy(policyOID uintptr, chain *CertChainContext, para *CertChainPolicyPara, status *CertChainPolicyStatus) (err error) { - r1, _, e1 := Syscall6(procCertVerifyCertificateChainPolicy.Addr(), 4, uintptr(policyOID), uintptr(unsafe.Pointer(chain)), uintptr(unsafe.Pointer(para)), uintptr(unsafe.Pointer(status)), 0, 0) + r1, _, e1 := SyscallN(procCertVerifyCertificateChainPolicy.Addr(), uintptr(policyOID), uintptr(unsafe.Pointer(chain)), uintptr(unsafe.Pointer(para)), uintptr(unsafe.Pointer(status))) if r1 == 0 { err = errnoErr(e1) } @@ -417,7 +419,7 @@ func CertVerifyCertificateChainPolicy(policyOID uintptr, chain *CertChainContext } func DnsNameCompare(name1 *uint16, name2 *uint16) (same bool) { - r0, _, _ := Syscall(procDnsNameCompare_W.Addr(), 2, uintptr(unsafe.Pointer(name1)), uintptr(unsafe.Pointer(name2)), 0) + r0, _, _ := SyscallN(procDnsNameCompare_W.Addr(), uintptr(unsafe.Pointer(name1)), uintptr(unsafe.Pointer(name2))) same = r0 != 0 return } @@ -432,7 +434,7 @@ func DnsQuery(name string, qtype uint16, options uint32, extra *byte, qrs **DNSR } func _DnsQuery(name *uint16, qtype uint16, options uint32, extra *byte, qrs **DNSRecord, pr *byte) (status error) { - r0, _, _ := Syscall6(procDnsQuery_W.Addr(), 6, uintptr(unsafe.Pointer(name)), uintptr(qtype), uintptr(options), uintptr(unsafe.Pointer(extra)), uintptr(unsafe.Pointer(qrs)), uintptr(unsafe.Pointer(pr))) + r0, _, _ := SyscallN(procDnsQuery_W.Addr(), uintptr(unsafe.Pointer(name)), uintptr(qtype), uintptr(options), uintptr(unsafe.Pointer(extra)), uintptr(unsafe.Pointer(qrs)), uintptr(unsafe.Pointer(pr))) if r0 != 0 { status = Errno(r0) } @@ -440,12 +442,12 @@ func _DnsQuery(name *uint16, qtype uint16, options uint32, extra *byte, qrs **DN } func DnsRecordListFree(rl *DNSRecord, freetype uint32) { - Syscall(procDnsRecordListFree.Addr(), 2, uintptr(unsafe.Pointer(rl)), uintptr(freetype), 0) + SyscallN(procDnsRecordListFree.Addr(), uintptr(unsafe.Pointer(rl)), uintptr(freetype)) return } func GetAdaptersInfo(ai *IpAdapterInfo, ol *uint32) (errcode error) { - r0, _, _ := Syscall(procGetAdaptersInfo.Addr(), 2, uintptr(unsafe.Pointer(ai)), uintptr(unsafe.Pointer(ol)), 0) + r0, _, _ := SyscallN(procGetAdaptersInfo.Addr(), uintptr(unsafe.Pointer(ai)), uintptr(unsafe.Pointer(ol))) if r0 != 0 { errcode = Errno(r0) } @@ -453,7 +455,7 @@ func GetAdaptersInfo(ai *IpAdapterInfo, ol *uint32) (errcode error) { } func GetIfEntry(pIfRow *MibIfRow) (errcode error) { - r0, _, _ := Syscall(procGetIfEntry.Addr(), 1, uintptr(unsafe.Pointer(pIfRow)), 0, 0) + r0, _, _ := SyscallN(procGetIfEntry.Addr(), uintptr(unsafe.Pointer(pIfRow))) if r0 != 0 { errcode = Errno(r0) } @@ -461,7 +463,7 @@ func GetIfEntry(pIfRow *MibIfRow) (errcode error) { } func CancelIo(s Handle) (err error) { - r1, _, e1 := Syscall(procCancelIo.Addr(), 1, uintptr(s), 0, 0) + r1, _, e1 := SyscallN(procCancelIo.Addr(), uintptr(s)) if r1 == 0 { err = errnoErr(e1) } @@ -469,7 +471,7 @@ func CancelIo(s Handle) (err error) { } func CancelIoEx(s Handle, o *Overlapped) (err error) { - r1, _, e1 := Syscall(procCancelIoEx.Addr(), 2, uintptr(s), uintptr(unsafe.Pointer(o)), 0) + r1, _, e1 := SyscallN(procCancelIoEx.Addr(), uintptr(s), uintptr(unsafe.Pointer(o))) if r1 == 0 { err = errnoErr(e1) } @@ -477,7 +479,7 @@ func CancelIoEx(s Handle, o *Overlapped) (err error) { } func CloseHandle(handle Handle) (err error) { - r1, _, e1 := Syscall(procCloseHandle.Addr(), 1, uintptr(handle), 0, 0) + r1, _, e1 := SyscallN(procCloseHandle.Addr(), uintptr(handle)) if r1 == 0 { err = errnoErr(e1) } @@ -485,7 +487,7 @@ func CloseHandle(handle Handle) (err error) { } func CreateDirectory(path *uint16, sa *SecurityAttributes) (err error) { - r1, _, e1 := Syscall(procCreateDirectoryW.Addr(), 2, uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(sa)), 0) + r1, _, e1 := SyscallN(procCreateDirectoryW.Addr(), uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(sa))) if r1 == 0 { err = errnoErr(e1) } @@ -493,7 +495,7 @@ func CreateDirectory(path *uint16, sa *SecurityAttributes) (err error) { } func CreateFileMapping(fhandle Handle, sa *SecurityAttributes, prot uint32, maxSizeHigh uint32, maxSizeLow uint32, name *uint16) (handle Handle, err error) { - r0, _, e1 := Syscall6(procCreateFileMappingW.Addr(), 6, uintptr(fhandle), uintptr(unsafe.Pointer(sa)), uintptr(prot), uintptr(maxSizeHigh), uintptr(maxSizeLow), uintptr(unsafe.Pointer(name))) + r0, _, e1 := SyscallN(procCreateFileMappingW.Addr(), uintptr(fhandle), uintptr(unsafe.Pointer(sa)), uintptr(prot), uintptr(maxSizeHigh), uintptr(maxSizeLow), uintptr(unsafe.Pointer(name))) handle = Handle(r0) if handle == 0 { err = errnoErr(e1) @@ -501,17 +503,17 @@ func CreateFileMapping(fhandle Handle, sa *SecurityAttributes, prot uint32, maxS return } -func CreateFile(name *uint16, access uint32, mode uint32, sa *SecurityAttributes, createmode uint32, attrs uint32, templatefile int32) (handle Handle, err error) { - r0, _, e1 := Syscall9(procCreateFileW.Addr(), 7, uintptr(unsafe.Pointer(name)), uintptr(access), uintptr(mode), uintptr(unsafe.Pointer(sa)), uintptr(createmode), uintptr(attrs), uintptr(templatefile), 0, 0) +func createFile(name *uint16, access uint32, mode uint32, sa *SecurityAttributes, createmode uint32, attrs uint32, templatefile int32) (handle Handle, err error) { + r0, _, e1 := SyscallN(procCreateFileW.Addr(), uintptr(unsafe.Pointer(name)), uintptr(access), uintptr(mode), uintptr(unsafe.Pointer(sa)), uintptr(createmode), uintptr(attrs), uintptr(templatefile)) handle = Handle(r0) - if handle == InvalidHandle { + if handle == InvalidHandle || e1 == ERROR_ALREADY_EXISTS { err = errnoErr(e1) } return } func CreateHardLink(filename *uint16, existingfilename *uint16, reserved uintptr) (err error) { - r1, _, e1 := Syscall(procCreateHardLinkW.Addr(), 3, uintptr(unsafe.Pointer(filename)), uintptr(unsafe.Pointer(existingfilename)), uintptr(reserved)) + r1, _, e1 := SyscallN(procCreateHardLinkW.Addr(), uintptr(unsafe.Pointer(filename)), uintptr(unsafe.Pointer(existingfilename)), uintptr(reserved)) if r1&0xff == 0 { err = errnoErr(e1) } @@ -519,7 +521,7 @@ func CreateHardLink(filename *uint16, existingfilename *uint16, reserved uintptr } func createIoCompletionPort(filehandle Handle, cphandle Handle, key uintptr, threadcnt uint32) (handle Handle, err error) { - r0, _, e1 := Syscall6(procCreateIoCompletionPort.Addr(), 4, uintptr(filehandle), uintptr(cphandle), uintptr(key), uintptr(threadcnt), 0, 0) + r0, _, e1 := SyscallN(procCreateIoCompletionPort.Addr(), uintptr(filehandle), uintptr(cphandle), uintptr(key), uintptr(threadcnt)) handle = Handle(r0) if handle == 0 { err = errnoErr(e1) @@ -528,7 +530,7 @@ func createIoCompletionPort(filehandle Handle, cphandle Handle, key uintptr, thr } func CreatePipe(readhandle *Handle, writehandle *Handle, sa *SecurityAttributes, size uint32) (err error) { - r1, _, e1 := Syscall6(procCreatePipe.Addr(), 4, uintptr(unsafe.Pointer(readhandle)), uintptr(unsafe.Pointer(writehandle)), uintptr(unsafe.Pointer(sa)), uintptr(size), 0, 0) + r1, _, e1 := SyscallN(procCreatePipe.Addr(), uintptr(unsafe.Pointer(readhandle)), uintptr(unsafe.Pointer(writehandle)), uintptr(unsafe.Pointer(sa)), uintptr(size)) if r1 == 0 { err = errnoErr(e1) } @@ -540,7 +542,7 @@ func CreateProcess(appName *uint16, commandLine *uint16, procSecurity *SecurityA if inheritHandles { _p0 = 1 } - r1, _, e1 := Syscall12(procCreateProcessW.Addr(), 10, uintptr(unsafe.Pointer(appName)), uintptr(unsafe.Pointer(commandLine)), uintptr(unsafe.Pointer(procSecurity)), uintptr(unsafe.Pointer(threadSecurity)), uintptr(_p0), uintptr(creationFlags), uintptr(unsafe.Pointer(env)), uintptr(unsafe.Pointer(currentDir)), uintptr(unsafe.Pointer(startupInfo)), uintptr(unsafe.Pointer(outProcInfo)), 0, 0) + r1, _, e1 := SyscallN(procCreateProcessW.Addr(), uintptr(unsafe.Pointer(appName)), uintptr(unsafe.Pointer(commandLine)), uintptr(unsafe.Pointer(procSecurity)), uintptr(unsafe.Pointer(threadSecurity)), uintptr(_p0), uintptr(creationFlags), uintptr(unsafe.Pointer(env)), uintptr(unsafe.Pointer(currentDir)), uintptr(unsafe.Pointer(startupInfo)), uintptr(unsafe.Pointer(outProcInfo))) if r1 == 0 { err = errnoErr(e1) } @@ -548,7 +550,7 @@ func CreateProcess(appName *uint16, commandLine *uint16, procSecurity *SecurityA } func CreateSymbolicLink(symlinkfilename *uint16, targetfilename *uint16, flags uint32) (err error) { - r1, _, e1 := Syscall(procCreateSymbolicLinkW.Addr(), 3, uintptr(unsafe.Pointer(symlinkfilename)), uintptr(unsafe.Pointer(targetfilename)), uintptr(flags)) + r1, _, e1 := SyscallN(procCreateSymbolicLinkW.Addr(), uintptr(unsafe.Pointer(symlinkfilename)), uintptr(unsafe.Pointer(targetfilename)), uintptr(flags)) if r1&0xff == 0 { err = errnoErr(e1) } @@ -556,7 +558,7 @@ func CreateSymbolicLink(symlinkfilename *uint16, targetfilename *uint16, flags u } func CreateToolhelp32Snapshot(flags uint32, processId uint32) (handle Handle, err error) { - r0, _, e1 := Syscall(procCreateToolhelp32Snapshot.Addr(), 2, uintptr(flags), uintptr(processId), 0) + r0, _, e1 := SyscallN(procCreateToolhelp32Snapshot.Addr(), uintptr(flags), uintptr(processId)) handle = Handle(r0) if handle == InvalidHandle { err = errnoErr(e1) @@ -565,7 +567,7 @@ func CreateToolhelp32Snapshot(flags uint32, processId uint32) (handle Handle, er } func DeleteFile(path *uint16) (err error) { - r1, _, e1 := Syscall(procDeleteFileW.Addr(), 1, uintptr(unsafe.Pointer(path)), 0, 0) + r1, _, e1 := SyscallN(procDeleteFileW.Addr(), uintptr(unsafe.Pointer(path))) if r1 == 0 { err = errnoErr(e1) } @@ -573,12 +575,12 @@ func DeleteFile(path *uint16) (err error) { } func deleteProcThreadAttributeList(attrlist *_PROC_THREAD_ATTRIBUTE_LIST) { - Syscall(procDeleteProcThreadAttributeList.Addr(), 1, uintptr(unsafe.Pointer(attrlist)), 0, 0) + SyscallN(procDeleteProcThreadAttributeList.Addr(), uintptr(unsafe.Pointer(attrlist))) return } func DeviceIoControl(handle Handle, ioControlCode uint32, inBuffer *byte, inBufferSize uint32, outBuffer *byte, outBufferSize uint32, bytesReturned *uint32, overlapped *Overlapped) (err error) { - r1, _, e1 := Syscall9(procDeviceIoControl.Addr(), 8, uintptr(handle), uintptr(ioControlCode), uintptr(unsafe.Pointer(inBuffer)), uintptr(inBufferSize), uintptr(unsafe.Pointer(outBuffer)), uintptr(outBufferSize), uintptr(unsafe.Pointer(bytesReturned)), uintptr(unsafe.Pointer(overlapped)), 0) + r1, _, e1 := SyscallN(procDeviceIoControl.Addr(), uintptr(handle), uintptr(ioControlCode), uintptr(unsafe.Pointer(inBuffer)), uintptr(inBufferSize), uintptr(unsafe.Pointer(outBuffer)), uintptr(outBufferSize), uintptr(unsafe.Pointer(bytesReturned)), uintptr(unsafe.Pointer(overlapped))) if r1 == 0 { err = errnoErr(e1) } @@ -590,7 +592,7 @@ func DuplicateHandle(hSourceProcessHandle Handle, hSourceHandle Handle, hTargetP if bInheritHandle { _p0 = 1 } - r1, _, e1 := Syscall9(procDuplicateHandle.Addr(), 7, uintptr(hSourceProcessHandle), uintptr(hSourceHandle), uintptr(hTargetProcessHandle), uintptr(unsafe.Pointer(lpTargetHandle)), uintptr(dwDesiredAccess), uintptr(_p0), uintptr(dwOptions), 0, 0) + r1, _, e1 := SyscallN(procDuplicateHandle.Addr(), uintptr(hSourceProcessHandle), uintptr(hSourceHandle), uintptr(hTargetProcessHandle), uintptr(unsafe.Pointer(lpTargetHandle)), uintptr(dwDesiredAccess), uintptr(_p0), uintptr(dwOptions)) if r1 == 0 { err = errnoErr(e1) } @@ -598,12 +600,12 @@ func DuplicateHandle(hSourceProcessHandle Handle, hSourceHandle Handle, hTargetP } func ExitProcess(exitcode uint32) { - Syscall(procExitProcess.Addr(), 1, uintptr(exitcode), 0, 0) + SyscallN(procExitProcess.Addr(), uintptr(exitcode)) return } func FindClose(handle Handle) (err error) { - r1, _, e1 := Syscall(procFindClose.Addr(), 1, uintptr(handle), 0, 0) + r1, _, e1 := SyscallN(procFindClose.Addr(), uintptr(handle)) if r1 == 0 { err = errnoErr(e1) } @@ -611,7 +613,7 @@ func FindClose(handle Handle) (err error) { } func findFirstFile1(name *uint16, data *win32finddata1) (handle Handle, err error) { - r0, _, e1 := Syscall(procFindFirstFileW.Addr(), 2, uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(data)), 0) + r0, _, e1 := SyscallN(procFindFirstFileW.Addr(), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(data))) handle = Handle(r0) if handle == InvalidHandle { err = errnoErr(e1) @@ -620,7 +622,7 @@ func findFirstFile1(name *uint16, data *win32finddata1) (handle Handle, err erro } func findNextFile1(handle Handle, data *win32finddata1) (err error) { - r1, _, e1 := Syscall(procFindNextFileW.Addr(), 2, uintptr(handle), uintptr(unsafe.Pointer(data)), 0) + r1, _, e1 := SyscallN(procFindNextFileW.Addr(), uintptr(handle), uintptr(unsafe.Pointer(data))) if r1 == 0 { err = errnoErr(e1) } @@ -628,7 +630,7 @@ func findNextFile1(handle Handle, data *win32finddata1) (err error) { } func FlushFileBuffers(handle Handle) (err error) { - r1, _, e1 := Syscall(procFlushFileBuffers.Addr(), 1, uintptr(handle), 0, 0) + r1, _, e1 := SyscallN(procFlushFileBuffers.Addr(), uintptr(handle)) if r1 == 0 { err = errnoErr(e1) } @@ -636,7 +638,7 @@ func FlushFileBuffers(handle Handle) (err error) { } func FlushViewOfFile(addr uintptr, length uintptr) (err error) { - r1, _, e1 := Syscall(procFlushViewOfFile.Addr(), 2, uintptr(addr), uintptr(length), 0) + r1, _, e1 := SyscallN(procFlushViewOfFile.Addr(), uintptr(addr), uintptr(length)) if r1 == 0 { err = errnoErr(e1) } @@ -648,7 +650,7 @@ func formatMessage(flags uint32, msgsrc uintptr, msgid uint32, langid uint32, bu if len(buf) > 0 { _p0 = &buf[0] } - r0, _, e1 := Syscall9(procFormatMessageW.Addr(), 7, uintptr(flags), uintptr(msgsrc), uintptr(msgid), uintptr(langid), uintptr(unsafe.Pointer(_p0)), uintptr(len(buf)), uintptr(unsafe.Pointer(args)), 0, 0) + r0, _, e1 := SyscallN(procFormatMessageW.Addr(), uintptr(flags), uintptr(msgsrc), uintptr(msgid), uintptr(langid), uintptr(unsafe.Pointer(_p0)), uintptr(len(buf)), uintptr(unsafe.Pointer(args))) n = uint32(r0) if n == 0 { err = errnoErr(e1) @@ -657,7 +659,7 @@ func formatMessage(flags uint32, msgsrc uintptr, msgid uint32, langid uint32, bu } func FreeEnvironmentStrings(envs *uint16) (err error) { - r1, _, e1 := Syscall(procFreeEnvironmentStringsW.Addr(), 1, uintptr(unsafe.Pointer(envs)), 0, 0) + r1, _, e1 := SyscallN(procFreeEnvironmentStringsW.Addr(), uintptr(unsafe.Pointer(envs))) if r1 == 0 { err = errnoErr(e1) } @@ -665,7 +667,7 @@ func FreeEnvironmentStrings(envs *uint16) (err error) { } func FreeLibrary(handle Handle) (err error) { - r1, _, e1 := Syscall(procFreeLibrary.Addr(), 1, uintptr(handle), 0, 0) + r1, _, e1 := SyscallN(procFreeLibrary.Addr(), uintptr(handle)) if r1 == 0 { err = errnoErr(e1) } @@ -673,13 +675,13 @@ func FreeLibrary(handle Handle) (err error) { } func GetCommandLine() (cmd *uint16) { - r0, _, _ := Syscall(procGetCommandLineW.Addr(), 0, 0, 0, 0) + r0, _, _ := SyscallN(procGetCommandLineW.Addr()) cmd = (*uint16)(unsafe.Pointer(r0)) return } func GetComputerName(buf *uint16, n *uint32) (err error) { - r1, _, e1 := Syscall(procGetComputerNameW.Addr(), 2, uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(n)), 0) + r1, _, e1 := SyscallN(procGetComputerNameW.Addr(), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(n))) if r1 == 0 { err = errnoErr(e1) } @@ -687,7 +689,7 @@ func GetComputerName(buf *uint16, n *uint32) (err error) { } func GetConsoleMode(console Handle, mode *uint32) (err error) { - r1, _, e1 := Syscall(procGetConsoleMode.Addr(), 2, uintptr(console), uintptr(unsafe.Pointer(mode)), 0) + r1, _, e1 := SyscallN(procGetConsoleMode.Addr(), uintptr(console), uintptr(unsafe.Pointer(mode))) if r1 == 0 { err = errnoErr(e1) } @@ -695,7 +697,7 @@ func GetConsoleMode(console Handle, mode *uint32) (err error) { } func GetCurrentDirectory(buflen uint32, buf *uint16) (n uint32, err error) { - r0, _, e1 := Syscall(procGetCurrentDirectoryW.Addr(), 2, uintptr(buflen), uintptr(unsafe.Pointer(buf)), 0) + r0, _, e1 := SyscallN(procGetCurrentDirectoryW.Addr(), uintptr(buflen), uintptr(unsafe.Pointer(buf))) n = uint32(r0) if n == 0 { err = errnoErr(e1) @@ -704,7 +706,7 @@ func GetCurrentDirectory(buflen uint32, buf *uint16) (n uint32, err error) { } func GetCurrentProcess() (pseudoHandle Handle, err error) { - r0, _, e1 := Syscall(procGetCurrentProcess.Addr(), 0, 0, 0, 0) + r0, _, e1 := SyscallN(procGetCurrentProcess.Addr()) pseudoHandle = Handle(r0) if pseudoHandle == 0 { err = errnoErr(e1) @@ -713,13 +715,13 @@ func GetCurrentProcess() (pseudoHandle Handle, err error) { } func getCurrentProcessId() (pid uint32) { - r0, _, _ := Syscall(procGetCurrentProcessId.Addr(), 0, 0, 0, 0) + r0, _, _ := SyscallN(procGetCurrentProcessId.Addr()) pid = uint32(r0) return } func GetEnvironmentStrings() (envs *uint16, err error) { - r0, _, e1 := Syscall(procGetEnvironmentStringsW.Addr(), 0, 0, 0, 0) + r0, _, e1 := SyscallN(procGetEnvironmentStringsW.Addr()) envs = (*uint16)(unsafe.Pointer(r0)) if envs == nil { err = errnoErr(e1) @@ -728,7 +730,7 @@ func GetEnvironmentStrings() (envs *uint16, err error) { } func GetEnvironmentVariable(name *uint16, buffer *uint16, size uint32) (n uint32, err error) { - r0, _, e1 := Syscall(procGetEnvironmentVariableW.Addr(), 3, uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(buffer)), uintptr(size)) + r0, _, e1 := SyscallN(procGetEnvironmentVariableW.Addr(), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(buffer)), uintptr(size)) n = uint32(r0) if n == 0 { err = errnoErr(e1) @@ -737,7 +739,7 @@ func GetEnvironmentVariable(name *uint16, buffer *uint16, size uint32) (n uint32 } func GetExitCodeProcess(handle Handle, exitcode *uint32) (err error) { - r1, _, e1 := Syscall(procGetExitCodeProcess.Addr(), 2, uintptr(handle), uintptr(unsafe.Pointer(exitcode)), 0) + r1, _, e1 := SyscallN(procGetExitCodeProcess.Addr(), uintptr(handle), uintptr(unsafe.Pointer(exitcode))) if r1 == 0 { err = errnoErr(e1) } @@ -745,7 +747,7 @@ func GetExitCodeProcess(handle Handle, exitcode *uint32) (err error) { } func GetFileAttributesEx(name *uint16, level uint32, info *byte) (err error) { - r1, _, e1 := Syscall(procGetFileAttributesExW.Addr(), 3, uintptr(unsafe.Pointer(name)), uintptr(level), uintptr(unsafe.Pointer(info))) + r1, _, e1 := SyscallN(procGetFileAttributesExW.Addr(), uintptr(unsafe.Pointer(name)), uintptr(level), uintptr(unsafe.Pointer(info))) if r1 == 0 { err = errnoErr(e1) } @@ -753,7 +755,7 @@ func GetFileAttributesEx(name *uint16, level uint32, info *byte) (err error) { } func GetFileAttributes(name *uint16) (attrs uint32, err error) { - r0, _, e1 := Syscall(procGetFileAttributesW.Addr(), 1, uintptr(unsafe.Pointer(name)), 0, 0) + r0, _, e1 := SyscallN(procGetFileAttributesW.Addr(), uintptr(unsafe.Pointer(name))) attrs = uint32(r0) if attrs == INVALID_FILE_ATTRIBUTES { err = errnoErr(e1) @@ -762,7 +764,7 @@ func GetFileAttributes(name *uint16) (attrs uint32, err error) { } func GetFileInformationByHandle(handle Handle, data *ByHandleFileInformation) (err error) { - r1, _, e1 := Syscall(procGetFileInformationByHandle.Addr(), 2, uintptr(handle), uintptr(unsafe.Pointer(data)), 0) + r1, _, e1 := SyscallN(procGetFileInformationByHandle.Addr(), uintptr(handle), uintptr(unsafe.Pointer(data))) if r1 == 0 { err = errnoErr(e1) } @@ -770,7 +772,7 @@ func GetFileInformationByHandle(handle Handle, data *ByHandleFileInformation) (e } func GetFileType(filehandle Handle) (n uint32, err error) { - r0, _, e1 := Syscall(procGetFileType.Addr(), 1, uintptr(filehandle), 0, 0) + r0, _, e1 := SyscallN(procGetFileType.Addr(), uintptr(filehandle)) n = uint32(r0) if n == 0 { err = errnoErr(e1) @@ -779,7 +781,7 @@ func GetFileType(filehandle Handle) (n uint32, err error) { } func getFinalPathNameByHandle(file Handle, filePath *uint16, filePathSize uint32, flags uint32) (n uint32, err error) { - r0, _, e1 := Syscall6(procGetFinalPathNameByHandleW.Addr(), 4, uintptr(file), uintptr(unsafe.Pointer(filePath)), uintptr(filePathSize), uintptr(flags), 0, 0) + r0, _, e1 := SyscallN(procGetFinalPathNameByHandleW.Addr(), uintptr(file), uintptr(unsafe.Pointer(filePath)), uintptr(filePathSize), uintptr(flags)) n = uint32(r0) if n == 0 || n >= filePathSize { err = errnoErr(e1) @@ -788,7 +790,7 @@ func getFinalPathNameByHandle(file Handle, filePath *uint16, filePathSize uint32 } func GetFullPathName(path *uint16, buflen uint32, buf *uint16, fname **uint16) (n uint32, err error) { - r0, _, e1 := Syscall6(procGetFullPathNameW.Addr(), 4, uintptr(unsafe.Pointer(path)), uintptr(buflen), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(fname)), 0, 0) + r0, _, e1 := SyscallN(procGetFullPathNameW.Addr(), uintptr(unsafe.Pointer(path)), uintptr(buflen), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(fname))) n = uint32(r0) if n == 0 { err = errnoErr(e1) @@ -797,7 +799,7 @@ func GetFullPathName(path *uint16, buflen uint32, buf *uint16, fname **uint16) ( } func GetLastError() (lasterr error) { - r0, _, _ := Syscall(procGetLastError.Addr(), 0, 0, 0, 0) + r0, _, _ := SyscallN(procGetLastError.Addr()) if r0 != 0 { lasterr = Errno(r0) } @@ -805,7 +807,7 @@ func GetLastError() (lasterr error) { } func GetLongPathName(path *uint16, buf *uint16, buflen uint32) (n uint32, err error) { - r0, _, e1 := Syscall(procGetLongPathNameW.Addr(), 3, uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(buf)), uintptr(buflen)) + r0, _, e1 := SyscallN(procGetLongPathNameW.Addr(), uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(buf)), uintptr(buflen)) n = uint32(r0) if n == 0 { err = errnoErr(e1) @@ -823,7 +825,7 @@ func GetProcAddress(module Handle, procname string) (proc uintptr, err error) { } func _GetProcAddress(module Handle, procname *byte) (proc uintptr, err error) { - r0, _, e1 := Syscall(procGetProcAddress.Addr(), 2, uintptr(module), uintptr(unsafe.Pointer(procname)), 0) + r0, _, e1 := SyscallN(procGetProcAddress.Addr(), uintptr(module), uintptr(unsafe.Pointer(procname))) proc = uintptr(r0) if proc == 0 { err = errnoErr(e1) @@ -832,7 +834,7 @@ func _GetProcAddress(module Handle, procname *byte) (proc uintptr, err error) { } func GetProcessTimes(handle Handle, creationTime *Filetime, exitTime *Filetime, kernelTime *Filetime, userTime *Filetime) (err error) { - r1, _, e1 := Syscall6(procGetProcessTimes.Addr(), 5, uintptr(handle), uintptr(unsafe.Pointer(creationTime)), uintptr(unsafe.Pointer(exitTime)), uintptr(unsafe.Pointer(kernelTime)), uintptr(unsafe.Pointer(userTime)), 0) + r1, _, e1 := SyscallN(procGetProcessTimes.Addr(), uintptr(handle), uintptr(unsafe.Pointer(creationTime)), uintptr(unsafe.Pointer(exitTime)), uintptr(unsafe.Pointer(kernelTime)), uintptr(unsafe.Pointer(userTime))) if r1 == 0 { err = errnoErr(e1) } @@ -840,7 +842,7 @@ func GetProcessTimes(handle Handle, creationTime *Filetime, exitTime *Filetime, } func getQueuedCompletionStatus(cphandle Handle, qty *uint32, key *uintptr, overlapped **Overlapped, timeout uint32) (err error) { - r1, _, e1 := Syscall6(procGetQueuedCompletionStatus.Addr(), 5, uintptr(cphandle), uintptr(unsafe.Pointer(qty)), uintptr(unsafe.Pointer(key)), uintptr(unsafe.Pointer(overlapped)), uintptr(timeout), 0) + r1, _, e1 := SyscallN(procGetQueuedCompletionStatus.Addr(), uintptr(cphandle), uintptr(unsafe.Pointer(qty)), uintptr(unsafe.Pointer(key)), uintptr(unsafe.Pointer(overlapped)), uintptr(timeout)) if r1 == 0 { err = errnoErr(e1) } @@ -848,7 +850,7 @@ func getQueuedCompletionStatus(cphandle Handle, qty *uint32, key *uintptr, overl } func GetShortPathName(longpath *uint16, shortpath *uint16, buflen uint32) (n uint32, err error) { - r0, _, e1 := Syscall(procGetShortPathNameW.Addr(), 3, uintptr(unsafe.Pointer(longpath)), uintptr(unsafe.Pointer(shortpath)), uintptr(buflen)) + r0, _, e1 := SyscallN(procGetShortPathNameW.Addr(), uintptr(unsafe.Pointer(longpath)), uintptr(unsafe.Pointer(shortpath)), uintptr(buflen)) n = uint32(r0) if n == 0 { err = errnoErr(e1) @@ -857,12 +859,12 @@ func GetShortPathName(longpath *uint16, shortpath *uint16, buflen uint32) (n uin } func getStartupInfo(startupInfo *StartupInfo) { - Syscall(procGetStartupInfoW.Addr(), 1, uintptr(unsafe.Pointer(startupInfo)), 0, 0) + SyscallN(procGetStartupInfoW.Addr(), uintptr(unsafe.Pointer(startupInfo))) return } func GetStdHandle(stdhandle int) (handle Handle, err error) { - r0, _, e1 := Syscall(procGetStdHandle.Addr(), 1, uintptr(stdhandle), 0, 0) + r0, _, e1 := SyscallN(procGetStdHandle.Addr(), uintptr(stdhandle)) handle = Handle(r0) if handle == InvalidHandle { err = errnoErr(e1) @@ -871,12 +873,12 @@ func GetStdHandle(stdhandle int) (handle Handle, err error) { } func GetSystemTimeAsFileTime(time *Filetime) { - Syscall(procGetSystemTimeAsFileTime.Addr(), 1, uintptr(unsafe.Pointer(time)), 0, 0) + SyscallN(procGetSystemTimeAsFileTime.Addr(), uintptr(unsafe.Pointer(time))) return } func GetTempPath(buflen uint32, buf *uint16) (n uint32, err error) { - r0, _, e1 := Syscall(procGetTempPathW.Addr(), 2, uintptr(buflen), uintptr(unsafe.Pointer(buf)), 0) + r0, _, e1 := SyscallN(procGetTempPathW.Addr(), uintptr(buflen), uintptr(unsafe.Pointer(buf))) n = uint32(r0) if n == 0 { err = errnoErr(e1) @@ -885,7 +887,7 @@ func GetTempPath(buflen uint32, buf *uint16) (n uint32, err error) { } func GetTimeZoneInformation(tzi *Timezoneinformation) (rc uint32, err error) { - r0, _, e1 := Syscall(procGetTimeZoneInformation.Addr(), 1, uintptr(unsafe.Pointer(tzi)), 0, 0) + r0, _, e1 := SyscallN(procGetTimeZoneInformation.Addr(), uintptr(unsafe.Pointer(tzi))) rc = uint32(r0) if rc == 0xffffffff { err = errnoErr(e1) @@ -894,7 +896,7 @@ func GetTimeZoneInformation(tzi *Timezoneinformation) (rc uint32, err error) { } func GetVersion() (ver uint32, err error) { - r0, _, e1 := Syscall(procGetVersion.Addr(), 0, 0, 0, 0) + r0, _, e1 := SyscallN(procGetVersion.Addr()) ver = uint32(r0) if ver == 0 { err = errnoErr(e1) @@ -903,7 +905,7 @@ func GetVersion() (ver uint32, err error) { } func initializeProcThreadAttributeList(attrlist *_PROC_THREAD_ATTRIBUTE_LIST, attrcount uint32, flags uint32, size *uintptr) (err error) { - r1, _, e1 := Syscall6(procInitializeProcThreadAttributeList.Addr(), 4, uintptr(unsafe.Pointer(attrlist)), uintptr(attrcount), uintptr(flags), uintptr(unsafe.Pointer(size)), 0, 0) + r1, _, e1 := SyscallN(procInitializeProcThreadAttributeList.Addr(), uintptr(unsafe.Pointer(attrlist)), uintptr(attrcount), uintptr(flags), uintptr(unsafe.Pointer(size))) if r1 == 0 { err = errnoErr(e1) } @@ -920,7 +922,7 @@ func LoadLibrary(libname string) (handle Handle, err error) { } func _LoadLibrary(libname *uint16) (handle Handle, err error) { - r0, _, e1 := Syscall(procLoadLibraryW.Addr(), 1, uintptr(unsafe.Pointer(libname)), 0, 0) + r0, _, e1 := SyscallN(procLoadLibraryW.Addr(), uintptr(unsafe.Pointer(libname))) handle = Handle(r0) if handle == 0 { err = errnoErr(e1) @@ -928,8 +930,17 @@ func _LoadLibrary(libname *uint16) (handle Handle, err error) { return } +func localAlloc(flags uint32, length uint32) (ptr uintptr, err error) { + r0, _, e1 := SyscallN(procLocalAlloc.Addr(), uintptr(flags), uintptr(length)) + ptr = uintptr(r0) + if ptr == 0 { + err = errnoErr(e1) + } + return +} + func LocalFree(hmem Handle) (handle Handle, err error) { - r0, _, e1 := Syscall(procLocalFree.Addr(), 1, uintptr(hmem), 0, 0) + r0, _, e1 := SyscallN(procLocalFree.Addr(), uintptr(hmem)) handle = Handle(r0) if handle != 0 { err = errnoErr(e1) @@ -938,7 +949,7 @@ func LocalFree(hmem Handle) (handle Handle, err error) { } func MapViewOfFile(handle Handle, access uint32, offsetHigh uint32, offsetLow uint32, length uintptr) (addr uintptr, err error) { - r0, _, e1 := Syscall6(procMapViewOfFile.Addr(), 5, uintptr(handle), uintptr(access), uintptr(offsetHigh), uintptr(offsetLow), uintptr(length), 0) + r0, _, e1 := SyscallN(procMapViewOfFile.Addr(), uintptr(handle), uintptr(access), uintptr(offsetHigh), uintptr(offsetLow), uintptr(length)) addr = uintptr(r0) if addr == 0 { err = errnoErr(e1) @@ -947,7 +958,7 @@ func MapViewOfFile(handle Handle, access uint32, offsetHigh uint32, offsetLow ui } func MoveFile(from *uint16, to *uint16) (err error) { - r1, _, e1 := Syscall(procMoveFileW.Addr(), 2, uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(to)), 0) + r1, _, e1 := SyscallN(procMoveFileW.Addr(), uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(to))) if r1 == 0 { err = errnoErr(e1) } @@ -959,7 +970,7 @@ func OpenProcess(da uint32, inheritHandle bool, pid uint32) (handle Handle, err if inheritHandle { _p0 = 1 } - r0, _, e1 := Syscall(procOpenProcess.Addr(), 3, uintptr(da), uintptr(_p0), uintptr(pid)) + r0, _, e1 := SyscallN(procOpenProcess.Addr(), uintptr(da), uintptr(_p0), uintptr(pid)) handle = Handle(r0) if handle == 0 { err = errnoErr(e1) @@ -968,7 +979,7 @@ func OpenProcess(da uint32, inheritHandle bool, pid uint32) (handle Handle, err } func postQueuedCompletionStatus(cphandle Handle, qty uint32, key uintptr, overlapped *Overlapped) (err error) { - r1, _, e1 := Syscall6(procPostQueuedCompletionStatus.Addr(), 4, uintptr(cphandle), uintptr(qty), uintptr(key), uintptr(unsafe.Pointer(overlapped)), 0, 0) + r1, _, e1 := SyscallN(procPostQueuedCompletionStatus.Addr(), uintptr(cphandle), uintptr(qty), uintptr(key), uintptr(unsafe.Pointer(overlapped))) if r1 == 0 { err = errnoErr(e1) } @@ -976,7 +987,7 @@ func postQueuedCompletionStatus(cphandle Handle, qty uint32, key uintptr, overla } func Process32First(snapshot Handle, procEntry *ProcessEntry32) (err error) { - r1, _, e1 := Syscall(procProcess32FirstW.Addr(), 2, uintptr(snapshot), uintptr(unsafe.Pointer(procEntry)), 0) + r1, _, e1 := SyscallN(procProcess32FirstW.Addr(), uintptr(snapshot), uintptr(unsafe.Pointer(procEntry))) if r1 == 0 { err = errnoErr(e1) } @@ -984,7 +995,7 @@ func Process32First(snapshot Handle, procEntry *ProcessEntry32) (err error) { } func Process32Next(snapshot Handle, procEntry *ProcessEntry32) (err error) { - r1, _, e1 := Syscall(procProcess32NextW.Addr(), 2, uintptr(snapshot), uintptr(unsafe.Pointer(procEntry)), 0) + r1, _, e1 := SyscallN(procProcess32NextW.Addr(), uintptr(snapshot), uintptr(unsafe.Pointer(procEntry))) if r1 == 0 { err = errnoErr(e1) } @@ -992,7 +1003,7 @@ func Process32Next(snapshot Handle, procEntry *ProcessEntry32) (err error) { } func ReadConsole(console Handle, buf *uint16, toread uint32, read *uint32, inputControl *byte) (err error) { - r1, _, e1 := Syscall6(procReadConsoleW.Addr(), 5, uintptr(console), uintptr(unsafe.Pointer(buf)), uintptr(toread), uintptr(unsafe.Pointer(read)), uintptr(unsafe.Pointer(inputControl)), 0) + r1, _, e1 := SyscallN(procReadConsoleW.Addr(), uintptr(console), uintptr(unsafe.Pointer(buf)), uintptr(toread), uintptr(unsafe.Pointer(read)), uintptr(unsafe.Pointer(inputControl))) if r1 == 0 { err = errnoErr(e1) } @@ -1004,7 +1015,7 @@ func ReadDirectoryChanges(handle Handle, buf *byte, buflen uint32, watchSubTree if watchSubTree { _p0 = 1 } - r1, _, e1 := Syscall9(procReadDirectoryChangesW.Addr(), 8, uintptr(handle), uintptr(unsafe.Pointer(buf)), uintptr(buflen), uintptr(_p0), uintptr(mask), uintptr(unsafe.Pointer(retlen)), uintptr(unsafe.Pointer(overlapped)), uintptr(completionRoutine), 0) + r1, _, e1 := SyscallN(procReadDirectoryChangesW.Addr(), uintptr(handle), uintptr(unsafe.Pointer(buf)), uintptr(buflen), uintptr(_p0), uintptr(mask), uintptr(unsafe.Pointer(retlen)), uintptr(unsafe.Pointer(overlapped)), uintptr(completionRoutine)) if r1 == 0 { err = errnoErr(e1) } @@ -1016,7 +1027,7 @@ func readFile(handle Handle, buf []byte, done *uint32, overlapped *Overlapped) ( if len(buf) > 0 { _p0 = &buf[0] } - r1, _, e1 := Syscall6(procReadFile.Addr(), 5, uintptr(handle), uintptr(unsafe.Pointer(_p0)), uintptr(len(buf)), uintptr(unsafe.Pointer(done)), uintptr(unsafe.Pointer(overlapped)), 0) + r1, _, e1 := SyscallN(procReadFile.Addr(), uintptr(handle), uintptr(unsafe.Pointer(_p0)), uintptr(len(buf)), uintptr(unsafe.Pointer(done)), uintptr(unsafe.Pointer(overlapped))) if r1 == 0 { err = errnoErr(e1) } @@ -1024,7 +1035,7 @@ func readFile(handle Handle, buf []byte, done *uint32, overlapped *Overlapped) ( } func RemoveDirectory(path *uint16) (err error) { - r1, _, e1 := Syscall(procRemoveDirectoryW.Addr(), 1, uintptr(unsafe.Pointer(path)), 0, 0) + r1, _, e1 := SyscallN(procRemoveDirectoryW.Addr(), uintptr(unsafe.Pointer(path))) if r1 == 0 { err = errnoErr(e1) } @@ -1032,7 +1043,7 @@ func RemoveDirectory(path *uint16) (err error) { } func SetCurrentDirectory(path *uint16) (err error) { - r1, _, e1 := Syscall(procSetCurrentDirectoryW.Addr(), 1, uintptr(unsafe.Pointer(path)), 0, 0) + r1, _, e1 := SyscallN(procSetCurrentDirectoryW.Addr(), uintptr(unsafe.Pointer(path))) if r1 == 0 { err = errnoErr(e1) } @@ -1040,7 +1051,7 @@ func SetCurrentDirectory(path *uint16) (err error) { } func SetEndOfFile(handle Handle) (err error) { - r1, _, e1 := Syscall(procSetEndOfFile.Addr(), 1, uintptr(handle), 0, 0) + r1, _, e1 := SyscallN(procSetEndOfFile.Addr(), uintptr(handle)) if r1 == 0 { err = errnoErr(e1) } @@ -1048,7 +1059,7 @@ func SetEndOfFile(handle Handle) (err error) { } func SetEnvironmentVariable(name *uint16, value *uint16) (err error) { - r1, _, e1 := Syscall(procSetEnvironmentVariableW.Addr(), 2, uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(value)), 0) + r1, _, e1 := SyscallN(procSetEnvironmentVariableW.Addr(), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(value))) if r1 == 0 { err = errnoErr(e1) } @@ -1056,7 +1067,7 @@ func SetEnvironmentVariable(name *uint16, value *uint16) (err error) { } func SetFileAttributes(name *uint16, attrs uint32) (err error) { - r1, _, e1 := Syscall(procSetFileAttributesW.Addr(), 2, uintptr(unsafe.Pointer(name)), uintptr(attrs), 0) + r1, _, e1 := SyscallN(procSetFileAttributesW.Addr(), uintptr(unsafe.Pointer(name)), uintptr(attrs)) if r1 == 0 { err = errnoErr(e1) } @@ -1064,7 +1075,15 @@ func SetFileAttributes(name *uint16, attrs uint32) (err error) { } func SetFileCompletionNotificationModes(handle Handle, flags uint8) (err error) { - r1, _, e1 := Syscall(procSetFileCompletionNotificationModes.Addr(), 2, uintptr(handle), uintptr(flags), 0) + r1, _, e1 := SyscallN(procSetFileCompletionNotificationModes.Addr(), uintptr(handle), uintptr(flags)) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func setFileInformationByHandle(handle Handle, fileInformationClass uint32, buf unsafe.Pointer, bufsize uint32) (err error) { + r1, _, e1 := SyscallN(procSetFileInformationByHandle.Addr(), uintptr(handle), uintptr(fileInformationClass), uintptr(buf), uintptr(bufsize)) if r1 == 0 { err = errnoErr(e1) } @@ -1072,7 +1091,7 @@ func SetFileCompletionNotificationModes(handle Handle, flags uint8) (err error) } func SetFilePointer(handle Handle, lowoffset int32, highoffsetptr *int32, whence uint32) (newlowoffset uint32, err error) { - r0, _, e1 := Syscall6(procSetFilePointer.Addr(), 4, uintptr(handle), uintptr(lowoffset), uintptr(unsafe.Pointer(highoffsetptr)), uintptr(whence), 0, 0) + r0, _, e1 := SyscallN(procSetFilePointer.Addr(), uintptr(handle), uintptr(lowoffset), uintptr(unsafe.Pointer(highoffsetptr)), uintptr(whence)) newlowoffset = uint32(r0) if newlowoffset == 0xffffffff { err = errnoErr(e1) @@ -1081,7 +1100,7 @@ func SetFilePointer(handle Handle, lowoffset int32, highoffsetptr *int32, whence } func SetFileTime(handle Handle, ctime *Filetime, atime *Filetime, wtime *Filetime) (err error) { - r1, _, e1 := Syscall6(procSetFileTime.Addr(), 4, uintptr(handle), uintptr(unsafe.Pointer(ctime)), uintptr(unsafe.Pointer(atime)), uintptr(unsafe.Pointer(wtime)), 0, 0) + r1, _, e1 := SyscallN(procSetFileTime.Addr(), uintptr(handle), uintptr(unsafe.Pointer(ctime)), uintptr(unsafe.Pointer(atime)), uintptr(unsafe.Pointer(wtime))) if r1 == 0 { err = errnoErr(e1) } @@ -1089,7 +1108,7 @@ func SetFileTime(handle Handle, ctime *Filetime, atime *Filetime, wtime *Filetim } func SetHandleInformation(handle Handle, mask uint32, flags uint32) (err error) { - r1, _, e1 := Syscall(procSetHandleInformation.Addr(), 3, uintptr(handle), uintptr(mask), uintptr(flags)) + r1, _, e1 := SyscallN(procSetHandleInformation.Addr(), uintptr(handle), uintptr(mask), uintptr(flags)) if r1 == 0 { err = errnoErr(e1) } @@ -1097,7 +1116,7 @@ func SetHandleInformation(handle Handle, mask uint32, flags uint32) (err error) } func TerminateProcess(handle Handle, exitcode uint32) (err error) { - r1, _, e1 := Syscall(procTerminateProcess.Addr(), 2, uintptr(handle), uintptr(exitcode), 0) + r1, _, e1 := SyscallN(procTerminateProcess.Addr(), uintptr(handle), uintptr(exitcode)) if r1 == 0 { err = errnoErr(e1) } @@ -1105,7 +1124,7 @@ func TerminateProcess(handle Handle, exitcode uint32) (err error) { } func UnmapViewOfFile(addr uintptr) (err error) { - r1, _, e1 := Syscall(procUnmapViewOfFile.Addr(), 1, uintptr(addr), 0, 0) + r1, _, e1 := SyscallN(procUnmapViewOfFile.Addr(), uintptr(addr)) if r1 == 0 { err = errnoErr(e1) } @@ -1113,7 +1132,7 @@ func UnmapViewOfFile(addr uintptr) (err error) { } func updateProcThreadAttribute(attrlist *_PROC_THREAD_ATTRIBUTE_LIST, flags uint32, attr uintptr, value unsafe.Pointer, size uintptr, prevvalue unsafe.Pointer, returnedsize *uintptr) (err error) { - r1, _, e1 := Syscall9(procUpdateProcThreadAttribute.Addr(), 7, uintptr(unsafe.Pointer(attrlist)), uintptr(flags), uintptr(attr), uintptr(value), uintptr(size), uintptr(prevvalue), uintptr(unsafe.Pointer(returnedsize)), 0, 0) + r1, _, e1 := SyscallN(procUpdateProcThreadAttribute.Addr(), uintptr(unsafe.Pointer(attrlist)), uintptr(flags), uintptr(attr), uintptr(value), uintptr(size), uintptr(prevvalue), uintptr(unsafe.Pointer(returnedsize))) if r1 == 0 { err = errnoErr(e1) } @@ -1121,7 +1140,7 @@ func updateProcThreadAttribute(attrlist *_PROC_THREAD_ATTRIBUTE_LIST, flags uint } func VirtualLock(addr uintptr, length uintptr) (err error) { - r1, _, e1 := Syscall(procVirtualLock.Addr(), 2, uintptr(addr), uintptr(length), 0) + r1, _, e1 := SyscallN(procVirtualLock.Addr(), uintptr(addr), uintptr(length)) if r1 == 0 { err = errnoErr(e1) } @@ -1129,7 +1148,7 @@ func VirtualLock(addr uintptr, length uintptr) (err error) { } func VirtualUnlock(addr uintptr, length uintptr) (err error) { - r1, _, e1 := Syscall(procVirtualUnlock.Addr(), 2, uintptr(addr), uintptr(length), 0) + r1, _, e1 := SyscallN(procVirtualUnlock.Addr(), uintptr(addr), uintptr(length)) if r1 == 0 { err = errnoErr(e1) } @@ -1137,7 +1156,7 @@ func VirtualUnlock(addr uintptr, length uintptr) (err error) { } func WaitForSingleObject(handle Handle, waitMilliseconds uint32) (event uint32, err error) { - r0, _, e1 := Syscall(procWaitForSingleObject.Addr(), 2, uintptr(handle), uintptr(waitMilliseconds), 0) + r0, _, e1 := SyscallN(procWaitForSingleObject.Addr(), uintptr(handle), uintptr(waitMilliseconds)) event = uint32(r0) if event == 0xffffffff { err = errnoErr(e1) @@ -1146,7 +1165,7 @@ func WaitForSingleObject(handle Handle, waitMilliseconds uint32) (event uint32, } func WriteConsole(console Handle, buf *uint16, towrite uint32, written *uint32, reserved *byte) (err error) { - r1, _, e1 := Syscall6(procWriteConsoleW.Addr(), 5, uintptr(console), uintptr(unsafe.Pointer(buf)), uintptr(towrite), uintptr(unsafe.Pointer(written)), uintptr(unsafe.Pointer(reserved)), 0) + r1, _, e1 := SyscallN(procWriteConsoleW.Addr(), uintptr(console), uintptr(unsafe.Pointer(buf)), uintptr(towrite), uintptr(unsafe.Pointer(written)), uintptr(unsafe.Pointer(reserved))) if r1 == 0 { err = errnoErr(e1) } @@ -1158,7 +1177,7 @@ func writeFile(handle Handle, buf []byte, done *uint32, overlapped *Overlapped) if len(buf) > 0 { _p0 = &buf[0] } - r1, _, e1 := Syscall6(procWriteFile.Addr(), 5, uintptr(handle), uintptr(unsafe.Pointer(_p0)), uintptr(len(buf)), uintptr(unsafe.Pointer(done)), uintptr(unsafe.Pointer(overlapped)), 0) + r1, _, e1 := SyscallN(procWriteFile.Addr(), uintptr(handle), uintptr(unsafe.Pointer(_p0)), uintptr(len(buf)), uintptr(unsafe.Pointer(done)), uintptr(unsafe.Pointer(overlapped))) if r1 == 0 { err = errnoErr(e1) } @@ -1166,7 +1185,7 @@ func writeFile(handle Handle, buf []byte, done *uint32, overlapped *Overlapped) } func AcceptEx(ls Handle, as Handle, buf *byte, rxdatalen uint32, laddrlen uint32, raddrlen uint32, recvd *uint32, overlapped *Overlapped) (err error) { - r1, _, e1 := Syscall9(procAcceptEx.Addr(), 8, uintptr(ls), uintptr(as), uintptr(unsafe.Pointer(buf)), uintptr(rxdatalen), uintptr(laddrlen), uintptr(raddrlen), uintptr(unsafe.Pointer(recvd)), uintptr(unsafe.Pointer(overlapped)), 0) + r1, _, e1 := SyscallN(procAcceptEx.Addr(), uintptr(ls), uintptr(as), uintptr(unsafe.Pointer(buf)), uintptr(rxdatalen), uintptr(laddrlen), uintptr(raddrlen), uintptr(unsafe.Pointer(recvd)), uintptr(unsafe.Pointer(overlapped))) if r1 == 0 { err = errnoErr(e1) } @@ -1174,12 +1193,12 @@ func AcceptEx(ls Handle, as Handle, buf *byte, rxdatalen uint32, laddrlen uint32 } func GetAcceptExSockaddrs(buf *byte, rxdatalen uint32, laddrlen uint32, raddrlen uint32, lrsa **RawSockaddrAny, lrsalen *int32, rrsa **RawSockaddrAny, rrsalen *int32) { - Syscall9(procGetAcceptExSockaddrs.Addr(), 8, uintptr(unsafe.Pointer(buf)), uintptr(rxdatalen), uintptr(laddrlen), uintptr(raddrlen), uintptr(unsafe.Pointer(lrsa)), uintptr(unsafe.Pointer(lrsalen)), uintptr(unsafe.Pointer(rrsa)), uintptr(unsafe.Pointer(rrsalen)), 0) + SyscallN(procGetAcceptExSockaddrs.Addr(), uintptr(unsafe.Pointer(buf)), uintptr(rxdatalen), uintptr(laddrlen), uintptr(raddrlen), uintptr(unsafe.Pointer(lrsa)), uintptr(unsafe.Pointer(lrsalen)), uintptr(unsafe.Pointer(rrsa)), uintptr(unsafe.Pointer(rrsalen))) return } func TransmitFile(s Handle, handle Handle, bytesToWrite uint32, bytsPerSend uint32, overlapped *Overlapped, transmitFileBuf *TransmitFileBuffers, flags uint32) (err error) { - r1, _, e1 := Syscall9(procTransmitFile.Addr(), 7, uintptr(s), uintptr(handle), uintptr(bytesToWrite), uintptr(bytsPerSend), uintptr(unsafe.Pointer(overlapped)), uintptr(unsafe.Pointer(transmitFileBuf)), uintptr(flags), 0, 0) + r1, _, e1 := SyscallN(procTransmitFile.Addr(), uintptr(s), uintptr(handle), uintptr(bytesToWrite), uintptr(bytsPerSend), uintptr(unsafe.Pointer(overlapped)), uintptr(unsafe.Pointer(transmitFileBuf)), uintptr(flags)) if r1 == 0 { err = errnoErr(e1) } @@ -1187,7 +1206,7 @@ func TransmitFile(s Handle, handle Handle, bytesToWrite uint32, bytsPerSend uint } func NetApiBufferFree(buf *byte) (neterr error) { - r0, _, _ := Syscall(procNetApiBufferFree.Addr(), 1, uintptr(unsafe.Pointer(buf)), 0, 0) + r0, _, _ := SyscallN(procNetApiBufferFree.Addr(), uintptr(unsafe.Pointer(buf))) if r0 != 0 { neterr = Errno(r0) } @@ -1195,7 +1214,7 @@ func NetApiBufferFree(buf *byte) (neterr error) { } func NetGetJoinInformation(server *uint16, name **uint16, bufType *uint32) (neterr error) { - r0, _, _ := Syscall(procNetGetJoinInformation.Addr(), 3, uintptr(unsafe.Pointer(server)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(bufType))) + r0, _, _ := SyscallN(procNetGetJoinInformation.Addr(), uintptr(unsafe.Pointer(server)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(bufType))) if r0 != 0 { neterr = Errno(r0) } @@ -1203,7 +1222,7 @@ func NetGetJoinInformation(server *uint16, name **uint16, bufType *uint32) (nete } func NetUserGetInfo(serverName *uint16, userName *uint16, level uint32, buf **byte) (neterr error) { - r0, _, _ := Syscall6(procNetUserGetInfo.Addr(), 4, uintptr(unsafe.Pointer(serverName)), uintptr(unsafe.Pointer(userName)), uintptr(level), uintptr(unsafe.Pointer(buf)), 0, 0) + r0, _, _ := SyscallN(procNetUserGetInfo.Addr(), uintptr(unsafe.Pointer(serverName)), uintptr(unsafe.Pointer(userName)), uintptr(level), uintptr(unsafe.Pointer(buf))) if r0 != 0 { neterr = Errno(r0) } @@ -1211,7 +1230,7 @@ func NetUserGetInfo(serverName *uint16, userName *uint16, level uint32, buf **by } func GetUserNameEx(nameFormat uint32, nameBuffre *uint16, nSize *uint32) (err error) { - r1, _, e1 := Syscall(procGetUserNameExW.Addr(), 3, uintptr(nameFormat), uintptr(unsafe.Pointer(nameBuffre)), uintptr(unsafe.Pointer(nSize))) + r1, _, e1 := SyscallN(procGetUserNameExW.Addr(), uintptr(nameFormat), uintptr(unsafe.Pointer(nameBuffre)), uintptr(unsafe.Pointer(nSize))) if r1&0xff == 0 { err = errnoErr(e1) } @@ -1219,7 +1238,7 @@ func GetUserNameEx(nameFormat uint32, nameBuffre *uint16, nSize *uint32) (err er } func TranslateName(accName *uint16, accNameFormat uint32, desiredNameFormat uint32, translatedName *uint16, nSize *uint32) (err error) { - r1, _, e1 := Syscall6(procTranslateNameW.Addr(), 5, uintptr(unsafe.Pointer(accName)), uintptr(accNameFormat), uintptr(desiredNameFormat), uintptr(unsafe.Pointer(translatedName)), uintptr(unsafe.Pointer(nSize)), 0) + r1, _, e1 := SyscallN(procTranslateNameW.Addr(), uintptr(unsafe.Pointer(accName)), uintptr(accNameFormat), uintptr(desiredNameFormat), uintptr(unsafe.Pointer(translatedName)), uintptr(unsafe.Pointer(nSize))) if r1&0xff == 0 { err = errnoErr(e1) } @@ -1227,7 +1246,7 @@ func TranslateName(accName *uint16, accNameFormat uint32, desiredNameFormat uint } func CommandLineToArgv(cmd *uint16, argc *int32) (argv *[8192]*[8192]uint16, err error) { - r0, _, e1 := Syscall(procCommandLineToArgvW.Addr(), 2, uintptr(unsafe.Pointer(cmd)), uintptr(unsafe.Pointer(argc)), 0) + r0, _, e1 := SyscallN(procCommandLineToArgvW.Addr(), uintptr(unsafe.Pointer(cmd)), uintptr(unsafe.Pointer(argc))) argv = (*[8192]*[8192]uint16)(unsafe.Pointer(r0)) if argv == nil { err = errnoErr(e1) @@ -1236,7 +1255,7 @@ func CommandLineToArgv(cmd *uint16, argc *int32) (argv *[8192]*[8192]uint16, err } func GetUserProfileDirectory(t Token, dir *uint16, dirLen *uint32) (err error) { - r1, _, e1 := Syscall(procGetUserProfileDirectoryW.Addr(), 3, uintptr(t), uintptr(unsafe.Pointer(dir)), uintptr(unsafe.Pointer(dirLen))) + r1, _, e1 := SyscallN(procGetUserProfileDirectoryW.Addr(), uintptr(t), uintptr(unsafe.Pointer(dir)), uintptr(unsafe.Pointer(dirLen))) if r1 == 0 { err = errnoErr(e1) } @@ -1244,12 +1263,12 @@ func GetUserProfileDirectory(t Token, dir *uint16, dirLen *uint32) (err error) { } func FreeAddrInfoW(addrinfo *AddrinfoW) { - Syscall(procFreeAddrInfoW.Addr(), 1, uintptr(unsafe.Pointer(addrinfo)), 0, 0) + SyscallN(procFreeAddrInfoW.Addr(), uintptr(unsafe.Pointer(addrinfo))) return } func GetAddrInfoW(nodename *uint16, servicename *uint16, hints *AddrinfoW, result **AddrinfoW) (sockerr error) { - r0, _, _ := Syscall6(procGetAddrInfoW.Addr(), 4, uintptr(unsafe.Pointer(nodename)), uintptr(unsafe.Pointer(servicename)), uintptr(unsafe.Pointer(hints)), uintptr(unsafe.Pointer(result)), 0, 0) + r0, _, _ := SyscallN(procGetAddrInfoW.Addr(), uintptr(unsafe.Pointer(nodename)), uintptr(unsafe.Pointer(servicename)), uintptr(unsafe.Pointer(hints)), uintptr(unsafe.Pointer(result))) if r0 != 0 { sockerr = Errno(r0) } @@ -1257,7 +1276,7 @@ func GetAddrInfoW(nodename *uint16, servicename *uint16, hints *AddrinfoW, resul } func WSACleanup() (err error) { - r1, _, e1 := Syscall(procWSACleanup.Addr(), 0, 0, 0, 0) + r1, _, e1 := SyscallN(procWSACleanup.Addr()) if r1 == socket_error { err = errnoErr(e1) } @@ -1265,7 +1284,7 @@ func WSACleanup() (err error) { } func WSAEnumProtocols(protocols *int32, protocolBuffer *WSAProtocolInfo, bufferLength *uint32) (n int32, err error) { - r0, _, e1 := Syscall(procWSAEnumProtocolsW.Addr(), 3, uintptr(unsafe.Pointer(protocols)), uintptr(unsafe.Pointer(protocolBuffer)), uintptr(unsafe.Pointer(bufferLength))) + r0, _, e1 := SyscallN(procWSAEnumProtocolsW.Addr(), uintptr(unsafe.Pointer(protocols)), uintptr(unsafe.Pointer(protocolBuffer)), uintptr(unsafe.Pointer(bufferLength))) n = int32(r0) if n == -1 { err = errnoErr(e1) @@ -1274,7 +1293,7 @@ func WSAEnumProtocols(protocols *int32, protocolBuffer *WSAProtocolInfo, bufferL } func WSAIoctl(s Handle, iocc uint32, inbuf *byte, cbif uint32, outbuf *byte, cbob uint32, cbbr *uint32, overlapped *Overlapped, completionRoutine uintptr) (err error) { - r1, _, e1 := Syscall9(procWSAIoctl.Addr(), 9, uintptr(s), uintptr(iocc), uintptr(unsafe.Pointer(inbuf)), uintptr(cbif), uintptr(unsafe.Pointer(outbuf)), uintptr(cbob), uintptr(unsafe.Pointer(cbbr)), uintptr(unsafe.Pointer(overlapped)), uintptr(completionRoutine)) + r1, _, e1 := SyscallN(procWSAIoctl.Addr(), uintptr(s), uintptr(iocc), uintptr(unsafe.Pointer(inbuf)), uintptr(cbif), uintptr(unsafe.Pointer(outbuf)), uintptr(cbob), uintptr(unsafe.Pointer(cbbr)), uintptr(unsafe.Pointer(overlapped)), uintptr(completionRoutine)) if r1 == socket_error { err = errnoErr(e1) } @@ -1282,7 +1301,7 @@ func WSAIoctl(s Handle, iocc uint32, inbuf *byte, cbif uint32, outbuf *byte, cbo } func WSARecv(s Handle, bufs *WSABuf, bufcnt uint32, recvd *uint32, flags *uint32, overlapped *Overlapped, croutine *byte) (err error) { - r1, _, e1 := Syscall9(procWSARecv.Addr(), 7, uintptr(s), uintptr(unsafe.Pointer(bufs)), uintptr(bufcnt), uintptr(unsafe.Pointer(recvd)), uintptr(unsafe.Pointer(flags)), uintptr(unsafe.Pointer(overlapped)), uintptr(unsafe.Pointer(croutine)), 0, 0) + r1, _, e1 := SyscallN(procWSARecv.Addr(), uintptr(s), uintptr(unsafe.Pointer(bufs)), uintptr(bufcnt), uintptr(unsafe.Pointer(recvd)), uintptr(unsafe.Pointer(flags)), uintptr(unsafe.Pointer(overlapped)), uintptr(unsafe.Pointer(croutine))) if r1 == socket_error { err = errnoErr(e1) } @@ -1290,7 +1309,7 @@ func WSARecv(s Handle, bufs *WSABuf, bufcnt uint32, recvd *uint32, flags *uint32 } func WSARecvFrom(s Handle, bufs *WSABuf, bufcnt uint32, recvd *uint32, flags *uint32, from *RawSockaddrAny, fromlen *int32, overlapped *Overlapped, croutine *byte) (err error) { - r1, _, e1 := Syscall9(procWSARecvFrom.Addr(), 9, uintptr(s), uintptr(unsafe.Pointer(bufs)), uintptr(bufcnt), uintptr(unsafe.Pointer(recvd)), uintptr(unsafe.Pointer(flags)), uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(fromlen)), uintptr(unsafe.Pointer(overlapped)), uintptr(unsafe.Pointer(croutine))) + r1, _, e1 := SyscallN(procWSARecvFrom.Addr(), uintptr(s), uintptr(unsafe.Pointer(bufs)), uintptr(bufcnt), uintptr(unsafe.Pointer(recvd)), uintptr(unsafe.Pointer(flags)), uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(fromlen)), uintptr(unsafe.Pointer(overlapped)), uintptr(unsafe.Pointer(croutine))) if r1 == socket_error { err = errnoErr(e1) } @@ -1298,7 +1317,7 @@ func WSARecvFrom(s Handle, bufs *WSABuf, bufcnt uint32, recvd *uint32, flags *ui } func WSASend(s Handle, bufs *WSABuf, bufcnt uint32, sent *uint32, flags uint32, overlapped *Overlapped, croutine *byte) (err error) { - r1, _, e1 := Syscall9(procWSASend.Addr(), 7, uintptr(s), uintptr(unsafe.Pointer(bufs)), uintptr(bufcnt), uintptr(unsafe.Pointer(sent)), uintptr(flags), uintptr(unsafe.Pointer(overlapped)), uintptr(unsafe.Pointer(croutine)), 0, 0) + r1, _, e1 := SyscallN(procWSASend.Addr(), uintptr(s), uintptr(unsafe.Pointer(bufs)), uintptr(bufcnt), uintptr(unsafe.Pointer(sent)), uintptr(flags), uintptr(unsafe.Pointer(overlapped)), uintptr(unsafe.Pointer(croutine))) if r1 == socket_error { err = errnoErr(e1) } @@ -1306,7 +1325,7 @@ func WSASend(s Handle, bufs *WSABuf, bufcnt uint32, sent *uint32, flags uint32, } func WSASendTo(s Handle, bufs *WSABuf, bufcnt uint32, sent *uint32, flags uint32, to *RawSockaddrAny, tolen int32, overlapped *Overlapped, croutine *byte) (err error) { - r1, _, e1 := Syscall9(procWSASendTo.Addr(), 9, uintptr(s), uintptr(unsafe.Pointer(bufs)), uintptr(bufcnt), uintptr(unsafe.Pointer(sent)), uintptr(flags), uintptr(unsafe.Pointer(to)), uintptr(tolen), uintptr(unsafe.Pointer(overlapped)), uintptr(unsafe.Pointer(croutine))) + r1, _, e1 := SyscallN(procWSASendTo.Addr(), uintptr(s), uintptr(unsafe.Pointer(bufs)), uintptr(bufcnt), uintptr(unsafe.Pointer(sent)), uintptr(flags), uintptr(unsafe.Pointer(to)), uintptr(tolen), uintptr(unsafe.Pointer(overlapped)), uintptr(unsafe.Pointer(croutine))) if r1 == socket_error { err = errnoErr(e1) } @@ -1314,7 +1333,7 @@ func WSASendTo(s Handle, bufs *WSABuf, bufcnt uint32, sent *uint32, flags uint32 } func WSAStartup(verreq uint32, data *WSAData) (sockerr error) { - r0, _, _ := Syscall(procWSAStartup.Addr(), 2, uintptr(verreq), uintptr(unsafe.Pointer(data)), 0) + r0, _, _ := SyscallN(procWSAStartup.Addr(), uintptr(verreq), uintptr(unsafe.Pointer(data))) if r0 != 0 { sockerr = Errno(r0) } @@ -1322,7 +1341,7 @@ func WSAStartup(verreq uint32, data *WSAData) (sockerr error) { } func bind(s Handle, name unsafe.Pointer, namelen int32) (err error) { - r1, _, e1 := Syscall(procbind.Addr(), 3, uintptr(s), uintptr(name), uintptr(namelen)) + r1, _, e1 := SyscallN(procbind.Addr(), uintptr(s), uintptr(name), uintptr(namelen)) if r1 == socket_error { err = errnoErr(e1) } @@ -1330,7 +1349,7 @@ func bind(s Handle, name unsafe.Pointer, namelen int32) (err error) { } func Closesocket(s Handle) (err error) { - r1, _, e1 := Syscall(procclosesocket.Addr(), 1, uintptr(s), 0, 0) + r1, _, e1 := SyscallN(procclosesocket.Addr(), uintptr(s)) if r1 == socket_error { err = errnoErr(e1) } @@ -1338,7 +1357,7 @@ func Closesocket(s Handle) (err error) { } func connect(s Handle, name unsafe.Pointer, namelen int32) (err error) { - r1, _, e1 := Syscall(procconnect.Addr(), 3, uintptr(s), uintptr(name), uintptr(namelen)) + r1, _, e1 := SyscallN(procconnect.Addr(), uintptr(s), uintptr(name), uintptr(namelen)) if r1 == socket_error { err = errnoErr(e1) } @@ -1355,7 +1374,7 @@ func GetHostByName(name string) (h *Hostent, err error) { } func _GetHostByName(name *byte) (h *Hostent, err error) { - r0, _, e1 := Syscall(procgethostbyname.Addr(), 1, uintptr(unsafe.Pointer(name)), 0, 0) + r0, _, e1 := SyscallN(procgethostbyname.Addr(), uintptr(unsafe.Pointer(name))) h = (*Hostent)(unsafe.Pointer(r0)) if h == nil { err = errnoErr(e1) @@ -1364,7 +1383,7 @@ func _GetHostByName(name *byte) (h *Hostent, err error) { } func getpeername(s Handle, rsa *RawSockaddrAny, addrlen *int32) (err error) { - r1, _, e1 := Syscall(procgetpeername.Addr(), 3, uintptr(s), uintptr(unsafe.Pointer(rsa)), uintptr(unsafe.Pointer(addrlen))) + r1, _, e1 := SyscallN(procgetpeername.Addr(), uintptr(s), uintptr(unsafe.Pointer(rsa)), uintptr(unsafe.Pointer(addrlen))) if r1 == socket_error { err = errnoErr(e1) } @@ -1381,7 +1400,7 @@ func GetProtoByName(name string) (p *Protoent, err error) { } func _GetProtoByName(name *byte) (p *Protoent, err error) { - r0, _, e1 := Syscall(procgetprotobyname.Addr(), 1, uintptr(unsafe.Pointer(name)), 0, 0) + r0, _, e1 := SyscallN(procgetprotobyname.Addr(), uintptr(unsafe.Pointer(name))) p = (*Protoent)(unsafe.Pointer(r0)) if p == nil { err = errnoErr(e1) @@ -1404,7 +1423,7 @@ func GetServByName(name string, proto string) (s *Servent, err error) { } func _GetServByName(name *byte, proto *byte) (s *Servent, err error) { - r0, _, e1 := Syscall(procgetservbyname.Addr(), 2, uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(proto)), 0) + r0, _, e1 := SyscallN(procgetservbyname.Addr(), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(proto))) s = (*Servent)(unsafe.Pointer(r0)) if s == nil { err = errnoErr(e1) @@ -1413,7 +1432,7 @@ func _GetServByName(name *byte, proto *byte) (s *Servent, err error) { } func getsockname(s Handle, rsa *RawSockaddrAny, addrlen *int32) (err error) { - r1, _, e1 := Syscall(procgetsockname.Addr(), 3, uintptr(s), uintptr(unsafe.Pointer(rsa)), uintptr(unsafe.Pointer(addrlen))) + r1, _, e1 := SyscallN(procgetsockname.Addr(), uintptr(s), uintptr(unsafe.Pointer(rsa)), uintptr(unsafe.Pointer(addrlen))) if r1 == socket_error { err = errnoErr(e1) } @@ -1421,7 +1440,7 @@ func getsockname(s Handle, rsa *RawSockaddrAny, addrlen *int32) (err error) { } func Getsockopt(s Handle, level int32, optname int32, optval *byte, optlen *int32) (err error) { - r1, _, e1 := Syscall6(procgetsockopt.Addr(), 5, uintptr(s), uintptr(level), uintptr(optname), uintptr(unsafe.Pointer(optval)), uintptr(unsafe.Pointer(optlen)), 0) + r1, _, e1 := SyscallN(procgetsockopt.Addr(), uintptr(s), uintptr(level), uintptr(optname), uintptr(unsafe.Pointer(optval)), uintptr(unsafe.Pointer(optlen))) if r1 == socket_error { err = errnoErr(e1) } @@ -1429,7 +1448,7 @@ func Getsockopt(s Handle, level int32, optname int32, optval *byte, optlen *int3 } func listen(s Handle, backlog int32) (err error) { - r1, _, e1 := Syscall(proclisten.Addr(), 2, uintptr(s), uintptr(backlog), 0) + r1, _, e1 := SyscallN(proclisten.Addr(), uintptr(s), uintptr(backlog)) if r1 == socket_error { err = errnoErr(e1) } @@ -1437,13 +1456,13 @@ func listen(s Handle, backlog int32) (err error) { } func Ntohs(netshort uint16) (u uint16) { - r0, _, _ := Syscall(procntohs.Addr(), 1, uintptr(netshort), 0, 0) + r0, _, _ := SyscallN(procntohs.Addr(), uintptr(netshort)) u = uint16(r0) return } func Setsockopt(s Handle, level int32, optname int32, optval *byte, optlen int32) (err error) { - r1, _, e1 := Syscall6(procsetsockopt.Addr(), 5, uintptr(s), uintptr(level), uintptr(optname), uintptr(unsafe.Pointer(optval)), uintptr(optlen), 0) + r1, _, e1 := SyscallN(procsetsockopt.Addr(), uintptr(s), uintptr(level), uintptr(optname), uintptr(unsafe.Pointer(optval)), uintptr(optlen)) if r1 == socket_error { err = errnoErr(e1) } @@ -1451,7 +1470,7 @@ func Setsockopt(s Handle, level int32, optname int32, optval *byte, optlen int32 } func shutdown(s Handle, how int32) (err error) { - r1, _, e1 := Syscall(procshutdown.Addr(), 2, uintptr(s), uintptr(how), 0) + r1, _, e1 := SyscallN(procshutdown.Addr(), uintptr(s), uintptr(how)) if r1 == socket_error { err = errnoErr(e1) } @@ -1459,7 +1478,7 @@ func shutdown(s Handle, how int32) (err error) { } func socket(af int32, typ int32, protocol int32) (handle Handle, err error) { - r0, _, e1 := Syscall(procsocket.Addr(), 3, uintptr(af), uintptr(typ), uintptr(protocol)) + r0, _, e1 := SyscallN(procsocket.Addr(), uintptr(af), uintptr(typ), uintptr(protocol)) handle = Handle(r0) if handle == InvalidHandle { err = errnoErr(e1) diff --git a/src/syscall/zsysnum_darwin_amd64.go b/src/syscall/zsysnum_darwin_amd64.go index 08e003f2926448..dcff3ca8346c72 100644 --- a/src/syscall/zsysnum_darwin_amd64.go +++ b/src/syscall/zsysnum_darwin_amd64.go @@ -1,8 +1,6 @@ // mksysnum_darwin.pl /usr/include/sys/syscall.h // Code generated by the command above; DO NOT EDIT. -//go:build amd64 && darwin - package syscall const ( diff --git a/src/syscall/zsysnum_darwin_arm64.go b/src/syscall/zsysnum_darwin_arm64.go index 71309bb4d6f7f8..d023dc3e036b4c 100644 --- a/src/syscall/zsysnum_darwin_arm64.go +++ b/src/syscall/zsysnum_darwin_arm64.go @@ -1,8 +1,6 @@ // mksysnum_darwin.pl /usr/include/sys/syscall.h // Code generated by the command above; DO NOT EDIT. -//go:build arm64 && darwin - package syscall const ( diff --git a/src/syscall/zsysnum_dragonfly_amd64.go b/src/syscall/zsysnum_dragonfly_amd64.go index 03d4b06a09b401..b9fb1c0807ab36 100644 --- a/src/syscall/zsysnum_dragonfly_amd64.go +++ b/src/syscall/zsysnum_dragonfly_amd64.go @@ -1,8 +1,6 @@ // mksysnum_dragonfly.pl // Code generated by the command above; DO NOT EDIT. -//go:build amd64 && dragonfly - package syscall const ( diff --git a/src/syscall/zsysnum_freebsd_386.go b/src/syscall/zsysnum_freebsd_386.go index ada885e458f4f3..348bcd014d2fe6 100644 --- a/src/syscall/zsysnum_freebsd_386.go +++ b/src/syscall/zsysnum_freebsd_386.go @@ -1,8 +1,6 @@ // mksysnum_freebsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build 386 && freebsd - package syscall const ( diff --git a/src/syscall/zsysnum_freebsd_amd64.go b/src/syscall/zsysnum_freebsd_amd64.go index 85a50ad94be8e2..348bcd014d2fe6 100644 --- a/src/syscall/zsysnum_freebsd_amd64.go +++ b/src/syscall/zsysnum_freebsd_amd64.go @@ -1,8 +1,6 @@ // mksysnum_freebsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build amd64 && freebsd - package syscall const ( diff --git a/src/syscall/zsysnum_freebsd_arm.go b/src/syscall/zsysnum_freebsd_arm.go index 7f6e3fc3f54ae3..348bcd014d2fe6 100644 --- a/src/syscall/zsysnum_freebsd_arm.go +++ b/src/syscall/zsysnum_freebsd_arm.go @@ -1,8 +1,6 @@ // mksysnum_freebsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build arm && freebsd - package syscall const ( diff --git a/src/syscall/zsysnum_freebsd_arm64.go b/src/syscall/zsysnum_freebsd_arm64.go index bde0f3bcb744a9..43870907f9f237 100644 --- a/src/syscall/zsysnum_freebsd_arm64.go +++ b/src/syscall/zsysnum_freebsd_arm64.go @@ -1,8 +1,6 @@ // mksysnum_freebsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build arm64 && freebsd - package syscall const ( diff --git a/src/syscall/zsysnum_freebsd_riscv64.go b/src/syscall/zsysnum_freebsd_riscv64.go index 48e70296437bfc..43870907f9f237 100644 --- a/src/syscall/zsysnum_freebsd_riscv64.go +++ b/src/syscall/zsysnum_freebsd_riscv64.go @@ -1,8 +1,6 @@ // mksysnum_freebsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build riscv64 && freebsd - package syscall const ( diff --git a/src/syscall/zsysnum_linux_386.go b/src/syscall/zsysnum_linux_386.go index 4966d2a94714e8..e5e474fb445101 100644 --- a/src/syscall/zsysnum_linux_386.go +++ b/src/syscall/zsysnum_linux_386.go @@ -1,8 +1,6 @@ // mksysnum_linux.pl /usr/include/asm/unistd_32.h // Code generated by the command above; DO NOT EDIT. -//go:build 386 && linux - package syscall const ( diff --git a/src/syscall/zsysnum_linux_amd64.go b/src/syscall/zsysnum_linux_amd64.go index 576c7c36a6b7eb..2a78e3a00d8d20 100644 --- a/src/syscall/zsysnum_linux_amd64.go +++ b/src/syscall/zsysnum_linux_amd64.go @@ -1,8 +1,6 @@ // mksysnum_linux.pl /usr/include/asm/unistd_64.h // Code generated by the command above; DO NOT EDIT. -//go:build amd64 && linux - package syscall const ( diff --git a/src/syscall/zsysnum_linux_arm.go b/src/syscall/zsysnum_linux_arm.go index b0da97c64bc32d..58e3a7813aad68 100644 --- a/src/syscall/zsysnum_linux_arm.go +++ b/src/syscall/zsysnum_linux_arm.go @@ -1,8 +1,6 @@ // mksysnum_linux.pl // Code generated by the command above; DO NOT EDIT. -//go:build arm && linux - package syscall const ( diff --git a/src/syscall/zsysnum_linux_arm64.go b/src/syscall/zsysnum_linux_arm64.go index 0136d9440b897a..0ccb57aad41fec 100644 --- a/src/syscall/zsysnum_linux_arm64.go +++ b/src/syscall/zsysnum_linux_arm64.go @@ -1,8 +1,6 @@ // mksysnum_linux.pl /usr/include/asm-generic/unistd.h // Code generated by the command above; DO NOT EDIT. -//go:build arm64 && linux - package syscall const ( diff --git a/src/syscall/zsysnum_linux_ppc64.go b/src/syscall/zsysnum_linux_ppc64.go index cc964c23e2eef0..c2e058b2b03a34 100644 --- a/src/syscall/zsysnum_linux_ppc64.go +++ b/src/syscall/zsysnum_linux_ppc64.go @@ -1,8 +1,6 @@ // mksysnum_linux.pl /usr/include/asm/unistd.h // Code generated by the command above; DO NOT EDIT. -//go:build ppc64 && linux - package syscall const ( diff --git a/src/syscall/zsysnum_linux_ppc64le.go b/src/syscall/zsysnum_linux_ppc64le.go index 57bfb7795f2341..0c8de6271cd9df 100644 --- a/src/syscall/zsysnum_linux_ppc64le.go +++ b/src/syscall/zsysnum_linux_ppc64le.go @@ -1,8 +1,6 @@ // mksysnum_linux.pl /usr/include/powerpc64le-linux-gnu/asm/unistd.h // Code generated by the command above; DO NOT EDIT. -//go:build ppc64le && linux - package syscall const ( diff --git a/src/syscall/zsysnum_netbsd_386.go b/src/syscall/zsysnum_netbsd_386.go index 5696c4befebc89..c28d0b4ea17c08 100644 --- a/src/syscall/zsysnum_netbsd_386.go +++ b/src/syscall/zsysnum_netbsd_386.go @@ -1,8 +1,6 @@ // mksysnum_netbsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build 386 && netbsd - package syscall const ( diff --git a/src/syscall/zsysnum_netbsd_amd64.go b/src/syscall/zsysnum_netbsd_amd64.go index 9fb85cddbaf858..c28d0b4ea17c08 100644 --- a/src/syscall/zsysnum_netbsd_amd64.go +++ b/src/syscall/zsysnum_netbsd_amd64.go @@ -1,8 +1,6 @@ // mksysnum_netbsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build amd64 && netbsd - package syscall const ( diff --git a/src/syscall/zsysnum_netbsd_arm.go b/src/syscall/zsysnum_netbsd_arm.go index e0e8994641ab71..c28d0b4ea17c08 100644 --- a/src/syscall/zsysnum_netbsd_arm.go +++ b/src/syscall/zsysnum_netbsd_arm.go @@ -1,8 +1,6 @@ // mksysnum_netbsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build arm && netbsd - package syscall const ( diff --git a/src/syscall/zsysnum_netbsd_arm64.go b/src/syscall/zsysnum_netbsd_arm64.go index 9653364dae9411..c28d0b4ea17c08 100644 --- a/src/syscall/zsysnum_netbsd_arm64.go +++ b/src/syscall/zsysnum_netbsd_arm64.go @@ -1,8 +1,6 @@ // mksysnum_netbsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build arm64 && netbsd - package syscall const ( diff --git a/src/syscall/zsysnum_openbsd_386.go b/src/syscall/zsysnum_openbsd_386.go index 3b12639bd73993..622efe19f2699f 100644 --- a/src/syscall/zsysnum_openbsd_386.go +++ b/src/syscall/zsysnum_openbsd_386.go @@ -1,8 +1,6 @@ // mksysnum_openbsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build 386 && openbsd - package syscall const ( diff --git a/src/syscall/zsysnum_openbsd_amd64.go b/src/syscall/zsysnum_openbsd_amd64.go index bce309dc5d94db..22d2ce387680ca 100644 --- a/src/syscall/zsysnum_openbsd_amd64.go +++ b/src/syscall/zsysnum_openbsd_amd64.go @@ -1,8 +1,6 @@ // mksysnum_openbsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build amd64 && openbsd - package syscall const ( diff --git a/src/syscall/zsysnum_openbsd_arm.go b/src/syscall/zsysnum_openbsd_arm.go index 05aed70762df58..80051fdf35a0f9 100644 --- a/src/syscall/zsysnum_openbsd_arm.go +++ b/src/syscall/zsysnum_openbsd_arm.go @@ -1,8 +1,6 @@ // mksysnum_openbsd.pl // Code generated by the command above; DO NOT EDIT. -//go:build arm && openbsd - package syscall const ( diff --git a/src/syscall/zsysnum_solaris_amd64.go b/src/syscall/zsysnum_solaris_amd64.go index ea244e535ec42a..43b3d8b40d6663 100644 --- a/src/syscall/zsysnum_solaris_amd64.go +++ b/src/syscall/zsysnum_solaris_amd64.go @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build amd64 && solaris - package syscall // TODO(aram): remove these before Go 1.3. diff --git a/src/syscall/ztypes_darwin_amd64.go b/src/syscall/ztypes_darwin_amd64.go index 551edc702578d1..8b05961b6f5b8f 100644 --- a/src/syscall/ztypes_darwin_amd64.go +++ b/src/syscall/ztypes_darwin_amd64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_darwin.go - -//go:build amd64 && darwin +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_darwin.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_darwin_arm64.go b/src/syscall/ztypes_darwin_arm64.go index 46f78a97eb2950..8b05961b6f5b8f 100644 --- a/src/syscall/ztypes_darwin_arm64.go +++ b/src/syscall/ztypes_darwin_arm64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_darwin.go - -//go:build arm64 && darwin +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_darwin.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_dragonfly_amd64.go b/src/syscall/ztypes_dragonfly_amd64.go index ec519b72ec1584..eae8221ef17cc6 100644 --- a/src/syscall/ztypes_dragonfly_amd64.go +++ b/src/syscall/ztypes_dragonfly_amd64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_dragonfly.go - -//go:build amd64 && dragonfly +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_dragonfly.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_freebsd_386.go b/src/syscall/ztypes_freebsd_386.go index 70ae808ef96940..bbb222e02d1aea 100644 --- a/src/syscall/ztypes_freebsd_386.go +++ b/src/syscall/ztypes_freebsd_386.go @@ -1,8 +1,6 @@ // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs types_freebsd.go | go run mkpost.go -//go:build 386 && freebsd - package syscall const ( diff --git a/src/syscall/ztypes_freebsd_amd64.go b/src/syscall/ztypes_freebsd_amd64.go index 050432a69338a8..3f97d1e4fe7016 100644 --- a/src/syscall/ztypes_freebsd_amd64.go +++ b/src/syscall/ztypes_freebsd_amd64.go @@ -1,8 +1,6 @@ // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs types_freebsd.go | go run mkpost.go -//go:build amd64 && freebsd - package syscall const ( diff --git a/src/syscall/ztypes_freebsd_arm.go b/src/syscall/ztypes_freebsd_arm.go index ae5ed56c88970a..f6b1ae4ef2f3b7 100644 --- a/src/syscall/ztypes_freebsd_arm.go +++ b/src/syscall/ztypes_freebsd_arm.go @@ -1,8 +1,6 @@ // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs -- -fsigned-char types_freebsd.go -//go:build arm && freebsd - package syscall const ( diff --git a/src/syscall/ztypes_freebsd_arm64.go b/src/syscall/ztypes_freebsd_arm64.go index b3ab991f8e01ed..3f97d1e4fe7016 100644 --- a/src/syscall/ztypes_freebsd_arm64.go +++ b/src/syscall/ztypes_freebsd_arm64.go @@ -1,8 +1,6 @@ // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs types_freebsd.go | go run mkpost.go -//go:build arm64 && freebsd - package syscall const ( diff --git a/src/syscall/ztypes_freebsd_riscv64.go b/src/syscall/ztypes_freebsd_riscv64.go index 288951840b1df0..3f97d1e4fe7016 100644 --- a/src/syscall/ztypes_freebsd_riscv64.go +++ b/src/syscall/ztypes_freebsd_riscv64.go @@ -1,8 +1,6 @@ // Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs types_freebsd.go | go run mkpost.go -//go:build riscv64 && freebsd - package syscall const ( diff --git a/src/syscall/ztypes_linux_386.go b/src/syscall/ztypes_linux_386.go index 79a46a4aba5b9d..3a3dfc1b56a242 100644 --- a/src/syscall/ztypes_linux_386.go +++ b/src/syscall/ztypes_linux_386.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go - -//go:build 386 && linux +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_amd64.go b/src/syscall/ztypes_linux_amd64.go index 3d223fe2565128..a3831c1e522d39 100644 --- a/src/syscall/ztypes_linux_amd64.go +++ b/src/syscall/ztypes_linux_amd64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go - -//go:build amd64 && linux +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_arm.go b/src/syscall/ztypes_linux_arm.go index 9db1142c22c7eb..f29c87331e19fc 100644 --- a/src/syscall/ztypes_linux_arm.go +++ b/src/syscall/ztypes_linux_arm.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go - -//go:build arm && linux +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_arm64.go b/src/syscall/ztypes_linux_arm64.go index 996950f4b05d0a..05a750bc29a477 100644 --- a/src/syscall/ztypes_linux_arm64.go +++ b/src/syscall/ztypes_linux_arm64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs -- -fsigned-char types_linux.go - -//go:build arm64 && linux +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_mips.go b/src/syscall/ztypes_linux_mips.go index 4ce84978b1ad6d..dac89f4f91b1cb 100644 --- a/src/syscall/ztypes_linux_mips.go +++ b/src/syscall/ztypes_linux_mips.go @@ -1,5 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_mips64.go b/src/syscall/ztypes_linux_mips64.go index de39e732c87f17..b7e55901061e3e 100644 --- a/src/syscall/ztypes_linux_mips64.go +++ b/src/syscall/ztypes_linux_mips64.go @@ -1,5 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_mips64le.go b/src/syscall/ztypes_linux_mips64le.go index de39e732c87f17..b7e55901061e3e 100644 --- a/src/syscall/ztypes_linux_mips64le.go +++ b/src/syscall/ztypes_linux_mips64le.go @@ -1,5 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_mipsle.go b/src/syscall/ztypes_linux_mipsle.go index 4ce84978b1ad6d..dac89f4f91b1cb 100644 --- a/src/syscall/ztypes_linux_mipsle.go +++ b/src/syscall/ztypes_linux_mipsle.go @@ -1,5 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_ppc64.go b/src/syscall/ztypes_linux_ppc64.go index 717e273425bc45..74e1435e0bca10 100644 --- a/src/syscall/ztypes_linux_ppc64.go +++ b/src/syscall/ztypes_linux_ppc64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go - -//go:build ppc64 && linux +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_ppc64le.go b/src/syscall/ztypes_linux_ppc64le.go index 177c1f1e37fd85..5828e9b1f301a7 100644 --- a/src/syscall/ztypes_linux_ppc64le.go +++ b/src/syscall/ztypes_linux_ppc64le.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go - -//go:build ppc64le && linux +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_riscv64.go b/src/syscall/ztypes_linux_riscv64.go index a6c4d7884ab8a9..1827e5df556434 100644 --- a/src/syscall/ztypes_linux_riscv64.go +++ b/src/syscall/ztypes_linux_riscv64.go @@ -1,5 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_linux.go +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_linux_s390x.go b/src/syscall/ztypes_linux_s390x.go index 5c5a71461e3efd..df3414db274c2b 100644 --- a/src/syscall/ztypes_linux_s390x.go +++ b/src/syscall/ztypes_linux_s390x.go @@ -1,4 +1,4 @@ -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs types_linux.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_netbsd_386.go b/src/syscall/ztypes_netbsd_386.go index 74eaa4a15034b0..b0e76adbd43bc3 100644 --- a/src/syscall/ztypes_netbsd_386.go +++ b/src/syscall/ztypes_netbsd_386.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_netbsd.go - -//go:build 386 && netbsd +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_netbsd.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_netbsd_amd64.go b/src/syscall/ztypes_netbsd_amd64.go index fc28fc9bb8f77f..6825623374f4c0 100644 --- a/src/syscall/ztypes_netbsd_amd64.go +++ b/src/syscall/ztypes_netbsd_amd64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_netbsd.go - -//go:build amd64 && netbsd +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_netbsd.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_netbsd_arm.go b/src/syscall/ztypes_netbsd_arm.go index 1f885048dd65f7..99ff41e0dbf43a 100644 --- a/src/syscall/ztypes_netbsd_arm.go +++ b/src/syscall/ztypes_netbsd_arm.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_netbsd.go - -//go:build arm && netbsd +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_netbsd.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_netbsd_arm64.go b/src/syscall/ztypes_netbsd_arm64.go index cac74693d756a8..6825623374f4c0 100644 --- a/src/syscall/ztypes_netbsd_arm64.go +++ b/src/syscall/ztypes_netbsd_arm64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_netbsd.go - -//go:build arm64 && netbsd +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_netbsd.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_openbsd_386.go b/src/syscall/ztypes_openbsd_386.go index f9ba685e25269d..8dfa22c29743f5 100644 --- a/src/syscall/ztypes_openbsd_386.go +++ b/src/syscall/ztypes_openbsd_386.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_openbsd.go - -//go:build 386 && openbsd +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_openbsd.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_openbsd_amd64.go b/src/syscall/ztypes_openbsd_amd64.go index 889b9551ae9024..9382727a8a85d5 100644 --- a/src/syscall/ztypes_openbsd_amd64.go +++ b/src/syscall/ztypes_openbsd_amd64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_openbsd.go - -//go:build amd64 && openbsd +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_openbsd.go | go run mkpost.go package syscall diff --git a/src/syscall/ztypes_solaris_amd64.go b/src/syscall/ztypes_solaris_amd64.go index d486cd002f967b..a9f14517f3a582 100644 --- a/src/syscall/ztypes_solaris_amd64.go +++ b/src/syscall/ztypes_solaris_amd64.go @@ -1,7 +1,5 @@ -// Created by cgo -godefs - DO NOT EDIT -// cgo -godefs types_solaris.go - -//go:build amd64 && solaris +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_solaris.go | go run mkpost.go package syscall diff --git a/src/testing/allocs.go b/src/testing/allocs.go index 1eeb2d4802c31c..ac005dd9342457 100644 --- a/src/testing/allocs.go +++ b/src/testing/allocs.go @@ -15,9 +15,12 @@ import ( // a warm-up. The average number of allocations over the specified number of // runs will then be measured and returned. // -// AllocsPerRun sets GOMAXPROCS to 1 during its measurement and will restore +// AllocsPerRun sets [runtime.GOMAXPROCS] to 1 during its measurement and will restore // it before returning. func AllocsPerRun(runs int, f func()) (avg float64) { + if parallelStart.Load() != parallelStop.Load() { + panic("testing: AllocsPerRun called during parallel test") + } defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1)) // Warm up the function diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go index 80a1b7de77bc6d..437c2ec741b182 100644 --- a/src/testing/benchmark.go +++ b/src/testing/benchmark.go @@ -5,6 +5,7 @@ package testing import ( + "context" "flag" "fmt" "internal/sysinfo" @@ -78,12 +79,12 @@ type InternalBenchmark struct { } // B is a type passed to [Benchmark] functions to manage benchmark -// timing and to specify the number of iterations to run. +// timing and control the number of iterations. // // A benchmark ends when its Benchmark function returns or calls any of the methods -// FailNow, Fatal, Fatalf, SkipNow, Skip, or Skipf. Those methods must be called -// only from the goroutine running the Benchmark function. -// The other reporting methods, such as the variations of Log and Error, +// [B.FailNow], [B.Fatal], [B.Fatalf], [B.SkipNow], [B.Skip], or [B.Skipf]. +// Those methods must be called only from the goroutine running the Benchmark function. +// The other reporting methods, such as the variations of [B.Log] and [B.Error], // may be called simultaneously from multiple goroutines. // // Like in tests, benchmark logs are accumulated during execution @@ -93,7 +94,7 @@ type InternalBenchmark struct { type B struct { common importPath string // import path of the package containing the benchmark - context *benchContext + bstate *benchState N int previousN int // number of iterations in the previous run previousDuration time.Duration // total duration of the previous run @@ -113,6 +114,22 @@ type B struct { netBytes uint64 // Extra metrics collected by ReportMetric. extra map[string]float64 + + // loop tracks the state of B.Loop + loop struct { + // n is the target number of iterations. It gets bumped up as we go. + // When the benchmark loop is done, we commit this to b.N so users can + // do reporting based on it, but we avoid exposing it until then. + n uint64 + // i is the current Loop iteration. It's strictly monotonically + // increasing toward n. + // + // The high bit is used to poison the Loop fast path and fall back to + // the slow path. + i uint64 + + done bool // set when B.Loop return false + } } // StartTimer starts timing a test. This function is called automatically @@ -125,12 +142,12 @@ func (b *B) StartTimer() { b.startBytes = memStats.TotalAlloc b.start = highPrecisionTimeNow() b.timerOn = true + b.loop.i &^= loopPoisonTimer } } // StopTimer stops timing a test. This can be used to pause the timer -// while performing complex initialization that you don't -// want to measure. +// while performing steps that you don't want to measure. func (b *B) StopTimer() { if b.timerOn { b.duration += highPrecisionTimeSince(b.start) @@ -138,6 +155,8 @@ func (b *B) StopTimer() { b.netAllocs += memStats.Mallocs - b.startAllocs b.netBytes += memStats.TotalAlloc - b.startBytes b.timerOn = false + // If we hit B.Loop with the timer stopped, fail. + b.loop.i |= loopPoisonTimer } } @@ -178,6 +197,7 @@ func (b *B) ReportAllocs() { func (b *B) runN(n int) { benchmarkLock.Lock() defer benchmarkLock.Unlock() + ctx, cancelCtx := context.WithCancel(context.Background()) defer func() { b.runCleanup(normalPanic) b.checkRaces() @@ -187,6 +207,12 @@ func (b *B) runN(n int) { runtime.GC() b.resetRaces() b.N = n + b.loop.n = 0 + b.loop.i = 0 + b.loop.done = false + b.ctx = ctx + b.cancelCtx = cancelCtx + b.parallelism = 1 b.ResetTimer() b.StartTimer() @@ -194,15 +220,19 @@ func (b *B) runN(n int) { b.StopTimer() b.previousN = n b.previousDuration = b.duration + + if b.loop.n > 0 && !b.loop.done && !b.failed { + b.Error("benchmark function returned without B.Loop() == false (break or return in loop?)") + } } // run1 runs the first iteration of benchFunc. It reports whether more // iterations of this benchmarks should be run. func (b *B) run1() bool { - if ctx := b.context; ctx != nil { + if bstate := b.bstate; bstate != nil { // Extend maxLen, if needed. - if n := len(b.name) + ctx.extLen + 1; n > ctx.maxLen { - ctx.maxLen = n + 8 // Add additional slack to avoid too many jumps in size. + if n := len(b.name) + bstate.extLen + 1; n > bstate.maxLen { + bstate.maxLen = n + 8 // Add additional slack to avoid too many jumps in size. } } go func() { @@ -253,9 +283,9 @@ func (b *B) run() { fmt.Fprintf(b.w, "cpu: %s\n", cpu) } }) - if b.context != nil { + if b.bstate != nil { // Running go test --test.bench - b.context.processBench(b) // Must call doBench. + b.bstate.processBench(b) // Must call doBench. } else { // Running func Benchmark. b.doBench() @@ -268,6 +298,32 @@ func (b *B) doBench() BenchmarkResult { return b.result } +// Don't run more than 1e9 times. (This also keeps n in int range on 32 bit platforms.) +const maxBenchPredictIters = 1_000_000_000 + +func predictN(goalns int64, prevIters int64, prevns int64, last int64) int { + if prevns == 0 { + // Round up to dodge divide by zero. See https://go.dev/issue/70709. + prevns = 1 + } + + // Order of operations matters. + // For very fast benchmarks, prevIters ~= prevns. + // If you divide first, you get 0 or 1, + // which can hide an order of magnitude in execution time. + // So multiply first, then divide. + n := goalns * prevIters / prevns + // Run more iterations than we think we'll need (1.2x). + n += n / 5 + // Don't grow too fast in case we had timing errors previously. + n = min(n, 100*last) + // Be sure to run at least one more than last time. + n = max(n, last+1) + // Don't run more than 1e9 times. (This also keeps n in int range on 32 bit platforms.) + n = min(n, maxBenchPredictIters) + return int(n) +} + // launch launches the benchmark function. It gradually increases the number // of benchmark iterations until the benchmark runs for the requested benchtime. // launch is run by the doBench function as a separate goroutine. @@ -279,41 +335,27 @@ func (b *B) launch() { b.signal <- true }() - // Run the benchmark for at least the specified amount of time. - if b.benchTime.n > 0 { - // We already ran a single iteration in run1. - // If -benchtime=1x was requested, use that result. - // See https://golang.org/issue/32051. - if b.benchTime.n > 1 { - b.runN(b.benchTime.n) - } - } else { - d := b.benchTime.d - for n := int64(1); !b.failed && b.duration < d && n < 1e9; { - last := n - // Predict required iterations. - goalns := d.Nanoseconds() - prevIters := int64(b.N) - prevns := b.duration.Nanoseconds() - if prevns <= 0 { - // Round up, to avoid div by zero. - prevns = 1 + // b.Loop does its own ramp-up logic so we just need to run it once. + // If b.loop.n is non zero, it means b.Loop has already run. + if b.loop.n == 0 { + // Run the benchmark for at least the specified amount of time. + if b.benchTime.n > 0 { + // We already ran a single iteration in run1. + // If -benchtime=1x was requested, use that result. + // See https://golang.org/issue/32051. + if b.benchTime.n > 1 { + b.runN(b.benchTime.n) + } + } else { + d := b.benchTime.d + for n := int64(1); !b.failed && b.duration < d && n < 1e9; { + last := n + // Predict required iterations. + goalns := d.Nanoseconds() + prevIters := int64(b.N) + n = int64(predictN(goalns, prevIters, b.duration.Nanoseconds(), last)) + b.runN(int(n)) } - // Order of operations matters. - // For very fast benchmarks, prevIters ~= prevns. - // If you divide first, you get 0 or 1, - // which can hide an order of magnitude in execution time. - // So multiply first, then divide. - n = goalns * prevIters / prevns - // Run more iterations than we think we'll need (1.2x). - n += n / 5 - // Don't grow too fast in case we had timing errors previously. - n = min(n, 100*last) - // Be sure to run at least one more than last time. - n = max(n, last+1) - // Don't run more than 1e9 times. (This also keeps n in int range on 32 bit platforms.) - n = min(n, 1e9) - b.runN(int(n)) } } b.result = BenchmarkResult{b.N, b.duration, b.bytes, b.netAllocs, b.netBytes, b.extra} @@ -349,6 +391,147 @@ func (b *B) ReportMetric(n float64, unit string) { b.extra[unit] = n } +func (b *B) stopOrScaleBLoop() bool { + t := b.Elapsed() + if t >= b.benchTime.d { + // We've reached the target + return false + } + // Loop scaling + goalns := b.benchTime.d.Nanoseconds() + prevIters := int64(b.loop.n) + b.loop.n = uint64(predictN(goalns, prevIters, t.Nanoseconds(), prevIters)) + if b.loop.n&loopPoisonMask != 0 { + // The iteration count should never get this high, but if it did we'd be + // in big trouble. + panic("loop iteration target overflow") + } + // predictN may have capped the number of iterations; make sure to + // terminate if we've already hit that cap. + return uint64(prevIters) < b.loop.n +} + +func (b *B) loopSlowPath() bool { + // Consistency checks + if !b.timerOn { + b.Fatal("B.Loop called with timer stopped") + } + if b.loop.i&loopPoisonMask != 0 { + panic(fmt.Sprintf("unknown loop stop condition: %#x", b.loop.i)) + } + + if b.loop.n == 0 { + // It's the first call to b.Loop() in the benchmark function. + if b.benchTime.n > 0 { + // Fixed iteration count. + b.loop.n = uint64(b.benchTime.n) + } else { + // Initialize target to 1 to kick start loop scaling. + b.loop.n = 1 + } + // Within a b.Loop loop, we don't use b.N (to avoid confusion). + b.N = 0 + b.ResetTimer() + + // Start the next iteration. + b.loop.i++ + return true + } + + // Should we keep iterating? + var more bool + if b.benchTime.n > 0 { + // The iteration count is fixed, so we should have run this many and now + // be done. + if b.loop.i != uint64(b.benchTime.n) { + // We shouldn't be able to reach the slow path in this case. + panic(fmt.Sprintf("iteration count %d < fixed target %d", b.loop.i, b.benchTime.n)) + } + more = false + } else { + // Handle fixed time case + more = b.stopOrScaleBLoop() + } + if !more { + b.StopTimer() + // Commit iteration count + b.N = int(b.loop.n) + b.loop.done = true + return false + } + + // Start the next iteration. + b.loop.i++ + return true +} + +// Loop returns true as long as the benchmark should continue running. +// +// A typical benchmark is structured like: +// +// func Benchmark(b *testing.B) { +// ... setup ... +// for b.Loop() { +// ... code to measure ... +// } +// ... cleanup ... +// } +// +// Loop resets the benchmark timer the first time it is called in a benchmark, +// so any setup performed prior to starting the benchmark loop does not count +// toward the benchmark measurement. Likewise, when it returns false, it stops +// the timer so cleanup code is not measured. +// +// Within the body of a "for b.Loop() { ... }" loop, arguments to and +// results from function calls within the loop are kept alive, preventing +// the compiler from fully optimizing away the loop body. Currently, this is +// implemented by disabling inlining of functions called in a b.Loop loop. +// This applies only to calls syntactically between the curly braces of the loop, +// and the loop condition must be written exactly as "b.Loop()". Optimizations +// are performed as usual in any functions called by the loop. +// +// After Loop returns false, b.N contains the total number of iterations that +// ran, so the benchmark may use b.N to compute other average metrics. +// +// Prior to the introduction of Loop, benchmarks were expected to contain an +// explicit loop from 0 to b.N. Benchmarks should either use Loop or contain a +// loop to b.N, but not both. Loop offers more automatic management of the +// benchmark timer, and runs each benchmark function only once per measurement, +// whereas b.N-based benchmarks must run the benchmark function (and any +// associated setup and cleanup) several times. +func (b *B) Loop() bool { + // This is written such that the fast path is as fast as possible and can be + // inlined. + // + // There are three cases where we'll fall out of the fast path: + // + // - On the first call, both i and n are 0. + // + // - If the loop reaches the n'th iteration, then i == n and we need + // to figure out the new target iteration count or if we're done. + // + // - If the timer is stopped, it poisons the top bit of i so the slow + // path can do consistency checks and fail. + if b.loop.i < b.loop.n { + b.loop.i++ + return true + } + return b.loopSlowPath() +} + +// The loopPoison constants can be OR'd into B.loop.i to cause it to fall back +// to the slow path. +const ( + loopPoisonTimer = uint64(1 << (63 - iota)) + // If necessary, add more poison bits here. + + // loopPoisonMask is the set of all loop poison bits. (iota-1) is the index + // of the bit we just set, from which we recreate that bit mask. We subtract + // 1 to set all of the bits below that bit, then complement the result to + // get the mask. Sorry, not sorry. + loopPoisonMask = ^uint64((1 << (63 - (iota - 1))) - 1) +) + // BenchmarkResult contains the results of a benchmark run. type BenchmarkResult struct { N int // The number of iterations. @@ -492,7 +675,7 @@ func benchmarkName(name string, n int) string { return name } -type benchContext struct { +type benchState struct { match *matcher maxLen int // The largest recorded benchmark name. @@ -517,17 +700,17 @@ func runBenchmarks(importPath string, matchString func(pat, str string) (bool, e maxprocs = procs } } - ctx := &benchContext{ + bstate := &benchState{ match: newMatcher(matchString, *matchBenchmarks, "-test.bench", *skip), extLen: len(benchmarkName("", maxprocs)), } var bs []InternalBenchmark for _, Benchmark := range benchmarks { - if _, matched, _ := ctx.match.fullName(nil, Benchmark.Name); matched { + if _, matched, _ := bstate.match.fullName(nil, Benchmark.Name); matched { bs = append(bs, Benchmark) benchName := benchmarkName(Benchmark.Name, maxprocs) - if l := len(benchName) + ctx.extLen + 1; l > ctx.maxLen { - ctx.maxLen = l + if l := len(benchName) + bstate.extLen + 1; l > bstate.maxLen { + bstate.maxLen = l } } } @@ -544,7 +727,7 @@ func runBenchmarks(importPath string, matchString func(pat, str string) (bool, e } }, benchTime: benchTime, - context: ctx, + bstate: bstate, } if Verbose() { main.chatty = newChattyPrinter(main.w) @@ -554,7 +737,7 @@ func runBenchmarks(importPath string, matchString func(pat, str string) (bool, e } // processBench runs bench b for the configured CPU counts and prints the results. -func (ctx *benchContext) processBench(b *B) { +func (s *benchState) processBench(b *B) { for i, procs := range cpuList { for j := uint(0); j < *count; j++ { runtime.GOMAXPROCS(procs) @@ -562,7 +745,7 @@ func (ctx *benchContext) processBench(b *B) { // If it's chatty, we've already printed this information. if b.chatty == nil { - fmt.Fprintf(b.w, "%-*s\t", ctx.maxLen, benchName) + fmt.Fprintf(b.w, "%-*s\t", s.maxLen, benchName) } // Recompute the running time for all but the first iteration. if i > 0 || j > 0 { @@ -577,6 +760,7 @@ func (ctx *benchContext) processBench(b *B) { benchFunc: b.benchFunc, benchTime: b.benchTime, } + b.setOutputWriter() b.run1() } r := b.doBench() @@ -589,7 +773,7 @@ func (ctx *benchContext) processBench(b *B) { } results := r.String() if b.chatty != nil { - fmt.Fprintf(b.w, "%-*s\t", ctx.maxLen, benchName) + fmt.Fprintf(b.w, "%-*s\t", s.maxLen, benchName) } if *benchmarkMemory || b.showAllocResult { results += "\t" + r.MemString() @@ -629,8 +813,8 @@ func (b *B) Run(name string, f func(b *B)) bool { defer benchmarkLock.Lock() benchName, ok, partial := b.name, true, false - if b.context != nil { - benchName, ok, partial = b.context.match.fullName(&b.common, name) + if b.bstate != nil { + benchName, ok, partial = b.bstate.match.fullName(&b.common, name) } if !ok { return true @@ -651,8 +835,9 @@ func (b *B) Run(name string, f func(b *B)) bool { importPath: b.importPath, benchFunc: f, benchTime: b.benchTime, - context: b.context, + bstate: b.bstate, } + sub.setOutputWriter() if partial { // Partial name match, like -bench=X/Y matching BenchmarkX. // Only process sub-benchmarks, if any. @@ -829,6 +1014,7 @@ func Benchmark(f func(b *B)) BenchmarkResult { benchFunc: f, benchTime: benchTime, } + b.setOutputWriter() if b.run1() { b.run() } diff --git a/src/testing/benchmark_test.go b/src/testing/benchmark_test.go index 66f555d1f1d05d..e2dd24c839d694 100644 --- a/src/testing/benchmark_test.go +++ b/src/testing/benchmark_test.go @@ -7,6 +7,8 @@ package testing_test import ( "bytes" "cmp" + "context" + "errors" "runtime" "slices" "strings" @@ -127,6 +129,34 @@ func TestRunParallelSkipNow(t *testing.T) { }) } +func TestBenchmarkContext(t *testing.T) { + testing.Benchmark(func(b *testing.B) { + ctx := b.Context() + if err := ctx.Err(); err != nil { + b.Fatalf("expected non-canceled context, got %v", err) + } + + var innerCtx context.Context + b.Run("inner", func(b *testing.B) { + innerCtx = b.Context() + if err := innerCtx.Err(); err != nil { + b.Fatalf("expected inner benchmark to not inherit canceled context, got %v", err) + } + }) + b.Run("inner2", func(b *testing.B) { + if !errors.Is(innerCtx.Err(), context.Canceled) { + t.Fatal("expected context of sibling benchmark to be canceled after its test function finished") + } + }) + + t.Cleanup(func() { + if !errors.Is(ctx.Err(), context.Canceled) { + t.Fatal("expected context canceled before cleanup") + } + }) + }) +} + func ExampleB_RunParallel() { // Parallel benchmark for text/template.Template.Execute on a single object. testing.Benchmark(func(b *testing.B) { @@ -167,7 +197,7 @@ func ExampleB_ReportMetric() { // specific algorithm (in this case, sorting). testing.Benchmark(func(b *testing.B) { var compares int64 - for i := 0; i < b.N; i++ { + for b.Loop() { s := []int{5, 4, 3, 2, 1} slices.SortFunc(s, func(a, b int) int { compares++ diff --git a/src/testing/cover.go b/src/testing/cover.go index 6ad43ab9ff4a8b..74c97d471e29d1 100644 --- a/src/testing/cover.go +++ b/src/testing/cover.go @@ -6,13 +6,6 @@ package testing -import ( - "fmt" - "internal/goexperiment" - "os" - "sync/atomic" -) - // CoverBlock records the coverage data for a single basic block. // The fields are 1-indexed, as in an editor: The opening line of // the file is number 1, for example. Columns are measured @@ -27,8 +20,6 @@ type CoverBlock struct { Stmts uint16 // Number of statements included in this block. } -var cover Cover - // Cover records information about test coverage checking. // NOTE: This struct is internal to the testing infrastructure and may change. // It is not covered (yet) by the Go 1 compatibility guidelines. @@ -39,86 +30,8 @@ type Cover struct { CoveredPackages string } -// Coverage reports the current code coverage as a fraction in the range [0, 1]. -// If coverage is not enabled, Coverage returns 0. -// -// When running a large set of sequential test cases, checking Coverage after each one -// can be useful for identifying which test cases exercise new code paths. -// It is not a replacement for the reports generated by 'go test -cover' and -// 'go tool cover'. -func Coverage() float64 { - if goexperiment.CoverageRedesign { - return coverage2() - } - var n, d int64 - for _, counters := range cover.Counters { - for i := range counters { - if atomic.LoadUint32(&counters[i]) > 0 { - n++ - } - d++ - } - } - if d == 0 { - return 0 - } - return float64(n) / float64(d) -} - // RegisterCover records the coverage data accumulators for the tests. // NOTE: This function is internal to the testing infrastructure and may change. // It is not covered (yet) by the Go 1 compatibility guidelines. func RegisterCover(c Cover) { - cover = c -} - -// mustBeNil checks the error and, if present, reports it and exits. -func mustBeNil(err error) { - if err != nil { - fmt.Fprintf(os.Stderr, "testing: %s\n", err) - os.Exit(2) - } -} - -// coverReport reports the coverage percentage and writes a coverage profile if requested. -func coverReport() { - if goexperiment.CoverageRedesign { - coverReport2() - return - } - var f *os.File - var err error - if *coverProfile != "" { - f, err = os.Create(toOutputDir(*coverProfile)) - mustBeNil(err) - fmt.Fprintf(f, "mode: %s\n", cover.Mode) - defer func() { mustBeNil(f.Close()) }() - } - - var active, total int64 - var count uint32 - for name, counts := range cover.Counters { - blocks := cover.Blocks[name] - for i := range counts { - stmts := int64(blocks[i].Stmts) - total += stmts - count = atomic.LoadUint32(&counts[i]) // For -mode=atomic. - if count > 0 { - active += stmts - } - if f != nil { - _, err := fmt.Fprintf(f, "%s:%d.%d,%d.%d %d %d\n", name, - blocks[i].Line0, blocks[i].Col0, - blocks[i].Line1, blocks[i].Col1, - stmts, - count) - mustBeNil(err) - } - } - } - if total == 0 { - fmt.Println("coverage: [no statements]") - return - } - fmt.Printf("coverage: %.1f%% of statements%s\n", 100*float64(active)/float64(total), cover.CoveredPackages) } diff --git a/src/testing/example.go b/src/testing/example.go index b14477a4069739..58c36f2edb187f 100644 --- a/src/testing/example.go +++ b/src/testing/example.go @@ -6,6 +6,7 @@ package testing import ( "fmt" + "runtime" "slices" "strings" "time" @@ -45,12 +46,6 @@ func runExamples(matchString func(pat, str string) (bool, error), examples []Int return ran, ok } -func sortLines(output string) string { - lines := strings.Split(output, "\n") - slices.Sort(lines) - return strings.Join(lines, "\n") -} - // processRunResult computes a summary and status of the result of running an example test. // stdout is the captured output from stdout of the test. // recovered is the result of invoking recover after running the test, in case it panicked. @@ -66,8 +61,14 @@ func (eg *InternalExample) processRunResult(stdout string, timeSpent time.Durati var fail string got := strings.TrimSpace(stdout) want := strings.TrimSpace(eg.Output) + if runtime.GOOS == "windows" { + got = strings.ReplaceAll(got, "\r\n", "\n") + want = strings.ReplaceAll(want, "\r\n", "\n") + } if eg.Unordered { - if sortLines(got) != sortLines(want) && recovered == nil { + gotLines := slices.Sorted(strings.SplitSeq(got, "\n")) + wantLines := slices.Sorted(strings.SplitSeq(want, "\n")) + if !slices.Equal(gotLines, wantLines) && recovered == nil { fail = fmt.Sprintf("got:\n%s\nwant (unordered):\n%s\n", stdout, eg.Output) } } else { diff --git a/src/testing/example_loop_test.go b/src/testing/example_loop_test.go new file mode 100644 index 00000000000000..eff8bab352058f --- /dev/null +++ b/src/testing/example_loop_test.go @@ -0,0 +1,48 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testing_test + +import ( + "math/rand/v2" + "testing" +) + +// ExBenchmark shows how to use b.Loop in a benchmark. +// +// (If this were a real benchmark, not an example, this would be named +// BenchmarkSomething.) +func ExBenchmark(b *testing.B) { + // Generate a large random slice to use as an input. + // Since this is done before the first call to b.Loop(), + // it doesn't count toward the benchmark time. + input := make([]int, 128<<10) + for i := range input { + input[i] = rand.Int() + } + + // Perform the benchmark. + for b.Loop() { + // Normally, the compiler would be allowed to optimize away the call + // to sum because it has no side effects and the result isn't used. + // However, inside a b.Loop loop, the compiler ensures function calls + // aren't optimized away. + sum(input) + } + + // Outside the loop, the timer is stopped, so we could perform + // cleanup if necessary without affecting the result. +} + +func sum(data []int) int { + total := 0 + for _, value := range data { + total += value + } + return total +} + +func ExampleB_Loop() { + testing.Benchmark(ExBenchmark) +} diff --git a/src/testing/export_test.go b/src/testing/export_test.go index 10a5b04aee38e2..a2dddc79b68f65 100644 --- a/src/testing/export_test.go +++ b/src/testing/export_test.go @@ -9,3 +9,5 @@ var PrettyPrint = prettyPrint type HighPrecisionTime = highPrecisionTime var HighPrecisionTimeNow = highPrecisionTimeNow + +const ParallelConflict = parallelConflict diff --git a/src/testing/flag_test.go b/src/testing/flag_test.go index 6f76c237c4bdf0..fb65e1592811e3 100644 --- a/src/testing/flag_test.go +++ b/src/testing/flag_test.go @@ -25,14 +25,9 @@ func TestFlag(t *testing.T) { testenv.MustHaveExec(t) for _, flag := range []string{"", "-test.v", "-test.v=test2json"} { - flag := flag t.Run(flag, func(t *testing.T) { t.Parallel() - exe, err := os.Executable() - if err != nil { - exe = os.Args[0] - } - cmd := exec.Command(exe, "-test.run=^TestFlag$", "-test_flag_arg="+flag) + cmd := exec.Command(testenv.Executable(t), "-test.run=^TestFlag$", "-test_flag_arg="+flag) if flag != "" { cmd.Args = append(cmd.Args, flag) } diff --git a/src/testing/fstest/mapfs.go b/src/testing/fstest/mapfs.go index 5e3720b0ed6e07..5ce03985e1ee5f 100644 --- a/src/testing/fstest/mapfs.go +++ b/src/testing/fstest/mapfs.go @@ -15,7 +15,7 @@ import ( // A MapFS is a simple in-memory file system for use in tests, // represented as a map from path names (arguments to Open) -// to information about the files or directories they represent. +// to information about the files, directories, or symbolic links they represent. // // The map need not include parent directories for files contained // in the map; those will be synthesized if needed. @@ -34,21 +34,27 @@ type MapFS map[string]*MapFile // A MapFile describes a single file in a [MapFS]. type MapFile struct { - Data []byte // file content + Data []byte // file content or symlink destination Mode fs.FileMode // fs.FileInfo.Mode ModTime time.Time // fs.FileInfo.ModTime Sys any // fs.FileInfo.Sys } var _ fs.FS = MapFS(nil) +var _ fs.ReadLinkFS = MapFS(nil) var _ fs.File = (*openMapFile)(nil) -// Open opens the named file. +// Open opens the named file after following any symbolic links. func (fsys MapFS) Open(name string) (fs.File, error) { if !fs.ValidPath(name) { return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} } - file := fsys[name] + realName, ok := fsys.resolveSymlinks(name) + if !ok { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} + } + + file := fsys[realName] if file != nil && file.Mode&fs.ModeDir == 0 { // Ordinary file return &openMapFile{name, mapFileInfo{path.Base(name), file}, 0}, nil @@ -59,10 +65,8 @@ func (fsys MapFS) Open(name string) (fs.File, error) { // But file can also be non-nil, in case the user wants to set metadata for the directory explicitly. // Either way, we need to construct the list of children of this directory. var list []mapFileInfo - var elem string var need = make(map[string]bool) - if name == "." { - elem = "." + if realName == "." { for fname, f := range fsys { i := strings.Index(fname, "/") if i < 0 { @@ -74,8 +78,7 @@ func (fsys MapFS) Open(name string) (fs.File, error) { } } } else { - elem = name[strings.LastIndex(name, "/")+1:] - prefix := name + "/" + prefix := realName + "/" for fname, f := range fsys { if strings.HasPrefix(fname, prefix) { felem := fname[len(prefix):] @@ -107,9 +110,103 @@ func (fsys MapFS) Open(name string) (fs.File, error) { if file == nil { file = &MapFile{Mode: fs.ModeDir | 0555} } + var elem string + if name == "." { + elem = "." + } else { + elem = name[strings.LastIndex(name, "/")+1:] + } return &mapDir{name, mapFileInfo{elem, file}, list, 0}, nil } +func (fsys MapFS) resolveSymlinks(name string) (_ string, ok bool) { + // Fast path: if a symlink is in the map, resolve it. + if file := fsys[name]; file != nil && file.Mode.Type() == fs.ModeSymlink { + target := string(file.Data) + if path.IsAbs(target) { + return "", false + } + return fsys.resolveSymlinks(path.Join(path.Dir(name), target)) + } + + // Check if each parent directory (starting at root) is a symlink. + for i := 0; i < len(name); { + j := strings.Index(name[i:], "/") + var dir string + if j < 0 { + dir = name + i = len(name) + } else { + dir = name[:i+j] + i += j + } + if file := fsys[dir]; file != nil && file.Mode.Type() == fs.ModeSymlink { + target := string(file.Data) + if path.IsAbs(target) { + return "", false + } + return fsys.resolveSymlinks(path.Join(path.Dir(dir), target) + name[i:]) + } + i += len("/") + } + return name, fs.ValidPath(name) +} + +// ReadLink returns the destination of the named symbolic link. +func (fsys MapFS) ReadLink(name string) (string, error) { + info, err := fsys.lstat(name) + if err != nil { + return "", &fs.PathError{Op: "readlink", Path: name, Err: err} + } + if info.f.Mode.Type() != fs.ModeSymlink { + return "", &fs.PathError{Op: "readlink", Path: name, Err: fs.ErrInvalid} + } + return string(info.f.Data), nil +} + +// Lstat returns a FileInfo describing the named file. +// If the file is a symbolic link, the returned FileInfo describes the symbolic link. +// Lstat makes no attempt to follow the link. +func (fsys MapFS) Lstat(name string) (fs.FileInfo, error) { + info, err := fsys.lstat(name) + if err != nil { + return nil, &fs.PathError{Op: "lstat", Path: name, Err: err} + } + return info, nil +} + +func (fsys MapFS) lstat(name string) (*mapFileInfo, error) { + if !fs.ValidPath(name) { + return nil, fs.ErrNotExist + } + realDir, ok := fsys.resolveSymlinks(path.Dir(name)) + if !ok { + return nil, fs.ErrNotExist + } + elem := path.Base(name) + realName := path.Join(realDir, elem) + + file := fsys[realName] + if file != nil { + return &mapFileInfo{elem, file}, nil + } + + if realName == "." { + return &mapFileInfo{elem, &MapFile{Mode: fs.ModeDir | 0555}}, nil + } + // Maybe a directory. + prefix := realName + "/" + for fname := range fsys { + if strings.HasPrefix(fname, prefix) { + return &mapFileInfo{elem, &MapFile{Mode: fs.ModeDir | 0555}}, nil + } + } + // If the directory name is not in the map, + // and there are no children of the name in the map, + // then the directory is treated as not existing. + return nil, fs.ErrNotExist +} + // fsOnly is a wrapper that hides all but the fs.FS methods, // to avoid an infinite recursion when implementing special // methods in terms of helpers that would use them. diff --git a/src/testing/fstest/mapfs_test.go b/src/testing/fstest/mapfs_test.go index 6381a2e56c99b3..e7ff4180ecc5e4 100644 --- a/src/testing/fstest/mapfs_test.go +++ b/src/testing/fstest/mapfs_test.go @@ -57,3 +57,69 @@ func TestMapFSFileInfoName(t *testing.T) { t.Errorf("MapFS FileInfo.Name want:\n%s\ngot:\n%s\n", want, got) } } + +func TestMapFSSymlink(t *testing.T) { + const fileContent = "If a program is too slow, it must have a loop.\n" + m := MapFS{ + "fortune/k/ken.txt": {Data: []byte(fileContent)}, + "dirlink": {Data: []byte("fortune/k"), Mode: fs.ModeSymlink}, + "linklink": {Data: []byte("dirlink"), Mode: fs.ModeSymlink}, + "ken.txt": {Data: []byte("dirlink/ken.txt"), Mode: fs.ModeSymlink}, + } + if err := TestFS(m, "fortune/k/ken.txt", "dirlink", "ken.txt", "linklink"); err != nil { + t.Error(err) + } + + gotData, err := fs.ReadFile(m, "ken.txt") + if string(gotData) != fileContent || err != nil { + t.Errorf("fs.ReadFile(m, \"ken.txt\") = %q, %v; want %q, ", gotData, err, fileContent) + } + gotLink, err := fs.ReadLink(m, "dirlink") + if want := "fortune/k"; gotLink != want || err != nil { + t.Errorf("fs.ReadLink(m, \"dirlink\") = %q, %v; want %q, ", gotLink, err, fileContent) + } + gotInfo, err := fs.Lstat(m, "dirlink") + if err != nil { + t.Errorf("fs.Lstat(m, \"dirlink\") = _, %v; want _, ", err) + } else { + if got, want := gotInfo.Name(), "dirlink"; got != want { + t.Errorf("fs.Lstat(m, \"dirlink\").Name() = %q; want %q", got, want) + } + if got, want := gotInfo.Mode(), fs.ModeSymlink; got != want { + t.Errorf("fs.Lstat(m, \"dirlink\").Mode() = %v; want %v", got, want) + } + } + gotInfo, err = fs.Stat(m, "dirlink") + if err != nil { + t.Errorf("fs.Stat(m, \"dirlink\") = _, %v; want _, ", err) + } else { + if got, want := gotInfo.Name(), "dirlink"; got != want { + t.Errorf("fs.Stat(m, \"dirlink\").Name() = %q; want %q", got, want) + } + if got, want := gotInfo.Mode(), fs.ModeDir|0555; got != want { + t.Errorf("fs.Stat(m, \"dirlink\").Mode() = %v; want %v", got, want) + } + } + gotInfo, err = fs.Lstat(m, "linklink") + if err != nil { + t.Errorf("fs.Lstat(m, \"linklink\") = _, %v; want _, ", err) + } else { + if got, want := gotInfo.Name(), "linklink"; got != want { + t.Errorf("fs.Lstat(m, \"linklink\").Name() = %q; want %q", got, want) + } + if got, want := gotInfo.Mode(), fs.ModeSymlink; got != want { + t.Errorf("fs.Lstat(m, \"linklink\").Mode() = %v; want %v", got, want) + } + } + gotInfo, err = fs.Stat(m, "linklink") + if err != nil { + t.Errorf("fs.Stat(m, \"linklink\") = _, %v; want _, ", err) + } else { + if got, want := gotInfo.Name(), "linklink"; got != want { + t.Errorf("fs.Stat(m, \"linklink\").Name() = %q; want %q", got, want) + } + if got, want := gotInfo.Mode(), fs.ModeDir|0555; got != want { + t.Errorf("fs.Stat(m, \"linklink\").Mode() = %v; want %v", got, want) + } + } +} diff --git a/src/testing/fstest/testfs.go b/src/testing/fstest/testfs.go index 76a2cff62d5608..72830a09a727bd 100644 --- a/src/testing/fstest/testfs.go +++ b/src/testing/fstest/testfs.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "io/fs" + "maps" "path" "slices" "strings" @@ -19,13 +20,16 @@ import ( // TestFS tests a file system implementation. // It walks the entire tree of files in fsys, // opening and checking that each file behaves correctly. +// Symbolic links are not followed, +// but their Lstat values are checked +// if the file system implements [fs.ReadLinkFS]. // It also checks that the file system contains at least the expected files. // As a special case, if no expected files are listed, fsys must be empty. // Otherwise, fsys must contain at least the listed files; it can also contain others. // The contents of fsys must not change concurrently with TestFS. // // If TestFS finds any misbehaviors, it returns either the first error or a -// list of errors. Use [errors.Is] or [errors.As] to inspect. +// list of errors. Use [errors.Is] or [errors.AsType] to inspect. // // Typical usage inside a test is: // @@ -71,13 +75,7 @@ func testFS(fsys fs.FS, expected ...string) error { } delete(found, ".") if len(expected) == 0 && len(found) > 0 { - var list []string - for k := range found { - if k != "." { - list = append(list, k) - } - } - slices.Sort(list) + list := slices.Sorted(maps.Keys(found)) if len(list) > 15 { list = append(list[:10], "...") } @@ -161,9 +159,14 @@ func (t *fsTester) checkDir(dir string) { path := prefix + name t.checkStat(path, info) t.checkOpen(path) - if info.IsDir() { + switch info.Type() { + case fs.ModeDir: t.checkDir(path) - } else { + case fs.ModeSymlink: + // No further processing. + // Avoid following symlinks to avoid potentially unbounded recursion. + t.files = append(t.files, path) + default: t.checkFile(path) } } @@ -445,6 +448,23 @@ func (t *fsTester) checkStat(path string, entry fs.DirEntry) { t.errorf("%s: fsys.Stat(...) = %s\n\twant %s", path, finfo2, finfo) } } + + if fsys, ok := t.fsys.(fs.ReadLinkFS); ok { + info2, err := fsys.Lstat(path) + if err != nil { + t.errorf("%s: fsys.Lstat: %v", path, err) + return + } + fientry2 := formatInfoEntry(info2) + if fentry != fientry2 { + t.errorf("%s: mismatch:\n\tentry = %s\n\tfsys.Lstat(...) = %s", path, fentry, fientry2) + } + feinfo := formatInfo(einfo) + finfo2 := formatInfo(info2) + if feinfo != finfo2 { + t.errorf("%s: mismatch:\n\tentry.Info() = %s\n\tfsys.Lstat(...) = %s\n", path, feinfo, finfo2) + } + } } // checkDirList checks that two directory lists contain the same files and file info. @@ -575,7 +595,7 @@ func (t *fsTester) checkFileRead(file, desc string, data1, data2 []byte) { } } -// checkBadPath checks that various invalid forms of file's name cannot be opened using t.fsys.Open. +// checkOpen validates file opening behavior by attempting to open and then close the given file path. func (t *fsTester) checkOpen(file string) { t.checkBadPath(file, "Open", func(file string) error { f, err := t.fsys.Open(file) diff --git a/src/testing/fstest/testfs_test.go b/src/testing/fstest/testfs_test.go index 2ef1053a01a646..a0561b6dfc661c 100644 --- a/src/testing/fstest/testfs_test.go +++ b/src/testing/fstest/testfs_test.go @@ -28,6 +28,9 @@ func TestSymlink(t *testing.T) { if err := os.Symlink(filepath.Join(tmp, "hello"), filepath.Join(tmp, "hello.link")); err != nil { t.Fatal(err) } + if err := os.Symlink("hello", filepath.Join(tmp, "hello_rel.link")); err != nil { + t.Fatal(err) + } if err := TestFS(tmpfs, "hello", "hello.link"); err != nil { t.Fatal(err) @@ -102,8 +105,12 @@ func TestTestFSWrappedErrors(t *testing.T) { // TestFS is expected to return a list of errors. // Enforce that the list can be extracted for browsing. - var errs interface{ Unwrap() []error } - if !errors.As(err, &errs) { + type wrapper interface { + error + Unwrap() []error + } + errs, ok := errors.AsType[wrapper](err) + if !ok { t.Errorf("caller should be able to extract the errors as a list: %#v", err) } else { for _, err := range errs.Unwrap() { diff --git a/src/testing/fuzz.go b/src/testing/fuzz.go index d561225b3c36e1..5867935b815899 100644 --- a/src/testing/fuzz.go +++ b/src/testing/fuzz.go @@ -5,6 +5,7 @@ package testing import ( + "context" "errors" "flag" "fmt" @@ -56,19 +57,19 @@ type InternalFuzzTarget struct { // find and report potential bugs in the code being tested. // // A fuzz test runs the seed corpus by default, which includes entries provided -// by (*F).Add and entries in the testdata/fuzz/ directory. After -// any necessary setup and calls to (*F).Add, the fuzz test must then call -// (*F).Fuzz to provide the fuzz target. See the testing package documentation +// by [F.Add] and entries in the testdata/fuzz/ directory. After +// any necessary setup and calls to [F.Add], the fuzz test must then call +// [F.Fuzz] to provide the fuzz target. See the testing package documentation // for an example, and see the [F.Fuzz] and [F.Add] method documentation for // details. // -// *F methods can only be called before (*F).Fuzz. Once the test is -// executing the fuzz target, only (*T) methods can be used. The only *F methods -// that are allowed in the (*F).Fuzz function are (*F).Failed and (*F).Name. +// *F methods can only be called before [F.Fuzz]. Once the test is +// executing the fuzz target, only [*T] methods can be used. The only *F methods +// that are allowed in the [F.Fuzz] function are [F.Failed] and [F.Name]. type F struct { common - fuzzContext *fuzzContext - testContext *testContext + fstate *fuzzState + tstate *testState // inFuzzFn is true when the fuzz function is running. Most F methods cannot // be called when inFuzzFn is true. @@ -162,29 +163,29 @@ func (f *F) Add(args ...any) { // supportedTypes represents all of the supported types which can be fuzzed. var supportedTypes = map[reflect.Type]bool{ - reflect.TypeOf(([]byte)("")): true, - reflect.TypeOf((string)("")): true, - reflect.TypeOf((bool)(false)): true, - reflect.TypeOf((byte)(0)): true, - reflect.TypeOf((rune)(0)): true, - reflect.TypeOf((float32)(0)): true, - reflect.TypeOf((float64)(0)): true, - reflect.TypeOf((int)(0)): true, - reflect.TypeOf((int8)(0)): true, - reflect.TypeOf((int16)(0)): true, - reflect.TypeOf((int32)(0)): true, - reflect.TypeOf((int64)(0)): true, - reflect.TypeOf((uint)(0)): true, - reflect.TypeOf((uint8)(0)): true, - reflect.TypeOf((uint16)(0)): true, - reflect.TypeOf((uint32)(0)): true, - reflect.TypeOf((uint64)(0)): true, + reflect.TypeFor[[]byte](): true, + reflect.TypeFor[string](): true, + reflect.TypeFor[bool](): true, + reflect.TypeFor[byte](): true, + reflect.TypeFor[rune](): true, + reflect.TypeFor[float32](): true, + reflect.TypeFor[float64](): true, + reflect.TypeFor[int](): true, + reflect.TypeFor[int8](): true, + reflect.TypeFor[int16](): true, + reflect.TypeFor[int32](): true, + reflect.TypeFor[int64](): true, + reflect.TypeFor[uint](): true, + reflect.TypeFor[uint8](): true, + reflect.TypeFor[uint16](): true, + reflect.TypeFor[uint32](): true, + reflect.TypeFor[uint64](): true, } // Fuzz runs the fuzz function, ff, for fuzz testing. If ff fails for a set of // arguments, those arguments will be added to the seed corpus. // -// ff must be a function with no return value whose first argument is *T and +// ff must be a function with no return value whose first argument is [*T] and // whose remaining arguments are the types to be fuzzed. // For example: // @@ -194,9 +195,9 @@ var supportedTypes = map[reflect.Type]bool{ // float64, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64. // More types may be supported in the future. // -// ff must not call any *F methods, e.g. (*F).Log, (*F).Error, (*F).Skip. Use -// the corresponding *T method instead. The only *F methods that are allowed in -// the (*F).Fuzz function are (*F).Failed and (*F).Name. +// ff must not call any [*F] methods, e.g. [F.Log], [F.Error], [F.Skip]. Use +// the corresponding [*T] method instead. The only [*F] methods that are allowed in +// the F.Fuzz function are [F.Failed] and [F.Name]. // // This function should be fast and deterministic, and its behavior should not // depend on shared state. No mutable input arguments, or pointers to them, @@ -206,7 +207,7 @@ var supportedTypes = map[reflect.Type]bool{ // // When fuzzing, F.Fuzz does not return until a problem is found, time runs out // (set with -fuzztime), or the test process is interrupted by a signal. F.Fuzz -// should be called exactly once, unless F.Skip or [F.Fail] is called beforehand. +// should be called exactly once, unless [F.Skip] or [F.Fail] is called beforehand. func (f *F) Fuzz(ff any) { if f.fuzzCalled { panic("testing: F.Fuzz called more than once") @@ -223,7 +224,7 @@ func (f *F) Fuzz(ff any) { if fnType.Kind() != reflect.Func { panic("testing: F.Fuzz must receive a function") } - if fnType.NumIn() < 2 || fnType.In(0) != reflect.TypeOf((*T)(nil)) { + if fnType.NumIn() < 2 || fnType.In(0) != reflect.TypeFor[*T]() { panic("testing: fuzz target must receive at least two arguments, where the first argument is a *T") } if fnType.NumOut() != 0 { @@ -244,22 +245,22 @@ func (f *F) Fuzz(ff any) { // corpus and entries declared with F.Add. // // Don't load the seed corpus if this is a worker process; we won't use it. - if f.fuzzContext.mode != fuzzWorker { + if f.fstate.mode != fuzzWorker { for _, c := range f.corpus { - if err := f.fuzzContext.deps.CheckCorpus(c.Values, types); err != nil { + if err := f.fstate.deps.CheckCorpus(c.Values, types); err != nil { // TODO(#48302): Report the source location of the F.Add call. f.Fatal(err) } } // Load seed corpus - c, err := f.fuzzContext.deps.ReadCorpus(filepath.Join(corpusDir, f.name), types) + c, err := f.fstate.deps.ReadCorpus(filepath.Join(corpusDir, f.name), types) if err != nil { f.Fatal(err) } for i := range c { c[i].IsSeed = true // these are all seed corpus values - if f.fuzzContext.mode == fuzzCoordinator { + if f.fstate.mode == fuzzCoordinator { // If this is the coordinator process, zero the values, since we don't need // to hold onto them. c[i].Values = nil @@ -285,14 +286,16 @@ func (f *F) Fuzz(ff any) { if e.Path != "" { testName = fmt.Sprintf("%s/%s", testName, filepath.Base(e.Path)) } - if f.testContext.isFuzzing { + if f.tstate.isFuzzing { // Don't preserve subtest names while fuzzing. If fn calls T.Run, // there will be a very large number of subtests with duplicate names, // which will use a large amount of memory. The subtest names aren't // useful since there's no way to re-run them deterministically. - f.testContext.match.clearSubNames() + f.tstate.match.clearSubNames() } + ctx, cancelCtx := context.WithCancel(f.ctx) + // Record the stack trace at the point of this call so that if the subtest // function - which runs in a separate stack - is marked as a helper, we can // continue walking the stack into the parent test. @@ -300,21 +303,24 @@ func (f *F) Fuzz(ff any) { n := runtime.Callers(2, pc[:]) t := &T{ common: common{ - barrier: make(chan bool), - signal: make(chan bool), - name: testName, - parent: &f.common, - level: f.level + 1, - creator: pc[:n], - chatty: f.chatty, + barrier: make(chan bool), + signal: make(chan bool), + name: testName, + parent: &f.common, + level: f.level + 1, + creator: pc[:n], + chatty: f.chatty, + ctx: ctx, + cancelCtx: cancelCtx, }, - context: f.testContext, + tstate: f.tstate, } if captureOut != nil { // t.parent aliases f.common. t.parent.w = captureOut } t.w = indenter{&t.common} + t.setOutputWriter() if t.chatty != nil { t.chatty.Updatef(t.name, "=== RUN %s\n", t.name) } @@ -328,9 +334,9 @@ func (f *F) Fuzz(ff any) { // we make sure it is called right before the tRunner function // exits, regardless of whether it was executed cleanly, panicked, // or if the fuzzFn called t.Fatal. - if f.testContext.isFuzzing { - defer f.fuzzContext.deps.SnapshotCoverage() - f.fuzzContext.deps.ResetCoverage() + if f.tstate.isFuzzing { + defer f.fstate.deps.SnapshotCoverage() + f.fstate.deps.ResetCoverage() } fn.Call(args) }) @@ -342,14 +348,14 @@ func (f *F) Fuzz(ff any) { return !t.Failed() } - switch f.fuzzContext.mode { + switch f.fstate.mode { case fuzzCoordinator: // Fuzzing is enabled, and this is the test process started by 'go test'. // Act as the coordinator process, and coordinate workers to perform the // actual fuzzing. corpusTargetDir := filepath.Join(corpusDir, f.name) cacheTargetDir := filepath.Join(*fuzzCacheDir, f.name) - err := f.fuzzContext.deps.CoordinateFuzzing( + err := f.fstate.deps.CoordinateFuzzing( fuzzDuration.d, int64(fuzzDuration.n), minimizeDuration.d, @@ -376,7 +382,7 @@ func (f *F) Fuzz(ff any) { case fuzzWorker: // Fuzzing is enabled, and this is a worker process. Follow instructions // from the coordinator. - if err := f.fuzzContext.deps.RunFuzzWorker(func(e corpusEntry) error { + if err := f.fstate.deps.RunFuzzWorker(func(e corpusEntry) error { // Don't write to f.w (which points to Stdout) if running from a // fuzz worker. This would become very verbose, particularly during // minimization. Return the error instead, and let the caller deal @@ -398,7 +404,7 @@ func (f *F) Fuzz(ff any) { // corpus now. for _, e := range f.corpus { name := fmt.Sprintf("%s/%s", f.name, filepath.Base(e.Path)) - if _, ok, _ := f.testContext.match.fullName(nil, name); ok { + if _, ok, _ := f.tstate.match.fullName(nil, name); ok { run(f.w, e) } } @@ -451,8 +457,8 @@ type fuzzCrashError interface { CrashPath() string } -// fuzzContext holds fields common to all fuzz tests. -type fuzzContext struct { +// fuzzState holds fields common to all fuzz tests. +type fuzzState struct { deps testDeps mode fuzzMode } @@ -486,9 +492,9 @@ func runFuzzTests(deps testDeps, fuzzTests []InternalFuzzTarget, deadline time.T break } - tctx := newTestContext(*parallel, m) - tctx.deadline = deadline - fctx := &fuzzContext{deps: deps, mode: seedCorpusOnly} + tstate := newTestState(*parallel, m) + tstate.deadline = deadline + fstate := &fuzzState{deps: deps, mode: seedCorpusOnly} root := common{w: os.Stdout} // gather output in one place if Verbose() { root.chatty = newChattyPrinter(root.w) @@ -497,7 +503,7 @@ func runFuzzTests(deps testDeps, fuzzTests []InternalFuzzTarget, deadline time.T if shouldFailFast() { break } - testName, matched, _ := tctx.match.fullName(nil, ft.Name) + testName, matched, _ := tstate.match.fullName(nil, ft.Name) if !matched { continue } @@ -508,19 +514,23 @@ func runFuzzTests(deps testDeps, fuzzTests []InternalFuzzTarget, deadline time.T continue } } + ctx, cancelCtx := context.WithCancel(context.Background()) f := &F{ common: common{ - signal: make(chan bool), - barrier: make(chan bool), - name: testName, - parent: &root, - level: root.level + 1, - chatty: root.chatty, + signal: make(chan bool), + barrier: make(chan bool), + name: testName, + parent: &root, + level: root.level + 1, + chatty: root.chatty, + ctx: ctx, + cancelCtx: cancelCtx, }, - testContext: tctx, - fuzzContext: fctx, + tstate: tstate, + fstate: fstate, } f.w = indenter{&f.common} + f.setOutputWriter() if f.chatty != nil { f.chatty.Updatef(f.name, "=== RUN %s\n", f.name) } @@ -554,17 +564,17 @@ func runFuzzing(deps testDeps, fuzzTests []InternalFuzzTarget) (ok bool) { return true } m := newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz", *skip) - tctx := newTestContext(1, m) - tctx.isFuzzing = true - fctx := &fuzzContext{ + tstate := newTestState(1, m) + tstate.isFuzzing = true + fstate := &fuzzState{ deps: deps, } root := common{w: os.Stdout} if *isFuzzWorker { root.w = io.Discard - fctx.mode = fuzzWorker + fstate.mode = fuzzWorker } else { - fctx.mode = fuzzCoordinator + fstate.mode = fuzzCoordinator } if Verbose() && !*isFuzzWorker { root.chatty = newChattyPrinter(root.w) @@ -573,7 +583,7 @@ func runFuzzing(deps testDeps, fuzzTests []InternalFuzzTarget) (ok bool) { var testName string var matched []string for i := range fuzzTests { - name, ok, _ := tctx.match.fullName(nil, fuzzTests[i].Name) + name, ok, _ := tstate.match.fullName(nil, fuzzTests[i].Name) if !ok { continue } @@ -590,19 +600,23 @@ func runFuzzing(deps testDeps, fuzzTests []InternalFuzzTarget) (ok bool) { return false } + ctx, cancelCtx := context.WithCancel(context.Background()) f := &F{ common: common{ - signal: make(chan bool), - barrier: nil, // T.Parallel has no effect when fuzzing. - name: testName, - parent: &root, - level: root.level + 1, - chatty: root.chatty, + signal: make(chan bool), + barrier: nil, // T.Parallel has no effect when fuzzing. + name: testName, + parent: &root, + level: root.level + 1, + chatty: root.chatty, + ctx: ctx, + cancelCtx: cancelCtx, }, - fuzzContext: fctx, - testContext: tctx, + fstate: fstate, + tstate: tstate, } f.w = indenter{&f.common} + f.setOutputWriter() if f.chatty != nil { f.chatty.Updatef(f.name, "=== RUN %s\n", f.name) } @@ -694,7 +708,7 @@ func fRunner(f *F, fn func(*F)) { // This only affects fuzz tests run as normal tests. // While fuzzing, T.Parallel has no effect, so f.sub is empty, and this // branch is not taken. f.barrier is nil in that case. - f.testContext.release() + f.tstate.release() close(f.barrier) // Wait for the subtests to complete. for _, sub := range f.sub { diff --git a/src/testing/helper_test.go b/src/testing/helper_test.go index da5622f85f6bbd..a698e79fa9cd69 100644 --- a/src/testing/helper_test.go +++ b/src/testing/helper_test.go @@ -23,15 +23,9 @@ func TestTBHelper(t *testing.T) { return } - testenv.MustHaveExec(t) t.Parallel() - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - - cmd := testenv.Command(t, exe, "-test.run=^TestTBHelper$") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^TestTBHelper$") cmd = testenv.CleanCmdEnv(cmd) cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=1") out, _ := cmd.CombinedOutput() @@ -66,15 +60,9 @@ func TestTBHelperParallel(t *testing.T) { return } - testenv.MustHaveExec(t) t.Parallel() - exe, err := os.Executable() - if err != nil { - t.Fatal(err) - } - - cmd := testenv.Command(t, exe, "-test.run=^TestTBHelperParallel$") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^TestTBHelperParallel$") cmd = testenv.CleanCmdEnv(cmd) cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=1") out, _ := cmd.CombinedOutput() diff --git a/src/testing/internal/testdeps/deps.go b/src/testing/internal/testdeps/deps.go index 6f42d4722ca00c..5ab377daeb61ed 100644 --- a/src/testing/internal/testdeps/deps.go +++ b/src/testing/internal/testdeps/deps.go @@ -66,6 +66,12 @@ func (TestDeps) ImportPath() string { return ImportPath } +var ModulePath string + +func (TestDeps) ModulePath() string { + return ModulePath +} + // testLog implements testlog.Interface, logging actions by package os. type testLog struct { mu sync.Mutex diff --git a/src/testing/iotest/reader_test.go b/src/testing/iotest/reader_test.go index 1d222372caf384..cecfbfce4920c4 100644 --- a/src/testing/iotest/reader_test.go +++ b/src/testing/iotest/reader_test.go @@ -238,7 +238,6 @@ func TestErrReader(t *testing.T) { } for _, tt := range cases { - tt := tt t.Run(tt.name, func(t *testing.T) { n, err := ErrReader(tt.err).Read(nil) if err != tt.err { diff --git a/src/testing/loop_test.go b/src/testing/loop_test.go new file mode 100644 index 00000000000000..71dbd35dc2ddf1 --- /dev/null +++ b/src/testing/loop_test.go @@ -0,0 +1,191 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testing + +import ( + "bytes" + "strings" + "time" +) + +// See also TestBenchmarkBLoop* in other files. + +func TestBenchmarkBLoop(t *T) { + var initialStart highPrecisionTime + var firstStart highPrecisionTime + var scaledStart highPrecisionTime + var runningEnd bool + runs := 0 + iters := 0 + firstBN := 0 + restBN := 0 + finalBN := 0 + bRet := Benchmark(func(b *B) { + initialStart = b.start + runs++ + for b.Loop() { + if iters == 0 { + firstStart = b.start + firstBN = b.N + } else { + restBN = max(restBN, b.N) + } + if iters == 1 { + scaledStart = b.start + } + iters++ + } + finalBN = b.N + runningEnd = b.timerOn + }) + // Verify that a b.Loop benchmark is invoked just once. + if runs != 1 { + t.Errorf("want runs == 1, got %d", runs) + } + // Verify that at least one iteration ran. + if iters == 0 { + t.Fatalf("no iterations ran") + } + // Verify that b.N, bRet.N, and the b.Loop() iteration count match. + if finalBN != iters || bRet.N != iters { + t.Errorf("benchmark iterations mismatch: %d loop iterations, final b.N=%d, bRet.N=%d", iters, finalBN, bRet.N) + } + // Verify that b.N was 0 inside the loop + if firstBN != 0 { + t.Errorf("want b.N == 0 on first iteration, got %d", firstBN) + } + if restBN != 0 { + t.Errorf("want b.N == 0 on subsequent iterations, got %d", restBN) + } + // Make sure the benchmark ran for an appropriate amount of time. + if bRet.T < benchTime.d { + t.Fatalf("benchmark ran for %s, want >= %s", bRet.T, benchTime.d) + } + // Verify that the timer is reset on the first loop, and then left alone. + if firstStart == initialStart { + t.Errorf("b.Loop did not reset the timer") + } + if scaledStart != firstStart { + t.Errorf("b.Loop stops and restarts the timer during iteration") + } + // Verify that it stopped the timer after the last loop. + if runningEnd { + t.Errorf("timer was still running after last iteration") + } +} + +func TestBenchmarkBLoopCheapEarlyTerminate(t *T) { + if Short() { + t.Skip("B.Loop test needs to run for > 1s to saturate 1e9 iterations") + } + runCnt := 0 + // Set the benchmark time high enough that we're likely to hit the 1B + // iteration limit even on very slow hardware. + // (on an AMD Ryzen 5900X, this benchmark runs in just over a second) + // + // Notably, the assertions below shouldn't fail if a test-run is slow + // enough that it doesn't saturate the limit. + const maxBenchTime = time.Second * 30 + res := Benchmark(func(b *B) { + // Set the benchmark time _much_ higher than required to hit 1e9 iterations. + b.benchTime.d = maxBenchTime + for b.Loop() { + runCnt++ + } + }) + if runCnt > maxBenchPredictIters { + t.Errorf("loop body ran more than max (%d) times: %d", maxBenchPredictIters, runCnt) + if res.T >= maxBenchTime { + t.Logf("cheap benchmark exhausted time budget: %s; ran for %s", maxBenchTime, res.T) + } + } + + if res.N != runCnt { + t.Errorf("disagreeing loop counts: res.N reported %d, while b.Loop() iterated %d times", res.N, runCnt) + } + + if res.N > maxBenchPredictIters { + t.Errorf("benchmark result claims more runs than max (%d) times: %d", maxBenchPredictIters, res.N) + } + +} + +func TestBenchmarkBLoopBreak(t *T) { + var bState *B + var bLog bytes.Buffer + bRet := Benchmark(func(b *B) { + // The Benchmark function provides no access to the failure state and + // discards the log, so capture the B and save its log. + bState = b + b.common.w = &bLog + + for i := 0; b.Loop(); i++ { + if i == 2 { + break + } + } + }) + if !bState.failed { + t.Errorf("benchmark should have failed") + } + const wantLog = "benchmark function returned without B.Loop" + if log := bLog.String(); !strings.Contains(log, wantLog) { + t.Errorf("missing error %q in output:\n%s", wantLog, log) + } + // A benchmark that exits early should not report its target iteration count + // because it's not meaningful. + if bRet.N != 0 { + t.Errorf("want N == 0, got %d", bRet.N) + } +} + +func TestBenchmarkBLoopError(t *T) { + // Test that a benchmark that exits early because of an error doesn't *also* + // complain that the benchmark exited early. + var bState *B + var bLog bytes.Buffer + bRet := Benchmark(func(b *B) { + bState = b + b.common.w = &bLog + + for i := 0; b.Loop(); i++ { + b.Error("error") + return + } + }) + if !bState.failed { + t.Errorf("benchmark should have failed") + } + const noWantLog = "benchmark function returned without B.Loop" + if log := bLog.String(); strings.Contains(log, noWantLog) { + t.Errorf("unexpected error %q in output:\n%s", noWantLog, log) + } + if bRet.N != 0 { + t.Errorf("want N == 0, got %d", bRet.N) + } +} + +func TestBenchmarkBLoopStop(t *T) { + var bState *B + var bLog bytes.Buffer + bRet := Benchmark(func(b *B) { + bState = b + b.common.w = &bLog + + for i := 0; b.Loop(); i++ { + b.StopTimer() + } + }) + if !bState.failed { + t.Errorf("benchmark should have failed") + } + const wantLog = "B.Loop called with timer stopped" + if log := bLog.String(); !strings.Contains(log, wantLog) { + t.Errorf("missing error %q in output:\n%s", wantLog, log) + } + if bRet.N != 0 { + t.Errorf("want N == 0, got %d", bRet.N) + } +} diff --git a/src/testing/newcover.go b/src/testing/newcover.go index ad2f622640e4d9..5a8d728831027f 100644 --- a/src/testing/newcover.go +++ b/src/testing/newcover.go @@ -8,49 +8,52 @@ package testing import ( "fmt" - "internal/goexperiment" "os" _ "unsafe" // for linkname ) -// cover2 variable stores the current coverage mode and a +// cover variable stores the current coverage mode and a // tear-down function to be called at the end of the testing run. -var cover2 struct { +var cover struct { mode string tearDown func(coverprofile string, gocoverdir string) (string, error) snapshotcov func() float64 } -// registerCover2 is invoked during "go test -cover" runs. +// registerCover is invoked during "go test -cover" runs. // It is used to record a 'tear down' function // (to be called when the test is complete) and the coverage mode. -func registerCover2(mode string, tearDown func(coverprofile string, gocoverdir string) (string, error), snapcov func() float64) { +func registerCover(mode string, tearDown func(coverprofile string, gocoverdir string) (string, error), snapcov func() float64) { if mode == "" { return } - cover2.mode = mode - cover2.tearDown = tearDown - cover2.snapshotcov = snapcov + cover.mode = mode + cover.tearDown = tearDown + cover.snapshotcov = snapcov } -// coverReport2 invokes a callback in _testmain.go that will +// coverReport reports the coverage percentage and +// writes a coverage profile if requested. +// This invokes a callback in _testmain.go that will // emit coverage data at the point where test execution is complete, // for "go test -cover" runs. -func coverReport2() { - if !goexperiment.CoverageRedesign { - panic("unexpected") - } - if errmsg, err := cover2.tearDown(*coverProfile, *gocoverdir); err != nil { +func coverReport() { + if errmsg, err := cover.tearDown(*coverProfile, *gocoverdir); err != nil { fmt.Fprintf(os.Stderr, "%s: %v\n", errmsg, err) os.Exit(2) } } -// coverage2 returns a rough "coverage percentage so far" -// number to support the testing.Coverage() function. -func coverage2() float64 { - if cover2.mode == "" { +// Coverage reports the current code coverage as a fraction in the range [0, 1]. +// If coverage is not enabled, Coverage returns 0. +// +// When running a large set of sequential test cases, checking Coverage after each one +// can be useful for identifying which test cases exercise new code paths. +// It is not a replacement for the reports generated by 'go test -cover' and +// 'go tool cover'. +func Coverage() float64 { + if cover.mode == "" { return 0.0 } - return cover2.snapshotcov() + return cover.snapshotcov() } diff --git a/src/testing/panic_test.go b/src/testing/panic_test.go index 6307b84a7a534b..01a34b08019edd 100644 --- a/src/testing/panic_test.go +++ b/src/testing/panic_test.go @@ -34,6 +34,7 @@ func TestPanic(t *testing.T) { want: ` --- FAIL: TestPanicHelper (N.NNs) panic_test.go:NNN: TestPanicHelper + TestPanicHelper `, }, { desc: "subtest panics", @@ -41,8 +42,10 @@ func TestPanic(t *testing.T) { want: ` --- FAIL: TestPanicHelper (N.NNs) panic_test.go:NNN: TestPanicHelper + TestPanicHelper --- FAIL: TestPanicHelper/1 (N.NNs) panic_test.go:NNN: TestPanicHelper/1 + TestPanicHelper/1 `, }, { desc: "subtest panics with cleanup", @@ -53,8 +56,10 @@ ran middle cleanup 1 ran outer cleanup --- FAIL: TestPanicHelper (N.NNs) panic_test.go:NNN: TestPanicHelper + TestPanicHelper --- FAIL: TestPanicHelper/1 (N.NNs) panic_test.go:NNN: TestPanicHelper/1 + TestPanicHelper/1 `, }, { desc: "subtest panics with outer cleanup panic", @@ -65,6 +70,7 @@ ran middle cleanup 1 ran outer cleanup --- FAIL: TestPanicHelper (N.NNs) panic_test.go:NNN: TestPanicHelper + TestPanicHelper `, }, { desc: "subtest panics with middle cleanup panic", @@ -75,8 +81,10 @@ ran middle cleanup 1 ran outer cleanup --- FAIL: TestPanicHelper (N.NNs) panic_test.go:NNN: TestPanicHelper + TestPanicHelper --- FAIL: TestPanicHelper/1 (N.NNs) panic_test.go:NNN: TestPanicHelper/1 + TestPanicHelper/1 `, }, { desc: "subtest panics with inner cleanup panic", @@ -87,8 +95,10 @@ ran middle cleanup 1 ran outer cleanup --- FAIL: TestPanicHelper (N.NNs) panic_test.go:NNN: TestPanicHelper + TestPanicHelper --- FAIL: TestPanicHelper/1 (N.NNs) panic_test.go:NNN: TestPanicHelper/1 + TestPanicHelper/1 `, }, { desc: "parallel subtest panics with cleanup", @@ -99,8 +109,10 @@ ran middle cleanup 1 ran outer cleanup --- FAIL: TestPanicHelper (N.NNs) panic_test.go:NNN: TestPanicHelper + TestPanicHelper --- FAIL: TestPanicHelper/1 (N.NNs) panic_test.go:NNN: TestPanicHelper/1 + TestPanicHelper/1 `, }, { desc: "parallel subtest panics with outer cleanup panic", @@ -111,6 +123,7 @@ ran middle cleanup 1 ran outer cleanup --- FAIL: TestPanicHelper (N.NNs) panic_test.go:NNN: TestPanicHelper + TestPanicHelper `, }, { desc: "parallel subtest panics with middle cleanup panic", @@ -121,8 +134,10 @@ ran middle cleanup 1 ran outer cleanup --- FAIL: TestPanicHelper (N.NNs) panic_test.go:NNN: TestPanicHelper + TestPanicHelper --- FAIL: TestPanicHelper/1 (N.NNs) panic_test.go:NNN: TestPanicHelper/1 + TestPanicHelper/1 `, }, { desc: "parallel subtest panics with inner cleanup panic", @@ -133,13 +148,15 @@ ran middle cleanup 1 ran outer cleanup --- FAIL: TestPanicHelper (N.NNs) panic_test.go:NNN: TestPanicHelper + TestPanicHelper --- FAIL: TestPanicHelper/1 (N.NNs) panic_test.go:NNN: TestPanicHelper/1 + TestPanicHelper/1 `, }} for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - cmd := exec.Command(os.Args[0], "-test.run=^TestPanicHelper$") + cmd := exec.Command(testenv.Executable(t), "-test.run=^TestPanicHelper$") cmd.Args = append(cmd.Args, tc.flags...) cmd.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=1") b, _ := cmd.CombinedOutput() @@ -165,6 +182,7 @@ func TestPanicHelper(t *testing.T) { return } t.Log(t.Name()) + t.Output().Write([]byte(t.Name())) if t.Name() == *testPanicTest { panic("panic") } @@ -180,7 +198,6 @@ func TestPanicHelper(t *testing.T) { } }) for i := 0; i < 3; i++ { - i := i t.Run(fmt.Sprintf("%v", i), func(t *testing.T) { chosen := t.Name() == *testPanicTest if chosen && *testPanicCleanup { @@ -195,6 +212,7 @@ func TestPanicHelper(t *testing.T) { t.Parallel() } t.Log(t.Name()) + t.Output().Write([]byte(t.Name())) if chosen { if *testPanicCleanup { t.Cleanup(func() { @@ -232,7 +250,7 @@ func TestMorePanic(t *testing.T) { } for _, tc := range testCases { - cmd := exec.Command(os.Args[0], tc.flags...) + cmd := exec.Command(testenv.Executable(t), tc.flags...) cmd.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=1") b, _ := cmd.CombinedOutput() got := string(b) diff --git a/src/testing/quick/quick.go b/src/testing/quick/quick.go index 8ef9cf7dda8211..6b3089d0499d9b 100644 --- a/src/testing/quick/quick.go +++ b/src/testing/quick/quick.go @@ -64,7 +64,7 @@ func Value(t reflect.Type, rand *rand.Rand) (value reflect.Value, ok bool) { // hint is used for shrinking as a function of indirection level so // that recursive data structures will terminate. func sizedValue(t reflect.Type, rand *rand.Rand, size int) (value reflect.Value, ok bool) { - if m, ok := reflect.Zero(t).Interface().(Generator); ok { + if m, ok := reflect.TypeAssert[Generator](reflect.Zero(t)); ok { return m.Generate(rand, size), true } diff --git a/src/testing/slogtest/example_test.go b/src/testing/slogtest/example_test.go index 0517a4b8574d8c..88fd2427b2e6d7 100644 --- a/src/testing/slogtest/example_test.go +++ b/src/testing/slogtest/example_test.go @@ -23,7 +23,7 @@ func Example_parsing() { results := func() []map[string]any { var ms []map[string]any - for _, line := range bytes.Split(buf.Bytes(), []byte{'\n'}) { + for line := range bytes.SplitSeq(buf.Bytes(), []byte{'\n'}) { if len(line) == 0 { continue } diff --git a/src/testing/slogtest/slogtest.go b/src/testing/slogtest/slogtest.go index 5c3aced65b62a6..7378b0613d8917 100644 --- a/src/testing/slogtest/slogtest.go +++ b/src/testing/slogtest/slogtest.go @@ -171,6 +171,22 @@ var cases = []testCase{ inGroup("G", missingKey("H")), }, }, + { + name: "nested-empty-group-record", + explanation: withSource("a Handler should not output nested groups if there are no attributes"), + f: func(l *slog.Logger) { + l.With("a", "b").WithGroup("G").With("c", "d").WithGroup("H").WithGroup("I").Info("msg") + }, + checks: []check{ + hasKey(slog.TimeKey), + hasKey(slog.LevelKey), + hasAttr(slog.MessageKey, "msg"), + hasAttr("a", "b"), + inGroup("G", hasAttr("c", "d")), + inGroup("G", missingKey("H")), + inGroup("G", missingKey("I")), + }, + }, { name: "resolve", explanation: withSource("a Handler should call Resolve on attribute values"), @@ -265,7 +281,7 @@ func TestHandler(h slog.Handler, results func() []map[string]any) error { if g, w := len(res), len(cases); g != w { return fmt.Errorf("got %d results, want %d", g, w) } - for i, got := range results() { + for i, got := range res { c := cases[i] for _, check := range c.checks { if problem := check(got); problem != "" { diff --git a/src/testing/sub_test.go b/src/testing/sub_test.go index 90c2afe605f41e..5d5573ccec506b 100644 --- a/src/testing/sub_test.go +++ b/src/testing/sub_test.go @@ -21,12 +21,11 @@ func init() { benchTime.d = 100 * time.Millisecond } -func TestTestContext(t *T) { +func TestTestState(t *T) { const ( add1 = 0 done = 1 ) - // After each of the calls are applied to the context, the type call struct { typ int // run or done // result from applying the call @@ -72,7 +71,7 @@ func TestTestContext(t *T) { }, }} for i, tc := range testCases { - ctx := &testContext{ + tstate := &testState{ startParallel: make(chan bool), maxParallel: tc.max, } @@ -88,18 +87,18 @@ func TestTestContext(t *T) { started := false switch call.typ { case add1: - signal := doCall(ctx.waitParallel) + signal := doCall(tstate.waitParallel) select { case <-signal: started = true - case ctx.startParallel <- true: + case tstate.startParallel <- true: <-signal } case done: - signal := doCall(ctx.release) + signal := doCall(tstate.release) select { case <-signal: - case <-ctx.startParallel: + case <-tstate.startParallel: started = true <-signal } @@ -107,11 +106,11 @@ func TestTestContext(t *T) { if started != call.started { t.Errorf("%d:%d:started: got %v; want %v", i, j, started, call.started) } - if ctx.running != call.running { - t.Errorf("%d:%d:running: got %v; want %v", i, j, ctx.running, call.running) + if tstate.running != call.running { + t.Errorf("%d:%d:running: got %v; want %v", i, j, tstate.running, call.running) } - if ctx.numWaiting != call.waiting { - t.Errorf("%d:%d:waiting: got %v; want %v", i, j, ctx.numWaiting, call.waiting) + if tstate.numWaiting != call.waiting { + t.Errorf("%d:%d:waiting: got %v; want %v", i, j, tstate.numWaiting, call.waiting) } } } @@ -281,7 +280,6 @@ func TestTRun(t *T) { t.Run("c", func(t *T) { t.Parallel() }) - }) }) }, @@ -306,7 +304,6 @@ func TestTRun(t *T) { time.Sleep(time.Nanosecond) }) } - }) } }) @@ -504,10 +501,104 @@ func TestTRun(t *T) { t2.FailNow() }) }, + }, { + desc: "buffered output gets flushed at test end", + ok: false, + output: ` +--- FAIL: buffered output gets flushed at test end (N.NNs) + --- FAIL: buffered output gets flushed at test end/#00 (N.NNs) + a + b`, + f: func(t *T) { + t.Run("", func(t *T) { + o := t.Output() + o.Write([]byte("a\n")) + o.Write([]byte("b")) + t.Fail() + }) + }, + }, { + desc: "output with chatty", + ok: true, + chatty: true, + output: ` +=== RUN output with chatty +=== RUN output with chatty/#00 + a + b +--- PASS: output with chatty (N.NNs) + --- PASS: output with chatty/#00 (N.NNs)`, + f: func(t *T) { + t.Run("", func(t *T) { + o := t.Output() + o.Write([]byte("a\n")) + o.Write([]byte("b")) + }) + }, + }, { + desc: "output with chatty and json", + ok: true, + chatty: true, + json: true, + output: ` +^V=== RUN output with chatty and json +^V=== RUN output with chatty and json/#00 + a + b +^V--- PASS: output with chatty and json/#00 (N.NNs) +^V=== NAME output with chatty and json +^V--- PASS: output with chatty and json (N.NNs) +^V=== NAME +`, + f: func(t *T) { + t.Run("", func(t *T) { + o := t.Output() + o.Write([]byte("a\n")) + o.Write([]byte("b")) + }) + }, + }, { + desc: "output in finished sub test outputs to parent", + ok: false, + output: ` + --- FAIL: output in finished sub test outputs to parent (N.NNs) + message2 + message1 + sub_test.go:NNN: error`, + f: func(t *T) { + ch := make(chan bool) + t.Run("sub", func(t2 *T) { + go func() { + <-ch + t2.Output().Write([]byte("message1\n")) + ch <- true + }() + }) + t.Output().Write([]byte("message2\n")) + ch <- true + <-ch + t.Errorf("error") + }, + }, { + desc: "newline between buffered log and log", + ok: false, + output: ` +--- FAIL: newline between buffered log and log (N.NNs) + --- FAIL: newline between buffered log and log/#00 (N.NNs) + buffered message + sub_test.go:NNN: log`, + f: func(t *T) { + t.Run("", func(t *T) { + o := t.Output() + o.Write([]byte("buffered message")) + t.Log("log") + t.Fail() + }) + }, }} for _, tc := range testCases { t.Run(tc.desc, func(t *T) { - ctx := newTestContext(tc.maxPar, allMatcher()) + tstate := newTestState(tc.maxPar, allMatcher()) buf := &strings.Builder{} root := &T{ common: common{ @@ -516,14 +607,14 @@ func TestTRun(t *T) { name: "", w: buf, }, - context: ctx, + tstate: tstate, } if tc.chatty { root.chatty = newChattyPrinter(root.w) root.chatty.json = tc.json } ok := root.Run(tc.desc, tc.f) - ctx.release() + tstate.release() if ok != tc.ok { t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, tc.ok) @@ -531,8 +622,8 @@ func TestTRun(t *T) { if ok != !root.Failed() { t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed()) } - if ctx.running != 0 || ctx.numWaiting != 0 { - t.Errorf("%s:running and waiting non-zero: got %d and %d", tc.desc, ctx.running, ctx.numWaiting) + if tstate.running != 0 || tstate.numWaiting != 0 { + t.Errorf("%s:running and waiting non-zero: got %d and %d", tc.desc, tstate.running, tstate.numWaiting) } got := strings.TrimSpace(buf.String()) want := strings.TrimSpace(tc.output) @@ -748,7 +839,7 @@ func TestBenchmarkOutput(t *T) { } func TestBenchmarkStartsFrom1(t *T) { - var first = true + first := true Benchmark(func(b *B) { if first && b.N != 1 { panic(fmt.Sprintf("Benchmark() first N=%v; want 1", b.N)) @@ -758,7 +849,7 @@ func TestBenchmarkStartsFrom1(t *T) { } func TestBenchmarkReadMemStatsBeforeFirstRun(t *T) { - var first = true + first := true Benchmark(func(b *B) { if first && (b.startAllocs == 0 || b.startBytes == 0) { panic("ReadMemStats not called before first run") @@ -790,8 +881,8 @@ func TestRacyOutput(t *T) { } root := &T{ - common: common{w: &funcWriter{raceDetector}}, - context: newTestContext(1, allMatcher()), + common: common{w: &funcWriter{raceDetector}}, + tstate: newTestState(1, allMatcher()), } root.chatty = newChattyPrinter(root.w) root.Run("", func(t *T) { @@ -815,7 +906,7 @@ func TestRacyOutput(t *T) { // The late log message did not include the test name. Issue 29388. func TestLogAfterComplete(t *T) { - ctx := newTestContext(1, allMatcher()) + tstate := newTestState(1, allMatcher()) var buf bytes.Buffer t1 := &T{ common: common{ @@ -824,7 +915,7 @@ func TestLogAfterComplete(t *T) { signal: make(chan bool, 1), w: &buf, }, - context: ctx, + tstate: tstate, } c1 := make(chan bool) @@ -841,7 +932,7 @@ func TestLogAfterComplete(t *T) { } s, ok := p.(string) if !ok { - c2 <- fmt.Sprintf("subtest panic with unexpected value %v", p) + c2 <- fmt.Sprintf("subtest panic with unexpected value %v of type %T", p, p) return } const want = "Log in goroutine after TestLateLog has completed: log after test" @@ -897,7 +988,6 @@ func TestConcurrentCleanup(t *T) { var wg sync.WaitGroup wg.Add(2) for i := 0; i < 2; i++ { - i := i go func() { t.Cleanup(func() { // Although the calls to Cleanup are concurrent, the functions passed @@ -990,3 +1080,188 @@ func TestNestedCleanup(t *T) { t.Errorf("unexpected cleanup count: got %d want 3", ranCleanup) } } + +// TestOutput checks that log messages are written, +// formatted and buffered as expected by Output. It +// checks both the chatty and non-chatty cases. +func TestOutput(t *T) { + tstate := newTestState(1, allMatcher()) + root := &T{ + tstate: tstate, + } + root.setOutputWriter() + o := root.Output() + + // Chatty case + tstateChatty := newTestState(1, allMatcher()) + bufChatty := &strings.Builder{} + rootChatty := &T{ + common: common{ + w: bufChatty, + }, + tstate: tstateChatty, + } + rootChatty.setOutputWriter() + rootChatty.chatty = newChattyPrinter(rootChatty.w) + oChatty := rootChatty.Output() + + testCases := []struct { + in string + out string + buf string + }{{ + in: "a", + out: "", + buf: "a", + }, { + in: "b", + out: "", + buf: "ab", + }, { + in: "\n", + out: " ab\n", + buf: "", + }, { + in: "\nc", + out: " ab\n \n", + buf: "c", + }, { + in: "d", + out: " ab\n \n", + buf: "cd", + }} + for _, tc := range testCases { + o.Write([]byte(tc.in)) + if string(root.output) != tc.out { + t.Errorf("output:\ngot:\n%s\nwant:\n%s", root.output, tc.out) + } + if string(root.o.partial) != tc.buf { + t.Errorf("buffer:\ngot:\n%s\nwant:\n%s", root.o.partial, tc.buf) + } + + // Chatty case + oChatty.Write([]byte(tc.in)) + if got := bufChatty.String(); got != tc.out { + t.Errorf("output:\ngot:\n%s\nwant:\n%s", got, tc.out) + } + } +} + +// TestOutputAfterComplete ensures that Output panics +// if called after a test function returns. +func TestOutputAfterComplete(t *T) { + tstate := newTestState(1, allMatcher()) + var buf bytes.Buffer + t1 := &T{ + common: common{ + // Use a buffered channel so that tRunner can write + // to it although nothing is reading from it. + signal: make(chan bool, 1), + w: &buf, + }, + tstate: tstate, + } + + c1 := make(chan bool) + c2 := make(chan string) + tRunner(t1, func(t *T) { + t.Run("TestLateOutput", func(t *T) { + go func() { + defer close(c2) + defer func() { + p := recover() + if p == nil { + c2 <- "subtest did not panic" + return + } + s, ok := p.(string) + if !ok { + c2 <- fmt.Sprintf("subtest panic with unexpected value %v of type %T", p, p) + return + } + const want = "Output called after TestLateOutput has completed" + if !strings.Contains(s, want) { + c2 <- fmt.Sprintf("subtest panic %q does not contain %q", s, want) + } + }() + + <-c1 + t.Output() + }() + }) + }) + close(c1) + + if s := <-c2; s != "" { + t.Error(s) + } +} + +// TestOutputWriteAfterComplete ensures that Write panics +// if called on t.Output() of a finished test t. +func TestOutputWriteAfterComplete(t *T) { + tstate := newTestState(1, allMatcher()) + var buf bytes.Buffer + t1 := &T{ + common: common{ + // Use a buffered channel so that tRunner can write + // to it although nothing is reading from it. + signal: make(chan bool, 1), + w: &buf, + }, + tstate: tstate, + } + + c1 := make(chan bool) + c2 := make(chan string) + tRunner(t1, func(t *T) { + t.Run("TestLateWrite", func(t *T) { + o := t.Output() + go func() { + defer close(c2) + defer func() { + p := recover() + if p == nil { + c2 <- "subtest did not panic" + return + } + s, ok := p.(string) + if !ok { + c2 <- fmt.Sprintf("subtest panic with unexpected value %v of type %T", p, p) + return + } + const want = "Write called after TestLateWrite has completed" + if !strings.Contains(s, want) { + c2 <- fmt.Sprintf("subtest panic %q does not contain %q", s, want) + } + }() + + <-c1 + o.Write([]byte("write after test")) + }() + }) + }) + close(c1) + + if s := <-c2; s != "" { + t.Error(s) + } +} + +// Verify that logging to an inactive top-level testing.T does not panic. +// These tests can run in either order. + +func TestOutputEscape1(t *T) { testOutputEscape(t) } +func TestOutputEscape2(t *T) { testOutputEscape(t) } + +var global *T + +func testOutputEscape(t *T) { + if global == nil { + // Store t in a global, to set up for the second execution. + global = t + } else { + // global is inactive here. + global.Log("hello") + } +} diff --git a/src/testing/synctest/example_test.go b/src/testing/synctest/example_test.go new file mode 100644 index 00000000000000..a86d87fcecdca1 --- /dev/null +++ b/src/testing/synctest/example_test.go @@ -0,0 +1,160 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package synctest_test + +import ( + "bufio" + "bytes" + "context" + "io" + "net" + "net/http" + "strings" + "testing" + "testing/synctest" + "time" +) + +// Keep the following tests in sync with the package documentation. + +func TestTime(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + start := time.Now() // always midnight UTC 2000-01-01 + go func() { + time.Sleep(1 * time.Nanosecond) + t.Log(time.Since(start)) // always logs "1ns" + }() + time.Sleep(2 * time.Nanosecond) // the AfterFunc will run before this Sleep returns + t.Log(time.Since(start)) // always logs "2ns" + }) +} + +func TestWait(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + done := false + go func() { + done = true + }() + // Wait will block until the goroutine above has finished. + synctest.Wait() + t.Log(done) // always logs "true" + }) +} + +func TestContextAfterFunc(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + // Create a context.Context which can be canceled. + ctx, cancel := context.WithCancel(t.Context()) + + // context.AfterFunc registers a function to be called + // when a context is canceled. + afterFuncCalled := false + context.AfterFunc(ctx, func() { + afterFuncCalled = true + }) + + // The context has not been canceled, so the AfterFunc is not called. + synctest.Wait() + if afterFuncCalled { + t.Fatalf("before context is canceled: AfterFunc called") + } + + // Cancel the context and wait for the AfterFunc to finish executing. + // Verify that the AfterFunc ran. + cancel() + synctest.Wait() + if !afterFuncCalled { + t.Fatalf("after context is canceled: AfterFunc not called") + } + }) +} + +func TestContextWithTimeout(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + // Create a context.Context which is canceled after a timeout. + const timeout = 5 * time.Second + ctx, cancel := context.WithTimeout(t.Context(), timeout) + defer cancel() + + // Wait just less than the timeout. + time.Sleep(timeout - time.Nanosecond) + synctest.Wait() + if err := ctx.Err(); err != nil { + t.Fatalf("before timeout: ctx.Err() = %v, want nil\n", err) + } + + // Wait the rest of the way until the timeout. + time.Sleep(time.Nanosecond) + synctest.Wait() + if err := ctx.Err(); err != context.DeadlineExceeded { + t.Fatalf("after timeout: ctx.Err() = %v, want DeadlineExceeded\n", err) + } + }) +} + +func TestHTTPTransport100Continue(t *testing.T) { + synctest.Test(t, func(*testing.T) { + // Create an in-process fake network connection. + // We cannot use a loopback network connection for this test, + // because goroutines blocked on network I/O prevent a synctest + // bubble from becoming idle. + srvConn, cliConn := net.Pipe() + defer cliConn.Close() + defer srvConn.Close() + + tr := &http.Transport{ + // Use the fake network connection created above. + DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + return cliConn, nil + }, + // Enable "Expect: 100-continue" handling. + ExpectContinueTimeout: 5 * time.Second, + } + + // Send a request with the "Expect: 100-continue" header set. + // Send it in a new goroutine, since it won't complete until the end of the test. + body := "request body" + go func() { + req, _ := http.NewRequest("PUT", "http://test.tld/", strings.NewReader(body)) + req.Header.Set("Expect", "100-continue") + resp, err := tr.RoundTrip(req) + if err != nil { + t.Errorf("RoundTrip: unexpected error %v\n", err) + } else { + resp.Body.Close() + } + }() + + // Read the request headers sent by the client. + req, err := http.ReadRequest(bufio.NewReader(srvConn)) + if err != nil { + t.Fatalf("ReadRequest: %v\n", err) + } + + // Start a new goroutine copying the body sent by the client into a buffer. + // Wait for all goroutines in the bubble to block and verify that we haven't + // read anything from the client yet. + var gotBody bytes.Buffer + go io.Copy(&gotBody, req.Body) + synctest.Wait() + if got, want := gotBody.String(), ""; got != want { + t.Fatalf("before sending 100 Continue, read body: %q, want %q\n", got, want) + } + + // Write a "100 Continue" response to the client and verify that + // it sends the request body. + srvConn.Write([]byte("HTTP/1.1 100 Continue\r\n\r\n")) + synctest.Wait() + if got, want := gotBody.String(), body; got != want { + t.Fatalf("after sending 100 Continue, read body: %q, want %q\n", got, want) + } + + // Finish up by sending the "200 OK" response to conclude the request. + srvConn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n")) + + // We started several goroutines during the test. + // The synctest.Test call will wait for all of them to exit before returning. + }) +} diff --git a/src/testing/synctest/helper_test.go b/src/testing/synctest/helper_test.go new file mode 100644 index 00000000000000..7547d3eac6991d --- /dev/null +++ b/src/testing/synctest/helper_test.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package synctest_test + +import "testing" + +// helperLog is a t.Helper which logs. +// Since it is a helper, the log prefix should contain +// the caller's file, not helper_test.go. +func helperLog(t *testing.T, s string) { + t.Helper() + t.Log(s) +} diff --git a/src/testing/synctest/synctest.go b/src/testing/synctest/synctest.go new file mode 100644 index 00000000000000..9f499515b8881a --- /dev/null +++ b/src/testing/synctest/synctest.go @@ -0,0 +1,311 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package synctest provides support for testing concurrent code. +// +// The [Test] function runs a function in an isolated "bubble". +// Any goroutines started within the bubble are also part of the bubble. +// +// Each test should be entirely self-contained: +// The following guidelines should apply to most tests: +// +// - Avoid interacting with goroutines not started from within the test. +// - Avoid using the network. Use a fake network implementation as needed. +// - Avoid interacting with external processes. +// - Avoid leaking goroutines in background tasks. +// +// # Time +// +// Within a bubble, the [time] package uses a fake clock. +// Each bubble has its own clock. +// The initial time is midnight UTC 2000-01-01. +// +// Time in a bubble only advances when every goroutine in the +// bubble is durably blocked. +// See below for the exact definition of "durably blocked". +// +// For example, this test runs immediately rather than taking +// two seconds: +// +// func TestTime(t *testing.T) { +// synctest.Test(t, func(t *testing.T) { +// start := time.Now() // always midnight UTC 2000-01-01 +// go func() { +// time.Sleep(1 * time.Second) +// t.Log(time.Since(start)) // always logs "1s" +// }() +// time.Sleep(2 * time.Second) // the goroutine above will run before this Sleep returns +// t.Log(time.Since(start)) // always logs "2s" +// }) +// } +// +// Time stops advancing when the root goroutine of the bubble exits. +// +// # Blocking +// +// A goroutine in a bubble is "durably blocked" when it is blocked +// and can only be unblocked by another goroutine in the same bubble. +// A goroutine which can be unblocked by an event from outside its +// bubble is not durably blocked. +// +// The [Wait] function blocks until all other goroutines in the +// bubble are durably blocked. +// +// For example: +// +// func TestWait(t *testing.T) { +// synctest.Test(t, func(t *testing.T) { +// done := false +// go func() { +// done = true +// }() +// // Wait will block until the goroutine above has finished. +// synctest.Wait() +// t.Log(done) // always logs "true" +// }) +// } +// +// When every goroutine in a bubble is durably blocked: +// +// - [Wait] returns, if it has been called. +// - Otherwise, time advances to the next time that will +// unblock at least one goroutine, if there is such a time +// and the root goroutine of the bubble has not exited. +// - Otherwise, there is a deadlock and [Test] panics. +// +// The following operations durably block a goroutine: +// +// - a blocking send or receive on a channel created within the bubble +// - a blocking select statement where every case is a channel created +// within the bubble +// - [sync.Cond.Wait] +// - [sync.WaitGroup.Wait], when [sync.WaitGroup.Add] was called within the bubble +// - [time.Sleep] +// +// Operations not in the above list are not durably blocking. +// In particular, the following operations may block a goroutine, +// but are not durably blocking because the goroutine can be unblocked +// by an event occurring outside its bubble: +// +// - locking a [sync.Mutex] or [sync.RWMutex] +// - blocking on I/O, such as reading from a network socket +// - system calls +// +// # Isolation +// +// A channel, [time.Timer], or [time.Ticker] created within a bubble +// is associated with it. Operating on a bubbled channel, timer, or +// ticker from outside the bubble panics. +// +// A [sync.WaitGroup] becomes associated with a bubble on the first +// call to Add or Go. Once a WaitGroup is associated with a bubble, +// calling Add or Go from outside that bubble is a fatal error. +// (As a technical limitation, a WaitGroup defined as a package +// variable, such as "var wg sync.WaitGroup", cannot be associated +// with a bubble and operations on it may not be durably blocking. +// This limitation does not apply to a *WaitGroup stored in a +// package variable, such as "var wg = new(sync.WaitGroup)".) +// +// [sync.Cond.Wait] is durably blocking. Waking a goroutine in a bubble +// blocked on Cond.Wait from outside the bubble is a fatal error. +// +// Cleanup functions and finalizers registered with +// [runtime.AddCleanup] and [runtime.SetFinalizer] +// run outside of any bubble. +// +// # Example: Context.AfterFunc +// +// This example demonstrates testing the [context.AfterFunc] function. +// +// AfterFunc registers a function to execute in a new goroutine +// after a context is canceled. +// +// The test verifies that the function is not run before the context is canceled, +// and is run after the context is canceled. +// +// func TestContextAfterFunc(t *testing.T) { +// synctest.Test(t, func(t *testing.T) { +// // Create a context.Context which can be canceled. +// ctx, cancel := context.WithCancel(t.Context()) +// +// // context.AfterFunc registers a function to be called +// // when a context is canceled. +// afterFuncCalled := false +// context.AfterFunc(ctx, func() { +// afterFuncCalled = true +// }) +// +// // The context has not been canceled, so the AfterFunc is not called. +// synctest.Wait() +// if afterFuncCalled { +// t.Fatalf("before context is canceled: AfterFunc called") +// } +// +// // Cancel the context and wait for the AfterFunc to finish executing. +// // Verify that the AfterFunc ran. +// cancel() +// synctest.Wait() +// if !afterFuncCalled { +// t.Fatalf("after context is canceled: AfterFunc not called") +// } +// }) +// } +// +// # Example: Context.WithTimeout +// +// This example demonstrates testing the [context.WithTimeout] function. +// +// WithTimeout creates a context which is canceled after a timeout. +// +// The test verifies that the context is not canceled before the timeout expires, +// and is canceled after the timeout expires. +// +// func TestContextWithTimeout(t *testing.T) { +// synctest.Test(t, func(t *testing.T) { +// // Create a context.Context which is canceled after a timeout. +// const timeout = 5 * time.Second +// ctx, cancel := context.WithTimeout(t.Context(), timeout) +// defer cancel() +// +// // Wait just less than the timeout. +// time.Sleep(timeout - time.Nanosecond) +// synctest.Wait() +// if err := ctx.Err(); err != nil { +// t.Fatalf("before timeout: ctx.Err() = %v, want nil\n", err) +// } +// +// // Wait the rest of the way until the timeout. +// time.Sleep(time.Nanosecond) +// synctest.Wait() +// if err := ctx.Err(); err != context.DeadlineExceeded { +// t.Fatalf("after timeout: ctx.Err() = %v, want DeadlineExceeded\n", err) +// } +// }) +// } +// +// # Example: HTTP 100 Continue +// +// This example demonstrates testing [http.Transport]'s 100 Continue handling. +// +// An HTTP client sending a request can include an "Expect: 100-continue" header +// to tell the server that the client has additional data to send. +// The server may then respond with an 100 Continue information response +// to request the data, or some other status to tell the client the data is not needed. +// For example, a client uploading a large file might use this feature to confirm +// that the server is willing to accept the file before sending it. +// +// This test confirms that when sending an "Expect: 100-continue" header +// the HTTP client does not send a request's content before the server requests it, +// and that it does send the content after receiving a 100 Continue response. +// +// func TestHTTPTransport100Continue(t *testing.T) { +// synctest.Test(t, func(*testing.T) { +// // Create an in-process fake network connection. +// // We cannot use a loopback network connection for this test, +// // because goroutines blocked on network I/O prevent a synctest +// // bubble from becoming idle. +// srvConn, cliConn := net.Pipe() +// defer cliConn.Close() +// defer srvConn.Close() +// +// tr := &http.Transport{ +// // Use the fake network connection created above. +// DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { +// return cliConn, nil +// }, +// // Enable "Expect: 100-continue" handling. +// ExpectContinueTimeout: 5 * time.Second, +// } +// +// // Send a request with the "Expect: 100-continue" header set. +// // Send it in a new goroutine, since it won't complete until the end of the test. +// body := "request body" +// go func() { +// req, _ := http.NewRequest("PUT", "http://test.tld/", strings.NewReader(body)) +// req.Header.Set("Expect", "100-continue") +// resp, err := tr.RoundTrip(req) +// if err != nil { +// t.Errorf("RoundTrip: unexpected error %v\n", err) +// } else { +// resp.Body.Close() +// } +// }() +// +// // Read the request headers sent by the client. +// req, err := http.ReadRequest(bufio.NewReader(srvConn)) +// if err != nil { +// t.Fatalf("ReadRequest: %v\n", err) +// } +// +// // Start a new goroutine copying the body sent by the client into a buffer. +// // Wait for all goroutines in the bubble to block and verify that we haven't +// // read anything from the client yet. +// var gotBody bytes.Buffer +// go io.Copy(&gotBody, req.Body) +// synctest.Wait() +// if got, want := gotBody.String(), ""; got != want { +// t.Fatalf("before sending 100 Continue, read body: %q, want %q\n", got, want) +// } +// +// // Write a "100 Continue" response to the client and verify that +// // it sends the request body. +// srvConn.Write([]byte("HTTP/1.1 100 Continue\r\n\r\n")) +// synctest.Wait() +// if got, want := gotBody.String(), body; got != want { +// t.Fatalf("after sending 100 Continue, read body: %q, want %q\n", got, want) +// } +// +// // Finish up by sending the "200 OK" response to conclude the request. +// srvConn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n")) +// +// // We started several goroutines during the test. +// // The synctest.Test call will wait for all of them to exit before returning. +// }) +// } +package synctest + +import ( + "internal/synctest" + "testing" + _ "unsafe" // for linkname +) + +// Test executes f in a new bubble. +// +// Test waits for all goroutines in the bubble to exit before returning. +// If the goroutines in the bubble become deadlocked, the test fails. +// +// Test must not be called from within a bubble. +// +// The [*testing.T] provided to f has the following properties: +// +// - T.Cleanup functions run inside the bubble, +// immediately before Test returns. +// - T.Context returns a [context.Context] with a Done channel +// associated with the bubble. +// - T.Run, T.Parallel, and T.Deadline must not be called. +func Test(t *testing.T, f func(*testing.T)) { + var ok bool + synctest.Run(func() { + ok = testingSynctestTest(t, f) + }) + if !ok { + // Fail the test outside the bubble, + // so test durations get set using real time. + t.FailNow() + } +} + +//go:linkname testingSynctestTest testing/synctest.testingSynctestTest +func testingSynctestTest(t *testing.T, f func(*testing.T)) bool + +// Wait blocks until every goroutine within the current bubble, +// other than the current goroutine, is durably blocked. +// +// Wait must not be called from outside a bubble. +// Wait must not be called concurrently by multiple goroutines +// in the same bubble. +func Wait() { + synctest.Wait() +} diff --git a/src/testing/synctest/synctest_test.go b/src/testing/synctest/synctest_test.go new file mode 100644 index 00000000000000..62f10d1b819b13 --- /dev/null +++ b/src/testing/synctest/synctest_test.go @@ -0,0 +1,190 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package synctest_test + +import ( + "fmt" + "internal/testenv" + "os" + "regexp" + "testing" + "testing/synctest" + "time" +) + +// Tests for interactions between synctest bubbles and the testing package. +// Other bubble behaviors are tested in internal/synctest. + +func TestSuccess(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + }) +} + +func TestFatal(t *testing.T) { + runTest(t, nil, func() { + synctest.Test(t, func(t *testing.T) { + t.Fatal("fatal") + }) + }, `^--- FAIL: TestFatal.* + synctest_test.go:.* fatal +FAIL +$`) +} + +func TestError(t *testing.T) { + runTest(t, nil, func() { + synctest.Test(t, func(t *testing.T) { + t.Error("error") + }) + }, `^--- FAIL: TestError.* + synctest_test.go:.* error +FAIL +$`) +} + +func TestVerboseError(t *testing.T) { + runTest(t, []string{"-test.v"}, func() { + synctest.Test(t, func(t *testing.T) { + t.Error("error") + }) + }, `^=== RUN TestVerboseError + synctest_test.go:.* error +--- FAIL: TestVerboseError.* +FAIL +$`) +} + +func TestSkip(t *testing.T) { + runTest(t, nil, func() { + synctest.Test(t, func(t *testing.T) { + t.Skip("skip") + }) + }, `^PASS +$`) +} + +func TestVerboseSkip(t *testing.T) { + runTest(t, []string{"-test.v"}, func() { + synctest.Test(t, func(t *testing.T) { + t.Skip("skip") + }) + }, `^=== RUN TestVerboseSkip + synctest_test.go:.* skip +--- PASS: TestVerboseSkip.* +PASS +$`) +} + +func TestCleanup(t *testing.T) { + done := false + synctest.Test(t, func(t *testing.T) { + ch := make(chan struct{}) + t.Cleanup(func() { + // This cleanup function should execute inside the test's bubble. + // (If it doesn't the runtime will panic.) + close(ch) + }) + // synctest.Test will wait for this goroutine to exit before returning. + // The cleanup function signals the goroutine to exit before the wait starts. + go func() { + <-ch + done = true + }() + }) + if !done { + t.Fatalf("background goroutine did not return") + } +} + +func TestContext(t *testing.T) { + state := "not started" + synctest.Test(t, func(t *testing.T) { + go func() { + state = "waiting on context" + <-t.Context().Done() + state = "done" + }() + // Wait blocks until the goroutine above is blocked on t.Context().Done(). + synctest.Wait() + if got, want := state, "waiting on context"; got != want { + t.Fatalf("state = %q, want %q", got, want) + } + }) + // t.Context() is canceled before the test completes, + // and synctest.Test does not return until the goroutine has set its state to "done". + if got, want := state, "done"; got != want { + t.Fatalf("state = %q, want %q", got, want) + } +} + +func TestDeadline(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + defer wantPanic(t, "testing: t.Deadline called inside synctest bubble") + _, _ = t.Deadline() + }) +} + +func TestParallel(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + defer wantPanic(t, "testing: t.Parallel called inside synctest bubble") + t.Parallel() + }) +} + +func TestRun(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + defer wantPanic(t, "testing: t.Run called inside synctest bubble") + t.Run("subtest", func(t *testing.T) { + }) + }) +} + +func TestHelper(t *testing.T) { + runTest(t, []string{"-test.v"}, func() { + synctest.Test(t, func(t *testing.T) { + helperLog(t, "log in helper") + }) + }, `^=== RUN TestHelper + synctest_test.go:.* log in helper +--- PASS: TestHelper.* +PASS +$`) +} + +func wantPanic(t *testing.T, want string) { + if e := recover(); e != nil { + if got := fmt.Sprint(e); got != want { + t.Errorf("got panic message %q, want %q", got, want) + } + } else { + t.Errorf("got no panic, want one") + } +} + +func runTest(t *testing.T, args []string, f func(), pattern string) { + if os.Getenv("GO_WANT_HELPER_PROCESS") == "1" { + f() + return + } + t.Helper() + re := regexp.MustCompile(pattern) + testenv.MustHaveExec(t) + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^"+regexp.QuoteMeta(t.Name())+"$", "-test.count=1") + cmd.Args = append(cmd.Args, args...) + cmd = testenv.CleanCmdEnv(cmd) + cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=1") + out, _ := cmd.CombinedOutput() + if !re.Match(out) { + t.Errorf("got output:\n%s\nwant matching:\n%s", out, pattern) + } +} + +func TestNow(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + if got, want := time.Now(), time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC); !got.Equal(want) { + t.Errorf("time.Now() = %v, want %v", got, want) + } + }) +} diff --git a/src/testing/testing.go b/src/testing/testing.go index 526cba39f8fc92..0d1d08ca89a5e6 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -11,7 +11,7 @@ // where Xxx does not start with a lowercase letter. The function name // serves to identify the test routine. // -// Within these functions, use the Error, Fail or related methods to signal failure. +// Within these functions, use [T.Error], [T.Fail] or related methods to signal failure. // // To write a new test suite, create a file that // contains the TestXxx functions as described here, @@ -30,9 +30,9 @@ // import "testing" // // func TestAbs(t *testing.T) { -// got := Abs(-1) +// got := abs(-1) // if got != 1 { -// t.Errorf("Abs(-1) = %d; want 1", got) +// t.Errorf("abs(-1) = %d; want 1", got) // } // } // @@ -55,7 +55,7 @@ // } // } // -// For more detail, run "go help test" and "go help testflag". +// For more detail, run [go help test] and [go help testflag]. // // # Benchmarks // @@ -66,33 +66,29 @@ // are considered benchmarks, and are executed by the "go test" command when // its -bench flag is provided. Benchmarks are run sequentially. // -// For a description of the testing flags, see -// https://golang.org/cmd/go/#hdr-Testing_flags. +// For a description of the testing flags, see [go help testflag]. // // A sample benchmark function looks like this: // // func BenchmarkRandInt(b *testing.B) { -// for range b.N { +// for b.Loop() { // rand.Int() // } // } // -// The benchmark function must run the target code b.N times. -// It is called multiple times with b.N adjusted until the -// benchmark function lasts long enough to be timed reliably. // The output // // BenchmarkRandInt-8 68453040 17.8 ns/op // -// means that the loop ran 68453040 times at a speed of 17.8 ns per loop. +// means that the body of the loop ran 68453040 times at a speed of 17.8 ns per loop. // -// If a benchmark needs some expensive setup before running, the timer -// may be reset: +// Only the body of the loop is timed, so benchmarks may do expensive +// setup before calling b.Loop, which will not be counted toward the +// benchmark measurement: // // func BenchmarkBigLen(b *testing.B) { // big := NewBig() -// b.ResetTimer() -// for range b.N { +// for b.Loop() { // big.Len() // } // } @@ -113,13 +109,44 @@ // } // // A detailed specification of the benchmark results format is given -// in https://golang.org/design/14313-benchmark-format. +// in https://go.dev/design/14313-benchmark-format. // // There are standard tools for working with benchmark results at -// https://golang.org/x/perf/cmd. -// In particular, https://golang.org/x/perf/cmd/benchstat performs +// [golang.org/x/perf/cmd]. +// In particular, [golang.org/x/perf/cmd/benchstat] performs // statistically robust A/B comparisons. // +// # b.N-style benchmarks +// +// Prior to the introduction of [B.Loop], benchmarks were written in a +// different style using B.N. For example: +// +// func BenchmarkRandInt(b *testing.B) { +// for range b.N { +// rand.Int() +// } +// } +// +// In this style of benchmark, the benchmark function must run +// the target code b.N times. The benchmark function is called +// multiple times with b.N adjusted until the benchmark function +// lasts long enough to be timed reliably. This also means any setup +// done before the loop may be run several times. +// +// If a benchmark needs some expensive setup before running, the timer +// should be explicitly reset: +// +// func BenchmarkBigLen(b *testing.B) { +// big := NewBig() +// b.ResetTimer() +// for range b.N { +// big.Len() +// } +// } +// +// New benchmarks should prefer using [B.Loop], which is more robust +// and more efficient. +// // # Examples // // The package also runs and verifies example code. Example functions may @@ -209,19 +236,19 @@ // // A fuzz test maintains a seed corpus, or a set of inputs which are run by // default, and can seed input generation. Seed inputs may be registered by -// calling (*F).Add or by storing files in the directory testdata/fuzz/ +// calling [F.Add] or by storing files in the directory testdata/fuzz/ // (where is the name of the fuzz test) within the package containing // the fuzz test. Seed inputs are optional, but the fuzzing engine may find // bugs more efficiently when provided with a set of small seed inputs with good // code coverage. These seed inputs can also serve as regression tests for bugs // identified through fuzzing. // -// The function passed to (*F).Fuzz within the fuzz test is considered the fuzz -// target. A fuzz target must accept a *T parameter, followed by one or more -// parameters for random inputs. The types of arguments passed to (*F).Add must +// The function passed to [F.Fuzz] within the fuzz test is considered the fuzz +// target. A fuzz target must accept a [*T] parameter, followed by one or more +// parameters for random inputs. The types of arguments passed to [F.Add] must // be identical to the types of these parameters. The fuzz target may signal -// that it's found a problem the same way tests do: by calling T.Fail (or any -// method that calls it like T.Error or T.Fatal) or by panicking. +// that it's found a problem the same way tests do: by calling [T.Fail] (or any +// method that calls it like [T.Error] or [T.Fatal]) or by panicking. // // When fuzzing is enabled (by setting the -fuzz flag to a regular expression // that matches a specific fuzz test), the fuzz target is called with arguments @@ -237,16 +264,16 @@ // the fuzz cache directory within the build cache instead. // // When fuzzing is disabled, the fuzz target is called with the seed inputs -// registered with F.Add and seed inputs from testdata/fuzz/. In this +// registered with [F.Add] and seed inputs from testdata/fuzz/. In this // mode, the fuzz test acts much like a regular test, with subtests started -// with F.Fuzz instead of T.Run. +// with [F.Fuzz] instead of [T.Run]. // // See https://go.dev/doc/fuzz for documentation about fuzzing. // // # Skipping // // Tests or benchmarks may be skipped at run time with a call to -// the Skip method of *T or *B: +// [T.Skip] or [B.Skip]: // // func TestTimeConsuming(t *testing.T) { // if testing.Short() { @@ -255,7 +282,7 @@ // ... // } // -// The Skip method of *T can be used in a fuzz target if the input is invalid, +// The [T.Skip] method can be used in a fuzz target if the input is invalid, // but should not be considered a failing input. For example: // // func FuzzJSONMarshaling(f *testing.F) { @@ -272,7 +299,7 @@ // // # Subtests and Sub-benchmarks // -// The Run methods of T and B allow defining subtests and sub-benchmarks, +// The [T.Run] and [B.Run] methods allow defining subtests and sub-benchmarks, // without having to define separate functions for each. This enables uses // like table-driven benchmarks and creating hierarchical tests. // It also provides a way to share common setup and tear-down code: @@ -317,7 +344,6 @@ // // func TestGroupedParallel(t *testing.T) { // for _, tc := range tests { -// tc := tc // capture range variable // t.Run(tc.Name, func(t *testing.T) { // t.Parallel() // ... @@ -350,12 +376,12 @@ // then the generated test will call TestMain(m) instead of running the tests or benchmarks // directly. TestMain runs in the main goroutine and can do whatever setup // and teardown is necessary around a call to m.Run. m.Run will return an exit -// code that may be passed to os.Exit. If TestMain returns, the test wrapper -// will pass the result of m.Run to os.Exit itself. +// code that may be passed to [os.Exit]. If TestMain returns, the test wrapper +// will pass the result of m.Run to [os.Exit] itself. // // When TestMain is called, flag.Parse has not been run. If TestMain depends on // command-line flags, including those of the testing package, it should call -// flag.Parse explicitly. Command line flags are always parsed by the time test +// [flag.Parse] explicitly. Command line flags are always parsed by the time test // or benchmark functions run. // // A simple implementation of TestMain is: @@ -367,18 +393,22 @@ // // TestMain is a low-level primitive and should not be necessary for casual // testing needs, where ordinary test functions suffice. +// +// [go help test]: https://pkg.go.dev/cmd/go#hdr-Test_packages +// [go help testflag]: https://pkg.go.dev/cmd/go#hdr-Testing_flags package testing import ( "bytes" + "context" "errors" "flag" "fmt" - "internal/goexperiment" "internal/race" "io" "math/rand" "os" + "path/filepath" "reflect" "runtime" "runtime/debug" @@ -390,11 +420,16 @@ import ( "sync/atomic" "time" "unicode" - "unicode/utf8" + _ "unsafe" // for linkname ) var initRan bool +var ( + parallelStart atomic.Int64 // number of parallel tests started + parallelStop atomic.Int64 // number of parallel tests stopped +) + // Init registers testing flags. These flags are automatically registered by // the "go test" command before running test functions, so Init is only needed // when calling functions such as Benchmark without using "go test". @@ -420,6 +455,7 @@ func Init() { // this flag lets "go test" tell the binary to write the files in the directory where // the "go test" command is run. outputDir = flag.String("test.outputdir", "", "write profiles to `dir`") + artifacts = flag.Bool("test.artifacts", false, "store test artifacts in test.,outputdir") // Report as tests are run; default is silent for success. flag.Var(&chatty, "test.v", "verbose: print additional output") count = flag.Uint("test.count", 1, "run tests and benchmarks `n` times") @@ -453,6 +489,7 @@ var ( short *bool failFast *bool outputDir *string + artifacts *bool chatty chattyFlag count *uint coverProfile *string @@ -480,6 +517,7 @@ var ( cpuList []int testlogFile *os.File + artifactDir string numFailed atomic.Uint32 // number of test failures @@ -596,6 +634,7 @@ type common struct { mu sync.RWMutex // guards this group of fields output []byte // Output generated by test or benchmark. w io.Writer // For flushToParent. + o *outputWriter // Writes output. ran bool // Test or benchmark (or one of its subtests) was executed. failed bool // Test or benchmark has failed. skipped bool // Test or benchmark has been skipped. @@ -607,6 +646,7 @@ type common struct { cleanupPc []uintptr // The stack trace at the point where Cleanup was called. finished bool // Test function has completed. inFuzzFn bool // Whether the fuzz target, if this is one, is running. + isSynctest bool chatty *chattyPrinter // A copy of chattyPrinter, if the chatty flag is set. bench bool // Whether the current test is a benchmark. @@ -615,15 +655,17 @@ type common struct { runner string // Function name of tRunner running the test. isParallel bool // Whether the test is parallel. - parent *common - level int // Nesting depth of test or benchmark. - creator []uintptr // If level > 0, the stack trace at the point where the parent called t.Run. - name string // Name of test or benchmark. - start highPrecisionTime // Time test or benchmark started - duration time.Duration - barrier chan bool // To signal parallel subtests they may start. Nil when T.Parallel is not present (B) or not usable (when fuzzing). - signal chan bool // To signal a test is done. - sub []*T // Queue of subtests to be run in parallel. + parent *common + level int // Nesting depth of test or benchmark. + creator []uintptr // If level > 0, the stack trace at the point where the parent called t.Run. + modulePath string + importPath string + name string // Name of test or benchmark. + start highPrecisionTime // Time test or benchmark started + duration time.Duration + barrier chan bool // To signal parallel subtests they may start. Nil when T.Parallel is not present (B) or not usable (when fuzzing). + signal chan bool // To signal a test is done. + sub []*T // Queue of subtests to be run in parallel. lastRaceErrors atomic.Int64 // Max value of race.Errors seen during the test or its subtests. raceErrorLogged atomic.Bool @@ -632,6 +674,13 @@ type common struct { tempDir string tempDirErr error tempDirSeq int32 + + artifactDirOnce sync.Once + artifactDir string + artifactDirErr error + + ctx context.Context + cancelCtx context.CancelFunc } // Short reports whether the -test.short flag is set. @@ -667,10 +716,7 @@ func Testing() bool { // values are "set", "count", or "atomic". The return value will be // empty if test coverage is not enabled. func CoverMode() string { - if goexperiment.CoverageRedesign { - return cover2.mode - } - return cover.Mode + return cover.mode } // Verbose reports whether the -test.v flag is set. @@ -764,44 +810,6 @@ func (c *common) frameSkip(skip int) runtime.Frame { return firstFrame } -// decorate prefixes the string with the file and line of the call site -// and inserts the final newline if needed and indentation spaces for formatting. -// This function must be called with c.mu held. -func (c *common) decorate(s string, skip int) string { - frame := c.frameSkip(skip) - file := frame.File - line := frame.Line - if file != "" { - if *fullPath { - // If relative path, truncate file name at last file name separator. - } else if index := strings.LastIndexAny(file, `/\`); index >= 0 { - file = file[index+1:] - } - } else { - file = "???" - } - if line == 0 { - line = 1 - } - buf := new(strings.Builder) - // Every line is indented at least 4 spaces. - buf.WriteString(" ") - fmt.Fprintf(buf, "%s:%d: ", file, line) - lines := strings.Split(s, "\n") - if l := len(lines); l > 1 && lines[l-1] == "" { - lines = lines[:l-1] - } - for i, line := range lines { - if i > 0 { - // Second and subsequent lines are indented an additional 4 spaces. - buf.WriteString("\n ") - } - buf.WriteString(line) - } - buf.WriteByte('\n') - return buf.String() -} - // flushToParent writes c.output to the parent after first writing the header // with the given format and arguments. func (c *common) flushToParent(testName, format string, args ...any) { @@ -847,6 +855,8 @@ type indenter struct { c *common } +const indent = " " + func (w indenter) Write(b []byte) (n int, err error) { n = len(b) for len(b) > 0 { @@ -863,7 +873,6 @@ func (w indenter) Write(b []byte) (n int, err error) { w.c.output = append(w.c.output, marker) line = line[1:] } - const indent = " " w.c.output = append(w.c.output, indent...) w.c.output = append(w.c.output, line...) b = b[end:] @@ -876,8 +885,10 @@ func fmtDuration(d time.Duration) string { return fmt.Sprintf("%.2fs", d.Seconds()) } -// TB is the interface common to T, B, and F. +// TB is the interface common to [T], [B], and [F]. type TB interface { + ArtifactDir() string + Attr(key, value string) Cleanup(func()) Error(args ...any) Errorf(format string, args ...any) @@ -891,11 +902,14 @@ type TB interface { Logf(format string, args ...any) Name() string Setenv(key, value string) + Chdir(dir string) Skip(args ...any) SkipNow() Skipf(format string, args ...any) Skipped() bool TempDir() string + Context() context.Context + Output() io.Writer // A private method to prevent users implementing the // interface and so future additions to it will not @@ -903,22 +917,24 @@ type TB interface { private() } -var _ TB = (*T)(nil) -var _ TB = (*B)(nil) +var ( + _ TB = (*T)(nil) + _ TB = (*B)(nil) +) // T is a type passed to Test functions to manage test state and support formatted test logs. // // A test ends when its Test function returns or calls any of the methods -// FailNow, Fatal, Fatalf, SkipNow, Skip, or Skipf. Those methods, as well as -// the Parallel method, must be called only from the goroutine running the +// [T.FailNow], [T.Fatal], [T.Fatalf], [T.SkipNow], [T.Skip], or [T.Skipf]. Those methods, as well as +// the [T.Parallel] method, must be called only from the goroutine running the // Test function. // -// The other reporting methods, such as the variations of Log and Error, +// The other reporting methods, such as the variations of [T.Log] and [T.Error], // may be called simultaneously from multiple goroutines. type T struct { common - isEnvSet bool - context *testContext // For running tests and subtests. + denyParallel bool + tstate *testState // For running tests and subtests. } func (c *common) private() {} @@ -970,7 +986,7 @@ func (c *common) Failed() bool { } // FailNow marks the function as having failed and stops its execution -// by calling runtime.Goexit (which then runs all deferred calls in the +// by calling [runtime.Goexit] (which then runs all deferred calls in the // current goroutine). // Execution will continue at the next test or benchmark. // FailNow must be called from the goroutine running the @@ -1006,57 +1022,179 @@ func (c *common) FailNow() { runtime.Goexit() } -// log generates the output. It's always at the same stack depth. +// log generates the output. It is always at the same stack depth. log inserts +// indentation and the final newline if necessary. It prefixes the string +// with the file and line of the call site. func (c *common) log(s string) { - c.logDepth(s, 3) // logDepth + log + public function + s = strings.TrimSuffix(s, "\n") + + // Second and subsequent lines are indented 4 spaces. This is in addition to + // the indentation provided by outputWriter. + s = strings.ReplaceAll(s, "\n", "\n"+indent) + s += "\n" + + n := c.destination() + if n == nil { + // The test and all its parents are done. The log cannot be output. + panic("Log in goroutine after " + c.name + " has completed: " + s) + } + + // Prefix with the call site. It is located by skipping 3 functions: + // callSite + log + public function + s = n.callSite(3) + s + + // Output buffered logs. + n.flushPartial() + + n.o.Write([]byte(s)) } -// logDepth generates the output at an arbitrary stack depth. -func (c *common) logDepth(s string, depth int) { +// destination selects the test to which output should be appended. It returns the +// test if it is incomplete. Otherwise, it finds its closest incomplete parent. +func (c *common) destination() *common { c.mu.Lock() defer c.mu.Unlock() - if c.done { - // This test has already finished. Try and log this message - // with our parent. If we don't have a parent, panic. - for parent := c.parent; parent != nil; parent = parent.parent { - parent.mu.Lock() - defer parent.mu.Unlock() - if !parent.done { - parent.output = append(parent.output, parent.decorate(s, depth+1)...) - return - } + + if !c.done && !c.isSynctest { + return c + } + for parent := c.parent; parent != nil; parent = parent.parent { + parent.mu.Lock() + defer parent.mu.Unlock() + if !parent.done { + return parent + } + } + return nil +} + +// callSite retrieves and formats the file and line of the call site. +func (c *common) callSite(skip int) string { + c.mu.Lock() + defer c.mu.Unlock() + + frame := c.frameSkip(skip) + file := frame.File + line := frame.Line + if file != "" { + if *fullPath { + // If relative path, truncate file name at last file name separator. + } else { + file = filepath.Base(file) } - panic("Log in goroutine after " + c.name + " has completed: " + s) } else { - if c.chatty != nil { - if c.bench { - // Benchmarks don't print === CONT, so we should skip the test - // printer and just print straight to stdout. - fmt.Print(c.decorate(s, depth+1)) - } else { - c.chatty.Printf(c.name, "%s", c.decorate(s, depth+1)) - } + file = "???" + } + if line == 0 { + line = 1 + } - return + return fmt.Sprintf("%s:%d: ", file, line) +} + +// flushPartial checks the buffer for partial logs and outputs them. +func (c *common) flushPartial() { + partial := func() bool { + c.mu.Lock() + defer c.mu.Unlock() + return (c.o != nil) && (len(c.o.partial) > 0) + } + + if partial() { + c.o.Write([]byte("\n")) + } +} + +// Output returns a Writer that writes to the same test output stream as TB.Log. +// The output is indented like TB.Log lines, but Output does not +// add source locations or newlines. The output is internally line +// buffered, and a call to TB.Log or the end of the test will implicitly +// flush the buffer, followed by a newline. After a test function and all its +// parents return, neither Output nor the Write method may be called. +func (c *common) Output() io.Writer { + c.checkFuzzFn("Output") + n := c.destination() + if n == nil { + panic("Output called after " + c.name + " has completed") + } + return n.o +} + +// setOutputWriter initializes an outputWriter and sets it as a common field. +func (c *common) setOutputWriter() { + c.o = &outputWriter{c: c} +} + +// outputWriter buffers, formats and writes log messages. +type outputWriter struct { + c *common + partial []byte // incomplete ('\n'-free) suffix of last Write +} + +// Write writes a log message to the test's output stream, properly formatted and +// indented. It may not be called after a test function and all its parents return. +func (o *outputWriter) Write(p []byte) (int, error) { + // o can be nil if this is called from a top-level *TB that is no longer active. + // Just ignore the message in that case. + if o == nil || o.c == nil { + return 0, nil + } + if o.c.destination() == nil { + panic("Write called after " + o.c.name + " has completed") + } + + o.c.mu.Lock() + defer o.c.mu.Unlock() + + // The last element is a partial line. + lines := bytes.SplitAfter(p, []byte("\n")) + last := len(lines) - 1 // Inv: 0 <= last + for i, line := range lines[:last] { + // Emit partial line from previous call. + if i == 0 && len(o.partial) > 0 { + line = slices.Concat(o.partial, line) + o.partial = o.partial[:0] } - c.output = append(c.output, c.decorate(s, depth+1)...) + o.writeLine(line) } + // Save partial line for next call. + o.partial = append(o.partial, lines[last]...) + + return len(p), nil } -// Log formats its arguments using default formatting, analogous to Println, +// writeLine generates the output for a given line. +func (o *outputWriter) writeLine(b []byte) { + if !o.c.done && (o.c.chatty != nil) { + if o.c.bench { + // Benchmarks don't print === CONT, so we should skip the test + // printer and just print straight to stdout. + fmt.Printf("%s%s", indent, b) + } else { + o.c.chatty.Printf(o.c.name, "%s%s", indent, b) + } + return + } + o.c.output = append(o.c.output, indent...) + o.c.output = append(o.c.output, b...) +} + +// Log formats its arguments using default formatting, analogous to [fmt.Println], // and records the text in the error log. For tests, the text will be printed only if // the test fails or the -test.v flag is set. For benchmarks, the text is always // printed to avoid having performance depend on the value of the -test.v flag. +// It is an error to call Log after a test or benchmark returns. func (c *common) Log(args ...any) { c.checkFuzzFn("Log") c.log(fmt.Sprintln(args...)) } -// Logf formats its arguments according to the format, analogous to Printf, and +// Logf formats its arguments according to the format, analogous to [fmt.Printf], and // records the text in the error log. A final newline is added if not provided. For // tests, the text will be printed only if the test fails or the -test.v flag is // set. For benchmarks, the text is always printed to avoid having performance // depend on the value of the -test.v flag. +// It is an error to call Logf after a test or benchmark returns. func (c *common) Logf(format string, args ...any) { c.checkFuzzFn("Logf") c.log(fmt.Sprintf(format, args...)) @@ -1132,6 +1270,9 @@ func (c *common) Skipped() bool { // When printing file and line information, that function will be skipped. // Helper may be called simultaneously from multiple goroutines. func (c *common) Helper() { + if c.isSynctest { + c = c.parent + } c.mu.Lock() defer c.mu.Unlock() if c.helperPCs == nil { @@ -1181,13 +1322,113 @@ func (c *common) Cleanup(f func()) { c.cleanups = append(c.cleanups, fn) } +// ArtifactDir returns a directory in which the test should store output files. +// When the -artifacts flag is provided, this directory is located +// under the output directory. Otherwise, ArtifactDir returns a temporary directory +// that is removed after the test completes. +// +// Each test or subtest within each test package has a unique artifact directory. +// Repeated calls to ArtifactDir in the same test or subtest return the same directory. +// Subtest outputs are not located under the parent test's output directory. +func (c *common) ArtifactDir() string { + c.checkFuzzFn("ArtifactDir") + c.artifactDirOnce.Do(func() { + c.artifactDir, c.artifactDirErr = c.makeArtifactDir() + }) + if c.artifactDirErr != nil { + c.Fatalf("ArtifactDir: %v", c.artifactDirErr) + } + return c.artifactDir +} + +func hashString(s string) (h uint64) { + // FNV, used here to avoid a dependency on maphash. + for i := 0; i < len(s); i++ { + h ^= uint64(s[i]) + h *= 1099511628211 + } + return +} + +// makeArtifactDir creates the artifact directory for a test. +// The artifact directory is: +// +// /_artifacts/// +// +// The test package is the package import path with the module name prefix removed. +// The test name is truncated if too long. +// Special characters are removed from the path. +func (c *common) makeArtifactDir() (string, error) { + if !*artifacts { + return c.makeTempDir() + } + + // If the test name is longer than maxNameSize, truncate it and replace the last + // hashSize bytes with a hash of the full name. + const maxNameSize = 64 + name := strings.ReplaceAll(c.name, "/", "__") + if len(name) > maxNameSize { + h := fmt.Sprintf("%0x", hashString(name)) + name = name[:maxNameSize-len(h)] + h + } + + // Remove the module path prefix from the import path. + pkg := strings.TrimPrefix(c.importPath, c.modulePath+"/") + + // Join with /, not filepath.Join: the import path is /-separated, + // and we don't want removeSymbolsExcept to strip \ separators on Windows. + base := "/" + pkg + "/" + name + base = removeSymbolsExcept(base, "!#$%&()+,-.=@^_{}~ /") + base, err := filepath.Localize(base) + if err != nil { + // This name can't be safely converted into a local filepath. + // Drop it and just use _artifacts/. + base = "" + } + + artifactBase := filepath.Join(artifactDir, base) + if err := os.MkdirAll(artifactBase, 0o777); err != nil { + return "", err + } + dir, err := os.MkdirTemp(artifactBase, "") + if err != nil { + return "", err + } + if c.chatty != nil { + c.chatty.Updatef(c.name, "=== ARTIFACTS %s %v\n", c.name, dir) + } + return dir, nil +} + +func removeSymbolsExcept(s, allowed string) string { + mapper := func(r rune) rune { + if unicode.IsLetter(r) || + unicode.IsNumber(r) || + strings.ContainsRune(allowed, r) { + return r + } + return -1 // disallowed symbol + } + return strings.Map(mapper, s) +} + // TempDir returns a temporary directory for the test to use. // The directory is automatically removed when the test and // all its subtests complete. -// Each subsequent call to t.TempDir returns a unique directory; +// Each subsequent call to TempDir returns a unique directory; // if the directory creation fails, TempDir terminates the test by calling Fatal. +// If the environment variable GOTMPDIR is set, the temporary directory will +// be created somewhere beneath it. func (c *common) TempDir() string { c.checkFuzzFn("TempDir") + dir, err := c.makeTempDir() + if err != nil { + c.Fatalf("TempDir: %v", err) + } + return dir +} + +func (c *common) makeTempDir() (string, error) { // Use a single parent directory for all the temporary directories // created by a test, each numbered sequentially. c.tempDirMu.Lock() @@ -1198,34 +1439,25 @@ func (c *common) TempDir() string { _, err := os.Stat(c.tempDir) nonExistent = os.IsNotExist(err) if err != nil && !nonExistent { - c.Fatalf("TempDir: %v", err) + return "", err } } if nonExistent { c.Helper() + pattern := c.Name() + // Limit length of file names on disk. + // Invalid runes from slicing are dropped by strings.Map below. + pattern = pattern[:min(len(pattern), 64)] + // Drop unusual characters (such as path separators or // characters interacting with globs) from the directory name to // avoid surprising os.MkdirTemp behavior. - mapper := func(r rune) rune { - if r < utf8.RuneSelf { - const allowed = "!#$%&()+,-.=@^_{}~ " - if '0' <= r && r <= '9' || - 'a' <= r && r <= 'z' || - 'A' <= r && r <= 'Z' { - return r - } - if strings.ContainsRune(allowed, r) { - return r - } - } else if unicode.IsLetter(r) || unicode.IsNumber(r) { - return r - } - return -1 - } - pattern := strings.Map(mapper, c.Name()) - c.tempDir, c.tempDirErr = os.MkdirTemp("", pattern) + const allowed = "!#$%&()+,-.=@^_{}~ " + pattern = removeSymbolsExcept(pattern, allowed) + + c.tempDir, c.tempDirErr = os.MkdirTemp(os.Getenv("GOTMPDIR"), pattern) if c.tempDirErr == nil { c.Cleanup(func() { if err := removeAll(c.tempDir); err != nil { @@ -1242,14 +1474,14 @@ func (c *common) TempDir() string { c.tempDirMu.Unlock() if c.tempDirErr != nil { - c.Fatalf("TempDir: %v", c.tempDirErr) + return "", c.tempDirErr } dir := fmt.Sprintf("%s%c%03d", c.tempDir, os.PathSeparator, seq) - if err := os.Mkdir(dir, 0777); err != nil { - c.Fatalf("TempDir: %v", err) + if err := os.Mkdir(dir, 0o777); err != nil { + return "", err } - return dir + return dir, nil } // removeAll is like os.RemoveAll, but retries Windows "Access is denied." @@ -1282,7 +1514,7 @@ func removeAll(path string) error { } } -// Setenv calls os.Setenv(key, value) and uses Cleanup to +// Setenv calls [os.Setenv] and uses Cleanup to // restore the environment variable to its original value // after the test. // @@ -1307,6 +1539,83 @@ func (c *common) Setenv(key, value string) { } } +// Chdir calls [os.Chdir] and uses Cleanup to restore the current +// working directory to its original value after the test. On Unix, it +// also sets PWD environment variable for the duration of the test. +// +// Because Chdir affects the whole process, it cannot be used +// in parallel tests or tests with parallel ancestors. +func (c *common) Chdir(dir string) { + c.checkFuzzFn("Chdir") + oldwd, err := os.Open(".") + if err != nil { + c.Fatal(err) + } + if err := os.Chdir(dir); err != nil { + c.Fatal(err) + } + // On POSIX platforms, PWD represents “an absolute pathname of the + // current working directory.” Since we are changing the working + // directory, we should also set or update PWD to reflect that. + switch runtime.GOOS { + case "windows", "plan9": + // Windows and Plan 9 do not use the PWD variable. + default: + if !filepath.IsAbs(dir) { + dir, err = os.Getwd() + if err != nil { + c.Fatal(err) + } + } + c.Setenv("PWD", dir) + } + c.Cleanup(func() { + err := oldwd.Chdir() + oldwd.Close() + if err != nil { + // It's not safe to continue with tests if we can't + // get back to the original working directory. Since + // we are holding a dirfd, this is highly unlikely. + panic("testing.Chdir: " + err.Error()) + } + }) +} + +// Context returns a context that is canceled just before +// Cleanup-registered functions are called. +// +// Cleanup functions can wait for any resources +// that shut down on [context.Context.Done] before the test or benchmark completes. +func (c *common) Context() context.Context { + c.checkFuzzFn("Context") + return c.ctx +} + +// Attr emits a test attribute associated with this test. +// +// The key must not contain whitespace. +// The value must not contain newlines or carriage returns. +// +// The meaning of different attribute keys is left up to +// continuous integration systems and test frameworks. +// +// Test attributes are emitted immediately in the test log, +// but they are intended to be treated as unordered. +func (c *common) Attr(key, value string) { + if strings.ContainsFunc(key, unicode.IsSpace) { + c.Errorf("disallowed whitespace in attribute key %q", key) + return + } + if strings.ContainsAny(value, "\r\n") { + c.Errorf("disallowed newline in attribute value %q", value) + return + } + if c.chatty == nil { + return + } + c.chatty.Updatef(c.name, "=== ATTR %s %v %v\n", c.name, key, value) +} + // panicHandling controls the panic handling used by runCleanup. type panicHandling int @@ -1339,6 +1648,10 @@ func (c *common) runCleanup(ph panicHandling) (panicVal any) { } }() + if c.cancelCtx != nil { + c.cancelCtx() + } + for { var cleanup func() c.mu.Lock() @@ -1436,6 +1749,8 @@ func pcToName(pc uintptr) string { return frame.Function } +const parallelConflict = `testing: test using t.Setenv or t.Chdir can not use t.Parallel` + // Parallel signals that this test is to be run in parallel with (and only with) // other parallel tests. When a test is run multiple times due to use of // -test.count or -test.cpu, multiple instances of a single test never run in @@ -1444,10 +1759,12 @@ func (t *T) Parallel() { if t.isParallel { panic("testing: t.Parallel called multiple times") } - if t.isEnvSet { - panic("testing: t.Parallel called after t.Setenv; cannot set environment variables in parallel tests") + if t.isSynctest { + panic("testing: t.Parallel called inside synctest bubble") + } + if t.denyParallel { + panic(parallelConflict) } - t.isParallel = true if t.parent.barrier == nil { // T.Parallel has no effect when fuzzing. // Multiple processes may run in parallel, but only one input can run at a @@ -1455,6 +1772,8 @@ func (t *T) Parallel() { return } + t.isParallel = true + // We don't want to include the time we spend waiting for serial tests // in the test duration. Record the elapsed time thus far and reset the // timer afterwards. @@ -1482,7 +1801,8 @@ func (t *T) Parallel() { t.signal <- true // Release calling test. <-t.parent.barrier // Wait for the parent test to complete. - t.context.waitParallel() + t.tstate.waitParallel() + parallelStart.Add(1) if t.chatty != nil { t.chatty.Updatef(t.name, "=== CONT %s\n", t.name) @@ -1500,34 +1820,43 @@ func (t *T) Parallel() { t.lastRaceErrors.Store(int64(race.Errors())) } -// Setenv calls os.Setenv(key, value) and uses Cleanup to -// restore the environment variable to its original value -// after the test. -// -// Because Setenv affects the whole process, it cannot be used -// in parallel tests or tests with parallel ancestors. -func (t *T) Setenv(key, value string) { +func (t *T) checkParallel() { // Non-parallel subtests that have parallel ancestors may still // run in parallel with other tests: they are only non-parallel // with respect to the other subtests of the same parent. - // Since SetEnv affects the whole process, we need to disallow it - // if the current test or any parent is parallel. - isParallel := false + // Since calls like SetEnv or Chdir affects the whole process, we need + // to deny those if the current test or any parent is parallel. for c := &t.common; c != nil; c = c.parent { if c.isParallel { - isParallel = true - break + panic(parallelConflict) } } - if isParallel { - panic("testing: t.Setenv called after t.Parallel; cannot set environment variables in parallel tests") - } - t.isEnvSet = true + t.denyParallel = true +} +// Setenv calls os.Setenv(key, value) and uses Cleanup to +// restore the environment variable to its original value +// after the test. +// +// Because Setenv affects the whole process, it cannot be used +// in parallel tests or tests with parallel ancestors. +func (t *T) Setenv(key, value string) { + t.checkParallel() t.common.Setenv(key, value) } +// Chdir calls [os.Chdir] and uses Cleanup to restore the current +// working directory to its original value after the test. On Unix, it +// also sets PWD environment variable for the duration of the test. +// +// Because Chdir affects the whole process, it cannot be used +// in parallel tests or tests with parallel ancestors. +func (t *T) Chdir(dir string) { + t.checkParallel() + t.common.Chdir(dir) +} + // InternalTest is an internal type but exported because it is cross-package; // it is part of the implementation of the "go test" command. type InternalTest struct { @@ -1583,7 +1912,7 @@ func tRunner(t *T, fn func(t *T)) { } } - if err != nil && t.context.isFuzzing { + if err != nil && t.tstate.isFuzzing { prefix := "panic: " if err == errNilPanicOrGoexit { prefix = "" @@ -1609,6 +1938,9 @@ func tRunner(t *T, fn func(t *T)) { panic(err) } running.Delete(t.name) + if t.isParallel { + parallelStop.Add(1) + } t.signal <- signal }() @@ -1618,11 +1950,14 @@ func tRunner(t *T, fn func(t *T)) { t.Logf("cleanup panicked with %v", r) } // Flush the output log up to the root before dying. - for root := &t.common; root.parent != nil; root = root.parent { + // Skip this if this *T is a synctest bubble, because we're not a subtest. + for root := &t.common; !root.isSynctest && root.parent != nil; root = root.parent { root.mu.Lock() root.duration += highPrecisionTimeSince(root.start) d := root.duration root.mu.Unlock() + // Output buffered logs. + root.flushPartial() root.flushToParent(root.name, "--- FAIL: %s (%s)\n", root.name, fmtDuration(d)) if r := root.parent.runCleanup(recoverAndReturnPanic); r != nil { fmt.Fprintf(root.parent.w, "cleanup panicked with %v", r) @@ -1641,7 +1976,7 @@ func tRunner(t *T, fn func(t *T)) { // Run parallel subtests. // Decrease the running count for this test and mark it as no longer running. - t.context.release() + t.tstate.release() running.Delete(t.name) // Release the parallel subtests. @@ -1663,12 +1998,16 @@ func tRunner(t *T, fn func(t *T)) { t.checkRaces() if !t.isParallel { // Reacquire the count for sequential tests. See comment in Run. - t.context.waitParallel() + t.tstate.waitParallel() } } else if t.isParallel { // Only release the count for this test if it was run as a parallel // test. See comment in Run method. - t.context.release() + t.tstate.release() + } + // Output buffered logs. + for root := &t.common; root.parent != nil; root = root.parent { + root.flushPartial() } t.report() // Report after all subtests have finished. @@ -1702,12 +2041,15 @@ func tRunner(t *T, fn func(t *T)) { // Run may be called simultaneously from multiple goroutines, but all such calls // must return before the outer test function for t returns. func (t *T) Run(name string, f func(t *T)) bool { + if t.isSynctest { + panic("testing: t.Run called inside synctest bubble") + } if t.cleanupStarted.Load() { panic("testing: t.Run called during t.Cleanup") } t.hasSub.Store(true) - testName, ok, _ := t.context.match.fullName(&t.common, name) + testName, ok, _ := t.tstate.match.fullName(&t.common, name) if !ok || shouldFailFast() { return true } @@ -1716,19 +2058,28 @@ func (t *T) Run(name string, f func(t *T)) bool { // continue walking the stack into the parent test. var pc [maxStackLen]uintptr n := runtime.Callers(2, pc[:]) + + // There's no reason to inherit this context from parent. The user's code can't observe + // the difference between the background context and the one from the parent test. + ctx, cancelCtx := context.WithCancel(context.Background()) t = &T{ common: common{ - barrier: make(chan bool), - signal: make(chan bool, 1), - name: testName, - parent: &t.common, - level: t.level + 1, - creator: pc[:n], - chatty: t.chatty, + barrier: make(chan bool), + signal: make(chan bool, 1), + name: testName, + modulePath: t.modulePath, + importPath: t.importPath, + parent: &t.common, + level: t.level + 1, + creator: pc[:n], + chatty: t.chatty, + ctx: ctx, + cancelCtx: cancelCtx, }, - context: t.context, + tstate: t.tstate, } t.w = indenter{&t.common} + t.setOutputWriter() if t.chatty != nil { t.chatty.Updatef(t.name, "=== RUN %s\n", t.name) @@ -1760,22 +2111,66 @@ func (t *T) Run(name string, f func(t *T)) bool { return !t.failed } +// testingSynctestTest runs f within a synctest bubble. +// It is called by synctest.Test, from within an already-created bubble. +// +//go:linkname testingSynctestTest testing/synctest.testingSynctestTest +func testingSynctestTest(t *T, f func(*T)) (ok bool) { + if t.cleanupStarted.Load() { + panic("testing: synctest.Run called during t.Cleanup") + } + + var pc [maxStackLen]uintptr + n := runtime.Callers(2, pc[:]) + + ctx, cancelCtx := context.WithCancel(context.Background()) + t2 := &T{ + common: common{ + barrier: make(chan bool), + signal: make(chan bool, 1), + name: t.name, + parent: &t.common, + level: t.level + 1, + creator: pc[:n], + chatty: t.chatty, + ctx: ctx, + cancelCtx: cancelCtx, + isSynctest: true, + }, + tstate: t.tstate, + } + + go tRunner(t2, f) + if !<-t2.signal { + // At this point, it is likely that FailNow was called on one of the + // parent tests by one of the subtests. Continue aborting up the chain. + runtime.Goexit() + } + return !t2.failed +} + // Deadline reports the time at which the test binary will have // exceeded the timeout specified by the -timeout flag. // // The ok result is false if the -timeout flag indicates “no timeout” (0). func (t *T) Deadline() (deadline time.Time, ok bool) { - deadline = t.context.deadline + if t.isSynctest { + // There's no point in returning a real-clock deadline to + // a test using a fake clock. We could return "no timeout", + // but panicking makes it easier for users to catch the error. + panic("testing: t.Deadline called inside synctest bubble") + } + deadline = t.tstate.deadline return deadline, !deadline.IsZero() } -// testContext holds all fields that are common to all tests. This includes +// testState holds all fields that are common to all tests. This includes // synchronization primitives to run at most *parallel tests. -type testContext struct { +type testState struct { match *matcher deadline time.Time - // isFuzzing is true in the context used when generating random inputs + // isFuzzing is true in the state used when generating random inputs // for fuzz targets. isFuzzing is false when running normal tests and // when running fuzz tests as unit tests (without -fuzz or when -fuzz // does not match). @@ -1797,8 +2192,8 @@ type testContext struct { maxParallel int } -func newTestContext(maxParallel int, m *matcher) *testContext { - return &testContext{ +func newTestState(maxParallel int, m *matcher) *testState { + return &testState{ match: m, startParallel: make(chan bool), maxParallel: maxParallel, @@ -1806,28 +2201,28 @@ func newTestContext(maxParallel int, m *matcher) *testContext { } } -func (c *testContext) waitParallel() { - c.mu.Lock() - if c.running < c.maxParallel { - c.running++ - c.mu.Unlock() +func (s *testState) waitParallel() { + s.mu.Lock() + if s.running < s.maxParallel { + s.running++ + s.mu.Unlock() return } - c.numWaiting++ - c.mu.Unlock() - <-c.startParallel + s.numWaiting++ + s.mu.Unlock() + <-s.startParallel } -func (c *testContext) release() { - c.mu.Lock() - if c.numWaiting == 0 { - c.running-- - c.mu.Unlock() +func (s *testState) release() { + s.mu.Lock() + if s.numWaiting == 0 { + s.running-- + s.mu.Unlock() return } - c.numWaiting-- - c.mu.Unlock() - c.startParallel <- true // Pick a waiting test to be run. + s.numWaiting-- + s.mu.Unlock() + s.startParallel <- true // Pick a waiting test to be run. } // No one should be using func Main anymore. @@ -1840,6 +2235,7 @@ func (f matchStringOnly) MatchString(pat, str string) (bool, error) { return f func (f matchStringOnly) StartCPUProfile(w io.Writer) error { return errMain } func (f matchStringOnly) StopCPUProfile() {} func (f matchStringOnly) WriteProfileTo(string, io.Writer, int) error { return errMain } +func (f matchStringOnly) ModulePath() string { return "" } func (f matchStringOnly) ImportPath() string { return "" } func (f matchStringOnly) StartTestLog(io.Writer) {} func (f matchStringOnly) StopTestLog() error { return errMain } @@ -1864,7 +2260,7 @@ func (f matchStringOnly) InitRuntimeCoverage() (mode string, tearDown func(strin // It is no longer used by "go test" but preserved, as much as possible, for other // systems that simulate "go test" using Main, but Main sometimes cannot be updated as // new functionality is added to the testing package. -// Systems simulating "go test" should be updated to use MainStart. +// Systems simulating "go test" should be updated to use [MainStart]. func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) { os.Exit(MainStart(matchStringOnly(matchString), tests, benchmarks, nil, examples).Run()) } @@ -1893,6 +2289,7 @@ type M struct { // testing/internal/testdeps's TestDeps. type testDeps interface { ImportPath() string + ModulePath() string MatchString(pat, str string) (bool, error) SetPanicOnExit0(bool) StartCPUProfile(io.Writer) error @@ -1913,7 +2310,7 @@ type testDeps interface { // It is not meant to be called directly and is not subject to the Go 1 compatibility document. // It may change signature from release to release. func MainStart(deps testDeps, tests []InternalTest, benchmarks []InternalBenchmark, fuzzTargets []InternalFuzzTarget, examples []InternalExample) *M { - registerCover2(deps.InitRuntimeCoverage()) + registerCover(deps.InitRuntimeCoverage()) Init() return &M{ deps: deps, @@ -1924,10 +2321,15 @@ func MainStart(deps testDeps, tests []InternalTest, benchmarks []InternalBenchma } } -var testingTesting bool -var realStderr *os.File +var ( + testingTesting bool + realStderr *os.File +) // Run runs the tests. It returns an exit code to pass to os.Exit. +// The exit code is zero when all tests pass, and non-zero for any kind +// of failure. For machine readable test results, parse the output of +// 'go test -json'. func (m *M) Run() (code int) { defer func() { code = m.exitCode @@ -2031,7 +2433,7 @@ func (m *M) Run() (code int) { if !*isFuzzWorker { deadline := m.startAlarm() haveExamples = len(m.examples) > 0 - testRan, testOk := runTests(m.deps.MatchString, m.tests, deadline) + testRan, testOk := runTests(m.deps.ModulePath(), m.deps.ImportPath(), m.deps.MatchString, m.tests, deadline) fuzzTargetsRan, fuzzTargetsOk := runFuzzTests(m.deps, m.fuzzTargets, deadline) exampleRan, exampleOk := runExamples(m.deps.MatchString, m.examples) m.stopAlarm() @@ -2081,6 +2483,9 @@ func (t *T) report() { if t.parent == nil { return } + if t.isSynctest { + return // t.parent will handle reporting + } dstr := fmtDuration(t.duration) format := "--- %s: %s (%s)\n" if t.Failed() { @@ -2129,14 +2534,14 @@ func RunTests(matchString func(pat, str string) (bool, error), tests []InternalT if *timeout > 0 { deadline = time.Now().Add(*timeout) } - ran, ok := runTests(matchString, tests, deadline) + ran, ok := runTests("", "", matchString, tests, deadline) if !ran && !haveExamples { fmt.Fprintln(os.Stderr, "testing: warning: no tests to run") } return ok } -func runTests(matchString func(pat, str string) (bool, error), tests []InternalTest, deadline time.Time) (ran, ok bool) { +func runTests(modulePath, importPath string, matchString func(pat, str string) (bool, error), tests []InternalTest, deadline time.Time) (ran, ok bool) { ok = true for _, procs := range cpuList { runtime.GOMAXPROCS(procs) @@ -2150,15 +2555,20 @@ func runTests(matchString func(pat, str string) (bool, error), tests []InternalT // to keep trying. break } - ctx := newTestContext(*parallel, newMatcher(matchString, *match, "-test.run", *skip)) - ctx.deadline = deadline + ctx, cancelCtx := context.WithCancel(context.Background()) + tstate := newTestState(*parallel, newMatcher(matchString, *match, "-test.run", *skip)) + tstate.deadline = deadline t := &T{ common: common{ - signal: make(chan bool, 1), - barrier: make(chan bool), - w: os.Stdout, + signal: make(chan bool, 1), + barrier: make(chan bool), + w: os.Stdout, + ctx: ctx, + cancelCtx: cancelCtx, + modulePath: modulePath, + importPath: importPath, }, - context: ctx, + tstate: tstate, } if Verbose() { t.chatty = newChattyPrinter(t.w) @@ -2225,6 +2635,18 @@ func (m *M) before() { fmt.Fprintf(os.Stderr, "testing: cannot use -test.gocoverdir because test binary was not built with coverage enabled\n") os.Exit(2) } + if *artifacts { + var err error + artifactDir, err = filepath.Abs(toOutputDir("_artifacts")) + if err != nil { + fmt.Fprintf(os.Stderr, "testing: cannot make -test.outputdir absolute: %v\n", err) + os.Exit(2) + } + if err := os.Mkdir(artifactDir, 0o777); err != nil && !errors.Is(err, os.ErrExist) { + fmt.Fprintf(os.Stderr, "testing: %v\n", err) + os.Exit(2) + } + } if *testlog != "" { // Note: Not using toOutputDir. // This file is for use by cmd/go, not users. @@ -2394,7 +2816,7 @@ func (m *M) stopAlarm() { } func parseCpuList() { - for _, val := range strings.Split(*cpuListStr, ",") { + for val := range strings.SplitSeq(*cpuListStr, ",") { val = strings.TrimSpace(val) if val == "" { continue diff --git a/src/testing/testing_test.go b/src/testing/testing_test.go index 4a9303952e79ca..167f4a0b457635 100644 --- a/src/testing/testing_test.go +++ b/src/testing/testing_test.go @@ -6,6 +6,8 @@ package testing_test import ( "bytes" + "context" + "errors" "fmt" "internal/race" "internal/testenv" @@ -13,6 +15,7 @@ import ( "os/exec" "path/filepath" "regexp" + "runtime" "slices" "strings" "sync" @@ -86,6 +89,7 @@ func TestTempDir(t *testing.T) { t.Run("test[]", testTempDir) t.Run("test*", testTempDir) t.Run("äöüéè", testTempDir) + t.Run(strings.Repeat("a", 300), testTempDir) } func testTempDir(t *testing.T) { @@ -143,6 +147,39 @@ func testTempDir(t *testing.T) { } } +func TestTempDirGOTMPDIR(t *testing.T) { + // The first call to t.TempDir will create a parent temporary directory + // that will contain all temporary directories created by TempDir. + // + // Use os.TempDir (not t.TempDir) to get a temporary directory, + // set GOTMPDIR to that directory, + // and then verify that t.TempDir creates a directory in GOTMPDIR. + customTmpDir := filepath.Join(os.TempDir(), "custom-gotmpdir-test") + if err := os.MkdirAll(customTmpDir, 0777); err != nil { + t.Fatal(err) + } + defer os.RemoveAll(customTmpDir) + + t.Setenv("GOTMPDIR", customTmpDir) + + dir := t.TempDir() + if dir == "" { + t.Fatal("expected dir") + } + + if !strings.HasPrefix(dir, customTmpDir) { + t.Errorf("TempDir did not use GOTMPDIR: got %q, want prefix %q", dir, customTmpDir) + } + + fi, err := os.Stat(dir) + if err != nil { + t.Fatal(err) + } + if !fi.IsDir() { + t.Errorf("dir %q is not a dir", dir) + } +} + func TestSetenv(t *testing.T) { tests := []struct { name string @@ -200,64 +237,177 @@ func TestSetenv(t *testing.T) { } } -func TestSetenvWithParallelAfterSetenv(t *testing.T) { - defer func() { - want := "testing: t.Parallel called after t.Setenv; cannot set environment variables in parallel tests" - if got := recover(); got != want { - t.Fatalf("expected panic; got %#v want %q", got, want) - } - }() +func expectParallelConflict(t *testing.T) { + want := testing.ParallelConflict + if got := recover(); got != want { + t.Fatalf("expected panic; got %#v want %q", got, want) + } +} - t.Setenv("GO_TEST_KEY_1", "value") +func testWithParallelAfter(t *testing.T, fn func(*testing.T)) { + defer expectParallelConflict(t) + fn(t) t.Parallel() } -func TestSetenvWithParallelBeforeSetenv(t *testing.T) { - defer func() { - want := "testing: t.Setenv called after t.Parallel; cannot set environment variables in parallel tests" - if got := recover(); got != want { - t.Fatalf("expected panic; got %#v want %q", got, want) - } - }() +func testWithParallelBefore(t *testing.T, fn func(*testing.T)) { + defer expectParallelConflict(t) t.Parallel() - - t.Setenv("GO_TEST_KEY_1", "value") + fn(t) } -func TestSetenvWithParallelParentBeforeSetenv(t *testing.T) { +func testWithParallelParentBefore(t *testing.T, fn func(*testing.T)) { t.Parallel() t.Run("child", func(t *testing.T) { - defer func() { - want := "testing: t.Setenv called after t.Parallel; cannot set environment variables in parallel tests" - if got := recover(); got != want { - t.Fatalf("expected panic; got %#v want %q", got, want) - } - }() + defer expectParallelConflict(t) - t.Setenv("GO_TEST_KEY_1", "value") + fn(t) }) } -func TestSetenvWithParallelGrandParentBeforeSetenv(t *testing.T) { +func testWithParallelGrandParentBefore(t *testing.T, fn func(*testing.T)) { t.Parallel() t.Run("child", func(t *testing.T) { t.Run("grand-child", func(t *testing.T) { - defer func() { - want := "testing: t.Setenv called after t.Parallel; cannot set environment variables in parallel tests" - if got := recover(); got != want { - t.Fatalf("expected panic; got %#v want %q", got, want) - } - }() + defer expectParallelConflict(t) - t.Setenv("GO_TEST_KEY_1", "value") + fn(t) }) }) } +func tSetenv(t *testing.T) { + t.Setenv("GO_TEST_KEY_1", "value") +} + +func TestSetenvWithParallelAfter(t *testing.T) { + testWithParallelAfter(t, tSetenv) +} + +func TestSetenvWithParallelBefore(t *testing.T) { + testWithParallelBefore(t, tSetenv) +} + +func TestSetenvWithParallelParentBefore(t *testing.T) { + testWithParallelParentBefore(t, tSetenv) +} + +func TestSetenvWithParallelGrandParentBefore(t *testing.T) { + testWithParallelGrandParentBefore(t, tSetenv) +} + +func tChdir(t *testing.T) { + t.Chdir(t.TempDir()) +} + +func TestChdirWithParallelAfter(t *testing.T) { + testWithParallelAfter(t, tChdir) +} + +func TestChdirWithParallelBefore(t *testing.T) { + testWithParallelBefore(t, tChdir) +} + +func TestChdirWithParallelParentBefore(t *testing.T) { + testWithParallelParentBefore(t, tChdir) +} + +func TestChdirWithParallelGrandParentBefore(t *testing.T) { + testWithParallelGrandParentBefore(t, tChdir) +} + +func TestChdir(t *testing.T) { + oldDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + defer os.Chdir(oldDir) + + // The "relative" test case relies on tmp not being a symlink. + tmp, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + rel, err := filepath.Rel(oldDir, tmp) + if err != nil { + // If GOROOT is on C: volume and tmp is on the D: volume, there + // is no relative path between them, so skip that test case. + rel = "skip" + } + + for _, tc := range []struct { + name, dir, pwd string + extraChdir bool + }{ + { + name: "absolute", + dir: tmp, + pwd: tmp, + }, + { + name: "relative", + dir: rel, + pwd: tmp, + }, + { + name: "current (absolute)", + dir: oldDir, + pwd: oldDir, + }, + { + name: "current (relative) with extra os.Chdir", + dir: ".", + pwd: oldDir, + + extraChdir: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.dir == "skip" { + t.Skipf("skipping test because there is no relative path between %s and %s", oldDir, tmp) + } + if !filepath.IsAbs(tc.pwd) { + t.Fatalf("Bad tc.pwd: %q (must be absolute)", tc.pwd) + } + + t.Chdir(tc.dir) + + newDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if newDir != tc.pwd { + t.Fatalf("failed to chdir to %q: getwd: got %q, want %q", tc.dir, newDir, tc.pwd) + } + + switch runtime.GOOS { + case "windows", "plan9": + // Windows and Plan 9 do not use the PWD variable. + default: + if pwd := os.Getenv("PWD"); pwd != tc.pwd { + t.Fatalf("PWD: got %q, want %q", pwd, tc.pwd) + } + } + + if tc.extraChdir { + os.Chdir("..") + } + }) + + newDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if newDir != oldDir { + t.Fatalf("failed to restore wd to %s: getwd: %s", oldDir, newDir) + } + } +} + // testingTrueInInit is part of TestTesting. var testingTrueInInit = false @@ -319,19 +469,15 @@ func TestTesting(t *testing.T) { // runTest runs a helper test with -test.v, ignoring its exit status. // runTest both logs and returns the test output. -func runTest(t *testing.T, test string) []byte { +func runTest(t *testing.T, test string, args ...string) []byte { t.Helper() testenv.MustHaveExec(t) - exe, err := os.Executable() - if err != nil { - t.Skipf("can't find test executable: %v", err) - } - - cmd := testenv.Command(t, exe, "-test.run=^"+test+"$", "-test.bench="+test, "-test.v", "-test.parallel=2", "-test.benchtime=2x") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^"+test+"$", "-test.bench="+test, "-test.v", "-test.parallel=2", "-test.benchtime=2x") cmd = testenv.CleanCmdEnv(cmd) cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=1") + cmd.Args = append(cmd.Args, args...) out, err := cmd.CombinedOutput() t.Logf("%v: %v\n%s", cmd, err, out) @@ -558,14 +704,7 @@ func TestRaceBeforeParallel(t *testing.T) { } func TestRaceBeforeTests(t *testing.T) { - testenv.MustHaveExec(t) - - exe, err := os.Executable() - if err != nil { - t.Skipf("can't find test executable: %v", err) - } - - cmd := testenv.Command(t, exe, "-test.run=^$") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^$") cmd = testenv.CleanCmdEnv(cmd) cmd.Env = append(cmd.Env, "GO_WANT_RACE_BEFORE_TESTS=1") out, _ := cmd.CombinedOutput() @@ -596,6 +735,20 @@ func TestBenchmarkRace(t *testing.T) { } } +func TestBenchmarkRaceBLoop(t *testing.T) { + out := runTest(t, "BenchmarkBLoopRacy") + c := bytes.Count(out, []byte("race detected during execution of test")) + + want := 0 + // We should see one race detector report. + if race.Enabled { + want = 1 + } + if c != want { + t.Errorf("got %d race reports; want %d", c, want) + } +} + func BenchmarkRacy(b *testing.B) { if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { b.Skipf("skipping intentionally-racy benchmark") @@ -605,15 +758,25 @@ func BenchmarkRacy(b *testing.B) { } } +func BenchmarkBLoopRacy(b *testing.B) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + b.Skipf("skipping intentionally-racy benchmark") + } + for b.Loop() { + doRace() + } +} + func TestBenchmarkSubRace(t *testing.T) { out := runTest(t, "BenchmarkSubRacy") c := bytes.Count(out, []byte("race detected during execution of test")) want := 0 - // We should see two race detector reports: - // one in the sub-bencmark, and one in the parent afterward. + // We should see 3 race detector reports: + // one in the sub-bencmark, one in the parent afterward, + // and one in b.Loop. if race.Enabled { - want = 2 + want = 3 } if c != want { t.Errorf("got %d race reports; want %d", c, want) @@ -639,6 +802,12 @@ func BenchmarkSubRacy(b *testing.B) { } }) + b.Run("racy-bLoop", func(b *testing.B) { + for b.Loop() { + doRace() + } + }) + doRace() // should be reported separately } @@ -667,7 +836,7 @@ func TestRunningTests(t *testing.T) { timeout := 10 * time.Millisecond for { - cmd := testenv.Command(t, os.Args[0], "-test.run=^"+t.Name()+"$", "-test.timeout="+timeout.String(), "-test.parallel=4") + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^"+t.Name()+"$", "-test.timeout="+timeout.String(), "-test.parallel=4") cmd.Env = append(cmd.Environ(), "GO_WANT_HELPER_PROCESS=1") out, err := cmd.CombinedOutput() t.Logf("%v:\n%s", cmd, out) @@ -726,7 +895,7 @@ func TestRunningTestsInCleanup(t *testing.T) { timeout := 10 * time.Millisecond for { - cmd := testenv.Command(t, os.Args[0], "-test.run=^"+t.Name()+"$", "-test.timeout="+timeout.String()) + cmd := testenv.Command(t, testenv.Executable(t), "-test.run=^"+t.Name()+"$", "-test.timeout="+timeout.String()) cmd.Env = append(cmd.Environ(), "GO_WANT_HELPER_PROCESS=1") out, err := cmd.CombinedOutput() t.Logf("%v:\n%s", cmd, out) @@ -760,7 +929,7 @@ func TestRunningTestsInCleanup(t *testing.T) { func parseRunningTests(out []byte) (runningTests []string, ok bool) { inRunningTests := false - for _, line := range strings.Split(string(out), "\n") { + for line := range strings.SplitSeq(string(out), "\n") { if inRunningTests { // Package testing adds one tab, the panic printer adds another. if trimmed, ok := strings.CutPrefix(line, "\t\t"); ok { @@ -813,3 +982,235 @@ func TestParentRun(t1 *testing.T) { }) }) } + +func TestContext(t *testing.T) { + ctx := t.Context() + if err := ctx.Err(); err != nil { + t.Fatalf("expected non-canceled context, got %v", err) + } + + var innerCtx context.Context + t.Run("inner", func(t *testing.T) { + innerCtx = t.Context() + if err := innerCtx.Err(); err != nil { + t.Fatalf("expected inner test to not inherit canceled context, got %v", err) + } + }) + t.Run("inner2", func(t *testing.T) { + if !errors.Is(innerCtx.Err(), context.Canceled) { + t.Fatal("expected context of sibling test to be canceled after its test function finished") + } + }) + + t.Cleanup(func() { + if !errors.Is(ctx.Err(), context.Canceled) { + t.Fatal("expected context canceled before cleanup") + } + }) +} + +// TestAttrExample is used by TestAttrSet, +// and also serves as a convenient test to run that sets an attribute. +func TestAttrExample(t *testing.T) { + t.Attr("key", "value") +} + +func TestAttrSet(t *testing.T) { + out := string(runTest(t, "TestAttrExample")) + + want := "=== ATTR TestAttrExample key value\n" + if !strings.Contains(out, want) { + t.Errorf("expected output containing %q, got:\n%q", want, out) + } +} + +func TestAttrInvalid(t *testing.T) { + tests := []struct { + key string + value string + }{ + {"k ey", "value"}, + {"k\tey", "value"}, + {"k\rey", "value"}, + {"k\ney", "value"}, + {"key", "val\rue"}, + {"key", "val\nue"}, + } + + if os.Getenv("GO_WANT_HELPER_PROCESS") == "1" { + for i, test := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + t.Attr(test.key, test.value) + }) + } + return + } + + out := string(runTest(t, "TestAttrInvalid")) + + for i := range tests { + want := fmt.Sprintf("--- FAIL: TestAttrInvalid/%v ", i) + if !strings.Contains(out, want) { + t.Errorf("expected output containing %q, got:\n%q", want, out) + } + } +} + +const artifactContent = "It belongs in a museum.\n" + +func TestArtifactDirExample(t *testing.T) { + os.WriteFile(filepath.Join(t.ArtifactDir(), "artifact"), []byte(artifactContent), 0o666) +} + +func TestArtifactDirDefault(t *testing.T) { + tempDir := t.TempDir() + t.Chdir(tempDir) + out := runTest(t, "TestArtifactDirExample", "-test.artifacts") + checkArtifactDir(t, out, "TestArtifactDirExample", tempDir) +} + +func TestArtifactDirSpecified(t *testing.T) { + tempDir := t.TempDir() + out := runTest(t, "TestArtifactDirExample", "-test.artifacts", "-test.outputdir="+tempDir) + checkArtifactDir(t, out, "TestArtifactDirExample", tempDir) +} + +func TestArtifactDirNoArtifacts(t *testing.T) { + t.Chdir(t.TempDir()) + out := string(runTest(t, "TestArtifactDirExample")) + if strings.Contains(out, "=== ARTIFACTS") { + t.Errorf("expected output with no === ARTIFACTS, got\n%q", out) + } + ents, err := os.ReadDir(".") + if err != nil { + t.Fatal(err) + } + for _, e := range ents { + t.Errorf("unexpected file in current directory after test: %v", e.Name()) + } +} + +func TestArtifactDirSubtestExample(t *testing.T) { + t.Run("Subtest", func(t *testing.T) { + os.WriteFile(filepath.Join(t.ArtifactDir(), "artifact"), []byte(artifactContent), 0o666) + }) +} + +func TestArtifactDirInSubtest(t *testing.T) { + tempDir := t.TempDir() + out := runTest(t, "TestArtifactDirSubtestExample/Subtest", "-test.artifacts", "-test.outputdir="+tempDir) + checkArtifactDir(t, out, "TestArtifactDirSubtestExample/Subtest", tempDir) +} + +func TestArtifactDirLongTestNameExample(t *testing.T) { + name := strings.Repeat("x", 256) + t.Run(name, func(t *testing.T) { + os.WriteFile(filepath.Join(t.ArtifactDir(), "artifact"), []byte(artifactContent), 0o666) + }) +} + +func TestArtifactDirWithLongTestName(t *testing.T) { + tempDir := t.TempDir() + out := runTest(t, "TestArtifactDirLongTestNameExample", "-test.artifacts", "-test.outputdir="+tempDir) + checkArtifactDir(t, out, `TestArtifactDirLongTestNameExample/\w+`, tempDir) +} + +func TestArtifactDirConsistent(t *testing.T) { + a := t.ArtifactDir() + b := t.ArtifactDir() + if a != b { + t.Errorf("t.ArtifactDir is not consistent between calls: %q, %q", a, b) + } +} + +func checkArtifactDir(t *testing.T, out []byte, testName, outputDir string) { + t.Helper() + + re := regexp.MustCompile(`=== ARTIFACTS ` + testName + ` ([^\n]+)`) + match := re.FindSubmatch(out) + if match == nil { + t.Fatalf("expected output matching %q, got\n%q", re, out) + } + artifactDir := string(match[1]) + + // Verify that the artifact directory is contained in the expected output directory. + relDir, err := filepath.Rel(outputDir, artifactDir) + if err != nil { + t.Fatal(err) + } + if !filepath.IsLocal(relDir) { + t.Fatalf("want artifact directory contained in %q, got %q", outputDir, artifactDir) + } + + for _, part := range strings.Split(relDir, string(os.PathSeparator)) { + const maxSize = 64 + if len(part) > maxSize { + t.Errorf("artifact directory %q contains component >%v characters long: %q", relDir, maxSize, part) + } + } + + got, err := os.ReadFile(filepath.Join(artifactDir, "artifact")) + if err != nil || string(got) != artifactContent { + t.Errorf("reading artifact in %q: got %q, %v; want %q", artifactDir, got, err, artifactContent) + } +} + +func TestBenchmarkBLoopIterationCorrect(t *testing.T) { + out := runTest(t, "BenchmarkBLoopPrint") + c := bytes.Count(out, []byte("Printing from BenchmarkBLoopPrint")) + + want := 2 + if c != want { + t.Errorf("got %d loop iterations; want %d", c, want) + } + + // b.Loop() will only rampup once. + c = bytes.Count(out, []byte("Ramping up from BenchmarkBLoopPrint")) + want = 1 + if c != want { + t.Errorf("got %d loop rampup; want %d", c, want) + } + + re := regexp.MustCompile(`BenchmarkBLoopPrint(-[0-9]+)?\s+2\s+[0-9]+\s+ns/op`) + if !re.Match(out) { + t.Error("missing benchmark output") + } +} + +func TestBenchmarkBNIterationCorrect(t *testing.T) { + out := runTest(t, "BenchmarkBNPrint") + c := bytes.Count(out, []byte("Printing from BenchmarkBNPrint")) + + // runTest sets benchtime=2x, with semantics specified in #32051 it should + // run 3 times. + want := 3 + if c != want { + t.Errorf("got %d loop iterations; want %d", c, want) + } + + // b.N style fixed iteration loop will rampup twice: + // One in run1(), the other in launch + c = bytes.Count(out, []byte("Ramping up from BenchmarkBNPrint")) + want = 2 + if c != want { + t.Errorf("got %d loop rampup; want %d", c, want) + } +} + +func BenchmarkBLoopPrint(b *testing.B) { + b.Logf("Ramping up from BenchmarkBLoopPrint") + for b.Loop() { + b.Logf("Printing from BenchmarkBLoopPrint") + } +} + +func BenchmarkBNPrint(b *testing.B) { + b.Logf("Ramping up from BenchmarkBNPrint") + for i := 0; i < b.N; i++ { + b.Logf("Printing from BenchmarkBNPrint") + } +} + +func TestArtifactDir(t *testing.T) { + t.Log(t.ArtifactDir()) +} diff --git a/src/text/scanner/scanner.go b/src/text/scanner/scanner.go index 6ae7a9b987423b..316fb4380f753c 100644 --- a/src/text/scanner/scanner.go +++ b/src/text/scanner/scanner.go @@ -50,7 +50,7 @@ func (pos Position) String() string { // to configure a [Scanner] such that it only recognizes (Go) identifiers, // integers, and skips comments, set the Scanner's Mode field to: // -// ScanIdents | ScanInts | SkipComments +// ScanIdents | ScanInts | ScanComments | SkipComments // // With the exceptions of comments, which are skipped if SkipComments is // set, unrecognized tokens are not ignored. Instead, the scanner simply diff --git a/src/text/template/doc.go b/src/text/template/doc.go index 12f6fe0d1c9cf2..01b99f0e2123a1 100644 --- a/src/text/template/doc.go +++ b/src/text/template/doc.go @@ -15,6 +15,11 @@ Execution of the template walks the structure and sets the cursor, represented by a period '.' and called "dot", to the value at the current location in the structure as execution proceeds. +The security model used by this package assumes that template authors are +trusted. The package does not auto-escape output, so injecting code into +a template can lead to arbitrary code execution if the template is executed +by an untrusted source. + The input text for a template is UTF-8-encoded text in any format. "Actions"--data evaluations or control structures--are delimited by "{{" and "}}"; all text outside actions is copied to the output unchanged. @@ -98,7 +103,8 @@ data, defined in detail in the corresponding sections that follow. {{if pipeline}} T1 {{else}}{{if pipeline}} T0 {{end}}{{end}} {{range pipeline}} T1 {{end}} - The value of the pipeline must be an array, slice, map, or channel. + The value of the pipeline must be an array, slice, map, iter.Seq, + iter.Seq2, integer or channel. If the value of the pipeline has length zero, nothing is output; otherwise, dot is set to the successive elements of the array, slice, or map and T1 is executed. If the value is a map and the @@ -106,7 +112,8 @@ data, defined in detail in the corresponding sections that follow. visited in sorted key order. {{range pipeline}} T1 {{else}} T0 {{end}} - The value of the pipeline must be an array, slice, map, or channel. + The value of the pipeline must be an array, slice, map, iter.Seq, + iter.Seq2, integer or channel. If the value of the pipeline has length zero, dot is unaffected and T0 is executed; otherwise, dot is set to the successive elements of the array, slice, or map and T1 is executed. diff --git a/src/text/template/examplefunc_test.go b/src/text/template/examplefunc_test.go index 080b5e3a0564b0..a1879e7c3f988e 100644 --- a/src/text/template/examplefunc_test.go +++ b/src/text/template/examplefunc_test.go @@ -52,3 +52,72 @@ Output 2: {{printf "%q" . | title}} // Output 1: "The Go Programming Language" // Output 2: "The Go Programming Language" } + +// This example demonstrates registering two custom template functions +// and how to overwite one of the functions after the template has been +// parsed. Overwriting can be used, for example, to alter the operation +// of cloned templates. +func ExampleTemplate_funcs() { + + // Define a simple template to test the functions. + const tmpl = `{{ . | lower | repeat }}` + + // Define the template funcMap with two functions. + var funcMap = template.FuncMap{ + "lower": strings.ToLower, + "repeat": func(s string) string { return strings.Repeat(s, 2) }, + } + + // Define a New template, add the funcMap using Funcs and then Parse + // the template. + parsedTmpl, err := template.New("t").Funcs(funcMap).Parse(tmpl) + if err != nil { + log.Fatal(err) + } + if err := parsedTmpl.Execute(os.Stdout, "ABC\n"); err != nil { + log.Fatal(err) + } + + // [Funcs] must be called before a template is parsed to add + // functions to the template. [Funcs] can also be used after a + // template is parsed to overwrite template functions. + // + // Here the function identified by "repeat" is overwritten. + parsedTmpl.Funcs(template.FuncMap{ + "repeat": func(s string) string { return strings.Repeat(s, 3) }, + }) + if err := parsedTmpl.Execute(os.Stdout, "DEF\n"); err != nil { + log.Fatal(err) + } + // Output: + // abc + // abc + // def + // def + // def +} + +// This example demonstrates how to use "if". +func ExampleTemplate_if() { + type book struct { + Stars float32 + Name string + } + + tpl, err := template.New("book").Parse(`{{ if (gt .Stars 4.0) }}"{{.Name }}" is a great book.{{ else }}"{{.Name}}" is not a great book.{{ end }}`) + if err != nil { + log.Fatalf("failed to parse template: %s", err) + } + + b := &book{ + Stars: 4.9, + Name: "Good Night, Gopher", + } + err = tpl.Execute(os.Stdout, b) + if err != nil { + log.Fatalf("failed to execute template: %s", err) + } + + // Output: + // "Good Night, Gopher" is a great book. +} diff --git a/src/text/template/exec.go b/src/text/template/exec.go index 5b35b3e5a85b17..7a67ec6824f5e5 100644 --- a/src/text/template/exec.go +++ b/src/text/template/exec.go @@ -333,7 +333,7 @@ func isTrue(val reflect.Value) (truth, ok bool) { truth = val.Bool() case reflect.Complex64, reflect.Complex128: truth = val.Complex() != 0 - case reflect.Chan, reflect.Func, reflect.Pointer, reflect.Interface: + case reflect.Chan, reflect.Func, reflect.Pointer, reflect.UnsafePointer, reflect.Interface: truth = !val.IsNil() case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: truth = val.Int() != 0 @@ -395,6 +395,22 @@ func (s *state) walkRange(dot reflect.Value, r *parse.RangeNode) { s.walk(elem, r.List) } switch val.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + if len(r.Pipe.Decl) > 1 { + s.errorf("can't use %v to iterate over more than one variable", val) + break + } + run := false + for v := range val.Seq() { + run = true + // Pass element as second value, as we do for channels. + oneIteration(reflect.Value{}, v) + } + if !run { + break + } + return case reflect.Array, reflect.Slice: if val.Len() == 0 { break @@ -434,6 +450,43 @@ func (s *state) walkRange(dot reflect.Value, r *parse.RangeNode) { return case reflect.Invalid: break // An invalid value is likely a nil map, etc. and acts like an empty map. + case reflect.Func: + if val.Type().CanSeq() { + if len(r.Pipe.Decl) > 1 { + s.errorf("can't use %v iterate over more than one variable", val) + break + } + run := false + for v := range val.Seq() { + run = true + // Pass element as second value, + // as we do for channels. + oneIteration(reflect.Value{}, v) + } + if !run { + break + } + return + } + if val.Type().CanSeq2() { + run := false + for i, v := range val.Seq2() { + run = true + if len(r.Pipe.Decl) > 1 { + oneIteration(i, v) + } else { + // If there is only one range variable, + // oneIteration will use the + // second value. + oneIteration(reflect.Value{}, i) + } + } + if !run { + break + } + return + } + fallthrough default: s.errorf("range can't iterate over %v", val) } @@ -757,7 +810,7 @@ func (s *state) evalCall(dot, fun reflect.Value, isBuiltin bool, node parse.Node return v } } - if final != missingVal { + if !final.Equal(missingVal) { // The last argument to and/or is coming from // the pipeline. We didn't short circuit on an earlier // argument, so we are going to return this one. @@ -803,7 +856,13 @@ func (s *state) evalCall(dot, fun reflect.Value, isBuiltin bool, node parse.Node // Special case for the "call" builtin. // Insert the name of the callee function as the first argument. if isBuiltin && name == "call" { - calleeName := args[0].String() + var calleeName string + if len(args) == 0 { + // final must be present or we would have errored out above. + calleeName = final.String() + } else { + calleeName = args[0].String() + } argv = append([]reflect.Value{reflect.ValueOf(calleeName)}, argv...) fun = reflect.ValueOf(call) } diff --git a/src/text/template/exec_test.go b/src/text/template/exec_test.go index 9903e17d0ef2c8..8665f3ad4987c2 100644 --- a/src/text/template/exec_test.go +++ b/src/text/template/exec_test.go @@ -10,10 +10,12 @@ import ( "flag" "fmt" "io" + "iter" "reflect" "strings" "sync" "testing" + "unsafe" ) var debug = flag.Bool("debug", false, "show the errors produced by the tests") @@ -70,10 +72,12 @@ type T struct { Str fmt.Stringer Err error // Pointers - PI *int - PS *string - PSI *[]int - NIL *int + PI *int + PS *string + PSI *[]int + NIL *int + UPI unsafe.Pointer + EmptyUPI unsafe.Pointer // Function (not method) BinaryFunc func(string, string) string VariadicFunc func(...string) string @@ -165,6 +169,7 @@ var tVal = &T{ PI: newInt(23), PS: newString("a string"), PSI: newIntSlice(21, 22, 23), + UPI: newUnsafePointer(23), BinaryFunc: func(a, b string) string { return fmt.Sprintf("[%s=%s]", a, b) }, VariadicFunc: func(s ...string) string { return fmt.Sprint("<", strings.Join(s, "+"), ">") }, VariadicFuncInt: func(a int, s ...string) string { return fmt.Sprint(a, "=<", strings.Join(s, "+"), ">") }, @@ -191,6 +196,10 @@ func newInt(n int) *int { return &n } +func newUnsafePointer(n int) unsafe.Pointer { + return unsafe.Pointer(&n) +} + func newString(s string) *string { return &s } @@ -399,6 +408,9 @@ var execTests = []execTest{ {"Interface Call", `{{stringer .S}}`, "foozle", map[string]any{"S": bytes.NewBufferString("foozle")}, true}, {".ErrFunc", "{{call .ErrFunc}}", "bla", tVal, true}, {"call nil", "{{call nil}}", "", tVal, false}, + {"empty call", "{{call}}", "", tVal, false}, + {"empty call after pipe valid", "{{.ErrFunc | call}}", "bla", tVal, true}, + {"empty call after pipe invalid", "{{1 | call}}", "", tVal, false}, // Erroneous function calls (check args). {".BinaryFuncTooFew", "{{call .BinaryFunc `1`}}", "", tVal, false}, @@ -439,6 +451,10 @@ var execTests = []execTest{ {"if 0.0", "{{if .FloatZero}}NON-ZERO{{else}}ZERO{{end}}", "ZERO", tVal, true}, {"if 1.5i", "{{if 1.5i}}NON-ZERO{{else}}ZERO{{end}}", "NON-ZERO", tVal, true}, {"if 0.0i", "{{if .ComplexZero}}NON-ZERO{{else}}ZERO{{end}}", "ZERO", tVal, true}, + {"if nonNilPointer", "{{if .PI}}NON-ZERO{{else}}ZERO{{end}}", "NON-ZERO", tVal, true}, + {"if nilPointer", "{{if .NIL}}NON-ZERO{{else}}ZERO{{end}}", "ZERO", tVal, true}, + {"if UPI", "{{if .UPI}}NON-ZERO{{else}}ZERO{{end}}", "NON-ZERO", tVal, true}, + {"if EmptyUPI", "{{if .EmptyUPI}}NON-ZERO{{else}}ZERO{{end}}", "ZERO", tVal, true}, {"if emptystring", "{{if ``}}NON-EMPTY{{else}}EMPTY{{end}}", "EMPTY", tVal, true}, {"if string", "{{if `notempty`}}NON-EMPTY{{else}}EMPTY{{end}}", "NON-EMPTY", tVal, true}, {"if emptyslice", "{{if .SIEmpty}}NON-EMPTY{{else}}EMPTY{{end}}", "EMPTY", tVal, true}, @@ -601,6 +617,30 @@ var execTests = []execTest{ {"declare in range", "{{range $x := .PSI}}<{{$foo:=$x}}{{$x}}>{{end}}", "<21><22><23>", tVal, true}, {"range count", `{{range $i, $x := count 5}}[{{$i}}]{{$x}}{{end}}`, "[0]a[1]b[2]c[3]d[4]e", tVal, true}, {"range nil count", `{{range $i, $x := count 0}}{{else}}empty{{end}}`, "empty", tVal, true}, + {"range iter.Seq[int]", `{{range $i := .}}{{$i}}{{end}}`, "01", fVal1(2), true}, + {"i = range iter.Seq[int]", `{{$i := 0}}{{range $i = .}}{{$i}}{{end}}`, "01", fVal1(2), true}, + {"range iter.Seq[int] over two var", `{{range $i, $c := .}}{{$c}}{{end}}`, "", fVal1(2), false}, + {"i, c := range iter.Seq2[int,int]", `{{range $i, $c := .}}{{$i}}{{$c}}{{end}}`, "0112", fVal2(2), true}, + {"i, c = range iter.Seq2[int,int]", `{{$i := 0}}{{$c := 0}}{{range $i, $c = .}}{{$i}}{{$c}}{{end}}`, "0112", fVal2(2), true}, + {"i = range iter.Seq2[int,int]", `{{$i := 0}}{{range $i = .}}{{$i}}{{end}}`, "01", fVal2(2), true}, + {"i := range iter.Seq2[int,int]", `{{range $i := .}}{{$i}}{{end}}`, "01", fVal2(2), true}, + {"i,c,x range iter.Seq2[int,int]", `{{$i := 0}}{{$c := 0}}{{$x := 0}}{{range $i, $c = .}}{{$i}}{{$c}}{{end}}`, "0112", fVal2(2), true}, + {"i,x range iter.Seq[int]", `{{$i := 0}}{{$x := 0}}{{range $i = .}}{{$i}}{{end}}`, "01", fVal1(2), true}, + {"range iter.Seq[int] else", `{{range $i := .}}{{$i}}{{else}}empty{{end}}`, "empty", fVal1(0), true}, + {"range iter.Seq2[int,int] else", `{{range $i := .}}{{$i}}{{else}}empty{{end}}`, "empty", fVal2(0), true}, + {"range int8", rangeTestInt, rangeTestData[int8](), int8(5), true}, + {"range int16", rangeTestInt, rangeTestData[int16](), int16(5), true}, + {"range int32", rangeTestInt, rangeTestData[int32](), int32(5), true}, + {"range int64", rangeTestInt, rangeTestData[int64](), int64(5), true}, + {"range int", rangeTestInt, rangeTestData[int](), int(5), true}, + {"range uint8", rangeTestInt, rangeTestData[uint8](), uint8(5), true}, + {"range uint16", rangeTestInt, rangeTestData[uint16](), uint16(5), true}, + {"range uint32", rangeTestInt, rangeTestData[uint32](), uint32(5), true}, + {"range uint64", rangeTestInt, rangeTestData[uint64](), uint64(5), true}, + {"range uint", rangeTestInt, rangeTestData[uint](), uint(5), true}, + {"range uintptr", rangeTestInt, rangeTestData[uintptr](), uintptr(5), true}, + {"range uintptr(0)", `{{range $v := .}}{{print $v}}{{else}}empty{{end}}`, "empty", uintptr(0), true}, + {"range 5", `{{range $v := 5}}{{printf "%T%d" $v $v}}{{end}}`, rangeTestData[int](), nil, true}, // Cute examples. {"or as if true", `{{or .SI "slice is empty"}}`, "[3 4 5]", tVal, true}, @@ -705,6 +745,37 @@ var execTests = []execTest{ {"issue60801", "{{$k := 0}}{{$v := 0}}{{range $k, $v = .AI}}{{$k}}={{$v}} {{end}}", "0=3 1=4 2=5 ", tVal, true}, } +func fVal1(i int) iter.Seq[int] { + return func(yield func(int) bool) { + for v := range i { + if !yield(v) { + break + } + } + } +} + +func fVal2(i int) iter.Seq2[int, int] { + return func(yield func(int, int) bool) { + for v := range i { + if !yield(v, v+1) { + break + } + } + } +} + +const rangeTestInt = `{{range $v := .}}{{printf "%T%d" $v $v}}{{end}}` + +func rangeTestData[T int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64 | uintptr]() string { + I := T(5) + var buf strings.Builder + for i := T(0); i < I; i++ { + fmt.Fprintf(&buf, "%T%d", i, i) + } + return buf.String() +} + func zeroArgs() string { return "zeroArgs" } @@ -944,8 +1015,7 @@ func TestExecError_CustomError(t *testing.T) { var b bytes.Buffer err := tmpl.Execute(&b, nil) - var e *CustomError - if !errors.As(err, &e) { + if _, ok := errors.AsType[*CustomError](err); !ok { t.Fatalf("expected custom error; got %s", err) } } @@ -1434,6 +1504,44 @@ func TestBadFuncNames(t *testing.T) { } } +func TestIsTrue(t *testing.T) { + var nil_ptr *int + var nil_chan chan int + tests := []struct { + v any + want bool + }{ + {1, true}, + {0, false}, + {uint8(1), true}, + {uint8(0), false}, + {float64(1.0), true}, + {float64(0.0), false}, + {complex64(1.0), true}, + {complex64(0.0), false}, + {true, true}, + {false, false}, + {[2]int{1, 2}, true}, + {[0]int{}, false}, + {[]byte("abc"), true}, + {[]byte(""), false}, + {map[string]int{"a": 1, "b": 2}, true}, + {map[string]int{}, false}, + {make(chan int), true}, + {nil_chan, false}, + {new(int), true}, + {nil_ptr, false}, + {unsafe.Pointer(new(int)), true}, + {unsafe.Pointer(nil_ptr), false}, + } + for _, test_case := range tests { + got, _ := IsTrue(test_case.v) + if got != test_case.want { + t.Fatalf("expect result %v, got %v", test_case.want, got) + } + } +} + func testBadFuncName(name string, t *testing.T) { t.Helper() defer func() { diff --git a/src/text/template/funcs.go b/src/text/template/funcs.go index 7d63cf8b7bb6db..30b3243a5a8416 100644 --- a/src/text/template/funcs.go +++ b/src/text/template/funcs.go @@ -22,7 +22,7 @@ import ( // return value evaluates to non-nil during execution, execution terminates and // Execute returns that error. // -// Errors returned by Execute wrap the underlying error; call [errors.As] to +// Errors returned by Execute wrap the underlying error; call [errors.AsType] to // unwrap them. // // When template execution invokes a function with an argument list, that list @@ -62,26 +62,13 @@ func builtins() FuncMap { } } -var builtinFuncsOnce struct { - sync.Once - v map[string]reflect.Value -} - -// builtinFuncsOnce lazily computes & caches the builtinFuncs map. -// TODO: revert this back to a global map once golang.org/issue/2559 is fixed. -func builtinFuncs() map[string]reflect.Value { - builtinFuncsOnce.Do(func() { - builtinFuncsOnce.v = createValueFuncs(builtins()) - }) - return builtinFuncsOnce.v -} - -// createValueFuncs turns a FuncMap into a map[string]reflect.Value -func createValueFuncs(funcMap FuncMap) map[string]reflect.Value { - m := make(map[string]reflect.Value) +// builtinFuncs lazily computes & caches the builtinFuncs map. +var builtinFuncs = sync.OnceValue(func() map[string]reflect.Value { + funcMap := builtins() + m := make(map[string]reflect.Value, len(funcMap)) addValueFuncs(m, funcMap) return m -} +}) // addValueFuncs adds to values the functions in funcs, converting them to reflect.Values. func addValueFuncs(out map[string]reflect.Value, in FuncMap) { @@ -409,7 +396,6 @@ func not(arg reflect.Value) bool { var ( errBadComparisonType = errors.New("invalid type for comparison") - errBadComparison = errors.New("incompatible types for comparison") errNoComparison = errors.New("missing argument for comparison") ) @@ -487,7 +473,7 @@ func eq(arg1 reflect.Value, arg2 ...reflect.Value) (bool, error) { truth = arg.Int() >= 0 && arg1.Uint() == uint64(arg.Int()) default: if arg1.IsValid() && arg.IsValid() { - return false, errBadComparison + return false, fmt.Errorf("incompatible types for comparison: %v and %v", arg1.Type(), arg.Type()) } } } else { @@ -553,7 +539,7 @@ func lt(arg1, arg2 reflect.Value) (bool, error) { case k1 == uintKind && k2 == intKind: truth = arg2.Int() >= 0 && arg1.Uint() < uint64(arg2.Int()) default: - return false, errBadComparison + return false, fmt.Errorf("incompatible types for comparison: %v and %v", arg1.Type(), arg2.Type()) } } else { switch k1 { diff --git a/src/text/template/multi_test.go b/src/text/template/multi_test.go index 63cd3f74b270a6..1e4f61b3f1bc4a 100644 --- a/src/text/template/multi_test.go +++ b/src/text/template/multi_test.go @@ -210,6 +210,7 @@ const ( cloneText2 = `{{define "b"}}b{{end}}` cloneText3 = `{{define "c"}}root{{end}}` cloneText4 = `{{define "c"}}clone{{end}}` + cloneText5 = `{{define "e"}}{{.Foo}}{{end}}` ) func TestClone(t *testing.T) { @@ -222,6 +223,8 @@ func TestClone(t *testing.T) { if err != nil { t.Fatal(err) } + root.Parse(cloneText5) + root.Option("missingkey=error") clone := Must(root.Clone()) // Add variants to both. _, err = root.Parse(cloneText3) @@ -259,6 +262,14 @@ func TestClone(t *testing.T) { if b.String() != "bclone" { t.Errorf("expected %q got %q", "bclone", b.String()) } + b.Reset() + rootErr := root.ExecuteTemplate(&b, "e", map[string]any{}) + cloneErr := clone.ExecuteTemplate(&b, "e", map[string]any{}) + if cloneErr == nil { + t.Errorf("expected error from missing key in cloned template") + } else if got, want := cloneErr.Error(), rootErr.Error(); got != want { + t.Errorf("got %q, wan t %q", got, want) + } } func TestAddParseTree(t *testing.T) { diff --git a/src/text/template/parse/lex.go b/src/text/template/parse/lex.go index 70fc86b63cccdc..a00f48e658ab64 100644 --- a/src/text/template/parse/lex.go +++ b/src/text/template/parse/lex.go @@ -352,6 +352,7 @@ func lexComment(l *lexer) stateFn { if !delim { return l.errorf("comment ends before closing delimiter") } + l.line += strings.Count(l.input[l.start:l.pos], "\n") i := l.thisItem(itemComment) if trimSpace { l.pos += trimMarkerLen diff --git a/src/text/template/parse/lex_test.go b/src/text/template/parse/lex_test.go index d47f10f9d5e9b9..bd535c4ce57005 100644 --- a/src/text/template/parse/lex_test.go +++ b/src/text/template/parse/lex_test.go @@ -545,6 +545,16 @@ var lexPosTests = []lexTest{ {itemRightDelim, 11, "}}", 2}, {itemEOF, 13, "", 2}, }}, + {"longcomment", "{{/*\n*/}}\n{{undefinedFunction \"test\"}}", []item{ + {itemComment, 2, "/*\n*/", 1}, + {itemText, 9, "\n", 2}, + {itemLeftDelim, 10, "{{", 3}, + {itemIdentifier, 12, "undefinedFunction", 3}, + {itemSpace, 29, " ", 3}, + {itemString, 30, "\"test\"", 3}, + {itemRightDelim, 36, "}}", 3}, + {itemEOF, 38, "", 3}, + }}, } // The other tests don't check position, to make the test cases easier to construct. @@ -568,15 +578,3 @@ func TestPos(t *testing.T) { } } } - -// parseLexer is a local version of parse that lets us pass in the lexer instead of building it. -// We expect an error, so the tree set and funcs list are explicitly nil. -func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) { - defer t.recover(&err) - t.ParseName = t.Name - t.startParse(nil, lex, map[string]*Tree{}) - t.parse() - t.add() - t.stopParse() - return t, nil -} diff --git a/src/text/template/parse/parse.go b/src/text/template/parse/parse.go index 27c84f31eb3a9b..b74dfb7f4eaa5c 100644 --- a/src/text/template/parse/parse.go +++ b/src/text/template/parse/parse.go @@ -32,9 +32,10 @@ type Tree struct { treeSet map[string]*Tree actionLine int // line of left delim starting action rangeDepth int + stackDepth int // depth of nested parenthesized expressions } -// A mode value is a set of flags (or 0). Modes control parser behavior. +// A Mode value is a set of flags (or 0). Modes control parser behavior. type Mode uint const ( @@ -42,6 +43,17 @@ const ( SkipFuncCheck // do not check that functions are defined ) +// maxStackDepth is the maximum depth permitted for nested +// parenthesized expressions. +var maxStackDepth = 10000 + +// init reduces maxStackDepth for WebAssembly due to its smaller stack size. +func init() { + if runtime.GOARCH == "wasm" { + maxStackDepth = 1000 + } +} + // Copy returns a copy of the [Tree]. Any parsing state is discarded. func (t *Tree) Copy() *Tree { if t == nil { @@ -223,6 +235,7 @@ func (t *Tree) startParse(funcs []map[string]any, lex *lexer, treeSet map[string t.vars = []string{"$"} t.funcs = funcs t.treeSet = treeSet + t.stackDepth = 0 lex.options = lexOptions{ emitComment: t.Mode&ParseComments != 0, breakOK: !t.hasFunction("break"), @@ -787,6 +800,11 @@ func (t *Tree) term() Node { } return number case itemLeftParen: + if t.stackDepth >= maxStackDepth { + t.errorf("max expression depth exceeded") + } + t.stackDepth++ + defer func() { t.stackDepth-- }() return t.pipeline("parenthesized pipeline", itemRightParen) case itemString, itemRawString: s, err := strconv.Unquote(token.val) diff --git a/src/text/template/parse/parse_test.go b/src/text/template/parse/parse_test.go index 26aff330fe8a27..e8e6fe9759d780 100644 --- a/src/text/template/parse/parse_test.go +++ b/src/text/template/parse/parse_test.go @@ -86,6 +86,11 @@ var numberTests = []numberTest{ {"0xef", true, true, true, false, 0xef, 0xef, 0xef, 0}, } +func init() { + // Use a small stack limit for testing to avoid creating huge expressions. + maxStackDepth = 3 +} + func TestNumberParse(t *testing.T) { for _, test := range numberTests { // If fmt.Sscan thinks it's complex, it's complex. We can't trust the output @@ -327,6 +332,15 @@ var parseTests = []parseTest{ {"empty pipeline", `{{printf "%d" ( ) }}`, hasError, ""}, // Missing pipeline in block {"block definition", `{{block "foo"}}hello{{end}}`, hasError, ""}, + + // Expression nested depth tests. + {"paren nesting normal", "{{ (( 1 )) }}", noError, "{{((1))}}"}, + {"paren nesting at limit", "{{ ((( 1 ))) }}", noError, "{{(((1)))}}"}, + {"paren nesting exceeds limit", "{{ (((( 1 )))) }}", hasError, "template: test:1: max expression depth exceeded"}, + {"paren nesting in pipeline", "{{ ((( 1 ))) | printf }}", noError, "{{(((1))) | printf}}"}, + {"paren nesting in pipeline exceeds limit", "{{ (((( 1 )))) | printf }}", hasError, "template: test:1: max expression depth exceeded"}, + {"paren nesting with other constructs", "{{ if ((( true ))) }}YES{{ end }}", noError, "{{if (((true)))}}\"YES\"{{end}}"}, + {"paren nesting with other constructs exceeds limit", "{{ if (((( true )))) }}YES{{ end }}", hasError, "template: test:1: max expression depth exceeded"}, } var builtins = map[string]any{ diff --git a/src/text/template/template.go b/src/text/template/template.go index 86fd3f122a1206..9ae5a6ca5bfa21 100644 --- a/src/text/template/template.go +++ b/src/text/template/template.go @@ -5,6 +5,7 @@ package template import ( + "maps" "reflect" "sync" "text/template/parse" @@ -89,6 +90,7 @@ func (t *Template) Clone() (*Template, error) { if t.common == nil { return nt, nil } + nt.option = t.option t.muTmpl.RLock() defer t.muTmpl.RUnlock() for k, v := range t.tmpl { @@ -102,12 +104,8 @@ func (t *Template) Clone() (*Template, error) { } t.muFuncs.RLock() defer t.muFuncs.RUnlock() - for k, v := range t.parseFuncs { - nt.parseFuncs[k] = v - } - for k, v := range t.execFuncs { - nt.execFuncs[k] = v - } + maps.Copy(nt.parseFuncs, t.parseFuncs) + maps.Copy(nt.execFuncs, t.execFuncs) return nt, nil } diff --git a/src/time/abs_test.go b/src/time/abs_test.go new file mode 100644 index 00000000000000..61d093a30e76a2 --- /dev/null +++ b/src/time/abs_test.go @@ -0,0 +1,183 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package time + +type testingT interface { + Error(args ...any) + Errorf(format string, args ...any) + Fail() + FailNow() + Failed() bool + Fatal(args ...any) + Fatalf(format string, args ...any) + Helper() + Log(args ...any) + Logf(format string, args ...any) + Skip(args ...any) + SkipNow() + Skipf(format string, args ...any) +} + +var InternalTests = []struct { + Name string + Test func(testingT) +}{ + {"AbsDaysSplit", testAbsDaysSplit}, + {"AbsYdaySplit", testAbsYdaySplit}, + {"AbsDate", testAbsDate}, + {"DateToAbsDays", testDateToAbsDays}, + {"DaysIn", testDaysIn}, + {"DaysBefore", testDaysBefore}, +} + +func testAbsDaysSplit(t testingT) { + isLeap := func(year uint64) bool { + return year%4 == 0 && (year%100 != 0 || year%400 == 0) + } + bad := 0 + wantYear := uint64(0) + wantYday := absYday(0) + for days := range absDays(1e6) { + century, cyear, yday := days.split() + if century != absCentury(wantYear/100) || cyear != absCyear(wantYear%100) || yday != wantYday { + t.Errorf("absDays(%d).split() = %d, %d, %d, want %d, %d, %d", + days, century, cyear, yday, + wantYear/100, wantYear%100, wantYday) + if bad++; bad >= 20 { + t.Fatalf("too many errors") + } + } + end := absYday(365) + if isLeap(wantYear + 1) { + end = 366 + } + if wantYday++; wantYday == end { + wantYear++ + wantYday = 0 + } + } +} + +func testAbsYdaySplit(t testingT) { + ends := []int{31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 31, 29} + bad := 0 + wantMonth := absMonth(3) + wantDay := 1 + for yday := range absYday(366) { + month, day := yday.split() + if month != wantMonth || day != wantDay { + t.Errorf("absYday(%d).split() = %d, %d, want %d, %d", yday, month, day, wantMonth, wantDay) + if bad++; bad >= 20 { + t.Fatalf("too many errors") + } + } + if wantDay++; wantDay > ends[wantMonth-3] { + wantMonth++ + wantDay = 1 + } + } +} + +func testAbsDate(t testingT) { + ends := []int{31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} + isLeap := func(year int) bool { + y := uint64(year) + absoluteYears + return y%4 == 0 && (y%100 != 0 || y%400 == 0) + } + wantYear := 0 + wantMonth := March + wantMday := 1 + wantYday := 31 + 29 + 1 + bad := 0 + absoluteYears := int64(absoluteYears) + for days := range absDays(1e6) { + year, month, mday := days.date() + year += int(absoluteYears) + if year != wantYear || month != wantMonth || mday != wantMday { + t.Errorf("days(%d).date() = %v, %v, %v, want %v, %v, %v", days, + year, month, mday, + wantYear, wantMonth, wantMday) + if bad++; bad >= 20 { + t.Fatalf("too many errors") + } + } + + year, yday := days.yearYday() + year += int(absoluteYears) + if year != wantYear || yday != wantYday { + t.Errorf("days(%d).yearYday() = %v, %v, want %v, %v, ", days, + year, yday, + wantYear, wantYday) + if bad++; bad >= 20 { + t.Fatalf("too many errors") + } + } + + if wantMday++; wantMday == ends[wantMonth-1]+1 || wantMonth == February && wantMday == 29 && !isLeap(year) { + wantMonth++ + wantMday = 1 + } + wantYday++ + if wantMonth == December+1 { + wantYear++ + wantMonth = January + wantMday = 1 + wantYday = 1 + } + } +} + +func testDateToAbsDays(t testingT) { + isLeap := func(year int64) bool { + return year%4 == 0 && (year%100 != 0 || year%400 == 0) + } + wantDays := absDays(marchThruDecember) + bad := 0 + for year := int64(1); year < 10000; year++ { + days := dateToAbsDays(year-absoluteYears, January, 1) + if days != wantDays { + t.Errorf("dateToAbsDays(abs %d, Jan, 1) = %d, want %d", year, days, wantDays) + if bad++; bad >= 20 { + t.Fatalf("too many errors") + } + } + wantDays += 365 + if isLeap(year) { + wantDays++ + } + } +} + +func testDaysIn(t testingT) { + isLeap := func(year int) bool { + return year%4 == 0 && (year%100 != 0 || year%400 == 0) + } + want := []int{0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} + bad := 0 + for year := 0; year <= 1600; year++ { + for m := January; m <= December; m++ { + w := want[m] + if m == February && isLeap(year) { + w++ + } + d := daysIn(m, year-800) + if d != w { + t.Errorf("daysIn(%v, %d) = %d, want %d", m, year-800, d, w) + if bad++; bad >= 20 { + t.Fatalf("too many errors") + } + } + } + } +} + +func testDaysBefore(t testingT) { + for m, want := range []int{0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365} { + d := daysBefore(Month(m + 1)) + if d != want { + t.Errorf("daysBefore(%d) = %d, want %d", m, d, want) + } + } +} diff --git a/src/time/example_test.go b/src/time/example_test.go index 53c20a051672a8..05eac86738db94 100644 --- a/src/time/example_test.go +++ b/src/time/example_test.go @@ -6,6 +6,7 @@ package time_test import ( "fmt" + "math" "time" ) @@ -108,6 +109,39 @@ func ExampleParseDuration() { // There are 1.00e-06 seconds in 1µs. } +func ExampleSince() { + start := time.Now() + expensiveCall() + elapsed := time.Since(start) + fmt.Printf("The call took %v to run.\n", elapsed) +} + +func ExampleUntil() { + futureTime := time.Now().Add(5 * time.Second) + durationUntil := time.Until(futureTime) + fmt.Printf("Duration until future time: %.0f seconds", math.Ceil(durationUntil.Seconds())) + // Output: Duration until future time: 5 seconds +} + +func ExampleDuration_Abs() { + positiveDuration := 5 * time.Second + negativeDuration := -3 * time.Second + minInt64CaseDuration := time.Duration(math.MinInt64) + + absPositive := positiveDuration.Abs() + absNegative := negativeDuration.Abs() + absSpecial := minInt64CaseDuration.Abs() == time.Duration(math.MaxInt64) + + fmt.Printf("Absolute value of positive duration: %v\n", absPositive) + fmt.Printf("Absolute value of negative duration: %v\n", absNegative) + fmt.Printf("Absolute value of MinInt64 equal to MaxInt64: %t\n", absSpecial) + + // Output: + // Absolute value of positive duration: 5s + // Absolute value of negative duration: 3s + // Absolute value of MinInt64 equal to MaxInt64: true +} + func ExampleDuration_Hours() { h, _ := time.ParseDuration("4h30m") fmt.Printf("I've got %.1f hours of work left.", h.Hours()) @@ -295,8 +329,8 @@ func ExampleTime_Format() { // default format: 2015-02-25 11:06:39 -0800 PST // Unix format: Wed Feb 25 11:06:39 PST 2015 // Same, in UTC: Wed Feb 25 19:06:39 UTC 2015 - //in Shanghai with seconds: 2015-02-26T03:06:39 +080000 - //in Shanghai with colon seconds: 2015-02-26T03:06:39 +08:00:00 + // in Shanghai with seconds: 2015-02-26T03:06:39 +080000 + // in Shanghai with colon seconds: 2015-02-26T03:06:39 +08:00:00 // // Formats: // @@ -701,8 +735,8 @@ func ExampleTime_String() { timeWithoutNanoseconds := time.Date(2000, 2, 1, 12, 13, 14, 0, time.UTC) withoutNanoseconds := timeWithoutNanoseconds.String() - fmt.Printf("withNanoseconds = %v\n", string(withNanoseconds)) - fmt.Printf("withoutNanoseconds = %v\n", string(withoutNanoseconds)) + fmt.Printf("withNanoseconds = %v\n", withNanoseconds) + fmt.Printf("withoutNanoseconds = %v\n", withoutNanoseconds) // Output: // withNanoseconds = 2000-02-01 12:13:14.000000015 +0000 UTC @@ -720,6 +754,31 @@ func ExampleTime_Sub() { // difference = 12h0m0s } +func ExampleTime_AppendBinary() { + t := time.Date(2025, 4, 1, 15, 30, 45, 123456789, time.UTC) + + var buffer []byte + buffer, err := t.AppendBinary(buffer) + if err != nil { + panic(err) + } + + var parseTime time.Time + err = parseTime.UnmarshalBinary(buffer[:]) + if err != nil { + panic(err) + } + + fmt.Printf("t: %v\n", t) + fmt.Printf("parseTime: %v\n", parseTime) + fmt.Printf("equal: %v\n", parseTime.Equal(t)) + + // Output: + // t: 2025-04-01 15:30:45.123456789 +0000 UTC + // parseTime: 2025-04-01 15:30:45.123456789 +0000 UTC + // equal: true +} + func ExampleTime_AppendFormat() { t := time.Date(2017, time.November, 4, 11, 0, 0, 0, time.UTC) text := []byte("Time: ") @@ -731,6 +790,21 @@ func ExampleTime_AppendFormat() { // Time: 11:00AM } +func ExampleTime_AppendText() { + t := time.Date(2025, 4, 1, 15, 30, 45, 123456789, time.UTC) + + buffer := []byte("t: ") + + buffer, err := t.AppendText(buffer) + if err != nil { + panic(err) + } + + fmt.Printf("%s\n", buffer) + + // Output: + // t: 2025-04-01T15:30:45.123456789Z +} func ExampleFixedZone() { loc := time.FixedZone("UTC-8", -8*60*60) t := time.Date(2009, time.November, 10, 23, 0, 0, 0, loc) diff --git a/src/time/format.go b/src/time/format.go index c9e68b3eb25417..ad5486f4d28f89 100644 --- a/src/time/format.go +++ b/src/time/format.go @@ -140,7 +140,7 @@ const ( stdDay // "2" stdUnderDay // "_2" stdZeroDay // "02" - stdUnderYearDay // "__2" + stdUnderYearDay = iota + stdNeedYday // "__2" stdZeroYearDay // "002" stdHour = iota + stdNeedClock // "15" stdHour12 // "3" @@ -168,7 +168,8 @@ const ( stdFracSecond9 // ".9", ".99", ..., trailing zeros omitted stdNeedDate = 1 << 8 // need month, day, year - stdNeedClock = 2 << 8 // need hour, minute, second + stdNeedYday = 1 << 9 // need yday + stdNeedClock = 1 << 10 // need hour, minute, second stdArgShift = 16 // extra argument in high bits, above low stdArgShift stdSeparatorShift = 28 // extra argument in high 4 bits for fractional second separators stdMask = 1<= i+2 && layout[i+1] == '2' { - //_2006 is really a literal _, followed by stdLongYear + // _2006 is really a literal _, followed by stdLongYear if len(layout) >= i+5 && layout[i+1:i+5] == "2006" { return layout[0 : i+1], stdLongYear, layout[i+5:] } @@ -404,7 +405,7 @@ func match(s1, s2 string) bool { func lookup(tab []string, val string) (int, string, error) { for i, v := range tab { - if len(val) >= len(v) && match(val[0:len(v)], v) { + if len(val) >= len(v) && match(val[:len(v)], v) { return i, val[len(v):], nil } } @@ -574,9 +575,9 @@ func (t Time) String() string { // GoString implements [fmt.GoStringer] and formats t to be printed in Go source // code. func (t Time) GoString() string { - abs := t.abs() - year, month, day, _ := absDate(abs, true) - hour, minute, second := absClock(abs) + abs := t.absSec() + year, month, day := abs.days().date() + hour, minute, second := abs.clock() buf := make([]byte, 0, len("time.Date(9999, time.September, 31, 23, 59, 59, 999999999, time.Local)")) buf = append(buf, "time.Date("...) @@ -664,13 +665,14 @@ func (t Time) AppendFormat(b []byte, layout string) []byte { } func (t Time) appendFormat(b []byte, layout string) []byte { - var ( - name, offset, abs = t.locabs() + name, offset, abs := t.locabs() + days := abs.days() + var ( year int = -1 month Month day int - yday int + yday int = -1 hour int = -1 min int sec int @@ -689,13 +691,15 @@ func (t Time) appendFormat(b []byte, layout string) []byte { // Compute year, month, day if needed. if year < 0 && std&stdNeedDate != 0 { - year, month, day, yday = absDate(abs, true) - yday++ + year, month, day = days.date() + } + if yday < 0 && std&stdNeedYday != 0 { + _, yday = days.yearYday() } // Compute hour, minute, second if needed. if hour < 0 && std&stdNeedClock != 0 { - hour, min, sec = absClock(abs) + hour, min, sec = abs.clock() } switch std & stdMask { @@ -717,9 +721,9 @@ func (t Time) appendFormat(b []byte, layout string) []byte { case stdZeroMonth: b = appendInt(b, int(month), 2) case stdWeekDay: - b = append(b, absWeekday(abs).String()[:3]...) + b = append(b, days.weekday().String()[:3]...) case stdLongWeekDay: - s := absWeekday(abs).String() + s := days.weekday().String() b = append(b, s...) case stdDay: b = appendInt(b, day, 0) @@ -887,7 +891,7 @@ func quote(s string) string { if c == '"' || c == '\\' { buf = append(buf, '\\') } - buf = append(buf, string(c)...) + buf = append(buf, byte(c)) } } buf = append(buf, '"') @@ -1255,9 +1259,9 @@ func parse(layout, value string, defaultLocation, local *Location) (Time, error) hr, _, err = getnum(hour, true) if err == nil { mm, _, err = getnum(min, true) - } - if err == nil { - ss, _, err = getnum(seconds, true) + if err == nil { + ss, _, err = getnum(seconds, true) + } } // The range test use > rather than >=, @@ -1350,10 +1354,10 @@ func parse(layout, value string, defaultLocation, local *Location) (Time, error) } if m == 0 { m = (yday-1)/31 + 1 - if int(daysBefore[m]) < yday { + if daysBefore(Month(m+1)) < yday { m++ } - d = yday - int(daysBefore[m-1]) + d = yday - daysBefore(Month(m)) } // If month, day already seen, yday's m, d must match. // Otherwise, set them from m, d. @@ -1598,6 +1602,16 @@ func leadingFraction(s string) (x uint64, scale float64, rem string) { return x, scale, s[i:] } +// parseDurationError describes a problem parsing a duration string. +type parseDurationError struct { + message string + value string +} + +func (e *parseDurationError) Error() string { + return "time: " + e.message + " " + quote(e.value) +} + var unitMap = map[string]uint64{ "ns": uint64(Nanosecond), "us": uint64(Microsecond), @@ -1633,7 +1647,7 @@ func ParseDuration(s string) (Duration, error) { return 0, nil } if s == "" { - return 0, errors.New("time: invalid duration " + quote(orig)) + return 0, &parseDurationError{"invalid duration", orig} } for s != "" { var ( @@ -1645,13 +1659,13 @@ func ParseDuration(s string) (Duration, error) { // The next character must be [0-9.] if !(s[0] == '.' || '0' <= s[0] && s[0] <= '9') { - return 0, errors.New("time: invalid duration " + quote(orig)) + return 0, &parseDurationError{"invalid duration", orig} } // Consume [0-9]* pl := len(s) v, s, err = leadingInt(s) if err != nil { - return 0, errors.New("time: invalid duration " + quote(orig)) + return 0, &parseDurationError{"invalid duration", orig} } pre := pl != len(s) // whether we consumed anything before a period @@ -1665,7 +1679,7 @@ func ParseDuration(s string) (Duration, error) { } if !pre && !post { // no digits (e.g. ".s" or "-.s") - return 0, errors.New("time: invalid duration " + quote(orig)) + return 0, &parseDurationError{"invalid duration", orig} } // Consume unit. @@ -1677,17 +1691,17 @@ func ParseDuration(s string) (Duration, error) { } } if i == 0 { - return 0, errors.New("time: missing unit in duration " + quote(orig)) + return 0, &parseDurationError{"missing unit in duration", orig} } u := s[:i] s = s[i:] unit, ok := unitMap[u] if !ok { - return 0, errors.New("time: unknown unit " + quote(u) + " in duration " + quote(orig)) + return 0, &parseDurationError{"unknown unit " + quote(u) + " in duration", orig} } if v > 1<<63/unit { // overflow - return 0, errors.New("time: invalid duration " + quote(orig)) + return 0, &parseDurationError{"invalid duration", orig} } v *= unit if f > 0 { @@ -1696,19 +1710,19 @@ func ParseDuration(s string) (Duration, error) { v += uint64(float64(f) * (float64(unit) / scale)) if v > 1<<63 { // overflow - return 0, errors.New("time: invalid duration " + quote(orig)) + return 0, &parseDurationError{"invalid duration", orig} } } d += v if d > 1<<63 { - return 0, errors.New("time: invalid duration " + quote(orig)) + return 0, &parseDurationError{"invalid duration", orig} } } if neg { return -Duration(d), nil } if d > 1<<63-1 { - return 0, errors.New("time: invalid duration " + quote(orig)) + return 0, &parseDurationError{"invalid duration", orig} } return Duration(d), nil } diff --git a/src/time/format_rfc3339.go b/src/time/format_rfc3339.go index 1151666c3e42f3..05fddfca89f64c 100644 --- a/src/time/format_rfc3339.go +++ b/src/time/format_rfc3339.go @@ -19,7 +19,7 @@ func (t Time) appendFormatRFC3339(b []byte, nanos bool) []byte { _, offset, abs := t.locabs() // Format date. - year, month, day, _ := absDate(abs, true) + year, month, day := abs.days().date() b = appendInt(b, year, 4) b = append(b, '-') b = appendInt(b, int(month), 2) @@ -29,7 +29,7 @@ func (t Time) appendFormatRFC3339(b []byte, nanos bool) []byte { b = append(b, 'T') // Format time. - hour, min, sec := absClock(abs) + hour, min, sec := abs.clock() b = appendInt(b, hour, 2) b = append(b, ':') b = appendInt(b, min, 2) diff --git a/src/time/linkname_test.go b/src/time/linkname_test.go new file mode 100644 index 00000000000000..3a47eaf33b0174 --- /dev/null +++ b/src/time/linkname_test.go @@ -0,0 +1,41 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package time_test + +import ( + "testing" + "time" + _ "unsafe" // for linkname +) + +//go:linkname timeAbs time.Time.abs +func timeAbs(time.Time) uint64 + +//go:linkname absClock time.absClock +func absClock(uint64) (hour, min, sec int) + +//go:linkname absDate time.absDate +func absDate(uint64, bool) (year int, month time.Month, day int, yday int) + +func TestLinkname(t *testing.T) { + tm := time.Date(2006, time.January, 2, 15, 4, 5, 6, time.UTC) + abs := timeAbs(tm) + // wantAbs should be Jan 1 based, not Mar 1 based. + // See absolute time description in time.go. + const wantAbs = 9223372029851535845 // NOT 9223372029877973939 + if abs != wantAbs { + t.Fatalf("timeAbs(2006-01-02 15:04:05 UTC) = %d, want %d", abs, uint64(wantAbs)) + } + + year, month, day, yday := absDate(abs, true) + if year != 2006 || month != time.January || day != 2 || yday != 1 { + t.Errorf("absDate() = %v, %v, %v, %v, want 2006, January, 2, 1", year, month, day, yday) + } + + hour, min, sec := absClock(abs) + if hour != 15 || min != 4 || sec != 5 { + t.Errorf("absClock() = %v, %v, %v, 15, 4, 5", hour, min, sec) + } +} diff --git a/src/time/sleep.go b/src/time/sleep.go index 7e2fa0c20af46e..4b7750eb9414e8 100644 --- a/src/time/sleep.go +++ b/src/time/sleep.go @@ -142,7 +142,7 @@ func (t *Timer) Stop() bool { // in Go 1.27 or later. func NewTimer(d Duration) *Timer { c := make(chan Time, 1) - t := (*Timer)(newTimer(when(d), 0, sendTime, c, syncTimer(c))) + t := newTimer(when(d), 0, sendTime, c, syncTimer(c)) t.C = c return t } @@ -165,8 +165,8 @@ func NewTimer(d Duration) *Timer { // to receive a time value corresponding to the previous timer settings; // if the program has not received from t.C already and the timer is // running, Reset is guaranteed to return true. -// Before Go 1.23, the only safe way to use Reset was to [Stop] and -// explicitly drain the timer first. +// Before Go 1.23, the only safe way to use Reset was to call [Timer.Stop] +// and explicitly drain the timer first. // See the [NewTimer] documentation for more details. func (t *Timer) Reset(d Duration) bool { if !t.initTimer { @@ -180,7 +180,7 @@ func (t *Timer) Reset(d Duration) bool { func sendTime(c any, seq uintptr, delta int64) { // delta is how long ago the channel send was supposed to happen. // The current time can be arbitrarily far into the future, because the runtime - // can delay a sendTime call until a goroutines tries to receive from + // can delay a sendTime call until a goroutine tries to receive from // the channel. Subtract delta to go back to the old time that we // used to send. select { @@ -208,7 +208,7 @@ func After(d Duration) <-chan Time { // be used to cancel the call using its Stop method. // The returned Timer's C field is not used and will be nil. func AfterFunc(d Duration, f func()) *Timer { - return (*Timer)(newTimer(when(d), 0, goFunc, f, nil)) + return newTimer(when(d), 0, goFunc, f, nil) } func goFunc(arg any, seq uintptr, delta int64) { diff --git a/src/time/sleep_test.go b/src/time/sleep_test.go index 29f56ef7520baa..c87f420f8f360e 100644 --- a/src/time/sleep_test.go +++ b/src/time/sleep_test.go @@ -785,6 +785,119 @@ func TestAdjustTimers(t *testing.T) { } } +func TestStopResult(t *testing.T) { + testStopResetResult(t, true) +} + +func TestResetResult(t *testing.T) { + testStopResetResult(t, false) +} + +// Test that when racing between running a timer and stopping a timer Stop +// consistently indicates whether a value can be read from the channel. +// Issue #69312. +func testStopResetResult(t *testing.T, testStop bool) { + for _, name := range []string{"0", "1", "2"} { + t.Run("asynctimerchan="+name, func(t *testing.T) { + testStopResetResultGODEBUG(t, testStop, name) + }) + } +} + +func testStopResetResultGODEBUG(t *testing.T, testStop bool, godebug string) { + t.Setenv("GODEBUG", "asynctimerchan="+godebug) + + stopOrReset := func(timer *Timer) bool { + if testStop { + return timer.Stop() + } else { + return timer.Reset(1 * Hour) + } + } + + start := make(chan struct{}) + var wg sync.WaitGroup + const N = 1000 + wg.Add(N) + for range N { + go func() { + defer wg.Done() + <-start + for j := 0; j < 100; j++ { + timer1 := NewTimer(1 * Millisecond) + timer2 := NewTimer(1 * Millisecond) + select { + case <-timer1.C: + if !stopOrReset(timer2) { + // The test fails if this + // channel read times out. + <-timer2.C + } + case <-timer2.C: + if !stopOrReset(timer1) { + // The test fails if this + // channel read times out. + <-timer1.C + } + } + } + }() + } + close(start) + wg.Wait() +} + +// Test having a large number of goroutines wake up a ticker simultaneously. +// This used to trigger a crash when run under x/tools/cmd/stress. +func TestMultiWakeupTicker(t *testing.T) { + if testing.Short() { + t.Skip("-short") + } + + goroutines := runtime.GOMAXPROCS(0) + timer := NewTicker(Microsecond) + var wg sync.WaitGroup + wg.Add(goroutines) + for range goroutines { + go func() { + defer wg.Done() + for range 100000 { + select { + case <-timer.C: + case <-After(Millisecond): + } + } + }() + } + wg.Wait() +} + +// Test having a large number of goroutines wake up a timer simultaneously. +// This used to trigger a crash when run under x/tools/cmd/stress. +func TestMultiWakeupTimer(t *testing.T) { + if testing.Short() { + t.Skip("-short") + } + + goroutines := runtime.GOMAXPROCS(0) + timer := NewTimer(Nanosecond) + var wg sync.WaitGroup + wg.Add(goroutines) + for range goroutines { + go func() { + defer wg.Done() + for range 10000 { + select { + case <-timer.C: + default: + } + timer.Reset(Nanosecond) + } + }() + } + wg.Wait() +} + // Benchmark timer latency when the thread that creates the timer is busy with // other work and the timers must be serviced by other threads. // https://golang.org/issue/38860 @@ -824,7 +937,6 @@ func BenchmarkParallelTimerLatency(b *testing.B) { wg.Add(timerCount) atomic.StoreInt32(&count, 0) for j := 0; j < timerCount; j++ { - j := j expectedWakeup := Now().Add(delay) AfterFunc(delay, func() { late := Since(expectedWakeup) @@ -855,17 +967,15 @@ func BenchmarkParallelTimerLatency(b *testing.B) { } var total float64 var samples float64 - max := Duration(0) + maximum := Duration(0) for _, s := range stats { - if s.max > max { - max = s.max - } + maximum = max(maximum, s.max) total += s.sum samples += float64(s.count) } b.ReportMetric(0, "ns/op") b.ReportMetric(total/samples, "avg-late-ns") - b.ReportMetric(float64(max.Nanoseconds()), "max-late-ns") + b.ReportMetric(float64(maximum.Nanoseconds()), "max-late-ns") } // Benchmark timer latency with staggered wakeup times and varying CPU bound @@ -900,7 +1010,6 @@ func BenchmarkStaggeredTickerLatency(b *testing.B) { var wg sync.WaitGroup wg.Add(tickerCount) for j := 0; j < tickerCount; j++ { - j := j doWork(delay / Duration(gmp)) expectedWakeup := Now().Add(delay) ticker := NewTicker(delay) diff --git a/src/time/tick_test.go b/src/time/tick_test.go index fce9002cfcbb63..dcbbcdb14509a7 100644 --- a/src/time/tick_test.go +++ b/src/time/tick_test.go @@ -151,9 +151,13 @@ func TestTickerResetLtZeroDuration(t *testing.T) { } func TestLongAdjustTimers(t *testing.T) { - if runtime.GOOS == "android" || runtime.GOOS == "ios" { + if runtime.GOOS == "android" || runtime.GOOS == "ios" || runtime.GOOS == "plan9" { t.Skipf("skipping on %s - too slow", runtime.GOOS) } + if testing.Short() && runtime.NumCPU() < 2 { + t.Skipf("skipping in short mode, insufficient CPUs") + } + t.Parallel() var wg sync.WaitGroup defer wg.Wait() @@ -462,7 +466,7 @@ func testTimerChan(t *testing.T, tim timer, C <-chan Time, synctimerchan bool) { tim.Reset(1) Sleep(sched) if l, c := len(C), cap(C); l != 0 || c != 0 { - //t.Fatalf("len(C), cap(C) = %d, %d, want 0, 0", l, c) + // t.Fatalf("len(C), cap(C) = %d, %d, want 0, 0", l, c) } assertTick() } else { diff --git a/src/time/time.go b/src/time/time.go index 43efe4b11d0d1c..cf9abc7196f9cf 100644 --- a/src/time/time.go +++ b/src/time/time.go @@ -54,8 +54,8 @@ // On some systems the monotonic clock will stop if the computer goes to sleep. // On such a system, t.Sub(u) may not accurately reflect the actual // time that passed between t and u. The same applies to other functions and -// methods that subtract times, such as [Since], [Until], [Before], [After], -// [Add], [Sub], [Equal] and [Compare]. In some cases, you may need to strip +// methods that subtract times, such as [Since], [Until], [Time.Before], [Time.After], +// [Time.Add], [Time.Equal] and [Time.Compare]. In some cases, you may need to strip // the monotonic clock to get accurate results. // // Because the monotonic clock reading has no meaning outside @@ -92,6 +92,7 @@ package time import ( "errors" + "math/bits" _ "unsafe" // for go:linkname ) @@ -118,9 +119,9 @@ import ( // these methods does not change the actual instant it represents, only the time // zone in which to interpret it. // -// Representations of a Time value saved by the [Time.GobEncode], [Time.MarshalBinary], -// [Time.MarshalJSON], and [Time.MarshalText] methods store the [Time.Location]'s offset, but not -// the location name. They therefore lose information about Daylight Saving Time. +// Representations of a Time value saved by the [Time.GobEncode], [Time.MarshalBinary], [Time.AppendBinary], +// [Time.MarshalJSON], [Time.MarshalText] and [Time.AppendText] methods store the [Time.Location]'s offset, +// but not the location name. They therefore lose information about Daylight Saving Time. // // In addition to the required “wall clock” reading, a Time may contain an optional // reading of the current process's monotonic clock, to provide additional precision @@ -256,6 +257,12 @@ func (t *Time) mono() int64 { return t.ext } +// IsZero reports whether t represents the zero time instant, +// January 1, year 1, 00:00:00 UTC. +func (t Time) IsZero() bool { + return t.sec() == 0 && t.nsec() == 0 +} + // After reports whether the time instant t is after u. func (t Time) After(u Time) bool { if t.wall&u.wall&hasMonotonic != 0 { @@ -360,7 +367,7 @@ func (d Weekday) String() string { return "%!Weekday(" + string(buf[n:]) + ")" } -// Computations on time. +// Computations on Times // // The zero value for a Time is defined to be // January 1, year 1, 00:00:00.000000000 UTC @@ -406,22 +413,22 @@ func (d Weekday) String() string { // // The calendar runs on an exact 400 year cycle: a 400-year calendar // printed for 1970-2369 will apply as well to 2370-2769. Even the days -// of the week match up. It simplifies the computations to choose the +// of the week match up. It simplifies date computations to choose the // cycle boundaries so that the exceptional years are always delayed as -// long as possible. That means choosing a year equal to 1 mod 400, so -// that the first leap year is the 4th year, the first missed leap year -// is the 100th year, and the missed missed leap year is the 400th year. -// So we'd prefer instead to print a calendar for 2001-2400 and reuse it -// for 2401-2800. +// long as possible: March 1, year 0 is such a day: +// the first leap day (Feb 29) is four years minus one day away, +// the first multiple-of-4 year without a Feb 29 is 100 years minus one day away, +// and the first multiple-of-100 year with a Feb 29 is 400 years minus one day away. +// March 1 year Y for any Y = 0 mod 400 is also such a day. // // Finally, it's convenient if the delta between the Unix epoch and // long-ago epoch is representable by an int64 constant. // // These three considerations—choose an epoch as early as possible, that -// uses a year equal to 1 mod 400, and that is no more than 2⁶³ seconds -// earlier than 1970—bring us to the year -292277022399. We refer to -// this year as the absolute zero year, and to times measured as a uint64 -// seconds since this year as absolute times. +// starts on March 1 of a year equal to 0 mod 400, and that is no more than +// 2⁶³ seconds earlier than 1970—bring us to the year -292277022400. +// We refer to this moment as the absolute zero instant, and to times +// measured as a uint64 seconds since this year as absolute times. // // Times measured as an int64 seconds since the year 1—the representation // used for Time's sec field—are called internal times. @@ -435,40 +442,327 @@ func (d Weekday) String() string { // west of UTC, since it is year 0. It doesn't seem tenable to say that // printing the zero time correctly isn't supported in half the time // zones. By comparison, it's reasonable to mishandle some times in -// the year -292277022399. +// the year -292277022400. // // All this is opaque to clients of the API and can be changed if a // better implementation presents itself. +// +// The date calculations are implemented using the following clever math from +// Cassio Neri and Lorenz Schneider, “Euclidean affine functions and their +// application to calendar algorithms,” SP&E 2023. https://doi.org/10.1002/spe.3172 +// +// Define a “calendrical division” (f, f°, f*) to be a triple of functions converting +// one time unit into a whole number of larger units and the remainder and back. +// For example, in a calendar with no leap years, (d/365, d%365, y*365) is the +// calendrical division for days into years: +// +// (f) year := days/365 +// (f°) yday := days%365 +// (f*) days := year*365 (+ yday) +// +// Note that f* is usually the “easy” function to write: it's the +// calendrical multiplication that inverts the more complex division. +// +// Neri and Schneider prove that when f* takes the form +// +// f*(n) = (a n + b) / c +// +// using integer division rounding down with a ≥ c > 0, +// which they call a Euclidean affine function or EAF, then: +// +// f(n) = (c n + c - b - 1) / a +// f°(n) = (c n + c - b - 1) % a / c +// +// This gives a fairly direct calculation for any calendrical division for which +// we can write the calendrical multiplication in EAF form. +// Because the epoch has been shifted to March 1, all the calendrical +// multiplications turn out to be possible to write in EAF form. +// When a date is broken into [century, cyear, amonth, mday], +// with century, cyear, and mday 0-based, +// and amonth 3-based (March = 3, ..., January = 13, February = 14), +// the calendrical multiplications written in EAF form are: +// +// yday = (153 (amonth-3) + 2) / 5 = (153 amonth - 457) / 5 +// cday = 365 cyear + cyear/4 = 1461 cyear / 4 +// centurydays = 36524 century + century/4 = 146097 century / 4 +// days = centurydays + cday + yday + mday. +// +// We can only handle one periodic cycle per equation, so the year +// calculation must be split into [century, cyear], handling both the +// 100-year cycle and the 400-year cycle. +// +// The yday calculation is not obvious but derives from the fact +// that the March through January calendar repeats the 5-month +// 153-day cycle 31, 30, 31, 30, 31 (we don't care about February +// because yday only ever count the days _before_ February 1, +// since February is the last month). +// +// Using the rule for deriving f and f° from f*, these multiplications +// convert to these divisions: +// +// century := (4 days + 3) / 146097 +// cdays := (4 days + 3) % 146097 / 4 +// cyear := (4 cdays + 3) / 1461 +// ayday := (4 cdays + 3) % 1461 / 4 +// amonth := (5 ayday + 461) / 153 +// mday := (5 ayday + 461) % 153 / 5 +// +// The a in ayday and amonth stands for absolute (March 1-based) +// to distinguish from the standard yday (January 1-based). +// +// After computing these, we can translate from the March 1 calendar +// to the standard January 1 calendar with branch-free math assuming a +// branch-free conversion from bool to int 0 or 1, denoted int(b) here: +// +// isJanFeb := int(yday >= marchThruDecember) +// month := amonth - isJanFeb*12 +// year := century*100 + cyear + isJanFeb +// isLeap := int(cyear%4 == 0) & (int(cyear != 0) | int(century%4 == 0)) +// day := 1 + mday +// yday := 1 + ayday + 31 + 28 + isLeap&^isJanFeb - 365*isJanFeb +// +// isLeap is the standard leap-year rule, but the split year form +// makes the divisions all reduce to binary masking. +// Note that day and yday are 1-based, in contrast to mday and ayday. +// To keep the various units separate, we define integer types +// for each. These are never stored in interfaces nor allocated, +// so their type information does not appear in Go binaries. const ( - // The unsigned zero year for internal calculations. - // Must be 1 mod 400, and times before it will not compute correctly, - // but otherwise can be changed at will. - absoluteZeroYear = -292277022399 + secondsPerMinute = 60 + secondsPerHour = 60 * secondsPerMinute + secondsPerDay = 24 * secondsPerHour + secondsPerWeek = 7 * secondsPerDay + daysPer400Years = 365*400 + 97 + + // Days from March 1 through end of year + marchThruDecember = 31 + 30 + 31 + 30 + 31 + 31 + 30 + 31 + 30 + 31 + + // absoluteYears is the number of years we subtract from internal time to get absolute time. + // This value must be 0 mod 400, and it defines the “absolute zero instant” + // mentioned in the “Computations on Times” comment above: March 1, -absoluteYears. + // Dates before the absolute epoch will not compute correctly, + // but otherwise the value can be changed as needed. + absoluteYears = 292277022400 // The year of the zero Time. // Assumed by the unixToInternal computation below. internalYear = 1 // Offsets to convert between internal and absolute or Unix times. - absoluteToInternal int64 = (absoluteZeroYear - internalYear) * 365.2425 * secondsPerDay + absoluteToInternal int64 = -(absoluteYears*365.2425 + marchThruDecember) * secondsPerDay internalToAbsolute = -absoluteToInternal unixToInternal int64 = (1969*365 + 1969/4 - 1969/100 + 1969/400) * secondsPerDay internalToUnix int64 = -unixToInternal + absoluteToUnix = absoluteToInternal + internalToUnix + unixToAbsolute = unixToInternal + internalToAbsolute + wallToInternal int64 = (1884*365 + 1884/4 - 1884/100 + 1884/400) * secondsPerDay ) -// IsZero reports whether t represents the zero time instant, -// January 1, year 1, 00:00:00 UTC. -func (t Time) IsZero() bool { - return t.sec() == 0 && t.nsec() == 0 +// An absSeconds counts the number of seconds since the absolute zero instant. +type absSeconds uint64 + +// An absDays counts the number of days since the absolute zero instant. +type absDays uint64 + +// An absCentury counts the number of centuries since the absolute zero instant. +type absCentury uint64 + +// An absCyear counts the number of years since the start of a century. +type absCyear int + +// An absYday counts the number of days since the start of a year. +// Note that absolute years start on March 1. +type absYday int + +// An absMonth counts the number of months since the start of a year. +// absMonth=0 denotes March. +type absMonth int + +// An absLeap is a single bit (0 or 1) denoting whether a given year is a leap year. +type absLeap int + +// An absJanFeb is a single bit (0 or 1) denoting whether a given day falls in January or February. +// That is a special case because the absolute years start in March (unlike normal calendar years). +type absJanFeb int + +// dateToAbsDays takes a standard year/month/day and returns the +// number of days from the absolute epoch to that day. +// The days argument can be out of range and in particular can be negative. +func dateToAbsDays(year int64, month Month, day int) absDays { + // See “Computations on Times” comment above. + amonth := uint32(month) + janFeb := uint32(0) + if amonth < 3 { + janFeb = 1 + } + amonth += 12 * janFeb + y := uint64(year) - uint64(janFeb) + absoluteYears + + // For amonth is in the range [3,14], we want: + // + // ayday := (153*amonth - 457) / 5 + // + // (See the “Computations on Times” comment above + // as well as Neri and Schneider, section 7.) + // + // That is equivalent to: + // + // ayday := (979*amonth - 2919) >> 5 + // + // and the latter form uses a couple fewer instructions, + // so use it, saving a few cycles. + // See Neri and Schneider, section 8.3 + // for more about this optimization. + // + // (Note that there is no saved division, because the compiler + // implements / 5 without division in all cases.) + ayday := (979*amonth - 2919) >> 5 + + century := y / 100 + cyear := uint32(y % 100) + cday := 1461 * cyear / 4 + centurydays := 146097 * century / 4 + + return absDays(centurydays + uint64(int64(cday+ayday)+int64(day)-1)) +} + +// days converts absolute seconds to absolute days. +func (abs absSeconds) days() absDays { + return absDays(abs / secondsPerDay) } -// abs returns the time t as an absolute time, adjusted by the zone offset. +// split splits days into century, cyear, ayday. +func (days absDays) split() (century absCentury, cyear absCyear, ayday absYday) { + // See “Computations on Times” comment above. + d := 4*uint64(days) + 3 + century = absCentury(d / 146097) + + // This should be + // cday := uint32(d % 146097) / 4 + // cd := 4*cday + 3 + // which is to say + // cday := uint32(d % 146097) >> 2 + // cd := cday<<2 + 3 + // but of course (x>>2<<2)+3 == x|3, + // so do that instead. + cd := uint32(d%146097) | 3 + + // For cdays in the range [0,146097] (100 years), we want: + // + // cyear := (4 cdays + 3) / 1461 + // yday := (4 cdays + 3) % 1461 / 4 + // + // (See the “Computations on Times” comment above + // as well as Neri and Schneider, section 7.) + // + // That is equivalent to: + // + // cyear := (2939745 cdays) >> 32 + // yday := (2939745 cdays) & 0xFFFFFFFF / 2939745 / 4 + // + // so do that instead, saving a few cycles. + // See Neri and Schneider, section 8.3 + // for more about this optimization. + hi, lo := bits.Mul32(2939745, cd) + cyear = absCyear(hi) + ayday = absYday(lo / 2939745 / 4) + return +} + +// split splits ayday into absolute month and standard (1-based) day-in-month. +func (ayday absYday) split() (m absMonth, mday int) { + // See “Computations on Times” comment above. + // + // For yday in the range [0,366], + // + // amonth := (5 yday + 461) / 153 + // mday := (5 yday + 461) % 153 / 5 + // + // is equivalent to: + // + // amonth = (2141 yday + 197913) >> 16 + // mday = (2141 yday + 197913) & 0xFFFF / 2141 + // + // so do that instead, saving a few cycles. + // See Neri and Schneider, section 8.3. + d := 2141*uint32(ayday) + 197913 + return absMonth(d >> 16), 1 + int((d&0xFFFF)/2141) +} + +// janFeb returns 1 if the March 1-based ayday is in January or February, 0 otherwise. +func (ayday absYday) janFeb() absJanFeb { + // See “Computations on Times” comment above. + jf := absJanFeb(0) + if ayday >= marchThruDecember { + jf = 1 + } + return jf +} + +// month returns the standard Month for (m, janFeb) +func (m absMonth) month(janFeb absJanFeb) Month { + // See “Computations on Times” comment above. + return Month(m) - Month(janFeb)*12 +} + +// leap returns 1 if (century, cyear) is a leap year, 0 otherwise. +func (century absCentury) leap(cyear absCyear) absLeap { + // See “Computations on Times” comment above. + y4ok := 0 + if cyear%4 == 0 { + y4ok = 1 + } + y100ok := 0 + if cyear != 0 { + y100ok = 1 + } + y400ok := 0 + if century%4 == 0 { + y400ok = 1 + } + return absLeap(y4ok & (y100ok | y400ok)) +} + +// year returns the standard year for (century, cyear, janFeb). +func (century absCentury) year(cyear absCyear, janFeb absJanFeb) int { + // See “Computations on Times” comment above. + return int(uint64(century)*100-absoluteYears) + int(cyear) + int(janFeb) +} + +// yday returns the standard 1-based yday for (ayday, janFeb, leap). +func (ayday absYday) yday(janFeb absJanFeb, leap absLeap) int { + // See “Computations on Times” comment above. + return int(ayday) + (1 + 31 + 28) + int(leap)&^int(janFeb) - 365*int(janFeb) +} + +// date converts days into standard year, month, day. +func (days absDays) date() (year int, month Month, day int) { + century, cyear, ayday := days.split() + amonth, day := ayday.split() + janFeb := ayday.janFeb() + year = century.year(cyear, janFeb) + month = amonth.month(janFeb) + return +} + +// yearYday converts days into the standard year and 1-based yday. +func (days absDays) yearYday() (year, yday int) { + century, cyear, ayday := days.split() + janFeb := ayday.janFeb() + year = century.year(cyear, janFeb) + yday = ayday.yday(janFeb, century.leap(cyear)) + return +} + +// absSec returns the time t as an absolute seconds, adjusted by the zone offset. // It is called when computing a presentation property like Month or Hour. -func (t Time) abs() uint64 { +// We'd rather call it abs, but there are linknames to abs that make that problematic. +// See timeAbs below. +func (t Time) absSec() absSeconds { l := t.loc // Avoid function calls when possible. if l == nil || l == &localLoc { @@ -483,12 +777,12 @@ func (t Time) abs() uint64 { sec += int64(offset) } } - return uint64(sec + (unixToInternal + internalToAbsolute)) + return absSeconds(sec + (unixToInternal + internalToAbsolute)) } // locabs is a combination of the Zone and abs methods, // extracting both return values from a single zone lookup. -func (t Time) locabs() (name string, offset int, abs uint64) { +func (t Time) locabs() (name string, offset int, abs absSeconds) { l := t.loc if l == nil || l == &localLoc { l = l.get() @@ -506,44 +800,45 @@ func (t Time) locabs() (name string, offset int, abs uint64) { } else { name = "UTC" } - abs = uint64(sec + (unixToInternal + internalToAbsolute)) + abs = absSeconds(sec + (unixToInternal + internalToAbsolute)) return } // Date returns the year, month, and day in which t occurs. func (t Time) Date() (year int, month Month, day int) { - year, month, day, _ = t.date(true) - return + return t.absSec().days().date() } // Year returns the year in which t occurs. func (t Time) Year() int { - year, _, _, _ := t.date(false) - return year + century, cyear, ayday := t.absSec().days().split() + janFeb := ayday.janFeb() + return century.year(cyear, janFeb) } // Month returns the month of the year specified by t. func (t Time) Month() Month { - _, month, _, _ := t.date(true) - return month + _, _, ayday := t.absSec().days().split() + amonth, _ := ayday.split() + return amonth.month(ayday.janFeb()) } // Day returns the day of the month specified by t. func (t Time) Day() int { - _, _, day, _ := t.date(true) + _, _, ayday := t.absSec().days().split() + _, day := ayday.split() return day } // Weekday returns the day of the week specified by t. func (t Time) Weekday() Weekday { - return absWeekday(t.abs()) + return t.absSec().days().weekday() } -// absWeekday is like Weekday but operates on an absolute time. -func absWeekday(abs uint64) Weekday { - // January 1 of the absolute year, like January 1 of 2001, was a Monday. - sec := (abs + uint64(Monday)*secondsPerDay) % secondsPerWeek - return Weekday(int(sec) / secondsPerDay) +// weekday returns the day of the week specified by days. +func (days absDays) weekday() Weekday { + // March 1 of the absolute year, like March 1 of 2000, was a Wednesday. + return Weekday((uint64(days) + uint64(Wednesday)) % 7) } // ISOWeek returns the ISO 8601 year and week number in which t occurs. @@ -561,35 +856,19 @@ func (t Time) ISOWeek() (year, week int) { // 1 2 3 4 5 6 7 // +3 +2 +1 0 -1 -2 -3 // the offset to Thursday - abs := t.abs() - d := Thursday - absWeekday(abs) - // handle Sunday - if d == 4 { - d = -3 - } - // find the Thursday of the calendar week - abs += uint64(d) * secondsPerDay - year, _, _, yday := absDate(abs, false) - return year, yday/7 + 1 + days := t.absSec().days() + thu := days + absDays(Thursday-((days-1).weekday()+1)) + year, yday := thu.yearYday() + return year, (yday-1)/7 + 1 } // Clock returns the hour, minute, and second within the day specified by t. func (t Time) Clock() (hour, min, sec int) { - return absClock(t.abs()) + return t.absSec().clock() } -// absClock is like clock but operates on an absolute time. -// -// absClock should be an internal detail, -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/phuslu/log -// -// Do not remove or change the type signature. -// See go.dev/issue/67401. -// -//go:linkname absClock -func absClock(abs uint64) (hour, min, sec int) { +// clock returns the hour, minute, and second within the day specified by abs. +func (abs absSeconds) clock() (hour, min, sec int) { sec = int(abs % secondsPerDay) hour = sec / secondsPerHour sec -= hour * secondsPerHour @@ -600,17 +879,17 @@ func absClock(abs uint64) (hour, min, sec int) { // Hour returns the hour within the day specified by t, in the range [0, 23]. func (t Time) Hour() int { - return int(t.abs()%secondsPerDay) / secondsPerHour + return int(t.absSec()%secondsPerDay) / secondsPerHour } // Minute returns the minute offset within the hour specified by t, in the range [0, 59]. func (t Time) Minute() int { - return int(t.abs()%secondsPerHour) / secondsPerMinute + return int(t.absSec()%secondsPerHour) / secondsPerMinute } // Second returns the second offset within the minute specified by t, in the range [0, 59]. func (t Time) Second() int { - return int(t.abs() % secondsPerMinute) + return int(t.absSec() % secondsPerMinute) } // Nanosecond returns the nanosecond offset within the second specified by t, @@ -622,8 +901,8 @@ func (t Time) Nanosecond() int { // YearDay returns the day of the year specified by t, in the range [1,365] for non-leap years, // and [1,366] in leap years. func (t Time) YearDay() int { - _, _, _, yday := t.date(false) - return yday + 1 + _, yday := t.absSec().days().yearYday() + return yday } // A Duration represents the elapsed time between two instants @@ -870,7 +1149,8 @@ func (d Duration) Round(m Duration) Duration { } // Abs returns the absolute value of d. -// As a special case, [math.MinInt64] is converted to [math.MaxInt64]. +// As a special case, Duration([math.MinInt64]) is converted to Duration([math.MaxInt64]), +// reducing its magnitude by 1 nanosecond. func (d Duration) Abs() Duration { switch { case d >= 0: @@ -941,7 +1221,7 @@ func subMono(t, u int64) Duration { // Since returns the time elapsed since t. // It is shorthand for time.Now().Sub(t). func Since(t Time) Duration { - if t.wall&hasMonotonic != 0 { + if t.wall&hasMonotonic != 0 && !runtimeIsBubbled() { // Common case optimization: if t has monotonic time, then Sub will use only it. return subMono(runtimeNano()-startNano, t.ext) } @@ -951,7 +1231,7 @@ func Since(t Time) Duration { // Until returns the duration until t. // It is shorthand for t.Sub(time.Now()). func Until(t Time) Duration { - if t.wall&hasMonotonic != 0 { + if t.wall&hasMonotonic != 0 && !runtimeIsBubbled() { // Common case optimization: if t has monotonic time, then Sub will use only it. return subMono(t.ext, runtimeNano()-startNano) } @@ -981,169 +1261,73 @@ func (t Time) AddDate(years int, months int, days int) Time { return Date(year+years, month+Month(months), day+days, hour, min, sec, int(t.nsec()), t.Location()) } -const ( - secondsPerMinute = 60 - secondsPerHour = 60 * secondsPerMinute - secondsPerDay = 24 * secondsPerHour - secondsPerWeek = 7 * secondsPerDay - daysPer400Years = 365*400 + 97 - daysPer100Years = 365*100 + 24 - daysPer4Years = 365*4 + 1 -) +// daysBefore returns the number of days in a non-leap year before month m. +// daysBefore(December+1) returns 365. +func daysBefore(m Month) int { + adj := 0 + if m >= March { + adj = -2 + } + + // With the -2 adjustment after February, + // we need to compute the running sum of: + // 0 31 30 31 30 31 30 31 31 30 31 30 31 + // which is: + // 0 31 61 92 122 153 183 214 245 275 306 336 367 + // This is almost exactly 367/12×(m-1) except for the + // occasonal off-by-one suggesting there may be an + // integer approximation of the form (a×m + b)/c. + // A brute force search over small a, b, c finds that + // (214×m - 211) / 7 computes the function perfectly. + return (214*int(m)-211)/7 + adj +} -// date computes the year, day of year, and when full=true, -// the month and day in which t occurs. -func (t Time) date(full bool) (year int, month Month, day int, yday int) { - return absDate(t.abs(), full) +func daysIn(m Month, year int) int { + if m == February { + if isLeap(year) { + return 29 + } + return 28 + } + // With the special case of February eliminated, the pattern is + // 31 30 31 30 31 30 31 31 30 31 30 31 + // Adding m&1 produces the basic alternation; + // adding (m>>3)&1 inverts the alternation starting in August. + return 30 + int((m+m>>3)&1) } -// absDate is like date but operates on an absolute time. +// Provided by package runtime. +// +// now returns the current real time, and is superseded by runtimeNow which returns +// the fake synctest clock when appropriate. // -// absDate should be an internal detail, +// now should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: -// - github.com/phuslu/log // - gitee.com/quant1x/gox +// - github.com/phuslu/log +// - github.com/sethvargo/go-limiter +// - github.com/ulule/limiter/v3 // // Do not remove or change the type signature. // See go.dev/issue/67401. -// -//go:linkname absDate -func absDate(abs uint64, full bool) (year int, month Month, day int, yday int) { - // Split into time and day. - d := abs / secondsPerDay - - // Account for 400 year cycles. - n := d / daysPer400Years - y := 400 * n - d -= daysPer400Years * n - - // Cut off 100-year cycles. - // The last cycle has one extra leap year, so on the last day - // of that year, day / daysPer100Years will be 4 instead of 3. - // Cut it back down to 3 by subtracting n>>2. - n = d / daysPer100Years - n -= n >> 2 - y += 100 * n - d -= daysPer100Years * n - - // Cut off 4-year cycles. - // The last cycle has a missing leap year, which does not - // affect the computation. - n = d / daysPer4Years - y += 4 * n - d -= daysPer4Years * n - - // Cut off years within a 4-year cycle. - // The last year is a leap year, so on the last day of that year, - // day / 365 will be 4 instead of 3. Cut it back down to 3 - // by subtracting n>>2. - n = d / 365 - n -= n >> 2 - y += n - d -= 365 * n - - year = int(int64(y) + absoluteZeroYear) - yday = int(d) - - if !full { - return - } - - day = yday - if isLeap(year) { - // Leap year - switch { - case day > 31+29-1: - // After leap day; pretend it wasn't there. - day-- - case day == 31+29-1: - // Leap day. - month = February - day = 29 - return - } - } - - // Estimate month on assumption that every month has 31 days. - // The estimate may be too low by at most one month, so adjust. - month = Month(day / 31) - end := int(daysBefore[month+1]) - var begin int - if day >= end { - month++ - begin = end - } else { - begin = int(daysBefore[month]) - } - - month++ // because January is 1 - day = day - begin + 1 - return -} - -// daysBefore[m] counts the number of days in a non-leap year -// before month m begins. There is an entry for m=12, counting -// the number of days before January of next year (365). -var daysBefore = [...]int32{ - 0, - 31, - 31 + 28, - 31 + 28 + 31, - 31 + 28 + 31 + 30, - 31 + 28 + 31 + 30 + 31, - 31 + 28 + 31 + 30 + 31 + 30, - 31 + 28 + 31 + 30 + 31 + 30 + 31, - 31 + 28 + 31 + 30 + 31 + 30 + 31 + 31, - 31 + 28 + 31 + 30 + 31 + 30 + 31 + 31 + 30, - 31 + 28 + 31 + 30 + 31 + 30 + 31 + 31 + 30 + 31, - 31 + 28 + 31 + 30 + 31 + 30 + 31 + 31 + 30 + 31 + 30, - 31 + 28 + 31 + 30 + 31 + 30 + 31 + 31 + 30 + 31 + 30 + 31, -} - -func daysIn(m Month, year int) int { - if m == February && isLeap(year) { - return 29 - } - return int(daysBefore[m] - daysBefore[m-1]) -} - -// daysSinceEpoch takes a year and returns the number of days from -// the absolute epoch to the start of that year. -// This is basically (year - zeroYear) * 365, but accounting for leap days. -func daysSinceEpoch(year int) uint64 { - y := uint64(int64(year) - absoluteZeroYear) - - // Add in days from 400-year cycles. - n := y / 400 - y -= 400 * n - d := daysPer400Years * n - - // Add in 100-year cycles. - n = y / 100 - y -= 100 * n - d += daysPer100Years * n - - // Add in 4-year cycles. - n = y / 4 - y -= 4 * n - d += daysPer4Years * n - - // Add in non-leap years. - n = y - d += 365 * n - - return d -} - -// Provided by package runtime. func now() (sec int64, nsec int32, mono int64) +// runtimeNow returns the current time. +// When called within a synctest.Run bubble, it returns the group's fake clock. +// +//go:linkname runtimeNow +func runtimeNow() (sec int64, nsec int32, mono int64) + // runtimeNano returns the current value of the runtime clock in nanoseconds. +// When called within a synctest.Run bubble, it returns the group's fake clock. // -//go:linkname runtimeNano runtime.nanotime +//go:linkname runtimeNano func runtimeNano() int64 +//go:linkname runtimeIsBubbled +func runtimeIsBubbled() bool + // Monotonic times are reported as offsets from startNano. // We initialize startNano to runtimeNano() - 1 so that on systems where // monotonic time resolution is fairly low (e.g. Windows 2008 @@ -1157,7 +1341,10 @@ var startNano int64 = runtimeNano() - 1 // Now returns the current local time. func Now() Time { - sec, nsec, mono := now() + sec, nsec, mono := runtimeNow() + if mono == 0 { + return Time{uint64(nsec), sec + unixToInternal, Local} + } mono -= startNano sec += unixToInternal - minWall if uint64(sec)>>33 != 0 { @@ -1275,8 +1462,8 @@ const ( timeBinaryVersionV2 // For LMT only ) -// MarshalBinary implements the encoding.BinaryMarshaler interface. -func (t Time) MarshalBinary() ([]byte, error) { +// AppendBinary implements the [encoding.BinaryAppender] interface. +func (t Time) AppendBinary(b []byte) ([]byte, error) { var offsetMin int16 // minutes east of UTC. -1 is UTC. var offsetSec int8 version := timeBinaryVersionV1 @@ -1292,38 +1479,46 @@ func (t Time) MarshalBinary() ([]byte, error) { offset /= 60 if offset < -32768 || offset == -1 || offset > 32767 { - return nil, errors.New("Time.MarshalBinary: unexpected zone offset") + return b, errors.New("Time.MarshalBinary: unexpected zone offset") } offsetMin = int16(offset) } sec := t.sec() nsec := t.nsec() - enc := []byte{ - version, // byte 0 : version - byte(sec >> 56), // bytes 1-8: seconds - byte(sec >> 48), - byte(sec >> 40), - byte(sec >> 32), - byte(sec >> 24), - byte(sec >> 16), - byte(sec >> 8), + b = append(b, + version, // byte 0 : version + byte(sec>>56), // bytes 1-8: seconds + byte(sec>>48), + byte(sec>>40), + byte(sec>>32), + byte(sec>>24), + byte(sec>>16), + byte(sec>>8), byte(sec), - byte(nsec >> 24), // bytes 9-12: nanoseconds - byte(nsec >> 16), - byte(nsec >> 8), + byte(nsec>>24), // bytes 9-12: nanoseconds + byte(nsec>>16), + byte(nsec>>8), byte(nsec), - byte(offsetMin >> 8), // bytes 13-14: zone offset in minutes + byte(offsetMin>>8), // bytes 13-14: zone offset in minutes byte(offsetMin), - } + ) if version == timeBinaryVersionV2 { - enc = append(enc, byte(offsetSec)) + b = append(b, byte(offsetSec)) } + return b, nil +} - return enc, nil +// MarshalBinary implements the [encoding.BinaryMarshaler] interface. +func (t Time) MarshalBinary() ([]byte, error) { + b, err := t.AppendBinary(make([]byte, 0, 16)) + if err != nil { + return nil, err + } + return b, nil } -// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. +// UnmarshalBinary implements the [encoding.BinaryUnmarshaler] interface. func (t *Time) UnmarshalBinary(data []byte) error { buf := data if len(buf) == 0 { @@ -1385,7 +1580,7 @@ func (t *Time) GobDecode(data []byte) error { return t.UnmarshalBinary(data) } -// MarshalJSON implements the [json.Marshaler] interface. +// MarshalJSON implements the [encoding/json.Marshaler] interface. // The time is a quoted string in the RFC 3339 format with sub-second precision. // If the timestamp cannot be represented as valid RFC 3339 // (e.g., the year is out of range), then an error is reported. @@ -1400,7 +1595,7 @@ func (t Time) MarshalJSON() ([]byte, error) { return b, nil } -// UnmarshalJSON implements the [json.Unmarshaler] interface. +// UnmarshalJSON implements the [encoding/json.Unmarshaler] interface. // The time must be a quoted string in the RFC 3339 format. func (t *Time) UnmarshalJSON(data []byte) error { if string(data) == "null" { @@ -1416,19 +1611,30 @@ func (t *Time) UnmarshalJSON(data []byte) error { return err } -// MarshalText implements the [encoding.TextMarshaler] interface. -// The time is formatted in RFC 3339 format with sub-second precision. -// If the timestamp cannot be represented as valid RFC 3339 -// (e.g., the year is out of range), then an error is reported. -func (t Time) MarshalText() ([]byte, error) { - b := make([]byte, 0, len(RFC3339Nano)) +func (t Time) appendTo(b []byte, errPrefix string) ([]byte, error) { b, err := t.appendStrictRFC3339(b) if err != nil { - return nil, errors.New("Time.MarshalText: " + err.Error()) + return nil, errors.New(errPrefix + err.Error()) } return b, nil } +// AppendText implements the [encoding.TextAppender] interface. +// The time is formatted in RFC 3339 format with sub-second precision. +// If the timestamp cannot be represented as valid RFC 3339 +// (e.g., the year is out of range), then an error is returned. +func (t Time) AppendText(b []byte) ([]byte, error) { + return t.appendTo(b, "Time.AppendText: ") +} + +// MarshalText implements the [encoding.TextMarshaler] interface. The output +// matches that of calling the [Time.AppendText] method. +// +// See [Time.AppendText] for more information. +func (t Time) MarshalText() ([]byte, error) { + return t.appendTo(make([]byte, 0, len(RFC3339Nano)), "Time.MarshalText: ") +} + // UnmarshalText implements the [encoding.TextUnmarshaler] interface. // The time must be in the RFC 3339 format. func (t *Time) UnmarshalText(data []byte) error { @@ -1474,7 +1680,15 @@ func (t Time) IsDST() bool { } func isLeap(year int) bool { - return year%4 == 0 && (year%100 != 0 || year%400 == 0) + // year%4 == 0 && (year%100 != 0 || year%400 == 0) + // Bottom 2 bits must be clear. + // For multiples of 25, bottom 4 bits must be clear. + // Thanks to Cassio Neri for this trick. + mask := 0xf + if year%25 != 0 { + mask = 3 + } + return year&mask == 0 } // norm returns nhi, nlo such that @@ -1529,23 +1743,10 @@ func Date(year int, month Month, day, hour, min, sec, nsec int, loc *Location) T hour, min = norm(hour, min, 60) day, hour = norm(day, hour, 24) - // Compute days since the absolute epoch. - d := daysSinceEpoch(year) - - // Add in days before this month. - d += uint64(daysBefore[month-1]) - if isLeap(year) && month >= March { - d++ // February 29 - } - - // Add in days before today. - d += uint64(day - 1) - - // Add in time elapsed today. - abs := d * secondsPerDay - abs += uint64(hour*secondsPerHour + min*secondsPerMinute + sec) - - unix := int64(abs) + (absoluteToInternal + internalToUnix) + // Convert to absolute time and then Unix time. + unix := int64(dateToAbsDays(int64(year), month, day))*secondsPerDay + + int64(hour*secondsPerHour+min*secondsPerMinute+sec) + + absoluteToUnix // Look for zone offset for expected time, so we can adjust to UTC. // The lookup function expects UTC, so first we pass unix in the @@ -1693,3 +1894,50 @@ func div(t Time, d Duration) (qmod2 int, r Duration) { } return } + +// Regrettable Linkname Compatibility +// +// timeAbs, absDate, and absClock mimic old internal details, no longer used. +// Widely used packages linknamed these to get “faster” time routines. +// Notable members of the hall of shame include: +// - gitee.com/quant1x/gox +// - github.com/phuslu/log +// +// phuslu hard-coded 'Unix time + 9223372028715321600' [sic] +// as the input to absDate and absClock, using the old Jan 1-based +// absolute times. +// quant1x linknamed the time.Time.abs method and passed the +// result of that method to absDate and absClock. +// +// Keeping both of these working forces us to provide these three +// routines here, operating on the old Jan 1-based epoch instead +// of the new March 1-based epoch. And the fact that time.Time.abs +// was linknamed means that we have to call the current abs method +// something different (time.Time.absSec, defined above) to make it +// possible to provide this simulation of the old routines here. +// +// None of this code is linked into the binary if not referenced by +// these linkname-happy packages. In particular, despite its name, +// time.Time.abs does not appear in the time.Time method table. +// +// Do not remove these routines or their linknames, or change the +// type signature or meaning of arguments. + +//go:linkname legacyTimeTimeAbs time.Time.abs +func legacyTimeTimeAbs(t Time) uint64 { + return uint64(t.absSec() - marchThruDecember*secondsPerDay) +} + +//go:linkname legacyAbsClock time.absClock +func legacyAbsClock(abs uint64) (hour, min, sec int) { + return absSeconds(abs + marchThruDecember*secondsPerDay).clock() +} + +//go:linkname legacyAbsDate time.absDate +func legacyAbsDate(abs uint64, full bool) (year int, month Month, day int, yday int) { + d := absSeconds(abs + marchThruDecember*secondsPerDay).days() + year, month, day = d.date() + _, yday = d.yearYday() + yday-- // yearYday is 1-based, old API was 0-based + return +} diff --git a/src/time/time_test.go b/src/time/time_test.go index 70eb61478480e0..a453ee043c4985 100644 --- a/src/time/time_test.go +++ b/src/time/time_test.go @@ -9,11 +9,13 @@ import ( "encoding/gob" "encoding/json" "fmt" + "internal/goexperiment" "math" "math/big" "math/rand" "os" "runtime" + "slices" "strings" "sync" "testing" @@ -21,6 +23,26 @@ import ( . "time" ) +func TestInternal(t *testing.T) { + for _, tt := range InternalTests { + t.Run(tt.Name, func(t *testing.T) { tt.Test(t) }) + } +} + +func TestZeroTime(t *testing.T) { + var zero Time + year, month, day := zero.Date() + hour, min, sec := zero.Clock() + nsec := zero.Nanosecond() + yday := zero.YearDay() + wday := zero.Weekday() + if year != 1 || month != January || day != 1 || hour != 0 || min != 0 || sec != 0 || nsec != 0 || yday != 1 || wday != Monday { + t.Errorf("zero time = %v %v %v year %v %02d:%02d:%02d.%09d yday %d want Monday Jan 1 year 1 00:00:00.000000000 yday 1", + wday, month, day, year, hour, min, sec, nsec, yday) + } + +} + // We should be in PST/PDT, but if the time zone files are missing we // won't be. The purpose of this test is to at least explain why some of // the subsequent tests fail. @@ -102,75 +124,75 @@ func same(t Time, u *parsedTime) bool { t.Weekday() == u.Weekday } -func TestSecondsToUTC(t *testing.T) { +func TestUnixUTC(t *testing.T) { for _, test := range utctests { sec := test.seconds golden := &test.golden tm := Unix(sec, 0).UTC() newsec := tm.Unix() if newsec != sec { - t.Errorf("SecondsToUTC(%d).Seconds() = %d", sec, newsec) + t.Errorf("Unix(%d, 0).Unix() = %d", sec, newsec) } if !same(tm, golden) { - t.Errorf("SecondsToUTC(%d): // %#v", sec, tm) + t.Errorf("Unix(%d, 0): // %#v", sec, tm) t.Errorf(" want=%+v", *golden) t.Errorf(" have=%v", tm.Format(RFC3339+" MST")) } } } -func TestNanosecondsToUTC(t *testing.T) { +func TestUnixNanoUTC(t *testing.T) { for _, test := range nanoutctests { golden := &test.golden nsec := test.seconds*1e9 + int64(golden.Nanosecond) tm := Unix(0, nsec).UTC() newnsec := tm.Unix()*1e9 + int64(tm.Nanosecond()) if newnsec != nsec { - t.Errorf("NanosecondsToUTC(%d).Nanoseconds() = %d", nsec, newnsec) + t.Errorf("Unix(0, %d).Nanoseconds() = %d", nsec, newnsec) } if !same(tm, golden) { - t.Errorf("NanosecondsToUTC(%d):", nsec) + t.Errorf("Unix(0, %d):", nsec) t.Errorf(" want=%+v", *golden) t.Errorf(" have=%+v", tm.Format(RFC3339+" MST")) } } } -func TestSecondsToLocalTime(t *testing.T) { +func TestUnix(t *testing.T) { for _, test := range localtests { sec := test.seconds golden := &test.golden tm := Unix(sec, 0) newsec := tm.Unix() if newsec != sec { - t.Errorf("SecondsToLocalTime(%d).Seconds() = %d", sec, newsec) + t.Errorf("Unix(%d, 0).Seconds() = %d", sec, newsec) } if !same(tm, golden) { - t.Errorf("SecondsToLocalTime(%d):", sec) + t.Errorf("Unix(%d, 0):", sec) t.Errorf(" want=%+v", *golden) t.Errorf(" have=%+v", tm.Format(RFC3339+" MST")) } } } -func TestNanosecondsToLocalTime(t *testing.T) { +func TestUnixNano(t *testing.T) { for _, test := range nanolocaltests { golden := &test.golden nsec := test.seconds*1e9 + int64(golden.Nanosecond) tm := Unix(0, nsec) newnsec := tm.Unix()*1e9 + int64(tm.Nanosecond()) if newnsec != nsec { - t.Errorf("NanosecondsToLocalTime(%d).Seconds() = %d", nsec, newnsec) + t.Errorf("Unix(0, %d).Seconds() = %d", nsec, newnsec) } if !same(tm, golden) { - t.Errorf("NanosecondsToLocalTime(%d):", nsec) + t.Errorf("Unix(0, %d):", nsec) t.Errorf(" want=%+v", *golden) t.Errorf(" have=%+v", tm.Format(RFC3339+" MST")) } } } -func TestSecondsToUTCAndBack(t *testing.T) { +func TestUnixUTCAndBack(t *testing.T) { f := func(sec int64) bool { return Unix(sec, 0).UTC().Unix() == sec } f32 := func(sec int32) bool { return f(int64(sec)) } cfg := &quick.Config{MaxCount: 10000} @@ -184,7 +206,7 @@ func TestSecondsToUTCAndBack(t *testing.T) { } } -func TestNanosecondsToUTCAndBack(t *testing.T) { +func TestUnixNanoUTCAndBack(t *testing.T) { f := func(nsec int64) bool { t := Unix(0, nsec).UTC() ns := t.Unix()*1e9 + int64(t.Nanosecond()) @@ -646,6 +668,9 @@ var dateTests = []struct { {2012, 1, -43, 7, 56, 35, 0, Local, 1321631795}, // Jan -52 7:56:35 2012 {2012, int(January - 2), 18, 7, 56, 35, 0, Local, 1321631795}, // (Jan-2) 18 7:56:35 2012 {2010, int(December + 11), 18, 7, 56, 35, 0, Local, 1321631795}, // (Dec+11) 18 7:56:35 2010 + {1970, 1, 15297, 7, 56, 35, 0, Local, 1321631795}, // large number of days + + {1970, 1, -25508, 0, 0, 0, 0, Local, -2203948800}, // negative Unix time } func TestDate(t *testing.T) { @@ -684,6 +709,13 @@ func TestAddDate(t *testing.T) { time, t1) } } + + t2 := Date(1899, 12, 31, 0, 0, 0, 0, UTC) + days := t2.Unix() / (24 * 60 * 60) + t3 := Unix(0, 0).AddDate(0, 0, int(days)) + if !t2.Equal(t3) { + t.Errorf("Adddate(0, 0, %d) = %v, want %v", days, t3, t2) + } } var daysInTests = []struct { @@ -828,8 +860,20 @@ func TestUnmarshalInvalidTimes(t *testing.T) { in string want string }{ - {`{}`, "Time.UnmarshalJSON: input is not a JSON string"}, - {`[]`, "Time.UnmarshalJSON: input is not a JSON string"}, + {`{}`, func() string { + if goexperiment.JSONv2 { + return "json: cannot unmarshal JSON object into Go type time.Time" + } else { + return "Time.UnmarshalJSON: input is not a JSON string" + } + }()}, + {`[]`, func() string { + if goexperiment.JSONv2 { + return "json: cannot unmarshal JSON array into Go type time.Time" + } else { + return "Time.UnmarshalJSON: input is not a JSON string" + } + }()}, {`"2000-01-01T1:12:34Z"`, ``}, {`"2000-01-01T00:00:00,000Z"`, ``}, {`"2000-01-01T00:00:00+24:00"`, ``}, @@ -885,6 +929,16 @@ func TestMarshalInvalidTimes(t *testing.T) { case err == nil || err.Error() != want: t.Errorf("(%v).MarshalText() error = %v, want %v", tt.time, err, want) } + + buf := make([]byte, 0, 64) + want = strings.ReplaceAll(tt.want, "MarshalJSON", "AppendText") + b, err = tt.time.AppendText(buf) + switch { + case b != nil: + t.Errorf("(%v).AppendText() = %q, want nil", tt.time, b) + case err == nil || err.Error() != want: + t.Errorf("(%v).AppendText() error = %v, want %v", tt.time, err, want) + } } } @@ -1084,10 +1138,15 @@ func TestLoadFixed(t *testing.T) { // So GMT+1 corresponds to -3600 in the Go zone, not +3600. name, offset := Now().In(loc).Zone() // The zone abbreviation is "-01" since tzdata-2016g, and "GMT+1" - // on earlier versions; we accept both. (Issue #17276). - if !(name == "GMT+1" || name == "-01") || offset != -1*60*60 { - t.Errorf("Now().In(loc).Zone() = %q, %d, want %q or %q, %d", - name, offset, "GMT+1", "-01", -1*60*60) + // on earlier versions; we accept both. (Issue 17276.) + wantName := []string{"GMT+1", "-01"} + // The zone abbreviation may be "+01" on OpenBSD. (Issue 69840.) + if runtime.GOOS == "openbsd" { + wantName = append(wantName, "+01") + } + if !slices.Contains(wantName, name) || offset != -1*60*60 { + t.Errorf("Now().In(loc).Zone() = %q, %d, want %q (one of), %d", + name, offset, wantName, -1*60*60) } } @@ -1354,7 +1413,7 @@ var defaultLocTests = []struct { {"Add", func(t1, t2 Time) bool { return t1.Add(Hour).Equal(t2.Add(Hour)) }}, {"Sub", func(t1, t2 Time) bool { return t1.Sub(t2) == t2.Sub(t1) }}, - //Original caus for this test case bug 15852 + // Original cause for this test case bug 15852 {"AddDate", func(t1, t2 Time) bool { return t1.AddDate(1991, 9, 3) == t2.AddDate(1991, 9, 3) }}, {"UTC", func(t1, t2 Time) bool { return t1.UTC() == t2.UTC() }}, @@ -1373,6 +1432,13 @@ var defaultLocTests = []struct { {"UnixMilli", func(t1, t2 Time) bool { return t1.UnixMilli() == t2.UnixMilli() }}, {"UnixMicro", func(t1, t2 Time) bool { return t1.UnixMicro() == t2.UnixMicro() }}, + {"AppendBinary", func(t1, t2 Time) bool { + buf1 := make([]byte, 4, 32) + buf2 := make([]byte, 4, 32) + a1, b1 := t1.AppendBinary(buf1) + a2, b2 := t2.AppendBinary(buf2) + return bytes.Equal(a1[4:], a2[4:]) && b1 == b2 + }}, {"MarshalBinary", func(t1, t2 Time) bool { a1, b1 := t1.MarshalBinary() a2, b2 := t2.MarshalBinary() @@ -1388,6 +1454,14 @@ var defaultLocTests = []struct { a2, b2 := t2.MarshalJSON() return bytes.Equal(a1, a2) && b1 == b2 }}, + {"AppendText", func(t1, t2 Time) bool { + maxCap := len(RFC3339Nano) + 4 + buf1 := make([]byte, 4, maxCap) + buf2 := make([]byte, 4, maxCap) + a1, b1 := t1.AppendText(buf1) + a2, b2 := t2.AppendText(buf2) + return bytes.Equal(a1[4:], a2[4:]) && b1 == b2 + }}, {"MarshalText", func(t1, t2 Time) bool { a1, b1 := t1.MarshalText() a2, b2 := t2.MarshalText() @@ -1436,6 +1510,20 @@ func BenchmarkNowUnixMicro(b *testing.B) { } } +func BenchmarkSince(b *testing.B) { + start := Now() + for b.Loop() { + u = int64(Since(start)) + } +} + +func BenchmarkUntil(b *testing.B) { + end := Now().Add(1 * Hour) + for b.Loop() { + u = int64(Until(end)) + } +} + func BenchmarkFormat(b *testing.B) { t := Unix(1265346057, 0) for i := 0; i < b.N; i++ { @@ -1480,6 +1568,13 @@ func BenchmarkMarshalText(b *testing.B) { } } +func BenchmarkMarshalBinary(b *testing.B) { + t := Now() + for i := 0; i < b.N; i++ { + t.MarshalBinary() + } +} + func BenchmarkParse(b *testing.B) { for i := 0; i < b.N; i++ { Parse(ANSIC, "Mon Jan 2 15:04:05 2006") @@ -1525,6 +1620,13 @@ func BenchmarkParseDuration(b *testing.B) { } } +func BenchmarkParseDurationError(b *testing.B) { + for i := 0; i < b.N; i++ { + ParseDuration("9223372036854775810ns") // overflow + ParseDuration("9007199254.740993") // missing unit + } +} + func BenchmarkHour(b *testing.B) { t := Now() for i := 0; i < b.N; i++ { diff --git a/src/time/zoneinfo.go b/src/time/zoneinfo.go index 0fe13630e9a20e..f0444a5d9daf17 100644 --- a/src/time/zoneinfo.go +++ b/src/time/zoneinfo.go @@ -331,14 +331,11 @@ func tzset(s string, lastTxSec, sec int64) (name string, offset int, start, end return "", 0, 0, 0, false, false } - year, _, _, yday := absDate(uint64(sec+unixToInternal+internalToAbsolute), false) - - ysec := int64(yday*secondsPerDay) + sec%secondsPerDay - - // Compute start of year in seconds since Unix epoch. - d := daysSinceEpoch(year) - abs := int64(d * secondsPerDay) - abs += absoluteToInternal + internalToUnix + // Compute start of year in seconds since Unix epoch, + // and seconds since then to get to sec. + year, yday := absSeconds(sec + unixToInternal + internalToAbsolute).days().yearYday() + ysec := int64((yday-1)*secondsPerDay) + sec%secondsPerDay + ystart := sec - ysec startSec := int64(tzruleTime(year, startRule, stdOffset)) endSec := int64(tzruleTime(year, endRule, dstOffset)) @@ -358,11 +355,11 @@ func tzset(s string, lastTxSec, sec int64) (name string, offset int, start, end // just the start and end of the year. That suffices for // the only caller that cares, which is Date. if ysec < startSec { - return stdName, stdOffset, abs, startSec + abs, stdIsDST, true + return stdName, stdOffset, ystart, startSec + ystart, stdIsDST, true } else if ysec >= endSec { - return stdName, stdOffset, endSec + abs, abs + 365*secondsPerDay, stdIsDST, true + return stdName, stdOffset, endSec + ystart, ystart + 365*secondsPerDay, stdIsDST, true } else { - return dstName, dstOffset, startSec + abs, endSec + abs, dstIsDST, true + return dstName, dstOffset, startSec + ystart, endSec + ystart, dstIsDST, true } } @@ -596,7 +593,7 @@ func tzruleTime(year int, r rule, off int) int { } d += 7 } - d += int(daysBefore[r.mon-1]) + d += daysBefore(Month(r.mon)) if isLeap(year) && r.mon > 2 { d++ } diff --git a/src/time/zoneinfo_abbrs_windows.go b/src/time/zoneinfo_abbrs_windows.go index 27831743e9e34e..814d9443c8642f 100644 --- a/src/time/zoneinfo_abbrs_windows.go +++ b/src/time/zoneinfo_abbrs_windows.go @@ -36,7 +36,7 @@ var abbrs = map[string]abbr{ "Central Standard Time": {"CST", "CDT"}, // America/Chicago "Central Brazilian Standard Time": {"-04", "-04"}, // America/Cuiaba "Mountain Standard Time": {"MST", "MDT"}, // America/Denver - "Greenland Standard Time": {"-03", "-02"}, // America/Godthab + "Greenland Standard Time": {"-02", "-01"}, // America/Godthab "Turks And Caicos Standard Time": {"EST", "EDT"}, // America/Grand_Turk "Central America Standard Time": {"CST", "CST"}, // America/Guatemala "Atlantic Standard Time": {"AST", "ADT"}, // America/Halifax @@ -58,13 +58,13 @@ var abbrs = map[string]abbr{ "Newfoundland Standard Time": {"NST", "NDT"}, // America/St_Johns "Pacific Standard Time (Mexico)": {"PST", "PDT"}, // America/Tijuana "Yukon Standard Time": {"MST", "MST"}, // America/Whitehorse - "Central Asia Standard Time": {"+06", "+06"}, // Asia/Almaty "Jordan Standard Time": {"+03", "+03"}, // Asia/Amman "Arabic Standard Time": {"+03", "+03"}, // Asia/Baghdad "Azerbaijan Standard Time": {"+04", "+04"}, // Asia/Baku "SE Asia Standard Time": {"+07", "+07"}, // Asia/Bangkok "Altai Standard Time": {"+07", "+07"}, // Asia/Barnaul "Middle East Standard Time": {"EET", "EEST"}, // Asia/Beirut + "Central Asia Standard Time": {"+06", "+06"}, // Asia/Bishkek "India Standard Time": {"IST", "IST"}, // Asia/Calcutta "Transbaikal Standard Time": {"+09", "+09"}, // Asia/Chita "Sri Lanka Standard Time": {"+0530", "+0530"}, // Asia/Colombo diff --git a/src/time/zoneinfo_plan9.go b/src/time/zoneinfo_plan9.go index d13b623a3738e5..036f669d8d7dc7 100644 --- a/src/time/zoneinfo_plan9.go +++ b/src/time/zoneinfo_plan9.go @@ -96,7 +96,7 @@ func loadZoneDataPlan9(s string) (l *Location, err error) { // Fill in the cache with information about right now, // since that will be the most common lookup. - sec, _, _ := now() + sec, _, _ := runtimeNow() for i := range tx { if tx[i].when <= sec && (i+1 == len(tx) || sec < tx[i+1].when) { l.cacheStart = tx[i].when diff --git a/src/time/zoneinfo_read.go b/src/time/zoneinfo_read.go index 5314b6ff9a1623..047e360d11ae9d 100644 --- a/src/time/zoneinfo_read.go +++ b/src/time/zoneinfo_read.go @@ -320,7 +320,7 @@ func LoadLocationFromTZData(name string, data []byte) (*Location, error) { // Fill in the cache with information about right now, // since that will be the most common lookup. - sec, _, _ := now() + sec, _, _ := runtimeNow() for i := range tx { if tx[i].when <= sec && (i+1 == len(tx) || sec < tx[i+1].when) { l.cacheStart = tx[i].when diff --git a/src/time/zoneinfo_windows.go b/src/time/zoneinfo_windows.go index c9f38ea3e058d8..55b21fa301584b 100644 --- a/src/time/zoneinfo_windows.go +++ b/src/time/zoneinfo_windows.go @@ -20,7 +20,7 @@ var platformZoneSources []string // none: Windows uses system calls instead // time apply to all previous and future years as well. // matchZoneKey checks if stdname and dstname match the corresponding key -// values "MUI_Std" and MUI_Dlt" or "Std" and "Dlt" in the kname key stored +// values "MUI_Std" and "MUI_Dlt" or "Std" and "Dlt" in the kname key stored // under the open registry key zones. func matchZoneKey(zones registry.Key, kname string, stdname, dstname string) (matched bool, err2 error) { k, err := registry.OpenKey(zones, kname, registry.READ) diff --git a/src/unicode/letter.go b/src/unicode/letter.go index 9e2cead631974d..3959314c979361 100644 --- a/src/unicode/letter.go +++ b/src/unicode/letter.go @@ -206,34 +206,17 @@ func IsTitle(r rune) bool { return isExcludingLatin(Title, r) } -// to maps the rune using the specified case mapping. -// It additionally reports whether caseRange contained a mapping for r. -func to(_case int, r rune, caseRange []CaseRange) (mappedRune rune, foundMapping bool) { - if _case < 0 || MaxCase <= _case { - return ReplacementChar, false // as reasonable an error as any - } +// lookupCaseRange returns the CaseRange mapping for rune r or nil if no +// mapping exists for r. +func lookupCaseRange(r rune, caseRange []CaseRange) *CaseRange { // binary search over ranges lo := 0 hi := len(caseRange) for lo < hi { m := int(uint(lo+hi) >> 1) - cr := caseRange[m] + cr := &caseRange[m] if rune(cr.Lo) <= r && r <= rune(cr.Hi) { - delta := cr.Delta[_case] - if delta > MaxRune { - // In an Upper-Lower sequence, which always starts with - // an UpperCase letter, the real deltas always look like: - // {0, 1, 0} UpperCase (Lower is next) - // {-1, 0, -1} LowerCase (Upper, Title are previous) - // The characters at even offsets from the beginning of the - // sequence are upper case; the ones at odd offsets are lower. - // The correct mapping can be done by clearing or setting the low - // bit in the sequence offset. - // The constants UpperCase and TitleCase are even while LowerCase - // is odd so we take the low bit from _case. - return rune(cr.Lo) + ((r-rune(cr.Lo))&^1 | rune(_case&1)), true - } - return r + delta, true + return cr } if r < rune(cr.Lo) { hi = m @@ -241,6 +224,37 @@ func to(_case int, r rune, caseRange []CaseRange) (mappedRune rune, foundMapping lo = m + 1 } } + return nil +} + +// convertCase converts r to _case using CaseRange cr. +func convertCase(_case int, r rune, cr *CaseRange) rune { + delta := cr.Delta[_case] + if delta > MaxRune { + // In an Upper-Lower sequence, which always starts with + // an UpperCase letter, the real deltas always look like: + // {0, 1, 0} UpperCase (Lower is next) + // {-1, 0, -1} LowerCase (Upper, Title are previous) + // The characters at even offsets from the beginning of the + // sequence are upper case; the ones at odd offsets are lower. + // The correct mapping can be done by clearing or setting the low + // bit in the sequence offset. + // The constants UpperCase and TitleCase are even while LowerCase + // is odd so we take the low bit from _case. + return rune(cr.Lo) + ((r-rune(cr.Lo))&^1 | rune(_case&1)) + } + return r + delta +} + +// to maps the rune using the specified case mapping. +// It additionally reports whether caseRange contained a mapping for r. +func to(_case int, r rune, caseRange []CaseRange) (mappedRune rune, foundMapping bool) { + if _case < 0 || MaxCase <= _case { + return ReplacementChar, false // as reasonable an error as any + } + if cr := lookupCaseRange(r, caseRange); cr != nil { + return convertCase(_case, r, cr), true + } return r, false } @@ -364,8 +378,11 @@ func SimpleFold(r rune) rune { // No folding specified. This is a one- or two-element // equivalence class containing rune and ToLower(rune) // and ToUpper(rune) if they are different from rune. - if l := ToLower(r); l != r { - return l + if cr := lookupCaseRange(r, CaseRanges); cr != nil { + if l := convertCase(LowerCase, r, cr); l != r { + return l + } + return convertCase(UpperCase, r, cr) } - return ToUpper(r) + return r } diff --git a/src/unicode/letter_test.go b/src/unicode/letter_test.go index 123f9a642e9bf6..75c8aeee90b2e1 100644 --- a/src/unicode/letter_test.go +++ b/src/unicode/letter_test.go @@ -642,3 +642,29 @@ func TestNegativeRune(t *testing.T) { } } } + +func BenchmarkToUpper(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = ToUpper('δ') + } +} + +func BenchmarkToLower(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = ToLower('Δ') + } +} + +func BenchmarkSimpleFold(b *testing.B) { + bench := func(name string, r rune) { + b.Run(name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = SimpleFold(r) + } + }) + } + bench("Upper", 'Δ') + bench("Lower", 'δ') + bench("Fold", '\u212A') + bench("NoFold", '習') +} diff --git a/src/unicode/script_test.go b/src/unicode/script_test.go index 66bfa3cb9e95d0..3ae2800426609c 100644 --- a/src/unicode/script_test.go +++ b/src/unicode/script_test.go @@ -52,6 +52,10 @@ var inCategoryTest = []T{ {0x00bb, "P"}, {0x00a2, "S"}, {0x00a0, "Z"}, + {0x0065, "LC"}, + // Unassigned + {0x0378, "Cn"}, + {0x0378, "C"}, } var inPropTest = []T{ diff --git a/src/unicode/tables.go b/src/unicode/tables.go index a0c0e1ca51c7ef..f66d553c5da652 100644 --- a/src/unicode/tables.go +++ b/src/unicode/tables.go @@ -10,9 +10,11 @@ var Categories = map[string]*RangeTable{ "C": C, "Cc": Cc, "Cf": Cf, + "Cn": Cn, "Co": Co, "Cs": Cs, "L": L, + "LC": LC, "Ll": Ll, "Lm": Lm, "Lo": Lo, @@ -45,33 +47,665 @@ var Categories = map[string]*RangeTable{ "Zs": Zs, } +// CategoryAliases maps category aliases to standard category names. +var CategoryAliases = map[string]string{ + "Cased_Letter": "LC", + "Close_Punctuation": "Pe", + "Combining_Mark": "M", + "Connector_Punctuation": "Pc", + "Control": "Cc", + "Currency_Symbol": "Sc", + "Dash_Punctuation": "Pd", + "Decimal_Number": "Nd", + "Enclosing_Mark": "Me", + "Final_Punctuation": "Pf", + "Format": "Cf", + "Initial_Punctuation": "Pi", + "Letter": "L", + "Letter_Number": "Nl", + "Line_Separator": "Zl", + "Lowercase_Letter": "Ll", + "Mark": "M", + "Math_Symbol": "Sm", + "Modifier_Letter": "Lm", + "Modifier_Symbol": "Sk", + "Nonspacing_Mark": "Mn", + "Number": "N", + "Open_Punctuation": "Ps", + "Other": "C", + "Other_Letter": "Lo", + "Other_Number": "No", + "Other_Punctuation": "Po", + "Other_Symbol": "So", + "Paragraph_Separator": "Zp", + "Private_Use": "Co", + "Punctuation": "P", + "Separator": "Z", + "Space_Separator": "Zs", + "Spacing_Mark": "Mc", + "Surrogate": "Cs", + "Symbol": "S", + "Titlecase_Letter": "Lt", + "Unassigned": "Cn", + "Uppercase_Letter": "Lu", + "cntrl": "Cc", + "digit": "Nd", + "punct": "P", +} + var _C = &RangeTable{ R16: []Range16{ {0x0000, 0x001f, 1}, {0x007f, 0x009f, 1}, - {0x00ad, 0x0600, 1363}, - {0x0601, 0x0605, 1}, + {0x00ad, 0x0378, 715}, + {0x0379, 0x0380, 7}, + {0x0381, 0x0383, 1}, + {0x038b, 0x038d, 2}, + {0x03a2, 0x0530, 398}, + {0x0557, 0x0558, 1}, + {0x058b, 0x058c, 1}, + {0x0590, 0x05c8, 56}, + {0x05c9, 0x05cf, 1}, + {0x05eb, 0x05ee, 1}, + {0x05f5, 0x0605, 1}, {0x061c, 0x06dd, 193}, - {0x070f, 0x0890, 385}, - {0x0891, 0x08e2, 81}, - {0x180e, 0x200b, 2045}, + {0x070e, 0x070f, 1}, + {0x074b, 0x074c, 1}, + {0x07b2, 0x07bf, 1}, + {0x07fb, 0x07fc, 1}, + {0x082e, 0x082f, 1}, + {0x083f, 0x085c, 29}, + {0x085d, 0x085f, 2}, + {0x086b, 0x086f, 1}, + {0x088f, 0x0897, 1}, + {0x08e2, 0x0984, 162}, + {0x098d, 0x098e, 1}, + {0x0991, 0x0992, 1}, + {0x09a9, 0x09b1, 8}, + {0x09b3, 0x09b5, 1}, + {0x09ba, 0x09bb, 1}, + {0x09c5, 0x09c6, 1}, + {0x09c9, 0x09ca, 1}, + {0x09cf, 0x09d6, 1}, + {0x09d8, 0x09db, 1}, + {0x09de, 0x09e4, 6}, + {0x09e5, 0x09ff, 26}, + {0x0a00, 0x0a04, 4}, + {0x0a0b, 0x0a0e, 1}, + {0x0a11, 0x0a12, 1}, + {0x0a29, 0x0a31, 8}, + {0x0a34, 0x0a3a, 3}, + {0x0a3b, 0x0a3d, 2}, + {0x0a43, 0x0a46, 1}, + {0x0a49, 0x0a4a, 1}, + {0x0a4e, 0x0a50, 1}, + {0x0a52, 0x0a58, 1}, + {0x0a5d, 0x0a5f, 2}, + {0x0a60, 0x0a65, 1}, + {0x0a77, 0x0a80, 1}, + {0x0a84, 0x0a8e, 10}, + {0x0a92, 0x0aa9, 23}, + {0x0ab1, 0x0ab4, 3}, + {0x0aba, 0x0abb, 1}, + {0x0ac6, 0x0ace, 4}, + {0x0acf, 0x0ad1, 2}, + {0x0ad2, 0x0adf, 1}, + {0x0ae4, 0x0ae5, 1}, + {0x0af2, 0x0af8, 1}, + {0x0b00, 0x0b04, 4}, + {0x0b0d, 0x0b0e, 1}, + {0x0b11, 0x0b12, 1}, + {0x0b29, 0x0b31, 8}, + {0x0b34, 0x0b3a, 6}, + {0x0b3b, 0x0b45, 10}, + {0x0b46, 0x0b49, 3}, + {0x0b4a, 0x0b4e, 4}, + {0x0b4f, 0x0b54, 1}, + {0x0b58, 0x0b5b, 1}, + {0x0b5e, 0x0b64, 6}, + {0x0b65, 0x0b78, 19}, + {0x0b79, 0x0b81, 1}, + {0x0b84, 0x0b8b, 7}, + {0x0b8c, 0x0b8d, 1}, + {0x0b91, 0x0b96, 5}, + {0x0b97, 0x0b98, 1}, + {0x0b9b, 0x0b9d, 2}, + {0x0ba0, 0x0ba2, 1}, + {0x0ba5, 0x0ba7, 1}, + {0x0bab, 0x0bad, 1}, + {0x0bba, 0x0bbd, 1}, + {0x0bc3, 0x0bc5, 1}, + {0x0bc9, 0x0bce, 5}, + {0x0bcf, 0x0bd1, 2}, + {0x0bd2, 0x0bd6, 1}, + {0x0bd8, 0x0be5, 1}, + {0x0bfb, 0x0bff, 1}, + {0x0c0d, 0x0c11, 4}, + {0x0c29, 0x0c3a, 17}, + {0x0c3b, 0x0c45, 10}, + {0x0c49, 0x0c4e, 5}, + {0x0c4f, 0x0c54, 1}, + {0x0c57, 0x0c5b, 4}, + {0x0c5c, 0x0c5e, 2}, + {0x0c5f, 0x0c64, 5}, + {0x0c65, 0x0c70, 11}, + {0x0c71, 0x0c76, 1}, + {0x0c8d, 0x0c91, 4}, + {0x0ca9, 0x0cb4, 11}, + {0x0cba, 0x0cbb, 1}, + {0x0cc5, 0x0cc9, 4}, + {0x0cce, 0x0cd4, 1}, + {0x0cd7, 0x0cdc, 1}, + {0x0cdf, 0x0ce4, 5}, + {0x0ce5, 0x0cf0, 11}, + {0x0cf4, 0x0cff, 1}, + {0x0d0d, 0x0d11, 4}, + {0x0d45, 0x0d49, 4}, + {0x0d50, 0x0d53, 1}, + {0x0d64, 0x0d65, 1}, + {0x0d80, 0x0d84, 4}, + {0x0d97, 0x0d99, 1}, + {0x0db2, 0x0dbc, 10}, + {0x0dbe, 0x0dbf, 1}, + {0x0dc7, 0x0dc9, 1}, + {0x0dcb, 0x0dce, 1}, + {0x0dd5, 0x0dd7, 2}, + {0x0de0, 0x0de5, 1}, + {0x0df0, 0x0df1, 1}, + {0x0df5, 0x0e00, 1}, + {0x0e3b, 0x0e3e, 1}, + {0x0e5c, 0x0e80, 1}, + {0x0e83, 0x0e85, 2}, + {0x0e8b, 0x0ea4, 25}, + {0x0ea6, 0x0ebe, 24}, + {0x0ebf, 0x0ec5, 6}, + {0x0ec7, 0x0ecf, 8}, + {0x0eda, 0x0edb, 1}, + {0x0ee0, 0x0eff, 1}, + {0x0f48, 0x0f6d, 37}, + {0x0f6e, 0x0f70, 1}, + {0x0f98, 0x0fbd, 37}, + {0x0fcd, 0x0fdb, 14}, + {0x0fdc, 0x0fff, 1}, + {0x10c6, 0x10c8, 2}, + {0x10c9, 0x10cc, 1}, + {0x10ce, 0x10cf, 1}, + {0x1249, 0x124e, 5}, + {0x124f, 0x1257, 8}, + {0x1259, 0x125e, 5}, + {0x125f, 0x1289, 42}, + {0x128e, 0x128f, 1}, + {0x12b1, 0x12b6, 5}, + {0x12b7, 0x12bf, 8}, + {0x12c1, 0x12c6, 5}, + {0x12c7, 0x12d7, 16}, + {0x1311, 0x1316, 5}, + {0x1317, 0x135b, 68}, + {0x135c, 0x137d, 33}, + {0x137e, 0x137f, 1}, + {0x139a, 0x139f, 1}, + {0x13f6, 0x13f7, 1}, + {0x13fe, 0x13ff, 1}, + {0x169d, 0x169f, 1}, + {0x16f9, 0x16ff, 1}, + {0x1716, 0x171e, 1}, + {0x1737, 0x173f, 1}, + {0x1754, 0x175f, 1}, + {0x176d, 0x1771, 4}, + {0x1774, 0x177f, 1}, + {0x17de, 0x17df, 1}, + {0x17ea, 0x17ef, 1}, + {0x17fa, 0x17ff, 1}, + {0x180e, 0x181a, 12}, + {0x181b, 0x181f, 1}, + {0x1879, 0x187f, 1}, + {0x18ab, 0x18af, 1}, + {0x18f6, 0x18ff, 1}, + {0x191f, 0x192c, 13}, + {0x192d, 0x192f, 1}, + {0x193c, 0x193f, 1}, + {0x1941, 0x1943, 1}, + {0x196e, 0x196f, 1}, + {0x1975, 0x197f, 1}, + {0x19ac, 0x19af, 1}, + {0x19ca, 0x19cf, 1}, + {0x19db, 0x19dd, 1}, + {0x1a1c, 0x1a1d, 1}, + {0x1a5f, 0x1a7d, 30}, + {0x1a7e, 0x1a8a, 12}, + {0x1a8b, 0x1a8f, 1}, + {0x1a9a, 0x1a9f, 1}, + {0x1aae, 0x1aaf, 1}, + {0x1acf, 0x1aff, 1}, + {0x1b4d, 0x1b4f, 1}, + {0x1b7f, 0x1bf4, 117}, + {0x1bf5, 0x1bfb, 1}, + {0x1c38, 0x1c3a, 1}, + {0x1c4a, 0x1c4c, 1}, + {0x1c89, 0x1c8f, 1}, + {0x1cbb, 0x1cbc, 1}, + {0x1cc8, 0x1ccf, 1}, + {0x1cfb, 0x1cff, 1}, + {0x1f16, 0x1f17, 1}, + {0x1f1e, 0x1f1f, 1}, + {0x1f46, 0x1f47, 1}, + {0x1f4e, 0x1f4f, 1}, + {0x1f58, 0x1f5e, 2}, + {0x1f7e, 0x1f7f, 1}, + {0x1fb5, 0x1fc5, 16}, + {0x1fd4, 0x1fd5, 1}, + {0x1fdc, 0x1ff0, 20}, + {0x1ff1, 0x1ff5, 4}, + {0x1fff, 0x200b, 12}, {0x200c, 0x200f, 1}, {0x202a, 0x202e, 1}, - {0x2060, 0x2064, 1}, - {0x2066, 0x206f, 1}, - {0xd800, 0xf8ff, 1}, - {0xfeff, 0xfff9, 250}, - {0xfffa, 0xfffb, 1}, + {0x2060, 0x206f, 1}, + {0x2072, 0x2073, 1}, + {0x208f, 0x209d, 14}, + {0x209e, 0x209f, 1}, + {0x20c1, 0x20cf, 1}, + {0x20f1, 0x20ff, 1}, + {0x218c, 0x218f, 1}, + {0x2427, 0x243f, 1}, + {0x244b, 0x245f, 1}, + {0x2b74, 0x2b75, 1}, + {0x2b96, 0x2cf4, 350}, + {0x2cf5, 0x2cf8, 1}, + {0x2d26, 0x2d28, 2}, + {0x2d29, 0x2d2c, 1}, + {0x2d2e, 0x2d2f, 1}, + {0x2d68, 0x2d6e, 1}, + {0x2d71, 0x2d7e, 1}, + {0x2d97, 0x2d9f, 1}, + {0x2da7, 0x2ddf, 8}, + {0x2e5e, 0x2e7f, 1}, + {0x2e9a, 0x2ef4, 90}, + {0x2ef5, 0x2eff, 1}, + {0x2fd6, 0x2fef, 1}, + {0x2ffc, 0x2fff, 1}, + {0x3040, 0x3097, 87}, + {0x3098, 0x3100, 104}, + {0x3101, 0x3104, 1}, + {0x3130, 0x318f, 95}, + {0x31e4, 0x31ef, 1}, + {0x321f, 0xa48d, 29294}, + {0xa48e, 0xa48f, 1}, + {0xa4c7, 0xa4cf, 1}, + {0xa62c, 0xa63f, 1}, + {0xa6f8, 0xa6ff, 1}, + {0xa7cb, 0xa7cf, 1}, + {0xa7d2, 0xa7d4, 2}, + {0xa7da, 0xa7f1, 1}, + {0xa82d, 0xa82f, 1}, + {0xa83a, 0xa83f, 1}, + {0xa878, 0xa87f, 1}, + {0xa8c6, 0xa8cd, 1}, + {0xa8da, 0xa8df, 1}, + {0xa954, 0xa95e, 1}, + {0xa97d, 0xa97f, 1}, + {0xa9ce, 0xa9da, 12}, + {0xa9db, 0xa9dd, 1}, + {0xa9ff, 0xaa37, 56}, + {0xaa38, 0xaa3f, 1}, + {0xaa4e, 0xaa4f, 1}, + {0xaa5a, 0xaa5b, 1}, + {0xaac3, 0xaada, 1}, + {0xaaf7, 0xab00, 1}, + {0xab07, 0xab08, 1}, + {0xab0f, 0xab10, 1}, + {0xab17, 0xab1f, 1}, + {0xab27, 0xab2f, 8}, + {0xab6c, 0xab6f, 1}, + {0xabee, 0xabef, 1}, + {0xabfa, 0xabff, 1}, + {0xd7a4, 0xd7af, 1}, + {0xd7c7, 0xd7ca, 1}, + {0xd7fc, 0xf8ff, 1}, + {0xfa6e, 0xfa6f, 1}, + {0xfada, 0xfaff, 1}, + {0xfb07, 0xfb12, 1}, + {0xfb18, 0xfb1c, 1}, + {0xfb37, 0xfb3d, 6}, + {0xfb3f, 0xfb45, 3}, + {0xfbc3, 0xfbd2, 1}, + {0xfd90, 0xfd91, 1}, + {0xfdc8, 0xfdce, 1}, + {0xfdd0, 0xfdef, 1}, + {0xfe1a, 0xfe1f, 1}, + {0xfe53, 0xfe67, 20}, + {0xfe6c, 0xfe6f, 1}, + {0xfe75, 0xfefd, 136}, + {0xfefe, 0xff00, 1}, + {0xffbf, 0xffc1, 1}, + {0xffc8, 0xffc9, 1}, + {0xffd0, 0xffd1, 1}, + {0xffd8, 0xffd9, 1}, + {0xffdd, 0xffdf, 1}, + {0xffe7, 0xffef, 8}, + {0xfff0, 0xfffb, 1}, + {0xfffe, 0xffff, 1}, }, R32: []Range32{ - {0x110bd, 0x110cd, 16}, + {0x1000c, 0x10027, 27}, + {0x1003b, 0x1003e, 3}, + {0x1004e, 0x1004f, 1}, + {0x1005e, 0x1007f, 1}, + {0x100fb, 0x100ff, 1}, + {0x10103, 0x10106, 1}, + {0x10134, 0x10136, 1}, + {0x1018f, 0x1019d, 14}, + {0x1019e, 0x1019f, 1}, + {0x101a1, 0x101cf, 1}, + {0x101fe, 0x1027f, 1}, + {0x1029d, 0x1029f, 1}, + {0x102d1, 0x102df, 1}, + {0x102fc, 0x102ff, 1}, + {0x10324, 0x1032c, 1}, + {0x1034b, 0x1034f, 1}, + {0x1037b, 0x1037f, 1}, + {0x1039e, 0x103c4, 38}, + {0x103c5, 0x103c7, 1}, + {0x103d6, 0x103ff, 1}, + {0x1049e, 0x1049f, 1}, + {0x104aa, 0x104af, 1}, + {0x104d4, 0x104d7, 1}, + {0x104fc, 0x104ff, 1}, + {0x10528, 0x1052f, 1}, + {0x10564, 0x1056e, 1}, + {0x1057b, 0x1058b, 16}, + {0x10593, 0x10596, 3}, + {0x105a2, 0x105b2, 16}, + {0x105ba, 0x105bd, 3}, + {0x105be, 0x105ff, 1}, + {0x10737, 0x1073f, 1}, + {0x10756, 0x1075f, 1}, + {0x10768, 0x1077f, 1}, + {0x10786, 0x107b1, 43}, + {0x107bb, 0x107ff, 1}, + {0x10806, 0x10807, 1}, + {0x10809, 0x10836, 45}, + {0x10839, 0x1083b, 1}, + {0x1083d, 0x1083e, 1}, + {0x10856, 0x1089f, 73}, + {0x108a0, 0x108a6, 1}, + {0x108b0, 0x108df, 1}, + {0x108f3, 0x108f6, 3}, + {0x108f7, 0x108fa, 1}, + {0x1091c, 0x1091e, 1}, + {0x1093a, 0x1093e, 1}, + {0x10940, 0x1097f, 1}, + {0x109b8, 0x109bb, 1}, + {0x109d0, 0x109d1, 1}, + {0x10a04, 0x10a07, 3}, + {0x10a08, 0x10a0b, 1}, + {0x10a14, 0x10a18, 4}, + {0x10a36, 0x10a37, 1}, + {0x10a3b, 0x10a3e, 1}, + {0x10a49, 0x10a4f, 1}, + {0x10a59, 0x10a5f, 1}, + {0x10aa0, 0x10abf, 1}, + {0x10ae7, 0x10aea, 1}, + {0x10af7, 0x10aff, 1}, + {0x10b36, 0x10b38, 1}, + {0x10b56, 0x10b57, 1}, + {0x10b73, 0x10b77, 1}, + {0x10b92, 0x10b98, 1}, + {0x10b9d, 0x10ba8, 1}, + {0x10bb0, 0x10bff, 1}, + {0x10c49, 0x10c7f, 1}, + {0x10cb3, 0x10cbf, 1}, + {0x10cf3, 0x10cf9, 1}, + {0x10d28, 0x10d2f, 1}, + {0x10d3a, 0x10e5f, 1}, + {0x10e7f, 0x10eaa, 43}, + {0x10eae, 0x10eaf, 1}, + {0x10eb2, 0x10efc, 1}, + {0x10f28, 0x10f2f, 1}, + {0x10f5a, 0x10f6f, 1}, + {0x10f8a, 0x10faf, 1}, + {0x10fcc, 0x10fdf, 1}, + {0x10ff7, 0x10fff, 1}, + {0x1104e, 0x11051, 1}, + {0x11076, 0x1107e, 1}, + {0x110bd, 0x110c3, 6}, + {0x110c4, 0x110cf, 1}, + {0x110e9, 0x110ef, 1}, + {0x110fa, 0x110ff, 1}, + {0x11135, 0x11148, 19}, + {0x11149, 0x1114f, 1}, + {0x11177, 0x1117f, 1}, + {0x111e0, 0x111f5, 21}, + {0x111f6, 0x111ff, 1}, + {0x11212, 0x11242, 48}, + {0x11243, 0x1127f, 1}, + {0x11287, 0x11289, 2}, + {0x1128e, 0x1129e, 16}, + {0x112aa, 0x112af, 1}, + {0x112eb, 0x112ef, 1}, + {0x112fa, 0x112ff, 1}, + {0x11304, 0x1130d, 9}, + {0x1130e, 0x11311, 3}, + {0x11312, 0x11329, 23}, + {0x11331, 0x11334, 3}, + {0x1133a, 0x11345, 11}, + {0x11346, 0x11349, 3}, + {0x1134a, 0x1134e, 4}, + {0x1134f, 0x11351, 2}, + {0x11352, 0x11356, 1}, + {0x11358, 0x1135c, 1}, + {0x11364, 0x11365, 1}, + {0x1136d, 0x1136f, 1}, + {0x11375, 0x113ff, 1}, + {0x1145c, 0x11462, 6}, + {0x11463, 0x1147f, 1}, + {0x114c8, 0x114cf, 1}, + {0x114da, 0x1157f, 1}, + {0x115b6, 0x115b7, 1}, + {0x115de, 0x115ff, 1}, + {0x11645, 0x1164f, 1}, + {0x1165a, 0x1165f, 1}, + {0x1166d, 0x1167f, 1}, + {0x116ba, 0x116bf, 1}, + {0x116ca, 0x116ff, 1}, + {0x1171b, 0x1171c, 1}, + {0x1172c, 0x1172f, 1}, + {0x11747, 0x117ff, 1}, + {0x1183c, 0x1189f, 1}, + {0x118f3, 0x118fe, 1}, + {0x11907, 0x11908, 1}, + {0x1190a, 0x1190b, 1}, + {0x11914, 0x11917, 3}, + {0x11936, 0x11939, 3}, + {0x1193a, 0x11947, 13}, + {0x11948, 0x1194f, 1}, + {0x1195a, 0x1199f, 1}, + {0x119a8, 0x119a9, 1}, + {0x119d8, 0x119d9, 1}, + {0x119e5, 0x119ff, 1}, + {0x11a48, 0x11a4f, 1}, + {0x11aa3, 0x11aaf, 1}, + {0x11af9, 0x11aff, 1}, + {0x11b0a, 0x11bff, 1}, + {0x11c09, 0x11c37, 46}, + {0x11c46, 0x11c4f, 1}, + {0x11c6d, 0x11c6f, 1}, + {0x11c90, 0x11c91, 1}, + {0x11ca8, 0x11cb7, 15}, + {0x11cb8, 0x11cff, 1}, + {0x11d07, 0x11d0a, 3}, + {0x11d37, 0x11d39, 1}, + {0x11d3b, 0x11d3e, 3}, + {0x11d48, 0x11d4f, 1}, + {0x11d5a, 0x11d5f, 1}, + {0x11d66, 0x11d69, 3}, + {0x11d8f, 0x11d92, 3}, + {0x11d99, 0x11d9f, 1}, + {0x11daa, 0x11edf, 1}, + {0x11ef9, 0x11eff, 1}, + {0x11f11, 0x11f3b, 42}, + {0x11f3c, 0x11f3d, 1}, + {0x11f5a, 0x11faf, 1}, + {0x11fb1, 0x11fbf, 1}, + {0x11ff2, 0x11ffe, 1}, + {0x1239a, 0x123ff, 1}, + {0x1246f, 0x12475, 6}, + {0x12476, 0x1247f, 1}, + {0x12544, 0x12f8f, 1}, + {0x12ff3, 0x12fff, 1}, {0x13430, 0x1343f, 1}, - {0x1bca0, 0x1bca3, 1}, + {0x13456, 0x143ff, 1}, + {0x14647, 0x167ff, 1}, + {0x16a39, 0x16a3f, 1}, + {0x16a5f, 0x16a6a, 11}, + {0x16a6b, 0x16a6d, 1}, + {0x16abf, 0x16aca, 11}, + {0x16acb, 0x16acf, 1}, + {0x16aee, 0x16aef, 1}, + {0x16af6, 0x16aff, 1}, + {0x16b46, 0x16b4f, 1}, + {0x16b5a, 0x16b62, 8}, + {0x16b78, 0x16b7c, 1}, + {0x16b90, 0x16e3f, 1}, + {0x16e9b, 0x16eff, 1}, + {0x16f4b, 0x16f4e, 1}, + {0x16f88, 0x16f8e, 1}, + {0x16fa0, 0x16fdf, 1}, + {0x16fe5, 0x16fef, 1}, + {0x16ff2, 0x16fff, 1}, + {0x187f8, 0x187ff, 1}, + {0x18cd6, 0x18cff, 1}, + {0x18d09, 0x1afef, 1}, + {0x1aff4, 0x1affc, 8}, + {0x1afff, 0x1b123, 292}, + {0x1b124, 0x1b131, 1}, + {0x1b133, 0x1b14f, 1}, + {0x1b153, 0x1b154, 1}, + {0x1b156, 0x1b163, 1}, + {0x1b168, 0x1b16f, 1}, + {0x1b2fc, 0x1bbff, 1}, + {0x1bc6b, 0x1bc6f, 1}, + {0x1bc7d, 0x1bc7f, 1}, + {0x1bc89, 0x1bc8f, 1}, + {0x1bc9a, 0x1bc9b, 1}, + {0x1bca0, 0x1ceff, 1}, + {0x1cf2e, 0x1cf2f, 1}, + {0x1cf47, 0x1cf4f, 1}, + {0x1cfc4, 0x1cfff, 1}, + {0x1d0f6, 0x1d0ff, 1}, + {0x1d127, 0x1d128, 1}, {0x1d173, 0x1d17a, 1}, - {0xe0001, 0xe0020, 31}, - {0xe0021, 0xe007f, 1}, - {0xf0000, 0xffffd, 1}, - {0x100000, 0x10fffd, 1}, + {0x1d1eb, 0x1d1ff, 1}, + {0x1d246, 0x1d2bf, 1}, + {0x1d2d4, 0x1d2df, 1}, + {0x1d2f4, 0x1d2ff, 1}, + {0x1d357, 0x1d35f, 1}, + {0x1d379, 0x1d3ff, 1}, + {0x1d455, 0x1d49d, 72}, + {0x1d4a0, 0x1d4a1, 1}, + {0x1d4a3, 0x1d4a4, 1}, + {0x1d4a7, 0x1d4a8, 1}, + {0x1d4ad, 0x1d4ba, 13}, + {0x1d4bc, 0x1d4c4, 8}, + {0x1d506, 0x1d50b, 5}, + {0x1d50c, 0x1d515, 9}, + {0x1d51d, 0x1d53a, 29}, + {0x1d53f, 0x1d545, 6}, + {0x1d547, 0x1d549, 1}, + {0x1d551, 0x1d6a6, 341}, + {0x1d6a7, 0x1d7cc, 293}, + {0x1d7cd, 0x1da8c, 703}, + {0x1da8d, 0x1da9a, 1}, + {0x1daa0, 0x1dab0, 16}, + {0x1dab1, 0x1deff, 1}, + {0x1df1f, 0x1df24, 1}, + {0x1df2b, 0x1dfff, 1}, + {0x1e007, 0x1e019, 18}, + {0x1e01a, 0x1e022, 8}, + {0x1e025, 0x1e02b, 6}, + {0x1e02c, 0x1e02f, 1}, + {0x1e06e, 0x1e08e, 1}, + {0x1e090, 0x1e0ff, 1}, + {0x1e12d, 0x1e12f, 1}, + {0x1e13e, 0x1e13f, 1}, + {0x1e14a, 0x1e14d, 1}, + {0x1e150, 0x1e28f, 1}, + {0x1e2af, 0x1e2bf, 1}, + {0x1e2fa, 0x1e2fe, 1}, + {0x1e300, 0x1e4cf, 1}, + {0x1e4fa, 0x1e7df, 1}, + {0x1e7e7, 0x1e7ec, 5}, + {0x1e7ef, 0x1e7ff, 16}, + {0x1e8c5, 0x1e8c6, 1}, + {0x1e8d7, 0x1e8ff, 1}, + {0x1e94c, 0x1e94f, 1}, + {0x1e95a, 0x1e95d, 1}, + {0x1e960, 0x1ec70, 1}, + {0x1ecb5, 0x1ed00, 1}, + {0x1ed3e, 0x1edff, 1}, + {0x1ee04, 0x1ee20, 28}, + {0x1ee23, 0x1ee25, 2}, + {0x1ee26, 0x1ee28, 2}, + {0x1ee33, 0x1ee38, 5}, + {0x1ee3a, 0x1ee3c, 2}, + {0x1ee3d, 0x1ee41, 1}, + {0x1ee43, 0x1ee46, 1}, + {0x1ee48, 0x1ee4c, 2}, + {0x1ee50, 0x1ee53, 3}, + {0x1ee55, 0x1ee56, 1}, + {0x1ee58, 0x1ee60, 2}, + {0x1ee63, 0x1ee65, 2}, + {0x1ee66, 0x1ee6b, 5}, + {0x1ee73, 0x1ee7d, 5}, + {0x1ee7f, 0x1ee8a, 11}, + {0x1ee9c, 0x1eea0, 1}, + {0x1eea4, 0x1eeaa, 6}, + {0x1eebc, 0x1eeef, 1}, + {0x1eef2, 0x1efff, 1}, + {0x1f02c, 0x1f02f, 1}, + {0x1f094, 0x1f09f, 1}, + {0x1f0af, 0x1f0b0, 1}, + {0x1f0c0, 0x1f0d0, 16}, + {0x1f0f6, 0x1f0ff, 1}, + {0x1f1ae, 0x1f1e5, 1}, + {0x1f203, 0x1f20f, 1}, + {0x1f23c, 0x1f23f, 1}, + {0x1f249, 0x1f24f, 1}, + {0x1f252, 0x1f25f, 1}, + {0x1f266, 0x1f2ff, 1}, + {0x1f6d8, 0x1f6db, 1}, + {0x1f6ed, 0x1f6ef, 1}, + {0x1f6fd, 0x1f6ff, 1}, + {0x1f777, 0x1f77a, 1}, + {0x1f7da, 0x1f7df, 1}, + {0x1f7ec, 0x1f7ef, 1}, + {0x1f7f1, 0x1f7ff, 1}, + {0x1f80c, 0x1f80f, 1}, + {0x1f848, 0x1f84f, 1}, + {0x1f85a, 0x1f85f, 1}, + {0x1f888, 0x1f88f, 1}, + {0x1f8ae, 0x1f8af, 1}, + {0x1f8b2, 0x1f8ff, 1}, + {0x1fa54, 0x1fa5f, 1}, + {0x1fa6e, 0x1fa6f, 1}, + {0x1fa7d, 0x1fa7f, 1}, + {0x1fa89, 0x1fa8f, 1}, + {0x1fabe, 0x1fac6, 8}, + {0x1fac7, 0x1facd, 1}, + {0x1fadc, 0x1fadf, 1}, + {0x1fae9, 0x1faef, 1}, + {0x1faf9, 0x1faff, 1}, + {0x1fb93, 0x1fbcb, 56}, + {0x1fbcc, 0x1fbef, 1}, + {0x1fbfa, 0x1ffff, 1}, + {0x2a6e0, 0x2a6ff, 1}, + {0x2b73a, 0x2b73f, 1}, + {0x2b81e, 0x2b81f, 1}, + {0x2cea2, 0x2ceaf, 1}, + {0x2ebe1, 0x2f7ff, 1}, + {0x2fa1e, 0x2ffff, 1}, + {0x3134b, 0x3134f, 1}, + {0x323b0, 0xe00ff, 1}, + {0xe01f0, 0x10ffff, 1}, }, LatinOffset: 2, } @@ -109,6 +743,617 @@ var _Cf = &RangeTable{ }, } +var _Cn = &RangeTable{ + R16: []Range16{ + {0x0378, 0x0379, 1}, + {0x0380, 0x0383, 1}, + {0x038b, 0x038d, 2}, + {0x03a2, 0x0530, 398}, + {0x0557, 0x0558, 1}, + {0x058b, 0x058c, 1}, + {0x0590, 0x05c8, 56}, + {0x05c9, 0x05cf, 1}, + {0x05eb, 0x05ee, 1}, + {0x05f5, 0x05ff, 1}, + {0x070e, 0x074b, 61}, + {0x074c, 0x07b2, 102}, + {0x07b3, 0x07bf, 1}, + {0x07fb, 0x07fc, 1}, + {0x082e, 0x082f, 1}, + {0x083f, 0x085c, 29}, + {0x085d, 0x085f, 2}, + {0x086b, 0x086f, 1}, + {0x088f, 0x0892, 3}, + {0x0893, 0x0897, 1}, + {0x0984, 0x098d, 9}, + {0x098e, 0x0991, 3}, + {0x0992, 0x09a9, 23}, + {0x09b1, 0x09b3, 2}, + {0x09b4, 0x09b5, 1}, + {0x09ba, 0x09bb, 1}, + {0x09c5, 0x09c6, 1}, + {0x09c9, 0x09ca, 1}, + {0x09cf, 0x09d6, 1}, + {0x09d8, 0x09db, 1}, + {0x09de, 0x09e4, 6}, + {0x09e5, 0x09ff, 26}, + {0x0a00, 0x0a04, 4}, + {0x0a0b, 0x0a0e, 1}, + {0x0a11, 0x0a12, 1}, + {0x0a29, 0x0a31, 8}, + {0x0a34, 0x0a3a, 3}, + {0x0a3b, 0x0a3d, 2}, + {0x0a43, 0x0a46, 1}, + {0x0a49, 0x0a4a, 1}, + {0x0a4e, 0x0a50, 1}, + {0x0a52, 0x0a58, 1}, + {0x0a5d, 0x0a5f, 2}, + {0x0a60, 0x0a65, 1}, + {0x0a77, 0x0a80, 1}, + {0x0a84, 0x0a8e, 10}, + {0x0a92, 0x0aa9, 23}, + {0x0ab1, 0x0ab4, 3}, + {0x0aba, 0x0abb, 1}, + {0x0ac6, 0x0ace, 4}, + {0x0acf, 0x0ad1, 2}, + {0x0ad2, 0x0adf, 1}, + {0x0ae4, 0x0ae5, 1}, + {0x0af2, 0x0af8, 1}, + {0x0b00, 0x0b04, 4}, + {0x0b0d, 0x0b0e, 1}, + {0x0b11, 0x0b12, 1}, + {0x0b29, 0x0b31, 8}, + {0x0b34, 0x0b3a, 6}, + {0x0b3b, 0x0b45, 10}, + {0x0b46, 0x0b49, 3}, + {0x0b4a, 0x0b4e, 4}, + {0x0b4f, 0x0b54, 1}, + {0x0b58, 0x0b5b, 1}, + {0x0b5e, 0x0b64, 6}, + {0x0b65, 0x0b78, 19}, + {0x0b79, 0x0b81, 1}, + {0x0b84, 0x0b8b, 7}, + {0x0b8c, 0x0b8d, 1}, + {0x0b91, 0x0b96, 5}, + {0x0b97, 0x0b98, 1}, + {0x0b9b, 0x0b9d, 2}, + {0x0ba0, 0x0ba2, 1}, + {0x0ba5, 0x0ba7, 1}, + {0x0bab, 0x0bad, 1}, + {0x0bba, 0x0bbd, 1}, + {0x0bc3, 0x0bc5, 1}, + {0x0bc9, 0x0bce, 5}, + {0x0bcf, 0x0bd1, 2}, + {0x0bd2, 0x0bd6, 1}, + {0x0bd8, 0x0be5, 1}, + {0x0bfb, 0x0bff, 1}, + {0x0c0d, 0x0c11, 4}, + {0x0c29, 0x0c3a, 17}, + {0x0c3b, 0x0c45, 10}, + {0x0c49, 0x0c4e, 5}, + {0x0c4f, 0x0c54, 1}, + {0x0c57, 0x0c5b, 4}, + {0x0c5c, 0x0c5e, 2}, + {0x0c5f, 0x0c64, 5}, + {0x0c65, 0x0c70, 11}, + {0x0c71, 0x0c76, 1}, + {0x0c8d, 0x0c91, 4}, + {0x0ca9, 0x0cb4, 11}, + {0x0cba, 0x0cbb, 1}, + {0x0cc5, 0x0cc9, 4}, + {0x0cce, 0x0cd4, 1}, + {0x0cd7, 0x0cdc, 1}, + {0x0cdf, 0x0ce4, 5}, + {0x0ce5, 0x0cf0, 11}, + {0x0cf4, 0x0cff, 1}, + {0x0d0d, 0x0d11, 4}, + {0x0d45, 0x0d49, 4}, + {0x0d50, 0x0d53, 1}, + {0x0d64, 0x0d65, 1}, + {0x0d80, 0x0d84, 4}, + {0x0d97, 0x0d99, 1}, + {0x0db2, 0x0dbc, 10}, + {0x0dbe, 0x0dbf, 1}, + {0x0dc7, 0x0dc9, 1}, + {0x0dcb, 0x0dce, 1}, + {0x0dd5, 0x0dd7, 2}, + {0x0de0, 0x0de5, 1}, + {0x0df0, 0x0df1, 1}, + {0x0df5, 0x0e00, 1}, + {0x0e3b, 0x0e3e, 1}, + {0x0e5c, 0x0e80, 1}, + {0x0e83, 0x0e85, 2}, + {0x0e8b, 0x0ea4, 25}, + {0x0ea6, 0x0ebe, 24}, + {0x0ebf, 0x0ec5, 6}, + {0x0ec7, 0x0ecf, 8}, + {0x0eda, 0x0edb, 1}, + {0x0ee0, 0x0eff, 1}, + {0x0f48, 0x0f6d, 37}, + {0x0f6e, 0x0f70, 1}, + {0x0f98, 0x0fbd, 37}, + {0x0fcd, 0x0fdb, 14}, + {0x0fdc, 0x0fff, 1}, + {0x10c6, 0x10c8, 2}, + {0x10c9, 0x10cc, 1}, + {0x10ce, 0x10cf, 1}, + {0x1249, 0x124e, 5}, + {0x124f, 0x1257, 8}, + {0x1259, 0x125e, 5}, + {0x125f, 0x1289, 42}, + {0x128e, 0x128f, 1}, + {0x12b1, 0x12b6, 5}, + {0x12b7, 0x12bf, 8}, + {0x12c1, 0x12c6, 5}, + {0x12c7, 0x12d7, 16}, + {0x1311, 0x1316, 5}, + {0x1317, 0x135b, 68}, + {0x135c, 0x137d, 33}, + {0x137e, 0x137f, 1}, + {0x139a, 0x139f, 1}, + {0x13f6, 0x13f7, 1}, + {0x13fe, 0x13ff, 1}, + {0x169d, 0x169f, 1}, + {0x16f9, 0x16ff, 1}, + {0x1716, 0x171e, 1}, + {0x1737, 0x173f, 1}, + {0x1754, 0x175f, 1}, + {0x176d, 0x1771, 4}, + {0x1774, 0x177f, 1}, + {0x17de, 0x17df, 1}, + {0x17ea, 0x17ef, 1}, + {0x17fa, 0x17ff, 1}, + {0x181a, 0x181f, 1}, + {0x1879, 0x187f, 1}, + {0x18ab, 0x18af, 1}, + {0x18f6, 0x18ff, 1}, + {0x191f, 0x192c, 13}, + {0x192d, 0x192f, 1}, + {0x193c, 0x193f, 1}, + {0x1941, 0x1943, 1}, + {0x196e, 0x196f, 1}, + {0x1975, 0x197f, 1}, + {0x19ac, 0x19af, 1}, + {0x19ca, 0x19cf, 1}, + {0x19db, 0x19dd, 1}, + {0x1a1c, 0x1a1d, 1}, + {0x1a5f, 0x1a7d, 30}, + {0x1a7e, 0x1a8a, 12}, + {0x1a8b, 0x1a8f, 1}, + {0x1a9a, 0x1a9f, 1}, + {0x1aae, 0x1aaf, 1}, + {0x1acf, 0x1aff, 1}, + {0x1b4d, 0x1b4f, 1}, + {0x1b7f, 0x1bf4, 117}, + {0x1bf5, 0x1bfb, 1}, + {0x1c38, 0x1c3a, 1}, + {0x1c4a, 0x1c4c, 1}, + {0x1c89, 0x1c8f, 1}, + {0x1cbb, 0x1cbc, 1}, + {0x1cc8, 0x1ccf, 1}, + {0x1cfb, 0x1cff, 1}, + {0x1f16, 0x1f17, 1}, + {0x1f1e, 0x1f1f, 1}, + {0x1f46, 0x1f47, 1}, + {0x1f4e, 0x1f4f, 1}, + {0x1f58, 0x1f5e, 2}, + {0x1f7e, 0x1f7f, 1}, + {0x1fb5, 0x1fc5, 16}, + {0x1fd4, 0x1fd5, 1}, + {0x1fdc, 0x1ff0, 20}, + {0x1ff1, 0x1ff5, 4}, + {0x1fff, 0x2065, 102}, + {0x2072, 0x2073, 1}, + {0x208f, 0x209d, 14}, + {0x209e, 0x209f, 1}, + {0x20c1, 0x20cf, 1}, + {0x20f1, 0x20ff, 1}, + {0x218c, 0x218f, 1}, + {0x2427, 0x243f, 1}, + {0x244b, 0x245f, 1}, + {0x2b74, 0x2b75, 1}, + {0x2b96, 0x2cf4, 350}, + {0x2cf5, 0x2cf8, 1}, + {0x2d26, 0x2d28, 2}, + {0x2d29, 0x2d2c, 1}, + {0x2d2e, 0x2d2f, 1}, + {0x2d68, 0x2d6e, 1}, + {0x2d71, 0x2d7e, 1}, + {0x2d97, 0x2d9f, 1}, + {0x2da7, 0x2ddf, 8}, + {0x2e5e, 0x2e7f, 1}, + {0x2e9a, 0x2ef4, 90}, + {0x2ef5, 0x2eff, 1}, + {0x2fd6, 0x2fef, 1}, + {0x2ffc, 0x2fff, 1}, + {0x3040, 0x3097, 87}, + {0x3098, 0x3100, 104}, + {0x3101, 0x3104, 1}, + {0x3130, 0x318f, 95}, + {0x31e4, 0x31ef, 1}, + {0x321f, 0xa48d, 29294}, + {0xa48e, 0xa48f, 1}, + {0xa4c7, 0xa4cf, 1}, + {0xa62c, 0xa63f, 1}, + {0xa6f8, 0xa6ff, 1}, + {0xa7cb, 0xa7cf, 1}, + {0xa7d2, 0xa7d4, 2}, + {0xa7da, 0xa7f1, 1}, + {0xa82d, 0xa82f, 1}, + {0xa83a, 0xa83f, 1}, + {0xa878, 0xa87f, 1}, + {0xa8c6, 0xa8cd, 1}, + {0xa8da, 0xa8df, 1}, + {0xa954, 0xa95e, 1}, + {0xa97d, 0xa97f, 1}, + {0xa9ce, 0xa9da, 12}, + {0xa9db, 0xa9dd, 1}, + {0xa9ff, 0xaa37, 56}, + {0xaa38, 0xaa3f, 1}, + {0xaa4e, 0xaa4f, 1}, + {0xaa5a, 0xaa5b, 1}, + {0xaac3, 0xaada, 1}, + {0xaaf7, 0xab00, 1}, + {0xab07, 0xab08, 1}, + {0xab0f, 0xab10, 1}, + {0xab17, 0xab1f, 1}, + {0xab27, 0xab2f, 8}, + {0xab6c, 0xab6f, 1}, + {0xabee, 0xabef, 1}, + {0xabfa, 0xabff, 1}, + {0xd7a4, 0xd7af, 1}, + {0xd7c7, 0xd7ca, 1}, + {0xd7fc, 0xd7ff, 1}, + {0xfa6e, 0xfa6f, 1}, + {0xfada, 0xfaff, 1}, + {0xfb07, 0xfb12, 1}, + {0xfb18, 0xfb1c, 1}, + {0xfb37, 0xfb3d, 6}, + {0xfb3f, 0xfb45, 3}, + {0xfbc3, 0xfbd2, 1}, + {0xfd90, 0xfd91, 1}, + {0xfdc8, 0xfdce, 1}, + {0xfdd0, 0xfdef, 1}, + {0xfe1a, 0xfe1f, 1}, + {0xfe53, 0xfe67, 20}, + {0xfe6c, 0xfe6f, 1}, + {0xfe75, 0xfefd, 136}, + {0xfefe, 0xff00, 2}, + {0xffbf, 0xffc1, 1}, + {0xffc8, 0xffc9, 1}, + {0xffd0, 0xffd1, 1}, + {0xffd8, 0xffd9, 1}, + {0xffdd, 0xffdf, 1}, + {0xffe7, 0xffef, 8}, + {0xfff0, 0xfff8, 1}, + {0xfffe, 0xffff, 1}, + }, + R32: []Range32{ + {0x1000c, 0x10027, 27}, + {0x1003b, 0x1003e, 3}, + {0x1004e, 0x1004f, 1}, + {0x1005e, 0x1007f, 1}, + {0x100fb, 0x100ff, 1}, + {0x10103, 0x10106, 1}, + {0x10134, 0x10136, 1}, + {0x1018f, 0x1019d, 14}, + {0x1019e, 0x1019f, 1}, + {0x101a1, 0x101cf, 1}, + {0x101fe, 0x1027f, 1}, + {0x1029d, 0x1029f, 1}, + {0x102d1, 0x102df, 1}, + {0x102fc, 0x102ff, 1}, + {0x10324, 0x1032c, 1}, + {0x1034b, 0x1034f, 1}, + {0x1037b, 0x1037f, 1}, + {0x1039e, 0x103c4, 38}, + {0x103c5, 0x103c7, 1}, + {0x103d6, 0x103ff, 1}, + {0x1049e, 0x1049f, 1}, + {0x104aa, 0x104af, 1}, + {0x104d4, 0x104d7, 1}, + {0x104fc, 0x104ff, 1}, + {0x10528, 0x1052f, 1}, + {0x10564, 0x1056e, 1}, + {0x1057b, 0x1058b, 16}, + {0x10593, 0x10596, 3}, + {0x105a2, 0x105b2, 16}, + {0x105ba, 0x105bd, 3}, + {0x105be, 0x105ff, 1}, + {0x10737, 0x1073f, 1}, + {0x10756, 0x1075f, 1}, + {0x10768, 0x1077f, 1}, + {0x10786, 0x107b1, 43}, + {0x107bb, 0x107ff, 1}, + {0x10806, 0x10807, 1}, + {0x10809, 0x10836, 45}, + {0x10839, 0x1083b, 1}, + {0x1083d, 0x1083e, 1}, + {0x10856, 0x1089f, 73}, + {0x108a0, 0x108a6, 1}, + {0x108b0, 0x108df, 1}, + {0x108f3, 0x108f6, 3}, + {0x108f7, 0x108fa, 1}, + {0x1091c, 0x1091e, 1}, + {0x1093a, 0x1093e, 1}, + {0x10940, 0x1097f, 1}, + {0x109b8, 0x109bb, 1}, + {0x109d0, 0x109d1, 1}, + {0x10a04, 0x10a07, 3}, + {0x10a08, 0x10a0b, 1}, + {0x10a14, 0x10a18, 4}, + {0x10a36, 0x10a37, 1}, + {0x10a3b, 0x10a3e, 1}, + {0x10a49, 0x10a4f, 1}, + {0x10a59, 0x10a5f, 1}, + {0x10aa0, 0x10abf, 1}, + {0x10ae7, 0x10aea, 1}, + {0x10af7, 0x10aff, 1}, + {0x10b36, 0x10b38, 1}, + {0x10b56, 0x10b57, 1}, + {0x10b73, 0x10b77, 1}, + {0x10b92, 0x10b98, 1}, + {0x10b9d, 0x10ba8, 1}, + {0x10bb0, 0x10bff, 1}, + {0x10c49, 0x10c7f, 1}, + {0x10cb3, 0x10cbf, 1}, + {0x10cf3, 0x10cf9, 1}, + {0x10d28, 0x10d2f, 1}, + {0x10d3a, 0x10e5f, 1}, + {0x10e7f, 0x10eaa, 43}, + {0x10eae, 0x10eaf, 1}, + {0x10eb2, 0x10efc, 1}, + {0x10f28, 0x10f2f, 1}, + {0x10f5a, 0x10f6f, 1}, + {0x10f8a, 0x10faf, 1}, + {0x10fcc, 0x10fdf, 1}, + {0x10ff7, 0x10fff, 1}, + {0x1104e, 0x11051, 1}, + {0x11076, 0x1107e, 1}, + {0x110c3, 0x110cc, 1}, + {0x110ce, 0x110cf, 1}, + {0x110e9, 0x110ef, 1}, + {0x110fa, 0x110ff, 1}, + {0x11135, 0x11148, 19}, + {0x11149, 0x1114f, 1}, + {0x11177, 0x1117f, 1}, + {0x111e0, 0x111f5, 21}, + {0x111f6, 0x111ff, 1}, + {0x11212, 0x11242, 48}, + {0x11243, 0x1127f, 1}, + {0x11287, 0x11289, 2}, + {0x1128e, 0x1129e, 16}, + {0x112aa, 0x112af, 1}, + {0x112eb, 0x112ef, 1}, + {0x112fa, 0x112ff, 1}, + {0x11304, 0x1130d, 9}, + {0x1130e, 0x11311, 3}, + {0x11312, 0x11329, 23}, + {0x11331, 0x11334, 3}, + {0x1133a, 0x11345, 11}, + {0x11346, 0x11349, 3}, + {0x1134a, 0x1134e, 4}, + {0x1134f, 0x11351, 2}, + {0x11352, 0x11356, 1}, + {0x11358, 0x1135c, 1}, + {0x11364, 0x11365, 1}, + {0x1136d, 0x1136f, 1}, + {0x11375, 0x113ff, 1}, + {0x1145c, 0x11462, 6}, + {0x11463, 0x1147f, 1}, + {0x114c8, 0x114cf, 1}, + {0x114da, 0x1157f, 1}, + {0x115b6, 0x115b7, 1}, + {0x115de, 0x115ff, 1}, + {0x11645, 0x1164f, 1}, + {0x1165a, 0x1165f, 1}, + {0x1166d, 0x1167f, 1}, + {0x116ba, 0x116bf, 1}, + {0x116ca, 0x116ff, 1}, + {0x1171b, 0x1171c, 1}, + {0x1172c, 0x1172f, 1}, + {0x11747, 0x117ff, 1}, + {0x1183c, 0x1189f, 1}, + {0x118f3, 0x118fe, 1}, + {0x11907, 0x11908, 1}, + {0x1190a, 0x1190b, 1}, + {0x11914, 0x11917, 3}, + {0x11936, 0x11939, 3}, + {0x1193a, 0x11947, 13}, + {0x11948, 0x1194f, 1}, + {0x1195a, 0x1199f, 1}, + {0x119a8, 0x119a9, 1}, + {0x119d8, 0x119d9, 1}, + {0x119e5, 0x119ff, 1}, + {0x11a48, 0x11a4f, 1}, + {0x11aa3, 0x11aaf, 1}, + {0x11af9, 0x11aff, 1}, + {0x11b0a, 0x11bff, 1}, + {0x11c09, 0x11c37, 46}, + {0x11c46, 0x11c4f, 1}, + {0x11c6d, 0x11c6f, 1}, + {0x11c90, 0x11c91, 1}, + {0x11ca8, 0x11cb7, 15}, + {0x11cb8, 0x11cff, 1}, + {0x11d07, 0x11d0a, 3}, + {0x11d37, 0x11d39, 1}, + {0x11d3b, 0x11d3e, 3}, + {0x11d48, 0x11d4f, 1}, + {0x11d5a, 0x11d5f, 1}, + {0x11d66, 0x11d69, 3}, + {0x11d8f, 0x11d92, 3}, + {0x11d99, 0x11d9f, 1}, + {0x11daa, 0x11edf, 1}, + {0x11ef9, 0x11eff, 1}, + {0x11f11, 0x11f3b, 42}, + {0x11f3c, 0x11f3d, 1}, + {0x11f5a, 0x11faf, 1}, + {0x11fb1, 0x11fbf, 1}, + {0x11ff2, 0x11ffe, 1}, + {0x1239a, 0x123ff, 1}, + {0x1246f, 0x12475, 6}, + {0x12476, 0x1247f, 1}, + {0x12544, 0x12f8f, 1}, + {0x12ff3, 0x12fff, 1}, + {0x13456, 0x143ff, 1}, + {0x14647, 0x167ff, 1}, + {0x16a39, 0x16a3f, 1}, + {0x16a5f, 0x16a6a, 11}, + {0x16a6b, 0x16a6d, 1}, + {0x16abf, 0x16aca, 11}, + {0x16acb, 0x16acf, 1}, + {0x16aee, 0x16aef, 1}, + {0x16af6, 0x16aff, 1}, + {0x16b46, 0x16b4f, 1}, + {0x16b5a, 0x16b62, 8}, + {0x16b78, 0x16b7c, 1}, + {0x16b90, 0x16e3f, 1}, + {0x16e9b, 0x16eff, 1}, + {0x16f4b, 0x16f4e, 1}, + {0x16f88, 0x16f8e, 1}, + {0x16fa0, 0x16fdf, 1}, + {0x16fe5, 0x16fef, 1}, + {0x16ff2, 0x16fff, 1}, + {0x187f8, 0x187ff, 1}, + {0x18cd6, 0x18cff, 1}, + {0x18d09, 0x1afef, 1}, + {0x1aff4, 0x1affc, 8}, + {0x1afff, 0x1b123, 292}, + {0x1b124, 0x1b131, 1}, + {0x1b133, 0x1b14f, 1}, + {0x1b153, 0x1b154, 1}, + {0x1b156, 0x1b163, 1}, + {0x1b168, 0x1b16f, 1}, + {0x1b2fc, 0x1bbff, 1}, + {0x1bc6b, 0x1bc6f, 1}, + {0x1bc7d, 0x1bc7f, 1}, + {0x1bc89, 0x1bc8f, 1}, + {0x1bc9a, 0x1bc9b, 1}, + {0x1bca4, 0x1ceff, 1}, + {0x1cf2e, 0x1cf2f, 1}, + {0x1cf47, 0x1cf4f, 1}, + {0x1cfc4, 0x1cfff, 1}, + {0x1d0f6, 0x1d0ff, 1}, + {0x1d127, 0x1d128, 1}, + {0x1d1eb, 0x1d1ff, 1}, + {0x1d246, 0x1d2bf, 1}, + {0x1d2d4, 0x1d2df, 1}, + {0x1d2f4, 0x1d2ff, 1}, + {0x1d357, 0x1d35f, 1}, + {0x1d379, 0x1d3ff, 1}, + {0x1d455, 0x1d49d, 72}, + {0x1d4a0, 0x1d4a1, 1}, + {0x1d4a3, 0x1d4a4, 1}, + {0x1d4a7, 0x1d4a8, 1}, + {0x1d4ad, 0x1d4ba, 13}, + {0x1d4bc, 0x1d4c4, 8}, + {0x1d506, 0x1d50b, 5}, + {0x1d50c, 0x1d515, 9}, + {0x1d51d, 0x1d53a, 29}, + {0x1d53f, 0x1d545, 6}, + {0x1d547, 0x1d549, 1}, + {0x1d551, 0x1d6a6, 341}, + {0x1d6a7, 0x1d7cc, 293}, + {0x1d7cd, 0x1da8c, 703}, + {0x1da8d, 0x1da9a, 1}, + {0x1daa0, 0x1dab0, 16}, + {0x1dab1, 0x1deff, 1}, + {0x1df1f, 0x1df24, 1}, + {0x1df2b, 0x1dfff, 1}, + {0x1e007, 0x1e019, 18}, + {0x1e01a, 0x1e022, 8}, + {0x1e025, 0x1e02b, 6}, + {0x1e02c, 0x1e02f, 1}, + {0x1e06e, 0x1e08e, 1}, + {0x1e090, 0x1e0ff, 1}, + {0x1e12d, 0x1e12f, 1}, + {0x1e13e, 0x1e13f, 1}, + {0x1e14a, 0x1e14d, 1}, + {0x1e150, 0x1e28f, 1}, + {0x1e2af, 0x1e2bf, 1}, + {0x1e2fa, 0x1e2fe, 1}, + {0x1e300, 0x1e4cf, 1}, + {0x1e4fa, 0x1e7df, 1}, + {0x1e7e7, 0x1e7ec, 5}, + {0x1e7ef, 0x1e7ff, 16}, + {0x1e8c5, 0x1e8c6, 1}, + {0x1e8d7, 0x1e8ff, 1}, + {0x1e94c, 0x1e94f, 1}, + {0x1e95a, 0x1e95d, 1}, + {0x1e960, 0x1ec70, 1}, + {0x1ecb5, 0x1ed00, 1}, + {0x1ed3e, 0x1edff, 1}, + {0x1ee04, 0x1ee20, 28}, + {0x1ee23, 0x1ee25, 2}, + {0x1ee26, 0x1ee28, 2}, + {0x1ee33, 0x1ee38, 5}, + {0x1ee3a, 0x1ee3c, 2}, + {0x1ee3d, 0x1ee41, 1}, + {0x1ee43, 0x1ee46, 1}, + {0x1ee48, 0x1ee4c, 2}, + {0x1ee50, 0x1ee53, 3}, + {0x1ee55, 0x1ee56, 1}, + {0x1ee58, 0x1ee60, 2}, + {0x1ee63, 0x1ee65, 2}, + {0x1ee66, 0x1ee6b, 5}, + {0x1ee73, 0x1ee7d, 5}, + {0x1ee7f, 0x1ee8a, 11}, + {0x1ee9c, 0x1eea0, 1}, + {0x1eea4, 0x1eeaa, 6}, + {0x1eebc, 0x1eeef, 1}, + {0x1eef2, 0x1efff, 1}, + {0x1f02c, 0x1f02f, 1}, + {0x1f094, 0x1f09f, 1}, + {0x1f0af, 0x1f0b0, 1}, + {0x1f0c0, 0x1f0d0, 16}, + {0x1f0f6, 0x1f0ff, 1}, + {0x1f1ae, 0x1f1e5, 1}, + {0x1f203, 0x1f20f, 1}, + {0x1f23c, 0x1f23f, 1}, + {0x1f249, 0x1f24f, 1}, + {0x1f252, 0x1f25f, 1}, + {0x1f266, 0x1f2ff, 1}, + {0x1f6d8, 0x1f6db, 1}, + {0x1f6ed, 0x1f6ef, 1}, + {0x1f6fd, 0x1f6ff, 1}, + {0x1f777, 0x1f77a, 1}, + {0x1f7da, 0x1f7df, 1}, + {0x1f7ec, 0x1f7ef, 1}, + {0x1f7f1, 0x1f7ff, 1}, + {0x1f80c, 0x1f80f, 1}, + {0x1f848, 0x1f84f, 1}, + {0x1f85a, 0x1f85f, 1}, + {0x1f888, 0x1f88f, 1}, + {0x1f8ae, 0x1f8af, 1}, + {0x1f8b2, 0x1f8ff, 1}, + {0x1fa54, 0x1fa5f, 1}, + {0x1fa6e, 0x1fa6f, 1}, + {0x1fa7d, 0x1fa7f, 1}, + {0x1fa89, 0x1fa8f, 1}, + {0x1fabe, 0x1fac6, 8}, + {0x1fac7, 0x1facd, 1}, + {0x1fadc, 0x1fadf, 1}, + {0x1fae9, 0x1faef, 1}, + {0x1faf9, 0x1faff, 1}, + {0x1fb93, 0x1fbcb, 56}, + {0x1fbcc, 0x1fbef, 1}, + {0x1fbfa, 0x1ffff, 1}, + {0x2a6e0, 0x2a6ff, 1}, + {0x2b73a, 0x2b73f, 1}, + {0x2b81e, 0x2b81f, 1}, + {0x2cea2, 0x2ceaf, 1}, + {0x2ebe1, 0x2f7ff, 1}, + {0x2fa1e, 0x2ffff, 1}, + {0x3134b, 0x3134f, 1}, + {0x323b0, 0xe0000, 1}, + {0xe0002, 0xe001f, 1}, + {0xe0080, 0xe00ff, 1}, + {0xe01f0, 0xeffff, 1}, + {0xffffe, 0xfffff, 1}, + {0x10fffe, 0x10ffff, 1}, + }, +} + var _Co = &RangeTable{ R16: []Range16{ {0xe000, 0xf8ff, 1}, @@ -756,6 +2001,149 @@ var _L = &RangeTable{ LatinOffset: 6, } +var _LC = &RangeTable{ + R16: []Range16{ + {0x0041, 0x005a, 1}, + {0x0061, 0x007a, 1}, + {0x00b5, 0x00c0, 11}, + {0x00c1, 0x00d6, 1}, + {0x00d8, 0x00f6, 1}, + {0x00f8, 0x01ba, 1}, + {0x01bc, 0x01bf, 1}, + {0x01c4, 0x0293, 1}, + {0x0295, 0x02af, 1}, + {0x0370, 0x0373, 1}, + {0x0376, 0x0377, 1}, + {0x037b, 0x037d, 1}, + {0x037f, 0x0386, 7}, + {0x0388, 0x038a, 1}, + {0x038c, 0x038e, 2}, + {0x038f, 0x03a1, 1}, + {0x03a3, 0x03f5, 1}, + {0x03f7, 0x0481, 1}, + {0x048a, 0x052f, 1}, + {0x0531, 0x0556, 1}, + {0x0560, 0x0588, 1}, + {0x10a0, 0x10c5, 1}, + {0x10c7, 0x10cd, 6}, + {0x10d0, 0x10fa, 1}, + {0x10fd, 0x10ff, 1}, + {0x13a0, 0x13f5, 1}, + {0x13f8, 0x13fd, 1}, + {0x1c80, 0x1c88, 1}, + {0x1c90, 0x1cba, 1}, + {0x1cbd, 0x1cbf, 1}, + {0x1d00, 0x1d2b, 1}, + {0x1d6b, 0x1d77, 1}, + {0x1d79, 0x1d9a, 1}, + {0x1e00, 0x1f15, 1}, + {0x1f18, 0x1f1d, 1}, + {0x1f20, 0x1f45, 1}, + {0x1f48, 0x1f4d, 1}, + {0x1f50, 0x1f57, 1}, + {0x1f59, 0x1f5f, 2}, + {0x1f60, 0x1f7d, 1}, + {0x1f80, 0x1fb4, 1}, + {0x1fb6, 0x1fbc, 1}, + {0x1fbe, 0x1fc2, 4}, + {0x1fc3, 0x1fc4, 1}, + {0x1fc6, 0x1fcc, 1}, + {0x1fd0, 0x1fd3, 1}, + {0x1fd6, 0x1fdb, 1}, + {0x1fe0, 0x1fec, 1}, + {0x1ff2, 0x1ff4, 1}, + {0x1ff6, 0x1ffc, 1}, + {0x2102, 0x2107, 5}, + {0x210a, 0x2113, 1}, + {0x2115, 0x2119, 4}, + {0x211a, 0x211d, 1}, + {0x2124, 0x212a, 2}, + {0x212b, 0x212d, 1}, + {0x212f, 0x2134, 1}, + {0x2139, 0x213c, 3}, + {0x213d, 0x213f, 1}, + {0x2145, 0x2149, 1}, + {0x214e, 0x2183, 53}, + {0x2184, 0x2c00, 2684}, + {0x2c01, 0x2c7b, 1}, + {0x2c7e, 0x2ce4, 1}, + {0x2ceb, 0x2cee, 1}, + {0x2cf2, 0x2cf3, 1}, + {0x2d00, 0x2d25, 1}, + {0x2d27, 0x2d2d, 6}, + {0xa640, 0xa66d, 1}, + {0xa680, 0xa69b, 1}, + {0xa722, 0xa76f, 1}, + {0xa771, 0xa787, 1}, + {0xa78b, 0xa78e, 1}, + {0xa790, 0xa7ca, 1}, + {0xa7d0, 0xa7d1, 1}, + {0xa7d3, 0xa7d5, 2}, + {0xa7d6, 0xa7d9, 1}, + {0xa7f5, 0xa7f6, 1}, + {0xa7fa, 0xab30, 822}, + {0xab31, 0xab5a, 1}, + {0xab60, 0xab68, 1}, + {0xab70, 0xabbf, 1}, + {0xfb00, 0xfb06, 1}, + {0xfb13, 0xfb17, 1}, + {0xff21, 0xff3a, 1}, + {0xff41, 0xff5a, 1}, + }, + R32: []Range32{ + {0x10400, 0x1044f, 1}, + {0x104b0, 0x104d3, 1}, + {0x104d8, 0x104fb, 1}, + {0x10570, 0x1057a, 1}, + {0x1057c, 0x1058a, 1}, + {0x1058c, 0x10592, 1}, + {0x10594, 0x10595, 1}, + {0x10597, 0x105a1, 1}, + {0x105a3, 0x105b1, 1}, + {0x105b3, 0x105b9, 1}, + {0x105bb, 0x105bc, 1}, + {0x10c80, 0x10cb2, 1}, + {0x10cc0, 0x10cf2, 1}, + {0x118a0, 0x118df, 1}, + {0x16e40, 0x16e7f, 1}, + {0x1d400, 0x1d454, 1}, + {0x1d456, 0x1d49c, 1}, + {0x1d49e, 0x1d49f, 1}, + {0x1d4a2, 0x1d4a5, 3}, + {0x1d4a6, 0x1d4a9, 3}, + {0x1d4aa, 0x1d4ac, 1}, + {0x1d4ae, 0x1d4b9, 1}, + {0x1d4bb, 0x1d4bd, 2}, + {0x1d4be, 0x1d4c3, 1}, + {0x1d4c5, 0x1d505, 1}, + {0x1d507, 0x1d50a, 1}, + {0x1d50d, 0x1d514, 1}, + {0x1d516, 0x1d51c, 1}, + {0x1d51e, 0x1d539, 1}, + {0x1d53b, 0x1d53e, 1}, + {0x1d540, 0x1d544, 1}, + {0x1d546, 0x1d54a, 4}, + {0x1d54b, 0x1d550, 1}, + {0x1d552, 0x1d6a5, 1}, + {0x1d6a8, 0x1d6c0, 1}, + {0x1d6c2, 0x1d6da, 1}, + {0x1d6dc, 0x1d6fa, 1}, + {0x1d6fc, 0x1d714, 1}, + {0x1d716, 0x1d734, 1}, + {0x1d736, 0x1d74e, 1}, + {0x1d750, 0x1d76e, 1}, + {0x1d770, 0x1d788, 1}, + {0x1d78a, 0x1d7a8, 1}, + {0x1d7aa, 0x1d7c2, 1}, + {0x1d7c4, 0x1d7cb, 1}, + {0x1df00, 0x1df09, 1}, + {0x1df0b, 0x1df1e, 1}, + {0x1df25, 0x1df2a, 1}, + {0x1e900, 0x1e943, 1}, + }, + LatinOffset: 5, +} + var _Ll = &RangeTable{ R16: []Range16{ {0x0061, 0x007a, 1}, @@ -3751,10 +5139,12 @@ var _Zs = &RangeTable{ var ( Cc = _Cc // Cc is the set of Unicode characters in category Cc (Other, control). Cf = _Cf // Cf is the set of Unicode characters in category Cf (Other, format). + Cn = _Cn // Cn is the set of Unicode characters in category Cn (Other, not assigned). Co = _Co // Co is the set of Unicode characters in category Co (Other, private use). Cs = _Cs // Cs is the set of Unicode characters in category Cs (Other, surrogate). Digit = _Nd // Digit is the set of Unicode characters with the "decimal digit" property. Nd = _Nd // Nd is the set of Unicode characters in category Nd (Number, decimal digit). + LC = _LC // LC is the set of Unicode characters in category LC (Letter, cased: Ll | Lt | Lu). Letter = _L // Letter/L is the set of Unicode letters, category L. L = _L Lm = _Lm // Lm is the set of Unicode characters in category Lm (Letter, modifier). @@ -3770,7 +5160,7 @@ var ( No = _No // No is the set of Unicode characters in category No (Number, other). Number = _N // Number/N is the set of Unicode number characters, category N. N = _N - Other = _C // Other/C is the set of Unicode control and special characters, category C. + Other = _C // Other/C is the set of Unicode control, special, and unassigned code points, category C. C = _C Pc = _Pc // Pc is the set of Unicode characters in category Pc (Punctuation, connector). Pd = _Pd // Pd is the set of Unicode characters in category Pd (Punctuation, dash). @@ -8372,7 +9762,7 @@ var foldInherited = &RangeTable{ }, } -// Range entries: 3535 16-bit, 2031 32-bit, 5566 total. -// Range bytes: 21210 16-bit, 24372 32-bit, 45582 total. +// Range entries: 4177 16-bit, 2714 32-bit, 6891 total. +// Range bytes: 25062 16-bit, 32568 32-bit, 57630 total. // Fold orbit bytes: 88 pairs, 352 bytes diff --git a/src/unicode/utf8/utf8.go b/src/unicode/utf8/utf8.go index 1c1391d55b53b7..68283341d92ace 100644 --- a/src/unicode/utf8/utf8.go +++ b/src/unicode/utf8/utf8.go @@ -155,6 +155,20 @@ func FullRuneInString(s string) bool { // out of range, or is not the shortest possible UTF-8 encoding for the // value. No other validation is performed. func DecodeRune(p []byte) (r rune, size int) { + // Inlineable fast path for ASCII characters; see #48195. + // This implementation is weird but effective at rendering the + // function inlineable. + for _, b := range p { + if b < RuneSelf { + return rune(b), 1 + } + break + } + r, size = decodeRuneSlow(p) + return +} + +func decodeRuneSlow(p []byte) (r rune, size int) { n := len(p) if n < 1 { return RuneError, 0 @@ -203,6 +217,18 @@ func DecodeRune(p []byte) (r rune, size int) { // out of range, or is not the shortest possible UTF-8 encoding for the // value. No other validation is performed. func DecodeRuneInString(s string) (r rune, size int) { + // Inlineable fast path for ASCII characters; see #48195. + // This implementation is a bit weird but effective at rendering the + // function inlineable. + if s != "" && s[0] < RuneSelf { + return rune(s[0]), 1 + } else { + r, size = decodeRuneInStringSlow(s) + } + return +} + +func decodeRuneInStringSlow(s string) (rune, int) { n := len(s) if n < 1 { return RuneError, 0 @@ -263,10 +289,7 @@ func DecodeLastRune(p []byte) (r rune, size int) { // guard against O(n^2) behavior when traversing // backwards through strings with long sequences of // invalid UTF-8. - lim := end - UTFMax - if lim < 0 { - lim = 0 - } + lim := max(end-UTFMax, 0) for start--; start >= lim; start-- { if RuneStart(p[start]) { break @@ -303,10 +326,7 @@ func DecodeLastRuneInString(s string) (r rune, size int) { // guard against O(n^2) behavior when traversing // backwards through strings with long sequences of // invalid UTF-8. - lim := end - UTFMax - if lim < 0 { - lim = 0 - } + lim := max(end-UTFMax, 0) for start--; start >= lim; start-- { if RuneStart(s[start]) { break @@ -414,70 +434,19 @@ func appendRuneNonASCII(p []byte, r rune) []byte { func RuneCount(p []byte) int { np := len(p) var n int - for i := 0; i < np; { - n++ - c := p[i] - if c < RuneSelf { - // ASCII fast path - i++ - continue - } - x := first[c] - if x == xx { - i++ // invalid. - continue - } - size := int(x & 7) - if i+size > np { - i++ // Short or invalid. - continue - } - accept := acceptRanges[x>>4] - if c := p[i+1]; c < accept.lo || accept.hi < c { - size = 1 - } else if size == 2 { - } else if c := p[i+2]; c < locb || hicb < c { - size = 1 - } else if size == 3 { - } else if c := p[i+3]; c < locb || hicb < c { - size = 1 + for ; n < np; n++ { + if c := p[n]; c >= RuneSelf { + // non-ASCII slow path + return n + RuneCountInString(string(p[n:])) } - i += size } return n } // RuneCountInString is like [RuneCount] but its input is a string. func RuneCountInString(s string) (n int) { - ns := len(s) - for i := 0; i < ns; n++ { - c := s[i] - if c < RuneSelf { - // ASCII fast path - i++ - continue - } - x := first[c] - if x == xx { - i++ // invalid. - continue - } - size := int(x & 7) - if i+size > ns { - i++ // Short or invalid. - continue - } - accept := acceptRanges[x>>4] - if c := s[i+1]; c < accept.lo || accept.hi < c { - size = 1 - } else if size == 2 { - } else if c := s[i+2]; c < locb || hicb < c { - size = 1 - } else if size == 3 { - } else if c := s[i+3]; c < locb || hicb < c { - size = 1 - } - i += size + for range s { + n++ } return n } @@ -487,99 +456,111 @@ func RuneCountInString(s string) (n int) { // bits set to 10. func RuneStart(b byte) bool { return b&0xC0 != 0x80 } +const ptrSize = 4 << (^uintptr(0) >> 63) +const hiBits = 0x8080808080808080 >> (64 - 8*ptrSize) + +func word[T string | []byte](s T) uintptr { + if ptrSize == 4 { + return uintptr(s[0]) | uintptr(s[1])<<8 | uintptr(s[2])<<16 | uintptr(s[3])<<24 + } + return uintptr(uint64(s[0]) | uint64(s[1])<<8 | uint64(s[2])<<16 | uint64(s[3])<<24 | uint64(s[4])<<32 | uint64(s[5])<<40 | uint64(s[6])<<48 | uint64(s[7])<<56) +} + // Valid reports whether p consists entirely of valid UTF-8-encoded runes. func Valid(p []byte) bool { // This optimization avoids the need to recompute the capacity - // when generating code for p[8:], bringing it to parity with + // when generating code for slicing p, bringing it to parity with // ValidString, which was 20% faster on long ASCII strings. p = p[:len(p):len(p)] - // Fast path. Check for and skip 8 bytes of ASCII characters per iteration. - for len(p) >= 8 { - // Combining two 32 bit loads allows the same code to be used - // for 32 and 64 bit platforms. - // The compiler can generate a 32bit load for first32 and second32 - // on many platforms. See test/codegen/memcombine.go. - first32 := uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24 - second32 := uint32(p[4]) | uint32(p[5])<<8 | uint32(p[6])<<16 | uint32(p[7])<<24 - if (first32|second32)&0x80808080 != 0 { - // Found a non ASCII byte (>= RuneSelf). - break - } - p = p[8:] - } - n := len(p) - for i := 0; i < n; { - pi := p[i] - if pi < RuneSelf { - i++ + for len(p) > 0 { + p0 := p[0] + if p0 < RuneSelf { + p = p[1:] + // If there's one ASCII byte, there are probably more. + // Advance quickly through ASCII-only data. + // Note: using > instead of >= here is intentional. That avoids + // needing pointing-past-the-end fixup on the slice operations. + if len(p) > ptrSize && word(p)&hiBits == 0 { + p = p[ptrSize:] + if len(p) > 2*ptrSize && (word(p)|word(p[ptrSize:]))&hiBits == 0 { + p = p[2*ptrSize:] + for len(p) > 4*ptrSize && ((word(p)|word(p[ptrSize:]))|(word(p[2*ptrSize:])|word(p[3*ptrSize:])))&hiBits == 0 { + p = p[4*ptrSize:] + } + } + } continue } - x := first[pi] - if x == xx { - return false // Illegal starter byte. - } + x := first[p0] size := int(x & 7) - if i+size > n { - return false // Short or invalid. - } accept := acceptRanges[x>>4] - if c := p[i+1]; c < accept.lo || accept.hi < c { - return false - } else if size == 2 { - } else if c := p[i+2]; c < locb || hicb < c { - return false - } else if size == 3 { - } else if c := p[i+3]; c < locb || hicb < c { - return false + switch size { + case 2: + if len(p) < 2 || p[1] < accept.lo || accept.hi < p[1] { + return false + } + p = p[2:] + case 3: + if len(p) < 3 || p[1] < accept.lo || accept.hi < p[1] || p[2] < locb || hicb < p[2] { + return false + } + p = p[3:] + case 4: + if len(p) < 4 || p[1] < accept.lo || accept.hi < p[1] || p[2] < locb || hicb < p[2] || p[3] < locb || hicb < p[3] { + return false + } + p = p[4:] + default: + return false // illegal starter byte } - i += size } return true } // ValidString reports whether s consists entirely of valid UTF-8-encoded runes. func ValidString(s string) bool { - // Fast path. Check for and skip 8 bytes of ASCII characters per iteration. - for len(s) >= 8 { - // Combining two 32 bit loads allows the same code to be used - // for 32 and 64 bit platforms. - // The compiler can generate a 32bit load for first32 and second32 - // on many platforms. See test/codegen/memcombine.go. - first32 := uint32(s[0]) | uint32(s[1])<<8 | uint32(s[2])<<16 | uint32(s[3])<<24 - second32 := uint32(s[4]) | uint32(s[5])<<8 | uint32(s[6])<<16 | uint32(s[7])<<24 - if (first32|second32)&0x80808080 != 0 { - // Found a non ASCII byte (>= RuneSelf). - break - } - s = s[8:] - } - n := len(s) - for i := 0; i < n; { - si := s[i] - if si < RuneSelf { - i++ + for len(s) > 0 { + s0 := s[0] + if s0 < RuneSelf { + s = s[1:] + // If there's one ASCII byte, there are probably more. + // Advance quickly through ASCII-only data. + // Note: using > instead of >= here is intentional. That avoids + // needing pointing-past-the-end fixup on the slice operations. + if len(s) > ptrSize && word(s)&hiBits == 0 { + s = s[ptrSize:] + if len(s) > 2*ptrSize && (word(s)|word(s[ptrSize:]))&hiBits == 0 { + s = s[2*ptrSize:] + for len(s) > 4*ptrSize && ((word(s)|word(s[ptrSize:]))|(word(s[2*ptrSize:])|word(s[3*ptrSize:])))&hiBits == 0 { + s = s[4*ptrSize:] + } + } + } continue } - x := first[si] - if x == xx { - return false // Illegal starter byte. - } + x := first[s0] size := int(x & 7) - if i+size > n { - return false // Short or invalid. - } accept := acceptRanges[x>>4] - if c := s[i+1]; c < accept.lo || accept.hi < c { - return false - } else if size == 2 { - } else if c := s[i+2]; c < locb || hicb < c { - return false - } else if size == 3 { - } else if c := s[i+3]; c < locb || hicb < c { - return false + switch size { + case 2: + if len(s) < 2 || s[1] < accept.lo || accept.hi < s[1] { + return false + } + s = s[2:] + case 3: + if len(s) < 3 || s[1] < accept.lo || accept.hi < s[1] || s[2] < locb || hicb < s[2] { + return false + } + s = s[3:] + case 4: + if len(s) < 4 || s[1] < accept.lo || accept.hi < s[1] || s[2] < locb || hicb < s[2] || s[3] < locb || hicb < s[3] { + return false + } + s = s[4:] + default: + return false // illegal starter byte } - i += size } return true } diff --git a/src/unicode/utf8/utf8_test.go b/src/unicode/utf8/utf8_test.go index fa23419b3649b7..bf4f074ffd0f5f 100644 --- a/src/unicode/utf8/utf8_test.go +++ b/src/unicode/utf8/utf8_test.go @@ -12,16 +12,6 @@ import ( . "unicode/utf8" ) -// Validate the constants redefined from unicode. -func init() { - if MaxRune != unicode.MaxRune { - panic("utf8.MaxRune is wrong") - } - if RuneError != unicode.ReplacementChar { - panic("utf8.RuneError is wrong") - } -} - // Validate the constants redefined from unicode. func TestConstants(t *testing.T) { if MaxRune != unicode.MaxRune { @@ -170,7 +160,7 @@ func TestDecodeRune(t *testing.T) { } r, size = DecodeRune(b[0 : len(b)-1]) if r != RuneError || size != wantsize { - t.Errorf("DecodeRune(%q) = %#04x, %d want %#04x, %d", b[0:len(b)-1], r, size, RuneError, wantsize) + t.Errorf("DecodeRune(%q) = %#04x, %d want %#04x, %d", b[:len(b)-1], r, size, RuneError, wantsize) } s = m.str[0 : len(m.str)-1] r, size = DecodeRuneInString(s) @@ -438,6 +428,15 @@ func TestRuneCount(t *testing.T) { } } +func TestRuneCountNonASCIIAllocation(t *testing.T) { + if n := testing.AllocsPerRun(10, func() { + s := []byte("日本語日本語日本語日") + _ = RuneCount(s) + }); n > 0 { + t.Errorf("unexpected RuneCount allocation, got %v, want 0", n) + } +} + type RuneLenTest struct { r rune size int @@ -490,6 +489,16 @@ var validTests = []ValidTest{ {string("\xed\xbf\xbf"), false}, // U+DFFF low surrogate (sic) } +func init() { + for i := range 100 { + validTests = append(validTests, ValidTest{in: strings.Repeat("a", i), out: true}) + validTests = append(validTests, ValidTest{in: strings.Repeat("a", i) + "Ж", out: true}) + validTests = append(validTests, ValidTest{in: strings.Repeat("a", i) + "\xe2", out: false}) + validTests = append(validTests, ValidTest{in: strings.Repeat("a", i) + "Ж" + strings.Repeat("b", i), out: true}) + validTests = append(validTests, ValidTest{in: strings.Repeat("a", i) + "\xe2" + strings.Repeat("b", i), out: false}) + } +} + func TestValid(t *testing.T) { for _, tt := range validTests { if Valid([]byte(tt.in)) != tt.out { @@ -738,18 +747,37 @@ func BenchmarkAppendInvalidRuneNegative(b *testing.B) { func BenchmarkDecodeASCIIRune(b *testing.B) { a := []byte{'a'} - for i := 0; i < b.N; i++ { - DecodeRune(a) + for range b.N { + runeSink, sizeSink = DecodeRune(a) } } func BenchmarkDecodeJapaneseRune(b *testing.B) { nihon := []byte("本") - for i := 0; i < b.N; i++ { - DecodeRune(nihon) + for range b.N { + runeSink, sizeSink = DecodeRune(nihon) + } +} + +func BenchmarkDecodeASCIIRuneInString(b *testing.B) { + a := "a" + for range b.N { + runeSink, sizeSink = DecodeRuneInString(a) + } +} + +func BenchmarkDecodeJapaneseRuneInString(b *testing.B) { + nihon := "本" + for range b.N { + runeSink, sizeSink = DecodeRuneInString(nihon) } } +var ( + runeSink rune + sizeSink int +) + // boolSink is used to reference the return value of benchmarked // functions to avoid dead code elimination. var boolSink bool diff --git a/src/unique/canonmap.go b/src/unique/canonmap.go new file mode 100644 index 00000000000000..74c95a12af9177 --- /dev/null +++ b/src/unique/canonmap.go @@ -0,0 +1,385 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unique + +import ( + "internal/abi" + "internal/goarch" + "runtime" + "sync" + "sync/atomic" + "unsafe" + "weak" +) + +// canonMap is a map of T -> *T. The map controls the creation +// of a canonical *T, and elements of the map are automatically +// deleted when the canonical *T is no longer referenced. +type canonMap[T comparable] struct { + root atomic.Pointer[indirect[T]] + hash func(unsafe.Pointer, uintptr) uintptr + seed uintptr +} + +func newCanonMap[T comparable]() *canonMap[T] { + cm := new(canonMap[T]) + cm.root.Store(newIndirectNode[T](nil)) + + var m map[T]struct{} + mapType := abi.TypeOf(m).MapType() + cm.hash = mapType.Hasher + cm.seed = uintptr(runtime_rand()) + return cm +} + +func (m *canonMap[T]) Load(key T) *T { + hash := m.hash(abi.NoEscape(unsafe.Pointer(&key)), m.seed) + + i := m.root.Load() + hashShift := 8 * goarch.PtrSize + for hashShift != 0 { + hashShift -= nChildrenLog2 + + n := i.children[(hash>>hashShift)&nChildrenMask].Load() + if n == nil { + return nil + } + if n.isEntry { + v, _ := n.entry().lookup(key) + return v + } + i = n.indirect() + } + panic("unique.canonMap: ran out of hash bits while iterating") +} + +func (m *canonMap[T]) LoadOrStore(key T) *T { + hash := m.hash(abi.NoEscape(unsafe.Pointer(&key)), m.seed) + + var i *indirect[T] + var hashShift uint + var slot *atomic.Pointer[node[T]] + var n *node[T] + for { + // Find the key or a candidate location for insertion. + i = m.root.Load() + hashShift = 8 * goarch.PtrSize + haveInsertPoint := false + for hashShift != 0 { + hashShift -= nChildrenLog2 + + slot = &i.children[(hash>>hashShift)&nChildrenMask] + n = slot.Load() + if n == nil { + // We found a nil slot which is a candidate for insertion. + haveInsertPoint = true + break + } + if n.isEntry { + // We found an existing entry, which is as far as we can go. + // If it stays this way, we'll have to replace it with an + // indirect node. + if v, _ := n.entry().lookup(key); v != nil { + return v + } + haveInsertPoint = true + break + } + i = n.indirect() + } + if !haveInsertPoint { + panic("unique.canonMap: ran out of hash bits while iterating") + } + + // Grab the lock and double-check what we saw. + i.mu.Lock() + n = slot.Load() + if (n == nil || n.isEntry) && !i.dead.Load() { + // What we saw is still true, so we can continue with the insert. + break + } + // We have to start over. + i.mu.Unlock() + } + // N.B. This lock is held from when we broke out of the outer loop above. + // We specifically break this out so that we can use defer here safely. + // One option is to break this out into a new function instead, but + // there's so much local iteration state used below that this turns out + // to be cleaner. + defer i.mu.Unlock() + + var oldEntry *entry[T] + if n != nil { + oldEntry = n.entry() + if v, _ := oldEntry.lookup(key); v != nil { + // Easy case: by loading again, it turns out exactly what we wanted is here! + return v + } + } + newEntry, canon, wp := newEntryNode(key, hash) + // Prune dead pointers. This is to avoid O(n) lookups when we store the exact same + // value in the set but the cleanup hasn't run yet because it got delayed for some + // reason. + oldEntry = oldEntry.prune() + if oldEntry == nil { + // Easy case: create a new entry and store it. + slot.Store(&newEntry.node) + } else { + // We possibly need to expand the entry already there into one or more new nodes. + // + // Publish the node last, which will make both oldEntry and newEntry visible. We + // don't want readers to be able to observe that oldEntry isn't in the tree. + slot.Store(m.expand(oldEntry, newEntry, hash, hashShift, i)) + } + runtime.AddCleanup(canon, func(_ struct{}) { + m.cleanup(hash, wp) + }, struct{}{}) + return canon +} + +// expand takes oldEntry and newEntry whose hashes conflict from bit 64 down to hashShift and +// produces a subtree of indirect nodes to hold the two new entries. newHash is the hash of +// the value in the new entry. +func (m *canonMap[T]) expand(oldEntry, newEntry *entry[T], newHash uintptr, hashShift uint, parent *indirect[T]) *node[T] { + // Check for a hash collision. + oldHash := oldEntry.hash + if oldHash == newHash { + // Store the old entry in the new entry's overflow list, then store + // the new entry. + newEntry.overflow.Store(oldEntry) + return &newEntry.node + } + // We have to add an indirect node. Worse still, we may need to add more than one. + newIndirect := newIndirectNode(parent) + top := newIndirect + for { + if hashShift == 0 { + panic("unique.canonMap: ran out of hash bits while inserting") + } + hashShift -= nChildrenLog2 // hashShift is for the level parent is at. We need to go deeper. + oi := (oldHash >> hashShift) & nChildrenMask + ni := (newHash >> hashShift) & nChildrenMask + if oi != ni { + newIndirect.children[oi].Store(&oldEntry.node) + newIndirect.children[ni].Store(&newEntry.node) + break + } + nextIndirect := newIndirectNode(newIndirect) + newIndirect.children[oi].Store(&nextIndirect.node) + newIndirect = nextIndirect + } + return &top.node +} + +// cleanup deletes the entry corresponding to wp in the canon map, if it's +// still in the map. wp must have a Value method that returns nil by the +// time this function is called. hash must be the hash of the value that +// wp once pointed to (that is, the hash of *wp.Value()). +func (m *canonMap[T]) cleanup(hash uintptr, wp weak.Pointer[T]) { + var i *indirect[T] + var hashShift uint + var slot *atomic.Pointer[node[T]] + var n *node[T] + for { + // Find wp in the map by following hash. + i = m.root.Load() + hashShift = 8 * goarch.PtrSize + haveEntry := false + for hashShift != 0 { + hashShift -= nChildrenLog2 + + slot = &i.children[(hash>>hashShift)&nChildrenMask] + n = slot.Load() + if n == nil { + // We found a nil slot, already deleted. + return + } + if n.isEntry { + if !n.entry().hasWeakPointer(wp) { + // The weak pointer was already pruned. + return + } + haveEntry = true + break + } + i = n.indirect() + } + if !haveEntry { + panic("unique.canonMap: ran out of hash bits while iterating") + } + + // Grab the lock and double-check what we saw. + i.mu.Lock() + n = slot.Load() + if n != nil && n.isEntry { + // Prune the entry node without thinking too hard. If we do + // somebody else's work, such as someone trying to insert an + // entry with the same hash (probably the same value) then + // great, they'll back out without taking the lock. + newEntry := n.entry().prune() + if newEntry == nil { + slot.Store(nil) + } else { + slot.Store(&newEntry.node) + } + + // Delete interior nodes that are empty, up the tree. + // + // We'll hand-over-hand lock our way up the tree as we do this, + // since we need to delete each empty node's link in its parent, + // which requires the parents' lock. + for i.parent != nil && i.empty() { + if hashShift == 8*goarch.PtrSize { + panic("unique.canonMap: ran out of hash bits while iterating") + } + hashShift += nChildrenLog2 + + // Delete the current node in the parent. + parent := i.parent + parent.mu.Lock() + i.dead.Store(true) // Could be done outside of parent's lock. + parent.children[(hash>>hashShift)&nChildrenMask].Store(nil) + i.mu.Unlock() + i = parent + } + i.mu.Unlock() + return + } + // We have to start over. + i.mu.Unlock() + } +} + +// node is the header for a node. It's polymorphic and +// is actually either an entry or an indirect. +type node[T comparable] struct { + isEntry bool +} + +func (n *node[T]) entry() *entry[T] { + if !n.isEntry { + panic("called entry on non-entry node") + } + return (*entry[T])(unsafe.Pointer(n)) +} + +func (n *node[T]) indirect() *indirect[T] { + if n.isEntry { + panic("called indirect on entry node") + } + return (*indirect[T])(unsafe.Pointer(n)) +} + +const ( + // 16 children. This seems to be the sweet spot for + // load performance: any smaller and we lose out on + // 50% or more in CPU performance. Any larger and the + // returns are minuscule (~1% improvement for 32 children). + nChildrenLog2 = 4 + nChildren = 1 << nChildrenLog2 + nChildrenMask = nChildren - 1 +) + +// indirect is an internal node in the hash-trie. +type indirect[T comparable] struct { + node[T] + dead atomic.Bool + parent *indirect[T] + mu sync.Mutex // Protects mutation to children and any children that are entry nodes. + children [nChildren]atomic.Pointer[node[T]] +} + +func newIndirectNode[T comparable](parent *indirect[T]) *indirect[T] { + return &indirect[T]{node: node[T]{isEntry: false}, parent: parent} +} + +func (i *indirect[T]) empty() bool { + for j := range i.children { + if i.children[j].Load() != nil { + return false + } + } + return true +} + +// entry is a leaf node in the hash-trie. +type entry[T comparable] struct { + node[T] + overflow atomic.Pointer[entry[T]] // Overflow for hash collisions. + key weak.Pointer[T] + hash uintptr +} + +func newEntryNode[T comparable](key T, hash uintptr) (*entry[T], *T, weak.Pointer[T]) { + k := new(T) + *k = key + wp := weak.Make(k) + return &entry[T]{ + node: node[T]{isEntry: true}, + key: wp, + hash: hash, + }, k, wp +} + +// lookup finds the entry in the overflow chain that has the provided key. +// +// Returns the key's canonical pointer and the weak pointer for that canonical pointer. +func (e *entry[T]) lookup(key T) (*T, weak.Pointer[T]) { + for e != nil { + s := e.key.Value() + if s != nil && *s == key { + return s, e.key + } + e = e.overflow.Load() + } + return nil, weak.Pointer[T]{} +} + +// hasWeakPointer returns true if the provided weak pointer can be found in the overflow chain. +func (e *entry[T]) hasWeakPointer(wp weak.Pointer[T]) bool { + for e != nil { + if e.key == wp { + return true + } + e = e.overflow.Load() + } + return false +} + +// prune removes all entries in the overflow chain whose keys are nil. +// +// The caller must hold the lock on e's parent node. +func (e *entry[T]) prune() *entry[T] { + // Prune the head of the list. + for e != nil { + if e.key.Value() != nil { + break + } + e = e.overflow.Load() + } + if e == nil { + return nil + } + + // Prune individual nodes in the list. + newHead := e + i := &e.overflow + e = i.Load() + for e != nil { + if e.key.Value() != nil { + i = &e.overflow + } else { + i.Store(e.overflow.Load()) + } + e = e.overflow.Load() + } + return newHead +} + +// Pull in runtime.rand so that we don't need to take a dependency +// on math/rand/v2. +// +//go:linkname runtime_rand runtime.rand +func runtime_rand() uint64 diff --git a/src/unique/canonmap_test.go b/src/unique/canonmap_test.go new file mode 100644 index 00000000000000..9609d7d422e67a --- /dev/null +++ b/src/unique/canonmap_test.go @@ -0,0 +1,198 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unique + +import ( + "internal/abi" + "runtime" + "strconv" + "sync" + "testing" + "unsafe" +) + +func TestCanonMap(t *testing.T) { + testCanonMap(t, func() *canonMap[string] { + return newCanonMap[string]() + }) +} + +func TestCanonMapBadHash(t *testing.T) { + testCanonMap(t, func() *canonMap[string] { + return newBadCanonMap[string]() + }) +} + +func TestCanonMapTruncHash(t *testing.T) { + testCanonMap(t, func() *canonMap[string] { + // Stub out the good hash function with a different terrible one + // (truncated hash). Everything should still work as expected. + // This is useful to test independently to catch issues with + // near collisions, where only the last few bits of the hash differ. + return newTruncCanonMap[string]() + }) +} + +func testCanonMap(t *testing.T, newMap func() *canonMap[string]) { + t.Run("LoadEmpty", func(t *testing.T) { + m := newMap() + + for _, s := range testData { + expectMissing(t, s)(m.Load(s)) + } + }) + t.Run("LoadOrStore", func(t *testing.T) { + t.Run("Sequential", func(t *testing.T) { + m := newMap() + + var refs []*string + for _, s := range testData { + expectMissing(t, s)(m.Load(s)) + refs = append(refs, expectPresent(t, s)(m.LoadOrStore(s))) + expectPresent(t, s)(m.Load(s)) + expectPresent(t, s)(m.LoadOrStore(s)) + } + drainCleanupQueue(t) + for _, s := range testData { + expectPresent(t, s)(m.Load(s)) + expectPresent(t, s)(m.LoadOrStore(s)) + } + runtime.KeepAlive(refs) + refs = nil + drainCleanupQueue(t) + for _, s := range testData { + expectMissing(t, s)(m.Load(s)) + expectPresent(t, s)(m.LoadOrStore(s)) + } + }) + t.Run("ConcurrentUnsharedKeys", func(t *testing.T) { + makeKey := func(s string, id int) string { + return s + "-" + strconv.Itoa(id) + } + + // Expand and shrink the map multiple times to try to get + // insertions and cleanups to overlap. + m := newMap() + gmp := runtime.GOMAXPROCS(-1) + for try := range 3 { + var wg sync.WaitGroup + for i := range gmp { + wg.Add(1) + go func(id int) { + defer wg.Done() + + var refs []*string + for _, s := range testData { + key := makeKey(s, id) + if try == 0 { + expectMissing(t, key)(m.Load(key)) + } + refs = append(refs, expectPresent(t, key)(m.LoadOrStore(key))) + expectPresent(t, key)(m.Load(key)) + expectPresent(t, key)(m.LoadOrStore(key)) + } + for i, s := range testData { + key := makeKey(s, id) + expectPresent(t, key)(m.Load(key)) + if got, want := expectPresent(t, key)(m.LoadOrStore(key)), refs[i]; got != want { + t.Errorf("canonical entry %p did not match ref %p", got, want) + } + } + // N.B. We avoid trying to test entry cleanup here + // because it's going to be very flaky, especially + // in the bad hash cases. + }(i) + } + wg.Wait() + } + + // Run an extra GC cycle to de-flake. Sometimes the cleanups + // fail to run in time, despite drainCleanupQueue. + // + // TODO(mknyszek): Figure out why the extra GC is necessary, + // and what is transiently keeping the cleanups live. + // * I have confirmed that they are not completely stuck, and + // they always eventually run. + // * I have also confirmed it's not asynchronous preemption + // keeping them around (though that is a possibility). + // * I have confirmed that they are not simply sitting on + // the queue, and that drainCleanupQueue is just failing + // to actually empty the queue. + // * I have confirmed that it's not a write barrier that's + // keeping it alive, nor is it a weak pointer dereference + // (which shades the object during the GC). + // The corresponding objects do seem to be transiently truly + // reachable, but I have no idea by what path. + runtime.GC() + + // Drain cleanups so everything is deleted. + drainCleanupQueue(t) + + // Double-check that it's all gone. + for id := range gmp { + makeKey := func(s string) string { + return s + "-" + strconv.Itoa(id) + } + for _, s := range testData { + key := makeKey(s) + expectMissing(t, key)(m.Load(key)) + } + } + }) + }) +} + +func expectMissing[T comparable](t *testing.T, key T) func(got *T) { + t.Helper() + return func(got *T) { + t.Helper() + + if got != nil { + t.Errorf("expected key %v to be missing from map, got %p", key, got) + } + } +} + +func expectPresent[T comparable](t *testing.T, key T) func(got *T) *T { + t.Helper() + return func(got *T) *T { + t.Helper() + + if got == nil { + t.Errorf("expected key %v to be present in map, got %p", key, got) + } + if got != nil && *got != key { + t.Errorf("key %v is present in map, but canonical version has the wrong value: got %v, want %v", key, *got, key) + } + return got + } +} + +// newBadCanonMap creates a new canonMap for the provided key type +// but with an intentionally bad hash function. +func newBadCanonMap[T comparable]() *canonMap[T] { + // Stub out the good hash function with a terrible one. + // Everything should still work as expected. + m := newCanonMap[T]() + m.hash = func(_ unsafe.Pointer, _ uintptr) uintptr { + return 0 + } + return m +} + +// newTruncCanonMap creates a new canonMap for the provided key type +// but with an intentionally bad hash function. +func newTruncCanonMap[T comparable]() *canonMap[T] { + // Stub out the good hash function with a terrible one. + // Everything should still work as expected. + m := newCanonMap[T]() + var mx map[string]int + mapType := abi.TypeOf(mx).MapType() + hasher := mapType.Hasher + m.hash = func(p unsafe.Pointer, n uintptr) uintptr { + return hasher(p, n) & ((uintptr(1) << 4) - 1) + } + return m +} diff --git a/src/unique/clone.go b/src/unique/clone.go index 36ced14ecea0a4..b67029b6549b20 100644 --- a/src/unique/clone.go +++ b/src/unique/clone.go @@ -23,7 +23,7 @@ func clone[T comparable](value T, seq *cloneSeq) T { ps := (*string)(unsafe.Pointer(uintptr(unsafe.Pointer(&value)) + offset)) *ps = stringslite.Clone(*ps) } - return value + return abi.EscapeToResultNonString(value) } // singleStringClone describes how to clone a single string. diff --git a/src/unique/clone_test.go b/src/unique/clone_test.go index 69a9a540c07fa0..b0ba5b312e1466 100644 --- a/src/unique/clone_test.go +++ b/src/unique/clone_test.go @@ -27,7 +27,7 @@ func cSeq(stringOffsets ...uintptr) cloneSeq { func testCloneSeq[T any](t *testing.T, want cloneSeq) { typName := reflect.TypeFor[T]().Name() - typ := abi.TypeOf(*new(T)) + typ := abi.TypeFor[T]() t.Run(typName, func(t *testing.T) { got := makeCloneSeq(typ) if !reflect.DeepEqual(got, want) { diff --git a/src/unique/handle.go b/src/unique/handle.go index 0842ae3185f2cc..93905e8185fdb8 100644 --- a/src/unique/handle.go +++ b/src/unique/handle.go @@ -6,13 +6,12 @@ package unique import ( "internal/abi" - "internal/concurrent" - "internal/weak" - "runtime" - "sync" - _ "unsafe" + isync "internal/sync" + "unsafe" ) +var zero uintptr + // Handle is a globally unique identity for some value of type T. // // Two handles compare equal exactly if the two values used to create the handles @@ -23,152 +22,48 @@ type Handle[T comparable] struct { } // Value returns a shallow copy of the T value that produced the Handle. +// Value is safe for concurrent use by multiple goroutines. func (h Handle[T]) Value() T { return *h.value } // Make returns a globally unique handle for a value of type T. Handles // are equal if and only if the values used to produce them are equal. +// Make is safe for concurrent use by multiple goroutines. func Make[T comparable](value T) Handle[T] { // Find the map for type T. - typ := abi.TypeOf(value) + typ := abi.TypeFor[T]() + if typ.Size() == 0 { + return Handle[T]{(*T)(unsafe.Pointer(&zero))} + } ma, ok := uniqueMaps.Load(typ) if !ok { - // This is a good time to initialize cleanup, since we must go through - // this path on the first use of Make, and it's not on the hot path. - setupMake.Do(registerCleanup) - ma = addUniqueMap[T](typ) + m := &uniqueMap[T]{canonMap: newCanonMap[T](), cloneSeq: makeCloneSeq(typ)} + ma, _ = uniqueMaps.LoadOrStore(typ, m) } m := ma.(*uniqueMap[T]) - // Keep around any values we allocate for insertion. There - // are a few different ways we can race with other threads - // and create values that we might discard. By keeping - // the first one we make around, we can avoid generating - // more than one per racing thread. - var ( - toInsert *T // Keep this around to keep it alive. - toInsertWeak weak.Pointer[T] - ) - newValue := func() weak.Pointer[T] { - if toInsert == nil { - toInsert = new(T) - *toInsert = clone(value, &m.cloneSeq) - toInsertWeak = weak.Make(toInsert) - } - return toInsertWeak - } - var ptr *T - for { - // Check the map. - wp, ok := m.Load(value) - if !ok { - // Try to insert a new value into the map. - wp, _ = m.LoadOrStore(value, newValue()) - } - // Now that we're sure there's a value in the map, let's - // try to get the pointer we need out of it. - ptr = wp.Strong() - if ptr != nil { - break - } - // The weak pointer is nil, so the old value is truly dead. - // Try to remove it and start over. - m.CompareAndDelete(value, wp) + // Find the value in the map. + ptr := m.Load(value) + if ptr == nil { + // Insert a new value into the map. + ptr = m.LoadOrStore(clone(value, &m.cloneSeq)) } - runtime.KeepAlive(toInsert) return Handle[T]{ptr} } -var ( - // uniqueMaps is an index of type-specific concurrent maps used for unique.Make. - // - // The two-level map might seem odd at first since the HashTrieMap could have "any" - // as its key type, but the issue is escape analysis. We do not want to force lookups - // to escape the argument, and using a type-specific map allows us to avoid that where - // possible (for example, for strings and plain-ol'-data structs). We also get the - // benefit of not cramming every different type into a single map, but that's certainly - // not enough to outweigh the cost of two map lookups. What is worth it though, is saving - // on those allocations. - uniqueMaps = concurrent.NewHashTrieMap[*abi.Type, any]() // any is always a *uniqueMap[T]. - - // cleanupFuncs are functions that clean up dead weak pointers in type-specific - // maps in uniqueMaps. We express cleanup this way because there's no way to iterate - // over the sync.Map and call functions on the type-specific data structures otherwise. - // These cleanup funcs each close over one of these type-specific maps. - // - // cleanupMu protects cleanupNotify and is held across the entire cleanup. Used for testing. - // cleanupNotify is a test-only mechanism that allow tests to wait for the cleanup to run. - cleanupMu sync.Mutex - cleanupFuncsMu sync.Mutex - cleanupFuncs []func() - cleanupNotify []func() // One-time notifications when cleanups finish. -) +// uniqueMaps is an index of type-specific concurrent maps used for unique.Make. +// +// The two-level map might seem odd at first since the HashTrieMap could have "any" +// as its key type, but the issue is escape analysis. We do not want to force lookups +// to escape the argument, and using a type-specific map allows us to avoid that where +// possible (for example, for strings and plain-ol'-data structs). We also get the +// benefit of not cramming every different type into a single map, but that's certainly +// not enough to outweigh the cost of two map lookups. What is worth it though, is saving +// on those allocations. +var uniqueMaps isync.HashTrieMap[*abi.Type, any] // any is always a *uniqueMap[T]. type uniqueMap[T comparable] struct { - *concurrent.HashTrieMap[T, weak.Pointer[T]] + *canonMap[T] cloneSeq } - -func addUniqueMap[T comparable](typ *abi.Type) *uniqueMap[T] { - // Create a map for T and try to register it. We could - // race with someone else, but that's fine; it's one - // small, stray allocation. The number of allocations - // this can create is bounded by a small constant. - m := &uniqueMap[T]{ - HashTrieMap: concurrent.NewHashTrieMap[T, weak.Pointer[T]](), - cloneSeq: makeCloneSeq(typ), - } - a, loaded := uniqueMaps.LoadOrStore(typ, m) - if !loaded { - // Add a cleanup function for the new map. - cleanupFuncsMu.Lock() - cleanupFuncs = append(cleanupFuncs, func() { - // Delete all the entries whose weak references are nil and clean up - // deleted entries. - m.All()(func(key T, wp weak.Pointer[T]) bool { - if wp.Strong() == nil { - m.CompareAndDelete(key, wp) - } - return true - }) - }) - cleanupFuncsMu.Unlock() - } - return a.(*uniqueMap[T]) -} - -// setupMake is used to perform initial setup for unique.Make. -var setupMake sync.Once - -// startBackgroundCleanup sets up a background goroutine to occasionally call cleanupFuncs. -func registerCleanup() { - runtime_registerUniqueMapCleanup(func() { - // Lock for cleanup. - cleanupMu.Lock() - - // Grab funcs to run. - cleanupFuncsMu.Lock() - cf := cleanupFuncs - cleanupFuncsMu.Unlock() - - // Run cleanup. - for _, f := range cf { - f() - } - - // Run cleanup notifications. - for _, f := range cleanupNotify { - f() - } - cleanupNotify = nil - - // Finished. - cleanupMu.Unlock() - }) -} - -// Implemented in runtime. - -//go:linkname runtime_registerUniqueMapCleanup -func runtime_registerUniqueMapCleanup(cleanup func()) diff --git a/src/unique/handle_test.go b/src/unique/handle_test.go index dffe10ac728189..4c7f1f9752e4d2 100644 --- a/src/unique/handle_test.go +++ b/src/unique/handle_test.go @@ -7,9 +7,18 @@ package unique import ( "fmt" "internal/abi" + "internal/asan" + "internal/msan" + "internal/race" + "internal/testenv" + "math/rand/v2" "reflect" "runtime" + "strconv" + "strings" "testing" + "time" + "unsafe" ) // Set up special types. Because the internal maps are sharded by type, @@ -28,26 +37,31 @@ type testStruct struct { z float64 b string } +type testZeroSize struct{} +type testNestedHandle struct { + next Handle[testNestedHandle] + arr [6]int +} func TestHandle(t *testing.T) { - testHandle[testString](t, "foo") - testHandle[testString](t, "bar") - testHandle[testString](t, "") - testHandle[testIntArray](t, [4]int{7, 77, 777, 7777}) - testHandle[testEface](t, nil) - testHandle[testStringArray](t, [3]string{"a", "b", "c"}) - testHandle[testStringStruct](t, testStringStruct{"x"}) - testHandle[testStringStructArrayStruct](t, testStringStructArrayStruct{ - s: [2]testStringStruct{testStringStruct{"y"}, testStringStruct{"z"}}, + testHandle(t, testString("foo")) + testHandle(t, testString("bar")) + testHandle(t, testString("")) + testHandle(t, testIntArray{7, 77, 777, 7777}) + testHandle(t, testEface(nil)) + testHandle(t, testStringArray{"a", "b", "c"}) + testHandle(t, testStringStruct{"x"}) + testHandle(t, testStringStructArrayStruct{ + s: [2]testStringStruct{{"y"}, {"z"}}, }) - testHandle[testStruct](t, testStruct{0.5, "184"}) + testHandle(t, testStruct{0.5, "184"}) + testHandle(t, testEface("hello")) + testHandle(t, testZeroSize(struct{}{})) } func testHandle[T comparable](t *testing.T, value T) { name := reflect.TypeFor[T]().Name() t.Run(fmt.Sprintf("%s/%#v", name, value), func(t *testing.T) { - t.Parallel() - v0 := Make(value) v1 := Make(value) @@ -61,51 +75,222 @@ func testHandle[T comparable](t *testing.T, value T) { t.Error("v0 != v1") } - drainMaps(t) + drainMaps[T](t) checkMapsFor(t, value) }) } // drainMaps ensures that the internal maps are drained. -func drainMaps(t *testing.T) { +func drainMaps[T comparable](t *testing.T) { t.Helper() - wait := make(chan struct{}, 1) - - // Set up a one-time notification for the next time the cleanup runs. - // Note: this will only run if there's no other active cleanup, so - // we can be sure that the next time cleanup runs, it'll see the new - // notification. - cleanupMu.Lock() - cleanupNotify = append(cleanupNotify, func() { - select { - case wait <- struct{}{}: - default: - } - }) + if unsafe.Sizeof(*(new(T))) == 0 { + return // zero-size types are not inserted. + } + drainCleanupQueue(t) +} - runtime.GC() - cleanupMu.Unlock() +func drainCleanupQueue(t *testing.T) { + t.Helper() - // Wait until cleanup runs. - <-wait + runtime.GC() // Queue up the cleanups. + runtime_blockUntilEmptyCleanupQueue(int64(5 * time.Second)) } func checkMapsFor[T comparable](t *testing.T, value T) { // Manually load the value out of the map. - typ := abi.TypeOf(value) + typ := abi.TypeFor[T]() a, ok := uniqueMaps.Load(typ) if !ok { return } m := a.(*uniqueMap[T]) - wp, ok := m.Load(value) - if !ok { - return + p := m.Load(value) + if p != nil { + t.Errorf("value %v still referenced by a handle (or tiny block?): internal pointer %p", value, p) } - if wp.Strong() != nil { - t.Errorf("value %v still referenced a handle (or tiny block?) ", value) - return +} + +func TestMakeClonesStrings(t *testing.T) { + s := strings.Clone("abcdefghijklmnopqrstuvwxyz") // N.B. Must be big enough to not be tiny-allocated. + ran := make(chan bool) + runtime.AddCleanup(unsafe.StringData(s), func(ch chan bool) { + ch <- true + }, ran) + h := Make(s) + + // Clean up s (hopefully) and run the cleanup. + runtime.GC() + + select { + case <-time.After(1 * time.Second): + t.Fatal("string was improperly retained") + case <-ran: + } + runtime.KeepAlive(h) +} + +func TestHandleUnsafeString(t *testing.T) { + var testData []string + for i := range 1024 { + testData = append(testData, strconv.Itoa(i)) + } + var buf []byte + var handles []Handle[string] + for _, s := range testData { + if len(buf) < len(s) { + buf = make([]byte, len(s)*2) + } + copy(buf, s) + sbuf := unsafe.String(&buf[0], len(s)) + handles = append(handles, Make(sbuf)) + } + for i, s := range testData { + h := Make(s) + if handles[i].Value() != h.Value() { + t.Fatal("unsafe string improperly retained internally") + } + } +} + +func nestHandle(n testNestedHandle) testNestedHandle { + return testNestedHandle{ + next: Make(n), + arr: n.arr, + } +} + +func TestNestedHandle(t *testing.T) { + n0 := testNestedHandle{arr: [6]int{1, 2, 3, 4, 5, 6}} + n1 := nestHandle(n0) + n2 := nestHandle(n1) + n3 := nestHandle(n2) + + if v := n3.next.Value(); v != n2 { + t.Errorf("n3.Value != n2: %#v vs. %#v", v, n2) + } + if v := n2.next.Value(); v != n1 { + t.Errorf("n2.Value != n1: %#v vs. %#v", v, n1) + } + if v := n1.next.Value(); v != n0 { + t.Errorf("n1.Value != n0: %#v vs. %#v", v, n0) } - t.Errorf("failed to drain internal maps of %v", value) + + // In a good implementation, the entire chain, down to the bottom-most + // value, should all be gone after we drain the maps. + drainMaps[testNestedHandle](t) + checkMapsFor(t, n0) +} + +// Implemented in runtime. +// +// Used only by tests. +// +//go:linkname runtime_blockUntilEmptyCleanupQueue +func runtime_blockUntilEmptyCleanupQueue(timeout int64) bool + +var ( + randomNumber = rand.IntN(1000000) + 1000000 + heapBytes = newHeapBytes() + heapString = newHeapString() + + stringHandle Handle[string] + intHandle Handle[int] + anyHandle Handle[any] + pairHandle Handle[[2]string] +) + +func TestMakeAllocs(t *testing.T) { + errorf := t.Errorf + if race.Enabled || msan.Enabled || asan.Enabled || testenv.OptimizationOff() { + errorf = t.Logf + } + + tests := []struct { + name string + allocs int + f func() + }{ + {name: "create heap bytes", allocs: 1, f: func() { + heapBytes = newHeapBytes() + }}, + + {name: "create heap string", allocs: 2, f: func() { + heapString = newHeapString() + }}, + + {name: "static string", allocs: 0, f: func() { + stringHandle = Make("this string is statically allocated") + }}, + + {name: "heap string", allocs: 0, f: func() { + stringHandle = Make(heapString) + }}, + + {name: "stack string", allocs: 0, f: func() { + var b [16]byte + b[8] = 'a' + stringHandle = Make(string(b[:])) + }}, + + {name: "bytes", allocs: 0, f: func() { + stringHandle = Make(string(heapBytes)) + }}, + + {name: "bytes truncated short", allocs: 0, f: func() { + stringHandle = Make(string(heapBytes[:16])) + }}, + + {name: "bytes truncated long", allocs: 0, f: func() { + stringHandle = Make(string(heapBytes[:40])) + }}, + + {name: "string to any", allocs: 1, f: func() { + anyHandle = Make[any](heapString) + }}, + + {name: "large number", allocs: 0, f: func() { + intHandle = Make(randomNumber) + }}, + + {name: "large number to any", allocs: 1, f: func() { + anyHandle = Make[any](randomNumber) + }}, + + {name: "pair", allocs: 0, f: func() { + pairHandle = Make([2]string{heapString, heapString}) + }}, + + {name: "pair from stack", allocs: 0, f: func() { + var b [16]byte + b[8] = 'a' + pairHandle = Make([2]string{string(b[:]), string(b[:])}) + }}, + + {name: "pair to any", allocs: 1, f: func() { + anyHandle = Make[any]([2]string{heapString, heapString}) + }}, + } + + for _, tt := range tests { + allocs := testing.AllocsPerRun(100, tt.f) + if allocs != float64(tt.allocs) { + errorf("%s: got %v allocs, want %v", tt.name, allocs, tt.allocs) + } + } +} + +//go:noinline +func newHeapBytes() []byte { + const N = 100 + b := make([]byte, N) + for i := range b { + b[i] = byte(i) + } + return b +} + +//go:noinline +func newHeapString() string { + return string(newHeapBytes()) } diff --git a/src/vendor/golang.org/x/crypto/chacha20/chacha_noasm.go b/src/vendor/golang.org/x/crypto/chacha20/chacha_noasm.go index db42e6676ab35b..c709b728477d10 100644 --- a/src/vendor/golang.org/x/crypto/chacha20/chacha_noasm.go +++ b/src/vendor/golang.org/x/crypto/chacha20/chacha_noasm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (!arm64 && !s390x && !ppc64le) || !gc || purego +//go:build (!arm64 && !s390x && !ppc64 && !ppc64le) || !gc || purego package chacha20 diff --git a/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.go b/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.go deleted file mode 100644 index 3a4287f9900e40..00000000000000 --- a/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build gc && !purego - -package chacha20 - -const bufSize = 256 - -//go:noescape -func chaCha20_ctr32_vsx(out, inp *byte, len int, key *[8]uint32, counter *uint32) - -func (c *Cipher) xorKeyStreamBlocks(dst, src []byte) { - chaCha20_ctr32_vsx(&dst[0], &src[0], len(src), &c.key, &c.counter) -} diff --git a/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.s b/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.s deleted file mode 100644 index c672ccf6986b08..00000000000000 --- a/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.s +++ /dev/null @@ -1,443 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Based on CRYPTOGAMS code with the following comment: -// # ==================================================================== -// # Written by Andy Polyakov for the OpenSSL -// # project. The module is, however, dual licensed under OpenSSL and -// # CRYPTOGAMS licenses depending on where you obtain it. For further -// # details see http://www.openssl.org/~appro/cryptogams/. -// # ==================================================================== - -// Code for the perl script that generates the ppc64 assembler -// can be found in the cryptogams repository at the link below. It is based on -// the original from openssl. - -// https://github.com/dot-asm/cryptogams/commit/a60f5b50ed908e91 - -// The differences in this and the original implementation are -// due to the calling conventions and initialization of constants. - -//go:build gc && !purego - -#include "textflag.h" - -#define OUT R3 -#define INP R4 -#define LEN R5 -#define KEY R6 -#define CNT R7 -#define TMP R15 - -#define CONSTBASE R16 -#define BLOCKS R17 - -// for VPERMXOR -#define MASK R18 - -DATA consts<>+0x00(SB)/8, $0x3320646e61707865 -DATA consts<>+0x08(SB)/8, $0x6b20657479622d32 -DATA consts<>+0x10(SB)/8, $0x0000000000000001 -DATA consts<>+0x18(SB)/8, $0x0000000000000000 -DATA consts<>+0x20(SB)/8, $0x0000000000000004 -DATA consts<>+0x28(SB)/8, $0x0000000000000000 -DATA consts<>+0x30(SB)/8, $0x0a0b08090e0f0c0d -DATA consts<>+0x38(SB)/8, $0x0203000106070405 -DATA consts<>+0x40(SB)/8, $0x090a0b080d0e0f0c -DATA consts<>+0x48(SB)/8, $0x0102030005060704 -DATA consts<>+0x50(SB)/8, $0x6170786561707865 -DATA consts<>+0x58(SB)/8, $0x6170786561707865 -DATA consts<>+0x60(SB)/8, $0x3320646e3320646e -DATA consts<>+0x68(SB)/8, $0x3320646e3320646e -DATA consts<>+0x70(SB)/8, $0x79622d3279622d32 -DATA consts<>+0x78(SB)/8, $0x79622d3279622d32 -DATA consts<>+0x80(SB)/8, $0x6b2065746b206574 -DATA consts<>+0x88(SB)/8, $0x6b2065746b206574 -DATA consts<>+0x90(SB)/8, $0x0000000100000000 -DATA consts<>+0x98(SB)/8, $0x0000000300000002 -DATA consts<>+0xa0(SB)/8, $0x5566774411223300 -DATA consts<>+0xa8(SB)/8, $0xddeeffcc99aabb88 -DATA consts<>+0xb0(SB)/8, $0x6677445522330011 -DATA consts<>+0xb8(SB)/8, $0xeeffccddaabb8899 -GLOBL consts<>(SB), RODATA, $0xc0 - -//func chaCha20_ctr32_vsx(out, inp *byte, len int, key *[8]uint32, counter *uint32) -TEXT ·chaCha20_ctr32_vsx(SB),NOSPLIT,$64-40 - MOVD out+0(FP), OUT - MOVD inp+8(FP), INP - MOVD len+16(FP), LEN - MOVD key+24(FP), KEY - MOVD counter+32(FP), CNT - - // Addressing for constants - MOVD $consts<>+0x00(SB), CONSTBASE - MOVD $16, R8 - MOVD $32, R9 - MOVD $48, R10 - MOVD $64, R11 - SRD $6, LEN, BLOCKS - // for VPERMXOR - MOVD $consts<>+0xa0(SB), MASK - MOVD $16, R20 - // V16 - LXVW4X (CONSTBASE)(R0), VS48 - ADD $80,CONSTBASE - - // Load key into V17,V18 - LXVW4X (KEY)(R0), VS49 - LXVW4X (KEY)(R8), VS50 - - // Load CNT, NONCE into V19 - LXVW4X (CNT)(R0), VS51 - - // Clear V27 - VXOR V27, V27, V27 - - // V28 - LXVW4X (CONSTBASE)(R11), VS60 - - // Load mask constants for VPERMXOR - LXVW4X (MASK)(R0), V20 - LXVW4X (MASK)(R20), V21 - - // splat slot from V19 -> V26 - VSPLTW $0, V19, V26 - - VSLDOI $4, V19, V27, V19 - VSLDOI $12, V27, V19, V19 - - VADDUWM V26, V28, V26 - - MOVD $10, R14 - MOVD R14, CTR - PCALIGN $16 -loop_outer_vsx: - // V0, V1, V2, V3 - LXVW4X (R0)(CONSTBASE), VS32 - LXVW4X (R8)(CONSTBASE), VS33 - LXVW4X (R9)(CONSTBASE), VS34 - LXVW4X (R10)(CONSTBASE), VS35 - - // splat values from V17, V18 into V4-V11 - VSPLTW $0, V17, V4 - VSPLTW $1, V17, V5 - VSPLTW $2, V17, V6 - VSPLTW $3, V17, V7 - VSPLTW $0, V18, V8 - VSPLTW $1, V18, V9 - VSPLTW $2, V18, V10 - VSPLTW $3, V18, V11 - - // VOR - VOR V26, V26, V12 - - // splat values from V19 -> V13, V14, V15 - VSPLTW $1, V19, V13 - VSPLTW $2, V19, V14 - VSPLTW $3, V19, V15 - - // splat const values - VSPLTISW $-16, V27 - VSPLTISW $12, V28 - VSPLTISW $8, V29 - VSPLTISW $7, V30 - PCALIGN $16 -loop_vsx: - VADDUWM V0, V4, V0 - VADDUWM V1, V5, V1 - VADDUWM V2, V6, V2 - VADDUWM V3, V7, V3 - - VPERMXOR V12, V0, V21, V12 - VPERMXOR V13, V1, V21, V13 - VPERMXOR V14, V2, V21, V14 - VPERMXOR V15, V3, V21, V15 - - VADDUWM V8, V12, V8 - VADDUWM V9, V13, V9 - VADDUWM V10, V14, V10 - VADDUWM V11, V15, V11 - - VXOR V4, V8, V4 - VXOR V5, V9, V5 - VXOR V6, V10, V6 - VXOR V7, V11, V7 - - VRLW V4, V28, V4 - VRLW V5, V28, V5 - VRLW V6, V28, V6 - VRLW V7, V28, V7 - - VADDUWM V0, V4, V0 - VADDUWM V1, V5, V1 - VADDUWM V2, V6, V2 - VADDUWM V3, V7, V3 - - VPERMXOR V12, V0, V20, V12 - VPERMXOR V13, V1, V20, V13 - VPERMXOR V14, V2, V20, V14 - VPERMXOR V15, V3, V20, V15 - - VADDUWM V8, V12, V8 - VADDUWM V9, V13, V9 - VADDUWM V10, V14, V10 - VADDUWM V11, V15, V11 - - VXOR V4, V8, V4 - VXOR V5, V9, V5 - VXOR V6, V10, V6 - VXOR V7, V11, V7 - - VRLW V4, V30, V4 - VRLW V5, V30, V5 - VRLW V6, V30, V6 - VRLW V7, V30, V7 - - VADDUWM V0, V5, V0 - VADDUWM V1, V6, V1 - VADDUWM V2, V7, V2 - VADDUWM V3, V4, V3 - - VPERMXOR V15, V0, V21, V15 - VPERMXOR V12, V1, V21, V12 - VPERMXOR V13, V2, V21, V13 - VPERMXOR V14, V3, V21, V14 - - VADDUWM V10, V15, V10 - VADDUWM V11, V12, V11 - VADDUWM V8, V13, V8 - VADDUWM V9, V14, V9 - - VXOR V5, V10, V5 - VXOR V6, V11, V6 - VXOR V7, V8, V7 - VXOR V4, V9, V4 - - VRLW V5, V28, V5 - VRLW V6, V28, V6 - VRLW V7, V28, V7 - VRLW V4, V28, V4 - - VADDUWM V0, V5, V0 - VADDUWM V1, V6, V1 - VADDUWM V2, V7, V2 - VADDUWM V3, V4, V3 - - VPERMXOR V15, V0, V20, V15 - VPERMXOR V12, V1, V20, V12 - VPERMXOR V13, V2, V20, V13 - VPERMXOR V14, V3, V20, V14 - - VADDUWM V10, V15, V10 - VADDUWM V11, V12, V11 - VADDUWM V8, V13, V8 - VADDUWM V9, V14, V9 - - VXOR V5, V10, V5 - VXOR V6, V11, V6 - VXOR V7, V8, V7 - VXOR V4, V9, V4 - - VRLW V5, V30, V5 - VRLW V6, V30, V6 - VRLW V7, V30, V7 - VRLW V4, V30, V4 - BDNZ loop_vsx - - VADDUWM V12, V26, V12 - - VMRGEW V0, V1, V27 - VMRGEW V2, V3, V28 - - VMRGOW V0, V1, V0 - VMRGOW V2, V3, V2 - - VMRGEW V4, V5, V29 - VMRGEW V6, V7, V30 - - XXPERMDI VS32, VS34, $0, VS33 - XXPERMDI VS32, VS34, $3, VS35 - XXPERMDI VS59, VS60, $0, VS32 - XXPERMDI VS59, VS60, $3, VS34 - - VMRGOW V4, V5, V4 - VMRGOW V6, V7, V6 - - VMRGEW V8, V9, V27 - VMRGEW V10, V11, V28 - - XXPERMDI VS36, VS38, $0, VS37 - XXPERMDI VS36, VS38, $3, VS39 - XXPERMDI VS61, VS62, $0, VS36 - XXPERMDI VS61, VS62, $3, VS38 - - VMRGOW V8, V9, V8 - VMRGOW V10, V11, V10 - - VMRGEW V12, V13, V29 - VMRGEW V14, V15, V30 - - XXPERMDI VS40, VS42, $0, VS41 - XXPERMDI VS40, VS42, $3, VS43 - XXPERMDI VS59, VS60, $0, VS40 - XXPERMDI VS59, VS60, $3, VS42 - - VMRGOW V12, V13, V12 - VMRGOW V14, V15, V14 - - VSPLTISW $4, V27 - VADDUWM V26, V27, V26 - - XXPERMDI VS44, VS46, $0, VS45 - XXPERMDI VS44, VS46, $3, VS47 - XXPERMDI VS61, VS62, $0, VS44 - XXPERMDI VS61, VS62, $3, VS46 - - VADDUWM V0, V16, V0 - VADDUWM V4, V17, V4 - VADDUWM V8, V18, V8 - VADDUWM V12, V19, V12 - - CMPU LEN, $64 - BLT tail_vsx - - // Bottom of loop - LXVW4X (INP)(R0), VS59 - LXVW4X (INP)(R8), VS60 - LXVW4X (INP)(R9), VS61 - LXVW4X (INP)(R10), VS62 - - VXOR V27, V0, V27 - VXOR V28, V4, V28 - VXOR V29, V8, V29 - VXOR V30, V12, V30 - - STXVW4X VS59, (OUT)(R0) - STXVW4X VS60, (OUT)(R8) - ADD $64, INP - STXVW4X VS61, (OUT)(R9) - ADD $-64, LEN - STXVW4X VS62, (OUT)(R10) - ADD $64, OUT - BEQ done_vsx - - VADDUWM V1, V16, V0 - VADDUWM V5, V17, V4 - VADDUWM V9, V18, V8 - VADDUWM V13, V19, V12 - - CMPU LEN, $64 - BLT tail_vsx - - LXVW4X (INP)(R0), VS59 - LXVW4X (INP)(R8), VS60 - LXVW4X (INP)(R9), VS61 - LXVW4X (INP)(R10), VS62 - VXOR V27, V0, V27 - - VXOR V28, V4, V28 - VXOR V29, V8, V29 - VXOR V30, V12, V30 - - STXVW4X VS59, (OUT)(R0) - STXVW4X VS60, (OUT)(R8) - ADD $64, INP - STXVW4X VS61, (OUT)(R9) - ADD $-64, LEN - STXVW4X VS62, (OUT)(V10) - ADD $64, OUT - BEQ done_vsx - - VADDUWM V2, V16, V0 - VADDUWM V6, V17, V4 - VADDUWM V10, V18, V8 - VADDUWM V14, V19, V12 - - CMPU LEN, $64 - BLT tail_vsx - - LXVW4X (INP)(R0), VS59 - LXVW4X (INP)(R8), VS60 - LXVW4X (INP)(R9), VS61 - LXVW4X (INP)(R10), VS62 - - VXOR V27, V0, V27 - VXOR V28, V4, V28 - VXOR V29, V8, V29 - VXOR V30, V12, V30 - - STXVW4X VS59, (OUT)(R0) - STXVW4X VS60, (OUT)(R8) - ADD $64, INP - STXVW4X VS61, (OUT)(R9) - ADD $-64, LEN - STXVW4X VS62, (OUT)(R10) - ADD $64, OUT - BEQ done_vsx - - VADDUWM V3, V16, V0 - VADDUWM V7, V17, V4 - VADDUWM V11, V18, V8 - VADDUWM V15, V19, V12 - - CMPU LEN, $64 - BLT tail_vsx - - LXVW4X (INP)(R0), VS59 - LXVW4X (INP)(R8), VS60 - LXVW4X (INP)(R9), VS61 - LXVW4X (INP)(R10), VS62 - - VXOR V27, V0, V27 - VXOR V28, V4, V28 - VXOR V29, V8, V29 - VXOR V30, V12, V30 - - STXVW4X VS59, (OUT)(R0) - STXVW4X VS60, (OUT)(R8) - ADD $64, INP - STXVW4X VS61, (OUT)(R9) - ADD $-64, LEN - STXVW4X VS62, (OUT)(R10) - ADD $64, OUT - - MOVD $10, R14 - MOVD R14, CTR - BNE loop_outer_vsx - -done_vsx: - // Increment counter by number of 64 byte blocks - MOVD (CNT), R14 - ADD BLOCKS, R14 - MOVD R14, (CNT) - RET - -tail_vsx: - ADD $32, R1, R11 - MOVD LEN, CTR - - // Save values on stack to copy from - STXVW4X VS32, (R11)(R0) - STXVW4X VS36, (R11)(R8) - STXVW4X VS40, (R11)(R9) - STXVW4X VS44, (R11)(R10) - ADD $-1, R11, R12 - ADD $-1, INP - ADD $-1, OUT - PCALIGN $16 -looptail_vsx: - // Copying the result to OUT - // in bytes. - MOVBZU 1(R12), KEY - MOVBZU 1(INP), TMP - XOR KEY, TMP, KEY - MOVBU KEY, 1(OUT) - BDNZ looptail_vsx - - // Clear the stack values - STXVW4X VS48, (R11)(R0) - STXVW4X VS48, (R11)(R8) - STXVW4X VS48, (R11)(R9) - STXVW4X VS48, (R11)(R10) - BR done_vsx diff --git a/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.go b/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.go new file mode 100644 index 00000000000000..bd183d9ba1243e --- /dev/null +++ b/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.go @@ -0,0 +1,16 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gc && !purego && (ppc64 || ppc64le) + +package chacha20 + +const bufSize = 256 + +//go:noescape +func chaCha20_ctr32_vsx(out, inp *byte, len int, key *[8]uint32, counter *uint32) + +func (c *Cipher) xorKeyStreamBlocks(dst, src []byte) { + chaCha20_ctr32_vsx(&dst[0], &src[0], len(src), &c.key, &c.counter) +} diff --git a/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.s b/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.s new file mode 100644 index 00000000000000..a660b4112fafd9 --- /dev/null +++ b/src/vendor/golang.org/x/crypto/chacha20/chacha_ppc64x.s @@ -0,0 +1,501 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Based on CRYPTOGAMS code with the following comment: +// # ==================================================================== +// # Written by Andy Polyakov for the OpenSSL +// # project. The module is, however, dual licensed under OpenSSL and +// # CRYPTOGAMS licenses depending on where you obtain it. For further +// # details see http://www.openssl.org/~appro/cryptogams/. +// # ==================================================================== + +// Code for the perl script that generates the ppc64 assembler +// can be found in the cryptogams repository at the link below. It is based on +// the original from openssl. + +// https://github.com/dot-asm/cryptogams/commit/a60f5b50ed908e91 + +// The differences in this and the original implementation are +// due to the calling conventions and initialization of constants. + +//go:build gc && !purego && (ppc64 || ppc64le) + +#include "textflag.h" + +#define OUT R3 +#define INP R4 +#define LEN R5 +#define KEY R6 +#define CNT R7 +#define TMP R15 + +#define CONSTBASE R16 +#define BLOCKS R17 + +// for VPERMXOR +#define MASK R18 + +DATA consts<>+0x00(SB)/4, $0x61707865 +DATA consts<>+0x04(SB)/4, $0x3320646e +DATA consts<>+0x08(SB)/4, $0x79622d32 +DATA consts<>+0x0c(SB)/4, $0x6b206574 +DATA consts<>+0x10(SB)/4, $0x00000001 +DATA consts<>+0x14(SB)/4, $0x00000000 +DATA consts<>+0x18(SB)/4, $0x00000000 +DATA consts<>+0x1c(SB)/4, $0x00000000 +DATA consts<>+0x20(SB)/4, $0x00000004 +DATA consts<>+0x24(SB)/4, $0x00000000 +DATA consts<>+0x28(SB)/4, $0x00000000 +DATA consts<>+0x2c(SB)/4, $0x00000000 +DATA consts<>+0x30(SB)/4, $0x0e0f0c0d +DATA consts<>+0x34(SB)/4, $0x0a0b0809 +DATA consts<>+0x38(SB)/4, $0x06070405 +DATA consts<>+0x3c(SB)/4, $0x02030001 +DATA consts<>+0x40(SB)/4, $0x0d0e0f0c +DATA consts<>+0x44(SB)/4, $0x090a0b08 +DATA consts<>+0x48(SB)/4, $0x05060704 +DATA consts<>+0x4c(SB)/4, $0x01020300 +DATA consts<>+0x50(SB)/4, $0x61707865 +DATA consts<>+0x54(SB)/4, $0x61707865 +DATA consts<>+0x58(SB)/4, $0x61707865 +DATA consts<>+0x5c(SB)/4, $0x61707865 +DATA consts<>+0x60(SB)/4, $0x3320646e +DATA consts<>+0x64(SB)/4, $0x3320646e +DATA consts<>+0x68(SB)/4, $0x3320646e +DATA consts<>+0x6c(SB)/4, $0x3320646e +DATA consts<>+0x70(SB)/4, $0x79622d32 +DATA consts<>+0x74(SB)/4, $0x79622d32 +DATA consts<>+0x78(SB)/4, $0x79622d32 +DATA consts<>+0x7c(SB)/4, $0x79622d32 +DATA consts<>+0x80(SB)/4, $0x6b206574 +DATA consts<>+0x84(SB)/4, $0x6b206574 +DATA consts<>+0x88(SB)/4, $0x6b206574 +DATA consts<>+0x8c(SB)/4, $0x6b206574 +DATA consts<>+0x90(SB)/4, $0x00000000 +DATA consts<>+0x94(SB)/4, $0x00000001 +DATA consts<>+0x98(SB)/4, $0x00000002 +DATA consts<>+0x9c(SB)/4, $0x00000003 +DATA consts<>+0xa0(SB)/4, $0x11223300 +DATA consts<>+0xa4(SB)/4, $0x55667744 +DATA consts<>+0xa8(SB)/4, $0x99aabb88 +DATA consts<>+0xac(SB)/4, $0xddeeffcc +DATA consts<>+0xb0(SB)/4, $0x22330011 +DATA consts<>+0xb4(SB)/4, $0x66774455 +DATA consts<>+0xb8(SB)/4, $0xaabb8899 +DATA consts<>+0xbc(SB)/4, $0xeeffccdd +GLOBL consts<>(SB), RODATA, $0xc0 + +#ifdef GOARCH_ppc64 +#define BE_XXBRW_INIT() \ + LVSL (R0)(R0), V24 \ + VSPLTISB $3, V25 \ + VXOR V24, V25, V24 \ + +#define BE_XXBRW(vr) VPERM vr, vr, V24, vr +#else +#define BE_XXBRW_INIT() +#define BE_XXBRW(vr) +#endif + +//func chaCha20_ctr32_vsx(out, inp *byte, len int, key *[8]uint32, counter *uint32) +TEXT ·chaCha20_ctr32_vsx(SB),NOSPLIT,$64-40 + MOVD out+0(FP), OUT + MOVD inp+8(FP), INP + MOVD len+16(FP), LEN + MOVD key+24(FP), KEY + MOVD counter+32(FP), CNT + + // Addressing for constants + MOVD $consts<>+0x00(SB), CONSTBASE + MOVD $16, R8 + MOVD $32, R9 + MOVD $48, R10 + MOVD $64, R11 + SRD $6, LEN, BLOCKS + // for VPERMXOR + MOVD $consts<>+0xa0(SB), MASK + MOVD $16, R20 + // V16 + LXVW4X (CONSTBASE)(R0), VS48 + ADD $80,CONSTBASE + + // Load key into V17,V18 + LXVW4X (KEY)(R0), VS49 + LXVW4X (KEY)(R8), VS50 + + // Load CNT, NONCE into V19 + LXVW4X (CNT)(R0), VS51 + + // Clear V27 + VXOR V27, V27, V27 + + BE_XXBRW_INIT() + + // V28 + LXVW4X (CONSTBASE)(R11), VS60 + + // Load mask constants for VPERMXOR + LXVW4X (MASK)(R0), V20 + LXVW4X (MASK)(R20), V21 + + // splat slot from V19 -> V26 + VSPLTW $0, V19, V26 + + VSLDOI $4, V19, V27, V19 + VSLDOI $12, V27, V19, V19 + + VADDUWM V26, V28, V26 + + MOVD $10, R14 + MOVD R14, CTR + PCALIGN $16 +loop_outer_vsx: + // V0, V1, V2, V3 + LXVW4X (R0)(CONSTBASE), VS32 + LXVW4X (R8)(CONSTBASE), VS33 + LXVW4X (R9)(CONSTBASE), VS34 + LXVW4X (R10)(CONSTBASE), VS35 + + // splat values from V17, V18 into V4-V11 + VSPLTW $0, V17, V4 + VSPLTW $1, V17, V5 + VSPLTW $2, V17, V6 + VSPLTW $3, V17, V7 + VSPLTW $0, V18, V8 + VSPLTW $1, V18, V9 + VSPLTW $2, V18, V10 + VSPLTW $3, V18, V11 + + // VOR + VOR V26, V26, V12 + + // splat values from V19 -> V13, V14, V15 + VSPLTW $1, V19, V13 + VSPLTW $2, V19, V14 + VSPLTW $3, V19, V15 + + // splat const values + VSPLTISW $-16, V27 + VSPLTISW $12, V28 + VSPLTISW $8, V29 + VSPLTISW $7, V30 + PCALIGN $16 +loop_vsx: + VADDUWM V0, V4, V0 + VADDUWM V1, V5, V1 + VADDUWM V2, V6, V2 + VADDUWM V3, V7, V3 + + VPERMXOR V12, V0, V21, V12 + VPERMXOR V13, V1, V21, V13 + VPERMXOR V14, V2, V21, V14 + VPERMXOR V15, V3, V21, V15 + + VADDUWM V8, V12, V8 + VADDUWM V9, V13, V9 + VADDUWM V10, V14, V10 + VADDUWM V11, V15, V11 + + VXOR V4, V8, V4 + VXOR V5, V9, V5 + VXOR V6, V10, V6 + VXOR V7, V11, V7 + + VRLW V4, V28, V4 + VRLW V5, V28, V5 + VRLW V6, V28, V6 + VRLW V7, V28, V7 + + VADDUWM V0, V4, V0 + VADDUWM V1, V5, V1 + VADDUWM V2, V6, V2 + VADDUWM V3, V7, V3 + + VPERMXOR V12, V0, V20, V12 + VPERMXOR V13, V1, V20, V13 + VPERMXOR V14, V2, V20, V14 + VPERMXOR V15, V3, V20, V15 + + VADDUWM V8, V12, V8 + VADDUWM V9, V13, V9 + VADDUWM V10, V14, V10 + VADDUWM V11, V15, V11 + + VXOR V4, V8, V4 + VXOR V5, V9, V5 + VXOR V6, V10, V6 + VXOR V7, V11, V7 + + VRLW V4, V30, V4 + VRLW V5, V30, V5 + VRLW V6, V30, V6 + VRLW V7, V30, V7 + + VADDUWM V0, V5, V0 + VADDUWM V1, V6, V1 + VADDUWM V2, V7, V2 + VADDUWM V3, V4, V3 + + VPERMXOR V15, V0, V21, V15 + VPERMXOR V12, V1, V21, V12 + VPERMXOR V13, V2, V21, V13 + VPERMXOR V14, V3, V21, V14 + + VADDUWM V10, V15, V10 + VADDUWM V11, V12, V11 + VADDUWM V8, V13, V8 + VADDUWM V9, V14, V9 + + VXOR V5, V10, V5 + VXOR V6, V11, V6 + VXOR V7, V8, V7 + VXOR V4, V9, V4 + + VRLW V5, V28, V5 + VRLW V6, V28, V6 + VRLW V7, V28, V7 + VRLW V4, V28, V4 + + VADDUWM V0, V5, V0 + VADDUWM V1, V6, V1 + VADDUWM V2, V7, V2 + VADDUWM V3, V4, V3 + + VPERMXOR V15, V0, V20, V15 + VPERMXOR V12, V1, V20, V12 + VPERMXOR V13, V2, V20, V13 + VPERMXOR V14, V3, V20, V14 + + VADDUWM V10, V15, V10 + VADDUWM V11, V12, V11 + VADDUWM V8, V13, V8 + VADDUWM V9, V14, V9 + + VXOR V5, V10, V5 + VXOR V6, V11, V6 + VXOR V7, V8, V7 + VXOR V4, V9, V4 + + VRLW V5, V30, V5 + VRLW V6, V30, V6 + VRLW V7, V30, V7 + VRLW V4, V30, V4 + BDNZ loop_vsx + + VADDUWM V12, V26, V12 + + VMRGEW V0, V1, V27 + VMRGEW V2, V3, V28 + + VMRGOW V0, V1, V0 + VMRGOW V2, V3, V2 + + VMRGEW V4, V5, V29 + VMRGEW V6, V7, V30 + + XXPERMDI VS32, VS34, $0, VS33 + XXPERMDI VS32, VS34, $3, VS35 + XXPERMDI VS59, VS60, $0, VS32 + XXPERMDI VS59, VS60, $3, VS34 + + VMRGOW V4, V5, V4 + VMRGOW V6, V7, V6 + + VMRGEW V8, V9, V27 + VMRGEW V10, V11, V28 + + XXPERMDI VS36, VS38, $0, VS37 + XXPERMDI VS36, VS38, $3, VS39 + XXPERMDI VS61, VS62, $0, VS36 + XXPERMDI VS61, VS62, $3, VS38 + + VMRGOW V8, V9, V8 + VMRGOW V10, V11, V10 + + VMRGEW V12, V13, V29 + VMRGEW V14, V15, V30 + + XXPERMDI VS40, VS42, $0, VS41 + XXPERMDI VS40, VS42, $3, VS43 + XXPERMDI VS59, VS60, $0, VS40 + XXPERMDI VS59, VS60, $3, VS42 + + VMRGOW V12, V13, V12 + VMRGOW V14, V15, V14 + + VSPLTISW $4, V27 + VADDUWM V26, V27, V26 + + XXPERMDI VS44, VS46, $0, VS45 + XXPERMDI VS44, VS46, $3, VS47 + XXPERMDI VS61, VS62, $0, VS44 + XXPERMDI VS61, VS62, $3, VS46 + + VADDUWM V0, V16, V0 + VADDUWM V4, V17, V4 + VADDUWM V8, V18, V8 + VADDUWM V12, V19, V12 + + BE_XXBRW(V0) + BE_XXBRW(V4) + BE_XXBRW(V8) + BE_XXBRW(V12) + + CMPU LEN, $64 + BLT tail_vsx + + // Bottom of loop + LXVW4X (INP)(R0), VS59 + LXVW4X (INP)(R8), VS60 + LXVW4X (INP)(R9), VS61 + LXVW4X (INP)(R10), VS62 + + VXOR V27, V0, V27 + VXOR V28, V4, V28 + VXOR V29, V8, V29 + VXOR V30, V12, V30 + + STXVW4X VS59, (OUT)(R0) + STXVW4X VS60, (OUT)(R8) + ADD $64, INP + STXVW4X VS61, (OUT)(R9) + ADD $-64, LEN + STXVW4X VS62, (OUT)(R10) + ADD $64, OUT + BEQ done_vsx + + VADDUWM V1, V16, V0 + VADDUWM V5, V17, V4 + VADDUWM V9, V18, V8 + VADDUWM V13, V19, V12 + + BE_XXBRW(V0) + BE_XXBRW(V4) + BE_XXBRW(V8) + BE_XXBRW(V12) + + CMPU LEN, $64 + BLT tail_vsx + + LXVW4X (INP)(R0), VS59 + LXVW4X (INP)(R8), VS60 + LXVW4X (INP)(R9), VS61 + LXVW4X (INP)(R10), VS62 + + VXOR V27, V0, V27 + VXOR V28, V4, V28 + VXOR V29, V8, V29 + VXOR V30, V12, V30 + + STXVW4X VS59, (OUT)(R0) + STXVW4X VS60, (OUT)(R8) + ADD $64, INP + STXVW4X VS61, (OUT)(R9) + ADD $-64, LEN + STXVW4X VS62, (OUT)(V10) + ADD $64, OUT + BEQ done_vsx + + VADDUWM V2, V16, V0 + VADDUWM V6, V17, V4 + VADDUWM V10, V18, V8 + VADDUWM V14, V19, V12 + + BE_XXBRW(V0) + BE_XXBRW(V4) + BE_XXBRW(V8) + BE_XXBRW(V12) + + CMPU LEN, $64 + BLT tail_vsx + + LXVW4X (INP)(R0), VS59 + LXVW4X (INP)(R8), VS60 + LXVW4X (INP)(R9), VS61 + LXVW4X (INP)(R10), VS62 + + VXOR V27, V0, V27 + VXOR V28, V4, V28 + VXOR V29, V8, V29 + VXOR V30, V12, V30 + + STXVW4X VS59, (OUT)(R0) + STXVW4X VS60, (OUT)(R8) + ADD $64, INP + STXVW4X VS61, (OUT)(R9) + ADD $-64, LEN + STXVW4X VS62, (OUT)(R10) + ADD $64, OUT + BEQ done_vsx + + VADDUWM V3, V16, V0 + VADDUWM V7, V17, V4 + VADDUWM V11, V18, V8 + VADDUWM V15, V19, V12 + + BE_XXBRW(V0) + BE_XXBRW(V4) + BE_XXBRW(V8) + BE_XXBRW(V12) + + CMPU LEN, $64 + BLT tail_vsx + + LXVW4X (INP)(R0), VS59 + LXVW4X (INP)(R8), VS60 + LXVW4X (INP)(R9), VS61 + LXVW4X (INP)(R10), VS62 + + VXOR V27, V0, V27 + VXOR V28, V4, V28 + VXOR V29, V8, V29 + VXOR V30, V12, V30 + + STXVW4X VS59, (OUT)(R0) + STXVW4X VS60, (OUT)(R8) + ADD $64, INP + STXVW4X VS61, (OUT)(R9) + ADD $-64, LEN + STXVW4X VS62, (OUT)(R10) + ADD $64, OUT + + MOVD $10, R14 + MOVD R14, CTR + BNE loop_outer_vsx + +done_vsx: + // Increment counter by number of 64 byte blocks + MOVWZ (CNT), R14 + ADD BLOCKS, R14 + MOVWZ R14, (CNT) + RET + +tail_vsx: + ADD $32, R1, R11 + MOVD LEN, CTR + + // Save values on stack to copy from + STXVW4X VS32, (R11)(R0) + STXVW4X VS36, (R11)(R8) + STXVW4X VS40, (R11)(R9) + STXVW4X VS44, (R11)(R10) + ADD $-1, R11, R12 + ADD $-1, INP + ADD $-1, OUT + PCALIGN $16 +looptail_vsx: + // Copying the result to OUT + // in bytes. + MOVBZU 1(R12), KEY + MOVBZU 1(INP), TMP + XOR KEY, TMP, KEY + MOVBU KEY, 1(OUT) + BDNZ looptail_vsx + + // Clear the stack values + STXVW4X VS48, (R11)(R0) + STXVW4X VS48, (R11)(R8) + STXVW4X VS48, (R11)(R9) + STXVW4X VS48, (R11)(R10) + BR done_vsx diff --git a/src/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s b/src/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s index 731d2ac6dbc11d..fd5ee845f9faac 100644 --- a/src/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s +++ b/src/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s @@ -1,2715 +1,9762 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file was originally from https://golang.org/cl/24717 by Vlad Krasnov of CloudFlare. +// Code generated by command: go run chacha20poly1305_amd64_asm.go -out ../chacha20poly1305_amd64.s -pkg chacha20poly1305. DO NOT EDIT. //go:build gc && !purego #include "textflag.h" -// General register allocation -#define oup DI -#define inp SI -#define inl BX -#define adp CX // free to reuse, after we hash the additional data -#define keyp R8 // free to reuse, when we copy the key to stack -#define itr2 R9 // general iterator -#define itr1 CX // general iterator -#define acc0 R10 -#define acc1 R11 -#define acc2 R12 -#define t0 R13 -#define t1 R14 -#define t2 R15 -#define t3 R8 -// Register and stack allocation for the SSE code -#define rStore (0*16)(BP) -#define sStore (1*16)(BP) -#define state1Store (2*16)(BP) -#define state2Store (3*16)(BP) -#define tmpStore (4*16)(BP) -#define ctr0Store (5*16)(BP) -#define ctr1Store (6*16)(BP) -#define ctr2Store (7*16)(BP) -#define ctr3Store (8*16)(BP) -#define A0 X0 -#define A1 X1 -#define A2 X2 -#define B0 X3 -#define B1 X4 -#define B2 X5 -#define C0 X6 -#define C1 X7 -#define C2 X8 -#define D0 X9 -#define D1 X10 -#define D2 X11 -#define T0 X12 -#define T1 X13 -#define T2 X14 -#define T3 X15 -#define A3 T0 -#define B3 T1 -#define C3 T2 -#define D3 T3 -// Register and stack allocation for the AVX2 code -#define rsStoreAVX2 (0*32)(BP) -#define state1StoreAVX2 (1*32)(BP) -#define state2StoreAVX2 (2*32)(BP) -#define ctr0StoreAVX2 (3*32)(BP) -#define ctr1StoreAVX2 (4*32)(BP) -#define ctr2StoreAVX2 (5*32)(BP) -#define ctr3StoreAVX2 (6*32)(BP) -#define tmpStoreAVX2 (7*32)(BP) // 256 bytes on stack -#define AA0 Y0 -#define AA1 Y5 -#define AA2 Y6 -#define AA3 Y7 -#define BB0 Y14 -#define BB1 Y9 -#define BB2 Y10 -#define BB3 Y11 -#define CC0 Y12 -#define CC1 Y13 -#define CC2 Y8 -#define CC3 Y15 -#define DD0 Y4 -#define DD1 Y1 -#define DD2 Y2 -#define DD3 Y3 -#define TT0 DD3 -#define TT1 AA3 -#define TT2 BB3 -#define TT3 CC3 -// ChaCha20 constants -DATA ·chacha20Constants<>+0x00(SB)/4, $0x61707865 -DATA ·chacha20Constants<>+0x04(SB)/4, $0x3320646e -DATA ·chacha20Constants<>+0x08(SB)/4, $0x79622d32 -DATA ·chacha20Constants<>+0x0c(SB)/4, $0x6b206574 -DATA ·chacha20Constants<>+0x10(SB)/4, $0x61707865 -DATA ·chacha20Constants<>+0x14(SB)/4, $0x3320646e -DATA ·chacha20Constants<>+0x18(SB)/4, $0x79622d32 -DATA ·chacha20Constants<>+0x1c(SB)/4, $0x6b206574 -// <<< 16 with PSHUFB -DATA ·rol16<>+0x00(SB)/8, $0x0504070601000302 -DATA ·rol16<>+0x08(SB)/8, $0x0D0C0F0E09080B0A -DATA ·rol16<>+0x10(SB)/8, $0x0504070601000302 -DATA ·rol16<>+0x18(SB)/8, $0x0D0C0F0E09080B0A -// <<< 8 with PSHUFB -DATA ·rol8<>+0x00(SB)/8, $0x0605040702010003 -DATA ·rol8<>+0x08(SB)/8, $0x0E0D0C0F0A09080B -DATA ·rol8<>+0x10(SB)/8, $0x0605040702010003 -DATA ·rol8<>+0x18(SB)/8, $0x0E0D0C0F0A09080B - -DATA ·avx2InitMask<>+0x00(SB)/8, $0x0 -DATA ·avx2InitMask<>+0x08(SB)/8, $0x0 -DATA ·avx2InitMask<>+0x10(SB)/8, $0x1 -DATA ·avx2InitMask<>+0x18(SB)/8, $0x0 - -DATA ·avx2IncMask<>+0x00(SB)/8, $0x2 -DATA ·avx2IncMask<>+0x08(SB)/8, $0x0 -DATA ·avx2IncMask<>+0x10(SB)/8, $0x2 -DATA ·avx2IncMask<>+0x18(SB)/8, $0x0 -// Poly1305 key clamp -DATA ·polyClampMask<>+0x00(SB)/8, $0x0FFFFFFC0FFFFFFF -DATA ·polyClampMask<>+0x08(SB)/8, $0x0FFFFFFC0FFFFFFC -DATA ·polyClampMask<>+0x10(SB)/8, $0xFFFFFFFFFFFFFFFF -DATA ·polyClampMask<>+0x18(SB)/8, $0xFFFFFFFFFFFFFFFF - -DATA ·sseIncMask<>+0x00(SB)/8, $0x1 -DATA ·sseIncMask<>+0x08(SB)/8, $0x0 -// To load/store the last < 16 bytes in a buffer -DATA ·andMask<>+0x00(SB)/8, $0x00000000000000ff -DATA ·andMask<>+0x08(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x10(SB)/8, $0x000000000000ffff -DATA ·andMask<>+0x18(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x20(SB)/8, $0x0000000000ffffff -DATA ·andMask<>+0x28(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x30(SB)/8, $0x00000000ffffffff -DATA ·andMask<>+0x38(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x40(SB)/8, $0x000000ffffffffff -DATA ·andMask<>+0x48(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x50(SB)/8, $0x0000ffffffffffff -DATA ·andMask<>+0x58(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x60(SB)/8, $0x00ffffffffffffff -DATA ·andMask<>+0x68(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x70(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0x78(SB)/8, $0x0000000000000000 -DATA ·andMask<>+0x80(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0x88(SB)/8, $0x00000000000000ff -DATA ·andMask<>+0x90(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0x98(SB)/8, $0x000000000000ffff -DATA ·andMask<>+0xa0(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0xa8(SB)/8, $0x0000000000ffffff -DATA ·andMask<>+0xb0(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0xb8(SB)/8, $0x00000000ffffffff -DATA ·andMask<>+0xc0(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0xc8(SB)/8, $0x000000ffffffffff -DATA ·andMask<>+0xd0(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0xd8(SB)/8, $0x0000ffffffffffff -DATA ·andMask<>+0xe0(SB)/8, $0xffffffffffffffff -DATA ·andMask<>+0xe8(SB)/8, $0x00ffffffffffffff - -GLOBL ·chacha20Constants<>(SB), (NOPTR+RODATA), $32 -GLOBL ·rol16<>(SB), (NOPTR+RODATA), $32 -GLOBL ·rol8<>(SB), (NOPTR+RODATA), $32 -GLOBL ·sseIncMask<>(SB), (NOPTR+RODATA), $16 -GLOBL ·avx2IncMask<>(SB), (NOPTR+RODATA), $32 -GLOBL ·avx2InitMask<>(SB), (NOPTR+RODATA), $32 -GLOBL ·polyClampMask<>(SB), (NOPTR+RODATA), $32 -GLOBL ·andMask<>(SB), (NOPTR+RODATA), $240 -// No PALIGNR in Go ASM yet (but VPALIGNR is present). -#define shiftB0Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x04 // PALIGNR $4, X3, X3 -#define shiftB1Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xe4; BYTE $0x04 // PALIGNR $4, X4, X4 -#define shiftB2Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x04 // PALIGNR $4, X5, X5 -#define shiftB3Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x04 // PALIGNR $4, X13, X13 -#define shiftC0Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xf6; BYTE $0x08 // PALIGNR $8, X6, X6 -#define shiftC1Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xff; BYTE $0x08 // PALIGNR $8, X7, X7 -#define shiftC2Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xc0; BYTE $0x08 // PALIGNR $8, X8, X8 -#define shiftC3Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xf6; BYTE $0x08 // PALIGNR $8, X14, X14 -#define shiftD0Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xc9; BYTE $0x0c // PALIGNR $12, X9, X9 -#define shiftD1Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xd2; BYTE $0x0c // PALIGNR $12, X10, X10 -#define shiftD2Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x0c // PALIGNR $12, X11, X11 -#define shiftD3Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xff; BYTE $0x0c // PALIGNR $12, X15, X15 -#define shiftB0Right BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x0c // PALIGNR $12, X3, X3 -#define shiftB1Right BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xe4; BYTE $0x0c // PALIGNR $12, X4, X4 -#define shiftB2Right BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x0c // PALIGNR $12, X5, X5 -#define shiftB3Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x0c // PALIGNR $12, X13, X13 -#define shiftC0Right shiftC0Left -#define shiftC1Right shiftC1Left -#define shiftC2Right shiftC2Left -#define shiftC3Right shiftC3Left -#define shiftD0Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xc9; BYTE $0x04 // PALIGNR $4, X9, X9 -#define shiftD1Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xd2; BYTE $0x04 // PALIGNR $4, X10, X10 -#define shiftD2Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x04 // PALIGNR $4, X11, X11 -#define shiftD3Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xff; BYTE $0x04 // PALIGNR $4, X15, X15 - -// Some macros - -// ROL rotates the uint32s in register R left by N bits, using temporary T. -#define ROL(N, R, T) \ - MOVO R, T; PSLLL $(N), T; PSRLL $(32-(N)), R; PXOR T, R - -// ROL16 rotates the uint32s in register R left by 16, using temporary T if needed. -#ifdef GOAMD64_v2 -#define ROL16(R, T) PSHUFB ·rol16<>(SB), R -#else -#define ROL16(R, T) ROL(16, R, T) -#endif - -// ROL8 rotates the uint32s in register R left by 8, using temporary T if needed. -#ifdef GOAMD64_v2 -#define ROL8(R, T) PSHUFB ·rol8<>(SB), R -#else -#define ROL8(R, T) ROL(8, R, T) -#endif - -#define chachaQR(A, B, C, D, T) \ - PADDD B, A; PXOR A, D; ROL16(D, T) \ - PADDD D, C; PXOR C, B; MOVO B, T; PSLLL $12, T; PSRLL $20, B; PXOR T, B \ - PADDD B, A; PXOR A, D; ROL8(D, T) \ - PADDD D, C; PXOR C, B; MOVO B, T; PSLLL $7, T; PSRLL $25, B; PXOR T, B - -#define chachaQR_AVX2(A, B, C, D, T) \ - VPADDD B, A, A; VPXOR A, D, D; VPSHUFB ·rol16<>(SB), D, D \ - VPADDD D, C, C; VPXOR C, B, B; VPSLLD $12, B, T; VPSRLD $20, B, B; VPXOR T, B, B \ - VPADDD B, A, A; VPXOR A, D, D; VPSHUFB ·rol8<>(SB), D, D \ - VPADDD D, C, C; VPXOR C, B, B; VPSLLD $7, B, T; VPSRLD $25, B, B; VPXOR T, B, B - -#define polyAdd(S) ADDQ S, acc0; ADCQ 8+S, acc1; ADCQ $1, acc2 -#define polyMulStage1 MOVQ (0*8)(BP), AX; MOVQ AX, t2; MULQ acc0; MOVQ AX, t0; MOVQ DX, t1; MOVQ (0*8)(BP), AX; MULQ acc1; IMULQ acc2, t2; ADDQ AX, t1; ADCQ DX, t2 -#define polyMulStage2 MOVQ (1*8)(BP), AX; MOVQ AX, t3; MULQ acc0; ADDQ AX, t1; ADCQ $0, DX; MOVQ DX, acc0; MOVQ (1*8)(BP), AX; MULQ acc1; ADDQ AX, t2; ADCQ $0, DX -#define polyMulStage3 IMULQ acc2, t3; ADDQ acc0, t2; ADCQ DX, t3 -#define polyMulReduceStage MOVQ t0, acc0; MOVQ t1, acc1; MOVQ t2, acc2; ANDQ $3, acc2; MOVQ t2, t0; ANDQ $-4, t0; MOVQ t3, t1; SHRQ $2, t3, t2; SHRQ $2, t3; ADDQ t0, acc0; ADCQ t1, acc1; ADCQ $0, acc2; ADDQ t2, acc0; ADCQ t3, acc1; ADCQ $0, acc2 - -#define polyMulStage1_AVX2 MOVQ (0*8)(BP), DX; MOVQ DX, t2; MULXQ acc0, t0, t1; IMULQ acc2, t2; MULXQ acc1, AX, DX; ADDQ AX, t1; ADCQ DX, t2 -#define polyMulStage2_AVX2 MOVQ (1*8)(BP), DX; MULXQ acc0, acc0, AX; ADDQ acc0, t1; MULXQ acc1, acc1, t3; ADCQ acc1, t2; ADCQ $0, t3 -#define polyMulStage3_AVX2 IMULQ acc2, DX; ADDQ AX, t2; ADCQ DX, t3 - -#define polyMul polyMulStage1; polyMulStage2; polyMulStage3; polyMulReduceStage -#define polyMulAVX2 polyMulStage1_AVX2; polyMulStage2_AVX2; polyMulStage3_AVX2; polyMulReduceStage -// ---------------------------------------------------------------------------- + +// func polyHashADInternal<>() TEXT polyHashADInternal<>(SB), NOSPLIT, $0 - // adp points to beginning of additional data - // itr2 holds ad length - XORQ acc0, acc0 - XORQ acc1, acc1 - XORQ acc2, acc2 - CMPQ itr2, $13 - JNE hashADLoop - -openFastTLSAD: - // Special treatment for the TLS case of 13 bytes - MOVQ (adp), acc0 - MOVQ 5(adp), acc1 - SHRQ $24, acc1 - MOVQ $1, acc2 - polyMul + // Hack: Must declare #define macros inside of a function due to Avo constraints + // ROL rotates the uint32s in register R left by N bits, using temporary T. + #define ROL(N, R, T) \ + MOVO R, T; \ + PSLLL $(N), T; \ + PSRLL $(32-(N)), R; \ + PXOR T, R + + // ROL8 rotates the uint32s in register R left by 8, using temporary T if needed. + #ifdef GOAMD64_v2 + #define ROL8(R, T) PSHUFB ·rol8<>(SB), R + #else + #define ROL8(R, T) ROL(8, R, T) + #endif + + // ROL16 rotates the uint32s in register R left by 16, using temporary T if needed. + #ifdef GOAMD64_v2 + #define ROL16(R, T) PSHUFB ·rol16<>(SB), R + #else + #define ROL16(R, T) ROL(16, R, T) + #endif + XORQ R10, R10 + XORQ R11, R11 + XORQ R12, R12 + CMPQ R9, $0x0d + JNE hashADLoop + MOVQ (CX), R10 + MOVQ 5(CX), R11 + SHRQ $0x18, R11 + MOVQ $0x00000001, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 RET hashADLoop: // Hash in 16 byte chunks - CMPQ itr2, $16 - JB hashADTail - polyAdd(0(adp)) - LEAQ (1*16)(adp), adp - SUBQ $16, itr2 - polyMul - JMP hashADLoop + CMPQ R9, $0x10 + JB hashADTail + ADDQ (CX), R10 + ADCQ 8(CX), R11 + ADCQ $0x01, R12 + LEAQ 16(CX), CX + SUBQ $0x10, R9 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + JMP hashADLoop hashADTail: - CMPQ itr2, $0 + CMPQ R9, $0x00 JE hashADDone // Hash last < 16 byte tail - XORQ t0, t0 - XORQ t1, t1 - XORQ t2, t2 - ADDQ itr2, adp + XORQ R13, R13 + XORQ R14, R14 + XORQ R15, R15 + ADDQ R9, CX hashADTailLoop: - SHLQ $8, t0, t1 - SHLQ $8, t0 - MOVB -1(adp), t2 - XORQ t2, t0 - DECQ adp - DECQ itr2 - JNE hashADTailLoop - -hashADTailFinish: - ADDQ t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2 - polyMul - - // Finished AD + SHLQ $0x08, R13, R14 + SHLQ $0x08, R13 + MOVB -1(CX), R15 + XORQ R15, R13 + DECQ CX + DECQ R9 + JNE hashADTailLoop + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + hashADDone: RET -// ---------------------------------------------------------------------------- -// func chacha20Poly1305Open(dst, key, src, ad []byte) bool -TEXT ·chacha20Poly1305Open(SB), 0, $288-97 +// func chacha20Poly1305Open(dst []byte, key []uint32, src []byte, ad []byte) bool +// Requires: AVX, AVX2, BMI2, CMOV, SSE2 +TEXT ·chacha20Poly1305Open(SB), $288-97 // For aligned stack access MOVQ SP, BP - ADDQ $32, BP + ADDQ $0x20, BP ANDQ $-32, BP - MOVQ dst+0(FP), oup - MOVQ key+24(FP), keyp - MOVQ src+48(FP), inp - MOVQ src_len+56(FP), inl - MOVQ ad+72(FP), adp + MOVQ dst_base+0(FP), DI + MOVQ key_base+24(FP), R8 + MOVQ src_base+48(FP), SI + MOVQ src_len+56(FP), BX + MOVQ ad_base+72(FP), CX // Check for AVX2 support - CMPB ·useAVX2(SB), $1 + CMPB ·useAVX2+0(SB), $0x01 JE chacha20Poly1305Open_AVX2 // Special optimization, for very short buffers - CMPQ inl, $128 - JBE openSSE128 // About 16% faster + CMPQ BX, $0x80 + JBE openSSE128 // For long buffers, prepare the poly key first - MOVOU ·chacha20Constants<>(SB), A0 - MOVOU (1*16)(keyp), B0 - MOVOU (2*16)(keyp), C0 - MOVOU (3*16)(keyp), D0 - MOVO D0, T1 + MOVOU ·chacha20Constants<>+0(SB), X0 + MOVOU 16(R8), X3 + MOVOU 32(R8), X6 + MOVOU 48(R8), X9 + MOVO X9, X13 // Store state on stack for future use - MOVO B0, state1Store - MOVO C0, state2Store - MOVO D0, ctr3Store - MOVQ $10, itr2 + MOVO X3, 32(BP) + MOVO X6, 48(BP) + MOVO X9, 128(BP) + MOVQ $0x0000000a, R9 openSSEPreparePolyKey: - chachaQR(A0, B0, C0, D0, T0) - shiftB0Left; shiftC0Left; shiftD0Left - chachaQR(A0, B0, C0, D0, T0) - shiftB0Right; shiftC0Right; shiftD0Right - DECQ itr2 - JNE openSSEPreparePolyKey + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + DECQ R9 + JNE openSSEPreparePolyKey // A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded - PADDL ·chacha20Constants<>(SB), A0; PADDL state1Store, B0 + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL 32(BP), X3 // Clamp and store the key - PAND ·polyClampMask<>(SB), A0 - MOVO A0, rStore; MOVO B0, sStore + PAND ·polyClampMask<>+0(SB), X0 + MOVO X0, (BP) + MOVO X3, 16(BP) // Hash AAD - MOVQ ad_len+80(FP), itr2 + MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) openSSEMainLoop: - CMPQ inl, $256 + CMPQ BX, $0x00000100 JB openSSEMainLoopDone // Load state, increment counter blocks - MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0 - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1 - MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2 - MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3 + MOVO ·chacha20Constants<>+0(SB), X0 + MOVO 32(BP), X3 + MOVO 48(BP), X6 + MOVO 128(BP), X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X1, X2 + MOVO X4, X5 + MOVO X7, X8 + MOVO X10, X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X2, X12 + MOVO X5, X13 + MOVO X8, X14 + MOVO X11, X15 + PADDL ·sseIncMask<>+0(SB), X15 // Store counters - MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store + MOVO X9, 80(BP) + MOVO X10, 96(BP) + MOVO X11, 112(BP) + MOVO X15, 128(BP) - // There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we hash 2 blocks, and for the remaining 4 only 1 block - for a total of 16 - MOVQ $4, itr1 - MOVQ inp, itr2 + // There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we hash + // 2 blocks, and for the remaining 4 only 1 block - for a total of 16 + MOVQ $0x00000004, CX + MOVQ SI, R9 openSSEInternalLoop: - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - polyAdd(0(itr2)) - shiftB0Left; shiftB1Left; shiftB2Left; shiftB3Left - shiftC0Left; shiftC1Left; shiftC2Left; shiftC3Left - shiftD0Left; shiftD1Left; shiftD2Left; shiftD3Left - polyMulStage1 - polyMulStage2 - LEAQ (2*8)(itr2), itr2 - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - polyMulStage3 - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - polyMulReduceStage - shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right - shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right - shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right - DECQ itr1 - JGE openSSEInternalLoop - - polyAdd(0(itr2)) - polyMul - LEAQ (2*8)(itr2), itr2 - - CMPQ itr1, $-6 - JG openSSEInternalLoop + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x0c + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + LEAQ 16(R9), R9 + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x04 + DECQ CX + JGE openSSEInternalLoop + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(R9), R9 + CMPQ CX, $-6 + JG openSSEInternalLoop // Add in the state - PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3 - PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3 - PADDD state2Store, C0; PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3 - PADDD ctr0Store, D0; PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3 + PADDD ·chacha20Constants<>+0(SB), X0 + PADDD ·chacha20Constants<>+0(SB), X1 + PADDD ·chacha20Constants<>+0(SB), X2 + PADDD ·chacha20Constants<>+0(SB), X12 + PADDD 32(BP), X3 + PADDD 32(BP), X4 + PADDD 32(BP), X5 + PADDD 32(BP), X13 + PADDD 48(BP), X6 + PADDD 48(BP), X7 + PADDD 48(BP), X8 + PADDD 48(BP), X14 + PADDD 80(BP), X9 + PADDD 96(BP), X10 + PADDD 112(BP), X11 + PADDD 128(BP), X15 // Load - xor - store - MOVO D3, tmpStore - MOVOU (0*16)(inp), D3; PXOR D3, A0; MOVOU A0, (0*16)(oup) - MOVOU (1*16)(inp), D3; PXOR D3, B0; MOVOU B0, (1*16)(oup) - MOVOU (2*16)(inp), D3; PXOR D3, C0; MOVOU C0, (2*16)(oup) - MOVOU (3*16)(inp), D3; PXOR D3, D0; MOVOU D0, (3*16)(oup) - MOVOU (4*16)(inp), D0; PXOR D0, A1; MOVOU A1, (4*16)(oup) - MOVOU (5*16)(inp), D0; PXOR D0, B1; MOVOU B1, (5*16)(oup) - MOVOU (6*16)(inp), D0; PXOR D0, C1; MOVOU C1, (6*16)(oup) - MOVOU (7*16)(inp), D0; PXOR D0, D1; MOVOU D1, (7*16)(oup) - MOVOU (8*16)(inp), D0; PXOR D0, A2; MOVOU A2, (8*16)(oup) - MOVOU (9*16)(inp), D0; PXOR D0, B2; MOVOU B2, (9*16)(oup) - MOVOU (10*16)(inp), D0; PXOR D0, C2; MOVOU C2, (10*16)(oup) - MOVOU (11*16)(inp), D0; PXOR D0, D2; MOVOU D2, (11*16)(oup) - MOVOU (12*16)(inp), D0; PXOR D0, A3; MOVOU A3, (12*16)(oup) - MOVOU (13*16)(inp), D0; PXOR D0, B3; MOVOU B3, (13*16)(oup) - MOVOU (14*16)(inp), D0; PXOR D0, C3; MOVOU C3, (14*16)(oup) - MOVOU (15*16)(inp), D0; PXOR tmpStore, D0; MOVOU D0, (15*16)(oup) - LEAQ 256(inp), inp - LEAQ 256(oup), oup - SUBQ $256, inl + MOVO X15, 64(BP) + MOVOU (SI), X15 + PXOR X15, X0 + MOVOU X0, (DI) + MOVOU 16(SI), X15 + PXOR X15, X3 + MOVOU X3, 16(DI) + MOVOU 32(SI), X15 + PXOR X15, X6 + MOVOU X6, 32(DI) + MOVOU 48(SI), X15 + PXOR X15, X9 + MOVOU X9, 48(DI) + MOVOU 64(SI), X9 + PXOR X9, X1 + MOVOU X1, 64(DI) + MOVOU 80(SI), X9 + PXOR X9, X4 + MOVOU X4, 80(DI) + MOVOU 96(SI), X9 + PXOR X9, X7 + MOVOU X7, 96(DI) + MOVOU 112(SI), X9 + PXOR X9, X10 + MOVOU X10, 112(DI) + MOVOU 128(SI), X9 + PXOR X9, X2 + MOVOU X2, 128(DI) + MOVOU 144(SI), X9 + PXOR X9, X5 + MOVOU X5, 144(DI) + MOVOU 160(SI), X9 + PXOR X9, X8 + MOVOU X8, 160(DI) + MOVOU 176(SI), X9 + PXOR X9, X11 + MOVOU X11, 176(DI) + MOVOU 192(SI), X9 + PXOR X9, X12 + MOVOU X12, 192(DI) + MOVOU 208(SI), X9 + PXOR X9, X13 + MOVOU X13, 208(DI) + MOVOU 224(SI), X9 + PXOR X9, X14 + MOVOU X14, 224(DI) + MOVOU 240(SI), X9 + PXOR 64(BP), X9 + MOVOU X9, 240(DI) + LEAQ 256(SI), SI + LEAQ 256(DI), DI + SUBQ $0x00000100, BX JMP openSSEMainLoop openSSEMainLoopDone: // Handle the various tail sizes efficiently - TESTQ inl, inl + TESTQ BX, BX JE openSSEFinalize - CMPQ inl, $64 + CMPQ BX, $0x40 JBE openSSETail64 - CMPQ inl, $128 + CMPQ BX, $0x80 JBE openSSETail128 - CMPQ inl, $192 + CMPQ BX, $0xc0 JBE openSSETail192 JMP openSSETail256 openSSEFinalize: // Hash in the PT, AAD lengths - ADDQ ad_len+80(FP), acc0; ADCQ src_len+56(FP), acc1; ADCQ $1, acc2 - polyMul + ADDQ ad_len+80(FP), R10 + ADCQ src_len+56(FP), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 // Final reduce - MOVQ acc0, t0 - MOVQ acc1, t1 - MOVQ acc2, t2 - SUBQ $-5, acc0 - SBBQ $-1, acc1 - SBBQ $3, acc2 - CMOVQCS t0, acc0 - CMOVQCS t1, acc1 - CMOVQCS t2, acc2 + MOVQ R10, R13 + MOVQ R11, R14 + MOVQ R12, R15 + SUBQ $-5, R10 + SBBQ $-1, R11 + SBBQ $0x03, R12 + CMOVQCS R13, R10 + CMOVQCS R14, R11 + CMOVQCS R15, R12 // Add in the "s" part of the key - ADDQ 0+sStore, acc0 - ADCQ 8+sStore, acc1 + ADDQ 16(BP), R10 + ADCQ 24(BP), R11 // Finally, constant time compare to the tag at the end of the message XORQ AX, AX - MOVQ $1, DX - XORQ (0*8)(inp), acc0 - XORQ (1*8)(inp), acc1 - ORQ acc1, acc0 + MOVQ $0x00000001, DX + XORQ (SI), R10 + XORQ 8(SI), R11 + ORQ R11, R10 CMOVQEQ DX, AX // Return true iff tags are equal MOVB AX, ret+96(FP) RET -// ---------------------------------------------------------------------------- -// Special optimization for buffers smaller than 129 bytes openSSE128: - // For up to 128 bytes of ciphertext and 64 bytes for the poly key, we require to process three blocks - MOVOU ·chacha20Constants<>(SB), A0; MOVOU (1*16)(keyp), B0; MOVOU (2*16)(keyp), C0; MOVOU (3*16)(keyp), D0 - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1 - MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2 - MOVO B0, T1; MOVO C0, T2; MOVO D1, T3 - MOVQ $10, itr2 + MOVOU ·chacha20Constants<>+0(SB), X0 + MOVOU 16(R8), X3 + MOVOU 32(R8), X6 + MOVOU 48(R8), X9 + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X1, X2 + MOVO X4, X5 + MOVO X7, X8 + MOVO X10, X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X3, X13 + MOVO X6, X14 + MOVO X10, X15 + MOVQ $0x0000000a, R9 openSSE128InnerCipherLoop: - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Left; shiftB1Left; shiftB2Left - shiftC0Left; shiftC1Left; shiftC2Left - shiftD0Left; shiftD1Left; shiftD2Left - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Right; shiftB1Right; shiftB2Right - shiftC0Right; shiftC1Right; shiftC2Right - shiftD0Right; shiftD1Right; shiftD2Right - DECQ itr2 - JNE openSSE128InnerCipherLoop + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + DECQ R9 + JNE openSSE128InnerCipherLoop // A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded - PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2 - PADDL T1, B0; PADDL T1, B1; PADDL T1, B2 - PADDL T2, C1; PADDL T2, C2 - PADDL T3, D1; PADDL ·sseIncMask<>(SB), T3; PADDL T3, D2 + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL ·chacha20Constants<>+0(SB), X1 + PADDL ·chacha20Constants<>+0(SB), X2 + PADDL X13, X3 + PADDL X13, X4 + PADDL X13, X5 + PADDL X14, X7 + PADDL X14, X8 + PADDL X15, X10 + PADDL ·sseIncMask<>+0(SB), X15 + PADDL X15, X11 // Clamp and store the key - PAND ·polyClampMask<>(SB), A0 - MOVOU A0, rStore; MOVOU B0, sStore + PAND ·polyClampMask<>+0(SB), X0 + MOVOU X0, (BP) + MOVOU X3, 16(BP) // Hash - MOVQ ad_len+80(FP), itr2 + MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) openSSE128Open: - CMPQ inl, $16 + CMPQ BX, $0x10 JB openSSETail16 - SUBQ $16, inl + SUBQ $0x10, BX // Load for hashing - polyAdd(0(inp)) + ADDQ (SI), R10 + ADCQ 8(SI), R11 + ADCQ $0x01, R12 // Load for decryption - MOVOU (inp), T0; PXOR T0, A1; MOVOU A1, (oup) - LEAQ (1*16)(inp), inp - LEAQ (1*16)(oup), oup - polyMul + MOVOU (SI), X12 + PXOR X12, X1 + MOVOU X1, (DI) + LEAQ 16(SI), SI + LEAQ 16(DI), DI + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 // Shift the stream "left" - MOVO B1, A1 - MOVO C1, B1 - MOVO D1, C1 - MOVO A2, D1 - MOVO B2, A2 - MOVO C2, B2 - MOVO D2, C2 + MOVO X4, X1 + MOVO X7, X4 + MOVO X10, X7 + MOVO X2, X10 + MOVO X5, X2 + MOVO X8, X5 + MOVO X11, X8 JMP openSSE128Open openSSETail16: - TESTQ inl, inl + TESTQ BX, BX JE openSSEFinalize // We can safely load the CT from the end, because it is padded with the MAC - MOVQ inl, itr2 - SHLQ $4, itr2 - LEAQ ·andMask<>(SB), t0 - MOVOU (inp), T0 - ADDQ inl, inp - PAND -16(t0)(itr2*1), T0 - MOVO T0, 0+tmpStore - MOVQ T0, t0 - MOVQ 8+tmpStore, t1 - PXOR A1, T0 + MOVQ BX, R9 + SHLQ $0x04, R9 + LEAQ ·andMask<>+0(SB), R13 + MOVOU (SI), X12 + ADDQ BX, SI + PAND -16(R13)(R9*1), X12 + MOVO X12, 64(BP) + MOVQ X12, R13 + MOVQ 72(BP), R14 + PXOR X1, X12 // We can only store one byte at a time, since plaintext can be shorter than 16 bytes openSSETail16Store: - MOVQ T0, t3 - MOVB t3, (oup) - PSRLDQ $1, T0 - INCQ oup - DECQ inl + MOVQ X12, R8 + MOVB R8, (DI) + PSRLDQ $0x01, X12 + INCQ DI + DECQ BX JNE openSSETail16Store - ADDQ t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2 - polyMul + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 JMP openSSEFinalize -// ---------------------------------------------------------------------------- -// Special optimization for the last 64 bytes of ciphertext openSSETail64: - // Need to decrypt up to 64 bytes - prepare single block - MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr0Store - XORQ itr2, itr2 - MOVQ inl, itr1 - CMPQ itr1, $16 - JB openSSETail64LoopB + MOVO ·chacha20Constants<>+0(SB), X0 + MOVO 32(BP), X3 + MOVO 48(BP), X6 + MOVO 128(BP), X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X9, 80(BP) + XORQ R9, R9 + MOVQ BX, CX + CMPQ CX, $0x10 + JB openSSETail64LoopB openSSETail64LoopA: - // Perform ChaCha rounds, while hashing the remaining input - polyAdd(0(inp)(itr2*1)) - polyMul - SUBQ $16, itr1 + ADDQ (SI)(R9*1), R10 + ADCQ 8(SI)(R9*1), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + SUBQ $0x10, CX openSSETail64LoopB: - ADDQ $16, itr2 - chachaQR(A0, B0, C0, D0, T0) - shiftB0Left; shiftC0Left; shiftD0Left - chachaQR(A0, B0, C0, D0, T0) - shiftB0Right; shiftC0Right; shiftD0Right - - CMPQ itr1, $16 - JAE openSSETail64LoopA - - CMPQ itr2, $160 - JNE openSSETail64LoopB - - PADDL ·chacha20Constants<>(SB), A0; PADDL state1Store, B0; PADDL state2Store, C0; PADDL ctr0Store, D0 + ADDQ $0x10, R9 + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + CMPQ CX, $0x10 + JAE openSSETail64LoopA + CMPQ R9, $0xa0 + JNE openSSETail64LoopB + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL 32(BP), X3 + PADDL 48(BP), X6 + PADDL 80(BP), X9 openSSETail64DecLoop: - CMPQ inl, $16 + CMPQ BX, $0x10 JB openSSETail64DecLoopDone - SUBQ $16, inl - MOVOU (inp), T0 - PXOR T0, A0 - MOVOU A0, (oup) - LEAQ 16(inp), inp - LEAQ 16(oup), oup - MOVO B0, A0 - MOVO C0, B0 - MOVO D0, C0 + SUBQ $0x10, BX + MOVOU (SI), X12 + PXOR X12, X0 + MOVOU X0, (DI) + LEAQ 16(SI), SI + LEAQ 16(DI), DI + MOVO X3, X0 + MOVO X6, X3 + MOVO X9, X6 JMP openSSETail64DecLoop openSSETail64DecLoopDone: - MOVO A0, A1 + MOVO X0, X1 JMP openSSETail16 -// ---------------------------------------------------------------------------- -// Special optimization for the last 128 bytes of ciphertext openSSETail128: - // Need to decrypt up to 128 bytes - prepare two blocks - MOVO ·chacha20Constants<>(SB), A1; MOVO state1Store, B1; MOVO state2Store, C1; MOVO ctr3Store, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr0Store - MOVO A1, A0; MOVO B1, B0; MOVO C1, C0; MOVO D1, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr1Store - XORQ itr2, itr2 - MOVQ inl, itr1 - ANDQ $-16, itr1 + MOVO ·chacha20Constants<>+0(SB), X1 + MOVO 32(BP), X4 + MOVO 48(BP), X7 + MOVO 128(BP), X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X10, 80(BP) + MOVO X1, X0 + MOVO X4, X3 + MOVO X7, X6 + MOVO X10, X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X9, 96(BP) + XORQ R9, R9 + MOVQ BX, CX + ANDQ $-16, CX openSSETail128LoopA: - // Perform ChaCha rounds, while hashing the remaining input - polyAdd(0(inp)(itr2*1)) - polyMul + ADDQ (SI)(R9*1), R10 + ADCQ 8(SI)(R9*1), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 openSSETail128LoopB: - ADDQ $16, itr2 - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0) - shiftB0Left; shiftC0Left; shiftD0Left - shiftB1Left; shiftC1Left; shiftD1Left - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0) - shiftB0Right; shiftC0Right; shiftD0Right - shiftB1Right; shiftC1Right; shiftD1Right - - CMPQ itr2, itr1 - JB openSSETail128LoopA - - CMPQ itr2, $160 - JNE openSSETail128LoopB - - PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1 - PADDL state1Store, B0; PADDL state1Store, B1 - PADDL state2Store, C0; PADDL state2Store, C1 - PADDL ctr1Store, D0; PADDL ctr0Store, D1 - - MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3 - PXOR T0, A1; PXOR T1, B1; PXOR T2, C1; PXOR T3, D1 - MOVOU A1, (0*16)(oup); MOVOU B1, (1*16)(oup); MOVOU C1, (2*16)(oup); MOVOU D1, (3*16)(oup) - - SUBQ $64, inl - LEAQ 64(inp), inp - LEAQ 64(oup), oup - JMP openSSETail64DecLoop - -// ---------------------------------------------------------------------------- -// Special optimization for the last 192 bytes of ciphertext + ADDQ $0x10, R9 + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + CMPQ R9, CX + JB openSSETail128LoopA + CMPQ R9, $0xa0 + JNE openSSETail128LoopB + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL ·chacha20Constants<>+0(SB), X1 + PADDL 32(BP), X3 + PADDL 32(BP), X4 + PADDL 48(BP), X6 + PADDL 48(BP), X7 + PADDL 96(BP), X9 + PADDL 80(BP), X10 + MOVOU (SI), X12 + MOVOU 16(SI), X13 + MOVOU 32(SI), X14 + MOVOU 48(SI), X15 + PXOR X12, X1 + PXOR X13, X4 + PXOR X14, X7 + PXOR X15, X10 + MOVOU X1, (DI) + MOVOU X4, 16(DI) + MOVOU X7, 32(DI) + MOVOU X10, 48(DI) + SUBQ $0x40, BX + LEAQ 64(SI), SI + LEAQ 64(DI), DI + JMP openSSETail64DecLoop + openSSETail192: - // Need to decrypt up to 192 bytes - prepare three blocks - MOVO ·chacha20Constants<>(SB), A2; MOVO state1Store, B2; MOVO state2Store, C2; MOVO ctr3Store, D2; PADDL ·sseIncMask<>(SB), D2; MOVO D2, ctr0Store - MOVO A2, A1; MOVO B2, B1; MOVO C2, C1; MOVO D2, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr1Store - MOVO A1, A0; MOVO B1, B0; MOVO C1, C0; MOVO D1, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr2Store - - MOVQ inl, itr1 - MOVQ $160, itr2 - CMPQ itr1, $160 - CMOVQGT itr2, itr1 - ANDQ $-16, itr1 - XORQ itr2, itr2 + MOVO ·chacha20Constants<>+0(SB), X2 + MOVO 32(BP), X5 + MOVO 48(BP), X8 + MOVO 128(BP), X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X11, 80(BP) + MOVO X2, X1 + MOVO X5, X4 + MOVO X8, X7 + MOVO X11, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X10, 96(BP) + MOVO X1, X0 + MOVO X4, X3 + MOVO X7, X6 + MOVO X10, X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X9, 112(BP) + MOVQ BX, CX + MOVQ $0x000000a0, R9 + CMPQ CX, $0xa0 + CMOVQGT R9, CX + ANDQ $-16, CX + XORQ R9, R9 openSSLTail192LoopA: - // Perform ChaCha rounds, while hashing the remaining input - polyAdd(0(inp)(itr2*1)) - polyMul + ADDQ (SI)(R9*1), R10 + ADCQ 8(SI)(R9*1), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 openSSLTail192LoopB: - ADDQ $16, itr2 - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Left; shiftC0Left; shiftD0Left - shiftB1Left; shiftC1Left; shiftD1Left - shiftB2Left; shiftC2Left; shiftD2Left - - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Right; shiftC0Right; shiftD0Right - shiftB1Right; shiftC1Right; shiftD1Right - shiftB2Right; shiftC2Right; shiftD2Right - - CMPQ itr2, itr1 - JB openSSLTail192LoopA - - CMPQ itr2, $160 - JNE openSSLTail192LoopB - - CMPQ inl, $176 - JB openSSLTail192Store - - polyAdd(160(inp)) - polyMul - - CMPQ inl, $192 - JB openSSLTail192Store - - polyAdd(176(inp)) - polyMul + ADDQ $0x10, R9 + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + CMPQ R9, CX + JB openSSLTail192LoopA + CMPQ R9, $0xa0 + JNE openSSLTail192LoopB + CMPQ BX, $0xb0 + JB openSSLTail192Store + ADDQ 160(SI), R10 + ADCQ 168(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + CMPQ BX, $0xc0 + JB openSSLTail192Store + ADDQ 176(SI), R10 + ADCQ 184(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 openSSLTail192Store: - PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2 - PADDL state1Store, B0; PADDL state1Store, B1; PADDL state1Store, B2 - PADDL state2Store, C0; PADDL state2Store, C1; PADDL state2Store, C2 - PADDL ctr2Store, D0; PADDL ctr1Store, D1; PADDL ctr0Store, D2 - - MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3 - PXOR T0, A2; PXOR T1, B2; PXOR T2, C2; PXOR T3, D2 - MOVOU A2, (0*16)(oup); MOVOU B2, (1*16)(oup); MOVOU C2, (2*16)(oup); MOVOU D2, (3*16)(oup) - - MOVOU (4*16)(inp), T0; MOVOU (5*16)(inp), T1; MOVOU (6*16)(inp), T2; MOVOU (7*16)(inp), T3 - PXOR T0, A1; PXOR T1, B1; PXOR T2, C1; PXOR T3, D1 - MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup) - - SUBQ $128, inl - LEAQ 128(inp), inp - LEAQ 128(oup), oup - JMP openSSETail64DecLoop - -// ---------------------------------------------------------------------------- -// Special optimization for the last 256 bytes of ciphertext + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL ·chacha20Constants<>+0(SB), X1 + PADDL ·chacha20Constants<>+0(SB), X2 + PADDL 32(BP), X3 + PADDL 32(BP), X4 + PADDL 32(BP), X5 + PADDL 48(BP), X6 + PADDL 48(BP), X7 + PADDL 48(BP), X8 + PADDL 112(BP), X9 + PADDL 96(BP), X10 + PADDL 80(BP), X11 + MOVOU (SI), X12 + MOVOU 16(SI), X13 + MOVOU 32(SI), X14 + MOVOU 48(SI), X15 + PXOR X12, X2 + PXOR X13, X5 + PXOR X14, X8 + PXOR X15, X11 + MOVOU X2, (DI) + MOVOU X5, 16(DI) + MOVOU X8, 32(DI) + MOVOU X11, 48(DI) + MOVOU 64(SI), X12 + MOVOU 80(SI), X13 + MOVOU 96(SI), X14 + MOVOU 112(SI), X15 + PXOR X12, X1 + PXOR X13, X4 + PXOR X14, X7 + PXOR X15, X10 + MOVOU X1, 64(DI) + MOVOU X4, 80(DI) + MOVOU X7, 96(DI) + MOVOU X10, 112(DI) + SUBQ $0x80, BX + LEAQ 128(SI), SI + LEAQ 128(DI), DI + JMP openSSETail64DecLoop + openSSETail256: - // Need to decrypt up to 256 bytes - prepare four blocks - MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0 - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1 - MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2 - MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3 + MOVO ·chacha20Constants<>+0(SB), X0 + MOVO 32(BP), X3 + MOVO 48(BP), X6 + MOVO 128(BP), X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X1, X2 + MOVO X4, X5 + MOVO X7, X8 + MOVO X10, X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X2, X12 + MOVO X5, X13 + MOVO X8, X14 + MOVO X11, X15 + PADDL ·sseIncMask<>+0(SB), X15 // Store counters - MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store - XORQ itr2, itr2 + MOVO X9, 80(BP) + MOVO X10, 96(BP) + MOVO X11, 112(BP) + MOVO X15, 128(BP) + XORQ R9, R9 openSSETail256Loop: - // This loop inteleaves 8 ChaCha quarter rounds with 1 poly multiplication - polyAdd(0(inp)(itr2*1)) - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - shiftB0Left; shiftB1Left; shiftB2Left; shiftB3Left - shiftC0Left; shiftC1Left; shiftC2Left; shiftC3Left - shiftD0Left; shiftD1Left; shiftD2Left; shiftD3Left - polyMulStage1 - polyMulStage2 - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - polyMulStage3 - polyMulReduceStage - shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right - shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right - shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right - ADDQ $2*8, itr2 - CMPQ itr2, $160 - JB openSSETail256Loop - MOVQ inl, itr1 - ANDQ $-16, itr1 + ADDQ (SI)(R9*1), R10 + ADCQ 8(SI)(R9*1), R11 + ADCQ $0x01, R12 + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x0c + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x04 + ADDQ $0x10, R9 + CMPQ R9, $0xa0 + JB openSSETail256Loop + MOVQ BX, CX + ANDQ $-16, CX openSSETail256HashLoop: - polyAdd(0(inp)(itr2*1)) - polyMul - ADDQ $2*8, itr2 - CMPQ itr2, itr1 - JB openSSETail256HashLoop + ADDQ (SI)(R9*1), R10 + ADCQ 8(SI)(R9*1), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + ADDQ $0x10, R9 + CMPQ R9, CX + JB openSSETail256HashLoop // Add in the state - PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3 - PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3 - PADDD state2Store, C0; PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3 - PADDD ctr0Store, D0; PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3 - MOVO D3, tmpStore + PADDD ·chacha20Constants<>+0(SB), X0 + PADDD ·chacha20Constants<>+0(SB), X1 + PADDD ·chacha20Constants<>+0(SB), X2 + PADDD ·chacha20Constants<>+0(SB), X12 + PADDD 32(BP), X3 + PADDD 32(BP), X4 + PADDD 32(BP), X5 + PADDD 32(BP), X13 + PADDD 48(BP), X6 + PADDD 48(BP), X7 + PADDD 48(BP), X8 + PADDD 48(BP), X14 + PADDD 80(BP), X9 + PADDD 96(BP), X10 + PADDD 112(BP), X11 + PADDD 128(BP), X15 + MOVO X15, 64(BP) // Load - xor - store - MOVOU (0*16)(inp), D3; PXOR D3, A0 - MOVOU (1*16)(inp), D3; PXOR D3, B0 - MOVOU (2*16)(inp), D3; PXOR D3, C0 - MOVOU (3*16)(inp), D3; PXOR D3, D0 - MOVOU A0, (0*16)(oup) - MOVOU B0, (1*16)(oup) - MOVOU C0, (2*16)(oup) - MOVOU D0, (3*16)(oup) - MOVOU (4*16)(inp), A0; MOVOU (5*16)(inp), B0; MOVOU (6*16)(inp), C0; MOVOU (7*16)(inp), D0 - PXOR A0, A1; PXOR B0, B1; PXOR C0, C1; PXOR D0, D1 - MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup) - MOVOU (8*16)(inp), A0; MOVOU (9*16)(inp), B0; MOVOU (10*16)(inp), C0; MOVOU (11*16)(inp), D0 - PXOR A0, A2; PXOR B0, B2; PXOR C0, C2; PXOR D0, D2 - MOVOU A2, (8*16)(oup); MOVOU B2, (9*16)(oup); MOVOU C2, (10*16)(oup); MOVOU D2, (11*16)(oup) - LEAQ 192(inp), inp - LEAQ 192(oup), oup - SUBQ $192, inl - MOVO A3, A0 - MOVO B3, B0 - MOVO C3, C0 - MOVO tmpStore, D0 - - JMP openSSETail64DecLoop - -// ---------------------------------------------------------------------------- -// ------------------------- AVX2 Code ---------------------------------------- + MOVOU (SI), X15 + PXOR X15, X0 + MOVOU 16(SI), X15 + PXOR X15, X3 + MOVOU 32(SI), X15 + PXOR X15, X6 + MOVOU 48(SI), X15 + PXOR X15, X9 + MOVOU X0, (DI) + MOVOU X3, 16(DI) + MOVOU X6, 32(DI) + MOVOU X9, 48(DI) + MOVOU 64(SI), X0 + MOVOU 80(SI), X3 + MOVOU 96(SI), X6 + MOVOU 112(SI), X9 + PXOR X0, X1 + PXOR X3, X4 + PXOR X6, X7 + PXOR X9, X10 + MOVOU X1, 64(DI) + MOVOU X4, 80(DI) + MOVOU X7, 96(DI) + MOVOU X10, 112(DI) + MOVOU 128(SI), X0 + MOVOU 144(SI), X3 + MOVOU 160(SI), X6 + MOVOU 176(SI), X9 + PXOR X0, X2 + PXOR X3, X5 + PXOR X6, X8 + PXOR X9, X11 + MOVOU X2, 128(DI) + MOVOU X5, 144(DI) + MOVOU X8, 160(DI) + MOVOU X11, 176(DI) + LEAQ 192(SI), SI + LEAQ 192(DI), DI + SUBQ $0xc0, BX + MOVO X12, X0 + MOVO X13, X3 + MOVO X14, X6 + MOVO 64(BP), X9 + JMP openSSETail64DecLoop + chacha20Poly1305Open_AVX2: VZEROUPPER - VMOVDQU ·chacha20Constants<>(SB), AA0 - BYTE $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x70; BYTE $0x10 // broadcasti128 16(r8), ymm14 - BYTE $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x20 // broadcasti128 32(r8), ymm12 - BYTE $0xc4; BYTE $0xc2; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x30 // broadcasti128 48(r8), ymm4 - VPADDD ·avx2InitMask<>(SB), DD0, DD0 + VMOVDQU ·chacha20Constants<>+0(SB), Y0 + BYTE $0xc4 + BYTE $0x42 + BYTE $0x7d + BYTE $0x5a + BYTE $0x70 + BYTE $0x10 + BYTE $0xc4 + BYTE $0x42 + BYTE $0x7d + BYTE $0x5a + BYTE $0x60 + BYTE $0x20 + BYTE $0xc4 + BYTE $0xc2 + BYTE $0x7d + BYTE $0x5a + BYTE $0x60 + BYTE $0x30 + VPADDD ·avx2InitMask<>+0(SB), Y4, Y4 // Special optimization, for very short buffers - CMPQ inl, $192 + CMPQ BX, $0xc0 JBE openAVX2192 - CMPQ inl, $320 + CMPQ BX, $0x00000140 JBE openAVX2320 // For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream - VMOVDQA BB0, state1StoreAVX2 - VMOVDQA CC0, state2StoreAVX2 - VMOVDQA DD0, ctr3StoreAVX2 - MOVQ $10, itr2 + VMOVDQA Y14, 32(BP) + VMOVDQA Y12, 64(BP) + VMOVDQA Y4, 192(BP) + MOVQ $0x0000000a, R9 openAVX2PreparePolyKey: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0) - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $12, DD0, DD0, DD0 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0) - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $4, DD0, DD0, DD0 - DECQ itr2 - JNE openAVX2PreparePolyKey - - VPADDD ·chacha20Constants<>(SB), AA0, AA0 - VPADDD state1StoreAVX2, BB0, BB0 - VPADDD state2StoreAVX2, CC0, CC0 - VPADDD ctr3StoreAVX2, DD0, DD0 - - VPERM2I128 $0x02, AA0, BB0, TT0 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x04, Y4, Y4, Y4 + DECQ R9 + JNE openAVX2PreparePolyKey + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD 32(BP), Y14, Y14 + VPADDD 64(BP), Y12, Y12 + VPADDD 192(BP), Y4, Y4 + VPERM2I128 $0x02, Y0, Y14, Y3 // Clamp and store poly key - VPAND ·polyClampMask<>(SB), TT0, TT0 - VMOVDQA TT0, rsStoreAVX2 + VPAND ·polyClampMask<>+0(SB), Y3, Y3 + VMOVDQA Y3, (BP) // Stream for the first 64 bytes - VPERM2I128 $0x13, AA0, BB0, AA0 - VPERM2I128 $0x13, CC0, DD0, BB0 + VPERM2I128 $0x13, Y0, Y14, Y0 + VPERM2I128 $0x13, Y12, Y4, Y14 // Hash AD + first 64 bytes - MOVQ ad_len+80(FP), itr2 + MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) - XORQ itr1, itr1 + XORQ CX, CX openAVX2InitialHash64: - polyAdd(0(inp)(itr1*1)) - polyMulAVX2 - ADDQ $16, itr1 - CMPQ itr1, $64 - JNE openAVX2InitialHash64 + ADDQ (SI)(CX*1), R10 + ADCQ 8(SI)(CX*1), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + ADDQ $0x10, CX + CMPQ CX, $0x40 + JNE openAVX2InitialHash64 // Decrypt the first 64 bytes - VPXOR (0*32)(inp), AA0, AA0 - VPXOR (1*32)(inp), BB0, BB0 - VMOVDQU AA0, (0*32)(oup) - VMOVDQU BB0, (1*32)(oup) - LEAQ (2*32)(inp), inp - LEAQ (2*32)(oup), oup - SUBQ $64, inl + VPXOR (SI), Y0, Y0 + VPXOR 32(SI), Y14, Y14 + VMOVDQU Y0, (DI) + VMOVDQU Y14, 32(DI) + LEAQ 64(SI), SI + LEAQ 64(DI), DI + SUBQ $0x40, BX openAVX2MainLoop: - CMPQ inl, $512 + CMPQ BX, $0x00000200 JB openAVX2MainLoopDone // Load state, increment counter blocks, store the incremented counters - VMOVDQU ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3 - VMOVDQA ctr3StoreAVX2, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3 - VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2 - XORQ itr1, itr1 + VMOVDQU ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA Y0, Y7 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA Y14, Y11 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA Y12, Y15 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 + VMOVDQA Y4, 96(BP) + VMOVDQA Y1, 128(BP) + VMOVDQA Y2, 160(BP) + VMOVDQA Y3, 192(BP) + XORQ CX, CX openAVX2InternalLoop: - // Lets just say this spaghetti loop interleaves 2 quarter rounds with 3 poly multiplications - // Effectively per 512 bytes of stream we hash 480 bytes of ciphertext - polyAdd(0*8(inp)(itr1*1)) - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - polyMulStage1_AVX2 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - polyMulStage2_AVX2 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - polyMulStage3_AVX2 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulReduceStage - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - polyAdd(2*8(inp)(itr1*1)) - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - polyMulStage1_AVX2 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulStage2_AVX2 - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - polyMulStage3_AVX2 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - polyMulReduceStage - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - polyAdd(4*8(inp)(itr1*1)) - LEAQ (6*8)(itr1), itr1 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulStage1_AVX2 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - polyMulStage2_AVX2 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - polyMulStage3_AVX2 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulReduceStage - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3 - CMPQ itr1, $480 + ADDQ (SI)(CX*1), R10 + ADCQ 8(SI)(CX*1), R11 + ADCQ $0x01, R12 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + ADDQ 16(SI)(CX*1), R10 + ADCQ 24(SI)(CX*1), R11 + ADCQ $0x01, R12 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x04, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPALIGNR $0x0c, Y3, Y3, Y3 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + ADDQ 32(SI)(CX*1), R10 + ADCQ 40(SI)(CX*1), R11 + ADCQ $0x01, R12 + LEAQ 48(CX), CX + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x0c, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + VPALIGNR $0x04, Y3, Y3, Y3 + CMPQ CX, $0x000001e0 JNE openAVX2InternalLoop - - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3 - VPADDD ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3 - VMOVDQA CC3, tmpStoreAVX2 + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 32(BP), Y11, Y11 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD 64(BP), Y15, Y15 + VPADDD 96(BP), Y4, Y4 + VPADDD 128(BP), Y1, Y1 + VPADDD 160(BP), Y2, Y2 + VPADDD 192(BP), Y3, Y3 + VMOVDQA Y15, 224(BP) // We only hashed 480 of the 512 bytes available - hash the remaining 32 here - polyAdd(480(inp)) - polyMulAVX2 - VPERM2I128 $0x02, AA0, BB0, CC3; VPERM2I128 $0x13, AA0, BB0, BB0; VPERM2I128 $0x02, CC0, DD0, AA0; VPERM2I128 $0x13, CC0, DD0, CC0 - VPXOR (0*32)(inp), CC3, CC3; VPXOR (1*32)(inp), AA0, AA0; VPXOR (2*32)(inp), BB0, BB0; VPXOR (3*32)(inp), CC0, CC0 - VMOVDQU CC3, (0*32)(oup); VMOVDQU AA0, (1*32)(oup); VMOVDQU BB0, (2*32)(oup); VMOVDQU CC0, (3*32)(oup) - VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0 - VPXOR (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0 - VMOVDQU AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup) + ADDQ 480(SI), R10 + ADCQ 488(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPERM2I128 $0x02, Y0, Y14, Y15 + VPERM2I128 $0x13, Y0, Y14, Y14 + VPERM2I128 $0x02, Y12, Y4, Y0 + VPERM2I128 $0x13, Y12, Y4, Y12 + VPXOR (SI), Y15, Y15 + VPXOR 32(SI), Y0, Y0 + VPXOR 64(SI), Y14, Y14 + VPXOR 96(SI), Y12, Y12 + VMOVDQU Y15, (DI) + VMOVDQU Y0, 32(DI) + VMOVDQU Y14, 64(DI) + VMOVDQU Y12, 96(DI) + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + VPXOR 128(SI), Y0, Y0 + VPXOR 160(SI), Y14, Y14 + VPXOR 192(SI), Y12, Y12 + VPXOR 224(SI), Y4, Y4 + VMOVDQU Y0, 128(DI) + VMOVDQU Y14, 160(DI) + VMOVDQU Y12, 192(DI) + VMOVDQU Y4, 224(DI) // and here - polyAdd(496(inp)) - polyMulAVX2 - VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0 - VPXOR (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0 - VMOVDQU AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup) - VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0 - VPXOR (12*32)(inp), AA0, AA0; VPXOR (13*32)(inp), BB0, BB0; VPXOR (14*32)(inp), CC0, CC0; VPXOR (15*32)(inp), DD0, DD0 - VMOVDQU AA0, (12*32)(oup); VMOVDQU BB0, (13*32)(oup); VMOVDQU CC0, (14*32)(oup); VMOVDQU DD0, (15*32)(oup) - LEAQ (32*16)(inp), inp - LEAQ (32*16)(oup), oup - SUBQ $(32*16), inl + ADDQ 496(SI), R10 + ADCQ 504(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + VPXOR 256(SI), Y0, Y0 + VPXOR 288(SI), Y14, Y14 + VPXOR 320(SI), Y12, Y12 + VPXOR 352(SI), Y4, Y4 + VMOVDQU Y0, 256(DI) + VMOVDQU Y14, 288(DI) + VMOVDQU Y12, 320(DI) + VMOVDQU Y4, 352(DI) + VPERM2I128 $0x02, Y7, Y11, Y0 + VPERM2I128 $0x02, 224(BP), Y3, Y14 + VPERM2I128 $0x13, Y7, Y11, Y12 + VPERM2I128 $0x13, 224(BP), Y3, Y4 + VPXOR 384(SI), Y0, Y0 + VPXOR 416(SI), Y14, Y14 + VPXOR 448(SI), Y12, Y12 + VPXOR 480(SI), Y4, Y4 + VMOVDQU Y0, 384(DI) + VMOVDQU Y14, 416(DI) + VMOVDQU Y12, 448(DI) + VMOVDQU Y4, 480(DI) + LEAQ 512(SI), SI + LEAQ 512(DI), DI + SUBQ $0x00000200, BX JMP openAVX2MainLoop openAVX2MainLoopDone: // Handle the various tail sizes efficiently - TESTQ inl, inl + TESTQ BX, BX JE openSSEFinalize - CMPQ inl, $128 + CMPQ BX, $0x80 JBE openAVX2Tail128 - CMPQ inl, $256 + CMPQ BX, $0x00000100 JBE openAVX2Tail256 - CMPQ inl, $384 + CMPQ BX, $0x00000180 JBE openAVX2Tail384 JMP openAVX2Tail512 -// ---------------------------------------------------------------------------- -// Special optimization for buffers smaller than 193 bytes openAVX2192: - // For up to 192 bytes of ciphertext and 64 bytes for the poly key, we process four blocks - VMOVDQA AA0, AA1 - VMOVDQA BB0, BB1 - VMOVDQA CC0, CC1 - VPADDD ·avx2IncMask<>(SB), DD0, DD1 - VMOVDQA AA0, AA2 - VMOVDQA BB0, BB2 - VMOVDQA CC0, CC2 - VMOVDQA DD0, DD2 - VMOVDQA DD1, TT3 - MOVQ $10, itr2 + VMOVDQA Y0, Y5 + VMOVDQA Y14, Y9 + VMOVDQA Y12, Y13 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y0, Y6 + VMOVDQA Y14, Y10 + VMOVDQA Y12, Y8 + VMOVDQA Y4, Y2 + VMOVDQA Y1, Y15 + MOVQ $0x0000000a, R9 openAVX2192InnerCipherLoop: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1 - DECQ itr2 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + DECQ R9 JNE openAVX2192InnerCipherLoop - VPADDD AA2, AA0, AA0; VPADDD AA2, AA1, AA1 - VPADDD BB2, BB0, BB0; VPADDD BB2, BB1, BB1 - VPADDD CC2, CC0, CC0; VPADDD CC2, CC1, CC1 - VPADDD DD2, DD0, DD0; VPADDD TT3, DD1, DD1 - VPERM2I128 $0x02, AA0, BB0, TT0 + VPADDD Y6, Y0, Y0 + VPADDD Y6, Y5, Y5 + VPADDD Y10, Y14, Y14 + VPADDD Y10, Y9, Y9 + VPADDD Y8, Y12, Y12 + VPADDD Y8, Y13, Y13 + VPADDD Y2, Y4, Y4 + VPADDD Y15, Y1, Y1 + VPERM2I128 $0x02, Y0, Y14, Y3 // Clamp and store poly key - VPAND ·polyClampMask<>(SB), TT0, TT0 - VMOVDQA TT0, rsStoreAVX2 + VPAND ·polyClampMask<>+0(SB), Y3, Y3 + VMOVDQA Y3, (BP) // Stream for up to 192 bytes - VPERM2I128 $0x13, AA0, BB0, AA0 - VPERM2I128 $0x13, CC0, DD0, BB0 - VPERM2I128 $0x02, AA1, BB1, CC0 - VPERM2I128 $0x02, CC1, DD1, DD0 - VPERM2I128 $0x13, AA1, BB1, AA1 - VPERM2I128 $0x13, CC1, DD1, BB1 + VPERM2I128 $0x13, Y0, Y14, Y0 + VPERM2I128 $0x13, Y12, Y4, Y14 + VPERM2I128 $0x02, Y5, Y9, Y12 + VPERM2I128 $0x02, Y13, Y1, Y4 + VPERM2I128 $0x13, Y5, Y9, Y5 + VPERM2I128 $0x13, Y13, Y1, Y9 openAVX2ShortOpen: // Hash - MOVQ ad_len+80(FP), itr2 + MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) openAVX2ShortOpenLoop: - CMPQ inl, $32 + CMPQ BX, $0x20 JB openAVX2ShortTail32 - SUBQ $32, inl + SUBQ $0x20, BX // Load for hashing - polyAdd(0*8(inp)) - polyMulAVX2 - polyAdd(2*8(inp)) - polyMulAVX2 + ADDQ (SI), R10 + ADCQ 8(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + ADDQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 // Load for decryption - VPXOR (inp), AA0, AA0 - VMOVDQU AA0, (oup) - LEAQ (1*32)(inp), inp - LEAQ (1*32)(oup), oup + VPXOR (SI), Y0, Y0 + VMOVDQU Y0, (DI) + LEAQ 32(SI), SI + LEAQ 32(DI), DI // Shift stream left - VMOVDQA BB0, AA0 - VMOVDQA CC0, BB0 - VMOVDQA DD0, CC0 - VMOVDQA AA1, DD0 - VMOVDQA BB1, AA1 - VMOVDQA CC1, BB1 - VMOVDQA DD1, CC1 - VMOVDQA AA2, DD1 - VMOVDQA BB2, AA2 + VMOVDQA Y14, Y0 + VMOVDQA Y12, Y14 + VMOVDQA Y4, Y12 + VMOVDQA Y5, Y4 + VMOVDQA Y9, Y5 + VMOVDQA Y13, Y9 + VMOVDQA Y1, Y13 + VMOVDQA Y6, Y1 + VMOVDQA Y10, Y6 JMP openAVX2ShortOpenLoop openAVX2ShortTail32: - CMPQ inl, $16 - VMOVDQA A0, A1 + CMPQ BX, $0x10 + VMOVDQA X0, X1 JB openAVX2ShortDone - - SUBQ $16, inl + SUBQ $0x10, BX // Load for hashing - polyAdd(0*8(inp)) - polyMulAVX2 + ADDQ (SI), R10 + ADCQ 8(SI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 // Load for decryption - VPXOR (inp), A0, T0 - VMOVDQU T0, (oup) - LEAQ (1*16)(inp), inp - LEAQ (1*16)(oup), oup - VPERM2I128 $0x11, AA0, AA0, AA0 - VMOVDQA A0, A1 + VPXOR (SI), X0, X12 + VMOVDQU X12, (DI) + LEAQ 16(SI), SI + LEAQ 16(DI), DI + VPERM2I128 $0x11, Y0, Y0, Y0 + VMOVDQA X0, X1 openAVX2ShortDone: VZEROUPPER JMP openSSETail16 -// ---------------------------------------------------------------------------- -// Special optimization for buffers smaller than 321 bytes openAVX2320: - // For up to 320 bytes of ciphertext and 64 bytes for the poly key, we process six blocks - VMOVDQA AA0, AA1; VMOVDQA BB0, BB1; VMOVDQA CC0, CC1; VPADDD ·avx2IncMask<>(SB), DD0, DD1 - VMOVDQA AA0, AA2; VMOVDQA BB0, BB2; VMOVDQA CC0, CC2; VPADDD ·avx2IncMask<>(SB), DD1, DD2 - VMOVDQA BB0, TT1; VMOVDQA CC0, TT2; VMOVDQA DD0, TT3 - MOVQ $10, itr2 + VMOVDQA Y0, Y5 + VMOVDQA Y14, Y9 + VMOVDQA Y12, Y13 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y0, Y6 + VMOVDQA Y14, Y10 + VMOVDQA Y12, Y8 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VMOVDQA Y14, Y7 + VMOVDQA Y12, Y11 + VMOVDQA Y4, Y15 + MOVQ $0x0000000a, R9 openAVX2320InnerCipherLoop: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2 - DECQ itr2 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + DECQ R9 JNE openAVX2320InnerCipherLoop - - VMOVDQA ·chacha20Constants<>(SB), TT0 - VPADDD TT0, AA0, AA0; VPADDD TT0, AA1, AA1; VPADDD TT0, AA2, AA2 - VPADDD TT1, BB0, BB0; VPADDD TT1, BB1, BB1; VPADDD TT1, BB2, BB2 - VPADDD TT2, CC0, CC0; VPADDD TT2, CC1, CC1; VPADDD TT2, CC2, CC2 - VMOVDQA ·avx2IncMask<>(SB), TT0 - VPADDD TT3, DD0, DD0; VPADDD TT0, TT3, TT3 - VPADDD TT3, DD1, DD1; VPADDD TT0, TT3, TT3 - VPADDD TT3, DD2, DD2 + VMOVDQA ·chacha20Constants<>+0(SB), Y3 + VPADDD Y3, Y0, Y0 + VPADDD Y3, Y5, Y5 + VPADDD Y3, Y6, Y6 + VPADDD Y7, Y14, Y14 + VPADDD Y7, Y9, Y9 + VPADDD Y7, Y10, Y10 + VPADDD Y11, Y12, Y12 + VPADDD Y11, Y13, Y13 + VPADDD Y11, Y8, Y8 + VMOVDQA ·avx2IncMask<>+0(SB), Y3 + VPADDD Y15, Y4, Y4 + VPADDD Y3, Y15, Y15 + VPADDD Y15, Y1, Y1 + VPADDD Y3, Y15, Y15 + VPADDD Y15, Y2, Y2 // Clamp and store poly key - VPERM2I128 $0x02, AA0, BB0, TT0 - VPAND ·polyClampMask<>(SB), TT0, TT0 - VMOVDQA TT0, rsStoreAVX2 + VPERM2I128 $0x02, Y0, Y14, Y3 + VPAND ·polyClampMask<>+0(SB), Y3, Y3 + VMOVDQA Y3, (BP) // Stream for up to 320 bytes - VPERM2I128 $0x13, AA0, BB0, AA0 - VPERM2I128 $0x13, CC0, DD0, BB0 - VPERM2I128 $0x02, AA1, BB1, CC0 - VPERM2I128 $0x02, CC1, DD1, DD0 - VPERM2I128 $0x13, AA1, BB1, AA1 - VPERM2I128 $0x13, CC1, DD1, BB1 - VPERM2I128 $0x02, AA2, BB2, CC1 - VPERM2I128 $0x02, CC2, DD2, DD1 - VPERM2I128 $0x13, AA2, BB2, AA2 - VPERM2I128 $0x13, CC2, DD2, BB2 + VPERM2I128 $0x13, Y0, Y14, Y0 + VPERM2I128 $0x13, Y12, Y4, Y14 + VPERM2I128 $0x02, Y5, Y9, Y12 + VPERM2I128 $0x02, Y13, Y1, Y4 + VPERM2I128 $0x13, Y5, Y9, Y5 + VPERM2I128 $0x13, Y13, Y1, Y9 + VPERM2I128 $0x02, Y6, Y10, Y13 + VPERM2I128 $0x02, Y8, Y2, Y1 + VPERM2I128 $0x13, Y6, Y10, Y6 + VPERM2I128 $0x13, Y8, Y2, Y10 JMP openAVX2ShortOpen -// ---------------------------------------------------------------------------- -// Special optimization for the last 128 bytes of ciphertext openAVX2Tail128: // Need to decrypt up to 128 bytes - prepare two blocks - VMOVDQA ·chacha20Constants<>(SB), AA1 - VMOVDQA state1StoreAVX2, BB1 - VMOVDQA state2StoreAVX2, CC1 - VMOVDQA ctr3StoreAVX2, DD1 - VPADDD ·avx2IncMask<>(SB), DD1, DD1 - VMOVDQA DD1, DD0 - - XORQ itr2, itr2 - MOVQ inl, itr1 - ANDQ $-16, itr1 - TESTQ itr1, itr1 - JE openAVX2Tail128LoopB + VMOVDQA ·chacha20Constants<>+0(SB), Y5 + VMOVDQA 32(BP), Y9 + VMOVDQA 64(BP), Y13 + VMOVDQA 192(BP), Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y1 + VMOVDQA Y1, Y4 + XORQ R9, R9 + MOVQ BX, CX + ANDQ $-16, CX + TESTQ CX, CX + JE openAVX2Tail128LoopB openAVX2Tail128LoopA: - // Perform ChaCha rounds, while hashing the remaining input - polyAdd(0(inp)(itr2*1)) - polyMulAVX2 + ADDQ (SI)(R9*1), R10 + ADCQ 8(SI)(R9*1), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 openAVX2Tail128LoopB: - ADDQ $16, itr2 - chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $4, BB1, BB1, BB1 - VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $12, DD1, DD1, DD1 - chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $12, BB1, BB1, BB1 - VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $4, DD1, DD1, DD1 - CMPQ itr2, itr1 - JB openAVX2Tail128LoopA - CMPQ itr2, $160 - JNE openAVX2Tail128LoopB - - VPADDD ·chacha20Constants<>(SB), AA1, AA1 - VPADDD state1StoreAVX2, BB1, BB1 - VPADDD state2StoreAVX2, CC1, CC1 - VPADDD DD0, DD1, DD1 - VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0 + ADDQ $0x10, R9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y1, Y1, Y1 + CMPQ R9, CX + JB openAVX2Tail128LoopA + CMPQ R9, $0xa0 + JNE openAVX2Tail128LoopB + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD 32(BP), Y9, Y9 + VPADDD 64(BP), Y13, Y13 + VPADDD Y4, Y1, Y1 + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 openAVX2TailLoop: - CMPQ inl, $32 + CMPQ BX, $0x20 JB openAVX2Tail - SUBQ $32, inl + SUBQ $0x20, BX // Load for decryption - VPXOR (inp), AA0, AA0 - VMOVDQU AA0, (oup) - LEAQ (1*32)(inp), inp - LEAQ (1*32)(oup), oup - VMOVDQA BB0, AA0 - VMOVDQA CC0, BB0 - VMOVDQA DD0, CC0 + VPXOR (SI), Y0, Y0 + VMOVDQU Y0, (DI) + LEAQ 32(SI), SI + LEAQ 32(DI), DI + VMOVDQA Y14, Y0 + VMOVDQA Y12, Y14 + VMOVDQA Y4, Y12 JMP openAVX2TailLoop openAVX2Tail: - CMPQ inl, $16 - VMOVDQA A0, A1 + CMPQ BX, $0x10 + VMOVDQA X0, X1 JB openAVX2TailDone - SUBQ $16, inl + SUBQ $0x10, BX // Load for decryption - VPXOR (inp), A0, T0 - VMOVDQU T0, (oup) - LEAQ (1*16)(inp), inp - LEAQ (1*16)(oup), oup - VPERM2I128 $0x11, AA0, AA0, AA0 - VMOVDQA A0, A1 + VPXOR (SI), X0, X12 + VMOVDQU X12, (DI) + LEAQ 16(SI), SI + LEAQ 16(DI), DI + VPERM2I128 $0x11, Y0, Y0, Y0 + VMOVDQA X0, X1 openAVX2TailDone: VZEROUPPER JMP openSSETail16 -// ---------------------------------------------------------------------------- -// Special optimization for the last 256 bytes of ciphertext openAVX2Tail256: - // Need to decrypt up to 256 bytes - prepare four blocks - VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1 - VMOVDQA ctr3StoreAVX2, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD1 - VMOVDQA DD0, TT1 - VMOVDQA DD1, TT2 + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y4, Y7 + VMOVDQA Y1, Y11 // Compute the number of iterations that will hash data - MOVQ inl, tmpStoreAVX2 - MOVQ inl, itr1 - SUBQ $128, itr1 - SHRQ $4, itr1 - MOVQ $10, itr2 - CMPQ itr1, $10 - CMOVQGT itr2, itr1 - MOVQ inp, inl - XORQ itr2, itr2 + MOVQ BX, 224(BP) + MOVQ BX, CX + SUBQ $0x80, CX + SHRQ $0x04, CX + MOVQ $0x0000000a, R9 + CMPQ CX, $0x0a + CMOVQGT R9, CX + MOVQ SI, BX + XORQ R9, R9 openAVX2Tail256LoopA: - polyAdd(0(inl)) - polyMulAVX2 - LEAQ 16(inl), inl + ADDQ (BX), R10 + ADCQ 8(BX), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(BX), BX - // Perform ChaCha rounds, while hashing the remaining input openAVX2Tail256LoopB: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1 - INCQ itr2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1 - CMPQ itr2, itr1 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + INCQ R9 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + CMPQ R9, CX JB openAVX2Tail256LoopA + CMPQ R9, $0x0a + JNE openAVX2Tail256LoopB + MOVQ BX, R9 + SUBQ SI, BX + MOVQ BX, CX + MOVQ 224(BP), BX - CMPQ itr2, $10 - JNE openAVX2Tail256LoopB - - MOVQ inl, itr2 - SUBQ inp, inl - MOVQ inl, itr1 - MOVQ tmpStoreAVX2, inl - - // Hash the remainder of data (if any) openAVX2Tail256Hash: - ADDQ $16, itr1 - CMPQ itr1, inl - JGT openAVX2Tail256HashEnd - polyAdd (0(itr2)) - polyMulAVX2 - LEAQ 16(itr2), itr2 - JMP openAVX2Tail256Hash - -// Store 128 bytes safely, then go to store loop + ADDQ $0x10, CX + CMPQ CX, BX + JGT openAVX2Tail256HashEnd + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(R9), R9 + JMP openAVX2Tail256Hash + openAVX2Tail256HashEnd: - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1 - VPADDD TT1, DD0, DD0; VPADDD TT2, DD1, DD1 - VPERM2I128 $0x02, AA0, BB0, AA2; VPERM2I128 $0x02, CC0, DD0, BB2; VPERM2I128 $0x13, AA0, BB0, CC2; VPERM2I128 $0x13, CC0, DD0, DD2 - VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0 - - VPXOR (0*32)(inp), AA2, AA2; VPXOR (1*32)(inp), BB2, BB2; VPXOR (2*32)(inp), CC2, CC2; VPXOR (3*32)(inp), DD2, DD2 - VMOVDQU AA2, (0*32)(oup); VMOVDQU BB2, (1*32)(oup); VMOVDQU CC2, (2*32)(oup); VMOVDQU DD2, (3*32)(oup) - LEAQ (4*32)(inp), inp - LEAQ (4*32)(oup), oup - SUBQ $4*32, inl - - JMP openAVX2TailLoop - -// ---------------------------------------------------------------------------- -// Special optimization for the last 384 bytes of ciphertext + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD Y7, Y4, Y4 + VPADDD Y11, Y1, Y1 + VPERM2I128 $0x02, Y0, Y14, Y6 + VPERM2I128 $0x02, Y12, Y4, Y10 + VPERM2I128 $0x13, Y0, Y14, Y8 + VPERM2I128 $0x13, Y12, Y4, Y2 + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + VPXOR (SI), Y6, Y6 + VPXOR 32(SI), Y10, Y10 + VPXOR 64(SI), Y8, Y8 + VPXOR 96(SI), Y2, Y2 + VMOVDQU Y6, (DI) + VMOVDQU Y10, 32(DI) + VMOVDQU Y8, 64(DI) + VMOVDQU Y2, 96(DI) + LEAQ 128(SI), SI + LEAQ 128(DI), DI + SUBQ $0x80, BX + JMP openAVX2TailLoop + openAVX2Tail384: // Need to decrypt up to 384 bytes - prepare six blocks - VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2 - VMOVDQA ctr3StoreAVX2, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD1 - VPADDD ·avx2IncMask<>(SB), DD1, DD2 - VMOVDQA DD0, ctr0StoreAVX2 - VMOVDQA DD1, ctr1StoreAVX2 - VMOVDQA DD2, ctr2StoreAVX2 + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VMOVDQA Y4, 96(BP) + VMOVDQA Y1, 128(BP) + VMOVDQA Y2, 160(BP) // Compute the number of iterations that will hash two blocks of data - MOVQ inl, tmpStoreAVX2 - MOVQ inl, itr1 - SUBQ $256, itr1 - SHRQ $4, itr1 - ADDQ $6, itr1 - MOVQ $10, itr2 - CMPQ itr1, $10 - CMOVQGT itr2, itr1 - MOVQ inp, inl - XORQ itr2, itr2 - - // Perform ChaCha rounds, while hashing the remaining input + MOVQ BX, 224(BP) + MOVQ BX, CX + SUBQ $0x00000100, CX + SHRQ $0x04, CX + ADDQ $0x06, CX + MOVQ $0x0000000a, R9 + CMPQ CX, $0x0a + CMOVQGT R9, CX + MOVQ SI, BX + XORQ R9, R9 + openAVX2Tail384LoopB: - polyAdd(0(inl)) - polyMulAVX2 - LEAQ 16(inl), inl + ADDQ (BX), R10 + ADCQ 8(BX), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(BX), BX openAVX2Tail384LoopA: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2 - polyAdd(0(inl)) - polyMulAVX2 - LEAQ 16(inl), inl - INCQ itr2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2 - - CMPQ itr2, itr1 - JB openAVX2Tail384LoopB - - CMPQ itr2, $10 - JNE openAVX2Tail384LoopA - - MOVQ inl, itr2 - SUBQ inp, inl - MOVQ inl, itr1 - MOVQ tmpStoreAVX2, inl + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + ADDQ (BX), R10 + ADCQ 8(BX), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(BX), BX + INCQ R9 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + CMPQ R9, CX + JB openAVX2Tail384LoopB + CMPQ R9, $0x0a + JNE openAVX2Tail384LoopA + MOVQ BX, R9 + SUBQ SI, BX + MOVQ BX, CX + MOVQ 224(BP), BX openAVX2Tail384Hash: - ADDQ $16, itr1 - CMPQ itr1, inl - JGT openAVX2Tail384HashEnd - polyAdd(0(itr2)) - polyMulAVX2 - LEAQ 16(itr2), itr2 - JMP openAVX2Tail384Hash - -// Store 256 bytes safely, then go to store loop + ADDQ $0x10, CX + CMPQ CX, BX + JGT openAVX2Tail384HashEnd + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(R9), R9 + JMP openAVX2Tail384Hash + openAVX2Tail384HashEnd: - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2 - VPADDD ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2 - VPERM2I128 $0x02, AA0, BB0, TT0; VPERM2I128 $0x02, CC0, DD0, TT1; VPERM2I128 $0x13, AA0, BB0, TT2; VPERM2I128 $0x13, CC0, DD0, TT3 - VPXOR (0*32)(inp), TT0, TT0; VPXOR (1*32)(inp), TT1, TT1; VPXOR (2*32)(inp), TT2, TT2; VPXOR (3*32)(inp), TT3, TT3 - VMOVDQU TT0, (0*32)(oup); VMOVDQU TT1, (1*32)(oup); VMOVDQU TT2, (2*32)(oup); VMOVDQU TT3, (3*32)(oup) - VPERM2I128 $0x02, AA1, BB1, TT0; VPERM2I128 $0x02, CC1, DD1, TT1; VPERM2I128 $0x13, AA1, BB1, TT2; VPERM2I128 $0x13, CC1, DD1, TT3 - VPXOR (4*32)(inp), TT0, TT0; VPXOR (5*32)(inp), TT1, TT1; VPXOR (6*32)(inp), TT2, TT2; VPXOR (7*32)(inp), TT3, TT3 - VMOVDQU TT0, (4*32)(oup); VMOVDQU TT1, (5*32)(oup); VMOVDQU TT2, (6*32)(oup); VMOVDQU TT3, (7*32)(oup) - VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0 - LEAQ (8*32)(inp), inp - LEAQ (8*32)(oup), oup - SUBQ $8*32, inl + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD 96(BP), Y4, Y4 + VPADDD 128(BP), Y1, Y1 + VPADDD 160(BP), Y2, Y2 + VPERM2I128 $0x02, Y0, Y14, Y3 + VPERM2I128 $0x02, Y12, Y4, Y7 + VPERM2I128 $0x13, Y0, Y14, Y11 + VPERM2I128 $0x13, Y12, Y4, Y15 + VPXOR (SI), Y3, Y3 + VPXOR 32(SI), Y7, Y7 + VPXOR 64(SI), Y11, Y11 + VPXOR 96(SI), Y15, Y15 + VMOVDQU Y3, (DI) + VMOVDQU Y7, 32(DI) + VMOVDQU Y11, 64(DI) + VMOVDQU Y15, 96(DI) + VPERM2I128 $0x02, Y5, Y9, Y3 + VPERM2I128 $0x02, Y13, Y1, Y7 + VPERM2I128 $0x13, Y5, Y9, Y11 + VPERM2I128 $0x13, Y13, Y1, Y15 + VPXOR 128(SI), Y3, Y3 + VPXOR 160(SI), Y7, Y7 + VPXOR 192(SI), Y11, Y11 + VPXOR 224(SI), Y15, Y15 + VMOVDQU Y3, 128(DI) + VMOVDQU Y7, 160(DI) + VMOVDQU Y11, 192(DI) + VMOVDQU Y15, 224(DI) + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + LEAQ 256(SI), SI + LEAQ 256(DI), DI + SUBQ $0x00000100, BX JMP openAVX2TailLoop -// ---------------------------------------------------------------------------- -// Special optimization for the last 512 bytes of ciphertext openAVX2Tail512: - VMOVDQU ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3 - VMOVDQA ctr3StoreAVX2, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3 - VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2 - XORQ itr1, itr1 - MOVQ inp, itr2 + VMOVDQU ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA Y0, Y7 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA Y14, Y11 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA Y12, Y15 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 + VMOVDQA Y4, 96(BP) + VMOVDQA Y1, 128(BP) + VMOVDQA Y2, 160(BP) + VMOVDQA Y3, 192(BP) + XORQ CX, CX + MOVQ SI, R9 openAVX2Tail512LoopB: - polyAdd(0(itr2)) - polyMulAVX2 - LEAQ (2*8)(itr2), itr2 + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(R9), R9 openAVX2Tail512LoopA: - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyAdd(0*8(itr2)) - polyMulAVX2 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - polyAdd(2*8(itr2)) - polyMulAVX2 - LEAQ (4*8)(itr2), itr2 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3 - INCQ itr1 - CMPQ itr1, $4 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x04, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPALIGNR $0x0c, Y3, Y3, Y3 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + ADDQ 16(R9), R10 + ADCQ 24(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(R9), R9 + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x0c, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + VPALIGNR $0x04, Y3, Y3, Y3 + INCQ CX + CMPQ CX, $0x04 JLT openAVX2Tail512LoopB - - CMPQ itr1, $10 - JNE openAVX2Tail512LoopA - - MOVQ inl, itr1 - SUBQ $384, itr1 - ANDQ $-16, itr1 + CMPQ CX, $0x0a + JNE openAVX2Tail512LoopA + MOVQ BX, CX + SUBQ $0x00000180, CX + ANDQ $-16, CX openAVX2Tail512HashLoop: - TESTQ itr1, itr1 + TESTQ CX, CX JE openAVX2Tail512HashEnd - polyAdd(0(itr2)) - polyMulAVX2 - LEAQ 16(itr2), itr2 - SUBQ $16, itr1 + ADDQ (R9), R10 + ADCQ 8(R9), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(R9), R9 + SUBQ $0x10, CX JMP openAVX2Tail512HashLoop openAVX2Tail512HashEnd: - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3 - VPADDD ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3 - VMOVDQA CC3, tmpStoreAVX2 - VPERM2I128 $0x02, AA0, BB0, CC3; VPERM2I128 $0x13, AA0, BB0, BB0; VPERM2I128 $0x02, CC0, DD0, AA0; VPERM2I128 $0x13, CC0, DD0, CC0 - VPXOR (0*32)(inp), CC3, CC3; VPXOR (1*32)(inp), AA0, AA0; VPXOR (2*32)(inp), BB0, BB0; VPXOR (3*32)(inp), CC0, CC0 - VMOVDQU CC3, (0*32)(oup); VMOVDQU AA0, (1*32)(oup); VMOVDQU BB0, (2*32)(oup); VMOVDQU CC0, (3*32)(oup) - VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0 - VPXOR (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0 - VMOVDQU AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup) - VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0 - VPXOR (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0 - VMOVDQU AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup) - VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0 - - LEAQ (12*32)(inp), inp - LEAQ (12*32)(oup), oup - SUBQ $12*32, inl - - JMP openAVX2TailLoop - -// ---------------------------------------------------------------------------- -// ---------------------------------------------------------------------------- -// func chacha20Poly1305Seal(dst, key, src, ad []byte) -TEXT ·chacha20Poly1305Seal(SB), 0, $288-96 - // For aligned stack access + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 32(BP), Y11, Y11 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD 64(BP), Y15, Y15 + VPADDD 96(BP), Y4, Y4 + VPADDD 128(BP), Y1, Y1 + VPADDD 160(BP), Y2, Y2 + VPADDD 192(BP), Y3, Y3 + VMOVDQA Y15, 224(BP) + VPERM2I128 $0x02, Y0, Y14, Y15 + VPERM2I128 $0x13, Y0, Y14, Y14 + VPERM2I128 $0x02, Y12, Y4, Y0 + VPERM2I128 $0x13, Y12, Y4, Y12 + VPXOR (SI), Y15, Y15 + VPXOR 32(SI), Y0, Y0 + VPXOR 64(SI), Y14, Y14 + VPXOR 96(SI), Y12, Y12 + VMOVDQU Y15, (DI) + VMOVDQU Y0, 32(DI) + VMOVDQU Y14, 64(DI) + VMOVDQU Y12, 96(DI) + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + VPXOR 128(SI), Y0, Y0 + VPXOR 160(SI), Y14, Y14 + VPXOR 192(SI), Y12, Y12 + VPXOR 224(SI), Y4, Y4 + VMOVDQU Y0, 128(DI) + VMOVDQU Y14, 160(DI) + VMOVDQU Y12, 192(DI) + VMOVDQU Y4, 224(DI) + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + VPXOR 256(SI), Y0, Y0 + VPXOR 288(SI), Y14, Y14 + VPXOR 320(SI), Y12, Y12 + VPXOR 352(SI), Y4, Y4 + VMOVDQU Y0, 256(DI) + VMOVDQU Y14, 288(DI) + VMOVDQU Y12, 320(DI) + VMOVDQU Y4, 352(DI) + VPERM2I128 $0x02, Y7, Y11, Y0 + VPERM2I128 $0x02, 224(BP), Y3, Y14 + VPERM2I128 $0x13, Y7, Y11, Y12 + VPERM2I128 $0x13, 224(BP), Y3, Y4 + LEAQ 384(SI), SI + LEAQ 384(DI), DI + SUBQ $0x00000180, BX + JMP openAVX2TailLoop + +DATA ·chacha20Constants<>+0(SB)/4, $0x61707865 +DATA ·chacha20Constants<>+4(SB)/4, $0x3320646e +DATA ·chacha20Constants<>+8(SB)/4, $0x79622d32 +DATA ·chacha20Constants<>+12(SB)/4, $0x6b206574 +DATA ·chacha20Constants<>+16(SB)/4, $0x61707865 +DATA ·chacha20Constants<>+20(SB)/4, $0x3320646e +DATA ·chacha20Constants<>+24(SB)/4, $0x79622d32 +DATA ·chacha20Constants<>+28(SB)/4, $0x6b206574 +GLOBL ·chacha20Constants<>(SB), RODATA|NOPTR, $32 + +DATA ·polyClampMask<>+0(SB)/8, $0x0ffffffc0fffffff +DATA ·polyClampMask<>+8(SB)/8, $0x0ffffffc0ffffffc +DATA ·polyClampMask<>+16(SB)/8, $0xffffffffffffffff +DATA ·polyClampMask<>+24(SB)/8, $0xffffffffffffffff +GLOBL ·polyClampMask<>(SB), RODATA|NOPTR, $32 + +DATA ·sseIncMask<>+0(SB)/8, $0x0000000000000001 +DATA ·sseIncMask<>+8(SB)/8, $0x0000000000000000 +GLOBL ·sseIncMask<>(SB), RODATA|NOPTR, $16 + +DATA ·andMask<>+0(SB)/8, $0x00000000000000ff +DATA ·andMask<>+8(SB)/8, $0x0000000000000000 +DATA ·andMask<>+16(SB)/8, $0x000000000000ffff +DATA ·andMask<>+24(SB)/8, $0x0000000000000000 +DATA ·andMask<>+32(SB)/8, $0x0000000000ffffff +DATA ·andMask<>+40(SB)/8, $0x0000000000000000 +DATA ·andMask<>+48(SB)/8, $0x00000000ffffffff +DATA ·andMask<>+56(SB)/8, $0x0000000000000000 +DATA ·andMask<>+64(SB)/8, $0x000000ffffffffff +DATA ·andMask<>+72(SB)/8, $0x0000000000000000 +DATA ·andMask<>+80(SB)/8, $0x0000ffffffffffff +DATA ·andMask<>+88(SB)/8, $0x0000000000000000 +DATA ·andMask<>+96(SB)/8, $0x00ffffffffffffff +DATA ·andMask<>+104(SB)/8, $0x0000000000000000 +DATA ·andMask<>+112(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+120(SB)/8, $0x0000000000000000 +DATA ·andMask<>+128(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+136(SB)/8, $0x00000000000000ff +DATA ·andMask<>+144(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+152(SB)/8, $0x000000000000ffff +DATA ·andMask<>+160(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+168(SB)/8, $0x0000000000ffffff +DATA ·andMask<>+176(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+184(SB)/8, $0x00000000ffffffff +DATA ·andMask<>+192(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+200(SB)/8, $0x000000ffffffffff +DATA ·andMask<>+208(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+216(SB)/8, $0x0000ffffffffffff +DATA ·andMask<>+224(SB)/8, $0xffffffffffffffff +DATA ·andMask<>+232(SB)/8, $0x00ffffffffffffff +GLOBL ·andMask<>(SB), RODATA|NOPTR, $240 + +DATA ·avx2InitMask<>+0(SB)/8, $0x0000000000000000 +DATA ·avx2InitMask<>+8(SB)/8, $0x0000000000000000 +DATA ·avx2InitMask<>+16(SB)/8, $0x0000000000000001 +DATA ·avx2InitMask<>+24(SB)/8, $0x0000000000000000 +GLOBL ·avx2InitMask<>(SB), RODATA|NOPTR, $32 + +DATA ·rol16<>+0(SB)/8, $0x0504070601000302 +DATA ·rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a +DATA ·rol16<>+16(SB)/8, $0x0504070601000302 +DATA ·rol16<>+24(SB)/8, $0x0d0c0f0e09080b0a +GLOBL ·rol16<>(SB), RODATA|NOPTR, $32 + +DATA ·rol8<>+0(SB)/8, $0x0605040702010003 +DATA ·rol8<>+8(SB)/8, $0x0e0d0c0f0a09080b +DATA ·rol8<>+16(SB)/8, $0x0605040702010003 +DATA ·rol8<>+24(SB)/8, $0x0e0d0c0f0a09080b +GLOBL ·rol8<>(SB), RODATA|NOPTR, $32 + +DATA ·avx2IncMask<>+0(SB)/8, $0x0000000000000002 +DATA ·avx2IncMask<>+8(SB)/8, $0x0000000000000000 +DATA ·avx2IncMask<>+16(SB)/8, $0x0000000000000002 +DATA ·avx2IncMask<>+24(SB)/8, $0x0000000000000000 +GLOBL ·avx2IncMask<>(SB), RODATA|NOPTR, $32 + +// func chacha20Poly1305Seal(dst []byte, key []uint32, src []byte, ad []byte) +// Requires: AVX, AVX2, BMI2, CMOV, SSE2 +TEXT ·chacha20Poly1305Seal(SB), $288-96 MOVQ SP, BP - ADDQ $32, BP + ADDQ $0x20, BP ANDQ $-32, BP - MOVQ dst+0(FP), oup - MOVQ key+24(FP), keyp - MOVQ src+48(FP), inp - MOVQ src_len+56(FP), inl - MOVQ ad+72(FP), adp - - CMPB ·useAVX2(SB), $1 + MOVQ dst_base+0(FP), DI + MOVQ key_base+24(FP), R8 + MOVQ src_base+48(FP), SI + MOVQ src_len+56(FP), BX + MOVQ ad_base+72(FP), CX + CMPB ·useAVX2+0(SB), $0x01 JE chacha20Poly1305Seal_AVX2 // Special optimization, for very short buffers - CMPQ inl, $128 - JBE sealSSE128 // About 15% faster + CMPQ BX, $0x80 + JBE sealSSE128 // In the seal case - prepare the poly key + 3 blocks of stream in the first iteration - MOVOU ·chacha20Constants<>(SB), A0 - MOVOU (1*16)(keyp), B0 - MOVOU (2*16)(keyp), C0 - MOVOU (3*16)(keyp), D0 + MOVOU ·chacha20Constants<>+0(SB), X0 + MOVOU 16(R8), X3 + MOVOU 32(R8), X6 + MOVOU 48(R8), X9 // Store state on stack for future use - MOVO B0, state1Store - MOVO C0, state2Store + MOVO X3, 32(BP) + MOVO X6, 48(BP) // Load state, increment counter blocks - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1 - MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2 - MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3 + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X1, X2 + MOVO X4, X5 + MOVO X7, X8 + MOVO X10, X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X2, X12 + MOVO X5, X13 + MOVO X8, X14 + MOVO X11, X15 + PADDL ·sseIncMask<>+0(SB), X15 // Store counters - MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store - MOVQ $10, itr2 + MOVO X9, 80(BP) + MOVO X10, 96(BP) + MOVO X11, 112(BP) + MOVO X15, 128(BP) + MOVQ $0x0000000a, R9 sealSSEIntroLoop: - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - shiftB0Left; shiftB1Left; shiftB2Left; shiftB3Left - shiftC0Left; shiftC1Left; shiftC2Left; shiftC3Left - shiftD0Left; shiftD1Left; shiftD2Left; shiftD3Left - - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right - shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right - shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right - DECQ itr2 - JNE sealSSEIntroLoop + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x0c + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x04 + DECQ R9 + JNE sealSSEIntroLoop // Add in the state - PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3 - PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3 - PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3 - PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3 + PADDD ·chacha20Constants<>+0(SB), X0 + PADDD ·chacha20Constants<>+0(SB), X1 + PADDD ·chacha20Constants<>+0(SB), X2 + PADDD ·chacha20Constants<>+0(SB), X12 + PADDD 32(BP), X3 + PADDD 32(BP), X4 + PADDD 32(BP), X5 + PADDD 32(BP), X13 + PADDD 48(BP), X7 + PADDD 48(BP), X8 + PADDD 48(BP), X14 + PADDD 96(BP), X10 + PADDD 112(BP), X11 + PADDD 128(BP), X15 // Clamp and store the key - PAND ·polyClampMask<>(SB), A0 - MOVO A0, rStore - MOVO B0, sStore + PAND ·polyClampMask<>+0(SB), X0 + MOVO X0, (BP) + MOVO X3, 16(BP) // Hash AAD - MOVQ ad_len+80(FP), itr2 - CALL polyHashADInternal<>(SB) - - MOVOU (0*16)(inp), A0; MOVOU (1*16)(inp), B0; MOVOU (2*16)(inp), C0; MOVOU (3*16)(inp), D0 - PXOR A0, A1; PXOR B0, B1; PXOR C0, C1; PXOR D0, D1 - MOVOU A1, (0*16)(oup); MOVOU B1, (1*16)(oup); MOVOU C1, (2*16)(oup); MOVOU D1, (3*16)(oup) - MOVOU (4*16)(inp), A0; MOVOU (5*16)(inp), B0; MOVOU (6*16)(inp), C0; MOVOU (7*16)(inp), D0 - PXOR A0, A2; PXOR B0, B2; PXOR C0, C2; PXOR D0, D2 - MOVOU A2, (4*16)(oup); MOVOU B2, (5*16)(oup); MOVOU C2, (6*16)(oup); MOVOU D2, (7*16)(oup) - - MOVQ $128, itr1 - SUBQ $128, inl - LEAQ 128(inp), inp - - MOVO A3, A1; MOVO B3, B1; MOVO C3, C1; MOVO D3, D1 - - CMPQ inl, $64 - JBE sealSSE128SealHash - - MOVOU (0*16)(inp), A0; MOVOU (1*16)(inp), B0; MOVOU (2*16)(inp), C0; MOVOU (3*16)(inp), D0 - PXOR A0, A3; PXOR B0, B3; PXOR C0, C3; PXOR D0, D3 - MOVOU A3, (8*16)(oup); MOVOU B3, (9*16)(oup); MOVOU C3, (10*16)(oup); MOVOU D3, (11*16)(oup) - - ADDQ $64, itr1 - SUBQ $64, inl - LEAQ 64(inp), inp - - MOVQ $2, itr1 - MOVQ $8, itr2 - - CMPQ inl, $64 - JBE sealSSETail64 - CMPQ inl, $128 - JBE sealSSETail128 - CMPQ inl, $192 - JBE sealSSETail192 + MOVQ ad_len+80(FP), R9 + CALL polyHashADInternal<>(SB) + MOVOU (SI), X0 + MOVOU 16(SI), X3 + MOVOU 32(SI), X6 + MOVOU 48(SI), X9 + PXOR X0, X1 + PXOR X3, X4 + PXOR X6, X7 + PXOR X9, X10 + MOVOU X1, (DI) + MOVOU X4, 16(DI) + MOVOU X7, 32(DI) + MOVOU X10, 48(DI) + MOVOU 64(SI), X0 + MOVOU 80(SI), X3 + MOVOU 96(SI), X6 + MOVOU 112(SI), X9 + PXOR X0, X2 + PXOR X3, X5 + PXOR X6, X8 + PXOR X9, X11 + MOVOU X2, 64(DI) + MOVOU X5, 80(DI) + MOVOU X8, 96(DI) + MOVOU X11, 112(DI) + MOVQ $0x00000080, CX + SUBQ $0x80, BX + LEAQ 128(SI), SI + MOVO X12, X1 + MOVO X13, X4 + MOVO X14, X7 + MOVO X15, X10 + CMPQ BX, $0x40 + JBE sealSSE128SealHash + MOVOU (SI), X0 + MOVOU 16(SI), X3 + MOVOU 32(SI), X6 + MOVOU 48(SI), X9 + PXOR X0, X12 + PXOR X3, X13 + PXOR X6, X14 + PXOR X9, X15 + MOVOU X12, 128(DI) + MOVOU X13, 144(DI) + MOVOU X14, 160(DI) + MOVOU X15, 176(DI) + ADDQ $0x40, CX + SUBQ $0x40, BX + LEAQ 64(SI), SI + MOVQ $0x00000002, CX + MOVQ $0x00000008, R9 + CMPQ BX, $0x40 + JBE sealSSETail64 + CMPQ BX, $0x80 + JBE sealSSETail128 + CMPQ BX, $0xc0 + JBE sealSSETail192 sealSSEMainLoop: // Load state, increment counter blocks - MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0 - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1 - MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2 - MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3 + MOVO ·chacha20Constants<>+0(SB), X0 + MOVO 32(BP), X3 + MOVO 48(BP), X6 + MOVO 128(BP), X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X1, X2 + MOVO X4, X5 + MOVO X7, X8 + MOVO X10, X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X2, X12 + MOVO X5, X13 + MOVO X8, X14 + MOVO X11, X15 + PADDL ·sseIncMask<>+0(SB), X15 // Store counters - MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store + MOVO X9, 80(BP) + MOVO X10, 96(BP) + MOVO X11, 112(BP) + MOVO X15, 128(BP) sealSSEInnerLoop: - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - polyAdd(0(oup)) - shiftB0Left; shiftB1Left; shiftB2Left; shiftB3Left - shiftC0Left; shiftC1Left; shiftC2Left; shiftC3Left - shiftD0Left; shiftD1Left; shiftD2Left; shiftD3Left - polyMulStage1 - polyMulStage2 - LEAQ (2*8)(oup), oup - MOVO C3, tmpStore - chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3) - MOVO tmpStore, C3 - MOVO C1, tmpStore - polyMulStage3 - chachaQR(A3, B3, C3, D3, C1) - MOVO tmpStore, C1 - polyMulReduceStage - shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right - shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right - shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right - DECQ itr2 - JGE sealSSEInnerLoop - polyAdd(0(oup)) - polyMul - LEAQ (2*8)(oup), oup - DECQ itr1 - JG sealSSEInnerLoop + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x0c + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + LEAQ 16(DI), DI + MOVO X14, 64(BP) + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X3 + PXOR X14, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X14) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X3 + PXOR X14, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X4 + PXOR X14, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X14) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X4 + PXOR X14, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x0c, X14 + PSRLL $0x14, X5 + PXOR X14, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X14) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X14 + PSLLL $0x07, X14 + PSRLL $0x19, X5 + PXOR X14, X5 + MOVO 64(BP), X14 + MOVO X7, 64(BP) + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + PADDD X13, X12 + PXOR X12, X15 + ROL16(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x0c, X7 + PSRLL $0x14, X13 + PXOR X7, X13 + PADDD X13, X12 + PXOR X12, X15 + ROL8(X15, X7) + PADDD X15, X14 + PXOR X14, X13 + MOVO X13, X7 + PSLLL $0x07, X7 + PSRLL $0x19, X13 + PXOR X7, X13 + MOVO 64(BP), X7 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x04 + DECQ R9 + JGE sealSSEInnerLoop + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI + DECQ CX + JG sealSSEInnerLoop // Add in the state - PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3 - PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3 - PADDD state2Store, C0; PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3 - PADDD ctr0Store, D0; PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3 - MOVO D3, tmpStore + PADDD ·chacha20Constants<>+0(SB), X0 + PADDD ·chacha20Constants<>+0(SB), X1 + PADDD ·chacha20Constants<>+0(SB), X2 + PADDD ·chacha20Constants<>+0(SB), X12 + PADDD 32(BP), X3 + PADDD 32(BP), X4 + PADDD 32(BP), X5 + PADDD 32(BP), X13 + PADDD 48(BP), X6 + PADDD 48(BP), X7 + PADDD 48(BP), X8 + PADDD 48(BP), X14 + PADDD 80(BP), X9 + PADDD 96(BP), X10 + PADDD 112(BP), X11 + PADDD 128(BP), X15 + MOVO X15, 64(BP) // Load - xor - store - MOVOU (0*16)(inp), D3; PXOR D3, A0 - MOVOU (1*16)(inp), D3; PXOR D3, B0 - MOVOU (2*16)(inp), D3; PXOR D3, C0 - MOVOU (3*16)(inp), D3; PXOR D3, D0 - MOVOU A0, (0*16)(oup) - MOVOU B0, (1*16)(oup) - MOVOU C0, (2*16)(oup) - MOVOU D0, (3*16)(oup) - MOVO tmpStore, D3 - - MOVOU (4*16)(inp), A0; MOVOU (5*16)(inp), B0; MOVOU (6*16)(inp), C0; MOVOU (7*16)(inp), D0 - PXOR A0, A1; PXOR B0, B1; PXOR C0, C1; PXOR D0, D1 - MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup) - MOVOU (8*16)(inp), A0; MOVOU (9*16)(inp), B0; MOVOU (10*16)(inp), C0; MOVOU (11*16)(inp), D0 - PXOR A0, A2; PXOR B0, B2; PXOR C0, C2; PXOR D0, D2 - MOVOU A2, (8*16)(oup); MOVOU B2, (9*16)(oup); MOVOU C2, (10*16)(oup); MOVOU D2, (11*16)(oup) - ADDQ $192, inp - MOVQ $192, itr1 - SUBQ $192, inl - MOVO A3, A1 - MOVO B3, B1 - MOVO C3, C1 - MOVO D3, D1 - CMPQ inl, $64 + MOVOU (SI), X15 + PXOR X15, X0 + MOVOU 16(SI), X15 + PXOR X15, X3 + MOVOU 32(SI), X15 + PXOR X15, X6 + MOVOU 48(SI), X15 + PXOR X15, X9 + MOVOU X0, (DI) + MOVOU X3, 16(DI) + MOVOU X6, 32(DI) + MOVOU X9, 48(DI) + MOVO 64(BP), X15 + MOVOU 64(SI), X0 + MOVOU 80(SI), X3 + MOVOU 96(SI), X6 + MOVOU 112(SI), X9 + PXOR X0, X1 + PXOR X3, X4 + PXOR X6, X7 + PXOR X9, X10 + MOVOU X1, 64(DI) + MOVOU X4, 80(DI) + MOVOU X7, 96(DI) + MOVOU X10, 112(DI) + MOVOU 128(SI), X0 + MOVOU 144(SI), X3 + MOVOU 160(SI), X6 + MOVOU 176(SI), X9 + PXOR X0, X2 + PXOR X3, X5 + PXOR X6, X8 + PXOR X9, X11 + MOVOU X2, 128(DI) + MOVOU X5, 144(DI) + MOVOU X8, 160(DI) + MOVOU X11, 176(DI) + ADDQ $0xc0, SI + MOVQ $0x000000c0, CX + SUBQ $0xc0, BX + MOVO X12, X1 + MOVO X13, X4 + MOVO X14, X7 + MOVO X15, X10 + CMPQ BX, $0x40 JBE sealSSE128SealHash - MOVOU (0*16)(inp), A0; MOVOU (1*16)(inp), B0; MOVOU (2*16)(inp), C0; MOVOU (3*16)(inp), D0 - PXOR A0, A3; PXOR B0, B3; PXOR C0, C3; PXOR D0, D3 - MOVOU A3, (12*16)(oup); MOVOU B3, (13*16)(oup); MOVOU C3, (14*16)(oup); MOVOU D3, (15*16)(oup) - LEAQ 64(inp), inp - SUBQ $64, inl - MOVQ $6, itr1 - MOVQ $4, itr2 - CMPQ inl, $192 + MOVOU (SI), X0 + MOVOU 16(SI), X3 + MOVOU 32(SI), X6 + MOVOU 48(SI), X9 + PXOR X0, X12 + PXOR X3, X13 + PXOR X6, X14 + PXOR X9, X15 + MOVOU X12, 192(DI) + MOVOU X13, 208(DI) + MOVOU X14, 224(DI) + MOVOU X15, 240(DI) + LEAQ 64(SI), SI + SUBQ $0x40, BX + MOVQ $0x00000006, CX + MOVQ $0x00000004, R9 + CMPQ BX, $0xc0 JG sealSSEMainLoop - - MOVQ inl, itr1 - TESTQ inl, inl + MOVQ BX, CX + TESTQ BX, BX JE sealSSE128SealHash - MOVQ $6, itr1 - CMPQ inl, $64 + MOVQ $0x00000006, CX + CMPQ BX, $0x40 JBE sealSSETail64 - CMPQ inl, $128 + CMPQ BX, $0x80 JBE sealSSETail128 JMP sealSSETail192 -// ---------------------------------------------------------------------------- -// Special optimization for the last 64 bytes of plaintext sealSSETail64: - // Need to encrypt up to 64 bytes - prepare single block, hash 192 or 256 bytes - MOVO ·chacha20Constants<>(SB), A1 - MOVO state1Store, B1 - MOVO state2Store, C1 - MOVO ctr3Store, D1 - PADDL ·sseIncMask<>(SB), D1 - MOVO D1, ctr0Store + MOVO ·chacha20Constants<>+0(SB), X1 + MOVO 32(BP), X4 + MOVO 48(BP), X7 + MOVO 128(BP), X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X10, 80(BP) sealSSETail64LoopA: - // Perform ChaCha rounds, while hashing the previously encrypted ciphertext - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI sealSSETail64LoopB: - chachaQR(A1, B1, C1, D1, T1) - shiftB1Left; shiftC1Left; shiftD1Left - chachaQR(A1, B1, C1, D1, T1) - shiftB1Right; shiftC1Right; shiftD1Right - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup - - DECQ itr1 - JG sealSSETail64LoopA - - DECQ itr2 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X13) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X13 + PSLLL $0x0c, X13 + PSRLL $0x14, X4 + PXOR X13, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X13) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X13 + PSLLL $0x07, X13 + PSRLL $0x19, X4 + PXOR X13, X4 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X13) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X13 + PSLLL $0x0c, X13 + PSRLL $0x14, X4 + PXOR X13, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X13) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X13 + PSLLL $0x07, X13 + PSRLL $0x19, X4 + PXOR X13, X4 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI + DECQ CX + JG sealSSETail64LoopA + DECQ R9 JGE sealSSETail64LoopB - PADDL ·chacha20Constants<>(SB), A1 - PADDL state1Store, B1 - PADDL state2Store, C1 - PADDL ctr0Store, D1 + PADDL ·chacha20Constants<>+0(SB), X1 + PADDL 32(BP), X4 + PADDL 48(BP), X7 + PADDL 80(BP), X10 + JMP sealSSE128Seal - JMP sealSSE128Seal - -// ---------------------------------------------------------------------------- -// Special optimization for the last 128 bytes of plaintext sealSSETail128: - // Need to encrypt up to 128 bytes - prepare two blocks, hash 192 or 256 bytes - MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr0Store - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr1Store + MOVO ·chacha20Constants<>+0(SB), X0 + MOVO 32(BP), X3 + MOVO 48(BP), X6 + MOVO 128(BP), X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X9, 80(BP) + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X10, 96(BP) sealSSETail128LoopA: - // Perform ChaCha rounds, while hashing the previously encrypted ciphertext - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI sealSSETail128LoopB: - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0) - shiftB0Left; shiftC0Left; shiftD0Left - shiftB1Left; shiftC1Left; shiftD1Left - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0) - shiftB0Right; shiftC0Right; shiftD0Right - shiftB1Right; shiftC1Right; shiftD1Right - - DECQ itr1 - JG sealSSETail128LoopA - - DECQ itr2 - JGE sealSSETail128LoopB - - PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1 - PADDL state1Store, B0; PADDL state1Store, B1 - PADDL state2Store, C0; PADDL state2Store, C1 - PADDL ctr0Store, D0; PADDL ctr1Store, D1 - - MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3 - PXOR T0, A0; PXOR T1, B0; PXOR T2, C0; PXOR T3, D0 - MOVOU A0, (0*16)(oup); MOVOU B0, (1*16)(oup); MOVOU C0, (2*16)(oup); MOVOU D0, (3*16)(oup) - - MOVQ $64, itr1 - LEAQ 64(inp), inp - SUBQ $64, inl - - JMP sealSSE128SealHash - -// ---------------------------------------------------------------------------- -// Special optimization for the last 192 bytes of plaintext + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + DECQ CX + JG sealSSETail128LoopA + DECQ R9 + JGE sealSSETail128LoopB + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL ·chacha20Constants<>+0(SB), X1 + PADDL 32(BP), X3 + PADDL 32(BP), X4 + PADDL 48(BP), X6 + PADDL 48(BP), X7 + PADDL 80(BP), X9 + PADDL 96(BP), X10 + MOVOU (SI), X12 + MOVOU 16(SI), X13 + MOVOU 32(SI), X14 + MOVOU 48(SI), X15 + PXOR X12, X0 + PXOR X13, X3 + PXOR X14, X6 + PXOR X15, X9 + MOVOU X0, (DI) + MOVOU X3, 16(DI) + MOVOU X6, 32(DI) + MOVOU X9, 48(DI) + MOVQ $0x00000040, CX + LEAQ 64(SI), SI + SUBQ $0x40, BX + JMP sealSSE128SealHash + sealSSETail192: - // Need to encrypt up to 192 bytes - prepare three blocks, hash 192 or 256 bytes - MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr0Store - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr1Store - MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2; MOVO D2, ctr2Store + MOVO ·chacha20Constants<>+0(SB), X0 + MOVO 32(BP), X3 + MOVO 48(BP), X6 + MOVO 128(BP), X9 + PADDL ·sseIncMask<>+0(SB), X9 + MOVO X9, 80(BP) + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X10, 96(BP) + MOVO X1, X2 + MOVO X4, X5 + MOVO X7, X8 + MOVO X10, X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X11, 112(BP) sealSSETail192LoopA: - // Perform ChaCha rounds, while hashing the previously encrypted ciphertext - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI sealSSETail192LoopB: - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Left; shiftC0Left; shiftD0Left - shiftB1Left; shiftC1Left; shiftD1Left - shiftB2Left; shiftC2Left; shiftD2Left - - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup - - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Right; shiftC0Right; shiftD0Right - shiftB1Right; shiftC1Right; shiftD1Right - shiftB2Right; shiftC2Right; shiftD2Right - - DECQ itr1 - JG sealSSETail192LoopA - - DECQ itr2 - JGE sealSSETail192LoopB - - PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2 - PADDL state1Store, B0; PADDL state1Store, B1; PADDL state1Store, B2 - PADDL state2Store, C0; PADDL state2Store, C1; PADDL state2Store, C2 - PADDL ctr0Store, D0; PADDL ctr1Store, D1; PADDL ctr2Store, D2 - - MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3 - PXOR T0, A0; PXOR T1, B0; PXOR T2, C0; PXOR T3, D0 - MOVOU A0, (0*16)(oup); MOVOU B0, (1*16)(oup); MOVOU C0, (2*16)(oup); MOVOU D0, (3*16)(oup) - MOVOU (4*16)(inp), T0; MOVOU (5*16)(inp), T1; MOVOU (6*16)(inp), T2; MOVOU (7*16)(inp), T3 - PXOR T0, A1; PXOR T1, B1; PXOR T2, C1; PXOR T3, D1 - MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup) - - MOVO A2, A1 - MOVO B2, B1 - MOVO C2, C1 - MOVO D2, D1 - MOVQ $128, itr1 - LEAQ 128(inp), inp - SUBQ $128, inl - - JMP sealSSE128SealHash - -// ---------------------------------------------------------------------------- -// Special seal optimization for buffers smaller than 129 bytes + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + DECQ CX + JG sealSSETail192LoopA + DECQ R9 + JGE sealSSETail192LoopB + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL ·chacha20Constants<>+0(SB), X1 + PADDL ·chacha20Constants<>+0(SB), X2 + PADDL 32(BP), X3 + PADDL 32(BP), X4 + PADDL 32(BP), X5 + PADDL 48(BP), X6 + PADDL 48(BP), X7 + PADDL 48(BP), X8 + PADDL 80(BP), X9 + PADDL 96(BP), X10 + PADDL 112(BP), X11 + MOVOU (SI), X12 + MOVOU 16(SI), X13 + MOVOU 32(SI), X14 + MOVOU 48(SI), X15 + PXOR X12, X0 + PXOR X13, X3 + PXOR X14, X6 + PXOR X15, X9 + MOVOU X0, (DI) + MOVOU X3, 16(DI) + MOVOU X6, 32(DI) + MOVOU X9, 48(DI) + MOVOU 64(SI), X12 + MOVOU 80(SI), X13 + MOVOU 96(SI), X14 + MOVOU 112(SI), X15 + PXOR X12, X1 + PXOR X13, X4 + PXOR X14, X7 + PXOR X15, X10 + MOVOU X1, 64(DI) + MOVOU X4, 80(DI) + MOVOU X7, 96(DI) + MOVOU X10, 112(DI) + MOVO X2, X1 + MOVO X5, X4 + MOVO X8, X7 + MOVO X11, X10 + MOVQ $0x00000080, CX + LEAQ 128(SI), SI + SUBQ $0x80, BX + JMP sealSSE128SealHash + sealSSE128: - // For up to 128 bytes of ciphertext and 64 bytes for the poly key, we require to process three blocks - MOVOU ·chacha20Constants<>(SB), A0; MOVOU (1*16)(keyp), B0; MOVOU (2*16)(keyp), C0; MOVOU (3*16)(keyp), D0 - MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1 - MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2 - MOVO B0, T1; MOVO C0, T2; MOVO D1, T3 - MOVQ $10, itr2 + MOVOU ·chacha20Constants<>+0(SB), X0 + MOVOU 16(R8), X3 + MOVOU 32(R8), X6 + MOVOU 48(R8), X9 + MOVO X0, X1 + MOVO X3, X4 + MOVO X6, X7 + MOVO X9, X10 + PADDL ·sseIncMask<>+0(SB), X10 + MOVO X1, X2 + MOVO X4, X5 + MOVO X7, X8 + MOVO X10, X11 + PADDL ·sseIncMask<>+0(SB), X11 + MOVO X3, X13 + MOVO X6, X14 + MOVO X10, X15 + MOVQ $0x0000000a, R9 sealSSE128InnerCipherLoop: - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Left; shiftB1Left; shiftB2Left - shiftC0Left; shiftC1Left; shiftC2Left - shiftD0Left; shiftD1Left; shiftD2Left - chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0) - shiftB0Right; shiftB1Right; shiftB2Right - shiftC0Right; shiftC1Right; shiftC2Right - shiftD0Right; shiftD1Right; shiftD2Right - DECQ itr2 - JNE sealSSE128InnerCipherLoop + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x04 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x0c + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + PADDD X3, X0 + PXOR X0, X9 + ROL16(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X3 + PXOR X12, X3 + PADDD X3, X0 + PXOR X0, X9 + ROL8(X9, X12) + PADDD X9, X6 + PXOR X6, X3 + MOVO X3, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X3 + PXOR X12, X3 + PADDD X4, X1 + PXOR X1, X10 + ROL16(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X4 + PXOR X12, X4 + PADDD X4, X1 + PXOR X1, X10 + ROL8(X10, X12) + PADDD X10, X7 + PXOR X7, X4 + MOVO X4, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X4 + PXOR X12, X4 + PADDD X5, X2 + PXOR X2, X11 + ROL16(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x0c, X12 + PSRLL $0x14, X5 + PXOR X12, X5 + PADDD X5, X2 + PXOR X2, X11 + ROL8(X11, X12) + PADDD X11, X8 + PXOR X8, X5 + MOVO X5, X12 + PSLLL $0x07, X12 + PSRLL $0x19, X5 + PXOR X12, X5 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xe4 + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xed + BYTE $0x0c + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xf6 + BYTE $0x08 + BYTE $0x66 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xff + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc0 + BYTE $0x08 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xc9 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xd2 + BYTE $0x04 + BYTE $0x66 + BYTE $0x45 + BYTE $0x0f + BYTE $0x3a + BYTE $0x0f + BYTE $0xdb + BYTE $0x04 + DECQ R9 + JNE sealSSE128InnerCipherLoop // A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded - PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2 - PADDL T1, B0; PADDL T1, B1; PADDL T1, B2 - PADDL T2, C1; PADDL T2, C2 - PADDL T3, D1; PADDL ·sseIncMask<>(SB), T3; PADDL T3, D2 - PAND ·polyClampMask<>(SB), A0 - MOVOU A0, rStore - MOVOU B0, sStore + PADDL ·chacha20Constants<>+0(SB), X0 + PADDL ·chacha20Constants<>+0(SB), X1 + PADDL ·chacha20Constants<>+0(SB), X2 + PADDL X13, X3 + PADDL X13, X4 + PADDL X13, X5 + PADDL X14, X7 + PADDL X14, X8 + PADDL X15, X10 + PADDL ·sseIncMask<>+0(SB), X15 + PADDL X15, X11 + PAND ·polyClampMask<>+0(SB), X0 + MOVOU X0, (BP) + MOVOU X3, 16(BP) // Hash - MOVQ ad_len+80(FP), itr2 + MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) - XORQ itr1, itr1 + XORQ CX, CX sealSSE128SealHash: - // itr1 holds the number of bytes encrypted but not yet hashed - CMPQ itr1, $16 - JB sealSSE128Seal - polyAdd(0(oup)) - polyMul - - SUBQ $16, itr1 - ADDQ $16, oup - - JMP sealSSE128SealHash + CMPQ CX, $0x10 + JB sealSSE128Seal + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + SUBQ $0x10, CX + ADDQ $0x10, DI + JMP sealSSE128SealHash sealSSE128Seal: - CMPQ inl, $16 + CMPQ BX, $0x10 JB sealSSETail - SUBQ $16, inl + SUBQ $0x10, BX // Load for decryption - MOVOU (inp), T0 - PXOR T0, A1 - MOVOU A1, (oup) - LEAQ (1*16)(inp), inp - LEAQ (1*16)(oup), oup + MOVOU (SI), X12 + PXOR X12, X1 + MOVOU X1, (DI) + LEAQ 16(SI), SI + LEAQ 16(DI), DI // Extract for hashing - MOVQ A1, t0 - PSRLDQ $8, A1 - MOVQ A1, t1 - ADDQ t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2 - polyMul + MOVQ X1, R13 + PSRLDQ $0x08, X1 + MOVQ X1, R14 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 // Shift the stream "left" - MOVO B1, A1 - MOVO C1, B1 - MOVO D1, C1 - MOVO A2, D1 - MOVO B2, A2 - MOVO C2, B2 - MOVO D2, C2 + MOVO X4, X1 + MOVO X7, X4 + MOVO X10, X7 + MOVO X2, X10 + MOVO X5, X2 + MOVO X8, X5 + MOVO X11, X8 JMP sealSSE128Seal sealSSETail: - TESTQ inl, inl + TESTQ BX, BX JE sealSSEFinalize // We can only load the PT one byte at a time to avoid read after end of buffer - MOVQ inl, itr2 - SHLQ $4, itr2 - LEAQ ·andMask<>(SB), t0 - MOVQ inl, itr1 - LEAQ -1(inp)(inl*1), inp - XORQ t2, t2 - XORQ t3, t3 + MOVQ BX, R9 + SHLQ $0x04, R9 + LEAQ ·andMask<>+0(SB), R13 + MOVQ BX, CX + LEAQ -1(SI)(BX*1), SI + XORQ R15, R15 + XORQ R8, R8 XORQ AX, AX sealSSETailLoadLoop: - SHLQ $8, t2, t3 - SHLQ $8, t2 - MOVB (inp), AX - XORQ AX, t2 - LEAQ -1(inp), inp - DECQ itr1 + SHLQ $0x08, R15, R8 + SHLQ $0x08, R15 + MOVB (SI), AX + XORQ AX, R15 + LEAQ -1(SI), SI + DECQ CX JNE sealSSETailLoadLoop - MOVQ t2, 0+tmpStore - MOVQ t3, 8+tmpStore - PXOR 0+tmpStore, A1 - MOVOU A1, (oup) - MOVOU -16(t0)(itr2*1), T0 - PAND T0, A1 - MOVQ A1, t0 - PSRLDQ $8, A1 - MOVQ A1, t1 - ADDQ t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2 - polyMul - - ADDQ inl, oup + MOVQ R15, 64(BP) + MOVQ R8, 72(BP) + PXOR 64(BP), X1 + MOVOU X1, (DI) + MOVOU -16(R13)(R9*1), X12 + PAND X12, X1 + MOVQ X1, R13 + PSRLDQ $0x08, X1 + MOVQ X1, R14 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + ADDQ BX, DI sealSSEFinalize: // Hash in the buffer lengths - ADDQ ad_len+80(FP), acc0 - ADCQ src_len+56(FP), acc1 - ADCQ $1, acc2 - polyMul + ADDQ ad_len+80(FP), R10 + ADCQ src_len+56(FP), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 // Final reduce - MOVQ acc0, t0 - MOVQ acc1, t1 - MOVQ acc2, t2 - SUBQ $-5, acc0 - SBBQ $-1, acc1 - SBBQ $3, acc2 - CMOVQCS t0, acc0 - CMOVQCS t1, acc1 - CMOVQCS t2, acc2 + MOVQ R10, R13 + MOVQ R11, R14 + MOVQ R12, R15 + SUBQ $-5, R10 + SBBQ $-1, R11 + SBBQ $0x03, R12 + CMOVQCS R13, R10 + CMOVQCS R14, R11 + CMOVQCS R15, R12 // Add in the "s" part of the key - ADDQ 0+sStore, acc0 - ADCQ 8+sStore, acc1 + ADDQ 16(BP), R10 + ADCQ 24(BP), R11 // Finally store the tag at the end of the message - MOVQ acc0, (0*8)(oup) - MOVQ acc1, (1*8)(oup) + MOVQ R10, (DI) + MOVQ R11, 8(DI) RET -// ---------------------------------------------------------------------------- -// ------------------------- AVX2 Code ---------------------------------------- chacha20Poly1305Seal_AVX2: VZEROUPPER - VMOVDQU ·chacha20Constants<>(SB), AA0 - BYTE $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x70; BYTE $0x10 // broadcasti128 16(r8), ymm14 - BYTE $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x20 // broadcasti128 32(r8), ymm12 - BYTE $0xc4; BYTE $0xc2; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x30 // broadcasti128 48(r8), ymm4 - VPADDD ·avx2InitMask<>(SB), DD0, DD0 + VMOVDQU ·chacha20Constants<>+0(SB), Y0 + BYTE $0xc4 + BYTE $0x42 + BYTE $0x7d + BYTE $0x5a + BYTE $0x70 + BYTE $0x10 + BYTE $0xc4 + BYTE $0x42 + BYTE $0x7d + BYTE $0x5a + BYTE $0x60 + BYTE $0x20 + BYTE $0xc4 + BYTE $0xc2 + BYTE $0x7d + BYTE $0x5a + BYTE $0x60 + BYTE $0x30 + VPADDD ·avx2InitMask<>+0(SB), Y4, Y4 // Special optimizations, for very short buffers - CMPQ inl, $192 - JBE seal192AVX2 // 33% faster - CMPQ inl, $320 - JBE seal320AVX2 // 17% faster + CMPQ BX, $0x000000c0 + JBE seal192AVX2 + CMPQ BX, $0x00000140 + JBE seal320AVX2 // For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream - VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3 - VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3; VMOVDQA BB0, state1StoreAVX2 - VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3; VMOVDQA CC0, state2StoreAVX2 - VPADDD ·avx2IncMask<>(SB), DD0, DD1; VMOVDQA DD0, ctr0StoreAVX2 - VPADDD ·avx2IncMask<>(SB), DD1, DD2; VMOVDQA DD1, ctr1StoreAVX2 - VPADDD ·avx2IncMask<>(SB), DD2, DD3; VMOVDQA DD2, ctr2StoreAVX2 - VMOVDQA DD3, ctr3StoreAVX2 - MOVQ $10, itr2 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA Y0, Y7 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA Y14, Y11 + VMOVDQA Y14, 32(BP) + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA Y12, Y15 + VMOVDQA Y12, 64(BP) + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y4, 96(BP) + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VMOVDQA Y1, 128(BP) + VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 + VMOVDQA Y2, 160(BP) + VMOVDQA Y3, 192(BP) + MOVQ $0x0000000a, R9 sealAVX2IntroLoop: - VMOVDQA CC3, tmpStoreAVX2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3) - VMOVDQA tmpStoreAVX2, CC3 - VMOVDQA CC1, tmpStoreAVX2 - chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1) - VMOVDQA tmpStoreAVX2, CC1 - - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $12, DD0, DD0, DD0 - VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $12, DD1, DD1, DD1 - VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $12, DD2, DD2, DD2 - VPALIGNR $4, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $12, DD3, DD3, DD3 - - VMOVDQA CC3, tmpStoreAVX2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3) - VMOVDQA tmpStoreAVX2, CC3 - VMOVDQA CC1, tmpStoreAVX2 - chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1) - VMOVDQA tmpStoreAVX2, CC1 - - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $4, DD0, DD0, DD0 - VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $4, DD1, DD1, DD1 - VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $4, DD2, DD2, DD2 - VPALIGNR $12, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $4, DD3, DD3, DD3 - DECQ itr2 - JNE sealAVX2IntroLoop - - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3 - VPADDD ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3 - - VPERM2I128 $0x13, CC0, DD0, CC0 // Stream bytes 96 - 127 - VPERM2I128 $0x02, AA0, BB0, DD0 // The Poly1305 key - VPERM2I128 $0x13, AA0, BB0, AA0 // Stream bytes 64 - 95 + VMOVDQA Y15, 224(BP) + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VMOVDQA 224(BP), Y15 + VMOVDQA Y13, 224(BP) + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x0c, Y11, Y13 + VPSRLD $0x14, Y11, Y11 + VPXOR Y13, Y11, Y11 + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x07, Y11, Y13 + VPSRLD $0x19, Y11, Y11 + VPXOR Y13, Y11, Y11 + VMOVDQA 224(BP), Y13 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPALIGNR $0x04, Y11, Y11, Y11 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x0c, Y3, Y3, Y3 + VMOVDQA Y15, 224(BP) + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VMOVDQA 224(BP), Y15 + VMOVDQA Y13, 224(BP) + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x0c, Y11, Y13 + VPSRLD $0x14, Y11, Y11 + VPXOR Y13, Y11, Y11 + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x07, Y11, Y13 + VPSRLD $0x19, Y11, Y11 + VPXOR Y13, Y11, Y11 + VMOVDQA 224(BP), Y13 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x04, Y2, Y2, Y2 + VPALIGNR $0x0c, Y11, Y11, Y11 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x04, Y3, Y3, Y3 + DECQ R9 + JNE sealAVX2IntroLoop + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 32(BP), Y11, Y11 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD 64(BP), Y15, Y15 + VPADDD 96(BP), Y4, Y4 + VPADDD 128(BP), Y1, Y1 + VPADDD 160(BP), Y2, Y2 + VPADDD 192(BP), Y3, Y3 + VPERM2I128 $0x13, Y12, Y4, Y12 + VPERM2I128 $0x02, Y0, Y14, Y4 + VPERM2I128 $0x13, Y0, Y14, Y0 // Clamp and store poly key - VPAND ·polyClampMask<>(SB), DD0, DD0 - VMOVDQA DD0, rsStoreAVX2 + VPAND ·polyClampMask<>+0(SB), Y4, Y4 + VMOVDQA Y4, (BP) // Hash AD - MOVQ ad_len+80(FP), itr2 + MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) // Can store at least 320 bytes - VPXOR (0*32)(inp), AA0, AA0 - VPXOR (1*32)(inp), CC0, CC0 - VMOVDQU AA0, (0*32)(oup) - VMOVDQU CC0, (1*32)(oup) - - VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0 - VPXOR (2*32)(inp), AA0, AA0; VPXOR (3*32)(inp), BB0, BB0; VPXOR (4*32)(inp), CC0, CC0; VPXOR (5*32)(inp), DD0, DD0 - VMOVDQU AA0, (2*32)(oup); VMOVDQU BB0, (3*32)(oup); VMOVDQU CC0, (4*32)(oup); VMOVDQU DD0, (5*32)(oup) - VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0 - VPXOR (6*32)(inp), AA0, AA0; VPXOR (7*32)(inp), BB0, BB0; VPXOR (8*32)(inp), CC0, CC0; VPXOR (9*32)(inp), DD0, DD0 - VMOVDQU AA0, (6*32)(oup); VMOVDQU BB0, (7*32)(oup); VMOVDQU CC0, (8*32)(oup); VMOVDQU DD0, (9*32)(oup) - - MOVQ $320, itr1 - SUBQ $320, inl - LEAQ 320(inp), inp - - VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, CC3, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, CC3, DD3, DD0 - CMPQ inl, $128 + VPXOR (SI), Y0, Y0 + VPXOR 32(SI), Y12, Y12 + VMOVDQU Y0, (DI) + VMOVDQU Y12, 32(DI) + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + VPXOR 64(SI), Y0, Y0 + VPXOR 96(SI), Y14, Y14 + VPXOR 128(SI), Y12, Y12 + VPXOR 160(SI), Y4, Y4 + VMOVDQU Y0, 64(DI) + VMOVDQU Y14, 96(DI) + VMOVDQU Y12, 128(DI) + VMOVDQU Y4, 160(DI) + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + VPXOR 192(SI), Y0, Y0 + VPXOR 224(SI), Y14, Y14 + VPXOR 256(SI), Y12, Y12 + VPXOR 288(SI), Y4, Y4 + VMOVDQU Y0, 192(DI) + VMOVDQU Y14, 224(DI) + VMOVDQU Y12, 256(DI) + VMOVDQU Y4, 288(DI) + MOVQ $0x00000140, CX + SUBQ $0x00000140, BX + LEAQ 320(SI), SI + VPERM2I128 $0x02, Y7, Y11, Y0 + VPERM2I128 $0x02, Y15, Y3, Y14 + VPERM2I128 $0x13, Y7, Y11, Y12 + VPERM2I128 $0x13, Y15, Y3, Y4 + CMPQ BX, $0x80 JBE sealAVX2SealHash - - VPXOR (0*32)(inp), AA0, AA0; VPXOR (1*32)(inp), BB0, BB0; VPXOR (2*32)(inp), CC0, CC0; VPXOR (3*32)(inp), DD0, DD0 - VMOVDQU AA0, (10*32)(oup); VMOVDQU BB0, (11*32)(oup); VMOVDQU CC0, (12*32)(oup); VMOVDQU DD0, (13*32)(oup) - SUBQ $128, inl - LEAQ 128(inp), inp - - MOVQ $8, itr1 - MOVQ $2, itr2 - - CMPQ inl, $128 - JBE sealAVX2Tail128 - CMPQ inl, $256 - JBE sealAVX2Tail256 - CMPQ inl, $384 - JBE sealAVX2Tail384 - CMPQ inl, $512 - JBE sealAVX2Tail512 + VPXOR (SI), Y0, Y0 + VPXOR 32(SI), Y14, Y14 + VPXOR 64(SI), Y12, Y12 + VPXOR 96(SI), Y4, Y4 + VMOVDQU Y0, 320(DI) + VMOVDQU Y14, 352(DI) + VMOVDQU Y12, 384(DI) + VMOVDQU Y4, 416(DI) + SUBQ $0x80, BX + LEAQ 128(SI), SI + MOVQ $0x00000008, CX + MOVQ $0x00000002, R9 + CMPQ BX, $0x80 + JBE sealAVX2Tail128 + CMPQ BX, $0x00000100 + JBE sealAVX2Tail256 + CMPQ BX, $0x00000180 + JBE sealAVX2Tail384 + CMPQ BX, $0x00000200 + JBE sealAVX2Tail512 // We have 448 bytes to hash, but main loop hashes 512 bytes at a time - perform some rounds, before the main loop - VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3 - VMOVDQA ctr3StoreAVX2, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3 - VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2 - - VMOVDQA CC3, tmpStoreAVX2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3) - VMOVDQA tmpStoreAVX2, CC3 - VMOVDQA CC1, tmpStoreAVX2 - chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1) - VMOVDQA tmpStoreAVX2, CC1 - - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $12, DD0, DD0, DD0 - VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $12, DD1, DD1, DD1 - VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $12, DD2, DD2, DD2 - VPALIGNR $4, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $12, DD3, DD3, DD3 - - VMOVDQA CC3, tmpStoreAVX2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3) - VMOVDQA tmpStoreAVX2, CC3 - VMOVDQA CC1, tmpStoreAVX2 - chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1) - VMOVDQA tmpStoreAVX2, CC1 - - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $4, DD0, DD0, DD0 - VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $4, DD1, DD1, DD1 - VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $4, DD2, DD2, DD2 - VPALIGNR $12, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $4, DD3, DD3, DD3 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - - SUBQ $16, oup // Adjust the pointer - MOVQ $9, itr1 - JMP sealAVX2InternalLoopStart + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA Y0, Y7 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA Y14, Y11 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA Y12, Y15 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 + VMOVDQA Y4, 96(BP) + VMOVDQA Y1, 128(BP) + VMOVDQA Y2, 160(BP) + VMOVDQA Y3, 192(BP) + VMOVDQA Y15, 224(BP) + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VMOVDQA 224(BP), Y15 + VMOVDQA Y13, 224(BP) + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x0c, Y11, Y13 + VPSRLD $0x14, Y11, Y11 + VPXOR Y13, Y11, Y11 + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x07, Y11, Y13 + VPSRLD $0x19, Y11, Y11 + VPXOR Y13, Y11, Y11 + VMOVDQA 224(BP), Y13 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPALIGNR $0x04, Y11, Y11, Y11 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x0c, Y3, Y3, Y3 + VMOVDQA Y15, 224(BP) + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VMOVDQA 224(BP), Y15 + VMOVDQA Y13, 224(BP) + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x0c, Y11, Y13 + VPSRLD $0x14, Y11, Y11 + VPXOR Y13, Y11, Y11 + VPADDD Y11, Y7, Y7 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y3, Y15, Y15 + VPXOR Y15, Y11, Y11 + VPSLLD $0x07, Y11, Y13 + VPSRLD $0x19, Y11, Y11 + VPXOR Y13, Y11, Y11 + VMOVDQA 224(BP), Y13 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x04, Y2, Y2, Y2 + VPALIGNR $0x0c, Y11, Y11, Y11 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x04, Y3, Y3, Y3 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + SUBQ $0x10, DI + MOVQ $0x00000009, CX + JMP sealAVX2InternalLoopStart sealAVX2MainLoop: - // Load state, increment counter blocks, store the incremented counters - VMOVDQU ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3 - VMOVDQA ctr3StoreAVX2, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3 - VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2 - MOVQ $10, itr1 + VMOVDQU ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA Y0, Y7 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA Y14, Y11 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA Y12, Y15 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 + VMOVDQA Y4, 96(BP) + VMOVDQA Y1, 128(BP) + VMOVDQA Y2, 160(BP) + VMOVDQA Y3, 192(BP) + MOVQ $0x0000000a, CX sealAVX2InternalLoop: - polyAdd(0*8(oup)) - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - polyMulStage1_AVX2 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - polyMulStage2_AVX2 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - polyMulStage3_AVX2 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulReduceStage + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 sealAVX2InternalLoopStart: - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - polyAdd(2*8(oup)) - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - polyMulStage1_AVX2 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulStage2_AVX2 - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - polyMulStage3_AVX2 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - polyMulReduceStage - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - polyAdd(4*8(oup)) - LEAQ (6*8)(oup), oup - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulStage1_AVX2 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - polyMulStage2_AVX2 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - polyMulStage3_AVX2 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyMulReduceStage - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3 - DECQ itr1 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + ADDQ 16(DI), R10 + ADCQ 24(DI), R11 + ADCQ $0x01, R12 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x04, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPALIGNR $0x0c, Y3, Y3, Y3 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + ADDQ 32(DI), R10 + ADCQ 40(DI), R11 + ADCQ $0x01, R12 + LEAQ 48(DI), DI + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x0c, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + VPALIGNR $0x04, Y3, Y3, Y3 + DECQ CX JNE sealAVX2InternalLoop - - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3 - VPADDD ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3 - VMOVDQA CC3, tmpStoreAVX2 + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 32(BP), Y11, Y11 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD 64(BP), Y15, Y15 + VPADDD 96(BP), Y4, Y4 + VPADDD 128(BP), Y1, Y1 + VPADDD 160(BP), Y2, Y2 + VPADDD 192(BP), Y3, Y3 + VMOVDQA Y15, 224(BP) // We only hashed 480 of the 512 bytes available - hash the remaining 32 here - polyAdd(0*8(oup)) - polyMulAVX2 - LEAQ (4*8)(oup), oup - VPERM2I128 $0x02, AA0, BB0, CC3; VPERM2I128 $0x13, AA0, BB0, BB0; VPERM2I128 $0x02, CC0, DD0, AA0; VPERM2I128 $0x13, CC0, DD0, CC0 - VPXOR (0*32)(inp), CC3, CC3; VPXOR (1*32)(inp), AA0, AA0; VPXOR (2*32)(inp), BB0, BB0; VPXOR (3*32)(inp), CC0, CC0 - VMOVDQU CC3, (0*32)(oup); VMOVDQU AA0, (1*32)(oup); VMOVDQU BB0, (2*32)(oup); VMOVDQU CC0, (3*32)(oup) - VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0 - VPXOR (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0 - VMOVDQU AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup) + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(DI), DI + VPERM2I128 $0x02, Y0, Y14, Y15 + VPERM2I128 $0x13, Y0, Y14, Y14 + VPERM2I128 $0x02, Y12, Y4, Y0 + VPERM2I128 $0x13, Y12, Y4, Y12 + VPXOR (SI), Y15, Y15 + VPXOR 32(SI), Y0, Y0 + VPXOR 64(SI), Y14, Y14 + VPXOR 96(SI), Y12, Y12 + VMOVDQU Y15, (DI) + VMOVDQU Y0, 32(DI) + VMOVDQU Y14, 64(DI) + VMOVDQU Y12, 96(DI) + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + VPXOR 128(SI), Y0, Y0 + VPXOR 160(SI), Y14, Y14 + VPXOR 192(SI), Y12, Y12 + VPXOR 224(SI), Y4, Y4 + VMOVDQU Y0, 128(DI) + VMOVDQU Y14, 160(DI) + VMOVDQU Y12, 192(DI) + VMOVDQU Y4, 224(DI) // and here - polyAdd(-2*8(oup)) - polyMulAVX2 - VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0 - VPXOR (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0 - VMOVDQU AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup) - VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0 - VPXOR (12*32)(inp), AA0, AA0; VPXOR (13*32)(inp), BB0, BB0; VPXOR (14*32)(inp), CC0, CC0; VPXOR (15*32)(inp), DD0, DD0 - VMOVDQU AA0, (12*32)(oup); VMOVDQU BB0, (13*32)(oup); VMOVDQU CC0, (14*32)(oup); VMOVDQU DD0, (15*32)(oup) - LEAQ (32*16)(inp), inp - SUBQ $(32*16), inl - CMPQ inl, $512 + ADDQ -16(DI), R10 + ADCQ -8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + VPXOR 256(SI), Y0, Y0 + VPXOR 288(SI), Y14, Y14 + VPXOR 320(SI), Y12, Y12 + VPXOR 352(SI), Y4, Y4 + VMOVDQU Y0, 256(DI) + VMOVDQU Y14, 288(DI) + VMOVDQU Y12, 320(DI) + VMOVDQU Y4, 352(DI) + VPERM2I128 $0x02, Y7, Y11, Y0 + VPERM2I128 $0x02, 224(BP), Y3, Y14 + VPERM2I128 $0x13, Y7, Y11, Y12 + VPERM2I128 $0x13, 224(BP), Y3, Y4 + VPXOR 384(SI), Y0, Y0 + VPXOR 416(SI), Y14, Y14 + VPXOR 448(SI), Y12, Y12 + VPXOR 480(SI), Y4, Y4 + VMOVDQU Y0, 384(DI) + VMOVDQU Y14, 416(DI) + VMOVDQU Y12, 448(DI) + VMOVDQU Y4, 480(DI) + LEAQ 512(SI), SI + SUBQ $0x00000200, BX + CMPQ BX, $0x00000200 JG sealAVX2MainLoop // Tail can only hash 480 bytes - polyAdd(0*8(oup)) - polyMulAVX2 - polyAdd(2*8(oup)) - polyMulAVX2 - LEAQ 32(oup), oup - - MOVQ $10, itr1 - MOVQ $0, itr2 - CMPQ inl, $128 - JBE sealAVX2Tail128 - CMPQ inl, $256 - JBE sealAVX2Tail256 - CMPQ inl, $384 - JBE sealAVX2Tail384 - JMP sealAVX2Tail512 - -// ---------------------------------------------------------------------------- -// Special optimization for buffers smaller than 193 bytes + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + ADDQ 16(DI), R10 + ADCQ 24(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(DI), DI + MOVQ $0x0000000a, CX + MOVQ $0x00000000, R9 + CMPQ BX, $0x80 + JBE sealAVX2Tail128 + CMPQ BX, $0x00000100 + JBE sealAVX2Tail256 + CMPQ BX, $0x00000180 + JBE sealAVX2Tail384 + JMP sealAVX2Tail512 + seal192AVX2: - // For up to 192 bytes of ciphertext and 64 bytes for the poly key, we process four blocks - VMOVDQA AA0, AA1 - VMOVDQA BB0, BB1 - VMOVDQA CC0, CC1 - VPADDD ·avx2IncMask<>(SB), DD0, DD1 - VMOVDQA AA0, AA2 - VMOVDQA BB0, BB2 - VMOVDQA CC0, CC2 - VMOVDQA DD0, DD2 - VMOVDQA DD1, TT3 - MOVQ $10, itr2 + VMOVDQA Y0, Y5 + VMOVDQA Y14, Y9 + VMOVDQA Y12, Y13 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y0, Y6 + VMOVDQA Y14, Y10 + VMOVDQA Y12, Y8 + VMOVDQA Y4, Y2 + VMOVDQA Y1, Y15 + MOVQ $0x0000000a, R9 sealAVX2192InnerCipherLoop: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1 - DECQ itr2 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + DECQ R9 JNE sealAVX2192InnerCipherLoop - VPADDD AA2, AA0, AA0; VPADDD AA2, AA1, AA1 - VPADDD BB2, BB0, BB0; VPADDD BB2, BB1, BB1 - VPADDD CC2, CC0, CC0; VPADDD CC2, CC1, CC1 - VPADDD DD2, DD0, DD0; VPADDD TT3, DD1, DD1 - VPERM2I128 $0x02, AA0, BB0, TT0 + VPADDD Y6, Y0, Y0 + VPADDD Y6, Y5, Y5 + VPADDD Y10, Y14, Y14 + VPADDD Y10, Y9, Y9 + VPADDD Y8, Y12, Y12 + VPADDD Y8, Y13, Y13 + VPADDD Y2, Y4, Y4 + VPADDD Y15, Y1, Y1 + VPERM2I128 $0x02, Y0, Y14, Y3 // Clamp and store poly key - VPAND ·polyClampMask<>(SB), TT0, TT0 - VMOVDQA TT0, rsStoreAVX2 + VPAND ·polyClampMask<>+0(SB), Y3, Y3 + VMOVDQA Y3, (BP) // Stream for up to 192 bytes - VPERM2I128 $0x13, AA0, BB0, AA0 - VPERM2I128 $0x13, CC0, DD0, BB0 - VPERM2I128 $0x02, AA1, BB1, CC0 - VPERM2I128 $0x02, CC1, DD1, DD0 - VPERM2I128 $0x13, AA1, BB1, AA1 - VPERM2I128 $0x13, CC1, DD1, BB1 + VPERM2I128 $0x13, Y0, Y14, Y0 + VPERM2I128 $0x13, Y12, Y4, Y14 + VPERM2I128 $0x02, Y5, Y9, Y12 + VPERM2I128 $0x02, Y13, Y1, Y4 + VPERM2I128 $0x13, Y5, Y9, Y5 + VPERM2I128 $0x13, Y13, Y1, Y9 sealAVX2ShortSeal: // Hash aad - MOVQ ad_len+80(FP), itr2 + MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) - XORQ itr1, itr1 + XORQ CX, CX sealAVX2SealHash: // itr1 holds the number of bytes encrypted but not yet hashed - CMPQ itr1, $16 - JB sealAVX2ShortSealLoop - polyAdd(0(oup)) - polyMul - SUBQ $16, itr1 - ADDQ $16, oup - JMP sealAVX2SealHash + CMPQ CX, $0x10 + JB sealAVX2ShortSealLoop + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + SUBQ $0x10, CX + ADDQ $0x10, DI + JMP sealAVX2SealHash sealAVX2ShortSealLoop: - CMPQ inl, $32 + CMPQ BX, $0x20 JB sealAVX2ShortTail32 - SUBQ $32, inl + SUBQ $0x20, BX // Load for encryption - VPXOR (inp), AA0, AA0 - VMOVDQU AA0, (oup) - LEAQ (1*32)(inp), inp + VPXOR (SI), Y0, Y0 + VMOVDQU Y0, (DI) + LEAQ 32(SI), SI // Now can hash - polyAdd(0*8(oup)) - polyMulAVX2 - polyAdd(2*8(oup)) - polyMulAVX2 - LEAQ (1*32)(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + ADDQ 16(DI), R10 + ADCQ 24(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(DI), DI // Shift stream left - VMOVDQA BB0, AA0 - VMOVDQA CC0, BB0 - VMOVDQA DD0, CC0 - VMOVDQA AA1, DD0 - VMOVDQA BB1, AA1 - VMOVDQA CC1, BB1 - VMOVDQA DD1, CC1 - VMOVDQA AA2, DD1 - VMOVDQA BB2, AA2 + VMOVDQA Y14, Y0 + VMOVDQA Y12, Y14 + VMOVDQA Y4, Y12 + VMOVDQA Y5, Y4 + VMOVDQA Y9, Y5 + VMOVDQA Y13, Y9 + VMOVDQA Y1, Y13 + VMOVDQA Y6, Y1 + VMOVDQA Y10, Y6 JMP sealAVX2ShortSealLoop sealAVX2ShortTail32: - CMPQ inl, $16 - VMOVDQA A0, A1 + CMPQ BX, $0x10 + VMOVDQA X0, X1 JB sealAVX2ShortDone - - SUBQ $16, inl + SUBQ $0x10, BX // Load for encryption - VPXOR (inp), A0, T0 - VMOVDQU T0, (oup) - LEAQ (1*16)(inp), inp + VPXOR (SI), X0, X12 + VMOVDQU X12, (DI) + LEAQ 16(SI), SI // Hash - polyAdd(0*8(oup)) - polyMulAVX2 - LEAQ (1*16)(oup), oup - VPERM2I128 $0x11, AA0, AA0, AA0 - VMOVDQA A0, A1 + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI + VPERM2I128 $0x11, Y0, Y0, Y0 + VMOVDQA X0, X1 sealAVX2ShortDone: VZEROUPPER JMP sealSSETail -// ---------------------------------------------------------------------------- -// Special optimization for buffers smaller than 321 bytes seal320AVX2: - // For up to 320 bytes of ciphertext and 64 bytes for the poly key, we process six blocks - VMOVDQA AA0, AA1; VMOVDQA BB0, BB1; VMOVDQA CC0, CC1; VPADDD ·avx2IncMask<>(SB), DD0, DD1 - VMOVDQA AA0, AA2; VMOVDQA BB0, BB2; VMOVDQA CC0, CC2; VPADDD ·avx2IncMask<>(SB), DD1, DD2 - VMOVDQA BB0, TT1; VMOVDQA CC0, TT2; VMOVDQA DD0, TT3 - MOVQ $10, itr2 + VMOVDQA Y0, Y5 + VMOVDQA Y14, Y9 + VMOVDQA Y12, Y13 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y0, Y6 + VMOVDQA Y14, Y10 + VMOVDQA Y12, Y8 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VMOVDQA Y14, Y7 + VMOVDQA Y12, Y11 + VMOVDQA Y4, Y15 + MOVQ $0x0000000a, R9 sealAVX2320InnerCipherLoop: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2 - DECQ itr2 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + DECQ R9 JNE sealAVX2320InnerCipherLoop - - VMOVDQA ·chacha20Constants<>(SB), TT0 - VPADDD TT0, AA0, AA0; VPADDD TT0, AA1, AA1; VPADDD TT0, AA2, AA2 - VPADDD TT1, BB0, BB0; VPADDD TT1, BB1, BB1; VPADDD TT1, BB2, BB2 - VPADDD TT2, CC0, CC0; VPADDD TT2, CC1, CC1; VPADDD TT2, CC2, CC2 - VMOVDQA ·avx2IncMask<>(SB), TT0 - VPADDD TT3, DD0, DD0; VPADDD TT0, TT3, TT3 - VPADDD TT3, DD1, DD1; VPADDD TT0, TT3, TT3 - VPADDD TT3, DD2, DD2 + VMOVDQA ·chacha20Constants<>+0(SB), Y3 + VPADDD Y3, Y0, Y0 + VPADDD Y3, Y5, Y5 + VPADDD Y3, Y6, Y6 + VPADDD Y7, Y14, Y14 + VPADDD Y7, Y9, Y9 + VPADDD Y7, Y10, Y10 + VPADDD Y11, Y12, Y12 + VPADDD Y11, Y13, Y13 + VPADDD Y11, Y8, Y8 + VMOVDQA ·avx2IncMask<>+0(SB), Y3 + VPADDD Y15, Y4, Y4 + VPADDD Y3, Y15, Y15 + VPADDD Y15, Y1, Y1 + VPADDD Y3, Y15, Y15 + VPADDD Y15, Y2, Y2 // Clamp and store poly key - VPERM2I128 $0x02, AA0, BB0, TT0 - VPAND ·polyClampMask<>(SB), TT0, TT0 - VMOVDQA TT0, rsStoreAVX2 + VPERM2I128 $0x02, Y0, Y14, Y3 + VPAND ·polyClampMask<>+0(SB), Y3, Y3 + VMOVDQA Y3, (BP) // Stream for up to 320 bytes - VPERM2I128 $0x13, AA0, BB0, AA0 - VPERM2I128 $0x13, CC0, DD0, BB0 - VPERM2I128 $0x02, AA1, BB1, CC0 - VPERM2I128 $0x02, CC1, DD1, DD0 - VPERM2I128 $0x13, AA1, BB1, AA1 - VPERM2I128 $0x13, CC1, DD1, BB1 - VPERM2I128 $0x02, AA2, BB2, CC1 - VPERM2I128 $0x02, CC2, DD2, DD1 - VPERM2I128 $0x13, AA2, BB2, AA2 - VPERM2I128 $0x13, CC2, DD2, BB2 + VPERM2I128 $0x13, Y0, Y14, Y0 + VPERM2I128 $0x13, Y12, Y4, Y14 + VPERM2I128 $0x02, Y5, Y9, Y12 + VPERM2I128 $0x02, Y13, Y1, Y4 + VPERM2I128 $0x13, Y5, Y9, Y5 + VPERM2I128 $0x13, Y13, Y1, Y9 + VPERM2I128 $0x02, Y6, Y10, Y13 + VPERM2I128 $0x02, Y8, Y2, Y1 + VPERM2I128 $0x13, Y6, Y10, Y6 + VPERM2I128 $0x13, Y8, Y2, Y10 JMP sealAVX2ShortSeal -// ---------------------------------------------------------------------------- -// Special optimization for the last 128 bytes of ciphertext sealAVX2Tail128: - // Need to decrypt up to 128 bytes - prepare two blocks - // If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed - // If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed - VMOVDQA ·chacha20Constants<>(SB), AA0 - VMOVDQA state1StoreAVX2, BB0 - VMOVDQA state2StoreAVX2, CC0 - VMOVDQA ctr3StoreAVX2, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD0 - VMOVDQA DD0, DD1 + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA 32(BP), Y14 + VMOVDQA 64(BP), Y12 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VMOVDQA Y4, Y1 sealAVX2Tail128LoopA: - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI sealAVX2Tail128LoopB: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0) - polyAdd(0(oup)) - polyMul - VPALIGNR $4, BB0, BB0, BB0 - VPALIGNR $8, CC0, CC0, CC0 - VPALIGNR $12, DD0, DD0, DD0 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0) - polyAdd(16(oup)) - polyMul - LEAQ 32(oup), oup - VPALIGNR $12, BB0, BB0, BB0 - VPALIGNR $8, CC0, CC0, CC0 - VPALIGNR $4, DD0, DD0, DD0 - DECQ itr1 - JG sealAVX2Tail128LoopA - DECQ itr2 - JGE sealAVX2Tail128LoopB - - VPADDD ·chacha20Constants<>(SB), AA0, AA1 - VPADDD state1StoreAVX2, BB0, BB1 - VPADDD state2StoreAVX2, CC0, CC1 - VPADDD DD1, DD0, DD1 - - VPERM2I128 $0x02, AA1, BB1, AA0 - VPERM2I128 $0x02, CC1, DD1, BB0 - VPERM2I128 $0x13, AA1, BB1, CC0 - VPERM2I128 $0x13, CC1, DD1, DD0 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + ADDQ 16(DI), R10 + ADCQ 24(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(DI), DI + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x04, Y4, Y4, Y4 + DECQ CX + JG sealAVX2Tail128LoopA + DECQ R9 + JGE sealAVX2Tail128LoopB + VPADDD ·chacha20Constants<>+0(SB), Y0, Y5 + VPADDD 32(BP), Y14, Y9 + VPADDD 64(BP), Y12, Y13 + VPADDD Y1, Y4, Y1 + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 JMP sealAVX2ShortSealLoop -// ---------------------------------------------------------------------------- -// Special optimization for the last 256 bytes of ciphertext sealAVX2Tail256: - // Need to decrypt up to 256 bytes - prepare two blocks - // If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed - // If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed - VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA ·chacha20Constants<>(SB), AA1 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA state1StoreAVX2, BB1 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA state2StoreAVX2, CC1 - VMOVDQA ctr3StoreAVX2, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD1 - VMOVDQA DD0, TT1 - VMOVDQA DD1, TT2 + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA ·chacha20Constants<>+0(SB), Y5 + VMOVDQA 32(BP), Y14 + VMOVDQA 32(BP), Y9 + VMOVDQA 64(BP), Y12 + VMOVDQA 64(BP), Y13 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VMOVDQA Y4, Y7 + VMOVDQA Y1, Y11 sealAVX2Tail256LoopA: - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI sealAVX2Tail256LoopB: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - polyAdd(0(oup)) - polyMul - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0) - polyAdd(16(oup)) - polyMul - LEAQ 32(oup), oup - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1 - DECQ itr1 - JG sealAVX2Tail256LoopA - DECQ itr2 - JGE sealAVX2Tail256LoopB - - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1 - VPADDD TT1, DD0, DD0; VPADDD TT2, DD1, DD1 - VPERM2I128 $0x02, AA0, BB0, TT0 - VPERM2I128 $0x02, CC0, DD0, TT1 - VPERM2I128 $0x13, AA0, BB0, TT2 - VPERM2I128 $0x13, CC0, DD0, TT3 - VPXOR (0*32)(inp), TT0, TT0; VPXOR (1*32)(inp), TT1, TT1; VPXOR (2*32)(inp), TT2, TT2; VPXOR (3*32)(inp), TT3, TT3 - VMOVDQU TT0, (0*32)(oup); VMOVDQU TT1, (1*32)(oup); VMOVDQU TT2, (2*32)(oup); VMOVDQU TT3, (3*32)(oup) - MOVQ $128, itr1 - LEAQ 128(inp), inp - SUBQ $128, inl - VPERM2I128 $0x02, AA1, BB1, AA0 - VPERM2I128 $0x02, CC1, DD1, BB0 - VPERM2I128 $0x13, AA1, BB1, CC0 - VPERM2I128 $0x13, CC1, DD1, DD0 - - JMP sealAVX2SealHash - -// ---------------------------------------------------------------------------- -// Special optimization for the last 384 bytes of ciphertext + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + ADDQ 16(DI), R10 + ADCQ 24(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(DI), DI + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + DECQ CX + JG sealAVX2Tail256LoopA + DECQ R9 + JGE sealAVX2Tail256LoopB + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD Y7, Y4, Y4 + VPADDD Y11, Y1, Y1 + VPERM2I128 $0x02, Y0, Y14, Y3 + VPERM2I128 $0x02, Y12, Y4, Y7 + VPERM2I128 $0x13, Y0, Y14, Y11 + VPERM2I128 $0x13, Y12, Y4, Y15 + VPXOR (SI), Y3, Y3 + VPXOR 32(SI), Y7, Y7 + VPXOR 64(SI), Y11, Y11 + VPXOR 96(SI), Y15, Y15 + VMOVDQU Y3, (DI) + VMOVDQU Y7, 32(DI) + VMOVDQU Y11, 64(DI) + VMOVDQU Y15, 96(DI) + MOVQ $0x00000080, CX + LEAQ 128(SI), SI + SUBQ $0x80, BX + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + JMP sealAVX2SealHash + sealAVX2Tail384: - // Need to decrypt up to 384 bytes - prepare two blocks - // If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed - // If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed - VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2 - VMOVDQA ctr3StoreAVX2, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2 - VMOVDQA DD0, TT1; VMOVDQA DD1, TT2; VMOVDQA DD2, TT3 + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VMOVDQA Y4, Y7 + VMOVDQA Y1, Y11 + VMOVDQA Y2, Y15 sealAVX2Tail384LoopA: - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI sealAVX2Tail384LoopB: - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - polyAdd(0(oup)) - polyMul - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2 - chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0) - polyAdd(16(oup)) - polyMul - LEAQ 32(oup), oup - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2 - DECQ itr1 - JG sealAVX2Tail384LoopA - DECQ itr2 - JGE sealAVX2Tail384LoopB - - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2 - VPADDD TT1, DD0, DD0; VPADDD TT2, DD1, DD1; VPADDD TT3, DD2, DD2 - VPERM2I128 $0x02, AA0, BB0, TT0 - VPERM2I128 $0x02, CC0, DD0, TT1 - VPERM2I128 $0x13, AA0, BB0, TT2 - VPERM2I128 $0x13, CC0, DD0, TT3 - VPXOR (0*32)(inp), TT0, TT0; VPXOR (1*32)(inp), TT1, TT1; VPXOR (2*32)(inp), TT2, TT2; VPXOR (3*32)(inp), TT3, TT3 - VMOVDQU TT0, (0*32)(oup); VMOVDQU TT1, (1*32)(oup); VMOVDQU TT2, (2*32)(oup); VMOVDQU TT3, (3*32)(oup) - VPERM2I128 $0x02, AA1, BB1, TT0 - VPERM2I128 $0x02, CC1, DD1, TT1 - VPERM2I128 $0x13, AA1, BB1, TT2 - VPERM2I128 $0x13, CC1, DD1, TT3 - VPXOR (4*32)(inp), TT0, TT0; VPXOR (5*32)(inp), TT1, TT1; VPXOR (6*32)(inp), TT2, TT2; VPXOR (7*32)(inp), TT3, TT3 - VMOVDQU TT0, (4*32)(oup); VMOVDQU TT1, (5*32)(oup); VMOVDQU TT2, (6*32)(oup); VMOVDQU TT3, (7*32)(oup) - MOVQ $256, itr1 - LEAQ 256(inp), inp - SUBQ $256, inl - VPERM2I128 $0x02, AA2, BB2, AA0 - VPERM2I128 $0x02, CC2, DD2, BB0 - VPERM2I128 $0x13, AA2, BB2, CC0 - VPERM2I128 $0x13, CC2, DD2, DD0 - - JMP sealAVX2SealHash - -// ---------------------------------------------------------------------------- -// Special optimization for the last 512 bytes of ciphertext + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x0c, Y14, Y3 + VPSRLD $0x14, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y14, Y0, Y0 + VPXOR Y0, Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPADDD Y4, Y12, Y12 + VPXOR Y12, Y14, Y14 + VPSLLD $0x07, Y14, Y3 + VPSRLD $0x19, Y14, Y14 + VPXOR Y3, Y14, Y14 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x0c, Y9, Y3 + VPSRLD $0x14, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y9, Y5, Y5 + VPXOR Y5, Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPADDD Y1, Y13, Y13 + VPXOR Y13, Y9, Y9 + VPSLLD $0x07, Y9, Y3 + VPSRLD $0x19, Y9, Y9 + VPXOR Y3, Y9, Y9 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x0c, Y10, Y3 + VPSRLD $0x14, Y10, Y10 + VPXOR Y3, Y10, Y10 + VPADDD Y10, Y6, Y6 + VPXOR Y6, Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPADDD Y2, Y8, Y8 + VPXOR Y8, Y10, Y10 + VPSLLD $0x07, Y10, Y3 + VPSRLD $0x19, Y10, Y10 + VPXOR Y3, Y10, Y10 + ADDQ 16(DI), R10 + ADCQ 24(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(DI), DI + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + DECQ CX + JG sealAVX2Tail384LoopA + DECQ R9 + JGE sealAVX2Tail384LoopB + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD Y7, Y4, Y4 + VPADDD Y11, Y1, Y1 + VPADDD Y15, Y2, Y2 + VPERM2I128 $0x02, Y0, Y14, Y3 + VPERM2I128 $0x02, Y12, Y4, Y7 + VPERM2I128 $0x13, Y0, Y14, Y11 + VPERM2I128 $0x13, Y12, Y4, Y15 + VPXOR (SI), Y3, Y3 + VPXOR 32(SI), Y7, Y7 + VPXOR 64(SI), Y11, Y11 + VPXOR 96(SI), Y15, Y15 + VMOVDQU Y3, (DI) + VMOVDQU Y7, 32(DI) + VMOVDQU Y11, 64(DI) + VMOVDQU Y15, 96(DI) + VPERM2I128 $0x02, Y5, Y9, Y3 + VPERM2I128 $0x02, Y13, Y1, Y7 + VPERM2I128 $0x13, Y5, Y9, Y11 + VPERM2I128 $0x13, Y13, Y1, Y15 + VPXOR 128(SI), Y3, Y3 + VPXOR 160(SI), Y7, Y7 + VPXOR 192(SI), Y11, Y11 + VPXOR 224(SI), Y15, Y15 + VMOVDQU Y3, 128(DI) + VMOVDQU Y7, 160(DI) + VMOVDQU Y11, 192(DI) + VMOVDQU Y15, 224(DI) + MOVQ $0x00000100, CX + LEAQ 256(SI), SI + SUBQ $0x00000100, BX + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + JMP sealAVX2SealHash + sealAVX2Tail512: - // Need to decrypt up to 512 bytes - prepare two blocks - // If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed - // If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed - VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3 - VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3 - VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3 - VMOVDQA ctr3StoreAVX2, DD0 - VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3 - VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2 + VMOVDQA ·chacha20Constants<>+0(SB), Y0 + VMOVDQA Y0, Y5 + VMOVDQA Y0, Y6 + VMOVDQA Y0, Y7 + VMOVDQA 32(BP), Y14 + VMOVDQA Y14, Y9 + VMOVDQA Y14, Y10 + VMOVDQA Y14, Y11 + VMOVDQA 64(BP), Y12 + VMOVDQA Y12, Y13 + VMOVDQA Y12, Y8 + VMOVDQA Y12, Y15 + VMOVDQA 192(BP), Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 + VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 + VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 + VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 + VMOVDQA Y4, 96(BP) + VMOVDQA Y1, 128(BP) + VMOVDQA Y2, 160(BP) + VMOVDQA Y3, 192(BP) sealAVX2Tail512LoopA: - polyAdd(0(oup)) - polyMul - LEAQ 16(oup), oup + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), AX + MOVQ AX, R15 + MULQ R10 + MOVQ AX, R13 + MOVQ DX, R14 + MOVQ (BP), AX + MULQ R11 + IMULQ R12, R15 + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), AX + MOVQ AX, R8 + MULQ R10 + ADDQ AX, R14 + ADCQ $0x00, DX + MOVQ DX, R10 + MOVQ 8(BP), AX + MULQ R11 + ADDQ AX, R15 + ADCQ $0x00, DX + IMULQ R12, R8 + ADDQ R10, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 16(DI), DI sealAVX2Tail512LoopB: - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - polyAdd(0*8(oup)) - polyMulAVX2 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - polyAdd(2*8(oup)) - polyMulAVX2 - LEAQ (4*8)(oup), oup - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - VPADDD BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3 - VPXOR AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3 - VPSHUFB ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3 - VPADDD DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3 - VPXOR CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3 - VMOVDQA CC3, tmpStoreAVX2 - VPSLLD $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0 - VPSLLD $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1 - VPSLLD $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2 - VPSLLD $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3 - VMOVDQA tmpStoreAVX2, CC3 - VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3 - VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3 - VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3 - - DECQ itr1 - JG sealAVX2Tail512LoopA - DECQ itr2 - JGE sealAVX2Tail512LoopB - - VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3 - VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3 - VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3 - VPADDD ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3 - VMOVDQA CC3, tmpStoreAVX2 - VPERM2I128 $0x02, AA0, BB0, CC3 - VPXOR (0*32)(inp), CC3, CC3 - VMOVDQU CC3, (0*32)(oup) - VPERM2I128 $0x02, CC0, DD0, CC3 - VPXOR (1*32)(inp), CC3, CC3 - VMOVDQU CC3, (1*32)(oup) - VPERM2I128 $0x13, AA0, BB0, CC3 - VPXOR (2*32)(inp), CC3, CC3 - VMOVDQU CC3, (2*32)(oup) - VPERM2I128 $0x13, CC0, DD0, CC3 - VPXOR (3*32)(inp), CC3, CC3 - VMOVDQU CC3, (3*32)(oup) - - VPERM2I128 $0x02, AA1, BB1, AA0 - VPERM2I128 $0x02, CC1, DD1, BB0 - VPERM2I128 $0x13, AA1, BB1, CC0 - VPERM2I128 $0x13, CC1, DD1, DD0 - VPXOR (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0 - VMOVDQU AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup) - - VPERM2I128 $0x02, AA2, BB2, AA0 - VPERM2I128 $0x02, CC2, DD2, BB0 - VPERM2I128 $0x13, AA2, BB2, CC0 - VPERM2I128 $0x13, CC2, DD2, DD0 - VPXOR (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0 - VMOVDQU AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup) - - MOVQ $384, itr1 - LEAQ 384(inp), inp - SUBQ $384, inl - VPERM2I128 $0x02, AA3, BB3, AA0 - VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0 - VPERM2I128 $0x13, AA3, BB3, CC0 - VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0 - - JMP sealAVX2SealHash + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + ADDQ (DI), R10 + ADCQ 8(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + VPALIGNR $0x04, Y14, Y14, Y14 + VPALIGNR $0x04, Y9, Y9, Y9 + VPALIGNR $0x04, Y10, Y10, Y10 + VPALIGNR $0x04, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x0c, Y4, Y4, Y4 + VPALIGNR $0x0c, Y1, Y1, Y1 + VPALIGNR $0x0c, Y2, Y2, Y2 + VPALIGNR $0x0c, Y3, Y3, Y3 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol16<>+0(SB), Y4, Y4 + VPSHUFB ·rol16<>+0(SB), Y1, Y1 + VPSHUFB ·rol16<>+0(SB), Y2, Y2 + VPSHUFB ·rol16<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + ADDQ 16(DI), R10 + ADCQ 24(DI), R11 + ADCQ $0x01, R12 + MOVQ (BP), DX + MOVQ DX, R15 + MULXQ R10, R13, R14 + IMULQ R12, R15 + MULXQ R11, AX, DX + ADDQ AX, R14 + ADCQ DX, R15 + MOVQ 8(BP), DX + MULXQ R10, R10, AX + ADDQ R10, R14 + MULXQ R11, R11, R8 + ADCQ R11, R15 + ADCQ $0x00, R8 + IMULQ R12, DX + ADDQ AX, R15 + ADCQ DX, R8 + MOVQ R13, R10 + MOVQ R14, R11 + MOVQ R15, R12 + ANDQ $0x03, R12 + MOVQ R15, R13 + ANDQ $-4, R13 + MOVQ R8, R14 + SHRQ $0x02, R8, R15 + SHRQ $0x02, R8 + ADDQ R13, R10 + ADCQ R14, R11 + ADCQ $0x00, R12 + ADDQ R15, R10 + ADCQ R8, R11 + ADCQ $0x00, R12 + LEAQ 32(DI), DI + VMOVDQA Y15, 224(BP) + VPSLLD $0x0c, Y14, Y15 + VPSRLD $0x14, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x0c, Y9, Y15 + VPSRLD $0x14, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x0c, Y10, Y15 + VPSRLD $0x14, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x0c, Y11, Y15 + VPSRLD $0x14, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + VPADDD Y14, Y0, Y0 + VPADDD Y9, Y5, Y5 + VPADDD Y10, Y6, Y6 + VPADDD Y11, Y7, Y7 + VPXOR Y0, Y4, Y4 + VPXOR Y5, Y1, Y1 + VPXOR Y6, Y2, Y2 + VPXOR Y7, Y3, Y3 + VPSHUFB ·rol8<>+0(SB), Y4, Y4 + VPSHUFB ·rol8<>+0(SB), Y1, Y1 + VPSHUFB ·rol8<>+0(SB), Y2, Y2 + VPSHUFB ·rol8<>+0(SB), Y3, Y3 + VPADDD Y4, Y12, Y12 + VPADDD Y1, Y13, Y13 + VPADDD Y2, Y8, Y8 + VPADDD Y3, Y15, Y15 + VPXOR Y12, Y14, Y14 + VPXOR Y13, Y9, Y9 + VPXOR Y8, Y10, Y10 + VPXOR Y15, Y11, Y11 + VMOVDQA Y15, 224(BP) + VPSLLD $0x07, Y14, Y15 + VPSRLD $0x19, Y14, Y14 + VPXOR Y15, Y14, Y14 + VPSLLD $0x07, Y9, Y15 + VPSRLD $0x19, Y9, Y9 + VPXOR Y15, Y9, Y9 + VPSLLD $0x07, Y10, Y15 + VPSRLD $0x19, Y10, Y10 + VPXOR Y15, Y10, Y10 + VPSLLD $0x07, Y11, Y15 + VPSRLD $0x19, Y11, Y11 + VPXOR Y15, Y11, Y11 + VMOVDQA 224(BP), Y15 + VPALIGNR $0x0c, Y14, Y14, Y14 + VPALIGNR $0x0c, Y9, Y9, Y9 + VPALIGNR $0x0c, Y10, Y10, Y10 + VPALIGNR $0x0c, Y11, Y11, Y11 + VPALIGNR $0x08, Y12, Y12, Y12 + VPALIGNR $0x08, Y13, Y13, Y13 + VPALIGNR $0x08, Y8, Y8, Y8 + VPALIGNR $0x08, Y15, Y15, Y15 + VPALIGNR $0x04, Y4, Y4, Y4 + VPALIGNR $0x04, Y1, Y1, Y1 + VPALIGNR $0x04, Y2, Y2, Y2 + VPALIGNR $0x04, Y3, Y3, Y3 + DECQ CX + JG sealAVX2Tail512LoopA + DECQ R9 + JGE sealAVX2Tail512LoopB + VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 + VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 + VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 + VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 + VPADDD 32(BP), Y14, Y14 + VPADDD 32(BP), Y9, Y9 + VPADDD 32(BP), Y10, Y10 + VPADDD 32(BP), Y11, Y11 + VPADDD 64(BP), Y12, Y12 + VPADDD 64(BP), Y13, Y13 + VPADDD 64(BP), Y8, Y8 + VPADDD 64(BP), Y15, Y15 + VPADDD 96(BP), Y4, Y4 + VPADDD 128(BP), Y1, Y1 + VPADDD 160(BP), Y2, Y2 + VPADDD 192(BP), Y3, Y3 + VMOVDQA Y15, 224(BP) + VPERM2I128 $0x02, Y0, Y14, Y15 + VPXOR (SI), Y15, Y15 + VMOVDQU Y15, (DI) + VPERM2I128 $0x02, Y12, Y4, Y15 + VPXOR 32(SI), Y15, Y15 + VMOVDQU Y15, 32(DI) + VPERM2I128 $0x13, Y0, Y14, Y15 + VPXOR 64(SI), Y15, Y15 + VMOVDQU Y15, 64(DI) + VPERM2I128 $0x13, Y12, Y4, Y15 + VPXOR 96(SI), Y15, Y15 + VMOVDQU Y15, 96(DI) + VPERM2I128 $0x02, Y5, Y9, Y0 + VPERM2I128 $0x02, Y13, Y1, Y14 + VPERM2I128 $0x13, Y5, Y9, Y12 + VPERM2I128 $0x13, Y13, Y1, Y4 + VPXOR 128(SI), Y0, Y0 + VPXOR 160(SI), Y14, Y14 + VPXOR 192(SI), Y12, Y12 + VPXOR 224(SI), Y4, Y4 + VMOVDQU Y0, 128(DI) + VMOVDQU Y14, 160(DI) + VMOVDQU Y12, 192(DI) + VMOVDQU Y4, 224(DI) + VPERM2I128 $0x02, Y6, Y10, Y0 + VPERM2I128 $0x02, Y8, Y2, Y14 + VPERM2I128 $0x13, Y6, Y10, Y12 + VPERM2I128 $0x13, Y8, Y2, Y4 + VPXOR 256(SI), Y0, Y0 + VPXOR 288(SI), Y14, Y14 + VPXOR 320(SI), Y12, Y12 + VPXOR 352(SI), Y4, Y4 + VMOVDQU Y0, 256(DI) + VMOVDQU Y14, 288(DI) + VMOVDQU Y12, 320(DI) + VMOVDQU Y4, 352(DI) + MOVQ $0x00000180, CX + LEAQ 384(SI), SI + SUBQ $0x00000180, BX + VPERM2I128 $0x02, Y7, Y11, Y0 + VPERM2I128 $0x02, 224(BP), Y3, Y14 + VPERM2I128 $0x13, Y7, Y11, Y12 + VPERM2I128 $0x13, 224(BP), Y3, Y4 + JMP sealAVX2SealHash diff --git a/src/vendor/golang.org/x/crypto/cryptobyte/asn1.go b/src/vendor/golang.org/x/crypto/cryptobyte/asn1.go index 2492f796af97f1..d25979d9f5330e 100644 --- a/src/vendor/golang.org/x/crypto/cryptobyte/asn1.go +++ b/src/vendor/golang.org/x/crypto/cryptobyte/asn1.go @@ -234,7 +234,7 @@ func (b *Builder) AddASN1(tag asn1.Tag, f BuilderContinuation) { // Identifiers with the low five bits set indicate high-tag-number format // (two or more octets), which we don't support. if tag&0x1f == 0x1f { - b.err = fmt.Errorf("cryptobyte: high-tag number identifier octects not supported: 0x%x", tag) + b.err = fmt.Errorf("cryptobyte: high-tag number identifier octets not supported: 0x%x", tag) return } b.AddUint8(uint8(tag)) diff --git a/src/vendor/golang.org/x/crypto/hkdf/hkdf.go b/src/vendor/golang.org/x/crypto/hkdf/hkdf.go deleted file mode 100644 index 3bee66294ecfc6..00000000000000 --- a/src/vendor/golang.org/x/crypto/hkdf/hkdf.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package hkdf implements the HMAC-based Extract-and-Expand Key Derivation -// Function (HKDF) as defined in RFC 5869. -// -// HKDF is a cryptographic key derivation function (KDF) with the goal of -// expanding limited input keying material into one or more cryptographically -// strong secret keys. -package hkdf - -import ( - "crypto/hmac" - "errors" - "hash" - "io" -) - -// Extract generates a pseudorandom key for use with Expand from an input secret -// and an optional independent salt. -// -// Only use this function if you need to reuse the extracted key with multiple -// Expand invocations and different context values. Most common scenarios, -// including the generation of multiple keys, should use New instead. -func Extract(hash func() hash.Hash, secret, salt []byte) []byte { - if salt == nil { - salt = make([]byte, hash().Size()) - } - extractor := hmac.New(hash, salt) - extractor.Write(secret) - return extractor.Sum(nil) -} - -type hkdf struct { - expander hash.Hash - size int - - info []byte - counter byte - - prev []byte - buf []byte -} - -func (f *hkdf) Read(p []byte) (int, error) { - // Check whether enough data can be generated - need := len(p) - remains := len(f.buf) + int(255-f.counter+1)*f.size - if remains < need { - return 0, errors.New("hkdf: entropy limit reached") - } - // Read any leftover from the buffer - n := copy(p, f.buf) - p = p[n:] - - // Fill the rest of the buffer - for len(p) > 0 { - if f.counter > 1 { - f.expander.Reset() - } - f.expander.Write(f.prev) - f.expander.Write(f.info) - f.expander.Write([]byte{f.counter}) - f.prev = f.expander.Sum(f.prev[:0]) - f.counter++ - - // Copy the new batch into p - f.buf = f.prev - n = copy(p, f.buf) - p = p[n:] - } - // Save leftovers for next run - f.buf = f.buf[n:] - - return need, nil -} - -// Expand returns a Reader, from which keys can be read, using the given -// pseudorandom key and optional context info, skipping the extraction step. -// -// The pseudorandomKey should have been generated by Extract, or be a uniformly -// random or pseudorandom cryptographically strong key. See RFC 5869, Section -// 3.3. Most common scenarios will want to use New instead. -func Expand(hash func() hash.Hash, pseudorandomKey, info []byte) io.Reader { - expander := hmac.New(hash, pseudorandomKey) - return &hkdf{expander, expander.Size(), info, 1, nil, nil} -} - -// New returns a Reader, from which keys can be read, using the given hash, -// secret, salt and context info. Salt and info can be nil. -func New(hash func() hash.Hash, secret, salt, info []byte) io.Reader { - prk := Extract(hash, secret, salt) - return Expand(hash, prk, info) -} diff --git a/src/vendor/golang.org/x/crypto/internal/poly1305/mac_noasm.go b/src/vendor/golang.org/x/crypto/internal/poly1305/mac_noasm.go index 333da285b32a34..8d99551feecbf6 100644 --- a/src/vendor/golang.org/x/crypto/internal/poly1305/mac_noasm.go +++ b/src/vendor/golang.org/x/crypto/internal/poly1305/mac_noasm.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (!amd64 && !ppc64le && !s390x) || !gc || purego +//go:build (!amd64 && !loong64 && !ppc64le && !ppc64 && !s390x) || !gc || purego package poly1305 diff --git a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.go b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.go deleted file mode 100644 index 164cd47d322a32..00000000000000 --- a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build gc && !purego - -package poly1305 - -//go:noescape -func update(state *macState, msg []byte) - -// mac is a wrapper for macGeneric that redirects calls that would have gone to -// updateGeneric to update. -// -// Its Write and Sum methods are otherwise identical to the macGeneric ones, but -// using function pointers would carry a major performance cost. -type mac struct{ macGeneric } - -func (h *mac) Write(p []byte) (int, error) { - nn := len(p) - if h.offset > 0 { - n := copy(h.buffer[h.offset:], p) - if h.offset+n < TagSize { - h.offset += n - return nn, nil - } - p = p[n:] - h.offset = 0 - update(&h.macState, h.buffer[:]) - } - if n := len(p) - (len(p) % TagSize); n > 0 { - update(&h.macState, p[:n]) - p = p[n:] - } - if len(p) > 0 { - h.offset += copy(h.buffer[h.offset:], p) - } - return nn, nil -} - -func (h *mac) Sum(out *[16]byte) { - state := h.macState - if h.offset > 0 { - update(&state, h.buffer[:h.offset]) - } - finalize(out, &state.h, &state.s) -} diff --git a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s index e0d3c64756692b..133757384b787f 100644 --- a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s +++ b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s @@ -1,108 +1,93 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. +// Code generated by command: go run sum_amd64_asm.go -out ../sum_amd64.s -pkg poly1305. DO NOT EDIT. //go:build gc && !purego -#include "textflag.h" - -#define POLY1305_ADD(msg, h0, h1, h2) \ - ADDQ 0(msg), h0; \ - ADCQ 8(msg), h1; \ - ADCQ $1, h2; \ - LEAQ 16(msg), msg - -#define POLY1305_MUL(h0, h1, h2, r0, r1, t0, t1, t2, t3) \ - MOVQ r0, AX; \ - MULQ h0; \ - MOVQ AX, t0; \ - MOVQ DX, t1; \ - MOVQ r0, AX; \ - MULQ h1; \ - ADDQ AX, t1; \ - ADCQ $0, DX; \ - MOVQ r0, t2; \ - IMULQ h2, t2; \ - ADDQ DX, t2; \ - \ - MOVQ r1, AX; \ - MULQ h0; \ - ADDQ AX, t1; \ - ADCQ $0, DX; \ - MOVQ DX, h0; \ - MOVQ r1, t3; \ - IMULQ h2, t3; \ - MOVQ r1, AX; \ - MULQ h1; \ - ADDQ AX, t2; \ - ADCQ DX, t3; \ - ADDQ h0, t2; \ - ADCQ $0, t3; \ - \ - MOVQ t0, h0; \ - MOVQ t1, h1; \ - MOVQ t2, h2; \ - ANDQ $3, h2; \ - MOVQ t2, t0; \ - ANDQ $0xFFFFFFFFFFFFFFFC, t0; \ - ADDQ t0, h0; \ - ADCQ t3, h1; \ - ADCQ $0, h2; \ - SHRQ $2, t3, t2; \ - SHRQ $2, t3; \ - ADDQ t2, h0; \ - ADCQ t3, h1; \ - ADCQ $0, h2 - -// func update(state *[7]uint64, msg []byte) +// func update(state *macState, msg []byte) TEXT ·update(SB), $0-32 MOVQ state+0(FP), DI MOVQ msg_base+8(FP), SI MOVQ msg_len+16(FP), R15 - - MOVQ 0(DI), R8 // h0 - MOVQ 8(DI), R9 // h1 - MOVQ 16(DI), R10 // h2 - MOVQ 24(DI), R11 // r0 - MOVQ 32(DI), R12 // r1 - - CMPQ R15, $16 + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + CMPQ R15, $0x10 JB bytes_between_0_and_15 loop: - POLY1305_ADD(SI, R8, R9, R10) + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ $0x01, R10 + LEAQ 16(SI), SI multiply: - POLY1305_MUL(R8, R9, R10, R11, R12, BX, CX, R13, R14) - SUBQ $16, R15 - CMPQ R15, $16 - JAE loop + MOVQ R11, AX + MULQ R8 + MOVQ AX, BX + MOVQ DX, CX + MOVQ R11, AX + MULQ R9 + ADDQ AX, CX + ADCQ $0x00, DX + MOVQ R11, R13 + IMULQ R10, R13 + ADDQ DX, R13 + MOVQ R12, AX + MULQ R8 + ADDQ AX, CX + ADCQ $0x00, DX + MOVQ DX, R8 + MOVQ R12, R14 + IMULQ R10, R14 + MOVQ R12, AX + MULQ R9 + ADDQ AX, R13 + ADCQ DX, R14 + ADDQ R8, R13 + ADCQ $0x00, R14 + MOVQ BX, R8 + MOVQ CX, R9 + MOVQ R13, R10 + ANDQ $0x03, R10 + MOVQ R13, BX + ANDQ $-4, BX + ADDQ BX, R8 + ADCQ R14, R9 + ADCQ $0x00, R10 + SHRQ $0x02, R14, R13 + SHRQ $0x02, R14 + ADDQ R13, R8 + ADCQ R14, R9 + ADCQ $0x00, R10 + SUBQ $0x10, R15 + CMPQ R15, $0x10 + JAE loop bytes_between_0_and_15: TESTQ R15, R15 JZ done - MOVQ $1, BX + MOVQ $0x00000001, BX XORQ CX, CX XORQ R13, R13 ADDQ R15, SI flush_buffer: - SHLQ $8, BX, CX - SHLQ $8, BX + SHLQ $0x08, BX, CX + SHLQ $0x08, BX MOVB -1(SI), R13 XORQ R13, BX DECQ SI DECQ R15 JNZ flush_buffer - ADDQ BX, R8 ADCQ CX, R9 - ADCQ $0, R10 - MOVQ $16, R15 + ADCQ $0x00, R10 + MOVQ $0x00000010, R15 JMP multiply done: - MOVQ R8, 0(DI) + MOVQ R8, (DI) MOVQ R9, 8(DI) MOVQ R10, 16(DI) RET diff --git a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_asm.go b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_asm.go new file mode 100644 index 00000000000000..315b84ac39bf63 --- /dev/null +++ b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_asm.go @@ -0,0 +1,47 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gc && !purego && (amd64 || loong64 || ppc64 || ppc64le) + +package poly1305 + +//go:noescape +func update(state *macState, msg []byte) + +// mac is a wrapper for macGeneric that redirects calls that would have gone to +// updateGeneric to update. +// +// Its Write and Sum methods are otherwise identical to the macGeneric ones, but +// using function pointers would carry a major performance cost. +type mac struct{ macGeneric } + +func (h *mac) Write(p []byte) (int, error) { + nn := len(p) + if h.offset > 0 { + n := copy(h.buffer[h.offset:], p) + if h.offset+n < TagSize { + h.offset += n + return nn, nil + } + p = p[n:] + h.offset = 0 + update(&h.macState, h.buffer[:]) + } + if n := len(p) - (len(p) % TagSize); n > 0 { + update(&h.macState, p[:n]) + p = p[n:] + } + if len(p) > 0 { + h.offset += copy(h.buffer[h.offset:], p) + } + return nn, nil +} + +func (h *mac) Sum(out *[16]byte) { + state := h.macState + if h.offset > 0 { + update(&state, h.buffer[:h.offset]) + } + finalize(out, &state.h, &state.s) +} diff --git a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_loong64.s b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_loong64.s new file mode 100644 index 00000000000000..bc8361da40244c --- /dev/null +++ b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_loong64.s @@ -0,0 +1,123 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gc && !purego + +// func update(state *macState, msg []byte) +TEXT ·update(SB), $0-32 + MOVV state+0(FP), R4 + MOVV msg_base+8(FP), R5 + MOVV msg_len+16(FP), R6 + + MOVV $0x10, R7 + + MOVV (R4), R8 // h0 + MOVV 8(R4), R9 // h1 + MOVV 16(R4), R10 // h2 + MOVV 24(R4), R11 // r0 + MOVV 32(R4), R12 // r1 + + BLT R6, R7, bytes_between_0_and_15 + +loop: + MOVV (R5), R14 // msg[0:8] + MOVV 8(R5), R16 // msg[8:16] + ADDV R14, R8, R8 // h0 (x1 + y1 = z1', if z1' < x1 then z1' overflow) + ADDV R16, R9, R27 + SGTU R14, R8, R24 // h0.carry + SGTU R9, R27, R28 + ADDV R27, R24, R9 // h1 + SGTU R27, R9, R24 + OR R24, R28, R24 // h1.carry + ADDV $0x01, R24, R24 + ADDV R10, R24, R10 // h2 + + ADDV $16, R5, R5 // msg = msg[16:] + +multiply: + MULV R8, R11, R14 // h0r0.lo + MULHVU R8, R11, R15 // h0r0.hi + MULV R9, R11, R13 // h1r0.lo + MULHVU R9, R11, R16 // h1r0.hi + ADDV R13, R15, R15 + SGTU R13, R15, R24 + ADDV R24, R16, R16 + MULV R10, R11, R25 + ADDV R16, R25, R25 + MULV R8, R12, R13 // h0r1.lo + MULHVU R8, R12, R16 // h0r1.hi + ADDV R13, R15, R15 + SGTU R13, R15, R24 + ADDV R24, R16, R16 + MOVV R16, R8 + MULV R10, R12, R26 // h2r1 + MULV R9, R12, R13 // h1r1.lo + MULHVU R9, R12, R16 // h1r1.hi + ADDV R13, R25, R25 + ADDV R16, R26, R27 + SGTU R13, R25, R24 + ADDV R27, R24, R26 + ADDV R8, R25, R25 + SGTU R8, R25, R24 + ADDV R24, R26, R26 + AND $3, R25, R10 + AND $-4, R25, R17 + ADDV R17, R14, R8 + ADDV R26, R15, R27 + SGTU R17, R8, R24 + SGTU R26, R27, R28 + ADDV R27, R24, R9 + SGTU R27, R9, R24 + OR R24, R28, R24 + ADDV R24, R10, R10 + SLLV $62, R26, R27 + SRLV $2, R25, R28 + SRLV $2, R26, R26 + OR R27, R28, R25 + ADDV R25, R8, R8 + ADDV R26, R9, R27 + SGTU R25, R8, R24 + SGTU R26, R27, R28 + ADDV R27, R24, R9 + SGTU R27, R9, R24 + OR R24, R28, R24 + ADDV R24, R10, R10 + + SUBV $16, R6, R6 + BGE R6, R7, loop + +bytes_between_0_and_15: + BEQ R6, R0, done + MOVV $1, R14 + XOR R15, R15 + ADDV R6, R5, R5 + +flush_buffer: + MOVBU -1(R5), R25 + SRLV $56, R14, R24 + SLLV $8, R15, R28 + SLLV $8, R14, R14 + OR R24, R28, R15 + XOR R25, R14, R14 + SUBV $1, R6, R6 + SUBV $1, R5, R5 + BNE R6, R0, flush_buffer + + ADDV R14, R8, R8 + SGTU R14, R8, R24 + ADDV R15, R9, R27 + SGTU R15, R27, R28 + ADDV R27, R24, R9 + SGTU R27, R9, R24 + OR R24, R28, R24 + ADDV R10, R24, R10 + + MOVV $16, R6 + JMP multiply + +done: + MOVV R8, (R4) + MOVV R9, 8(R4) + MOVV R10, 16(R4) + RET diff --git a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.go b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.go deleted file mode 100644 index 4aec4874b507e8..00000000000000 --- a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build gc && !purego - -package poly1305 - -//go:noescape -func update(state *macState, msg []byte) - -// mac is a wrapper for macGeneric that redirects calls that would have gone to -// updateGeneric to update. -// -// Its Write and Sum methods are otherwise identical to the macGeneric ones, but -// using function pointers would carry a major performance cost. -type mac struct{ macGeneric } - -func (h *mac) Write(p []byte) (int, error) { - nn := len(p) - if h.offset > 0 { - n := copy(h.buffer[h.offset:], p) - if h.offset+n < TagSize { - h.offset += n - return nn, nil - } - p = p[n:] - h.offset = 0 - update(&h.macState, h.buffer[:]) - } - if n := len(p) - (len(p) % TagSize); n > 0 { - update(&h.macState, p[:n]) - p = p[n:] - } - if len(p) > 0 { - h.offset += copy(h.buffer[h.offset:], p) - } - return nn, nil -} - -func (h *mac) Sum(out *[16]byte) { - state := h.macState - if h.offset > 0 { - update(&state, h.buffer[:h.offset]) - } - finalize(out, &state.h, &state.s) -} diff --git a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.s b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.s deleted file mode 100644 index b3c1699bff51a2..00000000000000 --- a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64le.s +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build gc && !purego - -#include "textflag.h" - -// This was ported from the amd64 implementation. - -#define POLY1305_ADD(msg, h0, h1, h2, t0, t1, t2) \ - MOVD (msg), t0; \ - MOVD 8(msg), t1; \ - MOVD $1, t2; \ - ADDC t0, h0, h0; \ - ADDE t1, h1, h1; \ - ADDE t2, h2; \ - ADD $16, msg - -#define POLY1305_MUL(h0, h1, h2, r0, r1, t0, t1, t2, t3, t4, t5) \ - MULLD r0, h0, t0; \ - MULHDU r0, h0, t1; \ - MULLD r0, h1, t4; \ - MULHDU r0, h1, t5; \ - ADDC t4, t1, t1; \ - MULLD r0, h2, t2; \ - MULHDU r1, h0, t4; \ - MULLD r1, h0, h0; \ - ADDE t5, t2, t2; \ - ADDC h0, t1, t1; \ - MULLD h2, r1, t3; \ - ADDZE t4, h0; \ - MULHDU r1, h1, t5; \ - MULLD r1, h1, t4; \ - ADDC t4, t2, t2; \ - ADDE t5, t3, t3; \ - ADDC h0, t2, t2; \ - MOVD $-4, t4; \ - ADDZE t3; \ - RLDICL $0, t2, $62, h2; \ - AND t2, t4, h0; \ - ADDC t0, h0, h0; \ - ADDE t3, t1, h1; \ - SLD $62, t3, t4; \ - SRD $2, t2; \ - ADDZE h2; \ - OR t4, t2, t2; \ - SRD $2, t3; \ - ADDC t2, h0, h0; \ - ADDE t3, h1, h1; \ - ADDZE h2 - -DATA ·poly1305Mask<>+0x00(SB)/8, $0x0FFFFFFC0FFFFFFF -DATA ·poly1305Mask<>+0x08(SB)/8, $0x0FFFFFFC0FFFFFFC -GLOBL ·poly1305Mask<>(SB), RODATA, $16 - -// func update(state *[7]uint64, msg []byte) -TEXT ·update(SB), $0-32 - MOVD state+0(FP), R3 - MOVD msg_base+8(FP), R4 - MOVD msg_len+16(FP), R5 - - MOVD 0(R3), R8 // h0 - MOVD 8(R3), R9 // h1 - MOVD 16(R3), R10 // h2 - MOVD 24(R3), R11 // r0 - MOVD 32(R3), R12 // r1 - - CMP R5, $16 - BLT bytes_between_0_and_15 - -loop: - POLY1305_ADD(R4, R8, R9, R10, R20, R21, R22) - - PCALIGN $16 -multiply: - POLY1305_MUL(R8, R9, R10, R11, R12, R16, R17, R18, R14, R20, R21) - ADD $-16, R5 - CMP R5, $16 - BGE loop - -bytes_between_0_and_15: - CMP R5, $0 - BEQ done - MOVD $0, R16 // h0 - MOVD $0, R17 // h1 - -flush_buffer: - CMP R5, $8 - BLE just1 - - MOVD $8, R21 - SUB R21, R5, R21 - - // Greater than 8 -- load the rightmost remaining bytes in msg - // and put into R17 (h1) - MOVD (R4)(R21), R17 - MOVD $16, R22 - - // Find the offset to those bytes - SUB R5, R22, R22 - SLD $3, R22 - - // Shift to get only the bytes in msg - SRD R22, R17, R17 - - // Put 1 at high end - MOVD $1, R23 - SLD $3, R21 - SLD R21, R23, R23 - OR R23, R17, R17 - - // Remainder is 8 - MOVD $8, R5 - -just1: - CMP R5, $8 - BLT less8 - - // Exactly 8 - MOVD (R4), R16 - - CMP R17, $0 - - // Check if we've already set R17; if not - // set 1 to indicate end of msg. - BNE carry - MOVD $1, R17 - BR carry - -less8: - MOVD $0, R16 // h0 - MOVD $0, R22 // shift count - CMP R5, $4 - BLT less4 - MOVWZ (R4), R16 - ADD $4, R4 - ADD $-4, R5 - MOVD $32, R22 - -less4: - CMP R5, $2 - BLT less2 - MOVHZ (R4), R21 - SLD R22, R21, R21 - OR R16, R21, R16 - ADD $16, R22 - ADD $-2, R5 - ADD $2, R4 - -less2: - CMP R5, $0 - BEQ insert1 - MOVBZ (R4), R21 - SLD R22, R21, R21 - OR R16, R21, R16 - ADD $8, R22 - -insert1: - // Insert 1 at end of msg - MOVD $1, R21 - SLD R22, R21, R21 - OR R16, R21, R16 - -carry: - // Add new values to h0, h1, h2 - ADDC R16, R8 - ADDE R17, R9 - ADDZE R10, R10 - MOVD $16, R5 - ADD R5, R4 - BR multiply - -done: - // Save h0, h1, h2 in state - MOVD R8, 0(R3) - MOVD R9, 8(R3) - MOVD R10, 16(R3) - RET diff --git a/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64x.s b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64x.s new file mode 100644 index 00000000000000..6899a1dabc0b3e --- /dev/null +++ b/src/vendor/golang.org/x/crypto/internal/poly1305/sum_ppc64x.s @@ -0,0 +1,187 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gc && !purego && (ppc64 || ppc64le) + +#include "textflag.h" + +// This was ported from the amd64 implementation. + +#ifdef GOARCH_ppc64le +#define LE_MOVD MOVD +#define LE_MOVWZ MOVWZ +#define LE_MOVHZ MOVHZ +#else +#define LE_MOVD MOVDBR +#define LE_MOVWZ MOVWBR +#define LE_MOVHZ MOVHBR +#endif + +#define POLY1305_ADD(msg, h0, h1, h2, t0, t1, t2) \ + LE_MOVD (msg)( R0), t0; \ + LE_MOVD (msg)(R24), t1; \ + MOVD $1, t2; \ + ADDC t0, h0, h0; \ + ADDE t1, h1, h1; \ + ADDE t2, h2; \ + ADD $16, msg + +#define POLY1305_MUL(h0, h1, h2, r0, r1, t0, t1, t2, t3, t4, t5) \ + MULLD r0, h0, t0; \ + MULHDU r0, h0, t1; \ + MULLD r0, h1, t4; \ + MULHDU r0, h1, t5; \ + ADDC t4, t1, t1; \ + MULLD r0, h2, t2; \ + MULHDU r1, h0, t4; \ + MULLD r1, h0, h0; \ + ADDE t5, t2, t2; \ + ADDC h0, t1, t1; \ + MULLD h2, r1, t3; \ + ADDZE t4, h0; \ + MULHDU r1, h1, t5; \ + MULLD r1, h1, t4; \ + ADDC t4, t2, t2; \ + ADDE t5, t3, t3; \ + ADDC h0, t2, t2; \ + MOVD $-4, t4; \ + ADDZE t3; \ + RLDICL $0, t2, $62, h2; \ + AND t2, t4, h0; \ + ADDC t0, h0, h0; \ + ADDE t3, t1, h1; \ + SLD $62, t3, t4; \ + SRD $2, t2; \ + ADDZE h2; \ + OR t4, t2, t2; \ + SRD $2, t3; \ + ADDC t2, h0, h0; \ + ADDE t3, h1, h1; \ + ADDZE h2 + +// func update(state *[7]uint64, msg []byte) +TEXT ·update(SB), $0-32 + MOVD state+0(FP), R3 + MOVD msg_base+8(FP), R4 + MOVD msg_len+16(FP), R5 + + MOVD 0(R3), R8 // h0 + MOVD 8(R3), R9 // h1 + MOVD 16(R3), R10 // h2 + MOVD 24(R3), R11 // r0 + MOVD 32(R3), R12 // r1 + + MOVD $8, R24 + + CMP R5, $16 + BLT bytes_between_0_and_15 + +loop: + POLY1305_ADD(R4, R8, R9, R10, R20, R21, R22) + + PCALIGN $16 +multiply: + POLY1305_MUL(R8, R9, R10, R11, R12, R16, R17, R18, R14, R20, R21) + ADD $-16, R5 + CMP R5, $16 + BGE loop + +bytes_between_0_and_15: + CMP R5, $0 + BEQ done + MOVD $0, R16 // h0 + MOVD $0, R17 // h1 + +flush_buffer: + CMP R5, $8 + BLE just1 + + MOVD $8, R21 + SUB R21, R5, R21 + + // Greater than 8 -- load the rightmost remaining bytes in msg + // and put into R17 (h1) + LE_MOVD (R4)(R21), R17 + MOVD $16, R22 + + // Find the offset to those bytes + SUB R5, R22, R22 + SLD $3, R22 + + // Shift to get only the bytes in msg + SRD R22, R17, R17 + + // Put 1 at high end + MOVD $1, R23 + SLD $3, R21 + SLD R21, R23, R23 + OR R23, R17, R17 + + // Remainder is 8 + MOVD $8, R5 + +just1: + CMP R5, $8 + BLT less8 + + // Exactly 8 + LE_MOVD (R4), R16 + + CMP R17, $0 + + // Check if we've already set R17; if not + // set 1 to indicate end of msg. + BNE carry + MOVD $1, R17 + BR carry + +less8: + MOVD $0, R16 // h0 + MOVD $0, R22 // shift count + CMP R5, $4 + BLT less4 + LE_MOVWZ (R4), R16 + ADD $4, R4 + ADD $-4, R5 + MOVD $32, R22 + +less4: + CMP R5, $2 + BLT less2 + LE_MOVHZ (R4), R21 + SLD R22, R21, R21 + OR R16, R21, R16 + ADD $16, R22 + ADD $-2, R5 + ADD $2, R4 + +less2: + CMP R5, $0 + BEQ insert1 + MOVBZ (R4), R21 + SLD R22, R21, R21 + OR R16, R21, R16 + ADD $8, R22 + +insert1: + // Insert 1 at end of msg + MOVD $1, R21 + SLD R22, R21, R21 + OR R16, R21, R16 + +carry: + // Add new values to h0, h1, h2 + ADDC R16, R8 + ADDE R17, R9 + ADDZE R10, R10 + MOVD $16, R5 + ADD R5, R4 + BR multiply + +done: + // Save h0, h1, h2 in state + MOVD R8, 0(R3) + MOVD R9, 8(R3) + MOVD R10, 16(R3) + RET diff --git a/src/vendor/golang.org/x/crypto/sha3/doc.go b/src/vendor/golang.org/x/crypto/sha3/doc.go deleted file mode 100644 index 7e023090707b8a..00000000000000 --- a/src/vendor/golang.org/x/crypto/sha3/doc.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package sha3 implements the SHA-3 fixed-output-length hash functions and -// the SHAKE variable-output-length hash functions defined by FIPS-202. -// -// Both types of hash function use the "sponge" construction and the Keccak -// permutation. For a detailed specification see http://keccak.noekeon.org/ -// -// # Guidance -// -// If you aren't sure what function you need, use SHAKE256 with at least 64 -// bytes of output. The SHAKE instances are faster than the SHA3 instances; -// the latter have to allocate memory to conform to the hash.Hash interface. -// -// If you need a secret-key MAC (message authentication code), prepend the -// secret key to the input, hash with SHAKE256 and read at least 32 bytes of -// output. -// -// # Security strengths -// -// The SHA3-x (x equals 224, 256, 384, or 512) functions have a security -// strength against preimage attacks of x bits. Since they only produce "x" -// bits of output, their collision-resistance is only "x/2" bits. -// -// The SHAKE-256 and -128 functions have a generic security strength of 256 and -// 128 bits against all attacks, provided that at least 2x bits of their output -// is used. Requesting more than 64 or 32 bytes of output, respectively, does -// not increase the collision-resistance of the SHAKE functions. -// -// # The sponge construction -// -// A sponge builds a pseudo-random function from a public pseudo-random -// permutation, by applying the permutation to a state of "rate + capacity" -// bytes, but hiding "capacity" of the bytes. -// -// A sponge starts out with a zero state. To hash an input using a sponge, up -// to "rate" bytes of the input are XORed into the sponge's state. The sponge -// is then "full" and the permutation is applied to "empty" it. This process is -// repeated until all the input has been "absorbed". The input is then padded. -// The digest is "squeezed" from the sponge in the same way, except that output -// is copied out instead of input being XORed in. -// -// A sponge is parameterized by its generic security strength, which is equal -// to half its capacity; capacity + rate is equal to the permutation's width. -// Since the KeccakF-1600 permutation is 1600 bits (200 bytes) wide, this means -// that the security strength of a sponge instance is equal to (1600 - bitrate) / 2. -// -// # Recommendations -// -// The SHAKE functions are recommended for most new uses. They can produce -// output of arbitrary length. SHAKE256, with an output length of at least -// 64 bytes, provides 256-bit security against all attacks. The Keccak team -// recommends it for most applications upgrading from SHA2-512. (NIST chose a -// much stronger, but much slower, sponge instance for SHA3-512.) -// -// The SHA-3 functions are "drop-in" replacements for the SHA-2 functions. -// They produce output of the same length, with the same security strengths -// against all attacks. This means, in particular, that SHA3-256 only has -// 128-bit collision resistance, because its output length is 32 bytes. -package sha3 diff --git a/src/vendor/golang.org/x/crypto/sha3/hashes.go b/src/vendor/golang.org/x/crypto/sha3/hashes.go deleted file mode 100644 index c544b29e5f2cce..00000000000000 --- a/src/vendor/golang.org/x/crypto/sha3/hashes.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package sha3 - -// This file provides functions for creating instances of the SHA-3 -// and SHAKE hash functions, as well as utility functions for hashing -// bytes. - -import ( - "crypto" - "hash" -) - -// New224 creates a new SHA3-224 hash. -// Its generic security strength is 224 bits against preimage attacks, -// and 112 bits against collision attacks. -func New224() hash.Hash { - return new224() -} - -// New256 creates a new SHA3-256 hash. -// Its generic security strength is 256 bits against preimage attacks, -// and 128 bits against collision attacks. -func New256() hash.Hash { - return new256() -} - -// New384 creates a new SHA3-384 hash. -// Its generic security strength is 384 bits against preimage attacks, -// and 192 bits against collision attacks. -func New384() hash.Hash { - return new384() -} - -// New512 creates a new SHA3-512 hash. -// Its generic security strength is 512 bits against preimage attacks, -// and 256 bits against collision attacks. -func New512() hash.Hash { - return new512() -} - -func init() { - crypto.RegisterHash(crypto.SHA3_224, New224) - crypto.RegisterHash(crypto.SHA3_256, New256) - crypto.RegisterHash(crypto.SHA3_384, New384) - crypto.RegisterHash(crypto.SHA3_512, New512) -} - -func new224Generic() *state { - return &state{rate: 144, outputLen: 28, dsbyte: 0x06} -} - -func new256Generic() *state { - return &state{rate: 136, outputLen: 32, dsbyte: 0x06} -} - -func new384Generic() *state { - return &state{rate: 104, outputLen: 48, dsbyte: 0x06} -} - -func new512Generic() *state { - return &state{rate: 72, outputLen: 64, dsbyte: 0x06} -} - -// NewLegacyKeccak256 creates a new Keccak-256 hash. -// -// Only use this function if you require compatibility with an existing cryptosystem -// that uses non-standard padding. All other users should use New256 instead. -func NewLegacyKeccak256() hash.Hash { return &state{rate: 136, outputLen: 32, dsbyte: 0x01} } - -// NewLegacyKeccak512 creates a new Keccak-512 hash. -// -// Only use this function if you require compatibility with an existing cryptosystem -// that uses non-standard padding. All other users should use New512 instead. -func NewLegacyKeccak512() hash.Hash { return &state{rate: 72, outputLen: 64, dsbyte: 0x01} } - -// Sum224 returns the SHA3-224 digest of the data. -func Sum224(data []byte) (digest [28]byte) { - h := New224() - h.Write(data) - h.Sum(digest[:0]) - return -} - -// Sum256 returns the SHA3-256 digest of the data. -func Sum256(data []byte) (digest [32]byte) { - h := New256() - h.Write(data) - h.Sum(digest[:0]) - return -} - -// Sum384 returns the SHA3-384 digest of the data. -func Sum384(data []byte) (digest [48]byte) { - h := New384() - h.Write(data) - h.Sum(digest[:0]) - return -} - -// Sum512 returns the SHA3-512 digest of the data. -func Sum512(data []byte) (digest [64]byte) { - h := New512() - h.Write(data) - h.Sum(digest[:0]) - return -} diff --git a/src/vendor/golang.org/x/crypto/sha3/hashes_noasm.go b/src/vendor/golang.org/x/crypto/sha3/hashes_noasm.go deleted file mode 100644 index 9d85fb621446d2..00000000000000 --- a/src/vendor/golang.org/x/crypto/sha3/hashes_noasm.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !gc || purego || !s390x - -package sha3 - -func new224() *state { - return new224Generic() -} - -func new256() *state { - return new256Generic() -} - -func new384() *state { - return new384Generic() -} - -func new512() *state { - return new512Generic() -} diff --git a/src/vendor/golang.org/x/crypto/sha3/keccakf_amd64.go b/src/vendor/golang.org/x/crypto/sha3/keccakf_amd64.go deleted file mode 100644 index b908696be58fb3..00000000000000 --- a/src/vendor/golang.org/x/crypto/sha3/keccakf_amd64.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build amd64 && !purego && gc - -package sha3 - -// This function is implemented in keccakf_amd64.s. - -//go:noescape - -func keccakF1600(a *[25]uint64) diff --git a/src/vendor/golang.org/x/crypto/sha3/keccakf_amd64.s b/src/vendor/golang.org/x/crypto/sha3/keccakf_amd64.s deleted file mode 100644 index 99e2f16e971937..00000000000000 --- a/src/vendor/golang.org/x/crypto/sha3/keccakf_amd64.s +++ /dev/null @@ -1,5419 +0,0 @@ -// Code generated by command: go run keccakf_amd64_asm.go -out ../keccakf_amd64.s -pkg sha3. DO NOT EDIT. - -//go:build amd64 && !purego && gc - -// func keccakF1600(a *[25]uint64) -TEXT ·keccakF1600(SB), $200-8 - MOVQ a+0(FP), DI - - // Convert the user state into an internal state - NOTQ 8(DI) - NOTQ 16(DI) - NOTQ 64(DI) - NOTQ 96(DI) - NOTQ 136(DI) - NOTQ 160(DI) - - // Execute the KeccakF permutation - MOVQ (DI), SI - MOVQ 8(DI), BP - MOVQ 32(DI), R15 - XORQ 40(DI), SI - XORQ 48(DI), BP - XORQ 72(DI), R15 - XORQ 80(DI), SI - XORQ 88(DI), BP - XORQ 112(DI), R15 - XORQ 120(DI), SI - XORQ 128(DI), BP - XORQ 152(DI), R15 - XORQ 160(DI), SI - XORQ 168(DI), BP - MOVQ 176(DI), DX - MOVQ 184(DI), R8 - XORQ 192(DI), R15 - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(DI), R12 - XORQ 56(DI), DX - XORQ R15, BX - XORQ 96(DI), R12 - XORQ 136(DI), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(DI), R13 - XORQ 64(DI), R8 - XORQ SI, CX - XORQ 104(DI), R13 - XORQ 144(DI), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (DI), R10 - MOVQ 48(DI), R11 - XORQ R13, R9 - MOVQ 96(DI), R12 - MOVQ 144(DI), R13 - MOVQ 192(DI), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x0000000000000001, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (SP) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(SP) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(SP) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(SP) - MOVQ R12, 8(SP) - MOVQ R12, BP - - // Result g - MOVQ 72(DI), R11 - XORQ R9, R11 - MOVQ 80(DI), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(DI), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(DI), R13 - MOVQ 176(DI), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(SP) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(SP) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(SP) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(SP) - - // Result k - MOVQ 8(DI), R10 - MOVQ 56(DI), R11 - MOVQ 104(DI), R12 - MOVQ 152(DI), R13 - MOVQ 160(DI), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(SP) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(SP) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(SP) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(SP) - XORQ R10, R15 - - // Result m - MOVQ 40(DI), R11 - XORQ BX, R11 - MOVQ 88(DI), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(DI), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(DI), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(DI), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(SP) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(SP) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(SP) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(SP) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(SP) - XORQ R11, R15 - - // Result s - MOVQ 16(DI), R10 - MOVQ 64(DI), R11 - MOVQ 112(DI), R12 - XORQ DX, R10 - MOVQ 120(DI), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(DI), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(SP) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(SP) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(SP) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(SP) - MOVQ R8, 184(SP) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(SP), R12 - XORQ 56(SP), DX - XORQ R15, BX - XORQ 96(SP), R12 - XORQ 136(SP), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(SP), R13 - XORQ 64(SP), R8 - XORQ SI, CX - XORQ 104(SP), R13 - XORQ 144(SP), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (SP), R10 - MOVQ 48(SP), R11 - XORQ R13, R9 - MOVQ 96(SP), R12 - MOVQ 144(SP), R13 - MOVQ 192(SP), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x0000000000008082, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (DI) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(DI) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(DI) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(DI) - MOVQ R12, 8(DI) - MOVQ R12, BP - - // Result g - MOVQ 72(SP), R11 - XORQ R9, R11 - MOVQ 80(SP), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(SP), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(SP), R13 - MOVQ 176(SP), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(DI) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(DI) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(DI) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(DI) - - // Result k - MOVQ 8(SP), R10 - MOVQ 56(SP), R11 - MOVQ 104(SP), R12 - MOVQ 152(SP), R13 - MOVQ 160(SP), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(DI) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(DI) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(DI) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(DI) - XORQ R10, R15 - - // Result m - MOVQ 40(SP), R11 - XORQ BX, R11 - MOVQ 88(SP), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(SP), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(SP), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(SP), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(DI) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(DI) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(DI) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(DI) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(DI) - XORQ R11, R15 - - // Result s - MOVQ 16(SP), R10 - MOVQ 64(SP), R11 - MOVQ 112(SP), R12 - XORQ DX, R10 - MOVQ 120(SP), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(SP), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(DI) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(DI) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(DI) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(DI) - MOVQ R8, 184(DI) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(DI), R12 - XORQ 56(DI), DX - XORQ R15, BX - XORQ 96(DI), R12 - XORQ 136(DI), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(DI), R13 - XORQ 64(DI), R8 - XORQ SI, CX - XORQ 104(DI), R13 - XORQ 144(DI), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (DI), R10 - MOVQ 48(DI), R11 - XORQ R13, R9 - MOVQ 96(DI), R12 - MOVQ 144(DI), R13 - MOVQ 192(DI), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x800000000000808a, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (SP) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(SP) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(SP) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(SP) - MOVQ R12, 8(SP) - MOVQ R12, BP - - // Result g - MOVQ 72(DI), R11 - XORQ R9, R11 - MOVQ 80(DI), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(DI), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(DI), R13 - MOVQ 176(DI), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(SP) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(SP) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(SP) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(SP) - - // Result k - MOVQ 8(DI), R10 - MOVQ 56(DI), R11 - MOVQ 104(DI), R12 - MOVQ 152(DI), R13 - MOVQ 160(DI), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(SP) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(SP) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(SP) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(SP) - XORQ R10, R15 - - // Result m - MOVQ 40(DI), R11 - XORQ BX, R11 - MOVQ 88(DI), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(DI), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(DI), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(DI), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(SP) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(SP) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(SP) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(SP) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(SP) - XORQ R11, R15 - - // Result s - MOVQ 16(DI), R10 - MOVQ 64(DI), R11 - MOVQ 112(DI), R12 - XORQ DX, R10 - MOVQ 120(DI), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(DI), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(SP) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(SP) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(SP) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(SP) - MOVQ R8, 184(SP) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(SP), R12 - XORQ 56(SP), DX - XORQ R15, BX - XORQ 96(SP), R12 - XORQ 136(SP), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(SP), R13 - XORQ 64(SP), R8 - XORQ SI, CX - XORQ 104(SP), R13 - XORQ 144(SP), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (SP), R10 - MOVQ 48(SP), R11 - XORQ R13, R9 - MOVQ 96(SP), R12 - MOVQ 144(SP), R13 - MOVQ 192(SP), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x8000000080008000, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (DI) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(DI) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(DI) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(DI) - MOVQ R12, 8(DI) - MOVQ R12, BP - - // Result g - MOVQ 72(SP), R11 - XORQ R9, R11 - MOVQ 80(SP), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(SP), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(SP), R13 - MOVQ 176(SP), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(DI) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(DI) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(DI) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(DI) - - // Result k - MOVQ 8(SP), R10 - MOVQ 56(SP), R11 - MOVQ 104(SP), R12 - MOVQ 152(SP), R13 - MOVQ 160(SP), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(DI) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(DI) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(DI) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(DI) - XORQ R10, R15 - - // Result m - MOVQ 40(SP), R11 - XORQ BX, R11 - MOVQ 88(SP), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(SP), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(SP), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(SP), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(DI) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(DI) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(DI) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(DI) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(DI) - XORQ R11, R15 - - // Result s - MOVQ 16(SP), R10 - MOVQ 64(SP), R11 - MOVQ 112(SP), R12 - XORQ DX, R10 - MOVQ 120(SP), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(SP), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(DI) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(DI) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(DI) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(DI) - MOVQ R8, 184(DI) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(DI), R12 - XORQ 56(DI), DX - XORQ R15, BX - XORQ 96(DI), R12 - XORQ 136(DI), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(DI), R13 - XORQ 64(DI), R8 - XORQ SI, CX - XORQ 104(DI), R13 - XORQ 144(DI), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (DI), R10 - MOVQ 48(DI), R11 - XORQ R13, R9 - MOVQ 96(DI), R12 - MOVQ 144(DI), R13 - MOVQ 192(DI), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x000000000000808b, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (SP) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(SP) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(SP) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(SP) - MOVQ R12, 8(SP) - MOVQ R12, BP - - // Result g - MOVQ 72(DI), R11 - XORQ R9, R11 - MOVQ 80(DI), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(DI), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(DI), R13 - MOVQ 176(DI), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(SP) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(SP) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(SP) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(SP) - - // Result k - MOVQ 8(DI), R10 - MOVQ 56(DI), R11 - MOVQ 104(DI), R12 - MOVQ 152(DI), R13 - MOVQ 160(DI), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(SP) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(SP) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(SP) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(SP) - XORQ R10, R15 - - // Result m - MOVQ 40(DI), R11 - XORQ BX, R11 - MOVQ 88(DI), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(DI), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(DI), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(DI), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(SP) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(SP) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(SP) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(SP) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(SP) - XORQ R11, R15 - - // Result s - MOVQ 16(DI), R10 - MOVQ 64(DI), R11 - MOVQ 112(DI), R12 - XORQ DX, R10 - MOVQ 120(DI), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(DI), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(SP) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(SP) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(SP) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(SP) - MOVQ R8, 184(SP) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(SP), R12 - XORQ 56(SP), DX - XORQ R15, BX - XORQ 96(SP), R12 - XORQ 136(SP), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(SP), R13 - XORQ 64(SP), R8 - XORQ SI, CX - XORQ 104(SP), R13 - XORQ 144(SP), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (SP), R10 - MOVQ 48(SP), R11 - XORQ R13, R9 - MOVQ 96(SP), R12 - MOVQ 144(SP), R13 - MOVQ 192(SP), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x0000000080000001, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (DI) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(DI) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(DI) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(DI) - MOVQ R12, 8(DI) - MOVQ R12, BP - - // Result g - MOVQ 72(SP), R11 - XORQ R9, R11 - MOVQ 80(SP), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(SP), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(SP), R13 - MOVQ 176(SP), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(DI) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(DI) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(DI) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(DI) - - // Result k - MOVQ 8(SP), R10 - MOVQ 56(SP), R11 - MOVQ 104(SP), R12 - MOVQ 152(SP), R13 - MOVQ 160(SP), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(DI) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(DI) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(DI) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(DI) - XORQ R10, R15 - - // Result m - MOVQ 40(SP), R11 - XORQ BX, R11 - MOVQ 88(SP), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(SP), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(SP), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(SP), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(DI) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(DI) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(DI) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(DI) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(DI) - XORQ R11, R15 - - // Result s - MOVQ 16(SP), R10 - MOVQ 64(SP), R11 - MOVQ 112(SP), R12 - XORQ DX, R10 - MOVQ 120(SP), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(SP), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(DI) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(DI) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(DI) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(DI) - MOVQ R8, 184(DI) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(DI), R12 - XORQ 56(DI), DX - XORQ R15, BX - XORQ 96(DI), R12 - XORQ 136(DI), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(DI), R13 - XORQ 64(DI), R8 - XORQ SI, CX - XORQ 104(DI), R13 - XORQ 144(DI), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (DI), R10 - MOVQ 48(DI), R11 - XORQ R13, R9 - MOVQ 96(DI), R12 - MOVQ 144(DI), R13 - MOVQ 192(DI), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x8000000080008081, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (SP) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(SP) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(SP) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(SP) - MOVQ R12, 8(SP) - MOVQ R12, BP - - // Result g - MOVQ 72(DI), R11 - XORQ R9, R11 - MOVQ 80(DI), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(DI), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(DI), R13 - MOVQ 176(DI), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(SP) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(SP) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(SP) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(SP) - - // Result k - MOVQ 8(DI), R10 - MOVQ 56(DI), R11 - MOVQ 104(DI), R12 - MOVQ 152(DI), R13 - MOVQ 160(DI), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(SP) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(SP) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(SP) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(SP) - XORQ R10, R15 - - // Result m - MOVQ 40(DI), R11 - XORQ BX, R11 - MOVQ 88(DI), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(DI), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(DI), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(DI), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(SP) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(SP) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(SP) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(SP) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(SP) - XORQ R11, R15 - - // Result s - MOVQ 16(DI), R10 - MOVQ 64(DI), R11 - MOVQ 112(DI), R12 - XORQ DX, R10 - MOVQ 120(DI), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(DI), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(SP) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(SP) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(SP) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(SP) - MOVQ R8, 184(SP) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(SP), R12 - XORQ 56(SP), DX - XORQ R15, BX - XORQ 96(SP), R12 - XORQ 136(SP), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(SP), R13 - XORQ 64(SP), R8 - XORQ SI, CX - XORQ 104(SP), R13 - XORQ 144(SP), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (SP), R10 - MOVQ 48(SP), R11 - XORQ R13, R9 - MOVQ 96(SP), R12 - MOVQ 144(SP), R13 - MOVQ 192(SP), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x8000000000008009, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (DI) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(DI) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(DI) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(DI) - MOVQ R12, 8(DI) - MOVQ R12, BP - - // Result g - MOVQ 72(SP), R11 - XORQ R9, R11 - MOVQ 80(SP), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(SP), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(SP), R13 - MOVQ 176(SP), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(DI) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(DI) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(DI) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(DI) - - // Result k - MOVQ 8(SP), R10 - MOVQ 56(SP), R11 - MOVQ 104(SP), R12 - MOVQ 152(SP), R13 - MOVQ 160(SP), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(DI) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(DI) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(DI) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(DI) - XORQ R10, R15 - - // Result m - MOVQ 40(SP), R11 - XORQ BX, R11 - MOVQ 88(SP), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(SP), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(SP), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(SP), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(DI) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(DI) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(DI) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(DI) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(DI) - XORQ R11, R15 - - // Result s - MOVQ 16(SP), R10 - MOVQ 64(SP), R11 - MOVQ 112(SP), R12 - XORQ DX, R10 - MOVQ 120(SP), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(SP), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(DI) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(DI) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(DI) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(DI) - MOVQ R8, 184(DI) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(DI), R12 - XORQ 56(DI), DX - XORQ R15, BX - XORQ 96(DI), R12 - XORQ 136(DI), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(DI), R13 - XORQ 64(DI), R8 - XORQ SI, CX - XORQ 104(DI), R13 - XORQ 144(DI), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (DI), R10 - MOVQ 48(DI), R11 - XORQ R13, R9 - MOVQ 96(DI), R12 - MOVQ 144(DI), R13 - MOVQ 192(DI), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x000000000000008a, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (SP) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(SP) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(SP) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(SP) - MOVQ R12, 8(SP) - MOVQ R12, BP - - // Result g - MOVQ 72(DI), R11 - XORQ R9, R11 - MOVQ 80(DI), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(DI), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(DI), R13 - MOVQ 176(DI), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(SP) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(SP) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(SP) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(SP) - - // Result k - MOVQ 8(DI), R10 - MOVQ 56(DI), R11 - MOVQ 104(DI), R12 - MOVQ 152(DI), R13 - MOVQ 160(DI), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(SP) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(SP) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(SP) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(SP) - XORQ R10, R15 - - // Result m - MOVQ 40(DI), R11 - XORQ BX, R11 - MOVQ 88(DI), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(DI), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(DI), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(DI), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(SP) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(SP) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(SP) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(SP) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(SP) - XORQ R11, R15 - - // Result s - MOVQ 16(DI), R10 - MOVQ 64(DI), R11 - MOVQ 112(DI), R12 - XORQ DX, R10 - MOVQ 120(DI), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(DI), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(SP) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(SP) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(SP) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(SP) - MOVQ R8, 184(SP) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(SP), R12 - XORQ 56(SP), DX - XORQ R15, BX - XORQ 96(SP), R12 - XORQ 136(SP), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(SP), R13 - XORQ 64(SP), R8 - XORQ SI, CX - XORQ 104(SP), R13 - XORQ 144(SP), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (SP), R10 - MOVQ 48(SP), R11 - XORQ R13, R9 - MOVQ 96(SP), R12 - MOVQ 144(SP), R13 - MOVQ 192(SP), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x0000000000000088, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (DI) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(DI) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(DI) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(DI) - MOVQ R12, 8(DI) - MOVQ R12, BP - - // Result g - MOVQ 72(SP), R11 - XORQ R9, R11 - MOVQ 80(SP), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(SP), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(SP), R13 - MOVQ 176(SP), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(DI) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(DI) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(DI) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(DI) - - // Result k - MOVQ 8(SP), R10 - MOVQ 56(SP), R11 - MOVQ 104(SP), R12 - MOVQ 152(SP), R13 - MOVQ 160(SP), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(DI) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(DI) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(DI) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(DI) - XORQ R10, R15 - - // Result m - MOVQ 40(SP), R11 - XORQ BX, R11 - MOVQ 88(SP), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(SP), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(SP), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(SP), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(DI) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(DI) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(DI) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(DI) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(DI) - XORQ R11, R15 - - // Result s - MOVQ 16(SP), R10 - MOVQ 64(SP), R11 - MOVQ 112(SP), R12 - XORQ DX, R10 - MOVQ 120(SP), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(SP), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(DI) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(DI) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(DI) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(DI) - MOVQ R8, 184(DI) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(DI), R12 - XORQ 56(DI), DX - XORQ R15, BX - XORQ 96(DI), R12 - XORQ 136(DI), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(DI), R13 - XORQ 64(DI), R8 - XORQ SI, CX - XORQ 104(DI), R13 - XORQ 144(DI), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (DI), R10 - MOVQ 48(DI), R11 - XORQ R13, R9 - MOVQ 96(DI), R12 - MOVQ 144(DI), R13 - MOVQ 192(DI), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x0000000080008009, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (SP) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(SP) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(SP) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(SP) - MOVQ R12, 8(SP) - MOVQ R12, BP - - // Result g - MOVQ 72(DI), R11 - XORQ R9, R11 - MOVQ 80(DI), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(DI), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(DI), R13 - MOVQ 176(DI), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(SP) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(SP) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(SP) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(SP) - - // Result k - MOVQ 8(DI), R10 - MOVQ 56(DI), R11 - MOVQ 104(DI), R12 - MOVQ 152(DI), R13 - MOVQ 160(DI), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(SP) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(SP) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(SP) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(SP) - XORQ R10, R15 - - // Result m - MOVQ 40(DI), R11 - XORQ BX, R11 - MOVQ 88(DI), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(DI), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(DI), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(DI), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(SP) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(SP) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(SP) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(SP) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(SP) - XORQ R11, R15 - - // Result s - MOVQ 16(DI), R10 - MOVQ 64(DI), R11 - MOVQ 112(DI), R12 - XORQ DX, R10 - MOVQ 120(DI), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(DI), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(SP) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(SP) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(SP) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(SP) - MOVQ R8, 184(SP) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(SP), R12 - XORQ 56(SP), DX - XORQ R15, BX - XORQ 96(SP), R12 - XORQ 136(SP), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(SP), R13 - XORQ 64(SP), R8 - XORQ SI, CX - XORQ 104(SP), R13 - XORQ 144(SP), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (SP), R10 - MOVQ 48(SP), R11 - XORQ R13, R9 - MOVQ 96(SP), R12 - MOVQ 144(SP), R13 - MOVQ 192(SP), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x000000008000000a, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (DI) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(DI) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(DI) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(DI) - MOVQ R12, 8(DI) - MOVQ R12, BP - - // Result g - MOVQ 72(SP), R11 - XORQ R9, R11 - MOVQ 80(SP), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(SP), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(SP), R13 - MOVQ 176(SP), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(DI) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(DI) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(DI) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(DI) - - // Result k - MOVQ 8(SP), R10 - MOVQ 56(SP), R11 - MOVQ 104(SP), R12 - MOVQ 152(SP), R13 - MOVQ 160(SP), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(DI) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(DI) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(DI) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(DI) - XORQ R10, R15 - - // Result m - MOVQ 40(SP), R11 - XORQ BX, R11 - MOVQ 88(SP), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(SP), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(SP), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(SP), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(DI) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(DI) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(DI) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(DI) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(DI) - XORQ R11, R15 - - // Result s - MOVQ 16(SP), R10 - MOVQ 64(SP), R11 - MOVQ 112(SP), R12 - XORQ DX, R10 - MOVQ 120(SP), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(SP), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(DI) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(DI) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(DI) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(DI) - MOVQ R8, 184(DI) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(DI), R12 - XORQ 56(DI), DX - XORQ R15, BX - XORQ 96(DI), R12 - XORQ 136(DI), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(DI), R13 - XORQ 64(DI), R8 - XORQ SI, CX - XORQ 104(DI), R13 - XORQ 144(DI), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (DI), R10 - MOVQ 48(DI), R11 - XORQ R13, R9 - MOVQ 96(DI), R12 - MOVQ 144(DI), R13 - MOVQ 192(DI), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x000000008000808b, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (SP) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(SP) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(SP) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(SP) - MOVQ R12, 8(SP) - MOVQ R12, BP - - // Result g - MOVQ 72(DI), R11 - XORQ R9, R11 - MOVQ 80(DI), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(DI), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(DI), R13 - MOVQ 176(DI), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(SP) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(SP) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(SP) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(SP) - - // Result k - MOVQ 8(DI), R10 - MOVQ 56(DI), R11 - MOVQ 104(DI), R12 - MOVQ 152(DI), R13 - MOVQ 160(DI), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(SP) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(SP) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(SP) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(SP) - XORQ R10, R15 - - // Result m - MOVQ 40(DI), R11 - XORQ BX, R11 - MOVQ 88(DI), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(DI), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(DI), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(DI), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(SP) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(SP) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(SP) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(SP) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(SP) - XORQ R11, R15 - - // Result s - MOVQ 16(DI), R10 - MOVQ 64(DI), R11 - MOVQ 112(DI), R12 - XORQ DX, R10 - MOVQ 120(DI), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(DI), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(SP) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(SP) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(SP) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(SP) - MOVQ R8, 184(SP) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(SP), R12 - XORQ 56(SP), DX - XORQ R15, BX - XORQ 96(SP), R12 - XORQ 136(SP), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(SP), R13 - XORQ 64(SP), R8 - XORQ SI, CX - XORQ 104(SP), R13 - XORQ 144(SP), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (SP), R10 - MOVQ 48(SP), R11 - XORQ R13, R9 - MOVQ 96(SP), R12 - MOVQ 144(SP), R13 - MOVQ 192(SP), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x800000000000008b, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (DI) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(DI) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(DI) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(DI) - MOVQ R12, 8(DI) - MOVQ R12, BP - - // Result g - MOVQ 72(SP), R11 - XORQ R9, R11 - MOVQ 80(SP), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(SP), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(SP), R13 - MOVQ 176(SP), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(DI) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(DI) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(DI) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(DI) - - // Result k - MOVQ 8(SP), R10 - MOVQ 56(SP), R11 - MOVQ 104(SP), R12 - MOVQ 152(SP), R13 - MOVQ 160(SP), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(DI) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(DI) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(DI) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(DI) - XORQ R10, R15 - - // Result m - MOVQ 40(SP), R11 - XORQ BX, R11 - MOVQ 88(SP), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(SP), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(SP), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(SP), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(DI) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(DI) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(DI) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(DI) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(DI) - XORQ R11, R15 - - // Result s - MOVQ 16(SP), R10 - MOVQ 64(SP), R11 - MOVQ 112(SP), R12 - XORQ DX, R10 - MOVQ 120(SP), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(SP), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(DI) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(DI) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(DI) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(DI) - MOVQ R8, 184(DI) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(DI), R12 - XORQ 56(DI), DX - XORQ R15, BX - XORQ 96(DI), R12 - XORQ 136(DI), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(DI), R13 - XORQ 64(DI), R8 - XORQ SI, CX - XORQ 104(DI), R13 - XORQ 144(DI), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (DI), R10 - MOVQ 48(DI), R11 - XORQ R13, R9 - MOVQ 96(DI), R12 - MOVQ 144(DI), R13 - MOVQ 192(DI), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x8000000000008089, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (SP) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(SP) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(SP) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(SP) - MOVQ R12, 8(SP) - MOVQ R12, BP - - // Result g - MOVQ 72(DI), R11 - XORQ R9, R11 - MOVQ 80(DI), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(DI), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(DI), R13 - MOVQ 176(DI), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(SP) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(SP) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(SP) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(SP) - - // Result k - MOVQ 8(DI), R10 - MOVQ 56(DI), R11 - MOVQ 104(DI), R12 - MOVQ 152(DI), R13 - MOVQ 160(DI), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(SP) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(SP) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(SP) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(SP) - XORQ R10, R15 - - // Result m - MOVQ 40(DI), R11 - XORQ BX, R11 - MOVQ 88(DI), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(DI), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(DI), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(DI), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(SP) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(SP) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(SP) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(SP) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(SP) - XORQ R11, R15 - - // Result s - MOVQ 16(DI), R10 - MOVQ 64(DI), R11 - MOVQ 112(DI), R12 - XORQ DX, R10 - MOVQ 120(DI), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(DI), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(SP) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(SP) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(SP) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(SP) - MOVQ R8, 184(SP) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(SP), R12 - XORQ 56(SP), DX - XORQ R15, BX - XORQ 96(SP), R12 - XORQ 136(SP), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(SP), R13 - XORQ 64(SP), R8 - XORQ SI, CX - XORQ 104(SP), R13 - XORQ 144(SP), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (SP), R10 - MOVQ 48(SP), R11 - XORQ R13, R9 - MOVQ 96(SP), R12 - MOVQ 144(SP), R13 - MOVQ 192(SP), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x8000000000008003, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (DI) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(DI) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(DI) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(DI) - MOVQ R12, 8(DI) - MOVQ R12, BP - - // Result g - MOVQ 72(SP), R11 - XORQ R9, R11 - MOVQ 80(SP), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(SP), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(SP), R13 - MOVQ 176(SP), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(DI) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(DI) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(DI) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(DI) - - // Result k - MOVQ 8(SP), R10 - MOVQ 56(SP), R11 - MOVQ 104(SP), R12 - MOVQ 152(SP), R13 - MOVQ 160(SP), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(DI) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(DI) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(DI) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(DI) - XORQ R10, R15 - - // Result m - MOVQ 40(SP), R11 - XORQ BX, R11 - MOVQ 88(SP), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(SP), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(SP), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(SP), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(DI) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(DI) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(DI) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(DI) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(DI) - XORQ R11, R15 - - // Result s - MOVQ 16(SP), R10 - MOVQ 64(SP), R11 - MOVQ 112(SP), R12 - XORQ DX, R10 - MOVQ 120(SP), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(SP), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(DI) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(DI) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(DI) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(DI) - MOVQ R8, 184(DI) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(DI), R12 - XORQ 56(DI), DX - XORQ R15, BX - XORQ 96(DI), R12 - XORQ 136(DI), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(DI), R13 - XORQ 64(DI), R8 - XORQ SI, CX - XORQ 104(DI), R13 - XORQ 144(DI), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (DI), R10 - MOVQ 48(DI), R11 - XORQ R13, R9 - MOVQ 96(DI), R12 - MOVQ 144(DI), R13 - MOVQ 192(DI), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x8000000000008002, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (SP) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(SP) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(SP) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(SP) - MOVQ R12, 8(SP) - MOVQ R12, BP - - // Result g - MOVQ 72(DI), R11 - XORQ R9, R11 - MOVQ 80(DI), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(DI), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(DI), R13 - MOVQ 176(DI), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(SP) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(SP) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(SP) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(SP) - - // Result k - MOVQ 8(DI), R10 - MOVQ 56(DI), R11 - MOVQ 104(DI), R12 - MOVQ 152(DI), R13 - MOVQ 160(DI), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(SP) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(SP) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(SP) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(SP) - XORQ R10, R15 - - // Result m - MOVQ 40(DI), R11 - XORQ BX, R11 - MOVQ 88(DI), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(DI), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(DI), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(DI), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(SP) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(SP) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(SP) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(SP) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(SP) - XORQ R11, R15 - - // Result s - MOVQ 16(DI), R10 - MOVQ 64(DI), R11 - MOVQ 112(DI), R12 - XORQ DX, R10 - MOVQ 120(DI), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(DI), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(SP) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(SP) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(SP) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(SP) - MOVQ R8, 184(SP) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(SP), R12 - XORQ 56(SP), DX - XORQ R15, BX - XORQ 96(SP), R12 - XORQ 136(SP), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(SP), R13 - XORQ 64(SP), R8 - XORQ SI, CX - XORQ 104(SP), R13 - XORQ 144(SP), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (SP), R10 - MOVQ 48(SP), R11 - XORQ R13, R9 - MOVQ 96(SP), R12 - MOVQ 144(SP), R13 - MOVQ 192(SP), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x8000000000000080, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (DI) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(DI) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(DI) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(DI) - MOVQ R12, 8(DI) - MOVQ R12, BP - - // Result g - MOVQ 72(SP), R11 - XORQ R9, R11 - MOVQ 80(SP), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(SP), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(SP), R13 - MOVQ 176(SP), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(DI) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(DI) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(DI) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(DI) - - // Result k - MOVQ 8(SP), R10 - MOVQ 56(SP), R11 - MOVQ 104(SP), R12 - MOVQ 152(SP), R13 - MOVQ 160(SP), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(DI) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(DI) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(DI) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(DI) - XORQ R10, R15 - - // Result m - MOVQ 40(SP), R11 - XORQ BX, R11 - MOVQ 88(SP), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(SP), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(SP), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(SP), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(DI) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(DI) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(DI) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(DI) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(DI) - XORQ R11, R15 - - // Result s - MOVQ 16(SP), R10 - MOVQ 64(SP), R11 - MOVQ 112(SP), R12 - XORQ DX, R10 - MOVQ 120(SP), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(SP), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(DI) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(DI) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(DI) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(DI) - MOVQ R8, 184(DI) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(DI), R12 - XORQ 56(DI), DX - XORQ R15, BX - XORQ 96(DI), R12 - XORQ 136(DI), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(DI), R13 - XORQ 64(DI), R8 - XORQ SI, CX - XORQ 104(DI), R13 - XORQ 144(DI), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (DI), R10 - MOVQ 48(DI), R11 - XORQ R13, R9 - MOVQ 96(DI), R12 - MOVQ 144(DI), R13 - MOVQ 192(DI), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x000000000000800a, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (SP) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(SP) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(SP) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(SP) - MOVQ R12, 8(SP) - MOVQ R12, BP - - // Result g - MOVQ 72(DI), R11 - XORQ R9, R11 - MOVQ 80(DI), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(DI), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(DI), R13 - MOVQ 176(DI), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(SP) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(SP) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(SP) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(SP) - - // Result k - MOVQ 8(DI), R10 - MOVQ 56(DI), R11 - MOVQ 104(DI), R12 - MOVQ 152(DI), R13 - MOVQ 160(DI), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(SP) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(SP) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(SP) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(SP) - XORQ R10, R15 - - // Result m - MOVQ 40(DI), R11 - XORQ BX, R11 - MOVQ 88(DI), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(DI), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(DI), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(DI), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(SP) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(SP) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(SP) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(SP) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(SP) - XORQ R11, R15 - - // Result s - MOVQ 16(DI), R10 - MOVQ 64(DI), R11 - MOVQ 112(DI), R12 - XORQ DX, R10 - MOVQ 120(DI), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(DI), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(SP) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(SP) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(SP) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(SP) - MOVQ R8, 184(SP) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(SP), R12 - XORQ 56(SP), DX - XORQ R15, BX - XORQ 96(SP), R12 - XORQ 136(SP), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(SP), R13 - XORQ 64(SP), R8 - XORQ SI, CX - XORQ 104(SP), R13 - XORQ 144(SP), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (SP), R10 - MOVQ 48(SP), R11 - XORQ R13, R9 - MOVQ 96(SP), R12 - MOVQ 144(SP), R13 - MOVQ 192(SP), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x800000008000000a, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (DI) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(DI) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(DI) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(DI) - MOVQ R12, 8(DI) - MOVQ R12, BP - - // Result g - MOVQ 72(SP), R11 - XORQ R9, R11 - MOVQ 80(SP), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(SP), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(SP), R13 - MOVQ 176(SP), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(DI) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(DI) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(DI) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(DI) - - // Result k - MOVQ 8(SP), R10 - MOVQ 56(SP), R11 - MOVQ 104(SP), R12 - MOVQ 152(SP), R13 - MOVQ 160(SP), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(DI) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(DI) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(DI) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(DI) - XORQ R10, R15 - - // Result m - MOVQ 40(SP), R11 - XORQ BX, R11 - MOVQ 88(SP), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(SP), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(SP), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(SP), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(DI) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(DI) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(DI) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(DI) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(DI) - XORQ R11, R15 - - // Result s - MOVQ 16(SP), R10 - MOVQ 64(SP), R11 - MOVQ 112(SP), R12 - XORQ DX, R10 - MOVQ 120(SP), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(SP), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(DI) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(DI) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(DI) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(DI) - MOVQ R8, 184(DI) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(DI), R12 - XORQ 56(DI), DX - XORQ R15, BX - XORQ 96(DI), R12 - XORQ 136(DI), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(DI), R13 - XORQ 64(DI), R8 - XORQ SI, CX - XORQ 104(DI), R13 - XORQ 144(DI), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (DI), R10 - MOVQ 48(DI), R11 - XORQ R13, R9 - MOVQ 96(DI), R12 - MOVQ 144(DI), R13 - MOVQ 192(DI), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x8000000080008081, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (SP) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(SP) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(SP) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(SP) - MOVQ R12, 8(SP) - MOVQ R12, BP - - // Result g - MOVQ 72(DI), R11 - XORQ R9, R11 - MOVQ 80(DI), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(DI), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(DI), R13 - MOVQ 176(DI), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(SP) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(SP) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(SP) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(SP) - - // Result k - MOVQ 8(DI), R10 - MOVQ 56(DI), R11 - MOVQ 104(DI), R12 - MOVQ 152(DI), R13 - MOVQ 160(DI), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(SP) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(SP) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(SP) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(SP) - XORQ R10, R15 - - // Result m - MOVQ 40(DI), R11 - XORQ BX, R11 - MOVQ 88(DI), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(DI), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(DI), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(DI), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(SP) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(SP) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(SP) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(SP) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(SP) - XORQ R11, R15 - - // Result s - MOVQ 16(DI), R10 - MOVQ 64(DI), R11 - MOVQ 112(DI), R12 - XORQ DX, R10 - MOVQ 120(DI), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(DI), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(SP) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(SP) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(SP) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(SP) - MOVQ R8, 184(SP) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(SP), R12 - XORQ 56(SP), DX - XORQ R15, BX - XORQ 96(SP), R12 - XORQ 136(SP), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(SP), R13 - XORQ 64(SP), R8 - XORQ SI, CX - XORQ 104(SP), R13 - XORQ 144(SP), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (SP), R10 - MOVQ 48(SP), R11 - XORQ R13, R9 - MOVQ 96(SP), R12 - MOVQ 144(SP), R13 - MOVQ 192(SP), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x8000000000008080, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (DI) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(DI) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(DI) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(DI) - MOVQ R12, 8(DI) - MOVQ R12, BP - - // Result g - MOVQ 72(SP), R11 - XORQ R9, R11 - MOVQ 80(SP), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(SP), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(SP), R13 - MOVQ 176(SP), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(DI) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(DI) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(DI) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(DI) - - // Result k - MOVQ 8(SP), R10 - MOVQ 56(SP), R11 - MOVQ 104(SP), R12 - MOVQ 152(SP), R13 - MOVQ 160(SP), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(DI) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(DI) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(DI) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(DI) - XORQ R10, R15 - - // Result m - MOVQ 40(SP), R11 - XORQ BX, R11 - MOVQ 88(SP), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(SP), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(SP), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(SP), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(DI) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(DI) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(DI) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(DI) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(DI) - XORQ R11, R15 - - // Result s - MOVQ 16(SP), R10 - MOVQ 64(SP), R11 - MOVQ 112(SP), R12 - XORQ DX, R10 - MOVQ 120(SP), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(SP), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(DI) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(DI) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(DI) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(DI) - MOVQ R8, 184(DI) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(DI), R12 - XORQ 56(DI), DX - XORQ R15, BX - XORQ 96(DI), R12 - XORQ 136(DI), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(DI), R13 - XORQ 64(DI), R8 - XORQ SI, CX - XORQ 104(DI), R13 - XORQ 144(DI), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (DI), R10 - MOVQ 48(DI), R11 - XORQ R13, R9 - MOVQ 96(DI), R12 - MOVQ 144(DI), R13 - MOVQ 192(DI), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x0000000080000001, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (SP) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(SP) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(SP) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(SP) - MOVQ R12, 8(SP) - MOVQ R12, BP - - // Result g - MOVQ 72(DI), R11 - XORQ R9, R11 - MOVQ 80(DI), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(DI), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(DI), R13 - MOVQ 176(DI), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(SP) - XORQ AX, SI - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(SP) - XORQ AX, BP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(SP) - NOTQ R14 - XORQ R10, R15 - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(SP) - - // Result k - MOVQ 8(DI), R10 - MOVQ 56(DI), R11 - MOVQ 104(DI), R12 - MOVQ 152(DI), R13 - MOVQ 160(DI), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(SP) - XORQ AX, SI - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(SP) - XORQ AX, BP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(SP) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(SP) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(SP) - XORQ R10, R15 - - // Result m - MOVQ 40(DI), R11 - XORQ BX, R11 - MOVQ 88(DI), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(DI), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(DI), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(DI), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(SP) - XORQ AX, SI - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(SP) - XORQ AX, BP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(SP) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(SP) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(SP) - XORQ R11, R15 - - // Result s - MOVQ 16(DI), R10 - MOVQ 64(DI), R11 - MOVQ 112(DI), R12 - XORQ DX, R10 - MOVQ 120(DI), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(DI), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(SP) - ROLQ $0x27, R12 - XORQ R9, R15 - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(SP) - XORQ BX, SI - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(SP) - XORQ CX, BP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(SP) - MOVQ R8, 184(SP) - - // Prepare round - MOVQ BP, BX - ROLQ $0x01, BX - MOVQ 16(SP), R12 - XORQ 56(SP), DX - XORQ R15, BX - XORQ 96(SP), R12 - XORQ 136(SP), DX - XORQ DX, R12 - MOVQ R12, CX - ROLQ $0x01, CX - MOVQ 24(SP), R13 - XORQ 64(SP), R8 - XORQ SI, CX - XORQ 104(SP), R13 - XORQ 144(SP), R8 - XORQ R8, R13 - MOVQ R13, DX - ROLQ $0x01, DX - MOVQ R15, R8 - XORQ BP, DX - ROLQ $0x01, R8 - MOVQ SI, R9 - XORQ R12, R8 - ROLQ $0x01, R9 - - // Result b - MOVQ (SP), R10 - MOVQ 48(SP), R11 - XORQ R13, R9 - MOVQ 96(SP), R12 - MOVQ 144(SP), R13 - MOVQ 192(SP), R14 - XORQ CX, R11 - ROLQ $0x2c, R11 - XORQ DX, R12 - XORQ BX, R10 - ROLQ $0x2b, R12 - MOVQ R11, SI - MOVQ $0x8000000080008008, AX - ORQ R12, SI - XORQ R10, AX - XORQ AX, SI - MOVQ SI, (DI) - XORQ R9, R14 - ROLQ $0x0e, R14 - MOVQ R10, R15 - ANDQ R11, R15 - XORQ R14, R15 - MOVQ R15, 32(DI) - XORQ R8, R13 - ROLQ $0x15, R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 16(DI) - NOTQ R12 - ORQ R10, R14 - ORQ R13, R12 - XORQ R13, R14 - XORQ R11, R12 - MOVQ R14, 24(DI) - MOVQ R12, 8(DI) - NOP - - // Result g - MOVQ 72(SP), R11 - XORQ R9, R11 - MOVQ 80(SP), R12 - ROLQ $0x14, R11 - XORQ BX, R12 - ROLQ $0x03, R12 - MOVQ 24(SP), R10 - MOVQ R11, AX - ORQ R12, AX - XORQ R8, R10 - MOVQ 128(SP), R13 - MOVQ 176(SP), R14 - ROLQ $0x1c, R10 - XORQ R10, AX - MOVQ AX, 40(DI) - NOP - XORQ CX, R13 - ROLQ $0x2d, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 48(DI) - NOP - XORQ DX, R14 - ROLQ $0x3d, R14 - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 64(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 72(DI) - NOTQ R14 - NOP - ORQ R14, R13 - XORQ R12, R13 - MOVQ R13, 56(DI) - - // Result k - MOVQ 8(SP), R10 - MOVQ 56(SP), R11 - MOVQ 104(SP), R12 - MOVQ 152(SP), R13 - MOVQ 160(SP), R14 - XORQ DX, R11 - ROLQ $0x06, R11 - XORQ R8, R12 - ROLQ $0x19, R12 - MOVQ R11, AX - ORQ R12, AX - XORQ CX, R10 - ROLQ $0x01, R10 - XORQ R10, AX - MOVQ AX, 80(DI) - NOP - XORQ R9, R13 - ROLQ $0x08, R13 - MOVQ R12, AX - ANDQ R13, AX - XORQ R11, AX - MOVQ AX, 88(DI) - NOP - XORQ BX, R14 - ROLQ $0x12, R14 - NOTQ R13 - MOVQ R13, AX - ANDQ R14, AX - XORQ R12, AX - MOVQ AX, 96(DI) - MOVQ R14, AX - ORQ R10, AX - XORQ R13, AX - MOVQ AX, 104(DI) - ANDQ R11, R10 - XORQ R14, R10 - MOVQ R10, 112(DI) - NOP - - // Result m - MOVQ 40(SP), R11 - XORQ BX, R11 - MOVQ 88(SP), R12 - ROLQ $0x24, R11 - XORQ CX, R12 - MOVQ 32(SP), R10 - ROLQ $0x0a, R12 - MOVQ R11, AX - MOVQ 136(SP), R13 - ANDQ R12, AX - XORQ R9, R10 - MOVQ 184(SP), R14 - ROLQ $0x1b, R10 - XORQ R10, AX - MOVQ AX, 120(DI) - NOP - XORQ DX, R13 - ROLQ $0x0f, R13 - MOVQ R12, AX - ORQ R13, AX - XORQ R11, AX - MOVQ AX, 128(DI) - NOP - XORQ R8, R14 - ROLQ $0x38, R14 - NOTQ R13 - MOVQ R13, AX - ORQ R14, AX - XORQ R12, AX - MOVQ AX, 136(DI) - ORQ R10, R11 - XORQ R14, R11 - MOVQ R11, 152(DI) - ANDQ R10, R14 - XORQ R13, R14 - MOVQ R14, 144(DI) - NOP - - // Result s - MOVQ 16(SP), R10 - MOVQ 64(SP), R11 - MOVQ 112(SP), R12 - XORQ DX, R10 - MOVQ 120(SP), R13 - ROLQ $0x3e, R10 - XORQ R8, R11 - MOVQ 168(SP), R14 - ROLQ $0x37, R11 - XORQ R9, R12 - MOVQ R10, R9 - XORQ CX, R14 - ROLQ $0x02, R14 - ANDQ R11, R9 - XORQ R14, R9 - MOVQ R9, 192(DI) - ROLQ $0x27, R12 - NOP - NOTQ R11 - XORQ BX, R13 - MOVQ R11, BX - ANDQ R12, BX - XORQ R10, BX - MOVQ BX, 160(DI) - NOP - ROLQ $0x29, R13 - MOVQ R12, CX - ORQ R13, CX - XORQ R11, CX - MOVQ CX, 168(DI) - NOP - MOVQ R13, DX - MOVQ R14, R8 - ANDQ R14, DX - ORQ R10, R8 - XORQ R12, DX - XORQ R13, R8 - MOVQ DX, 176(DI) - MOVQ R8, 184(DI) - - // Revert the internal state to the user state - NOTQ 8(DI) - NOTQ 16(DI) - NOTQ 64(DI) - NOTQ 96(DI) - NOTQ 136(DI) - NOTQ 160(DI) - RET diff --git a/src/vendor/golang.org/x/crypto/sha3/sha3.go b/src/vendor/golang.org/x/crypto/sha3/sha3.go deleted file mode 100644 index afedde5abf1fd4..00000000000000 --- a/src/vendor/golang.org/x/crypto/sha3/sha3.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package sha3 - -// spongeDirection indicates the direction bytes are flowing through the sponge. -type spongeDirection int - -const ( - // spongeAbsorbing indicates that the sponge is absorbing input. - spongeAbsorbing spongeDirection = iota - // spongeSqueezing indicates that the sponge is being squeezed. - spongeSqueezing -) - -const ( - // maxRate is the maximum size of the internal buffer. SHAKE-256 - // currently needs the largest buffer. - maxRate = 168 -) - -type state struct { - // Generic sponge components. - a [25]uint64 // main state of the hash - rate int // the number of bytes of state to use - - // dsbyte contains the "domain separation" bits and the first bit of - // the padding. Sections 6.1 and 6.2 of [1] separate the outputs of the - // SHA-3 and SHAKE functions by appending bitstrings to the message. - // Using a little-endian bit-ordering convention, these are "01" for SHA-3 - // and "1111" for SHAKE, or 00000010b and 00001111b, respectively. Then the - // padding rule from section 5.1 is applied to pad the message to a multiple - // of the rate, which involves adding a "1" bit, zero or more "0" bits, and - // a final "1" bit. We merge the first "1" bit from the padding into dsbyte, - // giving 00000110b (0x06) and 00011111b (0x1f). - // [1] http://csrc.nist.gov/publications/drafts/fips-202/fips_202_draft.pdf - // "Draft FIPS 202: SHA-3 Standard: Permutation-Based Hash and - // Extendable-Output Functions (May 2014)" - dsbyte byte - - i, n int // storage[i:n] is the buffer, i is only used while squeezing - storage [maxRate]byte - - // Specific to SHA-3 and SHAKE. - outputLen int // the default output size in bytes - state spongeDirection // whether the sponge is absorbing or squeezing -} - -// BlockSize returns the rate of sponge underlying this hash function. -func (d *state) BlockSize() int { return d.rate } - -// Size returns the output size of the hash function in bytes. -func (d *state) Size() int { return d.outputLen } - -// Reset clears the internal state by zeroing the sponge state and -// the buffer indexes, and setting Sponge.state to absorbing. -func (d *state) Reset() { - // Zero the permutation's state. - for i := range d.a { - d.a[i] = 0 - } - d.state = spongeAbsorbing - d.i, d.n = 0, 0 -} - -func (d *state) clone() *state { - ret := *d - return &ret -} - -// permute applies the KeccakF-1600 permutation. It handles -// any input-output buffering. -func (d *state) permute() { - switch d.state { - case spongeAbsorbing: - // If we're absorbing, we need to xor the input into the state - // before applying the permutation. - xorIn(d, d.storage[:d.rate]) - d.n = 0 - keccakF1600(&d.a) - case spongeSqueezing: - // If we're squeezing, we need to apply the permutation before - // copying more output. - keccakF1600(&d.a) - d.i = 0 - copyOut(d, d.storage[:d.rate]) - } -} - -// pads appends the domain separation bits in dsbyte, applies -// the multi-bitrate 10..1 padding rule, and permutes the state. -func (d *state) padAndPermute() { - // Pad with this instance's domain-separator bits. We know that there's - // at least one byte of space in d.buf because, if it were full, - // permute would have been called to empty it. dsbyte also contains the - // first one bit for the padding. See the comment in the state struct. - d.storage[d.n] = d.dsbyte - d.n++ - for d.n < d.rate { - d.storage[d.n] = 0 - d.n++ - } - // This adds the final one bit for the padding. Because of the way that - // bits are numbered from the LSB upwards, the final bit is the MSB of - // the last byte. - d.storage[d.rate-1] ^= 0x80 - // Apply the permutation - d.permute() - d.state = spongeSqueezing - d.n = d.rate - copyOut(d, d.storage[:d.rate]) -} - -// Write absorbs more data into the hash's state. It panics if any -// output has already been read. -func (d *state) Write(p []byte) (written int, err error) { - if d.state != spongeAbsorbing { - panic("sha3: Write after Read") - } - written = len(p) - - for len(p) > 0 { - if d.n == 0 && len(p) >= d.rate { - // The fast path; absorb a full "rate" bytes of input and apply the permutation. - xorIn(d, p[:d.rate]) - p = p[d.rate:] - keccakF1600(&d.a) - } else { - // The slow path; buffer the input until we can fill the sponge, and then xor it in. - todo := d.rate - d.n - if todo > len(p) { - todo = len(p) - } - d.n += copy(d.storage[d.n:], p[:todo]) - p = p[todo:] - - // If the sponge is full, apply the permutation. - if d.n == d.rate { - d.permute() - } - } - } - - return -} - -// Read squeezes an arbitrary number of bytes from the sponge. -func (d *state) Read(out []byte) (n int, err error) { - // If we're still absorbing, pad and apply the permutation. - if d.state == spongeAbsorbing { - d.padAndPermute() - } - - n = len(out) - - // Now, do the squeezing. - for len(out) > 0 { - n := copy(out, d.storage[d.i:d.n]) - d.i += n - out = out[n:] - - // Apply the permutation if we've squeezed the sponge dry. - if d.i == d.rate { - d.permute() - } - } - - return -} - -// Sum applies padding to the hash state and then squeezes out the desired -// number of output bytes. It panics if any output has already been read. -func (d *state) Sum(in []byte) []byte { - if d.state != spongeAbsorbing { - panic("sha3: Sum after Read") - } - - // Make a copy of the original hash so that caller can keep writing - // and summing. - dup := d.clone() - hash := make([]byte, dup.outputLen, 64) // explicit cap to allow stack allocation - dup.Read(hash) - return append(in, hash...) -} diff --git a/src/vendor/golang.org/x/crypto/sha3/sha3_s390x.go b/src/vendor/golang.org/x/crypto/sha3/sha3_s390x.go deleted file mode 100644 index 00d8034ae627bc..00000000000000 --- a/src/vendor/golang.org/x/crypto/sha3/sha3_s390x.go +++ /dev/null @@ -1,303 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build gc && !purego - -package sha3 - -// This file contains code for using the 'compute intermediate -// message digest' (KIMD) and 'compute last message digest' (KLMD) -// instructions to compute SHA-3 and SHAKE hashes on IBM Z. - -import ( - "hash" - - "golang.org/x/sys/cpu" -) - -// codes represent 7-bit KIMD/KLMD function codes as defined in -// the Principles of Operation. -type code uint64 - -const ( - // function codes for KIMD/KLMD - sha3_224 code = 32 - sha3_256 = 33 - sha3_384 = 34 - sha3_512 = 35 - shake_128 = 36 - shake_256 = 37 - nopad = 0x100 -) - -// kimd is a wrapper for the 'compute intermediate message digest' instruction. -// src must be a multiple of the rate for the given function code. -// -//go:noescape -func kimd(function code, chain *[200]byte, src []byte) - -// klmd is a wrapper for the 'compute last message digest' instruction. -// src padding is handled by the instruction. -// -//go:noescape -func klmd(function code, chain *[200]byte, dst, src []byte) - -type asmState struct { - a [200]byte // 1600 bit state - buf []byte // care must be taken to ensure cap(buf) is a multiple of rate - rate int // equivalent to block size - storage [3072]byte // underlying storage for buf - outputLen int // output length for full security - function code // KIMD/KLMD function code - state spongeDirection // whether the sponge is absorbing or squeezing -} - -func newAsmState(function code) *asmState { - var s asmState - s.function = function - switch function { - case sha3_224: - s.rate = 144 - s.outputLen = 28 - case sha3_256: - s.rate = 136 - s.outputLen = 32 - case sha3_384: - s.rate = 104 - s.outputLen = 48 - case sha3_512: - s.rate = 72 - s.outputLen = 64 - case shake_128: - s.rate = 168 - s.outputLen = 32 - case shake_256: - s.rate = 136 - s.outputLen = 64 - default: - panic("sha3: unrecognized function code") - } - - // limit s.buf size to a multiple of s.rate - s.resetBuf() - return &s -} - -func (s *asmState) clone() *asmState { - c := *s - c.buf = c.storage[:len(s.buf):cap(s.buf)] - return &c -} - -// copyIntoBuf copies b into buf. It will panic if there is not enough space to -// store all of b. -func (s *asmState) copyIntoBuf(b []byte) { - bufLen := len(s.buf) - s.buf = s.buf[:len(s.buf)+len(b)] - copy(s.buf[bufLen:], b) -} - -// resetBuf points buf at storage, sets the length to 0 and sets cap to be a -// multiple of the rate. -func (s *asmState) resetBuf() { - max := (cap(s.storage) / s.rate) * s.rate - s.buf = s.storage[:0:max] -} - -// Write (via the embedded io.Writer interface) adds more data to the running hash. -// It never returns an error. -func (s *asmState) Write(b []byte) (int, error) { - if s.state != spongeAbsorbing { - panic("sha3: Write after Read") - } - length := len(b) - for len(b) > 0 { - if len(s.buf) == 0 && len(b) >= cap(s.buf) { - // Hash the data directly and push any remaining bytes - // into the buffer. - remainder := len(b) % s.rate - kimd(s.function, &s.a, b[:len(b)-remainder]) - if remainder != 0 { - s.copyIntoBuf(b[len(b)-remainder:]) - } - return length, nil - } - - if len(s.buf) == cap(s.buf) { - // flush the buffer - kimd(s.function, &s.a, s.buf) - s.buf = s.buf[:0] - } - - // copy as much as we can into the buffer - n := len(b) - if len(b) > cap(s.buf)-len(s.buf) { - n = cap(s.buf) - len(s.buf) - } - s.copyIntoBuf(b[:n]) - b = b[n:] - } - return length, nil -} - -// Read squeezes an arbitrary number of bytes from the sponge. -func (s *asmState) Read(out []byte) (n int, err error) { - // The 'compute last message digest' instruction only stores the digest - // at the first operand (dst) for SHAKE functions. - if s.function != shake_128 && s.function != shake_256 { - panic("sha3: can only call Read for SHAKE functions") - } - - n = len(out) - - // need to pad if we were absorbing - if s.state == spongeAbsorbing { - s.state = spongeSqueezing - - // write hash directly into out if possible - if len(out)%s.rate == 0 { - klmd(s.function, &s.a, out, s.buf) // len(out) may be 0 - s.buf = s.buf[:0] - return - } - - // write hash into buffer - max := cap(s.buf) - if max > len(out) { - max = (len(out)/s.rate)*s.rate + s.rate - } - klmd(s.function, &s.a, s.buf[:max], s.buf) - s.buf = s.buf[:max] - } - - for len(out) > 0 { - // flush the buffer - if len(s.buf) != 0 { - c := copy(out, s.buf) - out = out[c:] - s.buf = s.buf[c:] - continue - } - - // write hash directly into out if possible - if len(out)%s.rate == 0 { - klmd(s.function|nopad, &s.a, out, nil) - return - } - - // write hash into buffer - s.resetBuf() - if cap(s.buf) > len(out) { - s.buf = s.buf[:(len(out)/s.rate)*s.rate+s.rate] - } - klmd(s.function|nopad, &s.a, s.buf, nil) - } - return -} - -// Sum appends the current hash to b and returns the resulting slice. -// It does not change the underlying hash state. -func (s *asmState) Sum(b []byte) []byte { - if s.state != spongeAbsorbing { - panic("sha3: Sum after Read") - } - - // Copy the state to preserve the original. - a := s.a - - // Hash the buffer. Note that we don't clear it because we - // aren't updating the state. - switch s.function { - case sha3_224, sha3_256, sha3_384, sha3_512: - klmd(s.function, &a, nil, s.buf) - return append(b, a[:s.outputLen]...) - case shake_128, shake_256: - d := make([]byte, s.outputLen, 64) - klmd(s.function, &a, d, s.buf) - return append(b, d[:s.outputLen]...) - default: - panic("sha3: unknown function") - } -} - -// Reset resets the Hash to its initial state. -func (s *asmState) Reset() { - for i := range s.a { - s.a[i] = 0 - } - s.resetBuf() - s.state = spongeAbsorbing -} - -// Size returns the number of bytes Sum will return. -func (s *asmState) Size() int { - return s.outputLen -} - -// BlockSize returns the hash's underlying block size. -// The Write method must be able to accept any amount -// of data, but it may operate more efficiently if all writes -// are a multiple of the block size. -func (s *asmState) BlockSize() int { - return s.rate -} - -// Clone returns a copy of the ShakeHash in its current state. -func (s *asmState) Clone() ShakeHash { - return s.clone() -} - -// new224 returns an assembly implementation of SHA3-224 if available, -// otherwise it returns a generic implementation. -func new224() hash.Hash { - if cpu.S390X.HasSHA3 { - return newAsmState(sha3_224) - } - return new224Generic() -} - -// new256 returns an assembly implementation of SHA3-256 if available, -// otherwise it returns a generic implementation. -func new256() hash.Hash { - if cpu.S390X.HasSHA3 { - return newAsmState(sha3_256) - } - return new256Generic() -} - -// new384 returns an assembly implementation of SHA3-384 if available, -// otherwise it returns a generic implementation. -func new384() hash.Hash { - if cpu.S390X.HasSHA3 { - return newAsmState(sha3_384) - } - return new384Generic() -} - -// new512 returns an assembly implementation of SHA3-512 if available, -// otherwise it returns a generic implementation. -func new512() hash.Hash { - if cpu.S390X.HasSHA3 { - return newAsmState(sha3_512) - } - return new512Generic() -} - -// newShake128 returns an assembly implementation of SHAKE-128 if available, -// otherwise it returns a generic implementation. -func newShake128() ShakeHash { - if cpu.S390X.HasSHA3 { - return newAsmState(shake_128) - } - return newShake128Generic() -} - -// newShake256 returns an assembly implementation of SHAKE-256 if available, -// otherwise it returns a generic implementation. -func newShake256() ShakeHash { - if cpu.S390X.HasSHA3 { - return newAsmState(shake_256) - } - return newShake256Generic() -} diff --git a/src/vendor/golang.org/x/crypto/sha3/sha3_s390x.s b/src/vendor/golang.org/x/crypto/sha3/sha3_s390x.s deleted file mode 100644 index 826b862c779622..00000000000000 --- a/src/vendor/golang.org/x/crypto/sha3/sha3_s390x.s +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build gc && !purego - -#include "textflag.h" - -// func kimd(function code, chain *[200]byte, src []byte) -TEXT ·kimd(SB), NOFRAME|NOSPLIT, $0-40 - MOVD function+0(FP), R0 - MOVD chain+8(FP), R1 - LMG src+16(FP), R2, R3 // R2=base, R3=len - -continue: - WORD $0xB93E0002 // KIMD --, R2 - BVS continue // continue if interrupted - MOVD $0, R0 // reset R0 for pre-go1.8 compilers - RET - -// func klmd(function code, chain *[200]byte, dst, src []byte) -TEXT ·klmd(SB), NOFRAME|NOSPLIT, $0-64 - // TODO: SHAKE support - MOVD function+0(FP), R0 - MOVD chain+8(FP), R1 - LMG dst+16(FP), R2, R3 // R2=base, R3=len - LMG src+40(FP), R4, R5 // R4=base, R5=len - -continue: - WORD $0xB93F0024 // KLMD R2, R4 - BVS continue // continue if interrupted - MOVD $0, R0 // reset R0 for pre-go1.8 compilers - RET diff --git a/src/vendor/golang.org/x/crypto/sha3/shake.go b/src/vendor/golang.org/x/crypto/sha3/shake.go deleted file mode 100644 index 1ea9275b8b7ac7..00000000000000 --- a/src/vendor/golang.org/x/crypto/sha3/shake.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package sha3 - -// This file defines the ShakeHash interface, and provides -// functions for creating SHAKE and cSHAKE instances, as well as utility -// functions for hashing bytes to arbitrary-length output. -// -// -// SHAKE implementation is based on FIPS PUB 202 [1] -// cSHAKE implementations is based on NIST SP 800-185 [2] -// -// [1] https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.202.pdf -// [2] https://doi.org/10.6028/NIST.SP.800-185 - -import ( - "encoding/binary" - "hash" - "io" -) - -// ShakeHash defines the interface to hash functions that support -// arbitrary-length output. When used as a plain [hash.Hash], it -// produces minimum-length outputs that provide full-strength generic -// security. -type ShakeHash interface { - hash.Hash - - // Read reads more output from the hash; reading affects the hash's - // state. (ShakeHash.Read is thus very different from Hash.Sum) - // It never returns an error, but subsequent calls to Write or Sum - // will panic. - io.Reader - - // Clone returns a copy of the ShakeHash in its current state. - Clone() ShakeHash -} - -// cSHAKE specific context -type cshakeState struct { - *state // SHA-3 state context and Read/Write operations - - // initBlock is the cSHAKE specific initialization set of bytes. It is initialized - // by newCShake function and stores concatenation of N followed by S, encoded - // by the method specified in 3.3 of [1]. - // It is stored here in order for Reset() to be able to put context into - // initial state. - initBlock []byte -} - -// Consts for configuring initial SHA-3 state -const ( - dsbyteShake = 0x1f - dsbyteCShake = 0x04 - rate128 = 168 - rate256 = 136 -) - -func bytepad(input []byte, w int) []byte { - // leftEncode always returns max 9 bytes - buf := make([]byte, 0, 9+len(input)+w) - buf = append(buf, leftEncode(uint64(w))...) - buf = append(buf, input...) - padlen := w - (len(buf) % w) - return append(buf, make([]byte, padlen)...) -} - -func leftEncode(value uint64) []byte { - var b [9]byte - binary.BigEndian.PutUint64(b[1:], value) - // Trim all but last leading zero bytes - i := byte(1) - for i < 8 && b[i] == 0 { - i++ - } - // Prepend number of encoded bytes - b[i-1] = 9 - i - return b[i-1:] -} - -func newCShake(N, S []byte, rate, outputLen int, dsbyte byte) ShakeHash { - c := cshakeState{state: &state{rate: rate, outputLen: outputLen, dsbyte: dsbyte}} - - // leftEncode returns max 9 bytes - c.initBlock = make([]byte, 0, 9*2+len(N)+len(S)) - c.initBlock = append(c.initBlock, leftEncode(uint64(len(N)*8))...) - c.initBlock = append(c.initBlock, N...) - c.initBlock = append(c.initBlock, leftEncode(uint64(len(S)*8))...) - c.initBlock = append(c.initBlock, S...) - c.Write(bytepad(c.initBlock, c.rate)) - return &c -} - -// Reset resets the hash to initial state. -func (c *cshakeState) Reset() { - c.state.Reset() - c.Write(bytepad(c.initBlock, c.rate)) -} - -// Clone returns copy of a cSHAKE context within its current state. -func (c *cshakeState) Clone() ShakeHash { - b := make([]byte, len(c.initBlock)) - copy(b, c.initBlock) - return &cshakeState{state: c.clone(), initBlock: b} -} - -// Clone returns copy of SHAKE context within its current state. -func (c *state) Clone() ShakeHash { - return c.clone() -} - -// NewShake128 creates a new SHAKE128 variable-output-length ShakeHash. -// Its generic security strength is 128 bits against all attacks if at -// least 32 bytes of its output are used. -func NewShake128() ShakeHash { - return newShake128() -} - -// NewShake256 creates a new SHAKE256 variable-output-length ShakeHash. -// Its generic security strength is 256 bits against all attacks if -// at least 64 bytes of its output are used. -func NewShake256() ShakeHash { - return newShake256() -} - -func newShake128Generic() *state { - return &state{rate: rate128, outputLen: 32, dsbyte: dsbyteShake} -} - -func newShake256Generic() *state { - return &state{rate: rate256, outputLen: 64, dsbyte: dsbyteShake} -} - -// NewCShake128 creates a new instance of cSHAKE128 variable-output-length ShakeHash, -// a customizable variant of SHAKE128. -// N is used to define functions based on cSHAKE, it can be empty when plain cSHAKE is -// desired. S is a customization byte string used for domain separation - two cSHAKE -// computations on same input with different S yield unrelated outputs. -// When N and S are both empty, this is equivalent to NewShake128. -func NewCShake128(N, S []byte) ShakeHash { - if len(N) == 0 && len(S) == 0 { - return NewShake128() - } - return newCShake(N, S, rate128, 32, dsbyteCShake) -} - -// NewCShake256 creates a new instance of cSHAKE256 variable-output-length ShakeHash, -// a customizable variant of SHAKE256. -// N is used to define functions based on cSHAKE, it can be empty when plain cSHAKE is -// desired. S is a customization byte string used for domain separation - two cSHAKE -// computations on same input with different S yield unrelated outputs. -// When N and S are both empty, this is equivalent to NewShake256. -func NewCShake256(N, S []byte) ShakeHash { - if len(N) == 0 && len(S) == 0 { - return NewShake256() - } - return newCShake(N, S, rate256, 64, dsbyteCShake) -} - -// ShakeSum128 writes an arbitrary-length digest of data into hash. -func ShakeSum128(hash, data []byte) { - h := NewShake128() - h.Write(data) - h.Read(hash) -} - -// ShakeSum256 writes an arbitrary-length digest of data into hash. -func ShakeSum256(hash, data []byte) { - h := NewShake256() - h.Write(data) - h.Read(hash) -} diff --git a/src/vendor/golang.org/x/crypto/sha3/shake_noasm.go b/src/vendor/golang.org/x/crypto/sha3/shake_noasm.go deleted file mode 100644 index 4276ba4ab2c417..00000000000000 --- a/src/vendor/golang.org/x/crypto/sha3/shake_noasm.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !gc || purego || !s390x - -package sha3 - -func newShake128() *state { - return newShake128Generic() -} - -func newShake256() *state { - return newShake256Generic() -} diff --git a/src/vendor/golang.org/x/crypto/sha3/xor.go b/src/vendor/golang.org/x/crypto/sha3/xor.go deleted file mode 100644 index 6ada5c9574e2af..00000000000000 --- a/src/vendor/golang.org/x/crypto/sha3/xor.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package sha3 - -import ( - "crypto/subtle" - "encoding/binary" - "unsafe" - - "golang.org/x/sys/cpu" -) - -// xorIn xors the bytes in buf into the state. -func xorIn(d *state, buf []byte) { - if cpu.IsBigEndian { - for i := 0; len(buf) >= 8; i++ { - a := binary.LittleEndian.Uint64(buf) - d.a[i] ^= a - buf = buf[8:] - } - } else { - ab := (*[25 * 64 / 8]byte)(unsafe.Pointer(&d.a)) - subtle.XORBytes(ab[:], ab[:], buf) - } -} - -// copyOut copies uint64s to a byte buffer. -func copyOut(d *state, b []byte) { - if cpu.IsBigEndian { - for i := 0; len(b) >= 8; i++ { - binary.LittleEndian.PutUint64(b, d.a[i]) - b = b[8:] - } - } else { - ab := (*[25 * 64 / 8]byte)(unsafe.Pointer(&d.a)) - copy(b, ab[:]) - } -} diff --git a/src/vendor/golang.org/x/net/http/httpproxy/proxy.go b/src/vendor/golang.org/x/net/http/httpproxy/proxy.go index 6404aaf157d6ad..d89c257ae72314 100644 --- a/src/vendor/golang.org/x/net/http/httpproxy/proxy.go +++ b/src/vendor/golang.org/x/net/http/httpproxy/proxy.go @@ -14,6 +14,7 @@ import ( "errors" "fmt" "net" + "net/netip" "net/url" "os" "strings" @@ -177,8 +178,10 @@ func (cfg *config) useProxy(addr string) bool { if host == "localhost" { return false } - ip := net.ParseIP(host) - if ip != nil { + nip, err := netip.ParseAddr(host) + var ip net.IP + if err == nil { + ip = net.IP(nip.AsSlice()) if ip.IsLoopback() { return false } @@ -360,6 +363,9 @@ type domainMatch struct { } func (m domainMatch) match(host, port string, ip net.IP) bool { + if ip != nil { + return false + } if strings.HasSuffix(host, m.host) || (m.matchHost && host == m.host[1:]) { return m.port == "" || m.port == port } diff --git a/src/vendor/golang.org/x/net/nettest/conntest.go b/src/vendor/golang.org/x/net/nettest/conntest.go index 4297d408c0477c..8b98dfe21c5d30 100644 --- a/src/vendor/golang.org/x/net/nettest/conntest.go +++ b/src/vendor/golang.org/x/net/nettest/conntest.go @@ -142,7 +142,7 @@ func testPingPong(t *testing.T, c1, c2 net.Conn) { } // testRacyRead tests that it is safe to mutate the input Read buffer -// immediately after cancelation has occurred. +// immediately after cancellation has occurred. func testRacyRead(t *testing.T, c1, c2 net.Conn) { go chunkedCopy(c2, rand.New(rand.NewSource(0))) @@ -170,7 +170,7 @@ func testRacyRead(t *testing.T, c1, c2 net.Conn) { } // testRacyWrite tests that it is safe to mutate the input Write buffer -// immediately after cancelation has occurred. +// immediately after cancellation has occurred. func testRacyWrite(t *testing.T, c1, c2 net.Conn) { go chunkedCopy(io.Discard, c2) @@ -318,7 +318,7 @@ func testCloseTimeout(t *testing.T, c1, c2 net.Conn) { defer wg.Wait() wg.Add(3) - // Test for cancelation upon connection closure. + // Test for cancellation upon connection closure. c1.SetDeadline(neverTimeout) go func() { defer wg.Done() diff --git a/src/vendor/golang.org/x/net/route/address.go b/src/vendor/golang.org/x/net/route/address.go deleted file mode 100644 index 5443d672236c2a..00000000000000 --- a/src/vendor/golang.org/x/net/route/address.go +++ /dev/null @@ -1,432 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build darwin || dragonfly || freebsd || netbsd || openbsd - -package route - -import ( - "runtime" - "syscall" -) - -// An Addr represents an address associated with packet routing. -type Addr interface { - // Family returns an address family. - Family() int -} - -// A LinkAddr represents a link-layer address. -type LinkAddr struct { - Index int // interface index when attached - Name string // interface name when attached - Addr []byte // link-layer address when attached -} - -// Family implements the Family method of Addr interface. -func (a *LinkAddr) Family() int { return syscall.AF_LINK } - -func (a *LinkAddr) lenAndSpace() (int, int) { - l := 8 + len(a.Name) + len(a.Addr) - return l, roundup(l) -} - -func (a *LinkAddr) marshal(b []byte) (int, error) { - l, ll := a.lenAndSpace() - if len(b) < ll { - return 0, errShortBuffer - } - nlen, alen := len(a.Name), len(a.Addr) - if nlen > 255 || alen > 255 { - return 0, errInvalidAddr - } - b[0] = byte(l) - b[1] = syscall.AF_LINK - if a.Index > 0 { - nativeEndian.PutUint16(b[2:4], uint16(a.Index)) - } - data := b[8:] - if nlen > 0 { - b[5] = byte(nlen) - copy(data[:nlen], a.Name) - data = data[nlen:] - } - if alen > 0 { - b[6] = byte(alen) - copy(data[:alen], a.Addr) - data = data[alen:] - } - return ll, nil -} - -func parseLinkAddr(b []byte) (Addr, error) { - if len(b) < 8 { - return nil, errInvalidAddr - } - _, a, err := parseKernelLinkAddr(syscall.AF_LINK, b[4:]) - if err != nil { - return nil, err - } - a.(*LinkAddr).Index = int(nativeEndian.Uint16(b[2:4])) - return a, nil -} - -// parseKernelLinkAddr parses b as a link-layer address in -// conventional BSD kernel form. -func parseKernelLinkAddr(_ int, b []byte) (int, Addr, error) { - // The encoding looks like the following: - // +----------------------------+ - // | Type (1 octet) | - // +----------------------------+ - // | Name length (1 octet) | - // +----------------------------+ - // | Address length (1 octet) | - // +----------------------------+ - // | Selector length (1 octet) | - // +----------------------------+ - // | Data (variable) | - // +----------------------------+ - // - // On some platforms, all-bit-one of length field means "don't - // care". - nlen, alen, slen := int(b[1]), int(b[2]), int(b[3]) - if nlen == 0xff { - nlen = 0 - } - if alen == 0xff { - alen = 0 - } - if slen == 0xff { - slen = 0 - } - l := 4 + nlen + alen + slen - if len(b) < l { - return 0, nil, errInvalidAddr - } - data := b[4:] - var name string - var addr []byte - if nlen > 0 { - name = string(data[:nlen]) - data = data[nlen:] - } - if alen > 0 { - addr = data[:alen] - data = data[alen:] - } - return l, &LinkAddr{Name: name, Addr: addr}, nil -} - -// An Inet4Addr represents an internet address for IPv4. -type Inet4Addr struct { - IP [4]byte // IP address -} - -// Family implements the Family method of Addr interface. -func (a *Inet4Addr) Family() int { return syscall.AF_INET } - -func (a *Inet4Addr) lenAndSpace() (int, int) { - return sizeofSockaddrInet, roundup(sizeofSockaddrInet) -} - -func (a *Inet4Addr) marshal(b []byte) (int, error) { - l, ll := a.lenAndSpace() - if len(b) < ll { - return 0, errShortBuffer - } - b[0] = byte(l) - b[1] = syscall.AF_INET - copy(b[4:8], a.IP[:]) - return ll, nil -} - -// An Inet6Addr represents an internet address for IPv6. -type Inet6Addr struct { - IP [16]byte // IP address - ZoneID int // zone identifier -} - -// Family implements the Family method of Addr interface. -func (a *Inet6Addr) Family() int { return syscall.AF_INET6 } - -func (a *Inet6Addr) lenAndSpace() (int, int) { - return sizeofSockaddrInet6, roundup(sizeofSockaddrInet6) -} - -func (a *Inet6Addr) marshal(b []byte) (int, error) { - l, ll := a.lenAndSpace() - if len(b) < ll { - return 0, errShortBuffer - } - b[0] = byte(l) - b[1] = syscall.AF_INET6 - copy(b[8:24], a.IP[:]) - if a.ZoneID > 0 { - nativeEndian.PutUint32(b[24:28], uint32(a.ZoneID)) - } - return ll, nil -} - -// parseInetAddr parses b as an internet address for IPv4 or IPv6. -func parseInetAddr(af int, b []byte) (Addr, error) { - switch af { - case syscall.AF_INET: - if len(b) < sizeofSockaddrInet { - return nil, errInvalidAddr - } - a := &Inet4Addr{} - copy(a.IP[:], b[4:8]) - return a, nil - case syscall.AF_INET6: - if len(b) < sizeofSockaddrInet6 { - return nil, errInvalidAddr - } - a := &Inet6Addr{ZoneID: int(nativeEndian.Uint32(b[24:28]))} - copy(a.IP[:], b[8:24]) - if a.IP[0] == 0xfe && a.IP[1]&0xc0 == 0x80 || a.IP[0] == 0xff && (a.IP[1]&0x0f == 0x01 || a.IP[1]&0x0f == 0x02) { - // KAME based IPv6 protocol stack usually - // embeds the interface index in the - // interface-local or link-local address as - // the kernel-internal form. - id := int(bigEndian.Uint16(a.IP[2:4])) - if id != 0 { - a.ZoneID = id - a.IP[2], a.IP[3] = 0, 0 - } - } - return a, nil - default: - return nil, errInvalidAddr - } -} - -// parseKernelInetAddr parses b as an internet address in conventional -// BSD kernel form. -func parseKernelInetAddr(af int, b []byte) (int, Addr, error) { - // The encoding looks similar to the NLRI encoding. - // +----------------------------+ - // | Length (1 octet) | - // +----------------------------+ - // | Address prefix (variable) | - // +----------------------------+ - // - // The differences between the kernel form and the NLRI - // encoding are: - // - // - The length field of the kernel form indicates the prefix - // length in bytes, not in bits - // - // - In the kernel form, zero value of the length field - // doesn't mean 0.0.0.0/0 or ::/0 - // - // - The kernel form appends leading bytes to the prefix field - // to make the tuple to be conformed with - // the routing message boundary - l := int(b[0]) - if runtime.GOOS == "darwin" || runtime.GOOS == "ios" { - // On Darwin, an address in the kernel form is also - // used as a message filler. - if l == 0 || len(b) > roundup(l) { - l = roundup(l) - } - } else { - l = roundup(l) - } - if len(b) < l { - return 0, nil, errInvalidAddr - } - // Don't reorder case expressions. - // The case expressions for IPv6 must come first. - const ( - off4 = 4 // offset of in_addr - off6 = 8 // offset of in6_addr - ) - switch { - case b[0] == sizeofSockaddrInet6: - a := &Inet6Addr{} - copy(a.IP[:], b[off6:off6+16]) - return int(b[0]), a, nil - case af == syscall.AF_INET6: - a := &Inet6Addr{} - if l-1 < off6 { - copy(a.IP[:], b[1:l]) - } else { - copy(a.IP[:], b[l-off6:l]) - } - return int(b[0]), a, nil - case b[0] == sizeofSockaddrInet: - a := &Inet4Addr{} - copy(a.IP[:], b[off4:off4+4]) - return int(b[0]), a, nil - default: // an old fashion, AF_UNSPEC or unknown means AF_INET - a := &Inet4Addr{} - if l-1 < off4 { - copy(a.IP[:], b[1:l]) - } else { - copy(a.IP[:], b[l-off4:l]) - } - return int(b[0]), a, nil - } -} - -// A DefaultAddr represents an address of various operating -// system-specific features. -type DefaultAddr struct { - af int - Raw []byte // raw format of address -} - -// Family implements the Family method of Addr interface. -func (a *DefaultAddr) Family() int { return a.af } - -func (a *DefaultAddr) lenAndSpace() (int, int) { - l := len(a.Raw) - return l, roundup(l) -} - -func (a *DefaultAddr) marshal(b []byte) (int, error) { - l, ll := a.lenAndSpace() - if len(b) < ll { - return 0, errShortBuffer - } - if l > 255 { - return 0, errInvalidAddr - } - b[1] = byte(l) - copy(b[:l], a.Raw) - return ll, nil -} - -func parseDefaultAddr(b []byte) (Addr, error) { - if len(b) < 2 || len(b) < int(b[0]) { - return nil, errInvalidAddr - } - a := &DefaultAddr{af: int(b[1]), Raw: b[:b[0]]} - return a, nil -} - -func addrsSpace(as []Addr) int { - var l int - for _, a := range as { - switch a := a.(type) { - case *LinkAddr: - _, ll := a.lenAndSpace() - l += ll - case *Inet4Addr: - _, ll := a.lenAndSpace() - l += ll - case *Inet6Addr: - _, ll := a.lenAndSpace() - l += ll - case *DefaultAddr: - _, ll := a.lenAndSpace() - l += ll - } - } - return l -} - -// marshalAddrs marshals as and returns a bitmap indicating which -// address is stored in b. -func marshalAddrs(b []byte, as []Addr) (uint, error) { - var attrs uint - for i, a := range as { - switch a := a.(type) { - case *LinkAddr: - l, err := a.marshal(b) - if err != nil { - return 0, err - } - b = b[l:] - attrs |= 1 << uint(i) - case *Inet4Addr: - l, err := a.marshal(b) - if err != nil { - return 0, err - } - b = b[l:] - attrs |= 1 << uint(i) - case *Inet6Addr: - l, err := a.marshal(b) - if err != nil { - return 0, err - } - b = b[l:] - attrs |= 1 << uint(i) - case *DefaultAddr: - l, err := a.marshal(b) - if err != nil { - return 0, err - } - b = b[l:] - attrs |= 1 << uint(i) - } - } - return attrs, nil -} - -func parseAddrs(attrs uint, fn func(int, []byte) (int, Addr, error), b []byte) ([]Addr, error) { - var as [syscall.RTAX_MAX]Addr - af := int(syscall.AF_UNSPEC) - for i := uint(0); i < syscall.RTAX_MAX && len(b) >= roundup(0); i++ { - if attrs&(1<> 8) -} - -func (binaryLittleEndian) Uint32(b []byte) uint32 { - _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 - return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 -} - -func (binaryLittleEndian) PutUint32(b []byte, v uint32) { - _ = b[3] // early bounds check to guarantee safety of writes below - b[0] = byte(v) - b[1] = byte(v >> 8) - b[2] = byte(v >> 16) - b[3] = byte(v >> 24) -} - -func (binaryLittleEndian) Uint64(b []byte) uint64 { - _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 -} - -type binaryBigEndian struct{} - -func (binaryBigEndian) Uint16(b []byte) uint16 { - _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 - return uint16(b[1]) | uint16(b[0])<<8 -} - -func (binaryBigEndian) PutUint16(b []byte, v uint16) { - _ = b[1] // early bounds check to guarantee safety of writes below - b[0] = byte(v >> 8) - b[1] = byte(v) -} - -func (binaryBigEndian) Uint32(b []byte) uint32 { - _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 - return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 -} - -func (binaryBigEndian) PutUint32(b []byte, v uint32) { - _ = b[3] // early bounds check to guarantee safety of writes below - b[0] = byte(v >> 24) - b[1] = byte(v >> 16) - b[2] = byte(v >> 8) - b[3] = byte(v) -} - -func (binaryBigEndian) Uint64(b []byte) uint64 { - _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 - return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | - uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 -} diff --git a/src/vendor/golang.org/x/net/route/empty.s b/src/vendor/golang.org/x/net/route/empty.s deleted file mode 100644 index 49d79791e01bb6..00000000000000 --- a/src/vendor/golang.org/x/net/route/empty.s +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build darwin && go1.12 - -// This exists solely so we can linkname in symbols from syscall. diff --git a/src/vendor/golang.org/x/net/route/interface.go b/src/vendor/golang.org/x/net/route/interface.go deleted file mode 100644 index 0aa70555ca6aef..00000000000000 --- a/src/vendor/golang.org/x/net/route/interface.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build darwin || dragonfly || freebsd || netbsd || openbsd - -package route - -// An InterfaceMessage represents an interface message. -type InterfaceMessage struct { - Version int // message version - Type int // message type - Flags int // interface flags - Index int // interface index - Name string // interface name - Addrs []Addr // addresses - - extOff int // offset of header extension - raw []byte // raw message -} - -// An InterfaceAddrMessage represents an interface address message. -type InterfaceAddrMessage struct { - Version int // message version - Type int // message type - Flags int // interface flags - Index int // interface index - Addrs []Addr // addresses - - raw []byte // raw message -} - -// Sys implements the Sys method of Message interface. -func (m *InterfaceAddrMessage) Sys() []Sys { return nil } - -// An InterfaceMulticastAddrMessage represents an interface multicast -// address message. -type InterfaceMulticastAddrMessage struct { - Version int // message version - Type int // message type - Flags int // interface flags - Index int // interface index - Addrs []Addr // addresses - - raw []byte // raw message -} - -// Sys implements the Sys method of Message interface. -func (m *InterfaceMulticastAddrMessage) Sys() []Sys { return nil } - -// An InterfaceAnnounceMessage represents an interface announcement -// message. -type InterfaceAnnounceMessage struct { - Version int // message version - Type int // message type - Index int // interface index - Name string // interface name - What int // what type of announcement - - raw []byte // raw message -} - -// Sys implements the Sys method of Message interface. -func (m *InterfaceAnnounceMessage) Sys() []Sys { return nil } diff --git a/src/vendor/golang.org/x/net/route/interface_announce.go b/src/vendor/golang.org/x/net/route/interface_announce.go deleted file mode 100644 index 70614c1b1a17ef..00000000000000 --- a/src/vendor/golang.org/x/net/route/interface_announce.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build dragonfly || freebsd || netbsd - -package route - -func (w *wireFormat) parseInterfaceAnnounceMessage(_ RIBType, b []byte) (Message, error) { - if len(b) < w.bodyOff { - return nil, errMessageTooShort - } - l := int(nativeEndian.Uint16(b[:2])) - if len(b) < l { - return nil, errInvalidMessage - } - m := &InterfaceAnnounceMessage{ - Version: int(b[2]), - Type: int(b[3]), - Index: int(nativeEndian.Uint16(b[4:6])), - What: int(nativeEndian.Uint16(b[22:24])), - raw: b[:l], - } - for i := 0; i < 16; i++ { - if b[6+i] != 0 { - continue - } - m.Name = string(b[6 : 6+i]) - break - } - return m, nil -} diff --git a/src/vendor/golang.org/x/net/route/interface_classic.go b/src/vendor/golang.org/x/net/route/interface_classic.go deleted file mode 100644 index be1bf2652e675a..00000000000000 --- a/src/vendor/golang.org/x/net/route/interface_classic.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build darwin || dragonfly || netbsd - -package route - -import ( - "runtime" - "syscall" -) - -func (w *wireFormat) parseInterfaceMessage(_ RIBType, b []byte) (Message, error) { - if len(b) < w.bodyOff { - return nil, errMessageTooShort - } - l := int(nativeEndian.Uint16(b[:2])) - if len(b) < l { - return nil, errInvalidMessage - } - attrs := uint(nativeEndian.Uint32(b[4:8])) - if attrs&syscall.RTA_IFP == 0 { - return nil, nil - } - m := &InterfaceMessage{ - Version: int(b[2]), - Type: int(b[3]), - Addrs: make([]Addr, syscall.RTAX_MAX), - Flags: int(nativeEndian.Uint32(b[8:12])), - Index: int(nativeEndian.Uint16(b[12:14])), - extOff: w.extOff, - raw: b[:l], - } - a, err := parseLinkAddr(b[w.bodyOff:]) - if err != nil { - return nil, err - } - m.Addrs[syscall.RTAX_IFP] = a - m.Name = a.(*LinkAddr).Name - return m, nil -} - -func (w *wireFormat) parseInterfaceAddrMessage(_ RIBType, b []byte) (Message, error) { - if len(b) < w.bodyOff { - return nil, errMessageTooShort - } - l := int(nativeEndian.Uint16(b[:2])) - if len(b) < l { - return nil, errInvalidMessage - } - m := &InterfaceAddrMessage{ - Version: int(b[2]), - Type: int(b[3]), - Flags: int(nativeEndian.Uint32(b[8:12])), - raw: b[:l], - } - if runtime.GOOS == "netbsd" { - m.Index = int(nativeEndian.Uint16(b[16:18])) - } else { - m.Index = int(nativeEndian.Uint16(b[12:14])) - } - var err error - m.Addrs, err = parseAddrs(uint(nativeEndian.Uint32(b[4:8])), parseKernelInetAddr, b[w.bodyOff:]) - if err != nil { - return nil, err - } - return m, nil -} diff --git a/src/vendor/golang.org/x/net/route/interface_freebsd.go b/src/vendor/golang.org/x/net/route/interface_freebsd.go deleted file mode 100644 index 14d251c94f9084..00000000000000 --- a/src/vendor/golang.org/x/net/route/interface_freebsd.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package route - -import "syscall" - -func (w *wireFormat) parseInterfaceMessage(typ RIBType, b []byte) (Message, error) { - var extOff, bodyOff int - if typ == syscall.NET_RT_IFLISTL { - if len(b) < 20 { - return nil, errMessageTooShort - } - extOff = int(nativeEndian.Uint16(b[18:20])) - bodyOff = int(nativeEndian.Uint16(b[16:18])) - } else { - extOff = w.extOff - bodyOff = w.bodyOff - } - if len(b) < extOff || len(b) < bodyOff { - return nil, errInvalidMessage - } - l := int(nativeEndian.Uint16(b[:2])) - if len(b) < l { - return nil, errInvalidMessage - } - attrs := uint(nativeEndian.Uint32(b[4:8])) - if attrs&syscall.RTA_IFP == 0 { - return nil, nil - } - m := &InterfaceMessage{ - Version: int(b[2]), - Type: int(b[3]), - Flags: int(nativeEndian.Uint32(b[8:12])), - Index: int(nativeEndian.Uint16(b[12:14])), - Addrs: make([]Addr, syscall.RTAX_MAX), - extOff: extOff, - raw: b[:l], - } - a, err := parseLinkAddr(b[bodyOff:]) - if err != nil { - return nil, err - } - m.Addrs[syscall.RTAX_IFP] = a - m.Name = a.(*LinkAddr).Name - return m, nil -} - -func (w *wireFormat) parseInterfaceAddrMessage(typ RIBType, b []byte) (Message, error) { - var bodyOff int - if typ == syscall.NET_RT_IFLISTL { - if len(b) < 24 { - return nil, errMessageTooShort - } - bodyOff = int(nativeEndian.Uint16(b[16:18])) - } else { - bodyOff = w.bodyOff - } - if len(b) < bodyOff { - return nil, errInvalidMessage - } - l := int(nativeEndian.Uint16(b[:2])) - if len(b) < l { - return nil, errInvalidMessage - } - m := &InterfaceAddrMessage{ - Version: int(b[2]), - Type: int(b[3]), - Flags: int(nativeEndian.Uint32(b[8:12])), - Index: int(nativeEndian.Uint16(b[12:14])), - raw: b[:l], - } - var err error - m.Addrs, err = parseAddrs(uint(nativeEndian.Uint32(b[4:8])), parseKernelInetAddr, b[bodyOff:]) - if err != nil { - return nil, err - } - return m, nil -} diff --git a/src/vendor/golang.org/x/net/route/interface_multicast.go b/src/vendor/golang.org/x/net/route/interface_multicast.go deleted file mode 100644 index 2ee37b9c749aef..00000000000000 --- a/src/vendor/golang.org/x/net/route/interface_multicast.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build darwin || dragonfly || freebsd - -package route - -func (w *wireFormat) parseInterfaceMulticastAddrMessage(_ RIBType, b []byte) (Message, error) { - if len(b) < w.bodyOff { - return nil, errMessageTooShort - } - l := int(nativeEndian.Uint16(b[:2])) - if len(b) < l { - return nil, errInvalidMessage - } - m := &InterfaceMulticastAddrMessage{ - Version: int(b[2]), - Type: int(b[3]), - Flags: int(nativeEndian.Uint32(b[8:12])), - Index: int(nativeEndian.Uint16(b[12:14])), - raw: b[:l], - } - var err error - m.Addrs, err = parseAddrs(uint(nativeEndian.Uint32(b[4:8])), parseKernelInetAddr, b[w.bodyOff:]) - if err != nil { - return nil, err - } - return m, nil -} diff --git a/src/vendor/golang.org/x/net/route/interface_openbsd.go b/src/vendor/golang.org/x/net/route/interface_openbsd.go deleted file mode 100644 index d369409a725227..00000000000000 --- a/src/vendor/golang.org/x/net/route/interface_openbsd.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package route - -import "syscall" - -func (*wireFormat) parseInterfaceMessage(_ RIBType, b []byte) (Message, error) { - if len(b) < 32 { - return nil, errMessageTooShort - } - l := int(nativeEndian.Uint16(b[:2])) - if len(b) < l { - return nil, errInvalidMessage - } - attrs := uint(nativeEndian.Uint32(b[12:16])) - if attrs&syscall.RTA_IFP == 0 { - return nil, nil - } - m := &InterfaceMessage{ - Version: int(b[2]), - Type: int(b[3]), - Flags: int(nativeEndian.Uint32(b[16:20])), - Index: int(nativeEndian.Uint16(b[6:8])), - Addrs: make([]Addr, syscall.RTAX_MAX), - raw: b[:l], - } - ll := int(nativeEndian.Uint16(b[4:6])) - if len(b) < ll { - return nil, errInvalidMessage - } - a, err := parseLinkAddr(b[ll:]) - if err != nil { - return nil, err - } - m.Addrs[syscall.RTAX_IFP] = a - m.Name = a.(*LinkAddr).Name - return m, nil -} - -func (*wireFormat) parseInterfaceAddrMessage(_ RIBType, b []byte) (Message, error) { - if len(b) < 24 { - return nil, errMessageTooShort - } - l := int(nativeEndian.Uint16(b[:2])) - if len(b) < l { - return nil, errInvalidMessage - } - bodyOff := int(nativeEndian.Uint16(b[4:6])) - if len(b) < bodyOff { - return nil, errInvalidMessage - } - m := &InterfaceAddrMessage{ - Version: int(b[2]), - Type: int(b[3]), - Flags: int(nativeEndian.Uint32(b[12:16])), - Index: int(nativeEndian.Uint16(b[6:8])), - raw: b[:l], - } - var err error - m.Addrs, err = parseAddrs(uint(nativeEndian.Uint32(b[12:16])), parseKernelInetAddr, b[bodyOff:]) - if err != nil { - return nil, err - } - return m, nil -} - -func (*wireFormat) parseInterfaceAnnounceMessage(_ RIBType, b []byte) (Message, error) { - if len(b) < 26 { - return nil, errMessageTooShort - } - l := int(nativeEndian.Uint16(b[:2])) - if len(b) < l { - return nil, errInvalidMessage - } - m := &InterfaceAnnounceMessage{ - Version: int(b[2]), - Type: int(b[3]), - Index: int(nativeEndian.Uint16(b[6:8])), - What: int(nativeEndian.Uint16(b[8:10])), - raw: b[:l], - } - for i := 0; i < 16; i++ { - if b[10+i] != 0 { - continue - } - m.Name = string(b[10 : 10+i]) - break - } - return m, nil -} diff --git a/src/vendor/golang.org/x/net/route/message.go b/src/vendor/golang.org/x/net/route/message.go deleted file mode 100644 index dc8bfc5b3aa728..00000000000000 --- a/src/vendor/golang.org/x/net/route/message.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build darwin || dragonfly || freebsd || netbsd || openbsd - -package route - -// A Message represents a routing message. -type Message interface { - // Sys returns operating system-specific information. - Sys() []Sys -} - -// A Sys reprensents operating system-specific information. -type Sys interface { - // SysType returns a type of operating system-specific - // information. - SysType() SysType -} - -// A SysType represents a type of operating system-specific -// information. -type SysType int - -const ( - SysMetrics SysType = iota - SysStats -) - -// ParseRIB parses b as a routing information base and returns a list -// of routing messages. -func ParseRIB(typ RIBType, b []byte) ([]Message, error) { - if !typ.parseable() { - return nil, errUnsupportedMessage - } - var msgs []Message - nmsgs, nskips := 0, 0 - for len(b) > 4 { - nmsgs++ - l := int(nativeEndian.Uint16(b[:2])) - if l == 0 { - return nil, errInvalidMessage - } - if len(b) < l { - return nil, errMessageTooShort - } - if b[2] != rtmVersion { - b = b[l:] - continue - } - if w, ok := wireFormats[int(b[3])]; !ok { - nskips++ - } else { - m, err := w.parse(typ, b[:l]) - if err != nil { - return nil, err - } - if m == nil { - nskips++ - } else { - msgs = append(msgs, m) - } - } - b = b[l:] - } - // We failed to parse any of the messages - version mismatch? - if nmsgs != len(msgs)+nskips { - return nil, errMessageMismatch - } - return msgs, nil -} diff --git a/src/vendor/golang.org/x/net/route/route.go b/src/vendor/golang.org/x/net/route/route.go deleted file mode 100644 index ca2ce2b88764cb..00000000000000 --- a/src/vendor/golang.org/x/net/route/route.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build darwin || dragonfly || freebsd || netbsd || openbsd - -// Package route provides basic functions for the manipulation of -// packet routing facilities on BSD variants. -// -// The package supports any version of Darwin, any version of -// DragonFly BSD, FreeBSD 7 and above, NetBSD 6 and above, and OpenBSD -// 5.6 and above. -package route - -import ( - "errors" - "os" - "syscall" -) - -var ( - errUnsupportedMessage = errors.New("unsupported message") - errMessageMismatch = errors.New("message mismatch") - errMessageTooShort = errors.New("message too short") - errInvalidMessage = errors.New("invalid message") - errInvalidAddr = errors.New("invalid address") - errShortBuffer = errors.New("short buffer") -) - -// A RouteMessage represents a message conveying an address prefix, a -// nexthop address and an output interface. -// -// Unlike other messages, this message can be used to query adjacency -// information for the given address prefix, to add a new route, and -// to delete or modify the existing route from the routing information -// base inside the kernel by writing and reading route messages on a -// routing socket. -// -// For the manipulation of routing information, the route message must -// contain appropriate fields that include: -// -// Version = -// Type = -// Flags = -// Index = -// ID = -// Seq = -// Addrs = -// -// The Type field specifies a type of manipulation, the Flags field -// specifies a class of target information and the Addrs field -// specifies target information like the following: -// -// route.RouteMessage{ -// Version: RTM_VERSION, -// Type: RTM_GET, -// Flags: RTF_UP | RTF_HOST, -// ID: uintptr(os.Getpid()), -// Seq: 1, -// Addrs: []route.Addrs{ -// RTAX_DST: &route.Inet4Addr{ ... }, -// RTAX_IFP: &route.LinkAddr{ ... }, -// RTAX_BRD: &route.Inet4Addr{ ... }, -// }, -// } -// -// The values for the above fields depend on the implementation of -// each operating system. -// -// The Err field on a response message contains an error value on the -// requested operation. If non-nil, the requested operation is failed. -type RouteMessage struct { - Version int // message version - Type int // message type - Flags int // route flags - Index int // interface index when attached - ID uintptr // sender's identifier; usually process ID - Seq int // sequence number - Err error // error on requested operation - Addrs []Addr // addresses - - extOff int // offset of header extension - raw []byte // raw message -} - -// Marshal returns the binary encoding of m. -func (m *RouteMessage) Marshal() ([]byte, error) { - return m.marshal() -} - -// A RIBType represents a type of routing information base. -type RIBType int - -const ( - RIBTypeRoute RIBType = syscall.NET_RT_DUMP - RIBTypeInterface RIBType = syscall.NET_RT_IFLIST -) - -// FetchRIB fetches a routing information base from the operating -// system. -// -// The provided af must be an address family. -// -// The provided arg must be a RIBType-specific argument. -// When RIBType is related to routes, arg might be a set of route -// flags. When RIBType is related to network interfaces, arg might be -// an interface index or a set of interface flags. In most cases, zero -// means a wildcard. -func FetchRIB(af int, typ RIBType, arg int) ([]byte, error) { - try := 0 - for { - try++ - mib := [6]int32{syscall.CTL_NET, syscall.AF_ROUTE, 0, int32(af), int32(typ), int32(arg)} - n := uintptr(0) - if err := sysctl(mib[:], nil, &n, nil, 0); err != nil { - return nil, os.NewSyscallError("sysctl", err) - } - if n == 0 { - return nil, nil - } - b := make([]byte, n) - if err := sysctl(mib[:], &b[0], &n, nil, 0); err != nil { - // If the sysctl failed because the data got larger - // between the two sysctl calls, try a few times - // before failing. (golang.org/issue/45736). - const maxTries = 3 - if err == syscall.ENOMEM && try < maxTries { - continue - } - return nil, os.NewSyscallError("sysctl", err) - } - return b[:n], nil - } -} diff --git a/src/vendor/golang.org/x/net/route/route_classic.go b/src/vendor/golang.org/x/net/route/route_classic.go deleted file mode 100644 index e273fe39ab32e1..00000000000000 --- a/src/vendor/golang.org/x/net/route/route_classic.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build darwin || dragonfly || freebsd || netbsd - -package route - -import ( - "runtime" - "syscall" -) - -func (m *RouteMessage) marshal() ([]byte, error) { - w, ok := wireFormats[m.Type] - if !ok { - return nil, errUnsupportedMessage - } - l := w.bodyOff + addrsSpace(m.Addrs) - if runtime.GOOS == "darwin" || runtime.GOOS == "ios" { - // Fix stray pointer writes on macOS. - // See golang.org/issue/22456. - l += 1024 - } - b := make([]byte, l) - nativeEndian.PutUint16(b[:2], uint16(l)) - if m.Version == 0 { - b[2] = rtmVersion - } else { - b[2] = byte(m.Version) - } - b[3] = byte(m.Type) - nativeEndian.PutUint32(b[8:12], uint32(m.Flags)) - nativeEndian.PutUint16(b[4:6], uint16(m.Index)) - nativeEndian.PutUint32(b[16:20], uint32(m.ID)) - nativeEndian.PutUint32(b[20:24], uint32(m.Seq)) - attrs, err := marshalAddrs(b[w.bodyOff:], m.Addrs) - if err != nil { - return nil, err - } - if attrs > 0 { - nativeEndian.PutUint32(b[12:16], uint32(attrs)) - } - return b, nil -} - -func (w *wireFormat) parseRouteMessage(typ RIBType, b []byte) (Message, error) { - if len(b) < w.bodyOff { - return nil, errMessageTooShort - } - l := int(nativeEndian.Uint16(b[:2])) - if len(b) < l { - return nil, errInvalidMessage - } - m := &RouteMessage{ - Version: int(b[2]), - Type: int(b[3]), - Flags: int(nativeEndian.Uint32(b[8:12])), - Index: int(nativeEndian.Uint16(b[4:6])), - ID: uintptr(nativeEndian.Uint32(b[16:20])), - Seq: int(nativeEndian.Uint32(b[20:24])), - extOff: w.extOff, - raw: b[:l], - } - errno := syscall.Errno(nativeEndian.Uint32(b[28:32])) - if errno != 0 { - m.Err = errno - } - var err error - m.Addrs, err = parseAddrs(uint(nativeEndian.Uint32(b[12:16])), parseKernelInetAddr, b[w.bodyOff:]) - if err != nil { - return nil, err - } - return m, nil -} diff --git a/src/vendor/golang.org/x/net/route/route_openbsd.go b/src/vendor/golang.org/x/net/route/route_openbsd.go deleted file mode 100644 index f848fb1f2453ae..00000000000000 --- a/src/vendor/golang.org/x/net/route/route_openbsd.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package route - -import ( - "syscall" -) - -func (m *RouteMessage) marshal() ([]byte, error) { - l := sizeofRtMsghdr + addrsSpace(m.Addrs) - b := make([]byte, l) - nativeEndian.PutUint16(b[:2], uint16(l)) - if m.Version == 0 { - b[2] = syscall.RTM_VERSION - } else { - b[2] = byte(m.Version) - } - b[3] = byte(m.Type) - nativeEndian.PutUint16(b[4:6], uint16(sizeofRtMsghdr)) - nativeEndian.PutUint32(b[16:20], uint32(m.Flags)) - nativeEndian.PutUint16(b[6:8], uint16(m.Index)) - nativeEndian.PutUint32(b[24:28], uint32(m.ID)) - nativeEndian.PutUint32(b[28:32], uint32(m.Seq)) - attrs, err := marshalAddrs(b[sizeofRtMsghdr:], m.Addrs) - if err != nil { - return nil, err - } - if attrs > 0 { - nativeEndian.PutUint32(b[12:16], uint32(attrs)) - } - return b, nil -} - -func (*wireFormat) parseRouteMessage(_ RIBType, b []byte) (Message, error) { - if len(b) < sizeofRtMsghdr { - return nil, errMessageTooShort - } - l := int(nativeEndian.Uint16(b[:2])) - if len(b) < l { - return nil, errInvalidMessage - } - m := &RouteMessage{ - Version: int(b[2]), - Type: int(b[3]), - Flags: int(nativeEndian.Uint32(b[16:20])), - Index: int(nativeEndian.Uint16(b[6:8])), - ID: uintptr(nativeEndian.Uint32(b[24:28])), - Seq: int(nativeEndian.Uint32(b[28:32])), - raw: b[:l], - } - ll := int(nativeEndian.Uint16(b[4:6])) - if len(b) < ll { - return nil, errInvalidMessage - } - errno := syscall.Errno(nativeEndian.Uint32(b[32:36])) - if errno != 0 { - m.Err = errno - } - as, err := parseAddrs(uint(nativeEndian.Uint32(b[12:16])), parseKernelInetAddr, b[ll:]) - if err != nil { - return nil, err - } - m.Addrs = as - return m, nil -} diff --git a/src/vendor/golang.org/x/net/route/sys_darwin.go b/src/vendor/golang.org/x/net/route/sys_darwin.go deleted file mode 100644 index c8c4eecb8e9c6d..00000000000000 --- a/src/vendor/golang.org/x/net/route/sys_darwin.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package route - -import "syscall" - -func (typ RIBType) parseable() bool { - switch typ { - case syscall.NET_RT_STAT, syscall.NET_RT_TRASH: - return false - default: - return true - } -} - -// RouteMetrics represents route metrics. -type RouteMetrics struct { - PathMTU int // path maximum transmission unit -} - -// SysType implements the SysType method of Sys interface. -func (rmx *RouteMetrics) SysType() SysType { return SysMetrics } - -// Sys implements the Sys method of Message interface. -func (m *RouteMessage) Sys() []Sys { - return []Sys{ - &RouteMetrics{ - PathMTU: int(nativeEndian.Uint32(m.raw[m.extOff+4 : m.extOff+8])), - }, - } -} - -// InterfaceMetrics represents interface metrics. -type InterfaceMetrics struct { - Type int // interface type - MTU int // maximum transmission unit -} - -// SysType implements the SysType method of Sys interface. -func (imx *InterfaceMetrics) SysType() SysType { return SysMetrics } - -// Sys implements the Sys method of Message interface. -func (m *InterfaceMessage) Sys() []Sys { - return []Sys{ - &InterfaceMetrics{ - Type: int(m.raw[m.extOff]), - MTU: int(nativeEndian.Uint32(m.raw[m.extOff+8 : m.extOff+12])), - }, - } -} - -func probeRoutingStack() (int, map[int]*wireFormat) { - rtm := &wireFormat{extOff: 36, bodyOff: sizeofRtMsghdrDarwin15} - rtm.parse = rtm.parseRouteMessage - rtm2 := &wireFormat{extOff: 36, bodyOff: sizeofRtMsghdr2Darwin15} - rtm2.parse = rtm2.parseRouteMessage - ifm := &wireFormat{extOff: 16, bodyOff: sizeofIfMsghdrDarwin15} - ifm.parse = ifm.parseInterfaceMessage - ifm2 := &wireFormat{extOff: 32, bodyOff: sizeofIfMsghdr2Darwin15} - ifm2.parse = ifm2.parseInterfaceMessage - ifam := &wireFormat{extOff: sizeofIfaMsghdrDarwin15, bodyOff: sizeofIfaMsghdrDarwin15} - ifam.parse = ifam.parseInterfaceAddrMessage - ifmam := &wireFormat{extOff: sizeofIfmaMsghdrDarwin15, bodyOff: sizeofIfmaMsghdrDarwin15} - ifmam.parse = ifmam.parseInterfaceMulticastAddrMessage - ifmam2 := &wireFormat{extOff: sizeofIfmaMsghdr2Darwin15, bodyOff: sizeofIfmaMsghdr2Darwin15} - ifmam2.parse = ifmam2.parseInterfaceMulticastAddrMessage - // Darwin kernels require 32-bit aligned access to routing facilities. - return 4, map[int]*wireFormat{ - syscall.RTM_ADD: rtm, - syscall.RTM_DELETE: rtm, - syscall.RTM_CHANGE: rtm, - syscall.RTM_GET: rtm, - syscall.RTM_LOSING: rtm, - syscall.RTM_REDIRECT: rtm, - syscall.RTM_MISS: rtm, - syscall.RTM_LOCK: rtm, - syscall.RTM_RESOLVE: rtm, - syscall.RTM_NEWADDR: ifam, - syscall.RTM_DELADDR: ifam, - syscall.RTM_IFINFO: ifm, - syscall.RTM_NEWMADDR: ifmam, - syscall.RTM_DELMADDR: ifmam, - syscall.RTM_IFINFO2: ifm2, - syscall.RTM_NEWMADDR2: ifmam2, - syscall.RTM_GET2: rtm2, - } -} diff --git a/src/vendor/golang.org/x/net/route/sys_dragonfly.go b/src/vendor/golang.org/x/net/route/sys_dragonfly.go deleted file mode 100644 index 577fb16eb4bb53..00000000000000 --- a/src/vendor/golang.org/x/net/route/sys_dragonfly.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package route - -import ( - "syscall" - "unsafe" -) - -func (typ RIBType) parseable() bool { return true } - -// RouteMetrics represents route metrics. -type RouteMetrics struct { - PathMTU int // path maximum transmission unit -} - -// SysType implements the SysType method of Sys interface. -func (rmx *RouteMetrics) SysType() SysType { return SysMetrics } - -// Sys implements the Sys method of Message interface. -func (m *RouteMessage) Sys() []Sys { - return []Sys{ - &RouteMetrics{ - PathMTU: int(nativeEndian.Uint64(m.raw[m.extOff+8 : m.extOff+16])), - }, - } -} - -// InterfaceMetrics represents interface metrics. -type InterfaceMetrics struct { - Type int // interface type - MTU int // maximum transmission unit -} - -// SysType implements the SysType method of Sys interface. -func (imx *InterfaceMetrics) SysType() SysType { return SysMetrics } - -// Sys implements the Sys method of Message interface. -func (m *InterfaceMessage) Sys() []Sys { - return []Sys{ - &InterfaceMetrics{ - Type: int(m.raw[m.extOff]), - MTU: int(nativeEndian.Uint32(m.raw[m.extOff+8 : m.extOff+12])), - }, - } -} - -func probeRoutingStack() (int, map[int]*wireFormat) { - var p uintptr - rtm := &wireFormat{extOff: 40, bodyOff: sizeofRtMsghdrDragonFlyBSD4} - rtm.parse = rtm.parseRouteMessage - ifm := &wireFormat{extOff: 16, bodyOff: sizeofIfMsghdrDragonFlyBSD4} - ifm.parse = ifm.parseInterfaceMessage - ifam := &wireFormat{extOff: sizeofIfaMsghdrDragonFlyBSD4, bodyOff: sizeofIfaMsghdrDragonFlyBSD4} - ifam.parse = ifam.parseInterfaceAddrMessage - ifmam := &wireFormat{extOff: sizeofIfmaMsghdrDragonFlyBSD4, bodyOff: sizeofIfmaMsghdrDragonFlyBSD4} - ifmam.parse = ifmam.parseInterfaceMulticastAddrMessage - ifanm := &wireFormat{extOff: sizeofIfAnnouncemsghdrDragonFlyBSD4, bodyOff: sizeofIfAnnouncemsghdrDragonFlyBSD4} - ifanm.parse = ifanm.parseInterfaceAnnounceMessage - - rel, _ := syscall.SysctlUint32("kern.osreldate") - if rel >= 500705 { - // https://github.com/DragonFlyBSD/DragonFlyBSD/commit/43a373152df2d405c9940983e584e6a25e76632d - // but only the size of struct ifa_msghdr actually changed - rtmVersion = 7 - ifam.bodyOff = sizeofIfaMsghdrDragonFlyBSD58 - } - - return int(unsafe.Sizeof(p)), map[int]*wireFormat{ - syscall.RTM_ADD: rtm, - syscall.RTM_DELETE: rtm, - syscall.RTM_CHANGE: rtm, - syscall.RTM_GET: rtm, - syscall.RTM_LOSING: rtm, - syscall.RTM_REDIRECT: rtm, - syscall.RTM_MISS: rtm, - syscall.RTM_LOCK: rtm, - syscall.RTM_RESOLVE: rtm, - syscall.RTM_NEWADDR: ifam, - syscall.RTM_DELADDR: ifam, - syscall.RTM_IFINFO: ifm, - syscall.RTM_NEWMADDR: ifmam, - syscall.RTM_DELMADDR: ifmam, - syscall.RTM_IFANNOUNCE: ifanm, - } -} diff --git a/src/vendor/golang.org/x/net/route/sys_freebsd.go b/src/vendor/golang.org/x/net/route/sys_freebsd.go deleted file mode 100644 index 0a66dcedb386d9..00000000000000 --- a/src/vendor/golang.org/x/net/route/sys_freebsd.go +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package route - -import ( - "syscall" - "unsafe" -) - -func (typ RIBType) parseable() bool { return true } - -// RouteMetrics represents route metrics. -type RouteMetrics struct { - PathMTU int // path maximum transmission unit -} - -// SysType implements the SysType method of Sys interface. -func (rmx *RouteMetrics) SysType() SysType { return SysMetrics } - -// Sys implements the Sys method of Message interface. -func (m *RouteMessage) Sys() []Sys { - if kernelAlign == 8 { - return []Sys{ - &RouteMetrics{ - PathMTU: int(nativeEndian.Uint64(m.raw[m.extOff+8 : m.extOff+16])), - }, - } - } - return []Sys{ - &RouteMetrics{ - PathMTU: int(nativeEndian.Uint32(m.raw[m.extOff+4 : m.extOff+8])), - }, - } -} - -// InterfaceMetrics represents interface metrics. -type InterfaceMetrics struct { - Type int // interface type - MTU int // maximum transmission unit -} - -// SysType implements the SysType method of Sys interface. -func (imx *InterfaceMetrics) SysType() SysType { return SysMetrics } - -// Sys implements the Sys method of Message interface. -func (m *InterfaceMessage) Sys() []Sys { - return []Sys{ - &InterfaceMetrics{ - Type: int(m.raw[m.extOff]), - MTU: int(nativeEndian.Uint32(m.raw[m.extOff+8 : m.extOff+12])), - }, - } -} - -var compatFreeBSD32 bool // 386 emulation on amd64 - -func probeRoutingStack() (int, map[int]*wireFormat) { - var p uintptr - wordSize := int(unsafe.Sizeof(p)) - align := wordSize - // In the case of kern.supported_archs="amd64 i386", we need - // to know the underlying kernel's architecture because the - // alignment for routing facilities are set at the build time - // of the kernel. - conf, _ := syscall.Sysctl("kern.conftxt") - for i, j := 0, 0; j < len(conf); j++ { - if conf[j] != '\n' { - continue - } - s := conf[i:j] - i = j + 1 - if len(s) > len("machine") && s[:len("machine")] == "machine" { - s = s[len("machine"):] - for k := 0; k < len(s); k++ { - if s[k] == ' ' || s[k] == '\t' { - s = s[1:] - } - break - } - if s == "amd64" { - align = 8 - } - break - } - } - if align != wordSize { - compatFreeBSD32 = true // 386 emulation on amd64 - } - var rtm, ifm, ifam, ifmam, ifanm *wireFormat - if compatFreeBSD32 { - rtm = &wireFormat{extOff: sizeofRtMsghdrFreeBSD10Emu - sizeofRtMetricsFreeBSD10Emu, bodyOff: sizeofRtMsghdrFreeBSD10Emu} - ifm = &wireFormat{extOff: 16} - ifam = &wireFormat{extOff: sizeofIfaMsghdrFreeBSD10Emu, bodyOff: sizeofIfaMsghdrFreeBSD10Emu} - ifmam = &wireFormat{extOff: sizeofIfmaMsghdrFreeBSD10Emu, bodyOff: sizeofIfmaMsghdrFreeBSD10Emu} - ifanm = &wireFormat{extOff: sizeofIfAnnouncemsghdrFreeBSD10Emu, bodyOff: sizeofIfAnnouncemsghdrFreeBSD10Emu} - } else { - rtm = &wireFormat{extOff: sizeofRtMsghdrFreeBSD10 - sizeofRtMetricsFreeBSD10, bodyOff: sizeofRtMsghdrFreeBSD10} - ifm = &wireFormat{extOff: 16} - ifam = &wireFormat{extOff: sizeofIfaMsghdrFreeBSD10, bodyOff: sizeofIfaMsghdrFreeBSD10} - ifmam = &wireFormat{extOff: sizeofIfmaMsghdrFreeBSD10, bodyOff: sizeofIfmaMsghdrFreeBSD10} - ifanm = &wireFormat{extOff: sizeofIfAnnouncemsghdrFreeBSD10, bodyOff: sizeofIfAnnouncemsghdrFreeBSD10} - } - rel, _ := syscall.SysctlUint32("kern.osreldate") - switch { - case rel < 800000: - if compatFreeBSD32 { - ifm.bodyOff = sizeofIfMsghdrFreeBSD7Emu - } else { - ifm.bodyOff = sizeofIfMsghdrFreeBSD7 - } - case 800000 <= rel && rel < 900000: - if compatFreeBSD32 { - ifm.bodyOff = sizeofIfMsghdrFreeBSD8Emu - } else { - ifm.bodyOff = sizeofIfMsghdrFreeBSD8 - } - case 900000 <= rel && rel < 1000000: - if compatFreeBSD32 { - ifm.bodyOff = sizeofIfMsghdrFreeBSD9Emu - } else { - ifm.bodyOff = sizeofIfMsghdrFreeBSD9 - } - case 1000000 <= rel && rel < 1100000: - if compatFreeBSD32 { - ifm.bodyOff = sizeofIfMsghdrFreeBSD10Emu - } else { - ifm.bodyOff = sizeofIfMsghdrFreeBSD10 - } - default: - if compatFreeBSD32 { - ifm.bodyOff = sizeofIfMsghdrFreeBSD11Emu - } else { - ifm.bodyOff = sizeofIfMsghdrFreeBSD11 - } - } - rtm.parse = rtm.parseRouteMessage - ifm.parse = ifm.parseInterfaceMessage - ifam.parse = ifam.parseInterfaceAddrMessage - ifmam.parse = ifmam.parseInterfaceMulticastAddrMessage - ifanm.parse = ifanm.parseInterfaceAnnounceMessage - return align, map[int]*wireFormat{ - syscall.RTM_ADD: rtm, - syscall.RTM_DELETE: rtm, - syscall.RTM_CHANGE: rtm, - syscall.RTM_GET: rtm, - syscall.RTM_LOSING: rtm, - syscall.RTM_REDIRECT: rtm, - syscall.RTM_MISS: rtm, - syscall.RTM_LOCK: rtm, - syscall.RTM_RESOLVE: rtm, - syscall.RTM_NEWADDR: ifam, - syscall.RTM_DELADDR: ifam, - syscall.RTM_IFINFO: ifm, - syscall.RTM_NEWMADDR: ifmam, - syscall.RTM_DELMADDR: ifmam, - syscall.RTM_IFANNOUNCE: ifanm, - } -} diff --git a/src/vendor/golang.org/x/net/route/sys_netbsd.go b/src/vendor/golang.org/x/net/route/sys_netbsd.go deleted file mode 100644 index be4460e13f044c..00000000000000 --- a/src/vendor/golang.org/x/net/route/sys_netbsd.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package route - -import "syscall" - -func (typ RIBType) parseable() bool { return true } - -// RouteMetrics represents route metrics. -type RouteMetrics struct { - PathMTU int // path maximum transmission unit -} - -// SysType implements the SysType method of Sys interface. -func (rmx *RouteMetrics) SysType() SysType { return SysMetrics } - -// Sys implements the Sys method of Message interface. -func (m *RouteMessage) Sys() []Sys { - return []Sys{ - &RouteMetrics{ - PathMTU: int(nativeEndian.Uint64(m.raw[m.extOff+8 : m.extOff+16])), - }, - } -} - -// RouteMetrics represents route metrics. -type InterfaceMetrics struct { - Type int // interface type - MTU int // maximum transmission unit -} - -// SysType implements the SysType method of Sys interface. -func (imx *InterfaceMetrics) SysType() SysType { return SysMetrics } - -// Sys implements the Sys method of Message interface. -func (m *InterfaceMessage) Sys() []Sys { - return []Sys{ - &InterfaceMetrics{ - Type: int(m.raw[m.extOff]), - MTU: int(nativeEndian.Uint32(m.raw[m.extOff+8 : m.extOff+12])), - }, - } -} - -func probeRoutingStack() (int, map[int]*wireFormat) { - rtm := &wireFormat{extOff: 40, bodyOff: sizeofRtMsghdrNetBSD7} - rtm.parse = rtm.parseRouteMessage - ifm := &wireFormat{extOff: 16, bodyOff: sizeofIfMsghdrNetBSD7} - ifm.parse = ifm.parseInterfaceMessage - ifam := &wireFormat{extOff: sizeofIfaMsghdrNetBSD7, bodyOff: sizeofIfaMsghdrNetBSD7} - ifam.parse = ifam.parseInterfaceAddrMessage - ifanm := &wireFormat{extOff: sizeofIfAnnouncemsghdrNetBSD7, bodyOff: sizeofIfAnnouncemsghdrNetBSD7} - ifanm.parse = ifanm.parseInterfaceAnnounceMessage - // NetBSD 6 and above kernels require 64-bit aligned access to - // routing facilities. - return 8, map[int]*wireFormat{ - syscall.RTM_ADD: rtm, - syscall.RTM_DELETE: rtm, - syscall.RTM_CHANGE: rtm, - syscall.RTM_GET: rtm, - syscall.RTM_LOSING: rtm, - syscall.RTM_REDIRECT: rtm, - syscall.RTM_MISS: rtm, - syscall.RTM_LOCK: rtm, - syscall.RTM_RESOLVE: rtm, - syscall.RTM_NEWADDR: ifam, - syscall.RTM_DELADDR: ifam, - syscall.RTM_IFANNOUNCE: ifanm, - syscall.RTM_IFINFO: ifm, - } -} diff --git a/src/vendor/golang.org/x/net/route/sys_openbsd.go b/src/vendor/golang.org/x/net/route/sys_openbsd.go deleted file mode 100644 index 7f4f93cbea088d..00000000000000 --- a/src/vendor/golang.org/x/net/route/sys_openbsd.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package route - -import ( - "syscall" - "unsafe" -) - -func (typ RIBType) parseable() bool { - switch typ { - case syscall.NET_RT_STATS, syscall.NET_RT_TABLE: - return false - default: - return true - } -} - -// RouteMetrics represents route metrics. -type RouteMetrics struct { - PathMTU int // path maximum transmission unit -} - -// SysType implements the SysType method of Sys interface. -func (rmx *RouteMetrics) SysType() SysType { return SysMetrics } - -// Sys implements the Sys method of Message interface. -func (m *RouteMessage) Sys() []Sys { - return []Sys{ - &RouteMetrics{ - PathMTU: int(nativeEndian.Uint32(m.raw[60:64])), - }, - } -} - -// InterfaceMetrics represents interface metrics. -type InterfaceMetrics struct { - Type int // interface type - MTU int // maximum transmission unit -} - -// SysType implements the SysType method of Sys interface. -func (imx *InterfaceMetrics) SysType() SysType { return SysMetrics } - -// Sys implements the Sys method of Message interface. -func (m *InterfaceMessage) Sys() []Sys { - return []Sys{ - &InterfaceMetrics{ - Type: int(m.raw[24]), - MTU: int(nativeEndian.Uint32(m.raw[28:32])), - }, - } -} - -func probeRoutingStack() (int, map[int]*wireFormat) { - var p uintptr - rtm := &wireFormat{extOff: -1, bodyOff: -1} - rtm.parse = rtm.parseRouteMessage - ifm := &wireFormat{extOff: -1, bodyOff: -1} - ifm.parse = ifm.parseInterfaceMessage - ifam := &wireFormat{extOff: -1, bodyOff: -1} - ifam.parse = ifam.parseInterfaceAddrMessage - ifanm := &wireFormat{extOff: -1, bodyOff: -1} - ifanm.parse = ifanm.parseInterfaceAnnounceMessage - return int(unsafe.Sizeof(p)), map[int]*wireFormat{ - syscall.RTM_ADD: rtm, - syscall.RTM_DELETE: rtm, - syscall.RTM_CHANGE: rtm, - syscall.RTM_GET: rtm, - syscall.RTM_LOSING: rtm, - syscall.RTM_REDIRECT: rtm, - syscall.RTM_MISS: rtm, - syscall.RTM_RESOLVE: rtm, - syscall.RTM_NEWADDR: ifam, - syscall.RTM_DELADDR: ifam, - syscall.RTM_IFINFO: ifm, - syscall.RTM_IFANNOUNCE: ifanm, - syscall.RTM_DESYNC: rtm, - } -} diff --git a/src/vendor/golang.org/x/net/route/syscall.go b/src/vendor/golang.org/x/net/route/syscall.go deleted file mode 100644 index 0ed53750a22f8a..00000000000000 --- a/src/vendor/golang.org/x/net/route/syscall.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build darwin || dragonfly || freebsd || netbsd || openbsd - -package route - -import _ "unsafe" // for linkname - -//go:linkname sysctl syscall.sysctl -func sysctl(mib []int32, old *byte, oldlen *uintptr, new *byte, newlen uintptr) error diff --git a/src/vendor/golang.org/x/net/route/zsys_darwin.go b/src/vendor/golang.org/x/net/route/zsys_darwin.go deleted file mode 100644 index 56a0c66f44f202..00000000000000 --- a/src/vendor/golang.org/x/net/route/zsys_darwin.go +++ /dev/null @@ -1,22 +0,0 @@ -// Code generated by cmd/cgo -godefs; DO NOT EDIT. -// cgo -godefs defs_darwin.go - -package route - -const ( - sizeofIfMsghdrDarwin15 = 0x70 - sizeofIfaMsghdrDarwin15 = 0x14 - sizeofIfmaMsghdrDarwin15 = 0x10 - sizeofIfMsghdr2Darwin15 = 0xa0 - sizeofIfmaMsghdr2Darwin15 = 0x14 - sizeofIfDataDarwin15 = 0x60 - sizeofIfData64Darwin15 = 0x80 - - sizeofRtMsghdrDarwin15 = 0x5c - sizeofRtMsghdr2Darwin15 = 0x5c - sizeofRtMetricsDarwin15 = 0x38 - - sizeofSockaddrStorage = 0x80 - sizeofSockaddrInet = 0x10 - sizeofSockaddrInet6 = 0x1c -) diff --git a/src/vendor/golang.org/x/net/route/zsys_dragonfly.go b/src/vendor/golang.org/x/net/route/zsys_dragonfly.go deleted file mode 100644 index f7c7a60cd64220..00000000000000 --- a/src/vendor/golang.org/x/net/route/zsys_dragonfly.go +++ /dev/null @@ -1,20 +0,0 @@ -// Code generated by cmd/cgo -godefs; DO NOT EDIT. -// cgo -godefs defs_dragonfly.go - -package route - -const ( - sizeofIfMsghdrDragonFlyBSD4 = 0xb0 - sizeofIfaMsghdrDragonFlyBSD4 = 0x14 - sizeofIfmaMsghdrDragonFlyBSD4 = 0x10 - sizeofIfAnnouncemsghdrDragonFlyBSD4 = 0x18 - - sizeofIfaMsghdrDragonFlyBSD58 = 0x18 - - sizeofRtMsghdrDragonFlyBSD4 = 0x98 - sizeofRtMetricsDragonFlyBSD4 = 0x70 - - sizeofSockaddrStorage = 0x80 - sizeofSockaddrInet = 0x10 - sizeofSockaddrInet6 = 0x1c -) diff --git a/src/vendor/golang.org/x/net/route/zsys_freebsd_386.go b/src/vendor/golang.org/x/net/route/zsys_freebsd_386.go deleted file mode 100644 index 3f985c7ee90c1d..00000000000000 --- a/src/vendor/golang.org/x/net/route/zsys_freebsd_386.go +++ /dev/null @@ -1,55 +0,0 @@ -// Code generated by cmd/cgo -godefs; DO NOT EDIT. -// cgo -godefs defs_freebsd.go - -package route - -const ( - sizeofIfMsghdrlFreeBSD10 = 0x68 - sizeofIfaMsghdrFreeBSD10 = 0x14 - sizeofIfaMsghdrlFreeBSD10 = 0x6c - sizeofIfmaMsghdrFreeBSD10 = 0x10 - sizeofIfAnnouncemsghdrFreeBSD10 = 0x18 - - sizeofRtMsghdrFreeBSD10 = 0x5c - sizeofRtMetricsFreeBSD10 = 0x38 - - sizeofIfMsghdrFreeBSD7 = 0x60 - sizeofIfMsghdrFreeBSD8 = 0x60 - sizeofIfMsghdrFreeBSD9 = 0x60 - sizeofIfMsghdrFreeBSD10 = 0x64 - sizeofIfMsghdrFreeBSD11 = 0xa8 - - sizeofIfDataFreeBSD7 = 0x50 - sizeofIfDataFreeBSD8 = 0x50 - sizeofIfDataFreeBSD9 = 0x50 - sizeofIfDataFreeBSD10 = 0x54 - sizeofIfDataFreeBSD11 = 0x98 - - // MODIFIED BY HAND FOR 386 EMULATION ON AMD64 - // 386 EMULATION USES THE UNDERLYING RAW DATA LAYOUT - - sizeofIfMsghdrlFreeBSD10Emu = 0xb0 - sizeofIfaMsghdrFreeBSD10Emu = 0x14 - sizeofIfaMsghdrlFreeBSD10Emu = 0xb0 - sizeofIfmaMsghdrFreeBSD10Emu = 0x10 - sizeofIfAnnouncemsghdrFreeBSD10Emu = 0x18 - - sizeofRtMsghdrFreeBSD10Emu = 0x98 - sizeofRtMetricsFreeBSD10Emu = 0x70 - - sizeofIfMsghdrFreeBSD7Emu = 0xa8 - sizeofIfMsghdrFreeBSD8Emu = 0xa8 - sizeofIfMsghdrFreeBSD9Emu = 0xa8 - sizeofIfMsghdrFreeBSD10Emu = 0xa8 - sizeofIfMsghdrFreeBSD11Emu = 0xa8 - - sizeofIfDataFreeBSD7Emu = 0x98 - sizeofIfDataFreeBSD8Emu = 0x98 - sizeofIfDataFreeBSD9Emu = 0x98 - sizeofIfDataFreeBSD10Emu = 0x98 - sizeofIfDataFreeBSD11Emu = 0x98 - - sizeofSockaddrStorage = 0x80 - sizeofSockaddrInet = 0x10 - sizeofSockaddrInet6 = 0x1c -) diff --git a/src/vendor/golang.org/x/net/route/zsys_freebsd_amd64.go b/src/vendor/golang.org/x/net/route/zsys_freebsd_amd64.go deleted file mode 100644 index 929339369829dc..00000000000000 --- a/src/vendor/golang.org/x/net/route/zsys_freebsd_amd64.go +++ /dev/null @@ -1,52 +0,0 @@ -// Code generated by cmd/cgo -godefs; DO NOT EDIT. -// cgo -godefs defs_freebsd.go - -package route - -const ( - sizeofIfMsghdrlFreeBSD10 = 0xb0 - sizeofIfaMsghdrFreeBSD10 = 0x14 - sizeofIfaMsghdrlFreeBSD10 = 0xb0 - sizeofIfmaMsghdrFreeBSD10 = 0x10 - sizeofIfAnnouncemsghdrFreeBSD10 = 0x18 - - sizeofRtMsghdrFreeBSD10 = 0x98 - sizeofRtMetricsFreeBSD10 = 0x70 - - sizeofIfMsghdrFreeBSD7 = 0xa8 - sizeofIfMsghdrFreeBSD8 = 0xa8 - sizeofIfMsghdrFreeBSD9 = 0xa8 - sizeofIfMsghdrFreeBSD10 = 0xa8 - sizeofIfMsghdrFreeBSD11 = 0xa8 - - sizeofIfDataFreeBSD7 = 0x98 - sizeofIfDataFreeBSD8 = 0x98 - sizeofIfDataFreeBSD9 = 0x98 - sizeofIfDataFreeBSD10 = 0x98 - sizeofIfDataFreeBSD11 = 0x98 - - sizeofIfMsghdrlFreeBSD10Emu = 0xb0 - sizeofIfaMsghdrFreeBSD10Emu = 0x14 - sizeofIfaMsghdrlFreeBSD10Emu = 0xb0 - sizeofIfmaMsghdrFreeBSD10Emu = 0x10 - sizeofIfAnnouncemsghdrFreeBSD10Emu = 0x18 - - sizeofRtMsghdrFreeBSD10Emu = 0x98 - sizeofRtMetricsFreeBSD10Emu = 0x70 - - sizeofIfMsghdrFreeBSD7Emu = 0xa8 - sizeofIfMsghdrFreeBSD8Emu = 0xa8 - sizeofIfMsghdrFreeBSD9Emu = 0xa8 - sizeofIfMsghdrFreeBSD10Emu = 0xa8 - sizeofIfMsghdrFreeBSD11Emu = 0xa8 - - sizeofIfDataFreeBSD7Emu = 0x98 - sizeofIfDataFreeBSD8Emu = 0x98 - sizeofIfDataFreeBSD9Emu = 0x98 - sizeofIfDataFreeBSD10Emu = 0x98 - sizeofIfDataFreeBSD11Emu = 0x98 - - sizeofSockaddrStorage = 0x80 - sizeofSockaddrInet = 0x10 - sizeofSockaddrInet6 = 0x1c -) diff --git a/src/vendor/golang.org/x/net/route/zsys_freebsd_arm.go b/src/vendor/golang.org/x/net/route/zsys_freebsd_arm.go deleted file mode 100644 index a2bdb4ad3b58ea..00000000000000 --- a/src/vendor/golang.org/x/net/route/zsys_freebsd_arm.go +++ /dev/null @@ -1,52 +0,0 @@ -// Code generated by cmd/cgo -godefs; DO NOT EDIT. -// cgo -godefs defs_freebsd.go - -package route - -const ( - sizeofIfMsghdrlFreeBSD10 = 0x68 - sizeofIfaMsghdrFreeBSD10 = 0x14 - sizeofIfaMsghdrlFreeBSD10 = 0x6c - sizeofIfmaMsghdrFreeBSD10 = 0x10 - sizeofIfAnnouncemsghdrFreeBSD10 = 0x18 - - sizeofRtMsghdrFreeBSD10 = 0x5c - sizeofRtMetricsFreeBSD10 = 0x38 - - sizeofIfMsghdrFreeBSD7 = 0x70 - sizeofIfMsghdrFreeBSD8 = 0x70 - sizeofIfMsghdrFreeBSD9 = 0x70 - sizeofIfMsghdrFreeBSD10 = 0x70 - sizeofIfMsghdrFreeBSD11 = 0xa8 - - sizeofIfDataFreeBSD7 = 0x60 - sizeofIfDataFreeBSD8 = 0x60 - sizeofIfDataFreeBSD9 = 0x60 - sizeofIfDataFreeBSD10 = 0x60 - sizeofIfDataFreeBSD11 = 0x98 - - sizeofIfMsghdrlFreeBSD10Emu = 0x68 - sizeofIfaMsghdrFreeBSD10Emu = 0x14 - sizeofIfaMsghdrlFreeBSD10Emu = 0x6c - sizeofIfmaMsghdrFreeBSD10Emu = 0x10 - sizeofIfAnnouncemsghdrFreeBSD10Emu = 0x18 - - sizeofRtMsghdrFreeBSD10Emu = 0x5c - sizeofRtMetricsFreeBSD10Emu = 0x38 - - sizeofIfMsghdrFreeBSD7Emu = 0x70 - sizeofIfMsghdrFreeBSD8Emu = 0x70 - sizeofIfMsghdrFreeBSD9Emu = 0x70 - sizeofIfMsghdrFreeBSD10Emu = 0x70 - sizeofIfMsghdrFreeBSD11Emu = 0xa8 - - sizeofIfDataFreeBSD7Emu = 0x60 - sizeofIfDataFreeBSD8Emu = 0x60 - sizeofIfDataFreeBSD9Emu = 0x60 - sizeofIfDataFreeBSD10Emu = 0x60 - sizeofIfDataFreeBSD11Emu = 0x98 - - sizeofSockaddrStorage = 0x80 - sizeofSockaddrInet = 0x10 - sizeofSockaddrInet6 = 0x1c -) diff --git a/src/vendor/golang.org/x/net/route/zsys_freebsd_arm64.go b/src/vendor/golang.org/x/net/route/zsys_freebsd_arm64.go deleted file mode 100644 index 929339369829dc..00000000000000 --- a/src/vendor/golang.org/x/net/route/zsys_freebsd_arm64.go +++ /dev/null @@ -1,52 +0,0 @@ -// Code generated by cmd/cgo -godefs; DO NOT EDIT. -// cgo -godefs defs_freebsd.go - -package route - -const ( - sizeofIfMsghdrlFreeBSD10 = 0xb0 - sizeofIfaMsghdrFreeBSD10 = 0x14 - sizeofIfaMsghdrlFreeBSD10 = 0xb0 - sizeofIfmaMsghdrFreeBSD10 = 0x10 - sizeofIfAnnouncemsghdrFreeBSD10 = 0x18 - - sizeofRtMsghdrFreeBSD10 = 0x98 - sizeofRtMetricsFreeBSD10 = 0x70 - - sizeofIfMsghdrFreeBSD7 = 0xa8 - sizeofIfMsghdrFreeBSD8 = 0xa8 - sizeofIfMsghdrFreeBSD9 = 0xa8 - sizeofIfMsghdrFreeBSD10 = 0xa8 - sizeofIfMsghdrFreeBSD11 = 0xa8 - - sizeofIfDataFreeBSD7 = 0x98 - sizeofIfDataFreeBSD8 = 0x98 - sizeofIfDataFreeBSD9 = 0x98 - sizeofIfDataFreeBSD10 = 0x98 - sizeofIfDataFreeBSD11 = 0x98 - - sizeofIfMsghdrlFreeBSD10Emu = 0xb0 - sizeofIfaMsghdrFreeBSD10Emu = 0x14 - sizeofIfaMsghdrlFreeBSD10Emu = 0xb0 - sizeofIfmaMsghdrFreeBSD10Emu = 0x10 - sizeofIfAnnouncemsghdrFreeBSD10Emu = 0x18 - - sizeofRtMsghdrFreeBSD10Emu = 0x98 - sizeofRtMetricsFreeBSD10Emu = 0x70 - - sizeofIfMsghdrFreeBSD7Emu = 0xa8 - sizeofIfMsghdrFreeBSD8Emu = 0xa8 - sizeofIfMsghdrFreeBSD9Emu = 0xa8 - sizeofIfMsghdrFreeBSD10Emu = 0xa8 - sizeofIfMsghdrFreeBSD11Emu = 0xa8 - - sizeofIfDataFreeBSD7Emu = 0x98 - sizeofIfDataFreeBSD8Emu = 0x98 - sizeofIfDataFreeBSD9Emu = 0x98 - sizeofIfDataFreeBSD10Emu = 0x98 - sizeofIfDataFreeBSD11Emu = 0x98 - - sizeofSockaddrStorage = 0x80 - sizeofSockaddrInet = 0x10 - sizeofSockaddrInet6 = 0x1c -) diff --git a/src/vendor/golang.org/x/net/route/zsys_freebsd_riscv64.go b/src/vendor/golang.org/x/net/route/zsys_freebsd_riscv64.go deleted file mode 100644 index 929339369829dc..00000000000000 --- a/src/vendor/golang.org/x/net/route/zsys_freebsd_riscv64.go +++ /dev/null @@ -1,52 +0,0 @@ -// Code generated by cmd/cgo -godefs; DO NOT EDIT. -// cgo -godefs defs_freebsd.go - -package route - -const ( - sizeofIfMsghdrlFreeBSD10 = 0xb0 - sizeofIfaMsghdrFreeBSD10 = 0x14 - sizeofIfaMsghdrlFreeBSD10 = 0xb0 - sizeofIfmaMsghdrFreeBSD10 = 0x10 - sizeofIfAnnouncemsghdrFreeBSD10 = 0x18 - - sizeofRtMsghdrFreeBSD10 = 0x98 - sizeofRtMetricsFreeBSD10 = 0x70 - - sizeofIfMsghdrFreeBSD7 = 0xa8 - sizeofIfMsghdrFreeBSD8 = 0xa8 - sizeofIfMsghdrFreeBSD9 = 0xa8 - sizeofIfMsghdrFreeBSD10 = 0xa8 - sizeofIfMsghdrFreeBSD11 = 0xa8 - - sizeofIfDataFreeBSD7 = 0x98 - sizeofIfDataFreeBSD8 = 0x98 - sizeofIfDataFreeBSD9 = 0x98 - sizeofIfDataFreeBSD10 = 0x98 - sizeofIfDataFreeBSD11 = 0x98 - - sizeofIfMsghdrlFreeBSD10Emu = 0xb0 - sizeofIfaMsghdrFreeBSD10Emu = 0x14 - sizeofIfaMsghdrlFreeBSD10Emu = 0xb0 - sizeofIfmaMsghdrFreeBSD10Emu = 0x10 - sizeofIfAnnouncemsghdrFreeBSD10Emu = 0x18 - - sizeofRtMsghdrFreeBSD10Emu = 0x98 - sizeofRtMetricsFreeBSD10Emu = 0x70 - - sizeofIfMsghdrFreeBSD7Emu = 0xa8 - sizeofIfMsghdrFreeBSD8Emu = 0xa8 - sizeofIfMsghdrFreeBSD9Emu = 0xa8 - sizeofIfMsghdrFreeBSD10Emu = 0xa8 - sizeofIfMsghdrFreeBSD11Emu = 0xa8 - - sizeofIfDataFreeBSD7Emu = 0x98 - sizeofIfDataFreeBSD8Emu = 0x98 - sizeofIfDataFreeBSD9Emu = 0x98 - sizeofIfDataFreeBSD10Emu = 0x98 - sizeofIfDataFreeBSD11Emu = 0x98 - - sizeofSockaddrStorage = 0x80 - sizeofSockaddrInet = 0x10 - sizeofSockaddrInet6 = 0x1c -) diff --git a/src/vendor/golang.org/x/net/route/zsys_netbsd.go b/src/vendor/golang.org/x/net/route/zsys_netbsd.go deleted file mode 100644 index eaffe8c40804cb..00000000000000 --- a/src/vendor/golang.org/x/net/route/zsys_netbsd.go +++ /dev/null @@ -1,17 +0,0 @@ -// Code generated by cmd/cgo -godefs; DO NOT EDIT. -// cgo -godefs defs_netbsd.go - -package route - -const ( - sizeofIfMsghdrNetBSD7 = 0x98 - sizeofIfaMsghdrNetBSD7 = 0x18 - sizeofIfAnnouncemsghdrNetBSD7 = 0x18 - - sizeofRtMsghdrNetBSD7 = 0x78 - sizeofRtMetricsNetBSD7 = 0x50 - - sizeofSockaddrStorage = 0x80 - sizeofSockaddrInet = 0x10 - sizeofSockaddrInet6 = 0x1c -) diff --git a/src/vendor/golang.org/x/net/route/zsys_openbsd.go b/src/vendor/golang.org/x/net/route/zsys_openbsd.go deleted file mode 100644 index b11b8126801b68..00000000000000 --- a/src/vendor/golang.org/x/net/route/zsys_openbsd.go +++ /dev/null @@ -1,12 +0,0 @@ -// Code generated by cmd/cgo -godefs; DO NOT EDIT. -// cgo -godefs defs_openbsd.go - -package route - -const ( - sizeofRtMsghdr = 0x60 - - sizeofSockaddrStorage = 0x100 - sizeofSockaddrInet = 0x10 - sizeofSockaddrInet6 = 0x1c -) diff --git a/src/vendor/golang.org/x/sys/cpu/asm_darwin_x86_gc.s b/src/vendor/golang.org/x/sys/cpu/asm_darwin_x86_gc.s new file mode 100644 index 00000000000000..ec2acfe540ea7b --- /dev/null +++ b/src/vendor/golang.org/x/sys/cpu/asm_darwin_x86_gc.s @@ -0,0 +1,17 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin && amd64 && gc + +#include "textflag.h" + +TEXT libc_sysctl_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_sysctl(SB) +GLOBL ·libc_sysctl_trampoline_addr(SB), RODATA, $8 +DATA ·libc_sysctl_trampoline_addr(SB)/8, $libc_sysctl_trampoline<>(SB) + +TEXT libc_sysctlbyname_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_sysctlbyname(SB) +GLOBL ·libc_sysctlbyname_trampoline_addr(SB), RODATA, $8 +DATA ·libc_sysctlbyname_trampoline_addr(SB)/8, $libc_sysctlbyname_trampoline<>(SB) diff --git a/src/vendor/golang.org/x/sys/cpu/cpu.go b/src/vendor/golang.org/x/sys/cpu/cpu.go index ec07aab05788c8..63541994ef0309 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu.go @@ -72,6 +72,9 @@ var X86 struct { HasSSSE3 bool // Supplemental streaming SIMD extension 3 HasSSE41 bool // Streaming SIMD extension 4 and 4.1 HasSSE42 bool // Streaming SIMD extension 4 and 4.2 + HasAVXIFMA bool // Advanced vector extension Integer Fused Multiply Add + HasAVXVNNI bool // Advanced vector extension Vector Neural Network Instructions + HasAVXVNNIInt8 bool // Advanced vector extension Vector Neural Network Int8 instructions _ CacheLinePad } @@ -146,6 +149,18 @@ var ARM struct { _ CacheLinePad } +// The booleans in Loong64 contain the correspondingly named cpu feature bit. +// The struct is padded to avoid false sharing. +var Loong64 struct { + _ CacheLinePad + HasLSX bool // support 128-bit vector extension + HasLASX bool // support 256-bit vector extension + HasCRC32 bool // support CRC instruction + HasLAM_BH bool // support AM{SWAP/ADD}[_DB].{B/H} instruction + HasLAMCAS bool // support AMCAS[_DB].{B/H/W/D} instruction + _ CacheLinePad +} + // MIPS64X contains the supported CPU features of the current mips64/mips64le // platforms. If the current platform is not mips64/mips64le or the current // operating system is not Linux then all feature flags are false. @@ -201,6 +216,36 @@ var S390X struct { _ CacheLinePad } +// RISCV64 contains the supported CPU features and performance characteristics for riscv64 +// platforms. The booleans in RISCV64, with the exception of HasFastMisaligned, indicate +// the presence of RISC-V extensions. +// +// It is safe to assume that all the RV64G extensions are supported and so they are omitted from +// this structure. As riscv64 Go programs require at least RV64G, the code that populates +// this structure cannot run successfully if some of the RV64G extensions are missing. +// The struct is padded to avoid false sharing. +var RISCV64 struct { + _ CacheLinePad + HasFastMisaligned bool // Fast misaligned accesses + HasC bool // Compressed instruction-set extension + HasV bool // Vector extension compatible with RVV 1.0 + HasZba bool // Address generation instructions extension + HasZbb bool // Basic bit-manipulation extension + HasZbs bool // Single-bit instructions extension + HasZvbb bool // Vector Basic Bit-manipulation + HasZvbc bool // Vector Carryless Multiplication + HasZvkb bool // Vector Cryptography Bit-manipulation + HasZvkt bool // Vector Data-Independent Execution Latency + HasZvkg bool // Vector GCM/GMAC + HasZvkn bool // NIST Algorithm Suite (AES/SHA256/SHA512) + HasZvknc bool // NIST Algorithm Suite with carryless multiply + HasZvkng bool // NIST Algorithm Suite with GCM + HasZvks bool // ShangMi Algorithm Suite + HasZvksc bool // ShangMi Algorithm Suite with carryless multiplication + HasZvksg bool // ShangMi Algorithm Suite with GCM + _ CacheLinePad +} + func init() { archInit() initOptions() diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_darwin_x86.go b/src/vendor/golang.org/x/sys/cpu/cpu_darwin_x86.go new file mode 100644 index 00000000000000..b838cb9e956e61 --- /dev/null +++ b/src/vendor/golang.org/x/sys/cpu/cpu_darwin_x86.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin && amd64 && gc + +package cpu + +// darwinSupportsAVX512 checks Darwin kernel for AVX512 support via sysctl +// call (see issue 43089). It also restricts AVX512 support for Darwin to +// kernel version 21.3.0 (MacOS 12.2.0) or later (see issue 49233). +// +// Background: +// Darwin implements a special mechanism to economize on thread state when +// AVX512 specific registers are not in use. This scheme minimizes state when +// preempting threads that haven't yet used any AVX512 instructions, but adds +// special requirements to check for AVX512 hardware support at runtime (e.g. +// via sysctl call or commpage inspection). See issue 43089 and link below for +// full background: +// https://github.com/apple-oss-distributions/xnu/blob/xnu-11215.1.10/osfmk/i386/fpu.c#L214-L240 +// +// Additionally, all versions of the Darwin kernel from 19.6.0 through 21.2.0 +// (corresponding to MacOS 10.15.6 - 12.1) have a bug that can cause corruption +// of the AVX512 mask registers (K0-K7) upon signal return. For this reason +// AVX512 is considered unsafe to use on Darwin for kernel versions prior to +// 21.3.0, where a fix has been confirmed. See issue 49233 for full background. +func darwinSupportsAVX512() bool { + return darwinSysctlEnabled([]byte("hw.optional.avx512f\x00")) && darwinKernelVersionCheck(21, 3, 0) +} + +// Ensure Darwin kernel version is at least major.minor.patch, avoiding dependencies +func darwinKernelVersionCheck(major, minor, patch int) bool { + var release [256]byte + err := darwinOSRelease(&release) + if err != nil { + return false + } + + var mmp [3]int + c := 0 +Loop: + for _, b := range release[:] { + switch { + case b >= '0' && b <= '9': + mmp[c] = 10*mmp[c] + int(b-'0') + case b == '.': + c++ + if c > 2 { + return false + } + case b == 0: + break Loop + default: + return false + } + } + if c != 2 { + return false + } + return mmp[0] > major || mmp[0] == major && (mmp[1] > minor || mmp[1] == minor && mmp[2] >= patch) +} diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go b/src/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go index 910728fb163f36..32a44514e245fe 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go @@ -6,10 +6,10 @@ package cpu -// cpuid is implemented in cpu_x86.s for gc compiler +// cpuid is implemented in cpu_gc_x86.s for gc compiler // and in cpu_gccgo.c for gccgo. func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) -// xgetbv with ecx = 0 is implemented in cpu_x86.s for gc compiler +// xgetbv with ecx = 0 is implemented in cpu_gc_x86.s for gc compiler // and in cpu_gccgo.c for gccgo. func xgetbv() (eax, edx uint32) diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_gc_x86.s b/src/vendor/golang.org/x/sys/cpu/cpu_gc_x86.s new file mode 100644 index 00000000000000..ce208ce6d6a3a6 --- /dev/null +++ b/src/vendor/golang.org/x/sys/cpu/cpu_gc_x86.s @@ -0,0 +1,26 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (386 || amd64 || amd64p32) && gc + +#include "textflag.h" + +// func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) +TEXT ·cpuid(SB), NOSPLIT, $0-24 + MOVL eaxArg+0(FP), AX + MOVL ecxArg+4(FP), CX + CPUID + MOVL AX, eax+8(FP) + MOVL BX, ebx+12(FP) + MOVL CX, ecx+16(FP) + MOVL DX, edx+20(FP) + RET + +// func xgetbv() (eax, edx uint32) +TEXT ·xgetbv(SB), NOSPLIT, $0-8 + MOVL $0, CX + XGETBV + MOVL AX, eax+0(FP) + MOVL DX, edx+4(FP) + RET diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go b/src/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go index 99c60fe9f9c657..170d21ddfda416 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go @@ -23,9 +23,3 @@ func xgetbv() (eax, edx uint32) { gccgoXgetbv(&a, &d) return a, d } - -// gccgo doesn't build on Darwin, per: -// https://github.com/Homebrew/homebrew-core/blob/HEAD/Formula/gcc.rb#L76 -func darwinSupportsAVX512() bool { - return false -} diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go b/src/vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go index 08f35ea17735fd..f1caf0f78e24b6 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go @@ -110,7 +110,6 @@ func doinit() { ARM64.HasASIMDFHM = isSet(hwCap, hwcap_ASIMDFHM) ARM64.HasDIT = isSet(hwCap, hwcap_DIT) - // HWCAP2 feature bits ARM64.HasSVE2 = isSet(hwCap2, hwcap2_SVE2) ARM64.HasI8MM = isSet(hwCap2, hwcap2_I8MM) diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_linux_loong64.go b/src/vendor/golang.org/x/sys/cpu/cpu_linux_loong64.go new file mode 100644 index 00000000000000..4f34114329eae0 --- /dev/null +++ b/src/vendor/golang.org/x/sys/cpu/cpu_linux_loong64.go @@ -0,0 +1,22 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +// HWCAP bits. These are exposed by the Linux kernel. +const ( + hwcap_LOONGARCH_LSX = 1 << 4 + hwcap_LOONGARCH_LASX = 1 << 5 +) + +func doinit() { + // TODO: Features that require kernel support like LSX and LASX can + // be detected here once needed in std library or by the compiler. + Loong64.HasLSX = hwcIsSet(hwCap, hwcap_LOONGARCH_LSX) + Loong64.HasLASX = hwcIsSet(hwCap, hwcap_LOONGARCH_LASX) +} + +func hwcIsSet(hwc uint, val uint) bool { + return hwc&val != 0 +} diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go b/src/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go index cd63e73355734c..a428dec9cde941 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build linux && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !s390x +//go:build linux && !arm && !arm64 && !loong64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !s390x && !riscv64 package cpu diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go b/src/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go new file mode 100644 index 00000000000000..ad741536f395b3 --- /dev/null +++ b/src/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go @@ -0,0 +1,160 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +import ( + "syscall" + "unsafe" +) + +// RISC-V extension discovery code for Linux. The approach here is to first try the riscv_hwprobe +// syscall falling back to HWCAP to check for the C extension if riscv_hwprobe is not available. +// +// A note on detection of the Vector extension using HWCAP. +// +// Support for the Vector extension version 1.0 was added to the Linux kernel in release 6.5. +// Support for the riscv_hwprobe syscall was added in 6.4. It follows that if the riscv_hwprobe +// syscall is not available then neither is the Vector extension (which needs kernel support). +// The riscv_hwprobe syscall should then be all we need to detect the Vector extension. +// However, some RISC-V board manufacturers ship boards with an older kernel on top of which +// they have back-ported various versions of the Vector extension patches but not the riscv_hwprobe +// patches. These kernels advertise support for the Vector extension using HWCAP. Falling +// back to HWCAP to detect the Vector extension, if riscv_hwprobe is not available, or simply not +// bothering with riscv_hwprobe at all and just using HWCAP may then seem like an attractive option. +// +// Unfortunately, simply checking the 'V' bit in AT_HWCAP will not work as this bit is used by +// RISC-V board and cloud instance providers to mean different things. The Lichee Pi 4A board +// and the Scaleway RV1 cloud instances use the 'V' bit to advertise their support for the unratified +// 0.7.1 version of the Vector Specification. The Banana Pi BPI-F3 and the CanMV-K230 board use +// it to advertise support for 1.0 of the Vector extension. Versions 0.7.1 and 1.0 of the Vector +// extension are binary incompatible. HWCAP can then not be used in isolation to populate the +// HasV field as this field indicates that the underlying CPU is compatible with RVV 1.0. +// +// There is a way at runtime to distinguish between versions 0.7.1 and 1.0 of the Vector +// specification by issuing a RVV 1.0 vsetvli instruction and checking the vill bit of the vtype +// register. This check would allow us to safely detect version 1.0 of the Vector extension +// with HWCAP, if riscv_hwprobe were not available. However, the check cannot +// be added until the assembler supports the Vector instructions. +// +// Note the riscv_hwprobe syscall does not suffer from these ambiguities by design as all of the +// extensions it advertises support for are explicitly versioned. It's also worth noting that +// the riscv_hwprobe syscall is the only way to detect multi-letter RISC-V extensions, e.g., Zba. +// These cannot be detected using HWCAP and so riscv_hwprobe must be used to detect the majority +// of RISC-V extensions. +// +// Please see https://docs.kernel.org/arch/riscv/hwprobe.html for more information. + +// golang.org/x/sys/cpu is not allowed to depend on golang.org/x/sys/unix so we must +// reproduce the constants, types and functions needed to make the riscv_hwprobe syscall +// here. + +const ( + // Copied from golang.org/x/sys/unix/ztypes_linux_riscv64.go. + riscv_HWPROBE_KEY_IMA_EXT_0 = 0x4 + riscv_HWPROBE_IMA_C = 0x2 + riscv_HWPROBE_IMA_V = 0x4 + riscv_HWPROBE_EXT_ZBA = 0x8 + riscv_HWPROBE_EXT_ZBB = 0x10 + riscv_HWPROBE_EXT_ZBS = 0x20 + riscv_HWPROBE_EXT_ZVBB = 0x20000 + riscv_HWPROBE_EXT_ZVBC = 0x40000 + riscv_HWPROBE_EXT_ZVKB = 0x80000 + riscv_HWPROBE_EXT_ZVKG = 0x100000 + riscv_HWPROBE_EXT_ZVKNED = 0x200000 + riscv_HWPROBE_EXT_ZVKNHB = 0x800000 + riscv_HWPROBE_EXT_ZVKSED = 0x1000000 + riscv_HWPROBE_EXT_ZVKSH = 0x2000000 + riscv_HWPROBE_EXT_ZVKT = 0x4000000 + riscv_HWPROBE_KEY_CPUPERF_0 = 0x5 + riscv_HWPROBE_MISALIGNED_FAST = 0x3 + riscv_HWPROBE_MISALIGNED_MASK = 0x7 +) + +const ( + // sys_RISCV_HWPROBE is copied from golang.org/x/sys/unix/zsysnum_linux_riscv64.go. + sys_RISCV_HWPROBE = 258 +) + +// riscvHWProbePairs is copied from golang.org/x/sys/unix/ztypes_linux_riscv64.go. +type riscvHWProbePairs struct { + key int64 + value uint64 +} + +const ( + // CPU features + hwcap_RISCV_ISA_C = 1 << ('C' - 'A') +) + +func doinit() { + // A slice of key/value pair structures is passed to the RISCVHWProbe syscall. The key + // field should be initialised with one of the key constants defined above, e.g., + // RISCV_HWPROBE_KEY_IMA_EXT_0. The syscall will set the value field to the appropriate value. + // If the kernel does not recognise a key it will set the key field to -1 and the value field to 0. + + pairs := []riscvHWProbePairs{ + {riscv_HWPROBE_KEY_IMA_EXT_0, 0}, + {riscv_HWPROBE_KEY_CPUPERF_0, 0}, + } + + // This call only indicates that extensions are supported if they are implemented on all cores. + if riscvHWProbe(pairs, 0) { + if pairs[0].key != -1 { + v := uint(pairs[0].value) + RISCV64.HasC = isSet(v, riscv_HWPROBE_IMA_C) + RISCV64.HasV = isSet(v, riscv_HWPROBE_IMA_V) + RISCV64.HasZba = isSet(v, riscv_HWPROBE_EXT_ZBA) + RISCV64.HasZbb = isSet(v, riscv_HWPROBE_EXT_ZBB) + RISCV64.HasZbs = isSet(v, riscv_HWPROBE_EXT_ZBS) + RISCV64.HasZvbb = isSet(v, riscv_HWPROBE_EXT_ZVBB) + RISCV64.HasZvbc = isSet(v, riscv_HWPROBE_EXT_ZVBC) + RISCV64.HasZvkb = isSet(v, riscv_HWPROBE_EXT_ZVKB) + RISCV64.HasZvkg = isSet(v, riscv_HWPROBE_EXT_ZVKG) + RISCV64.HasZvkt = isSet(v, riscv_HWPROBE_EXT_ZVKT) + // Cryptography shorthand extensions + RISCV64.HasZvkn = isSet(v, riscv_HWPROBE_EXT_ZVKNED) && + isSet(v, riscv_HWPROBE_EXT_ZVKNHB) && RISCV64.HasZvkb && RISCV64.HasZvkt + RISCV64.HasZvknc = RISCV64.HasZvkn && RISCV64.HasZvbc + RISCV64.HasZvkng = RISCV64.HasZvkn && RISCV64.HasZvkg + RISCV64.HasZvks = isSet(v, riscv_HWPROBE_EXT_ZVKSED) && + isSet(v, riscv_HWPROBE_EXT_ZVKSH) && RISCV64.HasZvkb && RISCV64.HasZvkt + RISCV64.HasZvksc = RISCV64.HasZvks && RISCV64.HasZvbc + RISCV64.HasZvksg = RISCV64.HasZvks && RISCV64.HasZvkg + } + if pairs[1].key != -1 { + v := pairs[1].value & riscv_HWPROBE_MISALIGNED_MASK + RISCV64.HasFastMisaligned = v == riscv_HWPROBE_MISALIGNED_FAST + } + } + + // Let's double check with HWCAP if the C extension does not appear to be supported. + // This may happen if we're running on a kernel older than 6.4. + + if !RISCV64.HasC { + RISCV64.HasC = isSet(hwCap, hwcap_RISCV_ISA_C) + } +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} + +// riscvHWProbe is a simplified version of the generated wrapper function found in +// golang.org/x/sys/unix/zsyscall_linux_riscv64.go. We simplify it by removing the +// cpuCount and cpus parameters which we do not need. We always want to pass 0 for +// these parameters here so the kernel only reports the extensions that are present +// on all cores. +func riscvHWProbe(pairs []riscvHWProbePairs, flags uint) bool { + var _zero uintptr + var p0 unsafe.Pointer + if len(pairs) > 0 { + p0 = unsafe.Pointer(&pairs[0]) + } else { + p0 = unsafe.Pointer(&_zero) + } + + _, _, e1 := syscall.Syscall6(sys_RISCV_HWPROBE, uintptr(p0), uintptr(len(pairs)), uintptr(0), uintptr(0), uintptr(flags), 0) + return e1 == 0 +} diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_loong64.go b/src/vendor/golang.org/x/sys/cpu/cpu_loong64.go index 558635850c7860..45ecb29ae78a51 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu_loong64.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu_loong64.go @@ -8,5 +8,43 @@ package cpu const cacheLineSize = 64 +// Bit fields for CPUCFG registers, Related reference documents: +// https://loongson.github.io/LoongArch-Documentation/LoongArch-Vol1-EN.html#_cpucfg +const ( + // CPUCFG1 bits + cpucfg1_CRC32 = 1 << 25 + + // CPUCFG2 bits + cpucfg2_LAM_BH = 1 << 27 + cpucfg2_LAMCAS = 1 << 28 +) + func initOptions() { + options = []option{ + {Name: "lsx", Feature: &Loong64.HasLSX}, + {Name: "lasx", Feature: &Loong64.HasLASX}, + {Name: "crc32", Feature: &Loong64.HasCRC32}, + {Name: "lam_bh", Feature: &Loong64.HasLAM_BH}, + {Name: "lamcas", Feature: &Loong64.HasLAMCAS}, + } + + // The CPUCFG data on Loong64 only reflects the hardware capabilities, + // not the kernel support status, so features such as LSX and LASX that + // require kernel support cannot be obtained from the CPUCFG data. + // + // These features only require hardware capability support and do not + // require kernel specific support, so they can be obtained directly + // through CPUCFG + cfg1 := get_cpucfg(1) + cfg2 := get_cpucfg(2) + + Loong64.HasCRC32 = cfgIsSet(cfg1, cpucfg1_CRC32) + Loong64.HasLAMCAS = cfgIsSet(cfg2, cpucfg2_LAMCAS) + Loong64.HasLAM_BH = cfgIsSet(cfg2, cpucfg2_LAM_BH) +} + +func get_cpucfg(reg uint32) uint32 + +func cfgIsSet(cfg uint32, val uint32) bool { + return cfg&val != 0 } diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_loong64.s b/src/vendor/golang.org/x/sys/cpu/cpu_loong64.s new file mode 100644 index 00000000000000..71cbaf1ce276de --- /dev/null +++ b/src/vendor/golang.org/x/sys/cpu/cpu_loong64.s @@ -0,0 +1,13 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// func get_cpucfg(reg uint32) uint32 +TEXT ·get_cpucfg(SB), NOSPLIT|NOFRAME, $0 + MOVW reg+0(FP), R5 + // CPUCFG R5, R4 = 0x00006ca4 + WORD $0x00006ca4 + MOVW R4, ret+8(FP) + RET diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_other_x86.go b/src/vendor/golang.org/x/sys/cpu/cpu_other_x86.go new file mode 100644 index 00000000000000..a0fd7e2f75d37f --- /dev/null +++ b/src/vendor/golang.org/x/sys/cpu/cpu_other_x86.go @@ -0,0 +1,11 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 || amd64p32 || (amd64 && (!darwin || !gc)) + +package cpu + +func darwinSupportsAVX512() bool { + panic("only implemented for gc && amd64 && darwin") +} diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_riscv64.go b/src/vendor/golang.org/x/sys/cpu/cpu_riscv64.go index 7f0c79c004b44a..0f617aef541b5c 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu_riscv64.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu_riscv64.go @@ -8,4 +8,25 @@ package cpu const cacheLineSize = 64 -func initOptions() {} +func initOptions() { + options = []option{ + {Name: "fastmisaligned", Feature: &RISCV64.HasFastMisaligned}, + {Name: "c", Feature: &RISCV64.HasC}, + {Name: "v", Feature: &RISCV64.HasV}, + {Name: "zba", Feature: &RISCV64.HasZba}, + {Name: "zbb", Feature: &RISCV64.HasZbb}, + {Name: "zbs", Feature: &RISCV64.HasZbs}, + // RISC-V Cryptography Extensions + {Name: "zvbb", Feature: &RISCV64.HasZvbb}, + {Name: "zvbc", Feature: &RISCV64.HasZvbc}, + {Name: "zvkb", Feature: &RISCV64.HasZvkb}, + {Name: "zvkg", Feature: &RISCV64.HasZvkg}, + {Name: "zvkt", Feature: &RISCV64.HasZvkt}, + {Name: "zvkn", Feature: &RISCV64.HasZvkn}, + {Name: "zvknc", Feature: &RISCV64.HasZvknc}, + {Name: "zvkng", Feature: &RISCV64.HasZvkng}, + {Name: "zvks", Feature: &RISCV64.HasZvks}, + {Name: "zvksc", Feature: &RISCV64.HasZvksc}, + {Name: "zvksg", Feature: &RISCV64.HasZvksg}, + } +} diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_x86.go b/src/vendor/golang.org/x/sys/cpu/cpu_x86.go index c29f5e4c5a6e4e..1e642f3304fa87 100644 --- a/src/vendor/golang.org/x/sys/cpu/cpu_x86.go +++ b/src/vendor/golang.org/x/sys/cpu/cpu_x86.go @@ -53,6 +53,9 @@ func initOptions() { {Name: "sse41", Feature: &X86.HasSSE41}, {Name: "sse42", Feature: &X86.HasSSE42}, {Name: "ssse3", Feature: &X86.HasSSSE3}, + {Name: "avxifma", Feature: &X86.HasAVXIFMA}, + {Name: "avxvnni", Feature: &X86.HasAVXVNNI}, + {Name: "avxvnniint8", Feature: &X86.HasAVXVNNIInt8}, // These capabilities should always be enabled on amd64: {Name: "sse2", Feature: &X86.HasSSE2, Required: runtime.GOARCH == "amd64"}, @@ -92,10 +95,8 @@ func archInit() { osSupportsAVX = isSet(1, eax) && isSet(2, eax) if runtime.GOOS == "darwin" { - // Darwin doesn't save/restore AVX-512 mask registers correctly across signal handlers. - // Since users can't rely on mask register contents, let's not advertise AVX-512 support. - // See issue 49233. - osSupportsAVX512 = false + // Darwin requires special AVX512 checks, see cpu_darwin_x86.go + osSupportsAVX512 = osSupportsAVX && darwinSupportsAVX512() } else { // Check if OPMASK and ZMM registers have OS support. osSupportsAVX512 = osSupportsAVX && isSet(5, eax) && isSet(6, eax) && isSet(7, eax) @@ -108,7 +109,7 @@ func archInit() { return } - _, ebx7, ecx7, edx7 := cpuid(7, 0) + eax7, ebx7, ecx7, edx7 := cpuid(7, 0) X86.HasBMI1 = isSet(3, ebx7) X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX X86.HasBMI2 = isSet(8, ebx7) @@ -136,14 +137,24 @@ func archInit() { X86.HasAVX512VAES = isSet(9, ecx7) X86.HasAVX512VBMI2 = isSet(6, ecx7) X86.HasAVX512BITALG = isSet(12, ecx7) - - eax71, _, _, _ := cpuid(7, 1) - X86.HasAVX512BF16 = isSet(5, eax71) } X86.HasAMXTile = isSet(24, edx7) X86.HasAMXInt8 = isSet(25, edx7) X86.HasAMXBF16 = isSet(22, edx7) + + // These features depend on the second level of extended features. + if eax7 >= 1 { + eax71, _, _, edx71 := cpuid(7, 1) + if X86.HasAVX512 { + X86.HasAVX512BF16 = isSet(5, eax71) + } + if X86.HasAVX { + X86.HasAVXIFMA = isSet(23, eax71) + X86.HasAVXVNNI = isSet(4, eax71) + X86.HasAVXVNNIInt8 = isSet(4, edx71) + } + } } func isSet(bitpos uint, value uint32) bool { diff --git a/src/vendor/golang.org/x/sys/cpu/cpu_x86.s b/src/vendor/golang.org/x/sys/cpu/cpu_x86.s deleted file mode 100644 index 7d7ba33efb8555..00000000000000 --- a/src/vendor/golang.org/x/sys/cpu/cpu_x86.s +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build (386 || amd64 || amd64p32) && gc - -#include "textflag.h" - -// func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) -TEXT ·cpuid(SB), NOSPLIT, $0-24 - MOVL eaxArg+0(FP), AX - MOVL ecxArg+4(FP), CX - CPUID - MOVL AX, eax+8(FP) - MOVL BX, ebx+12(FP) - MOVL CX, ecx+16(FP) - MOVL DX, edx+20(FP) - RET - -// func xgetbv() (eax, edx uint32) -TEXT ·xgetbv(SB),NOSPLIT,$0-8 - MOVL $0, CX - XGETBV - MOVL AX, eax+0(FP) - MOVL DX, edx+4(FP) - RET diff --git a/src/vendor/golang.org/x/sys/cpu/parse.go b/src/vendor/golang.org/x/sys/cpu/parse.go index 762b63d6882cd7..56a7e1a176f92e 100644 --- a/src/vendor/golang.org/x/sys/cpu/parse.go +++ b/src/vendor/golang.org/x/sys/cpu/parse.go @@ -13,7 +13,7 @@ import "strconv" // https://golang.org/cl/209597. func parseRelease(rel string) (major, minor, patch int, ok bool) { // Strip anything after a dash or plus. - for i := 0; i < len(rel); i++ { + for i := range len(rel) { if rel[i] == '-' || rel[i] == '+' { rel = rel[:i] break @@ -21,7 +21,7 @@ func parseRelease(rel string) (major, minor, patch int, ok bool) { } next := func() (int, bool) { - for i := 0; i < len(rel); i++ { + for i := range len(rel) { if rel[i] == '.' { ver, err := strconv.Atoi(rel[:i]) rel = rel[i+1:] diff --git a/src/vendor/golang.org/x/sys/cpu/syscall_darwin_x86_gc.go b/src/vendor/golang.org/x/sys/cpu/syscall_darwin_x86_gc.go new file mode 100644 index 00000000000000..4d0888b0c010f4 --- /dev/null +++ b/src/vendor/golang.org/x/sys/cpu/syscall_darwin_x86_gc.go @@ -0,0 +1,98 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Minimal copy of x/sys/unix so the cpu package can make a +// system call on Darwin without depending on x/sys/unix. + +//go:build darwin && amd64 && gc + +package cpu + +import ( + "syscall" + "unsafe" +) + +type _C_int int32 + +// adapted from unix.Uname() at x/sys/unix/syscall_darwin.go L419 +func darwinOSRelease(release *[256]byte) error { + // from x/sys/unix/zerrors_openbsd_amd64.go + const ( + CTL_KERN = 0x1 + KERN_OSRELEASE = 0x2 + ) + + mib := []_C_int{CTL_KERN, KERN_OSRELEASE} + n := unsafe.Sizeof(*release) + + return sysctl(mib, &release[0], &n, nil, 0) +} + +type Errno = syscall.Errno + +var _zero uintptr // Single-word zero for use when we need a valid pointer to 0 bytes. + +// from x/sys/unix/zsyscall_darwin_amd64.go L791-807 +func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) error { + var _p0 unsafe.Pointer + if len(mib) > 0 { + _p0 = unsafe.Pointer(&mib[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + if _, _, err := syscall_syscall6( + libc_sysctl_trampoline_addr, + uintptr(_p0), + uintptr(len(mib)), + uintptr(unsafe.Pointer(old)), + uintptr(unsafe.Pointer(oldlen)), + uintptr(unsafe.Pointer(new)), + uintptr(newlen), + ); err != 0 { + return err + } + + return nil +} + +var libc_sysctl_trampoline_addr uintptr + +// adapted from internal/cpu/cpu_arm64_darwin.go +func darwinSysctlEnabled(name []byte) bool { + out := int32(0) + nout := unsafe.Sizeof(out) + if ret := sysctlbyname(&name[0], (*byte)(unsafe.Pointer(&out)), &nout, nil, 0); ret != nil { + return false + } + return out > 0 +} + +//go:cgo_import_dynamic libc_sysctl sysctl "/usr/lib/libSystem.B.dylib" + +var libc_sysctlbyname_trampoline_addr uintptr + +// adapted from runtime/sys_darwin.go in the pattern of sysctl() above, as defined in x/sys/unix +func sysctlbyname(name *byte, old *byte, oldlen *uintptr, new *byte, newlen uintptr) error { + if _, _, err := syscall_syscall6( + libc_sysctlbyname_trampoline_addr, + uintptr(unsafe.Pointer(name)), + uintptr(unsafe.Pointer(old)), + uintptr(unsafe.Pointer(oldlen)), + uintptr(unsafe.Pointer(new)), + uintptr(newlen), + 0, + ); err != 0 { + return err + } + + return nil +} + +//go:cgo_import_dynamic libc_sysctlbyname sysctlbyname "/usr/lib/libSystem.B.dylib" + +// Implemented in the runtime package (runtime/sys_darwin.go) +func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) + +//go:linkname syscall_syscall6 syscall.syscall6 diff --git a/src/vendor/golang.org/x/text/unicode/bidi/core.go b/src/vendor/golang.org/x/text/unicode/bidi/core.go index 9d2ae547b5ed4d..fb8273236dde6e 100644 --- a/src/vendor/golang.org/x/text/unicode/bidi/core.go +++ b/src/vendor/golang.org/x/text/unicode/bidi/core.go @@ -427,13 +427,6 @@ type isolatingRunSequence struct { func (i *isolatingRunSequence) Len() int { return len(i.indexes) } -func maxLevel(a, b level) level { - if a > b { - return a - } - return b -} - // Rule X10, second bullet: Determine the start-of-sequence (sos) and end-of-sequence (eos) types, // either L or R, for each isolating run sequence. func (p *paragraph) isolatingRunSequence(indexes []int) *isolatingRunSequence { @@ -474,8 +467,8 @@ func (p *paragraph) isolatingRunSequence(indexes []int) *isolatingRunSequence { indexes: indexes, types: types, level: level, - sos: typeForLevel(maxLevel(prevLevel, level)), - eos: typeForLevel(maxLevel(succLevel, level)), + sos: typeForLevel(max(prevLevel, level)), + eos: typeForLevel(max(succLevel, level)), } } diff --git a/src/vendor/modules.txt b/src/vendor/modules.txt index 7562f74b39ada6..f1e33686ed0626 100644 --- a/src/vendor/modules.txt +++ b/src/vendor/modules.txt @@ -1,15 +1,13 @@ -# golang.org/x/crypto v0.25.1-0.20240722173533-bb80217080b0 -## explicit; go 1.20 +# golang.org/x/crypto v0.43.0 +## explicit; go 1.24.0 golang.org/x/crypto/chacha20 golang.org/x/crypto/chacha20poly1305 golang.org/x/crypto/cryptobyte golang.org/x/crypto/cryptobyte/asn1 -golang.org/x/crypto/hkdf golang.org/x/crypto/internal/alias golang.org/x/crypto/internal/poly1305 -golang.org/x/crypto/sha3 -# golang.org/x/net v0.27.1-0.20240722181819-765c7e89b3bd -## explicit; go 1.18 +# golang.org/x/net v0.46.0 +## explicit; go 1.24.0 golang.org/x/net/dns/dnsmessage golang.org/x/net/http/httpguts golang.org/x/net/http/httpproxy @@ -17,12 +15,11 @@ golang.org/x/net/http2/hpack golang.org/x/net/idna golang.org/x/net/lif golang.org/x/net/nettest -golang.org/x/net/route -# golang.org/x/sys v0.22.1-0.20240722205227-beb594982ddb -## explicit; go 1.18 +# golang.org/x/sys v0.37.0 +## explicit; go 1.24.0 golang.org/x/sys/cpu -# golang.org/x/text v0.16.1-0.20240716160804-ae0cf96bbcd9 -## explicit; go 1.18 +# golang.org/x/text v0.30.0 +## explicit; go 1.24.0 golang.org/x/text/secure/bidirule golang.org/x/text/transform golang.org/x/text/unicode/bidi diff --git a/src/weak/doc.go b/src/weak/doc.go new file mode 100644 index 00000000000000..1af8e4c69b2f86 --- /dev/null +++ b/src/weak/doc.go @@ -0,0 +1,9 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package weak provides ways to safely reference memory weakly, +that is, without preventing its reclamation. +*/ +package weak diff --git a/src/weak/pointer.go b/src/weak/pointer.go new file mode 100644 index 00000000000000..42d83afdb6a634 --- /dev/null +++ b/src/weak/pointer.go @@ -0,0 +1,98 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package weak + +import ( + "internal/abi" + "runtime" + "unsafe" +) + +// Pointer is a weak pointer to a value of type T. +// +// Just like regular pointers, Pointer may reference any part of an +// object, such as a field of a struct or an element of an array. +// Objects that are only pointed to by weak pointers are not considered +// reachable, and once the object becomes unreachable, [Pointer.Value] +// may return nil. +// +// The primary use-cases for weak pointers are for implementing caches, +// canonicalization maps (like the unique package), and for tying together +// the lifetimes of separate values (for example, through a map with weak +// keys). +// +// Two Pointer values compare equal if and only if the pointers from which they +// were created compare equal. +// This property is maintained even after the object referenced by the pointer +// used to create a weak reference is reclaimed. +// If multiple weak pointers are made to different offsets within the same object +// (for example, pointers to different fields of the same struct), those pointers +// will not compare equal. +// In other words, weak pointers map to objects and offsets within those +// objects, not plain addresses. +// If a weak pointer is created from an object that becomes unreachable, but is +// then resurrected due to a finalizer, that weak pointer will not compare equal +// with weak pointers created after the resurrection. +// +// Calling [Make] with a nil pointer returns a weak pointer whose [Pointer.Value] +// always returns nil. The zero value of a Pointer behaves as if it were created +// by passing nil to [Make] and compares equal with such pointers. +// +// [Pointer.Value] is not guaranteed to eventually return nil. +// [Pointer.Value] may return nil as soon as the object becomes +// unreachable. +// Values stored in global variables, or that can be found by tracing +// pointers from a global variable, are reachable. A function argument or +// receiver may become unreachable at the last point where the function +// mentions it. To ensure [Pointer.Value] does not return nil, +// pass a pointer to the object to the [runtime.KeepAlive] function after +// the last point where the object must remain reachable. +// +// Note that because [Pointer.Value] is not guaranteed to eventually return +// nil, even after an object is no longer referenced, the runtime is allowed to +// perform a space-saving optimization that batches objects together in a single +// allocation slot. The weak pointer for an unreferenced object in such an +// allocation may never become nil if it always exists in the same batch as a +// referenced object. Typically, this batching only happens for tiny +// (on the order of 16 bytes or less) and pointer-free objects. +type Pointer[T any] struct { + // Mention T in the type definition to prevent conversions + // between Pointer types, like we do for sync/atomic.Pointer. + _ [0]*T + u unsafe.Pointer +} + +// Make creates a weak pointer from a pointer to some value of type T. +func Make[T any](ptr *T) Pointer[T] { + // Explicitly force ptr to escape to the heap. + ptr = abi.Escape(ptr) + + var u unsafe.Pointer + if ptr != nil { + u = runtime_registerWeakPointer(unsafe.Pointer(ptr)) + } + runtime.KeepAlive(ptr) + return Pointer[T]{u: u} +} + +// Value returns the original pointer used to create the weak pointer. +// It returns nil if the value pointed to by the original pointer was reclaimed by +// the garbage collector. +// If a weak pointer points to an object with a finalizer, then Value will +// return nil as soon as the object's finalizer is queued for execution. +func (p Pointer[T]) Value() *T { + if p.u == nil { + return nil + } + return (*T)(runtime_makeStrongFromWeak(p.u)) +} + +// Implemented in runtime. + +//go:linkname runtime_registerWeakPointer +func runtime_registerWeakPointer(unsafe.Pointer) unsafe.Pointer + +//go:linkname runtime_makeStrongFromWeak +func runtime_makeStrongFromWeak(unsafe.Pointer) unsafe.Pointer diff --git a/src/weak/pointer_test.go b/src/weak/pointer_test.go new file mode 100644 index 00000000000000..da464a8d01f2d0 --- /dev/null +++ b/src/weak/pointer_test.go @@ -0,0 +1,329 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package weak_test + +import ( + "context" + "internal/goarch" + "runtime" + "sync" + "testing" + "time" + "unsafe" + "weak" +) + +type T struct { + // N.B. This must contain a pointer, otherwise the weak handle might get placed + // in a tiny block making the tests in this package flaky. + t *T + a int + b int +} + +func TestPointer(t *testing.T) { + var zero weak.Pointer[T] + if zero.Value() != nil { + t.Error("Value of zero value of weak.Pointer is not nil") + } + zeroNil := weak.Make[T](nil) + if zeroNil.Value() != nil { + t.Error("Value of weak.Make[T](nil) is not nil") + } + + bt := new(T) + wt := weak.Make(bt) + if st := wt.Value(); st != bt { + t.Fatalf("weak pointer is not the same as strong pointer: %p vs. %p", st, bt) + } + // bt is still referenced. + runtime.GC() + + if st := wt.Value(); st != bt { + t.Fatalf("weak pointer is not the same as strong pointer after GC: %p vs. %p", st, bt) + } + // bt is no longer referenced. + runtime.GC() + + if st := wt.Value(); st != nil { + t.Fatalf("expected weak pointer to be nil, got %p", st) + } +} + +func TestPointerEquality(t *testing.T) { + var zero weak.Pointer[T] + zeroNil := weak.Make[T](nil) + if zero != zeroNil { + t.Error("weak.Make[T](nil) != zero value of weak.Pointer[T]") + } + + bt := make([]*T, 10) + wt := make([]weak.Pointer[T], 10) + wo := make([]weak.Pointer[int], 10) + for i := range bt { + bt[i] = new(T) + wt[i] = weak.Make(bt[i]) + wo[i] = weak.Make(&bt[i].a) + } + for i := range bt { + st := wt[i].Value() + if st != bt[i] { + t.Fatalf("weak pointer is not the same as strong pointer: %p vs. %p", st, bt[i]) + } + if wp := weak.Make(st); wp != wt[i] { + t.Fatalf("new weak pointer not equal to existing weak pointer: %v vs. %v", wp, wt[i]) + } + if wp := weak.Make(&st.a); wp != wo[i] { + t.Fatalf("new weak pointer not equal to existing weak pointer: %v vs. %v", wp, wo[i]) + } + if i == 0 { + continue + } + if wt[i] == wt[i-1] { + t.Fatalf("expected weak pointers to not be equal to each other, but got %v", wt[i]) + } + } + // bt is still referenced. + runtime.GC() + for i := range bt { + st := wt[i].Value() + if st != bt[i] { + t.Fatalf("weak pointer is not the same as strong pointer: %p vs. %p", st, bt[i]) + } + if wp := weak.Make(st); wp != wt[i] { + t.Fatalf("new weak pointer not equal to existing weak pointer: %v vs. %v", wp, wt[i]) + } + if wp := weak.Make(&st.a); wp != wo[i] { + t.Fatalf("new weak pointer not equal to existing weak pointer: %v vs. %v", wp, wo[i]) + } + if i == 0 { + continue + } + if wt[i] == wt[i-1] { + t.Fatalf("expected weak pointers to not be equal to each other, but got %v", wt[i]) + } + } + bt = nil + // bt is no longer referenced. + runtime.GC() + for i := range bt { + st := wt[i].Value() + if st != nil { + t.Fatalf("expected weak pointer to be nil, got %p", st) + } + if i == 0 { + continue + } + if wt[i] == wt[i-1] { + t.Fatalf("expected weak pointers to not be equal to each other, but got %v", wt[i]) + } + } +} + +func TestPointerFinalizer(t *testing.T) { + bt := new(T) + wt := weak.Make(bt) + done := make(chan struct{}, 1) + runtime.SetFinalizer(bt, func(bt *T) { + if wt.Value() != nil { + t.Errorf("weak pointer did not go nil before finalizer ran") + } + done <- struct{}{} + }) + + // Make sure the weak pointer stays around while bt is live. + runtime.GC() + if wt.Value() == nil { + t.Errorf("weak pointer went nil too soon") + } + runtime.KeepAlive(bt) + + // bt is no longer referenced. + // + // Run one cycle to queue the finalizer. + runtime.GC() + if wt.Value() != nil { + t.Errorf("weak pointer did not go nil when finalizer was enqueued") + } + + // Wait for the finalizer to run. + <-done + + // The weak pointer should still be nil after the finalizer runs. + runtime.GC() + if wt.Value() != nil { + t.Errorf("weak pointer is non-nil even after finalization: %v", wt) + } +} + +func TestPointerCleanup(t *testing.T) { + bt := new(T) + wt := weak.Make(bt) + done := make(chan struct{}, 1) + runtime.AddCleanup(bt, func(_ bool) { + if wt.Value() != nil { + t.Errorf("weak pointer did not go nil before cleanup was executed") + } + done <- struct{}{} + }, true) + + // Make sure the weak pointer stays around while bt is live. + runtime.GC() + if wt.Value() == nil { + t.Errorf("weak pointer went nil too soon") + } + runtime.KeepAlive(bt) + + // bt is no longer referenced. + // + // Run one cycle to queue the cleanup. + runtime.GC() + if wt.Value() != nil { + t.Errorf("weak pointer did not go nil when cleanup was enqueued") + } + + // Wait for the cleanup to run. + <-done + + // The weak pointer should still be nil after the cleanup runs. + runtime.GC() + if wt.Value() != nil { + t.Errorf("weak pointer is non-nil even after cleanup: %v", wt) + } +} + +func TestPointerSize(t *testing.T) { + var p weak.Pointer[T] + size := unsafe.Sizeof(p) + if size != goarch.PtrSize { + t.Errorf("weak.Pointer[T] size = %d, want %d", size, goarch.PtrSize) + } +} + +// Regression test for issue 69210. +// +// Weak-to-strong conversions must shade the new strong pointer, otherwise +// that might be creating the only strong pointer to a white object which +// is hidden in a blackened stack. +// +// Never fails if correct, fails with some high probability if incorrect. +func TestIssue69210(t *testing.T) { + if testing.Short() { + t.Skip("this is a stress test that takes seconds to run on its own") + } + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + // What we're trying to do is manufacture the conditions under which this + // bug happens. Specifically, we want: + // + // 1. To create a whole bunch of objects that are only weakly-pointed-to, + // 2. To call Value while the GC is in the mark phase, + // 3. The new strong pointer to be missed by the GC, + // 4. The following GC cycle to mark a free object. + // + // Unfortunately, (2) and (3) are hard to control, but we can increase + // the likelihood by having several goroutines do (1) at once while + // another goroutine constantly keeps us in the GC with runtime.GC. + // Like throwing darts at a dart board until they land just right. + // We can increase the likelihood of (4) by adding some delay after + // creating the strong pointer, but only if it's non-nil. If it's nil, + // that means it was already collected in which case there's no chance + // of triggering the bug, so we want to retry as fast as possible. + // Our heap here is tiny, so the GCs will go by fast. + // + // As of 2024-09-03, removing the line that shades pointers during + // the weak-to-strong conversion causes this test to fail about 50% + // of the time. + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for { + runtime.GC() + + select { + case <-ctx.Done(): + return + default: + } + } + }() + for range max(runtime.GOMAXPROCS(-1)-1, 1) { + wg.Add(1) + go func() { + defer wg.Done() + for { + for range 5 { + bt := new(T) + wt := weak.Make(bt) + bt = nil + time.Sleep(1 * time.Millisecond) + bt = wt.Value() + if bt != nil { + time.Sleep(4 * time.Millisecond) + bt.t = bt + bt.a = 12 + } + runtime.KeepAlive(bt) + } + select { + case <-ctx.Done(): + return + default: + } + } + }() + } + wg.Wait() +} + +func TestIssue70739(t *testing.T) { + x := make([]*int, 4<<16) + wx1 := weak.Make(&x[1<<16]) + wx2 := weak.Make(&x[1<<16]) + if wx1 != wx2 { + t.Fatal("failed to look up special and made duplicate weak handle; see issue #70739") + } +} + +var immortal T + +func TestImmortalPointer(t *testing.T) { + w0 := weak.Make(&immortal) + if weak.Make(&immortal) != w0 { + t.Error("immortal weak pointers to the same pointer not equal") + } + w0a := weak.Make(&immortal.a) + w0b := weak.Make(&immortal.b) + if w0a == w0b { + t.Error("separate immortal pointers (same object) have the same pointer") + } + if got, want := w0.Value(), &immortal; got != want { + t.Errorf("immortal weak pointer to %p has unexpected Value %p", want, got) + } + if got, want := w0a.Value(), &immortal.a; got != want { + t.Errorf("immortal weak pointer to %p has unexpected Value %p", want, got) + } + if got, want := w0b.Value(), &immortal.b; got != want { + t.Errorf("immortal weak pointer to %p has unexpected Value %p", want, got) + } + + // Run a couple of cycles. + runtime.GC() + runtime.GC() + + // All immortal weak pointers should never get cleared. + if got, want := w0.Value(), &immortal; got != want { + t.Errorf("immortal weak pointer to %p has unexpected Value %p", want, got) + } + if got, want := w0a.Value(), &immortal.a; got != want { + t.Errorf("immortal weak pointer to %p has unexpected Value %p", want, got) + } + if got, want := w0b.Value(), &immortal.b; got != want { + t.Errorf("immortal weak pointer to %p has unexpected Value %p", want, got) + } +} diff --git a/test/alias2.go b/test/alias2.go index 95eb25a94b5ae2..7a926cc482af8f 100644 --- a/test/alias2.go +++ b/test/alias2.go @@ -46,8 +46,8 @@ var _ A0 = T0{} var _ T0 = A0{} // But aliases and original types cannot be used with new types based on them. -var _ N0 = T0{} // ERROR "cannot use T0{} \(value of type T0\) as N0 value in variable declaration" -var _ N0 = A0{} // ERROR "cannot use A0{} \(value of type A0\) as N0 value in variable declaration" +var _ N0 = T0{} // ERROR "cannot use T0{} \(value of struct type T0\) as N0 value in variable declaration" +var _ N0 = A0{} // ERROR "cannot use A0{} \(value of struct type A0\) as N0 value in variable declaration" var _ A5 = Value{} @@ -82,10 +82,10 @@ func _() { var _ A0 = T0{} var _ T0 = A0{} - var _ N0 = T0{} // ERROR "cannot use T0{} \(value of type T0\) as N0 value in variable declaration" - var _ N0 = A0{} // ERROR "cannot use A0{} \(value of type A0\) as N0 value in variable declaration" + var _ N0 = T0{} // ERROR "cannot use T0{} \(value of struct type T0\) as N0 value in variable declaration" + var _ N0 = A0{} // ERROR "cannot use A0{} \(value of struct type A0\) as N0 value in variable declaration" - var _ A5 = Value{} // ERROR "cannot use Value{} \(value of type reflect\.Value\) as A5 value in variable declaration" + var _ A5 = Value{} // ERROR "cannot use Value{} \(value of struct type reflect\.Value\) as A5 value in variable declaration" } // Invalid type alias declarations. diff --git a/test/assign.go b/test/assign.go index bdec58b710f421..5beffee6f36fc0 100644 --- a/test/assign.go +++ b/test/assign.go @@ -9,7 +9,10 @@ package main -import "sync" +import ( + "sync" + "time" +) type T struct { int @@ -38,7 +41,7 @@ func main() { _ = x } { - x := sync.Mutex{0, 0} // ERROR "assignment.*Mutex" + x := time.Time{0, 0, nil} // ERROR "assignment.*Time" _ = x } { diff --git a/test/checkbce.go b/test/checkbce.go index 71acfb71ac17d4..49d047443e09a4 100644 --- a/test/checkbce.go +++ b/test/checkbce.go @@ -159,16 +159,14 @@ func decode1(data []byte) (x uint64) { } func decode2(data []byte) (x uint64) { - // TODO(rasky): this should behave like decode1 and compile to no - // boundchecks. We're currently not able to remove all of them. for len(data) >= 32 { x += binary.BigEndian.Uint64(data) data = data[8:] - x += binary.BigEndian.Uint64(data) // ERROR "Found IsInBounds$" + x += binary.BigEndian.Uint64(data) data = data[8:] - x += binary.BigEndian.Uint64(data) // ERROR "Found IsInBounds$" + x += binary.BigEndian.Uint64(data) data = data[8:] - x += binary.BigEndian.Uint64(data) // ERROR "Found IsInBounds$" + x += binary.BigEndian.Uint64(data) data = data[8:] } return x diff --git a/test/closure3.dir/main.go b/test/closure3.dir/main.go index e3981a5161a8ce..7018a4734a6593 100644 --- a/test/closure3.dir/main.go +++ b/test/closure3.dir/main.go @@ -20,6 +20,7 @@ func main() { if x := func() int { // ERROR "can inline main.func2" "func literal does not escape" return 1 }; x() != 1 { // ERROR "inlining call to main.func2" + _ = x // prevent simple deadcode elimination after inlining ppanic("x() != 1") } } @@ -33,6 +34,7 @@ func main() { if y := func(x int) int { // ERROR "can inline main.func4" "func literal does not escape" return x + 2 }; y(40) != 42 { // ERROR "inlining call to main.func4" + _ = y // prevent simple deadcode elimination after inlining ppanic("y(40) != 42") } } @@ -50,7 +52,7 @@ func main() { } { - func() { // ERROR "func literal does not escape" + func() { // ERROR "can inline main.func7" y := func(x int) int { // ERROR "can inline main.func7.1" "func literal does not escape" return x + 2 } @@ -60,7 +62,7 @@ func main() { if y(40) != 41 { ppanic("y(40) != 41") } - }() + }() // ERROR "func literal does not escape" "inlining call to main.func7" } { @@ -76,7 +78,7 @@ func main() { } { - func() { // ERROR "func literal does not escape" + func() { // ERROR "can inline main.func10" y := func(x int) int { // ERROR "can inline main.func10.1" "func literal does not escape" return x + 2 } @@ -86,7 +88,7 @@ func main() { if y(40) != 41 { ppanic("y(40) != 41") } - }() + }() // ERROR "func literal does not escape" "inlining call to main.func10" } { @@ -104,11 +106,11 @@ func main() { } { - func() { // ERROR "func literal does not escape" + func() { // ERROR "can inline main.func13" y := func(x int) int { // ERROR "func literal does not escape" "can inline main.func13.1" return x + 2 } - y, sink = func() (func(int) int, int) { // ERROR "can inline main.func13.2" + y, sink = func() (func(int) int, int) { // ERROR "can inline main.func13.2" "can inline main.main.func13.func35" return func(x int) int { // ERROR "can inline main.func13.2" "func literal escapes to heap" return x + 1 }, 42 @@ -116,7 +118,7 @@ func main() { if y(40) != 41 { ppanic("y(40) != 41") } - }() + }() // ERROR "func literal does not escape" "inlining call to main.func13" "inlining call to main.main.func13.func35" } { @@ -132,7 +134,7 @@ func main() { } { - func() { // ERROR "func literal does not escape" + func() { // ERROR "can inline main.func16" y := func(x int) int { // ERROR "can inline main.func16.1" "func literal does not escape" return x + 2 } @@ -142,7 +144,7 @@ func main() { if y(40) != 41 { ppanic("y(40) != 41") } - }() + }() // ERROR "func literal does not escape" "inlining call to main.func16" "map\[int\]func\(int\) int{...} does not escape" "func literal escapes to heap" } { @@ -158,7 +160,7 @@ func main() { } { - func() { // ERROR "func literal does not escape" + func() { // ERROR "can inline main.func19" y := func(x int) int { // ERROR "can inline main.func19.1" "func literal does not escape" return x + 2 } @@ -168,7 +170,7 @@ func main() { if y(40) != 41 { ppanic("y(40) != 41") } - }() + }() // ERROR "func literal does not escape" "inlining call to main.func19" } { @@ -181,6 +183,7 @@ func main() { if y := func() int { // ERROR "can inline main.func21" "func literal does not escape" return x }; y() != 42 { // ERROR "inlining call to main.func21" + _ = y // prevent simple deadcode elimination after inlining ppanic("y() != 42") } } @@ -188,17 +191,18 @@ func main() { { x := 42 if z := func(y int) int { // ERROR "can inline main.func22" - return func() int { // ERROR "can inline main.func22.1" "can inline main.main.func22.func30" + return func() int { // ERROR "can inline main.func22.1" "can inline main.main.func22.func40" return x + y }() // ERROR "inlining call to main.func22.1" - }(1); z != 43 { // ERROR "inlining call to main.func22" "inlining call to main.main.func22.func30" + }(1); z != 43 { // ERROR "inlining call to main.func22" "inlining call to main.main.func22.func40" ppanic("z != 43") } if z := func(y int) int { // ERROR "func literal does not escape" "can inline main.func23" - return func() int { // ERROR "can inline main.func23.1" "can inline main.main.func23.func31" + return func() int { // ERROR "can inline main.func23.1" "can inline main.main.func23.func41" return x + y }() // ERROR "inlining call to main.func23.1" - }; z(1) != 43 { // ERROR "inlining call to main.func23" "inlining call to main.main.func23.func31" + }; z(1) != 43 { // ERROR "inlining call to main.func23" "inlining call to main.main.func23.func41" + _ = z // prevent simple deadcode elimination after inlining ppanic("z(1) != 43") } } @@ -206,10 +210,10 @@ func main() { { a := 1 func() { // ERROR "can inline main.func24" - func() { // ERROR "can inline main.func24" "can inline main.main.func24.func32" + func() { // ERROR "can inline main.func24" "can inline main.main.func24.func42" a = 2 }() // ERROR "inlining call to main.func24" - }() // ERROR "inlining call to main.func24" "inlining call to main.main.func24.func32" + }() // ERROR "inlining call to main.func24" "inlining call to main.main.func24.func42" if a != 2 { ppanic("a != 2") } @@ -217,14 +221,14 @@ func main() { { b := 2 - func(b int) { // ERROR "func literal does not escape" - func() { // ERROR "can inline main.func25.1" + func(b int) { // ERROR "can inline main.func25" + func() { // ERROR "can inline main.func25.1" "can inline main.main.func25.func43" b = 3 }() // ERROR "inlining call to main.func25.1" if b != 3 { ppanic("b != 3") } - }(b) + }(b) // ERROR "inlining call to main.func25" "inlining call to main.main.func25.func43" if b != 2 { ppanic("b != 2") } @@ -254,13 +258,13 @@ func main() { // revisit those. E.g., func34 and func36 are constructed by the inliner. if r := func(x int) int { // ERROR "can inline main.func27" b := 3 - return func(y int) int { // ERROR "can inline main.func27.1" "can inline main.main.func27.func34" + return func(y int) int { // ERROR "can inline main.func27.1" "can inline main.main.func27.func45" c := 5 - return func(z int) int { // ERROR "can inline main.func27.1.1" "can inline main.main.func27.func34.1" "can inline main.func27.main.func27.1.2" "can inline main.main.func27.main.main.func27.func34.func36" + return func(z int) int { // ERROR "can inline main.func27.1.1" "can inline main.main.func27.func45.1" "can inline main.func27.main.func27.1.2" "can inline main.main.func27.main.main.func27.func45.func48" return a*x + b*y + c*z }(10) // ERROR "inlining call to main.func27.1.1" }(100) // ERROR "inlining call to main.func27.1" "inlining call to main.func27.main.func27.1.2" - }(1000); r != 2350 { // ERROR "inlining call to main.func27" "inlining call to main.main.func27.func34" "inlining call to main.main.func27.main.main.func27.func34.func36" + }(1000); r != 2350 { // ERROR "inlining call to main.func27" "inlining call to main.main.func27.func45" "inlining call to main.main.func27.main.main.func27.func45.func48" ppanic("r != 2350") } } @@ -269,16 +273,16 @@ func main() { a := 2 if r := func(x int) int { // ERROR "can inline main.func28" b := 3 - return func(y int) int { // ERROR "can inline main.func28.1" "can inline main.main.func28.func35" + return func(y int) int { // ERROR "can inline main.func28.1" "can inline main.main.func28.func46" c := 5 - func(z int) { // ERROR "can inline main.func28.1.1" "can inline main.func28.main.func28.1.2" "can inline main.main.func28.func35.1" "can inline main.main.func28.main.main.func28.func35.func37" + func(z int) { // ERROR "can inline main.func28.1.1" "can inline main.func28.main.func28.1.2" "can inline main.main.func28.func46.1" "can inline main.main.func28.main.main.func28.func46.func49" a = a * x b = b * y c = c * z }(10) // ERROR "inlining call to main.func28.1.1" return a + c }(100) + b // ERROR "inlining call to main.func28.1" "inlining call to main.func28.main.func28.1.2" - }(1000); r != 2350 { // ERROR "inlining call to main.func28" "inlining call to main.main.func28.func35" "inlining call to main.main.func28.main.main.func28.func35.func37" + }(1000); r != 2350 { // ERROR "inlining call to main.func28" "inlining call to main.main.func28.func46" "inlining call to main.main.func28.main.main.func28.func46.func49" ppanic("r != 2350") } if a != 2000 { @@ -287,6 +291,25 @@ func main() { } } +//go:noinline +func notmain() { + { + // This duplicates the first block in main, but without the "_ = x" for closure x. + // This allows dead code elimination of x before escape analysis, + // thus "func literal does not escape" should not appear. + if x := func() int { // ERROR "can inline notmain.func1" + return 1 + }(); x != 1 { // ERROR "inlining call to notmain.func1" + ppanic("x != 1") + } + if x := func() int { // ERROR "can inline notmain.func2" + return 1 + }; x() != 1 { // ERROR "inlining call to notmain.func2" + ppanic("x() != 1") + } + } +} + //go:noinline func ppanic(s string) { // ERROR "leaking param: s" panic(s) // ERROR "s escapes to heap" diff --git a/test/codegen/README b/test/codegen/README index 19a73d031c7670..33b9fbc49c0e32 100644 --- a/test/codegen/README +++ b/test/codegen/README @@ -98,6 +98,13 @@ For example: verifies that NO memmove call is present in the assembly generated for the copy() line. +The expected number of matches for the regexp can be specified using a +positive number: + + func fb(a [4]int) (r [4]int) { + // amd64:2`MOVUPS[^,]+, X0$`,2`MOVUPS\sX0,[^\n]+$` + return a + } - Architecture specifiers diff --git a/test/codegen/alloc.go b/test/codegen/alloc.go index 31455fdabfe626..c6ff004eed36e9 100644 --- a/test/codegen/alloc.go +++ b/test/codegen/alloc.go @@ -10,25 +10,28 @@ package codegen func zeroAllocNew1() *struct{} { - // 386:-`CALL\truntime\.newobject` - // amd64:-`CALL\truntime\.newobject` - // arm:-`CALL\truntime\.newobject` - // arm64:-`CALL\truntime\.newobject` + // 386:-`CALL\truntime\.newobject`, `LEAL\truntime.zerobase` + // amd64:-`CALL\truntime\.newobject`, `LEAQ\truntime.zerobase` + // arm:-`CALL\truntime\.newobject`, `MOVW\t[$]runtime.zerobase` + // arm64:-`CALL\truntime\.newobject`, `MOVD\t[$]runtime.zerobase` + // riscv64:-`CALL\truntime\.newobject`, `MOV\t[$]runtime.zerobase` return new(struct{}) } func zeroAllocNew2() *[0]int { - // 386:-`CALL\truntime\.newobject` - // amd64:-`CALL\truntime\.newobject` - // arm:-`CALL\truntime\.newobject` - // arm64:-`CALL\truntime\.newobject` + // 386:-`CALL\truntime\.newobject`, `LEAL\truntime.zerobase` + // amd64:-`CALL\truntime\.newobject`, `LEAQ\truntime.zerobase` + // arm:-`CALL\truntime\.newobject`, `MOVW\t[$]runtime.zerobase` + // arm64:-`CALL\truntime\.newobject`, `MOVD\t[$]runtime.zerobase` + // riscv64:-`CALL\truntime\.newobject`, `MOV\t[$]runtime.zerobase` return new([0]int) } func zeroAllocSliceLit() []int { - // 386:-`CALL\truntime\.newobject` - // amd64:-`CALL\truntime\.newobject` - // arm:-`CALL\truntime\.newobject` - // arm64:-`CALL\truntime\.newobject` + // 386:-`CALL\truntime\.newobject`, `LEAL\truntime.zerobase` + // amd64:-`CALL\truntime\.newobject`, `LEAQ\truntime.zerobase` + // arm:-`CALL\truntime\.newobject`, `MOVW\t[$]runtime.zerobase` + // arm64:-`CALL\truntime\.newobject`, `MOVD\t[$]runtime.zerobase` + // riscv64:-`CALL\truntime\.newobject`, `MOV\t[$]runtime.zerobase` return []int{} } diff --git a/test/codegen/arithmetic.go b/test/codegen/arithmetic.go index e474a10ba2a793..7055db3dc9f72f 100644 --- a/test/codegen/arithmetic.go +++ b/test/codegen/arithmetic.go @@ -51,6 +51,11 @@ func AddLargeConst(a uint64, out []uint64) { out[9] = a - 32769 } +func AddLargeConst2(a int, out []int) { + // loong64: -"ADDVU","ADDV16" + out[0] = a + 0x10000 +} + // ----------------- // // Subtraction // // ----------------- // @@ -85,56 +90,92 @@ func SubMem(arr []int, b, c, d int) int { func SubFromConst(a int) int { // ppc64x: `SUBC\tR[0-9]+,\s[$]40,\sR` + // riscv64: "ADDI\t\\$-40","NEG" b := 40 - a return b } func SubFromConstNeg(a int) int { + // arm64: "ADD\t\\$40" + // loong64: "ADDV[U]\t\\$40" + // mips: "ADD[U]\t\\$40" + // mips64: "ADDV[U]\t\\$40" // ppc64x: `ADD\t[$]40,\sR[0-9]+,\sR` + // riscv64: "ADDI\t\\$40",-"NEG" c := 40 - (-a) return c } func SubSubFromConst(a int) int { + // arm64: "ADD\t\\$20" + // loong64: "ADDV[U]\t\\$20" + // mips: "ADD[U]\t\\$20" + // mips64: "ADDV[U]\t\\$20" // ppc64x: `ADD\t[$]20,\sR[0-9]+,\sR` + // riscv64: "ADDI\t\\$20",-"NEG" c := 40 - (20 - a) return c } func AddSubFromConst(a int) int { // ppc64x: `SUBC\tR[0-9]+,\s[$]60,\sR` + // riscv64: "ADDI\t\\$-60","NEG" c := 40 + (20 - a) return c } func NegSubFromConst(a int) int { + // arm64: "SUB\t\\$20" + // loong64: "ADDV[U]\t\\$-20" + // mips: "ADD[U]\t\\$-20" + // mips64: "ADDV[U]\t\\$-20" // ppc64x: `ADD\t[$]-20,\sR[0-9]+,\sR` + // riscv64: "ADDI\t\\$-20" c := -(20 - a) return c } func NegAddFromConstNeg(a int) int { + // arm64: "SUB\t\\$40","NEG" + // loong64: "ADDV[U]\t\\$-40","SUBV" + // mips: "ADD[U]\t\\$-40","SUB" + // mips64: "ADDV[U]\t\\$-40","SUBV" // ppc64x: `SUBC\tR[0-9]+,\s[$]40,\sR` + // riscv64: "ADDI\t\\$-40","NEG" c := -(-40 + a) return c } func SubSubNegSimplify(a, b int) int { // amd64:"NEGQ" + // arm64:"NEG" + // loong64:"SUBV" + // mips:"SUB" + // mips64:"SUBV" // ppc64x:"NEG" + // riscv64:"NEG",-"SUB" r := (a - b) - a return r } func SubAddSimplify(a, b int) int { // amd64:-"SUBQ",-"ADDQ" + // arm64:-"SUB",-"ADD" + // loong64:-"SUBV",-"ADDV" + // mips:-"SUB",-"ADD" + // mips64:-"SUBV",-"ADDV" // ppc64x:-"SUB",-"ADD" + // riscv64:-"SUB",-"ADD" r := a + (b - a) return r } func SubAddSimplify2(a, b, c int) (int, int, int, int, int, int) { // amd64:-"ADDQ" + // arm64:-"ADD" + // mips:"SUB",-"ADD" + // mips64:"SUBV",-"ADDV" + // loong64:"SUBV",-"ADDV" r := (a + b) - (a + c) // amd64:-"ADDQ" r1 := (a + b) - (c + a) @@ -143,6 +184,10 @@ func SubAddSimplify2(a, b, c int) (int, int, int, int, int, int) { // amd64:-"ADDQ" r3 := (b + a) - (c + a) // amd64:-"SUBQ" + // arm64:-"SUB" + // mips:"ADD",-"SUB" + // mips64:"ADDV",-"SUBV" + // loong64:"ADDV",-"SUBV" r4 := (a - c) + (c + b) // amd64:-"SUBQ" r5 := (a - c) + (b + c) @@ -151,18 +196,34 @@ func SubAddSimplify2(a, b, c int) (int, int, int, int, int, int) { func SubAddNegSimplify(a, b int) int { // amd64:"NEGQ",-"ADDQ",-"SUBQ" + // arm64:"NEG",-"ADD",-"SUB" + // loong64:"SUBV",-"ADDV" + // mips:"SUB",-"ADD" + // mips64:"SUBV",-"ADDV" // ppc64x:"NEG",-"ADD",-"SUB" + // riscv64:"NEG",-"ADD",-"SUB" r := a - (b + a) return r } func AddAddSubSimplify(a, b, c int) int { // amd64:-"SUBQ" + // arm64:"ADD",-"SUB" + // loong64:"ADDV",-"SUBV" + // mips:"ADD",-"SUB" + // mips64:"ADDV",-"SUBV" // ppc64x:-"SUB" + // riscv64:"ADD","ADD",-"SUB" r := a + (b + (c - a)) return r } +func NegToInt32(a int) int { + // riscv64: "NEGW",-"MOVW" + r := int(int32(-a)) + return r +} + // -------------------- // // Multiplication // // -------------------- // @@ -172,6 +233,7 @@ func Pow2Muls(n1, n2 int) (int, int) { // 386:"SHLL\t[$]5",-"IMULL" // arm:"SLL\t[$]5",-"MUL" // arm64:"LSL\t[$]5",-"MUL" + // loong64:"SLLV\t[$]5",-"MULV" // ppc64x:"SLD\t[$]5",-"MUL" a := n1 * 32 @@ -179,22 +241,34 @@ func Pow2Muls(n1, n2 int) (int, int) { // 386:"SHLL\t[$]6",-"IMULL" // arm:"SLL\t[$]6",-"MUL" // arm64:`NEG\sR[0-9]+<<6,\sR[0-9]+`,-`LSL`,-`MUL` + // loong64:"SLLV\t[$]6",-"MULV" // ppc64x:"SLD\t[$]6","NEG\\sR[0-9]+,\\sR[0-9]+",-"MUL" b := -64 * n2 return a, b } +func Mul_2(n1 int32, n2 int64) (int32, int64) { + // amd64:"ADDL", -"SHLL" + a := n1 * 2 + // amd64:"ADDQ", -"SHLQ" + b := n2 * 2 + + return a, b +} + func Mul_96(n int) int { // amd64:`SHLQ\t[$]5`,`LEAQ\t\(.*\)\(.*\*2\),`,-`IMULQ` // 386:`SHLL\t[$]5`,`LEAL\t\(.*\)\(.*\*2\),`,-`IMULL` // arm64:`LSL\t[$]5`,`ADD\sR[0-9]+<<1,\sR[0-9]+`,-`MUL` // arm:`SLL\t[$]5`,`ADD\sR[0-9]+<<1,\sR[0-9]+`,-`MUL` + // loong64:"SLLV\t[$]5","ALSLV\t[$]1," // s390x:`SLD\t[$]5`,`SLD\t[$]6`,-`MULLD` return n * 96 } func Mul_n120(n int) int { + // loong64:"SLLV\t[$]3","SLLV\t[$]7","SUBVU",-"MULV" // s390x:`SLD\t[$]3`,`SLD\t[$]7`,-`MULLD` return n * -120 } @@ -245,6 +319,18 @@ func MergeMuls5(a, n int) int { return a*n - 19*n // (a-19)n } +// Multiplications folded negation + +func FoldNegMul(a int) int { + // loong64:"SUBVU","ALSLV\t[$]2","ALSLV\t[$]1" + return (-a) * 11 +} + +func Fold2NegMul(a, b int) int { + // loong64:"MULV",-"SUBVU\tR[0-9], R0," + return (-a) * (-b) +} + // -------------- // // Division // // -------------- // @@ -543,6 +629,11 @@ func AddMul(x int) int { return 2*x + 1 } +func AddShift(a, b int) int { + // loong64: "ALSLV" + return a + (b << 4) +} + func MULA(a, b, c uint32) (uint32, uint32, uint32) { // arm:`MULA`,-`MUL\s` // arm64:`MADDW`,-`MULW` @@ -624,8 +715,44 @@ func constantFold2(i0, j0, i1, j1 int) (int, int) { } func constantFold3(i, j int) int { - // arm64: "MOVD\t[$]30","MUL",-"ADD",-"LSL" + // arm64: "LSL\t[$]5,","SUB\tR[0-9]+<<1,",-"ADD" // ppc64x:"MULLD\t[$]30","MULLD" r := (5 * i) * (6 * j) return r } + +// ----------------- // +// Integer Min/Max // +// ----------------- // + +func Int64Min(a, b int64) int64 { + // amd64: "CMPQ","CMOVQLT" + // arm64: "CMP","CSEL" + // riscv64/rva20u64:"BLT\t" + // riscv64/rva22u64,riscv64/rva23u64:"MIN\t" + return min(a, b) +} + +func Int64Max(a, b int64) int64 { + // amd64: "CMPQ","CMOVQGT" + // arm64: "CMP","CSEL" + // riscv64/rva20u64:"BLT\t" + // riscv64/rva22u64,riscv64/rva23u64:"MAX\t" + return max(a, b) +} + +func Uint64Min(a, b uint64) uint64 { + // amd64: "CMPQ","CMOVQCS" + // arm64: "CMP","CSEL" + // riscv64/rva20u64:"BLTU" + // riscv64/rva22u64,riscv64/rva23u64:"MINU" + return min(a, b) +} + +func Uint64Max(a, b uint64) uint64 { + // amd64: "CMPQ","CMOVQHI" + // arm64: "CMP","CSEL" + // riscv64/rva20u64:"BLTU" + // riscv64/rva22u64,riscv64/rva23u64:"MAXU" + return max(a, b) +} diff --git a/test/codegen/bitfield.go b/test/codegen/bitfield.go index 3276af3de03b5e..51221266e1097b 100644 --- a/test/codegen/bitfield.go +++ b/test/codegen/bitfield.go @@ -338,31 +338,41 @@ func shift_no_cmp(x int) int { func rev16(c uint64) (uint64, uint64, uint64) { // arm64:`REV16`,-`AND`,-`LSR`,-`AND`,-`ORR\tR[0-9]+<<8` + // loong64:`REVB4H`,-`MOVV`,-`AND`,-`SRLV`,-`AND`,-`SLLV`,-`OR` b1 := ((c & 0xff00ff00ff00ff00) >> 8) | ((c & 0x00ff00ff00ff00ff) << 8) // arm64:-`ADD\tR[0-9]+<<8` + // loong64:-`ADDV` b2 := ((c & 0xff00ff00ff00ff00) >> 8) + ((c & 0x00ff00ff00ff00ff) << 8) // arm64:-`EOR\tR[0-9]+<<8` + // loong64:-`XOR` b3 := ((c & 0xff00ff00ff00ff00) >> 8) ^ ((c & 0x00ff00ff00ff00ff) << 8) return b1, b2, b3 } func rev16w(c uint32) (uint32, uint32, uint32) { // arm64:`REV16W`,-`AND`,-`UBFX`,-`AND`,-`ORR\tR[0-9]+<<8` + // loong64:`REVB2H`,-`AND`,-`SRL`,-`AND`,-`SLL`,-`OR` b1 := ((c & 0xff00ff00) >> 8) | ((c & 0x00ff00ff) << 8) // arm64:-`ADD\tR[0-9]+<<8` + // loong64:-`ADDV` b2 := ((c & 0xff00ff00) >> 8) + ((c & 0x00ff00ff) << 8) // arm64:-`EOR\tR[0-9]+<<8` + // loong64:-`XOR` b3 := ((c & 0xff00ff00) >> 8) ^ ((c & 0x00ff00ff) << 8) return b1, b2, b3 } func shift(x uint32, y uint16, z uint8) uint64 { // arm64:-`MOVWU`,-`LSR\t[$]32` + // loong64:-`MOVWU`,-`SRLV\t[$]32` a := uint64(x) >> 32 // arm64:-`MOVHU + // loong64:-`MOVHU`,-`SRLV\t[$]16` b := uint64(y) >> 16 // arm64:-`MOVBU` + // loong64:-`MOVBU`,-`SRLV\t[$]8` c := uint64(z) >> 8 // arm64:`MOVD\tZR`,-`ADD\tR[0-9]+>>16`,-`ADD\tR[0-9]+>>8`, + // loong64:`MOVV\tR0`,-`ADDVU` return a + b + c } diff --git a/test/codegen/bits.go b/test/codegen/bits.go index 554e363ef52119..7974f471fcf526 100644 --- a/test/codegen/bits.go +++ b/test/codegen/bits.go @@ -120,6 +120,16 @@ func bitoff64(a, b uint64) (n uint64) { return n } +func clearLastBit(x int64, y int32) (int64, int32) { + // amd64:"ANDQ\t[$]-2" + a := (x >> 1) << 1 + + // amd64:"ANDL\t[$]-2" + b := (y >> 1) << 1 + + return a, b +} + func bitcompl64(a, b uint64) (n uint64) { // amd64:"BTCQ" n += b ^ (1 << (a & 63)) @@ -322,9 +332,24 @@ func op_eon(x, y, z uint32, a []uint32, n, m uint64) uint64 { func op_orn(x, y uint32) uint32 { // arm64:`ORN\t`,-`ORR` + // loong64:"ORN"\t,-"OR\t" return x | ^y } +func op_nor(x int64, a []int64) { + // loong64: "MOVV\t[$]0","NOR\tR" + a[0] = ^(0x1234 | x) + // loong64:"NOR",-"XOR" + a[1] = (-1) ^ x + // loong64: "MOVV\t[$]-55",-"OR",-"NOR" + a[2] = ^(0x12 | 0x34) +} + +func op_andn(x, y uint32) uint32 { + // loong64:"ANDN\t",-"AND\t" + return x &^ y +} + // check bitsets func bitSetPowerOf2Test(x int) bool { // amd64:"BTL\t[$]3" @@ -366,6 +391,7 @@ func issue48467(x, y uint64) uint64 { func foldConst(x, y uint64) uint64 { // arm64: "ADDS\t[$]7",-"MOVD\t[$]7" + // ppc64x: "ADDC\t[$]7," d, b := bits.Add64(x, 7, 0) return b & d } diff --git a/test/codegen/bool.go b/test/codegen/bool.go index 990a9ed1b1d8c7..760dbbcf7b55a7 100644 --- a/test/codegen/bool.go +++ b/test/codegen/bool.go @@ -47,6 +47,7 @@ func convertNeqBool32(x uint32) bool { func convertEqBool32(x uint32) bool { // ppc64x:"RLDICL",-"CMPW","XOR",-"ISEL" + // amd64:"ANDL","XORL",-"BTL",-"SETCC" return x&1 == 0 } @@ -57,9 +58,34 @@ func convertNeqBool64(x uint64) bool { func convertEqBool64(x uint64) bool { // ppc64x:"RLDICL","XOR",-"CMP",-"ISEL" + // amd64:"ANDL","XORL",-"BTL",-"SETCC" return x&1 == 0 } +func phiAnd(a, b bool) bool { + var x bool + // amd64:-"TESTB" + if a { + x = b + } else { + x = a + } + // amd64:"ANDL" + return x +} + +func phiOr(a, b bool) bool { + var x bool + // amd64:-"TESTB" + if a { + x = a + } else { + x = b + } + // amd64:"ORL" + return x +} + func TestSetEq64(x uint64, y uint64) bool { // ppc64x/power10:"SETBC\tCR0EQ",-"ISEL" // ppc64x/power9:"CMP","ISEL",-"SETBC\tCR0EQ" @@ -217,53 +243,53 @@ func TestSetInvGeFp64(x float64, y float64) bool { } func TestLogicalCompareZero(x *[64]uint64) { // ppc64x:"ANDCC",^"AND" - b := x[0]&3 - if b!=0 { + b := x[0] & 3 + if b != 0 { x[0] = b } // ppc64x:"ANDCC",^"AND" - b = x[1]&x[2] - if b!=0 { + b = x[1] & x[2] + if b != 0 { x[1] = b } // ppc64x:"ANDNCC",^"ANDN" - b = x[1]&^x[2] - if b!=0 { + b = x[1] &^ x[2] + if b != 0 { x[1] = b } // ppc64x:"ORCC",^"OR" - b = x[3]|x[4] - if b!=0 { + b = x[3] | x[4] + if b != 0 { x[3] = b } // ppc64x:"SUBCC",^"SUB" - b = x[5]-x[6] - if b!=0 { + b = x[5] - x[6] + if b != 0 { x[5] = b } // ppc64x:"NORCC",^"NOR" - b = ^(x[5]|x[6]) - if b!=0 { + b = ^(x[5] | x[6]) + if b != 0 { x[5] = b } // ppc64x:"XORCC",^"XOR" - b = x[7]^x[8] - if b!=0 { + b = x[7] ^ x[8] + if b != 0 { x[7] = b } // ppc64x:"ADDCC",^"ADD" - b = x[9]+x[10] - if b!=0 { + b = x[9] + x[10] + if b != 0 { x[9] = b } // ppc64x:"NEGCC",^"NEG" b = -x[11] - if b!=0 { + if b != 0 { x[11] = b } // ppc64x:"CNTLZDCC",^"CNTLZD" b = uint64(bits.LeadingZeros64(x[12])) - if b!=0 { + if b != 0 { x[12] = b } @@ -273,4 +299,17 @@ func TestLogicalCompareZero(x *[64]uint64) { x[12] = uint64(c) } + // ppc64x:"MULHDUCC",^"MULHDU" + hi, _ := bits.Mul64(x[13], x[14]) + if hi != 0 { + x[14] = hi + } + +} + +func constantWrite(b bool, p *bool) { + if b { + // amd64:`MOVB\t[$]1, \(` + *p = b + } } diff --git a/test/codegen/compare_and_branch.go b/test/codegen/compare_and_branch.go index c121f1d2cc235a..759dd263581885 100644 --- a/test/codegen/compare_and_branch.go +++ b/test/codegen/compare_and_branch.go @@ -241,4 +241,14 @@ func ui64x0(x chan uint64) { for <-x < 1 { dummy() } + + // riscv64:"BNEZ" + for 0 < <-x { + dummy() + } + + // riscv64:"BEQZ" + for 0 >= <-x { + dummy() + } } diff --git a/test/codegen/comparisons.go b/test/codegen/comparisons.go index e585045aa4730e..702ea275cc9b39 100644 --- a/test/codegen/comparisons.go +++ b/test/codegen/comparisons.go @@ -233,9 +233,9 @@ func CmpToZero(a, b, d int32, e, f int64, deOptC0, deOptC1 bool) int32 { // arm:`AND`,-`TST` // 386:`ANDL` c6 := a&d >= 0 - // arm64:`TST\sR[0-9]+<<3,\sR[0-9]+` + // For arm64, could be TST+BGE or AND+TBZ c7 := e&(f<<3) < 0 - // arm64:`CMN\sR[0-9]+<<3,\sR[0-9]+` + // For arm64, could be CMN+BPL or ADD+TBZ c8 := e+(f<<3) < 0 // arm64:`TST\sR[0-9],\sR[0-9]+` c9 := e&(-19) < 0 @@ -268,7 +268,7 @@ func CmpToZero(a, b, d int32, e, f int64, deOptC0, deOptC1 bool) int32 { } } -func CmpLogicalToZero(a, b, c uint32, d, e uint64) uint64 { +func CmpLogicalToZero(a, b, c uint32, d, e, f, g uint64) uint64 { // ppc64x:"ANDCC",-"CMPW" // wasm:"I64Eqz",-"I32Eqz",-"I64ExtendI32U",-"I32WrapI64" @@ -289,7 +289,7 @@ func CmpLogicalToZero(a, b, c uint32, d, e uint64) uint64 { } // ppc64x:"ORCC",-"CMP" // wasm:"I64Eqz",-"I32Eqz",-"I64ExtendI32U",-"I32WrapI64" - if d|e == 0 { + if f|g == 0 { return 1 } @@ -730,6 +730,54 @@ func cmpToCmnLessThan(a, b, c, d int) int { return c1 + c2 + c3 + c4 } +func less128Signed32(x int32) bool { + // amd64:`CMPL.*127` + // amd64:`SETLE` + return x < 128 +} + +func less128Signed64(x int64) bool { + // amd64:`CMPQ.*127` + // amd64:`SETLE` + return x < 128 +} + +func less128Unsigned32(x uint32) bool { + // amd64:`CMPL.*127` + // amd64:`SETLS` + return x < 128 +} + +func less128Unsigned64(x uint64) bool { + // amd64:`CMPQ.*127` + // amd64:`SETLS` + return x < 128 +} + +func ge128Unsigned32(x uint32) bool { + // amd64:`CMPL.*127` + // amd64:`SETHI` + return x >= 128 +} + +func ge128Unsigned64(x uint64) bool { + // amd64:`CMPQ.*127` + // amd64:`SETHI` + return x >= 128 +} + +func ge128Signed32(x int32) bool { + // amd64:`CMPL.*127` + // amd64:`SETGT` + return x >= 128 +} + +func ge128Signed64(x int64) bool { + // amd64:`CMPQ.*127` + // amd64:`SETGT` + return x >= 128 +} + func cmpToCmnGreaterThanEqual(a, b, c, d int) int { var c1, c2, c3, c4 int // arm64:`CMN`,`CSET\tPL`,-`CMP` diff --git a/test/codegen/condmove.go b/test/codegen/condmove.go index 1058910307ce4b..97be0ced75d19d 100644 --- a/test/codegen/condmove.go +++ b/test/codegen/condmove.go @@ -106,7 +106,7 @@ func cmovfloatint2(x, y float64) float64 { for r >= y { rfr, rexp := frexp(r) if rfr < yfr { - rexp = rexp - 1 + rexp = rexp - 42 } // amd64:"CMOVQHI" // arm64:"CSEL\tMI" @@ -205,7 +205,7 @@ func cmovinvert6(x, y uint64) uint64 { func cmovload(a []int, i int, b bool) int { if b { - i++ + i += 42 } // See issue 26306 // amd64:-"CMOVQNE" @@ -214,7 +214,7 @@ func cmovload(a []int, i int, b bool) int { func cmovstore(a []int, i int, b bool) { if b { - i++ + i += 42 } // amd64:"CMOVQNE" a[i] = 7 @@ -451,3 +451,59 @@ func cmovzeroreg1(a, b int) int { // ppc64x:"ISEL\t[$]2, R0, R[0-9]+, R[0-9]+" return x } + +func cmovmathadd(a uint, b bool) uint { + if b { + a++ + } + // amd64:"ADDQ", -"CMOV" + // arm64:"CSINC", -"CSEL" + // ppc64x:"ADD", -"ISEL" + // wasm:"I64Add", -"Select" + return a +} + +func cmovmathsub(a uint, b bool) uint { + if b { + a-- + } + // amd64:"SUBQ", -"CMOV" + // arm64:"SUB", -"CSEL" + // ppc64x:"SUB", -"ISEL" + // wasm:"I64Sub", -"Select" + return a +} + +func cmovmathdouble(a uint, b bool) uint { + if b { + a *= 2 + } + // amd64:"SHL", -"CMOV" + // amd64/v3:"SHL", -"CMOV", -"MOV" + // arm64:"LSL", -"CSEL" + // wasm:"I64Shl", -"Select" + return a +} + +func cmovmathhalvei(a int, b bool) int { + if b { + // For some reason the compiler attributes the shift to inside this block rather than where the Phi node is. + // arm64:"ASR", -"CSEL" + // wasm:"I64ShrS", -"Select" + a /= 2 + } + // arm64:-"CSEL" + // wasm:-"Select" + return a +} + +func cmovmathhalveu(a uint, b bool) uint { + if b { + a /= 2 + } + // amd64:"SHR", -"CMOV" + // amd64/v3:"SHR", -"CMOV", -"MOV" + // arm64:"LSR", -"CSEL" + // wasm:"I64ShrU", -"Select" + return a +} diff --git a/test/codegen/constants.go b/test/codegen/constants.go index 3ce17d0ad3a65b..0935a9e53a9d87 100644 --- a/test/codegen/constants.go +++ b/test/codegen/constants.go @@ -7,7 +7,7 @@ package codegen // A uint16 or sint16 constant shifted left. -func shifted16BitConstants(out [64]uint64) { +func shifted16BitConstants() (out [64]uint64) { // ppc64x: "MOVD\t[$]8193,", "SLD\t[$]27," out[0] = 0x0000010008000000 // ppc64x: "MOVD\t[$]-32767", "SLD\t[$]26," @@ -16,10 +16,11 @@ func shifted16BitConstants(out [64]uint64) { out[2] = 0xFFFF000000000000 // ppc64x: "MOVD\t[$]65535", "SLD\t[$]44," out[3] = 0x0FFFF00000000000 + return } // A contiguous set of 1 bits, potentially wrapping. -func contiguousMaskConstants(out [64]uint64) { +func contiguousMaskConstants() (out [64]uint64) { // ppc64x: "MOVD\t[$]-1", "RLDC\tR[0-9]+, [$]44, [$]63," out[0] = 0xFFFFF00000000001 // ppc64x: "MOVD\t[$]-1", "RLDC\tR[0-9]+, [$]43, [$]63," @@ -30,4 +31,5 @@ func contiguousMaskConstants(out [64]uint64) { // ppc64x/power9: "MOVD\t[$]-1", "RLDC\tR[0-9]+, [$]33, [$]63," // ppc64x/power10: "MOVD\t[$]-8589934591," out[3] = 0xFFFFFFFE00000001 + return } diff --git a/test/codegen/copy.go b/test/codegen/copy.go index 17ee8bc8076103..4329c6d78f3c30 100644 --- a/test/codegen/copy.go +++ b/test/codegen/copy.go @@ -95,6 +95,7 @@ func moveArchLowering1(b []byte, x *[1]byte) { _ = b[1] // amd64:-".*memmove" // arm64:-".*memmove" + // loong64:-".*memmove" // ppc64x:-".*memmove" copy(b, x[:]) } @@ -103,6 +104,7 @@ func moveArchLowering2(b []byte, x *[2]byte) { _ = b[2] // amd64:-".*memmove" // arm64:-".*memmove" + // loong64:-".*memmove" // ppc64x:-".*memmove" copy(b, x[:]) } @@ -111,6 +113,7 @@ func moveArchLowering4(b []byte, x *[4]byte) { _ = b[4] // amd64:-".*memmove" // arm64:-".*memmove" + // loong64:-".*memmove" // ppc64x:-".*memmove" copy(b, x[:]) } diff --git a/test/codegen/floats.go b/test/codegen/floats.go index d38df1cacbb8a7..b453cabf3b44e0 100644 --- a/test/codegen/floats.go +++ b/test/codegen/floats.go @@ -19,6 +19,7 @@ func Mul2(f float64) float64 { // amd64:"ADDSD",-"MULSD" // arm/7:"ADDD",-"MULD" // arm64:"FADDD",-"FMULD" + // loong64:"ADDD",-"MULD" // ppc64x:"FADD",-"FMUL" // riscv64:"FADDD",-"FMULD" return f * 2.0 @@ -29,6 +30,7 @@ func DivPow2(f1, f2, f3 float64) (float64, float64, float64) { // amd64:"MULSD",-"DIVSD" // arm/7:"MULD",-"DIVD" // arm64:"FMULD",-"FDIVD" + // loong64:"MULD",-"DIVD" // ppc64x:"FMUL",-"FDIV" // riscv64:"FMULD",-"FDIVD" x := f1 / 16.0 @@ -37,6 +39,7 @@ func DivPow2(f1, f2, f3 float64) (float64, float64, float64) { // amd64:"MULSD",-"DIVSD" // arm/7:"MULD",-"DIVD" // arm64:"FMULD",-"FDIVD" + // loong64:"MULD",-"DIVD" // ppc64x:"FMUL",-"FDIVD" // riscv64:"FMULD",-"FDIVD" y := f2 / 0.125 @@ -45,6 +48,7 @@ func DivPow2(f1, f2, f3 float64) (float64, float64, float64) { // amd64:"ADDSD",-"DIVSD",-"MULSD" // arm/7:"ADDD",-"MULD",-"DIVD" // arm64:"FADDD",-"FMULD",-"FDIVD" + // loong64:"ADDD",-"MULD",-"DIVD" // ppc64x:"FADD",-"FMUL",-"FDIV" // riscv64:"FADDD",-"FMULD",-"FDIVD" z := f3 / 0.5 @@ -54,11 +58,13 @@ func DivPow2(f1, f2, f3 float64) (float64, float64, float64) { func indexLoad(b0 []float32, b1 float32, idx int) float32 { // arm64:`FMOVS\s\(R[0-9]+\)\(R[0-9]+<<2\),\sF[0-9]+` + // loong64:`MOVF\s\(R[0-9]+\)\(R[0-9]+\),\sF[0-9]+` return b0[idx] * b1 } func indexStore(b0 []float64, b1 float64, idx int) { // arm64:`FMOVD\sF[0-9]+,\s\(R[0-9]+\)\(R[0-9]+<<3\)` + // loong64:`MOVD\sF[0-9]+,\s\(R[0-9]+\)\(R[0-9]+\)` b0[idx] = b1 } @@ -70,7 +76,9 @@ func FusedAdd32(x, y, z float32) float32 { // s390x:"FMADDS\t" // ppc64x:"FMADDS\t" // arm64:"FMADDS" + // loong64:"FMADDF\t" // riscv64:"FMADDS\t" + // amd64/v3:"VFMADD231SS\t" return x*y + z } @@ -78,11 +86,13 @@ func FusedSub32_a(x, y, z float32) float32 { // s390x:"FMSUBS\t" // ppc64x:"FMSUBS\t" // riscv64:"FMSUBS\t" + // loong64:"FMSUBF\t" return x*y - z } func FusedSub32_b(x, y, z float32) float32 { // arm64:"FMSUBS" + // loong64:"FNMSUBF\t" // riscv64:"FNMSUBS\t" return z - x*y } @@ -91,7 +101,9 @@ func FusedAdd64(x, y, z float64) float64 { // s390x:"FMADD\t" // ppc64x:"FMADD\t" // arm64:"FMADDD" + // loong64:"FMADDD\t" // riscv64:"FMADDD\t" + // amd64/v3:"VFMADD231SD\t" return x*y + z } @@ -99,11 +111,13 @@ func FusedSub64_a(x, y, z float64) float64 { // s390x:"FMSUB\t" // ppc64x:"FMSUB\t" // riscv64:"FMSUBD\t" + // loong64:"FMSUBD\t" return x*y - z } func FusedSub64_b(x, y, z float64) float64 { // arm64:"FMSUBD" + // loong64:"FNMSUBD\t" // riscv64:"FNMSUBD\t" return z - x*y } @@ -139,20 +153,14 @@ func CmpWithAdd(a float64, b float64) bool { // Non-floats // // ---------------- // -// We should make sure that the compiler doesn't generate floating point -// instructions for non-float operations on Plan 9, because floating point -// operations are not allowed in the note handler. - func ArrayZero() [16]byte { // amd64:"MOVUPS" - // plan9/amd64/:-"MOVUPS" var a [16]byte return a } func ArrayCopy(a [16]byte) (b [16]byte) { // amd64:"MOVUPS" - // plan9/amd64/:-"MOVUPS" b = a return } @@ -164,36 +172,44 @@ func ArrayCopy(a [16]byte) (b [16]byte) { func Float64Min(a, b float64) float64 { // amd64:"MINSD" // arm64:"FMIND" + // loong64:"FMIND" // riscv64:"FMIN" // ppc64/power9:"XSMINJDP" // ppc64/power10:"XSMINJDP" + // s390x: "WFMINDB" return min(a, b) } func Float64Max(a, b float64) float64 { // amd64:"MINSD" // arm64:"FMAXD" + // loong64:"FMAXD" // riscv64:"FMAX" // ppc64/power9:"XSMAXJDP" // ppc64/power10:"XSMAXJDP" + // s390x: "WFMAXDB" return max(a, b) } func Float32Min(a, b float32) float32 { // amd64:"MINSS" // arm64:"FMINS" + // loong64:"FMINF" // riscv64:"FMINS" // ppc64/power9:"XSMINJDP" // ppc64/power10:"XSMINJDP" + // s390x: "WFMINSB" return min(a, b) } func Float32Max(a, b float32) float32 { // amd64:"MINSS" // arm64:"FMAXS" + // loong64:"FMAXF" // riscv64:"FMAXS" // ppc64/power9:"XSMAXJDP" // ppc64/power10:"XSMAXJDP" + // s390x: "WFMAXSB" return max(a, b) } @@ -201,14 +217,36 @@ func Float32Max(a, b float32) float32 { // Constant Optimizations // // ------------------------ // +func Float32ConstantZero() float32 { + // arm64:"FMOVS\tZR," + return 0.0 +} + +func Float32ConstantChipFloat() float32 { + // arm64:"FMOVS\t[$]\\(2\\.25\\)," + return 2.25 +} + func Float32Constant() float32 { + // arm64:"FMOVS\t[$]f32\\.42440000\\(SB\\)" // ppc64x/power8:"FMOVS\t[$]f32\\.42440000\\(SB\\)" // ppc64x/power9:"FMOVS\t[$]f32\\.42440000\\(SB\\)" // ppc64x/power10:"XXSPLTIDP\t[$]1111752704," return 49.0 } +func Float64ConstantZero() float64 { + // arm64:"FMOVD\tZR," + return 0.0 +} + +func Float64ConstantChipFloat() float64 { + // arm64:"FMOVD\t[$]\\(2\\.25\\)," + return 2.25 +} + func Float64Constant() float64 { + // arm64:"FMOVD\t[$]f64\\.4048800000000000\\(SB\\)" // ppc64x/power8:"FMOVD\t[$]f64\\.4048800000000000\\(SB\\)" // ppc64x/power9:"FMOVD\t[$]f64\\.4048800000000000\\(SB\\)" // ppc64x/power10:"XXSPLTIDP\t[$]1111752704," @@ -228,11 +266,14 @@ func Float64DenormalFloat32Constant() float64 { return 0x1p-127 } -func Float64ConstantStore(p *float64) { - // amd64: "MOVQ\t[$]4617801906721357038" +func Float32ConstantStore(p *float32) { + // amd64:"MOVL\t[$]1085133554" + // riscv64: "MOVF\t[$]f32.40add2f2" *p = 5.432 } -func Float32ConstantStore(p *float32) { - // amd64: "MOVL\t[$]1085133554" + +func Float64ConstantStore(p *float64) { + // amd64: "MOVQ\t[$]4617801906721357038" + // riscv64: "MOVD\t[$]f64.4015ba5e353f7cee" *p = 5.432 } diff --git a/test/codegen/fuse.go b/test/codegen/fuse.go index 79dd337dee2234..561bac7224728a 100644 --- a/test/codegen/fuse.go +++ b/test/codegen/fuse.go @@ -6,6 +6,8 @@ package codegen +import "math" + // Notes: // - these examples use channels to provide a source of // unknown values that cannot be optimized away @@ -195,3 +197,102 @@ func ui4d(c <-chan uint8) { for x := <-c; x < 126 || x >= 128; x = <-c { } } + +// -------------------------------------// +// merge NaN checks // +// ------------------------------------ // + +func f64NaNOrPosInf(c <-chan float64) { + // This test assumes IsInf(x, 1) is implemented as x > MaxFloat rather than x == Inf(1). + + // amd64:"JCS",-"JNE",-"JPS",-"JPC" + // riscv64:"FCLASSD",-"FLED",-"FLTD",-"FNED",-"FEQD" + for x := <-c; math.IsNaN(x) || math.IsInf(x, 1); x = <-c { + } +} + +func f64NaNOrNegInf(c <-chan float64) { + // This test assumes IsInf(x, -1) is implemented as x < -MaxFloat rather than x == Inf(-1). + + // amd64:"JCS",-"JNE",-"JPS",-"JPC" + // riscv64:"FCLASSD",-"FLED",-"FLTD",-"FNED",-"FEQD" + for x := <-c; math.IsNaN(x) || math.IsInf(x, -1); x = <-c { + } +} + +func f64NaNOrLtOne(c <-chan float64) { + // amd64:"JCS",-"JNE",-"JPS",-"JPC" + // riscv64:"FLED",-"FLTD",-"FNED",-"FEQD" + for x := <-c; math.IsNaN(x) || x < 1; x = <-c { + } +} + +func f64NaNOrLteOne(c <-chan float64) { + // amd64:"JLS",-"JNE",-"JPS",-"JPC" + // riscv64:"FLTD",-"FLED",-"FNED",-"FEQD" + for x := <-c; x <= 1 || math.IsNaN(x); x = <-c { + } +} + +func f64NaNOrGtOne(c <-chan float64) { + // amd64:"JCS",-"JNE",-"JPS",-"JPC" + // riscv64:"FLED",-"FLTD",-"FNED",-"FEQD" + for x := <-c; math.IsNaN(x) || x > 1; x = <-c { + } +} + +func f64NaNOrGteOne(c <-chan float64) { + // amd64:"JLS",-"JNE",-"JPS",-"JPC" + // riscv64:"FLTD",-"FLED",-"FNED",-"FEQD" + for x := <-c; x >= 1 || math.IsNaN(x); x = <-c { + } +} + +func f32NaNOrLtOne(c <-chan float32) { + // amd64:"JCS",-"JNE",-"JPS",-"JPC" + // riscv64:"FLES",-"FLTS",-"FNES",-"FEQS" + for x := <-c; x < 1 || x != x; x = <-c { + } +} + +func f32NaNOrLteOne(c <-chan float32) { + // amd64:"JLS",-"JNE",-"JPS",-"JPC" + // riscv64:"FLTS",-"FLES",-"FNES",-"FEQS" + for x := <-c; x != x || x <= 1; x = <-c { + } +} + +func f32NaNOrGtOne(c <-chan float32) { + // amd64:"JCS",-"JNE",-"JPS",-"JPC" + // riscv64:"FLES",-"FLTS",-"FNES",-"FEQS" + for x := <-c; x > 1 || x != x; x = <-c { + } +} + +func f32NaNOrGteOne(c <-chan float32) { + // amd64:"JLS",-"JNE",-"JPS",-"JPC" + // riscv64:"FLTS",-"FLES",-"FNES",-"FEQS" + for x := <-c; x != x || x >= 1; x = <-c { + } +} + +// ------------------------------------ // +// regressions // +// ------------------------------------ // + +func gte4(x uint64) bool { + return x >= 4 +} + +func lt20(x uint64) bool { + return x < 20 +} + +func issue74915(c <-chan uint64) { + // Check that the optimization is not blocked by function inlining. + + // amd64:"CMPQ\t.+, [$]16","ADDQ\t[$]-4," + // s390x:"CLGIJ\t[$]4, R[0-9]+, [$]16","ADD\t[$]-4," + for x := <-c; gte4(x) && lt20(x); x = <-c { + } +} diff --git a/test/codegen/generics.go b/test/codegen/generics.go new file mode 100644 index 00000000000000..45c4ca8d6aee61 --- /dev/null +++ b/test/codegen/generics.go @@ -0,0 +1,40 @@ +// asmcheck + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +import "cmp" + +func isNaN[T cmp.Ordered](x T) bool { + return x != x +} + +func compare[T cmp.Ordered](x, y T) int { + // amd64:-"TESTB" + // arm64:-"MOVB" + xNaN := isNaN(x) + yNaN := isNaN(y) + if xNaN { + if yNaN { + return 0 + } + return -1 + } + if yNaN { + return +1 + } + if x < y { + return -1 + } + if x > y { + return +1 + } + return 0 +} + +func usesCompare(a, b int) int { + return compare(a, b) +} diff --git a/test/codegen/ifaces.go b/test/codegen/ifaces.go index 2be3fa5146a7ac..cc67a047405445 100644 --- a/test/codegen/ifaces.go +++ b/test/codegen/ifaces.go @@ -25,3 +25,39 @@ func ConvToM(x any) I { // arm64:`CALL\truntime.typeAssert`,`LDAR`,`MOVWU`,`MOVD\t\(R.*\)\(R.*\)` return x.(I) } + +func e1(x any, y *int) bool { + // amd64:-`.*faceeq`,`SETEQ` + // arm64:-`.*faceeq`,`CSET\tEQ` + return x == y +} + +func e2(x any, y *int) bool { + // amd64:-`.*faceeq`,`SETEQ` + // arm64:-`.*faceeq`,`CSET\tEQ` + return y == x +} + +type E *int + +func e3(x any, y E) bool { + // amd64:-`.*faceeq`,`SETEQ` + // arm64:-`.*faceeq`,`CSET\tEQ` + return x == y +} + +type T int + +func (t *T) M() {} + +func i1(x I, y *T) bool { + // amd64:-`.*faceeq`,`SETEQ` + // arm64:-`.*faceeq`,`CSET\tEQ` + return x == y +} + +func i2(x I, y *T) bool { + // amd64:-`.*faceeq`,`SETEQ` + // arm64:-`.*faceeq`,`CSET\tEQ` + return y == x +} diff --git a/test/codegen/issue52635.go b/test/codegen/issue52635.go index 9b08cade36f98f..65f2a021d63c01 100644 --- a/test/codegen/issue52635.go +++ b/test/codegen/issue52635.go @@ -12,30 +12,42 @@ package codegen type T struct { a *[10]int b [10]int + s []int } func (t *T) f() { // amd64:-".*runtime.memclrNoHeapPointers" - // amd64:"DUFFZERO" + // amd64:`MOVUPS\tX15,` for i := range t.a { t.a[i] = 0 } // amd64:-".*runtime.memclrNoHeapPointers" - // amd64:"DUFFZERO" + // amd64:`MOVUPS\tX15,` for i := range *t.a { t.a[i] = 0 } // amd64:-".*runtime.memclrNoHeapPointers" - // amd64:"DUFFZERO" + // amd64:`MOVUPS\tX15,` for i := range t.a { (*t.a)[i] = 0 } // amd64:-".*runtime.memclrNoHeapPointers" - // amd64:"DUFFZERO" + // amd64:`MOVUPS\tX15,` for i := range *t.a { (*t.a)[i] = 0 } + + // amd64:-".*runtime.memclrNoHeapPointers" + // amd64:`MOVUPS\tX15,` + for i := range t.b { + t.b[i] = 0 + } + + // amd64:".*runtime.memclrNoHeapPointers" + for i := range t.s { + t.s[i] = 0 + } } diff --git a/test/codegen/issue56440.go b/test/codegen/issue56440.go index c6c1e667890ab9..826dea85860e07 100644 --- a/test/codegen/issue56440.go +++ b/test/codegen/issue56440.go @@ -8,7 +8,7 @@ // is constant. We check this by making sure that the constant length // is folded into a load offset. -package p +package codegen func f(x []int) int { s := make([]int, 3) diff --git a/test/codegen/issue58166.go b/test/codegen/issue58166.go index 8be5aac8415734..b38f9c916e43e7 100644 --- a/test/codegen/issue58166.go +++ b/test/codegen/issue58166.go @@ -4,7 +4,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package p +package codegen func dgemmSerialNotNot(m, n, k int, a []float64, lda int, b []float64, ldb int, c []float64, ldc int, alpha float64) { for i := 0; i < m; i++ { diff --git a/test/codegen/issue59297.go b/test/codegen/issue59297.go new file mode 100644 index 00000000000000..1703a1a86399c7 --- /dev/null +++ b/test/codegen/issue59297.go @@ -0,0 +1,17 @@ +// asmcheck + +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +func f(x, y int, p *int) { + // amd64:`MOVQ\sAX, BX` + h(8, x) + *p = y +} + +//go:noinline +func h(a, b int) { +} diff --git a/test/codegen/issue66585.go b/test/codegen/issue66585.go index 32a180be0ee813..867a96218c868f 100644 --- a/test/codegen/issue66585.go +++ b/test/codegen/issue66585.go @@ -4,7 +4,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package p +package codegen var x = func() int { n := 0 diff --git a/test/codegen/issue68845.go b/test/codegen/issue68845.go new file mode 100644 index 00000000000000..3b22471d06f270 --- /dev/null +++ b/test/codegen/issue68845.go @@ -0,0 +1,52 @@ +// asmcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +type T1 struct { + x string +} + +func f1() *T1 { + // amd64:-`MOVQ\s[$]0`,-`MOVUPS\sX15` + return &T1{} +} + +type T2 struct { + x, y string +} + +func f2() *T2 { + // amd64:-`MOVQ\s[$]0`,-`MOVUPS\sX15` + return &T2{} +} + +type T3 struct { + x complex128 +} + +func f3() *T3 { + // amd64:-`MOVQ\s[$]0`,-`MOVUPS\sX15` + return &T3{} +} + +type T4 struct { + x []byte +} + +func f4() *T4 { + // amd64:-`MOVQ\s[$]0`,-`MOVUPS\sX15` + return &T4{} +} + +type T5 struct { + x any +} + +func f5() *T5 { + // amd64:-`MOVQ\s[$]0`,-`MOVUPS\sX15` + return &T5{} +} diff --git a/test/codegen/issue69635.go b/test/codegen/issue69635.go new file mode 100644 index 00000000000000..de68bf1287f565 --- /dev/null +++ b/test/codegen/issue69635.go @@ -0,0 +1,13 @@ +// asmcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +func calc(a uint64) uint64 { + v := a >> 20 & 0x7f + // amd64: `SHRQ\s\$17, AX$`, `ANDL\s\$1016, AX$` + return v << 3 +} diff --git a/test/codegen/issue70409.go b/test/codegen/issue70409.go new file mode 100644 index 00000000000000..bfb4560582e4f0 --- /dev/null +++ b/test/codegen/issue70409.go @@ -0,0 +1,20 @@ +// asmcheck -gcflags=-d=ssa/check/on + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +// amd64:-"MOVQ" +func foo(v uint64) (b [8]byte) { + b[0] = byte(v) + b[1] = byte(v >> 8) + b[2] = byte(v >> 16) + b[3] = byte(v >> 24) + b[4] = byte(v >> 32) + b[5] = byte(v >> 40) + b[6] = byte(v >> 48) + b[7] = byte(v >> 56) + return b +} diff --git a/test/codegen/issue72832.go b/test/codegen/issue72832.go new file mode 100644 index 00000000000000..392b41b1733773 --- /dev/null +++ b/test/codegen/issue72832.go @@ -0,0 +1,50 @@ +// asmcheck + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +type tile1 struct { + a uint16 + b uint16 + c uint32 +} + +func store_tile1(t *tile1) { + // amd64:`MOVQ` + t.a, t.b, t.c = 1, 1, 1 +} + +type tile2 struct { + a, b, c, d, e int8 +} + +func store_tile2(t *tile2) { + // amd64:`MOVW` + t.a, t.b = 1, 1 + // amd64:`MOVW` + t.d, t.e = 1, 1 +} + +type tile3 struct { + a, b uint8 + c uint16 +} + +func store_shifted(t *tile3, x uint32) { + // amd64:`MOVL` + // ppc64:`MOVHBR` + t.a = uint8(x) + t.b = uint8(x >> 8) + t.c = uint16(x >> 16) +} + +func store_const(t *tile3) { + // 0x00030201 + // amd64:`MOVL\s\$197121` + // 0x01020003 + // ppc64:`MOVD\s\$16908291` + t.a, t.b, t.c = 1, 2, 3 +} diff --git a/test/codegen/issue74485.go b/test/codegen/issue74485.go new file mode 100644 index 00000000000000..b5aba9568caca7 --- /dev/null +++ b/test/codegen/issue74485.go @@ -0,0 +1,47 @@ +// asmcheck + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +func divUint64(b uint64) uint64 { + // amd64:"SHRQ [$]63, AX" + return b / 9223372036854775808 +} + +func divUint32(b uint32) uint32 { + // amd64:"SHRL [$]31, AX" + return b / 2147483648 +} + +func divUint16(b uint16) uint16 { + // amd64:"SHRW [$]15, AX" + return b / 32768 +} + +func divUint8(b uint8) uint8 { + // amd64:"SHRB [$]7, AL" + return b / 128 +} + +func modUint64(b uint64) uint64 { + // amd64:"BTRQ [$]63, AX" + return b % 9223372036854775808 +} + +func modUint32(b uint32) uint32 { + // amd64:"ANDL [$]2147483647, AX" + return b % 2147483648 +} + +func modUint16(b uint16) uint16 { + // amd64:"ANDL [$]32767, AX" + return b % 32768 +} + +func modUint8(b uint8) uint8 { + // amd64:"ANDL [$]127, AX" + return b % 128 +} diff --git a/test/codegen/issue74788.go b/test/codegen/issue74788.go new file mode 100644 index 00000000000000..e102f638273cbc --- /dev/null +++ b/test/codegen/issue74788.go @@ -0,0 +1,17 @@ +// asmcheck + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +func fa(a [2]int) (r [2]int) { + // amd64:1`MOVUPS[^,]+, X[0-9]+$`,1`MOVUPS\sX[0-9]+,[^\n]+$` + return a +} + +func fb(a [4]int) (r [4]int) { + // amd64:2`MOVUPS[^,]+, X[0-9]+$`,2`MOVUPS\sX[0-9]+,[^\n]+$` + return a +} diff --git a/test/codegen/issue75203.go b/test/codegen/issue75203.go new file mode 100644 index 00000000000000..68e1794120ae8e --- /dev/null +++ b/test/codegen/issue75203.go @@ -0,0 +1,22 @@ +// asmcheck + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +import "reflect" + +func f() reflect.Type { + // amd64:`LEAQ\stype:\*int\(SB\)` + // arm64:`MOVD\s\$type:\*int\(SB\)` + return reflect.TypeFor[*int]() +} + +func g() reflect.Type { + // amd64:`LEAQ\stype:int\(SB\)` + // arm64:`MOVD\s\$type:int\(SB\)` + return reflect.TypeFor[int]() +} + diff --git a/test/codegen/load_type_from_itab.go b/test/codegen/load_type_from_itab.go new file mode 100644 index 00000000000000..b47044fcbd763b --- /dev/null +++ b/test/codegen/load_type_from_itab.go @@ -0,0 +1,24 @@ +// asmcheck + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This test makes sure that we statically load a type from an itab, instead +// of doing a indirect load from thet itab. + +package codegen + +type M interface{ M() } +type A interface{ A() } + +type Impl struct{} + +func (*Impl) M() {} +func (*Impl) A() {} + +func main() { + var a M = &Impl{} + // amd64:`LEAQ\ttype:.*Impl` + a.(A).A() +} diff --git a/test/codegen/maps.go b/test/codegen/maps.go index 25505799e9351d..860b2c2cbd23cd 100644 --- a/test/codegen/maps.go +++ b/test/codegen/maps.go @@ -66,6 +66,28 @@ func LookupStringConversionKeyedArrayLit(m map[[2]string]int, bytes []byte) int return m[[2]string{0: string(bytes)}] } +func LookupStringConversion1(m map[string]int, bytes []byte) int { + // amd64:-`.*runtime\.slicebytetostring\(` + s := string(bytes) + return m[s] +} +func LookupStringConversion2(m *map[string]int, bytes []byte) int { + // amd64:-`.*runtime\.slicebytetostring\(` + s := string(bytes) + return (*m)[s] +} +func LookupStringConversion3(m map[string]int, bytes []byte) (int, bool) { + // amd64:-`.*runtime\.slicebytetostring\(` + s := string(bytes) + r, ok := m[s] + return r, ok +} +func DeleteStringConversion(m map[string]int, bytes []byte) { + // amd64:-`.*runtime\.slicebytetostring\(` + s := string(bytes) + delete(m, s) +} + // ------------------- // // Map Clear // // ------------------- // @@ -74,7 +96,7 @@ func LookupStringConversionKeyedArrayLit(m map[[2]string]int, bytes []byte) int func MapClearReflexive(m map[int]int) { // amd64:`.*runtime\.mapclear` - // amd64:-`.*runtime\.mapiterinit` + // amd64:-`.*runtime\.(mapiterinit|mapIterStart)` for k := range m { delete(m, k) } @@ -83,7 +105,7 @@ func MapClearReflexive(m map[int]int) { func MapClearIndirect(m map[int]int) { s := struct{ m map[int]int }{m: m} // amd64:`.*runtime\.mapclear` - // amd64:-`.*runtime\.mapiterinit` + // amd64:-`.*runtime\.(mapiterinit|mapIterStart)` for k := range s.m { delete(s.m, k) } @@ -91,14 +113,14 @@ func MapClearIndirect(m map[int]int) { func MapClearPointer(m map[*byte]int) { // amd64:`.*runtime\.mapclear` - // amd64:-`.*runtime\.mapiterinit` + // amd64:-`.*runtime\.(mapiterinit|mapIterStart)` for k := range m { delete(m, k) } } func MapClearNotReflexive(m map[float64]int) { - // amd64:`.*runtime\.mapiterinit` + // amd64:`.*runtime\.(mapiterinit|mapIterStart)` // amd64:-`.*runtime\.mapclear` for k := range m { delete(m, k) @@ -106,7 +128,7 @@ func MapClearNotReflexive(m map[float64]int) { } func MapClearInterface(m map[interface{}]int) { - // amd64:`.*runtime\.mapiterinit` + // amd64:`.*runtime\.(mapiterinit|mapIterStart)` // amd64:-`.*runtime\.mapclear` for k := range m { delete(m, k) @@ -115,7 +137,7 @@ func MapClearInterface(m map[interface{}]int) { func MapClearSideEffect(m map[int]int) int { k := 0 - // amd64:`.*runtime\.mapiterinit` + // amd64:`.*runtime\.(mapiterinit|mapIterStart)` // amd64:-`.*runtime\.mapclear` for k = range m { delete(m, k) diff --git a/test/codegen/math.go b/test/codegen/math.go index eb6e927dec92fb..9ef881a9afc2f5 100644 --- a/test/codegen/math.go +++ b/test/codegen/math.go @@ -1,4 +1,4 @@ -// asmcheck +// asmcheck -gcflags=-d=converthash=qy // Copyright 2018 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style @@ -53,6 +53,7 @@ func sqrt(x float64) float64 { // 386/sse2:"SQRTSD" 386/softfloat:-"SQRTD" // arm64:"FSQRTD" // arm/7:"SQRTD" + // loong64:"SQRTD" // mips/hardfloat:"SQRTD" mips/softfloat:-"SQRTD" // mips64/hardfloat:"SQRTD" mips64/softfloat:-"SQRTD" // wasm:"F64Sqrt" @@ -66,6 +67,7 @@ func sqrt32(x float32) float32 { // 386/sse2:"SQRTSS" 386/softfloat:-"SQRTS" // arm64:"FSQRTS" // arm/7:"SQRTF" + // loong64:"SQRTF" // mips/hardfloat:"SQRTF" mips/softfloat:-"SQRTF" // mips64/hardfloat:"SQRTF" mips64/softfloat:-"SQRTF" // wasm:"F32Sqrt" @@ -78,6 +80,7 @@ func sqrt32(x float32) float32 { func abs(x, y float64) { // amd64:"BTRQ\t[$]63" // arm64:"FABSD\t" + // loong64:"ABSD\t" // s390x:"LPDFR\t",-"MOVD\t" (no integer load/store) // ppc64x:"FABS\t" // riscv64:"FABSD\t" @@ -102,6 +105,7 @@ func abs32(x float32) float32 { // Check that it's using integer registers func copysign(a, b, c float64) { // amd64:"BTRQ\t[$]63","ANDQ","ORQ" + // loong64:"FCOPYSGD" // s390x:"CPSDR",-"MOVD" (no integer load/store) // ppc64x:"FCPSGN" // riscv64:"FSGNJD" @@ -109,6 +113,7 @@ func copysign(a, b, c float64) { sink64[0] = math.Copysign(a, b) // amd64:"BTSQ\t[$]63" + // loong64:"FCOPYSGD" // s390x:"LNDFR\t",-"MOVD\t" (no integer load/store) // ppc64x:"FCPSGN" // riscv64:"FSGNJD" @@ -121,6 +126,7 @@ func copysign(a, b, c float64) { sink64[2] = math.Float64frombits(math.Float64bits(a) | 1<<63) // amd64:"ANDQ","ORQ" + // loong64:"FCOPYSGD" // s390x:"CPSDR\t",-"MOVD\t" (no integer load/store) // ppc64x:"FCPSGN" // riscv64:"FSGNJD" @@ -132,6 +138,7 @@ func fma(x, y, z float64) float64 { // amd64:"VFMADD231SD" // arm/6:"FMULAD" // arm64:"FMADDD" + // loong64:"FMADDD" // s390x:"FMADD" // ppc64x:"FMADD" // riscv64:"FMADDD" @@ -153,33 +160,101 @@ func fnma(x, y, z float64) float64 { return math.FMA(x, -y, -z) } +func isPosInf(x float64) bool { + // riscv64:"FCLASSD" + return math.IsInf(x, 1) +} + +func isPosInfEq(x float64) bool { + // riscv64:"FCLASSD" + return x == math.Inf(1) +} + +func isPosInfCmp(x float64) bool { + // riscv64:"FCLASSD" + return x > math.MaxFloat64 +} + +func isNotPosInf(x float64) bool { + // riscv64:"FCLASSD" + return !math.IsInf(x, 1) +} + +func isNotPosInfEq(x float64) bool { + // riscv64:"FCLASSD" + return x != math.Inf(1) +} + +func isNotPosInfCmp(x float64) bool { + // riscv64:"FCLASSD" + return x <= math.MaxFloat64 +} + +func isNegInf(x float64) bool { + // riscv64:"FCLASSD" + return math.IsInf(x, -1) +} + +func isNegInfEq(x float64) bool { + // riscv64:"FCLASSD" + return x == math.Inf(-1) +} + +func isNegInfCmp(x float64) bool { + // riscv64:"FCLASSD" + return x < -math.MaxFloat64 +} + +func isNotNegInf(x float64) bool { + // riscv64:"FCLASSD" + return !math.IsInf(x, -1) +} + +func isNotNegInfEq(x float64) bool { + // riscv64:"FCLASSD" + return x != math.Inf(-1) +} + +func isNotNegInfCmp(x float64) bool { + // riscv64:"FCLASSD" + return x >= -math.MaxFloat64 +} + func fromFloat64(f64 float64) uint64 { // amd64:"MOVQ\tX.*, [^X].*" // arm64:"FMOVD\tF.*, R.*" + // loong64:"MOVV\tF.*, R.*" // ppc64x:"MFVSRD" // mips64/hardfloat:"MOVV\tF.*, R.*" + // riscv64:"FMVXD" return math.Float64bits(f64+1) + 1 } func fromFloat32(f32 float32) uint32 { // amd64:"MOVL\tX.*, [^X].*" // arm64:"FMOVS\tF.*, R.*" + // loong64:"MOVW\tF.*, R.*" // mips64/hardfloat:"MOVW\tF.*, R.*" + // riscv64:"FMVXW" return math.Float32bits(f32+1) + 1 } func toFloat64(u64 uint64) float64 { // amd64:"MOVQ\t[^X].*, X.*" // arm64:"FMOVD\tR.*, F.*" + // loong64:"MOVV\tR.*, F.*" // ppc64x:"MTVSRD" // mips64/hardfloat:"MOVV\tR.*, F.*" + // riscv64:"FMVDX" return math.Float64frombits(u64+1) + 1 } func toFloat32(u32 uint32) float32 { // amd64:"MOVL\t[^X].*, X.*" // arm64:"FMOVS\tR.*, F.*" + // loong64:"MOVW\tR.*, F.*" // mips64/hardfloat:"MOVW\tR.*, F.*" + // riscv64:"FMVWX" return math.Float32frombits(u32+1) + 1 } @@ -235,10 +310,11 @@ func nanGenerate64() float64 { // amd64:"DIVSD" z0 := zero / zero - // amd64:"MULSD" + // amd64/v1,amd64/v2:"MULSD" z1 := zero * inf // amd64:"SQRTSD" z2 := math.Sqrt(negone) + // amd64/v3:"VFMADD231SD" return z0 + z1 + z2 } @@ -249,7 +325,53 @@ func nanGenerate32() float32 { // amd64:"DIVSS" z0 := zero / zero - // amd64:"MULSS" + // amd64/v1,amd64/v2:"MULSS" z1 := zero * inf + // amd64/v3:"VFMADD231SS" return z0 + z1 } + +func outOfBoundsConv(i32 *[2]int32, u32 *[2]uint32, i64 *[2]int64, u64 *[2]uint64) { + // arm64: "FCVTZSDW" + // amd64: "CVTTSD2SL", "CVTSD2SS" + i32[0] = int32(two40()) + // arm64: "FCVTZSDW" + // amd64: "CVTTSD2SL", "CVTSD2SS" + i32[1] = int32(-two40()) + // arm64: "FCVTZSDW" + // amd64: "CVTTSD2SL", "CVTSD2SS" + u32[0] = uint32(two41()) + // on arm64, this uses an explicit <0 comparison, so it constant folds. + // on amd64, this uses an explicit <0 comparison, so it constant folds. + // amd64: "MOVL\t[$]0," + u32[1] = uint32(minus1()) + // arm64: "FCVTZSD" + // amd64: "CVTTSD2SQ" + i64[0] = int64(two80()) + // arm64: "FCVTZSD" + // amd64: "CVTTSD2SQ" + i64[1] = int64(-two80()) + // arm64: "FCVTZUD" + // amd64: "CVTTSD2SQ" + u64[0] = uint64(two81()) + // arm64: "FCVTZUD" + // on amd64, this uses an explicit <0 comparison, so it constant folds. + // amd64: "MOVQ\t[$]0," + u64[1] = uint64(minus1()) +} + +func two40() float64 { + return 1 << 40 +} +func two41() float64 { + return 1 << 41 +} +func two80() float64 { + return 1 << 80 +} +func two81() float64 { + return 1 << 81 +} +func minus1() float64 { + return -1 +} diff --git a/test/codegen/mathbits.go b/test/codegen/mathbits.go index a743eaf3d3ae1f..f8fa374c0af483 100644 --- a/test/codegen/mathbits.go +++ b/test/codegen/mathbits.go @@ -6,7 +6,10 @@ package codegen -import "math/bits" +import ( + "math/bits" + "unsafe" +) // ----------------------- // // bits.LeadingZeros // @@ -15,55 +18,70 @@ import "math/bits" func LeadingZeros(n uint) int { // amd64/v1,amd64/v2:"BSRQ" // amd64/v3:"LZCNTQ", -"BSRQ" - // s390x:"FLOGR" - // arm:"CLZ" arm64:"CLZ" + // arm64:"CLZ" + // arm:"CLZ" + // loong64:"CLZV",-"SUB" // mips:"CLZ" - // wasm:"I64Clz" // ppc64x:"CNTLZD" + // riscv64/rva22u64,riscv64/rva23u64:"CLZ\t",-"SUB" + // s390x:"FLOGR" + // wasm:"I64Clz" return bits.LeadingZeros(n) } func LeadingZeros64(n uint64) int { // amd64/v1,amd64/v2:"BSRQ" // amd64/v3:"LZCNTQ", -"BSRQ" - // s390x:"FLOGR" - // arm:"CLZ" arm64:"CLZ" + // arm:"CLZ" + // arm64:"CLZ" + // loong64:"CLZV",-"SUB" // mips:"CLZ" - // wasm:"I64Clz" // ppc64x:"CNTLZD" + // riscv64/rva22u64,riscv64/rva23u64:"CLZ\t",-"ADDI" + // s390x:"FLOGR" + // wasm:"I64Clz" return bits.LeadingZeros64(n) } func LeadingZeros32(n uint32) int { // amd64/v1,amd64/v2:"BSRQ","LEAQ",-"CMOVQEQ" // amd64/v3: "LZCNTL",- "BSRL" - // s390x:"FLOGR" - // arm:"CLZ" arm64:"CLZW" + // arm:"CLZ" + // arm64:"CLZW" + // loong64:"CLZW",-"SUB" // mips:"CLZ" - // wasm:"I64Clz" // ppc64x:"CNTLZW" + // riscv64/rva22u64,riscv64/rva23u64:"CLZW",-"ADDI" + // s390x:"FLOGR" + // wasm:"I64Clz" return bits.LeadingZeros32(n) } func LeadingZeros16(n uint16) int { // amd64/v1,amd64/v2:"BSRL","LEAL",-"CMOVQEQ" // amd64/v3: "LZCNTL",- "BSRL" - // s390x:"FLOGR" - // arm:"CLZ" arm64:"CLZ" + // arm64:"CLZ" + // arm:"CLZ" + // loong64:"CLZV" // mips:"CLZ" - // wasm:"I64Clz" // ppc64x:"CNTLZD" + // riscv64/rva22u64,riscv64/rva23u64:"CLZ\t","ADDI\t\\$-48",-"NEG" + // s390x:"FLOGR" + // wasm:"I64Clz" return bits.LeadingZeros16(n) } func LeadingZeros8(n uint8) int { // amd64/v1,amd64/v2:"BSRL","LEAL",-"CMOVQEQ" // amd64/v3: "LZCNTL",- "BSRL" - // s390x:"FLOGR" - // arm:"CLZ" arm64:"CLZ" + // arm64:"CLZ" + // arm:"CLZ" + // loong64:"CLZV" // mips:"CLZ" - // wasm:"I64Clz" // ppc64x:"CNTLZD" + // riscv64/rva22u64,riscv64/rva23u64:"CLZ\t","ADDI\t\\$-56",-"NEG" + // s390x:"FLOGR" + // wasm:"I64Clz" return bits.LeadingZeros8(n) } @@ -74,60 +92,82 @@ func LeadingZeros8(n uint8) int { func Len(n uint) int { // amd64/v1,amd64/v2:"BSRQ" // amd64/v3: "LZCNTQ" - // s390x:"FLOGR" - // arm:"CLZ" arm64:"CLZ" + // arm64:"CLZ" + // arm:"CLZ" + // loong64:"CLZV" // mips:"CLZ" - // wasm:"I64Clz" // ppc64x:"SUBC","CNTLZD" + // riscv64/rva22u64,riscv64/rva23u64:"CLZ\t","ADDI\t\\$-64" + // s390x:"FLOGR" + // wasm:"I64Clz" return bits.Len(n) } func Len64(n uint64) int { // amd64/v1,amd64/v2:"BSRQ" // amd64/v3: "LZCNTQ" - // s390x:"FLOGR" - // arm:"CLZ" arm64:"CLZ" + // arm64:"CLZ" + // arm:"CLZ" + // loong64:"CLZV" // mips:"CLZ" - // wasm:"I64Clz" // ppc64x:"SUBC","CNTLZD" + // riscv64/rva22u64,riscv64/rva23u64:"CLZ\t","ADDI\t\\$-64" + // s390x:"FLOGR" + // wasm:"I64Clz" return bits.Len64(n) } func SubFromLen64(n uint64) int { + // loong64:"CLZV",-"ADD" // ppc64x:"CNTLZD",-"SUBC" + // riscv64/rva22u64,riscv64/rva23u64:"CLZ\t",-"ADDI",-"NEG" return 64 - bits.Len64(n) } +func CompareWithLen64(n uint64) bool { + // loong64:"CLZV",-"ADD",-"[$]64",-"[$]9" + return bits.Len64(n) < 9 +} + func Len32(n uint32) int { // amd64/v1,amd64/v2:"BSRQ","LEAQ",-"CMOVQEQ" // amd64/v3: "LZCNTL" - // s390x:"FLOGR" - // arm:"CLZ" arm64:"CLZ" + // arm64:"CLZ" + // arm:"CLZ" + // loong64:"CLZW" // mips:"CLZ" - // wasm:"I64Clz" // ppc64x: "CNTLZW" + // riscv64/rva22u64,riscv64/rva23u64:"CLZW","ADDI\t\\$-32" + // s390x:"FLOGR" + // wasm:"I64Clz" return bits.Len32(n) } func Len16(n uint16) int { // amd64/v1,amd64/v2:"BSRL","LEAL",-"CMOVQEQ" // amd64/v3: "LZCNTL" - // s390x:"FLOGR" - // arm:"CLZ" arm64:"CLZ" + // arm64:"CLZ" + // arm:"CLZ" + // loong64:"CLZV" // mips:"CLZ" - // wasm:"I64Clz" // ppc64x:"SUBC","CNTLZD" + // riscv64/rva22u64,riscv64/rva23u64:"CLZ\t","ADDI\t\\$-64" + // s390x:"FLOGR" + // wasm:"I64Clz" return bits.Len16(n) } func Len8(n uint8) int { // amd64/v1,amd64/v2:"BSRL","LEAL",-"CMOVQEQ" // amd64/v3: "LZCNTL" - // s390x:"FLOGR" - // arm:"CLZ" arm64:"CLZ" + // arm64:"CLZ" + // arm:"CLZ" + // loong64:"CLZV" // mips:"CLZ" - // wasm:"I64Clz" // ppc64x:"SUBC","CNTLZD" + // riscv64/rva22u64,riscv64/rva23u64:"CLZ\t","ADDI\t\\$-64" + // s390x:"FLOGR" + // wasm:"I64Clz" return bits.Len8(n) } @@ -140,8 +180,10 @@ func OnesCount(n uint) int { // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT" // amd64:"POPCNTQ" // arm64:"VCNT","VUADDLV" - // s390x:"POPCNT" + // loong64:"VPCNTV" // ppc64x:"POPCNTD" + // riscv64:"CPOP\t" + // s390x:"POPCNT" // wasm:"I64Popcnt" return bits.OnesCount(n) } @@ -150,8 +192,10 @@ func OnesCount64(n uint64) int { // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT" // amd64:"POPCNTQ" // arm64:"VCNT","VUADDLV" - // s390x:"POPCNT" + // loong64:"VPCNTV" // ppc64x:"POPCNTD" + // riscv64:"CPOP\t" + // s390x:"POPCNT" // wasm:"I64Popcnt" return bits.OnesCount64(n) } @@ -160,8 +204,10 @@ func OnesCount32(n uint32) int { // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT" // amd64:"POPCNTL" // arm64:"VCNT","VUADDLV" - // s390x:"POPCNT" + // loong64:"VPCNTW" // ppc64x:"POPCNTW" + // riscv64:"CPOPW" + // s390x:"POPCNT" // wasm:"I64Popcnt" return bits.OnesCount32(n) } @@ -170,56 +216,96 @@ func OnesCount16(n uint16) int { // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT" // amd64:"POPCNTL" // arm64:"VCNT","VUADDLV" - // s390x:"POPCNT" + // loong64:"VPCNTH" // ppc64x:"POPCNTW" + // riscv64:"CPOP\t" + // s390x:"POPCNT" // wasm:"I64Popcnt" return bits.OnesCount16(n) } func OnesCount8(n uint8) int { - // s390x:"POPCNT" // ppc64x:"POPCNTB" + // riscv64/rva22u64,riscv64/rva23u64:"CPOP\t" + // s390x:"POPCNT" // wasm:"I64Popcnt" return bits.OnesCount8(n) } +// ------------------ // +// bits.Reverse // +// ------------------ // + +func Reverse(n uint) uint { + // loong64:"BITREVV" + return bits.Reverse(n) +} + +func Reverse64(n uint64) uint64 { + // loong64:"BITREVV" + return bits.Reverse64(n) +} + +func Reverse32(n uint32) uint32 { + // loong64:"BITREVW" + return bits.Reverse32(n) +} + +func Reverse16(n uint16) uint16 { + // loong64:"BITREV4B","REVB2H" + return bits.Reverse16(n) +} + +func Reverse8(n uint8) uint8 { + // loong64:"BITREV4B" + return bits.Reverse8(n) +} + // ----------------------- // // bits.ReverseBytes // // ----------------------- // func ReverseBytes(n uint) uint { - // amd64:"BSWAPQ" // 386:"BSWAPL" - // s390x:"MOVDBR" + // amd64:"BSWAPQ" // arm64:"REV" + // loong64:"REVBV" + // riscv64/rva22u64,riscv64/rva23u64:"REV8" + // s390x:"MOVDBR" return bits.ReverseBytes(n) } func ReverseBytes64(n uint64) uint64 { - // amd64:"BSWAPQ" // 386:"BSWAPL" - // s390x:"MOVDBR" + // amd64:"BSWAPQ" // arm64:"REV" + // loong64:"REVBV" // ppc64x/power10: "BRD" + // riscv64/rva22u64,riscv64/rva23u64:"REV8" + // s390x:"MOVDBR" return bits.ReverseBytes64(n) } func ReverseBytes32(n uint32) uint32 { - // amd64:"BSWAPL" // 386:"BSWAPL" - // s390x:"MOVWBR" + // amd64:"BSWAPL" // arm64:"REVW" + // loong64:"REVB2W" // ppc64x/power10: "BRW" + // riscv64/rva22u64,riscv64/rva23u64:"REV8","SRLI\t\\$32" + // s390x:"MOVWBR" return bits.ReverseBytes32(n) } func ReverseBytes16(n uint16) uint16 { // amd64:"ROLW" - // arm64:"REV16W",-"UBFX",-"ORR" // arm/5:"SLL","SRL","ORR" // arm/6:"REV16" // arm/7:"REV16" + // arm64:"REV16W",-"UBFX",-"ORR" + // loong64:"REVB2H" // ppc64x/power10: "BRH" + // riscv64/rva22u64,riscv64/rva23u64:"REV8","SRLI\t\\$48" return bits.ReverseBytes16(n) } @@ -230,7 +316,9 @@ func ReverseBytes16(n uint16) uint16 { func RotateLeft64(n uint64) uint64 { // amd64:"ROLQ" // arm64:"ROR" + // loong64:"ROTRV" // ppc64x:"ROTL" + // riscv64:"RORI" // s390x:"RISBGZ\t[$]0, [$]63, [$]37, " // wasm:"I64Rotl" return bits.RotateLeft64(n, 37) @@ -240,7 +328,9 @@ func RotateLeft32(n uint32) uint32 { // amd64:"ROLL" 386:"ROLL" // arm:`MOVW\tR[0-9]+@>23` // arm64:"RORW" + // loong64:"ROTR\t" // ppc64x:"ROTLW" + // riscv64:"RORIW" // s390x:"RLL" // wasm:"I32Rotl" return bits.RotateLeft32(n, 9) @@ -249,19 +339,23 @@ func RotateLeft32(n uint32) uint32 { func RotateLeft16(n uint16, s int) uint16 { // amd64:"ROLW" 386:"ROLW" // arm64:"RORW",-"CSEL" + // loong64:"ROTR\t","SLLV" return bits.RotateLeft16(n, s) } func RotateLeft8(n uint8, s int) uint8 { // amd64:"ROLB" 386:"ROLB" // arm64:"LSL","LSR",-"CSEL" + // loong64:"OR","SLLV","SRLV" return bits.RotateLeft8(n, s) } func RotateLeftVariable(n uint, m int) uint { // amd64:"ROLQ" // arm64:"ROR" + // loong64:"ROTRV" // ppc64x:"ROTL" + // riscv64:"ROL" // s390x:"RLLG" // wasm:"I64Rotl" return bits.RotateLeft(n, m) @@ -270,7 +364,9 @@ func RotateLeftVariable(n uint, m int) uint { func RotateLeftVariable64(n uint64, m int) uint64 { // amd64:"ROLQ" // arm64:"ROR" + // loong64:"ROTRV" // ppc64x:"ROTL" + // riscv64:"ROL" // s390x:"RLLG" // wasm:"I64Rotl" return bits.RotateLeft64(n, m) @@ -280,7 +376,9 @@ func RotateLeftVariable32(n uint32, m int) uint32 { // arm:`MOVW\tR[0-9]+@>R[0-9]+` // amd64:"ROLL" // arm64:"RORW" + // loong64:"ROTR\t" // ppc64x:"ROTLW" + // riscv64:"ROLW" // s390x:"RLL" // wasm:"I32Rotl" return bits.RotateLeft32(n, m) @@ -291,26 +389,30 @@ func RotateLeftVariable32(n uint32, m int) uint32 { // ------------------------ // func TrailingZeros(n uint) int { + // 386:"BSFL" // amd64/v1,amd64/v2:"BSFQ","MOVL\t\\$64","CMOVQEQ" // amd64/v3:"TZCNTQ" - // 386:"BSFL" // arm:"CLZ" // arm64:"RBIT","CLZ" - // s390x:"FLOGR" + // loong64:"CTZV" // ppc64x/power8:"ANDN","POPCNTD" // ppc64x/power9: "CNTTZD" + // riscv64/rva22u64,riscv64/rva23u64: "CTZ\t" + // s390x:"FLOGR" // wasm:"I64Ctz" return bits.TrailingZeros(n) } func TrailingZeros64(n uint64) int { + // 386:"BSFL","JNE" // amd64/v1,amd64/v2:"BSFQ","MOVL\t\\$64","CMOVQEQ" // amd64/v3:"TZCNTQ" - // 386:"BSFL" // arm64:"RBIT","CLZ" - // s390x:"FLOGR" + // loong64:"CTZV" // ppc64x/power8:"ANDN","POPCNTD" // ppc64x/power9: "CNTTZD" + // riscv64/rva22u64,riscv64/rva23u64: "CTZ\t" + // s390x:"FLOGR" // wasm:"I64Ctz" return bits.TrailingZeros64(n) } @@ -322,35 +424,43 @@ func TrailingZeros64Subtract(n uint64) int { } func TrailingZeros32(n uint32) int { + // 386:"BSFL" // amd64/v1,amd64/v2:"BTSQ\\t\\$32","BSFQ" // amd64/v3:"TZCNTL" - // 386:"BSFL" // arm:"CLZ" // arm64:"RBITW","CLZW" - // s390x:"FLOGR","MOVWZ" + // loong64:"CTZW" // ppc64x/power8:"ANDN","POPCNTW" // ppc64x/power9: "CNTTZW" + // riscv64/rva22u64,riscv64/rva23u64: "CTZW" + // s390x:"FLOGR","MOVWZ" // wasm:"I64Ctz" return bits.TrailingZeros32(n) } func TrailingZeros16(n uint16) int { - // amd64:"BSFL","ORL\\t\\$65536" // 386:"BSFL\t" + // amd64:"BSFL","ORL\\t\\$65536" // arm:"ORR\t\\$65536","CLZ",-"MOVHU\tR" // arm64:"ORR\t\\$65536","RBITW","CLZW",-"MOVHU\tR",-"RBIT\t",-"CLZ\t" - // s390x:"FLOGR","OR\t\\$65536" - // ppc64x/power8:"POPCNTD","ORIS\\t\\$1" + // loong64:"CTZV" + // ppc64x/power8:"POPCNTW","ADD\t\\$-1" // ppc64x/power9:"CNTTZD","ORIS\\t\\$1" + // riscv64/rva22u64,riscv64/rva23u64: "ORI\t\\$65536","CTZW" + // s390x:"FLOGR","OR\t\\$65536" // wasm:"I64Ctz" return bits.TrailingZeros16(n) } func TrailingZeros8(n uint8) int { - // amd64:"BSFL","ORL\\t\\$256" // 386:"BSFL" + // amd64:"BSFL","ORL\\t\\$256" // arm:"ORR\t\\$256","CLZ",-"MOVBU\tR" // arm64:"ORR\t\\$256","RBITW","CLZW",-"MOVBU\tR",-"RBIT\t",-"CLZ\t" + // loong64:"CTZV" + // ppc64x/power8:"POPCNTB","ADD\t\\$-1" + // ppc64x/power9:"CNTTZD","OR\t\\$256" + // riscv64/rva22u64,riscv64/rva23u64: "ORI\t\\$256","CTZW" // s390x:"FLOGR","OR\t\\$256" // wasm:"I64Ctz" return bits.TrailingZeros8(n) @@ -374,6 +484,7 @@ func IterateBits64(n uint64) int { for n != 0 { // amd64/v1,amd64/v2:"BSFQ",-"CMOVEQ" // amd64/v3:"TZCNTQ" + // riscv64/rva22u64,riscv64/rva23u64: "CTZ\t" i += bits.TrailingZeros64(n) n &= n - 1 } @@ -385,6 +496,7 @@ func IterateBits32(n uint32) int { for n != 0 { // amd64/v1,amd64/v2:"BSFL",-"BTSQ" // amd64/v3:"TZCNTL" + // riscv64/rva22u64,riscv64/rva23u64: "CTZ\t" i += bits.TrailingZeros32(n) n &= n - 1 } @@ -397,6 +509,7 @@ func IterateBits16(n uint16) int { // amd64/v1,amd64/v2:"BSFL",-"BTSL" // amd64/v3:"TZCNTL" // arm64:"RBITW","CLZW",-"ORR" + // riscv64/rva22u64,riscv64/rva23u64: "CTZ\t",-"ORR" i += bits.TrailingZeros16(n) n &= n - 1 } @@ -409,6 +522,7 @@ func IterateBits8(n uint8) int { // amd64/v1,amd64/v2:"BSFL",-"BTSL" // amd64/v3:"TZCNTL" // arm64:"RBITW","CLZW",-"ORR" + // riscv64/rva22u64,riscv64/rva23u64: "CTZ\t",-"ORR" i += bits.TrailingZeros8(n) n &= n - 1 } @@ -617,7 +731,7 @@ func Add64MPanicOnOverflowGT(a, b [2]uint64) [2]uint64 { // // This is what happened on PPC64 when compiling // crypto/internal/edwards25519/field.feMulGeneric. -func Add64MultipleChains(a, b, c, d [2]uint64) { +func Add64MultipleChains(a, b, c, d [2]uint64) [2]uint64 { var cx, d1, d2 uint64 a1, a2 := a[0], a[1] b1, b2 := b[0], b[1] @@ -634,6 +748,7 @@ func Add64MultipleChains(a, b, c, d [2]uint64) { d2, _ = bits.Add64(c2, d2, cx) d[0] = d1 d[1] = d2 + return d } // --------------- // @@ -824,6 +939,7 @@ func Sub64MPanicOnOverflowGT(a, b [2]uint64) [2]uint64 { func Mul(x, y uint) (hi, lo uint) { // amd64:"MULQ" // arm64:"UMULH","MUL" + // loong64:"MULV","MULHVU" // ppc64x:"MULHDU","MULLD" // s390x:"MLGR" // mips64: "MULVU" @@ -834,6 +950,7 @@ func Mul(x, y uint) (hi, lo uint) { func Mul64(x, y uint64) (hi, lo uint64) { // amd64:"MULQ" // arm64:"UMULH","MUL" + // loong64:"MULV","MULHVU" // ppc64x:"MULHDU","MULLD" // s390x:"MLGR" // mips64: "MULVU" @@ -843,6 +960,7 @@ func Mul64(x, y uint64) (hi, lo uint64) { func Mul64HiOnly(x, y uint64) uint64 { // arm64:"UMULH",-"MUL" + // loong64:"MULHVU",-"MULV" // riscv64:"MULHU",-"MUL\t" hi, _ := bits.Mul64(x, y) return hi @@ -850,11 +968,24 @@ func Mul64HiOnly(x, y uint64) uint64 { func Mul64LoOnly(x, y uint64) uint64 { // arm64:"MUL",-"UMULH" + // loong64:"MULV",-"MULHVU" // riscv64:"MUL\t",-"MULHU" _, lo := bits.Mul64(x, y) return lo } +func Mul64Const() (uint64, uint64) { + // 7133701809754865664 == 99<<56 + // arm64:"MOVD\t[$]7133701809754865664, R1", "MOVD\t[$]88, R0" + // loong64:"MOVV\t[$]88, R4","MOVV\t[$]7133701809754865664, R5",-"MUL" + return bits.Mul64(99+88<<8, 1<<56) +} + +func MulUintOverflow(p *uint64) []uint64 { + // arm64:"CMP\t[$]72" + return unsafe.Slice(p, 9) +} + // --------------- // // bits.Div* // // --------------- // diff --git a/test/codegen/memcombine.go b/test/codegen/memcombine.go index ff67a442e4b223..fa0e902ac2b485 100644 --- a/test/codegen/memcombine.go +++ b/test/codegen/memcombine.go @@ -19,6 +19,7 @@ func load_le64(b []byte) uint64 { // amd64:`MOVQ\s\(.*\),`,-`MOV[BWL]\t[^$]`,-`OR` // s390x:`MOVDBR\s\(.*\),` // arm64:`MOVD\s\(R[0-9]+\),`,-`MOV[BHW]` + // loong64:`MOVV\s\(R[0-9]+\),` // ppc64le:`MOVD\s`,-`MOV[BHW]Z` // ppc64:`MOVDBR\s`,-`MOV[BHW]Z` return binary.LittleEndian.Uint64(b) @@ -28,6 +29,7 @@ func load_le64_idx(b []byte, idx int) uint64 { // amd64:`MOVQ\s\(.*\)\(.*\*1\),`,-`MOV[BWL]\t[^$]`,-`OR` // s390x:`MOVDBR\s\(.*\)\(.*\*1\),` // arm64:`MOVD\s\(R[0-9]+\)\(R[0-9]+\),`,-`MOV[BHW]` + // loong64:`MOVV\s\(R[0-9]+\)\(R[0-9]+\),` // ppc64le:`MOVD\s`,-`MOV[BHW]Z\s` // ppc64:`MOVDBR\s`,-`MOV[BHW]Z\s` return binary.LittleEndian.Uint64(b[idx:]) @@ -38,6 +40,7 @@ func load_le32(b []byte) uint32 { // 386:`MOVL\s\(.*\),`,-`MOV[BW]`,-`OR` // s390x:`MOVWBR\s\(.*\),` // arm64:`MOVWU\s\(R[0-9]+\),`,-`MOV[BH]` + // loong64:`MOVWU\s\(R[0-9]+\),` // ppc64le:`MOVWZ\s`,-`MOV[BH]Z\s` // ppc64:`MOVWBR\s`,-`MOV[BH]Z\s` return binary.LittleEndian.Uint32(b) @@ -48,6 +51,7 @@ func load_le32_idx(b []byte, idx int) uint32 { // 386:`MOVL\s\(.*\)\(.*\*1\),`,-`MOV[BW]`,-`OR` // s390x:`MOVWBR\s\(.*\)\(.*\*1\),` // arm64:`MOVWU\s\(R[0-9]+\)\(R[0-9]+\),`,-`MOV[BH]` + // loong64:`MOVWU\s\(R[0-9]+\)\(R[0-9]+\),` // ppc64le:`MOVWZ\s`,-`MOV[BH]Z\s` // ppc64:`MOVWBR\s`,-`MOV[BH]Z\s' return binary.LittleEndian.Uint32(b[idx:]) @@ -57,6 +61,7 @@ func load_le16(b []byte) uint16 { // amd64:`MOVWLZX\s\(.*\),`,-`MOVB`,-`OR` // ppc64le:`MOVHZ\s`,-`MOVBZ` // arm64:`MOVHU\s\(R[0-9]+\),`,-`MOVB` + // loong64:`MOVHU\s\(R[0-9]+\),` // s390x:`MOVHBR\s\(.*\),` // ppc64:`MOVHBR\s`,-`MOVBZ` return binary.LittleEndian.Uint16(b) @@ -67,6 +72,7 @@ func load_le16_idx(b []byte, idx int) uint16 { // ppc64le:`MOVHZ\s`,-`MOVBZ` // ppc64:`MOVHBR\s`,-`MOVBZ` // arm64:`MOVHU\s\(R[0-9]+\)\(R[0-9]+\),`,-`MOVB` + // loong64:`MOVHU\s\(R[0-9]+\)\(R[0-9]+\),` // s390x:`MOVHBR\s\(.*\)\(.*\*1\),` return binary.LittleEndian.Uint16(b[idx:]) } @@ -390,6 +396,15 @@ func load_op_no_merge(p, q *int) { } } +func load_op_in_loop(a []int) int { + r := 0 + for _, x := range a { + // amd64:`ADDQ\t\([A-Z]+\)\([A-Z]+\*8\), [A-Z]+` + r += x + } + return r +} + // Make sure offsets are folded into loads and stores. func offsets_fold(_, a [20]byte) (b [20]byte) { // arm64:`MOVD\tcommand-line-arguments\.a\+[0-9]+\(FP\), R[0-9]+`,`MOVD\tR[0-9]+, command-line-arguments\.b\+[0-9]+\(FP\)` @@ -893,9 +908,11 @@ func store32le(p *struct{ a, b uint32 }, x uint64) { p.b = uint32(x >> 32) } func store32be(p *struct{ a, b uint32 }, x uint64) { + // arm64:"STPW" // ppc64:"MOVD",-"MOVW",-"SRD" // s390x:"MOVD",-"MOVW",-"SRD" p.a = uint32(x >> 32) + // arm64:-"STPW" // ppc64:-"MOVW",-"SRD" // s390x:-"MOVW",-"SRD" p.b = uint32(x) @@ -938,3 +955,135 @@ func issue66413(p *struct { p.c = true p.d = 12 } + +func issue70300(v uint64) (b [8]byte) { + // amd64:"MOVQ",-"MOVB" + b[0] = byte(v) + b[1] = byte(v >> 8) + b[2] = byte(v >> 16) + b[3] = byte(v >> 24) + b[4] = byte(v >> 32) + b[5] = byte(v >> 40) + b[6] = byte(v >> 48) + b[7] = byte(v >> 56) + return b +} + +func issue70300Reverse(v uint64) (b [8]byte) { + // amd64:"MOVQ",-"MOVB" + b[7] = byte(v >> 56) + b[6] = byte(v >> 48) + b[5] = byte(v >> 40) + b[4] = byte(v >> 32) + b[3] = byte(v >> 24) + b[2] = byte(v >> 16) + b[1] = byte(v >> 8) + b[0] = byte(v) + return b +} + +// --------------------------------- // +// Arm64 double-register loads // +// --------------------------------- // + +func dwloadI64(p *struct{ a, b int64 }) int64 { + // arm64:"LDP\t" + return p.a + p.b +} +func dwloadI32(p *struct{ a, b int32 }) int32 { + // arm64:"LDPSW\t" + return p.a + p.b +} +func dwloadU32(p *struct{ a, b uint32 }) uint32 { + // arm64:"LDPW\t" + return p.a + p.b +} +func dwloadF64(p *struct{ a, b float64 }) float64 { + // arm64:"FLDPD\t" + return p.a + p.b +} +func dwloadF32(p *struct{ a, b float32 }) float32 { + // arm64:"FLDPS\t" + return p.a + p.b +} + +func dwloadBig(p *struct{ a, b, c, d, e, f int64 }) int64 { + // arm64:"LDP\t\\(", "LDP\t16", "LDP\t32" + return p.c + p.f + p.a + p.e + p.d + p.b +} + +func dwloadArg(a [2]int64) int64 { + // arm64:"LDP\t" + return a[0] + a[1] +} + +func dwloadResult1(p *string) string { + // arm64:"LDP\t\\(R0\\), \\(R0, R1\\)" + return *p +} + +func dwloadResult2(p *[2]int64) (int64, int64) { + // arm64:"LDP\t\\(R0\\), \\(R1, R0\\)" + return p[1], p[0] +} + +// ---------------------------------- // +// Arm64 double-register stores // +// ---------------------------------- // + +func dwstoreI64(p *struct{ a, b int64 }, x, y int64) { + // arm64:"STP\t" + p.a = x + p.b = y +} +func dwstoreI32(p *struct{ a, b int32 }, x, y int32) { + // arm64:"STPW\t" + p.a = x + p.b = y +} +func dwstoreF64(p *struct{ a, b float64 }, x, y float64) { + // arm64:"FSTPD\t" + p.a = x + p.b = y +} +func dwstoreF32(p *struct{ a, b float32 }, x, y float32) { + // arm64:"FSTPS\t" + p.a = x + p.b = y +} + +func dwstoreBig(p *struct{ a, b, c, d, e, f int64 }, a, b, c, d, e, f int64) { + // This is not perfect. We merge b+a, then d+e, then c and f have no pair. + p.c = c + p.f = f + // arm64:`STP\s\(R[0-9]+, R[0-9]+\), \(R[0-9]+\)` + p.a = a + // arm64:`STP\s\(R[0-9]+, R[0-9]+\), 24\(R[0-9]+\)` + p.e = e + p.d = d + p.b = b +} + +func dwstoreRet() [2]int { + // arm64:"STP\t" + return [2]int{5, 6} +} + +func dwstoreLocal(i int) int64 { + var a [2]int64 + a[0] = 5 + // arm64:"STP\t" + a[1] = 6 + return a[i] +} + +func dwstoreOrder(p *struct { + a, b int64 + c, d, e, f bool +}, a, b int64) { + // arm64:"STP\t" + p.a = a + p.c = true + p.e = true + p.b = b +} diff --git a/test/codegen/multiply.go b/test/codegen/multiply.go new file mode 100644 index 00000000000000..8c408cbfbabc19 --- /dev/null +++ b/test/codegen/multiply.go @@ -0,0 +1,372 @@ +// asmcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +// This file contains codegen tests related to strength +// reduction of integer multiply. + +func m0(x int64) int64 { + // amd64: "XORL" + // arm64: "MOVD\tZR" + // loong64: "MOVV\tR0" + return x * 0 +} +func m2(x int64) int64 { + // amd64: "ADDQ" + // arm64: "ADD" + // loong64: "ADDVU" + return x * 2 +} +func m3(x int64) int64 { + // amd64: "LEAQ\t.*[*]2" + // arm64: "ADD\tR[0-9]+<<1," + // loong64: "ALSLV\t[$]1," + return x * 3 +} +func m4(x int64) int64 { + // amd64: "SHLQ\t[$]2," + // arm64: "LSL\t[$]2," + // loong64: "SLLV\t[$]2," + return x * 4 +} +func m5(x int64) int64 { + // amd64: "LEAQ\t.*[*]4" + // arm64: "ADD\tR[0-9]+<<2," + // loong64: "ALSLV\t[$]2," + return x * 5 +} +func m6(x int64) int64 { + // amd64: "LEAQ\t.*[*]1", "LEAQ\t.*[*]2" + // arm64: "ADD\tR[0-9]+,", "ADD\tR[0-9]+<<1," + // loong64: "ADDVU", "ADDVU", "ADDVU" + return x * 6 +} +func m7(x int64) int64 { + // amd64: "LEAQ\t.*[*]2" + // arm64: "LSL\t[$]3,", "SUB\tR[0-9]+," + // loong64: "ALSLV\t[$]1,", "ALSLV\t[$]1," + return x * 7 +} +func m8(x int64) int64 { + // amd64: "SHLQ\t[$]3," + // arm64: "LSL\t[$]3," + // loong64: "SLLV\t[$]3," + return x * 8 +} +func m9(x int64) int64 { + // amd64: "LEAQ\t.*[*]8" + // arm64: "ADD\tR[0-9]+<<3," + // loong64: "ALSLV\t[$]3," + return x * 9 +} +func m10(x int64) int64 { + // amd64: "LEAQ\t.*[*]1", "LEAQ\t.*[*]4" + // arm64: "ADD\tR[0-9]+,", "ADD\tR[0-9]+<<2," + // loong64: "ADDVU", "ALSLV\t[$]2," + return x * 10 +} +func m11(x int64) int64 { + // amd64: "LEAQ\t.*[*]4", "LEAQ\t.*[*]2" + // arm64: "MOVD\t[$]11,", "MUL" + // loong64: "ALSLV\t[$]2,", "ALSLV\t[$]1," + return x * 11 +} +func m12(x int64) int64 { + // amd64: "LEAQ\t.*[*]2", "SHLQ\t[$]2," + // arm64: "LSL\t[$]2,", "ADD\tR[0-9]+<<1," + // loong64: "SLLV", "ALSLV\t[$]1," + return x * 12 +} +func m13(x int64) int64 { + // amd64: "LEAQ\t.*[*]2", "LEAQ\t.*[*]4" + // arm64: "MOVD\t[$]13,", "MUL" + // loong64: "ALSLV\t[$]1,", "ALSLV\t[$]2," + return x * 13 +} +func m14(x int64) int64 { + // amd64: "IMUL3Q\t[$]14," + // arm64: "LSL\t[$]4,", "SUB\tR[0-9]+<<1," + // loong64: "ADDVU", "ALSLV\t[$]1", "ALSLV\t[$]2" + return x * 14 +} +func m15(x int64) int64 { + // amd64: "LEAQ\t.*[*]2", "LEAQ\t.*[*]4" + // arm64: "LSL\t[$]4,", "SUB\tR[0-9]+," + // loong64: "ALSLV\t[$]1,", "ALSLV\t[$]2," + return x * 15 +} +func m16(x int64) int64 { + // amd64: "SHLQ\t[$]4," + // arm64: "LSL\t[$]4," + // loong64: "SLLV\t[$]4," + return x * 16 +} +func m17(x int64) int64 { + // amd64: "LEAQ\t.*[*]1", "LEAQ\t.*[*]8" + // arm64: "ADD\tR[0-9]+<<4," + // loong64: "ALSLV\t[$]" + return x * 17 +} +func m18(x int64) int64 { + // amd64: "LEAQ\t.*[*]1", "LEAQ\t.*[*]8" + // arm64: "ADD\tR[0-9]+,", "ADD\tR[0-9]+<<3," + // loong64: "ADDVU", "ALSLV\t[$]3," + return x * 18 +} +func m19(x int64) int64 { + // amd64: "LEAQ\t.*[*]8", "LEAQ\t.*[*]2" + // arm64: "MOVD\t[$]19,", "MUL" + // loong64: "ALSLV\t[$]3,", "ALSLV\t[$]1," + return x * 19 +} +func m20(x int64) int64 { + // amd64: "LEAQ\t.*[*]4", "SHLQ\t[$]2," + // arm64: "LSL\t[$]2,", "ADD\tR[0-9]+<<2," + // loong64: "SLLV\t[$]2,", "ALSLV\t[$]2," + return x * 20 +} +func m21(x int64) int64 { + // amd64: "LEAQ\t.*[*]4", "LEAQ\t.*[*]4" + // arm64: "MOVD\t[$]21,", "MUL" + // loong64: "ALSLV\t[$]2,", "ALSLV\t[$]2," + return x * 21 +} +func m22(x int64) int64 { + // amd64: "IMUL3Q\t[$]22," + // arm64: "MOVD\t[$]22,", "MUL" + // loong64: "ADDVU", "ALSLV\t[$]2,", "ALSLV\t[$]2," + return x * 22 +} +func m23(x int64) int64 { + // amd64: "IMUL3Q\t[$]23," + // arm64: "MOVD\t[$]23,", "MUL" + // loong64: "ALSLV\t[$]1,", "SUBVU", "ALSLV\t[$]3," + return x * 23 +} +func m24(x int64) int64 { + // amd64: "LEAQ\t.*[*]2", "SHLQ\t[$]3," + // arm64: "LSL\t[$]3,", "ADD\tR[0-9]+<<1," + // loong64: "SLLV\t[$]3", "ALSLV\t[$]1," + return x * 24 +} +func m25(x int64) int64 { + // amd64: "LEAQ\t.*[*]4", "LEAQ\t.*[*]4" + // arm64: "MOVD\t[$]25,", "MUL" + // loong64: "ALSLV\t[$]2,", "ALSLV\t[$]2," + return x * 25 +} +func m26(x int64) int64 { + // amd64: "IMUL3Q\t[$]26," + // arm64: "MOVD\t[$]26,", "MUL" + // loong64: "ADDVU", "ALSLV\t[$]1,", "ALSLV\t[$]3," + return x * 26 +} +func m27(x int64) int64 { + // amd64: "LEAQ\t.*[*]2", "LEAQ\t.*[*]8" + // arm64: "MOVD\t[$]27,", "MUL" + // loong64: "ALSLV\t[$]1,", "ALSLV\t[$]3," + return x * 27 +} +func m28(x int64) int64 { + // amd64: "IMUL3Q\t[$]28," + // arm64: "LSL\t[$]5, "SUB\tR[0-9]+<<2," + // loong64: "ALSLV\t[$]1,","SLLV\t[$]2,","ALSLV\t[$]3," + return x * 28 +} +func m29(x int64) int64 { + // amd64: "IMUL3Q\t[$]29," + // arm64: "MOVD\t[$]29,", "MUL" + // loong64: "ALSLV\t[$]1,","SLLV\t[$]5,","SUBVU" + return x * 29 +} +func m30(x int64) int64 { + // amd64: "IMUL3Q\t[$]30," + // arm64: "LSL\t[$]5,", "SUB\tR[0-9]+<<1," + // loong64: "ADDVU","SLLV\t[$]5,","SUBVU" + return x * 30 +} +func m31(x int64) int64 { + // amd64: "SHLQ\t[$]5,", "SUBQ" + // arm64: "LSL\t[$]5,", "SUB\tR[0-9]+," + // loong64: "SLLV\t[$]5,","SUBVU" + return x * 31 +} +func m32(x int64) int64 { + // amd64: "SHLQ\t[$]5," + // arm64: "LSL\t[$]5," + // loong64: "SLLV\t[$]5," + return x * 32 +} +func m33(x int64) int64 { + // amd64: "SHLQ\t[$]2,", "LEAQ\t.*[*]8" + // arm64: "ADD\tR[0-9]+<<5," + // loong64: "ADDVU", "ALSLV\t[$]4," + return x * 33 +} +func m34(x int64) int64 { + // amd64: "SHLQ\t[$]5,", "LEAQ\t.*[*]2" + // arm64: "ADD\tR[0-9]+,", "ADD\tR[0-9]+<<4," + // loong64: "ADDVU", "ALSLV\t[$]4," + return x * 34 +} +func m35(x int64) int64 { + // amd64: "IMUL3Q\t[$]35," + // arm64: "MOVD\t[$]35,", "MUL" + // loong64: "ALSLV\t[$]4,", "ALSLV\t[$]1," + return x * 35 +} +func m36(x int64) int64 { + // amd64: "LEAQ\t.*[*]8", "SHLQ\t[$]2," + // arm64: "LSL\t[$]2,", "ADD\tR[0-9]+<<3," + // loong64: "SLLV\t[$]2,", "ALSLV\t[$]3," + return x * 36 +} +func m37(x int64) int64 { + // amd64: "LEAQ\t.*[*]8", "LEAQ\t.*[*]4" + // arm64: "MOVD\t[$]37,", "MUL" + // loong64: "ALSLV\t[$]3,", "ALSLV\t[$]2," + return x * 37 +} +func m38(x int64) int64 { + // amd64: "IMUL3Q\t[$]38," + // arm64: "MOVD\t[$]38,", "MUL" + // loong64: "ALSLV\t[$]3,", "ALSLV\t[$]2," + return x * 38 +} +func m39(x int64) int64 { + // amd64: "IMUL3Q\t[$]39," + // arm64: "MOVD\t[$]39,", "MUL" + // loong64: "ALSLV\t[$]2,", "SUBVU", "ALSLV\t[$]3," + return x * 39 +} +func m40(x int64) int64 { + // amd64: "LEAQ\t.*[*]4", "SHLQ\t[$]3," + // arm64: "LSL\t[$]3,", "ADD\tR[0-9]+<<2," + // loong64: "SLLV\t[$]3,", "ALSLV\t[$]2," + return x * 40 +} + +func mn1(x int64) int64 { + // amd64: "NEGQ\t" + // arm64: "NEG\tR[0-9]+," + // loong64: "SUBVU\tR[0-9], R0," + return x * -1 +} +func mn2(x int64) int64 { + // amd64: "NEGQ", "ADDQ" + // arm64: "NEG\tR[0-9]+<<1," + // loong64: "ADDVU","SUBVU\tR[0-9], R0," + return x * -2 +} +func mn3(x int64) int64 { + // amd64: "NEGQ", "LEAQ\t.*[*]2" + // arm64: "SUB\tR[0-9]+<<2," + // loong64: "SUBVU", "ALSLV\t[$]1," + return x * -3 +} +func mn4(x int64) int64 { + // amd64: "NEGQ", "SHLQ\t[$]2," + // arm64: "NEG\tR[0-9]+<<2," + // loong64: "SLLV\t[$]2,","SUBVU\tR[0-9], R0," + return x * -4 +} +func mn5(x int64) int64 { + // amd64: "NEGQ", "LEAQ\t.*[*]4" + // arm64: "NEG\tR[0-9]+,", "ADD\tR[0-9]+<<2," + // loong64: "SUBVU", "ALSLV\t[$]2," + return x * -5 +} +func mn6(x int64) int64 { + // amd64: "IMUL3Q\t[$]-6," + // arm64: "ADD\tR[0-9]+,", "SUB\tR[0-9]+<<2," + // loong64: "ADDVU", "SUBVU", "ALSLV\t[$]3," + return x * -6 +} +func mn7(x int64) int64 { + // amd64: "NEGQ", "LEAQ\t.*[*]8" + // arm64: "SUB\tR[0-9]+<<3," + // loong64: "SUBVU", "ALSLV\t[$]3," + return x * -7 +} +func mn8(x int64) int64 { + // amd64: "NEGQ", "SHLQ\t[$]3," + // arm64: "NEG\tR[0-9]+<<3," + // loong64: "SLLV\t[$]3","SUBVU\tR[0-9], R0," + return x * -8 +} +func mn9(x int64) int64 { + // amd64: "NEGQ", "LEAQ\t.*[*]8" + // arm64: "NEG\tR[0-9]+,", "ADD\tR[0-9]+<<3," + // loong64: "SUBVU", "ALSLV\t[$]3," + return x * -9 +} +func mn10(x int64) int64 { + // amd64: "IMUL3Q\t[$]-10," + // arm64: "MOVD\t[$]-10,", "MUL" + // loong64: "ADDVU", "ALSLV\t[$]3", "SUBVU" + return x * -10 +} +func mn11(x int64) int64 { + // amd64: "IMUL3Q\t[$]-11," + // arm64: "MOVD\t[$]-11,", "MUL" + // loong64: "ALSLV\t[$]2,", "SUBVU", "ALSLV\t[$]4," + return x * -11 +} +func mn12(x int64) int64 { + // amd64: "IMUL3Q\t[$]-12," + // arm64: "LSL\t[$]2,", "SUB\tR[0-9]+<<2," + // loong64: "SUBVU", "SLLV\t[$]2,", "ALSLV\t[$]4," + return x * -12 +} +func mn13(x int64) int64 { + // amd64: "IMUL3Q\t[$]-13," + // arm64: "MOVD\t[$]-13,", "MUL" + // loong64: "ALSLV\t[$]4,", "SLLV\t[$]2, ", "SUBVU" + return x * -13 +} +func mn14(x int64) int64 { + // amd64: "IMUL3Q\t[$]-14," + // arm64: "ADD\tR[0-9]+,", "SUB\tR[0-9]+<<3," + // loong64: "ADDVU", "SUBVU", "ALSLV\t[$]4," + return x * -14 +} +func mn15(x int64) int64 { + // amd64: "SHLQ\t[$]4,", "SUBQ" + // arm64: "SUB\tR[0-9]+<<4," + // loong64: "SUBVU", "ALSLV\t[$]4," + return x * -15 +} +func mn16(x int64) int64 { + // amd64: "NEGQ", "SHLQ\t[$]4," + // arm64: "NEG\tR[0-9]+<<4," + // loong64: "SLLV\t[$]4,","SUBVU\tR[0-9], R0," + return x * -16 +} +func mn17(x int64) int64 { + // amd64: "IMUL3Q\t[$]-17," + // arm64: "NEG\tR[0-9]+,", "ADD\tR[0-9]+<<4," + // loong64: "SUBVU", "ALSLV\t[$]4," + return x * -17 +} +func mn18(x int64) int64 { + // amd64: "IMUL3Q\t[$]-18," + // arm64: "MOVD\t[$]-18,", "MUL" + // loong64: "ADDVU", "ALSLV\t[$]4,", "SUBVU" + return x * -18 +} +func mn19(x int64) int64 { + // amd64: "IMUL3Q\t[$]-19," + // arm64: "MOVD\t[$]-19,", "MUL" + // loong64: "ALSLV\t[$]1,", "ALSLV\t[$]4,", "SUBVU" + return x * -19 +} +func mn20(x int64) int64 { + // amd64: "IMUL3Q\t[$]-20," + // arm64: "MOVD\t[$]-20,", "MUL" + // loong64: "SLLV\t[$]2,", "ALSLV\t[$]4,", "SUBVU" + return x * -20 +} diff --git a/test/codegen/reflect_type.go b/test/codegen/reflect_type.go new file mode 100644 index 00000000000000..b92a9567f14eac --- /dev/null +++ b/test/codegen/reflect_type.go @@ -0,0 +1,21 @@ +// asmcheck + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +import "reflect" + +func intPtrTypeSize() uintptr { + // amd64:"MOVL\t[$]8,",-"CALL" + // arm64:"MOVD\t[$]8,",-"CALL" + return reflect.TypeFor[*int]().Size() +} + +func intPtrTypeKind() reflect.Kind { + // amd64:"MOVL\t[$]22,",-"CALL" + // arm64:"MOVD\t[$]22,",-"CALL" + return reflect.TypeFor[*int]().Kind() +} diff --git a/test/codegen/schedule.go b/test/codegen/schedule.go new file mode 100644 index 00000000000000..aafffd817ba7aa --- /dev/null +++ b/test/codegen/schedule.go @@ -0,0 +1,17 @@ +// asmcheck + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +func f(n int) int { + r := 0 + // arm64:-"MOVD\t R" + // amd64:-"LEAQ","INCQ" + for i := range n { + r += i + } + return r +} diff --git a/test/codegen/shift.go b/test/codegen/shift.go index 6a2a6c40cdaed4..4b0885a4dddd7b 100644 --- a/test/codegen/shift.go +++ b/test/codegen/shift.go @@ -11,95 +11,161 @@ package codegen // ------------------ // func lshConst64x64(v int64) int64 { + // loong64:"SLLV" // ppc64x:"SLD" // riscv64:"SLLI",-"AND",-"SLTIU" return v << uint64(33) } func rshConst64Ux64(v uint64) uint64 { + // loong64:"SRLV" // ppc64x:"SRD" // riscv64:"SRLI\t",-"AND",-"SLTIU" return v >> uint64(33) } func rshConst64Ux64Overflow32(v uint32) uint64 { + // loong64:"MOVV\tR0,",-"SRL\t" // riscv64:"MOV\t\\$0,",-"SRL" return uint64(v) >> 32 } func rshConst64Ux64Overflow16(v uint16) uint64 { + // loong64:"MOVV\tR0,",-"SRLV" // riscv64:"MOV\t\\$0,",-"SRL" return uint64(v) >> 16 } func rshConst64Ux64Overflow8(v uint8) uint64 { + // loong64:"MOVV\tR0,",-"SRLV" // riscv64:"MOV\t\\$0,",-"SRL" return uint64(v) >> 8 } func rshConst64x64(v int64) int64 { + // loong64:"SRAV" // ppc64x:"SRAD" // riscv64:"SRAI\t",-"OR",-"SLTIU" return v >> uint64(33) } func rshConst64x64Overflow32(v int32) int64 { + // loong64:"SRA\t\\$31" // riscv64:"SRAIW",-"SLLI",-"SRAI\t" return int64(v) >> 32 } func rshConst64x64Overflow16(v int16) int64 { + // loong64:"SLLV\t\\$48","SRAV\t\\$63" // riscv64:"SLLI","SRAI",-"SRAIW" return int64(v) >> 16 } func rshConst64x64Overflow8(v int8) int64 { + // loong64:"SLLV\t\\$56","SRAV\t\\$63" // riscv64:"SLLI","SRAI",-"SRAIW" return int64(v) >> 8 } +func lshConst32x1(v int32) int32 { + // amd64:"ADDL", -"SHLL" + return v << 1 +} + +func lshConst64x1(v int64) int64 { + // amd64:"ADDQ", -"SHLQ" + return v << 1 +} + func lshConst32x64(v int32) int32 { + // loong64:"SLL\t" // ppc64x:"SLW" // riscv64:"SLLI",-"AND",-"SLTIU", -"MOVW" return v << uint64(29) } func rshConst32Ux64(v uint32) uint32 { + // loong64:"SRL\t" // ppc64x:"SRW" // riscv64:"SRLIW",-"AND",-"SLTIU", -"MOVW" return v >> uint64(29) } func rshConst32x64(v int32) int32 { + // loong64:"SRA\t" // ppc64x:"SRAW" // riscv64:"SRAIW",-"OR",-"SLTIU", -"MOVW" return v >> uint64(29) } func lshConst64x32(v int64) int64 { + // loong64:"SLLV" // ppc64x:"SLD" // riscv64:"SLLI",-"AND",-"SLTIU" return v << uint32(33) } func rshConst64Ux32(v uint64) uint64 { + // loong64:"SRLV" // ppc64x:"SRD" // riscv64:"SRLI\t",-"AND",-"SLTIU" return v >> uint32(33) } func rshConst64x32(v int64) int64 { + // loong64:"SRAV" // ppc64x:"SRAD" // riscv64:"SRAI\t",-"OR",-"SLTIU" return v >> uint32(33) } +func lshConst32x1Add(x int32) int32 { + // amd64:"SHLL\t[$]2" + // loong64:"SLL\t[$]2" + // riscv64:"SLLI\t[$]2" + return (x + x) << 1 +} + +func lshConst64x1Add(x int64) int64 { + // amd64:"SHLQ\t[$]2" + // loong64:"SLLV\t[$]2" + // riscv64:"SLLI\t[$]2" + return (x + x) << 1 +} + +func lshConst32x2Add(x int32) int32 { + // amd64:"SHLL\t[$]3" + // loong64:"SLL\t[$]3" + // riscv64:"SLLI\t[$]3" + return (x + x) << 2 +} + +func lshConst64x2Add(x int64) int64 { + // amd64:"SHLQ\t[$]3" + // loong64:"SLLV\t[$]3" + // riscv64:"SLLI\t[$]3" + return (x + x) << 2 +} + +func lshConst32x31Add(x int32) int32 { + // loong64:-"SLL\t","MOVV\tR0" + // riscv64:-"SLLI","MOV\t[$]0" + return (x + x) << 31 +} + +func lshConst64x63Add(x int64) int64 { + // loong64:-"SLLV","MOVV\tR0" + // riscv64:-"SLLI","MOV\t[$]0" + return (x + x) << 63 +} + // ------------------ // // masked shifts // // ------------------ // func lshMask64x64(v int64, s uint64) int64 { // arm64:"LSL",-"AND" + // loong64:"SLLV",-"AND" // ppc64x:"RLDICL",-"ORN",-"ISEL" // riscv64:"SLL",-"AND\t",-"SLTIU" // s390x:-"RISBGZ",-"AND",-"LOCGR" @@ -108,6 +174,7 @@ func lshMask64x64(v int64, s uint64) int64 { func rshMask64Ux64(v uint64, s uint64) uint64 { // arm64:"LSR",-"AND",-"CSEL" + // loong64:"SRLV",-"AND" // ppc64x:"RLDICL",-"ORN",-"ISEL" // riscv64:"SRL\t",-"AND\t",-"SLTIU" // s390x:-"RISBGZ",-"AND",-"LOCGR" @@ -116,6 +183,7 @@ func rshMask64Ux64(v uint64, s uint64) uint64 { func rshMask64x64(v int64, s uint64) int64 { // arm64:"ASR",-"AND",-"CSEL" + // loong64:"SRAV",-"AND" // ppc64x:"RLDICL",-"ORN",-"ISEL" // riscv64:"SRA\t",-"OR",-"SLTIU" // s390x:-"RISBGZ",-"AND",-"LOCGR" @@ -124,14 +192,21 @@ func rshMask64x64(v int64, s uint64) int64 { func lshMask32x64(v int32, s uint64) int32 { // arm64:"LSL",-"AND" + // loong64:"SLL\t","AND","SGTU","MASKEQZ" // ppc64x:"ISEL",-"ORN" // riscv64:"SLL",-"AND\t",-"SLTIU" // s390x:-"RISBGZ",-"AND",-"LOCGR" return v << (s & 63) } +func lsh5Mask32x64(v int32, s uint64) int32 { + // loong64:"SLL\t",-"AND" + return v << (s & 31) +} + func rshMask32Ux64(v uint32, s uint64) uint32 { // arm64:"LSR",-"AND" + // loong64:"SRL\t","AND","SGTU","MASKEQZ" // ppc64x:"ISEL",-"ORN" // riscv64:"SRLW","SLTIU","NEG","AND\t",-"SRL\t" // s390x:-"RISBGZ",-"AND",-"LOCGR" @@ -139,12 +214,14 @@ func rshMask32Ux64(v uint32, s uint64) uint32 { } func rsh5Mask32Ux64(v uint32, s uint64) uint32 { + // loong64:"SRL\t",-"AND" // riscv64:"SRLW",-"AND\t",-"SLTIU",-"SRL\t" return v >> (s & 31) } func rshMask32x64(v int32, s uint64) int32 { // arm64:"ASR",-"AND" + // loong64:"SRA\t","AND","SGTU","SUBVU","OR" // ppc64x:"ISEL",-"ORN" // riscv64:"SRAW","OR","SLTIU" // s390x:-"RISBGZ",-"AND",-"LOCGR" @@ -152,12 +229,14 @@ func rshMask32x64(v int32, s uint64) int32 { } func rsh5Mask32x64(v int32, s uint64) int32 { + // loong64:"SRA\t",-"AND" // riscv64:"SRAW",-"OR",-"SLTIU" return v >> (s & 31) } func lshMask64x32(v int64, s uint32) int64 { // arm64:"LSL",-"AND" + // loong64:"SLLV",-"AND" // ppc64x:"RLDICL",-"ORN" // riscv64:"SLL",-"AND\t",-"SLTIU" // s390x:-"RISBGZ",-"AND",-"LOCGR" @@ -166,6 +245,7 @@ func lshMask64x32(v int64, s uint32) int64 { func rshMask64Ux32(v uint64, s uint32) uint64 { // arm64:"LSR",-"AND",-"CSEL" + // loong64:"SRLV",-"AND" // ppc64x:"RLDICL",-"ORN" // riscv64:"SRL\t",-"AND\t",-"SLTIU" // s390x:-"RISBGZ",-"AND",-"LOCGR" @@ -174,6 +254,7 @@ func rshMask64Ux32(v uint64, s uint32) uint64 { func rshMask64x32(v int64, s uint32) int64 { // arm64:"ASR",-"AND",-"CSEL" + // loong64:"SRAV",-"AND" // ppc64x:"RLDICL",-"ORN",-"ISEL" // riscv64:"SRA\t",-"OR",-"SLTIU" // s390x:-"RISBGZ",-"AND",-"LOCGR" @@ -462,21 +543,25 @@ func checkMergedShifts64(a [256]uint32, b [256]uint64, c [256]byte, v uint64) { a[2] = a[v>>25&0x7F] // ppc64x: -"CLRLSLDI", "RLWNM\t[$]3, R[0-9]+, [$]29, [$]29, R[0-9]+" a[3] = a[(v>>31)&0x01] - // ppc64x: "SRD", "CLRLSLDI", -"RLWNM" - a[4] = a[(v>>30)&0x07] - // ppc64x: "SRD", "CLRLSLDI", -"RLWNM" - a[5] = a[(v>>32)&0x01] - // ppc64x: "SRD", "CLRLSLDI", -"RLWNM" - a[6] = a[(v>>34)&0x03] // ppc64x: -"CLRLSLDI", "RLWNM\t[$]12, R[0-9]+, [$]21, [$]28, R[0-9]+" b[0] = b[uint8(v>>23)] // ppc64x: -"CLRLSLDI", "RLWNM\t[$]15, R[0-9]+, [$]21, [$]28, R[0-9]+" b[1] = b[(v>>20)&0xFF] // ppc64x: "RLWNM", -"SLD" b[2] = b[((uint64((uint32(v) >> 21)) & 0x3f) << 4)] + // ppc64x: -"RLWNM" + b[3] = (b[3] << 24) & 0xFFFFFF000000 + // ppc64x: "RLWNM\t[$]24, R[0-9]+, [$]0, [$]7," + b[4] = (b[4] << 24) & 0xFF000000 + // ppc64x: "RLWNM\t[$]24, R[0-9]+, [$]0, [$]7," + b[5] = (b[5] << 24) & 0xFF00000F + // ppc64x: -"RLWNM" + b[6] = (b[6] << 0) & 0xFF00000F + // ppc64x: "RLWNM\t[$]4, R[0-9]+, [$]28, [$]31," + b[7] = (b[7] >> 28) & 0xF // ppc64x: "RLWNM\t[$]11, R[0-9]+, [$]10, [$]15" c[0] = c[((v>>5)&0x3F)<<16] - // ppc64x: "RLWNM\t[$]0, R[0-9]+, [$]19, [$]24" + // ppc64x: "ANDCC\t[$]8064," c[1] = c[((v>>7)&0x3F)<<7] } @@ -520,3 +605,93 @@ func checkShiftToMask(u []uint64, s []int64) { // amd64:-"SHR",-"SHL","ANDQ" u[1] = u[1] << 5 >> 5 } + +// +// Left shift with addition. +// + +func checkLeftShiftWithAddition(a int64, b int64) int64 { + // riscv64/rva20u64: "SLLI","ADD" + // riscv64/rva22u64,riscv64/rva23u64: "SH1ADD" + a = a + b<<1 + // riscv64/rva20u64: "SLLI","ADD" + // riscv64/rva22u64,riscv64/rva23u64: "SH2ADD" + a = a + b<<2 + // riscv64/rva20u64: "SLLI","ADD" + // riscv64/rva22u64,riscv64/rva23u64: "SH3ADD" + a = a + b<<3 + return a +} + +// +// Convert and shift. +// + +func rsh64Uto32U(v uint64) uint32 { + x := uint32(v) + // riscv64:"MOVWU" + if x > 8 { + // riscv64:"SRLIW",-"MOVWU",-"SLLI" + x >>= 2 + } + return x +} + +func rsh64Uto16U(v uint64) uint16 { + x := uint16(v) + // riscv64:"MOVHU" + if x > 8 { + // riscv64:"SLLI","SRLI" + x >>= 2 + } + return x +} + +func rsh64Uto8U(v uint64) uint8 { + x := uint8(v) + // riscv64:"MOVBU" + if x > 8 { + // riscv64:"SLLI","SRLI" + x >>= 2 + } + return x +} + +func rsh64to32(v int64) int32 { + x := int32(v) + // riscv64:"MOVW" + if x > 8 { + // riscv64:"SRAIW",-"MOVW",-"SLLI" + x >>= 2 + } + return x +} + +func rsh64to16(v int64) int16 { + x := int16(v) + // riscv64:"MOVH" + if x > 8 { + // riscv64:"SLLI","SRAI" + x >>= 2 + } + return x +} + +func rsh64to8(v int64) int8 { + x := int8(v) + // riscv64:"MOVB" + if x > 8 { + // riscv64:"SLLI","SRAI" + x >>= 2 + } + return x +} + +// We don't need to worry about shifting +// more than the type size. +// (There is still a negative shift test, but +// no shift-too-big test.) +func signedModShift(i int) int64 { + // arm64:-"CMP",-"CSEL" + return 1 << (i % 64) +} diff --git a/test/codegen/slices.go b/test/codegen/slices.go index a38fe77e3f669d..1d918a3a0a686e 100644 --- a/test/codegen/slices.go +++ b/test/codegen/slices.go @@ -47,6 +47,7 @@ func SliceExtensionConst(s []int) []int { // amd64:-`.*runtime\.makeslice` // amd64:-`.*runtime\.panicmakeslicelen` // amd64:"MOVUPS\tX15" + // loong64:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.makeslice` // ppc64x:-`.*runtime\.panicmakeslicelen` @@ -58,6 +59,7 @@ func SliceExtensionConstInt64(s []int) []int { // amd64:-`.*runtime\.makeslice` // amd64:-`.*runtime\.panicmakeslicelen` // amd64:"MOVUPS\tX15" + // loong64:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.makeslice` // ppc64x:-`.*runtime\.panicmakeslicelen` @@ -69,6 +71,7 @@ func SliceExtensionConstUint64(s []int) []int { // amd64:-`.*runtime\.makeslice` // amd64:-`.*runtime\.panicmakeslicelen` // amd64:"MOVUPS\tX15" + // loong64:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.makeslice` // ppc64x:-`.*runtime\.panicmakeslicelen` @@ -80,16 +83,18 @@ func SliceExtensionConstUint(s []int) []int { // amd64:-`.*runtime\.makeslice` // amd64:-`.*runtime\.panicmakeslicelen` // amd64:"MOVUPS\tX15" + // loong64:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.memclrNoHeapPointers` // ppc64x:-`.*runtime\.makeslice` // ppc64x:-`.*runtime\.panicmakeslicelen` return append(s, make([]int, uint(1<<2))...) } -// On ppc64x continue to use memclrNoHeapPointers +// On ppc64x and loong64 continue to use memclrNoHeapPointers // for sizes >= 512. func SliceExtensionConst512(s []int) []int { // amd64:-`.*runtime\.memclrNoHeapPointers` + // loong64:`.*runtime\.memclrNoHeapPointers` // ppc64x:`.*runtime\.memclrNoHeapPointers` return append(s, make([]int, 1<<9)...) } @@ -412,6 +417,15 @@ func SliceWithSubtractBound(a []int, b int) []int { return a[(3 - b):] } +// --------------------------------------- // +// ARM64 folding for slice masks // +// --------------------------------------- // + +func SliceAndIndex(a []int, b int) int { + // arm64:"AND\tR[0-9]+->63","ADD\tR[0-9]+<<3" + return a[b:][b] +} + // --------------------------------------- // // Code generation for unsafe.Slice // // --------------------------------------- // @@ -424,3 +438,21 @@ func Slice0(p *struct{}, i int) []struct{} { // amd64:-"MULQ" return unsafe.Slice(p, i) } + +// --------------------------------------- // +// Code generation for slice bounds // +// checking comparison // +// --------------------------------------- // + +func SlicePut(a []byte, c uint8) []byte { + // arm64:`CBZ\tR1` + a[0] = c + // arm64:`CMP\t\$1, R1` + a = a[1:] + a[0] = c + // arm64:`CMP\t\$2, R1` + a = a[1:] + a[0] = c + a = a[1:] + return a +} diff --git a/test/codegen/spills.go b/test/codegen/spills.go new file mode 100644 index 00000000000000..c8ac9859a48c27 --- /dev/null +++ b/test/codegen/spills.go @@ -0,0 +1,31 @@ +// asmcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +func i64(a, b int64) int64 { // arm64:`STP\s`,`LDP\s` + g() + return a + b +} + +func i32(a, b int32) int32 { // arm64:`STPW`,`LDPW` + g() + return a + b +} + +func f64(a, b float64) float64 { // arm64:`FSTPD`,`FLDPD` + g() + return a + b +} + +func f32(a, b float32) float32 { // arm64:`FSTPS`,`FLDPS` + g() + return a + b +} + +//go:noinline +func g() { +} diff --git a/test/codegen/stack.go b/test/codegen/stack.go index 65c9868d67086d..59284ae88862a3 100644 --- a/test/codegen/stack.go +++ b/test/codegen/stack.go @@ -6,7 +6,10 @@ package codegen -import "runtime" +import ( + "runtime" + "unsafe" +) // This file contains code generation tests related to the use of the // stack. @@ -128,6 +131,29 @@ func spillSlotReuse() { getp2()[nopInt()] = 0 } +// Check that no stack frame space is needed for simple slice initialization with underlying structure. +type mySlice struct { + array unsafe.Pointer + len int + cap int +} + +// amd64:"TEXT\t.*, [$]0-" +func sliceInit(base uintptr) []uintptr { + const ptrSize = 8 + size := uintptr(4096) + bitmapSize := size / ptrSize / 8 + elements := int(bitmapSize / ptrSize) + var sl mySlice + sl = mySlice{ + unsafe.Pointer(base + size - bitmapSize), + elements, + elements, + } + // amd64:-"POPQ",-"SP" + return *(*[]uintptr)(unsafe.Pointer(&sl)) +} + //go:noinline func nopInt() int { return 0 @@ -142,3 +168,9 @@ func getp1() *[4]int { func getp2() *[4]int { return nil } + +// Store to an argument without read can be removed. +func storeArg(a [2]int) { + // amd64:-`MOVQ\t\$123,.*\.a\+\d+\(SP\)` + a[1] = 123 +} diff --git a/test/codegen/strings.go b/test/codegen/strings.go index f98c062d1bbd0e..0b3ca7016ffc58 100644 --- a/test/codegen/strings.go +++ b/test/codegen/strings.go @@ -6,6 +6,8 @@ package codegen +import "strings" + // This file contains code generation tests related to the handling of // string types. @@ -21,11 +23,16 @@ func CountBytes(s []byte) int { func ToByteSlice() []byte { // Issue #24698 // amd64:`LEAQ\ttype:\[3\]uint8` - // amd64:`CALL\truntime\.newobject` + // amd64:`CALL\truntime\.mallocTiny3` // amd64:-`.*runtime.stringtoslicebyte` return []byte("foo") } +func ConvertToByteSlice(a, b, c string) []byte { + // amd64:`.*runtime.concatbyte3` + return []byte(a + b + c) +} + // Loading from read-only symbols should get transformed into constants. func ConstantLoad() { // 12592 = 0x3130 @@ -34,6 +41,7 @@ func ConstantLoad() { // 386:`MOVW\t\$12592, \(`,`MOVB\t\$50, 2\(` // arm:`MOVW\t\$48`,`MOVW\t\$49`,`MOVW\t\$50` // arm64:`MOVD\t\$12592`,`MOVD\t\$50` + // loong64:`MOVV\t\$12592`,`MOVV\t\$50` // wasm:`I64Const\t\$12592`,`I64Store16\t\$0`,`I64Const\t\$50`,`I64Store8\t\$2` // mips64:`MOVV\t\$48`,`MOVV\t\$49`,`MOVV\t\$50` bsink = []byte("012") @@ -43,6 +51,7 @@ func ConstantLoad() { // amd64:`MOVL\t\$858927408`,`MOVW\t\$13620, 4\(` // 386:`MOVL\t\$858927408`,`MOVW\t\$13620, 4\(` // arm64:`MOVD\t\$858927408`,`MOVD\t\$13620` + // loong64:`MOVV\t\$858927408`,`MOVV\t\$13620` // wasm:`I64Const\t\$858927408`,`I64Store32\t\$0`,`I64Const\t\$13620`,`I64Store16\t\$4` bsink = []byte("012345") @@ -51,19 +60,23 @@ func ConstantLoad() { // amd64:`MOVQ\t\$3978425819141910832`,`MOVQ\t\$7306073769690871863` // 386:`MOVL\t\$858927408, \(`,`DUFFCOPY` // arm64:`MOVD\t\$3978425819141910832`,`MOVD\t\$7306073769690871863`,`MOVD\t\$15` + // loong64:`MOVV\t\$3978425819141910832`,`MOVV\t\$7306073769690871863`,`MOVV\t\$15` // wasm:`I64Const\t\$3978425819141910832`,`I64Store\t\$0`,`I64Const\t\$7306073769690871863`,`I64Store\t\$7` bsink = []byte("0123456789abcde") // 56 = 0x38 // amd64:`MOVQ\t\$3978425819141910832`,`MOVB\t\$56` + // loong64:`MOVV\t\$3978425819141910832`,`MOVV\t\$56` bsink = []byte("012345678") // 14648 = 0x3938 // amd64:`MOVQ\t\$3978425819141910832`,`MOVW\t\$14648` + // loong64:`MOVV\t\$3978425819141910832`,`MOVV\t\$14648` bsink = []byte("0123456789") // 1650538808 = 0x62613938 // amd64:`MOVQ\t\$3978425819141910832`,`MOVL\t\$1650538808` + // loong64:`MOVV\t\$3978425819141910832`,`MOVV\t\$1650538808` bsink = []byte("0123456789ab") } @@ -78,3 +91,23 @@ func NotEqualSelf(s string) bool { } var bsink []byte + +func HasPrefix3(s string) bool { + // amd64:-`.*memequal.*` + return strings.HasPrefix(s, "str") +} + +func HasPrefix5(s string) bool { + // amd64:-`.*memequal.*` + return strings.HasPrefix(s, "strin") +} + +func HasPrefix6(s string) bool { + // amd64:-`.*memequal.*` + return strings.HasPrefix(s, "string") +} + +func HasPrefix7(s string) bool { + // amd64:-`.*memequal.*` + return strings.HasPrefix(s, "strings") +} diff --git a/test/codegen/switch.go b/test/codegen/switch.go index 980ea7056192f3..1a66a5ddf84f81 100644 --- a/test/codegen/switch.go +++ b/test/codegen/switch.go @@ -25,6 +25,7 @@ func f(x string) int { func square(x int) int { // amd64:`JMP\s\(.*\)\(.*\)$` // arm64:`MOVD\s\(R.*\)\(R.*<<3\)`,`JMP\s\(R.*\)$` + // loong64: `ALSLV`,`MOVV`,`JMP` switch x { case 1: return 1 @@ -51,6 +52,7 @@ func square(x int) int { func length(x string) int { // amd64:`JMP\s\(.*\)\(.*\)$` // arm64:`MOVD\s\(R.*\)\(R.*<<3\)`,`JMP\s\(R.*\)$` + // loong64:`ALSLV`,`MOVV`,`JMP` switch x { case "a": return 1 @@ -183,3 +185,17 @@ func interfaceConv(x IJ) I { // arm64:`CALL\truntime.typeAssert`,`LDAR`,`MOVWU\t16\(R0\)`,`MOVD\t\(R.*\)\(R.*\)` return x } + +// Make sure we can constant fold after inlining. See issue 71699. +func stringSwitchInlineable(s string) { + switch s { + case "foo", "bar", "baz", "goo": + default: + println("no") + } +} +func stringSwitch() { + // amd64:-"CMP",-"CALL" + // arm64:-"CMP",-"CALL" + stringSwitchInlineable("foo") +} diff --git a/test/codegen/typeswitch.go b/test/codegen/typeswitch.go new file mode 100644 index 00000000000000..93f8e84269f5e5 --- /dev/null +++ b/test/codegen/typeswitch.go @@ -0,0 +1,67 @@ +// asmcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +type Ix interface { + X() +} + +type Iy interface { + Y() +} + +type Iz interface { + Z() +} + +func swXYZ(a Ix) { + switch t := a.(type) { + case Iy: // amd64:-".*typeAssert" + t.Y() + case Iz: // amd64:-".*typeAssert" + t.Z() + } +} + +type Ig[T any] interface { + G() T +} + +func swGYZ[T any](a Ig[T]) { + switch t := a.(type) { + case Iy: // amd64:-".*typeAssert" + t.Y() + case Iz: // amd64:-".*typeAssert" + t.Z() + case interface{ G() T }: // amd64:-".*typeAssert",-".*assertE2I\\(",".*assertE2I2" + t.G() + } +} + +func swE2G[T any](a any) { + switch t := a.(type) { + case Iy: + t.Y() + case Ig[T]: // amd64:-".*assertE2I\\(",".*assertE2I2" + t.G() + } +} + +func swI2G[T any](a Ix) { + switch t := a.(type) { + case Iy: + t.Y() + case Ig[T]: // amd64:-".*assertE2I\\(",".*assertE2I2" + t.G() + } +} + +func swCaller() { + swGYZ[int]((Ig[int])(nil)) + swE2G[int]((Ig[int])(nil)) + swI2G[int]((Ix)(nil)) +} diff --git a/test/codegen/unique.go b/test/codegen/unique.go new file mode 100644 index 00000000000000..8ddc986c26875f --- /dev/null +++ b/test/codegen/unique.go @@ -0,0 +1,24 @@ +// asmcheck + +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +import "unique" + +func BytesToHandle(b []byte) unique.Handle[string] { + // amd64:-`.*runtime\.slicebytetostring\(` + return unique.Make(string(b)) +} + +type Pair struct { + S1 string + S2 string +} + +func BytesPairToHandle(b1, b2 []byte) unique.Handle[Pair] { + // TODO: should not copy b1 and b2. + return unique.Make(Pair{string(b1), string(b2)}) +} diff --git a/test/codegen/unsafe.go b/test/codegen/unsafe.go new file mode 100644 index 00000000000000..1f1bdf2a951af0 --- /dev/null +++ b/test/codegen/unsafe.go @@ -0,0 +1,16 @@ +// asmcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +import "unsafe" + +func f(p unsafe.Pointer, x, y uintptr) int64 { + p = unsafe.Pointer(uintptr(p) + x + y) + // amd64:`MOVQ\s\(.*\)\(.*\*1\), ` + // arm64:`MOVD\s\(R[0-9]+\)\(R[0-9]+\), ` + return *(*int64)(p) +} diff --git a/test/codegen/writebarrier.go b/test/codegen/writebarrier.go index cfcfe15a403856..c3c39c58f7901c 100644 --- a/test/codegen/writebarrier.go +++ b/test/codegen/writebarrier.go @@ -53,3 +53,51 @@ func combine4slice(p *[4][]byte, a, b, c, d []byte) { // arm64:-`.*runtime[.]gcWriteBarrier` p[3] = d } + +func trickyWriteNil(p *int, q **int) { + if p == nil { + // We change "= p" to "= 0" in the prove pass, which + // means we have one less pointer that needs to go + // into the write barrier buffer. + // amd64:`.*runtime[.]gcWriteBarrier1` + *q = p + } +} + +type S struct { + a, b string + c *int +} + +var g1, g2 *int + +func issue71228(dst *S, ptr *int) { + // Make sure that the non-write-barrier write. + // "sp.c = ptr" happens before the large write + // barrier "*dst = *sp". We approximate testing + // that by ensuring that two global variable write + // barriers aren't combined. + _ = *dst + var s S + sp := &s + //amd64:`.*runtime[.]gcWriteBarrier1` + g1 = nil + sp.c = ptr // outside of any write barrier + //amd64:`.*runtime[.]gcWriteBarrier1` + g2 = nil + //amd64:`.*runtime[.]wbMove` + *dst = *sp +} + +func writeDouble(p *[2]*int, x, y *int) { + // arm64: `LDP\s`, `STP\s\(R[0-9]+, R[0-9]+\), \(`, + p[0] = x + // arm64: `STP\s\(R[0-9]+, R[0-9]+\), 16\(`, + p[1] = y +} + +func writeDoubleNil(p *[2]*int) { + // arm64: `LDP\s`, `STP\s\(R[0-9]+, R[0-9]+\),`, `STP\s\(ZR, ZR\),` + p[0] = nil + p[1] = nil +} diff --git a/test/codegen/zerosize.go b/test/codegen/zerosize.go index ecf33054617578..86c48194008cf2 100644 --- a/test/codegen/zerosize.go +++ b/test/codegen/zerosize.go @@ -18,8 +18,27 @@ func zeroSize() { g(&s, 1, 2, 3, 4, 5) // amd64:`LEAQ\tcommand-line-arguments\..*\+55\(SP\)` + c <- noliteral(struct{}{}) +} + +// Like zeroSize, but without hiding the zero-sized struct. +func zeroSize2() { + c := make(chan struct{}) + // amd64:`MOVQ\t\$0, command-line-arguments\.s\+48\(SP\)` + var s *int + // force s to be a stack object, also use some (fixed) stack space + g(&s, 1, 2, 3, 4, 5) + + // amd64:`LEAQ\tcommand-line-arguments\..*stmp_\d+\(SB\)` c <- struct{}{} } //go:noinline func g(**int, int, int, int, int, int) {} + +// noliteral prevents the compiler from recognizing a literal value. +// +//go:noinline +func noliteral[T any](t T) T { + return t +} diff --git a/test/complit1.go b/test/complit1.go index 8cbcd63ee0d176..9f33bc422ba503 100644 --- a/test/complit1.go +++ b/test/complit1.go @@ -18,9 +18,9 @@ func fp() *[3]int var mp map[int]*[3]int var ( - _ = [3]int{1, 2, 3}[:] // ERROR "slice of unaddressable value" - _ = m[0][:] // ERROR "slice of unaddressable value" - _ = f()[:] // ERROR "slice of unaddressable value" + _ = [3]int{1, 2, 3}[:] // ERROR "cannot slice unaddressable value" + _ = m[0][:] // ERROR "cannot slice unaddressable value" + _ = f()[:] // ERROR "cannot slice unaddressable value" _ = 301[:] // ERROR "cannot slice|attempt to slice object that is not" _ = 3.1[:] // ERROR "cannot slice|attempt to slice object that is not" diff --git a/test/convert5.go b/test/convert5.go new file mode 100644 index 00000000000000..df247ca0b9b238 --- /dev/null +++ b/test/convert5.go @@ -0,0 +1,268 @@ +// run -gcflags=-d=converthash=qy + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !wasm && !386 && !arm && !mips + +// TODO fix this to work for wasm and 32-bit architectures. +// Doing more than this, however, expands the change. + +package main + +import ( + "fmt" + "runtime" +) + +// This test checks that conversion from floats to (unsigned) 32 and 64-bit +// integers has the same sensible behavior for corner cases, and that the +// conversions to smaller integers agree. Because outliers are platform- +// independent, the "golden test" for smaller integers is more like of +// a "gold-ish test" and subject to change. + +//go:noinline +func id[T any](x T) T { + return x +} + +//go:noinline +func want[T comparable](name string, x, y T) { + if x != y { + _, _, line, _ := runtime.Caller(1) + fmt.Println("FAIL at line", line, "var =", name, "got =", x, "want =", y) + } +} + +//go:noinline +func log[T comparable](name string, x T) { + fmt.Println(name, x) +} + +const ( + // pX = max positive signed X bit + // nX = min negative signed X bit + // uX = max unsigned X bit + // tX = two to the X + p32 = 2147483647 + n32 = -2147483648 + u32 = 4294967295 + p64 = 9223372036854775807 + n64 = -9223372036854775808 + u64 = 18446744073709551615 + t44 = 1 << 44 +) + +func main() { + one := 1.0 + minus1_32 := id(float32(-1.0)) + minus1_64 := id(float64(-1.0)) + p32_plus4k_plus1 := id(float32(p32 + 4096 + 1)) // want this to be precise and fit in 24 bits mantissa + p64_plus4k_plus1 := id(float64(p64 + 4096 + 1)) // want this to be precise and fit in 53 bits mantissa + n32_minus4k := id(float32(n32 - 4096)) + n64_minus4k := id(float64(n64 - 4096)) + inf_32 := id(float32(one / 0)) + inf_64 := id(float64(one / 0)) + ninf_32 := id(float32(-one / 0)) + ninf_64 := id(float64(-one / 0)) + + // int32 conversions + int32Tests := []struct { + name string + input any // Use any to handle both float32 and float64 + expected int32 + }{ + {"minus1_32", minus1_32, -1}, + {"minus1_64", minus1_64, -1}, + {"p32_plus4k_plus1", p32_plus4k_plus1, p32}, + {"p64_plus4k_plus1", p64_plus4k_plus1, p32}, + {"n32_minus4k", n32_minus4k, n32}, + {"n64_minus4k", n64_minus4k, n32}, + {"inf_32", inf_32, p32}, + {"inf_64", inf_64, p32}, + {"ninf_32", ninf_32, n32}, + {"ninf_64", ninf_64, n32}, + } + + for _, test := range int32Tests { + var converted int32 + switch v := test.input.(type) { + case float32: + converted = int32(v) + case float64: + converted = int32(v) + } + want(test.name, converted, test.expected) + } + + // int64 conversions + int64Tests := []struct { + name string + input any + expected int64 + }{ + {"minus1_32", minus1_32, -1}, + {"minus1_64", minus1_64, -1}, + {"p32_plus4k_plus1", p32_plus4k_plus1, p32 + 4096 + 1}, + {"p64_plus4k_plus1", p64_plus4k_plus1, p64}, + {"n32_minus4k", n32_minus4k, n32 - 4096}, + {"n64_minus4k", n64_minus4k, n64}, + {"inf_32", inf_32, p64}, + {"inf_64", inf_64, p64}, + {"ninf_32", ninf_32, n64}, + {"ninf_64", ninf_64, n64}, + } + + for _, test := range int64Tests { + var converted int64 + switch v := test.input.(type) { + case float32: + converted = int64(v) + case float64: + converted = int64(v) + } + want(test.name, converted, test.expected) + } + + // uint32 conversions + uint32Tests := []struct { + name string + input any + expected uint32 + }{ + {"minus1_32", minus1_32, 0}, + {"minus1_64", minus1_64, 0}, + {"p32_plus4k_plus1", p32_plus4k_plus1, p32 + 4096 + 1}, + {"p64_plus4k_plus1", p64_plus4k_plus1, u32}, + {"n32_minus4k", n32_minus4k, 0}, + {"n64_minus4k", n64_minus4k, 0}, + {"inf_32", inf_32, u32}, + {"inf_64", inf_64, u32}, + {"ninf_32", ninf_32, 0}, + {"ninf_64", ninf_64, 0}, + } + + for _, test := range uint32Tests { + var converted uint32 + switch v := test.input.(type) { + case float32: + converted = uint32(v) + case float64: + converted = uint32(v) + } + want(test.name, converted, test.expected) + } + + u64_plus4k_plus1_64 := id(float64(u64 + 4096 + 1)) + u64_plust44_plus1_32 := id(float32(u64 + t44 + 1)) + + // uint64 conversions + uint64Tests := []struct { + name string + input any + expected uint64 + }{ + {"minus1_32", minus1_32, 0}, + {"minus1_64", minus1_64, 0}, + {"p32_plus4k_plus1", p32_plus4k_plus1, p32 + 4096 + 1}, + {"p64_plus4k_plus1", p64_plus4k_plus1, p64 + 4096 + 1}, + {"n32_minus4k", n32_minus4k, 0}, + {"n64_minus4k", n64_minus4k, 0}, + {"inf_32", inf_32, u64}, + {"inf_64", inf_64, u64}, + {"ninf_32", ninf_32, 0}, + {"ninf_64", ninf_64, 0}, + {"u64_plus4k_plus1_64", u64_plus4k_plus1_64, u64}, + {"u64_plust44_plus1_32", u64_plust44_plus1_32, u64}, + } + + for _, test := range uint64Tests { + var converted uint64 + switch v := test.input.(type) { + case float32: + converted = uint64(v) + case float64: + converted = uint64(v) + } + want(test.name, converted, test.expected) + } + + // for smaller integer types + // TODO the overflow behavior is dubious, maybe we should fix it to be more sensible, e.g. saturating. + fmt.Println("Below this are 'golden' results to check for consistency across platforms. Overflow behavior is not necessarily what we want") + + u8plus2 := id(float64(257)) + p8minus1 := id(float32(126)) + n8plus2 := id(float64(-126)) + n8minusone := id(float32(-129)) + + fmt.Println("\nuint8 conversions") + uint8Tests := []struct { + name string + input any + }{ + {"minus1_32", minus1_32}, + {"minus1_64", minus1_64}, + {"p32_plus4k_plus1", p32_plus4k_plus1}, + {"p64_plus4k_plus1", p64_plus4k_plus1}, + {"n32_minus4k", n32_minus4k}, + {"n64_minus4k", n64_minus4k}, + {"inf_32", inf_32}, + {"inf_64", inf_64}, + {"ninf_32", ninf_32}, + {"ninf_64", ninf_64}, + {"u64_plus4k_plus1_64", u64_plus4k_plus1_64}, + {"u64_plust44_plus1_32", u64_plust44_plus1_32}, + {"u8plus2", u8plus2}, + {"p8minus1", p8minus1}, + {"n8plus2", n8plus2}, + {"n8minusone", n8minusone}, + } + + for _, test := range uint8Tests { + var converted uint8 + switch v := test.input.(type) { + case float32: + converted = uint8(v) + case float64: + converted = uint8(v) + } + log(test.name, converted) + } + + fmt.Println("\nint8 conversions") + int8Tests := []struct { + name string + input any + }{ + {"minus1_32", minus1_32}, + {"minus1_64", minus1_64}, + {"p32_plus4k_plus1", p32_plus4k_plus1}, + {"p64_plus4k_plus1", p64_plus4k_plus1}, + {"n32_minus4k", n32_minus4k}, + {"n64_minus4k", n64_minus4k}, + {"inf_32", inf_32}, + {"inf_64", inf_64}, + {"ninf_32", ninf_32}, + {"ninf_64", ninf_64}, + {"u64_plus4k_plus1_64", u64_plus4k_plus1_64}, + {"u64_plust44_plus1_32", u64_plust44_plus1_32}, + {"u8plus2", u8plus2}, + {"p8minus1", p8minus1}, + {"n8plus2", n8plus2}, + {"n8minusone", n8minusone}, + } + + for _, test := range int8Tests { + var converted int8 + switch v := test.input.(type) { + case float32: + converted = int8(v) + case float64: + converted = int8(v) + } + log(test.name, converted) + } + +} diff --git a/test/convert5.out b/test/convert5.out new file mode 100644 index 00000000000000..47a8af67f961f2 --- /dev/null +++ b/test/convert5.out @@ -0,0 +1,37 @@ +Below this are 'golden' results to check for consistency across platforms. Overflow behavior is not necessarily what we want + +uint8 conversions +minus1_32 255 +minus1_64 255 +p32_plus4k_plus1 255 +p64_plus4k_plus1 255 +n32_minus4k 0 +n64_minus4k 0 +inf_32 255 +inf_64 255 +ninf_32 0 +ninf_64 0 +u64_plus4k_plus1_64 255 +u64_plust44_plus1_32 255 +u8plus2 1 +p8minus1 126 +n8plus2 130 +n8minusone 127 + +int8 conversions +minus1_32 -1 +minus1_64 -1 +p32_plus4k_plus1 -1 +p64_plus4k_plus1 -1 +n32_minus4k 0 +n64_minus4k 0 +inf_32 -1 +inf_64 -1 +ninf_32 0 +ninf_64 0 +u64_plus4k_plus1_64 -1 +u64_plust44_plus1_32 -1 +u8plus2 1 +p8minus1 126 +n8plus2 -126 +n8minusone 127 diff --git a/test/copy1.go b/test/copy1.go index c0760f71904879..0e929ac16c6fad 100644 --- a/test/copy1.go +++ b/test/copy1.go @@ -20,8 +20,8 @@ func main() { _ = copy(si, "hi") // ERROR "have different element types(.*int.*string| int and byte)" _ = copy(si, sf) // ERROR "have different element types.*int.*float64" - _ = copy(1, 2) // ERROR "must be slices; have int, int|expects slice arguments" - _ = copy(1, si) // ERROR "first argument to copy should be|expects slice arguments" - _ = copy(si, 2) // ERROR "second argument to copy should be|expects slice arguments" + _ = copy(1, 2) // ERROR "must be slices; have int, int|argument must be a slice; have 1" + _ = copy(1, si) // ERROR "first argument to copy should be|argument must be a slice; have 1" + _ = copy(si, 2) // ERROR "second argument to copy should be|argument must be a slice; have 2" } diff --git a/test/devirtualization.go b/test/devirtualization.go new file mode 100644 index 00000000000000..edabb94108d02d --- /dev/null +++ b/test/devirtualization.go @@ -0,0 +1,1283 @@ +// errorcheck -0 -m + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package escape + +type M interface{ M() } + +type A interface{ A() } + +type C interface{ C() } + +type Impl struct{} + +func (*Impl) M() {} // ERROR "can inline \(\*Impl\).M$" + +func (*Impl) A() {} // ERROR "can inline \(\*Impl\).A$" + +type Impl2 struct{} + +func (*Impl2) M() {} // ERROR "can inline \(\*Impl2\).M$" + +func (*Impl2) A() {} // ERROR "can inline \(\*Impl2\).A$" + +type CImpl struct{} + +func (CImpl) C() {} // ERROR "can inline CImpl.C$" + +func typeAsserts() { + var a M = &Impl{} // ERROR "&Impl{} does not escape$" + + a.(M).M() // ERROR "devirtualizing a.\(M\).M to \*Impl$" "inlining call to \(\*Impl\).M" + a.(A).A() // ERROR "devirtualizing a.\(A\).A to \*Impl$" "inlining call to \(\*Impl\).A" + a.(*Impl).M() // ERROR "inlining call to \(\*Impl\).M" + a.(*Impl).A() // ERROR "inlining call to \(\*Impl\).A" + + v := a.(M) + v.M() // ERROR "devirtualizing v.M to \*Impl$" "inlining call to \(\*Impl\).M" + v.(A).A() // ERROR "devirtualizing v.\(A\).A to \*Impl$" "inlining call to \(\*Impl\).A" + v.(*Impl).A() // ERROR "inlining call to \(\*Impl\).A" + v.(*Impl).M() // ERROR "inlining call to \(\*Impl\).M" + + v2 := a.(A) + v2.A() // ERROR "devirtualizing v2.A to \*Impl$" "inlining call to \(\*Impl\).A" + v2.(M).M() // ERROR "devirtualizing v2.\(M\).M to \*Impl$" "inlining call to \(\*Impl\).M" + v2.(*Impl).A() // ERROR "inlining call to \(\*Impl\).A" + v2.(*Impl).M() // ERROR "inlining call to \(\*Impl\).M" + + a.(M).(A).A() // ERROR "devirtualizing a.\(M\).\(A\).A to \*Impl$" "inlining call to \(\*Impl\).A" + a.(A).(M).M() // ERROR "devirtualizing a.\(A\).\(M\).M to \*Impl$" "inlining call to \(\*Impl\).M" + + a.(M).(A).(*Impl).A() // ERROR "inlining call to \(\*Impl\).A" + a.(A).(M).(*Impl).M() // ERROR "inlining call to \(\*Impl\).M" + + any(a).(M).M() // ERROR "devirtualizing any\(a\).\(M\).M to \*Impl$" "inlining call to \(\*Impl\).M" + any(a).(A).A() // ERROR "devirtualizing any\(a\).\(A\).A to \*Impl$" "inlining call to \(\*Impl\).A" + any(a).(M).(any).(A).A() // ERROR "devirtualizing any\(a\).\(M\).\(any\).\(A\).A to \*Impl$" "inlining call to \(\*Impl\).A" + + c := any(a) + c.(A).A() // ERROR "devirtualizing c.\(A\).A to \*Impl$" "inlining call to \(\*Impl\).A" + c.(M).M() // ERROR "devirtualizing c.\(M\).M to \*Impl$" "inlining call to \(\*Impl\).M" + + M(a).M() // ERROR "devirtualizing M\(a\).M to \*Impl$" "inlining call to \(\*Impl\).M" + M(M(a)).M() // ERROR "devirtualizing M\(M\(a\)\).M to \*Impl$" "inlining call to \(\*Impl\).M" + + a2 := a.(A) + A(a2).A() // ERROR "devirtualizing A\(a2\).A to \*Impl$" "inlining call to \(\*Impl\).A" + A(A(a2)).A() // ERROR "devirtualizing A\(A\(a2\)\).A to \*Impl$" "inlining call to \(\*Impl\).A" + + { + var a C = &CImpl{} // ERROR "&CImpl{} does not escape$" + a.(any).(C).C() // ERROR "devirtualizing a.\(any\).\(C\).C to \*CImpl$" "inlining call to CImpl.C" + a.(any).(*CImpl).C() // ERROR "inlining call to CImpl.C" + } +} + +func typeAssertsWithOkReturn() { + { + var a M = &Impl{} // ERROR "&Impl{} does not escape$" + if v, ok := a.(M); ok { + v.M() // ERROR "devirtualizing v.M to \*Impl$" "inlining call to \(\*Impl\).M" + } + } + { + var a M = &Impl{} // ERROR "&Impl{} does not escape$" + if v, ok := a.(A); ok { + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + } + { + var a M = &Impl{} // ERROR "&Impl{} does not escape$" + v, ok := a.(M) + if ok { + v.M() // ERROR "devirtualizing v.M to \*Impl$" "inlining call to \(\*Impl\).M" + } + } + { + var a M = &Impl{} // ERROR "&Impl{} does not escape$" + v, ok := a.(A) + if ok { + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + } + { + var a M = &Impl{} // ERROR "&Impl{} does not escape$" + v, ok := a.(*Impl) + if ok { + v.A() // ERROR "inlining call to \(\*Impl\).A" + v.M() // ERROR "inlining call to \(\*Impl\).M" + } + } + { + var a M = &Impl{} // ERROR "&Impl{} does not escape$" + v, _ := a.(M) + v.M() // ERROR "devirtualizing v.M to \*Impl$" "inlining call to \(\*Impl\).M" + } + { + var a M = &Impl{} // ERROR "&Impl{} does not escape$" + v, _ := a.(A) + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var a M = &Impl{} // ERROR "&Impl{} does not escape$" + v, _ := a.(*Impl) + v.A() // ERROR "inlining call to \(\*Impl\).A" + v.M() // ERROR "inlining call to \(\*Impl\).M" + } + { + a := newM() // ERROR "&Impl{} does not escape$" "inlining call to newM" + callA(a) // ERROR "devirtualizing m.\(A\).A to \*Impl$" "inlining call to \(\*Impl\).A" "inlining call to callA" + callIfA(a) // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" "inlining call to callIfA" + } + { + _, a := newM2ret() // ERROR "&Impl{} does not escape$" "inlining call to newM2ret" + callA(a) // ERROR "devirtualizing m.\(A\).A to \*Impl$" "inlining call to \(\*Impl\).A" "inlining call to callA" + callIfA(a) // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" "inlining call to callIfA" + } + { + var a M = &Impl{} // ERROR "&Impl{} does not escape$" + // Note the !ok condition, devirtualizing here is fine. + if v, ok := a.(M); !ok { + v.M() // ERROR "devirtualizing v.M to \*Impl$" "inlining call to \(\*Impl\).M" + } + } + { + var a A = newImplNoInline() + if v, ok := a.(M); ok { + v.M() // ERROR "devirtualizing v.M to \*Impl$" "inlining call to \(\*Impl\).M" + } + } + { + var impl2InA A = &Impl2{} // ERROR "&Impl2{} does not escape$" + var a A + a, _ = impl2InA.(*Impl) + // a now contains the zero value of *Impl + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + a := newANoInline() + a.A() + } + { + _, a := newANoInlineRet2() + a.A() + } +} + +func newM() M { // ERROR "can inline newM$" + return &Impl{} // ERROR "&Impl{} escapes to heap$" +} + +func newM2ret() (int, M) { // ERROR "can inline newM2ret$" + return -1, &Impl{} // ERROR "&Impl{} escapes to heap$" +} + +func callA(m M) { // ERROR "can inline callA$" "leaking param: m$" + m.(A).A() +} + +func callIfA(m M) { // ERROR "can inline callIfA$" "leaking param: m$" + if v, ok := m.(A); ok { + v.A() + } +} + +//go:noinline +func newImplNoInline() *Impl { + return &Impl{} // ERROR "&Impl{} escapes to heap$" +} + +//go:noinline +func newImpl2ret2() (string, *Impl2) { + return "str", &Impl2{} // ERROR "&Impl2{} escapes to heap$" +} + +//go:noinline +func newImpl2() *Impl2 { + return &Impl2{} // ERROR "&Impl2{} escapes to heap$" +} + +//go:noinline +func newANoInline() A { + return &Impl{} // ERROR "&Impl{} escapes to heap$" +} + +//go:noinline +func newANoInlineRet2() (string, A) { + return "", &Impl{} // ERROR "&Impl{} escapes to heap$" +} + +func testTypeSwitch() { + { + var v A = &Impl{} // ERROR "&Impl{} does not escape$" + switch v := v.(type) { + case A: + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + case M: + v.M() // ERROR "devirtualizing v.M to \*Impl$" "inlining call to \(\*Impl\).M" + } + } + { + var v A = &Impl{} // ERROR "&Impl{} does not escape$" + switch v := v.(type) { + case A: + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + case M: + v.M() // ERROR "devirtualizing v.M to \*Impl$" "inlining call to \(\*Impl\).M" + v = &Impl{} // ERROR "&Impl{} does not escape$" + v.M() // ERROR "devirtualizing v.M to \*Impl$" "inlining call to \(\*Impl\).M" + } + v.(M).M() // ERROR "devirtualizing v.\(M\).M to \*Impl$" "inlining call to \(\*Impl\).M" + } + { + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + switch v1 := v.(type) { + case A: + v1.A() + case M: + v1.M() + v = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + } + } + { + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + switch v := v.(type) { + case A: + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + case M: + v.M() // ERROR "devirtualizing v.M to \*Impl$" "inlining call to \(\*Impl\).M" + case C: + v.C() + } + } + { + var v A = &Impl{} // ERROR "&Impl{} does not escape$" + switch v := v.(type) { + case M: + v.M() // ERROR "devirtualizing v.M to \*Impl$" "inlining call to \(\*Impl\).M" + default: + panic("does not implement M") // ERROR ".does not implement M. escapes to heap$" + } + } +} + +func differentTypeAssign() { + { + var a A + a = &Impl{} // ERROR "&Impl{} escapes to heap$" + a = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + a.A() + } + { + a := A(&Impl{}) // ERROR "&Impl{} escapes to heap$" + a = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + a.A() + } + { + a := A(&Impl{}) // ERROR "&Impl{} escapes to heap$" + a.A() + a = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + } + { + a := A(&Impl{}) // ERROR "&Impl{} escapes to heap$" + a = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + var asAny any = a + asAny.(A).A() + } + { + a := A(&Impl{}) // ERROR "&Impl{} escapes to heap$" + var asAny any = a + asAny = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + asAny.(A).A() + } + { + a := A(&Impl{}) // ERROR "&Impl{} escapes to heap$" + var asAny any = a + asAny.(A).A() + asAny = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var a A + a = &Impl{} // ERROR "&Impl{} escapes to heap$" + a = newImpl2() + a.A() + } + { + var a A + a = &Impl{} // ERROR "&Impl{} escapes to heap$" + _, a = newImpl2ret2() + a.A() + } +} + +func assignWithTypeAssert() { + { + var i1 A = &Impl{} // ERROR "&Impl{} does not escape$" + var i2 A = &Impl2{} // ERROR "&Impl2{} does not escape$" + i1 = i2.(*Impl) // this will panic + i1.A() // ERROR "devirtualizing i1.A to \*Impl$" "inlining call to \(\*Impl\).A" + i2.A() // ERROR "devirtualizing i2.A to \*Impl2$" "inlining call to \(\*Impl2\).A" + } + { + var i1 A = &Impl{} // ERROR "&Impl{} does not escape$" + var i2 A = &Impl2{} // ERROR "&Impl2{} does not escape$" + i1, _ = i2.(*Impl) // i1 is going to be nil + i1.A() // ERROR "devirtualizing i1.A to \*Impl$" "inlining call to \(\*Impl\).A" + i2.A() // ERROR "devirtualizing i2.A to \*Impl2$" "inlining call to \(\*Impl2\).A" + } +} + +func nilIface() { + { + var v A = &Impl{} // ERROR "&Impl{} does not escape$" + v = nil + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var v A = &Impl{} // ERROR "&Impl{} does not escape$" + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + v = nil + } + { + var nilIface A + var v A = &Impl{} // ERROR "&Impl{} does not escape$" + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + v = nilIface + } + { + var nilIface A + var v A = &Impl{} // ERROR "&Impl{} does not escape$" + v = nilIface + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var v A + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + v = &Impl{} // ERROR "&Impl{} does not escape$" + } + { + var v A + var v2 A = v + v2.A() // ERROR "devirtualizing v2.A to \*Impl$" "inlining call to \(\*Impl\).A" + v2 = &Impl{} // ERROR "&Impl{} does not escape$" + } + { + var v A + v.A() + } + { + var v A + var v2 A = v + v2.A() + } + { + var v A + var v2 A + v2 = v + v2.A() + } +} + +func longDevirtTest() { + var a interface { + M + A + } = &Impl{} // ERROR "&Impl{} does not escape$" + + { + var b A = a + b.A() // ERROR "devirtualizing b.A to \*Impl$" "inlining call to \(\*Impl\).A" + b.(M).M() // ERROR "devirtualizing b.\(M\).M to \*Impl$" "inlining call to \(\*Impl\).M" + } + { + var b M = a + b.M() // ERROR "devirtualizing b.M to \*Impl$" "inlining call to \(\*Impl\).M" + b.(A).A() // ERROR "devirtualizing b.\(A\).A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var b A = a.(M).(A) + b.A() // ERROR "devirtualizing b.A to \*Impl$" "inlining call to \(\*Impl\).A" + b.(M).M() // ERROR "devirtualizing b.\(M\).M to \*Impl$" "inlining call to \(\*Impl\).M" + } + { + var b M = a.(A).(M) + b.M() // ERROR "devirtualizing b.M to \*Impl$" "inlining call to \(\*Impl\).M" + b.(A).A() // ERROR "devirtualizing b.\(A\).A to \*Impl$" "inlining call to \(\*Impl\).A" + } + + if v, ok := a.(A); ok { + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + + if v, ok := a.(M); ok { + v.M() // ERROR "devirtualizing v.M to \*Impl$" "inlining call to \(\*Impl\).M" + } + + { + var c A = a + + if v, ok := c.(A); ok { + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + + c = &Impl{} // ERROR "&Impl{} does not escape$" + + if v, ok := c.(M); ok { + v.M() // ERROR "devirtualizing v.M to \*Impl$" "inlining call to \(\*Impl\).M" + } + + if v, ok := c.(interface { + A + M + }); ok { + v.M() // ERROR "devirtualizing v.M to \*Impl$" "inlining call to \(\*Impl\).M" + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + } +} + +func deferDevirt() { + var a A + defer func() { // ERROR "can inline deferDevirt.func1$" "func literal does not escape$" + a = &Impl{} // ERROR "&Impl{} escapes to heap$" + }() + a = &Impl{} // ERROR "&Impl{} does not escape$" + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" +} + +func deferNoDevirt() { + var a A + defer func() { // ERROR "can inline deferNoDevirt.func1$" "func literal does not escape$" + a = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + }() + a = &Impl{} // ERROR "&Impl{} escapes to heap$" + a.A() +} + +//go:noinline +func closureDevirt() { + var a A + func() { // ERROR "func literal does not escape$" + // defer so that it does not lnline. + defer func() {}() // ERROR "can inline closureDevirt.func1.1$" "func literal does not escape$" + a = &Impl{} // ERROR "&Impl{} escapes to heap$" + }() + a = &Impl{} // ERROR "&Impl{} does not escape$" + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" +} + +//go:noinline +func closureNoDevirt() { + var a A + func() { // ERROR "func literal does not escape$" + // defer so that it does not lnline. + defer func() {}() // ERROR "can inline closureNoDevirt.func1.1$" "func literal does not escape$" + a = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + }() + a = &Impl{} // ERROR "&Impl{} escapes to heap$" + a.A() +} + +var global = "1" + +func closureDevirt2() { + var a A + a = &Impl{} // ERROR "&Impl{} does not escape$" + c := func() { // ERROR "can inline closureDevirt2.func1$" "func literal does not escape$" + a = &Impl{} // ERROR "&Impl{} escapes to heap$" + } + if global == "1" { + c = func() { // ERROR "can inline closureDevirt2.func2$" "func literal does not escape$" + a = &Impl{} // ERROR "&Impl{} escapes to heap$" + } + } + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + c() +} + +func closureNoDevirt2() { + var a A + a = &Impl{} // ERROR "&Impl{} escapes to heap$" + c := func() { // ERROR "can inline closureNoDevirt2.func1$" "func literal does not escape$" + a = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + } + if global == "1" { + c = func() { // ERROR "can inline closureNoDevirt2.func2$" "func literal does not escape$" + a = &Impl{} // ERROR "&Impl{} escapes to heap$" + } + } + a.A() + c() +} + +//go:noinline +func closureDevirt3() { + var a A = &Impl{} // ERROR "&Impl{} does not escape$" + func() { // ERROR "func literal does not escape$" + // defer so that it does not lnline. + defer func() {}() // ERROR "can inline closureDevirt3.func1.1$" "func literal does not escape$" + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + }() + func() { // ERROR "can inline closureDevirt3.func2$" + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + }() // ERROR "inlining call to closureDevirt3.func2" "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" +} + +//go:noinline +func closureNoDevirt3() { + var a A = &Impl{} // ERROR "&Impl{} escapes to heap$" + func() { // ERROR "func literal does not escape$" + // defer so that it does not lnline. + defer func() {}() // ERROR "can inline closureNoDevirt3.func1.1$" "func literal does not escape$" + a.A() + }() + func() { // ERROR "can inline closureNoDevirt3.func2$" + a.A() + }() // ERROR "inlining call to closureNoDevirt3.func2" + a = &Impl2{} // ERROR "&Impl2{} escapes to heap$" +} + +//go:noinline +func varDeclaredInClosureReferencesOuter() { + var a A = &Impl{} // ERROR "&Impl{} does not escape$" + func() { // ERROR "func literal does not escape$" + // defer for noinline + defer func() {}() // ERROR "can inline varDeclaredInClosureReferencesOuter.func1.1$" "func literal does not escape$" + var v A = a + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + }() + func() { // ERROR "func literal does not escape$" + // defer for noinline + defer func() {}() // ERROR "can inline varDeclaredInClosureReferencesOuter.func2.1$" "func literal does not escape$" + var v A = a + v = &Impl{} // ERROR "&Impl{} does not escape$" + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + }() + + var b A = &Impl{} // ERROR "&Impl{} escapes to heap$" + func() { // ERROR "func literal does not escape$" + // defer for noinline + defer func() {}() // ERROR "can inline varDeclaredInClosureReferencesOuter.func3.1$" "func literal does not escape$" + var v A = b + v = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + v.A() + }() + func() { // ERROR "func literal does not escape$" + // defer for noinline + defer func() {}() // ERROR "can inline varDeclaredInClosureReferencesOuter.func4.1$" "func literal does not escape$" + var v A = b + v.A() + v = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + }() +} + +//go:noinline +func testNamedReturn0() (v A) { + v = &Impl{} // ERROR "&Impl{} escapes to heap$" + v.A() + return +} + +//go:noinline +func testNamedReturn1() (v A) { + v = &Impl{} // ERROR "&Impl{} escapes to heap$" + v.A() + return &Impl{} // ERROR "&Impl{} escapes to heap$" +} + +func testNamedReturns3() (v A) { + v = &Impl{} // ERROR "&Impl{} escapes to heap$" + defer func() { // ERROR "can inline testNamedReturns3.func1$" "func literal does not escape$" + v.A() + }() + v.A() + return &Impl2{} // ERROR "&Impl2{} escapes to heap$" +} + +var ( + globalImpl = &Impl{} + globalImpl2 = &Impl2{} + globalA A = &Impl{} + globalM M = &Impl{} +) + +func globals() { + { + globalA.A() + globalA.(M).M() + globalM.M() + globalM.(A).A() + + a := globalA + a.A() + a.(M).M() + + m := globalM + m.M() + m.(A).A() + } + { + var a A = &Impl{} // ERROR "&Impl{} does not escape$" + a = globalImpl + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var a A = &Impl{} // ERROR "&Impl{} does not escape$" + a = A(globalImpl) + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var a A = &Impl{} // ERROR "&Impl{} does not escape$" + a = M(globalImpl).(A) + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var a A = &Impl{} // ERROR "&Impl{} does not escape$" + a = globalA.(*Impl) + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + a = globalM.(*Impl) + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var a A = &Impl{} // ERROR "&Impl{} escapes to heap$" + a = globalImpl2 + a.A() + } + { + var a A = &Impl{} // ERROR "&Impl{} escapes to heap$" + a = globalA + a.A() + } + { + var a A = &Impl{} // ERROR "&Impl{} escapes to heap$" + a = globalM.(A) + a.A() + } +} + +func mapsDevirt() { + { + m := make(map[int]*Impl) // ERROR "make\(map\[int\]\*Impl\) does not escape$" + var v A = m[0] + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + v.(M).M() // ERROR "devirtualizing v.\(M\).M to \*Impl$" "inlining call to \(\*Impl\).M" + } + { + m := make(map[int]*Impl) // ERROR "make\(map\[int\]\*Impl\) does not escape$" + var v A + var ok bool + if v, ok = m[0]; ok { + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + m := make(map[int]*Impl) // ERROR "make\(map\[int\]\*Impl\) does not escape$" + var v A + v, _ = m[0] + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } +} + +func mapsNoDevirt() { + { + m := make(map[int]*Impl) // ERROR "make\(map\[int\]\*Impl\) does not escape$" + var v A = m[0] + v.A() + v = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + v.(M).M() + } + { + m := make(map[int]*Impl) // ERROR "make\(map\[int\]\*Impl\) does not escape$" + var v A + var ok bool + if v, ok = m[0]; ok { + v.A() + } + v = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + v.A() + } + { + m := make(map[int]*Impl) // ERROR "make\(map\[int\]\*Impl\) does not escape$" + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + v, _ = m[0] + v = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + v.A() + } + + { + m := make(map[int]A) // ERROR "make\(map\[int\]A\) does not escape$" + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + v = m[0] + v.A() + } + { + m := make(map[int]A) // ERROR "make\(map\[int\]A\) does not escape$" + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + var ok bool + if v, ok = m[0]; ok { + v.A() + } + v.A() + } + { + m := make(map[int]A) // ERROR "make\(map\[int\]A\) does not escape$" + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + v, _ = m[0] + v.A() + } +} + +func chanDevirt() { + { + m := make(chan *Impl) + var v A = <-m + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + m := make(chan *Impl) + var v A + v = <-m + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + m := make(chan *Impl) + var v A + v, _ = <-m + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + m := make(chan *Impl) + var v A + var ok bool + if v, ok = <-m; ok { + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + m := make(chan *Impl) + var v A + var ok bool + if v, ok = <-m; ok { + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + select { + case <-m: + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + case v = <-m: + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + case v, ok = <-m: + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + } +} + +func chanNoDevirt() { + { + m := make(chan *Impl) + var v A = <-m + v = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + v.A() + } + { + m := make(chan *Impl) + var v A + v = <-m + v = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + v.A() + } + { + m := make(chan *Impl) + var v A + v, _ = <-m + v = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + v.A() + } + { + m := make(chan *Impl) + var v A + var ok bool + if v, ok = <-m; ok { + v.A() + } + v = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + v.A() + } + { + m := make(chan *Impl) + var v A = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + var ok bool + if v, ok = <-m; ok { + v.A() + } + } + { + m := make(chan *Impl) + var v A = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + select { + case v = <-m: + v.A() + } + v.A() + } + { + m := make(chan *Impl) + var v A = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + select { + case v, _ = <-m: + v.A() + } + v.A() + } + + { + m := make(chan A) + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + v = <-m + v.A() + } + { + m := make(chan A) + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + v, _ = <-m + v.A() + } + { + m := make(chan A) + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + var ok bool + if v, ok = <-m; ok { + v.A() + } + } + { + m := make(chan A) + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + select { + case v = <-m: + v.A() + } + v.A() + } + { + m := make(chan A) + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + select { + case v, _ = <-m: + v.A() + } + v.A() + } +} + +func rangeDevirt() { + { + var v A + m := make(map[*Impl]struct{}) // ERROR "make\(map\[\*Impl\]struct {}\) does not escape$" + v = &Impl{} // ERROR "&Impl{} does not escape$" + for v = range m { + } + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var v A + m := make(map[*Impl]*Impl) // ERROR "make\(map\[\*Impl\]\*Impl\) does not escape$" + v = &Impl{} // ERROR "&Impl{} does not escape$" + for v = range m { + } + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var v A + m := make(map[*Impl]*Impl) // ERROR "make\(map\[\*Impl\]\*Impl\) does not escape$" + v = &Impl{} // ERROR "&Impl{} does not escape$" + for _, v = range m { + } + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var v A + m := make(chan *Impl) + v = &Impl{} // ERROR "&Impl{} does not escape$" + for v = range m { + } + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var v A + m := []*Impl{} // ERROR "\[\]\*Impl{} does not escape$" + v = &Impl{} // ERROR "&Impl{} does not escape$" + for _, v = range m { + } + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var v A + v = &Impl{} // ERROR "&Impl{} does not escape$" + impl := &Impl{} // ERROR "&Impl{} does not escape$" + i := 0 + for v = impl; i < 10; i++ { + } + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var v A + v = &Impl{} // ERROR "&Impl{} does not escape$" + impl := &Impl{} // ERROR "&Impl{} does not escape$" + i := 0 + for v = impl; i < 10; i++ { + } + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var v A + m := [1]*Impl{&Impl{}} // ERROR "&Impl{} does not escape$" + v = &Impl{} // ERROR "&Impl{} does not escape$" + for _, v = range m { + } + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var v A + m := [1]*Impl{&Impl{}} // ERROR "&Impl{} does not escape$" + v = &Impl{} // ERROR "&Impl{} does not escape$" + for _, v = range &m { + } + v.A() // ERROR "devirtualizing v.A to \*Impl$" "inlining call to \(\*Impl\).A" + } +} + +func rangeNoDevirt() { + { + var v A = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + m := make(map[*Impl]struct{}) // ERROR "make\(map\[\*Impl\]struct {}\) does not escape$" + for v = range m { + } + v.A() + } + { + var v A = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + m := make(map[*Impl]*Impl) // ERROR "make\(map\[\*Impl\]\*Impl\) does not escape$" + for v = range m { + } + v.A() + } + { + var v A = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + m := make(map[*Impl]*Impl) // ERROR "make\(map\[\*Impl\]\*Impl\) does not escape$" + for _, v = range m { + } + v.A() + } + { + var v A = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + m := make(chan *Impl) + for v = range m { + } + v.A() + } + { + var v A = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + m := []*Impl{} // ERROR "\[\]\*Impl{} does not escape$" + for _, v = range m { + } + v.A() + } + { + var v A + v = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + impl := &Impl{} // ERROR "&Impl{} escapes to heap$" + i := 0 + for v = impl; i < 10; i++ { + } + v.A() + } + { + var v A + v = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + impl := &Impl{} // ERROR "&Impl{} escapes to heap$" + i := 0 + for v = impl; i < 10; i++ { + } + v.A() + } + { + var v A + m := [1]*Impl{&Impl{}} // ERROR "&Impl{} escapes to heap$" + v = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + for _, v = range m { + } + v.A() + } + { + var v A + m := [1]*Impl{&Impl{}} // ERROR "&Impl{} escapes to heap$" + v = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + for _, v = range &m { + } + v.A() + } + + { + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + m := make(map[A]struct{}) // ERROR "make\(map\[A\]struct {}\) does not escape$" + for v = range m { + } + v.A() + } + { + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + m := make(map[A]A) // ERROR "make\(map\[A\]A\) does not escape$" + for v = range m { + } + v.A() + } + { + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + m := make(map[A]A) // ERROR "make\(map\[A\]A\) does not escape$" + for _, v = range m { + } + v.A() + } + { + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + m := make(chan A) + for v = range m { + } + v.A() + } + { + var v A = &Impl{} // ERROR "&Impl{} escapes to heap$" + m := []A{} // ERROR "\[\]A{} does not escape$" + for _, v = range m { + } + v.A() + } + + { + var v A + m := [1]A{&Impl{}} // ERROR "&Impl{} escapes to heap$" + v = &Impl{} // ERROR "&Impl{} escapes to heap$" + for _, v = range m { + } + v.A() + } + { + var v A + m := [1]A{&Impl{}} // ERROR "&Impl{} escapes to heap$" + v = &Impl{} // ERROR "&Impl{} escapes to heap$" + for _, v = range &m { + } + v.A() + } +} + +var globalInt = 1 + +func testIfInit() { + { + var a A = &Impl{} // ERROR "&Impl{} does not escape$" + var i = &Impl{} // ERROR "&Impl{} does not escape$" + if a = i; globalInt == 1 { + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + a.(M).M() // ERROR "devirtualizing a.\(M\).M to \*Impl$" "inlining call to \(\*Impl\).M" + } + { + var a A = &Impl{} // ERROR "&Impl{} escapes to heap$" + var i2 = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + if a = i2; globalInt == 1 { + a.A() + } + a.A() + } +} + +func testSwitchInit() { + { + var a A = &Impl{} // ERROR "&Impl{} does not escape$" + var i = &Impl{} // ERROR "&Impl{} does not escape$" + switch a = i; globalInt { + case 12: + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + a.(M).M() // ERROR "devirtualizing a.\(M\).M to \*Impl$" "inlining call to \(\*Impl\).M" + } + { + var a A = &Impl{} // ERROR "&Impl{} escapes to heap$" + var i2 = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + switch a = i2; globalInt { + case 12: + a.A() + } + a.A() + } +} + +type implWrapper Impl + +func (implWrapper) A() {} // ERROR "can inline implWrapper.A$" + +//go:noinline +func devirtWrapperType() { + { + i := &Impl{} // ERROR "&Impl{} does not escape$" + // This is an OCONVNOP, so we have to be careful, not to devirtualize it to Impl.A. + var a A = (*implWrapper)(i) + a.A() // ERROR "devirtualizing a.A to \*implWrapper$" "inlining call to implWrapper.A" + } + { + i := Impl{} + // This is an OCONVNOP, so we have to be careful, not to devirtualize it to Impl.A. + var a A = (implWrapper)(i) // ERROR "implWrapper\(i\) does not escape$" + a.A() // ERROR "devirtualizing a.A to implWrapper$" "inlining call to implWrapper.A" + } + { + type anyWrapper any + var foo any = &Impl{} // ERROR "&Impl\{\} does not escape" + var bar anyWrapper = foo + bar.(M).M() // ERROR "devirtualizing bar\.\(M\).M to \*Impl" "inlining call to \(\*Impl\)\.M" + } +} + +func selfAssigns() { + { + var a A = &Impl{} // ERROR "&Impl{} does not escape$" + a = a + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var a A = &Impl{} // ERROR "&Impl{} does not escape" + var asAny any = a + asAny = asAny + asAny.(A).A() // ERROR "devirtualizing asAny.\(A\).A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var a A = &Impl{} // ERROR "&Impl{} does not escape" + var asAny any = a + a = asAny.(A) + asAny.(A).A() // ERROR "devirtualizing asAny.\(A\).A to \*Impl$" "inlining call to \(\*Impl\).A" + a.(A).A() // ERROR "devirtualizing a.\(A\).A to \*Impl$" "inlining call to \(\*Impl\).A" + b := a + b.(A).A() // ERROR "devirtualizing b.\(A\).A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var a A = &Impl{} // ERROR "&Impl{} does not escape" + var asAny any = a + asAny = asAny + a = asAny.(A) + asAny = a + asAny.(A).A() // ERROR "devirtualizing asAny.\(A\).A to \*Impl$" "inlining call to \(\*Impl\).A" + asAny.(M).M() // ERROR "devirtualizing asAny.\(M\).M to \*Impl$" "inlining call to \(\*Impl\).M" + } + { + var a A = &Impl{} // ERROR "&Impl{} does not escape" + var asAny A = a + a = asAny.(A) + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + } + { + var a, b, c A + c = &Impl{} // ERROR "&Impl{} does not escape$" + a = c + c = b + b = c + a = b + b = a + c = a + a.A() // ERROR "devirtualizing a.A to \*Impl$" "inlining call to \(\*Impl\).A" + } +} + +func boolNoDevirt() { + { + m := make(map[int]*Impl) // ERROR "make\(map\[int\]\*Impl\) does not escape$" + var v any = &Impl{} // ERROR "&Impl{} escapes to heap$" + _, v = m[0] // ERROR ".autotmp_[0-9]+ escapes to heap$" + v.(A).A() + } + { + m := make(chan *Impl) + var v any = &Impl{} // ERROR "&Impl{} escapes to heap$" + select { + case _, v = <-m: // ERROR ".autotmp_[0-9]+ escapes to heap$" + } + v.(A).A() + } + { + m := make(chan *Impl) + var v any = &Impl{} // ERROR "&Impl{} escapes to heap$" + _, v = <-m // ERROR ".autotmp_[0-9]+ escapes to heap$" + v.(A).A() + } + { + var a any = 4 // ERROR "4 does not escape$" + var v any = &Impl{} // ERROR "&Impl{} escapes to heap$" + _, v = a.(int) // ERROR ".autotmp_[0-9]+ escapes to heap$" + v.(A).A() + } +} + +func addrTaken() { + { + var a A = &Impl{} // ERROR "&Impl{} escapes to heap$" + var ptrA = &a + a.A() + _ = ptrA + } + { + var a A = &Impl{} // ERROR "&Impl{} escapes to heap$" + var ptrA = &a + *ptrA = &Impl{} // ERROR "&Impl{} escapes to heap$" + a.A() + } + { + var a A = &Impl{} // ERROR "&Impl{} escapes to heap$" + var ptrA = &a + *ptrA = &Impl2{} // ERROR "&Impl2{} escapes to heap$" + a.A() + } +} + +func testInvalidAsserts() { + any(0).(interface{ A() }).A() // ERROR "any\(0\) escapes to heap$" + { + var a M = &Impl{} // ERROR "&Impl{} escapes to heap$" + a.(C).C() // this will panic + a.(any).(C).C() // this will panic + } + { + var a C = &CImpl{} // ERROR "&CImpl{} escapes to heap$" + a.(M).M() // this will panic + a.(any).(M).M() // this will panic + } + { + var a C = &CImpl{} // ERROR "&CImpl{} does not escape$" + + // this will panic + a.(M).(*Impl).M() // ERROR "inlining call to \(\*Impl\).M" + + // this will panic + a.(any).(M).(*Impl).M() // ERROR "inlining call to \(\*Impl\).M" + } +} + +type namedBool bool + +func (namedBool) M() {} // ERROR "can inline namedBool.M$" + +//go:noinline +func namedBoolTest() { + m := map[int]int{} // ERROR "map\[int\]int{} does not escape" + var ok namedBool + _, ok = m[5] + var i M = ok // ERROR "ok does not escape" + i.M() // ERROR "devirtualizing i.M to namedBool$" "inlining call to namedBool.M" +} diff --git a/test/devirtualization_nil_panics.go b/test/devirtualization_nil_panics.go new file mode 100644 index 00000000000000..59da454be7f910 --- /dev/null +++ b/test/devirtualization_nil_panics.go @@ -0,0 +1,100 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "runtime" + "strings" +) + +type A interface{ A() } + +type Impl struct{} + +func (*Impl) A() {} + +type Impl2 struct{} + +func (*Impl2) A() {} + +func main() { + shouldNilPanic(28, func() { + var v A + v.A() + v = &Impl{} + }) + shouldNilPanic(36, func() { + var v A + defer func() { + v = &Impl{} + }() + v.A() + }) + shouldNilPanic(43, func() { + var v A + f := func() { + v = &Impl{} + } + v.A() + f() + }) + + // Make sure that both devirtualized and non devirtualized + // variants have the panic at the same line. + shouldNilPanic(55, func() { + var v A + defer func() { + v = &Impl{} + }() + v. // A() is on a sepearate line + A() + }) + shouldNilPanic(64, func() { + var v A + defer func() { + v = &Impl{} + v = &Impl2{} // assign different type, such that the call below does not get devirtualized + }() + v. // A() is on a sepearate line + A() + }) +} + +var cnt = 0 + +func shouldNilPanic(wantLine int, f func()) { + cnt++ + defer func() { + p := recover() + if p == nil { + panic("no nil deref panic") + } + if strings.Contains(fmt.Sprintf("%s", p), "invalid memory address or nil pointer dereference") { + callers := make([]uintptr, 128) + n := runtime.Callers(0, callers) + callers = callers[:n] + + frames := runtime.CallersFrames(callers) + line := -1 + for f, next := frames.Next(); next; f, next = frames.Next() { + if f.Func.Name() == fmt.Sprintf("main.main.func%v", cnt) { + line = f.Line + break + } + } + + if line != wantLine { + panic(fmt.Sprintf("invalid line number in panic = %v; want = %v", line, wantLine)) + } + + return + } + panic(p) + }() + f() +} diff --git a/test/devirtualization_with_type_assertions_interleaved.go b/test/devirtualization_with_type_assertions_interleaved.go new file mode 100644 index 00000000000000..6bad3beb9aa0d2 --- /dev/null +++ b/test/devirtualization_with_type_assertions_interleaved.go @@ -0,0 +1,139 @@ +// errorcheck -0 -m + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package escape + +type hashIface interface { + Sum() []byte +} + +type cloneableHashIface interface { + hashIface + Clone() hashIface +} + +type hash struct{ state [32]byte } + +func (h *hash) Sum() []byte { // ERROR "can inline \(\*hash\).Sum$" "h does not escape$" + return make([]byte, 32) // ERROR "make\(\[\]byte, 32\) escapes to heap$" +} + +func (h *hash) Clone() hashIface { // ERROR "can inline \(\*hash\).Clone$" "h does not escape$" + c := *h // ERROR "moved to heap: c$" + return &c +} + +type hash2 struct{ state [32]byte } + +func (h *hash2) Sum() []byte { // ERROR "can inline \(\*hash2\).Sum$" "h does not escape$" + return make([]byte, 32) // ERROR "make\(\[\]byte, 32\) escapes to heap$" +} + +func (h *hash2) Clone() hashIface { // ERROR "can inline \(\*hash2\).Clone$" "h does not escape$" + c := *h // ERROR "moved to heap: c$" + return &c +} + +func newHash() hashIface { // ERROR "can inline newHash$" + return &hash{} // ERROR "&hash{} escapes to heap$" +} + +func cloneHash1(h hashIface) hashIface { // ERROR "can inline cloneHash1$" "leaking param: h$" + if h, ok := h.(cloneableHashIface); ok { + return h.Clone() + } + return &hash{} // ERROR "&hash{} escapes to heap$" +} + +func cloneHash2(h hashIface) hashIface { // ERROR "can inline cloneHash2$" "leaking param: h$" + if h, ok := h.(cloneableHashIface); ok { + return h.Clone() + } + return nil +} + +func cloneHash3(h hashIface) hashIface { // ERROR "can inline cloneHash3$" "leaking param: h$" + if h, ok := h.(cloneableHashIface); ok { + return h.Clone() + } + return &hash2{} // ERROR "&hash2{} escapes to heap$" +} + +func cloneHashWithBool1(h hashIface) (hashIface, bool) { // ERROR "can inline cloneHashWithBool1$" "leaking param: h$" + if h, ok := h.(cloneableHashIface); ok { + return h.Clone(), true + } + return &hash{}, false // ERROR "&hash{} escapes to heap$" +} + +func cloneHashWithBool2(h hashIface) (hashIface, bool) { // ERROR "can inline cloneHashWithBool2$" "leaking param: h$" + if h, ok := h.(cloneableHashIface); ok { + return h.Clone(), true + } + return nil, false +} + +func cloneHashWithBool3(h hashIface) (hashIface, bool) { // ERROR "can inline cloneHashWithBool3$" "leaking param: h$" + if h, ok := h.(cloneableHashIface); ok { + return h.Clone(), true + } + return &hash2{}, false // ERROR "&hash2{} escapes to heap$" +} + +func interleavedWithTypeAssertions() { + h1 := newHash() // ERROR "&hash{} does not escape$" "inlining call to newHash" + _ = h1.Sum() // ERROR "devirtualizing h1.Sum to \*hash$" "inlining call to \(\*hash\).Sum" "make\(\[\]byte, 32\) does not escape$" + + h2 := cloneHash1(h1) // ERROR "&hash{} does not escape$" "devirtualizing h.Clone to \*hash$" "inlining call to \(\*hash\).Clone" "inlining call to cloneHash1" + _ = h2.Sum() // ERROR "devirtualizing h2.Sum to \*hash$" "inlining call to \(\*hash\).Sum" "make\(\[\]byte, 32\) does not escape$" + + h3 := cloneHash2(h1) // ERROR "devirtualizing h.Clone to \*hash$" "inlining call to \(\*hash\).Clone" "inlining call to cloneHash2" + _ = h3.Sum() // ERROR "devirtualizing h3.Sum to \*hash$" "inlining call to \(\*hash\).Sum" "make\(\[\]byte, 32\) does not escape$" + + h4 := cloneHash3(h1) // ERROR "&hash2{} escapes to heap$" "devirtualizing h.Clone to \*hash$" "inlining call to \(\*hash\).Clone" "inlining call to cloneHash3" "moved to heap: c$" + _ = h4.Sum() + + h5, _ := cloneHashWithBool1(h1) // ERROR "&hash{} does not escape$" "devirtualizing h.Clone to \*hash$" "inlining call to \(\*hash\).Clone" "inlining call to cloneHashWithBool1" + _ = h5.Sum() // ERROR "devirtualizing h5.Sum to \*hash$" "inlining call to \(\*hash\).Sum" "make\(\[\]byte, 32\) does not escape$" + + h6, _ := cloneHashWithBool2(h1) // ERROR "devirtualizing h.Clone to \*hash$" "inlining call to \(\*hash\).Clone" "inlining call to cloneHashWithBool2" + _ = h6.Sum() // ERROR "devirtualizing h6.Sum to \*hash$" "inlining call to \(\*hash\).Sum" "make\(\[\]byte, 32\) does not escape$" + + h7, _ := cloneHashWithBool3(h1) // ERROR "&hash2{} escapes to heap$" "devirtualizing h.Clone to \*hash$" "inlining call to \(\*hash\).Clone" "inlining call to cloneHashWithBool3" "moved to heap: c$" + _ = h7.Sum() +} + +type cloneableHashError interface { + hashIface + Clone() (hashIface, error) +} + +type hash3 struct{ state [32]byte } + +func newHash3() hashIface { // ERROR "can inline newHash3$" + return &hash3{} // ERROR "&hash3{} escapes to heap$" +} + +func (h *hash3) Sum() []byte { // ERROR "can inline \(\*hash3\).Sum$" "h does not escape$" + return make([]byte, 32) // ERROR "make\(\[\]byte, 32\) escapes to heap$" +} + +func (h *hash3) Clone() (hashIface, error) { // ERROR "can inline \(\*hash3\).Clone$" "h does not escape$" + c := *h // ERROR "moved to heap: c$" + return &c, nil +} + +func interleavedCloneableHashError() { + h1 := newHash3() // ERROR "&hash3{} does not escape$" "inlining call to newHash3" + _ = h1.Sum() // ERROR "devirtualizing h1.Sum to \*hash3$" "inlining call to \(\*hash3\).Sum" "make\(\[\]byte, 32\) does not escape$" + + if h1, ok := h1.(cloneableHashError); ok { + h2, err := h1.Clone() // ERROR "devirtualizing h1.Clone to \*hash3$" "inlining call to \(\*hash3\).Clone" + if err == nil { + _ = h2.Sum() // ERROR "devirtualizing h2.Sum to \*hash3$" "inlining call to \(\*hash3\).Sum" "make\(\[\]byte, 32\) does not escape$" + } + } +} diff --git a/test/escape2.go b/test/escape2.go index 3e5d11f88e1aba..a23d732061a908 100644 --- a/test/escape2.go +++ b/test/escape2.go @@ -494,13 +494,13 @@ func foo70(mv1 *MV, m M) { // ERROR "leaking param: m$" "leaking param: mv1$" func foo71(x *int) []*int { // ERROR "leaking param: x$" var y []*int - y = append(y, x) + y = append(y, x) // ERROR "append escapes to heap" return y } func foo71a(x int) []*int { // ERROR "moved to heap: x$" var y []*int - y = append(y, &x) + y = append(y, &x) // ERROR "append escapes to heap" return y } @@ -860,12 +860,12 @@ func foo104(x []*int) { // ERROR "leaking param content: x" // does not leak x but does leak content func foo105(x []*int) { // ERROR "leaking param content: x" - _ = append(y, x...) + _ = append(y, x...) // ERROR "append does not escape" } // does leak x func foo106(x *int) { // ERROR "leaking param: x$" - _ = append(y, x) + _ = append(y, x) // ERROR "append does not escape" } func foo107(x *int) map[*int]*int { // ERROR "leaking param: x$" diff --git a/test/escape2n.go b/test/escape2n.go index 26131521504cd4..23d1ea2b9a74e5 100644 --- a/test/escape2n.go +++ b/test/escape2n.go @@ -494,13 +494,13 @@ func foo70(mv1 *MV, m M) { // ERROR "leaking param: m$" "leaking param: mv1$" func foo71(x *int) []*int { // ERROR "leaking param: x$" var y []*int - y = append(y, x) + y = append(y, x) // ERROR "append escapes to heap" return y } func foo71a(x int) []*int { // ERROR "moved to heap: x$" var y []*int - y = append(y, &x) + y = append(y, &x) // ERROR "append escapes to heap" return y } @@ -860,12 +860,12 @@ func foo104(x []*int) { // ERROR "leaking param content: x" // does not leak x but does leak content func foo105(x []*int) { // ERROR "leaking param content: x" - _ = append(y, x...) + _ = append(y, x...) // ERROR "append does not escape" } // does leak x func foo106(x *int) { // ERROR "leaking param: x$" - _ = append(y, x) + _ = append(y, x) // ERROR "append does not escape" } func foo107(x *int) map[*int]*int { // ERROR "leaking param: x$" diff --git a/test/escape5.go b/test/escape5.go index 133d973ba5d5f8..2ed8b5789ddd7b 100644 --- a/test/escape5.go +++ b/test/escape5.go @@ -252,7 +252,7 @@ func f29000(_ int, x interface{}) { // ERROR "leaking param: x" func g29000() { x := 1 - f29000(2, x) // ERROR "x escapes to heap" + f29000(2, x) // ERROR "1 escapes to heap" } // Issue 28369: taking an address of a parameter and converting it into a uintptr causes an diff --git a/test/escape6.go b/test/escape6.go new file mode 100644 index 00000000000000..c45eb023fc7725 --- /dev/null +++ b/test/escape6.go @@ -0,0 +1,59 @@ +// errorcheck -0 -m -l + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Tests for escaping variable-sized allocations. +// In particular, we need to make sure things assigned into +// variable-sized allocations escape even when the variable-sized +// allocations themselves don't escape. + +package foo + +type T string + +func f1(n int, v T) { // ERROR "leaking param: v" + s := make([]T, n) // ERROR "make\(\[\]T, n\) does not escape" + s[0] = v + g(s) +} + +func f2(n int, v T) { // ERROR "leaking param: v" + s := make([]T, n) // ERROR "make\(\[\]T, n\) does not escape" + p := &s[0] + *p = v + g(s) +} + +func f3(n int, v T) { // ERROR "leaking param: v" + s := make([]T, n) // ERROR "make\(\[\]T, n\) does not escape" + t := (*[4]T)(s) + t[0] = v + g(s) +} + +// TODO: imprecise: this does not need to leak v. +func f4(v T) { // ERROR "leaking param: v" + s := make([]T, 4) // ERROR "make\(\[\]T, 4\) does not escape" + s[0] = v + g(s) +} + +// TODO: imprecise: this does not need to leak v. +func f5(v T) { // ERROR "leaking param: v" + var b [4]T + s := b[:] + s[0] = v + g(s) +} + +func f6(v T) { // ERROR "v does not escape" + var b [4]T + s := b[:] + b[0] = v + g(s) +} + +func g(s []T) { // ERROR "s does not escape" +} diff --git a/test/escape_array.go b/test/escape_array.go index 83062c9436e71d..1a1eb5e4aafb6f 100644 --- a/test/escape_array.go +++ b/test/escape_array.go @@ -123,7 +123,7 @@ func doesMakeSlice(x *string, y *string) { // ERROR "leaking param: x" "leaking func nonconstArray() { n := 32 - s1 := make([]int, n) // ERROR "make\(\[\]int, n\) escapes to heap" - s2 := make([]int, 0, n) // ERROR "make\(\[\]int, 0, n\) escapes to heap" + s1 := make([]int, n) // ERROR "make\(\[\]int, 32\) does not escape" + s2 := make([]int, 0, n) // ERROR "make\(\[\]int, 0, 32\) does not escape" _, _ = s1, s2 } diff --git a/test/escape_calls.go b/test/escape_calls.go index 5424c006ee4d0d..2525ef61396652 100644 --- a/test/escape_calls.go +++ b/test/escape_calls.go @@ -48,7 +48,7 @@ func prototype(xyz []string) {} // ERROR "xyz does not escape" func bar() { var got [][]string f := prototype - f = func(ss []string) { got = append(got, ss) } // ERROR "leaking param: ss" "func literal does not escape" + f = func(ss []string) { got = append(got, ss) } // ERROR "leaking param: ss" "func literal does not escape" "append escapes to heap" s := "string" f([]string{s}) // ERROR "\[\]string{...} escapes to heap" } diff --git a/test/escape_iface.go b/test/escape_iface.go index d822cca2f8485f..78b5209a629f3a 100644 --- a/test/escape_iface.go +++ b/test/escape_iface.go @@ -228,8 +228,8 @@ func dotTypeEscape2() { // #13805, #15796 j := 0 var v int var ok bool - var x interface{} = i // ERROR "i does not escape" - var y interface{} = j // ERROR "j does not escape" + var x interface{} = i // ERROR "0 does not escape" + var y interface{} = j // ERROR "0 does not escape" *(&v) = x.(int) *(&v), *(&ok) = y.(int) @@ -238,8 +238,8 @@ func dotTypeEscape2() { // #13805, #15796 i := 0 j := 0 var ok bool - var x interface{} = i // ERROR "i does not escape" - var y interface{} = j // ERROR "j does not escape" + var x interface{} = i // ERROR "0 does not escape" + var y interface{} = j // ERROR "0 does not escape" sink = x.(int) // ERROR "x.\(int\) escapes to heap" sink, *(&ok) = y.(int) // ERROR "autotmp_.* escapes to heap" diff --git a/test/escape_iface_data.go b/test/escape_iface_data.go new file mode 100644 index 00000000000000..b42974c4860f0c --- /dev/null +++ b/test/escape_iface_data.go @@ -0,0 +1,297 @@ +// errorcheck -0 -d=escapedebug=1 + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test the data word used for interface conversions +// that might otherwise allocate. + +package dataword + +var sink interface{} + +func string1() { + sink = "abc" // ERROR "using global for interface value" +} + +func string2() { + v := "abc" + sink = v // ERROR "using global for interface value" +} + +func string3() { + sink = "" // ERROR "using global for interface value" +} + +func string4() { + v := "" + sink = v // ERROR "using global for interface value" +} + +func string5() { + var a any = "abc" // ERROR "using global for interface value" + _ = a +} + +func string6() { + var a any + v := "abc" + a = v // ERROR "using global for interface value" + _ = a +} + +// string7 can be inlined. +func string7(v string) { + sink = v +} + +func string8() { + v0 := "abc" + v := v0 + string7(v) // ERROR "using global for interface value" +} + +func string9() { + v0 := "abc" + v := v0 + f := func() { + string7(v) + } + f() // ERROR "using global for interface value" +} + +func string10() { + v0 := "abc" + v := v0 + f := func() { + f2 := func() { + string7(v) + } + f2() + } + f() // ERROR "using global for interface value" +} + +func string11() { + v0 := "abc" + v := v0 + defer func() { + string7(v) // ERROR "using global for interface value" + }() +} + +func integer1() { + sink = 42 // ERROR "using global for interface value" +} + +func integer2() { + v := 42 + sink = v // ERROR "using global for interface value" +} + +func integer3() { + sink = 0 // ERROR "using global for interface value" +} + +func integer4a() { + v := 0 + sink = v // ERROR "using global for interface value" +} + +func integer4b() { + v := uint8(0) + sink = v // ERROR "using global for single-byte interface value" +} + +func integer5() { + var a any = 42 // ERROR "using global for interface value" + _ = a +} + +func integer6() { + var a any + v := 42 + a = v // ERROR "using global for interface value" + _ = a +} + +func integer7(v int) { + sink = v +} + +type M interface{ M() } + +type MyInt int + +func (m MyInt) M() {} + +func escapes(m M) { + sink = m +} + +func named1a() { + sink = MyInt(42) // ERROR "using global for interface value" +} + +func named1b() { + escapes(MyInt(42)) // ERROR "using global for interface value" +} + +func named2a() { + v := MyInt(0) + sink = v // ERROR "using global for interface value" +} + +func named2b() { + v := MyInt(42) + escapes(v) // ERROR "using global for interface value" +} + +func named2c() { + v := 42 + sink = MyInt(v) // ERROR "using global for interface value" +} + +func named2d() { + v := 42 + escapes(MyInt(v)) // ERROR "using global for interface value" +} +func named3a() { + sink = MyInt(42) // ERROR "using global for interface value" +} + +func named3b() { + escapes(MyInt(0)) // ERROR "using global for interface value" +} + +func named4a() { + v := MyInt(0) + sink = v // ERROR "using global for interface value" +} + +func named4b() { + v := MyInt(0) + escapes(v) // ERROR "using global for interface value" +} + +func named4c() { + v := 0 + sink = MyInt(v) // ERROR "using global for interface value" +} + +func named4d() { + v := 0 + escapes(MyInt(v)) // ERROR "using global for interface value" +} + +func named5() { + var a any = MyInt(42) // ERROR "using global for interface value" + _ = a +} + +func named6() { + var a any + v := MyInt(42) + a = v // ERROR "using global for interface value" + _ = a +} + +func named7a(v MyInt) { + sink = v +} + +func named7b(v MyInt) { + escapes(v) +} + +type S struct{ a, b int64 } + +func struct1() { + sink = S{1, 1} // ERROR "using global for interface value" +} + +func struct2() { + v := S{1, 1} + sink = v // ERROR "using global for interface value" +} + +func struct3() { + sink = S{} // ERROR "using global for zero value interface value" +} + +func struct4() { + v := S{} + sink = v // ERROR "using global for zero value interface value" +} + +func struct5() { + var a any = S{1, 1} // ERROR "using global for interface value" + _ = a +} + +func struct6() { + var a any + v := S{1, 1} + a = v // ERROR "using global for interface value" + _ = a +} + +func struct7(v S) { + sink = v +} + +func emptyStruct1() { + sink = struct{}{} // ERROR "using global for zero-sized interface value" +} + +func emptyStruct2() { + v := struct{}{} + sink = v // ERROR "using global for zero-sized interface value" +} + +func emptyStruct3(v struct{}) { // ERROR "using global for zero-sized interface value" + sink = v +} + +// Some light emulation of conditional debug printing (such as in #53465). + +func Printf(format string, args ...any) { + for _, arg := range args { + sink = arg + } +} + +var enabled = true + +func debugf(format string, args ...interface{}) { + if enabled { + Printf(format, args...) + } +} + +//go:noinline +func debugf2(format string, args ...interface{}) { + if enabled { + Printf(format, args...) + } +} + +func f1() { + v := 1000 + debugf("hello %d", v) // ERROR "using global for interface value" +} + +func f2() { + v := 1000 + debugf2("hello %d", v) // ERROR "using global for interface value" +} + +//go:noinline +func f3(i int) { + debugf("hello %d", i) +} + +func f4() { + f3(1000) +} diff --git a/test/escape_make_non_const.go b/test/escape_make_non_const.go new file mode 100644 index 00000000000000..11854ac4f4731a --- /dev/null +++ b/test/escape_make_non_const.go @@ -0,0 +1,123 @@ +// errorcheck -0 -m + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package escape + +const globalConstSize = 128 + +var globalVarSize = 128 + +//go:noinline +func testSlices() { + { + size := 128 + _ = make([]byte, size) // ERROR "make\(\[\]byte, 128\) does not escape" + } + + { + s := 128 + size := s + _ = make([]byte, size) // ERROR "make\(\[\]byte, 128\) does not escape" + } + + { + size := 128 + _ = make([]byte, size) // ERROR "make\(\[\]byte, 128\) does not escape" + } + + { + s := 128 + size := s + _ = make([]byte, size) // ERROR "make\(\[\]byte, 128\) does not escape" + } + + { + s1 := 128 + s2 := 256 + _ = make([]byte, s2, s1) // ERROR "make\(\[\]byte, s2, 128\) does not escape" + } + + allocLen(256) // ERROR "make\(\[\]byte, 256\) does not escape" "inlining call" + allocCap(256) // ERROR "make\(\[\]byte, 0, 256\) does not escape" "inlining call" + _ = newT(256) // ERROR "make\(\[\]byte, 256\) does not escape" "inlining call" + + { + size := globalConstSize + _ = make([]byte, size) // ERROR "make\(\[\]byte, 128\) does not escape" + } + + allocLen(globalConstSize) // ERROR "make\(\[\]byte, 128\) does not escape" "inlining call" + allocCap(globalConstSize) // ERROR "make\(\[\]byte, 0, 128\) does not escape" "inlining call" + _ = newT(globalConstSize) // ERROR "make\(\[\]byte, 128\) does not escape" "inlining call" + + { + c := 128 + s := 256 + _ = make([]byte, s, c) // ERROR "make\(\[\]byte, s, 128\) does not escape" + } + + { + s := 256 + _ = make([]byte, s, globalConstSize) // ERROR "make\(\[\]byte, s, 128\) does not escape" + } + + { + _ = make([]byte, globalVarSize) // ERROR "make\(\[\]byte, globalVarSize\) does not escape" + _ = make([]byte, globalVarSize, globalConstSize) // ERROR "make\(\[\]byte, globalVarSize, 128\) does not escape" + } +} + +func allocLen(l int) []byte { // ERROR "can inline" + return make([]byte, l) // ERROR "escapes to heap" +} + +func allocCap(l int) []byte { // ERROR "can inline" + return make([]byte, 0, l) // ERROR "escapes to heap" +} + +type t struct { + s []byte +} + +func newT(l int) t { // ERROR "can inline" + return t{make([]byte, l)} // ERROR "make.*escapes to heap" +} + +//go:noinline +func testMaps() { + size := 128 + _ = make(map[string]int, size) // ERROR "does not escape" + + _ = allocMapLen(128) // ERROR "does not escape" "inlining call" + _ = newM(128) // ERROR "does not escape" "inlining call" +} + +func allocMapLen(l int) map[string]int { // ERROR "can inline" + return make(map[string]int, l) // ERROR "escapes to heap" +} + +type m struct { + m map[string]int +} + +func newM(l int) m { // ERROR "can inline" + return m{make(map[string]int, l)} // ERROR "make.*escapes to heap" +} + +//go:noinline +func testLenOfSliceLit() { + ints := []int{0, 1, 2, 3, 4, 5} // ERROR "\[\]int\{\.\.\.\} does not escape"' + _ = make([]int, len(ints)) // ERROR "make\(\[\]int, 6\) does not escape" + _ = allocLenOf(ints) // ERROR "inlining call", "make\(\[\]int, 6\) does not escape" + + _ = make([]int, 2, len(ints)) // ERROR "make\(\[\]int, 2, 6\) does not escape" + _ = make([]int, len(ints), 2) // ERROR "make\(\[\]int, len\(ints\), 2\) does not escape" + _ = make([]int, 10, len(ints)) // ERROR "make\(\[\]int, 10, 6\) does not escape" +} + +func allocLenOf(s []int) []int { // ERROR "can inline" "s does not escape" + return make([]int, len(s)) // ERROR "escapes to heap" +} diff --git a/test/escape_map.go b/test/escape_map.go index 23abaa1e0cc5e9..ef6f79b039a1ac 100644 --- a/test/escape_map.go +++ b/test/escape_map.go @@ -45,7 +45,7 @@ func map3() []*int { m[&i] = &j var r []*int for k := range m { - r = append(r, k) + r = append(r, k) // ERROR "append escapes to heap" } return r } @@ -61,7 +61,7 @@ func map4() []*int { // We want to test exactly "for k, v := range m" rather than "for _, v := range m". // The following if is merely to use (but not leak) k. if k != nil { - r = append(r, v) + r = append(r, v) // ERROR "append escapes to heap" } } return r diff --git a/test/escape_reflect.go b/test/escape_reflect.go index 99fbada9a98eb4..e50323702e9162 100644 --- a/test/escape_reflect.go +++ b/test/escape_reflect.go @@ -423,7 +423,7 @@ func mapdelete(m map[string]string, k string) { // ERROR "m does not escape" "le } // Unfortunate: v doesn't need to leak. -func setiterkey1(v reflect.Value, it *reflect.MapIter) { // ERROR "leaking param: v$" "it does not escape" +func setiterkey1(v reflect.Value, it *reflect.MapIter) { // ERROR "leaking param: v$" "leaking param content: it$" v.SetIterKey(it) } @@ -434,7 +434,7 @@ func setiterkey2(v reflect.Value, m map[string]string) { // ERROR "leaking param } // Unfortunate: v doesn't need to leak. -func setitervalue1(v reflect.Value, it *reflect.MapIter) { // ERROR "leaking param: v$" "it does not escape" +func setitervalue1(v reflect.Value, it *reflect.MapIter) { // ERROR "leaking param: v$" "leaking param content: it$" v.SetIterValue(it) } diff --git a/test/escape_slice.go b/test/escape_slice.go index 65181e57d7edd3..9ac94e48ba5b39 100644 --- a/test/escape_slice.go +++ b/test/escape_slice.go @@ -18,29 +18,29 @@ var sink interface{} func slice0() { var s []*int // BAD: i should not escape - i := 0 // ERROR "moved to heap: i" - s = append(s, &i) + i := 0 // ERROR "moved to heap: i" + s = append(s, &i) // ERROR "append does not escape" _ = s } func slice1() *int { var s []*int - i := 0 // ERROR "moved to heap: i" - s = append(s, &i) + i := 0 // ERROR "moved to heap: i" + s = append(s, &i) // ERROR "append does not escape" return s[0] } func slice2() []*int { var s []*int - i := 0 // ERROR "moved to heap: i" - s = append(s, &i) + i := 0 // ERROR "moved to heap: i" + s = append(s, &i) // ERROR "append escapes to heap" return s } func slice3() *int { var s []*int - i := 0 // ERROR "moved to heap: i" - s = append(s, &i) + i := 0 // ERROR "moved to heap: i" + s = append(s, &i) // ERROR "append does not escape" for _, p := range s { return p } @@ -124,7 +124,7 @@ NextVar: continue NextVar } } - out = append(out, inkv) + out = append(out, inkv) // ERROR "append escapes to heap" } return out } @@ -167,7 +167,7 @@ var resolveIPAddrTests = []resolveIPAddrTest{ } func setupTestData() { - resolveIPAddrTests = append(resolveIPAddrTests, + resolveIPAddrTests = append(resolveIPAddrTests, // ERROR "append escapes to heap" []resolveIPAddrTest{ // ERROR "\[\]resolveIPAddrTest{...} does not escape" {"ip", "localhost", diff --git a/test/escape_unique.go b/test/escape_unique.go new file mode 100644 index 00000000000000..78d6eeb777de7a --- /dev/null +++ b/test/escape_unique.go @@ -0,0 +1,62 @@ +// errorcheck -0 -m -l + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test escape analysis for unique. + +package escape + +import "unique" + +type T string + +func f1(s string) unique.Handle[string] { // ERROR "s does not escape$" + return unique.Make(s) +} + +func f1a(s []byte) unique.Handle[string] { // ERROR "s does not escape$" + return unique.Make(string(s)) // ERROR "string\(s\) does not escape$" +} + +func gen[S ~string](s S) unique.Handle[S] { + return unique.Make(s) +} + +func f2(s T) unique.Handle[T] { // ERROR "s does not escape$" + return unique.Make(s) +} + +func f3(s T) unique.Handle[T] { // ERROR "s does not escape$" + return gen(s) +} + +type pair struct { + s1 string + s2 string +} + +func f4(s1 string, s2 string) unique.Handle[pair] { // ERROR "s1 does not escape$" "s2 does not escape$" + return unique.Make(pair{s1, s2}) +} + +type viaInterface struct { + s any +} + +func f5(s string) unique.Handle[viaInterface] { // ERROR "leaking param: s$" + return unique.Make(viaInterface{s}) // ERROR "s escapes to heap$" +} + +var sink any + +func f6(s string) unique.Handle[string] { // ERROR "leaking param: s$" + sink = s // ERROR "s escapes to heap$" + return unique.Make(s) +} + +func f6a(s []byte) unique.Handle[string] { // ERROR "leaking param: s$" + sink = s // ERROR "s escapes to heap$" + return unique.Make(string(s)) // ERROR "string\(s\) does not escape$" +} diff --git a/test/fixedbugs/bug022.go b/test/fixedbugs/bug022.go index 65a8bfe9a1989a..9e991bb49cb115 100644 --- a/test/fixedbugs/bug022.go +++ b/test/fixedbugs/bug022.go @@ -9,7 +9,7 @@ package main func putint(digits *string) { var i byte; i = (*digits)[7]; // compiles - i = digits[7]; // ERROR "illegal|is not|invalid" + i = digits[7]; // ERROR "illegal|is not|cannot index" _ = i; } diff --git a/test/fixedbugs/bug195.go b/test/fixedbugs/bug195.go index 769ed050b3704c..fa9cec75aae876 100644 --- a/test/fixedbugs/bug195.go +++ b/test/fixedbugs/bug195.go @@ -18,7 +18,7 @@ type I4 interface { // GC_ERROR "invalid recursive type: I4 refers to itself" I4 // GCCGO_ERROR "interface" } -type I5 interface { // GC_ERROR "invalid recursive type I5\n\tLINE:.* I5 refers to\n\tLINE+4:.* I6 refers to\n\tLINE:.* I5$" +type I5 interface { // GC_ERROR "invalid recursive type I5\n\tLINE:.* I5 refers to I6\n\tLINE+4:.* I6 refers to I5$" I6 } diff --git a/test/fixedbugs/bug385_64.go b/test/fixedbugs/bug385_64.go index deba9c9faecf9f..49201d953f7ab9 100644 --- a/test/fixedbugs/bug385_64.go +++ b/test/fixedbugs/bug385_64.go @@ -11,423 +11,49449 @@ package main -var z [10 << 20]byte +// seq 1 16480 | sed 's/.*/var z& [1 << 17]byte/' +var z1 [1 << 17]byte +var z2 [1 << 17]byte +var z3 [1 << 17]byte +var z4 [1 << 17]byte +var z5 [1 << 17]byte +var z6 [1 << 17]byte +var z7 [1 << 17]byte +var z8 [1 << 17]byte +var z9 [1 << 17]byte +var z10 [1 << 17]byte +var z11 [1 << 17]byte +var z12 [1 << 17]byte +var z13 [1 << 17]byte +var z14 [1 << 17]byte +var z15 [1 << 17]byte +var z16 [1 << 17]byte +var z17 [1 << 17]byte +var z18 [1 << 17]byte +var z19 [1 << 17]byte +var z20 [1 << 17]byte +var z21 [1 << 17]byte +var z22 [1 << 17]byte +var z23 [1 << 17]byte +var z24 [1 << 17]byte +var z25 [1 << 17]byte +var z26 [1 << 17]byte +var z27 [1 << 17]byte +var z28 [1 << 17]byte +var z29 [1 << 17]byte +var z30 [1 << 17]byte +var z31 [1 << 17]byte +var z32 [1 << 17]byte +var z33 [1 << 17]byte +var z34 [1 << 17]byte +var z35 [1 << 17]byte +var z36 [1 << 17]byte +var z37 [1 << 17]byte +var z38 [1 << 17]byte +var z39 [1 << 17]byte +var z40 [1 << 17]byte +var z41 [1 << 17]byte +var z42 [1 << 17]byte +var z43 [1 << 17]byte +var z44 [1 << 17]byte +var z45 [1 << 17]byte +var z46 [1 << 17]byte +var z47 [1 << 17]byte +var z48 [1 << 17]byte +var z49 [1 << 17]byte +var z50 [1 << 17]byte +var z51 [1 << 17]byte +var z52 [1 << 17]byte +var z53 [1 << 17]byte +var z54 [1 << 17]byte +var z55 [1 << 17]byte +var z56 [1 << 17]byte +var z57 [1 << 17]byte +var z58 [1 << 17]byte +var z59 [1 << 17]byte +var z60 [1 << 17]byte +var z61 [1 << 17]byte +var z62 [1 << 17]byte +var z63 [1 << 17]byte +var z64 [1 << 17]byte +var z65 [1 << 17]byte +var z66 [1 << 17]byte +var z67 [1 << 17]byte +var z68 [1 << 17]byte +var z69 [1 << 17]byte +var z70 [1 << 17]byte +var z71 [1 << 17]byte +var z72 [1 << 17]byte +var z73 [1 << 17]byte +var z74 [1 << 17]byte +var z75 [1 << 17]byte +var z76 [1 << 17]byte +var z77 [1 << 17]byte +var z78 [1 << 17]byte +var z79 [1 << 17]byte +var z80 [1 << 17]byte +var z81 [1 << 17]byte +var z82 [1 << 17]byte +var z83 [1 << 17]byte +var z84 [1 << 17]byte +var z85 [1 << 17]byte +var z86 [1 << 17]byte +var z87 [1 << 17]byte +var z88 [1 << 17]byte +var z89 [1 << 17]byte +var z90 [1 << 17]byte +var z91 [1 << 17]byte +var z92 [1 << 17]byte +var z93 [1 << 17]byte +var z94 [1 << 17]byte +var z95 [1 << 17]byte +var z96 [1 << 17]byte +var z97 [1 << 17]byte +var z98 [1 << 17]byte +var z99 [1 << 17]byte +var z100 [1 << 17]byte +var z101 [1 << 17]byte +var z102 [1 << 17]byte +var z103 [1 << 17]byte +var z104 [1 << 17]byte +var z105 [1 << 17]byte +var z106 [1 << 17]byte +var z107 [1 << 17]byte +var z108 [1 << 17]byte +var z109 [1 << 17]byte +var z110 [1 << 17]byte +var z111 [1 << 17]byte +var z112 [1 << 17]byte +var z113 [1 << 17]byte +var z114 [1 << 17]byte +var z115 [1 << 17]byte +var z116 [1 << 17]byte +var z117 [1 << 17]byte +var z118 [1 << 17]byte +var z119 [1 << 17]byte +var z120 [1 << 17]byte +var z121 [1 << 17]byte +var z122 [1 << 17]byte +var z123 [1 << 17]byte +var z124 [1 << 17]byte +var z125 [1 << 17]byte +var z126 [1 << 17]byte +var z127 [1 << 17]byte +var z128 [1 << 17]byte +var z129 [1 << 17]byte +var z130 [1 << 17]byte +var z131 [1 << 17]byte +var z132 [1 << 17]byte +var z133 [1 << 17]byte +var z134 [1 << 17]byte +var z135 [1 << 17]byte +var z136 [1 << 17]byte +var z137 [1 << 17]byte +var z138 [1 << 17]byte +var z139 [1 << 17]byte +var z140 [1 << 17]byte +var z141 [1 << 17]byte +var z142 [1 << 17]byte +var z143 [1 << 17]byte +var z144 [1 << 17]byte +var z145 [1 << 17]byte +var z146 [1 << 17]byte +var z147 [1 << 17]byte +var z148 [1 << 17]byte +var z149 [1 << 17]byte +var z150 [1 << 17]byte +var z151 [1 << 17]byte +var z152 [1 << 17]byte +var z153 [1 << 17]byte +var z154 [1 << 17]byte +var z155 [1 << 17]byte +var z156 [1 << 17]byte +var z157 [1 << 17]byte +var z158 [1 << 17]byte +var z159 [1 << 17]byte +var z160 [1 << 17]byte +var z161 [1 << 17]byte +var z162 [1 << 17]byte +var z163 [1 << 17]byte +var z164 [1 << 17]byte +var z165 [1 << 17]byte +var z166 [1 << 17]byte +var z167 [1 << 17]byte +var z168 [1 << 17]byte +var z169 [1 << 17]byte +var z170 [1 << 17]byte +var z171 [1 << 17]byte +var z172 [1 << 17]byte +var z173 [1 << 17]byte +var z174 [1 << 17]byte +var z175 [1 << 17]byte +var z176 [1 << 17]byte +var z177 [1 << 17]byte +var z178 [1 << 17]byte +var z179 [1 << 17]byte +var z180 [1 << 17]byte +var z181 [1 << 17]byte +var z182 [1 << 17]byte +var z183 [1 << 17]byte +var z184 [1 << 17]byte +var z185 [1 << 17]byte +var z186 [1 << 17]byte +var z187 [1 << 17]byte +var z188 [1 << 17]byte +var z189 [1 << 17]byte +var z190 [1 << 17]byte +var z191 [1 << 17]byte +var z192 [1 << 17]byte +var z193 [1 << 17]byte +var z194 [1 << 17]byte +var z195 [1 << 17]byte +var z196 [1 << 17]byte +var z197 [1 << 17]byte +var z198 [1 << 17]byte +var z199 [1 << 17]byte +var z200 [1 << 17]byte +var z201 [1 << 17]byte +var z202 [1 << 17]byte +var z203 [1 << 17]byte +var z204 [1 << 17]byte +var z205 [1 << 17]byte +var z206 [1 << 17]byte +var z207 [1 << 17]byte +var z208 [1 << 17]byte +var z209 [1 << 17]byte +var z210 [1 << 17]byte +var z211 [1 << 17]byte +var z212 [1 << 17]byte +var z213 [1 << 17]byte +var z214 [1 << 17]byte +var z215 [1 << 17]byte +var z216 [1 << 17]byte +var z217 [1 << 17]byte +var z218 [1 << 17]byte +var z219 [1 << 17]byte +var z220 [1 << 17]byte +var z221 [1 << 17]byte +var z222 [1 << 17]byte +var z223 [1 << 17]byte +var z224 [1 << 17]byte +var z225 [1 << 17]byte +var z226 [1 << 17]byte +var z227 [1 << 17]byte +var z228 [1 << 17]byte +var z229 [1 << 17]byte +var z230 [1 << 17]byte +var z231 [1 << 17]byte +var z232 [1 << 17]byte +var z233 [1 << 17]byte +var z234 [1 << 17]byte +var z235 [1 << 17]byte +var z236 [1 << 17]byte +var z237 [1 << 17]byte +var z238 [1 << 17]byte +var z239 [1 << 17]byte +var z240 [1 << 17]byte +var z241 [1 << 17]byte +var z242 [1 << 17]byte +var z243 [1 << 17]byte +var z244 [1 << 17]byte +var z245 [1 << 17]byte +var z246 [1 << 17]byte +var z247 [1 << 17]byte +var z248 [1 << 17]byte +var z249 [1 << 17]byte +var z250 [1 << 17]byte +var z251 [1 << 17]byte +var z252 [1 << 17]byte +var z253 [1 << 17]byte +var z254 [1 << 17]byte +var z255 [1 << 17]byte +var z256 [1 << 17]byte +var z257 [1 << 17]byte +var z258 [1 << 17]byte +var z259 [1 << 17]byte +var z260 [1 << 17]byte +var z261 [1 << 17]byte +var z262 [1 << 17]byte +var z263 [1 << 17]byte +var z264 [1 << 17]byte +var z265 [1 << 17]byte +var z266 [1 << 17]byte +var z267 [1 << 17]byte +var z268 [1 << 17]byte +var z269 [1 << 17]byte +var z270 [1 << 17]byte +var z271 [1 << 17]byte +var z272 [1 << 17]byte +var z273 [1 << 17]byte +var z274 [1 << 17]byte +var z275 [1 << 17]byte +var z276 [1 << 17]byte +var z277 [1 << 17]byte +var z278 [1 << 17]byte +var z279 [1 << 17]byte +var z280 [1 << 17]byte +var z281 [1 << 17]byte +var z282 [1 << 17]byte +var z283 [1 << 17]byte +var z284 [1 << 17]byte +var z285 [1 << 17]byte +var z286 [1 << 17]byte +var z287 [1 << 17]byte +var z288 [1 << 17]byte +var z289 [1 << 17]byte +var z290 [1 << 17]byte +var z291 [1 << 17]byte +var z292 [1 << 17]byte +var z293 [1 << 17]byte +var z294 [1 << 17]byte +var z295 [1 << 17]byte +var z296 [1 << 17]byte +var z297 [1 << 17]byte +var z298 [1 << 17]byte +var z299 [1 << 17]byte +var z300 [1 << 17]byte +var z301 [1 << 17]byte +var z302 [1 << 17]byte +var z303 [1 << 17]byte +var z304 [1 << 17]byte +var z305 [1 << 17]byte +var z306 [1 << 17]byte +var z307 [1 << 17]byte +var z308 [1 << 17]byte +var z309 [1 << 17]byte +var z310 [1 << 17]byte +var z311 [1 << 17]byte +var z312 [1 << 17]byte +var z313 [1 << 17]byte +var z314 [1 << 17]byte +var z315 [1 << 17]byte +var z316 [1 << 17]byte +var z317 [1 << 17]byte +var z318 [1 << 17]byte +var z319 [1 << 17]byte +var z320 [1 << 17]byte +var z321 [1 << 17]byte +var z322 [1 << 17]byte +var z323 [1 << 17]byte +var z324 [1 << 17]byte +var z325 [1 << 17]byte +var z326 [1 << 17]byte +var z327 [1 << 17]byte +var z328 [1 << 17]byte +var z329 [1 << 17]byte +var z330 [1 << 17]byte +var z331 [1 << 17]byte +var z332 [1 << 17]byte +var z333 [1 << 17]byte +var z334 [1 << 17]byte +var z335 [1 << 17]byte +var z336 [1 << 17]byte +var z337 [1 << 17]byte +var z338 [1 << 17]byte +var z339 [1 << 17]byte +var z340 [1 << 17]byte +var z341 [1 << 17]byte +var z342 [1 << 17]byte +var z343 [1 << 17]byte +var z344 [1 << 17]byte +var z345 [1 << 17]byte +var z346 [1 << 17]byte +var z347 [1 << 17]byte +var z348 [1 << 17]byte +var z349 [1 << 17]byte +var z350 [1 << 17]byte +var z351 [1 << 17]byte +var z352 [1 << 17]byte +var z353 [1 << 17]byte +var z354 [1 << 17]byte +var z355 [1 << 17]byte +var z356 [1 << 17]byte +var z357 [1 << 17]byte +var z358 [1 << 17]byte +var z359 [1 << 17]byte +var z360 [1 << 17]byte +var z361 [1 << 17]byte +var z362 [1 << 17]byte +var z363 [1 << 17]byte +var z364 [1 << 17]byte +var z365 [1 << 17]byte +var z366 [1 << 17]byte +var z367 [1 << 17]byte +var z368 [1 << 17]byte +var z369 [1 << 17]byte +var z370 [1 << 17]byte +var z371 [1 << 17]byte +var z372 [1 << 17]byte +var z373 [1 << 17]byte +var z374 [1 << 17]byte +var z375 [1 << 17]byte +var z376 [1 << 17]byte +var z377 [1 << 17]byte +var z378 [1 << 17]byte +var z379 [1 << 17]byte +var z380 [1 << 17]byte +var z381 [1 << 17]byte +var z382 [1 << 17]byte +var z383 [1 << 17]byte +var z384 [1 << 17]byte +var z385 [1 << 17]byte +var z386 [1 << 17]byte +var z387 [1 << 17]byte +var z388 [1 << 17]byte +var z389 [1 << 17]byte +var z390 [1 << 17]byte +var z391 [1 << 17]byte +var z392 [1 << 17]byte +var z393 [1 << 17]byte +var z394 [1 << 17]byte +var z395 [1 << 17]byte +var z396 [1 << 17]byte +var z397 [1 << 17]byte +var z398 [1 << 17]byte +var z399 [1 << 17]byte +var z400 [1 << 17]byte +var z401 [1 << 17]byte +var z402 [1 << 17]byte +var z403 [1 << 17]byte +var z404 [1 << 17]byte +var z405 [1 << 17]byte +var z406 [1 << 17]byte +var z407 [1 << 17]byte +var z408 [1 << 17]byte +var z409 [1 << 17]byte +var z410 [1 << 17]byte +var z411 [1 << 17]byte +var z412 [1 << 17]byte +var z413 [1 << 17]byte +var z414 [1 << 17]byte +var z415 [1 << 17]byte +var z416 [1 << 17]byte +var z417 [1 << 17]byte +var z418 [1 << 17]byte +var z419 [1 << 17]byte +var z420 [1 << 17]byte +var z421 [1 << 17]byte +var z422 [1 << 17]byte +var z423 [1 << 17]byte +var z424 [1 << 17]byte +var z425 [1 << 17]byte +var z426 [1 << 17]byte +var z427 [1 << 17]byte +var z428 [1 << 17]byte +var z429 [1 << 17]byte +var z430 [1 << 17]byte +var z431 [1 << 17]byte +var z432 [1 << 17]byte +var z433 [1 << 17]byte +var z434 [1 << 17]byte +var z435 [1 << 17]byte +var z436 [1 << 17]byte +var z437 [1 << 17]byte +var z438 [1 << 17]byte +var z439 [1 << 17]byte +var z440 [1 << 17]byte +var z441 [1 << 17]byte +var z442 [1 << 17]byte +var z443 [1 << 17]byte +var z444 [1 << 17]byte +var z445 [1 << 17]byte +var z446 [1 << 17]byte +var z447 [1 << 17]byte +var z448 [1 << 17]byte +var z449 [1 << 17]byte +var z450 [1 << 17]byte +var z451 [1 << 17]byte +var z452 [1 << 17]byte +var z453 [1 << 17]byte +var z454 [1 << 17]byte +var z455 [1 << 17]byte +var z456 [1 << 17]byte +var z457 [1 << 17]byte +var z458 [1 << 17]byte +var z459 [1 << 17]byte +var z460 [1 << 17]byte +var z461 [1 << 17]byte +var z462 [1 << 17]byte +var z463 [1 << 17]byte +var z464 [1 << 17]byte +var z465 [1 << 17]byte +var z466 [1 << 17]byte +var z467 [1 << 17]byte +var z468 [1 << 17]byte +var z469 [1 << 17]byte +var z470 [1 << 17]byte +var z471 [1 << 17]byte +var z472 [1 << 17]byte +var z473 [1 << 17]byte +var z474 [1 << 17]byte +var z475 [1 << 17]byte +var z476 [1 << 17]byte +var z477 [1 << 17]byte +var z478 [1 << 17]byte +var z479 [1 << 17]byte +var z480 [1 << 17]byte +var z481 [1 << 17]byte +var z482 [1 << 17]byte +var z483 [1 << 17]byte +var z484 [1 << 17]byte +var z485 [1 << 17]byte +var z486 [1 << 17]byte +var z487 [1 << 17]byte +var z488 [1 << 17]byte +var z489 [1 << 17]byte +var z490 [1 << 17]byte +var z491 [1 << 17]byte +var z492 [1 << 17]byte +var z493 [1 << 17]byte +var z494 [1 << 17]byte +var z495 [1 << 17]byte +var z496 [1 << 17]byte +var z497 [1 << 17]byte +var z498 [1 << 17]byte +var z499 [1 << 17]byte +var z500 [1 << 17]byte +var z501 [1 << 17]byte +var z502 [1 << 17]byte +var z503 [1 << 17]byte +var z504 [1 << 17]byte +var z505 [1 << 17]byte +var z506 [1 << 17]byte +var z507 [1 << 17]byte +var z508 [1 << 17]byte +var z509 [1 << 17]byte +var z510 [1 << 17]byte +var z511 [1 << 17]byte +var z512 [1 << 17]byte +var z513 [1 << 17]byte +var z514 [1 << 17]byte +var z515 [1 << 17]byte +var z516 [1 << 17]byte +var z517 [1 << 17]byte +var z518 [1 << 17]byte +var z519 [1 << 17]byte +var z520 [1 << 17]byte +var z521 [1 << 17]byte +var z522 [1 << 17]byte +var z523 [1 << 17]byte +var z524 [1 << 17]byte +var z525 [1 << 17]byte +var z526 [1 << 17]byte +var z527 [1 << 17]byte +var z528 [1 << 17]byte +var z529 [1 << 17]byte +var z530 [1 << 17]byte +var z531 [1 << 17]byte +var z532 [1 << 17]byte +var z533 [1 << 17]byte +var z534 [1 << 17]byte +var z535 [1 << 17]byte +var z536 [1 << 17]byte +var z537 [1 << 17]byte +var z538 [1 << 17]byte +var z539 [1 << 17]byte +var z540 [1 << 17]byte +var z541 [1 << 17]byte +var z542 [1 << 17]byte +var z543 [1 << 17]byte +var z544 [1 << 17]byte +var z545 [1 << 17]byte +var z546 [1 << 17]byte +var z547 [1 << 17]byte +var z548 [1 << 17]byte +var z549 [1 << 17]byte +var z550 [1 << 17]byte +var z551 [1 << 17]byte +var z552 [1 << 17]byte +var z553 [1 << 17]byte +var z554 [1 << 17]byte +var z555 [1 << 17]byte +var z556 [1 << 17]byte +var z557 [1 << 17]byte +var z558 [1 << 17]byte +var z559 [1 << 17]byte +var z560 [1 << 17]byte +var z561 [1 << 17]byte +var z562 [1 << 17]byte +var z563 [1 << 17]byte +var z564 [1 << 17]byte +var z565 [1 << 17]byte +var z566 [1 << 17]byte +var z567 [1 << 17]byte +var z568 [1 << 17]byte +var z569 [1 << 17]byte +var z570 [1 << 17]byte +var z571 [1 << 17]byte +var z572 [1 << 17]byte +var z573 [1 << 17]byte +var z574 [1 << 17]byte +var z575 [1 << 17]byte +var z576 [1 << 17]byte +var z577 [1 << 17]byte +var z578 [1 << 17]byte +var z579 [1 << 17]byte +var z580 [1 << 17]byte +var z581 [1 << 17]byte +var z582 [1 << 17]byte +var z583 [1 << 17]byte +var z584 [1 << 17]byte +var z585 [1 << 17]byte +var z586 [1 << 17]byte +var z587 [1 << 17]byte +var z588 [1 << 17]byte +var z589 [1 << 17]byte +var z590 [1 << 17]byte +var z591 [1 << 17]byte +var z592 [1 << 17]byte +var z593 [1 << 17]byte +var z594 [1 << 17]byte +var z595 [1 << 17]byte +var z596 [1 << 17]byte +var z597 [1 << 17]byte +var z598 [1 << 17]byte +var z599 [1 << 17]byte +var z600 [1 << 17]byte +var z601 [1 << 17]byte +var z602 [1 << 17]byte +var z603 [1 << 17]byte +var z604 [1 << 17]byte +var z605 [1 << 17]byte +var z606 [1 << 17]byte +var z607 [1 << 17]byte +var z608 [1 << 17]byte +var z609 [1 << 17]byte +var z610 [1 << 17]byte +var z611 [1 << 17]byte +var z612 [1 << 17]byte +var z613 [1 << 17]byte +var z614 [1 << 17]byte +var z615 [1 << 17]byte +var z616 [1 << 17]byte +var z617 [1 << 17]byte +var z618 [1 << 17]byte +var z619 [1 << 17]byte +var z620 [1 << 17]byte +var z621 [1 << 17]byte +var z622 [1 << 17]byte +var z623 [1 << 17]byte +var z624 [1 << 17]byte +var z625 [1 << 17]byte +var z626 [1 << 17]byte +var z627 [1 << 17]byte +var z628 [1 << 17]byte +var z629 [1 << 17]byte +var z630 [1 << 17]byte +var z631 [1 << 17]byte +var z632 [1 << 17]byte +var z633 [1 << 17]byte +var z634 [1 << 17]byte +var z635 [1 << 17]byte +var z636 [1 << 17]byte +var z637 [1 << 17]byte +var z638 [1 << 17]byte +var z639 [1 << 17]byte +var z640 [1 << 17]byte +var z641 [1 << 17]byte +var z642 [1 << 17]byte +var z643 [1 << 17]byte +var z644 [1 << 17]byte +var z645 [1 << 17]byte +var z646 [1 << 17]byte +var z647 [1 << 17]byte +var z648 [1 << 17]byte +var z649 [1 << 17]byte +var z650 [1 << 17]byte +var z651 [1 << 17]byte +var z652 [1 << 17]byte +var z653 [1 << 17]byte +var z654 [1 << 17]byte +var z655 [1 << 17]byte +var z656 [1 << 17]byte +var z657 [1 << 17]byte +var z658 [1 << 17]byte +var z659 [1 << 17]byte +var z660 [1 << 17]byte +var z661 [1 << 17]byte +var z662 [1 << 17]byte +var z663 [1 << 17]byte +var z664 [1 << 17]byte +var z665 [1 << 17]byte +var z666 [1 << 17]byte +var z667 [1 << 17]byte +var z668 [1 << 17]byte +var z669 [1 << 17]byte +var z670 [1 << 17]byte +var z671 [1 << 17]byte +var z672 [1 << 17]byte +var z673 [1 << 17]byte +var z674 [1 << 17]byte +var z675 [1 << 17]byte +var z676 [1 << 17]byte +var z677 [1 << 17]byte +var z678 [1 << 17]byte +var z679 [1 << 17]byte +var z680 [1 << 17]byte +var z681 [1 << 17]byte +var z682 [1 << 17]byte +var z683 [1 << 17]byte +var z684 [1 << 17]byte +var z685 [1 << 17]byte +var z686 [1 << 17]byte +var z687 [1 << 17]byte +var z688 [1 << 17]byte +var z689 [1 << 17]byte +var z690 [1 << 17]byte +var z691 [1 << 17]byte +var z692 [1 << 17]byte +var z693 [1 << 17]byte +var z694 [1 << 17]byte +var z695 [1 << 17]byte +var z696 [1 << 17]byte +var z697 [1 << 17]byte +var z698 [1 << 17]byte +var z699 [1 << 17]byte +var z700 [1 << 17]byte +var z701 [1 << 17]byte +var z702 [1 << 17]byte +var z703 [1 << 17]byte +var z704 [1 << 17]byte +var z705 [1 << 17]byte +var z706 [1 << 17]byte +var z707 [1 << 17]byte +var z708 [1 << 17]byte +var z709 [1 << 17]byte +var z710 [1 << 17]byte +var z711 [1 << 17]byte +var z712 [1 << 17]byte +var z713 [1 << 17]byte +var z714 [1 << 17]byte +var z715 [1 << 17]byte +var z716 [1 << 17]byte +var z717 [1 << 17]byte +var z718 [1 << 17]byte +var z719 [1 << 17]byte +var z720 [1 << 17]byte +var z721 [1 << 17]byte +var z722 [1 << 17]byte +var z723 [1 << 17]byte +var z724 [1 << 17]byte +var z725 [1 << 17]byte +var z726 [1 << 17]byte +var z727 [1 << 17]byte +var z728 [1 << 17]byte +var z729 [1 << 17]byte +var z730 [1 << 17]byte +var z731 [1 << 17]byte +var z732 [1 << 17]byte +var z733 [1 << 17]byte +var z734 [1 << 17]byte +var z735 [1 << 17]byte +var z736 [1 << 17]byte +var z737 [1 << 17]byte +var z738 [1 << 17]byte +var z739 [1 << 17]byte +var z740 [1 << 17]byte +var z741 [1 << 17]byte +var z742 [1 << 17]byte +var z743 [1 << 17]byte +var z744 [1 << 17]byte +var z745 [1 << 17]byte +var z746 [1 << 17]byte +var z747 [1 << 17]byte +var z748 [1 << 17]byte +var z749 [1 << 17]byte +var z750 [1 << 17]byte +var z751 [1 << 17]byte +var z752 [1 << 17]byte +var z753 [1 << 17]byte +var z754 [1 << 17]byte +var z755 [1 << 17]byte +var z756 [1 << 17]byte +var z757 [1 << 17]byte +var z758 [1 << 17]byte +var z759 [1 << 17]byte +var z760 [1 << 17]byte +var z761 [1 << 17]byte +var z762 [1 << 17]byte +var z763 [1 << 17]byte +var z764 [1 << 17]byte +var z765 [1 << 17]byte +var z766 [1 << 17]byte +var z767 [1 << 17]byte +var z768 [1 << 17]byte +var z769 [1 << 17]byte +var z770 [1 << 17]byte +var z771 [1 << 17]byte +var z772 [1 << 17]byte +var z773 [1 << 17]byte +var z774 [1 << 17]byte +var z775 [1 << 17]byte +var z776 [1 << 17]byte +var z777 [1 << 17]byte +var z778 [1 << 17]byte +var z779 [1 << 17]byte +var z780 [1 << 17]byte +var z781 [1 << 17]byte +var z782 [1 << 17]byte +var z783 [1 << 17]byte +var z784 [1 << 17]byte +var z785 [1 << 17]byte +var z786 [1 << 17]byte +var z787 [1 << 17]byte +var z788 [1 << 17]byte +var z789 [1 << 17]byte +var z790 [1 << 17]byte +var z791 [1 << 17]byte +var z792 [1 << 17]byte +var z793 [1 << 17]byte +var z794 [1 << 17]byte +var z795 [1 << 17]byte +var z796 [1 << 17]byte +var z797 [1 << 17]byte +var z798 [1 << 17]byte +var z799 [1 << 17]byte +var z800 [1 << 17]byte +var z801 [1 << 17]byte +var z802 [1 << 17]byte +var z803 [1 << 17]byte +var z804 [1 << 17]byte +var z805 [1 << 17]byte +var z806 [1 << 17]byte +var z807 [1 << 17]byte +var z808 [1 << 17]byte +var z809 [1 << 17]byte +var z810 [1 << 17]byte +var z811 [1 << 17]byte +var z812 [1 << 17]byte +var z813 [1 << 17]byte +var z814 [1 << 17]byte +var z815 [1 << 17]byte +var z816 [1 << 17]byte +var z817 [1 << 17]byte +var z818 [1 << 17]byte +var z819 [1 << 17]byte +var z820 [1 << 17]byte +var z821 [1 << 17]byte +var z822 [1 << 17]byte +var z823 [1 << 17]byte +var z824 [1 << 17]byte +var z825 [1 << 17]byte +var z826 [1 << 17]byte +var z827 [1 << 17]byte +var z828 [1 << 17]byte +var z829 [1 << 17]byte +var z830 [1 << 17]byte +var z831 [1 << 17]byte +var z832 [1 << 17]byte +var z833 [1 << 17]byte +var z834 [1 << 17]byte +var z835 [1 << 17]byte +var z836 [1 << 17]byte +var z837 [1 << 17]byte +var z838 [1 << 17]byte +var z839 [1 << 17]byte +var z840 [1 << 17]byte +var z841 [1 << 17]byte +var z842 [1 << 17]byte +var z843 [1 << 17]byte +var z844 [1 << 17]byte +var z845 [1 << 17]byte +var z846 [1 << 17]byte +var z847 [1 << 17]byte +var z848 [1 << 17]byte +var z849 [1 << 17]byte +var z850 [1 << 17]byte +var z851 [1 << 17]byte +var z852 [1 << 17]byte +var z853 [1 << 17]byte +var z854 [1 << 17]byte +var z855 [1 << 17]byte +var z856 [1 << 17]byte +var z857 [1 << 17]byte +var z858 [1 << 17]byte +var z859 [1 << 17]byte +var z860 [1 << 17]byte +var z861 [1 << 17]byte +var z862 [1 << 17]byte +var z863 [1 << 17]byte +var z864 [1 << 17]byte +var z865 [1 << 17]byte +var z866 [1 << 17]byte +var z867 [1 << 17]byte +var z868 [1 << 17]byte +var z869 [1 << 17]byte +var z870 [1 << 17]byte +var z871 [1 << 17]byte +var z872 [1 << 17]byte +var z873 [1 << 17]byte +var z874 [1 << 17]byte +var z875 [1 << 17]byte +var z876 [1 << 17]byte +var z877 [1 << 17]byte +var z878 [1 << 17]byte +var z879 [1 << 17]byte +var z880 [1 << 17]byte +var z881 [1 << 17]byte +var z882 [1 << 17]byte +var z883 [1 << 17]byte +var z884 [1 << 17]byte +var z885 [1 << 17]byte +var z886 [1 << 17]byte +var z887 [1 << 17]byte +var z888 [1 << 17]byte +var z889 [1 << 17]byte +var z890 [1 << 17]byte +var z891 [1 << 17]byte +var z892 [1 << 17]byte +var z893 [1 << 17]byte +var z894 [1 << 17]byte +var z895 [1 << 17]byte +var z896 [1 << 17]byte +var z897 [1 << 17]byte +var z898 [1 << 17]byte +var z899 [1 << 17]byte +var z900 [1 << 17]byte +var z901 [1 << 17]byte +var z902 [1 << 17]byte +var z903 [1 << 17]byte +var z904 [1 << 17]byte +var z905 [1 << 17]byte +var z906 [1 << 17]byte +var z907 [1 << 17]byte +var z908 [1 << 17]byte +var z909 [1 << 17]byte +var z910 [1 << 17]byte +var z911 [1 << 17]byte +var z912 [1 << 17]byte +var z913 [1 << 17]byte +var z914 [1 << 17]byte +var z915 [1 << 17]byte +var z916 [1 << 17]byte +var z917 [1 << 17]byte +var z918 [1 << 17]byte +var z919 [1 << 17]byte +var z920 [1 << 17]byte +var z921 [1 << 17]byte +var z922 [1 << 17]byte +var z923 [1 << 17]byte +var z924 [1 << 17]byte +var z925 [1 << 17]byte +var z926 [1 << 17]byte +var z927 [1 << 17]byte +var z928 [1 << 17]byte +var z929 [1 << 17]byte +var z930 [1 << 17]byte +var z931 [1 << 17]byte +var z932 [1 << 17]byte +var z933 [1 << 17]byte +var z934 [1 << 17]byte +var z935 [1 << 17]byte +var z936 [1 << 17]byte +var z937 [1 << 17]byte +var z938 [1 << 17]byte +var z939 [1 << 17]byte +var z940 [1 << 17]byte +var z941 [1 << 17]byte +var z942 [1 << 17]byte +var z943 [1 << 17]byte +var z944 [1 << 17]byte +var z945 [1 << 17]byte +var z946 [1 << 17]byte +var z947 [1 << 17]byte +var z948 [1 << 17]byte +var z949 [1 << 17]byte +var z950 [1 << 17]byte +var z951 [1 << 17]byte +var z952 [1 << 17]byte +var z953 [1 << 17]byte +var z954 [1 << 17]byte +var z955 [1 << 17]byte +var z956 [1 << 17]byte +var z957 [1 << 17]byte +var z958 [1 << 17]byte +var z959 [1 << 17]byte +var z960 [1 << 17]byte +var z961 [1 << 17]byte +var z962 [1 << 17]byte +var z963 [1 << 17]byte +var z964 [1 << 17]byte +var z965 [1 << 17]byte +var z966 [1 << 17]byte +var z967 [1 << 17]byte +var z968 [1 << 17]byte +var z969 [1 << 17]byte +var z970 [1 << 17]byte +var z971 [1 << 17]byte +var z972 [1 << 17]byte +var z973 [1 << 17]byte +var z974 [1 << 17]byte +var z975 [1 << 17]byte +var z976 [1 << 17]byte +var z977 [1 << 17]byte +var z978 [1 << 17]byte +var z979 [1 << 17]byte +var z980 [1 << 17]byte +var z981 [1 << 17]byte +var z982 [1 << 17]byte +var z983 [1 << 17]byte +var z984 [1 << 17]byte +var z985 [1 << 17]byte +var z986 [1 << 17]byte +var z987 [1 << 17]byte +var z988 [1 << 17]byte +var z989 [1 << 17]byte +var z990 [1 << 17]byte +var z991 [1 << 17]byte +var z992 [1 << 17]byte +var z993 [1 << 17]byte +var z994 [1 << 17]byte +var z995 [1 << 17]byte +var z996 [1 << 17]byte +var z997 [1 << 17]byte +var z998 [1 << 17]byte +var z999 [1 << 17]byte +var z1000 [1 << 17]byte +var z1001 [1 << 17]byte +var z1002 [1 << 17]byte +var z1003 [1 << 17]byte +var z1004 [1 << 17]byte +var z1005 [1 << 17]byte +var z1006 [1 << 17]byte +var z1007 [1 << 17]byte +var z1008 [1 << 17]byte +var z1009 [1 << 17]byte +var z1010 [1 << 17]byte +var z1011 [1 << 17]byte +var z1012 [1 << 17]byte +var z1013 [1 << 17]byte +var z1014 [1 << 17]byte +var z1015 [1 << 17]byte +var z1016 [1 << 17]byte +var z1017 [1 << 17]byte +var z1018 [1 << 17]byte +var z1019 [1 << 17]byte +var z1020 [1 << 17]byte +var z1021 [1 << 17]byte +var z1022 [1 << 17]byte +var z1023 [1 << 17]byte +var z1024 [1 << 17]byte +var z1025 [1 << 17]byte +var z1026 [1 << 17]byte +var z1027 [1 << 17]byte +var z1028 [1 << 17]byte +var z1029 [1 << 17]byte +var z1030 [1 << 17]byte +var z1031 [1 << 17]byte +var z1032 [1 << 17]byte +var z1033 [1 << 17]byte +var z1034 [1 << 17]byte +var z1035 [1 << 17]byte +var z1036 [1 << 17]byte +var z1037 [1 << 17]byte +var z1038 [1 << 17]byte +var z1039 [1 << 17]byte +var z1040 [1 << 17]byte +var z1041 [1 << 17]byte +var z1042 [1 << 17]byte +var z1043 [1 << 17]byte +var z1044 [1 << 17]byte +var z1045 [1 << 17]byte +var z1046 [1 << 17]byte +var z1047 [1 << 17]byte +var z1048 [1 << 17]byte +var z1049 [1 << 17]byte +var z1050 [1 << 17]byte +var z1051 [1 << 17]byte +var z1052 [1 << 17]byte +var z1053 [1 << 17]byte +var z1054 [1 << 17]byte +var z1055 [1 << 17]byte +var z1056 [1 << 17]byte +var z1057 [1 << 17]byte +var z1058 [1 << 17]byte +var z1059 [1 << 17]byte +var z1060 [1 << 17]byte +var z1061 [1 << 17]byte +var z1062 [1 << 17]byte +var z1063 [1 << 17]byte +var z1064 [1 << 17]byte +var z1065 [1 << 17]byte +var z1066 [1 << 17]byte +var z1067 [1 << 17]byte +var z1068 [1 << 17]byte +var z1069 [1 << 17]byte +var z1070 [1 << 17]byte +var z1071 [1 << 17]byte +var z1072 [1 << 17]byte +var z1073 [1 << 17]byte +var z1074 [1 << 17]byte +var z1075 [1 << 17]byte +var z1076 [1 << 17]byte +var z1077 [1 << 17]byte +var z1078 [1 << 17]byte +var z1079 [1 << 17]byte +var z1080 [1 << 17]byte +var z1081 [1 << 17]byte +var z1082 [1 << 17]byte +var z1083 [1 << 17]byte +var z1084 [1 << 17]byte +var z1085 [1 << 17]byte +var z1086 [1 << 17]byte +var z1087 [1 << 17]byte +var z1088 [1 << 17]byte +var z1089 [1 << 17]byte +var z1090 [1 << 17]byte +var z1091 [1 << 17]byte +var z1092 [1 << 17]byte +var z1093 [1 << 17]byte +var z1094 [1 << 17]byte +var z1095 [1 << 17]byte +var z1096 [1 << 17]byte +var z1097 [1 << 17]byte +var z1098 [1 << 17]byte +var z1099 [1 << 17]byte +var z1100 [1 << 17]byte +var z1101 [1 << 17]byte +var z1102 [1 << 17]byte +var z1103 [1 << 17]byte +var z1104 [1 << 17]byte +var z1105 [1 << 17]byte +var z1106 [1 << 17]byte +var z1107 [1 << 17]byte +var z1108 [1 << 17]byte +var z1109 [1 << 17]byte +var z1110 [1 << 17]byte +var z1111 [1 << 17]byte +var z1112 [1 << 17]byte +var z1113 [1 << 17]byte +var z1114 [1 << 17]byte +var z1115 [1 << 17]byte +var z1116 [1 << 17]byte +var z1117 [1 << 17]byte +var z1118 [1 << 17]byte +var z1119 [1 << 17]byte +var z1120 [1 << 17]byte +var z1121 [1 << 17]byte +var z1122 [1 << 17]byte +var z1123 [1 << 17]byte +var z1124 [1 << 17]byte +var z1125 [1 << 17]byte +var z1126 [1 << 17]byte +var z1127 [1 << 17]byte +var z1128 [1 << 17]byte +var z1129 [1 << 17]byte +var z1130 [1 << 17]byte +var z1131 [1 << 17]byte +var z1132 [1 << 17]byte +var z1133 [1 << 17]byte +var z1134 [1 << 17]byte +var z1135 [1 << 17]byte +var z1136 [1 << 17]byte +var z1137 [1 << 17]byte +var z1138 [1 << 17]byte +var z1139 [1 << 17]byte +var z1140 [1 << 17]byte +var z1141 [1 << 17]byte +var z1142 [1 << 17]byte +var z1143 [1 << 17]byte +var z1144 [1 << 17]byte +var z1145 [1 << 17]byte +var z1146 [1 << 17]byte +var z1147 [1 << 17]byte +var z1148 [1 << 17]byte +var z1149 [1 << 17]byte +var z1150 [1 << 17]byte +var z1151 [1 << 17]byte +var z1152 [1 << 17]byte +var z1153 [1 << 17]byte +var z1154 [1 << 17]byte +var z1155 [1 << 17]byte +var z1156 [1 << 17]byte +var z1157 [1 << 17]byte +var z1158 [1 << 17]byte +var z1159 [1 << 17]byte +var z1160 [1 << 17]byte +var z1161 [1 << 17]byte +var z1162 [1 << 17]byte +var z1163 [1 << 17]byte +var z1164 [1 << 17]byte +var z1165 [1 << 17]byte +var z1166 [1 << 17]byte +var z1167 [1 << 17]byte +var z1168 [1 << 17]byte +var z1169 [1 << 17]byte +var z1170 [1 << 17]byte +var z1171 [1 << 17]byte +var z1172 [1 << 17]byte +var z1173 [1 << 17]byte +var z1174 [1 << 17]byte +var z1175 [1 << 17]byte +var z1176 [1 << 17]byte +var z1177 [1 << 17]byte +var z1178 [1 << 17]byte +var z1179 [1 << 17]byte +var z1180 [1 << 17]byte +var z1181 [1 << 17]byte +var z1182 [1 << 17]byte +var z1183 [1 << 17]byte +var z1184 [1 << 17]byte +var z1185 [1 << 17]byte +var z1186 [1 << 17]byte +var z1187 [1 << 17]byte +var z1188 [1 << 17]byte +var z1189 [1 << 17]byte +var z1190 [1 << 17]byte +var z1191 [1 << 17]byte +var z1192 [1 << 17]byte +var z1193 [1 << 17]byte +var z1194 [1 << 17]byte +var z1195 [1 << 17]byte +var z1196 [1 << 17]byte +var z1197 [1 << 17]byte +var z1198 [1 << 17]byte +var z1199 [1 << 17]byte +var z1200 [1 << 17]byte +var z1201 [1 << 17]byte +var z1202 [1 << 17]byte +var z1203 [1 << 17]byte +var z1204 [1 << 17]byte +var z1205 [1 << 17]byte +var z1206 [1 << 17]byte +var z1207 [1 << 17]byte +var z1208 [1 << 17]byte +var z1209 [1 << 17]byte +var z1210 [1 << 17]byte +var z1211 [1 << 17]byte +var z1212 [1 << 17]byte +var z1213 [1 << 17]byte +var z1214 [1 << 17]byte +var z1215 [1 << 17]byte +var z1216 [1 << 17]byte +var z1217 [1 << 17]byte +var z1218 [1 << 17]byte +var z1219 [1 << 17]byte +var z1220 [1 << 17]byte +var z1221 [1 << 17]byte +var z1222 [1 << 17]byte +var z1223 [1 << 17]byte +var z1224 [1 << 17]byte +var z1225 [1 << 17]byte +var z1226 [1 << 17]byte +var z1227 [1 << 17]byte +var z1228 [1 << 17]byte +var z1229 [1 << 17]byte +var z1230 [1 << 17]byte +var z1231 [1 << 17]byte +var z1232 [1 << 17]byte +var z1233 [1 << 17]byte +var z1234 [1 << 17]byte +var z1235 [1 << 17]byte +var z1236 [1 << 17]byte +var z1237 [1 << 17]byte +var z1238 [1 << 17]byte +var z1239 [1 << 17]byte +var z1240 [1 << 17]byte +var z1241 [1 << 17]byte +var z1242 [1 << 17]byte +var z1243 [1 << 17]byte +var z1244 [1 << 17]byte +var z1245 [1 << 17]byte +var z1246 [1 << 17]byte +var z1247 [1 << 17]byte +var z1248 [1 << 17]byte +var z1249 [1 << 17]byte +var z1250 [1 << 17]byte +var z1251 [1 << 17]byte +var z1252 [1 << 17]byte +var z1253 [1 << 17]byte +var z1254 [1 << 17]byte +var z1255 [1 << 17]byte +var z1256 [1 << 17]byte +var z1257 [1 << 17]byte +var z1258 [1 << 17]byte +var z1259 [1 << 17]byte +var z1260 [1 << 17]byte +var z1261 [1 << 17]byte +var z1262 [1 << 17]byte +var z1263 [1 << 17]byte +var z1264 [1 << 17]byte +var z1265 [1 << 17]byte +var z1266 [1 << 17]byte +var z1267 [1 << 17]byte +var z1268 [1 << 17]byte +var z1269 [1 << 17]byte +var z1270 [1 << 17]byte +var z1271 [1 << 17]byte +var z1272 [1 << 17]byte +var z1273 [1 << 17]byte +var z1274 [1 << 17]byte +var z1275 [1 << 17]byte +var z1276 [1 << 17]byte +var z1277 [1 << 17]byte +var z1278 [1 << 17]byte +var z1279 [1 << 17]byte +var z1280 [1 << 17]byte +var z1281 [1 << 17]byte +var z1282 [1 << 17]byte +var z1283 [1 << 17]byte +var z1284 [1 << 17]byte +var z1285 [1 << 17]byte +var z1286 [1 << 17]byte +var z1287 [1 << 17]byte +var z1288 [1 << 17]byte +var z1289 [1 << 17]byte +var z1290 [1 << 17]byte +var z1291 [1 << 17]byte +var z1292 [1 << 17]byte +var z1293 [1 << 17]byte +var z1294 [1 << 17]byte +var z1295 [1 << 17]byte +var z1296 [1 << 17]byte +var z1297 [1 << 17]byte +var z1298 [1 << 17]byte +var z1299 [1 << 17]byte +var z1300 [1 << 17]byte +var z1301 [1 << 17]byte +var z1302 [1 << 17]byte +var z1303 [1 << 17]byte +var z1304 [1 << 17]byte +var z1305 [1 << 17]byte +var z1306 [1 << 17]byte +var z1307 [1 << 17]byte +var z1308 [1 << 17]byte +var z1309 [1 << 17]byte +var z1310 [1 << 17]byte +var z1311 [1 << 17]byte +var z1312 [1 << 17]byte +var z1313 [1 << 17]byte +var z1314 [1 << 17]byte +var z1315 [1 << 17]byte +var z1316 [1 << 17]byte +var z1317 [1 << 17]byte +var z1318 [1 << 17]byte +var z1319 [1 << 17]byte +var z1320 [1 << 17]byte +var z1321 [1 << 17]byte +var z1322 [1 << 17]byte +var z1323 [1 << 17]byte +var z1324 [1 << 17]byte +var z1325 [1 << 17]byte +var z1326 [1 << 17]byte +var z1327 [1 << 17]byte +var z1328 [1 << 17]byte +var z1329 [1 << 17]byte +var z1330 [1 << 17]byte +var z1331 [1 << 17]byte +var z1332 [1 << 17]byte +var z1333 [1 << 17]byte +var z1334 [1 << 17]byte +var z1335 [1 << 17]byte +var z1336 [1 << 17]byte +var z1337 [1 << 17]byte +var z1338 [1 << 17]byte +var z1339 [1 << 17]byte +var z1340 [1 << 17]byte +var z1341 [1 << 17]byte +var z1342 [1 << 17]byte +var z1343 [1 << 17]byte +var z1344 [1 << 17]byte +var z1345 [1 << 17]byte +var z1346 [1 << 17]byte +var z1347 [1 << 17]byte +var z1348 [1 << 17]byte +var z1349 [1 << 17]byte +var z1350 [1 << 17]byte +var z1351 [1 << 17]byte +var z1352 [1 << 17]byte +var z1353 [1 << 17]byte +var z1354 [1 << 17]byte +var z1355 [1 << 17]byte +var z1356 [1 << 17]byte +var z1357 [1 << 17]byte +var z1358 [1 << 17]byte +var z1359 [1 << 17]byte +var z1360 [1 << 17]byte +var z1361 [1 << 17]byte +var z1362 [1 << 17]byte +var z1363 [1 << 17]byte +var z1364 [1 << 17]byte +var z1365 [1 << 17]byte +var z1366 [1 << 17]byte +var z1367 [1 << 17]byte +var z1368 [1 << 17]byte +var z1369 [1 << 17]byte +var z1370 [1 << 17]byte +var z1371 [1 << 17]byte +var z1372 [1 << 17]byte +var z1373 [1 << 17]byte +var z1374 [1 << 17]byte +var z1375 [1 << 17]byte +var z1376 [1 << 17]byte +var z1377 [1 << 17]byte +var z1378 [1 << 17]byte +var z1379 [1 << 17]byte +var z1380 [1 << 17]byte +var z1381 [1 << 17]byte +var z1382 [1 << 17]byte +var z1383 [1 << 17]byte +var z1384 [1 << 17]byte +var z1385 [1 << 17]byte +var z1386 [1 << 17]byte +var z1387 [1 << 17]byte +var z1388 [1 << 17]byte +var z1389 [1 << 17]byte +var z1390 [1 << 17]byte +var z1391 [1 << 17]byte +var z1392 [1 << 17]byte +var z1393 [1 << 17]byte +var z1394 [1 << 17]byte +var z1395 [1 << 17]byte +var z1396 [1 << 17]byte +var z1397 [1 << 17]byte +var z1398 [1 << 17]byte +var z1399 [1 << 17]byte +var z1400 [1 << 17]byte +var z1401 [1 << 17]byte +var z1402 [1 << 17]byte +var z1403 [1 << 17]byte +var z1404 [1 << 17]byte +var z1405 [1 << 17]byte +var z1406 [1 << 17]byte +var z1407 [1 << 17]byte +var z1408 [1 << 17]byte +var z1409 [1 << 17]byte +var z1410 [1 << 17]byte +var z1411 [1 << 17]byte +var z1412 [1 << 17]byte +var z1413 [1 << 17]byte +var z1414 [1 << 17]byte +var z1415 [1 << 17]byte +var z1416 [1 << 17]byte +var z1417 [1 << 17]byte +var z1418 [1 << 17]byte +var z1419 [1 << 17]byte +var z1420 [1 << 17]byte +var z1421 [1 << 17]byte +var z1422 [1 << 17]byte +var z1423 [1 << 17]byte +var z1424 [1 << 17]byte +var z1425 [1 << 17]byte +var z1426 [1 << 17]byte +var z1427 [1 << 17]byte +var z1428 [1 << 17]byte +var z1429 [1 << 17]byte +var z1430 [1 << 17]byte +var z1431 [1 << 17]byte +var z1432 [1 << 17]byte +var z1433 [1 << 17]byte +var z1434 [1 << 17]byte +var z1435 [1 << 17]byte +var z1436 [1 << 17]byte +var z1437 [1 << 17]byte +var z1438 [1 << 17]byte +var z1439 [1 << 17]byte +var z1440 [1 << 17]byte +var z1441 [1 << 17]byte +var z1442 [1 << 17]byte +var z1443 [1 << 17]byte +var z1444 [1 << 17]byte +var z1445 [1 << 17]byte +var z1446 [1 << 17]byte +var z1447 [1 << 17]byte +var z1448 [1 << 17]byte +var z1449 [1 << 17]byte +var z1450 [1 << 17]byte +var z1451 [1 << 17]byte +var z1452 [1 << 17]byte +var z1453 [1 << 17]byte +var z1454 [1 << 17]byte +var z1455 [1 << 17]byte +var z1456 [1 << 17]byte +var z1457 [1 << 17]byte +var z1458 [1 << 17]byte +var z1459 [1 << 17]byte +var z1460 [1 << 17]byte +var z1461 [1 << 17]byte +var z1462 [1 << 17]byte +var z1463 [1 << 17]byte +var z1464 [1 << 17]byte +var z1465 [1 << 17]byte +var z1466 [1 << 17]byte +var z1467 [1 << 17]byte +var z1468 [1 << 17]byte +var z1469 [1 << 17]byte +var z1470 [1 << 17]byte +var z1471 [1 << 17]byte +var z1472 [1 << 17]byte +var z1473 [1 << 17]byte +var z1474 [1 << 17]byte +var z1475 [1 << 17]byte +var z1476 [1 << 17]byte +var z1477 [1 << 17]byte +var z1478 [1 << 17]byte +var z1479 [1 << 17]byte +var z1480 [1 << 17]byte +var z1481 [1 << 17]byte +var z1482 [1 << 17]byte +var z1483 [1 << 17]byte +var z1484 [1 << 17]byte +var z1485 [1 << 17]byte +var z1486 [1 << 17]byte +var z1487 [1 << 17]byte +var z1488 [1 << 17]byte +var z1489 [1 << 17]byte +var z1490 [1 << 17]byte +var z1491 [1 << 17]byte +var z1492 [1 << 17]byte +var z1493 [1 << 17]byte +var z1494 [1 << 17]byte +var z1495 [1 << 17]byte +var z1496 [1 << 17]byte +var z1497 [1 << 17]byte +var z1498 [1 << 17]byte +var z1499 [1 << 17]byte +var z1500 [1 << 17]byte +var z1501 [1 << 17]byte +var z1502 [1 << 17]byte +var z1503 [1 << 17]byte +var z1504 [1 << 17]byte +var z1505 [1 << 17]byte +var z1506 [1 << 17]byte +var z1507 [1 << 17]byte +var z1508 [1 << 17]byte +var z1509 [1 << 17]byte +var z1510 [1 << 17]byte +var z1511 [1 << 17]byte +var z1512 [1 << 17]byte +var z1513 [1 << 17]byte +var z1514 [1 << 17]byte +var z1515 [1 << 17]byte +var z1516 [1 << 17]byte +var z1517 [1 << 17]byte +var z1518 [1 << 17]byte +var z1519 [1 << 17]byte +var z1520 [1 << 17]byte +var z1521 [1 << 17]byte +var z1522 [1 << 17]byte +var z1523 [1 << 17]byte +var z1524 [1 << 17]byte +var z1525 [1 << 17]byte +var z1526 [1 << 17]byte +var z1527 [1 << 17]byte +var z1528 [1 << 17]byte +var z1529 [1 << 17]byte +var z1530 [1 << 17]byte +var z1531 [1 << 17]byte +var z1532 [1 << 17]byte +var z1533 [1 << 17]byte +var z1534 [1 << 17]byte +var z1535 [1 << 17]byte +var z1536 [1 << 17]byte +var z1537 [1 << 17]byte +var z1538 [1 << 17]byte +var z1539 [1 << 17]byte +var z1540 [1 << 17]byte +var z1541 [1 << 17]byte +var z1542 [1 << 17]byte +var z1543 [1 << 17]byte +var z1544 [1 << 17]byte +var z1545 [1 << 17]byte +var z1546 [1 << 17]byte +var z1547 [1 << 17]byte +var z1548 [1 << 17]byte +var z1549 [1 << 17]byte +var z1550 [1 << 17]byte +var z1551 [1 << 17]byte +var z1552 [1 << 17]byte +var z1553 [1 << 17]byte +var z1554 [1 << 17]byte +var z1555 [1 << 17]byte +var z1556 [1 << 17]byte +var z1557 [1 << 17]byte +var z1558 [1 << 17]byte +var z1559 [1 << 17]byte +var z1560 [1 << 17]byte +var z1561 [1 << 17]byte +var z1562 [1 << 17]byte +var z1563 [1 << 17]byte +var z1564 [1 << 17]byte +var z1565 [1 << 17]byte +var z1566 [1 << 17]byte +var z1567 [1 << 17]byte +var z1568 [1 << 17]byte +var z1569 [1 << 17]byte +var z1570 [1 << 17]byte +var z1571 [1 << 17]byte +var z1572 [1 << 17]byte +var z1573 [1 << 17]byte +var z1574 [1 << 17]byte +var z1575 [1 << 17]byte +var z1576 [1 << 17]byte +var z1577 [1 << 17]byte +var z1578 [1 << 17]byte +var z1579 [1 << 17]byte +var z1580 [1 << 17]byte +var z1581 [1 << 17]byte +var z1582 [1 << 17]byte +var z1583 [1 << 17]byte +var z1584 [1 << 17]byte +var z1585 [1 << 17]byte +var z1586 [1 << 17]byte +var z1587 [1 << 17]byte +var z1588 [1 << 17]byte +var z1589 [1 << 17]byte +var z1590 [1 << 17]byte +var z1591 [1 << 17]byte +var z1592 [1 << 17]byte +var z1593 [1 << 17]byte +var z1594 [1 << 17]byte +var z1595 [1 << 17]byte +var z1596 [1 << 17]byte +var z1597 [1 << 17]byte +var z1598 [1 << 17]byte +var z1599 [1 << 17]byte +var z1600 [1 << 17]byte +var z1601 [1 << 17]byte +var z1602 [1 << 17]byte +var z1603 [1 << 17]byte +var z1604 [1 << 17]byte +var z1605 [1 << 17]byte +var z1606 [1 << 17]byte +var z1607 [1 << 17]byte +var z1608 [1 << 17]byte +var z1609 [1 << 17]byte +var z1610 [1 << 17]byte +var z1611 [1 << 17]byte +var z1612 [1 << 17]byte +var z1613 [1 << 17]byte +var z1614 [1 << 17]byte +var z1615 [1 << 17]byte +var z1616 [1 << 17]byte +var z1617 [1 << 17]byte +var z1618 [1 << 17]byte +var z1619 [1 << 17]byte +var z1620 [1 << 17]byte +var z1621 [1 << 17]byte +var z1622 [1 << 17]byte +var z1623 [1 << 17]byte +var z1624 [1 << 17]byte +var z1625 [1 << 17]byte +var z1626 [1 << 17]byte +var z1627 [1 << 17]byte +var z1628 [1 << 17]byte +var z1629 [1 << 17]byte +var z1630 [1 << 17]byte +var z1631 [1 << 17]byte +var z1632 [1 << 17]byte +var z1633 [1 << 17]byte +var z1634 [1 << 17]byte +var z1635 [1 << 17]byte +var z1636 [1 << 17]byte +var z1637 [1 << 17]byte +var z1638 [1 << 17]byte +var z1639 [1 << 17]byte +var z1640 [1 << 17]byte +var z1641 [1 << 17]byte +var z1642 [1 << 17]byte +var z1643 [1 << 17]byte +var z1644 [1 << 17]byte +var z1645 [1 << 17]byte +var z1646 [1 << 17]byte +var z1647 [1 << 17]byte +var z1648 [1 << 17]byte +var z1649 [1 << 17]byte +var z1650 [1 << 17]byte +var z1651 [1 << 17]byte +var z1652 [1 << 17]byte +var z1653 [1 << 17]byte +var z1654 [1 << 17]byte +var z1655 [1 << 17]byte +var z1656 [1 << 17]byte +var z1657 [1 << 17]byte +var z1658 [1 << 17]byte +var z1659 [1 << 17]byte +var z1660 [1 << 17]byte +var z1661 [1 << 17]byte +var z1662 [1 << 17]byte +var z1663 [1 << 17]byte +var z1664 [1 << 17]byte +var z1665 [1 << 17]byte +var z1666 [1 << 17]byte +var z1667 [1 << 17]byte +var z1668 [1 << 17]byte +var z1669 [1 << 17]byte +var z1670 [1 << 17]byte +var z1671 [1 << 17]byte +var z1672 [1 << 17]byte +var z1673 [1 << 17]byte +var z1674 [1 << 17]byte +var z1675 [1 << 17]byte +var z1676 [1 << 17]byte +var z1677 [1 << 17]byte +var z1678 [1 << 17]byte +var z1679 [1 << 17]byte +var z1680 [1 << 17]byte +var z1681 [1 << 17]byte +var z1682 [1 << 17]byte +var z1683 [1 << 17]byte +var z1684 [1 << 17]byte +var z1685 [1 << 17]byte +var z1686 [1 << 17]byte +var z1687 [1 << 17]byte +var z1688 [1 << 17]byte +var z1689 [1 << 17]byte +var z1690 [1 << 17]byte +var z1691 [1 << 17]byte +var z1692 [1 << 17]byte +var z1693 [1 << 17]byte +var z1694 [1 << 17]byte +var z1695 [1 << 17]byte +var z1696 [1 << 17]byte +var z1697 [1 << 17]byte +var z1698 [1 << 17]byte +var z1699 [1 << 17]byte +var z1700 [1 << 17]byte +var z1701 [1 << 17]byte +var z1702 [1 << 17]byte +var z1703 [1 << 17]byte +var z1704 [1 << 17]byte +var z1705 [1 << 17]byte +var z1706 [1 << 17]byte +var z1707 [1 << 17]byte +var z1708 [1 << 17]byte +var z1709 [1 << 17]byte +var z1710 [1 << 17]byte +var z1711 [1 << 17]byte +var z1712 [1 << 17]byte +var z1713 [1 << 17]byte +var z1714 [1 << 17]byte +var z1715 [1 << 17]byte +var z1716 [1 << 17]byte +var z1717 [1 << 17]byte +var z1718 [1 << 17]byte +var z1719 [1 << 17]byte +var z1720 [1 << 17]byte +var z1721 [1 << 17]byte +var z1722 [1 << 17]byte +var z1723 [1 << 17]byte +var z1724 [1 << 17]byte +var z1725 [1 << 17]byte +var z1726 [1 << 17]byte +var z1727 [1 << 17]byte +var z1728 [1 << 17]byte +var z1729 [1 << 17]byte +var z1730 [1 << 17]byte +var z1731 [1 << 17]byte +var z1732 [1 << 17]byte +var z1733 [1 << 17]byte +var z1734 [1 << 17]byte +var z1735 [1 << 17]byte +var z1736 [1 << 17]byte +var z1737 [1 << 17]byte +var z1738 [1 << 17]byte +var z1739 [1 << 17]byte +var z1740 [1 << 17]byte +var z1741 [1 << 17]byte +var z1742 [1 << 17]byte +var z1743 [1 << 17]byte +var z1744 [1 << 17]byte +var z1745 [1 << 17]byte +var z1746 [1 << 17]byte +var z1747 [1 << 17]byte +var z1748 [1 << 17]byte +var z1749 [1 << 17]byte +var z1750 [1 << 17]byte +var z1751 [1 << 17]byte +var z1752 [1 << 17]byte +var z1753 [1 << 17]byte +var z1754 [1 << 17]byte +var z1755 [1 << 17]byte +var z1756 [1 << 17]byte +var z1757 [1 << 17]byte +var z1758 [1 << 17]byte +var z1759 [1 << 17]byte +var z1760 [1 << 17]byte +var z1761 [1 << 17]byte +var z1762 [1 << 17]byte +var z1763 [1 << 17]byte +var z1764 [1 << 17]byte +var z1765 [1 << 17]byte +var z1766 [1 << 17]byte +var z1767 [1 << 17]byte +var z1768 [1 << 17]byte +var z1769 [1 << 17]byte +var z1770 [1 << 17]byte +var z1771 [1 << 17]byte +var z1772 [1 << 17]byte +var z1773 [1 << 17]byte +var z1774 [1 << 17]byte +var z1775 [1 << 17]byte +var z1776 [1 << 17]byte +var z1777 [1 << 17]byte +var z1778 [1 << 17]byte +var z1779 [1 << 17]byte +var z1780 [1 << 17]byte +var z1781 [1 << 17]byte +var z1782 [1 << 17]byte +var z1783 [1 << 17]byte +var z1784 [1 << 17]byte +var z1785 [1 << 17]byte +var z1786 [1 << 17]byte +var z1787 [1 << 17]byte +var z1788 [1 << 17]byte +var z1789 [1 << 17]byte +var z1790 [1 << 17]byte +var z1791 [1 << 17]byte +var z1792 [1 << 17]byte +var z1793 [1 << 17]byte +var z1794 [1 << 17]byte +var z1795 [1 << 17]byte +var z1796 [1 << 17]byte +var z1797 [1 << 17]byte +var z1798 [1 << 17]byte +var z1799 [1 << 17]byte +var z1800 [1 << 17]byte +var z1801 [1 << 17]byte +var z1802 [1 << 17]byte +var z1803 [1 << 17]byte +var z1804 [1 << 17]byte +var z1805 [1 << 17]byte +var z1806 [1 << 17]byte +var z1807 [1 << 17]byte +var z1808 [1 << 17]byte +var z1809 [1 << 17]byte +var z1810 [1 << 17]byte +var z1811 [1 << 17]byte +var z1812 [1 << 17]byte +var z1813 [1 << 17]byte +var z1814 [1 << 17]byte +var z1815 [1 << 17]byte +var z1816 [1 << 17]byte +var z1817 [1 << 17]byte +var z1818 [1 << 17]byte +var z1819 [1 << 17]byte +var z1820 [1 << 17]byte +var z1821 [1 << 17]byte +var z1822 [1 << 17]byte +var z1823 [1 << 17]byte +var z1824 [1 << 17]byte +var z1825 [1 << 17]byte +var z1826 [1 << 17]byte +var z1827 [1 << 17]byte +var z1828 [1 << 17]byte +var z1829 [1 << 17]byte +var z1830 [1 << 17]byte +var z1831 [1 << 17]byte +var z1832 [1 << 17]byte +var z1833 [1 << 17]byte +var z1834 [1 << 17]byte +var z1835 [1 << 17]byte +var z1836 [1 << 17]byte +var z1837 [1 << 17]byte +var z1838 [1 << 17]byte +var z1839 [1 << 17]byte +var z1840 [1 << 17]byte +var z1841 [1 << 17]byte +var z1842 [1 << 17]byte +var z1843 [1 << 17]byte +var z1844 [1 << 17]byte +var z1845 [1 << 17]byte +var z1846 [1 << 17]byte +var z1847 [1 << 17]byte +var z1848 [1 << 17]byte +var z1849 [1 << 17]byte +var z1850 [1 << 17]byte +var z1851 [1 << 17]byte +var z1852 [1 << 17]byte +var z1853 [1 << 17]byte +var z1854 [1 << 17]byte +var z1855 [1 << 17]byte +var z1856 [1 << 17]byte +var z1857 [1 << 17]byte +var z1858 [1 << 17]byte +var z1859 [1 << 17]byte +var z1860 [1 << 17]byte +var z1861 [1 << 17]byte +var z1862 [1 << 17]byte +var z1863 [1 << 17]byte +var z1864 [1 << 17]byte +var z1865 [1 << 17]byte +var z1866 [1 << 17]byte +var z1867 [1 << 17]byte +var z1868 [1 << 17]byte +var z1869 [1 << 17]byte +var z1870 [1 << 17]byte +var z1871 [1 << 17]byte +var z1872 [1 << 17]byte +var z1873 [1 << 17]byte +var z1874 [1 << 17]byte +var z1875 [1 << 17]byte +var z1876 [1 << 17]byte +var z1877 [1 << 17]byte +var z1878 [1 << 17]byte +var z1879 [1 << 17]byte +var z1880 [1 << 17]byte +var z1881 [1 << 17]byte +var z1882 [1 << 17]byte +var z1883 [1 << 17]byte +var z1884 [1 << 17]byte +var z1885 [1 << 17]byte +var z1886 [1 << 17]byte +var z1887 [1 << 17]byte +var z1888 [1 << 17]byte +var z1889 [1 << 17]byte +var z1890 [1 << 17]byte +var z1891 [1 << 17]byte +var z1892 [1 << 17]byte +var z1893 [1 << 17]byte +var z1894 [1 << 17]byte +var z1895 [1 << 17]byte +var z1896 [1 << 17]byte +var z1897 [1 << 17]byte +var z1898 [1 << 17]byte +var z1899 [1 << 17]byte +var z1900 [1 << 17]byte +var z1901 [1 << 17]byte +var z1902 [1 << 17]byte +var z1903 [1 << 17]byte +var z1904 [1 << 17]byte +var z1905 [1 << 17]byte +var z1906 [1 << 17]byte +var z1907 [1 << 17]byte +var z1908 [1 << 17]byte +var z1909 [1 << 17]byte +var z1910 [1 << 17]byte +var z1911 [1 << 17]byte +var z1912 [1 << 17]byte +var z1913 [1 << 17]byte +var z1914 [1 << 17]byte +var z1915 [1 << 17]byte +var z1916 [1 << 17]byte +var z1917 [1 << 17]byte +var z1918 [1 << 17]byte +var z1919 [1 << 17]byte +var z1920 [1 << 17]byte +var z1921 [1 << 17]byte +var z1922 [1 << 17]byte +var z1923 [1 << 17]byte +var z1924 [1 << 17]byte +var z1925 [1 << 17]byte +var z1926 [1 << 17]byte +var z1927 [1 << 17]byte +var z1928 [1 << 17]byte +var z1929 [1 << 17]byte +var z1930 [1 << 17]byte +var z1931 [1 << 17]byte +var z1932 [1 << 17]byte +var z1933 [1 << 17]byte +var z1934 [1 << 17]byte +var z1935 [1 << 17]byte +var z1936 [1 << 17]byte +var z1937 [1 << 17]byte +var z1938 [1 << 17]byte +var z1939 [1 << 17]byte +var z1940 [1 << 17]byte +var z1941 [1 << 17]byte +var z1942 [1 << 17]byte +var z1943 [1 << 17]byte +var z1944 [1 << 17]byte +var z1945 [1 << 17]byte +var z1946 [1 << 17]byte +var z1947 [1 << 17]byte +var z1948 [1 << 17]byte +var z1949 [1 << 17]byte +var z1950 [1 << 17]byte +var z1951 [1 << 17]byte +var z1952 [1 << 17]byte +var z1953 [1 << 17]byte +var z1954 [1 << 17]byte +var z1955 [1 << 17]byte +var z1956 [1 << 17]byte +var z1957 [1 << 17]byte +var z1958 [1 << 17]byte +var z1959 [1 << 17]byte +var z1960 [1 << 17]byte +var z1961 [1 << 17]byte +var z1962 [1 << 17]byte +var z1963 [1 << 17]byte +var z1964 [1 << 17]byte +var z1965 [1 << 17]byte +var z1966 [1 << 17]byte +var z1967 [1 << 17]byte +var z1968 [1 << 17]byte +var z1969 [1 << 17]byte +var z1970 [1 << 17]byte +var z1971 [1 << 17]byte +var z1972 [1 << 17]byte +var z1973 [1 << 17]byte +var z1974 [1 << 17]byte +var z1975 [1 << 17]byte +var z1976 [1 << 17]byte +var z1977 [1 << 17]byte +var z1978 [1 << 17]byte +var z1979 [1 << 17]byte +var z1980 [1 << 17]byte +var z1981 [1 << 17]byte +var z1982 [1 << 17]byte +var z1983 [1 << 17]byte +var z1984 [1 << 17]byte +var z1985 [1 << 17]byte +var z1986 [1 << 17]byte +var z1987 [1 << 17]byte +var z1988 [1 << 17]byte +var z1989 [1 << 17]byte +var z1990 [1 << 17]byte +var z1991 [1 << 17]byte +var z1992 [1 << 17]byte +var z1993 [1 << 17]byte +var z1994 [1 << 17]byte +var z1995 [1 << 17]byte +var z1996 [1 << 17]byte +var z1997 [1 << 17]byte +var z1998 [1 << 17]byte +var z1999 [1 << 17]byte +var z2000 [1 << 17]byte +var z2001 [1 << 17]byte +var z2002 [1 << 17]byte +var z2003 [1 << 17]byte +var z2004 [1 << 17]byte +var z2005 [1 << 17]byte +var z2006 [1 << 17]byte +var z2007 [1 << 17]byte +var z2008 [1 << 17]byte +var z2009 [1 << 17]byte +var z2010 [1 << 17]byte +var z2011 [1 << 17]byte +var z2012 [1 << 17]byte +var z2013 [1 << 17]byte +var z2014 [1 << 17]byte +var z2015 [1 << 17]byte +var z2016 [1 << 17]byte +var z2017 [1 << 17]byte +var z2018 [1 << 17]byte +var z2019 [1 << 17]byte +var z2020 [1 << 17]byte +var z2021 [1 << 17]byte +var z2022 [1 << 17]byte +var z2023 [1 << 17]byte +var z2024 [1 << 17]byte +var z2025 [1 << 17]byte +var z2026 [1 << 17]byte +var z2027 [1 << 17]byte +var z2028 [1 << 17]byte +var z2029 [1 << 17]byte +var z2030 [1 << 17]byte +var z2031 [1 << 17]byte +var z2032 [1 << 17]byte +var z2033 [1 << 17]byte +var z2034 [1 << 17]byte +var z2035 [1 << 17]byte +var z2036 [1 << 17]byte +var z2037 [1 << 17]byte +var z2038 [1 << 17]byte +var z2039 [1 << 17]byte +var z2040 [1 << 17]byte +var z2041 [1 << 17]byte +var z2042 [1 << 17]byte +var z2043 [1 << 17]byte +var z2044 [1 << 17]byte +var z2045 [1 << 17]byte +var z2046 [1 << 17]byte +var z2047 [1 << 17]byte +var z2048 [1 << 17]byte +var z2049 [1 << 17]byte +var z2050 [1 << 17]byte +var z2051 [1 << 17]byte +var z2052 [1 << 17]byte +var z2053 [1 << 17]byte +var z2054 [1 << 17]byte +var z2055 [1 << 17]byte +var z2056 [1 << 17]byte +var z2057 [1 << 17]byte +var z2058 [1 << 17]byte +var z2059 [1 << 17]byte +var z2060 [1 << 17]byte +var z2061 [1 << 17]byte +var z2062 [1 << 17]byte +var z2063 [1 << 17]byte +var z2064 [1 << 17]byte +var z2065 [1 << 17]byte +var z2066 [1 << 17]byte +var z2067 [1 << 17]byte +var z2068 [1 << 17]byte +var z2069 [1 << 17]byte +var z2070 [1 << 17]byte +var z2071 [1 << 17]byte +var z2072 [1 << 17]byte +var z2073 [1 << 17]byte +var z2074 [1 << 17]byte +var z2075 [1 << 17]byte +var z2076 [1 << 17]byte +var z2077 [1 << 17]byte +var z2078 [1 << 17]byte +var z2079 [1 << 17]byte +var z2080 [1 << 17]byte +var z2081 [1 << 17]byte +var z2082 [1 << 17]byte +var z2083 [1 << 17]byte +var z2084 [1 << 17]byte +var z2085 [1 << 17]byte +var z2086 [1 << 17]byte +var z2087 [1 << 17]byte +var z2088 [1 << 17]byte +var z2089 [1 << 17]byte +var z2090 [1 << 17]byte +var z2091 [1 << 17]byte +var z2092 [1 << 17]byte +var z2093 [1 << 17]byte +var z2094 [1 << 17]byte +var z2095 [1 << 17]byte +var z2096 [1 << 17]byte +var z2097 [1 << 17]byte +var z2098 [1 << 17]byte +var z2099 [1 << 17]byte +var z2100 [1 << 17]byte +var z2101 [1 << 17]byte +var z2102 [1 << 17]byte +var z2103 [1 << 17]byte +var z2104 [1 << 17]byte +var z2105 [1 << 17]byte +var z2106 [1 << 17]byte +var z2107 [1 << 17]byte +var z2108 [1 << 17]byte +var z2109 [1 << 17]byte +var z2110 [1 << 17]byte +var z2111 [1 << 17]byte +var z2112 [1 << 17]byte +var z2113 [1 << 17]byte +var z2114 [1 << 17]byte +var z2115 [1 << 17]byte +var z2116 [1 << 17]byte +var z2117 [1 << 17]byte +var z2118 [1 << 17]byte +var z2119 [1 << 17]byte +var z2120 [1 << 17]byte +var z2121 [1 << 17]byte +var z2122 [1 << 17]byte +var z2123 [1 << 17]byte +var z2124 [1 << 17]byte +var z2125 [1 << 17]byte +var z2126 [1 << 17]byte +var z2127 [1 << 17]byte +var z2128 [1 << 17]byte +var z2129 [1 << 17]byte +var z2130 [1 << 17]byte +var z2131 [1 << 17]byte +var z2132 [1 << 17]byte +var z2133 [1 << 17]byte +var z2134 [1 << 17]byte +var z2135 [1 << 17]byte +var z2136 [1 << 17]byte +var z2137 [1 << 17]byte +var z2138 [1 << 17]byte +var z2139 [1 << 17]byte +var z2140 [1 << 17]byte +var z2141 [1 << 17]byte +var z2142 [1 << 17]byte +var z2143 [1 << 17]byte +var z2144 [1 << 17]byte +var z2145 [1 << 17]byte +var z2146 [1 << 17]byte +var z2147 [1 << 17]byte +var z2148 [1 << 17]byte +var z2149 [1 << 17]byte +var z2150 [1 << 17]byte +var z2151 [1 << 17]byte +var z2152 [1 << 17]byte +var z2153 [1 << 17]byte +var z2154 [1 << 17]byte +var z2155 [1 << 17]byte +var z2156 [1 << 17]byte +var z2157 [1 << 17]byte +var z2158 [1 << 17]byte +var z2159 [1 << 17]byte +var z2160 [1 << 17]byte +var z2161 [1 << 17]byte +var z2162 [1 << 17]byte +var z2163 [1 << 17]byte +var z2164 [1 << 17]byte +var z2165 [1 << 17]byte +var z2166 [1 << 17]byte +var z2167 [1 << 17]byte +var z2168 [1 << 17]byte +var z2169 [1 << 17]byte +var z2170 [1 << 17]byte +var z2171 [1 << 17]byte +var z2172 [1 << 17]byte +var z2173 [1 << 17]byte +var z2174 [1 << 17]byte +var z2175 [1 << 17]byte +var z2176 [1 << 17]byte +var z2177 [1 << 17]byte +var z2178 [1 << 17]byte +var z2179 [1 << 17]byte +var z2180 [1 << 17]byte +var z2181 [1 << 17]byte +var z2182 [1 << 17]byte +var z2183 [1 << 17]byte +var z2184 [1 << 17]byte +var z2185 [1 << 17]byte +var z2186 [1 << 17]byte +var z2187 [1 << 17]byte +var z2188 [1 << 17]byte +var z2189 [1 << 17]byte +var z2190 [1 << 17]byte +var z2191 [1 << 17]byte +var z2192 [1 << 17]byte +var z2193 [1 << 17]byte +var z2194 [1 << 17]byte +var z2195 [1 << 17]byte +var z2196 [1 << 17]byte +var z2197 [1 << 17]byte +var z2198 [1 << 17]byte +var z2199 [1 << 17]byte +var z2200 [1 << 17]byte +var z2201 [1 << 17]byte +var z2202 [1 << 17]byte +var z2203 [1 << 17]byte +var z2204 [1 << 17]byte +var z2205 [1 << 17]byte +var z2206 [1 << 17]byte +var z2207 [1 << 17]byte +var z2208 [1 << 17]byte +var z2209 [1 << 17]byte +var z2210 [1 << 17]byte +var z2211 [1 << 17]byte +var z2212 [1 << 17]byte +var z2213 [1 << 17]byte +var z2214 [1 << 17]byte +var z2215 [1 << 17]byte +var z2216 [1 << 17]byte +var z2217 [1 << 17]byte +var z2218 [1 << 17]byte +var z2219 [1 << 17]byte +var z2220 [1 << 17]byte +var z2221 [1 << 17]byte +var z2222 [1 << 17]byte +var z2223 [1 << 17]byte +var z2224 [1 << 17]byte +var z2225 [1 << 17]byte +var z2226 [1 << 17]byte +var z2227 [1 << 17]byte +var z2228 [1 << 17]byte +var z2229 [1 << 17]byte +var z2230 [1 << 17]byte +var z2231 [1 << 17]byte +var z2232 [1 << 17]byte +var z2233 [1 << 17]byte +var z2234 [1 << 17]byte +var z2235 [1 << 17]byte +var z2236 [1 << 17]byte +var z2237 [1 << 17]byte +var z2238 [1 << 17]byte +var z2239 [1 << 17]byte +var z2240 [1 << 17]byte +var z2241 [1 << 17]byte +var z2242 [1 << 17]byte +var z2243 [1 << 17]byte +var z2244 [1 << 17]byte +var z2245 [1 << 17]byte +var z2246 [1 << 17]byte +var z2247 [1 << 17]byte +var z2248 [1 << 17]byte +var z2249 [1 << 17]byte +var z2250 [1 << 17]byte +var z2251 [1 << 17]byte +var z2252 [1 << 17]byte +var z2253 [1 << 17]byte +var z2254 [1 << 17]byte +var z2255 [1 << 17]byte +var z2256 [1 << 17]byte +var z2257 [1 << 17]byte +var z2258 [1 << 17]byte +var z2259 [1 << 17]byte +var z2260 [1 << 17]byte +var z2261 [1 << 17]byte +var z2262 [1 << 17]byte +var z2263 [1 << 17]byte +var z2264 [1 << 17]byte +var z2265 [1 << 17]byte +var z2266 [1 << 17]byte +var z2267 [1 << 17]byte +var z2268 [1 << 17]byte +var z2269 [1 << 17]byte +var z2270 [1 << 17]byte +var z2271 [1 << 17]byte +var z2272 [1 << 17]byte +var z2273 [1 << 17]byte +var z2274 [1 << 17]byte +var z2275 [1 << 17]byte +var z2276 [1 << 17]byte +var z2277 [1 << 17]byte +var z2278 [1 << 17]byte +var z2279 [1 << 17]byte +var z2280 [1 << 17]byte +var z2281 [1 << 17]byte +var z2282 [1 << 17]byte +var z2283 [1 << 17]byte +var z2284 [1 << 17]byte +var z2285 [1 << 17]byte +var z2286 [1 << 17]byte +var z2287 [1 << 17]byte +var z2288 [1 << 17]byte +var z2289 [1 << 17]byte +var z2290 [1 << 17]byte +var z2291 [1 << 17]byte +var z2292 [1 << 17]byte +var z2293 [1 << 17]byte +var z2294 [1 << 17]byte +var z2295 [1 << 17]byte +var z2296 [1 << 17]byte +var z2297 [1 << 17]byte +var z2298 [1 << 17]byte +var z2299 [1 << 17]byte +var z2300 [1 << 17]byte +var z2301 [1 << 17]byte +var z2302 [1 << 17]byte +var z2303 [1 << 17]byte +var z2304 [1 << 17]byte +var z2305 [1 << 17]byte +var z2306 [1 << 17]byte +var z2307 [1 << 17]byte +var z2308 [1 << 17]byte +var z2309 [1 << 17]byte +var z2310 [1 << 17]byte +var z2311 [1 << 17]byte +var z2312 [1 << 17]byte +var z2313 [1 << 17]byte +var z2314 [1 << 17]byte +var z2315 [1 << 17]byte +var z2316 [1 << 17]byte +var z2317 [1 << 17]byte +var z2318 [1 << 17]byte +var z2319 [1 << 17]byte +var z2320 [1 << 17]byte +var z2321 [1 << 17]byte +var z2322 [1 << 17]byte +var z2323 [1 << 17]byte +var z2324 [1 << 17]byte +var z2325 [1 << 17]byte +var z2326 [1 << 17]byte +var z2327 [1 << 17]byte +var z2328 [1 << 17]byte +var z2329 [1 << 17]byte +var z2330 [1 << 17]byte +var z2331 [1 << 17]byte +var z2332 [1 << 17]byte +var z2333 [1 << 17]byte +var z2334 [1 << 17]byte +var z2335 [1 << 17]byte +var z2336 [1 << 17]byte +var z2337 [1 << 17]byte +var z2338 [1 << 17]byte +var z2339 [1 << 17]byte +var z2340 [1 << 17]byte +var z2341 [1 << 17]byte +var z2342 [1 << 17]byte +var z2343 [1 << 17]byte +var z2344 [1 << 17]byte +var z2345 [1 << 17]byte +var z2346 [1 << 17]byte +var z2347 [1 << 17]byte +var z2348 [1 << 17]byte +var z2349 [1 << 17]byte +var z2350 [1 << 17]byte +var z2351 [1 << 17]byte +var z2352 [1 << 17]byte +var z2353 [1 << 17]byte +var z2354 [1 << 17]byte +var z2355 [1 << 17]byte +var z2356 [1 << 17]byte +var z2357 [1 << 17]byte +var z2358 [1 << 17]byte +var z2359 [1 << 17]byte +var z2360 [1 << 17]byte +var z2361 [1 << 17]byte +var z2362 [1 << 17]byte +var z2363 [1 << 17]byte +var z2364 [1 << 17]byte +var z2365 [1 << 17]byte +var z2366 [1 << 17]byte +var z2367 [1 << 17]byte +var z2368 [1 << 17]byte +var z2369 [1 << 17]byte +var z2370 [1 << 17]byte +var z2371 [1 << 17]byte +var z2372 [1 << 17]byte +var z2373 [1 << 17]byte +var z2374 [1 << 17]byte +var z2375 [1 << 17]byte +var z2376 [1 << 17]byte +var z2377 [1 << 17]byte +var z2378 [1 << 17]byte +var z2379 [1 << 17]byte +var z2380 [1 << 17]byte +var z2381 [1 << 17]byte +var z2382 [1 << 17]byte +var z2383 [1 << 17]byte +var z2384 [1 << 17]byte +var z2385 [1 << 17]byte +var z2386 [1 << 17]byte +var z2387 [1 << 17]byte +var z2388 [1 << 17]byte +var z2389 [1 << 17]byte +var z2390 [1 << 17]byte +var z2391 [1 << 17]byte +var z2392 [1 << 17]byte +var z2393 [1 << 17]byte +var z2394 [1 << 17]byte +var z2395 [1 << 17]byte +var z2396 [1 << 17]byte +var z2397 [1 << 17]byte +var z2398 [1 << 17]byte +var z2399 [1 << 17]byte +var z2400 [1 << 17]byte +var z2401 [1 << 17]byte +var z2402 [1 << 17]byte +var z2403 [1 << 17]byte +var z2404 [1 << 17]byte +var z2405 [1 << 17]byte +var z2406 [1 << 17]byte +var z2407 [1 << 17]byte +var z2408 [1 << 17]byte +var z2409 [1 << 17]byte +var z2410 [1 << 17]byte +var z2411 [1 << 17]byte +var z2412 [1 << 17]byte +var z2413 [1 << 17]byte +var z2414 [1 << 17]byte +var z2415 [1 << 17]byte +var z2416 [1 << 17]byte +var z2417 [1 << 17]byte +var z2418 [1 << 17]byte +var z2419 [1 << 17]byte +var z2420 [1 << 17]byte +var z2421 [1 << 17]byte +var z2422 [1 << 17]byte +var z2423 [1 << 17]byte +var z2424 [1 << 17]byte +var z2425 [1 << 17]byte +var z2426 [1 << 17]byte +var z2427 [1 << 17]byte +var z2428 [1 << 17]byte +var z2429 [1 << 17]byte +var z2430 [1 << 17]byte +var z2431 [1 << 17]byte +var z2432 [1 << 17]byte +var z2433 [1 << 17]byte +var z2434 [1 << 17]byte +var z2435 [1 << 17]byte +var z2436 [1 << 17]byte +var z2437 [1 << 17]byte +var z2438 [1 << 17]byte +var z2439 [1 << 17]byte +var z2440 [1 << 17]byte +var z2441 [1 << 17]byte +var z2442 [1 << 17]byte +var z2443 [1 << 17]byte +var z2444 [1 << 17]byte +var z2445 [1 << 17]byte +var z2446 [1 << 17]byte +var z2447 [1 << 17]byte +var z2448 [1 << 17]byte +var z2449 [1 << 17]byte +var z2450 [1 << 17]byte +var z2451 [1 << 17]byte +var z2452 [1 << 17]byte +var z2453 [1 << 17]byte +var z2454 [1 << 17]byte +var z2455 [1 << 17]byte +var z2456 [1 << 17]byte +var z2457 [1 << 17]byte +var z2458 [1 << 17]byte +var z2459 [1 << 17]byte +var z2460 [1 << 17]byte +var z2461 [1 << 17]byte +var z2462 [1 << 17]byte +var z2463 [1 << 17]byte +var z2464 [1 << 17]byte +var z2465 [1 << 17]byte +var z2466 [1 << 17]byte +var z2467 [1 << 17]byte +var z2468 [1 << 17]byte +var z2469 [1 << 17]byte +var z2470 [1 << 17]byte +var z2471 [1 << 17]byte +var z2472 [1 << 17]byte +var z2473 [1 << 17]byte +var z2474 [1 << 17]byte +var z2475 [1 << 17]byte +var z2476 [1 << 17]byte +var z2477 [1 << 17]byte +var z2478 [1 << 17]byte +var z2479 [1 << 17]byte +var z2480 [1 << 17]byte +var z2481 [1 << 17]byte +var z2482 [1 << 17]byte +var z2483 [1 << 17]byte +var z2484 [1 << 17]byte +var z2485 [1 << 17]byte +var z2486 [1 << 17]byte +var z2487 [1 << 17]byte +var z2488 [1 << 17]byte +var z2489 [1 << 17]byte +var z2490 [1 << 17]byte +var z2491 [1 << 17]byte +var z2492 [1 << 17]byte +var z2493 [1 << 17]byte +var z2494 [1 << 17]byte +var z2495 [1 << 17]byte +var z2496 [1 << 17]byte +var z2497 [1 << 17]byte +var z2498 [1 << 17]byte +var z2499 [1 << 17]byte +var z2500 [1 << 17]byte +var z2501 [1 << 17]byte +var z2502 [1 << 17]byte +var z2503 [1 << 17]byte +var z2504 [1 << 17]byte +var z2505 [1 << 17]byte +var z2506 [1 << 17]byte +var z2507 [1 << 17]byte +var z2508 [1 << 17]byte +var z2509 [1 << 17]byte +var z2510 [1 << 17]byte +var z2511 [1 << 17]byte +var z2512 [1 << 17]byte +var z2513 [1 << 17]byte +var z2514 [1 << 17]byte +var z2515 [1 << 17]byte +var z2516 [1 << 17]byte +var z2517 [1 << 17]byte +var z2518 [1 << 17]byte +var z2519 [1 << 17]byte +var z2520 [1 << 17]byte +var z2521 [1 << 17]byte +var z2522 [1 << 17]byte +var z2523 [1 << 17]byte +var z2524 [1 << 17]byte +var z2525 [1 << 17]byte +var z2526 [1 << 17]byte +var z2527 [1 << 17]byte +var z2528 [1 << 17]byte +var z2529 [1 << 17]byte +var z2530 [1 << 17]byte +var z2531 [1 << 17]byte +var z2532 [1 << 17]byte +var z2533 [1 << 17]byte +var z2534 [1 << 17]byte +var z2535 [1 << 17]byte +var z2536 [1 << 17]byte +var z2537 [1 << 17]byte +var z2538 [1 << 17]byte +var z2539 [1 << 17]byte +var z2540 [1 << 17]byte +var z2541 [1 << 17]byte +var z2542 [1 << 17]byte +var z2543 [1 << 17]byte +var z2544 [1 << 17]byte +var z2545 [1 << 17]byte +var z2546 [1 << 17]byte +var z2547 [1 << 17]byte +var z2548 [1 << 17]byte +var z2549 [1 << 17]byte +var z2550 [1 << 17]byte +var z2551 [1 << 17]byte +var z2552 [1 << 17]byte +var z2553 [1 << 17]byte +var z2554 [1 << 17]byte +var z2555 [1 << 17]byte +var z2556 [1 << 17]byte +var z2557 [1 << 17]byte +var z2558 [1 << 17]byte +var z2559 [1 << 17]byte +var z2560 [1 << 17]byte +var z2561 [1 << 17]byte +var z2562 [1 << 17]byte +var z2563 [1 << 17]byte +var z2564 [1 << 17]byte +var z2565 [1 << 17]byte +var z2566 [1 << 17]byte +var z2567 [1 << 17]byte +var z2568 [1 << 17]byte +var z2569 [1 << 17]byte +var z2570 [1 << 17]byte +var z2571 [1 << 17]byte +var z2572 [1 << 17]byte +var z2573 [1 << 17]byte +var z2574 [1 << 17]byte +var z2575 [1 << 17]byte +var z2576 [1 << 17]byte +var z2577 [1 << 17]byte +var z2578 [1 << 17]byte +var z2579 [1 << 17]byte +var z2580 [1 << 17]byte +var z2581 [1 << 17]byte +var z2582 [1 << 17]byte +var z2583 [1 << 17]byte +var z2584 [1 << 17]byte +var z2585 [1 << 17]byte +var z2586 [1 << 17]byte +var z2587 [1 << 17]byte +var z2588 [1 << 17]byte +var z2589 [1 << 17]byte +var z2590 [1 << 17]byte +var z2591 [1 << 17]byte +var z2592 [1 << 17]byte +var z2593 [1 << 17]byte +var z2594 [1 << 17]byte +var z2595 [1 << 17]byte +var z2596 [1 << 17]byte +var z2597 [1 << 17]byte +var z2598 [1 << 17]byte +var z2599 [1 << 17]byte +var z2600 [1 << 17]byte +var z2601 [1 << 17]byte +var z2602 [1 << 17]byte +var z2603 [1 << 17]byte +var z2604 [1 << 17]byte +var z2605 [1 << 17]byte +var z2606 [1 << 17]byte +var z2607 [1 << 17]byte +var z2608 [1 << 17]byte +var z2609 [1 << 17]byte +var z2610 [1 << 17]byte +var z2611 [1 << 17]byte +var z2612 [1 << 17]byte +var z2613 [1 << 17]byte +var z2614 [1 << 17]byte +var z2615 [1 << 17]byte +var z2616 [1 << 17]byte +var z2617 [1 << 17]byte +var z2618 [1 << 17]byte +var z2619 [1 << 17]byte +var z2620 [1 << 17]byte +var z2621 [1 << 17]byte +var z2622 [1 << 17]byte +var z2623 [1 << 17]byte +var z2624 [1 << 17]byte +var z2625 [1 << 17]byte +var z2626 [1 << 17]byte +var z2627 [1 << 17]byte +var z2628 [1 << 17]byte +var z2629 [1 << 17]byte +var z2630 [1 << 17]byte +var z2631 [1 << 17]byte +var z2632 [1 << 17]byte +var z2633 [1 << 17]byte +var z2634 [1 << 17]byte +var z2635 [1 << 17]byte +var z2636 [1 << 17]byte +var z2637 [1 << 17]byte +var z2638 [1 << 17]byte +var z2639 [1 << 17]byte +var z2640 [1 << 17]byte +var z2641 [1 << 17]byte +var z2642 [1 << 17]byte +var z2643 [1 << 17]byte +var z2644 [1 << 17]byte +var z2645 [1 << 17]byte +var z2646 [1 << 17]byte +var z2647 [1 << 17]byte +var z2648 [1 << 17]byte +var z2649 [1 << 17]byte +var z2650 [1 << 17]byte +var z2651 [1 << 17]byte +var z2652 [1 << 17]byte +var z2653 [1 << 17]byte +var z2654 [1 << 17]byte +var z2655 [1 << 17]byte +var z2656 [1 << 17]byte +var z2657 [1 << 17]byte +var z2658 [1 << 17]byte +var z2659 [1 << 17]byte +var z2660 [1 << 17]byte +var z2661 [1 << 17]byte +var z2662 [1 << 17]byte +var z2663 [1 << 17]byte +var z2664 [1 << 17]byte +var z2665 [1 << 17]byte +var z2666 [1 << 17]byte +var z2667 [1 << 17]byte +var z2668 [1 << 17]byte +var z2669 [1 << 17]byte +var z2670 [1 << 17]byte +var z2671 [1 << 17]byte +var z2672 [1 << 17]byte +var z2673 [1 << 17]byte +var z2674 [1 << 17]byte +var z2675 [1 << 17]byte +var z2676 [1 << 17]byte +var z2677 [1 << 17]byte +var z2678 [1 << 17]byte +var z2679 [1 << 17]byte +var z2680 [1 << 17]byte +var z2681 [1 << 17]byte +var z2682 [1 << 17]byte +var z2683 [1 << 17]byte +var z2684 [1 << 17]byte +var z2685 [1 << 17]byte +var z2686 [1 << 17]byte +var z2687 [1 << 17]byte +var z2688 [1 << 17]byte +var z2689 [1 << 17]byte +var z2690 [1 << 17]byte +var z2691 [1 << 17]byte +var z2692 [1 << 17]byte +var z2693 [1 << 17]byte +var z2694 [1 << 17]byte +var z2695 [1 << 17]byte +var z2696 [1 << 17]byte +var z2697 [1 << 17]byte +var z2698 [1 << 17]byte +var z2699 [1 << 17]byte +var z2700 [1 << 17]byte +var z2701 [1 << 17]byte +var z2702 [1 << 17]byte +var z2703 [1 << 17]byte +var z2704 [1 << 17]byte +var z2705 [1 << 17]byte +var z2706 [1 << 17]byte +var z2707 [1 << 17]byte +var z2708 [1 << 17]byte +var z2709 [1 << 17]byte +var z2710 [1 << 17]byte +var z2711 [1 << 17]byte +var z2712 [1 << 17]byte +var z2713 [1 << 17]byte +var z2714 [1 << 17]byte +var z2715 [1 << 17]byte +var z2716 [1 << 17]byte +var z2717 [1 << 17]byte +var z2718 [1 << 17]byte +var z2719 [1 << 17]byte +var z2720 [1 << 17]byte +var z2721 [1 << 17]byte +var z2722 [1 << 17]byte +var z2723 [1 << 17]byte +var z2724 [1 << 17]byte +var z2725 [1 << 17]byte +var z2726 [1 << 17]byte +var z2727 [1 << 17]byte +var z2728 [1 << 17]byte +var z2729 [1 << 17]byte +var z2730 [1 << 17]byte +var z2731 [1 << 17]byte +var z2732 [1 << 17]byte +var z2733 [1 << 17]byte +var z2734 [1 << 17]byte +var z2735 [1 << 17]byte +var z2736 [1 << 17]byte +var z2737 [1 << 17]byte +var z2738 [1 << 17]byte +var z2739 [1 << 17]byte +var z2740 [1 << 17]byte +var z2741 [1 << 17]byte +var z2742 [1 << 17]byte +var z2743 [1 << 17]byte +var z2744 [1 << 17]byte +var z2745 [1 << 17]byte +var z2746 [1 << 17]byte +var z2747 [1 << 17]byte +var z2748 [1 << 17]byte +var z2749 [1 << 17]byte +var z2750 [1 << 17]byte +var z2751 [1 << 17]byte +var z2752 [1 << 17]byte +var z2753 [1 << 17]byte +var z2754 [1 << 17]byte +var z2755 [1 << 17]byte +var z2756 [1 << 17]byte +var z2757 [1 << 17]byte +var z2758 [1 << 17]byte +var z2759 [1 << 17]byte +var z2760 [1 << 17]byte +var z2761 [1 << 17]byte +var z2762 [1 << 17]byte +var z2763 [1 << 17]byte +var z2764 [1 << 17]byte +var z2765 [1 << 17]byte +var z2766 [1 << 17]byte +var z2767 [1 << 17]byte +var z2768 [1 << 17]byte +var z2769 [1 << 17]byte +var z2770 [1 << 17]byte +var z2771 [1 << 17]byte +var z2772 [1 << 17]byte +var z2773 [1 << 17]byte +var z2774 [1 << 17]byte +var z2775 [1 << 17]byte +var z2776 [1 << 17]byte +var z2777 [1 << 17]byte +var z2778 [1 << 17]byte +var z2779 [1 << 17]byte +var z2780 [1 << 17]byte +var z2781 [1 << 17]byte +var z2782 [1 << 17]byte +var z2783 [1 << 17]byte +var z2784 [1 << 17]byte +var z2785 [1 << 17]byte +var z2786 [1 << 17]byte +var z2787 [1 << 17]byte +var z2788 [1 << 17]byte +var z2789 [1 << 17]byte +var z2790 [1 << 17]byte +var z2791 [1 << 17]byte +var z2792 [1 << 17]byte +var z2793 [1 << 17]byte +var z2794 [1 << 17]byte +var z2795 [1 << 17]byte +var z2796 [1 << 17]byte +var z2797 [1 << 17]byte +var z2798 [1 << 17]byte +var z2799 [1 << 17]byte +var z2800 [1 << 17]byte +var z2801 [1 << 17]byte +var z2802 [1 << 17]byte +var z2803 [1 << 17]byte +var z2804 [1 << 17]byte +var z2805 [1 << 17]byte +var z2806 [1 << 17]byte +var z2807 [1 << 17]byte +var z2808 [1 << 17]byte +var z2809 [1 << 17]byte +var z2810 [1 << 17]byte +var z2811 [1 << 17]byte +var z2812 [1 << 17]byte +var z2813 [1 << 17]byte +var z2814 [1 << 17]byte +var z2815 [1 << 17]byte +var z2816 [1 << 17]byte +var z2817 [1 << 17]byte +var z2818 [1 << 17]byte +var z2819 [1 << 17]byte +var z2820 [1 << 17]byte +var z2821 [1 << 17]byte +var z2822 [1 << 17]byte +var z2823 [1 << 17]byte +var z2824 [1 << 17]byte +var z2825 [1 << 17]byte +var z2826 [1 << 17]byte +var z2827 [1 << 17]byte +var z2828 [1 << 17]byte +var z2829 [1 << 17]byte +var z2830 [1 << 17]byte +var z2831 [1 << 17]byte +var z2832 [1 << 17]byte +var z2833 [1 << 17]byte +var z2834 [1 << 17]byte +var z2835 [1 << 17]byte +var z2836 [1 << 17]byte +var z2837 [1 << 17]byte +var z2838 [1 << 17]byte +var z2839 [1 << 17]byte +var z2840 [1 << 17]byte +var z2841 [1 << 17]byte +var z2842 [1 << 17]byte +var z2843 [1 << 17]byte +var z2844 [1 << 17]byte +var z2845 [1 << 17]byte +var z2846 [1 << 17]byte +var z2847 [1 << 17]byte +var z2848 [1 << 17]byte +var z2849 [1 << 17]byte +var z2850 [1 << 17]byte +var z2851 [1 << 17]byte +var z2852 [1 << 17]byte +var z2853 [1 << 17]byte +var z2854 [1 << 17]byte +var z2855 [1 << 17]byte +var z2856 [1 << 17]byte +var z2857 [1 << 17]byte +var z2858 [1 << 17]byte +var z2859 [1 << 17]byte +var z2860 [1 << 17]byte +var z2861 [1 << 17]byte +var z2862 [1 << 17]byte +var z2863 [1 << 17]byte +var z2864 [1 << 17]byte +var z2865 [1 << 17]byte +var z2866 [1 << 17]byte +var z2867 [1 << 17]byte +var z2868 [1 << 17]byte +var z2869 [1 << 17]byte +var z2870 [1 << 17]byte +var z2871 [1 << 17]byte +var z2872 [1 << 17]byte +var z2873 [1 << 17]byte +var z2874 [1 << 17]byte +var z2875 [1 << 17]byte +var z2876 [1 << 17]byte +var z2877 [1 << 17]byte +var z2878 [1 << 17]byte +var z2879 [1 << 17]byte +var z2880 [1 << 17]byte +var z2881 [1 << 17]byte +var z2882 [1 << 17]byte +var z2883 [1 << 17]byte +var z2884 [1 << 17]byte +var z2885 [1 << 17]byte +var z2886 [1 << 17]byte +var z2887 [1 << 17]byte +var z2888 [1 << 17]byte +var z2889 [1 << 17]byte +var z2890 [1 << 17]byte +var z2891 [1 << 17]byte +var z2892 [1 << 17]byte +var z2893 [1 << 17]byte +var z2894 [1 << 17]byte +var z2895 [1 << 17]byte +var z2896 [1 << 17]byte +var z2897 [1 << 17]byte +var z2898 [1 << 17]byte +var z2899 [1 << 17]byte +var z2900 [1 << 17]byte +var z2901 [1 << 17]byte +var z2902 [1 << 17]byte +var z2903 [1 << 17]byte +var z2904 [1 << 17]byte +var z2905 [1 << 17]byte +var z2906 [1 << 17]byte +var z2907 [1 << 17]byte +var z2908 [1 << 17]byte +var z2909 [1 << 17]byte +var z2910 [1 << 17]byte +var z2911 [1 << 17]byte +var z2912 [1 << 17]byte +var z2913 [1 << 17]byte +var z2914 [1 << 17]byte +var z2915 [1 << 17]byte +var z2916 [1 << 17]byte +var z2917 [1 << 17]byte +var z2918 [1 << 17]byte +var z2919 [1 << 17]byte +var z2920 [1 << 17]byte +var z2921 [1 << 17]byte +var z2922 [1 << 17]byte +var z2923 [1 << 17]byte +var z2924 [1 << 17]byte +var z2925 [1 << 17]byte +var z2926 [1 << 17]byte +var z2927 [1 << 17]byte +var z2928 [1 << 17]byte +var z2929 [1 << 17]byte +var z2930 [1 << 17]byte +var z2931 [1 << 17]byte +var z2932 [1 << 17]byte +var z2933 [1 << 17]byte +var z2934 [1 << 17]byte +var z2935 [1 << 17]byte +var z2936 [1 << 17]byte +var z2937 [1 << 17]byte +var z2938 [1 << 17]byte +var z2939 [1 << 17]byte +var z2940 [1 << 17]byte +var z2941 [1 << 17]byte +var z2942 [1 << 17]byte +var z2943 [1 << 17]byte +var z2944 [1 << 17]byte +var z2945 [1 << 17]byte +var z2946 [1 << 17]byte +var z2947 [1 << 17]byte +var z2948 [1 << 17]byte +var z2949 [1 << 17]byte +var z2950 [1 << 17]byte +var z2951 [1 << 17]byte +var z2952 [1 << 17]byte +var z2953 [1 << 17]byte +var z2954 [1 << 17]byte +var z2955 [1 << 17]byte +var z2956 [1 << 17]byte +var z2957 [1 << 17]byte +var z2958 [1 << 17]byte +var z2959 [1 << 17]byte +var z2960 [1 << 17]byte +var z2961 [1 << 17]byte +var z2962 [1 << 17]byte +var z2963 [1 << 17]byte +var z2964 [1 << 17]byte +var z2965 [1 << 17]byte +var z2966 [1 << 17]byte +var z2967 [1 << 17]byte +var z2968 [1 << 17]byte +var z2969 [1 << 17]byte +var z2970 [1 << 17]byte +var z2971 [1 << 17]byte +var z2972 [1 << 17]byte +var z2973 [1 << 17]byte +var z2974 [1 << 17]byte +var z2975 [1 << 17]byte +var z2976 [1 << 17]byte +var z2977 [1 << 17]byte +var z2978 [1 << 17]byte +var z2979 [1 << 17]byte +var z2980 [1 << 17]byte +var z2981 [1 << 17]byte +var z2982 [1 << 17]byte +var z2983 [1 << 17]byte +var z2984 [1 << 17]byte +var z2985 [1 << 17]byte +var z2986 [1 << 17]byte +var z2987 [1 << 17]byte +var z2988 [1 << 17]byte +var z2989 [1 << 17]byte +var z2990 [1 << 17]byte +var z2991 [1 << 17]byte +var z2992 [1 << 17]byte +var z2993 [1 << 17]byte +var z2994 [1 << 17]byte +var z2995 [1 << 17]byte +var z2996 [1 << 17]byte +var z2997 [1 << 17]byte +var z2998 [1 << 17]byte +var z2999 [1 << 17]byte +var z3000 [1 << 17]byte +var z3001 [1 << 17]byte +var z3002 [1 << 17]byte +var z3003 [1 << 17]byte +var z3004 [1 << 17]byte +var z3005 [1 << 17]byte +var z3006 [1 << 17]byte +var z3007 [1 << 17]byte +var z3008 [1 << 17]byte +var z3009 [1 << 17]byte +var z3010 [1 << 17]byte +var z3011 [1 << 17]byte +var z3012 [1 << 17]byte +var z3013 [1 << 17]byte +var z3014 [1 << 17]byte +var z3015 [1 << 17]byte +var z3016 [1 << 17]byte +var z3017 [1 << 17]byte +var z3018 [1 << 17]byte +var z3019 [1 << 17]byte +var z3020 [1 << 17]byte +var z3021 [1 << 17]byte +var z3022 [1 << 17]byte +var z3023 [1 << 17]byte +var z3024 [1 << 17]byte +var z3025 [1 << 17]byte +var z3026 [1 << 17]byte +var z3027 [1 << 17]byte +var z3028 [1 << 17]byte +var z3029 [1 << 17]byte +var z3030 [1 << 17]byte +var z3031 [1 << 17]byte +var z3032 [1 << 17]byte +var z3033 [1 << 17]byte +var z3034 [1 << 17]byte +var z3035 [1 << 17]byte +var z3036 [1 << 17]byte +var z3037 [1 << 17]byte +var z3038 [1 << 17]byte +var z3039 [1 << 17]byte +var z3040 [1 << 17]byte +var z3041 [1 << 17]byte +var z3042 [1 << 17]byte +var z3043 [1 << 17]byte +var z3044 [1 << 17]byte +var z3045 [1 << 17]byte +var z3046 [1 << 17]byte +var z3047 [1 << 17]byte +var z3048 [1 << 17]byte +var z3049 [1 << 17]byte +var z3050 [1 << 17]byte +var z3051 [1 << 17]byte +var z3052 [1 << 17]byte +var z3053 [1 << 17]byte +var z3054 [1 << 17]byte +var z3055 [1 << 17]byte +var z3056 [1 << 17]byte +var z3057 [1 << 17]byte +var z3058 [1 << 17]byte +var z3059 [1 << 17]byte +var z3060 [1 << 17]byte +var z3061 [1 << 17]byte +var z3062 [1 << 17]byte +var z3063 [1 << 17]byte +var z3064 [1 << 17]byte +var z3065 [1 << 17]byte +var z3066 [1 << 17]byte +var z3067 [1 << 17]byte +var z3068 [1 << 17]byte +var z3069 [1 << 17]byte +var z3070 [1 << 17]byte +var z3071 [1 << 17]byte +var z3072 [1 << 17]byte +var z3073 [1 << 17]byte +var z3074 [1 << 17]byte +var z3075 [1 << 17]byte +var z3076 [1 << 17]byte +var z3077 [1 << 17]byte +var z3078 [1 << 17]byte +var z3079 [1 << 17]byte +var z3080 [1 << 17]byte +var z3081 [1 << 17]byte +var z3082 [1 << 17]byte +var z3083 [1 << 17]byte +var z3084 [1 << 17]byte +var z3085 [1 << 17]byte +var z3086 [1 << 17]byte +var z3087 [1 << 17]byte +var z3088 [1 << 17]byte +var z3089 [1 << 17]byte +var z3090 [1 << 17]byte +var z3091 [1 << 17]byte +var z3092 [1 << 17]byte +var z3093 [1 << 17]byte +var z3094 [1 << 17]byte +var z3095 [1 << 17]byte +var z3096 [1 << 17]byte +var z3097 [1 << 17]byte +var z3098 [1 << 17]byte +var z3099 [1 << 17]byte +var z3100 [1 << 17]byte +var z3101 [1 << 17]byte +var z3102 [1 << 17]byte +var z3103 [1 << 17]byte +var z3104 [1 << 17]byte +var z3105 [1 << 17]byte +var z3106 [1 << 17]byte +var z3107 [1 << 17]byte +var z3108 [1 << 17]byte +var z3109 [1 << 17]byte +var z3110 [1 << 17]byte +var z3111 [1 << 17]byte +var z3112 [1 << 17]byte +var z3113 [1 << 17]byte +var z3114 [1 << 17]byte +var z3115 [1 << 17]byte +var z3116 [1 << 17]byte +var z3117 [1 << 17]byte +var z3118 [1 << 17]byte +var z3119 [1 << 17]byte +var z3120 [1 << 17]byte +var z3121 [1 << 17]byte +var z3122 [1 << 17]byte +var z3123 [1 << 17]byte +var z3124 [1 << 17]byte +var z3125 [1 << 17]byte +var z3126 [1 << 17]byte +var z3127 [1 << 17]byte +var z3128 [1 << 17]byte +var z3129 [1 << 17]byte +var z3130 [1 << 17]byte +var z3131 [1 << 17]byte +var z3132 [1 << 17]byte +var z3133 [1 << 17]byte +var z3134 [1 << 17]byte +var z3135 [1 << 17]byte +var z3136 [1 << 17]byte +var z3137 [1 << 17]byte +var z3138 [1 << 17]byte +var z3139 [1 << 17]byte +var z3140 [1 << 17]byte +var z3141 [1 << 17]byte +var z3142 [1 << 17]byte +var z3143 [1 << 17]byte +var z3144 [1 << 17]byte +var z3145 [1 << 17]byte +var z3146 [1 << 17]byte +var z3147 [1 << 17]byte +var z3148 [1 << 17]byte +var z3149 [1 << 17]byte +var z3150 [1 << 17]byte +var z3151 [1 << 17]byte +var z3152 [1 << 17]byte +var z3153 [1 << 17]byte +var z3154 [1 << 17]byte +var z3155 [1 << 17]byte +var z3156 [1 << 17]byte +var z3157 [1 << 17]byte +var z3158 [1 << 17]byte +var z3159 [1 << 17]byte +var z3160 [1 << 17]byte +var z3161 [1 << 17]byte +var z3162 [1 << 17]byte +var z3163 [1 << 17]byte +var z3164 [1 << 17]byte +var z3165 [1 << 17]byte +var z3166 [1 << 17]byte +var z3167 [1 << 17]byte +var z3168 [1 << 17]byte +var z3169 [1 << 17]byte +var z3170 [1 << 17]byte +var z3171 [1 << 17]byte +var z3172 [1 << 17]byte +var z3173 [1 << 17]byte +var z3174 [1 << 17]byte +var z3175 [1 << 17]byte +var z3176 [1 << 17]byte +var z3177 [1 << 17]byte +var z3178 [1 << 17]byte +var z3179 [1 << 17]byte +var z3180 [1 << 17]byte +var z3181 [1 << 17]byte +var z3182 [1 << 17]byte +var z3183 [1 << 17]byte +var z3184 [1 << 17]byte +var z3185 [1 << 17]byte +var z3186 [1 << 17]byte +var z3187 [1 << 17]byte +var z3188 [1 << 17]byte +var z3189 [1 << 17]byte +var z3190 [1 << 17]byte +var z3191 [1 << 17]byte +var z3192 [1 << 17]byte +var z3193 [1 << 17]byte +var z3194 [1 << 17]byte +var z3195 [1 << 17]byte +var z3196 [1 << 17]byte +var z3197 [1 << 17]byte +var z3198 [1 << 17]byte +var z3199 [1 << 17]byte +var z3200 [1 << 17]byte +var z3201 [1 << 17]byte +var z3202 [1 << 17]byte +var z3203 [1 << 17]byte +var z3204 [1 << 17]byte +var z3205 [1 << 17]byte +var z3206 [1 << 17]byte +var z3207 [1 << 17]byte +var z3208 [1 << 17]byte +var z3209 [1 << 17]byte +var z3210 [1 << 17]byte +var z3211 [1 << 17]byte +var z3212 [1 << 17]byte +var z3213 [1 << 17]byte +var z3214 [1 << 17]byte +var z3215 [1 << 17]byte +var z3216 [1 << 17]byte +var z3217 [1 << 17]byte +var z3218 [1 << 17]byte +var z3219 [1 << 17]byte +var z3220 [1 << 17]byte +var z3221 [1 << 17]byte +var z3222 [1 << 17]byte +var z3223 [1 << 17]byte +var z3224 [1 << 17]byte +var z3225 [1 << 17]byte +var z3226 [1 << 17]byte +var z3227 [1 << 17]byte +var z3228 [1 << 17]byte +var z3229 [1 << 17]byte +var z3230 [1 << 17]byte +var z3231 [1 << 17]byte +var z3232 [1 << 17]byte +var z3233 [1 << 17]byte +var z3234 [1 << 17]byte +var z3235 [1 << 17]byte +var z3236 [1 << 17]byte +var z3237 [1 << 17]byte +var z3238 [1 << 17]byte +var z3239 [1 << 17]byte +var z3240 [1 << 17]byte +var z3241 [1 << 17]byte +var z3242 [1 << 17]byte +var z3243 [1 << 17]byte +var z3244 [1 << 17]byte +var z3245 [1 << 17]byte +var z3246 [1 << 17]byte +var z3247 [1 << 17]byte +var z3248 [1 << 17]byte +var z3249 [1 << 17]byte +var z3250 [1 << 17]byte +var z3251 [1 << 17]byte +var z3252 [1 << 17]byte +var z3253 [1 << 17]byte +var z3254 [1 << 17]byte +var z3255 [1 << 17]byte +var z3256 [1 << 17]byte +var z3257 [1 << 17]byte +var z3258 [1 << 17]byte +var z3259 [1 << 17]byte +var z3260 [1 << 17]byte +var z3261 [1 << 17]byte +var z3262 [1 << 17]byte +var z3263 [1 << 17]byte +var z3264 [1 << 17]byte +var z3265 [1 << 17]byte +var z3266 [1 << 17]byte +var z3267 [1 << 17]byte +var z3268 [1 << 17]byte +var z3269 [1 << 17]byte +var z3270 [1 << 17]byte +var z3271 [1 << 17]byte +var z3272 [1 << 17]byte +var z3273 [1 << 17]byte +var z3274 [1 << 17]byte +var z3275 [1 << 17]byte +var z3276 [1 << 17]byte +var z3277 [1 << 17]byte +var z3278 [1 << 17]byte +var z3279 [1 << 17]byte +var z3280 [1 << 17]byte +var z3281 [1 << 17]byte +var z3282 [1 << 17]byte +var z3283 [1 << 17]byte +var z3284 [1 << 17]byte +var z3285 [1 << 17]byte +var z3286 [1 << 17]byte +var z3287 [1 << 17]byte +var z3288 [1 << 17]byte +var z3289 [1 << 17]byte +var z3290 [1 << 17]byte +var z3291 [1 << 17]byte +var z3292 [1 << 17]byte +var z3293 [1 << 17]byte +var z3294 [1 << 17]byte +var z3295 [1 << 17]byte +var z3296 [1 << 17]byte +var z3297 [1 << 17]byte +var z3298 [1 << 17]byte +var z3299 [1 << 17]byte +var z3300 [1 << 17]byte +var z3301 [1 << 17]byte +var z3302 [1 << 17]byte +var z3303 [1 << 17]byte +var z3304 [1 << 17]byte +var z3305 [1 << 17]byte +var z3306 [1 << 17]byte +var z3307 [1 << 17]byte +var z3308 [1 << 17]byte +var z3309 [1 << 17]byte +var z3310 [1 << 17]byte +var z3311 [1 << 17]byte +var z3312 [1 << 17]byte +var z3313 [1 << 17]byte +var z3314 [1 << 17]byte +var z3315 [1 << 17]byte +var z3316 [1 << 17]byte +var z3317 [1 << 17]byte +var z3318 [1 << 17]byte +var z3319 [1 << 17]byte +var z3320 [1 << 17]byte +var z3321 [1 << 17]byte +var z3322 [1 << 17]byte +var z3323 [1 << 17]byte +var z3324 [1 << 17]byte +var z3325 [1 << 17]byte +var z3326 [1 << 17]byte +var z3327 [1 << 17]byte +var z3328 [1 << 17]byte +var z3329 [1 << 17]byte +var z3330 [1 << 17]byte +var z3331 [1 << 17]byte +var z3332 [1 << 17]byte +var z3333 [1 << 17]byte +var z3334 [1 << 17]byte +var z3335 [1 << 17]byte +var z3336 [1 << 17]byte +var z3337 [1 << 17]byte +var z3338 [1 << 17]byte +var z3339 [1 << 17]byte +var z3340 [1 << 17]byte +var z3341 [1 << 17]byte +var z3342 [1 << 17]byte +var z3343 [1 << 17]byte +var z3344 [1 << 17]byte +var z3345 [1 << 17]byte +var z3346 [1 << 17]byte +var z3347 [1 << 17]byte +var z3348 [1 << 17]byte +var z3349 [1 << 17]byte +var z3350 [1 << 17]byte +var z3351 [1 << 17]byte +var z3352 [1 << 17]byte +var z3353 [1 << 17]byte +var z3354 [1 << 17]byte +var z3355 [1 << 17]byte +var z3356 [1 << 17]byte +var z3357 [1 << 17]byte +var z3358 [1 << 17]byte +var z3359 [1 << 17]byte +var z3360 [1 << 17]byte +var z3361 [1 << 17]byte +var z3362 [1 << 17]byte +var z3363 [1 << 17]byte +var z3364 [1 << 17]byte +var z3365 [1 << 17]byte +var z3366 [1 << 17]byte +var z3367 [1 << 17]byte +var z3368 [1 << 17]byte +var z3369 [1 << 17]byte +var z3370 [1 << 17]byte +var z3371 [1 << 17]byte +var z3372 [1 << 17]byte +var z3373 [1 << 17]byte +var z3374 [1 << 17]byte +var z3375 [1 << 17]byte +var z3376 [1 << 17]byte +var z3377 [1 << 17]byte +var z3378 [1 << 17]byte +var z3379 [1 << 17]byte +var z3380 [1 << 17]byte +var z3381 [1 << 17]byte +var z3382 [1 << 17]byte +var z3383 [1 << 17]byte +var z3384 [1 << 17]byte +var z3385 [1 << 17]byte +var z3386 [1 << 17]byte +var z3387 [1 << 17]byte +var z3388 [1 << 17]byte +var z3389 [1 << 17]byte +var z3390 [1 << 17]byte +var z3391 [1 << 17]byte +var z3392 [1 << 17]byte +var z3393 [1 << 17]byte +var z3394 [1 << 17]byte +var z3395 [1 << 17]byte +var z3396 [1 << 17]byte +var z3397 [1 << 17]byte +var z3398 [1 << 17]byte +var z3399 [1 << 17]byte +var z3400 [1 << 17]byte +var z3401 [1 << 17]byte +var z3402 [1 << 17]byte +var z3403 [1 << 17]byte +var z3404 [1 << 17]byte +var z3405 [1 << 17]byte +var z3406 [1 << 17]byte +var z3407 [1 << 17]byte +var z3408 [1 << 17]byte +var z3409 [1 << 17]byte +var z3410 [1 << 17]byte +var z3411 [1 << 17]byte +var z3412 [1 << 17]byte +var z3413 [1 << 17]byte +var z3414 [1 << 17]byte +var z3415 [1 << 17]byte +var z3416 [1 << 17]byte +var z3417 [1 << 17]byte +var z3418 [1 << 17]byte +var z3419 [1 << 17]byte +var z3420 [1 << 17]byte +var z3421 [1 << 17]byte +var z3422 [1 << 17]byte +var z3423 [1 << 17]byte +var z3424 [1 << 17]byte +var z3425 [1 << 17]byte +var z3426 [1 << 17]byte +var z3427 [1 << 17]byte +var z3428 [1 << 17]byte +var z3429 [1 << 17]byte +var z3430 [1 << 17]byte +var z3431 [1 << 17]byte +var z3432 [1 << 17]byte +var z3433 [1 << 17]byte +var z3434 [1 << 17]byte +var z3435 [1 << 17]byte +var z3436 [1 << 17]byte +var z3437 [1 << 17]byte +var z3438 [1 << 17]byte +var z3439 [1 << 17]byte +var z3440 [1 << 17]byte +var z3441 [1 << 17]byte +var z3442 [1 << 17]byte +var z3443 [1 << 17]byte +var z3444 [1 << 17]byte +var z3445 [1 << 17]byte +var z3446 [1 << 17]byte +var z3447 [1 << 17]byte +var z3448 [1 << 17]byte +var z3449 [1 << 17]byte +var z3450 [1 << 17]byte +var z3451 [1 << 17]byte +var z3452 [1 << 17]byte +var z3453 [1 << 17]byte +var z3454 [1 << 17]byte +var z3455 [1 << 17]byte +var z3456 [1 << 17]byte +var z3457 [1 << 17]byte +var z3458 [1 << 17]byte +var z3459 [1 << 17]byte +var z3460 [1 << 17]byte +var z3461 [1 << 17]byte +var z3462 [1 << 17]byte +var z3463 [1 << 17]byte +var z3464 [1 << 17]byte +var z3465 [1 << 17]byte +var z3466 [1 << 17]byte +var z3467 [1 << 17]byte +var z3468 [1 << 17]byte +var z3469 [1 << 17]byte +var z3470 [1 << 17]byte +var z3471 [1 << 17]byte +var z3472 [1 << 17]byte +var z3473 [1 << 17]byte +var z3474 [1 << 17]byte +var z3475 [1 << 17]byte +var z3476 [1 << 17]byte +var z3477 [1 << 17]byte +var z3478 [1 << 17]byte +var z3479 [1 << 17]byte +var z3480 [1 << 17]byte +var z3481 [1 << 17]byte +var z3482 [1 << 17]byte +var z3483 [1 << 17]byte +var z3484 [1 << 17]byte +var z3485 [1 << 17]byte +var z3486 [1 << 17]byte +var z3487 [1 << 17]byte +var z3488 [1 << 17]byte +var z3489 [1 << 17]byte +var z3490 [1 << 17]byte +var z3491 [1 << 17]byte +var z3492 [1 << 17]byte +var z3493 [1 << 17]byte +var z3494 [1 << 17]byte +var z3495 [1 << 17]byte +var z3496 [1 << 17]byte +var z3497 [1 << 17]byte +var z3498 [1 << 17]byte +var z3499 [1 << 17]byte +var z3500 [1 << 17]byte +var z3501 [1 << 17]byte +var z3502 [1 << 17]byte +var z3503 [1 << 17]byte +var z3504 [1 << 17]byte +var z3505 [1 << 17]byte +var z3506 [1 << 17]byte +var z3507 [1 << 17]byte +var z3508 [1 << 17]byte +var z3509 [1 << 17]byte +var z3510 [1 << 17]byte +var z3511 [1 << 17]byte +var z3512 [1 << 17]byte +var z3513 [1 << 17]byte +var z3514 [1 << 17]byte +var z3515 [1 << 17]byte +var z3516 [1 << 17]byte +var z3517 [1 << 17]byte +var z3518 [1 << 17]byte +var z3519 [1 << 17]byte +var z3520 [1 << 17]byte +var z3521 [1 << 17]byte +var z3522 [1 << 17]byte +var z3523 [1 << 17]byte +var z3524 [1 << 17]byte +var z3525 [1 << 17]byte +var z3526 [1 << 17]byte +var z3527 [1 << 17]byte +var z3528 [1 << 17]byte +var z3529 [1 << 17]byte +var z3530 [1 << 17]byte +var z3531 [1 << 17]byte +var z3532 [1 << 17]byte +var z3533 [1 << 17]byte +var z3534 [1 << 17]byte +var z3535 [1 << 17]byte +var z3536 [1 << 17]byte +var z3537 [1 << 17]byte +var z3538 [1 << 17]byte +var z3539 [1 << 17]byte +var z3540 [1 << 17]byte +var z3541 [1 << 17]byte +var z3542 [1 << 17]byte +var z3543 [1 << 17]byte +var z3544 [1 << 17]byte +var z3545 [1 << 17]byte +var z3546 [1 << 17]byte +var z3547 [1 << 17]byte +var z3548 [1 << 17]byte +var z3549 [1 << 17]byte +var z3550 [1 << 17]byte +var z3551 [1 << 17]byte +var z3552 [1 << 17]byte +var z3553 [1 << 17]byte +var z3554 [1 << 17]byte +var z3555 [1 << 17]byte +var z3556 [1 << 17]byte +var z3557 [1 << 17]byte +var z3558 [1 << 17]byte +var z3559 [1 << 17]byte +var z3560 [1 << 17]byte +var z3561 [1 << 17]byte +var z3562 [1 << 17]byte +var z3563 [1 << 17]byte +var z3564 [1 << 17]byte +var z3565 [1 << 17]byte +var z3566 [1 << 17]byte +var z3567 [1 << 17]byte +var z3568 [1 << 17]byte +var z3569 [1 << 17]byte +var z3570 [1 << 17]byte +var z3571 [1 << 17]byte +var z3572 [1 << 17]byte +var z3573 [1 << 17]byte +var z3574 [1 << 17]byte +var z3575 [1 << 17]byte +var z3576 [1 << 17]byte +var z3577 [1 << 17]byte +var z3578 [1 << 17]byte +var z3579 [1 << 17]byte +var z3580 [1 << 17]byte +var z3581 [1 << 17]byte +var z3582 [1 << 17]byte +var z3583 [1 << 17]byte +var z3584 [1 << 17]byte +var z3585 [1 << 17]byte +var z3586 [1 << 17]byte +var z3587 [1 << 17]byte +var z3588 [1 << 17]byte +var z3589 [1 << 17]byte +var z3590 [1 << 17]byte +var z3591 [1 << 17]byte +var z3592 [1 << 17]byte +var z3593 [1 << 17]byte +var z3594 [1 << 17]byte +var z3595 [1 << 17]byte +var z3596 [1 << 17]byte +var z3597 [1 << 17]byte +var z3598 [1 << 17]byte +var z3599 [1 << 17]byte +var z3600 [1 << 17]byte +var z3601 [1 << 17]byte +var z3602 [1 << 17]byte +var z3603 [1 << 17]byte +var z3604 [1 << 17]byte +var z3605 [1 << 17]byte +var z3606 [1 << 17]byte +var z3607 [1 << 17]byte +var z3608 [1 << 17]byte +var z3609 [1 << 17]byte +var z3610 [1 << 17]byte +var z3611 [1 << 17]byte +var z3612 [1 << 17]byte +var z3613 [1 << 17]byte +var z3614 [1 << 17]byte +var z3615 [1 << 17]byte +var z3616 [1 << 17]byte +var z3617 [1 << 17]byte +var z3618 [1 << 17]byte +var z3619 [1 << 17]byte +var z3620 [1 << 17]byte +var z3621 [1 << 17]byte +var z3622 [1 << 17]byte +var z3623 [1 << 17]byte +var z3624 [1 << 17]byte +var z3625 [1 << 17]byte +var z3626 [1 << 17]byte +var z3627 [1 << 17]byte +var z3628 [1 << 17]byte +var z3629 [1 << 17]byte +var z3630 [1 << 17]byte +var z3631 [1 << 17]byte +var z3632 [1 << 17]byte +var z3633 [1 << 17]byte +var z3634 [1 << 17]byte +var z3635 [1 << 17]byte +var z3636 [1 << 17]byte +var z3637 [1 << 17]byte +var z3638 [1 << 17]byte +var z3639 [1 << 17]byte +var z3640 [1 << 17]byte +var z3641 [1 << 17]byte +var z3642 [1 << 17]byte +var z3643 [1 << 17]byte +var z3644 [1 << 17]byte +var z3645 [1 << 17]byte +var z3646 [1 << 17]byte +var z3647 [1 << 17]byte +var z3648 [1 << 17]byte +var z3649 [1 << 17]byte +var z3650 [1 << 17]byte +var z3651 [1 << 17]byte +var z3652 [1 << 17]byte +var z3653 [1 << 17]byte +var z3654 [1 << 17]byte +var z3655 [1 << 17]byte +var z3656 [1 << 17]byte +var z3657 [1 << 17]byte +var z3658 [1 << 17]byte +var z3659 [1 << 17]byte +var z3660 [1 << 17]byte +var z3661 [1 << 17]byte +var z3662 [1 << 17]byte +var z3663 [1 << 17]byte +var z3664 [1 << 17]byte +var z3665 [1 << 17]byte +var z3666 [1 << 17]byte +var z3667 [1 << 17]byte +var z3668 [1 << 17]byte +var z3669 [1 << 17]byte +var z3670 [1 << 17]byte +var z3671 [1 << 17]byte +var z3672 [1 << 17]byte +var z3673 [1 << 17]byte +var z3674 [1 << 17]byte +var z3675 [1 << 17]byte +var z3676 [1 << 17]byte +var z3677 [1 << 17]byte +var z3678 [1 << 17]byte +var z3679 [1 << 17]byte +var z3680 [1 << 17]byte +var z3681 [1 << 17]byte +var z3682 [1 << 17]byte +var z3683 [1 << 17]byte +var z3684 [1 << 17]byte +var z3685 [1 << 17]byte +var z3686 [1 << 17]byte +var z3687 [1 << 17]byte +var z3688 [1 << 17]byte +var z3689 [1 << 17]byte +var z3690 [1 << 17]byte +var z3691 [1 << 17]byte +var z3692 [1 << 17]byte +var z3693 [1 << 17]byte +var z3694 [1 << 17]byte +var z3695 [1 << 17]byte +var z3696 [1 << 17]byte +var z3697 [1 << 17]byte +var z3698 [1 << 17]byte +var z3699 [1 << 17]byte +var z3700 [1 << 17]byte +var z3701 [1 << 17]byte +var z3702 [1 << 17]byte +var z3703 [1 << 17]byte +var z3704 [1 << 17]byte +var z3705 [1 << 17]byte +var z3706 [1 << 17]byte +var z3707 [1 << 17]byte +var z3708 [1 << 17]byte +var z3709 [1 << 17]byte +var z3710 [1 << 17]byte +var z3711 [1 << 17]byte +var z3712 [1 << 17]byte +var z3713 [1 << 17]byte +var z3714 [1 << 17]byte +var z3715 [1 << 17]byte +var z3716 [1 << 17]byte +var z3717 [1 << 17]byte +var z3718 [1 << 17]byte +var z3719 [1 << 17]byte +var z3720 [1 << 17]byte +var z3721 [1 << 17]byte +var z3722 [1 << 17]byte +var z3723 [1 << 17]byte +var z3724 [1 << 17]byte +var z3725 [1 << 17]byte +var z3726 [1 << 17]byte +var z3727 [1 << 17]byte +var z3728 [1 << 17]byte +var z3729 [1 << 17]byte +var z3730 [1 << 17]byte +var z3731 [1 << 17]byte +var z3732 [1 << 17]byte +var z3733 [1 << 17]byte +var z3734 [1 << 17]byte +var z3735 [1 << 17]byte +var z3736 [1 << 17]byte +var z3737 [1 << 17]byte +var z3738 [1 << 17]byte +var z3739 [1 << 17]byte +var z3740 [1 << 17]byte +var z3741 [1 << 17]byte +var z3742 [1 << 17]byte +var z3743 [1 << 17]byte +var z3744 [1 << 17]byte +var z3745 [1 << 17]byte +var z3746 [1 << 17]byte +var z3747 [1 << 17]byte +var z3748 [1 << 17]byte +var z3749 [1 << 17]byte +var z3750 [1 << 17]byte +var z3751 [1 << 17]byte +var z3752 [1 << 17]byte +var z3753 [1 << 17]byte +var z3754 [1 << 17]byte +var z3755 [1 << 17]byte +var z3756 [1 << 17]byte +var z3757 [1 << 17]byte +var z3758 [1 << 17]byte +var z3759 [1 << 17]byte +var z3760 [1 << 17]byte +var z3761 [1 << 17]byte +var z3762 [1 << 17]byte +var z3763 [1 << 17]byte +var z3764 [1 << 17]byte +var z3765 [1 << 17]byte +var z3766 [1 << 17]byte +var z3767 [1 << 17]byte +var z3768 [1 << 17]byte +var z3769 [1 << 17]byte +var z3770 [1 << 17]byte +var z3771 [1 << 17]byte +var z3772 [1 << 17]byte +var z3773 [1 << 17]byte +var z3774 [1 << 17]byte +var z3775 [1 << 17]byte +var z3776 [1 << 17]byte +var z3777 [1 << 17]byte +var z3778 [1 << 17]byte +var z3779 [1 << 17]byte +var z3780 [1 << 17]byte +var z3781 [1 << 17]byte +var z3782 [1 << 17]byte +var z3783 [1 << 17]byte +var z3784 [1 << 17]byte +var z3785 [1 << 17]byte +var z3786 [1 << 17]byte +var z3787 [1 << 17]byte +var z3788 [1 << 17]byte +var z3789 [1 << 17]byte +var z3790 [1 << 17]byte +var z3791 [1 << 17]byte +var z3792 [1 << 17]byte +var z3793 [1 << 17]byte +var z3794 [1 << 17]byte +var z3795 [1 << 17]byte +var z3796 [1 << 17]byte +var z3797 [1 << 17]byte +var z3798 [1 << 17]byte +var z3799 [1 << 17]byte +var z3800 [1 << 17]byte +var z3801 [1 << 17]byte +var z3802 [1 << 17]byte +var z3803 [1 << 17]byte +var z3804 [1 << 17]byte +var z3805 [1 << 17]byte +var z3806 [1 << 17]byte +var z3807 [1 << 17]byte +var z3808 [1 << 17]byte +var z3809 [1 << 17]byte +var z3810 [1 << 17]byte +var z3811 [1 << 17]byte +var z3812 [1 << 17]byte +var z3813 [1 << 17]byte +var z3814 [1 << 17]byte +var z3815 [1 << 17]byte +var z3816 [1 << 17]byte +var z3817 [1 << 17]byte +var z3818 [1 << 17]byte +var z3819 [1 << 17]byte +var z3820 [1 << 17]byte +var z3821 [1 << 17]byte +var z3822 [1 << 17]byte +var z3823 [1 << 17]byte +var z3824 [1 << 17]byte +var z3825 [1 << 17]byte +var z3826 [1 << 17]byte +var z3827 [1 << 17]byte +var z3828 [1 << 17]byte +var z3829 [1 << 17]byte +var z3830 [1 << 17]byte +var z3831 [1 << 17]byte +var z3832 [1 << 17]byte +var z3833 [1 << 17]byte +var z3834 [1 << 17]byte +var z3835 [1 << 17]byte +var z3836 [1 << 17]byte +var z3837 [1 << 17]byte +var z3838 [1 << 17]byte +var z3839 [1 << 17]byte +var z3840 [1 << 17]byte +var z3841 [1 << 17]byte +var z3842 [1 << 17]byte +var z3843 [1 << 17]byte +var z3844 [1 << 17]byte +var z3845 [1 << 17]byte +var z3846 [1 << 17]byte +var z3847 [1 << 17]byte +var z3848 [1 << 17]byte +var z3849 [1 << 17]byte +var z3850 [1 << 17]byte +var z3851 [1 << 17]byte +var z3852 [1 << 17]byte +var z3853 [1 << 17]byte +var z3854 [1 << 17]byte +var z3855 [1 << 17]byte +var z3856 [1 << 17]byte +var z3857 [1 << 17]byte +var z3858 [1 << 17]byte +var z3859 [1 << 17]byte +var z3860 [1 << 17]byte +var z3861 [1 << 17]byte +var z3862 [1 << 17]byte +var z3863 [1 << 17]byte +var z3864 [1 << 17]byte +var z3865 [1 << 17]byte +var z3866 [1 << 17]byte +var z3867 [1 << 17]byte +var z3868 [1 << 17]byte +var z3869 [1 << 17]byte +var z3870 [1 << 17]byte +var z3871 [1 << 17]byte +var z3872 [1 << 17]byte +var z3873 [1 << 17]byte +var z3874 [1 << 17]byte +var z3875 [1 << 17]byte +var z3876 [1 << 17]byte +var z3877 [1 << 17]byte +var z3878 [1 << 17]byte +var z3879 [1 << 17]byte +var z3880 [1 << 17]byte +var z3881 [1 << 17]byte +var z3882 [1 << 17]byte +var z3883 [1 << 17]byte +var z3884 [1 << 17]byte +var z3885 [1 << 17]byte +var z3886 [1 << 17]byte +var z3887 [1 << 17]byte +var z3888 [1 << 17]byte +var z3889 [1 << 17]byte +var z3890 [1 << 17]byte +var z3891 [1 << 17]byte +var z3892 [1 << 17]byte +var z3893 [1 << 17]byte +var z3894 [1 << 17]byte +var z3895 [1 << 17]byte +var z3896 [1 << 17]byte +var z3897 [1 << 17]byte +var z3898 [1 << 17]byte +var z3899 [1 << 17]byte +var z3900 [1 << 17]byte +var z3901 [1 << 17]byte +var z3902 [1 << 17]byte +var z3903 [1 << 17]byte +var z3904 [1 << 17]byte +var z3905 [1 << 17]byte +var z3906 [1 << 17]byte +var z3907 [1 << 17]byte +var z3908 [1 << 17]byte +var z3909 [1 << 17]byte +var z3910 [1 << 17]byte +var z3911 [1 << 17]byte +var z3912 [1 << 17]byte +var z3913 [1 << 17]byte +var z3914 [1 << 17]byte +var z3915 [1 << 17]byte +var z3916 [1 << 17]byte +var z3917 [1 << 17]byte +var z3918 [1 << 17]byte +var z3919 [1 << 17]byte +var z3920 [1 << 17]byte +var z3921 [1 << 17]byte +var z3922 [1 << 17]byte +var z3923 [1 << 17]byte +var z3924 [1 << 17]byte +var z3925 [1 << 17]byte +var z3926 [1 << 17]byte +var z3927 [1 << 17]byte +var z3928 [1 << 17]byte +var z3929 [1 << 17]byte +var z3930 [1 << 17]byte +var z3931 [1 << 17]byte +var z3932 [1 << 17]byte +var z3933 [1 << 17]byte +var z3934 [1 << 17]byte +var z3935 [1 << 17]byte +var z3936 [1 << 17]byte +var z3937 [1 << 17]byte +var z3938 [1 << 17]byte +var z3939 [1 << 17]byte +var z3940 [1 << 17]byte +var z3941 [1 << 17]byte +var z3942 [1 << 17]byte +var z3943 [1 << 17]byte +var z3944 [1 << 17]byte +var z3945 [1 << 17]byte +var z3946 [1 << 17]byte +var z3947 [1 << 17]byte +var z3948 [1 << 17]byte +var z3949 [1 << 17]byte +var z3950 [1 << 17]byte +var z3951 [1 << 17]byte +var z3952 [1 << 17]byte +var z3953 [1 << 17]byte +var z3954 [1 << 17]byte +var z3955 [1 << 17]byte +var z3956 [1 << 17]byte +var z3957 [1 << 17]byte +var z3958 [1 << 17]byte +var z3959 [1 << 17]byte +var z3960 [1 << 17]byte +var z3961 [1 << 17]byte +var z3962 [1 << 17]byte +var z3963 [1 << 17]byte +var z3964 [1 << 17]byte +var z3965 [1 << 17]byte +var z3966 [1 << 17]byte +var z3967 [1 << 17]byte +var z3968 [1 << 17]byte +var z3969 [1 << 17]byte +var z3970 [1 << 17]byte +var z3971 [1 << 17]byte +var z3972 [1 << 17]byte +var z3973 [1 << 17]byte +var z3974 [1 << 17]byte +var z3975 [1 << 17]byte +var z3976 [1 << 17]byte +var z3977 [1 << 17]byte +var z3978 [1 << 17]byte +var z3979 [1 << 17]byte +var z3980 [1 << 17]byte +var z3981 [1 << 17]byte +var z3982 [1 << 17]byte +var z3983 [1 << 17]byte +var z3984 [1 << 17]byte +var z3985 [1 << 17]byte +var z3986 [1 << 17]byte +var z3987 [1 << 17]byte +var z3988 [1 << 17]byte +var z3989 [1 << 17]byte +var z3990 [1 << 17]byte +var z3991 [1 << 17]byte +var z3992 [1 << 17]byte +var z3993 [1 << 17]byte +var z3994 [1 << 17]byte +var z3995 [1 << 17]byte +var z3996 [1 << 17]byte +var z3997 [1 << 17]byte +var z3998 [1 << 17]byte +var z3999 [1 << 17]byte +var z4000 [1 << 17]byte +var z4001 [1 << 17]byte +var z4002 [1 << 17]byte +var z4003 [1 << 17]byte +var z4004 [1 << 17]byte +var z4005 [1 << 17]byte +var z4006 [1 << 17]byte +var z4007 [1 << 17]byte +var z4008 [1 << 17]byte +var z4009 [1 << 17]byte +var z4010 [1 << 17]byte +var z4011 [1 << 17]byte +var z4012 [1 << 17]byte +var z4013 [1 << 17]byte +var z4014 [1 << 17]byte +var z4015 [1 << 17]byte +var z4016 [1 << 17]byte +var z4017 [1 << 17]byte +var z4018 [1 << 17]byte +var z4019 [1 << 17]byte +var z4020 [1 << 17]byte +var z4021 [1 << 17]byte +var z4022 [1 << 17]byte +var z4023 [1 << 17]byte +var z4024 [1 << 17]byte +var z4025 [1 << 17]byte +var z4026 [1 << 17]byte +var z4027 [1 << 17]byte +var z4028 [1 << 17]byte +var z4029 [1 << 17]byte +var z4030 [1 << 17]byte +var z4031 [1 << 17]byte +var z4032 [1 << 17]byte +var z4033 [1 << 17]byte +var z4034 [1 << 17]byte +var z4035 [1 << 17]byte +var z4036 [1 << 17]byte +var z4037 [1 << 17]byte +var z4038 [1 << 17]byte +var z4039 [1 << 17]byte +var z4040 [1 << 17]byte +var z4041 [1 << 17]byte +var z4042 [1 << 17]byte +var z4043 [1 << 17]byte +var z4044 [1 << 17]byte +var z4045 [1 << 17]byte +var z4046 [1 << 17]byte +var z4047 [1 << 17]byte +var z4048 [1 << 17]byte +var z4049 [1 << 17]byte +var z4050 [1 << 17]byte +var z4051 [1 << 17]byte +var z4052 [1 << 17]byte +var z4053 [1 << 17]byte +var z4054 [1 << 17]byte +var z4055 [1 << 17]byte +var z4056 [1 << 17]byte +var z4057 [1 << 17]byte +var z4058 [1 << 17]byte +var z4059 [1 << 17]byte +var z4060 [1 << 17]byte +var z4061 [1 << 17]byte +var z4062 [1 << 17]byte +var z4063 [1 << 17]byte +var z4064 [1 << 17]byte +var z4065 [1 << 17]byte +var z4066 [1 << 17]byte +var z4067 [1 << 17]byte +var z4068 [1 << 17]byte +var z4069 [1 << 17]byte +var z4070 [1 << 17]byte +var z4071 [1 << 17]byte +var z4072 [1 << 17]byte +var z4073 [1 << 17]byte +var z4074 [1 << 17]byte +var z4075 [1 << 17]byte +var z4076 [1 << 17]byte +var z4077 [1 << 17]byte +var z4078 [1 << 17]byte +var z4079 [1 << 17]byte +var z4080 [1 << 17]byte +var z4081 [1 << 17]byte +var z4082 [1 << 17]byte +var z4083 [1 << 17]byte +var z4084 [1 << 17]byte +var z4085 [1 << 17]byte +var z4086 [1 << 17]byte +var z4087 [1 << 17]byte +var z4088 [1 << 17]byte +var z4089 [1 << 17]byte +var z4090 [1 << 17]byte +var z4091 [1 << 17]byte +var z4092 [1 << 17]byte +var z4093 [1 << 17]byte +var z4094 [1 << 17]byte +var z4095 [1 << 17]byte +var z4096 [1 << 17]byte +var z4097 [1 << 17]byte +var z4098 [1 << 17]byte +var z4099 [1 << 17]byte +var z4100 [1 << 17]byte +var z4101 [1 << 17]byte +var z4102 [1 << 17]byte +var z4103 [1 << 17]byte +var z4104 [1 << 17]byte +var z4105 [1 << 17]byte +var z4106 [1 << 17]byte +var z4107 [1 << 17]byte +var z4108 [1 << 17]byte +var z4109 [1 << 17]byte +var z4110 [1 << 17]byte +var z4111 [1 << 17]byte +var z4112 [1 << 17]byte +var z4113 [1 << 17]byte +var z4114 [1 << 17]byte +var z4115 [1 << 17]byte +var z4116 [1 << 17]byte +var z4117 [1 << 17]byte +var z4118 [1 << 17]byte +var z4119 [1 << 17]byte +var z4120 [1 << 17]byte +var z4121 [1 << 17]byte +var z4122 [1 << 17]byte +var z4123 [1 << 17]byte +var z4124 [1 << 17]byte +var z4125 [1 << 17]byte +var z4126 [1 << 17]byte +var z4127 [1 << 17]byte +var z4128 [1 << 17]byte +var z4129 [1 << 17]byte +var z4130 [1 << 17]byte +var z4131 [1 << 17]byte +var z4132 [1 << 17]byte +var z4133 [1 << 17]byte +var z4134 [1 << 17]byte +var z4135 [1 << 17]byte +var z4136 [1 << 17]byte +var z4137 [1 << 17]byte +var z4138 [1 << 17]byte +var z4139 [1 << 17]byte +var z4140 [1 << 17]byte +var z4141 [1 << 17]byte +var z4142 [1 << 17]byte +var z4143 [1 << 17]byte +var z4144 [1 << 17]byte +var z4145 [1 << 17]byte +var z4146 [1 << 17]byte +var z4147 [1 << 17]byte +var z4148 [1 << 17]byte +var z4149 [1 << 17]byte +var z4150 [1 << 17]byte +var z4151 [1 << 17]byte +var z4152 [1 << 17]byte +var z4153 [1 << 17]byte +var z4154 [1 << 17]byte +var z4155 [1 << 17]byte +var z4156 [1 << 17]byte +var z4157 [1 << 17]byte +var z4158 [1 << 17]byte +var z4159 [1 << 17]byte +var z4160 [1 << 17]byte +var z4161 [1 << 17]byte +var z4162 [1 << 17]byte +var z4163 [1 << 17]byte +var z4164 [1 << 17]byte +var z4165 [1 << 17]byte +var z4166 [1 << 17]byte +var z4167 [1 << 17]byte +var z4168 [1 << 17]byte +var z4169 [1 << 17]byte +var z4170 [1 << 17]byte +var z4171 [1 << 17]byte +var z4172 [1 << 17]byte +var z4173 [1 << 17]byte +var z4174 [1 << 17]byte +var z4175 [1 << 17]byte +var z4176 [1 << 17]byte +var z4177 [1 << 17]byte +var z4178 [1 << 17]byte +var z4179 [1 << 17]byte +var z4180 [1 << 17]byte +var z4181 [1 << 17]byte +var z4182 [1 << 17]byte +var z4183 [1 << 17]byte +var z4184 [1 << 17]byte +var z4185 [1 << 17]byte +var z4186 [1 << 17]byte +var z4187 [1 << 17]byte +var z4188 [1 << 17]byte +var z4189 [1 << 17]byte +var z4190 [1 << 17]byte +var z4191 [1 << 17]byte +var z4192 [1 << 17]byte +var z4193 [1 << 17]byte +var z4194 [1 << 17]byte +var z4195 [1 << 17]byte +var z4196 [1 << 17]byte +var z4197 [1 << 17]byte +var z4198 [1 << 17]byte +var z4199 [1 << 17]byte +var z4200 [1 << 17]byte +var z4201 [1 << 17]byte +var z4202 [1 << 17]byte +var z4203 [1 << 17]byte +var z4204 [1 << 17]byte +var z4205 [1 << 17]byte +var z4206 [1 << 17]byte +var z4207 [1 << 17]byte +var z4208 [1 << 17]byte +var z4209 [1 << 17]byte +var z4210 [1 << 17]byte +var z4211 [1 << 17]byte +var z4212 [1 << 17]byte +var z4213 [1 << 17]byte +var z4214 [1 << 17]byte +var z4215 [1 << 17]byte +var z4216 [1 << 17]byte +var z4217 [1 << 17]byte +var z4218 [1 << 17]byte +var z4219 [1 << 17]byte +var z4220 [1 << 17]byte +var z4221 [1 << 17]byte +var z4222 [1 << 17]byte +var z4223 [1 << 17]byte +var z4224 [1 << 17]byte +var z4225 [1 << 17]byte +var z4226 [1 << 17]byte +var z4227 [1 << 17]byte +var z4228 [1 << 17]byte +var z4229 [1 << 17]byte +var z4230 [1 << 17]byte +var z4231 [1 << 17]byte +var z4232 [1 << 17]byte +var z4233 [1 << 17]byte +var z4234 [1 << 17]byte +var z4235 [1 << 17]byte +var z4236 [1 << 17]byte +var z4237 [1 << 17]byte +var z4238 [1 << 17]byte +var z4239 [1 << 17]byte +var z4240 [1 << 17]byte +var z4241 [1 << 17]byte +var z4242 [1 << 17]byte +var z4243 [1 << 17]byte +var z4244 [1 << 17]byte +var z4245 [1 << 17]byte +var z4246 [1 << 17]byte +var z4247 [1 << 17]byte +var z4248 [1 << 17]byte +var z4249 [1 << 17]byte +var z4250 [1 << 17]byte +var z4251 [1 << 17]byte +var z4252 [1 << 17]byte +var z4253 [1 << 17]byte +var z4254 [1 << 17]byte +var z4255 [1 << 17]byte +var z4256 [1 << 17]byte +var z4257 [1 << 17]byte +var z4258 [1 << 17]byte +var z4259 [1 << 17]byte +var z4260 [1 << 17]byte +var z4261 [1 << 17]byte +var z4262 [1 << 17]byte +var z4263 [1 << 17]byte +var z4264 [1 << 17]byte +var z4265 [1 << 17]byte +var z4266 [1 << 17]byte +var z4267 [1 << 17]byte +var z4268 [1 << 17]byte +var z4269 [1 << 17]byte +var z4270 [1 << 17]byte +var z4271 [1 << 17]byte +var z4272 [1 << 17]byte +var z4273 [1 << 17]byte +var z4274 [1 << 17]byte +var z4275 [1 << 17]byte +var z4276 [1 << 17]byte +var z4277 [1 << 17]byte +var z4278 [1 << 17]byte +var z4279 [1 << 17]byte +var z4280 [1 << 17]byte +var z4281 [1 << 17]byte +var z4282 [1 << 17]byte +var z4283 [1 << 17]byte +var z4284 [1 << 17]byte +var z4285 [1 << 17]byte +var z4286 [1 << 17]byte +var z4287 [1 << 17]byte +var z4288 [1 << 17]byte +var z4289 [1 << 17]byte +var z4290 [1 << 17]byte +var z4291 [1 << 17]byte +var z4292 [1 << 17]byte +var z4293 [1 << 17]byte +var z4294 [1 << 17]byte +var z4295 [1 << 17]byte +var z4296 [1 << 17]byte +var z4297 [1 << 17]byte +var z4298 [1 << 17]byte +var z4299 [1 << 17]byte +var z4300 [1 << 17]byte +var z4301 [1 << 17]byte +var z4302 [1 << 17]byte +var z4303 [1 << 17]byte +var z4304 [1 << 17]byte +var z4305 [1 << 17]byte +var z4306 [1 << 17]byte +var z4307 [1 << 17]byte +var z4308 [1 << 17]byte +var z4309 [1 << 17]byte +var z4310 [1 << 17]byte +var z4311 [1 << 17]byte +var z4312 [1 << 17]byte +var z4313 [1 << 17]byte +var z4314 [1 << 17]byte +var z4315 [1 << 17]byte +var z4316 [1 << 17]byte +var z4317 [1 << 17]byte +var z4318 [1 << 17]byte +var z4319 [1 << 17]byte +var z4320 [1 << 17]byte +var z4321 [1 << 17]byte +var z4322 [1 << 17]byte +var z4323 [1 << 17]byte +var z4324 [1 << 17]byte +var z4325 [1 << 17]byte +var z4326 [1 << 17]byte +var z4327 [1 << 17]byte +var z4328 [1 << 17]byte +var z4329 [1 << 17]byte +var z4330 [1 << 17]byte +var z4331 [1 << 17]byte +var z4332 [1 << 17]byte +var z4333 [1 << 17]byte +var z4334 [1 << 17]byte +var z4335 [1 << 17]byte +var z4336 [1 << 17]byte +var z4337 [1 << 17]byte +var z4338 [1 << 17]byte +var z4339 [1 << 17]byte +var z4340 [1 << 17]byte +var z4341 [1 << 17]byte +var z4342 [1 << 17]byte +var z4343 [1 << 17]byte +var z4344 [1 << 17]byte +var z4345 [1 << 17]byte +var z4346 [1 << 17]byte +var z4347 [1 << 17]byte +var z4348 [1 << 17]byte +var z4349 [1 << 17]byte +var z4350 [1 << 17]byte +var z4351 [1 << 17]byte +var z4352 [1 << 17]byte +var z4353 [1 << 17]byte +var z4354 [1 << 17]byte +var z4355 [1 << 17]byte +var z4356 [1 << 17]byte +var z4357 [1 << 17]byte +var z4358 [1 << 17]byte +var z4359 [1 << 17]byte +var z4360 [1 << 17]byte +var z4361 [1 << 17]byte +var z4362 [1 << 17]byte +var z4363 [1 << 17]byte +var z4364 [1 << 17]byte +var z4365 [1 << 17]byte +var z4366 [1 << 17]byte +var z4367 [1 << 17]byte +var z4368 [1 << 17]byte +var z4369 [1 << 17]byte +var z4370 [1 << 17]byte +var z4371 [1 << 17]byte +var z4372 [1 << 17]byte +var z4373 [1 << 17]byte +var z4374 [1 << 17]byte +var z4375 [1 << 17]byte +var z4376 [1 << 17]byte +var z4377 [1 << 17]byte +var z4378 [1 << 17]byte +var z4379 [1 << 17]byte +var z4380 [1 << 17]byte +var z4381 [1 << 17]byte +var z4382 [1 << 17]byte +var z4383 [1 << 17]byte +var z4384 [1 << 17]byte +var z4385 [1 << 17]byte +var z4386 [1 << 17]byte +var z4387 [1 << 17]byte +var z4388 [1 << 17]byte +var z4389 [1 << 17]byte +var z4390 [1 << 17]byte +var z4391 [1 << 17]byte +var z4392 [1 << 17]byte +var z4393 [1 << 17]byte +var z4394 [1 << 17]byte +var z4395 [1 << 17]byte +var z4396 [1 << 17]byte +var z4397 [1 << 17]byte +var z4398 [1 << 17]byte +var z4399 [1 << 17]byte +var z4400 [1 << 17]byte +var z4401 [1 << 17]byte +var z4402 [1 << 17]byte +var z4403 [1 << 17]byte +var z4404 [1 << 17]byte +var z4405 [1 << 17]byte +var z4406 [1 << 17]byte +var z4407 [1 << 17]byte +var z4408 [1 << 17]byte +var z4409 [1 << 17]byte +var z4410 [1 << 17]byte +var z4411 [1 << 17]byte +var z4412 [1 << 17]byte +var z4413 [1 << 17]byte +var z4414 [1 << 17]byte +var z4415 [1 << 17]byte +var z4416 [1 << 17]byte +var z4417 [1 << 17]byte +var z4418 [1 << 17]byte +var z4419 [1 << 17]byte +var z4420 [1 << 17]byte +var z4421 [1 << 17]byte +var z4422 [1 << 17]byte +var z4423 [1 << 17]byte +var z4424 [1 << 17]byte +var z4425 [1 << 17]byte +var z4426 [1 << 17]byte +var z4427 [1 << 17]byte +var z4428 [1 << 17]byte +var z4429 [1 << 17]byte +var z4430 [1 << 17]byte +var z4431 [1 << 17]byte +var z4432 [1 << 17]byte +var z4433 [1 << 17]byte +var z4434 [1 << 17]byte +var z4435 [1 << 17]byte +var z4436 [1 << 17]byte +var z4437 [1 << 17]byte +var z4438 [1 << 17]byte +var z4439 [1 << 17]byte +var z4440 [1 << 17]byte +var z4441 [1 << 17]byte +var z4442 [1 << 17]byte +var z4443 [1 << 17]byte +var z4444 [1 << 17]byte +var z4445 [1 << 17]byte +var z4446 [1 << 17]byte +var z4447 [1 << 17]byte +var z4448 [1 << 17]byte +var z4449 [1 << 17]byte +var z4450 [1 << 17]byte +var z4451 [1 << 17]byte +var z4452 [1 << 17]byte +var z4453 [1 << 17]byte +var z4454 [1 << 17]byte +var z4455 [1 << 17]byte +var z4456 [1 << 17]byte +var z4457 [1 << 17]byte +var z4458 [1 << 17]byte +var z4459 [1 << 17]byte +var z4460 [1 << 17]byte +var z4461 [1 << 17]byte +var z4462 [1 << 17]byte +var z4463 [1 << 17]byte +var z4464 [1 << 17]byte +var z4465 [1 << 17]byte +var z4466 [1 << 17]byte +var z4467 [1 << 17]byte +var z4468 [1 << 17]byte +var z4469 [1 << 17]byte +var z4470 [1 << 17]byte +var z4471 [1 << 17]byte +var z4472 [1 << 17]byte +var z4473 [1 << 17]byte +var z4474 [1 << 17]byte +var z4475 [1 << 17]byte +var z4476 [1 << 17]byte +var z4477 [1 << 17]byte +var z4478 [1 << 17]byte +var z4479 [1 << 17]byte +var z4480 [1 << 17]byte +var z4481 [1 << 17]byte +var z4482 [1 << 17]byte +var z4483 [1 << 17]byte +var z4484 [1 << 17]byte +var z4485 [1 << 17]byte +var z4486 [1 << 17]byte +var z4487 [1 << 17]byte +var z4488 [1 << 17]byte +var z4489 [1 << 17]byte +var z4490 [1 << 17]byte +var z4491 [1 << 17]byte +var z4492 [1 << 17]byte +var z4493 [1 << 17]byte +var z4494 [1 << 17]byte +var z4495 [1 << 17]byte +var z4496 [1 << 17]byte +var z4497 [1 << 17]byte +var z4498 [1 << 17]byte +var z4499 [1 << 17]byte +var z4500 [1 << 17]byte +var z4501 [1 << 17]byte +var z4502 [1 << 17]byte +var z4503 [1 << 17]byte +var z4504 [1 << 17]byte +var z4505 [1 << 17]byte +var z4506 [1 << 17]byte +var z4507 [1 << 17]byte +var z4508 [1 << 17]byte +var z4509 [1 << 17]byte +var z4510 [1 << 17]byte +var z4511 [1 << 17]byte +var z4512 [1 << 17]byte +var z4513 [1 << 17]byte +var z4514 [1 << 17]byte +var z4515 [1 << 17]byte +var z4516 [1 << 17]byte +var z4517 [1 << 17]byte +var z4518 [1 << 17]byte +var z4519 [1 << 17]byte +var z4520 [1 << 17]byte +var z4521 [1 << 17]byte +var z4522 [1 << 17]byte +var z4523 [1 << 17]byte +var z4524 [1 << 17]byte +var z4525 [1 << 17]byte +var z4526 [1 << 17]byte +var z4527 [1 << 17]byte +var z4528 [1 << 17]byte +var z4529 [1 << 17]byte +var z4530 [1 << 17]byte +var z4531 [1 << 17]byte +var z4532 [1 << 17]byte +var z4533 [1 << 17]byte +var z4534 [1 << 17]byte +var z4535 [1 << 17]byte +var z4536 [1 << 17]byte +var z4537 [1 << 17]byte +var z4538 [1 << 17]byte +var z4539 [1 << 17]byte +var z4540 [1 << 17]byte +var z4541 [1 << 17]byte +var z4542 [1 << 17]byte +var z4543 [1 << 17]byte +var z4544 [1 << 17]byte +var z4545 [1 << 17]byte +var z4546 [1 << 17]byte +var z4547 [1 << 17]byte +var z4548 [1 << 17]byte +var z4549 [1 << 17]byte +var z4550 [1 << 17]byte +var z4551 [1 << 17]byte +var z4552 [1 << 17]byte +var z4553 [1 << 17]byte +var z4554 [1 << 17]byte +var z4555 [1 << 17]byte +var z4556 [1 << 17]byte +var z4557 [1 << 17]byte +var z4558 [1 << 17]byte +var z4559 [1 << 17]byte +var z4560 [1 << 17]byte +var z4561 [1 << 17]byte +var z4562 [1 << 17]byte +var z4563 [1 << 17]byte +var z4564 [1 << 17]byte +var z4565 [1 << 17]byte +var z4566 [1 << 17]byte +var z4567 [1 << 17]byte +var z4568 [1 << 17]byte +var z4569 [1 << 17]byte +var z4570 [1 << 17]byte +var z4571 [1 << 17]byte +var z4572 [1 << 17]byte +var z4573 [1 << 17]byte +var z4574 [1 << 17]byte +var z4575 [1 << 17]byte +var z4576 [1 << 17]byte +var z4577 [1 << 17]byte +var z4578 [1 << 17]byte +var z4579 [1 << 17]byte +var z4580 [1 << 17]byte +var z4581 [1 << 17]byte +var z4582 [1 << 17]byte +var z4583 [1 << 17]byte +var z4584 [1 << 17]byte +var z4585 [1 << 17]byte +var z4586 [1 << 17]byte +var z4587 [1 << 17]byte +var z4588 [1 << 17]byte +var z4589 [1 << 17]byte +var z4590 [1 << 17]byte +var z4591 [1 << 17]byte +var z4592 [1 << 17]byte +var z4593 [1 << 17]byte +var z4594 [1 << 17]byte +var z4595 [1 << 17]byte +var z4596 [1 << 17]byte +var z4597 [1 << 17]byte +var z4598 [1 << 17]byte +var z4599 [1 << 17]byte +var z4600 [1 << 17]byte +var z4601 [1 << 17]byte +var z4602 [1 << 17]byte +var z4603 [1 << 17]byte +var z4604 [1 << 17]byte +var z4605 [1 << 17]byte +var z4606 [1 << 17]byte +var z4607 [1 << 17]byte +var z4608 [1 << 17]byte +var z4609 [1 << 17]byte +var z4610 [1 << 17]byte +var z4611 [1 << 17]byte +var z4612 [1 << 17]byte +var z4613 [1 << 17]byte +var z4614 [1 << 17]byte +var z4615 [1 << 17]byte +var z4616 [1 << 17]byte +var z4617 [1 << 17]byte +var z4618 [1 << 17]byte +var z4619 [1 << 17]byte +var z4620 [1 << 17]byte +var z4621 [1 << 17]byte +var z4622 [1 << 17]byte +var z4623 [1 << 17]byte +var z4624 [1 << 17]byte +var z4625 [1 << 17]byte +var z4626 [1 << 17]byte +var z4627 [1 << 17]byte +var z4628 [1 << 17]byte +var z4629 [1 << 17]byte +var z4630 [1 << 17]byte +var z4631 [1 << 17]byte +var z4632 [1 << 17]byte +var z4633 [1 << 17]byte +var z4634 [1 << 17]byte +var z4635 [1 << 17]byte +var z4636 [1 << 17]byte +var z4637 [1 << 17]byte +var z4638 [1 << 17]byte +var z4639 [1 << 17]byte +var z4640 [1 << 17]byte +var z4641 [1 << 17]byte +var z4642 [1 << 17]byte +var z4643 [1 << 17]byte +var z4644 [1 << 17]byte +var z4645 [1 << 17]byte +var z4646 [1 << 17]byte +var z4647 [1 << 17]byte +var z4648 [1 << 17]byte +var z4649 [1 << 17]byte +var z4650 [1 << 17]byte +var z4651 [1 << 17]byte +var z4652 [1 << 17]byte +var z4653 [1 << 17]byte +var z4654 [1 << 17]byte +var z4655 [1 << 17]byte +var z4656 [1 << 17]byte +var z4657 [1 << 17]byte +var z4658 [1 << 17]byte +var z4659 [1 << 17]byte +var z4660 [1 << 17]byte +var z4661 [1 << 17]byte +var z4662 [1 << 17]byte +var z4663 [1 << 17]byte +var z4664 [1 << 17]byte +var z4665 [1 << 17]byte +var z4666 [1 << 17]byte +var z4667 [1 << 17]byte +var z4668 [1 << 17]byte +var z4669 [1 << 17]byte +var z4670 [1 << 17]byte +var z4671 [1 << 17]byte +var z4672 [1 << 17]byte +var z4673 [1 << 17]byte +var z4674 [1 << 17]byte +var z4675 [1 << 17]byte +var z4676 [1 << 17]byte +var z4677 [1 << 17]byte +var z4678 [1 << 17]byte +var z4679 [1 << 17]byte +var z4680 [1 << 17]byte +var z4681 [1 << 17]byte +var z4682 [1 << 17]byte +var z4683 [1 << 17]byte +var z4684 [1 << 17]byte +var z4685 [1 << 17]byte +var z4686 [1 << 17]byte +var z4687 [1 << 17]byte +var z4688 [1 << 17]byte +var z4689 [1 << 17]byte +var z4690 [1 << 17]byte +var z4691 [1 << 17]byte +var z4692 [1 << 17]byte +var z4693 [1 << 17]byte +var z4694 [1 << 17]byte +var z4695 [1 << 17]byte +var z4696 [1 << 17]byte +var z4697 [1 << 17]byte +var z4698 [1 << 17]byte +var z4699 [1 << 17]byte +var z4700 [1 << 17]byte +var z4701 [1 << 17]byte +var z4702 [1 << 17]byte +var z4703 [1 << 17]byte +var z4704 [1 << 17]byte +var z4705 [1 << 17]byte +var z4706 [1 << 17]byte +var z4707 [1 << 17]byte +var z4708 [1 << 17]byte +var z4709 [1 << 17]byte +var z4710 [1 << 17]byte +var z4711 [1 << 17]byte +var z4712 [1 << 17]byte +var z4713 [1 << 17]byte +var z4714 [1 << 17]byte +var z4715 [1 << 17]byte +var z4716 [1 << 17]byte +var z4717 [1 << 17]byte +var z4718 [1 << 17]byte +var z4719 [1 << 17]byte +var z4720 [1 << 17]byte +var z4721 [1 << 17]byte +var z4722 [1 << 17]byte +var z4723 [1 << 17]byte +var z4724 [1 << 17]byte +var z4725 [1 << 17]byte +var z4726 [1 << 17]byte +var z4727 [1 << 17]byte +var z4728 [1 << 17]byte +var z4729 [1 << 17]byte +var z4730 [1 << 17]byte +var z4731 [1 << 17]byte +var z4732 [1 << 17]byte +var z4733 [1 << 17]byte +var z4734 [1 << 17]byte +var z4735 [1 << 17]byte +var z4736 [1 << 17]byte +var z4737 [1 << 17]byte +var z4738 [1 << 17]byte +var z4739 [1 << 17]byte +var z4740 [1 << 17]byte +var z4741 [1 << 17]byte +var z4742 [1 << 17]byte +var z4743 [1 << 17]byte +var z4744 [1 << 17]byte +var z4745 [1 << 17]byte +var z4746 [1 << 17]byte +var z4747 [1 << 17]byte +var z4748 [1 << 17]byte +var z4749 [1 << 17]byte +var z4750 [1 << 17]byte +var z4751 [1 << 17]byte +var z4752 [1 << 17]byte +var z4753 [1 << 17]byte +var z4754 [1 << 17]byte +var z4755 [1 << 17]byte +var z4756 [1 << 17]byte +var z4757 [1 << 17]byte +var z4758 [1 << 17]byte +var z4759 [1 << 17]byte +var z4760 [1 << 17]byte +var z4761 [1 << 17]byte +var z4762 [1 << 17]byte +var z4763 [1 << 17]byte +var z4764 [1 << 17]byte +var z4765 [1 << 17]byte +var z4766 [1 << 17]byte +var z4767 [1 << 17]byte +var z4768 [1 << 17]byte +var z4769 [1 << 17]byte +var z4770 [1 << 17]byte +var z4771 [1 << 17]byte +var z4772 [1 << 17]byte +var z4773 [1 << 17]byte +var z4774 [1 << 17]byte +var z4775 [1 << 17]byte +var z4776 [1 << 17]byte +var z4777 [1 << 17]byte +var z4778 [1 << 17]byte +var z4779 [1 << 17]byte +var z4780 [1 << 17]byte +var z4781 [1 << 17]byte +var z4782 [1 << 17]byte +var z4783 [1 << 17]byte +var z4784 [1 << 17]byte +var z4785 [1 << 17]byte +var z4786 [1 << 17]byte +var z4787 [1 << 17]byte +var z4788 [1 << 17]byte +var z4789 [1 << 17]byte +var z4790 [1 << 17]byte +var z4791 [1 << 17]byte +var z4792 [1 << 17]byte +var z4793 [1 << 17]byte +var z4794 [1 << 17]byte +var z4795 [1 << 17]byte +var z4796 [1 << 17]byte +var z4797 [1 << 17]byte +var z4798 [1 << 17]byte +var z4799 [1 << 17]byte +var z4800 [1 << 17]byte +var z4801 [1 << 17]byte +var z4802 [1 << 17]byte +var z4803 [1 << 17]byte +var z4804 [1 << 17]byte +var z4805 [1 << 17]byte +var z4806 [1 << 17]byte +var z4807 [1 << 17]byte +var z4808 [1 << 17]byte +var z4809 [1 << 17]byte +var z4810 [1 << 17]byte +var z4811 [1 << 17]byte +var z4812 [1 << 17]byte +var z4813 [1 << 17]byte +var z4814 [1 << 17]byte +var z4815 [1 << 17]byte +var z4816 [1 << 17]byte +var z4817 [1 << 17]byte +var z4818 [1 << 17]byte +var z4819 [1 << 17]byte +var z4820 [1 << 17]byte +var z4821 [1 << 17]byte +var z4822 [1 << 17]byte +var z4823 [1 << 17]byte +var z4824 [1 << 17]byte +var z4825 [1 << 17]byte +var z4826 [1 << 17]byte +var z4827 [1 << 17]byte +var z4828 [1 << 17]byte +var z4829 [1 << 17]byte +var z4830 [1 << 17]byte +var z4831 [1 << 17]byte +var z4832 [1 << 17]byte +var z4833 [1 << 17]byte +var z4834 [1 << 17]byte +var z4835 [1 << 17]byte +var z4836 [1 << 17]byte +var z4837 [1 << 17]byte +var z4838 [1 << 17]byte +var z4839 [1 << 17]byte +var z4840 [1 << 17]byte +var z4841 [1 << 17]byte +var z4842 [1 << 17]byte +var z4843 [1 << 17]byte +var z4844 [1 << 17]byte +var z4845 [1 << 17]byte +var z4846 [1 << 17]byte +var z4847 [1 << 17]byte +var z4848 [1 << 17]byte +var z4849 [1 << 17]byte +var z4850 [1 << 17]byte +var z4851 [1 << 17]byte +var z4852 [1 << 17]byte +var z4853 [1 << 17]byte +var z4854 [1 << 17]byte +var z4855 [1 << 17]byte +var z4856 [1 << 17]byte +var z4857 [1 << 17]byte +var z4858 [1 << 17]byte +var z4859 [1 << 17]byte +var z4860 [1 << 17]byte +var z4861 [1 << 17]byte +var z4862 [1 << 17]byte +var z4863 [1 << 17]byte +var z4864 [1 << 17]byte +var z4865 [1 << 17]byte +var z4866 [1 << 17]byte +var z4867 [1 << 17]byte +var z4868 [1 << 17]byte +var z4869 [1 << 17]byte +var z4870 [1 << 17]byte +var z4871 [1 << 17]byte +var z4872 [1 << 17]byte +var z4873 [1 << 17]byte +var z4874 [1 << 17]byte +var z4875 [1 << 17]byte +var z4876 [1 << 17]byte +var z4877 [1 << 17]byte +var z4878 [1 << 17]byte +var z4879 [1 << 17]byte +var z4880 [1 << 17]byte +var z4881 [1 << 17]byte +var z4882 [1 << 17]byte +var z4883 [1 << 17]byte +var z4884 [1 << 17]byte +var z4885 [1 << 17]byte +var z4886 [1 << 17]byte +var z4887 [1 << 17]byte +var z4888 [1 << 17]byte +var z4889 [1 << 17]byte +var z4890 [1 << 17]byte +var z4891 [1 << 17]byte +var z4892 [1 << 17]byte +var z4893 [1 << 17]byte +var z4894 [1 << 17]byte +var z4895 [1 << 17]byte +var z4896 [1 << 17]byte +var z4897 [1 << 17]byte +var z4898 [1 << 17]byte +var z4899 [1 << 17]byte +var z4900 [1 << 17]byte +var z4901 [1 << 17]byte +var z4902 [1 << 17]byte +var z4903 [1 << 17]byte +var z4904 [1 << 17]byte +var z4905 [1 << 17]byte +var z4906 [1 << 17]byte +var z4907 [1 << 17]byte +var z4908 [1 << 17]byte +var z4909 [1 << 17]byte +var z4910 [1 << 17]byte +var z4911 [1 << 17]byte +var z4912 [1 << 17]byte +var z4913 [1 << 17]byte +var z4914 [1 << 17]byte +var z4915 [1 << 17]byte +var z4916 [1 << 17]byte +var z4917 [1 << 17]byte +var z4918 [1 << 17]byte +var z4919 [1 << 17]byte +var z4920 [1 << 17]byte +var z4921 [1 << 17]byte +var z4922 [1 << 17]byte +var z4923 [1 << 17]byte +var z4924 [1 << 17]byte +var z4925 [1 << 17]byte +var z4926 [1 << 17]byte +var z4927 [1 << 17]byte +var z4928 [1 << 17]byte +var z4929 [1 << 17]byte +var z4930 [1 << 17]byte +var z4931 [1 << 17]byte +var z4932 [1 << 17]byte +var z4933 [1 << 17]byte +var z4934 [1 << 17]byte +var z4935 [1 << 17]byte +var z4936 [1 << 17]byte +var z4937 [1 << 17]byte +var z4938 [1 << 17]byte +var z4939 [1 << 17]byte +var z4940 [1 << 17]byte +var z4941 [1 << 17]byte +var z4942 [1 << 17]byte +var z4943 [1 << 17]byte +var z4944 [1 << 17]byte +var z4945 [1 << 17]byte +var z4946 [1 << 17]byte +var z4947 [1 << 17]byte +var z4948 [1 << 17]byte +var z4949 [1 << 17]byte +var z4950 [1 << 17]byte +var z4951 [1 << 17]byte +var z4952 [1 << 17]byte +var z4953 [1 << 17]byte +var z4954 [1 << 17]byte +var z4955 [1 << 17]byte +var z4956 [1 << 17]byte +var z4957 [1 << 17]byte +var z4958 [1 << 17]byte +var z4959 [1 << 17]byte +var z4960 [1 << 17]byte +var z4961 [1 << 17]byte +var z4962 [1 << 17]byte +var z4963 [1 << 17]byte +var z4964 [1 << 17]byte +var z4965 [1 << 17]byte +var z4966 [1 << 17]byte +var z4967 [1 << 17]byte +var z4968 [1 << 17]byte +var z4969 [1 << 17]byte +var z4970 [1 << 17]byte +var z4971 [1 << 17]byte +var z4972 [1 << 17]byte +var z4973 [1 << 17]byte +var z4974 [1 << 17]byte +var z4975 [1 << 17]byte +var z4976 [1 << 17]byte +var z4977 [1 << 17]byte +var z4978 [1 << 17]byte +var z4979 [1 << 17]byte +var z4980 [1 << 17]byte +var z4981 [1 << 17]byte +var z4982 [1 << 17]byte +var z4983 [1 << 17]byte +var z4984 [1 << 17]byte +var z4985 [1 << 17]byte +var z4986 [1 << 17]byte +var z4987 [1 << 17]byte +var z4988 [1 << 17]byte +var z4989 [1 << 17]byte +var z4990 [1 << 17]byte +var z4991 [1 << 17]byte +var z4992 [1 << 17]byte +var z4993 [1 << 17]byte +var z4994 [1 << 17]byte +var z4995 [1 << 17]byte +var z4996 [1 << 17]byte +var z4997 [1 << 17]byte +var z4998 [1 << 17]byte +var z4999 [1 << 17]byte +var z5000 [1 << 17]byte +var z5001 [1 << 17]byte +var z5002 [1 << 17]byte +var z5003 [1 << 17]byte +var z5004 [1 << 17]byte +var z5005 [1 << 17]byte +var z5006 [1 << 17]byte +var z5007 [1 << 17]byte +var z5008 [1 << 17]byte +var z5009 [1 << 17]byte +var z5010 [1 << 17]byte +var z5011 [1 << 17]byte +var z5012 [1 << 17]byte +var z5013 [1 << 17]byte +var z5014 [1 << 17]byte +var z5015 [1 << 17]byte +var z5016 [1 << 17]byte +var z5017 [1 << 17]byte +var z5018 [1 << 17]byte +var z5019 [1 << 17]byte +var z5020 [1 << 17]byte +var z5021 [1 << 17]byte +var z5022 [1 << 17]byte +var z5023 [1 << 17]byte +var z5024 [1 << 17]byte +var z5025 [1 << 17]byte +var z5026 [1 << 17]byte +var z5027 [1 << 17]byte +var z5028 [1 << 17]byte +var z5029 [1 << 17]byte +var z5030 [1 << 17]byte +var z5031 [1 << 17]byte +var z5032 [1 << 17]byte +var z5033 [1 << 17]byte +var z5034 [1 << 17]byte +var z5035 [1 << 17]byte +var z5036 [1 << 17]byte +var z5037 [1 << 17]byte +var z5038 [1 << 17]byte +var z5039 [1 << 17]byte +var z5040 [1 << 17]byte +var z5041 [1 << 17]byte +var z5042 [1 << 17]byte +var z5043 [1 << 17]byte +var z5044 [1 << 17]byte +var z5045 [1 << 17]byte +var z5046 [1 << 17]byte +var z5047 [1 << 17]byte +var z5048 [1 << 17]byte +var z5049 [1 << 17]byte +var z5050 [1 << 17]byte +var z5051 [1 << 17]byte +var z5052 [1 << 17]byte +var z5053 [1 << 17]byte +var z5054 [1 << 17]byte +var z5055 [1 << 17]byte +var z5056 [1 << 17]byte +var z5057 [1 << 17]byte +var z5058 [1 << 17]byte +var z5059 [1 << 17]byte +var z5060 [1 << 17]byte +var z5061 [1 << 17]byte +var z5062 [1 << 17]byte +var z5063 [1 << 17]byte +var z5064 [1 << 17]byte +var z5065 [1 << 17]byte +var z5066 [1 << 17]byte +var z5067 [1 << 17]byte +var z5068 [1 << 17]byte +var z5069 [1 << 17]byte +var z5070 [1 << 17]byte +var z5071 [1 << 17]byte +var z5072 [1 << 17]byte +var z5073 [1 << 17]byte +var z5074 [1 << 17]byte +var z5075 [1 << 17]byte +var z5076 [1 << 17]byte +var z5077 [1 << 17]byte +var z5078 [1 << 17]byte +var z5079 [1 << 17]byte +var z5080 [1 << 17]byte +var z5081 [1 << 17]byte +var z5082 [1 << 17]byte +var z5083 [1 << 17]byte +var z5084 [1 << 17]byte +var z5085 [1 << 17]byte +var z5086 [1 << 17]byte +var z5087 [1 << 17]byte +var z5088 [1 << 17]byte +var z5089 [1 << 17]byte +var z5090 [1 << 17]byte +var z5091 [1 << 17]byte +var z5092 [1 << 17]byte +var z5093 [1 << 17]byte +var z5094 [1 << 17]byte +var z5095 [1 << 17]byte +var z5096 [1 << 17]byte +var z5097 [1 << 17]byte +var z5098 [1 << 17]byte +var z5099 [1 << 17]byte +var z5100 [1 << 17]byte +var z5101 [1 << 17]byte +var z5102 [1 << 17]byte +var z5103 [1 << 17]byte +var z5104 [1 << 17]byte +var z5105 [1 << 17]byte +var z5106 [1 << 17]byte +var z5107 [1 << 17]byte +var z5108 [1 << 17]byte +var z5109 [1 << 17]byte +var z5110 [1 << 17]byte +var z5111 [1 << 17]byte +var z5112 [1 << 17]byte +var z5113 [1 << 17]byte +var z5114 [1 << 17]byte +var z5115 [1 << 17]byte +var z5116 [1 << 17]byte +var z5117 [1 << 17]byte +var z5118 [1 << 17]byte +var z5119 [1 << 17]byte +var z5120 [1 << 17]byte +var z5121 [1 << 17]byte +var z5122 [1 << 17]byte +var z5123 [1 << 17]byte +var z5124 [1 << 17]byte +var z5125 [1 << 17]byte +var z5126 [1 << 17]byte +var z5127 [1 << 17]byte +var z5128 [1 << 17]byte +var z5129 [1 << 17]byte +var z5130 [1 << 17]byte +var z5131 [1 << 17]byte +var z5132 [1 << 17]byte +var z5133 [1 << 17]byte +var z5134 [1 << 17]byte +var z5135 [1 << 17]byte +var z5136 [1 << 17]byte +var z5137 [1 << 17]byte +var z5138 [1 << 17]byte +var z5139 [1 << 17]byte +var z5140 [1 << 17]byte +var z5141 [1 << 17]byte +var z5142 [1 << 17]byte +var z5143 [1 << 17]byte +var z5144 [1 << 17]byte +var z5145 [1 << 17]byte +var z5146 [1 << 17]byte +var z5147 [1 << 17]byte +var z5148 [1 << 17]byte +var z5149 [1 << 17]byte +var z5150 [1 << 17]byte +var z5151 [1 << 17]byte +var z5152 [1 << 17]byte +var z5153 [1 << 17]byte +var z5154 [1 << 17]byte +var z5155 [1 << 17]byte +var z5156 [1 << 17]byte +var z5157 [1 << 17]byte +var z5158 [1 << 17]byte +var z5159 [1 << 17]byte +var z5160 [1 << 17]byte +var z5161 [1 << 17]byte +var z5162 [1 << 17]byte +var z5163 [1 << 17]byte +var z5164 [1 << 17]byte +var z5165 [1 << 17]byte +var z5166 [1 << 17]byte +var z5167 [1 << 17]byte +var z5168 [1 << 17]byte +var z5169 [1 << 17]byte +var z5170 [1 << 17]byte +var z5171 [1 << 17]byte +var z5172 [1 << 17]byte +var z5173 [1 << 17]byte +var z5174 [1 << 17]byte +var z5175 [1 << 17]byte +var z5176 [1 << 17]byte +var z5177 [1 << 17]byte +var z5178 [1 << 17]byte +var z5179 [1 << 17]byte +var z5180 [1 << 17]byte +var z5181 [1 << 17]byte +var z5182 [1 << 17]byte +var z5183 [1 << 17]byte +var z5184 [1 << 17]byte +var z5185 [1 << 17]byte +var z5186 [1 << 17]byte +var z5187 [1 << 17]byte +var z5188 [1 << 17]byte +var z5189 [1 << 17]byte +var z5190 [1 << 17]byte +var z5191 [1 << 17]byte +var z5192 [1 << 17]byte +var z5193 [1 << 17]byte +var z5194 [1 << 17]byte +var z5195 [1 << 17]byte +var z5196 [1 << 17]byte +var z5197 [1 << 17]byte +var z5198 [1 << 17]byte +var z5199 [1 << 17]byte +var z5200 [1 << 17]byte +var z5201 [1 << 17]byte +var z5202 [1 << 17]byte +var z5203 [1 << 17]byte +var z5204 [1 << 17]byte +var z5205 [1 << 17]byte +var z5206 [1 << 17]byte +var z5207 [1 << 17]byte +var z5208 [1 << 17]byte +var z5209 [1 << 17]byte +var z5210 [1 << 17]byte +var z5211 [1 << 17]byte +var z5212 [1 << 17]byte +var z5213 [1 << 17]byte +var z5214 [1 << 17]byte +var z5215 [1 << 17]byte +var z5216 [1 << 17]byte +var z5217 [1 << 17]byte +var z5218 [1 << 17]byte +var z5219 [1 << 17]byte +var z5220 [1 << 17]byte +var z5221 [1 << 17]byte +var z5222 [1 << 17]byte +var z5223 [1 << 17]byte +var z5224 [1 << 17]byte +var z5225 [1 << 17]byte +var z5226 [1 << 17]byte +var z5227 [1 << 17]byte +var z5228 [1 << 17]byte +var z5229 [1 << 17]byte +var z5230 [1 << 17]byte +var z5231 [1 << 17]byte +var z5232 [1 << 17]byte +var z5233 [1 << 17]byte +var z5234 [1 << 17]byte +var z5235 [1 << 17]byte +var z5236 [1 << 17]byte +var z5237 [1 << 17]byte +var z5238 [1 << 17]byte +var z5239 [1 << 17]byte +var z5240 [1 << 17]byte +var z5241 [1 << 17]byte +var z5242 [1 << 17]byte +var z5243 [1 << 17]byte +var z5244 [1 << 17]byte +var z5245 [1 << 17]byte +var z5246 [1 << 17]byte +var z5247 [1 << 17]byte +var z5248 [1 << 17]byte +var z5249 [1 << 17]byte +var z5250 [1 << 17]byte +var z5251 [1 << 17]byte +var z5252 [1 << 17]byte +var z5253 [1 << 17]byte +var z5254 [1 << 17]byte +var z5255 [1 << 17]byte +var z5256 [1 << 17]byte +var z5257 [1 << 17]byte +var z5258 [1 << 17]byte +var z5259 [1 << 17]byte +var z5260 [1 << 17]byte +var z5261 [1 << 17]byte +var z5262 [1 << 17]byte +var z5263 [1 << 17]byte +var z5264 [1 << 17]byte +var z5265 [1 << 17]byte +var z5266 [1 << 17]byte +var z5267 [1 << 17]byte +var z5268 [1 << 17]byte +var z5269 [1 << 17]byte +var z5270 [1 << 17]byte +var z5271 [1 << 17]byte +var z5272 [1 << 17]byte +var z5273 [1 << 17]byte +var z5274 [1 << 17]byte +var z5275 [1 << 17]byte +var z5276 [1 << 17]byte +var z5277 [1 << 17]byte +var z5278 [1 << 17]byte +var z5279 [1 << 17]byte +var z5280 [1 << 17]byte +var z5281 [1 << 17]byte +var z5282 [1 << 17]byte +var z5283 [1 << 17]byte +var z5284 [1 << 17]byte +var z5285 [1 << 17]byte +var z5286 [1 << 17]byte +var z5287 [1 << 17]byte +var z5288 [1 << 17]byte +var z5289 [1 << 17]byte +var z5290 [1 << 17]byte +var z5291 [1 << 17]byte +var z5292 [1 << 17]byte +var z5293 [1 << 17]byte +var z5294 [1 << 17]byte +var z5295 [1 << 17]byte +var z5296 [1 << 17]byte +var z5297 [1 << 17]byte +var z5298 [1 << 17]byte +var z5299 [1 << 17]byte +var z5300 [1 << 17]byte +var z5301 [1 << 17]byte +var z5302 [1 << 17]byte +var z5303 [1 << 17]byte +var z5304 [1 << 17]byte +var z5305 [1 << 17]byte +var z5306 [1 << 17]byte +var z5307 [1 << 17]byte +var z5308 [1 << 17]byte +var z5309 [1 << 17]byte +var z5310 [1 << 17]byte +var z5311 [1 << 17]byte +var z5312 [1 << 17]byte +var z5313 [1 << 17]byte +var z5314 [1 << 17]byte +var z5315 [1 << 17]byte +var z5316 [1 << 17]byte +var z5317 [1 << 17]byte +var z5318 [1 << 17]byte +var z5319 [1 << 17]byte +var z5320 [1 << 17]byte +var z5321 [1 << 17]byte +var z5322 [1 << 17]byte +var z5323 [1 << 17]byte +var z5324 [1 << 17]byte +var z5325 [1 << 17]byte +var z5326 [1 << 17]byte +var z5327 [1 << 17]byte +var z5328 [1 << 17]byte +var z5329 [1 << 17]byte +var z5330 [1 << 17]byte +var z5331 [1 << 17]byte +var z5332 [1 << 17]byte +var z5333 [1 << 17]byte +var z5334 [1 << 17]byte +var z5335 [1 << 17]byte +var z5336 [1 << 17]byte +var z5337 [1 << 17]byte +var z5338 [1 << 17]byte +var z5339 [1 << 17]byte +var z5340 [1 << 17]byte +var z5341 [1 << 17]byte +var z5342 [1 << 17]byte +var z5343 [1 << 17]byte +var z5344 [1 << 17]byte +var z5345 [1 << 17]byte +var z5346 [1 << 17]byte +var z5347 [1 << 17]byte +var z5348 [1 << 17]byte +var z5349 [1 << 17]byte +var z5350 [1 << 17]byte +var z5351 [1 << 17]byte +var z5352 [1 << 17]byte +var z5353 [1 << 17]byte +var z5354 [1 << 17]byte +var z5355 [1 << 17]byte +var z5356 [1 << 17]byte +var z5357 [1 << 17]byte +var z5358 [1 << 17]byte +var z5359 [1 << 17]byte +var z5360 [1 << 17]byte +var z5361 [1 << 17]byte +var z5362 [1 << 17]byte +var z5363 [1 << 17]byte +var z5364 [1 << 17]byte +var z5365 [1 << 17]byte +var z5366 [1 << 17]byte +var z5367 [1 << 17]byte +var z5368 [1 << 17]byte +var z5369 [1 << 17]byte +var z5370 [1 << 17]byte +var z5371 [1 << 17]byte +var z5372 [1 << 17]byte +var z5373 [1 << 17]byte +var z5374 [1 << 17]byte +var z5375 [1 << 17]byte +var z5376 [1 << 17]byte +var z5377 [1 << 17]byte +var z5378 [1 << 17]byte +var z5379 [1 << 17]byte +var z5380 [1 << 17]byte +var z5381 [1 << 17]byte +var z5382 [1 << 17]byte +var z5383 [1 << 17]byte +var z5384 [1 << 17]byte +var z5385 [1 << 17]byte +var z5386 [1 << 17]byte +var z5387 [1 << 17]byte +var z5388 [1 << 17]byte +var z5389 [1 << 17]byte +var z5390 [1 << 17]byte +var z5391 [1 << 17]byte +var z5392 [1 << 17]byte +var z5393 [1 << 17]byte +var z5394 [1 << 17]byte +var z5395 [1 << 17]byte +var z5396 [1 << 17]byte +var z5397 [1 << 17]byte +var z5398 [1 << 17]byte +var z5399 [1 << 17]byte +var z5400 [1 << 17]byte +var z5401 [1 << 17]byte +var z5402 [1 << 17]byte +var z5403 [1 << 17]byte +var z5404 [1 << 17]byte +var z5405 [1 << 17]byte +var z5406 [1 << 17]byte +var z5407 [1 << 17]byte +var z5408 [1 << 17]byte +var z5409 [1 << 17]byte +var z5410 [1 << 17]byte +var z5411 [1 << 17]byte +var z5412 [1 << 17]byte +var z5413 [1 << 17]byte +var z5414 [1 << 17]byte +var z5415 [1 << 17]byte +var z5416 [1 << 17]byte +var z5417 [1 << 17]byte +var z5418 [1 << 17]byte +var z5419 [1 << 17]byte +var z5420 [1 << 17]byte +var z5421 [1 << 17]byte +var z5422 [1 << 17]byte +var z5423 [1 << 17]byte +var z5424 [1 << 17]byte +var z5425 [1 << 17]byte +var z5426 [1 << 17]byte +var z5427 [1 << 17]byte +var z5428 [1 << 17]byte +var z5429 [1 << 17]byte +var z5430 [1 << 17]byte +var z5431 [1 << 17]byte +var z5432 [1 << 17]byte +var z5433 [1 << 17]byte +var z5434 [1 << 17]byte +var z5435 [1 << 17]byte +var z5436 [1 << 17]byte +var z5437 [1 << 17]byte +var z5438 [1 << 17]byte +var z5439 [1 << 17]byte +var z5440 [1 << 17]byte +var z5441 [1 << 17]byte +var z5442 [1 << 17]byte +var z5443 [1 << 17]byte +var z5444 [1 << 17]byte +var z5445 [1 << 17]byte +var z5446 [1 << 17]byte +var z5447 [1 << 17]byte +var z5448 [1 << 17]byte +var z5449 [1 << 17]byte +var z5450 [1 << 17]byte +var z5451 [1 << 17]byte +var z5452 [1 << 17]byte +var z5453 [1 << 17]byte +var z5454 [1 << 17]byte +var z5455 [1 << 17]byte +var z5456 [1 << 17]byte +var z5457 [1 << 17]byte +var z5458 [1 << 17]byte +var z5459 [1 << 17]byte +var z5460 [1 << 17]byte +var z5461 [1 << 17]byte +var z5462 [1 << 17]byte +var z5463 [1 << 17]byte +var z5464 [1 << 17]byte +var z5465 [1 << 17]byte +var z5466 [1 << 17]byte +var z5467 [1 << 17]byte +var z5468 [1 << 17]byte +var z5469 [1 << 17]byte +var z5470 [1 << 17]byte +var z5471 [1 << 17]byte +var z5472 [1 << 17]byte +var z5473 [1 << 17]byte +var z5474 [1 << 17]byte +var z5475 [1 << 17]byte +var z5476 [1 << 17]byte +var z5477 [1 << 17]byte +var z5478 [1 << 17]byte +var z5479 [1 << 17]byte +var z5480 [1 << 17]byte +var z5481 [1 << 17]byte +var z5482 [1 << 17]byte +var z5483 [1 << 17]byte +var z5484 [1 << 17]byte +var z5485 [1 << 17]byte +var z5486 [1 << 17]byte +var z5487 [1 << 17]byte +var z5488 [1 << 17]byte +var z5489 [1 << 17]byte +var z5490 [1 << 17]byte +var z5491 [1 << 17]byte +var z5492 [1 << 17]byte +var z5493 [1 << 17]byte +var z5494 [1 << 17]byte +var z5495 [1 << 17]byte +var z5496 [1 << 17]byte +var z5497 [1 << 17]byte +var z5498 [1 << 17]byte +var z5499 [1 << 17]byte +var z5500 [1 << 17]byte +var z5501 [1 << 17]byte +var z5502 [1 << 17]byte +var z5503 [1 << 17]byte +var z5504 [1 << 17]byte +var z5505 [1 << 17]byte +var z5506 [1 << 17]byte +var z5507 [1 << 17]byte +var z5508 [1 << 17]byte +var z5509 [1 << 17]byte +var z5510 [1 << 17]byte +var z5511 [1 << 17]byte +var z5512 [1 << 17]byte +var z5513 [1 << 17]byte +var z5514 [1 << 17]byte +var z5515 [1 << 17]byte +var z5516 [1 << 17]byte +var z5517 [1 << 17]byte +var z5518 [1 << 17]byte +var z5519 [1 << 17]byte +var z5520 [1 << 17]byte +var z5521 [1 << 17]byte +var z5522 [1 << 17]byte +var z5523 [1 << 17]byte +var z5524 [1 << 17]byte +var z5525 [1 << 17]byte +var z5526 [1 << 17]byte +var z5527 [1 << 17]byte +var z5528 [1 << 17]byte +var z5529 [1 << 17]byte +var z5530 [1 << 17]byte +var z5531 [1 << 17]byte +var z5532 [1 << 17]byte +var z5533 [1 << 17]byte +var z5534 [1 << 17]byte +var z5535 [1 << 17]byte +var z5536 [1 << 17]byte +var z5537 [1 << 17]byte +var z5538 [1 << 17]byte +var z5539 [1 << 17]byte +var z5540 [1 << 17]byte +var z5541 [1 << 17]byte +var z5542 [1 << 17]byte +var z5543 [1 << 17]byte +var z5544 [1 << 17]byte +var z5545 [1 << 17]byte +var z5546 [1 << 17]byte +var z5547 [1 << 17]byte +var z5548 [1 << 17]byte +var z5549 [1 << 17]byte +var z5550 [1 << 17]byte +var z5551 [1 << 17]byte +var z5552 [1 << 17]byte +var z5553 [1 << 17]byte +var z5554 [1 << 17]byte +var z5555 [1 << 17]byte +var z5556 [1 << 17]byte +var z5557 [1 << 17]byte +var z5558 [1 << 17]byte +var z5559 [1 << 17]byte +var z5560 [1 << 17]byte +var z5561 [1 << 17]byte +var z5562 [1 << 17]byte +var z5563 [1 << 17]byte +var z5564 [1 << 17]byte +var z5565 [1 << 17]byte +var z5566 [1 << 17]byte +var z5567 [1 << 17]byte +var z5568 [1 << 17]byte +var z5569 [1 << 17]byte +var z5570 [1 << 17]byte +var z5571 [1 << 17]byte +var z5572 [1 << 17]byte +var z5573 [1 << 17]byte +var z5574 [1 << 17]byte +var z5575 [1 << 17]byte +var z5576 [1 << 17]byte +var z5577 [1 << 17]byte +var z5578 [1 << 17]byte +var z5579 [1 << 17]byte +var z5580 [1 << 17]byte +var z5581 [1 << 17]byte +var z5582 [1 << 17]byte +var z5583 [1 << 17]byte +var z5584 [1 << 17]byte +var z5585 [1 << 17]byte +var z5586 [1 << 17]byte +var z5587 [1 << 17]byte +var z5588 [1 << 17]byte +var z5589 [1 << 17]byte +var z5590 [1 << 17]byte +var z5591 [1 << 17]byte +var z5592 [1 << 17]byte +var z5593 [1 << 17]byte +var z5594 [1 << 17]byte +var z5595 [1 << 17]byte +var z5596 [1 << 17]byte +var z5597 [1 << 17]byte +var z5598 [1 << 17]byte +var z5599 [1 << 17]byte +var z5600 [1 << 17]byte +var z5601 [1 << 17]byte +var z5602 [1 << 17]byte +var z5603 [1 << 17]byte +var z5604 [1 << 17]byte +var z5605 [1 << 17]byte +var z5606 [1 << 17]byte +var z5607 [1 << 17]byte +var z5608 [1 << 17]byte +var z5609 [1 << 17]byte +var z5610 [1 << 17]byte +var z5611 [1 << 17]byte +var z5612 [1 << 17]byte +var z5613 [1 << 17]byte +var z5614 [1 << 17]byte +var z5615 [1 << 17]byte +var z5616 [1 << 17]byte +var z5617 [1 << 17]byte +var z5618 [1 << 17]byte +var z5619 [1 << 17]byte +var z5620 [1 << 17]byte +var z5621 [1 << 17]byte +var z5622 [1 << 17]byte +var z5623 [1 << 17]byte +var z5624 [1 << 17]byte +var z5625 [1 << 17]byte +var z5626 [1 << 17]byte +var z5627 [1 << 17]byte +var z5628 [1 << 17]byte +var z5629 [1 << 17]byte +var z5630 [1 << 17]byte +var z5631 [1 << 17]byte +var z5632 [1 << 17]byte +var z5633 [1 << 17]byte +var z5634 [1 << 17]byte +var z5635 [1 << 17]byte +var z5636 [1 << 17]byte +var z5637 [1 << 17]byte +var z5638 [1 << 17]byte +var z5639 [1 << 17]byte +var z5640 [1 << 17]byte +var z5641 [1 << 17]byte +var z5642 [1 << 17]byte +var z5643 [1 << 17]byte +var z5644 [1 << 17]byte +var z5645 [1 << 17]byte +var z5646 [1 << 17]byte +var z5647 [1 << 17]byte +var z5648 [1 << 17]byte +var z5649 [1 << 17]byte +var z5650 [1 << 17]byte +var z5651 [1 << 17]byte +var z5652 [1 << 17]byte +var z5653 [1 << 17]byte +var z5654 [1 << 17]byte +var z5655 [1 << 17]byte +var z5656 [1 << 17]byte +var z5657 [1 << 17]byte +var z5658 [1 << 17]byte +var z5659 [1 << 17]byte +var z5660 [1 << 17]byte +var z5661 [1 << 17]byte +var z5662 [1 << 17]byte +var z5663 [1 << 17]byte +var z5664 [1 << 17]byte +var z5665 [1 << 17]byte +var z5666 [1 << 17]byte +var z5667 [1 << 17]byte +var z5668 [1 << 17]byte +var z5669 [1 << 17]byte +var z5670 [1 << 17]byte +var z5671 [1 << 17]byte +var z5672 [1 << 17]byte +var z5673 [1 << 17]byte +var z5674 [1 << 17]byte +var z5675 [1 << 17]byte +var z5676 [1 << 17]byte +var z5677 [1 << 17]byte +var z5678 [1 << 17]byte +var z5679 [1 << 17]byte +var z5680 [1 << 17]byte +var z5681 [1 << 17]byte +var z5682 [1 << 17]byte +var z5683 [1 << 17]byte +var z5684 [1 << 17]byte +var z5685 [1 << 17]byte +var z5686 [1 << 17]byte +var z5687 [1 << 17]byte +var z5688 [1 << 17]byte +var z5689 [1 << 17]byte +var z5690 [1 << 17]byte +var z5691 [1 << 17]byte +var z5692 [1 << 17]byte +var z5693 [1 << 17]byte +var z5694 [1 << 17]byte +var z5695 [1 << 17]byte +var z5696 [1 << 17]byte +var z5697 [1 << 17]byte +var z5698 [1 << 17]byte +var z5699 [1 << 17]byte +var z5700 [1 << 17]byte +var z5701 [1 << 17]byte +var z5702 [1 << 17]byte +var z5703 [1 << 17]byte +var z5704 [1 << 17]byte +var z5705 [1 << 17]byte +var z5706 [1 << 17]byte +var z5707 [1 << 17]byte +var z5708 [1 << 17]byte +var z5709 [1 << 17]byte +var z5710 [1 << 17]byte +var z5711 [1 << 17]byte +var z5712 [1 << 17]byte +var z5713 [1 << 17]byte +var z5714 [1 << 17]byte +var z5715 [1 << 17]byte +var z5716 [1 << 17]byte +var z5717 [1 << 17]byte +var z5718 [1 << 17]byte +var z5719 [1 << 17]byte +var z5720 [1 << 17]byte +var z5721 [1 << 17]byte +var z5722 [1 << 17]byte +var z5723 [1 << 17]byte +var z5724 [1 << 17]byte +var z5725 [1 << 17]byte +var z5726 [1 << 17]byte +var z5727 [1 << 17]byte +var z5728 [1 << 17]byte +var z5729 [1 << 17]byte +var z5730 [1 << 17]byte +var z5731 [1 << 17]byte +var z5732 [1 << 17]byte +var z5733 [1 << 17]byte +var z5734 [1 << 17]byte +var z5735 [1 << 17]byte +var z5736 [1 << 17]byte +var z5737 [1 << 17]byte +var z5738 [1 << 17]byte +var z5739 [1 << 17]byte +var z5740 [1 << 17]byte +var z5741 [1 << 17]byte +var z5742 [1 << 17]byte +var z5743 [1 << 17]byte +var z5744 [1 << 17]byte +var z5745 [1 << 17]byte +var z5746 [1 << 17]byte +var z5747 [1 << 17]byte +var z5748 [1 << 17]byte +var z5749 [1 << 17]byte +var z5750 [1 << 17]byte +var z5751 [1 << 17]byte +var z5752 [1 << 17]byte +var z5753 [1 << 17]byte +var z5754 [1 << 17]byte +var z5755 [1 << 17]byte +var z5756 [1 << 17]byte +var z5757 [1 << 17]byte +var z5758 [1 << 17]byte +var z5759 [1 << 17]byte +var z5760 [1 << 17]byte +var z5761 [1 << 17]byte +var z5762 [1 << 17]byte +var z5763 [1 << 17]byte +var z5764 [1 << 17]byte +var z5765 [1 << 17]byte +var z5766 [1 << 17]byte +var z5767 [1 << 17]byte +var z5768 [1 << 17]byte +var z5769 [1 << 17]byte +var z5770 [1 << 17]byte +var z5771 [1 << 17]byte +var z5772 [1 << 17]byte +var z5773 [1 << 17]byte +var z5774 [1 << 17]byte +var z5775 [1 << 17]byte +var z5776 [1 << 17]byte +var z5777 [1 << 17]byte +var z5778 [1 << 17]byte +var z5779 [1 << 17]byte +var z5780 [1 << 17]byte +var z5781 [1 << 17]byte +var z5782 [1 << 17]byte +var z5783 [1 << 17]byte +var z5784 [1 << 17]byte +var z5785 [1 << 17]byte +var z5786 [1 << 17]byte +var z5787 [1 << 17]byte +var z5788 [1 << 17]byte +var z5789 [1 << 17]byte +var z5790 [1 << 17]byte +var z5791 [1 << 17]byte +var z5792 [1 << 17]byte +var z5793 [1 << 17]byte +var z5794 [1 << 17]byte +var z5795 [1 << 17]byte +var z5796 [1 << 17]byte +var z5797 [1 << 17]byte +var z5798 [1 << 17]byte +var z5799 [1 << 17]byte +var z5800 [1 << 17]byte +var z5801 [1 << 17]byte +var z5802 [1 << 17]byte +var z5803 [1 << 17]byte +var z5804 [1 << 17]byte +var z5805 [1 << 17]byte +var z5806 [1 << 17]byte +var z5807 [1 << 17]byte +var z5808 [1 << 17]byte +var z5809 [1 << 17]byte +var z5810 [1 << 17]byte +var z5811 [1 << 17]byte +var z5812 [1 << 17]byte +var z5813 [1 << 17]byte +var z5814 [1 << 17]byte +var z5815 [1 << 17]byte +var z5816 [1 << 17]byte +var z5817 [1 << 17]byte +var z5818 [1 << 17]byte +var z5819 [1 << 17]byte +var z5820 [1 << 17]byte +var z5821 [1 << 17]byte +var z5822 [1 << 17]byte +var z5823 [1 << 17]byte +var z5824 [1 << 17]byte +var z5825 [1 << 17]byte +var z5826 [1 << 17]byte +var z5827 [1 << 17]byte +var z5828 [1 << 17]byte +var z5829 [1 << 17]byte +var z5830 [1 << 17]byte +var z5831 [1 << 17]byte +var z5832 [1 << 17]byte +var z5833 [1 << 17]byte +var z5834 [1 << 17]byte +var z5835 [1 << 17]byte +var z5836 [1 << 17]byte +var z5837 [1 << 17]byte +var z5838 [1 << 17]byte +var z5839 [1 << 17]byte +var z5840 [1 << 17]byte +var z5841 [1 << 17]byte +var z5842 [1 << 17]byte +var z5843 [1 << 17]byte +var z5844 [1 << 17]byte +var z5845 [1 << 17]byte +var z5846 [1 << 17]byte +var z5847 [1 << 17]byte +var z5848 [1 << 17]byte +var z5849 [1 << 17]byte +var z5850 [1 << 17]byte +var z5851 [1 << 17]byte +var z5852 [1 << 17]byte +var z5853 [1 << 17]byte +var z5854 [1 << 17]byte +var z5855 [1 << 17]byte +var z5856 [1 << 17]byte +var z5857 [1 << 17]byte +var z5858 [1 << 17]byte +var z5859 [1 << 17]byte +var z5860 [1 << 17]byte +var z5861 [1 << 17]byte +var z5862 [1 << 17]byte +var z5863 [1 << 17]byte +var z5864 [1 << 17]byte +var z5865 [1 << 17]byte +var z5866 [1 << 17]byte +var z5867 [1 << 17]byte +var z5868 [1 << 17]byte +var z5869 [1 << 17]byte +var z5870 [1 << 17]byte +var z5871 [1 << 17]byte +var z5872 [1 << 17]byte +var z5873 [1 << 17]byte +var z5874 [1 << 17]byte +var z5875 [1 << 17]byte +var z5876 [1 << 17]byte +var z5877 [1 << 17]byte +var z5878 [1 << 17]byte +var z5879 [1 << 17]byte +var z5880 [1 << 17]byte +var z5881 [1 << 17]byte +var z5882 [1 << 17]byte +var z5883 [1 << 17]byte +var z5884 [1 << 17]byte +var z5885 [1 << 17]byte +var z5886 [1 << 17]byte +var z5887 [1 << 17]byte +var z5888 [1 << 17]byte +var z5889 [1 << 17]byte +var z5890 [1 << 17]byte +var z5891 [1 << 17]byte +var z5892 [1 << 17]byte +var z5893 [1 << 17]byte +var z5894 [1 << 17]byte +var z5895 [1 << 17]byte +var z5896 [1 << 17]byte +var z5897 [1 << 17]byte +var z5898 [1 << 17]byte +var z5899 [1 << 17]byte +var z5900 [1 << 17]byte +var z5901 [1 << 17]byte +var z5902 [1 << 17]byte +var z5903 [1 << 17]byte +var z5904 [1 << 17]byte +var z5905 [1 << 17]byte +var z5906 [1 << 17]byte +var z5907 [1 << 17]byte +var z5908 [1 << 17]byte +var z5909 [1 << 17]byte +var z5910 [1 << 17]byte +var z5911 [1 << 17]byte +var z5912 [1 << 17]byte +var z5913 [1 << 17]byte +var z5914 [1 << 17]byte +var z5915 [1 << 17]byte +var z5916 [1 << 17]byte +var z5917 [1 << 17]byte +var z5918 [1 << 17]byte +var z5919 [1 << 17]byte +var z5920 [1 << 17]byte +var z5921 [1 << 17]byte +var z5922 [1 << 17]byte +var z5923 [1 << 17]byte +var z5924 [1 << 17]byte +var z5925 [1 << 17]byte +var z5926 [1 << 17]byte +var z5927 [1 << 17]byte +var z5928 [1 << 17]byte +var z5929 [1 << 17]byte +var z5930 [1 << 17]byte +var z5931 [1 << 17]byte +var z5932 [1 << 17]byte +var z5933 [1 << 17]byte +var z5934 [1 << 17]byte +var z5935 [1 << 17]byte +var z5936 [1 << 17]byte +var z5937 [1 << 17]byte +var z5938 [1 << 17]byte +var z5939 [1 << 17]byte +var z5940 [1 << 17]byte +var z5941 [1 << 17]byte +var z5942 [1 << 17]byte +var z5943 [1 << 17]byte +var z5944 [1 << 17]byte +var z5945 [1 << 17]byte +var z5946 [1 << 17]byte +var z5947 [1 << 17]byte +var z5948 [1 << 17]byte +var z5949 [1 << 17]byte +var z5950 [1 << 17]byte +var z5951 [1 << 17]byte +var z5952 [1 << 17]byte +var z5953 [1 << 17]byte +var z5954 [1 << 17]byte +var z5955 [1 << 17]byte +var z5956 [1 << 17]byte +var z5957 [1 << 17]byte +var z5958 [1 << 17]byte +var z5959 [1 << 17]byte +var z5960 [1 << 17]byte +var z5961 [1 << 17]byte +var z5962 [1 << 17]byte +var z5963 [1 << 17]byte +var z5964 [1 << 17]byte +var z5965 [1 << 17]byte +var z5966 [1 << 17]byte +var z5967 [1 << 17]byte +var z5968 [1 << 17]byte +var z5969 [1 << 17]byte +var z5970 [1 << 17]byte +var z5971 [1 << 17]byte +var z5972 [1 << 17]byte +var z5973 [1 << 17]byte +var z5974 [1 << 17]byte +var z5975 [1 << 17]byte +var z5976 [1 << 17]byte +var z5977 [1 << 17]byte +var z5978 [1 << 17]byte +var z5979 [1 << 17]byte +var z5980 [1 << 17]byte +var z5981 [1 << 17]byte +var z5982 [1 << 17]byte +var z5983 [1 << 17]byte +var z5984 [1 << 17]byte +var z5985 [1 << 17]byte +var z5986 [1 << 17]byte +var z5987 [1 << 17]byte +var z5988 [1 << 17]byte +var z5989 [1 << 17]byte +var z5990 [1 << 17]byte +var z5991 [1 << 17]byte +var z5992 [1 << 17]byte +var z5993 [1 << 17]byte +var z5994 [1 << 17]byte +var z5995 [1 << 17]byte +var z5996 [1 << 17]byte +var z5997 [1 << 17]byte +var z5998 [1 << 17]byte +var z5999 [1 << 17]byte +var z6000 [1 << 17]byte +var z6001 [1 << 17]byte +var z6002 [1 << 17]byte +var z6003 [1 << 17]byte +var z6004 [1 << 17]byte +var z6005 [1 << 17]byte +var z6006 [1 << 17]byte +var z6007 [1 << 17]byte +var z6008 [1 << 17]byte +var z6009 [1 << 17]byte +var z6010 [1 << 17]byte +var z6011 [1 << 17]byte +var z6012 [1 << 17]byte +var z6013 [1 << 17]byte +var z6014 [1 << 17]byte +var z6015 [1 << 17]byte +var z6016 [1 << 17]byte +var z6017 [1 << 17]byte +var z6018 [1 << 17]byte +var z6019 [1 << 17]byte +var z6020 [1 << 17]byte +var z6021 [1 << 17]byte +var z6022 [1 << 17]byte +var z6023 [1 << 17]byte +var z6024 [1 << 17]byte +var z6025 [1 << 17]byte +var z6026 [1 << 17]byte +var z6027 [1 << 17]byte +var z6028 [1 << 17]byte +var z6029 [1 << 17]byte +var z6030 [1 << 17]byte +var z6031 [1 << 17]byte +var z6032 [1 << 17]byte +var z6033 [1 << 17]byte +var z6034 [1 << 17]byte +var z6035 [1 << 17]byte +var z6036 [1 << 17]byte +var z6037 [1 << 17]byte +var z6038 [1 << 17]byte +var z6039 [1 << 17]byte +var z6040 [1 << 17]byte +var z6041 [1 << 17]byte +var z6042 [1 << 17]byte +var z6043 [1 << 17]byte +var z6044 [1 << 17]byte +var z6045 [1 << 17]byte +var z6046 [1 << 17]byte +var z6047 [1 << 17]byte +var z6048 [1 << 17]byte +var z6049 [1 << 17]byte +var z6050 [1 << 17]byte +var z6051 [1 << 17]byte +var z6052 [1 << 17]byte +var z6053 [1 << 17]byte +var z6054 [1 << 17]byte +var z6055 [1 << 17]byte +var z6056 [1 << 17]byte +var z6057 [1 << 17]byte +var z6058 [1 << 17]byte +var z6059 [1 << 17]byte +var z6060 [1 << 17]byte +var z6061 [1 << 17]byte +var z6062 [1 << 17]byte +var z6063 [1 << 17]byte +var z6064 [1 << 17]byte +var z6065 [1 << 17]byte +var z6066 [1 << 17]byte +var z6067 [1 << 17]byte +var z6068 [1 << 17]byte +var z6069 [1 << 17]byte +var z6070 [1 << 17]byte +var z6071 [1 << 17]byte +var z6072 [1 << 17]byte +var z6073 [1 << 17]byte +var z6074 [1 << 17]byte +var z6075 [1 << 17]byte +var z6076 [1 << 17]byte +var z6077 [1 << 17]byte +var z6078 [1 << 17]byte +var z6079 [1 << 17]byte +var z6080 [1 << 17]byte +var z6081 [1 << 17]byte +var z6082 [1 << 17]byte +var z6083 [1 << 17]byte +var z6084 [1 << 17]byte +var z6085 [1 << 17]byte +var z6086 [1 << 17]byte +var z6087 [1 << 17]byte +var z6088 [1 << 17]byte +var z6089 [1 << 17]byte +var z6090 [1 << 17]byte +var z6091 [1 << 17]byte +var z6092 [1 << 17]byte +var z6093 [1 << 17]byte +var z6094 [1 << 17]byte +var z6095 [1 << 17]byte +var z6096 [1 << 17]byte +var z6097 [1 << 17]byte +var z6098 [1 << 17]byte +var z6099 [1 << 17]byte +var z6100 [1 << 17]byte +var z6101 [1 << 17]byte +var z6102 [1 << 17]byte +var z6103 [1 << 17]byte +var z6104 [1 << 17]byte +var z6105 [1 << 17]byte +var z6106 [1 << 17]byte +var z6107 [1 << 17]byte +var z6108 [1 << 17]byte +var z6109 [1 << 17]byte +var z6110 [1 << 17]byte +var z6111 [1 << 17]byte +var z6112 [1 << 17]byte +var z6113 [1 << 17]byte +var z6114 [1 << 17]byte +var z6115 [1 << 17]byte +var z6116 [1 << 17]byte +var z6117 [1 << 17]byte +var z6118 [1 << 17]byte +var z6119 [1 << 17]byte +var z6120 [1 << 17]byte +var z6121 [1 << 17]byte +var z6122 [1 << 17]byte +var z6123 [1 << 17]byte +var z6124 [1 << 17]byte +var z6125 [1 << 17]byte +var z6126 [1 << 17]byte +var z6127 [1 << 17]byte +var z6128 [1 << 17]byte +var z6129 [1 << 17]byte +var z6130 [1 << 17]byte +var z6131 [1 << 17]byte +var z6132 [1 << 17]byte +var z6133 [1 << 17]byte +var z6134 [1 << 17]byte +var z6135 [1 << 17]byte +var z6136 [1 << 17]byte +var z6137 [1 << 17]byte +var z6138 [1 << 17]byte +var z6139 [1 << 17]byte +var z6140 [1 << 17]byte +var z6141 [1 << 17]byte +var z6142 [1 << 17]byte +var z6143 [1 << 17]byte +var z6144 [1 << 17]byte +var z6145 [1 << 17]byte +var z6146 [1 << 17]byte +var z6147 [1 << 17]byte +var z6148 [1 << 17]byte +var z6149 [1 << 17]byte +var z6150 [1 << 17]byte +var z6151 [1 << 17]byte +var z6152 [1 << 17]byte +var z6153 [1 << 17]byte +var z6154 [1 << 17]byte +var z6155 [1 << 17]byte +var z6156 [1 << 17]byte +var z6157 [1 << 17]byte +var z6158 [1 << 17]byte +var z6159 [1 << 17]byte +var z6160 [1 << 17]byte +var z6161 [1 << 17]byte +var z6162 [1 << 17]byte +var z6163 [1 << 17]byte +var z6164 [1 << 17]byte +var z6165 [1 << 17]byte +var z6166 [1 << 17]byte +var z6167 [1 << 17]byte +var z6168 [1 << 17]byte +var z6169 [1 << 17]byte +var z6170 [1 << 17]byte +var z6171 [1 << 17]byte +var z6172 [1 << 17]byte +var z6173 [1 << 17]byte +var z6174 [1 << 17]byte +var z6175 [1 << 17]byte +var z6176 [1 << 17]byte +var z6177 [1 << 17]byte +var z6178 [1 << 17]byte +var z6179 [1 << 17]byte +var z6180 [1 << 17]byte +var z6181 [1 << 17]byte +var z6182 [1 << 17]byte +var z6183 [1 << 17]byte +var z6184 [1 << 17]byte +var z6185 [1 << 17]byte +var z6186 [1 << 17]byte +var z6187 [1 << 17]byte +var z6188 [1 << 17]byte +var z6189 [1 << 17]byte +var z6190 [1 << 17]byte +var z6191 [1 << 17]byte +var z6192 [1 << 17]byte +var z6193 [1 << 17]byte +var z6194 [1 << 17]byte +var z6195 [1 << 17]byte +var z6196 [1 << 17]byte +var z6197 [1 << 17]byte +var z6198 [1 << 17]byte +var z6199 [1 << 17]byte +var z6200 [1 << 17]byte +var z6201 [1 << 17]byte +var z6202 [1 << 17]byte +var z6203 [1 << 17]byte +var z6204 [1 << 17]byte +var z6205 [1 << 17]byte +var z6206 [1 << 17]byte +var z6207 [1 << 17]byte +var z6208 [1 << 17]byte +var z6209 [1 << 17]byte +var z6210 [1 << 17]byte +var z6211 [1 << 17]byte +var z6212 [1 << 17]byte +var z6213 [1 << 17]byte +var z6214 [1 << 17]byte +var z6215 [1 << 17]byte +var z6216 [1 << 17]byte +var z6217 [1 << 17]byte +var z6218 [1 << 17]byte +var z6219 [1 << 17]byte +var z6220 [1 << 17]byte +var z6221 [1 << 17]byte +var z6222 [1 << 17]byte +var z6223 [1 << 17]byte +var z6224 [1 << 17]byte +var z6225 [1 << 17]byte +var z6226 [1 << 17]byte +var z6227 [1 << 17]byte +var z6228 [1 << 17]byte +var z6229 [1 << 17]byte +var z6230 [1 << 17]byte +var z6231 [1 << 17]byte +var z6232 [1 << 17]byte +var z6233 [1 << 17]byte +var z6234 [1 << 17]byte +var z6235 [1 << 17]byte +var z6236 [1 << 17]byte +var z6237 [1 << 17]byte +var z6238 [1 << 17]byte +var z6239 [1 << 17]byte +var z6240 [1 << 17]byte +var z6241 [1 << 17]byte +var z6242 [1 << 17]byte +var z6243 [1 << 17]byte +var z6244 [1 << 17]byte +var z6245 [1 << 17]byte +var z6246 [1 << 17]byte +var z6247 [1 << 17]byte +var z6248 [1 << 17]byte +var z6249 [1 << 17]byte +var z6250 [1 << 17]byte +var z6251 [1 << 17]byte +var z6252 [1 << 17]byte +var z6253 [1 << 17]byte +var z6254 [1 << 17]byte +var z6255 [1 << 17]byte +var z6256 [1 << 17]byte +var z6257 [1 << 17]byte +var z6258 [1 << 17]byte +var z6259 [1 << 17]byte +var z6260 [1 << 17]byte +var z6261 [1 << 17]byte +var z6262 [1 << 17]byte +var z6263 [1 << 17]byte +var z6264 [1 << 17]byte +var z6265 [1 << 17]byte +var z6266 [1 << 17]byte +var z6267 [1 << 17]byte +var z6268 [1 << 17]byte +var z6269 [1 << 17]byte +var z6270 [1 << 17]byte +var z6271 [1 << 17]byte +var z6272 [1 << 17]byte +var z6273 [1 << 17]byte +var z6274 [1 << 17]byte +var z6275 [1 << 17]byte +var z6276 [1 << 17]byte +var z6277 [1 << 17]byte +var z6278 [1 << 17]byte +var z6279 [1 << 17]byte +var z6280 [1 << 17]byte +var z6281 [1 << 17]byte +var z6282 [1 << 17]byte +var z6283 [1 << 17]byte +var z6284 [1 << 17]byte +var z6285 [1 << 17]byte +var z6286 [1 << 17]byte +var z6287 [1 << 17]byte +var z6288 [1 << 17]byte +var z6289 [1 << 17]byte +var z6290 [1 << 17]byte +var z6291 [1 << 17]byte +var z6292 [1 << 17]byte +var z6293 [1 << 17]byte +var z6294 [1 << 17]byte +var z6295 [1 << 17]byte +var z6296 [1 << 17]byte +var z6297 [1 << 17]byte +var z6298 [1 << 17]byte +var z6299 [1 << 17]byte +var z6300 [1 << 17]byte +var z6301 [1 << 17]byte +var z6302 [1 << 17]byte +var z6303 [1 << 17]byte +var z6304 [1 << 17]byte +var z6305 [1 << 17]byte +var z6306 [1 << 17]byte +var z6307 [1 << 17]byte +var z6308 [1 << 17]byte +var z6309 [1 << 17]byte +var z6310 [1 << 17]byte +var z6311 [1 << 17]byte +var z6312 [1 << 17]byte +var z6313 [1 << 17]byte +var z6314 [1 << 17]byte +var z6315 [1 << 17]byte +var z6316 [1 << 17]byte +var z6317 [1 << 17]byte +var z6318 [1 << 17]byte +var z6319 [1 << 17]byte +var z6320 [1 << 17]byte +var z6321 [1 << 17]byte +var z6322 [1 << 17]byte +var z6323 [1 << 17]byte +var z6324 [1 << 17]byte +var z6325 [1 << 17]byte +var z6326 [1 << 17]byte +var z6327 [1 << 17]byte +var z6328 [1 << 17]byte +var z6329 [1 << 17]byte +var z6330 [1 << 17]byte +var z6331 [1 << 17]byte +var z6332 [1 << 17]byte +var z6333 [1 << 17]byte +var z6334 [1 << 17]byte +var z6335 [1 << 17]byte +var z6336 [1 << 17]byte +var z6337 [1 << 17]byte +var z6338 [1 << 17]byte +var z6339 [1 << 17]byte +var z6340 [1 << 17]byte +var z6341 [1 << 17]byte +var z6342 [1 << 17]byte +var z6343 [1 << 17]byte +var z6344 [1 << 17]byte +var z6345 [1 << 17]byte +var z6346 [1 << 17]byte +var z6347 [1 << 17]byte +var z6348 [1 << 17]byte +var z6349 [1 << 17]byte +var z6350 [1 << 17]byte +var z6351 [1 << 17]byte +var z6352 [1 << 17]byte +var z6353 [1 << 17]byte +var z6354 [1 << 17]byte +var z6355 [1 << 17]byte +var z6356 [1 << 17]byte +var z6357 [1 << 17]byte +var z6358 [1 << 17]byte +var z6359 [1 << 17]byte +var z6360 [1 << 17]byte +var z6361 [1 << 17]byte +var z6362 [1 << 17]byte +var z6363 [1 << 17]byte +var z6364 [1 << 17]byte +var z6365 [1 << 17]byte +var z6366 [1 << 17]byte +var z6367 [1 << 17]byte +var z6368 [1 << 17]byte +var z6369 [1 << 17]byte +var z6370 [1 << 17]byte +var z6371 [1 << 17]byte +var z6372 [1 << 17]byte +var z6373 [1 << 17]byte +var z6374 [1 << 17]byte +var z6375 [1 << 17]byte +var z6376 [1 << 17]byte +var z6377 [1 << 17]byte +var z6378 [1 << 17]byte +var z6379 [1 << 17]byte +var z6380 [1 << 17]byte +var z6381 [1 << 17]byte +var z6382 [1 << 17]byte +var z6383 [1 << 17]byte +var z6384 [1 << 17]byte +var z6385 [1 << 17]byte +var z6386 [1 << 17]byte +var z6387 [1 << 17]byte +var z6388 [1 << 17]byte +var z6389 [1 << 17]byte +var z6390 [1 << 17]byte +var z6391 [1 << 17]byte +var z6392 [1 << 17]byte +var z6393 [1 << 17]byte +var z6394 [1 << 17]byte +var z6395 [1 << 17]byte +var z6396 [1 << 17]byte +var z6397 [1 << 17]byte +var z6398 [1 << 17]byte +var z6399 [1 << 17]byte +var z6400 [1 << 17]byte +var z6401 [1 << 17]byte +var z6402 [1 << 17]byte +var z6403 [1 << 17]byte +var z6404 [1 << 17]byte +var z6405 [1 << 17]byte +var z6406 [1 << 17]byte +var z6407 [1 << 17]byte +var z6408 [1 << 17]byte +var z6409 [1 << 17]byte +var z6410 [1 << 17]byte +var z6411 [1 << 17]byte +var z6412 [1 << 17]byte +var z6413 [1 << 17]byte +var z6414 [1 << 17]byte +var z6415 [1 << 17]byte +var z6416 [1 << 17]byte +var z6417 [1 << 17]byte +var z6418 [1 << 17]byte +var z6419 [1 << 17]byte +var z6420 [1 << 17]byte +var z6421 [1 << 17]byte +var z6422 [1 << 17]byte +var z6423 [1 << 17]byte +var z6424 [1 << 17]byte +var z6425 [1 << 17]byte +var z6426 [1 << 17]byte +var z6427 [1 << 17]byte +var z6428 [1 << 17]byte +var z6429 [1 << 17]byte +var z6430 [1 << 17]byte +var z6431 [1 << 17]byte +var z6432 [1 << 17]byte +var z6433 [1 << 17]byte +var z6434 [1 << 17]byte +var z6435 [1 << 17]byte +var z6436 [1 << 17]byte +var z6437 [1 << 17]byte +var z6438 [1 << 17]byte +var z6439 [1 << 17]byte +var z6440 [1 << 17]byte +var z6441 [1 << 17]byte +var z6442 [1 << 17]byte +var z6443 [1 << 17]byte +var z6444 [1 << 17]byte +var z6445 [1 << 17]byte +var z6446 [1 << 17]byte +var z6447 [1 << 17]byte +var z6448 [1 << 17]byte +var z6449 [1 << 17]byte +var z6450 [1 << 17]byte +var z6451 [1 << 17]byte +var z6452 [1 << 17]byte +var z6453 [1 << 17]byte +var z6454 [1 << 17]byte +var z6455 [1 << 17]byte +var z6456 [1 << 17]byte +var z6457 [1 << 17]byte +var z6458 [1 << 17]byte +var z6459 [1 << 17]byte +var z6460 [1 << 17]byte +var z6461 [1 << 17]byte +var z6462 [1 << 17]byte +var z6463 [1 << 17]byte +var z6464 [1 << 17]byte +var z6465 [1 << 17]byte +var z6466 [1 << 17]byte +var z6467 [1 << 17]byte +var z6468 [1 << 17]byte +var z6469 [1 << 17]byte +var z6470 [1 << 17]byte +var z6471 [1 << 17]byte +var z6472 [1 << 17]byte +var z6473 [1 << 17]byte +var z6474 [1 << 17]byte +var z6475 [1 << 17]byte +var z6476 [1 << 17]byte +var z6477 [1 << 17]byte +var z6478 [1 << 17]byte +var z6479 [1 << 17]byte +var z6480 [1 << 17]byte +var z6481 [1 << 17]byte +var z6482 [1 << 17]byte +var z6483 [1 << 17]byte +var z6484 [1 << 17]byte +var z6485 [1 << 17]byte +var z6486 [1 << 17]byte +var z6487 [1 << 17]byte +var z6488 [1 << 17]byte +var z6489 [1 << 17]byte +var z6490 [1 << 17]byte +var z6491 [1 << 17]byte +var z6492 [1 << 17]byte +var z6493 [1 << 17]byte +var z6494 [1 << 17]byte +var z6495 [1 << 17]byte +var z6496 [1 << 17]byte +var z6497 [1 << 17]byte +var z6498 [1 << 17]byte +var z6499 [1 << 17]byte +var z6500 [1 << 17]byte +var z6501 [1 << 17]byte +var z6502 [1 << 17]byte +var z6503 [1 << 17]byte +var z6504 [1 << 17]byte +var z6505 [1 << 17]byte +var z6506 [1 << 17]byte +var z6507 [1 << 17]byte +var z6508 [1 << 17]byte +var z6509 [1 << 17]byte +var z6510 [1 << 17]byte +var z6511 [1 << 17]byte +var z6512 [1 << 17]byte +var z6513 [1 << 17]byte +var z6514 [1 << 17]byte +var z6515 [1 << 17]byte +var z6516 [1 << 17]byte +var z6517 [1 << 17]byte +var z6518 [1 << 17]byte +var z6519 [1 << 17]byte +var z6520 [1 << 17]byte +var z6521 [1 << 17]byte +var z6522 [1 << 17]byte +var z6523 [1 << 17]byte +var z6524 [1 << 17]byte +var z6525 [1 << 17]byte +var z6526 [1 << 17]byte +var z6527 [1 << 17]byte +var z6528 [1 << 17]byte +var z6529 [1 << 17]byte +var z6530 [1 << 17]byte +var z6531 [1 << 17]byte +var z6532 [1 << 17]byte +var z6533 [1 << 17]byte +var z6534 [1 << 17]byte +var z6535 [1 << 17]byte +var z6536 [1 << 17]byte +var z6537 [1 << 17]byte +var z6538 [1 << 17]byte +var z6539 [1 << 17]byte +var z6540 [1 << 17]byte +var z6541 [1 << 17]byte +var z6542 [1 << 17]byte +var z6543 [1 << 17]byte +var z6544 [1 << 17]byte +var z6545 [1 << 17]byte +var z6546 [1 << 17]byte +var z6547 [1 << 17]byte +var z6548 [1 << 17]byte +var z6549 [1 << 17]byte +var z6550 [1 << 17]byte +var z6551 [1 << 17]byte +var z6552 [1 << 17]byte +var z6553 [1 << 17]byte +var z6554 [1 << 17]byte +var z6555 [1 << 17]byte +var z6556 [1 << 17]byte +var z6557 [1 << 17]byte +var z6558 [1 << 17]byte +var z6559 [1 << 17]byte +var z6560 [1 << 17]byte +var z6561 [1 << 17]byte +var z6562 [1 << 17]byte +var z6563 [1 << 17]byte +var z6564 [1 << 17]byte +var z6565 [1 << 17]byte +var z6566 [1 << 17]byte +var z6567 [1 << 17]byte +var z6568 [1 << 17]byte +var z6569 [1 << 17]byte +var z6570 [1 << 17]byte +var z6571 [1 << 17]byte +var z6572 [1 << 17]byte +var z6573 [1 << 17]byte +var z6574 [1 << 17]byte +var z6575 [1 << 17]byte +var z6576 [1 << 17]byte +var z6577 [1 << 17]byte +var z6578 [1 << 17]byte +var z6579 [1 << 17]byte +var z6580 [1 << 17]byte +var z6581 [1 << 17]byte +var z6582 [1 << 17]byte +var z6583 [1 << 17]byte +var z6584 [1 << 17]byte +var z6585 [1 << 17]byte +var z6586 [1 << 17]byte +var z6587 [1 << 17]byte +var z6588 [1 << 17]byte +var z6589 [1 << 17]byte +var z6590 [1 << 17]byte +var z6591 [1 << 17]byte +var z6592 [1 << 17]byte +var z6593 [1 << 17]byte +var z6594 [1 << 17]byte +var z6595 [1 << 17]byte +var z6596 [1 << 17]byte +var z6597 [1 << 17]byte +var z6598 [1 << 17]byte +var z6599 [1 << 17]byte +var z6600 [1 << 17]byte +var z6601 [1 << 17]byte +var z6602 [1 << 17]byte +var z6603 [1 << 17]byte +var z6604 [1 << 17]byte +var z6605 [1 << 17]byte +var z6606 [1 << 17]byte +var z6607 [1 << 17]byte +var z6608 [1 << 17]byte +var z6609 [1 << 17]byte +var z6610 [1 << 17]byte +var z6611 [1 << 17]byte +var z6612 [1 << 17]byte +var z6613 [1 << 17]byte +var z6614 [1 << 17]byte +var z6615 [1 << 17]byte +var z6616 [1 << 17]byte +var z6617 [1 << 17]byte +var z6618 [1 << 17]byte +var z6619 [1 << 17]byte +var z6620 [1 << 17]byte +var z6621 [1 << 17]byte +var z6622 [1 << 17]byte +var z6623 [1 << 17]byte +var z6624 [1 << 17]byte +var z6625 [1 << 17]byte +var z6626 [1 << 17]byte +var z6627 [1 << 17]byte +var z6628 [1 << 17]byte +var z6629 [1 << 17]byte +var z6630 [1 << 17]byte +var z6631 [1 << 17]byte +var z6632 [1 << 17]byte +var z6633 [1 << 17]byte +var z6634 [1 << 17]byte +var z6635 [1 << 17]byte +var z6636 [1 << 17]byte +var z6637 [1 << 17]byte +var z6638 [1 << 17]byte +var z6639 [1 << 17]byte +var z6640 [1 << 17]byte +var z6641 [1 << 17]byte +var z6642 [1 << 17]byte +var z6643 [1 << 17]byte +var z6644 [1 << 17]byte +var z6645 [1 << 17]byte +var z6646 [1 << 17]byte +var z6647 [1 << 17]byte +var z6648 [1 << 17]byte +var z6649 [1 << 17]byte +var z6650 [1 << 17]byte +var z6651 [1 << 17]byte +var z6652 [1 << 17]byte +var z6653 [1 << 17]byte +var z6654 [1 << 17]byte +var z6655 [1 << 17]byte +var z6656 [1 << 17]byte +var z6657 [1 << 17]byte +var z6658 [1 << 17]byte +var z6659 [1 << 17]byte +var z6660 [1 << 17]byte +var z6661 [1 << 17]byte +var z6662 [1 << 17]byte +var z6663 [1 << 17]byte +var z6664 [1 << 17]byte +var z6665 [1 << 17]byte +var z6666 [1 << 17]byte +var z6667 [1 << 17]byte +var z6668 [1 << 17]byte +var z6669 [1 << 17]byte +var z6670 [1 << 17]byte +var z6671 [1 << 17]byte +var z6672 [1 << 17]byte +var z6673 [1 << 17]byte +var z6674 [1 << 17]byte +var z6675 [1 << 17]byte +var z6676 [1 << 17]byte +var z6677 [1 << 17]byte +var z6678 [1 << 17]byte +var z6679 [1 << 17]byte +var z6680 [1 << 17]byte +var z6681 [1 << 17]byte +var z6682 [1 << 17]byte +var z6683 [1 << 17]byte +var z6684 [1 << 17]byte +var z6685 [1 << 17]byte +var z6686 [1 << 17]byte +var z6687 [1 << 17]byte +var z6688 [1 << 17]byte +var z6689 [1 << 17]byte +var z6690 [1 << 17]byte +var z6691 [1 << 17]byte +var z6692 [1 << 17]byte +var z6693 [1 << 17]byte +var z6694 [1 << 17]byte +var z6695 [1 << 17]byte +var z6696 [1 << 17]byte +var z6697 [1 << 17]byte +var z6698 [1 << 17]byte +var z6699 [1 << 17]byte +var z6700 [1 << 17]byte +var z6701 [1 << 17]byte +var z6702 [1 << 17]byte +var z6703 [1 << 17]byte +var z6704 [1 << 17]byte +var z6705 [1 << 17]byte +var z6706 [1 << 17]byte +var z6707 [1 << 17]byte +var z6708 [1 << 17]byte +var z6709 [1 << 17]byte +var z6710 [1 << 17]byte +var z6711 [1 << 17]byte +var z6712 [1 << 17]byte +var z6713 [1 << 17]byte +var z6714 [1 << 17]byte +var z6715 [1 << 17]byte +var z6716 [1 << 17]byte +var z6717 [1 << 17]byte +var z6718 [1 << 17]byte +var z6719 [1 << 17]byte +var z6720 [1 << 17]byte +var z6721 [1 << 17]byte +var z6722 [1 << 17]byte +var z6723 [1 << 17]byte +var z6724 [1 << 17]byte +var z6725 [1 << 17]byte +var z6726 [1 << 17]byte +var z6727 [1 << 17]byte +var z6728 [1 << 17]byte +var z6729 [1 << 17]byte +var z6730 [1 << 17]byte +var z6731 [1 << 17]byte +var z6732 [1 << 17]byte +var z6733 [1 << 17]byte +var z6734 [1 << 17]byte +var z6735 [1 << 17]byte +var z6736 [1 << 17]byte +var z6737 [1 << 17]byte +var z6738 [1 << 17]byte +var z6739 [1 << 17]byte +var z6740 [1 << 17]byte +var z6741 [1 << 17]byte +var z6742 [1 << 17]byte +var z6743 [1 << 17]byte +var z6744 [1 << 17]byte +var z6745 [1 << 17]byte +var z6746 [1 << 17]byte +var z6747 [1 << 17]byte +var z6748 [1 << 17]byte +var z6749 [1 << 17]byte +var z6750 [1 << 17]byte +var z6751 [1 << 17]byte +var z6752 [1 << 17]byte +var z6753 [1 << 17]byte +var z6754 [1 << 17]byte +var z6755 [1 << 17]byte +var z6756 [1 << 17]byte +var z6757 [1 << 17]byte +var z6758 [1 << 17]byte +var z6759 [1 << 17]byte +var z6760 [1 << 17]byte +var z6761 [1 << 17]byte +var z6762 [1 << 17]byte +var z6763 [1 << 17]byte +var z6764 [1 << 17]byte +var z6765 [1 << 17]byte +var z6766 [1 << 17]byte +var z6767 [1 << 17]byte +var z6768 [1 << 17]byte +var z6769 [1 << 17]byte +var z6770 [1 << 17]byte +var z6771 [1 << 17]byte +var z6772 [1 << 17]byte +var z6773 [1 << 17]byte +var z6774 [1 << 17]byte +var z6775 [1 << 17]byte +var z6776 [1 << 17]byte +var z6777 [1 << 17]byte +var z6778 [1 << 17]byte +var z6779 [1 << 17]byte +var z6780 [1 << 17]byte +var z6781 [1 << 17]byte +var z6782 [1 << 17]byte +var z6783 [1 << 17]byte +var z6784 [1 << 17]byte +var z6785 [1 << 17]byte +var z6786 [1 << 17]byte +var z6787 [1 << 17]byte +var z6788 [1 << 17]byte +var z6789 [1 << 17]byte +var z6790 [1 << 17]byte +var z6791 [1 << 17]byte +var z6792 [1 << 17]byte +var z6793 [1 << 17]byte +var z6794 [1 << 17]byte +var z6795 [1 << 17]byte +var z6796 [1 << 17]byte +var z6797 [1 << 17]byte +var z6798 [1 << 17]byte +var z6799 [1 << 17]byte +var z6800 [1 << 17]byte +var z6801 [1 << 17]byte +var z6802 [1 << 17]byte +var z6803 [1 << 17]byte +var z6804 [1 << 17]byte +var z6805 [1 << 17]byte +var z6806 [1 << 17]byte +var z6807 [1 << 17]byte +var z6808 [1 << 17]byte +var z6809 [1 << 17]byte +var z6810 [1 << 17]byte +var z6811 [1 << 17]byte +var z6812 [1 << 17]byte +var z6813 [1 << 17]byte +var z6814 [1 << 17]byte +var z6815 [1 << 17]byte +var z6816 [1 << 17]byte +var z6817 [1 << 17]byte +var z6818 [1 << 17]byte +var z6819 [1 << 17]byte +var z6820 [1 << 17]byte +var z6821 [1 << 17]byte +var z6822 [1 << 17]byte +var z6823 [1 << 17]byte +var z6824 [1 << 17]byte +var z6825 [1 << 17]byte +var z6826 [1 << 17]byte +var z6827 [1 << 17]byte +var z6828 [1 << 17]byte +var z6829 [1 << 17]byte +var z6830 [1 << 17]byte +var z6831 [1 << 17]byte +var z6832 [1 << 17]byte +var z6833 [1 << 17]byte +var z6834 [1 << 17]byte +var z6835 [1 << 17]byte +var z6836 [1 << 17]byte +var z6837 [1 << 17]byte +var z6838 [1 << 17]byte +var z6839 [1 << 17]byte +var z6840 [1 << 17]byte +var z6841 [1 << 17]byte +var z6842 [1 << 17]byte +var z6843 [1 << 17]byte +var z6844 [1 << 17]byte +var z6845 [1 << 17]byte +var z6846 [1 << 17]byte +var z6847 [1 << 17]byte +var z6848 [1 << 17]byte +var z6849 [1 << 17]byte +var z6850 [1 << 17]byte +var z6851 [1 << 17]byte +var z6852 [1 << 17]byte +var z6853 [1 << 17]byte +var z6854 [1 << 17]byte +var z6855 [1 << 17]byte +var z6856 [1 << 17]byte +var z6857 [1 << 17]byte +var z6858 [1 << 17]byte +var z6859 [1 << 17]byte +var z6860 [1 << 17]byte +var z6861 [1 << 17]byte +var z6862 [1 << 17]byte +var z6863 [1 << 17]byte +var z6864 [1 << 17]byte +var z6865 [1 << 17]byte +var z6866 [1 << 17]byte +var z6867 [1 << 17]byte +var z6868 [1 << 17]byte +var z6869 [1 << 17]byte +var z6870 [1 << 17]byte +var z6871 [1 << 17]byte +var z6872 [1 << 17]byte +var z6873 [1 << 17]byte +var z6874 [1 << 17]byte +var z6875 [1 << 17]byte +var z6876 [1 << 17]byte +var z6877 [1 << 17]byte +var z6878 [1 << 17]byte +var z6879 [1 << 17]byte +var z6880 [1 << 17]byte +var z6881 [1 << 17]byte +var z6882 [1 << 17]byte +var z6883 [1 << 17]byte +var z6884 [1 << 17]byte +var z6885 [1 << 17]byte +var z6886 [1 << 17]byte +var z6887 [1 << 17]byte +var z6888 [1 << 17]byte +var z6889 [1 << 17]byte +var z6890 [1 << 17]byte +var z6891 [1 << 17]byte +var z6892 [1 << 17]byte +var z6893 [1 << 17]byte +var z6894 [1 << 17]byte +var z6895 [1 << 17]byte +var z6896 [1 << 17]byte +var z6897 [1 << 17]byte +var z6898 [1 << 17]byte +var z6899 [1 << 17]byte +var z6900 [1 << 17]byte +var z6901 [1 << 17]byte +var z6902 [1 << 17]byte +var z6903 [1 << 17]byte +var z6904 [1 << 17]byte +var z6905 [1 << 17]byte +var z6906 [1 << 17]byte +var z6907 [1 << 17]byte +var z6908 [1 << 17]byte +var z6909 [1 << 17]byte +var z6910 [1 << 17]byte +var z6911 [1 << 17]byte +var z6912 [1 << 17]byte +var z6913 [1 << 17]byte +var z6914 [1 << 17]byte +var z6915 [1 << 17]byte +var z6916 [1 << 17]byte +var z6917 [1 << 17]byte +var z6918 [1 << 17]byte +var z6919 [1 << 17]byte +var z6920 [1 << 17]byte +var z6921 [1 << 17]byte +var z6922 [1 << 17]byte +var z6923 [1 << 17]byte +var z6924 [1 << 17]byte +var z6925 [1 << 17]byte +var z6926 [1 << 17]byte +var z6927 [1 << 17]byte +var z6928 [1 << 17]byte +var z6929 [1 << 17]byte +var z6930 [1 << 17]byte +var z6931 [1 << 17]byte +var z6932 [1 << 17]byte +var z6933 [1 << 17]byte +var z6934 [1 << 17]byte +var z6935 [1 << 17]byte +var z6936 [1 << 17]byte +var z6937 [1 << 17]byte +var z6938 [1 << 17]byte +var z6939 [1 << 17]byte +var z6940 [1 << 17]byte +var z6941 [1 << 17]byte +var z6942 [1 << 17]byte +var z6943 [1 << 17]byte +var z6944 [1 << 17]byte +var z6945 [1 << 17]byte +var z6946 [1 << 17]byte +var z6947 [1 << 17]byte +var z6948 [1 << 17]byte +var z6949 [1 << 17]byte +var z6950 [1 << 17]byte +var z6951 [1 << 17]byte +var z6952 [1 << 17]byte +var z6953 [1 << 17]byte +var z6954 [1 << 17]byte +var z6955 [1 << 17]byte +var z6956 [1 << 17]byte +var z6957 [1 << 17]byte +var z6958 [1 << 17]byte +var z6959 [1 << 17]byte +var z6960 [1 << 17]byte +var z6961 [1 << 17]byte +var z6962 [1 << 17]byte +var z6963 [1 << 17]byte +var z6964 [1 << 17]byte +var z6965 [1 << 17]byte +var z6966 [1 << 17]byte +var z6967 [1 << 17]byte +var z6968 [1 << 17]byte +var z6969 [1 << 17]byte +var z6970 [1 << 17]byte +var z6971 [1 << 17]byte +var z6972 [1 << 17]byte +var z6973 [1 << 17]byte +var z6974 [1 << 17]byte +var z6975 [1 << 17]byte +var z6976 [1 << 17]byte +var z6977 [1 << 17]byte +var z6978 [1 << 17]byte +var z6979 [1 << 17]byte +var z6980 [1 << 17]byte +var z6981 [1 << 17]byte +var z6982 [1 << 17]byte +var z6983 [1 << 17]byte +var z6984 [1 << 17]byte +var z6985 [1 << 17]byte +var z6986 [1 << 17]byte +var z6987 [1 << 17]byte +var z6988 [1 << 17]byte +var z6989 [1 << 17]byte +var z6990 [1 << 17]byte +var z6991 [1 << 17]byte +var z6992 [1 << 17]byte +var z6993 [1 << 17]byte +var z6994 [1 << 17]byte +var z6995 [1 << 17]byte +var z6996 [1 << 17]byte +var z6997 [1 << 17]byte +var z6998 [1 << 17]byte +var z6999 [1 << 17]byte +var z7000 [1 << 17]byte +var z7001 [1 << 17]byte +var z7002 [1 << 17]byte +var z7003 [1 << 17]byte +var z7004 [1 << 17]byte +var z7005 [1 << 17]byte +var z7006 [1 << 17]byte +var z7007 [1 << 17]byte +var z7008 [1 << 17]byte +var z7009 [1 << 17]byte +var z7010 [1 << 17]byte +var z7011 [1 << 17]byte +var z7012 [1 << 17]byte +var z7013 [1 << 17]byte +var z7014 [1 << 17]byte +var z7015 [1 << 17]byte +var z7016 [1 << 17]byte +var z7017 [1 << 17]byte +var z7018 [1 << 17]byte +var z7019 [1 << 17]byte +var z7020 [1 << 17]byte +var z7021 [1 << 17]byte +var z7022 [1 << 17]byte +var z7023 [1 << 17]byte +var z7024 [1 << 17]byte +var z7025 [1 << 17]byte +var z7026 [1 << 17]byte +var z7027 [1 << 17]byte +var z7028 [1 << 17]byte +var z7029 [1 << 17]byte +var z7030 [1 << 17]byte +var z7031 [1 << 17]byte +var z7032 [1 << 17]byte +var z7033 [1 << 17]byte +var z7034 [1 << 17]byte +var z7035 [1 << 17]byte +var z7036 [1 << 17]byte +var z7037 [1 << 17]byte +var z7038 [1 << 17]byte +var z7039 [1 << 17]byte +var z7040 [1 << 17]byte +var z7041 [1 << 17]byte +var z7042 [1 << 17]byte +var z7043 [1 << 17]byte +var z7044 [1 << 17]byte +var z7045 [1 << 17]byte +var z7046 [1 << 17]byte +var z7047 [1 << 17]byte +var z7048 [1 << 17]byte +var z7049 [1 << 17]byte +var z7050 [1 << 17]byte +var z7051 [1 << 17]byte +var z7052 [1 << 17]byte +var z7053 [1 << 17]byte +var z7054 [1 << 17]byte +var z7055 [1 << 17]byte +var z7056 [1 << 17]byte +var z7057 [1 << 17]byte +var z7058 [1 << 17]byte +var z7059 [1 << 17]byte +var z7060 [1 << 17]byte +var z7061 [1 << 17]byte +var z7062 [1 << 17]byte +var z7063 [1 << 17]byte +var z7064 [1 << 17]byte +var z7065 [1 << 17]byte +var z7066 [1 << 17]byte +var z7067 [1 << 17]byte +var z7068 [1 << 17]byte +var z7069 [1 << 17]byte +var z7070 [1 << 17]byte +var z7071 [1 << 17]byte +var z7072 [1 << 17]byte +var z7073 [1 << 17]byte +var z7074 [1 << 17]byte +var z7075 [1 << 17]byte +var z7076 [1 << 17]byte +var z7077 [1 << 17]byte +var z7078 [1 << 17]byte +var z7079 [1 << 17]byte +var z7080 [1 << 17]byte +var z7081 [1 << 17]byte +var z7082 [1 << 17]byte +var z7083 [1 << 17]byte +var z7084 [1 << 17]byte +var z7085 [1 << 17]byte +var z7086 [1 << 17]byte +var z7087 [1 << 17]byte +var z7088 [1 << 17]byte +var z7089 [1 << 17]byte +var z7090 [1 << 17]byte +var z7091 [1 << 17]byte +var z7092 [1 << 17]byte +var z7093 [1 << 17]byte +var z7094 [1 << 17]byte +var z7095 [1 << 17]byte +var z7096 [1 << 17]byte +var z7097 [1 << 17]byte +var z7098 [1 << 17]byte +var z7099 [1 << 17]byte +var z7100 [1 << 17]byte +var z7101 [1 << 17]byte +var z7102 [1 << 17]byte +var z7103 [1 << 17]byte +var z7104 [1 << 17]byte +var z7105 [1 << 17]byte +var z7106 [1 << 17]byte +var z7107 [1 << 17]byte +var z7108 [1 << 17]byte +var z7109 [1 << 17]byte +var z7110 [1 << 17]byte +var z7111 [1 << 17]byte +var z7112 [1 << 17]byte +var z7113 [1 << 17]byte +var z7114 [1 << 17]byte +var z7115 [1 << 17]byte +var z7116 [1 << 17]byte +var z7117 [1 << 17]byte +var z7118 [1 << 17]byte +var z7119 [1 << 17]byte +var z7120 [1 << 17]byte +var z7121 [1 << 17]byte +var z7122 [1 << 17]byte +var z7123 [1 << 17]byte +var z7124 [1 << 17]byte +var z7125 [1 << 17]byte +var z7126 [1 << 17]byte +var z7127 [1 << 17]byte +var z7128 [1 << 17]byte +var z7129 [1 << 17]byte +var z7130 [1 << 17]byte +var z7131 [1 << 17]byte +var z7132 [1 << 17]byte +var z7133 [1 << 17]byte +var z7134 [1 << 17]byte +var z7135 [1 << 17]byte +var z7136 [1 << 17]byte +var z7137 [1 << 17]byte +var z7138 [1 << 17]byte +var z7139 [1 << 17]byte +var z7140 [1 << 17]byte +var z7141 [1 << 17]byte +var z7142 [1 << 17]byte +var z7143 [1 << 17]byte +var z7144 [1 << 17]byte +var z7145 [1 << 17]byte +var z7146 [1 << 17]byte +var z7147 [1 << 17]byte +var z7148 [1 << 17]byte +var z7149 [1 << 17]byte +var z7150 [1 << 17]byte +var z7151 [1 << 17]byte +var z7152 [1 << 17]byte +var z7153 [1 << 17]byte +var z7154 [1 << 17]byte +var z7155 [1 << 17]byte +var z7156 [1 << 17]byte +var z7157 [1 << 17]byte +var z7158 [1 << 17]byte +var z7159 [1 << 17]byte +var z7160 [1 << 17]byte +var z7161 [1 << 17]byte +var z7162 [1 << 17]byte +var z7163 [1 << 17]byte +var z7164 [1 << 17]byte +var z7165 [1 << 17]byte +var z7166 [1 << 17]byte +var z7167 [1 << 17]byte +var z7168 [1 << 17]byte +var z7169 [1 << 17]byte +var z7170 [1 << 17]byte +var z7171 [1 << 17]byte +var z7172 [1 << 17]byte +var z7173 [1 << 17]byte +var z7174 [1 << 17]byte +var z7175 [1 << 17]byte +var z7176 [1 << 17]byte +var z7177 [1 << 17]byte +var z7178 [1 << 17]byte +var z7179 [1 << 17]byte +var z7180 [1 << 17]byte +var z7181 [1 << 17]byte +var z7182 [1 << 17]byte +var z7183 [1 << 17]byte +var z7184 [1 << 17]byte +var z7185 [1 << 17]byte +var z7186 [1 << 17]byte +var z7187 [1 << 17]byte +var z7188 [1 << 17]byte +var z7189 [1 << 17]byte +var z7190 [1 << 17]byte +var z7191 [1 << 17]byte +var z7192 [1 << 17]byte +var z7193 [1 << 17]byte +var z7194 [1 << 17]byte +var z7195 [1 << 17]byte +var z7196 [1 << 17]byte +var z7197 [1 << 17]byte +var z7198 [1 << 17]byte +var z7199 [1 << 17]byte +var z7200 [1 << 17]byte +var z7201 [1 << 17]byte +var z7202 [1 << 17]byte +var z7203 [1 << 17]byte +var z7204 [1 << 17]byte +var z7205 [1 << 17]byte +var z7206 [1 << 17]byte +var z7207 [1 << 17]byte +var z7208 [1 << 17]byte +var z7209 [1 << 17]byte +var z7210 [1 << 17]byte +var z7211 [1 << 17]byte +var z7212 [1 << 17]byte +var z7213 [1 << 17]byte +var z7214 [1 << 17]byte +var z7215 [1 << 17]byte +var z7216 [1 << 17]byte +var z7217 [1 << 17]byte +var z7218 [1 << 17]byte +var z7219 [1 << 17]byte +var z7220 [1 << 17]byte +var z7221 [1 << 17]byte +var z7222 [1 << 17]byte +var z7223 [1 << 17]byte +var z7224 [1 << 17]byte +var z7225 [1 << 17]byte +var z7226 [1 << 17]byte +var z7227 [1 << 17]byte +var z7228 [1 << 17]byte +var z7229 [1 << 17]byte +var z7230 [1 << 17]byte +var z7231 [1 << 17]byte +var z7232 [1 << 17]byte +var z7233 [1 << 17]byte +var z7234 [1 << 17]byte +var z7235 [1 << 17]byte +var z7236 [1 << 17]byte +var z7237 [1 << 17]byte +var z7238 [1 << 17]byte +var z7239 [1 << 17]byte +var z7240 [1 << 17]byte +var z7241 [1 << 17]byte +var z7242 [1 << 17]byte +var z7243 [1 << 17]byte +var z7244 [1 << 17]byte +var z7245 [1 << 17]byte +var z7246 [1 << 17]byte +var z7247 [1 << 17]byte +var z7248 [1 << 17]byte +var z7249 [1 << 17]byte +var z7250 [1 << 17]byte +var z7251 [1 << 17]byte +var z7252 [1 << 17]byte +var z7253 [1 << 17]byte +var z7254 [1 << 17]byte +var z7255 [1 << 17]byte +var z7256 [1 << 17]byte +var z7257 [1 << 17]byte +var z7258 [1 << 17]byte +var z7259 [1 << 17]byte +var z7260 [1 << 17]byte +var z7261 [1 << 17]byte +var z7262 [1 << 17]byte +var z7263 [1 << 17]byte +var z7264 [1 << 17]byte +var z7265 [1 << 17]byte +var z7266 [1 << 17]byte +var z7267 [1 << 17]byte +var z7268 [1 << 17]byte +var z7269 [1 << 17]byte +var z7270 [1 << 17]byte +var z7271 [1 << 17]byte +var z7272 [1 << 17]byte +var z7273 [1 << 17]byte +var z7274 [1 << 17]byte +var z7275 [1 << 17]byte +var z7276 [1 << 17]byte +var z7277 [1 << 17]byte +var z7278 [1 << 17]byte +var z7279 [1 << 17]byte +var z7280 [1 << 17]byte +var z7281 [1 << 17]byte +var z7282 [1 << 17]byte +var z7283 [1 << 17]byte +var z7284 [1 << 17]byte +var z7285 [1 << 17]byte +var z7286 [1 << 17]byte +var z7287 [1 << 17]byte +var z7288 [1 << 17]byte +var z7289 [1 << 17]byte +var z7290 [1 << 17]byte +var z7291 [1 << 17]byte +var z7292 [1 << 17]byte +var z7293 [1 << 17]byte +var z7294 [1 << 17]byte +var z7295 [1 << 17]byte +var z7296 [1 << 17]byte +var z7297 [1 << 17]byte +var z7298 [1 << 17]byte +var z7299 [1 << 17]byte +var z7300 [1 << 17]byte +var z7301 [1 << 17]byte +var z7302 [1 << 17]byte +var z7303 [1 << 17]byte +var z7304 [1 << 17]byte +var z7305 [1 << 17]byte +var z7306 [1 << 17]byte +var z7307 [1 << 17]byte +var z7308 [1 << 17]byte +var z7309 [1 << 17]byte +var z7310 [1 << 17]byte +var z7311 [1 << 17]byte +var z7312 [1 << 17]byte +var z7313 [1 << 17]byte +var z7314 [1 << 17]byte +var z7315 [1 << 17]byte +var z7316 [1 << 17]byte +var z7317 [1 << 17]byte +var z7318 [1 << 17]byte +var z7319 [1 << 17]byte +var z7320 [1 << 17]byte +var z7321 [1 << 17]byte +var z7322 [1 << 17]byte +var z7323 [1 << 17]byte +var z7324 [1 << 17]byte +var z7325 [1 << 17]byte +var z7326 [1 << 17]byte +var z7327 [1 << 17]byte +var z7328 [1 << 17]byte +var z7329 [1 << 17]byte +var z7330 [1 << 17]byte +var z7331 [1 << 17]byte +var z7332 [1 << 17]byte +var z7333 [1 << 17]byte +var z7334 [1 << 17]byte +var z7335 [1 << 17]byte +var z7336 [1 << 17]byte +var z7337 [1 << 17]byte +var z7338 [1 << 17]byte +var z7339 [1 << 17]byte +var z7340 [1 << 17]byte +var z7341 [1 << 17]byte +var z7342 [1 << 17]byte +var z7343 [1 << 17]byte +var z7344 [1 << 17]byte +var z7345 [1 << 17]byte +var z7346 [1 << 17]byte +var z7347 [1 << 17]byte +var z7348 [1 << 17]byte +var z7349 [1 << 17]byte +var z7350 [1 << 17]byte +var z7351 [1 << 17]byte +var z7352 [1 << 17]byte +var z7353 [1 << 17]byte +var z7354 [1 << 17]byte +var z7355 [1 << 17]byte +var z7356 [1 << 17]byte +var z7357 [1 << 17]byte +var z7358 [1 << 17]byte +var z7359 [1 << 17]byte +var z7360 [1 << 17]byte +var z7361 [1 << 17]byte +var z7362 [1 << 17]byte +var z7363 [1 << 17]byte +var z7364 [1 << 17]byte +var z7365 [1 << 17]byte +var z7366 [1 << 17]byte +var z7367 [1 << 17]byte +var z7368 [1 << 17]byte +var z7369 [1 << 17]byte +var z7370 [1 << 17]byte +var z7371 [1 << 17]byte +var z7372 [1 << 17]byte +var z7373 [1 << 17]byte +var z7374 [1 << 17]byte +var z7375 [1 << 17]byte +var z7376 [1 << 17]byte +var z7377 [1 << 17]byte +var z7378 [1 << 17]byte +var z7379 [1 << 17]byte +var z7380 [1 << 17]byte +var z7381 [1 << 17]byte +var z7382 [1 << 17]byte +var z7383 [1 << 17]byte +var z7384 [1 << 17]byte +var z7385 [1 << 17]byte +var z7386 [1 << 17]byte +var z7387 [1 << 17]byte +var z7388 [1 << 17]byte +var z7389 [1 << 17]byte +var z7390 [1 << 17]byte +var z7391 [1 << 17]byte +var z7392 [1 << 17]byte +var z7393 [1 << 17]byte +var z7394 [1 << 17]byte +var z7395 [1 << 17]byte +var z7396 [1 << 17]byte +var z7397 [1 << 17]byte +var z7398 [1 << 17]byte +var z7399 [1 << 17]byte +var z7400 [1 << 17]byte +var z7401 [1 << 17]byte +var z7402 [1 << 17]byte +var z7403 [1 << 17]byte +var z7404 [1 << 17]byte +var z7405 [1 << 17]byte +var z7406 [1 << 17]byte +var z7407 [1 << 17]byte +var z7408 [1 << 17]byte +var z7409 [1 << 17]byte +var z7410 [1 << 17]byte +var z7411 [1 << 17]byte +var z7412 [1 << 17]byte +var z7413 [1 << 17]byte +var z7414 [1 << 17]byte +var z7415 [1 << 17]byte +var z7416 [1 << 17]byte +var z7417 [1 << 17]byte +var z7418 [1 << 17]byte +var z7419 [1 << 17]byte +var z7420 [1 << 17]byte +var z7421 [1 << 17]byte +var z7422 [1 << 17]byte +var z7423 [1 << 17]byte +var z7424 [1 << 17]byte +var z7425 [1 << 17]byte +var z7426 [1 << 17]byte +var z7427 [1 << 17]byte +var z7428 [1 << 17]byte +var z7429 [1 << 17]byte +var z7430 [1 << 17]byte +var z7431 [1 << 17]byte +var z7432 [1 << 17]byte +var z7433 [1 << 17]byte +var z7434 [1 << 17]byte +var z7435 [1 << 17]byte +var z7436 [1 << 17]byte +var z7437 [1 << 17]byte +var z7438 [1 << 17]byte +var z7439 [1 << 17]byte +var z7440 [1 << 17]byte +var z7441 [1 << 17]byte +var z7442 [1 << 17]byte +var z7443 [1 << 17]byte +var z7444 [1 << 17]byte +var z7445 [1 << 17]byte +var z7446 [1 << 17]byte +var z7447 [1 << 17]byte +var z7448 [1 << 17]byte +var z7449 [1 << 17]byte +var z7450 [1 << 17]byte +var z7451 [1 << 17]byte +var z7452 [1 << 17]byte +var z7453 [1 << 17]byte +var z7454 [1 << 17]byte +var z7455 [1 << 17]byte +var z7456 [1 << 17]byte +var z7457 [1 << 17]byte +var z7458 [1 << 17]byte +var z7459 [1 << 17]byte +var z7460 [1 << 17]byte +var z7461 [1 << 17]byte +var z7462 [1 << 17]byte +var z7463 [1 << 17]byte +var z7464 [1 << 17]byte +var z7465 [1 << 17]byte +var z7466 [1 << 17]byte +var z7467 [1 << 17]byte +var z7468 [1 << 17]byte +var z7469 [1 << 17]byte +var z7470 [1 << 17]byte +var z7471 [1 << 17]byte +var z7472 [1 << 17]byte +var z7473 [1 << 17]byte +var z7474 [1 << 17]byte +var z7475 [1 << 17]byte +var z7476 [1 << 17]byte +var z7477 [1 << 17]byte +var z7478 [1 << 17]byte +var z7479 [1 << 17]byte +var z7480 [1 << 17]byte +var z7481 [1 << 17]byte +var z7482 [1 << 17]byte +var z7483 [1 << 17]byte +var z7484 [1 << 17]byte +var z7485 [1 << 17]byte +var z7486 [1 << 17]byte +var z7487 [1 << 17]byte +var z7488 [1 << 17]byte +var z7489 [1 << 17]byte +var z7490 [1 << 17]byte +var z7491 [1 << 17]byte +var z7492 [1 << 17]byte +var z7493 [1 << 17]byte +var z7494 [1 << 17]byte +var z7495 [1 << 17]byte +var z7496 [1 << 17]byte +var z7497 [1 << 17]byte +var z7498 [1 << 17]byte +var z7499 [1 << 17]byte +var z7500 [1 << 17]byte +var z7501 [1 << 17]byte +var z7502 [1 << 17]byte +var z7503 [1 << 17]byte +var z7504 [1 << 17]byte +var z7505 [1 << 17]byte +var z7506 [1 << 17]byte +var z7507 [1 << 17]byte +var z7508 [1 << 17]byte +var z7509 [1 << 17]byte +var z7510 [1 << 17]byte +var z7511 [1 << 17]byte +var z7512 [1 << 17]byte +var z7513 [1 << 17]byte +var z7514 [1 << 17]byte +var z7515 [1 << 17]byte +var z7516 [1 << 17]byte +var z7517 [1 << 17]byte +var z7518 [1 << 17]byte +var z7519 [1 << 17]byte +var z7520 [1 << 17]byte +var z7521 [1 << 17]byte +var z7522 [1 << 17]byte +var z7523 [1 << 17]byte +var z7524 [1 << 17]byte +var z7525 [1 << 17]byte +var z7526 [1 << 17]byte +var z7527 [1 << 17]byte +var z7528 [1 << 17]byte +var z7529 [1 << 17]byte +var z7530 [1 << 17]byte +var z7531 [1 << 17]byte +var z7532 [1 << 17]byte +var z7533 [1 << 17]byte +var z7534 [1 << 17]byte +var z7535 [1 << 17]byte +var z7536 [1 << 17]byte +var z7537 [1 << 17]byte +var z7538 [1 << 17]byte +var z7539 [1 << 17]byte +var z7540 [1 << 17]byte +var z7541 [1 << 17]byte +var z7542 [1 << 17]byte +var z7543 [1 << 17]byte +var z7544 [1 << 17]byte +var z7545 [1 << 17]byte +var z7546 [1 << 17]byte +var z7547 [1 << 17]byte +var z7548 [1 << 17]byte +var z7549 [1 << 17]byte +var z7550 [1 << 17]byte +var z7551 [1 << 17]byte +var z7552 [1 << 17]byte +var z7553 [1 << 17]byte +var z7554 [1 << 17]byte +var z7555 [1 << 17]byte +var z7556 [1 << 17]byte +var z7557 [1 << 17]byte +var z7558 [1 << 17]byte +var z7559 [1 << 17]byte +var z7560 [1 << 17]byte +var z7561 [1 << 17]byte +var z7562 [1 << 17]byte +var z7563 [1 << 17]byte +var z7564 [1 << 17]byte +var z7565 [1 << 17]byte +var z7566 [1 << 17]byte +var z7567 [1 << 17]byte +var z7568 [1 << 17]byte +var z7569 [1 << 17]byte +var z7570 [1 << 17]byte +var z7571 [1 << 17]byte +var z7572 [1 << 17]byte +var z7573 [1 << 17]byte +var z7574 [1 << 17]byte +var z7575 [1 << 17]byte +var z7576 [1 << 17]byte +var z7577 [1 << 17]byte +var z7578 [1 << 17]byte +var z7579 [1 << 17]byte +var z7580 [1 << 17]byte +var z7581 [1 << 17]byte +var z7582 [1 << 17]byte +var z7583 [1 << 17]byte +var z7584 [1 << 17]byte +var z7585 [1 << 17]byte +var z7586 [1 << 17]byte +var z7587 [1 << 17]byte +var z7588 [1 << 17]byte +var z7589 [1 << 17]byte +var z7590 [1 << 17]byte +var z7591 [1 << 17]byte +var z7592 [1 << 17]byte +var z7593 [1 << 17]byte +var z7594 [1 << 17]byte +var z7595 [1 << 17]byte +var z7596 [1 << 17]byte +var z7597 [1 << 17]byte +var z7598 [1 << 17]byte +var z7599 [1 << 17]byte +var z7600 [1 << 17]byte +var z7601 [1 << 17]byte +var z7602 [1 << 17]byte +var z7603 [1 << 17]byte +var z7604 [1 << 17]byte +var z7605 [1 << 17]byte +var z7606 [1 << 17]byte +var z7607 [1 << 17]byte +var z7608 [1 << 17]byte +var z7609 [1 << 17]byte +var z7610 [1 << 17]byte +var z7611 [1 << 17]byte +var z7612 [1 << 17]byte +var z7613 [1 << 17]byte +var z7614 [1 << 17]byte +var z7615 [1 << 17]byte +var z7616 [1 << 17]byte +var z7617 [1 << 17]byte +var z7618 [1 << 17]byte +var z7619 [1 << 17]byte +var z7620 [1 << 17]byte +var z7621 [1 << 17]byte +var z7622 [1 << 17]byte +var z7623 [1 << 17]byte +var z7624 [1 << 17]byte +var z7625 [1 << 17]byte +var z7626 [1 << 17]byte +var z7627 [1 << 17]byte +var z7628 [1 << 17]byte +var z7629 [1 << 17]byte +var z7630 [1 << 17]byte +var z7631 [1 << 17]byte +var z7632 [1 << 17]byte +var z7633 [1 << 17]byte +var z7634 [1 << 17]byte +var z7635 [1 << 17]byte +var z7636 [1 << 17]byte +var z7637 [1 << 17]byte +var z7638 [1 << 17]byte +var z7639 [1 << 17]byte +var z7640 [1 << 17]byte +var z7641 [1 << 17]byte +var z7642 [1 << 17]byte +var z7643 [1 << 17]byte +var z7644 [1 << 17]byte +var z7645 [1 << 17]byte +var z7646 [1 << 17]byte +var z7647 [1 << 17]byte +var z7648 [1 << 17]byte +var z7649 [1 << 17]byte +var z7650 [1 << 17]byte +var z7651 [1 << 17]byte +var z7652 [1 << 17]byte +var z7653 [1 << 17]byte +var z7654 [1 << 17]byte +var z7655 [1 << 17]byte +var z7656 [1 << 17]byte +var z7657 [1 << 17]byte +var z7658 [1 << 17]byte +var z7659 [1 << 17]byte +var z7660 [1 << 17]byte +var z7661 [1 << 17]byte +var z7662 [1 << 17]byte +var z7663 [1 << 17]byte +var z7664 [1 << 17]byte +var z7665 [1 << 17]byte +var z7666 [1 << 17]byte +var z7667 [1 << 17]byte +var z7668 [1 << 17]byte +var z7669 [1 << 17]byte +var z7670 [1 << 17]byte +var z7671 [1 << 17]byte +var z7672 [1 << 17]byte +var z7673 [1 << 17]byte +var z7674 [1 << 17]byte +var z7675 [1 << 17]byte +var z7676 [1 << 17]byte +var z7677 [1 << 17]byte +var z7678 [1 << 17]byte +var z7679 [1 << 17]byte +var z7680 [1 << 17]byte +var z7681 [1 << 17]byte +var z7682 [1 << 17]byte +var z7683 [1 << 17]byte +var z7684 [1 << 17]byte +var z7685 [1 << 17]byte +var z7686 [1 << 17]byte +var z7687 [1 << 17]byte +var z7688 [1 << 17]byte +var z7689 [1 << 17]byte +var z7690 [1 << 17]byte +var z7691 [1 << 17]byte +var z7692 [1 << 17]byte +var z7693 [1 << 17]byte +var z7694 [1 << 17]byte +var z7695 [1 << 17]byte +var z7696 [1 << 17]byte +var z7697 [1 << 17]byte +var z7698 [1 << 17]byte +var z7699 [1 << 17]byte +var z7700 [1 << 17]byte +var z7701 [1 << 17]byte +var z7702 [1 << 17]byte +var z7703 [1 << 17]byte +var z7704 [1 << 17]byte +var z7705 [1 << 17]byte +var z7706 [1 << 17]byte +var z7707 [1 << 17]byte +var z7708 [1 << 17]byte +var z7709 [1 << 17]byte +var z7710 [1 << 17]byte +var z7711 [1 << 17]byte +var z7712 [1 << 17]byte +var z7713 [1 << 17]byte +var z7714 [1 << 17]byte +var z7715 [1 << 17]byte +var z7716 [1 << 17]byte +var z7717 [1 << 17]byte +var z7718 [1 << 17]byte +var z7719 [1 << 17]byte +var z7720 [1 << 17]byte +var z7721 [1 << 17]byte +var z7722 [1 << 17]byte +var z7723 [1 << 17]byte +var z7724 [1 << 17]byte +var z7725 [1 << 17]byte +var z7726 [1 << 17]byte +var z7727 [1 << 17]byte +var z7728 [1 << 17]byte +var z7729 [1 << 17]byte +var z7730 [1 << 17]byte +var z7731 [1 << 17]byte +var z7732 [1 << 17]byte +var z7733 [1 << 17]byte +var z7734 [1 << 17]byte +var z7735 [1 << 17]byte +var z7736 [1 << 17]byte +var z7737 [1 << 17]byte +var z7738 [1 << 17]byte +var z7739 [1 << 17]byte +var z7740 [1 << 17]byte +var z7741 [1 << 17]byte +var z7742 [1 << 17]byte +var z7743 [1 << 17]byte +var z7744 [1 << 17]byte +var z7745 [1 << 17]byte +var z7746 [1 << 17]byte +var z7747 [1 << 17]byte +var z7748 [1 << 17]byte +var z7749 [1 << 17]byte +var z7750 [1 << 17]byte +var z7751 [1 << 17]byte +var z7752 [1 << 17]byte +var z7753 [1 << 17]byte +var z7754 [1 << 17]byte +var z7755 [1 << 17]byte +var z7756 [1 << 17]byte +var z7757 [1 << 17]byte +var z7758 [1 << 17]byte +var z7759 [1 << 17]byte +var z7760 [1 << 17]byte +var z7761 [1 << 17]byte +var z7762 [1 << 17]byte +var z7763 [1 << 17]byte +var z7764 [1 << 17]byte +var z7765 [1 << 17]byte +var z7766 [1 << 17]byte +var z7767 [1 << 17]byte +var z7768 [1 << 17]byte +var z7769 [1 << 17]byte +var z7770 [1 << 17]byte +var z7771 [1 << 17]byte +var z7772 [1 << 17]byte +var z7773 [1 << 17]byte +var z7774 [1 << 17]byte +var z7775 [1 << 17]byte +var z7776 [1 << 17]byte +var z7777 [1 << 17]byte +var z7778 [1 << 17]byte +var z7779 [1 << 17]byte +var z7780 [1 << 17]byte +var z7781 [1 << 17]byte +var z7782 [1 << 17]byte +var z7783 [1 << 17]byte +var z7784 [1 << 17]byte +var z7785 [1 << 17]byte +var z7786 [1 << 17]byte +var z7787 [1 << 17]byte +var z7788 [1 << 17]byte +var z7789 [1 << 17]byte +var z7790 [1 << 17]byte +var z7791 [1 << 17]byte +var z7792 [1 << 17]byte +var z7793 [1 << 17]byte +var z7794 [1 << 17]byte +var z7795 [1 << 17]byte +var z7796 [1 << 17]byte +var z7797 [1 << 17]byte +var z7798 [1 << 17]byte +var z7799 [1 << 17]byte +var z7800 [1 << 17]byte +var z7801 [1 << 17]byte +var z7802 [1 << 17]byte +var z7803 [1 << 17]byte +var z7804 [1 << 17]byte +var z7805 [1 << 17]byte +var z7806 [1 << 17]byte +var z7807 [1 << 17]byte +var z7808 [1 << 17]byte +var z7809 [1 << 17]byte +var z7810 [1 << 17]byte +var z7811 [1 << 17]byte +var z7812 [1 << 17]byte +var z7813 [1 << 17]byte +var z7814 [1 << 17]byte +var z7815 [1 << 17]byte +var z7816 [1 << 17]byte +var z7817 [1 << 17]byte +var z7818 [1 << 17]byte +var z7819 [1 << 17]byte +var z7820 [1 << 17]byte +var z7821 [1 << 17]byte +var z7822 [1 << 17]byte +var z7823 [1 << 17]byte +var z7824 [1 << 17]byte +var z7825 [1 << 17]byte +var z7826 [1 << 17]byte +var z7827 [1 << 17]byte +var z7828 [1 << 17]byte +var z7829 [1 << 17]byte +var z7830 [1 << 17]byte +var z7831 [1 << 17]byte +var z7832 [1 << 17]byte +var z7833 [1 << 17]byte +var z7834 [1 << 17]byte +var z7835 [1 << 17]byte +var z7836 [1 << 17]byte +var z7837 [1 << 17]byte +var z7838 [1 << 17]byte +var z7839 [1 << 17]byte +var z7840 [1 << 17]byte +var z7841 [1 << 17]byte +var z7842 [1 << 17]byte +var z7843 [1 << 17]byte +var z7844 [1 << 17]byte +var z7845 [1 << 17]byte +var z7846 [1 << 17]byte +var z7847 [1 << 17]byte +var z7848 [1 << 17]byte +var z7849 [1 << 17]byte +var z7850 [1 << 17]byte +var z7851 [1 << 17]byte +var z7852 [1 << 17]byte +var z7853 [1 << 17]byte +var z7854 [1 << 17]byte +var z7855 [1 << 17]byte +var z7856 [1 << 17]byte +var z7857 [1 << 17]byte +var z7858 [1 << 17]byte +var z7859 [1 << 17]byte +var z7860 [1 << 17]byte +var z7861 [1 << 17]byte +var z7862 [1 << 17]byte +var z7863 [1 << 17]byte +var z7864 [1 << 17]byte +var z7865 [1 << 17]byte +var z7866 [1 << 17]byte +var z7867 [1 << 17]byte +var z7868 [1 << 17]byte +var z7869 [1 << 17]byte +var z7870 [1 << 17]byte +var z7871 [1 << 17]byte +var z7872 [1 << 17]byte +var z7873 [1 << 17]byte +var z7874 [1 << 17]byte +var z7875 [1 << 17]byte +var z7876 [1 << 17]byte +var z7877 [1 << 17]byte +var z7878 [1 << 17]byte +var z7879 [1 << 17]byte +var z7880 [1 << 17]byte +var z7881 [1 << 17]byte +var z7882 [1 << 17]byte +var z7883 [1 << 17]byte +var z7884 [1 << 17]byte +var z7885 [1 << 17]byte +var z7886 [1 << 17]byte +var z7887 [1 << 17]byte +var z7888 [1 << 17]byte +var z7889 [1 << 17]byte +var z7890 [1 << 17]byte +var z7891 [1 << 17]byte +var z7892 [1 << 17]byte +var z7893 [1 << 17]byte +var z7894 [1 << 17]byte +var z7895 [1 << 17]byte +var z7896 [1 << 17]byte +var z7897 [1 << 17]byte +var z7898 [1 << 17]byte +var z7899 [1 << 17]byte +var z7900 [1 << 17]byte +var z7901 [1 << 17]byte +var z7902 [1 << 17]byte +var z7903 [1 << 17]byte +var z7904 [1 << 17]byte +var z7905 [1 << 17]byte +var z7906 [1 << 17]byte +var z7907 [1 << 17]byte +var z7908 [1 << 17]byte +var z7909 [1 << 17]byte +var z7910 [1 << 17]byte +var z7911 [1 << 17]byte +var z7912 [1 << 17]byte +var z7913 [1 << 17]byte +var z7914 [1 << 17]byte +var z7915 [1 << 17]byte +var z7916 [1 << 17]byte +var z7917 [1 << 17]byte +var z7918 [1 << 17]byte +var z7919 [1 << 17]byte +var z7920 [1 << 17]byte +var z7921 [1 << 17]byte +var z7922 [1 << 17]byte +var z7923 [1 << 17]byte +var z7924 [1 << 17]byte +var z7925 [1 << 17]byte +var z7926 [1 << 17]byte +var z7927 [1 << 17]byte +var z7928 [1 << 17]byte +var z7929 [1 << 17]byte +var z7930 [1 << 17]byte +var z7931 [1 << 17]byte +var z7932 [1 << 17]byte +var z7933 [1 << 17]byte +var z7934 [1 << 17]byte +var z7935 [1 << 17]byte +var z7936 [1 << 17]byte +var z7937 [1 << 17]byte +var z7938 [1 << 17]byte +var z7939 [1 << 17]byte +var z7940 [1 << 17]byte +var z7941 [1 << 17]byte +var z7942 [1 << 17]byte +var z7943 [1 << 17]byte +var z7944 [1 << 17]byte +var z7945 [1 << 17]byte +var z7946 [1 << 17]byte +var z7947 [1 << 17]byte +var z7948 [1 << 17]byte +var z7949 [1 << 17]byte +var z7950 [1 << 17]byte +var z7951 [1 << 17]byte +var z7952 [1 << 17]byte +var z7953 [1 << 17]byte +var z7954 [1 << 17]byte +var z7955 [1 << 17]byte +var z7956 [1 << 17]byte +var z7957 [1 << 17]byte +var z7958 [1 << 17]byte +var z7959 [1 << 17]byte +var z7960 [1 << 17]byte +var z7961 [1 << 17]byte +var z7962 [1 << 17]byte +var z7963 [1 << 17]byte +var z7964 [1 << 17]byte +var z7965 [1 << 17]byte +var z7966 [1 << 17]byte +var z7967 [1 << 17]byte +var z7968 [1 << 17]byte +var z7969 [1 << 17]byte +var z7970 [1 << 17]byte +var z7971 [1 << 17]byte +var z7972 [1 << 17]byte +var z7973 [1 << 17]byte +var z7974 [1 << 17]byte +var z7975 [1 << 17]byte +var z7976 [1 << 17]byte +var z7977 [1 << 17]byte +var z7978 [1 << 17]byte +var z7979 [1 << 17]byte +var z7980 [1 << 17]byte +var z7981 [1 << 17]byte +var z7982 [1 << 17]byte +var z7983 [1 << 17]byte +var z7984 [1 << 17]byte +var z7985 [1 << 17]byte +var z7986 [1 << 17]byte +var z7987 [1 << 17]byte +var z7988 [1 << 17]byte +var z7989 [1 << 17]byte +var z7990 [1 << 17]byte +var z7991 [1 << 17]byte +var z7992 [1 << 17]byte +var z7993 [1 << 17]byte +var z7994 [1 << 17]byte +var z7995 [1 << 17]byte +var z7996 [1 << 17]byte +var z7997 [1 << 17]byte +var z7998 [1 << 17]byte +var z7999 [1 << 17]byte +var z8000 [1 << 17]byte +var z8001 [1 << 17]byte +var z8002 [1 << 17]byte +var z8003 [1 << 17]byte +var z8004 [1 << 17]byte +var z8005 [1 << 17]byte +var z8006 [1 << 17]byte +var z8007 [1 << 17]byte +var z8008 [1 << 17]byte +var z8009 [1 << 17]byte +var z8010 [1 << 17]byte +var z8011 [1 << 17]byte +var z8012 [1 << 17]byte +var z8013 [1 << 17]byte +var z8014 [1 << 17]byte +var z8015 [1 << 17]byte +var z8016 [1 << 17]byte +var z8017 [1 << 17]byte +var z8018 [1 << 17]byte +var z8019 [1 << 17]byte +var z8020 [1 << 17]byte +var z8021 [1 << 17]byte +var z8022 [1 << 17]byte +var z8023 [1 << 17]byte +var z8024 [1 << 17]byte +var z8025 [1 << 17]byte +var z8026 [1 << 17]byte +var z8027 [1 << 17]byte +var z8028 [1 << 17]byte +var z8029 [1 << 17]byte +var z8030 [1 << 17]byte +var z8031 [1 << 17]byte +var z8032 [1 << 17]byte +var z8033 [1 << 17]byte +var z8034 [1 << 17]byte +var z8035 [1 << 17]byte +var z8036 [1 << 17]byte +var z8037 [1 << 17]byte +var z8038 [1 << 17]byte +var z8039 [1 << 17]byte +var z8040 [1 << 17]byte +var z8041 [1 << 17]byte +var z8042 [1 << 17]byte +var z8043 [1 << 17]byte +var z8044 [1 << 17]byte +var z8045 [1 << 17]byte +var z8046 [1 << 17]byte +var z8047 [1 << 17]byte +var z8048 [1 << 17]byte +var z8049 [1 << 17]byte +var z8050 [1 << 17]byte +var z8051 [1 << 17]byte +var z8052 [1 << 17]byte +var z8053 [1 << 17]byte +var z8054 [1 << 17]byte +var z8055 [1 << 17]byte +var z8056 [1 << 17]byte +var z8057 [1 << 17]byte +var z8058 [1 << 17]byte +var z8059 [1 << 17]byte +var z8060 [1 << 17]byte +var z8061 [1 << 17]byte +var z8062 [1 << 17]byte +var z8063 [1 << 17]byte +var z8064 [1 << 17]byte +var z8065 [1 << 17]byte +var z8066 [1 << 17]byte +var z8067 [1 << 17]byte +var z8068 [1 << 17]byte +var z8069 [1 << 17]byte +var z8070 [1 << 17]byte +var z8071 [1 << 17]byte +var z8072 [1 << 17]byte +var z8073 [1 << 17]byte +var z8074 [1 << 17]byte +var z8075 [1 << 17]byte +var z8076 [1 << 17]byte +var z8077 [1 << 17]byte +var z8078 [1 << 17]byte +var z8079 [1 << 17]byte +var z8080 [1 << 17]byte +var z8081 [1 << 17]byte +var z8082 [1 << 17]byte +var z8083 [1 << 17]byte +var z8084 [1 << 17]byte +var z8085 [1 << 17]byte +var z8086 [1 << 17]byte +var z8087 [1 << 17]byte +var z8088 [1 << 17]byte +var z8089 [1 << 17]byte +var z8090 [1 << 17]byte +var z8091 [1 << 17]byte +var z8092 [1 << 17]byte +var z8093 [1 << 17]byte +var z8094 [1 << 17]byte +var z8095 [1 << 17]byte +var z8096 [1 << 17]byte +var z8097 [1 << 17]byte +var z8098 [1 << 17]byte +var z8099 [1 << 17]byte +var z8100 [1 << 17]byte +var z8101 [1 << 17]byte +var z8102 [1 << 17]byte +var z8103 [1 << 17]byte +var z8104 [1 << 17]byte +var z8105 [1 << 17]byte +var z8106 [1 << 17]byte +var z8107 [1 << 17]byte +var z8108 [1 << 17]byte +var z8109 [1 << 17]byte +var z8110 [1 << 17]byte +var z8111 [1 << 17]byte +var z8112 [1 << 17]byte +var z8113 [1 << 17]byte +var z8114 [1 << 17]byte +var z8115 [1 << 17]byte +var z8116 [1 << 17]byte +var z8117 [1 << 17]byte +var z8118 [1 << 17]byte +var z8119 [1 << 17]byte +var z8120 [1 << 17]byte +var z8121 [1 << 17]byte +var z8122 [1 << 17]byte +var z8123 [1 << 17]byte +var z8124 [1 << 17]byte +var z8125 [1 << 17]byte +var z8126 [1 << 17]byte +var z8127 [1 << 17]byte +var z8128 [1 << 17]byte +var z8129 [1 << 17]byte +var z8130 [1 << 17]byte +var z8131 [1 << 17]byte +var z8132 [1 << 17]byte +var z8133 [1 << 17]byte +var z8134 [1 << 17]byte +var z8135 [1 << 17]byte +var z8136 [1 << 17]byte +var z8137 [1 << 17]byte +var z8138 [1 << 17]byte +var z8139 [1 << 17]byte +var z8140 [1 << 17]byte +var z8141 [1 << 17]byte +var z8142 [1 << 17]byte +var z8143 [1 << 17]byte +var z8144 [1 << 17]byte +var z8145 [1 << 17]byte +var z8146 [1 << 17]byte +var z8147 [1 << 17]byte +var z8148 [1 << 17]byte +var z8149 [1 << 17]byte +var z8150 [1 << 17]byte +var z8151 [1 << 17]byte +var z8152 [1 << 17]byte +var z8153 [1 << 17]byte +var z8154 [1 << 17]byte +var z8155 [1 << 17]byte +var z8156 [1 << 17]byte +var z8157 [1 << 17]byte +var z8158 [1 << 17]byte +var z8159 [1 << 17]byte +var z8160 [1 << 17]byte +var z8161 [1 << 17]byte +var z8162 [1 << 17]byte +var z8163 [1 << 17]byte +var z8164 [1 << 17]byte +var z8165 [1 << 17]byte +var z8166 [1 << 17]byte +var z8167 [1 << 17]byte +var z8168 [1 << 17]byte +var z8169 [1 << 17]byte +var z8170 [1 << 17]byte +var z8171 [1 << 17]byte +var z8172 [1 << 17]byte +var z8173 [1 << 17]byte +var z8174 [1 << 17]byte +var z8175 [1 << 17]byte +var z8176 [1 << 17]byte +var z8177 [1 << 17]byte +var z8178 [1 << 17]byte +var z8179 [1 << 17]byte +var z8180 [1 << 17]byte +var z8181 [1 << 17]byte +var z8182 [1 << 17]byte +var z8183 [1 << 17]byte +var z8184 [1 << 17]byte +var z8185 [1 << 17]byte +var z8186 [1 << 17]byte +var z8187 [1 << 17]byte +var z8188 [1 << 17]byte +var z8189 [1 << 17]byte +var z8190 [1 << 17]byte +var z8191 [1 << 17]byte +var z8192 [1 << 17]byte +var z8193 [1 << 17]byte +var z8194 [1 << 17]byte +var z8195 [1 << 17]byte +var z8196 [1 << 17]byte +var z8197 [1 << 17]byte +var z8198 [1 << 17]byte +var z8199 [1 << 17]byte +var z8200 [1 << 17]byte +var z8201 [1 << 17]byte +var z8202 [1 << 17]byte +var z8203 [1 << 17]byte +var z8204 [1 << 17]byte +var z8205 [1 << 17]byte +var z8206 [1 << 17]byte +var z8207 [1 << 17]byte +var z8208 [1 << 17]byte +var z8209 [1 << 17]byte +var z8210 [1 << 17]byte +var z8211 [1 << 17]byte +var z8212 [1 << 17]byte +var z8213 [1 << 17]byte +var z8214 [1 << 17]byte +var z8215 [1 << 17]byte +var z8216 [1 << 17]byte +var z8217 [1 << 17]byte +var z8218 [1 << 17]byte +var z8219 [1 << 17]byte +var z8220 [1 << 17]byte +var z8221 [1 << 17]byte +var z8222 [1 << 17]byte +var z8223 [1 << 17]byte +var z8224 [1 << 17]byte +var z8225 [1 << 17]byte +var z8226 [1 << 17]byte +var z8227 [1 << 17]byte +var z8228 [1 << 17]byte +var z8229 [1 << 17]byte +var z8230 [1 << 17]byte +var z8231 [1 << 17]byte +var z8232 [1 << 17]byte +var z8233 [1 << 17]byte +var z8234 [1 << 17]byte +var z8235 [1 << 17]byte +var z8236 [1 << 17]byte +var z8237 [1 << 17]byte +var z8238 [1 << 17]byte +var z8239 [1 << 17]byte +var z8240 [1 << 17]byte +var z8241 [1 << 17]byte +var z8242 [1 << 17]byte +var z8243 [1 << 17]byte +var z8244 [1 << 17]byte +var z8245 [1 << 17]byte +var z8246 [1 << 17]byte +var z8247 [1 << 17]byte +var z8248 [1 << 17]byte +var z8249 [1 << 17]byte +var z8250 [1 << 17]byte +var z8251 [1 << 17]byte +var z8252 [1 << 17]byte +var z8253 [1 << 17]byte +var z8254 [1 << 17]byte +var z8255 [1 << 17]byte +var z8256 [1 << 17]byte +var z8257 [1 << 17]byte +var z8258 [1 << 17]byte +var z8259 [1 << 17]byte +var z8260 [1 << 17]byte +var z8261 [1 << 17]byte +var z8262 [1 << 17]byte +var z8263 [1 << 17]byte +var z8264 [1 << 17]byte +var z8265 [1 << 17]byte +var z8266 [1 << 17]byte +var z8267 [1 << 17]byte +var z8268 [1 << 17]byte +var z8269 [1 << 17]byte +var z8270 [1 << 17]byte +var z8271 [1 << 17]byte +var z8272 [1 << 17]byte +var z8273 [1 << 17]byte +var z8274 [1 << 17]byte +var z8275 [1 << 17]byte +var z8276 [1 << 17]byte +var z8277 [1 << 17]byte +var z8278 [1 << 17]byte +var z8279 [1 << 17]byte +var z8280 [1 << 17]byte +var z8281 [1 << 17]byte +var z8282 [1 << 17]byte +var z8283 [1 << 17]byte +var z8284 [1 << 17]byte +var z8285 [1 << 17]byte +var z8286 [1 << 17]byte +var z8287 [1 << 17]byte +var z8288 [1 << 17]byte +var z8289 [1 << 17]byte +var z8290 [1 << 17]byte +var z8291 [1 << 17]byte +var z8292 [1 << 17]byte +var z8293 [1 << 17]byte +var z8294 [1 << 17]byte +var z8295 [1 << 17]byte +var z8296 [1 << 17]byte +var z8297 [1 << 17]byte +var z8298 [1 << 17]byte +var z8299 [1 << 17]byte +var z8300 [1 << 17]byte +var z8301 [1 << 17]byte +var z8302 [1 << 17]byte +var z8303 [1 << 17]byte +var z8304 [1 << 17]byte +var z8305 [1 << 17]byte +var z8306 [1 << 17]byte +var z8307 [1 << 17]byte +var z8308 [1 << 17]byte +var z8309 [1 << 17]byte +var z8310 [1 << 17]byte +var z8311 [1 << 17]byte +var z8312 [1 << 17]byte +var z8313 [1 << 17]byte +var z8314 [1 << 17]byte +var z8315 [1 << 17]byte +var z8316 [1 << 17]byte +var z8317 [1 << 17]byte +var z8318 [1 << 17]byte +var z8319 [1 << 17]byte +var z8320 [1 << 17]byte +var z8321 [1 << 17]byte +var z8322 [1 << 17]byte +var z8323 [1 << 17]byte +var z8324 [1 << 17]byte +var z8325 [1 << 17]byte +var z8326 [1 << 17]byte +var z8327 [1 << 17]byte +var z8328 [1 << 17]byte +var z8329 [1 << 17]byte +var z8330 [1 << 17]byte +var z8331 [1 << 17]byte +var z8332 [1 << 17]byte +var z8333 [1 << 17]byte +var z8334 [1 << 17]byte +var z8335 [1 << 17]byte +var z8336 [1 << 17]byte +var z8337 [1 << 17]byte +var z8338 [1 << 17]byte +var z8339 [1 << 17]byte +var z8340 [1 << 17]byte +var z8341 [1 << 17]byte +var z8342 [1 << 17]byte +var z8343 [1 << 17]byte +var z8344 [1 << 17]byte +var z8345 [1 << 17]byte +var z8346 [1 << 17]byte +var z8347 [1 << 17]byte +var z8348 [1 << 17]byte +var z8349 [1 << 17]byte +var z8350 [1 << 17]byte +var z8351 [1 << 17]byte +var z8352 [1 << 17]byte +var z8353 [1 << 17]byte +var z8354 [1 << 17]byte +var z8355 [1 << 17]byte +var z8356 [1 << 17]byte +var z8357 [1 << 17]byte +var z8358 [1 << 17]byte +var z8359 [1 << 17]byte +var z8360 [1 << 17]byte +var z8361 [1 << 17]byte +var z8362 [1 << 17]byte +var z8363 [1 << 17]byte +var z8364 [1 << 17]byte +var z8365 [1 << 17]byte +var z8366 [1 << 17]byte +var z8367 [1 << 17]byte +var z8368 [1 << 17]byte +var z8369 [1 << 17]byte +var z8370 [1 << 17]byte +var z8371 [1 << 17]byte +var z8372 [1 << 17]byte +var z8373 [1 << 17]byte +var z8374 [1 << 17]byte +var z8375 [1 << 17]byte +var z8376 [1 << 17]byte +var z8377 [1 << 17]byte +var z8378 [1 << 17]byte +var z8379 [1 << 17]byte +var z8380 [1 << 17]byte +var z8381 [1 << 17]byte +var z8382 [1 << 17]byte +var z8383 [1 << 17]byte +var z8384 [1 << 17]byte +var z8385 [1 << 17]byte +var z8386 [1 << 17]byte +var z8387 [1 << 17]byte +var z8388 [1 << 17]byte +var z8389 [1 << 17]byte +var z8390 [1 << 17]byte +var z8391 [1 << 17]byte +var z8392 [1 << 17]byte +var z8393 [1 << 17]byte +var z8394 [1 << 17]byte +var z8395 [1 << 17]byte +var z8396 [1 << 17]byte +var z8397 [1 << 17]byte +var z8398 [1 << 17]byte +var z8399 [1 << 17]byte +var z8400 [1 << 17]byte +var z8401 [1 << 17]byte +var z8402 [1 << 17]byte +var z8403 [1 << 17]byte +var z8404 [1 << 17]byte +var z8405 [1 << 17]byte +var z8406 [1 << 17]byte +var z8407 [1 << 17]byte +var z8408 [1 << 17]byte +var z8409 [1 << 17]byte +var z8410 [1 << 17]byte +var z8411 [1 << 17]byte +var z8412 [1 << 17]byte +var z8413 [1 << 17]byte +var z8414 [1 << 17]byte +var z8415 [1 << 17]byte +var z8416 [1 << 17]byte +var z8417 [1 << 17]byte +var z8418 [1 << 17]byte +var z8419 [1 << 17]byte +var z8420 [1 << 17]byte +var z8421 [1 << 17]byte +var z8422 [1 << 17]byte +var z8423 [1 << 17]byte +var z8424 [1 << 17]byte +var z8425 [1 << 17]byte +var z8426 [1 << 17]byte +var z8427 [1 << 17]byte +var z8428 [1 << 17]byte +var z8429 [1 << 17]byte +var z8430 [1 << 17]byte +var z8431 [1 << 17]byte +var z8432 [1 << 17]byte +var z8433 [1 << 17]byte +var z8434 [1 << 17]byte +var z8435 [1 << 17]byte +var z8436 [1 << 17]byte +var z8437 [1 << 17]byte +var z8438 [1 << 17]byte +var z8439 [1 << 17]byte +var z8440 [1 << 17]byte +var z8441 [1 << 17]byte +var z8442 [1 << 17]byte +var z8443 [1 << 17]byte +var z8444 [1 << 17]byte +var z8445 [1 << 17]byte +var z8446 [1 << 17]byte +var z8447 [1 << 17]byte +var z8448 [1 << 17]byte +var z8449 [1 << 17]byte +var z8450 [1 << 17]byte +var z8451 [1 << 17]byte +var z8452 [1 << 17]byte +var z8453 [1 << 17]byte +var z8454 [1 << 17]byte +var z8455 [1 << 17]byte +var z8456 [1 << 17]byte +var z8457 [1 << 17]byte +var z8458 [1 << 17]byte +var z8459 [1 << 17]byte +var z8460 [1 << 17]byte +var z8461 [1 << 17]byte +var z8462 [1 << 17]byte +var z8463 [1 << 17]byte +var z8464 [1 << 17]byte +var z8465 [1 << 17]byte +var z8466 [1 << 17]byte +var z8467 [1 << 17]byte +var z8468 [1 << 17]byte +var z8469 [1 << 17]byte +var z8470 [1 << 17]byte +var z8471 [1 << 17]byte +var z8472 [1 << 17]byte +var z8473 [1 << 17]byte +var z8474 [1 << 17]byte +var z8475 [1 << 17]byte +var z8476 [1 << 17]byte +var z8477 [1 << 17]byte +var z8478 [1 << 17]byte +var z8479 [1 << 17]byte +var z8480 [1 << 17]byte +var z8481 [1 << 17]byte +var z8482 [1 << 17]byte +var z8483 [1 << 17]byte +var z8484 [1 << 17]byte +var z8485 [1 << 17]byte +var z8486 [1 << 17]byte +var z8487 [1 << 17]byte +var z8488 [1 << 17]byte +var z8489 [1 << 17]byte +var z8490 [1 << 17]byte +var z8491 [1 << 17]byte +var z8492 [1 << 17]byte +var z8493 [1 << 17]byte +var z8494 [1 << 17]byte +var z8495 [1 << 17]byte +var z8496 [1 << 17]byte +var z8497 [1 << 17]byte +var z8498 [1 << 17]byte +var z8499 [1 << 17]byte +var z8500 [1 << 17]byte +var z8501 [1 << 17]byte +var z8502 [1 << 17]byte +var z8503 [1 << 17]byte +var z8504 [1 << 17]byte +var z8505 [1 << 17]byte +var z8506 [1 << 17]byte +var z8507 [1 << 17]byte +var z8508 [1 << 17]byte +var z8509 [1 << 17]byte +var z8510 [1 << 17]byte +var z8511 [1 << 17]byte +var z8512 [1 << 17]byte +var z8513 [1 << 17]byte +var z8514 [1 << 17]byte +var z8515 [1 << 17]byte +var z8516 [1 << 17]byte +var z8517 [1 << 17]byte +var z8518 [1 << 17]byte +var z8519 [1 << 17]byte +var z8520 [1 << 17]byte +var z8521 [1 << 17]byte +var z8522 [1 << 17]byte +var z8523 [1 << 17]byte +var z8524 [1 << 17]byte +var z8525 [1 << 17]byte +var z8526 [1 << 17]byte +var z8527 [1 << 17]byte +var z8528 [1 << 17]byte +var z8529 [1 << 17]byte +var z8530 [1 << 17]byte +var z8531 [1 << 17]byte +var z8532 [1 << 17]byte +var z8533 [1 << 17]byte +var z8534 [1 << 17]byte +var z8535 [1 << 17]byte +var z8536 [1 << 17]byte +var z8537 [1 << 17]byte +var z8538 [1 << 17]byte +var z8539 [1 << 17]byte +var z8540 [1 << 17]byte +var z8541 [1 << 17]byte +var z8542 [1 << 17]byte +var z8543 [1 << 17]byte +var z8544 [1 << 17]byte +var z8545 [1 << 17]byte +var z8546 [1 << 17]byte +var z8547 [1 << 17]byte +var z8548 [1 << 17]byte +var z8549 [1 << 17]byte +var z8550 [1 << 17]byte +var z8551 [1 << 17]byte +var z8552 [1 << 17]byte +var z8553 [1 << 17]byte +var z8554 [1 << 17]byte +var z8555 [1 << 17]byte +var z8556 [1 << 17]byte +var z8557 [1 << 17]byte +var z8558 [1 << 17]byte +var z8559 [1 << 17]byte +var z8560 [1 << 17]byte +var z8561 [1 << 17]byte +var z8562 [1 << 17]byte +var z8563 [1 << 17]byte +var z8564 [1 << 17]byte +var z8565 [1 << 17]byte +var z8566 [1 << 17]byte +var z8567 [1 << 17]byte +var z8568 [1 << 17]byte +var z8569 [1 << 17]byte +var z8570 [1 << 17]byte +var z8571 [1 << 17]byte +var z8572 [1 << 17]byte +var z8573 [1 << 17]byte +var z8574 [1 << 17]byte +var z8575 [1 << 17]byte +var z8576 [1 << 17]byte +var z8577 [1 << 17]byte +var z8578 [1 << 17]byte +var z8579 [1 << 17]byte +var z8580 [1 << 17]byte +var z8581 [1 << 17]byte +var z8582 [1 << 17]byte +var z8583 [1 << 17]byte +var z8584 [1 << 17]byte +var z8585 [1 << 17]byte +var z8586 [1 << 17]byte +var z8587 [1 << 17]byte +var z8588 [1 << 17]byte +var z8589 [1 << 17]byte +var z8590 [1 << 17]byte +var z8591 [1 << 17]byte +var z8592 [1 << 17]byte +var z8593 [1 << 17]byte +var z8594 [1 << 17]byte +var z8595 [1 << 17]byte +var z8596 [1 << 17]byte +var z8597 [1 << 17]byte +var z8598 [1 << 17]byte +var z8599 [1 << 17]byte +var z8600 [1 << 17]byte +var z8601 [1 << 17]byte +var z8602 [1 << 17]byte +var z8603 [1 << 17]byte +var z8604 [1 << 17]byte +var z8605 [1 << 17]byte +var z8606 [1 << 17]byte +var z8607 [1 << 17]byte +var z8608 [1 << 17]byte +var z8609 [1 << 17]byte +var z8610 [1 << 17]byte +var z8611 [1 << 17]byte +var z8612 [1 << 17]byte +var z8613 [1 << 17]byte +var z8614 [1 << 17]byte +var z8615 [1 << 17]byte +var z8616 [1 << 17]byte +var z8617 [1 << 17]byte +var z8618 [1 << 17]byte +var z8619 [1 << 17]byte +var z8620 [1 << 17]byte +var z8621 [1 << 17]byte +var z8622 [1 << 17]byte +var z8623 [1 << 17]byte +var z8624 [1 << 17]byte +var z8625 [1 << 17]byte +var z8626 [1 << 17]byte +var z8627 [1 << 17]byte +var z8628 [1 << 17]byte +var z8629 [1 << 17]byte +var z8630 [1 << 17]byte +var z8631 [1 << 17]byte +var z8632 [1 << 17]byte +var z8633 [1 << 17]byte +var z8634 [1 << 17]byte +var z8635 [1 << 17]byte +var z8636 [1 << 17]byte +var z8637 [1 << 17]byte +var z8638 [1 << 17]byte +var z8639 [1 << 17]byte +var z8640 [1 << 17]byte +var z8641 [1 << 17]byte +var z8642 [1 << 17]byte +var z8643 [1 << 17]byte +var z8644 [1 << 17]byte +var z8645 [1 << 17]byte +var z8646 [1 << 17]byte +var z8647 [1 << 17]byte +var z8648 [1 << 17]byte +var z8649 [1 << 17]byte +var z8650 [1 << 17]byte +var z8651 [1 << 17]byte +var z8652 [1 << 17]byte +var z8653 [1 << 17]byte +var z8654 [1 << 17]byte +var z8655 [1 << 17]byte +var z8656 [1 << 17]byte +var z8657 [1 << 17]byte +var z8658 [1 << 17]byte +var z8659 [1 << 17]byte +var z8660 [1 << 17]byte +var z8661 [1 << 17]byte +var z8662 [1 << 17]byte +var z8663 [1 << 17]byte +var z8664 [1 << 17]byte +var z8665 [1 << 17]byte +var z8666 [1 << 17]byte +var z8667 [1 << 17]byte +var z8668 [1 << 17]byte +var z8669 [1 << 17]byte +var z8670 [1 << 17]byte +var z8671 [1 << 17]byte +var z8672 [1 << 17]byte +var z8673 [1 << 17]byte +var z8674 [1 << 17]byte +var z8675 [1 << 17]byte +var z8676 [1 << 17]byte +var z8677 [1 << 17]byte +var z8678 [1 << 17]byte +var z8679 [1 << 17]byte +var z8680 [1 << 17]byte +var z8681 [1 << 17]byte +var z8682 [1 << 17]byte +var z8683 [1 << 17]byte +var z8684 [1 << 17]byte +var z8685 [1 << 17]byte +var z8686 [1 << 17]byte +var z8687 [1 << 17]byte +var z8688 [1 << 17]byte +var z8689 [1 << 17]byte +var z8690 [1 << 17]byte +var z8691 [1 << 17]byte +var z8692 [1 << 17]byte +var z8693 [1 << 17]byte +var z8694 [1 << 17]byte +var z8695 [1 << 17]byte +var z8696 [1 << 17]byte +var z8697 [1 << 17]byte +var z8698 [1 << 17]byte +var z8699 [1 << 17]byte +var z8700 [1 << 17]byte +var z8701 [1 << 17]byte +var z8702 [1 << 17]byte +var z8703 [1 << 17]byte +var z8704 [1 << 17]byte +var z8705 [1 << 17]byte +var z8706 [1 << 17]byte +var z8707 [1 << 17]byte +var z8708 [1 << 17]byte +var z8709 [1 << 17]byte +var z8710 [1 << 17]byte +var z8711 [1 << 17]byte +var z8712 [1 << 17]byte +var z8713 [1 << 17]byte +var z8714 [1 << 17]byte +var z8715 [1 << 17]byte +var z8716 [1 << 17]byte +var z8717 [1 << 17]byte +var z8718 [1 << 17]byte +var z8719 [1 << 17]byte +var z8720 [1 << 17]byte +var z8721 [1 << 17]byte +var z8722 [1 << 17]byte +var z8723 [1 << 17]byte +var z8724 [1 << 17]byte +var z8725 [1 << 17]byte +var z8726 [1 << 17]byte +var z8727 [1 << 17]byte +var z8728 [1 << 17]byte +var z8729 [1 << 17]byte +var z8730 [1 << 17]byte +var z8731 [1 << 17]byte +var z8732 [1 << 17]byte +var z8733 [1 << 17]byte +var z8734 [1 << 17]byte +var z8735 [1 << 17]byte +var z8736 [1 << 17]byte +var z8737 [1 << 17]byte +var z8738 [1 << 17]byte +var z8739 [1 << 17]byte +var z8740 [1 << 17]byte +var z8741 [1 << 17]byte +var z8742 [1 << 17]byte +var z8743 [1 << 17]byte +var z8744 [1 << 17]byte +var z8745 [1 << 17]byte +var z8746 [1 << 17]byte +var z8747 [1 << 17]byte +var z8748 [1 << 17]byte +var z8749 [1 << 17]byte +var z8750 [1 << 17]byte +var z8751 [1 << 17]byte +var z8752 [1 << 17]byte +var z8753 [1 << 17]byte +var z8754 [1 << 17]byte +var z8755 [1 << 17]byte +var z8756 [1 << 17]byte +var z8757 [1 << 17]byte +var z8758 [1 << 17]byte +var z8759 [1 << 17]byte +var z8760 [1 << 17]byte +var z8761 [1 << 17]byte +var z8762 [1 << 17]byte +var z8763 [1 << 17]byte +var z8764 [1 << 17]byte +var z8765 [1 << 17]byte +var z8766 [1 << 17]byte +var z8767 [1 << 17]byte +var z8768 [1 << 17]byte +var z8769 [1 << 17]byte +var z8770 [1 << 17]byte +var z8771 [1 << 17]byte +var z8772 [1 << 17]byte +var z8773 [1 << 17]byte +var z8774 [1 << 17]byte +var z8775 [1 << 17]byte +var z8776 [1 << 17]byte +var z8777 [1 << 17]byte +var z8778 [1 << 17]byte +var z8779 [1 << 17]byte +var z8780 [1 << 17]byte +var z8781 [1 << 17]byte +var z8782 [1 << 17]byte +var z8783 [1 << 17]byte +var z8784 [1 << 17]byte +var z8785 [1 << 17]byte +var z8786 [1 << 17]byte +var z8787 [1 << 17]byte +var z8788 [1 << 17]byte +var z8789 [1 << 17]byte +var z8790 [1 << 17]byte +var z8791 [1 << 17]byte +var z8792 [1 << 17]byte +var z8793 [1 << 17]byte +var z8794 [1 << 17]byte +var z8795 [1 << 17]byte +var z8796 [1 << 17]byte +var z8797 [1 << 17]byte +var z8798 [1 << 17]byte +var z8799 [1 << 17]byte +var z8800 [1 << 17]byte +var z8801 [1 << 17]byte +var z8802 [1 << 17]byte +var z8803 [1 << 17]byte +var z8804 [1 << 17]byte +var z8805 [1 << 17]byte +var z8806 [1 << 17]byte +var z8807 [1 << 17]byte +var z8808 [1 << 17]byte +var z8809 [1 << 17]byte +var z8810 [1 << 17]byte +var z8811 [1 << 17]byte +var z8812 [1 << 17]byte +var z8813 [1 << 17]byte +var z8814 [1 << 17]byte +var z8815 [1 << 17]byte +var z8816 [1 << 17]byte +var z8817 [1 << 17]byte +var z8818 [1 << 17]byte +var z8819 [1 << 17]byte +var z8820 [1 << 17]byte +var z8821 [1 << 17]byte +var z8822 [1 << 17]byte +var z8823 [1 << 17]byte +var z8824 [1 << 17]byte +var z8825 [1 << 17]byte +var z8826 [1 << 17]byte +var z8827 [1 << 17]byte +var z8828 [1 << 17]byte +var z8829 [1 << 17]byte +var z8830 [1 << 17]byte +var z8831 [1 << 17]byte +var z8832 [1 << 17]byte +var z8833 [1 << 17]byte +var z8834 [1 << 17]byte +var z8835 [1 << 17]byte +var z8836 [1 << 17]byte +var z8837 [1 << 17]byte +var z8838 [1 << 17]byte +var z8839 [1 << 17]byte +var z8840 [1 << 17]byte +var z8841 [1 << 17]byte +var z8842 [1 << 17]byte +var z8843 [1 << 17]byte +var z8844 [1 << 17]byte +var z8845 [1 << 17]byte +var z8846 [1 << 17]byte +var z8847 [1 << 17]byte +var z8848 [1 << 17]byte +var z8849 [1 << 17]byte +var z8850 [1 << 17]byte +var z8851 [1 << 17]byte +var z8852 [1 << 17]byte +var z8853 [1 << 17]byte +var z8854 [1 << 17]byte +var z8855 [1 << 17]byte +var z8856 [1 << 17]byte +var z8857 [1 << 17]byte +var z8858 [1 << 17]byte +var z8859 [1 << 17]byte +var z8860 [1 << 17]byte +var z8861 [1 << 17]byte +var z8862 [1 << 17]byte +var z8863 [1 << 17]byte +var z8864 [1 << 17]byte +var z8865 [1 << 17]byte +var z8866 [1 << 17]byte +var z8867 [1 << 17]byte +var z8868 [1 << 17]byte +var z8869 [1 << 17]byte +var z8870 [1 << 17]byte +var z8871 [1 << 17]byte +var z8872 [1 << 17]byte +var z8873 [1 << 17]byte +var z8874 [1 << 17]byte +var z8875 [1 << 17]byte +var z8876 [1 << 17]byte +var z8877 [1 << 17]byte +var z8878 [1 << 17]byte +var z8879 [1 << 17]byte +var z8880 [1 << 17]byte +var z8881 [1 << 17]byte +var z8882 [1 << 17]byte +var z8883 [1 << 17]byte +var z8884 [1 << 17]byte +var z8885 [1 << 17]byte +var z8886 [1 << 17]byte +var z8887 [1 << 17]byte +var z8888 [1 << 17]byte +var z8889 [1 << 17]byte +var z8890 [1 << 17]byte +var z8891 [1 << 17]byte +var z8892 [1 << 17]byte +var z8893 [1 << 17]byte +var z8894 [1 << 17]byte +var z8895 [1 << 17]byte +var z8896 [1 << 17]byte +var z8897 [1 << 17]byte +var z8898 [1 << 17]byte +var z8899 [1 << 17]byte +var z8900 [1 << 17]byte +var z8901 [1 << 17]byte +var z8902 [1 << 17]byte +var z8903 [1 << 17]byte +var z8904 [1 << 17]byte +var z8905 [1 << 17]byte +var z8906 [1 << 17]byte +var z8907 [1 << 17]byte +var z8908 [1 << 17]byte +var z8909 [1 << 17]byte +var z8910 [1 << 17]byte +var z8911 [1 << 17]byte +var z8912 [1 << 17]byte +var z8913 [1 << 17]byte +var z8914 [1 << 17]byte +var z8915 [1 << 17]byte +var z8916 [1 << 17]byte +var z8917 [1 << 17]byte +var z8918 [1 << 17]byte +var z8919 [1 << 17]byte +var z8920 [1 << 17]byte +var z8921 [1 << 17]byte +var z8922 [1 << 17]byte +var z8923 [1 << 17]byte +var z8924 [1 << 17]byte +var z8925 [1 << 17]byte +var z8926 [1 << 17]byte +var z8927 [1 << 17]byte +var z8928 [1 << 17]byte +var z8929 [1 << 17]byte +var z8930 [1 << 17]byte +var z8931 [1 << 17]byte +var z8932 [1 << 17]byte +var z8933 [1 << 17]byte +var z8934 [1 << 17]byte +var z8935 [1 << 17]byte +var z8936 [1 << 17]byte +var z8937 [1 << 17]byte +var z8938 [1 << 17]byte +var z8939 [1 << 17]byte +var z8940 [1 << 17]byte +var z8941 [1 << 17]byte +var z8942 [1 << 17]byte +var z8943 [1 << 17]byte +var z8944 [1 << 17]byte +var z8945 [1 << 17]byte +var z8946 [1 << 17]byte +var z8947 [1 << 17]byte +var z8948 [1 << 17]byte +var z8949 [1 << 17]byte +var z8950 [1 << 17]byte +var z8951 [1 << 17]byte +var z8952 [1 << 17]byte +var z8953 [1 << 17]byte +var z8954 [1 << 17]byte +var z8955 [1 << 17]byte +var z8956 [1 << 17]byte +var z8957 [1 << 17]byte +var z8958 [1 << 17]byte +var z8959 [1 << 17]byte +var z8960 [1 << 17]byte +var z8961 [1 << 17]byte +var z8962 [1 << 17]byte +var z8963 [1 << 17]byte +var z8964 [1 << 17]byte +var z8965 [1 << 17]byte +var z8966 [1 << 17]byte +var z8967 [1 << 17]byte +var z8968 [1 << 17]byte +var z8969 [1 << 17]byte +var z8970 [1 << 17]byte +var z8971 [1 << 17]byte +var z8972 [1 << 17]byte +var z8973 [1 << 17]byte +var z8974 [1 << 17]byte +var z8975 [1 << 17]byte +var z8976 [1 << 17]byte +var z8977 [1 << 17]byte +var z8978 [1 << 17]byte +var z8979 [1 << 17]byte +var z8980 [1 << 17]byte +var z8981 [1 << 17]byte +var z8982 [1 << 17]byte +var z8983 [1 << 17]byte +var z8984 [1 << 17]byte +var z8985 [1 << 17]byte +var z8986 [1 << 17]byte +var z8987 [1 << 17]byte +var z8988 [1 << 17]byte +var z8989 [1 << 17]byte +var z8990 [1 << 17]byte +var z8991 [1 << 17]byte +var z8992 [1 << 17]byte +var z8993 [1 << 17]byte +var z8994 [1 << 17]byte +var z8995 [1 << 17]byte +var z8996 [1 << 17]byte +var z8997 [1 << 17]byte +var z8998 [1 << 17]byte +var z8999 [1 << 17]byte +var z9000 [1 << 17]byte +var z9001 [1 << 17]byte +var z9002 [1 << 17]byte +var z9003 [1 << 17]byte +var z9004 [1 << 17]byte +var z9005 [1 << 17]byte +var z9006 [1 << 17]byte +var z9007 [1 << 17]byte +var z9008 [1 << 17]byte +var z9009 [1 << 17]byte +var z9010 [1 << 17]byte +var z9011 [1 << 17]byte +var z9012 [1 << 17]byte +var z9013 [1 << 17]byte +var z9014 [1 << 17]byte +var z9015 [1 << 17]byte +var z9016 [1 << 17]byte +var z9017 [1 << 17]byte +var z9018 [1 << 17]byte +var z9019 [1 << 17]byte +var z9020 [1 << 17]byte +var z9021 [1 << 17]byte +var z9022 [1 << 17]byte +var z9023 [1 << 17]byte +var z9024 [1 << 17]byte +var z9025 [1 << 17]byte +var z9026 [1 << 17]byte +var z9027 [1 << 17]byte +var z9028 [1 << 17]byte +var z9029 [1 << 17]byte +var z9030 [1 << 17]byte +var z9031 [1 << 17]byte +var z9032 [1 << 17]byte +var z9033 [1 << 17]byte +var z9034 [1 << 17]byte +var z9035 [1 << 17]byte +var z9036 [1 << 17]byte +var z9037 [1 << 17]byte +var z9038 [1 << 17]byte +var z9039 [1 << 17]byte +var z9040 [1 << 17]byte +var z9041 [1 << 17]byte +var z9042 [1 << 17]byte +var z9043 [1 << 17]byte +var z9044 [1 << 17]byte +var z9045 [1 << 17]byte +var z9046 [1 << 17]byte +var z9047 [1 << 17]byte +var z9048 [1 << 17]byte +var z9049 [1 << 17]byte +var z9050 [1 << 17]byte +var z9051 [1 << 17]byte +var z9052 [1 << 17]byte +var z9053 [1 << 17]byte +var z9054 [1 << 17]byte +var z9055 [1 << 17]byte +var z9056 [1 << 17]byte +var z9057 [1 << 17]byte +var z9058 [1 << 17]byte +var z9059 [1 << 17]byte +var z9060 [1 << 17]byte +var z9061 [1 << 17]byte +var z9062 [1 << 17]byte +var z9063 [1 << 17]byte +var z9064 [1 << 17]byte +var z9065 [1 << 17]byte +var z9066 [1 << 17]byte +var z9067 [1 << 17]byte +var z9068 [1 << 17]byte +var z9069 [1 << 17]byte +var z9070 [1 << 17]byte +var z9071 [1 << 17]byte +var z9072 [1 << 17]byte +var z9073 [1 << 17]byte +var z9074 [1 << 17]byte +var z9075 [1 << 17]byte +var z9076 [1 << 17]byte +var z9077 [1 << 17]byte +var z9078 [1 << 17]byte +var z9079 [1 << 17]byte +var z9080 [1 << 17]byte +var z9081 [1 << 17]byte +var z9082 [1 << 17]byte +var z9083 [1 << 17]byte +var z9084 [1 << 17]byte +var z9085 [1 << 17]byte +var z9086 [1 << 17]byte +var z9087 [1 << 17]byte +var z9088 [1 << 17]byte +var z9089 [1 << 17]byte +var z9090 [1 << 17]byte +var z9091 [1 << 17]byte +var z9092 [1 << 17]byte +var z9093 [1 << 17]byte +var z9094 [1 << 17]byte +var z9095 [1 << 17]byte +var z9096 [1 << 17]byte +var z9097 [1 << 17]byte +var z9098 [1 << 17]byte +var z9099 [1 << 17]byte +var z9100 [1 << 17]byte +var z9101 [1 << 17]byte +var z9102 [1 << 17]byte +var z9103 [1 << 17]byte +var z9104 [1 << 17]byte +var z9105 [1 << 17]byte +var z9106 [1 << 17]byte +var z9107 [1 << 17]byte +var z9108 [1 << 17]byte +var z9109 [1 << 17]byte +var z9110 [1 << 17]byte +var z9111 [1 << 17]byte +var z9112 [1 << 17]byte +var z9113 [1 << 17]byte +var z9114 [1 << 17]byte +var z9115 [1 << 17]byte +var z9116 [1 << 17]byte +var z9117 [1 << 17]byte +var z9118 [1 << 17]byte +var z9119 [1 << 17]byte +var z9120 [1 << 17]byte +var z9121 [1 << 17]byte +var z9122 [1 << 17]byte +var z9123 [1 << 17]byte +var z9124 [1 << 17]byte +var z9125 [1 << 17]byte +var z9126 [1 << 17]byte +var z9127 [1 << 17]byte +var z9128 [1 << 17]byte +var z9129 [1 << 17]byte +var z9130 [1 << 17]byte +var z9131 [1 << 17]byte +var z9132 [1 << 17]byte +var z9133 [1 << 17]byte +var z9134 [1 << 17]byte +var z9135 [1 << 17]byte +var z9136 [1 << 17]byte +var z9137 [1 << 17]byte +var z9138 [1 << 17]byte +var z9139 [1 << 17]byte +var z9140 [1 << 17]byte +var z9141 [1 << 17]byte +var z9142 [1 << 17]byte +var z9143 [1 << 17]byte +var z9144 [1 << 17]byte +var z9145 [1 << 17]byte +var z9146 [1 << 17]byte +var z9147 [1 << 17]byte +var z9148 [1 << 17]byte +var z9149 [1 << 17]byte +var z9150 [1 << 17]byte +var z9151 [1 << 17]byte +var z9152 [1 << 17]byte +var z9153 [1 << 17]byte +var z9154 [1 << 17]byte +var z9155 [1 << 17]byte +var z9156 [1 << 17]byte +var z9157 [1 << 17]byte +var z9158 [1 << 17]byte +var z9159 [1 << 17]byte +var z9160 [1 << 17]byte +var z9161 [1 << 17]byte +var z9162 [1 << 17]byte +var z9163 [1 << 17]byte +var z9164 [1 << 17]byte +var z9165 [1 << 17]byte +var z9166 [1 << 17]byte +var z9167 [1 << 17]byte +var z9168 [1 << 17]byte +var z9169 [1 << 17]byte +var z9170 [1 << 17]byte +var z9171 [1 << 17]byte +var z9172 [1 << 17]byte +var z9173 [1 << 17]byte +var z9174 [1 << 17]byte +var z9175 [1 << 17]byte +var z9176 [1 << 17]byte +var z9177 [1 << 17]byte +var z9178 [1 << 17]byte +var z9179 [1 << 17]byte +var z9180 [1 << 17]byte +var z9181 [1 << 17]byte +var z9182 [1 << 17]byte +var z9183 [1 << 17]byte +var z9184 [1 << 17]byte +var z9185 [1 << 17]byte +var z9186 [1 << 17]byte +var z9187 [1 << 17]byte +var z9188 [1 << 17]byte +var z9189 [1 << 17]byte +var z9190 [1 << 17]byte +var z9191 [1 << 17]byte +var z9192 [1 << 17]byte +var z9193 [1 << 17]byte +var z9194 [1 << 17]byte +var z9195 [1 << 17]byte +var z9196 [1 << 17]byte +var z9197 [1 << 17]byte +var z9198 [1 << 17]byte +var z9199 [1 << 17]byte +var z9200 [1 << 17]byte +var z9201 [1 << 17]byte +var z9202 [1 << 17]byte +var z9203 [1 << 17]byte +var z9204 [1 << 17]byte +var z9205 [1 << 17]byte +var z9206 [1 << 17]byte +var z9207 [1 << 17]byte +var z9208 [1 << 17]byte +var z9209 [1 << 17]byte +var z9210 [1 << 17]byte +var z9211 [1 << 17]byte +var z9212 [1 << 17]byte +var z9213 [1 << 17]byte +var z9214 [1 << 17]byte +var z9215 [1 << 17]byte +var z9216 [1 << 17]byte +var z9217 [1 << 17]byte +var z9218 [1 << 17]byte +var z9219 [1 << 17]byte +var z9220 [1 << 17]byte +var z9221 [1 << 17]byte +var z9222 [1 << 17]byte +var z9223 [1 << 17]byte +var z9224 [1 << 17]byte +var z9225 [1 << 17]byte +var z9226 [1 << 17]byte +var z9227 [1 << 17]byte +var z9228 [1 << 17]byte +var z9229 [1 << 17]byte +var z9230 [1 << 17]byte +var z9231 [1 << 17]byte +var z9232 [1 << 17]byte +var z9233 [1 << 17]byte +var z9234 [1 << 17]byte +var z9235 [1 << 17]byte +var z9236 [1 << 17]byte +var z9237 [1 << 17]byte +var z9238 [1 << 17]byte +var z9239 [1 << 17]byte +var z9240 [1 << 17]byte +var z9241 [1 << 17]byte +var z9242 [1 << 17]byte +var z9243 [1 << 17]byte +var z9244 [1 << 17]byte +var z9245 [1 << 17]byte +var z9246 [1 << 17]byte +var z9247 [1 << 17]byte +var z9248 [1 << 17]byte +var z9249 [1 << 17]byte +var z9250 [1 << 17]byte +var z9251 [1 << 17]byte +var z9252 [1 << 17]byte +var z9253 [1 << 17]byte +var z9254 [1 << 17]byte +var z9255 [1 << 17]byte +var z9256 [1 << 17]byte +var z9257 [1 << 17]byte +var z9258 [1 << 17]byte +var z9259 [1 << 17]byte +var z9260 [1 << 17]byte +var z9261 [1 << 17]byte +var z9262 [1 << 17]byte +var z9263 [1 << 17]byte +var z9264 [1 << 17]byte +var z9265 [1 << 17]byte +var z9266 [1 << 17]byte +var z9267 [1 << 17]byte +var z9268 [1 << 17]byte +var z9269 [1 << 17]byte +var z9270 [1 << 17]byte +var z9271 [1 << 17]byte +var z9272 [1 << 17]byte +var z9273 [1 << 17]byte +var z9274 [1 << 17]byte +var z9275 [1 << 17]byte +var z9276 [1 << 17]byte +var z9277 [1 << 17]byte +var z9278 [1 << 17]byte +var z9279 [1 << 17]byte +var z9280 [1 << 17]byte +var z9281 [1 << 17]byte +var z9282 [1 << 17]byte +var z9283 [1 << 17]byte +var z9284 [1 << 17]byte +var z9285 [1 << 17]byte +var z9286 [1 << 17]byte +var z9287 [1 << 17]byte +var z9288 [1 << 17]byte +var z9289 [1 << 17]byte +var z9290 [1 << 17]byte +var z9291 [1 << 17]byte +var z9292 [1 << 17]byte +var z9293 [1 << 17]byte +var z9294 [1 << 17]byte +var z9295 [1 << 17]byte +var z9296 [1 << 17]byte +var z9297 [1 << 17]byte +var z9298 [1 << 17]byte +var z9299 [1 << 17]byte +var z9300 [1 << 17]byte +var z9301 [1 << 17]byte +var z9302 [1 << 17]byte +var z9303 [1 << 17]byte +var z9304 [1 << 17]byte +var z9305 [1 << 17]byte +var z9306 [1 << 17]byte +var z9307 [1 << 17]byte +var z9308 [1 << 17]byte +var z9309 [1 << 17]byte +var z9310 [1 << 17]byte +var z9311 [1 << 17]byte +var z9312 [1 << 17]byte +var z9313 [1 << 17]byte +var z9314 [1 << 17]byte +var z9315 [1 << 17]byte +var z9316 [1 << 17]byte +var z9317 [1 << 17]byte +var z9318 [1 << 17]byte +var z9319 [1 << 17]byte +var z9320 [1 << 17]byte +var z9321 [1 << 17]byte +var z9322 [1 << 17]byte +var z9323 [1 << 17]byte +var z9324 [1 << 17]byte +var z9325 [1 << 17]byte +var z9326 [1 << 17]byte +var z9327 [1 << 17]byte +var z9328 [1 << 17]byte +var z9329 [1 << 17]byte +var z9330 [1 << 17]byte +var z9331 [1 << 17]byte +var z9332 [1 << 17]byte +var z9333 [1 << 17]byte +var z9334 [1 << 17]byte +var z9335 [1 << 17]byte +var z9336 [1 << 17]byte +var z9337 [1 << 17]byte +var z9338 [1 << 17]byte +var z9339 [1 << 17]byte +var z9340 [1 << 17]byte +var z9341 [1 << 17]byte +var z9342 [1 << 17]byte +var z9343 [1 << 17]byte +var z9344 [1 << 17]byte +var z9345 [1 << 17]byte +var z9346 [1 << 17]byte +var z9347 [1 << 17]byte +var z9348 [1 << 17]byte +var z9349 [1 << 17]byte +var z9350 [1 << 17]byte +var z9351 [1 << 17]byte +var z9352 [1 << 17]byte +var z9353 [1 << 17]byte +var z9354 [1 << 17]byte +var z9355 [1 << 17]byte +var z9356 [1 << 17]byte +var z9357 [1 << 17]byte +var z9358 [1 << 17]byte +var z9359 [1 << 17]byte +var z9360 [1 << 17]byte +var z9361 [1 << 17]byte +var z9362 [1 << 17]byte +var z9363 [1 << 17]byte +var z9364 [1 << 17]byte +var z9365 [1 << 17]byte +var z9366 [1 << 17]byte +var z9367 [1 << 17]byte +var z9368 [1 << 17]byte +var z9369 [1 << 17]byte +var z9370 [1 << 17]byte +var z9371 [1 << 17]byte +var z9372 [1 << 17]byte +var z9373 [1 << 17]byte +var z9374 [1 << 17]byte +var z9375 [1 << 17]byte +var z9376 [1 << 17]byte +var z9377 [1 << 17]byte +var z9378 [1 << 17]byte +var z9379 [1 << 17]byte +var z9380 [1 << 17]byte +var z9381 [1 << 17]byte +var z9382 [1 << 17]byte +var z9383 [1 << 17]byte +var z9384 [1 << 17]byte +var z9385 [1 << 17]byte +var z9386 [1 << 17]byte +var z9387 [1 << 17]byte +var z9388 [1 << 17]byte +var z9389 [1 << 17]byte +var z9390 [1 << 17]byte +var z9391 [1 << 17]byte +var z9392 [1 << 17]byte +var z9393 [1 << 17]byte +var z9394 [1 << 17]byte +var z9395 [1 << 17]byte +var z9396 [1 << 17]byte +var z9397 [1 << 17]byte +var z9398 [1 << 17]byte +var z9399 [1 << 17]byte +var z9400 [1 << 17]byte +var z9401 [1 << 17]byte +var z9402 [1 << 17]byte +var z9403 [1 << 17]byte +var z9404 [1 << 17]byte +var z9405 [1 << 17]byte +var z9406 [1 << 17]byte +var z9407 [1 << 17]byte +var z9408 [1 << 17]byte +var z9409 [1 << 17]byte +var z9410 [1 << 17]byte +var z9411 [1 << 17]byte +var z9412 [1 << 17]byte +var z9413 [1 << 17]byte +var z9414 [1 << 17]byte +var z9415 [1 << 17]byte +var z9416 [1 << 17]byte +var z9417 [1 << 17]byte +var z9418 [1 << 17]byte +var z9419 [1 << 17]byte +var z9420 [1 << 17]byte +var z9421 [1 << 17]byte +var z9422 [1 << 17]byte +var z9423 [1 << 17]byte +var z9424 [1 << 17]byte +var z9425 [1 << 17]byte +var z9426 [1 << 17]byte +var z9427 [1 << 17]byte +var z9428 [1 << 17]byte +var z9429 [1 << 17]byte +var z9430 [1 << 17]byte +var z9431 [1 << 17]byte +var z9432 [1 << 17]byte +var z9433 [1 << 17]byte +var z9434 [1 << 17]byte +var z9435 [1 << 17]byte +var z9436 [1 << 17]byte +var z9437 [1 << 17]byte +var z9438 [1 << 17]byte +var z9439 [1 << 17]byte +var z9440 [1 << 17]byte +var z9441 [1 << 17]byte +var z9442 [1 << 17]byte +var z9443 [1 << 17]byte +var z9444 [1 << 17]byte +var z9445 [1 << 17]byte +var z9446 [1 << 17]byte +var z9447 [1 << 17]byte +var z9448 [1 << 17]byte +var z9449 [1 << 17]byte +var z9450 [1 << 17]byte +var z9451 [1 << 17]byte +var z9452 [1 << 17]byte +var z9453 [1 << 17]byte +var z9454 [1 << 17]byte +var z9455 [1 << 17]byte +var z9456 [1 << 17]byte +var z9457 [1 << 17]byte +var z9458 [1 << 17]byte +var z9459 [1 << 17]byte +var z9460 [1 << 17]byte +var z9461 [1 << 17]byte +var z9462 [1 << 17]byte +var z9463 [1 << 17]byte +var z9464 [1 << 17]byte +var z9465 [1 << 17]byte +var z9466 [1 << 17]byte +var z9467 [1 << 17]byte +var z9468 [1 << 17]byte +var z9469 [1 << 17]byte +var z9470 [1 << 17]byte +var z9471 [1 << 17]byte +var z9472 [1 << 17]byte +var z9473 [1 << 17]byte +var z9474 [1 << 17]byte +var z9475 [1 << 17]byte +var z9476 [1 << 17]byte +var z9477 [1 << 17]byte +var z9478 [1 << 17]byte +var z9479 [1 << 17]byte +var z9480 [1 << 17]byte +var z9481 [1 << 17]byte +var z9482 [1 << 17]byte +var z9483 [1 << 17]byte +var z9484 [1 << 17]byte +var z9485 [1 << 17]byte +var z9486 [1 << 17]byte +var z9487 [1 << 17]byte +var z9488 [1 << 17]byte +var z9489 [1 << 17]byte +var z9490 [1 << 17]byte +var z9491 [1 << 17]byte +var z9492 [1 << 17]byte +var z9493 [1 << 17]byte +var z9494 [1 << 17]byte +var z9495 [1 << 17]byte +var z9496 [1 << 17]byte +var z9497 [1 << 17]byte +var z9498 [1 << 17]byte +var z9499 [1 << 17]byte +var z9500 [1 << 17]byte +var z9501 [1 << 17]byte +var z9502 [1 << 17]byte +var z9503 [1 << 17]byte +var z9504 [1 << 17]byte +var z9505 [1 << 17]byte +var z9506 [1 << 17]byte +var z9507 [1 << 17]byte +var z9508 [1 << 17]byte +var z9509 [1 << 17]byte +var z9510 [1 << 17]byte +var z9511 [1 << 17]byte +var z9512 [1 << 17]byte +var z9513 [1 << 17]byte +var z9514 [1 << 17]byte +var z9515 [1 << 17]byte +var z9516 [1 << 17]byte +var z9517 [1 << 17]byte +var z9518 [1 << 17]byte +var z9519 [1 << 17]byte +var z9520 [1 << 17]byte +var z9521 [1 << 17]byte +var z9522 [1 << 17]byte +var z9523 [1 << 17]byte +var z9524 [1 << 17]byte +var z9525 [1 << 17]byte +var z9526 [1 << 17]byte +var z9527 [1 << 17]byte +var z9528 [1 << 17]byte +var z9529 [1 << 17]byte +var z9530 [1 << 17]byte +var z9531 [1 << 17]byte +var z9532 [1 << 17]byte +var z9533 [1 << 17]byte +var z9534 [1 << 17]byte +var z9535 [1 << 17]byte +var z9536 [1 << 17]byte +var z9537 [1 << 17]byte +var z9538 [1 << 17]byte +var z9539 [1 << 17]byte +var z9540 [1 << 17]byte +var z9541 [1 << 17]byte +var z9542 [1 << 17]byte +var z9543 [1 << 17]byte +var z9544 [1 << 17]byte +var z9545 [1 << 17]byte +var z9546 [1 << 17]byte +var z9547 [1 << 17]byte +var z9548 [1 << 17]byte +var z9549 [1 << 17]byte +var z9550 [1 << 17]byte +var z9551 [1 << 17]byte +var z9552 [1 << 17]byte +var z9553 [1 << 17]byte +var z9554 [1 << 17]byte +var z9555 [1 << 17]byte +var z9556 [1 << 17]byte +var z9557 [1 << 17]byte +var z9558 [1 << 17]byte +var z9559 [1 << 17]byte +var z9560 [1 << 17]byte +var z9561 [1 << 17]byte +var z9562 [1 << 17]byte +var z9563 [1 << 17]byte +var z9564 [1 << 17]byte +var z9565 [1 << 17]byte +var z9566 [1 << 17]byte +var z9567 [1 << 17]byte +var z9568 [1 << 17]byte +var z9569 [1 << 17]byte +var z9570 [1 << 17]byte +var z9571 [1 << 17]byte +var z9572 [1 << 17]byte +var z9573 [1 << 17]byte +var z9574 [1 << 17]byte +var z9575 [1 << 17]byte +var z9576 [1 << 17]byte +var z9577 [1 << 17]byte +var z9578 [1 << 17]byte +var z9579 [1 << 17]byte +var z9580 [1 << 17]byte +var z9581 [1 << 17]byte +var z9582 [1 << 17]byte +var z9583 [1 << 17]byte +var z9584 [1 << 17]byte +var z9585 [1 << 17]byte +var z9586 [1 << 17]byte +var z9587 [1 << 17]byte +var z9588 [1 << 17]byte +var z9589 [1 << 17]byte +var z9590 [1 << 17]byte +var z9591 [1 << 17]byte +var z9592 [1 << 17]byte +var z9593 [1 << 17]byte +var z9594 [1 << 17]byte +var z9595 [1 << 17]byte +var z9596 [1 << 17]byte +var z9597 [1 << 17]byte +var z9598 [1 << 17]byte +var z9599 [1 << 17]byte +var z9600 [1 << 17]byte +var z9601 [1 << 17]byte +var z9602 [1 << 17]byte +var z9603 [1 << 17]byte +var z9604 [1 << 17]byte +var z9605 [1 << 17]byte +var z9606 [1 << 17]byte +var z9607 [1 << 17]byte +var z9608 [1 << 17]byte +var z9609 [1 << 17]byte +var z9610 [1 << 17]byte +var z9611 [1 << 17]byte +var z9612 [1 << 17]byte +var z9613 [1 << 17]byte +var z9614 [1 << 17]byte +var z9615 [1 << 17]byte +var z9616 [1 << 17]byte +var z9617 [1 << 17]byte +var z9618 [1 << 17]byte +var z9619 [1 << 17]byte +var z9620 [1 << 17]byte +var z9621 [1 << 17]byte +var z9622 [1 << 17]byte +var z9623 [1 << 17]byte +var z9624 [1 << 17]byte +var z9625 [1 << 17]byte +var z9626 [1 << 17]byte +var z9627 [1 << 17]byte +var z9628 [1 << 17]byte +var z9629 [1 << 17]byte +var z9630 [1 << 17]byte +var z9631 [1 << 17]byte +var z9632 [1 << 17]byte +var z9633 [1 << 17]byte +var z9634 [1 << 17]byte +var z9635 [1 << 17]byte +var z9636 [1 << 17]byte +var z9637 [1 << 17]byte +var z9638 [1 << 17]byte +var z9639 [1 << 17]byte +var z9640 [1 << 17]byte +var z9641 [1 << 17]byte +var z9642 [1 << 17]byte +var z9643 [1 << 17]byte +var z9644 [1 << 17]byte +var z9645 [1 << 17]byte +var z9646 [1 << 17]byte +var z9647 [1 << 17]byte +var z9648 [1 << 17]byte +var z9649 [1 << 17]byte +var z9650 [1 << 17]byte +var z9651 [1 << 17]byte +var z9652 [1 << 17]byte +var z9653 [1 << 17]byte +var z9654 [1 << 17]byte +var z9655 [1 << 17]byte +var z9656 [1 << 17]byte +var z9657 [1 << 17]byte +var z9658 [1 << 17]byte +var z9659 [1 << 17]byte +var z9660 [1 << 17]byte +var z9661 [1 << 17]byte +var z9662 [1 << 17]byte +var z9663 [1 << 17]byte +var z9664 [1 << 17]byte +var z9665 [1 << 17]byte +var z9666 [1 << 17]byte +var z9667 [1 << 17]byte +var z9668 [1 << 17]byte +var z9669 [1 << 17]byte +var z9670 [1 << 17]byte +var z9671 [1 << 17]byte +var z9672 [1 << 17]byte +var z9673 [1 << 17]byte +var z9674 [1 << 17]byte +var z9675 [1 << 17]byte +var z9676 [1 << 17]byte +var z9677 [1 << 17]byte +var z9678 [1 << 17]byte +var z9679 [1 << 17]byte +var z9680 [1 << 17]byte +var z9681 [1 << 17]byte +var z9682 [1 << 17]byte +var z9683 [1 << 17]byte +var z9684 [1 << 17]byte +var z9685 [1 << 17]byte +var z9686 [1 << 17]byte +var z9687 [1 << 17]byte +var z9688 [1 << 17]byte +var z9689 [1 << 17]byte +var z9690 [1 << 17]byte +var z9691 [1 << 17]byte +var z9692 [1 << 17]byte +var z9693 [1 << 17]byte +var z9694 [1 << 17]byte +var z9695 [1 << 17]byte +var z9696 [1 << 17]byte +var z9697 [1 << 17]byte +var z9698 [1 << 17]byte +var z9699 [1 << 17]byte +var z9700 [1 << 17]byte +var z9701 [1 << 17]byte +var z9702 [1 << 17]byte +var z9703 [1 << 17]byte +var z9704 [1 << 17]byte +var z9705 [1 << 17]byte +var z9706 [1 << 17]byte +var z9707 [1 << 17]byte +var z9708 [1 << 17]byte +var z9709 [1 << 17]byte +var z9710 [1 << 17]byte +var z9711 [1 << 17]byte +var z9712 [1 << 17]byte +var z9713 [1 << 17]byte +var z9714 [1 << 17]byte +var z9715 [1 << 17]byte +var z9716 [1 << 17]byte +var z9717 [1 << 17]byte +var z9718 [1 << 17]byte +var z9719 [1 << 17]byte +var z9720 [1 << 17]byte +var z9721 [1 << 17]byte +var z9722 [1 << 17]byte +var z9723 [1 << 17]byte +var z9724 [1 << 17]byte +var z9725 [1 << 17]byte +var z9726 [1 << 17]byte +var z9727 [1 << 17]byte +var z9728 [1 << 17]byte +var z9729 [1 << 17]byte +var z9730 [1 << 17]byte +var z9731 [1 << 17]byte +var z9732 [1 << 17]byte +var z9733 [1 << 17]byte +var z9734 [1 << 17]byte +var z9735 [1 << 17]byte +var z9736 [1 << 17]byte +var z9737 [1 << 17]byte +var z9738 [1 << 17]byte +var z9739 [1 << 17]byte +var z9740 [1 << 17]byte +var z9741 [1 << 17]byte +var z9742 [1 << 17]byte +var z9743 [1 << 17]byte +var z9744 [1 << 17]byte +var z9745 [1 << 17]byte +var z9746 [1 << 17]byte +var z9747 [1 << 17]byte +var z9748 [1 << 17]byte +var z9749 [1 << 17]byte +var z9750 [1 << 17]byte +var z9751 [1 << 17]byte +var z9752 [1 << 17]byte +var z9753 [1 << 17]byte +var z9754 [1 << 17]byte +var z9755 [1 << 17]byte +var z9756 [1 << 17]byte +var z9757 [1 << 17]byte +var z9758 [1 << 17]byte +var z9759 [1 << 17]byte +var z9760 [1 << 17]byte +var z9761 [1 << 17]byte +var z9762 [1 << 17]byte +var z9763 [1 << 17]byte +var z9764 [1 << 17]byte +var z9765 [1 << 17]byte +var z9766 [1 << 17]byte +var z9767 [1 << 17]byte +var z9768 [1 << 17]byte +var z9769 [1 << 17]byte +var z9770 [1 << 17]byte +var z9771 [1 << 17]byte +var z9772 [1 << 17]byte +var z9773 [1 << 17]byte +var z9774 [1 << 17]byte +var z9775 [1 << 17]byte +var z9776 [1 << 17]byte +var z9777 [1 << 17]byte +var z9778 [1 << 17]byte +var z9779 [1 << 17]byte +var z9780 [1 << 17]byte +var z9781 [1 << 17]byte +var z9782 [1 << 17]byte +var z9783 [1 << 17]byte +var z9784 [1 << 17]byte +var z9785 [1 << 17]byte +var z9786 [1 << 17]byte +var z9787 [1 << 17]byte +var z9788 [1 << 17]byte +var z9789 [1 << 17]byte +var z9790 [1 << 17]byte +var z9791 [1 << 17]byte +var z9792 [1 << 17]byte +var z9793 [1 << 17]byte +var z9794 [1 << 17]byte +var z9795 [1 << 17]byte +var z9796 [1 << 17]byte +var z9797 [1 << 17]byte +var z9798 [1 << 17]byte +var z9799 [1 << 17]byte +var z9800 [1 << 17]byte +var z9801 [1 << 17]byte +var z9802 [1 << 17]byte +var z9803 [1 << 17]byte +var z9804 [1 << 17]byte +var z9805 [1 << 17]byte +var z9806 [1 << 17]byte +var z9807 [1 << 17]byte +var z9808 [1 << 17]byte +var z9809 [1 << 17]byte +var z9810 [1 << 17]byte +var z9811 [1 << 17]byte +var z9812 [1 << 17]byte +var z9813 [1 << 17]byte +var z9814 [1 << 17]byte +var z9815 [1 << 17]byte +var z9816 [1 << 17]byte +var z9817 [1 << 17]byte +var z9818 [1 << 17]byte +var z9819 [1 << 17]byte +var z9820 [1 << 17]byte +var z9821 [1 << 17]byte +var z9822 [1 << 17]byte +var z9823 [1 << 17]byte +var z9824 [1 << 17]byte +var z9825 [1 << 17]byte +var z9826 [1 << 17]byte +var z9827 [1 << 17]byte +var z9828 [1 << 17]byte +var z9829 [1 << 17]byte +var z9830 [1 << 17]byte +var z9831 [1 << 17]byte +var z9832 [1 << 17]byte +var z9833 [1 << 17]byte +var z9834 [1 << 17]byte +var z9835 [1 << 17]byte +var z9836 [1 << 17]byte +var z9837 [1 << 17]byte +var z9838 [1 << 17]byte +var z9839 [1 << 17]byte +var z9840 [1 << 17]byte +var z9841 [1 << 17]byte +var z9842 [1 << 17]byte +var z9843 [1 << 17]byte +var z9844 [1 << 17]byte +var z9845 [1 << 17]byte +var z9846 [1 << 17]byte +var z9847 [1 << 17]byte +var z9848 [1 << 17]byte +var z9849 [1 << 17]byte +var z9850 [1 << 17]byte +var z9851 [1 << 17]byte +var z9852 [1 << 17]byte +var z9853 [1 << 17]byte +var z9854 [1 << 17]byte +var z9855 [1 << 17]byte +var z9856 [1 << 17]byte +var z9857 [1 << 17]byte +var z9858 [1 << 17]byte +var z9859 [1 << 17]byte +var z9860 [1 << 17]byte +var z9861 [1 << 17]byte +var z9862 [1 << 17]byte +var z9863 [1 << 17]byte +var z9864 [1 << 17]byte +var z9865 [1 << 17]byte +var z9866 [1 << 17]byte +var z9867 [1 << 17]byte +var z9868 [1 << 17]byte +var z9869 [1 << 17]byte +var z9870 [1 << 17]byte +var z9871 [1 << 17]byte +var z9872 [1 << 17]byte +var z9873 [1 << 17]byte +var z9874 [1 << 17]byte +var z9875 [1 << 17]byte +var z9876 [1 << 17]byte +var z9877 [1 << 17]byte +var z9878 [1 << 17]byte +var z9879 [1 << 17]byte +var z9880 [1 << 17]byte +var z9881 [1 << 17]byte +var z9882 [1 << 17]byte +var z9883 [1 << 17]byte +var z9884 [1 << 17]byte +var z9885 [1 << 17]byte +var z9886 [1 << 17]byte +var z9887 [1 << 17]byte +var z9888 [1 << 17]byte +var z9889 [1 << 17]byte +var z9890 [1 << 17]byte +var z9891 [1 << 17]byte +var z9892 [1 << 17]byte +var z9893 [1 << 17]byte +var z9894 [1 << 17]byte +var z9895 [1 << 17]byte +var z9896 [1 << 17]byte +var z9897 [1 << 17]byte +var z9898 [1 << 17]byte +var z9899 [1 << 17]byte +var z9900 [1 << 17]byte +var z9901 [1 << 17]byte +var z9902 [1 << 17]byte +var z9903 [1 << 17]byte +var z9904 [1 << 17]byte +var z9905 [1 << 17]byte +var z9906 [1 << 17]byte +var z9907 [1 << 17]byte +var z9908 [1 << 17]byte +var z9909 [1 << 17]byte +var z9910 [1 << 17]byte +var z9911 [1 << 17]byte +var z9912 [1 << 17]byte +var z9913 [1 << 17]byte +var z9914 [1 << 17]byte +var z9915 [1 << 17]byte +var z9916 [1 << 17]byte +var z9917 [1 << 17]byte +var z9918 [1 << 17]byte +var z9919 [1 << 17]byte +var z9920 [1 << 17]byte +var z9921 [1 << 17]byte +var z9922 [1 << 17]byte +var z9923 [1 << 17]byte +var z9924 [1 << 17]byte +var z9925 [1 << 17]byte +var z9926 [1 << 17]byte +var z9927 [1 << 17]byte +var z9928 [1 << 17]byte +var z9929 [1 << 17]byte +var z9930 [1 << 17]byte +var z9931 [1 << 17]byte +var z9932 [1 << 17]byte +var z9933 [1 << 17]byte +var z9934 [1 << 17]byte +var z9935 [1 << 17]byte +var z9936 [1 << 17]byte +var z9937 [1 << 17]byte +var z9938 [1 << 17]byte +var z9939 [1 << 17]byte +var z9940 [1 << 17]byte +var z9941 [1 << 17]byte +var z9942 [1 << 17]byte +var z9943 [1 << 17]byte +var z9944 [1 << 17]byte +var z9945 [1 << 17]byte +var z9946 [1 << 17]byte +var z9947 [1 << 17]byte +var z9948 [1 << 17]byte +var z9949 [1 << 17]byte +var z9950 [1 << 17]byte +var z9951 [1 << 17]byte +var z9952 [1 << 17]byte +var z9953 [1 << 17]byte +var z9954 [1 << 17]byte +var z9955 [1 << 17]byte +var z9956 [1 << 17]byte +var z9957 [1 << 17]byte +var z9958 [1 << 17]byte +var z9959 [1 << 17]byte +var z9960 [1 << 17]byte +var z9961 [1 << 17]byte +var z9962 [1 << 17]byte +var z9963 [1 << 17]byte +var z9964 [1 << 17]byte +var z9965 [1 << 17]byte +var z9966 [1 << 17]byte +var z9967 [1 << 17]byte +var z9968 [1 << 17]byte +var z9969 [1 << 17]byte +var z9970 [1 << 17]byte +var z9971 [1 << 17]byte +var z9972 [1 << 17]byte +var z9973 [1 << 17]byte +var z9974 [1 << 17]byte +var z9975 [1 << 17]byte +var z9976 [1 << 17]byte +var z9977 [1 << 17]byte +var z9978 [1 << 17]byte +var z9979 [1 << 17]byte +var z9980 [1 << 17]byte +var z9981 [1 << 17]byte +var z9982 [1 << 17]byte +var z9983 [1 << 17]byte +var z9984 [1 << 17]byte +var z9985 [1 << 17]byte +var z9986 [1 << 17]byte +var z9987 [1 << 17]byte +var z9988 [1 << 17]byte +var z9989 [1 << 17]byte +var z9990 [1 << 17]byte +var z9991 [1 << 17]byte +var z9992 [1 << 17]byte +var z9993 [1 << 17]byte +var z9994 [1 << 17]byte +var z9995 [1 << 17]byte +var z9996 [1 << 17]byte +var z9997 [1 << 17]byte +var z9998 [1 << 17]byte +var z9999 [1 << 17]byte +var z10000 [1 << 17]byte +var z10001 [1 << 17]byte +var z10002 [1 << 17]byte +var z10003 [1 << 17]byte +var z10004 [1 << 17]byte +var z10005 [1 << 17]byte +var z10006 [1 << 17]byte +var z10007 [1 << 17]byte +var z10008 [1 << 17]byte +var z10009 [1 << 17]byte +var z10010 [1 << 17]byte +var z10011 [1 << 17]byte +var z10012 [1 << 17]byte +var z10013 [1 << 17]byte +var z10014 [1 << 17]byte +var z10015 [1 << 17]byte +var z10016 [1 << 17]byte +var z10017 [1 << 17]byte +var z10018 [1 << 17]byte +var z10019 [1 << 17]byte +var z10020 [1 << 17]byte +var z10021 [1 << 17]byte +var z10022 [1 << 17]byte +var z10023 [1 << 17]byte +var z10024 [1 << 17]byte +var z10025 [1 << 17]byte +var z10026 [1 << 17]byte +var z10027 [1 << 17]byte +var z10028 [1 << 17]byte +var z10029 [1 << 17]byte +var z10030 [1 << 17]byte +var z10031 [1 << 17]byte +var z10032 [1 << 17]byte +var z10033 [1 << 17]byte +var z10034 [1 << 17]byte +var z10035 [1 << 17]byte +var z10036 [1 << 17]byte +var z10037 [1 << 17]byte +var z10038 [1 << 17]byte +var z10039 [1 << 17]byte +var z10040 [1 << 17]byte +var z10041 [1 << 17]byte +var z10042 [1 << 17]byte +var z10043 [1 << 17]byte +var z10044 [1 << 17]byte +var z10045 [1 << 17]byte +var z10046 [1 << 17]byte +var z10047 [1 << 17]byte +var z10048 [1 << 17]byte +var z10049 [1 << 17]byte +var z10050 [1 << 17]byte +var z10051 [1 << 17]byte +var z10052 [1 << 17]byte +var z10053 [1 << 17]byte +var z10054 [1 << 17]byte +var z10055 [1 << 17]byte +var z10056 [1 << 17]byte +var z10057 [1 << 17]byte +var z10058 [1 << 17]byte +var z10059 [1 << 17]byte +var z10060 [1 << 17]byte +var z10061 [1 << 17]byte +var z10062 [1 << 17]byte +var z10063 [1 << 17]byte +var z10064 [1 << 17]byte +var z10065 [1 << 17]byte +var z10066 [1 << 17]byte +var z10067 [1 << 17]byte +var z10068 [1 << 17]byte +var z10069 [1 << 17]byte +var z10070 [1 << 17]byte +var z10071 [1 << 17]byte +var z10072 [1 << 17]byte +var z10073 [1 << 17]byte +var z10074 [1 << 17]byte +var z10075 [1 << 17]byte +var z10076 [1 << 17]byte +var z10077 [1 << 17]byte +var z10078 [1 << 17]byte +var z10079 [1 << 17]byte +var z10080 [1 << 17]byte +var z10081 [1 << 17]byte +var z10082 [1 << 17]byte +var z10083 [1 << 17]byte +var z10084 [1 << 17]byte +var z10085 [1 << 17]byte +var z10086 [1 << 17]byte +var z10087 [1 << 17]byte +var z10088 [1 << 17]byte +var z10089 [1 << 17]byte +var z10090 [1 << 17]byte +var z10091 [1 << 17]byte +var z10092 [1 << 17]byte +var z10093 [1 << 17]byte +var z10094 [1 << 17]byte +var z10095 [1 << 17]byte +var z10096 [1 << 17]byte +var z10097 [1 << 17]byte +var z10098 [1 << 17]byte +var z10099 [1 << 17]byte +var z10100 [1 << 17]byte +var z10101 [1 << 17]byte +var z10102 [1 << 17]byte +var z10103 [1 << 17]byte +var z10104 [1 << 17]byte +var z10105 [1 << 17]byte +var z10106 [1 << 17]byte +var z10107 [1 << 17]byte +var z10108 [1 << 17]byte +var z10109 [1 << 17]byte +var z10110 [1 << 17]byte +var z10111 [1 << 17]byte +var z10112 [1 << 17]byte +var z10113 [1 << 17]byte +var z10114 [1 << 17]byte +var z10115 [1 << 17]byte +var z10116 [1 << 17]byte +var z10117 [1 << 17]byte +var z10118 [1 << 17]byte +var z10119 [1 << 17]byte +var z10120 [1 << 17]byte +var z10121 [1 << 17]byte +var z10122 [1 << 17]byte +var z10123 [1 << 17]byte +var z10124 [1 << 17]byte +var z10125 [1 << 17]byte +var z10126 [1 << 17]byte +var z10127 [1 << 17]byte +var z10128 [1 << 17]byte +var z10129 [1 << 17]byte +var z10130 [1 << 17]byte +var z10131 [1 << 17]byte +var z10132 [1 << 17]byte +var z10133 [1 << 17]byte +var z10134 [1 << 17]byte +var z10135 [1 << 17]byte +var z10136 [1 << 17]byte +var z10137 [1 << 17]byte +var z10138 [1 << 17]byte +var z10139 [1 << 17]byte +var z10140 [1 << 17]byte +var z10141 [1 << 17]byte +var z10142 [1 << 17]byte +var z10143 [1 << 17]byte +var z10144 [1 << 17]byte +var z10145 [1 << 17]byte +var z10146 [1 << 17]byte +var z10147 [1 << 17]byte +var z10148 [1 << 17]byte +var z10149 [1 << 17]byte +var z10150 [1 << 17]byte +var z10151 [1 << 17]byte +var z10152 [1 << 17]byte +var z10153 [1 << 17]byte +var z10154 [1 << 17]byte +var z10155 [1 << 17]byte +var z10156 [1 << 17]byte +var z10157 [1 << 17]byte +var z10158 [1 << 17]byte +var z10159 [1 << 17]byte +var z10160 [1 << 17]byte +var z10161 [1 << 17]byte +var z10162 [1 << 17]byte +var z10163 [1 << 17]byte +var z10164 [1 << 17]byte +var z10165 [1 << 17]byte +var z10166 [1 << 17]byte +var z10167 [1 << 17]byte +var z10168 [1 << 17]byte +var z10169 [1 << 17]byte +var z10170 [1 << 17]byte +var z10171 [1 << 17]byte +var z10172 [1 << 17]byte +var z10173 [1 << 17]byte +var z10174 [1 << 17]byte +var z10175 [1 << 17]byte +var z10176 [1 << 17]byte +var z10177 [1 << 17]byte +var z10178 [1 << 17]byte +var z10179 [1 << 17]byte +var z10180 [1 << 17]byte +var z10181 [1 << 17]byte +var z10182 [1 << 17]byte +var z10183 [1 << 17]byte +var z10184 [1 << 17]byte +var z10185 [1 << 17]byte +var z10186 [1 << 17]byte +var z10187 [1 << 17]byte +var z10188 [1 << 17]byte +var z10189 [1 << 17]byte +var z10190 [1 << 17]byte +var z10191 [1 << 17]byte +var z10192 [1 << 17]byte +var z10193 [1 << 17]byte +var z10194 [1 << 17]byte +var z10195 [1 << 17]byte +var z10196 [1 << 17]byte +var z10197 [1 << 17]byte +var z10198 [1 << 17]byte +var z10199 [1 << 17]byte +var z10200 [1 << 17]byte +var z10201 [1 << 17]byte +var z10202 [1 << 17]byte +var z10203 [1 << 17]byte +var z10204 [1 << 17]byte +var z10205 [1 << 17]byte +var z10206 [1 << 17]byte +var z10207 [1 << 17]byte +var z10208 [1 << 17]byte +var z10209 [1 << 17]byte +var z10210 [1 << 17]byte +var z10211 [1 << 17]byte +var z10212 [1 << 17]byte +var z10213 [1 << 17]byte +var z10214 [1 << 17]byte +var z10215 [1 << 17]byte +var z10216 [1 << 17]byte +var z10217 [1 << 17]byte +var z10218 [1 << 17]byte +var z10219 [1 << 17]byte +var z10220 [1 << 17]byte +var z10221 [1 << 17]byte +var z10222 [1 << 17]byte +var z10223 [1 << 17]byte +var z10224 [1 << 17]byte +var z10225 [1 << 17]byte +var z10226 [1 << 17]byte +var z10227 [1 << 17]byte +var z10228 [1 << 17]byte +var z10229 [1 << 17]byte +var z10230 [1 << 17]byte +var z10231 [1 << 17]byte +var z10232 [1 << 17]byte +var z10233 [1 << 17]byte +var z10234 [1 << 17]byte +var z10235 [1 << 17]byte +var z10236 [1 << 17]byte +var z10237 [1 << 17]byte +var z10238 [1 << 17]byte +var z10239 [1 << 17]byte +var z10240 [1 << 17]byte +var z10241 [1 << 17]byte +var z10242 [1 << 17]byte +var z10243 [1 << 17]byte +var z10244 [1 << 17]byte +var z10245 [1 << 17]byte +var z10246 [1 << 17]byte +var z10247 [1 << 17]byte +var z10248 [1 << 17]byte +var z10249 [1 << 17]byte +var z10250 [1 << 17]byte +var z10251 [1 << 17]byte +var z10252 [1 << 17]byte +var z10253 [1 << 17]byte +var z10254 [1 << 17]byte +var z10255 [1 << 17]byte +var z10256 [1 << 17]byte +var z10257 [1 << 17]byte +var z10258 [1 << 17]byte +var z10259 [1 << 17]byte +var z10260 [1 << 17]byte +var z10261 [1 << 17]byte +var z10262 [1 << 17]byte +var z10263 [1 << 17]byte +var z10264 [1 << 17]byte +var z10265 [1 << 17]byte +var z10266 [1 << 17]byte +var z10267 [1 << 17]byte +var z10268 [1 << 17]byte +var z10269 [1 << 17]byte +var z10270 [1 << 17]byte +var z10271 [1 << 17]byte +var z10272 [1 << 17]byte +var z10273 [1 << 17]byte +var z10274 [1 << 17]byte +var z10275 [1 << 17]byte +var z10276 [1 << 17]byte +var z10277 [1 << 17]byte +var z10278 [1 << 17]byte +var z10279 [1 << 17]byte +var z10280 [1 << 17]byte +var z10281 [1 << 17]byte +var z10282 [1 << 17]byte +var z10283 [1 << 17]byte +var z10284 [1 << 17]byte +var z10285 [1 << 17]byte +var z10286 [1 << 17]byte +var z10287 [1 << 17]byte +var z10288 [1 << 17]byte +var z10289 [1 << 17]byte +var z10290 [1 << 17]byte +var z10291 [1 << 17]byte +var z10292 [1 << 17]byte +var z10293 [1 << 17]byte +var z10294 [1 << 17]byte +var z10295 [1 << 17]byte +var z10296 [1 << 17]byte +var z10297 [1 << 17]byte +var z10298 [1 << 17]byte +var z10299 [1 << 17]byte +var z10300 [1 << 17]byte +var z10301 [1 << 17]byte +var z10302 [1 << 17]byte +var z10303 [1 << 17]byte +var z10304 [1 << 17]byte +var z10305 [1 << 17]byte +var z10306 [1 << 17]byte +var z10307 [1 << 17]byte +var z10308 [1 << 17]byte +var z10309 [1 << 17]byte +var z10310 [1 << 17]byte +var z10311 [1 << 17]byte +var z10312 [1 << 17]byte +var z10313 [1 << 17]byte +var z10314 [1 << 17]byte +var z10315 [1 << 17]byte +var z10316 [1 << 17]byte +var z10317 [1 << 17]byte +var z10318 [1 << 17]byte +var z10319 [1 << 17]byte +var z10320 [1 << 17]byte +var z10321 [1 << 17]byte +var z10322 [1 << 17]byte +var z10323 [1 << 17]byte +var z10324 [1 << 17]byte +var z10325 [1 << 17]byte +var z10326 [1 << 17]byte +var z10327 [1 << 17]byte +var z10328 [1 << 17]byte +var z10329 [1 << 17]byte +var z10330 [1 << 17]byte +var z10331 [1 << 17]byte +var z10332 [1 << 17]byte +var z10333 [1 << 17]byte +var z10334 [1 << 17]byte +var z10335 [1 << 17]byte +var z10336 [1 << 17]byte +var z10337 [1 << 17]byte +var z10338 [1 << 17]byte +var z10339 [1 << 17]byte +var z10340 [1 << 17]byte +var z10341 [1 << 17]byte +var z10342 [1 << 17]byte +var z10343 [1 << 17]byte +var z10344 [1 << 17]byte +var z10345 [1 << 17]byte +var z10346 [1 << 17]byte +var z10347 [1 << 17]byte +var z10348 [1 << 17]byte +var z10349 [1 << 17]byte +var z10350 [1 << 17]byte +var z10351 [1 << 17]byte +var z10352 [1 << 17]byte +var z10353 [1 << 17]byte +var z10354 [1 << 17]byte +var z10355 [1 << 17]byte +var z10356 [1 << 17]byte +var z10357 [1 << 17]byte +var z10358 [1 << 17]byte +var z10359 [1 << 17]byte +var z10360 [1 << 17]byte +var z10361 [1 << 17]byte +var z10362 [1 << 17]byte +var z10363 [1 << 17]byte +var z10364 [1 << 17]byte +var z10365 [1 << 17]byte +var z10366 [1 << 17]byte +var z10367 [1 << 17]byte +var z10368 [1 << 17]byte +var z10369 [1 << 17]byte +var z10370 [1 << 17]byte +var z10371 [1 << 17]byte +var z10372 [1 << 17]byte +var z10373 [1 << 17]byte +var z10374 [1 << 17]byte +var z10375 [1 << 17]byte +var z10376 [1 << 17]byte +var z10377 [1 << 17]byte +var z10378 [1 << 17]byte +var z10379 [1 << 17]byte +var z10380 [1 << 17]byte +var z10381 [1 << 17]byte +var z10382 [1 << 17]byte +var z10383 [1 << 17]byte +var z10384 [1 << 17]byte +var z10385 [1 << 17]byte +var z10386 [1 << 17]byte +var z10387 [1 << 17]byte +var z10388 [1 << 17]byte +var z10389 [1 << 17]byte +var z10390 [1 << 17]byte +var z10391 [1 << 17]byte +var z10392 [1 << 17]byte +var z10393 [1 << 17]byte +var z10394 [1 << 17]byte +var z10395 [1 << 17]byte +var z10396 [1 << 17]byte +var z10397 [1 << 17]byte +var z10398 [1 << 17]byte +var z10399 [1 << 17]byte +var z10400 [1 << 17]byte +var z10401 [1 << 17]byte +var z10402 [1 << 17]byte +var z10403 [1 << 17]byte +var z10404 [1 << 17]byte +var z10405 [1 << 17]byte +var z10406 [1 << 17]byte +var z10407 [1 << 17]byte +var z10408 [1 << 17]byte +var z10409 [1 << 17]byte +var z10410 [1 << 17]byte +var z10411 [1 << 17]byte +var z10412 [1 << 17]byte +var z10413 [1 << 17]byte +var z10414 [1 << 17]byte +var z10415 [1 << 17]byte +var z10416 [1 << 17]byte +var z10417 [1 << 17]byte +var z10418 [1 << 17]byte +var z10419 [1 << 17]byte +var z10420 [1 << 17]byte +var z10421 [1 << 17]byte +var z10422 [1 << 17]byte +var z10423 [1 << 17]byte +var z10424 [1 << 17]byte +var z10425 [1 << 17]byte +var z10426 [1 << 17]byte +var z10427 [1 << 17]byte +var z10428 [1 << 17]byte +var z10429 [1 << 17]byte +var z10430 [1 << 17]byte +var z10431 [1 << 17]byte +var z10432 [1 << 17]byte +var z10433 [1 << 17]byte +var z10434 [1 << 17]byte +var z10435 [1 << 17]byte +var z10436 [1 << 17]byte +var z10437 [1 << 17]byte +var z10438 [1 << 17]byte +var z10439 [1 << 17]byte +var z10440 [1 << 17]byte +var z10441 [1 << 17]byte +var z10442 [1 << 17]byte +var z10443 [1 << 17]byte +var z10444 [1 << 17]byte +var z10445 [1 << 17]byte +var z10446 [1 << 17]byte +var z10447 [1 << 17]byte +var z10448 [1 << 17]byte +var z10449 [1 << 17]byte +var z10450 [1 << 17]byte +var z10451 [1 << 17]byte +var z10452 [1 << 17]byte +var z10453 [1 << 17]byte +var z10454 [1 << 17]byte +var z10455 [1 << 17]byte +var z10456 [1 << 17]byte +var z10457 [1 << 17]byte +var z10458 [1 << 17]byte +var z10459 [1 << 17]byte +var z10460 [1 << 17]byte +var z10461 [1 << 17]byte +var z10462 [1 << 17]byte +var z10463 [1 << 17]byte +var z10464 [1 << 17]byte +var z10465 [1 << 17]byte +var z10466 [1 << 17]byte +var z10467 [1 << 17]byte +var z10468 [1 << 17]byte +var z10469 [1 << 17]byte +var z10470 [1 << 17]byte +var z10471 [1 << 17]byte +var z10472 [1 << 17]byte +var z10473 [1 << 17]byte +var z10474 [1 << 17]byte +var z10475 [1 << 17]byte +var z10476 [1 << 17]byte +var z10477 [1 << 17]byte +var z10478 [1 << 17]byte +var z10479 [1 << 17]byte +var z10480 [1 << 17]byte +var z10481 [1 << 17]byte +var z10482 [1 << 17]byte +var z10483 [1 << 17]byte +var z10484 [1 << 17]byte +var z10485 [1 << 17]byte +var z10486 [1 << 17]byte +var z10487 [1 << 17]byte +var z10488 [1 << 17]byte +var z10489 [1 << 17]byte +var z10490 [1 << 17]byte +var z10491 [1 << 17]byte +var z10492 [1 << 17]byte +var z10493 [1 << 17]byte +var z10494 [1 << 17]byte +var z10495 [1 << 17]byte +var z10496 [1 << 17]byte +var z10497 [1 << 17]byte +var z10498 [1 << 17]byte +var z10499 [1 << 17]byte +var z10500 [1 << 17]byte +var z10501 [1 << 17]byte +var z10502 [1 << 17]byte +var z10503 [1 << 17]byte +var z10504 [1 << 17]byte +var z10505 [1 << 17]byte +var z10506 [1 << 17]byte +var z10507 [1 << 17]byte +var z10508 [1 << 17]byte +var z10509 [1 << 17]byte +var z10510 [1 << 17]byte +var z10511 [1 << 17]byte +var z10512 [1 << 17]byte +var z10513 [1 << 17]byte +var z10514 [1 << 17]byte +var z10515 [1 << 17]byte +var z10516 [1 << 17]byte +var z10517 [1 << 17]byte +var z10518 [1 << 17]byte +var z10519 [1 << 17]byte +var z10520 [1 << 17]byte +var z10521 [1 << 17]byte +var z10522 [1 << 17]byte +var z10523 [1 << 17]byte +var z10524 [1 << 17]byte +var z10525 [1 << 17]byte +var z10526 [1 << 17]byte +var z10527 [1 << 17]byte +var z10528 [1 << 17]byte +var z10529 [1 << 17]byte +var z10530 [1 << 17]byte +var z10531 [1 << 17]byte +var z10532 [1 << 17]byte +var z10533 [1 << 17]byte +var z10534 [1 << 17]byte +var z10535 [1 << 17]byte +var z10536 [1 << 17]byte +var z10537 [1 << 17]byte +var z10538 [1 << 17]byte +var z10539 [1 << 17]byte +var z10540 [1 << 17]byte +var z10541 [1 << 17]byte +var z10542 [1 << 17]byte +var z10543 [1 << 17]byte +var z10544 [1 << 17]byte +var z10545 [1 << 17]byte +var z10546 [1 << 17]byte +var z10547 [1 << 17]byte +var z10548 [1 << 17]byte +var z10549 [1 << 17]byte +var z10550 [1 << 17]byte +var z10551 [1 << 17]byte +var z10552 [1 << 17]byte +var z10553 [1 << 17]byte +var z10554 [1 << 17]byte +var z10555 [1 << 17]byte +var z10556 [1 << 17]byte +var z10557 [1 << 17]byte +var z10558 [1 << 17]byte +var z10559 [1 << 17]byte +var z10560 [1 << 17]byte +var z10561 [1 << 17]byte +var z10562 [1 << 17]byte +var z10563 [1 << 17]byte +var z10564 [1 << 17]byte +var z10565 [1 << 17]byte +var z10566 [1 << 17]byte +var z10567 [1 << 17]byte +var z10568 [1 << 17]byte +var z10569 [1 << 17]byte +var z10570 [1 << 17]byte +var z10571 [1 << 17]byte +var z10572 [1 << 17]byte +var z10573 [1 << 17]byte +var z10574 [1 << 17]byte +var z10575 [1 << 17]byte +var z10576 [1 << 17]byte +var z10577 [1 << 17]byte +var z10578 [1 << 17]byte +var z10579 [1 << 17]byte +var z10580 [1 << 17]byte +var z10581 [1 << 17]byte +var z10582 [1 << 17]byte +var z10583 [1 << 17]byte +var z10584 [1 << 17]byte +var z10585 [1 << 17]byte +var z10586 [1 << 17]byte +var z10587 [1 << 17]byte +var z10588 [1 << 17]byte +var z10589 [1 << 17]byte +var z10590 [1 << 17]byte +var z10591 [1 << 17]byte +var z10592 [1 << 17]byte +var z10593 [1 << 17]byte +var z10594 [1 << 17]byte +var z10595 [1 << 17]byte +var z10596 [1 << 17]byte +var z10597 [1 << 17]byte +var z10598 [1 << 17]byte +var z10599 [1 << 17]byte +var z10600 [1 << 17]byte +var z10601 [1 << 17]byte +var z10602 [1 << 17]byte +var z10603 [1 << 17]byte +var z10604 [1 << 17]byte +var z10605 [1 << 17]byte +var z10606 [1 << 17]byte +var z10607 [1 << 17]byte +var z10608 [1 << 17]byte +var z10609 [1 << 17]byte +var z10610 [1 << 17]byte +var z10611 [1 << 17]byte +var z10612 [1 << 17]byte +var z10613 [1 << 17]byte +var z10614 [1 << 17]byte +var z10615 [1 << 17]byte +var z10616 [1 << 17]byte +var z10617 [1 << 17]byte +var z10618 [1 << 17]byte +var z10619 [1 << 17]byte +var z10620 [1 << 17]byte +var z10621 [1 << 17]byte +var z10622 [1 << 17]byte +var z10623 [1 << 17]byte +var z10624 [1 << 17]byte +var z10625 [1 << 17]byte +var z10626 [1 << 17]byte +var z10627 [1 << 17]byte +var z10628 [1 << 17]byte +var z10629 [1 << 17]byte +var z10630 [1 << 17]byte +var z10631 [1 << 17]byte +var z10632 [1 << 17]byte +var z10633 [1 << 17]byte +var z10634 [1 << 17]byte +var z10635 [1 << 17]byte +var z10636 [1 << 17]byte +var z10637 [1 << 17]byte +var z10638 [1 << 17]byte +var z10639 [1 << 17]byte +var z10640 [1 << 17]byte +var z10641 [1 << 17]byte +var z10642 [1 << 17]byte +var z10643 [1 << 17]byte +var z10644 [1 << 17]byte +var z10645 [1 << 17]byte +var z10646 [1 << 17]byte +var z10647 [1 << 17]byte +var z10648 [1 << 17]byte +var z10649 [1 << 17]byte +var z10650 [1 << 17]byte +var z10651 [1 << 17]byte +var z10652 [1 << 17]byte +var z10653 [1 << 17]byte +var z10654 [1 << 17]byte +var z10655 [1 << 17]byte +var z10656 [1 << 17]byte +var z10657 [1 << 17]byte +var z10658 [1 << 17]byte +var z10659 [1 << 17]byte +var z10660 [1 << 17]byte +var z10661 [1 << 17]byte +var z10662 [1 << 17]byte +var z10663 [1 << 17]byte +var z10664 [1 << 17]byte +var z10665 [1 << 17]byte +var z10666 [1 << 17]byte +var z10667 [1 << 17]byte +var z10668 [1 << 17]byte +var z10669 [1 << 17]byte +var z10670 [1 << 17]byte +var z10671 [1 << 17]byte +var z10672 [1 << 17]byte +var z10673 [1 << 17]byte +var z10674 [1 << 17]byte +var z10675 [1 << 17]byte +var z10676 [1 << 17]byte +var z10677 [1 << 17]byte +var z10678 [1 << 17]byte +var z10679 [1 << 17]byte +var z10680 [1 << 17]byte +var z10681 [1 << 17]byte +var z10682 [1 << 17]byte +var z10683 [1 << 17]byte +var z10684 [1 << 17]byte +var z10685 [1 << 17]byte +var z10686 [1 << 17]byte +var z10687 [1 << 17]byte +var z10688 [1 << 17]byte +var z10689 [1 << 17]byte +var z10690 [1 << 17]byte +var z10691 [1 << 17]byte +var z10692 [1 << 17]byte +var z10693 [1 << 17]byte +var z10694 [1 << 17]byte +var z10695 [1 << 17]byte +var z10696 [1 << 17]byte +var z10697 [1 << 17]byte +var z10698 [1 << 17]byte +var z10699 [1 << 17]byte +var z10700 [1 << 17]byte +var z10701 [1 << 17]byte +var z10702 [1 << 17]byte +var z10703 [1 << 17]byte +var z10704 [1 << 17]byte +var z10705 [1 << 17]byte +var z10706 [1 << 17]byte +var z10707 [1 << 17]byte +var z10708 [1 << 17]byte +var z10709 [1 << 17]byte +var z10710 [1 << 17]byte +var z10711 [1 << 17]byte +var z10712 [1 << 17]byte +var z10713 [1 << 17]byte +var z10714 [1 << 17]byte +var z10715 [1 << 17]byte +var z10716 [1 << 17]byte +var z10717 [1 << 17]byte +var z10718 [1 << 17]byte +var z10719 [1 << 17]byte +var z10720 [1 << 17]byte +var z10721 [1 << 17]byte +var z10722 [1 << 17]byte +var z10723 [1 << 17]byte +var z10724 [1 << 17]byte +var z10725 [1 << 17]byte +var z10726 [1 << 17]byte +var z10727 [1 << 17]byte +var z10728 [1 << 17]byte +var z10729 [1 << 17]byte +var z10730 [1 << 17]byte +var z10731 [1 << 17]byte +var z10732 [1 << 17]byte +var z10733 [1 << 17]byte +var z10734 [1 << 17]byte +var z10735 [1 << 17]byte +var z10736 [1 << 17]byte +var z10737 [1 << 17]byte +var z10738 [1 << 17]byte +var z10739 [1 << 17]byte +var z10740 [1 << 17]byte +var z10741 [1 << 17]byte +var z10742 [1 << 17]byte +var z10743 [1 << 17]byte +var z10744 [1 << 17]byte +var z10745 [1 << 17]byte +var z10746 [1 << 17]byte +var z10747 [1 << 17]byte +var z10748 [1 << 17]byte +var z10749 [1 << 17]byte +var z10750 [1 << 17]byte +var z10751 [1 << 17]byte +var z10752 [1 << 17]byte +var z10753 [1 << 17]byte +var z10754 [1 << 17]byte +var z10755 [1 << 17]byte +var z10756 [1 << 17]byte +var z10757 [1 << 17]byte +var z10758 [1 << 17]byte +var z10759 [1 << 17]byte +var z10760 [1 << 17]byte +var z10761 [1 << 17]byte +var z10762 [1 << 17]byte +var z10763 [1 << 17]byte +var z10764 [1 << 17]byte +var z10765 [1 << 17]byte +var z10766 [1 << 17]byte +var z10767 [1 << 17]byte +var z10768 [1 << 17]byte +var z10769 [1 << 17]byte +var z10770 [1 << 17]byte +var z10771 [1 << 17]byte +var z10772 [1 << 17]byte +var z10773 [1 << 17]byte +var z10774 [1 << 17]byte +var z10775 [1 << 17]byte +var z10776 [1 << 17]byte +var z10777 [1 << 17]byte +var z10778 [1 << 17]byte +var z10779 [1 << 17]byte +var z10780 [1 << 17]byte +var z10781 [1 << 17]byte +var z10782 [1 << 17]byte +var z10783 [1 << 17]byte +var z10784 [1 << 17]byte +var z10785 [1 << 17]byte +var z10786 [1 << 17]byte +var z10787 [1 << 17]byte +var z10788 [1 << 17]byte +var z10789 [1 << 17]byte +var z10790 [1 << 17]byte +var z10791 [1 << 17]byte +var z10792 [1 << 17]byte +var z10793 [1 << 17]byte +var z10794 [1 << 17]byte +var z10795 [1 << 17]byte +var z10796 [1 << 17]byte +var z10797 [1 << 17]byte +var z10798 [1 << 17]byte +var z10799 [1 << 17]byte +var z10800 [1 << 17]byte +var z10801 [1 << 17]byte +var z10802 [1 << 17]byte +var z10803 [1 << 17]byte +var z10804 [1 << 17]byte +var z10805 [1 << 17]byte +var z10806 [1 << 17]byte +var z10807 [1 << 17]byte +var z10808 [1 << 17]byte +var z10809 [1 << 17]byte +var z10810 [1 << 17]byte +var z10811 [1 << 17]byte +var z10812 [1 << 17]byte +var z10813 [1 << 17]byte +var z10814 [1 << 17]byte +var z10815 [1 << 17]byte +var z10816 [1 << 17]byte +var z10817 [1 << 17]byte +var z10818 [1 << 17]byte +var z10819 [1 << 17]byte +var z10820 [1 << 17]byte +var z10821 [1 << 17]byte +var z10822 [1 << 17]byte +var z10823 [1 << 17]byte +var z10824 [1 << 17]byte +var z10825 [1 << 17]byte +var z10826 [1 << 17]byte +var z10827 [1 << 17]byte +var z10828 [1 << 17]byte +var z10829 [1 << 17]byte +var z10830 [1 << 17]byte +var z10831 [1 << 17]byte +var z10832 [1 << 17]byte +var z10833 [1 << 17]byte +var z10834 [1 << 17]byte +var z10835 [1 << 17]byte +var z10836 [1 << 17]byte +var z10837 [1 << 17]byte +var z10838 [1 << 17]byte +var z10839 [1 << 17]byte +var z10840 [1 << 17]byte +var z10841 [1 << 17]byte +var z10842 [1 << 17]byte +var z10843 [1 << 17]byte +var z10844 [1 << 17]byte +var z10845 [1 << 17]byte +var z10846 [1 << 17]byte +var z10847 [1 << 17]byte +var z10848 [1 << 17]byte +var z10849 [1 << 17]byte +var z10850 [1 << 17]byte +var z10851 [1 << 17]byte +var z10852 [1 << 17]byte +var z10853 [1 << 17]byte +var z10854 [1 << 17]byte +var z10855 [1 << 17]byte +var z10856 [1 << 17]byte +var z10857 [1 << 17]byte +var z10858 [1 << 17]byte +var z10859 [1 << 17]byte +var z10860 [1 << 17]byte +var z10861 [1 << 17]byte +var z10862 [1 << 17]byte +var z10863 [1 << 17]byte +var z10864 [1 << 17]byte +var z10865 [1 << 17]byte +var z10866 [1 << 17]byte +var z10867 [1 << 17]byte +var z10868 [1 << 17]byte +var z10869 [1 << 17]byte +var z10870 [1 << 17]byte +var z10871 [1 << 17]byte +var z10872 [1 << 17]byte +var z10873 [1 << 17]byte +var z10874 [1 << 17]byte +var z10875 [1 << 17]byte +var z10876 [1 << 17]byte +var z10877 [1 << 17]byte +var z10878 [1 << 17]byte +var z10879 [1 << 17]byte +var z10880 [1 << 17]byte +var z10881 [1 << 17]byte +var z10882 [1 << 17]byte +var z10883 [1 << 17]byte +var z10884 [1 << 17]byte +var z10885 [1 << 17]byte +var z10886 [1 << 17]byte +var z10887 [1 << 17]byte +var z10888 [1 << 17]byte +var z10889 [1 << 17]byte +var z10890 [1 << 17]byte +var z10891 [1 << 17]byte +var z10892 [1 << 17]byte +var z10893 [1 << 17]byte +var z10894 [1 << 17]byte +var z10895 [1 << 17]byte +var z10896 [1 << 17]byte +var z10897 [1 << 17]byte +var z10898 [1 << 17]byte +var z10899 [1 << 17]byte +var z10900 [1 << 17]byte +var z10901 [1 << 17]byte +var z10902 [1 << 17]byte +var z10903 [1 << 17]byte +var z10904 [1 << 17]byte +var z10905 [1 << 17]byte +var z10906 [1 << 17]byte +var z10907 [1 << 17]byte +var z10908 [1 << 17]byte +var z10909 [1 << 17]byte +var z10910 [1 << 17]byte +var z10911 [1 << 17]byte +var z10912 [1 << 17]byte +var z10913 [1 << 17]byte +var z10914 [1 << 17]byte +var z10915 [1 << 17]byte +var z10916 [1 << 17]byte +var z10917 [1 << 17]byte +var z10918 [1 << 17]byte +var z10919 [1 << 17]byte +var z10920 [1 << 17]byte +var z10921 [1 << 17]byte +var z10922 [1 << 17]byte +var z10923 [1 << 17]byte +var z10924 [1 << 17]byte +var z10925 [1 << 17]byte +var z10926 [1 << 17]byte +var z10927 [1 << 17]byte +var z10928 [1 << 17]byte +var z10929 [1 << 17]byte +var z10930 [1 << 17]byte +var z10931 [1 << 17]byte +var z10932 [1 << 17]byte +var z10933 [1 << 17]byte +var z10934 [1 << 17]byte +var z10935 [1 << 17]byte +var z10936 [1 << 17]byte +var z10937 [1 << 17]byte +var z10938 [1 << 17]byte +var z10939 [1 << 17]byte +var z10940 [1 << 17]byte +var z10941 [1 << 17]byte +var z10942 [1 << 17]byte +var z10943 [1 << 17]byte +var z10944 [1 << 17]byte +var z10945 [1 << 17]byte +var z10946 [1 << 17]byte +var z10947 [1 << 17]byte +var z10948 [1 << 17]byte +var z10949 [1 << 17]byte +var z10950 [1 << 17]byte +var z10951 [1 << 17]byte +var z10952 [1 << 17]byte +var z10953 [1 << 17]byte +var z10954 [1 << 17]byte +var z10955 [1 << 17]byte +var z10956 [1 << 17]byte +var z10957 [1 << 17]byte +var z10958 [1 << 17]byte +var z10959 [1 << 17]byte +var z10960 [1 << 17]byte +var z10961 [1 << 17]byte +var z10962 [1 << 17]byte +var z10963 [1 << 17]byte +var z10964 [1 << 17]byte +var z10965 [1 << 17]byte +var z10966 [1 << 17]byte +var z10967 [1 << 17]byte +var z10968 [1 << 17]byte +var z10969 [1 << 17]byte +var z10970 [1 << 17]byte +var z10971 [1 << 17]byte +var z10972 [1 << 17]byte +var z10973 [1 << 17]byte +var z10974 [1 << 17]byte +var z10975 [1 << 17]byte +var z10976 [1 << 17]byte +var z10977 [1 << 17]byte +var z10978 [1 << 17]byte +var z10979 [1 << 17]byte +var z10980 [1 << 17]byte +var z10981 [1 << 17]byte +var z10982 [1 << 17]byte +var z10983 [1 << 17]byte +var z10984 [1 << 17]byte +var z10985 [1 << 17]byte +var z10986 [1 << 17]byte +var z10987 [1 << 17]byte +var z10988 [1 << 17]byte +var z10989 [1 << 17]byte +var z10990 [1 << 17]byte +var z10991 [1 << 17]byte +var z10992 [1 << 17]byte +var z10993 [1 << 17]byte +var z10994 [1 << 17]byte +var z10995 [1 << 17]byte +var z10996 [1 << 17]byte +var z10997 [1 << 17]byte +var z10998 [1 << 17]byte +var z10999 [1 << 17]byte +var z11000 [1 << 17]byte +var z11001 [1 << 17]byte +var z11002 [1 << 17]byte +var z11003 [1 << 17]byte +var z11004 [1 << 17]byte +var z11005 [1 << 17]byte +var z11006 [1 << 17]byte +var z11007 [1 << 17]byte +var z11008 [1 << 17]byte +var z11009 [1 << 17]byte +var z11010 [1 << 17]byte +var z11011 [1 << 17]byte +var z11012 [1 << 17]byte +var z11013 [1 << 17]byte +var z11014 [1 << 17]byte +var z11015 [1 << 17]byte +var z11016 [1 << 17]byte +var z11017 [1 << 17]byte +var z11018 [1 << 17]byte +var z11019 [1 << 17]byte +var z11020 [1 << 17]byte +var z11021 [1 << 17]byte +var z11022 [1 << 17]byte +var z11023 [1 << 17]byte +var z11024 [1 << 17]byte +var z11025 [1 << 17]byte +var z11026 [1 << 17]byte +var z11027 [1 << 17]byte +var z11028 [1 << 17]byte +var z11029 [1 << 17]byte +var z11030 [1 << 17]byte +var z11031 [1 << 17]byte +var z11032 [1 << 17]byte +var z11033 [1 << 17]byte +var z11034 [1 << 17]byte +var z11035 [1 << 17]byte +var z11036 [1 << 17]byte +var z11037 [1 << 17]byte +var z11038 [1 << 17]byte +var z11039 [1 << 17]byte +var z11040 [1 << 17]byte +var z11041 [1 << 17]byte +var z11042 [1 << 17]byte +var z11043 [1 << 17]byte +var z11044 [1 << 17]byte +var z11045 [1 << 17]byte +var z11046 [1 << 17]byte +var z11047 [1 << 17]byte +var z11048 [1 << 17]byte +var z11049 [1 << 17]byte +var z11050 [1 << 17]byte +var z11051 [1 << 17]byte +var z11052 [1 << 17]byte +var z11053 [1 << 17]byte +var z11054 [1 << 17]byte +var z11055 [1 << 17]byte +var z11056 [1 << 17]byte +var z11057 [1 << 17]byte +var z11058 [1 << 17]byte +var z11059 [1 << 17]byte +var z11060 [1 << 17]byte +var z11061 [1 << 17]byte +var z11062 [1 << 17]byte +var z11063 [1 << 17]byte +var z11064 [1 << 17]byte +var z11065 [1 << 17]byte +var z11066 [1 << 17]byte +var z11067 [1 << 17]byte +var z11068 [1 << 17]byte +var z11069 [1 << 17]byte +var z11070 [1 << 17]byte +var z11071 [1 << 17]byte +var z11072 [1 << 17]byte +var z11073 [1 << 17]byte +var z11074 [1 << 17]byte +var z11075 [1 << 17]byte +var z11076 [1 << 17]byte +var z11077 [1 << 17]byte +var z11078 [1 << 17]byte +var z11079 [1 << 17]byte +var z11080 [1 << 17]byte +var z11081 [1 << 17]byte +var z11082 [1 << 17]byte +var z11083 [1 << 17]byte +var z11084 [1 << 17]byte +var z11085 [1 << 17]byte +var z11086 [1 << 17]byte +var z11087 [1 << 17]byte +var z11088 [1 << 17]byte +var z11089 [1 << 17]byte +var z11090 [1 << 17]byte +var z11091 [1 << 17]byte +var z11092 [1 << 17]byte +var z11093 [1 << 17]byte +var z11094 [1 << 17]byte +var z11095 [1 << 17]byte +var z11096 [1 << 17]byte +var z11097 [1 << 17]byte +var z11098 [1 << 17]byte +var z11099 [1 << 17]byte +var z11100 [1 << 17]byte +var z11101 [1 << 17]byte +var z11102 [1 << 17]byte +var z11103 [1 << 17]byte +var z11104 [1 << 17]byte +var z11105 [1 << 17]byte +var z11106 [1 << 17]byte +var z11107 [1 << 17]byte +var z11108 [1 << 17]byte +var z11109 [1 << 17]byte +var z11110 [1 << 17]byte +var z11111 [1 << 17]byte +var z11112 [1 << 17]byte +var z11113 [1 << 17]byte +var z11114 [1 << 17]byte +var z11115 [1 << 17]byte +var z11116 [1 << 17]byte +var z11117 [1 << 17]byte +var z11118 [1 << 17]byte +var z11119 [1 << 17]byte +var z11120 [1 << 17]byte +var z11121 [1 << 17]byte +var z11122 [1 << 17]byte +var z11123 [1 << 17]byte +var z11124 [1 << 17]byte +var z11125 [1 << 17]byte +var z11126 [1 << 17]byte +var z11127 [1 << 17]byte +var z11128 [1 << 17]byte +var z11129 [1 << 17]byte +var z11130 [1 << 17]byte +var z11131 [1 << 17]byte +var z11132 [1 << 17]byte +var z11133 [1 << 17]byte +var z11134 [1 << 17]byte +var z11135 [1 << 17]byte +var z11136 [1 << 17]byte +var z11137 [1 << 17]byte +var z11138 [1 << 17]byte +var z11139 [1 << 17]byte +var z11140 [1 << 17]byte +var z11141 [1 << 17]byte +var z11142 [1 << 17]byte +var z11143 [1 << 17]byte +var z11144 [1 << 17]byte +var z11145 [1 << 17]byte +var z11146 [1 << 17]byte +var z11147 [1 << 17]byte +var z11148 [1 << 17]byte +var z11149 [1 << 17]byte +var z11150 [1 << 17]byte +var z11151 [1 << 17]byte +var z11152 [1 << 17]byte +var z11153 [1 << 17]byte +var z11154 [1 << 17]byte +var z11155 [1 << 17]byte +var z11156 [1 << 17]byte +var z11157 [1 << 17]byte +var z11158 [1 << 17]byte +var z11159 [1 << 17]byte +var z11160 [1 << 17]byte +var z11161 [1 << 17]byte +var z11162 [1 << 17]byte +var z11163 [1 << 17]byte +var z11164 [1 << 17]byte +var z11165 [1 << 17]byte +var z11166 [1 << 17]byte +var z11167 [1 << 17]byte +var z11168 [1 << 17]byte +var z11169 [1 << 17]byte +var z11170 [1 << 17]byte +var z11171 [1 << 17]byte +var z11172 [1 << 17]byte +var z11173 [1 << 17]byte +var z11174 [1 << 17]byte +var z11175 [1 << 17]byte +var z11176 [1 << 17]byte +var z11177 [1 << 17]byte +var z11178 [1 << 17]byte +var z11179 [1 << 17]byte +var z11180 [1 << 17]byte +var z11181 [1 << 17]byte +var z11182 [1 << 17]byte +var z11183 [1 << 17]byte +var z11184 [1 << 17]byte +var z11185 [1 << 17]byte +var z11186 [1 << 17]byte +var z11187 [1 << 17]byte +var z11188 [1 << 17]byte +var z11189 [1 << 17]byte +var z11190 [1 << 17]byte +var z11191 [1 << 17]byte +var z11192 [1 << 17]byte +var z11193 [1 << 17]byte +var z11194 [1 << 17]byte +var z11195 [1 << 17]byte +var z11196 [1 << 17]byte +var z11197 [1 << 17]byte +var z11198 [1 << 17]byte +var z11199 [1 << 17]byte +var z11200 [1 << 17]byte +var z11201 [1 << 17]byte +var z11202 [1 << 17]byte +var z11203 [1 << 17]byte +var z11204 [1 << 17]byte +var z11205 [1 << 17]byte +var z11206 [1 << 17]byte +var z11207 [1 << 17]byte +var z11208 [1 << 17]byte +var z11209 [1 << 17]byte +var z11210 [1 << 17]byte +var z11211 [1 << 17]byte +var z11212 [1 << 17]byte +var z11213 [1 << 17]byte +var z11214 [1 << 17]byte +var z11215 [1 << 17]byte +var z11216 [1 << 17]byte +var z11217 [1 << 17]byte +var z11218 [1 << 17]byte +var z11219 [1 << 17]byte +var z11220 [1 << 17]byte +var z11221 [1 << 17]byte +var z11222 [1 << 17]byte +var z11223 [1 << 17]byte +var z11224 [1 << 17]byte +var z11225 [1 << 17]byte +var z11226 [1 << 17]byte +var z11227 [1 << 17]byte +var z11228 [1 << 17]byte +var z11229 [1 << 17]byte +var z11230 [1 << 17]byte +var z11231 [1 << 17]byte +var z11232 [1 << 17]byte +var z11233 [1 << 17]byte +var z11234 [1 << 17]byte +var z11235 [1 << 17]byte +var z11236 [1 << 17]byte +var z11237 [1 << 17]byte +var z11238 [1 << 17]byte +var z11239 [1 << 17]byte +var z11240 [1 << 17]byte +var z11241 [1 << 17]byte +var z11242 [1 << 17]byte +var z11243 [1 << 17]byte +var z11244 [1 << 17]byte +var z11245 [1 << 17]byte +var z11246 [1 << 17]byte +var z11247 [1 << 17]byte +var z11248 [1 << 17]byte +var z11249 [1 << 17]byte +var z11250 [1 << 17]byte +var z11251 [1 << 17]byte +var z11252 [1 << 17]byte +var z11253 [1 << 17]byte +var z11254 [1 << 17]byte +var z11255 [1 << 17]byte +var z11256 [1 << 17]byte +var z11257 [1 << 17]byte +var z11258 [1 << 17]byte +var z11259 [1 << 17]byte +var z11260 [1 << 17]byte +var z11261 [1 << 17]byte +var z11262 [1 << 17]byte +var z11263 [1 << 17]byte +var z11264 [1 << 17]byte +var z11265 [1 << 17]byte +var z11266 [1 << 17]byte +var z11267 [1 << 17]byte +var z11268 [1 << 17]byte +var z11269 [1 << 17]byte +var z11270 [1 << 17]byte +var z11271 [1 << 17]byte +var z11272 [1 << 17]byte +var z11273 [1 << 17]byte +var z11274 [1 << 17]byte +var z11275 [1 << 17]byte +var z11276 [1 << 17]byte +var z11277 [1 << 17]byte +var z11278 [1 << 17]byte +var z11279 [1 << 17]byte +var z11280 [1 << 17]byte +var z11281 [1 << 17]byte +var z11282 [1 << 17]byte +var z11283 [1 << 17]byte +var z11284 [1 << 17]byte +var z11285 [1 << 17]byte +var z11286 [1 << 17]byte +var z11287 [1 << 17]byte +var z11288 [1 << 17]byte +var z11289 [1 << 17]byte +var z11290 [1 << 17]byte +var z11291 [1 << 17]byte +var z11292 [1 << 17]byte +var z11293 [1 << 17]byte +var z11294 [1 << 17]byte +var z11295 [1 << 17]byte +var z11296 [1 << 17]byte +var z11297 [1 << 17]byte +var z11298 [1 << 17]byte +var z11299 [1 << 17]byte +var z11300 [1 << 17]byte +var z11301 [1 << 17]byte +var z11302 [1 << 17]byte +var z11303 [1 << 17]byte +var z11304 [1 << 17]byte +var z11305 [1 << 17]byte +var z11306 [1 << 17]byte +var z11307 [1 << 17]byte +var z11308 [1 << 17]byte +var z11309 [1 << 17]byte +var z11310 [1 << 17]byte +var z11311 [1 << 17]byte +var z11312 [1 << 17]byte +var z11313 [1 << 17]byte +var z11314 [1 << 17]byte +var z11315 [1 << 17]byte +var z11316 [1 << 17]byte +var z11317 [1 << 17]byte +var z11318 [1 << 17]byte +var z11319 [1 << 17]byte +var z11320 [1 << 17]byte +var z11321 [1 << 17]byte +var z11322 [1 << 17]byte +var z11323 [1 << 17]byte +var z11324 [1 << 17]byte +var z11325 [1 << 17]byte +var z11326 [1 << 17]byte +var z11327 [1 << 17]byte +var z11328 [1 << 17]byte +var z11329 [1 << 17]byte +var z11330 [1 << 17]byte +var z11331 [1 << 17]byte +var z11332 [1 << 17]byte +var z11333 [1 << 17]byte +var z11334 [1 << 17]byte +var z11335 [1 << 17]byte +var z11336 [1 << 17]byte +var z11337 [1 << 17]byte +var z11338 [1 << 17]byte +var z11339 [1 << 17]byte +var z11340 [1 << 17]byte +var z11341 [1 << 17]byte +var z11342 [1 << 17]byte +var z11343 [1 << 17]byte +var z11344 [1 << 17]byte +var z11345 [1 << 17]byte +var z11346 [1 << 17]byte +var z11347 [1 << 17]byte +var z11348 [1 << 17]byte +var z11349 [1 << 17]byte +var z11350 [1 << 17]byte +var z11351 [1 << 17]byte +var z11352 [1 << 17]byte +var z11353 [1 << 17]byte +var z11354 [1 << 17]byte +var z11355 [1 << 17]byte +var z11356 [1 << 17]byte +var z11357 [1 << 17]byte +var z11358 [1 << 17]byte +var z11359 [1 << 17]byte +var z11360 [1 << 17]byte +var z11361 [1 << 17]byte +var z11362 [1 << 17]byte +var z11363 [1 << 17]byte +var z11364 [1 << 17]byte +var z11365 [1 << 17]byte +var z11366 [1 << 17]byte +var z11367 [1 << 17]byte +var z11368 [1 << 17]byte +var z11369 [1 << 17]byte +var z11370 [1 << 17]byte +var z11371 [1 << 17]byte +var z11372 [1 << 17]byte +var z11373 [1 << 17]byte +var z11374 [1 << 17]byte +var z11375 [1 << 17]byte +var z11376 [1 << 17]byte +var z11377 [1 << 17]byte +var z11378 [1 << 17]byte +var z11379 [1 << 17]byte +var z11380 [1 << 17]byte +var z11381 [1 << 17]byte +var z11382 [1 << 17]byte +var z11383 [1 << 17]byte +var z11384 [1 << 17]byte +var z11385 [1 << 17]byte +var z11386 [1 << 17]byte +var z11387 [1 << 17]byte +var z11388 [1 << 17]byte +var z11389 [1 << 17]byte +var z11390 [1 << 17]byte +var z11391 [1 << 17]byte +var z11392 [1 << 17]byte +var z11393 [1 << 17]byte +var z11394 [1 << 17]byte +var z11395 [1 << 17]byte +var z11396 [1 << 17]byte +var z11397 [1 << 17]byte +var z11398 [1 << 17]byte +var z11399 [1 << 17]byte +var z11400 [1 << 17]byte +var z11401 [1 << 17]byte +var z11402 [1 << 17]byte +var z11403 [1 << 17]byte +var z11404 [1 << 17]byte +var z11405 [1 << 17]byte +var z11406 [1 << 17]byte +var z11407 [1 << 17]byte +var z11408 [1 << 17]byte +var z11409 [1 << 17]byte +var z11410 [1 << 17]byte +var z11411 [1 << 17]byte +var z11412 [1 << 17]byte +var z11413 [1 << 17]byte +var z11414 [1 << 17]byte +var z11415 [1 << 17]byte +var z11416 [1 << 17]byte +var z11417 [1 << 17]byte +var z11418 [1 << 17]byte +var z11419 [1 << 17]byte +var z11420 [1 << 17]byte +var z11421 [1 << 17]byte +var z11422 [1 << 17]byte +var z11423 [1 << 17]byte +var z11424 [1 << 17]byte +var z11425 [1 << 17]byte +var z11426 [1 << 17]byte +var z11427 [1 << 17]byte +var z11428 [1 << 17]byte +var z11429 [1 << 17]byte +var z11430 [1 << 17]byte +var z11431 [1 << 17]byte +var z11432 [1 << 17]byte +var z11433 [1 << 17]byte +var z11434 [1 << 17]byte +var z11435 [1 << 17]byte +var z11436 [1 << 17]byte +var z11437 [1 << 17]byte +var z11438 [1 << 17]byte +var z11439 [1 << 17]byte +var z11440 [1 << 17]byte +var z11441 [1 << 17]byte +var z11442 [1 << 17]byte +var z11443 [1 << 17]byte +var z11444 [1 << 17]byte +var z11445 [1 << 17]byte +var z11446 [1 << 17]byte +var z11447 [1 << 17]byte +var z11448 [1 << 17]byte +var z11449 [1 << 17]byte +var z11450 [1 << 17]byte +var z11451 [1 << 17]byte +var z11452 [1 << 17]byte +var z11453 [1 << 17]byte +var z11454 [1 << 17]byte +var z11455 [1 << 17]byte +var z11456 [1 << 17]byte +var z11457 [1 << 17]byte +var z11458 [1 << 17]byte +var z11459 [1 << 17]byte +var z11460 [1 << 17]byte +var z11461 [1 << 17]byte +var z11462 [1 << 17]byte +var z11463 [1 << 17]byte +var z11464 [1 << 17]byte +var z11465 [1 << 17]byte +var z11466 [1 << 17]byte +var z11467 [1 << 17]byte +var z11468 [1 << 17]byte +var z11469 [1 << 17]byte +var z11470 [1 << 17]byte +var z11471 [1 << 17]byte +var z11472 [1 << 17]byte +var z11473 [1 << 17]byte +var z11474 [1 << 17]byte +var z11475 [1 << 17]byte +var z11476 [1 << 17]byte +var z11477 [1 << 17]byte +var z11478 [1 << 17]byte +var z11479 [1 << 17]byte +var z11480 [1 << 17]byte +var z11481 [1 << 17]byte +var z11482 [1 << 17]byte +var z11483 [1 << 17]byte +var z11484 [1 << 17]byte +var z11485 [1 << 17]byte +var z11486 [1 << 17]byte +var z11487 [1 << 17]byte +var z11488 [1 << 17]byte +var z11489 [1 << 17]byte +var z11490 [1 << 17]byte +var z11491 [1 << 17]byte +var z11492 [1 << 17]byte +var z11493 [1 << 17]byte +var z11494 [1 << 17]byte +var z11495 [1 << 17]byte +var z11496 [1 << 17]byte +var z11497 [1 << 17]byte +var z11498 [1 << 17]byte +var z11499 [1 << 17]byte +var z11500 [1 << 17]byte +var z11501 [1 << 17]byte +var z11502 [1 << 17]byte +var z11503 [1 << 17]byte +var z11504 [1 << 17]byte +var z11505 [1 << 17]byte +var z11506 [1 << 17]byte +var z11507 [1 << 17]byte +var z11508 [1 << 17]byte +var z11509 [1 << 17]byte +var z11510 [1 << 17]byte +var z11511 [1 << 17]byte +var z11512 [1 << 17]byte +var z11513 [1 << 17]byte +var z11514 [1 << 17]byte +var z11515 [1 << 17]byte +var z11516 [1 << 17]byte +var z11517 [1 << 17]byte +var z11518 [1 << 17]byte +var z11519 [1 << 17]byte +var z11520 [1 << 17]byte +var z11521 [1 << 17]byte +var z11522 [1 << 17]byte +var z11523 [1 << 17]byte +var z11524 [1 << 17]byte +var z11525 [1 << 17]byte +var z11526 [1 << 17]byte +var z11527 [1 << 17]byte +var z11528 [1 << 17]byte +var z11529 [1 << 17]byte +var z11530 [1 << 17]byte +var z11531 [1 << 17]byte +var z11532 [1 << 17]byte +var z11533 [1 << 17]byte +var z11534 [1 << 17]byte +var z11535 [1 << 17]byte +var z11536 [1 << 17]byte +var z11537 [1 << 17]byte +var z11538 [1 << 17]byte +var z11539 [1 << 17]byte +var z11540 [1 << 17]byte +var z11541 [1 << 17]byte +var z11542 [1 << 17]byte +var z11543 [1 << 17]byte +var z11544 [1 << 17]byte +var z11545 [1 << 17]byte +var z11546 [1 << 17]byte +var z11547 [1 << 17]byte +var z11548 [1 << 17]byte +var z11549 [1 << 17]byte +var z11550 [1 << 17]byte +var z11551 [1 << 17]byte +var z11552 [1 << 17]byte +var z11553 [1 << 17]byte +var z11554 [1 << 17]byte +var z11555 [1 << 17]byte +var z11556 [1 << 17]byte +var z11557 [1 << 17]byte +var z11558 [1 << 17]byte +var z11559 [1 << 17]byte +var z11560 [1 << 17]byte +var z11561 [1 << 17]byte +var z11562 [1 << 17]byte +var z11563 [1 << 17]byte +var z11564 [1 << 17]byte +var z11565 [1 << 17]byte +var z11566 [1 << 17]byte +var z11567 [1 << 17]byte +var z11568 [1 << 17]byte +var z11569 [1 << 17]byte +var z11570 [1 << 17]byte +var z11571 [1 << 17]byte +var z11572 [1 << 17]byte +var z11573 [1 << 17]byte +var z11574 [1 << 17]byte +var z11575 [1 << 17]byte +var z11576 [1 << 17]byte +var z11577 [1 << 17]byte +var z11578 [1 << 17]byte +var z11579 [1 << 17]byte +var z11580 [1 << 17]byte +var z11581 [1 << 17]byte +var z11582 [1 << 17]byte +var z11583 [1 << 17]byte +var z11584 [1 << 17]byte +var z11585 [1 << 17]byte +var z11586 [1 << 17]byte +var z11587 [1 << 17]byte +var z11588 [1 << 17]byte +var z11589 [1 << 17]byte +var z11590 [1 << 17]byte +var z11591 [1 << 17]byte +var z11592 [1 << 17]byte +var z11593 [1 << 17]byte +var z11594 [1 << 17]byte +var z11595 [1 << 17]byte +var z11596 [1 << 17]byte +var z11597 [1 << 17]byte +var z11598 [1 << 17]byte +var z11599 [1 << 17]byte +var z11600 [1 << 17]byte +var z11601 [1 << 17]byte +var z11602 [1 << 17]byte +var z11603 [1 << 17]byte +var z11604 [1 << 17]byte +var z11605 [1 << 17]byte +var z11606 [1 << 17]byte +var z11607 [1 << 17]byte +var z11608 [1 << 17]byte +var z11609 [1 << 17]byte +var z11610 [1 << 17]byte +var z11611 [1 << 17]byte +var z11612 [1 << 17]byte +var z11613 [1 << 17]byte +var z11614 [1 << 17]byte +var z11615 [1 << 17]byte +var z11616 [1 << 17]byte +var z11617 [1 << 17]byte +var z11618 [1 << 17]byte +var z11619 [1 << 17]byte +var z11620 [1 << 17]byte +var z11621 [1 << 17]byte +var z11622 [1 << 17]byte +var z11623 [1 << 17]byte +var z11624 [1 << 17]byte +var z11625 [1 << 17]byte +var z11626 [1 << 17]byte +var z11627 [1 << 17]byte +var z11628 [1 << 17]byte +var z11629 [1 << 17]byte +var z11630 [1 << 17]byte +var z11631 [1 << 17]byte +var z11632 [1 << 17]byte +var z11633 [1 << 17]byte +var z11634 [1 << 17]byte +var z11635 [1 << 17]byte +var z11636 [1 << 17]byte +var z11637 [1 << 17]byte +var z11638 [1 << 17]byte +var z11639 [1 << 17]byte +var z11640 [1 << 17]byte +var z11641 [1 << 17]byte +var z11642 [1 << 17]byte +var z11643 [1 << 17]byte +var z11644 [1 << 17]byte +var z11645 [1 << 17]byte +var z11646 [1 << 17]byte +var z11647 [1 << 17]byte +var z11648 [1 << 17]byte +var z11649 [1 << 17]byte +var z11650 [1 << 17]byte +var z11651 [1 << 17]byte +var z11652 [1 << 17]byte +var z11653 [1 << 17]byte +var z11654 [1 << 17]byte +var z11655 [1 << 17]byte +var z11656 [1 << 17]byte +var z11657 [1 << 17]byte +var z11658 [1 << 17]byte +var z11659 [1 << 17]byte +var z11660 [1 << 17]byte +var z11661 [1 << 17]byte +var z11662 [1 << 17]byte +var z11663 [1 << 17]byte +var z11664 [1 << 17]byte +var z11665 [1 << 17]byte +var z11666 [1 << 17]byte +var z11667 [1 << 17]byte +var z11668 [1 << 17]byte +var z11669 [1 << 17]byte +var z11670 [1 << 17]byte +var z11671 [1 << 17]byte +var z11672 [1 << 17]byte +var z11673 [1 << 17]byte +var z11674 [1 << 17]byte +var z11675 [1 << 17]byte +var z11676 [1 << 17]byte +var z11677 [1 << 17]byte +var z11678 [1 << 17]byte +var z11679 [1 << 17]byte +var z11680 [1 << 17]byte +var z11681 [1 << 17]byte +var z11682 [1 << 17]byte +var z11683 [1 << 17]byte +var z11684 [1 << 17]byte +var z11685 [1 << 17]byte +var z11686 [1 << 17]byte +var z11687 [1 << 17]byte +var z11688 [1 << 17]byte +var z11689 [1 << 17]byte +var z11690 [1 << 17]byte +var z11691 [1 << 17]byte +var z11692 [1 << 17]byte +var z11693 [1 << 17]byte +var z11694 [1 << 17]byte +var z11695 [1 << 17]byte +var z11696 [1 << 17]byte +var z11697 [1 << 17]byte +var z11698 [1 << 17]byte +var z11699 [1 << 17]byte +var z11700 [1 << 17]byte +var z11701 [1 << 17]byte +var z11702 [1 << 17]byte +var z11703 [1 << 17]byte +var z11704 [1 << 17]byte +var z11705 [1 << 17]byte +var z11706 [1 << 17]byte +var z11707 [1 << 17]byte +var z11708 [1 << 17]byte +var z11709 [1 << 17]byte +var z11710 [1 << 17]byte +var z11711 [1 << 17]byte +var z11712 [1 << 17]byte +var z11713 [1 << 17]byte +var z11714 [1 << 17]byte +var z11715 [1 << 17]byte +var z11716 [1 << 17]byte +var z11717 [1 << 17]byte +var z11718 [1 << 17]byte +var z11719 [1 << 17]byte +var z11720 [1 << 17]byte +var z11721 [1 << 17]byte +var z11722 [1 << 17]byte +var z11723 [1 << 17]byte +var z11724 [1 << 17]byte +var z11725 [1 << 17]byte +var z11726 [1 << 17]byte +var z11727 [1 << 17]byte +var z11728 [1 << 17]byte +var z11729 [1 << 17]byte +var z11730 [1 << 17]byte +var z11731 [1 << 17]byte +var z11732 [1 << 17]byte +var z11733 [1 << 17]byte +var z11734 [1 << 17]byte +var z11735 [1 << 17]byte +var z11736 [1 << 17]byte +var z11737 [1 << 17]byte +var z11738 [1 << 17]byte +var z11739 [1 << 17]byte +var z11740 [1 << 17]byte +var z11741 [1 << 17]byte +var z11742 [1 << 17]byte +var z11743 [1 << 17]byte +var z11744 [1 << 17]byte +var z11745 [1 << 17]byte +var z11746 [1 << 17]byte +var z11747 [1 << 17]byte +var z11748 [1 << 17]byte +var z11749 [1 << 17]byte +var z11750 [1 << 17]byte +var z11751 [1 << 17]byte +var z11752 [1 << 17]byte +var z11753 [1 << 17]byte +var z11754 [1 << 17]byte +var z11755 [1 << 17]byte +var z11756 [1 << 17]byte +var z11757 [1 << 17]byte +var z11758 [1 << 17]byte +var z11759 [1 << 17]byte +var z11760 [1 << 17]byte +var z11761 [1 << 17]byte +var z11762 [1 << 17]byte +var z11763 [1 << 17]byte +var z11764 [1 << 17]byte +var z11765 [1 << 17]byte +var z11766 [1 << 17]byte +var z11767 [1 << 17]byte +var z11768 [1 << 17]byte +var z11769 [1 << 17]byte +var z11770 [1 << 17]byte +var z11771 [1 << 17]byte +var z11772 [1 << 17]byte +var z11773 [1 << 17]byte +var z11774 [1 << 17]byte +var z11775 [1 << 17]byte +var z11776 [1 << 17]byte +var z11777 [1 << 17]byte +var z11778 [1 << 17]byte +var z11779 [1 << 17]byte +var z11780 [1 << 17]byte +var z11781 [1 << 17]byte +var z11782 [1 << 17]byte +var z11783 [1 << 17]byte +var z11784 [1 << 17]byte +var z11785 [1 << 17]byte +var z11786 [1 << 17]byte +var z11787 [1 << 17]byte +var z11788 [1 << 17]byte +var z11789 [1 << 17]byte +var z11790 [1 << 17]byte +var z11791 [1 << 17]byte +var z11792 [1 << 17]byte +var z11793 [1 << 17]byte +var z11794 [1 << 17]byte +var z11795 [1 << 17]byte +var z11796 [1 << 17]byte +var z11797 [1 << 17]byte +var z11798 [1 << 17]byte +var z11799 [1 << 17]byte +var z11800 [1 << 17]byte +var z11801 [1 << 17]byte +var z11802 [1 << 17]byte +var z11803 [1 << 17]byte +var z11804 [1 << 17]byte +var z11805 [1 << 17]byte +var z11806 [1 << 17]byte +var z11807 [1 << 17]byte +var z11808 [1 << 17]byte +var z11809 [1 << 17]byte +var z11810 [1 << 17]byte +var z11811 [1 << 17]byte +var z11812 [1 << 17]byte +var z11813 [1 << 17]byte +var z11814 [1 << 17]byte +var z11815 [1 << 17]byte +var z11816 [1 << 17]byte +var z11817 [1 << 17]byte +var z11818 [1 << 17]byte +var z11819 [1 << 17]byte +var z11820 [1 << 17]byte +var z11821 [1 << 17]byte +var z11822 [1 << 17]byte +var z11823 [1 << 17]byte +var z11824 [1 << 17]byte +var z11825 [1 << 17]byte +var z11826 [1 << 17]byte +var z11827 [1 << 17]byte +var z11828 [1 << 17]byte +var z11829 [1 << 17]byte +var z11830 [1 << 17]byte +var z11831 [1 << 17]byte +var z11832 [1 << 17]byte +var z11833 [1 << 17]byte +var z11834 [1 << 17]byte +var z11835 [1 << 17]byte +var z11836 [1 << 17]byte +var z11837 [1 << 17]byte +var z11838 [1 << 17]byte +var z11839 [1 << 17]byte +var z11840 [1 << 17]byte +var z11841 [1 << 17]byte +var z11842 [1 << 17]byte +var z11843 [1 << 17]byte +var z11844 [1 << 17]byte +var z11845 [1 << 17]byte +var z11846 [1 << 17]byte +var z11847 [1 << 17]byte +var z11848 [1 << 17]byte +var z11849 [1 << 17]byte +var z11850 [1 << 17]byte +var z11851 [1 << 17]byte +var z11852 [1 << 17]byte +var z11853 [1 << 17]byte +var z11854 [1 << 17]byte +var z11855 [1 << 17]byte +var z11856 [1 << 17]byte +var z11857 [1 << 17]byte +var z11858 [1 << 17]byte +var z11859 [1 << 17]byte +var z11860 [1 << 17]byte +var z11861 [1 << 17]byte +var z11862 [1 << 17]byte +var z11863 [1 << 17]byte +var z11864 [1 << 17]byte +var z11865 [1 << 17]byte +var z11866 [1 << 17]byte +var z11867 [1 << 17]byte +var z11868 [1 << 17]byte +var z11869 [1 << 17]byte +var z11870 [1 << 17]byte +var z11871 [1 << 17]byte +var z11872 [1 << 17]byte +var z11873 [1 << 17]byte +var z11874 [1 << 17]byte +var z11875 [1 << 17]byte +var z11876 [1 << 17]byte +var z11877 [1 << 17]byte +var z11878 [1 << 17]byte +var z11879 [1 << 17]byte +var z11880 [1 << 17]byte +var z11881 [1 << 17]byte +var z11882 [1 << 17]byte +var z11883 [1 << 17]byte +var z11884 [1 << 17]byte +var z11885 [1 << 17]byte +var z11886 [1 << 17]byte +var z11887 [1 << 17]byte +var z11888 [1 << 17]byte +var z11889 [1 << 17]byte +var z11890 [1 << 17]byte +var z11891 [1 << 17]byte +var z11892 [1 << 17]byte +var z11893 [1 << 17]byte +var z11894 [1 << 17]byte +var z11895 [1 << 17]byte +var z11896 [1 << 17]byte +var z11897 [1 << 17]byte +var z11898 [1 << 17]byte +var z11899 [1 << 17]byte +var z11900 [1 << 17]byte +var z11901 [1 << 17]byte +var z11902 [1 << 17]byte +var z11903 [1 << 17]byte +var z11904 [1 << 17]byte +var z11905 [1 << 17]byte +var z11906 [1 << 17]byte +var z11907 [1 << 17]byte +var z11908 [1 << 17]byte +var z11909 [1 << 17]byte +var z11910 [1 << 17]byte +var z11911 [1 << 17]byte +var z11912 [1 << 17]byte +var z11913 [1 << 17]byte +var z11914 [1 << 17]byte +var z11915 [1 << 17]byte +var z11916 [1 << 17]byte +var z11917 [1 << 17]byte +var z11918 [1 << 17]byte +var z11919 [1 << 17]byte +var z11920 [1 << 17]byte +var z11921 [1 << 17]byte +var z11922 [1 << 17]byte +var z11923 [1 << 17]byte +var z11924 [1 << 17]byte +var z11925 [1 << 17]byte +var z11926 [1 << 17]byte +var z11927 [1 << 17]byte +var z11928 [1 << 17]byte +var z11929 [1 << 17]byte +var z11930 [1 << 17]byte +var z11931 [1 << 17]byte +var z11932 [1 << 17]byte +var z11933 [1 << 17]byte +var z11934 [1 << 17]byte +var z11935 [1 << 17]byte +var z11936 [1 << 17]byte +var z11937 [1 << 17]byte +var z11938 [1 << 17]byte +var z11939 [1 << 17]byte +var z11940 [1 << 17]byte +var z11941 [1 << 17]byte +var z11942 [1 << 17]byte +var z11943 [1 << 17]byte +var z11944 [1 << 17]byte +var z11945 [1 << 17]byte +var z11946 [1 << 17]byte +var z11947 [1 << 17]byte +var z11948 [1 << 17]byte +var z11949 [1 << 17]byte +var z11950 [1 << 17]byte +var z11951 [1 << 17]byte +var z11952 [1 << 17]byte +var z11953 [1 << 17]byte +var z11954 [1 << 17]byte +var z11955 [1 << 17]byte +var z11956 [1 << 17]byte +var z11957 [1 << 17]byte +var z11958 [1 << 17]byte +var z11959 [1 << 17]byte +var z11960 [1 << 17]byte +var z11961 [1 << 17]byte +var z11962 [1 << 17]byte +var z11963 [1 << 17]byte +var z11964 [1 << 17]byte +var z11965 [1 << 17]byte +var z11966 [1 << 17]byte +var z11967 [1 << 17]byte +var z11968 [1 << 17]byte +var z11969 [1 << 17]byte +var z11970 [1 << 17]byte +var z11971 [1 << 17]byte +var z11972 [1 << 17]byte +var z11973 [1 << 17]byte +var z11974 [1 << 17]byte +var z11975 [1 << 17]byte +var z11976 [1 << 17]byte +var z11977 [1 << 17]byte +var z11978 [1 << 17]byte +var z11979 [1 << 17]byte +var z11980 [1 << 17]byte +var z11981 [1 << 17]byte +var z11982 [1 << 17]byte +var z11983 [1 << 17]byte +var z11984 [1 << 17]byte +var z11985 [1 << 17]byte +var z11986 [1 << 17]byte +var z11987 [1 << 17]byte +var z11988 [1 << 17]byte +var z11989 [1 << 17]byte +var z11990 [1 << 17]byte +var z11991 [1 << 17]byte +var z11992 [1 << 17]byte +var z11993 [1 << 17]byte +var z11994 [1 << 17]byte +var z11995 [1 << 17]byte +var z11996 [1 << 17]byte +var z11997 [1 << 17]byte +var z11998 [1 << 17]byte +var z11999 [1 << 17]byte +var z12000 [1 << 17]byte +var z12001 [1 << 17]byte +var z12002 [1 << 17]byte +var z12003 [1 << 17]byte +var z12004 [1 << 17]byte +var z12005 [1 << 17]byte +var z12006 [1 << 17]byte +var z12007 [1 << 17]byte +var z12008 [1 << 17]byte +var z12009 [1 << 17]byte +var z12010 [1 << 17]byte +var z12011 [1 << 17]byte +var z12012 [1 << 17]byte +var z12013 [1 << 17]byte +var z12014 [1 << 17]byte +var z12015 [1 << 17]byte +var z12016 [1 << 17]byte +var z12017 [1 << 17]byte +var z12018 [1 << 17]byte +var z12019 [1 << 17]byte +var z12020 [1 << 17]byte +var z12021 [1 << 17]byte +var z12022 [1 << 17]byte +var z12023 [1 << 17]byte +var z12024 [1 << 17]byte +var z12025 [1 << 17]byte +var z12026 [1 << 17]byte +var z12027 [1 << 17]byte +var z12028 [1 << 17]byte +var z12029 [1 << 17]byte +var z12030 [1 << 17]byte +var z12031 [1 << 17]byte +var z12032 [1 << 17]byte +var z12033 [1 << 17]byte +var z12034 [1 << 17]byte +var z12035 [1 << 17]byte +var z12036 [1 << 17]byte +var z12037 [1 << 17]byte +var z12038 [1 << 17]byte +var z12039 [1 << 17]byte +var z12040 [1 << 17]byte +var z12041 [1 << 17]byte +var z12042 [1 << 17]byte +var z12043 [1 << 17]byte +var z12044 [1 << 17]byte +var z12045 [1 << 17]byte +var z12046 [1 << 17]byte +var z12047 [1 << 17]byte +var z12048 [1 << 17]byte +var z12049 [1 << 17]byte +var z12050 [1 << 17]byte +var z12051 [1 << 17]byte +var z12052 [1 << 17]byte +var z12053 [1 << 17]byte +var z12054 [1 << 17]byte +var z12055 [1 << 17]byte +var z12056 [1 << 17]byte +var z12057 [1 << 17]byte +var z12058 [1 << 17]byte +var z12059 [1 << 17]byte +var z12060 [1 << 17]byte +var z12061 [1 << 17]byte +var z12062 [1 << 17]byte +var z12063 [1 << 17]byte +var z12064 [1 << 17]byte +var z12065 [1 << 17]byte +var z12066 [1 << 17]byte +var z12067 [1 << 17]byte +var z12068 [1 << 17]byte +var z12069 [1 << 17]byte +var z12070 [1 << 17]byte +var z12071 [1 << 17]byte +var z12072 [1 << 17]byte +var z12073 [1 << 17]byte +var z12074 [1 << 17]byte +var z12075 [1 << 17]byte +var z12076 [1 << 17]byte +var z12077 [1 << 17]byte +var z12078 [1 << 17]byte +var z12079 [1 << 17]byte +var z12080 [1 << 17]byte +var z12081 [1 << 17]byte +var z12082 [1 << 17]byte +var z12083 [1 << 17]byte +var z12084 [1 << 17]byte +var z12085 [1 << 17]byte +var z12086 [1 << 17]byte +var z12087 [1 << 17]byte +var z12088 [1 << 17]byte +var z12089 [1 << 17]byte +var z12090 [1 << 17]byte +var z12091 [1 << 17]byte +var z12092 [1 << 17]byte +var z12093 [1 << 17]byte +var z12094 [1 << 17]byte +var z12095 [1 << 17]byte +var z12096 [1 << 17]byte +var z12097 [1 << 17]byte +var z12098 [1 << 17]byte +var z12099 [1 << 17]byte +var z12100 [1 << 17]byte +var z12101 [1 << 17]byte +var z12102 [1 << 17]byte +var z12103 [1 << 17]byte +var z12104 [1 << 17]byte +var z12105 [1 << 17]byte +var z12106 [1 << 17]byte +var z12107 [1 << 17]byte +var z12108 [1 << 17]byte +var z12109 [1 << 17]byte +var z12110 [1 << 17]byte +var z12111 [1 << 17]byte +var z12112 [1 << 17]byte +var z12113 [1 << 17]byte +var z12114 [1 << 17]byte +var z12115 [1 << 17]byte +var z12116 [1 << 17]byte +var z12117 [1 << 17]byte +var z12118 [1 << 17]byte +var z12119 [1 << 17]byte +var z12120 [1 << 17]byte +var z12121 [1 << 17]byte +var z12122 [1 << 17]byte +var z12123 [1 << 17]byte +var z12124 [1 << 17]byte +var z12125 [1 << 17]byte +var z12126 [1 << 17]byte +var z12127 [1 << 17]byte +var z12128 [1 << 17]byte +var z12129 [1 << 17]byte +var z12130 [1 << 17]byte +var z12131 [1 << 17]byte +var z12132 [1 << 17]byte +var z12133 [1 << 17]byte +var z12134 [1 << 17]byte +var z12135 [1 << 17]byte +var z12136 [1 << 17]byte +var z12137 [1 << 17]byte +var z12138 [1 << 17]byte +var z12139 [1 << 17]byte +var z12140 [1 << 17]byte +var z12141 [1 << 17]byte +var z12142 [1 << 17]byte +var z12143 [1 << 17]byte +var z12144 [1 << 17]byte +var z12145 [1 << 17]byte +var z12146 [1 << 17]byte +var z12147 [1 << 17]byte +var z12148 [1 << 17]byte +var z12149 [1 << 17]byte +var z12150 [1 << 17]byte +var z12151 [1 << 17]byte +var z12152 [1 << 17]byte +var z12153 [1 << 17]byte +var z12154 [1 << 17]byte +var z12155 [1 << 17]byte +var z12156 [1 << 17]byte +var z12157 [1 << 17]byte +var z12158 [1 << 17]byte +var z12159 [1 << 17]byte +var z12160 [1 << 17]byte +var z12161 [1 << 17]byte +var z12162 [1 << 17]byte +var z12163 [1 << 17]byte +var z12164 [1 << 17]byte +var z12165 [1 << 17]byte +var z12166 [1 << 17]byte +var z12167 [1 << 17]byte +var z12168 [1 << 17]byte +var z12169 [1 << 17]byte +var z12170 [1 << 17]byte +var z12171 [1 << 17]byte +var z12172 [1 << 17]byte +var z12173 [1 << 17]byte +var z12174 [1 << 17]byte +var z12175 [1 << 17]byte +var z12176 [1 << 17]byte +var z12177 [1 << 17]byte +var z12178 [1 << 17]byte +var z12179 [1 << 17]byte +var z12180 [1 << 17]byte +var z12181 [1 << 17]byte +var z12182 [1 << 17]byte +var z12183 [1 << 17]byte +var z12184 [1 << 17]byte +var z12185 [1 << 17]byte +var z12186 [1 << 17]byte +var z12187 [1 << 17]byte +var z12188 [1 << 17]byte +var z12189 [1 << 17]byte +var z12190 [1 << 17]byte +var z12191 [1 << 17]byte +var z12192 [1 << 17]byte +var z12193 [1 << 17]byte +var z12194 [1 << 17]byte +var z12195 [1 << 17]byte +var z12196 [1 << 17]byte +var z12197 [1 << 17]byte +var z12198 [1 << 17]byte +var z12199 [1 << 17]byte +var z12200 [1 << 17]byte +var z12201 [1 << 17]byte +var z12202 [1 << 17]byte +var z12203 [1 << 17]byte +var z12204 [1 << 17]byte +var z12205 [1 << 17]byte +var z12206 [1 << 17]byte +var z12207 [1 << 17]byte +var z12208 [1 << 17]byte +var z12209 [1 << 17]byte +var z12210 [1 << 17]byte +var z12211 [1 << 17]byte +var z12212 [1 << 17]byte +var z12213 [1 << 17]byte +var z12214 [1 << 17]byte +var z12215 [1 << 17]byte +var z12216 [1 << 17]byte +var z12217 [1 << 17]byte +var z12218 [1 << 17]byte +var z12219 [1 << 17]byte +var z12220 [1 << 17]byte +var z12221 [1 << 17]byte +var z12222 [1 << 17]byte +var z12223 [1 << 17]byte +var z12224 [1 << 17]byte +var z12225 [1 << 17]byte +var z12226 [1 << 17]byte +var z12227 [1 << 17]byte +var z12228 [1 << 17]byte +var z12229 [1 << 17]byte +var z12230 [1 << 17]byte +var z12231 [1 << 17]byte +var z12232 [1 << 17]byte +var z12233 [1 << 17]byte +var z12234 [1 << 17]byte +var z12235 [1 << 17]byte +var z12236 [1 << 17]byte +var z12237 [1 << 17]byte +var z12238 [1 << 17]byte +var z12239 [1 << 17]byte +var z12240 [1 << 17]byte +var z12241 [1 << 17]byte +var z12242 [1 << 17]byte +var z12243 [1 << 17]byte +var z12244 [1 << 17]byte +var z12245 [1 << 17]byte +var z12246 [1 << 17]byte +var z12247 [1 << 17]byte +var z12248 [1 << 17]byte +var z12249 [1 << 17]byte +var z12250 [1 << 17]byte +var z12251 [1 << 17]byte +var z12252 [1 << 17]byte +var z12253 [1 << 17]byte +var z12254 [1 << 17]byte +var z12255 [1 << 17]byte +var z12256 [1 << 17]byte +var z12257 [1 << 17]byte +var z12258 [1 << 17]byte +var z12259 [1 << 17]byte +var z12260 [1 << 17]byte +var z12261 [1 << 17]byte +var z12262 [1 << 17]byte +var z12263 [1 << 17]byte +var z12264 [1 << 17]byte +var z12265 [1 << 17]byte +var z12266 [1 << 17]byte +var z12267 [1 << 17]byte +var z12268 [1 << 17]byte +var z12269 [1 << 17]byte +var z12270 [1 << 17]byte +var z12271 [1 << 17]byte +var z12272 [1 << 17]byte +var z12273 [1 << 17]byte +var z12274 [1 << 17]byte +var z12275 [1 << 17]byte +var z12276 [1 << 17]byte +var z12277 [1 << 17]byte +var z12278 [1 << 17]byte +var z12279 [1 << 17]byte +var z12280 [1 << 17]byte +var z12281 [1 << 17]byte +var z12282 [1 << 17]byte +var z12283 [1 << 17]byte +var z12284 [1 << 17]byte +var z12285 [1 << 17]byte +var z12286 [1 << 17]byte +var z12287 [1 << 17]byte +var z12288 [1 << 17]byte +var z12289 [1 << 17]byte +var z12290 [1 << 17]byte +var z12291 [1 << 17]byte +var z12292 [1 << 17]byte +var z12293 [1 << 17]byte +var z12294 [1 << 17]byte +var z12295 [1 << 17]byte +var z12296 [1 << 17]byte +var z12297 [1 << 17]byte +var z12298 [1 << 17]byte +var z12299 [1 << 17]byte +var z12300 [1 << 17]byte +var z12301 [1 << 17]byte +var z12302 [1 << 17]byte +var z12303 [1 << 17]byte +var z12304 [1 << 17]byte +var z12305 [1 << 17]byte +var z12306 [1 << 17]byte +var z12307 [1 << 17]byte +var z12308 [1 << 17]byte +var z12309 [1 << 17]byte +var z12310 [1 << 17]byte +var z12311 [1 << 17]byte +var z12312 [1 << 17]byte +var z12313 [1 << 17]byte +var z12314 [1 << 17]byte +var z12315 [1 << 17]byte +var z12316 [1 << 17]byte +var z12317 [1 << 17]byte +var z12318 [1 << 17]byte +var z12319 [1 << 17]byte +var z12320 [1 << 17]byte +var z12321 [1 << 17]byte +var z12322 [1 << 17]byte +var z12323 [1 << 17]byte +var z12324 [1 << 17]byte +var z12325 [1 << 17]byte +var z12326 [1 << 17]byte +var z12327 [1 << 17]byte +var z12328 [1 << 17]byte +var z12329 [1 << 17]byte +var z12330 [1 << 17]byte +var z12331 [1 << 17]byte +var z12332 [1 << 17]byte +var z12333 [1 << 17]byte +var z12334 [1 << 17]byte +var z12335 [1 << 17]byte +var z12336 [1 << 17]byte +var z12337 [1 << 17]byte +var z12338 [1 << 17]byte +var z12339 [1 << 17]byte +var z12340 [1 << 17]byte +var z12341 [1 << 17]byte +var z12342 [1 << 17]byte +var z12343 [1 << 17]byte +var z12344 [1 << 17]byte +var z12345 [1 << 17]byte +var z12346 [1 << 17]byte +var z12347 [1 << 17]byte +var z12348 [1 << 17]byte +var z12349 [1 << 17]byte +var z12350 [1 << 17]byte +var z12351 [1 << 17]byte +var z12352 [1 << 17]byte +var z12353 [1 << 17]byte +var z12354 [1 << 17]byte +var z12355 [1 << 17]byte +var z12356 [1 << 17]byte +var z12357 [1 << 17]byte +var z12358 [1 << 17]byte +var z12359 [1 << 17]byte +var z12360 [1 << 17]byte +var z12361 [1 << 17]byte +var z12362 [1 << 17]byte +var z12363 [1 << 17]byte +var z12364 [1 << 17]byte +var z12365 [1 << 17]byte +var z12366 [1 << 17]byte +var z12367 [1 << 17]byte +var z12368 [1 << 17]byte +var z12369 [1 << 17]byte +var z12370 [1 << 17]byte +var z12371 [1 << 17]byte +var z12372 [1 << 17]byte +var z12373 [1 << 17]byte +var z12374 [1 << 17]byte +var z12375 [1 << 17]byte +var z12376 [1 << 17]byte +var z12377 [1 << 17]byte +var z12378 [1 << 17]byte +var z12379 [1 << 17]byte +var z12380 [1 << 17]byte +var z12381 [1 << 17]byte +var z12382 [1 << 17]byte +var z12383 [1 << 17]byte +var z12384 [1 << 17]byte +var z12385 [1 << 17]byte +var z12386 [1 << 17]byte +var z12387 [1 << 17]byte +var z12388 [1 << 17]byte +var z12389 [1 << 17]byte +var z12390 [1 << 17]byte +var z12391 [1 << 17]byte +var z12392 [1 << 17]byte +var z12393 [1 << 17]byte +var z12394 [1 << 17]byte +var z12395 [1 << 17]byte +var z12396 [1 << 17]byte +var z12397 [1 << 17]byte +var z12398 [1 << 17]byte +var z12399 [1 << 17]byte +var z12400 [1 << 17]byte +var z12401 [1 << 17]byte +var z12402 [1 << 17]byte +var z12403 [1 << 17]byte +var z12404 [1 << 17]byte +var z12405 [1 << 17]byte +var z12406 [1 << 17]byte +var z12407 [1 << 17]byte +var z12408 [1 << 17]byte +var z12409 [1 << 17]byte +var z12410 [1 << 17]byte +var z12411 [1 << 17]byte +var z12412 [1 << 17]byte +var z12413 [1 << 17]byte +var z12414 [1 << 17]byte +var z12415 [1 << 17]byte +var z12416 [1 << 17]byte +var z12417 [1 << 17]byte +var z12418 [1 << 17]byte +var z12419 [1 << 17]byte +var z12420 [1 << 17]byte +var z12421 [1 << 17]byte +var z12422 [1 << 17]byte +var z12423 [1 << 17]byte +var z12424 [1 << 17]byte +var z12425 [1 << 17]byte +var z12426 [1 << 17]byte +var z12427 [1 << 17]byte +var z12428 [1 << 17]byte +var z12429 [1 << 17]byte +var z12430 [1 << 17]byte +var z12431 [1 << 17]byte +var z12432 [1 << 17]byte +var z12433 [1 << 17]byte +var z12434 [1 << 17]byte +var z12435 [1 << 17]byte +var z12436 [1 << 17]byte +var z12437 [1 << 17]byte +var z12438 [1 << 17]byte +var z12439 [1 << 17]byte +var z12440 [1 << 17]byte +var z12441 [1 << 17]byte +var z12442 [1 << 17]byte +var z12443 [1 << 17]byte +var z12444 [1 << 17]byte +var z12445 [1 << 17]byte +var z12446 [1 << 17]byte +var z12447 [1 << 17]byte +var z12448 [1 << 17]byte +var z12449 [1 << 17]byte +var z12450 [1 << 17]byte +var z12451 [1 << 17]byte +var z12452 [1 << 17]byte +var z12453 [1 << 17]byte +var z12454 [1 << 17]byte +var z12455 [1 << 17]byte +var z12456 [1 << 17]byte +var z12457 [1 << 17]byte +var z12458 [1 << 17]byte +var z12459 [1 << 17]byte +var z12460 [1 << 17]byte +var z12461 [1 << 17]byte +var z12462 [1 << 17]byte +var z12463 [1 << 17]byte +var z12464 [1 << 17]byte +var z12465 [1 << 17]byte +var z12466 [1 << 17]byte +var z12467 [1 << 17]byte +var z12468 [1 << 17]byte +var z12469 [1 << 17]byte +var z12470 [1 << 17]byte +var z12471 [1 << 17]byte +var z12472 [1 << 17]byte +var z12473 [1 << 17]byte +var z12474 [1 << 17]byte +var z12475 [1 << 17]byte +var z12476 [1 << 17]byte +var z12477 [1 << 17]byte +var z12478 [1 << 17]byte +var z12479 [1 << 17]byte +var z12480 [1 << 17]byte +var z12481 [1 << 17]byte +var z12482 [1 << 17]byte +var z12483 [1 << 17]byte +var z12484 [1 << 17]byte +var z12485 [1 << 17]byte +var z12486 [1 << 17]byte +var z12487 [1 << 17]byte +var z12488 [1 << 17]byte +var z12489 [1 << 17]byte +var z12490 [1 << 17]byte +var z12491 [1 << 17]byte +var z12492 [1 << 17]byte +var z12493 [1 << 17]byte +var z12494 [1 << 17]byte +var z12495 [1 << 17]byte +var z12496 [1 << 17]byte +var z12497 [1 << 17]byte +var z12498 [1 << 17]byte +var z12499 [1 << 17]byte +var z12500 [1 << 17]byte +var z12501 [1 << 17]byte +var z12502 [1 << 17]byte +var z12503 [1 << 17]byte +var z12504 [1 << 17]byte +var z12505 [1 << 17]byte +var z12506 [1 << 17]byte +var z12507 [1 << 17]byte +var z12508 [1 << 17]byte +var z12509 [1 << 17]byte +var z12510 [1 << 17]byte +var z12511 [1 << 17]byte +var z12512 [1 << 17]byte +var z12513 [1 << 17]byte +var z12514 [1 << 17]byte +var z12515 [1 << 17]byte +var z12516 [1 << 17]byte +var z12517 [1 << 17]byte +var z12518 [1 << 17]byte +var z12519 [1 << 17]byte +var z12520 [1 << 17]byte +var z12521 [1 << 17]byte +var z12522 [1 << 17]byte +var z12523 [1 << 17]byte +var z12524 [1 << 17]byte +var z12525 [1 << 17]byte +var z12526 [1 << 17]byte +var z12527 [1 << 17]byte +var z12528 [1 << 17]byte +var z12529 [1 << 17]byte +var z12530 [1 << 17]byte +var z12531 [1 << 17]byte +var z12532 [1 << 17]byte +var z12533 [1 << 17]byte +var z12534 [1 << 17]byte +var z12535 [1 << 17]byte +var z12536 [1 << 17]byte +var z12537 [1 << 17]byte +var z12538 [1 << 17]byte +var z12539 [1 << 17]byte +var z12540 [1 << 17]byte +var z12541 [1 << 17]byte +var z12542 [1 << 17]byte +var z12543 [1 << 17]byte +var z12544 [1 << 17]byte +var z12545 [1 << 17]byte +var z12546 [1 << 17]byte +var z12547 [1 << 17]byte +var z12548 [1 << 17]byte +var z12549 [1 << 17]byte +var z12550 [1 << 17]byte +var z12551 [1 << 17]byte +var z12552 [1 << 17]byte +var z12553 [1 << 17]byte +var z12554 [1 << 17]byte +var z12555 [1 << 17]byte +var z12556 [1 << 17]byte +var z12557 [1 << 17]byte +var z12558 [1 << 17]byte +var z12559 [1 << 17]byte +var z12560 [1 << 17]byte +var z12561 [1 << 17]byte +var z12562 [1 << 17]byte +var z12563 [1 << 17]byte +var z12564 [1 << 17]byte +var z12565 [1 << 17]byte +var z12566 [1 << 17]byte +var z12567 [1 << 17]byte +var z12568 [1 << 17]byte +var z12569 [1 << 17]byte +var z12570 [1 << 17]byte +var z12571 [1 << 17]byte +var z12572 [1 << 17]byte +var z12573 [1 << 17]byte +var z12574 [1 << 17]byte +var z12575 [1 << 17]byte +var z12576 [1 << 17]byte +var z12577 [1 << 17]byte +var z12578 [1 << 17]byte +var z12579 [1 << 17]byte +var z12580 [1 << 17]byte +var z12581 [1 << 17]byte +var z12582 [1 << 17]byte +var z12583 [1 << 17]byte +var z12584 [1 << 17]byte +var z12585 [1 << 17]byte +var z12586 [1 << 17]byte +var z12587 [1 << 17]byte +var z12588 [1 << 17]byte +var z12589 [1 << 17]byte +var z12590 [1 << 17]byte +var z12591 [1 << 17]byte +var z12592 [1 << 17]byte +var z12593 [1 << 17]byte +var z12594 [1 << 17]byte +var z12595 [1 << 17]byte +var z12596 [1 << 17]byte +var z12597 [1 << 17]byte +var z12598 [1 << 17]byte +var z12599 [1 << 17]byte +var z12600 [1 << 17]byte +var z12601 [1 << 17]byte +var z12602 [1 << 17]byte +var z12603 [1 << 17]byte +var z12604 [1 << 17]byte +var z12605 [1 << 17]byte +var z12606 [1 << 17]byte +var z12607 [1 << 17]byte +var z12608 [1 << 17]byte +var z12609 [1 << 17]byte +var z12610 [1 << 17]byte +var z12611 [1 << 17]byte +var z12612 [1 << 17]byte +var z12613 [1 << 17]byte +var z12614 [1 << 17]byte +var z12615 [1 << 17]byte +var z12616 [1 << 17]byte +var z12617 [1 << 17]byte +var z12618 [1 << 17]byte +var z12619 [1 << 17]byte +var z12620 [1 << 17]byte +var z12621 [1 << 17]byte +var z12622 [1 << 17]byte +var z12623 [1 << 17]byte +var z12624 [1 << 17]byte +var z12625 [1 << 17]byte +var z12626 [1 << 17]byte +var z12627 [1 << 17]byte +var z12628 [1 << 17]byte +var z12629 [1 << 17]byte +var z12630 [1 << 17]byte +var z12631 [1 << 17]byte +var z12632 [1 << 17]byte +var z12633 [1 << 17]byte +var z12634 [1 << 17]byte +var z12635 [1 << 17]byte +var z12636 [1 << 17]byte +var z12637 [1 << 17]byte +var z12638 [1 << 17]byte +var z12639 [1 << 17]byte +var z12640 [1 << 17]byte +var z12641 [1 << 17]byte +var z12642 [1 << 17]byte +var z12643 [1 << 17]byte +var z12644 [1 << 17]byte +var z12645 [1 << 17]byte +var z12646 [1 << 17]byte +var z12647 [1 << 17]byte +var z12648 [1 << 17]byte +var z12649 [1 << 17]byte +var z12650 [1 << 17]byte +var z12651 [1 << 17]byte +var z12652 [1 << 17]byte +var z12653 [1 << 17]byte +var z12654 [1 << 17]byte +var z12655 [1 << 17]byte +var z12656 [1 << 17]byte +var z12657 [1 << 17]byte +var z12658 [1 << 17]byte +var z12659 [1 << 17]byte +var z12660 [1 << 17]byte +var z12661 [1 << 17]byte +var z12662 [1 << 17]byte +var z12663 [1 << 17]byte +var z12664 [1 << 17]byte +var z12665 [1 << 17]byte +var z12666 [1 << 17]byte +var z12667 [1 << 17]byte +var z12668 [1 << 17]byte +var z12669 [1 << 17]byte +var z12670 [1 << 17]byte +var z12671 [1 << 17]byte +var z12672 [1 << 17]byte +var z12673 [1 << 17]byte +var z12674 [1 << 17]byte +var z12675 [1 << 17]byte +var z12676 [1 << 17]byte +var z12677 [1 << 17]byte +var z12678 [1 << 17]byte +var z12679 [1 << 17]byte +var z12680 [1 << 17]byte +var z12681 [1 << 17]byte +var z12682 [1 << 17]byte +var z12683 [1 << 17]byte +var z12684 [1 << 17]byte +var z12685 [1 << 17]byte +var z12686 [1 << 17]byte +var z12687 [1 << 17]byte +var z12688 [1 << 17]byte +var z12689 [1 << 17]byte +var z12690 [1 << 17]byte +var z12691 [1 << 17]byte +var z12692 [1 << 17]byte +var z12693 [1 << 17]byte +var z12694 [1 << 17]byte +var z12695 [1 << 17]byte +var z12696 [1 << 17]byte +var z12697 [1 << 17]byte +var z12698 [1 << 17]byte +var z12699 [1 << 17]byte +var z12700 [1 << 17]byte +var z12701 [1 << 17]byte +var z12702 [1 << 17]byte +var z12703 [1 << 17]byte +var z12704 [1 << 17]byte +var z12705 [1 << 17]byte +var z12706 [1 << 17]byte +var z12707 [1 << 17]byte +var z12708 [1 << 17]byte +var z12709 [1 << 17]byte +var z12710 [1 << 17]byte +var z12711 [1 << 17]byte +var z12712 [1 << 17]byte +var z12713 [1 << 17]byte +var z12714 [1 << 17]byte +var z12715 [1 << 17]byte +var z12716 [1 << 17]byte +var z12717 [1 << 17]byte +var z12718 [1 << 17]byte +var z12719 [1 << 17]byte +var z12720 [1 << 17]byte +var z12721 [1 << 17]byte +var z12722 [1 << 17]byte +var z12723 [1 << 17]byte +var z12724 [1 << 17]byte +var z12725 [1 << 17]byte +var z12726 [1 << 17]byte +var z12727 [1 << 17]byte +var z12728 [1 << 17]byte +var z12729 [1 << 17]byte +var z12730 [1 << 17]byte +var z12731 [1 << 17]byte +var z12732 [1 << 17]byte +var z12733 [1 << 17]byte +var z12734 [1 << 17]byte +var z12735 [1 << 17]byte +var z12736 [1 << 17]byte +var z12737 [1 << 17]byte +var z12738 [1 << 17]byte +var z12739 [1 << 17]byte +var z12740 [1 << 17]byte +var z12741 [1 << 17]byte +var z12742 [1 << 17]byte +var z12743 [1 << 17]byte +var z12744 [1 << 17]byte +var z12745 [1 << 17]byte +var z12746 [1 << 17]byte +var z12747 [1 << 17]byte +var z12748 [1 << 17]byte +var z12749 [1 << 17]byte +var z12750 [1 << 17]byte +var z12751 [1 << 17]byte +var z12752 [1 << 17]byte +var z12753 [1 << 17]byte +var z12754 [1 << 17]byte +var z12755 [1 << 17]byte +var z12756 [1 << 17]byte +var z12757 [1 << 17]byte +var z12758 [1 << 17]byte +var z12759 [1 << 17]byte +var z12760 [1 << 17]byte +var z12761 [1 << 17]byte +var z12762 [1 << 17]byte +var z12763 [1 << 17]byte +var z12764 [1 << 17]byte +var z12765 [1 << 17]byte +var z12766 [1 << 17]byte +var z12767 [1 << 17]byte +var z12768 [1 << 17]byte +var z12769 [1 << 17]byte +var z12770 [1 << 17]byte +var z12771 [1 << 17]byte +var z12772 [1 << 17]byte +var z12773 [1 << 17]byte +var z12774 [1 << 17]byte +var z12775 [1 << 17]byte +var z12776 [1 << 17]byte +var z12777 [1 << 17]byte +var z12778 [1 << 17]byte +var z12779 [1 << 17]byte +var z12780 [1 << 17]byte +var z12781 [1 << 17]byte +var z12782 [1 << 17]byte +var z12783 [1 << 17]byte +var z12784 [1 << 17]byte +var z12785 [1 << 17]byte +var z12786 [1 << 17]byte +var z12787 [1 << 17]byte +var z12788 [1 << 17]byte +var z12789 [1 << 17]byte +var z12790 [1 << 17]byte +var z12791 [1 << 17]byte +var z12792 [1 << 17]byte +var z12793 [1 << 17]byte +var z12794 [1 << 17]byte +var z12795 [1 << 17]byte +var z12796 [1 << 17]byte +var z12797 [1 << 17]byte +var z12798 [1 << 17]byte +var z12799 [1 << 17]byte +var z12800 [1 << 17]byte +var z12801 [1 << 17]byte +var z12802 [1 << 17]byte +var z12803 [1 << 17]byte +var z12804 [1 << 17]byte +var z12805 [1 << 17]byte +var z12806 [1 << 17]byte +var z12807 [1 << 17]byte +var z12808 [1 << 17]byte +var z12809 [1 << 17]byte +var z12810 [1 << 17]byte +var z12811 [1 << 17]byte +var z12812 [1 << 17]byte +var z12813 [1 << 17]byte +var z12814 [1 << 17]byte +var z12815 [1 << 17]byte +var z12816 [1 << 17]byte +var z12817 [1 << 17]byte +var z12818 [1 << 17]byte +var z12819 [1 << 17]byte +var z12820 [1 << 17]byte +var z12821 [1 << 17]byte +var z12822 [1 << 17]byte +var z12823 [1 << 17]byte +var z12824 [1 << 17]byte +var z12825 [1 << 17]byte +var z12826 [1 << 17]byte +var z12827 [1 << 17]byte +var z12828 [1 << 17]byte +var z12829 [1 << 17]byte +var z12830 [1 << 17]byte +var z12831 [1 << 17]byte +var z12832 [1 << 17]byte +var z12833 [1 << 17]byte +var z12834 [1 << 17]byte +var z12835 [1 << 17]byte +var z12836 [1 << 17]byte +var z12837 [1 << 17]byte +var z12838 [1 << 17]byte +var z12839 [1 << 17]byte +var z12840 [1 << 17]byte +var z12841 [1 << 17]byte +var z12842 [1 << 17]byte +var z12843 [1 << 17]byte +var z12844 [1 << 17]byte +var z12845 [1 << 17]byte +var z12846 [1 << 17]byte +var z12847 [1 << 17]byte +var z12848 [1 << 17]byte +var z12849 [1 << 17]byte +var z12850 [1 << 17]byte +var z12851 [1 << 17]byte +var z12852 [1 << 17]byte +var z12853 [1 << 17]byte +var z12854 [1 << 17]byte +var z12855 [1 << 17]byte +var z12856 [1 << 17]byte +var z12857 [1 << 17]byte +var z12858 [1 << 17]byte +var z12859 [1 << 17]byte +var z12860 [1 << 17]byte +var z12861 [1 << 17]byte +var z12862 [1 << 17]byte +var z12863 [1 << 17]byte +var z12864 [1 << 17]byte +var z12865 [1 << 17]byte +var z12866 [1 << 17]byte +var z12867 [1 << 17]byte +var z12868 [1 << 17]byte +var z12869 [1 << 17]byte +var z12870 [1 << 17]byte +var z12871 [1 << 17]byte +var z12872 [1 << 17]byte +var z12873 [1 << 17]byte +var z12874 [1 << 17]byte +var z12875 [1 << 17]byte +var z12876 [1 << 17]byte +var z12877 [1 << 17]byte +var z12878 [1 << 17]byte +var z12879 [1 << 17]byte +var z12880 [1 << 17]byte +var z12881 [1 << 17]byte +var z12882 [1 << 17]byte +var z12883 [1 << 17]byte +var z12884 [1 << 17]byte +var z12885 [1 << 17]byte +var z12886 [1 << 17]byte +var z12887 [1 << 17]byte +var z12888 [1 << 17]byte +var z12889 [1 << 17]byte +var z12890 [1 << 17]byte +var z12891 [1 << 17]byte +var z12892 [1 << 17]byte +var z12893 [1 << 17]byte +var z12894 [1 << 17]byte +var z12895 [1 << 17]byte +var z12896 [1 << 17]byte +var z12897 [1 << 17]byte +var z12898 [1 << 17]byte +var z12899 [1 << 17]byte +var z12900 [1 << 17]byte +var z12901 [1 << 17]byte +var z12902 [1 << 17]byte +var z12903 [1 << 17]byte +var z12904 [1 << 17]byte +var z12905 [1 << 17]byte +var z12906 [1 << 17]byte +var z12907 [1 << 17]byte +var z12908 [1 << 17]byte +var z12909 [1 << 17]byte +var z12910 [1 << 17]byte +var z12911 [1 << 17]byte +var z12912 [1 << 17]byte +var z12913 [1 << 17]byte +var z12914 [1 << 17]byte +var z12915 [1 << 17]byte +var z12916 [1 << 17]byte +var z12917 [1 << 17]byte +var z12918 [1 << 17]byte +var z12919 [1 << 17]byte +var z12920 [1 << 17]byte +var z12921 [1 << 17]byte +var z12922 [1 << 17]byte +var z12923 [1 << 17]byte +var z12924 [1 << 17]byte +var z12925 [1 << 17]byte +var z12926 [1 << 17]byte +var z12927 [1 << 17]byte +var z12928 [1 << 17]byte +var z12929 [1 << 17]byte +var z12930 [1 << 17]byte +var z12931 [1 << 17]byte +var z12932 [1 << 17]byte +var z12933 [1 << 17]byte +var z12934 [1 << 17]byte +var z12935 [1 << 17]byte +var z12936 [1 << 17]byte +var z12937 [1 << 17]byte +var z12938 [1 << 17]byte +var z12939 [1 << 17]byte +var z12940 [1 << 17]byte +var z12941 [1 << 17]byte +var z12942 [1 << 17]byte +var z12943 [1 << 17]byte +var z12944 [1 << 17]byte +var z12945 [1 << 17]byte +var z12946 [1 << 17]byte +var z12947 [1 << 17]byte +var z12948 [1 << 17]byte +var z12949 [1 << 17]byte +var z12950 [1 << 17]byte +var z12951 [1 << 17]byte +var z12952 [1 << 17]byte +var z12953 [1 << 17]byte +var z12954 [1 << 17]byte +var z12955 [1 << 17]byte +var z12956 [1 << 17]byte +var z12957 [1 << 17]byte +var z12958 [1 << 17]byte +var z12959 [1 << 17]byte +var z12960 [1 << 17]byte +var z12961 [1 << 17]byte +var z12962 [1 << 17]byte +var z12963 [1 << 17]byte +var z12964 [1 << 17]byte +var z12965 [1 << 17]byte +var z12966 [1 << 17]byte +var z12967 [1 << 17]byte +var z12968 [1 << 17]byte +var z12969 [1 << 17]byte +var z12970 [1 << 17]byte +var z12971 [1 << 17]byte +var z12972 [1 << 17]byte +var z12973 [1 << 17]byte +var z12974 [1 << 17]byte +var z12975 [1 << 17]byte +var z12976 [1 << 17]byte +var z12977 [1 << 17]byte +var z12978 [1 << 17]byte +var z12979 [1 << 17]byte +var z12980 [1 << 17]byte +var z12981 [1 << 17]byte +var z12982 [1 << 17]byte +var z12983 [1 << 17]byte +var z12984 [1 << 17]byte +var z12985 [1 << 17]byte +var z12986 [1 << 17]byte +var z12987 [1 << 17]byte +var z12988 [1 << 17]byte +var z12989 [1 << 17]byte +var z12990 [1 << 17]byte +var z12991 [1 << 17]byte +var z12992 [1 << 17]byte +var z12993 [1 << 17]byte +var z12994 [1 << 17]byte +var z12995 [1 << 17]byte +var z12996 [1 << 17]byte +var z12997 [1 << 17]byte +var z12998 [1 << 17]byte +var z12999 [1 << 17]byte +var z13000 [1 << 17]byte +var z13001 [1 << 17]byte +var z13002 [1 << 17]byte +var z13003 [1 << 17]byte +var z13004 [1 << 17]byte +var z13005 [1 << 17]byte +var z13006 [1 << 17]byte +var z13007 [1 << 17]byte +var z13008 [1 << 17]byte +var z13009 [1 << 17]byte +var z13010 [1 << 17]byte +var z13011 [1 << 17]byte +var z13012 [1 << 17]byte +var z13013 [1 << 17]byte +var z13014 [1 << 17]byte +var z13015 [1 << 17]byte +var z13016 [1 << 17]byte +var z13017 [1 << 17]byte +var z13018 [1 << 17]byte +var z13019 [1 << 17]byte +var z13020 [1 << 17]byte +var z13021 [1 << 17]byte +var z13022 [1 << 17]byte +var z13023 [1 << 17]byte +var z13024 [1 << 17]byte +var z13025 [1 << 17]byte +var z13026 [1 << 17]byte +var z13027 [1 << 17]byte +var z13028 [1 << 17]byte +var z13029 [1 << 17]byte +var z13030 [1 << 17]byte +var z13031 [1 << 17]byte +var z13032 [1 << 17]byte +var z13033 [1 << 17]byte +var z13034 [1 << 17]byte +var z13035 [1 << 17]byte +var z13036 [1 << 17]byte +var z13037 [1 << 17]byte +var z13038 [1 << 17]byte +var z13039 [1 << 17]byte +var z13040 [1 << 17]byte +var z13041 [1 << 17]byte +var z13042 [1 << 17]byte +var z13043 [1 << 17]byte +var z13044 [1 << 17]byte +var z13045 [1 << 17]byte +var z13046 [1 << 17]byte +var z13047 [1 << 17]byte +var z13048 [1 << 17]byte +var z13049 [1 << 17]byte +var z13050 [1 << 17]byte +var z13051 [1 << 17]byte +var z13052 [1 << 17]byte +var z13053 [1 << 17]byte +var z13054 [1 << 17]byte +var z13055 [1 << 17]byte +var z13056 [1 << 17]byte +var z13057 [1 << 17]byte +var z13058 [1 << 17]byte +var z13059 [1 << 17]byte +var z13060 [1 << 17]byte +var z13061 [1 << 17]byte +var z13062 [1 << 17]byte +var z13063 [1 << 17]byte +var z13064 [1 << 17]byte +var z13065 [1 << 17]byte +var z13066 [1 << 17]byte +var z13067 [1 << 17]byte +var z13068 [1 << 17]byte +var z13069 [1 << 17]byte +var z13070 [1 << 17]byte +var z13071 [1 << 17]byte +var z13072 [1 << 17]byte +var z13073 [1 << 17]byte +var z13074 [1 << 17]byte +var z13075 [1 << 17]byte +var z13076 [1 << 17]byte +var z13077 [1 << 17]byte +var z13078 [1 << 17]byte +var z13079 [1 << 17]byte +var z13080 [1 << 17]byte +var z13081 [1 << 17]byte +var z13082 [1 << 17]byte +var z13083 [1 << 17]byte +var z13084 [1 << 17]byte +var z13085 [1 << 17]byte +var z13086 [1 << 17]byte +var z13087 [1 << 17]byte +var z13088 [1 << 17]byte +var z13089 [1 << 17]byte +var z13090 [1 << 17]byte +var z13091 [1 << 17]byte +var z13092 [1 << 17]byte +var z13093 [1 << 17]byte +var z13094 [1 << 17]byte +var z13095 [1 << 17]byte +var z13096 [1 << 17]byte +var z13097 [1 << 17]byte +var z13098 [1 << 17]byte +var z13099 [1 << 17]byte +var z13100 [1 << 17]byte +var z13101 [1 << 17]byte +var z13102 [1 << 17]byte +var z13103 [1 << 17]byte +var z13104 [1 << 17]byte +var z13105 [1 << 17]byte +var z13106 [1 << 17]byte +var z13107 [1 << 17]byte +var z13108 [1 << 17]byte +var z13109 [1 << 17]byte +var z13110 [1 << 17]byte +var z13111 [1 << 17]byte +var z13112 [1 << 17]byte +var z13113 [1 << 17]byte +var z13114 [1 << 17]byte +var z13115 [1 << 17]byte +var z13116 [1 << 17]byte +var z13117 [1 << 17]byte +var z13118 [1 << 17]byte +var z13119 [1 << 17]byte +var z13120 [1 << 17]byte +var z13121 [1 << 17]byte +var z13122 [1 << 17]byte +var z13123 [1 << 17]byte +var z13124 [1 << 17]byte +var z13125 [1 << 17]byte +var z13126 [1 << 17]byte +var z13127 [1 << 17]byte +var z13128 [1 << 17]byte +var z13129 [1 << 17]byte +var z13130 [1 << 17]byte +var z13131 [1 << 17]byte +var z13132 [1 << 17]byte +var z13133 [1 << 17]byte +var z13134 [1 << 17]byte +var z13135 [1 << 17]byte +var z13136 [1 << 17]byte +var z13137 [1 << 17]byte +var z13138 [1 << 17]byte +var z13139 [1 << 17]byte +var z13140 [1 << 17]byte +var z13141 [1 << 17]byte +var z13142 [1 << 17]byte +var z13143 [1 << 17]byte +var z13144 [1 << 17]byte +var z13145 [1 << 17]byte +var z13146 [1 << 17]byte +var z13147 [1 << 17]byte +var z13148 [1 << 17]byte +var z13149 [1 << 17]byte +var z13150 [1 << 17]byte +var z13151 [1 << 17]byte +var z13152 [1 << 17]byte +var z13153 [1 << 17]byte +var z13154 [1 << 17]byte +var z13155 [1 << 17]byte +var z13156 [1 << 17]byte +var z13157 [1 << 17]byte +var z13158 [1 << 17]byte +var z13159 [1 << 17]byte +var z13160 [1 << 17]byte +var z13161 [1 << 17]byte +var z13162 [1 << 17]byte +var z13163 [1 << 17]byte +var z13164 [1 << 17]byte +var z13165 [1 << 17]byte +var z13166 [1 << 17]byte +var z13167 [1 << 17]byte +var z13168 [1 << 17]byte +var z13169 [1 << 17]byte +var z13170 [1 << 17]byte +var z13171 [1 << 17]byte +var z13172 [1 << 17]byte +var z13173 [1 << 17]byte +var z13174 [1 << 17]byte +var z13175 [1 << 17]byte +var z13176 [1 << 17]byte +var z13177 [1 << 17]byte +var z13178 [1 << 17]byte +var z13179 [1 << 17]byte +var z13180 [1 << 17]byte +var z13181 [1 << 17]byte +var z13182 [1 << 17]byte +var z13183 [1 << 17]byte +var z13184 [1 << 17]byte +var z13185 [1 << 17]byte +var z13186 [1 << 17]byte +var z13187 [1 << 17]byte +var z13188 [1 << 17]byte +var z13189 [1 << 17]byte +var z13190 [1 << 17]byte +var z13191 [1 << 17]byte +var z13192 [1 << 17]byte +var z13193 [1 << 17]byte +var z13194 [1 << 17]byte +var z13195 [1 << 17]byte +var z13196 [1 << 17]byte +var z13197 [1 << 17]byte +var z13198 [1 << 17]byte +var z13199 [1 << 17]byte +var z13200 [1 << 17]byte +var z13201 [1 << 17]byte +var z13202 [1 << 17]byte +var z13203 [1 << 17]byte +var z13204 [1 << 17]byte +var z13205 [1 << 17]byte +var z13206 [1 << 17]byte +var z13207 [1 << 17]byte +var z13208 [1 << 17]byte +var z13209 [1 << 17]byte +var z13210 [1 << 17]byte +var z13211 [1 << 17]byte +var z13212 [1 << 17]byte +var z13213 [1 << 17]byte +var z13214 [1 << 17]byte +var z13215 [1 << 17]byte +var z13216 [1 << 17]byte +var z13217 [1 << 17]byte +var z13218 [1 << 17]byte +var z13219 [1 << 17]byte +var z13220 [1 << 17]byte +var z13221 [1 << 17]byte +var z13222 [1 << 17]byte +var z13223 [1 << 17]byte +var z13224 [1 << 17]byte +var z13225 [1 << 17]byte +var z13226 [1 << 17]byte +var z13227 [1 << 17]byte +var z13228 [1 << 17]byte +var z13229 [1 << 17]byte +var z13230 [1 << 17]byte +var z13231 [1 << 17]byte +var z13232 [1 << 17]byte +var z13233 [1 << 17]byte +var z13234 [1 << 17]byte +var z13235 [1 << 17]byte +var z13236 [1 << 17]byte +var z13237 [1 << 17]byte +var z13238 [1 << 17]byte +var z13239 [1 << 17]byte +var z13240 [1 << 17]byte +var z13241 [1 << 17]byte +var z13242 [1 << 17]byte +var z13243 [1 << 17]byte +var z13244 [1 << 17]byte +var z13245 [1 << 17]byte +var z13246 [1 << 17]byte +var z13247 [1 << 17]byte +var z13248 [1 << 17]byte +var z13249 [1 << 17]byte +var z13250 [1 << 17]byte +var z13251 [1 << 17]byte +var z13252 [1 << 17]byte +var z13253 [1 << 17]byte +var z13254 [1 << 17]byte +var z13255 [1 << 17]byte +var z13256 [1 << 17]byte +var z13257 [1 << 17]byte +var z13258 [1 << 17]byte +var z13259 [1 << 17]byte +var z13260 [1 << 17]byte +var z13261 [1 << 17]byte +var z13262 [1 << 17]byte +var z13263 [1 << 17]byte +var z13264 [1 << 17]byte +var z13265 [1 << 17]byte +var z13266 [1 << 17]byte +var z13267 [1 << 17]byte +var z13268 [1 << 17]byte +var z13269 [1 << 17]byte +var z13270 [1 << 17]byte +var z13271 [1 << 17]byte +var z13272 [1 << 17]byte +var z13273 [1 << 17]byte +var z13274 [1 << 17]byte +var z13275 [1 << 17]byte +var z13276 [1 << 17]byte +var z13277 [1 << 17]byte +var z13278 [1 << 17]byte +var z13279 [1 << 17]byte +var z13280 [1 << 17]byte +var z13281 [1 << 17]byte +var z13282 [1 << 17]byte +var z13283 [1 << 17]byte +var z13284 [1 << 17]byte +var z13285 [1 << 17]byte +var z13286 [1 << 17]byte +var z13287 [1 << 17]byte +var z13288 [1 << 17]byte +var z13289 [1 << 17]byte +var z13290 [1 << 17]byte +var z13291 [1 << 17]byte +var z13292 [1 << 17]byte +var z13293 [1 << 17]byte +var z13294 [1 << 17]byte +var z13295 [1 << 17]byte +var z13296 [1 << 17]byte +var z13297 [1 << 17]byte +var z13298 [1 << 17]byte +var z13299 [1 << 17]byte +var z13300 [1 << 17]byte +var z13301 [1 << 17]byte +var z13302 [1 << 17]byte +var z13303 [1 << 17]byte +var z13304 [1 << 17]byte +var z13305 [1 << 17]byte +var z13306 [1 << 17]byte +var z13307 [1 << 17]byte +var z13308 [1 << 17]byte +var z13309 [1 << 17]byte +var z13310 [1 << 17]byte +var z13311 [1 << 17]byte +var z13312 [1 << 17]byte +var z13313 [1 << 17]byte +var z13314 [1 << 17]byte +var z13315 [1 << 17]byte +var z13316 [1 << 17]byte +var z13317 [1 << 17]byte +var z13318 [1 << 17]byte +var z13319 [1 << 17]byte +var z13320 [1 << 17]byte +var z13321 [1 << 17]byte +var z13322 [1 << 17]byte +var z13323 [1 << 17]byte +var z13324 [1 << 17]byte +var z13325 [1 << 17]byte +var z13326 [1 << 17]byte +var z13327 [1 << 17]byte +var z13328 [1 << 17]byte +var z13329 [1 << 17]byte +var z13330 [1 << 17]byte +var z13331 [1 << 17]byte +var z13332 [1 << 17]byte +var z13333 [1 << 17]byte +var z13334 [1 << 17]byte +var z13335 [1 << 17]byte +var z13336 [1 << 17]byte +var z13337 [1 << 17]byte +var z13338 [1 << 17]byte +var z13339 [1 << 17]byte +var z13340 [1 << 17]byte +var z13341 [1 << 17]byte +var z13342 [1 << 17]byte +var z13343 [1 << 17]byte +var z13344 [1 << 17]byte +var z13345 [1 << 17]byte +var z13346 [1 << 17]byte +var z13347 [1 << 17]byte +var z13348 [1 << 17]byte +var z13349 [1 << 17]byte +var z13350 [1 << 17]byte +var z13351 [1 << 17]byte +var z13352 [1 << 17]byte +var z13353 [1 << 17]byte +var z13354 [1 << 17]byte +var z13355 [1 << 17]byte +var z13356 [1 << 17]byte +var z13357 [1 << 17]byte +var z13358 [1 << 17]byte +var z13359 [1 << 17]byte +var z13360 [1 << 17]byte +var z13361 [1 << 17]byte +var z13362 [1 << 17]byte +var z13363 [1 << 17]byte +var z13364 [1 << 17]byte +var z13365 [1 << 17]byte +var z13366 [1 << 17]byte +var z13367 [1 << 17]byte +var z13368 [1 << 17]byte +var z13369 [1 << 17]byte +var z13370 [1 << 17]byte +var z13371 [1 << 17]byte +var z13372 [1 << 17]byte +var z13373 [1 << 17]byte +var z13374 [1 << 17]byte +var z13375 [1 << 17]byte +var z13376 [1 << 17]byte +var z13377 [1 << 17]byte +var z13378 [1 << 17]byte +var z13379 [1 << 17]byte +var z13380 [1 << 17]byte +var z13381 [1 << 17]byte +var z13382 [1 << 17]byte +var z13383 [1 << 17]byte +var z13384 [1 << 17]byte +var z13385 [1 << 17]byte +var z13386 [1 << 17]byte +var z13387 [1 << 17]byte +var z13388 [1 << 17]byte +var z13389 [1 << 17]byte +var z13390 [1 << 17]byte +var z13391 [1 << 17]byte +var z13392 [1 << 17]byte +var z13393 [1 << 17]byte +var z13394 [1 << 17]byte +var z13395 [1 << 17]byte +var z13396 [1 << 17]byte +var z13397 [1 << 17]byte +var z13398 [1 << 17]byte +var z13399 [1 << 17]byte +var z13400 [1 << 17]byte +var z13401 [1 << 17]byte +var z13402 [1 << 17]byte +var z13403 [1 << 17]byte +var z13404 [1 << 17]byte +var z13405 [1 << 17]byte +var z13406 [1 << 17]byte +var z13407 [1 << 17]byte +var z13408 [1 << 17]byte +var z13409 [1 << 17]byte +var z13410 [1 << 17]byte +var z13411 [1 << 17]byte +var z13412 [1 << 17]byte +var z13413 [1 << 17]byte +var z13414 [1 << 17]byte +var z13415 [1 << 17]byte +var z13416 [1 << 17]byte +var z13417 [1 << 17]byte +var z13418 [1 << 17]byte +var z13419 [1 << 17]byte +var z13420 [1 << 17]byte +var z13421 [1 << 17]byte +var z13422 [1 << 17]byte +var z13423 [1 << 17]byte +var z13424 [1 << 17]byte +var z13425 [1 << 17]byte +var z13426 [1 << 17]byte +var z13427 [1 << 17]byte +var z13428 [1 << 17]byte +var z13429 [1 << 17]byte +var z13430 [1 << 17]byte +var z13431 [1 << 17]byte +var z13432 [1 << 17]byte +var z13433 [1 << 17]byte +var z13434 [1 << 17]byte +var z13435 [1 << 17]byte +var z13436 [1 << 17]byte +var z13437 [1 << 17]byte +var z13438 [1 << 17]byte +var z13439 [1 << 17]byte +var z13440 [1 << 17]byte +var z13441 [1 << 17]byte +var z13442 [1 << 17]byte +var z13443 [1 << 17]byte +var z13444 [1 << 17]byte +var z13445 [1 << 17]byte +var z13446 [1 << 17]byte +var z13447 [1 << 17]byte +var z13448 [1 << 17]byte +var z13449 [1 << 17]byte +var z13450 [1 << 17]byte +var z13451 [1 << 17]byte +var z13452 [1 << 17]byte +var z13453 [1 << 17]byte +var z13454 [1 << 17]byte +var z13455 [1 << 17]byte +var z13456 [1 << 17]byte +var z13457 [1 << 17]byte +var z13458 [1 << 17]byte +var z13459 [1 << 17]byte +var z13460 [1 << 17]byte +var z13461 [1 << 17]byte +var z13462 [1 << 17]byte +var z13463 [1 << 17]byte +var z13464 [1 << 17]byte +var z13465 [1 << 17]byte +var z13466 [1 << 17]byte +var z13467 [1 << 17]byte +var z13468 [1 << 17]byte +var z13469 [1 << 17]byte +var z13470 [1 << 17]byte +var z13471 [1 << 17]byte +var z13472 [1 << 17]byte +var z13473 [1 << 17]byte +var z13474 [1 << 17]byte +var z13475 [1 << 17]byte +var z13476 [1 << 17]byte +var z13477 [1 << 17]byte +var z13478 [1 << 17]byte +var z13479 [1 << 17]byte +var z13480 [1 << 17]byte +var z13481 [1 << 17]byte +var z13482 [1 << 17]byte +var z13483 [1 << 17]byte +var z13484 [1 << 17]byte +var z13485 [1 << 17]byte +var z13486 [1 << 17]byte +var z13487 [1 << 17]byte +var z13488 [1 << 17]byte +var z13489 [1 << 17]byte +var z13490 [1 << 17]byte +var z13491 [1 << 17]byte +var z13492 [1 << 17]byte +var z13493 [1 << 17]byte +var z13494 [1 << 17]byte +var z13495 [1 << 17]byte +var z13496 [1 << 17]byte +var z13497 [1 << 17]byte +var z13498 [1 << 17]byte +var z13499 [1 << 17]byte +var z13500 [1 << 17]byte +var z13501 [1 << 17]byte +var z13502 [1 << 17]byte +var z13503 [1 << 17]byte +var z13504 [1 << 17]byte +var z13505 [1 << 17]byte +var z13506 [1 << 17]byte +var z13507 [1 << 17]byte +var z13508 [1 << 17]byte +var z13509 [1 << 17]byte +var z13510 [1 << 17]byte +var z13511 [1 << 17]byte +var z13512 [1 << 17]byte +var z13513 [1 << 17]byte +var z13514 [1 << 17]byte +var z13515 [1 << 17]byte +var z13516 [1 << 17]byte +var z13517 [1 << 17]byte +var z13518 [1 << 17]byte +var z13519 [1 << 17]byte +var z13520 [1 << 17]byte +var z13521 [1 << 17]byte +var z13522 [1 << 17]byte +var z13523 [1 << 17]byte +var z13524 [1 << 17]byte +var z13525 [1 << 17]byte +var z13526 [1 << 17]byte +var z13527 [1 << 17]byte +var z13528 [1 << 17]byte +var z13529 [1 << 17]byte +var z13530 [1 << 17]byte +var z13531 [1 << 17]byte +var z13532 [1 << 17]byte +var z13533 [1 << 17]byte +var z13534 [1 << 17]byte +var z13535 [1 << 17]byte +var z13536 [1 << 17]byte +var z13537 [1 << 17]byte +var z13538 [1 << 17]byte +var z13539 [1 << 17]byte +var z13540 [1 << 17]byte +var z13541 [1 << 17]byte +var z13542 [1 << 17]byte +var z13543 [1 << 17]byte +var z13544 [1 << 17]byte +var z13545 [1 << 17]byte +var z13546 [1 << 17]byte +var z13547 [1 << 17]byte +var z13548 [1 << 17]byte +var z13549 [1 << 17]byte +var z13550 [1 << 17]byte +var z13551 [1 << 17]byte +var z13552 [1 << 17]byte +var z13553 [1 << 17]byte +var z13554 [1 << 17]byte +var z13555 [1 << 17]byte +var z13556 [1 << 17]byte +var z13557 [1 << 17]byte +var z13558 [1 << 17]byte +var z13559 [1 << 17]byte +var z13560 [1 << 17]byte +var z13561 [1 << 17]byte +var z13562 [1 << 17]byte +var z13563 [1 << 17]byte +var z13564 [1 << 17]byte +var z13565 [1 << 17]byte +var z13566 [1 << 17]byte +var z13567 [1 << 17]byte +var z13568 [1 << 17]byte +var z13569 [1 << 17]byte +var z13570 [1 << 17]byte +var z13571 [1 << 17]byte +var z13572 [1 << 17]byte +var z13573 [1 << 17]byte +var z13574 [1 << 17]byte +var z13575 [1 << 17]byte +var z13576 [1 << 17]byte +var z13577 [1 << 17]byte +var z13578 [1 << 17]byte +var z13579 [1 << 17]byte +var z13580 [1 << 17]byte +var z13581 [1 << 17]byte +var z13582 [1 << 17]byte +var z13583 [1 << 17]byte +var z13584 [1 << 17]byte +var z13585 [1 << 17]byte +var z13586 [1 << 17]byte +var z13587 [1 << 17]byte +var z13588 [1 << 17]byte +var z13589 [1 << 17]byte +var z13590 [1 << 17]byte +var z13591 [1 << 17]byte +var z13592 [1 << 17]byte +var z13593 [1 << 17]byte +var z13594 [1 << 17]byte +var z13595 [1 << 17]byte +var z13596 [1 << 17]byte +var z13597 [1 << 17]byte +var z13598 [1 << 17]byte +var z13599 [1 << 17]byte +var z13600 [1 << 17]byte +var z13601 [1 << 17]byte +var z13602 [1 << 17]byte +var z13603 [1 << 17]byte +var z13604 [1 << 17]byte +var z13605 [1 << 17]byte +var z13606 [1 << 17]byte +var z13607 [1 << 17]byte +var z13608 [1 << 17]byte +var z13609 [1 << 17]byte +var z13610 [1 << 17]byte +var z13611 [1 << 17]byte +var z13612 [1 << 17]byte +var z13613 [1 << 17]byte +var z13614 [1 << 17]byte +var z13615 [1 << 17]byte +var z13616 [1 << 17]byte +var z13617 [1 << 17]byte +var z13618 [1 << 17]byte +var z13619 [1 << 17]byte +var z13620 [1 << 17]byte +var z13621 [1 << 17]byte +var z13622 [1 << 17]byte +var z13623 [1 << 17]byte +var z13624 [1 << 17]byte +var z13625 [1 << 17]byte +var z13626 [1 << 17]byte +var z13627 [1 << 17]byte +var z13628 [1 << 17]byte +var z13629 [1 << 17]byte +var z13630 [1 << 17]byte +var z13631 [1 << 17]byte +var z13632 [1 << 17]byte +var z13633 [1 << 17]byte +var z13634 [1 << 17]byte +var z13635 [1 << 17]byte +var z13636 [1 << 17]byte +var z13637 [1 << 17]byte +var z13638 [1 << 17]byte +var z13639 [1 << 17]byte +var z13640 [1 << 17]byte +var z13641 [1 << 17]byte +var z13642 [1 << 17]byte +var z13643 [1 << 17]byte +var z13644 [1 << 17]byte +var z13645 [1 << 17]byte +var z13646 [1 << 17]byte +var z13647 [1 << 17]byte +var z13648 [1 << 17]byte +var z13649 [1 << 17]byte +var z13650 [1 << 17]byte +var z13651 [1 << 17]byte +var z13652 [1 << 17]byte +var z13653 [1 << 17]byte +var z13654 [1 << 17]byte +var z13655 [1 << 17]byte +var z13656 [1 << 17]byte +var z13657 [1 << 17]byte +var z13658 [1 << 17]byte +var z13659 [1 << 17]byte +var z13660 [1 << 17]byte +var z13661 [1 << 17]byte +var z13662 [1 << 17]byte +var z13663 [1 << 17]byte +var z13664 [1 << 17]byte +var z13665 [1 << 17]byte +var z13666 [1 << 17]byte +var z13667 [1 << 17]byte +var z13668 [1 << 17]byte +var z13669 [1 << 17]byte +var z13670 [1 << 17]byte +var z13671 [1 << 17]byte +var z13672 [1 << 17]byte +var z13673 [1 << 17]byte +var z13674 [1 << 17]byte +var z13675 [1 << 17]byte +var z13676 [1 << 17]byte +var z13677 [1 << 17]byte +var z13678 [1 << 17]byte +var z13679 [1 << 17]byte +var z13680 [1 << 17]byte +var z13681 [1 << 17]byte +var z13682 [1 << 17]byte +var z13683 [1 << 17]byte +var z13684 [1 << 17]byte +var z13685 [1 << 17]byte +var z13686 [1 << 17]byte +var z13687 [1 << 17]byte +var z13688 [1 << 17]byte +var z13689 [1 << 17]byte +var z13690 [1 << 17]byte +var z13691 [1 << 17]byte +var z13692 [1 << 17]byte +var z13693 [1 << 17]byte +var z13694 [1 << 17]byte +var z13695 [1 << 17]byte +var z13696 [1 << 17]byte +var z13697 [1 << 17]byte +var z13698 [1 << 17]byte +var z13699 [1 << 17]byte +var z13700 [1 << 17]byte +var z13701 [1 << 17]byte +var z13702 [1 << 17]byte +var z13703 [1 << 17]byte +var z13704 [1 << 17]byte +var z13705 [1 << 17]byte +var z13706 [1 << 17]byte +var z13707 [1 << 17]byte +var z13708 [1 << 17]byte +var z13709 [1 << 17]byte +var z13710 [1 << 17]byte +var z13711 [1 << 17]byte +var z13712 [1 << 17]byte +var z13713 [1 << 17]byte +var z13714 [1 << 17]byte +var z13715 [1 << 17]byte +var z13716 [1 << 17]byte +var z13717 [1 << 17]byte +var z13718 [1 << 17]byte +var z13719 [1 << 17]byte +var z13720 [1 << 17]byte +var z13721 [1 << 17]byte +var z13722 [1 << 17]byte +var z13723 [1 << 17]byte +var z13724 [1 << 17]byte +var z13725 [1 << 17]byte +var z13726 [1 << 17]byte +var z13727 [1 << 17]byte +var z13728 [1 << 17]byte +var z13729 [1 << 17]byte +var z13730 [1 << 17]byte +var z13731 [1 << 17]byte +var z13732 [1 << 17]byte +var z13733 [1 << 17]byte +var z13734 [1 << 17]byte +var z13735 [1 << 17]byte +var z13736 [1 << 17]byte +var z13737 [1 << 17]byte +var z13738 [1 << 17]byte +var z13739 [1 << 17]byte +var z13740 [1 << 17]byte +var z13741 [1 << 17]byte +var z13742 [1 << 17]byte +var z13743 [1 << 17]byte +var z13744 [1 << 17]byte +var z13745 [1 << 17]byte +var z13746 [1 << 17]byte +var z13747 [1 << 17]byte +var z13748 [1 << 17]byte +var z13749 [1 << 17]byte +var z13750 [1 << 17]byte +var z13751 [1 << 17]byte +var z13752 [1 << 17]byte +var z13753 [1 << 17]byte +var z13754 [1 << 17]byte +var z13755 [1 << 17]byte +var z13756 [1 << 17]byte +var z13757 [1 << 17]byte +var z13758 [1 << 17]byte +var z13759 [1 << 17]byte +var z13760 [1 << 17]byte +var z13761 [1 << 17]byte +var z13762 [1 << 17]byte +var z13763 [1 << 17]byte +var z13764 [1 << 17]byte +var z13765 [1 << 17]byte +var z13766 [1 << 17]byte +var z13767 [1 << 17]byte +var z13768 [1 << 17]byte +var z13769 [1 << 17]byte +var z13770 [1 << 17]byte +var z13771 [1 << 17]byte +var z13772 [1 << 17]byte +var z13773 [1 << 17]byte +var z13774 [1 << 17]byte +var z13775 [1 << 17]byte +var z13776 [1 << 17]byte +var z13777 [1 << 17]byte +var z13778 [1 << 17]byte +var z13779 [1 << 17]byte +var z13780 [1 << 17]byte +var z13781 [1 << 17]byte +var z13782 [1 << 17]byte +var z13783 [1 << 17]byte +var z13784 [1 << 17]byte +var z13785 [1 << 17]byte +var z13786 [1 << 17]byte +var z13787 [1 << 17]byte +var z13788 [1 << 17]byte +var z13789 [1 << 17]byte +var z13790 [1 << 17]byte +var z13791 [1 << 17]byte +var z13792 [1 << 17]byte +var z13793 [1 << 17]byte +var z13794 [1 << 17]byte +var z13795 [1 << 17]byte +var z13796 [1 << 17]byte +var z13797 [1 << 17]byte +var z13798 [1 << 17]byte +var z13799 [1 << 17]byte +var z13800 [1 << 17]byte +var z13801 [1 << 17]byte +var z13802 [1 << 17]byte +var z13803 [1 << 17]byte +var z13804 [1 << 17]byte +var z13805 [1 << 17]byte +var z13806 [1 << 17]byte +var z13807 [1 << 17]byte +var z13808 [1 << 17]byte +var z13809 [1 << 17]byte +var z13810 [1 << 17]byte +var z13811 [1 << 17]byte +var z13812 [1 << 17]byte +var z13813 [1 << 17]byte +var z13814 [1 << 17]byte +var z13815 [1 << 17]byte +var z13816 [1 << 17]byte +var z13817 [1 << 17]byte +var z13818 [1 << 17]byte +var z13819 [1 << 17]byte +var z13820 [1 << 17]byte +var z13821 [1 << 17]byte +var z13822 [1 << 17]byte +var z13823 [1 << 17]byte +var z13824 [1 << 17]byte +var z13825 [1 << 17]byte +var z13826 [1 << 17]byte +var z13827 [1 << 17]byte +var z13828 [1 << 17]byte +var z13829 [1 << 17]byte +var z13830 [1 << 17]byte +var z13831 [1 << 17]byte +var z13832 [1 << 17]byte +var z13833 [1 << 17]byte +var z13834 [1 << 17]byte +var z13835 [1 << 17]byte +var z13836 [1 << 17]byte +var z13837 [1 << 17]byte +var z13838 [1 << 17]byte +var z13839 [1 << 17]byte +var z13840 [1 << 17]byte +var z13841 [1 << 17]byte +var z13842 [1 << 17]byte +var z13843 [1 << 17]byte +var z13844 [1 << 17]byte +var z13845 [1 << 17]byte +var z13846 [1 << 17]byte +var z13847 [1 << 17]byte +var z13848 [1 << 17]byte +var z13849 [1 << 17]byte +var z13850 [1 << 17]byte +var z13851 [1 << 17]byte +var z13852 [1 << 17]byte +var z13853 [1 << 17]byte +var z13854 [1 << 17]byte +var z13855 [1 << 17]byte +var z13856 [1 << 17]byte +var z13857 [1 << 17]byte +var z13858 [1 << 17]byte +var z13859 [1 << 17]byte +var z13860 [1 << 17]byte +var z13861 [1 << 17]byte +var z13862 [1 << 17]byte +var z13863 [1 << 17]byte +var z13864 [1 << 17]byte +var z13865 [1 << 17]byte +var z13866 [1 << 17]byte +var z13867 [1 << 17]byte +var z13868 [1 << 17]byte +var z13869 [1 << 17]byte +var z13870 [1 << 17]byte +var z13871 [1 << 17]byte +var z13872 [1 << 17]byte +var z13873 [1 << 17]byte +var z13874 [1 << 17]byte +var z13875 [1 << 17]byte +var z13876 [1 << 17]byte +var z13877 [1 << 17]byte +var z13878 [1 << 17]byte +var z13879 [1 << 17]byte +var z13880 [1 << 17]byte +var z13881 [1 << 17]byte +var z13882 [1 << 17]byte +var z13883 [1 << 17]byte +var z13884 [1 << 17]byte +var z13885 [1 << 17]byte +var z13886 [1 << 17]byte +var z13887 [1 << 17]byte +var z13888 [1 << 17]byte +var z13889 [1 << 17]byte +var z13890 [1 << 17]byte +var z13891 [1 << 17]byte +var z13892 [1 << 17]byte +var z13893 [1 << 17]byte +var z13894 [1 << 17]byte +var z13895 [1 << 17]byte +var z13896 [1 << 17]byte +var z13897 [1 << 17]byte +var z13898 [1 << 17]byte +var z13899 [1 << 17]byte +var z13900 [1 << 17]byte +var z13901 [1 << 17]byte +var z13902 [1 << 17]byte +var z13903 [1 << 17]byte +var z13904 [1 << 17]byte +var z13905 [1 << 17]byte +var z13906 [1 << 17]byte +var z13907 [1 << 17]byte +var z13908 [1 << 17]byte +var z13909 [1 << 17]byte +var z13910 [1 << 17]byte +var z13911 [1 << 17]byte +var z13912 [1 << 17]byte +var z13913 [1 << 17]byte +var z13914 [1 << 17]byte +var z13915 [1 << 17]byte +var z13916 [1 << 17]byte +var z13917 [1 << 17]byte +var z13918 [1 << 17]byte +var z13919 [1 << 17]byte +var z13920 [1 << 17]byte +var z13921 [1 << 17]byte +var z13922 [1 << 17]byte +var z13923 [1 << 17]byte +var z13924 [1 << 17]byte +var z13925 [1 << 17]byte +var z13926 [1 << 17]byte +var z13927 [1 << 17]byte +var z13928 [1 << 17]byte +var z13929 [1 << 17]byte +var z13930 [1 << 17]byte +var z13931 [1 << 17]byte +var z13932 [1 << 17]byte +var z13933 [1 << 17]byte +var z13934 [1 << 17]byte +var z13935 [1 << 17]byte +var z13936 [1 << 17]byte +var z13937 [1 << 17]byte +var z13938 [1 << 17]byte +var z13939 [1 << 17]byte +var z13940 [1 << 17]byte +var z13941 [1 << 17]byte +var z13942 [1 << 17]byte +var z13943 [1 << 17]byte +var z13944 [1 << 17]byte +var z13945 [1 << 17]byte +var z13946 [1 << 17]byte +var z13947 [1 << 17]byte +var z13948 [1 << 17]byte +var z13949 [1 << 17]byte +var z13950 [1 << 17]byte +var z13951 [1 << 17]byte +var z13952 [1 << 17]byte +var z13953 [1 << 17]byte +var z13954 [1 << 17]byte +var z13955 [1 << 17]byte +var z13956 [1 << 17]byte +var z13957 [1 << 17]byte +var z13958 [1 << 17]byte +var z13959 [1 << 17]byte +var z13960 [1 << 17]byte +var z13961 [1 << 17]byte +var z13962 [1 << 17]byte +var z13963 [1 << 17]byte +var z13964 [1 << 17]byte +var z13965 [1 << 17]byte +var z13966 [1 << 17]byte +var z13967 [1 << 17]byte +var z13968 [1 << 17]byte +var z13969 [1 << 17]byte +var z13970 [1 << 17]byte +var z13971 [1 << 17]byte +var z13972 [1 << 17]byte +var z13973 [1 << 17]byte +var z13974 [1 << 17]byte +var z13975 [1 << 17]byte +var z13976 [1 << 17]byte +var z13977 [1 << 17]byte +var z13978 [1 << 17]byte +var z13979 [1 << 17]byte +var z13980 [1 << 17]byte +var z13981 [1 << 17]byte +var z13982 [1 << 17]byte +var z13983 [1 << 17]byte +var z13984 [1 << 17]byte +var z13985 [1 << 17]byte +var z13986 [1 << 17]byte +var z13987 [1 << 17]byte +var z13988 [1 << 17]byte +var z13989 [1 << 17]byte +var z13990 [1 << 17]byte +var z13991 [1 << 17]byte +var z13992 [1 << 17]byte +var z13993 [1 << 17]byte +var z13994 [1 << 17]byte +var z13995 [1 << 17]byte +var z13996 [1 << 17]byte +var z13997 [1 << 17]byte +var z13998 [1 << 17]byte +var z13999 [1 << 17]byte +var z14000 [1 << 17]byte +var z14001 [1 << 17]byte +var z14002 [1 << 17]byte +var z14003 [1 << 17]byte +var z14004 [1 << 17]byte +var z14005 [1 << 17]byte +var z14006 [1 << 17]byte +var z14007 [1 << 17]byte +var z14008 [1 << 17]byte +var z14009 [1 << 17]byte +var z14010 [1 << 17]byte +var z14011 [1 << 17]byte +var z14012 [1 << 17]byte +var z14013 [1 << 17]byte +var z14014 [1 << 17]byte +var z14015 [1 << 17]byte +var z14016 [1 << 17]byte +var z14017 [1 << 17]byte +var z14018 [1 << 17]byte +var z14019 [1 << 17]byte +var z14020 [1 << 17]byte +var z14021 [1 << 17]byte +var z14022 [1 << 17]byte +var z14023 [1 << 17]byte +var z14024 [1 << 17]byte +var z14025 [1 << 17]byte +var z14026 [1 << 17]byte +var z14027 [1 << 17]byte +var z14028 [1 << 17]byte +var z14029 [1 << 17]byte +var z14030 [1 << 17]byte +var z14031 [1 << 17]byte +var z14032 [1 << 17]byte +var z14033 [1 << 17]byte +var z14034 [1 << 17]byte +var z14035 [1 << 17]byte +var z14036 [1 << 17]byte +var z14037 [1 << 17]byte +var z14038 [1 << 17]byte +var z14039 [1 << 17]byte +var z14040 [1 << 17]byte +var z14041 [1 << 17]byte +var z14042 [1 << 17]byte +var z14043 [1 << 17]byte +var z14044 [1 << 17]byte +var z14045 [1 << 17]byte +var z14046 [1 << 17]byte +var z14047 [1 << 17]byte +var z14048 [1 << 17]byte +var z14049 [1 << 17]byte +var z14050 [1 << 17]byte +var z14051 [1 << 17]byte +var z14052 [1 << 17]byte +var z14053 [1 << 17]byte +var z14054 [1 << 17]byte +var z14055 [1 << 17]byte +var z14056 [1 << 17]byte +var z14057 [1 << 17]byte +var z14058 [1 << 17]byte +var z14059 [1 << 17]byte +var z14060 [1 << 17]byte +var z14061 [1 << 17]byte +var z14062 [1 << 17]byte +var z14063 [1 << 17]byte +var z14064 [1 << 17]byte +var z14065 [1 << 17]byte +var z14066 [1 << 17]byte +var z14067 [1 << 17]byte +var z14068 [1 << 17]byte +var z14069 [1 << 17]byte +var z14070 [1 << 17]byte +var z14071 [1 << 17]byte +var z14072 [1 << 17]byte +var z14073 [1 << 17]byte +var z14074 [1 << 17]byte +var z14075 [1 << 17]byte +var z14076 [1 << 17]byte +var z14077 [1 << 17]byte +var z14078 [1 << 17]byte +var z14079 [1 << 17]byte +var z14080 [1 << 17]byte +var z14081 [1 << 17]byte +var z14082 [1 << 17]byte +var z14083 [1 << 17]byte +var z14084 [1 << 17]byte +var z14085 [1 << 17]byte +var z14086 [1 << 17]byte +var z14087 [1 << 17]byte +var z14088 [1 << 17]byte +var z14089 [1 << 17]byte +var z14090 [1 << 17]byte +var z14091 [1 << 17]byte +var z14092 [1 << 17]byte +var z14093 [1 << 17]byte +var z14094 [1 << 17]byte +var z14095 [1 << 17]byte +var z14096 [1 << 17]byte +var z14097 [1 << 17]byte +var z14098 [1 << 17]byte +var z14099 [1 << 17]byte +var z14100 [1 << 17]byte +var z14101 [1 << 17]byte +var z14102 [1 << 17]byte +var z14103 [1 << 17]byte +var z14104 [1 << 17]byte +var z14105 [1 << 17]byte +var z14106 [1 << 17]byte +var z14107 [1 << 17]byte +var z14108 [1 << 17]byte +var z14109 [1 << 17]byte +var z14110 [1 << 17]byte +var z14111 [1 << 17]byte +var z14112 [1 << 17]byte +var z14113 [1 << 17]byte +var z14114 [1 << 17]byte +var z14115 [1 << 17]byte +var z14116 [1 << 17]byte +var z14117 [1 << 17]byte +var z14118 [1 << 17]byte +var z14119 [1 << 17]byte +var z14120 [1 << 17]byte +var z14121 [1 << 17]byte +var z14122 [1 << 17]byte +var z14123 [1 << 17]byte +var z14124 [1 << 17]byte +var z14125 [1 << 17]byte +var z14126 [1 << 17]byte +var z14127 [1 << 17]byte +var z14128 [1 << 17]byte +var z14129 [1 << 17]byte +var z14130 [1 << 17]byte +var z14131 [1 << 17]byte +var z14132 [1 << 17]byte +var z14133 [1 << 17]byte +var z14134 [1 << 17]byte +var z14135 [1 << 17]byte +var z14136 [1 << 17]byte +var z14137 [1 << 17]byte +var z14138 [1 << 17]byte +var z14139 [1 << 17]byte +var z14140 [1 << 17]byte +var z14141 [1 << 17]byte +var z14142 [1 << 17]byte +var z14143 [1 << 17]byte +var z14144 [1 << 17]byte +var z14145 [1 << 17]byte +var z14146 [1 << 17]byte +var z14147 [1 << 17]byte +var z14148 [1 << 17]byte +var z14149 [1 << 17]byte +var z14150 [1 << 17]byte +var z14151 [1 << 17]byte +var z14152 [1 << 17]byte +var z14153 [1 << 17]byte +var z14154 [1 << 17]byte +var z14155 [1 << 17]byte +var z14156 [1 << 17]byte +var z14157 [1 << 17]byte +var z14158 [1 << 17]byte +var z14159 [1 << 17]byte +var z14160 [1 << 17]byte +var z14161 [1 << 17]byte +var z14162 [1 << 17]byte +var z14163 [1 << 17]byte +var z14164 [1 << 17]byte +var z14165 [1 << 17]byte +var z14166 [1 << 17]byte +var z14167 [1 << 17]byte +var z14168 [1 << 17]byte +var z14169 [1 << 17]byte +var z14170 [1 << 17]byte +var z14171 [1 << 17]byte +var z14172 [1 << 17]byte +var z14173 [1 << 17]byte +var z14174 [1 << 17]byte +var z14175 [1 << 17]byte +var z14176 [1 << 17]byte +var z14177 [1 << 17]byte +var z14178 [1 << 17]byte +var z14179 [1 << 17]byte +var z14180 [1 << 17]byte +var z14181 [1 << 17]byte +var z14182 [1 << 17]byte +var z14183 [1 << 17]byte +var z14184 [1 << 17]byte +var z14185 [1 << 17]byte +var z14186 [1 << 17]byte +var z14187 [1 << 17]byte +var z14188 [1 << 17]byte +var z14189 [1 << 17]byte +var z14190 [1 << 17]byte +var z14191 [1 << 17]byte +var z14192 [1 << 17]byte +var z14193 [1 << 17]byte +var z14194 [1 << 17]byte +var z14195 [1 << 17]byte +var z14196 [1 << 17]byte +var z14197 [1 << 17]byte +var z14198 [1 << 17]byte +var z14199 [1 << 17]byte +var z14200 [1 << 17]byte +var z14201 [1 << 17]byte +var z14202 [1 << 17]byte +var z14203 [1 << 17]byte +var z14204 [1 << 17]byte +var z14205 [1 << 17]byte +var z14206 [1 << 17]byte +var z14207 [1 << 17]byte +var z14208 [1 << 17]byte +var z14209 [1 << 17]byte +var z14210 [1 << 17]byte +var z14211 [1 << 17]byte +var z14212 [1 << 17]byte +var z14213 [1 << 17]byte +var z14214 [1 << 17]byte +var z14215 [1 << 17]byte +var z14216 [1 << 17]byte +var z14217 [1 << 17]byte +var z14218 [1 << 17]byte +var z14219 [1 << 17]byte +var z14220 [1 << 17]byte +var z14221 [1 << 17]byte +var z14222 [1 << 17]byte +var z14223 [1 << 17]byte +var z14224 [1 << 17]byte +var z14225 [1 << 17]byte +var z14226 [1 << 17]byte +var z14227 [1 << 17]byte +var z14228 [1 << 17]byte +var z14229 [1 << 17]byte +var z14230 [1 << 17]byte +var z14231 [1 << 17]byte +var z14232 [1 << 17]byte +var z14233 [1 << 17]byte +var z14234 [1 << 17]byte +var z14235 [1 << 17]byte +var z14236 [1 << 17]byte +var z14237 [1 << 17]byte +var z14238 [1 << 17]byte +var z14239 [1 << 17]byte +var z14240 [1 << 17]byte +var z14241 [1 << 17]byte +var z14242 [1 << 17]byte +var z14243 [1 << 17]byte +var z14244 [1 << 17]byte +var z14245 [1 << 17]byte +var z14246 [1 << 17]byte +var z14247 [1 << 17]byte +var z14248 [1 << 17]byte +var z14249 [1 << 17]byte +var z14250 [1 << 17]byte +var z14251 [1 << 17]byte +var z14252 [1 << 17]byte +var z14253 [1 << 17]byte +var z14254 [1 << 17]byte +var z14255 [1 << 17]byte +var z14256 [1 << 17]byte +var z14257 [1 << 17]byte +var z14258 [1 << 17]byte +var z14259 [1 << 17]byte +var z14260 [1 << 17]byte +var z14261 [1 << 17]byte +var z14262 [1 << 17]byte +var z14263 [1 << 17]byte +var z14264 [1 << 17]byte +var z14265 [1 << 17]byte +var z14266 [1 << 17]byte +var z14267 [1 << 17]byte +var z14268 [1 << 17]byte +var z14269 [1 << 17]byte +var z14270 [1 << 17]byte +var z14271 [1 << 17]byte +var z14272 [1 << 17]byte +var z14273 [1 << 17]byte +var z14274 [1 << 17]byte +var z14275 [1 << 17]byte +var z14276 [1 << 17]byte +var z14277 [1 << 17]byte +var z14278 [1 << 17]byte +var z14279 [1 << 17]byte +var z14280 [1 << 17]byte +var z14281 [1 << 17]byte +var z14282 [1 << 17]byte +var z14283 [1 << 17]byte +var z14284 [1 << 17]byte +var z14285 [1 << 17]byte +var z14286 [1 << 17]byte +var z14287 [1 << 17]byte +var z14288 [1 << 17]byte +var z14289 [1 << 17]byte +var z14290 [1 << 17]byte +var z14291 [1 << 17]byte +var z14292 [1 << 17]byte +var z14293 [1 << 17]byte +var z14294 [1 << 17]byte +var z14295 [1 << 17]byte +var z14296 [1 << 17]byte +var z14297 [1 << 17]byte +var z14298 [1 << 17]byte +var z14299 [1 << 17]byte +var z14300 [1 << 17]byte +var z14301 [1 << 17]byte +var z14302 [1 << 17]byte +var z14303 [1 << 17]byte +var z14304 [1 << 17]byte +var z14305 [1 << 17]byte +var z14306 [1 << 17]byte +var z14307 [1 << 17]byte +var z14308 [1 << 17]byte +var z14309 [1 << 17]byte +var z14310 [1 << 17]byte +var z14311 [1 << 17]byte +var z14312 [1 << 17]byte +var z14313 [1 << 17]byte +var z14314 [1 << 17]byte +var z14315 [1 << 17]byte +var z14316 [1 << 17]byte +var z14317 [1 << 17]byte +var z14318 [1 << 17]byte +var z14319 [1 << 17]byte +var z14320 [1 << 17]byte +var z14321 [1 << 17]byte +var z14322 [1 << 17]byte +var z14323 [1 << 17]byte +var z14324 [1 << 17]byte +var z14325 [1 << 17]byte +var z14326 [1 << 17]byte +var z14327 [1 << 17]byte +var z14328 [1 << 17]byte +var z14329 [1 << 17]byte +var z14330 [1 << 17]byte +var z14331 [1 << 17]byte +var z14332 [1 << 17]byte +var z14333 [1 << 17]byte +var z14334 [1 << 17]byte +var z14335 [1 << 17]byte +var z14336 [1 << 17]byte +var z14337 [1 << 17]byte +var z14338 [1 << 17]byte +var z14339 [1 << 17]byte +var z14340 [1 << 17]byte +var z14341 [1 << 17]byte +var z14342 [1 << 17]byte +var z14343 [1 << 17]byte +var z14344 [1 << 17]byte +var z14345 [1 << 17]byte +var z14346 [1 << 17]byte +var z14347 [1 << 17]byte +var z14348 [1 << 17]byte +var z14349 [1 << 17]byte +var z14350 [1 << 17]byte +var z14351 [1 << 17]byte +var z14352 [1 << 17]byte +var z14353 [1 << 17]byte +var z14354 [1 << 17]byte +var z14355 [1 << 17]byte +var z14356 [1 << 17]byte +var z14357 [1 << 17]byte +var z14358 [1 << 17]byte +var z14359 [1 << 17]byte +var z14360 [1 << 17]byte +var z14361 [1 << 17]byte +var z14362 [1 << 17]byte +var z14363 [1 << 17]byte +var z14364 [1 << 17]byte +var z14365 [1 << 17]byte +var z14366 [1 << 17]byte +var z14367 [1 << 17]byte +var z14368 [1 << 17]byte +var z14369 [1 << 17]byte +var z14370 [1 << 17]byte +var z14371 [1 << 17]byte +var z14372 [1 << 17]byte +var z14373 [1 << 17]byte +var z14374 [1 << 17]byte +var z14375 [1 << 17]byte +var z14376 [1 << 17]byte +var z14377 [1 << 17]byte +var z14378 [1 << 17]byte +var z14379 [1 << 17]byte +var z14380 [1 << 17]byte +var z14381 [1 << 17]byte +var z14382 [1 << 17]byte +var z14383 [1 << 17]byte +var z14384 [1 << 17]byte +var z14385 [1 << 17]byte +var z14386 [1 << 17]byte +var z14387 [1 << 17]byte +var z14388 [1 << 17]byte +var z14389 [1 << 17]byte +var z14390 [1 << 17]byte +var z14391 [1 << 17]byte +var z14392 [1 << 17]byte +var z14393 [1 << 17]byte +var z14394 [1 << 17]byte +var z14395 [1 << 17]byte +var z14396 [1 << 17]byte +var z14397 [1 << 17]byte +var z14398 [1 << 17]byte +var z14399 [1 << 17]byte +var z14400 [1 << 17]byte +var z14401 [1 << 17]byte +var z14402 [1 << 17]byte +var z14403 [1 << 17]byte +var z14404 [1 << 17]byte +var z14405 [1 << 17]byte +var z14406 [1 << 17]byte +var z14407 [1 << 17]byte +var z14408 [1 << 17]byte +var z14409 [1 << 17]byte +var z14410 [1 << 17]byte +var z14411 [1 << 17]byte +var z14412 [1 << 17]byte +var z14413 [1 << 17]byte +var z14414 [1 << 17]byte +var z14415 [1 << 17]byte +var z14416 [1 << 17]byte +var z14417 [1 << 17]byte +var z14418 [1 << 17]byte +var z14419 [1 << 17]byte +var z14420 [1 << 17]byte +var z14421 [1 << 17]byte +var z14422 [1 << 17]byte +var z14423 [1 << 17]byte +var z14424 [1 << 17]byte +var z14425 [1 << 17]byte +var z14426 [1 << 17]byte +var z14427 [1 << 17]byte +var z14428 [1 << 17]byte +var z14429 [1 << 17]byte +var z14430 [1 << 17]byte +var z14431 [1 << 17]byte +var z14432 [1 << 17]byte +var z14433 [1 << 17]byte +var z14434 [1 << 17]byte +var z14435 [1 << 17]byte +var z14436 [1 << 17]byte +var z14437 [1 << 17]byte +var z14438 [1 << 17]byte +var z14439 [1 << 17]byte +var z14440 [1 << 17]byte +var z14441 [1 << 17]byte +var z14442 [1 << 17]byte +var z14443 [1 << 17]byte +var z14444 [1 << 17]byte +var z14445 [1 << 17]byte +var z14446 [1 << 17]byte +var z14447 [1 << 17]byte +var z14448 [1 << 17]byte +var z14449 [1 << 17]byte +var z14450 [1 << 17]byte +var z14451 [1 << 17]byte +var z14452 [1 << 17]byte +var z14453 [1 << 17]byte +var z14454 [1 << 17]byte +var z14455 [1 << 17]byte +var z14456 [1 << 17]byte +var z14457 [1 << 17]byte +var z14458 [1 << 17]byte +var z14459 [1 << 17]byte +var z14460 [1 << 17]byte +var z14461 [1 << 17]byte +var z14462 [1 << 17]byte +var z14463 [1 << 17]byte +var z14464 [1 << 17]byte +var z14465 [1 << 17]byte +var z14466 [1 << 17]byte +var z14467 [1 << 17]byte +var z14468 [1 << 17]byte +var z14469 [1 << 17]byte +var z14470 [1 << 17]byte +var z14471 [1 << 17]byte +var z14472 [1 << 17]byte +var z14473 [1 << 17]byte +var z14474 [1 << 17]byte +var z14475 [1 << 17]byte +var z14476 [1 << 17]byte +var z14477 [1 << 17]byte +var z14478 [1 << 17]byte +var z14479 [1 << 17]byte +var z14480 [1 << 17]byte +var z14481 [1 << 17]byte +var z14482 [1 << 17]byte +var z14483 [1 << 17]byte +var z14484 [1 << 17]byte +var z14485 [1 << 17]byte +var z14486 [1 << 17]byte +var z14487 [1 << 17]byte +var z14488 [1 << 17]byte +var z14489 [1 << 17]byte +var z14490 [1 << 17]byte +var z14491 [1 << 17]byte +var z14492 [1 << 17]byte +var z14493 [1 << 17]byte +var z14494 [1 << 17]byte +var z14495 [1 << 17]byte +var z14496 [1 << 17]byte +var z14497 [1 << 17]byte +var z14498 [1 << 17]byte +var z14499 [1 << 17]byte +var z14500 [1 << 17]byte +var z14501 [1 << 17]byte +var z14502 [1 << 17]byte +var z14503 [1 << 17]byte +var z14504 [1 << 17]byte +var z14505 [1 << 17]byte +var z14506 [1 << 17]byte +var z14507 [1 << 17]byte +var z14508 [1 << 17]byte +var z14509 [1 << 17]byte +var z14510 [1 << 17]byte +var z14511 [1 << 17]byte +var z14512 [1 << 17]byte +var z14513 [1 << 17]byte +var z14514 [1 << 17]byte +var z14515 [1 << 17]byte +var z14516 [1 << 17]byte +var z14517 [1 << 17]byte +var z14518 [1 << 17]byte +var z14519 [1 << 17]byte +var z14520 [1 << 17]byte +var z14521 [1 << 17]byte +var z14522 [1 << 17]byte +var z14523 [1 << 17]byte +var z14524 [1 << 17]byte +var z14525 [1 << 17]byte +var z14526 [1 << 17]byte +var z14527 [1 << 17]byte +var z14528 [1 << 17]byte +var z14529 [1 << 17]byte +var z14530 [1 << 17]byte +var z14531 [1 << 17]byte +var z14532 [1 << 17]byte +var z14533 [1 << 17]byte +var z14534 [1 << 17]byte +var z14535 [1 << 17]byte +var z14536 [1 << 17]byte +var z14537 [1 << 17]byte +var z14538 [1 << 17]byte +var z14539 [1 << 17]byte +var z14540 [1 << 17]byte +var z14541 [1 << 17]byte +var z14542 [1 << 17]byte +var z14543 [1 << 17]byte +var z14544 [1 << 17]byte +var z14545 [1 << 17]byte +var z14546 [1 << 17]byte +var z14547 [1 << 17]byte +var z14548 [1 << 17]byte +var z14549 [1 << 17]byte +var z14550 [1 << 17]byte +var z14551 [1 << 17]byte +var z14552 [1 << 17]byte +var z14553 [1 << 17]byte +var z14554 [1 << 17]byte +var z14555 [1 << 17]byte +var z14556 [1 << 17]byte +var z14557 [1 << 17]byte +var z14558 [1 << 17]byte +var z14559 [1 << 17]byte +var z14560 [1 << 17]byte +var z14561 [1 << 17]byte +var z14562 [1 << 17]byte +var z14563 [1 << 17]byte +var z14564 [1 << 17]byte +var z14565 [1 << 17]byte +var z14566 [1 << 17]byte +var z14567 [1 << 17]byte +var z14568 [1 << 17]byte +var z14569 [1 << 17]byte +var z14570 [1 << 17]byte +var z14571 [1 << 17]byte +var z14572 [1 << 17]byte +var z14573 [1 << 17]byte +var z14574 [1 << 17]byte +var z14575 [1 << 17]byte +var z14576 [1 << 17]byte +var z14577 [1 << 17]byte +var z14578 [1 << 17]byte +var z14579 [1 << 17]byte +var z14580 [1 << 17]byte +var z14581 [1 << 17]byte +var z14582 [1 << 17]byte +var z14583 [1 << 17]byte +var z14584 [1 << 17]byte +var z14585 [1 << 17]byte +var z14586 [1 << 17]byte +var z14587 [1 << 17]byte +var z14588 [1 << 17]byte +var z14589 [1 << 17]byte +var z14590 [1 << 17]byte +var z14591 [1 << 17]byte +var z14592 [1 << 17]byte +var z14593 [1 << 17]byte +var z14594 [1 << 17]byte +var z14595 [1 << 17]byte +var z14596 [1 << 17]byte +var z14597 [1 << 17]byte +var z14598 [1 << 17]byte +var z14599 [1 << 17]byte +var z14600 [1 << 17]byte +var z14601 [1 << 17]byte +var z14602 [1 << 17]byte +var z14603 [1 << 17]byte +var z14604 [1 << 17]byte +var z14605 [1 << 17]byte +var z14606 [1 << 17]byte +var z14607 [1 << 17]byte +var z14608 [1 << 17]byte +var z14609 [1 << 17]byte +var z14610 [1 << 17]byte +var z14611 [1 << 17]byte +var z14612 [1 << 17]byte +var z14613 [1 << 17]byte +var z14614 [1 << 17]byte +var z14615 [1 << 17]byte +var z14616 [1 << 17]byte +var z14617 [1 << 17]byte +var z14618 [1 << 17]byte +var z14619 [1 << 17]byte +var z14620 [1 << 17]byte +var z14621 [1 << 17]byte +var z14622 [1 << 17]byte +var z14623 [1 << 17]byte +var z14624 [1 << 17]byte +var z14625 [1 << 17]byte +var z14626 [1 << 17]byte +var z14627 [1 << 17]byte +var z14628 [1 << 17]byte +var z14629 [1 << 17]byte +var z14630 [1 << 17]byte +var z14631 [1 << 17]byte +var z14632 [1 << 17]byte +var z14633 [1 << 17]byte +var z14634 [1 << 17]byte +var z14635 [1 << 17]byte +var z14636 [1 << 17]byte +var z14637 [1 << 17]byte +var z14638 [1 << 17]byte +var z14639 [1 << 17]byte +var z14640 [1 << 17]byte +var z14641 [1 << 17]byte +var z14642 [1 << 17]byte +var z14643 [1 << 17]byte +var z14644 [1 << 17]byte +var z14645 [1 << 17]byte +var z14646 [1 << 17]byte +var z14647 [1 << 17]byte +var z14648 [1 << 17]byte +var z14649 [1 << 17]byte +var z14650 [1 << 17]byte +var z14651 [1 << 17]byte +var z14652 [1 << 17]byte +var z14653 [1 << 17]byte +var z14654 [1 << 17]byte +var z14655 [1 << 17]byte +var z14656 [1 << 17]byte +var z14657 [1 << 17]byte +var z14658 [1 << 17]byte +var z14659 [1 << 17]byte +var z14660 [1 << 17]byte +var z14661 [1 << 17]byte +var z14662 [1 << 17]byte +var z14663 [1 << 17]byte +var z14664 [1 << 17]byte +var z14665 [1 << 17]byte +var z14666 [1 << 17]byte +var z14667 [1 << 17]byte +var z14668 [1 << 17]byte +var z14669 [1 << 17]byte +var z14670 [1 << 17]byte +var z14671 [1 << 17]byte +var z14672 [1 << 17]byte +var z14673 [1 << 17]byte +var z14674 [1 << 17]byte +var z14675 [1 << 17]byte +var z14676 [1 << 17]byte +var z14677 [1 << 17]byte +var z14678 [1 << 17]byte +var z14679 [1 << 17]byte +var z14680 [1 << 17]byte +var z14681 [1 << 17]byte +var z14682 [1 << 17]byte +var z14683 [1 << 17]byte +var z14684 [1 << 17]byte +var z14685 [1 << 17]byte +var z14686 [1 << 17]byte +var z14687 [1 << 17]byte +var z14688 [1 << 17]byte +var z14689 [1 << 17]byte +var z14690 [1 << 17]byte +var z14691 [1 << 17]byte +var z14692 [1 << 17]byte +var z14693 [1 << 17]byte +var z14694 [1 << 17]byte +var z14695 [1 << 17]byte +var z14696 [1 << 17]byte +var z14697 [1 << 17]byte +var z14698 [1 << 17]byte +var z14699 [1 << 17]byte +var z14700 [1 << 17]byte +var z14701 [1 << 17]byte +var z14702 [1 << 17]byte +var z14703 [1 << 17]byte +var z14704 [1 << 17]byte +var z14705 [1 << 17]byte +var z14706 [1 << 17]byte +var z14707 [1 << 17]byte +var z14708 [1 << 17]byte +var z14709 [1 << 17]byte +var z14710 [1 << 17]byte +var z14711 [1 << 17]byte +var z14712 [1 << 17]byte +var z14713 [1 << 17]byte +var z14714 [1 << 17]byte +var z14715 [1 << 17]byte +var z14716 [1 << 17]byte +var z14717 [1 << 17]byte +var z14718 [1 << 17]byte +var z14719 [1 << 17]byte +var z14720 [1 << 17]byte +var z14721 [1 << 17]byte +var z14722 [1 << 17]byte +var z14723 [1 << 17]byte +var z14724 [1 << 17]byte +var z14725 [1 << 17]byte +var z14726 [1 << 17]byte +var z14727 [1 << 17]byte +var z14728 [1 << 17]byte +var z14729 [1 << 17]byte +var z14730 [1 << 17]byte +var z14731 [1 << 17]byte +var z14732 [1 << 17]byte +var z14733 [1 << 17]byte +var z14734 [1 << 17]byte +var z14735 [1 << 17]byte +var z14736 [1 << 17]byte +var z14737 [1 << 17]byte +var z14738 [1 << 17]byte +var z14739 [1 << 17]byte +var z14740 [1 << 17]byte +var z14741 [1 << 17]byte +var z14742 [1 << 17]byte +var z14743 [1 << 17]byte +var z14744 [1 << 17]byte +var z14745 [1 << 17]byte +var z14746 [1 << 17]byte +var z14747 [1 << 17]byte +var z14748 [1 << 17]byte +var z14749 [1 << 17]byte +var z14750 [1 << 17]byte +var z14751 [1 << 17]byte +var z14752 [1 << 17]byte +var z14753 [1 << 17]byte +var z14754 [1 << 17]byte +var z14755 [1 << 17]byte +var z14756 [1 << 17]byte +var z14757 [1 << 17]byte +var z14758 [1 << 17]byte +var z14759 [1 << 17]byte +var z14760 [1 << 17]byte +var z14761 [1 << 17]byte +var z14762 [1 << 17]byte +var z14763 [1 << 17]byte +var z14764 [1 << 17]byte +var z14765 [1 << 17]byte +var z14766 [1 << 17]byte +var z14767 [1 << 17]byte +var z14768 [1 << 17]byte +var z14769 [1 << 17]byte +var z14770 [1 << 17]byte +var z14771 [1 << 17]byte +var z14772 [1 << 17]byte +var z14773 [1 << 17]byte +var z14774 [1 << 17]byte +var z14775 [1 << 17]byte +var z14776 [1 << 17]byte +var z14777 [1 << 17]byte +var z14778 [1 << 17]byte +var z14779 [1 << 17]byte +var z14780 [1 << 17]byte +var z14781 [1 << 17]byte +var z14782 [1 << 17]byte +var z14783 [1 << 17]byte +var z14784 [1 << 17]byte +var z14785 [1 << 17]byte +var z14786 [1 << 17]byte +var z14787 [1 << 17]byte +var z14788 [1 << 17]byte +var z14789 [1 << 17]byte +var z14790 [1 << 17]byte +var z14791 [1 << 17]byte +var z14792 [1 << 17]byte +var z14793 [1 << 17]byte +var z14794 [1 << 17]byte +var z14795 [1 << 17]byte +var z14796 [1 << 17]byte +var z14797 [1 << 17]byte +var z14798 [1 << 17]byte +var z14799 [1 << 17]byte +var z14800 [1 << 17]byte +var z14801 [1 << 17]byte +var z14802 [1 << 17]byte +var z14803 [1 << 17]byte +var z14804 [1 << 17]byte +var z14805 [1 << 17]byte +var z14806 [1 << 17]byte +var z14807 [1 << 17]byte +var z14808 [1 << 17]byte +var z14809 [1 << 17]byte +var z14810 [1 << 17]byte +var z14811 [1 << 17]byte +var z14812 [1 << 17]byte +var z14813 [1 << 17]byte +var z14814 [1 << 17]byte +var z14815 [1 << 17]byte +var z14816 [1 << 17]byte +var z14817 [1 << 17]byte +var z14818 [1 << 17]byte +var z14819 [1 << 17]byte +var z14820 [1 << 17]byte +var z14821 [1 << 17]byte +var z14822 [1 << 17]byte +var z14823 [1 << 17]byte +var z14824 [1 << 17]byte +var z14825 [1 << 17]byte +var z14826 [1 << 17]byte +var z14827 [1 << 17]byte +var z14828 [1 << 17]byte +var z14829 [1 << 17]byte +var z14830 [1 << 17]byte +var z14831 [1 << 17]byte +var z14832 [1 << 17]byte +var z14833 [1 << 17]byte +var z14834 [1 << 17]byte +var z14835 [1 << 17]byte +var z14836 [1 << 17]byte +var z14837 [1 << 17]byte +var z14838 [1 << 17]byte +var z14839 [1 << 17]byte +var z14840 [1 << 17]byte +var z14841 [1 << 17]byte +var z14842 [1 << 17]byte +var z14843 [1 << 17]byte +var z14844 [1 << 17]byte +var z14845 [1 << 17]byte +var z14846 [1 << 17]byte +var z14847 [1 << 17]byte +var z14848 [1 << 17]byte +var z14849 [1 << 17]byte +var z14850 [1 << 17]byte +var z14851 [1 << 17]byte +var z14852 [1 << 17]byte +var z14853 [1 << 17]byte +var z14854 [1 << 17]byte +var z14855 [1 << 17]byte +var z14856 [1 << 17]byte +var z14857 [1 << 17]byte +var z14858 [1 << 17]byte +var z14859 [1 << 17]byte +var z14860 [1 << 17]byte +var z14861 [1 << 17]byte +var z14862 [1 << 17]byte +var z14863 [1 << 17]byte +var z14864 [1 << 17]byte +var z14865 [1 << 17]byte +var z14866 [1 << 17]byte +var z14867 [1 << 17]byte +var z14868 [1 << 17]byte +var z14869 [1 << 17]byte +var z14870 [1 << 17]byte +var z14871 [1 << 17]byte +var z14872 [1 << 17]byte +var z14873 [1 << 17]byte +var z14874 [1 << 17]byte +var z14875 [1 << 17]byte +var z14876 [1 << 17]byte +var z14877 [1 << 17]byte +var z14878 [1 << 17]byte +var z14879 [1 << 17]byte +var z14880 [1 << 17]byte +var z14881 [1 << 17]byte +var z14882 [1 << 17]byte +var z14883 [1 << 17]byte +var z14884 [1 << 17]byte +var z14885 [1 << 17]byte +var z14886 [1 << 17]byte +var z14887 [1 << 17]byte +var z14888 [1 << 17]byte +var z14889 [1 << 17]byte +var z14890 [1 << 17]byte +var z14891 [1 << 17]byte +var z14892 [1 << 17]byte +var z14893 [1 << 17]byte +var z14894 [1 << 17]byte +var z14895 [1 << 17]byte +var z14896 [1 << 17]byte +var z14897 [1 << 17]byte +var z14898 [1 << 17]byte +var z14899 [1 << 17]byte +var z14900 [1 << 17]byte +var z14901 [1 << 17]byte +var z14902 [1 << 17]byte +var z14903 [1 << 17]byte +var z14904 [1 << 17]byte +var z14905 [1 << 17]byte +var z14906 [1 << 17]byte +var z14907 [1 << 17]byte +var z14908 [1 << 17]byte +var z14909 [1 << 17]byte +var z14910 [1 << 17]byte +var z14911 [1 << 17]byte +var z14912 [1 << 17]byte +var z14913 [1 << 17]byte +var z14914 [1 << 17]byte +var z14915 [1 << 17]byte +var z14916 [1 << 17]byte +var z14917 [1 << 17]byte +var z14918 [1 << 17]byte +var z14919 [1 << 17]byte +var z14920 [1 << 17]byte +var z14921 [1 << 17]byte +var z14922 [1 << 17]byte +var z14923 [1 << 17]byte +var z14924 [1 << 17]byte +var z14925 [1 << 17]byte +var z14926 [1 << 17]byte +var z14927 [1 << 17]byte +var z14928 [1 << 17]byte +var z14929 [1 << 17]byte +var z14930 [1 << 17]byte +var z14931 [1 << 17]byte +var z14932 [1 << 17]byte +var z14933 [1 << 17]byte +var z14934 [1 << 17]byte +var z14935 [1 << 17]byte +var z14936 [1 << 17]byte +var z14937 [1 << 17]byte +var z14938 [1 << 17]byte +var z14939 [1 << 17]byte +var z14940 [1 << 17]byte +var z14941 [1 << 17]byte +var z14942 [1 << 17]byte +var z14943 [1 << 17]byte +var z14944 [1 << 17]byte +var z14945 [1 << 17]byte +var z14946 [1 << 17]byte +var z14947 [1 << 17]byte +var z14948 [1 << 17]byte +var z14949 [1 << 17]byte +var z14950 [1 << 17]byte +var z14951 [1 << 17]byte +var z14952 [1 << 17]byte +var z14953 [1 << 17]byte +var z14954 [1 << 17]byte +var z14955 [1 << 17]byte +var z14956 [1 << 17]byte +var z14957 [1 << 17]byte +var z14958 [1 << 17]byte +var z14959 [1 << 17]byte +var z14960 [1 << 17]byte +var z14961 [1 << 17]byte +var z14962 [1 << 17]byte +var z14963 [1 << 17]byte +var z14964 [1 << 17]byte +var z14965 [1 << 17]byte +var z14966 [1 << 17]byte +var z14967 [1 << 17]byte +var z14968 [1 << 17]byte +var z14969 [1 << 17]byte +var z14970 [1 << 17]byte +var z14971 [1 << 17]byte +var z14972 [1 << 17]byte +var z14973 [1 << 17]byte +var z14974 [1 << 17]byte +var z14975 [1 << 17]byte +var z14976 [1 << 17]byte +var z14977 [1 << 17]byte +var z14978 [1 << 17]byte +var z14979 [1 << 17]byte +var z14980 [1 << 17]byte +var z14981 [1 << 17]byte +var z14982 [1 << 17]byte +var z14983 [1 << 17]byte +var z14984 [1 << 17]byte +var z14985 [1 << 17]byte +var z14986 [1 << 17]byte +var z14987 [1 << 17]byte +var z14988 [1 << 17]byte +var z14989 [1 << 17]byte +var z14990 [1 << 17]byte +var z14991 [1 << 17]byte +var z14992 [1 << 17]byte +var z14993 [1 << 17]byte +var z14994 [1 << 17]byte +var z14995 [1 << 17]byte +var z14996 [1 << 17]byte +var z14997 [1 << 17]byte +var z14998 [1 << 17]byte +var z14999 [1 << 17]byte +var z15000 [1 << 17]byte +var z15001 [1 << 17]byte +var z15002 [1 << 17]byte +var z15003 [1 << 17]byte +var z15004 [1 << 17]byte +var z15005 [1 << 17]byte +var z15006 [1 << 17]byte +var z15007 [1 << 17]byte +var z15008 [1 << 17]byte +var z15009 [1 << 17]byte +var z15010 [1 << 17]byte +var z15011 [1 << 17]byte +var z15012 [1 << 17]byte +var z15013 [1 << 17]byte +var z15014 [1 << 17]byte +var z15015 [1 << 17]byte +var z15016 [1 << 17]byte +var z15017 [1 << 17]byte +var z15018 [1 << 17]byte +var z15019 [1 << 17]byte +var z15020 [1 << 17]byte +var z15021 [1 << 17]byte +var z15022 [1 << 17]byte +var z15023 [1 << 17]byte +var z15024 [1 << 17]byte +var z15025 [1 << 17]byte +var z15026 [1 << 17]byte +var z15027 [1 << 17]byte +var z15028 [1 << 17]byte +var z15029 [1 << 17]byte +var z15030 [1 << 17]byte +var z15031 [1 << 17]byte +var z15032 [1 << 17]byte +var z15033 [1 << 17]byte +var z15034 [1 << 17]byte +var z15035 [1 << 17]byte +var z15036 [1 << 17]byte +var z15037 [1 << 17]byte +var z15038 [1 << 17]byte +var z15039 [1 << 17]byte +var z15040 [1 << 17]byte +var z15041 [1 << 17]byte +var z15042 [1 << 17]byte +var z15043 [1 << 17]byte +var z15044 [1 << 17]byte +var z15045 [1 << 17]byte +var z15046 [1 << 17]byte +var z15047 [1 << 17]byte +var z15048 [1 << 17]byte +var z15049 [1 << 17]byte +var z15050 [1 << 17]byte +var z15051 [1 << 17]byte +var z15052 [1 << 17]byte +var z15053 [1 << 17]byte +var z15054 [1 << 17]byte +var z15055 [1 << 17]byte +var z15056 [1 << 17]byte +var z15057 [1 << 17]byte +var z15058 [1 << 17]byte +var z15059 [1 << 17]byte +var z15060 [1 << 17]byte +var z15061 [1 << 17]byte +var z15062 [1 << 17]byte +var z15063 [1 << 17]byte +var z15064 [1 << 17]byte +var z15065 [1 << 17]byte +var z15066 [1 << 17]byte +var z15067 [1 << 17]byte +var z15068 [1 << 17]byte +var z15069 [1 << 17]byte +var z15070 [1 << 17]byte +var z15071 [1 << 17]byte +var z15072 [1 << 17]byte +var z15073 [1 << 17]byte +var z15074 [1 << 17]byte +var z15075 [1 << 17]byte +var z15076 [1 << 17]byte +var z15077 [1 << 17]byte +var z15078 [1 << 17]byte +var z15079 [1 << 17]byte +var z15080 [1 << 17]byte +var z15081 [1 << 17]byte +var z15082 [1 << 17]byte +var z15083 [1 << 17]byte +var z15084 [1 << 17]byte +var z15085 [1 << 17]byte +var z15086 [1 << 17]byte +var z15087 [1 << 17]byte +var z15088 [1 << 17]byte +var z15089 [1 << 17]byte +var z15090 [1 << 17]byte +var z15091 [1 << 17]byte +var z15092 [1 << 17]byte +var z15093 [1 << 17]byte +var z15094 [1 << 17]byte +var z15095 [1 << 17]byte +var z15096 [1 << 17]byte +var z15097 [1 << 17]byte +var z15098 [1 << 17]byte +var z15099 [1 << 17]byte +var z15100 [1 << 17]byte +var z15101 [1 << 17]byte +var z15102 [1 << 17]byte +var z15103 [1 << 17]byte +var z15104 [1 << 17]byte +var z15105 [1 << 17]byte +var z15106 [1 << 17]byte +var z15107 [1 << 17]byte +var z15108 [1 << 17]byte +var z15109 [1 << 17]byte +var z15110 [1 << 17]byte +var z15111 [1 << 17]byte +var z15112 [1 << 17]byte +var z15113 [1 << 17]byte +var z15114 [1 << 17]byte +var z15115 [1 << 17]byte +var z15116 [1 << 17]byte +var z15117 [1 << 17]byte +var z15118 [1 << 17]byte +var z15119 [1 << 17]byte +var z15120 [1 << 17]byte +var z15121 [1 << 17]byte +var z15122 [1 << 17]byte +var z15123 [1 << 17]byte +var z15124 [1 << 17]byte +var z15125 [1 << 17]byte +var z15126 [1 << 17]byte +var z15127 [1 << 17]byte +var z15128 [1 << 17]byte +var z15129 [1 << 17]byte +var z15130 [1 << 17]byte +var z15131 [1 << 17]byte +var z15132 [1 << 17]byte +var z15133 [1 << 17]byte +var z15134 [1 << 17]byte +var z15135 [1 << 17]byte +var z15136 [1 << 17]byte +var z15137 [1 << 17]byte +var z15138 [1 << 17]byte +var z15139 [1 << 17]byte +var z15140 [1 << 17]byte +var z15141 [1 << 17]byte +var z15142 [1 << 17]byte +var z15143 [1 << 17]byte +var z15144 [1 << 17]byte +var z15145 [1 << 17]byte +var z15146 [1 << 17]byte +var z15147 [1 << 17]byte +var z15148 [1 << 17]byte +var z15149 [1 << 17]byte +var z15150 [1 << 17]byte +var z15151 [1 << 17]byte +var z15152 [1 << 17]byte +var z15153 [1 << 17]byte +var z15154 [1 << 17]byte +var z15155 [1 << 17]byte +var z15156 [1 << 17]byte +var z15157 [1 << 17]byte +var z15158 [1 << 17]byte +var z15159 [1 << 17]byte +var z15160 [1 << 17]byte +var z15161 [1 << 17]byte +var z15162 [1 << 17]byte +var z15163 [1 << 17]byte +var z15164 [1 << 17]byte +var z15165 [1 << 17]byte +var z15166 [1 << 17]byte +var z15167 [1 << 17]byte +var z15168 [1 << 17]byte +var z15169 [1 << 17]byte +var z15170 [1 << 17]byte +var z15171 [1 << 17]byte +var z15172 [1 << 17]byte +var z15173 [1 << 17]byte +var z15174 [1 << 17]byte +var z15175 [1 << 17]byte +var z15176 [1 << 17]byte +var z15177 [1 << 17]byte +var z15178 [1 << 17]byte +var z15179 [1 << 17]byte +var z15180 [1 << 17]byte +var z15181 [1 << 17]byte +var z15182 [1 << 17]byte +var z15183 [1 << 17]byte +var z15184 [1 << 17]byte +var z15185 [1 << 17]byte +var z15186 [1 << 17]byte +var z15187 [1 << 17]byte +var z15188 [1 << 17]byte +var z15189 [1 << 17]byte +var z15190 [1 << 17]byte +var z15191 [1 << 17]byte +var z15192 [1 << 17]byte +var z15193 [1 << 17]byte +var z15194 [1 << 17]byte +var z15195 [1 << 17]byte +var z15196 [1 << 17]byte +var z15197 [1 << 17]byte +var z15198 [1 << 17]byte +var z15199 [1 << 17]byte +var z15200 [1 << 17]byte +var z15201 [1 << 17]byte +var z15202 [1 << 17]byte +var z15203 [1 << 17]byte +var z15204 [1 << 17]byte +var z15205 [1 << 17]byte +var z15206 [1 << 17]byte +var z15207 [1 << 17]byte +var z15208 [1 << 17]byte +var z15209 [1 << 17]byte +var z15210 [1 << 17]byte +var z15211 [1 << 17]byte +var z15212 [1 << 17]byte +var z15213 [1 << 17]byte +var z15214 [1 << 17]byte +var z15215 [1 << 17]byte +var z15216 [1 << 17]byte +var z15217 [1 << 17]byte +var z15218 [1 << 17]byte +var z15219 [1 << 17]byte +var z15220 [1 << 17]byte +var z15221 [1 << 17]byte +var z15222 [1 << 17]byte +var z15223 [1 << 17]byte +var z15224 [1 << 17]byte +var z15225 [1 << 17]byte +var z15226 [1 << 17]byte +var z15227 [1 << 17]byte +var z15228 [1 << 17]byte +var z15229 [1 << 17]byte +var z15230 [1 << 17]byte +var z15231 [1 << 17]byte +var z15232 [1 << 17]byte +var z15233 [1 << 17]byte +var z15234 [1 << 17]byte +var z15235 [1 << 17]byte +var z15236 [1 << 17]byte +var z15237 [1 << 17]byte +var z15238 [1 << 17]byte +var z15239 [1 << 17]byte +var z15240 [1 << 17]byte +var z15241 [1 << 17]byte +var z15242 [1 << 17]byte +var z15243 [1 << 17]byte +var z15244 [1 << 17]byte +var z15245 [1 << 17]byte +var z15246 [1 << 17]byte +var z15247 [1 << 17]byte +var z15248 [1 << 17]byte +var z15249 [1 << 17]byte +var z15250 [1 << 17]byte +var z15251 [1 << 17]byte +var z15252 [1 << 17]byte +var z15253 [1 << 17]byte +var z15254 [1 << 17]byte +var z15255 [1 << 17]byte +var z15256 [1 << 17]byte +var z15257 [1 << 17]byte +var z15258 [1 << 17]byte +var z15259 [1 << 17]byte +var z15260 [1 << 17]byte +var z15261 [1 << 17]byte +var z15262 [1 << 17]byte +var z15263 [1 << 17]byte +var z15264 [1 << 17]byte +var z15265 [1 << 17]byte +var z15266 [1 << 17]byte +var z15267 [1 << 17]byte +var z15268 [1 << 17]byte +var z15269 [1 << 17]byte +var z15270 [1 << 17]byte +var z15271 [1 << 17]byte +var z15272 [1 << 17]byte +var z15273 [1 << 17]byte +var z15274 [1 << 17]byte +var z15275 [1 << 17]byte +var z15276 [1 << 17]byte +var z15277 [1 << 17]byte +var z15278 [1 << 17]byte +var z15279 [1 << 17]byte +var z15280 [1 << 17]byte +var z15281 [1 << 17]byte +var z15282 [1 << 17]byte +var z15283 [1 << 17]byte +var z15284 [1 << 17]byte +var z15285 [1 << 17]byte +var z15286 [1 << 17]byte +var z15287 [1 << 17]byte +var z15288 [1 << 17]byte +var z15289 [1 << 17]byte +var z15290 [1 << 17]byte +var z15291 [1 << 17]byte +var z15292 [1 << 17]byte +var z15293 [1 << 17]byte +var z15294 [1 << 17]byte +var z15295 [1 << 17]byte +var z15296 [1 << 17]byte +var z15297 [1 << 17]byte +var z15298 [1 << 17]byte +var z15299 [1 << 17]byte +var z15300 [1 << 17]byte +var z15301 [1 << 17]byte +var z15302 [1 << 17]byte +var z15303 [1 << 17]byte +var z15304 [1 << 17]byte +var z15305 [1 << 17]byte +var z15306 [1 << 17]byte +var z15307 [1 << 17]byte +var z15308 [1 << 17]byte +var z15309 [1 << 17]byte +var z15310 [1 << 17]byte +var z15311 [1 << 17]byte +var z15312 [1 << 17]byte +var z15313 [1 << 17]byte +var z15314 [1 << 17]byte +var z15315 [1 << 17]byte +var z15316 [1 << 17]byte +var z15317 [1 << 17]byte +var z15318 [1 << 17]byte +var z15319 [1 << 17]byte +var z15320 [1 << 17]byte +var z15321 [1 << 17]byte +var z15322 [1 << 17]byte +var z15323 [1 << 17]byte +var z15324 [1 << 17]byte +var z15325 [1 << 17]byte +var z15326 [1 << 17]byte +var z15327 [1 << 17]byte +var z15328 [1 << 17]byte +var z15329 [1 << 17]byte +var z15330 [1 << 17]byte +var z15331 [1 << 17]byte +var z15332 [1 << 17]byte +var z15333 [1 << 17]byte +var z15334 [1 << 17]byte +var z15335 [1 << 17]byte +var z15336 [1 << 17]byte +var z15337 [1 << 17]byte +var z15338 [1 << 17]byte +var z15339 [1 << 17]byte +var z15340 [1 << 17]byte +var z15341 [1 << 17]byte +var z15342 [1 << 17]byte +var z15343 [1 << 17]byte +var z15344 [1 << 17]byte +var z15345 [1 << 17]byte +var z15346 [1 << 17]byte +var z15347 [1 << 17]byte +var z15348 [1 << 17]byte +var z15349 [1 << 17]byte +var z15350 [1 << 17]byte +var z15351 [1 << 17]byte +var z15352 [1 << 17]byte +var z15353 [1 << 17]byte +var z15354 [1 << 17]byte +var z15355 [1 << 17]byte +var z15356 [1 << 17]byte +var z15357 [1 << 17]byte +var z15358 [1 << 17]byte +var z15359 [1 << 17]byte +var z15360 [1 << 17]byte +var z15361 [1 << 17]byte +var z15362 [1 << 17]byte +var z15363 [1 << 17]byte +var z15364 [1 << 17]byte +var z15365 [1 << 17]byte +var z15366 [1 << 17]byte +var z15367 [1 << 17]byte +var z15368 [1 << 17]byte +var z15369 [1 << 17]byte +var z15370 [1 << 17]byte +var z15371 [1 << 17]byte +var z15372 [1 << 17]byte +var z15373 [1 << 17]byte +var z15374 [1 << 17]byte +var z15375 [1 << 17]byte +var z15376 [1 << 17]byte +var z15377 [1 << 17]byte +var z15378 [1 << 17]byte +var z15379 [1 << 17]byte +var z15380 [1 << 17]byte +var z15381 [1 << 17]byte +var z15382 [1 << 17]byte +var z15383 [1 << 17]byte +var z15384 [1 << 17]byte +var z15385 [1 << 17]byte +var z15386 [1 << 17]byte +var z15387 [1 << 17]byte +var z15388 [1 << 17]byte +var z15389 [1 << 17]byte +var z15390 [1 << 17]byte +var z15391 [1 << 17]byte +var z15392 [1 << 17]byte +var z15393 [1 << 17]byte +var z15394 [1 << 17]byte +var z15395 [1 << 17]byte +var z15396 [1 << 17]byte +var z15397 [1 << 17]byte +var z15398 [1 << 17]byte +var z15399 [1 << 17]byte +var z15400 [1 << 17]byte +var z15401 [1 << 17]byte +var z15402 [1 << 17]byte +var z15403 [1 << 17]byte +var z15404 [1 << 17]byte +var z15405 [1 << 17]byte +var z15406 [1 << 17]byte +var z15407 [1 << 17]byte +var z15408 [1 << 17]byte +var z15409 [1 << 17]byte +var z15410 [1 << 17]byte +var z15411 [1 << 17]byte +var z15412 [1 << 17]byte +var z15413 [1 << 17]byte +var z15414 [1 << 17]byte +var z15415 [1 << 17]byte +var z15416 [1 << 17]byte +var z15417 [1 << 17]byte +var z15418 [1 << 17]byte +var z15419 [1 << 17]byte +var z15420 [1 << 17]byte +var z15421 [1 << 17]byte +var z15422 [1 << 17]byte +var z15423 [1 << 17]byte +var z15424 [1 << 17]byte +var z15425 [1 << 17]byte +var z15426 [1 << 17]byte +var z15427 [1 << 17]byte +var z15428 [1 << 17]byte +var z15429 [1 << 17]byte +var z15430 [1 << 17]byte +var z15431 [1 << 17]byte +var z15432 [1 << 17]byte +var z15433 [1 << 17]byte +var z15434 [1 << 17]byte +var z15435 [1 << 17]byte +var z15436 [1 << 17]byte +var z15437 [1 << 17]byte +var z15438 [1 << 17]byte +var z15439 [1 << 17]byte +var z15440 [1 << 17]byte +var z15441 [1 << 17]byte +var z15442 [1 << 17]byte +var z15443 [1 << 17]byte +var z15444 [1 << 17]byte +var z15445 [1 << 17]byte +var z15446 [1 << 17]byte +var z15447 [1 << 17]byte +var z15448 [1 << 17]byte +var z15449 [1 << 17]byte +var z15450 [1 << 17]byte +var z15451 [1 << 17]byte +var z15452 [1 << 17]byte +var z15453 [1 << 17]byte +var z15454 [1 << 17]byte +var z15455 [1 << 17]byte +var z15456 [1 << 17]byte +var z15457 [1 << 17]byte +var z15458 [1 << 17]byte +var z15459 [1 << 17]byte +var z15460 [1 << 17]byte +var z15461 [1 << 17]byte +var z15462 [1 << 17]byte +var z15463 [1 << 17]byte +var z15464 [1 << 17]byte +var z15465 [1 << 17]byte +var z15466 [1 << 17]byte +var z15467 [1 << 17]byte +var z15468 [1 << 17]byte +var z15469 [1 << 17]byte +var z15470 [1 << 17]byte +var z15471 [1 << 17]byte +var z15472 [1 << 17]byte +var z15473 [1 << 17]byte +var z15474 [1 << 17]byte +var z15475 [1 << 17]byte +var z15476 [1 << 17]byte +var z15477 [1 << 17]byte +var z15478 [1 << 17]byte +var z15479 [1 << 17]byte +var z15480 [1 << 17]byte +var z15481 [1 << 17]byte +var z15482 [1 << 17]byte +var z15483 [1 << 17]byte +var z15484 [1 << 17]byte +var z15485 [1 << 17]byte +var z15486 [1 << 17]byte +var z15487 [1 << 17]byte +var z15488 [1 << 17]byte +var z15489 [1 << 17]byte +var z15490 [1 << 17]byte +var z15491 [1 << 17]byte +var z15492 [1 << 17]byte +var z15493 [1 << 17]byte +var z15494 [1 << 17]byte +var z15495 [1 << 17]byte +var z15496 [1 << 17]byte +var z15497 [1 << 17]byte +var z15498 [1 << 17]byte +var z15499 [1 << 17]byte +var z15500 [1 << 17]byte +var z15501 [1 << 17]byte +var z15502 [1 << 17]byte +var z15503 [1 << 17]byte +var z15504 [1 << 17]byte +var z15505 [1 << 17]byte +var z15506 [1 << 17]byte +var z15507 [1 << 17]byte +var z15508 [1 << 17]byte +var z15509 [1 << 17]byte +var z15510 [1 << 17]byte +var z15511 [1 << 17]byte +var z15512 [1 << 17]byte +var z15513 [1 << 17]byte +var z15514 [1 << 17]byte +var z15515 [1 << 17]byte +var z15516 [1 << 17]byte +var z15517 [1 << 17]byte +var z15518 [1 << 17]byte +var z15519 [1 << 17]byte +var z15520 [1 << 17]byte +var z15521 [1 << 17]byte +var z15522 [1 << 17]byte +var z15523 [1 << 17]byte +var z15524 [1 << 17]byte +var z15525 [1 << 17]byte +var z15526 [1 << 17]byte +var z15527 [1 << 17]byte +var z15528 [1 << 17]byte +var z15529 [1 << 17]byte +var z15530 [1 << 17]byte +var z15531 [1 << 17]byte +var z15532 [1 << 17]byte +var z15533 [1 << 17]byte +var z15534 [1 << 17]byte +var z15535 [1 << 17]byte +var z15536 [1 << 17]byte +var z15537 [1 << 17]byte +var z15538 [1 << 17]byte +var z15539 [1 << 17]byte +var z15540 [1 << 17]byte +var z15541 [1 << 17]byte +var z15542 [1 << 17]byte +var z15543 [1 << 17]byte +var z15544 [1 << 17]byte +var z15545 [1 << 17]byte +var z15546 [1 << 17]byte +var z15547 [1 << 17]byte +var z15548 [1 << 17]byte +var z15549 [1 << 17]byte +var z15550 [1 << 17]byte +var z15551 [1 << 17]byte +var z15552 [1 << 17]byte +var z15553 [1 << 17]byte +var z15554 [1 << 17]byte +var z15555 [1 << 17]byte +var z15556 [1 << 17]byte +var z15557 [1 << 17]byte +var z15558 [1 << 17]byte +var z15559 [1 << 17]byte +var z15560 [1 << 17]byte +var z15561 [1 << 17]byte +var z15562 [1 << 17]byte +var z15563 [1 << 17]byte +var z15564 [1 << 17]byte +var z15565 [1 << 17]byte +var z15566 [1 << 17]byte +var z15567 [1 << 17]byte +var z15568 [1 << 17]byte +var z15569 [1 << 17]byte +var z15570 [1 << 17]byte +var z15571 [1 << 17]byte +var z15572 [1 << 17]byte +var z15573 [1 << 17]byte +var z15574 [1 << 17]byte +var z15575 [1 << 17]byte +var z15576 [1 << 17]byte +var z15577 [1 << 17]byte +var z15578 [1 << 17]byte +var z15579 [1 << 17]byte +var z15580 [1 << 17]byte +var z15581 [1 << 17]byte +var z15582 [1 << 17]byte +var z15583 [1 << 17]byte +var z15584 [1 << 17]byte +var z15585 [1 << 17]byte +var z15586 [1 << 17]byte +var z15587 [1 << 17]byte +var z15588 [1 << 17]byte +var z15589 [1 << 17]byte +var z15590 [1 << 17]byte +var z15591 [1 << 17]byte +var z15592 [1 << 17]byte +var z15593 [1 << 17]byte +var z15594 [1 << 17]byte +var z15595 [1 << 17]byte +var z15596 [1 << 17]byte +var z15597 [1 << 17]byte +var z15598 [1 << 17]byte +var z15599 [1 << 17]byte +var z15600 [1 << 17]byte +var z15601 [1 << 17]byte +var z15602 [1 << 17]byte +var z15603 [1 << 17]byte +var z15604 [1 << 17]byte +var z15605 [1 << 17]byte +var z15606 [1 << 17]byte +var z15607 [1 << 17]byte +var z15608 [1 << 17]byte +var z15609 [1 << 17]byte +var z15610 [1 << 17]byte +var z15611 [1 << 17]byte +var z15612 [1 << 17]byte +var z15613 [1 << 17]byte +var z15614 [1 << 17]byte +var z15615 [1 << 17]byte +var z15616 [1 << 17]byte +var z15617 [1 << 17]byte +var z15618 [1 << 17]byte +var z15619 [1 << 17]byte +var z15620 [1 << 17]byte +var z15621 [1 << 17]byte +var z15622 [1 << 17]byte +var z15623 [1 << 17]byte +var z15624 [1 << 17]byte +var z15625 [1 << 17]byte +var z15626 [1 << 17]byte +var z15627 [1 << 17]byte +var z15628 [1 << 17]byte +var z15629 [1 << 17]byte +var z15630 [1 << 17]byte +var z15631 [1 << 17]byte +var z15632 [1 << 17]byte +var z15633 [1 << 17]byte +var z15634 [1 << 17]byte +var z15635 [1 << 17]byte +var z15636 [1 << 17]byte +var z15637 [1 << 17]byte +var z15638 [1 << 17]byte +var z15639 [1 << 17]byte +var z15640 [1 << 17]byte +var z15641 [1 << 17]byte +var z15642 [1 << 17]byte +var z15643 [1 << 17]byte +var z15644 [1 << 17]byte +var z15645 [1 << 17]byte +var z15646 [1 << 17]byte +var z15647 [1 << 17]byte +var z15648 [1 << 17]byte +var z15649 [1 << 17]byte +var z15650 [1 << 17]byte +var z15651 [1 << 17]byte +var z15652 [1 << 17]byte +var z15653 [1 << 17]byte +var z15654 [1 << 17]byte +var z15655 [1 << 17]byte +var z15656 [1 << 17]byte +var z15657 [1 << 17]byte +var z15658 [1 << 17]byte +var z15659 [1 << 17]byte +var z15660 [1 << 17]byte +var z15661 [1 << 17]byte +var z15662 [1 << 17]byte +var z15663 [1 << 17]byte +var z15664 [1 << 17]byte +var z15665 [1 << 17]byte +var z15666 [1 << 17]byte +var z15667 [1 << 17]byte +var z15668 [1 << 17]byte +var z15669 [1 << 17]byte +var z15670 [1 << 17]byte +var z15671 [1 << 17]byte +var z15672 [1 << 17]byte +var z15673 [1 << 17]byte +var z15674 [1 << 17]byte +var z15675 [1 << 17]byte +var z15676 [1 << 17]byte +var z15677 [1 << 17]byte +var z15678 [1 << 17]byte +var z15679 [1 << 17]byte +var z15680 [1 << 17]byte +var z15681 [1 << 17]byte +var z15682 [1 << 17]byte +var z15683 [1 << 17]byte +var z15684 [1 << 17]byte +var z15685 [1 << 17]byte +var z15686 [1 << 17]byte +var z15687 [1 << 17]byte +var z15688 [1 << 17]byte +var z15689 [1 << 17]byte +var z15690 [1 << 17]byte +var z15691 [1 << 17]byte +var z15692 [1 << 17]byte +var z15693 [1 << 17]byte +var z15694 [1 << 17]byte +var z15695 [1 << 17]byte +var z15696 [1 << 17]byte +var z15697 [1 << 17]byte +var z15698 [1 << 17]byte +var z15699 [1 << 17]byte +var z15700 [1 << 17]byte +var z15701 [1 << 17]byte +var z15702 [1 << 17]byte +var z15703 [1 << 17]byte +var z15704 [1 << 17]byte +var z15705 [1 << 17]byte +var z15706 [1 << 17]byte +var z15707 [1 << 17]byte +var z15708 [1 << 17]byte +var z15709 [1 << 17]byte +var z15710 [1 << 17]byte +var z15711 [1 << 17]byte +var z15712 [1 << 17]byte +var z15713 [1 << 17]byte +var z15714 [1 << 17]byte +var z15715 [1 << 17]byte +var z15716 [1 << 17]byte +var z15717 [1 << 17]byte +var z15718 [1 << 17]byte +var z15719 [1 << 17]byte +var z15720 [1 << 17]byte +var z15721 [1 << 17]byte +var z15722 [1 << 17]byte +var z15723 [1 << 17]byte +var z15724 [1 << 17]byte +var z15725 [1 << 17]byte +var z15726 [1 << 17]byte +var z15727 [1 << 17]byte +var z15728 [1 << 17]byte +var z15729 [1 << 17]byte +var z15730 [1 << 17]byte +var z15731 [1 << 17]byte +var z15732 [1 << 17]byte +var z15733 [1 << 17]byte +var z15734 [1 << 17]byte +var z15735 [1 << 17]byte +var z15736 [1 << 17]byte +var z15737 [1 << 17]byte +var z15738 [1 << 17]byte +var z15739 [1 << 17]byte +var z15740 [1 << 17]byte +var z15741 [1 << 17]byte +var z15742 [1 << 17]byte +var z15743 [1 << 17]byte +var z15744 [1 << 17]byte +var z15745 [1 << 17]byte +var z15746 [1 << 17]byte +var z15747 [1 << 17]byte +var z15748 [1 << 17]byte +var z15749 [1 << 17]byte +var z15750 [1 << 17]byte +var z15751 [1 << 17]byte +var z15752 [1 << 17]byte +var z15753 [1 << 17]byte +var z15754 [1 << 17]byte +var z15755 [1 << 17]byte +var z15756 [1 << 17]byte +var z15757 [1 << 17]byte +var z15758 [1 << 17]byte +var z15759 [1 << 17]byte +var z15760 [1 << 17]byte +var z15761 [1 << 17]byte +var z15762 [1 << 17]byte +var z15763 [1 << 17]byte +var z15764 [1 << 17]byte +var z15765 [1 << 17]byte +var z15766 [1 << 17]byte +var z15767 [1 << 17]byte +var z15768 [1 << 17]byte +var z15769 [1 << 17]byte +var z15770 [1 << 17]byte +var z15771 [1 << 17]byte +var z15772 [1 << 17]byte +var z15773 [1 << 17]byte +var z15774 [1 << 17]byte +var z15775 [1 << 17]byte +var z15776 [1 << 17]byte +var z15777 [1 << 17]byte +var z15778 [1 << 17]byte +var z15779 [1 << 17]byte +var z15780 [1 << 17]byte +var z15781 [1 << 17]byte +var z15782 [1 << 17]byte +var z15783 [1 << 17]byte +var z15784 [1 << 17]byte +var z15785 [1 << 17]byte +var z15786 [1 << 17]byte +var z15787 [1 << 17]byte +var z15788 [1 << 17]byte +var z15789 [1 << 17]byte +var z15790 [1 << 17]byte +var z15791 [1 << 17]byte +var z15792 [1 << 17]byte +var z15793 [1 << 17]byte +var z15794 [1 << 17]byte +var z15795 [1 << 17]byte +var z15796 [1 << 17]byte +var z15797 [1 << 17]byte +var z15798 [1 << 17]byte +var z15799 [1 << 17]byte +var z15800 [1 << 17]byte +var z15801 [1 << 17]byte +var z15802 [1 << 17]byte +var z15803 [1 << 17]byte +var z15804 [1 << 17]byte +var z15805 [1 << 17]byte +var z15806 [1 << 17]byte +var z15807 [1 << 17]byte +var z15808 [1 << 17]byte +var z15809 [1 << 17]byte +var z15810 [1 << 17]byte +var z15811 [1 << 17]byte +var z15812 [1 << 17]byte +var z15813 [1 << 17]byte +var z15814 [1 << 17]byte +var z15815 [1 << 17]byte +var z15816 [1 << 17]byte +var z15817 [1 << 17]byte +var z15818 [1 << 17]byte +var z15819 [1 << 17]byte +var z15820 [1 << 17]byte +var z15821 [1 << 17]byte +var z15822 [1 << 17]byte +var z15823 [1 << 17]byte +var z15824 [1 << 17]byte +var z15825 [1 << 17]byte +var z15826 [1 << 17]byte +var z15827 [1 << 17]byte +var z15828 [1 << 17]byte +var z15829 [1 << 17]byte +var z15830 [1 << 17]byte +var z15831 [1 << 17]byte +var z15832 [1 << 17]byte +var z15833 [1 << 17]byte +var z15834 [1 << 17]byte +var z15835 [1 << 17]byte +var z15836 [1 << 17]byte +var z15837 [1 << 17]byte +var z15838 [1 << 17]byte +var z15839 [1 << 17]byte +var z15840 [1 << 17]byte +var z15841 [1 << 17]byte +var z15842 [1 << 17]byte +var z15843 [1 << 17]byte +var z15844 [1 << 17]byte +var z15845 [1 << 17]byte +var z15846 [1 << 17]byte +var z15847 [1 << 17]byte +var z15848 [1 << 17]byte +var z15849 [1 << 17]byte +var z15850 [1 << 17]byte +var z15851 [1 << 17]byte +var z15852 [1 << 17]byte +var z15853 [1 << 17]byte +var z15854 [1 << 17]byte +var z15855 [1 << 17]byte +var z15856 [1 << 17]byte +var z15857 [1 << 17]byte +var z15858 [1 << 17]byte +var z15859 [1 << 17]byte +var z15860 [1 << 17]byte +var z15861 [1 << 17]byte +var z15862 [1 << 17]byte +var z15863 [1 << 17]byte +var z15864 [1 << 17]byte +var z15865 [1 << 17]byte +var z15866 [1 << 17]byte +var z15867 [1 << 17]byte +var z15868 [1 << 17]byte +var z15869 [1 << 17]byte +var z15870 [1 << 17]byte +var z15871 [1 << 17]byte +var z15872 [1 << 17]byte +var z15873 [1 << 17]byte +var z15874 [1 << 17]byte +var z15875 [1 << 17]byte +var z15876 [1 << 17]byte +var z15877 [1 << 17]byte +var z15878 [1 << 17]byte +var z15879 [1 << 17]byte +var z15880 [1 << 17]byte +var z15881 [1 << 17]byte +var z15882 [1 << 17]byte +var z15883 [1 << 17]byte +var z15884 [1 << 17]byte +var z15885 [1 << 17]byte +var z15886 [1 << 17]byte +var z15887 [1 << 17]byte +var z15888 [1 << 17]byte +var z15889 [1 << 17]byte +var z15890 [1 << 17]byte +var z15891 [1 << 17]byte +var z15892 [1 << 17]byte +var z15893 [1 << 17]byte +var z15894 [1 << 17]byte +var z15895 [1 << 17]byte +var z15896 [1 << 17]byte +var z15897 [1 << 17]byte +var z15898 [1 << 17]byte +var z15899 [1 << 17]byte +var z15900 [1 << 17]byte +var z15901 [1 << 17]byte +var z15902 [1 << 17]byte +var z15903 [1 << 17]byte +var z15904 [1 << 17]byte +var z15905 [1 << 17]byte +var z15906 [1 << 17]byte +var z15907 [1 << 17]byte +var z15908 [1 << 17]byte +var z15909 [1 << 17]byte +var z15910 [1 << 17]byte +var z15911 [1 << 17]byte +var z15912 [1 << 17]byte +var z15913 [1 << 17]byte +var z15914 [1 << 17]byte +var z15915 [1 << 17]byte +var z15916 [1 << 17]byte +var z15917 [1 << 17]byte +var z15918 [1 << 17]byte +var z15919 [1 << 17]byte +var z15920 [1 << 17]byte +var z15921 [1 << 17]byte +var z15922 [1 << 17]byte +var z15923 [1 << 17]byte +var z15924 [1 << 17]byte +var z15925 [1 << 17]byte +var z15926 [1 << 17]byte +var z15927 [1 << 17]byte +var z15928 [1 << 17]byte +var z15929 [1 << 17]byte +var z15930 [1 << 17]byte +var z15931 [1 << 17]byte +var z15932 [1 << 17]byte +var z15933 [1 << 17]byte +var z15934 [1 << 17]byte +var z15935 [1 << 17]byte +var z15936 [1 << 17]byte +var z15937 [1 << 17]byte +var z15938 [1 << 17]byte +var z15939 [1 << 17]byte +var z15940 [1 << 17]byte +var z15941 [1 << 17]byte +var z15942 [1 << 17]byte +var z15943 [1 << 17]byte +var z15944 [1 << 17]byte +var z15945 [1 << 17]byte +var z15946 [1 << 17]byte +var z15947 [1 << 17]byte +var z15948 [1 << 17]byte +var z15949 [1 << 17]byte +var z15950 [1 << 17]byte +var z15951 [1 << 17]byte +var z15952 [1 << 17]byte +var z15953 [1 << 17]byte +var z15954 [1 << 17]byte +var z15955 [1 << 17]byte +var z15956 [1 << 17]byte +var z15957 [1 << 17]byte +var z15958 [1 << 17]byte +var z15959 [1 << 17]byte +var z15960 [1 << 17]byte +var z15961 [1 << 17]byte +var z15962 [1 << 17]byte +var z15963 [1 << 17]byte +var z15964 [1 << 17]byte +var z15965 [1 << 17]byte +var z15966 [1 << 17]byte +var z15967 [1 << 17]byte +var z15968 [1 << 17]byte +var z15969 [1 << 17]byte +var z15970 [1 << 17]byte +var z15971 [1 << 17]byte +var z15972 [1 << 17]byte +var z15973 [1 << 17]byte +var z15974 [1 << 17]byte +var z15975 [1 << 17]byte +var z15976 [1 << 17]byte +var z15977 [1 << 17]byte +var z15978 [1 << 17]byte +var z15979 [1 << 17]byte +var z15980 [1 << 17]byte +var z15981 [1 << 17]byte +var z15982 [1 << 17]byte +var z15983 [1 << 17]byte +var z15984 [1 << 17]byte +var z15985 [1 << 17]byte +var z15986 [1 << 17]byte +var z15987 [1 << 17]byte +var z15988 [1 << 17]byte +var z15989 [1 << 17]byte +var z15990 [1 << 17]byte +var z15991 [1 << 17]byte +var z15992 [1 << 17]byte +var z15993 [1 << 17]byte +var z15994 [1 << 17]byte +var z15995 [1 << 17]byte +var z15996 [1 << 17]byte +var z15997 [1 << 17]byte +var z15998 [1 << 17]byte +var z15999 [1 << 17]byte +var z16000 [1 << 17]byte +var z16001 [1 << 17]byte +var z16002 [1 << 17]byte +var z16003 [1 << 17]byte +var z16004 [1 << 17]byte +var z16005 [1 << 17]byte +var z16006 [1 << 17]byte +var z16007 [1 << 17]byte +var z16008 [1 << 17]byte +var z16009 [1 << 17]byte +var z16010 [1 << 17]byte +var z16011 [1 << 17]byte +var z16012 [1 << 17]byte +var z16013 [1 << 17]byte +var z16014 [1 << 17]byte +var z16015 [1 << 17]byte +var z16016 [1 << 17]byte +var z16017 [1 << 17]byte +var z16018 [1 << 17]byte +var z16019 [1 << 17]byte +var z16020 [1 << 17]byte +var z16021 [1 << 17]byte +var z16022 [1 << 17]byte +var z16023 [1 << 17]byte +var z16024 [1 << 17]byte +var z16025 [1 << 17]byte +var z16026 [1 << 17]byte +var z16027 [1 << 17]byte +var z16028 [1 << 17]byte +var z16029 [1 << 17]byte +var z16030 [1 << 17]byte +var z16031 [1 << 17]byte +var z16032 [1 << 17]byte +var z16033 [1 << 17]byte +var z16034 [1 << 17]byte +var z16035 [1 << 17]byte +var z16036 [1 << 17]byte +var z16037 [1 << 17]byte +var z16038 [1 << 17]byte +var z16039 [1 << 17]byte +var z16040 [1 << 17]byte +var z16041 [1 << 17]byte +var z16042 [1 << 17]byte +var z16043 [1 << 17]byte +var z16044 [1 << 17]byte +var z16045 [1 << 17]byte +var z16046 [1 << 17]byte +var z16047 [1 << 17]byte +var z16048 [1 << 17]byte +var z16049 [1 << 17]byte +var z16050 [1 << 17]byte +var z16051 [1 << 17]byte +var z16052 [1 << 17]byte +var z16053 [1 << 17]byte +var z16054 [1 << 17]byte +var z16055 [1 << 17]byte +var z16056 [1 << 17]byte +var z16057 [1 << 17]byte +var z16058 [1 << 17]byte +var z16059 [1 << 17]byte +var z16060 [1 << 17]byte +var z16061 [1 << 17]byte +var z16062 [1 << 17]byte +var z16063 [1 << 17]byte +var z16064 [1 << 17]byte +var z16065 [1 << 17]byte +var z16066 [1 << 17]byte +var z16067 [1 << 17]byte +var z16068 [1 << 17]byte +var z16069 [1 << 17]byte +var z16070 [1 << 17]byte +var z16071 [1 << 17]byte +var z16072 [1 << 17]byte +var z16073 [1 << 17]byte +var z16074 [1 << 17]byte +var z16075 [1 << 17]byte +var z16076 [1 << 17]byte +var z16077 [1 << 17]byte +var z16078 [1 << 17]byte +var z16079 [1 << 17]byte +var z16080 [1 << 17]byte +var z16081 [1 << 17]byte +var z16082 [1 << 17]byte +var z16083 [1 << 17]byte +var z16084 [1 << 17]byte +var z16085 [1 << 17]byte +var z16086 [1 << 17]byte +var z16087 [1 << 17]byte +var z16088 [1 << 17]byte +var z16089 [1 << 17]byte +var z16090 [1 << 17]byte +var z16091 [1 << 17]byte +var z16092 [1 << 17]byte +var z16093 [1 << 17]byte +var z16094 [1 << 17]byte +var z16095 [1 << 17]byte +var z16096 [1 << 17]byte +var z16097 [1 << 17]byte +var z16098 [1 << 17]byte +var z16099 [1 << 17]byte +var z16100 [1 << 17]byte +var z16101 [1 << 17]byte +var z16102 [1 << 17]byte +var z16103 [1 << 17]byte +var z16104 [1 << 17]byte +var z16105 [1 << 17]byte +var z16106 [1 << 17]byte +var z16107 [1 << 17]byte +var z16108 [1 << 17]byte +var z16109 [1 << 17]byte +var z16110 [1 << 17]byte +var z16111 [1 << 17]byte +var z16112 [1 << 17]byte +var z16113 [1 << 17]byte +var z16114 [1 << 17]byte +var z16115 [1 << 17]byte +var z16116 [1 << 17]byte +var z16117 [1 << 17]byte +var z16118 [1 << 17]byte +var z16119 [1 << 17]byte +var z16120 [1 << 17]byte +var z16121 [1 << 17]byte +var z16122 [1 << 17]byte +var z16123 [1 << 17]byte +var z16124 [1 << 17]byte +var z16125 [1 << 17]byte +var z16126 [1 << 17]byte +var z16127 [1 << 17]byte +var z16128 [1 << 17]byte +var z16129 [1 << 17]byte +var z16130 [1 << 17]byte +var z16131 [1 << 17]byte +var z16132 [1 << 17]byte +var z16133 [1 << 17]byte +var z16134 [1 << 17]byte +var z16135 [1 << 17]byte +var z16136 [1 << 17]byte +var z16137 [1 << 17]byte +var z16138 [1 << 17]byte +var z16139 [1 << 17]byte +var z16140 [1 << 17]byte +var z16141 [1 << 17]byte +var z16142 [1 << 17]byte +var z16143 [1 << 17]byte +var z16144 [1 << 17]byte +var z16145 [1 << 17]byte +var z16146 [1 << 17]byte +var z16147 [1 << 17]byte +var z16148 [1 << 17]byte +var z16149 [1 << 17]byte +var z16150 [1 << 17]byte +var z16151 [1 << 17]byte +var z16152 [1 << 17]byte +var z16153 [1 << 17]byte +var z16154 [1 << 17]byte +var z16155 [1 << 17]byte +var z16156 [1 << 17]byte +var z16157 [1 << 17]byte +var z16158 [1 << 17]byte +var z16159 [1 << 17]byte +var z16160 [1 << 17]byte +var z16161 [1 << 17]byte +var z16162 [1 << 17]byte +var z16163 [1 << 17]byte +var z16164 [1 << 17]byte +var z16165 [1 << 17]byte +var z16166 [1 << 17]byte +var z16167 [1 << 17]byte +var z16168 [1 << 17]byte +var z16169 [1 << 17]byte +var z16170 [1 << 17]byte +var z16171 [1 << 17]byte +var z16172 [1 << 17]byte +var z16173 [1 << 17]byte +var z16174 [1 << 17]byte +var z16175 [1 << 17]byte +var z16176 [1 << 17]byte +var z16177 [1 << 17]byte +var z16178 [1 << 17]byte +var z16179 [1 << 17]byte +var z16180 [1 << 17]byte +var z16181 [1 << 17]byte +var z16182 [1 << 17]byte +var z16183 [1 << 17]byte +var z16184 [1 << 17]byte +var z16185 [1 << 17]byte +var z16186 [1 << 17]byte +var z16187 [1 << 17]byte +var z16188 [1 << 17]byte +var z16189 [1 << 17]byte +var z16190 [1 << 17]byte +var z16191 [1 << 17]byte +var z16192 [1 << 17]byte +var z16193 [1 << 17]byte +var z16194 [1 << 17]byte +var z16195 [1 << 17]byte +var z16196 [1 << 17]byte +var z16197 [1 << 17]byte +var z16198 [1 << 17]byte +var z16199 [1 << 17]byte +var z16200 [1 << 17]byte +var z16201 [1 << 17]byte +var z16202 [1 << 17]byte +var z16203 [1 << 17]byte +var z16204 [1 << 17]byte +var z16205 [1 << 17]byte +var z16206 [1 << 17]byte +var z16207 [1 << 17]byte +var z16208 [1 << 17]byte +var z16209 [1 << 17]byte +var z16210 [1 << 17]byte +var z16211 [1 << 17]byte +var z16212 [1 << 17]byte +var z16213 [1 << 17]byte +var z16214 [1 << 17]byte +var z16215 [1 << 17]byte +var z16216 [1 << 17]byte +var z16217 [1 << 17]byte +var z16218 [1 << 17]byte +var z16219 [1 << 17]byte +var z16220 [1 << 17]byte +var z16221 [1 << 17]byte +var z16222 [1 << 17]byte +var z16223 [1 << 17]byte +var z16224 [1 << 17]byte +var z16225 [1 << 17]byte +var z16226 [1 << 17]byte +var z16227 [1 << 17]byte +var z16228 [1 << 17]byte +var z16229 [1 << 17]byte +var z16230 [1 << 17]byte +var z16231 [1 << 17]byte +var z16232 [1 << 17]byte +var z16233 [1 << 17]byte +var z16234 [1 << 17]byte +var z16235 [1 << 17]byte +var z16236 [1 << 17]byte +var z16237 [1 << 17]byte +var z16238 [1 << 17]byte +var z16239 [1 << 17]byte +var z16240 [1 << 17]byte +var z16241 [1 << 17]byte +var z16242 [1 << 17]byte +var z16243 [1 << 17]byte +var z16244 [1 << 17]byte +var z16245 [1 << 17]byte +var z16246 [1 << 17]byte +var z16247 [1 << 17]byte +var z16248 [1 << 17]byte +var z16249 [1 << 17]byte +var z16250 [1 << 17]byte +var z16251 [1 << 17]byte +var z16252 [1 << 17]byte +var z16253 [1 << 17]byte +var z16254 [1 << 17]byte +var z16255 [1 << 17]byte +var z16256 [1 << 17]byte +var z16257 [1 << 17]byte +var z16258 [1 << 17]byte +var z16259 [1 << 17]byte +var z16260 [1 << 17]byte +var z16261 [1 << 17]byte +var z16262 [1 << 17]byte +var z16263 [1 << 17]byte +var z16264 [1 << 17]byte +var z16265 [1 << 17]byte +var z16266 [1 << 17]byte +var z16267 [1 << 17]byte +var z16268 [1 << 17]byte +var z16269 [1 << 17]byte +var z16270 [1 << 17]byte +var z16271 [1 << 17]byte +var z16272 [1 << 17]byte +var z16273 [1 << 17]byte +var z16274 [1 << 17]byte +var z16275 [1 << 17]byte +var z16276 [1 << 17]byte +var z16277 [1 << 17]byte +var z16278 [1 << 17]byte +var z16279 [1 << 17]byte +var z16280 [1 << 17]byte +var z16281 [1 << 17]byte +var z16282 [1 << 17]byte +var z16283 [1 << 17]byte +var z16284 [1 << 17]byte +var z16285 [1 << 17]byte +var z16286 [1 << 17]byte +var z16287 [1 << 17]byte +var z16288 [1 << 17]byte +var z16289 [1 << 17]byte +var z16290 [1 << 17]byte +var z16291 [1 << 17]byte +var z16292 [1 << 17]byte +var z16293 [1 << 17]byte +var z16294 [1 << 17]byte +var z16295 [1 << 17]byte +var z16296 [1 << 17]byte +var z16297 [1 << 17]byte +var z16298 [1 << 17]byte +var z16299 [1 << 17]byte +var z16300 [1 << 17]byte +var z16301 [1 << 17]byte +var z16302 [1 << 17]byte +var z16303 [1 << 17]byte +var z16304 [1 << 17]byte +var z16305 [1 << 17]byte +var z16306 [1 << 17]byte +var z16307 [1 << 17]byte +var z16308 [1 << 17]byte +var z16309 [1 << 17]byte +var z16310 [1 << 17]byte +var z16311 [1 << 17]byte +var z16312 [1 << 17]byte +var z16313 [1 << 17]byte +var z16314 [1 << 17]byte +var z16315 [1 << 17]byte +var z16316 [1 << 17]byte +var z16317 [1 << 17]byte +var z16318 [1 << 17]byte +var z16319 [1 << 17]byte +var z16320 [1 << 17]byte +var z16321 [1 << 17]byte +var z16322 [1 << 17]byte +var z16323 [1 << 17]byte +var z16324 [1 << 17]byte +var z16325 [1 << 17]byte +var z16326 [1 << 17]byte +var z16327 [1 << 17]byte +var z16328 [1 << 17]byte +var z16329 [1 << 17]byte +var z16330 [1 << 17]byte +var z16331 [1 << 17]byte +var z16332 [1 << 17]byte +var z16333 [1 << 17]byte +var z16334 [1 << 17]byte +var z16335 [1 << 17]byte +var z16336 [1 << 17]byte +var z16337 [1 << 17]byte +var z16338 [1 << 17]byte +var z16339 [1 << 17]byte +var z16340 [1 << 17]byte +var z16341 [1 << 17]byte +var z16342 [1 << 17]byte +var z16343 [1 << 17]byte +var z16344 [1 << 17]byte +var z16345 [1 << 17]byte +var z16346 [1 << 17]byte +var z16347 [1 << 17]byte +var z16348 [1 << 17]byte +var z16349 [1 << 17]byte +var z16350 [1 << 17]byte +var z16351 [1 << 17]byte +var z16352 [1 << 17]byte +var z16353 [1 << 17]byte +var z16354 [1 << 17]byte +var z16355 [1 << 17]byte +var z16356 [1 << 17]byte +var z16357 [1 << 17]byte +var z16358 [1 << 17]byte +var z16359 [1 << 17]byte +var z16360 [1 << 17]byte +var z16361 [1 << 17]byte +var z16362 [1 << 17]byte +var z16363 [1 << 17]byte +var z16364 [1 << 17]byte +var z16365 [1 << 17]byte +var z16366 [1 << 17]byte +var z16367 [1 << 17]byte +var z16368 [1 << 17]byte +var z16369 [1 << 17]byte +var z16370 [1 << 17]byte +var z16371 [1 << 17]byte +var z16372 [1 << 17]byte +var z16373 [1 << 17]byte +var z16374 [1 << 17]byte +var z16375 [1 << 17]byte +var z16376 [1 << 17]byte +var z16377 [1 << 17]byte +var z16378 [1 << 17]byte +var z16379 [1 << 17]byte +var z16380 [1 << 17]byte +var z16381 [1 << 17]byte +var z16382 [1 << 17]byte +var z16383 [1 << 17]byte +var z16384 [1 << 17]byte +var z16385 [1 << 17]byte +var z16386 [1 << 17]byte +var z16387 [1 << 17]byte +var z16388 [1 << 17]byte +var z16389 [1 << 17]byte +var z16390 [1 << 17]byte +var z16391 [1 << 17]byte +var z16392 [1 << 17]byte +var z16393 [1 << 17]byte +var z16394 [1 << 17]byte +var z16395 [1 << 17]byte +var z16396 [1 << 17]byte +var z16397 [1 << 17]byte +var z16398 [1 << 17]byte +var z16399 [1 << 17]byte +var z16400 [1 << 17]byte +var z16401 [1 << 17]byte +var z16402 [1 << 17]byte +var z16403 [1 << 17]byte +var z16404 [1 << 17]byte +var z16405 [1 << 17]byte +var z16406 [1 << 17]byte +var z16407 [1 << 17]byte +var z16408 [1 << 17]byte +var z16409 [1 << 17]byte +var z16410 [1 << 17]byte +var z16411 [1 << 17]byte +var z16412 [1 << 17]byte +var z16413 [1 << 17]byte +var z16414 [1 << 17]byte +var z16415 [1 << 17]byte +var z16416 [1 << 17]byte +var z16417 [1 << 17]byte +var z16418 [1 << 17]byte +var z16419 [1 << 17]byte +var z16420 [1 << 17]byte +var z16421 [1 << 17]byte +var z16422 [1 << 17]byte +var z16423 [1 << 17]byte +var z16424 [1 << 17]byte +var z16425 [1 << 17]byte +var z16426 [1 << 17]byte +var z16427 [1 << 17]byte +var z16428 [1 << 17]byte +var z16429 [1 << 17]byte +var z16430 [1 << 17]byte +var z16431 [1 << 17]byte +var z16432 [1 << 17]byte +var z16433 [1 << 17]byte +var z16434 [1 << 17]byte +var z16435 [1 << 17]byte +var z16436 [1 << 17]byte +var z16437 [1 << 17]byte +var z16438 [1 << 17]byte +var z16439 [1 << 17]byte +var z16440 [1 << 17]byte +var z16441 [1 << 17]byte +var z16442 [1 << 17]byte +var z16443 [1 << 17]byte +var z16444 [1 << 17]byte +var z16445 [1 << 17]byte +var z16446 [1 << 17]byte +var z16447 [1 << 17]byte +var z16448 [1 << 17]byte +var z16449 [1 << 17]byte +var z16450 [1 << 17]byte +var z16451 [1 << 17]byte +var z16452 [1 << 17]byte +var z16453 [1 << 17]byte +var z16454 [1 << 17]byte +var z16455 [1 << 17]byte +var z16456 [1 << 17]byte +var z16457 [1 << 17]byte +var z16458 [1 << 17]byte +var z16459 [1 << 17]byte +var z16460 [1 << 17]byte +var z16461 [1 << 17]byte +var z16462 [1 << 17]byte +var z16463 [1 << 17]byte +var z16464 [1 << 17]byte +var z16465 [1 << 17]byte +var z16466 [1 << 17]byte +var z16467 [1 << 17]byte +var z16468 [1 << 17]byte +var z16469 [1 << 17]byte +var z16470 [1 << 17]byte +var z16471 [1 << 17]byte +var z16472 [1 << 17]byte +var z16473 [1 << 17]byte +var z16474 [1 << 17]byte +var z16475 [1 << 17]byte +var z16476 [1 << 17]byte +var z16477 [1 << 17]byte +var z16478 [1 << 17]byte +var z16479 [1 << 17]byte +var z16480 [1 << 17]byte func main() { // GC_ERROR "stack frame too large" - // seq 1 206 | sed 's/.*/ var x& [10<<20]byte/' - // seq 1 206 | sed 's/.*/ z = x&/' - var x1 [10<<20]byte - var x2 [10<<20]byte - var x3 [10<<20]byte - var x4 [10<<20]byte - var x5 [10<<20]byte - var x6 [10<<20]byte - var x7 [10<<20]byte - var x8 [10<<20]byte - var x9 [10<<20]byte - var x10 [10<<20]byte - var x11 [10<<20]byte - var x12 [10<<20]byte - var x13 [10<<20]byte - var x14 [10<<20]byte - var x15 [10<<20]byte - var x16 [10<<20]byte - var x17 [10<<20]byte - var x18 [10<<20]byte - var x19 [10<<20]byte - var x20 [10<<20]byte - var x21 [10<<20]byte - var x22 [10<<20]byte - var x23 [10<<20]byte - var x24 [10<<20]byte - var x25 [10<<20]byte - var x26 [10<<20]byte - var x27 [10<<20]byte - var x28 [10<<20]byte - var x29 [10<<20]byte - var x30 [10<<20]byte - var x31 [10<<20]byte - var x32 [10<<20]byte - var x33 [10<<20]byte - var x34 [10<<20]byte - var x35 [10<<20]byte - var x36 [10<<20]byte - var x37 [10<<20]byte - var x38 [10<<20]byte - var x39 [10<<20]byte - var x40 [10<<20]byte - var x41 [10<<20]byte - var x42 [10<<20]byte - var x43 [10<<20]byte - var x44 [10<<20]byte - var x45 [10<<20]byte - var x46 [10<<20]byte - var x47 [10<<20]byte - var x48 [10<<20]byte - var x49 [10<<20]byte - var x50 [10<<20]byte - var x51 [10<<20]byte - var x52 [10<<20]byte - var x53 [10<<20]byte - var x54 [10<<20]byte - var x55 [10<<20]byte - var x56 [10<<20]byte - var x57 [10<<20]byte - var x58 [10<<20]byte - var x59 [10<<20]byte - var x60 [10<<20]byte - var x61 [10<<20]byte - var x62 [10<<20]byte - var x63 [10<<20]byte - var x64 [10<<20]byte - var x65 [10<<20]byte - var x66 [10<<20]byte - var x67 [10<<20]byte - var x68 [10<<20]byte - var x69 [10<<20]byte - var x70 [10<<20]byte - var x71 [10<<20]byte - var x72 [10<<20]byte - var x73 [10<<20]byte - var x74 [10<<20]byte - var x75 [10<<20]byte - var x76 [10<<20]byte - var x77 [10<<20]byte - var x78 [10<<20]byte - var x79 [10<<20]byte - var x80 [10<<20]byte - var x81 [10<<20]byte - var x82 [10<<20]byte - var x83 [10<<20]byte - var x84 [10<<20]byte - var x85 [10<<20]byte - var x86 [10<<20]byte - var x87 [10<<20]byte - var x88 [10<<20]byte - var x89 [10<<20]byte - var x90 [10<<20]byte - var x91 [10<<20]byte - var x92 [10<<20]byte - var x93 [10<<20]byte - var x94 [10<<20]byte - var x95 [10<<20]byte - var x96 [10<<20]byte - var x97 [10<<20]byte - var x98 [10<<20]byte - var x99 [10<<20]byte - var x100 [10<<20]byte - var x101 [10<<20]byte - var x102 [10<<20]byte - var x103 [10<<20]byte - var x104 [10<<20]byte - var x105 [10<<20]byte - var x106 [10<<20]byte - var x107 [10<<20]byte - var x108 [10<<20]byte - var x109 [10<<20]byte - var x110 [10<<20]byte - var x111 [10<<20]byte - var x112 [10<<20]byte - var x113 [10<<20]byte - var x114 [10<<20]byte - var x115 [10<<20]byte - var x116 [10<<20]byte - var x117 [10<<20]byte - var x118 [10<<20]byte - var x119 [10<<20]byte - var x120 [10<<20]byte - var x121 [10<<20]byte - var x122 [10<<20]byte - var x123 [10<<20]byte - var x124 [10<<20]byte - var x125 [10<<20]byte - var x126 [10<<20]byte - var x127 [10<<20]byte - var x128 [10<<20]byte - var x129 [10<<20]byte - var x130 [10<<20]byte - var x131 [10<<20]byte - var x132 [10<<20]byte - var x133 [10<<20]byte - var x134 [10<<20]byte - var x135 [10<<20]byte - var x136 [10<<20]byte - var x137 [10<<20]byte - var x138 [10<<20]byte - var x139 [10<<20]byte - var x140 [10<<20]byte - var x141 [10<<20]byte - var x142 [10<<20]byte - var x143 [10<<20]byte - var x144 [10<<20]byte - var x145 [10<<20]byte - var x146 [10<<20]byte - var x147 [10<<20]byte - var x148 [10<<20]byte - var x149 [10<<20]byte - var x150 [10<<20]byte - var x151 [10<<20]byte - var x152 [10<<20]byte - var x153 [10<<20]byte - var x154 [10<<20]byte - var x155 [10<<20]byte - var x156 [10<<20]byte - var x157 [10<<20]byte - var x158 [10<<20]byte - var x159 [10<<20]byte - var x160 [10<<20]byte - var x161 [10<<20]byte - var x162 [10<<20]byte - var x163 [10<<20]byte - var x164 [10<<20]byte - var x165 [10<<20]byte - var x166 [10<<20]byte - var x167 [10<<20]byte - var x168 [10<<20]byte - var x169 [10<<20]byte - var x170 [10<<20]byte - var x171 [10<<20]byte - var x172 [10<<20]byte - var x173 [10<<20]byte - var x174 [10<<20]byte - var x175 [10<<20]byte - var x176 [10<<20]byte - var x177 [10<<20]byte - var x178 [10<<20]byte - var x179 [10<<20]byte - var x180 [10<<20]byte - var x181 [10<<20]byte - var x182 [10<<20]byte - var x183 [10<<20]byte - var x184 [10<<20]byte - var x185 [10<<20]byte - var x186 [10<<20]byte - var x187 [10<<20]byte - var x188 [10<<20]byte - var x189 [10<<20]byte - var x190 [10<<20]byte - var x191 [10<<20]byte - var x192 [10<<20]byte - var x193 [10<<20]byte - var x194 [10<<20]byte - var x195 [10<<20]byte - var x196 [10<<20]byte - var x197 [10<<20]byte - var x198 [10<<20]byte - var x199 [10<<20]byte - var x200 [10<<20]byte - var x201 [10<<20]byte - var x202 [10<<20]byte - var x203 [10<<20]byte - var x204 [10<<20]byte - var x205 [10<<20]byte - var x206 [10<<20]byte - var x207 [10<<20]byte - z = x1 - z = x2 - z = x3 - z = x4 - z = x5 - z = x6 - z = x7 - z = x8 - z = x9 - z = x10 - z = x11 - z = x12 - z = x13 - z = x14 - z = x15 - z = x16 - z = x17 - z = x18 - z = x19 - z = x20 - z = x21 - z = x22 - z = x23 - z = x24 - z = x25 - z = x26 - z = x27 - z = x28 - z = x29 - z = x30 - z = x31 - z = x32 - z = x33 - z = x34 - z = x35 - z = x36 - z = x37 - z = x38 - z = x39 - z = x40 - z = x41 - z = x42 - z = x43 - z = x44 - z = x45 - z = x46 - z = x47 - z = x48 - z = x49 - z = x50 - z = x51 - z = x52 - z = x53 - z = x54 - z = x55 - z = x56 - z = x57 - z = x58 - z = x59 - z = x60 - z = x61 - z = x62 - z = x63 - z = x64 - z = x65 - z = x66 - z = x67 - z = x68 - z = x69 - z = x70 - z = x71 - z = x72 - z = x73 - z = x74 - z = x75 - z = x76 - z = x77 - z = x78 - z = x79 - z = x80 - z = x81 - z = x82 - z = x83 - z = x84 - z = x85 - z = x86 - z = x87 - z = x88 - z = x89 - z = x90 - z = x91 - z = x92 - z = x93 - z = x94 - z = x95 - z = x96 - z = x97 - z = x98 - z = x99 - z = x100 - z = x101 - z = x102 - z = x103 - z = x104 - z = x105 - z = x106 - z = x107 - z = x108 - z = x109 - z = x110 - z = x111 - z = x112 - z = x113 - z = x114 - z = x115 - z = x116 - z = x117 - z = x118 - z = x119 - z = x120 - z = x121 - z = x122 - z = x123 - z = x124 - z = x125 - z = x126 - z = x127 - z = x128 - z = x129 - z = x130 - z = x131 - z = x132 - z = x133 - z = x134 - z = x135 - z = x136 - z = x137 - z = x138 - z = x139 - z = x140 - z = x141 - z = x142 - z = x143 - z = x144 - z = x145 - z = x146 - z = x147 - z = x148 - z = x149 - z = x150 - z = x151 - z = x152 - z = x153 - z = x154 - z = x155 - z = x156 - z = x157 - z = x158 - z = x159 - z = x160 - z = x161 - z = x162 - z = x163 - z = x164 - z = x165 - z = x166 - z = x167 - z = x168 - z = x169 - z = x170 - z = x171 - z = x172 - z = x173 - z = x174 - z = x175 - z = x176 - z = x177 - z = x178 - z = x179 - z = x180 - z = x181 - z = x182 - z = x183 - z = x184 - z = x185 - z = x186 - z = x187 - z = x188 - z = x189 - z = x190 - z = x191 - z = x192 - z = x193 - z = x194 - z = x195 - z = x196 - z = x197 - z = x198 - z = x199 - z = x200 - z = x201 - z = x202 - z = x203 - z = x204 - z = x205 - z = x206 - z = x207 + // seq 1 16480 | sed 's/.*/ var x& [1 << 17]byte/' + // seq 1 16480 | sed 's/.*/ z& = x&/' + var x1 [1 << 17]byte + var x2 [1 << 17]byte + var x3 [1 << 17]byte + var x4 [1 << 17]byte + var x5 [1 << 17]byte + var x6 [1 << 17]byte + var x7 [1 << 17]byte + var x8 [1 << 17]byte + var x9 [1 << 17]byte + var x10 [1 << 17]byte + var x11 [1 << 17]byte + var x12 [1 << 17]byte + var x13 [1 << 17]byte + var x14 [1 << 17]byte + var x15 [1 << 17]byte + var x16 [1 << 17]byte + var x17 [1 << 17]byte + var x18 [1 << 17]byte + var x19 [1 << 17]byte + var x20 [1 << 17]byte + var x21 [1 << 17]byte + var x22 [1 << 17]byte + var x23 [1 << 17]byte + var x24 [1 << 17]byte + var x25 [1 << 17]byte + var x26 [1 << 17]byte + var x27 [1 << 17]byte + var x28 [1 << 17]byte + var x29 [1 << 17]byte + var x30 [1 << 17]byte + var x31 [1 << 17]byte + var x32 [1 << 17]byte + var x33 [1 << 17]byte + var x34 [1 << 17]byte + var x35 [1 << 17]byte + var x36 [1 << 17]byte + var x37 [1 << 17]byte + var x38 [1 << 17]byte + var x39 [1 << 17]byte + var x40 [1 << 17]byte + var x41 [1 << 17]byte + var x42 [1 << 17]byte + var x43 [1 << 17]byte + var x44 [1 << 17]byte + var x45 [1 << 17]byte + var x46 [1 << 17]byte + var x47 [1 << 17]byte + var x48 [1 << 17]byte + var x49 [1 << 17]byte + var x50 [1 << 17]byte + var x51 [1 << 17]byte + var x52 [1 << 17]byte + var x53 [1 << 17]byte + var x54 [1 << 17]byte + var x55 [1 << 17]byte + var x56 [1 << 17]byte + var x57 [1 << 17]byte + var x58 [1 << 17]byte + var x59 [1 << 17]byte + var x60 [1 << 17]byte + var x61 [1 << 17]byte + var x62 [1 << 17]byte + var x63 [1 << 17]byte + var x64 [1 << 17]byte + var x65 [1 << 17]byte + var x66 [1 << 17]byte + var x67 [1 << 17]byte + var x68 [1 << 17]byte + var x69 [1 << 17]byte + var x70 [1 << 17]byte + var x71 [1 << 17]byte + var x72 [1 << 17]byte + var x73 [1 << 17]byte + var x74 [1 << 17]byte + var x75 [1 << 17]byte + var x76 [1 << 17]byte + var x77 [1 << 17]byte + var x78 [1 << 17]byte + var x79 [1 << 17]byte + var x80 [1 << 17]byte + var x81 [1 << 17]byte + var x82 [1 << 17]byte + var x83 [1 << 17]byte + var x84 [1 << 17]byte + var x85 [1 << 17]byte + var x86 [1 << 17]byte + var x87 [1 << 17]byte + var x88 [1 << 17]byte + var x89 [1 << 17]byte + var x90 [1 << 17]byte + var x91 [1 << 17]byte + var x92 [1 << 17]byte + var x93 [1 << 17]byte + var x94 [1 << 17]byte + var x95 [1 << 17]byte + var x96 [1 << 17]byte + var x97 [1 << 17]byte + var x98 [1 << 17]byte + var x99 [1 << 17]byte + var x100 [1 << 17]byte + var x101 [1 << 17]byte + var x102 [1 << 17]byte + var x103 [1 << 17]byte + var x104 [1 << 17]byte + var x105 [1 << 17]byte + var x106 [1 << 17]byte + var x107 [1 << 17]byte + var x108 [1 << 17]byte + var x109 [1 << 17]byte + var x110 [1 << 17]byte + var x111 [1 << 17]byte + var x112 [1 << 17]byte + var x113 [1 << 17]byte + var x114 [1 << 17]byte + var x115 [1 << 17]byte + var x116 [1 << 17]byte + var x117 [1 << 17]byte + var x118 [1 << 17]byte + var x119 [1 << 17]byte + var x120 [1 << 17]byte + var x121 [1 << 17]byte + var x122 [1 << 17]byte + var x123 [1 << 17]byte + var x124 [1 << 17]byte + var x125 [1 << 17]byte + var x126 [1 << 17]byte + var x127 [1 << 17]byte + var x128 [1 << 17]byte + var x129 [1 << 17]byte + var x130 [1 << 17]byte + var x131 [1 << 17]byte + var x132 [1 << 17]byte + var x133 [1 << 17]byte + var x134 [1 << 17]byte + var x135 [1 << 17]byte + var x136 [1 << 17]byte + var x137 [1 << 17]byte + var x138 [1 << 17]byte + var x139 [1 << 17]byte + var x140 [1 << 17]byte + var x141 [1 << 17]byte + var x142 [1 << 17]byte + var x143 [1 << 17]byte + var x144 [1 << 17]byte + var x145 [1 << 17]byte + var x146 [1 << 17]byte + var x147 [1 << 17]byte + var x148 [1 << 17]byte + var x149 [1 << 17]byte + var x150 [1 << 17]byte + var x151 [1 << 17]byte + var x152 [1 << 17]byte + var x153 [1 << 17]byte + var x154 [1 << 17]byte + var x155 [1 << 17]byte + var x156 [1 << 17]byte + var x157 [1 << 17]byte + var x158 [1 << 17]byte + var x159 [1 << 17]byte + var x160 [1 << 17]byte + var x161 [1 << 17]byte + var x162 [1 << 17]byte + var x163 [1 << 17]byte + var x164 [1 << 17]byte + var x165 [1 << 17]byte + var x166 [1 << 17]byte + var x167 [1 << 17]byte + var x168 [1 << 17]byte + var x169 [1 << 17]byte + var x170 [1 << 17]byte + var x171 [1 << 17]byte + var x172 [1 << 17]byte + var x173 [1 << 17]byte + var x174 [1 << 17]byte + var x175 [1 << 17]byte + var x176 [1 << 17]byte + var x177 [1 << 17]byte + var x178 [1 << 17]byte + var x179 [1 << 17]byte + var x180 [1 << 17]byte + var x181 [1 << 17]byte + var x182 [1 << 17]byte + var x183 [1 << 17]byte + var x184 [1 << 17]byte + var x185 [1 << 17]byte + var x186 [1 << 17]byte + var x187 [1 << 17]byte + var x188 [1 << 17]byte + var x189 [1 << 17]byte + var x190 [1 << 17]byte + var x191 [1 << 17]byte + var x192 [1 << 17]byte + var x193 [1 << 17]byte + var x194 [1 << 17]byte + var x195 [1 << 17]byte + var x196 [1 << 17]byte + var x197 [1 << 17]byte + var x198 [1 << 17]byte + var x199 [1 << 17]byte + var x200 [1 << 17]byte + var x201 [1 << 17]byte + var x202 [1 << 17]byte + var x203 [1 << 17]byte + var x204 [1 << 17]byte + var x205 [1 << 17]byte + var x206 [1 << 17]byte + var x207 [1 << 17]byte + var x208 [1 << 17]byte + var x209 [1 << 17]byte + var x210 [1 << 17]byte + var x211 [1 << 17]byte + var x212 [1 << 17]byte + var x213 [1 << 17]byte + var x214 [1 << 17]byte + var x215 [1 << 17]byte + var x216 [1 << 17]byte + var x217 [1 << 17]byte + var x218 [1 << 17]byte + var x219 [1 << 17]byte + var x220 [1 << 17]byte + var x221 [1 << 17]byte + var x222 [1 << 17]byte + var x223 [1 << 17]byte + var x224 [1 << 17]byte + var x225 [1 << 17]byte + var x226 [1 << 17]byte + var x227 [1 << 17]byte + var x228 [1 << 17]byte + var x229 [1 << 17]byte + var x230 [1 << 17]byte + var x231 [1 << 17]byte + var x232 [1 << 17]byte + var x233 [1 << 17]byte + var x234 [1 << 17]byte + var x235 [1 << 17]byte + var x236 [1 << 17]byte + var x237 [1 << 17]byte + var x238 [1 << 17]byte + var x239 [1 << 17]byte + var x240 [1 << 17]byte + var x241 [1 << 17]byte + var x242 [1 << 17]byte + var x243 [1 << 17]byte + var x244 [1 << 17]byte + var x245 [1 << 17]byte + var x246 [1 << 17]byte + var x247 [1 << 17]byte + var x248 [1 << 17]byte + var x249 [1 << 17]byte + var x250 [1 << 17]byte + var x251 [1 << 17]byte + var x252 [1 << 17]byte + var x253 [1 << 17]byte + var x254 [1 << 17]byte + var x255 [1 << 17]byte + var x256 [1 << 17]byte + var x257 [1 << 17]byte + var x258 [1 << 17]byte + var x259 [1 << 17]byte + var x260 [1 << 17]byte + var x261 [1 << 17]byte + var x262 [1 << 17]byte + var x263 [1 << 17]byte + var x264 [1 << 17]byte + var x265 [1 << 17]byte + var x266 [1 << 17]byte + var x267 [1 << 17]byte + var x268 [1 << 17]byte + var x269 [1 << 17]byte + var x270 [1 << 17]byte + var x271 [1 << 17]byte + var x272 [1 << 17]byte + var x273 [1 << 17]byte + var x274 [1 << 17]byte + var x275 [1 << 17]byte + var x276 [1 << 17]byte + var x277 [1 << 17]byte + var x278 [1 << 17]byte + var x279 [1 << 17]byte + var x280 [1 << 17]byte + var x281 [1 << 17]byte + var x282 [1 << 17]byte + var x283 [1 << 17]byte + var x284 [1 << 17]byte + var x285 [1 << 17]byte + var x286 [1 << 17]byte + var x287 [1 << 17]byte + var x288 [1 << 17]byte + var x289 [1 << 17]byte + var x290 [1 << 17]byte + var x291 [1 << 17]byte + var x292 [1 << 17]byte + var x293 [1 << 17]byte + var x294 [1 << 17]byte + var x295 [1 << 17]byte + var x296 [1 << 17]byte + var x297 [1 << 17]byte + var x298 [1 << 17]byte + var x299 [1 << 17]byte + var x300 [1 << 17]byte + var x301 [1 << 17]byte + var x302 [1 << 17]byte + var x303 [1 << 17]byte + var x304 [1 << 17]byte + var x305 [1 << 17]byte + var x306 [1 << 17]byte + var x307 [1 << 17]byte + var x308 [1 << 17]byte + var x309 [1 << 17]byte + var x310 [1 << 17]byte + var x311 [1 << 17]byte + var x312 [1 << 17]byte + var x313 [1 << 17]byte + var x314 [1 << 17]byte + var x315 [1 << 17]byte + var x316 [1 << 17]byte + var x317 [1 << 17]byte + var x318 [1 << 17]byte + var x319 [1 << 17]byte + var x320 [1 << 17]byte + var x321 [1 << 17]byte + var x322 [1 << 17]byte + var x323 [1 << 17]byte + var x324 [1 << 17]byte + var x325 [1 << 17]byte + var x326 [1 << 17]byte + var x327 [1 << 17]byte + var x328 [1 << 17]byte + var x329 [1 << 17]byte + var x330 [1 << 17]byte + var x331 [1 << 17]byte + var x332 [1 << 17]byte + var x333 [1 << 17]byte + var x334 [1 << 17]byte + var x335 [1 << 17]byte + var x336 [1 << 17]byte + var x337 [1 << 17]byte + var x338 [1 << 17]byte + var x339 [1 << 17]byte + var x340 [1 << 17]byte + var x341 [1 << 17]byte + var x342 [1 << 17]byte + var x343 [1 << 17]byte + var x344 [1 << 17]byte + var x345 [1 << 17]byte + var x346 [1 << 17]byte + var x347 [1 << 17]byte + var x348 [1 << 17]byte + var x349 [1 << 17]byte + var x350 [1 << 17]byte + var x351 [1 << 17]byte + var x352 [1 << 17]byte + var x353 [1 << 17]byte + var x354 [1 << 17]byte + var x355 [1 << 17]byte + var x356 [1 << 17]byte + var x357 [1 << 17]byte + var x358 [1 << 17]byte + var x359 [1 << 17]byte + var x360 [1 << 17]byte + var x361 [1 << 17]byte + var x362 [1 << 17]byte + var x363 [1 << 17]byte + var x364 [1 << 17]byte + var x365 [1 << 17]byte + var x366 [1 << 17]byte + var x367 [1 << 17]byte + var x368 [1 << 17]byte + var x369 [1 << 17]byte + var x370 [1 << 17]byte + var x371 [1 << 17]byte + var x372 [1 << 17]byte + var x373 [1 << 17]byte + var x374 [1 << 17]byte + var x375 [1 << 17]byte + var x376 [1 << 17]byte + var x377 [1 << 17]byte + var x378 [1 << 17]byte + var x379 [1 << 17]byte + var x380 [1 << 17]byte + var x381 [1 << 17]byte + var x382 [1 << 17]byte + var x383 [1 << 17]byte + var x384 [1 << 17]byte + var x385 [1 << 17]byte + var x386 [1 << 17]byte + var x387 [1 << 17]byte + var x388 [1 << 17]byte + var x389 [1 << 17]byte + var x390 [1 << 17]byte + var x391 [1 << 17]byte + var x392 [1 << 17]byte + var x393 [1 << 17]byte + var x394 [1 << 17]byte + var x395 [1 << 17]byte + var x396 [1 << 17]byte + var x397 [1 << 17]byte + var x398 [1 << 17]byte + var x399 [1 << 17]byte + var x400 [1 << 17]byte + var x401 [1 << 17]byte + var x402 [1 << 17]byte + var x403 [1 << 17]byte + var x404 [1 << 17]byte + var x405 [1 << 17]byte + var x406 [1 << 17]byte + var x407 [1 << 17]byte + var x408 [1 << 17]byte + var x409 [1 << 17]byte + var x410 [1 << 17]byte + var x411 [1 << 17]byte + var x412 [1 << 17]byte + var x413 [1 << 17]byte + var x414 [1 << 17]byte + var x415 [1 << 17]byte + var x416 [1 << 17]byte + var x417 [1 << 17]byte + var x418 [1 << 17]byte + var x419 [1 << 17]byte + var x420 [1 << 17]byte + var x421 [1 << 17]byte + var x422 [1 << 17]byte + var x423 [1 << 17]byte + var x424 [1 << 17]byte + var x425 [1 << 17]byte + var x426 [1 << 17]byte + var x427 [1 << 17]byte + var x428 [1 << 17]byte + var x429 [1 << 17]byte + var x430 [1 << 17]byte + var x431 [1 << 17]byte + var x432 [1 << 17]byte + var x433 [1 << 17]byte + var x434 [1 << 17]byte + var x435 [1 << 17]byte + var x436 [1 << 17]byte + var x437 [1 << 17]byte + var x438 [1 << 17]byte + var x439 [1 << 17]byte + var x440 [1 << 17]byte + var x441 [1 << 17]byte + var x442 [1 << 17]byte + var x443 [1 << 17]byte + var x444 [1 << 17]byte + var x445 [1 << 17]byte + var x446 [1 << 17]byte + var x447 [1 << 17]byte + var x448 [1 << 17]byte + var x449 [1 << 17]byte + var x450 [1 << 17]byte + var x451 [1 << 17]byte + var x452 [1 << 17]byte + var x453 [1 << 17]byte + var x454 [1 << 17]byte + var x455 [1 << 17]byte + var x456 [1 << 17]byte + var x457 [1 << 17]byte + var x458 [1 << 17]byte + var x459 [1 << 17]byte + var x460 [1 << 17]byte + var x461 [1 << 17]byte + var x462 [1 << 17]byte + var x463 [1 << 17]byte + var x464 [1 << 17]byte + var x465 [1 << 17]byte + var x466 [1 << 17]byte + var x467 [1 << 17]byte + var x468 [1 << 17]byte + var x469 [1 << 17]byte + var x470 [1 << 17]byte + var x471 [1 << 17]byte + var x472 [1 << 17]byte + var x473 [1 << 17]byte + var x474 [1 << 17]byte + var x475 [1 << 17]byte + var x476 [1 << 17]byte + var x477 [1 << 17]byte + var x478 [1 << 17]byte + var x479 [1 << 17]byte + var x480 [1 << 17]byte + var x481 [1 << 17]byte + var x482 [1 << 17]byte + var x483 [1 << 17]byte + var x484 [1 << 17]byte + var x485 [1 << 17]byte + var x486 [1 << 17]byte + var x487 [1 << 17]byte + var x488 [1 << 17]byte + var x489 [1 << 17]byte + var x490 [1 << 17]byte + var x491 [1 << 17]byte + var x492 [1 << 17]byte + var x493 [1 << 17]byte + var x494 [1 << 17]byte + var x495 [1 << 17]byte + var x496 [1 << 17]byte + var x497 [1 << 17]byte + var x498 [1 << 17]byte + var x499 [1 << 17]byte + var x500 [1 << 17]byte + var x501 [1 << 17]byte + var x502 [1 << 17]byte + var x503 [1 << 17]byte + var x504 [1 << 17]byte + var x505 [1 << 17]byte + var x506 [1 << 17]byte + var x507 [1 << 17]byte + var x508 [1 << 17]byte + var x509 [1 << 17]byte + var x510 [1 << 17]byte + var x511 [1 << 17]byte + var x512 [1 << 17]byte + var x513 [1 << 17]byte + var x514 [1 << 17]byte + var x515 [1 << 17]byte + var x516 [1 << 17]byte + var x517 [1 << 17]byte + var x518 [1 << 17]byte + var x519 [1 << 17]byte + var x520 [1 << 17]byte + var x521 [1 << 17]byte + var x522 [1 << 17]byte + var x523 [1 << 17]byte + var x524 [1 << 17]byte + var x525 [1 << 17]byte + var x526 [1 << 17]byte + var x527 [1 << 17]byte + var x528 [1 << 17]byte + var x529 [1 << 17]byte + var x530 [1 << 17]byte + var x531 [1 << 17]byte + var x532 [1 << 17]byte + var x533 [1 << 17]byte + var x534 [1 << 17]byte + var x535 [1 << 17]byte + var x536 [1 << 17]byte + var x537 [1 << 17]byte + var x538 [1 << 17]byte + var x539 [1 << 17]byte + var x540 [1 << 17]byte + var x541 [1 << 17]byte + var x542 [1 << 17]byte + var x543 [1 << 17]byte + var x544 [1 << 17]byte + var x545 [1 << 17]byte + var x546 [1 << 17]byte + var x547 [1 << 17]byte + var x548 [1 << 17]byte + var x549 [1 << 17]byte + var x550 [1 << 17]byte + var x551 [1 << 17]byte + var x552 [1 << 17]byte + var x553 [1 << 17]byte + var x554 [1 << 17]byte + var x555 [1 << 17]byte + var x556 [1 << 17]byte + var x557 [1 << 17]byte + var x558 [1 << 17]byte + var x559 [1 << 17]byte + var x560 [1 << 17]byte + var x561 [1 << 17]byte + var x562 [1 << 17]byte + var x563 [1 << 17]byte + var x564 [1 << 17]byte + var x565 [1 << 17]byte + var x566 [1 << 17]byte + var x567 [1 << 17]byte + var x568 [1 << 17]byte + var x569 [1 << 17]byte + var x570 [1 << 17]byte + var x571 [1 << 17]byte + var x572 [1 << 17]byte + var x573 [1 << 17]byte + var x574 [1 << 17]byte + var x575 [1 << 17]byte + var x576 [1 << 17]byte + var x577 [1 << 17]byte + var x578 [1 << 17]byte + var x579 [1 << 17]byte + var x580 [1 << 17]byte + var x581 [1 << 17]byte + var x582 [1 << 17]byte + var x583 [1 << 17]byte + var x584 [1 << 17]byte + var x585 [1 << 17]byte + var x586 [1 << 17]byte + var x587 [1 << 17]byte + var x588 [1 << 17]byte + var x589 [1 << 17]byte + var x590 [1 << 17]byte + var x591 [1 << 17]byte + var x592 [1 << 17]byte + var x593 [1 << 17]byte + var x594 [1 << 17]byte + var x595 [1 << 17]byte + var x596 [1 << 17]byte + var x597 [1 << 17]byte + var x598 [1 << 17]byte + var x599 [1 << 17]byte + var x600 [1 << 17]byte + var x601 [1 << 17]byte + var x602 [1 << 17]byte + var x603 [1 << 17]byte + var x604 [1 << 17]byte + var x605 [1 << 17]byte + var x606 [1 << 17]byte + var x607 [1 << 17]byte + var x608 [1 << 17]byte + var x609 [1 << 17]byte + var x610 [1 << 17]byte + var x611 [1 << 17]byte + var x612 [1 << 17]byte + var x613 [1 << 17]byte + var x614 [1 << 17]byte + var x615 [1 << 17]byte + var x616 [1 << 17]byte + var x617 [1 << 17]byte + var x618 [1 << 17]byte + var x619 [1 << 17]byte + var x620 [1 << 17]byte + var x621 [1 << 17]byte + var x622 [1 << 17]byte + var x623 [1 << 17]byte + var x624 [1 << 17]byte + var x625 [1 << 17]byte + var x626 [1 << 17]byte + var x627 [1 << 17]byte + var x628 [1 << 17]byte + var x629 [1 << 17]byte + var x630 [1 << 17]byte + var x631 [1 << 17]byte + var x632 [1 << 17]byte + var x633 [1 << 17]byte + var x634 [1 << 17]byte + var x635 [1 << 17]byte + var x636 [1 << 17]byte + var x637 [1 << 17]byte + var x638 [1 << 17]byte + var x639 [1 << 17]byte + var x640 [1 << 17]byte + var x641 [1 << 17]byte + var x642 [1 << 17]byte + var x643 [1 << 17]byte + var x644 [1 << 17]byte + var x645 [1 << 17]byte + var x646 [1 << 17]byte + var x647 [1 << 17]byte + var x648 [1 << 17]byte + var x649 [1 << 17]byte + var x650 [1 << 17]byte + var x651 [1 << 17]byte + var x652 [1 << 17]byte + var x653 [1 << 17]byte + var x654 [1 << 17]byte + var x655 [1 << 17]byte + var x656 [1 << 17]byte + var x657 [1 << 17]byte + var x658 [1 << 17]byte + var x659 [1 << 17]byte + var x660 [1 << 17]byte + var x661 [1 << 17]byte + var x662 [1 << 17]byte + var x663 [1 << 17]byte + var x664 [1 << 17]byte + var x665 [1 << 17]byte + var x666 [1 << 17]byte + var x667 [1 << 17]byte + var x668 [1 << 17]byte + var x669 [1 << 17]byte + var x670 [1 << 17]byte + var x671 [1 << 17]byte + var x672 [1 << 17]byte + var x673 [1 << 17]byte + var x674 [1 << 17]byte + var x675 [1 << 17]byte + var x676 [1 << 17]byte + var x677 [1 << 17]byte + var x678 [1 << 17]byte + var x679 [1 << 17]byte + var x680 [1 << 17]byte + var x681 [1 << 17]byte + var x682 [1 << 17]byte + var x683 [1 << 17]byte + var x684 [1 << 17]byte + var x685 [1 << 17]byte + var x686 [1 << 17]byte + var x687 [1 << 17]byte + var x688 [1 << 17]byte + var x689 [1 << 17]byte + var x690 [1 << 17]byte + var x691 [1 << 17]byte + var x692 [1 << 17]byte + var x693 [1 << 17]byte + var x694 [1 << 17]byte + var x695 [1 << 17]byte + var x696 [1 << 17]byte + var x697 [1 << 17]byte + var x698 [1 << 17]byte + var x699 [1 << 17]byte + var x700 [1 << 17]byte + var x701 [1 << 17]byte + var x702 [1 << 17]byte + var x703 [1 << 17]byte + var x704 [1 << 17]byte + var x705 [1 << 17]byte + var x706 [1 << 17]byte + var x707 [1 << 17]byte + var x708 [1 << 17]byte + var x709 [1 << 17]byte + var x710 [1 << 17]byte + var x711 [1 << 17]byte + var x712 [1 << 17]byte + var x713 [1 << 17]byte + var x714 [1 << 17]byte + var x715 [1 << 17]byte + var x716 [1 << 17]byte + var x717 [1 << 17]byte + var x718 [1 << 17]byte + var x719 [1 << 17]byte + var x720 [1 << 17]byte + var x721 [1 << 17]byte + var x722 [1 << 17]byte + var x723 [1 << 17]byte + var x724 [1 << 17]byte + var x725 [1 << 17]byte + var x726 [1 << 17]byte + var x727 [1 << 17]byte + var x728 [1 << 17]byte + var x729 [1 << 17]byte + var x730 [1 << 17]byte + var x731 [1 << 17]byte + var x732 [1 << 17]byte + var x733 [1 << 17]byte + var x734 [1 << 17]byte + var x735 [1 << 17]byte + var x736 [1 << 17]byte + var x737 [1 << 17]byte + var x738 [1 << 17]byte + var x739 [1 << 17]byte + var x740 [1 << 17]byte + var x741 [1 << 17]byte + var x742 [1 << 17]byte + var x743 [1 << 17]byte + var x744 [1 << 17]byte + var x745 [1 << 17]byte + var x746 [1 << 17]byte + var x747 [1 << 17]byte + var x748 [1 << 17]byte + var x749 [1 << 17]byte + var x750 [1 << 17]byte + var x751 [1 << 17]byte + var x752 [1 << 17]byte + var x753 [1 << 17]byte + var x754 [1 << 17]byte + var x755 [1 << 17]byte + var x756 [1 << 17]byte + var x757 [1 << 17]byte + var x758 [1 << 17]byte + var x759 [1 << 17]byte + var x760 [1 << 17]byte + var x761 [1 << 17]byte + var x762 [1 << 17]byte + var x763 [1 << 17]byte + var x764 [1 << 17]byte + var x765 [1 << 17]byte + var x766 [1 << 17]byte + var x767 [1 << 17]byte + var x768 [1 << 17]byte + var x769 [1 << 17]byte + var x770 [1 << 17]byte + var x771 [1 << 17]byte + var x772 [1 << 17]byte + var x773 [1 << 17]byte + var x774 [1 << 17]byte + var x775 [1 << 17]byte + var x776 [1 << 17]byte + var x777 [1 << 17]byte + var x778 [1 << 17]byte + var x779 [1 << 17]byte + var x780 [1 << 17]byte + var x781 [1 << 17]byte + var x782 [1 << 17]byte + var x783 [1 << 17]byte + var x784 [1 << 17]byte + var x785 [1 << 17]byte + var x786 [1 << 17]byte + var x787 [1 << 17]byte + var x788 [1 << 17]byte + var x789 [1 << 17]byte + var x790 [1 << 17]byte + var x791 [1 << 17]byte + var x792 [1 << 17]byte + var x793 [1 << 17]byte + var x794 [1 << 17]byte + var x795 [1 << 17]byte + var x796 [1 << 17]byte + var x797 [1 << 17]byte + var x798 [1 << 17]byte + var x799 [1 << 17]byte + var x800 [1 << 17]byte + var x801 [1 << 17]byte + var x802 [1 << 17]byte + var x803 [1 << 17]byte + var x804 [1 << 17]byte + var x805 [1 << 17]byte + var x806 [1 << 17]byte + var x807 [1 << 17]byte + var x808 [1 << 17]byte + var x809 [1 << 17]byte + var x810 [1 << 17]byte + var x811 [1 << 17]byte + var x812 [1 << 17]byte + var x813 [1 << 17]byte + var x814 [1 << 17]byte + var x815 [1 << 17]byte + var x816 [1 << 17]byte + var x817 [1 << 17]byte + var x818 [1 << 17]byte + var x819 [1 << 17]byte + var x820 [1 << 17]byte + var x821 [1 << 17]byte + var x822 [1 << 17]byte + var x823 [1 << 17]byte + var x824 [1 << 17]byte + var x825 [1 << 17]byte + var x826 [1 << 17]byte + var x827 [1 << 17]byte + var x828 [1 << 17]byte + var x829 [1 << 17]byte + var x830 [1 << 17]byte + var x831 [1 << 17]byte + var x832 [1 << 17]byte + var x833 [1 << 17]byte + var x834 [1 << 17]byte + var x835 [1 << 17]byte + var x836 [1 << 17]byte + var x837 [1 << 17]byte + var x838 [1 << 17]byte + var x839 [1 << 17]byte + var x840 [1 << 17]byte + var x841 [1 << 17]byte + var x842 [1 << 17]byte + var x843 [1 << 17]byte + var x844 [1 << 17]byte + var x845 [1 << 17]byte + var x846 [1 << 17]byte + var x847 [1 << 17]byte + var x848 [1 << 17]byte + var x849 [1 << 17]byte + var x850 [1 << 17]byte + var x851 [1 << 17]byte + var x852 [1 << 17]byte + var x853 [1 << 17]byte + var x854 [1 << 17]byte + var x855 [1 << 17]byte + var x856 [1 << 17]byte + var x857 [1 << 17]byte + var x858 [1 << 17]byte + var x859 [1 << 17]byte + var x860 [1 << 17]byte + var x861 [1 << 17]byte + var x862 [1 << 17]byte + var x863 [1 << 17]byte + var x864 [1 << 17]byte + var x865 [1 << 17]byte + var x866 [1 << 17]byte + var x867 [1 << 17]byte + var x868 [1 << 17]byte + var x869 [1 << 17]byte + var x870 [1 << 17]byte + var x871 [1 << 17]byte + var x872 [1 << 17]byte + var x873 [1 << 17]byte + var x874 [1 << 17]byte + var x875 [1 << 17]byte + var x876 [1 << 17]byte + var x877 [1 << 17]byte + var x878 [1 << 17]byte + var x879 [1 << 17]byte + var x880 [1 << 17]byte + var x881 [1 << 17]byte + var x882 [1 << 17]byte + var x883 [1 << 17]byte + var x884 [1 << 17]byte + var x885 [1 << 17]byte + var x886 [1 << 17]byte + var x887 [1 << 17]byte + var x888 [1 << 17]byte + var x889 [1 << 17]byte + var x890 [1 << 17]byte + var x891 [1 << 17]byte + var x892 [1 << 17]byte + var x893 [1 << 17]byte + var x894 [1 << 17]byte + var x895 [1 << 17]byte + var x896 [1 << 17]byte + var x897 [1 << 17]byte + var x898 [1 << 17]byte + var x899 [1 << 17]byte + var x900 [1 << 17]byte + var x901 [1 << 17]byte + var x902 [1 << 17]byte + var x903 [1 << 17]byte + var x904 [1 << 17]byte + var x905 [1 << 17]byte + var x906 [1 << 17]byte + var x907 [1 << 17]byte + var x908 [1 << 17]byte + var x909 [1 << 17]byte + var x910 [1 << 17]byte + var x911 [1 << 17]byte + var x912 [1 << 17]byte + var x913 [1 << 17]byte + var x914 [1 << 17]byte + var x915 [1 << 17]byte + var x916 [1 << 17]byte + var x917 [1 << 17]byte + var x918 [1 << 17]byte + var x919 [1 << 17]byte + var x920 [1 << 17]byte + var x921 [1 << 17]byte + var x922 [1 << 17]byte + var x923 [1 << 17]byte + var x924 [1 << 17]byte + var x925 [1 << 17]byte + var x926 [1 << 17]byte + var x927 [1 << 17]byte + var x928 [1 << 17]byte + var x929 [1 << 17]byte + var x930 [1 << 17]byte + var x931 [1 << 17]byte + var x932 [1 << 17]byte + var x933 [1 << 17]byte + var x934 [1 << 17]byte + var x935 [1 << 17]byte + var x936 [1 << 17]byte + var x937 [1 << 17]byte + var x938 [1 << 17]byte + var x939 [1 << 17]byte + var x940 [1 << 17]byte + var x941 [1 << 17]byte + var x942 [1 << 17]byte + var x943 [1 << 17]byte + var x944 [1 << 17]byte + var x945 [1 << 17]byte + var x946 [1 << 17]byte + var x947 [1 << 17]byte + var x948 [1 << 17]byte + var x949 [1 << 17]byte + var x950 [1 << 17]byte + var x951 [1 << 17]byte + var x952 [1 << 17]byte + var x953 [1 << 17]byte + var x954 [1 << 17]byte + var x955 [1 << 17]byte + var x956 [1 << 17]byte + var x957 [1 << 17]byte + var x958 [1 << 17]byte + var x959 [1 << 17]byte + var x960 [1 << 17]byte + var x961 [1 << 17]byte + var x962 [1 << 17]byte + var x963 [1 << 17]byte + var x964 [1 << 17]byte + var x965 [1 << 17]byte + var x966 [1 << 17]byte + var x967 [1 << 17]byte + var x968 [1 << 17]byte + var x969 [1 << 17]byte + var x970 [1 << 17]byte + var x971 [1 << 17]byte + var x972 [1 << 17]byte + var x973 [1 << 17]byte + var x974 [1 << 17]byte + var x975 [1 << 17]byte + var x976 [1 << 17]byte + var x977 [1 << 17]byte + var x978 [1 << 17]byte + var x979 [1 << 17]byte + var x980 [1 << 17]byte + var x981 [1 << 17]byte + var x982 [1 << 17]byte + var x983 [1 << 17]byte + var x984 [1 << 17]byte + var x985 [1 << 17]byte + var x986 [1 << 17]byte + var x987 [1 << 17]byte + var x988 [1 << 17]byte + var x989 [1 << 17]byte + var x990 [1 << 17]byte + var x991 [1 << 17]byte + var x992 [1 << 17]byte + var x993 [1 << 17]byte + var x994 [1 << 17]byte + var x995 [1 << 17]byte + var x996 [1 << 17]byte + var x997 [1 << 17]byte + var x998 [1 << 17]byte + var x999 [1 << 17]byte + var x1000 [1 << 17]byte + var x1001 [1 << 17]byte + var x1002 [1 << 17]byte + var x1003 [1 << 17]byte + var x1004 [1 << 17]byte + var x1005 [1 << 17]byte + var x1006 [1 << 17]byte + var x1007 [1 << 17]byte + var x1008 [1 << 17]byte + var x1009 [1 << 17]byte + var x1010 [1 << 17]byte + var x1011 [1 << 17]byte + var x1012 [1 << 17]byte + var x1013 [1 << 17]byte + var x1014 [1 << 17]byte + var x1015 [1 << 17]byte + var x1016 [1 << 17]byte + var x1017 [1 << 17]byte + var x1018 [1 << 17]byte + var x1019 [1 << 17]byte + var x1020 [1 << 17]byte + var x1021 [1 << 17]byte + var x1022 [1 << 17]byte + var x1023 [1 << 17]byte + var x1024 [1 << 17]byte + var x1025 [1 << 17]byte + var x1026 [1 << 17]byte + var x1027 [1 << 17]byte + var x1028 [1 << 17]byte + var x1029 [1 << 17]byte + var x1030 [1 << 17]byte + var x1031 [1 << 17]byte + var x1032 [1 << 17]byte + var x1033 [1 << 17]byte + var x1034 [1 << 17]byte + var x1035 [1 << 17]byte + var x1036 [1 << 17]byte + var x1037 [1 << 17]byte + var x1038 [1 << 17]byte + var x1039 [1 << 17]byte + var x1040 [1 << 17]byte + var x1041 [1 << 17]byte + var x1042 [1 << 17]byte + var x1043 [1 << 17]byte + var x1044 [1 << 17]byte + var x1045 [1 << 17]byte + var x1046 [1 << 17]byte + var x1047 [1 << 17]byte + var x1048 [1 << 17]byte + var x1049 [1 << 17]byte + var x1050 [1 << 17]byte + var x1051 [1 << 17]byte + var x1052 [1 << 17]byte + var x1053 [1 << 17]byte + var x1054 [1 << 17]byte + var x1055 [1 << 17]byte + var x1056 [1 << 17]byte + var x1057 [1 << 17]byte + var x1058 [1 << 17]byte + var x1059 [1 << 17]byte + var x1060 [1 << 17]byte + var x1061 [1 << 17]byte + var x1062 [1 << 17]byte + var x1063 [1 << 17]byte + var x1064 [1 << 17]byte + var x1065 [1 << 17]byte + var x1066 [1 << 17]byte + var x1067 [1 << 17]byte + var x1068 [1 << 17]byte + var x1069 [1 << 17]byte + var x1070 [1 << 17]byte + var x1071 [1 << 17]byte + var x1072 [1 << 17]byte + var x1073 [1 << 17]byte + var x1074 [1 << 17]byte + var x1075 [1 << 17]byte + var x1076 [1 << 17]byte + var x1077 [1 << 17]byte + var x1078 [1 << 17]byte + var x1079 [1 << 17]byte + var x1080 [1 << 17]byte + var x1081 [1 << 17]byte + var x1082 [1 << 17]byte + var x1083 [1 << 17]byte + var x1084 [1 << 17]byte + var x1085 [1 << 17]byte + var x1086 [1 << 17]byte + var x1087 [1 << 17]byte + var x1088 [1 << 17]byte + var x1089 [1 << 17]byte + var x1090 [1 << 17]byte + var x1091 [1 << 17]byte + var x1092 [1 << 17]byte + var x1093 [1 << 17]byte + var x1094 [1 << 17]byte + var x1095 [1 << 17]byte + var x1096 [1 << 17]byte + var x1097 [1 << 17]byte + var x1098 [1 << 17]byte + var x1099 [1 << 17]byte + var x1100 [1 << 17]byte + var x1101 [1 << 17]byte + var x1102 [1 << 17]byte + var x1103 [1 << 17]byte + var x1104 [1 << 17]byte + var x1105 [1 << 17]byte + var x1106 [1 << 17]byte + var x1107 [1 << 17]byte + var x1108 [1 << 17]byte + var x1109 [1 << 17]byte + var x1110 [1 << 17]byte + var x1111 [1 << 17]byte + var x1112 [1 << 17]byte + var x1113 [1 << 17]byte + var x1114 [1 << 17]byte + var x1115 [1 << 17]byte + var x1116 [1 << 17]byte + var x1117 [1 << 17]byte + var x1118 [1 << 17]byte + var x1119 [1 << 17]byte + var x1120 [1 << 17]byte + var x1121 [1 << 17]byte + var x1122 [1 << 17]byte + var x1123 [1 << 17]byte + var x1124 [1 << 17]byte + var x1125 [1 << 17]byte + var x1126 [1 << 17]byte + var x1127 [1 << 17]byte + var x1128 [1 << 17]byte + var x1129 [1 << 17]byte + var x1130 [1 << 17]byte + var x1131 [1 << 17]byte + var x1132 [1 << 17]byte + var x1133 [1 << 17]byte + var x1134 [1 << 17]byte + var x1135 [1 << 17]byte + var x1136 [1 << 17]byte + var x1137 [1 << 17]byte + var x1138 [1 << 17]byte + var x1139 [1 << 17]byte + var x1140 [1 << 17]byte + var x1141 [1 << 17]byte + var x1142 [1 << 17]byte + var x1143 [1 << 17]byte + var x1144 [1 << 17]byte + var x1145 [1 << 17]byte + var x1146 [1 << 17]byte + var x1147 [1 << 17]byte + var x1148 [1 << 17]byte + var x1149 [1 << 17]byte + var x1150 [1 << 17]byte + var x1151 [1 << 17]byte + var x1152 [1 << 17]byte + var x1153 [1 << 17]byte + var x1154 [1 << 17]byte + var x1155 [1 << 17]byte + var x1156 [1 << 17]byte + var x1157 [1 << 17]byte + var x1158 [1 << 17]byte + var x1159 [1 << 17]byte + var x1160 [1 << 17]byte + var x1161 [1 << 17]byte + var x1162 [1 << 17]byte + var x1163 [1 << 17]byte + var x1164 [1 << 17]byte + var x1165 [1 << 17]byte + var x1166 [1 << 17]byte + var x1167 [1 << 17]byte + var x1168 [1 << 17]byte + var x1169 [1 << 17]byte + var x1170 [1 << 17]byte + var x1171 [1 << 17]byte + var x1172 [1 << 17]byte + var x1173 [1 << 17]byte + var x1174 [1 << 17]byte + var x1175 [1 << 17]byte + var x1176 [1 << 17]byte + var x1177 [1 << 17]byte + var x1178 [1 << 17]byte + var x1179 [1 << 17]byte + var x1180 [1 << 17]byte + var x1181 [1 << 17]byte + var x1182 [1 << 17]byte + var x1183 [1 << 17]byte + var x1184 [1 << 17]byte + var x1185 [1 << 17]byte + var x1186 [1 << 17]byte + var x1187 [1 << 17]byte + var x1188 [1 << 17]byte + var x1189 [1 << 17]byte + var x1190 [1 << 17]byte + var x1191 [1 << 17]byte + var x1192 [1 << 17]byte + var x1193 [1 << 17]byte + var x1194 [1 << 17]byte + var x1195 [1 << 17]byte + var x1196 [1 << 17]byte + var x1197 [1 << 17]byte + var x1198 [1 << 17]byte + var x1199 [1 << 17]byte + var x1200 [1 << 17]byte + var x1201 [1 << 17]byte + var x1202 [1 << 17]byte + var x1203 [1 << 17]byte + var x1204 [1 << 17]byte + var x1205 [1 << 17]byte + var x1206 [1 << 17]byte + var x1207 [1 << 17]byte + var x1208 [1 << 17]byte + var x1209 [1 << 17]byte + var x1210 [1 << 17]byte + var x1211 [1 << 17]byte + var x1212 [1 << 17]byte + var x1213 [1 << 17]byte + var x1214 [1 << 17]byte + var x1215 [1 << 17]byte + var x1216 [1 << 17]byte + var x1217 [1 << 17]byte + var x1218 [1 << 17]byte + var x1219 [1 << 17]byte + var x1220 [1 << 17]byte + var x1221 [1 << 17]byte + var x1222 [1 << 17]byte + var x1223 [1 << 17]byte + var x1224 [1 << 17]byte + var x1225 [1 << 17]byte + var x1226 [1 << 17]byte + var x1227 [1 << 17]byte + var x1228 [1 << 17]byte + var x1229 [1 << 17]byte + var x1230 [1 << 17]byte + var x1231 [1 << 17]byte + var x1232 [1 << 17]byte + var x1233 [1 << 17]byte + var x1234 [1 << 17]byte + var x1235 [1 << 17]byte + var x1236 [1 << 17]byte + var x1237 [1 << 17]byte + var x1238 [1 << 17]byte + var x1239 [1 << 17]byte + var x1240 [1 << 17]byte + var x1241 [1 << 17]byte + var x1242 [1 << 17]byte + var x1243 [1 << 17]byte + var x1244 [1 << 17]byte + var x1245 [1 << 17]byte + var x1246 [1 << 17]byte + var x1247 [1 << 17]byte + var x1248 [1 << 17]byte + var x1249 [1 << 17]byte + var x1250 [1 << 17]byte + var x1251 [1 << 17]byte + var x1252 [1 << 17]byte + var x1253 [1 << 17]byte + var x1254 [1 << 17]byte + var x1255 [1 << 17]byte + var x1256 [1 << 17]byte + var x1257 [1 << 17]byte + var x1258 [1 << 17]byte + var x1259 [1 << 17]byte + var x1260 [1 << 17]byte + var x1261 [1 << 17]byte + var x1262 [1 << 17]byte + var x1263 [1 << 17]byte + var x1264 [1 << 17]byte + var x1265 [1 << 17]byte + var x1266 [1 << 17]byte + var x1267 [1 << 17]byte + var x1268 [1 << 17]byte + var x1269 [1 << 17]byte + var x1270 [1 << 17]byte + var x1271 [1 << 17]byte + var x1272 [1 << 17]byte + var x1273 [1 << 17]byte + var x1274 [1 << 17]byte + var x1275 [1 << 17]byte + var x1276 [1 << 17]byte + var x1277 [1 << 17]byte + var x1278 [1 << 17]byte + var x1279 [1 << 17]byte + var x1280 [1 << 17]byte + var x1281 [1 << 17]byte + var x1282 [1 << 17]byte + var x1283 [1 << 17]byte + var x1284 [1 << 17]byte + var x1285 [1 << 17]byte + var x1286 [1 << 17]byte + var x1287 [1 << 17]byte + var x1288 [1 << 17]byte + var x1289 [1 << 17]byte + var x1290 [1 << 17]byte + var x1291 [1 << 17]byte + var x1292 [1 << 17]byte + var x1293 [1 << 17]byte + var x1294 [1 << 17]byte + var x1295 [1 << 17]byte + var x1296 [1 << 17]byte + var x1297 [1 << 17]byte + var x1298 [1 << 17]byte + var x1299 [1 << 17]byte + var x1300 [1 << 17]byte + var x1301 [1 << 17]byte + var x1302 [1 << 17]byte + var x1303 [1 << 17]byte + var x1304 [1 << 17]byte + var x1305 [1 << 17]byte + var x1306 [1 << 17]byte + var x1307 [1 << 17]byte + var x1308 [1 << 17]byte + var x1309 [1 << 17]byte + var x1310 [1 << 17]byte + var x1311 [1 << 17]byte + var x1312 [1 << 17]byte + var x1313 [1 << 17]byte + var x1314 [1 << 17]byte + var x1315 [1 << 17]byte + var x1316 [1 << 17]byte + var x1317 [1 << 17]byte + var x1318 [1 << 17]byte + var x1319 [1 << 17]byte + var x1320 [1 << 17]byte + var x1321 [1 << 17]byte + var x1322 [1 << 17]byte + var x1323 [1 << 17]byte + var x1324 [1 << 17]byte + var x1325 [1 << 17]byte + var x1326 [1 << 17]byte + var x1327 [1 << 17]byte + var x1328 [1 << 17]byte + var x1329 [1 << 17]byte + var x1330 [1 << 17]byte + var x1331 [1 << 17]byte + var x1332 [1 << 17]byte + var x1333 [1 << 17]byte + var x1334 [1 << 17]byte + var x1335 [1 << 17]byte + var x1336 [1 << 17]byte + var x1337 [1 << 17]byte + var x1338 [1 << 17]byte + var x1339 [1 << 17]byte + var x1340 [1 << 17]byte + var x1341 [1 << 17]byte + var x1342 [1 << 17]byte + var x1343 [1 << 17]byte + var x1344 [1 << 17]byte + var x1345 [1 << 17]byte + var x1346 [1 << 17]byte + var x1347 [1 << 17]byte + var x1348 [1 << 17]byte + var x1349 [1 << 17]byte + var x1350 [1 << 17]byte + var x1351 [1 << 17]byte + var x1352 [1 << 17]byte + var x1353 [1 << 17]byte + var x1354 [1 << 17]byte + var x1355 [1 << 17]byte + var x1356 [1 << 17]byte + var x1357 [1 << 17]byte + var x1358 [1 << 17]byte + var x1359 [1 << 17]byte + var x1360 [1 << 17]byte + var x1361 [1 << 17]byte + var x1362 [1 << 17]byte + var x1363 [1 << 17]byte + var x1364 [1 << 17]byte + var x1365 [1 << 17]byte + var x1366 [1 << 17]byte + var x1367 [1 << 17]byte + var x1368 [1 << 17]byte + var x1369 [1 << 17]byte + var x1370 [1 << 17]byte + var x1371 [1 << 17]byte + var x1372 [1 << 17]byte + var x1373 [1 << 17]byte + var x1374 [1 << 17]byte + var x1375 [1 << 17]byte + var x1376 [1 << 17]byte + var x1377 [1 << 17]byte + var x1378 [1 << 17]byte + var x1379 [1 << 17]byte + var x1380 [1 << 17]byte + var x1381 [1 << 17]byte + var x1382 [1 << 17]byte + var x1383 [1 << 17]byte + var x1384 [1 << 17]byte + var x1385 [1 << 17]byte + var x1386 [1 << 17]byte + var x1387 [1 << 17]byte + var x1388 [1 << 17]byte + var x1389 [1 << 17]byte + var x1390 [1 << 17]byte + var x1391 [1 << 17]byte + var x1392 [1 << 17]byte + var x1393 [1 << 17]byte + var x1394 [1 << 17]byte + var x1395 [1 << 17]byte + var x1396 [1 << 17]byte + var x1397 [1 << 17]byte + var x1398 [1 << 17]byte + var x1399 [1 << 17]byte + var x1400 [1 << 17]byte + var x1401 [1 << 17]byte + var x1402 [1 << 17]byte + var x1403 [1 << 17]byte + var x1404 [1 << 17]byte + var x1405 [1 << 17]byte + var x1406 [1 << 17]byte + var x1407 [1 << 17]byte + var x1408 [1 << 17]byte + var x1409 [1 << 17]byte + var x1410 [1 << 17]byte + var x1411 [1 << 17]byte + var x1412 [1 << 17]byte + var x1413 [1 << 17]byte + var x1414 [1 << 17]byte + var x1415 [1 << 17]byte + var x1416 [1 << 17]byte + var x1417 [1 << 17]byte + var x1418 [1 << 17]byte + var x1419 [1 << 17]byte + var x1420 [1 << 17]byte + var x1421 [1 << 17]byte + var x1422 [1 << 17]byte + var x1423 [1 << 17]byte + var x1424 [1 << 17]byte + var x1425 [1 << 17]byte + var x1426 [1 << 17]byte + var x1427 [1 << 17]byte + var x1428 [1 << 17]byte + var x1429 [1 << 17]byte + var x1430 [1 << 17]byte + var x1431 [1 << 17]byte + var x1432 [1 << 17]byte + var x1433 [1 << 17]byte + var x1434 [1 << 17]byte + var x1435 [1 << 17]byte + var x1436 [1 << 17]byte + var x1437 [1 << 17]byte + var x1438 [1 << 17]byte + var x1439 [1 << 17]byte + var x1440 [1 << 17]byte + var x1441 [1 << 17]byte + var x1442 [1 << 17]byte + var x1443 [1 << 17]byte + var x1444 [1 << 17]byte + var x1445 [1 << 17]byte + var x1446 [1 << 17]byte + var x1447 [1 << 17]byte + var x1448 [1 << 17]byte + var x1449 [1 << 17]byte + var x1450 [1 << 17]byte + var x1451 [1 << 17]byte + var x1452 [1 << 17]byte + var x1453 [1 << 17]byte + var x1454 [1 << 17]byte + var x1455 [1 << 17]byte + var x1456 [1 << 17]byte + var x1457 [1 << 17]byte + var x1458 [1 << 17]byte + var x1459 [1 << 17]byte + var x1460 [1 << 17]byte + var x1461 [1 << 17]byte + var x1462 [1 << 17]byte + var x1463 [1 << 17]byte + var x1464 [1 << 17]byte + var x1465 [1 << 17]byte + var x1466 [1 << 17]byte + var x1467 [1 << 17]byte + var x1468 [1 << 17]byte + var x1469 [1 << 17]byte + var x1470 [1 << 17]byte + var x1471 [1 << 17]byte + var x1472 [1 << 17]byte + var x1473 [1 << 17]byte + var x1474 [1 << 17]byte + var x1475 [1 << 17]byte + var x1476 [1 << 17]byte + var x1477 [1 << 17]byte + var x1478 [1 << 17]byte + var x1479 [1 << 17]byte + var x1480 [1 << 17]byte + var x1481 [1 << 17]byte + var x1482 [1 << 17]byte + var x1483 [1 << 17]byte + var x1484 [1 << 17]byte + var x1485 [1 << 17]byte + var x1486 [1 << 17]byte + var x1487 [1 << 17]byte + var x1488 [1 << 17]byte + var x1489 [1 << 17]byte + var x1490 [1 << 17]byte + var x1491 [1 << 17]byte + var x1492 [1 << 17]byte + var x1493 [1 << 17]byte + var x1494 [1 << 17]byte + var x1495 [1 << 17]byte + var x1496 [1 << 17]byte + var x1497 [1 << 17]byte + var x1498 [1 << 17]byte + var x1499 [1 << 17]byte + var x1500 [1 << 17]byte + var x1501 [1 << 17]byte + var x1502 [1 << 17]byte + var x1503 [1 << 17]byte + var x1504 [1 << 17]byte + var x1505 [1 << 17]byte + var x1506 [1 << 17]byte + var x1507 [1 << 17]byte + var x1508 [1 << 17]byte + var x1509 [1 << 17]byte + var x1510 [1 << 17]byte + var x1511 [1 << 17]byte + var x1512 [1 << 17]byte + var x1513 [1 << 17]byte + var x1514 [1 << 17]byte + var x1515 [1 << 17]byte + var x1516 [1 << 17]byte + var x1517 [1 << 17]byte + var x1518 [1 << 17]byte + var x1519 [1 << 17]byte + var x1520 [1 << 17]byte + var x1521 [1 << 17]byte + var x1522 [1 << 17]byte + var x1523 [1 << 17]byte + var x1524 [1 << 17]byte + var x1525 [1 << 17]byte + var x1526 [1 << 17]byte + var x1527 [1 << 17]byte + var x1528 [1 << 17]byte + var x1529 [1 << 17]byte + var x1530 [1 << 17]byte + var x1531 [1 << 17]byte + var x1532 [1 << 17]byte + var x1533 [1 << 17]byte + var x1534 [1 << 17]byte + var x1535 [1 << 17]byte + var x1536 [1 << 17]byte + var x1537 [1 << 17]byte + var x1538 [1 << 17]byte + var x1539 [1 << 17]byte + var x1540 [1 << 17]byte + var x1541 [1 << 17]byte + var x1542 [1 << 17]byte + var x1543 [1 << 17]byte + var x1544 [1 << 17]byte + var x1545 [1 << 17]byte + var x1546 [1 << 17]byte + var x1547 [1 << 17]byte + var x1548 [1 << 17]byte + var x1549 [1 << 17]byte + var x1550 [1 << 17]byte + var x1551 [1 << 17]byte + var x1552 [1 << 17]byte + var x1553 [1 << 17]byte + var x1554 [1 << 17]byte + var x1555 [1 << 17]byte + var x1556 [1 << 17]byte + var x1557 [1 << 17]byte + var x1558 [1 << 17]byte + var x1559 [1 << 17]byte + var x1560 [1 << 17]byte + var x1561 [1 << 17]byte + var x1562 [1 << 17]byte + var x1563 [1 << 17]byte + var x1564 [1 << 17]byte + var x1565 [1 << 17]byte + var x1566 [1 << 17]byte + var x1567 [1 << 17]byte + var x1568 [1 << 17]byte + var x1569 [1 << 17]byte + var x1570 [1 << 17]byte + var x1571 [1 << 17]byte + var x1572 [1 << 17]byte + var x1573 [1 << 17]byte + var x1574 [1 << 17]byte + var x1575 [1 << 17]byte + var x1576 [1 << 17]byte + var x1577 [1 << 17]byte + var x1578 [1 << 17]byte + var x1579 [1 << 17]byte + var x1580 [1 << 17]byte + var x1581 [1 << 17]byte + var x1582 [1 << 17]byte + var x1583 [1 << 17]byte + var x1584 [1 << 17]byte + var x1585 [1 << 17]byte + var x1586 [1 << 17]byte + var x1587 [1 << 17]byte + var x1588 [1 << 17]byte + var x1589 [1 << 17]byte + var x1590 [1 << 17]byte + var x1591 [1 << 17]byte + var x1592 [1 << 17]byte + var x1593 [1 << 17]byte + var x1594 [1 << 17]byte + var x1595 [1 << 17]byte + var x1596 [1 << 17]byte + var x1597 [1 << 17]byte + var x1598 [1 << 17]byte + var x1599 [1 << 17]byte + var x1600 [1 << 17]byte + var x1601 [1 << 17]byte + var x1602 [1 << 17]byte + var x1603 [1 << 17]byte + var x1604 [1 << 17]byte + var x1605 [1 << 17]byte + var x1606 [1 << 17]byte + var x1607 [1 << 17]byte + var x1608 [1 << 17]byte + var x1609 [1 << 17]byte + var x1610 [1 << 17]byte + var x1611 [1 << 17]byte + var x1612 [1 << 17]byte + var x1613 [1 << 17]byte + var x1614 [1 << 17]byte + var x1615 [1 << 17]byte + var x1616 [1 << 17]byte + var x1617 [1 << 17]byte + var x1618 [1 << 17]byte + var x1619 [1 << 17]byte + var x1620 [1 << 17]byte + var x1621 [1 << 17]byte + var x1622 [1 << 17]byte + var x1623 [1 << 17]byte + var x1624 [1 << 17]byte + var x1625 [1 << 17]byte + var x1626 [1 << 17]byte + var x1627 [1 << 17]byte + var x1628 [1 << 17]byte + var x1629 [1 << 17]byte + var x1630 [1 << 17]byte + var x1631 [1 << 17]byte + var x1632 [1 << 17]byte + var x1633 [1 << 17]byte + var x1634 [1 << 17]byte + var x1635 [1 << 17]byte + var x1636 [1 << 17]byte + var x1637 [1 << 17]byte + var x1638 [1 << 17]byte + var x1639 [1 << 17]byte + var x1640 [1 << 17]byte + var x1641 [1 << 17]byte + var x1642 [1 << 17]byte + var x1643 [1 << 17]byte + var x1644 [1 << 17]byte + var x1645 [1 << 17]byte + var x1646 [1 << 17]byte + var x1647 [1 << 17]byte + var x1648 [1 << 17]byte + var x1649 [1 << 17]byte + var x1650 [1 << 17]byte + var x1651 [1 << 17]byte + var x1652 [1 << 17]byte + var x1653 [1 << 17]byte + var x1654 [1 << 17]byte + var x1655 [1 << 17]byte + var x1656 [1 << 17]byte + var x1657 [1 << 17]byte + var x1658 [1 << 17]byte + var x1659 [1 << 17]byte + var x1660 [1 << 17]byte + var x1661 [1 << 17]byte + var x1662 [1 << 17]byte + var x1663 [1 << 17]byte + var x1664 [1 << 17]byte + var x1665 [1 << 17]byte + var x1666 [1 << 17]byte + var x1667 [1 << 17]byte + var x1668 [1 << 17]byte + var x1669 [1 << 17]byte + var x1670 [1 << 17]byte + var x1671 [1 << 17]byte + var x1672 [1 << 17]byte + var x1673 [1 << 17]byte + var x1674 [1 << 17]byte + var x1675 [1 << 17]byte + var x1676 [1 << 17]byte + var x1677 [1 << 17]byte + var x1678 [1 << 17]byte + var x1679 [1 << 17]byte + var x1680 [1 << 17]byte + var x1681 [1 << 17]byte + var x1682 [1 << 17]byte + var x1683 [1 << 17]byte + var x1684 [1 << 17]byte + var x1685 [1 << 17]byte + var x1686 [1 << 17]byte + var x1687 [1 << 17]byte + var x1688 [1 << 17]byte + var x1689 [1 << 17]byte + var x1690 [1 << 17]byte + var x1691 [1 << 17]byte + var x1692 [1 << 17]byte + var x1693 [1 << 17]byte + var x1694 [1 << 17]byte + var x1695 [1 << 17]byte + var x1696 [1 << 17]byte + var x1697 [1 << 17]byte + var x1698 [1 << 17]byte + var x1699 [1 << 17]byte + var x1700 [1 << 17]byte + var x1701 [1 << 17]byte + var x1702 [1 << 17]byte + var x1703 [1 << 17]byte + var x1704 [1 << 17]byte + var x1705 [1 << 17]byte + var x1706 [1 << 17]byte + var x1707 [1 << 17]byte + var x1708 [1 << 17]byte + var x1709 [1 << 17]byte + var x1710 [1 << 17]byte + var x1711 [1 << 17]byte + var x1712 [1 << 17]byte + var x1713 [1 << 17]byte + var x1714 [1 << 17]byte + var x1715 [1 << 17]byte + var x1716 [1 << 17]byte + var x1717 [1 << 17]byte + var x1718 [1 << 17]byte + var x1719 [1 << 17]byte + var x1720 [1 << 17]byte + var x1721 [1 << 17]byte + var x1722 [1 << 17]byte + var x1723 [1 << 17]byte + var x1724 [1 << 17]byte + var x1725 [1 << 17]byte + var x1726 [1 << 17]byte + var x1727 [1 << 17]byte + var x1728 [1 << 17]byte + var x1729 [1 << 17]byte + var x1730 [1 << 17]byte + var x1731 [1 << 17]byte + var x1732 [1 << 17]byte + var x1733 [1 << 17]byte + var x1734 [1 << 17]byte + var x1735 [1 << 17]byte + var x1736 [1 << 17]byte + var x1737 [1 << 17]byte + var x1738 [1 << 17]byte + var x1739 [1 << 17]byte + var x1740 [1 << 17]byte + var x1741 [1 << 17]byte + var x1742 [1 << 17]byte + var x1743 [1 << 17]byte + var x1744 [1 << 17]byte + var x1745 [1 << 17]byte + var x1746 [1 << 17]byte + var x1747 [1 << 17]byte + var x1748 [1 << 17]byte + var x1749 [1 << 17]byte + var x1750 [1 << 17]byte + var x1751 [1 << 17]byte + var x1752 [1 << 17]byte + var x1753 [1 << 17]byte + var x1754 [1 << 17]byte + var x1755 [1 << 17]byte + var x1756 [1 << 17]byte + var x1757 [1 << 17]byte + var x1758 [1 << 17]byte + var x1759 [1 << 17]byte + var x1760 [1 << 17]byte + var x1761 [1 << 17]byte + var x1762 [1 << 17]byte + var x1763 [1 << 17]byte + var x1764 [1 << 17]byte + var x1765 [1 << 17]byte + var x1766 [1 << 17]byte + var x1767 [1 << 17]byte + var x1768 [1 << 17]byte + var x1769 [1 << 17]byte + var x1770 [1 << 17]byte + var x1771 [1 << 17]byte + var x1772 [1 << 17]byte + var x1773 [1 << 17]byte + var x1774 [1 << 17]byte + var x1775 [1 << 17]byte + var x1776 [1 << 17]byte + var x1777 [1 << 17]byte + var x1778 [1 << 17]byte + var x1779 [1 << 17]byte + var x1780 [1 << 17]byte + var x1781 [1 << 17]byte + var x1782 [1 << 17]byte + var x1783 [1 << 17]byte + var x1784 [1 << 17]byte + var x1785 [1 << 17]byte + var x1786 [1 << 17]byte + var x1787 [1 << 17]byte + var x1788 [1 << 17]byte + var x1789 [1 << 17]byte + var x1790 [1 << 17]byte + var x1791 [1 << 17]byte + var x1792 [1 << 17]byte + var x1793 [1 << 17]byte + var x1794 [1 << 17]byte + var x1795 [1 << 17]byte + var x1796 [1 << 17]byte + var x1797 [1 << 17]byte + var x1798 [1 << 17]byte + var x1799 [1 << 17]byte + var x1800 [1 << 17]byte + var x1801 [1 << 17]byte + var x1802 [1 << 17]byte + var x1803 [1 << 17]byte + var x1804 [1 << 17]byte + var x1805 [1 << 17]byte + var x1806 [1 << 17]byte + var x1807 [1 << 17]byte + var x1808 [1 << 17]byte + var x1809 [1 << 17]byte + var x1810 [1 << 17]byte + var x1811 [1 << 17]byte + var x1812 [1 << 17]byte + var x1813 [1 << 17]byte + var x1814 [1 << 17]byte + var x1815 [1 << 17]byte + var x1816 [1 << 17]byte + var x1817 [1 << 17]byte + var x1818 [1 << 17]byte + var x1819 [1 << 17]byte + var x1820 [1 << 17]byte + var x1821 [1 << 17]byte + var x1822 [1 << 17]byte + var x1823 [1 << 17]byte + var x1824 [1 << 17]byte + var x1825 [1 << 17]byte + var x1826 [1 << 17]byte + var x1827 [1 << 17]byte + var x1828 [1 << 17]byte + var x1829 [1 << 17]byte + var x1830 [1 << 17]byte + var x1831 [1 << 17]byte + var x1832 [1 << 17]byte + var x1833 [1 << 17]byte + var x1834 [1 << 17]byte + var x1835 [1 << 17]byte + var x1836 [1 << 17]byte + var x1837 [1 << 17]byte + var x1838 [1 << 17]byte + var x1839 [1 << 17]byte + var x1840 [1 << 17]byte + var x1841 [1 << 17]byte + var x1842 [1 << 17]byte + var x1843 [1 << 17]byte + var x1844 [1 << 17]byte + var x1845 [1 << 17]byte + var x1846 [1 << 17]byte + var x1847 [1 << 17]byte + var x1848 [1 << 17]byte + var x1849 [1 << 17]byte + var x1850 [1 << 17]byte + var x1851 [1 << 17]byte + var x1852 [1 << 17]byte + var x1853 [1 << 17]byte + var x1854 [1 << 17]byte + var x1855 [1 << 17]byte + var x1856 [1 << 17]byte + var x1857 [1 << 17]byte + var x1858 [1 << 17]byte + var x1859 [1 << 17]byte + var x1860 [1 << 17]byte + var x1861 [1 << 17]byte + var x1862 [1 << 17]byte + var x1863 [1 << 17]byte + var x1864 [1 << 17]byte + var x1865 [1 << 17]byte + var x1866 [1 << 17]byte + var x1867 [1 << 17]byte + var x1868 [1 << 17]byte + var x1869 [1 << 17]byte + var x1870 [1 << 17]byte + var x1871 [1 << 17]byte + var x1872 [1 << 17]byte + var x1873 [1 << 17]byte + var x1874 [1 << 17]byte + var x1875 [1 << 17]byte + var x1876 [1 << 17]byte + var x1877 [1 << 17]byte + var x1878 [1 << 17]byte + var x1879 [1 << 17]byte + var x1880 [1 << 17]byte + var x1881 [1 << 17]byte + var x1882 [1 << 17]byte + var x1883 [1 << 17]byte + var x1884 [1 << 17]byte + var x1885 [1 << 17]byte + var x1886 [1 << 17]byte + var x1887 [1 << 17]byte + var x1888 [1 << 17]byte + var x1889 [1 << 17]byte + var x1890 [1 << 17]byte + var x1891 [1 << 17]byte + var x1892 [1 << 17]byte + var x1893 [1 << 17]byte + var x1894 [1 << 17]byte + var x1895 [1 << 17]byte + var x1896 [1 << 17]byte + var x1897 [1 << 17]byte + var x1898 [1 << 17]byte + var x1899 [1 << 17]byte + var x1900 [1 << 17]byte + var x1901 [1 << 17]byte + var x1902 [1 << 17]byte + var x1903 [1 << 17]byte + var x1904 [1 << 17]byte + var x1905 [1 << 17]byte + var x1906 [1 << 17]byte + var x1907 [1 << 17]byte + var x1908 [1 << 17]byte + var x1909 [1 << 17]byte + var x1910 [1 << 17]byte + var x1911 [1 << 17]byte + var x1912 [1 << 17]byte + var x1913 [1 << 17]byte + var x1914 [1 << 17]byte + var x1915 [1 << 17]byte + var x1916 [1 << 17]byte + var x1917 [1 << 17]byte + var x1918 [1 << 17]byte + var x1919 [1 << 17]byte + var x1920 [1 << 17]byte + var x1921 [1 << 17]byte + var x1922 [1 << 17]byte + var x1923 [1 << 17]byte + var x1924 [1 << 17]byte + var x1925 [1 << 17]byte + var x1926 [1 << 17]byte + var x1927 [1 << 17]byte + var x1928 [1 << 17]byte + var x1929 [1 << 17]byte + var x1930 [1 << 17]byte + var x1931 [1 << 17]byte + var x1932 [1 << 17]byte + var x1933 [1 << 17]byte + var x1934 [1 << 17]byte + var x1935 [1 << 17]byte + var x1936 [1 << 17]byte + var x1937 [1 << 17]byte + var x1938 [1 << 17]byte + var x1939 [1 << 17]byte + var x1940 [1 << 17]byte + var x1941 [1 << 17]byte + var x1942 [1 << 17]byte + var x1943 [1 << 17]byte + var x1944 [1 << 17]byte + var x1945 [1 << 17]byte + var x1946 [1 << 17]byte + var x1947 [1 << 17]byte + var x1948 [1 << 17]byte + var x1949 [1 << 17]byte + var x1950 [1 << 17]byte + var x1951 [1 << 17]byte + var x1952 [1 << 17]byte + var x1953 [1 << 17]byte + var x1954 [1 << 17]byte + var x1955 [1 << 17]byte + var x1956 [1 << 17]byte + var x1957 [1 << 17]byte + var x1958 [1 << 17]byte + var x1959 [1 << 17]byte + var x1960 [1 << 17]byte + var x1961 [1 << 17]byte + var x1962 [1 << 17]byte + var x1963 [1 << 17]byte + var x1964 [1 << 17]byte + var x1965 [1 << 17]byte + var x1966 [1 << 17]byte + var x1967 [1 << 17]byte + var x1968 [1 << 17]byte + var x1969 [1 << 17]byte + var x1970 [1 << 17]byte + var x1971 [1 << 17]byte + var x1972 [1 << 17]byte + var x1973 [1 << 17]byte + var x1974 [1 << 17]byte + var x1975 [1 << 17]byte + var x1976 [1 << 17]byte + var x1977 [1 << 17]byte + var x1978 [1 << 17]byte + var x1979 [1 << 17]byte + var x1980 [1 << 17]byte + var x1981 [1 << 17]byte + var x1982 [1 << 17]byte + var x1983 [1 << 17]byte + var x1984 [1 << 17]byte + var x1985 [1 << 17]byte + var x1986 [1 << 17]byte + var x1987 [1 << 17]byte + var x1988 [1 << 17]byte + var x1989 [1 << 17]byte + var x1990 [1 << 17]byte + var x1991 [1 << 17]byte + var x1992 [1 << 17]byte + var x1993 [1 << 17]byte + var x1994 [1 << 17]byte + var x1995 [1 << 17]byte + var x1996 [1 << 17]byte + var x1997 [1 << 17]byte + var x1998 [1 << 17]byte + var x1999 [1 << 17]byte + var x2000 [1 << 17]byte + var x2001 [1 << 17]byte + var x2002 [1 << 17]byte + var x2003 [1 << 17]byte + var x2004 [1 << 17]byte + var x2005 [1 << 17]byte + var x2006 [1 << 17]byte + var x2007 [1 << 17]byte + var x2008 [1 << 17]byte + var x2009 [1 << 17]byte + var x2010 [1 << 17]byte + var x2011 [1 << 17]byte + var x2012 [1 << 17]byte + var x2013 [1 << 17]byte + var x2014 [1 << 17]byte + var x2015 [1 << 17]byte + var x2016 [1 << 17]byte + var x2017 [1 << 17]byte + var x2018 [1 << 17]byte + var x2019 [1 << 17]byte + var x2020 [1 << 17]byte + var x2021 [1 << 17]byte + var x2022 [1 << 17]byte + var x2023 [1 << 17]byte + var x2024 [1 << 17]byte + var x2025 [1 << 17]byte + var x2026 [1 << 17]byte + var x2027 [1 << 17]byte + var x2028 [1 << 17]byte + var x2029 [1 << 17]byte + var x2030 [1 << 17]byte + var x2031 [1 << 17]byte + var x2032 [1 << 17]byte + var x2033 [1 << 17]byte + var x2034 [1 << 17]byte + var x2035 [1 << 17]byte + var x2036 [1 << 17]byte + var x2037 [1 << 17]byte + var x2038 [1 << 17]byte + var x2039 [1 << 17]byte + var x2040 [1 << 17]byte + var x2041 [1 << 17]byte + var x2042 [1 << 17]byte + var x2043 [1 << 17]byte + var x2044 [1 << 17]byte + var x2045 [1 << 17]byte + var x2046 [1 << 17]byte + var x2047 [1 << 17]byte + var x2048 [1 << 17]byte + var x2049 [1 << 17]byte + var x2050 [1 << 17]byte + var x2051 [1 << 17]byte + var x2052 [1 << 17]byte + var x2053 [1 << 17]byte + var x2054 [1 << 17]byte + var x2055 [1 << 17]byte + var x2056 [1 << 17]byte + var x2057 [1 << 17]byte + var x2058 [1 << 17]byte + var x2059 [1 << 17]byte + var x2060 [1 << 17]byte + var x2061 [1 << 17]byte + var x2062 [1 << 17]byte + var x2063 [1 << 17]byte + var x2064 [1 << 17]byte + var x2065 [1 << 17]byte + var x2066 [1 << 17]byte + var x2067 [1 << 17]byte + var x2068 [1 << 17]byte + var x2069 [1 << 17]byte + var x2070 [1 << 17]byte + var x2071 [1 << 17]byte + var x2072 [1 << 17]byte + var x2073 [1 << 17]byte + var x2074 [1 << 17]byte + var x2075 [1 << 17]byte + var x2076 [1 << 17]byte + var x2077 [1 << 17]byte + var x2078 [1 << 17]byte + var x2079 [1 << 17]byte + var x2080 [1 << 17]byte + var x2081 [1 << 17]byte + var x2082 [1 << 17]byte + var x2083 [1 << 17]byte + var x2084 [1 << 17]byte + var x2085 [1 << 17]byte + var x2086 [1 << 17]byte + var x2087 [1 << 17]byte + var x2088 [1 << 17]byte + var x2089 [1 << 17]byte + var x2090 [1 << 17]byte + var x2091 [1 << 17]byte + var x2092 [1 << 17]byte + var x2093 [1 << 17]byte + var x2094 [1 << 17]byte + var x2095 [1 << 17]byte + var x2096 [1 << 17]byte + var x2097 [1 << 17]byte + var x2098 [1 << 17]byte + var x2099 [1 << 17]byte + var x2100 [1 << 17]byte + var x2101 [1 << 17]byte + var x2102 [1 << 17]byte + var x2103 [1 << 17]byte + var x2104 [1 << 17]byte + var x2105 [1 << 17]byte + var x2106 [1 << 17]byte + var x2107 [1 << 17]byte + var x2108 [1 << 17]byte + var x2109 [1 << 17]byte + var x2110 [1 << 17]byte + var x2111 [1 << 17]byte + var x2112 [1 << 17]byte + var x2113 [1 << 17]byte + var x2114 [1 << 17]byte + var x2115 [1 << 17]byte + var x2116 [1 << 17]byte + var x2117 [1 << 17]byte + var x2118 [1 << 17]byte + var x2119 [1 << 17]byte + var x2120 [1 << 17]byte + var x2121 [1 << 17]byte + var x2122 [1 << 17]byte + var x2123 [1 << 17]byte + var x2124 [1 << 17]byte + var x2125 [1 << 17]byte + var x2126 [1 << 17]byte + var x2127 [1 << 17]byte + var x2128 [1 << 17]byte + var x2129 [1 << 17]byte + var x2130 [1 << 17]byte + var x2131 [1 << 17]byte + var x2132 [1 << 17]byte + var x2133 [1 << 17]byte + var x2134 [1 << 17]byte + var x2135 [1 << 17]byte + var x2136 [1 << 17]byte + var x2137 [1 << 17]byte + var x2138 [1 << 17]byte + var x2139 [1 << 17]byte + var x2140 [1 << 17]byte + var x2141 [1 << 17]byte + var x2142 [1 << 17]byte + var x2143 [1 << 17]byte + var x2144 [1 << 17]byte + var x2145 [1 << 17]byte + var x2146 [1 << 17]byte + var x2147 [1 << 17]byte + var x2148 [1 << 17]byte + var x2149 [1 << 17]byte + var x2150 [1 << 17]byte + var x2151 [1 << 17]byte + var x2152 [1 << 17]byte + var x2153 [1 << 17]byte + var x2154 [1 << 17]byte + var x2155 [1 << 17]byte + var x2156 [1 << 17]byte + var x2157 [1 << 17]byte + var x2158 [1 << 17]byte + var x2159 [1 << 17]byte + var x2160 [1 << 17]byte + var x2161 [1 << 17]byte + var x2162 [1 << 17]byte + var x2163 [1 << 17]byte + var x2164 [1 << 17]byte + var x2165 [1 << 17]byte + var x2166 [1 << 17]byte + var x2167 [1 << 17]byte + var x2168 [1 << 17]byte + var x2169 [1 << 17]byte + var x2170 [1 << 17]byte + var x2171 [1 << 17]byte + var x2172 [1 << 17]byte + var x2173 [1 << 17]byte + var x2174 [1 << 17]byte + var x2175 [1 << 17]byte + var x2176 [1 << 17]byte + var x2177 [1 << 17]byte + var x2178 [1 << 17]byte + var x2179 [1 << 17]byte + var x2180 [1 << 17]byte + var x2181 [1 << 17]byte + var x2182 [1 << 17]byte + var x2183 [1 << 17]byte + var x2184 [1 << 17]byte + var x2185 [1 << 17]byte + var x2186 [1 << 17]byte + var x2187 [1 << 17]byte + var x2188 [1 << 17]byte + var x2189 [1 << 17]byte + var x2190 [1 << 17]byte + var x2191 [1 << 17]byte + var x2192 [1 << 17]byte + var x2193 [1 << 17]byte + var x2194 [1 << 17]byte + var x2195 [1 << 17]byte + var x2196 [1 << 17]byte + var x2197 [1 << 17]byte + var x2198 [1 << 17]byte + var x2199 [1 << 17]byte + var x2200 [1 << 17]byte + var x2201 [1 << 17]byte + var x2202 [1 << 17]byte + var x2203 [1 << 17]byte + var x2204 [1 << 17]byte + var x2205 [1 << 17]byte + var x2206 [1 << 17]byte + var x2207 [1 << 17]byte + var x2208 [1 << 17]byte + var x2209 [1 << 17]byte + var x2210 [1 << 17]byte + var x2211 [1 << 17]byte + var x2212 [1 << 17]byte + var x2213 [1 << 17]byte + var x2214 [1 << 17]byte + var x2215 [1 << 17]byte + var x2216 [1 << 17]byte + var x2217 [1 << 17]byte + var x2218 [1 << 17]byte + var x2219 [1 << 17]byte + var x2220 [1 << 17]byte + var x2221 [1 << 17]byte + var x2222 [1 << 17]byte + var x2223 [1 << 17]byte + var x2224 [1 << 17]byte + var x2225 [1 << 17]byte + var x2226 [1 << 17]byte + var x2227 [1 << 17]byte + var x2228 [1 << 17]byte + var x2229 [1 << 17]byte + var x2230 [1 << 17]byte + var x2231 [1 << 17]byte + var x2232 [1 << 17]byte + var x2233 [1 << 17]byte + var x2234 [1 << 17]byte + var x2235 [1 << 17]byte + var x2236 [1 << 17]byte + var x2237 [1 << 17]byte + var x2238 [1 << 17]byte + var x2239 [1 << 17]byte + var x2240 [1 << 17]byte + var x2241 [1 << 17]byte + var x2242 [1 << 17]byte + var x2243 [1 << 17]byte + var x2244 [1 << 17]byte + var x2245 [1 << 17]byte + var x2246 [1 << 17]byte + var x2247 [1 << 17]byte + var x2248 [1 << 17]byte + var x2249 [1 << 17]byte + var x2250 [1 << 17]byte + var x2251 [1 << 17]byte + var x2252 [1 << 17]byte + var x2253 [1 << 17]byte + var x2254 [1 << 17]byte + var x2255 [1 << 17]byte + var x2256 [1 << 17]byte + var x2257 [1 << 17]byte + var x2258 [1 << 17]byte + var x2259 [1 << 17]byte + var x2260 [1 << 17]byte + var x2261 [1 << 17]byte + var x2262 [1 << 17]byte + var x2263 [1 << 17]byte + var x2264 [1 << 17]byte + var x2265 [1 << 17]byte + var x2266 [1 << 17]byte + var x2267 [1 << 17]byte + var x2268 [1 << 17]byte + var x2269 [1 << 17]byte + var x2270 [1 << 17]byte + var x2271 [1 << 17]byte + var x2272 [1 << 17]byte + var x2273 [1 << 17]byte + var x2274 [1 << 17]byte + var x2275 [1 << 17]byte + var x2276 [1 << 17]byte + var x2277 [1 << 17]byte + var x2278 [1 << 17]byte + var x2279 [1 << 17]byte + var x2280 [1 << 17]byte + var x2281 [1 << 17]byte + var x2282 [1 << 17]byte + var x2283 [1 << 17]byte + var x2284 [1 << 17]byte + var x2285 [1 << 17]byte + var x2286 [1 << 17]byte + var x2287 [1 << 17]byte + var x2288 [1 << 17]byte + var x2289 [1 << 17]byte + var x2290 [1 << 17]byte + var x2291 [1 << 17]byte + var x2292 [1 << 17]byte + var x2293 [1 << 17]byte + var x2294 [1 << 17]byte + var x2295 [1 << 17]byte + var x2296 [1 << 17]byte + var x2297 [1 << 17]byte + var x2298 [1 << 17]byte + var x2299 [1 << 17]byte + var x2300 [1 << 17]byte + var x2301 [1 << 17]byte + var x2302 [1 << 17]byte + var x2303 [1 << 17]byte + var x2304 [1 << 17]byte + var x2305 [1 << 17]byte + var x2306 [1 << 17]byte + var x2307 [1 << 17]byte + var x2308 [1 << 17]byte + var x2309 [1 << 17]byte + var x2310 [1 << 17]byte + var x2311 [1 << 17]byte + var x2312 [1 << 17]byte + var x2313 [1 << 17]byte + var x2314 [1 << 17]byte + var x2315 [1 << 17]byte + var x2316 [1 << 17]byte + var x2317 [1 << 17]byte + var x2318 [1 << 17]byte + var x2319 [1 << 17]byte + var x2320 [1 << 17]byte + var x2321 [1 << 17]byte + var x2322 [1 << 17]byte + var x2323 [1 << 17]byte + var x2324 [1 << 17]byte + var x2325 [1 << 17]byte + var x2326 [1 << 17]byte + var x2327 [1 << 17]byte + var x2328 [1 << 17]byte + var x2329 [1 << 17]byte + var x2330 [1 << 17]byte + var x2331 [1 << 17]byte + var x2332 [1 << 17]byte + var x2333 [1 << 17]byte + var x2334 [1 << 17]byte + var x2335 [1 << 17]byte + var x2336 [1 << 17]byte + var x2337 [1 << 17]byte + var x2338 [1 << 17]byte + var x2339 [1 << 17]byte + var x2340 [1 << 17]byte + var x2341 [1 << 17]byte + var x2342 [1 << 17]byte + var x2343 [1 << 17]byte + var x2344 [1 << 17]byte + var x2345 [1 << 17]byte + var x2346 [1 << 17]byte + var x2347 [1 << 17]byte + var x2348 [1 << 17]byte + var x2349 [1 << 17]byte + var x2350 [1 << 17]byte + var x2351 [1 << 17]byte + var x2352 [1 << 17]byte + var x2353 [1 << 17]byte + var x2354 [1 << 17]byte + var x2355 [1 << 17]byte + var x2356 [1 << 17]byte + var x2357 [1 << 17]byte + var x2358 [1 << 17]byte + var x2359 [1 << 17]byte + var x2360 [1 << 17]byte + var x2361 [1 << 17]byte + var x2362 [1 << 17]byte + var x2363 [1 << 17]byte + var x2364 [1 << 17]byte + var x2365 [1 << 17]byte + var x2366 [1 << 17]byte + var x2367 [1 << 17]byte + var x2368 [1 << 17]byte + var x2369 [1 << 17]byte + var x2370 [1 << 17]byte + var x2371 [1 << 17]byte + var x2372 [1 << 17]byte + var x2373 [1 << 17]byte + var x2374 [1 << 17]byte + var x2375 [1 << 17]byte + var x2376 [1 << 17]byte + var x2377 [1 << 17]byte + var x2378 [1 << 17]byte + var x2379 [1 << 17]byte + var x2380 [1 << 17]byte + var x2381 [1 << 17]byte + var x2382 [1 << 17]byte + var x2383 [1 << 17]byte + var x2384 [1 << 17]byte + var x2385 [1 << 17]byte + var x2386 [1 << 17]byte + var x2387 [1 << 17]byte + var x2388 [1 << 17]byte + var x2389 [1 << 17]byte + var x2390 [1 << 17]byte + var x2391 [1 << 17]byte + var x2392 [1 << 17]byte + var x2393 [1 << 17]byte + var x2394 [1 << 17]byte + var x2395 [1 << 17]byte + var x2396 [1 << 17]byte + var x2397 [1 << 17]byte + var x2398 [1 << 17]byte + var x2399 [1 << 17]byte + var x2400 [1 << 17]byte + var x2401 [1 << 17]byte + var x2402 [1 << 17]byte + var x2403 [1 << 17]byte + var x2404 [1 << 17]byte + var x2405 [1 << 17]byte + var x2406 [1 << 17]byte + var x2407 [1 << 17]byte + var x2408 [1 << 17]byte + var x2409 [1 << 17]byte + var x2410 [1 << 17]byte + var x2411 [1 << 17]byte + var x2412 [1 << 17]byte + var x2413 [1 << 17]byte + var x2414 [1 << 17]byte + var x2415 [1 << 17]byte + var x2416 [1 << 17]byte + var x2417 [1 << 17]byte + var x2418 [1 << 17]byte + var x2419 [1 << 17]byte + var x2420 [1 << 17]byte + var x2421 [1 << 17]byte + var x2422 [1 << 17]byte + var x2423 [1 << 17]byte + var x2424 [1 << 17]byte + var x2425 [1 << 17]byte + var x2426 [1 << 17]byte + var x2427 [1 << 17]byte + var x2428 [1 << 17]byte + var x2429 [1 << 17]byte + var x2430 [1 << 17]byte + var x2431 [1 << 17]byte + var x2432 [1 << 17]byte + var x2433 [1 << 17]byte + var x2434 [1 << 17]byte + var x2435 [1 << 17]byte + var x2436 [1 << 17]byte + var x2437 [1 << 17]byte + var x2438 [1 << 17]byte + var x2439 [1 << 17]byte + var x2440 [1 << 17]byte + var x2441 [1 << 17]byte + var x2442 [1 << 17]byte + var x2443 [1 << 17]byte + var x2444 [1 << 17]byte + var x2445 [1 << 17]byte + var x2446 [1 << 17]byte + var x2447 [1 << 17]byte + var x2448 [1 << 17]byte + var x2449 [1 << 17]byte + var x2450 [1 << 17]byte + var x2451 [1 << 17]byte + var x2452 [1 << 17]byte + var x2453 [1 << 17]byte + var x2454 [1 << 17]byte + var x2455 [1 << 17]byte + var x2456 [1 << 17]byte + var x2457 [1 << 17]byte + var x2458 [1 << 17]byte + var x2459 [1 << 17]byte + var x2460 [1 << 17]byte + var x2461 [1 << 17]byte + var x2462 [1 << 17]byte + var x2463 [1 << 17]byte + var x2464 [1 << 17]byte + var x2465 [1 << 17]byte + var x2466 [1 << 17]byte + var x2467 [1 << 17]byte + var x2468 [1 << 17]byte + var x2469 [1 << 17]byte + var x2470 [1 << 17]byte + var x2471 [1 << 17]byte + var x2472 [1 << 17]byte + var x2473 [1 << 17]byte + var x2474 [1 << 17]byte + var x2475 [1 << 17]byte + var x2476 [1 << 17]byte + var x2477 [1 << 17]byte + var x2478 [1 << 17]byte + var x2479 [1 << 17]byte + var x2480 [1 << 17]byte + var x2481 [1 << 17]byte + var x2482 [1 << 17]byte + var x2483 [1 << 17]byte + var x2484 [1 << 17]byte + var x2485 [1 << 17]byte + var x2486 [1 << 17]byte + var x2487 [1 << 17]byte + var x2488 [1 << 17]byte + var x2489 [1 << 17]byte + var x2490 [1 << 17]byte + var x2491 [1 << 17]byte + var x2492 [1 << 17]byte + var x2493 [1 << 17]byte + var x2494 [1 << 17]byte + var x2495 [1 << 17]byte + var x2496 [1 << 17]byte + var x2497 [1 << 17]byte + var x2498 [1 << 17]byte + var x2499 [1 << 17]byte + var x2500 [1 << 17]byte + var x2501 [1 << 17]byte + var x2502 [1 << 17]byte + var x2503 [1 << 17]byte + var x2504 [1 << 17]byte + var x2505 [1 << 17]byte + var x2506 [1 << 17]byte + var x2507 [1 << 17]byte + var x2508 [1 << 17]byte + var x2509 [1 << 17]byte + var x2510 [1 << 17]byte + var x2511 [1 << 17]byte + var x2512 [1 << 17]byte + var x2513 [1 << 17]byte + var x2514 [1 << 17]byte + var x2515 [1 << 17]byte + var x2516 [1 << 17]byte + var x2517 [1 << 17]byte + var x2518 [1 << 17]byte + var x2519 [1 << 17]byte + var x2520 [1 << 17]byte + var x2521 [1 << 17]byte + var x2522 [1 << 17]byte + var x2523 [1 << 17]byte + var x2524 [1 << 17]byte + var x2525 [1 << 17]byte + var x2526 [1 << 17]byte + var x2527 [1 << 17]byte + var x2528 [1 << 17]byte + var x2529 [1 << 17]byte + var x2530 [1 << 17]byte + var x2531 [1 << 17]byte + var x2532 [1 << 17]byte + var x2533 [1 << 17]byte + var x2534 [1 << 17]byte + var x2535 [1 << 17]byte + var x2536 [1 << 17]byte + var x2537 [1 << 17]byte + var x2538 [1 << 17]byte + var x2539 [1 << 17]byte + var x2540 [1 << 17]byte + var x2541 [1 << 17]byte + var x2542 [1 << 17]byte + var x2543 [1 << 17]byte + var x2544 [1 << 17]byte + var x2545 [1 << 17]byte + var x2546 [1 << 17]byte + var x2547 [1 << 17]byte + var x2548 [1 << 17]byte + var x2549 [1 << 17]byte + var x2550 [1 << 17]byte + var x2551 [1 << 17]byte + var x2552 [1 << 17]byte + var x2553 [1 << 17]byte + var x2554 [1 << 17]byte + var x2555 [1 << 17]byte + var x2556 [1 << 17]byte + var x2557 [1 << 17]byte + var x2558 [1 << 17]byte + var x2559 [1 << 17]byte + var x2560 [1 << 17]byte + var x2561 [1 << 17]byte + var x2562 [1 << 17]byte + var x2563 [1 << 17]byte + var x2564 [1 << 17]byte + var x2565 [1 << 17]byte + var x2566 [1 << 17]byte + var x2567 [1 << 17]byte + var x2568 [1 << 17]byte + var x2569 [1 << 17]byte + var x2570 [1 << 17]byte + var x2571 [1 << 17]byte + var x2572 [1 << 17]byte + var x2573 [1 << 17]byte + var x2574 [1 << 17]byte + var x2575 [1 << 17]byte + var x2576 [1 << 17]byte + var x2577 [1 << 17]byte + var x2578 [1 << 17]byte + var x2579 [1 << 17]byte + var x2580 [1 << 17]byte + var x2581 [1 << 17]byte + var x2582 [1 << 17]byte + var x2583 [1 << 17]byte + var x2584 [1 << 17]byte + var x2585 [1 << 17]byte + var x2586 [1 << 17]byte + var x2587 [1 << 17]byte + var x2588 [1 << 17]byte + var x2589 [1 << 17]byte + var x2590 [1 << 17]byte + var x2591 [1 << 17]byte + var x2592 [1 << 17]byte + var x2593 [1 << 17]byte + var x2594 [1 << 17]byte + var x2595 [1 << 17]byte + var x2596 [1 << 17]byte + var x2597 [1 << 17]byte + var x2598 [1 << 17]byte + var x2599 [1 << 17]byte + var x2600 [1 << 17]byte + var x2601 [1 << 17]byte + var x2602 [1 << 17]byte + var x2603 [1 << 17]byte + var x2604 [1 << 17]byte + var x2605 [1 << 17]byte + var x2606 [1 << 17]byte + var x2607 [1 << 17]byte + var x2608 [1 << 17]byte + var x2609 [1 << 17]byte + var x2610 [1 << 17]byte + var x2611 [1 << 17]byte + var x2612 [1 << 17]byte + var x2613 [1 << 17]byte + var x2614 [1 << 17]byte + var x2615 [1 << 17]byte + var x2616 [1 << 17]byte + var x2617 [1 << 17]byte + var x2618 [1 << 17]byte + var x2619 [1 << 17]byte + var x2620 [1 << 17]byte + var x2621 [1 << 17]byte + var x2622 [1 << 17]byte + var x2623 [1 << 17]byte + var x2624 [1 << 17]byte + var x2625 [1 << 17]byte + var x2626 [1 << 17]byte + var x2627 [1 << 17]byte + var x2628 [1 << 17]byte + var x2629 [1 << 17]byte + var x2630 [1 << 17]byte + var x2631 [1 << 17]byte + var x2632 [1 << 17]byte + var x2633 [1 << 17]byte + var x2634 [1 << 17]byte + var x2635 [1 << 17]byte + var x2636 [1 << 17]byte + var x2637 [1 << 17]byte + var x2638 [1 << 17]byte + var x2639 [1 << 17]byte + var x2640 [1 << 17]byte + var x2641 [1 << 17]byte + var x2642 [1 << 17]byte + var x2643 [1 << 17]byte + var x2644 [1 << 17]byte + var x2645 [1 << 17]byte + var x2646 [1 << 17]byte + var x2647 [1 << 17]byte + var x2648 [1 << 17]byte + var x2649 [1 << 17]byte + var x2650 [1 << 17]byte + var x2651 [1 << 17]byte + var x2652 [1 << 17]byte + var x2653 [1 << 17]byte + var x2654 [1 << 17]byte + var x2655 [1 << 17]byte + var x2656 [1 << 17]byte + var x2657 [1 << 17]byte + var x2658 [1 << 17]byte + var x2659 [1 << 17]byte + var x2660 [1 << 17]byte + var x2661 [1 << 17]byte + var x2662 [1 << 17]byte + var x2663 [1 << 17]byte + var x2664 [1 << 17]byte + var x2665 [1 << 17]byte + var x2666 [1 << 17]byte + var x2667 [1 << 17]byte + var x2668 [1 << 17]byte + var x2669 [1 << 17]byte + var x2670 [1 << 17]byte + var x2671 [1 << 17]byte + var x2672 [1 << 17]byte + var x2673 [1 << 17]byte + var x2674 [1 << 17]byte + var x2675 [1 << 17]byte + var x2676 [1 << 17]byte + var x2677 [1 << 17]byte + var x2678 [1 << 17]byte + var x2679 [1 << 17]byte + var x2680 [1 << 17]byte + var x2681 [1 << 17]byte + var x2682 [1 << 17]byte + var x2683 [1 << 17]byte + var x2684 [1 << 17]byte + var x2685 [1 << 17]byte + var x2686 [1 << 17]byte + var x2687 [1 << 17]byte + var x2688 [1 << 17]byte + var x2689 [1 << 17]byte + var x2690 [1 << 17]byte + var x2691 [1 << 17]byte + var x2692 [1 << 17]byte + var x2693 [1 << 17]byte + var x2694 [1 << 17]byte + var x2695 [1 << 17]byte + var x2696 [1 << 17]byte + var x2697 [1 << 17]byte + var x2698 [1 << 17]byte + var x2699 [1 << 17]byte + var x2700 [1 << 17]byte + var x2701 [1 << 17]byte + var x2702 [1 << 17]byte + var x2703 [1 << 17]byte + var x2704 [1 << 17]byte + var x2705 [1 << 17]byte + var x2706 [1 << 17]byte + var x2707 [1 << 17]byte + var x2708 [1 << 17]byte + var x2709 [1 << 17]byte + var x2710 [1 << 17]byte + var x2711 [1 << 17]byte + var x2712 [1 << 17]byte + var x2713 [1 << 17]byte + var x2714 [1 << 17]byte + var x2715 [1 << 17]byte + var x2716 [1 << 17]byte + var x2717 [1 << 17]byte + var x2718 [1 << 17]byte + var x2719 [1 << 17]byte + var x2720 [1 << 17]byte + var x2721 [1 << 17]byte + var x2722 [1 << 17]byte + var x2723 [1 << 17]byte + var x2724 [1 << 17]byte + var x2725 [1 << 17]byte + var x2726 [1 << 17]byte + var x2727 [1 << 17]byte + var x2728 [1 << 17]byte + var x2729 [1 << 17]byte + var x2730 [1 << 17]byte + var x2731 [1 << 17]byte + var x2732 [1 << 17]byte + var x2733 [1 << 17]byte + var x2734 [1 << 17]byte + var x2735 [1 << 17]byte + var x2736 [1 << 17]byte + var x2737 [1 << 17]byte + var x2738 [1 << 17]byte + var x2739 [1 << 17]byte + var x2740 [1 << 17]byte + var x2741 [1 << 17]byte + var x2742 [1 << 17]byte + var x2743 [1 << 17]byte + var x2744 [1 << 17]byte + var x2745 [1 << 17]byte + var x2746 [1 << 17]byte + var x2747 [1 << 17]byte + var x2748 [1 << 17]byte + var x2749 [1 << 17]byte + var x2750 [1 << 17]byte + var x2751 [1 << 17]byte + var x2752 [1 << 17]byte + var x2753 [1 << 17]byte + var x2754 [1 << 17]byte + var x2755 [1 << 17]byte + var x2756 [1 << 17]byte + var x2757 [1 << 17]byte + var x2758 [1 << 17]byte + var x2759 [1 << 17]byte + var x2760 [1 << 17]byte + var x2761 [1 << 17]byte + var x2762 [1 << 17]byte + var x2763 [1 << 17]byte + var x2764 [1 << 17]byte + var x2765 [1 << 17]byte + var x2766 [1 << 17]byte + var x2767 [1 << 17]byte + var x2768 [1 << 17]byte + var x2769 [1 << 17]byte + var x2770 [1 << 17]byte + var x2771 [1 << 17]byte + var x2772 [1 << 17]byte + var x2773 [1 << 17]byte + var x2774 [1 << 17]byte + var x2775 [1 << 17]byte + var x2776 [1 << 17]byte + var x2777 [1 << 17]byte + var x2778 [1 << 17]byte + var x2779 [1 << 17]byte + var x2780 [1 << 17]byte + var x2781 [1 << 17]byte + var x2782 [1 << 17]byte + var x2783 [1 << 17]byte + var x2784 [1 << 17]byte + var x2785 [1 << 17]byte + var x2786 [1 << 17]byte + var x2787 [1 << 17]byte + var x2788 [1 << 17]byte + var x2789 [1 << 17]byte + var x2790 [1 << 17]byte + var x2791 [1 << 17]byte + var x2792 [1 << 17]byte + var x2793 [1 << 17]byte + var x2794 [1 << 17]byte + var x2795 [1 << 17]byte + var x2796 [1 << 17]byte + var x2797 [1 << 17]byte + var x2798 [1 << 17]byte + var x2799 [1 << 17]byte + var x2800 [1 << 17]byte + var x2801 [1 << 17]byte + var x2802 [1 << 17]byte + var x2803 [1 << 17]byte + var x2804 [1 << 17]byte + var x2805 [1 << 17]byte + var x2806 [1 << 17]byte + var x2807 [1 << 17]byte + var x2808 [1 << 17]byte + var x2809 [1 << 17]byte + var x2810 [1 << 17]byte + var x2811 [1 << 17]byte + var x2812 [1 << 17]byte + var x2813 [1 << 17]byte + var x2814 [1 << 17]byte + var x2815 [1 << 17]byte + var x2816 [1 << 17]byte + var x2817 [1 << 17]byte + var x2818 [1 << 17]byte + var x2819 [1 << 17]byte + var x2820 [1 << 17]byte + var x2821 [1 << 17]byte + var x2822 [1 << 17]byte + var x2823 [1 << 17]byte + var x2824 [1 << 17]byte + var x2825 [1 << 17]byte + var x2826 [1 << 17]byte + var x2827 [1 << 17]byte + var x2828 [1 << 17]byte + var x2829 [1 << 17]byte + var x2830 [1 << 17]byte + var x2831 [1 << 17]byte + var x2832 [1 << 17]byte + var x2833 [1 << 17]byte + var x2834 [1 << 17]byte + var x2835 [1 << 17]byte + var x2836 [1 << 17]byte + var x2837 [1 << 17]byte + var x2838 [1 << 17]byte + var x2839 [1 << 17]byte + var x2840 [1 << 17]byte + var x2841 [1 << 17]byte + var x2842 [1 << 17]byte + var x2843 [1 << 17]byte + var x2844 [1 << 17]byte + var x2845 [1 << 17]byte + var x2846 [1 << 17]byte + var x2847 [1 << 17]byte + var x2848 [1 << 17]byte + var x2849 [1 << 17]byte + var x2850 [1 << 17]byte + var x2851 [1 << 17]byte + var x2852 [1 << 17]byte + var x2853 [1 << 17]byte + var x2854 [1 << 17]byte + var x2855 [1 << 17]byte + var x2856 [1 << 17]byte + var x2857 [1 << 17]byte + var x2858 [1 << 17]byte + var x2859 [1 << 17]byte + var x2860 [1 << 17]byte + var x2861 [1 << 17]byte + var x2862 [1 << 17]byte + var x2863 [1 << 17]byte + var x2864 [1 << 17]byte + var x2865 [1 << 17]byte + var x2866 [1 << 17]byte + var x2867 [1 << 17]byte + var x2868 [1 << 17]byte + var x2869 [1 << 17]byte + var x2870 [1 << 17]byte + var x2871 [1 << 17]byte + var x2872 [1 << 17]byte + var x2873 [1 << 17]byte + var x2874 [1 << 17]byte + var x2875 [1 << 17]byte + var x2876 [1 << 17]byte + var x2877 [1 << 17]byte + var x2878 [1 << 17]byte + var x2879 [1 << 17]byte + var x2880 [1 << 17]byte + var x2881 [1 << 17]byte + var x2882 [1 << 17]byte + var x2883 [1 << 17]byte + var x2884 [1 << 17]byte + var x2885 [1 << 17]byte + var x2886 [1 << 17]byte + var x2887 [1 << 17]byte + var x2888 [1 << 17]byte + var x2889 [1 << 17]byte + var x2890 [1 << 17]byte + var x2891 [1 << 17]byte + var x2892 [1 << 17]byte + var x2893 [1 << 17]byte + var x2894 [1 << 17]byte + var x2895 [1 << 17]byte + var x2896 [1 << 17]byte + var x2897 [1 << 17]byte + var x2898 [1 << 17]byte + var x2899 [1 << 17]byte + var x2900 [1 << 17]byte + var x2901 [1 << 17]byte + var x2902 [1 << 17]byte + var x2903 [1 << 17]byte + var x2904 [1 << 17]byte + var x2905 [1 << 17]byte + var x2906 [1 << 17]byte + var x2907 [1 << 17]byte + var x2908 [1 << 17]byte + var x2909 [1 << 17]byte + var x2910 [1 << 17]byte + var x2911 [1 << 17]byte + var x2912 [1 << 17]byte + var x2913 [1 << 17]byte + var x2914 [1 << 17]byte + var x2915 [1 << 17]byte + var x2916 [1 << 17]byte + var x2917 [1 << 17]byte + var x2918 [1 << 17]byte + var x2919 [1 << 17]byte + var x2920 [1 << 17]byte + var x2921 [1 << 17]byte + var x2922 [1 << 17]byte + var x2923 [1 << 17]byte + var x2924 [1 << 17]byte + var x2925 [1 << 17]byte + var x2926 [1 << 17]byte + var x2927 [1 << 17]byte + var x2928 [1 << 17]byte + var x2929 [1 << 17]byte + var x2930 [1 << 17]byte + var x2931 [1 << 17]byte + var x2932 [1 << 17]byte + var x2933 [1 << 17]byte + var x2934 [1 << 17]byte + var x2935 [1 << 17]byte + var x2936 [1 << 17]byte + var x2937 [1 << 17]byte + var x2938 [1 << 17]byte + var x2939 [1 << 17]byte + var x2940 [1 << 17]byte + var x2941 [1 << 17]byte + var x2942 [1 << 17]byte + var x2943 [1 << 17]byte + var x2944 [1 << 17]byte + var x2945 [1 << 17]byte + var x2946 [1 << 17]byte + var x2947 [1 << 17]byte + var x2948 [1 << 17]byte + var x2949 [1 << 17]byte + var x2950 [1 << 17]byte + var x2951 [1 << 17]byte + var x2952 [1 << 17]byte + var x2953 [1 << 17]byte + var x2954 [1 << 17]byte + var x2955 [1 << 17]byte + var x2956 [1 << 17]byte + var x2957 [1 << 17]byte + var x2958 [1 << 17]byte + var x2959 [1 << 17]byte + var x2960 [1 << 17]byte + var x2961 [1 << 17]byte + var x2962 [1 << 17]byte + var x2963 [1 << 17]byte + var x2964 [1 << 17]byte + var x2965 [1 << 17]byte + var x2966 [1 << 17]byte + var x2967 [1 << 17]byte + var x2968 [1 << 17]byte + var x2969 [1 << 17]byte + var x2970 [1 << 17]byte + var x2971 [1 << 17]byte + var x2972 [1 << 17]byte + var x2973 [1 << 17]byte + var x2974 [1 << 17]byte + var x2975 [1 << 17]byte + var x2976 [1 << 17]byte + var x2977 [1 << 17]byte + var x2978 [1 << 17]byte + var x2979 [1 << 17]byte + var x2980 [1 << 17]byte + var x2981 [1 << 17]byte + var x2982 [1 << 17]byte + var x2983 [1 << 17]byte + var x2984 [1 << 17]byte + var x2985 [1 << 17]byte + var x2986 [1 << 17]byte + var x2987 [1 << 17]byte + var x2988 [1 << 17]byte + var x2989 [1 << 17]byte + var x2990 [1 << 17]byte + var x2991 [1 << 17]byte + var x2992 [1 << 17]byte + var x2993 [1 << 17]byte + var x2994 [1 << 17]byte + var x2995 [1 << 17]byte + var x2996 [1 << 17]byte + var x2997 [1 << 17]byte + var x2998 [1 << 17]byte + var x2999 [1 << 17]byte + var x3000 [1 << 17]byte + var x3001 [1 << 17]byte + var x3002 [1 << 17]byte + var x3003 [1 << 17]byte + var x3004 [1 << 17]byte + var x3005 [1 << 17]byte + var x3006 [1 << 17]byte + var x3007 [1 << 17]byte + var x3008 [1 << 17]byte + var x3009 [1 << 17]byte + var x3010 [1 << 17]byte + var x3011 [1 << 17]byte + var x3012 [1 << 17]byte + var x3013 [1 << 17]byte + var x3014 [1 << 17]byte + var x3015 [1 << 17]byte + var x3016 [1 << 17]byte + var x3017 [1 << 17]byte + var x3018 [1 << 17]byte + var x3019 [1 << 17]byte + var x3020 [1 << 17]byte + var x3021 [1 << 17]byte + var x3022 [1 << 17]byte + var x3023 [1 << 17]byte + var x3024 [1 << 17]byte + var x3025 [1 << 17]byte + var x3026 [1 << 17]byte + var x3027 [1 << 17]byte + var x3028 [1 << 17]byte + var x3029 [1 << 17]byte + var x3030 [1 << 17]byte + var x3031 [1 << 17]byte + var x3032 [1 << 17]byte + var x3033 [1 << 17]byte + var x3034 [1 << 17]byte + var x3035 [1 << 17]byte + var x3036 [1 << 17]byte + var x3037 [1 << 17]byte + var x3038 [1 << 17]byte + var x3039 [1 << 17]byte + var x3040 [1 << 17]byte + var x3041 [1 << 17]byte + var x3042 [1 << 17]byte + var x3043 [1 << 17]byte + var x3044 [1 << 17]byte + var x3045 [1 << 17]byte + var x3046 [1 << 17]byte + var x3047 [1 << 17]byte + var x3048 [1 << 17]byte + var x3049 [1 << 17]byte + var x3050 [1 << 17]byte + var x3051 [1 << 17]byte + var x3052 [1 << 17]byte + var x3053 [1 << 17]byte + var x3054 [1 << 17]byte + var x3055 [1 << 17]byte + var x3056 [1 << 17]byte + var x3057 [1 << 17]byte + var x3058 [1 << 17]byte + var x3059 [1 << 17]byte + var x3060 [1 << 17]byte + var x3061 [1 << 17]byte + var x3062 [1 << 17]byte + var x3063 [1 << 17]byte + var x3064 [1 << 17]byte + var x3065 [1 << 17]byte + var x3066 [1 << 17]byte + var x3067 [1 << 17]byte + var x3068 [1 << 17]byte + var x3069 [1 << 17]byte + var x3070 [1 << 17]byte + var x3071 [1 << 17]byte + var x3072 [1 << 17]byte + var x3073 [1 << 17]byte + var x3074 [1 << 17]byte + var x3075 [1 << 17]byte + var x3076 [1 << 17]byte + var x3077 [1 << 17]byte + var x3078 [1 << 17]byte + var x3079 [1 << 17]byte + var x3080 [1 << 17]byte + var x3081 [1 << 17]byte + var x3082 [1 << 17]byte + var x3083 [1 << 17]byte + var x3084 [1 << 17]byte + var x3085 [1 << 17]byte + var x3086 [1 << 17]byte + var x3087 [1 << 17]byte + var x3088 [1 << 17]byte + var x3089 [1 << 17]byte + var x3090 [1 << 17]byte + var x3091 [1 << 17]byte + var x3092 [1 << 17]byte + var x3093 [1 << 17]byte + var x3094 [1 << 17]byte + var x3095 [1 << 17]byte + var x3096 [1 << 17]byte + var x3097 [1 << 17]byte + var x3098 [1 << 17]byte + var x3099 [1 << 17]byte + var x3100 [1 << 17]byte + var x3101 [1 << 17]byte + var x3102 [1 << 17]byte + var x3103 [1 << 17]byte + var x3104 [1 << 17]byte + var x3105 [1 << 17]byte + var x3106 [1 << 17]byte + var x3107 [1 << 17]byte + var x3108 [1 << 17]byte + var x3109 [1 << 17]byte + var x3110 [1 << 17]byte + var x3111 [1 << 17]byte + var x3112 [1 << 17]byte + var x3113 [1 << 17]byte + var x3114 [1 << 17]byte + var x3115 [1 << 17]byte + var x3116 [1 << 17]byte + var x3117 [1 << 17]byte + var x3118 [1 << 17]byte + var x3119 [1 << 17]byte + var x3120 [1 << 17]byte + var x3121 [1 << 17]byte + var x3122 [1 << 17]byte + var x3123 [1 << 17]byte + var x3124 [1 << 17]byte + var x3125 [1 << 17]byte + var x3126 [1 << 17]byte + var x3127 [1 << 17]byte + var x3128 [1 << 17]byte + var x3129 [1 << 17]byte + var x3130 [1 << 17]byte + var x3131 [1 << 17]byte + var x3132 [1 << 17]byte + var x3133 [1 << 17]byte + var x3134 [1 << 17]byte + var x3135 [1 << 17]byte + var x3136 [1 << 17]byte + var x3137 [1 << 17]byte + var x3138 [1 << 17]byte + var x3139 [1 << 17]byte + var x3140 [1 << 17]byte + var x3141 [1 << 17]byte + var x3142 [1 << 17]byte + var x3143 [1 << 17]byte + var x3144 [1 << 17]byte + var x3145 [1 << 17]byte + var x3146 [1 << 17]byte + var x3147 [1 << 17]byte + var x3148 [1 << 17]byte + var x3149 [1 << 17]byte + var x3150 [1 << 17]byte + var x3151 [1 << 17]byte + var x3152 [1 << 17]byte + var x3153 [1 << 17]byte + var x3154 [1 << 17]byte + var x3155 [1 << 17]byte + var x3156 [1 << 17]byte + var x3157 [1 << 17]byte + var x3158 [1 << 17]byte + var x3159 [1 << 17]byte + var x3160 [1 << 17]byte + var x3161 [1 << 17]byte + var x3162 [1 << 17]byte + var x3163 [1 << 17]byte + var x3164 [1 << 17]byte + var x3165 [1 << 17]byte + var x3166 [1 << 17]byte + var x3167 [1 << 17]byte + var x3168 [1 << 17]byte + var x3169 [1 << 17]byte + var x3170 [1 << 17]byte + var x3171 [1 << 17]byte + var x3172 [1 << 17]byte + var x3173 [1 << 17]byte + var x3174 [1 << 17]byte + var x3175 [1 << 17]byte + var x3176 [1 << 17]byte + var x3177 [1 << 17]byte + var x3178 [1 << 17]byte + var x3179 [1 << 17]byte + var x3180 [1 << 17]byte + var x3181 [1 << 17]byte + var x3182 [1 << 17]byte + var x3183 [1 << 17]byte + var x3184 [1 << 17]byte + var x3185 [1 << 17]byte + var x3186 [1 << 17]byte + var x3187 [1 << 17]byte + var x3188 [1 << 17]byte + var x3189 [1 << 17]byte + var x3190 [1 << 17]byte + var x3191 [1 << 17]byte + var x3192 [1 << 17]byte + var x3193 [1 << 17]byte + var x3194 [1 << 17]byte + var x3195 [1 << 17]byte + var x3196 [1 << 17]byte + var x3197 [1 << 17]byte + var x3198 [1 << 17]byte + var x3199 [1 << 17]byte + var x3200 [1 << 17]byte + var x3201 [1 << 17]byte + var x3202 [1 << 17]byte + var x3203 [1 << 17]byte + var x3204 [1 << 17]byte + var x3205 [1 << 17]byte + var x3206 [1 << 17]byte + var x3207 [1 << 17]byte + var x3208 [1 << 17]byte + var x3209 [1 << 17]byte + var x3210 [1 << 17]byte + var x3211 [1 << 17]byte + var x3212 [1 << 17]byte + var x3213 [1 << 17]byte + var x3214 [1 << 17]byte + var x3215 [1 << 17]byte + var x3216 [1 << 17]byte + var x3217 [1 << 17]byte + var x3218 [1 << 17]byte + var x3219 [1 << 17]byte + var x3220 [1 << 17]byte + var x3221 [1 << 17]byte + var x3222 [1 << 17]byte + var x3223 [1 << 17]byte + var x3224 [1 << 17]byte + var x3225 [1 << 17]byte + var x3226 [1 << 17]byte + var x3227 [1 << 17]byte + var x3228 [1 << 17]byte + var x3229 [1 << 17]byte + var x3230 [1 << 17]byte + var x3231 [1 << 17]byte + var x3232 [1 << 17]byte + var x3233 [1 << 17]byte + var x3234 [1 << 17]byte + var x3235 [1 << 17]byte + var x3236 [1 << 17]byte + var x3237 [1 << 17]byte + var x3238 [1 << 17]byte + var x3239 [1 << 17]byte + var x3240 [1 << 17]byte + var x3241 [1 << 17]byte + var x3242 [1 << 17]byte + var x3243 [1 << 17]byte + var x3244 [1 << 17]byte + var x3245 [1 << 17]byte + var x3246 [1 << 17]byte + var x3247 [1 << 17]byte + var x3248 [1 << 17]byte + var x3249 [1 << 17]byte + var x3250 [1 << 17]byte + var x3251 [1 << 17]byte + var x3252 [1 << 17]byte + var x3253 [1 << 17]byte + var x3254 [1 << 17]byte + var x3255 [1 << 17]byte + var x3256 [1 << 17]byte + var x3257 [1 << 17]byte + var x3258 [1 << 17]byte + var x3259 [1 << 17]byte + var x3260 [1 << 17]byte + var x3261 [1 << 17]byte + var x3262 [1 << 17]byte + var x3263 [1 << 17]byte + var x3264 [1 << 17]byte + var x3265 [1 << 17]byte + var x3266 [1 << 17]byte + var x3267 [1 << 17]byte + var x3268 [1 << 17]byte + var x3269 [1 << 17]byte + var x3270 [1 << 17]byte + var x3271 [1 << 17]byte + var x3272 [1 << 17]byte + var x3273 [1 << 17]byte + var x3274 [1 << 17]byte + var x3275 [1 << 17]byte + var x3276 [1 << 17]byte + var x3277 [1 << 17]byte + var x3278 [1 << 17]byte + var x3279 [1 << 17]byte + var x3280 [1 << 17]byte + var x3281 [1 << 17]byte + var x3282 [1 << 17]byte + var x3283 [1 << 17]byte + var x3284 [1 << 17]byte + var x3285 [1 << 17]byte + var x3286 [1 << 17]byte + var x3287 [1 << 17]byte + var x3288 [1 << 17]byte + var x3289 [1 << 17]byte + var x3290 [1 << 17]byte + var x3291 [1 << 17]byte + var x3292 [1 << 17]byte + var x3293 [1 << 17]byte + var x3294 [1 << 17]byte + var x3295 [1 << 17]byte + var x3296 [1 << 17]byte + var x3297 [1 << 17]byte + var x3298 [1 << 17]byte + var x3299 [1 << 17]byte + var x3300 [1 << 17]byte + var x3301 [1 << 17]byte + var x3302 [1 << 17]byte + var x3303 [1 << 17]byte + var x3304 [1 << 17]byte + var x3305 [1 << 17]byte + var x3306 [1 << 17]byte + var x3307 [1 << 17]byte + var x3308 [1 << 17]byte + var x3309 [1 << 17]byte + var x3310 [1 << 17]byte + var x3311 [1 << 17]byte + var x3312 [1 << 17]byte + var x3313 [1 << 17]byte + var x3314 [1 << 17]byte + var x3315 [1 << 17]byte + var x3316 [1 << 17]byte + var x3317 [1 << 17]byte + var x3318 [1 << 17]byte + var x3319 [1 << 17]byte + var x3320 [1 << 17]byte + var x3321 [1 << 17]byte + var x3322 [1 << 17]byte + var x3323 [1 << 17]byte + var x3324 [1 << 17]byte + var x3325 [1 << 17]byte + var x3326 [1 << 17]byte + var x3327 [1 << 17]byte + var x3328 [1 << 17]byte + var x3329 [1 << 17]byte + var x3330 [1 << 17]byte + var x3331 [1 << 17]byte + var x3332 [1 << 17]byte + var x3333 [1 << 17]byte + var x3334 [1 << 17]byte + var x3335 [1 << 17]byte + var x3336 [1 << 17]byte + var x3337 [1 << 17]byte + var x3338 [1 << 17]byte + var x3339 [1 << 17]byte + var x3340 [1 << 17]byte + var x3341 [1 << 17]byte + var x3342 [1 << 17]byte + var x3343 [1 << 17]byte + var x3344 [1 << 17]byte + var x3345 [1 << 17]byte + var x3346 [1 << 17]byte + var x3347 [1 << 17]byte + var x3348 [1 << 17]byte + var x3349 [1 << 17]byte + var x3350 [1 << 17]byte + var x3351 [1 << 17]byte + var x3352 [1 << 17]byte + var x3353 [1 << 17]byte + var x3354 [1 << 17]byte + var x3355 [1 << 17]byte + var x3356 [1 << 17]byte + var x3357 [1 << 17]byte + var x3358 [1 << 17]byte + var x3359 [1 << 17]byte + var x3360 [1 << 17]byte + var x3361 [1 << 17]byte + var x3362 [1 << 17]byte + var x3363 [1 << 17]byte + var x3364 [1 << 17]byte + var x3365 [1 << 17]byte + var x3366 [1 << 17]byte + var x3367 [1 << 17]byte + var x3368 [1 << 17]byte + var x3369 [1 << 17]byte + var x3370 [1 << 17]byte + var x3371 [1 << 17]byte + var x3372 [1 << 17]byte + var x3373 [1 << 17]byte + var x3374 [1 << 17]byte + var x3375 [1 << 17]byte + var x3376 [1 << 17]byte + var x3377 [1 << 17]byte + var x3378 [1 << 17]byte + var x3379 [1 << 17]byte + var x3380 [1 << 17]byte + var x3381 [1 << 17]byte + var x3382 [1 << 17]byte + var x3383 [1 << 17]byte + var x3384 [1 << 17]byte + var x3385 [1 << 17]byte + var x3386 [1 << 17]byte + var x3387 [1 << 17]byte + var x3388 [1 << 17]byte + var x3389 [1 << 17]byte + var x3390 [1 << 17]byte + var x3391 [1 << 17]byte + var x3392 [1 << 17]byte + var x3393 [1 << 17]byte + var x3394 [1 << 17]byte + var x3395 [1 << 17]byte + var x3396 [1 << 17]byte + var x3397 [1 << 17]byte + var x3398 [1 << 17]byte + var x3399 [1 << 17]byte + var x3400 [1 << 17]byte + var x3401 [1 << 17]byte + var x3402 [1 << 17]byte + var x3403 [1 << 17]byte + var x3404 [1 << 17]byte + var x3405 [1 << 17]byte + var x3406 [1 << 17]byte + var x3407 [1 << 17]byte + var x3408 [1 << 17]byte + var x3409 [1 << 17]byte + var x3410 [1 << 17]byte + var x3411 [1 << 17]byte + var x3412 [1 << 17]byte + var x3413 [1 << 17]byte + var x3414 [1 << 17]byte + var x3415 [1 << 17]byte + var x3416 [1 << 17]byte + var x3417 [1 << 17]byte + var x3418 [1 << 17]byte + var x3419 [1 << 17]byte + var x3420 [1 << 17]byte + var x3421 [1 << 17]byte + var x3422 [1 << 17]byte + var x3423 [1 << 17]byte + var x3424 [1 << 17]byte + var x3425 [1 << 17]byte + var x3426 [1 << 17]byte + var x3427 [1 << 17]byte + var x3428 [1 << 17]byte + var x3429 [1 << 17]byte + var x3430 [1 << 17]byte + var x3431 [1 << 17]byte + var x3432 [1 << 17]byte + var x3433 [1 << 17]byte + var x3434 [1 << 17]byte + var x3435 [1 << 17]byte + var x3436 [1 << 17]byte + var x3437 [1 << 17]byte + var x3438 [1 << 17]byte + var x3439 [1 << 17]byte + var x3440 [1 << 17]byte + var x3441 [1 << 17]byte + var x3442 [1 << 17]byte + var x3443 [1 << 17]byte + var x3444 [1 << 17]byte + var x3445 [1 << 17]byte + var x3446 [1 << 17]byte + var x3447 [1 << 17]byte + var x3448 [1 << 17]byte + var x3449 [1 << 17]byte + var x3450 [1 << 17]byte + var x3451 [1 << 17]byte + var x3452 [1 << 17]byte + var x3453 [1 << 17]byte + var x3454 [1 << 17]byte + var x3455 [1 << 17]byte + var x3456 [1 << 17]byte + var x3457 [1 << 17]byte + var x3458 [1 << 17]byte + var x3459 [1 << 17]byte + var x3460 [1 << 17]byte + var x3461 [1 << 17]byte + var x3462 [1 << 17]byte + var x3463 [1 << 17]byte + var x3464 [1 << 17]byte + var x3465 [1 << 17]byte + var x3466 [1 << 17]byte + var x3467 [1 << 17]byte + var x3468 [1 << 17]byte + var x3469 [1 << 17]byte + var x3470 [1 << 17]byte + var x3471 [1 << 17]byte + var x3472 [1 << 17]byte + var x3473 [1 << 17]byte + var x3474 [1 << 17]byte + var x3475 [1 << 17]byte + var x3476 [1 << 17]byte + var x3477 [1 << 17]byte + var x3478 [1 << 17]byte + var x3479 [1 << 17]byte + var x3480 [1 << 17]byte + var x3481 [1 << 17]byte + var x3482 [1 << 17]byte + var x3483 [1 << 17]byte + var x3484 [1 << 17]byte + var x3485 [1 << 17]byte + var x3486 [1 << 17]byte + var x3487 [1 << 17]byte + var x3488 [1 << 17]byte + var x3489 [1 << 17]byte + var x3490 [1 << 17]byte + var x3491 [1 << 17]byte + var x3492 [1 << 17]byte + var x3493 [1 << 17]byte + var x3494 [1 << 17]byte + var x3495 [1 << 17]byte + var x3496 [1 << 17]byte + var x3497 [1 << 17]byte + var x3498 [1 << 17]byte + var x3499 [1 << 17]byte + var x3500 [1 << 17]byte + var x3501 [1 << 17]byte + var x3502 [1 << 17]byte + var x3503 [1 << 17]byte + var x3504 [1 << 17]byte + var x3505 [1 << 17]byte + var x3506 [1 << 17]byte + var x3507 [1 << 17]byte + var x3508 [1 << 17]byte + var x3509 [1 << 17]byte + var x3510 [1 << 17]byte + var x3511 [1 << 17]byte + var x3512 [1 << 17]byte + var x3513 [1 << 17]byte + var x3514 [1 << 17]byte + var x3515 [1 << 17]byte + var x3516 [1 << 17]byte + var x3517 [1 << 17]byte + var x3518 [1 << 17]byte + var x3519 [1 << 17]byte + var x3520 [1 << 17]byte + var x3521 [1 << 17]byte + var x3522 [1 << 17]byte + var x3523 [1 << 17]byte + var x3524 [1 << 17]byte + var x3525 [1 << 17]byte + var x3526 [1 << 17]byte + var x3527 [1 << 17]byte + var x3528 [1 << 17]byte + var x3529 [1 << 17]byte + var x3530 [1 << 17]byte + var x3531 [1 << 17]byte + var x3532 [1 << 17]byte + var x3533 [1 << 17]byte + var x3534 [1 << 17]byte + var x3535 [1 << 17]byte + var x3536 [1 << 17]byte + var x3537 [1 << 17]byte + var x3538 [1 << 17]byte + var x3539 [1 << 17]byte + var x3540 [1 << 17]byte + var x3541 [1 << 17]byte + var x3542 [1 << 17]byte + var x3543 [1 << 17]byte + var x3544 [1 << 17]byte + var x3545 [1 << 17]byte + var x3546 [1 << 17]byte + var x3547 [1 << 17]byte + var x3548 [1 << 17]byte + var x3549 [1 << 17]byte + var x3550 [1 << 17]byte + var x3551 [1 << 17]byte + var x3552 [1 << 17]byte + var x3553 [1 << 17]byte + var x3554 [1 << 17]byte + var x3555 [1 << 17]byte + var x3556 [1 << 17]byte + var x3557 [1 << 17]byte + var x3558 [1 << 17]byte + var x3559 [1 << 17]byte + var x3560 [1 << 17]byte + var x3561 [1 << 17]byte + var x3562 [1 << 17]byte + var x3563 [1 << 17]byte + var x3564 [1 << 17]byte + var x3565 [1 << 17]byte + var x3566 [1 << 17]byte + var x3567 [1 << 17]byte + var x3568 [1 << 17]byte + var x3569 [1 << 17]byte + var x3570 [1 << 17]byte + var x3571 [1 << 17]byte + var x3572 [1 << 17]byte + var x3573 [1 << 17]byte + var x3574 [1 << 17]byte + var x3575 [1 << 17]byte + var x3576 [1 << 17]byte + var x3577 [1 << 17]byte + var x3578 [1 << 17]byte + var x3579 [1 << 17]byte + var x3580 [1 << 17]byte + var x3581 [1 << 17]byte + var x3582 [1 << 17]byte + var x3583 [1 << 17]byte + var x3584 [1 << 17]byte + var x3585 [1 << 17]byte + var x3586 [1 << 17]byte + var x3587 [1 << 17]byte + var x3588 [1 << 17]byte + var x3589 [1 << 17]byte + var x3590 [1 << 17]byte + var x3591 [1 << 17]byte + var x3592 [1 << 17]byte + var x3593 [1 << 17]byte + var x3594 [1 << 17]byte + var x3595 [1 << 17]byte + var x3596 [1 << 17]byte + var x3597 [1 << 17]byte + var x3598 [1 << 17]byte + var x3599 [1 << 17]byte + var x3600 [1 << 17]byte + var x3601 [1 << 17]byte + var x3602 [1 << 17]byte + var x3603 [1 << 17]byte + var x3604 [1 << 17]byte + var x3605 [1 << 17]byte + var x3606 [1 << 17]byte + var x3607 [1 << 17]byte + var x3608 [1 << 17]byte + var x3609 [1 << 17]byte + var x3610 [1 << 17]byte + var x3611 [1 << 17]byte + var x3612 [1 << 17]byte + var x3613 [1 << 17]byte + var x3614 [1 << 17]byte + var x3615 [1 << 17]byte + var x3616 [1 << 17]byte + var x3617 [1 << 17]byte + var x3618 [1 << 17]byte + var x3619 [1 << 17]byte + var x3620 [1 << 17]byte + var x3621 [1 << 17]byte + var x3622 [1 << 17]byte + var x3623 [1 << 17]byte + var x3624 [1 << 17]byte + var x3625 [1 << 17]byte + var x3626 [1 << 17]byte + var x3627 [1 << 17]byte + var x3628 [1 << 17]byte + var x3629 [1 << 17]byte + var x3630 [1 << 17]byte + var x3631 [1 << 17]byte + var x3632 [1 << 17]byte + var x3633 [1 << 17]byte + var x3634 [1 << 17]byte + var x3635 [1 << 17]byte + var x3636 [1 << 17]byte + var x3637 [1 << 17]byte + var x3638 [1 << 17]byte + var x3639 [1 << 17]byte + var x3640 [1 << 17]byte + var x3641 [1 << 17]byte + var x3642 [1 << 17]byte + var x3643 [1 << 17]byte + var x3644 [1 << 17]byte + var x3645 [1 << 17]byte + var x3646 [1 << 17]byte + var x3647 [1 << 17]byte + var x3648 [1 << 17]byte + var x3649 [1 << 17]byte + var x3650 [1 << 17]byte + var x3651 [1 << 17]byte + var x3652 [1 << 17]byte + var x3653 [1 << 17]byte + var x3654 [1 << 17]byte + var x3655 [1 << 17]byte + var x3656 [1 << 17]byte + var x3657 [1 << 17]byte + var x3658 [1 << 17]byte + var x3659 [1 << 17]byte + var x3660 [1 << 17]byte + var x3661 [1 << 17]byte + var x3662 [1 << 17]byte + var x3663 [1 << 17]byte + var x3664 [1 << 17]byte + var x3665 [1 << 17]byte + var x3666 [1 << 17]byte + var x3667 [1 << 17]byte + var x3668 [1 << 17]byte + var x3669 [1 << 17]byte + var x3670 [1 << 17]byte + var x3671 [1 << 17]byte + var x3672 [1 << 17]byte + var x3673 [1 << 17]byte + var x3674 [1 << 17]byte + var x3675 [1 << 17]byte + var x3676 [1 << 17]byte + var x3677 [1 << 17]byte + var x3678 [1 << 17]byte + var x3679 [1 << 17]byte + var x3680 [1 << 17]byte + var x3681 [1 << 17]byte + var x3682 [1 << 17]byte + var x3683 [1 << 17]byte + var x3684 [1 << 17]byte + var x3685 [1 << 17]byte + var x3686 [1 << 17]byte + var x3687 [1 << 17]byte + var x3688 [1 << 17]byte + var x3689 [1 << 17]byte + var x3690 [1 << 17]byte + var x3691 [1 << 17]byte + var x3692 [1 << 17]byte + var x3693 [1 << 17]byte + var x3694 [1 << 17]byte + var x3695 [1 << 17]byte + var x3696 [1 << 17]byte + var x3697 [1 << 17]byte + var x3698 [1 << 17]byte + var x3699 [1 << 17]byte + var x3700 [1 << 17]byte + var x3701 [1 << 17]byte + var x3702 [1 << 17]byte + var x3703 [1 << 17]byte + var x3704 [1 << 17]byte + var x3705 [1 << 17]byte + var x3706 [1 << 17]byte + var x3707 [1 << 17]byte + var x3708 [1 << 17]byte + var x3709 [1 << 17]byte + var x3710 [1 << 17]byte + var x3711 [1 << 17]byte + var x3712 [1 << 17]byte + var x3713 [1 << 17]byte + var x3714 [1 << 17]byte + var x3715 [1 << 17]byte + var x3716 [1 << 17]byte + var x3717 [1 << 17]byte + var x3718 [1 << 17]byte + var x3719 [1 << 17]byte + var x3720 [1 << 17]byte + var x3721 [1 << 17]byte + var x3722 [1 << 17]byte + var x3723 [1 << 17]byte + var x3724 [1 << 17]byte + var x3725 [1 << 17]byte + var x3726 [1 << 17]byte + var x3727 [1 << 17]byte + var x3728 [1 << 17]byte + var x3729 [1 << 17]byte + var x3730 [1 << 17]byte + var x3731 [1 << 17]byte + var x3732 [1 << 17]byte + var x3733 [1 << 17]byte + var x3734 [1 << 17]byte + var x3735 [1 << 17]byte + var x3736 [1 << 17]byte + var x3737 [1 << 17]byte + var x3738 [1 << 17]byte + var x3739 [1 << 17]byte + var x3740 [1 << 17]byte + var x3741 [1 << 17]byte + var x3742 [1 << 17]byte + var x3743 [1 << 17]byte + var x3744 [1 << 17]byte + var x3745 [1 << 17]byte + var x3746 [1 << 17]byte + var x3747 [1 << 17]byte + var x3748 [1 << 17]byte + var x3749 [1 << 17]byte + var x3750 [1 << 17]byte + var x3751 [1 << 17]byte + var x3752 [1 << 17]byte + var x3753 [1 << 17]byte + var x3754 [1 << 17]byte + var x3755 [1 << 17]byte + var x3756 [1 << 17]byte + var x3757 [1 << 17]byte + var x3758 [1 << 17]byte + var x3759 [1 << 17]byte + var x3760 [1 << 17]byte + var x3761 [1 << 17]byte + var x3762 [1 << 17]byte + var x3763 [1 << 17]byte + var x3764 [1 << 17]byte + var x3765 [1 << 17]byte + var x3766 [1 << 17]byte + var x3767 [1 << 17]byte + var x3768 [1 << 17]byte + var x3769 [1 << 17]byte + var x3770 [1 << 17]byte + var x3771 [1 << 17]byte + var x3772 [1 << 17]byte + var x3773 [1 << 17]byte + var x3774 [1 << 17]byte + var x3775 [1 << 17]byte + var x3776 [1 << 17]byte + var x3777 [1 << 17]byte + var x3778 [1 << 17]byte + var x3779 [1 << 17]byte + var x3780 [1 << 17]byte + var x3781 [1 << 17]byte + var x3782 [1 << 17]byte + var x3783 [1 << 17]byte + var x3784 [1 << 17]byte + var x3785 [1 << 17]byte + var x3786 [1 << 17]byte + var x3787 [1 << 17]byte + var x3788 [1 << 17]byte + var x3789 [1 << 17]byte + var x3790 [1 << 17]byte + var x3791 [1 << 17]byte + var x3792 [1 << 17]byte + var x3793 [1 << 17]byte + var x3794 [1 << 17]byte + var x3795 [1 << 17]byte + var x3796 [1 << 17]byte + var x3797 [1 << 17]byte + var x3798 [1 << 17]byte + var x3799 [1 << 17]byte + var x3800 [1 << 17]byte + var x3801 [1 << 17]byte + var x3802 [1 << 17]byte + var x3803 [1 << 17]byte + var x3804 [1 << 17]byte + var x3805 [1 << 17]byte + var x3806 [1 << 17]byte + var x3807 [1 << 17]byte + var x3808 [1 << 17]byte + var x3809 [1 << 17]byte + var x3810 [1 << 17]byte + var x3811 [1 << 17]byte + var x3812 [1 << 17]byte + var x3813 [1 << 17]byte + var x3814 [1 << 17]byte + var x3815 [1 << 17]byte + var x3816 [1 << 17]byte + var x3817 [1 << 17]byte + var x3818 [1 << 17]byte + var x3819 [1 << 17]byte + var x3820 [1 << 17]byte + var x3821 [1 << 17]byte + var x3822 [1 << 17]byte + var x3823 [1 << 17]byte + var x3824 [1 << 17]byte + var x3825 [1 << 17]byte + var x3826 [1 << 17]byte + var x3827 [1 << 17]byte + var x3828 [1 << 17]byte + var x3829 [1 << 17]byte + var x3830 [1 << 17]byte + var x3831 [1 << 17]byte + var x3832 [1 << 17]byte + var x3833 [1 << 17]byte + var x3834 [1 << 17]byte + var x3835 [1 << 17]byte + var x3836 [1 << 17]byte + var x3837 [1 << 17]byte + var x3838 [1 << 17]byte + var x3839 [1 << 17]byte + var x3840 [1 << 17]byte + var x3841 [1 << 17]byte + var x3842 [1 << 17]byte + var x3843 [1 << 17]byte + var x3844 [1 << 17]byte + var x3845 [1 << 17]byte + var x3846 [1 << 17]byte + var x3847 [1 << 17]byte + var x3848 [1 << 17]byte + var x3849 [1 << 17]byte + var x3850 [1 << 17]byte + var x3851 [1 << 17]byte + var x3852 [1 << 17]byte + var x3853 [1 << 17]byte + var x3854 [1 << 17]byte + var x3855 [1 << 17]byte + var x3856 [1 << 17]byte + var x3857 [1 << 17]byte + var x3858 [1 << 17]byte + var x3859 [1 << 17]byte + var x3860 [1 << 17]byte + var x3861 [1 << 17]byte + var x3862 [1 << 17]byte + var x3863 [1 << 17]byte + var x3864 [1 << 17]byte + var x3865 [1 << 17]byte + var x3866 [1 << 17]byte + var x3867 [1 << 17]byte + var x3868 [1 << 17]byte + var x3869 [1 << 17]byte + var x3870 [1 << 17]byte + var x3871 [1 << 17]byte + var x3872 [1 << 17]byte + var x3873 [1 << 17]byte + var x3874 [1 << 17]byte + var x3875 [1 << 17]byte + var x3876 [1 << 17]byte + var x3877 [1 << 17]byte + var x3878 [1 << 17]byte + var x3879 [1 << 17]byte + var x3880 [1 << 17]byte + var x3881 [1 << 17]byte + var x3882 [1 << 17]byte + var x3883 [1 << 17]byte + var x3884 [1 << 17]byte + var x3885 [1 << 17]byte + var x3886 [1 << 17]byte + var x3887 [1 << 17]byte + var x3888 [1 << 17]byte + var x3889 [1 << 17]byte + var x3890 [1 << 17]byte + var x3891 [1 << 17]byte + var x3892 [1 << 17]byte + var x3893 [1 << 17]byte + var x3894 [1 << 17]byte + var x3895 [1 << 17]byte + var x3896 [1 << 17]byte + var x3897 [1 << 17]byte + var x3898 [1 << 17]byte + var x3899 [1 << 17]byte + var x3900 [1 << 17]byte + var x3901 [1 << 17]byte + var x3902 [1 << 17]byte + var x3903 [1 << 17]byte + var x3904 [1 << 17]byte + var x3905 [1 << 17]byte + var x3906 [1 << 17]byte + var x3907 [1 << 17]byte + var x3908 [1 << 17]byte + var x3909 [1 << 17]byte + var x3910 [1 << 17]byte + var x3911 [1 << 17]byte + var x3912 [1 << 17]byte + var x3913 [1 << 17]byte + var x3914 [1 << 17]byte + var x3915 [1 << 17]byte + var x3916 [1 << 17]byte + var x3917 [1 << 17]byte + var x3918 [1 << 17]byte + var x3919 [1 << 17]byte + var x3920 [1 << 17]byte + var x3921 [1 << 17]byte + var x3922 [1 << 17]byte + var x3923 [1 << 17]byte + var x3924 [1 << 17]byte + var x3925 [1 << 17]byte + var x3926 [1 << 17]byte + var x3927 [1 << 17]byte + var x3928 [1 << 17]byte + var x3929 [1 << 17]byte + var x3930 [1 << 17]byte + var x3931 [1 << 17]byte + var x3932 [1 << 17]byte + var x3933 [1 << 17]byte + var x3934 [1 << 17]byte + var x3935 [1 << 17]byte + var x3936 [1 << 17]byte + var x3937 [1 << 17]byte + var x3938 [1 << 17]byte + var x3939 [1 << 17]byte + var x3940 [1 << 17]byte + var x3941 [1 << 17]byte + var x3942 [1 << 17]byte + var x3943 [1 << 17]byte + var x3944 [1 << 17]byte + var x3945 [1 << 17]byte + var x3946 [1 << 17]byte + var x3947 [1 << 17]byte + var x3948 [1 << 17]byte + var x3949 [1 << 17]byte + var x3950 [1 << 17]byte + var x3951 [1 << 17]byte + var x3952 [1 << 17]byte + var x3953 [1 << 17]byte + var x3954 [1 << 17]byte + var x3955 [1 << 17]byte + var x3956 [1 << 17]byte + var x3957 [1 << 17]byte + var x3958 [1 << 17]byte + var x3959 [1 << 17]byte + var x3960 [1 << 17]byte + var x3961 [1 << 17]byte + var x3962 [1 << 17]byte + var x3963 [1 << 17]byte + var x3964 [1 << 17]byte + var x3965 [1 << 17]byte + var x3966 [1 << 17]byte + var x3967 [1 << 17]byte + var x3968 [1 << 17]byte + var x3969 [1 << 17]byte + var x3970 [1 << 17]byte + var x3971 [1 << 17]byte + var x3972 [1 << 17]byte + var x3973 [1 << 17]byte + var x3974 [1 << 17]byte + var x3975 [1 << 17]byte + var x3976 [1 << 17]byte + var x3977 [1 << 17]byte + var x3978 [1 << 17]byte + var x3979 [1 << 17]byte + var x3980 [1 << 17]byte + var x3981 [1 << 17]byte + var x3982 [1 << 17]byte + var x3983 [1 << 17]byte + var x3984 [1 << 17]byte + var x3985 [1 << 17]byte + var x3986 [1 << 17]byte + var x3987 [1 << 17]byte + var x3988 [1 << 17]byte + var x3989 [1 << 17]byte + var x3990 [1 << 17]byte + var x3991 [1 << 17]byte + var x3992 [1 << 17]byte + var x3993 [1 << 17]byte + var x3994 [1 << 17]byte + var x3995 [1 << 17]byte + var x3996 [1 << 17]byte + var x3997 [1 << 17]byte + var x3998 [1 << 17]byte + var x3999 [1 << 17]byte + var x4000 [1 << 17]byte + var x4001 [1 << 17]byte + var x4002 [1 << 17]byte + var x4003 [1 << 17]byte + var x4004 [1 << 17]byte + var x4005 [1 << 17]byte + var x4006 [1 << 17]byte + var x4007 [1 << 17]byte + var x4008 [1 << 17]byte + var x4009 [1 << 17]byte + var x4010 [1 << 17]byte + var x4011 [1 << 17]byte + var x4012 [1 << 17]byte + var x4013 [1 << 17]byte + var x4014 [1 << 17]byte + var x4015 [1 << 17]byte + var x4016 [1 << 17]byte + var x4017 [1 << 17]byte + var x4018 [1 << 17]byte + var x4019 [1 << 17]byte + var x4020 [1 << 17]byte + var x4021 [1 << 17]byte + var x4022 [1 << 17]byte + var x4023 [1 << 17]byte + var x4024 [1 << 17]byte + var x4025 [1 << 17]byte + var x4026 [1 << 17]byte + var x4027 [1 << 17]byte + var x4028 [1 << 17]byte + var x4029 [1 << 17]byte + var x4030 [1 << 17]byte + var x4031 [1 << 17]byte + var x4032 [1 << 17]byte + var x4033 [1 << 17]byte + var x4034 [1 << 17]byte + var x4035 [1 << 17]byte + var x4036 [1 << 17]byte + var x4037 [1 << 17]byte + var x4038 [1 << 17]byte + var x4039 [1 << 17]byte + var x4040 [1 << 17]byte + var x4041 [1 << 17]byte + var x4042 [1 << 17]byte + var x4043 [1 << 17]byte + var x4044 [1 << 17]byte + var x4045 [1 << 17]byte + var x4046 [1 << 17]byte + var x4047 [1 << 17]byte + var x4048 [1 << 17]byte + var x4049 [1 << 17]byte + var x4050 [1 << 17]byte + var x4051 [1 << 17]byte + var x4052 [1 << 17]byte + var x4053 [1 << 17]byte + var x4054 [1 << 17]byte + var x4055 [1 << 17]byte + var x4056 [1 << 17]byte + var x4057 [1 << 17]byte + var x4058 [1 << 17]byte + var x4059 [1 << 17]byte + var x4060 [1 << 17]byte + var x4061 [1 << 17]byte + var x4062 [1 << 17]byte + var x4063 [1 << 17]byte + var x4064 [1 << 17]byte + var x4065 [1 << 17]byte + var x4066 [1 << 17]byte + var x4067 [1 << 17]byte + var x4068 [1 << 17]byte + var x4069 [1 << 17]byte + var x4070 [1 << 17]byte + var x4071 [1 << 17]byte + var x4072 [1 << 17]byte + var x4073 [1 << 17]byte + var x4074 [1 << 17]byte + var x4075 [1 << 17]byte + var x4076 [1 << 17]byte + var x4077 [1 << 17]byte + var x4078 [1 << 17]byte + var x4079 [1 << 17]byte + var x4080 [1 << 17]byte + var x4081 [1 << 17]byte + var x4082 [1 << 17]byte + var x4083 [1 << 17]byte + var x4084 [1 << 17]byte + var x4085 [1 << 17]byte + var x4086 [1 << 17]byte + var x4087 [1 << 17]byte + var x4088 [1 << 17]byte + var x4089 [1 << 17]byte + var x4090 [1 << 17]byte + var x4091 [1 << 17]byte + var x4092 [1 << 17]byte + var x4093 [1 << 17]byte + var x4094 [1 << 17]byte + var x4095 [1 << 17]byte + var x4096 [1 << 17]byte + var x4097 [1 << 17]byte + var x4098 [1 << 17]byte + var x4099 [1 << 17]byte + var x4100 [1 << 17]byte + var x4101 [1 << 17]byte + var x4102 [1 << 17]byte + var x4103 [1 << 17]byte + var x4104 [1 << 17]byte + var x4105 [1 << 17]byte + var x4106 [1 << 17]byte + var x4107 [1 << 17]byte + var x4108 [1 << 17]byte + var x4109 [1 << 17]byte + var x4110 [1 << 17]byte + var x4111 [1 << 17]byte + var x4112 [1 << 17]byte + var x4113 [1 << 17]byte + var x4114 [1 << 17]byte + var x4115 [1 << 17]byte + var x4116 [1 << 17]byte + var x4117 [1 << 17]byte + var x4118 [1 << 17]byte + var x4119 [1 << 17]byte + var x4120 [1 << 17]byte + var x4121 [1 << 17]byte + var x4122 [1 << 17]byte + var x4123 [1 << 17]byte + var x4124 [1 << 17]byte + var x4125 [1 << 17]byte + var x4126 [1 << 17]byte + var x4127 [1 << 17]byte + var x4128 [1 << 17]byte + var x4129 [1 << 17]byte + var x4130 [1 << 17]byte + var x4131 [1 << 17]byte + var x4132 [1 << 17]byte + var x4133 [1 << 17]byte + var x4134 [1 << 17]byte + var x4135 [1 << 17]byte + var x4136 [1 << 17]byte + var x4137 [1 << 17]byte + var x4138 [1 << 17]byte + var x4139 [1 << 17]byte + var x4140 [1 << 17]byte + var x4141 [1 << 17]byte + var x4142 [1 << 17]byte + var x4143 [1 << 17]byte + var x4144 [1 << 17]byte + var x4145 [1 << 17]byte + var x4146 [1 << 17]byte + var x4147 [1 << 17]byte + var x4148 [1 << 17]byte + var x4149 [1 << 17]byte + var x4150 [1 << 17]byte + var x4151 [1 << 17]byte + var x4152 [1 << 17]byte + var x4153 [1 << 17]byte + var x4154 [1 << 17]byte + var x4155 [1 << 17]byte + var x4156 [1 << 17]byte + var x4157 [1 << 17]byte + var x4158 [1 << 17]byte + var x4159 [1 << 17]byte + var x4160 [1 << 17]byte + var x4161 [1 << 17]byte + var x4162 [1 << 17]byte + var x4163 [1 << 17]byte + var x4164 [1 << 17]byte + var x4165 [1 << 17]byte + var x4166 [1 << 17]byte + var x4167 [1 << 17]byte + var x4168 [1 << 17]byte + var x4169 [1 << 17]byte + var x4170 [1 << 17]byte + var x4171 [1 << 17]byte + var x4172 [1 << 17]byte + var x4173 [1 << 17]byte + var x4174 [1 << 17]byte + var x4175 [1 << 17]byte + var x4176 [1 << 17]byte + var x4177 [1 << 17]byte + var x4178 [1 << 17]byte + var x4179 [1 << 17]byte + var x4180 [1 << 17]byte + var x4181 [1 << 17]byte + var x4182 [1 << 17]byte + var x4183 [1 << 17]byte + var x4184 [1 << 17]byte + var x4185 [1 << 17]byte + var x4186 [1 << 17]byte + var x4187 [1 << 17]byte + var x4188 [1 << 17]byte + var x4189 [1 << 17]byte + var x4190 [1 << 17]byte + var x4191 [1 << 17]byte + var x4192 [1 << 17]byte + var x4193 [1 << 17]byte + var x4194 [1 << 17]byte + var x4195 [1 << 17]byte + var x4196 [1 << 17]byte + var x4197 [1 << 17]byte + var x4198 [1 << 17]byte + var x4199 [1 << 17]byte + var x4200 [1 << 17]byte + var x4201 [1 << 17]byte + var x4202 [1 << 17]byte + var x4203 [1 << 17]byte + var x4204 [1 << 17]byte + var x4205 [1 << 17]byte + var x4206 [1 << 17]byte + var x4207 [1 << 17]byte + var x4208 [1 << 17]byte + var x4209 [1 << 17]byte + var x4210 [1 << 17]byte + var x4211 [1 << 17]byte + var x4212 [1 << 17]byte + var x4213 [1 << 17]byte + var x4214 [1 << 17]byte + var x4215 [1 << 17]byte + var x4216 [1 << 17]byte + var x4217 [1 << 17]byte + var x4218 [1 << 17]byte + var x4219 [1 << 17]byte + var x4220 [1 << 17]byte + var x4221 [1 << 17]byte + var x4222 [1 << 17]byte + var x4223 [1 << 17]byte + var x4224 [1 << 17]byte + var x4225 [1 << 17]byte + var x4226 [1 << 17]byte + var x4227 [1 << 17]byte + var x4228 [1 << 17]byte + var x4229 [1 << 17]byte + var x4230 [1 << 17]byte + var x4231 [1 << 17]byte + var x4232 [1 << 17]byte + var x4233 [1 << 17]byte + var x4234 [1 << 17]byte + var x4235 [1 << 17]byte + var x4236 [1 << 17]byte + var x4237 [1 << 17]byte + var x4238 [1 << 17]byte + var x4239 [1 << 17]byte + var x4240 [1 << 17]byte + var x4241 [1 << 17]byte + var x4242 [1 << 17]byte + var x4243 [1 << 17]byte + var x4244 [1 << 17]byte + var x4245 [1 << 17]byte + var x4246 [1 << 17]byte + var x4247 [1 << 17]byte + var x4248 [1 << 17]byte + var x4249 [1 << 17]byte + var x4250 [1 << 17]byte + var x4251 [1 << 17]byte + var x4252 [1 << 17]byte + var x4253 [1 << 17]byte + var x4254 [1 << 17]byte + var x4255 [1 << 17]byte + var x4256 [1 << 17]byte + var x4257 [1 << 17]byte + var x4258 [1 << 17]byte + var x4259 [1 << 17]byte + var x4260 [1 << 17]byte + var x4261 [1 << 17]byte + var x4262 [1 << 17]byte + var x4263 [1 << 17]byte + var x4264 [1 << 17]byte + var x4265 [1 << 17]byte + var x4266 [1 << 17]byte + var x4267 [1 << 17]byte + var x4268 [1 << 17]byte + var x4269 [1 << 17]byte + var x4270 [1 << 17]byte + var x4271 [1 << 17]byte + var x4272 [1 << 17]byte + var x4273 [1 << 17]byte + var x4274 [1 << 17]byte + var x4275 [1 << 17]byte + var x4276 [1 << 17]byte + var x4277 [1 << 17]byte + var x4278 [1 << 17]byte + var x4279 [1 << 17]byte + var x4280 [1 << 17]byte + var x4281 [1 << 17]byte + var x4282 [1 << 17]byte + var x4283 [1 << 17]byte + var x4284 [1 << 17]byte + var x4285 [1 << 17]byte + var x4286 [1 << 17]byte + var x4287 [1 << 17]byte + var x4288 [1 << 17]byte + var x4289 [1 << 17]byte + var x4290 [1 << 17]byte + var x4291 [1 << 17]byte + var x4292 [1 << 17]byte + var x4293 [1 << 17]byte + var x4294 [1 << 17]byte + var x4295 [1 << 17]byte + var x4296 [1 << 17]byte + var x4297 [1 << 17]byte + var x4298 [1 << 17]byte + var x4299 [1 << 17]byte + var x4300 [1 << 17]byte + var x4301 [1 << 17]byte + var x4302 [1 << 17]byte + var x4303 [1 << 17]byte + var x4304 [1 << 17]byte + var x4305 [1 << 17]byte + var x4306 [1 << 17]byte + var x4307 [1 << 17]byte + var x4308 [1 << 17]byte + var x4309 [1 << 17]byte + var x4310 [1 << 17]byte + var x4311 [1 << 17]byte + var x4312 [1 << 17]byte + var x4313 [1 << 17]byte + var x4314 [1 << 17]byte + var x4315 [1 << 17]byte + var x4316 [1 << 17]byte + var x4317 [1 << 17]byte + var x4318 [1 << 17]byte + var x4319 [1 << 17]byte + var x4320 [1 << 17]byte + var x4321 [1 << 17]byte + var x4322 [1 << 17]byte + var x4323 [1 << 17]byte + var x4324 [1 << 17]byte + var x4325 [1 << 17]byte + var x4326 [1 << 17]byte + var x4327 [1 << 17]byte + var x4328 [1 << 17]byte + var x4329 [1 << 17]byte + var x4330 [1 << 17]byte + var x4331 [1 << 17]byte + var x4332 [1 << 17]byte + var x4333 [1 << 17]byte + var x4334 [1 << 17]byte + var x4335 [1 << 17]byte + var x4336 [1 << 17]byte + var x4337 [1 << 17]byte + var x4338 [1 << 17]byte + var x4339 [1 << 17]byte + var x4340 [1 << 17]byte + var x4341 [1 << 17]byte + var x4342 [1 << 17]byte + var x4343 [1 << 17]byte + var x4344 [1 << 17]byte + var x4345 [1 << 17]byte + var x4346 [1 << 17]byte + var x4347 [1 << 17]byte + var x4348 [1 << 17]byte + var x4349 [1 << 17]byte + var x4350 [1 << 17]byte + var x4351 [1 << 17]byte + var x4352 [1 << 17]byte + var x4353 [1 << 17]byte + var x4354 [1 << 17]byte + var x4355 [1 << 17]byte + var x4356 [1 << 17]byte + var x4357 [1 << 17]byte + var x4358 [1 << 17]byte + var x4359 [1 << 17]byte + var x4360 [1 << 17]byte + var x4361 [1 << 17]byte + var x4362 [1 << 17]byte + var x4363 [1 << 17]byte + var x4364 [1 << 17]byte + var x4365 [1 << 17]byte + var x4366 [1 << 17]byte + var x4367 [1 << 17]byte + var x4368 [1 << 17]byte + var x4369 [1 << 17]byte + var x4370 [1 << 17]byte + var x4371 [1 << 17]byte + var x4372 [1 << 17]byte + var x4373 [1 << 17]byte + var x4374 [1 << 17]byte + var x4375 [1 << 17]byte + var x4376 [1 << 17]byte + var x4377 [1 << 17]byte + var x4378 [1 << 17]byte + var x4379 [1 << 17]byte + var x4380 [1 << 17]byte + var x4381 [1 << 17]byte + var x4382 [1 << 17]byte + var x4383 [1 << 17]byte + var x4384 [1 << 17]byte + var x4385 [1 << 17]byte + var x4386 [1 << 17]byte + var x4387 [1 << 17]byte + var x4388 [1 << 17]byte + var x4389 [1 << 17]byte + var x4390 [1 << 17]byte + var x4391 [1 << 17]byte + var x4392 [1 << 17]byte + var x4393 [1 << 17]byte + var x4394 [1 << 17]byte + var x4395 [1 << 17]byte + var x4396 [1 << 17]byte + var x4397 [1 << 17]byte + var x4398 [1 << 17]byte + var x4399 [1 << 17]byte + var x4400 [1 << 17]byte + var x4401 [1 << 17]byte + var x4402 [1 << 17]byte + var x4403 [1 << 17]byte + var x4404 [1 << 17]byte + var x4405 [1 << 17]byte + var x4406 [1 << 17]byte + var x4407 [1 << 17]byte + var x4408 [1 << 17]byte + var x4409 [1 << 17]byte + var x4410 [1 << 17]byte + var x4411 [1 << 17]byte + var x4412 [1 << 17]byte + var x4413 [1 << 17]byte + var x4414 [1 << 17]byte + var x4415 [1 << 17]byte + var x4416 [1 << 17]byte + var x4417 [1 << 17]byte + var x4418 [1 << 17]byte + var x4419 [1 << 17]byte + var x4420 [1 << 17]byte + var x4421 [1 << 17]byte + var x4422 [1 << 17]byte + var x4423 [1 << 17]byte + var x4424 [1 << 17]byte + var x4425 [1 << 17]byte + var x4426 [1 << 17]byte + var x4427 [1 << 17]byte + var x4428 [1 << 17]byte + var x4429 [1 << 17]byte + var x4430 [1 << 17]byte + var x4431 [1 << 17]byte + var x4432 [1 << 17]byte + var x4433 [1 << 17]byte + var x4434 [1 << 17]byte + var x4435 [1 << 17]byte + var x4436 [1 << 17]byte + var x4437 [1 << 17]byte + var x4438 [1 << 17]byte + var x4439 [1 << 17]byte + var x4440 [1 << 17]byte + var x4441 [1 << 17]byte + var x4442 [1 << 17]byte + var x4443 [1 << 17]byte + var x4444 [1 << 17]byte + var x4445 [1 << 17]byte + var x4446 [1 << 17]byte + var x4447 [1 << 17]byte + var x4448 [1 << 17]byte + var x4449 [1 << 17]byte + var x4450 [1 << 17]byte + var x4451 [1 << 17]byte + var x4452 [1 << 17]byte + var x4453 [1 << 17]byte + var x4454 [1 << 17]byte + var x4455 [1 << 17]byte + var x4456 [1 << 17]byte + var x4457 [1 << 17]byte + var x4458 [1 << 17]byte + var x4459 [1 << 17]byte + var x4460 [1 << 17]byte + var x4461 [1 << 17]byte + var x4462 [1 << 17]byte + var x4463 [1 << 17]byte + var x4464 [1 << 17]byte + var x4465 [1 << 17]byte + var x4466 [1 << 17]byte + var x4467 [1 << 17]byte + var x4468 [1 << 17]byte + var x4469 [1 << 17]byte + var x4470 [1 << 17]byte + var x4471 [1 << 17]byte + var x4472 [1 << 17]byte + var x4473 [1 << 17]byte + var x4474 [1 << 17]byte + var x4475 [1 << 17]byte + var x4476 [1 << 17]byte + var x4477 [1 << 17]byte + var x4478 [1 << 17]byte + var x4479 [1 << 17]byte + var x4480 [1 << 17]byte + var x4481 [1 << 17]byte + var x4482 [1 << 17]byte + var x4483 [1 << 17]byte + var x4484 [1 << 17]byte + var x4485 [1 << 17]byte + var x4486 [1 << 17]byte + var x4487 [1 << 17]byte + var x4488 [1 << 17]byte + var x4489 [1 << 17]byte + var x4490 [1 << 17]byte + var x4491 [1 << 17]byte + var x4492 [1 << 17]byte + var x4493 [1 << 17]byte + var x4494 [1 << 17]byte + var x4495 [1 << 17]byte + var x4496 [1 << 17]byte + var x4497 [1 << 17]byte + var x4498 [1 << 17]byte + var x4499 [1 << 17]byte + var x4500 [1 << 17]byte + var x4501 [1 << 17]byte + var x4502 [1 << 17]byte + var x4503 [1 << 17]byte + var x4504 [1 << 17]byte + var x4505 [1 << 17]byte + var x4506 [1 << 17]byte + var x4507 [1 << 17]byte + var x4508 [1 << 17]byte + var x4509 [1 << 17]byte + var x4510 [1 << 17]byte + var x4511 [1 << 17]byte + var x4512 [1 << 17]byte + var x4513 [1 << 17]byte + var x4514 [1 << 17]byte + var x4515 [1 << 17]byte + var x4516 [1 << 17]byte + var x4517 [1 << 17]byte + var x4518 [1 << 17]byte + var x4519 [1 << 17]byte + var x4520 [1 << 17]byte + var x4521 [1 << 17]byte + var x4522 [1 << 17]byte + var x4523 [1 << 17]byte + var x4524 [1 << 17]byte + var x4525 [1 << 17]byte + var x4526 [1 << 17]byte + var x4527 [1 << 17]byte + var x4528 [1 << 17]byte + var x4529 [1 << 17]byte + var x4530 [1 << 17]byte + var x4531 [1 << 17]byte + var x4532 [1 << 17]byte + var x4533 [1 << 17]byte + var x4534 [1 << 17]byte + var x4535 [1 << 17]byte + var x4536 [1 << 17]byte + var x4537 [1 << 17]byte + var x4538 [1 << 17]byte + var x4539 [1 << 17]byte + var x4540 [1 << 17]byte + var x4541 [1 << 17]byte + var x4542 [1 << 17]byte + var x4543 [1 << 17]byte + var x4544 [1 << 17]byte + var x4545 [1 << 17]byte + var x4546 [1 << 17]byte + var x4547 [1 << 17]byte + var x4548 [1 << 17]byte + var x4549 [1 << 17]byte + var x4550 [1 << 17]byte + var x4551 [1 << 17]byte + var x4552 [1 << 17]byte + var x4553 [1 << 17]byte + var x4554 [1 << 17]byte + var x4555 [1 << 17]byte + var x4556 [1 << 17]byte + var x4557 [1 << 17]byte + var x4558 [1 << 17]byte + var x4559 [1 << 17]byte + var x4560 [1 << 17]byte + var x4561 [1 << 17]byte + var x4562 [1 << 17]byte + var x4563 [1 << 17]byte + var x4564 [1 << 17]byte + var x4565 [1 << 17]byte + var x4566 [1 << 17]byte + var x4567 [1 << 17]byte + var x4568 [1 << 17]byte + var x4569 [1 << 17]byte + var x4570 [1 << 17]byte + var x4571 [1 << 17]byte + var x4572 [1 << 17]byte + var x4573 [1 << 17]byte + var x4574 [1 << 17]byte + var x4575 [1 << 17]byte + var x4576 [1 << 17]byte + var x4577 [1 << 17]byte + var x4578 [1 << 17]byte + var x4579 [1 << 17]byte + var x4580 [1 << 17]byte + var x4581 [1 << 17]byte + var x4582 [1 << 17]byte + var x4583 [1 << 17]byte + var x4584 [1 << 17]byte + var x4585 [1 << 17]byte + var x4586 [1 << 17]byte + var x4587 [1 << 17]byte + var x4588 [1 << 17]byte + var x4589 [1 << 17]byte + var x4590 [1 << 17]byte + var x4591 [1 << 17]byte + var x4592 [1 << 17]byte + var x4593 [1 << 17]byte + var x4594 [1 << 17]byte + var x4595 [1 << 17]byte + var x4596 [1 << 17]byte + var x4597 [1 << 17]byte + var x4598 [1 << 17]byte + var x4599 [1 << 17]byte + var x4600 [1 << 17]byte + var x4601 [1 << 17]byte + var x4602 [1 << 17]byte + var x4603 [1 << 17]byte + var x4604 [1 << 17]byte + var x4605 [1 << 17]byte + var x4606 [1 << 17]byte + var x4607 [1 << 17]byte + var x4608 [1 << 17]byte + var x4609 [1 << 17]byte + var x4610 [1 << 17]byte + var x4611 [1 << 17]byte + var x4612 [1 << 17]byte + var x4613 [1 << 17]byte + var x4614 [1 << 17]byte + var x4615 [1 << 17]byte + var x4616 [1 << 17]byte + var x4617 [1 << 17]byte + var x4618 [1 << 17]byte + var x4619 [1 << 17]byte + var x4620 [1 << 17]byte + var x4621 [1 << 17]byte + var x4622 [1 << 17]byte + var x4623 [1 << 17]byte + var x4624 [1 << 17]byte + var x4625 [1 << 17]byte + var x4626 [1 << 17]byte + var x4627 [1 << 17]byte + var x4628 [1 << 17]byte + var x4629 [1 << 17]byte + var x4630 [1 << 17]byte + var x4631 [1 << 17]byte + var x4632 [1 << 17]byte + var x4633 [1 << 17]byte + var x4634 [1 << 17]byte + var x4635 [1 << 17]byte + var x4636 [1 << 17]byte + var x4637 [1 << 17]byte + var x4638 [1 << 17]byte + var x4639 [1 << 17]byte + var x4640 [1 << 17]byte + var x4641 [1 << 17]byte + var x4642 [1 << 17]byte + var x4643 [1 << 17]byte + var x4644 [1 << 17]byte + var x4645 [1 << 17]byte + var x4646 [1 << 17]byte + var x4647 [1 << 17]byte + var x4648 [1 << 17]byte + var x4649 [1 << 17]byte + var x4650 [1 << 17]byte + var x4651 [1 << 17]byte + var x4652 [1 << 17]byte + var x4653 [1 << 17]byte + var x4654 [1 << 17]byte + var x4655 [1 << 17]byte + var x4656 [1 << 17]byte + var x4657 [1 << 17]byte + var x4658 [1 << 17]byte + var x4659 [1 << 17]byte + var x4660 [1 << 17]byte + var x4661 [1 << 17]byte + var x4662 [1 << 17]byte + var x4663 [1 << 17]byte + var x4664 [1 << 17]byte + var x4665 [1 << 17]byte + var x4666 [1 << 17]byte + var x4667 [1 << 17]byte + var x4668 [1 << 17]byte + var x4669 [1 << 17]byte + var x4670 [1 << 17]byte + var x4671 [1 << 17]byte + var x4672 [1 << 17]byte + var x4673 [1 << 17]byte + var x4674 [1 << 17]byte + var x4675 [1 << 17]byte + var x4676 [1 << 17]byte + var x4677 [1 << 17]byte + var x4678 [1 << 17]byte + var x4679 [1 << 17]byte + var x4680 [1 << 17]byte + var x4681 [1 << 17]byte + var x4682 [1 << 17]byte + var x4683 [1 << 17]byte + var x4684 [1 << 17]byte + var x4685 [1 << 17]byte + var x4686 [1 << 17]byte + var x4687 [1 << 17]byte + var x4688 [1 << 17]byte + var x4689 [1 << 17]byte + var x4690 [1 << 17]byte + var x4691 [1 << 17]byte + var x4692 [1 << 17]byte + var x4693 [1 << 17]byte + var x4694 [1 << 17]byte + var x4695 [1 << 17]byte + var x4696 [1 << 17]byte + var x4697 [1 << 17]byte + var x4698 [1 << 17]byte + var x4699 [1 << 17]byte + var x4700 [1 << 17]byte + var x4701 [1 << 17]byte + var x4702 [1 << 17]byte + var x4703 [1 << 17]byte + var x4704 [1 << 17]byte + var x4705 [1 << 17]byte + var x4706 [1 << 17]byte + var x4707 [1 << 17]byte + var x4708 [1 << 17]byte + var x4709 [1 << 17]byte + var x4710 [1 << 17]byte + var x4711 [1 << 17]byte + var x4712 [1 << 17]byte + var x4713 [1 << 17]byte + var x4714 [1 << 17]byte + var x4715 [1 << 17]byte + var x4716 [1 << 17]byte + var x4717 [1 << 17]byte + var x4718 [1 << 17]byte + var x4719 [1 << 17]byte + var x4720 [1 << 17]byte + var x4721 [1 << 17]byte + var x4722 [1 << 17]byte + var x4723 [1 << 17]byte + var x4724 [1 << 17]byte + var x4725 [1 << 17]byte + var x4726 [1 << 17]byte + var x4727 [1 << 17]byte + var x4728 [1 << 17]byte + var x4729 [1 << 17]byte + var x4730 [1 << 17]byte + var x4731 [1 << 17]byte + var x4732 [1 << 17]byte + var x4733 [1 << 17]byte + var x4734 [1 << 17]byte + var x4735 [1 << 17]byte + var x4736 [1 << 17]byte + var x4737 [1 << 17]byte + var x4738 [1 << 17]byte + var x4739 [1 << 17]byte + var x4740 [1 << 17]byte + var x4741 [1 << 17]byte + var x4742 [1 << 17]byte + var x4743 [1 << 17]byte + var x4744 [1 << 17]byte + var x4745 [1 << 17]byte + var x4746 [1 << 17]byte + var x4747 [1 << 17]byte + var x4748 [1 << 17]byte + var x4749 [1 << 17]byte + var x4750 [1 << 17]byte + var x4751 [1 << 17]byte + var x4752 [1 << 17]byte + var x4753 [1 << 17]byte + var x4754 [1 << 17]byte + var x4755 [1 << 17]byte + var x4756 [1 << 17]byte + var x4757 [1 << 17]byte + var x4758 [1 << 17]byte + var x4759 [1 << 17]byte + var x4760 [1 << 17]byte + var x4761 [1 << 17]byte + var x4762 [1 << 17]byte + var x4763 [1 << 17]byte + var x4764 [1 << 17]byte + var x4765 [1 << 17]byte + var x4766 [1 << 17]byte + var x4767 [1 << 17]byte + var x4768 [1 << 17]byte + var x4769 [1 << 17]byte + var x4770 [1 << 17]byte + var x4771 [1 << 17]byte + var x4772 [1 << 17]byte + var x4773 [1 << 17]byte + var x4774 [1 << 17]byte + var x4775 [1 << 17]byte + var x4776 [1 << 17]byte + var x4777 [1 << 17]byte + var x4778 [1 << 17]byte + var x4779 [1 << 17]byte + var x4780 [1 << 17]byte + var x4781 [1 << 17]byte + var x4782 [1 << 17]byte + var x4783 [1 << 17]byte + var x4784 [1 << 17]byte + var x4785 [1 << 17]byte + var x4786 [1 << 17]byte + var x4787 [1 << 17]byte + var x4788 [1 << 17]byte + var x4789 [1 << 17]byte + var x4790 [1 << 17]byte + var x4791 [1 << 17]byte + var x4792 [1 << 17]byte + var x4793 [1 << 17]byte + var x4794 [1 << 17]byte + var x4795 [1 << 17]byte + var x4796 [1 << 17]byte + var x4797 [1 << 17]byte + var x4798 [1 << 17]byte + var x4799 [1 << 17]byte + var x4800 [1 << 17]byte + var x4801 [1 << 17]byte + var x4802 [1 << 17]byte + var x4803 [1 << 17]byte + var x4804 [1 << 17]byte + var x4805 [1 << 17]byte + var x4806 [1 << 17]byte + var x4807 [1 << 17]byte + var x4808 [1 << 17]byte + var x4809 [1 << 17]byte + var x4810 [1 << 17]byte + var x4811 [1 << 17]byte + var x4812 [1 << 17]byte + var x4813 [1 << 17]byte + var x4814 [1 << 17]byte + var x4815 [1 << 17]byte + var x4816 [1 << 17]byte + var x4817 [1 << 17]byte + var x4818 [1 << 17]byte + var x4819 [1 << 17]byte + var x4820 [1 << 17]byte + var x4821 [1 << 17]byte + var x4822 [1 << 17]byte + var x4823 [1 << 17]byte + var x4824 [1 << 17]byte + var x4825 [1 << 17]byte + var x4826 [1 << 17]byte + var x4827 [1 << 17]byte + var x4828 [1 << 17]byte + var x4829 [1 << 17]byte + var x4830 [1 << 17]byte + var x4831 [1 << 17]byte + var x4832 [1 << 17]byte + var x4833 [1 << 17]byte + var x4834 [1 << 17]byte + var x4835 [1 << 17]byte + var x4836 [1 << 17]byte + var x4837 [1 << 17]byte + var x4838 [1 << 17]byte + var x4839 [1 << 17]byte + var x4840 [1 << 17]byte + var x4841 [1 << 17]byte + var x4842 [1 << 17]byte + var x4843 [1 << 17]byte + var x4844 [1 << 17]byte + var x4845 [1 << 17]byte + var x4846 [1 << 17]byte + var x4847 [1 << 17]byte + var x4848 [1 << 17]byte + var x4849 [1 << 17]byte + var x4850 [1 << 17]byte + var x4851 [1 << 17]byte + var x4852 [1 << 17]byte + var x4853 [1 << 17]byte + var x4854 [1 << 17]byte + var x4855 [1 << 17]byte + var x4856 [1 << 17]byte + var x4857 [1 << 17]byte + var x4858 [1 << 17]byte + var x4859 [1 << 17]byte + var x4860 [1 << 17]byte + var x4861 [1 << 17]byte + var x4862 [1 << 17]byte + var x4863 [1 << 17]byte + var x4864 [1 << 17]byte + var x4865 [1 << 17]byte + var x4866 [1 << 17]byte + var x4867 [1 << 17]byte + var x4868 [1 << 17]byte + var x4869 [1 << 17]byte + var x4870 [1 << 17]byte + var x4871 [1 << 17]byte + var x4872 [1 << 17]byte + var x4873 [1 << 17]byte + var x4874 [1 << 17]byte + var x4875 [1 << 17]byte + var x4876 [1 << 17]byte + var x4877 [1 << 17]byte + var x4878 [1 << 17]byte + var x4879 [1 << 17]byte + var x4880 [1 << 17]byte + var x4881 [1 << 17]byte + var x4882 [1 << 17]byte + var x4883 [1 << 17]byte + var x4884 [1 << 17]byte + var x4885 [1 << 17]byte + var x4886 [1 << 17]byte + var x4887 [1 << 17]byte + var x4888 [1 << 17]byte + var x4889 [1 << 17]byte + var x4890 [1 << 17]byte + var x4891 [1 << 17]byte + var x4892 [1 << 17]byte + var x4893 [1 << 17]byte + var x4894 [1 << 17]byte + var x4895 [1 << 17]byte + var x4896 [1 << 17]byte + var x4897 [1 << 17]byte + var x4898 [1 << 17]byte + var x4899 [1 << 17]byte + var x4900 [1 << 17]byte + var x4901 [1 << 17]byte + var x4902 [1 << 17]byte + var x4903 [1 << 17]byte + var x4904 [1 << 17]byte + var x4905 [1 << 17]byte + var x4906 [1 << 17]byte + var x4907 [1 << 17]byte + var x4908 [1 << 17]byte + var x4909 [1 << 17]byte + var x4910 [1 << 17]byte + var x4911 [1 << 17]byte + var x4912 [1 << 17]byte + var x4913 [1 << 17]byte + var x4914 [1 << 17]byte + var x4915 [1 << 17]byte + var x4916 [1 << 17]byte + var x4917 [1 << 17]byte + var x4918 [1 << 17]byte + var x4919 [1 << 17]byte + var x4920 [1 << 17]byte + var x4921 [1 << 17]byte + var x4922 [1 << 17]byte + var x4923 [1 << 17]byte + var x4924 [1 << 17]byte + var x4925 [1 << 17]byte + var x4926 [1 << 17]byte + var x4927 [1 << 17]byte + var x4928 [1 << 17]byte + var x4929 [1 << 17]byte + var x4930 [1 << 17]byte + var x4931 [1 << 17]byte + var x4932 [1 << 17]byte + var x4933 [1 << 17]byte + var x4934 [1 << 17]byte + var x4935 [1 << 17]byte + var x4936 [1 << 17]byte + var x4937 [1 << 17]byte + var x4938 [1 << 17]byte + var x4939 [1 << 17]byte + var x4940 [1 << 17]byte + var x4941 [1 << 17]byte + var x4942 [1 << 17]byte + var x4943 [1 << 17]byte + var x4944 [1 << 17]byte + var x4945 [1 << 17]byte + var x4946 [1 << 17]byte + var x4947 [1 << 17]byte + var x4948 [1 << 17]byte + var x4949 [1 << 17]byte + var x4950 [1 << 17]byte + var x4951 [1 << 17]byte + var x4952 [1 << 17]byte + var x4953 [1 << 17]byte + var x4954 [1 << 17]byte + var x4955 [1 << 17]byte + var x4956 [1 << 17]byte + var x4957 [1 << 17]byte + var x4958 [1 << 17]byte + var x4959 [1 << 17]byte + var x4960 [1 << 17]byte + var x4961 [1 << 17]byte + var x4962 [1 << 17]byte + var x4963 [1 << 17]byte + var x4964 [1 << 17]byte + var x4965 [1 << 17]byte + var x4966 [1 << 17]byte + var x4967 [1 << 17]byte + var x4968 [1 << 17]byte + var x4969 [1 << 17]byte + var x4970 [1 << 17]byte + var x4971 [1 << 17]byte + var x4972 [1 << 17]byte + var x4973 [1 << 17]byte + var x4974 [1 << 17]byte + var x4975 [1 << 17]byte + var x4976 [1 << 17]byte + var x4977 [1 << 17]byte + var x4978 [1 << 17]byte + var x4979 [1 << 17]byte + var x4980 [1 << 17]byte + var x4981 [1 << 17]byte + var x4982 [1 << 17]byte + var x4983 [1 << 17]byte + var x4984 [1 << 17]byte + var x4985 [1 << 17]byte + var x4986 [1 << 17]byte + var x4987 [1 << 17]byte + var x4988 [1 << 17]byte + var x4989 [1 << 17]byte + var x4990 [1 << 17]byte + var x4991 [1 << 17]byte + var x4992 [1 << 17]byte + var x4993 [1 << 17]byte + var x4994 [1 << 17]byte + var x4995 [1 << 17]byte + var x4996 [1 << 17]byte + var x4997 [1 << 17]byte + var x4998 [1 << 17]byte + var x4999 [1 << 17]byte + var x5000 [1 << 17]byte + var x5001 [1 << 17]byte + var x5002 [1 << 17]byte + var x5003 [1 << 17]byte + var x5004 [1 << 17]byte + var x5005 [1 << 17]byte + var x5006 [1 << 17]byte + var x5007 [1 << 17]byte + var x5008 [1 << 17]byte + var x5009 [1 << 17]byte + var x5010 [1 << 17]byte + var x5011 [1 << 17]byte + var x5012 [1 << 17]byte + var x5013 [1 << 17]byte + var x5014 [1 << 17]byte + var x5015 [1 << 17]byte + var x5016 [1 << 17]byte + var x5017 [1 << 17]byte + var x5018 [1 << 17]byte + var x5019 [1 << 17]byte + var x5020 [1 << 17]byte + var x5021 [1 << 17]byte + var x5022 [1 << 17]byte + var x5023 [1 << 17]byte + var x5024 [1 << 17]byte + var x5025 [1 << 17]byte + var x5026 [1 << 17]byte + var x5027 [1 << 17]byte + var x5028 [1 << 17]byte + var x5029 [1 << 17]byte + var x5030 [1 << 17]byte + var x5031 [1 << 17]byte + var x5032 [1 << 17]byte + var x5033 [1 << 17]byte + var x5034 [1 << 17]byte + var x5035 [1 << 17]byte + var x5036 [1 << 17]byte + var x5037 [1 << 17]byte + var x5038 [1 << 17]byte + var x5039 [1 << 17]byte + var x5040 [1 << 17]byte + var x5041 [1 << 17]byte + var x5042 [1 << 17]byte + var x5043 [1 << 17]byte + var x5044 [1 << 17]byte + var x5045 [1 << 17]byte + var x5046 [1 << 17]byte + var x5047 [1 << 17]byte + var x5048 [1 << 17]byte + var x5049 [1 << 17]byte + var x5050 [1 << 17]byte + var x5051 [1 << 17]byte + var x5052 [1 << 17]byte + var x5053 [1 << 17]byte + var x5054 [1 << 17]byte + var x5055 [1 << 17]byte + var x5056 [1 << 17]byte + var x5057 [1 << 17]byte + var x5058 [1 << 17]byte + var x5059 [1 << 17]byte + var x5060 [1 << 17]byte + var x5061 [1 << 17]byte + var x5062 [1 << 17]byte + var x5063 [1 << 17]byte + var x5064 [1 << 17]byte + var x5065 [1 << 17]byte + var x5066 [1 << 17]byte + var x5067 [1 << 17]byte + var x5068 [1 << 17]byte + var x5069 [1 << 17]byte + var x5070 [1 << 17]byte + var x5071 [1 << 17]byte + var x5072 [1 << 17]byte + var x5073 [1 << 17]byte + var x5074 [1 << 17]byte + var x5075 [1 << 17]byte + var x5076 [1 << 17]byte + var x5077 [1 << 17]byte + var x5078 [1 << 17]byte + var x5079 [1 << 17]byte + var x5080 [1 << 17]byte + var x5081 [1 << 17]byte + var x5082 [1 << 17]byte + var x5083 [1 << 17]byte + var x5084 [1 << 17]byte + var x5085 [1 << 17]byte + var x5086 [1 << 17]byte + var x5087 [1 << 17]byte + var x5088 [1 << 17]byte + var x5089 [1 << 17]byte + var x5090 [1 << 17]byte + var x5091 [1 << 17]byte + var x5092 [1 << 17]byte + var x5093 [1 << 17]byte + var x5094 [1 << 17]byte + var x5095 [1 << 17]byte + var x5096 [1 << 17]byte + var x5097 [1 << 17]byte + var x5098 [1 << 17]byte + var x5099 [1 << 17]byte + var x5100 [1 << 17]byte + var x5101 [1 << 17]byte + var x5102 [1 << 17]byte + var x5103 [1 << 17]byte + var x5104 [1 << 17]byte + var x5105 [1 << 17]byte + var x5106 [1 << 17]byte + var x5107 [1 << 17]byte + var x5108 [1 << 17]byte + var x5109 [1 << 17]byte + var x5110 [1 << 17]byte + var x5111 [1 << 17]byte + var x5112 [1 << 17]byte + var x5113 [1 << 17]byte + var x5114 [1 << 17]byte + var x5115 [1 << 17]byte + var x5116 [1 << 17]byte + var x5117 [1 << 17]byte + var x5118 [1 << 17]byte + var x5119 [1 << 17]byte + var x5120 [1 << 17]byte + var x5121 [1 << 17]byte + var x5122 [1 << 17]byte + var x5123 [1 << 17]byte + var x5124 [1 << 17]byte + var x5125 [1 << 17]byte + var x5126 [1 << 17]byte + var x5127 [1 << 17]byte + var x5128 [1 << 17]byte + var x5129 [1 << 17]byte + var x5130 [1 << 17]byte + var x5131 [1 << 17]byte + var x5132 [1 << 17]byte + var x5133 [1 << 17]byte + var x5134 [1 << 17]byte + var x5135 [1 << 17]byte + var x5136 [1 << 17]byte + var x5137 [1 << 17]byte + var x5138 [1 << 17]byte + var x5139 [1 << 17]byte + var x5140 [1 << 17]byte + var x5141 [1 << 17]byte + var x5142 [1 << 17]byte + var x5143 [1 << 17]byte + var x5144 [1 << 17]byte + var x5145 [1 << 17]byte + var x5146 [1 << 17]byte + var x5147 [1 << 17]byte + var x5148 [1 << 17]byte + var x5149 [1 << 17]byte + var x5150 [1 << 17]byte + var x5151 [1 << 17]byte + var x5152 [1 << 17]byte + var x5153 [1 << 17]byte + var x5154 [1 << 17]byte + var x5155 [1 << 17]byte + var x5156 [1 << 17]byte + var x5157 [1 << 17]byte + var x5158 [1 << 17]byte + var x5159 [1 << 17]byte + var x5160 [1 << 17]byte + var x5161 [1 << 17]byte + var x5162 [1 << 17]byte + var x5163 [1 << 17]byte + var x5164 [1 << 17]byte + var x5165 [1 << 17]byte + var x5166 [1 << 17]byte + var x5167 [1 << 17]byte + var x5168 [1 << 17]byte + var x5169 [1 << 17]byte + var x5170 [1 << 17]byte + var x5171 [1 << 17]byte + var x5172 [1 << 17]byte + var x5173 [1 << 17]byte + var x5174 [1 << 17]byte + var x5175 [1 << 17]byte + var x5176 [1 << 17]byte + var x5177 [1 << 17]byte + var x5178 [1 << 17]byte + var x5179 [1 << 17]byte + var x5180 [1 << 17]byte + var x5181 [1 << 17]byte + var x5182 [1 << 17]byte + var x5183 [1 << 17]byte + var x5184 [1 << 17]byte + var x5185 [1 << 17]byte + var x5186 [1 << 17]byte + var x5187 [1 << 17]byte + var x5188 [1 << 17]byte + var x5189 [1 << 17]byte + var x5190 [1 << 17]byte + var x5191 [1 << 17]byte + var x5192 [1 << 17]byte + var x5193 [1 << 17]byte + var x5194 [1 << 17]byte + var x5195 [1 << 17]byte + var x5196 [1 << 17]byte + var x5197 [1 << 17]byte + var x5198 [1 << 17]byte + var x5199 [1 << 17]byte + var x5200 [1 << 17]byte + var x5201 [1 << 17]byte + var x5202 [1 << 17]byte + var x5203 [1 << 17]byte + var x5204 [1 << 17]byte + var x5205 [1 << 17]byte + var x5206 [1 << 17]byte + var x5207 [1 << 17]byte + var x5208 [1 << 17]byte + var x5209 [1 << 17]byte + var x5210 [1 << 17]byte + var x5211 [1 << 17]byte + var x5212 [1 << 17]byte + var x5213 [1 << 17]byte + var x5214 [1 << 17]byte + var x5215 [1 << 17]byte + var x5216 [1 << 17]byte + var x5217 [1 << 17]byte + var x5218 [1 << 17]byte + var x5219 [1 << 17]byte + var x5220 [1 << 17]byte + var x5221 [1 << 17]byte + var x5222 [1 << 17]byte + var x5223 [1 << 17]byte + var x5224 [1 << 17]byte + var x5225 [1 << 17]byte + var x5226 [1 << 17]byte + var x5227 [1 << 17]byte + var x5228 [1 << 17]byte + var x5229 [1 << 17]byte + var x5230 [1 << 17]byte + var x5231 [1 << 17]byte + var x5232 [1 << 17]byte + var x5233 [1 << 17]byte + var x5234 [1 << 17]byte + var x5235 [1 << 17]byte + var x5236 [1 << 17]byte + var x5237 [1 << 17]byte + var x5238 [1 << 17]byte + var x5239 [1 << 17]byte + var x5240 [1 << 17]byte + var x5241 [1 << 17]byte + var x5242 [1 << 17]byte + var x5243 [1 << 17]byte + var x5244 [1 << 17]byte + var x5245 [1 << 17]byte + var x5246 [1 << 17]byte + var x5247 [1 << 17]byte + var x5248 [1 << 17]byte + var x5249 [1 << 17]byte + var x5250 [1 << 17]byte + var x5251 [1 << 17]byte + var x5252 [1 << 17]byte + var x5253 [1 << 17]byte + var x5254 [1 << 17]byte + var x5255 [1 << 17]byte + var x5256 [1 << 17]byte + var x5257 [1 << 17]byte + var x5258 [1 << 17]byte + var x5259 [1 << 17]byte + var x5260 [1 << 17]byte + var x5261 [1 << 17]byte + var x5262 [1 << 17]byte + var x5263 [1 << 17]byte + var x5264 [1 << 17]byte + var x5265 [1 << 17]byte + var x5266 [1 << 17]byte + var x5267 [1 << 17]byte + var x5268 [1 << 17]byte + var x5269 [1 << 17]byte + var x5270 [1 << 17]byte + var x5271 [1 << 17]byte + var x5272 [1 << 17]byte + var x5273 [1 << 17]byte + var x5274 [1 << 17]byte + var x5275 [1 << 17]byte + var x5276 [1 << 17]byte + var x5277 [1 << 17]byte + var x5278 [1 << 17]byte + var x5279 [1 << 17]byte + var x5280 [1 << 17]byte + var x5281 [1 << 17]byte + var x5282 [1 << 17]byte + var x5283 [1 << 17]byte + var x5284 [1 << 17]byte + var x5285 [1 << 17]byte + var x5286 [1 << 17]byte + var x5287 [1 << 17]byte + var x5288 [1 << 17]byte + var x5289 [1 << 17]byte + var x5290 [1 << 17]byte + var x5291 [1 << 17]byte + var x5292 [1 << 17]byte + var x5293 [1 << 17]byte + var x5294 [1 << 17]byte + var x5295 [1 << 17]byte + var x5296 [1 << 17]byte + var x5297 [1 << 17]byte + var x5298 [1 << 17]byte + var x5299 [1 << 17]byte + var x5300 [1 << 17]byte + var x5301 [1 << 17]byte + var x5302 [1 << 17]byte + var x5303 [1 << 17]byte + var x5304 [1 << 17]byte + var x5305 [1 << 17]byte + var x5306 [1 << 17]byte + var x5307 [1 << 17]byte + var x5308 [1 << 17]byte + var x5309 [1 << 17]byte + var x5310 [1 << 17]byte + var x5311 [1 << 17]byte + var x5312 [1 << 17]byte + var x5313 [1 << 17]byte + var x5314 [1 << 17]byte + var x5315 [1 << 17]byte + var x5316 [1 << 17]byte + var x5317 [1 << 17]byte + var x5318 [1 << 17]byte + var x5319 [1 << 17]byte + var x5320 [1 << 17]byte + var x5321 [1 << 17]byte + var x5322 [1 << 17]byte + var x5323 [1 << 17]byte + var x5324 [1 << 17]byte + var x5325 [1 << 17]byte + var x5326 [1 << 17]byte + var x5327 [1 << 17]byte + var x5328 [1 << 17]byte + var x5329 [1 << 17]byte + var x5330 [1 << 17]byte + var x5331 [1 << 17]byte + var x5332 [1 << 17]byte + var x5333 [1 << 17]byte + var x5334 [1 << 17]byte + var x5335 [1 << 17]byte + var x5336 [1 << 17]byte + var x5337 [1 << 17]byte + var x5338 [1 << 17]byte + var x5339 [1 << 17]byte + var x5340 [1 << 17]byte + var x5341 [1 << 17]byte + var x5342 [1 << 17]byte + var x5343 [1 << 17]byte + var x5344 [1 << 17]byte + var x5345 [1 << 17]byte + var x5346 [1 << 17]byte + var x5347 [1 << 17]byte + var x5348 [1 << 17]byte + var x5349 [1 << 17]byte + var x5350 [1 << 17]byte + var x5351 [1 << 17]byte + var x5352 [1 << 17]byte + var x5353 [1 << 17]byte + var x5354 [1 << 17]byte + var x5355 [1 << 17]byte + var x5356 [1 << 17]byte + var x5357 [1 << 17]byte + var x5358 [1 << 17]byte + var x5359 [1 << 17]byte + var x5360 [1 << 17]byte + var x5361 [1 << 17]byte + var x5362 [1 << 17]byte + var x5363 [1 << 17]byte + var x5364 [1 << 17]byte + var x5365 [1 << 17]byte + var x5366 [1 << 17]byte + var x5367 [1 << 17]byte + var x5368 [1 << 17]byte + var x5369 [1 << 17]byte + var x5370 [1 << 17]byte + var x5371 [1 << 17]byte + var x5372 [1 << 17]byte + var x5373 [1 << 17]byte + var x5374 [1 << 17]byte + var x5375 [1 << 17]byte + var x5376 [1 << 17]byte + var x5377 [1 << 17]byte + var x5378 [1 << 17]byte + var x5379 [1 << 17]byte + var x5380 [1 << 17]byte + var x5381 [1 << 17]byte + var x5382 [1 << 17]byte + var x5383 [1 << 17]byte + var x5384 [1 << 17]byte + var x5385 [1 << 17]byte + var x5386 [1 << 17]byte + var x5387 [1 << 17]byte + var x5388 [1 << 17]byte + var x5389 [1 << 17]byte + var x5390 [1 << 17]byte + var x5391 [1 << 17]byte + var x5392 [1 << 17]byte + var x5393 [1 << 17]byte + var x5394 [1 << 17]byte + var x5395 [1 << 17]byte + var x5396 [1 << 17]byte + var x5397 [1 << 17]byte + var x5398 [1 << 17]byte + var x5399 [1 << 17]byte + var x5400 [1 << 17]byte + var x5401 [1 << 17]byte + var x5402 [1 << 17]byte + var x5403 [1 << 17]byte + var x5404 [1 << 17]byte + var x5405 [1 << 17]byte + var x5406 [1 << 17]byte + var x5407 [1 << 17]byte + var x5408 [1 << 17]byte + var x5409 [1 << 17]byte + var x5410 [1 << 17]byte + var x5411 [1 << 17]byte + var x5412 [1 << 17]byte + var x5413 [1 << 17]byte + var x5414 [1 << 17]byte + var x5415 [1 << 17]byte + var x5416 [1 << 17]byte + var x5417 [1 << 17]byte + var x5418 [1 << 17]byte + var x5419 [1 << 17]byte + var x5420 [1 << 17]byte + var x5421 [1 << 17]byte + var x5422 [1 << 17]byte + var x5423 [1 << 17]byte + var x5424 [1 << 17]byte + var x5425 [1 << 17]byte + var x5426 [1 << 17]byte + var x5427 [1 << 17]byte + var x5428 [1 << 17]byte + var x5429 [1 << 17]byte + var x5430 [1 << 17]byte + var x5431 [1 << 17]byte + var x5432 [1 << 17]byte + var x5433 [1 << 17]byte + var x5434 [1 << 17]byte + var x5435 [1 << 17]byte + var x5436 [1 << 17]byte + var x5437 [1 << 17]byte + var x5438 [1 << 17]byte + var x5439 [1 << 17]byte + var x5440 [1 << 17]byte + var x5441 [1 << 17]byte + var x5442 [1 << 17]byte + var x5443 [1 << 17]byte + var x5444 [1 << 17]byte + var x5445 [1 << 17]byte + var x5446 [1 << 17]byte + var x5447 [1 << 17]byte + var x5448 [1 << 17]byte + var x5449 [1 << 17]byte + var x5450 [1 << 17]byte + var x5451 [1 << 17]byte + var x5452 [1 << 17]byte + var x5453 [1 << 17]byte + var x5454 [1 << 17]byte + var x5455 [1 << 17]byte + var x5456 [1 << 17]byte + var x5457 [1 << 17]byte + var x5458 [1 << 17]byte + var x5459 [1 << 17]byte + var x5460 [1 << 17]byte + var x5461 [1 << 17]byte + var x5462 [1 << 17]byte + var x5463 [1 << 17]byte + var x5464 [1 << 17]byte + var x5465 [1 << 17]byte + var x5466 [1 << 17]byte + var x5467 [1 << 17]byte + var x5468 [1 << 17]byte + var x5469 [1 << 17]byte + var x5470 [1 << 17]byte + var x5471 [1 << 17]byte + var x5472 [1 << 17]byte + var x5473 [1 << 17]byte + var x5474 [1 << 17]byte + var x5475 [1 << 17]byte + var x5476 [1 << 17]byte + var x5477 [1 << 17]byte + var x5478 [1 << 17]byte + var x5479 [1 << 17]byte + var x5480 [1 << 17]byte + var x5481 [1 << 17]byte + var x5482 [1 << 17]byte + var x5483 [1 << 17]byte + var x5484 [1 << 17]byte + var x5485 [1 << 17]byte + var x5486 [1 << 17]byte + var x5487 [1 << 17]byte + var x5488 [1 << 17]byte + var x5489 [1 << 17]byte + var x5490 [1 << 17]byte + var x5491 [1 << 17]byte + var x5492 [1 << 17]byte + var x5493 [1 << 17]byte + var x5494 [1 << 17]byte + var x5495 [1 << 17]byte + var x5496 [1 << 17]byte + var x5497 [1 << 17]byte + var x5498 [1 << 17]byte + var x5499 [1 << 17]byte + var x5500 [1 << 17]byte + var x5501 [1 << 17]byte + var x5502 [1 << 17]byte + var x5503 [1 << 17]byte + var x5504 [1 << 17]byte + var x5505 [1 << 17]byte + var x5506 [1 << 17]byte + var x5507 [1 << 17]byte + var x5508 [1 << 17]byte + var x5509 [1 << 17]byte + var x5510 [1 << 17]byte + var x5511 [1 << 17]byte + var x5512 [1 << 17]byte + var x5513 [1 << 17]byte + var x5514 [1 << 17]byte + var x5515 [1 << 17]byte + var x5516 [1 << 17]byte + var x5517 [1 << 17]byte + var x5518 [1 << 17]byte + var x5519 [1 << 17]byte + var x5520 [1 << 17]byte + var x5521 [1 << 17]byte + var x5522 [1 << 17]byte + var x5523 [1 << 17]byte + var x5524 [1 << 17]byte + var x5525 [1 << 17]byte + var x5526 [1 << 17]byte + var x5527 [1 << 17]byte + var x5528 [1 << 17]byte + var x5529 [1 << 17]byte + var x5530 [1 << 17]byte + var x5531 [1 << 17]byte + var x5532 [1 << 17]byte + var x5533 [1 << 17]byte + var x5534 [1 << 17]byte + var x5535 [1 << 17]byte + var x5536 [1 << 17]byte + var x5537 [1 << 17]byte + var x5538 [1 << 17]byte + var x5539 [1 << 17]byte + var x5540 [1 << 17]byte + var x5541 [1 << 17]byte + var x5542 [1 << 17]byte + var x5543 [1 << 17]byte + var x5544 [1 << 17]byte + var x5545 [1 << 17]byte + var x5546 [1 << 17]byte + var x5547 [1 << 17]byte + var x5548 [1 << 17]byte + var x5549 [1 << 17]byte + var x5550 [1 << 17]byte + var x5551 [1 << 17]byte + var x5552 [1 << 17]byte + var x5553 [1 << 17]byte + var x5554 [1 << 17]byte + var x5555 [1 << 17]byte + var x5556 [1 << 17]byte + var x5557 [1 << 17]byte + var x5558 [1 << 17]byte + var x5559 [1 << 17]byte + var x5560 [1 << 17]byte + var x5561 [1 << 17]byte + var x5562 [1 << 17]byte + var x5563 [1 << 17]byte + var x5564 [1 << 17]byte + var x5565 [1 << 17]byte + var x5566 [1 << 17]byte + var x5567 [1 << 17]byte + var x5568 [1 << 17]byte + var x5569 [1 << 17]byte + var x5570 [1 << 17]byte + var x5571 [1 << 17]byte + var x5572 [1 << 17]byte + var x5573 [1 << 17]byte + var x5574 [1 << 17]byte + var x5575 [1 << 17]byte + var x5576 [1 << 17]byte + var x5577 [1 << 17]byte + var x5578 [1 << 17]byte + var x5579 [1 << 17]byte + var x5580 [1 << 17]byte + var x5581 [1 << 17]byte + var x5582 [1 << 17]byte + var x5583 [1 << 17]byte + var x5584 [1 << 17]byte + var x5585 [1 << 17]byte + var x5586 [1 << 17]byte + var x5587 [1 << 17]byte + var x5588 [1 << 17]byte + var x5589 [1 << 17]byte + var x5590 [1 << 17]byte + var x5591 [1 << 17]byte + var x5592 [1 << 17]byte + var x5593 [1 << 17]byte + var x5594 [1 << 17]byte + var x5595 [1 << 17]byte + var x5596 [1 << 17]byte + var x5597 [1 << 17]byte + var x5598 [1 << 17]byte + var x5599 [1 << 17]byte + var x5600 [1 << 17]byte + var x5601 [1 << 17]byte + var x5602 [1 << 17]byte + var x5603 [1 << 17]byte + var x5604 [1 << 17]byte + var x5605 [1 << 17]byte + var x5606 [1 << 17]byte + var x5607 [1 << 17]byte + var x5608 [1 << 17]byte + var x5609 [1 << 17]byte + var x5610 [1 << 17]byte + var x5611 [1 << 17]byte + var x5612 [1 << 17]byte + var x5613 [1 << 17]byte + var x5614 [1 << 17]byte + var x5615 [1 << 17]byte + var x5616 [1 << 17]byte + var x5617 [1 << 17]byte + var x5618 [1 << 17]byte + var x5619 [1 << 17]byte + var x5620 [1 << 17]byte + var x5621 [1 << 17]byte + var x5622 [1 << 17]byte + var x5623 [1 << 17]byte + var x5624 [1 << 17]byte + var x5625 [1 << 17]byte + var x5626 [1 << 17]byte + var x5627 [1 << 17]byte + var x5628 [1 << 17]byte + var x5629 [1 << 17]byte + var x5630 [1 << 17]byte + var x5631 [1 << 17]byte + var x5632 [1 << 17]byte + var x5633 [1 << 17]byte + var x5634 [1 << 17]byte + var x5635 [1 << 17]byte + var x5636 [1 << 17]byte + var x5637 [1 << 17]byte + var x5638 [1 << 17]byte + var x5639 [1 << 17]byte + var x5640 [1 << 17]byte + var x5641 [1 << 17]byte + var x5642 [1 << 17]byte + var x5643 [1 << 17]byte + var x5644 [1 << 17]byte + var x5645 [1 << 17]byte + var x5646 [1 << 17]byte + var x5647 [1 << 17]byte + var x5648 [1 << 17]byte + var x5649 [1 << 17]byte + var x5650 [1 << 17]byte + var x5651 [1 << 17]byte + var x5652 [1 << 17]byte + var x5653 [1 << 17]byte + var x5654 [1 << 17]byte + var x5655 [1 << 17]byte + var x5656 [1 << 17]byte + var x5657 [1 << 17]byte + var x5658 [1 << 17]byte + var x5659 [1 << 17]byte + var x5660 [1 << 17]byte + var x5661 [1 << 17]byte + var x5662 [1 << 17]byte + var x5663 [1 << 17]byte + var x5664 [1 << 17]byte + var x5665 [1 << 17]byte + var x5666 [1 << 17]byte + var x5667 [1 << 17]byte + var x5668 [1 << 17]byte + var x5669 [1 << 17]byte + var x5670 [1 << 17]byte + var x5671 [1 << 17]byte + var x5672 [1 << 17]byte + var x5673 [1 << 17]byte + var x5674 [1 << 17]byte + var x5675 [1 << 17]byte + var x5676 [1 << 17]byte + var x5677 [1 << 17]byte + var x5678 [1 << 17]byte + var x5679 [1 << 17]byte + var x5680 [1 << 17]byte + var x5681 [1 << 17]byte + var x5682 [1 << 17]byte + var x5683 [1 << 17]byte + var x5684 [1 << 17]byte + var x5685 [1 << 17]byte + var x5686 [1 << 17]byte + var x5687 [1 << 17]byte + var x5688 [1 << 17]byte + var x5689 [1 << 17]byte + var x5690 [1 << 17]byte + var x5691 [1 << 17]byte + var x5692 [1 << 17]byte + var x5693 [1 << 17]byte + var x5694 [1 << 17]byte + var x5695 [1 << 17]byte + var x5696 [1 << 17]byte + var x5697 [1 << 17]byte + var x5698 [1 << 17]byte + var x5699 [1 << 17]byte + var x5700 [1 << 17]byte + var x5701 [1 << 17]byte + var x5702 [1 << 17]byte + var x5703 [1 << 17]byte + var x5704 [1 << 17]byte + var x5705 [1 << 17]byte + var x5706 [1 << 17]byte + var x5707 [1 << 17]byte + var x5708 [1 << 17]byte + var x5709 [1 << 17]byte + var x5710 [1 << 17]byte + var x5711 [1 << 17]byte + var x5712 [1 << 17]byte + var x5713 [1 << 17]byte + var x5714 [1 << 17]byte + var x5715 [1 << 17]byte + var x5716 [1 << 17]byte + var x5717 [1 << 17]byte + var x5718 [1 << 17]byte + var x5719 [1 << 17]byte + var x5720 [1 << 17]byte + var x5721 [1 << 17]byte + var x5722 [1 << 17]byte + var x5723 [1 << 17]byte + var x5724 [1 << 17]byte + var x5725 [1 << 17]byte + var x5726 [1 << 17]byte + var x5727 [1 << 17]byte + var x5728 [1 << 17]byte + var x5729 [1 << 17]byte + var x5730 [1 << 17]byte + var x5731 [1 << 17]byte + var x5732 [1 << 17]byte + var x5733 [1 << 17]byte + var x5734 [1 << 17]byte + var x5735 [1 << 17]byte + var x5736 [1 << 17]byte + var x5737 [1 << 17]byte + var x5738 [1 << 17]byte + var x5739 [1 << 17]byte + var x5740 [1 << 17]byte + var x5741 [1 << 17]byte + var x5742 [1 << 17]byte + var x5743 [1 << 17]byte + var x5744 [1 << 17]byte + var x5745 [1 << 17]byte + var x5746 [1 << 17]byte + var x5747 [1 << 17]byte + var x5748 [1 << 17]byte + var x5749 [1 << 17]byte + var x5750 [1 << 17]byte + var x5751 [1 << 17]byte + var x5752 [1 << 17]byte + var x5753 [1 << 17]byte + var x5754 [1 << 17]byte + var x5755 [1 << 17]byte + var x5756 [1 << 17]byte + var x5757 [1 << 17]byte + var x5758 [1 << 17]byte + var x5759 [1 << 17]byte + var x5760 [1 << 17]byte + var x5761 [1 << 17]byte + var x5762 [1 << 17]byte + var x5763 [1 << 17]byte + var x5764 [1 << 17]byte + var x5765 [1 << 17]byte + var x5766 [1 << 17]byte + var x5767 [1 << 17]byte + var x5768 [1 << 17]byte + var x5769 [1 << 17]byte + var x5770 [1 << 17]byte + var x5771 [1 << 17]byte + var x5772 [1 << 17]byte + var x5773 [1 << 17]byte + var x5774 [1 << 17]byte + var x5775 [1 << 17]byte + var x5776 [1 << 17]byte + var x5777 [1 << 17]byte + var x5778 [1 << 17]byte + var x5779 [1 << 17]byte + var x5780 [1 << 17]byte + var x5781 [1 << 17]byte + var x5782 [1 << 17]byte + var x5783 [1 << 17]byte + var x5784 [1 << 17]byte + var x5785 [1 << 17]byte + var x5786 [1 << 17]byte + var x5787 [1 << 17]byte + var x5788 [1 << 17]byte + var x5789 [1 << 17]byte + var x5790 [1 << 17]byte + var x5791 [1 << 17]byte + var x5792 [1 << 17]byte + var x5793 [1 << 17]byte + var x5794 [1 << 17]byte + var x5795 [1 << 17]byte + var x5796 [1 << 17]byte + var x5797 [1 << 17]byte + var x5798 [1 << 17]byte + var x5799 [1 << 17]byte + var x5800 [1 << 17]byte + var x5801 [1 << 17]byte + var x5802 [1 << 17]byte + var x5803 [1 << 17]byte + var x5804 [1 << 17]byte + var x5805 [1 << 17]byte + var x5806 [1 << 17]byte + var x5807 [1 << 17]byte + var x5808 [1 << 17]byte + var x5809 [1 << 17]byte + var x5810 [1 << 17]byte + var x5811 [1 << 17]byte + var x5812 [1 << 17]byte + var x5813 [1 << 17]byte + var x5814 [1 << 17]byte + var x5815 [1 << 17]byte + var x5816 [1 << 17]byte + var x5817 [1 << 17]byte + var x5818 [1 << 17]byte + var x5819 [1 << 17]byte + var x5820 [1 << 17]byte + var x5821 [1 << 17]byte + var x5822 [1 << 17]byte + var x5823 [1 << 17]byte + var x5824 [1 << 17]byte + var x5825 [1 << 17]byte + var x5826 [1 << 17]byte + var x5827 [1 << 17]byte + var x5828 [1 << 17]byte + var x5829 [1 << 17]byte + var x5830 [1 << 17]byte + var x5831 [1 << 17]byte + var x5832 [1 << 17]byte + var x5833 [1 << 17]byte + var x5834 [1 << 17]byte + var x5835 [1 << 17]byte + var x5836 [1 << 17]byte + var x5837 [1 << 17]byte + var x5838 [1 << 17]byte + var x5839 [1 << 17]byte + var x5840 [1 << 17]byte + var x5841 [1 << 17]byte + var x5842 [1 << 17]byte + var x5843 [1 << 17]byte + var x5844 [1 << 17]byte + var x5845 [1 << 17]byte + var x5846 [1 << 17]byte + var x5847 [1 << 17]byte + var x5848 [1 << 17]byte + var x5849 [1 << 17]byte + var x5850 [1 << 17]byte + var x5851 [1 << 17]byte + var x5852 [1 << 17]byte + var x5853 [1 << 17]byte + var x5854 [1 << 17]byte + var x5855 [1 << 17]byte + var x5856 [1 << 17]byte + var x5857 [1 << 17]byte + var x5858 [1 << 17]byte + var x5859 [1 << 17]byte + var x5860 [1 << 17]byte + var x5861 [1 << 17]byte + var x5862 [1 << 17]byte + var x5863 [1 << 17]byte + var x5864 [1 << 17]byte + var x5865 [1 << 17]byte + var x5866 [1 << 17]byte + var x5867 [1 << 17]byte + var x5868 [1 << 17]byte + var x5869 [1 << 17]byte + var x5870 [1 << 17]byte + var x5871 [1 << 17]byte + var x5872 [1 << 17]byte + var x5873 [1 << 17]byte + var x5874 [1 << 17]byte + var x5875 [1 << 17]byte + var x5876 [1 << 17]byte + var x5877 [1 << 17]byte + var x5878 [1 << 17]byte + var x5879 [1 << 17]byte + var x5880 [1 << 17]byte + var x5881 [1 << 17]byte + var x5882 [1 << 17]byte + var x5883 [1 << 17]byte + var x5884 [1 << 17]byte + var x5885 [1 << 17]byte + var x5886 [1 << 17]byte + var x5887 [1 << 17]byte + var x5888 [1 << 17]byte + var x5889 [1 << 17]byte + var x5890 [1 << 17]byte + var x5891 [1 << 17]byte + var x5892 [1 << 17]byte + var x5893 [1 << 17]byte + var x5894 [1 << 17]byte + var x5895 [1 << 17]byte + var x5896 [1 << 17]byte + var x5897 [1 << 17]byte + var x5898 [1 << 17]byte + var x5899 [1 << 17]byte + var x5900 [1 << 17]byte + var x5901 [1 << 17]byte + var x5902 [1 << 17]byte + var x5903 [1 << 17]byte + var x5904 [1 << 17]byte + var x5905 [1 << 17]byte + var x5906 [1 << 17]byte + var x5907 [1 << 17]byte + var x5908 [1 << 17]byte + var x5909 [1 << 17]byte + var x5910 [1 << 17]byte + var x5911 [1 << 17]byte + var x5912 [1 << 17]byte + var x5913 [1 << 17]byte + var x5914 [1 << 17]byte + var x5915 [1 << 17]byte + var x5916 [1 << 17]byte + var x5917 [1 << 17]byte + var x5918 [1 << 17]byte + var x5919 [1 << 17]byte + var x5920 [1 << 17]byte + var x5921 [1 << 17]byte + var x5922 [1 << 17]byte + var x5923 [1 << 17]byte + var x5924 [1 << 17]byte + var x5925 [1 << 17]byte + var x5926 [1 << 17]byte + var x5927 [1 << 17]byte + var x5928 [1 << 17]byte + var x5929 [1 << 17]byte + var x5930 [1 << 17]byte + var x5931 [1 << 17]byte + var x5932 [1 << 17]byte + var x5933 [1 << 17]byte + var x5934 [1 << 17]byte + var x5935 [1 << 17]byte + var x5936 [1 << 17]byte + var x5937 [1 << 17]byte + var x5938 [1 << 17]byte + var x5939 [1 << 17]byte + var x5940 [1 << 17]byte + var x5941 [1 << 17]byte + var x5942 [1 << 17]byte + var x5943 [1 << 17]byte + var x5944 [1 << 17]byte + var x5945 [1 << 17]byte + var x5946 [1 << 17]byte + var x5947 [1 << 17]byte + var x5948 [1 << 17]byte + var x5949 [1 << 17]byte + var x5950 [1 << 17]byte + var x5951 [1 << 17]byte + var x5952 [1 << 17]byte + var x5953 [1 << 17]byte + var x5954 [1 << 17]byte + var x5955 [1 << 17]byte + var x5956 [1 << 17]byte + var x5957 [1 << 17]byte + var x5958 [1 << 17]byte + var x5959 [1 << 17]byte + var x5960 [1 << 17]byte + var x5961 [1 << 17]byte + var x5962 [1 << 17]byte + var x5963 [1 << 17]byte + var x5964 [1 << 17]byte + var x5965 [1 << 17]byte + var x5966 [1 << 17]byte + var x5967 [1 << 17]byte + var x5968 [1 << 17]byte + var x5969 [1 << 17]byte + var x5970 [1 << 17]byte + var x5971 [1 << 17]byte + var x5972 [1 << 17]byte + var x5973 [1 << 17]byte + var x5974 [1 << 17]byte + var x5975 [1 << 17]byte + var x5976 [1 << 17]byte + var x5977 [1 << 17]byte + var x5978 [1 << 17]byte + var x5979 [1 << 17]byte + var x5980 [1 << 17]byte + var x5981 [1 << 17]byte + var x5982 [1 << 17]byte + var x5983 [1 << 17]byte + var x5984 [1 << 17]byte + var x5985 [1 << 17]byte + var x5986 [1 << 17]byte + var x5987 [1 << 17]byte + var x5988 [1 << 17]byte + var x5989 [1 << 17]byte + var x5990 [1 << 17]byte + var x5991 [1 << 17]byte + var x5992 [1 << 17]byte + var x5993 [1 << 17]byte + var x5994 [1 << 17]byte + var x5995 [1 << 17]byte + var x5996 [1 << 17]byte + var x5997 [1 << 17]byte + var x5998 [1 << 17]byte + var x5999 [1 << 17]byte + var x6000 [1 << 17]byte + var x6001 [1 << 17]byte + var x6002 [1 << 17]byte + var x6003 [1 << 17]byte + var x6004 [1 << 17]byte + var x6005 [1 << 17]byte + var x6006 [1 << 17]byte + var x6007 [1 << 17]byte + var x6008 [1 << 17]byte + var x6009 [1 << 17]byte + var x6010 [1 << 17]byte + var x6011 [1 << 17]byte + var x6012 [1 << 17]byte + var x6013 [1 << 17]byte + var x6014 [1 << 17]byte + var x6015 [1 << 17]byte + var x6016 [1 << 17]byte + var x6017 [1 << 17]byte + var x6018 [1 << 17]byte + var x6019 [1 << 17]byte + var x6020 [1 << 17]byte + var x6021 [1 << 17]byte + var x6022 [1 << 17]byte + var x6023 [1 << 17]byte + var x6024 [1 << 17]byte + var x6025 [1 << 17]byte + var x6026 [1 << 17]byte + var x6027 [1 << 17]byte + var x6028 [1 << 17]byte + var x6029 [1 << 17]byte + var x6030 [1 << 17]byte + var x6031 [1 << 17]byte + var x6032 [1 << 17]byte + var x6033 [1 << 17]byte + var x6034 [1 << 17]byte + var x6035 [1 << 17]byte + var x6036 [1 << 17]byte + var x6037 [1 << 17]byte + var x6038 [1 << 17]byte + var x6039 [1 << 17]byte + var x6040 [1 << 17]byte + var x6041 [1 << 17]byte + var x6042 [1 << 17]byte + var x6043 [1 << 17]byte + var x6044 [1 << 17]byte + var x6045 [1 << 17]byte + var x6046 [1 << 17]byte + var x6047 [1 << 17]byte + var x6048 [1 << 17]byte + var x6049 [1 << 17]byte + var x6050 [1 << 17]byte + var x6051 [1 << 17]byte + var x6052 [1 << 17]byte + var x6053 [1 << 17]byte + var x6054 [1 << 17]byte + var x6055 [1 << 17]byte + var x6056 [1 << 17]byte + var x6057 [1 << 17]byte + var x6058 [1 << 17]byte + var x6059 [1 << 17]byte + var x6060 [1 << 17]byte + var x6061 [1 << 17]byte + var x6062 [1 << 17]byte + var x6063 [1 << 17]byte + var x6064 [1 << 17]byte + var x6065 [1 << 17]byte + var x6066 [1 << 17]byte + var x6067 [1 << 17]byte + var x6068 [1 << 17]byte + var x6069 [1 << 17]byte + var x6070 [1 << 17]byte + var x6071 [1 << 17]byte + var x6072 [1 << 17]byte + var x6073 [1 << 17]byte + var x6074 [1 << 17]byte + var x6075 [1 << 17]byte + var x6076 [1 << 17]byte + var x6077 [1 << 17]byte + var x6078 [1 << 17]byte + var x6079 [1 << 17]byte + var x6080 [1 << 17]byte + var x6081 [1 << 17]byte + var x6082 [1 << 17]byte + var x6083 [1 << 17]byte + var x6084 [1 << 17]byte + var x6085 [1 << 17]byte + var x6086 [1 << 17]byte + var x6087 [1 << 17]byte + var x6088 [1 << 17]byte + var x6089 [1 << 17]byte + var x6090 [1 << 17]byte + var x6091 [1 << 17]byte + var x6092 [1 << 17]byte + var x6093 [1 << 17]byte + var x6094 [1 << 17]byte + var x6095 [1 << 17]byte + var x6096 [1 << 17]byte + var x6097 [1 << 17]byte + var x6098 [1 << 17]byte + var x6099 [1 << 17]byte + var x6100 [1 << 17]byte + var x6101 [1 << 17]byte + var x6102 [1 << 17]byte + var x6103 [1 << 17]byte + var x6104 [1 << 17]byte + var x6105 [1 << 17]byte + var x6106 [1 << 17]byte + var x6107 [1 << 17]byte + var x6108 [1 << 17]byte + var x6109 [1 << 17]byte + var x6110 [1 << 17]byte + var x6111 [1 << 17]byte + var x6112 [1 << 17]byte + var x6113 [1 << 17]byte + var x6114 [1 << 17]byte + var x6115 [1 << 17]byte + var x6116 [1 << 17]byte + var x6117 [1 << 17]byte + var x6118 [1 << 17]byte + var x6119 [1 << 17]byte + var x6120 [1 << 17]byte + var x6121 [1 << 17]byte + var x6122 [1 << 17]byte + var x6123 [1 << 17]byte + var x6124 [1 << 17]byte + var x6125 [1 << 17]byte + var x6126 [1 << 17]byte + var x6127 [1 << 17]byte + var x6128 [1 << 17]byte + var x6129 [1 << 17]byte + var x6130 [1 << 17]byte + var x6131 [1 << 17]byte + var x6132 [1 << 17]byte + var x6133 [1 << 17]byte + var x6134 [1 << 17]byte + var x6135 [1 << 17]byte + var x6136 [1 << 17]byte + var x6137 [1 << 17]byte + var x6138 [1 << 17]byte + var x6139 [1 << 17]byte + var x6140 [1 << 17]byte + var x6141 [1 << 17]byte + var x6142 [1 << 17]byte + var x6143 [1 << 17]byte + var x6144 [1 << 17]byte + var x6145 [1 << 17]byte + var x6146 [1 << 17]byte + var x6147 [1 << 17]byte + var x6148 [1 << 17]byte + var x6149 [1 << 17]byte + var x6150 [1 << 17]byte + var x6151 [1 << 17]byte + var x6152 [1 << 17]byte + var x6153 [1 << 17]byte + var x6154 [1 << 17]byte + var x6155 [1 << 17]byte + var x6156 [1 << 17]byte + var x6157 [1 << 17]byte + var x6158 [1 << 17]byte + var x6159 [1 << 17]byte + var x6160 [1 << 17]byte + var x6161 [1 << 17]byte + var x6162 [1 << 17]byte + var x6163 [1 << 17]byte + var x6164 [1 << 17]byte + var x6165 [1 << 17]byte + var x6166 [1 << 17]byte + var x6167 [1 << 17]byte + var x6168 [1 << 17]byte + var x6169 [1 << 17]byte + var x6170 [1 << 17]byte + var x6171 [1 << 17]byte + var x6172 [1 << 17]byte + var x6173 [1 << 17]byte + var x6174 [1 << 17]byte + var x6175 [1 << 17]byte + var x6176 [1 << 17]byte + var x6177 [1 << 17]byte + var x6178 [1 << 17]byte + var x6179 [1 << 17]byte + var x6180 [1 << 17]byte + var x6181 [1 << 17]byte + var x6182 [1 << 17]byte + var x6183 [1 << 17]byte + var x6184 [1 << 17]byte + var x6185 [1 << 17]byte + var x6186 [1 << 17]byte + var x6187 [1 << 17]byte + var x6188 [1 << 17]byte + var x6189 [1 << 17]byte + var x6190 [1 << 17]byte + var x6191 [1 << 17]byte + var x6192 [1 << 17]byte + var x6193 [1 << 17]byte + var x6194 [1 << 17]byte + var x6195 [1 << 17]byte + var x6196 [1 << 17]byte + var x6197 [1 << 17]byte + var x6198 [1 << 17]byte + var x6199 [1 << 17]byte + var x6200 [1 << 17]byte + var x6201 [1 << 17]byte + var x6202 [1 << 17]byte + var x6203 [1 << 17]byte + var x6204 [1 << 17]byte + var x6205 [1 << 17]byte + var x6206 [1 << 17]byte + var x6207 [1 << 17]byte + var x6208 [1 << 17]byte + var x6209 [1 << 17]byte + var x6210 [1 << 17]byte + var x6211 [1 << 17]byte + var x6212 [1 << 17]byte + var x6213 [1 << 17]byte + var x6214 [1 << 17]byte + var x6215 [1 << 17]byte + var x6216 [1 << 17]byte + var x6217 [1 << 17]byte + var x6218 [1 << 17]byte + var x6219 [1 << 17]byte + var x6220 [1 << 17]byte + var x6221 [1 << 17]byte + var x6222 [1 << 17]byte + var x6223 [1 << 17]byte + var x6224 [1 << 17]byte + var x6225 [1 << 17]byte + var x6226 [1 << 17]byte + var x6227 [1 << 17]byte + var x6228 [1 << 17]byte + var x6229 [1 << 17]byte + var x6230 [1 << 17]byte + var x6231 [1 << 17]byte + var x6232 [1 << 17]byte + var x6233 [1 << 17]byte + var x6234 [1 << 17]byte + var x6235 [1 << 17]byte + var x6236 [1 << 17]byte + var x6237 [1 << 17]byte + var x6238 [1 << 17]byte + var x6239 [1 << 17]byte + var x6240 [1 << 17]byte + var x6241 [1 << 17]byte + var x6242 [1 << 17]byte + var x6243 [1 << 17]byte + var x6244 [1 << 17]byte + var x6245 [1 << 17]byte + var x6246 [1 << 17]byte + var x6247 [1 << 17]byte + var x6248 [1 << 17]byte + var x6249 [1 << 17]byte + var x6250 [1 << 17]byte + var x6251 [1 << 17]byte + var x6252 [1 << 17]byte + var x6253 [1 << 17]byte + var x6254 [1 << 17]byte + var x6255 [1 << 17]byte + var x6256 [1 << 17]byte + var x6257 [1 << 17]byte + var x6258 [1 << 17]byte + var x6259 [1 << 17]byte + var x6260 [1 << 17]byte + var x6261 [1 << 17]byte + var x6262 [1 << 17]byte + var x6263 [1 << 17]byte + var x6264 [1 << 17]byte + var x6265 [1 << 17]byte + var x6266 [1 << 17]byte + var x6267 [1 << 17]byte + var x6268 [1 << 17]byte + var x6269 [1 << 17]byte + var x6270 [1 << 17]byte + var x6271 [1 << 17]byte + var x6272 [1 << 17]byte + var x6273 [1 << 17]byte + var x6274 [1 << 17]byte + var x6275 [1 << 17]byte + var x6276 [1 << 17]byte + var x6277 [1 << 17]byte + var x6278 [1 << 17]byte + var x6279 [1 << 17]byte + var x6280 [1 << 17]byte + var x6281 [1 << 17]byte + var x6282 [1 << 17]byte + var x6283 [1 << 17]byte + var x6284 [1 << 17]byte + var x6285 [1 << 17]byte + var x6286 [1 << 17]byte + var x6287 [1 << 17]byte + var x6288 [1 << 17]byte + var x6289 [1 << 17]byte + var x6290 [1 << 17]byte + var x6291 [1 << 17]byte + var x6292 [1 << 17]byte + var x6293 [1 << 17]byte + var x6294 [1 << 17]byte + var x6295 [1 << 17]byte + var x6296 [1 << 17]byte + var x6297 [1 << 17]byte + var x6298 [1 << 17]byte + var x6299 [1 << 17]byte + var x6300 [1 << 17]byte + var x6301 [1 << 17]byte + var x6302 [1 << 17]byte + var x6303 [1 << 17]byte + var x6304 [1 << 17]byte + var x6305 [1 << 17]byte + var x6306 [1 << 17]byte + var x6307 [1 << 17]byte + var x6308 [1 << 17]byte + var x6309 [1 << 17]byte + var x6310 [1 << 17]byte + var x6311 [1 << 17]byte + var x6312 [1 << 17]byte + var x6313 [1 << 17]byte + var x6314 [1 << 17]byte + var x6315 [1 << 17]byte + var x6316 [1 << 17]byte + var x6317 [1 << 17]byte + var x6318 [1 << 17]byte + var x6319 [1 << 17]byte + var x6320 [1 << 17]byte + var x6321 [1 << 17]byte + var x6322 [1 << 17]byte + var x6323 [1 << 17]byte + var x6324 [1 << 17]byte + var x6325 [1 << 17]byte + var x6326 [1 << 17]byte + var x6327 [1 << 17]byte + var x6328 [1 << 17]byte + var x6329 [1 << 17]byte + var x6330 [1 << 17]byte + var x6331 [1 << 17]byte + var x6332 [1 << 17]byte + var x6333 [1 << 17]byte + var x6334 [1 << 17]byte + var x6335 [1 << 17]byte + var x6336 [1 << 17]byte + var x6337 [1 << 17]byte + var x6338 [1 << 17]byte + var x6339 [1 << 17]byte + var x6340 [1 << 17]byte + var x6341 [1 << 17]byte + var x6342 [1 << 17]byte + var x6343 [1 << 17]byte + var x6344 [1 << 17]byte + var x6345 [1 << 17]byte + var x6346 [1 << 17]byte + var x6347 [1 << 17]byte + var x6348 [1 << 17]byte + var x6349 [1 << 17]byte + var x6350 [1 << 17]byte + var x6351 [1 << 17]byte + var x6352 [1 << 17]byte + var x6353 [1 << 17]byte + var x6354 [1 << 17]byte + var x6355 [1 << 17]byte + var x6356 [1 << 17]byte + var x6357 [1 << 17]byte + var x6358 [1 << 17]byte + var x6359 [1 << 17]byte + var x6360 [1 << 17]byte + var x6361 [1 << 17]byte + var x6362 [1 << 17]byte + var x6363 [1 << 17]byte + var x6364 [1 << 17]byte + var x6365 [1 << 17]byte + var x6366 [1 << 17]byte + var x6367 [1 << 17]byte + var x6368 [1 << 17]byte + var x6369 [1 << 17]byte + var x6370 [1 << 17]byte + var x6371 [1 << 17]byte + var x6372 [1 << 17]byte + var x6373 [1 << 17]byte + var x6374 [1 << 17]byte + var x6375 [1 << 17]byte + var x6376 [1 << 17]byte + var x6377 [1 << 17]byte + var x6378 [1 << 17]byte + var x6379 [1 << 17]byte + var x6380 [1 << 17]byte + var x6381 [1 << 17]byte + var x6382 [1 << 17]byte + var x6383 [1 << 17]byte + var x6384 [1 << 17]byte + var x6385 [1 << 17]byte + var x6386 [1 << 17]byte + var x6387 [1 << 17]byte + var x6388 [1 << 17]byte + var x6389 [1 << 17]byte + var x6390 [1 << 17]byte + var x6391 [1 << 17]byte + var x6392 [1 << 17]byte + var x6393 [1 << 17]byte + var x6394 [1 << 17]byte + var x6395 [1 << 17]byte + var x6396 [1 << 17]byte + var x6397 [1 << 17]byte + var x6398 [1 << 17]byte + var x6399 [1 << 17]byte + var x6400 [1 << 17]byte + var x6401 [1 << 17]byte + var x6402 [1 << 17]byte + var x6403 [1 << 17]byte + var x6404 [1 << 17]byte + var x6405 [1 << 17]byte + var x6406 [1 << 17]byte + var x6407 [1 << 17]byte + var x6408 [1 << 17]byte + var x6409 [1 << 17]byte + var x6410 [1 << 17]byte + var x6411 [1 << 17]byte + var x6412 [1 << 17]byte + var x6413 [1 << 17]byte + var x6414 [1 << 17]byte + var x6415 [1 << 17]byte + var x6416 [1 << 17]byte + var x6417 [1 << 17]byte + var x6418 [1 << 17]byte + var x6419 [1 << 17]byte + var x6420 [1 << 17]byte + var x6421 [1 << 17]byte + var x6422 [1 << 17]byte + var x6423 [1 << 17]byte + var x6424 [1 << 17]byte + var x6425 [1 << 17]byte + var x6426 [1 << 17]byte + var x6427 [1 << 17]byte + var x6428 [1 << 17]byte + var x6429 [1 << 17]byte + var x6430 [1 << 17]byte + var x6431 [1 << 17]byte + var x6432 [1 << 17]byte + var x6433 [1 << 17]byte + var x6434 [1 << 17]byte + var x6435 [1 << 17]byte + var x6436 [1 << 17]byte + var x6437 [1 << 17]byte + var x6438 [1 << 17]byte + var x6439 [1 << 17]byte + var x6440 [1 << 17]byte + var x6441 [1 << 17]byte + var x6442 [1 << 17]byte + var x6443 [1 << 17]byte + var x6444 [1 << 17]byte + var x6445 [1 << 17]byte + var x6446 [1 << 17]byte + var x6447 [1 << 17]byte + var x6448 [1 << 17]byte + var x6449 [1 << 17]byte + var x6450 [1 << 17]byte + var x6451 [1 << 17]byte + var x6452 [1 << 17]byte + var x6453 [1 << 17]byte + var x6454 [1 << 17]byte + var x6455 [1 << 17]byte + var x6456 [1 << 17]byte + var x6457 [1 << 17]byte + var x6458 [1 << 17]byte + var x6459 [1 << 17]byte + var x6460 [1 << 17]byte + var x6461 [1 << 17]byte + var x6462 [1 << 17]byte + var x6463 [1 << 17]byte + var x6464 [1 << 17]byte + var x6465 [1 << 17]byte + var x6466 [1 << 17]byte + var x6467 [1 << 17]byte + var x6468 [1 << 17]byte + var x6469 [1 << 17]byte + var x6470 [1 << 17]byte + var x6471 [1 << 17]byte + var x6472 [1 << 17]byte + var x6473 [1 << 17]byte + var x6474 [1 << 17]byte + var x6475 [1 << 17]byte + var x6476 [1 << 17]byte + var x6477 [1 << 17]byte + var x6478 [1 << 17]byte + var x6479 [1 << 17]byte + var x6480 [1 << 17]byte + var x6481 [1 << 17]byte + var x6482 [1 << 17]byte + var x6483 [1 << 17]byte + var x6484 [1 << 17]byte + var x6485 [1 << 17]byte + var x6486 [1 << 17]byte + var x6487 [1 << 17]byte + var x6488 [1 << 17]byte + var x6489 [1 << 17]byte + var x6490 [1 << 17]byte + var x6491 [1 << 17]byte + var x6492 [1 << 17]byte + var x6493 [1 << 17]byte + var x6494 [1 << 17]byte + var x6495 [1 << 17]byte + var x6496 [1 << 17]byte + var x6497 [1 << 17]byte + var x6498 [1 << 17]byte + var x6499 [1 << 17]byte + var x6500 [1 << 17]byte + var x6501 [1 << 17]byte + var x6502 [1 << 17]byte + var x6503 [1 << 17]byte + var x6504 [1 << 17]byte + var x6505 [1 << 17]byte + var x6506 [1 << 17]byte + var x6507 [1 << 17]byte + var x6508 [1 << 17]byte + var x6509 [1 << 17]byte + var x6510 [1 << 17]byte + var x6511 [1 << 17]byte + var x6512 [1 << 17]byte + var x6513 [1 << 17]byte + var x6514 [1 << 17]byte + var x6515 [1 << 17]byte + var x6516 [1 << 17]byte + var x6517 [1 << 17]byte + var x6518 [1 << 17]byte + var x6519 [1 << 17]byte + var x6520 [1 << 17]byte + var x6521 [1 << 17]byte + var x6522 [1 << 17]byte + var x6523 [1 << 17]byte + var x6524 [1 << 17]byte + var x6525 [1 << 17]byte + var x6526 [1 << 17]byte + var x6527 [1 << 17]byte + var x6528 [1 << 17]byte + var x6529 [1 << 17]byte + var x6530 [1 << 17]byte + var x6531 [1 << 17]byte + var x6532 [1 << 17]byte + var x6533 [1 << 17]byte + var x6534 [1 << 17]byte + var x6535 [1 << 17]byte + var x6536 [1 << 17]byte + var x6537 [1 << 17]byte + var x6538 [1 << 17]byte + var x6539 [1 << 17]byte + var x6540 [1 << 17]byte + var x6541 [1 << 17]byte + var x6542 [1 << 17]byte + var x6543 [1 << 17]byte + var x6544 [1 << 17]byte + var x6545 [1 << 17]byte + var x6546 [1 << 17]byte + var x6547 [1 << 17]byte + var x6548 [1 << 17]byte + var x6549 [1 << 17]byte + var x6550 [1 << 17]byte + var x6551 [1 << 17]byte + var x6552 [1 << 17]byte + var x6553 [1 << 17]byte + var x6554 [1 << 17]byte + var x6555 [1 << 17]byte + var x6556 [1 << 17]byte + var x6557 [1 << 17]byte + var x6558 [1 << 17]byte + var x6559 [1 << 17]byte + var x6560 [1 << 17]byte + var x6561 [1 << 17]byte + var x6562 [1 << 17]byte + var x6563 [1 << 17]byte + var x6564 [1 << 17]byte + var x6565 [1 << 17]byte + var x6566 [1 << 17]byte + var x6567 [1 << 17]byte + var x6568 [1 << 17]byte + var x6569 [1 << 17]byte + var x6570 [1 << 17]byte + var x6571 [1 << 17]byte + var x6572 [1 << 17]byte + var x6573 [1 << 17]byte + var x6574 [1 << 17]byte + var x6575 [1 << 17]byte + var x6576 [1 << 17]byte + var x6577 [1 << 17]byte + var x6578 [1 << 17]byte + var x6579 [1 << 17]byte + var x6580 [1 << 17]byte + var x6581 [1 << 17]byte + var x6582 [1 << 17]byte + var x6583 [1 << 17]byte + var x6584 [1 << 17]byte + var x6585 [1 << 17]byte + var x6586 [1 << 17]byte + var x6587 [1 << 17]byte + var x6588 [1 << 17]byte + var x6589 [1 << 17]byte + var x6590 [1 << 17]byte + var x6591 [1 << 17]byte + var x6592 [1 << 17]byte + var x6593 [1 << 17]byte + var x6594 [1 << 17]byte + var x6595 [1 << 17]byte + var x6596 [1 << 17]byte + var x6597 [1 << 17]byte + var x6598 [1 << 17]byte + var x6599 [1 << 17]byte + var x6600 [1 << 17]byte + var x6601 [1 << 17]byte + var x6602 [1 << 17]byte + var x6603 [1 << 17]byte + var x6604 [1 << 17]byte + var x6605 [1 << 17]byte + var x6606 [1 << 17]byte + var x6607 [1 << 17]byte + var x6608 [1 << 17]byte + var x6609 [1 << 17]byte + var x6610 [1 << 17]byte + var x6611 [1 << 17]byte + var x6612 [1 << 17]byte + var x6613 [1 << 17]byte + var x6614 [1 << 17]byte + var x6615 [1 << 17]byte + var x6616 [1 << 17]byte + var x6617 [1 << 17]byte + var x6618 [1 << 17]byte + var x6619 [1 << 17]byte + var x6620 [1 << 17]byte + var x6621 [1 << 17]byte + var x6622 [1 << 17]byte + var x6623 [1 << 17]byte + var x6624 [1 << 17]byte + var x6625 [1 << 17]byte + var x6626 [1 << 17]byte + var x6627 [1 << 17]byte + var x6628 [1 << 17]byte + var x6629 [1 << 17]byte + var x6630 [1 << 17]byte + var x6631 [1 << 17]byte + var x6632 [1 << 17]byte + var x6633 [1 << 17]byte + var x6634 [1 << 17]byte + var x6635 [1 << 17]byte + var x6636 [1 << 17]byte + var x6637 [1 << 17]byte + var x6638 [1 << 17]byte + var x6639 [1 << 17]byte + var x6640 [1 << 17]byte + var x6641 [1 << 17]byte + var x6642 [1 << 17]byte + var x6643 [1 << 17]byte + var x6644 [1 << 17]byte + var x6645 [1 << 17]byte + var x6646 [1 << 17]byte + var x6647 [1 << 17]byte + var x6648 [1 << 17]byte + var x6649 [1 << 17]byte + var x6650 [1 << 17]byte + var x6651 [1 << 17]byte + var x6652 [1 << 17]byte + var x6653 [1 << 17]byte + var x6654 [1 << 17]byte + var x6655 [1 << 17]byte + var x6656 [1 << 17]byte + var x6657 [1 << 17]byte + var x6658 [1 << 17]byte + var x6659 [1 << 17]byte + var x6660 [1 << 17]byte + var x6661 [1 << 17]byte + var x6662 [1 << 17]byte + var x6663 [1 << 17]byte + var x6664 [1 << 17]byte + var x6665 [1 << 17]byte + var x6666 [1 << 17]byte + var x6667 [1 << 17]byte + var x6668 [1 << 17]byte + var x6669 [1 << 17]byte + var x6670 [1 << 17]byte + var x6671 [1 << 17]byte + var x6672 [1 << 17]byte + var x6673 [1 << 17]byte + var x6674 [1 << 17]byte + var x6675 [1 << 17]byte + var x6676 [1 << 17]byte + var x6677 [1 << 17]byte + var x6678 [1 << 17]byte + var x6679 [1 << 17]byte + var x6680 [1 << 17]byte + var x6681 [1 << 17]byte + var x6682 [1 << 17]byte + var x6683 [1 << 17]byte + var x6684 [1 << 17]byte + var x6685 [1 << 17]byte + var x6686 [1 << 17]byte + var x6687 [1 << 17]byte + var x6688 [1 << 17]byte + var x6689 [1 << 17]byte + var x6690 [1 << 17]byte + var x6691 [1 << 17]byte + var x6692 [1 << 17]byte + var x6693 [1 << 17]byte + var x6694 [1 << 17]byte + var x6695 [1 << 17]byte + var x6696 [1 << 17]byte + var x6697 [1 << 17]byte + var x6698 [1 << 17]byte + var x6699 [1 << 17]byte + var x6700 [1 << 17]byte + var x6701 [1 << 17]byte + var x6702 [1 << 17]byte + var x6703 [1 << 17]byte + var x6704 [1 << 17]byte + var x6705 [1 << 17]byte + var x6706 [1 << 17]byte + var x6707 [1 << 17]byte + var x6708 [1 << 17]byte + var x6709 [1 << 17]byte + var x6710 [1 << 17]byte + var x6711 [1 << 17]byte + var x6712 [1 << 17]byte + var x6713 [1 << 17]byte + var x6714 [1 << 17]byte + var x6715 [1 << 17]byte + var x6716 [1 << 17]byte + var x6717 [1 << 17]byte + var x6718 [1 << 17]byte + var x6719 [1 << 17]byte + var x6720 [1 << 17]byte + var x6721 [1 << 17]byte + var x6722 [1 << 17]byte + var x6723 [1 << 17]byte + var x6724 [1 << 17]byte + var x6725 [1 << 17]byte + var x6726 [1 << 17]byte + var x6727 [1 << 17]byte + var x6728 [1 << 17]byte + var x6729 [1 << 17]byte + var x6730 [1 << 17]byte + var x6731 [1 << 17]byte + var x6732 [1 << 17]byte + var x6733 [1 << 17]byte + var x6734 [1 << 17]byte + var x6735 [1 << 17]byte + var x6736 [1 << 17]byte + var x6737 [1 << 17]byte + var x6738 [1 << 17]byte + var x6739 [1 << 17]byte + var x6740 [1 << 17]byte + var x6741 [1 << 17]byte + var x6742 [1 << 17]byte + var x6743 [1 << 17]byte + var x6744 [1 << 17]byte + var x6745 [1 << 17]byte + var x6746 [1 << 17]byte + var x6747 [1 << 17]byte + var x6748 [1 << 17]byte + var x6749 [1 << 17]byte + var x6750 [1 << 17]byte + var x6751 [1 << 17]byte + var x6752 [1 << 17]byte + var x6753 [1 << 17]byte + var x6754 [1 << 17]byte + var x6755 [1 << 17]byte + var x6756 [1 << 17]byte + var x6757 [1 << 17]byte + var x6758 [1 << 17]byte + var x6759 [1 << 17]byte + var x6760 [1 << 17]byte + var x6761 [1 << 17]byte + var x6762 [1 << 17]byte + var x6763 [1 << 17]byte + var x6764 [1 << 17]byte + var x6765 [1 << 17]byte + var x6766 [1 << 17]byte + var x6767 [1 << 17]byte + var x6768 [1 << 17]byte + var x6769 [1 << 17]byte + var x6770 [1 << 17]byte + var x6771 [1 << 17]byte + var x6772 [1 << 17]byte + var x6773 [1 << 17]byte + var x6774 [1 << 17]byte + var x6775 [1 << 17]byte + var x6776 [1 << 17]byte + var x6777 [1 << 17]byte + var x6778 [1 << 17]byte + var x6779 [1 << 17]byte + var x6780 [1 << 17]byte + var x6781 [1 << 17]byte + var x6782 [1 << 17]byte + var x6783 [1 << 17]byte + var x6784 [1 << 17]byte + var x6785 [1 << 17]byte + var x6786 [1 << 17]byte + var x6787 [1 << 17]byte + var x6788 [1 << 17]byte + var x6789 [1 << 17]byte + var x6790 [1 << 17]byte + var x6791 [1 << 17]byte + var x6792 [1 << 17]byte + var x6793 [1 << 17]byte + var x6794 [1 << 17]byte + var x6795 [1 << 17]byte + var x6796 [1 << 17]byte + var x6797 [1 << 17]byte + var x6798 [1 << 17]byte + var x6799 [1 << 17]byte + var x6800 [1 << 17]byte + var x6801 [1 << 17]byte + var x6802 [1 << 17]byte + var x6803 [1 << 17]byte + var x6804 [1 << 17]byte + var x6805 [1 << 17]byte + var x6806 [1 << 17]byte + var x6807 [1 << 17]byte + var x6808 [1 << 17]byte + var x6809 [1 << 17]byte + var x6810 [1 << 17]byte + var x6811 [1 << 17]byte + var x6812 [1 << 17]byte + var x6813 [1 << 17]byte + var x6814 [1 << 17]byte + var x6815 [1 << 17]byte + var x6816 [1 << 17]byte + var x6817 [1 << 17]byte + var x6818 [1 << 17]byte + var x6819 [1 << 17]byte + var x6820 [1 << 17]byte + var x6821 [1 << 17]byte + var x6822 [1 << 17]byte + var x6823 [1 << 17]byte + var x6824 [1 << 17]byte + var x6825 [1 << 17]byte + var x6826 [1 << 17]byte + var x6827 [1 << 17]byte + var x6828 [1 << 17]byte + var x6829 [1 << 17]byte + var x6830 [1 << 17]byte + var x6831 [1 << 17]byte + var x6832 [1 << 17]byte + var x6833 [1 << 17]byte + var x6834 [1 << 17]byte + var x6835 [1 << 17]byte + var x6836 [1 << 17]byte + var x6837 [1 << 17]byte + var x6838 [1 << 17]byte + var x6839 [1 << 17]byte + var x6840 [1 << 17]byte + var x6841 [1 << 17]byte + var x6842 [1 << 17]byte + var x6843 [1 << 17]byte + var x6844 [1 << 17]byte + var x6845 [1 << 17]byte + var x6846 [1 << 17]byte + var x6847 [1 << 17]byte + var x6848 [1 << 17]byte + var x6849 [1 << 17]byte + var x6850 [1 << 17]byte + var x6851 [1 << 17]byte + var x6852 [1 << 17]byte + var x6853 [1 << 17]byte + var x6854 [1 << 17]byte + var x6855 [1 << 17]byte + var x6856 [1 << 17]byte + var x6857 [1 << 17]byte + var x6858 [1 << 17]byte + var x6859 [1 << 17]byte + var x6860 [1 << 17]byte + var x6861 [1 << 17]byte + var x6862 [1 << 17]byte + var x6863 [1 << 17]byte + var x6864 [1 << 17]byte + var x6865 [1 << 17]byte + var x6866 [1 << 17]byte + var x6867 [1 << 17]byte + var x6868 [1 << 17]byte + var x6869 [1 << 17]byte + var x6870 [1 << 17]byte + var x6871 [1 << 17]byte + var x6872 [1 << 17]byte + var x6873 [1 << 17]byte + var x6874 [1 << 17]byte + var x6875 [1 << 17]byte + var x6876 [1 << 17]byte + var x6877 [1 << 17]byte + var x6878 [1 << 17]byte + var x6879 [1 << 17]byte + var x6880 [1 << 17]byte + var x6881 [1 << 17]byte + var x6882 [1 << 17]byte + var x6883 [1 << 17]byte + var x6884 [1 << 17]byte + var x6885 [1 << 17]byte + var x6886 [1 << 17]byte + var x6887 [1 << 17]byte + var x6888 [1 << 17]byte + var x6889 [1 << 17]byte + var x6890 [1 << 17]byte + var x6891 [1 << 17]byte + var x6892 [1 << 17]byte + var x6893 [1 << 17]byte + var x6894 [1 << 17]byte + var x6895 [1 << 17]byte + var x6896 [1 << 17]byte + var x6897 [1 << 17]byte + var x6898 [1 << 17]byte + var x6899 [1 << 17]byte + var x6900 [1 << 17]byte + var x6901 [1 << 17]byte + var x6902 [1 << 17]byte + var x6903 [1 << 17]byte + var x6904 [1 << 17]byte + var x6905 [1 << 17]byte + var x6906 [1 << 17]byte + var x6907 [1 << 17]byte + var x6908 [1 << 17]byte + var x6909 [1 << 17]byte + var x6910 [1 << 17]byte + var x6911 [1 << 17]byte + var x6912 [1 << 17]byte + var x6913 [1 << 17]byte + var x6914 [1 << 17]byte + var x6915 [1 << 17]byte + var x6916 [1 << 17]byte + var x6917 [1 << 17]byte + var x6918 [1 << 17]byte + var x6919 [1 << 17]byte + var x6920 [1 << 17]byte + var x6921 [1 << 17]byte + var x6922 [1 << 17]byte + var x6923 [1 << 17]byte + var x6924 [1 << 17]byte + var x6925 [1 << 17]byte + var x6926 [1 << 17]byte + var x6927 [1 << 17]byte + var x6928 [1 << 17]byte + var x6929 [1 << 17]byte + var x6930 [1 << 17]byte + var x6931 [1 << 17]byte + var x6932 [1 << 17]byte + var x6933 [1 << 17]byte + var x6934 [1 << 17]byte + var x6935 [1 << 17]byte + var x6936 [1 << 17]byte + var x6937 [1 << 17]byte + var x6938 [1 << 17]byte + var x6939 [1 << 17]byte + var x6940 [1 << 17]byte + var x6941 [1 << 17]byte + var x6942 [1 << 17]byte + var x6943 [1 << 17]byte + var x6944 [1 << 17]byte + var x6945 [1 << 17]byte + var x6946 [1 << 17]byte + var x6947 [1 << 17]byte + var x6948 [1 << 17]byte + var x6949 [1 << 17]byte + var x6950 [1 << 17]byte + var x6951 [1 << 17]byte + var x6952 [1 << 17]byte + var x6953 [1 << 17]byte + var x6954 [1 << 17]byte + var x6955 [1 << 17]byte + var x6956 [1 << 17]byte + var x6957 [1 << 17]byte + var x6958 [1 << 17]byte + var x6959 [1 << 17]byte + var x6960 [1 << 17]byte + var x6961 [1 << 17]byte + var x6962 [1 << 17]byte + var x6963 [1 << 17]byte + var x6964 [1 << 17]byte + var x6965 [1 << 17]byte + var x6966 [1 << 17]byte + var x6967 [1 << 17]byte + var x6968 [1 << 17]byte + var x6969 [1 << 17]byte + var x6970 [1 << 17]byte + var x6971 [1 << 17]byte + var x6972 [1 << 17]byte + var x6973 [1 << 17]byte + var x6974 [1 << 17]byte + var x6975 [1 << 17]byte + var x6976 [1 << 17]byte + var x6977 [1 << 17]byte + var x6978 [1 << 17]byte + var x6979 [1 << 17]byte + var x6980 [1 << 17]byte + var x6981 [1 << 17]byte + var x6982 [1 << 17]byte + var x6983 [1 << 17]byte + var x6984 [1 << 17]byte + var x6985 [1 << 17]byte + var x6986 [1 << 17]byte + var x6987 [1 << 17]byte + var x6988 [1 << 17]byte + var x6989 [1 << 17]byte + var x6990 [1 << 17]byte + var x6991 [1 << 17]byte + var x6992 [1 << 17]byte + var x6993 [1 << 17]byte + var x6994 [1 << 17]byte + var x6995 [1 << 17]byte + var x6996 [1 << 17]byte + var x6997 [1 << 17]byte + var x6998 [1 << 17]byte + var x6999 [1 << 17]byte + var x7000 [1 << 17]byte + var x7001 [1 << 17]byte + var x7002 [1 << 17]byte + var x7003 [1 << 17]byte + var x7004 [1 << 17]byte + var x7005 [1 << 17]byte + var x7006 [1 << 17]byte + var x7007 [1 << 17]byte + var x7008 [1 << 17]byte + var x7009 [1 << 17]byte + var x7010 [1 << 17]byte + var x7011 [1 << 17]byte + var x7012 [1 << 17]byte + var x7013 [1 << 17]byte + var x7014 [1 << 17]byte + var x7015 [1 << 17]byte + var x7016 [1 << 17]byte + var x7017 [1 << 17]byte + var x7018 [1 << 17]byte + var x7019 [1 << 17]byte + var x7020 [1 << 17]byte + var x7021 [1 << 17]byte + var x7022 [1 << 17]byte + var x7023 [1 << 17]byte + var x7024 [1 << 17]byte + var x7025 [1 << 17]byte + var x7026 [1 << 17]byte + var x7027 [1 << 17]byte + var x7028 [1 << 17]byte + var x7029 [1 << 17]byte + var x7030 [1 << 17]byte + var x7031 [1 << 17]byte + var x7032 [1 << 17]byte + var x7033 [1 << 17]byte + var x7034 [1 << 17]byte + var x7035 [1 << 17]byte + var x7036 [1 << 17]byte + var x7037 [1 << 17]byte + var x7038 [1 << 17]byte + var x7039 [1 << 17]byte + var x7040 [1 << 17]byte + var x7041 [1 << 17]byte + var x7042 [1 << 17]byte + var x7043 [1 << 17]byte + var x7044 [1 << 17]byte + var x7045 [1 << 17]byte + var x7046 [1 << 17]byte + var x7047 [1 << 17]byte + var x7048 [1 << 17]byte + var x7049 [1 << 17]byte + var x7050 [1 << 17]byte + var x7051 [1 << 17]byte + var x7052 [1 << 17]byte + var x7053 [1 << 17]byte + var x7054 [1 << 17]byte + var x7055 [1 << 17]byte + var x7056 [1 << 17]byte + var x7057 [1 << 17]byte + var x7058 [1 << 17]byte + var x7059 [1 << 17]byte + var x7060 [1 << 17]byte + var x7061 [1 << 17]byte + var x7062 [1 << 17]byte + var x7063 [1 << 17]byte + var x7064 [1 << 17]byte + var x7065 [1 << 17]byte + var x7066 [1 << 17]byte + var x7067 [1 << 17]byte + var x7068 [1 << 17]byte + var x7069 [1 << 17]byte + var x7070 [1 << 17]byte + var x7071 [1 << 17]byte + var x7072 [1 << 17]byte + var x7073 [1 << 17]byte + var x7074 [1 << 17]byte + var x7075 [1 << 17]byte + var x7076 [1 << 17]byte + var x7077 [1 << 17]byte + var x7078 [1 << 17]byte + var x7079 [1 << 17]byte + var x7080 [1 << 17]byte + var x7081 [1 << 17]byte + var x7082 [1 << 17]byte + var x7083 [1 << 17]byte + var x7084 [1 << 17]byte + var x7085 [1 << 17]byte + var x7086 [1 << 17]byte + var x7087 [1 << 17]byte + var x7088 [1 << 17]byte + var x7089 [1 << 17]byte + var x7090 [1 << 17]byte + var x7091 [1 << 17]byte + var x7092 [1 << 17]byte + var x7093 [1 << 17]byte + var x7094 [1 << 17]byte + var x7095 [1 << 17]byte + var x7096 [1 << 17]byte + var x7097 [1 << 17]byte + var x7098 [1 << 17]byte + var x7099 [1 << 17]byte + var x7100 [1 << 17]byte + var x7101 [1 << 17]byte + var x7102 [1 << 17]byte + var x7103 [1 << 17]byte + var x7104 [1 << 17]byte + var x7105 [1 << 17]byte + var x7106 [1 << 17]byte + var x7107 [1 << 17]byte + var x7108 [1 << 17]byte + var x7109 [1 << 17]byte + var x7110 [1 << 17]byte + var x7111 [1 << 17]byte + var x7112 [1 << 17]byte + var x7113 [1 << 17]byte + var x7114 [1 << 17]byte + var x7115 [1 << 17]byte + var x7116 [1 << 17]byte + var x7117 [1 << 17]byte + var x7118 [1 << 17]byte + var x7119 [1 << 17]byte + var x7120 [1 << 17]byte + var x7121 [1 << 17]byte + var x7122 [1 << 17]byte + var x7123 [1 << 17]byte + var x7124 [1 << 17]byte + var x7125 [1 << 17]byte + var x7126 [1 << 17]byte + var x7127 [1 << 17]byte + var x7128 [1 << 17]byte + var x7129 [1 << 17]byte + var x7130 [1 << 17]byte + var x7131 [1 << 17]byte + var x7132 [1 << 17]byte + var x7133 [1 << 17]byte + var x7134 [1 << 17]byte + var x7135 [1 << 17]byte + var x7136 [1 << 17]byte + var x7137 [1 << 17]byte + var x7138 [1 << 17]byte + var x7139 [1 << 17]byte + var x7140 [1 << 17]byte + var x7141 [1 << 17]byte + var x7142 [1 << 17]byte + var x7143 [1 << 17]byte + var x7144 [1 << 17]byte + var x7145 [1 << 17]byte + var x7146 [1 << 17]byte + var x7147 [1 << 17]byte + var x7148 [1 << 17]byte + var x7149 [1 << 17]byte + var x7150 [1 << 17]byte + var x7151 [1 << 17]byte + var x7152 [1 << 17]byte + var x7153 [1 << 17]byte + var x7154 [1 << 17]byte + var x7155 [1 << 17]byte + var x7156 [1 << 17]byte + var x7157 [1 << 17]byte + var x7158 [1 << 17]byte + var x7159 [1 << 17]byte + var x7160 [1 << 17]byte + var x7161 [1 << 17]byte + var x7162 [1 << 17]byte + var x7163 [1 << 17]byte + var x7164 [1 << 17]byte + var x7165 [1 << 17]byte + var x7166 [1 << 17]byte + var x7167 [1 << 17]byte + var x7168 [1 << 17]byte + var x7169 [1 << 17]byte + var x7170 [1 << 17]byte + var x7171 [1 << 17]byte + var x7172 [1 << 17]byte + var x7173 [1 << 17]byte + var x7174 [1 << 17]byte + var x7175 [1 << 17]byte + var x7176 [1 << 17]byte + var x7177 [1 << 17]byte + var x7178 [1 << 17]byte + var x7179 [1 << 17]byte + var x7180 [1 << 17]byte + var x7181 [1 << 17]byte + var x7182 [1 << 17]byte + var x7183 [1 << 17]byte + var x7184 [1 << 17]byte + var x7185 [1 << 17]byte + var x7186 [1 << 17]byte + var x7187 [1 << 17]byte + var x7188 [1 << 17]byte + var x7189 [1 << 17]byte + var x7190 [1 << 17]byte + var x7191 [1 << 17]byte + var x7192 [1 << 17]byte + var x7193 [1 << 17]byte + var x7194 [1 << 17]byte + var x7195 [1 << 17]byte + var x7196 [1 << 17]byte + var x7197 [1 << 17]byte + var x7198 [1 << 17]byte + var x7199 [1 << 17]byte + var x7200 [1 << 17]byte + var x7201 [1 << 17]byte + var x7202 [1 << 17]byte + var x7203 [1 << 17]byte + var x7204 [1 << 17]byte + var x7205 [1 << 17]byte + var x7206 [1 << 17]byte + var x7207 [1 << 17]byte + var x7208 [1 << 17]byte + var x7209 [1 << 17]byte + var x7210 [1 << 17]byte + var x7211 [1 << 17]byte + var x7212 [1 << 17]byte + var x7213 [1 << 17]byte + var x7214 [1 << 17]byte + var x7215 [1 << 17]byte + var x7216 [1 << 17]byte + var x7217 [1 << 17]byte + var x7218 [1 << 17]byte + var x7219 [1 << 17]byte + var x7220 [1 << 17]byte + var x7221 [1 << 17]byte + var x7222 [1 << 17]byte + var x7223 [1 << 17]byte + var x7224 [1 << 17]byte + var x7225 [1 << 17]byte + var x7226 [1 << 17]byte + var x7227 [1 << 17]byte + var x7228 [1 << 17]byte + var x7229 [1 << 17]byte + var x7230 [1 << 17]byte + var x7231 [1 << 17]byte + var x7232 [1 << 17]byte + var x7233 [1 << 17]byte + var x7234 [1 << 17]byte + var x7235 [1 << 17]byte + var x7236 [1 << 17]byte + var x7237 [1 << 17]byte + var x7238 [1 << 17]byte + var x7239 [1 << 17]byte + var x7240 [1 << 17]byte + var x7241 [1 << 17]byte + var x7242 [1 << 17]byte + var x7243 [1 << 17]byte + var x7244 [1 << 17]byte + var x7245 [1 << 17]byte + var x7246 [1 << 17]byte + var x7247 [1 << 17]byte + var x7248 [1 << 17]byte + var x7249 [1 << 17]byte + var x7250 [1 << 17]byte + var x7251 [1 << 17]byte + var x7252 [1 << 17]byte + var x7253 [1 << 17]byte + var x7254 [1 << 17]byte + var x7255 [1 << 17]byte + var x7256 [1 << 17]byte + var x7257 [1 << 17]byte + var x7258 [1 << 17]byte + var x7259 [1 << 17]byte + var x7260 [1 << 17]byte + var x7261 [1 << 17]byte + var x7262 [1 << 17]byte + var x7263 [1 << 17]byte + var x7264 [1 << 17]byte + var x7265 [1 << 17]byte + var x7266 [1 << 17]byte + var x7267 [1 << 17]byte + var x7268 [1 << 17]byte + var x7269 [1 << 17]byte + var x7270 [1 << 17]byte + var x7271 [1 << 17]byte + var x7272 [1 << 17]byte + var x7273 [1 << 17]byte + var x7274 [1 << 17]byte + var x7275 [1 << 17]byte + var x7276 [1 << 17]byte + var x7277 [1 << 17]byte + var x7278 [1 << 17]byte + var x7279 [1 << 17]byte + var x7280 [1 << 17]byte + var x7281 [1 << 17]byte + var x7282 [1 << 17]byte + var x7283 [1 << 17]byte + var x7284 [1 << 17]byte + var x7285 [1 << 17]byte + var x7286 [1 << 17]byte + var x7287 [1 << 17]byte + var x7288 [1 << 17]byte + var x7289 [1 << 17]byte + var x7290 [1 << 17]byte + var x7291 [1 << 17]byte + var x7292 [1 << 17]byte + var x7293 [1 << 17]byte + var x7294 [1 << 17]byte + var x7295 [1 << 17]byte + var x7296 [1 << 17]byte + var x7297 [1 << 17]byte + var x7298 [1 << 17]byte + var x7299 [1 << 17]byte + var x7300 [1 << 17]byte + var x7301 [1 << 17]byte + var x7302 [1 << 17]byte + var x7303 [1 << 17]byte + var x7304 [1 << 17]byte + var x7305 [1 << 17]byte + var x7306 [1 << 17]byte + var x7307 [1 << 17]byte + var x7308 [1 << 17]byte + var x7309 [1 << 17]byte + var x7310 [1 << 17]byte + var x7311 [1 << 17]byte + var x7312 [1 << 17]byte + var x7313 [1 << 17]byte + var x7314 [1 << 17]byte + var x7315 [1 << 17]byte + var x7316 [1 << 17]byte + var x7317 [1 << 17]byte + var x7318 [1 << 17]byte + var x7319 [1 << 17]byte + var x7320 [1 << 17]byte + var x7321 [1 << 17]byte + var x7322 [1 << 17]byte + var x7323 [1 << 17]byte + var x7324 [1 << 17]byte + var x7325 [1 << 17]byte + var x7326 [1 << 17]byte + var x7327 [1 << 17]byte + var x7328 [1 << 17]byte + var x7329 [1 << 17]byte + var x7330 [1 << 17]byte + var x7331 [1 << 17]byte + var x7332 [1 << 17]byte + var x7333 [1 << 17]byte + var x7334 [1 << 17]byte + var x7335 [1 << 17]byte + var x7336 [1 << 17]byte + var x7337 [1 << 17]byte + var x7338 [1 << 17]byte + var x7339 [1 << 17]byte + var x7340 [1 << 17]byte + var x7341 [1 << 17]byte + var x7342 [1 << 17]byte + var x7343 [1 << 17]byte + var x7344 [1 << 17]byte + var x7345 [1 << 17]byte + var x7346 [1 << 17]byte + var x7347 [1 << 17]byte + var x7348 [1 << 17]byte + var x7349 [1 << 17]byte + var x7350 [1 << 17]byte + var x7351 [1 << 17]byte + var x7352 [1 << 17]byte + var x7353 [1 << 17]byte + var x7354 [1 << 17]byte + var x7355 [1 << 17]byte + var x7356 [1 << 17]byte + var x7357 [1 << 17]byte + var x7358 [1 << 17]byte + var x7359 [1 << 17]byte + var x7360 [1 << 17]byte + var x7361 [1 << 17]byte + var x7362 [1 << 17]byte + var x7363 [1 << 17]byte + var x7364 [1 << 17]byte + var x7365 [1 << 17]byte + var x7366 [1 << 17]byte + var x7367 [1 << 17]byte + var x7368 [1 << 17]byte + var x7369 [1 << 17]byte + var x7370 [1 << 17]byte + var x7371 [1 << 17]byte + var x7372 [1 << 17]byte + var x7373 [1 << 17]byte + var x7374 [1 << 17]byte + var x7375 [1 << 17]byte + var x7376 [1 << 17]byte + var x7377 [1 << 17]byte + var x7378 [1 << 17]byte + var x7379 [1 << 17]byte + var x7380 [1 << 17]byte + var x7381 [1 << 17]byte + var x7382 [1 << 17]byte + var x7383 [1 << 17]byte + var x7384 [1 << 17]byte + var x7385 [1 << 17]byte + var x7386 [1 << 17]byte + var x7387 [1 << 17]byte + var x7388 [1 << 17]byte + var x7389 [1 << 17]byte + var x7390 [1 << 17]byte + var x7391 [1 << 17]byte + var x7392 [1 << 17]byte + var x7393 [1 << 17]byte + var x7394 [1 << 17]byte + var x7395 [1 << 17]byte + var x7396 [1 << 17]byte + var x7397 [1 << 17]byte + var x7398 [1 << 17]byte + var x7399 [1 << 17]byte + var x7400 [1 << 17]byte + var x7401 [1 << 17]byte + var x7402 [1 << 17]byte + var x7403 [1 << 17]byte + var x7404 [1 << 17]byte + var x7405 [1 << 17]byte + var x7406 [1 << 17]byte + var x7407 [1 << 17]byte + var x7408 [1 << 17]byte + var x7409 [1 << 17]byte + var x7410 [1 << 17]byte + var x7411 [1 << 17]byte + var x7412 [1 << 17]byte + var x7413 [1 << 17]byte + var x7414 [1 << 17]byte + var x7415 [1 << 17]byte + var x7416 [1 << 17]byte + var x7417 [1 << 17]byte + var x7418 [1 << 17]byte + var x7419 [1 << 17]byte + var x7420 [1 << 17]byte + var x7421 [1 << 17]byte + var x7422 [1 << 17]byte + var x7423 [1 << 17]byte + var x7424 [1 << 17]byte + var x7425 [1 << 17]byte + var x7426 [1 << 17]byte + var x7427 [1 << 17]byte + var x7428 [1 << 17]byte + var x7429 [1 << 17]byte + var x7430 [1 << 17]byte + var x7431 [1 << 17]byte + var x7432 [1 << 17]byte + var x7433 [1 << 17]byte + var x7434 [1 << 17]byte + var x7435 [1 << 17]byte + var x7436 [1 << 17]byte + var x7437 [1 << 17]byte + var x7438 [1 << 17]byte + var x7439 [1 << 17]byte + var x7440 [1 << 17]byte + var x7441 [1 << 17]byte + var x7442 [1 << 17]byte + var x7443 [1 << 17]byte + var x7444 [1 << 17]byte + var x7445 [1 << 17]byte + var x7446 [1 << 17]byte + var x7447 [1 << 17]byte + var x7448 [1 << 17]byte + var x7449 [1 << 17]byte + var x7450 [1 << 17]byte + var x7451 [1 << 17]byte + var x7452 [1 << 17]byte + var x7453 [1 << 17]byte + var x7454 [1 << 17]byte + var x7455 [1 << 17]byte + var x7456 [1 << 17]byte + var x7457 [1 << 17]byte + var x7458 [1 << 17]byte + var x7459 [1 << 17]byte + var x7460 [1 << 17]byte + var x7461 [1 << 17]byte + var x7462 [1 << 17]byte + var x7463 [1 << 17]byte + var x7464 [1 << 17]byte + var x7465 [1 << 17]byte + var x7466 [1 << 17]byte + var x7467 [1 << 17]byte + var x7468 [1 << 17]byte + var x7469 [1 << 17]byte + var x7470 [1 << 17]byte + var x7471 [1 << 17]byte + var x7472 [1 << 17]byte + var x7473 [1 << 17]byte + var x7474 [1 << 17]byte + var x7475 [1 << 17]byte + var x7476 [1 << 17]byte + var x7477 [1 << 17]byte + var x7478 [1 << 17]byte + var x7479 [1 << 17]byte + var x7480 [1 << 17]byte + var x7481 [1 << 17]byte + var x7482 [1 << 17]byte + var x7483 [1 << 17]byte + var x7484 [1 << 17]byte + var x7485 [1 << 17]byte + var x7486 [1 << 17]byte + var x7487 [1 << 17]byte + var x7488 [1 << 17]byte + var x7489 [1 << 17]byte + var x7490 [1 << 17]byte + var x7491 [1 << 17]byte + var x7492 [1 << 17]byte + var x7493 [1 << 17]byte + var x7494 [1 << 17]byte + var x7495 [1 << 17]byte + var x7496 [1 << 17]byte + var x7497 [1 << 17]byte + var x7498 [1 << 17]byte + var x7499 [1 << 17]byte + var x7500 [1 << 17]byte + var x7501 [1 << 17]byte + var x7502 [1 << 17]byte + var x7503 [1 << 17]byte + var x7504 [1 << 17]byte + var x7505 [1 << 17]byte + var x7506 [1 << 17]byte + var x7507 [1 << 17]byte + var x7508 [1 << 17]byte + var x7509 [1 << 17]byte + var x7510 [1 << 17]byte + var x7511 [1 << 17]byte + var x7512 [1 << 17]byte + var x7513 [1 << 17]byte + var x7514 [1 << 17]byte + var x7515 [1 << 17]byte + var x7516 [1 << 17]byte + var x7517 [1 << 17]byte + var x7518 [1 << 17]byte + var x7519 [1 << 17]byte + var x7520 [1 << 17]byte + var x7521 [1 << 17]byte + var x7522 [1 << 17]byte + var x7523 [1 << 17]byte + var x7524 [1 << 17]byte + var x7525 [1 << 17]byte + var x7526 [1 << 17]byte + var x7527 [1 << 17]byte + var x7528 [1 << 17]byte + var x7529 [1 << 17]byte + var x7530 [1 << 17]byte + var x7531 [1 << 17]byte + var x7532 [1 << 17]byte + var x7533 [1 << 17]byte + var x7534 [1 << 17]byte + var x7535 [1 << 17]byte + var x7536 [1 << 17]byte + var x7537 [1 << 17]byte + var x7538 [1 << 17]byte + var x7539 [1 << 17]byte + var x7540 [1 << 17]byte + var x7541 [1 << 17]byte + var x7542 [1 << 17]byte + var x7543 [1 << 17]byte + var x7544 [1 << 17]byte + var x7545 [1 << 17]byte + var x7546 [1 << 17]byte + var x7547 [1 << 17]byte + var x7548 [1 << 17]byte + var x7549 [1 << 17]byte + var x7550 [1 << 17]byte + var x7551 [1 << 17]byte + var x7552 [1 << 17]byte + var x7553 [1 << 17]byte + var x7554 [1 << 17]byte + var x7555 [1 << 17]byte + var x7556 [1 << 17]byte + var x7557 [1 << 17]byte + var x7558 [1 << 17]byte + var x7559 [1 << 17]byte + var x7560 [1 << 17]byte + var x7561 [1 << 17]byte + var x7562 [1 << 17]byte + var x7563 [1 << 17]byte + var x7564 [1 << 17]byte + var x7565 [1 << 17]byte + var x7566 [1 << 17]byte + var x7567 [1 << 17]byte + var x7568 [1 << 17]byte + var x7569 [1 << 17]byte + var x7570 [1 << 17]byte + var x7571 [1 << 17]byte + var x7572 [1 << 17]byte + var x7573 [1 << 17]byte + var x7574 [1 << 17]byte + var x7575 [1 << 17]byte + var x7576 [1 << 17]byte + var x7577 [1 << 17]byte + var x7578 [1 << 17]byte + var x7579 [1 << 17]byte + var x7580 [1 << 17]byte + var x7581 [1 << 17]byte + var x7582 [1 << 17]byte + var x7583 [1 << 17]byte + var x7584 [1 << 17]byte + var x7585 [1 << 17]byte + var x7586 [1 << 17]byte + var x7587 [1 << 17]byte + var x7588 [1 << 17]byte + var x7589 [1 << 17]byte + var x7590 [1 << 17]byte + var x7591 [1 << 17]byte + var x7592 [1 << 17]byte + var x7593 [1 << 17]byte + var x7594 [1 << 17]byte + var x7595 [1 << 17]byte + var x7596 [1 << 17]byte + var x7597 [1 << 17]byte + var x7598 [1 << 17]byte + var x7599 [1 << 17]byte + var x7600 [1 << 17]byte + var x7601 [1 << 17]byte + var x7602 [1 << 17]byte + var x7603 [1 << 17]byte + var x7604 [1 << 17]byte + var x7605 [1 << 17]byte + var x7606 [1 << 17]byte + var x7607 [1 << 17]byte + var x7608 [1 << 17]byte + var x7609 [1 << 17]byte + var x7610 [1 << 17]byte + var x7611 [1 << 17]byte + var x7612 [1 << 17]byte + var x7613 [1 << 17]byte + var x7614 [1 << 17]byte + var x7615 [1 << 17]byte + var x7616 [1 << 17]byte + var x7617 [1 << 17]byte + var x7618 [1 << 17]byte + var x7619 [1 << 17]byte + var x7620 [1 << 17]byte + var x7621 [1 << 17]byte + var x7622 [1 << 17]byte + var x7623 [1 << 17]byte + var x7624 [1 << 17]byte + var x7625 [1 << 17]byte + var x7626 [1 << 17]byte + var x7627 [1 << 17]byte + var x7628 [1 << 17]byte + var x7629 [1 << 17]byte + var x7630 [1 << 17]byte + var x7631 [1 << 17]byte + var x7632 [1 << 17]byte + var x7633 [1 << 17]byte + var x7634 [1 << 17]byte + var x7635 [1 << 17]byte + var x7636 [1 << 17]byte + var x7637 [1 << 17]byte + var x7638 [1 << 17]byte + var x7639 [1 << 17]byte + var x7640 [1 << 17]byte + var x7641 [1 << 17]byte + var x7642 [1 << 17]byte + var x7643 [1 << 17]byte + var x7644 [1 << 17]byte + var x7645 [1 << 17]byte + var x7646 [1 << 17]byte + var x7647 [1 << 17]byte + var x7648 [1 << 17]byte + var x7649 [1 << 17]byte + var x7650 [1 << 17]byte + var x7651 [1 << 17]byte + var x7652 [1 << 17]byte + var x7653 [1 << 17]byte + var x7654 [1 << 17]byte + var x7655 [1 << 17]byte + var x7656 [1 << 17]byte + var x7657 [1 << 17]byte + var x7658 [1 << 17]byte + var x7659 [1 << 17]byte + var x7660 [1 << 17]byte + var x7661 [1 << 17]byte + var x7662 [1 << 17]byte + var x7663 [1 << 17]byte + var x7664 [1 << 17]byte + var x7665 [1 << 17]byte + var x7666 [1 << 17]byte + var x7667 [1 << 17]byte + var x7668 [1 << 17]byte + var x7669 [1 << 17]byte + var x7670 [1 << 17]byte + var x7671 [1 << 17]byte + var x7672 [1 << 17]byte + var x7673 [1 << 17]byte + var x7674 [1 << 17]byte + var x7675 [1 << 17]byte + var x7676 [1 << 17]byte + var x7677 [1 << 17]byte + var x7678 [1 << 17]byte + var x7679 [1 << 17]byte + var x7680 [1 << 17]byte + var x7681 [1 << 17]byte + var x7682 [1 << 17]byte + var x7683 [1 << 17]byte + var x7684 [1 << 17]byte + var x7685 [1 << 17]byte + var x7686 [1 << 17]byte + var x7687 [1 << 17]byte + var x7688 [1 << 17]byte + var x7689 [1 << 17]byte + var x7690 [1 << 17]byte + var x7691 [1 << 17]byte + var x7692 [1 << 17]byte + var x7693 [1 << 17]byte + var x7694 [1 << 17]byte + var x7695 [1 << 17]byte + var x7696 [1 << 17]byte + var x7697 [1 << 17]byte + var x7698 [1 << 17]byte + var x7699 [1 << 17]byte + var x7700 [1 << 17]byte + var x7701 [1 << 17]byte + var x7702 [1 << 17]byte + var x7703 [1 << 17]byte + var x7704 [1 << 17]byte + var x7705 [1 << 17]byte + var x7706 [1 << 17]byte + var x7707 [1 << 17]byte + var x7708 [1 << 17]byte + var x7709 [1 << 17]byte + var x7710 [1 << 17]byte + var x7711 [1 << 17]byte + var x7712 [1 << 17]byte + var x7713 [1 << 17]byte + var x7714 [1 << 17]byte + var x7715 [1 << 17]byte + var x7716 [1 << 17]byte + var x7717 [1 << 17]byte + var x7718 [1 << 17]byte + var x7719 [1 << 17]byte + var x7720 [1 << 17]byte + var x7721 [1 << 17]byte + var x7722 [1 << 17]byte + var x7723 [1 << 17]byte + var x7724 [1 << 17]byte + var x7725 [1 << 17]byte + var x7726 [1 << 17]byte + var x7727 [1 << 17]byte + var x7728 [1 << 17]byte + var x7729 [1 << 17]byte + var x7730 [1 << 17]byte + var x7731 [1 << 17]byte + var x7732 [1 << 17]byte + var x7733 [1 << 17]byte + var x7734 [1 << 17]byte + var x7735 [1 << 17]byte + var x7736 [1 << 17]byte + var x7737 [1 << 17]byte + var x7738 [1 << 17]byte + var x7739 [1 << 17]byte + var x7740 [1 << 17]byte + var x7741 [1 << 17]byte + var x7742 [1 << 17]byte + var x7743 [1 << 17]byte + var x7744 [1 << 17]byte + var x7745 [1 << 17]byte + var x7746 [1 << 17]byte + var x7747 [1 << 17]byte + var x7748 [1 << 17]byte + var x7749 [1 << 17]byte + var x7750 [1 << 17]byte + var x7751 [1 << 17]byte + var x7752 [1 << 17]byte + var x7753 [1 << 17]byte + var x7754 [1 << 17]byte + var x7755 [1 << 17]byte + var x7756 [1 << 17]byte + var x7757 [1 << 17]byte + var x7758 [1 << 17]byte + var x7759 [1 << 17]byte + var x7760 [1 << 17]byte + var x7761 [1 << 17]byte + var x7762 [1 << 17]byte + var x7763 [1 << 17]byte + var x7764 [1 << 17]byte + var x7765 [1 << 17]byte + var x7766 [1 << 17]byte + var x7767 [1 << 17]byte + var x7768 [1 << 17]byte + var x7769 [1 << 17]byte + var x7770 [1 << 17]byte + var x7771 [1 << 17]byte + var x7772 [1 << 17]byte + var x7773 [1 << 17]byte + var x7774 [1 << 17]byte + var x7775 [1 << 17]byte + var x7776 [1 << 17]byte + var x7777 [1 << 17]byte + var x7778 [1 << 17]byte + var x7779 [1 << 17]byte + var x7780 [1 << 17]byte + var x7781 [1 << 17]byte + var x7782 [1 << 17]byte + var x7783 [1 << 17]byte + var x7784 [1 << 17]byte + var x7785 [1 << 17]byte + var x7786 [1 << 17]byte + var x7787 [1 << 17]byte + var x7788 [1 << 17]byte + var x7789 [1 << 17]byte + var x7790 [1 << 17]byte + var x7791 [1 << 17]byte + var x7792 [1 << 17]byte + var x7793 [1 << 17]byte + var x7794 [1 << 17]byte + var x7795 [1 << 17]byte + var x7796 [1 << 17]byte + var x7797 [1 << 17]byte + var x7798 [1 << 17]byte + var x7799 [1 << 17]byte + var x7800 [1 << 17]byte + var x7801 [1 << 17]byte + var x7802 [1 << 17]byte + var x7803 [1 << 17]byte + var x7804 [1 << 17]byte + var x7805 [1 << 17]byte + var x7806 [1 << 17]byte + var x7807 [1 << 17]byte + var x7808 [1 << 17]byte + var x7809 [1 << 17]byte + var x7810 [1 << 17]byte + var x7811 [1 << 17]byte + var x7812 [1 << 17]byte + var x7813 [1 << 17]byte + var x7814 [1 << 17]byte + var x7815 [1 << 17]byte + var x7816 [1 << 17]byte + var x7817 [1 << 17]byte + var x7818 [1 << 17]byte + var x7819 [1 << 17]byte + var x7820 [1 << 17]byte + var x7821 [1 << 17]byte + var x7822 [1 << 17]byte + var x7823 [1 << 17]byte + var x7824 [1 << 17]byte + var x7825 [1 << 17]byte + var x7826 [1 << 17]byte + var x7827 [1 << 17]byte + var x7828 [1 << 17]byte + var x7829 [1 << 17]byte + var x7830 [1 << 17]byte + var x7831 [1 << 17]byte + var x7832 [1 << 17]byte + var x7833 [1 << 17]byte + var x7834 [1 << 17]byte + var x7835 [1 << 17]byte + var x7836 [1 << 17]byte + var x7837 [1 << 17]byte + var x7838 [1 << 17]byte + var x7839 [1 << 17]byte + var x7840 [1 << 17]byte + var x7841 [1 << 17]byte + var x7842 [1 << 17]byte + var x7843 [1 << 17]byte + var x7844 [1 << 17]byte + var x7845 [1 << 17]byte + var x7846 [1 << 17]byte + var x7847 [1 << 17]byte + var x7848 [1 << 17]byte + var x7849 [1 << 17]byte + var x7850 [1 << 17]byte + var x7851 [1 << 17]byte + var x7852 [1 << 17]byte + var x7853 [1 << 17]byte + var x7854 [1 << 17]byte + var x7855 [1 << 17]byte + var x7856 [1 << 17]byte + var x7857 [1 << 17]byte + var x7858 [1 << 17]byte + var x7859 [1 << 17]byte + var x7860 [1 << 17]byte + var x7861 [1 << 17]byte + var x7862 [1 << 17]byte + var x7863 [1 << 17]byte + var x7864 [1 << 17]byte + var x7865 [1 << 17]byte + var x7866 [1 << 17]byte + var x7867 [1 << 17]byte + var x7868 [1 << 17]byte + var x7869 [1 << 17]byte + var x7870 [1 << 17]byte + var x7871 [1 << 17]byte + var x7872 [1 << 17]byte + var x7873 [1 << 17]byte + var x7874 [1 << 17]byte + var x7875 [1 << 17]byte + var x7876 [1 << 17]byte + var x7877 [1 << 17]byte + var x7878 [1 << 17]byte + var x7879 [1 << 17]byte + var x7880 [1 << 17]byte + var x7881 [1 << 17]byte + var x7882 [1 << 17]byte + var x7883 [1 << 17]byte + var x7884 [1 << 17]byte + var x7885 [1 << 17]byte + var x7886 [1 << 17]byte + var x7887 [1 << 17]byte + var x7888 [1 << 17]byte + var x7889 [1 << 17]byte + var x7890 [1 << 17]byte + var x7891 [1 << 17]byte + var x7892 [1 << 17]byte + var x7893 [1 << 17]byte + var x7894 [1 << 17]byte + var x7895 [1 << 17]byte + var x7896 [1 << 17]byte + var x7897 [1 << 17]byte + var x7898 [1 << 17]byte + var x7899 [1 << 17]byte + var x7900 [1 << 17]byte + var x7901 [1 << 17]byte + var x7902 [1 << 17]byte + var x7903 [1 << 17]byte + var x7904 [1 << 17]byte + var x7905 [1 << 17]byte + var x7906 [1 << 17]byte + var x7907 [1 << 17]byte + var x7908 [1 << 17]byte + var x7909 [1 << 17]byte + var x7910 [1 << 17]byte + var x7911 [1 << 17]byte + var x7912 [1 << 17]byte + var x7913 [1 << 17]byte + var x7914 [1 << 17]byte + var x7915 [1 << 17]byte + var x7916 [1 << 17]byte + var x7917 [1 << 17]byte + var x7918 [1 << 17]byte + var x7919 [1 << 17]byte + var x7920 [1 << 17]byte + var x7921 [1 << 17]byte + var x7922 [1 << 17]byte + var x7923 [1 << 17]byte + var x7924 [1 << 17]byte + var x7925 [1 << 17]byte + var x7926 [1 << 17]byte + var x7927 [1 << 17]byte + var x7928 [1 << 17]byte + var x7929 [1 << 17]byte + var x7930 [1 << 17]byte + var x7931 [1 << 17]byte + var x7932 [1 << 17]byte + var x7933 [1 << 17]byte + var x7934 [1 << 17]byte + var x7935 [1 << 17]byte + var x7936 [1 << 17]byte + var x7937 [1 << 17]byte + var x7938 [1 << 17]byte + var x7939 [1 << 17]byte + var x7940 [1 << 17]byte + var x7941 [1 << 17]byte + var x7942 [1 << 17]byte + var x7943 [1 << 17]byte + var x7944 [1 << 17]byte + var x7945 [1 << 17]byte + var x7946 [1 << 17]byte + var x7947 [1 << 17]byte + var x7948 [1 << 17]byte + var x7949 [1 << 17]byte + var x7950 [1 << 17]byte + var x7951 [1 << 17]byte + var x7952 [1 << 17]byte + var x7953 [1 << 17]byte + var x7954 [1 << 17]byte + var x7955 [1 << 17]byte + var x7956 [1 << 17]byte + var x7957 [1 << 17]byte + var x7958 [1 << 17]byte + var x7959 [1 << 17]byte + var x7960 [1 << 17]byte + var x7961 [1 << 17]byte + var x7962 [1 << 17]byte + var x7963 [1 << 17]byte + var x7964 [1 << 17]byte + var x7965 [1 << 17]byte + var x7966 [1 << 17]byte + var x7967 [1 << 17]byte + var x7968 [1 << 17]byte + var x7969 [1 << 17]byte + var x7970 [1 << 17]byte + var x7971 [1 << 17]byte + var x7972 [1 << 17]byte + var x7973 [1 << 17]byte + var x7974 [1 << 17]byte + var x7975 [1 << 17]byte + var x7976 [1 << 17]byte + var x7977 [1 << 17]byte + var x7978 [1 << 17]byte + var x7979 [1 << 17]byte + var x7980 [1 << 17]byte + var x7981 [1 << 17]byte + var x7982 [1 << 17]byte + var x7983 [1 << 17]byte + var x7984 [1 << 17]byte + var x7985 [1 << 17]byte + var x7986 [1 << 17]byte + var x7987 [1 << 17]byte + var x7988 [1 << 17]byte + var x7989 [1 << 17]byte + var x7990 [1 << 17]byte + var x7991 [1 << 17]byte + var x7992 [1 << 17]byte + var x7993 [1 << 17]byte + var x7994 [1 << 17]byte + var x7995 [1 << 17]byte + var x7996 [1 << 17]byte + var x7997 [1 << 17]byte + var x7998 [1 << 17]byte + var x7999 [1 << 17]byte + var x8000 [1 << 17]byte + var x8001 [1 << 17]byte + var x8002 [1 << 17]byte + var x8003 [1 << 17]byte + var x8004 [1 << 17]byte + var x8005 [1 << 17]byte + var x8006 [1 << 17]byte + var x8007 [1 << 17]byte + var x8008 [1 << 17]byte + var x8009 [1 << 17]byte + var x8010 [1 << 17]byte + var x8011 [1 << 17]byte + var x8012 [1 << 17]byte + var x8013 [1 << 17]byte + var x8014 [1 << 17]byte + var x8015 [1 << 17]byte + var x8016 [1 << 17]byte + var x8017 [1 << 17]byte + var x8018 [1 << 17]byte + var x8019 [1 << 17]byte + var x8020 [1 << 17]byte + var x8021 [1 << 17]byte + var x8022 [1 << 17]byte + var x8023 [1 << 17]byte + var x8024 [1 << 17]byte + var x8025 [1 << 17]byte + var x8026 [1 << 17]byte + var x8027 [1 << 17]byte + var x8028 [1 << 17]byte + var x8029 [1 << 17]byte + var x8030 [1 << 17]byte + var x8031 [1 << 17]byte + var x8032 [1 << 17]byte + var x8033 [1 << 17]byte + var x8034 [1 << 17]byte + var x8035 [1 << 17]byte + var x8036 [1 << 17]byte + var x8037 [1 << 17]byte + var x8038 [1 << 17]byte + var x8039 [1 << 17]byte + var x8040 [1 << 17]byte + var x8041 [1 << 17]byte + var x8042 [1 << 17]byte + var x8043 [1 << 17]byte + var x8044 [1 << 17]byte + var x8045 [1 << 17]byte + var x8046 [1 << 17]byte + var x8047 [1 << 17]byte + var x8048 [1 << 17]byte + var x8049 [1 << 17]byte + var x8050 [1 << 17]byte + var x8051 [1 << 17]byte + var x8052 [1 << 17]byte + var x8053 [1 << 17]byte + var x8054 [1 << 17]byte + var x8055 [1 << 17]byte + var x8056 [1 << 17]byte + var x8057 [1 << 17]byte + var x8058 [1 << 17]byte + var x8059 [1 << 17]byte + var x8060 [1 << 17]byte + var x8061 [1 << 17]byte + var x8062 [1 << 17]byte + var x8063 [1 << 17]byte + var x8064 [1 << 17]byte + var x8065 [1 << 17]byte + var x8066 [1 << 17]byte + var x8067 [1 << 17]byte + var x8068 [1 << 17]byte + var x8069 [1 << 17]byte + var x8070 [1 << 17]byte + var x8071 [1 << 17]byte + var x8072 [1 << 17]byte + var x8073 [1 << 17]byte + var x8074 [1 << 17]byte + var x8075 [1 << 17]byte + var x8076 [1 << 17]byte + var x8077 [1 << 17]byte + var x8078 [1 << 17]byte + var x8079 [1 << 17]byte + var x8080 [1 << 17]byte + var x8081 [1 << 17]byte + var x8082 [1 << 17]byte + var x8083 [1 << 17]byte + var x8084 [1 << 17]byte + var x8085 [1 << 17]byte + var x8086 [1 << 17]byte + var x8087 [1 << 17]byte + var x8088 [1 << 17]byte + var x8089 [1 << 17]byte + var x8090 [1 << 17]byte + var x8091 [1 << 17]byte + var x8092 [1 << 17]byte + var x8093 [1 << 17]byte + var x8094 [1 << 17]byte + var x8095 [1 << 17]byte + var x8096 [1 << 17]byte + var x8097 [1 << 17]byte + var x8098 [1 << 17]byte + var x8099 [1 << 17]byte + var x8100 [1 << 17]byte + var x8101 [1 << 17]byte + var x8102 [1 << 17]byte + var x8103 [1 << 17]byte + var x8104 [1 << 17]byte + var x8105 [1 << 17]byte + var x8106 [1 << 17]byte + var x8107 [1 << 17]byte + var x8108 [1 << 17]byte + var x8109 [1 << 17]byte + var x8110 [1 << 17]byte + var x8111 [1 << 17]byte + var x8112 [1 << 17]byte + var x8113 [1 << 17]byte + var x8114 [1 << 17]byte + var x8115 [1 << 17]byte + var x8116 [1 << 17]byte + var x8117 [1 << 17]byte + var x8118 [1 << 17]byte + var x8119 [1 << 17]byte + var x8120 [1 << 17]byte + var x8121 [1 << 17]byte + var x8122 [1 << 17]byte + var x8123 [1 << 17]byte + var x8124 [1 << 17]byte + var x8125 [1 << 17]byte + var x8126 [1 << 17]byte + var x8127 [1 << 17]byte + var x8128 [1 << 17]byte + var x8129 [1 << 17]byte + var x8130 [1 << 17]byte + var x8131 [1 << 17]byte + var x8132 [1 << 17]byte + var x8133 [1 << 17]byte + var x8134 [1 << 17]byte + var x8135 [1 << 17]byte + var x8136 [1 << 17]byte + var x8137 [1 << 17]byte + var x8138 [1 << 17]byte + var x8139 [1 << 17]byte + var x8140 [1 << 17]byte + var x8141 [1 << 17]byte + var x8142 [1 << 17]byte + var x8143 [1 << 17]byte + var x8144 [1 << 17]byte + var x8145 [1 << 17]byte + var x8146 [1 << 17]byte + var x8147 [1 << 17]byte + var x8148 [1 << 17]byte + var x8149 [1 << 17]byte + var x8150 [1 << 17]byte + var x8151 [1 << 17]byte + var x8152 [1 << 17]byte + var x8153 [1 << 17]byte + var x8154 [1 << 17]byte + var x8155 [1 << 17]byte + var x8156 [1 << 17]byte + var x8157 [1 << 17]byte + var x8158 [1 << 17]byte + var x8159 [1 << 17]byte + var x8160 [1 << 17]byte + var x8161 [1 << 17]byte + var x8162 [1 << 17]byte + var x8163 [1 << 17]byte + var x8164 [1 << 17]byte + var x8165 [1 << 17]byte + var x8166 [1 << 17]byte + var x8167 [1 << 17]byte + var x8168 [1 << 17]byte + var x8169 [1 << 17]byte + var x8170 [1 << 17]byte + var x8171 [1 << 17]byte + var x8172 [1 << 17]byte + var x8173 [1 << 17]byte + var x8174 [1 << 17]byte + var x8175 [1 << 17]byte + var x8176 [1 << 17]byte + var x8177 [1 << 17]byte + var x8178 [1 << 17]byte + var x8179 [1 << 17]byte + var x8180 [1 << 17]byte + var x8181 [1 << 17]byte + var x8182 [1 << 17]byte + var x8183 [1 << 17]byte + var x8184 [1 << 17]byte + var x8185 [1 << 17]byte + var x8186 [1 << 17]byte + var x8187 [1 << 17]byte + var x8188 [1 << 17]byte + var x8189 [1 << 17]byte + var x8190 [1 << 17]byte + var x8191 [1 << 17]byte + var x8192 [1 << 17]byte + var x8193 [1 << 17]byte + var x8194 [1 << 17]byte + var x8195 [1 << 17]byte + var x8196 [1 << 17]byte + var x8197 [1 << 17]byte + var x8198 [1 << 17]byte + var x8199 [1 << 17]byte + var x8200 [1 << 17]byte + var x8201 [1 << 17]byte + var x8202 [1 << 17]byte + var x8203 [1 << 17]byte + var x8204 [1 << 17]byte + var x8205 [1 << 17]byte + var x8206 [1 << 17]byte + var x8207 [1 << 17]byte + var x8208 [1 << 17]byte + var x8209 [1 << 17]byte + var x8210 [1 << 17]byte + var x8211 [1 << 17]byte + var x8212 [1 << 17]byte + var x8213 [1 << 17]byte + var x8214 [1 << 17]byte + var x8215 [1 << 17]byte + var x8216 [1 << 17]byte + var x8217 [1 << 17]byte + var x8218 [1 << 17]byte + var x8219 [1 << 17]byte + var x8220 [1 << 17]byte + var x8221 [1 << 17]byte + var x8222 [1 << 17]byte + var x8223 [1 << 17]byte + var x8224 [1 << 17]byte + var x8225 [1 << 17]byte + var x8226 [1 << 17]byte + var x8227 [1 << 17]byte + var x8228 [1 << 17]byte + var x8229 [1 << 17]byte + var x8230 [1 << 17]byte + var x8231 [1 << 17]byte + var x8232 [1 << 17]byte + var x8233 [1 << 17]byte + var x8234 [1 << 17]byte + var x8235 [1 << 17]byte + var x8236 [1 << 17]byte + var x8237 [1 << 17]byte + var x8238 [1 << 17]byte + var x8239 [1 << 17]byte + var x8240 [1 << 17]byte + var x8241 [1 << 17]byte + var x8242 [1 << 17]byte + var x8243 [1 << 17]byte + var x8244 [1 << 17]byte + var x8245 [1 << 17]byte + var x8246 [1 << 17]byte + var x8247 [1 << 17]byte + var x8248 [1 << 17]byte + var x8249 [1 << 17]byte + var x8250 [1 << 17]byte + var x8251 [1 << 17]byte + var x8252 [1 << 17]byte + var x8253 [1 << 17]byte + var x8254 [1 << 17]byte + var x8255 [1 << 17]byte + var x8256 [1 << 17]byte + var x8257 [1 << 17]byte + var x8258 [1 << 17]byte + var x8259 [1 << 17]byte + var x8260 [1 << 17]byte + var x8261 [1 << 17]byte + var x8262 [1 << 17]byte + var x8263 [1 << 17]byte + var x8264 [1 << 17]byte + var x8265 [1 << 17]byte + var x8266 [1 << 17]byte + var x8267 [1 << 17]byte + var x8268 [1 << 17]byte + var x8269 [1 << 17]byte + var x8270 [1 << 17]byte + var x8271 [1 << 17]byte + var x8272 [1 << 17]byte + var x8273 [1 << 17]byte + var x8274 [1 << 17]byte + var x8275 [1 << 17]byte + var x8276 [1 << 17]byte + var x8277 [1 << 17]byte + var x8278 [1 << 17]byte + var x8279 [1 << 17]byte + var x8280 [1 << 17]byte + var x8281 [1 << 17]byte + var x8282 [1 << 17]byte + var x8283 [1 << 17]byte + var x8284 [1 << 17]byte + var x8285 [1 << 17]byte + var x8286 [1 << 17]byte + var x8287 [1 << 17]byte + var x8288 [1 << 17]byte + var x8289 [1 << 17]byte + var x8290 [1 << 17]byte + var x8291 [1 << 17]byte + var x8292 [1 << 17]byte + var x8293 [1 << 17]byte + var x8294 [1 << 17]byte + var x8295 [1 << 17]byte + var x8296 [1 << 17]byte + var x8297 [1 << 17]byte + var x8298 [1 << 17]byte + var x8299 [1 << 17]byte + var x8300 [1 << 17]byte + var x8301 [1 << 17]byte + var x8302 [1 << 17]byte + var x8303 [1 << 17]byte + var x8304 [1 << 17]byte + var x8305 [1 << 17]byte + var x8306 [1 << 17]byte + var x8307 [1 << 17]byte + var x8308 [1 << 17]byte + var x8309 [1 << 17]byte + var x8310 [1 << 17]byte + var x8311 [1 << 17]byte + var x8312 [1 << 17]byte + var x8313 [1 << 17]byte + var x8314 [1 << 17]byte + var x8315 [1 << 17]byte + var x8316 [1 << 17]byte + var x8317 [1 << 17]byte + var x8318 [1 << 17]byte + var x8319 [1 << 17]byte + var x8320 [1 << 17]byte + var x8321 [1 << 17]byte + var x8322 [1 << 17]byte + var x8323 [1 << 17]byte + var x8324 [1 << 17]byte + var x8325 [1 << 17]byte + var x8326 [1 << 17]byte + var x8327 [1 << 17]byte + var x8328 [1 << 17]byte + var x8329 [1 << 17]byte + var x8330 [1 << 17]byte + var x8331 [1 << 17]byte + var x8332 [1 << 17]byte + var x8333 [1 << 17]byte + var x8334 [1 << 17]byte + var x8335 [1 << 17]byte + var x8336 [1 << 17]byte + var x8337 [1 << 17]byte + var x8338 [1 << 17]byte + var x8339 [1 << 17]byte + var x8340 [1 << 17]byte + var x8341 [1 << 17]byte + var x8342 [1 << 17]byte + var x8343 [1 << 17]byte + var x8344 [1 << 17]byte + var x8345 [1 << 17]byte + var x8346 [1 << 17]byte + var x8347 [1 << 17]byte + var x8348 [1 << 17]byte + var x8349 [1 << 17]byte + var x8350 [1 << 17]byte + var x8351 [1 << 17]byte + var x8352 [1 << 17]byte + var x8353 [1 << 17]byte + var x8354 [1 << 17]byte + var x8355 [1 << 17]byte + var x8356 [1 << 17]byte + var x8357 [1 << 17]byte + var x8358 [1 << 17]byte + var x8359 [1 << 17]byte + var x8360 [1 << 17]byte + var x8361 [1 << 17]byte + var x8362 [1 << 17]byte + var x8363 [1 << 17]byte + var x8364 [1 << 17]byte + var x8365 [1 << 17]byte + var x8366 [1 << 17]byte + var x8367 [1 << 17]byte + var x8368 [1 << 17]byte + var x8369 [1 << 17]byte + var x8370 [1 << 17]byte + var x8371 [1 << 17]byte + var x8372 [1 << 17]byte + var x8373 [1 << 17]byte + var x8374 [1 << 17]byte + var x8375 [1 << 17]byte + var x8376 [1 << 17]byte + var x8377 [1 << 17]byte + var x8378 [1 << 17]byte + var x8379 [1 << 17]byte + var x8380 [1 << 17]byte + var x8381 [1 << 17]byte + var x8382 [1 << 17]byte + var x8383 [1 << 17]byte + var x8384 [1 << 17]byte + var x8385 [1 << 17]byte + var x8386 [1 << 17]byte + var x8387 [1 << 17]byte + var x8388 [1 << 17]byte + var x8389 [1 << 17]byte + var x8390 [1 << 17]byte + var x8391 [1 << 17]byte + var x8392 [1 << 17]byte + var x8393 [1 << 17]byte + var x8394 [1 << 17]byte + var x8395 [1 << 17]byte + var x8396 [1 << 17]byte + var x8397 [1 << 17]byte + var x8398 [1 << 17]byte + var x8399 [1 << 17]byte + var x8400 [1 << 17]byte + var x8401 [1 << 17]byte + var x8402 [1 << 17]byte + var x8403 [1 << 17]byte + var x8404 [1 << 17]byte + var x8405 [1 << 17]byte + var x8406 [1 << 17]byte + var x8407 [1 << 17]byte + var x8408 [1 << 17]byte + var x8409 [1 << 17]byte + var x8410 [1 << 17]byte + var x8411 [1 << 17]byte + var x8412 [1 << 17]byte + var x8413 [1 << 17]byte + var x8414 [1 << 17]byte + var x8415 [1 << 17]byte + var x8416 [1 << 17]byte + var x8417 [1 << 17]byte + var x8418 [1 << 17]byte + var x8419 [1 << 17]byte + var x8420 [1 << 17]byte + var x8421 [1 << 17]byte + var x8422 [1 << 17]byte + var x8423 [1 << 17]byte + var x8424 [1 << 17]byte + var x8425 [1 << 17]byte + var x8426 [1 << 17]byte + var x8427 [1 << 17]byte + var x8428 [1 << 17]byte + var x8429 [1 << 17]byte + var x8430 [1 << 17]byte + var x8431 [1 << 17]byte + var x8432 [1 << 17]byte + var x8433 [1 << 17]byte + var x8434 [1 << 17]byte + var x8435 [1 << 17]byte + var x8436 [1 << 17]byte + var x8437 [1 << 17]byte + var x8438 [1 << 17]byte + var x8439 [1 << 17]byte + var x8440 [1 << 17]byte + var x8441 [1 << 17]byte + var x8442 [1 << 17]byte + var x8443 [1 << 17]byte + var x8444 [1 << 17]byte + var x8445 [1 << 17]byte + var x8446 [1 << 17]byte + var x8447 [1 << 17]byte + var x8448 [1 << 17]byte + var x8449 [1 << 17]byte + var x8450 [1 << 17]byte + var x8451 [1 << 17]byte + var x8452 [1 << 17]byte + var x8453 [1 << 17]byte + var x8454 [1 << 17]byte + var x8455 [1 << 17]byte + var x8456 [1 << 17]byte + var x8457 [1 << 17]byte + var x8458 [1 << 17]byte + var x8459 [1 << 17]byte + var x8460 [1 << 17]byte + var x8461 [1 << 17]byte + var x8462 [1 << 17]byte + var x8463 [1 << 17]byte + var x8464 [1 << 17]byte + var x8465 [1 << 17]byte + var x8466 [1 << 17]byte + var x8467 [1 << 17]byte + var x8468 [1 << 17]byte + var x8469 [1 << 17]byte + var x8470 [1 << 17]byte + var x8471 [1 << 17]byte + var x8472 [1 << 17]byte + var x8473 [1 << 17]byte + var x8474 [1 << 17]byte + var x8475 [1 << 17]byte + var x8476 [1 << 17]byte + var x8477 [1 << 17]byte + var x8478 [1 << 17]byte + var x8479 [1 << 17]byte + var x8480 [1 << 17]byte + var x8481 [1 << 17]byte + var x8482 [1 << 17]byte + var x8483 [1 << 17]byte + var x8484 [1 << 17]byte + var x8485 [1 << 17]byte + var x8486 [1 << 17]byte + var x8487 [1 << 17]byte + var x8488 [1 << 17]byte + var x8489 [1 << 17]byte + var x8490 [1 << 17]byte + var x8491 [1 << 17]byte + var x8492 [1 << 17]byte + var x8493 [1 << 17]byte + var x8494 [1 << 17]byte + var x8495 [1 << 17]byte + var x8496 [1 << 17]byte + var x8497 [1 << 17]byte + var x8498 [1 << 17]byte + var x8499 [1 << 17]byte + var x8500 [1 << 17]byte + var x8501 [1 << 17]byte + var x8502 [1 << 17]byte + var x8503 [1 << 17]byte + var x8504 [1 << 17]byte + var x8505 [1 << 17]byte + var x8506 [1 << 17]byte + var x8507 [1 << 17]byte + var x8508 [1 << 17]byte + var x8509 [1 << 17]byte + var x8510 [1 << 17]byte + var x8511 [1 << 17]byte + var x8512 [1 << 17]byte + var x8513 [1 << 17]byte + var x8514 [1 << 17]byte + var x8515 [1 << 17]byte + var x8516 [1 << 17]byte + var x8517 [1 << 17]byte + var x8518 [1 << 17]byte + var x8519 [1 << 17]byte + var x8520 [1 << 17]byte + var x8521 [1 << 17]byte + var x8522 [1 << 17]byte + var x8523 [1 << 17]byte + var x8524 [1 << 17]byte + var x8525 [1 << 17]byte + var x8526 [1 << 17]byte + var x8527 [1 << 17]byte + var x8528 [1 << 17]byte + var x8529 [1 << 17]byte + var x8530 [1 << 17]byte + var x8531 [1 << 17]byte + var x8532 [1 << 17]byte + var x8533 [1 << 17]byte + var x8534 [1 << 17]byte + var x8535 [1 << 17]byte + var x8536 [1 << 17]byte + var x8537 [1 << 17]byte + var x8538 [1 << 17]byte + var x8539 [1 << 17]byte + var x8540 [1 << 17]byte + var x8541 [1 << 17]byte + var x8542 [1 << 17]byte + var x8543 [1 << 17]byte + var x8544 [1 << 17]byte + var x8545 [1 << 17]byte + var x8546 [1 << 17]byte + var x8547 [1 << 17]byte + var x8548 [1 << 17]byte + var x8549 [1 << 17]byte + var x8550 [1 << 17]byte + var x8551 [1 << 17]byte + var x8552 [1 << 17]byte + var x8553 [1 << 17]byte + var x8554 [1 << 17]byte + var x8555 [1 << 17]byte + var x8556 [1 << 17]byte + var x8557 [1 << 17]byte + var x8558 [1 << 17]byte + var x8559 [1 << 17]byte + var x8560 [1 << 17]byte + var x8561 [1 << 17]byte + var x8562 [1 << 17]byte + var x8563 [1 << 17]byte + var x8564 [1 << 17]byte + var x8565 [1 << 17]byte + var x8566 [1 << 17]byte + var x8567 [1 << 17]byte + var x8568 [1 << 17]byte + var x8569 [1 << 17]byte + var x8570 [1 << 17]byte + var x8571 [1 << 17]byte + var x8572 [1 << 17]byte + var x8573 [1 << 17]byte + var x8574 [1 << 17]byte + var x8575 [1 << 17]byte + var x8576 [1 << 17]byte + var x8577 [1 << 17]byte + var x8578 [1 << 17]byte + var x8579 [1 << 17]byte + var x8580 [1 << 17]byte + var x8581 [1 << 17]byte + var x8582 [1 << 17]byte + var x8583 [1 << 17]byte + var x8584 [1 << 17]byte + var x8585 [1 << 17]byte + var x8586 [1 << 17]byte + var x8587 [1 << 17]byte + var x8588 [1 << 17]byte + var x8589 [1 << 17]byte + var x8590 [1 << 17]byte + var x8591 [1 << 17]byte + var x8592 [1 << 17]byte + var x8593 [1 << 17]byte + var x8594 [1 << 17]byte + var x8595 [1 << 17]byte + var x8596 [1 << 17]byte + var x8597 [1 << 17]byte + var x8598 [1 << 17]byte + var x8599 [1 << 17]byte + var x8600 [1 << 17]byte + var x8601 [1 << 17]byte + var x8602 [1 << 17]byte + var x8603 [1 << 17]byte + var x8604 [1 << 17]byte + var x8605 [1 << 17]byte + var x8606 [1 << 17]byte + var x8607 [1 << 17]byte + var x8608 [1 << 17]byte + var x8609 [1 << 17]byte + var x8610 [1 << 17]byte + var x8611 [1 << 17]byte + var x8612 [1 << 17]byte + var x8613 [1 << 17]byte + var x8614 [1 << 17]byte + var x8615 [1 << 17]byte + var x8616 [1 << 17]byte + var x8617 [1 << 17]byte + var x8618 [1 << 17]byte + var x8619 [1 << 17]byte + var x8620 [1 << 17]byte + var x8621 [1 << 17]byte + var x8622 [1 << 17]byte + var x8623 [1 << 17]byte + var x8624 [1 << 17]byte + var x8625 [1 << 17]byte + var x8626 [1 << 17]byte + var x8627 [1 << 17]byte + var x8628 [1 << 17]byte + var x8629 [1 << 17]byte + var x8630 [1 << 17]byte + var x8631 [1 << 17]byte + var x8632 [1 << 17]byte + var x8633 [1 << 17]byte + var x8634 [1 << 17]byte + var x8635 [1 << 17]byte + var x8636 [1 << 17]byte + var x8637 [1 << 17]byte + var x8638 [1 << 17]byte + var x8639 [1 << 17]byte + var x8640 [1 << 17]byte + var x8641 [1 << 17]byte + var x8642 [1 << 17]byte + var x8643 [1 << 17]byte + var x8644 [1 << 17]byte + var x8645 [1 << 17]byte + var x8646 [1 << 17]byte + var x8647 [1 << 17]byte + var x8648 [1 << 17]byte + var x8649 [1 << 17]byte + var x8650 [1 << 17]byte + var x8651 [1 << 17]byte + var x8652 [1 << 17]byte + var x8653 [1 << 17]byte + var x8654 [1 << 17]byte + var x8655 [1 << 17]byte + var x8656 [1 << 17]byte + var x8657 [1 << 17]byte + var x8658 [1 << 17]byte + var x8659 [1 << 17]byte + var x8660 [1 << 17]byte + var x8661 [1 << 17]byte + var x8662 [1 << 17]byte + var x8663 [1 << 17]byte + var x8664 [1 << 17]byte + var x8665 [1 << 17]byte + var x8666 [1 << 17]byte + var x8667 [1 << 17]byte + var x8668 [1 << 17]byte + var x8669 [1 << 17]byte + var x8670 [1 << 17]byte + var x8671 [1 << 17]byte + var x8672 [1 << 17]byte + var x8673 [1 << 17]byte + var x8674 [1 << 17]byte + var x8675 [1 << 17]byte + var x8676 [1 << 17]byte + var x8677 [1 << 17]byte + var x8678 [1 << 17]byte + var x8679 [1 << 17]byte + var x8680 [1 << 17]byte + var x8681 [1 << 17]byte + var x8682 [1 << 17]byte + var x8683 [1 << 17]byte + var x8684 [1 << 17]byte + var x8685 [1 << 17]byte + var x8686 [1 << 17]byte + var x8687 [1 << 17]byte + var x8688 [1 << 17]byte + var x8689 [1 << 17]byte + var x8690 [1 << 17]byte + var x8691 [1 << 17]byte + var x8692 [1 << 17]byte + var x8693 [1 << 17]byte + var x8694 [1 << 17]byte + var x8695 [1 << 17]byte + var x8696 [1 << 17]byte + var x8697 [1 << 17]byte + var x8698 [1 << 17]byte + var x8699 [1 << 17]byte + var x8700 [1 << 17]byte + var x8701 [1 << 17]byte + var x8702 [1 << 17]byte + var x8703 [1 << 17]byte + var x8704 [1 << 17]byte + var x8705 [1 << 17]byte + var x8706 [1 << 17]byte + var x8707 [1 << 17]byte + var x8708 [1 << 17]byte + var x8709 [1 << 17]byte + var x8710 [1 << 17]byte + var x8711 [1 << 17]byte + var x8712 [1 << 17]byte + var x8713 [1 << 17]byte + var x8714 [1 << 17]byte + var x8715 [1 << 17]byte + var x8716 [1 << 17]byte + var x8717 [1 << 17]byte + var x8718 [1 << 17]byte + var x8719 [1 << 17]byte + var x8720 [1 << 17]byte + var x8721 [1 << 17]byte + var x8722 [1 << 17]byte + var x8723 [1 << 17]byte + var x8724 [1 << 17]byte + var x8725 [1 << 17]byte + var x8726 [1 << 17]byte + var x8727 [1 << 17]byte + var x8728 [1 << 17]byte + var x8729 [1 << 17]byte + var x8730 [1 << 17]byte + var x8731 [1 << 17]byte + var x8732 [1 << 17]byte + var x8733 [1 << 17]byte + var x8734 [1 << 17]byte + var x8735 [1 << 17]byte + var x8736 [1 << 17]byte + var x8737 [1 << 17]byte + var x8738 [1 << 17]byte + var x8739 [1 << 17]byte + var x8740 [1 << 17]byte + var x8741 [1 << 17]byte + var x8742 [1 << 17]byte + var x8743 [1 << 17]byte + var x8744 [1 << 17]byte + var x8745 [1 << 17]byte + var x8746 [1 << 17]byte + var x8747 [1 << 17]byte + var x8748 [1 << 17]byte + var x8749 [1 << 17]byte + var x8750 [1 << 17]byte + var x8751 [1 << 17]byte + var x8752 [1 << 17]byte + var x8753 [1 << 17]byte + var x8754 [1 << 17]byte + var x8755 [1 << 17]byte + var x8756 [1 << 17]byte + var x8757 [1 << 17]byte + var x8758 [1 << 17]byte + var x8759 [1 << 17]byte + var x8760 [1 << 17]byte + var x8761 [1 << 17]byte + var x8762 [1 << 17]byte + var x8763 [1 << 17]byte + var x8764 [1 << 17]byte + var x8765 [1 << 17]byte + var x8766 [1 << 17]byte + var x8767 [1 << 17]byte + var x8768 [1 << 17]byte + var x8769 [1 << 17]byte + var x8770 [1 << 17]byte + var x8771 [1 << 17]byte + var x8772 [1 << 17]byte + var x8773 [1 << 17]byte + var x8774 [1 << 17]byte + var x8775 [1 << 17]byte + var x8776 [1 << 17]byte + var x8777 [1 << 17]byte + var x8778 [1 << 17]byte + var x8779 [1 << 17]byte + var x8780 [1 << 17]byte + var x8781 [1 << 17]byte + var x8782 [1 << 17]byte + var x8783 [1 << 17]byte + var x8784 [1 << 17]byte + var x8785 [1 << 17]byte + var x8786 [1 << 17]byte + var x8787 [1 << 17]byte + var x8788 [1 << 17]byte + var x8789 [1 << 17]byte + var x8790 [1 << 17]byte + var x8791 [1 << 17]byte + var x8792 [1 << 17]byte + var x8793 [1 << 17]byte + var x8794 [1 << 17]byte + var x8795 [1 << 17]byte + var x8796 [1 << 17]byte + var x8797 [1 << 17]byte + var x8798 [1 << 17]byte + var x8799 [1 << 17]byte + var x8800 [1 << 17]byte + var x8801 [1 << 17]byte + var x8802 [1 << 17]byte + var x8803 [1 << 17]byte + var x8804 [1 << 17]byte + var x8805 [1 << 17]byte + var x8806 [1 << 17]byte + var x8807 [1 << 17]byte + var x8808 [1 << 17]byte + var x8809 [1 << 17]byte + var x8810 [1 << 17]byte + var x8811 [1 << 17]byte + var x8812 [1 << 17]byte + var x8813 [1 << 17]byte + var x8814 [1 << 17]byte + var x8815 [1 << 17]byte + var x8816 [1 << 17]byte + var x8817 [1 << 17]byte + var x8818 [1 << 17]byte + var x8819 [1 << 17]byte + var x8820 [1 << 17]byte + var x8821 [1 << 17]byte + var x8822 [1 << 17]byte + var x8823 [1 << 17]byte + var x8824 [1 << 17]byte + var x8825 [1 << 17]byte + var x8826 [1 << 17]byte + var x8827 [1 << 17]byte + var x8828 [1 << 17]byte + var x8829 [1 << 17]byte + var x8830 [1 << 17]byte + var x8831 [1 << 17]byte + var x8832 [1 << 17]byte + var x8833 [1 << 17]byte + var x8834 [1 << 17]byte + var x8835 [1 << 17]byte + var x8836 [1 << 17]byte + var x8837 [1 << 17]byte + var x8838 [1 << 17]byte + var x8839 [1 << 17]byte + var x8840 [1 << 17]byte + var x8841 [1 << 17]byte + var x8842 [1 << 17]byte + var x8843 [1 << 17]byte + var x8844 [1 << 17]byte + var x8845 [1 << 17]byte + var x8846 [1 << 17]byte + var x8847 [1 << 17]byte + var x8848 [1 << 17]byte + var x8849 [1 << 17]byte + var x8850 [1 << 17]byte + var x8851 [1 << 17]byte + var x8852 [1 << 17]byte + var x8853 [1 << 17]byte + var x8854 [1 << 17]byte + var x8855 [1 << 17]byte + var x8856 [1 << 17]byte + var x8857 [1 << 17]byte + var x8858 [1 << 17]byte + var x8859 [1 << 17]byte + var x8860 [1 << 17]byte + var x8861 [1 << 17]byte + var x8862 [1 << 17]byte + var x8863 [1 << 17]byte + var x8864 [1 << 17]byte + var x8865 [1 << 17]byte + var x8866 [1 << 17]byte + var x8867 [1 << 17]byte + var x8868 [1 << 17]byte + var x8869 [1 << 17]byte + var x8870 [1 << 17]byte + var x8871 [1 << 17]byte + var x8872 [1 << 17]byte + var x8873 [1 << 17]byte + var x8874 [1 << 17]byte + var x8875 [1 << 17]byte + var x8876 [1 << 17]byte + var x8877 [1 << 17]byte + var x8878 [1 << 17]byte + var x8879 [1 << 17]byte + var x8880 [1 << 17]byte + var x8881 [1 << 17]byte + var x8882 [1 << 17]byte + var x8883 [1 << 17]byte + var x8884 [1 << 17]byte + var x8885 [1 << 17]byte + var x8886 [1 << 17]byte + var x8887 [1 << 17]byte + var x8888 [1 << 17]byte + var x8889 [1 << 17]byte + var x8890 [1 << 17]byte + var x8891 [1 << 17]byte + var x8892 [1 << 17]byte + var x8893 [1 << 17]byte + var x8894 [1 << 17]byte + var x8895 [1 << 17]byte + var x8896 [1 << 17]byte + var x8897 [1 << 17]byte + var x8898 [1 << 17]byte + var x8899 [1 << 17]byte + var x8900 [1 << 17]byte + var x8901 [1 << 17]byte + var x8902 [1 << 17]byte + var x8903 [1 << 17]byte + var x8904 [1 << 17]byte + var x8905 [1 << 17]byte + var x8906 [1 << 17]byte + var x8907 [1 << 17]byte + var x8908 [1 << 17]byte + var x8909 [1 << 17]byte + var x8910 [1 << 17]byte + var x8911 [1 << 17]byte + var x8912 [1 << 17]byte + var x8913 [1 << 17]byte + var x8914 [1 << 17]byte + var x8915 [1 << 17]byte + var x8916 [1 << 17]byte + var x8917 [1 << 17]byte + var x8918 [1 << 17]byte + var x8919 [1 << 17]byte + var x8920 [1 << 17]byte + var x8921 [1 << 17]byte + var x8922 [1 << 17]byte + var x8923 [1 << 17]byte + var x8924 [1 << 17]byte + var x8925 [1 << 17]byte + var x8926 [1 << 17]byte + var x8927 [1 << 17]byte + var x8928 [1 << 17]byte + var x8929 [1 << 17]byte + var x8930 [1 << 17]byte + var x8931 [1 << 17]byte + var x8932 [1 << 17]byte + var x8933 [1 << 17]byte + var x8934 [1 << 17]byte + var x8935 [1 << 17]byte + var x8936 [1 << 17]byte + var x8937 [1 << 17]byte + var x8938 [1 << 17]byte + var x8939 [1 << 17]byte + var x8940 [1 << 17]byte + var x8941 [1 << 17]byte + var x8942 [1 << 17]byte + var x8943 [1 << 17]byte + var x8944 [1 << 17]byte + var x8945 [1 << 17]byte + var x8946 [1 << 17]byte + var x8947 [1 << 17]byte + var x8948 [1 << 17]byte + var x8949 [1 << 17]byte + var x8950 [1 << 17]byte + var x8951 [1 << 17]byte + var x8952 [1 << 17]byte + var x8953 [1 << 17]byte + var x8954 [1 << 17]byte + var x8955 [1 << 17]byte + var x8956 [1 << 17]byte + var x8957 [1 << 17]byte + var x8958 [1 << 17]byte + var x8959 [1 << 17]byte + var x8960 [1 << 17]byte + var x8961 [1 << 17]byte + var x8962 [1 << 17]byte + var x8963 [1 << 17]byte + var x8964 [1 << 17]byte + var x8965 [1 << 17]byte + var x8966 [1 << 17]byte + var x8967 [1 << 17]byte + var x8968 [1 << 17]byte + var x8969 [1 << 17]byte + var x8970 [1 << 17]byte + var x8971 [1 << 17]byte + var x8972 [1 << 17]byte + var x8973 [1 << 17]byte + var x8974 [1 << 17]byte + var x8975 [1 << 17]byte + var x8976 [1 << 17]byte + var x8977 [1 << 17]byte + var x8978 [1 << 17]byte + var x8979 [1 << 17]byte + var x8980 [1 << 17]byte + var x8981 [1 << 17]byte + var x8982 [1 << 17]byte + var x8983 [1 << 17]byte + var x8984 [1 << 17]byte + var x8985 [1 << 17]byte + var x8986 [1 << 17]byte + var x8987 [1 << 17]byte + var x8988 [1 << 17]byte + var x8989 [1 << 17]byte + var x8990 [1 << 17]byte + var x8991 [1 << 17]byte + var x8992 [1 << 17]byte + var x8993 [1 << 17]byte + var x8994 [1 << 17]byte + var x8995 [1 << 17]byte + var x8996 [1 << 17]byte + var x8997 [1 << 17]byte + var x8998 [1 << 17]byte + var x8999 [1 << 17]byte + var x9000 [1 << 17]byte + var x9001 [1 << 17]byte + var x9002 [1 << 17]byte + var x9003 [1 << 17]byte + var x9004 [1 << 17]byte + var x9005 [1 << 17]byte + var x9006 [1 << 17]byte + var x9007 [1 << 17]byte + var x9008 [1 << 17]byte + var x9009 [1 << 17]byte + var x9010 [1 << 17]byte + var x9011 [1 << 17]byte + var x9012 [1 << 17]byte + var x9013 [1 << 17]byte + var x9014 [1 << 17]byte + var x9015 [1 << 17]byte + var x9016 [1 << 17]byte + var x9017 [1 << 17]byte + var x9018 [1 << 17]byte + var x9019 [1 << 17]byte + var x9020 [1 << 17]byte + var x9021 [1 << 17]byte + var x9022 [1 << 17]byte + var x9023 [1 << 17]byte + var x9024 [1 << 17]byte + var x9025 [1 << 17]byte + var x9026 [1 << 17]byte + var x9027 [1 << 17]byte + var x9028 [1 << 17]byte + var x9029 [1 << 17]byte + var x9030 [1 << 17]byte + var x9031 [1 << 17]byte + var x9032 [1 << 17]byte + var x9033 [1 << 17]byte + var x9034 [1 << 17]byte + var x9035 [1 << 17]byte + var x9036 [1 << 17]byte + var x9037 [1 << 17]byte + var x9038 [1 << 17]byte + var x9039 [1 << 17]byte + var x9040 [1 << 17]byte + var x9041 [1 << 17]byte + var x9042 [1 << 17]byte + var x9043 [1 << 17]byte + var x9044 [1 << 17]byte + var x9045 [1 << 17]byte + var x9046 [1 << 17]byte + var x9047 [1 << 17]byte + var x9048 [1 << 17]byte + var x9049 [1 << 17]byte + var x9050 [1 << 17]byte + var x9051 [1 << 17]byte + var x9052 [1 << 17]byte + var x9053 [1 << 17]byte + var x9054 [1 << 17]byte + var x9055 [1 << 17]byte + var x9056 [1 << 17]byte + var x9057 [1 << 17]byte + var x9058 [1 << 17]byte + var x9059 [1 << 17]byte + var x9060 [1 << 17]byte + var x9061 [1 << 17]byte + var x9062 [1 << 17]byte + var x9063 [1 << 17]byte + var x9064 [1 << 17]byte + var x9065 [1 << 17]byte + var x9066 [1 << 17]byte + var x9067 [1 << 17]byte + var x9068 [1 << 17]byte + var x9069 [1 << 17]byte + var x9070 [1 << 17]byte + var x9071 [1 << 17]byte + var x9072 [1 << 17]byte + var x9073 [1 << 17]byte + var x9074 [1 << 17]byte + var x9075 [1 << 17]byte + var x9076 [1 << 17]byte + var x9077 [1 << 17]byte + var x9078 [1 << 17]byte + var x9079 [1 << 17]byte + var x9080 [1 << 17]byte + var x9081 [1 << 17]byte + var x9082 [1 << 17]byte + var x9083 [1 << 17]byte + var x9084 [1 << 17]byte + var x9085 [1 << 17]byte + var x9086 [1 << 17]byte + var x9087 [1 << 17]byte + var x9088 [1 << 17]byte + var x9089 [1 << 17]byte + var x9090 [1 << 17]byte + var x9091 [1 << 17]byte + var x9092 [1 << 17]byte + var x9093 [1 << 17]byte + var x9094 [1 << 17]byte + var x9095 [1 << 17]byte + var x9096 [1 << 17]byte + var x9097 [1 << 17]byte + var x9098 [1 << 17]byte + var x9099 [1 << 17]byte + var x9100 [1 << 17]byte + var x9101 [1 << 17]byte + var x9102 [1 << 17]byte + var x9103 [1 << 17]byte + var x9104 [1 << 17]byte + var x9105 [1 << 17]byte + var x9106 [1 << 17]byte + var x9107 [1 << 17]byte + var x9108 [1 << 17]byte + var x9109 [1 << 17]byte + var x9110 [1 << 17]byte + var x9111 [1 << 17]byte + var x9112 [1 << 17]byte + var x9113 [1 << 17]byte + var x9114 [1 << 17]byte + var x9115 [1 << 17]byte + var x9116 [1 << 17]byte + var x9117 [1 << 17]byte + var x9118 [1 << 17]byte + var x9119 [1 << 17]byte + var x9120 [1 << 17]byte + var x9121 [1 << 17]byte + var x9122 [1 << 17]byte + var x9123 [1 << 17]byte + var x9124 [1 << 17]byte + var x9125 [1 << 17]byte + var x9126 [1 << 17]byte + var x9127 [1 << 17]byte + var x9128 [1 << 17]byte + var x9129 [1 << 17]byte + var x9130 [1 << 17]byte + var x9131 [1 << 17]byte + var x9132 [1 << 17]byte + var x9133 [1 << 17]byte + var x9134 [1 << 17]byte + var x9135 [1 << 17]byte + var x9136 [1 << 17]byte + var x9137 [1 << 17]byte + var x9138 [1 << 17]byte + var x9139 [1 << 17]byte + var x9140 [1 << 17]byte + var x9141 [1 << 17]byte + var x9142 [1 << 17]byte + var x9143 [1 << 17]byte + var x9144 [1 << 17]byte + var x9145 [1 << 17]byte + var x9146 [1 << 17]byte + var x9147 [1 << 17]byte + var x9148 [1 << 17]byte + var x9149 [1 << 17]byte + var x9150 [1 << 17]byte + var x9151 [1 << 17]byte + var x9152 [1 << 17]byte + var x9153 [1 << 17]byte + var x9154 [1 << 17]byte + var x9155 [1 << 17]byte + var x9156 [1 << 17]byte + var x9157 [1 << 17]byte + var x9158 [1 << 17]byte + var x9159 [1 << 17]byte + var x9160 [1 << 17]byte + var x9161 [1 << 17]byte + var x9162 [1 << 17]byte + var x9163 [1 << 17]byte + var x9164 [1 << 17]byte + var x9165 [1 << 17]byte + var x9166 [1 << 17]byte + var x9167 [1 << 17]byte + var x9168 [1 << 17]byte + var x9169 [1 << 17]byte + var x9170 [1 << 17]byte + var x9171 [1 << 17]byte + var x9172 [1 << 17]byte + var x9173 [1 << 17]byte + var x9174 [1 << 17]byte + var x9175 [1 << 17]byte + var x9176 [1 << 17]byte + var x9177 [1 << 17]byte + var x9178 [1 << 17]byte + var x9179 [1 << 17]byte + var x9180 [1 << 17]byte + var x9181 [1 << 17]byte + var x9182 [1 << 17]byte + var x9183 [1 << 17]byte + var x9184 [1 << 17]byte + var x9185 [1 << 17]byte + var x9186 [1 << 17]byte + var x9187 [1 << 17]byte + var x9188 [1 << 17]byte + var x9189 [1 << 17]byte + var x9190 [1 << 17]byte + var x9191 [1 << 17]byte + var x9192 [1 << 17]byte + var x9193 [1 << 17]byte + var x9194 [1 << 17]byte + var x9195 [1 << 17]byte + var x9196 [1 << 17]byte + var x9197 [1 << 17]byte + var x9198 [1 << 17]byte + var x9199 [1 << 17]byte + var x9200 [1 << 17]byte + var x9201 [1 << 17]byte + var x9202 [1 << 17]byte + var x9203 [1 << 17]byte + var x9204 [1 << 17]byte + var x9205 [1 << 17]byte + var x9206 [1 << 17]byte + var x9207 [1 << 17]byte + var x9208 [1 << 17]byte + var x9209 [1 << 17]byte + var x9210 [1 << 17]byte + var x9211 [1 << 17]byte + var x9212 [1 << 17]byte + var x9213 [1 << 17]byte + var x9214 [1 << 17]byte + var x9215 [1 << 17]byte + var x9216 [1 << 17]byte + var x9217 [1 << 17]byte + var x9218 [1 << 17]byte + var x9219 [1 << 17]byte + var x9220 [1 << 17]byte + var x9221 [1 << 17]byte + var x9222 [1 << 17]byte + var x9223 [1 << 17]byte + var x9224 [1 << 17]byte + var x9225 [1 << 17]byte + var x9226 [1 << 17]byte + var x9227 [1 << 17]byte + var x9228 [1 << 17]byte + var x9229 [1 << 17]byte + var x9230 [1 << 17]byte + var x9231 [1 << 17]byte + var x9232 [1 << 17]byte + var x9233 [1 << 17]byte + var x9234 [1 << 17]byte + var x9235 [1 << 17]byte + var x9236 [1 << 17]byte + var x9237 [1 << 17]byte + var x9238 [1 << 17]byte + var x9239 [1 << 17]byte + var x9240 [1 << 17]byte + var x9241 [1 << 17]byte + var x9242 [1 << 17]byte + var x9243 [1 << 17]byte + var x9244 [1 << 17]byte + var x9245 [1 << 17]byte + var x9246 [1 << 17]byte + var x9247 [1 << 17]byte + var x9248 [1 << 17]byte + var x9249 [1 << 17]byte + var x9250 [1 << 17]byte + var x9251 [1 << 17]byte + var x9252 [1 << 17]byte + var x9253 [1 << 17]byte + var x9254 [1 << 17]byte + var x9255 [1 << 17]byte + var x9256 [1 << 17]byte + var x9257 [1 << 17]byte + var x9258 [1 << 17]byte + var x9259 [1 << 17]byte + var x9260 [1 << 17]byte + var x9261 [1 << 17]byte + var x9262 [1 << 17]byte + var x9263 [1 << 17]byte + var x9264 [1 << 17]byte + var x9265 [1 << 17]byte + var x9266 [1 << 17]byte + var x9267 [1 << 17]byte + var x9268 [1 << 17]byte + var x9269 [1 << 17]byte + var x9270 [1 << 17]byte + var x9271 [1 << 17]byte + var x9272 [1 << 17]byte + var x9273 [1 << 17]byte + var x9274 [1 << 17]byte + var x9275 [1 << 17]byte + var x9276 [1 << 17]byte + var x9277 [1 << 17]byte + var x9278 [1 << 17]byte + var x9279 [1 << 17]byte + var x9280 [1 << 17]byte + var x9281 [1 << 17]byte + var x9282 [1 << 17]byte + var x9283 [1 << 17]byte + var x9284 [1 << 17]byte + var x9285 [1 << 17]byte + var x9286 [1 << 17]byte + var x9287 [1 << 17]byte + var x9288 [1 << 17]byte + var x9289 [1 << 17]byte + var x9290 [1 << 17]byte + var x9291 [1 << 17]byte + var x9292 [1 << 17]byte + var x9293 [1 << 17]byte + var x9294 [1 << 17]byte + var x9295 [1 << 17]byte + var x9296 [1 << 17]byte + var x9297 [1 << 17]byte + var x9298 [1 << 17]byte + var x9299 [1 << 17]byte + var x9300 [1 << 17]byte + var x9301 [1 << 17]byte + var x9302 [1 << 17]byte + var x9303 [1 << 17]byte + var x9304 [1 << 17]byte + var x9305 [1 << 17]byte + var x9306 [1 << 17]byte + var x9307 [1 << 17]byte + var x9308 [1 << 17]byte + var x9309 [1 << 17]byte + var x9310 [1 << 17]byte + var x9311 [1 << 17]byte + var x9312 [1 << 17]byte + var x9313 [1 << 17]byte + var x9314 [1 << 17]byte + var x9315 [1 << 17]byte + var x9316 [1 << 17]byte + var x9317 [1 << 17]byte + var x9318 [1 << 17]byte + var x9319 [1 << 17]byte + var x9320 [1 << 17]byte + var x9321 [1 << 17]byte + var x9322 [1 << 17]byte + var x9323 [1 << 17]byte + var x9324 [1 << 17]byte + var x9325 [1 << 17]byte + var x9326 [1 << 17]byte + var x9327 [1 << 17]byte + var x9328 [1 << 17]byte + var x9329 [1 << 17]byte + var x9330 [1 << 17]byte + var x9331 [1 << 17]byte + var x9332 [1 << 17]byte + var x9333 [1 << 17]byte + var x9334 [1 << 17]byte + var x9335 [1 << 17]byte + var x9336 [1 << 17]byte + var x9337 [1 << 17]byte + var x9338 [1 << 17]byte + var x9339 [1 << 17]byte + var x9340 [1 << 17]byte + var x9341 [1 << 17]byte + var x9342 [1 << 17]byte + var x9343 [1 << 17]byte + var x9344 [1 << 17]byte + var x9345 [1 << 17]byte + var x9346 [1 << 17]byte + var x9347 [1 << 17]byte + var x9348 [1 << 17]byte + var x9349 [1 << 17]byte + var x9350 [1 << 17]byte + var x9351 [1 << 17]byte + var x9352 [1 << 17]byte + var x9353 [1 << 17]byte + var x9354 [1 << 17]byte + var x9355 [1 << 17]byte + var x9356 [1 << 17]byte + var x9357 [1 << 17]byte + var x9358 [1 << 17]byte + var x9359 [1 << 17]byte + var x9360 [1 << 17]byte + var x9361 [1 << 17]byte + var x9362 [1 << 17]byte + var x9363 [1 << 17]byte + var x9364 [1 << 17]byte + var x9365 [1 << 17]byte + var x9366 [1 << 17]byte + var x9367 [1 << 17]byte + var x9368 [1 << 17]byte + var x9369 [1 << 17]byte + var x9370 [1 << 17]byte + var x9371 [1 << 17]byte + var x9372 [1 << 17]byte + var x9373 [1 << 17]byte + var x9374 [1 << 17]byte + var x9375 [1 << 17]byte + var x9376 [1 << 17]byte + var x9377 [1 << 17]byte + var x9378 [1 << 17]byte + var x9379 [1 << 17]byte + var x9380 [1 << 17]byte + var x9381 [1 << 17]byte + var x9382 [1 << 17]byte + var x9383 [1 << 17]byte + var x9384 [1 << 17]byte + var x9385 [1 << 17]byte + var x9386 [1 << 17]byte + var x9387 [1 << 17]byte + var x9388 [1 << 17]byte + var x9389 [1 << 17]byte + var x9390 [1 << 17]byte + var x9391 [1 << 17]byte + var x9392 [1 << 17]byte + var x9393 [1 << 17]byte + var x9394 [1 << 17]byte + var x9395 [1 << 17]byte + var x9396 [1 << 17]byte + var x9397 [1 << 17]byte + var x9398 [1 << 17]byte + var x9399 [1 << 17]byte + var x9400 [1 << 17]byte + var x9401 [1 << 17]byte + var x9402 [1 << 17]byte + var x9403 [1 << 17]byte + var x9404 [1 << 17]byte + var x9405 [1 << 17]byte + var x9406 [1 << 17]byte + var x9407 [1 << 17]byte + var x9408 [1 << 17]byte + var x9409 [1 << 17]byte + var x9410 [1 << 17]byte + var x9411 [1 << 17]byte + var x9412 [1 << 17]byte + var x9413 [1 << 17]byte + var x9414 [1 << 17]byte + var x9415 [1 << 17]byte + var x9416 [1 << 17]byte + var x9417 [1 << 17]byte + var x9418 [1 << 17]byte + var x9419 [1 << 17]byte + var x9420 [1 << 17]byte + var x9421 [1 << 17]byte + var x9422 [1 << 17]byte + var x9423 [1 << 17]byte + var x9424 [1 << 17]byte + var x9425 [1 << 17]byte + var x9426 [1 << 17]byte + var x9427 [1 << 17]byte + var x9428 [1 << 17]byte + var x9429 [1 << 17]byte + var x9430 [1 << 17]byte + var x9431 [1 << 17]byte + var x9432 [1 << 17]byte + var x9433 [1 << 17]byte + var x9434 [1 << 17]byte + var x9435 [1 << 17]byte + var x9436 [1 << 17]byte + var x9437 [1 << 17]byte + var x9438 [1 << 17]byte + var x9439 [1 << 17]byte + var x9440 [1 << 17]byte + var x9441 [1 << 17]byte + var x9442 [1 << 17]byte + var x9443 [1 << 17]byte + var x9444 [1 << 17]byte + var x9445 [1 << 17]byte + var x9446 [1 << 17]byte + var x9447 [1 << 17]byte + var x9448 [1 << 17]byte + var x9449 [1 << 17]byte + var x9450 [1 << 17]byte + var x9451 [1 << 17]byte + var x9452 [1 << 17]byte + var x9453 [1 << 17]byte + var x9454 [1 << 17]byte + var x9455 [1 << 17]byte + var x9456 [1 << 17]byte + var x9457 [1 << 17]byte + var x9458 [1 << 17]byte + var x9459 [1 << 17]byte + var x9460 [1 << 17]byte + var x9461 [1 << 17]byte + var x9462 [1 << 17]byte + var x9463 [1 << 17]byte + var x9464 [1 << 17]byte + var x9465 [1 << 17]byte + var x9466 [1 << 17]byte + var x9467 [1 << 17]byte + var x9468 [1 << 17]byte + var x9469 [1 << 17]byte + var x9470 [1 << 17]byte + var x9471 [1 << 17]byte + var x9472 [1 << 17]byte + var x9473 [1 << 17]byte + var x9474 [1 << 17]byte + var x9475 [1 << 17]byte + var x9476 [1 << 17]byte + var x9477 [1 << 17]byte + var x9478 [1 << 17]byte + var x9479 [1 << 17]byte + var x9480 [1 << 17]byte + var x9481 [1 << 17]byte + var x9482 [1 << 17]byte + var x9483 [1 << 17]byte + var x9484 [1 << 17]byte + var x9485 [1 << 17]byte + var x9486 [1 << 17]byte + var x9487 [1 << 17]byte + var x9488 [1 << 17]byte + var x9489 [1 << 17]byte + var x9490 [1 << 17]byte + var x9491 [1 << 17]byte + var x9492 [1 << 17]byte + var x9493 [1 << 17]byte + var x9494 [1 << 17]byte + var x9495 [1 << 17]byte + var x9496 [1 << 17]byte + var x9497 [1 << 17]byte + var x9498 [1 << 17]byte + var x9499 [1 << 17]byte + var x9500 [1 << 17]byte + var x9501 [1 << 17]byte + var x9502 [1 << 17]byte + var x9503 [1 << 17]byte + var x9504 [1 << 17]byte + var x9505 [1 << 17]byte + var x9506 [1 << 17]byte + var x9507 [1 << 17]byte + var x9508 [1 << 17]byte + var x9509 [1 << 17]byte + var x9510 [1 << 17]byte + var x9511 [1 << 17]byte + var x9512 [1 << 17]byte + var x9513 [1 << 17]byte + var x9514 [1 << 17]byte + var x9515 [1 << 17]byte + var x9516 [1 << 17]byte + var x9517 [1 << 17]byte + var x9518 [1 << 17]byte + var x9519 [1 << 17]byte + var x9520 [1 << 17]byte + var x9521 [1 << 17]byte + var x9522 [1 << 17]byte + var x9523 [1 << 17]byte + var x9524 [1 << 17]byte + var x9525 [1 << 17]byte + var x9526 [1 << 17]byte + var x9527 [1 << 17]byte + var x9528 [1 << 17]byte + var x9529 [1 << 17]byte + var x9530 [1 << 17]byte + var x9531 [1 << 17]byte + var x9532 [1 << 17]byte + var x9533 [1 << 17]byte + var x9534 [1 << 17]byte + var x9535 [1 << 17]byte + var x9536 [1 << 17]byte + var x9537 [1 << 17]byte + var x9538 [1 << 17]byte + var x9539 [1 << 17]byte + var x9540 [1 << 17]byte + var x9541 [1 << 17]byte + var x9542 [1 << 17]byte + var x9543 [1 << 17]byte + var x9544 [1 << 17]byte + var x9545 [1 << 17]byte + var x9546 [1 << 17]byte + var x9547 [1 << 17]byte + var x9548 [1 << 17]byte + var x9549 [1 << 17]byte + var x9550 [1 << 17]byte + var x9551 [1 << 17]byte + var x9552 [1 << 17]byte + var x9553 [1 << 17]byte + var x9554 [1 << 17]byte + var x9555 [1 << 17]byte + var x9556 [1 << 17]byte + var x9557 [1 << 17]byte + var x9558 [1 << 17]byte + var x9559 [1 << 17]byte + var x9560 [1 << 17]byte + var x9561 [1 << 17]byte + var x9562 [1 << 17]byte + var x9563 [1 << 17]byte + var x9564 [1 << 17]byte + var x9565 [1 << 17]byte + var x9566 [1 << 17]byte + var x9567 [1 << 17]byte + var x9568 [1 << 17]byte + var x9569 [1 << 17]byte + var x9570 [1 << 17]byte + var x9571 [1 << 17]byte + var x9572 [1 << 17]byte + var x9573 [1 << 17]byte + var x9574 [1 << 17]byte + var x9575 [1 << 17]byte + var x9576 [1 << 17]byte + var x9577 [1 << 17]byte + var x9578 [1 << 17]byte + var x9579 [1 << 17]byte + var x9580 [1 << 17]byte + var x9581 [1 << 17]byte + var x9582 [1 << 17]byte + var x9583 [1 << 17]byte + var x9584 [1 << 17]byte + var x9585 [1 << 17]byte + var x9586 [1 << 17]byte + var x9587 [1 << 17]byte + var x9588 [1 << 17]byte + var x9589 [1 << 17]byte + var x9590 [1 << 17]byte + var x9591 [1 << 17]byte + var x9592 [1 << 17]byte + var x9593 [1 << 17]byte + var x9594 [1 << 17]byte + var x9595 [1 << 17]byte + var x9596 [1 << 17]byte + var x9597 [1 << 17]byte + var x9598 [1 << 17]byte + var x9599 [1 << 17]byte + var x9600 [1 << 17]byte + var x9601 [1 << 17]byte + var x9602 [1 << 17]byte + var x9603 [1 << 17]byte + var x9604 [1 << 17]byte + var x9605 [1 << 17]byte + var x9606 [1 << 17]byte + var x9607 [1 << 17]byte + var x9608 [1 << 17]byte + var x9609 [1 << 17]byte + var x9610 [1 << 17]byte + var x9611 [1 << 17]byte + var x9612 [1 << 17]byte + var x9613 [1 << 17]byte + var x9614 [1 << 17]byte + var x9615 [1 << 17]byte + var x9616 [1 << 17]byte + var x9617 [1 << 17]byte + var x9618 [1 << 17]byte + var x9619 [1 << 17]byte + var x9620 [1 << 17]byte + var x9621 [1 << 17]byte + var x9622 [1 << 17]byte + var x9623 [1 << 17]byte + var x9624 [1 << 17]byte + var x9625 [1 << 17]byte + var x9626 [1 << 17]byte + var x9627 [1 << 17]byte + var x9628 [1 << 17]byte + var x9629 [1 << 17]byte + var x9630 [1 << 17]byte + var x9631 [1 << 17]byte + var x9632 [1 << 17]byte + var x9633 [1 << 17]byte + var x9634 [1 << 17]byte + var x9635 [1 << 17]byte + var x9636 [1 << 17]byte + var x9637 [1 << 17]byte + var x9638 [1 << 17]byte + var x9639 [1 << 17]byte + var x9640 [1 << 17]byte + var x9641 [1 << 17]byte + var x9642 [1 << 17]byte + var x9643 [1 << 17]byte + var x9644 [1 << 17]byte + var x9645 [1 << 17]byte + var x9646 [1 << 17]byte + var x9647 [1 << 17]byte + var x9648 [1 << 17]byte + var x9649 [1 << 17]byte + var x9650 [1 << 17]byte + var x9651 [1 << 17]byte + var x9652 [1 << 17]byte + var x9653 [1 << 17]byte + var x9654 [1 << 17]byte + var x9655 [1 << 17]byte + var x9656 [1 << 17]byte + var x9657 [1 << 17]byte + var x9658 [1 << 17]byte + var x9659 [1 << 17]byte + var x9660 [1 << 17]byte + var x9661 [1 << 17]byte + var x9662 [1 << 17]byte + var x9663 [1 << 17]byte + var x9664 [1 << 17]byte + var x9665 [1 << 17]byte + var x9666 [1 << 17]byte + var x9667 [1 << 17]byte + var x9668 [1 << 17]byte + var x9669 [1 << 17]byte + var x9670 [1 << 17]byte + var x9671 [1 << 17]byte + var x9672 [1 << 17]byte + var x9673 [1 << 17]byte + var x9674 [1 << 17]byte + var x9675 [1 << 17]byte + var x9676 [1 << 17]byte + var x9677 [1 << 17]byte + var x9678 [1 << 17]byte + var x9679 [1 << 17]byte + var x9680 [1 << 17]byte + var x9681 [1 << 17]byte + var x9682 [1 << 17]byte + var x9683 [1 << 17]byte + var x9684 [1 << 17]byte + var x9685 [1 << 17]byte + var x9686 [1 << 17]byte + var x9687 [1 << 17]byte + var x9688 [1 << 17]byte + var x9689 [1 << 17]byte + var x9690 [1 << 17]byte + var x9691 [1 << 17]byte + var x9692 [1 << 17]byte + var x9693 [1 << 17]byte + var x9694 [1 << 17]byte + var x9695 [1 << 17]byte + var x9696 [1 << 17]byte + var x9697 [1 << 17]byte + var x9698 [1 << 17]byte + var x9699 [1 << 17]byte + var x9700 [1 << 17]byte + var x9701 [1 << 17]byte + var x9702 [1 << 17]byte + var x9703 [1 << 17]byte + var x9704 [1 << 17]byte + var x9705 [1 << 17]byte + var x9706 [1 << 17]byte + var x9707 [1 << 17]byte + var x9708 [1 << 17]byte + var x9709 [1 << 17]byte + var x9710 [1 << 17]byte + var x9711 [1 << 17]byte + var x9712 [1 << 17]byte + var x9713 [1 << 17]byte + var x9714 [1 << 17]byte + var x9715 [1 << 17]byte + var x9716 [1 << 17]byte + var x9717 [1 << 17]byte + var x9718 [1 << 17]byte + var x9719 [1 << 17]byte + var x9720 [1 << 17]byte + var x9721 [1 << 17]byte + var x9722 [1 << 17]byte + var x9723 [1 << 17]byte + var x9724 [1 << 17]byte + var x9725 [1 << 17]byte + var x9726 [1 << 17]byte + var x9727 [1 << 17]byte + var x9728 [1 << 17]byte + var x9729 [1 << 17]byte + var x9730 [1 << 17]byte + var x9731 [1 << 17]byte + var x9732 [1 << 17]byte + var x9733 [1 << 17]byte + var x9734 [1 << 17]byte + var x9735 [1 << 17]byte + var x9736 [1 << 17]byte + var x9737 [1 << 17]byte + var x9738 [1 << 17]byte + var x9739 [1 << 17]byte + var x9740 [1 << 17]byte + var x9741 [1 << 17]byte + var x9742 [1 << 17]byte + var x9743 [1 << 17]byte + var x9744 [1 << 17]byte + var x9745 [1 << 17]byte + var x9746 [1 << 17]byte + var x9747 [1 << 17]byte + var x9748 [1 << 17]byte + var x9749 [1 << 17]byte + var x9750 [1 << 17]byte + var x9751 [1 << 17]byte + var x9752 [1 << 17]byte + var x9753 [1 << 17]byte + var x9754 [1 << 17]byte + var x9755 [1 << 17]byte + var x9756 [1 << 17]byte + var x9757 [1 << 17]byte + var x9758 [1 << 17]byte + var x9759 [1 << 17]byte + var x9760 [1 << 17]byte + var x9761 [1 << 17]byte + var x9762 [1 << 17]byte + var x9763 [1 << 17]byte + var x9764 [1 << 17]byte + var x9765 [1 << 17]byte + var x9766 [1 << 17]byte + var x9767 [1 << 17]byte + var x9768 [1 << 17]byte + var x9769 [1 << 17]byte + var x9770 [1 << 17]byte + var x9771 [1 << 17]byte + var x9772 [1 << 17]byte + var x9773 [1 << 17]byte + var x9774 [1 << 17]byte + var x9775 [1 << 17]byte + var x9776 [1 << 17]byte + var x9777 [1 << 17]byte + var x9778 [1 << 17]byte + var x9779 [1 << 17]byte + var x9780 [1 << 17]byte + var x9781 [1 << 17]byte + var x9782 [1 << 17]byte + var x9783 [1 << 17]byte + var x9784 [1 << 17]byte + var x9785 [1 << 17]byte + var x9786 [1 << 17]byte + var x9787 [1 << 17]byte + var x9788 [1 << 17]byte + var x9789 [1 << 17]byte + var x9790 [1 << 17]byte + var x9791 [1 << 17]byte + var x9792 [1 << 17]byte + var x9793 [1 << 17]byte + var x9794 [1 << 17]byte + var x9795 [1 << 17]byte + var x9796 [1 << 17]byte + var x9797 [1 << 17]byte + var x9798 [1 << 17]byte + var x9799 [1 << 17]byte + var x9800 [1 << 17]byte + var x9801 [1 << 17]byte + var x9802 [1 << 17]byte + var x9803 [1 << 17]byte + var x9804 [1 << 17]byte + var x9805 [1 << 17]byte + var x9806 [1 << 17]byte + var x9807 [1 << 17]byte + var x9808 [1 << 17]byte + var x9809 [1 << 17]byte + var x9810 [1 << 17]byte + var x9811 [1 << 17]byte + var x9812 [1 << 17]byte + var x9813 [1 << 17]byte + var x9814 [1 << 17]byte + var x9815 [1 << 17]byte + var x9816 [1 << 17]byte + var x9817 [1 << 17]byte + var x9818 [1 << 17]byte + var x9819 [1 << 17]byte + var x9820 [1 << 17]byte + var x9821 [1 << 17]byte + var x9822 [1 << 17]byte + var x9823 [1 << 17]byte + var x9824 [1 << 17]byte + var x9825 [1 << 17]byte + var x9826 [1 << 17]byte + var x9827 [1 << 17]byte + var x9828 [1 << 17]byte + var x9829 [1 << 17]byte + var x9830 [1 << 17]byte + var x9831 [1 << 17]byte + var x9832 [1 << 17]byte + var x9833 [1 << 17]byte + var x9834 [1 << 17]byte + var x9835 [1 << 17]byte + var x9836 [1 << 17]byte + var x9837 [1 << 17]byte + var x9838 [1 << 17]byte + var x9839 [1 << 17]byte + var x9840 [1 << 17]byte + var x9841 [1 << 17]byte + var x9842 [1 << 17]byte + var x9843 [1 << 17]byte + var x9844 [1 << 17]byte + var x9845 [1 << 17]byte + var x9846 [1 << 17]byte + var x9847 [1 << 17]byte + var x9848 [1 << 17]byte + var x9849 [1 << 17]byte + var x9850 [1 << 17]byte + var x9851 [1 << 17]byte + var x9852 [1 << 17]byte + var x9853 [1 << 17]byte + var x9854 [1 << 17]byte + var x9855 [1 << 17]byte + var x9856 [1 << 17]byte + var x9857 [1 << 17]byte + var x9858 [1 << 17]byte + var x9859 [1 << 17]byte + var x9860 [1 << 17]byte + var x9861 [1 << 17]byte + var x9862 [1 << 17]byte + var x9863 [1 << 17]byte + var x9864 [1 << 17]byte + var x9865 [1 << 17]byte + var x9866 [1 << 17]byte + var x9867 [1 << 17]byte + var x9868 [1 << 17]byte + var x9869 [1 << 17]byte + var x9870 [1 << 17]byte + var x9871 [1 << 17]byte + var x9872 [1 << 17]byte + var x9873 [1 << 17]byte + var x9874 [1 << 17]byte + var x9875 [1 << 17]byte + var x9876 [1 << 17]byte + var x9877 [1 << 17]byte + var x9878 [1 << 17]byte + var x9879 [1 << 17]byte + var x9880 [1 << 17]byte + var x9881 [1 << 17]byte + var x9882 [1 << 17]byte + var x9883 [1 << 17]byte + var x9884 [1 << 17]byte + var x9885 [1 << 17]byte + var x9886 [1 << 17]byte + var x9887 [1 << 17]byte + var x9888 [1 << 17]byte + var x9889 [1 << 17]byte + var x9890 [1 << 17]byte + var x9891 [1 << 17]byte + var x9892 [1 << 17]byte + var x9893 [1 << 17]byte + var x9894 [1 << 17]byte + var x9895 [1 << 17]byte + var x9896 [1 << 17]byte + var x9897 [1 << 17]byte + var x9898 [1 << 17]byte + var x9899 [1 << 17]byte + var x9900 [1 << 17]byte + var x9901 [1 << 17]byte + var x9902 [1 << 17]byte + var x9903 [1 << 17]byte + var x9904 [1 << 17]byte + var x9905 [1 << 17]byte + var x9906 [1 << 17]byte + var x9907 [1 << 17]byte + var x9908 [1 << 17]byte + var x9909 [1 << 17]byte + var x9910 [1 << 17]byte + var x9911 [1 << 17]byte + var x9912 [1 << 17]byte + var x9913 [1 << 17]byte + var x9914 [1 << 17]byte + var x9915 [1 << 17]byte + var x9916 [1 << 17]byte + var x9917 [1 << 17]byte + var x9918 [1 << 17]byte + var x9919 [1 << 17]byte + var x9920 [1 << 17]byte + var x9921 [1 << 17]byte + var x9922 [1 << 17]byte + var x9923 [1 << 17]byte + var x9924 [1 << 17]byte + var x9925 [1 << 17]byte + var x9926 [1 << 17]byte + var x9927 [1 << 17]byte + var x9928 [1 << 17]byte + var x9929 [1 << 17]byte + var x9930 [1 << 17]byte + var x9931 [1 << 17]byte + var x9932 [1 << 17]byte + var x9933 [1 << 17]byte + var x9934 [1 << 17]byte + var x9935 [1 << 17]byte + var x9936 [1 << 17]byte + var x9937 [1 << 17]byte + var x9938 [1 << 17]byte + var x9939 [1 << 17]byte + var x9940 [1 << 17]byte + var x9941 [1 << 17]byte + var x9942 [1 << 17]byte + var x9943 [1 << 17]byte + var x9944 [1 << 17]byte + var x9945 [1 << 17]byte + var x9946 [1 << 17]byte + var x9947 [1 << 17]byte + var x9948 [1 << 17]byte + var x9949 [1 << 17]byte + var x9950 [1 << 17]byte + var x9951 [1 << 17]byte + var x9952 [1 << 17]byte + var x9953 [1 << 17]byte + var x9954 [1 << 17]byte + var x9955 [1 << 17]byte + var x9956 [1 << 17]byte + var x9957 [1 << 17]byte + var x9958 [1 << 17]byte + var x9959 [1 << 17]byte + var x9960 [1 << 17]byte + var x9961 [1 << 17]byte + var x9962 [1 << 17]byte + var x9963 [1 << 17]byte + var x9964 [1 << 17]byte + var x9965 [1 << 17]byte + var x9966 [1 << 17]byte + var x9967 [1 << 17]byte + var x9968 [1 << 17]byte + var x9969 [1 << 17]byte + var x9970 [1 << 17]byte + var x9971 [1 << 17]byte + var x9972 [1 << 17]byte + var x9973 [1 << 17]byte + var x9974 [1 << 17]byte + var x9975 [1 << 17]byte + var x9976 [1 << 17]byte + var x9977 [1 << 17]byte + var x9978 [1 << 17]byte + var x9979 [1 << 17]byte + var x9980 [1 << 17]byte + var x9981 [1 << 17]byte + var x9982 [1 << 17]byte + var x9983 [1 << 17]byte + var x9984 [1 << 17]byte + var x9985 [1 << 17]byte + var x9986 [1 << 17]byte + var x9987 [1 << 17]byte + var x9988 [1 << 17]byte + var x9989 [1 << 17]byte + var x9990 [1 << 17]byte + var x9991 [1 << 17]byte + var x9992 [1 << 17]byte + var x9993 [1 << 17]byte + var x9994 [1 << 17]byte + var x9995 [1 << 17]byte + var x9996 [1 << 17]byte + var x9997 [1 << 17]byte + var x9998 [1 << 17]byte + var x9999 [1 << 17]byte + var x10000 [1 << 17]byte + var x10001 [1 << 17]byte + var x10002 [1 << 17]byte + var x10003 [1 << 17]byte + var x10004 [1 << 17]byte + var x10005 [1 << 17]byte + var x10006 [1 << 17]byte + var x10007 [1 << 17]byte + var x10008 [1 << 17]byte + var x10009 [1 << 17]byte + var x10010 [1 << 17]byte + var x10011 [1 << 17]byte + var x10012 [1 << 17]byte + var x10013 [1 << 17]byte + var x10014 [1 << 17]byte + var x10015 [1 << 17]byte + var x10016 [1 << 17]byte + var x10017 [1 << 17]byte + var x10018 [1 << 17]byte + var x10019 [1 << 17]byte + var x10020 [1 << 17]byte + var x10021 [1 << 17]byte + var x10022 [1 << 17]byte + var x10023 [1 << 17]byte + var x10024 [1 << 17]byte + var x10025 [1 << 17]byte + var x10026 [1 << 17]byte + var x10027 [1 << 17]byte + var x10028 [1 << 17]byte + var x10029 [1 << 17]byte + var x10030 [1 << 17]byte + var x10031 [1 << 17]byte + var x10032 [1 << 17]byte + var x10033 [1 << 17]byte + var x10034 [1 << 17]byte + var x10035 [1 << 17]byte + var x10036 [1 << 17]byte + var x10037 [1 << 17]byte + var x10038 [1 << 17]byte + var x10039 [1 << 17]byte + var x10040 [1 << 17]byte + var x10041 [1 << 17]byte + var x10042 [1 << 17]byte + var x10043 [1 << 17]byte + var x10044 [1 << 17]byte + var x10045 [1 << 17]byte + var x10046 [1 << 17]byte + var x10047 [1 << 17]byte + var x10048 [1 << 17]byte + var x10049 [1 << 17]byte + var x10050 [1 << 17]byte + var x10051 [1 << 17]byte + var x10052 [1 << 17]byte + var x10053 [1 << 17]byte + var x10054 [1 << 17]byte + var x10055 [1 << 17]byte + var x10056 [1 << 17]byte + var x10057 [1 << 17]byte + var x10058 [1 << 17]byte + var x10059 [1 << 17]byte + var x10060 [1 << 17]byte + var x10061 [1 << 17]byte + var x10062 [1 << 17]byte + var x10063 [1 << 17]byte + var x10064 [1 << 17]byte + var x10065 [1 << 17]byte + var x10066 [1 << 17]byte + var x10067 [1 << 17]byte + var x10068 [1 << 17]byte + var x10069 [1 << 17]byte + var x10070 [1 << 17]byte + var x10071 [1 << 17]byte + var x10072 [1 << 17]byte + var x10073 [1 << 17]byte + var x10074 [1 << 17]byte + var x10075 [1 << 17]byte + var x10076 [1 << 17]byte + var x10077 [1 << 17]byte + var x10078 [1 << 17]byte + var x10079 [1 << 17]byte + var x10080 [1 << 17]byte + var x10081 [1 << 17]byte + var x10082 [1 << 17]byte + var x10083 [1 << 17]byte + var x10084 [1 << 17]byte + var x10085 [1 << 17]byte + var x10086 [1 << 17]byte + var x10087 [1 << 17]byte + var x10088 [1 << 17]byte + var x10089 [1 << 17]byte + var x10090 [1 << 17]byte + var x10091 [1 << 17]byte + var x10092 [1 << 17]byte + var x10093 [1 << 17]byte + var x10094 [1 << 17]byte + var x10095 [1 << 17]byte + var x10096 [1 << 17]byte + var x10097 [1 << 17]byte + var x10098 [1 << 17]byte + var x10099 [1 << 17]byte + var x10100 [1 << 17]byte + var x10101 [1 << 17]byte + var x10102 [1 << 17]byte + var x10103 [1 << 17]byte + var x10104 [1 << 17]byte + var x10105 [1 << 17]byte + var x10106 [1 << 17]byte + var x10107 [1 << 17]byte + var x10108 [1 << 17]byte + var x10109 [1 << 17]byte + var x10110 [1 << 17]byte + var x10111 [1 << 17]byte + var x10112 [1 << 17]byte + var x10113 [1 << 17]byte + var x10114 [1 << 17]byte + var x10115 [1 << 17]byte + var x10116 [1 << 17]byte + var x10117 [1 << 17]byte + var x10118 [1 << 17]byte + var x10119 [1 << 17]byte + var x10120 [1 << 17]byte + var x10121 [1 << 17]byte + var x10122 [1 << 17]byte + var x10123 [1 << 17]byte + var x10124 [1 << 17]byte + var x10125 [1 << 17]byte + var x10126 [1 << 17]byte + var x10127 [1 << 17]byte + var x10128 [1 << 17]byte + var x10129 [1 << 17]byte + var x10130 [1 << 17]byte + var x10131 [1 << 17]byte + var x10132 [1 << 17]byte + var x10133 [1 << 17]byte + var x10134 [1 << 17]byte + var x10135 [1 << 17]byte + var x10136 [1 << 17]byte + var x10137 [1 << 17]byte + var x10138 [1 << 17]byte + var x10139 [1 << 17]byte + var x10140 [1 << 17]byte + var x10141 [1 << 17]byte + var x10142 [1 << 17]byte + var x10143 [1 << 17]byte + var x10144 [1 << 17]byte + var x10145 [1 << 17]byte + var x10146 [1 << 17]byte + var x10147 [1 << 17]byte + var x10148 [1 << 17]byte + var x10149 [1 << 17]byte + var x10150 [1 << 17]byte + var x10151 [1 << 17]byte + var x10152 [1 << 17]byte + var x10153 [1 << 17]byte + var x10154 [1 << 17]byte + var x10155 [1 << 17]byte + var x10156 [1 << 17]byte + var x10157 [1 << 17]byte + var x10158 [1 << 17]byte + var x10159 [1 << 17]byte + var x10160 [1 << 17]byte + var x10161 [1 << 17]byte + var x10162 [1 << 17]byte + var x10163 [1 << 17]byte + var x10164 [1 << 17]byte + var x10165 [1 << 17]byte + var x10166 [1 << 17]byte + var x10167 [1 << 17]byte + var x10168 [1 << 17]byte + var x10169 [1 << 17]byte + var x10170 [1 << 17]byte + var x10171 [1 << 17]byte + var x10172 [1 << 17]byte + var x10173 [1 << 17]byte + var x10174 [1 << 17]byte + var x10175 [1 << 17]byte + var x10176 [1 << 17]byte + var x10177 [1 << 17]byte + var x10178 [1 << 17]byte + var x10179 [1 << 17]byte + var x10180 [1 << 17]byte + var x10181 [1 << 17]byte + var x10182 [1 << 17]byte + var x10183 [1 << 17]byte + var x10184 [1 << 17]byte + var x10185 [1 << 17]byte + var x10186 [1 << 17]byte + var x10187 [1 << 17]byte + var x10188 [1 << 17]byte + var x10189 [1 << 17]byte + var x10190 [1 << 17]byte + var x10191 [1 << 17]byte + var x10192 [1 << 17]byte + var x10193 [1 << 17]byte + var x10194 [1 << 17]byte + var x10195 [1 << 17]byte + var x10196 [1 << 17]byte + var x10197 [1 << 17]byte + var x10198 [1 << 17]byte + var x10199 [1 << 17]byte + var x10200 [1 << 17]byte + var x10201 [1 << 17]byte + var x10202 [1 << 17]byte + var x10203 [1 << 17]byte + var x10204 [1 << 17]byte + var x10205 [1 << 17]byte + var x10206 [1 << 17]byte + var x10207 [1 << 17]byte + var x10208 [1 << 17]byte + var x10209 [1 << 17]byte + var x10210 [1 << 17]byte + var x10211 [1 << 17]byte + var x10212 [1 << 17]byte + var x10213 [1 << 17]byte + var x10214 [1 << 17]byte + var x10215 [1 << 17]byte + var x10216 [1 << 17]byte + var x10217 [1 << 17]byte + var x10218 [1 << 17]byte + var x10219 [1 << 17]byte + var x10220 [1 << 17]byte + var x10221 [1 << 17]byte + var x10222 [1 << 17]byte + var x10223 [1 << 17]byte + var x10224 [1 << 17]byte + var x10225 [1 << 17]byte + var x10226 [1 << 17]byte + var x10227 [1 << 17]byte + var x10228 [1 << 17]byte + var x10229 [1 << 17]byte + var x10230 [1 << 17]byte + var x10231 [1 << 17]byte + var x10232 [1 << 17]byte + var x10233 [1 << 17]byte + var x10234 [1 << 17]byte + var x10235 [1 << 17]byte + var x10236 [1 << 17]byte + var x10237 [1 << 17]byte + var x10238 [1 << 17]byte + var x10239 [1 << 17]byte + var x10240 [1 << 17]byte + var x10241 [1 << 17]byte + var x10242 [1 << 17]byte + var x10243 [1 << 17]byte + var x10244 [1 << 17]byte + var x10245 [1 << 17]byte + var x10246 [1 << 17]byte + var x10247 [1 << 17]byte + var x10248 [1 << 17]byte + var x10249 [1 << 17]byte + var x10250 [1 << 17]byte + var x10251 [1 << 17]byte + var x10252 [1 << 17]byte + var x10253 [1 << 17]byte + var x10254 [1 << 17]byte + var x10255 [1 << 17]byte + var x10256 [1 << 17]byte + var x10257 [1 << 17]byte + var x10258 [1 << 17]byte + var x10259 [1 << 17]byte + var x10260 [1 << 17]byte + var x10261 [1 << 17]byte + var x10262 [1 << 17]byte + var x10263 [1 << 17]byte + var x10264 [1 << 17]byte + var x10265 [1 << 17]byte + var x10266 [1 << 17]byte + var x10267 [1 << 17]byte + var x10268 [1 << 17]byte + var x10269 [1 << 17]byte + var x10270 [1 << 17]byte + var x10271 [1 << 17]byte + var x10272 [1 << 17]byte + var x10273 [1 << 17]byte + var x10274 [1 << 17]byte + var x10275 [1 << 17]byte + var x10276 [1 << 17]byte + var x10277 [1 << 17]byte + var x10278 [1 << 17]byte + var x10279 [1 << 17]byte + var x10280 [1 << 17]byte + var x10281 [1 << 17]byte + var x10282 [1 << 17]byte + var x10283 [1 << 17]byte + var x10284 [1 << 17]byte + var x10285 [1 << 17]byte + var x10286 [1 << 17]byte + var x10287 [1 << 17]byte + var x10288 [1 << 17]byte + var x10289 [1 << 17]byte + var x10290 [1 << 17]byte + var x10291 [1 << 17]byte + var x10292 [1 << 17]byte + var x10293 [1 << 17]byte + var x10294 [1 << 17]byte + var x10295 [1 << 17]byte + var x10296 [1 << 17]byte + var x10297 [1 << 17]byte + var x10298 [1 << 17]byte + var x10299 [1 << 17]byte + var x10300 [1 << 17]byte + var x10301 [1 << 17]byte + var x10302 [1 << 17]byte + var x10303 [1 << 17]byte + var x10304 [1 << 17]byte + var x10305 [1 << 17]byte + var x10306 [1 << 17]byte + var x10307 [1 << 17]byte + var x10308 [1 << 17]byte + var x10309 [1 << 17]byte + var x10310 [1 << 17]byte + var x10311 [1 << 17]byte + var x10312 [1 << 17]byte + var x10313 [1 << 17]byte + var x10314 [1 << 17]byte + var x10315 [1 << 17]byte + var x10316 [1 << 17]byte + var x10317 [1 << 17]byte + var x10318 [1 << 17]byte + var x10319 [1 << 17]byte + var x10320 [1 << 17]byte + var x10321 [1 << 17]byte + var x10322 [1 << 17]byte + var x10323 [1 << 17]byte + var x10324 [1 << 17]byte + var x10325 [1 << 17]byte + var x10326 [1 << 17]byte + var x10327 [1 << 17]byte + var x10328 [1 << 17]byte + var x10329 [1 << 17]byte + var x10330 [1 << 17]byte + var x10331 [1 << 17]byte + var x10332 [1 << 17]byte + var x10333 [1 << 17]byte + var x10334 [1 << 17]byte + var x10335 [1 << 17]byte + var x10336 [1 << 17]byte + var x10337 [1 << 17]byte + var x10338 [1 << 17]byte + var x10339 [1 << 17]byte + var x10340 [1 << 17]byte + var x10341 [1 << 17]byte + var x10342 [1 << 17]byte + var x10343 [1 << 17]byte + var x10344 [1 << 17]byte + var x10345 [1 << 17]byte + var x10346 [1 << 17]byte + var x10347 [1 << 17]byte + var x10348 [1 << 17]byte + var x10349 [1 << 17]byte + var x10350 [1 << 17]byte + var x10351 [1 << 17]byte + var x10352 [1 << 17]byte + var x10353 [1 << 17]byte + var x10354 [1 << 17]byte + var x10355 [1 << 17]byte + var x10356 [1 << 17]byte + var x10357 [1 << 17]byte + var x10358 [1 << 17]byte + var x10359 [1 << 17]byte + var x10360 [1 << 17]byte + var x10361 [1 << 17]byte + var x10362 [1 << 17]byte + var x10363 [1 << 17]byte + var x10364 [1 << 17]byte + var x10365 [1 << 17]byte + var x10366 [1 << 17]byte + var x10367 [1 << 17]byte + var x10368 [1 << 17]byte + var x10369 [1 << 17]byte + var x10370 [1 << 17]byte + var x10371 [1 << 17]byte + var x10372 [1 << 17]byte + var x10373 [1 << 17]byte + var x10374 [1 << 17]byte + var x10375 [1 << 17]byte + var x10376 [1 << 17]byte + var x10377 [1 << 17]byte + var x10378 [1 << 17]byte + var x10379 [1 << 17]byte + var x10380 [1 << 17]byte + var x10381 [1 << 17]byte + var x10382 [1 << 17]byte + var x10383 [1 << 17]byte + var x10384 [1 << 17]byte + var x10385 [1 << 17]byte + var x10386 [1 << 17]byte + var x10387 [1 << 17]byte + var x10388 [1 << 17]byte + var x10389 [1 << 17]byte + var x10390 [1 << 17]byte + var x10391 [1 << 17]byte + var x10392 [1 << 17]byte + var x10393 [1 << 17]byte + var x10394 [1 << 17]byte + var x10395 [1 << 17]byte + var x10396 [1 << 17]byte + var x10397 [1 << 17]byte + var x10398 [1 << 17]byte + var x10399 [1 << 17]byte + var x10400 [1 << 17]byte + var x10401 [1 << 17]byte + var x10402 [1 << 17]byte + var x10403 [1 << 17]byte + var x10404 [1 << 17]byte + var x10405 [1 << 17]byte + var x10406 [1 << 17]byte + var x10407 [1 << 17]byte + var x10408 [1 << 17]byte + var x10409 [1 << 17]byte + var x10410 [1 << 17]byte + var x10411 [1 << 17]byte + var x10412 [1 << 17]byte + var x10413 [1 << 17]byte + var x10414 [1 << 17]byte + var x10415 [1 << 17]byte + var x10416 [1 << 17]byte + var x10417 [1 << 17]byte + var x10418 [1 << 17]byte + var x10419 [1 << 17]byte + var x10420 [1 << 17]byte + var x10421 [1 << 17]byte + var x10422 [1 << 17]byte + var x10423 [1 << 17]byte + var x10424 [1 << 17]byte + var x10425 [1 << 17]byte + var x10426 [1 << 17]byte + var x10427 [1 << 17]byte + var x10428 [1 << 17]byte + var x10429 [1 << 17]byte + var x10430 [1 << 17]byte + var x10431 [1 << 17]byte + var x10432 [1 << 17]byte + var x10433 [1 << 17]byte + var x10434 [1 << 17]byte + var x10435 [1 << 17]byte + var x10436 [1 << 17]byte + var x10437 [1 << 17]byte + var x10438 [1 << 17]byte + var x10439 [1 << 17]byte + var x10440 [1 << 17]byte + var x10441 [1 << 17]byte + var x10442 [1 << 17]byte + var x10443 [1 << 17]byte + var x10444 [1 << 17]byte + var x10445 [1 << 17]byte + var x10446 [1 << 17]byte + var x10447 [1 << 17]byte + var x10448 [1 << 17]byte + var x10449 [1 << 17]byte + var x10450 [1 << 17]byte + var x10451 [1 << 17]byte + var x10452 [1 << 17]byte + var x10453 [1 << 17]byte + var x10454 [1 << 17]byte + var x10455 [1 << 17]byte + var x10456 [1 << 17]byte + var x10457 [1 << 17]byte + var x10458 [1 << 17]byte + var x10459 [1 << 17]byte + var x10460 [1 << 17]byte + var x10461 [1 << 17]byte + var x10462 [1 << 17]byte + var x10463 [1 << 17]byte + var x10464 [1 << 17]byte + var x10465 [1 << 17]byte + var x10466 [1 << 17]byte + var x10467 [1 << 17]byte + var x10468 [1 << 17]byte + var x10469 [1 << 17]byte + var x10470 [1 << 17]byte + var x10471 [1 << 17]byte + var x10472 [1 << 17]byte + var x10473 [1 << 17]byte + var x10474 [1 << 17]byte + var x10475 [1 << 17]byte + var x10476 [1 << 17]byte + var x10477 [1 << 17]byte + var x10478 [1 << 17]byte + var x10479 [1 << 17]byte + var x10480 [1 << 17]byte + var x10481 [1 << 17]byte + var x10482 [1 << 17]byte + var x10483 [1 << 17]byte + var x10484 [1 << 17]byte + var x10485 [1 << 17]byte + var x10486 [1 << 17]byte + var x10487 [1 << 17]byte + var x10488 [1 << 17]byte + var x10489 [1 << 17]byte + var x10490 [1 << 17]byte + var x10491 [1 << 17]byte + var x10492 [1 << 17]byte + var x10493 [1 << 17]byte + var x10494 [1 << 17]byte + var x10495 [1 << 17]byte + var x10496 [1 << 17]byte + var x10497 [1 << 17]byte + var x10498 [1 << 17]byte + var x10499 [1 << 17]byte + var x10500 [1 << 17]byte + var x10501 [1 << 17]byte + var x10502 [1 << 17]byte + var x10503 [1 << 17]byte + var x10504 [1 << 17]byte + var x10505 [1 << 17]byte + var x10506 [1 << 17]byte + var x10507 [1 << 17]byte + var x10508 [1 << 17]byte + var x10509 [1 << 17]byte + var x10510 [1 << 17]byte + var x10511 [1 << 17]byte + var x10512 [1 << 17]byte + var x10513 [1 << 17]byte + var x10514 [1 << 17]byte + var x10515 [1 << 17]byte + var x10516 [1 << 17]byte + var x10517 [1 << 17]byte + var x10518 [1 << 17]byte + var x10519 [1 << 17]byte + var x10520 [1 << 17]byte + var x10521 [1 << 17]byte + var x10522 [1 << 17]byte + var x10523 [1 << 17]byte + var x10524 [1 << 17]byte + var x10525 [1 << 17]byte + var x10526 [1 << 17]byte + var x10527 [1 << 17]byte + var x10528 [1 << 17]byte + var x10529 [1 << 17]byte + var x10530 [1 << 17]byte + var x10531 [1 << 17]byte + var x10532 [1 << 17]byte + var x10533 [1 << 17]byte + var x10534 [1 << 17]byte + var x10535 [1 << 17]byte + var x10536 [1 << 17]byte + var x10537 [1 << 17]byte + var x10538 [1 << 17]byte + var x10539 [1 << 17]byte + var x10540 [1 << 17]byte + var x10541 [1 << 17]byte + var x10542 [1 << 17]byte + var x10543 [1 << 17]byte + var x10544 [1 << 17]byte + var x10545 [1 << 17]byte + var x10546 [1 << 17]byte + var x10547 [1 << 17]byte + var x10548 [1 << 17]byte + var x10549 [1 << 17]byte + var x10550 [1 << 17]byte + var x10551 [1 << 17]byte + var x10552 [1 << 17]byte + var x10553 [1 << 17]byte + var x10554 [1 << 17]byte + var x10555 [1 << 17]byte + var x10556 [1 << 17]byte + var x10557 [1 << 17]byte + var x10558 [1 << 17]byte + var x10559 [1 << 17]byte + var x10560 [1 << 17]byte + var x10561 [1 << 17]byte + var x10562 [1 << 17]byte + var x10563 [1 << 17]byte + var x10564 [1 << 17]byte + var x10565 [1 << 17]byte + var x10566 [1 << 17]byte + var x10567 [1 << 17]byte + var x10568 [1 << 17]byte + var x10569 [1 << 17]byte + var x10570 [1 << 17]byte + var x10571 [1 << 17]byte + var x10572 [1 << 17]byte + var x10573 [1 << 17]byte + var x10574 [1 << 17]byte + var x10575 [1 << 17]byte + var x10576 [1 << 17]byte + var x10577 [1 << 17]byte + var x10578 [1 << 17]byte + var x10579 [1 << 17]byte + var x10580 [1 << 17]byte + var x10581 [1 << 17]byte + var x10582 [1 << 17]byte + var x10583 [1 << 17]byte + var x10584 [1 << 17]byte + var x10585 [1 << 17]byte + var x10586 [1 << 17]byte + var x10587 [1 << 17]byte + var x10588 [1 << 17]byte + var x10589 [1 << 17]byte + var x10590 [1 << 17]byte + var x10591 [1 << 17]byte + var x10592 [1 << 17]byte + var x10593 [1 << 17]byte + var x10594 [1 << 17]byte + var x10595 [1 << 17]byte + var x10596 [1 << 17]byte + var x10597 [1 << 17]byte + var x10598 [1 << 17]byte + var x10599 [1 << 17]byte + var x10600 [1 << 17]byte + var x10601 [1 << 17]byte + var x10602 [1 << 17]byte + var x10603 [1 << 17]byte + var x10604 [1 << 17]byte + var x10605 [1 << 17]byte + var x10606 [1 << 17]byte + var x10607 [1 << 17]byte + var x10608 [1 << 17]byte + var x10609 [1 << 17]byte + var x10610 [1 << 17]byte + var x10611 [1 << 17]byte + var x10612 [1 << 17]byte + var x10613 [1 << 17]byte + var x10614 [1 << 17]byte + var x10615 [1 << 17]byte + var x10616 [1 << 17]byte + var x10617 [1 << 17]byte + var x10618 [1 << 17]byte + var x10619 [1 << 17]byte + var x10620 [1 << 17]byte + var x10621 [1 << 17]byte + var x10622 [1 << 17]byte + var x10623 [1 << 17]byte + var x10624 [1 << 17]byte + var x10625 [1 << 17]byte + var x10626 [1 << 17]byte + var x10627 [1 << 17]byte + var x10628 [1 << 17]byte + var x10629 [1 << 17]byte + var x10630 [1 << 17]byte + var x10631 [1 << 17]byte + var x10632 [1 << 17]byte + var x10633 [1 << 17]byte + var x10634 [1 << 17]byte + var x10635 [1 << 17]byte + var x10636 [1 << 17]byte + var x10637 [1 << 17]byte + var x10638 [1 << 17]byte + var x10639 [1 << 17]byte + var x10640 [1 << 17]byte + var x10641 [1 << 17]byte + var x10642 [1 << 17]byte + var x10643 [1 << 17]byte + var x10644 [1 << 17]byte + var x10645 [1 << 17]byte + var x10646 [1 << 17]byte + var x10647 [1 << 17]byte + var x10648 [1 << 17]byte + var x10649 [1 << 17]byte + var x10650 [1 << 17]byte + var x10651 [1 << 17]byte + var x10652 [1 << 17]byte + var x10653 [1 << 17]byte + var x10654 [1 << 17]byte + var x10655 [1 << 17]byte + var x10656 [1 << 17]byte + var x10657 [1 << 17]byte + var x10658 [1 << 17]byte + var x10659 [1 << 17]byte + var x10660 [1 << 17]byte + var x10661 [1 << 17]byte + var x10662 [1 << 17]byte + var x10663 [1 << 17]byte + var x10664 [1 << 17]byte + var x10665 [1 << 17]byte + var x10666 [1 << 17]byte + var x10667 [1 << 17]byte + var x10668 [1 << 17]byte + var x10669 [1 << 17]byte + var x10670 [1 << 17]byte + var x10671 [1 << 17]byte + var x10672 [1 << 17]byte + var x10673 [1 << 17]byte + var x10674 [1 << 17]byte + var x10675 [1 << 17]byte + var x10676 [1 << 17]byte + var x10677 [1 << 17]byte + var x10678 [1 << 17]byte + var x10679 [1 << 17]byte + var x10680 [1 << 17]byte + var x10681 [1 << 17]byte + var x10682 [1 << 17]byte + var x10683 [1 << 17]byte + var x10684 [1 << 17]byte + var x10685 [1 << 17]byte + var x10686 [1 << 17]byte + var x10687 [1 << 17]byte + var x10688 [1 << 17]byte + var x10689 [1 << 17]byte + var x10690 [1 << 17]byte + var x10691 [1 << 17]byte + var x10692 [1 << 17]byte + var x10693 [1 << 17]byte + var x10694 [1 << 17]byte + var x10695 [1 << 17]byte + var x10696 [1 << 17]byte + var x10697 [1 << 17]byte + var x10698 [1 << 17]byte + var x10699 [1 << 17]byte + var x10700 [1 << 17]byte + var x10701 [1 << 17]byte + var x10702 [1 << 17]byte + var x10703 [1 << 17]byte + var x10704 [1 << 17]byte + var x10705 [1 << 17]byte + var x10706 [1 << 17]byte + var x10707 [1 << 17]byte + var x10708 [1 << 17]byte + var x10709 [1 << 17]byte + var x10710 [1 << 17]byte + var x10711 [1 << 17]byte + var x10712 [1 << 17]byte + var x10713 [1 << 17]byte + var x10714 [1 << 17]byte + var x10715 [1 << 17]byte + var x10716 [1 << 17]byte + var x10717 [1 << 17]byte + var x10718 [1 << 17]byte + var x10719 [1 << 17]byte + var x10720 [1 << 17]byte + var x10721 [1 << 17]byte + var x10722 [1 << 17]byte + var x10723 [1 << 17]byte + var x10724 [1 << 17]byte + var x10725 [1 << 17]byte + var x10726 [1 << 17]byte + var x10727 [1 << 17]byte + var x10728 [1 << 17]byte + var x10729 [1 << 17]byte + var x10730 [1 << 17]byte + var x10731 [1 << 17]byte + var x10732 [1 << 17]byte + var x10733 [1 << 17]byte + var x10734 [1 << 17]byte + var x10735 [1 << 17]byte + var x10736 [1 << 17]byte + var x10737 [1 << 17]byte + var x10738 [1 << 17]byte + var x10739 [1 << 17]byte + var x10740 [1 << 17]byte + var x10741 [1 << 17]byte + var x10742 [1 << 17]byte + var x10743 [1 << 17]byte + var x10744 [1 << 17]byte + var x10745 [1 << 17]byte + var x10746 [1 << 17]byte + var x10747 [1 << 17]byte + var x10748 [1 << 17]byte + var x10749 [1 << 17]byte + var x10750 [1 << 17]byte + var x10751 [1 << 17]byte + var x10752 [1 << 17]byte + var x10753 [1 << 17]byte + var x10754 [1 << 17]byte + var x10755 [1 << 17]byte + var x10756 [1 << 17]byte + var x10757 [1 << 17]byte + var x10758 [1 << 17]byte + var x10759 [1 << 17]byte + var x10760 [1 << 17]byte + var x10761 [1 << 17]byte + var x10762 [1 << 17]byte + var x10763 [1 << 17]byte + var x10764 [1 << 17]byte + var x10765 [1 << 17]byte + var x10766 [1 << 17]byte + var x10767 [1 << 17]byte + var x10768 [1 << 17]byte + var x10769 [1 << 17]byte + var x10770 [1 << 17]byte + var x10771 [1 << 17]byte + var x10772 [1 << 17]byte + var x10773 [1 << 17]byte + var x10774 [1 << 17]byte + var x10775 [1 << 17]byte + var x10776 [1 << 17]byte + var x10777 [1 << 17]byte + var x10778 [1 << 17]byte + var x10779 [1 << 17]byte + var x10780 [1 << 17]byte + var x10781 [1 << 17]byte + var x10782 [1 << 17]byte + var x10783 [1 << 17]byte + var x10784 [1 << 17]byte + var x10785 [1 << 17]byte + var x10786 [1 << 17]byte + var x10787 [1 << 17]byte + var x10788 [1 << 17]byte + var x10789 [1 << 17]byte + var x10790 [1 << 17]byte + var x10791 [1 << 17]byte + var x10792 [1 << 17]byte + var x10793 [1 << 17]byte + var x10794 [1 << 17]byte + var x10795 [1 << 17]byte + var x10796 [1 << 17]byte + var x10797 [1 << 17]byte + var x10798 [1 << 17]byte + var x10799 [1 << 17]byte + var x10800 [1 << 17]byte + var x10801 [1 << 17]byte + var x10802 [1 << 17]byte + var x10803 [1 << 17]byte + var x10804 [1 << 17]byte + var x10805 [1 << 17]byte + var x10806 [1 << 17]byte + var x10807 [1 << 17]byte + var x10808 [1 << 17]byte + var x10809 [1 << 17]byte + var x10810 [1 << 17]byte + var x10811 [1 << 17]byte + var x10812 [1 << 17]byte + var x10813 [1 << 17]byte + var x10814 [1 << 17]byte + var x10815 [1 << 17]byte + var x10816 [1 << 17]byte + var x10817 [1 << 17]byte + var x10818 [1 << 17]byte + var x10819 [1 << 17]byte + var x10820 [1 << 17]byte + var x10821 [1 << 17]byte + var x10822 [1 << 17]byte + var x10823 [1 << 17]byte + var x10824 [1 << 17]byte + var x10825 [1 << 17]byte + var x10826 [1 << 17]byte + var x10827 [1 << 17]byte + var x10828 [1 << 17]byte + var x10829 [1 << 17]byte + var x10830 [1 << 17]byte + var x10831 [1 << 17]byte + var x10832 [1 << 17]byte + var x10833 [1 << 17]byte + var x10834 [1 << 17]byte + var x10835 [1 << 17]byte + var x10836 [1 << 17]byte + var x10837 [1 << 17]byte + var x10838 [1 << 17]byte + var x10839 [1 << 17]byte + var x10840 [1 << 17]byte + var x10841 [1 << 17]byte + var x10842 [1 << 17]byte + var x10843 [1 << 17]byte + var x10844 [1 << 17]byte + var x10845 [1 << 17]byte + var x10846 [1 << 17]byte + var x10847 [1 << 17]byte + var x10848 [1 << 17]byte + var x10849 [1 << 17]byte + var x10850 [1 << 17]byte + var x10851 [1 << 17]byte + var x10852 [1 << 17]byte + var x10853 [1 << 17]byte + var x10854 [1 << 17]byte + var x10855 [1 << 17]byte + var x10856 [1 << 17]byte + var x10857 [1 << 17]byte + var x10858 [1 << 17]byte + var x10859 [1 << 17]byte + var x10860 [1 << 17]byte + var x10861 [1 << 17]byte + var x10862 [1 << 17]byte + var x10863 [1 << 17]byte + var x10864 [1 << 17]byte + var x10865 [1 << 17]byte + var x10866 [1 << 17]byte + var x10867 [1 << 17]byte + var x10868 [1 << 17]byte + var x10869 [1 << 17]byte + var x10870 [1 << 17]byte + var x10871 [1 << 17]byte + var x10872 [1 << 17]byte + var x10873 [1 << 17]byte + var x10874 [1 << 17]byte + var x10875 [1 << 17]byte + var x10876 [1 << 17]byte + var x10877 [1 << 17]byte + var x10878 [1 << 17]byte + var x10879 [1 << 17]byte + var x10880 [1 << 17]byte + var x10881 [1 << 17]byte + var x10882 [1 << 17]byte + var x10883 [1 << 17]byte + var x10884 [1 << 17]byte + var x10885 [1 << 17]byte + var x10886 [1 << 17]byte + var x10887 [1 << 17]byte + var x10888 [1 << 17]byte + var x10889 [1 << 17]byte + var x10890 [1 << 17]byte + var x10891 [1 << 17]byte + var x10892 [1 << 17]byte + var x10893 [1 << 17]byte + var x10894 [1 << 17]byte + var x10895 [1 << 17]byte + var x10896 [1 << 17]byte + var x10897 [1 << 17]byte + var x10898 [1 << 17]byte + var x10899 [1 << 17]byte + var x10900 [1 << 17]byte + var x10901 [1 << 17]byte + var x10902 [1 << 17]byte + var x10903 [1 << 17]byte + var x10904 [1 << 17]byte + var x10905 [1 << 17]byte + var x10906 [1 << 17]byte + var x10907 [1 << 17]byte + var x10908 [1 << 17]byte + var x10909 [1 << 17]byte + var x10910 [1 << 17]byte + var x10911 [1 << 17]byte + var x10912 [1 << 17]byte + var x10913 [1 << 17]byte + var x10914 [1 << 17]byte + var x10915 [1 << 17]byte + var x10916 [1 << 17]byte + var x10917 [1 << 17]byte + var x10918 [1 << 17]byte + var x10919 [1 << 17]byte + var x10920 [1 << 17]byte + var x10921 [1 << 17]byte + var x10922 [1 << 17]byte + var x10923 [1 << 17]byte + var x10924 [1 << 17]byte + var x10925 [1 << 17]byte + var x10926 [1 << 17]byte + var x10927 [1 << 17]byte + var x10928 [1 << 17]byte + var x10929 [1 << 17]byte + var x10930 [1 << 17]byte + var x10931 [1 << 17]byte + var x10932 [1 << 17]byte + var x10933 [1 << 17]byte + var x10934 [1 << 17]byte + var x10935 [1 << 17]byte + var x10936 [1 << 17]byte + var x10937 [1 << 17]byte + var x10938 [1 << 17]byte + var x10939 [1 << 17]byte + var x10940 [1 << 17]byte + var x10941 [1 << 17]byte + var x10942 [1 << 17]byte + var x10943 [1 << 17]byte + var x10944 [1 << 17]byte + var x10945 [1 << 17]byte + var x10946 [1 << 17]byte + var x10947 [1 << 17]byte + var x10948 [1 << 17]byte + var x10949 [1 << 17]byte + var x10950 [1 << 17]byte + var x10951 [1 << 17]byte + var x10952 [1 << 17]byte + var x10953 [1 << 17]byte + var x10954 [1 << 17]byte + var x10955 [1 << 17]byte + var x10956 [1 << 17]byte + var x10957 [1 << 17]byte + var x10958 [1 << 17]byte + var x10959 [1 << 17]byte + var x10960 [1 << 17]byte + var x10961 [1 << 17]byte + var x10962 [1 << 17]byte + var x10963 [1 << 17]byte + var x10964 [1 << 17]byte + var x10965 [1 << 17]byte + var x10966 [1 << 17]byte + var x10967 [1 << 17]byte + var x10968 [1 << 17]byte + var x10969 [1 << 17]byte + var x10970 [1 << 17]byte + var x10971 [1 << 17]byte + var x10972 [1 << 17]byte + var x10973 [1 << 17]byte + var x10974 [1 << 17]byte + var x10975 [1 << 17]byte + var x10976 [1 << 17]byte + var x10977 [1 << 17]byte + var x10978 [1 << 17]byte + var x10979 [1 << 17]byte + var x10980 [1 << 17]byte + var x10981 [1 << 17]byte + var x10982 [1 << 17]byte + var x10983 [1 << 17]byte + var x10984 [1 << 17]byte + var x10985 [1 << 17]byte + var x10986 [1 << 17]byte + var x10987 [1 << 17]byte + var x10988 [1 << 17]byte + var x10989 [1 << 17]byte + var x10990 [1 << 17]byte + var x10991 [1 << 17]byte + var x10992 [1 << 17]byte + var x10993 [1 << 17]byte + var x10994 [1 << 17]byte + var x10995 [1 << 17]byte + var x10996 [1 << 17]byte + var x10997 [1 << 17]byte + var x10998 [1 << 17]byte + var x10999 [1 << 17]byte + var x11000 [1 << 17]byte + var x11001 [1 << 17]byte + var x11002 [1 << 17]byte + var x11003 [1 << 17]byte + var x11004 [1 << 17]byte + var x11005 [1 << 17]byte + var x11006 [1 << 17]byte + var x11007 [1 << 17]byte + var x11008 [1 << 17]byte + var x11009 [1 << 17]byte + var x11010 [1 << 17]byte + var x11011 [1 << 17]byte + var x11012 [1 << 17]byte + var x11013 [1 << 17]byte + var x11014 [1 << 17]byte + var x11015 [1 << 17]byte + var x11016 [1 << 17]byte + var x11017 [1 << 17]byte + var x11018 [1 << 17]byte + var x11019 [1 << 17]byte + var x11020 [1 << 17]byte + var x11021 [1 << 17]byte + var x11022 [1 << 17]byte + var x11023 [1 << 17]byte + var x11024 [1 << 17]byte + var x11025 [1 << 17]byte + var x11026 [1 << 17]byte + var x11027 [1 << 17]byte + var x11028 [1 << 17]byte + var x11029 [1 << 17]byte + var x11030 [1 << 17]byte + var x11031 [1 << 17]byte + var x11032 [1 << 17]byte + var x11033 [1 << 17]byte + var x11034 [1 << 17]byte + var x11035 [1 << 17]byte + var x11036 [1 << 17]byte + var x11037 [1 << 17]byte + var x11038 [1 << 17]byte + var x11039 [1 << 17]byte + var x11040 [1 << 17]byte + var x11041 [1 << 17]byte + var x11042 [1 << 17]byte + var x11043 [1 << 17]byte + var x11044 [1 << 17]byte + var x11045 [1 << 17]byte + var x11046 [1 << 17]byte + var x11047 [1 << 17]byte + var x11048 [1 << 17]byte + var x11049 [1 << 17]byte + var x11050 [1 << 17]byte + var x11051 [1 << 17]byte + var x11052 [1 << 17]byte + var x11053 [1 << 17]byte + var x11054 [1 << 17]byte + var x11055 [1 << 17]byte + var x11056 [1 << 17]byte + var x11057 [1 << 17]byte + var x11058 [1 << 17]byte + var x11059 [1 << 17]byte + var x11060 [1 << 17]byte + var x11061 [1 << 17]byte + var x11062 [1 << 17]byte + var x11063 [1 << 17]byte + var x11064 [1 << 17]byte + var x11065 [1 << 17]byte + var x11066 [1 << 17]byte + var x11067 [1 << 17]byte + var x11068 [1 << 17]byte + var x11069 [1 << 17]byte + var x11070 [1 << 17]byte + var x11071 [1 << 17]byte + var x11072 [1 << 17]byte + var x11073 [1 << 17]byte + var x11074 [1 << 17]byte + var x11075 [1 << 17]byte + var x11076 [1 << 17]byte + var x11077 [1 << 17]byte + var x11078 [1 << 17]byte + var x11079 [1 << 17]byte + var x11080 [1 << 17]byte + var x11081 [1 << 17]byte + var x11082 [1 << 17]byte + var x11083 [1 << 17]byte + var x11084 [1 << 17]byte + var x11085 [1 << 17]byte + var x11086 [1 << 17]byte + var x11087 [1 << 17]byte + var x11088 [1 << 17]byte + var x11089 [1 << 17]byte + var x11090 [1 << 17]byte + var x11091 [1 << 17]byte + var x11092 [1 << 17]byte + var x11093 [1 << 17]byte + var x11094 [1 << 17]byte + var x11095 [1 << 17]byte + var x11096 [1 << 17]byte + var x11097 [1 << 17]byte + var x11098 [1 << 17]byte + var x11099 [1 << 17]byte + var x11100 [1 << 17]byte + var x11101 [1 << 17]byte + var x11102 [1 << 17]byte + var x11103 [1 << 17]byte + var x11104 [1 << 17]byte + var x11105 [1 << 17]byte + var x11106 [1 << 17]byte + var x11107 [1 << 17]byte + var x11108 [1 << 17]byte + var x11109 [1 << 17]byte + var x11110 [1 << 17]byte + var x11111 [1 << 17]byte + var x11112 [1 << 17]byte + var x11113 [1 << 17]byte + var x11114 [1 << 17]byte + var x11115 [1 << 17]byte + var x11116 [1 << 17]byte + var x11117 [1 << 17]byte + var x11118 [1 << 17]byte + var x11119 [1 << 17]byte + var x11120 [1 << 17]byte + var x11121 [1 << 17]byte + var x11122 [1 << 17]byte + var x11123 [1 << 17]byte + var x11124 [1 << 17]byte + var x11125 [1 << 17]byte + var x11126 [1 << 17]byte + var x11127 [1 << 17]byte + var x11128 [1 << 17]byte + var x11129 [1 << 17]byte + var x11130 [1 << 17]byte + var x11131 [1 << 17]byte + var x11132 [1 << 17]byte + var x11133 [1 << 17]byte + var x11134 [1 << 17]byte + var x11135 [1 << 17]byte + var x11136 [1 << 17]byte + var x11137 [1 << 17]byte + var x11138 [1 << 17]byte + var x11139 [1 << 17]byte + var x11140 [1 << 17]byte + var x11141 [1 << 17]byte + var x11142 [1 << 17]byte + var x11143 [1 << 17]byte + var x11144 [1 << 17]byte + var x11145 [1 << 17]byte + var x11146 [1 << 17]byte + var x11147 [1 << 17]byte + var x11148 [1 << 17]byte + var x11149 [1 << 17]byte + var x11150 [1 << 17]byte + var x11151 [1 << 17]byte + var x11152 [1 << 17]byte + var x11153 [1 << 17]byte + var x11154 [1 << 17]byte + var x11155 [1 << 17]byte + var x11156 [1 << 17]byte + var x11157 [1 << 17]byte + var x11158 [1 << 17]byte + var x11159 [1 << 17]byte + var x11160 [1 << 17]byte + var x11161 [1 << 17]byte + var x11162 [1 << 17]byte + var x11163 [1 << 17]byte + var x11164 [1 << 17]byte + var x11165 [1 << 17]byte + var x11166 [1 << 17]byte + var x11167 [1 << 17]byte + var x11168 [1 << 17]byte + var x11169 [1 << 17]byte + var x11170 [1 << 17]byte + var x11171 [1 << 17]byte + var x11172 [1 << 17]byte + var x11173 [1 << 17]byte + var x11174 [1 << 17]byte + var x11175 [1 << 17]byte + var x11176 [1 << 17]byte + var x11177 [1 << 17]byte + var x11178 [1 << 17]byte + var x11179 [1 << 17]byte + var x11180 [1 << 17]byte + var x11181 [1 << 17]byte + var x11182 [1 << 17]byte + var x11183 [1 << 17]byte + var x11184 [1 << 17]byte + var x11185 [1 << 17]byte + var x11186 [1 << 17]byte + var x11187 [1 << 17]byte + var x11188 [1 << 17]byte + var x11189 [1 << 17]byte + var x11190 [1 << 17]byte + var x11191 [1 << 17]byte + var x11192 [1 << 17]byte + var x11193 [1 << 17]byte + var x11194 [1 << 17]byte + var x11195 [1 << 17]byte + var x11196 [1 << 17]byte + var x11197 [1 << 17]byte + var x11198 [1 << 17]byte + var x11199 [1 << 17]byte + var x11200 [1 << 17]byte + var x11201 [1 << 17]byte + var x11202 [1 << 17]byte + var x11203 [1 << 17]byte + var x11204 [1 << 17]byte + var x11205 [1 << 17]byte + var x11206 [1 << 17]byte + var x11207 [1 << 17]byte + var x11208 [1 << 17]byte + var x11209 [1 << 17]byte + var x11210 [1 << 17]byte + var x11211 [1 << 17]byte + var x11212 [1 << 17]byte + var x11213 [1 << 17]byte + var x11214 [1 << 17]byte + var x11215 [1 << 17]byte + var x11216 [1 << 17]byte + var x11217 [1 << 17]byte + var x11218 [1 << 17]byte + var x11219 [1 << 17]byte + var x11220 [1 << 17]byte + var x11221 [1 << 17]byte + var x11222 [1 << 17]byte + var x11223 [1 << 17]byte + var x11224 [1 << 17]byte + var x11225 [1 << 17]byte + var x11226 [1 << 17]byte + var x11227 [1 << 17]byte + var x11228 [1 << 17]byte + var x11229 [1 << 17]byte + var x11230 [1 << 17]byte + var x11231 [1 << 17]byte + var x11232 [1 << 17]byte + var x11233 [1 << 17]byte + var x11234 [1 << 17]byte + var x11235 [1 << 17]byte + var x11236 [1 << 17]byte + var x11237 [1 << 17]byte + var x11238 [1 << 17]byte + var x11239 [1 << 17]byte + var x11240 [1 << 17]byte + var x11241 [1 << 17]byte + var x11242 [1 << 17]byte + var x11243 [1 << 17]byte + var x11244 [1 << 17]byte + var x11245 [1 << 17]byte + var x11246 [1 << 17]byte + var x11247 [1 << 17]byte + var x11248 [1 << 17]byte + var x11249 [1 << 17]byte + var x11250 [1 << 17]byte + var x11251 [1 << 17]byte + var x11252 [1 << 17]byte + var x11253 [1 << 17]byte + var x11254 [1 << 17]byte + var x11255 [1 << 17]byte + var x11256 [1 << 17]byte + var x11257 [1 << 17]byte + var x11258 [1 << 17]byte + var x11259 [1 << 17]byte + var x11260 [1 << 17]byte + var x11261 [1 << 17]byte + var x11262 [1 << 17]byte + var x11263 [1 << 17]byte + var x11264 [1 << 17]byte + var x11265 [1 << 17]byte + var x11266 [1 << 17]byte + var x11267 [1 << 17]byte + var x11268 [1 << 17]byte + var x11269 [1 << 17]byte + var x11270 [1 << 17]byte + var x11271 [1 << 17]byte + var x11272 [1 << 17]byte + var x11273 [1 << 17]byte + var x11274 [1 << 17]byte + var x11275 [1 << 17]byte + var x11276 [1 << 17]byte + var x11277 [1 << 17]byte + var x11278 [1 << 17]byte + var x11279 [1 << 17]byte + var x11280 [1 << 17]byte + var x11281 [1 << 17]byte + var x11282 [1 << 17]byte + var x11283 [1 << 17]byte + var x11284 [1 << 17]byte + var x11285 [1 << 17]byte + var x11286 [1 << 17]byte + var x11287 [1 << 17]byte + var x11288 [1 << 17]byte + var x11289 [1 << 17]byte + var x11290 [1 << 17]byte + var x11291 [1 << 17]byte + var x11292 [1 << 17]byte + var x11293 [1 << 17]byte + var x11294 [1 << 17]byte + var x11295 [1 << 17]byte + var x11296 [1 << 17]byte + var x11297 [1 << 17]byte + var x11298 [1 << 17]byte + var x11299 [1 << 17]byte + var x11300 [1 << 17]byte + var x11301 [1 << 17]byte + var x11302 [1 << 17]byte + var x11303 [1 << 17]byte + var x11304 [1 << 17]byte + var x11305 [1 << 17]byte + var x11306 [1 << 17]byte + var x11307 [1 << 17]byte + var x11308 [1 << 17]byte + var x11309 [1 << 17]byte + var x11310 [1 << 17]byte + var x11311 [1 << 17]byte + var x11312 [1 << 17]byte + var x11313 [1 << 17]byte + var x11314 [1 << 17]byte + var x11315 [1 << 17]byte + var x11316 [1 << 17]byte + var x11317 [1 << 17]byte + var x11318 [1 << 17]byte + var x11319 [1 << 17]byte + var x11320 [1 << 17]byte + var x11321 [1 << 17]byte + var x11322 [1 << 17]byte + var x11323 [1 << 17]byte + var x11324 [1 << 17]byte + var x11325 [1 << 17]byte + var x11326 [1 << 17]byte + var x11327 [1 << 17]byte + var x11328 [1 << 17]byte + var x11329 [1 << 17]byte + var x11330 [1 << 17]byte + var x11331 [1 << 17]byte + var x11332 [1 << 17]byte + var x11333 [1 << 17]byte + var x11334 [1 << 17]byte + var x11335 [1 << 17]byte + var x11336 [1 << 17]byte + var x11337 [1 << 17]byte + var x11338 [1 << 17]byte + var x11339 [1 << 17]byte + var x11340 [1 << 17]byte + var x11341 [1 << 17]byte + var x11342 [1 << 17]byte + var x11343 [1 << 17]byte + var x11344 [1 << 17]byte + var x11345 [1 << 17]byte + var x11346 [1 << 17]byte + var x11347 [1 << 17]byte + var x11348 [1 << 17]byte + var x11349 [1 << 17]byte + var x11350 [1 << 17]byte + var x11351 [1 << 17]byte + var x11352 [1 << 17]byte + var x11353 [1 << 17]byte + var x11354 [1 << 17]byte + var x11355 [1 << 17]byte + var x11356 [1 << 17]byte + var x11357 [1 << 17]byte + var x11358 [1 << 17]byte + var x11359 [1 << 17]byte + var x11360 [1 << 17]byte + var x11361 [1 << 17]byte + var x11362 [1 << 17]byte + var x11363 [1 << 17]byte + var x11364 [1 << 17]byte + var x11365 [1 << 17]byte + var x11366 [1 << 17]byte + var x11367 [1 << 17]byte + var x11368 [1 << 17]byte + var x11369 [1 << 17]byte + var x11370 [1 << 17]byte + var x11371 [1 << 17]byte + var x11372 [1 << 17]byte + var x11373 [1 << 17]byte + var x11374 [1 << 17]byte + var x11375 [1 << 17]byte + var x11376 [1 << 17]byte + var x11377 [1 << 17]byte + var x11378 [1 << 17]byte + var x11379 [1 << 17]byte + var x11380 [1 << 17]byte + var x11381 [1 << 17]byte + var x11382 [1 << 17]byte + var x11383 [1 << 17]byte + var x11384 [1 << 17]byte + var x11385 [1 << 17]byte + var x11386 [1 << 17]byte + var x11387 [1 << 17]byte + var x11388 [1 << 17]byte + var x11389 [1 << 17]byte + var x11390 [1 << 17]byte + var x11391 [1 << 17]byte + var x11392 [1 << 17]byte + var x11393 [1 << 17]byte + var x11394 [1 << 17]byte + var x11395 [1 << 17]byte + var x11396 [1 << 17]byte + var x11397 [1 << 17]byte + var x11398 [1 << 17]byte + var x11399 [1 << 17]byte + var x11400 [1 << 17]byte + var x11401 [1 << 17]byte + var x11402 [1 << 17]byte + var x11403 [1 << 17]byte + var x11404 [1 << 17]byte + var x11405 [1 << 17]byte + var x11406 [1 << 17]byte + var x11407 [1 << 17]byte + var x11408 [1 << 17]byte + var x11409 [1 << 17]byte + var x11410 [1 << 17]byte + var x11411 [1 << 17]byte + var x11412 [1 << 17]byte + var x11413 [1 << 17]byte + var x11414 [1 << 17]byte + var x11415 [1 << 17]byte + var x11416 [1 << 17]byte + var x11417 [1 << 17]byte + var x11418 [1 << 17]byte + var x11419 [1 << 17]byte + var x11420 [1 << 17]byte + var x11421 [1 << 17]byte + var x11422 [1 << 17]byte + var x11423 [1 << 17]byte + var x11424 [1 << 17]byte + var x11425 [1 << 17]byte + var x11426 [1 << 17]byte + var x11427 [1 << 17]byte + var x11428 [1 << 17]byte + var x11429 [1 << 17]byte + var x11430 [1 << 17]byte + var x11431 [1 << 17]byte + var x11432 [1 << 17]byte + var x11433 [1 << 17]byte + var x11434 [1 << 17]byte + var x11435 [1 << 17]byte + var x11436 [1 << 17]byte + var x11437 [1 << 17]byte + var x11438 [1 << 17]byte + var x11439 [1 << 17]byte + var x11440 [1 << 17]byte + var x11441 [1 << 17]byte + var x11442 [1 << 17]byte + var x11443 [1 << 17]byte + var x11444 [1 << 17]byte + var x11445 [1 << 17]byte + var x11446 [1 << 17]byte + var x11447 [1 << 17]byte + var x11448 [1 << 17]byte + var x11449 [1 << 17]byte + var x11450 [1 << 17]byte + var x11451 [1 << 17]byte + var x11452 [1 << 17]byte + var x11453 [1 << 17]byte + var x11454 [1 << 17]byte + var x11455 [1 << 17]byte + var x11456 [1 << 17]byte + var x11457 [1 << 17]byte + var x11458 [1 << 17]byte + var x11459 [1 << 17]byte + var x11460 [1 << 17]byte + var x11461 [1 << 17]byte + var x11462 [1 << 17]byte + var x11463 [1 << 17]byte + var x11464 [1 << 17]byte + var x11465 [1 << 17]byte + var x11466 [1 << 17]byte + var x11467 [1 << 17]byte + var x11468 [1 << 17]byte + var x11469 [1 << 17]byte + var x11470 [1 << 17]byte + var x11471 [1 << 17]byte + var x11472 [1 << 17]byte + var x11473 [1 << 17]byte + var x11474 [1 << 17]byte + var x11475 [1 << 17]byte + var x11476 [1 << 17]byte + var x11477 [1 << 17]byte + var x11478 [1 << 17]byte + var x11479 [1 << 17]byte + var x11480 [1 << 17]byte + var x11481 [1 << 17]byte + var x11482 [1 << 17]byte + var x11483 [1 << 17]byte + var x11484 [1 << 17]byte + var x11485 [1 << 17]byte + var x11486 [1 << 17]byte + var x11487 [1 << 17]byte + var x11488 [1 << 17]byte + var x11489 [1 << 17]byte + var x11490 [1 << 17]byte + var x11491 [1 << 17]byte + var x11492 [1 << 17]byte + var x11493 [1 << 17]byte + var x11494 [1 << 17]byte + var x11495 [1 << 17]byte + var x11496 [1 << 17]byte + var x11497 [1 << 17]byte + var x11498 [1 << 17]byte + var x11499 [1 << 17]byte + var x11500 [1 << 17]byte + var x11501 [1 << 17]byte + var x11502 [1 << 17]byte + var x11503 [1 << 17]byte + var x11504 [1 << 17]byte + var x11505 [1 << 17]byte + var x11506 [1 << 17]byte + var x11507 [1 << 17]byte + var x11508 [1 << 17]byte + var x11509 [1 << 17]byte + var x11510 [1 << 17]byte + var x11511 [1 << 17]byte + var x11512 [1 << 17]byte + var x11513 [1 << 17]byte + var x11514 [1 << 17]byte + var x11515 [1 << 17]byte + var x11516 [1 << 17]byte + var x11517 [1 << 17]byte + var x11518 [1 << 17]byte + var x11519 [1 << 17]byte + var x11520 [1 << 17]byte + var x11521 [1 << 17]byte + var x11522 [1 << 17]byte + var x11523 [1 << 17]byte + var x11524 [1 << 17]byte + var x11525 [1 << 17]byte + var x11526 [1 << 17]byte + var x11527 [1 << 17]byte + var x11528 [1 << 17]byte + var x11529 [1 << 17]byte + var x11530 [1 << 17]byte + var x11531 [1 << 17]byte + var x11532 [1 << 17]byte + var x11533 [1 << 17]byte + var x11534 [1 << 17]byte + var x11535 [1 << 17]byte + var x11536 [1 << 17]byte + var x11537 [1 << 17]byte + var x11538 [1 << 17]byte + var x11539 [1 << 17]byte + var x11540 [1 << 17]byte + var x11541 [1 << 17]byte + var x11542 [1 << 17]byte + var x11543 [1 << 17]byte + var x11544 [1 << 17]byte + var x11545 [1 << 17]byte + var x11546 [1 << 17]byte + var x11547 [1 << 17]byte + var x11548 [1 << 17]byte + var x11549 [1 << 17]byte + var x11550 [1 << 17]byte + var x11551 [1 << 17]byte + var x11552 [1 << 17]byte + var x11553 [1 << 17]byte + var x11554 [1 << 17]byte + var x11555 [1 << 17]byte + var x11556 [1 << 17]byte + var x11557 [1 << 17]byte + var x11558 [1 << 17]byte + var x11559 [1 << 17]byte + var x11560 [1 << 17]byte + var x11561 [1 << 17]byte + var x11562 [1 << 17]byte + var x11563 [1 << 17]byte + var x11564 [1 << 17]byte + var x11565 [1 << 17]byte + var x11566 [1 << 17]byte + var x11567 [1 << 17]byte + var x11568 [1 << 17]byte + var x11569 [1 << 17]byte + var x11570 [1 << 17]byte + var x11571 [1 << 17]byte + var x11572 [1 << 17]byte + var x11573 [1 << 17]byte + var x11574 [1 << 17]byte + var x11575 [1 << 17]byte + var x11576 [1 << 17]byte + var x11577 [1 << 17]byte + var x11578 [1 << 17]byte + var x11579 [1 << 17]byte + var x11580 [1 << 17]byte + var x11581 [1 << 17]byte + var x11582 [1 << 17]byte + var x11583 [1 << 17]byte + var x11584 [1 << 17]byte + var x11585 [1 << 17]byte + var x11586 [1 << 17]byte + var x11587 [1 << 17]byte + var x11588 [1 << 17]byte + var x11589 [1 << 17]byte + var x11590 [1 << 17]byte + var x11591 [1 << 17]byte + var x11592 [1 << 17]byte + var x11593 [1 << 17]byte + var x11594 [1 << 17]byte + var x11595 [1 << 17]byte + var x11596 [1 << 17]byte + var x11597 [1 << 17]byte + var x11598 [1 << 17]byte + var x11599 [1 << 17]byte + var x11600 [1 << 17]byte + var x11601 [1 << 17]byte + var x11602 [1 << 17]byte + var x11603 [1 << 17]byte + var x11604 [1 << 17]byte + var x11605 [1 << 17]byte + var x11606 [1 << 17]byte + var x11607 [1 << 17]byte + var x11608 [1 << 17]byte + var x11609 [1 << 17]byte + var x11610 [1 << 17]byte + var x11611 [1 << 17]byte + var x11612 [1 << 17]byte + var x11613 [1 << 17]byte + var x11614 [1 << 17]byte + var x11615 [1 << 17]byte + var x11616 [1 << 17]byte + var x11617 [1 << 17]byte + var x11618 [1 << 17]byte + var x11619 [1 << 17]byte + var x11620 [1 << 17]byte + var x11621 [1 << 17]byte + var x11622 [1 << 17]byte + var x11623 [1 << 17]byte + var x11624 [1 << 17]byte + var x11625 [1 << 17]byte + var x11626 [1 << 17]byte + var x11627 [1 << 17]byte + var x11628 [1 << 17]byte + var x11629 [1 << 17]byte + var x11630 [1 << 17]byte + var x11631 [1 << 17]byte + var x11632 [1 << 17]byte + var x11633 [1 << 17]byte + var x11634 [1 << 17]byte + var x11635 [1 << 17]byte + var x11636 [1 << 17]byte + var x11637 [1 << 17]byte + var x11638 [1 << 17]byte + var x11639 [1 << 17]byte + var x11640 [1 << 17]byte + var x11641 [1 << 17]byte + var x11642 [1 << 17]byte + var x11643 [1 << 17]byte + var x11644 [1 << 17]byte + var x11645 [1 << 17]byte + var x11646 [1 << 17]byte + var x11647 [1 << 17]byte + var x11648 [1 << 17]byte + var x11649 [1 << 17]byte + var x11650 [1 << 17]byte + var x11651 [1 << 17]byte + var x11652 [1 << 17]byte + var x11653 [1 << 17]byte + var x11654 [1 << 17]byte + var x11655 [1 << 17]byte + var x11656 [1 << 17]byte + var x11657 [1 << 17]byte + var x11658 [1 << 17]byte + var x11659 [1 << 17]byte + var x11660 [1 << 17]byte + var x11661 [1 << 17]byte + var x11662 [1 << 17]byte + var x11663 [1 << 17]byte + var x11664 [1 << 17]byte + var x11665 [1 << 17]byte + var x11666 [1 << 17]byte + var x11667 [1 << 17]byte + var x11668 [1 << 17]byte + var x11669 [1 << 17]byte + var x11670 [1 << 17]byte + var x11671 [1 << 17]byte + var x11672 [1 << 17]byte + var x11673 [1 << 17]byte + var x11674 [1 << 17]byte + var x11675 [1 << 17]byte + var x11676 [1 << 17]byte + var x11677 [1 << 17]byte + var x11678 [1 << 17]byte + var x11679 [1 << 17]byte + var x11680 [1 << 17]byte + var x11681 [1 << 17]byte + var x11682 [1 << 17]byte + var x11683 [1 << 17]byte + var x11684 [1 << 17]byte + var x11685 [1 << 17]byte + var x11686 [1 << 17]byte + var x11687 [1 << 17]byte + var x11688 [1 << 17]byte + var x11689 [1 << 17]byte + var x11690 [1 << 17]byte + var x11691 [1 << 17]byte + var x11692 [1 << 17]byte + var x11693 [1 << 17]byte + var x11694 [1 << 17]byte + var x11695 [1 << 17]byte + var x11696 [1 << 17]byte + var x11697 [1 << 17]byte + var x11698 [1 << 17]byte + var x11699 [1 << 17]byte + var x11700 [1 << 17]byte + var x11701 [1 << 17]byte + var x11702 [1 << 17]byte + var x11703 [1 << 17]byte + var x11704 [1 << 17]byte + var x11705 [1 << 17]byte + var x11706 [1 << 17]byte + var x11707 [1 << 17]byte + var x11708 [1 << 17]byte + var x11709 [1 << 17]byte + var x11710 [1 << 17]byte + var x11711 [1 << 17]byte + var x11712 [1 << 17]byte + var x11713 [1 << 17]byte + var x11714 [1 << 17]byte + var x11715 [1 << 17]byte + var x11716 [1 << 17]byte + var x11717 [1 << 17]byte + var x11718 [1 << 17]byte + var x11719 [1 << 17]byte + var x11720 [1 << 17]byte + var x11721 [1 << 17]byte + var x11722 [1 << 17]byte + var x11723 [1 << 17]byte + var x11724 [1 << 17]byte + var x11725 [1 << 17]byte + var x11726 [1 << 17]byte + var x11727 [1 << 17]byte + var x11728 [1 << 17]byte + var x11729 [1 << 17]byte + var x11730 [1 << 17]byte + var x11731 [1 << 17]byte + var x11732 [1 << 17]byte + var x11733 [1 << 17]byte + var x11734 [1 << 17]byte + var x11735 [1 << 17]byte + var x11736 [1 << 17]byte + var x11737 [1 << 17]byte + var x11738 [1 << 17]byte + var x11739 [1 << 17]byte + var x11740 [1 << 17]byte + var x11741 [1 << 17]byte + var x11742 [1 << 17]byte + var x11743 [1 << 17]byte + var x11744 [1 << 17]byte + var x11745 [1 << 17]byte + var x11746 [1 << 17]byte + var x11747 [1 << 17]byte + var x11748 [1 << 17]byte + var x11749 [1 << 17]byte + var x11750 [1 << 17]byte + var x11751 [1 << 17]byte + var x11752 [1 << 17]byte + var x11753 [1 << 17]byte + var x11754 [1 << 17]byte + var x11755 [1 << 17]byte + var x11756 [1 << 17]byte + var x11757 [1 << 17]byte + var x11758 [1 << 17]byte + var x11759 [1 << 17]byte + var x11760 [1 << 17]byte + var x11761 [1 << 17]byte + var x11762 [1 << 17]byte + var x11763 [1 << 17]byte + var x11764 [1 << 17]byte + var x11765 [1 << 17]byte + var x11766 [1 << 17]byte + var x11767 [1 << 17]byte + var x11768 [1 << 17]byte + var x11769 [1 << 17]byte + var x11770 [1 << 17]byte + var x11771 [1 << 17]byte + var x11772 [1 << 17]byte + var x11773 [1 << 17]byte + var x11774 [1 << 17]byte + var x11775 [1 << 17]byte + var x11776 [1 << 17]byte + var x11777 [1 << 17]byte + var x11778 [1 << 17]byte + var x11779 [1 << 17]byte + var x11780 [1 << 17]byte + var x11781 [1 << 17]byte + var x11782 [1 << 17]byte + var x11783 [1 << 17]byte + var x11784 [1 << 17]byte + var x11785 [1 << 17]byte + var x11786 [1 << 17]byte + var x11787 [1 << 17]byte + var x11788 [1 << 17]byte + var x11789 [1 << 17]byte + var x11790 [1 << 17]byte + var x11791 [1 << 17]byte + var x11792 [1 << 17]byte + var x11793 [1 << 17]byte + var x11794 [1 << 17]byte + var x11795 [1 << 17]byte + var x11796 [1 << 17]byte + var x11797 [1 << 17]byte + var x11798 [1 << 17]byte + var x11799 [1 << 17]byte + var x11800 [1 << 17]byte + var x11801 [1 << 17]byte + var x11802 [1 << 17]byte + var x11803 [1 << 17]byte + var x11804 [1 << 17]byte + var x11805 [1 << 17]byte + var x11806 [1 << 17]byte + var x11807 [1 << 17]byte + var x11808 [1 << 17]byte + var x11809 [1 << 17]byte + var x11810 [1 << 17]byte + var x11811 [1 << 17]byte + var x11812 [1 << 17]byte + var x11813 [1 << 17]byte + var x11814 [1 << 17]byte + var x11815 [1 << 17]byte + var x11816 [1 << 17]byte + var x11817 [1 << 17]byte + var x11818 [1 << 17]byte + var x11819 [1 << 17]byte + var x11820 [1 << 17]byte + var x11821 [1 << 17]byte + var x11822 [1 << 17]byte + var x11823 [1 << 17]byte + var x11824 [1 << 17]byte + var x11825 [1 << 17]byte + var x11826 [1 << 17]byte + var x11827 [1 << 17]byte + var x11828 [1 << 17]byte + var x11829 [1 << 17]byte + var x11830 [1 << 17]byte + var x11831 [1 << 17]byte + var x11832 [1 << 17]byte + var x11833 [1 << 17]byte + var x11834 [1 << 17]byte + var x11835 [1 << 17]byte + var x11836 [1 << 17]byte + var x11837 [1 << 17]byte + var x11838 [1 << 17]byte + var x11839 [1 << 17]byte + var x11840 [1 << 17]byte + var x11841 [1 << 17]byte + var x11842 [1 << 17]byte + var x11843 [1 << 17]byte + var x11844 [1 << 17]byte + var x11845 [1 << 17]byte + var x11846 [1 << 17]byte + var x11847 [1 << 17]byte + var x11848 [1 << 17]byte + var x11849 [1 << 17]byte + var x11850 [1 << 17]byte + var x11851 [1 << 17]byte + var x11852 [1 << 17]byte + var x11853 [1 << 17]byte + var x11854 [1 << 17]byte + var x11855 [1 << 17]byte + var x11856 [1 << 17]byte + var x11857 [1 << 17]byte + var x11858 [1 << 17]byte + var x11859 [1 << 17]byte + var x11860 [1 << 17]byte + var x11861 [1 << 17]byte + var x11862 [1 << 17]byte + var x11863 [1 << 17]byte + var x11864 [1 << 17]byte + var x11865 [1 << 17]byte + var x11866 [1 << 17]byte + var x11867 [1 << 17]byte + var x11868 [1 << 17]byte + var x11869 [1 << 17]byte + var x11870 [1 << 17]byte + var x11871 [1 << 17]byte + var x11872 [1 << 17]byte + var x11873 [1 << 17]byte + var x11874 [1 << 17]byte + var x11875 [1 << 17]byte + var x11876 [1 << 17]byte + var x11877 [1 << 17]byte + var x11878 [1 << 17]byte + var x11879 [1 << 17]byte + var x11880 [1 << 17]byte + var x11881 [1 << 17]byte + var x11882 [1 << 17]byte + var x11883 [1 << 17]byte + var x11884 [1 << 17]byte + var x11885 [1 << 17]byte + var x11886 [1 << 17]byte + var x11887 [1 << 17]byte + var x11888 [1 << 17]byte + var x11889 [1 << 17]byte + var x11890 [1 << 17]byte + var x11891 [1 << 17]byte + var x11892 [1 << 17]byte + var x11893 [1 << 17]byte + var x11894 [1 << 17]byte + var x11895 [1 << 17]byte + var x11896 [1 << 17]byte + var x11897 [1 << 17]byte + var x11898 [1 << 17]byte + var x11899 [1 << 17]byte + var x11900 [1 << 17]byte + var x11901 [1 << 17]byte + var x11902 [1 << 17]byte + var x11903 [1 << 17]byte + var x11904 [1 << 17]byte + var x11905 [1 << 17]byte + var x11906 [1 << 17]byte + var x11907 [1 << 17]byte + var x11908 [1 << 17]byte + var x11909 [1 << 17]byte + var x11910 [1 << 17]byte + var x11911 [1 << 17]byte + var x11912 [1 << 17]byte + var x11913 [1 << 17]byte + var x11914 [1 << 17]byte + var x11915 [1 << 17]byte + var x11916 [1 << 17]byte + var x11917 [1 << 17]byte + var x11918 [1 << 17]byte + var x11919 [1 << 17]byte + var x11920 [1 << 17]byte + var x11921 [1 << 17]byte + var x11922 [1 << 17]byte + var x11923 [1 << 17]byte + var x11924 [1 << 17]byte + var x11925 [1 << 17]byte + var x11926 [1 << 17]byte + var x11927 [1 << 17]byte + var x11928 [1 << 17]byte + var x11929 [1 << 17]byte + var x11930 [1 << 17]byte + var x11931 [1 << 17]byte + var x11932 [1 << 17]byte + var x11933 [1 << 17]byte + var x11934 [1 << 17]byte + var x11935 [1 << 17]byte + var x11936 [1 << 17]byte + var x11937 [1 << 17]byte + var x11938 [1 << 17]byte + var x11939 [1 << 17]byte + var x11940 [1 << 17]byte + var x11941 [1 << 17]byte + var x11942 [1 << 17]byte + var x11943 [1 << 17]byte + var x11944 [1 << 17]byte + var x11945 [1 << 17]byte + var x11946 [1 << 17]byte + var x11947 [1 << 17]byte + var x11948 [1 << 17]byte + var x11949 [1 << 17]byte + var x11950 [1 << 17]byte + var x11951 [1 << 17]byte + var x11952 [1 << 17]byte + var x11953 [1 << 17]byte + var x11954 [1 << 17]byte + var x11955 [1 << 17]byte + var x11956 [1 << 17]byte + var x11957 [1 << 17]byte + var x11958 [1 << 17]byte + var x11959 [1 << 17]byte + var x11960 [1 << 17]byte + var x11961 [1 << 17]byte + var x11962 [1 << 17]byte + var x11963 [1 << 17]byte + var x11964 [1 << 17]byte + var x11965 [1 << 17]byte + var x11966 [1 << 17]byte + var x11967 [1 << 17]byte + var x11968 [1 << 17]byte + var x11969 [1 << 17]byte + var x11970 [1 << 17]byte + var x11971 [1 << 17]byte + var x11972 [1 << 17]byte + var x11973 [1 << 17]byte + var x11974 [1 << 17]byte + var x11975 [1 << 17]byte + var x11976 [1 << 17]byte + var x11977 [1 << 17]byte + var x11978 [1 << 17]byte + var x11979 [1 << 17]byte + var x11980 [1 << 17]byte + var x11981 [1 << 17]byte + var x11982 [1 << 17]byte + var x11983 [1 << 17]byte + var x11984 [1 << 17]byte + var x11985 [1 << 17]byte + var x11986 [1 << 17]byte + var x11987 [1 << 17]byte + var x11988 [1 << 17]byte + var x11989 [1 << 17]byte + var x11990 [1 << 17]byte + var x11991 [1 << 17]byte + var x11992 [1 << 17]byte + var x11993 [1 << 17]byte + var x11994 [1 << 17]byte + var x11995 [1 << 17]byte + var x11996 [1 << 17]byte + var x11997 [1 << 17]byte + var x11998 [1 << 17]byte + var x11999 [1 << 17]byte + var x12000 [1 << 17]byte + var x12001 [1 << 17]byte + var x12002 [1 << 17]byte + var x12003 [1 << 17]byte + var x12004 [1 << 17]byte + var x12005 [1 << 17]byte + var x12006 [1 << 17]byte + var x12007 [1 << 17]byte + var x12008 [1 << 17]byte + var x12009 [1 << 17]byte + var x12010 [1 << 17]byte + var x12011 [1 << 17]byte + var x12012 [1 << 17]byte + var x12013 [1 << 17]byte + var x12014 [1 << 17]byte + var x12015 [1 << 17]byte + var x12016 [1 << 17]byte + var x12017 [1 << 17]byte + var x12018 [1 << 17]byte + var x12019 [1 << 17]byte + var x12020 [1 << 17]byte + var x12021 [1 << 17]byte + var x12022 [1 << 17]byte + var x12023 [1 << 17]byte + var x12024 [1 << 17]byte + var x12025 [1 << 17]byte + var x12026 [1 << 17]byte + var x12027 [1 << 17]byte + var x12028 [1 << 17]byte + var x12029 [1 << 17]byte + var x12030 [1 << 17]byte + var x12031 [1 << 17]byte + var x12032 [1 << 17]byte + var x12033 [1 << 17]byte + var x12034 [1 << 17]byte + var x12035 [1 << 17]byte + var x12036 [1 << 17]byte + var x12037 [1 << 17]byte + var x12038 [1 << 17]byte + var x12039 [1 << 17]byte + var x12040 [1 << 17]byte + var x12041 [1 << 17]byte + var x12042 [1 << 17]byte + var x12043 [1 << 17]byte + var x12044 [1 << 17]byte + var x12045 [1 << 17]byte + var x12046 [1 << 17]byte + var x12047 [1 << 17]byte + var x12048 [1 << 17]byte + var x12049 [1 << 17]byte + var x12050 [1 << 17]byte + var x12051 [1 << 17]byte + var x12052 [1 << 17]byte + var x12053 [1 << 17]byte + var x12054 [1 << 17]byte + var x12055 [1 << 17]byte + var x12056 [1 << 17]byte + var x12057 [1 << 17]byte + var x12058 [1 << 17]byte + var x12059 [1 << 17]byte + var x12060 [1 << 17]byte + var x12061 [1 << 17]byte + var x12062 [1 << 17]byte + var x12063 [1 << 17]byte + var x12064 [1 << 17]byte + var x12065 [1 << 17]byte + var x12066 [1 << 17]byte + var x12067 [1 << 17]byte + var x12068 [1 << 17]byte + var x12069 [1 << 17]byte + var x12070 [1 << 17]byte + var x12071 [1 << 17]byte + var x12072 [1 << 17]byte + var x12073 [1 << 17]byte + var x12074 [1 << 17]byte + var x12075 [1 << 17]byte + var x12076 [1 << 17]byte + var x12077 [1 << 17]byte + var x12078 [1 << 17]byte + var x12079 [1 << 17]byte + var x12080 [1 << 17]byte + var x12081 [1 << 17]byte + var x12082 [1 << 17]byte + var x12083 [1 << 17]byte + var x12084 [1 << 17]byte + var x12085 [1 << 17]byte + var x12086 [1 << 17]byte + var x12087 [1 << 17]byte + var x12088 [1 << 17]byte + var x12089 [1 << 17]byte + var x12090 [1 << 17]byte + var x12091 [1 << 17]byte + var x12092 [1 << 17]byte + var x12093 [1 << 17]byte + var x12094 [1 << 17]byte + var x12095 [1 << 17]byte + var x12096 [1 << 17]byte + var x12097 [1 << 17]byte + var x12098 [1 << 17]byte + var x12099 [1 << 17]byte + var x12100 [1 << 17]byte + var x12101 [1 << 17]byte + var x12102 [1 << 17]byte + var x12103 [1 << 17]byte + var x12104 [1 << 17]byte + var x12105 [1 << 17]byte + var x12106 [1 << 17]byte + var x12107 [1 << 17]byte + var x12108 [1 << 17]byte + var x12109 [1 << 17]byte + var x12110 [1 << 17]byte + var x12111 [1 << 17]byte + var x12112 [1 << 17]byte + var x12113 [1 << 17]byte + var x12114 [1 << 17]byte + var x12115 [1 << 17]byte + var x12116 [1 << 17]byte + var x12117 [1 << 17]byte + var x12118 [1 << 17]byte + var x12119 [1 << 17]byte + var x12120 [1 << 17]byte + var x12121 [1 << 17]byte + var x12122 [1 << 17]byte + var x12123 [1 << 17]byte + var x12124 [1 << 17]byte + var x12125 [1 << 17]byte + var x12126 [1 << 17]byte + var x12127 [1 << 17]byte + var x12128 [1 << 17]byte + var x12129 [1 << 17]byte + var x12130 [1 << 17]byte + var x12131 [1 << 17]byte + var x12132 [1 << 17]byte + var x12133 [1 << 17]byte + var x12134 [1 << 17]byte + var x12135 [1 << 17]byte + var x12136 [1 << 17]byte + var x12137 [1 << 17]byte + var x12138 [1 << 17]byte + var x12139 [1 << 17]byte + var x12140 [1 << 17]byte + var x12141 [1 << 17]byte + var x12142 [1 << 17]byte + var x12143 [1 << 17]byte + var x12144 [1 << 17]byte + var x12145 [1 << 17]byte + var x12146 [1 << 17]byte + var x12147 [1 << 17]byte + var x12148 [1 << 17]byte + var x12149 [1 << 17]byte + var x12150 [1 << 17]byte + var x12151 [1 << 17]byte + var x12152 [1 << 17]byte + var x12153 [1 << 17]byte + var x12154 [1 << 17]byte + var x12155 [1 << 17]byte + var x12156 [1 << 17]byte + var x12157 [1 << 17]byte + var x12158 [1 << 17]byte + var x12159 [1 << 17]byte + var x12160 [1 << 17]byte + var x12161 [1 << 17]byte + var x12162 [1 << 17]byte + var x12163 [1 << 17]byte + var x12164 [1 << 17]byte + var x12165 [1 << 17]byte + var x12166 [1 << 17]byte + var x12167 [1 << 17]byte + var x12168 [1 << 17]byte + var x12169 [1 << 17]byte + var x12170 [1 << 17]byte + var x12171 [1 << 17]byte + var x12172 [1 << 17]byte + var x12173 [1 << 17]byte + var x12174 [1 << 17]byte + var x12175 [1 << 17]byte + var x12176 [1 << 17]byte + var x12177 [1 << 17]byte + var x12178 [1 << 17]byte + var x12179 [1 << 17]byte + var x12180 [1 << 17]byte + var x12181 [1 << 17]byte + var x12182 [1 << 17]byte + var x12183 [1 << 17]byte + var x12184 [1 << 17]byte + var x12185 [1 << 17]byte + var x12186 [1 << 17]byte + var x12187 [1 << 17]byte + var x12188 [1 << 17]byte + var x12189 [1 << 17]byte + var x12190 [1 << 17]byte + var x12191 [1 << 17]byte + var x12192 [1 << 17]byte + var x12193 [1 << 17]byte + var x12194 [1 << 17]byte + var x12195 [1 << 17]byte + var x12196 [1 << 17]byte + var x12197 [1 << 17]byte + var x12198 [1 << 17]byte + var x12199 [1 << 17]byte + var x12200 [1 << 17]byte + var x12201 [1 << 17]byte + var x12202 [1 << 17]byte + var x12203 [1 << 17]byte + var x12204 [1 << 17]byte + var x12205 [1 << 17]byte + var x12206 [1 << 17]byte + var x12207 [1 << 17]byte + var x12208 [1 << 17]byte + var x12209 [1 << 17]byte + var x12210 [1 << 17]byte + var x12211 [1 << 17]byte + var x12212 [1 << 17]byte + var x12213 [1 << 17]byte + var x12214 [1 << 17]byte + var x12215 [1 << 17]byte + var x12216 [1 << 17]byte + var x12217 [1 << 17]byte + var x12218 [1 << 17]byte + var x12219 [1 << 17]byte + var x12220 [1 << 17]byte + var x12221 [1 << 17]byte + var x12222 [1 << 17]byte + var x12223 [1 << 17]byte + var x12224 [1 << 17]byte + var x12225 [1 << 17]byte + var x12226 [1 << 17]byte + var x12227 [1 << 17]byte + var x12228 [1 << 17]byte + var x12229 [1 << 17]byte + var x12230 [1 << 17]byte + var x12231 [1 << 17]byte + var x12232 [1 << 17]byte + var x12233 [1 << 17]byte + var x12234 [1 << 17]byte + var x12235 [1 << 17]byte + var x12236 [1 << 17]byte + var x12237 [1 << 17]byte + var x12238 [1 << 17]byte + var x12239 [1 << 17]byte + var x12240 [1 << 17]byte + var x12241 [1 << 17]byte + var x12242 [1 << 17]byte + var x12243 [1 << 17]byte + var x12244 [1 << 17]byte + var x12245 [1 << 17]byte + var x12246 [1 << 17]byte + var x12247 [1 << 17]byte + var x12248 [1 << 17]byte + var x12249 [1 << 17]byte + var x12250 [1 << 17]byte + var x12251 [1 << 17]byte + var x12252 [1 << 17]byte + var x12253 [1 << 17]byte + var x12254 [1 << 17]byte + var x12255 [1 << 17]byte + var x12256 [1 << 17]byte + var x12257 [1 << 17]byte + var x12258 [1 << 17]byte + var x12259 [1 << 17]byte + var x12260 [1 << 17]byte + var x12261 [1 << 17]byte + var x12262 [1 << 17]byte + var x12263 [1 << 17]byte + var x12264 [1 << 17]byte + var x12265 [1 << 17]byte + var x12266 [1 << 17]byte + var x12267 [1 << 17]byte + var x12268 [1 << 17]byte + var x12269 [1 << 17]byte + var x12270 [1 << 17]byte + var x12271 [1 << 17]byte + var x12272 [1 << 17]byte + var x12273 [1 << 17]byte + var x12274 [1 << 17]byte + var x12275 [1 << 17]byte + var x12276 [1 << 17]byte + var x12277 [1 << 17]byte + var x12278 [1 << 17]byte + var x12279 [1 << 17]byte + var x12280 [1 << 17]byte + var x12281 [1 << 17]byte + var x12282 [1 << 17]byte + var x12283 [1 << 17]byte + var x12284 [1 << 17]byte + var x12285 [1 << 17]byte + var x12286 [1 << 17]byte + var x12287 [1 << 17]byte + var x12288 [1 << 17]byte + var x12289 [1 << 17]byte + var x12290 [1 << 17]byte + var x12291 [1 << 17]byte + var x12292 [1 << 17]byte + var x12293 [1 << 17]byte + var x12294 [1 << 17]byte + var x12295 [1 << 17]byte + var x12296 [1 << 17]byte + var x12297 [1 << 17]byte + var x12298 [1 << 17]byte + var x12299 [1 << 17]byte + var x12300 [1 << 17]byte + var x12301 [1 << 17]byte + var x12302 [1 << 17]byte + var x12303 [1 << 17]byte + var x12304 [1 << 17]byte + var x12305 [1 << 17]byte + var x12306 [1 << 17]byte + var x12307 [1 << 17]byte + var x12308 [1 << 17]byte + var x12309 [1 << 17]byte + var x12310 [1 << 17]byte + var x12311 [1 << 17]byte + var x12312 [1 << 17]byte + var x12313 [1 << 17]byte + var x12314 [1 << 17]byte + var x12315 [1 << 17]byte + var x12316 [1 << 17]byte + var x12317 [1 << 17]byte + var x12318 [1 << 17]byte + var x12319 [1 << 17]byte + var x12320 [1 << 17]byte + var x12321 [1 << 17]byte + var x12322 [1 << 17]byte + var x12323 [1 << 17]byte + var x12324 [1 << 17]byte + var x12325 [1 << 17]byte + var x12326 [1 << 17]byte + var x12327 [1 << 17]byte + var x12328 [1 << 17]byte + var x12329 [1 << 17]byte + var x12330 [1 << 17]byte + var x12331 [1 << 17]byte + var x12332 [1 << 17]byte + var x12333 [1 << 17]byte + var x12334 [1 << 17]byte + var x12335 [1 << 17]byte + var x12336 [1 << 17]byte + var x12337 [1 << 17]byte + var x12338 [1 << 17]byte + var x12339 [1 << 17]byte + var x12340 [1 << 17]byte + var x12341 [1 << 17]byte + var x12342 [1 << 17]byte + var x12343 [1 << 17]byte + var x12344 [1 << 17]byte + var x12345 [1 << 17]byte + var x12346 [1 << 17]byte + var x12347 [1 << 17]byte + var x12348 [1 << 17]byte + var x12349 [1 << 17]byte + var x12350 [1 << 17]byte + var x12351 [1 << 17]byte + var x12352 [1 << 17]byte + var x12353 [1 << 17]byte + var x12354 [1 << 17]byte + var x12355 [1 << 17]byte + var x12356 [1 << 17]byte + var x12357 [1 << 17]byte + var x12358 [1 << 17]byte + var x12359 [1 << 17]byte + var x12360 [1 << 17]byte + var x12361 [1 << 17]byte + var x12362 [1 << 17]byte + var x12363 [1 << 17]byte + var x12364 [1 << 17]byte + var x12365 [1 << 17]byte + var x12366 [1 << 17]byte + var x12367 [1 << 17]byte + var x12368 [1 << 17]byte + var x12369 [1 << 17]byte + var x12370 [1 << 17]byte + var x12371 [1 << 17]byte + var x12372 [1 << 17]byte + var x12373 [1 << 17]byte + var x12374 [1 << 17]byte + var x12375 [1 << 17]byte + var x12376 [1 << 17]byte + var x12377 [1 << 17]byte + var x12378 [1 << 17]byte + var x12379 [1 << 17]byte + var x12380 [1 << 17]byte + var x12381 [1 << 17]byte + var x12382 [1 << 17]byte + var x12383 [1 << 17]byte + var x12384 [1 << 17]byte + var x12385 [1 << 17]byte + var x12386 [1 << 17]byte + var x12387 [1 << 17]byte + var x12388 [1 << 17]byte + var x12389 [1 << 17]byte + var x12390 [1 << 17]byte + var x12391 [1 << 17]byte + var x12392 [1 << 17]byte + var x12393 [1 << 17]byte + var x12394 [1 << 17]byte + var x12395 [1 << 17]byte + var x12396 [1 << 17]byte + var x12397 [1 << 17]byte + var x12398 [1 << 17]byte + var x12399 [1 << 17]byte + var x12400 [1 << 17]byte + var x12401 [1 << 17]byte + var x12402 [1 << 17]byte + var x12403 [1 << 17]byte + var x12404 [1 << 17]byte + var x12405 [1 << 17]byte + var x12406 [1 << 17]byte + var x12407 [1 << 17]byte + var x12408 [1 << 17]byte + var x12409 [1 << 17]byte + var x12410 [1 << 17]byte + var x12411 [1 << 17]byte + var x12412 [1 << 17]byte + var x12413 [1 << 17]byte + var x12414 [1 << 17]byte + var x12415 [1 << 17]byte + var x12416 [1 << 17]byte + var x12417 [1 << 17]byte + var x12418 [1 << 17]byte + var x12419 [1 << 17]byte + var x12420 [1 << 17]byte + var x12421 [1 << 17]byte + var x12422 [1 << 17]byte + var x12423 [1 << 17]byte + var x12424 [1 << 17]byte + var x12425 [1 << 17]byte + var x12426 [1 << 17]byte + var x12427 [1 << 17]byte + var x12428 [1 << 17]byte + var x12429 [1 << 17]byte + var x12430 [1 << 17]byte + var x12431 [1 << 17]byte + var x12432 [1 << 17]byte + var x12433 [1 << 17]byte + var x12434 [1 << 17]byte + var x12435 [1 << 17]byte + var x12436 [1 << 17]byte + var x12437 [1 << 17]byte + var x12438 [1 << 17]byte + var x12439 [1 << 17]byte + var x12440 [1 << 17]byte + var x12441 [1 << 17]byte + var x12442 [1 << 17]byte + var x12443 [1 << 17]byte + var x12444 [1 << 17]byte + var x12445 [1 << 17]byte + var x12446 [1 << 17]byte + var x12447 [1 << 17]byte + var x12448 [1 << 17]byte + var x12449 [1 << 17]byte + var x12450 [1 << 17]byte + var x12451 [1 << 17]byte + var x12452 [1 << 17]byte + var x12453 [1 << 17]byte + var x12454 [1 << 17]byte + var x12455 [1 << 17]byte + var x12456 [1 << 17]byte + var x12457 [1 << 17]byte + var x12458 [1 << 17]byte + var x12459 [1 << 17]byte + var x12460 [1 << 17]byte + var x12461 [1 << 17]byte + var x12462 [1 << 17]byte + var x12463 [1 << 17]byte + var x12464 [1 << 17]byte + var x12465 [1 << 17]byte + var x12466 [1 << 17]byte + var x12467 [1 << 17]byte + var x12468 [1 << 17]byte + var x12469 [1 << 17]byte + var x12470 [1 << 17]byte + var x12471 [1 << 17]byte + var x12472 [1 << 17]byte + var x12473 [1 << 17]byte + var x12474 [1 << 17]byte + var x12475 [1 << 17]byte + var x12476 [1 << 17]byte + var x12477 [1 << 17]byte + var x12478 [1 << 17]byte + var x12479 [1 << 17]byte + var x12480 [1 << 17]byte + var x12481 [1 << 17]byte + var x12482 [1 << 17]byte + var x12483 [1 << 17]byte + var x12484 [1 << 17]byte + var x12485 [1 << 17]byte + var x12486 [1 << 17]byte + var x12487 [1 << 17]byte + var x12488 [1 << 17]byte + var x12489 [1 << 17]byte + var x12490 [1 << 17]byte + var x12491 [1 << 17]byte + var x12492 [1 << 17]byte + var x12493 [1 << 17]byte + var x12494 [1 << 17]byte + var x12495 [1 << 17]byte + var x12496 [1 << 17]byte + var x12497 [1 << 17]byte + var x12498 [1 << 17]byte + var x12499 [1 << 17]byte + var x12500 [1 << 17]byte + var x12501 [1 << 17]byte + var x12502 [1 << 17]byte + var x12503 [1 << 17]byte + var x12504 [1 << 17]byte + var x12505 [1 << 17]byte + var x12506 [1 << 17]byte + var x12507 [1 << 17]byte + var x12508 [1 << 17]byte + var x12509 [1 << 17]byte + var x12510 [1 << 17]byte + var x12511 [1 << 17]byte + var x12512 [1 << 17]byte + var x12513 [1 << 17]byte + var x12514 [1 << 17]byte + var x12515 [1 << 17]byte + var x12516 [1 << 17]byte + var x12517 [1 << 17]byte + var x12518 [1 << 17]byte + var x12519 [1 << 17]byte + var x12520 [1 << 17]byte + var x12521 [1 << 17]byte + var x12522 [1 << 17]byte + var x12523 [1 << 17]byte + var x12524 [1 << 17]byte + var x12525 [1 << 17]byte + var x12526 [1 << 17]byte + var x12527 [1 << 17]byte + var x12528 [1 << 17]byte + var x12529 [1 << 17]byte + var x12530 [1 << 17]byte + var x12531 [1 << 17]byte + var x12532 [1 << 17]byte + var x12533 [1 << 17]byte + var x12534 [1 << 17]byte + var x12535 [1 << 17]byte + var x12536 [1 << 17]byte + var x12537 [1 << 17]byte + var x12538 [1 << 17]byte + var x12539 [1 << 17]byte + var x12540 [1 << 17]byte + var x12541 [1 << 17]byte + var x12542 [1 << 17]byte + var x12543 [1 << 17]byte + var x12544 [1 << 17]byte + var x12545 [1 << 17]byte + var x12546 [1 << 17]byte + var x12547 [1 << 17]byte + var x12548 [1 << 17]byte + var x12549 [1 << 17]byte + var x12550 [1 << 17]byte + var x12551 [1 << 17]byte + var x12552 [1 << 17]byte + var x12553 [1 << 17]byte + var x12554 [1 << 17]byte + var x12555 [1 << 17]byte + var x12556 [1 << 17]byte + var x12557 [1 << 17]byte + var x12558 [1 << 17]byte + var x12559 [1 << 17]byte + var x12560 [1 << 17]byte + var x12561 [1 << 17]byte + var x12562 [1 << 17]byte + var x12563 [1 << 17]byte + var x12564 [1 << 17]byte + var x12565 [1 << 17]byte + var x12566 [1 << 17]byte + var x12567 [1 << 17]byte + var x12568 [1 << 17]byte + var x12569 [1 << 17]byte + var x12570 [1 << 17]byte + var x12571 [1 << 17]byte + var x12572 [1 << 17]byte + var x12573 [1 << 17]byte + var x12574 [1 << 17]byte + var x12575 [1 << 17]byte + var x12576 [1 << 17]byte + var x12577 [1 << 17]byte + var x12578 [1 << 17]byte + var x12579 [1 << 17]byte + var x12580 [1 << 17]byte + var x12581 [1 << 17]byte + var x12582 [1 << 17]byte + var x12583 [1 << 17]byte + var x12584 [1 << 17]byte + var x12585 [1 << 17]byte + var x12586 [1 << 17]byte + var x12587 [1 << 17]byte + var x12588 [1 << 17]byte + var x12589 [1 << 17]byte + var x12590 [1 << 17]byte + var x12591 [1 << 17]byte + var x12592 [1 << 17]byte + var x12593 [1 << 17]byte + var x12594 [1 << 17]byte + var x12595 [1 << 17]byte + var x12596 [1 << 17]byte + var x12597 [1 << 17]byte + var x12598 [1 << 17]byte + var x12599 [1 << 17]byte + var x12600 [1 << 17]byte + var x12601 [1 << 17]byte + var x12602 [1 << 17]byte + var x12603 [1 << 17]byte + var x12604 [1 << 17]byte + var x12605 [1 << 17]byte + var x12606 [1 << 17]byte + var x12607 [1 << 17]byte + var x12608 [1 << 17]byte + var x12609 [1 << 17]byte + var x12610 [1 << 17]byte + var x12611 [1 << 17]byte + var x12612 [1 << 17]byte + var x12613 [1 << 17]byte + var x12614 [1 << 17]byte + var x12615 [1 << 17]byte + var x12616 [1 << 17]byte + var x12617 [1 << 17]byte + var x12618 [1 << 17]byte + var x12619 [1 << 17]byte + var x12620 [1 << 17]byte + var x12621 [1 << 17]byte + var x12622 [1 << 17]byte + var x12623 [1 << 17]byte + var x12624 [1 << 17]byte + var x12625 [1 << 17]byte + var x12626 [1 << 17]byte + var x12627 [1 << 17]byte + var x12628 [1 << 17]byte + var x12629 [1 << 17]byte + var x12630 [1 << 17]byte + var x12631 [1 << 17]byte + var x12632 [1 << 17]byte + var x12633 [1 << 17]byte + var x12634 [1 << 17]byte + var x12635 [1 << 17]byte + var x12636 [1 << 17]byte + var x12637 [1 << 17]byte + var x12638 [1 << 17]byte + var x12639 [1 << 17]byte + var x12640 [1 << 17]byte + var x12641 [1 << 17]byte + var x12642 [1 << 17]byte + var x12643 [1 << 17]byte + var x12644 [1 << 17]byte + var x12645 [1 << 17]byte + var x12646 [1 << 17]byte + var x12647 [1 << 17]byte + var x12648 [1 << 17]byte + var x12649 [1 << 17]byte + var x12650 [1 << 17]byte + var x12651 [1 << 17]byte + var x12652 [1 << 17]byte + var x12653 [1 << 17]byte + var x12654 [1 << 17]byte + var x12655 [1 << 17]byte + var x12656 [1 << 17]byte + var x12657 [1 << 17]byte + var x12658 [1 << 17]byte + var x12659 [1 << 17]byte + var x12660 [1 << 17]byte + var x12661 [1 << 17]byte + var x12662 [1 << 17]byte + var x12663 [1 << 17]byte + var x12664 [1 << 17]byte + var x12665 [1 << 17]byte + var x12666 [1 << 17]byte + var x12667 [1 << 17]byte + var x12668 [1 << 17]byte + var x12669 [1 << 17]byte + var x12670 [1 << 17]byte + var x12671 [1 << 17]byte + var x12672 [1 << 17]byte + var x12673 [1 << 17]byte + var x12674 [1 << 17]byte + var x12675 [1 << 17]byte + var x12676 [1 << 17]byte + var x12677 [1 << 17]byte + var x12678 [1 << 17]byte + var x12679 [1 << 17]byte + var x12680 [1 << 17]byte + var x12681 [1 << 17]byte + var x12682 [1 << 17]byte + var x12683 [1 << 17]byte + var x12684 [1 << 17]byte + var x12685 [1 << 17]byte + var x12686 [1 << 17]byte + var x12687 [1 << 17]byte + var x12688 [1 << 17]byte + var x12689 [1 << 17]byte + var x12690 [1 << 17]byte + var x12691 [1 << 17]byte + var x12692 [1 << 17]byte + var x12693 [1 << 17]byte + var x12694 [1 << 17]byte + var x12695 [1 << 17]byte + var x12696 [1 << 17]byte + var x12697 [1 << 17]byte + var x12698 [1 << 17]byte + var x12699 [1 << 17]byte + var x12700 [1 << 17]byte + var x12701 [1 << 17]byte + var x12702 [1 << 17]byte + var x12703 [1 << 17]byte + var x12704 [1 << 17]byte + var x12705 [1 << 17]byte + var x12706 [1 << 17]byte + var x12707 [1 << 17]byte + var x12708 [1 << 17]byte + var x12709 [1 << 17]byte + var x12710 [1 << 17]byte + var x12711 [1 << 17]byte + var x12712 [1 << 17]byte + var x12713 [1 << 17]byte + var x12714 [1 << 17]byte + var x12715 [1 << 17]byte + var x12716 [1 << 17]byte + var x12717 [1 << 17]byte + var x12718 [1 << 17]byte + var x12719 [1 << 17]byte + var x12720 [1 << 17]byte + var x12721 [1 << 17]byte + var x12722 [1 << 17]byte + var x12723 [1 << 17]byte + var x12724 [1 << 17]byte + var x12725 [1 << 17]byte + var x12726 [1 << 17]byte + var x12727 [1 << 17]byte + var x12728 [1 << 17]byte + var x12729 [1 << 17]byte + var x12730 [1 << 17]byte + var x12731 [1 << 17]byte + var x12732 [1 << 17]byte + var x12733 [1 << 17]byte + var x12734 [1 << 17]byte + var x12735 [1 << 17]byte + var x12736 [1 << 17]byte + var x12737 [1 << 17]byte + var x12738 [1 << 17]byte + var x12739 [1 << 17]byte + var x12740 [1 << 17]byte + var x12741 [1 << 17]byte + var x12742 [1 << 17]byte + var x12743 [1 << 17]byte + var x12744 [1 << 17]byte + var x12745 [1 << 17]byte + var x12746 [1 << 17]byte + var x12747 [1 << 17]byte + var x12748 [1 << 17]byte + var x12749 [1 << 17]byte + var x12750 [1 << 17]byte + var x12751 [1 << 17]byte + var x12752 [1 << 17]byte + var x12753 [1 << 17]byte + var x12754 [1 << 17]byte + var x12755 [1 << 17]byte + var x12756 [1 << 17]byte + var x12757 [1 << 17]byte + var x12758 [1 << 17]byte + var x12759 [1 << 17]byte + var x12760 [1 << 17]byte + var x12761 [1 << 17]byte + var x12762 [1 << 17]byte + var x12763 [1 << 17]byte + var x12764 [1 << 17]byte + var x12765 [1 << 17]byte + var x12766 [1 << 17]byte + var x12767 [1 << 17]byte + var x12768 [1 << 17]byte + var x12769 [1 << 17]byte + var x12770 [1 << 17]byte + var x12771 [1 << 17]byte + var x12772 [1 << 17]byte + var x12773 [1 << 17]byte + var x12774 [1 << 17]byte + var x12775 [1 << 17]byte + var x12776 [1 << 17]byte + var x12777 [1 << 17]byte + var x12778 [1 << 17]byte + var x12779 [1 << 17]byte + var x12780 [1 << 17]byte + var x12781 [1 << 17]byte + var x12782 [1 << 17]byte + var x12783 [1 << 17]byte + var x12784 [1 << 17]byte + var x12785 [1 << 17]byte + var x12786 [1 << 17]byte + var x12787 [1 << 17]byte + var x12788 [1 << 17]byte + var x12789 [1 << 17]byte + var x12790 [1 << 17]byte + var x12791 [1 << 17]byte + var x12792 [1 << 17]byte + var x12793 [1 << 17]byte + var x12794 [1 << 17]byte + var x12795 [1 << 17]byte + var x12796 [1 << 17]byte + var x12797 [1 << 17]byte + var x12798 [1 << 17]byte + var x12799 [1 << 17]byte + var x12800 [1 << 17]byte + var x12801 [1 << 17]byte + var x12802 [1 << 17]byte + var x12803 [1 << 17]byte + var x12804 [1 << 17]byte + var x12805 [1 << 17]byte + var x12806 [1 << 17]byte + var x12807 [1 << 17]byte + var x12808 [1 << 17]byte + var x12809 [1 << 17]byte + var x12810 [1 << 17]byte + var x12811 [1 << 17]byte + var x12812 [1 << 17]byte + var x12813 [1 << 17]byte + var x12814 [1 << 17]byte + var x12815 [1 << 17]byte + var x12816 [1 << 17]byte + var x12817 [1 << 17]byte + var x12818 [1 << 17]byte + var x12819 [1 << 17]byte + var x12820 [1 << 17]byte + var x12821 [1 << 17]byte + var x12822 [1 << 17]byte + var x12823 [1 << 17]byte + var x12824 [1 << 17]byte + var x12825 [1 << 17]byte + var x12826 [1 << 17]byte + var x12827 [1 << 17]byte + var x12828 [1 << 17]byte + var x12829 [1 << 17]byte + var x12830 [1 << 17]byte + var x12831 [1 << 17]byte + var x12832 [1 << 17]byte + var x12833 [1 << 17]byte + var x12834 [1 << 17]byte + var x12835 [1 << 17]byte + var x12836 [1 << 17]byte + var x12837 [1 << 17]byte + var x12838 [1 << 17]byte + var x12839 [1 << 17]byte + var x12840 [1 << 17]byte + var x12841 [1 << 17]byte + var x12842 [1 << 17]byte + var x12843 [1 << 17]byte + var x12844 [1 << 17]byte + var x12845 [1 << 17]byte + var x12846 [1 << 17]byte + var x12847 [1 << 17]byte + var x12848 [1 << 17]byte + var x12849 [1 << 17]byte + var x12850 [1 << 17]byte + var x12851 [1 << 17]byte + var x12852 [1 << 17]byte + var x12853 [1 << 17]byte + var x12854 [1 << 17]byte + var x12855 [1 << 17]byte + var x12856 [1 << 17]byte + var x12857 [1 << 17]byte + var x12858 [1 << 17]byte + var x12859 [1 << 17]byte + var x12860 [1 << 17]byte + var x12861 [1 << 17]byte + var x12862 [1 << 17]byte + var x12863 [1 << 17]byte + var x12864 [1 << 17]byte + var x12865 [1 << 17]byte + var x12866 [1 << 17]byte + var x12867 [1 << 17]byte + var x12868 [1 << 17]byte + var x12869 [1 << 17]byte + var x12870 [1 << 17]byte + var x12871 [1 << 17]byte + var x12872 [1 << 17]byte + var x12873 [1 << 17]byte + var x12874 [1 << 17]byte + var x12875 [1 << 17]byte + var x12876 [1 << 17]byte + var x12877 [1 << 17]byte + var x12878 [1 << 17]byte + var x12879 [1 << 17]byte + var x12880 [1 << 17]byte + var x12881 [1 << 17]byte + var x12882 [1 << 17]byte + var x12883 [1 << 17]byte + var x12884 [1 << 17]byte + var x12885 [1 << 17]byte + var x12886 [1 << 17]byte + var x12887 [1 << 17]byte + var x12888 [1 << 17]byte + var x12889 [1 << 17]byte + var x12890 [1 << 17]byte + var x12891 [1 << 17]byte + var x12892 [1 << 17]byte + var x12893 [1 << 17]byte + var x12894 [1 << 17]byte + var x12895 [1 << 17]byte + var x12896 [1 << 17]byte + var x12897 [1 << 17]byte + var x12898 [1 << 17]byte + var x12899 [1 << 17]byte + var x12900 [1 << 17]byte + var x12901 [1 << 17]byte + var x12902 [1 << 17]byte + var x12903 [1 << 17]byte + var x12904 [1 << 17]byte + var x12905 [1 << 17]byte + var x12906 [1 << 17]byte + var x12907 [1 << 17]byte + var x12908 [1 << 17]byte + var x12909 [1 << 17]byte + var x12910 [1 << 17]byte + var x12911 [1 << 17]byte + var x12912 [1 << 17]byte + var x12913 [1 << 17]byte + var x12914 [1 << 17]byte + var x12915 [1 << 17]byte + var x12916 [1 << 17]byte + var x12917 [1 << 17]byte + var x12918 [1 << 17]byte + var x12919 [1 << 17]byte + var x12920 [1 << 17]byte + var x12921 [1 << 17]byte + var x12922 [1 << 17]byte + var x12923 [1 << 17]byte + var x12924 [1 << 17]byte + var x12925 [1 << 17]byte + var x12926 [1 << 17]byte + var x12927 [1 << 17]byte + var x12928 [1 << 17]byte + var x12929 [1 << 17]byte + var x12930 [1 << 17]byte + var x12931 [1 << 17]byte + var x12932 [1 << 17]byte + var x12933 [1 << 17]byte + var x12934 [1 << 17]byte + var x12935 [1 << 17]byte + var x12936 [1 << 17]byte + var x12937 [1 << 17]byte + var x12938 [1 << 17]byte + var x12939 [1 << 17]byte + var x12940 [1 << 17]byte + var x12941 [1 << 17]byte + var x12942 [1 << 17]byte + var x12943 [1 << 17]byte + var x12944 [1 << 17]byte + var x12945 [1 << 17]byte + var x12946 [1 << 17]byte + var x12947 [1 << 17]byte + var x12948 [1 << 17]byte + var x12949 [1 << 17]byte + var x12950 [1 << 17]byte + var x12951 [1 << 17]byte + var x12952 [1 << 17]byte + var x12953 [1 << 17]byte + var x12954 [1 << 17]byte + var x12955 [1 << 17]byte + var x12956 [1 << 17]byte + var x12957 [1 << 17]byte + var x12958 [1 << 17]byte + var x12959 [1 << 17]byte + var x12960 [1 << 17]byte + var x12961 [1 << 17]byte + var x12962 [1 << 17]byte + var x12963 [1 << 17]byte + var x12964 [1 << 17]byte + var x12965 [1 << 17]byte + var x12966 [1 << 17]byte + var x12967 [1 << 17]byte + var x12968 [1 << 17]byte + var x12969 [1 << 17]byte + var x12970 [1 << 17]byte + var x12971 [1 << 17]byte + var x12972 [1 << 17]byte + var x12973 [1 << 17]byte + var x12974 [1 << 17]byte + var x12975 [1 << 17]byte + var x12976 [1 << 17]byte + var x12977 [1 << 17]byte + var x12978 [1 << 17]byte + var x12979 [1 << 17]byte + var x12980 [1 << 17]byte + var x12981 [1 << 17]byte + var x12982 [1 << 17]byte + var x12983 [1 << 17]byte + var x12984 [1 << 17]byte + var x12985 [1 << 17]byte + var x12986 [1 << 17]byte + var x12987 [1 << 17]byte + var x12988 [1 << 17]byte + var x12989 [1 << 17]byte + var x12990 [1 << 17]byte + var x12991 [1 << 17]byte + var x12992 [1 << 17]byte + var x12993 [1 << 17]byte + var x12994 [1 << 17]byte + var x12995 [1 << 17]byte + var x12996 [1 << 17]byte + var x12997 [1 << 17]byte + var x12998 [1 << 17]byte + var x12999 [1 << 17]byte + var x13000 [1 << 17]byte + var x13001 [1 << 17]byte + var x13002 [1 << 17]byte + var x13003 [1 << 17]byte + var x13004 [1 << 17]byte + var x13005 [1 << 17]byte + var x13006 [1 << 17]byte + var x13007 [1 << 17]byte + var x13008 [1 << 17]byte + var x13009 [1 << 17]byte + var x13010 [1 << 17]byte + var x13011 [1 << 17]byte + var x13012 [1 << 17]byte + var x13013 [1 << 17]byte + var x13014 [1 << 17]byte + var x13015 [1 << 17]byte + var x13016 [1 << 17]byte + var x13017 [1 << 17]byte + var x13018 [1 << 17]byte + var x13019 [1 << 17]byte + var x13020 [1 << 17]byte + var x13021 [1 << 17]byte + var x13022 [1 << 17]byte + var x13023 [1 << 17]byte + var x13024 [1 << 17]byte + var x13025 [1 << 17]byte + var x13026 [1 << 17]byte + var x13027 [1 << 17]byte + var x13028 [1 << 17]byte + var x13029 [1 << 17]byte + var x13030 [1 << 17]byte + var x13031 [1 << 17]byte + var x13032 [1 << 17]byte + var x13033 [1 << 17]byte + var x13034 [1 << 17]byte + var x13035 [1 << 17]byte + var x13036 [1 << 17]byte + var x13037 [1 << 17]byte + var x13038 [1 << 17]byte + var x13039 [1 << 17]byte + var x13040 [1 << 17]byte + var x13041 [1 << 17]byte + var x13042 [1 << 17]byte + var x13043 [1 << 17]byte + var x13044 [1 << 17]byte + var x13045 [1 << 17]byte + var x13046 [1 << 17]byte + var x13047 [1 << 17]byte + var x13048 [1 << 17]byte + var x13049 [1 << 17]byte + var x13050 [1 << 17]byte + var x13051 [1 << 17]byte + var x13052 [1 << 17]byte + var x13053 [1 << 17]byte + var x13054 [1 << 17]byte + var x13055 [1 << 17]byte + var x13056 [1 << 17]byte + var x13057 [1 << 17]byte + var x13058 [1 << 17]byte + var x13059 [1 << 17]byte + var x13060 [1 << 17]byte + var x13061 [1 << 17]byte + var x13062 [1 << 17]byte + var x13063 [1 << 17]byte + var x13064 [1 << 17]byte + var x13065 [1 << 17]byte + var x13066 [1 << 17]byte + var x13067 [1 << 17]byte + var x13068 [1 << 17]byte + var x13069 [1 << 17]byte + var x13070 [1 << 17]byte + var x13071 [1 << 17]byte + var x13072 [1 << 17]byte + var x13073 [1 << 17]byte + var x13074 [1 << 17]byte + var x13075 [1 << 17]byte + var x13076 [1 << 17]byte + var x13077 [1 << 17]byte + var x13078 [1 << 17]byte + var x13079 [1 << 17]byte + var x13080 [1 << 17]byte + var x13081 [1 << 17]byte + var x13082 [1 << 17]byte + var x13083 [1 << 17]byte + var x13084 [1 << 17]byte + var x13085 [1 << 17]byte + var x13086 [1 << 17]byte + var x13087 [1 << 17]byte + var x13088 [1 << 17]byte + var x13089 [1 << 17]byte + var x13090 [1 << 17]byte + var x13091 [1 << 17]byte + var x13092 [1 << 17]byte + var x13093 [1 << 17]byte + var x13094 [1 << 17]byte + var x13095 [1 << 17]byte + var x13096 [1 << 17]byte + var x13097 [1 << 17]byte + var x13098 [1 << 17]byte + var x13099 [1 << 17]byte + var x13100 [1 << 17]byte + var x13101 [1 << 17]byte + var x13102 [1 << 17]byte + var x13103 [1 << 17]byte + var x13104 [1 << 17]byte + var x13105 [1 << 17]byte + var x13106 [1 << 17]byte + var x13107 [1 << 17]byte + var x13108 [1 << 17]byte + var x13109 [1 << 17]byte + var x13110 [1 << 17]byte + var x13111 [1 << 17]byte + var x13112 [1 << 17]byte + var x13113 [1 << 17]byte + var x13114 [1 << 17]byte + var x13115 [1 << 17]byte + var x13116 [1 << 17]byte + var x13117 [1 << 17]byte + var x13118 [1 << 17]byte + var x13119 [1 << 17]byte + var x13120 [1 << 17]byte + var x13121 [1 << 17]byte + var x13122 [1 << 17]byte + var x13123 [1 << 17]byte + var x13124 [1 << 17]byte + var x13125 [1 << 17]byte + var x13126 [1 << 17]byte + var x13127 [1 << 17]byte + var x13128 [1 << 17]byte + var x13129 [1 << 17]byte + var x13130 [1 << 17]byte + var x13131 [1 << 17]byte + var x13132 [1 << 17]byte + var x13133 [1 << 17]byte + var x13134 [1 << 17]byte + var x13135 [1 << 17]byte + var x13136 [1 << 17]byte + var x13137 [1 << 17]byte + var x13138 [1 << 17]byte + var x13139 [1 << 17]byte + var x13140 [1 << 17]byte + var x13141 [1 << 17]byte + var x13142 [1 << 17]byte + var x13143 [1 << 17]byte + var x13144 [1 << 17]byte + var x13145 [1 << 17]byte + var x13146 [1 << 17]byte + var x13147 [1 << 17]byte + var x13148 [1 << 17]byte + var x13149 [1 << 17]byte + var x13150 [1 << 17]byte + var x13151 [1 << 17]byte + var x13152 [1 << 17]byte + var x13153 [1 << 17]byte + var x13154 [1 << 17]byte + var x13155 [1 << 17]byte + var x13156 [1 << 17]byte + var x13157 [1 << 17]byte + var x13158 [1 << 17]byte + var x13159 [1 << 17]byte + var x13160 [1 << 17]byte + var x13161 [1 << 17]byte + var x13162 [1 << 17]byte + var x13163 [1 << 17]byte + var x13164 [1 << 17]byte + var x13165 [1 << 17]byte + var x13166 [1 << 17]byte + var x13167 [1 << 17]byte + var x13168 [1 << 17]byte + var x13169 [1 << 17]byte + var x13170 [1 << 17]byte + var x13171 [1 << 17]byte + var x13172 [1 << 17]byte + var x13173 [1 << 17]byte + var x13174 [1 << 17]byte + var x13175 [1 << 17]byte + var x13176 [1 << 17]byte + var x13177 [1 << 17]byte + var x13178 [1 << 17]byte + var x13179 [1 << 17]byte + var x13180 [1 << 17]byte + var x13181 [1 << 17]byte + var x13182 [1 << 17]byte + var x13183 [1 << 17]byte + var x13184 [1 << 17]byte + var x13185 [1 << 17]byte + var x13186 [1 << 17]byte + var x13187 [1 << 17]byte + var x13188 [1 << 17]byte + var x13189 [1 << 17]byte + var x13190 [1 << 17]byte + var x13191 [1 << 17]byte + var x13192 [1 << 17]byte + var x13193 [1 << 17]byte + var x13194 [1 << 17]byte + var x13195 [1 << 17]byte + var x13196 [1 << 17]byte + var x13197 [1 << 17]byte + var x13198 [1 << 17]byte + var x13199 [1 << 17]byte + var x13200 [1 << 17]byte + var x13201 [1 << 17]byte + var x13202 [1 << 17]byte + var x13203 [1 << 17]byte + var x13204 [1 << 17]byte + var x13205 [1 << 17]byte + var x13206 [1 << 17]byte + var x13207 [1 << 17]byte + var x13208 [1 << 17]byte + var x13209 [1 << 17]byte + var x13210 [1 << 17]byte + var x13211 [1 << 17]byte + var x13212 [1 << 17]byte + var x13213 [1 << 17]byte + var x13214 [1 << 17]byte + var x13215 [1 << 17]byte + var x13216 [1 << 17]byte + var x13217 [1 << 17]byte + var x13218 [1 << 17]byte + var x13219 [1 << 17]byte + var x13220 [1 << 17]byte + var x13221 [1 << 17]byte + var x13222 [1 << 17]byte + var x13223 [1 << 17]byte + var x13224 [1 << 17]byte + var x13225 [1 << 17]byte + var x13226 [1 << 17]byte + var x13227 [1 << 17]byte + var x13228 [1 << 17]byte + var x13229 [1 << 17]byte + var x13230 [1 << 17]byte + var x13231 [1 << 17]byte + var x13232 [1 << 17]byte + var x13233 [1 << 17]byte + var x13234 [1 << 17]byte + var x13235 [1 << 17]byte + var x13236 [1 << 17]byte + var x13237 [1 << 17]byte + var x13238 [1 << 17]byte + var x13239 [1 << 17]byte + var x13240 [1 << 17]byte + var x13241 [1 << 17]byte + var x13242 [1 << 17]byte + var x13243 [1 << 17]byte + var x13244 [1 << 17]byte + var x13245 [1 << 17]byte + var x13246 [1 << 17]byte + var x13247 [1 << 17]byte + var x13248 [1 << 17]byte + var x13249 [1 << 17]byte + var x13250 [1 << 17]byte + var x13251 [1 << 17]byte + var x13252 [1 << 17]byte + var x13253 [1 << 17]byte + var x13254 [1 << 17]byte + var x13255 [1 << 17]byte + var x13256 [1 << 17]byte + var x13257 [1 << 17]byte + var x13258 [1 << 17]byte + var x13259 [1 << 17]byte + var x13260 [1 << 17]byte + var x13261 [1 << 17]byte + var x13262 [1 << 17]byte + var x13263 [1 << 17]byte + var x13264 [1 << 17]byte + var x13265 [1 << 17]byte + var x13266 [1 << 17]byte + var x13267 [1 << 17]byte + var x13268 [1 << 17]byte + var x13269 [1 << 17]byte + var x13270 [1 << 17]byte + var x13271 [1 << 17]byte + var x13272 [1 << 17]byte + var x13273 [1 << 17]byte + var x13274 [1 << 17]byte + var x13275 [1 << 17]byte + var x13276 [1 << 17]byte + var x13277 [1 << 17]byte + var x13278 [1 << 17]byte + var x13279 [1 << 17]byte + var x13280 [1 << 17]byte + var x13281 [1 << 17]byte + var x13282 [1 << 17]byte + var x13283 [1 << 17]byte + var x13284 [1 << 17]byte + var x13285 [1 << 17]byte + var x13286 [1 << 17]byte + var x13287 [1 << 17]byte + var x13288 [1 << 17]byte + var x13289 [1 << 17]byte + var x13290 [1 << 17]byte + var x13291 [1 << 17]byte + var x13292 [1 << 17]byte + var x13293 [1 << 17]byte + var x13294 [1 << 17]byte + var x13295 [1 << 17]byte + var x13296 [1 << 17]byte + var x13297 [1 << 17]byte + var x13298 [1 << 17]byte + var x13299 [1 << 17]byte + var x13300 [1 << 17]byte + var x13301 [1 << 17]byte + var x13302 [1 << 17]byte + var x13303 [1 << 17]byte + var x13304 [1 << 17]byte + var x13305 [1 << 17]byte + var x13306 [1 << 17]byte + var x13307 [1 << 17]byte + var x13308 [1 << 17]byte + var x13309 [1 << 17]byte + var x13310 [1 << 17]byte + var x13311 [1 << 17]byte + var x13312 [1 << 17]byte + var x13313 [1 << 17]byte + var x13314 [1 << 17]byte + var x13315 [1 << 17]byte + var x13316 [1 << 17]byte + var x13317 [1 << 17]byte + var x13318 [1 << 17]byte + var x13319 [1 << 17]byte + var x13320 [1 << 17]byte + var x13321 [1 << 17]byte + var x13322 [1 << 17]byte + var x13323 [1 << 17]byte + var x13324 [1 << 17]byte + var x13325 [1 << 17]byte + var x13326 [1 << 17]byte + var x13327 [1 << 17]byte + var x13328 [1 << 17]byte + var x13329 [1 << 17]byte + var x13330 [1 << 17]byte + var x13331 [1 << 17]byte + var x13332 [1 << 17]byte + var x13333 [1 << 17]byte + var x13334 [1 << 17]byte + var x13335 [1 << 17]byte + var x13336 [1 << 17]byte + var x13337 [1 << 17]byte + var x13338 [1 << 17]byte + var x13339 [1 << 17]byte + var x13340 [1 << 17]byte + var x13341 [1 << 17]byte + var x13342 [1 << 17]byte + var x13343 [1 << 17]byte + var x13344 [1 << 17]byte + var x13345 [1 << 17]byte + var x13346 [1 << 17]byte + var x13347 [1 << 17]byte + var x13348 [1 << 17]byte + var x13349 [1 << 17]byte + var x13350 [1 << 17]byte + var x13351 [1 << 17]byte + var x13352 [1 << 17]byte + var x13353 [1 << 17]byte + var x13354 [1 << 17]byte + var x13355 [1 << 17]byte + var x13356 [1 << 17]byte + var x13357 [1 << 17]byte + var x13358 [1 << 17]byte + var x13359 [1 << 17]byte + var x13360 [1 << 17]byte + var x13361 [1 << 17]byte + var x13362 [1 << 17]byte + var x13363 [1 << 17]byte + var x13364 [1 << 17]byte + var x13365 [1 << 17]byte + var x13366 [1 << 17]byte + var x13367 [1 << 17]byte + var x13368 [1 << 17]byte + var x13369 [1 << 17]byte + var x13370 [1 << 17]byte + var x13371 [1 << 17]byte + var x13372 [1 << 17]byte + var x13373 [1 << 17]byte + var x13374 [1 << 17]byte + var x13375 [1 << 17]byte + var x13376 [1 << 17]byte + var x13377 [1 << 17]byte + var x13378 [1 << 17]byte + var x13379 [1 << 17]byte + var x13380 [1 << 17]byte + var x13381 [1 << 17]byte + var x13382 [1 << 17]byte + var x13383 [1 << 17]byte + var x13384 [1 << 17]byte + var x13385 [1 << 17]byte + var x13386 [1 << 17]byte + var x13387 [1 << 17]byte + var x13388 [1 << 17]byte + var x13389 [1 << 17]byte + var x13390 [1 << 17]byte + var x13391 [1 << 17]byte + var x13392 [1 << 17]byte + var x13393 [1 << 17]byte + var x13394 [1 << 17]byte + var x13395 [1 << 17]byte + var x13396 [1 << 17]byte + var x13397 [1 << 17]byte + var x13398 [1 << 17]byte + var x13399 [1 << 17]byte + var x13400 [1 << 17]byte + var x13401 [1 << 17]byte + var x13402 [1 << 17]byte + var x13403 [1 << 17]byte + var x13404 [1 << 17]byte + var x13405 [1 << 17]byte + var x13406 [1 << 17]byte + var x13407 [1 << 17]byte + var x13408 [1 << 17]byte + var x13409 [1 << 17]byte + var x13410 [1 << 17]byte + var x13411 [1 << 17]byte + var x13412 [1 << 17]byte + var x13413 [1 << 17]byte + var x13414 [1 << 17]byte + var x13415 [1 << 17]byte + var x13416 [1 << 17]byte + var x13417 [1 << 17]byte + var x13418 [1 << 17]byte + var x13419 [1 << 17]byte + var x13420 [1 << 17]byte + var x13421 [1 << 17]byte + var x13422 [1 << 17]byte + var x13423 [1 << 17]byte + var x13424 [1 << 17]byte + var x13425 [1 << 17]byte + var x13426 [1 << 17]byte + var x13427 [1 << 17]byte + var x13428 [1 << 17]byte + var x13429 [1 << 17]byte + var x13430 [1 << 17]byte + var x13431 [1 << 17]byte + var x13432 [1 << 17]byte + var x13433 [1 << 17]byte + var x13434 [1 << 17]byte + var x13435 [1 << 17]byte + var x13436 [1 << 17]byte + var x13437 [1 << 17]byte + var x13438 [1 << 17]byte + var x13439 [1 << 17]byte + var x13440 [1 << 17]byte + var x13441 [1 << 17]byte + var x13442 [1 << 17]byte + var x13443 [1 << 17]byte + var x13444 [1 << 17]byte + var x13445 [1 << 17]byte + var x13446 [1 << 17]byte + var x13447 [1 << 17]byte + var x13448 [1 << 17]byte + var x13449 [1 << 17]byte + var x13450 [1 << 17]byte + var x13451 [1 << 17]byte + var x13452 [1 << 17]byte + var x13453 [1 << 17]byte + var x13454 [1 << 17]byte + var x13455 [1 << 17]byte + var x13456 [1 << 17]byte + var x13457 [1 << 17]byte + var x13458 [1 << 17]byte + var x13459 [1 << 17]byte + var x13460 [1 << 17]byte + var x13461 [1 << 17]byte + var x13462 [1 << 17]byte + var x13463 [1 << 17]byte + var x13464 [1 << 17]byte + var x13465 [1 << 17]byte + var x13466 [1 << 17]byte + var x13467 [1 << 17]byte + var x13468 [1 << 17]byte + var x13469 [1 << 17]byte + var x13470 [1 << 17]byte + var x13471 [1 << 17]byte + var x13472 [1 << 17]byte + var x13473 [1 << 17]byte + var x13474 [1 << 17]byte + var x13475 [1 << 17]byte + var x13476 [1 << 17]byte + var x13477 [1 << 17]byte + var x13478 [1 << 17]byte + var x13479 [1 << 17]byte + var x13480 [1 << 17]byte + var x13481 [1 << 17]byte + var x13482 [1 << 17]byte + var x13483 [1 << 17]byte + var x13484 [1 << 17]byte + var x13485 [1 << 17]byte + var x13486 [1 << 17]byte + var x13487 [1 << 17]byte + var x13488 [1 << 17]byte + var x13489 [1 << 17]byte + var x13490 [1 << 17]byte + var x13491 [1 << 17]byte + var x13492 [1 << 17]byte + var x13493 [1 << 17]byte + var x13494 [1 << 17]byte + var x13495 [1 << 17]byte + var x13496 [1 << 17]byte + var x13497 [1 << 17]byte + var x13498 [1 << 17]byte + var x13499 [1 << 17]byte + var x13500 [1 << 17]byte + var x13501 [1 << 17]byte + var x13502 [1 << 17]byte + var x13503 [1 << 17]byte + var x13504 [1 << 17]byte + var x13505 [1 << 17]byte + var x13506 [1 << 17]byte + var x13507 [1 << 17]byte + var x13508 [1 << 17]byte + var x13509 [1 << 17]byte + var x13510 [1 << 17]byte + var x13511 [1 << 17]byte + var x13512 [1 << 17]byte + var x13513 [1 << 17]byte + var x13514 [1 << 17]byte + var x13515 [1 << 17]byte + var x13516 [1 << 17]byte + var x13517 [1 << 17]byte + var x13518 [1 << 17]byte + var x13519 [1 << 17]byte + var x13520 [1 << 17]byte + var x13521 [1 << 17]byte + var x13522 [1 << 17]byte + var x13523 [1 << 17]byte + var x13524 [1 << 17]byte + var x13525 [1 << 17]byte + var x13526 [1 << 17]byte + var x13527 [1 << 17]byte + var x13528 [1 << 17]byte + var x13529 [1 << 17]byte + var x13530 [1 << 17]byte + var x13531 [1 << 17]byte + var x13532 [1 << 17]byte + var x13533 [1 << 17]byte + var x13534 [1 << 17]byte + var x13535 [1 << 17]byte + var x13536 [1 << 17]byte + var x13537 [1 << 17]byte + var x13538 [1 << 17]byte + var x13539 [1 << 17]byte + var x13540 [1 << 17]byte + var x13541 [1 << 17]byte + var x13542 [1 << 17]byte + var x13543 [1 << 17]byte + var x13544 [1 << 17]byte + var x13545 [1 << 17]byte + var x13546 [1 << 17]byte + var x13547 [1 << 17]byte + var x13548 [1 << 17]byte + var x13549 [1 << 17]byte + var x13550 [1 << 17]byte + var x13551 [1 << 17]byte + var x13552 [1 << 17]byte + var x13553 [1 << 17]byte + var x13554 [1 << 17]byte + var x13555 [1 << 17]byte + var x13556 [1 << 17]byte + var x13557 [1 << 17]byte + var x13558 [1 << 17]byte + var x13559 [1 << 17]byte + var x13560 [1 << 17]byte + var x13561 [1 << 17]byte + var x13562 [1 << 17]byte + var x13563 [1 << 17]byte + var x13564 [1 << 17]byte + var x13565 [1 << 17]byte + var x13566 [1 << 17]byte + var x13567 [1 << 17]byte + var x13568 [1 << 17]byte + var x13569 [1 << 17]byte + var x13570 [1 << 17]byte + var x13571 [1 << 17]byte + var x13572 [1 << 17]byte + var x13573 [1 << 17]byte + var x13574 [1 << 17]byte + var x13575 [1 << 17]byte + var x13576 [1 << 17]byte + var x13577 [1 << 17]byte + var x13578 [1 << 17]byte + var x13579 [1 << 17]byte + var x13580 [1 << 17]byte + var x13581 [1 << 17]byte + var x13582 [1 << 17]byte + var x13583 [1 << 17]byte + var x13584 [1 << 17]byte + var x13585 [1 << 17]byte + var x13586 [1 << 17]byte + var x13587 [1 << 17]byte + var x13588 [1 << 17]byte + var x13589 [1 << 17]byte + var x13590 [1 << 17]byte + var x13591 [1 << 17]byte + var x13592 [1 << 17]byte + var x13593 [1 << 17]byte + var x13594 [1 << 17]byte + var x13595 [1 << 17]byte + var x13596 [1 << 17]byte + var x13597 [1 << 17]byte + var x13598 [1 << 17]byte + var x13599 [1 << 17]byte + var x13600 [1 << 17]byte + var x13601 [1 << 17]byte + var x13602 [1 << 17]byte + var x13603 [1 << 17]byte + var x13604 [1 << 17]byte + var x13605 [1 << 17]byte + var x13606 [1 << 17]byte + var x13607 [1 << 17]byte + var x13608 [1 << 17]byte + var x13609 [1 << 17]byte + var x13610 [1 << 17]byte + var x13611 [1 << 17]byte + var x13612 [1 << 17]byte + var x13613 [1 << 17]byte + var x13614 [1 << 17]byte + var x13615 [1 << 17]byte + var x13616 [1 << 17]byte + var x13617 [1 << 17]byte + var x13618 [1 << 17]byte + var x13619 [1 << 17]byte + var x13620 [1 << 17]byte + var x13621 [1 << 17]byte + var x13622 [1 << 17]byte + var x13623 [1 << 17]byte + var x13624 [1 << 17]byte + var x13625 [1 << 17]byte + var x13626 [1 << 17]byte + var x13627 [1 << 17]byte + var x13628 [1 << 17]byte + var x13629 [1 << 17]byte + var x13630 [1 << 17]byte + var x13631 [1 << 17]byte + var x13632 [1 << 17]byte + var x13633 [1 << 17]byte + var x13634 [1 << 17]byte + var x13635 [1 << 17]byte + var x13636 [1 << 17]byte + var x13637 [1 << 17]byte + var x13638 [1 << 17]byte + var x13639 [1 << 17]byte + var x13640 [1 << 17]byte + var x13641 [1 << 17]byte + var x13642 [1 << 17]byte + var x13643 [1 << 17]byte + var x13644 [1 << 17]byte + var x13645 [1 << 17]byte + var x13646 [1 << 17]byte + var x13647 [1 << 17]byte + var x13648 [1 << 17]byte + var x13649 [1 << 17]byte + var x13650 [1 << 17]byte + var x13651 [1 << 17]byte + var x13652 [1 << 17]byte + var x13653 [1 << 17]byte + var x13654 [1 << 17]byte + var x13655 [1 << 17]byte + var x13656 [1 << 17]byte + var x13657 [1 << 17]byte + var x13658 [1 << 17]byte + var x13659 [1 << 17]byte + var x13660 [1 << 17]byte + var x13661 [1 << 17]byte + var x13662 [1 << 17]byte + var x13663 [1 << 17]byte + var x13664 [1 << 17]byte + var x13665 [1 << 17]byte + var x13666 [1 << 17]byte + var x13667 [1 << 17]byte + var x13668 [1 << 17]byte + var x13669 [1 << 17]byte + var x13670 [1 << 17]byte + var x13671 [1 << 17]byte + var x13672 [1 << 17]byte + var x13673 [1 << 17]byte + var x13674 [1 << 17]byte + var x13675 [1 << 17]byte + var x13676 [1 << 17]byte + var x13677 [1 << 17]byte + var x13678 [1 << 17]byte + var x13679 [1 << 17]byte + var x13680 [1 << 17]byte + var x13681 [1 << 17]byte + var x13682 [1 << 17]byte + var x13683 [1 << 17]byte + var x13684 [1 << 17]byte + var x13685 [1 << 17]byte + var x13686 [1 << 17]byte + var x13687 [1 << 17]byte + var x13688 [1 << 17]byte + var x13689 [1 << 17]byte + var x13690 [1 << 17]byte + var x13691 [1 << 17]byte + var x13692 [1 << 17]byte + var x13693 [1 << 17]byte + var x13694 [1 << 17]byte + var x13695 [1 << 17]byte + var x13696 [1 << 17]byte + var x13697 [1 << 17]byte + var x13698 [1 << 17]byte + var x13699 [1 << 17]byte + var x13700 [1 << 17]byte + var x13701 [1 << 17]byte + var x13702 [1 << 17]byte + var x13703 [1 << 17]byte + var x13704 [1 << 17]byte + var x13705 [1 << 17]byte + var x13706 [1 << 17]byte + var x13707 [1 << 17]byte + var x13708 [1 << 17]byte + var x13709 [1 << 17]byte + var x13710 [1 << 17]byte + var x13711 [1 << 17]byte + var x13712 [1 << 17]byte + var x13713 [1 << 17]byte + var x13714 [1 << 17]byte + var x13715 [1 << 17]byte + var x13716 [1 << 17]byte + var x13717 [1 << 17]byte + var x13718 [1 << 17]byte + var x13719 [1 << 17]byte + var x13720 [1 << 17]byte + var x13721 [1 << 17]byte + var x13722 [1 << 17]byte + var x13723 [1 << 17]byte + var x13724 [1 << 17]byte + var x13725 [1 << 17]byte + var x13726 [1 << 17]byte + var x13727 [1 << 17]byte + var x13728 [1 << 17]byte + var x13729 [1 << 17]byte + var x13730 [1 << 17]byte + var x13731 [1 << 17]byte + var x13732 [1 << 17]byte + var x13733 [1 << 17]byte + var x13734 [1 << 17]byte + var x13735 [1 << 17]byte + var x13736 [1 << 17]byte + var x13737 [1 << 17]byte + var x13738 [1 << 17]byte + var x13739 [1 << 17]byte + var x13740 [1 << 17]byte + var x13741 [1 << 17]byte + var x13742 [1 << 17]byte + var x13743 [1 << 17]byte + var x13744 [1 << 17]byte + var x13745 [1 << 17]byte + var x13746 [1 << 17]byte + var x13747 [1 << 17]byte + var x13748 [1 << 17]byte + var x13749 [1 << 17]byte + var x13750 [1 << 17]byte + var x13751 [1 << 17]byte + var x13752 [1 << 17]byte + var x13753 [1 << 17]byte + var x13754 [1 << 17]byte + var x13755 [1 << 17]byte + var x13756 [1 << 17]byte + var x13757 [1 << 17]byte + var x13758 [1 << 17]byte + var x13759 [1 << 17]byte + var x13760 [1 << 17]byte + var x13761 [1 << 17]byte + var x13762 [1 << 17]byte + var x13763 [1 << 17]byte + var x13764 [1 << 17]byte + var x13765 [1 << 17]byte + var x13766 [1 << 17]byte + var x13767 [1 << 17]byte + var x13768 [1 << 17]byte + var x13769 [1 << 17]byte + var x13770 [1 << 17]byte + var x13771 [1 << 17]byte + var x13772 [1 << 17]byte + var x13773 [1 << 17]byte + var x13774 [1 << 17]byte + var x13775 [1 << 17]byte + var x13776 [1 << 17]byte + var x13777 [1 << 17]byte + var x13778 [1 << 17]byte + var x13779 [1 << 17]byte + var x13780 [1 << 17]byte + var x13781 [1 << 17]byte + var x13782 [1 << 17]byte + var x13783 [1 << 17]byte + var x13784 [1 << 17]byte + var x13785 [1 << 17]byte + var x13786 [1 << 17]byte + var x13787 [1 << 17]byte + var x13788 [1 << 17]byte + var x13789 [1 << 17]byte + var x13790 [1 << 17]byte + var x13791 [1 << 17]byte + var x13792 [1 << 17]byte + var x13793 [1 << 17]byte + var x13794 [1 << 17]byte + var x13795 [1 << 17]byte + var x13796 [1 << 17]byte + var x13797 [1 << 17]byte + var x13798 [1 << 17]byte + var x13799 [1 << 17]byte + var x13800 [1 << 17]byte + var x13801 [1 << 17]byte + var x13802 [1 << 17]byte + var x13803 [1 << 17]byte + var x13804 [1 << 17]byte + var x13805 [1 << 17]byte + var x13806 [1 << 17]byte + var x13807 [1 << 17]byte + var x13808 [1 << 17]byte + var x13809 [1 << 17]byte + var x13810 [1 << 17]byte + var x13811 [1 << 17]byte + var x13812 [1 << 17]byte + var x13813 [1 << 17]byte + var x13814 [1 << 17]byte + var x13815 [1 << 17]byte + var x13816 [1 << 17]byte + var x13817 [1 << 17]byte + var x13818 [1 << 17]byte + var x13819 [1 << 17]byte + var x13820 [1 << 17]byte + var x13821 [1 << 17]byte + var x13822 [1 << 17]byte + var x13823 [1 << 17]byte + var x13824 [1 << 17]byte + var x13825 [1 << 17]byte + var x13826 [1 << 17]byte + var x13827 [1 << 17]byte + var x13828 [1 << 17]byte + var x13829 [1 << 17]byte + var x13830 [1 << 17]byte + var x13831 [1 << 17]byte + var x13832 [1 << 17]byte + var x13833 [1 << 17]byte + var x13834 [1 << 17]byte + var x13835 [1 << 17]byte + var x13836 [1 << 17]byte + var x13837 [1 << 17]byte + var x13838 [1 << 17]byte + var x13839 [1 << 17]byte + var x13840 [1 << 17]byte + var x13841 [1 << 17]byte + var x13842 [1 << 17]byte + var x13843 [1 << 17]byte + var x13844 [1 << 17]byte + var x13845 [1 << 17]byte + var x13846 [1 << 17]byte + var x13847 [1 << 17]byte + var x13848 [1 << 17]byte + var x13849 [1 << 17]byte + var x13850 [1 << 17]byte + var x13851 [1 << 17]byte + var x13852 [1 << 17]byte + var x13853 [1 << 17]byte + var x13854 [1 << 17]byte + var x13855 [1 << 17]byte + var x13856 [1 << 17]byte + var x13857 [1 << 17]byte + var x13858 [1 << 17]byte + var x13859 [1 << 17]byte + var x13860 [1 << 17]byte + var x13861 [1 << 17]byte + var x13862 [1 << 17]byte + var x13863 [1 << 17]byte + var x13864 [1 << 17]byte + var x13865 [1 << 17]byte + var x13866 [1 << 17]byte + var x13867 [1 << 17]byte + var x13868 [1 << 17]byte + var x13869 [1 << 17]byte + var x13870 [1 << 17]byte + var x13871 [1 << 17]byte + var x13872 [1 << 17]byte + var x13873 [1 << 17]byte + var x13874 [1 << 17]byte + var x13875 [1 << 17]byte + var x13876 [1 << 17]byte + var x13877 [1 << 17]byte + var x13878 [1 << 17]byte + var x13879 [1 << 17]byte + var x13880 [1 << 17]byte + var x13881 [1 << 17]byte + var x13882 [1 << 17]byte + var x13883 [1 << 17]byte + var x13884 [1 << 17]byte + var x13885 [1 << 17]byte + var x13886 [1 << 17]byte + var x13887 [1 << 17]byte + var x13888 [1 << 17]byte + var x13889 [1 << 17]byte + var x13890 [1 << 17]byte + var x13891 [1 << 17]byte + var x13892 [1 << 17]byte + var x13893 [1 << 17]byte + var x13894 [1 << 17]byte + var x13895 [1 << 17]byte + var x13896 [1 << 17]byte + var x13897 [1 << 17]byte + var x13898 [1 << 17]byte + var x13899 [1 << 17]byte + var x13900 [1 << 17]byte + var x13901 [1 << 17]byte + var x13902 [1 << 17]byte + var x13903 [1 << 17]byte + var x13904 [1 << 17]byte + var x13905 [1 << 17]byte + var x13906 [1 << 17]byte + var x13907 [1 << 17]byte + var x13908 [1 << 17]byte + var x13909 [1 << 17]byte + var x13910 [1 << 17]byte + var x13911 [1 << 17]byte + var x13912 [1 << 17]byte + var x13913 [1 << 17]byte + var x13914 [1 << 17]byte + var x13915 [1 << 17]byte + var x13916 [1 << 17]byte + var x13917 [1 << 17]byte + var x13918 [1 << 17]byte + var x13919 [1 << 17]byte + var x13920 [1 << 17]byte + var x13921 [1 << 17]byte + var x13922 [1 << 17]byte + var x13923 [1 << 17]byte + var x13924 [1 << 17]byte + var x13925 [1 << 17]byte + var x13926 [1 << 17]byte + var x13927 [1 << 17]byte + var x13928 [1 << 17]byte + var x13929 [1 << 17]byte + var x13930 [1 << 17]byte + var x13931 [1 << 17]byte + var x13932 [1 << 17]byte + var x13933 [1 << 17]byte + var x13934 [1 << 17]byte + var x13935 [1 << 17]byte + var x13936 [1 << 17]byte + var x13937 [1 << 17]byte + var x13938 [1 << 17]byte + var x13939 [1 << 17]byte + var x13940 [1 << 17]byte + var x13941 [1 << 17]byte + var x13942 [1 << 17]byte + var x13943 [1 << 17]byte + var x13944 [1 << 17]byte + var x13945 [1 << 17]byte + var x13946 [1 << 17]byte + var x13947 [1 << 17]byte + var x13948 [1 << 17]byte + var x13949 [1 << 17]byte + var x13950 [1 << 17]byte + var x13951 [1 << 17]byte + var x13952 [1 << 17]byte + var x13953 [1 << 17]byte + var x13954 [1 << 17]byte + var x13955 [1 << 17]byte + var x13956 [1 << 17]byte + var x13957 [1 << 17]byte + var x13958 [1 << 17]byte + var x13959 [1 << 17]byte + var x13960 [1 << 17]byte + var x13961 [1 << 17]byte + var x13962 [1 << 17]byte + var x13963 [1 << 17]byte + var x13964 [1 << 17]byte + var x13965 [1 << 17]byte + var x13966 [1 << 17]byte + var x13967 [1 << 17]byte + var x13968 [1 << 17]byte + var x13969 [1 << 17]byte + var x13970 [1 << 17]byte + var x13971 [1 << 17]byte + var x13972 [1 << 17]byte + var x13973 [1 << 17]byte + var x13974 [1 << 17]byte + var x13975 [1 << 17]byte + var x13976 [1 << 17]byte + var x13977 [1 << 17]byte + var x13978 [1 << 17]byte + var x13979 [1 << 17]byte + var x13980 [1 << 17]byte + var x13981 [1 << 17]byte + var x13982 [1 << 17]byte + var x13983 [1 << 17]byte + var x13984 [1 << 17]byte + var x13985 [1 << 17]byte + var x13986 [1 << 17]byte + var x13987 [1 << 17]byte + var x13988 [1 << 17]byte + var x13989 [1 << 17]byte + var x13990 [1 << 17]byte + var x13991 [1 << 17]byte + var x13992 [1 << 17]byte + var x13993 [1 << 17]byte + var x13994 [1 << 17]byte + var x13995 [1 << 17]byte + var x13996 [1 << 17]byte + var x13997 [1 << 17]byte + var x13998 [1 << 17]byte + var x13999 [1 << 17]byte + var x14000 [1 << 17]byte + var x14001 [1 << 17]byte + var x14002 [1 << 17]byte + var x14003 [1 << 17]byte + var x14004 [1 << 17]byte + var x14005 [1 << 17]byte + var x14006 [1 << 17]byte + var x14007 [1 << 17]byte + var x14008 [1 << 17]byte + var x14009 [1 << 17]byte + var x14010 [1 << 17]byte + var x14011 [1 << 17]byte + var x14012 [1 << 17]byte + var x14013 [1 << 17]byte + var x14014 [1 << 17]byte + var x14015 [1 << 17]byte + var x14016 [1 << 17]byte + var x14017 [1 << 17]byte + var x14018 [1 << 17]byte + var x14019 [1 << 17]byte + var x14020 [1 << 17]byte + var x14021 [1 << 17]byte + var x14022 [1 << 17]byte + var x14023 [1 << 17]byte + var x14024 [1 << 17]byte + var x14025 [1 << 17]byte + var x14026 [1 << 17]byte + var x14027 [1 << 17]byte + var x14028 [1 << 17]byte + var x14029 [1 << 17]byte + var x14030 [1 << 17]byte + var x14031 [1 << 17]byte + var x14032 [1 << 17]byte + var x14033 [1 << 17]byte + var x14034 [1 << 17]byte + var x14035 [1 << 17]byte + var x14036 [1 << 17]byte + var x14037 [1 << 17]byte + var x14038 [1 << 17]byte + var x14039 [1 << 17]byte + var x14040 [1 << 17]byte + var x14041 [1 << 17]byte + var x14042 [1 << 17]byte + var x14043 [1 << 17]byte + var x14044 [1 << 17]byte + var x14045 [1 << 17]byte + var x14046 [1 << 17]byte + var x14047 [1 << 17]byte + var x14048 [1 << 17]byte + var x14049 [1 << 17]byte + var x14050 [1 << 17]byte + var x14051 [1 << 17]byte + var x14052 [1 << 17]byte + var x14053 [1 << 17]byte + var x14054 [1 << 17]byte + var x14055 [1 << 17]byte + var x14056 [1 << 17]byte + var x14057 [1 << 17]byte + var x14058 [1 << 17]byte + var x14059 [1 << 17]byte + var x14060 [1 << 17]byte + var x14061 [1 << 17]byte + var x14062 [1 << 17]byte + var x14063 [1 << 17]byte + var x14064 [1 << 17]byte + var x14065 [1 << 17]byte + var x14066 [1 << 17]byte + var x14067 [1 << 17]byte + var x14068 [1 << 17]byte + var x14069 [1 << 17]byte + var x14070 [1 << 17]byte + var x14071 [1 << 17]byte + var x14072 [1 << 17]byte + var x14073 [1 << 17]byte + var x14074 [1 << 17]byte + var x14075 [1 << 17]byte + var x14076 [1 << 17]byte + var x14077 [1 << 17]byte + var x14078 [1 << 17]byte + var x14079 [1 << 17]byte + var x14080 [1 << 17]byte + var x14081 [1 << 17]byte + var x14082 [1 << 17]byte + var x14083 [1 << 17]byte + var x14084 [1 << 17]byte + var x14085 [1 << 17]byte + var x14086 [1 << 17]byte + var x14087 [1 << 17]byte + var x14088 [1 << 17]byte + var x14089 [1 << 17]byte + var x14090 [1 << 17]byte + var x14091 [1 << 17]byte + var x14092 [1 << 17]byte + var x14093 [1 << 17]byte + var x14094 [1 << 17]byte + var x14095 [1 << 17]byte + var x14096 [1 << 17]byte + var x14097 [1 << 17]byte + var x14098 [1 << 17]byte + var x14099 [1 << 17]byte + var x14100 [1 << 17]byte + var x14101 [1 << 17]byte + var x14102 [1 << 17]byte + var x14103 [1 << 17]byte + var x14104 [1 << 17]byte + var x14105 [1 << 17]byte + var x14106 [1 << 17]byte + var x14107 [1 << 17]byte + var x14108 [1 << 17]byte + var x14109 [1 << 17]byte + var x14110 [1 << 17]byte + var x14111 [1 << 17]byte + var x14112 [1 << 17]byte + var x14113 [1 << 17]byte + var x14114 [1 << 17]byte + var x14115 [1 << 17]byte + var x14116 [1 << 17]byte + var x14117 [1 << 17]byte + var x14118 [1 << 17]byte + var x14119 [1 << 17]byte + var x14120 [1 << 17]byte + var x14121 [1 << 17]byte + var x14122 [1 << 17]byte + var x14123 [1 << 17]byte + var x14124 [1 << 17]byte + var x14125 [1 << 17]byte + var x14126 [1 << 17]byte + var x14127 [1 << 17]byte + var x14128 [1 << 17]byte + var x14129 [1 << 17]byte + var x14130 [1 << 17]byte + var x14131 [1 << 17]byte + var x14132 [1 << 17]byte + var x14133 [1 << 17]byte + var x14134 [1 << 17]byte + var x14135 [1 << 17]byte + var x14136 [1 << 17]byte + var x14137 [1 << 17]byte + var x14138 [1 << 17]byte + var x14139 [1 << 17]byte + var x14140 [1 << 17]byte + var x14141 [1 << 17]byte + var x14142 [1 << 17]byte + var x14143 [1 << 17]byte + var x14144 [1 << 17]byte + var x14145 [1 << 17]byte + var x14146 [1 << 17]byte + var x14147 [1 << 17]byte + var x14148 [1 << 17]byte + var x14149 [1 << 17]byte + var x14150 [1 << 17]byte + var x14151 [1 << 17]byte + var x14152 [1 << 17]byte + var x14153 [1 << 17]byte + var x14154 [1 << 17]byte + var x14155 [1 << 17]byte + var x14156 [1 << 17]byte + var x14157 [1 << 17]byte + var x14158 [1 << 17]byte + var x14159 [1 << 17]byte + var x14160 [1 << 17]byte + var x14161 [1 << 17]byte + var x14162 [1 << 17]byte + var x14163 [1 << 17]byte + var x14164 [1 << 17]byte + var x14165 [1 << 17]byte + var x14166 [1 << 17]byte + var x14167 [1 << 17]byte + var x14168 [1 << 17]byte + var x14169 [1 << 17]byte + var x14170 [1 << 17]byte + var x14171 [1 << 17]byte + var x14172 [1 << 17]byte + var x14173 [1 << 17]byte + var x14174 [1 << 17]byte + var x14175 [1 << 17]byte + var x14176 [1 << 17]byte + var x14177 [1 << 17]byte + var x14178 [1 << 17]byte + var x14179 [1 << 17]byte + var x14180 [1 << 17]byte + var x14181 [1 << 17]byte + var x14182 [1 << 17]byte + var x14183 [1 << 17]byte + var x14184 [1 << 17]byte + var x14185 [1 << 17]byte + var x14186 [1 << 17]byte + var x14187 [1 << 17]byte + var x14188 [1 << 17]byte + var x14189 [1 << 17]byte + var x14190 [1 << 17]byte + var x14191 [1 << 17]byte + var x14192 [1 << 17]byte + var x14193 [1 << 17]byte + var x14194 [1 << 17]byte + var x14195 [1 << 17]byte + var x14196 [1 << 17]byte + var x14197 [1 << 17]byte + var x14198 [1 << 17]byte + var x14199 [1 << 17]byte + var x14200 [1 << 17]byte + var x14201 [1 << 17]byte + var x14202 [1 << 17]byte + var x14203 [1 << 17]byte + var x14204 [1 << 17]byte + var x14205 [1 << 17]byte + var x14206 [1 << 17]byte + var x14207 [1 << 17]byte + var x14208 [1 << 17]byte + var x14209 [1 << 17]byte + var x14210 [1 << 17]byte + var x14211 [1 << 17]byte + var x14212 [1 << 17]byte + var x14213 [1 << 17]byte + var x14214 [1 << 17]byte + var x14215 [1 << 17]byte + var x14216 [1 << 17]byte + var x14217 [1 << 17]byte + var x14218 [1 << 17]byte + var x14219 [1 << 17]byte + var x14220 [1 << 17]byte + var x14221 [1 << 17]byte + var x14222 [1 << 17]byte + var x14223 [1 << 17]byte + var x14224 [1 << 17]byte + var x14225 [1 << 17]byte + var x14226 [1 << 17]byte + var x14227 [1 << 17]byte + var x14228 [1 << 17]byte + var x14229 [1 << 17]byte + var x14230 [1 << 17]byte + var x14231 [1 << 17]byte + var x14232 [1 << 17]byte + var x14233 [1 << 17]byte + var x14234 [1 << 17]byte + var x14235 [1 << 17]byte + var x14236 [1 << 17]byte + var x14237 [1 << 17]byte + var x14238 [1 << 17]byte + var x14239 [1 << 17]byte + var x14240 [1 << 17]byte + var x14241 [1 << 17]byte + var x14242 [1 << 17]byte + var x14243 [1 << 17]byte + var x14244 [1 << 17]byte + var x14245 [1 << 17]byte + var x14246 [1 << 17]byte + var x14247 [1 << 17]byte + var x14248 [1 << 17]byte + var x14249 [1 << 17]byte + var x14250 [1 << 17]byte + var x14251 [1 << 17]byte + var x14252 [1 << 17]byte + var x14253 [1 << 17]byte + var x14254 [1 << 17]byte + var x14255 [1 << 17]byte + var x14256 [1 << 17]byte + var x14257 [1 << 17]byte + var x14258 [1 << 17]byte + var x14259 [1 << 17]byte + var x14260 [1 << 17]byte + var x14261 [1 << 17]byte + var x14262 [1 << 17]byte + var x14263 [1 << 17]byte + var x14264 [1 << 17]byte + var x14265 [1 << 17]byte + var x14266 [1 << 17]byte + var x14267 [1 << 17]byte + var x14268 [1 << 17]byte + var x14269 [1 << 17]byte + var x14270 [1 << 17]byte + var x14271 [1 << 17]byte + var x14272 [1 << 17]byte + var x14273 [1 << 17]byte + var x14274 [1 << 17]byte + var x14275 [1 << 17]byte + var x14276 [1 << 17]byte + var x14277 [1 << 17]byte + var x14278 [1 << 17]byte + var x14279 [1 << 17]byte + var x14280 [1 << 17]byte + var x14281 [1 << 17]byte + var x14282 [1 << 17]byte + var x14283 [1 << 17]byte + var x14284 [1 << 17]byte + var x14285 [1 << 17]byte + var x14286 [1 << 17]byte + var x14287 [1 << 17]byte + var x14288 [1 << 17]byte + var x14289 [1 << 17]byte + var x14290 [1 << 17]byte + var x14291 [1 << 17]byte + var x14292 [1 << 17]byte + var x14293 [1 << 17]byte + var x14294 [1 << 17]byte + var x14295 [1 << 17]byte + var x14296 [1 << 17]byte + var x14297 [1 << 17]byte + var x14298 [1 << 17]byte + var x14299 [1 << 17]byte + var x14300 [1 << 17]byte + var x14301 [1 << 17]byte + var x14302 [1 << 17]byte + var x14303 [1 << 17]byte + var x14304 [1 << 17]byte + var x14305 [1 << 17]byte + var x14306 [1 << 17]byte + var x14307 [1 << 17]byte + var x14308 [1 << 17]byte + var x14309 [1 << 17]byte + var x14310 [1 << 17]byte + var x14311 [1 << 17]byte + var x14312 [1 << 17]byte + var x14313 [1 << 17]byte + var x14314 [1 << 17]byte + var x14315 [1 << 17]byte + var x14316 [1 << 17]byte + var x14317 [1 << 17]byte + var x14318 [1 << 17]byte + var x14319 [1 << 17]byte + var x14320 [1 << 17]byte + var x14321 [1 << 17]byte + var x14322 [1 << 17]byte + var x14323 [1 << 17]byte + var x14324 [1 << 17]byte + var x14325 [1 << 17]byte + var x14326 [1 << 17]byte + var x14327 [1 << 17]byte + var x14328 [1 << 17]byte + var x14329 [1 << 17]byte + var x14330 [1 << 17]byte + var x14331 [1 << 17]byte + var x14332 [1 << 17]byte + var x14333 [1 << 17]byte + var x14334 [1 << 17]byte + var x14335 [1 << 17]byte + var x14336 [1 << 17]byte + var x14337 [1 << 17]byte + var x14338 [1 << 17]byte + var x14339 [1 << 17]byte + var x14340 [1 << 17]byte + var x14341 [1 << 17]byte + var x14342 [1 << 17]byte + var x14343 [1 << 17]byte + var x14344 [1 << 17]byte + var x14345 [1 << 17]byte + var x14346 [1 << 17]byte + var x14347 [1 << 17]byte + var x14348 [1 << 17]byte + var x14349 [1 << 17]byte + var x14350 [1 << 17]byte + var x14351 [1 << 17]byte + var x14352 [1 << 17]byte + var x14353 [1 << 17]byte + var x14354 [1 << 17]byte + var x14355 [1 << 17]byte + var x14356 [1 << 17]byte + var x14357 [1 << 17]byte + var x14358 [1 << 17]byte + var x14359 [1 << 17]byte + var x14360 [1 << 17]byte + var x14361 [1 << 17]byte + var x14362 [1 << 17]byte + var x14363 [1 << 17]byte + var x14364 [1 << 17]byte + var x14365 [1 << 17]byte + var x14366 [1 << 17]byte + var x14367 [1 << 17]byte + var x14368 [1 << 17]byte + var x14369 [1 << 17]byte + var x14370 [1 << 17]byte + var x14371 [1 << 17]byte + var x14372 [1 << 17]byte + var x14373 [1 << 17]byte + var x14374 [1 << 17]byte + var x14375 [1 << 17]byte + var x14376 [1 << 17]byte + var x14377 [1 << 17]byte + var x14378 [1 << 17]byte + var x14379 [1 << 17]byte + var x14380 [1 << 17]byte + var x14381 [1 << 17]byte + var x14382 [1 << 17]byte + var x14383 [1 << 17]byte + var x14384 [1 << 17]byte + var x14385 [1 << 17]byte + var x14386 [1 << 17]byte + var x14387 [1 << 17]byte + var x14388 [1 << 17]byte + var x14389 [1 << 17]byte + var x14390 [1 << 17]byte + var x14391 [1 << 17]byte + var x14392 [1 << 17]byte + var x14393 [1 << 17]byte + var x14394 [1 << 17]byte + var x14395 [1 << 17]byte + var x14396 [1 << 17]byte + var x14397 [1 << 17]byte + var x14398 [1 << 17]byte + var x14399 [1 << 17]byte + var x14400 [1 << 17]byte + var x14401 [1 << 17]byte + var x14402 [1 << 17]byte + var x14403 [1 << 17]byte + var x14404 [1 << 17]byte + var x14405 [1 << 17]byte + var x14406 [1 << 17]byte + var x14407 [1 << 17]byte + var x14408 [1 << 17]byte + var x14409 [1 << 17]byte + var x14410 [1 << 17]byte + var x14411 [1 << 17]byte + var x14412 [1 << 17]byte + var x14413 [1 << 17]byte + var x14414 [1 << 17]byte + var x14415 [1 << 17]byte + var x14416 [1 << 17]byte + var x14417 [1 << 17]byte + var x14418 [1 << 17]byte + var x14419 [1 << 17]byte + var x14420 [1 << 17]byte + var x14421 [1 << 17]byte + var x14422 [1 << 17]byte + var x14423 [1 << 17]byte + var x14424 [1 << 17]byte + var x14425 [1 << 17]byte + var x14426 [1 << 17]byte + var x14427 [1 << 17]byte + var x14428 [1 << 17]byte + var x14429 [1 << 17]byte + var x14430 [1 << 17]byte + var x14431 [1 << 17]byte + var x14432 [1 << 17]byte + var x14433 [1 << 17]byte + var x14434 [1 << 17]byte + var x14435 [1 << 17]byte + var x14436 [1 << 17]byte + var x14437 [1 << 17]byte + var x14438 [1 << 17]byte + var x14439 [1 << 17]byte + var x14440 [1 << 17]byte + var x14441 [1 << 17]byte + var x14442 [1 << 17]byte + var x14443 [1 << 17]byte + var x14444 [1 << 17]byte + var x14445 [1 << 17]byte + var x14446 [1 << 17]byte + var x14447 [1 << 17]byte + var x14448 [1 << 17]byte + var x14449 [1 << 17]byte + var x14450 [1 << 17]byte + var x14451 [1 << 17]byte + var x14452 [1 << 17]byte + var x14453 [1 << 17]byte + var x14454 [1 << 17]byte + var x14455 [1 << 17]byte + var x14456 [1 << 17]byte + var x14457 [1 << 17]byte + var x14458 [1 << 17]byte + var x14459 [1 << 17]byte + var x14460 [1 << 17]byte + var x14461 [1 << 17]byte + var x14462 [1 << 17]byte + var x14463 [1 << 17]byte + var x14464 [1 << 17]byte + var x14465 [1 << 17]byte + var x14466 [1 << 17]byte + var x14467 [1 << 17]byte + var x14468 [1 << 17]byte + var x14469 [1 << 17]byte + var x14470 [1 << 17]byte + var x14471 [1 << 17]byte + var x14472 [1 << 17]byte + var x14473 [1 << 17]byte + var x14474 [1 << 17]byte + var x14475 [1 << 17]byte + var x14476 [1 << 17]byte + var x14477 [1 << 17]byte + var x14478 [1 << 17]byte + var x14479 [1 << 17]byte + var x14480 [1 << 17]byte + var x14481 [1 << 17]byte + var x14482 [1 << 17]byte + var x14483 [1 << 17]byte + var x14484 [1 << 17]byte + var x14485 [1 << 17]byte + var x14486 [1 << 17]byte + var x14487 [1 << 17]byte + var x14488 [1 << 17]byte + var x14489 [1 << 17]byte + var x14490 [1 << 17]byte + var x14491 [1 << 17]byte + var x14492 [1 << 17]byte + var x14493 [1 << 17]byte + var x14494 [1 << 17]byte + var x14495 [1 << 17]byte + var x14496 [1 << 17]byte + var x14497 [1 << 17]byte + var x14498 [1 << 17]byte + var x14499 [1 << 17]byte + var x14500 [1 << 17]byte + var x14501 [1 << 17]byte + var x14502 [1 << 17]byte + var x14503 [1 << 17]byte + var x14504 [1 << 17]byte + var x14505 [1 << 17]byte + var x14506 [1 << 17]byte + var x14507 [1 << 17]byte + var x14508 [1 << 17]byte + var x14509 [1 << 17]byte + var x14510 [1 << 17]byte + var x14511 [1 << 17]byte + var x14512 [1 << 17]byte + var x14513 [1 << 17]byte + var x14514 [1 << 17]byte + var x14515 [1 << 17]byte + var x14516 [1 << 17]byte + var x14517 [1 << 17]byte + var x14518 [1 << 17]byte + var x14519 [1 << 17]byte + var x14520 [1 << 17]byte + var x14521 [1 << 17]byte + var x14522 [1 << 17]byte + var x14523 [1 << 17]byte + var x14524 [1 << 17]byte + var x14525 [1 << 17]byte + var x14526 [1 << 17]byte + var x14527 [1 << 17]byte + var x14528 [1 << 17]byte + var x14529 [1 << 17]byte + var x14530 [1 << 17]byte + var x14531 [1 << 17]byte + var x14532 [1 << 17]byte + var x14533 [1 << 17]byte + var x14534 [1 << 17]byte + var x14535 [1 << 17]byte + var x14536 [1 << 17]byte + var x14537 [1 << 17]byte + var x14538 [1 << 17]byte + var x14539 [1 << 17]byte + var x14540 [1 << 17]byte + var x14541 [1 << 17]byte + var x14542 [1 << 17]byte + var x14543 [1 << 17]byte + var x14544 [1 << 17]byte + var x14545 [1 << 17]byte + var x14546 [1 << 17]byte + var x14547 [1 << 17]byte + var x14548 [1 << 17]byte + var x14549 [1 << 17]byte + var x14550 [1 << 17]byte + var x14551 [1 << 17]byte + var x14552 [1 << 17]byte + var x14553 [1 << 17]byte + var x14554 [1 << 17]byte + var x14555 [1 << 17]byte + var x14556 [1 << 17]byte + var x14557 [1 << 17]byte + var x14558 [1 << 17]byte + var x14559 [1 << 17]byte + var x14560 [1 << 17]byte + var x14561 [1 << 17]byte + var x14562 [1 << 17]byte + var x14563 [1 << 17]byte + var x14564 [1 << 17]byte + var x14565 [1 << 17]byte + var x14566 [1 << 17]byte + var x14567 [1 << 17]byte + var x14568 [1 << 17]byte + var x14569 [1 << 17]byte + var x14570 [1 << 17]byte + var x14571 [1 << 17]byte + var x14572 [1 << 17]byte + var x14573 [1 << 17]byte + var x14574 [1 << 17]byte + var x14575 [1 << 17]byte + var x14576 [1 << 17]byte + var x14577 [1 << 17]byte + var x14578 [1 << 17]byte + var x14579 [1 << 17]byte + var x14580 [1 << 17]byte + var x14581 [1 << 17]byte + var x14582 [1 << 17]byte + var x14583 [1 << 17]byte + var x14584 [1 << 17]byte + var x14585 [1 << 17]byte + var x14586 [1 << 17]byte + var x14587 [1 << 17]byte + var x14588 [1 << 17]byte + var x14589 [1 << 17]byte + var x14590 [1 << 17]byte + var x14591 [1 << 17]byte + var x14592 [1 << 17]byte + var x14593 [1 << 17]byte + var x14594 [1 << 17]byte + var x14595 [1 << 17]byte + var x14596 [1 << 17]byte + var x14597 [1 << 17]byte + var x14598 [1 << 17]byte + var x14599 [1 << 17]byte + var x14600 [1 << 17]byte + var x14601 [1 << 17]byte + var x14602 [1 << 17]byte + var x14603 [1 << 17]byte + var x14604 [1 << 17]byte + var x14605 [1 << 17]byte + var x14606 [1 << 17]byte + var x14607 [1 << 17]byte + var x14608 [1 << 17]byte + var x14609 [1 << 17]byte + var x14610 [1 << 17]byte + var x14611 [1 << 17]byte + var x14612 [1 << 17]byte + var x14613 [1 << 17]byte + var x14614 [1 << 17]byte + var x14615 [1 << 17]byte + var x14616 [1 << 17]byte + var x14617 [1 << 17]byte + var x14618 [1 << 17]byte + var x14619 [1 << 17]byte + var x14620 [1 << 17]byte + var x14621 [1 << 17]byte + var x14622 [1 << 17]byte + var x14623 [1 << 17]byte + var x14624 [1 << 17]byte + var x14625 [1 << 17]byte + var x14626 [1 << 17]byte + var x14627 [1 << 17]byte + var x14628 [1 << 17]byte + var x14629 [1 << 17]byte + var x14630 [1 << 17]byte + var x14631 [1 << 17]byte + var x14632 [1 << 17]byte + var x14633 [1 << 17]byte + var x14634 [1 << 17]byte + var x14635 [1 << 17]byte + var x14636 [1 << 17]byte + var x14637 [1 << 17]byte + var x14638 [1 << 17]byte + var x14639 [1 << 17]byte + var x14640 [1 << 17]byte + var x14641 [1 << 17]byte + var x14642 [1 << 17]byte + var x14643 [1 << 17]byte + var x14644 [1 << 17]byte + var x14645 [1 << 17]byte + var x14646 [1 << 17]byte + var x14647 [1 << 17]byte + var x14648 [1 << 17]byte + var x14649 [1 << 17]byte + var x14650 [1 << 17]byte + var x14651 [1 << 17]byte + var x14652 [1 << 17]byte + var x14653 [1 << 17]byte + var x14654 [1 << 17]byte + var x14655 [1 << 17]byte + var x14656 [1 << 17]byte + var x14657 [1 << 17]byte + var x14658 [1 << 17]byte + var x14659 [1 << 17]byte + var x14660 [1 << 17]byte + var x14661 [1 << 17]byte + var x14662 [1 << 17]byte + var x14663 [1 << 17]byte + var x14664 [1 << 17]byte + var x14665 [1 << 17]byte + var x14666 [1 << 17]byte + var x14667 [1 << 17]byte + var x14668 [1 << 17]byte + var x14669 [1 << 17]byte + var x14670 [1 << 17]byte + var x14671 [1 << 17]byte + var x14672 [1 << 17]byte + var x14673 [1 << 17]byte + var x14674 [1 << 17]byte + var x14675 [1 << 17]byte + var x14676 [1 << 17]byte + var x14677 [1 << 17]byte + var x14678 [1 << 17]byte + var x14679 [1 << 17]byte + var x14680 [1 << 17]byte + var x14681 [1 << 17]byte + var x14682 [1 << 17]byte + var x14683 [1 << 17]byte + var x14684 [1 << 17]byte + var x14685 [1 << 17]byte + var x14686 [1 << 17]byte + var x14687 [1 << 17]byte + var x14688 [1 << 17]byte + var x14689 [1 << 17]byte + var x14690 [1 << 17]byte + var x14691 [1 << 17]byte + var x14692 [1 << 17]byte + var x14693 [1 << 17]byte + var x14694 [1 << 17]byte + var x14695 [1 << 17]byte + var x14696 [1 << 17]byte + var x14697 [1 << 17]byte + var x14698 [1 << 17]byte + var x14699 [1 << 17]byte + var x14700 [1 << 17]byte + var x14701 [1 << 17]byte + var x14702 [1 << 17]byte + var x14703 [1 << 17]byte + var x14704 [1 << 17]byte + var x14705 [1 << 17]byte + var x14706 [1 << 17]byte + var x14707 [1 << 17]byte + var x14708 [1 << 17]byte + var x14709 [1 << 17]byte + var x14710 [1 << 17]byte + var x14711 [1 << 17]byte + var x14712 [1 << 17]byte + var x14713 [1 << 17]byte + var x14714 [1 << 17]byte + var x14715 [1 << 17]byte + var x14716 [1 << 17]byte + var x14717 [1 << 17]byte + var x14718 [1 << 17]byte + var x14719 [1 << 17]byte + var x14720 [1 << 17]byte + var x14721 [1 << 17]byte + var x14722 [1 << 17]byte + var x14723 [1 << 17]byte + var x14724 [1 << 17]byte + var x14725 [1 << 17]byte + var x14726 [1 << 17]byte + var x14727 [1 << 17]byte + var x14728 [1 << 17]byte + var x14729 [1 << 17]byte + var x14730 [1 << 17]byte + var x14731 [1 << 17]byte + var x14732 [1 << 17]byte + var x14733 [1 << 17]byte + var x14734 [1 << 17]byte + var x14735 [1 << 17]byte + var x14736 [1 << 17]byte + var x14737 [1 << 17]byte + var x14738 [1 << 17]byte + var x14739 [1 << 17]byte + var x14740 [1 << 17]byte + var x14741 [1 << 17]byte + var x14742 [1 << 17]byte + var x14743 [1 << 17]byte + var x14744 [1 << 17]byte + var x14745 [1 << 17]byte + var x14746 [1 << 17]byte + var x14747 [1 << 17]byte + var x14748 [1 << 17]byte + var x14749 [1 << 17]byte + var x14750 [1 << 17]byte + var x14751 [1 << 17]byte + var x14752 [1 << 17]byte + var x14753 [1 << 17]byte + var x14754 [1 << 17]byte + var x14755 [1 << 17]byte + var x14756 [1 << 17]byte + var x14757 [1 << 17]byte + var x14758 [1 << 17]byte + var x14759 [1 << 17]byte + var x14760 [1 << 17]byte + var x14761 [1 << 17]byte + var x14762 [1 << 17]byte + var x14763 [1 << 17]byte + var x14764 [1 << 17]byte + var x14765 [1 << 17]byte + var x14766 [1 << 17]byte + var x14767 [1 << 17]byte + var x14768 [1 << 17]byte + var x14769 [1 << 17]byte + var x14770 [1 << 17]byte + var x14771 [1 << 17]byte + var x14772 [1 << 17]byte + var x14773 [1 << 17]byte + var x14774 [1 << 17]byte + var x14775 [1 << 17]byte + var x14776 [1 << 17]byte + var x14777 [1 << 17]byte + var x14778 [1 << 17]byte + var x14779 [1 << 17]byte + var x14780 [1 << 17]byte + var x14781 [1 << 17]byte + var x14782 [1 << 17]byte + var x14783 [1 << 17]byte + var x14784 [1 << 17]byte + var x14785 [1 << 17]byte + var x14786 [1 << 17]byte + var x14787 [1 << 17]byte + var x14788 [1 << 17]byte + var x14789 [1 << 17]byte + var x14790 [1 << 17]byte + var x14791 [1 << 17]byte + var x14792 [1 << 17]byte + var x14793 [1 << 17]byte + var x14794 [1 << 17]byte + var x14795 [1 << 17]byte + var x14796 [1 << 17]byte + var x14797 [1 << 17]byte + var x14798 [1 << 17]byte + var x14799 [1 << 17]byte + var x14800 [1 << 17]byte + var x14801 [1 << 17]byte + var x14802 [1 << 17]byte + var x14803 [1 << 17]byte + var x14804 [1 << 17]byte + var x14805 [1 << 17]byte + var x14806 [1 << 17]byte + var x14807 [1 << 17]byte + var x14808 [1 << 17]byte + var x14809 [1 << 17]byte + var x14810 [1 << 17]byte + var x14811 [1 << 17]byte + var x14812 [1 << 17]byte + var x14813 [1 << 17]byte + var x14814 [1 << 17]byte + var x14815 [1 << 17]byte + var x14816 [1 << 17]byte + var x14817 [1 << 17]byte + var x14818 [1 << 17]byte + var x14819 [1 << 17]byte + var x14820 [1 << 17]byte + var x14821 [1 << 17]byte + var x14822 [1 << 17]byte + var x14823 [1 << 17]byte + var x14824 [1 << 17]byte + var x14825 [1 << 17]byte + var x14826 [1 << 17]byte + var x14827 [1 << 17]byte + var x14828 [1 << 17]byte + var x14829 [1 << 17]byte + var x14830 [1 << 17]byte + var x14831 [1 << 17]byte + var x14832 [1 << 17]byte + var x14833 [1 << 17]byte + var x14834 [1 << 17]byte + var x14835 [1 << 17]byte + var x14836 [1 << 17]byte + var x14837 [1 << 17]byte + var x14838 [1 << 17]byte + var x14839 [1 << 17]byte + var x14840 [1 << 17]byte + var x14841 [1 << 17]byte + var x14842 [1 << 17]byte + var x14843 [1 << 17]byte + var x14844 [1 << 17]byte + var x14845 [1 << 17]byte + var x14846 [1 << 17]byte + var x14847 [1 << 17]byte + var x14848 [1 << 17]byte + var x14849 [1 << 17]byte + var x14850 [1 << 17]byte + var x14851 [1 << 17]byte + var x14852 [1 << 17]byte + var x14853 [1 << 17]byte + var x14854 [1 << 17]byte + var x14855 [1 << 17]byte + var x14856 [1 << 17]byte + var x14857 [1 << 17]byte + var x14858 [1 << 17]byte + var x14859 [1 << 17]byte + var x14860 [1 << 17]byte + var x14861 [1 << 17]byte + var x14862 [1 << 17]byte + var x14863 [1 << 17]byte + var x14864 [1 << 17]byte + var x14865 [1 << 17]byte + var x14866 [1 << 17]byte + var x14867 [1 << 17]byte + var x14868 [1 << 17]byte + var x14869 [1 << 17]byte + var x14870 [1 << 17]byte + var x14871 [1 << 17]byte + var x14872 [1 << 17]byte + var x14873 [1 << 17]byte + var x14874 [1 << 17]byte + var x14875 [1 << 17]byte + var x14876 [1 << 17]byte + var x14877 [1 << 17]byte + var x14878 [1 << 17]byte + var x14879 [1 << 17]byte + var x14880 [1 << 17]byte + var x14881 [1 << 17]byte + var x14882 [1 << 17]byte + var x14883 [1 << 17]byte + var x14884 [1 << 17]byte + var x14885 [1 << 17]byte + var x14886 [1 << 17]byte + var x14887 [1 << 17]byte + var x14888 [1 << 17]byte + var x14889 [1 << 17]byte + var x14890 [1 << 17]byte + var x14891 [1 << 17]byte + var x14892 [1 << 17]byte + var x14893 [1 << 17]byte + var x14894 [1 << 17]byte + var x14895 [1 << 17]byte + var x14896 [1 << 17]byte + var x14897 [1 << 17]byte + var x14898 [1 << 17]byte + var x14899 [1 << 17]byte + var x14900 [1 << 17]byte + var x14901 [1 << 17]byte + var x14902 [1 << 17]byte + var x14903 [1 << 17]byte + var x14904 [1 << 17]byte + var x14905 [1 << 17]byte + var x14906 [1 << 17]byte + var x14907 [1 << 17]byte + var x14908 [1 << 17]byte + var x14909 [1 << 17]byte + var x14910 [1 << 17]byte + var x14911 [1 << 17]byte + var x14912 [1 << 17]byte + var x14913 [1 << 17]byte + var x14914 [1 << 17]byte + var x14915 [1 << 17]byte + var x14916 [1 << 17]byte + var x14917 [1 << 17]byte + var x14918 [1 << 17]byte + var x14919 [1 << 17]byte + var x14920 [1 << 17]byte + var x14921 [1 << 17]byte + var x14922 [1 << 17]byte + var x14923 [1 << 17]byte + var x14924 [1 << 17]byte + var x14925 [1 << 17]byte + var x14926 [1 << 17]byte + var x14927 [1 << 17]byte + var x14928 [1 << 17]byte + var x14929 [1 << 17]byte + var x14930 [1 << 17]byte + var x14931 [1 << 17]byte + var x14932 [1 << 17]byte + var x14933 [1 << 17]byte + var x14934 [1 << 17]byte + var x14935 [1 << 17]byte + var x14936 [1 << 17]byte + var x14937 [1 << 17]byte + var x14938 [1 << 17]byte + var x14939 [1 << 17]byte + var x14940 [1 << 17]byte + var x14941 [1 << 17]byte + var x14942 [1 << 17]byte + var x14943 [1 << 17]byte + var x14944 [1 << 17]byte + var x14945 [1 << 17]byte + var x14946 [1 << 17]byte + var x14947 [1 << 17]byte + var x14948 [1 << 17]byte + var x14949 [1 << 17]byte + var x14950 [1 << 17]byte + var x14951 [1 << 17]byte + var x14952 [1 << 17]byte + var x14953 [1 << 17]byte + var x14954 [1 << 17]byte + var x14955 [1 << 17]byte + var x14956 [1 << 17]byte + var x14957 [1 << 17]byte + var x14958 [1 << 17]byte + var x14959 [1 << 17]byte + var x14960 [1 << 17]byte + var x14961 [1 << 17]byte + var x14962 [1 << 17]byte + var x14963 [1 << 17]byte + var x14964 [1 << 17]byte + var x14965 [1 << 17]byte + var x14966 [1 << 17]byte + var x14967 [1 << 17]byte + var x14968 [1 << 17]byte + var x14969 [1 << 17]byte + var x14970 [1 << 17]byte + var x14971 [1 << 17]byte + var x14972 [1 << 17]byte + var x14973 [1 << 17]byte + var x14974 [1 << 17]byte + var x14975 [1 << 17]byte + var x14976 [1 << 17]byte + var x14977 [1 << 17]byte + var x14978 [1 << 17]byte + var x14979 [1 << 17]byte + var x14980 [1 << 17]byte + var x14981 [1 << 17]byte + var x14982 [1 << 17]byte + var x14983 [1 << 17]byte + var x14984 [1 << 17]byte + var x14985 [1 << 17]byte + var x14986 [1 << 17]byte + var x14987 [1 << 17]byte + var x14988 [1 << 17]byte + var x14989 [1 << 17]byte + var x14990 [1 << 17]byte + var x14991 [1 << 17]byte + var x14992 [1 << 17]byte + var x14993 [1 << 17]byte + var x14994 [1 << 17]byte + var x14995 [1 << 17]byte + var x14996 [1 << 17]byte + var x14997 [1 << 17]byte + var x14998 [1 << 17]byte + var x14999 [1 << 17]byte + var x15000 [1 << 17]byte + var x15001 [1 << 17]byte + var x15002 [1 << 17]byte + var x15003 [1 << 17]byte + var x15004 [1 << 17]byte + var x15005 [1 << 17]byte + var x15006 [1 << 17]byte + var x15007 [1 << 17]byte + var x15008 [1 << 17]byte + var x15009 [1 << 17]byte + var x15010 [1 << 17]byte + var x15011 [1 << 17]byte + var x15012 [1 << 17]byte + var x15013 [1 << 17]byte + var x15014 [1 << 17]byte + var x15015 [1 << 17]byte + var x15016 [1 << 17]byte + var x15017 [1 << 17]byte + var x15018 [1 << 17]byte + var x15019 [1 << 17]byte + var x15020 [1 << 17]byte + var x15021 [1 << 17]byte + var x15022 [1 << 17]byte + var x15023 [1 << 17]byte + var x15024 [1 << 17]byte + var x15025 [1 << 17]byte + var x15026 [1 << 17]byte + var x15027 [1 << 17]byte + var x15028 [1 << 17]byte + var x15029 [1 << 17]byte + var x15030 [1 << 17]byte + var x15031 [1 << 17]byte + var x15032 [1 << 17]byte + var x15033 [1 << 17]byte + var x15034 [1 << 17]byte + var x15035 [1 << 17]byte + var x15036 [1 << 17]byte + var x15037 [1 << 17]byte + var x15038 [1 << 17]byte + var x15039 [1 << 17]byte + var x15040 [1 << 17]byte + var x15041 [1 << 17]byte + var x15042 [1 << 17]byte + var x15043 [1 << 17]byte + var x15044 [1 << 17]byte + var x15045 [1 << 17]byte + var x15046 [1 << 17]byte + var x15047 [1 << 17]byte + var x15048 [1 << 17]byte + var x15049 [1 << 17]byte + var x15050 [1 << 17]byte + var x15051 [1 << 17]byte + var x15052 [1 << 17]byte + var x15053 [1 << 17]byte + var x15054 [1 << 17]byte + var x15055 [1 << 17]byte + var x15056 [1 << 17]byte + var x15057 [1 << 17]byte + var x15058 [1 << 17]byte + var x15059 [1 << 17]byte + var x15060 [1 << 17]byte + var x15061 [1 << 17]byte + var x15062 [1 << 17]byte + var x15063 [1 << 17]byte + var x15064 [1 << 17]byte + var x15065 [1 << 17]byte + var x15066 [1 << 17]byte + var x15067 [1 << 17]byte + var x15068 [1 << 17]byte + var x15069 [1 << 17]byte + var x15070 [1 << 17]byte + var x15071 [1 << 17]byte + var x15072 [1 << 17]byte + var x15073 [1 << 17]byte + var x15074 [1 << 17]byte + var x15075 [1 << 17]byte + var x15076 [1 << 17]byte + var x15077 [1 << 17]byte + var x15078 [1 << 17]byte + var x15079 [1 << 17]byte + var x15080 [1 << 17]byte + var x15081 [1 << 17]byte + var x15082 [1 << 17]byte + var x15083 [1 << 17]byte + var x15084 [1 << 17]byte + var x15085 [1 << 17]byte + var x15086 [1 << 17]byte + var x15087 [1 << 17]byte + var x15088 [1 << 17]byte + var x15089 [1 << 17]byte + var x15090 [1 << 17]byte + var x15091 [1 << 17]byte + var x15092 [1 << 17]byte + var x15093 [1 << 17]byte + var x15094 [1 << 17]byte + var x15095 [1 << 17]byte + var x15096 [1 << 17]byte + var x15097 [1 << 17]byte + var x15098 [1 << 17]byte + var x15099 [1 << 17]byte + var x15100 [1 << 17]byte + var x15101 [1 << 17]byte + var x15102 [1 << 17]byte + var x15103 [1 << 17]byte + var x15104 [1 << 17]byte + var x15105 [1 << 17]byte + var x15106 [1 << 17]byte + var x15107 [1 << 17]byte + var x15108 [1 << 17]byte + var x15109 [1 << 17]byte + var x15110 [1 << 17]byte + var x15111 [1 << 17]byte + var x15112 [1 << 17]byte + var x15113 [1 << 17]byte + var x15114 [1 << 17]byte + var x15115 [1 << 17]byte + var x15116 [1 << 17]byte + var x15117 [1 << 17]byte + var x15118 [1 << 17]byte + var x15119 [1 << 17]byte + var x15120 [1 << 17]byte + var x15121 [1 << 17]byte + var x15122 [1 << 17]byte + var x15123 [1 << 17]byte + var x15124 [1 << 17]byte + var x15125 [1 << 17]byte + var x15126 [1 << 17]byte + var x15127 [1 << 17]byte + var x15128 [1 << 17]byte + var x15129 [1 << 17]byte + var x15130 [1 << 17]byte + var x15131 [1 << 17]byte + var x15132 [1 << 17]byte + var x15133 [1 << 17]byte + var x15134 [1 << 17]byte + var x15135 [1 << 17]byte + var x15136 [1 << 17]byte + var x15137 [1 << 17]byte + var x15138 [1 << 17]byte + var x15139 [1 << 17]byte + var x15140 [1 << 17]byte + var x15141 [1 << 17]byte + var x15142 [1 << 17]byte + var x15143 [1 << 17]byte + var x15144 [1 << 17]byte + var x15145 [1 << 17]byte + var x15146 [1 << 17]byte + var x15147 [1 << 17]byte + var x15148 [1 << 17]byte + var x15149 [1 << 17]byte + var x15150 [1 << 17]byte + var x15151 [1 << 17]byte + var x15152 [1 << 17]byte + var x15153 [1 << 17]byte + var x15154 [1 << 17]byte + var x15155 [1 << 17]byte + var x15156 [1 << 17]byte + var x15157 [1 << 17]byte + var x15158 [1 << 17]byte + var x15159 [1 << 17]byte + var x15160 [1 << 17]byte + var x15161 [1 << 17]byte + var x15162 [1 << 17]byte + var x15163 [1 << 17]byte + var x15164 [1 << 17]byte + var x15165 [1 << 17]byte + var x15166 [1 << 17]byte + var x15167 [1 << 17]byte + var x15168 [1 << 17]byte + var x15169 [1 << 17]byte + var x15170 [1 << 17]byte + var x15171 [1 << 17]byte + var x15172 [1 << 17]byte + var x15173 [1 << 17]byte + var x15174 [1 << 17]byte + var x15175 [1 << 17]byte + var x15176 [1 << 17]byte + var x15177 [1 << 17]byte + var x15178 [1 << 17]byte + var x15179 [1 << 17]byte + var x15180 [1 << 17]byte + var x15181 [1 << 17]byte + var x15182 [1 << 17]byte + var x15183 [1 << 17]byte + var x15184 [1 << 17]byte + var x15185 [1 << 17]byte + var x15186 [1 << 17]byte + var x15187 [1 << 17]byte + var x15188 [1 << 17]byte + var x15189 [1 << 17]byte + var x15190 [1 << 17]byte + var x15191 [1 << 17]byte + var x15192 [1 << 17]byte + var x15193 [1 << 17]byte + var x15194 [1 << 17]byte + var x15195 [1 << 17]byte + var x15196 [1 << 17]byte + var x15197 [1 << 17]byte + var x15198 [1 << 17]byte + var x15199 [1 << 17]byte + var x15200 [1 << 17]byte + var x15201 [1 << 17]byte + var x15202 [1 << 17]byte + var x15203 [1 << 17]byte + var x15204 [1 << 17]byte + var x15205 [1 << 17]byte + var x15206 [1 << 17]byte + var x15207 [1 << 17]byte + var x15208 [1 << 17]byte + var x15209 [1 << 17]byte + var x15210 [1 << 17]byte + var x15211 [1 << 17]byte + var x15212 [1 << 17]byte + var x15213 [1 << 17]byte + var x15214 [1 << 17]byte + var x15215 [1 << 17]byte + var x15216 [1 << 17]byte + var x15217 [1 << 17]byte + var x15218 [1 << 17]byte + var x15219 [1 << 17]byte + var x15220 [1 << 17]byte + var x15221 [1 << 17]byte + var x15222 [1 << 17]byte + var x15223 [1 << 17]byte + var x15224 [1 << 17]byte + var x15225 [1 << 17]byte + var x15226 [1 << 17]byte + var x15227 [1 << 17]byte + var x15228 [1 << 17]byte + var x15229 [1 << 17]byte + var x15230 [1 << 17]byte + var x15231 [1 << 17]byte + var x15232 [1 << 17]byte + var x15233 [1 << 17]byte + var x15234 [1 << 17]byte + var x15235 [1 << 17]byte + var x15236 [1 << 17]byte + var x15237 [1 << 17]byte + var x15238 [1 << 17]byte + var x15239 [1 << 17]byte + var x15240 [1 << 17]byte + var x15241 [1 << 17]byte + var x15242 [1 << 17]byte + var x15243 [1 << 17]byte + var x15244 [1 << 17]byte + var x15245 [1 << 17]byte + var x15246 [1 << 17]byte + var x15247 [1 << 17]byte + var x15248 [1 << 17]byte + var x15249 [1 << 17]byte + var x15250 [1 << 17]byte + var x15251 [1 << 17]byte + var x15252 [1 << 17]byte + var x15253 [1 << 17]byte + var x15254 [1 << 17]byte + var x15255 [1 << 17]byte + var x15256 [1 << 17]byte + var x15257 [1 << 17]byte + var x15258 [1 << 17]byte + var x15259 [1 << 17]byte + var x15260 [1 << 17]byte + var x15261 [1 << 17]byte + var x15262 [1 << 17]byte + var x15263 [1 << 17]byte + var x15264 [1 << 17]byte + var x15265 [1 << 17]byte + var x15266 [1 << 17]byte + var x15267 [1 << 17]byte + var x15268 [1 << 17]byte + var x15269 [1 << 17]byte + var x15270 [1 << 17]byte + var x15271 [1 << 17]byte + var x15272 [1 << 17]byte + var x15273 [1 << 17]byte + var x15274 [1 << 17]byte + var x15275 [1 << 17]byte + var x15276 [1 << 17]byte + var x15277 [1 << 17]byte + var x15278 [1 << 17]byte + var x15279 [1 << 17]byte + var x15280 [1 << 17]byte + var x15281 [1 << 17]byte + var x15282 [1 << 17]byte + var x15283 [1 << 17]byte + var x15284 [1 << 17]byte + var x15285 [1 << 17]byte + var x15286 [1 << 17]byte + var x15287 [1 << 17]byte + var x15288 [1 << 17]byte + var x15289 [1 << 17]byte + var x15290 [1 << 17]byte + var x15291 [1 << 17]byte + var x15292 [1 << 17]byte + var x15293 [1 << 17]byte + var x15294 [1 << 17]byte + var x15295 [1 << 17]byte + var x15296 [1 << 17]byte + var x15297 [1 << 17]byte + var x15298 [1 << 17]byte + var x15299 [1 << 17]byte + var x15300 [1 << 17]byte + var x15301 [1 << 17]byte + var x15302 [1 << 17]byte + var x15303 [1 << 17]byte + var x15304 [1 << 17]byte + var x15305 [1 << 17]byte + var x15306 [1 << 17]byte + var x15307 [1 << 17]byte + var x15308 [1 << 17]byte + var x15309 [1 << 17]byte + var x15310 [1 << 17]byte + var x15311 [1 << 17]byte + var x15312 [1 << 17]byte + var x15313 [1 << 17]byte + var x15314 [1 << 17]byte + var x15315 [1 << 17]byte + var x15316 [1 << 17]byte + var x15317 [1 << 17]byte + var x15318 [1 << 17]byte + var x15319 [1 << 17]byte + var x15320 [1 << 17]byte + var x15321 [1 << 17]byte + var x15322 [1 << 17]byte + var x15323 [1 << 17]byte + var x15324 [1 << 17]byte + var x15325 [1 << 17]byte + var x15326 [1 << 17]byte + var x15327 [1 << 17]byte + var x15328 [1 << 17]byte + var x15329 [1 << 17]byte + var x15330 [1 << 17]byte + var x15331 [1 << 17]byte + var x15332 [1 << 17]byte + var x15333 [1 << 17]byte + var x15334 [1 << 17]byte + var x15335 [1 << 17]byte + var x15336 [1 << 17]byte + var x15337 [1 << 17]byte + var x15338 [1 << 17]byte + var x15339 [1 << 17]byte + var x15340 [1 << 17]byte + var x15341 [1 << 17]byte + var x15342 [1 << 17]byte + var x15343 [1 << 17]byte + var x15344 [1 << 17]byte + var x15345 [1 << 17]byte + var x15346 [1 << 17]byte + var x15347 [1 << 17]byte + var x15348 [1 << 17]byte + var x15349 [1 << 17]byte + var x15350 [1 << 17]byte + var x15351 [1 << 17]byte + var x15352 [1 << 17]byte + var x15353 [1 << 17]byte + var x15354 [1 << 17]byte + var x15355 [1 << 17]byte + var x15356 [1 << 17]byte + var x15357 [1 << 17]byte + var x15358 [1 << 17]byte + var x15359 [1 << 17]byte + var x15360 [1 << 17]byte + var x15361 [1 << 17]byte + var x15362 [1 << 17]byte + var x15363 [1 << 17]byte + var x15364 [1 << 17]byte + var x15365 [1 << 17]byte + var x15366 [1 << 17]byte + var x15367 [1 << 17]byte + var x15368 [1 << 17]byte + var x15369 [1 << 17]byte + var x15370 [1 << 17]byte + var x15371 [1 << 17]byte + var x15372 [1 << 17]byte + var x15373 [1 << 17]byte + var x15374 [1 << 17]byte + var x15375 [1 << 17]byte + var x15376 [1 << 17]byte + var x15377 [1 << 17]byte + var x15378 [1 << 17]byte + var x15379 [1 << 17]byte + var x15380 [1 << 17]byte + var x15381 [1 << 17]byte + var x15382 [1 << 17]byte + var x15383 [1 << 17]byte + var x15384 [1 << 17]byte + var x15385 [1 << 17]byte + var x15386 [1 << 17]byte + var x15387 [1 << 17]byte + var x15388 [1 << 17]byte + var x15389 [1 << 17]byte + var x15390 [1 << 17]byte + var x15391 [1 << 17]byte + var x15392 [1 << 17]byte + var x15393 [1 << 17]byte + var x15394 [1 << 17]byte + var x15395 [1 << 17]byte + var x15396 [1 << 17]byte + var x15397 [1 << 17]byte + var x15398 [1 << 17]byte + var x15399 [1 << 17]byte + var x15400 [1 << 17]byte + var x15401 [1 << 17]byte + var x15402 [1 << 17]byte + var x15403 [1 << 17]byte + var x15404 [1 << 17]byte + var x15405 [1 << 17]byte + var x15406 [1 << 17]byte + var x15407 [1 << 17]byte + var x15408 [1 << 17]byte + var x15409 [1 << 17]byte + var x15410 [1 << 17]byte + var x15411 [1 << 17]byte + var x15412 [1 << 17]byte + var x15413 [1 << 17]byte + var x15414 [1 << 17]byte + var x15415 [1 << 17]byte + var x15416 [1 << 17]byte + var x15417 [1 << 17]byte + var x15418 [1 << 17]byte + var x15419 [1 << 17]byte + var x15420 [1 << 17]byte + var x15421 [1 << 17]byte + var x15422 [1 << 17]byte + var x15423 [1 << 17]byte + var x15424 [1 << 17]byte + var x15425 [1 << 17]byte + var x15426 [1 << 17]byte + var x15427 [1 << 17]byte + var x15428 [1 << 17]byte + var x15429 [1 << 17]byte + var x15430 [1 << 17]byte + var x15431 [1 << 17]byte + var x15432 [1 << 17]byte + var x15433 [1 << 17]byte + var x15434 [1 << 17]byte + var x15435 [1 << 17]byte + var x15436 [1 << 17]byte + var x15437 [1 << 17]byte + var x15438 [1 << 17]byte + var x15439 [1 << 17]byte + var x15440 [1 << 17]byte + var x15441 [1 << 17]byte + var x15442 [1 << 17]byte + var x15443 [1 << 17]byte + var x15444 [1 << 17]byte + var x15445 [1 << 17]byte + var x15446 [1 << 17]byte + var x15447 [1 << 17]byte + var x15448 [1 << 17]byte + var x15449 [1 << 17]byte + var x15450 [1 << 17]byte + var x15451 [1 << 17]byte + var x15452 [1 << 17]byte + var x15453 [1 << 17]byte + var x15454 [1 << 17]byte + var x15455 [1 << 17]byte + var x15456 [1 << 17]byte + var x15457 [1 << 17]byte + var x15458 [1 << 17]byte + var x15459 [1 << 17]byte + var x15460 [1 << 17]byte + var x15461 [1 << 17]byte + var x15462 [1 << 17]byte + var x15463 [1 << 17]byte + var x15464 [1 << 17]byte + var x15465 [1 << 17]byte + var x15466 [1 << 17]byte + var x15467 [1 << 17]byte + var x15468 [1 << 17]byte + var x15469 [1 << 17]byte + var x15470 [1 << 17]byte + var x15471 [1 << 17]byte + var x15472 [1 << 17]byte + var x15473 [1 << 17]byte + var x15474 [1 << 17]byte + var x15475 [1 << 17]byte + var x15476 [1 << 17]byte + var x15477 [1 << 17]byte + var x15478 [1 << 17]byte + var x15479 [1 << 17]byte + var x15480 [1 << 17]byte + var x15481 [1 << 17]byte + var x15482 [1 << 17]byte + var x15483 [1 << 17]byte + var x15484 [1 << 17]byte + var x15485 [1 << 17]byte + var x15486 [1 << 17]byte + var x15487 [1 << 17]byte + var x15488 [1 << 17]byte + var x15489 [1 << 17]byte + var x15490 [1 << 17]byte + var x15491 [1 << 17]byte + var x15492 [1 << 17]byte + var x15493 [1 << 17]byte + var x15494 [1 << 17]byte + var x15495 [1 << 17]byte + var x15496 [1 << 17]byte + var x15497 [1 << 17]byte + var x15498 [1 << 17]byte + var x15499 [1 << 17]byte + var x15500 [1 << 17]byte + var x15501 [1 << 17]byte + var x15502 [1 << 17]byte + var x15503 [1 << 17]byte + var x15504 [1 << 17]byte + var x15505 [1 << 17]byte + var x15506 [1 << 17]byte + var x15507 [1 << 17]byte + var x15508 [1 << 17]byte + var x15509 [1 << 17]byte + var x15510 [1 << 17]byte + var x15511 [1 << 17]byte + var x15512 [1 << 17]byte + var x15513 [1 << 17]byte + var x15514 [1 << 17]byte + var x15515 [1 << 17]byte + var x15516 [1 << 17]byte + var x15517 [1 << 17]byte + var x15518 [1 << 17]byte + var x15519 [1 << 17]byte + var x15520 [1 << 17]byte + var x15521 [1 << 17]byte + var x15522 [1 << 17]byte + var x15523 [1 << 17]byte + var x15524 [1 << 17]byte + var x15525 [1 << 17]byte + var x15526 [1 << 17]byte + var x15527 [1 << 17]byte + var x15528 [1 << 17]byte + var x15529 [1 << 17]byte + var x15530 [1 << 17]byte + var x15531 [1 << 17]byte + var x15532 [1 << 17]byte + var x15533 [1 << 17]byte + var x15534 [1 << 17]byte + var x15535 [1 << 17]byte + var x15536 [1 << 17]byte + var x15537 [1 << 17]byte + var x15538 [1 << 17]byte + var x15539 [1 << 17]byte + var x15540 [1 << 17]byte + var x15541 [1 << 17]byte + var x15542 [1 << 17]byte + var x15543 [1 << 17]byte + var x15544 [1 << 17]byte + var x15545 [1 << 17]byte + var x15546 [1 << 17]byte + var x15547 [1 << 17]byte + var x15548 [1 << 17]byte + var x15549 [1 << 17]byte + var x15550 [1 << 17]byte + var x15551 [1 << 17]byte + var x15552 [1 << 17]byte + var x15553 [1 << 17]byte + var x15554 [1 << 17]byte + var x15555 [1 << 17]byte + var x15556 [1 << 17]byte + var x15557 [1 << 17]byte + var x15558 [1 << 17]byte + var x15559 [1 << 17]byte + var x15560 [1 << 17]byte + var x15561 [1 << 17]byte + var x15562 [1 << 17]byte + var x15563 [1 << 17]byte + var x15564 [1 << 17]byte + var x15565 [1 << 17]byte + var x15566 [1 << 17]byte + var x15567 [1 << 17]byte + var x15568 [1 << 17]byte + var x15569 [1 << 17]byte + var x15570 [1 << 17]byte + var x15571 [1 << 17]byte + var x15572 [1 << 17]byte + var x15573 [1 << 17]byte + var x15574 [1 << 17]byte + var x15575 [1 << 17]byte + var x15576 [1 << 17]byte + var x15577 [1 << 17]byte + var x15578 [1 << 17]byte + var x15579 [1 << 17]byte + var x15580 [1 << 17]byte + var x15581 [1 << 17]byte + var x15582 [1 << 17]byte + var x15583 [1 << 17]byte + var x15584 [1 << 17]byte + var x15585 [1 << 17]byte + var x15586 [1 << 17]byte + var x15587 [1 << 17]byte + var x15588 [1 << 17]byte + var x15589 [1 << 17]byte + var x15590 [1 << 17]byte + var x15591 [1 << 17]byte + var x15592 [1 << 17]byte + var x15593 [1 << 17]byte + var x15594 [1 << 17]byte + var x15595 [1 << 17]byte + var x15596 [1 << 17]byte + var x15597 [1 << 17]byte + var x15598 [1 << 17]byte + var x15599 [1 << 17]byte + var x15600 [1 << 17]byte + var x15601 [1 << 17]byte + var x15602 [1 << 17]byte + var x15603 [1 << 17]byte + var x15604 [1 << 17]byte + var x15605 [1 << 17]byte + var x15606 [1 << 17]byte + var x15607 [1 << 17]byte + var x15608 [1 << 17]byte + var x15609 [1 << 17]byte + var x15610 [1 << 17]byte + var x15611 [1 << 17]byte + var x15612 [1 << 17]byte + var x15613 [1 << 17]byte + var x15614 [1 << 17]byte + var x15615 [1 << 17]byte + var x15616 [1 << 17]byte + var x15617 [1 << 17]byte + var x15618 [1 << 17]byte + var x15619 [1 << 17]byte + var x15620 [1 << 17]byte + var x15621 [1 << 17]byte + var x15622 [1 << 17]byte + var x15623 [1 << 17]byte + var x15624 [1 << 17]byte + var x15625 [1 << 17]byte + var x15626 [1 << 17]byte + var x15627 [1 << 17]byte + var x15628 [1 << 17]byte + var x15629 [1 << 17]byte + var x15630 [1 << 17]byte + var x15631 [1 << 17]byte + var x15632 [1 << 17]byte + var x15633 [1 << 17]byte + var x15634 [1 << 17]byte + var x15635 [1 << 17]byte + var x15636 [1 << 17]byte + var x15637 [1 << 17]byte + var x15638 [1 << 17]byte + var x15639 [1 << 17]byte + var x15640 [1 << 17]byte + var x15641 [1 << 17]byte + var x15642 [1 << 17]byte + var x15643 [1 << 17]byte + var x15644 [1 << 17]byte + var x15645 [1 << 17]byte + var x15646 [1 << 17]byte + var x15647 [1 << 17]byte + var x15648 [1 << 17]byte + var x15649 [1 << 17]byte + var x15650 [1 << 17]byte + var x15651 [1 << 17]byte + var x15652 [1 << 17]byte + var x15653 [1 << 17]byte + var x15654 [1 << 17]byte + var x15655 [1 << 17]byte + var x15656 [1 << 17]byte + var x15657 [1 << 17]byte + var x15658 [1 << 17]byte + var x15659 [1 << 17]byte + var x15660 [1 << 17]byte + var x15661 [1 << 17]byte + var x15662 [1 << 17]byte + var x15663 [1 << 17]byte + var x15664 [1 << 17]byte + var x15665 [1 << 17]byte + var x15666 [1 << 17]byte + var x15667 [1 << 17]byte + var x15668 [1 << 17]byte + var x15669 [1 << 17]byte + var x15670 [1 << 17]byte + var x15671 [1 << 17]byte + var x15672 [1 << 17]byte + var x15673 [1 << 17]byte + var x15674 [1 << 17]byte + var x15675 [1 << 17]byte + var x15676 [1 << 17]byte + var x15677 [1 << 17]byte + var x15678 [1 << 17]byte + var x15679 [1 << 17]byte + var x15680 [1 << 17]byte + var x15681 [1 << 17]byte + var x15682 [1 << 17]byte + var x15683 [1 << 17]byte + var x15684 [1 << 17]byte + var x15685 [1 << 17]byte + var x15686 [1 << 17]byte + var x15687 [1 << 17]byte + var x15688 [1 << 17]byte + var x15689 [1 << 17]byte + var x15690 [1 << 17]byte + var x15691 [1 << 17]byte + var x15692 [1 << 17]byte + var x15693 [1 << 17]byte + var x15694 [1 << 17]byte + var x15695 [1 << 17]byte + var x15696 [1 << 17]byte + var x15697 [1 << 17]byte + var x15698 [1 << 17]byte + var x15699 [1 << 17]byte + var x15700 [1 << 17]byte + var x15701 [1 << 17]byte + var x15702 [1 << 17]byte + var x15703 [1 << 17]byte + var x15704 [1 << 17]byte + var x15705 [1 << 17]byte + var x15706 [1 << 17]byte + var x15707 [1 << 17]byte + var x15708 [1 << 17]byte + var x15709 [1 << 17]byte + var x15710 [1 << 17]byte + var x15711 [1 << 17]byte + var x15712 [1 << 17]byte + var x15713 [1 << 17]byte + var x15714 [1 << 17]byte + var x15715 [1 << 17]byte + var x15716 [1 << 17]byte + var x15717 [1 << 17]byte + var x15718 [1 << 17]byte + var x15719 [1 << 17]byte + var x15720 [1 << 17]byte + var x15721 [1 << 17]byte + var x15722 [1 << 17]byte + var x15723 [1 << 17]byte + var x15724 [1 << 17]byte + var x15725 [1 << 17]byte + var x15726 [1 << 17]byte + var x15727 [1 << 17]byte + var x15728 [1 << 17]byte + var x15729 [1 << 17]byte + var x15730 [1 << 17]byte + var x15731 [1 << 17]byte + var x15732 [1 << 17]byte + var x15733 [1 << 17]byte + var x15734 [1 << 17]byte + var x15735 [1 << 17]byte + var x15736 [1 << 17]byte + var x15737 [1 << 17]byte + var x15738 [1 << 17]byte + var x15739 [1 << 17]byte + var x15740 [1 << 17]byte + var x15741 [1 << 17]byte + var x15742 [1 << 17]byte + var x15743 [1 << 17]byte + var x15744 [1 << 17]byte + var x15745 [1 << 17]byte + var x15746 [1 << 17]byte + var x15747 [1 << 17]byte + var x15748 [1 << 17]byte + var x15749 [1 << 17]byte + var x15750 [1 << 17]byte + var x15751 [1 << 17]byte + var x15752 [1 << 17]byte + var x15753 [1 << 17]byte + var x15754 [1 << 17]byte + var x15755 [1 << 17]byte + var x15756 [1 << 17]byte + var x15757 [1 << 17]byte + var x15758 [1 << 17]byte + var x15759 [1 << 17]byte + var x15760 [1 << 17]byte + var x15761 [1 << 17]byte + var x15762 [1 << 17]byte + var x15763 [1 << 17]byte + var x15764 [1 << 17]byte + var x15765 [1 << 17]byte + var x15766 [1 << 17]byte + var x15767 [1 << 17]byte + var x15768 [1 << 17]byte + var x15769 [1 << 17]byte + var x15770 [1 << 17]byte + var x15771 [1 << 17]byte + var x15772 [1 << 17]byte + var x15773 [1 << 17]byte + var x15774 [1 << 17]byte + var x15775 [1 << 17]byte + var x15776 [1 << 17]byte + var x15777 [1 << 17]byte + var x15778 [1 << 17]byte + var x15779 [1 << 17]byte + var x15780 [1 << 17]byte + var x15781 [1 << 17]byte + var x15782 [1 << 17]byte + var x15783 [1 << 17]byte + var x15784 [1 << 17]byte + var x15785 [1 << 17]byte + var x15786 [1 << 17]byte + var x15787 [1 << 17]byte + var x15788 [1 << 17]byte + var x15789 [1 << 17]byte + var x15790 [1 << 17]byte + var x15791 [1 << 17]byte + var x15792 [1 << 17]byte + var x15793 [1 << 17]byte + var x15794 [1 << 17]byte + var x15795 [1 << 17]byte + var x15796 [1 << 17]byte + var x15797 [1 << 17]byte + var x15798 [1 << 17]byte + var x15799 [1 << 17]byte + var x15800 [1 << 17]byte + var x15801 [1 << 17]byte + var x15802 [1 << 17]byte + var x15803 [1 << 17]byte + var x15804 [1 << 17]byte + var x15805 [1 << 17]byte + var x15806 [1 << 17]byte + var x15807 [1 << 17]byte + var x15808 [1 << 17]byte + var x15809 [1 << 17]byte + var x15810 [1 << 17]byte + var x15811 [1 << 17]byte + var x15812 [1 << 17]byte + var x15813 [1 << 17]byte + var x15814 [1 << 17]byte + var x15815 [1 << 17]byte + var x15816 [1 << 17]byte + var x15817 [1 << 17]byte + var x15818 [1 << 17]byte + var x15819 [1 << 17]byte + var x15820 [1 << 17]byte + var x15821 [1 << 17]byte + var x15822 [1 << 17]byte + var x15823 [1 << 17]byte + var x15824 [1 << 17]byte + var x15825 [1 << 17]byte + var x15826 [1 << 17]byte + var x15827 [1 << 17]byte + var x15828 [1 << 17]byte + var x15829 [1 << 17]byte + var x15830 [1 << 17]byte + var x15831 [1 << 17]byte + var x15832 [1 << 17]byte + var x15833 [1 << 17]byte + var x15834 [1 << 17]byte + var x15835 [1 << 17]byte + var x15836 [1 << 17]byte + var x15837 [1 << 17]byte + var x15838 [1 << 17]byte + var x15839 [1 << 17]byte + var x15840 [1 << 17]byte + var x15841 [1 << 17]byte + var x15842 [1 << 17]byte + var x15843 [1 << 17]byte + var x15844 [1 << 17]byte + var x15845 [1 << 17]byte + var x15846 [1 << 17]byte + var x15847 [1 << 17]byte + var x15848 [1 << 17]byte + var x15849 [1 << 17]byte + var x15850 [1 << 17]byte + var x15851 [1 << 17]byte + var x15852 [1 << 17]byte + var x15853 [1 << 17]byte + var x15854 [1 << 17]byte + var x15855 [1 << 17]byte + var x15856 [1 << 17]byte + var x15857 [1 << 17]byte + var x15858 [1 << 17]byte + var x15859 [1 << 17]byte + var x15860 [1 << 17]byte + var x15861 [1 << 17]byte + var x15862 [1 << 17]byte + var x15863 [1 << 17]byte + var x15864 [1 << 17]byte + var x15865 [1 << 17]byte + var x15866 [1 << 17]byte + var x15867 [1 << 17]byte + var x15868 [1 << 17]byte + var x15869 [1 << 17]byte + var x15870 [1 << 17]byte + var x15871 [1 << 17]byte + var x15872 [1 << 17]byte + var x15873 [1 << 17]byte + var x15874 [1 << 17]byte + var x15875 [1 << 17]byte + var x15876 [1 << 17]byte + var x15877 [1 << 17]byte + var x15878 [1 << 17]byte + var x15879 [1 << 17]byte + var x15880 [1 << 17]byte + var x15881 [1 << 17]byte + var x15882 [1 << 17]byte + var x15883 [1 << 17]byte + var x15884 [1 << 17]byte + var x15885 [1 << 17]byte + var x15886 [1 << 17]byte + var x15887 [1 << 17]byte + var x15888 [1 << 17]byte + var x15889 [1 << 17]byte + var x15890 [1 << 17]byte + var x15891 [1 << 17]byte + var x15892 [1 << 17]byte + var x15893 [1 << 17]byte + var x15894 [1 << 17]byte + var x15895 [1 << 17]byte + var x15896 [1 << 17]byte + var x15897 [1 << 17]byte + var x15898 [1 << 17]byte + var x15899 [1 << 17]byte + var x15900 [1 << 17]byte + var x15901 [1 << 17]byte + var x15902 [1 << 17]byte + var x15903 [1 << 17]byte + var x15904 [1 << 17]byte + var x15905 [1 << 17]byte + var x15906 [1 << 17]byte + var x15907 [1 << 17]byte + var x15908 [1 << 17]byte + var x15909 [1 << 17]byte + var x15910 [1 << 17]byte + var x15911 [1 << 17]byte + var x15912 [1 << 17]byte + var x15913 [1 << 17]byte + var x15914 [1 << 17]byte + var x15915 [1 << 17]byte + var x15916 [1 << 17]byte + var x15917 [1 << 17]byte + var x15918 [1 << 17]byte + var x15919 [1 << 17]byte + var x15920 [1 << 17]byte + var x15921 [1 << 17]byte + var x15922 [1 << 17]byte + var x15923 [1 << 17]byte + var x15924 [1 << 17]byte + var x15925 [1 << 17]byte + var x15926 [1 << 17]byte + var x15927 [1 << 17]byte + var x15928 [1 << 17]byte + var x15929 [1 << 17]byte + var x15930 [1 << 17]byte + var x15931 [1 << 17]byte + var x15932 [1 << 17]byte + var x15933 [1 << 17]byte + var x15934 [1 << 17]byte + var x15935 [1 << 17]byte + var x15936 [1 << 17]byte + var x15937 [1 << 17]byte + var x15938 [1 << 17]byte + var x15939 [1 << 17]byte + var x15940 [1 << 17]byte + var x15941 [1 << 17]byte + var x15942 [1 << 17]byte + var x15943 [1 << 17]byte + var x15944 [1 << 17]byte + var x15945 [1 << 17]byte + var x15946 [1 << 17]byte + var x15947 [1 << 17]byte + var x15948 [1 << 17]byte + var x15949 [1 << 17]byte + var x15950 [1 << 17]byte + var x15951 [1 << 17]byte + var x15952 [1 << 17]byte + var x15953 [1 << 17]byte + var x15954 [1 << 17]byte + var x15955 [1 << 17]byte + var x15956 [1 << 17]byte + var x15957 [1 << 17]byte + var x15958 [1 << 17]byte + var x15959 [1 << 17]byte + var x15960 [1 << 17]byte + var x15961 [1 << 17]byte + var x15962 [1 << 17]byte + var x15963 [1 << 17]byte + var x15964 [1 << 17]byte + var x15965 [1 << 17]byte + var x15966 [1 << 17]byte + var x15967 [1 << 17]byte + var x15968 [1 << 17]byte + var x15969 [1 << 17]byte + var x15970 [1 << 17]byte + var x15971 [1 << 17]byte + var x15972 [1 << 17]byte + var x15973 [1 << 17]byte + var x15974 [1 << 17]byte + var x15975 [1 << 17]byte + var x15976 [1 << 17]byte + var x15977 [1 << 17]byte + var x15978 [1 << 17]byte + var x15979 [1 << 17]byte + var x15980 [1 << 17]byte + var x15981 [1 << 17]byte + var x15982 [1 << 17]byte + var x15983 [1 << 17]byte + var x15984 [1 << 17]byte + var x15985 [1 << 17]byte + var x15986 [1 << 17]byte + var x15987 [1 << 17]byte + var x15988 [1 << 17]byte + var x15989 [1 << 17]byte + var x15990 [1 << 17]byte + var x15991 [1 << 17]byte + var x15992 [1 << 17]byte + var x15993 [1 << 17]byte + var x15994 [1 << 17]byte + var x15995 [1 << 17]byte + var x15996 [1 << 17]byte + var x15997 [1 << 17]byte + var x15998 [1 << 17]byte + var x15999 [1 << 17]byte + var x16000 [1 << 17]byte + var x16001 [1 << 17]byte + var x16002 [1 << 17]byte + var x16003 [1 << 17]byte + var x16004 [1 << 17]byte + var x16005 [1 << 17]byte + var x16006 [1 << 17]byte + var x16007 [1 << 17]byte + var x16008 [1 << 17]byte + var x16009 [1 << 17]byte + var x16010 [1 << 17]byte + var x16011 [1 << 17]byte + var x16012 [1 << 17]byte + var x16013 [1 << 17]byte + var x16014 [1 << 17]byte + var x16015 [1 << 17]byte + var x16016 [1 << 17]byte + var x16017 [1 << 17]byte + var x16018 [1 << 17]byte + var x16019 [1 << 17]byte + var x16020 [1 << 17]byte + var x16021 [1 << 17]byte + var x16022 [1 << 17]byte + var x16023 [1 << 17]byte + var x16024 [1 << 17]byte + var x16025 [1 << 17]byte + var x16026 [1 << 17]byte + var x16027 [1 << 17]byte + var x16028 [1 << 17]byte + var x16029 [1 << 17]byte + var x16030 [1 << 17]byte + var x16031 [1 << 17]byte + var x16032 [1 << 17]byte + var x16033 [1 << 17]byte + var x16034 [1 << 17]byte + var x16035 [1 << 17]byte + var x16036 [1 << 17]byte + var x16037 [1 << 17]byte + var x16038 [1 << 17]byte + var x16039 [1 << 17]byte + var x16040 [1 << 17]byte + var x16041 [1 << 17]byte + var x16042 [1 << 17]byte + var x16043 [1 << 17]byte + var x16044 [1 << 17]byte + var x16045 [1 << 17]byte + var x16046 [1 << 17]byte + var x16047 [1 << 17]byte + var x16048 [1 << 17]byte + var x16049 [1 << 17]byte + var x16050 [1 << 17]byte + var x16051 [1 << 17]byte + var x16052 [1 << 17]byte + var x16053 [1 << 17]byte + var x16054 [1 << 17]byte + var x16055 [1 << 17]byte + var x16056 [1 << 17]byte + var x16057 [1 << 17]byte + var x16058 [1 << 17]byte + var x16059 [1 << 17]byte + var x16060 [1 << 17]byte + var x16061 [1 << 17]byte + var x16062 [1 << 17]byte + var x16063 [1 << 17]byte + var x16064 [1 << 17]byte + var x16065 [1 << 17]byte + var x16066 [1 << 17]byte + var x16067 [1 << 17]byte + var x16068 [1 << 17]byte + var x16069 [1 << 17]byte + var x16070 [1 << 17]byte + var x16071 [1 << 17]byte + var x16072 [1 << 17]byte + var x16073 [1 << 17]byte + var x16074 [1 << 17]byte + var x16075 [1 << 17]byte + var x16076 [1 << 17]byte + var x16077 [1 << 17]byte + var x16078 [1 << 17]byte + var x16079 [1 << 17]byte + var x16080 [1 << 17]byte + var x16081 [1 << 17]byte + var x16082 [1 << 17]byte + var x16083 [1 << 17]byte + var x16084 [1 << 17]byte + var x16085 [1 << 17]byte + var x16086 [1 << 17]byte + var x16087 [1 << 17]byte + var x16088 [1 << 17]byte + var x16089 [1 << 17]byte + var x16090 [1 << 17]byte + var x16091 [1 << 17]byte + var x16092 [1 << 17]byte + var x16093 [1 << 17]byte + var x16094 [1 << 17]byte + var x16095 [1 << 17]byte + var x16096 [1 << 17]byte + var x16097 [1 << 17]byte + var x16098 [1 << 17]byte + var x16099 [1 << 17]byte + var x16100 [1 << 17]byte + var x16101 [1 << 17]byte + var x16102 [1 << 17]byte + var x16103 [1 << 17]byte + var x16104 [1 << 17]byte + var x16105 [1 << 17]byte + var x16106 [1 << 17]byte + var x16107 [1 << 17]byte + var x16108 [1 << 17]byte + var x16109 [1 << 17]byte + var x16110 [1 << 17]byte + var x16111 [1 << 17]byte + var x16112 [1 << 17]byte + var x16113 [1 << 17]byte + var x16114 [1 << 17]byte + var x16115 [1 << 17]byte + var x16116 [1 << 17]byte + var x16117 [1 << 17]byte + var x16118 [1 << 17]byte + var x16119 [1 << 17]byte + var x16120 [1 << 17]byte + var x16121 [1 << 17]byte + var x16122 [1 << 17]byte + var x16123 [1 << 17]byte + var x16124 [1 << 17]byte + var x16125 [1 << 17]byte + var x16126 [1 << 17]byte + var x16127 [1 << 17]byte + var x16128 [1 << 17]byte + var x16129 [1 << 17]byte + var x16130 [1 << 17]byte + var x16131 [1 << 17]byte + var x16132 [1 << 17]byte + var x16133 [1 << 17]byte + var x16134 [1 << 17]byte + var x16135 [1 << 17]byte + var x16136 [1 << 17]byte + var x16137 [1 << 17]byte + var x16138 [1 << 17]byte + var x16139 [1 << 17]byte + var x16140 [1 << 17]byte + var x16141 [1 << 17]byte + var x16142 [1 << 17]byte + var x16143 [1 << 17]byte + var x16144 [1 << 17]byte + var x16145 [1 << 17]byte + var x16146 [1 << 17]byte + var x16147 [1 << 17]byte + var x16148 [1 << 17]byte + var x16149 [1 << 17]byte + var x16150 [1 << 17]byte + var x16151 [1 << 17]byte + var x16152 [1 << 17]byte + var x16153 [1 << 17]byte + var x16154 [1 << 17]byte + var x16155 [1 << 17]byte + var x16156 [1 << 17]byte + var x16157 [1 << 17]byte + var x16158 [1 << 17]byte + var x16159 [1 << 17]byte + var x16160 [1 << 17]byte + var x16161 [1 << 17]byte + var x16162 [1 << 17]byte + var x16163 [1 << 17]byte + var x16164 [1 << 17]byte + var x16165 [1 << 17]byte + var x16166 [1 << 17]byte + var x16167 [1 << 17]byte + var x16168 [1 << 17]byte + var x16169 [1 << 17]byte + var x16170 [1 << 17]byte + var x16171 [1 << 17]byte + var x16172 [1 << 17]byte + var x16173 [1 << 17]byte + var x16174 [1 << 17]byte + var x16175 [1 << 17]byte + var x16176 [1 << 17]byte + var x16177 [1 << 17]byte + var x16178 [1 << 17]byte + var x16179 [1 << 17]byte + var x16180 [1 << 17]byte + var x16181 [1 << 17]byte + var x16182 [1 << 17]byte + var x16183 [1 << 17]byte + var x16184 [1 << 17]byte + var x16185 [1 << 17]byte + var x16186 [1 << 17]byte + var x16187 [1 << 17]byte + var x16188 [1 << 17]byte + var x16189 [1 << 17]byte + var x16190 [1 << 17]byte + var x16191 [1 << 17]byte + var x16192 [1 << 17]byte + var x16193 [1 << 17]byte + var x16194 [1 << 17]byte + var x16195 [1 << 17]byte + var x16196 [1 << 17]byte + var x16197 [1 << 17]byte + var x16198 [1 << 17]byte + var x16199 [1 << 17]byte + var x16200 [1 << 17]byte + var x16201 [1 << 17]byte + var x16202 [1 << 17]byte + var x16203 [1 << 17]byte + var x16204 [1 << 17]byte + var x16205 [1 << 17]byte + var x16206 [1 << 17]byte + var x16207 [1 << 17]byte + var x16208 [1 << 17]byte + var x16209 [1 << 17]byte + var x16210 [1 << 17]byte + var x16211 [1 << 17]byte + var x16212 [1 << 17]byte + var x16213 [1 << 17]byte + var x16214 [1 << 17]byte + var x16215 [1 << 17]byte + var x16216 [1 << 17]byte + var x16217 [1 << 17]byte + var x16218 [1 << 17]byte + var x16219 [1 << 17]byte + var x16220 [1 << 17]byte + var x16221 [1 << 17]byte + var x16222 [1 << 17]byte + var x16223 [1 << 17]byte + var x16224 [1 << 17]byte + var x16225 [1 << 17]byte + var x16226 [1 << 17]byte + var x16227 [1 << 17]byte + var x16228 [1 << 17]byte + var x16229 [1 << 17]byte + var x16230 [1 << 17]byte + var x16231 [1 << 17]byte + var x16232 [1 << 17]byte + var x16233 [1 << 17]byte + var x16234 [1 << 17]byte + var x16235 [1 << 17]byte + var x16236 [1 << 17]byte + var x16237 [1 << 17]byte + var x16238 [1 << 17]byte + var x16239 [1 << 17]byte + var x16240 [1 << 17]byte + var x16241 [1 << 17]byte + var x16242 [1 << 17]byte + var x16243 [1 << 17]byte + var x16244 [1 << 17]byte + var x16245 [1 << 17]byte + var x16246 [1 << 17]byte + var x16247 [1 << 17]byte + var x16248 [1 << 17]byte + var x16249 [1 << 17]byte + var x16250 [1 << 17]byte + var x16251 [1 << 17]byte + var x16252 [1 << 17]byte + var x16253 [1 << 17]byte + var x16254 [1 << 17]byte + var x16255 [1 << 17]byte + var x16256 [1 << 17]byte + var x16257 [1 << 17]byte + var x16258 [1 << 17]byte + var x16259 [1 << 17]byte + var x16260 [1 << 17]byte + var x16261 [1 << 17]byte + var x16262 [1 << 17]byte + var x16263 [1 << 17]byte + var x16264 [1 << 17]byte + var x16265 [1 << 17]byte + var x16266 [1 << 17]byte + var x16267 [1 << 17]byte + var x16268 [1 << 17]byte + var x16269 [1 << 17]byte + var x16270 [1 << 17]byte + var x16271 [1 << 17]byte + var x16272 [1 << 17]byte + var x16273 [1 << 17]byte + var x16274 [1 << 17]byte + var x16275 [1 << 17]byte + var x16276 [1 << 17]byte + var x16277 [1 << 17]byte + var x16278 [1 << 17]byte + var x16279 [1 << 17]byte + var x16280 [1 << 17]byte + var x16281 [1 << 17]byte + var x16282 [1 << 17]byte + var x16283 [1 << 17]byte + var x16284 [1 << 17]byte + var x16285 [1 << 17]byte + var x16286 [1 << 17]byte + var x16287 [1 << 17]byte + var x16288 [1 << 17]byte + var x16289 [1 << 17]byte + var x16290 [1 << 17]byte + var x16291 [1 << 17]byte + var x16292 [1 << 17]byte + var x16293 [1 << 17]byte + var x16294 [1 << 17]byte + var x16295 [1 << 17]byte + var x16296 [1 << 17]byte + var x16297 [1 << 17]byte + var x16298 [1 << 17]byte + var x16299 [1 << 17]byte + var x16300 [1 << 17]byte + var x16301 [1 << 17]byte + var x16302 [1 << 17]byte + var x16303 [1 << 17]byte + var x16304 [1 << 17]byte + var x16305 [1 << 17]byte + var x16306 [1 << 17]byte + var x16307 [1 << 17]byte + var x16308 [1 << 17]byte + var x16309 [1 << 17]byte + var x16310 [1 << 17]byte + var x16311 [1 << 17]byte + var x16312 [1 << 17]byte + var x16313 [1 << 17]byte + var x16314 [1 << 17]byte + var x16315 [1 << 17]byte + var x16316 [1 << 17]byte + var x16317 [1 << 17]byte + var x16318 [1 << 17]byte + var x16319 [1 << 17]byte + var x16320 [1 << 17]byte + var x16321 [1 << 17]byte + var x16322 [1 << 17]byte + var x16323 [1 << 17]byte + var x16324 [1 << 17]byte + var x16325 [1 << 17]byte + var x16326 [1 << 17]byte + var x16327 [1 << 17]byte + var x16328 [1 << 17]byte + var x16329 [1 << 17]byte + var x16330 [1 << 17]byte + var x16331 [1 << 17]byte + var x16332 [1 << 17]byte + var x16333 [1 << 17]byte + var x16334 [1 << 17]byte + var x16335 [1 << 17]byte + var x16336 [1 << 17]byte + var x16337 [1 << 17]byte + var x16338 [1 << 17]byte + var x16339 [1 << 17]byte + var x16340 [1 << 17]byte + var x16341 [1 << 17]byte + var x16342 [1 << 17]byte + var x16343 [1 << 17]byte + var x16344 [1 << 17]byte + var x16345 [1 << 17]byte + var x16346 [1 << 17]byte + var x16347 [1 << 17]byte + var x16348 [1 << 17]byte + var x16349 [1 << 17]byte + var x16350 [1 << 17]byte + var x16351 [1 << 17]byte + var x16352 [1 << 17]byte + var x16353 [1 << 17]byte + var x16354 [1 << 17]byte + var x16355 [1 << 17]byte + var x16356 [1 << 17]byte + var x16357 [1 << 17]byte + var x16358 [1 << 17]byte + var x16359 [1 << 17]byte + var x16360 [1 << 17]byte + var x16361 [1 << 17]byte + var x16362 [1 << 17]byte + var x16363 [1 << 17]byte + var x16364 [1 << 17]byte + var x16365 [1 << 17]byte + var x16366 [1 << 17]byte + var x16367 [1 << 17]byte + var x16368 [1 << 17]byte + var x16369 [1 << 17]byte + var x16370 [1 << 17]byte + var x16371 [1 << 17]byte + var x16372 [1 << 17]byte + var x16373 [1 << 17]byte + var x16374 [1 << 17]byte + var x16375 [1 << 17]byte + var x16376 [1 << 17]byte + var x16377 [1 << 17]byte + var x16378 [1 << 17]byte + var x16379 [1 << 17]byte + var x16380 [1 << 17]byte + var x16381 [1 << 17]byte + var x16382 [1 << 17]byte + var x16383 [1 << 17]byte + var x16384 [1 << 17]byte + var x16385 [1 << 17]byte + var x16386 [1 << 17]byte + var x16387 [1 << 17]byte + var x16388 [1 << 17]byte + var x16389 [1 << 17]byte + var x16390 [1 << 17]byte + var x16391 [1 << 17]byte + var x16392 [1 << 17]byte + var x16393 [1 << 17]byte + var x16394 [1 << 17]byte + var x16395 [1 << 17]byte + var x16396 [1 << 17]byte + var x16397 [1 << 17]byte + var x16398 [1 << 17]byte + var x16399 [1 << 17]byte + var x16400 [1 << 17]byte + var x16401 [1 << 17]byte + var x16402 [1 << 17]byte + var x16403 [1 << 17]byte + var x16404 [1 << 17]byte + var x16405 [1 << 17]byte + var x16406 [1 << 17]byte + var x16407 [1 << 17]byte + var x16408 [1 << 17]byte + var x16409 [1 << 17]byte + var x16410 [1 << 17]byte + var x16411 [1 << 17]byte + var x16412 [1 << 17]byte + var x16413 [1 << 17]byte + var x16414 [1 << 17]byte + var x16415 [1 << 17]byte + var x16416 [1 << 17]byte + var x16417 [1 << 17]byte + var x16418 [1 << 17]byte + var x16419 [1 << 17]byte + var x16420 [1 << 17]byte + var x16421 [1 << 17]byte + var x16422 [1 << 17]byte + var x16423 [1 << 17]byte + var x16424 [1 << 17]byte + var x16425 [1 << 17]byte + var x16426 [1 << 17]byte + var x16427 [1 << 17]byte + var x16428 [1 << 17]byte + var x16429 [1 << 17]byte + var x16430 [1 << 17]byte + var x16431 [1 << 17]byte + var x16432 [1 << 17]byte + var x16433 [1 << 17]byte + var x16434 [1 << 17]byte + var x16435 [1 << 17]byte + var x16436 [1 << 17]byte + var x16437 [1 << 17]byte + var x16438 [1 << 17]byte + var x16439 [1 << 17]byte + var x16440 [1 << 17]byte + var x16441 [1 << 17]byte + var x16442 [1 << 17]byte + var x16443 [1 << 17]byte + var x16444 [1 << 17]byte + var x16445 [1 << 17]byte + var x16446 [1 << 17]byte + var x16447 [1 << 17]byte + var x16448 [1 << 17]byte + var x16449 [1 << 17]byte + var x16450 [1 << 17]byte + var x16451 [1 << 17]byte + var x16452 [1 << 17]byte + var x16453 [1 << 17]byte + var x16454 [1 << 17]byte + var x16455 [1 << 17]byte + var x16456 [1 << 17]byte + var x16457 [1 << 17]byte + var x16458 [1 << 17]byte + var x16459 [1 << 17]byte + var x16460 [1 << 17]byte + var x16461 [1 << 17]byte + var x16462 [1 << 17]byte + var x16463 [1 << 17]byte + var x16464 [1 << 17]byte + var x16465 [1 << 17]byte + var x16466 [1 << 17]byte + var x16467 [1 << 17]byte + var x16468 [1 << 17]byte + var x16469 [1 << 17]byte + var x16470 [1 << 17]byte + var x16471 [1 << 17]byte + var x16472 [1 << 17]byte + var x16473 [1 << 17]byte + var x16474 [1 << 17]byte + var x16475 [1 << 17]byte + var x16476 [1 << 17]byte + var x16477 [1 << 17]byte + var x16478 [1 << 17]byte + var x16479 [1 << 17]byte + var x16480 [1 << 17]byte + z1 = x1 + z2 = x2 + z3 = x3 + z4 = x4 + z5 = x5 + z6 = x6 + z7 = x7 + z8 = x8 + z9 = x9 + z10 = x10 + z11 = x11 + z12 = x12 + z13 = x13 + z14 = x14 + z15 = x15 + z16 = x16 + z17 = x17 + z18 = x18 + z19 = x19 + z20 = x20 + z21 = x21 + z22 = x22 + z23 = x23 + z24 = x24 + z25 = x25 + z26 = x26 + z27 = x27 + z28 = x28 + z29 = x29 + z30 = x30 + z31 = x31 + z32 = x32 + z33 = x33 + z34 = x34 + z35 = x35 + z36 = x36 + z37 = x37 + z38 = x38 + z39 = x39 + z40 = x40 + z41 = x41 + z42 = x42 + z43 = x43 + z44 = x44 + z45 = x45 + z46 = x46 + z47 = x47 + z48 = x48 + z49 = x49 + z50 = x50 + z51 = x51 + z52 = x52 + z53 = x53 + z54 = x54 + z55 = x55 + z56 = x56 + z57 = x57 + z58 = x58 + z59 = x59 + z60 = x60 + z61 = x61 + z62 = x62 + z63 = x63 + z64 = x64 + z65 = x65 + z66 = x66 + z67 = x67 + z68 = x68 + z69 = x69 + z70 = x70 + z71 = x71 + z72 = x72 + z73 = x73 + z74 = x74 + z75 = x75 + z76 = x76 + z77 = x77 + z78 = x78 + z79 = x79 + z80 = x80 + z81 = x81 + z82 = x82 + z83 = x83 + z84 = x84 + z85 = x85 + z86 = x86 + z87 = x87 + z88 = x88 + z89 = x89 + z90 = x90 + z91 = x91 + z92 = x92 + z93 = x93 + z94 = x94 + z95 = x95 + z96 = x96 + z97 = x97 + z98 = x98 + z99 = x99 + z100 = x100 + z101 = x101 + z102 = x102 + z103 = x103 + z104 = x104 + z105 = x105 + z106 = x106 + z107 = x107 + z108 = x108 + z109 = x109 + z110 = x110 + z111 = x111 + z112 = x112 + z113 = x113 + z114 = x114 + z115 = x115 + z116 = x116 + z117 = x117 + z118 = x118 + z119 = x119 + z120 = x120 + z121 = x121 + z122 = x122 + z123 = x123 + z124 = x124 + z125 = x125 + z126 = x126 + z127 = x127 + z128 = x128 + z129 = x129 + z130 = x130 + z131 = x131 + z132 = x132 + z133 = x133 + z134 = x134 + z135 = x135 + z136 = x136 + z137 = x137 + z138 = x138 + z139 = x139 + z140 = x140 + z141 = x141 + z142 = x142 + z143 = x143 + z144 = x144 + z145 = x145 + z146 = x146 + z147 = x147 + z148 = x148 + z149 = x149 + z150 = x150 + z151 = x151 + z152 = x152 + z153 = x153 + z154 = x154 + z155 = x155 + z156 = x156 + z157 = x157 + z158 = x158 + z159 = x159 + z160 = x160 + z161 = x161 + z162 = x162 + z163 = x163 + z164 = x164 + z165 = x165 + z166 = x166 + z167 = x167 + z168 = x168 + z169 = x169 + z170 = x170 + z171 = x171 + z172 = x172 + z173 = x173 + z174 = x174 + z175 = x175 + z176 = x176 + z177 = x177 + z178 = x178 + z179 = x179 + z180 = x180 + z181 = x181 + z182 = x182 + z183 = x183 + z184 = x184 + z185 = x185 + z186 = x186 + z187 = x187 + z188 = x188 + z189 = x189 + z190 = x190 + z191 = x191 + z192 = x192 + z193 = x193 + z194 = x194 + z195 = x195 + z196 = x196 + z197 = x197 + z198 = x198 + z199 = x199 + z200 = x200 + z201 = x201 + z202 = x202 + z203 = x203 + z204 = x204 + z205 = x205 + z206 = x206 + z207 = x207 + z208 = x208 + z209 = x209 + z210 = x210 + z211 = x211 + z212 = x212 + z213 = x213 + z214 = x214 + z215 = x215 + z216 = x216 + z217 = x217 + z218 = x218 + z219 = x219 + z220 = x220 + z221 = x221 + z222 = x222 + z223 = x223 + z224 = x224 + z225 = x225 + z226 = x226 + z227 = x227 + z228 = x228 + z229 = x229 + z230 = x230 + z231 = x231 + z232 = x232 + z233 = x233 + z234 = x234 + z235 = x235 + z236 = x236 + z237 = x237 + z238 = x238 + z239 = x239 + z240 = x240 + z241 = x241 + z242 = x242 + z243 = x243 + z244 = x244 + z245 = x245 + z246 = x246 + z247 = x247 + z248 = x248 + z249 = x249 + z250 = x250 + z251 = x251 + z252 = x252 + z253 = x253 + z254 = x254 + z255 = x255 + z256 = x256 + z257 = x257 + z258 = x258 + z259 = x259 + z260 = x260 + z261 = x261 + z262 = x262 + z263 = x263 + z264 = x264 + z265 = x265 + z266 = x266 + z267 = x267 + z268 = x268 + z269 = x269 + z270 = x270 + z271 = x271 + z272 = x272 + z273 = x273 + z274 = x274 + z275 = x275 + z276 = x276 + z277 = x277 + z278 = x278 + z279 = x279 + z280 = x280 + z281 = x281 + z282 = x282 + z283 = x283 + z284 = x284 + z285 = x285 + z286 = x286 + z287 = x287 + z288 = x288 + z289 = x289 + z290 = x290 + z291 = x291 + z292 = x292 + z293 = x293 + z294 = x294 + z295 = x295 + z296 = x296 + z297 = x297 + z298 = x298 + z299 = x299 + z300 = x300 + z301 = x301 + z302 = x302 + z303 = x303 + z304 = x304 + z305 = x305 + z306 = x306 + z307 = x307 + z308 = x308 + z309 = x309 + z310 = x310 + z311 = x311 + z312 = x312 + z313 = x313 + z314 = x314 + z315 = x315 + z316 = x316 + z317 = x317 + z318 = x318 + z319 = x319 + z320 = x320 + z321 = x321 + z322 = x322 + z323 = x323 + z324 = x324 + z325 = x325 + z326 = x326 + z327 = x327 + z328 = x328 + z329 = x329 + z330 = x330 + z331 = x331 + z332 = x332 + z333 = x333 + z334 = x334 + z335 = x335 + z336 = x336 + z337 = x337 + z338 = x338 + z339 = x339 + z340 = x340 + z341 = x341 + z342 = x342 + z343 = x343 + z344 = x344 + z345 = x345 + z346 = x346 + z347 = x347 + z348 = x348 + z349 = x349 + z350 = x350 + z351 = x351 + z352 = x352 + z353 = x353 + z354 = x354 + z355 = x355 + z356 = x356 + z357 = x357 + z358 = x358 + z359 = x359 + z360 = x360 + z361 = x361 + z362 = x362 + z363 = x363 + z364 = x364 + z365 = x365 + z366 = x366 + z367 = x367 + z368 = x368 + z369 = x369 + z370 = x370 + z371 = x371 + z372 = x372 + z373 = x373 + z374 = x374 + z375 = x375 + z376 = x376 + z377 = x377 + z378 = x378 + z379 = x379 + z380 = x380 + z381 = x381 + z382 = x382 + z383 = x383 + z384 = x384 + z385 = x385 + z386 = x386 + z387 = x387 + z388 = x388 + z389 = x389 + z390 = x390 + z391 = x391 + z392 = x392 + z393 = x393 + z394 = x394 + z395 = x395 + z396 = x396 + z397 = x397 + z398 = x398 + z399 = x399 + z400 = x400 + z401 = x401 + z402 = x402 + z403 = x403 + z404 = x404 + z405 = x405 + z406 = x406 + z407 = x407 + z408 = x408 + z409 = x409 + z410 = x410 + z411 = x411 + z412 = x412 + z413 = x413 + z414 = x414 + z415 = x415 + z416 = x416 + z417 = x417 + z418 = x418 + z419 = x419 + z420 = x420 + z421 = x421 + z422 = x422 + z423 = x423 + z424 = x424 + z425 = x425 + z426 = x426 + z427 = x427 + z428 = x428 + z429 = x429 + z430 = x430 + z431 = x431 + z432 = x432 + z433 = x433 + z434 = x434 + z435 = x435 + z436 = x436 + z437 = x437 + z438 = x438 + z439 = x439 + z440 = x440 + z441 = x441 + z442 = x442 + z443 = x443 + z444 = x444 + z445 = x445 + z446 = x446 + z447 = x447 + z448 = x448 + z449 = x449 + z450 = x450 + z451 = x451 + z452 = x452 + z453 = x453 + z454 = x454 + z455 = x455 + z456 = x456 + z457 = x457 + z458 = x458 + z459 = x459 + z460 = x460 + z461 = x461 + z462 = x462 + z463 = x463 + z464 = x464 + z465 = x465 + z466 = x466 + z467 = x467 + z468 = x468 + z469 = x469 + z470 = x470 + z471 = x471 + z472 = x472 + z473 = x473 + z474 = x474 + z475 = x475 + z476 = x476 + z477 = x477 + z478 = x478 + z479 = x479 + z480 = x480 + z481 = x481 + z482 = x482 + z483 = x483 + z484 = x484 + z485 = x485 + z486 = x486 + z487 = x487 + z488 = x488 + z489 = x489 + z490 = x490 + z491 = x491 + z492 = x492 + z493 = x493 + z494 = x494 + z495 = x495 + z496 = x496 + z497 = x497 + z498 = x498 + z499 = x499 + z500 = x500 + z501 = x501 + z502 = x502 + z503 = x503 + z504 = x504 + z505 = x505 + z506 = x506 + z507 = x507 + z508 = x508 + z509 = x509 + z510 = x510 + z511 = x511 + z512 = x512 + z513 = x513 + z514 = x514 + z515 = x515 + z516 = x516 + z517 = x517 + z518 = x518 + z519 = x519 + z520 = x520 + z521 = x521 + z522 = x522 + z523 = x523 + z524 = x524 + z525 = x525 + z526 = x526 + z527 = x527 + z528 = x528 + z529 = x529 + z530 = x530 + z531 = x531 + z532 = x532 + z533 = x533 + z534 = x534 + z535 = x535 + z536 = x536 + z537 = x537 + z538 = x538 + z539 = x539 + z540 = x540 + z541 = x541 + z542 = x542 + z543 = x543 + z544 = x544 + z545 = x545 + z546 = x546 + z547 = x547 + z548 = x548 + z549 = x549 + z550 = x550 + z551 = x551 + z552 = x552 + z553 = x553 + z554 = x554 + z555 = x555 + z556 = x556 + z557 = x557 + z558 = x558 + z559 = x559 + z560 = x560 + z561 = x561 + z562 = x562 + z563 = x563 + z564 = x564 + z565 = x565 + z566 = x566 + z567 = x567 + z568 = x568 + z569 = x569 + z570 = x570 + z571 = x571 + z572 = x572 + z573 = x573 + z574 = x574 + z575 = x575 + z576 = x576 + z577 = x577 + z578 = x578 + z579 = x579 + z580 = x580 + z581 = x581 + z582 = x582 + z583 = x583 + z584 = x584 + z585 = x585 + z586 = x586 + z587 = x587 + z588 = x588 + z589 = x589 + z590 = x590 + z591 = x591 + z592 = x592 + z593 = x593 + z594 = x594 + z595 = x595 + z596 = x596 + z597 = x597 + z598 = x598 + z599 = x599 + z600 = x600 + z601 = x601 + z602 = x602 + z603 = x603 + z604 = x604 + z605 = x605 + z606 = x606 + z607 = x607 + z608 = x608 + z609 = x609 + z610 = x610 + z611 = x611 + z612 = x612 + z613 = x613 + z614 = x614 + z615 = x615 + z616 = x616 + z617 = x617 + z618 = x618 + z619 = x619 + z620 = x620 + z621 = x621 + z622 = x622 + z623 = x623 + z624 = x624 + z625 = x625 + z626 = x626 + z627 = x627 + z628 = x628 + z629 = x629 + z630 = x630 + z631 = x631 + z632 = x632 + z633 = x633 + z634 = x634 + z635 = x635 + z636 = x636 + z637 = x637 + z638 = x638 + z639 = x639 + z640 = x640 + z641 = x641 + z642 = x642 + z643 = x643 + z644 = x644 + z645 = x645 + z646 = x646 + z647 = x647 + z648 = x648 + z649 = x649 + z650 = x650 + z651 = x651 + z652 = x652 + z653 = x653 + z654 = x654 + z655 = x655 + z656 = x656 + z657 = x657 + z658 = x658 + z659 = x659 + z660 = x660 + z661 = x661 + z662 = x662 + z663 = x663 + z664 = x664 + z665 = x665 + z666 = x666 + z667 = x667 + z668 = x668 + z669 = x669 + z670 = x670 + z671 = x671 + z672 = x672 + z673 = x673 + z674 = x674 + z675 = x675 + z676 = x676 + z677 = x677 + z678 = x678 + z679 = x679 + z680 = x680 + z681 = x681 + z682 = x682 + z683 = x683 + z684 = x684 + z685 = x685 + z686 = x686 + z687 = x687 + z688 = x688 + z689 = x689 + z690 = x690 + z691 = x691 + z692 = x692 + z693 = x693 + z694 = x694 + z695 = x695 + z696 = x696 + z697 = x697 + z698 = x698 + z699 = x699 + z700 = x700 + z701 = x701 + z702 = x702 + z703 = x703 + z704 = x704 + z705 = x705 + z706 = x706 + z707 = x707 + z708 = x708 + z709 = x709 + z710 = x710 + z711 = x711 + z712 = x712 + z713 = x713 + z714 = x714 + z715 = x715 + z716 = x716 + z717 = x717 + z718 = x718 + z719 = x719 + z720 = x720 + z721 = x721 + z722 = x722 + z723 = x723 + z724 = x724 + z725 = x725 + z726 = x726 + z727 = x727 + z728 = x728 + z729 = x729 + z730 = x730 + z731 = x731 + z732 = x732 + z733 = x733 + z734 = x734 + z735 = x735 + z736 = x736 + z737 = x737 + z738 = x738 + z739 = x739 + z740 = x740 + z741 = x741 + z742 = x742 + z743 = x743 + z744 = x744 + z745 = x745 + z746 = x746 + z747 = x747 + z748 = x748 + z749 = x749 + z750 = x750 + z751 = x751 + z752 = x752 + z753 = x753 + z754 = x754 + z755 = x755 + z756 = x756 + z757 = x757 + z758 = x758 + z759 = x759 + z760 = x760 + z761 = x761 + z762 = x762 + z763 = x763 + z764 = x764 + z765 = x765 + z766 = x766 + z767 = x767 + z768 = x768 + z769 = x769 + z770 = x770 + z771 = x771 + z772 = x772 + z773 = x773 + z774 = x774 + z775 = x775 + z776 = x776 + z777 = x777 + z778 = x778 + z779 = x779 + z780 = x780 + z781 = x781 + z782 = x782 + z783 = x783 + z784 = x784 + z785 = x785 + z786 = x786 + z787 = x787 + z788 = x788 + z789 = x789 + z790 = x790 + z791 = x791 + z792 = x792 + z793 = x793 + z794 = x794 + z795 = x795 + z796 = x796 + z797 = x797 + z798 = x798 + z799 = x799 + z800 = x800 + z801 = x801 + z802 = x802 + z803 = x803 + z804 = x804 + z805 = x805 + z806 = x806 + z807 = x807 + z808 = x808 + z809 = x809 + z810 = x810 + z811 = x811 + z812 = x812 + z813 = x813 + z814 = x814 + z815 = x815 + z816 = x816 + z817 = x817 + z818 = x818 + z819 = x819 + z820 = x820 + z821 = x821 + z822 = x822 + z823 = x823 + z824 = x824 + z825 = x825 + z826 = x826 + z827 = x827 + z828 = x828 + z829 = x829 + z830 = x830 + z831 = x831 + z832 = x832 + z833 = x833 + z834 = x834 + z835 = x835 + z836 = x836 + z837 = x837 + z838 = x838 + z839 = x839 + z840 = x840 + z841 = x841 + z842 = x842 + z843 = x843 + z844 = x844 + z845 = x845 + z846 = x846 + z847 = x847 + z848 = x848 + z849 = x849 + z850 = x850 + z851 = x851 + z852 = x852 + z853 = x853 + z854 = x854 + z855 = x855 + z856 = x856 + z857 = x857 + z858 = x858 + z859 = x859 + z860 = x860 + z861 = x861 + z862 = x862 + z863 = x863 + z864 = x864 + z865 = x865 + z866 = x866 + z867 = x867 + z868 = x868 + z869 = x869 + z870 = x870 + z871 = x871 + z872 = x872 + z873 = x873 + z874 = x874 + z875 = x875 + z876 = x876 + z877 = x877 + z878 = x878 + z879 = x879 + z880 = x880 + z881 = x881 + z882 = x882 + z883 = x883 + z884 = x884 + z885 = x885 + z886 = x886 + z887 = x887 + z888 = x888 + z889 = x889 + z890 = x890 + z891 = x891 + z892 = x892 + z893 = x893 + z894 = x894 + z895 = x895 + z896 = x896 + z897 = x897 + z898 = x898 + z899 = x899 + z900 = x900 + z901 = x901 + z902 = x902 + z903 = x903 + z904 = x904 + z905 = x905 + z906 = x906 + z907 = x907 + z908 = x908 + z909 = x909 + z910 = x910 + z911 = x911 + z912 = x912 + z913 = x913 + z914 = x914 + z915 = x915 + z916 = x916 + z917 = x917 + z918 = x918 + z919 = x919 + z920 = x920 + z921 = x921 + z922 = x922 + z923 = x923 + z924 = x924 + z925 = x925 + z926 = x926 + z927 = x927 + z928 = x928 + z929 = x929 + z930 = x930 + z931 = x931 + z932 = x932 + z933 = x933 + z934 = x934 + z935 = x935 + z936 = x936 + z937 = x937 + z938 = x938 + z939 = x939 + z940 = x940 + z941 = x941 + z942 = x942 + z943 = x943 + z944 = x944 + z945 = x945 + z946 = x946 + z947 = x947 + z948 = x948 + z949 = x949 + z950 = x950 + z951 = x951 + z952 = x952 + z953 = x953 + z954 = x954 + z955 = x955 + z956 = x956 + z957 = x957 + z958 = x958 + z959 = x959 + z960 = x960 + z961 = x961 + z962 = x962 + z963 = x963 + z964 = x964 + z965 = x965 + z966 = x966 + z967 = x967 + z968 = x968 + z969 = x969 + z970 = x970 + z971 = x971 + z972 = x972 + z973 = x973 + z974 = x974 + z975 = x975 + z976 = x976 + z977 = x977 + z978 = x978 + z979 = x979 + z980 = x980 + z981 = x981 + z982 = x982 + z983 = x983 + z984 = x984 + z985 = x985 + z986 = x986 + z987 = x987 + z988 = x988 + z989 = x989 + z990 = x990 + z991 = x991 + z992 = x992 + z993 = x993 + z994 = x994 + z995 = x995 + z996 = x996 + z997 = x997 + z998 = x998 + z999 = x999 + z1000 = x1000 + z1001 = x1001 + z1002 = x1002 + z1003 = x1003 + z1004 = x1004 + z1005 = x1005 + z1006 = x1006 + z1007 = x1007 + z1008 = x1008 + z1009 = x1009 + z1010 = x1010 + z1011 = x1011 + z1012 = x1012 + z1013 = x1013 + z1014 = x1014 + z1015 = x1015 + z1016 = x1016 + z1017 = x1017 + z1018 = x1018 + z1019 = x1019 + z1020 = x1020 + z1021 = x1021 + z1022 = x1022 + z1023 = x1023 + z1024 = x1024 + z1025 = x1025 + z1026 = x1026 + z1027 = x1027 + z1028 = x1028 + z1029 = x1029 + z1030 = x1030 + z1031 = x1031 + z1032 = x1032 + z1033 = x1033 + z1034 = x1034 + z1035 = x1035 + z1036 = x1036 + z1037 = x1037 + z1038 = x1038 + z1039 = x1039 + z1040 = x1040 + z1041 = x1041 + z1042 = x1042 + z1043 = x1043 + z1044 = x1044 + z1045 = x1045 + z1046 = x1046 + z1047 = x1047 + z1048 = x1048 + z1049 = x1049 + z1050 = x1050 + z1051 = x1051 + z1052 = x1052 + z1053 = x1053 + z1054 = x1054 + z1055 = x1055 + z1056 = x1056 + z1057 = x1057 + z1058 = x1058 + z1059 = x1059 + z1060 = x1060 + z1061 = x1061 + z1062 = x1062 + z1063 = x1063 + z1064 = x1064 + z1065 = x1065 + z1066 = x1066 + z1067 = x1067 + z1068 = x1068 + z1069 = x1069 + z1070 = x1070 + z1071 = x1071 + z1072 = x1072 + z1073 = x1073 + z1074 = x1074 + z1075 = x1075 + z1076 = x1076 + z1077 = x1077 + z1078 = x1078 + z1079 = x1079 + z1080 = x1080 + z1081 = x1081 + z1082 = x1082 + z1083 = x1083 + z1084 = x1084 + z1085 = x1085 + z1086 = x1086 + z1087 = x1087 + z1088 = x1088 + z1089 = x1089 + z1090 = x1090 + z1091 = x1091 + z1092 = x1092 + z1093 = x1093 + z1094 = x1094 + z1095 = x1095 + z1096 = x1096 + z1097 = x1097 + z1098 = x1098 + z1099 = x1099 + z1100 = x1100 + z1101 = x1101 + z1102 = x1102 + z1103 = x1103 + z1104 = x1104 + z1105 = x1105 + z1106 = x1106 + z1107 = x1107 + z1108 = x1108 + z1109 = x1109 + z1110 = x1110 + z1111 = x1111 + z1112 = x1112 + z1113 = x1113 + z1114 = x1114 + z1115 = x1115 + z1116 = x1116 + z1117 = x1117 + z1118 = x1118 + z1119 = x1119 + z1120 = x1120 + z1121 = x1121 + z1122 = x1122 + z1123 = x1123 + z1124 = x1124 + z1125 = x1125 + z1126 = x1126 + z1127 = x1127 + z1128 = x1128 + z1129 = x1129 + z1130 = x1130 + z1131 = x1131 + z1132 = x1132 + z1133 = x1133 + z1134 = x1134 + z1135 = x1135 + z1136 = x1136 + z1137 = x1137 + z1138 = x1138 + z1139 = x1139 + z1140 = x1140 + z1141 = x1141 + z1142 = x1142 + z1143 = x1143 + z1144 = x1144 + z1145 = x1145 + z1146 = x1146 + z1147 = x1147 + z1148 = x1148 + z1149 = x1149 + z1150 = x1150 + z1151 = x1151 + z1152 = x1152 + z1153 = x1153 + z1154 = x1154 + z1155 = x1155 + z1156 = x1156 + z1157 = x1157 + z1158 = x1158 + z1159 = x1159 + z1160 = x1160 + z1161 = x1161 + z1162 = x1162 + z1163 = x1163 + z1164 = x1164 + z1165 = x1165 + z1166 = x1166 + z1167 = x1167 + z1168 = x1168 + z1169 = x1169 + z1170 = x1170 + z1171 = x1171 + z1172 = x1172 + z1173 = x1173 + z1174 = x1174 + z1175 = x1175 + z1176 = x1176 + z1177 = x1177 + z1178 = x1178 + z1179 = x1179 + z1180 = x1180 + z1181 = x1181 + z1182 = x1182 + z1183 = x1183 + z1184 = x1184 + z1185 = x1185 + z1186 = x1186 + z1187 = x1187 + z1188 = x1188 + z1189 = x1189 + z1190 = x1190 + z1191 = x1191 + z1192 = x1192 + z1193 = x1193 + z1194 = x1194 + z1195 = x1195 + z1196 = x1196 + z1197 = x1197 + z1198 = x1198 + z1199 = x1199 + z1200 = x1200 + z1201 = x1201 + z1202 = x1202 + z1203 = x1203 + z1204 = x1204 + z1205 = x1205 + z1206 = x1206 + z1207 = x1207 + z1208 = x1208 + z1209 = x1209 + z1210 = x1210 + z1211 = x1211 + z1212 = x1212 + z1213 = x1213 + z1214 = x1214 + z1215 = x1215 + z1216 = x1216 + z1217 = x1217 + z1218 = x1218 + z1219 = x1219 + z1220 = x1220 + z1221 = x1221 + z1222 = x1222 + z1223 = x1223 + z1224 = x1224 + z1225 = x1225 + z1226 = x1226 + z1227 = x1227 + z1228 = x1228 + z1229 = x1229 + z1230 = x1230 + z1231 = x1231 + z1232 = x1232 + z1233 = x1233 + z1234 = x1234 + z1235 = x1235 + z1236 = x1236 + z1237 = x1237 + z1238 = x1238 + z1239 = x1239 + z1240 = x1240 + z1241 = x1241 + z1242 = x1242 + z1243 = x1243 + z1244 = x1244 + z1245 = x1245 + z1246 = x1246 + z1247 = x1247 + z1248 = x1248 + z1249 = x1249 + z1250 = x1250 + z1251 = x1251 + z1252 = x1252 + z1253 = x1253 + z1254 = x1254 + z1255 = x1255 + z1256 = x1256 + z1257 = x1257 + z1258 = x1258 + z1259 = x1259 + z1260 = x1260 + z1261 = x1261 + z1262 = x1262 + z1263 = x1263 + z1264 = x1264 + z1265 = x1265 + z1266 = x1266 + z1267 = x1267 + z1268 = x1268 + z1269 = x1269 + z1270 = x1270 + z1271 = x1271 + z1272 = x1272 + z1273 = x1273 + z1274 = x1274 + z1275 = x1275 + z1276 = x1276 + z1277 = x1277 + z1278 = x1278 + z1279 = x1279 + z1280 = x1280 + z1281 = x1281 + z1282 = x1282 + z1283 = x1283 + z1284 = x1284 + z1285 = x1285 + z1286 = x1286 + z1287 = x1287 + z1288 = x1288 + z1289 = x1289 + z1290 = x1290 + z1291 = x1291 + z1292 = x1292 + z1293 = x1293 + z1294 = x1294 + z1295 = x1295 + z1296 = x1296 + z1297 = x1297 + z1298 = x1298 + z1299 = x1299 + z1300 = x1300 + z1301 = x1301 + z1302 = x1302 + z1303 = x1303 + z1304 = x1304 + z1305 = x1305 + z1306 = x1306 + z1307 = x1307 + z1308 = x1308 + z1309 = x1309 + z1310 = x1310 + z1311 = x1311 + z1312 = x1312 + z1313 = x1313 + z1314 = x1314 + z1315 = x1315 + z1316 = x1316 + z1317 = x1317 + z1318 = x1318 + z1319 = x1319 + z1320 = x1320 + z1321 = x1321 + z1322 = x1322 + z1323 = x1323 + z1324 = x1324 + z1325 = x1325 + z1326 = x1326 + z1327 = x1327 + z1328 = x1328 + z1329 = x1329 + z1330 = x1330 + z1331 = x1331 + z1332 = x1332 + z1333 = x1333 + z1334 = x1334 + z1335 = x1335 + z1336 = x1336 + z1337 = x1337 + z1338 = x1338 + z1339 = x1339 + z1340 = x1340 + z1341 = x1341 + z1342 = x1342 + z1343 = x1343 + z1344 = x1344 + z1345 = x1345 + z1346 = x1346 + z1347 = x1347 + z1348 = x1348 + z1349 = x1349 + z1350 = x1350 + z1351 = x1351 + z1352 = x1352 + z1353 = x1353 + z1354 = x1354 + z1355 = x1355 + z1356 = x1356 + z1357 = x1357 + z1358 = x1358 + z1359 = x1359 + z1360 = x1360 + z1361 = x1361 + z1362 = x1362 + z1363 = x1363 + z1364 = x1364 + z1365 = x1365 + z1366 = x1366 + z1367 = x1367 + z1368 = x1368 + z1369 = x1369 + z1370 = x1370 + z1371 = x1371 + z1372 = x1372 + z1373 = x1373 + z1374 = x1374 + z1375 = x1375 + z1376 = x1376 + z1377 = x1377 + z1378 = x1378 + z1379 = x1379 + z1380 = x1380 + z1381 = x1381 + z1382 = x1382 + z1383 = x1383 + z1384 = x1384 + z1385 = x1385 + z1386 = x1386 + z1387 = x1387 + z1388 = x1388 + z1389 = x1389 + z1390 = x1390 + z1391 = x1391 + z1392 = x1392 + z1393 = x1393 + z1394 = x1394 + z1395 = x1395 + z1396 = x1396 + z1397 = x1397 + z1398 = x1398 + z1399 = x1399 + z1400 = x1400 + z1401 = x1401 + z1402 = x1402 + z1403 = x1403 + z1404 = x1404 + z1405 = x1405 + z1406 = x1406 + z1407 = x1407 + z1408 = x1408 + z1409 = x1409 + z1410 = x1410 + z1411 = x1411 + z1412 = x1412 + z1413 = x1413 + z1414 = x1414 + z1415 = x1415 + z1416 = x1416 + z1417 = x1417 + z1418 = x1418 + z1419 = x1419 + z1420 = x1420 + z1421 = x1421 + z1422 = x1422 + z1423 = x1423 + z1424 = x1424 + z1425 = x1425 + z1426 = x1426 + z1427 = x1427 + z1428 = x1428 + z1429 = x1429 + z1430 = x1430 + z1431 = x1431 + z1432 = x1432 + z1433 = x1433 + z1434 = x1434 + z1435 = x1435 + z1436 = x1436 + z1437 = x1437 + z1438 = x1438 + z1439 = x1439 + z1440 = x1440 + z1441 = x1441 + z1442 = x1442 + z1443 = x1443 + z1444 = x1444 + z1445 = x1445 + z1446 = x1446 + z1447 = x1447 + z1448 = x1448 + z1449 = x1449 + z1450 = x1450 + z1451 = x1451 + z1452 = x1452 + z1453 = x1453 + z1454 = x1454 + z1455 = x1455 + z1456 = x1456 + z1457 = x1457 + z1458 = x1458 + z1459 = x1459 + z1460 = x1460 + z1461 = x1461 + z1462 = x1462 + z1463 = x1463 + z1464 = x1464 + z1465 = x1465 + z1466 = x1466 + z1467 = x1467 + z1468 = x1468 + z1469 = x1469 + z1470 = x1470 + z1471 = x1471 + z1472 = x1472 + z1473 = x1473 + z1474 = x1474 + z1475 = x1475 + z1476 = x1476 + z1477 = x1477 + z1478 = x1478 + z1479 = x1479 + z1480 = x1480 + z1481 = x1481 + z1482 = x1482 + z1483 = x1483 + z1484 = x1484 + z1485 = x1485 + z1486 = x1486 + z1487 = x1487 + z1488 = x1488 + z1489 = x1489 + z1490 = x1490 + z1491 = x1491 + z1492 = x1492 + z1493 = x1493 + z1494 = x1494 + z1495 = x1495 + z1496 = x1496 + z1497 = x1497 + z1498 = x1498 + z1499 = x1499 + z1500 = x1500 + z1501 = x1501 + z1502 = x1502 + z1503 = x1503 + z1504 = x1504 + z1505 = x1505 + z1506 = x1506 + z1507 = x1507 + z1508 = x1508 + z1509 = x1509 + z1510 = x1510 + z1511 = x1511 + z1512 = x1512 + z1513 = x1513 + z1514 = x1514 + z1515 = x1515 + z1516 = x1516 + z1517 = x1517 + z1518 = x1518 + z1519 = x1519 + z1520 = x1520 + z1521 = x1521 + z1522 = x1522 + z1523 = x1523 + z1524 = x1524 + z1525 = x1525 + z1526 = x1526 + z1527 = x1527 + z1528 = x1528 + z1529 = x1529 + z1530 = x1530 + z1531 = x1531 + z1532 = x1532 + z1533 = x1533 + z1534 = x1534 + z1535 = x1535 + z1536 = x1536 + z1537 = x1537 + z1538 = x1538 + z1539 = x1539 + z1540 = x1540 + z1541 = x1541 + z1542 = x1542 + z1543 = x1543 + z1544 = x1544 + z1545 = x1545 + z1546 = x1546 + z1547 = x1547 + z1548 = x1548 + z1549 = x1549 + z1550 = x1550 + z1551 = x1551 + z1552 = x1552 + z1553 = x1553 + z1554 = x1554 + z1555 = x1555 + z1556 = x1556 + z1557 = x1557 + z1558 = x1558 + z1559 = x1559 + z1560 = x1560 + z1561 = x1561 + z1562 = x1562 + z1563 = x1563 + z1564 = x1564 + z1565 = x1565 + z1566 = x1566 + z1567 = x1567 + z1568 = x1568 + z1569 = x1569 + z1570 = x1570 + z1571 = x1571 + z1572 = x1572 + z1573 = x1573 + z1574 = x1574 + z1575 = x1575 + z1576 = x1576 + z1577 = x1577 + z1578 = x1578 + z1579 = x1579 + z1580 = x1580 + z1581 = x1581 + z1582 = x1582 + z1583 = x1583 + z1584 = x1584 + z1585 = x1585 + z1586 = x1586 + z1587 = x1587 + z1588 = x1588 + z1589 = x1589 + z1590 = x1590 + z1591 = x1591 + z1592 = x1592 + z1593 = x1593 + z1594 = x1594 + z1595 = x1595 + z1596 = x1596 + z1597 = x1597 + z1598 = x1598 + z1599 = x1599 + z1600 = x1600 + z1601 = x1601 + z1602 = x1602 + z1603 = x1603 + z1604 = x1604 + z1605 = x1605 + z1606 = x1606 + z1607 = x1607 + z1608 = x1608 + z1609 = x1609 + z1610 = x1610 + z1611 = x1611 + z1612 = x1612 + z1613 = x1613 + z1614 = x1614 + z1615 = x1615 + z1616 = x1616 + z1617 = x1617 + z1618 = x1618 + z1619 = x1619 + z1620 = x1620 + z1621 = x1621 + z1622 = x1622 + z1623 = x1623 + z1624 = x1624 + z1625 = x1625 + z1626 = x1626 + z1627 = x1627 + z1628 = x1628 + z1629 = x1629 + z1630 = x1630 + z1631 = x1631 + z1632 = x1632 + z1633 = x1633 + z1634 = x1634 + z1635 = x1635 + z1636 = x1636 + z1637 = x1637 + z1638 = x1638 + z1639 = x1639 + z1640 = x1640 + z1641 = x1641 + z1642 = x1642 + z1643 = x1643 + z1644 = x1644 + z1645 = x1645 + z1646 = x1646 + z1647 = x1647 + z1648 = x1648 + z1649 = x1649 + z1650 = x1650 + z1651 = x1651 + z1652 = x1652 + z1653 = x1653 + z1654 = x1654 + z1655 = x1655 + z1656 = x1656 + z1657 = x1657 + z1658 = x1658 + z1659 = x1659 + z1660 = x1660 + z1661 = x1661 + z1662 = x1662 + z1663 = x1663 + z1664 = x1664 + z1665 = x1665 + z1666 = x1666 + z1667 = x1667 + z1668 = x1668 + z1669 = x1669 + z1670 = x1670 + z1671 = x1671 + z1672 = x1672 + z1673 = x1673 + z1674 = x1674 + z1675 = x1675 + z1676 = x1676 + z1677 = x1677 + z1678 = x1678 + z1679 = x1679 + z1680 = x1680 + z1681 = x1681 + z1682 = x1682 + z1683 = x1683 + z1684 = x1684 + z1685 = x1685 + z1686 = x1686 + z1687 = x1687 + z1688 = x1688 + z1689 = x1689 + z1690 = x1690 + z1691 = x1691 + z1692 = x1692 + z1693 = x1693 + z1694 = x1694 + z1695 = x1695 + z1696 = x1696 + z1697 = x1697 + z1698 = x1698 + z1699 = x1699 + z1700 = x1700 + z1701 = x1701 + z1702 = x1702 + z1703 = x1703 + z1704 = x1704 + z1705 = x1705 + z1706 = x1706 + z1707 = x1707 + z1708 = x1708 + z1709 = x1709 + z1710 = x1710 + z1711 = x1711 + z1712 = x1712 + z1713 = x1713 + z1714 = x1714 + z1715 = x1715 + z1716 = x1716 + z1717 = x1717 + z1718 = x1718 + z1719 = x1719 + z1720 = x1720 + z1721 = x1721 + z1722 = x1722 + z1723 = x1723 + z1724 = x1724 + z1725 = x1725 + z1726 = x1726 + z1727 = x1727 + z1728 = x1728 + z1729 = x1729 + z1730 = x1730 + z1731 = x1731 + z1732 = x1732 + z1733 = x1733 + z1734 = x1734 + z1735 = x1735 + z1736 = x1736 + z1737 = x1737 + z1738 = x1738 + z1739 = x1739 + z1740 = x1740 + z1741 = x1741 + z1742 = x1742 + z1743 = x1743 + z1744 = x1744 + z1745 = x1745 + z1746 = x1746 + z1747 = x1747 + z1748 = x1748 + z1749 = x1749 + z1750 = x1750 + z1751 = x1751 + z1752 = x1752 + z1753 = x1753 + z1754 = x1754 + z1755 = x1755 + z1756 = x1756 + z1757 = x1757 + z1758 = x1758 + z1759 = x1759 + z1760 = x1760 + z1761 = x1761 + z1762 = x1762 + z1763 = x1763 + z1764 = x1764 + z1765 = x1765 + z1766 = x1766 + z1767 = x1767 + z1768 = x1768 + z1769 = x1769 + z1770 = x1770 + z1771 = x1771 + z1772 = x1772 + z1773 = x1773 + z1774 = x1774 + z1775 = x1775 + z1776 = x1776 + z1777 = x1777 + z1778 = x1778 + z1779 = x1779 + z1780 = x1780 + z1781 = x1781 + z1782 = x1782 + z1783 = x1783 + z1784 = x1784 + z1785 = x1785 + z1786 = x1786 + z1787 = x1787 + z1788 = x1788 + z1789 = x1789 + z1790 = x1790 + z1791 = x1791 + z1792 = x1792 + z1793 = x1793 + z1794 = x1794 + z1795 = x1795 + z1796 = x1796 + z1797 = x1797 + z1798 = x1798 + z1799 = x1799 + z1800 = x1800 + z1801 = x1801 + z1802 = x1802 + z1803 = x1803 + z1804 = x1804 + z1805 = x1805 + z1806 = x1806 + z1807 = x1807 + z1808 = x1808 + z1809 = x1809 + z1810 = x1810 + z1811 = x1811 + z1812 = x1812 + z1813 = x1813 + z1814 = x1814 + z1815 = x1815 + z1816 = x1816 + z1817 = x1817 + z1818 = x1818 + z1819 = x1819 + z1820 = x1820 + z1821 = x1821 + z1822 = x1822 + z1823 = x1823 + z1824 = x1824 + z1825 = x1825 + z1826 = x1826 + z1827 = x1827 + z1828 = x1828 + z1829 = x1829 + z1830 = x1830 + z1831 = x1831 + z1832 = x1832 + z1833 = x1833 + z1834 = x1834 + z1835 = x1835 + z1836 = x1836 + z1837 = x1837 + z1838 = x1838 + z1839 = x1839 + z1840 = x1840 + z1841 = x1841 + z1842 = x1842 + z1843 = x1843 + z1844 = x1844 + z1845 = x1845 + z1846 = x1846 + z1847 = x1847 + z1848 = x1848 + z1849 = x1849 + z1850 = x1850 + z1851 = x1851 + z1852 = x1852 + z1853 = x1853 + z1854 = x1854 + z1855 = x1855 + z1856 = x1856 + z1857 = x1857 + z1858 = x1858 + z1859 = x1859 + z1860 = x1860 + z1861 = x1861 + z1862 = x1862 + z1863 = x1863 + z1864 = x1864 + z1865 = x1865 + z1866 = x1866 + z1867 = x1867 + z1868 = x1868 + z1869 = x1869 + z1870 = x1870 + z1871 = x1871 + z1872 = x1872 + z1873 = x1873 + z1874 = x1874 + z1875 = x1875 + z1876 = x1876 + z1877 = x1877 + z1878 = x1878 + z1879 = x1879 + z1880 = x1880 + z1881 = x1881 + z1882 = x1882 + z1883 = x1883 + z1884 = x1884 + z1885 = x1885 + z1886 = x1886 + z1887 = x1887 + z1888 = x1888 + z1889 = x1889 + z1890 = x1890 + z1891 = x1891 + z1892 = x1892 + z1893 = x1893 + z1894 = x1894 + z1895 = x1895 + z1896 = x1896 + z1897 = x1897 + z1898 = x1898 + z1899 = x1899 + z1900 = x1900 + z1901 = x1901 + z1902 = x1902 + z1903 = x1903 + z1904 = x1904 + z1905 = x1905 + z1906 = x1906 + z1907 = x1907 + z1908 = x1908 + z1909 = x1909 + z1910 = x1910 + z1911 = x1911 + z1912 = x1912 + z1913 = x1913 + z1914 = x1914 + z1915 = x1915 + z1916 = x1916 + z1917 = x1917 + z1918 = x1918 + z1919 = x1919 + z1920 = x1920 + z1921 = x1921 + z1922 = x1922 + z1923 = x1923 + z1924 = x1924 + z1925 = x1925 + z1926 = x1926 + z1927 = x1927 + z1928 = x1928 + z1929 = x1929 + z1930 = x1930 + z1931 = x1931 + z1932 = x1932 + z1933 = x1933 + z1934 = x1934 + z1935 = x1935 + z1936 = x1936 + z1937 = x1937 + z1938 = x1938 + z1939 = x1939 + z1940 = x1940 + z1941 = x1941 + z1942 = x1942 + z1943 = x1943 + z1944 = x1944 + z1945 = x1945 + z1946 = x1946 + z1947 = x1947 + z1948 = x1948 + z1949 = x1949 + z1950 = x1950 + z1951 = x1951 + z1952 = x1952 + z1953 = x1953 + z1954 = x1954 + z1955 = x1955 + z1956 = x1956 + z1957 = x1957 + z1958 = x1958 + z1959 = x1959 + z1960 = x1960 + z1961 = x1961 + z1962 = x1962 + z1963 = x1963 + z1964 = x1964 + z1965 = x1965 + z1966 = x1966 + z1967 = x1967 + z1968 = x1968 + z1969 = x1969 + z1970 = x1970 + z1971 = x1971 + z1972 = x1972 + z1973 = x1973 + z1974 = x1974 + z1975 = x1975 + z1976 = x1976 + z1977 = x1977 + z1978 = x1978 + z1979 = x1979 + z1980 = x1980 + z1981 = x1981 + z1982 = x1982 + z1983 = x1983 + z1984 = x1984 + z1985 = x1985 + z1986 = x1986 + z1987 = x1987 + z1988 = x1988 + z1989 = x1989 + z1990 = x1990 + z1991 = x1991 + z1992 = x1992 + z1993 = x1993 + z1994 = x1994 + z1995 = x1995 + z1996 = x1996 + z1997 = x1997 + z1998 = x1998 + z1999 = x1999 + z2000 = x2000 + z2001 = x2001 + z2002 = x2002 + z2003 = x2003 + z2004 = x2004 + z2005 = x2005 + z2006 = x2006 + z2007 = x2007 + z2008 = x2008 + z2009 = x2009 + z2010 = x2010 + z2011 = x2011 + z2012 = x2012 + z2013 = x2013 + z2014 = x2014 + z2015 = x2015 + z2016 = x2016 + z2017 = x2017 + z2018 = x2018 + z2019 = x2019 + z2020 = x2020 + z2021 = x2021 + z2022 = x2022 + z2023 = x2023 + z2024 = x2024 + z2025 = x2025 + z2026 = x2026 + z2027 = x2027 + z2028 = x2028 + z2029 = x2029 + z2030 = x2030 + z2031 = x2031 + z2032 = x2032 + z2033 = x2033 + z2034 = x2034 + z2035 = x2035 + z2036 = x2036 + z2037 = x2037 + z2038 = x2038 + z2039 = x2039 + z2040 = x2040 + z2041 = x2041 + z2042 = x2042 + z2043 = x2043 + z2044 = x2044 + z2045 = x2045 + z2046 = x2046 + z2047 = x2047 + z2048 = x2048 + z2049 = x2049 + z2050 = x2050 + z2051 = x2051 + z2052 = x2052 + z2053 = x2053 + z2054 = x2054 + z2055 = x2055 + z2056 = x2056 + z2057 = x2057 + z2058 = x2058 + z2059 = x2059 + z2060 = x2060 + z2061 = x2061 + z2062 = x2062 + z2063 = x2063 + z2064 = x2064 + z2065 = x2065 + z2066 = x2066 + z2067 = x2067 + z2068 = x2068 + z2069 = x2069 + z2070 = x2070 + z2071 = x2071 + z2072 = x2072 + z2073 = x2073 + z2074 = x2074 + z2075 = x2075 + z2076 = x2076 + z2077 = x2077 + z2078 = x2078 + z2079 = x2079 + z2080 = x2080 + z2081 = x2081 + z2082 = x2082 + z2083 = x2083 + z2084 = x2084 + z2085 = x2085 + z2086 = x2086 + z2087 = x2087 + z2088 = x2088 + z2089 = x2089 + z2090 = x2090 + z2091 = x2091 + z2092 = x2092 + z2093 = x2093 + z2094 = x2094 + z2095 = x2095 + z2096 = x2096 + z2097 = x2097 + z2098 = x2098 + z2099 = x2099 + z2100 = x2100 + z2101 = x2101 + z2102 = x2102 + z2103 = x2103 + z2104 = x2104 + z2105 = x2105 + z2106 = x2106 + z2107 = x2107 + z2108 = x2108 + z2109 = x2109 + z2110 = x2110 + z2111 = x2111 + z2112 = x2112 + z2113 = x2113 + z2114 = x2114 + z2115 = x2115 + z2116 = x2116 + z2117 = x2117 + z2118 = x2118 + z2119 = x2119 + z2120 = x2120 + z2121 = x2121 + z2122 = x2122 + z2123 = x2123 + z2124 = x2124 + z2125 = x2125 + z2126 = x2126 + z2127 = x2127 + z2128 = x2128 + z2129 = x2129 + z2130 = x2130 + z2131 = x2131 + z2132 = x2132 + z2133 = x2133 + z2134 = x2134 + z2135 = x2135 + z2136 = x2136 + z2137 = x2137 + z2138 = x2138 + z2139 = x2139 + z2140 = x2140 + z2141 = x2141 + z2142 = x2142 + z2143 = x2143 + z2144 = x2144 + z2145 = x2145 + z2146 = x2146 + z2147 = x2147 + z2148 = x2148 + z2149 = x2149 + z2150 = x2150 + z2151 = x2151 + z2152 = x2152 + z2153 = x2153 + z2154 = x2154 + z2155 = x2155 + z2156 = x2156 + z2157 = x2157 + z2158 = x2158 + z2159 = x2159 + z2160 = x2160 + z2161 = x2161 + z2162 = x2162 + z2163 = x2163 + z2164 = x2164 + z2165 = x2165 + z2166 = x2166 + z2167 = x2167 + z2168 = x2168 + z2169 = x2169 + z2170 = x2170 + z2171 = x2171 + z2172 = x2172 + z2173 = x2173 + z2174 = x2174 + z2175 = x2175 + z2176 = x2176 + z2177 = x2177 + z2178 = x2178 + z2179 = x2179 + z2180 = x2180 + z2181 = x2181 + z2182 = x2182 + z2183 = x2183 + z2184 = x2184 + z2185 = x2185 + z2186 = x2186 + z2187 = x2187 + z2188 = x2188 + z2189 = x2189 + z2190 = x2190 + z2191 = x2191 + z2192 = x2192 + z2193 = x2193 + z2194 = x2194 + z2195 = x2195 + z2196 = x2196 + z2197 = x2197 + z2198 = x2198 + z2199 = x2199 + z2200 = x2200 + z2201 = x2201 + z2202 = x2202 + z2203 = x2203 + z2204 = x2204 + z2205 = x2205 + z2206 = x2206 + z2207 = x2207 + z2208 = x2208 + z2209 = x2209 + z2210 = x2210 + z2211 = x2211 + z2212 = x2212 + z2213 = x2213 + z2214 = x2214 + z2215 = x2215 + z2216 = x2216 + z2217 = x2217 + z2218 = x2218 + z2219 = x2219 + z2220 = x2220 + z2221 = x2221 + z2222 = x2222 + z2223 = x2223 + z2224 = x2224 + z2225 = x2225 + z2226 = x2226 + z2227 = x2227 + z2228 = x2228 + z2229 = x2229 + z2230 = x2230 + z2231 = x2231 + z2232 = x2232 + z2233 = x2233 + z2234 = x2234 + z2235 = x2235 + z2236 = x2236 + z2237 = x2237 + z2238 = x2238 + z2239 = x2239 + z2240 = x2240 + z2241 = x2241 + z2242 = x2242 + z2243 = x2243 + z2244 = x2244 + z2245 = x2245 + z2246 = x2246 + z2247 = x2247 + z2248 = x2248 + z2249 = x2249 + z2250 = x2250 + z2251 = x2251 + z2252 = x2252 + z2253 = x2253 + z2254 = x2254 + z2255 = x2255 + z2256 = x2256 + z2257 = x2257 + z2258 = x2258 + z2259 = x2259 + z2260 = x2260 + z2261 = x2261 + z2262 = x2262 + z2263 = x2263 + z2264 = x2264 + z2265 = x2265 + z2266 = x2266 + z2267 = x2267 + z2268 = x2268 + z2269 = x2269 + z2270 = x2270 + z2271 = x2271 + z2272 = x2272 + z2273 = x2273 + z2274 = x2274 + z2275 = x2275 + z2276 = x2276 + z2277 = x2277 + z2278 = x2278 + z2279 = x2279 + z2280 = x2280 + z2281 = x2281 + z2282 = x2282 + z2283 = x2283 + z2284 = x2284 + z2285 = x2285 + z2286 = x2286 + z2287 = x2287 + z2288 = x2288 + z2289 = x2289 + z2290 = x2290 + z2291 = x2291 + z2292 = x2292 + z2293 = x2293 + z2294 = x2294 + z2295 = x2295 + z2296 = x2296 + z2297 = x2297 + z2298 = x2298 + z2299 = x2299 + z2300 = x2300 + z2301 = x2301 + z2302 = x2302 + z2303 = x2303 + z2304 = x2304 + z2305 = x2305 + z2306 = x2306 + z2307 = x2307 + z2308 = x2308 + z2309 = x2309 + z2310 = x2310 + z2311 = x2311 + z2312 = x2312 + z2313 = x2313 + z2314 = x2314 + z2315 = x2315 + z2316 = x2316 + z2317 = x2317 + z2318 = x2318 + z2319 = x2319 + z2320 = x2320 + z2321 = x2321 + z2322 = x2322 + z2323 = x2323 + z2324 = x2324 + z2325 = x2325 + z2326 = x2326 + z2327 = x2327 + z2328 = x2328 + z2329 = x2329 + z2330 = x2330 + z2331 = x2331 + z2332 = x2332 + z2333 = x2333 + z2334 = x2334 + z2335 = x2335 + z2336 = x2336 + z2337 = x2337 + z2338 = x2338 + z2339 = x2339 + z2340 = x2340 + z2341 = x2341 + z2342 = x2342 + z2343 = x2343 + z2344 = x2344 + z2345 = x2345 + z2346 = x2346 + z2347 = x2347 + z2348 = x2348 + z2349 = x2349 + z2350 = x2350 + z2351 = x2351 + z2352 = x2352 + z2353 = x2353 + z2354 = x2354 + z2355 = x2355 + z2356 = x2356 + z2357 = x2357 + z2358 = x2358 + z2359 = x2359 + z2360 = x2360 + z2361 = x2361 + z2362 = x2362 + z2363 = x2363 + z2364 = x2364 + z2365 = x2365 + z2366 = x2366 + z2367 = x2367 + z2368 = x2368 + z2369 = x2369 + z2370 = x2370 + z2371 = x2371 + z2372 = x2372 + z2373 = x2373 + z2374 = x2374 + z2375 = x2375 + z2376 = x2376 + z2377 = x2377 + z2378 = x2378 + z2379 = x2379 + z2380 = x2380 + z2381 = x2381 + z2382 = x2382 + z2383 = x2383 + z2384 = x2384 + z2385 = x2385 + z2386 = x2386 + z2387 = x2387 + z2388 = x2388 + z2389 = x2389 + z2390 = x2390 + z2391 = x2391 + z2392 = x2392 + z2393 = x2393 + z2394 = x2394 + z2395 = x2395 + z2396 = x2396 + z2397 = x2397 + z2398 = x2398 + z2399 = x2399 + z2400 = x2400 + z2401 = x2401 + z2402 = x2402 + z2403 = x2403 + z2404 = x2404 + z2405 = x2405 + z2406 = x2406 + z2407 = x2407 + z2408 = x2408 + z2409 = x2409 + z2410 = x2410 + z2411 = x2411 + z2412 = x2412 + z2413 = x2413 + z2414 = x2414 + z2415 = x2415 + z2416 = x2416 + z2417 = x2417 + z2418 = x2418 + z2419 = x2419 + z2420 = x2420 + z2421 = x2421 + z2422 = x2422 + z2423 = x2423 + z2424 = x2424 + z2425 = x2425 + z2426 = x2426 + z2427 = x2427 + z2428 = x2428 + z2429 = x2429 + z2430 = x2430 + z2431 = x2431 + z2432 = x2432 + z2433 = x2433 + z2434 = x2434 + z2435 = x2435 + z2436 = x2436 + z2437 = x2437 + z2438 = x2438 + z2439 = x2439 + z2440 = x2440 + z2441 = x2441 + z2442 = x2442 + z2443 = x2443 + z2444 = x2444 + z2445 = x2445 + z2446 = x2446 + z2447 = x2447 + z2448 = x2448 + z2449 = x2449 + z2450 = x2450 + z2451 = x2451 + z2452 = x2452 + z2453 = x2453 + z2454 = x2454 + z2455 = x2455 + z2456 = x2456 + z2457 = x2457 + z2458 = x2458 + z2459 = x2459 + z2460 = x2460 + z2461 = x2461 + z2462 = x2462 + z2463 = x2463 + z2464 = x2464 + z2465 = x2465 + z2466 = x2466 + z2467 = x2467 + z2468 = x2468 + z2469 = x2469 + z2470 = x2470 + z2471 = x2471 + z2472 = x2472 + z2473 = x2473 + z2474 = x2474 + z2475 = x2475 + z2476 = x2476 + z2477 = x2477 + z2478 = x2478 + z2479 = x2479 + z2480 = x2480 + z2481 = x2481 + z2482 = x2482 + z2483 = x2483 + z2484 = x2484 + z2485 = x2485 + z2486 = x2486 + z2487 = x2487 + z2488 = x2488 + z2489 = x2489 + z2490 = x2490 + z2491 = x2491 + z2492 = x2492 + z2493 = x2493 + z2494 = x2494 + z2495 = x2495 + z2496 = x2496 + z2497 = x2497 + z2498 = x2498 + z2499 = x2499 + z2500 = x2500 + z2501 = x2501 + z2502 = x2502 + z2503 = x2503 + z2504 = x2504 + z2505 = x2505 + z2506 = x2506 + z2507 = x2507 + z2508 = x2508 + z2509 = x2509 + z2510 = x2510 + z2511 = x2511 + z2512 = x2512 + z2513 = x2513 + z2514 = x2514 + z2515 = x2515 + z2516 = x2516 + z2517 = x2517 + z2518 = x2518 + z2519 = x2519 + z2520 = x2520 + z2521 = x2521 + z2522 = x2522 + z2523 = x2523 + z2524 = x2524 + z2525 = x2525 + z2526 = x2526 + z2527 = x2527 + z2528 = x2528 + z2529 = x2529 + z2530 = x2530 + z2531 = x2531 + z2532 = x2532 + z2533 = x2533 + z2534 = x2534 + z2535 = x2535 + z2536 = x2536 + z2537 = x2537 + z2538 = x2538 + z2539 = x2539 + z2540 = x2540 + z2541 = x2541 + z2542 = x2542 + z2543 = x2543 + z2544 = x2544 + z2545 = x2545 + z2546 = x2546 + z2547 = x2547 + z2548 = x2548 + z2549 = x2549 + z2550 = x2550 + z2551 = x2551 + z2552 = x2552 + z2553 = x2553 + z2554 = x2554 + z2555 = x2555 + z2556 = x2556 + z2557 = x2557 + z2558 = x2558 + z2559 = x2559 + z2560 = x2560 + z2561 = x2561 + z2562 = x2562 + z2563 = x2563 + z2564 = x2564 + z2565 = x2565 + z2566 = x2566 + z2567 = x2567 + z2568 = x2568 + z2569 = x2569 + z2570 = x2570 + z2571 = x2571 + z2572 = x2572 + z2573 = x2573 + z2574 = x2574 + z2575 = x2575 + z2576 = x2576 + z2577 = x2577 + z2578 = x2578 + z2579 = x2579 + z2580 = x2580 + z2581 = x2581 + z2582 = x2582 + z2583 = x2583 + z2584 = x2584 + z2585 = x2585 + z2586 = x2586 + z2587 = x2587 + z2588 = x2588 + z2589 = x2589 + z2590 = x2590 + z2591 = x2591 + z2592 = x2592 + z2593 = x2593 + z2594 = x2594 + z2595 = x2595 + z2596 = x2596 + z2597 = x2597 + z2598 = x2598 + z2599 = x2599 + z2600 = x2600 + z2601 = x2601 + z2602 = x2602 + z2603 = x2603 + z2604 = x2604 + z2605 = x2605 + z2606 = x2606 + z2607 = x2607 + z2608 = x2608 + z2609 = x2609 + z2610 = x2610 + z2611 = x2611 + z2612 = x2612 + z2613 = x2613 + z2614 = x2614 + z2615 = x2615 + z2616 = x2616 + z2617 = x2617 + z2618 = x2618 + z2619 = x2619 + z2620 = x2620 + z2621 = x2621 + z2622 = x2622 + z2623 = x2623 + z2624 = x2624 + z2625 = x2625 + z2626 = x2626 + z2627 = x2627 + z2628 = x2628 + z2629 = x2629 + z2630 = x2630 + z2631 = x2631 + z2632 = x2632 + z2633 = x2633 + z2634 = x2634 + z2635 = x2635 + z2636 = x2636 + z2637 = x2637 + z2638 = x2638 + z2639 = x2639 + z2640 = x2640 + z2641 = x2641 + z2642 = x2642 + z2643 = x2643 + z2644 = x2644 + z2645 = x2645 + z2646 = x2646 + z2647 = x2647 + z2648 = x2648 + z2649 = x2649 + z2650 = x2650 + z2651 = x2651 + z2652 = x2652 + z2653 = x2653 + z2654 = x2654 + z2655 = x2655 + z2656 = x2656 + z2657 = x2657 + z2658 = x2658 + z2659 = x2659 + z2660 = x2660 + z2661 = x2661 + z2662 = x2662 + z2663 = x2663 + z2664 = x2664 + z2665 = x2665 + z2666 = x2666 + z2667 = x2667 + z2668 = x2668 + z2669 = x2669 + z2670 = x2670 + z2671 = x2671 + z2672 = x2672 + z2673 = x2673 + z2674 = x2674 + z2675 = x2675 + z2676 = x2676 + z2677 = x2677 + z2678 = x2678 + z2679 = x2679 + z2680 = x2680 + z2681 = x2681 + z2682 = x2682 + z2683 = x2683 + z2684 = x2684 + z2685 = x2685 + z2686 = x2686 + z2687 = x2687 + z2688 = x2688 + z2689 = x2689 + z2690 = x2690 + z2691 = x2691 + z2692 = x2692 + z2693 = x2693 + z2694 = x2694 + z2695 = x2695 + z2696 = x2696 + z2697 = x2697 + z2698 = x2698 + z2699 = x2699 + z2700 = x2700 + z2701 = x2701 + z2702 = x2702 + z2703 = x2703 + z2704 = x2704 + z2705 = x2705 + z2706 = x2706 + z2707 = x2707 + z2708 = x2708 + z2709 = x2709 + z2710 = x2710 + z2711 = x2711 + z2712 = x2712 + z2713 = x2713 + z2714 = x2714 + z2715 = x2715 + z2716 = x2716 + z2717 = x2717 + z2718 = x2718 + z2719 = x2719 + z2720 = x2720 + z2721 = x2721 + z2722 = x2722 + z2723 = x2723 + z2724 = x2724 + z2725 = x2725 + z2726 = x2726 + z2727 = x2727 + z2728 = x2728 + z2729 = x2729 + z2730 = x2730 + z2731 = x2731 + z2732 = x2732 + z2733 = x2733 + z2734 = x2734 + z2735 = x2735 + z2736 = x2736 + z2737 = x2737 + z2738 = x2738 + z2739 = x2739 + z2740 = x2740 + z2741 = x2741 + z2742 = x2742 + z2743 = x2743 + z2744 = x2744 + z2745 = x2745 + z2746 = x2746 + z2747 = x2747 + z2748 = x2748 + z2749 = x2749 + z2750 = x2750 + z2751 = x2751 + z2752 = x2752 + z2753 = x2753 + z2754 = x2754 + z2755 = x2755 + z2756 = x2756 + z2757 = x2757 + z2758 = x2758 + z2759 = x2759 + z2760 = x2760 + z2761 = x2761 + z2762 = x2762 + z2763 = x2763 + z2764 = x2764 + z2765 = x2765 + z2766 = x2766 + z2767 = x2767 + z2768 = x2768 + z2769 = x2769 + z2770 = x2770 + z2771 = x2771 + z2772 = x2772 + z2773 = x2773 + z2774 = x2774 + z2775 = x2775 + z2776 = x2776 + z2777 = x2777 + z2778 = x2778 + z2779 = x2779 + z2780 = x2780 + z2781 = x2781 + z2782 = x2782 + z2783 = x2783 + z2784 = x2784 + z2785 = x2785 + z2786 = x2786 + z2787 = x2787 + z2788 = x2788 + z2789 = x2789 + z2790 = x2790 + z2791 = x2791 + z2792 = x2792 + z2793 = x2793 + z2794 = x2794 + z2795 = x2795 + z2796 = x2796 + z2797 = x2797 + z2798 = x2798 + z2799 = x2799 + z2800 = x2800 + z2801 = x2801 + z2802 = x2802 + z2803 = x2803 + z2804 = x2804 + z2805 = x2805 + z2806 = x2806 + z2807 = x2807 + z2808 = x2808 + z2809 = x2809 + z2810 = x2810 + z2811 = x2811 + z2812 = x2812 + z2813 = x2813 + z2814 = x2814 + z2815 = x2815 + z2816 = x2816 + z2817 = x2817 + z2818 = x2818 + z2819 = x2819 + z2820 = x2820 + z2821 = x2821 + z2822 = x2822 + z2823 = x2823 + z2824 = x2824 + z2825 = x2825 + z2826 = x2826 + z2827 = x2827 + z2828 = x2828 + z2829 = x2829 + z2830 = x2830 + z2831 = x2831 + z2832 = x2832 + z2833 = x2833 + z2834 = x2834 + z2835 = x2835 + z2836 = x2836 + z2837 = x2837 + z2838 = x2838 + z2839 = x2839 + z2840 = x2840 + z2841 = x2841 + z2842 = x2842 + z2843 = x2843 + z2844 = x2844 + z2845 = x2845 + z2846 = x2846 + z2847 = x2847 + z2848 = x2848 + z2849 = x2849 + z2850 = x2850 + z2851 = x2851 + z2852 = x2852 + z2853 = x2853 + z2854 = x2854 + z2855 = x2855 + z2856 = x2856 + z2857 = x2857 + z2858 = x2858 + z2859 = x2859 + z2860 = x2860 + z2861 = x2861 + z2862 = x2862 + z2863 = x2863 + z2864 = x2864 + z2865 = x2865 + z2866 = x2866 + z2867 = x2867 + z2868 = x2868 + z2869 = x2869 + z2870 = x2870 + z2871 = x2871 + z2872 = x2872 + z2873 = x2873 + z2874 = x2874 + z2875 = x2875 + z2876 = x2876 + z2877 = x2877 + z2878 = x2878 + z2879 = x2879 + z2880 = x2880 + z2881 = x2881 + z2882 = x2882 + z2883 = x2883 + z2884 = x2884 + z2885 = x2885 + z2886 = x2886 + z2887 = x2887 + z2888 = x2888 + z2889 = x2889 + z2890 = x2890 + z2891 = x2891 + z2892 = x2892 + z2893 = x2893 + z2894 = x2894 + z2895 = x2895 + z2896 = x2896 + z2897 = x2897 + z2898 = x2898 + z2899 = x2899 + z2900 = x2900 + z2901 = x2901 + z2902 = x2902 + z2903 = x2903 + z2904 = x2904 + z2905 = x2905 + z2906 = x2906 + z2907 = x2907 + z2908 = x2908 + z2909 = x2909 + z2910 = x2910 + z2911 = x2911 + z2912 = x2912 + z2913 = x2913 + z2914 = x2914 + z2915 = x2915 + z2916 = x2916 + z2917 = x2917 + z2918 = x2918 + z2919 = x2919 + z2920 = x2920 + z2921 = x2921 + z2922 = x2922 + z2923 = x2923 + z2924 = x2924 + z2925 = x2925 + z2926 = x2926 + z2927 = x2927 + z2928 = x2928 + z2929 = x2929 + z2930 = x2930 + z2931 = x2931 + z2932 = x2932 + z2933 = x2933 + z2934 = x2934 + z2935 = x2935 + z2936 = x2936 + z2937 = x2937 + z2938 = x2938 + z2939 = x2939 + z2940 = x2940 + z2941 = x2941 + z2942 = x2942 + z2943 = x2943 + z2944 = x2944 + z2945 = x2945 + z2946 = x2946 + z2947 = x2947 + z2948 = x2948 + z2949 = x2949 + z2950 = x2950 + z2951 = x2951 + z2952 = x2952 + z2953 = x2953 + z2954 = x2954 + z2955 = x2955 + z2956 = x2956 + z2957 = x2957 + z2958 = x2958 + z2959 = x2959 + z2960 = x2960 + z2961 = x2961 + z2962 = x2962 + z2963 = x2963 + z2964 = x2964 + z2965 = x2965 + z2966 = x2966 + z2967 = x2967 + z2968 = x2968 + z2969 = x2969 + z2970 = x2970 + z2971 = x2971 + z2972 = x2972 + z2973 = x2973 + z2974 = x2974 + z2975 = x2975 + z2976 = x2976 + z2977 = x2977 + z2978 = x2978 + z2979 = x2979 + z2980 = x2980 + z2981 = x2981 + z2982 = x2982 + z2983 = x2983 + z2984 = x2984 + z2985 = x2985 + z2986 = x2986 + z2987 = x2987 + z2988 = x2988 + z2989 = x2989 + z2990 = x2990 + z2991 = x2991 + z2992 = x2992 + z2993 = x2993 + z2994 = x2994 + z2995 = x2995 + z2996 = x2996 + z2997 = x2997 + z2998 = x2998 + z2999 = x2999 + z3000 = x3000 + z3001 = x3001 + z3002 = x3002 + z3003 = x3003 + z3004 = x3004 + z3005 = x3005 + z3006 = x3006 + z3007 = x3007 + z3008 = x3008 + z3009 = x3009 + z3010 = x3010 + z3011 = x3011 + z3012 = x3012 + z3013 = x3013 + z3014 = x3014 + z3015 = x3015 + z3016 = x3016 + z3017 = x3017 + z3018 = x3018 + z3019 = x3019 + z3020 = x3020 + z3021 = x3021 + z3022 = x3022 + z3023 = x3023 + z3024 = x3024 + z3025 = x3025 + z3026 = x3026 + z3027 = x3027 + z3028 = x3028 + z3029 = x3029 + z3030 = x3030 + z3031 = x3031 + z3032 = x3032 + z3033 = x3033 + z3034 = x3034 + z3035 = x3035 + z3036 = x3036 + z3037 = x3037 + z3038 = x3038 + z3039 = x3039 + z3040 = x3040 + z3041 = x3041 + z3042 = x3042 + z3043 = x3043 + z3044 = x3044 + z3045 = x3045 + z3046 = x3046 + z3047 = x3047 + z3048 = x3048 + z3049 = x3049 + z3050 = x3050 + z3051 = x3051 + z3052 = x3052 + z3053 = x3053 + z3054 = x3054 + z3055 = x3055 + z3056 = x3056 + z3057 = x3057 + z3058 = x3058 + z3059 = x3059 + z3060 = x3060 + z3061 = x3061 + z3062 = x3062 + z3063 = x3063 + z3064 = x3064 + z3065 = x3065 + z3066 = x3066 + z3067 = x3067 + z3068 = x3068 + z3069 = x3069 + z3070 = x3070 + z3071 = x3071 + z3072 = x3072 + z3073 = x3073 + z3074 = x3074 + z3075 = x3075 + z3076 = x3076 + z3077 = x3077 + z3078 = x3078 + z3079 = x3079 + z3080 = x3080 + z3081 = x3081 + z3082 = x3082 + z3083 = x3083 + z3084 = x3084 + z3085 = x3085 + z3086 = x3086 + z3087 = x3087 + z3088 = x3088 + z3089 = x3089 + z3090 = x3090 + z3091 = x3091 + z3092 = x3092 + z3093 = x3093 + z3094 = x3094 + z3095 = x3095 + z3096 = x3096 + z3097 = x3097 + z3098 = x3098 + z3099 = x3099 + z3100 = x3100 + z3101 = x3101 + z3102 = x3102 + z3103 = x3103 + z3104 = x3104 + z3105 = x3105 + z3106 = x3106 + z3107 = x3107 + z3108 = x3108 + z3109 = x3109 + z3110 = x3110 + z3111 = x3111 + z3112 = x3112 + z3113 = x3113 + z3114 = x3114 + z3115 = x3115 + z3116 = x3116 + z3117 = x3117 + z3118 = x3118 + z3119 = x3119 + z3120 = x3120 + z3121 = x3121 + z3122 = x3122 + z3123 = x3123 + z3124 = x3124 + z3125 = x3125 + z3126 = x3126 + z3127 = x3127 + z3128 = x3128 + z3129 = x3129 + z3130 = x3130 + z3131 = x3131 + z3132 = x3132 + z3133 = x3133 + z3134 = x3134 + z3135 = x3135 + z3136 = x3136 + z3137 = x3137 + z3138 = x3138 + z3139 = x3139 + z3140 = x3140 + z3141 = x3141 + z3142 = x3142 + z3143 = x3143 + z3144 = x3144 + z3145 = x3145 + z3146 = x3146 + z3147 = x3147 + z3148 = x3148 + z3149 = x3149 + z3150 = x3150 + z3151 = x3151 + z3152 = x3152 + z3153 = x3153 + z3154 = x3154 + z3155 = x3155 + z3156 = x3156 + z3157 = x3157 + z3158 = x3158 + z3159 = x3159 + z3160 = x3160 + z3161 = x3161 + z3162 = x3162 + z3163 = x3163 + z3164 = x3164 + z3165 = x3165 + z3166 = x3166 + z3167 = x3167 + z3168 = x3168 + z3169 = x3169 + z3170 = x3170 + z3171 = x3171 + z3172 = x3172 + z3173 = x3173 + z3174 = x3174 + z3175 = x3175 + z3176 = x3176 + z3177 = x3177 + z3178 = x3178 + z3179 = x3179 + z3180 = x3180 + z3181 = x3181 + z3182 = x3182 + z3183 = x3183 + z3184 = x3184 + z3185 = x3185 + z3186 = x3186 + z3187 = x3187 + z3188 = x3188 + z3189 = x3189 + z3190 = x3190 + z3191 = x3191 + z3192 = x3192 + z3193 = x3193 + z3194 = x3194 + z3195 = x3195 + z3196 = x3196 + z3197 = x3197 + z3198 = x3198 + z3199 = x3199 + z3200 = x3200 + z3201 = x3201 + z3202 = x3202 + z3203 = x3203 + z3204 = x3204 + z3205 = x3205 + z3206 = x3206 + z3207 = x3207 + z3208 = x3208 + z3209 = x3209 + z3210 = x3210 + z3211 = x3211 + z3212 = x3212 + z3213 = x3213 + z3214 = x3214 + z3215 = x3215 + z3216 = x3216 + z3217 = x3217 + z3218 = x3218 + z3219 = x3219 + z3220 = x3220 + z3221 = x3221 + z3222 = x3222 + z3223 = x3223 + z3224 = x3224 + z3225 = x3225 + z3226 = x3226 + z3227 = x3227 + z3228 = x3228 + z3229 = x3229 + z3230 = x3230 + z3231 = x3231 + z3232 = x3232 + z3233 = x3233 + z3234 = x3234 + z3235 = x3235 + z3236 = x3236 + z3237 = x3237 + z3238 = x3238 + z3239 = x3239 + z3240 = x3240 + z3241 = x3241 + z3242 = x3242 + z3243 = x3243 + z3244 = x3244 + z3245 = x3245 + z3246 = x3246 + z3247 = x3247 + z3248 = x3248 + z3249 = x3249 + z3250 = x3250 + z3251 = x3251 + z3252 = x3252 + z3253 = x3253 + z3254 = x3254 + z3255 = x3255 + z3256 = x3256 + z3257 = x3257 + z3258 = x3258 + z3259 = x3259 + z3260 = x3260 + z3261 = x3261 + z3262 = x3262 + z3263 = x3263 + z3264 = x3264 + z3265 = x3265 + z3266 = x3266 + z3267 = x3267 + z3268 = x3268 + z3269 = x3269 + z3270 = x3270 + z3271 = x3271 + z3272 = x3272 + z3273 = x3273 + z3274 = x3274 + z3275 = x3275 + z3276 = x3276 + z3277 = x3277 + z3278 = x3278 + z3279 = x3279 + z3280 = x3280 + z3281 = x3281 + z3282 = x3282 + z3283 = x3283 + z3284 = x3284 + z3285 = x3285 + z3286 = x3286 + z3287 = x3287 + z3288 = x3288 + z3289 = x3289 + z3290 = x3290 + z3291 = x3291 + z3292 = x3292 + z3293 = x3293 + z3294 = x3294 + z3295 = x3295 + z3296 = x3296 + z3297 = x3297 + z3298 = x3298 + z3299 = x3299 + z3300 = x3300 + z3301 = x3301 + z3302 = x3302 + z3303 = x3303 + z3304 = x3304 + z3305 = x3305 + z3306 = x3306 + z3307 = x3307 + z3308 = x3308 + z3309 = x3309 + z3310 = x3310 + z3311 = x3311 + z3312 = x3312 + z3313 = x3313 + z3314 = x3314 + z3315 = x3315 + z3316 = x3316 + z3317 = x3317 + z3318 = x3318 + z3319 = x3319 + z3320 = x3320 + z3321 = x3321 + z3322 = x3322 + z3323 = x3323 + z3324 = x3324 + z3325 = x3325 + z3326 = x3326 + z3327 = x3327 + z3328 = x3328 + z3329 = x3329 + z3330 = x3330 + z3331 = x3331 + z3332 = x3332 + z3333 = x3333 + z3334 = x3334 + z3335 = x3335 + z3336 = x3336 + z3337 = x3337 + z3338 = x3338 + z3339 = x3339 + z3340 = x3340 + z3341 = x3341 + z3342 = x3342 + z3343 = x3343 + z3344 = x3344 + z3345 = x3345 + z3346 = x3346 + z3347 = x3347 + z3348 = x3348 + z3349 = x3349 + z3350 = x3350 + z3351 = x3351 + z3352 = x3352 + z3353 = x3353 + z3354 = x3354 + z3355 = x3355 + z3356 = x3356 + z3357 = x3357 + z3358 = x3358 + z3359 = x3359 + z3360 = x3360 + z3361 = x3361 + z3362 = x3362 + z3363 = x3363 + z3364 = x3364 + z3365 = x3365 + z3366 = x3366 + z3367 = x3367 + z3368 = x3368 + z3369 = x3369 + z3370 = x3370 + z3371 = x3371 + z3372 = x3372 + z3373 = x3373 + z3374 = x3374 + z3375 = x3375 + z3376 = x3376 + z3377 = x3377 + z3378 = x3378 + z3379 = x3379 + z3380 = x3380 + z3381 = x3381 + z3382 = x3382 + z3383 = x3383 + z3384 = x3384 + z3385 = x3385 + z3386 = x3386 + z3387 = x3387 + z3388 = x3388 + z3389 = x3389 + z3390 = x3390 + z3391 = x3391 + z3392 = x3392 + z3393 = x3393 + z3394 = x3394 + z3395 = x3395 + z3396 = x3396 + z3397 = x3397 + z3398 = x3398 + z3399 = x3399 + z3400 = x3400 + z3401 = x3401 + z3402 = x3402 + z3403 = x3403 + z3404 = x3404 + z3405 = x3405 + z3406 = x3406 + z3407 = x3407 + z3408 = x3408 + z3409 = x3409 + z3410 = x3410 + z3411 = x3411 + z3412 = x3412 + z3413 = x3413 + z3414 = x3414 + z3415 = x3415 + z3416 = x3416 + z3417 = x3417 + z3418 = x3418 + z3419 = x3419 + z3420 = x3420 + z3421 = x3421 + z3422 = x3422 + z3423 = x3423 + z3424 = x3424 + z3425 = x3425 + z3426 = x3426 + z3427 = x3427 + z3428 = x3428 + z3429 = x3429 + z3430 = x3430 + z3431 = x3431 + z3432 = x3432 + z3433 = x3433 + z3434 = x3434 + z3435 = x3435 + z3436 = x3436 + z3437 = x3437 + z3438 = x3438 + z3439 = x3439 + z3440 = x3440 + z3441 = x3441 + z3442 = x3442 + z3443 = x3443 + z3444 = x3444 + z3445 = x3445 + z3446 = x3446 + z3447 = x3447 + z3448 = x3448 + z3449 = x3449 + z3450 = x3450 + z3451 = x3451 + z3452 = x3452 + z3453 = x3453 + z3454 = x3454 + z3455 = x3455 + z3456 = x3456 + z3457 = x3457 + z3458 = x3458 + z3459 = x3459 + z3460 = x3460 + z3461 = x3461 + z3462 = x3462 + z3463 = x3463 + z3464 = x3464 + z3465 = x3465 + z3466 = x3466 + z3467 = x3467 + z3468 = x3468 + z3469 = x3469 + z3470 = x3470 + z3471 = x3471 + z3472 = x3472 + z3473 = x3473 + z3474 = x3474 + z3475 = x3475 + z3476 = x3476 + z3477 = x3477 + z3478 = x3478 + z3479 = x3479 + z3480 = x3480 + z3481 = x3481 + z3482 = x3482 + z3483 = x3483 + z3484 = x3484 + z3485 = x3485 + z3486 = x3486 + z3487 = x3487 + z3488 = x3488 + z3489 = x3489 + z3490 = x3490 + z3491 = x3491 + z3492 = x3492 + z3493 = x3493 + z3494 = x3494 + z3495 = x3495 + z3496 = x3496 + z3497 = x3497 + z3498 = x3498 + z3499 = x3499 + z3500 = x3500 + z3501 = x3501 + z3502 = x3502 + z3503 = x3503 + z3504 = x3504 + z3505 = x3505 + z3506 = x3506 + z3507 = x3507 + z3508 = x3508 + z3509 = x3509 + z3510 = x3510 + z3511 = x3511 + z3512 = x3512 + z3513 = x3513 + z3514 = x3514 + z3515 = x3515 + z3516 = x3516 + z3517 = x3517 + z3518 = x3518 + z3519 = x3519 + z3520 = x3520 + z3521 = x3521 + z3522 = x3522 + z3523 = x3523 + z3524 = x3524 + z3525 = x3525 + z3526 = x3526 + z3527 = x3527 + z3528 = x3528 + z3529 = x3529 + z3530 = x3530 + z3531 = x3531 + z3532 = x3532 + z3533 = x3533 + z3534 = x3534 + z3535 = x3535 + z3536 = x3536 + z3537 = x3537 + z3538 = x3538 + z3539 = x3539 + z3540 = x3540 + z3541 = x3541 + z3542 = x3542 + z3543 = x3543 + z3544 = x3544 + z3545 = x3545 + z3546 = x3546 + z3547 = x3547 + z3548 = x3548 + z3549 = x3549 + z3550 = x3550 + z3551 = x3551 + z3552 = x3552 + z3553 = x3553 + z3554 = x3554 + z3555 = x3555 + z3556 = x3556 + z3557 = x3557 + z3558 = x3558 + z3559 = x3559 + z3560 = x3560 + z3561 = x3561 + z3562 = x3562 + z3563 = x3563 + z3564 = x3564 + z3565 = x3565 + z3566 = x3566 + z3567 = x3567 + z3568 = x3568 + z3569 = x3569 + z3570 = x3570 + z3571 = x3571 + z3572 = x3572 + z3573 = x3573 + z3574 = x3574 + z3575 = x3575 + z3576 = x3576 + z3577 = x3577 + z3578 = x3578 + z3579 = x3579 + z3580 = x3580 + z3581 = x3581 + z3582 = x3582 + z3583 = x3583 + z3584 = x3584 + z3585 = x3585 + z3586 = x3586 + z3587 = x3587 + z3588 = x3588 + z3589 = x3589 + z3590 = x3590 + z3591 = x3591 + z3592 = x3592 + z3593 = x3593 + z3594 = x3594 + z3595 = x3595 + z3596 = x3596 + z3597 = x3597 + z3598 = x3598 + z3599 = x3599 + z3600 = x3600 + z3601 = x3601 + z3602 = x3602 + z3603 = x3603 + z3604 = x3604 + z3605 = x3605 + z3606 = x3606 + z3607 = x3607 + z3608 = x3608 + z3609 = x3609 + z3610 = x3610 + z3611 = x3611 + z3612 = x3612 + z3613 = x3613 + z3614 = x3614 + z3615 = x3615 + z3616 = x3616 + z3617 = x3617 + z3618 = x3618 + z3619 = x3619 + z3620 = x3620 + z3621 = x3621 + z3622 = x3622 + z3623 = x3623 + z3624 = x3624 + z3625 = x3625 + z3626 = x3626 + z3627 = x3627 + z3628 = x3628 + z3629 = x3629 + z3630 = x3630 + z3631 = x3631 + z3632 = x3632 + z3633 = x3633 + z3634 = x3634 + z3635 = x3635 + z3636 = x3636 + z3637 = x3637 + z3638 = x3638 + z3639 = x3639 + z3640 = x3640 + z3641 = x3641 + z3642 = x3642 + z3643 = x3643 + z3644 = x3644 + z3645 = x3645 + z3646 = x3646 + z3647 = x3647 + z3648 = x3648 + z3649 = x3649 + z3650 = x3650 + z3651 = x3651 + z3652 = x3652 + z3653 = x3653 + z3654 = x3654 + z3655 = x3655 + z3656 = x3656 + z3657 = x3657 + z3658 = x3658 + z3659 = x3659 + z3660 = x3660 + z3661 = x3661 + z3662 = x3662 + z3663 = x3663 + z3664 = x3664 + z3665 = x3665 + z3666 = x3666 + z3667 = x3667 + z3668 = x3668 + z3669 = x3669 + z3670 = x3670 + z3671 = x3671 + z3672 = x3672 + z3673 = x3673 + z3674 = x3674 + z3675 = x3675 + z3676 = x3676 + z3677 = x3677 + z3678 = x3678 + z3679 = x3679 + z3680 = x3680 + z3681 = x3681 + z3682 = x3682 + z3683 = x3683 + z3684 = x3684 + z3685 = x3685 + z3686 = x3686 + z3687 = x3687 + z3688 = x3688 + z3689 = x3689 + z3690 = x3690 + z3691 = x3691 + z3692 = x3692 + z3693 = x3693 + z3694 = x3694 + z3695 = x3695 + z3696 = x3696 + z3697 = x3697 + z3698 = x3698 + z3699 = x3699 + z3700 = x3700 + z3701 = x3701 + z3702 = x3702 + z3703 = x3703 + z3704 = x3704 + z3705 = x3705 + z3706 = x3706 + z3707 = x3707 + z3708 = x3708 + z3709 = x3709 + z3710 = x3710 + z3711 = x3711 + z3712 = x3712 + z3713 = x3713 + z3714 = x3714 + z3715 = x3715 + z3716 = x3716 + z3717 = x3717 + z3718 = x3718 + z3719 = x3719 + z3720 = x3720 + z3721 = x3721 + z3722 = x3722 + z3723 = x3723 + z3724 = x3724 + z3725 = x3725 + z3726 = x3726 + z3727 = x3727 + z3728 = x3728 + z3729 = x3729 + z3730 = x3730 + z3731 = x3731 + z3732 = x3732 + z3733 = x3733 + z3734 = x3734 + z3735 = x3735 + z3736 = x3736 + z3737 = x3737 + z3738 = x3738 + z3739 = x3739 + z3740 = x3740 + z3741 = x3741 + z3742 = x3742 + z3743 = x3743 + z3744 = x3744 + z3745 = x3745 + z3746 = x3746 + z3747 = x3747 + z3748 = x3748 + z3749 = x3749 + z3750 = x3750 + z3751 = x3751 + z3752 = x3752 + z3753 = x3753 + z3754 = x3754 + z3755 = x3755 + z3756 = x3756 + z3757 = x3757 + z3758 = x3758 + z3759 = x3759 + z3760 = x3760 + z3761 = x3761 + z3762 = x3762 + z3763 = x3763 + z3764 = x3764 + z3765 = x3765 + z3766 = x3766 + z3767 = x3767 + z3768 = x3768 + z3769 = x3769 + z3770 = x3770 + z3771 = x3771 + z3772 = x3772 + z3773 = x3773 + z3774 = x3774 + z3775 = x3775 + z3776 = x3776 + z3777 = x3777 + z3778 = x3778 + z3779 = x3779 + z3780 = x3780 + z3781 = x3781 + z3782 = x3782 + z3783 = x3783 + z3784 = x3784 + z3785 = x3785 + z3786 = x3786 + z3787 = x3787 + z3788 = x3788 + z3789 = x3789 + z3790 = x3790 + z3791 = x3791 + z3792 = x3792 + z3793 = x3793 + z3794 = x3794 + z3795 = x3795 + z3796 = x3796 + z3797 = x3797 + z3798 = x3798 + z3799 = x3799 + z3800 = x3800 + z3801 = x3801 + z3802 = x3802 + z3803 = x3803 + z3804 = x3804 + z3805 = x3805 + z3806 = x3806 + z3807 = x3807 + z3808 = x3808 + z3809 = x3809 + z3810 = x3810 + z3811 = x3811 + z3812 = x3812 + z3813 = x3813 + z3814 = x3814 + z3815 = x3815 + z3816 = x3816 + z3817 = x3817 + z3818 = x3818 + z3819 = x3819 + z3820 = x3820 + z3821 = x3821 + z3822 = x3822 + z3823 = x3823 + z3824 = x3824 + z3825 = x3825 + z3826 = x3826 + z3827 = x3827 + z3828 = x3828 + z3829 = x3829 + z3830 = x3830 + z3831 = x3831 + z3832 = x3832 + z3833 = x3833 + z3834 = x3834 + z3835 = x3835 + z3836 = x3836 + z3837 = x3837 + z3838 = x3838 + z3839 = x3839 + z3840 = x3840 + z3841 = x3841 + z3842 = x3842 + z3843 = x3843 + z3844 = x3844 + z3845 = x3845 + z3846 = x3846 + z3847 = x3847 + z3848 = x3848 + z3849 = x3849 + z3850 = x3850 + z3851 = x3851 + z3852 = x3852 + z3853 = x3853 + z3854 = x3854 + z3855 = x3855 + z3856 = x3856 + z3857 = x3857 + z3858 = x3858 + z3859 = x3859 + z3860 = x3860 + z3861 = x3861 + z3862 = x3862 + z3863 = x3863 + z3864 = x3864 + z3865 = x3865 + z3866 = x3866 + z3867 = x3867 + z3868 = x3868 + z3869 = x3869 + z3870 = x3870 + z3871 = x3871 + z3872 = x3872 + z3873 = x3873 + z3874 = x3874 + z3875 = x3875 + z3876 = x3876 + z3877 = x3877 + z3878 = x3878 + z3879 = x3879 + z3880 = x3880 + z3881 = x3881 + z3882 = x3882 + z3883 = x3883 + z3884 = x3884 + z3885 = x3885 + z3886 = x3886 + z3887 = x3887 + z3888 = x3888 + z3889 = x3889 + z3890 = x3890 + z3891 = x3891 + z3892 = x3892 + z3893 = x3893 + z3894 = x3894 + z3895 = x3895 + z3896 = x3896 + z3897 = x3897 + z3898 = x3898 + z3899 = x3899 + z3900 = x3900 + z3901 = x3901 + z3902 = x3902 + z3903 = x3903 + z3904 = x3904 + z3905 = x3905 + z3906 = x3906 + z3907 = x3907 + z3908 = x3908 + z3909 = x3909 + z3910 = x3910 + z3911 = x3911 + z3912 = x3912 + z3913 = x3913 + z3914 = x3914 + z3915 = x3915 + z3916 = x3916 + z3917 = x3917 + z3918 = x3918 + z3919 = x3919 + z3920 = x3920 + z3921 = x3921 + z3922 = x3922 + z3923 = x3923 + z3924 = x3924 + z3925 = x3925 + z3926 = x3926 + z3927 = x3927 + z3928 = x3928 + z3929 = x3929 + z3930 = x3930 + z3931 = x3931 + z3932 = x3932 + z3933 = x3933 + z3934 = x3934 + z3935 = x3935 + z3936 = x3936 + z3937 = x3937 + z3938 = x3938 + z3939 = x3939 + z3940 = x3940 + z3941 = x3941 + z3942 = x3942 + z3943 = x3943 + z3944 = x3944 + z3945 = x3945 + z3946 = x3946 + z3947 = x3947 + z3948 = x3948 + z3949 = x3949 + z3950 = x3950 + z3951 = x3951 + z3952 = x3952 + z3953 = x3953 + z3954 = x3954 + z3955 = x3955 + z3956 = x3956 + z3957 = x3957 + z3958 = x3958 + z3959 = x3959 + z3960 = x3960 + z3961 = x3961 + z3962 = x3962 + z3963 = x3963 + z3964 = x3964 + z3965 = x3965 + z3966 = x3966 + z3967 = x3967 + z3968 = x3968 + z3969 = x3969 + z3970 = x3970 + z3971 = x3971 + z3972 = x3972 + z3973 = x3973 + z3974 = x3974 + z3975 = x3975 + z3976 = x3976 + z3977 = x3977 + z3978 = x3978 + z3979 = x3979 + z3980 = x3980 + z3981 = x3981 + z3982 = x3982 + z3983 = x3983 + z3984 = x3984 + z3985 = x3985 + z3986 = x3986 + z3987 = x3987 + z3988 = x3988 + z3989 = x3989 + z3990 = x3990 + z3991 = x3991 + z3992 = x3992 + z3993 = x3993 + z3994 = x3994 + z3995 = x3995 + z3996 = x3996 + z3997 = x3997 + z3998 = x3998 + z3999 = x3999 + z4000 = x4000 + z4001 = x4001 + z4002 = x4002 + z4003 = x4003 + z4004 = x4004 + z4005 = x4005 + z4006 = x4006 + z4007 = x4007 + z4008 = x4008 + z4009 = x4009 + z4010 = x4010 + z4011 = x4011 + z4012 = x4012 + z4013 = x4013 + z4014 = x4014 + z4015 = x4015 + z4016 = x4016 + z4017 = x4017 + z4018 = x4018 + z4019 = x4019 + z4020 = x4020 + z4021 = x4021 + z4022 = x4022 + z4023 = x4023 + z4024 = x4024 + z4025 = x4025 + z4026 = x4026 + z4027 = x4027 + z4028 = x4028 + z4029 = x4029 + z4030 = x4030 + z4031 = x4031 + z4032 = x4032 + z4033 = x4033 + z4034 = x4034 + z4035 = x4035 + z4036 = x4036 + z4037 = x4037 + z4038 = x4038 + z4039 = x4039 + z4040 = x4040 + z4041 = x4041 + z4042 = x4042 + z4043 = x4043 + z4044 = x4044 + z4045 = x4045 + z4046 = x4046 + z4047 = x4047 + z4048 = x4048 + z4049 = x4049 + z4050 = x4050 + z4051 = x4051 + z4052 = x4052 + z4053 = x4053 + z4054 = x4054 + z4055 = x4055 + z4056 = x4056 + z4057 = x4057 + z4058 = x4058 + z4059 = x4059 + z4060 = x4060 + z4061 = x4061 + z4062 = x4062 + z4063 = x4063 + z4064 = x4064 + z4065 = x4065 + z4066 = x4066 + z4067 = x4067 + z4068 = x4068 + z4069 = x4069 + z4070 = x4070 + z4071 = x4071 + z4072 = x4072 + z4073 = x4073 + z4074 = x4074 + z4075 = x4075 + z4076 = x4076 + z4077 = x4077 + z4078 = x4078 + z4079 = x4079 + z4080 = x4080 + z4081 = x4081 + z4082 = x4082 + z4083 = x4083 + z4084 = x4084 + z4085 = x4085 + z4086 = x4086 + z4087 = x4087 + z4088 = x4088 + z4089 = x4089 + z4090 = x4090 + z4091 = x4091 + z4092 = x4092 + z4093 = x4093 + z4094 = x4094 + z4095 = x4095 + z4096 = x4096 + z4097 = x4097 + z4098 = x4098 + z4099 = x4099 + z4100 = x4100 + z4101 = x4101 + z4102 = x4102 + z4103 = x4103 + z4104 = x4104 + z4105 = x4105 + z4106 = x4106 + z4107 = x4107 + z4108 = x4108 + z4109 = x4109 + z4110 = x4110 + z4111 = x4111 + z4112 = x4112 + z4113 = x4113 + z4114 = x4114 + z4115 = x4115 + z4116 = x4116 + z4117 = x4117 + z4118 = x4118 + z4119 = x4119 + z4120 = x4120 + z4121 = x4121 + z4122 = x4122 + z4123 = x4123 + z4124 = x4124 + z4125 = x4125 + z4126 = x4126 + z4127 = x4127 + z4128 = x4128 + z4129 = x4129 + z4130 = x4130 + z4131 = x4131 + z4132 = x4132 + z4133 = x4133 + z4134 = x4134 + z4135 = x4135 + z4136 = x4136 + z4137 = x4137 + z4138 = x4138 + z4139 = x4139 + z4140 = x4140 + z4141 = x4141 + z4142 = x4142 + z4143 = x4143 + z4144 = x4144 + z4145 = x4145 + z4146 = x4146 + z4147 = x4147 + z4148 = x4148 + z4149 = x4149 + z4150 = x4150 + z4151 = x4151 + z4152 = x4152 + z4153 = x4153 + z4154 = x4154 + z4155 = x4155 + z4156 = x4156 + z4157 = x4157 + z4158 = x4158 + z4159 = x4159 + z4160 = x4160 + z4161 = x4161 + z4162 = x4162 + z4163 = x4163 + z4164 = x4164 + z4165 = x4165 + z4166 = x4166 + z4167 = x4167 + z4168 = x4168 + z4169 = x4169 + z4170 = x4170 + z4171 = x4171 + z4172 = x4172 + z4173 = x4173 + z4174 = x4174 + z4175 = x4175 + z4176 = x4176 + z4177 = x4177 + z4178 = x4178 + z4179 = x4179 + z4180 = x4180 + z4181 = x4181 + z4182 = x4182 + z4183 = x4183 + z4184 = x4184 + z4185 = x4185 + z4186 = x4186 + z4187 = x4187 + z4188 = x4188 + z4189 = x4189 + z4190 = x4190 + z4191 = x4191 + z4192 = x4192 + z4193 = x4193 + z4194 = x4194 + z4195 = x4195 + z4196 = x4196 + z4197 = x4197 + z4198 = x4198 + z4199 = x4199 + z4200 = x4200 + z4201 = x4201 + z4202 = x4202 + z4203 = x4203 + z4204 = x4204 + z4205 = x4205 + z4206 = x4206 + z4207 = x4207 + z4208 = x4208 + z4209 = x4209 + z4210 = x4210 + z4211 = x4211 + z4212 = x4212 + z4213 = x4213 + z4214 = x4214 + z4215 = x4215 + z4216 = x4216 + z4217 = x4217 + z4218 = x4218 + z4219 = x4219 + z4220 = x4220 + z4221 = x4221 + z4222 = x4222 + z4223 = x4223 + z4224 = x4224 + z4225 = x4225 + z4226 = x4226 + z4227 = x4227 + z4228 = x4228 + z4229 = x4229 + z4230 = x4230 + z4231 = x4231 + z4232 = x4232 + z4233 = x4233 + z4234 = x4234 + z4235 = x4235 + z4236 = x4236 + z4237 = x4237 + z4238 = x4238 + z4239 = x4239 + z4240 = x4240 + z4241 = x4241 + z4242 = x4242 + z4243 = x4243 + z4244 = x4244 + z4245 = x4245 + z4246 = x4246 + z4247 = x4247 + z4248 = x4248 + z4249 = x4249 + z4250 = x4250 + z4251 = x4251 + z4252 = x4252 + z4253 = x4253 + z4254 = x4254 + z4255 = x4255 + z4256 = x4256 + z4257 = x4257 + z4258 = x4258 + z4259 = x4259 + z4260 = x4260 + z4261 = x4261 + z4262 = x4262 + z4263 = x4263 + z4264 = x4264 + z4265 = x4265 + z4266 = x4266 + z4267 = x4267 + z4268 = x4268 + z4269 = x4269 + z4270 = x4270 + z4271 = x4271 + z4272 = x4272 + z4273 = x4273 + z4274 = x4274 + z4275 = x4275 + z4276 = x4276 + z4277 = x4277 + z4278 = x4278 + z4279 = x4279 + z4280 = x4280 + z4281 = x4281 + z4282 = x4282 + z4283 = x4283 + z4284 = x4284 + z4285 = x4285 + z4286 = x4286 + z4287 = x4287 + z4288 = x4288 + z4289 = x4289 + z4290 = x4290 + z4291 = x4291 + z4292 = x4292 + z4293 = x4293 + z4294 = x4294 + z4295 = x4295 + z4296 = x4296 + z4297 = x4297 + z4298 = x4298 + z4299 = x4299 + z4300 = x4300 + z4301 = x4301 + z4302 = x4302 + z4303 = x4303 + z4304 = x4304 + z4305 = x4305 + z4306 = x4306 + z4307 = x4307 + z4308 = x4308 + z4309 = x4309 + z4310 = x4310 + z4311 = x4311 + z4312 = x4312 + z4313 = x4313 + z4314 = x4314 + z4315 = x4315 + z4316 = x4316 + z4317 = x4317 + z4318 = x4318 + z4319 = x4319 + z4320 = x4320 + z4321 = x4321 + z4322 = x4322 + z4323 = x4323 + z4324 = x4324 + z4325 = x4325 + z4326 = x4326 + z4327 = x4327 + z4328 = x4328 + z4329 = x4329 + z4330 = x4330 + z4331 = x4331 + z4332 = x4332 + z4333 = x4333 + z4334 = x4334 + z4335 = x4335 + z4336 = x4336 + z4337 = x4337 + z4338 = x4338 + z4339 = x4339 + z4340 = x4340 + z4341 = x4341 + z4342 = x4342 + z4343 = x4343 + z4344 = x4344 + z4345 = x4345 + z4346 = x4346 + z4347 = x4347 + z4348 = x4348 + z4349 = x4349 + z4350 = x4350 + z4351 = x4351 + z4352 = x4352 + z4353 = x4353 + z4354 = x4354 + z4355 = x4355 + z4356 = x4356 + z4357 = x4357 + z4358 = x4358 + z4359 = x4359 + z4360 = x4360 + z4361 = x4361 + z4362 = x4362 + z4363 = x4363 + z4364 = x4364 + z4365 = x4365 + z4366 = x4366 + z4367 = x4367 + z4368 = x4368 + z4369 = x4369 + z4370 = x4370 + z4371 = x4371 + z4372 = x4372 + z4373 = x4373 + z4374 = x4374 + z4375 = x4375 + z4376 = x4376 + z4377 = x4377 + z4378 = x4378 + z4379 = x4379 + z4380 = x4380 + z4381 = x4381 + z4382 = x4382 + z4383 = x4383 + z4384 = x4384 + z4385 = x4385 + z4386 = x4386 + z4387 = x4387 + z4388 = x4388 + z4389 = x4389 + z4390 = x4390 + z4391 = x4391 + z4392 = x4392 + z4393 = x4393 + z4394 = x4394 + z4395 = x4395 + z4396 = x4396 + z4397 = x4397 + z4398 = x4398 + z4399 = x4399 + z4400 = x4400 + z4401 = x4401 + z4402 = x4402 + z4403 = x4403 + z4404 = x4404 + z4405 = x4405 + z4406 = x4406 + z4407 = x4407 + z4408 = x4408 + z4409 = x4409 + z4410 = x4410 + z4411 = x4411 + z4412 = x4412 + z4413 = x4413 + z4414 = x4414 + z4415 = x4415 + z4416 = x4416 + z4417 = x4417 + z4418 = x4418 + z4419 = x4419 + z4420 = x4420 + z4421 = x4421 + z4422 = x4422 + z4423 = x4423 + z4424 = x4424 + z4425 = x4425 + z4426 = x4426 + z4427 = x4427 + z4428 = x4428 + z4429 = x4429 + z4430 = x4430 + z4431 = x4431 + z4432 = x4432 + z4433 = x4433 + z4434 = x4434 + z4435 = x4435 + z4436 = x4436 + z4437 = x4437 + z4438 = x4438 + z4439 = x4439 + z4440 = x4440 + z4441 = x4441 + z4442 = x4442 + z4443 = x4443 + z4444 = x4444 + z4445 = x4445 + z4446 = x4446 + z4447 = x4447 + z4448 = x4448 + z4449 = x4449 + z4450 = x4450 + z4451 = x4451 + z4452 = x4452 + z4453 = x4453 + z4454 = x4454 + z4455 = x4455 + z4456 = x4456 + z4457 = x4457 + z4458 = x4458 + z4459 = x4459 + z4460 = x4460 + z4461 = x4461 + z4462 = x4462 + z4463 = x4463 + z4464 = x4464 + z4465 = x4465 + z4466 = x4466 + z4467 = x4467 + z4468 = x4468 + z4469 = x4469 + z4470 = x4470 + z4471 = x4471 + z4472 = x4472 + z4473 = x4473 + z4474 = x4474 + z4475 = x4475 + z4476 = x4476 + z4477 = x4477 + z4478 = x4478 + z4479 = x4479 + z4480 = x4480 + z4481 = x4481 + z4482 = x4482 + z4483 = x4483 + z4484 = x4484 + z4485 = x4485 + z4486 = x4486 + z4487 = x4487 + z4488 = x4488 + z4489 = x4489 + z4490 = x4490 + z4491 = x4491 + z4492 = x4492 + z4493 = x4493 + z4494 = x4494 + z4495 = x4495 + z4496 = x4496 + z4497 = x4497 + z4498 = x4498 + z4499 = x4499 + z4500 = x4500 + z4501 = x4501 + z4502 = x4502 + z4503 = x4503 + z4504 = x4504 + z4505 = x4505 + z4506 = x4506 + z4507 = x4507 + z4508 = x4508 + z4509 = x4509 + z4510 = x4510 + z4511 = x4511 + z4512 = x4512 + z4513 = x4513 + z4514 = x4514 + z4515 = x4515 + z4516 = x4516 + z4517 = x4517 + z4518 = x4518 + z4519 = x4519 + z4520 = x4520 + z4521 = x4521 + z4522 = x4522 + z4523 = x4523 + z4524 = x4524 + z4525 = x4525 + z4526 = x4526 + z4527 = x4527 + z4528 = x4528 + z4529 = x4529 + z4530 = x4530 + z4531 = x4531 + z4532 = x4532 + z4533 = x4533 + z4534 = x4534 + z4535 = x4535 + z4536 = x4536 + z4537 = x4537 + z4538 = x4538 + z4539 = x4539 + z4540 = x4540 + z4541 = x4541 + z4542 = x4542 + z4543 = x4543 + z4544 = x4544 + z4545 = x4545 + z4546 = x4546 + z4547 = x4547 + z4548 = x4548 + z4549 = x4549 + z4550 = x4550 + z4551 = x4551 + z4552 = x4552 + z4553 = x4553 + z4554 = x4554 + z4555 = x4555 + z4556 = x4556 + z4557 = x4557 + z4558 = x4558 + z4559 = x4559 + z4560 = x4560 + z4561 = x4561 + z4562 = x4562 + z4563 = x4563 + z4564 = x4564 + z4565 = x4565 + z4566 = x4566 + z4567 = x4567 + z4568 = x4568 + z4569 = x4569 + z4570 = x4570 + z4571 = x4571 + z4572 = x4572 + z4573 = x4573 + z4574 = x4574 + z4575 = x4575 + z4576 = x4576 + z4577 = x4577 + z4578 = x4578 + z4579 = x4579 + z4580 = x4580 + z4581 = x4581 + z4582 = x4582 + z4583 = x4583 + z4584 = x4584 + z4585 = x4585 + z4586 = x4586 + z4587 = x4587 + z4588 = x4588 + z4589 = x4589 + z4590 = x4590 + z4591 = x4591 + z4592 = x4592 + z4593 = x4593 + z4594 = x4594 + z4595 = x4595 + z4596 = x4596 + z4597 = x4597 + z4598 = x4598 + z4599 = x4599 + z4600 = x4600 + z4601 = x4601 + z4602 = x4602 + z4603 = x4603 + z4604 = x4604 + z4605 = x4605 + z4606 = x4606 + z4607 = x4607 + z4608 = x4608 + z4609 = x4609 + z4610 = x4610 + z4611 = x4611 + z4612 = x4612 + z4613 = x4613 + z4614 = x4614 + z4615 = x4615 + z4616 = x4616 + z4617 = x4617 + z4618 = x4618 + z4619 = x4619 + z4620 = x4620 + z4621 = x4621 + z4622 = x4622 + z4623 = x4623 + z4624 = x4624 + z4625 = x4625 + z4626 = x4626 + z4627 = x4627 + z4628 = x4628 + z4629 = x4629 + z4630 = x4630 + z4631 = x4631 + z4632 = x4632 + z4633 = x4633 + z4634 = x4634 + z4635 = x4635 + z4636 = x4636 + z4637 = x4637 + z4638 = x4638 + z4639 = x4639 + z4640 = x4640 + z4641 = x4641 + z4642 = x4642 + z4643 = x4643 + z4644 = x4644 + z4645 = x4645 + z4646 = x4646 + z4647 = x4647 + z4648 = x4648 + z4649 = x4649 + z4650 = x4650 + z4651 = x4651 + z4652 = x4652 + z4653 = x4653 + z4654 = x4654 + z4655 = x4655 + z4656 = x4656 + z4657 = x4657 + z4658 = x4658 + z4659 = x4659 + z4660 = x4660 + z4661 = x4661 + z4662 = x4662 + z4663 = x4663 + z4664 = x4664 + z4665 = x4665 + z4666 = x4666 + z4667 = x4667 + z4668 = x4668 + z4669 = x4669 + z4670 = x4670 + z4671 = x4671 + z4672 = x4672 + z4673 = x4673 + z4674 = x4674 + z4675 = x4675 + z4676 = x4676 + z4677 = x4677 + z4678 = x4678 + z4679 = x4679 + z4680 = x4680 + z4681 = x4681 + z4682 = x4682 + z4683 = x4683 + z4684 = x4684 + z4685 = x4685 + z4686 = x4686 + z4687 = x4687 + z4688 = x4688 + z4689 = x4689 + z4690 = x4690 + z4691 = x4691 + z4692 = x4692 + z4693 = x4693 + z4694 = x4694 + z4695 = x4695 + z4696 = x4696 + z4697 = x4697 + z4698 = x4698 + z4699 = x4699 + z4700 = x4700 + z4701 = x4701 + z4702 = x4702 + z4703 = x4703 + z4704 = x4704 + z4705 = x4705 + z4706 = x4706 + z4707 = x4707 + z4708 = x4708 + z4709 = x4709 + z4710 = x4710 + z4711 = x4711 + z4712 = x4712 + z4713 = x4713 + z4714 = x4714 + z4715 = x4715 + z4716 = x4716 + z4717 = x4717 + z4718 = x4718 + z4719 = x4719 + z4720 = x4720 + z4721 = x4721 + z4722 = x4722 + z4723 = x4723 + z4724 = x4724 + z4725 = x4725 + z4726 = x4726 + z4727 = x4727 + z4728 = x4728 + z4729 = x4729 + z4730 = x4730 + z4731 = x4731 + z4732 = x4732 + z4733 = x4733 + z4734 = x4734 + z4735 = x4735 + z4736 = x4736 + z4737 = x4737 + z4738 = x4738 + z4739 = x4739 + z4740 = x4740 + z4741 = x4741 + z4742 = x4742 + z4743 = x4743 + z4744 = x4744 + z4745 = x4745 + z4746 = x4746 + z4747 = x4747 + z4748 = x4748 + z4749 = x4749 + z4750 = x4750 + z4751 = x4751 + z4752 = x4752 + z4753 = x4753 + z4754 = x4754 + z4755 = x4755 + z4756 = x4756 + z4757 = x4757 + z4758 = x4758 + z4759 = x4759 + z4760 = x4760 + z4761 = x4761 + z4762 = x4762 + z4763 = x4763 + z4764 = x4764 + z4765 = x4765 + z4766 = x4766 + z4767 = x4767 + z4768 = x4768 + z4769 = x4769 + z4770 = x4770 + z4771 = x4771 + z4772 = x4772 + z4773 = x4773 + z4774 = x4774 + z4775 = x4775 + z4776 = x4776 + z4777 = x4777 + z4778 = x4778 + z4779 = x4779 + z4780 = x4780 + z4781 = x4781 + z4782 = x4782 + z4783 = x4783 + z4784 = x4784 + z4785 = x4785 + z4786 = x4786 + z4787 = x4787 + z4788 = x4788 + z4789 = x4789 + z4790 = x4790 + z4791 = x4791 + z4792 = x4792 + z4793 = x4793 + z4794 = x4794 + z4795 = x4795 + z4796 = x4796 + z4797 = x4797 + z4798 = x4798 + z4799 = x4799 + z4800 = x4800 + z4801 = x4801 + z4802 = x4802 + z4803 = x4803 + z4804 = x4804 + z4805 = x4805 + z4806 = x4806 + z4807 = x4807 + z4808 = x4808 + z4809 = x4809 + z4810 = x4810 + z4811 = x4811 + z4812 = x4812 + z4813 = x4813 + z4814 = x4814 + z4815 = x4815 + z4816 = x4816 + z4817 = x4817 + z4818 = x4818 + z4819 = x4819 + z4820 = x4820 + z4821 = x4821 + z4822 = x4822 + z4823 = x4823 + z4824 = x4824 + z4825 = x4825 + z4826 = x4826 + z4827 = x4827 + z4828 = x4828 + z4829 = x4829 + z4830 = x4830 + z4831 = x4831 + z4832 = x4832 + z4833 = x4833 + z4834 = x4834 + z4835 = x4835 + z4836 = x4836 + z4837 = x4837 + z4838 = x4838 + z4839 = x4839 + z4840 = x4840 + z4841 = x4841 + z4842 = x4842 + z4843 = x4843 + z4844 = x4844 + z4845 = x4845 + z4846 = x4846 + z4847 = x4847 + z4848 = x4848 + z4849 = x4849 + z4850 = x4850 + z4851 = x4851 + z4852 = x4852 + z4853 = x4853 + z4854 = x4854 + z4855 = x4855 + z4856 = x4856 + z4857 = x4857 + z4858 = x4858 + z4859 = x4859 + z4860 = x4860 + z4861 = x4861 + z4862 = x4862 + z4863 = x4863 + z4864 = x4864 + z4865 = x4865 + z4866 = x4866 + z4867 = x4867 + z4868 = x4868 + z4869 = x4869 + z4870 = x4870 + z4871 = x4871 + z4872 = x4872 + z4873 = x4873 + z4874 = x4874 + z4875 = x4875 + z4876 = x4876 + z4877 = x4877 + z4878 = x4878 + z4879 = x4879 + z4880 = x4880 + z4881 = x4881 + z4882 = x4882 + z4883 = x4883 + z4884 = x4884 + z4885 = x4885 + z4886 = x4886 + z4887 = x4887 + z4888 = x4888 + z4889 = x4889 + z4890 = x4890 + z4891 = x4891 + z4892 = x4892 + z4893 = x4893 + z4894 = x4894 + z4895 = x4895 + z4896 = x4896 + z4897 = x4897 + z4898 = x4898 + z4899 = x4899 + z4900 = x4900 + z4901 = x4901 + z4902 = x4902 + z4903 = x4903 + z4904 = x4904 + z4905 = x4905 + z4906 = x4906 + z4907 = x4907 + z4908 = x4908 + z4909 = x4909 + z4910 = x4910 + z4911 = x4911 + z4912 = x4912 + z4913 = x4913 + z4914 = x4914 + z4915 = x4915 + z4916 = x4916 + z4917 = x4917 + z4918 = x4918 + z4919 = x4919 + z4920 = x4920 + z4921 = x4921 + z4922 = x4922 + z4923 = x4923 + z4924 = x4924 + z4925 = x4925 + z4926 = x4926 + z4927 = x4927 + z4928 = x4928 + z4929 = x4929 + z4930 = x4930 + z4931 = x4931 + z4932 = x4932 + z4933 = x4933 + z4934 = x4934 + z4935 = x4935 + z4936 = x4936 + z4937 = x4937 + z4938 = x4938 + z4939 = x4939 + z4940 = x4940 + z4941 = x4941 + z4942 = x4942 + z4943 = x4943 + z4944 = x4944 + z4945 = x4945 + z4946 = x4946 + z4947 = x4947 + z4948 = x4948 + z4949 = x4949 + z4950 = x4950 + z4951 = x4951 + z4952 = x4952 + z4953 = x4953 + z4954 = x4954 + z4955 = x4955 + z4956 = x4956 + z4957 = x4957 + z4958 = x4958 + z4959 = x4959 + z4960 = x4960 + z4961 = x4961 + z4962 = x4962 + z4963 = x4963 + z4964 = x4964 + z4965 = x4965 + z4966 = x4966 + z4967 = x4967 + z4968 = x4968 + z4969 = x4969 + z4970 = x4970 + z4971 = x4971 + z4972 = x4972 + z4973 = x4973 + z4974 = x4974 + z4975 = x4975 + z4976 = x4976 + z4977 = x4977 + z4978 = x4978 + z4979 = x4979 + z4980 = x4980 + z4981 = x4981 + z4982 = x4982 + z4983 = x4983 + z4984 = x4984 + z4985 = x4985 + z4986 = x4986 + z4987 = x4987 + z4988 = x4988 + z4989 = x4989 + z4990 = x4990 + z4991 = x4991 + z4992 = x4992 + z4993 = x4993 + z4994 = x4994 + z4995 = x4995 + z4996 = x4996 + z4997 = x4997 + z4998 = x4998 + z4999 = x4999 + z5000 = x5000 + z5001 = x5001 + z5002 = x5002 + z5003 = x5003 + z5004 = x5004 + z5005 = x5005 + z5006 = x5006 + z5007 = x5007 + z5008 = x5008 + z5009 = x5009 + z5010 = x5010 + z5011 = x5011 + z5012 = x5012 + z5013 = x5013 + z5014 = x5014 + z5015 = x5015 + z5016 = x5016 + z5017 = x5017 + z5018 = x5018 + z5019 = x5019 + z5020 = x5020 + z5021 = x5021 + z5022 = x5022 + z5023 = x5023 + z5024 = x5024 + z5025 = x5025 + z5026 = x5026 + z5027 = x5027 + z5028 = x5028 + z5029 = x5029 + z5030 = x5030 + z5031 = x5031 + z5032 = x5032 + z5033 = x5033 + z5034 = x5034 + z5035 = x5035 + z5036 = x5036 + z5037 = x5037 + z5038 = x5038 + z5039 = x5039 + z5040 = x5040 + z5041 = x5041 + z5042 = x5042 + z5043 = x5043 + z5044 = x5044 + z5045 = x5045 + z5046 = x5046 + z5047 = x5047 + z5048 = x5048 + z5049 = x5049 + z5050 = x5050 + z5051 = x5051 + z5052 = x5052 + z5053 = x5053 + z5054 = x5054 + z5055 = x5055 + z5056 = x5056 + z5057 = x5057 + z5058 = x5058 + z5059 = x5059 + z5060 = x5060 + z5061 = x5061 + z5062 = x5062 + z5063 = x5063 + z5064 = x5064 + z5065 = x5065 + z5066 = x5066 + z5067 = x5067 + z5068 = x5068 + z5069 = x5069 + z5070 = x5070 + z5071 = x5071 + z5072 = x5072 + z5073 = x5073 + z5074 = x5074 + z5075 = x5075 + z5076 = x5076 + z5077 = x5077 + z5078 = x5078 + z5079 = x5079 + z5080 = x5080 + z5081 = x5081 + z5082 = x5082 + z5083 = x5083 + z5084 = x5084 + z5085 = x5085 + z5086 = x5086 + z5087 = x5087 + z5088 = x5088 + z5089 = x5089 + z5090 = x5090 + z5091 = x5091 + z5092 = x5092 + z5093 = x5093 + z5094 = x5094 + z5095 = x5095 + z5096 = x5096 + z5097 = x5097 + z5098 = x5098 + z5099 = x5099 + z5100 = x5100 + z5101 = x5101 + z5102 = x5102 + z5103 = x5103 + z5104 = x5104 + z5105 = x5105 + z5106 = x5106 + z5107 = x5107 + z5108 = x5108 + z5109 = x5109 + z5110 = x5110 + z5111 = x5111 + z5112 = x5112 + z5113 = x5113 + z5114 = x5114 + z5115 = x5115 + z5116 = x5116 + z5117 = x5117 + z5118 = x5118 + z5119 = x5119 + z5120 = x5120 + z5121 = x5121 + z5122 = x5122 + z5123 = x5123 + z5124 = x5124 + z5125 = x5125 + z5126 = x5126 + z5127 = x5127 + z5128 = x5128 + z5129 = x5129 + z5130 = x5130 + z5131 = x5131 + z5132 = x5132 + z5133 = x5133 + z5134 = x5134 + z5135 = x5135 + z5136 = x5136 + z5137 = x5137 + z5138 = x5138 + z5139 = x5139 + z5140 = x5140 + z5141 = x5141 + z5142 = x5142 + z5143 = x5143 + z5144 = x5144 + z5145 = x5145 + z5146 = x5146 + z5147 = x5147 + z5148 = x5148 + z5149 = x5149 + z5150 = x5150 + z5151 = x5151 + z5152 = x5152 + z5153 = x5153 + z5154 = x5154 + z5155 = x5155 + z5156 = x5156 + z5157 = x5157 + z5158 = x5158 + z5159 = x5159 + z5160 = x5160 + z5161 = x5161 + z5162 = x5162 + z5163 = x5163 + z5164 = x5164 + z5165 = x5165 + z5166 = x5166 + z5167 = x5167 + z5168 = x5168 + z5169 = x5169 + z5170 = x5170 + z5171 = x5171 + z5172 = x5172 + z5173 = x5173 + z5174 = x5174 + z5175 = x5175 + z5176 = x5176 + z5177 = x5177 + z5178 = x5178 + z5179 = x5179 + z5180 = x5180 + z5181 = x5181 + z5182 = x5182 + z5183 = x5183 + z5184 = x5184 + z5185 = x5185 + z5186 = x5186 + z5187 = x5187 + z5188 = x5188 + z5189 = x5189 + z5190 = x5190 + z5191 = x5191 + z5192 = x5192 + z5193 = x5193 + z5194 = x5194 + z5195 = x5195 + z5196 = x5196 + z5197 = x5197 + z5198 = x5198 + z5199 = x5199 + z5200 = x5200 + z5201 = x5201 + z5202 = x5202 + z5203 = x5203 + z5204 = x5204 + z5205 = x5205 + z5206 = x5206 + z5207 = x5207 + z5208 = x5208 + z5209 = x5209 + z5210 = x5210 + z5211 = x5211 + z5212 = x5212 + z5213 = x5213 + z5214 = x5214 + z5215 = x5215 + z5216 = x5216 + z5217 = x5217 + z5218 = x5218 + z5219 = x5219 + z5220 = x5220 + z5221 = x5221 + z5222 = x5222 + z5223 = x5223 + z5224 = x5224 + z5225 = x5225 + z5226 = x5226 + z5227 = x5227 + z5228 = x5228 + z5229 = x5229 + z5230 = x5230 + z5231 = x5231 + z5232 = x5232 + z5233 = x5233 + z5234 = x5234 + z5235 = x5235 + z5236 = x5236 + z5237 = x5237 + z5238 = x5238 + z5239 = x5239 + z5240 = x5240 + z5241 = x5241 + z5242 = x5242 + z5243 = x5243 + z5244 = x5244 + z5245 = x5245 + z5246 = x5246 + z5247 = x5247 + z5248 = x5248 + z5249 = x5249 + z5250 = x5250 + z5251 = x5251 + z5252 = x5252 + z5253 = x5253 + z5254 = x5254 + z5255 = x5255 + z5256 = x5256 + z5257 = x5257 + z5258 = x5258 + z5259 = x5259 + z5260 = x5260 + z5261 = x5261 + z5262 = x5262 + z5263 = x5263 + z5264 = x5264 + z5265 = x5265 + z5266 = x5266 + z5267 = x5267 + z5268 = x5268 + z5269 = x5269 + z5270 = x5270 + z5271 = x5271 + z5272 = x5272 + z5273 = x5273 + z5274 = x5274 + z5275 = x5275 + z5276 = x5276 + z5277 = x5277 + z5278 = x5278 + z5279 = x5279 + z5280 = x5280 + z5281 = x5281 + z5282 = x5282 + z5283 = x5283 + z5284 = x5284 + z5285 = x5285 + z5286 = x5286 + z5287 = x5287 + z5288 = x5288 + z5289 = x5289 + z5290 = x5290 + z5291 = x5291 + z5292 = x5292 + z5293 = x5293 + z5294 = x5294 + z5295 = x5295 + z5296 = x5296 + z5297 = x5297 + z5298 = x5298 + z5299 = x5299 + z5300 = x5300 + z5301 = x5301 + z5302 = x5302 + z5303 = x5303 + z5304 = x5304 + z5305 = x5305 + z5306 = x5306 + z5307 = x5307 + z5308 = x5308 + z5309 = x5309 + z5310 = x5310 + z5311 = x5311 + z5312 = x5312 + z5313 = x5313 + z5314 = x5314 + z5315 = x5315 + z5316 = x5316 + z5317 = x5317 + z5318 = x5318 + z5319 = x5319 + z5320 = x5320 + z5321 = x5321 + z5322 = x5322 + z5323 = x5323 + z5324 = x5324 + z5325 = x5325 + z5326 = x5326 + z5327 = x5327 + z5328 = x5328 + z5329 = x5329 + z5330 = x5330 + z5331 = x5331 + z5332 = x5332 + z5333 = x5333 + z5334 = x5334 + z5335 = x5335 + z5336 = x5336 + z5337 = x5337 + z5338 = x5338 + z5339 = x5339 + z5340 = x5340 + z5341 = x5341 + z5342 = x5342 + z5343 = x5343 + z5344 = x5344 + z5345 = x5345 + z5346 = x5346 + z5347 = x5347 + z5348 = x5348 + z5349 = x5349 + z5350 = x5350 + z5351 = x5351 + z5352 = x5352 + z5353 = x5353 + z5354 = x5354 + z5355 = x5355 + z5356 = x5356 + z5357 = x5357 + z5358 = x5358 + z5359 = x5359 + z5360 = x5360 + z5361 = x5361 + z5362 = x5362 + z5363 = x5363 + z5364 = x5364 + z5365 = x5365 + z5366 = x5366 + z5367 = x5367 + z5368 = x5368 + z5369 = x5369 + z5370 = x5370 + z5371 = x5371 + z5372 = x5372 + z5373 = x5373 + z5374 = x5374 + z5375 = x5375 + z5376 = x5376 + z5377 = x5377 + z5378 = x5378 + z5379 = x5379 + z5380 = x5380 + z5381 = x5381 + z5382 = x5382 + z5383 = x5383 + z5384 = x5384 + z5385 = x5385 + z5386 = x5386 + z5387 = x5387 + z5388 = x5388 + z5389 = x5389 + z5390 = x5390 + z5391 = x5391 + z5392 = x5392 + z5393 = x5393 + z5394 = x5394 + z5395 = x5395 + z5396 = x5396 + z5397 = x5397 + z5398 = x5398 + z5399 = x5399 + z5400 = x5400 + z5401 = x5401 + z5402 = x5402 + z5403 = x5403 + z5404 = x5404 + z5405 = x5405 + z5406 = x5406 + z5407 = x5407 + z5408 = x5408 + z5409 = x5409 + z5410 = x5410 + z5411 = x5411 + z5412 = x5412 + z5413 = x5413 + z5414 = x5414 + z5415 = x5415 + z5416 = x5416 + z5417 = x5417 + z5418 = x5418 + z5419 = x5419 + z5420 = x5420 + z5421 = x5421 + z5422 = x5422 + z5423 = x5423 + z5424 = x5424 + z5425 = x5425 + z5426 = x5426 + z5427 = x5427 + z5428 = x5428 + z5429 = x5429 + z5430 = x5430 + z5431 = x5431 + z5432 = x5432 + z5433 = x5433 + z5434 = x5434 + z5435 = x5435 + z5436 = x5436 + z5437 = x5437 + z5438 = x5438 + z5439 = x5439 + z5440 = x5440 + z5441 = x5441 + z5442 = x5442 + z5443 = x5443 + z5444 = x5444 + z5445 = x5445 + z5446 = x5446 + z5447 = x5447 + z5448 = x5448 + z5449 = x5449 + z5450 = x5450 + z5451 = x5451 + z5452 = x5452 + z5453 = x5453 + z5454 = x5454 + z5455 = x5455 + z5456 = x5456 + z5457 = x5457 + z5458 = x5458 + z5459 = x5459 + z5460 = x5460 + z5461 = x5461 + z5462 = x5462 + z5463 = x5463 + z5464 = x5464 + z5465 = x5465 + z5466 = x5466 + z5467 = x5467 + z5468 = x5468 + z5469 = x5469 + z5470 = x5470 + z5471 = x5471 + z5472 = x5472 + z5473 = x5473 + z5474 = x5474 + z5475 = x5475 + z5476 = x5476 + z5477 = x5477 + z5478 = x5478 + z5479 = x5479 + z5480 = x5480 + z5481 = x5481 + z5482 = x5482 + z5483 = x5483 + z5484 = x5484 + z5485 = x5485 + z5486 = x5486 + z5487 = x5487 + z5488 = x5488 + z5489 = x5489 + z5490 = x5490 + z5491 = x5491 + z5492 = x5492 + z5493 = x5493 + z5494 = x5494 + z5495 = x5495 + z5496 = x5496 + z5497 = x5497 + z5498 = x5498 + z5499 = x5499 + z5500 = x5500 + z5501 = x5501 + z5502 = x5502 + z5503 = x5503 + z5504 = x5504 + z5505 = x5505 + z5506 = x5506 + z5507 = x5507 + z5508 = x5508 + z5509 = x5509 + z5510 = x5510 + z5511 = x5511 + z5512 = x5512 + z5513 = x5513 + z5514 = x5514 + z5515 = x5515 + z5516 = x5516 + z5517 = x5517 + z5518 = x5518 + z5519 = x5519 + z5520 = x5520 + z5521 = x5521 + z5522 = x5522 + z5523 = x5523 + z5524 = x5524 + z5525 = x5525 + z5526 = x5526 + z5527 = x5527 + z5528 = x5528 + z5529 = x5529 + z5530 = x5530 + z5531 = x5531 + z5532 = x5532 + z5533 = x5533 + z5534 = x5534 + z5535 = x5535 + z5536 = x5536 + z5537 = x5537 + z5538 = x5538 + z5539 = x5539 + z5540 = x5540 + z5541 = x5541 + z5542 = x5542 + z5543 = x5543 + z5544 = x5544 + z5545 = x5545 + z5546 = x5546 + z5547 = x5547 + z5548 = x5548 + z5549 = x5549 + z5550 = x5550 + z5551 = x5551 + z5552 = x5552 + z5553 = x5553 + z5554 = x5554 + z5555 = x5555 + z5556 = x5556 + z5557 = x5557 + z5558 = x5558 + z5559 = x5559 + z5560 = x5560 + z5561 = x5561 + z5562 = x5562 + z5563 = x5563 + z5564 = x5564 + z5565 = x5565 + z5566 = x5566 + z5567 = x5567 + z5568 = x5568 + z5569 = x5569 + z5570 = x5570 + z5571 = x5571 + z5572 = x5572 + z5573 = x5573 + z5574 = x5574 + z5575 = x5575 + z5576 = x5576 + z5577 = x5577 + z5578 = x5578 + z5579 = x5579 + z5580 = x5580 + z5581 = x5581 + z5582 = x5582 + z5583 = x5583 + z5584 = x5584 + z5585 = x5585 + z5586 = x5586 + z5587 = x5587 + z5588 = x5588 + z5589 = x5589 + z5590 = x5590 + z5591 = x5591 + z5592 = x5592 + z5593 = x5593 + z5594 = x5594 + z5595 = x5595 + z5596 = x5596 + z5597 = x5597 + z5598 = x5598 + z5599 = x5599 + z5600 = x5600 + z5601 = x5601 + z5602 = x5602 + z5603 = x5603 + z5604 = x5604 + z5605 = x5605 + z5606 = x5606 + z5607 = x5607 + z5608 = x5608 + z5609 = x5609 + z5610 = x5610 + z5611 = x5611 + z5612 = x5612 + z5613 = x5613 + z5614 = x5614 + z5615 = x5615 + z5616 = x5616 + z5617 = x5617 + z5618 = x5618 + z5619 = x5619 + z5620 = x5620 + z5621 = x5621 + z5622 = x5622 + z5623 = x5623 + z5624 = x5624 + z5625 = x5625 + z5626 = x5626 + z5627 = x5627 + z5628 = x5628 + z5629 = x5629 + z5630 = x5630 + z5631 = x5631 + z5632 = x5632 + z5633 = x5633 + z5634 = x5634 + z5635 = x5635 + z5636 = x5636 + z5637 = x5637 + z5638 = x5638 + z5639 = x5639 + z5640 = x5640 + z5641 = x5641 + z5642 = x5642 + z5643 = x5643 + z5644 = x5644 + z5645 = x5645 + z5646 = x5646 + z5647 = x5647 + z5648 = x5648 + z5649 = x5649 + z5650 = x5650 + z5651 = x5651 + z5652 = x5652 + z5653 = x5653 + z5654 = x5654 + z5655 = x5655 + z5656 = x5656 + z5657 = x5657 + z5658 = x5658 + z5659 = x5659 + z5660 = x5660 + z5661 = x5661 + z5662 = x5662 + z5663 = x5663 + z5664 = x5664 + z5665 = x5665 + z5666 = x5666 + z5667 = x5667 + z5668 = x5668 + z5669 = x5669 + z5670 = x5670 + z5671 = x5671 + z5672 = x5672 + z5673 = x5673 + z5674 = x5674 + z5675 = x5675 + z5676 = x5676 + z5677 = x5677 + z5678 = x5678 + z5679 = x5679 + z5680 = x5680 + z5681 = x5681 + z5682 = x5682 + z5683 = x5683 + z5684 = x5684 + z5685 = x5685 + z5686 = x5686 + z5687 = x5687 + z5688 = x5688 + z5689 = x5689 + z5690 = x5690 + z5691 = x5691 + z5692 = x5692 + z5693 = x5693 + z5694 = x5694 + z5695 = x5695 + z5696 = x5696 + z5697 = x5697 + z5698 = x5698 + z5699 = x5699 + z5700 = x5700 + z5701 = x5701 + z5702 = x5702 + z5703 = x5703 + z5704 = x5704 + z5705 = x5705 + z5706 = x5706 + z5707 = x5707 + z5708 = x5708 + z5709 = x5709 + z5710 = x5710 + z5711 = x5711 + z5712 = x5712 + z5713 = x5713 + z5714 = x5714 + z5715 = x5715 + z5716 = x5716 + z5717 = x5717 + z5718 = x5718 + z5719 = x5719 + z5720 = x5720 + z5721 = x5721 + z5722 = x5722 + z5723 = x5723 + z5724 = x5724 + z5725 = x5725 + z5726 = x5726 + z5727 = x5727 + z5728 = x5728 + z5729 = x5729 + z5730 = x5730 + z5731 = x5731 + z5732 = x5732 + z5733 = x5733 + z5734 = x5734 + z5735 = x5735 + z5736 = x5736 + z5737 = x5737 + z5738 = x5738 + z5739 = x5739 + z5740 = x5740 + z5741 = x5741 + z5742 = x5742 + z5743 = x5743 + z5744 = x5744 + z5745 = x5745 + z5746 = x5746 + z5747 = x5747 + z5748 = x5748 + z5749 = x5749 + z5750 = x5750 + z5751 = x5751 + z5752 = x5752 + z5753 = x5753 + z5754 = x5754 + z5755 = x5755 + z5756 = x5756 + z5757 = x5757 + z5758 = x5758 + z5759 = x5759 + z5760 = x5760 + z5761 = x5761 + z5762 = x5762 + z5763 = x5763 + z5764 = x5764 + z5765 = x5765 + z5766 = x5766 + z5767 = x5767 + z5768 = x5768 + z5769 = x5769 + z5770 = x5770 + z5771 = x5771 + z5772 = x5772 + z5773 = x5773 + z5774 = x5774 + z5775 = x5775 + z5776 = x5776 + z5777 = x5777 + z5778 = x5778 + z5779 = x5779 + z5780 = x5780 + z5781 = x5781 + z5782 = x5782 + z5783 = x5783 + z5784 = x5784 + z5785 = x5785 + z5786 = x5786 + z5787 = x5787 + z5788 = x5788 + z5789 = x5789 + z5790 = x5790 + z5791 = x5791 + z5792 = x5792 + z5793 = x5793 + z5794 = x5794 + z5795 = x5795 + z5796 = x5796 + z5797 = x5797 + z5798 = x5798 + z5799 = x5799 + z5800 = x5800 + z5801 = x5801 + z5802 = x5802 + z5803 = x5803 + z5804 = x5804 + z5805 = x5805 + z5806 = x5806 + z5807 = x5807 + z5808 = x5808 + z5809 = x5809 + z5810 = x5810 + z5811 = x5811 + z5812 = x5812 + z5813 = x5813 + z5814 = x5814 + z5815 = x5815 + z5816 = x5816 + z5817 = x5817 + z5818 = x5818 + z5819 = x5819 + z5820 = x5820 + z5821 = x5821 + z5822 = x5822 + z5823 = x5823 + z5824 = x5824 + z5825 = x5825 + z5826 = x5826 + z5827 = x5827 + z5828 = x5828 + z5829 = x5829 + z5830 = x5830 + z5831 = x5831 + z5832 = x5832 + z5833 = x5833 + z5834 = x5834 + z5835 = x5835 + z5836 = x5836 + z5837 = x5837 + z5838 = x5838 + z5839 = x5839 + z5840 = x5840 + z5841 = x5841 + z5842 = x5842 + z5843 = x5843 + z5844 = x5844 + z5845 = x5845 + z5846 = x5846 + z5847 = x5847 + z5848 = x5848 + z5849 = x5849 + z5850 = x5850 + z5851 = x5851 + z5852 = x5852 + z5853 = x5853 + z5854 = x5854 + z5855 = x5855 + z5856 = x5856 + z5857 = x5857 + z5858 = x5858 + z5859 = x5859 + z5860 = x5860 + z5861 = x5861 + z5862 = x5862 + z5863 = x5863 + z5864 = x5864 + z5865 = x5865 + z5866 = x5866 + z5867 = x5867 + z5868 = x5868 + z5869 = x5869 + z5870 = x5870 + z5871 = x5871 + z5872 = x5872 + z5873 = x5873 + z5874 = x5874 + z5875 = x5875 + z5876 = x5876 + z5877 = x5877 + z5878 = x5878 + z5879 = x5879 + z5880 = x5880 + z5881 = x5881 + z5882 = x5882 + z5883 = x5883 + z5884 = x5884 + z5885 = x5885 + z5886 = x5886 + z5887 = x5887 + z5888 = x5888 + z5889 = x5889 + z5890 = x5890 + z5891 = x5891 + z5892 = x5892 + z5893 = x5893 + z5894 = x5894 + z5895 = x5895 + z5896 = x5896 + z5897 = x5897 + z5898 = x5898 + z5899 = x5899 + z5900 = x5900 + z5901 = x5901 + z5902 = x5902 + z5903 = x5903 + z5904 = x5904 + z5905 = x5905 + z5906 = x5906 + z5907 = x5907 + z5908 = x5908 + z5909 = x5909 + z5910 = x5910 + z5911 = x5911 + z5912 = x5912 + z5913 = x5913 + z5914 = x5914 + z5915 = x5915 + z5916 = x5916 + z5917 = x5917 + z5918 = x5918 + z5919 = x5919 + z5920 = x5920 + z5921 = x5921 + z5922 = x5922 + z5923 = x5923 + z5924 = x5924 + z5925 = x5925 + z5926 = x5926 + z5927 = x5927 + z5928 = x5928 + z5929 = x5929 + z5930 = x5930 + z5931 = x5931 + z5932 = x5932 + z5933 = x5933 + z5934 = x5934 + z5935 = x5935 + z5936 = x5936 + z5937 = x5937 + z5938 = x5938 + z5939 = x5939 + z5940 = x5940 + z5941 = x5941 + z5942 = x5942 + z5943 = x5943 + z5944 = x5944 + z5945 = x5945 + z5946 = x5946 + z5947 = x5947 + z5948 = x5948 + z5949 = x5949 + z5950 = x5950 + z5951 = x5951 + z5952 = x5952 + z5953 = x5953 + z5954 = x5954 + z5955 = x5955 + z5956 = x5956 + z5957 = x5957 + z5958 = x5958 + z5959 = x5959 + z5960 = x5960 + z5961 = x5961 + z5962 = x5962 + z5963 = x5963 + z5964 = x5964 + z5965 = x5965 + z5966 = x5966 + z5967 = x5967 + z5968 = x5968 + z5969 = x5969 + z5970 = x5970 + z5971 = x5971 + z5972 = x5972 + z5973 = x5973 + z5974 = x5974 + z5975 = x5975 + z5976 = x5976 + z5977 = x5977 + z5978 = x5978 + z5979 = x5979 + z5980 = x5980 + z5981 = x5981 + z5982 = x5982 + z5983 = x5983 + z5984 = x5984 + z5985 = x5985 + z5986 = x5986 + z5987 = x5987 + z5988 = x5988 + z5989 = x5989 + z5990 = x5990 + z5991 = x5991 + z5992 = x5992 + z5993 = x5993 + z5994 = x5994 + z5995 = x5995 + z5996 = x5996 + z5997 = x5997 + z5998 = x5998 + z5999 = x5999 + z6000 = x6000 + z6001 = x6001 + z6002 = x6002 + z6003 = x6003 + z6004 = x6004 + z6005 = x6005 + z6006 = x6006 + z6007 = x6007 + z6008 = x6008 + z6009 = x6009 + z6010 = x6010 + z6011 = x6011 + z6012 = x6012 + z6013 = x6013 + z6014 = x6014 + z6015 = x6015 + z6016 = x6016 + z6017 = x6017 + z6018 = x6018 + z6019 = x6019 + z6020 = x6020 + z6021 = x6021 + z6022 = x6022 + z6023 = x6023 + z6024 = x6024 + z6025 = x6025 + z6026 = x6026 + z6027 = x6027 + z6028 = x6028 + z6029 = x6029 + z6030 = x6030 + z6031 = x6031 + z6032 = x6032 + z6033 = x6033 + z6034 = x6034 + z6035 = x6035 + z6036 = x6036 + z6037 = x6037 + z6038 = x6038 + z6039 = x6039 + z6040 = x6040 + z6041 = x6041 + z6042 = x6042 + z6043 = x6043 + z6044 = x6044 + z6045 = x6045 + z6046 = x6046 + z6047 = x6047 + z6048 = x6048 + z6049 = x6049 + z6050 = x6050 + z6051 = x6051 + z6052 = x6052 + z6053 = x6053 + z6054 = x6054 + z6055 = x6055 + z6056 = x6056 + z6057 = x6057 + z6058 = x6058 + z6059 = x6059 + z6060 = x6060 + z6061 = x6061 + z6062 = x6062 + z6063 = x6063 + z6064 = x6064 + z6065 = x6065 + z6066 = x6066 + z6067 = x6067 + z6068 = x6068 + z6069 = x6069 + z6070 = x6070 + z6071 = x6071 + z6072 = x6072 + z6073 = x6073 + z6074 = x6074 + z6075 = x6075 + z6076 = x6076 + z6077 = x6077 + z6078 = x6078 + z6079 = x6079 + z6080 = x6080 + z6081 = x6081 + z6082 = x6082 + z6083 = x6083 + z6084 = x6084 + z6085 = x6085 + z6086 = x6086 + z6087 = x6087 + z6088 = x6088 + z6089 = x6089 + z6090 = x6090 + z6091 = x6091 + z6092 = x6092 + z6093 = x6093 + z6094 = x6094 + z6095 = x6095 + z6096 = x6096 + z6097 = x6097 + z6098 = x6098 + z6099 = x6099 + z6100 = x6100 + z6101 = x6101 + z6102 = x6102 + z6103 = x6103 + z6104 = x6104 + z6105 = x6105 + z6106 = x6106 + z6107 = x6107 + z6108 = x6108 + z6109 = x6109 + z6110 = x6110 + z6111 = x6111 + z6112 = x6112 + z6113 = x6113 + z6114 = x6114 + z6115 = x6115 + z6116 = x6116 + z6117 = x6117 + z6118 = x6118 + z6119 = x6119 + z6120 = x6120 + z6121 = x6121 + z6122 = x6122 + z6123 = x6123 + z6124 = x6124 + z6125 = x6125 + z6126 = x6126 + z6127 = x6127 + z6128 = x6128 + z6129 = x6129 + z6130 = x6130 + z6131 = x6131 + z6132 = x6132 + z6133 = x6133 + z6134 = x6134 + z6135 = x6135 + z6136 = x6136 + z6137 = x6137 + z6138 = x6138 + z6139 = x6139 + z6140 = x6140 + z6141 = x6141 + z6142 = x6142 + z6143 = x6143 + z6144 = x6144 + z6145 = x6145 + z6146 = x6146 + z6147 = x6147 + z6148 = x6148 + z6149 = x6149 + z6150 = x6150 + z6151 = x6151 + z6152 = x6152 + z6153 = x6153 + z6154 = x6154 + z6155 = x6155 + z6156 = x6156 + z6157 = x6157 + z6158 = x6158 + z6159 = x6159 + z6160 = x6160 + z6161 = x6161 + z6162 = x6162 + z6163 = x6163 + z6164 = x6164 + z6165 = x6165 + z6166 = x6166 + z6167 = x6167 + z6168 = x6168 + z6169 = x6169 + z6170 = x6170 + z6171 = x6171 + z6172 = x6172 + z6173 = x6173 + z6174 = x6174 + z6175 = x6175 + z6176 = x6176 + z6177 = x6177 + z6178 = x6178 + z6179 = x6179 + z6180 = x6180 + z6181 = x6181 + z6182 = x6182 + z6183 = x6183 + z6184 = x6184 + z6185 = x6185 + z6186 = x6186 + z6187 = x6187 + z6188 = x6188 + z6189 = x6189 + z6190 = x6190 + z6191 = x6191 + z6192 = x6192 + z6193 = x6193 + z6194 = x6194 + z6195 = x6195 + z6196 = x6196 + z6197 = x6197 + z6198 = x6198 + z6199 = x6199 + z6200 = x6200 + z6201 = x6201 + z6202 = x6202 + z6203 = x6203 + z6204 = x6204 + z6205 = x6205 + z6206 = x6206 + z6207 = x6207 + z6208 = x6208 + z6209 = x6209 + z6210 = x6210 + z6211 = x6211 + z6212 = x6212 + z6213 = x6213 + z6214 = x6214 + z6215 = x6215 + z6216 = x6216 + z6217 = x6217 + z6218 = x6218 + z6219 = x6219 + z6220 = x6220 + z6221 = x6221 + z6222 = x6222 + z6223 = x6223 + z6224 = x6224 + z6225 = x6225 + z6226 = x6226 + z6227 = x6227 + z6228 = x6228 + z6229 = x6229 + z6230 = x6230 + z6231 = x6231 + z6232 = x6232 + z6233 = x6233 + z6234 = x6234 + z6235 = x6235 + z6236 = x6236 + z6237 = x6237 + z6238 = x6238 + z6239 = x6239 + z6240 = x6240 + z6241 = x6241 + z6242 = x6242 + z6243 = x6243 + z6244 = x6244 + z6245 = x6245 + z6246 = x6246 + z6247 = x6247 + z6248 = x6248 + z6249 = x6249 + z6250 = x6250 + z6251 = x6251 + z6252 = x6252 + z6253 = x6253 + z6254 = x6254 + z6255 = x6255 + z6256 = x6256 + z6257 = x6257 + z6258 = x6258 + z6259 = x6259 + z6260 = x6260 + z6261 = x6261 + z6262 = x6262 + z6263 = x6263 + z6264 = x6264 + z6265 = x6265 + z6266 = x6266 + z6267 = x6267 + z6268 = x6268 + z6269 = x6269 + z6270 = x6270 + z6271 = x6271 + z6272 = x6272 + z6273 = x6273 + z6274 = x6274 + z6275 = x6275 + z6276 = x6276 + z6277 = x6277 + z6278 = x6278 + z6279 = x6279 + z6280 = x6280 + z6281 = x6281 + z6282 = x6282 + z6283 = x6283 + z6284 = x6284 + z6285 = x6285 + z6286 = x6286 + z6287 = x6287 + z6288 = x6288 + z6289 = x6289 + z6290 = x6290 + z6291 = x6291 + z6292 = x6292 + z6293 = x6293 + z6294 = x6294 + z6295 = x6295 + z6296 = x6296 + z6297 = x6297 + z6298 = x6298 + z6299 = x6299 + z6300 = x6300 + z6301 = x6301 + z6302 = x6302 + z6303 = x6303 + z6304 = x6304 + z6305 = x6305 + z6306 = x6306 + z6307 = x6307 + z6308 = x6308 + z6309 = x6309 + z6310 = x6310 + z6311 = x6311 + z6312 = x6312 + z6313 = x6313 + z6314 = x6314 + z6315 = x6315 + z6316 = x6316 + z6317 = x6317 + z6318 = x6318 + z6319 = x6319 + z6320 = x6320 + z6321 = x6321 + z6322 = x6322 + z6323 = x6323 + z6324 = x6324 + z6325 = x6325 + z6326 = x6326 + z6327 = x6327 + z6328 = x6328 + z6329 = x6329 + z6330 = x6330 + z6331 = x6331 + z6332 = x6332 + z6333 = x6333 + z6334 = x6334 + z6335 = x6335 + z6336 = x6336 + z6337 = x6337 + z6338 = x6338 + z6339 = x6339 + z6340 = x6340 + z6341 = x6341 + z6342 = x6342 + z6343 = x6343 + z6344 = x6344 + z6345 = x6345 + z6346 = x6346 + z6347 = x6347 + z6348 = x6348 + z6349 = x6349 + z6350 = x6350 + z6351 = x6351 + z6352 = x6352 + z6353 = x6353 + z6354 = x6354 + z6355 = x6355 + z6356 = x6356 + z6357 = x6357 + z6358 = x6358 + z6359 = x6359 + z6360 = x6360 + z6361 = x6361 + z6362 = x6362 + z6363 = x6363 + z6364 = x6364 + z6365 = x6365 + z6366 = x6366 + z6367 = x6367 + z6368 = x6368 + z6369 = x6369 + z6370 = x6370 + z6371 = x6371 + z6372 = x6372 + z6373 = x6373 + z6374 = x6374 + z6375 = x6375 + z6376 = x6376 + z6377 = x6377 + z6378 = x6378 + z6379 = x6379 + z6380 = x6380 + z6381 = x6381 + z6382 = x6382 + z6383 = x6383 + z6384 = x6384 + z6385 = x6385 + z6386 = x6386 + z6387 = x6387 + z6388 = x6388 + z6389 = x6389 + z6390 = x6390 + z6391 = x6391 + z6392 = x6392 + z6393 = x6393 + z6394 = x6394 + z6395 = x6395 + z6396 = x6396 + z6397 = x6397 + z6398 = x6398 + z6399 = x6399 + z6400 = x6400 + z6401 = x6401 + z6402 = x6402 + z6403 = x6403 + z6404 = x6404 + z6405 = x6405 + z6406 = x6406 + z6407 = x6407 + z6408 = x6408 + z6409 = x6409 + z6410 = x6410 + z6411 = x6411 + z6412 = x6412 + z6413 = x6413 + z6414 = x6414 + z6415 = x6415 + z6416 = x6416 + z6417 = x6417 + z6418 = x6418 + z6419 = x6419 + z6420 = x6420 + z6421 = x6421 + z6422 = x6422 + z6423 = x6423 + z6424 = x6424 + z6425 = x6425 + z6426 = x6426 + z6427 = x6427 + z6428 = x6428 + z6429 = x6429 + z6430 = x6430 + z6431 = x6431 + z6432 = x6432 + z6433 = x6433 + z6434 = x6434 + z6435 = x6435 + z6436 = x6436 + z6437 = x6437 + z6438 = x6438 + z6439 = x6439 + z6440 = x6440 + z6441 = x6441 + z6442 = x6442 + z6443 = x6443 + z6444 = x6444 + z6445 = x6445 + z6446 = x6446 + z6447 = x6447 + z6448 = x6448 + z6449 = x6449 + z6450 = x6450 + z6451 = x6451 + z6452 = x6452 + z6453 = x6453 + z6454 = x6454 + z6455 = x6455 + z6456 = x6456 + z6457 = x6457 + z6458 = x6458 + z6459 = x6459 + z6460 = x6460 + z6461 = x6461 + z6462 = x6462 + z6463 = x6463 + z6464 = x6464 + z6465 = x6465 + z6466 = x6466 + z6467 = x6467 + z6468 = x6468 + z6469 = x6469 + z6470 = x6470 + z6471 = x6471 + z6472 = x6472 + z6473 = x6473 + z6474 = x6474 + z6475 = x6475 + z6476 = x6476 + z6477 = x6477 + z6478 = x6478 + z6479 = x6479 + z6480 = x6480 + z6481 = x6481 + z6482 = x6482 + z6483 = x6483 + z6484 = x6484 + z6485 = x6485 + z6486 = x6486 + z6487 = x6487 + z6488 = x6488 + z6489 = x6489 + z6490 = x6490 + z6491 = x6491 + z6492 = x6492 + z6493 = x6493 + z6494 = x6494 + z6495 = x6495 + z6496 = x6496 + z6497 = x6497 + z6498 = x6498 + z6499 = x6499 + z6500 = x6500 + z6501 = x6501 + z6502 = x6502 + z6503 = x6503 + z6504 = x6504 + z6505 = x6505 + z6506 = x6506 + z6507 = x6507 + z6508 = x6508 + z6509 = x6509 + z6510 = x6510 + z6511 = x6511 + z6512 = x6512 + z6513 = x6513 + z6514 = x6514 + z6515 = x6515 + z6516 = x6516 + z6517 = x6517 + z6518 = x6518 + z6519 = x6519 + z6520 = x6520 + z6521 = x6521 + z6522 = x6522 + z6523 = x6523 + z6524 = x6524 + z6525 = x6525 + z6526 = x6526 + z6527 = x6527 + z6528 = x6528 + z6529 = x6529 + z6530 = x6530 + z6531 = x6531 + z6532 = x6532 + z6533 = x6533 + z6534 = x6534 + z6535 = x6535 + z6536 = x6536 + z6537 = x6537 + z6538 = x6538 + z6539 = x6539 + z6540 = x6540 + z6541 = x6541 + z6542 = x6542 + z6543 = x6543 + z6544 = x6544 + z6545 = x6545 + z6546 = x6546 + z6547 = x6547 + z6548 = x6548 + z6549 = x6549 + z6550 = x6550 + z6551 = x6551 + z6552 = x6552 + z6553 = x6553 + z6554 = x6554 + z6555 = x6555 + z6556 = x6556 + z6557 = x6557 + z6558 = x6558 + z6559 = x6559 + z6560 = x6560 + z6561 = x6561 + z6562 = x6562 + z6563 = x6563 + z6564 = x6564 + z6565 = x6565 + z6566 = x6566 + z6567 = x6567 + z6568 = x6568 + z6569 = x6569 + z6570 = x6570 + z6571 = x6571 + z6572 = x6572 + z6573 = x6573 + z6574 = x6574 + z6575 = x6575 + z6576 = x6576 + z6577 = x6577 + z6578 = x6578 + z6579 = x6579 + z6580 = x6580 + z6581 = x6581 + z6582 = x6582 + z6583 = x6583 + z6584 = x6584 + z6585 = x6585 + z6586 = x6586 + z6587 = x6587 + z6588 = x6588 + z6589 = x6589 + z6590 = x6590 + z6591 = x6591 + z6592 = x6592 + z6593 = x6593 + z6594 = x6594 + z6595 = x6595 + z6596 = x6596 + z6597 = x6597 + z6598 = x6598 + z6599 = x6599 + z6600 = x6600 + z6601 = x6601 + z6602 = x6602 + z6603 = x6603 + z6604 = x6604 + z6605 = x6605 + z6606 = x6606 + z6607 = x6607 + z6608 = x6608 + z6609 = x6609 + z6610 = x6610 + z6611 = x6611 + z6612 = x6612 + z6613 = x6613 + z6614 = x6614 + z6615 = x6615 + z6616 = x6616 + z6617 = x6617 + z6618 = x6618 + z6619 = x6619 + z6620 = x6620 + z6621 = x6621 + z6622 = x6622 + z6623 = x6623 + z6624 = x6624 + z6625 = x6625 + z6626 = x6626 + z6627 = x6627 + z6628 = x6628 + z6629 = x6629 + z6630 = x6630 + z6631 = x6631 + z6632 = x6632 + z6633 = x6633 + z6634 = x6634 + z6635 = x6635 + z6636 = x6636 + z6637 = x6637 + z6638 = x6638 + z6639 = x6639 + z6640 = x6640 + z6641 = x6641 + z6642 = x6642 + z6643 = x6643 + z6644 = x6644 + z6645 = x6645 + z6646 = x6646 + z6647 = x6647 + z6648 = x6648 + z6649 = x6649 + z6650 = x6650 + z6651 = x6651 + z6652 = x6652 + z6653 = x6653 + z6654 = x6654 + z6655 = x6655 + z6656 = x6656 + z6657 = x6657 + z6658 = x6658 + z6659 = x6659 + z6660 = x6660 + z6661 = x6661 + z6662 = x6662 + z6663 = x6663 + z6664 = x6664 + z6665 = x6665 + z6666 = x6666 + z6667 = x6667 + z6668 = x6668 + z6669 = x6669 + z6670 = x6670 + z6671 = x6671 + z6672 = x6672 + z6673 = x6673 + z6674 = x6674 + z6675 = x6675 + z6676 = x6676 + z6677 = x6677 + z6678 = x6678 + z6679 = x6679 + z6680 = x6680 + z6681 = x6681 + z6682 = x6682 + z6683 = x6683 + z6684 = x6684 + z6685 = x6685 + z6686 = x6686 + z6687 = x6687 + z6688 = x6688 + z6689 = x6689 + z6690 = x6690 + z6691 = x6691 + z6692 = x6692 + z6693 = x6693 + z6694 = x6694 + z6695 = x6695 + z6696 = x6696 + z6697 = x6697 + z6698 = x6698 + z6699 = x6699 + z6700 = x6700 + z6701 = x6701 + z6702 = x6702 + z6703 = x6703 + z6704 = x6704 + z6705 = x6705 + z6706 = x6706 + z6707 = x6707 + z6708 = x6708 + z6709 = x6709 + z6710 = x6710 + z6711 = x6711 + z6712 = x6712 + z6713 = x6713 + z6714 = x6714 + z6715 = x6715 + z6716 = x6716 + z6717 = x6717 + z6718 = x6718 + z6719 = x6719 + z6720 = x6720 + z6721 = x6721 + z6722 = x6722 + z6723 = x6723 + z6724 = x6724 + z6725 = x6725 + z6726 = x6726 + z6727 = x6727 + z6728 = x6728 + z6729 = x6729 + z6730 = x6730 + z6731 = x6731 + z6732 = x6732 + z6733 = x6733 + z6734 = x6734 + z6735 = x6735 + z6736 = x6736 + z6737 = x6737 + z6738 = x6738 + z6739 = x6739 + z6740 = x6740 + z6741 = x6741 + z6742 = x6742 + z6743 = x6743 + z6744 = x6744 + z6745 = x6745 + z6746 = x6746 + z6747 = x6747 + z6748 = x6748 + z6749 = x6749 + z6750 = x6750 + z6751 = x6751 + z6752 = x6752 + z6753 = x6753 + z6754 = x6754 + z6755 = x6755 + z6756 = x6756 + z6757 = x6757 + z6758 = x6758 + z6759 = x6759 + z6760 = x6760 + z6761 = x6761 + z6762 = x6762 + z6763 = x6763 + z6764 = x6764 + z6765 = x6765 + z6766 = x6766 + z6767 = x6767 + z6768 = x6768 + z6769 = x6769 + z6770 = x6770 + z6771 = x6771 + z6772 = x6772 + z6773 = x6773 + z6774 = x6774 + z6775 = x6775 + z6776 = x6776 + z6777 = x6777 + z6778 = x6778 + z6779 = x6779 + z6780 = x6780 + z6781 = x6781 + z6782 = x6782 + z6783 = x6783 + z6784 = x6784 + z6785 = x6785 + z6786 = x6786 + z6787 = x6787 + z6788 = x6788 + z6789 = x6789 + z6790 = x6790 + z6791 = x6791 + z6792 = x6792 + z6793 = x6793 + z6794 = x6794 + z6795 = x6795 + z6796 = x6796 + z6797 = x6797 + z6798 = x6798 + z6799 = x6799 + z6800 = x6800 + z6801 = x6801 + z6802 = x6802 + z6803 = x6803 + z6804 = x6804 + z6805 = x6805 + z6806 = x6806 + z6807 = x6807 + z6808 = x6808 + z6809 = x6809 + z6810 = x6810 + z6811 = x6811 + z6812 = x6812 + z6813 = x6813 + z6814 = x6814 + z6815 = x6815 + z6816 = x6816 + z6817 = x6817 + z6818 = x6818 + z6819 = x6819 + z6820 = x6820 + z6821 = x6821 + z6822 = x6822 + z6823 = x6823 + z6824 = x6824 + z6825 = x6825 + z6826 = x6826 + z6827 = x6827 + z6828 = x6828 + z6829 = x6829 + z6830 = x6830 + z6831 = x6831 + z6832 = x6832 + z6833 = x6833 + z6834 = x6834 + z6835 = x6835 + z6836 = x6836 + z6837 = x6837 + z6838 = x6838 + z6839 = x6839 + z6840 = x6840 + z6841 = x6841 + z6842 = x6842 + z6843 = x6843 + z6844 = x6844 + z6845 = x6845 + z6846 = x6846 + z6847 = x6847 + z6848 = x6848 + z6849 = x6849 + z6850 = x6850 + z6851 = x6851 + z6852 = x6852 + z6853 = x6853 + z6854 = x6854 + z6855 = x6855 + z6856 = x6856 + z6857 = x6857 + z6858 = x6858 + z6859 = x6859 + z6860 = x6860 + z6861 = x6861 + z6862 = x6862 + z6863 = x6863 + z6864 = x6864 + z6865 = x6865 + z6866 = x6866 + z6867 = x6867 + z6868 = x6868 + z6869 = x6869 + z6870 = x6870 + z6871 = x6871 + z6872 = x6872 + z6873 = x6873 + z6874 = x6874 + z6875 = x6875 + z6876 = x6876 + z6877 = x6877 + z6878 = x6878 + z6879 = x6879 + z6880 = x6880 + z6881 = x6881 + z6882 = x6882 + z6883 = x6883 + z6884 = x6884 + z6885 = x6885 + z6886 = x6886 + z6887 = x6887 + z6888 = x6888 + z6889 = x6889 + z6890 = x6890 + z6891 = x6891 + z6892 = x6892 + z6893 = x6893 + z6894 = x6894 + z6895 = x6895 + z6896 = x6896 + z6897 = x6897 + z6898 = x6898 + z6899 = x6899 + z6900 = x6900 + z6901 = x6901 + z6902 = x6902 + z6903 = x6903 + z6904 = x6904 + z6905 = x6905 + z6906 = x6906 + z6907 = x6907 + z6908 = x6908 + z6909 = x6909 + z6910 = x6910 + z6911 = x6911 + z6912 = x6912 + z6913 = x6913 + z6914 = x6914 + z6915 = x6915 + z6916 = x6916 + z6917 = x6917 + z6918 = x6918 + z6919 = x6919 + z6920 = x6920 + z6921 = x6921 + z6922 = x6922 + z6923 = x6923 + z6924 = x6924 + z6925 = x6925 + z6926 = x6926 + z6927 = x6927 + z6928 = x6928 + z6929 = x6929 + z6930 = x6930 + z6931 = x6931 + z6932 = x6932 + z6933 = x6933 + z6934 = x6934 + z6935 = x6935 + z6936 = x6936 + z6937 = x6937 + z6938 = x6938 + z6939 = x6939 + z6940 = x6940 + z6941 = x6941 + z6942 = x6942 + z6943 = x6943 + z6944 = x6944 + z6945 = x6945 + z6946 = x6946 + z6947 = x6947 + z6948 = x6948 + z6949 = x6949 + z6950 = x6950 + z6951 = x6951 + z6952 = x6952 + z6953 = x6953 + z6954 = x6954 + z6955 = x6955 + z6956 = x6956 + z6957 = x6957 + z6958 = x6958 + z6959 = x6959 + z6960 = x6960 + z6961 = x6961 + z6962 = x6962 + z6963 = x6963 + z6964 = x6964 + z6965 = x6965 + z6966 = x6966 + z6967 = x6967 + z6968 = x6968 + z6969 = x6969 + z6970 = x6970 + z6971 = x6971 + z6972 = x6972 + z6973 = x6973 + z6974 = x6974 + z6975 = x6975 + z6976 = x6976 + z6977 = x6977 + z6978 = x6978 + z6979 = x6979 + z6980 = x6980 + z6981 = x6981 + z6982 = x6982 + z6983 = x6983 + z6984 = x6984 + z6985 = x6985 + z6986 = x6986 + z6987 = x6987 + z6988 = x6988 + z6989 = x6989 + z6990 = x6990 + z6991 = x6991 + z6992 = x6992 + z6993 = x6993 + z6994 = x6994 + z6995 = x6995 + z6996 = x6996 + z6997 = x6997 + z6998 = x6998 + z6999 = x6999 + z7000 = x7000 + z7001 = x7001 + z7002 = x7002 + z7003 = x7003 + z7004 = x7004 + z7005 = x7005 + z7006 = x7006 + z7007 = x7007 + z7008 = x7008 + z7009 = x7009 + z7010 = x7010 + z7011 = x7011 + z7012 = x7012 + z7013 = x7013 + z7014 = x7014 + z7015 = x7015 + z7016 = x7016 + z7017 = x7017 + z7018 = x7018 + z7019 = x7019 + z7020 = x7020 + z7021 = x7021 + z7022 = x7022 + z7023 = x7023 + z7024 = x7024 + z7025 = x7025 + z7026 = x7026 + z7027 = x7027 + z7028 = x7028 + z7029 = x7029 + z7030 = x7030 + z7031 = x7031 + z7032 = x7032 + z7033 = x7033 + z7034 = x7034 + z7035 = x7035 + z7036 = x7036 + z7037 = x7037 + z7038 = x7038 + z7039 = x7039 + z7040 = x7040 + z7041 = x7041 + z7042 = x7042 + z7043 = x7043 + z7044 = x7044 + z7045 = x7045 + z7046 = x7046 + z7047 = x7047 + z7048 = x7048 + z7049 = x7049 + z7050 = x7050 + z7051 = x7051 + z7052 = x7052 + z7053 = x7053 + z7054 = x7054 + z7055 = x7055 + z7056 = x7056 + z7057 = x7057 + z7058 = x7058 + z7059 = x7059 + z7060 = x7060 + z7061 = x7061 + z7062 = x7062 + z7063 = x7063 + z7064 = x7064 + z7065 = x7065 + z7066 = x7066 + z7067 = x7067 + z7068 = x7068 + z7069 = x7069 + z7070 = x7070 + z7071 = x7071 + z7072 = x7072 + z7073 = x7073 + z7074 = x7074 + z7075 = x7075 + z7076 = x7076 + z7077 = x7077 + z7078 = x7078 + z7079 = x7079 + z7080 = x7080 + z7081 = x7081 + z7082 = x7082 + z7083 = x7083 + z7084 = x7084 + z7085 = x7085 + z7086 = x7086 + z7087 = x7087 + z7088 = x7088 + z7089 = x7089 + z7090 = x7090 + z7091 = x7091 + z7092 = x7092 + z7093 = x7093 + z7094 = x7094 + z7095 = x7095 + z7096 = x7096 + z7097 = x7097 + z7098 = x7098 + z7099 = x7099 + z7100 = x7100 + z7101 = x7101 + z7102 = x7102 + z7103 = x7103 + z7104 = x7104 + z7105 = x7105 + z7106 = x7106 + z7107 = x7107 + z7108 = x7108 + z7109 = x7109 + z7110 = x7110 + z7111 = x7111 + z7112 = x7112 + z7113 = x7113 + z7114 = x7114 + z7115 = x7115 + z7116 = x7116 + z7117 = x7117 + z7118 = x7118 + z7119 = x7119 + z7120 = x7120 + z7121 = x7121 + z7122 = x7122 + z7123 = x7123 + z7124 = x7124 + z7125 = x7125 + z7126 = x7126 + z7127 = x7127 + z7128 = x7128 + z7129 = x7129 + z7130 = x7130 + z7131 = x7131 + z7132 = x7132 + z7133 = x7133 + z7134 = x7134 + z7135 = x7135 + z7136 = x7136 + z7137 = x7137 + z7138 = x7138 + z7139 = x7139 + z7140 = x7140 + z7141 = x7141 + z7142 = x7142 + z7143 = x7143 + z7144 = x7144 + z7145 = x7145 + z7146 = x7146 + z7147 = x7147 + z7148 = x7148 + z7149 = x7149 + z7150 = x7150 + z7151 = x7151 + z7152 = x7152 + z7153 = x7153 + z7154 = x7154 + z7155 = x7155 + z7156 = x7156 + z7157 = x7157 + z7158 = x7158 + z7159 = x7159 + z7160 = x7160 + z7161 = x7161 + z7162 = x7162 + z7163 = x7163 + z7164 = x7164 + z7165 = x7165 + z7166 = x7166 + z7167 = x7167 + z7168 = x7168 + z7169 = x7169 + z7170 = x7170 + z7171 = x7171 + z7172 = x7172 + z7173 = x7173 + z7174 = x7174 + z7175 = x7175 + z7176 = x7176 + z7177 = x7177 + z7178 = x7178 + z7179 = x7179 + z7180 = x7180 + z7181 = x7181 + z7182 = x7182 + z7183 = x7183 + z7184 = x7184 + z7185 = x7185 + z7186 = x7186 + z7187 = x7187 + z7188 = x7188 + z7189 = x7189 + z7190 = x7190 + z7191 = x7191 + z7192 = x7192 + z7193 = x7193 + z7194 = x7194 + z7195 = x7195 + z7196 = x7196 + z7197 = x7197 + z7198 = x7198 + z7199 = x7199 + z7200 = x7200 + z7201 = x7201 + z7202 = x7202 + z7203 = x7203 + z7204 = x7204 + z7205 = x7205 + z7206 = x7206 + z7207 = x7207 + z7208 = x7208 + z7209 = x7209 + z7210 = x7210 + z7211 = x7211 + z7212 = x7212 + z7213 = x7213 + z7214 = x7214 + z7215 = x7215 + z7216 = x7216 + z7217 = x7217 + z7218 = x7218 + z7219 = x7219 + z7220 = x7220 + z7221 = x7221 + z7222 = x7222 + z7223 = x7223 + z7224 = x7224 + z7225 = x7225 + z7226 = x7226 + z7227 = x7227 + z7228 = x7228 + z7229 = x7229 + z7230 = x7230 + z7231 = x7231 + z7232 = x7232 + z7233 = x7233 + z7234 = x7234 + z7235 = x7235 + z7236 = x7236 + z7237 = x7237 + z7238 = x7238 + z7239 = x7239 + z7240 = x7240 + z7241 = x7241 + z7242 = x7242 + z7243 = x7243 + z7244 = x7244 + z7245 = x7245 + z7246 = x7246 + z7247 = x7247 + z7248 = x7248 + z7249 = x7249 + z7250 = x7250 + z7251 = x7251 + z7252 = x7252 + z7253 = x7253 + z7254 = x7254 + z7255 = x7255 + z7256 = x7256 + z7257 = x7257 + z7258 = x7258 + z7259 = x7259 + z7260 = x7260 + z7261 = x7261 + z7262 = x7262 + z7263 = x7263 + z7264 = x7264 + z7265 = x7265 + z7266 = x7266 + z7267 = x7267 + z7268 = x7268 + z7269 = x7269 + z7270 = x7270 + z7271 = x7271 + z7272 = x7272 + z7273 = x7273 + z7274 = x7274 + z7275 = x7275 + z7276 = x7276 + z7277 = x7277 + z7278 = x7278 + z7279 = x7279 + z7280 = x7280 + z7281 = x7281 + z7282 = x7282 + z7283 = x7283 + z7284 = x7284 + z7285 = x7285 + z7286 = x7286 + z7287 = x7287 + z7288 = x7288 + z7289 = x7289 + z7290 = x7290 + z7291 = x7291 + z7292 = x7292 + z7293 = x7293 + z7294 = x7294 + z7295 = x7295 + z7296 = x7296 + z7297 = x7297 + z7298 = x7298 + z7299 = x7299 + z7300 = x7300 + z7301 = x7301 + z7302 = x7302 + z7303 = x7303 + z7304 = x7304 + z7305 = x7305 + z7306 = x7306 + z7307 = x7307 + z7308 = x7308 + z7309 = x7309 + z7310 = x7310 + z7311 = x7311 + z7312 = x7312 + z7313 = x7313 + z7314 = x7314 + z7315 = x7315 + z7316 = x7316 + z7317 = x7317 + z7318 = x7318 + z7319 = x7319 + z7320 = x7320 + z7321 = x7321 + z7322 = x7322 + z7323 = x7323 + z7324 = x7324 + z7325 = x7325 + z7326 = x7326 + z7327 = x7327 + z7328 = x7328 + z7329 = x7329 + z7330 = x7330 + z7331 = x7331 + z7332 = x7332 + z7333 = x7333 + z7334 = x7334 + z7335 = x7335 + z7336 = x7336 + z7337 = x7337 + z7338 = x7338 + z7339 = x7339 + z7340 = x7340 + z7341 = x7341 + z7342 = x7342 + z7343 = x7343 + z7344 = x7344 + z7345 = x7345 + z7346 = x7346 + z7347 = x7347 + z7348 = x7348 + z7349 = x7349 + z7350 = x7350 + z7351 = x7351 + z7352 = x7352 + z7353 = x7353 + z7354 = x7354 + z7355 = x7355 + z7356 = x7356 + z7357 = x7357 + z7358 = x7358 + z7359 = x7359 + z7360 = x7360 + z7361 = x7361 + z7362 = x7362 + z7363 = x7363 + z7364 = x7364 + z7365 = x7365 + z7366 = x7366 + z7367 = x7367 + z7368 = x7368 + z7369 = x7369 + z7370 = x7370 + z7371 = x7371 + z7372 = x7372 + z7373 = x7373 + z7374 = x7374 + z7375 = x7375 + z7376 = x7376 + z7377 = x7377 + z7378 = x7378 + z7379 = x7379 + z7380 = x7380 + z7381 = x7381 + z7382 = x7382 + z7383 = x7383 + z7384 = x7384 + z7385 = x7385 + z7386 = x7386 + z7387 = x7387 + z7388 = x7388 + z7389 = x7389 + z7390 = x7390 + z7391 = x7391 + z7392 = x7392 + z7393 = x7393 + z7394 = x7394 + z7395 = x7395 + z7396 = x7396 + z7397 = x7397 + z7398 = x7398 + z7399 = x7399 + z7400 = x7400 + z7401 = x7401 + z7402 = x7402 + z7403 = x7403 + z7404 = x7404 + z7405 = x7405 + z7406 = x7406 + z7407 = x7407 + z7408 = x7408 + z7409 = x7409 + z7410 = x7410 + z7411 = x7411 + z7412 = x7412 + z7413 = x7413 + z7414 = x7414 + z7415 = x7415 + z7416 = x7416 + z7417 = x7417 + z7418 = x7418 + z7419 = x7419 + z7420 = x7420 + z7421 = x7421 + z7422 = x7422 + z7423 = x7423 + z7424 = x7424 + z7425 = x7425 + z7426 = x7426 + z7427 = x7427 + z7428 = x7428 + z7429 = x7429 + z7430 = x7430 + z7431 = x7431 + z7432 = x7432 + z7433 = x7433 + z7434 = x7434 + z7435 = x7435 + z7436 = x7436 + z7437 = x7437 + z7438 = x7438 + z7439 = x7439 + z7440 = x7440 + z7441 = x7441 + z7442 = x7442 + z7443 = x7443 + z7444 = x7444 + z7445 = x7445 + z7446 = x7446 + z7447 = x7447 + z7448 = x7448 + z7449 = x7449 + z7450 = x7450 + z7451 = x7451 + z7452 = x7452 + z7453 = x7453 + z7454 = x7454 + z7455 = x7455 + z7456 = x7456 + z7457 = x7457 + z7458 = x7458 + z7459 = x7459 + z7460 = x7460 + z7461 = x7461 + z7462 = x7462 + z7463 = x7463 + z7464 = x7464 + z7465 = x7465 + z7466 = x7466 + z7467 = x7467 + z7468 = x7468 + z7469 = x7469 + z7470 = x7470 + z7471 = x7471 + z7472 = x7472 + z7473 = x7473 + z7474 = x7474 + z7475 = x7475 + z7476 = x7476 + z7477 = x7477 + z7478 = x7478 + z7479 = x7479 + z7480 = x7480 + z7481 = x7481 + z7482 = x7482 + z7483 = x7483 + z7484 = x7484 + z7485 = x7485 + z7486 = x7486 + z7487 = x7487 + z7488 = x7488 + z7489 = x7489 + z7490 = x7490 + z7491 = x7491 + z7492 = x7492 + z7493 = x7493 + z7494 = x7494 + z7495 = x7495 + z7496 = x7496 + z7497 = x7497 + z7498 = x7498 + z7499 = x7499 + z7500 = x7500 + z7501 = x7501 + z7502 = x7502 + z7503 = x7503 + z7504 = x7504 + z7505 = x7505 + z7506 = x7506 + z7507 = x7507 + z7508 = x7508 + z7509 = x7509 + z7510 = x7510 + z7511 = x7511 + z7512 = x7512 + z7513 = x7513 + z7514 = x7514 + z7515 = x7515 + z7516 = x7516 + z7517 = x7517 + z7518 = x7518 + z7519 = x7519 + z7520 = x7520 + z7521 = x7521 + z7522 = x7522 + z7523 = x7523 + z7524 = x7524 + z7525 = x7525 + z7526 = x7526 + z7527 = x7527 + z7528 = x7528 + z7529 = x7529 + z7530 = x7530 + z7531 = x7531 + z7532 = x7532 + z7533 = x7533 + z7534 = x7534 + z7535 = x7535 + z7536 = x7536 + z7537 = x7537 + z7538 = x7538 + z7539 = x7539 + z7540 = x7540 + z7541 = x7541 + z7542 = x7542 + z7543 = x7543 + z7544 = x7544 + z7545 = x7545 + z7546 = x7546 + z7547 = x7547 + z7548 = x7548 + z7549 = x7549 + z7550 = x7550 + z7551 = x7551 + z7552 = x7552 + z7553 = x7553 + z7554 = x7554 + z7555 = x7555 + z7556 = x7556 + z7557 = x7557 + z7558 = x7558 + z7559 = x7559 + z7560 = x7560 + z7561 = x7561 + z7562 = x7562 + z7563 = x7563 + z7564 = x7564 + z7565 = x7565 + z7566 = x7566 + z7567 = x7567 + z7568 = x7568 + z7569 = x7569 + z7570 = x7570 + z7571 = x7571 + z7572 = x7572 + z7573 = x7573 + z7574 = x7574 + z7575 = x7575 + z7576 = x7576 + z7577 = x7577 + z7578 = x7578 + z7579 = x7579 + z7580 = x7580 + z7581 = x7581 + z7582 = x7582 + z7583 = x7583 + z7584 = x7584 + z7585 = x7585 + z7586 = x7586 + z7587 = x7587 + z7588 = x7588 + z7589 = x7589 + z7590 = x7590 + z7591 = x7591 + z7592 = x7592 + z7593 = x7593 + z7594 = x7594 + z7595 = x7595 + z7596 = x7596 + z7597 = x7597 + z7598 = x7598 + z7599 = x7599 + z7600 = x7600 + z7601 = x7601 + z7602 = x7602 + z7603 = x7603 + z7604 = x7604 + z7605 = x7605 + z7606 = x7606 + z7607 = x7607 + z7608 = x7608 + z7609 = x7609 + z7610 = x7610 + z7611 = x7611 + z7612 = x7612 + z7613 = x7613 + z7614 = x7614 + z7615 = x7615 + z7616 = x7616 + z7617 = x7617 + z7618 = x7618 + z7619 = x7619 + z7620 = x7620 + z7621 = x7621 + z7622 = x7622 + z7623 = x7623 + z7624 = x7624 + z7625 = x7625 + z7626 = x7626 + z7627 = x7627 + z7628 = x7628 + z7629 = x7629 + z7630 = x7630 + z7631 = x7631 + z7632 = x7632 + z7633 = x7633 + z7634 = x7634 + z7635 = x7635 + z7636 = x7636 + z7637 = x7637 + z7638 = x7638 + z7639 = x7639 + z7640 = x7640 + z7641 = x7641 + z7642 = x7642 + z7643 = x7643 + z7644 = x7644 + z7645 = x7645 + z7646 = x7646 + z7647 = x7647 + z7648 = x7648 + z7649 = x7649 + z7650 = x7650 + z7651 = x7651 + z7652 = x7652 + z7653 = x7653 + z7654 = x7654 + z7655 = x7655 + z7656 = x7656 + z7657 = x7657 + z7658 = x7658 + z7659 = x7659 + z7660 = x7660 + z7661 = x7661 + z7662 = x7662 + z7663 = x7663 + z7664 = x7664 + z7665 = x7665 + z7666 = x7666 + z7667 = x7667 + z7668 = x7668 + z7669 = x7669 + z7670 = x7670 + z7671 = x7671 + z7672 = x7672 + z7673 = x7673 + z7674 = x7674 + z7675 = x7675 + z7676 = x7676 + z7677 = x7677 + z7678 = x7678 + z7679 = x7679 + z7680 = x7680 + z7681 = x7681 + z7682 = x7682 + z7683 = x7683 + z7684 = x7684 + z7685 = x7685 + z7686 = x7686 + z7687 = x7687 + z7688 = x7688 + z7689 = x7689 + z7690 = x7690 + z7691 = x7691 + z7692 = x7692 + z7693 = x7693 + z7694 = x7694 + z7695 = x7695 + z7696 = x7696 + z7697 = x7697 + z7698 = x7698 + z7699 = x7699 + z7700 = x7700 + z7701 = x7701 + z7702 = x7702 + z7703 = x7703 + z7704 = x7704 + z7705 = x7705 + z7706 = x7706 + z7707 = x7707 + z7708 = x7708 + z7709 = x7709 + z7710 = x7710 + z7711 = x7711 + z7712 = x7712 + z7713 = x7713 + z7714 = x7714 + z7715 = x7715 + z7716 = x7716 + z7717 = x7717 + z7718 = x7718 + z7719 = x7719 + z7720 = x7720 + z7721 = x7721 + z7722 = x7722 + z7723 = x7723 + z7724 = x7724 + z7725 = x7725 + z7726 = x7726 + z7727 = x7727 + z7728 = x7728 + z7729 = x7729 + z7730 = x7730 + z7731 = x7731 + z7732 = x7732 + z7733 = x7733 + z7734 = x7734 + z7735 = x7735 + z7736 = x7736 + z7737 = x7737 + z7738 = x7738 + z7739 = x7739 + z7740 = x7740 + z7741 = x7741 + z7742 = x7742 + z7743 = x7743 + z7744 = x7744 + z7745 = x7745 + z7746 = x7746 + z7747 = x7747 + z7748 = x7748 + z7749 = x7749 + z7750 = x7750 + z7751 = x7751 + z7752 = x7752 + z7753 = x7753 + z7754 = x7754 + z7755 = x7755 + z7756 = x7756 + z7757 = x7757 + z7758 = x7758 + z7759 = x7759 + z7760 = x7760 + z7761 = x7761 + z7762 = x7762 + z7763 = x7763 + z7764 = x7764 + z7765 = x7765 + z7766 = x7766 + z7767 = x7767 + z7768 = x7768 + z7769 = x7769 + z7770 = x7770 + z7771 = x7771 + z7772 = x7772 + z7773 = x7773 + z7774 = x7774 + z7775 = x7775 + z7776 = x7776 + z7777 = x7777 + z7778 = x7778 + z7779 = x7779 + z7780 = x7780 + z7781 = x7781 + z7782 = x7782 + z7783 = x7783 + z7784 = x7784 + z7785 = x7785 + z7786 = x7786 + z7787 = x7787 + z7788 = x7788 + z7789 = x7789 + z7790 = x7790 + z7791 = x7791 + z7792 = x7792 + z7793 = x7793 + z7794 = x7794 + z7795 = x7795 + z7796 = x7796 + z7797 = x7797 + z7798 = x7798 + z7799 = x7799 + z7800 = x7800 + z7801 = x7801 + z7802 = x7802 + z7803 = x7803 + z7804 = x7804 + z7805 = x7805 + z7806 = x7806 + z7807 = x7807 + z7808 = x7808 + z7809 = x7809 + z7810 = x7810 + z7811 = x7811 + z7812 = x7812 + z7813 = x7813 + z7814 = x7814 + z7815 = x7815 + z7816 = x7816 + z7817 = x7817 + z7818 = x7818 + z7819 = x7819 + z7820 = x7820 + z7821 = x7821 + z7822 = x7822 + z7823 = x7823 + z7824 = x7824 + z7825 = x7825 + z7826 = x7826 + z7827 = x7827 + z7828 = x7828 + z7829 = x7829 + z7830 = x7830 + z7831 = x7831 + z7832 = x7832 + z7833 = x7833 + z7834 = x7834 + z7835 = x7835 + z7836 = x7836 + z7837 = x7837 + z7838 = x7838 + z7839 = x7839 + z7840 = x7840 + z7841 = x7841 + z7842 = x7842 + z7843 = x7843 + z7844 = x7844 + z7845 = x7845 + z7846 = x7846 + z7847 = x7847 + z7848 = x7848 + z7849 = x7849 + z7850 = x7850 + z7851 = x7851 + z7852 = x7852 + z7853 = x7853 + z7854 = x7854 + z7855 = x7855 + z7856 = x7856 + z7857 = x7857 + z7858 = x7858 + z7859 = x7859 + z7860 = x7860 + z7861 = x7861 + z7862 = x7862 + z7863 = x7863 + z7864 = x7864 + z7865 = x7865 + z7866 = x7866 + z7867 = x7867 + z7868 = x7868 + z7869 = x7869 + z7870 = x7870 + z7871 = x7871 + z7872 = x7872 + z7873 = x7873 + z7874 = x7874 + z7875 = x7875 + z7876 = x7876 + z7877 = x7877 + z7878 = x7878 + z7879 = x7879 + z7880 = x7880 + z7881 = x7881 + z7882 = x7882 + z7883 = x7883 + z7884 = x7884 + z7885 = x7885 + z7886 = x7886 + z7887 = x7887 + z7888 = x7888 + z7889 = x7889 + z7890 = x7890 + z7891 = x7891 + z7892 = x7892 + z7893 = x7893 + z7894 = x7894 + z7895 = x7895 + z7896 = x7896 + z7897 = x7897 + z7898 = x7898 + z7899 = x7899 + z7900 = x7900 + z7901 = x7901 + z7902 = x7902 + z7903 = x7903 + z7904 = x7904 + z7905 = x7905 + z7906 = x7906 + z7907 = x7907 + z7908 = x7908 + z7909 = x7909 + z7910 = x7910 + z7911 = x7911 + z7912 = x7912 + z7913 = x7913 + z7914 = x7914 + z7915 = x7915 + z7916 = x7916 + z7917 = x7917 + z7918 = x7918 + z7919 = x7919 + z7920 = x7920 + z7921 = x7921 + z7922 = x7922 + z7923 = x7923 + z7924 = x7924 + z7925 = x7925 + z7926 = x7926 + z7927 = x7927 + z7928 = x7928 + z7929 = x7929 + z7930 = x7930 + z7931 = x7931 + z7932 = x7932 + z7933 = x7933 + z7934 = x7934 + z7935 = x7935 + z7936 = x7936 + z7937 = x7937 + z7938 = x7938 + z7939 = x7939 + z7940 = x7940 + z7941 = x7941 + z7942 = x7942 + z7943 = x7943 + z7944 = x7944 + z7945 = x7945 + z7946 = x7946 + z7947 = x7947 + z7948 = x7948 + z7949 = x7949 + z7950 = x7950 + z7951 = x7951 + z7952 = x7952 + z7953 = x7953 + z7954 = x7954 + z7955 = x7955 + z7956 = x7956 + z7957 = x7957 + z7958 = x7958 + z7959 = x7959 + z7960 = x7960 + z7961 = x7961 + z7962 = x7962 + z7963 = x7963 + z7964 = x7964 + z7965 = x7965 + z7966 = x7966 + z7967 = x7967 + z7968 = x7968 + z7969 = x7969 + z7970 = x7970 + z7971 = x7971 + z7972 = x7972 + z7973 = x7973 + z7974 = x7974 + z7975 = x7975 + z7976 = x7976 + z7977 = x7977 + z7978 = x7978 + z7979 = x7979 + z7980 = x7980 + z7981 = x7981 + z7982 = x7982 + z7983 = x7983 + z7984 = x7984 + z7985 = x7985 + z7986 = x7986 + z7987 = x7987 + z7988 = x7988 + z7989 = x7989 + z7990 = x7990 + z7991 = x7991 + z7992 = x7992 + z7993 = x7993 + z7994 = x7994 + z7995 = x7995 + z7996 = x7996 + z7997 = x7997 + z7998 = x7998 + z7999 = x7999 + z8000 = x8000 + z8001 = x8001 + z8002 = x8002 + z8003 = x8003 + z8004 = x8004 + z8005 = x8005 + z8006 = x8006 + z8007 = x8007 + z8008 = x8008 + z8009 = x8009 + z8010 = x8010 + z8011 = x8011 + z8012 = x8012 + z8013 = x8013 + z8014 = x8014 + z8015 = x8015 + z8016 = x8016 + z8017 = x8017 + z8018 = x8018 + z8019 = x8019 + z8020 = x8020 + z8021 = x8021 + z8022 = x8022 + z8023 = x8023 + z8024 = x8024 + z8025 = x8025 + z8026 = x8026 + z8027 = x8027 + z8028 = x8028 + z8029 = x8029 + z8030 = x8030 + z8031 = x8031 + z8032 = x8032 + z8033 = x8033 + z8034 = x8034 + z8035 = x8035 + z8036 = x8036 + z8037 = x8037 + z8038 = x8038 + z8039 = x8039 + z8040 = x8040 + z8041 = x8041 + z8042 = x8042 + z8043 = x8043 + z8044 = x8044 + z8045 = x8045 + z8046 = x8046 + z8047 = x8047 + z8048 = x8048 + z8049 = x8049 + z8050 = x8050 + z8051 = x8051 + z8052 = x8052 + z8053 = x8053 + z8054 = x8054 + z8055 = x8055 + z8056 = x8056 + z8057 = x8057 + z8058 = x8058 + z8059 = x8059 + z8060 = x8060 + z8061 = x8061 + z8062 = x8062 + z8063 = x8063 + z8064 = x8064 + z8065 = x8065 + z8066 = x8066 + z8067 = x8067 + z8068 = x8068 + z8069 = x8069 + z8070 = x8070 + z8071 = x8071 + z8072 = x8072 + z8073 = x8073 + z8074 = x8074 + z8075 = x8075 + z8076 = x8076 + z8077 = x8077 + z8078 = x8078 + z8079 = x8079 + z8080 = x8080 + z8081 = x8081 + z8082 = x8082 + z8083 = x8083 + z8084 = x8084 + z8085 = x8085 + z8086 = x8086 + z8087 = x8087 + z8088 = x8088 + z8089 = x8089 + z8090 = x8090 + z8091 = x8091 + z8092 = x8092 + z8093 = x8093 + z8094 = x8094 + z8095 = x8095 + z8096 = x8096 + z8097 = x8097 + z8098 = x8098 + z8099 = x8099 + z8100 = x8100 + z8101 = x8101 + z8102 = x8102 + z8103 = x8103 + z8104 = x8104 + z8105 = x8105 + z8106 = x8106 + z8107 = x8107 + z8108 = x8108 + z8109 = x8109 + z8110 = x8110 + z8111 = x8111 + z8112 = x8112 + z8113 = x8113 + z8114 = x8114 + z8115 = x8115 + z8116 = x8116 + z8117 = x8117 + z8118 = x8118 + z8119 = x8119 + z8120 = x8120 + z8121 = x8121 + z8122 = x8122 + z8123 = x8123 + z8124 = x8124 + z8125 = x8125 + z8126 = x8126 + z8127 = x8127 + z8128 = x8128 + z8129 = x8129 + z8130 = x8130 + z8131 = x8131 + z8132 = x8132 + z8133 = x8133 + z8134 = x8134 + z8135 = x8135 + z8136 = x8136 + z8137 = x8137 + z8138 = x8138 + z8139 = x8139 + z8140 = x8140 + z8141 = x8141 + z8142 = x8142 + z8143 = x8143 + z8144 = x8144 + z8145 = x8145 + z8146 = x8146 + z8147 = x8147 + z8148 = x8148 + z8149 = x8149 + z8150 = x8150 + z8151 = x8151 + z8152 = x8152 + z8153 = x8153 + z8154 = x8154 + z8155 = x8155 + z8156 = x8156 + z8157 = x8157 + z8158 = x8158 + z8159 = x8159 + z8160 = x8160 + z8161 = x8161 + z8162 = x8162 + z8163 = x8163 + z8164 = x8164 + z8165 = x8165 + z8166 = x8166 + z8167 = x8167 + z8168 = x8168 + z8169 = x8169 + z8170 = x8170 + z8171 = x8171 + z8172 = x8172 + z8173 = x8173 + z8174 = x8174 + z8175 = x8175 + z8176 = x8176 + z8177 = x8177 + z8178 = x8178 + z8179 = x8179 + z8180 = x8180 + z8181 = x8181 + z8182 = x8182 + z8183 = x8183 + z8184 = x8184 + z8185 = x8185 + z8186 = x8186 + z8187 = x8187 + z8188 = x8188 + z8189 = x8189 + z8190 = x8190 + z8191 = x8191 + z8192 = x8192 + z8193 = x8193 + z8194 = x8194 + z8195 = x8195 + z8196 = x8196 + z8197 = x8197 + z8198 = x8198 + z8199 = x8199 + z8200 = x8200 + z8201 = x8201 + z8202 = x8202 + z8203 = x8203 + z8204 = x8204 + z8205 = x8205 + z8206 = x8206 + z8207 = x8207 + z8208 = x8208 + z8209 = x8209 + z8210 = x8210 + z8211 = x8211 + z8212 = x8212 + z8213 = x8213 + z8214 = x8214 + z8215 = x8215 + z8216 = x8216 + z8217 = x8217 + z8218 = x8218 + z8219 = x8219 + z8220 = x8220 + z8221 = x8221 + z8222 = x8222 + z8223 = x8223 + z8224 = x8224 + z8225 = x8225 + z8226 = x8226 + z8227 = x8227 + z8228 = x8228 + z8229 = x8229 + z8230 = x8230 + z8231 = x8231 + z8232 = x8232 + z8233 = x8233 + z8234 = x8234 + z8235 = x8235 + z8236 = x8236 + z8237 = x8237 + z8238 = x8238 + z8239 = x8239 + z8240 = x8240 + z8241 = x8241 + z8242 = x8242 + z8243 = x8243 + z8244 = x8244 + z8245 = x8245 + z8246 = x8246 + z8247 = x8247 + z8248 = x8248 + z8249 = x8249 + z8250 = x8250 + z8251 = x8251 + z8252 = x8252 + z8253 = x8253 + z8254 = x8254 + z8255 = x8255 + z8256 = x8256 + z8257 = x8257 + z8258 = x8258 + z8259 = x8259 + z8260 = x8260 + z8261 = x8261 + z8262 = x8262 + z8263 = x8263 + z8264 = x8264 + z8265 = x8265 + z8266 = x8266 + z8267 = x8267 + z8268 = x8268 + z8269 = x8269 + z8270 = x8270 + z8271 = x8271 + z8272 = x8272 + z8273 = x8273 + z8274 = x8274 + z8275 = x8275 + z8276 = x8276 + z8277 = x8277 + z8278 = x8278 + z8279 = x8279 + z8280 = x8280 + z8281 = x8281 + z8282 = x8282 + z8283 = x8283 + z8284 = x8284 + z8285 = x8285 + z8286 = x8286 + z8287 = x8287 + z8288 = x8288 + z8289 = x8289 + z8290 = x8290 + z8291 = x8291 + z8292 = x8292 + z8293 = x8293 + z8294 = x8294 + z8295 = x8295 + z8296 = x8296 + z8297 = x8297 + z8298 = x8298 + z8299 = x8299 + z8300 = x8300 + z8301 = x8301 + z8302 = x8302 + z8303 = x8303 + z8304 = x8304 + z8305 = x8305 + z8306 = x8306 + z8307 = x8307 + z8308 = x8308 + z8309 = x8309 + z8310 = x8310 + z8311 = x8311 + z8312 = x8312 + z8313 = x8313 + z8314 = x8314 + z8315 = x8315 + z8316 = x8316 + z8317 = x8317 + z8318 = x8318 + z8319 = x8319 + z8320 = x8320 + z8321 = x8321 + z8322 = x8322 + z8323 = x8323 + z8324 = x8324 + z8325 = x8325 + z8326 = x8326 + z8327 = x8327 + z8328 = x8328 + z8329 = x8329 + z8330 = x8330 + z8331 = x8331 + z8332 = x8332 + z8333 = x8333 + z8334 = x8334 + z8335 = x8335 + z8336 = x8336 + z8337 = x8337 + z8338 = x8338 + z8339 = x8339 + z8340 = x8340 + z8341 = x8341 + z8342 = x8342 + z8343 = x8343 + z8344 = x8344 + z8345 = x8345 + z8346 = x8346 + z8347 = x8347 + z8348 = x8348 + z8349 = x8349 + z8350 = x8350 + z8351 = x8351 + z8352 = x8352 + z8353 = x8353 + z8354 = x8354 + z8355 = x8355 + z8356 = x8356 + z8357 = x8357 + z8358 = x8358 + z8359 = x8359 + z8360 = x8360 + z8361 = x8361 + z8362 = x8362 + z8363 = x8363 + z8364 = x8364 + z8365 = x8365 + z8366 = x8366 + z8367 = x8367 + z8368 = x8368 + z8369 = x8369 + z8370 = x8370 + z8371 = x8371 + z8372 = x8372 + z8373 = x8373 + z8374 = x8374 + z8375 = x8375 + z8376 = x8376 + z8377 = x8377 + z8378 = x8378 + z8379 = x8379 + z8380 = x8380 + z8381 = x8381 + z8382 = x8382 + z8383 = x8383 + z8384 = x8384 + z8385 = x8385 + z8386 = x8386 + z8387 = x8387 + z8388 = x8388 + z8389 = x8389 + z8390 = x8390 + z8391 = x8391 + z8392 = x8392 + z8393 = x8393 + z8394 = x8394 + z8395 = x8395 + z8396 = x8396 + z8397 = x8397 + z8398 = x8398 + z8399 = x8399 + z8400 = x8400 + z8401 = x8401 + z8402 = x8402 + z8403 = x8403 + z8404 = x8404 + z8405 = x8405 + z8406 = x8406 + z8407 = x8407 + z8408 = x8408 + z8409 = x8409 + z8410 = x8410 + z8411 = x8411 + z8412 = x8412 + z8413 = x8413 + z8414 = x8414 + z8415 = x8415 + z8416 = x8416 + z8417 = x8417 + z8418 = x8418 + z8419 = x8419 + z8420 = x8420 + z8421 = x8421 + z8422 = x8422 + z8423 = x8423 + z8424 = x8424 + z8425 = x8425 + z8426 = x8426 + z8427 = x8427 + z8428 = x8428 + z8429 = x8429 + z8430 = x8430 + z8431 = x8431 + z8432 = x8432 + z8433 = x8433 + z8434 = x8434 + z8435 = x8435 + z8436 = x8436 + z8437 = x8437 + z8438 = x8438 + z8439 = x8439 + z8440 = x8440 + z8441 = x8441 + z8442 = x8442 + z8443 = x8443 + z8444 = x8444 + z8445 = x8445 + z8446 = x8446 + z8447 = x8447 + z8448 = x8448 + z8449 = x8449 + z8450 = x8450 + z8451 = x8451 + z8452 = x8452 + z8453 = x8453 + z8454 = x8454 + z8455 = x8455 + z8456 = x8456 + z8457 = x8457 + z8458 = x8458 + z8459 = x8459 + z8460 = x8460 + z8461 = x8461 + z8462 = x8462 + z8463 = x8463 + z8464 = x8464 + z8465 = x8465 + z8466 = x8466 + z8467 = x8467 + z8468 = x8468 + z8469 = x8469 + z8470 = x8470 + z8471 = x8471 + z8472 = x8472 + z8473 = x8473 + z8474 = x8474 + z8475 = x8475 + z8476 = x8476 + z8477 = x8477 + z8478 = x8478 + z8479 = x8479 + z8480 = x8480 + z8481 = x8481 + z8482 = x8482 + z8483 = x8483 + z8484 = x8484 + z8485 = x8485 + z8486 = x8486 + z8487 = x8487 + z8488 = x8488 + z8489 = x8489 + z8490 = x8490 + z8491 = x8491 + z8492 = x8492 + z8493 = x8493 + z8494 = x8494 + z8495 = x8495 + z8496 = x8496 + z8497 = x8497 + z8498 = x8498 + z8499 = x8499 + z8500 = x8500 + z8501 = x8501 + z8502 = x8502 + z8503 = x8503 + z8504 = x8504 + z8505 = x8505 + z8506 = x8506 + z8507 = x8507 + z8508 = x8508 + z8509 = x8509 + z8510 = x8510 + z8511 = x8511 + z8512 = x8512 + z8513 = x8513 + z8514 = x8514 + z8515 = x8515 + z8516 = x8516 + z8517 = x8517 + z8518 = x8518 + z8519 = x8519 + z8520 = x8520 + z8521 = x8521 + z8522 = x8522 + z8523 = x8523 + z8524 = x8524 + z8525 = x8525 + z8526 = x8526 + z8527 = x8527 + z8528 = x8528 + z8529 = x8529 + z8530 = x8530 + z8531 = x8531 + z8532 = x8532 + z8533 = x8533 + z8534 = x8534 + z8535 = x8535 + z8536 = x8536 + z8537 = x8537 + z8538 = x8538 + z8539 = x8539 + z8540 = x8540 + z8541 = x8541 + z8542 = x8542 + z8543 = x8543 + z8544 = x8544 + z8545 = x8545 + z8546 = x8546 + z8547 = x8547 + z8548 = x8548 + z8549 = x8549 + z8550 = x8550 + z8551 = x8551 + z8552 = x8552 + z8553 = x8553 + z8554 = x8554 + z8555 = x8555 + z8556 = x8556 + z8557 = x8557 + z8558 = x8558 + z8559 = x8559 + z8560 = x8560 + z8561 = x8561 + z8562 = x8562 + z8563 = x8563 + z8564 = x8564 + z8565 = x8565 + z8566 = x8566 + z8567 = x8567 + z8568 = x8568 + z8569 = x8569 + z8570 = x8570 + z8571 = x8571 + z8572 = x8572 + z8573 = x8573 + z8574 = x8574 + z8575 = x8575 + z8576 = x8576 + z8577 = x8577 + z8578 = x8578 + z8579 = x8579 + z8580 = x8580 + z8581 = x8581 + z8582 = x8582 + z8583 = x8583 + z8584 = x8584 + z8585 = x8585 + z8586 = x8586 + z8587 = x8587 + z8588 = x8588 + z8589 = x8589 + z8590 = x8590 + z8591 = x8591 + z8592 = x8592 + z8593 = x8593 + z8594 = x8594 + z8595 = x8595 + z8596 = x8596 + z8597 = x8597 + z8598 = x8598 + z8599 = x8599 + z8600 = x8600 + z8601 = x8601 + z8602 = x8602 + z8603 = x8603 + z8604 = x8604 + z8605 = x8605 + z8606 = x8606 + z8607 = x8607 + z8608 = x8608 + z8609 = x8609 + z8610 = x8610 + z8611 = x8611 + z8612 = x8612 + z8613 = x8613 + z8614 = x8614 + z8615 = x8615 + z8616 = x8616 + z8617 = x8617 + z8618 = x8618 + z8619 = x8619 + z8620 = x8620 + z8621 = x8621 + z8622 = x8622 + z8623 = x8623 + z8624 = x8624 + z8625 = x8625 + z8626 = x8626 + z8627 = x8627 + z8628 = x8628 + z8629 = x8629 + z8630 = x8630 + z8631 = x8631 + z8632 = x8632 + z8633 = x8633 + z8634 = x8634 + z8635 = x8635 + z8636 = x8636 + z8637 = x8637 + z8638 = x8638 + z8639 = x8639 + z8640 = x8640 + z8641 = x8641 + z8642 = x8642 + z8643 = x8643 + z8644 = x8644 + z8645 = x8645 + z8646 = x8646 + z8647 = x8647 + z8648 = x8648 + z8649 = x8649 + z8650 = x8650 + z8651 = x8651 + z8652 = x8652 + z8653 = x8653 + z8654 = x8654 + z8655 = x8655 + z8656 = x8656 + z8657 = x8657 + z8658 = x8658 + z8659 = x8659 + z8660 = x8660 + z8661 = x8661 + z8662 = x8662 + z8663 = x8663 + z8664 = x8664 + z8665 = x8665 + z8666 = x8666 + z8667 = x8667 + z8668 = x8668 + z8669 = x8669 + z8670 = x8670 + z8671 = x8671 + z8672 = x8672 + z8673 = x8673 + z8674 = x8674 + z8675 = x8675 + z8676 = x8676 + z8677 = x8677 + z8678 = x8678 + z8679 = x8679 + z8680 = x8680 + z8681 = x8681 + z8682 = x8682 + z8683 = x8683 + z8684 = x8684 + z8685 = x8685 + z8686 = x8686 + z8687 = x8687 + z8688 = x8688 + z8689 = x8689 + z8690 = x8690 + z8691 = x8691 + z8692 = x8692 + z8693 = x8693 + z8694 = x8694 + z8695 = x8695 + z8696 = x8696 + z8697 = x8697 + z8698 = x8698 + z8699 = x8699 + z8700 = x8700 + z8701 = x8701 + z8702 = x8702 + z8703 = x8703 + z8704 = x8704 + z8705 = x8705 + z8706 = x8706 + z8707 = x8707 + z8708 = x8708 + z8709 = x8709 + z8710 = x8710 + z8711 = x8711 + z8712 = x8712 + z8713 = x8713 + z8714 = x8714 + z8715 = x8715 + z8716 = x8716 + z8717 = x8717 + z8718 = x8718 + z8719 = x8719 + z8720 = x8720 + z8721 = x8721 + z8722 = x8722 + z8723 = x8723 + z8724 = x8724 + z8725 = x8725 + z8726 = x8726 + z8727 = x8727 + z8728 = x8728 + z8729 = x8729 + z8730 = x8730 + z8731 = x8731 + z8732 = x8732 + z8733 = x8733 + z8734 = x8734 + z8735 = x8735 + z8736 = x8736 + z8737 = x8737 + z8738 = x8738 + z8739 = x8739 + z8740 = x8740 + z8741 = x8741 + z8742 = x8742 + z8743 = x8743 + z8744 = x8744 + z8745 = x8745 + z8746 = x8746 + z8747 = x8747 + z8748 = x8748 + z8749 = x8749 + z8750 = x8750 + z8751 = x8751 + z8752 = x8752 + z8753 = x8753 + z8754 = x8754 + z8755 = x8755 + z8756 = x8756 + z8757 = x8757 + z8758 = x8758 + z8759 = x8759 + z8760 = x8760 + z8761 = x8761 + z8762 = x8762 + z8763 = x8763 + z8764 = x8764 + z8765 = x8765 + z8766 = x8766 + z8767 = x8767 + z8768 = x8768 + z8769 = x8769 + z8770 = x8770 + z8771 = x8771 + z8772 = x8772 + z8773 = x8773 + z8774 = x8774 + z8775 = x8775 + z8776 = x8776 + z8777 = x8777 + z8778 = x8778 + z8779 = x8779 + z8780 = x8780 + z8781 = x8781 + z8782 = x8782 + z8783 = x8783 + z8784 = x8784 + z8785 = x8785 + z8786 = x8786 + z8787 = x8787 + z8788 = x8788 + z8789 = x8789 + z8790 = x8790 + z8791 = x8791 + z8792 = x8792 + z8793 = x8793 + z8794 = x8794 + z8795 = x8795 + z8796 = x8796 + z8797 = x8797 + z8798 = x8798 + z8799 = x8799 + z8800 = x8800 + z8801 = x8801 + z8802 = x8802 + z8803 = x8803 + z8804 = x8804 + z8805 = x8805 + z8806 = x8806 + z8807 = x8807 + z8808 = x8808 + z8809 = x8809 + z8810 = x8810 + z8811 = x8811 + z8812 = x8812 + z8813 = x8813 + z8814 = x8814 + z8815 = x8815 + z8816 = x8816 + z8817 = x8817 + z8818 = x8818 + z8819 = x8819 + z8820 = x8820 + z8821 = x8821 + z8822 = x8822 + z8823 = x8823 + z8824 = x8824 + z8825 = x8825 + z8826 = x8826 + z8827 = x8827 + z8828 = x8828 + z8829 = x8829 + z8830 = x8830 + z8831 = x8831 + z8832 = x8832 + z8833 = x8833 + z8834 = x8834 + z8835 = x8835 + z8836 = x8836 + z8837 = x8837 + z8838 = x8838 + z8839 = x8839 + z8840 = x8840 + z8841 = x8841 + z8842 = x8842 + z8843 = x8843 + z8844 = x8844 + z8845 = x8845 + z8846 = x8846 + z8847 = x8847 + z8848 = x8848 + z8849 = x8849 + z8850 = x8850 + z8851 = x8851 + z8852 = x8852 + z8853 = x8853 + z8854 = x8854 + z8855 = x8855 + z8856 = x8856 + z8857 = x8857 + z8858 = x8858 + z8859 = x8859 + z8860 = x8860 + z8861 = x8861 + z8862 = x8862 + z8863 = x8863 + z8864 = x8864 + z8865 = x8865 + z8866 = x8866 + z8867 = x8867 + z8868 = x8868 + z8869 = x8869 + z8870 = x8870 + z8871 = x8871 + z8872 = x8872 + z8873 = x8873 + z8874 = x8874 + z8875 = x8875 + z8876 = x8876 + z8877 = x8877 + z8878 = x8878 + z8879 = x8879 + z8880 = x8880 + z8881 = x8881 + z8882 = x8882 + z8883 = x8883 + z8884 = x8884 + z8885 = x8885 + z8886 = x8886 + z8887 = x8887 + z8888 = x8888 + z8889 = x8889 + z8890 = x8890 + z8891 = x8891 + z8892 = x8892 + z8893 = x8893 + z8894 = x8894 + z8895 = x8895 + z8896 = x8896 + z8897 = x8897 + z8898 = x8898 + z8899 = x8899 + z8900 = x8900 + z8901 = x8901 + z8902 = x8902 + z8903 = x8903 + z8904 = x8904 + z8905 = x8905 + z8906 = x8906 + z8907 = x8907 + z8908 = x8908 + z8909 = x8909 + z8910 = x8910 + z8911 = x8911 + z8912 = x8912 + z8913 = x8913 + z8914 = x8914 + z8915 = x8915 + z8916 = x8916 + z8917 = x8917 + z8918 = x8918 + z8919 = x8919 + z8920 = x8920 + z8921 = x8921 + z8922 = x8922 + z8923 = x8923 + z8924 = x8924 + z8925 = x8925 + z8926 = x8926 + z8927 = x8927 + z8928 = x8928 + z8929 = x8929 + z8930 = x8930 + z8931 = x8931 + z8932 = x8932 + z8933 = x8933 + z8934 = x8934 + z8935 = x8935 + z8936 = x8936 + z8937 = x8937 + z8938 = x8938 + z8939 = x8939 + z8940 = x8940 + z8941 = x8941 + z8942 = x8942 + z8943 = x8943 + z8944 = x8944 + z8945 = x8945 + z8946 = x8946 + z8947 = x8947 + z8948 = x8948 + z8949 = x8949 + z8950 = x8950 + z8951 = x8951 + z8952 = x8952 + z8953 = x8953 + z8954 = x8954 + z8955 = x8955 + z8956 = x8956 + z8957 = x8957 + z8958 = x8958 + z8959 = x8959 + z8960 = x8960 + z8961 = x8961 + z8962 = x8962 + z8963 = x8963 + z8964 = x8964 + z8965 = x8965 + z8966 = x8966 + z8967 = x8967 + z8968 = x8968 + z8969 = x8969 + z8970 = x8970 + z8971 = x8971 + z8972 = x8972 + z8973 = x8973 + z8974 = x8974 + z8975 = x8975 + z8976 = x8976 + z8977 = x8977 + z8978 = x8978 + z8979 = x8979 + z8980 = x8980 + z8981 = x8981 + z8982 = x8982 + z8983 = x8983 + z8984 = x8984 + z8985 = x8985 + z8986 = x8986 + z8987 = x8987 + z8988 = x8988 + z8989 = x8989 + z8990 = x8990 + z8991 = x8991 + z8992 = x8992 + z8993 = x8993 + z8994 = x8994 + z8995 = x8995 + z8996 = x8996 + z8997 = x8997 + z8998 = x8998 + z8999 = x8999 + z9000 = x9000 + z9001 = x9001 + z9002 = x9002 + z9003 = x9003 + z9004 = x9004 + z9005 = x9005 + z9006 = x9006 + z9007 = x9007 + z9008 = x9008 + z9009 = x9009 + z9010 = x9010 + z9011 = x9011 + z9012 = x9012 + z9013 = x9013 + z9014 = x9014 + z9015 = x9015 + z9016 = x9016 + z9017 = x9017 + z9018 = x9018 + z9019 = x9019 + z9020 = x9020 + z9021 = x9021 + z9022 = x9022 + z9023 = x9023 + z9024 = x9024 + z9025 = x9025 + z9026 = x9026 + z9027 = x9027 + z9028 = x9028 + z9029 = x9029 + z9030 = x9030 + z9031 = x9031 + z9032 = x9032 + z9033 = x9033 + z9034 = x9034 + z9035 = x9035 + z9036 = x9036 + z9037 = x9037 + z9038 = x9038 + z9039 = x9039 + z9040 = x9040 + z9041 = x9041 + z9042 = x9042 + z9043 = x9043 + z9044 = x9044 + z9045 = x9045 + z9046 = x9046 + z9047 = x9047 + z9048 = x9048 + z9049 = x9049 + z9050 = x9050 + z9051 = x9051 + z9052 = x9052 + z9053 = x9053 + z9054 = x9054 + z9055 = x9055 + z9056 = x9056 + z9057 = x9057 + z9058 = x9058 + z9059 = x9059 + z9060 = x9060 + z9061 = x9061 + z9062 = x9062 + z9063 = x9063 + z9064 = x9064 + z9065 = x9065 + z9066 = x9066 + z9067 = x9067 + z9068 = x9068 + z9069 = x9069 + z9070 = x9070 + z9071 = x9071 + z9072 = x9072 + z9073 = x9073 + z9074 = x9074 + z9075 = x9075 + z9076 = x9076 + z9077 = x9077 + z9078 = x9078 + z9079 = x9079 + z9080 = x9080 + z9081 = x9081 + z9082 = x9082 + z9083 = x9083 + z9084 = x9084 + z9085 = x9085 + z9086 = x9086 + z9087 = x9087 + z9088 = x9088 + z9089 = x9089 + z9090 = x9090 + z9091 = x9091 + z9092 = x9092 + z9093 = x9093 + z9094 = x9094 + z9095 = x9095 + z9096 = x9096 + z9097 = x9097 + z9098 = x9098 + z9099 = x9099 + z9100 = x9100 + z9101 = x9101 + z9102 = x9102 + z9103 = x9103 + z9104 = x9104 + z9105 = x9105 + z9106 = x9106 + z9107 = x9107 + z9108 = x9108 + z9109 = x9109 + z9110 = x9110 + z9111 = x9111 + z9112 = x9112 + z9113 = x9113 + z9114 = x9114 + z9115 = x9115 + z9116 = x9116 + z9117 = x9117 + z9118 = x9118 + z9119 = x9119 + z9120 = x9120 + z9121 = x9121 + z9122 = x9122 + z9123 = x9123 + z9124 = x9124 + z9125 = x9125 + z9126 = x9126 + z9127 = x9127 + z9128 = x9128 + z9129 = x9129 + z9130 = x9130 + z9131 = x9131 + z9132 = x9132 + z9133 = x9133 + z9134 = x9134 + z9135 = x9135 + z9136 = x9136 + z9137 = x9137 + z9138 = x9138 + z9139 = x9139 + z9140 = x9140 + z9141 = x9141 + z9142 = x9142 + z9143 = x9143 + z9144 = x9144 + z9145 = x9145 + z9146 = x9146 + z9147 = x9147 + z9148 = x9148 + z9149 = x9149 + z9150 = x9150 + z9151 = x9151 + z9152 = x9152 + z9153 = x9153 + z9154 = x9154 + z9155 = x9155 + z9156 = x9156 + z9157 = x9157 + z9158 = x9158 + z9159 = x9159 + z9160 = x9160 + z9161 = x9161 + z9162 = x9162 + z9163 = x9163 + z9164 = x9164 + z9165 = x9165 + z9166 = x9166 + z9167 = x9167 + z9168 = x9168 + z9169 = x9169 + z9170 = x9170 + z9171 = x9171 + z9172 = x9172 + z9173 = x9173 + z9174 = x9174 + z9175 = x9175 + z9176 = x9176 + z9177 = x9177 + z9178 = x9178 + z9179 = x9179 + z9180 = x9180 + z9181 = x9181 + z9182 = x9182 + z9183 = x9183 + z9184 = x9184 + z9185 = x9185 + z9186 = x9186 + z9187 = x9187 + z9188 = x9188 + z9189 = x9189 + z9190 = x9190 + z9191 = x9191 + z9192 = x9192 + z9193 = x9193 + z9194 = x9194 + z9195 = x9195 + z9196 = x9196 + z9197 = x9197 + z9198 = x9198 + z9199 = x9199 + z9200 = x9200 + z9201 = x9201 + z9202 = x9202 + z9203 = x9203 + z9204 = x9204 + z9205 = x9205 + z9206 = x9206 + z9207 = x9207 + z9208 = x9208 + z9209 = x9209 + z9210 = x9210 + z9211 = x9211 + z9212 = x9212 + z9213 = x9213 + z9214 = x9214 + z9215 = x9215 + z9216 = x9216 + z9217 = x9217 + z9218 = x9218 + z9219 = x9219 + z9220 = x9220 + z9221 = x9221 + z9222 = x9222 + z9223 = x9223 + z9224 = x9224 + z9225 = x9225 + z9226 = x9226 + z9227 = x9227 + z9228 = x9228 + z9229 = x9229 + z9230 = x9230 + z9231 = x9231 + z9232 = x9232 + z9233 = x9233 + z9234 = x9234 + z9235 = x9235 + z9236 = x9236 + z9237 = x9237 + z9238 = x9238 + z9239 = x9239 + z9240 = x9240 + z9241 = x9241 + z9242 = x9242 + z9243 = x9243 + z9244 = x9244 + z9245 = x9245 + z9246 = x9246 + z9247 = x9247 + z9248 = x9248 + z9249 = x9249 + z9250 = x9250 + z9251 = x9251 + z9252 = x9252 + z9253 = x9253 + z9254 = x9254 + z9255 = x9255 + z9256 = x9256 + z9257 = x9257 + z9258 = x9258 + z9259 = x9259 + z9260 = x9260 + z9261 = x9261 + z9262 = x9262 + z9263 = x9263 + z9264 = x9264 + z9265 = x9265 + z9266 = x9266 + z9267 = x9267 + z9268 = x9268 + z9269 = x9269 + z9270 = x9270 + z9271 = x9271 + z9272 = x9272 + z9273 = x9273 + z9274 = x9274 + z9275 = x9275 + z9276 = x9276 + z9277 = x9277 + z9278 = x9278 + z9279 = x9279 + z9280 = x9280 + z9281 = x9281 + z9282 = x9282 + z9283 = x9283 + z9284 = x9284 + z9285 = x9285 + z9286 = x9286 + z9287 = x9287 + z9288 = x9288 + z9289 = x9289 + z9290 = x9290 + z9291 = x9291 + z9292 = x9292 + z9293 = x9293 + z9294 = x9294 + z9295 = x9295 + z9296 = x9296 + z9297 = x9297 + z9298 = x9298 + z9299 = x9299 + z9300 = x9300 + z9301 = x9301 + z9302 = x9302 + z9303 = x9303 + z9304 = x9304 + z9305 = x9305 + z9306 = x9306 + z9307 = x9307 + z9308 = x9308 + z9309 = x9309 + z9310 = x9310 + z9311 = x9311 + z9312 = x9312 + z9313 = x9313 + z9314 = x9314 + z9315 = x9315 + z9316 = x9316 + z9317 = x9317 + z9318 = x9318 + z9319 = x9319 + z9320 = x9320 + z9321 = x9321 + z9322 = x9322 + z9323 = x9323 + z9324 = x9324 + z9325 = x9325 + z9326 = x9326 + z9327 = x9327 + z9328 = x9328 + z9329 = x9329 + z9330 = x9330 + z9331 = x9331 + z9332 = x9332 + z9333 = x9333 + z9334 = x9334 + z9335 = x9335 + z9336 = x9336 + z9337 = x9337 + z9338 = x9338 + z9339 = x9339 + z9340 = x9340 + z9341 = x9341 + z9342 = x9342 + z9343 = x9343 + z9344 = x9344 + z9345 = x9345 + z9346 = x9346 + z9347 = x9347 + z9348 = x9348 + z9349 = x9349 + z9350 = x9350 + z9351 = x9351 + z9352 = x9352 + z9353 = x9353 + z9354 = x9354 + z9355 = x9355 + z9356 = x9356 + z9357 = x9357 + z9358 = x9358 + z9359 = x9359 + z9360 = x9360 + z9361 = x9361 + z9362 = x9362 + z9363 = x9363 + z9364 = x9364 + z9365 = x9365 + z9366 = x9366 + z9367 = x9367 + z9368 = x9368 + z9369 = x9369 + z9370 = x9370 + z9371 = x9371 + z9372 = x9372 + z9373 = x9373 + z9374 = x9374 + z9375 = x9375 + z9376 = x9376 + z9377 = x9377 + z9378 = x9378 + z9379 = x9379 + z9380 = x9380 + z9381 = x9381 + z9382 = x9382 + z9383 = x9383 + z9384 = x9384 + z9385 = x9385 + z9386 = x9386 + z9387 = x9387 + z9388 = x9388 + z9389 = x9389 + z9390 = x9390 + z9391 = x9391 + z9392 = x9392 + z9393 = x9393 + z9394 = x9394 + z9395 = x9395 + z9396 = x9396 + z9397 = x9397 + z9398 = x9398 + z9399 = x9399 + z9400 = x9400 + z9401 = x9401 + z9402 = x9402 + z9403 = x9403 + z9404 = x9404 + z9405 = x9405 + z9406 = x9406 + z9407 = x9407 + z9408 = x9408 + z9409 = x9409 + z9410 = x9410 + z9411 = x9411 + z9412 = x9412 + z9413 = x9413 + z9414 = x9414 + z9415 = x9415 + z9416 = x9416 + z9417 = x9417 + z9418 = x9418 + z9419 = x9419 + z9420 = x9420 + z9421 = x9421 + z9422 = x9422 + z9423 = x9423 + z9424 = x9424 + z9425 = x9425 + z9426 = x9426 + z9427 = x9427 + z9428 = x9428 + z9429 = x9429 + z9430 = x9430 + z9431 = x9431 + z9432 = x9432 + z9433 = x9433 + z9434 = x9434 + z9435 = x9435 + z9436 = x9436 + z9437 = x9437 + z9438 = x9438 + z9439 = x9439 + z9440 = x9440 + z9441 = x9441 + z9442 = x9442 + z9443 = x9443 + z9444 = x9444 + z9445 = x9445 + z9446 = x9446 + z9447 = x9447 + z9448 = x9448 + z9449 = x9449 + z9450 = x9450 + z9451 = x9451 + z9452 = x9452 + z9453 = x9453 + z9454 = x9454 + z9455 = x9455 + z9456 = x9456 + z9457 = x9457 + z9458 = x9458 + z9459 = x9459 + z9460 = x9460 + z9461 = x9461 + z9462 = x9462 + z9463 = x9463 + z9464 = x9464 + z9465 = x9465 + z9466 = x9466 + z9467 = x9467 + z9468 = x9468 + z9469 = x9469 + z9470 = x9470 + z9471 = x9471 + z9472 = x9472 + z9473 = x9473 + z9474 = x9474 + z9475 = x9475 + z9476 = x9476 + z9477 = x9477 + z9478 = x9478 + z9479 = x9479 + z9480 = x9480 + z9481 = x9481 + z9482 = x9482 + z9483 = x9483 + z9484 = x9484 + z9485 = x9485 + z9486 = x9486 + z9487 = x9487 + z9488 = x9488 + z9489 = x9489 + z9490 = x9490 + z9491 = x9491 + z9492 = x9492 + z9493 = x9493 + z9494 = x9494 + z9495 = x9495 + z9496 = x9496 + z9497 = x9497 + z9498 = x9498 + z9499 = x9499 + z9500 = x9500 + z9501 = x9501 + z9502 = x9502 + z9503 = x9503 + z9504 = x9504 + z9505 = x9505 + z9506 = x9506 + z9507 = x9507 + z9508 = x9508 + z9509 = x9509 + z9510 = x9510 + z9511 = x9511 + z9512 = x9512 + z9513 = x9513 + z9514 = x9514 + z9515 = x9515 + z9516 = x9516 + z9517 = x9517 + z9518 = x9518 + z9519 = x9519 + z9520 = x9520 + z9521 = x9521 + z9522 = x9522 + z9523 = x9523 + z9524 = x9524 + z9525 = x9525 + z9526 = x9526 + z9527 = x9527 + z9528 = x9528 + z9529 = x9529 + z9530 = x9530 + z9531 = x9531 + z9532 = x9532 + z9533 = x9533 + z9534 = x9534 + z9535 = x9535 + z9536 = x9536 + z9537 = x9537 + z9538 = x9538 + z9539 = x9539 + z9540 = x9540 + z9541 = x9541 + z9542 = x9542 + z9543 = x9543 + z9544 = x9544 + z9545 = x9545 + z9546 = x9546 + z9547 = x9547 + z9548 = x9548 + z9549 = x9549 + z9550 = x9550 + z9551 = x9551 + z9552 = x9552 + z9553 = x9553 + z9554 = x9554 + z9555 = x9555 + z9556 = x9556 + z9557 = x9557 + z9558 = x9558 + z9559 = x9559 + z9560 = x9560 + z9561 = x9561 + z9562 = x9562 + z9563 = x9563 + z9564 = x9564 + z9565 = x9565 + z9566 = x9566 + z9567 = x9567 + z9568 = x9568 + z9569 = x9569 + z9570 = x9570 + z9571 = x9571 + z9572 = x9572 + z9573 = x9573 + z9574 = x9574 + z9575 = x9575 + z9576 = x9576 + z9577 = x9577 + z9578 = x9578 + z9579 = x9579 + z9580 = x9580 + z9581 = x9581 + z9582 = x9582 + z9583 = x9583 + z9584 = x9584 + z9585 = x9585 + z9586 = x9586 + z9587 = x9587 + z9588 = x9588 + z9589 = x9589 + z9590 = x9590 + z9591 = x9591 + z9592 = x9592 + z9593 = x9593 + z9594 = x9594 + z9595 = x9595 + z9596 = x9596 + z9597 = x9597 + z9598 = x9598 + z9599 = x9599 + z9600 = x9600 + z9601 = x9601 + z9602 = x9602 + z9603 = x9603 + z9604 = x9604 + z9605 = x9605 + z9606 = x9606 + z9607 = x9607 + z9608 = x9608 + z9609 = x9609 + z9610 = x9610 + z9611 = x9611 + z9612 = x9612 + z9613 = x9613 + z9614 = x9614 + z9615 = x9615 + z9616 = x9616 + z9617 = x9617 + z9618 = x9618 + z9619 = x9619 + z9620 = x9620 + z9621 = x9621 + z9622 = x9622 + z9623 = x9623 + z9624 = x9624 + z9625 = x9625 + z9626 = x9626 + z9627 = x9627 + z9628 = x9628 + z9629 = x9629 + z9630 = x9630 + z9631 = x9631 + z9632 = x9632 + z9633 = x9633 + z9634 = x9634 + z9635 = x9635 + z9636 = x9636 + z9637 = x9637 + z9638 = x9638 + z9639 = x9639 + z9640 = x9640 + z9641 = x9641 + z9642 = x9642 + z9643 = x9643 + z9644 = x9644 + z9645 = x9645 + z9646 = x9646 + z9647 = x9647 + z9648 = x9648 + z9649 = x9649 + z9650 = x9650 + z9651 = x9651 + z9652 = x9652 + z9653 = x9653 + z9654 = x9654 + z9655 = x9655 + z9656 = x9656 + z9657 = x9657 + z9658 = x9658 + z9659 = x9659 + z9660 = x9660 + z9661 = x9661 + z9662 = x9662 + z9663 = x9663 + z9664 = x9664 + z9665 = x9665 + z9666 = x9666 + z9667 = x9667 + z9668 = x9668 + z9669 = x9669 + z9670 = x9670 + z9671 = x9671 + z9672 = x9672 + z9673 = x9673 + z9674 = x9674 + z9675 = x9675 + z9676 = x9676 + z9677 = x9677 + z9678 = x9678 + z9679 = x9679 + z9680 = x9680 + z9681 = x9681 + z9682 = x9682 + z9683 = x9683 + z9684 = x9684 + z9685 = x9685 + z9686 = x9686 + z9687 = x9687 + z9688 = x9688 + z9689 = x9689 + z9690 = x9690 + z9691 = x9691 + z9692 = x9692 + z9693 = x9693 + z9694 = x9694 + z9695 = x9695 + z9696 = x9696 + z9697 = x9697 + z9698 = x9698 + z9699 = x9699 + z9700 = x9700 + z9701 = x9701 + z9702 = x9702 + z9703 = x9703 + z9704 = x9704 + z9705 = x9705 + z9706 = x9706 + z9707 = x9707 + z9708 = x9708 + z9709 = x9709 + z9710 = x9710 + z9711 = x9711 + z9712 = x9712 + z9713 = x9713 + z9714 = x9714 + z9715 = x9715 + z9716 = x9716 + z9717 = x9717 + z9718 = x9718 + z9719 = x9719 + z9720 = x9720 + z9721 = x9721 + z9722 = x9722 + z9723 = x9723 + z9724 = x9724 + z9725 = x9725 + z9726 = x9726 + z9727 = x9727 + z9728 = x9728 + z9729 = x9729 + z9730 = x9730 + z9731 = x9731 + z9732 = x9732 + z9733 = x9733 + z9734 = x9734 + z9735 = x9735 + z9736 = x9736 + z9737 = x9737 + z9738 = x9738 + z9739 = x9739 + z9740 = x9740 + z9741 = x9741 + z9742 = x9742 + z9743 = x9743 + z9744 = x9744 + z9745 = x9745 + z9746 = x9746 + z9747 = x9747 + z9748 = x9748 + z9749 = x9749 + z9750 = x9750 + z9751 = x9751 + z9752 = x9752 + z9753 = x9753 + z9754 = x9754 + z9755 = x9755 + z9756 = x9756 + z9757 = x9757 + z9758 = x9758 + z9759 = x9759 + z9760 = x9760 + z9761 = x9761 + z9762 = x9762 + z9763 = x9763 + z9764 = x9764 + z9765 = x9765 + z9766 = x9766 + z9767 = x9767 + z9768 = x9768 + z9769 = x9769 + z9770 = x9770 + z9771 = x9771 + z9772 = x9772 + z9773 = x9773 + z9774 = x9774 + z9775 = x9775 + z9776 = x9776 + z9777 = x9777 + z9778 = x9778 + z9779 = x9779 + z9780 = x9780 + z9781 = x9781 + z9782 = x9782 + z9783 = x9783 + z9784 = x9784 + z9785 = x9785 + z9786 = x9786 + z9787 = x9787 + z9788 = x9788 + z9789 = x9789 + z9790 = x9790 + z9791 = x9791 + z9792 = x9792 + z9793 = x9793 + z9794 = x9794 + z9795 = x9795 + z9796 = x9796 + z9797 = x9797 + z9798 = x9798 + z9799 = x9799 + z9800 = x9800 + z9801 = x9801 + z9802 = x9802 + z9803 = x9803 + z9804 = x9804 + z9805 = x9805 + z9806 = x9806 + z9807 = x9807 + z9808 = x9808 + z9809 = x9809 + z9810 = x9810 + z9811 = x9811 + z9812 = x9812 + z9813 = x9813 + z9814 = x9814 + z9815 = x9815 + z9816 = x9816 + z9817 = x9817 + z9818 = x9818 + z9819 = x9819 + z9820 = x9820 + z9821 = x9821 + z9822 = x9822 + z9823 = x9823 + z9824 = x9824 + z9825 = x9825 + z9826 = x9826 + z9827 = x9827 + z9828 = x9828 + z9829 = x9829 + z9830 = x9830 + z9831 = x9831 + z9832 = x9832 + z9833 = x9833 + z9834 = x9834 + z9835 = x9835 + z9836 = x9836 + z9837 = x9837 + z9838 = x9838 + z9839 = x9839 + z9840 = x9840 + z9841 = x9841 + z9842 = x9842 + z9843 = x9843 + z9844 = x9844 + z9845 = x9845 + z9846 = x9846 + z9847 = x9847 + z9848 = x9848 + z9849 = x9849 + z9850 = x9850 + z9851 = x9851 + z9852 = x9852 + z9853 = x9853 + z9854 = x9854 + z9855 = x9855 + z9856 = x9856 + z9857 = x9857 + z9858 = x9858 + z9859 = x9859 + z9860 = x9860 + z9861 = x9861 + z9862 = x9862 + z9863 = x9863 + z9864 = x9864 + z9865 = x9865 + z9866 = x9866 + z9867 = x9867 + z9868 = x9868 + z9869 = x9869 + z9870 = x9870 + z9871 = x9871 + z9872 = x9872 + z9873 = x9873 + z9874 = x9874 + z9875 = x9875 + z9876 = x9876 + z9877 = x9877 + z9878 = x9878 + z9879 = x9879 + z9880 = x9880 + z9881 = x9881 + z9882 = x9882 + z9883 = x9883 + z9884 = x9884 + z9885 = x9885 + z9886 = x9886 + z9887 = x9887 + z9888 = x9888 + z9889 = x9889 + z9890 = x9890 + z9891 = x9891 + z9892 = x9892 + z9893 = x9893 + z9894 = x9894 + z9895 = x9895 + z9896 = x9896 + z9897 = x9897 + z9898 = x9898 + z9899 = x9899 + z9900 = x9900 + z9901 = x9901 + z9902 = x9902 + z9903 = x9903 + z9904 = x9904 + z9905 = x9905 + z9906 = x9906 + z9907 = x9907 + z9908 = x9908 + z9909 = x9909 + z9910 = x9910 + z9911 = x9911 + z9912 = x9912 + z9913 = x9913 + z9914 = x9914 + z9915 = x9915 + z9916 = x9916 + z9917 = x9917 + z9918 = x9918 + z9919 = x9919 + z9920 = x9920 + z9921 = x9921 + z9922 = x9922 + z9923 = x9923 + z9924 = x9924 + z9925 = x9925 + z9926 = x9926 + z9927 = x9927 + z9928 = x9928 + z9929 = x9929 + z9930 = x9930 + z9931 = x9931 + z9932 = x9932 + z9933 = x9933 + z9934 = x9934 + z9935 = x9935 + z9936 = x9936 + z9937 = x9937 + z9938 = x9938 + z9939 = x9939 + z9940 = x9940 + z9941 = x9941 + z9942 = x9942 + z9943 = x9943 + z9944 = x9944 + z9945 = x9945 + z9946 = x9946 + z9947 = x9947 + z9948 = x9948 + z9949 = x9949 + z9950 = x9950 + z9951 = x9951 + z9952 = x9952 + z9953 = x9953 + z9954 = x9954 + z9955 = x9955 + z9956 = x9956 + z9957 = x9957 + z9958 = x9958 + z9959 = x9959 + z9960 = x9960 + z9961 = x9961 + z9962 = x9962 + z9963 = x9963 + z9964 = x9964 + z9965 = x9965 + z9966 = x9966 + z9967 = x9967 + z9968 = x9968 + z9969 = x9969 + z9970 = x9970 + z9971 = x9971 + z9972 = x9972 + z9973 = x9973 + z9974 = x9974 + z9975 = x9975 + z9976 = x9976 + z9977 = x9977 + z9978 = x9978 + z9979 = x9979 + z9980 = x9980 + z9981 = x9981 + z9982 = x9982 + z9983 = x9983 + z9984 = x9984 + z9985 = x9985 + z9986 = x9986 + z9987 = x9987 + z9988 = x9988 + z9989 = x9989 + z9990 = x9990 + z9991 = x9991 + z9992 = x9992 + z9993 = x9993 + z9994 = x9994 + z9995 = x9995 + z9996 = x9996 + z9997 = x9997 + z9998 = x9998 + z9999 = x9999 + z10000 = x10000 + z10001 = x10001 + z10002 = x10002 + z10003 = x10003 + z10004 = x10004 + z10005 = x10005 + z10006 = x10006 + z10007 = x10007 + z10008 = x10008 + z10009 = x10009 + z10010 = x10010 + z10011 = x10011 + z10012 = x10012 + z10013 = x10013 + z10014 = x10014 + z10015 = x10015 + z10016 = x10016 + z10017 = x10017 + z10018 = x10018 + z10019 = x10019 + z10020 = x10020 + z10021 = x10021 + z10022 = x10022 + z10023 = x10023 + z10024 = x10024 + z10025 = x10025 + z10026 = x10026 + z10027 = x10027 + z10028 = x10028 + z10029 = x10029 + z10030 = x10030 + z10031 = x10031 + z10032 = x10032 + z10033 = x10033 + z10034 = x10034 + z10035 = x10035 + z10036 = x10036 + z10037 = x10037 + z10038 = x10038 + z10039 = x10039 + z10040 = x10040 + z10041 = x10041 + z10042 = x10042 + z10043 = x10043 + z10044 = x10044 + z10045 = x10045 + z10046 = x10046 + z10047 = x10047 + z10048 = x10048 + z10049 = x10049 + z10050 = x10050 + z10051 = x10051 + z10052 = x10052 + z10053 = x10053 + z10054 = x10054 + z10055 = x10055 + z10056 = x10056 + z10057 = x10057 + z10058 = x10058 + z10059 = x10059 + z10060 = x10060 + z10061 = x10061 + z10062 = x10062 + z10063 = x10063 + z10064 = x10064 + z10065 = x10065 + z10066 = x10066 + z10067 = x10067 + z10068 = x10068 + z10069 = x10069 + z10070 = x10070 + z10071 = x10071 + z10072 = x10072 + z10073 = x10073 + z10074 = x10074 + z10075 = x10075 + z10076 = x10076 + z10077 = x10077 + z10078 = x10078 + z10079 = x10079 + z10080 = x10080 + z10081 = x10081 + z10082 = x10082 + z10083 = x10083 + z10084 = x10084 + z10085 = x10085 + z10086 = x10086 + z10087 = x10087 + z10088 = x10088 + z10089 = x10089 + z10090 = x10090 + z10091 = x10091 + z10092 = x10092 + z10093 = x10093 + z10094 = x10094 + z10095 = x10095 + z10096 = x10096 + z10097 = x10097 + z10098 = x10098 + z10099 = x10099 + z10100 = x10100 + z10101 = x10101 + z10102 = x10102 + z10103 = x10103 + z10104 = x10104 + z10105 = x10105 + z10106 = x10106 + z10107 = x10107 + z10108 = x10108 + z10109 = x10109 + z10110 = x10110 + z10111 = x10111 + z10112 = x10112 + z10113 = x10113 + z10114 = x10114 + z10115 = x10115 + z10116 = x10116 + z10117 = x10117 + z10118 = x10118 + z10119 = x10119 + z10120 = x10120 + z10121 = x10121 + z10122 = x10122 + z10123 = x10123 + z10124 = x10124 + z10125 = x10125 + z10126 = x10126 + z10127 = x10127 + z10128 = x10128 + z10129 = x10129 + z10130 = x10130 + z10131 = x10131 + z10132 = x10132 + z10133 = x10133 + z10134 = x10134 + z10135 = x10135 + z10136 = x10136 + z10137 = x10137 + z10138 = x10138 + z10139 = x10139 + z10140 = x10140 + z10141 = x10141 + z10142 = x10142 + z10143 = x10143 + z10144 = x10144 + z10145 = x10145 + z10146 = x10146 + z10147 = x10147 + z10148 = x10148 + z10149 = x10149 + z10150 = x10150 + z10151 = x10151 + z10152 = x10152 + z10153 = x10153 + z10154 = x10154 + z10155 = x10155 + z10156 = x10156 + z10157 = x10157 + z10158 = x10158 + z10159 = x10159 + z10160 = x10160 + z10161 = x10161 + z10162 = x10162 + z10163 = x10163 + z10164 = x10164 + z10165 = x10165 + z10166 = x10166 + z10167 = x10167 + z10168 = x10168 + z10169 = x10169 + z10170 = x10170 + z10171 = x10171 + z10172 = x10172 + z10173 = x10173 + z10174 = x10174 + z10175 = x10175 + z10176 = x10176 + z10177 = x10177 + z10178 = x10178 + z10179 = x10179 + z10180 = x10180 + z10181 = x10181 + z10182 = x10182 + z10183 = x10183 + z10184 = x10184 + z10185 = x10185 + z10186 = x10186 + z10187 = x10187 + z10188 = x10188 + z10189 = x10189 + z10190 = x10190 + z10191 = x10191 + z10192 = x10192 + z10193 = x10193 + z10194 = x10194 + z10195 = x10195 + z10196 = x10196 + z10197 = x10197 + z10198 = x10198 + z10199 = x10199 + z10200 = x10200 + z10201 = x10201 + z10202 = x10202 + z10203 = x10203 + z10204 = x10204 + z10205 = x10205 + z10206 = x10206 + z10207 = x10207 + z10208 = x10208 + z10209 = x10209 + z10210 = x10210 + z10211 = x10211 + z10212 = x10212 + z10213 = x10213 + z10214 = x10214 + z10215 = x10215 + z10216 = x10216 + z10217 = x10217 + z10218 = x10218 + z10219 = x10219 + z10220 = x10220 + z10221 = x10221 + z10222 = x10222 + z10223 = x10223 + z10224 = x10224 + z10225 = x10225 + z10226 = x10226 + z10227 = x10227 + z10228 = x10228 + z10229 = x10229 + z10230 = x10230 + z10231 = x10231 + z10232 = x10232 + z10233 = x10233 + z10234 = x10234 + z10235 = x10235 + z10236 = x10236 + z10237 = x10237 + z10238 = x10238 + z10239 = x10239 + z10240 = x10240 + z10241 = x10241 + z10242 = x10242 + z10243 = x10243 + z10244 = x10244 + z10245 = x10245 + z10246 = x10246 + z10247 = x10247 + z10248 = x10248 + z10249 = x10249 + z10250 = x10250 + z10251 = x10251 + z10252 = x10252 + z10253 = x10253 + z10254 = x10254 + z10255 = x10255 + z10256 = x10256 + z10257 = x10257 + z10258 = x10258 + z10259 = x10259 + z10260 = x10260 + z10261 = x10261 + z10262 = x10262 + z10263 = x10263 + z10264 = x10264 + z10265 = x10265 + z10266 = x10266 + z10267 = x10267 + z10268 = x10268 + z10269 = x10269 + z10270 = x10270 + z10271 = x10271 + z10272 = x10272 + z10273 = x10273 + z10274 = x10274 + z10275 = x10275 + z10276 = x10276 + z10277 = x10277 + z10278 = x10278 + z10279 = x10279 + z10280 = x10280 + z10281 = x10281 + z10282 = x10282 + z10283 = x10283 + z10284 = x10284 + z10285 = x10285 + z10286 = x10286 + z10287 = x10287 + z10288 = x10288 + z10289 = x10289 + z10290 = x10290 + z10291 = x10291 + z10292 = x10292 + z10293 = x10293 + z10294 = x10294 + z10295 = x10295 + z10296 = x10296 + z10297 = x10297 + z10298 = x10298 + z10299 = x10299 + z10300 = x10300 + z10301 = x10301 + z10302 = x10302 + z10303 = x10303 + z10304 = x10304 + z10305 = x10305 + z10306 = x10306 + z10307 = x10307 + z10308 = x10308 + z10309 = x10309 + z10310 = x10310 + z10311 = x10311 + z10312 = x10312 + z10313 = x10313 + z10314 = x10314 + z10315 = x10315 + z10316 = x10316 + z10317 = x10317 + z10318 = x10318 + z10319 = x10319 + z10320 = x10320 + z10321 = x10321 + z10322 = x10322 + z10323 = x10323 + z10324 = x10324 + z10325 = x10325 + z10326 = x10326 + z10327 = x10327 + z10328 = x10328 + z10329 = x10329 + z10330 = x10330 + z10331 = x10331 + z10332 = x10332 + z10333 = x10333 + z10334 = x10334 + z10335 = x10335 + z10336 = x10336 + z10337 = x10337 + z10338 = x10338 + z10339 = x10339 + z10340 = x10340 + z10341 = x10341 + z10342 = x10342 + z10343 = x10343 + z10344 = x10344 + z10345 = x10345 + z10346 = x10346 + z10347 = x10347 + z10348 = x10348 + z10349 = x10349 + z10350 = x10350 + z10351 = x10351 + z10352 = x10352 + z10353 = x10353 + z10354 = x10354 + z10355 = x10355 + z10356 = x10356 + z10357 = x10357 + z10358 = x10358 + z10359 = x10359 + z10360 = x10360 + z10361 = x10361 + z10362 = x10362 + z10363 = x10363 + z10364 = x10364 + z10365 = x10365 + z10366 = x10366 + z10367 = x10367 + z10368 = x10368 + z10369 = x10369 + z10370 = x10370 + z10371 = x10371 + z10372 = x10372 + z10373 = x10373 + z10374 = x10374 + z10375 = x10375 + z10376 = x10376 + z10377 = x10377 + z10378 = x10378 + z10379 = x10379 + z10380 = x10380 + z10381 = x10381 + z10382 = x10382 + z10383 = x10383 + z10384 = x10384 + z10385 = x10385 + z10386 = x10386 + z10387 = x10387 + z10388 = x10388 + z10389 = x10389 + z10390 = x10390 + z10391 = x10391 + z10392 = x10392 + z10393 = x10393 + z10394 = x10394 + z10395 = x10395 + z10396 = x10396 + z10397 = x10397 + z10398 = x10398 + z10399 = x10399 + z10400 = x10400 + z10401 = x10401 + z10402 = x10402 + z10403 = x10403 + z10404 = x10404 + z10405 = x10405 + z10406 = x10406 + z10407 = x10407 + z10408 = x10408 + z10409 = x10409 + z10410 = x10410 + z10411 = x10411 + z10412 = x10412 + z10413 = x10413 + z10414 = x10414 + z10415 = x10415 + z10416 = x10416 + z10417 = x10417 + z10418 = x10418 + z10419 = x10419 + z10420 = x10420 + z10421 = x10421 + z10422 = x10422 + z10423 = x10423 + z10424 = x10424 + z10425 = x10425 + z10426 = x10426 + z10427 = x10427 + z10428 = x10428 + z10429 = x10429 + z10430 = x10430 + z10431 = x10431 + z10432 = x10432 + z10433 = x10433 + z10434 = x10434 + z10435 = x10435 + z10436 = x10436 + z10437 = x10437 + z10438 = x10438 + z10439 = x10439 + z10440 = x10440 + z10441 = x10441 + z10442 = x10442 + z10443 = x10443 + z10444 = x10444 + z10445 = x10445 + z10446 = x10446 + z10447 = x10447 + z10448 = x10448 + z10449 = x10449 + z10450 = x10450 + z10451 = x10451 + z10452 = x10452 + z10453 = x10453 + z10454 = x10454 + z10455 = x10455 + z10456 = x10456 + z10457 = x10457 + z10458 = x10458 + z10459 = x10459 + z10460 = x10460 + z10461 = x10461 + z10462 = x10462 + z10463 = x10463 + z10464 = x10464 + z10465 = x10465 + z10466 = x10466 + z10467 = x10467 + z10468 = x10468 + z10469 = x10469 + z10470 = x10470 + z10471 = x10471 + z10472 = x10472 + z10473 = x10473 + z10474 = x10474 + z10475 = x10475 + z10476 = x10476 + z10477 = x10477 + z10478 = x10478 + z10479 = x10479 + z10480 = x10480 + z10481 = x10481 + z10482 = x10482 + z10483 = x10483 + z10484 = x10484 + z10485 = x10485 + z10486 = x10486 + z10487 = x10487 + z10488 = x10488 + z10489 = x10489 + z10490 = x10490 + z10491 = x10491 + z10492 = x10492 + z10493 = x10493 + z10494 = x10494 + z10495 = x10495 + z10496 = x10496 + z10497 = x10497 + z10498 = x10498 + z10499 = x10499 + z10500 = x10500 + z10501 = x10501 + z10502 = x10502 + z10503 = x10503 + z10504 = x10504 + z10505 = x10505 + z10506 = x10506 + z10507 = x10507 + z10508 = x10508 + z10509 = x10509 + z10510 = x10510 + z10511 = x10511 + z10512 = x10512 + z10513 = x10513 + z10514 = x10514 + z10515 = x10515 + z10516 = x10516 + z10517 = x10517 + z10518 = x10518 + z10519 = x10519 + z10520 = x10520 + z10521 = x10521 + z10522 = x10522 + z10523 = x10523 + z10524 = x10524 + z10525 = x10525 + z10526 = x10526 + z10527 = x10527 + z10528 = x10528 + z10529 = x10529 + z10530 = x10530 + z10531 = x10531 + z10532 = x10532 + z10533 = x10533 + z10534 = x10534 + z10535 = x10535 + z10536 = x10536 + z10537 = x10537 + z10538 = x10538 + z10539 = x10539 + z10540 = x10540 + z10541 = x10541 + z10542 = x10542 + z10543 = x10543 + z10544 = x10544 + z10545 = x10545 + z10546 = x10546 + z10547 = x10547 + z10548 = x10548 + z10549 = x10549 + z10550 = x10550 + z10551 = x10551 + z10552 = x10552 + z10553 = x10553 + z10554 = x10554 + z10555 = x10555 + z10556 = x10556 + z10557 = x10557 + z10558 = x10558 + z10559 = x10559 + z10560 = x10560 + z10561 = x10561 + z10562 = x10562 + z10563 = x10563 + z10564 = x10564 + z10565 = x10565 + z10566 = x10566 + z10567 = x10567 + z10568 = x10568 + z10569 = x10569 + z10570 = x10570 + z10571 = x10571 + z10572 = x10572 + z10573 = x10573 + z10574 = x10574 + z10575 = x10575 + z10576 = x10576 + z10577 = x10577 + z10578 = x10578 + z10579 = x10579 + z10580 = x10580 + z10581 = x10581 + z10582 = x10582 + z10583 = x10583 + z10584 = x10584 + z10585 = x10585 + z10586 = x10586 + z10587 = x10587 + z10588 = x10588 + z10589 = x10589 + z10590 = x10590 + z10591 = x10591 + z10592 = x10592 + z10593 = x10593 + z10594 = x10594 + z10595 = x10595 + z10596 = x10596 + z10597 = x10597 + z10598 = x10598 + z10599 = x10599 + z10600 = x10600 + z10601 = x10601 + z10602 = x10602 + z10603 = x10603 + z10604 = x10604 + z10605 = x10605 + z10606 = x10606 + z10607 = x10607 + z10608 = x10608 + z10609 = x10609 + z10610 = x10610 + z10611 = x10611 + z10612 = x10612 + z10613 = x10613 + z10614 = x10614 + z10615 = x10615 + z10616 = x10616 + z10617 = x10617 + z10618 = x10618 + z10619 = x10619 + z10620 = x10620 + z10621 = x10621 + z10622 = x10622 + z10623 = x10623 + z10624 = x10624 + z10625 = x10625 + z10626 = x10626 + z10627 = x10627 + z10628 = x10628 + z10629 = x10629 + z10630 = x10630 + z10631 = x10631 + z10632 = x10632 + z10633 = x10633 + z10634 = x10634 + z10635 = x10635 + z10636 = x10636 + z10637 = x10637 + z10638 = x10638 + z10639 = x10639 + z10640 = x10640 + z10641 = x10641 + z10642 = x10642 + z10643 = x10643 + z10644 = x10644 + z10645 = x10645 + z10646 = x10646 + z10647 = x10647 + z10648 = x10648 + z10649 = x10649 + z10650 = x10650 + z10651 = x10651 + z10652 = x10652 + z10653 = x10653 + z10654 = x10654 + z10655 = x10655 + z10656 = x10656 + z10657 = x10657 + z10658 = x10658 + z10659 = x10659 + z10660 = x10660 + z10661 = x10661 + z10662 = x10662 + z10663 = x10663 + z10664 = x10664 + z10665 = x10665 + z10666 = x10666 + z10667 = x10667 + z10668 = x10668 + z10669 = x10669 + z10670 = x10670 + z10671 = x10671 + z10672 = x10672 + z10673 = x10673 + z10674 = x10674 + z10675 = x10675 + z10676 = x10676 + z10677 = x10677 + z10678 = x10678 + z10679 = x10679 + z10680 = x10680 + z10681 = x10681 + z10682 = x10682 + z10683 = x10683 + z10684 = x10684 + z10685 = x10685 + z10686 = x10686 + z10687 = x10687 + z10688 = x10688 + z10689 = x10689 + z10690 = x10690 + z10691 = x10691 + z10692 = x10692 + z10693 = x10693 + z10694 = x10694 + z10695 = x10695 + z10696 = x10696 + z10697 = x10697 + z10698 = x10698 + z10699 = x10699 + z10700 = x10700 + z10701 = x10701 + z10702 = x10702 + z10703 = x10703 + z10704 = x10704 + z10705 = x10705 + z10706 = x10706 + z10707 = x10707 + z10708 = x10708 + z10709 = x10709 + z10710 = x10710 + z10711 = x10711 + z10712 = x10712 + z10713 = x10713 + z10714 = x10714 + z10715 = x10715 + z10716 = x10716 + z10717 = x10717 + z10718 = x10718 + z10719 = x10719 + z10720 = x10720 + z10721 = x10721 + z10722 = x10722 + z10723 = x10723 + z10724 = x10724 + z10725 = x10725 + z10726 = x10726 + z10727 = x10727 + z10728 = x10728 + z10729 = x10729 + z10730 = x10730 + z10731 = x10731 + z10732 = x10732 + z10733 = x10733 + z10734 = x10734 + z10735 = x10735 + z10736 = x10736 + z10737 = x10737 + z10738 = x10738 + z10739 = x10739 + z10740 = x10740 + z10741 = x10741 + z10742 = x10742 + z10743 = x10743 + z10744 = x10744 + z10745 = x10745 + z10746 = x10746 + z10747 = x10747 + z10748 = x10748 + z10749 = x10749 + z10750 = x10750 + z10751 = x10751 + z10752 = x10752 + z10753 = x10753 + z10754 = x10754 + z10755 = x10755 + z10756 = x10756 + z10757 = x10757 + z10758 = x10758 + z10759 = x10759 + z10760 = x10760 + z10761 = x10761 + z10762 = x10762 + z10763 = x10763 + z10764 = x10764 + z10765 = x10765 + z10766 = x10766 + z10767 = x10767 + z10768 = x10768 + z10769 = x10769 + z10770 = x10770 + z10771 = x10771 + z10772 = x10772 + z10773 = x10773 + z10774 = x10774 + z10775 = x10775 + z10776 = x10776 + z10777 = x10777 + z10778 = x10778 + z10779 = x10779 + z10780 = x10780 + z10781 = x10781 + z10782 = x10782 + z10783 = x10783 + z10784 = x10784 + z10785 = x10785 + z10786 = x10786 + z10787 = x10787 + z10788 = x10788 + z10789 = x10789 + z10790 = x10790 + z10791 = x10791 + z10792 = x10792 + z10793 = x10793 + z10794 = x10794 + z10795 = x10795 + z10796 = x10796 + z10797 = x10797 + z10798 = x10798 + z10799 = x10799 + z10800 = x10800 + z10801 = x10801 + z10802 = x10802 + z10803 = x10803 + z10804 = x10804 + z10805 = x10805 + z10806 = x10806 + z10807 = x10807 + z10808 = x10808 + z10809 = x10809 + z10810 = x10810 + z10811 = x10811 + z10812 = x10812 + z10813 = x10813 + z10814 = x10814 + z10815 = x10815 + z10816 = x10816 + z10817 = x10817 + z10818 = x10818 + z10819 = x10819 + z10820 = x10820 + z10821 = x10821 + z10822 = x10822 + z10823 = x10823 + z10824 = x10824 + z10825 = x10825 + z10826 = x10826 + z10827 = x10827 + z10828 = x10828 + z10829 = x10829 + z10830 = x10830 + z10831 = x10831 + z10832 = x10832 + z10833 = x10833 + z10834 = x10834 + z10835 = x10835 + z10836 = x10836 + z10837 = x10837 + z10838 = x10838 + z10839 = x10839 + z10840 = x10840 + z10841 = x10841 + z10842 = x10842 + z10843 = x10843 + z10844 = x10844 + z10845 = x10845 + z10846 = x10846 + z10847 = x10847 + z10848 = x10848 + z10849 = x10849 + z10850 = x10850 + z10851 = x10851 + z10852 = x10852 + z10853 = x10853 + z10854 = x10854 + z10855 = x10855 + z10856 = x10856 + z10857 = x10857 + z10858 = x10858 + z10859 = x10859 + z10860 = x10860 + z10861 = x10861 + z10862 = x10862 + z10863 = x10863 + z10864 = x10864 + z10865 = x10865 + z10866 = x10866 + z10867 = x10867 + z10868 = x10868 + z10869 = x10869 + z10870 = x10870 + z10871 = x10871 + z10872 = x10872 + z10873 = x10873 + z10874 = x10874 + z10875 = x10875 + z10876 = x10876 + z10877 = x10877 + z10878 = x10878 + z10879 = x10879 + z10880 = x10880 + z10881 = x10881 + z10882 = x10882 + z10883 = x10883 + z10884 = x10884 + z10885 = x10885 + z10886 = x10886 + z10887 = x10887 + z10888 = x10888 + z10889 = x10889 + z10890 = x10890 + z10891 = x10891 + z10892 = x10892 + z10893 = x10893 + z10894 = x10894 + z10895 = x10895 + z10896 = x10896 + z10897 = x10897 + z10898 = x10898 + z10899 = x10899 + z10900 = x10900 + z10901 = x10901 + z10902 = x10902 + z10903 = x10903 + z10904 = x10904 + z10905 = x10905 + z10906 = x10906 + z10907 = x10907 + z10908 = x10908 + z10909 = x10909 + z10910 = x10910 + z10911 = x10911 + z10912 = x10912 + z10913 = x10913 + z10914 = x10914 + z10915 = x10915 + z10916 = x10916 + z10917 = x10917 + z10918 = x10918 + z10919 = x10919 + z10920 = x10920 + z10921 = x10921 + z10922 = x10922 + z10923 = x10923 + z10924 = x10924 + z10925 = x10925 + z10926 = x10926 + z10927 = x10927 + z10928 = x10928 + z10929 = x10929 + z10930 = x10930 + z10931 = x10931 + z10932 = x10932 + z10933 = x10933 + z10934 = x10934 + z10935 = x10935 + z10936 = x10936 + z10937 = x10937 + z10938 = x10938 + z10939 = x10939 + z10940 = x10940 + z10941 = x10941 + z10942 = x10942 + z10943 = x10943 + z10944 = x10944 + z10945 = x10945 + z10946 = x10946 + z10947 = x10947 + z10948 = x10948 + z10949 = x10949 + z10950 = x10950 + z10951 = x10951 + z10952 = x10952 + z10953 = x10953 + z10954 = x10954 + z10955 = x10955 + z10956 = x10956 + z10957 = x10957 + z10958 = x10958 + z10959 = x10959 + z10960 = x10960 + z10961 = x10961 + z10962 = x10962 + z10963 = x10963 + z10964 = x10964 + z10965 = x10965 + z10966 = x10966 + z10967 = x10967 + z10968 = x10968 + z10969 = x10969 + z10970 = x10970 + z10971 = x10971 + z10972 = x10972 + z10973 = x10973 + z10974 = x10974 + z10975 = x10975 + z10976 = x10976 + z10977 = x10977 + z10978 = x10978 + z10979 = x10979 + z10980 = x10980 + z10981 = x10981 + z10982 = x10982 + z10983 = x10983 + z10984 = x10984 + z10985 = x10985 + z10986 = x10986 + z10987 = x10987 + z10988 = x10988 + z10989 = x10989 + z10990 = x10990 + z10991 = x10991 + z10992 = x10992 + z10993 = x10993 + z10994 = x10994 + z10995 = x10995 + z10996 = x10996 + z10997 = x10997 + z10998 = x10998 + z10999 = x10999 + z11000 = x11000 + z11001 = x11001 + z11002 = x11002 + z11003 = x11003 + z11004 = x11004 + z11005 = x11005 + z11006 = x11006 + z11007 = x11007 + z11008 = x11008 + z11009 = x11009 + z11010 = x11010 + z11011 = x11011 + z11012 = x11012 + z11013 = x11013 + z11014 = x11014 + z11015 = x11015 + z11016 = x11016 + z11017 = x11017 + z11018 = x11018 + z11019 = x11019 + z11020 = x11020 + z11021 = x11021 + z11022 = x11022 + z11023 = x11023 + z11024 = x11024 + z11025 = x11025 + z11026 = x11026 + z11027 = x11027 + z11028 = x11028 + z11029 = x11029 + z11030 = x11030 + z11031 = x11031 + z11032 = x11032 + z11033 = x11033 + z11034 = x11034 + z11035 = x11035 + z11036 = x11036 + z11037 = x11037 + z11038 = x11038 + z11039 = x11039 + z11040 = x11040 + z11041 = x11041 + z11042 = x11042 + z11043 = x11043 + z11044 = x11044 + z11045 = x11045 + z11046 = x11046 + z11047 = x11047 + z11048 = x11048 + z11049 = x11049 + z11050 = x11050 + z11051 = x11051 + z11052 = x11052 + z11053 = x11053 + z11054 = x11054 + z11055 = x11055 + z11056 = x11056 + z11057 = x11057 + z11058 = x11058 + z11059 = x11059 + z11060 = x11060 + z11061 = x11061 + z11062 = x11062 + z11063 = x11063 + z11064 = x11064 + z11065 = x11065 + z11066 = x11066 + z11067 = x11067 + z11068 = x11068 + z11069 = x11069 + z11070 = x11070 + z11071 = x11071 + z11072 = x11072 + z11073 = x11073 + z11074 = x11074 + z11075 = x11075 + z11076 = x11076 + z11077 = x11077 + z11078 = x11078 + z11079 = x11079 + z11080 = x11080 + z11081 = x11081 + z11082 = x11082 + z11083 = x11083 + z11084 = x11084 + z11085 = x11085 + z11086 = x11086 + z11087 = x11087 + z11088 = x11088 + z11089 = x11089 + z11090 = x11090 + z11091 = x11091 + z11092 = x11092 + z11093 = x11093 + z11094 = x11094 + z11095 = x11095 + z11096 = x11096 + z11097 = x11097 + z11098 = x11098 + z11099 = x11099 + z11100 = x11100 + z11101 = x11101 + z11102 = x11102 + z11103 = x11103 + z11104 = x11104 + z11105 = x11105 + z11106 = x11106 + z11107 = x11107 + z11108 = x11108 + z11109 = x11109 + z11110 = x11110 + z11111 = x11111 + z11112 = x11112 + z11113 = x11113 + z11114 = x11114 + z11115 = x11115 + z11116 = x11116 + z11117 = x11117 + z11118 = x11118 + z11119 = x11119 + z11120 = x11120 + z11121 = x11121 + z11122 = x11122 + z11123 = x11123 + z11124 = x11124 + z11125 = x11125 + z11126 = x11126 + z11127 = x11127 + z11128 = x11128 + z11129 = x11129 + z11130 = x11130 + z11131 = x11131 + z11132 = x11132 + z11133 = x11133 + z11134 = x11134 + z11135 = x11135 + z11136 = x11136 + z11137 = x11137 + z11138 = x11138 + z11139 = x11139 + z11140 = x11140 + z11141 = x11141 + z11142 = x11142 + z11143 = x11143 + z11144 = x11144 + z11145 = x11145 + z11146 = x11146 + z11147 = x11147 + z11148 = x11148 + z11149 = x11149 + z11150 = x11150 + z11151 = x11151 + z11152 = x11152 + z11153 = x11153 + z11154 = x11154 + z11155 = x11155 + z11156 = x11156 + z11157 = x11157 + z11158 = x11158 + z11159 = x11159 + z11160 = x11160 + z11161 = x11161 + z11162 = x11162 + z11163 = x11163 + z11164 = x11164 + z11165 = x11165 + z11166 = x11166 + z11167 = x11167 + z11168 = x11168 + z11169 = x11169 + z11170 = x11170 + z11171 = x11171 + z11172 = x11172 + z11173 = x11173 + z11174 = x11174 + z11175 = x11175 + z11176 = x11176 + z11177 = x11177 + z11178 = x11178 + z11179 = x11179 + z11180 = x11180 + z11181 = x11181 + z11182 = x11182 + z11183 = x11183 + z11184 = x11184 + z11185 = x11185 + z11186 = x11186 + z11187 = x11187 + z11188 = x11188 + z11189 = x11189 + z11190 = x11190 + z11191 = x11191 + z11192 = x11192 + z11193 = x11193 + z11194 = x11194 + z11195 = x11195 + z11196 = x11196 + z11197 = x11197 + z11198 = x11198 + z11199 = x11199 + z11200 = x11200 + z11201 = x11201 + z11202 = x11202 + z11203 = x11203 + z11204 = x11204 + z11205 = x11205 + z11206 = x11206 + z11207 = x11207 + z11208 = x11208 + z11209 = x11209 + z11210 = x11210 + z11211 = x11211 + z11212 = x11212 + z11213 = x11213 + z11214 = x11214 + z11215 = x11215 + z11216 = x11216 + z11217 = x11217 + z11218 = x11218 + z11219 = x11219 + z11220 = x11220 + z11221 = x11221 + z11222 = x11222 + z11223 = x11223 + z11224 = x11224 + z11225 = x11225 + z11226 = x11226 + z11227 = x11227 + z11228 = x11228 + z11229 = x11229 + z11230 = x11230 + z11231 = x11231 + z11232 = x11232 + z11233 = x11233 + z11234 = x11234 + z11235 = x11235 + z11236 = x11236 + z11237 = x11237 + z11238 = x11238 + z11239 = x11239 + z11240 = x11240 + z11241 = x11241 + z11242 = x11242 + z11243 = x11243 + z11244 = x11244 + z11245 = x11245 + z11246 = x11246 + z11247 = x11247 + z11248 = x11248 + z11249 = x11249 + z11250 = x11250 + z11251 = x11251 + z11252 = x11252 + z11253 = x11253 + z11254 = x11254 + z11255 = x11255 + z11256 = x11256 + z11257 = x11257 + z11258 = x11258 + z11259 = x11259 + z11260 = x11260 + z11261 = x11261 + z11262 = x11262 + z11263 = x11263 + z11264 = x11264 + z11265 = x11265 + z11266 = x11266 + z11267 = x11267 + z11268 = x11268 + z11269 = x11269 + z11270 = x11270 + z11271 = x11271 + z11272 = x11272 + z11273 = x11273 + z11274 = x11274 + z11275 = x11275 + z11276 = x11276 + z11277 = x11277 + z11278 = x11278 + z11279 = x11279 + z11280 = x11280 + z11281 = x11281 + z11282 = x11282 + z11283 = x11283 + z11284 = x11284 + z11285 = x11285 + z11286 = x11286 + z11287 = x11287 + z11288 = x11288 + z11289 = x11289 + z11290 = x11290 + z11291 = x11291 + z11292 = x11292 + z11293 = x11293 + z11294 = x11294 + z11295 = x11295 + z11296 = x11296 + z11297 = x11297 + z11298 = x11298 + z11299 = x11299 + z11300 = x11300 + z11301 = x11301 + z11302 = x11302 + z11303 = x11303 + z11304 = x11304 + z11305 = x11305 + z11306 = x11306 + z11307 = x11307 + z11308 = x11308 + z11309 = x11309 + z11310 = x11310 + z11311 = x11311 + z11312 = x11312 + z11313 = x11313 + z11314 = x11314 + z11315 = x11315 + z11316 = x11316 + z11317 = x11317 + z11318 = x11318 + z11319 = x11319 + z11320 = x11320 + z11321 = x11321 + z11322 = x11322 + z11323 = x11323 + z11324 = x11324 + z11325 = x11325 + z11326 = x11326 + z11327 = x11327 + z11328 = x11328 + z11329 = x11329 + z11330 = x11330 + z11331 = x11331 + z11332 = x11332 + z11333 = x11333 + z11334 = x11334 + z11335 = x11335 + z11336 = x11336 + z11337 = x11337 + z11338 = x11338 + z11339 = x11339 + z11340 = x11340 + z11341 = x11341 + z11342 = x11342 + z11343 = x11343 + z11344 = x11344 + z11345 = x11345 + z11346 = x11346 + z11347 = x11347 + z11348 = x11348 + z11349 = x11349 + z11350 = x11350 + z11351 = x11351 + z11352 = x11352 + z11353 = x11353 + z11354 = x11354 + z11355 = x11355 + z11356 = x11356 + z11357 = x11357 + z11358 = x11358 + z11359 = x11359 + z11360 = x11360 + z11361 = x11361 + z11362 = x11362 + z11363 = x11363 + z11364 = x11364 + z11365 = x11365 + z11366 = x11366 + z11367 = x11367 + z11368 = x11368 + z11369 = x11369 + z11370 = x11370 + z11371 = x11371 + z11372 = x11372 + z11373 = x11373 + z11374 = x11374 + z11375 = x11375 + z11376 = x11376 + z11377 = x11377 + z11378 = x11378 + z11379 = x11379 + z11380 = x11380 + z11381 = x11381 + z11382 = x11382 + z11383 = x11383 + z11384 = x11384 + z11385 = x11385 + z11386 = x11386 + z11387 = x11387 + z11388 = x11388 + z11389 = x11389 + z11390 = x11390 + z11391 = x11391 + z11392 = x11392 + z11393 = x11393 + z11394 = x11394 + z11395 = x11395 + z11396 = x11396 + z11397 = x11397 + z11398 = x11398 + z11399 = x11399 + z11400 = x11400 + z11401 = x11401 + z11402 = x11402 + z11403 = x11403 + z11404 = x11404 + z11405 = x11405 + z11406 = x11406 + z11407 = x11407 + z11408 = x11408 + z11409 = x11409 + z11410 = x11410 + z11411 = x11411 + z11412 = x11412 + z11413 = x11413 + z11414 = x11414 + z11415 = x11415 + z11416 = x11416 + z11417 = x11417 + z11418 = x11418 + z11419 = x11419 + z11420 = x11420 + z11421 = x11421 + z11422 = x11422 + z11423 = x11423 + z11424 = x11424 + z11425 = x11425 + z11426 = x11426 + z11427 = x11427 + z11428 = x11428 + z11429 = x11429 + z11430 = x11430 + z11431 = x11431 + z11432 = x11432 + z11433 = x11433 + z11434 = x11434 + z11435 = x11435 + z11436 = x11436 + z11437 = x11437 + z11438 = x11438 + z11439 = x11439 + z11440 = x11440 + z11441 = x11441 + z11442 = x11442 + z11443 = x11443 + z11444 = x11444 + z11445 = x11445 + z11446 = x11446 + z11447 = x11447 + z11448 = x11448 + z11449 = x11449 + z11450 = x11450 + z11451 = x11451 + z11452 = x11452 + z11453 = x11453 + z11454 = x11454 + z11455 = x11455 + z11456 = x11456 + z11457 = x11457 + z11458 = x11458 + z11459 = x11459 + z11460 = x11460 + z11461 = x11461 + z11462 = x11462 + z11463 = x11463 + z11464 = x11464 + z11465 = x11465 + z11466 = x11466 + z11467 = x11467 + z11468 = x11468 + z11469 = x11469 + z11470 = x11470 + z11471 = x11471 + z11472 = x11472 + z11473 = x11473 + z11474 = x11474 + z11475 = x11475 + z11476 = x11476 + z11477 = x11477 + z11478 = x11478 + z11479 = x11479 + z11480 = x11480 + z11481 = x11481 + z11482 = x11482 + z11483 = x11483 + z11484 = x11484 + z11485 = x11485 + z11486 = x11486 + z11487 = x11487 + z11488 = x11488 + z11489 = x11489 + z11490 = x11490 + z11491 = x11491 + z11492 = x11492 + z11493 = x11493 + z11494 = x11494 + z11495 = x11495 + z11496 = x11496 + z11497 = x11497 + z11498 = x11498 + z11499 = x11499 + z11500 = x11500 + z11501 = x11501 + z11502 = x11502 + z11503 = x11503 + z11504 = x11504 + z11505 = x11505 + z11506 = x11506 + z11507 = x11507 + z11508 = x11508 + z11509 = x11509 + z11510 = x11510 + z11511 = x11511 + z11512 = x11512 + z11513 = x11513 + z11514 = x11514 + z11515 = x11515 + z11516 = x11516 + z11517 = x11517 + z11518 = x11518 + z11519 = x11519 + z11520 = x11520 + z11521 = x11521 + z11522 = x11522 + z11523 = x11523 + z11524 = x11524 + z11525 = x11525 + z11526 = x11526 + z11527 = x11527 + z11528 = x11528 + z11529 = x11529 + z11530 = x11530 + z11531 = x11531 + z11532 = x11532 + z11533 = x11533 + z11534 = x11534 + z11535 = x11535 + z11536 = x11536 + z11537 = x11537 + z11538 = x11538 + z11539 = x11539 + z11540 = x11540 + z11541 = x11541 + z11542 = x11542 + z11543 = x11543 + z11544 = x11544 + z11545 = x11545 + z11546 = x11546 + z11547 = x11547 + z11548 = x11548 + z11549 = x11549 + z11550 = x11550 + z11551 = x11551 + z11552 = x11552 + z11553 = x11553 + z11554 = x11554 + z11555 = x11555 + z11556 = x11556 + z11557 = x11557 + z11558 = x11558 + z11559 = x11559 + z11560 = x11560 + z11561 = x11561 + z11562 = x11562 + z11563 = x11563 + z11564 = x11564 + z11565 = x11565 + z11566 = x11566 + z11567 = x11567 + z11568 = x11568 + z11569 = x11569 + z11570 = x11570 + z11571 = x11571 + z11572 = x11572 + z11573 = x11573 + z11574 = x11574 + z11575 = x11575 + z11576 = x11576 + z11577 = x11577 + z11578 = x11578 + z11579 = x11579 + z11580 = x11580 + z11581 = x11581 + z11582 = x11582 + z11583 = x11583 + z11584 = x11584 + z11585 = x11585 + z11586 = x11586 + z11587 = x11587 + z11588 = x11588 + z11589 = x11589 + z11590 = x11590 + z11591 = x11591 + z11592 = x11592 + z11593 = x11593 + z11594 = x11594 + z11595 = x11595 + z11596 = x11596 + z11597 = x11597 + z11598 = x11598 + z11599 = x11599 + z11600 = x11600 + z11601 = x11601 + z11602 = x11602 + z11603 = x11603 + z11604 = x11604 + z11605 = x11605 + z11606 = x11606 + z11607 = x11607 + z11608 = x11608 + z11609 = x11609 + z11610 = x11610 + z11611 = x11611 + z11612 = x11612 + z11613 = x11613 + z11614 = x11614 + z11615 = x11615 + z11616 = x11616 + z11617 = x11617 + z11618 = x11618 + z11619 = x11619 + z11620 = x11620 + z11621 = x11621 + z11622 = x11622 + z11623 = x11623 + z11624 = x11624 + z11625 = x11625 + z11626 = x11626 + z11627 = x11627 + z11628 = x11628 + z11629 = x11629 + z11630 = x11630 + z11631 = x11631 + z11632 = x11632 + z11633 = x11633 + z11634 = x11634 + z11635 = x11635 + z11636 = x11636 + z11637 = x11637 + z11638 = x11638 + z11639 = x11639 + z11640 = x11640 + z11641 = x11641 + z11642 = x11642 + z11643 = x11643 + z11644 = x11644 + z11645 = x11645 + z11646 = x11646 + z11647 = x11647 + z11648 = x11648 + z11649 = x11649 + z11650 = x11650 + z11651 = x11651 + z11652 = x11652 + z11653 = x11653 + z11654 = x11654 + z11655 = x11655 + z11656 = x11656 + z11657 = x11657 + z11658 = x11658 + z11659 = x11659 + z11660 = x11660 + z11661 = x11661 + z11662 = x11662 + z11663 = x11663 + z11664 = x11664 + z11665 = x11665 + z11666 = x11666 + z11667 = x11667 + z11668 = x11668 + z11669 = x11669 + z11670 = x11670 + z11671 = x11671 + z11672 = x11672 + z11673 = x11673 + z11674 = x11674 + z11675 = x11675 + z11676 = x11676 + z11677 = x11677 + z11678 = x11678 + z11679 = x11679 + z11680 = x11680 + z11681 = x11681 + z11682 = x11682 + z11683 = x11683 + z11684 = x11684 + z11685 = x11685 + z11686 = x11686 + z11687 = x11687 + z11688 = x11688 + z11689 = x11689 + z11690 = x11690 + z11691 = x11691 + z11692 = x11692 + z11693 = x11693 + z11694 = x11694 + z11695 = x11695 + z11696 = x11696 + z11697 = x11697 + z11698 = x11698 + z11699 = x11699 + z11700 = x11700 + z11701 = x11701 + z11702 = x11702 + z11703 = x11703 + z11704 = x11704 + z11705 = x11705 + z11706 = x11706 + z11707 = x11707 + z11708 = x11708 + z11709 = x11709 + z11710 = x11710 + z11711 = x11711 + z11712 = x11712 + z11713 = x11713 + z11714 = x11714 + z11715 = x11715 + z11716 = x11716 + z11717 = x11717 + z11718 = x11718 + z11719 = x11719 + z11720 = x11720 + z11721 = x11721 + z11722 = x11722 + z11723 = x11723 + z11724 = x11724 + z11725 = x11725 + z11726 = x11726 + z11727 = x11727 + z11728 = x11728 + z11729 = x11729 + z11730 = x11730 + z11731 = x11731 + z11732 = x11732 + z11733 = x11733 + z11734 = x11734 + z11735 = x11735 + z11736 = x11736 + z11737 = x11737 + z11738 = x11738 + z11739 = x11739 + z11740 = x11740 + z11741 = x11741 + z11742 = x11742 + z11743 = x11743 + z11744 = x11744 + z11745 = x11745 + z11746 = x11746 + z11747 = x11747 + z11748 = x11748 + z11749 = x11749 + z11750 = x11750 + z11751 = x11751 + z11752 = x11752 + z11753 = x11753 + z11754 = x11754 + z11755 = x11755 + z11756 = x11756 + z11757 = x11757 + z11758 = x11758 + z11759 = x11759 + z11760 = x11760 + z11761 = x11761 + z11762 = x11762 + z11763 = x11763 + z11764 = x11764 + z11765 = x11765 + z11766 = x11766 + z11767 = x11767 + z11768 = x11768 + z11769 = x11769 + z11770 = x11770 + z11771 = x11771 + z11772 = x11772 + z11773 = x11773 + z11774 = x11774 + z11775 = x11775 + z11776 = x11776 + z11777 = x11777 + z11778 = x11778 + z11779 = x11779 + z11780 = x11780 + z11781 = x11781 + z11782 = x11782 + z11783 = x11783 + z11784 = x11784 + z11785 = x11785 + z11786 = x11786 + z11787 = x11787 + z11788 = x11788 + z11789 = x11789 + z11790 = x11790 + z11791 = x11791 + z11792 = x11792 + z11793 = x11793 + z11794 = x11794 + z11795 = x11795 + z11796 = x11796 + z11797 = x11797 + z11798 = x11798 + z11799 = x11799 + z11800 = x11800 + z11801 = x11801 + z11802 = x11802 + z11803 = x11803 + z11804 = x11804 + z11805 = x11805 + z11806 = x11806 + z11807 = x11807 + z11808 = x11808 + z11809 = x11809 + z11810 = x11810 + z11811 = x11811 + z11812 = x11812 + z11813 = x11813 + z11814 = x11814 + z11815 = x11815 + z11816 = x11816 + z11817 = x11817 + z11818 = x11818 + z11819 = x11819 + z11820 = x11820 + z11821 = x11821 + z11822 = x11822 + z11823 = x11823 + z11824 = x11824 + z11825 = x11825 + z11826 = x11826 + z11827 = x11827 + z11828 = x11828 + z11829 = x11829 + z11830 = x11830 + z11831 = x11831 + z11832 = x11832 + z11833 = x11833 + z11834 = x11834 + z11835 = x11835 + z11836 = x11836 + z11837 = x11837 + z11838 = x11838 + z11839 = x11839 + z11840 = x11840 + z11841 = x11841 + z11842 = x11842 + z11843 = x11843 + z11844 = x11844 + z11845 = x11845 + z11846 = x11846 + z11847 = x11847 + z11848 = x11848 + z11849 = x11849 + z11850 = x11850 + z11851 = x11851 + z11852 = x11852 + z11853 = x11853 + z11854 = x11854 + z11855 = x11855 + z11856 = x11856 + z11857 = x11857 + z11858 = x11858 + z11859 = x11859 + z11860 = x11860 + z11861 = x11861 + z11862 = x11862 + z11863 = x11863 + z11864 = x11864 + z11865 = x11865 + z11866 = x11866 + z11867 = x11867 + z11868 = x11868 + z11869 = x11869 + z11870 = x11870 + z11871 = x11871 + z11872 = x11872 + z11873 = x11873 + z11874 = x11874 + z11875 = x11875 + z11876 = x11876 + z11877 = x11877 + z11878 = x11878 + z11879 = x11879 + z11880 = x11880 + z11881 = x11881 + z11882 = x11882 + z11883 = x11883 + z11884 = x11884 + z11885 = x11885 + z11886 = x11886 + z11887 = x11887 + z11888 = x11888 + z11889 = x11889 + z11890 = x11890 + z11891 = x11891 + z11892 = x11892 + z11893 = x11893 + z11894 = x11894 + z11895 = x11895 + z11896 = x11896 + z11897 = x11897 + z11898 = x11898 + z11899 = x11899 + z11900 = x11900 + z11901 = x11901 + z11902 = x11902 + z11903 = x11903 + z11904 = x11904 + z11905 = x11905 + z11906 = x11906 + z11907 = x11907 + z11908 = x11908 + z11909 = x11909 + z11910 = x11910 + z11911 = x11911 + z11912 = x11912 + z11913 = x11913 + z11914 = x11914 + z11915 = x11915 + z11916 = x11916 + z11917 = x11917 + z11918 = x11918 + z11919 = x11919 + z11920 = x11920 + z11921 = x11921 + z11922 = x11922 + z11923 = x11923 + z11924 = x11924 + z11925 = x11925 + z11926 = x11926 + z11927 = x11927 + z11928 = x11928 + z11929 = x11929 + z11930 = x11930 + z11931 = x11931 + z11932 = x11932 + z11933 = x11933 + z11934 = x11934 + z11935 = x11935 + z11936 = x11936 + z11937 = x11937 + z11938 = x11938 + z11939 = x11939 + z11940 = x11940 + z11941 = x11941 + z11942 = x11942 + z11943 = x11943 + z11944 = x11944 + z11945 = x11945 + z11946 = x11946 + z11947 = x11947 + z11948 = x11948 + z11949 = x11949 + z11950 = x11950 + z11951 = x11951 + z11952 = x11952 + z11953 = x11953 + z11954 = x11954 + z11955 = x11955 + z11956 = x11956 + z11957 = x11957 + z11958 = x11958 + z11959 = x11959 + z11960 = x11960 + z11961 = x11961 + z11962 = x11962 + z11963 = x11963 + z11964 = x11964 + z11965 = x11965 + z11966 = x11966 + z11967 = x11967 + z11968 = x11968 + z11969 = x11969 + z11970 = x11970 + z11971 = x11971 + z11972 = x11972 + z11973 = x11973 + z11974 = x11974 + z11975 = x11975 + z11976 = x11976 + z11977 = x11977 + z11978 = x11978 + z11979 = x11979 + z11980 = x11980 + z11981 = x11981 + z11982 = x11982 + z11983 = x11983 + z11984 = x11984 + z11985 = x11985 + z11986 = x11986 + z11987 = x11987 + z11988 = x11988 + z11989 = x11989 + z11990 = x11990 + z11991 = x11991 + z11992 = x11992 + z11993 = x11993 + z11994 = x11994 + z11995 = x11995 + z11996 = x11996 + z11997 = x11997 + z11998 = x11998 + z11999 = x11999 + z12000 = x12000 + z12001 = x12001 + z12002 = x12002 + z12003 = x12003 + z12004 = x12004 + z12005 = x12005 + z12006 = x12006 + z12007 = x12007 + z12008 = x12008 + z12009 = x12009 + z12010 = x12010 + z12011 = x12011 + z12012 = x12012 + z12013 = x12013 + z12014 = x12014 + z12015 = x12015 + z12016 = x12016 + z12017 = x12017 + z12018 = x12018 + z12019 = x12019 + z12020 = x12020 + z12021 = x12021 + z12022 = x12022 + z12023 = x12023 + z12024 = x12024 + z12025 = x12025 + z12026 = x12026 + z12027 = x12027 + z12028 = x12028 + z12029 = x12029 + z12030 = x12030 + z12031 = x12031 + z12032 = x12032 + z12033 = x12033 + z12034 = x12034 + z12035 = x12035 + z12036 = x12036 + z12037 = x12037 + z12038 = x12038 + z12039 = x12039 + z12040 = x12040 + z12041 = x12041 + z12042 = x12042 + z12043 = x12043 + z12044 = x12044 + z12045 = x12045 + z12046 = x12046 + z12047 = x12047 + z12048 = x12048 + z12049 = x12049 + z12050 = x12050 + z12051 = x12051 + z12052 = x12052 + z12053 = x12053 + z12054 = x12054 + z12055 = x12055 + z12056 = x12056 + z12057 = x12057 + z12058 = x12058 + z12059 = x12059 + z12060 = x12060 + z12061 = x12061 + z12062 = x12062 + z12063 = x12063 + z12064 = x12064 + z12065 = x12065 + z12066 = x12066 + z12067 = x12067 + z12068 = x12068 + z12069 = x12069 + z12070 = x12070 + z12071 = x12071 + z12072 = x12072 + z12073 = x12073 + z12074 = x12074 + z12075 = x12075 + z12076 = x12076 + z12077 = x12077 + z12078 = x12078 + z12079 = x12079 + z12080 = x12080 + z12081 = x12081 + z12082 = x12082 + z12083 = x12083 + z12084 = x12084 + z12085 = x12085 + z12086 = x12086 + z12087 = x12087 + z12088 = x12088 + z12089 = x12089 + z12090 = x12090 + z12091 = x12091 + z12092 = x12092 + z12093 = x12093 + z12094 = x12094 + z12095 = x12095 + z12096 = x12096 + z12097 = x12097 + z12098 = x12098 + z12099 = x12099 + z12100 = x12100 + z12101 = x12101 + z12102 = x12102 + z12103 = x12103 + z12104 = x12104 + z12105 = x12105 + z12106 = x12106 + z12107 = x12107 + z12108 = x12108 + z12109 = x12109 + z12110 = x12110 + z12111 = x12111 + z12112 = x12112 + z12113 = x12113 + z12114 = x12114 + z12115 = x12115 + z12116 = x12116 + z12117 = x12117 + z12118 = x12118 + z12119 = x12119 + z12120 = x12120 + z12121 = x12121 + z12122 = x12122 + z12123 = x12123 + z12124 = x12124 + z12125 = x12125 + z12126 = x12126 + z12127 = x12127 + z12128 = x12128 + z12129 = x12129 + z12130 = x12130 + z12131 = x12131 + z12132 = x12132 + z12133 = x12133 + z12134 = x12134 + z12135 = x12135 + z12136 = x12136 + z12137 = x12137 + z12138 = x12138 + z12139 = x12139 + z12140 = x12140 + z12141 = x12141 + z12142 = x12142 + z12143 = x12143 + z12144 = x12144 + z12145 = x12145 + z12146 = x12146 + z12147 = x12147 + z12148 = x12148 + z12149 = x12149 + z12150 = x12150 + z12151 = x12151 + z12152 = x12152 + z12153 = x12153 + z12154 = x12154 + z12155 = x12155 + z12156 = x12156 + z12157 = x12157 + z12158 = x12158 + z12159 = x12159 + z12160 = x12160 + z12161 = x12161 + z12162 = x12162 + z12163 = x12163 + z12164 = x12164 + z12165 = x12165 + z12166 = x12166 + z12167 = x12167 + z12168 = x12168 + z12169 = x12169 + z12170 = x12170 + z12171 = x12171 + z12172 = x12172 + z12173 = x12173 + z12174 = x12174 + z12175 = x12175 + z12176 = x12176 + z12177 = x12177 + z12178 = x12178 + z12179 = x12179 + z12180 = x12180 + z12181 = x12181 + z12182 = x12182 + z12183 = x12183 + z12184 = x12184 + z12185 = x12185 + z12186 = x12186 + z12187 = x12187 + z12188 = x12188 + z12189 = x12189 + z12190 = x12190 + z12191 = x12191 + z12192 = x12192 + z12193 = x12193 + z12194 = x12194 + z12195 = x12195 + z12196 = x12196 + z12197 = x12197 + z12198 = x12198 + z12199 = x12199 + z12200 = x12200 + z12201 = x12201 + z12202 = x12202 + z12203 = x12203 + z12204 = x12204 + z12205 = x12205 + z12206 = x12206 + z12207 = x12207 + z12208 = x12208 + z12209 = x12209 + z12210 = x12210 + z12211 = x12211 + z12212 = x12212 + z12213 = x12213 + z12214 = x12214 + z12215 = x12215 + z12216 = x12216 + z12217 = x12217 + z12218 = x12218 + z12219 = x12219 + z12220 = x12220 + z12221 = x12221 + z12222 = x12222 + z12223 = x12223 + z12224 = x12224 + z12225 = x12225 + z12226 = x12226 + z12227 = x12227 + z12228 = x12228 + z12229 = x12229 + z12230 = x12230 + z12231 = x12231 + z12232 = x12232 + z12233 = x12233 + z12234 = x12234 + z12235 = x12235 + z12236 = x12236 + z12237 = x12237 + z12238 = x12238 + z12239 = x12239 + z12240 = x12240 + z12241 = x12241 + z12242 = x12242 + z12243 = x12243 + z12244 = x12244 + z12245 = x12245 + z12246 = x12246 + z12247 = x12247 + z12248 = x12248 + z12249 = x12249 + z12250 = x12250 + z12251 = x12251 + z12252 = x12252 + z12253 = x12253 + z12254 = x12254 + z12255 = x12255 + z12256 = x12256 + z12257 = x12257 + z12258 = x12258 + z12259 = x12259 + z12260 = x12260 + z12261 = x12261 + z12262 = x12262 + z12263 = x12263 + z12264 = x12264 + z12265 = x12265 + z12266 = x12266 + z12267 = x12267 + z12268 = x12268 + z12269 = x12269 + z12270 = x12270 + z12271 = x12271 + z12272 = x12272 + z12273 = x12273 + z12274 = x12274 + z12275 = x12275 + z12276 = x12276 + z12277 = x12277 + z12278 = x12278 + z12279 = x12279 + z12280 = x12280 + z12281 = x12281 + z12282 = x12282 + z12283 = x12283 + z12284 = x12284 + z12285 = x12285 + z12286 = x12286 + z12287 = x12287 + z12288 = x12288 + z12289 = x12289 + z12290 = x12290 + z12291 = x12291 + z12292 = x12292 + z12293 = x12293 + z12294 = x12294 + z12295 = x12295 + z12296 = x12296 + z12297 = x12297 + z12298 = x12298 + z12299 = x12299 + z12300 = x12300 + z12301 = x12301 + z12302 = x12302 + z12303 = x12303 + z12304 = x12304 + z12305 = x12305 + z12306 = x12306 + z12307 = x12307 + z12308 = x12308 + z12309 = x12309 + z12310 = x12310 + z12311 = x12311 + z12312 = x12312 + z12313 = x12313 + z12314 = x12314 + z12315 = x12315 + z12316 = x12316 + z12317 = x12317 + z12318 = x12318 + z12319 = x12319 + z12320 = x12320 + z12321 = x12321 + z12322 = x12322 + z12323 = x12323 + z12324 = x12324 + z12325 = x12325 + z12326 = x12326 + z12327 = x12327 + z12328 = x12328 + z12329 = x12329 + z12330 = x12330 + z12331 = x12331 + z12332 = x12332 + z12333 = x12333 + z12334 = x12334 + z12335 = x12335 + z12336 = x12336 + z12337 = x12337 + z12338 = x12338 + z12339 = x12339 + z12340 = x12340 + z12341 = x12341 + z12342 = x12342 + z12343 = x12343 + z12344 = x12344 + z12345 = x12345 + z12346 = x12346 + z12347 = x12347 + z12348 = x12348 + z12349 = x12349 + z12350 = x12350 + z12351 = x12351 + z12352 = x12352 + z12353 = x12353 + z12354 = x12354 + z12355 = x12355 + z12356 = x12356 + z12357 = x12357 + z12358 = x12358 + z12359 = x12359 + z12360 = x12360 + z12361 = x12361 + z12362 = x12362 + z12363 = x12363 + z12364 = x12364 + z12365 = x12365 + z12366 = x12366 + z12367 = x12367 + z12368 = x12368 + z12369 = x12369 + z12370 = x12370 + z12371 = x12371 + z12372 = x12372 + z12373 = x12373 + z12374 = x12374 + z12375 = x12375 + z12376 = x12376 + z12377 = x12377 + z12378 = x12378 + z12379 = x12379 + z12380 = x12380 + z12381 = x12381 + z12382 = x12382 + z12383 = x12383 + z12384 = x12384 + z12385 = x12385 + z12386 = x12386 + z12387 = x12387 + z12388 = x12388 + z12389 = x12389 + z12390 = x12390 + z12391 = x12391 + z12392 = x12392 + z12393 = x12393 + z12394 = x12394 + z12395 = x12395 + z12396 = x12396 + z12397 = x12397 + z12398 = x12398 + z12399 = x12399 + z12400 = x12400 + z12401 = x12401 + z12402 = x12402 + z12403 = x12403 + z12404 = x12404 + z12405 = x12405 + z12406 = x12406 + z12407 = x12407 + z12408 = x12408 + z12409 = x12409 + z12410 = x12410 + z12411 = x12411 + z12412 = x12412 + z12413 = x12413 + z12414 = x12414 + z12415 = x12415 + z12416 = x12416 + z12417 = x12417 + z12418 = x12418 + z12419 = x12419 + z12420 = x12420 + z12421 = x12421 + z12422 = x12422 + z12423 = x12423 + z12424 = x12424 + z12425 = x12425 + z12426 = x12426 + z12427 = x12427 + z12428 = x12428 + z12429 = x12429 + z12430 = x12430 + z12431 = x12431 + z12432 = x12432 + z12433 = x12433 + z12434 = x12434 + z12435 = x12435 + z12436 = x12436 + z12437 = x12437 + z12438 = x12438 + z12439 = x12439 + z12440 = x12440 + z12441 = x12441 + z12442 = x12442 + z12443 = x12443 + z12444 = x12444 + z12445 = x12445 + z12446 = x12446 + z12447 = x12447 + z12448 = x12448 + z12449 = x12449 + z12450 = x12450 + z12451 = x12451 + z12452 = x12452 + z12453 = x12453 + z12454 = x12454 + z12455 = x12455 + z12456 = x12456 + z12457 = x12457 + z12458 = x12458 + z12459 = x12459 + z12460 = x12460 + z12461 = x12461 + z12462 = x12462 + z12463 = x12463 + z12464 = x12464 + z12465 = x12465 + z12466 = x12466 + z12467 = x12467 + z12468 = x12468 + z12469 = x12469 + z12470 = x12470 + z12471 = x12471 + z12472 = x12472 + z12473 = x12473 + z12474 = x12474 + z12475 = x12475 + z12476 = x12476 + z12477 = x12477 + z12478 = x12478 + z12479 = x12479 + z12480 = x12480 + z12481 = x12481 + z12482 = x12482 + z12483 = x12483 + z12484 = x12484 + z12485 = x12485 + z12486 = x12486 + z12487 = x12487 + z12488 = x12488 + z12489 = x12489 + z12490 = x12490 + z12491 = x12491 + z12492 = x12492 + z12493 = x12493 + z12494 = x12494 + z12495 = x12495 + z12496 = x12496 + z12497 = x12497 + z12498 = x12498 + z12499 = x12499 + z12500 = x12500 + z12501 = x12501 + z12502 = x12502 + z12503 = x12503 + z12504 = x12504 + z12505 = x12505 + z12506 = x12506 + z12507 = x12507 + z12508 = x12508 + z12509 = x12509 + z12510 = x12510 + z12511 = x12511 + z12512 = x12512 + z12513 = x12513 + z12514 = x12514 + z12515 = x12515 + z12516 = x12516 + z12517 = x12517 + z12518 = x12518 + z12519 = x12519 + z12520 = x12520 + z12521 = x12521 + z12522 = x12522 + z12523 = x12523 + z12524 = x12524 + z12525 = x12525 + z12526 = x12526 + z12527 = x12527 + z12528 = x12528 + z12529 = x12529 + z12530 = x12530 + z12531 = x12531 + z12532 = x12532 + z12533 = x12533 + z12534 = x12534 + z12535 = x12535 + z12536 = x12536 + z12537 = x12537 + z12538 = x12538 + z12539 = x12539 + z12540 = x12540 + z12541 = x12541 + z12542 = x12542 + z12543 = x12543 + z12544 = x12544 + z12545 = x12545 + z12546 = x12546 + z12547 = x12547 + z12548 = x12548 + z12549 = x12549 + z12550 = x12550 + z12551 = x12551 + z12552 = x12552 + z12553 = x12553 + z12554 = x12554 + z12555 = x12555 + z12556 = x12556 + z12557 = x12557 + z12558 = x12558 + z12559 = x12559 + z12560 = x12560 + z12561 = x12561 + z12562 = x12562 + z12563 = x12563 + z12564 = x12564 + z12565 = x12565 + z12566 = x12566 + z12567 = x12567 + z12568 = x12568 + z12569 = x12569 + z12570 = x12570 + z12571 = x12571 + z12572 = x12572 + z12573 = x12573 + z12574 = x12574 + z12575 = x12575 + z12576 = x12576 + z12577 = x12577 + z12578 = x12578 + z12579 = x12579 + z12580 = x12580 + z12581 = x12581 + z12582 = x12582 + z12583 = x12583 + z12584 = x12584 + z12585 = x12585 + z12586 = x12586 + z12587 = x12587 + z12588 = x12588 + z12589 = x12589 + z12590 = x12590 + z12591 = x12591 + z12592 = x12592 + z12593 = x12593 + z12594 = x12594 + z12595 = x12595 + z12596 = x12596 + z12597 = x12597 + z12598 = x12598 + z12599 = x12599 + z12600 = x12600 + z12601 = x12601 + z12602 = x12602 + z12603 = x12603 + z12604 = x12604 + z12605 = x12605 + z12606 = x12606 + z12607 = x12607 + z12608 = x12608 + z12609 = x12609 + z12610 = x12610 + z12611 = x12611 + z12612 = x12612 + z12613 = x12613 + z12614 = x12614 + z12615 = x12615 + z12616 = x12616 + z12617 = x12617 + z12618 = x12618 + z12619 = x12619 + z12620 = x12620 + z12621 = x12621 + z12622 = x12622 + z12623 = x12623 + z12624 = x12624 + z12625 = x12625 + z12626 = x12626 + z12627 = x12627 + z12628 = x12628 + z12629 = x12629 + z12630 = x12630 + z12631 = x12631 + z12632 = x12632 + z12633 = x12633 + z12634 = x12634 + z12635 = x12635 + z12636 = x12636 + z12637 = x12637 + z12638 = x12638 + z12639 = x12639 + z12640 = x12640 + z12641 = x12641 + z12642 = x12642 + z12643 = x12643 + z12644 = x12644 + z12645 = x12645 + z12646 = x12646 + z12647 = x12647 + z12648 = x12648 + z12649 = x12649 + z12650 = x12650 + z12651 = x12651 + z12652 = x12652 + z12653 = x12653 + z12654 = x12654 + z12655 = x12655 + z12656 = x12656 + z12657 = x12657 + z12658 = x12658 + z12659 = x12659 + z12660 = x12660 + z12661 = x12661 + z12662 = x12662 + z12663 = x12663 + z12664 = x12664 + z12665 = x12665 + z12666 = x12666 + z12667 = x12667 + z12668 = x12668 + z12669 = x12669 + z12670 = x12670 + z12671 = x12671 + z12672 = x12672 + z12673 = x12673 + z12674 = x12674 + z12675 = x12675 + z12676 = x12676 + z12677 = x12677 + z12678 = x12678 + z12679 = x12679 + z12680 = x12680 + z12681 = x12681 + z12682 = x12682 + z12683 = x12683 + z12684 = x12684 + z12685 = x12685 + z12686 = x12686 + z12687 = x12687 + z12688 = x12688 + z12689 = x12689 + z12690 = x12690 + z12691 = x12691 + z12692 = x12692 + z12693 = x12693 + z12694 = x12694 + z12695 = x12695 + z12696 = x12696 + z12697 = x12697 + z12698 = x12698 + z12699 = x12699 + z12700 = x12700 + z12701 = x12701 + z12702 = x12702 + z12703 = x12703 + z12704 = x12704 + z12705 = x12705 + z12706 = x12706 + z12707 = x12707 + z12708 = x12708 + z12709 = x12709 + z12710 = x12710 + z12711 = x12711 + z12712 = x12712 + z12713 = x12713 + z12714 = x12714 + z12715 = x12715 + z12716 = x12716 + z12717 = x12717 + z12718 = x12718 + z12719 = x12719 + z12720 = x12720 + z12721 = x12721 + z12722 = x12722 + z12723 = x12723 + z12724 = x12724 + z12725 = x12725 + z12726 = x12726 + z12727 = x12727 + z12728 = x12728 + z12729 = x12729 + z12730 = x12730 + z12731 = x12731 + z12732 = x12732 + z12733 = x12733 + z12734 = x12734 + z12735 = x12735 + z12736 = x12736 + z12737 = x12737 + z12738 = x12738 + z12739 = x12739 + z12740 = x12740 + z12741 = x12741 + z12742 = x12742 + z12743 = x12743 + z12744 = x12744 + z12745 = x12745 + z12746 = x12746 + z12747 = x12747 + z12748 = x12748 + z12749 = x12749 + z12750 = x12750 + z12751 = x12751 + z12752 = x12752 + z12753 = x12753 + z12754 = x12754 + z12755 = x12755 + z12756 = x12756 + z12757 = x12757 + z12758 = x12758 + z12759 = x12759 + z12760 = x12760 + z12761 = x12761 + z12762 = x12762 + z12763 = x12763 + z12764 = x12764 + z12765 = x12765 + z12766 = x12766 + z12767 = x12767 + z12768 = x12768 + z12769 = x12769 + z12770 = x12770 + z12771 = x12771 + z12772 = x12772 + z12773 = x12773 + z12774 = x12774 + z12775 = x12775 + z12776 = x12776 + z12777 = x12777 + z12778 = x12778 + z12779 = x12779 + z12780 = x12780 + z12781 = x12781 + z12782 = x12782 + z12783 = x12783 + z12784 = x12784 + z12785 = x12785 + z12786 = x12786 + z12787 = x12787 + z12788 = x12788 + z12789 = x12789 + z12790 = x12790 + z12791 = x12791 + z12792 = x12792 + z12793 = x12793 + z12794 = x12794 + z12795 = x12795 + z12796 = x12796 + z12797 = x12797 + z12798 = x12798 + z12799 = x12799 + z12800 = x12800 + z12801 = x12801 + z12802 = x12802 + z12803 = x12803 + z12804 = x12804 + z12805 = x12805 + z12806 = x12806 + z12807 = x12807 + z12808 = x12808 + z12809 = x12809 + z12810 = x12810 + z12811 = x12811 + z12812 = x12812 + z12813 = x12813 + z12814 = x12814 + z12815 = x12815 + z12816 = x12816 + z12817 = x12817 + z12818 = x12818 + z12819 = x12819 + z12820 = x12820 + z12821 = x12821 + z12822 = x12822 + z12823 = x12823 + z12824 = x12824 + z12825 = x12825 + z12826 = x12826 + z12827 = x12827 + z12828 = x12828 + z12829 = x12829 + z12830 = x12830 + z12831 = x12831 + z12832 = x12832 + z12833 = x12833 + z12834 = x12834 + z12835 = x12835 + z12836 = x12836 + z12837 = x12837 + z12838 = x12838 + z12839 = x12839 + z12840 = x12840 + z12841 = x12841 + z12842 = x12842 + z12843 = x12843 + z12844 = x12844 + z12845 = x12845 + z12846 = x12846 + z12847 = x12847 + z12848 = x12848 + z12849 = x12849 + z12850 = x12850 + z12851 = x12851 + z12852 = x12852 + z12853 = x12853 + z12854 = x12854 + z12855 = x12855 + z12856 = x12856 + z12857 = x12857 + z12858 = x12858 + z12859 = x12859 + z12860 = x12860 + z12861 = x12861 + z12862 = x12862 + z12863 = x12863 + z12864 = x12864 + z12865 = x12865 + z12866 = x12866 + z12867 = x12867 + z12868 = x12868 + z12869 = x12869 + z12870 = x12870 + z12871 = x12871 + z12872 = x12872 + z12873 = x12873 + z12874 = x12874 + z12875 = x12875 + z12876 = x12876 + z12877 = x12877 + z12878 = x12878 + z12879 = x12879 + z12880 = x12880 + z12881 = x12881 + z12882 = x12882 + z12883 = x12883 + z12884 = x12884 + z12885 = x12885 + z12886 = x12886 + z12887 = x12887 + z12888 = x12888 + z12889 = x12889 + z12890 = x12890 + z12891 = x12891 + z12892 = x12892 + z12893 = x12893 + z12894 = x12894 + z12895 = x12895 + z12896 = x12896 + z12897 = x12897 + z12898 = x12898 + z12899 = x12899 + z12900 = x12900 + z12901 = x12901 + z12902 = x12902 + z12903 = x12903 + z12904 = x12904 + z12905 = x12905 + z12906 = x12906 + z12907 = x12907 + z12908 = x12908 + z12909 = x12909 + z12910 = x12910 + z12911 = x12911 + z12912 = x12912 + z12913 = x12913 + z12914 = x12914 + z12915 = x12915 + z12916 = x12916 + z12917 = x12917 + z12918 = x12918 + z12919 = x12919 + z12920 = x12920 + z12921 = x12921 + z12922 = x12922 + z12923 = x12923 + z12924 = x12924 + z12925 = x12925 + z12926 = x12926 + z12927 = x12927 + z12928 = x12928 + z12929 = x12929 + z12930 = x12930 + z12931 = x12931 + z12932 = x12932 + z12933 = x12933 + z12934 = x12934 + z12935 = x12935 + z12936 = x12936 + z12937 = x12937 + z12938 = x12938 + z12939 = x12939 + z12940 = x12940 + z12941 = x12941 + z12942 = x12942 + z12943 = x12943 + z12944 = x12944 + z12945 = x12945 + z12946 = x12946 + z12947 = x12947 + z12948 = x12948 + z12949 = x12949 + z12950 = x12950 + z12951 = x12951 + z12952 = x12952 + z12953 = x12953 + z12954 = x12954 + z12955 = x12955 + z12956 = x12956 + z12957 = x12957 + z12958 = x12958 + z12959 = x12959 + z12960 = x12960 + z12961 = x12961 + z12962 = x12962 + z12963 = x12963 + z12964 = x12964 + z12965 = x12965 + z12966 = x12966 + z12967 = x12967 + z12968 = x12968 + z12969 = x12969 + z12970 = x12970 + z12971 = x12971 + z12972 = x12972 + z12973 = x12973 + z12974 = x12974 + z12975 = x12975 + z12976 = x12976 + z12977 = x12977 + z12978 = x12978 + z12979 = x12979 + z12980 = x12980 + z12981 = x12981 + z12982 = x12982 + z12983 = x12983 + z12984 = x12984 + z12985 = x12985 + z12986 = x12986 + z12987 = x12987 + z12988 = x12988 + z12989 = x12989 + z12990 = x12990 + z12991 = x12991 + z12992 = x12992 + z12993 = x12993 + z12994 = x12994 + z12995 = x12995 + z12996 = x12996 + z12997 = x12997 + z12998 = x12998 + z12999 = x12999 + z13000 = x13000 + z13001 = x13001 + z13002 = x13002 + z13003 = x13003 + z13004 = x13004 + z13005 = x13005 + z13006 = x13006 + z13007 = x13007 + z13008 = x13008 + z13009 = x13009 + z13010 = x13010 + z13011 = x13011 + z13012 = x13012 + z13013 = x13013 + z13014 = x13014 + z13015 = x13015 + z13016 = x13016 + z13017 = x13017 + z13018 = x13018 + z13019 = x13019 + z13020 = x13020 + z13021 = x13021 + z13022 = x13022 + z13023 = x13023 + z13024 = x13024 + z13025 = x13025 + z13026 = x13026 + z13027 = x13027 + z13028 = x13028 + z13029 = x13029 + z13030 = x13030 + z13031 = x13031 + z13032 = x13032 + z13033 = x13033 + z13034 = x13034 + z13035 = x13035 + z13036 = x13036 + z13037 = x13037 + z13038 = x13038 + z13039 = x13039 + z13040 = x13040 + z13041 = x13041 + z13042 = x13042 + z13043 = x13043 + z13044 = x13044 + z13045 = x13045 + z13046 = x13046 + z13047 = x13047 + z13048 = x13048 + z13049 = x13049 + z13050 = x13050 + z13051 = x13051 + z13052 = x13052 + z13053 = x13053 + z13054 = x13054 + z13055 = x13055 + z13056 = x13056 + z13057 = x13057 + z13058 = x13058 + z13059 = x13059 + z13060 = x13060 + z13061 = x13061 + z13062 = x13062 + z13063 = x13063 + z13064 = x13064 + z13065 = x13065 + z13066 = x13066 + z13067 = x13067 + z13068 = x13068 + z13069 = x13069 + z13070 = x13070 + z13071 = x13071 + z13072 = x13072 + z13073 = x13073 + z13074 = x13074 + z13075 = x13075 + z13076 = x13076 + z13077 = x13077 + z13078 = x13078 + z13079 = x13079 + z13080 = x13080 + z13081 = x13081 + z13082 = x13082 + z13083 = x13083 + z13084 = x13084 + z13085 = x13085 + z13086 = x13086 + z13087 = x13087 + z13088 = x13088 + z13089 = x13089 + z13090 = x13090 + z13091 = x13091 + z13092 = x13092 + z13093 = x13093 + z13094 = x13094 + z13095 = x13095 + z13096 = x13096 + z13097 = x13097 + z13098 = x13098 + z13099 = x13099 + z13100 = x13100 + z13101 = x13101 + z13102 = x13102 + z13103 = x13103 + z13104 = x13104 + z13105 = x13105 + z13106 = x13106 + z13107 = x13107 + z13108 = x13108 + z13109 = x13109 + z13110 = x13110 + z13111 = x13111 + z13112 = x13112 + z13113 = x13113 + z13114 = x13114 + z13115 = x13115 + z13116 = x13116 + z13117 = x13117 + z13118 = x13118 + z13119 = x13119 + z13120 = x13120 + z13121 = x13121 + z13122 = x13122 + z13123 = x13123 + z13124 = x13124 + z13125 = x13125 + z13126 = x13126 + z13127 = x13127 + z13128 = x13128 + z13129 = x13129 + z13130 = x13130 + z13131 = x13131 + z13132 = x13132 + z13133 = x13133 + z13134 = x13134 + z13135 = x13135 + z13136 = x13136 + z13137 = x13137 + z13138 = x13138 + z13139 = x13139 + z13140 = x13140 + z13141 = x13141 + z13142 = x13142 + z13143 = x13143 + z13144 = x13144 + z13145 = x13145 + z13146 = x13146 + z13147 = x13147 + z13148 = x13148 + z13149 = x13149 + z13150 = x13150 + z13151 = x13151 + z13152 = x13152 + z13153 = x13153 + z13154 = x13154 + z13155 = x13155 + z13156 = x13156 + z13157 = x13157 + z13158 = x13158 + z13159 = x13159 + z13160 = x13160 + z13161 = x13161 + z13162 = x13162 + z13163 = x13163 + z13164 = x13164 + z13165 = x13165 + z13166 = x13166 + z13167 = x13167 + z13168 = x13168 + z13169 = x13169 + z13170 = x13170 + z13171 = x13171 + z13172 = x13172 + z13173 = x13173 + z13174 = x13174 + z13175 = x13175 + z13176 = x13176 + z13177 = x13177 + z13178 = x13178 + z13179 = x13179 + z13180 = x13180 + z13181 = x13181 + z13182 = x13182 + z13183 = x13183 + z13184 = x13184 + z13185 = x13185 + z13186 = x13186 + z13187 = x13187 + z13188 = x13188 + z13189 = x13189 + z13190 = x13190 + z13191 = x13191 + z13192 = x13192 + z13193 = x13193 + z13194 = x13194 + z13195 = x13195 + z13196 = x13196 + z13197 = x13197 + z13198 = x13198 + z13199 = x13199 + z13200 = x13200 + z13201 = x13201 + z13202 = x13202 + z13203 = x13203 + z13204 = x13204 + z13205 = x13205 + z13206 = x13206 + z13207 = x13207 + z13208 = x13208 + z13209 = x13209 + z13210 = x13210 + z13211 = x13211 + z13212 = x13212 + z13213 = x13213 + z13214 = x13214 + z13215 = x13215 + z13216 = x13216 + z13217 = x13217 + z13218 = x13218 + z13219 = x13219 + z13220 = x13220 + z13221 = x13221 + z13222 = x13222 + z13223 = x13223 + z13224 = x13224 + z13225 = x13225 + z13226 = x13226 + z13227 = x13227 + z13228 = x13228 + z13229 = x13229 + z13230 = x13230 + z13231 = x13231 + z13232 = x13232 + z13233 = x13233 + z13234 = x13234 + z13235 = x13235 + z13236 = x13236 + z13237 = x13237 + z13238 = x13238 + z13239 = x13239 + z13240 = x13240 + z13241 = x13241 + z13242 = x13242 + z13243 = x13243 + z13244 = x13244 + z13245 = x13245 + z13246 = x13246 + z13247 = x13247 + z13248 = x13248 + z13249 = x13249 + z13250 = x13250 + z13251 = x13251 + z13252 = x13252 + z13253 = x13253 + z13254 = x13254 + z13255 = x13255 + z13256 = x13256 + z13257 = x13257 + z13258 = x13258 + z13259 = x13259 + z13260 = x13260 + z13261 = x13261 + z13262 = x13262 + z13263 = x13263 + z13264 = x13264 + z13265 = x13265 + z13266 = x13266 + z13267 = x13267 + z13268 = x13268 + z13269 = x13269 + z13270 = x13270 + z13271 = x13271 + z13272 = x13272 + z13273 = x13273 + z13274 = x13274 + z13275 = x13275 + z13276 = x13276 + z13277 = x13277 + z13278 = x13278 + z13279 = x13279 + z13280 = x13280 + z13281 = x13281 + z13282 = x13282 + z13283 = x13283 + z13284 = x13284 + z13285 = x13285 + z13286 = x13286 + z13287 = x13287 + z13288 = x13288 + z13289 = x13289 + z13290 = x13290 + z13291 = x13291 + z13292 = x13292 + z13293 = x13293 + z13294 = x13294 + z13295 = x13295 + z13296 = x13296 + z13297 = x13297 + z13298 = x13298 + z13299 = x13299 + z13300 = x13300 + z13301 = x13301 + z13302 = x13302 + z13303 = x13303 + z13304 = x13304 + z13305 = x13305 + z13306 = x13306 + z13307 = x13307 + z13308 = x13308 + z13309 = x13309 + z13310 = x13310 + z13311 = x13311 + z13312 = x13312 + z13313 = x13313 + z13314 = x13314 + z13315 = x13315 + z13316 = x13316 + z13317 = x13317 + z13318 = x13318 + z13319 = x13319 + z13320 = x13320 + z13321 = x13321 + z13322 = x13322 + z13323 = x13323 + z13324 = x13324 + z13325 = x13325 + z13326 = x13326 + z13327 = x13327 + z13328 = x13328 + z13329 = x13329 + z13330 = x13330 + z13331 = x13331 + z13332 = x13332 + z13333 = x13333 + z13334 = x13334 + z13335 = x13335 + z13336 = x13336 + z13337 = x13337 + z13338 = x13338 + z13339 = x13339 + z13340 = x13340 + z13341 = x13341 + z13342 = x13342 + z13343 = x13343 + z13344 = x13344 + z13345 = x13345 + z13346 = x13346 + z13347 = x13347 + z13348 = x13348 + z13349 = x13349 + z13350 = x13350 + z13351 = x13351 + z13352 = x13352 + z13353 = x13353 + z13354 = x13354 + z13355 = x13355 + z13356 = x13356 + z13357 = x13357 + z13358 = x13358 + z13359 = x13359 + z13360 = x13360 + z13361 = x13361 + z13362 = x13362 + z13363 = x13363 + z13364 = x13364 + z13365 = x13365 + z13366 = x13366 + z13367 = x13367 + z13368 = x13368 + z13369 = x13369 + z13370 = x13370 + z13371 = x13371 + z13372 = x13372 + z13373 = x13373 + z13374 = x13374 + z13375 = x13375 + z13376 = x13376 + z13377 = x13377 + z13378 = x13378 + z13379 = x13379 + z13380 = x13380 + z13381 = x13381 + z13382 = x13382 + z13383 = x13383 + z13384 = x13384 + z13385 = x13385 + z13386 = x13386 + z13387 = x13387 + z13388 = x13388 + z13389 = x13389 + z13390 = x13390 + z13391 = x13391 + z13392 = x13392 + z13393 = x13393 + z13394 = x13394 + z13395 = x13395 + z13396 = x13396 + z13397 = x13397 + z13398 = x13398 + z13399 = x13399 + z13400 = x13400 + z13401 = x13401 + z13402 = x13402 + z13403 = x13403 + z13404 = x13404 + z13405 = x13405 + z13406 = x13406 + z13407 = x13407 + z13408 = x13408 + z13409 = x13409 + z13410 = x13410 + z13411 = x13411 + z13412 = x13412 + z13413 = x13413 + z13414 = x13414 + z13415 = x13415 + z13416 = x13416 + z13417 = x13417 + z13418 = x13418 + z13419 = x13419 + z13420 = x13420 + z13421 = x13421 + z13422 = x13422 + z13423 = x13423 + z13424 = x13424 + z13425 = x13425 + z13426 = x13426 + z13427 = x13427 + z13428 = x13428 + z13429 = x13429 + z13430 = x13430 + z13431 = x13431 + z13432 = x13432 + z13433 = x13433 + z13434 = x13434 + z13435 = x13435 + z13436 = x13436 + z13437 = x13437 + z13438 = x13438 + z13439 = x13439 + z13440 = x13440 + z13441 = x13441 + z13442 = x13442 + z13443 = x13443 + z13444 = x13444 + z13445 = x13445 + z13446 = x13446 + z13447 = x13447 + z13448 = x13448 + z13449 = x13449 + z13450 = x13450 + z13451 = x13451 + z13452 = x13452 + z13453 = x13453 + z13454 = x13454 + z13455 = x13455 + z13456 = x13456 + z13457 = x13457 + z13458 = x13458 + z13459 = x13459 + z13460 = x13460 + z13461 = x13461 + z13462 = x13462 + z13463 = x13463 + z13464 = x13464 + z13465 = x13465 + z13466 = x13466 + z13467 = x13467 + z13468 = x13468 + z13469 = x13469 + z13470 = x13470 + z13471 = x13471 + z13472 = x13472 + z13473 = x13473 + z13474 = x13474 + z13475 = x13475 + z13476 = x13476 + z13477 = x13477 + z13478 = x13478 + z13479 = x13479 + z13480 = x13480 + z13481 = x13481 + z13482 = x13482 + z13483 = x13483 + z13484 = x13484 + z13485 = x13485 + z13486 = x13486 + z13487 = x13487 + z13488 = x13488 + z13489 = x13489 + z13490 = x13490 + z13491 = x13491 + z13492 = x13492 + z13493 = x13493 + z13494 = x13494 + z13495 = x13495 + z13496 = x13496 + z13497 = x13497 + z13498 = x13498 + z13499 = x13499 + z13500 = x13500 + z13501 = x13501 + z13502 = x13502 + z13503 = x13503 + z13504 = x13504 + z13505 = x13505 + z13506 = x13506 + z13507 = x13507 + z13508 = x13508 + z13509 = x13509 + z13510 = x13510 + z13511 = x13511 + z13512 = x13512 + z13513 = x13513 + z13514 = x13514 + z13515 = x13515 + z13516 = x13516 + z13517 = x13517 + z13518 = x13518 + z13519 = x13519 + z13520 = x13520 + z13521 = x13521 + z13522 = x13522 + z13523 = x13523 + z13524 = x13524 + z13525 = x13525 + z13526 = x13526 + z13527 = x13527 + z13528 = x13528 + z13529 = x13529 + z13530 = x13530 + z13531 = x13531 + z13532 = x13532 + z13533 = x13533 + z13534 = x13534 + z13535 = x13535 + z13536 = x13536 + z13537 = x13537 + z13538 = x13538 + z13539 = x13539 + z13540 = x13540 + z13541 = x13541 + z13542 = x13542 + z13543 = x13543 + z13544 = x13544 + z13545 = x13545 + z13546 = x13546 + z13547 = x13547 + z13548 = x13548 + z13549 = x13549 + z13550 = x13550 + z13551 = x13551 + z13552 = x13552 + z13553 = x13553 + z13554 = x13554 + z13555 = x13555 + z13556 = x13556 + z13557 = x13557 + z13558 = x13558 + z13559 = x13559 + z13560 = x13560 + z13561 = x13561 + z13562 = x13562 + z13563 = x13563 + z13564 = x13564 + z13565 = x13565 + z13566 = x13566 + z13567 = x13567 + z13568 = x13568 + z13569 = x13569 + z13570 = x13570 + z13571 = x13571 + z13572 = x13572 + z13573 = x13573 + z13574 = x13574 + z13575 = x13575 + z13576 = x13576 + z13577 = x13577 + z13578 = x13578 + z13579 = x13579 + z13580 = x13580 + z13581 = x13581 + z13582 = x13582 + z13583 = x13583 + z13584 = x13584 + z13585 = x13585 + z13586 = x13586 + z13587 = x13587 + z13588 = x13588 + z13589 = x13589 + z13590 = x13590 + z13591 = x13591 + z13592 = x13592 + z13593 = x13593 + z13594 = x13594 + z13595 = x13595 + z13596 = x13596 + z13597 = x13597 + z13598 = x13598 + z13599 = x13599 + z13600 = x13600 + z13601 = x13601 + z13602 = x13602 + z13603 = x13603 + z13604 = x13604 + z13605 = x13605 + z13606 = x13606 + z13607 = x13607 + z13608 = x13608 + z13609 = x13609 + z13610 = x13610 + z13611 = x13611 + z13612 = x13612 + z13613 = x13613 + z13614 = x13614 + z13615 = x13615 + z13616 = x13616 + z13617 = x13617 + z13618 = x13618 + z13619 = x13619 + z13620 = x13620 + z13621 = x13621 + z13622 = x13622 + z13623 = x13623 + z13624 = x13624 + z13625 = x13625 + z13626 = x13626 + z13627 = x13627 + z13628 = x13628 + z13629 = x13629 + z13630 = x13630 + z13631 = x13631 + z13632 = x13632 + z13633 = x13633 + z13634 = x13634 + z13635 = x13635 + z13636 = x13636 + z13637 = x13637 + z13638 = x13638 + z13639 = x13639 + z13640 = x13640 + z13641 = x13641 + z13642 = x13642 + z13643 = x13643 + z13644 = x13644 + z13645 = x13645 + z13646 = x13646 + z13647 = x13647 + z13648 = x13648 + z13649 = x13649 + z13650 = x13650 + z13651 = x13651 + z13652 = x13652 + z13653 = x13653 + z13654 = x13654 + z13655 = x13655 + z13656 = x13656 + z13657 = x13657 + z13658 = x13658 + z13659 = x13659 + z13660 = x13660 + z13661 = x13661 + z13662 = x13662 + z13663 = x13663 + z13664 = x13664 + z13665 = x13665 + z13666 = x13666 + z13667 = x13667 + z13668 = x13668 + z13669 = x13669 + z13670 = x13670 + z13671 = x13671 + z13672 = x13672 + z13673 = x13673 + z13674 = x13674 + z13675 = x13675 + z13676 = x13676 + z13677 = x13677 + z13678 = x13678 + z13679 = x13679 + z13680 = x13680 + z13681 = x13681 + z13682 = x13682 + z13683 = x13683 + z13684 = x13684 + z13685 = x13685 + z13686 = x13686 + z13687 = x13687 + z13688 = x13688 + z13689 = x13689 + z13690 = x13690 + z13691 = x13691 + z13692 = x13692 + z13693 = x13693 + z13694 = x13694 + z13695 = x13695 + z13696 = x13696 + z13697 = x13697 + z13698 = x13698 + z13699 = x13699 + z13700 = x13700 + z13701 = x13701 + z13702 = x13702 + z13703 = x13703 + z13704 = x13704 + z13705 = x13705 + z13706 = x13706 + z13707 = x13707 + z13708 = x13708 + z13709 = x13709 + z13710 = x13710 + z13711 = x13711 + z13712 = x13712 + z13713 = x13713 + z13714 = x13714 + z13715 = x13715 + z13716 = x13716 + z13717 = x13717 + z13718 = x13718 + z13719 = x13719 + z13720 = x13720 + z13721 = x13721 + z13722 = x13722 + z13723 = x13723 + z13724 = x13724 + z13725 = x13725 + z13726 = x13726 + z13727 = x13727 + z13728 = x13728 + z13729 = x13729 + z13730 = x13730 + z13731 = x13731 + z13732 = x13732 + z13733 = x13733 + z13734 = x13734 + z13735 = x13735 + z13736 = x13736 + z13737 = x13737 + z13738 = x13738 + z13739 = x13739 + z13740 = x13740 + z13741 = x13741 + z13742 = x13742 + z13743 = x13743 + z13744 = x13744 + z13745 = x13745 + z13746 = x13746 + z13747 = x13747 + z13748 = x13748 + z13749 = x13749 + z13750 = x13750 + z13751 = x13751 + z13752 = x13752 + z13753 = x13753 + z13754 = x13754 + z13755 = x13755 + z13756 = x13756 + z13757 = x13757 + z13758 = x13758 + z13759 = x13759 + z13760 = x13760 + z13761 = x13761 + z13762 = x13762 + z13763 = x13763 + z13764 = x13764 + z13765 = x13765 + z13766 = x13766 + z13767 = x13767 + z13768 = x13768 + z13769 = x13769 + z13770 = x13770 + z13771 = x13771 + z13772 = x13772 + z13773 = x13773 + z13774 = x13774 + z13775 = x13775 + z13776 = x13776 + z13777 = x13777 + z13778 = x13778 + z13779 = x13779 + z13780 = x13780 + z13781 = x13781 + z13782 = x13782 + z13783 = x13783 + z13784 = x13784 + z13785 = x13785 + z13786 = x13786 + z13787 = x13787 + z13788 = x13788 + z13789 = x13789 + z13790 = x13790 + z13791 = x13791 + z13792 = x13792 + z13793 = x13793 + z13794 = x13794 + z13795 = x13795 + z13796 = x13796 + z13797 = x13797 + z13798 = x13798 + z13799 = x13799 + z13800 = x13800 + z13801 = x13801 + z13802 = x13802 + z13803 = x13803 + z13804 = x13804 + z13805 = x13805 + z13806 = x13806 + z13807 = x13807 + z13808 = x13808 + z13809 = x13809 + z13810 = x13810 + z13811 = x13811 + z13812 = x13812 + z13813 = x13813 + z13814 = x13814 + z13815 = x13815 + z13816 = x13816 + z13817 = x13817 + z13818 = x13818 + z13819 = x13819 + z13820 = x13820 + z13821 = x13821 + z13822 = x13822 + z13823 = x13823 + z13824 = x13824 + z13825 = x13825 + z13826 = x13826 + z13827 = x13827 + z13828 = x13828 + z13829 = x13829 + z13830 = x13830 + z13831 = x13831 + z13832 = x13832 + z13833 = x13833 + z13834 = x13834 + z13835 = x13835 + z13836 = x13836 + z13837 = x13837 + z13838 = x13838 + z13839 = x13839 + z13840 = x13840 + z13841 = x13841 + z13842 = x13842 + z13843 = x13843 + z13844 = x13844 + z13845 = x13845 + z13846 = x13846 + z13847 = x13847 + z13848 = x13848 + z13849 = x13849 + z13850 = x13850 + z13851 = x13851 + z13852 = x13852 + z13853 = x13853 + z13854 = x13854 + z13855 = x13855 + z13856 = x13856 + z13857 = x13857 + z13858 = x13858 + z13859 = x13859 + z13860 = x13860 + z13861 = x13861 + z13862 = x13862 + z13863 = x13863 + z13864 = x13864 + z13865 = x13865 + z13866 = x13866 + z13867 = x13867 + z13868 = x13868 + z13869 = x13869 + z13870 = x13870 + z13871 = x13871 + z13872 = x13872 + z13873 = x13873 + z13874 = x13874 + z13875 = x13875 + z13876 = x13876 + z13877 = x13877 + z13878 = x13878 + z13879 = x13879 + z13880 = x13880 + z13881 = x13881 + z13882 = x13882 + z13883 = x13883 + z13884 = x13884 + z13885 = x13885 + z13886 = x13886 + z13887 = x13887 + z13888 = x13888 + z13889 = x13889 + z13890 = x13890 + z13891 = x13891 + z13892 = x13892 + z13893 = x13893 + z13894 = x13894 + z13895 = x13895 + z13896 = x13896 + z13897 = x13897 + z13898 = x13898 + z13899 = x13899 + z13900 = x13900 + z13901 = x13901 + z13902 = x13902 + z13903 = x13903 + z13904 = x13904 + z13905 = x13905 + z13906 = x13906 + z13907 = x13907 + z13908 = x13908 + z13909 = x13909 + z13910 = x13910 + z13911 = x13911 + z13912 = x13912 + z13913 = x13913 + z13914 = x13914 + z13915 = x13915 + z13916 = x13916 + z13917 = x13917 + z13918 = x13918 + z13919 = x13919 + z13920 = x13920 + z13921 = x13921 + z13922 = x13922 + z13923 = x13923 + z13924 = x13924 + z13925 = x13925 + z13926 = x13926 + z13927 = x13927 + z13928 = x13928 + z13929 = x13929 + z13930 = x13930 + z13931 = x13931 + z13932 = x13932 + z13933 = x13933 + z13934 = x13934 + z13935 = x13935 + z13936 = x13936 + z13937 = x13937 + z13938 = x13938 + z13939 = x13939 + z13940 = x13940 + z13941 = x13941 + z13942 = x13942 + z13943 = x13943 + z13944 = x13944 + z13945 = x13945 + z13946 = x13946 + z13947 = x13947 + z13948 = x13948 + z13949 = x13949 + z13950 = x13950 + z13951 = x13951 + z13952 = x13952 + z13953 = x13953 + z13954 = x13954 + z13955 = x13955 + z13956 = x13956 + z13957 = x13957 + z13958 = x13958 + z13959 = x13959 + z13960 = x13960 + z13961 = x13961 + z13962 = x13962 + z13963 = x13963 + z13964 = x13964 + z13965 = x13965 + z13966 = x13966 + z13967 = x13967 + z13968 = x13968 + z13969 = x13969 + z13970 = x13970 + z13971 = x13971 + z13972 = x13972 + z13973 = x13973 + z13974 = x13974 + z13975 = x13975 + z13976 = x13976 + z13977 = x13977 + z13978 = x13978 + z13979 = x13979 + z13980 = x13980 + z13981 = x13981 + z13982 = x13982 + z13983 = x13983 + z13984 = x13984 + z13985 = x13985 + z13986 = x13986 + z13987 = x13987 + z13988 = x13988 + z13989 = x13989 + z13990 = x13990 + z13991 = x13991 + z13992 = x13992 + z13993 = x13993 + z13994 = x13994 + z13995 = x13995 + z13996 = x13996 + z13997 = x13997 + z13998 = x13998 + z13999 = x13999 + z14000 = x14000 + z14001 = x14001 + z14002 = x14002 + z14003 = x14003 + z14004 = x14004 + z14005 = x14005 + z14006 = x14006 + z14007 = x14007 + z14008 = x14008 + z14009 = x14009 + z14010 = x14010 + z14011 = x14011 + z14012 = x14012 + z14013 = x14013 + z14014 = x14014 + z14015 = x14015 + z14016 = x14016 + z14017 = x14017 + z14018 = x14018 + z14019 = x14019 + z14020 = x14020 + z14021 = x14021 + z14022 = x14022 + z14023 = x14023 + z14024 = x14024 + z14025 = x14025 + z14026 = x14026 + z14027 = x14027 + z14028 = x14028 + z14029 = x14029 + z14030 = x14030 + z14031 = x14031 + z14032 = x14032 + z14033 = x14033 + z14034 = x14034 + z14035 = x14035 + z14036 = x14036 + z14037 = x14037 + z14038 = x14038 + z14039 = x14039 + z14040 = x14040 + z14041 = x14041 + z14042 = x14042 + z14043 = x14043 + z14044 = x14044 + z14045 = x14045 + z14046 = x14046 + z14047 = x14047 + z14048 = x14048 + z14049 = x14049 + z14050 = x14050 + z14051 = x14051 + z14052 = x14052 + z14053 = x14053 + z14054 = x14054 + z14055 = x14055 + z14056 = x14056 + z14057 = x14057 + z14058 = x14058 + z14059 = x14059 + z14060 = x14060 + z14061 = x14061 + z14062 = x14062 + z14063 = x14063 + z14064 = x14064 + z14065 = x14065 + z14066 = x14066 + z14067 = x14067 + z14068 = x14068 + z14069 = x14069 + z14070 = x14070 + z14071 = x14071 + z14072 = x14072 + z14073 = x14073 + z14074 = x14074 + z14075 = x14075 + z14076 = x14076 + z14077 = x14077 + z14078 = x14078 + z14079 = x14079 + z14080 = x14080 + z14081 = x14081 + z14082 = x14082 + z14083 = x14083 + z14084 = x14084 + z14085 = x14085 + z14086 = x14086 + z14087 = x14087 + z14088 = x14088 + z14089 = x14089 + z14090 = x14090 + z14091 = x14091 + z14092 = x14092 + z14093 = x14093 + z14094 = x14094 + z14095 = x14095 + z14096 = x14096 + z14097 = x14097 + z14098 = x14098 + z14099 = x14099 + z14100 = x14100 + z14101 = x14101 + z14102 = x14102 + z14103 = x14103 + z14104 = x14104 + z14105 = x14105 + z14106 = x14106 + z14107 = x14107 + z14108 = x14108 + z14109 = x14109 + z14110 = x14110 + z14111 = x14111 + z14112 = x14112 + z14113 = x14113 + z14114 = x14114 + z14115 = x14115 + z14116 = x14116 + z14117 = x14117 + z14118 = x14118 + z14119 = x14119 + z14120 = x14120 + z14121 = x14121 + z14122 = x14122 + z14123 = x14123 + z14124 = x14124 + z14125 = x14125 + z14126 = x14126 + z14127 = x14127 + z14128 = x14128 + z14129 = x14129 + z14130 = x14130 + z14131 = x14131 + z14132 = x14132 + z14133 = x14133 + z14134 = x14134 + z14135 = x14135 + z14136 = x14136 + z14137 = x14137 + z14138 = x14138 + z14139 = x14139 + z14140 = x14140 + z14141 = x14141 + z14142 = x14142 + z14143 = x14143 + z14144 = x14144 + z14145 = x14145 + z14146 = x14146 + z14147 = x14147 + z14148 = x14148 + z14149 = x14149 + z14150 = x14150 + z14151 = x14151 + z14152 = x14152 + z14153 = x14153 + z14154 = x14154 + z14155 = x14155 + z14156 = x14156 + z14157 = x14157 + z14158 = x14158 + z14159 = x14159 + z14160 = x14160 + z14161 = x14161 + z14162 = x14162 + z14163 = x14163 + z14164 = x14164 + z14165 = x14165 + z14166 = x14166 + z14167 = x14167 + z14168 = x14168 + z14169 = x14169 + z14170 = x14170 + z14171 = x14171 + z14172 = x14172 + z14173 = x14173 + z14174 = x14174 + z14175 = x14175 + z14176 = x14176 + z14177 = x14177 + z14178 = x14178 + z14179 = x14179 + z14180 = x14180 + z14181 = x14181 + z14182 = x14182 + z14183 = x14183 + z14184 = x14184 + z14185 = x14185 + z14186 = x14186 + z14187 = x14187 + z14188 = x14188 + z14189 = x14189 + z14190 = x14190 + z14191 = x14191 + z14192 = x14192 + z14193 = x14193 + z14194 = x14194 + z14195 = x14195 + z14196 = x14196 + z14197 = x14197 + z14198 = x14198 + z14199 = x14199 + z14200 = x14200 + z14201 = x14201 + z14202 = x14202 + z14203 = x14203 + z14204 = x14204 + z14205 = x14205 + z14206 = x14206 + z14207 = x14207 + z14208 = x14208 + z14209 = x14209 + z14210 = x14210 + z14211 = x14211 + z14212 = x14212 + z14213 = x14213 + z14214 = x14214 + z14215 = x14215 + z14216 = x14216 + z14217 = x14217 + z14218 = x14218 + z14219 = x14219 + z14220 = x14220 + z14221 = x14221 + z14222 = x14222 + z14223 = x14223 + z14224 = x14224 + z14225 = x14225 + z14226 = x14226 + z14227 = x14227 + z14228 = x14228 + z14229 = x14229 + z14230 = x14230 + z14231 = x14231 + z14232 = x14232 + z14233 = x14233 + z14234 = x14234 + z14235 = x14235 + z14236 = x14236 + z14237 = x14237 + z14238 = x14238 + z14239 = x14239 + z14240 = x14240 + z14241 = x14241 + z14242 = x14242 + z14243 = x14243 + z14244 = x14244 + z14245 = x14245 + z14246 = x14246 + z14247 = x14247 + z14248 = x14248 + z14249 = x14249 + z14250 = x14250 + z14251 = x14251 + z14252 = x14252 + z14253 = x14253 + z14254 = x14254 + z14255 = x14255 + z14256 = x14256 + z14257 = x14257 + z14258 = x14258 + z14259 = x14259 + z14260 = x14260 + z14261 = x14261 + z14262 = x14262 + z14263 = x14263 + z14264 = x14264 + z14265 = x14265 + z14266 = x14266 + z14267 = x14267 + z14268 = x14268 + z14269 = x14269 + z14270 = x14270 + z14271 = x14271 + z14272 = x14272 + z14273 = x14273 + z14274 = x14274 + z14275 = x14275 + z14276 = x14276 + z14277 = x14277 + z14278 = x14278 + z14279 = x14279 + z14280 = x14280 + z14281 = x14281 + z14282 = x14282 + z14283 = x14283 + z14284 = x14284 + z14285 = x14285 + z14286 = x14286 + z14287 = x14287 + z14288 = x14288 + z14289 = x14289 + z14290 = x14290 + z14291 = x14291 + z14292 = x14292 + z14293 = x14293 + z14294 = x14294 + z14295 = x14295 + z14296 = x14296 + z14297 = x14297 + z14298 = x14298 + z14299 = x14299 + z14300 = x14300 + z14301 = x14301 + z14302 = x14302 + z14303 = x14303 + z14304 = x14304 + z14305 = x14305 + z14306 = x14306 + z14307 = x14307 + z14308 = x14308 + z14309 = x14309 + z14310 = x14310 + z14311 = x14311 + z14312 = x14312 + z14313 = x14313 + z14314 = x14314 + z14315 = x14315 + z14316 = x14316 + z14317 = x14317 + z14318 = x14318 + z14319 = x14319 + z14320 = x14320 + z14321 = x14321 + z14322 = x14322 + z14323 = x14323 + z14324 = x14324 + z14325 = x14325 + z14326 = x14326 + z14327 = x14327 + z14328 = x14328 + z14329 = x14329 + z14330 = x14330 + z14331 = x14331 + z14332 = x14332 + z14333 = x14333 + z14334 = x14334 + z14335 = x14335 + z14336 = x14336 + z14337 = x14337 + z14338 = x14338 + z14339 = x14339 + z14340 = x14340 + z14341 = x14341 + z14342 = x14342 + z14343 = x14343 + z14344 = x14344 + z14345 = x14345 + z14346 = x14346 + z14347 = x14347 + z14348 = x14348 + z14349 = x14349 + z14350 = x14350 + z14351 = x14351 + z14352 = x14352 + z14353 = x14353 + z14354 = x14354 + z14355 = x14355 + z14356 = x14356 + z14357 = x14357 + z14358 = x14358 + z14359 = x14359 + z14360 = x14360 + z14361 = x14361 + z14362 = x14362 + z14363 = x14363 + z14364 = x14364 + z14365 = x14365 + z14366 = x14366 + z14367 = x14367 + z14368 = x14368 + z14369 = x14369 + z14370 = x14370 + z14371 = x14371 + z14372 = x14372 + z14373 = x14373 + z14374 = x14374 + z14375 = x14375 + z14376 = x14376 + z14377 = x14377 + z14378 = x14378 + z14379 = x14379 + z14380 = x14380 + z14381 = x14381 + z14382 = x14382 + z14383 = x14383 + z14384 = x14384 + z14385 = x14385 + z14386 = x14386 + z14387 = x14387 + z14388 = x14388 + z14389 = x14389 + z14390 = x14390 + z14391 = x14391 + z14392 = x14392 + z14393 = x14393 + z14394 = x14394 + z14395 = x14395 + z14396 = x14396 + z14397 = x14397 + z14398 = x14398 + z14399 = x14399 + z14400 = x14400 + z14401 = x14401 + z14402 = x14402 + z14403 = x14403 + z14404 = x14404 + z14405 = x14405 + z14406 = x14406 + z14407 = x14407 + z14408 = x14408 + z14409 = x14409 + z14410 = x14410 + z14411 = x14411 + z14412 = x14412 + z14413 = x14413 + z14414 = x14414 + z14415 = x14415 + z14416 = x14416 + z14417 = x14417 + z14418 = x14418 + z14419 = x14419 + z14420 = x14420 + z14421 = x14421 + z14422 = x14422 + z14423 = x14423 + z14424 = x14424 + z14425 = x14425 + z14426 = x14426 + z14427 = x14427 + z14428 = x14428 + z14429 = x14429 + z14430 = x14430 + z14431 = x14431 + z14432 = x14432 + z14433 = x14433 + z14434 = x14434 + z14435 = x14435 + z14436 = x14436 + z14437 = x14437 + z14438 = x14438 + z14439 = x14439 + z14440 = x14440 + z14441 = x14441 + z14442 = x14442 + z14443 = x14443 + z14444 = x14444 + z14445 = x14445 + z14446 = x14446 + z14447 = x14447 + z14448 = x14448 + z14449 = x14449 + z14450 = x14450 + z14451 = x14451 + z14452 = x14452 + z14453 = x14453 + z14454 = x14454 + z14455 = x14455 + z14456 = x14456 + z14457 = x14457 + z14458 = x14458 + z14459 = x14459 + z14460 = x14460 + z14461 = x14461 + z14462 = x14462 + z14463 = x14463 + z14464 = x14464 + z14465 = x14465 + z14466 = x14466 + z14467 = x14467 + z14468 = x14468 + z14469 = x14469 + z14470 = x14470 + z14471 = x14471 + z14472 = x14472 + z14473 = x14473 + z14474 = x14474 + z14475 = x14475 + z14476 = x14476 + z14477 = x14477 + z14478 = x14478 + z14479 = x14479 + z14480 = x14480 + z14481 = x14481 + z14482 = x14482 + z14483 = x14483 + z14484 = x14484 + z14485 = x14485 + z14486 = x14486 + z14487 = x14487 + z14488 = x14488 + z14489 = x14489 + z14490 = x14490 + z14491 = x14491 + z14492 = x14492 + z14493 = x14493 + z14494 = x14494 + z14495 = x14495 + z14496 = x14496 + z14497 = x14497 + z14498 = x14498 + z14499 = x14499 + z14500 = x14500 + z14501 = x14501 + z14502 = x14502 + z14503 = x14503 + z14504 = x14504 + z14505 = x14505 + z14506 = x14506 + z14507 = x14507 + z14508 = x14508 + z14509 = x14509 + z14510 = x14510 + z14511 = x14511 + z14512 = x14512 + z14513 = x14513 + z14514 = x14514 + z14515 = x14515 + z14516 = x14516 + z14517 = x14517 + z14518 = x14518 + z14519 = x14519 + z14520 = x14520 + z14521 = x14521 + z14522 = x14522 + z14523 = x14523 + z14524 = x14524 + z14525 = x14525 + z14526 = x14526 + z14527 = x14527 + z14528 = x14528 + z14529 = x14529 + z14530 = x14530 + z14531 = x14531 + z14532 = x14532 + z14533 = x14533 + z14534 = x14534 + z14535 = x14535 + z14536 = x14536 + z14537 = x14537 + z14538 = x14538 + z14539 = x14539 + z14540 = x14540 + z14541 = x14541 + z14542 = x14542 + z14543 = x14543 + z14544 = x14544 + z14545 = x14545 + z14546 = x14546 + z14547 = x14547 + z14548 = x14548 + z14549 = x14549 + z14550 = x14550 + z14551 = x14551 + z14552 = x14552 + z14553 = x14553 + z14554 = x14554 + z14555 = x14555 + z14556 = x14556 + z14557 = x14557 + z14558 = x14558 + z14559 = x14559 + z14560 = x14560 + z14561 = x14561 + z14562 = x14562 + z14563 = x14563 + z14564 = x14564 + z14565 = x14565 + z14566 = x14566 + z14567 = x14567 + z14568 = x14568 + z14569 = x14569 + z14570 = x14570 + z14571 = x14571 + z14572 = x14572 + z14573 = x14573 + z14574 = x14574 + z14575 = x14575 + z14576 = x14576 + z14577 = x14577 + z14578 = x14578 + z14579 = x14579 + z14580 = x14580 + z14581 = x14581 + z14582 = x14582 + z14583 = x14583 + z14584 = x14584 + z14585 = x14585 + z14586 = x14586 + z14587 = x14587 + z14588 = x14588 + z14589 = x14589 + z14590 = x14590 + z14591 = x14591 + z14592 = x14592 + z14593 = x14593 + z14594 = x14594 + z14595 = x14595 + z14596 = x14596 + z14597 = x14597 + z14598 = x14598 + z14599 = x14599 + z14600 = x14600 + z14601 = x14601 + z14602 = x14602 + z14603 = x14603 + z14604 = x14604 + z14605 = x14605 + z14606 = x14606 + z14607 = x14607 + z14608 = x14608 + z14609 = x14609 + z14610 = x14610 + z14611 = x14611 + z14612 = x14612 + z14613 = x14613 + z14614 = x14614 + z14615 = x14615 + z14616 = x14616 + z14617 = x14617 + z14618 = x14618 + z14619 = x14619 + z14620 = x14620 + z14621 = x14621 + z14622 = x14622 + z14623 = x14623 + z14624 = x14624 + z14625 = x14625 + z14626 = x14626 + z14627 = x14627 + z14628 = x14628 + z14629 = x14629 + z14630 = x14630 + z14631 = x14631 + z14632 = x14632 + z14633 = x14633 + z14634 = x14634 + z14635 = x14635 + z14636 = x14636 + z14637 = x14637 + z14638 = x14638 + z14639 = x14639 + z14640 = x14640 + z14641 = x14641 + z14642 = x14642 + z14643 = x14643 + z14644 = x14644 + z14645 = x14645 + z14646 = x14646 + z14647 = x14647 + z14648 = x14648 + z14649 = x14649 + z14650 = x14650 + z14651 = x14651 + z14652 = x14652 + z14653 = x14653 + z14654 = x14654 + z14655 = x14655 + z14656 = x14656 + z14657 = x14657 + z14658 = x14658 + z14659 = x14659 + z14660 = x14660 + z14661 = x14661 + z14662 = x14662 + z14663 = x14663 + z14664 = x14664 + z14665 = x14665 + z14666 = x14666 + z14667 = x14667 + z14668 = x14668 + z14669 = x14669 + z14670 = x14670 + z14671 = x14671 + z14672 = x14672 + z14673 = x14673 + z14674 = x14674 + z14675 = x14675 + z14676 = x14676 + z14677 = x14677 + z14678 = x14678 + z14679 = x14679 + z14680 = x14680 + z14681 = x14681 + z14682 = x14682 + z14683 = x14683 + z14684 = x14684 + z14685 = x14685 + z14686 = x14686 + z14687 = x14687 + z14688 = x14688 + z14689 = x14689 + z14690 = x14690 + z14691 = x14691 + z14692 = x14692 + z14693 = x14693 + z14694 = x14694 + z14695 = x14695 + z14696 = x14696 + z14697 = x14697 + z14698 = x14698 + z14699 = x14699 + z14700 = x14700 + z14701 = x14701 + z14702 = x14702 + z14703 = x14703 + z14704 = x14704 + z14705 = x14705 + z14706 = x14706 + z14707 = x14707 + z14708 = x14708 + z14709 = x14709 + z14710 = x14710 + z14711 = x14711 + z14712 = x14712 + z14713 = x14713 + z14714 = x14714 + z14715 = x14715 + z14716 = x14716 + z14717 = x14717 + z14718 = x14718 + z14719 = x14719 + z14720 = x14720 + z14721 = x14721 + z14722 = x14722 + z14723 = x14723 + z14724 = x14724 + z14725 = x14725 + z14726 = x14726 + z14727 = x14727 + z14728 = x14728 + z14729 = x14729 + z14730 = x14730 + z14731 = x14731 + z14732 = x14732 + z14733 = x14733 + z14734 = x14734 + z14735 = x14735 + z14736 = x14736 + z14737 = x14737 + z14738 = x14738 + z14739 = x14739 + z14740 = x14740 + z14741 = x14741 + z14742 = x14742 + z14743 = x14743 + z14744 = x14744 + z14745 = x14745 + z14746 = x14746 + z14747 = x14747 + z14748 = x14748 + z14749 = x14749 + z14750 = x14750 + z14751 = x14751 + z14752 = x14752 + z14753 = x14753 + z14754 = x14754 + z14755 = x14755 + z14756 = x14756 + z14757 = x14757 + z14758 = x14758 + z14759 = x14759 + z14760 = x14760 + z14761 = x14761 + z14762 = x14762 + z14763 = x14763 + z14764 = x14764 + z14765 = x14765 + z14766 = x14766 + z14767 = x14767 + z14768 = x14768 + z14769 = x14769 + z14770 = x14770 + z14771 = x14771 + z14772 = x14772 + z14773 = x14773 + z14774 = x14774 + z14775 = x14775 + z14776 = x14776 + z14777 = x14777 + z14778 = x14778 + z14779 = x14779 + z14780 = x14780 + z14781 = x14781 + z14782 = x14782 + z14783 = x14783 + z14784 = x14784 + z14785 = x14785 + z14786 = x14786 + z14787 = x14787 + z14788 = x14788 + z14789 = x14789 + z14790 = x14790 + z14791 = x14791 + z14792 = x14792 + z14793 = x14793 + z14794 = x14794 + z14795 = x14795 + z14796 = x14796 + z14797 = x14797 + z14798 = x14798 + z14799 = x14799 + z14800 = x14800 + z14801 = x14801 + z14802 = x14802 + z14803 = x14803 + z14804 = x14804 + z14805 = x14805 + z14806 = x14806 + z14807 = x14807 + z14808 = x14808 + z14809 = x14809 + z14810 = x14810 + z14811 = x14811 + z14812 = x14812 + z14813 = x14813 + z14814 = x14814 + z14815 = x14815 + z14816 = x14816 + z14817 = x14817 + z14818 = x14818 + z14819 = x14819 + z14820 = x14820 + z14821 = x14821 + z14822 = x14822 + z14823 = x14823 + z14824 = x14824 + z14825 = x14825 + z14826 = x14826 + z14827 = x14827 + z14828 = x14828 + z14829 = x14829 + z14830 = x14830 + z14831 = x14831 + z14832 = x14832 + z14833 = x14833 + z14834 = x14834 + z14835 = x14835 + z14836 = x14836 + z14837 = x14837 + z14838 = x14838 + z14839 = x14839 + z14840 = x14840 + z14841 = x14841 + z14842 = x14842 + z14843 = x14843 + z14844 = x14844 + z14845 = x14845 + z14846 = x14846 + z14847 = x14847 + z14848 = x14848 + z14849 = x14849 + z14850 = x14850 + z14851 = x14851 + z14852 = x14852 + z14853 = x14853 + z14854 = x14854 + z14855 = x14855 + z14856 = x14856 + z14857 = x14857 + z14858 = x14858 + z14859 = x14859 + z14860 = x14860 + z14861 = x14861 + z14862 = x14862 + z14863 = x14863 + z14864 = x14864 + z14865 = x14865 + z14866 = x14866 + z14867 = x14867 + z14868 = x14868 + z14869 = x14869 + z14870 = x14870 + z14871 = x14871 + z14872 = x14872 + z14873 = x14873 + z14874 = x14874 + z14875 = x14875 + z14876 = x14876 + z14877 = x14877 + z14878 = x14878 + z14879 = x14879 + z14880 = x14880 + z14881 = x14881 + z14882 = x14882 + z14883 = x14883 + z14884 = x14884 + z14885 = x14885 + z14886 = x14886 + z14887 = x14887 + z14888 = x14888 + z14889 = x14889 + z14890 = x14890 + z14891 = x14891 + z14892 = x14892 + z14893 = x14893 + z14894 = x14894 + z14895 = x14895 + z14896 = x14896 + z14897 = x14897 + z14898 = x14898 + z14899 = x14899 + z14900 = x14900 + z14901 = x14901 + z14902 = x14902 + z14903 = x14903 + z14904 = x14904 + z14905 = x14905 + z14906 = x14906 + z14907 = x14907 + z14908 = x14908 + z14909 = x14909 + z14910 = x14910 + z14911 = x14911 + z14912 = x14912 + z14913 = x14913 + z14914 = x14914 + z14915 = x14915 + z14916 = x14916 + z14917 = x14917 + z14918 = x14918 + z14919 = x14919 + z14920 = x14920 + z14921 = x14921 + z14922 = x14922 + z14923 = x14923 + z14924 = x14924 + z14925 = x14925 + z14926 = x14926 + z14927 = x14927 + z14928 = x14928 + z14929 = x14929 + z14930 = x14930 + z14931 = x14931 + z14932 = x14932 + z14933 = x14933 + z14934 = x14934 + z14935 = x14935 + z14936 = x14936 + z14937 = x14937 + z14938 = x14938 + z14939 = x14939 + z14940 = x14940 + z14941 = x14941 + z14942 = x14942 + z14943 = x14943 + z14944 = x14944 + z14945 = x14945 + z14946 = x14946 + z14947 = x14947 + z14948 = x14948 + z14949 = x14949 + z14950 = x14950 + z14951 = x14951 + z14952 = x14952 + z14953 = x14953 + z14954 = x14954 + z14955 = x14955 + z14956 = x14956 + z14957 = x14957 + z14958 = x14958 + z14959 = x14959 + z14960 = x14960 + z14961 = x14961 + z14962 = x14962 + z14963 = x14963 + z14964 = x14964 + z14965 = x14965 + z14966 = x14966 + z14967 = x14967 + z14968 = x14968 + z14969 = x14969 + z14970 = x14970 + z14971 = x14971 + z14972 = x14972 + z14973 = x14973 + z14974 = x14974 + z14975 = x14975 + z14976 = x14976 + z14977 = x14977 + z14978 = x14978 + z14979 = x14979 + z14980 = x14980 + z14981 = x14981 + z14982 = x14982 + z14983 = x14983 + z14984 = x14984 + z14985 = x14985 + z14986 = x14986 + z14987 = x14987 + z14988 = x14988 + z14989 = x14989 + z14990 = x14990 + z14991 = x14991 + z14992 = x14992 + z14993 = x14993 + z14994 = x14994 + z14995 = x14995 + z14996 = x14996 + z14997 = x14997 + z14998 = x14998 + z14999 = x14999 + z15000 = x15000 + z15001 = x15001 + z15002 = x15002 + z15003 = x15003 + z15004 = x15004 + z15005 = x15005 + z15006 = x15006 + z15007 = x15007 + z15008 = x15008 + z15009 = x15009 + z15010 = x15010 + z15011 = x15011 + z15012 = x15012 + z15013 = x15013 + z15014 = x15014 + z15015 = x15015 + z15016 = x15016 + z15017 = x15017 + z15018 = x15018 + z15019 = x15019 + z15020 = x15020 + z15021 = x15021 + z15022 = x15022 + z15023 = x15023 + z15024 = x15024 + z15025 = x15025 + z15026 = x15026 + z15027 = x15027 + z15028 = x15028 + z15029 = x15029 + z15030 = x15030 + z15031 = x15031 + z15032 = x15032 + z15033 = x15033 + z15034 = x15034 + z15035 = x15035 + z15036 = x15036 + z15037 = x15037 + z15038 = x15038 + z15039 = x15039 + z15040 = x15040 + z15041 = x15041 + z15042 = x15042 + z15043 = x15043 + z15044 = x15044 + z15045 = x15045 + z15046 = x15046 + z15047 = x15047 + z15048 = x15048 + z15049 = x15049 + z15050 = x15050 + z15051 = x15051 + z15052 = x15052 + z15053 = x15053 + z15054 = x15054 + z15055 = x15055 + z15056 = x15056 + z15057 = x15057 + z15058 = x15058 + z15059 = x15059 + z15060 = x15060 + z15061 = x15061 + z15062 = x15062 + z15063 = x15063 + z15064 = x15064 + z15065 = x15065 + z15066 = x15066 + z15067 = x15067 + z15068 = x15068 + z15069 = x15069 + z15070 = x15070 + z15071 = x15071 + z15072 = x15072 + z15073 = x15073 + z15074 = x15074 + z15075 = x15075 + z15076 = x15076 + z15077 = x15077 + z15078 = x15078 + z15079 = x15079 + z15080 = x15080 + z15081 = x15081 + z15082 = x15082 + z15083 = x15083 + z15084 = x15084 + z15085 = x15085 + z15086 = x15086 + z15087 = x15087 + z15088 = x15088 + z15089 = x15089 + z15090 = x15090 + z15091 = x15091 + z15092 = x15092 + z15093 = x15093 + z15094 = x15094 + z15095 = x15095 + z15096 = x15096 + z15097 = x15097 + z15098 = x15098 + z15099 = x15099 + z15100 = x15100 + z15101 = x15101 + z15102 = x15102 + z15103 = x15103 + z15104 = x15104 + z15105 = x15105 + z15106 = x15106 + z15107 = x15107 + z15108 = x15108 + z15109 = x15109 + z15110 = x15110 + z15111 = x15111 + z15112 = x15112 + z15113 = x15113 + z15114 = x15114 + z15115 = x15115 + z15116 = x15116 + z15117 = x15117 + z15118 = x15118 + z15119 = x15119 + z15120 = x15120 + z15121 = x15121 + z15122 = x15122 + z15123 = x15123 + z15124 = x15124 + z15125 = x15125 + z15126 = x15126 + z15127 = x15127 + z15128 = x15128 + z15129 = x15129 + z15130 = x15130 + z15131 = x15131 + z15132 = x15132 + z15133 = x15133 + z15134 = x15134 + z15135 = x15135 + z15136 = x15136 + z15137 = x15137 + z15138 = x15138 + z15139 = x15139 + z15140 = x15140 + z15141 = x15141 + z15142 = x15142 + z15143 = x15143 + z15144 = x15144 + z15145 = x15145 + z15146 = x15146 + z15147 = x15147 + z15148 = x15148 + z15149 = x15149 + z15150 = x15150 + z15151 = x15151 + z15152 = x15152 + z15153 = x15153 + z15154 = x15154 + z15155 = x15155 + z15156 = x15156 + z15157 = x15157 + z15158 = x15158 + z15159 = x15159 + z15160 = x15160 + z15161 = x15161 + z15162 = x15162 + z15163 = x15163 + z15164 = x15164 + z15165 = x15165 + z15166 = x15166 + z15167 = x15167 + z15168 = x15168 + z15169 = x15169 + z15170 = x15170 + z15171 = x15171 + z15172 = x15172 + z15173 = x15173 + z15174 = x15174 + z15175 = x15175 + z15176 = x15176 + z15177 = x15177 + z15178 = x15178 + z15179 = x15179 + z15180 = x15180 + z15181 = x15181 + z15182 = x15182 + z15183 = x15183 + z15184 = x15184 + z15185 = x15185 + z15186 = x15186 + z15187 = x15187 + z15188 = x15188 + z15189 = x15189 + z15190 = x15190 + z15191 = x15191 + z15192 = x15192 + z15193 = x15193 + z15194 = x15194 + z15195 = x15195 + z15196 = x15196 + z15197 = x15197 + z15198 = x15198 + z15199 = x15199 + z15200 = x15200 + z15201 = x15201 + z15202 = x15202 + z15203 = x15203 + z15204 = x15204 + z15205 = x15205 + z15206 = x15206 + z15207 = x15207 + z15208 = x15208 + z15209 = x15209 + z15210 = x15210 + z15211 = x15211 + z15212 = x15212 + z15213 = x15213 + z15214 = x15214 + z15215 = x15215 + z15216 = x15216 + z15217 = x15217 + z15218 = x15218 + z15219 = x15219 + z15220 = x15220 + z15221 = x15221 + z15222 = x15222 + z15223 = x15223 + z15224 = x15224 + z15225 = x15225 + z15226 = x15226 + z15227 = x15227 + z15228 = x15228 + z15229 = x15229 + z15230 = x15230 + z15231 = x15231 + z15232 = x15232 + z15233 = x15233 + z15234 = x15234 + z15235 = x15235 + z15236 = x15236 + z15237 = x15237 + z15238 = x15238 + z15239 = x15239 + z15240 = x15240 + z15241 = x15241 + z15242 = x15242 + z15243 = x15243 + z15244 = x15244 + z15245 = x15245 + z15246 = x15246 + z15247 = x15247 + z15248 = x15248 + z15249 = x15249 + z15250 = x15250 + z15251 = x15251 + z15252 = x15252 + z15253 = x15253 + z15254 = x15254 + z15255 = x15255 + z15256 = x15256 + z15257 = x15257 + z15258 = x15258 + z15259 = x15259 + z15260 = x15260 + z15261 = x15261 + z15262 = x15262 + z15263 = x15263 + z15264 = x15264 + z15265 = x15265 + z15266 = x15266 + z15267 = x15267 + z15268 = x15268 + z15269 = x15269 + z15270 = x15270 + z15271 = x15271 + z15272 = x15272 + z15273 = x15273 + z15274 = x15274 + z15275 = x15275 + z15276 = x15276 + z15277 = x15277 + z15278 = x15278 + z15279 = x15279 + z15280 = x15280 + z15281 = x15281 + z15282 = x15282 + z15283 = x15283 + z15284 = x15284 + z15285 = x15285 + z15286 = x15286 + z15287 = x15287 + z15288 = x15288 + z15289 = x15289 + z15290 = x15290 + z15291 = x15291 + z15292 = x15292 + z15293 = x15293 + z15294 = x15294 + z15295 = x15295 + z15296 = x15296 + z15297 = x15297 + z15298 = x15298 + z15299 = x15299 + z15300 = x15300 + z15301 = x15301 + z15302 = x15302 + z15303 = x15303 + z15304 = x15304 + z15305 = x15305 + z15306 = x15306 + z15307 = x15307 + z15308 = x15308 + z15309 = x15309 + z15310 = x15310 + z15311 = x15311 + z15312 = x15312 + z15313 = x15313 + z15314 = x15314 + z15315 = x15315 + z15316 = x15316 + z15317 = x15317 + z15318 = x15318 + z15319 = x15319 + z15320 = x15320 + z15321 = x15321 + z15322 = x15322 + z15323 = x15323 + z15324 = x15324 + z15325 = x15325 + z15326 = x15326 + z15327 = x15327 + z15328 = x15328 + z15329 = x15329 + z15330 = x15330 + z15331 = x15331 + z15332 = x15332 + z15333 = x15333 + z15334 = x15334 + z15335 = x15335 + z15336 = x15336 + z15337 = x15337 + z15338 = x15338 + z15339 = x15339 + z15340 = x15340 + z15341 = x15341 + z15342 = x15342 + z15343 = x15343 + z15344 = x15344 + z15345 = x15345 + z15346 = x15346 + z15347 = x15347 + z15348 = x15348 + z15349 = x15349 + z15350 = x15350 + z15351 = x15351 + z15352 = x15352 + z15353 = x15353 + z15354 = x15354 + z15355 = x15355 + z15356 = x15356 + z15357 = x15357 + z15358 = x15358 + z15359 = x15359 + z15360 = x15360 + z15361 = x15361 + z15362 = x15362 + z15363 = x15363 + z15364 = x15364 + z15365 = x15365 + z15366 = x15366 + z15367 = x15367 + z15368 = x15368 + z15369 = x15369 + z15370 = x15370 + z15371 = x15371 + z15372 = x15372 + z15373 = x15373 + z15374 = x15374 + z15375 = x15375 + z15376 = x15376 + z15377 = x15377 + z15378 = x15378 + z15379 = x15379 + z15380 = x15380 + z15381 = x15381 + z15382 = x15382 + z15383 = x15383 + z15384 = x15384 + z15385 = x15385 + z15386 = x15386 + z15387 = x15387 + z15388 = x15388 + z15389 = x15389 + z15390 = x15390 + z15391 = x15391 + z15392 = x15392 + z15393 = x15393 + z15394 = x15394 + z15395 = x15395 + z15396 = x15396 + z15397 = x15397 + z15398 = x15398 + z15399 = x15399 + z15400 = x15400 + z15401 = x15401 + z15402 = x15402 + z15403 = x15403 + z15404 = x15404 + z15405 = x15405 + z15406 = x15406 + z15407 = x15407 + z15408 = x15408 + z15409 = x15409 + z15410 = x15410 + z15411 = x15411 + z15412 = x15412 + z15413 = x15413 + z15414 = x15414 + z15415 = x15415 + z15416 = x15416 + z15417 = x15417 + z15418 = x15418 + z15419 = x15419 + z15420 = x15420 + z15421 = x15421 + z15422 = x15422 + z15423 = x15423 + z15424 = x15424 + z15425 = x15425 + z15426 = x15426 + z15427 = x15427 + z15428 = x15428 + z15429 = x15429 + z15430 = x15430 + z15431 = x15431 + z15432 = x15432 + z15433 = x15433 + z15434 = x15434 + z15435 = x15435 + z15436 = x15436 + z15437 = x15437 + z15438 = x15438 + z15439 = x15439 + z15440 = x15440 + z15441 = x15441 + z15442 = x15442 + z15443 = x15443 + z15444 = x15444 + z15445 = x15445 + z15446 = x15446 + z15447 = x15447 + z15448 = x15448 + z15449 = x15449 + z15450 = x15450 + z15451 = x15451 + z15452 = x15452 + z15453 = x15453 + z15454 = x15454 + z15455 = x15455 + z15456 = x15456 + z15457 = x15457 + z15458 = x15458 + z15459 = x15459 + z15460 = x15460 + z15461 = x15461 + z15462 = x15462 + z15463 = x15463 + z15464 = x15464 + z15465 = x15465 + z15466 = x15466 + z15467 = x15467 + z15468 = x15468 + z15469 = x15469 + z15470 = x15470 + z15471 = x15471 + z15472 = x15472 + z15473 = x15473 + z15474 = x15474 + z15475 = x15475 + z15476 = x15476 + z15477 = x15477 + z15478 = x15478 + z15479 = x15479 + z15480 = x15480 + z15481 = x15481 + z15482 = x15482 + z15483 = x15483 + z15484 = x15484 + z15485 = x15485 + z15486 = x15486 + z15487 = x15487 + z15488 = x15488 + z15489 = x15489 + z15490 = x15490 + z15491 = x15491 + z15492 = x15492 + z15493 = x15493 + z15494 = x15494 + z15495 = x15495 + z15496 = x15496 + z15497 = x15497 + z15498 = x15498 + z15499 = x15499 + z15500 = x15500 + z15501 = x15501 + z15502 = x15502 + z15503 = x15503 + z15504 = x15504 + z15505 = x15505 + z15506 = x15506 + z15507 = x15507 + z15508 = x15508 + z15509 = x15509 + z15510 = x15510 + z15511 = x15511 + z15512 = x15512 + z15513 = x15513 + z15514 = x15514 + z15515 = x15515 + z15516 = x15516 + z15517 = x15517 + z15518 = x15518 + z15519 = x15519 + z15520 = x15520 + z15521 = x15521 + z15522 = x15522 + z15523 = x15523 + z15524 = x15524 + z15525 = x15525 + z15526 = x15526 + z15527 = x15527 + z15528 = x15528 + z15529 = x15529 + z15530 = x15530 + z15531 = x15531 + z15532 = x15532 + z15533 = x15533 + z15534 = x15534 + z15535 = x15535 + z15536 = x15536 + z15537 = x15537 + z15538 = x15538 + z15539 = x15539 + z15540 = x15540 + z15541 = x15541 + z15542 = x15542 + z15543 = x15543 + z15544 = x15544 + z15545 = x15545 + z15546 = x15546 + z15547 = x15547 + z15548 = x15548 + z15549 = x15549 + z15550 = x15550 + z15551 = x15551 + z15552 = x15552 + z15553 = x15553 + z15554 = x15554 + z15555 = x15555 + z15556 = x15556 + z15557 = x15557 + z15558 = x15558 + z15559 = x15559 + z15560 = x15560 + z15561 = x15561 + z15562 = x15562 + z15563 = x15563 + z15564 = x15564 + z15565 = x15565 + z15566 = x15566 + z15567 = x15567 + z15568 = x15568 + z15569 = x15569 + z15570 = x15570 + z15571 = x15571 + z15572 = x15572 + z15573 = x15573 + z15574 = x15574 + z15575 = x15575 + z15576 = x15576 + z15577 = x15577 + z15578 = x15578 + z15579 = x15579 + z15580 = x15580 + z15581 = x15581 + z15582 = x15582 + z15583 = x15583 + z15584 = x15584 + z15585 = x15585 + z15586 = x15586 + z15587 = x15587 + z15588 = x15588 + z15589 = x15589 + z15590 = x15590 + z15591 = x15591 + z15592 = x15592 + z15593 = x15593 + z15594 = x15594 + z15595 = x15595 + z15596 = x15596 + z15597 = x15597 + z15598 = x15598 + z15599 = x15599 + z15600 = x15600 + z15601 = x15601 + z15602 = x15602 + z15603 = x15603 + z15604 = x15604 + z15605 = x15605 + z15606 = x15606 + z15607 = x15607 + z15608 = x15608 + z15609 = x15609 + z15610 = x15610 + z15611 = x15611 + z15612 = x15612 + z15613 = x15613 + z15614 = x15614 + z15615 = x15615 + z15616 = x15616 + z15617 = x15617 + z15618 = x15618 + z15619 = x15619 + z15620 = x15620 + z15621 = x15621 + z15622 = x15622 + z15623 = x15623 + z15624 = x15624 + z15625 = x15625 + z15626 = x15626 + z15627 = x15627 + z15628 = x15628 + z15629 = x15629 + z15630 = x15630 + z15631 = x15631 + z15632 = x15632 + z15633 = x15633 + z15634 = x15634 + z15635 = x15635 + z15636 = x15636 + z15637 = x15637 + z15638 = x15638 + z15639 = x15639 + z15640 = x15640 + z15641 = x15641 + z15642 = x15642 + z15643 = x15643 + z15644 = x15644 + z15645 = x15645 + z15646 = x15646 + z15647 = x15647 + z15648 = x15648 + z15649 = x15649 + z15650 = x15650 + z15651 = x15651 + z15652 = x15652 + z15653 = x15653 + z15654 = x15654 + z15655 = x15655 + z15656 = x15656 + z15657 = x15657 + z15658 = x15658 + z15659 = x15659 + z15660 = x15660 + z15661 = x15661 + z15662 = x15662 + z15663 = x15663 + z15664 = x15664 + z15665 = x15665 + z15666 = x15666 + z15667 = x15667 + z15668 = x15668 + z15669 = x15669 + z15670 = x15670 + z15671 = x15671 + z15672 = x15672 + z15673 = x15673 + z15674 = x15674 + z15675 = x15675 + z15676 = x15676 + z15677 = x15677 + z15678 = x15678 + z15679 = x15679 + z15680 = x15680 + z15681 = x15681 + z15682 = x15682 + z15683 = x15683 + z15684 = x15684 + z15685 = x15685 + z15686 = x15686 + z15687 = x15687 + z15688 = x15688 + z15689 = x15689 + z15690 = x15690 + z15691 = x15691 + z15692 = x15692 + z15693 = x15693 + z15694 = x15694 + z15695 = x15695 + z15696 = x15696 + z15697 = x15697 + z15698 = x15698 + z15699 = x15699 + z15700 = x15700 + z15701 = x15701 + z15702 = x15702 + z15703 = x15703 + z15704 = x15704 + z15705 = x15705 + z15706 = x15706 + z15707 = x15707 + z15708 = x15708 + z15709 = x15709 + z15710 = x15710 + z15711 = x15711 + z15712 = x15712 + z15713 = x15713 + z15714 = x15714 + z15715 = x15715 + z15716 = x15716 + z15717 = x15717 + z15718 = x15718 + z15719 = x15719 + z15720 = x15720 + z15721 = x15721 + z15722 = x15722 + z15723 = x15723 + z15724 = x15724 + z15725 = x15725 + z15726 = x15726 + z15727 = x15727 + z15728 = x15728 + z15729 = x15729 + z15730 = x15730 + z15731 = x15731 + z15732 = x15732 + z15733 = x15733 + z15734 = x15734 + z15735 = x15735 + z15736 = x15736 + z15737 = x15737 + z15738 = x15738 + z15739 = x15739 + z15740 = x15740 + z15741 = x15741 + z15742 = x15742 + z15743 = x15743 + z15744 = x15744 + z15745 = x15745 + z15746 = x15746 + z15747 = x15747 + z15748 = x15748 + z15749 = x15749 + z15750 = x15750 + z15751 = x15751 + z15752 = x15752 + z15753 = x15753 + z15754 = x15754 + z15755 = x15755 + z15756 = x15756 + z15757 = x15757 + z15758 = x15758 + z15759 = x15759 + z15760 = x15760 + z15761 = x15761 + z15762 = x15762 + z15763 = x15763 + z15764 = x15764 + z15765 = x15765 + z15766 = x15766 + z15767 = x15767 + z15768 = x15768 + z15769 = x15769 + z15770 = x15770 + z15771 = x15771 + z15772 = x15772 + z15773 = x15773 + z15774 = x15774 + z15775 = x15775 + z15776 = x15776 + z15777 = x15777 + z15778 = x15778 + z15779 = x15779 + z15780 = x15780 + z15781 = x15781 + z15782 = x15782 + z15783 = x15783 + z15784 = x15784 + z15785 = x15785 + z15786 = x15786 + z15787 = x15787 + z15788 = x15788 + z15789 = x15789 + z15790 = x15790 + z15791 = x15791 + z15792 = x15792 + z15793 = x15793 + z15794 = x15794 + z15795 = x15795 + z15796 = x15796 + z15797 = x15797 + z15798 = x15798 + z15799 = x15799 + z15800 = x15800 + z15801 = x15801 + z15802 = x15802 + z15803 = x15803 + z15804 = x15804 + z15805 = x15805 + z15806 = x15806 + z15807 = x15807 + z15808 = x15808 + z15809 = x15809 + z15810 = x15810 + z15811 = x15811 + z15812 = x15812 + z15813 = x15813 + z15814 = x15814 + z15815 = x15815 + z15816 = x15816 + z15817 = x15817 + z15818 = x15818 + z15819 = x15819 + z15820 = x15820 + z15821 = x15821 + z15822 = x15822 + z15823 = x15823 + z15824 = x15824 + z15825 = x15825 + z15826 = x15826 + z15827 = x15827 + z15828 = x15828 + z15829 = x15829 + z15830 = x15830 + z15831 = x15831 + z15832 = x15832 + z15833 = x15833 + z15834 = x15834 + z15835 = x15835 + z15836 = x15836 + z15837 = x15837 + z15838 = x15838 + z15839 = x15839 + z15840 = x15840 + z15841 = x15841 + z15842 = x15842 + z15843 = x15843 + z15844 = x15844 + z15845 = x15845 + z15846 = x15846 + z15847 = x15847 + z15848 = x15848 + z15849 = x15849 + z15850 = x15850 + z15851 = x15851 + z15852 = x15852 + z15853 = x15853 + z15854 = x15854 + z15855 = x15855 + z15856 = x15856 + z15857 = x15857 + z15858 = x15858 + z15859 = x15859 + z15860 = x15860 + z15861 = x15861 + z15862 = x15862 + z15863 = x15863 + z15864 = x15864 + z15865 = x15865 + z15866 = x15866 + z15867 = x15867 + z15868 = x15868 + z15869 = x15869 + z15870 = x15870 + z15871 = x15871 + z15872 = x15872 + z15873 = x15873 + z15874 = x15874 + z15875 = x15875 + z15876 = x15876 + z15877 = x15877 + z15878 = x15878 + z15879 = x15879 + z15880 = x15880 + z15881 = x15881 + z15882 = x15882 + z15883 = x15883 + z15884 = x15884 + z15885 = x15885 + z15886 = x15886 + z15887 = x15887 + z15888 = x15888 + z15889 = x15889 + z15890 = x15890 + z15891 = x15891 + z15892 = x15892 + z15893 = x15893 + z15894 = x15894 + z15895 = x15895 + z15896 = x15896 + z15897 = x15897 + z15898 = x15898 + z15899 = x15899 + z15900 = x15900 + z15901 = x15901 + z15902 = x15902 + z15903 = x15903 + z15904 = x15904 + z15905 = x15905 + z15906 = x15906 + z15907 = x15907 + z15908 = x15908 + z15909 = x15909 + z15910 = x15910 + z15911 = x15911 + z15912 = x15912 + z15913 = x15913 + z15914 = x15914 + z15915 = x15915 + z15916 = x15916 + z15917 = x15917 + z15918 = x15918 + z15919 = x15919 + z15920 = x15920 + z15921 = x15921 + z15922 = x15922 + z15923 = x15923 + z15924 = x15924 + z15925 = x15925 + z15926 = x15926 + z15927 = x15927 + z15928 = x15928 + z15929 = x15929 + z15930 = x15930 + z15931 = x15931 + z15932 = x15932 + z15933 = x15933 + z15934 = x15934 + z15935 = x15935 + z15936 = x15936 + z15937 = x15937 + z15938 = x15938 + z15939 = x15939 + z15940 = x15940 + z15941 = x15941 + z15942 = x15942 + z15943 = x15943 + z15944 = x15944 + z15945 = x15945 + z15946 = x15946 + z15947 = x15947 + z15948 = x15948 + z15949 = x15949 + z15950 = x15950 + z15951 = x15951 + z15952 = x15952 + z15953 = x15953 + z15954 = x15954 + z15955 = x15955 + z15956 = x15956 + z15957 = x15957 + z15958 = x15958 + z15959 = x15959 + z15960 = x15960 + z15961 = x15961 + z15962 = x15962 + z15963 = x15963 + z15964 = x15964 + z15965 = x15965 + z15966 = x15966 + z15967 = x15967 + z15968 = x15968 + z15969 = x15969 + z15970 = x15970 + z15971 = x15971 + z15972 = x15972 + z15973 = x15973 + z15974 = x15974 + z15975 = x15975 + z15976 = x15976 + z15977 = x15977 + z15978 = x15978 + z15979 = x15979 + z15980 = x15980 + z15981 = x15981 + z15982 = x15982 + z15983 = x15983 + z15984 = x15984 + z15985 = x15985 + z15986 = x15986 + z15987 = x15987 + z15988 = x15988 + z15989 = x15989 + z15990 = x15990 + z15991 = x15991 + z15992 = x15992 + z15993 = x15993 + z15994 = x15994 + z15995 = x15995 + z15996 = x15996 + z15997 = x15997 + z15998 = x15998 + z15999 = x15999 + z16000 = x16000 + z16001 = x16001 + z16002 = x16002 + z16003 = x16003 + z16004 = x16004 + z16005 = x16005 + z16006 = x16006 + z16007 = x16007 + z16008 = x16008 + z16009 = x16009 + z16010 = x16010 + z16011 = x16011 + z16012 = x16012 + z16013 = x16013 + z16014 = x16014 + z16015 = x16015 + z16016 = x16016 + z16017 = x16017 + z16018 = x16018 + z16019 = x16019 + z16020 = x16020 + z16021 = x16021 + z16022 = x16022 + z16023 = x16023 + z16024 = x16024 + z16025 = x16025 + z16026 = x16026 + z16027 = x16027 + z16028 = x16028 + z16029 = x16029 + z16030 = x16030 + z16031 = x16031 + z16032 = x16032 + z16033 = x16033 + z16034 = x16034 + z16035 = x16035 + z16036 = x16036 + z16037 = x16037 + z16038 = x16038 + z16039 = x16039 + z16040 = x16040 + z16041 = x16041 + z16042 = x16042 + z16043 = x16043 + z16044 = x16044 + z16045 = x16045 + z16046 = x16046 + z16047 = x16047 + z16048 = x16048 + z16049 = x16049 + z16050 = x16050 + z16051 = x16051 + z16052 = x16052 + z16053 = x16053 + z16054 = x16054 + z16055 = x16055 + z16056 = x16056 + z16057 = x16057 + z16058 = x16058 + z16059 = x16059 + z16060 = x16060 + z16061 = x16061 + z16062 = x16062 + z16063 = x16063 + z16064 = x16064 + z16065 = x16065 + z16066 = x16066 + z16067 = x16067 + z16068 = x16068 + z16069 = x16069 + z16070 = x16070 + z16071 = x16071 + z16072 = x16072 + z16073 = x16073 + z16074 = x16074 + z16075 = x16075 + z16076 = x16076 + z16077 = x16077 + z16078 = x16078 + z16079 = x16079 + z16080 = x16080 + z16081 = x16081 + z16082 = x16082 + z16083 = x16083 + z16084 = x16084 + z16085 = x16085 + z16086 = x16086 + z16087 = x16087 + z16088 = x16088 + z16089 = x16089 + z16090 = x16090 + z16091 = x16091 + z16092 = x16092 + z16093 = x16093 + z16094 = x16094 + z16095 = x16095 + z16096 = x16096 + z16097 = x16097 + z16098 = x16098 + z16099 = x16099 + z16100 = x16100 + z16101 = x16101 + z16102 = x16102 + z16103 = x16103 + z16104 = x16104 + z16105 = x16105 + z16106 = x16106 + z16107 = x16107 + z16108 = x16108 + z16109 = x16109 + z16110 = x16110 + z16111 = x16111 + z16112 = x16112 + z16113 = x16113 + z16114 = x16114 + z16115 = x16115 + z16116 = x16116 + z16117 = x16117 + z16118 = x16118 + z16119 = x16119 + z16120 = x16120 + z16121 = x16121 + z16122 = x16122 + z16123 = x16123 + z16124 = x16124 + z16125 = x16125 + z16126 = x16126 + z16127 = x16127 + z16128 = x16128 + z16129 = x16129 + z16130 = x16130 + z16131 = x16131 + z16132 = x16132 + z16133 = x16133 + z16134 = x16134 + z16135 = x16135 + z16136 = x16136 + z16137 = x16137 + z16138 = x16138 + z16139 = x16139 + z16140 = x16140 + z16141 = x16141 + z16142 = x16142 + z16143 = x16143 + z16144 = x16144 + z16145 = x16145 + z16146 = x16146 + z16147 = x16147 + z16148 = x16148 + z16149 = x16149 + z16150 = x16150 + z16151 = x16151 + z16152 = x16152 + z16153 = x16153 + z16154 = x16154 + z16155 = x16155 + z16156 = x16156 + z16157 = x16157 + z16158 = x16158 + z16159 = x16159 + z16160 = x16160 + z16161 = x16161 + z16162 = x16162 + z16163 = x16163 + z16164 = x16164 + z16165 = x16165 + z16166 = x16166 + z16167 = x16167 + z16168 = x16168 + z16169 = x16169 + z16170 = x16170 + z16171 = x16171 + z16172 = x16172 + z16173 = x16173 + z16174 = x16174 + z16175 = x16175 + z16176 = x16176 + z16177 = x16177 + z16178 = x16178 + z16179 = x16179 + z16180 = x16180 + z16181 = x16181 + z16182 = x16182 + z16183 = x16183 + z16184 = x16184 + z16185 = x16185 + z16186 = x16186 + z16187 = x16187 + z16188 = x16188 + z16189 = x16189 + z16190 = x16190 + z16191 = x16191 + z16192 = x16192 + z16193 = x16193 + z16194 = x16194 + z16195 = x16195 + z16196 = x16196 + z16197 = x16197 + z16198 = x16198 + z16199 = x16199 + z16200 = x16200 + z16201 = x16201 + z16202 = x16202 + z16203 = x16203 + z16204 = x16204 + z16205 = x16205 + z16206 = x16206 + z16207 = x16207 + z16208 = x16208 + z16209 = x16209 + z16210 = x16210 + z16211 = x16211 + z16212 = x16212 + z16213 = x16213 + z16214 = x16214 + z16215 = x16215 + z16216 = x16216 + z16217 = x16217 + z16218 = x16218 + z16219 = x16219 + z16220 = x16220 + z16221 = x16221 + z16222 = x16222 + z16223 = x16223 + z16224 = x16224 + z16225 = x16225 + z16226 = x16226 + z16227 = x16227 + z16228 = x16228 + z16229 = x16229 + z16230 = x16230 + z16231 = x16231 + z16232 = x16232 + z16233 = x16233 + z16234 = x16234 + z16235 = x16235 + z16236 = x16236 + z16237 = x16237 + z16238 = x16238 + z16239 = x16239 + z16240 = x16240 + z16241 = x16241 + z16242 = x16242 + z16243 = x16243 + z16244 = x16244 + z16245 = x16245 + z16246 = x16246 + z16247 = x16247 + z16248 = x16248 + z16249 = x16249 + z16250 = x16250 + z16251 = x16251 + z16252 = x16252 + z16253 = x16253 + z16254 = x16254 + z16255 = x16255 + z16256 = x16256 + z16257 = x16257 + z16258 = x16258 + z16259 = x16259 + z16260 = x16260 + z16261 = x16261 + z16262 = x16262 + z16263 = x16263 + z16264 = x16264 + z16265 = x16265 + z16266 = x16266 + z16267 = x16267 + z16268 = x16268 + z16269 = x16269 + z16270 = x16270 + z16271 = x16271 + z16272 = x16272 + z16273 = x16273 + z16274 = x16274 + z16275 = x16275 + z16276 = x16276 + z16277 = x16277 + z16278 = x16278 + z16279 = x16279 + z16280 = x16280 + z16281 = x16281 + z16282 = x16282 + z16283 = x16283 + z16284 = x16284 + z16285 = x16285 + z16286 = x16286 + z16287 = x16287 + z16288 = x16288 + z16289 = x16289 + z16290 = x16290 + z16291 = x16291 + z16292 = x16292 + z16293 = x16293 + z16294 = x16294 + z16295 = x16295 + z16296 = x16296 + z16297 = x16297 + z16298 = x16298 + z16299 = x16299 + z16300 = x16300 + z16301 = x16301 + z16302 = x16302 + z16303 = x16303 + z16304 = x16304 + z16305 = x16305 + z16306 = x16306 + z16307 = x16307 + z16308 = x16308 + z16309 = x16309 + z16310 = x16310 + z16311 = x16311 + z16312 = x16312 + z16313 = x16313 + z16314 = x16314 + z16315 = x16315 + z16316 = x16316 + z16317 = x16317 + z16318 = x16318 + z16319 = x16319 + z16320 = x16320 + z16321 = x16321 + z16322 = x16322 + z16323 = x16323 + z16324 = x16324 + z16325 = x16325 + z16326 = x16326 + z16327 = x16327 + z16328 = x16328 + z16329 = x16329 + z16330 = x16330 + z16331 = x16331 + z16332 = x16332 + z16333 = x16333 + z16334 = x16334 + z16335 = x16335 + z16336 = x16336 + z16337 = x16337 + z16338 = x16338 + z16339 = x16339 + z16340 = x16340 + z16341 = x16341 + z16342 = x16342 + z16343 = x16343 + z16344 = x16344 + z16345 = x16345 + z16346 = x16346 + z16347 = x16347 + z16348 = x16348 + z16349 = x16349 + z16350 = x16350 + z16351 = x16351 + z16352 = x16352 + z16353 = x16353 + z16354 = x16354 + z16355 = x16355 + z16356 = x16356 + z16357 = x16357 + z16358 = x16358 + z16359 = x16359 + z16360 = x16360 + z16361 = x16361 + z16362 = x16362 + z16363 = x16363 + z16364 = x16364 + z16365 = x16365 + z16366 = x16366 + z16367 = x16367 + z16368 = x16368 + z16369 = x16369 + z16370 = x16370 + z16371 = x16371 + z16372 = x16372 + z16373 = x16373 + z16374 = x16374 + z16375 = x16375 + z16376 = x16376 + z16377 = x16377 + z16378 = x16378 + z16379 = x16379 + z16380 = x16380 + z16381 = x16381 + z16382 = x16382 + z16383 = x16383 + z16384 = x16384 + z16385 = x16385 + z16386 = x16386 + z16387 = x16387 + z16388 = x16388 + z16389 = x16389 + z16390 = x16390 + z16391 = x16391 + z16392 = x16392 + z16393 = x16393 + z16394 = x16394 + z16395 = x16395 + z16396 = x16396 + z16397 = x16397 + z16398 = x16398 + z16399 = x16399 + z16400 = x16400 + z16401 = x16401 + z16402 = x16402 + z16403 = x16403 + z16404 = x16404 + z16405 = x16405 + z16406 = x16406 + z16407 = x16407 + z16408 = x16408 + z16409 = x16409 + z16410 = x16410 + z16411 = x16411 + z16412 = x16412 + z16413 = x16413 + z16414 = x16414 + z16415 = x16415 + z16416 = x16416 + z16417 = x16417 + z16418 = x16418 + z16419 = x16419 + z16420 = x16420 + z16421 = x16421 + z16422 = x16422 + z16423 = x16423 + z16424 = x16424 + z16425 = x16425 + z16426 = x16426 + z16427 = x16427 + z16428 = x16428 + z16429 = x16429 + z16430 = x16430 + z16431 = x16431 + z16432 = x16432 + z16433 = x16433 + z16434 = x16434 + z16435 = x16435 + z16436 = x16436 + z16437 = x16437 + z16438 = x16438 + z16439 = x16439 + z16440 = x16440 + z16441 = x16441 + z16442 = x16442 + z16443 = x16443 + z16444 = x16444 + z16445 = x16445 + z16446 = x16446 + z16447 = x16447 + z16448 = x16448 + z16449 = x16449 + z16450 = x16450 + z16451 = x16451 + z16452 = x16452 + z16453 = x16453 + z16454 = x16454 + z16455 = x16455 + z16456 = x16456 + z16457 = x16457 + z16458 = x16458 + z16459 = x16459 + z16460 = x16460 + z16461 = x16461 + z16462 = x16462 + z16463 = x16463 + z16464 = x16464 + z16465 = x16465 + z16466 = x16466 + z16467 = x16467 + z16468 = x16468 + z16469 = x16469 + z16470 = x16470 + z16471 = x16471 + z16472 = x16472 + z16473 = x16473 + z16474 = x16474 + z16475 = x16475 + z16476 = x16476 + z16477 = x16477 + z16478 = x16478 + z16479 = x16479 + z16480 = x16480 } diff --git a/test/fixedbugs/bug388.go b/test/fixedbugs/bug388.go index a060c9fd5a75c6..0524534bb3f6dc 100644 --- a/test/fixedbugs/bug388.go +++ b/test/fixedbugs/bug388.go @@ -9,7 +9,7 @@ package main import "runtime" -func foo(runtime.UintType, i int) { // ERROR "cannot declare name runtime.UintType|mixed named and unnamed|undefined identifier" +func foo(runtime.UintType, i int) { // ERROR "cannot declare name runtime.UintType|missing parameter name|undefined identifier" println(i, runtime.UintType) // GCCGO_ERROR "undefined identifier" } diff --git a/test/fixedbugs/issue11610.go b/test/fixedbugs/issue11610.go index 8d68c98f2d437d..ec4d8e84c14b09 100644 --- a/test/fixedbugs/issue11610.go +++ b/test/fixedbugs/issue11610.go @@ -10,7 +10,7 @@ package a var? // ERROR "invalid character U\+003F '\?'|invalid character 0x3f in input file" -var x int // ERROR "unexpected var|expected identifier|expected type" +var x int // ERROR "unexpected keyword var|expected identifier|expected type" func main() { } diff --git a/test/fixedbugs/issue12006.go b/test/fixedbugs/issue12006.go index e878bc48e241e1..94ff52442cdfdf 100644 --- a/test/fixedbugs/issue12006.go +++ b/test/fixedbugs/issue12006.go @@ -17,14 +17,14 @@ func FooN(vals ...*int) (s int) { // ERROR "vals does not escape" // Append forces heap allocation and copies entries in vals to heap, therefore they escape to heap. func FooNx(x *int, vals ...*int) (s int) { // ERROR "leaking param: x" "leaking param content: vals" - vals = append(vals, x) + vals = append(vals, x) // ERROR "append does not escape" return FooN(vals...) } var sink []*int func FooNy(x *int, vals ...*int) (s int) { // ERROR "leaking param: x" "leaking param: vals" - vals = append(vals, x) + vals = append(vals, x) // ERROR "append escapes to heap" sink = vals return FooN(vals...) } @@ -84,7 +84,7 @@ func TFooI() { a := int32(1) // ERROR "moved to heap: a" b := "cat" c := &a - FooI(a, b, c) // ERROR "a escapes to heap" "b escapes to heap" "... argument does not escape" + FooI(a, b, c) // ERROR "a escapes to heap" ".cat. escapes to heap" "... argument does not escape" } func FooJ(args ...interface{}) *int32 { // ERROR "leaking param: args to result ~r0 level=1" @@ -108,14 +108,14 @@ func TFooJ1() { a := int32(1) b := "cat" c := &a - FooJ(a, b, c) // ERROR "a does not escape" "b does not escape" "... argument does not escape" + FooJ(a, b, c) // ERROR "a does not escape" ".cat. does not escape" "... argument does not escape" } func TFooJ2() { a := int32(1) // ERROR "moved to heap: a" b := "cat" c := &a - isink = FooJ(a, b, c) // ERROR "a escapes to heap" "b escapes to heap" "... argument does not escape" + isink = FooJ(a, b, c) // ERROR "a escapes to heap" ".cat. escapes to heap" "... argument does not escape" } type fakeSlice struct { @@ -144,7 +144,7 @@ func TFooK2() { a := int32(1) // ERROR "moved to heap: a" b := "cat" c := &a - fs := fakeSlice{3, &[4]interface{}{a, b, c, nil}} // ERROR "a escapes to heap" "b escapes to heap" "&\[4\]interface {}{...} does not escape" + fs := fakeSlice{3, &[4]interface{}{a, b, c, nil}} // ERROR "a escapes to heap" ".cat. escapes to heap" "&\[4\]interface {}{...} does not escape" isink = FooK(fs) } @@ -169,6 +169,6 @@ func TFooL2() { a := int32(1) // ERROR "moved to heap: a" b := "cat" c := &a - s := []interface{}{a, b, c} // ERROR "a escapes to heap" "b escapes to heap" "\[\]interface {}{...} does not escape" + s := []interface{}{a, b, c} // ERROR "a escapes to heap" ".cat. escapes to heap" "\[\]interface {}{...} does not escape" isink = FooL(s) } diff --git a/test/fixedbugs/issue13799.go b/test/fixedbugs/issue13799.go index f06f19829e78ea..68a0c4af22f880 100644 --- a/test/fixedbugs/issue13799.go +++ b/test/fixedbugs/issue13799.go @@ -51,7 +51,7 @@ func test1(iter int) { // var fn func() // this makes it work, because fn stays off heap j := 0 // ERROR "moved to heap: j$" fn = func() { // ERROR "func literal escapes to heap$" - m[i] = append(m[i], 0) + m[i] = append(m[i], 0) // ERROR "append escapes to heap" if j < 25 { j++ fn() @@ -75,7 +75,7 @@ func test2(iter int) { var fn func() // this makes it work, because fn stays off heap j := 0 fn = func() { // ERROR "func literal does not escape$" - m[i] = append(m[i], 0) + m[i] = append(m[i], 0) // ERROR "append escapes to heap" if j < 25 { j++ fn() diff --git a/test/fixedbugs/issue14636.go b/test/fixedbugs/issue14636.go index c8e751fb613c2e..a866c9a9e30e8e 100644 --- a/test/fixedbugs/issue14636.go +++ b/test/fixedbugs/issue14636.go @@ -12,22 +12,29 @@ import ( "bytes" "log" "os/exec" + "runtime" "strings" ) func main() { - checkLinkOutput("", "-B argument must start with 0x") + // The cannot open file error indicates that the parsing of -B flag + // succeeded and it failed at a later step. checkLinkOutput("0", "-B argument must start with 0x") - checkLinkOutput("0x", "usage") + checkLinkOutput("0x", "cannot open file nonexistent.o") checkLinkOutput("0x0", "-B argument must have even number of digits") - checkLinkOutput("0x00", "usage") + checkLinkOutput("0x00", "cannot open file nonexistent.o") checkLinkOutput("0xYZ", "-B argument contains invalid hex digit") - checkLinkOutput("0x"+strings.Repeat("00", 32), "usage") - checkLinkOutput("0x"+strings.Repeat("00", 33), "-B option too long (max 32 digits)") + + maxLen := 32 + if runtime.GOOS == "darwin" || runtime.GOOS == "ios" { + maxLen = 16 + } + checkLinkOutput("0x"+strings.Repeat("00", maxLen), "cannot open file nonexistent.o") + checkLinkOutput("0x"+strings.Repeat("00", maxLen+1), "-B option too long") } func checkLinkOutput(buildid string, message string) { - cmd := exec.Command("go", "tool", "link", "-B", buildid) + cmd := exec.Command("go", "tool", "link", "-B", buildid, "nonexistent.o") out, err := cmd.CombinedOutput() if err == nil { log.Fatalf("expected cmd/link to fail") @@ -39,6 +46,6 @@ func checkLinkOutput(buildid string, message string) { } if !strings.Contains(firstLine, message) { - log.Fatalf("cmd/link output did not include expected message %q: %s", message, firstLine) + log.Fatalf("%s: cmd/link output did not include expected message %q: %s", buildid, message, firstLine) } } diff --git a/test/fixedbugs/issue15747.go b/test/fixedbugs/issue15747.go index 92e762c4e923ac..743adb6a8ffb1f 100644 --- a/test/fixedbugs/issue15747.go +++ b/test/fixedbugs/issue15747.go @@ -19,7 +19,7 @@ type T struct{ M string } var b bool -func f1(q *Q, xx []byte) interface{} { // ERROR "live at call to newobject: xx$" "live at entry to f1: xx$" +func f1(q *Q, xx []byte) interface{} { // ERROR "live at call to mallocgcSmallScanNoHeaderSC[0-9]+: xx$" "live at entry to f1: xx$" // xx was copied from the stack to the heap on the previous line: // xx was live for the first two prints but then it switched to &xx // being live. We should not see plain xx again. @@ -36,7 +36,7 @@ func f1(q *Q, xx []byte) interface{} { // ERROR "live at call to newobject: xx$" //go:noinline func f2(d []byte, n int) (odata, res []byte, e interface{}) { // ERROR "live at entry to f2: d$" if n > len(d) { - return d, nil, &T{M: "hello"} // ERROR "live at call to newobject: d" + return d, nil, &T{M: "hello"} // ERROR "live at call to mallocgcSmallScanNoHeaderSC[0-9]+: d" } res = d[:n] odata = d[n:] diff --git a/test/fixedbugs/issue16241.go b/test/fixedbugs/issue16241.go new file mode 100644 index 00000000000000..a88dc9a40aa842 --- /dev/null +++ b/test/fixedbugs/issue16241.go @@ -0,0 +1,79 @@ +// errorcheck -0 -m -l + +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package foo + +import "sync/atomic" + +func AddInt32(x *int32) { // ERROR "x does not escape$" + atomic.AddInt32(x, 42) +} +func AddUint32(x *uint32) { // ERROR "x does not escape$" + atomic.AddUint32(x, 42) +} +func AddUintptr(x *uintptr) { // ERROR "x does not escape$" + atomic.AddUintptr(x, 42) +} + +func AndInt32(x *int32) { // ERROR "x does not escape$" + atomic.AndInt32(x, 42) +} +func AndUint32(x *uint32) { // ERROR "x does not escape$" + atomic.AndUint32(x, 42) +} +func AndUintptr(x *uintptr) { // ERROR "x does not escape$" + atomic.AndUintptr(x, 42) +} + +func CompareAndSwapInt32(x *int32) { // ERROR "x does not escape$" + atomic.CompareAndSwapInt32(x, 42, 42) +} +func CompareAndSwapUint32(x *uint32) { // ERROR "x does not escape$" + atomic.CompareAndSwapUint32(x, 42, 42) +} +func CompareAndSwapUintptr(x *uintptr) { // ERROR "x does not escape$" + atomic.CompareAndSwapUintptr(x, 42, 42) +} + +func LoadInt32(x *int32) { // ERROR "x does not escape$" + atomic.LoadInt32(x) +} +func LoadUint32(x *uint32) { // ERROR "x does not escape$" + atomic.LoadUint32(x) +} +func LoadUintptr(x *uintptr) { // ERROR "x does not escape$" + atomic.LoadUintptr(x) +} + +func OrInt32(x *int32) { // ERROR "x does not escape$" + atomic.OrInt32(x, 42) +} +func OrUint32(x *uint32) { // ERROR "x does not escape$" + atomic.OrUint32(x, 42) +} +func OrUintptr(x *uintptr) { // ERROR "x does not escape$" + atomic.OrUintptr(x, 42) +} + +func StoreInt32(x *int32) { // ERROR "x does not escape$" + atomic.StoreInt32(x, 42) +} +func StoreUint32(x *uint32) { // ERROR "x does not escape$" + atomic.StoreUint32(x, 42) +} +func StoreUintptr(x *uintptr) { // ERROR "x does not escape$" + atomic.StoreUintptr(x, 42) +} + +func SwapInt32(x *int32) { // ERROR "x does not escape$" + atomic.SwapInt32(x, 42) +} +func SwapUint32(x *uint32) { // ERROR "x does not escape$" + atomic.SwapUint32(x, 42) +} +func SwapUintptr(x *uintptr) { // ERROR "x does not escape$" + atomic.SwapUintptr(x, 42) +} diff --git a/test/fixedbugs/issue16241_64.go b/test/fixedbugs/issue16241_64.go new file mode 100644 index 00000000000000..468444bdd778a3 --- /dev/null +++ b/test/fixedbugs/issue16241_64.go @@ -0,0 +1,60 @@ +//go:build !(386 || arm || mips || mipsle) + +// errorcheck -0 -m -l + +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package foo + +import "sync/atomic" + +func AddInt64(x *int64) { // ERROR "x does not escape$" + atomic.AddInt64(x, 42) +} +func AddUint64(x *uint64) { // ERROR "x does not escape$" + atomic.AddUint64(x, 42) +} + +func AndInt64(x *int64) { // ERROR "x does not escape$" + atomic.AndInt64(x, 42) +} +func AndUint64(x *uint64) { // ERROR "x does not escape$" + atomic.AndUint64(x, 42) +} + +func CompareAndSwapInt64(x *int64) { // ERROR "x does not escape$" + atomic.CompareAndSwapInt64(x, 42, 42) +} +func CompareAndSwapUint64(x *uint64) { // ERROR "x does not escape$" + atomic.CompareAndSwapUint64(x, 42, 42) +} + +func LoadInt64(x *int64) { // ERROR "x does not escape$" + atomic.LoadInt64(x) +} +func LoadUint64(x *uint64) { // ERROR "x does not escape$" + atomic.LoadUint64(x) +} + +func OrInt64(x *int64) { // ERROR "x does not escape$" + atomic.OrInt64(x, 42) +} +func OrUint64(x *uint64) { // ERROR "x does not escape$" + atomic.OrUint64(x, 42) +} + +func StoreInt64(x *int64) { // ERROR "x does not escape$" + atomic.StoreInt64(x, 42) +} +func StoreUint64(x *uint64) { // ERROR "x does not escape$" + atomic.StoreUint64(x, 42) +} + +func SwapInt64(x *int64) { // ERROR "x does not escape$" + atomic.SwapInt64(x, 42) +} +func SwapUint64(x *uint64) { // ERROR "x does not escape$" + atomic.SwapUint64(x, 42) +} diff --git a/test/fixedbugs/issue17038.go b/test/fixedbugs/issue17038.go index 1b65ffc1f0eec6..32d3c9320b43d1 100644 --- a/test/fixedbugs/issue17038.go +++ b/test/fixedbugs/issue17038.go @@ -6,4 +6,4 @@ package main -const A = complex(0()) // ERROR "cannot call non-function" +const A = complex(0()) // ERROR "cannot call .* not a function" diff --git a/test/fixedbugs/issue20027.go b/test/fixedbugs/issue20027.go new file mode 100644 index 00000000000000..e93b359aebedd4 --- /dev/null +++ b/test/fixedbugs/issue20027.go @@ -0,0 +1,13 @@ +// errorcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +var _ chan [0x2FFFF]byte // ERROR "channel element type too large" +var _ = make(chan [0x2FFFF]byte) // ERROR "channel element type too large" + +var c1 chan [0x2FFFF]byte // ERROR "channel element type too large" +var c2 = make(chan [0x2FFFF]byte) // ERROR "channel element type too large" diff --git a/test/fixedbugs/issue22822.go b/test/fixedbugs/issue22822.go index 9483c9cab0ab1c..c760a8b9edb653 100644 --- a/test/fixedbugs/issue22822.go +++ b/test/fixedbugs/issue22822.go @@ -13,7 +13,7 @@ func F() { slice := []int{1, 2, 3} _ = slice len := int(2) - println(len(slice)) // ERROR "cannot call non-function len .type int., declared at LINE-1|expected function|cannot call non-function len" + println(len(slice)) // ERROR "cannot call non-function len .type int., declared at LINE-1|expected function|cannot call len" const iota = 1 - println(iota(slice)) // ERROR "cannot call non-function iota .type int., declared at LINE-1|expected function|cannot call non-function iota" + println(iota(slice)) // ERROR "cannot call non-function iota .type int., declared at LINE-1|expected function|cannot call iota" } diff --git a/test/fixedbugs/issue24755.go b/test/fixedbugs/issue24755.go new file mode 100644 index 00000000000000..705b15fbdc1840 --- /dev/null +++ b/test/fixedbugs/issue24755.go @@ -0,0 +1,19 @@ +// errorcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type I interface { + F() +} + +type T struct { +} + +const _ = I((*T)(nil)) // ERROR "is not constant" + +func (*T) F() { +} diff --git a/test/fixedbugs/issue27356.go b/test/fixedbugs/issue27356.go index c3e686df339544..8be5d04af732a6 100644 --- a/test/fixedbugs/issue27356.go +++ b/test/fixedbugs/issue27356.go @@ -11,9 +11,9 @@ package p var a = []int{1,2,3} func _(len int) { - _ = len(a) // ERROR "cannot call non-function|expected function" + _ = len(a) // ERROR "cannot call|expected function" } var cap = false -var _ = cap(a) // ERROR "cannot call non-function|expected function" +var _ = cap(a) // ERROR "cannot call|expected function" diff --git a/test/fixedbugs/issue30898.go b/test/fixedbugs/issue30898.go index c7f6f2d3712b3c..38358949e23799 100644 --- a/test/fixedbugs/issue30898.go +++ b/test/fixedbugs/issue30898.go @@ -15,5 +15,5 @@ func debugf(format string, args ...interface{}) { // ERROR "can inline debugf" " func bar() { // ERROR "can inline bar" value := 10 - debugf("value is %d", value) // ERROR "inlining call to debugf" "value does not escape" "\.\.\. argument does not escape" + debugf("value is %d", value) // ERROR "inlining call to debugf" "10 does not escape" "\.\.\. argument does not escape" } diff --git a/test/fixedbugs/issue30908.go b/test/fixedbugs/issue30908.go index 98dd641066fc6e..c7679e2ba247b7 100644 --- a/test/fixedbugs/issue30908.go +++ b/test/fixedbugs/issue30908.go @@ -4,6 +4,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !nacl && !js +//go:build !nacl && !js && !wasip1 package ignored diff --git a/test/fixedbugs/issue3705.go b/test/fixedbugs/issue3705.go index ed0a193dcfbb54..b75094288fa576 100644 --- a/test/fixedbugs/issue3705.go +++ b/test/fixedbugs/issue3705.go @@ -6,4 +6,4 @@ package p -func init() // ERROR "missing function body|cannot declare init" +func init() // ERROR "func init must have a body|cannot declare init" diff --git a/test/fixedbugs/issue39292.go b/test/fixedbugs/issue39292.go index 7dac2e5fc6b639..9a4840d0284f3c 100644 --- a/test/fixedbugs/issue39292.go +++ b/test/fixedbugs/issue39292.go @@ -27,3 +27,12 @@ func z() { z := t{&i}.f // ERROR "t{...}.f escapes to heap" z() } + +// Should match cmd/compile/internal/ir/cfg.go:MaxStackVarSize. +const maxStack = 128 * 1024 + +func w(i int) byte { + var x [maxStack]byte + var y [maxStack + 1]byte // ERROR "moved to heap: y" + return x[i] + y[i] +} diff --git a/test/fixedbugs/issue40954.go b/test/fixedbugs/issue40954.go index 0beaabb7439755..8b303b12e65a2c 100644 --- a/test/fixedbugs/issue40954.go +++ b/test/fixedbugs/issue40954.go @@ -30,6 +30,8 @@ func main() { // should not be adjusted when the stack is copied. recurse(100, p, v) } + +//go:noinline func recurse(n int, p *S, v uintptr) { if n > 0 { recurse(n-1, p, v) diff --git a/test/fixedbugs/issue41575.go b/test/fixedbugs/issue41575.go index 456873038fc891..2eed37a9497047 100644 --- a/test/fixedbugs/issue41575.go +++ b/test/fixedbugs/issue41575.go @@ -6,7 +6,7 @@ package p -type T1 struct { // ERROR "invalid recursive type T1\n\tLINE: T1 refers to\n\tLINE+4: T2 refers to\n\tLINE: T1$|invalid recursive type" +type T1 struct { // ERROR "invalid recursive type T1\n.*T1 refers to T2\n.*T2 refers to T1|invalid recursive type" f2 T2 } @@ -15,21 +15,21 @@ type T2 struct { // GCCGO_ERROR "invalid recursive type" } type a b // GCCGO_ERROR "invalid recursive type" -type b c // ERROR "invalid recursive type b\n\tLINE: b refers to\n\tLINE+1: c refers to\n\tLINE: b$|invalid recursive type" +type b c // ERROR "invalid recursive type b\n.*b refers to c\n.*c refers to b|invalid recursive type|invalid recursive type" type c b // GCCGO_ERROR "invalid recursive type" type d e type e f -type f f // ERROR "invalid recursive type f\n\tLINE: f refers to\n\tLINE: f$|invalid recursive type" +type f f // ERROR "invalid recursive type: f refers to itself|invalid recursive type|invalid recursive type" -type g struct { // ERROR "invalid recursive type g\n\tLINE: g refers to\n\tLINE: g$|invalid recursive type" +type g struct { // ERROR "invalid recursive type: g refers to itself|invalid recursive type" h struct { g } } type w x -type x y // ERROR "invalid recursive type x\n\tLINE: x refers to\n\tLINE+1: y refers to\n\tLINE+2: z refers to\n\tLINE: x$|invalid recursive type" +type x y // ERROR "invalid recursive type x\n.*x refers to y\n.*y refers to z\n.*z refers to x|invalid recursive type" type y struct{ z } // GCCGO_ERROR "invalid recursive type" type z [10]x diff --git a/test/fixedbugs/issue41635.go b/test/fixedbugs/issue41635.go index 35c0034cdd40e0..ede8a4f2c9c64b 100644 --- a/test/fixedbugs/issue41635.go +++ b/test/fixedbugs/issue41635.go @@ -12,6 +12,6 @@ func f() { // ERROR "" _ = make([]byte, 100, 1<<17) // ERROR "too large for stack" "" _ = make([]byte, n, 1<<17) // ERROR "too large for stack" "" - _ = make([]byte, n) // ERROR "non-constant size" "" - _ = make([]byte, 100, m) // ERROR "non-constant size" "" + _ = make([]byte, n) // ERROR "does not escape" + _ = make([]byte, 100, m) // ERROR "does not escape" } diff --git a/test/fixedbugs/issue42284.dir/a.go b/test/fixedbugs/issue42284.dir/a.go index ccf54fad54a03c..e55f190d7ee571 100644 --- a/test/fixedbugs/issue42284.dir/a.go +++ b/test/fixedbugs/issue42284.dir/a.go @@ -22,9 +22,8 @@ func g() { h := E() // ERROR "inlining call to E" "T\(0\) does not escape" h.M() // ERROR "devirtualizing h.M to T" "inlining call to T.M" - // BAD: T(0) could be stack allocated. - i := F(T(0)) // ERROR "inlining call to F" "T\(0\) escapes to heap" + i := F(T(0)) // ERROR "inlining call to F" "T\(0\) does not escape" - // Testing that we do NOT devirtualize here: - i.M() + // It is fine that we devirtualize here, as we add an additional nilcheck. + i.M() // ERROR "devirtualizing i.M to T" "inlining call to T.M" } diff --git a/test/fixedbugs/issue42284.dir/b.go b/test/fixedbugs/issue42284.dir/b.go index 559de59184460a..4a0b7cea102e88 100644 --- a/test/fixedbugs/issue42284.dir/b.go +++ b/test/fixedbugs/issue42284.dir/b.go @@ -10,9 +10,8 @@ func g() { h := a.E() // ERROR "inlining call to a.E" "T\(0\) does not escape" h.M() // ERROR "devirtualizing h.M to a.T" "inlining call to a.T.M" - // BAD: T(0) could be stack allocated. - i := a.F(a.T(0)) // ERROR "inlining call to a.F" "a.T\(0\) escapes to heap" + i := a.F(a.T(0)) // ERROR "inlining call to a.F" "a.T\(0\) does not escape" - // Testing that we do NOT devirtualize here: - i.M() + // It is fine that we devirtualize here, as we add an additional nilcheck. + i.M() // ERROR "devirtualizing i.M to a.T" "inlining call to a.T.M" } diff --git a/test/fixedbugs/issue46749.go b/test/fixedbugs/issue46749.go index faf1f884a6b40c..86f57a6159d6f2 100644 --- a/test/fixedbugs/issue46749.go +++ b/test/fixedbugs/issue46749.go @@ -31,7 +31,7 @@ var ( var ( _ = b + 1 // ERROR "invalid operation.*mismatched types.*bool and untyped int" _ = i + false // ERROR "invalid operation.*mismatched types.*int and untyped bool" - _ = iface + 1 // ERROR "invalid operation.*mismatched types.*interface *{} and int" - _ = iface + 1.0 // ERROR "invalid operation.*mismatched types.*interface *{} and float64" + _ = iface + 1 // ERROR "invalid operation.*mismatched types.*interface *{} and untyped int" + _ = iface + 1.0 // ERROR "invalid operation.*mismatched types.*interface *{} and untyped float" _ = iface + false // ERROR "invalid operation.*mismatched types.*interface *{} and bool" ) diff --git a/test/fixedbugs/issue48471.go b/test/fixedbugs/issue48471.go index 75875c40041c4d..a834c8067558b4 100644 --- a/test/fixedbugs/issue48471.go +++ b/test/fixedbugs/issue48471.go @@ -51,6 +51,6 @@ func g() { _ = i.(T6) // ERROR "impossible type assertion: i.\(T6\)\n\tT6 does not implement I \(missing method M\)\n\t\thave m\(int\) string\n\t\twant M\(int\)" var t *T4 - t = i // ERROR "cannot use i \(variable of type I\) as \*T4 value in assignment: need type assertion" + t = i // ERROR "cannot use i \(variable of interface type I\) as \*T4 value in assignment: need type assertion" _ = t } diff --git a/test/fixedbugs/issue50788.dir/b.go b/test/fixedbugs/issue50788.dir/b.go index e17afc7b433099..97ae20801985ab 100644 --- a/test/fixedbugs/issue50788.dir/b.go +++ b/test/fixedbugs/issue50788.dir/b.go @@ -6,4 +6,4 @@ package b import "./a" -type T a.T[T] // ERROR "invalid recursive type T\n.*T refers to\n.*a\.T refers to\n.*T" +type T a.T[T] // ERROR "invalid recursive type T\n.*T refers to a\.T\n.*a\.T refers to T" diff --git a/test/fixedbugs/issue52193.go b/test/fixedbugs/issue52193.go index 32375d114f0fa9..c7afc9312f5576 100644 --- a/test/fixedbugs/issue52193.go +++ b/test/fixedbugs/issue52193.go @@ -6,40 +6,19 @@ package p -import ( - "crypto/ecdh" - "crypto/rand" -) - -func F(peerShare []byte) ([]byte, error) { // ERROR "leaking param: peerShare" - p256 := ecdh.P256() // ERROR "inlining call to ecdh.P256" - - ourKey, err := p256.GenerateKey(rand.Reader) // ERROR "devirtualizing p256.GenerateKey" "inlining call to ecdh.*GenerateKey" - if err != nil { - return nil, err - } - - peerPublic, err := p256.NewPublicKey(peerShare) // ERROR "devirtualizing p256.NewPublicKey" "inlining call to ecdh.*NewPublicKey" - if err != nil { - return nil, err - } - - return ourKey.ECDH(peerPublic) -} - // Test that inlining doesn't break if devirtualization exposes a new // inlinable callee. func f() { // ERROR "can inline f" var i interface{ m() } = T(0) // ERROR "T\(0\) does not escape" - i.m() // ERROR "devirtualizing i.m" "inlining call to T.m" + i.m() // ERROR "devirtualizing i.m" "inlining call to T.m" "inlining call to f" "T\(0\) does not escape" } type T int func (T) m() { // ERROR "can inline T.m" if never { - f() // ERROR "inlining call to f" "devirtualizing i.m" "T\(0\) does not escape" + f() // ERROR "inlining call to f" "devirtualizing i.m" "T\(0\) does not escape" "inlining call to T.m" } } diff --git a/test/fixedbugs/issue54159.go b/test/fixedbugs/issue54159.go index 8ef0e684837edd..8a29bc5cbaab5e 100644 --- a/test/fixedbugs/issue54159.go +++ b/test/fixedbugs/issue54159.go @@ -6,11 +6,13 @@ package main -func run() { // ERROR "cannot inline run: recursive" +//go:noinline +func run() { // ERROR "cannot inline run: marked go:noinline" f := func() { // ERROR "can inline run.func1 with cost .* as:.*" "func literal does not escape" g() // ERROR "inlining call to g" } f() // ERROR "inlining call to run.func1" "inlining call to g" + _ = f run() } diff --git a/test/fixedbugs/issue63489a.go b/test/fixedbugs/issue63489a.go index b88120f2c045ef..2b46814f9566de 100644 --- a/test/fixedbugs/issue63489a.go +++ b/test/fixedbugs/issue63489a.go @@ -1,16 +1,20 @@ -// errorcheck -lang=go1.21 +// errorcheck -lang=go1.22 // Copyright 2023 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build go1.4 +// This file has been changed from its original version as +// //go:build file versions below 1.21 set the language version to 1.21. +// The original tested a -lang version of 1.21 with a file version of +// go1.4 while this new version tests a -lang version of go1.22 +// with a file version of go1.21. -package p - -const c = 0o123 // ERROR "file declares //go:build go1.4" +//go:build go1.21 -// ERROR "file declares //go:build go1.4" +package p -//line issue63489a.go:13:1 -const d = 0o124 +func f() { + for _ = range 10 { // ERROR "file declares //go:build go1.21" + } +} diff --git a/test/fixedbugs/issue63489b.go b/test/fixedbugs/issue63489b.go index 2ad590dfc33347..fd897dea97cb88 100644 --- a/test/fixedbugs/issue63489b.go +++ b/test/fixedbugs/issue63489b.go @@ -1,11 +1,20 @@ -// errorcheck -lang=go1.4 +// errorcheck -lang=go1.21 // Copyright 2023 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build go1.4 +// This file has been changed from its original version as +// //go:build file versions below 1.21 set the language version to 1.21. +// The original tested a -lang version of 1.4 with a file version of +// go1.4 while this new version tests a -lang version of go1.1 +// with a file version of go1.21. + +//go:build go1.21 package p -const c = 0o123 // ERROR "file declares //go:build go1.4" +func f() { + for _ = range 10 { // ERROR "file declares //go:build go1.21" + } +} diff --git a/test/fixedbugs/issue68054.go b/test/fixedbugs/issue68054.go index 5409fc90818003..e9f95efa14991e 100644 --- a/test/fixedbugs/issue68054.go +++ b/test/fixedbugs/issue68054.go @@ -1,4 +1,4 @@ -// compile -goexperiment aliastypeparams +// compile // Copyright 2024 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/fixedbugs/issue68292.go b/test/fixedbugs/issue68292.go new file mode 100644 index 00000000000000..2a0d8267451f54 --- /dev/null +++ b/test/fixedbugs/issue68292.go @@ -0,0 +1,12 @@ +// errorcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func f[S any, T any](T) {} +func g() { + f(0) // ERROR "in call to f, cannot infer S \(declared at issue68292.go:9:8\)" +} diff --git a/test/fixedbugs/issue68526.dir/a/a.go b/test/fixedbugs/issue68526.dir/a/a.go new file mode 100644 index 00000000000000..6249eb59df032b --- /dev/null +++ b/test/fixedbugs/issue68526.dir/a/a.go @@ -0,0 +1,14 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package a + +type A[T any] = struct{ F T } + +type B = struct{ F int } + +func F() B { + type a[T any] = struct{ F T } + return a[int]{} +} diff --git a/test/fixedbugs/issue68526.dir/main.go b/test/fixedbugs/issue68526.dir/main.go new file mode 100644 index 00000000000000..8b72ea37b69ebd --- /dev/null +++ b/test/fixedbugs/issue68526.dir/main.go @@ -0,0 +1,43 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + + "issue68526.dir/a" +) + +func main() { + unexported() + exported() +} + +func unexported() { + var want struct{ F int } + + if any(want) != any(a.B{}) || any(want) != any(a.F()) { + panic("zero value of alias and concrete type not identical") + } +} + +func exported() { + var ( + astr a.A[string] + aint a.A[int] + ) + + if any(astr) != any(struct{ F string }{}) || any(aint) != any(struct{ F int }{}) { + panic("zero value of alias and concrete type not identical") + } + + if any(astr) == any(aint) { + panic("zero value of struct{ F string } and struct{ F int } are not distinct") + } + + if got := fmt.Sprintf("%T", astr); got != "struct { F string }" { + panic(got) + } +} diff --git a/test/fixedbugs/issue68526.go b/test/fixedbugs/issue68526.go new file mode 100644 index 00000000000000..aca6354b7c6948 --- /dev/null +++ b/test/fixedbugs/issue68526.go @@ -0,0 +1,7 @@ +// runindir -gomodversion "1.23" + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ignored diff --git a/test/fixedbugs/issue68580.go b/test/fixedbugs/issue68580.go index b60a7447aaa77b..5c25d15b4362d2 100644 --- a/test/fixedbugs/issue68580.go +++ b/test/fixedbugs/issue68580.go @@ -1,4 +1,4 @@ -// compile -goexperiment aliastypeparams +// compile // Copyright 2024 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/fixedbugs/issue68734.go b/test/fixedbugs/issue68734.go new file mode 100644 index 00000000000000..e60dbc5482230c --- /dev/null +++ b/test/fixedbugs/issue68734.go @@ -0,0 +1,17 @@ +// compile + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The gofrontend had a bug handling panic of an untyped constant expression. + +package issue68734 + +func F1() { + panic(1 + 2) +} + +func F2() { + panic("a" + "b") +} diff --git a/test/fixedbugs/issue68809.go b/test/fixedbugs/issue68809.go new file mode 100644 index 00000000000000..67afda67085496 --- /dev/null +++ b/test/fixedbugs/issue68809.go @@ -0,0 +1,19 @@ +// run + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func main() { + cnt := 0 + for i := 1; i <= 11; i++ { + if i-6 > 4 { + cnt++ + } + } + if cnt != 1 { + panic("bad") + } +} diff --git a/test/fixedbugs/issue68816.go b/test/fixedbugs/issue68816.go new file mode 100644 index 00000000000000..8622c9aa98006a --- /dev/null +++ b/test/fixedbugs/issue68816.go @@ -0,0 +1,41 @@ +// run + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func main() { + mustPanic(func() { + f1(1) + }) + f2(1, 0) // must not panic + mustPanic(func() { + f2(1, 2) + }) +} + +var v []func() + +//go:noinline +func f1(i int) { + v = make([]func(), -2|i) +} + +//go:noinline +func f2(i, j int) { + if j > 0 { + v = make([]func(), -2|i) + } +} + +func mustPanic(f func()) { + defer func() { + r := recover() + if r == nil { + panic("didn't panic") + } + }() + f() +} diff --git a/test/fixedbugs/issue69434.go b/test/fixedbugs/issue69434.go new file mode 100644 index 00000000000000..026d3246063c56 --- /dev/null +++ b/test/fixedbugs/issue69434.go @@ -0,0 +1,61 @@ +// run + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "iter" +) + +func All() iter.Seq[int] { + return func(yield func(int) bool) { + for i := 0; i < 10; i++ { + growStack(512) + if !yield(i) { + return + } + } + } +} + +type S struct { + round int +} + +func NewS(round int) *S { + s := &S{round: round} + return s +} + +func (s *S) check(round int) { + if s.round != round { + panic("bad round") + } +} + +func f() { + rounds := 0 + s := NewS(rounds) + s.check(rounds) + + for range All() { + s.check(rounds) + rounds++ + s = NewS(rounds) + s.check(rounds) + } +} + +func growStack(i int) { + if i == 0 { + return + } + growStack(i - 1) +} + +func main() { + f() +} diff --git a/test/fixedbugs/issue69507.go b/test/fixedbugs/issue69507.go new file mode 100644 index 00000000000000..fc300c848ee62f --- /dev/null +++ b/test/fixedbugs/issue69507.go @@ -0,0 +1,133 @@ +// run + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func main() { + err := run() + if err != nil { + panic(err) + } +} + +func run() error { + methods := "AB" + + type node struct { + tag string + choices []string + } + all := []node{ + {"000", permutations(methods)}, + } + + next := 1 + for len(all) > 0 { + cur := all[0] + k := copy(all, all[1:]) + all = all[:k] + + if len(cur.choices) == 1 { + continue + } + + var bestM map[byte][]string + bMax := len(cur.choices) + 1 + bMin := -1 + for sel := range selections(methods) { + m := make(map[byte][]string) + for _, order := range cur.choices { + x := findFirstMatch(order, sel) + m[x] = append(m[x], order) + } + + min := len(cur.choices) + 1 + max := -1 + for _, v := range m { + if len(v) < min { + min = len(v) + } + if len(v) > max { + max = len(v) + } + } + if max < bMax || (max == bMax && min > bMin) { + bestM = m + bMin = min + bMax = max + } + } + + if bMax == len(cur.choices) { + continue + } + + cc := Keys(bestM) + for c := range cc { + choices := bestM[c] + next++ + + switch c { + case 'A': + case 'B': + default: + panic("unexpected selector type " + string(c)) + } + all = append(all, node{"", choices}) + } + } + return nil +} + +func permutations(s string) []string { + if len(s) <= 1 { + return []string{s} + } + + var result []string + for i, char := range s { + rest := s[:i] + s[i+1:] + for _, perm := range permutations(rest) { + result = append(result, string(char)+perm) + } + } + return result +} + +type Seq[V any] func(yield func(V) bool) + +func selections(s string) Seq[string] { + return func(yield func(string) bool) { + for bits := 1; bits < 1< + +static void CFn(_GoString_ gostr) { + printf("%.*s\n", (int)(_GoStringLen(gostr)), _GoStringPtr(gostr)); +} +*/ +import "C" + +func main() { + C.CFn("hello, world") +} + +// The bug only occurs if there is an exported function. +//export Fn +func Fn() { +} diff --git a/test/fixedbugs/issue71675.go b/test/fixedbugs/issue71675.go new file mode 100644 index 00000000000000..c5c65f5b4c8f85 --- /dev/null +++ b/test/fixedbugs/issue71675.go @@ -0,0 +1,99 @@ +// run +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +package main + +//go:noinline +func i() { + for range yieldInts { + defer func() { + println("I") + recover() + }() + } + // This panic causes dead code elimination of the return block. + // The compiler should nonetheless emit a deferreturn. + panic("i panic") +} + +//go:noinline +func h() { + defer func() { + println("H first") + }() + for range yieldInts { + defer func() { + println("H second") + }() + } + defer func() { + println("H third") + }() + for range yieldIntsPanic { + defer func() { + println("h recover:called") + recover() + }() + } +} + +//go:noinline +func yieldInts(yield func(int) bool) { + if !yield(0) { + return + } +} + +//go:noinline +func g() { + defer func() { + println("G first") + }() + for range yieldIntsPanic { + defer func() { + println("g recover:called") + recover() + }() + } +} + +//go:noinline +func yieldIntsPanic(yield func(int) bool) { + if !yield(0) { + return + } + panic("yield stop") +} + +//go:noinline +func next(i int) int { + if i == 0 { + panic("next stop") + } + return i + 1 +} + +//go:noinline +func f() { + defer func() { + println("F first") + }() + for i := 0; i < 1; i = next(i) { + defer func() { + println("f recover:called") + recover() + }() + } +} +func main() { + f() + println("f returned") + g() + println("g returned") + h() + println("h returned") + i() + println("i returned") + +} diff --git a/test/fixedbugs/issue71675.out b/test/fixedbugs/issue71675.out new file mode 100644 index 00000000000000..077359ba149087 --- /dev/null +++ b/test/fixedbugs/issue71675.out @@ -0,0 +1,13 @@ +f recover:called +F first +f returned +g recover:called +G first +g returned +h recover:called +H third +H second +H first +h returned +I +i returned diff --git a/test/fixedbugs/issue71680.go b/test/fixedbugs/issue71680.go new file mode 100644 index 00000000000000..1013b8f43407c2 --- /dev/null +++ b/test/fixedbugs/issue71680.go @@ -0,0 +1,28 @@ +// compile + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type Parser struct{} +type Node struct{} + +type parserState func(p *Parser) parserState + +func parserStateData(root *Node) parserState { + return func(p *Parser) parserState { + return parserStateOpenMap(root)(p) + } +} + +func parserStateOpenMap(root *Node) parserState { + return func(p *Parser) parserState { + switch { + case p != nil: + return parserStateData(root)(p) + } + return parserStateOpenMap(root)(p) + } +} diff --git a/test/fixedbugs/issue71759.go b/test/fixedbugs/issue71759.go new file mode 100644 index 00000000000000..8134ff041b1505 --- /dev/null +++ b/test/fixedbugs/issue71759.go @@ -0,0 +1,20 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +//go:noinline +func f(p *[2]int32) (int64, int64) { + return int64(p[0]), int64(p[1]) +} + +func main() { + p := [2]int32{-1, -1} + x, y := f(&p) + if x != -1 || y != -1 { + println(x, y) + } +} diff --git a/test/fixedbugs/issue71852.go b/test/fixedbugs/issue71852.go new file mode 100644 index 00000000000000..a0bc0925e32b5c --- /dev/null +++ b/test/fixedbugs/issue71852.go @@ -0,0 +1,23 @@ +// compile + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "math" +) + +func main() { + test(2) +} + +func test(i int) { + if i <= 0 { + return + } + + _ = math.Pow10(i + 2) +} diff --git a/test/fixedbugs/issue71857.go b/test/fixedbugs/issue71857.go new file mode 100644 index 00000000000000..34d29281c06aa0 --- /dev/null +++ b/test/fixedbugs/issue71857.go @@ -0,0 +1,29 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "sync/atomic" + +//go:noinline +func f(p0, p1, p2, p3, p4, p5, p6, p7 *uint64, a *atomic.Uint64) { + old := a.Or(0xaaa) + *p0 = old + *p1 = old + *p2 = old + *p3 = old + *p4 = old + *p5 = old + *p6 = old + *p7 = old +} + +func main() { + a := new(atomic.Uint64) + p := new(uint64) + f(p, p, p, p, p, p, p, p, a) + +} diff --git a/test/fixedbugs/issue71932.go b/test/fixedbugs/issue71932.go new file mode 100644 index 00000000000000..d69b2416bb9002 --- /dev/null +++ b/test/fixedbugs/issue71932.go @@ -0,0 +1,50 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "runtime" + +const C = 16 + +type T [C * C]byte + +func main() { + var ts []*T + + for i := 0; i < 100; i++ { + t := new(T) + // Save every even object. + if i%2 == 0 { + ts = append(ts, t) + } + } + // Make sure the odd objects are collected. + runtime.GC() + + for _, t := range ts { + f(t, C, C) + } +} + +//go:noinline +func f(t *T, i, j uint) { + if i == 0 || i > C || j == 0 || j > C { + return // gets rid of bounds check below (via prove pass) + } + p := &t[i*j-1] + *p = 0 + runtime.GC() + *p = 0 + + // This goes badly if compiled to + // q := &t[i*j] + // *(q-1) = 0 + // runtime.GC() + // *(q-1) = 0 + // as at the GC call, q is an invalid pointer + // (it points past the end of t's allocation). +} diff --git a/test/fixedbugs/issue72063.go b/test/fixedbugs/issue72063.go new file mode 100644 index 00000000000000..7b9c13e4527d20 --- /dev/null +++ b/test/fixedbugs/issue72063.go @@ -0,0 +1,40 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "fmt" + +// Y is the Y-combinator based on https://dreamsongs.com/Files/WhyOfY.pdf +func Y[Endo ~func(RecFct) RecFct, RecFct ~func(T) R, T, R any](f Endo) RecFct { + + type internal[RecFct ~func(T) R, T, R any] func(internal[RecFct, T, R]) RecFct + + g := func(h internal[RecFct, T, R]) RecFct { + return func(t T) R { + return f(h(h))(t) + } + } + return g(g) +} + +func main() { + + fct := Y(func(r func(int) int) func(int) int { + return func(n int) int { + if n <= 0 { + return 1 + } + return n * r(n-1) + } + }) + + want := 3628800 + if got := fct(10); got != want { + msg := fmt.Sprintf("unexpected result, got: %d, want: %d", got, want) + panic(msg) + } +} diff --git a/test/fixedbugs/issue72090.go b/test/fixedbugs/issue72090.go new file mode 100644 index 00000000000000..ca8dc34292d9e9 --- /dev/null +++ b/test/fixedbugs/issue72090.go @@ -0,0 +1,85 @@ +// build + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "iter" +) + +type leafSet map[rune]struct{} + +type branchMap map[rune]*node + +func (bm branchMap) findOrCreateBranch(r rune) *node { + if _, ok := bm[r]; !ok { + bm[r] = newNode() + } + return bm[r] +} + +func (bm branchMap) allSuffixes() iter.Seq[string] { + return func(yield func(string) bool) { + for r, n := range bm { + for s := range n.allStrings() { + if !yield(string(r) + s) { + return + } + } + } + } +} + +type node struct { + leafSet + branchMap +} + +func newNode() *node { + return &node{make(leafSet), make(branchMap)} +} + +func (n *node) add(s []rune) { + switch len(s) { + case 0: + return + case 1: + n.leafSet[s[0]] = struct{}{} + default: + n.branchMap.findOrCreateBranch(s[0]).add(s[1:]) + } +} + +func (n *node) addString(s string) { + n.add([]rune(s)) +} + +func (n *node) allStrings() iter.Seq[string] { + return func(yield func(string) bool) { + for s := range n.leafSet { + if !yield(string(s)) { + return + } + } + for r, n := range n.branchMap { + for s := range n.allSuffixes() { + if !yield(string(r) + s) { + return + } + } + } + } +} + +func main() { + root := newNode() + for _, s := range []string{"foo", "bar", "baz", "a", "b", "c", "hello", "world"} { + root.addString(s) + } + for s := range root.allStrings() { + println(s) + } +} diff --git a/test/fixedbugs/issue72844.go b/test/fixedbugs/issue72844.go new file mode 100644 index 00000000000000..65f1d342753ac2 --- /dev/null +++ b/test/fixedbugs/issue72844.go @@ -0,0 +1,70 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +//go:noinline +func nilPtrFunc() *[4]int { + return nil +} + +var nilPtrVar *[4]int + +func testLen1() { + _ = len(*nilPtrFunc()) +} + +func testLen2() { + _ = len(nilPtrFunc()) +} + +func testLen3() { + _ = len(*nilPtrVar) +} + +func testLen4() { + _ = len(nilPtrVar) +} + +func testRange1() { + for range *nilPtrFunc() { + } +} +func testRange2() { + for range nilPtrFunc() { + } +} +func testRange3() { + for range *nilPtrVar { + } +} +func testRange4() { + for range nilPtrVar { + } +} + +func main() { + shouldPanic(testLen1) + shouldNotPanic(testLen2) + shouldNotPanic(testLen3) + shouldNotPanic(testLen4) + shouldPanic(testRange1) + shouldNotPanic(testRange2) + shouldNotPanic(testRange3) + shouldNotPanic(testRange4) +} + +func shouldPanic(f func()) { + defer func() { + if e := recover(); e == nil { + panic("should have panicked") + } + }() + f() +} +func shouldNotPanic(f func()) { + f() +} diff --git a/test/fixedbugs/issue72860.go b/test/fixedbugs/issue72860.go new file mode 100644 index 00000000000000..5b6199da98b569 --- /dev/null +++ b/test/fixedbugs/issue72860.go @@ -0,0 +1,24 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +//go:noinline +func f(p *int, b bool) int { + valid := *p >= 0 + if !b || !valid { + return 5 + } + return 6 +} +func main() { + defer func() { + if e := recover(); e == nil { + println("should have panicked") + } + }() + f(nil, false) +} diff --git a/test/fixedbugs/issue7310.go b/test/fixedbugs/issue7310.go index 4d10e47e64a518..a7ecbfbcba71f5 100644 --- a/test/fixedbugs/issue7310.go +++ b/test/fixedbugs/issue7310.go @@ -9,7 +9,7 @@ package main func main() { - _ = copy(nil, []int{}) // ERROR "use of untyped nil|left argument must be a slice|expects slice arguments" - _ = copy([]int{}, nil) // ERROR "use of untyped nil|second argument must be slice or string|expects slice arguments" + _ = copy(nil, []int{}) // ERROR "use of untyped nil|left argument must be a slice|argument must be a slice; have untyped nil" + _ = copy([]int{}, nil) // ERROR "use of untyped nil|second argument must be slice or string|argument must be a slice; have untyped nil" _ = 1 + true // ERROR "mismatched types untyped int and untyped bool|incompatible types|cannot convert" } diff --git a/test/fixedbugs/issue73180.go b/test/fixedbugs/issue73180.go new file mode 100644 index 00000000000000..421f42f8d6750e --- /dev/null +++ b/test/fixedbugs/issue73180.go @@ -0,0 +1,15 @@ +// build + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func F(a, b map[float32]int) int { + var st *struct { + n int + f float32 + } + return a[0] + b[st.f] +} diff --git a/test/fixedbugs/issue73200.go b/test/fixedbugs/issue73200.go new file mode 100644 index 00000000000000..060f7945651308 --- /dev/null +++ b/test/fixedbugs/issue73200.go @@ -0,0 +1,36 @@ +// build + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +var g bool + +func main() { + l_4 := uint32(0x6E54EE87) + v4 := int8(-Int64FromInt64(1)) + g = int32(v4) >= safe_mod_func_int32_t_s_s(BoolInt32(l_4 >= 1), 7) +} + +func safe_mod_func_int32_t_s_s(si1 int32, si2 int32) (r int32) { + var v1 int32 + if si2 == 0 { + v1 = si1 + } else { + v1 = si1 % si2 + } + return v1 +} + +func Int64FromInt64(n int64) int64 { + return n +} + +func BoolInt32(b bool) int32 { + if b { + return 1 + } + return 0 +} diff --git a/test/fixedbugs/issue73309.go b/test/fixedbugs/issue73309.go new file mode 100644 index 00000000000000..5e96e6513b1327 --- /dev/null +++ b/test/fixedbugs/issue73309.go @@ -0,0 +1,18 @@ +// compile + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +type B[T any] struct { + a A[T] +} + +type A[T any] = func(B[T]) bool + +func main() { + var s A[int] + println(s) +} diff --git a/test/fixedbugs/issue73309b.go b/test/fixedbugs/issue73309b.go new file mode 100644 index 00000000000000..1e29781ba99778 --- /dev/null +++ b/test/fixedbugs/issue73309b.go @@ -0,0 +1,88 @@ +// compile + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +type Unsigned interface { + ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr +} + +// a Validator instance +type Validator []Validable + +type Numeric interface { + ~int | ~int8 | ~int16 | ~int32 | ~int64 | ~float32 | ~float64 +} + +func (v Validator) Valid() bool { + for _, field := range v { + if !field.Validate() { + return false + } + } + return true +} + +type Validable interface { + Validate() bool +} + +type FieldDef[T any] struct { + value T + rules []Rule[T] +} + +func (f FieldDef[T]) Validate() bool { + for _, rule := range f.rules { + if !rule(f) { + return false + } + } + return true +} + +type Rule[T any] = func(FieldDef[T]) bool + +func Field[T any](value T, rules ...Rule[T]) *FieldDef[T] { + return &FieldDef[T]{value: value, rules: rules} +} + +type StringRule = Rule[string] + +type NumericRule[T Numeric] = Rule[T] + +type UnsignedRule[T Unsigned] = Rule[T] + +func MinS(n int) StringRule { + return func(fd FieldDef[string]) bool { + return len(fd.value) < n + } +} + +func MinD[T Numeric](n T) NumericRule[T] { + return func(fd FieldDef[T]) bool { + return fd.value < n + } +} + +func MinU[T Unsigned](n T) UnsignedRule[T] { + return func(fd FieldDef[T]) bool { + return fd.value < n + } +} + +func main() { + v := Validator{ + Field("test", MinS(5)), + } + + if !v.Valid() { + println("invalid") + return + } + + println("valid") +} diff --git a/test/fixedbugs/issue73476.go b/test/fixedbugs/issue73476.go new file mode 100644 index 00000000000000..cc54d9c1e4498c --- /dev/null +++ b/test/fixedbugs/issue73476.go @@ -0,0 +1,17 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +//go:noinline +func f(p *[4]int) { + for i := range (*p) { // Note the parentheses! gofmt wants to remove them - don't let it! + println(i) + } +} +func main() { + f(nil) +} diff --git a/test/fixedbugs/issue73476.out b/test/fixedbugs/issue73476.out new file mode 100644 index 00000000000000..bc856dafab0941 --- /dev/null +++ b/test/fixedbugs/issue73476.out @@ -0,0 +1,4 @@ +0 +1 +2 +3 diff --git a/test/fixedbugs/issue73483.go b/test/fixedbugs/issue73483.go new file mode 100644 index 00000000000000..8cd0c3433a9b3a --- /dev/null +++ b/test/fixedbugs/issue73483.go @@ -0,0 +1,20 @@ +// run -race + +//go:build race && cgo + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +/* + int v[8192]; +*/ +import "C" + +var x [8192]C.int + +func main() { + copy(C.v[:], x[:]) +} diff --git a/test/fixedbugs/issue73491.go b/test/fixedbugs/issue73491.go new file mode 100644 index 00000000000000..4137088bded696 --- /dev/null +++ b/test/fixedbugs/issue73491.go @@ -0,0 +1,25 @@ +// build + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +type T int + +const K T = 5 + +type P struct { + a [K]*byte +} + +//go:noinline +func f(p *P) { + for i := range K { + p.a[i] = nil + } +} +func main() { + f(nil) +} diff --git a/test/fixedbugs/issue73716.go b/test/fixedbugs/issue73716.go new file mode 100644 index 00000000000000..4680b362c60c5a --- /dev/null +++ b/test/fixedbugs/issue73716.go @@ -0,0 +1,37 @@ +// build + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Issue 73716: cmd/compile: unnamed functions missing FuncInfo + +package main + +import "fmt" + +type EP func() +type F func(EP) EP + +func main() { + eps := []EP{ep1, ep2} + var h EP + + for _, ep := range eps { + h = F(func(e EP) EP { + return func() { + ep() + e() + } + })(h) + } + h() +} + +func ep1() { + fmt.Printf("ep1\n") +} + +func ep2() { + fmt.Printf("ep2\n") +} diff --git a/test/fixedbugs/issue73748a.go b/test/fixedbugs/issue73748a.go new file mode 100644 index 00000000000000..c8ac10c29c2471 --- /dev/null +++ b/test/fixedbugs/issue73748a.go @@ -0,0 +1,32 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "context" + "io" + "runtime/trace" +) + +type T struct { + a [16]int +} + +//go:noinline +func f(x *T) { + *x = T{} +} + +func main() { + trace.Start(io.Discard) + defer func() { + recover() + trace.Log(context.Background(), "a", "b") + + }() + f(nil) +} diff --git a/test/fixedbugs/issue73748b.go b/test/fixedbugs/issue73748b.go new file mode 100644 index 00000000000000..ff094a97640f48 --- /dev/null +++ b/test/fixedbugs/issue73748b.go @@ -0,0 +1,32 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "context" + "io" + "runtime/trace" +) + +type T struct { + a [16]int +} + +//go:noinline +func f(x, y *T) { + *x = *y +} + +func main() { + trace.Start(io.Discard) + defer func() { + recover() + trace.Log(context.Background(), "a", "b") + + }() + f(nil, nil) +} diff --git a/test/fixedbugs/issue73823.go b/test/fixedbugs/issue73823.go new file mode 100644 index 00000000000000..2f662662548bcf --- /dev/null +++ b/test/fixedbugs/issue73823.go @@ -0,0 +1,58 @@ +// compile + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type Backend interface { + Hash(ignores func(bucketName, keyName []byte) bool) (uint32, error) +} + +type backend struct { +} + +func first() (key []byte, value []byte) { + return +} + +func (b *backend) View(fn func() error) error { + return nil +} + +func (b *backend) Hash(ignores func(bucketName, keyName []byte) bool) (uint32, error) { + err := b.View(func() error { + for next, _ := first(); next != nil; next, _ = first() { + _ = next + } + return nil + }) + return 0, err +} + +func defragdb() error { + for next, _ := first(); next != nil; next, _ = first() { + _ = f(next) + ForEach(func(k, v []byte) error { + _ = next + return nil + }) + } + + return nil +} + +func ForEach(fn func(k, v []byte) error) error { + for k, v := first(); k != nil; k, v = first() { + if err := fn(k, v); err != nil { + return err + } + } + return nil +} + +//go:noinline +func f(any) string { + return "" +} diff --git a/test/fixedbugs/issue73888.go b/test/fixedbugs/issue73888.go new file mode 100644 index 00000000000000..b3c1ff768fc317 --- /dev/null +++ b/test/fixedbugs/issue73888.go @@ -0,0 +1,34 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +type SourceRange struct { + x, y int +} + +func (r *SourceRange) String() string { + return "hello" +} + +type SourceNode interface { + SourceRange() +} + +type testNode SourceRange + +func (tn testNode) SourceRange() { +} + +func main() { + n := testNode(SourceRange{}) // zero value + Errorf(n) +} + +//go:noinline +func Errorf(n SourceNode) { + n.SourceRange() +} diff --git a/test/fixedbugs/issue73888b.go b/test/fixedbugs/issue73888b.go new file mode 100644 index 00000000000000..b6e0289cc4eb75 --- /dev/null +++ b/test/fixedbugs/issue73888b.go @@ -0,0 +1,34 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +type SourceRange struct { + x, y int +} + +func (r *SourceRange) String() string { + return "hello" +} + +type SourceNode interface { + SourceRange() +} + +type testNode SourceRange + +func (tn testNode) SourceRange() { +} + +func main() { + n := testNode(SourceRange{1, 1}) // not zero value + Errorf(n) +} + +//go:noinline +func Errorf(n SourceNode) { + n.SourceRange() +} diff --git a/test/fixedbugs/issue73916.go b/test/fixedbugs/issue73916.go new file mode 100644 index 00000000000000..6060c3f235a8e3 --- /dev/null +++ b/test/fixedbugs/issue73916.go @@ -0,0 +1,32 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func callRecover() { + if recover() != nil { + println("recovered") + } +} + +func F(int) { callRecover() } + +func main() { + mustPanic(func() { + defer F(1) + panic("XXX") + }) +} + +func mustPanic(f func()) { + defer func() { + r := recover() + if r == nil { + panic("didn't panic") + } + }() + f() +} diff --git a/test/fixedbugs/issue73916b.go b/test/fixedbugs/issue73916b.go new file mode 100644 index 00000000000000..29393fa502a38c --- /dev/null +++ b/test/fixedbugs/issue73916b.go @@ -0,0 +1,34 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func callRecover() { + func() { + if recover() != nil { + println("recovered") + } + }() +} + +func F() int { callRecover(); return 0 } + +func main() { + mustPanic(func() { + defer F() + panic("XXX") + }) +} + +func mustPanic(f func()) { + defer func() { + r := recover() + if r == nil { + panic("didn't panic") + } + }() + f() +} diff --git a/test/fixedbugs/issue73917.go b/test/fixedbugs/issue73917.go new file mode 100644 index 00000000000000..9c0330d19a3b23 --- /dev/null +++ b/test/fixedbugs/issue73917.go @@ -0,0 +1,40 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func callRecover() { + if recover() != nil { + println("recovered") + } +} + +type T int + +func (*T) M() { callRecover() } + +type S struct{ *T } // has a wrapper S.M wrapping (*T.M) + +var p = S{new(T)} + +var fn = S.M // using a function pointer to force using the wrapper + +func main() { + mustPanic(func() { + defer fn(p) + panic("XXX") + }) +} + +func mustPanic(f func()) { + defer func() { + r := recover() + if r == nil { + panic("didn't panic") + } + }() + f() +} diff --git a/test/fixedbugs/issue73920.go b/test/fixedbugs/issue73920.go new file mode 100644 index 00000000000000..f0a711b0451fe6 --- /dev/null +++ b/test/fixedbugs/issue73920.go @@ -0,0 +1,40 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func callRecover() { + if recover() != nil { + println("recovered") + } +} + +type T int + +func (*T) M() { callRecover() } + +type S struct{ *T } // has a wrapper (*S).M wrapping (*T.M) + +var p = &S{new(T)} + +var fn = (*S).M // using a function pointer to force using the wrapper + +func main() { + mustPanic(func() { + defer fn(p) + panic("XXX") + }) +} + +func mustPanic(f func()) { + defer func() { + r := recover() + if r == nil { + panic("didn't panic") + } + }() + f() +} diff --git a/test/fixedbugs/issue74379.go b/test/fixedbugs/issue74379.go new file mode 100644 index 00000000000000..e516505fbe5407 --- /dev/null +++ b/test/fixedbugs/issue74379.go @@ -0,0 +1,30 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "errors" + "fmt" + "os" +) + +func crashOnErr(err error) bool { + if err != nil { + panic(err) + } + return false +} + +func main() { + defer func() { + if recover() == nil { + fmt.Println("failed to have expected panic") + os.Exit(1) + } + }() + fmt.Println(crashOnErr(errors.New("test error"))) +} diff --git a/test/fixedbugs/issue74379b.go b/test/fixedbugs/issue74379b.go new file mode 100644 index 00000000000000..2603587914ac51 --- /dev/null +++ b/test/fixedbugs/issue74379b.go @@ -0,0 +1,32 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "errors" + "fmt" + "os" +) + +func crashOnErr(err error) int { + if err != nil { + panic(err) + } + return 10 +} + +func main() { + defer func() { + if recover() == nil { + fmt.Println("failed to have expected panic") + os.Exit(1) + } + }() + + s := make([]int, crashOnErr(errors.New("test error"))) + println("unreachable: len(s) =", len(s)) +} diff --git a/test/fixedbugs/issue74379c.go b/test/fixedbugs/issue74379c.go new file mode 100644 index 00000000000000..871307bf89a4e9 --- /dev/null +++ b/test/fixedbugs/issue74379c.go @@ -0,0 +1,54 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "errors" + "fmt" + "os" +) + +type S struct{ a, b int } + +func crashOnErr1(err error) S { + if err != nil { + panic(err) + } + return S{} // zero value struct +} + +func f1() { + defer func() { + if recover() == nil { + fmt.Println("failed to have expected panic") + os.Exit(1) + } + }() + fmt.Println(crashOnErr1(errors.New("test error"))) +} + +func crashOnErr2(err error) S { + if err != nil { + panic(err) + } + return S{1, 2} // not zero value struct +} + +func f2() { + defer func() { + if recover() == nil { + fmt.Println("failed to have expected panic") + os.Exit(1) + } + }() + fmt.Println(crashOnErr2(errors.New("test error"))) +} + +func main() { + f1() + f2() +} diff --git a/test/fixedbugs/issue74648.dir/a.s b/test/fixedbugs/issue74648.dir/a.s new file mode 100644 index 00000000000000..39a8d7684c4eba --- /dev/null +++ b/test/fixedbugs/issue74648.dir/a.s @@ -0,0 +1,11 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +TEXT ·F(SB), $0 + JMP prealigned + INT $3 // should never be reached +prealigned: + PCALIGN $0x10 +aligned: + RET diff --git a/test/fixedbugs/issue74648.dir/x.go b/test/fixedbugs/issue74648.dir/x.go new file mode 100644 index 00000000000000..afde832d4a593a --- /dev/null +++ b/test/fixedbugs/issue74648.dir/x.go @@ -0,0 +1,13 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Issue 74648: wrong jump target when using PCALIGN. + +package main + +func F() + +func main() { + F() +} diff --git a/test/fixedbugs/issue74648.go b/test/fixedbugs/issue74648.go new file mode 100644 index 00000000000000..5db4b8517430e5 --- /dev/null +++ b/test/fixedbugs/issue74648.go @@ -0,0 +1,9 @@ +// runindir + +//go:build amd64 + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ignored diff --git a/test/fixedbugs/issue74836.go b/test/fixedbugs/issue74836.go new file mode 100644 index 00000000000000..3e61c7109318cc --- /dev/null +++ b/test/fixedbugs/issue74836.go @@ -0,0 +1,17 @@ +// compile + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +type T struct { + a [20]int +} + +func f(x [4]int) { + g(T{}, x) +} + +func g(t T, x [4]int) diff --git a/test/fixedbugs/issue75063.go b/test/fixedbugs/issue75063.go new file mode 100644 index 00000000000000..5caa7cb9abbe8f --- /dev/null +++ b/test/fixedbugs/issue75063.go @@ -0,0 +1,75 @@ +// compile + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package reorder + +type Element struct { + A string + B string + C string + D string + E string + Text []string + List []string + Child Elements + F string + G bool + H bool + I string +} + +type Elements []Element + +func DoesNotCompile(ve Elements) Elements { + aa := Elements{} + bb := Elements{} + cc := Elements{} + dd := Elements{} + ee := Elements{} + ff := Elements{} + gg := Elements{} + hh := Elements{} + ii := Elements{} + + if len(ve) != 1 { + return ve + } + for _, e := range ve[0].Child { + if len(e.Text) == 1 && (e.Text[0] == "xx") { + ee = append(ee, e) + } else if len(e.Text) == 1 && e.Text[0] == "yy" { + for _, c := range e.Child { + if len(c.Text) == 1 && c.Text[0] == "zz" { + ii = append(ii, c) + } else { + hh = append(hh, c) + } + } + ii = append(ii, hh...) + e.Child = ii + gg = append(gg, e) + } else if len(e.Text) == 1 && e.Text[0] == "tt" { + for _, entry := range e.Child { + for _, c := range entry.Child { + if len(c.Text) == 1 && c.Text[0] == "ee" { + cc = append(cc, c) + } else { + dd = append(dd, c) + } + } + cc = append(cc, dd...) + entry.Child = cc + bb = append(bb, entry) + cc, dd = Elements{}, Elements{} + } + e.Child = bb + aa = append(aa, e) + } else { + ff = append(ff, e) + } + } + return ve +} diff --git a/test/fixedbugs/issue75278.go b/test/fixedbugs/issue75278.go new file mode 100644 index 00000000000000..bcf5dc3076a24b --- /dev/null +++ b/test/fixedbugs/issue75278.go @@ -0,0 +1,25 @@ +// errorcheck -0 -m=2 + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +var a, b []int + +func NoIndices() { // ERROR "can inline NoIndices with cost 4 as:.*" + b = a[:] +} + +func LowIndex() { // ERROR "can inline LowIndex with cost 4 as:.*" + b = a[0:] +} + +func HighIndex() { // ERROR "can inline HighIndex with cost 4 as:.*" + b = a[:len(a)] +} + +func BothIndices() { // ERROR "can inline BothIndices with cost 4 as:.*" + b = a[0:len(a)] +} diff --git a/test/fixedbugs/issue75327.go b/test/fixedbugs/issue75327.go new file mode 100644 index 00000000000000..9ca7c24e9ba4be --- /dev/null +++ b/test/fixedbugs/issue75327.go @@ -0,0 +1,33 @@ +// run + +package main + +import ( + "fmt" + "strings" +) + +func main() { + defer func() { + err := recover() + txt := fmt.Sprintf("%s", err) + if !strings.HasSuffix(txt, "with length 1") { + panic("bad error: " + txt) + } + }() + foo([]uint64{0}) +} + +//go:noinline +func foo(haystack []uint64) { + for n := range len(haystack) { + _ = n + _ = haystack[1] + } + + xxx := haystack[0:len(haystack)] + sink(xxx) +} + +//go:noinline +func sink([]uint64) {} diff --git a/test/fixedbugs/issue75569.go b/test/fixedbugs/issue75569.go new file mode 100644 index 00000000000000..8420641db2234a --- /dev/null +++ b/test/fixedbugs/issue75569.go @@ -0,0 +1,77 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func fff(a []int, b bool, p, q *int) { +outer: + n := a[0] + a = a[1:] + switch n { + case 1: + goto one + case 2: + goto two + case 3: + goto three + case 4: + goto four + } + +one: + goto inner +two: + goto outer +three: + goto inner +four: + goto innerSideEntry + +inner: + n = a[0] + a = a[1:] + switch n { + case 1: + goto outer + case 2: + goto inner + case 3: + goto innerSideEntry + default: + return + } +innerSideEntry: + n = a[0] + a = a[1:] + switch n { + case 1: + goto outer + case 2: + goto inner + case 3: + goto inner + } + ggg(p, q) + goto inner +} + +var b bool + +func ggg(p, q *int) { + n := *p + 5 // this +5 ends up in the entry block, well before the *p load + if b { + *q = 0 + } + *p = n +} + +func main() { + var x, y int + fff([]int{4, 4, 4}, false, &x, &y) + if x != 5 { + panic(x) + } +} diff --git a/test/func3.go b/test/func3.go index 6be3bf0184d386..861ab2cba52406 100644 --- a/test/func3.go +++ b/test/func3.go @@ -13,8 +13,8 @@ type t1 int type t2 int type t3 int -func f1(*t2, x t3) // ERROR "named" -func f2(t1, *t2, x t3) // ERROR "named" -func f3() (x int, *string) // ERROR "named" +func f1(*t2, x t3) // ERROR "missing parameter name" +func f2(t1, *t2, x t3) // ERROR "missing parameter name" +func f3() (x int, *string) // ERROR "missing parameter name" func f4() (t1 t1) // legal - scope of parameter named t1 starts in body of f4. diff --git a/test/fuse.go b/test/fuse.go index e9205dcc2355fb..9366b218587f76 100644 --- a/test/fuse.go +++ b/test/fuse.go @@ -148,11 +148,11 @@ func fEqInterEqInter(a interface{}, f float64) bool { } func fEqInterNeqInter(a interface{}, f float64) bool { - return a == nil && f > Cf2 || a != nil && f < -Cf2 + return a == nil && f > Cf2 || a != nil && f < -Cf2 // ERROR "Redirect IsNonNil based on IsNonNil" } func fNeqInterEqInter(a interface{}, f float64) bool { - return a != nil && f > Cf2 || a == nil && f < -Cf2 + return a != nil && f > Cf2 || a == nil && f < -Cf2 // ERROR "Redirect IsNonNil based on IsNonNil" } func fNeqInterNeqInter(a interface{}, f float64) bool { @@ -164,11 +164,11 @@ func fEqSliceEqSlice(a []int, f float64) bool { } func fEqSliceNeqSlice(a []int, f float64) bool { - return a == nil && f > Cf2 || a != nil && f < -Cf2 + return a == nil && f > Cf2 || a != nil && f < -Cf2 // ERROR "Redirect IsNonNil based on IsNonNil" } func fNeqSliceEqSlice(a []int, f float64) bool { - return a != nil && f > Cf2 || a == nil && f < -Cf2 + return a != nil && f > Cf2 || a == nil && f < -Cf2 // ERROR "Redirect IsNonNil based on IsNonNil" } func fNeqSliceNeqSlice(a []int, f float64) bool { diff --git a/test/heapsampling.go b/test/heapsampling.go index 741db74f894d45..db93b253c90bab 100644 --- a/test/heapsampling.go +++ b/test/heapsampling.go @@ -48,22 +48,23 @@ func testInterleavedAllocations() error { const iters = 50000 // Sizes of the allocations performed by each experiment. frames := []string{"main.allocInterleaved1", "main.allocInterleaved2", "main.allocInterleaved3"} + leafFrame := "main.allocInterleaved" // Pass if at least one of three experiments has no errors. Use a separate // function for each experiment to identify each experiment in the profile. allocInterleaved1(iters) - if checkAllocations(getMemProfileRecords(), frames[0:1], iters, allocInterleavedSizes) == nil { + if checkAllocations(getMemProfileRecords(), leafFrame, frames[0:1], iters, allocInterleavedSizes) == nil { // Passed on first try, report no error. return nil } allocInterleaved2(iters) - if checkAllocations(getMemProfileRecords(), frames[0:2], iters, allocInterleavedSizes) == nil { + if checkAllocations(getMemProfileRecords(), leafFrame, frames[0:2], iters, allocInterleavedSizes) == nil { // Passed on second try, report no error. return nil } allocInterleaved3(iters) // If it fails a third time, we may be onto something. - return checkAllocations(getMemProfileRecords(), frames[0:3], iters, allocInterleavedSizes) + return checkAllocations(getMemProfileRecords(), leafFrame, frames[0:3], iters, allocInterleavedSizes) } var allocInterleavedSizes = []int64{17 * 1024, 1024, 18 * 1024, 512, 16 * 1024, 256} @@ -108,22 +109,23 @@ func testSmallAllocations() error { // Sizes of the allocations performed by each experiment. sizes := []int64{1024, 512, 256} frames := []string{"main.allocSmall1", "main.allocSmall2", "main.allocSmall3"} + leafFrame := "main.allocSmall" // Pass if at least one of three experiments has no errors. Use a separate // function for each experiment to identify each experiment in the profile. allocSmall1(iters) - if checkAllocations(getMemProfileRecords(), frames[0:1], iters, sizes) == nil { + if checkAllocations(getMemProfileRecords(), leafFrame, frames[0:1], iters, sizes) == nil { // Passed on first try, report no error. return nil } allocSmall2(iters) - if checkAllocations(getMemProfileRecords(), frames[0:2], iters, sizes) == nil { + if checkAllocations(getMemProfileRecords(), leafFrame, frames[0:2], iters, sizes) == nil { // Passed on second try, report no error. return nil } allocSmall3(iters) // If it fails a third time, we may be onto something. - return checkAllocations(getMemProfileRecords(), frames[0:3], iters, sizes) + return checkAllocations(getMemProfileRecords(), leafFrame, frames[0:3], iters, sizes) } // allocSmall performs only small allocations for sanity testing. @@ -161,21 +163,21 @@ func allocSmall3(n int) { // Look only at samples that include the named frames, and group the // allocations by their line number. All these allocations are done from // the same leaf function, so their line numbers are the same. -func checkAllocations(records []runtime.MemProfileRecord, frames []string, count int64, size []int64) error { +func checkAllocations(records []runtime.MemProfileRecord, leafFrame string, frames []string, count int64, size []int64) error { objectsPerLine := map[int][]int64{} bytesPerLine := map[int][]int64{} totalCount := []int64{} // Compute the line number of the first allocation. All the // allocations are from the same leaf, so pick the first one. var firstLine int - for ln := range allocObjects(records, frames[0]) { + for ln := range allocObjects(records, leafFrame, frames[0]) { if firstLine == 0 || firstLine > ln { firstLine = ln } } for _, frame := range frames { var objectCount int64 - a := allocObjects(records, frame) + a := allocObjects(records, leafFrame, frame) for s := range size { // Allocations of size size[s] should be on line firstLine + s. ln := firstLine + s @@ -258,7 +260,7 @@ type allocStat struct { // allocObjects examines the profile records for samples including the // named function and returns the allocation stats aggregated by // source line number of the allocation (at the leaf frame). -func allocObjects(records []runtime.MemProfileRecord, function string) map[int]allocStat { +func allocObjects(records []runtime.MemProfileRecord, leafFrame, function string) map[int]allocStat { a := make(map[int]allocStat) for _, r := range records { var pcs []uintptr @@ -273,7 +275,7 @@ func allocObjects(records []runtime.MemProfileRecord, function string) map[int]a for { frame, more := frames.Next() name := frame.Function - if line == 0 { + if name == leafFrame && line == 0 { line = frame.Line } if name == function { diff --git a/test/initloop.go b/test/initloop.go index b1a8470b3a0b15..c4530f36d6c24d 100644 --- a/test/initloop.go +++ b/test/initloop.go @@ -11,7 +11,7 @@ package main var ( x int = a - a int = b // ERROR "a refers to\n.*b refers to\n.*c refers to\n.*a|initialization loop" + a int = b // ERROR "a refers to b\n.*b refers to c\n.*c refers to a|initialization loop" b int = c c int = a ) diff --git a/test/inline.go b/test/inline.go index fd14f259833555..3ed4b1de451ff1 100644 --- a/test/inline.go +++ b/test/inline.go @@ -73,6 +73,7 @@ func l(x, y int) (int, int, error) { // ERROR "can inline l" f := e f(nil) // ERROR "inlining call to l.func1" } + _ = e // prevent simple deadcode elimination after inlining return y, x, nil } @@ -109,6 +110,7 @@ func p() int { // ERROR "can inline p" func q(x int) int { // ERROR "can inline q" foo := func() int { return x * 2 } // ERROR "can inline q.func1" "func literal does not escape" + _ = foo // prevent simple deadcode elimination after inlining return foo() // ERROR "inlining call to q.func1" } @@ -121,6 +123,8 @@ func r(z int) int { return 2*y + x*z }(x) // ERROR "inlining call to r.func2.1" } + _, _ = foo, bar // prevent simple deadcode elimination after inlining + return foo(42) + bar(42) // ERROR "inlining call to r.func1" "inlining call to r.func2" "inlining call to r.r.func2.func3" } @@ -128,7 +132,8 @@ func s0(x int) int { // ERROR "can inline s0" foo := func() { // ERROR "can inline s0.func1" "func literal does not escape" x = x + 1 } - foo() // ERROR "inlining call to s0.func1" + foo() // ERROR "inlining call to s0.func1" + _ = foo // prevent simple deadcode elimination after inlining return x } @@ -137,6 +142,7 @@ func s1(x int) int { // ERROR "can inline s1" return x } x = x + 1 + _ = foo // prevent simple deadcode elimination after inlining return foo() // ERROR "inlining call to s1.func1" } @@ -274,13 +280,13 @@ func ff(x int) { // ERROR "can inline ff" if x < 0 { return } - gg(x - 1) // ERROR "inlining call to gg" "inlining call to hh" + gg(x - 1) // ERROR "inlining call to gg" "inlining call to hh" "inlining call to ff" } func gg(x int) { // ERROR "can inline gg" - hh(x - 1) // ERROR "inlining call to hh" "inlining call to ff" + hh(x - 1) // ERROR "inlining call to hh" "inlining call to ff" "inlining call to gg" } func hh(x int) { // ERROR "can inline hh" - ff(x - 1) // ERROR "inlining call to ff" "inlining call to gg" + ff(x - 1) // ERROR "inlining call to ff" "inlining call to gg" "inlining call to hh" } // Issue #14768 - make sure we can inline for loops. @@ -326,9 +332,9 @@ func ii() { // ERROR "can inline ii" // Issue #42194 - make sure that functions evaluated in // go and defer statements can be inlined. func gd1(int) { - defer gd1(gd2()) // ERROR "inlining call to gd2" + defer gd1(gd2()) // ERROR "inlining call to gd2" "can inline gd1.deferwrap1" defer gd3()() // ERROR "inlining call to gd3" - go gd1(gd2()) // ERROR "inlining call to gd2" + go gd1(gd2()) // ERROR "inlining call to gd2" "can inline gd1.gowrap2" go gd3()() // ERROR "inlining call to gd3" } diff --git a/test/inline_endian.go b/test/inline_endian.go index fc94321de03b94..0466b4adeef02c 100644 --- a/test/inline_endian.go +++ b/test/inline_endian.go @@ -21,15 +21,15 @@ func endian(b []byte) uint64 { // ERROR "can inline endian" "b does not escape" } func appendLittleEndian(b []byte) []byte { // ERROR "can inline appendLittleEndian" "leaking param: b to result ~r0 level=0" - b = binary.LittleEndian.AppendUint64(b, 64) // ERROR "inlining call to binary.littleEndian.AppendUint64" - b = binary.LittleEndian.AppendUint32(b, 32) // ERROR "inlining call to binary.littleEndian.AppendUint32" - b = binary.LittleEndian.AppendUint16(b, 16) // ERROR "inlining call to binary.littleEndian.AppendUint16" + b = binary.LittleEndian.AppendUint64(b, 64) // ERROR "inlining call to binary.littleEndian.AppendUint64" "append escapes to heap" + b = binary.LittleEndian.AppendUint32(b, 32) // ERROR "inlining call to binary.littleEndian.AppendUint32" "append escapes to heap" + b = binary.LittleEndian.AppendUint16(b, 16) // ERROR "inlining call to binary.littleEndian.AppendUint16" "append escapes to heap" return b } func appendBigEndian(b []byte) []byte { // ERROR "can inline appendBigEndian" "leaking param: b to result ~r0 level=0" - b = binary.BigEndian.AppendUint64(b, 64) // ERROR "inlining call to binary.bigEndian.AppendUint64" - b = binary.BigEndian.AppendUint32(b, 32) // ERROR "inlining call to binary.bigEndian.AppendUint32" - b = binary.BigEndian.AppendUint16(b, 16) // ERROR "inlining call to binary.bigEndian.AppendUint16" + b = binary.BigEndian.AppendUint64(b, 64) // ERROR "inlining call to binary.bigEndian.AppendUint64" "append escapes to heap" + b = binary.BigEndian.AppendUint32(b, 32) // ERROR "inlining call to binary.bigEndian.AppendUint32" "append escapes to heap" + b = binary.BigEndian.AppendUint16(b, 16) // ERROR "inlining call to binary.bigEndian.AppendUint16" "append escapes to heap" return b } diff --git a/test/inline_sync.go b/test/inline_sync.go index eaa2176d5fdc3d..8359aa3aa22cd2 100644 --- a/test/inline_sync.go +++ b/test/inline_sync.go @@ -37,7 +37,7 @@ var once *sync.Once func small7() { // ERROR "can inline small7" // the Do fast path should be inlined - once.Do(small5) // ERROR "inlining call to sync\.\(\*Once\)\.Do" "inlining call to atomic\.\(\*Uint32\)\.Load" + once.Do(small5) // ERROR "inlining call to sync\.\(\*Once\)\.Do" "inlining call to atomic\.\(\*Bool\)\.Load" } var rwmutex *sync.RWMutex diff --git a/test/inline_testingbloop.go b/test/inline_testingbloop.go new file mode 100644 index 00000000000000..702a652f562b57 --- /dev/null +++ b/test/inline_testingbloop.go @@ -0,0 +1,37 @@ +// errorcheck -0 -m + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test no inlining of function calls in testing.B.Loop. +// See issue #61515. + +package foo + +import "testing" + +func caninline(x int) int { // ERROR "can inline caninline" + return x +} + +func test(b *testing.B) { // ERROR "leaking param: b" + for i := 0; i < b.N; i++ { + caninline(1) // ERROR "inlining call to caninline" + } + for b.Loop() { // ERROR "skip inlining within testing.B.loop" "inlining call to testing\.\(\*B\)\.Loop" + caninline(1) + } + for i := 0; i < b.N; i++ { + caninline(1) // ERROR "inlining call to caninline" + } + for b.Loop() { // ERROR "skip inlining within testing.B.loop" "inlining call to testing\.\(\*B\)\.Loop" + caninline(1) + } + for i := 0; i < b.N; i++ { + caninline(1) // ERROR "inlining call to caninline" + } + for b.Loop() { // ERROR "skip inlining within testing.B.loop" "inlining call to testing\.\(\*B\)\.Loop" + caninline(1) + } +} diff --git a/test/internal/runtime/sys/README b/test/internal/runtime/sys/README new file mode 100644 index 00000000000000..919e0bb03f45df --- /dev/null +++ b/test/internal/runtime/sys/README @@ -0,0 +1,7 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +The internal/runtime/sys directory contains tests that specifically need to be +compiled as-if in the internal/runtime/sys package. For error-check tests, +these require the additional flags -+ and -p=internal/runtime/sys. diff --git a/test/internal/runtime/sys/inlinegcpc.go b/test/internal/runtime/sys/inlinegcpc.go new file mode 100644 index 00000000000000..7cadc639d10169 --- /dev/null +++ b/test/internal/runtime/sys/inlinegcpc.go @@ -0,0 +1,29 @@ +// errorcheck -0 -+ -p=internal/runtime/sys -m + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sys + +// A function that calls sys.GetCallerPC or sys.GetCallerSP +// cannot be inlined, no matter how small it is. + +func GetCallerPC() uintptr +func GetCallerSP() uintptr + +func pc() uintptr { + return GetCallerPC() + 1 +} + +func cpc() uintptr { // ERROR "can inline cpc" + return pc() + 2 +} + +func sp() uintptr { + return GetCallerSP() + 3 +} + +func csp() uintptr { // ERROR "can inline csp" + return sp() + 4 +} diff --git a/test/live.go b/test/live.go index 5658c8ba06763e..56b78ccf8b4af2 100644 --- a/test/live.go +++ b/test/live.go @@ -337,23 +337,47 @@ func f20() { ch <- byteptr() } -func f21() { +func f21(x, y string) { // ERROR "live at entry to f21: x y" // key temporary for mapaccess using array literal key. var z *byte if b { - z = m2[[2]string{"x", "y"}] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" + z = m2[[2]string{x, y}] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" } z = m2[[2]string{"x", "y"}] z = m2[[2]string{"x", "y"}] printbytepointer(z) } -func f23() { +func f21b() { + // key temporary for mapaccess using array literal key. + var z *byte + if b { + z = m2[[2]string{"x", "y"}] + } + z = m2[[2]string{"x", "y"}] + z = m2[[2]string{"x", "y"}] + printbytepointer(z) +} + +func f23(x, y string) { // ERROR "live at entry to f23: x y" + // key temporary for two-result map access using array literal key. + var z *byte + var ok bool + if b { + z, ok = m2[[2]string{x, y}] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" + } + z, ok = m2[[2]string{"x", "y"}] + z, ok = m2[[2]string{"x", "y"}] + printbytepointer(z) + print(ok) +} + +func f23b() { // key temporary for two-result map access using array literal key. var z *byte var ok bool if b { - z, ok = m2[[2]string{"x", "y"}] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" + z, ok = m2[[2]string{"x", "y"}] } z, ok = m2[[2]string{"x", "y"}] z, ok = m2[[2]string{"x", "y"}] @@ -361,11 +385,21 @@ func f23() { print(ok) } -func f24() { +func f24(x, y string) { // ERROR "live at entry to f24: x y" // key temporary for map access using array literal key. // value temporary too. if b { - m2[[2]string{"x", "y"}] = nil // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" + m2[[2]string{x, y}] = nil // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" + } + m2[[2]string{"x", "y"}] = nil + m2[[2]string{"x", "y"}] = nil +} + +func f24b() { + // key temporary for map access using array literal key. + // value temporary too. + if b { + m2[[2]string{"x", "y"}] = nil } m2[[2]string{"x", "y"}] = nil m2[[2]string{"x", "y"}] = nil @@ -433,9 +467,9 @@ func f27defer(b bool) { func f27go(b bool) { x := 0 if b { - go call27(func() { x++ }) // ERROR "live at call to newobject: &x$" "live at call to newobject: &x .autotmp_[0-9]+$" "live at call to newproc: &x$" // allocate two closures, the func literal, and the wrapper for go + go call27(func() { x++ }) // ERROR "live at call to mallocgcSmallScanNoHeaderSC[0-9]+: &x$" "live at call to mallocgcSmallScanNoHeaderSC[0-9]+: &x .autotmp_[0-9]+$" "live at call to newproc: &x$" // allocate two closures, the func literal, and the wrapper for go } - go call27(func() { x++ }) // ERROR "live at call to newobject: &x$" "live at call to newobject: .autotmp_[0-9]+$" // allocate two closures, the func literal, and the wrapper for go + go call27(func() { x++ }) // ERROR "live at call to mallocgcSmallScanNoHeaderSC[0-9]+: &x$" "live at call to mallocgcSmallScanNoHeaderSC[0-9]+: .autotmp_[0-9]+$" // allocate two closures, the func literal, and the wrapper for go printnl() } @@ -458,14 +492,14 @@ func f28(b bool) { func f29(b bool) { if b { - for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ runtime.hiter$" + for k := range m { // ERROR "live at call to (mapiterinit|mapIterStart): .autotmp_[0-9]+$" "live at call to (mapiternext|mapIterNext): .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ internal/runtime/maps.Iter$" printstring(k) // ERROR "live at call to printstring: .autotmp_[0-9]+$" } } - for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" + for k := range m { // ERROR "live at call to (mapiterinit|mapIterStart): .autotmp_[0-9]+$" "live at call to (mapiternext|mapIterNext): .autotmp_[0-9]+$" printstring(k) // ERROR "live at call to printstring: .autotmp_[0-9]+$" } - for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" + for k := range m { // ERROR "live at call to (mapiterinit|mapIterStart): .autotmp_[0-9]+$" "live at call to (mapiternext|mapIterNext): .autotmp_[0-9]+$" printstring(k) // ERROR "live at call to printstring: .autotmp_[0-9]+$" } } @@ -504,7 +538,7 @@ func f31(b1, b2, b3 bool) { g31(g18()) // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" } if b2 { - h31(g18()) // ERROR "live at call to convT: .autotmp_[0-9]+$" "live at call to newobject: .autotmp_[0-9]+$" + h31(g18()) // ERROR "live at call to convT: .autotmp_[0-9]+$" "live at call to mallocgcSmallScanNoHeaderSC[0-9]+: .autotmp_[0-9]+$" } if b3 { panic(g18()) @@ -631,14 +665,14 @@ func f39a() (x []int) { func f39b() (x [10]*int) { x = [10]*int{} - x[0] = new(int) // ERROR "live at call to newobject: x$" + x[0] = new(int) // ERROR "live at call to mallocTiny[48]: x$" printnl() // ERROR "live at call to printnl: x$" return x } func f39c() (x [10]*int) { x = [10]*int{} - x[0] = new(int) // ERROR "live at call to newobject: x$" + x[0] = new(int) // ERROR "live at call to mallocTiny[48]: x$" printnl() // ERROR "live at call to printnl: x$" return } @@ -659,15 +693,9 @@ func newT40() *T40 { return &ret } -func bad40() { - t := newT40() - _ = t - printnl() -} - func good40() { ret := T40{} // ERROR "stack object ret T40$" - ret.m = make(map[int]int) // ERROR "live at call to rand32: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ runtime.hmap$" + ret.m = make(map[int]int) // ERROR "live at call to rand(32)?: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ internal/runtime/maps.Map$" t := &ret printnl() // ERROR "live at call to printnl: ret$" // Note: ret is live at the printnl because the compiler moves &ret @@ -675,6 +703,12 @@ func good40() { useT40(t) } +func bad40() { + t := newT40() + _ = t + printnl() +} + func ddd1(x, y *int) { // ERROR "live at entry to ddd1: x y$" ddd2(x, y) // ERROR "stack object .autotmp_[0-9]+ \[2\]\*int$" printnl() diff --git a/test/live2.go b/test/live2.go index 2beac4f8d2bbe4..d88ee6df2ce348 100644 --- a/test/live2.go +++ b/test/live2.go @@ -27,14 +27,14 @@ func newT40() *T40 { } func bad40() { - t := newT40() // ERROR "stack object ret T40$" "stack object .autotmp_[0-9]+ runtime.hmap$" + t := newT40() // ERROR "stack object ret T40$" "stack object .autotmp_[0-9]+ internal/runtime/maps.Map$" printnl() // ERROR "live at call to printnl: ret$" useT40(t) } func good40() { ret := T40{} // ERROR "stack object ret T40$" - ret.m = make(map[int]int, 42) // ERROR "stack object .autotmp_[0-9]+ runtime.hmap$" + ret.m = make(map[int]int, 42) // ERROR "stack object .autotmp_[0-9]+ internal/runtime/maps.Map$" t := &ret printnl() // ERROR "live at call to printnl: ret$" useT40(t) diff --git a/test/live_regabi.go b/test/live_regabi.go index a335126b3f558f..838cbdefad7c5b 100644 --- a/test/live_regabi.go +++ b/test/live_regabi.go @@ -278,6 +278,7 @@ func f17b(p *byte) { // ERROR "live at entry to f17b: p$" // key temporary if b { m2s[str()] = p // ERROR "live at call to mapassign_faststr: p$" "live at call to str: p$" + } m2s[str()] = p // ERROR "live at call to mapassign_faststr: p$" "live at call to str: p$" m2s[str()] = p // ERROR "live at call to mapassign_faststr: p$" "live at call to str: p$" @@ -334,23 +335,47 @@ func f20() { ch <- byteptr() } -func f21() { +func f21(x, y string) { // ERROR "live at entry to f21: x y" + // key temporary for mapaccess using array literal key. + var z *byte + if b { + z = m2[[2]string{x, y}] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" + } + z = m2[[2]string{"x", "y"}] + z = m2[[2]string{"x", "y"}] + printbytepointer(z) +} + +func f21b() { // key temporary for mapaccess using array literal key. var z *byte if b { - z = m2[[2]string{"x", "y"}] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" + z = m2[[2]string{"x", "y"}] } z = m2[[2]string{"x", "y"}] z = m2[[2]string{"x", "y"}] printbytepointer(z) } -func f23() { +func f23(x, y string) { // ERROR "live at entry to f23: x y" + // key temporary for two-result map access using array literal key. + var z *byte + var ok bool + if b { + z, ok = m2[[2]string{x, y}] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" + } + z, ok = m2[[2]string{"x", "y"}] + z, ok = m2[[2]string{"x", "y"}] + printbytepointer(z) + print(ok) +} + +func f23b() { // key temporary for two-result map access using array literal key. var z *byte var ok bool if b { - z, ok = m2[[2]string{"x", "y"}] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" + z, ok = m2[[2]string{"x", "y"}] } z, ok = m2[[2]string{"x", "y"}] z, ok = m2[[2]string{"x", "y"}] @@ -358,11 +383,21 @@ func f23() { print(ok) } -func f24() { +func f24(x, y string) { // ERROR "live at entry to f24: x y" + // key temporary for map access using array lit3ral key. + // value temporary too. + if b { + m2[[2]string{x, y}] = nil // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" + } + m2[[2]string{"x", "y"}] = nil + m2[[2]string{"x", "y"}] = nil +} + +func f24b() { // key temporary for map access using array literal key. // value temporary too. if b { - m2[[2]string{"x", "y"}] = nil // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" + m2[[2]string{"x", "y"}] = nil } m2[[2]string{"x", "y"}] = nil m2[[2]string{"x", "y"}] = nil @@ -430,9 +465,9 @@ func f27defer(b bool) { func f27go(b bool) { x := 0 if b { - go call27(func() { x++ }) // ERROR "live at call to newobject: &x$" "live at call to newobject: &x .autotmp_[0-9]+$" "live at call to newproc: &x$" // allocate two closures, the func literal, and the wrapper for go + go call27(func() { x++ }) // ERROR "live at call to mallocgcSmallScanNoHeaderSC[0-9]+: &x$" "live at call to mallocgcSmallScanNoHeaderSC[0-9]+: &x .autotmp_[0-9]+$" "live at call to newproc: &x$" // allocate two closures, the func literal, and the wrapper for go } - go call27(func() { x++ }) // ERROR "live at call to newobject: &x$" "live at call to newobject: .autotmp_[0-9]+$" // allocate two closures, the func literal, and the wrapper for go + go call27(func() { x++ }) // ERROR "live at call to mallocgcSmallScanNoHeaderSC[0-9]+: &x$" "live at call to mallocgcSmallScanNoHeaderSC[0-9]+: .autotmp_[0-9]+$" // allocate two closures, the func literal, and the wrapper for go printnl() } @@ -455,14 +490,14 @@ func f28(b bool) { func f29(b bool) { if b { - for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ runtime.hiter$" + for k := range m { // ERROR "live at call to (mapiterinit|mapIterStart): .autotmp_[0-9]+$" "live at call to (mapiternext|mapIterNext): .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ internal/runtime/maps.Iter$" printstring(k) // ERROR "live at call to printstring: .autotmp_[0-9]+$" } } - for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" + for k := range m { // ERROR "live at call to (mapiterinit|mapIterStart): .autotmp_[0-9]+$" "live at call to (mapiternext|mapIterNext): .autotmp_[0-9]+$" printstring(k) // ERROR "live at call to printstring: .autotmp_[0-9]+$" } - for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" + for k := range m { // ERROR "live at call to (mapiterinit|mapIterStart): .autotmp_[0-9]+$" "live at call to (mapiternext|mapIterNext): .autotmp_[0-9]+$" printstring(k) // ERROR "live at call to printstring: .autotmp_[0-9]+$" } } @@ -501,7 +536,7 @@ func f31(b1, b2, b3 bool) { g31(g18()) // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" } if b2 { - h31(g18()) // ERROR "live at call to convT: .autotmp_[0-9]+$" "live at call to newobject: .autotmp_[0-9]+$" + h31(g18()) // ERROR "live at call to convT: .autotmp_[0-9]+$" "live at call to mallocgcSmallScanNoHeaderSC[0-9]+: .autotmp_[0-9]+$" } if b3 { panic(g18()) @@ -628,14 +663,14 @@ func f39a() (x []int) { func f39b() (x [10]*int) { x = [10]*int{} - x[0] = new(int) // ERROR "live at call to newobject: x$" + x[0] = new(int) // ERROR "live at call to mallocTiny[48]: x$" printnl() // ERROR "live at call to printnl: x$" return x } func f39c() (x [10]*int) { x = [10]*int{} - x[0] = new(int) // ERROR "live at call to newobject: x$" + x[0] = new(int) // ERROR "live at call to mallocTiny[48]: x$" printnl() // ERROR "live at call to printnl: x$" return } @@ -656,15 +691,9 @@ func newT40() *T40 { return &ret } -func bad40() { - t := newT40() - _ = t - printnl() -} - func good40() { ret := T40{} // ERROR "stack object ret T40$" - ret.m = make(map[int]int) // ERROR "live at call to rand32: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ runtime.hmap$" + ret.m = make(map[int]int) // ERROR "live at call to rand(32)?: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ internal/runtime/maps.Map$" t := &ret printnl() // ERROR "live at call to printnl: ret$" // Note: ret is live at the printnl because the compiler moves &ret @@ -672,6 +701,12 @@ func good40() { useT40(t) } +func bad40() { + t := newT40() + _ = t + printnl() +} + func ddd1(x, y *int) { // ERROR "live at entry to ddd1: x y$" ddd2(x, y) // ERROR "stack object .autotmp_[0-9]+ \[2\]\*int$" printnl() diff --git a/test/loopbce.go b/test/loopbce.go index 04c186be0eb194..8a58d942361221 100644 --- a/test/loopbce.go +++ b/test/loopbce.go @@ -9,7 +9,7 @@ import "math" func f0a(a []int) int { x := 0 for i := range a { // ERROR "Induction variable: limits \[0,\?\), increment 1$" - x += a[i] // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + x += a[i] // ERROR "Proved IsInBounds$" } return x } @@ -17,7 +17,7 @@ func f0a(a []int) int { func f0b(a []int) int { x := 0 for i := range a { // ERROR "Induction variable: limits \[0,\?\), increment 1$" - b := a[i:] // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" + b := a[i:] // ERROR "Proved IsSliceInBounds$" x += b[0] } return x @@ -26,8 +26,8 @@ func f0b(a []int) int { func f0c(a []int) int { x := 0 for i := range a { // ERROR "Induction variable: limits \[0,\?\), increment 1$" - b := a[:i+1] // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" - x += b[0] + b := a[:i+1] // ERROR "Proved IsSliceInBounds$" + x += b[0] // ERROR "Proved IsInBounds$" } return x } @@ -43,7 +43,7 @@ func f1(a []int) int { func f2(a []int) int { x := 0 for i := 1; i < len(a); i++ { // ERROR "Induction variable: limits \[1,\?\), increment 1$" - x += a[i] // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + x += a[i] // ERROR "Proved IsInBounds$" } return x } @@ -51,7 +51,7 @@ func f2(a []int) int { func f4(a [10]int) int { x := 0 for i := 0; i < len(a); i += 2 { // ERROR "Induction variable: limits \[0,8\], increment 2$" - x += a[i] // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + x += a[i] // ERROR "Proved IsInBounds$" } return x } @@ -88,9 +88,10 @@ func f5_int8(a [10]int) int { return x } +//go:noinline func f6(a []int) { for i := range a { // ERROR "Induction variable: limits \[0,\?\), increment 1$" - b := a[0:i] // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" + b := a[0:i] // ERROR "Proved IsSliceInBounds$" f6(b) } } @@ -98,7 +99,7 @@ func f6(a []int) { func g0a(a string) int { x := 0 for i := 0; i < len(a); i++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" - x += int(a[i]) // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + x += int(a[i]) // ERROR "Proved IsInBounds$" } return x } @@ -106,7 +107,7 @@ func g0a(a string) int { func g0b(a string) int { x := 0 for i := 0; len(a) > i; i++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" - x += int(a[i]) // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + x += int(a[i]) // ERROR "Proved IsInBounds$" } return x } @@ -114,7 +115,7 @@ func g0b(a string) int { func g0c(a string) int { x := 0 for i := len(a); i > 0; i-- { // ERROR "Induction variable: limits \(0,\?\], increment 1$" - x += int(a[i-1]) // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + x += int(a[i-1]) // ERROR "Proved IsInBounds$" } return x } @@ -122,7 +123,7 @@ func g0c(a string) int { func g0d(a string) int { x := 0 for i := len(a); 0 < i; i-- { // ERROR "Induction variable: limits \(0,\?\], increment 1$" - x += int(a[i-1]) // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + x += int(a[i-1]) // ERROR "Proved IsInBounds$" } return x } @@ -130,7 +131,7 @@ func g0d(a string) int { func g0e(a string) int { x := 0 for i := len(a) - 1; i >= 0; i-- { // ERROR "Induction variable: limits \[0,\?\], increment 1$" - x += int(a[i]) // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + x += int(a[i]) // ERROR "Proved IsInBounds$" } return x } @@ -138,7 +139,7 @@ func g0e(a string) int { func g0f(a string) int { x := 0 for i := len(a) - 1; 0 <= i; i-- { // ERROR "Induction variable: limits \[0,\?\], increment 1$" - x += int(a[i]) // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + x += int(a[i]) // ERROR "Proved IsInBounds$" } return x } @@ -147,7 +148,7 @@ func g1() int { a := "evenlength" x := 0 for i := 0; i < len(a); i += 2 { // ERROR "Induction variable: limits \[0,8\], increment 2$" - x += int(a[i]) // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + x += int(a[i]) // ERROR "Proved IsInBounds$" } return x } @@ -157,7 +158,7 @@ func g2() int { x := 0 for i := 0; i < len(a); i += 2 { // ERROR "Induction variable: limits \[0,8\], increment 2$" j := i - if a[i] == 'e' { // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + if a[i] == 'e' { // ERROR "Proved IsInBounds$" j = j + 1 } x += int(a[j]) @@ -168,29 +169,29 @@ func g2() int { func g3a() { a := "this string has length 25" for i := 0; i < len(a); i += 5 { // ERROR "Induction variable: limits \[0,20\], increment 5$" - useString(a[i:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" - useString(a[:i+3]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" - useString(a[:i+5]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" + useString(a[i:]) // ERROR "Proved IsSliceInBounds$" "Proved slicemask not needed \(by limit\)$" + useString(a[:i+3]) // ERROR "Proved IsSliceInBounds$" + useString(a[:i+5]) // ERROR "Proved IsSliceInBounds$" useString(a[:i+6]) } } func g3b(a string) { for i := 0; i < len(a); i++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" - useString(a[i+1:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" + useString(a[i+1:]) // ERROR "Proved IsSliceInBounds$" } } func g3c(a string) { for i := 0; i < len(a); i++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" - useString(a[:i+1]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" + useString(a[:i+1]) // ERROR "Proved IsSliceInBounds$" } } func h1(a []byte) { c := a[:128] for i := range c { // ERROR "Induction variable: limits \[0,128\), increment 1$" - c[i] = byte(i) // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + c[i] = byte(i) // ERROR "Proved IsInBounds$" } } @@ -207,11 +208,11 @@ func k0(a [100]int) [100]int { continue } a[i-11] = i - a[i-10] = i // ERROR "(\([0-9]+\) )?Proved IsInBounds$" - a[i-5] = i // ERROR "(\([0-9]+\) )?Proved IsInBounds$" - a[i] = i // ERROR "(\([0-9]+\) )?Proved IsInBounds$" - a[i+5] = i // ERROR "(\([0-9]+\) )?Proved IsInBounds$" - a[i+10] = i // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + a[i-10] = i // ERROR "Proved IsInBounds$" + a[i-5] = i // ERROR "Proved IsInBounds$" + a[i] = i // ERROR "Proved IsInBounds$" + a[i+5] = i // ERROR "Proved IsInBounds$" + a[i+10] = i // ERROR "Proved IsInBounds$" a[i+11] = i } return a @@ -224,12 +225,12 @@ func k1(a [100]int) [100]int { continue } useSlice(a[:i-11]) - useSlice(a[:i-10]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" - useSlice(a[:i-5]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" - useSlice(a[:i]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" - useSlice(a[:i+5]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" - useSlice(a[:i+10]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" - useSlice(a[:i+11]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" + useSlice(a[:i-10]) // ERROR "Proved IsSliceInBounds$" + useSlice(a[:i-5]) // ERROR "Proved IsSliceInBounds$" + useSlice(a[:i]) // ERROR "Proved IsSliceInBounds$" + useSlice(a[:i+5]) // ERROR "Proved IsSliceInBounds$" + useSlice(a[:i+10]) // ERROR "Proved IsSliceInBounds$" + useSlice(a[:i+11]) // ERROR "Proved IsSliceInBounds$" useSlice(a[:i+12]) } @@ -242,13 +243,13 @@ func k2(a [100]int) [100]int { // This is a trick to prohibit sccp to optimize out the following out of bound check continue } - useSlice(a[i-11:]) - useSlice(a[i-10:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" - useSlice(a[i-5:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" - useSlice(a[i:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" - useSlice(a[i+5:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" - useSlice(a[i+10:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" - useSlice(a[i+11:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" + useSlice(a[i-11:]) // ERROR "Proved slicemask not needed \(by limit\)$" + useSlice(a[i-10:]) // ERROR "Proved IsSliceInBounds$" "Proved slicemask not needed \(by limit\)$" + useSlice(a[i-5:]) // ERROR "Proved IsSliceInBounds$" "Proved slicemask not needed \(by limit\)$" + useSlice(a[i:]) // ERROR "Proved IsSliceInBounds$" "Proved slicemask not needed \(by limit\)$" + useSlice(a[i+5:]) // ERROR "Proved IsSliceInBounds$" "Proved slicemask not needed \(by limit\)$" + useSlice(a[i+10:]) // ERROR "Proved IsSliceInBounds$" "Proved slicemask not needed \(by limit\)$" + useSlice(a[i+11:]) // ERROR "Proved IsSliceInBounds$" useSlice(a[i+12:]) } return a @@ -261,7 +262,7 @@ func k3(a [100]int) [100]int { continue } a[i+9] = i - a[i+10] = i // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + a[i+10] = i // ERROR "Proved IsInBounds$" a[i+11] = i } return a @@ -274,7 +275,7 @@ func k3neg(a [100]int) [100]int { continue } a[i+9] = i - a[i+10] = i // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + a[i+10] = i // ERROR "Proved IsInBounds$" a[i+11] = i } return a @@ -287,16 +288,18 @@ func k3neg2(a [100]int) [100]int { continue } a[i+9] = i - a[i+10] = i // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + a[i+10] = i // ERROR "Proved IsInBounds$" a[i+11] = i } return a } func k4(a [100]int) [100]int { - min := (-1) << 63 - for i := min; i < min+50; i++ { // ERROR "Induction variable: limits \[-9223372036854775808,-9223372036854775758\), increment 1$" - a[i-min] = i // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + // Note: can't use (-1)<<63 here, because i-min doesn't get rewritten to i+(-min), + // and it isn't worth adding that special case to prove. + min := (-1)<<63 + 1 + for i := min; i < min+50; i++ { // ERROR "Induction variable: limits \[-9223372036854775807,-9223372036854775757\), increment 1$" + a[i-min] = i // ERROR "Proved IsInBounds$" } return a } @@ -304,8 +307,8 @@ func k4(a [100]int) [100]int { func k5(a [100]int) [100]int { max := (1 << 63) - 1 for i := max - 50; i < max; i++ { // ERROR "Induction variable: limits \[9223372036854775757,9223372036854775807\), increment 1$" - a[i-max+50] = i // ERROR "(\([0-9]+\) )?Proved IsInBounds$" - a[i-(max-70)] = i // ERROR "(\([0-9]+\) )?Proved IsInBounds$" + a[i-max+50] = i // ERROR "Proved IsInBounds$" + a[i-(max-70)] = i // ERROR "Proved IsInBounds$" } return a } @@ -314,7 +317,7 @@ func d1(a [100]int) [100]int { for i := 0; i < 100; i++ { // ERROR "Induction variable: limits \[0,100\), increment 1$" for j := 0; j < i; j++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" a[j] = 0 // ERROR "Proved IsInBounds$" - a[j+1] = 0 // FIXME: this boundcheck should be eliminated + a[j+1] = 0 // ERROR "Proved IsInBounds$" a[j+2] = 0 } } @@ -325,7 +328,7 @@ func d2(a [100]int) [100]int { for i := 0; i < 100; i++ { // ERROR "Induction variable: limits \[0,100\), increment 1$" for j := 0; i > j; j++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" a[j] = 0 // ERROR "Proved IsInBounds$" - a[j+1] = 0 // FIXME: this boundcheck should be eliminated + a[j+1] = 0 // ERROR "Proved IsInBounds$" a[j+2] = 0 } } @@ -371,22 +374,22 @@ func d4() { } func d5() { - for i := int64(math.MinInt64 + 9); i > math.MinInt64+2; i -= 4 { // ERROR "Induction variable: limits \[-9223372036854775803,-9223372036854775799\], increment 4" + for i := int64(math.MinInt64 + 9); i > math.MinInt64+2; i -= 4 { // ERROR "Induction variable: limits \[-9223372036854775803,-9223372036854775799\], increment 4$" useString("foo") } - for i := int64(math.MinInt64 + 8); i > math.MinInt64+2; i -= 4 { // ERROR "Induction variable: limits \[-9223372036854775804,-9223372036854775800\], increment 4" + for i := int64(math.MinInt64 + 8); i > math.MinInt64+2; i -= 4 { // ERROR "Induction variable: limits \[-9223372036854775804,-9223372036854775800\], increment 4$" useString("foo") } for i := int64(math.MinInt64 + 7); i > math.MinInt64+2; i -= 4 { useString("foo") } - for i := int64(math.MinInt64 + 6); i > math.MinInt64+2; i -= 4 { // ERROR "Induction variable: limits \[-9223372036854775802,-9223372036854775802\], increment 4" + for i := int64(math.MinInt64 + 6); i > math.MinInt64+2; i -= 4 { // ERROR "Induction variable: limits \[-9223372036854775802,-9223372036854775802\], increment 4$" useString("foo") } - for i := int64(math.MinInt64 + 9); i >= math.MinInt64+2; i -= 4 { // ERROR "Induction variable: limits \[-9223372036854775803,-9223372036854775799\], increment 4" + for i := int64(math.MinInt64 + 9); i >= math.MinInt64+2; i -= 4 { // ERROR "Induction variable: limits \[-9223372036854775803,-9223372036854775799\], increment 4$" useString("foo") } - for i := int64(math.MinInt64 + 8); i >= math.MinInt64+2; i -= 4 { // ERROR "Induction variable: limits \[-9223372036854775804,-9223372036854775800\], increment 4" + for i := int64(math.MinInt64 + 8); i >= math.MinInt64+2; i -= 4 { // ERROR "Induction variable: limits \[-9223372036854775804,-9223372036854775800\], increment 4$" useString("foo") } for i := int64(math.MinInt64 + 7); i >= math.MinInt64+2; i -= 4 { @@ -407,24 +410,24 @@ func bce1() { panic("invalid test: modulos should differ") } - for i := b; i < a; i += z { // ERROR "Induction variable: limits \[-1547,9223372036854772720\], increment 1337" + for i := b; i < a; i += z { // ERROR "Induction variable: limits \[-1547,9223372036854772720\], increment 1337$" useString("foobar") } } func nobce2(a string) { for i := int64(0); i < int64(len(a)); i++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" - useString(a[i:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" + useString(a[i:]) // ERROR "Proved IsSliceInBounds$" } for i := int64(0); i < int64(len(a))-31337; i++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" - useString(a[i:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" + useString(a[i:]) // ERROR "Proved IsSliceInBounds$" } - for i := int64(0); i < int64(len(a))+int64(-1<<63); i++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" - useString(a[i:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" + for i := int64(0); i < int64(len(a))+int64(-1<<63); i++ { // ERROR "Disproved Less64$" "Induction variable: limits \[0,\?\), increment 1$" + useString(a[i:]) } j := int64(len(a)) - 123 - for i := int64(0); i < j+123+int64(-1<<63); i++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" - useString(a[i:]) // ERROR "(\([0-9]+\) )?Proved IsSliceInBounds$" + for i := int64(0); i < j+123+int64(-1<<63); i++ { // ERROR "Disproved Less64$" "Induction variable: limits \[0,\?\), increment 1$" + useString(a[i:]) } for i := int64(0); i < j+122+int64(-1<<63); i++ { // ERROR "Induction variable: limits \[0,\?\), increment 1$" // len(a)-123+122+MinInt overflows when len(a) == 0, so a bound check is needed here @@ -452,16 +455,16 @@ func issue26116a(a []int) { func stride1(x *[7]int) int { s := 0 - for i := 0; i <= 8; i += 3 { // ERROR "Induction variable: limits \[0,6\], increment 3" - s += x[i] // ERROR "Proved IsInBounds" + for i := 0; i <= 8; i += 3 { // ERROR "Induction variable: limits \[0,6\], increment 3$" + s += x[i] // ERROR "Proved IsInBounds$" } return s } func stride2(x *[7]int) int { s := 0 - for i := 0; i < 9; i += 3 { // ERROR "Induction variable: limits \[0,6\], increment 3" - s += x[i] // ERROR "Proved IsInBounds" + for i := 0; i < 9; i += 3 { // ERROR "Induction variable: limits \[0,6\], increment 3$" + s += x[i] // ERROR "Proved IsInBounds$" } return s } diff --git a/test/newexpr.go b/test/newexpr.go new file mode 100644 index 00000000000000..c9a8804d4eb31a --- /dev/null +++ b/test/newexpr.go @@ -0,0 +1,39 @@ +// run + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +// Issue #45624 is the proposal to accept new(expr) in go1.26. +// Here we test its run-time behavior. +func main() { + { + p := new(123) // untyped constant expr + if *p != 123 { + panic("wrong value") + } + } + { + x := 42 + p := new(x) // non-constant expr + if *p != x { + panic("wrong value") + } + } + { + x := [2]int{123, 456} + p := new(x) // composite value + if *p != x { + panic("wrong value") + } + } + { + var i int + v := new(i > 0) // untyped expression, see issue #75617 + if *v != false { + panic("wrong value") + } + } +} diff --git a/test/newinline.go b/test/newinline.go index 69f1310ab2f3f4..da299d5543e64b 100644 --- a/test/newinline.go +++ b/test/newinline.go @@ -73,6 +73,7 @@ func l(x, y int) (int, int, error) { // ERROR "can inline l" f := e f(nil) // ERROR "inlining call to l.func1" } + _ = e // prevent simple deadcode elimination return y, x, nil } @@ -109,6 +110,7 @@ func p() int { // ERROR "can inline p" func q(x int) int { // ERROR "can inline q" foo := func() int { return x * 2 } // ERROR "can inline q.func1" "func literal does not escape" + _ = foo // prevent simple deadcode elimination return foo() // ERROR "inlining call to q.func1" } @@ -121,6 +123,8 @@ func r(z int) int { // ERROR "can inline r" return 2*y + x*z }(x) // ERROR "inlining call to r.func2.1" } + _ = foo // prevent simple deadcode elimination + _ = bar // prevent simple deadcode elimination return foo(42) + bar(42) // ERROR "inlining call to r.func1" "inlining call to r.func2" "inlining call to r.r.func2.func3" } @@ -128,7 +132,8 @@ func s0(x int) int { // ERROR "can inline s0" foo := func() { // ERROR "can inline s0.func1" "func literal does not escape" x = x + 1 } - foo() // ERROR "inlining call to s0.func1" + foo() // ERROR "inlining call to s0.func1" + _ = foo // prevent simple deadcode elimination return x } @@ -137,6 +142,7 @@ func s1(x int) int { // ERROR "can inline s1" return x } x = x + 1 + _ = foo // prevent simple deadcode elimination return foo() // ERROR "inlining call to s1.func1" } @@ -274,13 +280,13 @@ func ff(x int) { // ERROR "can inline ff" if x < 0 { return } - gg(x - 1) // ERROR "inlining call to gg" "inlining call to hh" + gg(x - 1) // ERROR "inlining call to gg" "inlining call to hh" "inlining call to ff" } func gg(x int) { // ERROR "can inline gg" - hh(x - 1) // ERROR "inlining call to hh" "inlining call to ff" + hh(x - 1) // ERROR "inlining call to hh" "inlining call to ff" "inlining call to gg" } func hh(x int) { // ERROR "can inline hh" - ff(x - 1) // ERROR "inlining call to ff" "inlining call to gg" + ff(x - 1) // ERROR "inlining call to ff" "inlining call to gg" "inlining call to hh" } // Issue #14768 - make sure we can inline for loops. diff --git a/test/nilptr.go b/test/nilptr.go index 7f42e930bdee7e..6a60b18fc110cf 100644 --- a/test/nilptr.go +++ b/test/nilptr.go @@ -7,8 +7,8 @@ // Test that the implementation catches nil ptr indirection // in a large address space. -// Address space starts at 1<<32 on AIX and on darwin/arm64 and on windows/arm64, so dummy is too far. -//go:build !aix && (!darwin || !arm64) && (!windows || !arm64) +// Address space starts at 1<<32 on AIX and on darwin/arm64 and on windows/[amd64/arm64], so dummy is too far. +//go:build !aix && (!darwin || !arm64) && (!windows || (!amd64 && !arm64)) package main diff --git a/test/nilptr5.go b/test/nilptr5.go index 51a302f2521827..a76592abadc61f 100644 --- a/test/nilptr5.go +++ b/test/nilptr5.go @@ -30,3 +30,9 @@ func f6(p, q *T) { func f8(t *struct{ b [8]int }) struct{ b [8]int } { return *t // ERROR "removed nil check" } + +// nil check is removed for pointer write (which involves a +// write barrier). +func f9(x **int, y *int) { + *x = y // ERROR "removed nil check" +} diff --git a/test/nilptr5_aix.go b/test/nilptr5_aix.go index 3b116ca19f5e70..cd8ecb1e49600f 100644 --- a/test/nilptr5_aix.go +++ b/test/nilptr5_aix.go @@ -30,3 +30,9 @@ func f6(p, q *T) { func f8(t *[8]int) [8]int { return *t // ERROR "generated nil check" } + +// On AIX, a write nil check is removed, but a read nil check +// remains (for the write barrier). +func f9(x **int, y *int) { + *x = y // ERROR "generated nil check" "removed nil check" +} diff --git a/test/nilptr5_wasm.go b/test/nilptr5_wasm.go index ea380d1384f702..e653ffd2870fcf 100644 --- a/test/nilptr5_wasm.go +++ b/test/nilptr5_wasm.go @@ -30,3 +30,8 @@ func f6(p, q *T) { func f8(t *[8]int) [8]int { return *t // ERROR "generated nil check" } + +// nil check is not removed on Wasm. +func f9(x **int, y *int) { + *x = y // ERROR "generated nil check" +} diff --git a/test/nosplit.go b/test/nosplit.go index e171d1da6618ec..4b4c93b1d067c5 100644 --- a/test/nosplit.go +++ b/test/nosplit.go @@ -1,6 +1,6 @@ // run -//go:build !nacl && !js && !aix && !wasip1 && !gcflags_noopt && gc +//go:build !nacl && !js && !aix && !openbsd && !wasip1 && !gcflags_noopt && gc // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/phiopt.go b/test/phiopt.go index 9e21bfdba55aea..206a117fbb2c4a 100644 --- a/test/phiopt.go +++ b/test/phiopt.go @@ -129,5 +129,27 @@ func f9(a, b int) bool { return c } +//go:noinline +func f10and(a bool, b bool) bool { + var x bool + if a { + x = b + } else { + x = a + } + return x // ERROR "converted OpPhi to AndB$" +} + +//go:noinline +func f11or(a bool, b bool) bool { + var x bool + if a { + x = a + } else { + x = b + } + return x // ERROR "converted OpPhi to OrB$" +} + func main() { } diff --git a/test/prove.go b/test/prove.go index 0d93db905ae871..c48280aa31e7b5 100644 --- a/test/prove.go +++ b/test/prove.go @@ -8,7 +8,10 @@ package main -import "math" +import ( + "math" + "math/bits" +) func f0(a []int) int { a[0] = 1 @@ -400,8 +403,8 @@ func f13f(a, b int64) int64 { if b != math.MaxInt64 { return 42 } - if a > b { - if a == 0 { // ERROR "Disproved Eq64$" + if a > b { // ERROR "Disproved Less64$" + if a == 0 { return 1 } } @@ -508,10 +511,10 @@ func f19() (e int64, err error) { func sm1(b []int, x int) { // Test constant argument to slicemask. - useSlice(b[2:8]) // ERROR "Proved slicemask not needed$" + useSlice(b[2:8]) // optimized away earlier by rewrite // Test non-constant argument with known limits. if cap(b) > 10 { - useSlice(b[2:]) + useSlice(b[2:]) // ERROR "Proved slicemask not needed$" } } @@ -684,20 +687,6 @@ func constsuffix(s string) bool { return suffix(s, "abc") // ERROR "Proved IsSliceInBounds$" } -// oforuntil tests the pattern created by OFORUNTIL blocks. These are -// handled by addLocalInductiveFacts rather than findIndVar. -func oforuntil(b []int) { - i := 0 - if len(b) > i { - top: - println(b[i]) // ERROR "Induction variable: limits \[0,\?\), increment 1$" "Proved IsInBounds$" - i++ - if i < len(b) { - goto top - } - } -} - func atexit(foobar []func()) { for i := len(foobar) - 1; i >= 0; i-- { // ERROR "Induction variable: limits \[0,\?\], increment 1" f := foobar[i] @@ -784,8 +773,8 @@ func indexGT0(b []byte, n int) { func unrollUpExcl(a []int) int { var i, x int for i = 0; i < len(a)-1; i += 2 { // ERROR "Induction variable: limits \[0,\?\), increment 2$" - x += a[i] // ERROR "Proved IsInBounds$" - x += a[i+1] + x += a[i] // ERROR "Proved IsInBounds$" + x += a[i+1] // ERROR "Proved IsInBounds( for blocked indexing)?$" } if i == len(a)-1 { x += a[i] @@ -797,8 +786,8 @@ func unrollUpExcl(a []int) int { func unrollUpIncl(a []int) int { var i, x int for i = 0; i <= len(a)-2; i += 2 { // ERROR "Induction variable: limits \[0,\?\], increment 2$" - x += a[i] // ERROR "Proved IsInBounds$" - x += a[i+1] + x += a[i] // ERROR "Proved IsInBounds$" + x += a[i+1] // ERROR "Proved IsInBounds( for blocked indexing)?$" } if i == len(a)-1 { x += a[i] @@ -850,7 +839,7 @@ func unrollExclStepTooLarge(a []int) int { var i, x int for i = 0; i < len(a)-1; i += 3 { x += a[i] - x += a[i+1] + x += a[i+1] // ERROR "Proved IsInBounds( for blocked indexing)?$" } if i == len(a)-1 { x += a[i] @@ -863,7 +852,7 @@ func unrollInclStepTooLarge(a []int) int { var i, x int for i = 0; i <= len(a)-2; i += 3 { x += a[i] - x += a[i+1] + x += a[i+1] // ERROR "Proved IsInBounds( for blocked indexing)?$" } if i == len(a)-1 { x += a[i] @@ -877,11 +866,11 @@ func unrollDecMin(a []int, b int) int { return 42 } var i, x int - for i = len(a); i >= b; i -= 2 { + for i = len(a); i >= b; i -= 2 { // ERROR "Proved Leq64" x += a[i-1] x += a[i-2] } - if i == 1 { // ERROR "Disproved Eq64$" + if i == 1 { x += a[i-1] } return x @@ -893,11 +882,11 @@ func unrollIncMin(a []int, b int) int { return 42 } var i, x int - for i = len(a); i >= b; i += 2 { + for i = len(a); i >= b; i += 2 { // ERROR "Proved Leq64" x += a[i-1] x += a[i-2] } - if i == 1 { // ERROR "Disproved Eq64$" + if i == 1 { x += a[i-1] } return x @@ -1107,7 +1096,7 @@ func modu2(x, y uint) int { func issue57077(s []int) (left, right []int) { middle := len(s) / 2 - left = s[:middle] // ERROR "Proved IsSliceInBounds$" + left = s[:middle] // ERROR "Proved IsSliceInBounds$" right = s[middle:] // ERROR "Proved IsSliceInBounds$" return } @@ -1124,6 +1113,1377 @@ func issue45928(x int) { useInt(combinedFrac) } +func constantBounds1(i, j uint) int { + var a [10]int + if j < 11 && i < j { + return a[i] // ERROR "Proved IsInBounds$" + } + return 0 +} + +func constantBounds2(i, j uint) int { + var a [10]int + if i < j && j < 11 { + return a[i] // ERROR "Proved IsInBounds" + } + return 0 +} + +func constantBounds3(i, j, k, l uint) int { + var a [8]int + if i < j && j < k && k < l && l < 11 { + return a[i] // ERROR "Proved IsInBounds" + } + return 0 +} + +func equalityPropagation(a [1]int, i, j uint) int { + if i == j && i == 5 { + return a[j-5] // ERROR "Proved IsInBounds" + } + return 0 +} +func inequalityPropagation(a [1]int, i, j uint) int { + if i != j && j >= 5 && j <= 6 && i == 5 { + return a[j-6] // ERROR "Proved IsInBounds" + } + return 0 +} + +func issue66826a(a [21]byte) { + for i := 0; i <= 10; i++ { // ERROR "Induction variable: limits \[0,10\], increment 1$" + _ = a[2*i] // ERROR "Proved IsInBounds" + } +} +func issue66826b(a [31]byte, i int) { + if i < 0 || i > 10 { + return + } + _ = a[3*i] // ERROR "Proved IsInBounds" +} + +func f20(a, b bool) int { + if a == b { + if a { + if b { // ERROR "Proved Arg" + return 1 + } + } + } + return 0 +} + +func f21(a, b *int) int { + if a == b { + if a != nil { + if b != nil { // ERROR "Proved IsNonNil" + return 1 + } + } + } + return 0 +} + +func f22(b bool, x, y int) int { + b2 := x < y + if b == b2 { + if b { + if x >= y { // ERROR "Disproved Leq64$" + return 1 + } + } + } + return 0 +} + +func ctz64(x uint64, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint64 + sz := bits.Len64(max) + + log2half := uint64(max) >> (sz / 2) + if x >= log2half || x == 0 { + return 42 + } + + y := bits.TrailingZeros64(x) // ERROR "Proved Ctz64 non-zero$"" + + z := sz / 2 + if ensureBothBranchesCouldHappen { + if y < z { // ERROR "Proved Less64$" + return -42 + } + } else { + if y >= z { // ERROR "Disproved Leq64$" + return 1337 + } + } + + return y +} +func ctz32(x uint32, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint32 + sz := bits.Len32(max) + + log2half := uint32(max) >> (sz / 2) + if x >= log2half || x == 0 { + return 42 + } + + y := bits.TrailingZeros32(x) // ERROR "Proved Ctz32 non-zero$"" + + z := sz / 2 + if ensureBothBranchesCouldHappen { + if y < z { // ERROR "Proved Less64$" + return -42 + } + } else { + if y >= z { // ERROR "Disproved Leq64$" + return 1337 + } + } + + return y +} +func ctz16(x uint16, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint16 + sz := bits.Len16(max) + + log2half := uint16(max) >> (sz / 2) + if x >= log2half || x == 0 { + return 42 + } + + y := bits.TrailingZeros16(x) // ERROR "Proved Ctz16 non-zero$"" + + z := sz / 2 + if ensureBothBranchesCouldHappen { + if y < z { // ERROR "Proved Less64$" + return -42 + } + } else { + if y >= z { // ERROR "Disproved Leq64$" + return 1337 + } + } + + return y +} +func ctz8(x uint8, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint8 + sz := bits.Len8(max) + + log2half := uint8(max) >> (sz / 2) + if x >= log2half || x == 0 { + return 42 + } + + y := bits.TrailingZeros8(x) // ERROR "Proved Ctz8 non-zero$"" + + z := sz / 2 + if ensureBothBranchesCouldHappen { + if y < z { // ERROR "Proved Less64$" + return -42 + } + } else { + if y >= z { // ERROR "Disproved Leq64$" + return 1337 + } + } + + return y +} + +func bitLen64(x uint64, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint64 + sz := bits.Len64(max) + + if x >= max>>3 { + return 42 + } + if x <= max>>6 { + return 42 + } + + y := bits.Len64(x) + + if ensureBothBranchesCouldHappen { + if sz-6 <= y && y <= sz-3 { // ERROR "Proved Leq64$" + return -42 + } + } else { + if y < sz-6 || sz-3 < y { // ERROR "Disproved Less64$" + return 1337 + } + } + return y +} +func bitLen32(x uint32, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint32 + sz := bits.Len32(max) + + if x >= max>>3 { + return 42 + } + if x <= max>>6 { + return 42 + } + + y := bits.Len32(x) + + if ensureBothBranchesCouldHappen { + if sz-6 <= y && y <= sz-3 { // ERROR "Proved Leq64$" + return -42 + } + } else { + if y < sz-6 || sz-3 < y { // ERROR "Disproved Less64$" + return 1337 + } + } + return y +} +func bitLen16(x uint16, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint16 + sz := bits.Len16(max) + + if x >= max>>3 { + return 42 + } + if x <= max>>6 { + return 42 + } + + y := bits.Len16(x) + + if ensureBothBranchesCouldHappen { + if sz-6 <= y && y <= sz-3 { // ERROR "Proved Leq64$" + return -42 + } + } else { + if y < sz-6 || sz-3 < y { // ERROR "Disproved Less64$" + return 1337 + } + } + return y +} +func bitLen8(x uint8, ensureBothBranchesCouldHappen bool) int { + const max = math.MaxUint8 + sz := bits.Len8(max) + + if x >= max>>3 { + return 42 + } + if x <= max>>6 { + return 42 + } + + y := bits.Len8(x) + + if ensureBothBranchesCouldHappen { + if sz-6 <= y && y <= sz-3 { // ERROR "Proved Leq64$" + return -42 + } + } else { + if y < sz-6 || sz-3 < y { // ERROR "Disproved Less64$" + return 1337 + } + } + return y +} + +func xor64(a, b uint64, ensureBothBranchesCouldHappen bool) int { + a = min(a, 0xff) + b = min(b, 0xfff) + + z := a ^ b + + if ensureBothBranchesCouldHappen { + if z > 0xfff { // ERROR "Disproved Less64U$" + return 42 + } + } else { + if z <= 0xfff { // ERROR "Proved Leq64U$" + return 1337 + } + } + return int(z) +} + +func or64(a, b uint64, ensureBothBranchesCouldHappen bool) int { + a = min(a, 0xff) + b = min(b, 0xfff) + + z := a | b + + if ensureBothBranchesCouldHappen { + if z > 0xfff { // ERROR "Disproved Less64U$" + return 42 + } + } else { + if z <= 0xfff { // ERROR "Proved Leq64U$" + return 1337 + } + } + return int(z) +} + +func mod64uWithSmallerDividendMax(a, b uint64, ensureBothBranchesCouldHappen bool) int { + a = min(a, 0xff) + b = min(b, 0xfff) + + z := bits.Len64(a % b) // see go.dev/issue/68857 for bits.Len64 + + if ensureBothBranchesCouldHappen { + if z > bits.Len64(0xff) { // ERROR "Disproved Less64$" + return 42 + } + } else { + if z <= bits.Len64(0xff) { // ERROR "Proved Leq64$" + return 1337 + } + } + return z +} +func mod64uWithSmallerDivisorMax(a, b uint64, ensureBothBranchesCouldHappen bool) int { + a = min(a, 0xfff) + b = min(b, 0x10) // we need bits.Len64(b.umax) != bits.Len64(b.umax-1) + + z := bits.Len64(a % b) // see go.dev/issue/68857 for bits.Len64 + + if ensureBothBranchesCouldHappen { + if z > bits.Len64(0x10-1) { // ERROR "Disproved Less64$" + return 42 + } + } else { + if z <= bits.Len64(0x10-1) { // ERROR "Proved Leq64$" + return 1337 + } + } + return z +} +func mod64uWithIdenticalMax(a, b uint64, ensureBothBranchesCouldHappen bool) int { + a = min(a, 0x10) + b = min(b, 0x10) // we need bits.Len64(b.umax) != bits.Len64(b.umax-1) + + z := bits.Len64(a % b) // see go.dev/issue/68857 for bits.Len64 + + if ensureBothBranchesCouldHappen { + if z > bits.Len64(0x10-1) { // ERROR "Disproved Less64$" + return 42 + } + } else { + if z <= bits.Len64(0x10-1) { // ERROR "Proved Leq64$" + return 1337 + } + } + return z +} +func mod64sPositiveWithSmallerDividendMax(a, b int64, ensureBothBranchesCouldHappen bool) int64 { + if a < 0 || b < 0 { + return 42 + } + a = min(a, 0xff) + b = min(b, 0xfff) + + z := a % b // ERROR "Proved Mod64 does not need fix-up$" + + if ensureBothBranchesCouldHappen { + if z > 0xff { // ERROR "Disproved Less64$" + return 42 + } + } else { + if z <= 0xff { // ERROR "Proved Leq64$" + return 1337 + } + } + return z +} +func mod64sPositiveWithSmallerDivisorMax(a, b int64, ensureBothBranchesCouldHappen bool) int64 { + if a < 0 || b < 0 { + return 42 + } + a = min(a, 0xfff) + b = min(b, 0xff) + + z := a % b // ERROR "Proved Mod64 does not need fix-up$" + + if ensureBothBranchesCouldHappen { + if z > 0xff-1 { // ERROR "Disproved Less64$" + return 42 + } + } else { + if z <= 0xff-1 { // ERROR "Proved Leq64$" + return 1337 + } + } + return z +} +func mod64sPositiveWithIdenticalMax(a, b int64, ensureBothBranchesCouldHappen bool) int64 { + if a < 0 || b < 0 { + return 42 + } + a = min(a, 0xfff) + b = min(b, 0xfff) + + z := a % b // ERROR "Proved Mod64 does not need fix-up$" + + if ensureBothBranchesCouldHappen { + if z > 0xfff-1 { // ERROR "Disproved Less64$" + return 42 + } + } else { + if z <= 0xfff-1 { // ERROR "Proved Leq64$" + return 1337 + } + } + return z +} + +func div64u(a, b uint64, ensureAllBranchesCouldHappen func() bool) uint64 { + a = min(a, 0xffff) + a = max(a, 0xfff) + b = min(b, 0xff) + b = max(b, 0xf) + + z := a / b // ERROR "Proved Neq64$" + + if ensureAllBranchesCouldHappen() && z > 0xffff/0xf { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z <= 0xffff/0xf { // ERROR "Proved Leq64U$" + return 1337 + } + if ensureAllBranchesCouldHappen() && z < 0xfff/0xff { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z >= 0xfff/0xff { // ERROR "Proved Leq64U$" + return 42 + } + return z +} +func div64s(a, b int64, ensureAllBranchesCouldHappen func() bool) int64 { + if a < 0 || b < 0 { + return 42 + } + a = min(a, 0xffff) + a = max(a, 0xfff) + b = min(b, 0xff) + b = max(b, 0xf) + + z := a / b // ERROR "(Proved Div64 does not need fix-up|Proved Neq64)$" + + if ensureAllBranchesCouldHappen() && z > 0xffff/0xf { // ERROR "Disproved Less64$" + return 42 + } + if ensureAllBranchesCouldHappen() && z <= 0xffff/0xf { // ERROR "Proved Leq64$" + return 1337 + } + if ensureAllBranchesCouldHappen() && z < 0xfff/0xff { // ERROR "Disproved Less64$" + return 42 + } + if ensureAllBranchesCouldHappen() && z >= 0xfff/0xff { // ERROR "Proved Leq64$" + return 42 + } + return z +} + +func trunc64to16(a uint64, ensureAllBranchesCouldHappen func() bool) uint16 { + a = min(a, 0xfff) + a = max(a, 0xff) + + z := uint16(a) + if ensureAllBranchesCouldHappen() && z > 0xfff { // ERROR "Disproved Less16U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z <= 0xfff { // ERROR "Proved Leq16U$" + return 1337 + } + if ensureAllBranchesCouldHappen() && z < 0xff { // ERROR "Disproved Less16U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z >= 0xff { // ERROR "Proved Leq16U$" + return 1337 + } + return z +} + +func com64(a uint64, ensureAllBranchesCouldHappen func() bool) uint64 { + a = min(a, 0xffff) + a = max(a, 0xff) + + z := ^a + + if ensureAllBranchesCouldHappen() && z > ^uint64(0xff) { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z <= ^uint64(0xff) { // ERROR "Proved Leq64U$" + return 1337 + } + if ensureAllBranchesCouldHappen() && z < ^uint64(0xffff) { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z >= ^uint64(0xffff) { // ERROR "Proved Leq64U$" + return 1337 + } + return z +} + +func neg64(a uint64, ensureAllBranchesCouldHappen func() bool) uint64 { + var lo, hi uint64 = 0xff, 0xfff + a = min(a, hi) + a = max(a, lo) + + z := -a + + if ensureAllBranchesCouldHappen() && z > -lo { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z <= -lo { // ERROR "Proved Leq64U$" + return 1337 + } + if ensureAllBranchesCouldHappen() && z < -hi { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && z >= -hi { // ERROR "Proved Leq64U$" + return 1337 + } + return z +} +func neg64mightOverflowDuringNeg(a uint64, ensureAllBranchesCouldHappen func() bool) uint64 { + var lo, hi uint64 = 0, 0xfff + a = min(a, hi) + a = max(a, lo) + + z := -a + + if ensureAllBranchesCouldHappen() && z > -lo { + return 42 + } + if ensureAllBranchesCouldHappen() && z <= -lo { + return 1337 + } + if ensureAllBranchesCouldHappen() && z < -hi { + return 42 + } + if ensureAllBranchesCouldHappen() && z >= -hi { + return 1337 + } + return z +} + +func phiMin(a, b []byte) { + _ = a[:min(len(a), len(b))] // ERROR "Proved IsSliceInBounds" + _ = b[:min(len(a), len(b))] // ERROR "Proved IsSliceInBounds" + _ = a[:max(len(a), len(b))] + _ = b[:max(len(a), len(b))] + x := len(a) + if x > len(b) { + x = len(b) + useInt(0) + } + _ = a[:x] // ERROR "Proved IsSliceInBounds" + y := len(a) + if y > len(b) { + y = len(b) + useInt(0) + } else { + useInt(1) + } + _ = b[:y] // ERROR "Proved IsSliceInBounds" +} + +func minPhiLeq[T uint | int](x, y T) (z T) { + if x <= y { + z = x + } else { + z = y + } + return z +} +func maxPhiLeq[T uint | int](x, y T) (z T) { + if y <= x { + z = x + } else { + z = y + } + return z +} +func mathBasedOnPhiLosangeMinUFirstLeq(x uint, ensureAllBranchesCouldHappen func() bool) uint { + const maxc = 0xf2a + const minc = 0xf0a + x = minPhiLeq(x, maxc) + x = maxPhiLeq(x, minc) + + const k = 1 + x += k + + if ensureAllBranchesCouldHappen() && x > maxc+k { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && x <= maxc+k { // ERROR "Proved Leq64U$" + return 4242 + } + if ensureAllBranchesCouldHappen() && x < minc+k { // ERROR "Disproved Less64U$" + return 424242 + } + if ensureAllBranchesCouldHappen() && x >= minc+k { // ERROR "Proved Leq64U$" + return 42424242 + } + return x +} +func mathBasedOnPhiLosangeMinUSecondLeq(x uint, ensureAllBranchesCouldHappen func() bool) uint { + const maxc = 0xf2a + const minc = 0xf0a + x = maxPhiLeq(x, minc) + x = minPhiLeq(x, maxc) + + const k = 1 + x += k + + if ensureAllBranchesCouldHappen() && x > maxc+k { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && x <= maxc+k { // ERROR "Proved Leq64U$" + return 4242 + } + if ensureAllBranchesCouldHappen() && x < minc+k { // ERROR "Disproved Less64U$" + return 424242 + } + if ensureAllBranchesCouldHappen() && x >= minc+k { // ERROR "Proved Leq64U$" + return 42424242 + } + return x +} +func mathBasedOnPhiLosangeMinFirstLeq(x int, ensureAllBranchesCouldHappen func() bool) int { + const maxc = 0xf2a + const minc = 0xf0a + x = minPhiLeq(x, maxc) + x = maxPhiLeq(x, minc) + + const k = 1 + x += k + + if ensureAllBranchesCouldHappen() && x > maxc+k { // ERROR "Disproved Less64$" + return 42 + } + if ensureAllBranchesCouldHappen() && x <= maxc+k { // ERROR "Proved Leq64$" + return 4242 + } + if ensureAllBranchesCouldHappen() && x < minc+k { // ERROR "Disproved Less64$" + return 424242 + } + if ensureAllBranchesCouldHappen() && x >= minc+k { // ERROR "Proved Leq64$" + return 42424242 + } + return x +} +func mathBasedOnPhiLosangeMinSecondLeq(x int, ensureAllBranchesCouldHappen func() bool) int { + const maxc = 0xf2a + const minc = 0xf0a + x = maxPhiLeq(x, minc) + x = minPhiLeq(x, maxc) + + const k = 1 + x += k + + if ensureAllBranchesCouldHappen() && x > maxc+k { // ERROR "Disproved Less64$" + return 42 + } + if ensureAllBranchesCouldHappen() && x <= maxc+k { // ERROR "Proved Leq64$" + return 4242 + } + if ensureAllBranchesCouldHappen() && x < minc+k { // ERROR "Disproved Less64$" + return 424242 + } + if ensureAllBranchesCouldHappen() && x >= minc+k { // ERROR "Proved Leq64$" + return 42424242 + } + return x +} + +func minPhiLess[T uint | int](x, y T) (z T) { + if x < y { + z = x + } else { + z = y + } + return z +} +func maxPhiLess[T uint | int](x, y T) (z T) { + if y < x { + z = x + } else { + z = y + } + return z +} +func mathBasedOnPhiLosangeMinUFirstLess(x uint, ensureAllBranchesCouldHappen func() bool) uint { + const maxc = 0xf2a + const minc = 0xf0a + x = minPhiLess(x, maxc) + x = maxPhiLess(x, minc) + + const k = 1 + x += k + + if ensureAllBranchesCouldHappen() && x > maxc+k { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && x <= maxc+k { // ERROR "Proved Leq64U$" + return 4242 + } + if ensureAllBranchesCouldHappen() && x < minc+k { // ERROR "Disproved Less64U$" + return 424242 + } + if ensureAllBranchesCouldHappen() && x >= minc+k { // ERROR "Proved Leq64U$" + return 42424242 + } + return x +} +func mathBasedOnPhiLosangeMinUSecondLess(x uint, ensureAllBranchesCouldHappen func() bool) uint { + const maxc = 0xf2a + const minc = 0xf0a + x = maxPhiLess(x, minc) + x = minPhiLess(x, maxc) + + const k = 1 + x += k + + if ensureAllBranchesCouldHappen() && x > maxc+k { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && x <= maxc+k { // ERROR "Proved Leq64U$" + return 4242 + } + if ensureAllBranchesCouldHappen() && x < minc+k { // ERROR "Disproved Less64U$" + return 424242 + } + if ensureAllBranchesCouldHappen() && x >= minc+k { // ERROR "Proved Leq64U$" + return 42424242 + } + return x +} +func mathBasedOnPhiLosangeMinFirstLess(x int, ensureAllBranchesCouldHappen func() bool) int { + const maxc = 0xf2a + const minc = 0xf0a + x = minPhiLess(x, maxc) + x = maxPhiLess(x, minc) + + const k = 1 + x += k + + if ensureAllBranchesCouldHappen() && x > maxc+k { // ERROR "Disproved Less64$" + return 42 + } + if ensureAllBranchesCouldHappen() && x <= maxc+k { // ERROR "Proved Leq64$" + return 4242 + } + if ensureAllBranchesCouldHappen() && x < minc+k { // ERROR "Disproved Less64$" + return 424242 + } + if ensureAllBranchesCouldHappen() && x >= minc+k { // ERROR "Proved Leq64$" + return 42424242 + } + return x +} +func mathBasedOnPhiLosangeMinSecondLess(x int, ensureAllBranchesCouldHappen func() bool) int { + const maxc = 0xf2a + const minc = 0xf0a + x = maxPhiLess(x, minc) + x = minPhiLess(x, maxc) + + const k = 1 + x += k + + if ensureAllBranchesCouldHappen() && x > maxc+k { // ERROR "Disproved Less64$" + return 42 + } + if ensureAllBranchesCouldHappen() && x <= maxc+k { // ERROR "Proved Leq64$" + return 4242 + } + if ensureAllBranchesCouldHappen() && x < minc+k { // ERROR "Disproved Less64$" + return 424242 + } + if ensureAllBranchesCouldHappen() && x >= minc+k { // ERROR "Proved Leq64$" + return 42424242 + } + return x +} + +func mathBasedOnPhiBuiltinMinUFirst(x uint, ensureAllBranchesCouldHappen func() bool) uint { + const maxc = 0xf2a + const minc = 0xf0a + x = min(x, maxc) + x = max(x, minc) + + const k = 1 + x += k + + if ensureAllBranchesCouldHappen() && x > maxc+k { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && x <= maxc+k { // ERROR "Proved Leq64U$" + return 4242 + } + if ensureAllBranchesCouldHappen() && x < minc+k { // ERROR "Disproved Less64U$" + return 424242 + } + if ensureAllBranchesCouldHappen() && x >= minc+k { // ERROR "Proved Leq64U$" + return 42424242 + } + return x +} +func mathBasedOnPhiBuiltinMinUSecond(x uint, ensureAllBranchesCouldHappen func() bool) uint { + const maxc = 0xf2a + const minc = 0xf0a + x = max(x, minc) + x = min(x, maxc) + + const k = 1 + x += k + + if ensureAllBranchesCouldHappen() && x > maxc+k { // ERROR "Disproved Less64U$" + return 42 + } + if ensureAllBranchesCouldHappen() && x <= maxc+k { // ERROR "Proved Leq64U$" + return 4242 + } + if ensureAllBranchesCouldHappen() && x < minc+k { // ERROR "Disproved Less64U$" + return 424242 + } + if ensureAllBranchesCouldHappen() && x >= minc+k { // ERROR "Proved Leq64U$" + return 42424242 + } + return x +} +func mathBasedOnPhiBuiltinMinFirst(x int, ensureAllBranchesCouldHappen func() bool) int { + const maxc = 0xf2a + const minc = 0xf0a + x = min(x, maxc) + x = max(x, minc) + + const k = 1 + x += k + + if ensureAllBranchesCouldHappen() && x > maxc+k { // ERROR "Disproved Less64$" + return 42 + } + if ensureAllBranchesCouldHappen() && x <= maxc+k { // ERROR "Proved Leq64$" + return 4242 + } + if ensureAllBranchesCouldHappen() && x < minc+k { // ERROR "Disproved Less64$" + return 424242 + } + if ensureAllBranchesCouldHappen() && x >= minc+k { // ERROR "Proved Leq64$" + return 42424242 + } + return x +} +func mathBasedOnPhiBuiltinMinSecond(x int, ensureAllBranchesCouldHappen func() bool) int { + const maxc = 0xf2a + const minc = 0xf0a + x = max(x, minc) + x = min(x, maxc) + + const k = 1 + x += k + + if ensureAllBranchesCouldHappen() && x > maxc+k { // ERROR "Disproved Less64$" + return 42 + } + if ensureAllBranchesCouldHappen() && x <= maxc+k { // ERROR "Proved Leq64$" + return 4242 + } + if ensureAllBranchesCouldHappen() && x < minc+k { // ERROR "Disproved Less64$" + return 424242 + } + if ensureAllBranchesCouldHappen() && x >= minc+k { // ERROR "Proved Leq64$" + return 42424242 + } + return x +} + +func issue16833(a, b []byte) { + n := copy(a, b) + _ = a[n:] // ERROR "Proved IsSliceInBounds" + _ = b[n:] // ERROR "Proved IsSliceInBounds" + _ = a[:n] // ERROR "Proved IsSliceInBounds" + _ = b[:n] // ERROR "Proved IsSliceInBounds" +} + +func clampedIdx1(x []int, i int) int { + if len(x) == 0 { + return 0 + } + return x[min(max(0, i), len(x)-1)] // ERROR "Proved IsInBounds" +} +func clampedIdx2(x []int, i int) int { + if len(x) == 0 { + return 0 + } + return x[max(min(i, len(x)-1), 0)] // TODO: can't get rid of this bounds check yet +} + +func cvtBoolToUint8Disprove(b bool) byte { + var c byte + if b { + c = 1 + } + if c == 2 { // ERROR "Disproved Eq8" + c = 3 + } + return c +} +func cvtBoolToUint8BCE(b bool, a [2]int64) int64 { + c := byte(0) + if b { + c = 1 + } + return a[c] // ERROR "Proved IsInBounds$" +} + +func transitiveProofsThroughNonOverflowingUnsignedAdd(x, y, z uint64) { + x &= 1<<63 - 1 + y &= 1<<63 - 1 + + a := x + y + if a > z { + return + } + + if x > z { // ERROR "Disproved Less64U$" + return + } + if y > z { // ERROR "Disproved Less64U$" + return + } + if a == x { + return + } + if a == y { + return + } + + x |= 1 + y |= 1 + a = x + y + if a == x { // ERROR "Disproved Eq64$" + return + } + if a == y { // ERROR "Disproved Eq64$" + return + } +} + +func transitiveProofsThroughOverflowingUnsignedAdd(x, y, z uint64) { + a := x + y + if a > z { + return + } + + if x > z { + return + } + if y > z { + return + } + if a == x { + return + } + if a == y { + return + } + + x |= 1 + y |= 1 + a = x + y + if a == x { + return + } + if a == y { + return + } +} + +func transitiveProofsThroughNonOverflowingSignedAddPositive(x, y, z int64) { + x &= 1<<62 - 1 + y &= 1<<62 - 1 + + a := x + y + if a > z { + return + } + + if x > z { // ERROR "Disproved Less64$" + return + } + if y > z { // ERROR "Disproved Less64$" + return + } + if a == x { + return + } + if a == y { + return + } + + x |= 1 + y |= 1 + a = x + y + if a == x { // ERROR "Disproved Eq64$" + return + } + if a == y { // ERROR "Disproved Eq64$" + return + } +} + +func transitiveProofsThroughOverflowingSignedAddPositive(x, y, z int64) { + if x < 0 || y < 0 { + return + } + + a := x + y + if a > z { + return + } + + if x > z { + return + } + if y > z { + return + } + if a == x { + return + } + if a == y { + return + } + + x |= 1 + y |= 1 + a = x + y + if a == x { // ERROR "Disproved Eq64$" + return + } + if a == y { // ERROR "Disproved Eq64$" + return + } +} + +func transitiveProofsThroughNonOverflowingSignedAddNegative(x, y, z int64) { + if x < math.MinInt64>>1 || x > 0 || + y < math.MinInt64>>1 || y > 0 { + return + } + + a := x + y + if a < z { + return + } + + if x < z { // ERROR "Disproved Less64$" + return + } + if y < z { // ERROR "Disproved Less64$" + return + } + if a == x { + return + } + if a == y { + return + } + + if x == 0 && y == 0 { + return + } + a = x + y + if a == x { // ERROR "Disproved Eq64$" + return + } + if a == y { // ERROR "Disproved Eq64$" + return + } +} + +func transitiveProofsThroughOverflowingSignedAddNegative(x, y, z int64) { + if x >= 0 || y >= 0 { + return + } + + a := x + y + if a < z { + return + } + + if x < z { + return + } + if y < z { + return + } + if a == x { + return + } + if a == y { + return + } + + x |= 1 + y |= 1 + a = x + y + if a == x { + return + } + if a == y { + return + } +} + +func transitiveProofsThroughNonOverflowingUnsignedSub(x, y, z uint64) { + x |= 0xfff + y &= 0xfff + + a := x - y + if a < z { + return + } + + if x < z { // ERROR "Disproved Less64U$" + return + } + if y < z { + return + } + if a == x { + return + } + if a == y { + return + } + + y |= 1 + a = x - y + if a == x { // ERROR "Disproved Eq64$" + return + } + if a == y { + return + } +} + +func transitiveProofsThroughOverflowingUnsignedSub(x, y, z uint64) { + a := x - y + if a < z { + return + } + + if x < z { + return + } + if y < z { + return + } + if a == x { + return + } + if a == y { + return + } + + y |= 1 + a = x - y + if a == x { + return + } + if a == y { + return + } +} + +func resliceString(s string) byte { + if len(s) >= 4 { + s = s[2:] // ERROR "Proved IsSliceInBounds" "Proved slicemask not needed" + s = s[1:] // ERROR "Proved IsSliceInBounds" "Proved slicemask not needed" + return s[0] // ERROR "Proved IsInBounds" + } + return 0 +} +func resliceBytes(b []byte) byte { + if len(b) >= 4 { + b = b[2:] // ERROR "Proved IsSliceInBounds" "Proved slicemask not needed" + b = b[1:] // ERROR "Proved IsSliceInBounds" "Proved slicemask not needed" + return b[0] // ERROR "Proved IsInBounds" + } + return 0 +} + +func issue74473(s []uint) { + i := 0 + for { + if i >= len(s) { // ERROR "Induction variable: limits \[0,\?\), increment 1$" + break + } + _ = s[i] // ERROR "Proved IsInBounds$" + i++ + } +} + +func setCapMaxBasedOnElementSize(x []uint64) int { + c := uintptr(cap(x)) + max := ^uintptr(0) >> 3 + if c > max { // ERROR "Disproved Less" + return 42 + } + if c <= max { // ERROR "Proved Leq" + return 1337 + } + return 0 +} + +func issue75144for(a, b []uint64) bool { + if len(a) == len(b) { + for len(a) > 4 { + a = a[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + b = b[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + } + if len(a) == len(b) { // ERROR "Proved Eq64$" + return true + } + } + return false +} + +func issue75144if(a, b []uint64) bool { + if len(a) == len(b) { + if len(a) > 4 { + a = a[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + b = b[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + } + if len(a) == len(b) { // ERROR "Proved Eq64$" + return true + } + } + return false +} + +func issue75144if2(a, b, c, d []uint64) (r bool) { + if len(a) != len(b) || len(c) != len(d) { + return + } + if len(a) <= 4 || len(c) <= 4 { + return + } + if len(a) < len(c) { + c = c[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + d = d[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + } else { + a = a[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + b = b[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + } + if len(a) == len(c) { + return + } + if len(a) == len(b) { // ERROR "Proved Eq64$" + r = true + } + if len(c) == len(d) { // ERROR "Proved Eq64$" + r = true + } + return +} + +func issue75144forCannot(a, b []uint64) bool { + if len(a) == len(b) { + for len(a) > 4 { + a = a[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + b = b[4:] + for len(a) > 2 { + a = a[2:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + b = b[2:] + } + } + if len(a) == len(b) { + return true + } + } + return false +} + +func issue75144ifCannot(a, b []uint64) bool { + if len(a) == len(b) { + if len(a) > 4 { + a = a[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + b = b[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + if len(a) > 2 { + a = a[2:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + b = b[2:] + } + } + if len(a) == len(b) { + return true + } + } + return false +} + +func issue75144ifCannot2(a, b []uint64) bool { + if len(a) == len(b) { + if len(a) > 4 { + a = a[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + b = b[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + } else if len(a) > 2 { + a = a[2:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + b = b[2:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + } + if len(a) == len(b) { + return true + } + } + return false +} + +func issue75144forNot(a, b []uint64) bool { + if len(a) == len(b) { + for len(a) > 4 { + a = a[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + b = b[3:] + } + if len(a) == len(b) { + return true + } + } + return false +} + +func issue75144forNot2(a, b, c []uint64) bool { + if len(a) == len(b) { + for len(a) > 4 { + a = a[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + b = c[4:] + } + if len(a) == len(b) { + return true + } + } + return false +} + +func issue75144ifNot(a, b []uint64) bool { + if len(a) == len(b) { + if len(a) > 4 { + a = a[4:] // ERROR "Proved slicemask not needed$" "Proved IsSliceInBounds$" + } else { + b = b[4:] + } + if len(a) == len(b) { + return true + } + } + return false +} + //go:noinline func useInt(a int) { } diff --git a/test/prove_constant_folding.go b/test/prove_constant_folding.go index ed63e6851c264b..366c446b8303f8 100644 --- a/test/prove_constant_folding.go +++ b/test/prove_constant_folding.go @@ -9,25 +9,25 @@ package main func f0i(x int) int { - if x == 20 { - return x // ERROR "Proved.+is constant 20$" - } + if x == 20 { + return x // ERROR "Proved.+is constant 20$" + } - if (x + 20) == 20 { - return x + 5 // ERROR "Proved.+is constant 0$" - } + if (x + 20) == 20 { + return x + 5 // ERROR "Proved.+is constant 0$" "Proved.+is constant 5$" "x\+d >=? w" + } - return x / 2 + return x / 2 } func f0u(x uint) uint { - if x == 20 { - return x // ERROR "Proved.+is constant 20$" - } + if x == 20 { + return x // ERROR "Proved.+is constant 20$" + } - if (x + 20) == 20 { - return x + 5 // ERROR "Proved.+is constant 0$" - } + if (x + 20) == 20 { + return x + 5 // ERROR "Proved.+is constant 0$" "Proved.+is constant 5$" "x\+d >=? w" + } - return x / 2 + return x / 2 } diff --git a/test/prove_popcount.go b/test/prove_popcount.go new file mode 100644 index 00000000000000..1fe53defb097a0 --- /dev/null +++ b/test/prove_popcount.go @@ -0,0 +1,60 @@ +// errorcheck -0 -d=ssa/prove/debug=1 + +//go:build amd64.v3 || arm64 + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// FIXME(@Jorropo): this file exists because I havn't yet bothered to +// make prove work on the pure go function call fallback. +// My idea was to wait until CL 637936 is merged, then we can always emit +// the PopCount SSA operation and translate them to pure function calls +// in late-opt. + +package main + +import "math/bits" + +func onesCountsBounds(x uint64, ensureAllBranchesCouldHappen func() bool) int { + z := bits.OnesCount64(x) + if ensureAllBranchesCouldHappen() && z > 64 { // ERROR "Disproved Less64$" + return 42 + } + if ensureAllBranchesCouldHappen() && z <= 64 { // ERROR "Proved Leq64$" + return 4242 + } + if ensureAllBranchesCouldHappen() && z < 0 { // ERROR "Disproved Less64$" + return 424242 + } + if ensureAllBranchesCouldHappen() && z >= 0 { // ERROR "Proved Leq64$" + return 42424242 + } + return z +} + +func onesCountsTight(x uint64, ensureAllBranchesCouldHappen func() bool) int { + const maxv = 0xff0f + const minv = 0xff00 + x = max(x, minv) + x = min(x, maxv) + + z := bits.OnesCount64(x) + + if ensureAllBranchesCouldHappen() && z > bits.OnesCount64(maxv) { // ERROR "Disproved Less64$" + return 42 + } + if ensureAllBranchesCouldHappen() && z <= bits.OnesCount64(maxv) { // ERROR "Proved Leq64$" + return 4242 + } + if ensureAllBranchesCouldHappen() && z < bits.OnesCount64(minv) { // ERROR "Disproved Less64$" + return 424242 + } + if ensureAllBranchesCouldHappen() && z >= bits.OnesCount64(minv) { // ERROR "Proved Leq64$" + return 42424242 + } + return z +} + +func main() { +} diff --git a/test/range2.go b/test/range2.go index 6ccf1e53d82e23..83f19adc20c73a 100644 --- a/test/range2.go +++ b/test/range2.go @@ -1,4 +1,4 @@ -// errorcheck -goexperiment rangefunc +// errorcheck // Copyright 2023 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/range4.go b/test/range4.go index 0b051f6d3c0f4b..e8c8da0ae36ef5 100644 --- a/test/range4.go +++ b/test/range4.go @@ -1,4 +1,4 @@ -// run -goexperiment rangefunc +// run // Copyright 2023 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/rangegen.go b/test/rangegen.go index 8231c64db75847..9d7025bd2adc19 100644 --- a/test/rangegen.go +++ b/test/rangegen.go @@ -1,4 +1,4 @@ -// runoutput -goexperiment rangefunc +// runoutput // Copyright 2023 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style diff --git a/test/runtime/README b/test/runtime/README deleted file mode 100644 index 249031afc18274..00000000000000 --- a/test/runtime/README +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -The runtime directory contains tests that specifically need -to be compiled as-if in the runtime package. For error-check -tests, these require the additional flags -+ and -p=runtime. diff --git a/test/runtime/inlinegcpc.go b/test/runtime/inlinegcpc.go deleted file mode 100644 index c423993b713428..00000000000000 --- a/test/runtime/inlinegcpc.go +++ /dev/null @@ -1,29 +0,0 @@ -// errorcheck -0 -+ -p=runtime -m - -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -// A function that calls runtime.getcallerpc or runtime.getcallersp() -// cannot be inlined, no matter how small it is. - -func getcallerpc() uintptr -func getcallersp() uintptr - -func pc() uintptr { - return getcallerpc() + 1 -} - -func cpc() uintptr { // ERROR "can inline cpc" - return pc() + 2 -} - -func sp() uintptr { - return getcallersp() + 3 -} - -func csp() uintptr { // ERROR "can inline csp" - return sp() + 4 -} diff --git a/test/switch2.go b/test/switch2.go index 66e89fda19e2c3..113d81be854fc6 100644 --- a/test/switch2.go +++ b/test/switch2.go @@ -25,12 +25,12 @@ func f() { switch { case 0: f(); case 0: - case 0: f() case 0: // ERROR "unexpected case at end of statement" + case 0: f() case 0: // ERROR "unexpected keyword case at end of statement" } switch { case 0: f(); default: - case 0: f() default: // ERROR "unexpected default at end of statement" + case 0: f() default: // ERROR "unexpected keyword default at end of statement" } switch { diff --git a/test/syntax/semi7.go b/test/syntax/semi7.go index a1948b0f7d38b6..acd2f83597391a 100644 --- a/test/syntax/semi7.go +++ b/test/syntax/semi7.go @@ -8,7 +8,7 @@ package main func main() { if x { } // GCCGO_ERROR "undefined" - else { } // ERROR "unexpected semicolon or newline before .?else.?|unexpected else" + else { } // ERROR "unexpected semicolon or newline before .?else.?|unexpected keyword else" } diff --git a/test/tailcall.go b/test/tailcall.go new file mode 100644 index 00000000000000..6b14a2f1b77b6c --- /dev/null +++ b/test/tailcall.go @@ -0,0 +1,29 @@ +// errorcheck -0 -d=tailcall=1 + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +// Test that when generating wrappers for methods, we generate a tail call to the pointer version of +// the method, if that method is not inlineable. We use go:noinline here to force the non-inlineability +// condition. + +//go:noinline +func (f *Foo) Get2Vals() [2]int { return [2]int{f.Val, f.Val + 1} } +func (f *Foo) Get3Vals() [3]int { return [3]int{f.Val, f.Val + 1, f.Val + 2} } + +type Foo struct{ Val int } + +type Bar struct { // ERROR "tail call emitted for the method \(\*Foo\).Get2Vals wrapper" + int64 + *Foo // needs a method wrapper + string +} + +var i any + +func init() { + i = Bar{1, nil, "first"} +} diff --git a/test/tighten.go b/test/tighten.go index 92ed2492b26e74..d85dfecbb0237e 100644 --- a/test/tighten.go +++ b/test/tighten.go @@ -9,14 +9,20 @@ package main var ( - e any - ts uint16 + ga, gb, gc, gd int ) func moveValuesWithMemoryArg(len int) { for n := 0; n < len; n++ { - // Load of e.data is lowed as a MOVDload op, which has a memory - // argument. It's moved near where it's used. - _ = e != ts // ERROR "MOVDload is moved$" "MOVDaddr is moved$" + // Loads of b and d can be delayed until inside the outer "if". + a := ga + b := gb // ERROR "MOVDload is moved$" + c := gc + d := gd // ERROR "MOVDload is moved$" + if a == c { + if b == d { + return + } + } } } diff --git a/test/typeparam/issue46461.go b/test/typeparam/issue46461.go index 363a87cfe08f10..7e35106c15b42e 100644 --- a/test/typeparam/issue46461.go +++ b/test/typeparam/issue46461.go @@ -1,4 +1,4 @@ -// errorcheck +// compile // Copyright 2021 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style @@ -6,7 +6,7 @@ package p -type T[U interface{ M() T[U] }] int // ERROR "invalid recursive type: T refers to itself" +type T[U interface{ M() T[U] }] int type X int diff --git a/test/typeparam/issue46461b.dir/a.go b/test/typeparam/issue46461b.dir/a.go index fcb414266d741c..0d53b3e20429cd 100644 --- a/test/typeparam/issue46461b.dir/a.go +++ b/test/typeparam/issue46461b.dir/a.go @@ -4,4 +4,4 @@ package a -type T[U interface{ M() int }] int +type T[U interface{ M() T[U] }] int diff --git a/test/typeparam/issue46461b.dir/b.go b/test/typeparam/issue46461b.dir/b.go index a4583257ffd657..3393a375c20570 100644 --- a/test/typeparam/issue46461b.dir/b.go +++ b/test/typeparam/issue46461b.dir/b.go @@ -8,6 +8,4 @@ import "./a" type X int -func (X) M() int { return 0 } - -type _ a.T[X] +func (X) M() a.T[X] { return 0 } diff --git a/test/typeparam/issue48280.dir/a.go b/test/typeparam/issue48280.dir/a.go index f66fd30e34ee43..17859e6aa902f0 100644 --- a/test/typeparam/issue48280.dir/a.go +++ b/test/typeparam/issue48280.dir/a.go @@ -4,7 +4,7 @@ package a -type I[T any] interface { +type I[T I[T]] interface { F() T } diff --git a/test/typeparam/issue48306.dir/a.go b/test/typeparam/issue48306.dir/a.go index fdfd86cb6d4a5d..739750b20b35e6 100644 --- a/test/typeparam/issue48306.dir/a.go +++ b/test/typeparam/issue48306.dir/a.go @@ -4,6 +4,6 @@ package a -type I[T any] interface { +type I[T I[T]] interface { F() T } diff --git a/test/uintptrescapes2.go b/test/uintptrescapes2.go index 656286c0ff2bd7..e111d47fab95c4 100644 --- a/test/uintptrescapes2.go +++ b/test/uintptrescapes2.go @@ -33,8 +33,8 @@ func (T) M1(a uintptr) {} // ERROR "escaping uintptr" func (T) M2(a ...uintptr) {} // ERROR "escaping ...uintptr" func TestF1() { - var t int // ERROR "moved to heap" - F1(uintptr(unsafe.Pointer(&t))) // ERROR "live at call to F1: .?autotmp" "stack object .autotmp_[0-9]+ unsafe.Pointer$" + var t int // ERROR "moved to heap" + F1(uintptr(unsafe.Pointer(&t))) // ERROR "live at call to F1: .?autotmp" "stack object .autotmp_[0-9]+ unsafe.Pointer$" } func TestF3() { @@ -49,17 +49,17 @@ func TestM1() { } func TestF2() { - var v int // ERROR "moved to heap" - F2(0, 1, uintptr(unsafe.Pointer(&v)), 2) // ERROR "live at call to newobject: .?autotmp" "live at call to F2: .?autotmp" "escapes to heap" "stack object .autotmp_[0-9]+ unsafe.Pointer$" + var v int // ERROR "moved to heap" + F2(0, 1, uintptr(unsafe.Pointer(&v)), 2) // ERROR "live at call to mallocgcSmallNoScanSC[0-9]+: .?autotmp" "live at call to F2: .?autotmp" "escapes to heap" "stack object .autotmp_[0-9]+ unsafe.Pointer$" } func TestF4() { var v2 int // ERROR "moved to heap" - F4(0, 1, uintptr(unsafe.Pointer(&v2)), 2) // ERROR "live at call to newobject: .?autotmp" "live at call to F4: .?autotmp" "escapes to heap" "stack object .autotmp_[0-9]+ unsafe.Pointer$" + F4(0, 1, uintptr(unsafe.Pointer(&v2)), 2) // ERROR "live at call to mallocgcSmallNoScanSC[0-9]+: .?autotmp" "live at call to F4: .?autotmp" "escapes to heap" "stack object .autotmp_[0-9]+ unsafe.Pointer$" } func TestM2() { var t T var v int // ERROR "moved to heap" - t.M2(0, 1, uintptr(unsafe.Pointer(&v)), 2) // ERROR "live at call to newobject: .?autotmp" "live at call to T.M2: .?autotmp" "escapes to heap" "stack object .autotmp_[0-9]+ unsafe.Pointer$" + t.M2(0, 1, uintptr(unsafe.Pointer(&v)), 2) // ERROR "live at call to mallocgcSmallNoScanSC[0-9]+: .?autotmp" "live at call to T.M2: .?autotmp" "escapes to heap" "stack object .autotmp_[0-9]+ unsafe.Pointer$" } diff --git a/test/used.go b/test/used.go index 5bdc5a73188691..33e1140cef74ac 100644 --- a/test/used.go +++ b/test/used.go @@ -139,7 +139,6 @@ func _() { unsafe.Sizeof(t) // ERROR "unsafe.Sizeof\(t\) .* not used" _ = int // ERROR "type int is not an expression|not an expression" (x) // ERROR "x .* not used|not used" - _ = new(x2) // ERROR "x2 is not a type|not a type" - // Disabled due to issue #43125. - // _ = new(1 + 1) // DISABLED "1 \+ 1 is not a type" + _ = new(len) // ERROR "len.*must be called" + _ = new(1 + 1) // ok } diff --git a/test/wasmexport.go b/test/wasmexport.go new file mode 100644 index 00000000000000..3b92ae93c954c8 --- /dev/null +++ b/test/wasmexport.go @@ -0,0 +1,19 @@ +// errorcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Verify that misplaced directives are diagnosed. + +//go:build wasm + +package p + +//go:wasmexport F +func F() {} // OK + +type S int32 + +//go:wasmexport M +func (S) M() {} // ERROR "cannot use //go:wasmexport on a method" diff --git a/test/wasmexport2.go b/test/wasmexport2.go new file mode 100644 index 00000000000000..cfbfab99b03a4d --- /dev/null +++ b/test/wasmexport2.go @@ -0,0 +1,92 @@ +// errorcheck + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Verify that wasmexport supports allowed types and rejects +// unallowed types. + +//go:build wasm + +package p + +import ( + "structs" + "unsafe" +) + +//go:wasmexport good1 +func good1(int32, uint32, int64, uint64, float32, float64, unsafe.Pointer) {} // allowed types + +type MyInt32 int32 + +//go:wasmexport good2 +func good2(MyInt32) {} // named type is ok + +//go:wasmexport good3 +func good3() int32 { return 0 } // one result is ok + +//go:wasmexport good4 +func good4() unsafe.Pointer { return nil } // one result is ok + +//go:wasmexport good5 +func good5(string, uintptr) bool { return false } // bool, string, and uintptr are allowed + +//go:wasmexport bad1 +func bad1(any) {} // ERROR "go:wasmexport: unsupported parameter type" + +//go:wasmexport bad2 +func bad2(func()) {} // ERROR "go:wasmexport: unsupported parameter type" + +//go:wasmexport bad3 +func bad3(uint8) {} // ERROR "go:wasmexport: unsupported parameter type" + +//go:wasmexport bad4 +func bad4(int) {} // ERROR "go:wasmexport: unsupported parameter type" + +// Struct and array types are also not allowed. + +type S struct { x, y int32 } + +type H struct { _ structs.HostLayout; x, y int32 } + +type A = structs.HostLayout + +type AH struct { _ A; x, y int32 } + +//go:wasmexport bad5 +func bad5(S) {} // ERROR "go:wasmexport: unsupported parameter type" + +//go:wasmexport bad6 +func bad6(H) {} // ERROR "go:wasmexport: unsupported parameter type" + +//go:wasmexport bad7 +func bad7([4]int32) {} // ERROR "go:wasmexport: unsupported parameter type" + +// Pointer types are not allowed, with resitrictions on +// the element type. + +//go:wasmexport good6 +func good6(*int32, *uint8, *bool) {} + +//go:wasmexport bad8 +func bad8(*S) {} // ERROR "go:wasmexport: unsupported parameter type" // without HostLayout, not allowed + +//go:wasmexport bad9 +func bad9() *S { return nil } // ERROR "go:wasmexport: unsupported result type" + +//go:wasmexport good7 +func good7(*H, *AH) {} // pointer to struct with HostLayout is allowed + +//go:wasmexport good8 +func good8(*struct{}) {} // pointer to empty struct is allowed + +//go:wasmexport good9 +func good9(*[4]int32, *[2]H) {} // pointer to array is allowed, if the element type is okay + +//go:wasmexport toomanyresults +func toomanyresults() (int32, int32) { return 0, 0 } // ERROR "go:wasmexport: too many return values" + +//go:wasmexport bad10 +func bad10() string { return "" } // ERROR "go:wasmexport: unsupported result type" // string cannot be a result diff --git a/test/wasmmemsize.dir/asm_wasm.s b/test/wasmmemsize.dir/asm_wasm.s new file mode 100644 index 00000000000000..daccfcc753d8d3 --- /dev/null +++ b/test/wasmmemsize.dir/asm_wasm.s @@ -0,0 +1,11 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +TEXT ·currentMemory(SB), NOSPLIT, $0 + Get SP + CurrentMemory + I32Store ret+0(FP) + RET diff --git a/test/wasmmemsize.dir/main.go b/test/wasmmemsize.dir/main.go new file mode 100644 index 00000000000000..c51e6b3b0476c9 --- /dev/null +++ b/test/wasmmemsize.dir/main.go @@ -0,0 +1,31 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "io" +) + +// Wasm page size. +const pageSize = 64 * 1024 + +// Expect less than 3 MB + 1 page of memory usage for a small wasm +// program. This reflects the current allocator. If the allocator +// changes, update this value. +const want = 3<<20 + pageSize + +var w = io.Discard + +func main() { + fmt.Fprintln(w, "hello world") + + sz := uintptr(currentMemory()) * pageSize + if sz > want { + fmt.Printf("FAIL: unexpected memory size %d, want <= %d\n", sz, want) + } +} + +func currentMemory() int32 // implemented in assembly diff --git a/test/wasmmemsize.go b/test/wasmmemsize.go new file mode 100644 index 00000000000000..449816908719f7 --- /dev/null +++ b/test/wasmmemsize.go @@ -0,0 +1,11 @@ +// runindir + +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This test checks the memory size of a small wasm program. + +//go:build wasm + +package ignored diff --git a/test/weak.go b/test/weak.go new file mode 100644 index 00000000000000..ca3ec797fc2366 --- /dev/null +++ b/test/weak.go @@ -0,0 +1,24 @@ +// errorcheck + +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test weak pointers. + +package p + +import ( + "runtime" + "weak" +) + +// Adapted from example in https://github.com/golang/go/issues/67552#issuecomment-2639661220 +func conversion() { + p := "hello" + a := weak.Make(&p) + b := (weak.Pointer[*byte])(a) // ERROR "cannot convert a \(variable of struct type weak\.Pointer\[string\]\) to type weak.Pointer\[\*byte\]" + c := b.Value() + println(**c) + runtime.KeepAlive(p) +}